From 44d5412e10dd1448c8a41db3768534b8b4b79fd0 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 31 Mar 2023 12:51:12 +0300 Subject: [PATCH 0001/1943] [#181] ir: Do not deposit notary GAS by non-alphabet nodes Signed-off-by: Pavel Karpy --- pkg/innerring/innerring.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 063d0f7cd..38023932f 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -149,14 +149,16 @@ func (s *Server) Start(ctx context.Context, intError chan<- error) (err error) { return err } - err = s.initMainNotary(ctx) - if err != nil { - return err - } + if s.IsAlphabet() { + err = s.initMainNotary(ctx) + if err != nil { + return err + } - err = s.initSideNotary(ctx) - if err != nil { - return err + err = s.initSideNotary(ctx) + if err != nil { + return err + } } prm := governance.VoteValidatorPrm{} From db5321309d073a0869a271897d866357afa30f60 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 31 Mar 2023 12:52:16 +0300 Subject: [PATCH 0002/1943] [#181] ir: Do not sync the Alphabet by non-alphabet nodes Signed-off-by: Pavel Karpy --- pkg/innerring/initialization.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 31a1bcd60..5a8fdcc3d 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/balance" cont "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/container" - frostfs "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/frostfs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap" nodevalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation" @@ -90,7 +90,9 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, AuditSettlementsHandler: s.onlyAlphabetEventHandler( settlementProcessor.HandleAuditEvent, ), - AlphabetSyncHandler: alphaSync, + AlphabetSyncHandler: s.onlyAlphabetEventHandler( + alphaSync, + ), NodeValidator: nodevalidator.New( &netMapCandidateStateValidator, addrvalidator.New(), From f09ee27af937cb6608e24a95030db79ceb508ffb Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 30 Mar 2023 18:24:07 +0300 Subject: [PATCH 0003/1943] [#181] ir: Do not process container estimations by non-alphabet nodes Signed-off-by: Pavel Karpy --- pkg/innerring/blocktimer.go | 11 +++++++++++ pkg/innerring/processors/netmap/process_epoch.go | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/innerring/blocktimer.go b/pkg/innerring/blocktimer.go index 747f36fdf..eb74e44d4 100644 --- a/pkg/innerring/blocktimer.go +++ b/pkg/innerring/blocktimer.go @@ -20,6 +20,10 @@ type ( EpochDuration() uint64 } + alphaState interface { + IsAlphabet() bool + } + subEpochEventHandler struct { handler event.Handler // handle to execute durationMul uint32 // X: X/Y of epoch in blocks @@ -31,6 +35,8 @@ type ( epochTimerArgs struct { l *logger.Logger + alphabetState alphaState + newEpochHandlers []newEpochHandler cnrWrapper *container.Client // to invoke stop container estimation @@ -91,6 +97,11 @@ func newEpochTimer(args *epochTimerArgs) *timer.BlockTimer { args.stopEstimationDMul, args.stopEstimationDDiv, func() { + if !args.alphabetState.IsAlphabet() { + args.l.Debug("non-alphabet mode, do not stop container estimations") + return + } + epochN := args.epoch.EpochCounter() if epochN == 0 { // estimates are invalid in genesis epoch return diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index 48de528e1..ffcddc497 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -50,7 +50,7 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { prm.SetEpoch(epoch - 1) prm.SetHash(ev.TxHash()) - if epoch > 0 { // estimates are invalid in genesis epoch + if epoch > 0 && np.alphabetState.IsAlphabet() { // estimates are invalid in genesis epoch err = np.containerWrp.StartEstimation(prm) if err != nil { From aa92f977ef88addc3de6f7814a79592ba79011a9 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 30 Mar 2023 18:24:37 +0300 Subject: [PATCH 0004/1943] [#181] ir: Do not pay for audit by non-alphabet nodes Signed-off-by: Pavel Karpy --- pkg/innerring/processors/settlement/calls.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/innerring/processors/settlement/calls.go b/pkg/innerring/processors/settlement/calls.go index b4f44543e..33191662b 100644 --- a/pkg/innerring/processors/settlement/calls.go +++ b/pkg/innerring/processors/settlement/calls.go @@ -13,6 +13,12 @@ func (p *Processor) HandleAuditEvent(e event.Event) { epoch := ev.Epoch() + if !p.state.IsAlphabet() { + p.log.Info("non alphabet mode, ignore audit payments") + + return + } + log := &logger.Logger{Logger: p.log.With( zap.Uint64("epoch", epoch), )} From 2bdf7126b89217941712c854dc4818c4851c7e25 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 30 Mar 2023 18:25:45 +0300 Subject: [PATCH 0005/1943] [#181] Update CHANGELOG Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc2f4adad..ed92f3a53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ Changelog for FrostFS Node - Adding of public key for nns group `group.frostfs` at init step (#130) - Iterating over just removed files by FSTree (#98) - Parts of a locked object could not be removed anymore (#141) +- Non-alphabet nodes do not try to handle alphabet events (#181) ### Removed ### Updated From a69c6d1ec9e43f49e6e588d8770ea05ccbcef050 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Mon, 27 Feb 2023 17:17:55 +0300 Subject: [PATCH 0006/1943] [#2272] morph: Do not subscribe to events without listening It led to a neo-go dead-lock in the `subscriber` component. Subscribing to notifications is the same RPC as any others, so it could also be blocked forever if no async listening (reading the notification channel) routine exists. If a number of subscriptions is big enough (or a caller is lucky enough) subscribing loop might have not finished subscribing before the first notification is received and then: subscribing RPC is blocked by received notification (non)handling and listening notifications routine is blocked by not finished subscription loop. That commit starts listening notification channel _before_ any subscription actions. Signed-off-by: Pavel Karpy --- CHANGELOG.md | 2 +- pkg/morph/event/listener.go | 87 ++++++++++++++---------------- pkg/morph/subscriber/subscriber.go | 41 +++++++++----- 3 files changed, 69 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed92f3a53..6793ed340 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,7 +44,7 @@ Changelog for FrostFS Node - Possible deadlock in write-cache (#2239) - Fix `*_req_count` and `*_req_count_success` metric values (#2241) - Storage ID update by write-cache (#2244) -- `neo-go` client deadlock on subscription restoration (#2244) +- `neo-go` client deadlock on subscription (#2244, #2272) - Possible panic during write-cache initialization (#2234) - Do not fetch an object if `meta` is missing it (#61) - Create contract wallet only by `init` and `update-config` command (#63) diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index ed2b95026..64fdc3df3 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/subscriber" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" "github.com/nspcc-dev/neo-go/pkg/util" @@ -158,6 +157,19 @@ func (l *listener) ListenWithError(ctx context.Context, intError chan<- error) { } func (l *listener) listen(ctx context.Context, intError chan<- error) error { + // mark listener as started + l.started = true + + subErrCh := make(chan error) + + go l.subscribe(subErrCh) + + l.listenLoop(ctx, intError, subErrCh) + + return nil +} + +func (l *listener) subscribe(errCh chan error) { // create the list of listening contract hashes hashes := make([]util.Uint160, 0) @@ -175,71 +187,50 @@ func (l *listener) listen(ctx context.Context, intError chan<- error) error { hashes = append(hashes, hashType.ScriptHash()) } - - // mark listener as started - l.started = true - l.mtx.RUnlock() - chEvent, err := l.subscriber.SubscribeForNotification(hashes...) + err := l.subscriber.SubscribeForNotification(hashes...) if err != nil { - return err + errCh <- fmt.Errorf("could not subscribe for notifications: %w", err) + return } - l.listenLoop(ctx, chEvent, intError) - - return nil -} - -// nolint: funlen, gocognit -func (l *listener) listenLoop(ctx context.Context, chEvent <-chan *state.ContainedNotificationEvent, intErr chan<- error) { - var ( - blockChan <-chan *block.Block - - notaryChan <-chan *result.NotaryRequestEvent - - err error - ) - if len(l.blockHandlers) > 0 { - if blockChan, err = l.subscriber.BlockNotifications(); err != nil { - if intErr != nil { - intErr <- fmt.Errorf("could not open block notifications channel: %w", err) - } else { - l.log.Debug("could not open block notifications channel", - zap.String("error", err.Error()), - ) - } - + if err = l.subscriber.BlockNotifications(); err != nil { + errCh <- fmt.Errorf("could not subscribe for blocks: %w", err) return } - } else { - blockChan = make(chan *block.Block) } if l.listenNotary { - if notaryChan, err = l.subscriber.SubscribeForNotaryRequests(l.notaryMainTXSigner); err != nil { - if intErr != nil { - intErr <- fmt.Errorf("could not open notary notifications channel: %w", err) - } else { - l.log.Debug("could not open notary notifications channel", - zap.String("error", err.Error()), - ) - } - + if err = l.subscriber.SubscribeForNotaryRequests(l.notaryMainTXSigner); err != nil { + errCh <- fmt.Errorf("could not subscribe for notary requests: %w", err) return } } +} + +// nolint: funlen, gocognit +func (l *listener) listenLoop(ctx context.Context, intErr chan<- error, subErrCh chan error) { + chs := l.subscriber.NotificationChannels() loop: for { select { + case err := <-subErrCh: + if intErr != nil { + intErr <- err + } else { + l.log.Error("stop event listener by error", zap.Error(err)) + } + + break loop case <-ctx.Done(): l.log.Info("stop event listener by context", zap.String("reason", ctx.Err().Error()), ) break loop - case notifyEvent, ok := <-chEvent: + case notifyEvent, ok := <-chs.NotificationsCh: if !ok { l.log.Warn("stop event listener by notification channel") if intErr != nil { @@ -252,13 +243,13 @@ loop: continue loop } - if err = l.pool.Submit(func() { + if err := l.pool.Submit(func() { l.parseAndHandleNotification(notifyEvent) }); err != nil { l.log.Warn("listener worker pool drained", zap.Int("capacity", l.pool.Cap())) } - case notaryEvent, ok := <-notaryChan: + case notaryEvent, ok := <-chs.NotaryRequestsCh: if !ok { l.log.Warn("stop event listener by notary channel") if intErr != nil { @@ -271,13 +262,13 @@ loop: continue loop } - if err = l.pool.Submit(func() { + if err := l.pool.Submit(func() { l.parseAndHandleNotary(notaryEvent) }); err != nil { l.log.Warn("listener worker pool drained", zap.Int("capacity", l.pool.Cap())) } - case b, ok := <-blockChan: + case b, ok := <-chs.BlockCh: if !ok { l.log.Warn("stop event listener by block channel") if intErr != nil { @@ -290,7 +281,7 @@ loop: continue loop } - if err = l.pool.Submit(func() { + if err := l.pool.Submit(func() { for i := range l.blockHandlers { l.blockHandlers[i](b) } diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 6229e6f30..17bed5b2d 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -17,12 +17,21 @@ import ( ) type ( + NotificationChannels struct { + BlockCh <-chan *block.Block + NotificationsCh <-chan *state.ContainedNotificationEvent + NotaryRequestsCh <-chan *result.NotaryRequestEvent + } + // Subscriber is an interface of the NotificationEvent listener. Subscriber interface { - SubscribeForNotification(...util.Uint160) (<-chan *state.ContainedNotificationEvent, error) + SubscribeForNotification(...util.Uint160) error UnsubscribeForNotification() - BlockNotifications() (<-chan *block.Block, error) - SubscribeForNotaryRequests(mainTXSigner util.Uint160) (<-chan *result.NotaryRequestEvent, error) + BlockNotifications() error + SubscribeForNotaryRequests(mainTXSigner util.Uint160) error + + NotificationChannels() NotificationChannels + Close() } @@ -46,6 +55,14 @@ type ( } ) +func (s *subscriber) NotificationChannels() NotificationChannels { + return NotificationChannels{ + BlockCh: s.blockChan, + NotificationsCh: s.notifyChan, + NotaryRequestsCh: s.notaryChan, + } +} + var ( errNilParams = errors.New("chain/subscriber: config was not provided to the constructor") @@ -54,7 +71,7 @@ var ( errNilClient = errors.New("chain/subscriber: client was not provided to the constructor") ) -func (s *subscriber) SubscribeForNotification(contracts ...util.Uint160) (<-chan *state.ContainedNotificationEvent, error) { +func (s *subscriber) SubscribeForNotification(contracts ...util.Uint160) error { s.Lock() defer s.Unlock() @@ -69,14 +86,14 @@ func (s *subscriber) SubscribeForNotification(contracts ...util.Uint160) (<-chan _ = s.client.UnsubscribeContract(hash) } - return nil, err + return err } // save notification id notifyIDs[contracts[i]] = struct{}{} } - return s.notifyChan, nil + return nil } func (s *subscriber) UnsubscribeForNotification() { @@ -91,20 +108,20 @@ func (s *subscriber) Close() { s.client.Close() } -func (s *subscriber) BlockNotifications() (<-chan *block.Block, error) { +func (s *subscriber) BlockNotifications() error { if err := s.client.SubscribeForNewBlocks(); err != nil { - return nil, fmt.Errorf("could not subscribe for new block events: %w", err) + return fmt.Errorf("could not subscribe for new block events: %w", err) } - return s.blockChan, nil + return nil } -func (s *subscriber) SubscribeForNotaryRequests(mainTXSigner util.Uint160) (<-chan *result.NotaryRequestEvent, error) { +func (s *subscriber) SubscribeForNotaryRequests(mainTXSigner util.Uint160) error { if err := s.client.SubscribeForNotaryRequests(mainTXSigner); err != nil { - return nil, fmt.Errorf("could not subscribe for notary request events: %w", err) + return fmt.Errorf("could not subscribe for notary request events: %w", err) } - return s.notaryChan, nil + return nil } func (s *subscriber) routeNotifications(ctx context.Context) { From 00377dca8343e326eb4d497ed733e6acd51fd793 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 12:24:01 +0300 Subject: [PATCH 0007/1943] [#199] putsvc: Refactor placement iterator Resolve funlen linter for iteratePlacement method Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/distributed.go | 111 ++++++++++++------------- 1 file changed, 53 insertions(+), 58 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 47104b323..3b5f4ec53 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -164,7 +164,6 @@ func (t *distributedTarget) sendObject(node nodeDesc) error { return nil } -// nolint: funlen func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transformer.AccessIdentifiers, error) { id, _ := t.obj.ID() @@ -175,72 +174,22 @@ func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transform return nil, fmt.Errorf("(%T) could not create object placement traverser: %w", t, err) } - var resErr atomic.Value + resErr := &atomic.Value{} -loop: for { addrs := traverser.Next() if len(addrs) == 0 { break } - wg := new(sync.WaitGroup) - - for i := range addrs { - if t.traversal.processed(addrs[i]) { - // it can happen only during additional container broadcast - continue - } - - wg.Add(1) - - addr := addrs[i] - - isLocal := t.isLocalKey(addr.PublicKey()) - - var workerPool util.WorkerPool - - if isLocal { - workerPool = t.localPool - } else { - workerPool = t.remotePool - } - - if err := workerPool.Submit(func() { - defer wg.Done() - - err := f(nodeDesc{local: isLocal, info: addr}) - - // mark the container node as processed in order to exclude it - // in subsequent container broadcast. Note that we don't - // process this node during broadcast if primary placement - // on it failed. - t.traversal.submitProcessed(addr) - - if err != nil { - resErr.Store(err) - svcutil.LogServiceError(t.log, "PUT", addr.Addresses(), err) - return - } - - traverser.SubmitSuccess() - }); err != nil { - wg.Done() - - svcutil.LogWorkerPoolError(t.log, "PUT", err) - - break loop - } + if t.iterateAddresses(traverser, addrs, f, resErr) { + break } - - wg.Wait() } if !traverser.Success() { var err errIncompletePut - err.singleErr, _ = resErr.Load().(error) - return nil, err } @@ -248,10 +197,7 @@ loop: if t.traversal.submitPrimaryPlacementFinish() { _, err = t.iteratePlacement(f) if err != nil { - t.log.Error("additional container broadcast failure", - zap.Error(err), - ) - + t.log.Error("additional container broadcast failure", zap.Error(err)) // we don't fail primary operation because of broadcast failure } } @@ -261,3 +207,52 @@ loop: return new(transformer.AccessIdentifiers). WithSelfID(id), nil } + +func (t *distributedTarget) iterateAddresses(traverser *placement.Traverser, addrs []placement.Node, f func(nodeDesc) error, resErr *atomic.Value) bool { + wg := &sync.WaitGroup{} + + for i := range addrs { + if t.traversal.processed(addrs[i]) { + // it can happen only during additional container broadcast + continue + } + + wg.Add(1) + + addr := addrs[i] + isLocal := t.isLocalKey(addr.PublicKey()) + + workerPool := t.remotePool + if isLocal { + workerPool = t.localPool + } + + if err := workerPool.Submit(func() { + defer wg.Done() + + err := f(nodeDesc{local: isLocal, info: addr}) + + // mark the container node as processed in order to exclude it + // in subsequent container broadcast. Note that we don't + // process this node during broadcast if primary placement + // on it failed. + t.traversal.submitProcessed(addr) + + if err != nil { + resErr.Store(err) + svcutil.LogServiceError(t.log, "PUT", addr.Addresses(), err) + return + } + + traverser.SubmitSuccess() + }); err != nil { + wg.Done() + svcutil.LogWorkerPoolError(t.log, "PUT", err) + return true + } + } + + wg.Wait() + + return false +} From c635164b4e36f7c15b30a1c7a742c9c26d4bce16 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 13:09:44 +0300 Subject: [PATCH 0008/1943] [#199] putsvc: Refactor streamer pool Resolve staticcheck linter for putBytesPool Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/distributed.go | 6 +++--- pkg/services/object/put/pool.go | 16 ++++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 3b5f4ec53..e4566157e 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -28,7 +28,7 @@ type distributedTarget struct { obj *objectSDK.Object objMeta object.ContentMeta - payload []byte + payload *payload nodeTargetInitializer func(nodeDesc) preparedObjectTarget @@ -122,7 +122,7 @@ func (t *distributedTarget) WriteHeader(obj *objectSDK.Object) error { } func (t *distributedTarget) Write(p []byte) (n int, err error) { - t.payload = append(t.payload, p...) + t.payload.Data = append(t.payload.Data, p...) return len(p), nil } @@ -133,7 +133,7 @@ func (t *distributedTarget) Close() (*transformer.AccessIdentifiers, error) { t.payload = nil }() - t.obj.SetPayload(t.payload) + t.obj.SetPayload(t.payload.Data) var err error diff --git a/pkg/services/object/put/pool.go b/pkg/services/object/put/pool.go index 705273227..5726856e5 100644 --- a/pkg/services/object/put/pool.go +++ b/pkg/services/object/put/pool.go @@ -6,15 +6,19 @@ import ( const defaultAllocSize = 1024 +type payload struct { + Data []byte +} + var putBytesPool = &sync.Pool{ - New: func() any { return make([]byte, 0, defaultAllocSize) }, + New: func() any { return &payload{Data: make([]byte, 0, defaultAllocSize)} }, } -func getPayload() []byte { - return putBytesPool.Get().([]byte) +func getPayload() *payload { + return putBytesPool.Get().(*payload) } -func putPayload(p []byte) { - //nolint:staticcheck - putBytesPool.Put(p[:0]) +func putPayload(p *payload) { + p.Data = p.Data[:0] + putBytesPool.Put(p) } From 5b59966b6a64e7765bd7eb46b2b5e1f576f2ff92 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 14:23:53 +0300 Subject: [PATCH 0009/1943] [#199] putsvc: Refactor put object Resolve containedctx linter for streamer and remote target Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 4 +- pkg/network/transport/object/grpc/service.go | 8 +-- pkg/services/object/acl/v2/service.go | 12 ++--- pkg/services/object/common.go | 4 +- pkg/services/object/delete/util.go | 6 +-- pkg/services/object/metrics.go | 14 ++--- pkg/services/object/put/distributed.go | 23 ++++---- pkg/services/object/put/local.go | 3 +- pkg/services/object/put/remote.go | 10 ++-- pkg/services/object/put/service.go | 5 +- pkg/services/object/put/streamer.go | 12 ++--- pkg/services/object/put/v2/service.go | 5 +- pkg/services/object/put/v2/streamer.go | 9 ++-- pkg/services/object/put/validation.go | 9 ++-- pkg/services/object/response.go | 20 +++---- pkg/services/object/server.go | 6 +-- pkg/services/object/sign.go | 20 +++---- pkg/services/object/transport_splitter.go | 4 +- .../object_manager/transformer/fmt.go | 9 ++-- .../object_manager/transformer/transformer.go | 34 ++++++------ .../object_manager/transformer/types.go | 6 +-- .../object_manager/transformer/writer.go | 52 +++++++++++++++++++ pkg/services/util/response/client_stream.go | 9 ++-- pkg/services/util/sign.go | 12 ++--- 24 files changed, 171 insertions(+), 125 deletions(-) create mode 100644 pkg/services/object_manager/transformer/writer.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 8680aac28..3b6bdcc7d 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -70,8 +70,8 @@ func (c *cfg) MaxObjectSize() uint64 { return sz } -func (s *objectSvc) Put(ctx context.Context) (objectService.PutObjectStream, error) { - return s.put.Put(ctx) +func (s *objectSvc) Put() (objectService.PutObjectStream, error) { + return s.put.Put() } func (s *objectSvc) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { diff --git a/pkg/network/transport/object/grpc/service.go b/pkg/network/transport/object/grpc/service.go index 82e323a3c..7fa60f99c 100644 --- a/pkg/network/transport/object/grpc/service.go +++ b/pkg/network/transport/object/grpc/service.go @@ -26,7 +26,7 @@ func New(c objectSvc.ServiceServer) *Server { // Put opens internal Object service Put stream and overtakes data from gRPC stream to it. func (s *Server) Put(gStream objectGRPC.ObjectService_PutServer) error { - stream, err := s.srv.Put(gStream.Context()) + stream, err := s.srv.Put() if err != nil { return err } @@ -35,7 +35,7 @@ func (s *Server) Put(gStream objectGRPC.ObjectService_PutServer) error { req, err := gStream.Recv() if err != nil { if errors.Is(err, io.EOF) { - resp, err := stream.CloseAndRecv() + resp, err := stream.CloseAndRecv(gStream.Context()) if err != nil { return err } @@ -51,9 +51,9 @@ func (s *Server) Put(gStream objectGRPC.ObjectService_PutServer) error { return err } - if err := stream.Send(putReq); err != nil { + if err := stream.Send(gStream.Context(), putReq); err != nil { if errors.Is(err, util.ErrAbortStream) { - resp, err := stream.CloseAndRecv() + resp, err := stream.CloseAndRecv(gStream.Context()) if err != nil { return err } diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 6bf8c4405..1e451a99f 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -165,8 +165,8 @@ func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream }) } -func (b Service) Put(ctx context.Context) (object.PutObjectStream, error) { - streamer, err := b.next.Put(ctx) +func (b Service) Put() (object.PutObjectStream, error) { + streamer, err := b.next.Put() return putStreamBasicChecker{ source: &b, @@ -444,7 +444,7 @@ func (b Service) GetRangeHash( } // nolint: funlen -func (p putStreamBasicChecker) Send(request *objectV2.PutRequest) error { +func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { body := request.GetBody() if body == nil { return errEmptyBody @@ -531,11 +531,11 @@ func (p putStreamBasicChecker) Send(request *objectV2.PutRequest) error { } } - return p.next.Send(request) + return p.next.Send(ctx, request) } -func (p putStreamBasicChecker) CloseAndRecv() (*objectV2.PutResponse, error) { - return p.next.CloseAndRecv() +func (p putStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PutResponse, error) { + return p.next.CloseAndRecv(ctx) } func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { diff --git a/pkg/services/object/common.go b/pkg/services/object/common.go index e797f1a64..5b139d8eb 100644 --- a/pkg/services/object/common.go +++ b/pkg/services/object/common.go @@ -42,12 +42,12 @@ func (x *Common) Get(req *objectV2.GetRequest, stream GetObjectStream) error { return x.nextHandler.Get(req, stream) } -func (x *Common) Put(ctx context.Context) (PutObjectStream, error) { +func (x *Common) Put() (PutObjectStream, error) { if x.state.IsMaintenance() { return nil, errMaintenance } - return x.nextHandler.Put(ctx) + return x.nextHandler.Put() } func (x *Common) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index a8ebb3065..cc5433740 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -108,7 +108,7 @@ func (s *simpleIDWriter) WriteIDs(ids []oid.ID) error { } func (w *putSvcWrapper) put(exec *execCtx) (*oid.ID, error) { - streamer, err := (*putsvc.Service)(w).Put(exec.context()) + streamer, err := (*putsvc.Service)(w).Put() if err != nil { return nil, err } @@ -124,12 +124,12 @@ func (w *putSvcWrapper) put(exec *execCtx) (*oid.ID, error) { return nil, err } - err = streamer.SendChunk(new(putsvc.PutChunkPrm).WithChunk(payload)) + err = streamer.SendChunk(exec.context(), new(putsvc.PutChunkPrm).WithChunk(payload)) if err != nil { return nil, err } - r, err := streamer.Close() + r, err := streamer.Close(exec.context()) if err != nil { return nil, err } diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index 9f15e834a..3ea16dafd 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -75,11 +75,11 @@ func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (er return } -func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) { +func (m MetricCollector) Put() (PutObjectStream, error) { if m.enabled { t := time.Now() - stream, err := m.next.Put(ctx) + stream, err := m.next.Put() if err != nil { return nil, err } @@ -90,7 +90,7 @@ func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) { start: t, }, nil } - return m.next.Put(ctx) + return m.next.Put() } func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) { @@ -179,17 +179,17 @@ func (s getStreamMetric) Send(resp *object.GetResponse) error { return s.stream.Send(resp) } -func (s putStreamMetric) Send(req *object.PutRequest) error { +func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error { chunk, ok := req.GetBody().GetObjectPart().(*object.PutObjectPartChunk) if ok { s.metrics.AddPutPayload(len(chunk.GetChunk())) } - return s.stream.Send(req) + return s.stream.Send(ctx, req) } -func (s putStreamMetric) CloseAndRecv() (*object.PutResponse, error) { - res, err := s.stream.CloseAndRecv() +func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { + res, err := s.stream.CloseAndRecv(ctx) s.metrics.IncPutReqCounter(err == nil) s.metrics.AddPutReqDuration(time.Since(s.start)) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index e4566157e..d8b59487e 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -1,6 +1,7 @@ package putsvc import ( + "context" "fmt" "sync" "sync/atomic" @@ -17,7 +18,7 @@ import ( type preparedObjectTarget interface { WriteObject(*objectSDK.Object, object.ContentMeta) error - Close() (*transformer.AccessIdentifiers, error) + Close(ctx context.Context) (*transformer.AccessIdentifiers, error) } type distributedTarget struct { @@ -121,13 +122,13 @@ func (t *distributedTarget) WriteHeader(obj *objectSDK.Object) error { return nil } -func (t *distributedTarget) Write(p []byte) (n int, err error) { +func (t *distributedTarget) Write(_ context.Context, p []byte) (n int, err error) { t.payload.Data = append(t.payload.Data, p...) return len(p), nil } -func (t *distributedTarget) Close() (*transformer.AccessIdentifiers, error) { +func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { defer func() { putPayload(t.payload) t.payload = nil @@ -146,10 +147,10 @@ func (t *distributedTarget) Close() (*transformer.AccessIdentifiers, error) { t.traversal.extraBroadcastEnabled = true } - return t.iteratePlacement(t.sendObject) + return t.iteratePlacement(ctx) } -func (t *distributedTarget) sendObject(node nodeDesc) error { +func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error { if !node.local && t.relay != nil { return t.relay(node) } @@ -158,13 +159,13 @@ func (t *distributedTarget) sendObject(node nodeDesc) error { if err := target.WriteObject(t.obj, t.objMeta); err != nil { return fmt.Errorf("could not write header: %w", err) - } else if _, err := target.Close(); err != nil { + } else if _, err := target.Close(ctx); err != nil { return fmt.Errorf("could not close object stream: %w", err) } return nil } -func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transformer.AccessIdentifiers, error) { +func (t *distributedTarget) iteratePlacement(ctx context.Context) (*transformer.AccessIdentifiers, error) { id, _ := t.obj.ID() traverser, err := placement.NewTraverser( @@ -182,7 +183,7 @@ func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transform break } - if t.iterateAddresses(traverser, addrs, f, resErr) { + if t.iterateAddresses(ctx, traverser, addrs, resErr) { break } } @@ -195,7 +196,7 @@ func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transform // perform additional container broadcast if needed if t.traversal.submitPrimaryPlacementFinish() { - _, err = t.iteratePlacement(f) + _, err = t.iteratePlacement(ctx) if err != nil { t.log.Error("additional container broadcast failure", zap.Error(err)) // we don't fail primary operation because of broadcast failure @@ -208,7 +209,7 @@ func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transform WithSelfID(id), nil } -func (t *distributedTarget) iterateAddresses(traverser *placement.Traverser, addrs []placement.Node, f func(nodeDesc) error, resErr *atomic.Value) bool { +func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *placement.Traverser, addrs []placement.Node, resErr *atomic.Value) bool { wg := &sync.WaitGroup{} for i := range addrs { @@ -230,7 +231,7 @@ func (t *distributedTarget) iterateAddresses(traverser *placement.Traverser, add if err := workerPool.Submit(func() { defer wg.Done() - err := f(nodeDesc{local: isLocal, info: addr}) + err := t.sendObject(ctx, nodeDesc{local: isLocal, info: addr}) // mark the container node as processed in order to exclude it // in subsequent container broadcast. Note that we don't diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index f344f77e9..12e3a2eee 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -1,6 +1,7 @@ package putsvc import ( + "context" "fmt" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -38,7 +39,7 @@ func (t *localTarget) WriteObject(obj *object.Object, meta objectCore.ContentMet return nil } -func (t *localTarget) Close() (*transformer.AccessIdentifiers, error) { +func (t *localTarget) Close(_ context.Context) (*transformer.AccessIdentifiers, error) { switch t.meta.Type() { case object.TypeTombstone: err := t.storage.Delete(objectCore.AddressOf(t.obj), t.meta.Objects()) diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/put/remote.go index 760de7508..6933abca6 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/put/remote.go @@ -15,10 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) -// nolint: containedctx type remoteTarget struct { - ctx context.Context - privateKey *ecdsa.PrivateKey commonPrm *util.CommonPrm @@ -51,7 +48,7 @@ func (t *remoteTarget) WriteObject(obj *object.Object, _ objectcore.ContentMeta) return nil } -func (t *remoteTarget) Close() (*transformer.AccessIdentifiers, error) { +func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { c, err := t.clientConstructor.Get(t.nodeInfo) if err != nil { return nil, fmt.Errorf("(%T) could not create SDK client %s: %w", t, t.nodeInfo, err) @@ -59,7 +56,7 @@ func (t *remoteTarget) Close() (*transformer.AccessIdentifiers, error) { var prm internalclient.PutObjectPrm - prm.SetContext(t.ctx) + prm.SetContext(ctx) prm.SetClient(c) prm.SetPrivateKey(t.privateKey) prm.SetSessionToken(t.commonPrm.SessionToken()) @@ -110,7 +107,6 @@ func (s *RemoteSender) PutObject(ctx context.Context, p *RemotePutPrm) error { } t := &remoteTarget{ - ctx: ctx, privateKey: key, clientConstructor: s.clientConstructor, } @@ -122,7 +118,7 @@ func (s *RemoteSender) PutObject(ctx context.Context, p *RemotePutPrm) error { if err := t.WriteObject(p.obj, objectcore.ContentMeta{}); err != nil { return fmt.Errorf("(%T) could not send object header: %w", s, err) - } else if _, err := t.Close(); err != nil { + } else if _, err := t.Close(ctx); err != nil { return fmt.Errorf("(%T) could not send object: %w", s, err) } diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index b74c97d49..567a3fea1 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -1,8 +1,6 @@ package putsvc import ( - "context" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -79,10 +77,9 @@ func NewService(opts ...Option) *Service { } } -func (p *Service) Put(ctx context.Context) (*Streamer, error) { +func (p *Service) Put() (*Streamer, error) { return &Streamer{ cfg: p.cfg, - ctx: ctx, }, nil } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 915b718a3..678cff572 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -16,12 +16,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) -// nolint: containedctx type Streamer struct { *cfg - ctx context.Context - sessionKey *ecdsa.PrivateKey target transformer.ObjectTarget @@ -232,7 +229,6 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { } rt := &remoteTarget{ - ctx: p.ctx, privateKey: p.sessionKey, commonPrm: prm.common, clientConstructor: p.clientConstructor, @@ -250,24 +246,24 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { } } -func (p *Streamer) SendChunk(prm *PutChunkPrm) error { +func (p *Streamer) SendChunk(ctx context.Context, prm *PutChunkPrm) error { if p.target == nil { return errNotInit } - if _, err := p.target.Write(prm.chunk); err != nil { + if _, err := p.target.Write(ctx, prm.chunk); err != nil { return fmt.Errorf("(%T) could not write payload chunk to target: %w", p, err) } return nil } -func (p *Streamer) Close() (*PutResponse, error) { +func (p *Streamer) Close(ctx context.Context) (*PutResponse, error) { if p.target == nil { return nil, errNotInit } - ids, err := p.target.Close() + ids, err := p.target.Close(ctx) if err != nil { return nil, fmt.Errorf("(%T) could not close object target: %w", p, err) } diff --git a/pkg/services/object/put/v2/service.go b/pkg/services/object/put/v2/service.go index 7d0dfc613..656f8df9c 100644 --- a/pkg/services/object/put/v2/service.go +++ b/pkg/services/object/put/v2/service.go @@ -1,7 +1,6 @@ package putsvc import ( - "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" @@ -36,8 +35,8 @@ func NewService(opts ...Option) *Service { } // Put calls internal service and returns v2 object streamer. -func (s *Service) Put(ctx context.Context) (object.PutObjectStream, error) { - stream, err := s.svc.Put(ctx) +func (s *Service) Put() (object.PutObjectStream, error) { + stream, err := s.svc.Put() if err != nil { return nil, fmt.Errorf("(%T) could not open object put stream: %w", s, err) } diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 85827cd4c..65846ea9f 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -1,6 +1,7 @@ package putsvc import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" @@ -32,7 +33,7 @@ type sizes struct { writtenPayload uint64 // sum size of already cached chunks } -func (s *streamer) Send(req *object.PutRequest) (err error) { +func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) { switch v := req.GetBody().GetObjectPart().(type) { case *object.PutObjectPartInit: var initPrm *putsvc.PutInitPrm @@ -71,7 +72,7 @@ func (s *streamer) Send(req *object.PutRequest) (err error) { } } - if err = s.stream.SendChunk(toChunkPrm(v)); err != nil { + if err = s.stream.SendChunk(ctx, toChunkPrm(v)); err != nil { err = fmt.Errorf("(%T) could not send payload chunk: %w", s, err) } @@ -103,7 +104,7 @@ func (s *streamer) Send(req *object.PutRequest) (err error) { return signature.SignServiceMessage(key, req) } -func (s *streamer) CloseAndRecv() (*object.PutResponse, error) { +func (s *streamer) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { if s.saveChunks { // check payload size correctness if s.writtenPayload != s.payloadSz { @@ -111,7 +112,7 @@ func (s *streamer) CloseAndRecv() (*object.PutResponse, error) { } } - resp, err := s.stream.Close() + resp, err := s.stream.Close(ctx) if err != nil { return nil, fmt.Errorf("(%T) could not object put stream: %w", s, err) } diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/put/validation.go index 2d6ada5a1..70c6974d3 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/put/validation.go @@ -2,6 +2,7 @@ package putsvc import ( "bytes" + "context" "crypto/sha256" "errors" "fmt" @@ -92,7 +93,7 @@ func (t *validatingTarget) WriteHeader(obj *objectSDK.Object) error { return nil } -func (t *validatingTarget) Write(p []byte) (n int, err error) { +func (t *validatingTarget) Write(ctx context.Context, p []byte) (n int, err error) { chunkLn := uint64(len(p)) if !t.unpreparedObject { @@ -107,7 +108,7 @@ func (t *validatingTarget) Write(p []byte) (n int, err error) { } } - n, err = t.nextTarget.Write(p) + n, err = t.nextTarget.Write(ctx, p) if err == nil { t.writtenPayload += uint64(n) } @@ -115,7 +116,7 @@ func (t *validatingTarget) Write(p []byte) (n int, err error) { return } -func (t *validatingTarget) Close() (*transformer.AccessIdentifiers, error) { +func (t *validatingTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { if !t.unpreparedObject { // check payload size correctness if t.payloadSz != t.writtenPayload { @@ -127,5 +128,5 @@ func (t *validatingTarget) Close() (*transformer.AccessIdentifiers, error) { } } - return t.nextTarget.Close() + return t.nextTarget.Close(ctx) } diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index 4da2b23a7..def934ea6 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -59,12 +59,12 @@ func (s *ResponseService) Get(req *object.GetRequest, stream GetObjectStream) er }) } -func (s *putStreamResponser) Send(req *object.PutRequest) error { - return s.stream.Send(req) +func (s *putStreamResponser) Send(ctx context.Context, req *object.PutRequest) error { + return s.stream.Send(ctx, req) } -func (s *putStreamResponser) CloseAndRecv() (*object.PutResponse, error) { - r, err := s.stream.CloseAndRecv() +func (s *putStreamResponser) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { + r, err := s.stream.CloseAndRecv(ctx) if err != nil { return nil, fmt.Errorf("(%T) could not receive response: %w", s, err) } @@ -72,19 +72,19 @@ func (s *putStreamResponser) CloseAndRecv() (*object.PutResponse, error) { return r.(*object.PutResponse), nil } -func (s *ResponseService) Put(ctx context.Context) (PutObjectStream, error) { - stream, err := s.svc.Put(ctx) +func (s *ResponseService) Put() (PutObjectStream, error) { + stream, err := s.svc.Put() if err != nil { return nil, fmt.Errorf("could not create Put object streamer: %w", err) } return &putStreamResponser{ stream: s.respSvc.CreateRequestStreamer( - func(req any) error { - return stream.Send(req.(*object.PutRequest)) + func(ctx context.Context, req any) error { + return stream.Send(ctx, req.(*object.PutRequest)) }, - func() (util.ResponseMessage, error) { - return stream.CloseAndRecv() + func(ctx context.Context) (util.ResponseMessage, error) { + return stream.CloseAndRecv(ctx) }, ), }, nil diff --git a/pkg/services/object/server.go b/pkg/services/object/server.go index d95c6c906..ccce9c4f4 100644 --- a/pkg/services/object/server.go +++ b/pkg/services/object/server.go @@ -27,15 +27,15 @@ type SearchStream interface { // PutObjectStream is an interface of FrostFS API v2 compatible client's object streamer. type PutObjectStream interface { - Send(*object.PutRequest) error - CloseAndRecv() (*object.PutResponse, error) + Send(context.Context, *object.PutRequest) error + CloseAndRecv(context.Context) (*object.PutResponse, error) } // ServiceServer is an interface of utility // serving v2 Object service. type ServiceServer interface { Get(*object.GetRequest, GetObjectStream) error - Put(context.Context) (PutObjectStream, error) + Put() (PutObjectStream, error) Head(context.Context, *object.HeadRequest) (*object.HeadResponse, error) Search(*object.SearchRequest, SearchStream) error Delete(context.Context, *object.DeleteRequest) (*object.DeleteResponse, error) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 585fc659a..9d66c76ba 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -70,12 +70,12 @@ func (s *SignService) Get(req *object.GetRequest, stream GetObjectStream) error ) } -func (s *putStreamSigner) Send(req *object.PutRequest) error { - return s.stream.Send(req) +func (s *putStreamSigner) Send(ctx context.Context, req *object.PutRequest) error { + return s.stream.Send(ctx, req) } -func (s *putStreamSigner) CloseAndRecv() (*object.PutResponse, error) { - r, err := s.stream.CloseAndRecv() +func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { + r, err := s.stream.CloseAndRecv(ctx) if err != nil { return nil, fmt.Errorf("could not receive response: %w", err) } @@ -83,19 +83,19 @@ func (s *putStreamSigner) CloseAndRecv() (*object.PutResponse, error) { return r.(*object.PutResponse), nil } -func (s *SignService) Put(ctx context.Context) (PutObjectStream, error) { - stream, err := s.svc.Put(ctx) +func (s *SignService) Put() (PutObjectStream, error) { + stream, err := s.svc.Put() if err != nil { return nil, fmt.Errorf("could not create Put object streamer: %w", err) } return &putStreamSigner{ stream: s.sigSvc.CreateRequestStreamer( - func(req any) error { - return stream.Send(req.(*object.PutRequest)) + func(ctx context.Context, req any) error { + return stream.Send(ctx, req.(*object.PutRequest)) }, - func() (util.ResponseMessage, error) { - return stream.CloseAndRecv() + func(ctx context.Context) (util.ResponseMessage, error) { + return stream.CloseAndRecv(ctx) }, func() util.ResponseMessage { return new(object.PutResponse) diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index 3836103de..a7d1c486a 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -87,8 +87,8 @@ func (c *TransportSplitter) Get(req *object.GetRequest, stream GetObjectStream) }) } -func (c TransportSplitter) Put(ctx context.Context) (PutObjectStream, error) { - return c.next.Put(ctx) +func (c TransportSplitter) Put() (PutObjectStream, error) { + return c.next.Put() } func (c TransportSplitter) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) { diff --git a/pkg/services/object_manager/transformer/fmt.go b/pkg/services/object_manager/transformer/fmt.go index c9b5dc967..462cc7474 100644 --- a/pkg/services/object_manager/transformer/fmt.go +++ b/pkg/services/object_manager/transformer/fmt.go @@ -1,6 +1,7 @@ package transformer import ( + "context" "crypto/ecdsa" "fmt" @@ -53,15 +54,15 @@ func (f *formatter) WriteHeader(obj *object.Object) error { return nil } -func (f *formatter) Write(p []byte) (n int, err error) { - n, err = f.prm.NextTarget.Write(p) +func (f *formatter) Write(ctx context.Context, p []byte) (n int, err error) { + n, err = f.prm.NextTarget.Write(ctx, p) f.sz += uint64(n) return } -func (f *formatter) Close() (*AccessIdentifiers, error) { +func (f *formatter) Close(ctx context.Context) (*AccessIdentifiers, error) { curEpoch := f.prm.NetworkState.CurrentEpoch() ver := version.Current() @@ -100,7 +101,7 @@ func (f *formatter) Close() (*AccessIdentifiers, error) { return nil, fmt.Errorf("could not write header to next target: %w", err) } - if _, err := f.prm.NextTarget.Close(); err != nil { + if _, err := f.prm.NextTarget.Close(ctx); err != nil { return nil, fmt.Errorf("could not close next target: %w", err) } diff --git a/pkg/services/object_manager/transformer/transformer.go b/pkg/services/object_manager/transformer/transformer.go index 7b717d3df..199f5d0c1 100644 --- a/pkg/services/object_manager/transformer/transformer.go +++ b/pkg/services/object_manager/transformer/transformer.go @@ -1,10 +1,10 @@ package transformer import ( + "context" "crypto/sha256" "fmt" "hash" - "io" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -27,7 +27,7 @@ type payloadSizeLimiter struct { previous []oid.ID - chunkWriter io.Writer + chunkWriter writer splitID *object.SplitID @@ -64,16 +64,16 @@ func (s *payloadSizeLimiter) WriteHeader(hdr *object.Object) error { return nil } -func (s *payloadSizeLimiter) Write(p []byte) (int, error) { - if err := s.writeChunk(p); err != nil { +func (s *payloadSizeLimiter) Write(ctx context.Context, p []byte) (int, error) { + if err := s.writeChunk(ctx, p); err != nil { return 0, err } return len(p), nil } -func (s *payloadSizeLimiter) Close() (*AccessIdentifiers, error) { - return s.release(true) +func (s *payloadSizeLimiter) Close(ctx context.Context) (*AccessIdentifiers, error) { + return s.release(ctx, true) } func (s *payloadSizeLimiter) initialize() { @@ -117,19 +117,19 @@ func (s *payloadSizeLimiter) initializeCurrent() { s.currentHashers = payloadHashersForObject(s.current, s.withoutHomomorphicHash) // compose multi-writer from target and all payload hashers - ws := make([]io.Writer, 0, 1+len(s.currentHashers)+len(s.parentHashers)) + ws := make([]writer, 0, 1+len(s.currentHashers)+len(s.parentHashers)) ws = append(ws, s.target) for i := range s.currentHashers { - ws = append(ws, s.currentHashers[i].hasher) + ws = append(ws, newWriter(s.currentHashers[i].hasher)) } for i := range s.parentHashers { - ws = append(ws, s.parentHashers[i].hasher) + ws = append(ws, newWriter(s.parentHashers[i].hasher)) } - s.chunkWriter = io.MultiWriter(ws...) + s.chunkWriter = newMultiWriter(ws...) } func payloadHashersForObject(obj *object.Object, withoutHomomorphicHash bool) []*payloadChecksumHasher { @@ -174,7 +174,7 @@ func payloadHashersForObject(obj *object.Object, withoutHomomorphicHash bool) [] return hashers } -func (s *payloadSizeLimiter) release(finalize bool) (*AccessIdentifiers, error) { +func (s *payloadSizeLimiter) release(ctx context.Context, finalize bool) (*AccessIdentifiers, error) { // Arg finalize is true only when called from Close method. // We finalize parent and generate linking objects only if it is more // than 1 object in split-chain. @@ -194,7 +194,7 @@ func (s *payloadSizeLimiter) release(finalize bool) (*AccessIdentifiers, error) return nil, fmt.Errorf("could not write header: %w", err) } - ids, err := s.target.Close() + ids, err := s.target.Close(ctx) if err != nil { return nil, fmt.Errorf("could not close target: %w", err) } @@ -207,7 +207,7 @@ func (s *payloadSizeLimiter) release(finalize bool) (*AccessIdentifiers, error) s.initializeLinking(ids.Parent()) s.initializeCurrent() - if _, err := s.release(false); err != nil { + if _, err := s.release(ctx, false); err != nil { return nil, fmt.Errorf("could not release linking object: %w", err) } } @@ -228,7 +228,7 @@ func (s *payloadSizeLimiter) initializeLinking(parHdr *object.Object) { s.current.SetSplitID(s.splitID) } -func (s *payloadSizeLimiter) writeChunk(chunk []byte) error { +func (s *payloadSizeLimiter) writeChunk(ctx context.Context, chunk []byte) error { // statement is true if the previous write of bytes reached exactly the boundary. if s.written > 0 && s.written%s.maxSize == 0 { if s.written == s.maxSize { @@ -236,7 +236,7 @@ func (s *payloadSizeLimiter) writeChunk(chunk []byte) error { } // we need to release current object - if _, err := s.release(false); err != nil { + if _, err := s.release(ctx, false); err != nil { return fmt.Errorf("could not release object: %w", err) } @@ -255,7 +255,7 @@ func (s *payloadSizeLimiter) writeChunk(chunk []byte) error { cut = leftToEdge } - if _, err := s.chunkWriter.Write(chunk[:cut]); err != nil { + if _, err := s.chunkWriter.Write(ctx, chunk[:cut]); err != nil { return fmt.Errorf("could not write chunk to target: %w", err) } @@ -264,7 +264,7 @@ func (s *payloadSizeLimiter) writeChunk(chunk []byte) error { // if there are more bytes in buffer we call method again to start filling another object if ln > leftToEdge { - return s.writeChunk(chunk[cut:]) + return s.writeChunk(ctx, chunk[cut:]) } return nil diff --git a/pkg/services/object_manager/transformer/types.go b/pkg/services/object_manager/transformer/types.go index 0fa3b6436..3e6e2feff 100644 --- a/pkg/services/object_manager/transformer/types.go +++ b/pkg/services/object_manager/transformer/types.go @@ -1,7 +1,7 @@ package transformer import ( - "io" + "context" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -35,7 +35,7 @@ type ObjectTarget interface { // Can be called multiple times. // // Must not be called after Close call. - io.Writer + Write(ctx context.Context, p []byte) (n int, err error) // Close is used to finish object writing. // @@ -45,7 +45,7 @@ type ObjectTarget interface { // Must be called no more than once. Control remains with the caller. // Re-calling can lead to undefined behavior // that depends on the implementation. - Close() (*AccessIdentifiers, error) + Close(ctx context.Context) (*AccessIdentifiers, error) } // TargetInitializer represents ObjectTarget constructor. diff --git a/pkg/services/object_manager/transformer/writer.go b/pkg/services/object_manager/transformer/writer.go new file mode 100644 index 000000000..27aed16ff --- /dev/null +++ b/pkg/services/object_manager/transformer/writer.go @@ -0,0 +1,52 @@ +package transformer + +import ( + "context" + "io" +) + +type writer interface { + Write(ctx context.Context, p []byte) (n int, err error) +} + +type multiWriter struct { + writers []writer +} + +func (t *multiWriter) Write(ctx context.Context, p []byte) (n int, err error) { + for _, w := range t.writers { + n, err = w.Write(ctx, p) + if err != nil { + return + } + if n != len(p) { + err = io.ErrShortWrite + return + } + } + return len(p), nil +} + +func newMultiWriter(writers ...writer) writer { + allWriters := make([]writer, 0, len(writers)) + for _, w := range writers { + if mw, ok := w.(*multiWriter); ok { + allWriters = append(allWriters, mw.writers...) + } else { + allWriters = append(allWriters, w) + } + } + return &multiWriter{allWriters} +} + +type writerWrapper struct { + Writer io.Writer +} + +func (w *writerWrapper) Write(_ context.Context, p []byte) (n int, err error) { + return w.Writer.Write(p) +} + +func newWriter(w io.Writer) writer { + return &writerWrapper{Writer: w} +} diff --git a/pkg/services/util/response/client_stream.go b/pkg/services/util/response/client_stream.go index f167f005a..b541c73db 100644 --- a/pkg/services/util/response/client_stream.go +++ b/pkg/services/util/response/client_stream.go @@ -1,6 +1,7 @@ package response import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" @@ -17,8 +18,8 @@ type ClientMessageStreamer struct { } // Send calls send method of internal streamer. -func (s *ClientMessageStreamer) Send(req any) error { - if err := s.send(req); err != nil { +func (s *ClientMessageStreamer) Send(ctx context.Context, req any) error { + if err := s.send(ctx, req); err != nil { return fmt.Errorf("(%T) could not send the request: %w", s, err) } return nil @@ -26,8 +27,8 @@ func (s *ClientMessageStreamer) Send(req any) error { // CloseAndRecv closes internal stream, receivers the response, // sets meta values and returns the result. -func (s *ClientMessageStreamer) CloseAndRecv() (util.ResponseMessage, error) { - resp, err := s.close() +func (s *ClientMessageStreamer) CloseAndRecv(ctx context.Context) (util.ResponseMessage, error) { + resp, err := s.close(ctx) if err != nil { return nil, fmt.Errorf("(%T) could not close stream and receive response: %w", s, err) } diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index 2478e6256..cb4be3084 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -37,9 +37,9 @@ var ErrAbortStream = errors.New("abort message stream") type ResponseConstructor func() ResponseMessage -type RequestMessageWriter func(any) error +type RequestMessageWriter func(context.Context, any) error -type ClientStreamCloser func() (ResponseMessage, error) +type ClientStreamCloser func(context.Context) (ResponseMessage, error) type RequestMessageStreamer struct { key *ecdsa.PrivateKey @@ -61,7 +61,7 @@ func NewUnarySignService(key *ecdsa.PrivateKey) *SignService { } } -func (s *RequestMessageStreamer) Send(req any) error { +func (s *RequestMessageStreamer) Send(ctx context.Context, req any) error { // req argument should be strengthen with type RequestMessage s.statusSupported = isStatusSupported(req.(RequestMessage)) // panic is OK here for now @@ -71,7 +71,7 @@ func (s *RequestMessageStreamer) Send(req any) error { if err = signature.VerifyServiceMessage(req); err != nil { err = fmt.Errorf("could not verify request: %w", err) } else { - err = s.send(req) + err = s.send(ctx, req) } if err != nil { @@ -87,7 +87,7 @@ func (s *RequestMessageStreamer) Send(req any) error { return nil } -func (s *RequestMessageStreamer) CloseAndRecv() (ResponseMessage, error) { +func (s *RequestMessageStreamer) CloseAndRecv(ctx context.Context) (ResponseMessage, error) { var ( resp ResponseMessage err error @@ -96,7 +96,7 @@ func (s *RequestMessageStreamer) CloseAndRecv() (ResponseMessage, error) { if s.sendErr != nil { err = s.sendErr } else { - resp, err = s.close() + resp, err = s.close(ctx) if err != nil { err = fmt.Errorf("could not close stream and receive response: %w", err) } From f6614e2a0b3a6663bbf27adc4d30deed9b3b5e3f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 15:40:52 +0300 Subject: [PATCH 0010/1943] [#199] putsvc: Refactor streamer initialization Resolve funlen linter for initTarget method Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/streamer.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 678cff572..fed161e03 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -51,7 +51,6 @@ func (p *Streamer) MaxObjectSize() uint64 { return p.maxPayloadSz } -// nolint: funlen func (p *Streamer) initTarget(prm *PutInitPrm) error { // prevent re-calling if p.target != nil { @@ -69,19 +68,26 @@ func (p *Streamer) initTarget(prm *PutInitPrm) error { } if prm.hdr.Signature() != nil { - p.relay = prm.relay + return p.initUntrustedTarget(prm) + } + return p.initTrustedTarget(prm) +} - // prepare untrusted-Put object target - p.target = &validatingTarget{ - nextTarget: p.newCommonTarget(prm), - fmt: p.fmtValidator, +func (p *Streamer) initUntrustedTarget(prm *PutInitPrm) error { + p.relay = prm.relay - maxPayloadSz: p.maxPayloadSz, - } + // prepare untrusted-Put object target + p.target = &validatingTarget{ + nextTarget: p.newCommonTarget(prm), + fmt: p.fmtValidator, - return nil + maxPayloadSz: p.maxPayloadSz, } + return nil +} + +func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { sToken := prm.common.SessionToken() // prepare trusted-Put object target From 14d894178e29be1e97de160905f3fc7504bd7693 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 12:24:01 +0300 Subject: [PATCH 0011/1943] [#199] putsvc: Refactor placement iterator Resolve funlen linter for iteratePlacement method Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/distributed.go | 111 ++++++++++++------------- 1 file changed, 53 insertions(+), 58 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 47104b323..3b5f4ec53 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -164,7 +164,6 @@ func (t *distributedTarget) sendObject(node nodeDesc) error { return nil } -// nolint: funlen func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transformer.AccessIdentifiers, error) { id, _ := t.obj.ID() @@ -175,72 +174,22 @@ func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transform return nil, fmt.Errorf("(%T) could not create object placement traverser: %w", t, err) } - var resErr atomic.Value + resErr := &atomic.Value{} -loop: for { addrs := traverser.Next() if len(addrs) == 0 { break } - wg := new(sync.WaitGroup) - - for i := range addrs { - if t.traversal.processed(addrs[i]) { - // it can happen only during additional container broadcast - continue - } - - wg.Add(1) - - addr := addrs[i] - - isLocal := t.isLocalKey(addr.PublicKey()) - - var workerPool util.WorkerPool - - if isLocal { - workerPool = t.localPool - } else { - workerPool = t.remotePool - } - - if err := workerPool.Submit(func() { - defer wg.Done() - - err := f(nodeDesc{local: isLocal, info: addr}) - - // mark the container node as processed in order to exclude it - // in subsequent container broadcast. Note that we don't - // process this node during broadcast if primary placement - // on it failed. - t.traversal.submitProcessed(addr) - - if err != nil { - resErr.Store(err) - svcutil.LogServiceError(t.log, "PUT", addr.Addresses(), err) - return - } - - traverser.SubmitSuccess() - }); err != nil { - wg.Done() - - svcutil.LogWorkerPoolError(t.log, "PUT", err) - - break loop - } + if t.iterateAddresses(traverser, addrs, f, resErr) { + break } - - wg.Wait() } if !traverser.Success() { var err errIncompletePut - err.singleErr, _ = resErr.Load().(error) - return nil, err } @@ -248,10 +197,7 @@ loop: if t.traversal.submitPrimaryPlacementFinish() { _, err = t.iteratePlacement(f) if err != nil { - t.log.Error("additional container broadcast failure", - zap.Error(err), - ) - + t.log.Error("additional container broadcast failure", zap.Error(err)) // we don't fail primary operation because of broadcast failure } } @@ -261,3 +207,52 @@ loop: return new(transformer.AccessIdentifiers). WithSelfID(id), nil } + +func (t *distributedTarget) iterateAddresses(traverser *placement.Traverser, addrs []placement.Node, f func(nodeDesc) error, resErr *atomic.Value) bool { + wg := &sync.WaitGroup{} + + for i := range addrs { + if t.traversal.processed(addrs[i]) { + // it can happen only during additional container broadcast + continue + } + + wg.Add(1) + + addr := addrs[i] + isLocal := t.isLocalKey(addr.PublicKey()) + + workerPool := t.remotePool + if isLocal { + workerPool = t.localPool + } + + if err := workerPool.Submit(func() { + defer wg.Done() + + err := f(nodeDesc{local: isLocal, info: addr}) + + // mark the container node as processed in order to exclude it + // in subsequent container broadcast. Note that we don't + // process this node during broadcast if primary placement + // on it failed. + t.traversal.submitProcessed(addr) + + if err != nil { + resErr.Store(err) + svcutil.LogServiceError(t.log, "PUT", addr.Addresses(), err) + return + } + + traverser.SubmitSuccess() + }); err != nil { + wg.Done() + svcutil.LogWorkerPoolError(t.log, "PUT", err) + return true + } + } + + wg.Wait() + + return false +} From cecea8053a8e394994a8f85765a7be07b0da394f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 13:09:44 +0300 Subject: [PATCH 0012/1943] [#199] putsvc: Refactor streamer pool Resolve staticcheck linter for putBytesPool Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/distributed.go | 6 +++--- pkg/services/object/put/pool.go | 16 ++++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 3b5f4ec53..e4566157e 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -28,7 +28,7 @@ type distributedTarget struct { obj *objectSDK.Object objMeta object.ContentMeta - payload []byte + payload *payload nodeTargetInitializer func(nodeDesc) preparedObjectTarget @@ -122,7 +122,7 @@ func (t *distributedTarget) WriteHeader(obj *objectSDK.Object) error { } func (t *distributedTarget) Write(p []byte) (n int, err error) { - t.payload = append(t.payload, p...) + t.payload.Data = append(t.payload.Data, p...) return len(p), nil } @@ -133,7 +133,7 @@ func (t *distributedTarget) Close() (*transformer.AccessIdentifiers, error) { t.payload = nil }() - t.obj.SetPayload(t.payload) + t.obj.SetPayload(t.payload.Data) var err error diff --git a/pkg/services/object/put/pool.go b/pkg/services/object/put/pool.go index 705273227..5726856e5 100644 --- a/pkg/services/object/put/pool.go +++ b/pkg/services/object/put/pool.go @@ -6,15 +6,19 @@ import ( const defaultAllocSize = 1024 +type payload struct { + Data []byte +} + var putBytesPool = &sync.Pool{ - New: func() any { return make([]byte, 0, defaultAllocSize) }, + New: func() any { return &payload{Data: make([]byte, 0, defaultAllocSize)} }, } -func getPayload() []byte { - return putBytesPool.Get().([]byte) +func getPayload() *payload { + return putBytesPool.Get().(*payload) } -func putPayload(p []byte) { - //nolint:staticcheck - putBytesPool.Put(p[:0]) +func putPayload(p *payload) { + p.Data = p.Data[:0] + putBytesPool.Put(p) } From 27bdddc48f1ae68d9eee638b28906fab1ee22938 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 14:23:53 +0300 Subject: [PATCH 0013/1943] [#199] putsvc: Refactor put object Resolve containedctx linter for streamer and remote target Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 4 +- pkg/network/transport/object/grpc/service.go | 8 +-- pkg/services/object/acl/v2/service.go | 12 ++--- pkg/services/object/common.go | 4 +- pkg/services/object/delete/util.go | 6 +-- pkg/services/object/metrics.go | 14 ++--- pkg/services/object/put/distributed.go | 23 ++++---- pkg/services/object/put/local.go | 3 +- pkg/services/object/put/remote.go | 10 ++-- pkg/services/object/put/service.go | 5 +- pkg/services/object/put/streamer.go | 12 ++--- pkg/services/object/put/v2/service.go | 5 +- pkg/services/object/put/v2/streamer.go | 9 ++-- pkg/services/object/put/validation.go | 9 ++-- pkg/services/object/response.go | 20 +++---- pkg/services/object/server.go | 6 +-- pkg/services/object/sign.go | 20 +++---- pkg/services/object/transport_splitter.go | 4 +- .../object_manager/transformer/fmt.go | 9 ++-- .../object_manager/transformer/transformer.go | 34 ++++++------ .../object_manager/transformer/types.go | 6 +-- .../object_manager/transformer/writer.go | 52 +++++++++++++++++++ pkg/services/util/response/client_stream.go | 9 ++-- pkg/services/util/sign.go | 12 ++--- 24 files changed, 171 insertions(+), 125 deletions(-) create mode 100644 pkg/services/object_manager/transformer/writer.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 8680aac28..3b6bdcc7d 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -70,8 +70,8 @@ func (c *cfg) MaxObjectSize() uint64 { return sz } -func (s *objectSvc) Put(ctx context.Context) (objectService.PutObjectStream, error) { - return s.put.Put(ctx) +func (s *objectSvc) Put() (objectService.PutObjectStream, error) { + return s.put.Put() } func (s *objectSvc) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { diff --git a/pkg/network/transport/object/grpc/service.go b/pkg/network/transport/object/grpc/service.go index 82e323a3c..7fa60f99c 100644 --- a/pkg/network/transport/object/grpc/service.go +++ b/pkg/network/transport/object/grpc/service.go @@ -26,7 +26,7 @@ func New(c objectSvc.ServiceServer) *Server { // Put opens internal Object service Put stream and overtakes data from gRPC stream to it. func (s *Server) Put(gStream objectGRPC.ObjectService_PutServer) error { - stream, err := s.srv.Put(gStream.Context()) + stream, err := s.srv.Put() if err != nil { return err } @@ -35,7 +35,7 @@ func (s *Server) Put(gStream objectGRPC.ObjectService_PutServer) error { req, err := gStream.Recv() if err != nil { if errors.Is(err, io.EOF) { - resp, err := stream.CloseAndRecv() + resp, err := stream.CloseAndRecv(gStream.Context()) if err != nil { return err } @@ -51,9 +51,9 @@ func (s *Server) Put(gStream objectGRPC.ObjectService_PutServer) error { return err } - if err := stream.Send(putReq); err != nil { + if err := stream.Send(gStream.Context(), putReq); err != nil { if errors.Is(err, util.ErrAbortStream) { - resp, err := stream.CloseAndRecv() + resp, err := stream.CloseAndRecv(gStream.Context()) if err != nil { return err } diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 6bf8c4405..1e451a99f 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -165,8 +165,8 @@ func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream }) } -func (b Service) Put(ctx context.Context) (object.PutObjectStream, error) { - streamer, err := b.next.Put(ctx) +func (b Service) Put() (object.PutObjectStream, error) { + streamer, err := b.next.Put() return putStreamBasicChecker{ source: &b, @@ -444,7 +444,7 @@ func (b Service) GetRangeHash( } // nolint: funlen -func (p putStreamBasicChecker) Send(request *objectV2.PutRequest) error { +func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { body := request.GetBody() if body == nil { return errEmptyBody @@ -531,11 +531,11 @@ func (p putStreamBasicChecker) Send(request *objectV2.PutRequest) error { } } - return p.next.Send(request) + return p.next.Send(ctx, request) } -func (p putStreamBasicChecker) CloseAndRecv() (*objectV2.PutResponse, error) { - return p.next.CloseAndRecv() +func (p putStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PutResponse, error) { + return p.next.CloseAndRecv(ctx) } func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { diff --git a/pkg/services/object/common.go b/pkg/services/object/common.go index e797f1a64..5b139d8eb 100644 --- a/pkg/services/object/common.go +++ b/pkg/services/object/common.go @@ -42,12 +42,12 @@ func (x *Common) Get(req *objectV2.GetRequest, stream GetObjectStream) error { return x.nextHandler.Get(req, stream) } -func (x *Common) Put(ctx context.Context) (PutObjectStream, error) { +func (x *Common) Put() (PutObjectStream, error) { if x.state.IsMaintenance() { return nil, errMaintenance } - return x.nextHandler.Put(ctx) + return x.nextHandler.Put() } func (x *Common) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index a8ebb3065..cc5433740 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -108,7 +108,7 @@ func (s *simpleIDWriter) WriteIDs(ids []oid.ID) error { } func (w *putSvcWrapper) put(exec *execCtx) (*oid.ID, error) { - streamer, err := (*putsvc.Service)(w).Put(exec.context()) + streamer, err := (*putsvc.Service)(w).Put() if err != nil { return nil, err } @@ -124,12 +124,12 @@ func (w *putSvcWrapper) put(exec *execCtx) (*oid.ID, error) { return nil, err } - err = streamer.SendChunk(new(putsvc.PutChunkPrm).WithChunk(payload)) + err = streamer.SendChunk(exec.context(), new(putsvc.PutChunkPrm).WithChunk(payload)) if err != nil { return nil, err } - r, err := streamer.Close() + r, err := streamer.Close(exec.context()) if err != nil { return nil, err } diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index 9f15e834a..3ea16dafd 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -75,11 +75,11 @@ func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (er return } -func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) { +func (m MetricCollector) Put() (PutObjectStream, error) { if m.enabled { t := time.Now() - stream, err := m.next.Put(ctx) + stream, err := m.next.Put() if err != nil { return nil, err } @@ -90,7 +90,7 @@ func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) { start: t, }, nil } - return m.next.Put(ctx) + return m.next.Put() } func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) { @@ -179,17 +179,17 @@ func (s getStreamMetric) Send(resp *object.GetResponse) error { return s.stream.Send(resp) } -func (s putStreamMetric) Send(req *object.PutRequest) error { +func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error { chunk, ok := req.GetBody().GetObjectPart().(*object.PutObjectPartChunk) if ok { s.metrics.AddPutPayload(len(chunk.GetChunk())) } - return s.stream.Send(req) + return s.stream.Send(ctx, req) } -func (s putStreamMetric) CloseAndRecv() (*object.PutResponse, error) { - res, err := s.stream.CloseAndRecv() +func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { + res, err := s.stream.CloseAndRecv(ctx) s.metrics.IncPutReqCounter(err == nil) s.metrics.AddPutReqDuration(time.Since(s.start)) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index e4566157e..d8b59487e 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -1,6 +1,7 @@ package putsvc import ( + "context" "fmt" "sync" "sync/atomic" @@ -17,7 +18,7 @@ import ( type preparedObjectTarget interface { WriteObject(*objectSDK.Object, object.ContentMeta) error - Close() (*transformer.AccessIdentifiers, error) + Close(ctx context.Context) (*transformer.AccessIdentifiers, error) } type distributedTarget struct { @@ -121,13 +122,13 @@ func (t *distributedTarget) WriteHeader(obj *objectSDK.Object) error { return nil } -func (t *distributedTarget) Write(p []byte) (n int, err error) { +func (t *distributedTarget) Write(_ context.Context, p []byte) (n int, err error) { t.payload.Data = append(t.payload.Data, p...) return len(p), nil } -func (t *distributedTarget) Close() (*transformer.AccessIdentifiers, error) { +func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { defer func() { putPayload(t.payload) t.payload = nil @@ -146,10 +147,10 @@ func (t *distributedTarget) Close() (*transformer.AccessIdentifiers, error) { t.traversal.extraBroadcastEnabled = true } - return t.iteratePlacement(t.sendObject) + return t.iteratePlacement(ctx) } -func (t *distributedTarget) sendObject(node nodeDesc) error { +func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error { if !node.local && t.relay != nil { return t.relay(node) } @@ -158,13 +159,13 @@ func (t *distributedTarget) sendObject(node nodeDesc) error { if err := target.WriteObject(t.obj, t.objMeta); err != nil { return fmt.Errorf("could not write header: %w", err) - } else if _, err := target.Close(); err != nil { + } else if _, err := target.Close(ctx); err != nil { return fmt.Errorf("could not close object stream: %w", err) } return nil } -func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transformer.AccessIdentifiers, error) { +func (t *distributedTarget) iteratePlacement(ctx context.Context) (*transformer.AccessIdentifiers, error) { id, _ := t.obj.ID() traverser, err := placement.NewTraverser( @@ -182,7 +183,7 @@ func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transform break } - if t.iterateAddresses(traverser, addrs, f, resErr) { + if t.iterateAddresses(ctx, traverser, addrs, resErr) { break } } @@ -195,7 +196,7 @@ func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transform // perform additional container broadcast if needed if t.traversal.submitPrimaryPlacementFinish() { - _, err = t.iteratePlacement(f) + _, err = t.iteratePlacement(ctx) if err != nil { t.log.Error("additional container broadcast failure", zap.Error(err)) // we don't fail primary operation because of broadcast failure @@ -208,7 +209,7 @@ func (t *distributedTarget) iteratePlacement(f func(nodeDesc) error) (*transform WithSelfID(id), nil } -func (t *distributedTarget) iterateAddresses(traverser *placement.Traverser, addrs []placement.Node, f func(nodeDesc) error, resErr *atomic.Value) bool { +func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *placement.Traverser, addrs []placement.Node, resErr *atomic.Value) bool { wg := &sync.WaitGroup{} for i := range addrs { @@ -230,7 +231,7 @@ func (t *distributedTarget) iterateAddresses(traverser *placement.Traverser, add if err := workerPool.Submit(func() { defer wg.Done() - err := f(nodeDesc{local: isLocal, info: addr}) + err := t.sendObject(ctx, nodeDesc{local: isLocal, info: addr}) // mark the container node as processed in order to exclude it // in subsequent container broadcast. Note that we don't diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index f344f77e9..12e3a2eee 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -1,6 +1,7 @@ package putsvc import ( + "context" "fmt" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -38,7 +39,7 @@ func (t *localTarget) WriteObject(obj *object.Object, meta objectCore.ContentMet return nil } -func (t *localTarget) Close() (*transformer.AccessIdentifiers, error) { +func (t *localTarget) Close(_ context.Context) (*transformer.AccessIdentifiers, error) { switch t.meta.Type() { case object.TypeTombstone: err := t.storage.Delete(objectCore.AddressOf(t.obj), t.meta.Objects()) diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/put/remote.go index 760de7508..6933abca6 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/put/remote.go @@ -15,10 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) -// nolint: containedctx type remoteTarget struct { - ctx context.Context - privateKey *ecdsa.PrivateKey commonPrm *util.CommonPrm @@ -51,7 +48,7 @@ func (t *remoteTarget) WriteObject(obj *object.Object, _ objectcore.ContentMeta) return nil } -func (t *remoteTarget) Close() (*transformer.AccessIdentifiers, error) { +func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { c, err := t.clientConstructor.Get(t.nodeInfo) if err != nil { return nil, fmt.Errorf("(%T) could not create SDK client %s: %w", t, t.nodeInfo, err) @@ -59,7 +56,7 @@ func (t *remoteTarget) Close() (*transformer.AccessIdentifiers, error) { var prm internalclient.PutObjectPrm - prm.SetContext(t.ctx) + prm.SetContext(ctx) prm.SetClient(c) prm.SetPrivateKey(t.privateKey) prm.SetSessionToken(t.commonPrm.SessionToken()) @@ -110,7 +107,6 @@ func (s *RemoteSender) PutObject(ctx context.Context, p *RemotePutPrm) error { } t := &remoteTarget{ - ctx: ctx, privateKey: key, clientConstructor: s.clientConstructor, } @@ -122,7 +118,7 @@ func (s *RemoteSender) PutObject(ctx context.Context, p *RemotePutPrm) error { if err := t.WriteObject(p.obj, objectcore.ContentMeta{}); err != nil { return fmt.Errorf("(%T) could not send object header: %w", s, err) - } else if _, err := t.Close(); err != nil { + } else if _, err := t.Close(ctx); err != nil { return fmt.Errorf("(%T) could not send object: %w", s, err) } diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index b74c97d49..567a3fea1 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -1,8 +1,6 @@ package putsvc import ( - "context" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -79,10 +77,9 @@ func NewService(opts ...Option) *Service { } } -func (p *Service) Put(ctx context.Context) (*Streamer, error) { +func (p *Service) Put() (*Streamer, error) { return &Streamer{ cfg: p.cfg, - ctx: ctx, }, nil } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 915b718a3..678cff572 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -16,12 +16,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) -// nolint: containedctx type Streamer struct { *cfg - ctx context.Context - sessionKey *ecdsa.PrivateKey target transformer.ObjectTarget @@ -232,7 +229,6 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { } rt := &remoteTarget{ - ctx: p.ctx, privateKey: p.sessionKey, commonPrm: prm.common, clientConstructor: p.clientConstructor, @@ -250,24 +246,24 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { } } -func (p *Streamer) SendChunk(prm *PutChunkPrm) error { +func (p *Streamer) SendChunk(ctx context.Context, prm *PutChunkPrm) error { if p.target == nil { return errNotInit } - if _, err := p.target.Write(prm.chunk); err != nil { + if _, err := p.target.Write(ctx, prm.chunk); err != nil { return fmt.Errorf("(%T) could not write payload chunk to target: %w", p, err) } return nil } -func (p *Streamer) Close() (*PutResponse, error) { +func (p *Streamer) Close(ctx context.Context) (*PutResponse, error) { if p.target == nil { return nil, errNotInit } - ids, err := p.target.Close() + ids, err := p.target.Close(ctx) if err != nil { return nil, fmt.Errorf("(%T) could not close object target: %w", p, err) } diff --git a/pkg/services/object/put/v2/service.go b/pkg/services/object/put/v2/service.go index 7d0dfc613..656f8df9c 100644 --- a/pkg/services/object/put/v2/service.go +++ b/pkg/services/object/put/v2/service.go @@ -1,7 +1,6 @@ package putsvc import ( - "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" @@ -36,8 +35,8 @@ func NewService(opts ...Option) *Service { } // Put calls internal service and returns v2 object streamer. -func (s *Service) Put(ctx context.Context) (object.PutObjectStream, error) { - stream, err := s.svc.Put(ctx) +func (s *Service) Put() (object.PutObjectStream, error) { + stream, err := s.svc.Put() if err != nil { return nil, fmt.Errorf("(%T) could not open object put stream: %w", s, err) } diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 85827cd4c..65846ea9f 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -1,6 +1,7 @@ package putsvc import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" @@ -32,7 +33,7 @@ type sizes struct { writtenPayload uint64 // sum size of already cached chunks } -func (s *streamer) Send(req *object.PutRequest) (err error) { +func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) { switch v := req.GetBody().GetObjectPart().(type) { case *object.PutObjectPartInit: var initPrm *putsvc.PutInitPrm @@ -71,7 +72,7 @@ func (s *streamer) Send(req *object.PutRequest) (err error) { } } - if err = s.stream.SendChunk(toChunkPrm(v)); err != nil { + if err = s.stream.SendChunk(ctx, toChunkPrm(v)); err != nil { err = fmt.Errorf("(%T) could not send payload chunk: %w", s, err) } @@ -103,7 +104,7 @@ func (s *streamer) Send(req *object.PutRequest) (err error) { return signature.SignServiceMessage(key, req) } -func (s *streamer) CloseAndRecv() (*object.PutResponse, error) { +func (s *streamer) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { if s.saveChunks { // check payload size correctness if s.writtenPayload != s.payloadSz { @@ -111,7 +112,7 @@ func (s *streamer) CloseAndRecv() (*object.PutResponse, error) { } } - resp, err := s.stream.Close() + resp, err := s.stream.Close(ctx) if err != nil { return nil, fmt.Errorf("(%T) could not object put stream: %w", s, err) } diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/put/validation.go index 2d6ada5a1..70c6974d3 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/put/validation.go @@ -2,6 +2,7 @@ package putsvc import ( "bytes" + "context" "crypto/sha256" "errors" "fmt" @@ -92,7 +93,7 @@ func (t *validatingTarget) WriteHeader(obj *objectSDK.Object) error { return nil } -func (t *validatingTarget) Write(p []byte) (n int, err error) { +func (t *validatingTarget) Write(ctx context.Context, p []byte) (n int, err error) { chunkLn := uint64(len(p)) if !t.unpreparedObject { @@ -107,7 +108,7 @@ func (t *validatingTarget) Write(p []byte) (n int, err error) { } } - n, err = t.nextTarget.Write(p) + n, err = t.nextTarget.Write(ctx, p) if err == nil { t.writtenPayload += uint64(n) } @@ -115,7 +116,7 @@ func (t *validatingTarget) Write(p []byte) (n int, err error) { return } -func (t *validatingTarget) Close() (*transformer.AccessIdentifiers, error) { +func (t *validatingTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { if !t.unpreparedObject { // check payload size correctness if t.payloadSz != t.writtenPayload { @@ -127,5 +128,5 @@ func (t *validatingTarget) Close() (*transformer.AccessIdentifiers, error) { } } - return t.nextTarget.Close() + return t.nextTarget.Close(ctx) } diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index 4da2b23a7..def934ea6 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -59,12 +59,12 @@ func (s *ResponseService) Get(req *object.GetRequest, stream GetObjectStream) er }) } -func (s *putStreamResponser) Send(req *object.PutRequest) error { - return s.stream.Send(req) +func (s *putStreamResponser) Send(ctx context.Context, req *object.PutRequest) error { + return s.stream.Send(ctx, req) } -func (s *putStreamResponser) CloseAndRecv() (*object.PutResponse, error) { - r, err := s.stream.CloseAndRecv() +func (s *putStreamResponser) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { + r, err := s.stream.CloseAndRecv(ctx) if err != nil { return nil, fmt.Errorf("(%T) could not receive response: %w", s, err) } @@ -72,19 +72,19 @@ func (s *putStreamResponser) CloseAndRecv() (*object.PutResponse, error) { return r.(*object.PutResponse), nil } -func (s *ResponseService) Put(ctx context.Context) (PutObjectStream, error) { - stream, err := s.svc.Put(ctx) +func (s *ResponseService) Put() (PutObjectStream, error) { + stream, err := s.svc.Put() if err != nil { return nil, fmt.Errorf("could not create Put object streamer: %w", err) } return &putStreamResponser{ stream: s.respSvc.CreateRequestStreamer( - func(req any) error { - return stream.Send(req.(*object.PutRequest)) + func(ctx context.Context, req any) error { + return stream.Send(ctx, req.(*object.PutRequest)) }, - func() (util.ResponseMessage, error) { - return stream.CloseAndRecv() + func(ctx context.Context) (util.ResponseMessage, error) { + return stream.CloseAndRecv(ctx) }, ), }, nil diff --git a/pkg/services/object/server.go b/pkg/services/object/server.go index d95c6c906..ccce9c4f4 100644 --- a/pkg/services/object/server.go +++ b/pkg/services/object/server.go @@ -27,15 +27,15 @@ type SearchStream interface { // PutObjectStream is an interface of FrostFS API v2 compatible client's object streamer. type PutObjectStream interface { - Send(*object.PutRequest) error - CloseAndRecv() (*object.PutResponse, error) + Send(context.Context, *object.PutRequest) error + CloseAndRecv(context.Context) (*object.PutResponse, error) } // ServiceServer is an interface of utility // serving v2 Object service. type ServiceServer interface { Get(*object.GetRequest, GetObjectStream) error - Put(context.Context) (PutObjectStream, error) + Put() (PutObjectStream, error) Head(context.Context, *object.HeadRequest) (*object.HeadResponse, error) Search(*object.SearchRequest, SearchStream) error Delete(context.Context, *object.DeleteRequest) (*object.DeleteResponse, error) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 585fc659a..9d66c76ba 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -70,12 +70,12 @@ func (s *SignService) Get(req *object.GetRequest, stream GetObjectStream) error ) } -func (s *putStreamSigner) Send(req *object.PutRequest) error { - return s.stream.Send(req) +func (s *putStreamSigner) Send(ctx context.Context, req *object.PutRequest) error { + return s.stream.Send(ctx, req) } -func (s *putStreamSigner) CloseAndRecv() (*object.PutResponse, error) { - r, err := s.stream.CloseAndRecv() +func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { + r, err := s.stream.CloseAndRecv(ctx) if err != nil { return nil, fmt.Errorf("could not receive response: %w", err) } @@ -83,19 +83,19 @@ func (s *putStreamSigner) CloseAndRecv() (*object.PutResponse, error) { return r.(*object.PutResponse), nil } -func (s *SignService) Put(ctx context.Context) (PutObjectStream, error) { - stream, err := s.svc.Put(ctx) +func (s *SignService) Put() (PutObjectStream, error) { + stream, err := s.svc.Put() if err != nil { return nil, fmt.Errorf("could not create Put object streamer: %w", err) } return &putStreamSigner{ stream: s.sigSvc.CreateRequestStreamer( - func(req any) error { - return stream.Send(req.(*object.PutRequest)) + func(ctx context.Context, req any) error { + return stream.Send(ctx, req.(*object.PutRequest)) }, - func() (util.ResponseMessage, error) { - return stream.CloseAndRecv() + func(ctx context.Context) (util.ResponseMessage, error) { + return stream.CloseAndRecv(ctx) }, func() util.ResponseMessage { return new(object.PutResponse) diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index 3836103de..a7d1c486a 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -87,8 +87,8 @@ func (c *TransportSplitter) Get(req *object.GetRequest, stream GetObjectStream) }) } -func (c TransportSplitter) Put(ctx context.Context) (PutObjectStream, error) { - return c.next.Put(ctx) +func (c TransportSplitter) Put() (PutObjectStream, error) { + return c.next.Put() } func (c TransportSplitter) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) { diff --git a/pkg/services/object_manager/transformer/fmt.go b/pkg/services/object_manager/transformer/fmt.go index c9b5dc967..462cc7474 100644 --- a/pkg/services/object_manager/transformer/fmt.go +++ b/pkg/services/object_manager/transformer/fmt.go @@ -1,6 +1,7 @@ package transformer import ( + "context" "crypto/ecdsa" "fmt" @@ -53,15 +54,15 @@ func (f *formatter) WriteHeader(obj *object.Object) error { return nil } -func (f *formatter) Write(p []byte) (n int, err error) { - n, err = f.prm.NextTarget.Write(p) +func (f *formatter) Write(ctx context.Context, p []byte) (n int, err error) { + n, err = f.prm.NextTarget.Write(ctx, p) f.sz += uint64(n) return } -func (f *formatter) Close() (*AccessIdentifiers, error) { +func (f *formatter) Close(ctx context.Context) (*AccessIdentifiers, error) { curEpoch := f.prm.NetworkState.CurrentEpoch() ver := version.Current() @@ -100,7 +101,7 @@ func (f *formatter) Close() (*AccessIdentifiers, error) { return nil, fmt.Errorf("could not write header to next target: %w", err) } - if _, err := f.prm.NextTarget.Close(); err != nil { + if _, err := f.prm.NextTarget.Close(ctx); err != nil { return nil, fmt.Errorf("could not close next target: %w", err) } diff --git a/pkg/services/object_manager/transformer/transformer.go b/pkg/services/object_manager/transformer/transformer.go index 7b717d3df..199f5d0c1 100644 --- a/pkg/services/object_manager/transformer/transformer.go +++ b/pkg/services/object_manager/transformer/transformer.go @@ -1,10 +1,10 @@ package transformer import ( + "context" "crypto/sha256" "fmt" "hash" - "io" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -27,7 +27,7 @@ type payloadSizeLimiter struct { previous []oid.ID - chunkWriter io.Writer + chunkWriter writer splitID *object.SplitID @@ -64,16 +64,16 @@ func (s *payloadSizeLimiter) WriteHeader(hdr *object.Object) error { return nil } -func (s *payloadSizeLimiter) Write(p []byte) (int, error) { - if err := s.writeChunk(p); err != nil { +func (s *payloadSizeLimiter) Write(ctx context.Context, p []byte) (int, error) { + if err := s.writeChunk(ctx, p); err != nil { return 0, err } return len(p), nil } -func (s *payloadSizeLimiter) Close() (*AccessIdentifiers, error) { - return s.release(true) +func (s *payloadSizeLimiter) Close(ctx context.Context) (*AccessIdentifiers, error) { + return s.release(ctx, true) } func (s *payloadSizeLimiter) initialize() { @@ -117,19 +117,19 @@ func (s *payloadSizeLimiter) initializeCurrent() { s.currentHashers = payloadHashersForObject(s.current, s.withoutHomomorphicHash) // compose multi-writer from target and all payload hashers - ws := make([]io.Writer, 0, 1+len(s.currentHashers)+len(s.parentHashers)) + ws := make([]writer, 0, 1+len(s.currentHashers)+len(s.parentHashers)) ws = append(ws, s.target) for i := range s.currentHashers { - ws = append(ws, s.currentHashers[i].hasher) + ws = append(ws, newWriter(s.currentHashers[i].hasher)) } for i := range s.parentHashers { - ws = append(ws, s.parentHashers[i].hasher) + ws = append(ws, newWriter(s.parentHashers[i].hasher)) } - s.chunkWriter = io.MultiWriter(ws...) + s.chunkWriter = newMultiWriter(ws...) } func payloadHashersForObject(obj *object.Object, withoutHomomorphicHash bool) []*payloadChecksumHasher { @@ -174,7 +174,7 @@ func payloadHashersForObject(obj *object.Object, withoutHomomorphicHash bool) [] return hashers } -func (s *payloadSizeLimiter) release(finalize bool) (*AccessIdentifiers, error) { +func (s *payloadSizeLimiter) release(ctx context.Context, finalize bool) (*AccessIdentifiers, error) { // Arg finalize is true only when called from Close method. // We finalize parent and generate linking objects only if it is more // than 1 object in split-chain. @@ -194,7 +194,7 @@ func (s *payloadSizeLimiter) release(finalize bool) (*AccessIdentifiers, error) return nil, fmt.Errorf("could not write header: %w", err) } - ids, err := s.target.Close() + ids, err := s.target.Close(ctx) if err != nil { return nil, fmt.Errorf("could not close target: %w", err) } @@ -207,7 +207,7 @@ func (s *payloadSizeLimiter) release(finalize bool) (*AccessIdentifiers, error) s.initializeLinking(ids.Parent()) s.initializeCurrent() - if _, err := s.release(false); err != nil { + if _, err := s.release(ctx, false); err != nil { return nil, fmt.Errorf("could not release linking object: %w", err) } } @@ -228,7 +228,7 @@ func (s *payloadSizeLimiter) initializeLinking(parHdr *object.Object) { s.current.SetSplitID(s.splitID) } -func (s *payloadSizeLimiter) writeChunk(chunk []byte) error { +func (s *payloadSizeLimiter) writeChunk(ctx context.Context, chunk []byte) error { // statement is true if the previous write of bytes reached exactly the boundary. if s.written > 0 && s.written%s.maxSize == 0 { if s.written == s.maxSize { @@ -236,7 +236,7 @@ func (s *payloadSizeLimiter) writeChunk(chunk []byte) error { } // we need to release current object - if _, err := s.release(false); err != nil { + if _, err := s.release(ctx, false); err != nil { return fmt.Errorf("could not release object: %w", err) } @@ -255,7 +255,7 @@ func (s *payloadSizeLimiter) writeChunk(chunk []byte) error { cut = leftToEdge } - if _, err := s.chunkWriter.Write(chunk[:cut]); err != nil { + if _, err := s.chunkWriter.Write(ctx, chunk[:cut]); err != nil { return fmt.Errorf("could not write chunk to target: %w", err) } @@ -264,7 +264,7 @@ func (s *payloadSizeLimiter) writeChunk(chunk []byte) error { // if there are more bytes in buffer we call method again to start filling another object if ln > leftToEdge { - return s.writeChunk(chunk[cut:]) + return s.writeChunk(ctx, chunk[cut:]) } return nil diff --git a/pkg/services/object_manager/transformer/types.go b/pkg/services/object_manager/transformer/types.go index 0fa3b6436..3e6e2feff 100644 --- a/pkg/services/object_manager/transformer/types.go +++ b/pkg/services/object_manager/transformer/types.go @@ -1,7 +1,7 @@ package transformer import ( - "io" + "context" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -35,7 +35,7 @@ type ObjectTarget interface { // Can be called multiple times. // // Must not be called after Close call. - io.Writer + Write(ctx context.Context, p []byte) (n int, err error) // Close is used to finish object writing. // @@ -45,7 +45,7 @@ type ObjectTarget interface { // Must be called no more than once. Control remains with the caller. // Re-calling can lead to undefined behavior // that depends on the implementation. - Close() (*AccessIdentifiers, error) + Close(ctx context.Context) (*AccessIdentifiers, error) } // TargetInitializer represents ObjectTarget constructor. diff --git a/pkg/services/object_manager/transformer/writer.go b/pkg/services/object_manager/transformer/writer.go new file mode 100644 index 000000000..27aed16ff --- /dev/null +++ b/pkg/services/object_manager/transformer/writer.go @@ -0,0 +1,52 @@ +package transformer + +import ( + "context" + "io" +) + +type writer interface { + Write(ctx context.Context, p []byte) (n int, err error) +} + +type multiWriter struct { + writers []writer +} + +func (t *multiWriter) Write(ctx context.Context, p []byte) (n int, err error) { + for _, w := range t.writers { + n, err = w.Write(ctx, p) + if err != nil { + return + } + if n != len(p) { + err = io.ErrShortWrite + return + } + } + return len(p), nil +} + +func newMultiWriter(writers ...writer) writer { + allWriters := make([]writer, 0, len(writers)) + for _, w := range writers { + if mw, ok := w.(*multiWriter); ok { + allWriters = append(allWriters, mw.writers...) + } else { + allWriters = append(allWriters, w) + } + } + return &multiWriter{allWriters} +} + +type writerWrapper struct { + Writer io.Writer +} + +func (w *writerWrapper) Write(_ context.Context, p []byte) (n int, err error) { + return w.Writer.Write(p) +} + +func newWriter(w io.Writer) writer { + return &writerWrapper{Writer: w} +} diff --git a/pkg/services/util/response/client_stream.go b/pkg/services/util/response/client_stream.go index f167f005a..b541c73db 100644 --- a/pkg/services/util/response/client_stream.go +++ b/pkg/services/util/response/client_stream.go @@ -1,6 +1,7 @@ package response import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" @@ -17,8 +18,8 @@ type ClientMessageStreamer struct { } // Send calls send method of internal streamer. -func (s *ClientMessageStreamer) Send(req any) error { - if err := s.send(req); err != nil { +func (s *ClientMessageStreamer) Send(ctx context.Context, req any) error { + if err := s.send(ctx, req); err != nil { return fmt.Errorf("(%T) could not send the request: %w", s, err) } return nil @@ -26,8 +27,8 @@ func (s *ClientMessageStreamer) Send(req any) error { // CloseAndRecv closes internal stream, receivers the response, // sets meta values and returns the result. -func (s *ClientMessageStreamer) CloseAndRecv() (util.ResponseMessage, error) { - resp, err := s.close() +func (s *ClientMessageStreamer) CloseAndRecv(ctx context.Context) (util.ResponseMessage, error) { + resp, err := s.close(ctx) if err != nil { return nil, fmt.Errorf("(%T) could not close stream and receive response: %w", s, err) } diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index 2478e6256..cb4be3084 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -37,9 +37,9 @@ var ErrAbortStream = errors.New("abort message stream") type ResponseConstructor func() ResponseMessage -type RequestMessageWriter func(any) error +type RequestMessageWriter func(context.Context, any) error -type ClientStreamCloser func() (ResponseMessage, error) +type ClientStreamCloser func(context.Context) (ResponseMessage, error) type RequestMessageStreamer struct { key *ecdsa.PrivateKey @@ -61,7 +61,7 @@ func NewUnarySignService(key *ecdsa.PrivateKey) *SignService { } } -func (s *RequestMessageStreamer) Send(req any) error { +func (s *RequestMessageStreamer) Send(ctx context.Context, req any) error { // req argument should be strengthen with type RequestMessage s.statusSupported = isStatusSupported(req.(RequestMessage)) // panic is OK here for now @@ -71,7 +71,7 @@ func (s *RequestMessageStreamer) Send(req any) error { if err = signature.VerifyServiceMessage(req); err != nil { err = fmt.Errorf("could not verify request: %w", err) } else { - err = s.send(req) + err = s.send(ctx, req) } if err != nil { @@ -87,7 +87,7 @@ func (s *RequestMessageStreamer) Send(req any) error { return nil } -func (s *RequestMessageStreamer) CloseAndRecv() (ResponseMessage, error) { +func (s *RequestMessageStreamer) CloseAndRecv(ctx context.Context) (ResponseMessage, error) { var ( resp ResponseMessage err error @@ -96,7 +96,7 @@ func (s *RequestMessageStreamer) CloseAndRecv() (ResponseMessage, error) { if s.sendErr != nil { err = s.sendErr } else { - resp, err = s.close() + resp, err = s.close(ctx) if err != nil { err = fmt.Errorf("could not close stream and receive response: %w", err) } From ece6c820e70d8a58ab0a20f72c45708c8d411958 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 15:40:52 +0300 Subject: [PATCH 0014/1943] [#199] putsvc: Refactor streamer initialization Resolve funlen linter for initTarget method Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/streamer.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 678cff572..fed161e03 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -51,7 +51,6 @@ func (p *Streamer) MaxObjectSize() uint64 { return p.maxPayloadSz } -// nolint: funlen func (p *Streamer) initTarget(prm *PutInitPrm) error { // prevent re-calling if p.target != nil { @@ -69,19 +68,26 @@ func (p *Streamer) initTarget(prm *PutInitPrm) error { } if prm.hdr.Signature() != nil { - p.relay = prm.relay + return p.initUntrustedTarget(prm) + } + return p.initTrustedTarget(prm) +} - // prepare untrusted-Put object target - p.target = &validatingTarget{ - nextTarget: p.newCommonTarget(prm), - fmt: p.fmtValidator, +func (p *Streamer) initUntrustedTarget(prm *PutInitPrm) error { + p.relay = prm.relay - maxPayloadSz: p.maxPayloadSz, - } + // prepare untrusted-Put object target + p.target = &validatingTarget{ + nextTarget: p.newCommonTarget(prm), + fmt: p.fmtValidator, - return nil + maxPayloadSz: p.maxPayloadSz, } + return nil +} + +func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { sToken := prm.common.SessionToken() // prepare trusted-Put object target From 0948a280fa0dd5341c25885e8790e6af16c40f88 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 3 Apr 2023 10:44:05 +0300 Subject: [PATCH 0015/1943] [#195] morph: use blocking unlimited pool for notifications With non-blocking pool restricted by 10 in capacity, the probability of dropping events is unexpectedly big. Notifications are an essential part of the FrostFS, we should not drop anything, especially new epochs. ``` Mar 31 07:07:03 vedi neofs-ir[19164]: 2023-03-31T07:07:03.901Z debug subscriber/subscriber.go:154 new notification event from sidechain {"name": "NewEpoch"} Mar 31 07:07:03 vedi neofs-ir[19164]: 2023-03-31T07:07:03.901Z warn event/listener.go:248 listener worker pool drained {"chain": "morph", "capacity": 10} ``` Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/morph.go | 13 ++----------- pkg/innerring/innerring.go | 13 ++----------- pkg/morph/event/listener.go | 14 +++----------- 3 files changed, 7 insertions(+), 33 deletions(-) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 439de3a9e..2dfbe5c18 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -181,14 +181,6 @@ func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error { } func listenMorphNotifications(ctx context.Context, c *cfg) { - // listenerPoolCap is a capacity of a - // worker pool inside the listener. It - // is used to prevent blocking in neo-go: - // the client cannot make RPC requests if - // the notification channel is not being - // read by another goroutine. - const listenerPoolCap = 10 - var ( err error subs subscriber.Subscriber @@ -208,9 +200,8 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { fatalOnErr(err) lis, err := event.NewListener(event.ListenerParams{ - Logger: c.log, - Subscriber: subs, - WorkerPoolCapacity: listenerPoolCap, + Logger: c.log, + Subscriber: subs, }) fatalOnErr(err) diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 38023932f..3b42a5853 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -409,14 +409,6 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan } func createListener(ctx context.Context, cli *client.Client, p *chainParams) (event.Listener, error) { - // listenerPoolCap is a capacity of a - // worker pool inside the listener. It - // is used to prevent blocking in neo-go: - // the client cannot make RPC requests if - // the notification channel is not being - // read by another goroutine. - const listenerPoolCap = 10 - var ( sub subscriber.Subscriber err error @@ -432,9 +424,8 @@ func createListener(ctx context.Context, cli *client.Client, p *chainParams) (ev } listener, err := event.NewListener(event.ListenerParams{ - Logger: &logger.Logger{Logger: p.log.With(zap.String("chain", p.name))}, - Subscriber: sub, - WorkerPoolCapacity: listenerPoolCap, + Logger: &logger.Logger{Logger: p.log.With(zap.String("chain", p.name))}, + Subscriber: sub, }) if err != nil { return nil, err diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 64fdc3df3..0bc7e89f8 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -600,11 +600,6 @@ func (l *listener) RegisterBlockHandler(handler BlockHandler) { // NewListener create the notification event listener instance and returns Listener interface. func NewListener(p ListenerParams) (Listener, error) { - // defaultPoolCap is a default worker - // pool capacity if it was not specified - // via params - const defaultPoolCap = 10 - switch { case p.Logger == nil: return nil, fmt.Errorf("%s: %w", newListenerFailMsg, errNilLogger) @@ -612,12 +607,9 @@ func NewListener(p ListenerParams) (Listener, error) { return nil, fmt.Errorf("%s: %w", newListenerFailMsg, errNilSubscriber) } - poolCap := p.WorkerPoolCapacity - if poolCap == 0 { - poolCap = defaultPoolCap - } - - pool, err := ants.NewPool(poolCap, ants.WithNonblocking(true)) + // The pool here must be blocking, otherwise notifications could be dropped. + // The default capacity is 0, which means "infinite". + pool, err := ants.NewPool(p.WorkerPoolCapacity) if err != nil { return nil, fmt.Errorf("could not init worker pool: %w", err) } From 8e5a0dcf272dda711449a31746f602d5a72c81f1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 19:15:13 +0300 Subject: [PATCH 0016/1943] [#204] gc: Fix GC handlers start Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index c3bb841d2..6f18e6c3a 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -136,7 +136,8 @@ func (gc *gc) listenEvents(ctx context.Context) { v.cancelFunc() v.prevGroup.Wait() - ctx, v.cancelFunc = context.WithCancel(ctx) + var runCtx context.Context + runCtx, v.cancelFunc = context.WithCancel(ctx) v.prevGroup.Add(len(v.handlers)) @@ -144,7 +145,7 @@ func (gc *gc) listenEvents(ctx context.Context) { h := v.handlers[i] err := gc.workerPool.Submit(func() { - h(ctx, event) + h(runCtx, event) v.prevGroup.Done() }) if err != nil { From e85e5382e4bebc574aa7569635137be1b96f6ed2 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 3 Apr 2023 14:06:57 +0300 Subject: [PATCH 0017/1943] [#175] adm: list containers using `containersOf` Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/container.go | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/container.go b/cmd/frostfs-adm/internal/modules/morph/container.go index 9bc6cae41..b5447fcfe 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container.go @@ -42,15 +42,27 @@ func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker, c Client } func getContainersList(inv *invoker.Invoker, ch util.Uint160) ([][]byte, error) { - res, err := inv.Call(ch, "list", "") + sid, r, err := unwrap.SessionIterator(inv.Call(ch, "containersOf", "")) if err != nil { return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err) } - itm, err := unwrap.Item(res, err) - if _, ok := itm.(stackitem.Null); !ok { - return unwrap.ArrayOfBytes(res, err) + // Nothing bad, except live session on the server, do not report to the user. + defer func() { _ = inv.TerminateSession(sid) }() + + var lst [][]byte + + items, err := inv.TraverseIterator(sid, &r, 0) + for err == nil && len(items) != 0 { + for j := range items { + b, err := items[j].TryBytes() + if err != nil { + return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err) + } + lst = append(lst, b) + } + items, err = inv.TraverseIterator(sid, &r, 0) } - return nil, nil + return lst, err } func dumpContainers(cmd *cobra.Command, _ []string) error { From 49cc23e03c8da451fe67f67509183441d3ee740d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 3 Apr 2023 14:37:05 +0300 Subject: [PATCH 0018/1943] [#175] adm: pipeline container iteration Do not accumulate everything in memory. Also, CLI should be responsive. Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/container.go | 134 ++++++++++-------- 1 file changed, 77 insertions(+), 57 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/container.go b/cmd/frostfs-adm/internal/modules/morph/container.go index b5447fcfe..687d7e84e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container.go @@ -41,28 +41,28 @@ func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker, c Client return ch, nil } -func getContainersList(inv *invoker.Invoker, ch util.Uint160) ([][]byte, error) { +func iterateContainerList(inv *invoker.Invoker, ch util.Uint160, f func([]byte) error) error { sid, r, err := unwrap.SessionIterator(inv.Call(ch, "containersOf", "")) if err != nil { - return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err) + return fmt.Errorf("%w: %v", errInvalidContainerResponse, err) } // Nothing bad, except live session on the server, do not report to the user. defer func() { _ = inv.TerminateSession(sid) }() - var lst [][]byte - items, err := inv.TraverseIterator(sid, &r, 0) for err == nil && len(items) != 0 { for j := range items { b, err := items[j].TryBytes() if err != nil { - return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err) + return fmt.Errorf("%w: %v", errInvalidContainerResponse, err) + } + if err := f(b); err != nil { + return err } - lst = append(lst, b) } items, err = inv.TraverseIterator(sid, &r, 0) } - return lst, err + return err } func dumpContainers(cmd *cobra.Command, _ []string) error { @@ -83,56 +83,81 @@ func dumpContainers(cmd *cobra.Command, _ []string) error { return fmt.Errorf("unable to get contaract hash: %w", err) } - cids, err := getContainersList(inv, ch) - if err != nil { - return fmt.Errorf("%w: %v", errInvalidContainerResponse, err) - } - isOK, err := getCIDFilterFunc(cmd) if err != nil { return err } - var containers []*Container - bw := io.NewBufBinWriter() - for _, id := range cids { - if !isOK(id) { - continue - } - bw.Reset() - emit.AppCall(bw.BinWriter, ch, "get", callflag.All, id) - emit.AppCall(bw.BinWriter, ch, "eACL", callflag.All, id) - res, err := inv.Run(bw.Bytes()) - if err != nil { - return fmt.Errorf("can't get container info: %w", err) - } - if len(res.Stack) != 2 { - return fmt.Errorf("%w: expected 2 items on stack", errInvalidContainerResponse) - } - - cnt := new(Container) - err = cnt.FromStackItem(res.Stack[0]) - if err != nil { - return fmt.Errorf("%w: %v", errInvalidContainerResponse, err) - } - - ea := new(EACL) - err = ea.FromStackItem(res.Stack[1]) - if err != nil { - return fmt.Errorf("%w: %v", errInvalidContainerResponse, err) - } - if len(ea.Value) != 0 { - cnt.EACL = ea - } - - containers = append(containers, cnt) - } - - out, err := json.Marshal(containers) + f, err := os.OpenFile(filename, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0o660) if err != nil { return err } - return os.WriteFile(filename, out, 0o660) + defer f.Close() + + _, err = f.Write([]byte{'['}) + if err != nil { + return err + } + + written := 0 + enc := json.NewEncoder(f) + bw := io.NewBufBinWriter() + iterErr := iterateContainerList(inv, ch, func(id []byte) error { + if !isOK(id) { + return nil + } + + cnt, err := dumpSingleContainer(bw, ch, inv, id) + if err != nil { + return err + } + + // Writing directly to the file is ok, because json.Encoder does no internal buffering. + if written != 0 { + _, err = f.Write([]byte{','}) + if err != nil { + return err + } + } + + written++ + return enc.Encode(cnt) + }) + if iterErr != nil { + return iterErr + } + + _, err = f.Write([]byte{']'}) + return err +} + +func dumpSingleContainer(bw *io.BufBinWriter, ch util.Uint160, inv *invoker.Invoker, id []byte) (*Container, error) { + bw.Reset() + emit.AppCall(bw.BinWriter, ch, "get", callflag.All, id) + emit.AppCall(bw.BinWriter, ch, "eACL", callflag.All, id) + res, err := inv.Run(bw.Bytes()) + if err != nil { + return nil, fmt.Errorf("can't get container info: %w", err) + } + if len(res.Stack) != 2 { + return nil, fmt.Errorf("%w: expected 2 items on stack", errInvalidContainerResponse) + } + + cnt := new(Container) + err = cnt.FromStackItem(res.Stack[0]) + if err != nil { + return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err) + } + + ea := new(EACL) + err = ea.FromStackItem(res.Stack[1]) + if err != nil { + return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err) + } + if len(ea.Value) != 0 { + cnt.EACL = ea + } + return cnt, nil } func listContainers(cmd *cobra.Command, _ []string) error { @@ -148,20 +173,15 @@ func listContainers(cmd *cobra.Command, _ []string) error { return fmt.Errorf("unable to get contaract hash: %w", err) } - cids, err := getContainersList(inv, ch) - if err != nil { - return fmt.Errorf("%w: %v", errInvalidContainerResponse, err) - } - - for _, id := range cids { + return iterateContainerList(inv, ch, func(id []byte) error { var idCnr cid.ID err = idCnr.Decode(id) if err != nil { return fmt.Errorf("unable to decode container id: %w", err) } cmd.Println(idCnr) - } - return nil + return nil + }) } func restoreContainers(cmd *cobra.Command, _ []string) error { From ee7468daa73fe6230a5aa165506710ba2dca1f96 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 4 Apr 2023 09:36:22 +0300 Subject: [PATCH 0019/1943] [#205] innerring: Provide `alphabetState` param to `epochTimer` Fix NPE, introduced in f09ee27a. Signed-off-by: Evgenii Stratonikov --- pkg/innerring/initialization.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 5a8fdcc3d..30b8d43a0 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -323,6 +323,7 @@ func (s *Server) createIRFetcher() irFetcher { func (s *Server) initTimers(cfg *viper.Viper, processors *serverProcessors, morphClients *serverMorphClients) { s.epochTimer = newEpochTimer(&epochTimerArgs{ l: s.log, + alphabetState: s, newEpochHandlers: s.newEpochTickHandlers(), cnrWrapper: morphClients.CnrClient, epoch: s, From 594b5821edff66368876895fe86c6b8c07144eee Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 30 Mar 2023 12:31:29 +0300 Subject: [PATCH 0020/1943] [#188] blobstor: Refactor put data to blobovniczas Resolve funlen linter for Blobovniczas.Put method Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/put.go | 140 +++++++++--------- 1 file changed, 73 insertions(+), 67 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 438c2e233..db7ca1082 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -13,8 +13,6 @@ import ( // Put saves object in the maximum weight blobobnicza. // // returns error if could not save object in any blobovnicza. -// -// nolint: funlen func (b *Blobovniczas) Put(prm common.PutPrm) (common.PutRes, error) { if b.readOnly { return common.PutRes{}, common.ErrReadOnly @@ -28,77 +26,85 @@ func (b *Blobovniczas) Put(prm common.PutPrm) (common.PutRes, error) { putPrm.SetAddress(prm.Address) putPrm.SetMarshaledObject(prm.RawData) - var ( - fn func(string) (bool, error) - id *blobovnicza.ID - allFull = true - ) - - fn = func(p string) (bool, error) { - active, err := b.getActivated(p) - if err != nil { - if !isLogical(err) { - b.reportError("could not get active blobovnicza", err) - } else { - b.log.Debug("could not get active blobovnicza", - zap.String("error", err.Error())) - } - - return false, nil - } - - if _, err := active.blz.Put(putPrm); err != nil { - // Check if blobovnicza is full. We could either receive `blobovnicza.ErrFull` error - // or update active blobovnicza in other thread. In the latter case the database will be closed - // and `updateActive` takes care of not updating the active blobovnicza twice. - if isFull := errors.Is(err, blobovnicza.ErrFull); isFull || errors.Is(err, bbolt.ErrDatabaseNotOpen) { - if isFull { - b.log.Debug("blobovnicza overflowed", - zap.String("path", filepath.Join(p, u64ToHexString(active.ind)))) - } - - if err := b.updateActive(p, &active.ind); err != nil { - if !isLogical(err) { - b.reportError("could not update active blobovnicza", err) - } else { - b.log.Debug("could not update active blobovnicza", - zap.String("level", p), - zap.String("error", err.Error())) - } - - return false, nil - } - - return fn(p) - } - - allFull = false - if !isLogical(err) { - b.reportError("could not put object to active blobovnicza", err) - } else { - b.log.Debug("could not put object to active blobovnicza", - zap.String("path", filepath.Join(p, u64ToHexString(active.ind))), - zap.String("error", err.Error())) - } - - return false, nil - } - - p = filepath.Join(p, u64ToHexString(active.ind)) - - id = blobovnicza.NewIDFromBytes([]byte(p)) - - return true, nil + it := &putIterator{ + B: b, + ID: nil, + AllFull: true, + PutPrm: putPrm, } - if err := b.iterateDeepest(prm.Address, fn); err != nil { + if err := b.iterateDeepest(prm.Address, it.iterate); err != nil { return common.PutRes{}, err - } else if id == nil { - if allFull { + } else if it.ID == nil { + if it.AllFull { return common.PutRes{}, common.ErrNoSpace } return common.PutRes{}, errPutFailed } - return common.PutRes{StorageID: id.Bytes()}, nil + return common.PutRes{StorageID: it.ID.Bytes()}, nil +} + +type putIterator struct { + B *Blobovniczas + ID *blobovnicza.ID + AllFull bool + PutPrm blobovnicza.PutPrm +} + +func (i *putIterator) iterate(path string) (bool, error) { + active, err := i.B.getActivated(path) + if err != nil { + if !isLogical(err) { + i.B.reportError("could not get active blobovnicza", err) + } else { + i.B.log.Debug("could not get active blobovnicza", + zap.String("error", err.Error())) + } + + return false, nil + } + + if _, err := active.blz.Put(i.PutPrm); err != nil { + // Check if blobovnicza is full. We could either receive `blobovnicza.ErrFull` error + // or update active blobovnicza in other thread. In the latter case the database will be closed + // and `updateActive` takes care of not updating the active blobovnicza twice. + if isFull := errors.Is(err, blobovnicza.ErrFull); isFull || errors.Is(err, bbolt.ErrDatabaseNotOpen) { + if isFull { + i.B.log.Debug("blobovnicza overflowed", + zap.String("path", filepath.Join(path, u64ToHexString(active.ind)))) + } + + if err := i.B.updateActive(path, &active.ind); err != nil { + if !isLogical(err) { + i.B.reportError("could not update active blobovnicza", err) + } else { + i.B.log.Debug("could not update active blobovnicza", + zap.String("level", path), + zap.String("error", err.Error())) + } + + return false, nil + } + + return i.iterate(path) + } + + i.AllFull = false + if !isLogical(err) { + i.B.reportError("could not put object to active blobovnicza", err) + } else { + i.B.log.Debug("could not put object to active blobovnicza", + zap.String("path", filepath.Join(path, u64ToHexString(active.ind))), + zap.String("error", err.Error())) + } + + return false, nil + } + + path = filepath.Join(path, u64ToHexString(active.ind)) + + i.ID = blobovnicza.NewIDFromBytes([]byte(path)) + + return true, nil } From 8273a3dfb26665bd4f29d0d3aad6c7e6c44f1c8c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 30 Mar 2023 13:15:29 +0300 Subject: [PATCH 0021/1943] [#188] blobstor: Refactor blobstor test Resolve funlen linter for TestIterate function Signed-off-by: Dmitrii Stepanov --- .../blobstor/internal/blobstortest/iterate.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index e2a310efc..f98cca638 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/require" ) -// nolint: funlen func TestIterate(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) require.NoError(t, s.Open(false)) @@ -28,6 +27,14 @@ func TestIterate(t *testing.T, cons Constructor, min, max uint64) { objects = append(objects[:delID], objects[delID+1:]...) + runTestNormalHandler(t, s, objects) + + runTestLazyHandler(t, s, objects) + + runTestIgnoreLogicalErrors(t, s, objects) +} + +func runTestNormalHandler(t *testing.T, s common.Storage, objects []objectDesc) { t.Run("normal handler", func(t *testing.T) { seen := make(map[string]objectDesc) @@ -52,7 +59,9 @@ func TestIterate(t *testing.T, cons Constructor, min, max uint64) { require.Equal(t, objects[i].storageID, d.storageID) } }) +} +func runTestLazyHandler(t *testing.T, s common.Storage, objects []objectDesc) { t.Run("lazy handler", func(t *testing.T) { seen := make(map[string]func() ([]byte, error)) @@ -74,7 +83,9 @@ func TestIterate(t *testing.T, cons Constructor, min, max uint64) { require.Equal(t, objects[i].raw, data) } }) +} +func runTestIgnoreLogicalErrors(t *testing.T, s common.Storage, objects []objectDesc) { t.Run("ignore errors doesn't work for logical errors", func(t *testing.T) { seen := make(map[string]objectDesc) From 0739c36a3b336e8bdce8de34b611a8009ad57c02 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 30 Mar 2023 13:42:10 +0300 Subject: [PATCH 0022/1943] [#188] metabase: Refactor object put to metabase Resolve funlen linter for db.put method Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/put.go | 53 ++++++++++++++---------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 07bb4c2b5..8e11c5d9c 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -10,6 +10,7 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/io" @@ -76,10 +77,11 @@ func (db *DB) Put(prm PutPrm) (res PutRes, err error) { return } -// nolint: funlen -func (db *DB) put( - tx *bbolt.Tx, obj *objectSDK.Object, id []byte, - si *objectSDK.SplitInfo, currEpoch uint64) error { +func (db *DB) put(tx *bbolt.Tx, + obj *objectSDK.Object, + id []byte, + si *objectSDK.SplitInfo, + currEpoch uint64) error { cnr, ok := obj.ContainerID() if !ok { return errors.New("missing container in object") @@ -95,25 +97,34 @@ func (db *DB) put( return err // return any error besides SplitInfoError } - // most right child and split header overlap parent so we have to - // check if object exists to not overwrite it twice if exists { - // When storage engine moves objects between different sub-storages, - // it calls metabase.Put method with new storage ID, thus triggering this code. - if !isParent && id != nil { - return updateStorageID(tx, object.AddressOf(obj), id) - } - - // when storage already has last object in split hierarchy and there is - // a linking object to put (or vice versa), we should update split info - // with object ids of these objects - if isParent { - return updateSplitInfo(tx, object.AddressOf(obj), si) - } - - return nil + return db.updateObj(tx, obj, id, si, isParent) } + return db.insertObject(tx, obj, id, si, isParent, cnr, currEpoch) +} + +func (db *DB) updateObj(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *objectSDK.SplitInfo, isParent bool) error { + // most right child and split header overlap parent so we have to + // check if object exists to not overwrite it twice + + // When storage engine moves objects between different sub-storages, + // it calls metabase.Put method with new storage ID, thus triggering this code. + if !isParent && id != nil { + return updateStorageID(tx, object.AddressOf(obj), id) + } + + // when storage already has last object in split hierarchy and there is + // a linking object to put (or vice versa), we should update split info + // with object ids of these objects + if isParent { + return updateSplitInfo(tx, object.AddressOf(obj), si) + } + + return nil +} + +func (db *DB) insertObject(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *objectSDK.SplitInfo, isParent bool, cnr cid.ID, currEpoch uint64) error { if par := obj.Parent(); par != nil && !isParent { // limit depth by two parentSI, err := splitInfoFromObject(obj) if err != nil { @@ -126,7 +137,7 @@ func (db *DB) put( } } - err = putUniqueIndexes(tx, obj, si, id) + err := putUniqueIndexes(tx, obj, si, id) if err != nil { return fmt.Errorf("can't put unique indexes: %w", err) } From 3010ca26494d63ca81651dd2a22765fdaf3b6d52 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 30 Mar 2023 17:32:33 +0300 Subject: [PATCH 0023/1943] [#188] engine: Refactor get object from engine Resolve funlen linter for StorageEngine.get method Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/get.go | 196 +++++++++++++------------ 1 file changed, 103 insertions(+), 93 deletions(-) diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 787a7bac8..4d0a30bc8 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -52,124 +52,134 @@ func (e *StorageEngine) Get(prm GetPrm) (res GetRes, err error) { return } -// nolint: funlen func (e *StorageEngine) get(prm GetPrm) (GetRes, error) { if e.metrics != nil { defer elapsed(e.metrics.AddGetDuration)() } - var ( - obj *objectSDK.Object - siErr *objectSDK.SplitInfoError - - errNotFound apistatus.ObjectNotFound - - outSI *objectSDK.SplitInfo - outError error = errNotFound - - shardWithMeta hashedShard - metaError error - ) + var errNotFound apistatus.ObjectNotFound var shPrm shard.GetPrm shPrm.SetAddress(prm.addr) - var hasDegraded bool - var objectExpired bool - - e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { - noMeta := sh.GetMode().NoMetabase() - shPrm.SetIgnoreMeta(noMeta) - - hasDegraded = hasDegraded || noMeta - - res, err := sh.Get(shPrm) - if err != nil { - if res.HasMeta() { - shardWithMeta = sh - metaError = err - } - switch { - case shard.IsErrNotFound(err): - return false // ignore, go to next shard - case errors.As(err, &siErr): - if outSI == nil { - outSI = objectSDK.NewSplitInfo() - } - - util.MergeSplitInfo(siErr.SplitInfo(), outSI) - - _, withLink := outSI.Link() - _, withLast := outSI.LastPart() - - // stop iterating over shards if SplitInfo structure is complete - if withLink && withLast { - return true - } - - return false - case shard.IsErrRemoved(err): - outError = err - - return true // stop, return it back - case shard.IsErrObjectExpired(err): - // object is found but should not - // be returned - objectExpired = true - return true - default: - e.reportShardError(sh, "could not get object from shard", err) - return false - } - } - - obj = res.Object() - - return true - }) - - if outSI != nil { - return GetRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(outSI)) + it := &getShardIterator{ + OutError: errNotFound, + ShardPrm: shPrm, + Address: prm.addr, + Engine: e, } - if objectExpired { + it.tryGetWithMeta() + + if it.SplitInfo != nil { + return GetRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(it.SplitInfo)) + } + + if it.ObjectExpired { return GetRes{}, errNotFound } - if obj == nil { - if !hasDegraded && shardWithMeta.Shard == nil || !shard.IsErrNotFound(outError) { - return GetRes{}, outError + if it.Object == nil { + if !it.HasDegraded && it.ShardWithMeta.Shard == nil || !shard.IsErrNotFound(it.OutError) { + return GetRes{}, it.OutError } - // If the object is not found but is present in metabase, - // try to fetch it from blobstor directly. If it is found in any - // blobstor, increase the error counter for the shard which contains the meta. - shPrm.SetIgnoreMeta(true) + it.tryGetFromBlobstore() - e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { - if sh.GetMode().NoMetabase() { - // Already visited. - return false - } - - res, err := sh.Get(shPrm) - obj = res.Object() - return err == nil - }) - if obj == nil { - return GetRes{}, outError + if it.Object == nil { + return GetRes{}, it.OutError } - if shardWithMeta.Shard != nil { - e.reportShardError(shardWithMeta, "meta info was present, but object is missing", - metaError, zap.Stringer("address", prm.addr)) + if it.ShardWithMeta.Shard != nil { + e.reportShardError(it.ShardWithMeta, "meta info was present, but object is missing", + it.MetaError, zap.Stringer("address", prm.addr)) } } return GetRes{ - obj: obj, + obj: it.Object, }, nil } +type getShardIterator struct { + Object *objectSDK.Object + SplitInfo *objectSDK.SplitInfo + OutError error + ShardWithMeta hashedShard + MetaError error + HasDegraded bool + ObjectExpired bool + + ShardPrm shard.GetPrm + Address oid.Address + Engine *StorageEngine + + splitInfoErr *objectSDK.SplitInfoError +} + +func (i *getShardIterator) tryGetWithMeta() { + i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { + noMeta := sh.GetMode().NoMetabase() + i.ShardPrm.SetIgnoreMeta(noMeta) + + i.HasDegraded = i.HasDegraded || noMeta + + res, err := sh.Get(i.ShardPrm) + if err == nil { + i.Object = res.Object() + return true + } + + if res.HasMeta() { + i.ShardWithMeta = sh + i.MetaError = err + } + switch { + case shard.IsErrNotFound(err): + return false // ignore, go to next shard + case errors.As(err, &i.splitInfoErr): + if i.SplitInfo == nil { + i.SplitInfo = objectSDK.NewSplitInfo() + } + + util.MergeSplitInfo(i.splitInfoErr.SplitInfo(), i.SplitInfo) + + _, withLink := i.SplitInfo.Link() + _, withLast := i.SplitInfo.LastPart() + + // stop iterating over shards if SplitInfo structure is complete + return withLink && withLast + case shard.IsErrRemoved(err): + i.OutError = err + return true // stop, return it back + case shard.IsErrObjectExpired(err): + // object is found but should not be returned + i.ObjectExpired = true + return true + default: + i.Engine.reportShardError(sh, "could not get object from shard", err) + return false + } + }) +} + +func (i *getShardIterator) tryGetFromBlobstore() { + // If the object is not found but is present in metabase, + // try to fetch it from blobstor directly. If it is found in any + // blobstor, increase the error counter for the shard which contains the meta. + i.ShardPrm.SetIgnoreMeta(true) + + i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { + if sh.GetMode().NoMetabase() { + // Already visited. + return false + } + + res, err := sh.Get(i.ShardPrm) + i.Object = res.Object() + return err == nil + }) +} + // Get reads object from local storage by provided address. func Get(storage *StorageEngine, addr oid.Address) (*objectSDK.Object, error) { var getPrm GetPrm From 456bc097f7144c7bde37d7a5e8a5fc8f2b8afb85 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 30 Mar 2023 18:44:43 +0300 Subject: [PATCH 0024/1943] [#188] engine: Refactor get range from engine Resolve funlen linter for StorageEngine.getRange method Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/range.go | 197 ++++++++++++----------- 1 file changed, 104 insertions(+), 93 deletions(-) diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 491b226bf..25b533bd4 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -65,122 +65,51 @@ func (e *StorageEngine) GetRange(prm RngPrm) (res RngRes, err error) { return } -// nolint: funlen func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) { if e.metrics != nil { defer elapsed(e.metrics.AddRangeDuration)() } - var ( - obj *objectSDK.Object - siErr *objectSDK.SplitInfoError - - errNotFound apistatus.ObjectNotFound - - outSI *objectSDK.SplitInfo - outError error = errNotFound - - shardWithMeta hashedShard - metaError error - ) - - var hasDegraded bool + var errNotFound apistatus.ObjectNotFound var shPrm shard.RngPrm shPrm.SetAddress(prm.addr) shPrm.SetRange(prm.off, prm.ln) - e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { - noMeta := sh.GetMode().NoMetabase() - hasDegraded = hasDegraded || noMeta - shPrm.SetIgnoreMeta(noMeta) - - res, err := sh.GetRange(shPrm) - if err != nil { - if res.HasMeta() { - shardWithMeta = sh - metaError = err - } - switch { - case shard.IsErrNotFound(err): - return false // ignore, go to next shard - case errors.As(err, &siErr): - if outSI == nil { - outSI = objectSDK.NewSplitInfo() - } - - util.MergeSplitInfo(siErr.SplitInfo(), outSI) - - _, withLink := outSI.Link() - _, withLast := outSI.LastPart() - - // stop iterating over shards if SplitInfo structure is complete - if withLink && withLast { - return true - } - - return false - case - shard.IsErrRemoved(err), - shard.IsErrOutOfRange(err): - outError = err - - return true // stop, return it back - default: - e.reportShardError(sh, "could not get object from shard", err) - return false - } - } - - obj = res.Object() - - return true - }) - - if outSI != nil { - return RngRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(outSI)) + it := &getRangeShardIterator{ + OutError: errNotFound, + ShardPrm: shPrm, + Address: prm.addr, + Engine: e, } - if obj == nil { + it.tryGetWithMeta() + + if it.SplitInfo != nil { + return RngRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(it.SplitInfo)) + } + + if it.Object == nil { // If any shard is in a degraded mode, we should assume that metabase could store // info about some object. - if shardWithMeta.Shard == nil && !hasDegraded || !shard.IsErrNotFound(outError) { - return RngRes{}, outError + if it.ShardWithMeta.Shard == nil && !it.HasDegraded || !shard.IsErrNotFound(it.OutError) { + return RngRes{}, it.OutError } - // If the object is not found but is present in metabase, - // try to fetch it from blobstor directly. If it is found in any - // blobstor, increase the error counter for the shard which contains the meta. - shPrm.SetIgnoreMeta(true) + it.tryGetFromBlobstor() - e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { - if sh.GetMode().NoMetabase() { - // Already processed it without a metabase. - return false - } - - res, err := sh.GetRange(shPrm) - if shard.IsErrOutOfRange(err) { - var errOutOfRange apistatus.ObjectOutOfRange - - outError = errOutOfRange - return true - } - obj = res.Object() - return err == nil - }) - if obj == nil { - return RngRes{}, outError + if it.Object == nil { + return RngRes{}, it.OutError } - if shardWithMeta.Shard != nil { - e.reportShardError(shardWithMeta, "meta info was present, but object is missing", - metaError, + if it.ShardWithMeta.Shard != nil { + e.reportShardError(it.ShardWithMeta, "meta info was present, but object is missing", + it.MetaError, zap.Stringer("address", prm.addr)) } } return RngRes{ - obj: obj, + obj: it.Object, }, nil } @@ -197,3 +126,85 @@ func GetRange(storage *StorageEngine, addr oid.Address, rng *objectSDK.Range) ([ return res.Object().Payload(), nil } + +type getRangeShardIterator struct { + Object *objectSDK.Object + SplitInfoError *objectSDK.SplitInfoError + SplitInfo *objectSDK.SplitInfo + OutError error + ShardWithMeta hashedShard + MetaError error + HasDegraded bool + + ShardPrm shard.RngPrm + Address oid.Address + Engine *StorageEngine +} + +func (i *getRangeShardIterator) tryGetWithMeta() { + i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { + noMeta := sh.GetMode().NoMetabase() + i.HasDegraded = i.HasDegraded || noMeta + i.ShardPrm.SetIgnoreMeta(noMeta) + + res, err := sh.GetRange(i.ShardPrm) + if err == nil { + i.Object = res.Object() + return true + } + + if res.HasMeta() { + i.ShardWithMeta = sh + i.MetaError = err + } + switch { + case shard.IsErrNotFound(err): + return false // ignore, go to next shard + case errors.As(err, &i.SplitInfoError): + if i.SplitInfo == nil { + i.SplitInfo = objectSDK.NewSplitInfo() + } + + util.MergeSplitInfo(i.SplitInfoError.SplitInfo(), i.SplitInfo) + + _, withLink := i.SplitInfo.Link() + _, withLast := i.SplitInfo.LastPart() + + // stop iterating over shards if SplitInfo structure is complete + return withLink && withLast + case + shard.IsErrRemoved(err), + shard.IsErrOutOfRange(err): + i.OutError = err + + return true // stop, return it back + default: + i.Engine.reportShardError(sh, "could not get object from shard", err) + return false + } + }) +} + +func (i *getRangeShardIterator) tryGetFromBlobstor() { + // If the object is not found but is present in metabase, + // try to fetch it from blobstor directly. If it is found in any + // blobstor, increase the error counter for the shard which contains the meta. + i.ShardPrm.SetIgnoreMeta(true) + + i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { + if sh.GetMode().NoMetabase() { + // Already processed it without a metabase. + return false + } + + res, err := sh.GetRange(i.ShardPrm) + if shard.IsErrOutOfRange(err) { + var errOutOfRange apistatus.ObjectOutOfRange + + i.OutError = errOutOfRange + return true + } + i.Object = res.Object() + return err == nil + }) +} From 5a66db80c564c1d45bad0e74f952c1884828b26f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 31 Mar 2023 11:33:08 +0300 Subject: [PATCH 0025/1943] [#188] engine: Refactor shard evacuation Resolve funlen and gocognit linter for StorageEngine.Evacuate method Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 216 +++++++++++--------- 1 file changed, 120 insertions(+), 96 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 4bc7eac1a..457228bb2 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -57,35 +58,89 @@ var errMustHaveTwoShards = errors.New("must have at least 1 spare shard") // Evacuate moves data from one shard to the others. // The shard being moved must be in read-only mode. -// -// nolint: funlen, gocognit func (e *StorageEngine) Evacuate(prm EvacuateShardPrm) (EvacuateShardRes, error) { - sidList := make([]string, len(prm.shardID)) + shardIDs := make([]string, len(prm.shardID)) for i := range prm.shardID { - sidList[i] = prm.shardID[i].String() + shardIDs[i] = prm.shardID[i].String() } + shards, weights, err := e.getActualShards(shardIDs, prm.handler != nil) + if err != nil { + return EvacuateShardRes{}, err + } + + shardsToEvacuate := make(map[string]*shard.Shard) + for i := range shardIDs { + for j := range shards { + if shards[j].ID().String() == shardIDs[i] { + shardsToEvacuate[shardIDs[i]] = shards[j].Shard + } + } + } + + e.log.Info("started shards evacuation", zap.Strings("shard_ids", shardIDs)) + + var res EvacuateShardRes + + for _, shardID := range shardIDs { + if err = e.evacuateShard(shardID, prm, &res, shards, weights, shardsToEvacuate); err != nil { + return res, err + } + } + + e.log.Info("finished shards evacuation", zap.Strings("shard_ids", shardIDs)) + return res, nil +} + +func (e *StorageEngine) evacuateShard(shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + var listPrm shard.ListWithCursorPrm + listPrm.WithCount(defaultEvacuateBatchSize) + + sh := shardsToEvacuate[shardID] + + var c *meta.Cursor + for { + listPrm.WithCursor(c) + + // TODO (@fyrchik): #1731 this approach doesn't work in degraded modes + // because ListWithCursor works only with the metabase. + listRes, err := sh.ListWithCursor(listPrm) + if err != nil { + if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { + break + } + return err + } + + if err = e.evacuateObjects(sh, listRes.AddressList(), prm, res, shards, weights, shardsToEvacuate); err != nil { + return err + } + + c = listRes.Cursor() + } + return nil +} + +func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) ([]pooledShard, []float64, error) { e.mtx.RLock() - for i := range sidList { - sh, ok := e.shards[sidList[i]] + defer e.mtx.RUnlock() + + for i := range shardIDs { + sh, ok := e.shards[shardIDs[i]] if !ok { - e.mtx.RUnlock() - return EvacuateShardRes{}, errShardNotFound + return nil, nil, errShardNotFound } if !sh.GetMode().ReadOnly() { - e.mtx.RUnlock() - return EvacuateShardRes{}, shard.ErrMustBeReadOnly + return nil, nil, shard.ErrMustBeReadOnly } } - if len(e.shards)-len(sidList) < 1 && prm.handler == nil { - e.mtx.RUnlock() - return EvacuateShardRes{}, errMustHaveTwoShards + if len(e.shards)-len(shardIDs) < 1 && !handlerDefined { + return nil, nil, errMustHaveTwoShards } - e.log.Info("started shards evacuation", zap.Strings("shard_ids", sidList)) - // We must have all shards, to have correct information about their // indexes in a sorted slice and set appropriate marks in the metabase. // Evacuated shard is skipped during put. @@ -96,100 +151,69 @@ func (e *StorageEngine) Evacuate(prm EvacuateShardPrm) (EvacuateShardRes, error) pool: e.shardPools[id], }) } - e.mtx.RUnlock() weights := make([]float64, 0, len(shards)) for i := range shards { weights = append(weights, e.shardWeight(shards[i].Shard)) } - shardMap := make(map[string]*shard.Shard) - for i := range sidList { - for j := range shards { - if shards[j].ID().String() == sidList[i] { - shardMap[sidList[i]] = shards[j].Shard + return shards, weights, nil +} + +func (e *StorageEngine) evacuateObjects(sh *shard.Shard, toEvacuate []object.AddressWithType, prm EvacuateShardPrm, res *EvacuateShardRes, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + for i := range toEvacuate { + addr := toEvacuate[i].Address + + var getPrm shard.GetPrm + getPrm.SetAddress(addr) + + getRes, err := sh.Get(getPrm) + if err != nil { + if prm.ignoreErrors { + continue } + return err } + + if e.tryEvacuateObject(addr, getRes.Object(), sh, res, shards, weights, shardsToEvacuate) { + continue + } + + if prm.handler == nil { + // Do not check ignoreErrors flag here because + // ignoring errors on put make this command kinda useless. + return fmt.Errorf("%w: %s", errPutShard, toEvacuate[i]) + } + + err = prm.handler(addr, getRes.Object()) + if err != nil { + return err + } + res.count++ } + return nil +} - var listPrm shard.ListWithCursorPrm - listPrm.WithCount(defaultEvacuateBatchSize) - - var res EvacuateShardRes - -mainLoop: - for n := range sidList { - sh := shardMap[sidList[n]] - - var c *meta.Cursor - for { - listPrm.WithCursor(c) - - // TODO (@fyrchik): #1731 this approach doesn't work in degraded modes - // because ListWithCursor works only with the metabase. - listRes, err := sh.ListWithCursor(listPrm) - if err != nil { - if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { - continue mainLoop - } - return res, err - } - - // TODO (@fyrchik): #1731 parallelize the loop - lst := listRes.AddressList() - - loop: - for i := range lst { - addr := lst[i].Address - - var getPrm shard.GetPrm - getPrm.SetAddress(addr) - - getRes, err := sh.Get(getPrm) - if err != nil { - if prm.ignoreErrors { - continue - } - return res, err - } - - hrw.SortHasherSliceByWeightValue(shards, weights, hrw.Hash([]byte(addr.EncodeToString()))) - for j := range shards { - if _, ok := shardMap[shards[j].ID().String()]; ok { - continue - } - putDone, exists := e.putToShard(shards[j].hashedShard, j, shards[j].pool, addr, getRes.Object()) - if putDone || exists { - if putDone { - e.log.Debug("object is moved to another shard", - zap.String("from", sidList[n]), - zap.Stringer("to", shards[j].ID()), - zap.Stringer("addr", addr)) - - res.count++ - } - continue loop - } - } - - if prm.handler == nil { - // Do not check ignoreErrors flag here because - // ignoring errors on put make this command kinda useless. - return res, fmt.Errorf("%w: %s", errPutShard, lst[i]) - } - - err = prm.handler(addr, getRes.Object()) - if err != nil { - return res, err - } +func (e *StorageEngine) tryEvacuateObject(addr oid.Address, object *objectSDK.Object, sh *shard.Shard, res *EvacuateShardRes, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) bool { + hrw.SortHasherSliceByWeightValue(shards, weights, hrw.Hash([]byte(addr.EncodeToString()))) + for j := range shards { + if _, ok := shardsToEvacuate[shards[j].ID().String()]; ok { + continue + } + putDone, exists := e.putToShard(shards[j].hashedShard, j, shards[j].pool, addr, object) + if putDone || exists { + if putDone { + e.log.Debug("object is moved to another shard", + zap.Stringer("from", sh.ID()), + zap.Stringer("to", shards[j].ID()), + zap.Stringer("addr", addr)) res.count++ } - - c = listRes.Cursor() + return true } } - e.log.Info("finished shards evacuation", - zap.Strings("shard_ids", sidList)) - return res, nil + return false } From 1f1aed87be8acbf577fe65bd49cbd5ceb59bf8c5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 31 Mar 2023 14:12:18 +0300 Subject: [PATCH 0026/1943] [#188] metabase: Refactor object inhume Resolve funlen linter for db.Inhume method Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/inhume.go | 271 +++++++++++--------- 1 file changed, 150 insertions(+), 121 deletions(-) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 3f33f990c..b6e6cadf1 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -118,8 +118,6 @@ var ErrLockObjectRemoval = logicerr.New("lock object removal") // // NOTE: Marks any object with GC mark (despite any prohibitions on operations // with that object) if WithForceGCMark option has been provided. -// -// nolint: funlen, gocognit func (db *DB) Inhume(prm InhumePrm) (res InhumeRes, err error) { db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -131,145 +129,176 @@ func (db *DB) Inhume(prm InhumePrm) (res InhumeRes, err error) { } currEpoch := db.epochState.CurrentEpoch() - var inhumed uint64 - err = db.boltDB.Update(func(tx *bbolt.Tx) error { - garbageBKT := tx.Bucket(garbageBucketName) - graveyardBKT := tx.Bucket(graveyardBucketName) + return db.inhumeTx(tx, currEpoch, prm, &res) + }) - var ( - // target bucket of the operation, one of the: - // 1. Graveyard if Inhume was called with a Tombstone - // 2. Garbage if Inhume was called with a GC mark - bkt *bbolt.Bucket - // value that will be put in the bucket, one of the: - // 1. tombstone address if Inhume was called with - // a Tombstone - // 2. zeroValue if Inhume was called with a GC mark - value []byte - ) + return +} - if prm.tomb != nil { - bkt = graveyardBKT - tombKey := addressKey(*prm.tomb, make([]byte, addressKeySize)) +func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes) error { + garbageBKT := tx.Bucket(garbageBucketName) + graveyardBKT := tx.Bucket(graveyardBucketName) - // it is forbidden to have a tomb-on-tomb in FrostFS, - // so graveyard keys must not be addresses of tombstones - data := bkt.Get(tombKey) - if data != nil { - err := bkt.Delete(tombKey) - if err != nil { - return fmt.Errorf("could not remove grave with tombstone key: %w", err) - } - } + bkt, value, err := db.getInhumeTargetBucketAndValue(garbageBKT, graveyardBKT, &prm) + if err != nil { + return err + } - value = tombKey - } else { - bkt = garbageBKT - value = zeroValue + buf := make([]byte, addressKeySize) + for i := range prm.target { + id := prm.target[i].Object() + cnr := prm.target[i].Container() + + // prevent locked objects to be inhumed + if !prm.forceRemoval && objectLocked(tx, cnr, id) { + return apistatus.ObjectLocked{} } - buf := make([]byte, addressKeySize) - for i := range prm.target { - id := prm.target[i].Object() - cnr := prm.target[i].Container() + var lockWasChecked bool - // prevent locked objects to be inhumed - if !prm.forceRemoval && objectLocked(tx, cnr, id) { - return apistatus.ObjectLocked{} + // prevent lock objects to be inhumed + // if `Inhume` was called not with the + // `WithForceGCMark` option + if !prm.forceRemoval { + if isLockObject(tx, cnr, id) { + return ErrLockObjectRemoval } - var lockWasChecked bool + lockWasChecked = true + } - // prevent lock objects to be inhumed - // if `Inhume` was called not with the - // `WithForceGCMark` option - if !prm.forceRemoval { - if isLockObject(tx, cnr, id) { - return ErrLockObjectRemoval - } - - lockWasChecked = true + obj, err := db.get(tx, prm.target[i], buf, false, true, epoch) + targetKey := addressKey(prm.target[i], buf) + if err == nil { + err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, targetKey, cnr, obj, res) + if err != nil { + return err } + } - obj, err := db.get(tx, prm.target[i], buf, false, true, currEpoch) - targetKey := addressKey(prm.target[i], buf) - if err == nil { - containerID, _ := obj.ContainerID() - if inGraveyardWithKey(targetKey, graveyardBKT, garbageBKT) == 0 { - inhumed++ - res.storeDeletionInfo(containerID, obj.PayloadSize()) - } - - // if object is stored, and it is regular object then update bucket - // with container size estimations - if obj.Type() == object.TypeRegular { - err := changeContainerSize(tx, cnr, obj.PayloadSize(), false) - if err != nil { - return err - } - } - } - - if prm.tomb != nil { - targetIsTomb := false - - // iterate over graveyard and check if target address - // is the address of tombstone in graveyard. - err = bkt.ForEach(func(k, v []byte) error { - // check if graveyard has record with key corresponding - // to tombstone address (at least one) - targetIsTomb = bytes.Equal(v, targetKey) - - if targetIsTomb { - // break bucket iterator - return errBreakBucketForEach - } - - return nil - }) - if err != nil && !errors.Is(err, errBreakBucketForEach) { - return err - } - - // do not add grave if target is a tombstone - if targetIsTomb { - continue - } - - // if tombstone appears object must be - // additionally marked with GC - err = garbageBKT.Put(targetKey, zeroValue) - if err != nil { - return err - } - } - - // consider checking if target is already in graveyard? - err = bkt.Put(targetKey, value) + if prm.tomb != nil { + var isTomb bool + isTomb, err = db.markAsGC(graveyardBKT, garbageBKT, targetKey) if err != nil { return err } - if prm.lockObjectHandling { - // do not perform lock check if - // it was already called - if lockWasChecked { - // inhumed object is not of - // the LOCK type - continue - } - - if isLockObject(tx, cnr, id) { - res.deletedLockObj = append(res.deletedLockObj, prm.target[i]) - } + if isTomb { + continue } } - return db.updateCounter(tx, logical, inhumed, false) - }) + // consider checking if target is already in graveyard? + err = bkt.Put(targetKey, value) + if err != nil { + return err + } - res.availableImhumed = inhumed + if prm.lockObjectHandling { + // do not perform lock check if + // it was already called + if lockWasChecked { + // inhumed object is not of + // the LOCK type + continue + } - return + if isLockObject(tx, cnr, id) { + res.deletedLockObj = append(res.deletedLockObj, prm.target[i]) + } + } + } + + return db.updateCounter(tx, logical, res.availableImhumed, false) +} + +// getInhumeTargetBucketAndValue return target bucket to store inhume result and value that will be put in the bucket. +// +// target bucket of the operation, one of the: +// 1. Graveyard if Inhume was called with a Tombstone +// 2. Garbage if Inhume was called with a GC mark +// +// value that will be put in the bucket, one of the: +// 1. tombstone address if Inhume was called with +// a Tombstone +// 2. zeroValue if Inhume was called with a GC mark +func (db *DB) getInhumeTargetBucketAndValue(garbageBKT, graveyardBKT *bbolt.Bucket, prm *InhumePrm) (targetBucket *bbolt.Bucket, value []byte, err error) { + if prm.tomb != nil { + targetBucket = graveyardBKT + tombKey := addressKey(*prm.tomb, make([]byte, addressKeySize)) + + // it is forbidden to have a tomb-on-tomb in FrostFS, + // so graveyard keys must not be addresses of tombstones + data := targetBucket.Get(tombKey) + if data != nil { + err := targetBucket.Delete(tombKey) + if err != nil { + return nil, nil, fmt.Errorf("could not remove grave with tombstone key: %w", err) + } + } + + value = tombKey + } else { + targetBucket = garbageBKT + value = zeroValue + } + return targetBucket, value, nil +} + +func (db *DB) markAsGC(graveyardBKT, garbageBKT *bbolt.Bucket, key []byte) (bool, error) { + targetIsTomb, err := isTomb(graveyardBKT, key) + if err != nil { + return false, err + } + + // do not add grave if target is a tombstone + if targetIsTomb { + return true, nil + } + + // if tombstone appears object must be + // additionally marked with GC + return false, garbageBKT.Put(key, zeroValue) +} + +func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Bucket, targetKey []byte, cnr cid.ID, obj *object.Object, res *InhumeRes) error { + containerID, _ := obj.ContainerID() + if inGraveyardWithKey(targetKey, graveyardBKT, garbageBKT) == 0 { + res.availableImhumed++ + res.storeDeletionInfo(containerID, obj.PayloadSize()) + } + + // if object is stored, and it is regular object then update bucket + // with container size estimations + if obj.Type() == object.TypeRegular { + err := changeContainerSize(tx, cnr, obj.PayloadSize(), false) + if err != nil { + return err + } + } + return nil +} + +func isTomb(graveyardBucket *bbolt.Bucket, key []byte) (bool, error) { + targetIsTomb := false + + // iterate over graveyard and check if target address + // is the address of tombstone in graveyard. + err := graveyardBucket.ForEach(func(k, v []byte) error { + // check if graveyard has record with key corresponding + // to tombstone address (at least one) + targetIsTomb = bytes.Equal(v, key) + + if targetIsTomb { + // break bucket iterator + return errBreakBucketForEach + } + + return nil + }) + if err != nil && !errors.Is(err, errBreakBucketForEach) { + return false, err + } + return targetIsTomb, nil } From cd33a57f443b2c3348b6eca3e13dbbda4de63fb8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 11:34:47 +0300 Subject: [PATCH 0027/1943] [#207] aclsvc: Refactor EACL check Resolve funlen linter for CheckEACL method. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/acl/acl.go | 70 ++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/pkg/services/object/acl/acl.go b/pkg/services/object/acl/acl.go index a068eadb8..87d2f9c82 100644 --- a/pkg/services/object/acl/acl.go +++ b/pkg/services/object/acl/acl.go @@ -14,6 +14,7 @@ import ( bearerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -118,8 +119,6 @@ func (c *Checker) StickyBitCheck(info v2.RequestInfo, owner user.ID) bool { } // CheckEACL is a main check function for extended ACL. -// -// nolint: funlen func (c *Checker) CheckEACL(msg any, reqInfo v2.RequestInfo) error { basicACL := reqInfo.BasicACL() if !basicACL.Extendable() { @@ -154,6 +153,44 @@ func (c *Checker) CheckEACL(msg any, reqInfo v2.RequestInfo) error { return err } + hdrSrc, err := c.getHeaderSource(cnr, msg, reqInfo) + if err != nil { + return err + } + + eaclRole := getRole(reqInfo) + + action, _ := c.validator.CalculateAction(new(eaclSDK.ValidationUnit). + WithRole(eaclRole). + WithOperation(eaclSDK.Operation(reqInfo.Operation())). + WithContainerID(&cnr). + WithSenderKey(reqInfo.SenderKey()). + WithHeaderSource(hdrSrc). + WithEACLTable(&table), + ) + + if action != eaclSDK.ActionAllow { + return errEACLDeniedByRule + } + return nil +} + +func getRole(reqInfo v2.RequestInfo) eaclSDK.Role { + var eaclRole eaclSDK.Role + switch op := reqInfo.RequestRole(); op { + default: + eaclRole = eaclSDK.Role(op) + case acl.RoleOwner: + eaclRole = eaclSDK.RoleUser + case acl.RoleInnerRing, acl.RoleContainer: + eaclRole = eaclSDK.RoleSystem + case acl.RoleOthers: + eaclRole = eaclSDK.RoleOthers + } + return eaclRole +} + +func (c *Checker) getHeaderSource(cnr cid.ID, msg any, reqInfo v2.RequestInfo) (eaclSDK.TypedHeaderSource, error) { hdrSrcOpts := make([]eaclV2.Option, 0, 3) hdrSrcOpts = append(hdrSrcOpts, @@ -175,34 +212,9 @@ func (c *Checker) CheckEACL(msg any, reqInfo v2.RequestInfo) error { hdrSrc, err := eaclV2.NewMessageHeaderSource(hdrSrcOpts...) if err != nil { - return fmt.Errorf("can't parse headers: %w", err) + return nil, fmt.Errorf("can't parse headers: %w", err) } - - var eaclRole eaclSDK.Role - switch op := reqInfo.RequestRole(); op { - default: - eaclRole = eaclSDK.Role(op) - case acl.RoleOwner: - eaclRole = eaclSDK.RoleUser - case acl.RoleInnerRing, acl.RoleContainer: - eaclRole = eaclSDK.RoleSystem - case acl.RoleOthers: - eaclRole = eaclSDK.RoleOthers - } - - action, _ := c.validator.CalculateAction(new(eaclSDK.ValidationUnit). - WithRole(eaclRole). - WithOperation(eaclSDK.Operation(reqInfo.Operation())). - WithContainerID(&cnr). - WithSenderKey(reqInfo.SenderKey()). - WithHeaderSource(hdrSrc). - WithEACLTable(&table), - ) - - if action != eaclSDK.ActionAllow { - return errEACLDeniedByRule - } - return nil + return hdrSrc, nil } // isValidBearer checks whether bearer token was correctly signed by authorized From 9ef790f7824259733990ef4ec60d8676b04fd49b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 12:03:19 +0300 Subject: [PATCH 0028/1943] [#207] aclsvc: Refactor object headers read Resolve funlen linter for readObjectHeaders method. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/acl/eacl/v2/headers.go | 161 +++++++++++---------- 1 file changed, 84 insertions(+), 77 deletions(-) diff --git a/pkg/services/object/acl/eacl/v2/headers.go b/pkg/services/object/acl/eacl/v2/headers.go index 63e733cf6..736c05763 100644 --- a/pkg/services/object/acl/eacl/v2/headers.go +++ b/pkg/services/object/acl/eacl/v2/headers.go @@ -101,96 +101,103 @@ func requestHeaders(msg xHeaderSource) []eaclSDK.Header { var errMissingOID = errors.New("object ID is missing") -// nolint: funlen func (h *cfg) readObjectHeaders(dst *headerSource) error { switch m := h.msg.(type) { default: panic(fmt.Sprintf("unexpected message type %T", h.msg)) case requestXHeaderSource: - switch req := m.req.(type) { - case - *objectV2.GetRequest, - *objectV2.HeadRequest: - if h.obj == nil { - return errMissingOID - } - - objHeaders, completed := h.localObjectHeaders(h.cnr, h.obj) - - dst.objectHeaders = objHeaders - dst.incompleteObjectHeaders = !completed - case - *objectV2.GetRangeRequest, - *objectV2.GetRangeHashRequest, - *objectV2.DeleteRequest: - if h.obj == nil { - return errMissingOID - } - - dst.objectHeaders = addressHeaders(h.cnr, h.obj) - case *objectV2.PutRequest: - if v, ok := req.GetBody().GetObjectPart().(*objectV2.PutObjectPartInit); ok { - oV2 := new(objectV2.Object) - oV2.SetObjectID(v.GetObjectID()) - oV2.SetHeader(v.GetHeader()) - - dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) - } - case *objectV2.SearchRequest: - cnrV2 := req.GetBody().GetContainerID() - var cnr cid.ID - - if cnrV2 != nil { - if err := cnr.ReadFromV2(*cnrV2); err != nil { - return fmt.Errorf("can't parse container ID: %w", err) - } - } - - dst.objectHeaders = []eaclSDK.Header{cidHeader(cnr)} - } + return h.readObjectHeadersFromRequestXHeaderSource(m, dst) case responseXHeaderSource: - switch resp := m.resp.(type) { - default: - objectHeaders, completed := h.localObjectHeaders(h.cnr, h.obj) + return h.readObjectHeadersResponseXHeaderSource(m, dst) + } +} - dst.objectHeaders = objectHeaders - dst.incompleteObjectHeaders = !completed - case *objectV2.GetResponse: - if v, ok := resp.GetBody().GetObjectPart().(*objectV2.GetObjectPartInit); ok { - oV2 := new(objectV2.Object) - oV2.SetObjectID(v.GetObjectID()) - oV2.SetHeader(v.GetHeader()) +func (h *cfg) readObjectHeadersFromRequestXHeaderSource(m requestXHeaderSource, dst *headerSource) error { + switch req := m.req.(type) { + case + *objectV2.GetRequest, + *objectV2.HeadRequest: + if h.obj == nil { + return errMissingOID + } - dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) - } - case *objectV2.HeadResponse: + objHeaders, completed := h.localObjectHeaders(h.cnr, h.obj) + + dst.objectHeaders = objHeaders + dst.incompleteObjectHeaders = !completed + case + *objectV2.GetRangeRequest, + *objectV2.GetRangeHashRequest, + *objectV2.DeleteRequest: + if h.obj == nil { + return errMissingOID + } + + dst.objectHeaders = addressHeaders(h.cnr, h.obj) + case *objectV2.PutRequest: + if v, ok := req.GetBody().GetObjectPart().(*objectV2.PutObjectPartInit); ok { oV2 := new(objectV2.Object) - - var hdr *objectV2.Header - - switch v := resp.GetBody().GetHeaderPart().(type) { - case *objectV2.ShortHeader: - hdr = new(objectV2.Header) - - var idV2 refsV2.ContainerID - h.cnr.WriteToV2(&idV2) - - hdr.SetContainerID(&idV2) - hdr.SetVersion(v.GetVersion()) - hdr.SetCreationEpoch(v.GetCreationEpoch()) - hdr.SetOwnerID(v.GetOwnerID()) - hdr.SetObjectType(v.GetObjectType()) - hdr.SetPayloadLength(v.GetPayloadLength()) - case *objectV2.HeaderWithSignature: - hdr = v.GetHeader() - } - - oV2.SetHeader(hdr) + oV2.SetObjectID(v.GetObjectID()) + oV2.SetHeader(v.GetHeader()) dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) } - } + case *objectV2.SearchRequest: + cnrV2 := req.GetBody().GetContainerID() + var cnr cid.ID + if cnrV2 != nil { + if err := cnr.ReadFromV2(*cnrV2); err != nil { + return fmt.Errorf("can't parse container ID: %w", err) + } + } + + dst.objectHeaders = []eaclSDK.Header{cidHeader(cnr)} + } + return nil +} + +func (h *cfg) readObjectHeadersResponseXHeaderSource(m responseXHeaderSource, dst *headerSource) error { + switch resp := m.resp.(type) { + default: + objectHeaders, completed := h.localObjectHeaders(h.cnr, h.obj) + + dst.objectHeaders = objectHeaders + dst.incompleteObjectHeaders = !completed + case *objectV2.GetResponse: + if v, ok := resp.GetBody().GetObjectPart().(*objectV2.GetObjectPartInit); ok { + oV2 := new(objectV2.Object) + oV2.SetObjectID(v.GetObjectID()) + oV2.SetHeader(v.GetHeader()) + + dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) + } + case *objectV2.HeadResponse: + oV2 := new(objectV2.Object) + + var hdr *objectV2.Header + + switch v := resp.GetBody().GetHeaderPart().(type) { + case *objectV2.ShortHeader: + hdr = new(objectV2.Header) + + var idV2 refsV2.ContainerID + h.cnr.WriteToV2(&idV2) + + hdr.SetContainerID(&idV2) + hdr.SetVersion(v.GetVersion()) + hdr.SetCreationEpoch(v.GetCreationEpoch()) + hdr.SetOwnerID(v.GetOwnerID()) + hdr.SetObjectType(v.GetObjectType()) + hdr.SetPayloadLength(v.GetPayloadLength()) + case *objectV2.HeaderWithSignature: + hdr = v.GetHeader() + } + + oV2.SetHeader(hdr) + + dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) + } return nil } From 585415fa92782dc157ef0681bb74428fc33793b9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 12:18:43 +0300 Subject: [PATCH 0029/1943] [#207] aclsvc: Refactor send checker Resolve funlen linter for putStreamBasicChecker.Send method. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/acl/v2/service.go | 53 ++++++++++++++++----------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 1e451a99f..6544d78d7 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -443,7 +443,6 @@ func (b Service) GetRangeHash( return b.next.GetRangeHash(ctx, request) } -// nolint: funlen func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { body := request.GetBody() if body == nil { @@ -482,27 +481,9 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe } var sTok *sessionSDK.Object - - if tokV2 := request.GetMetaHeader().GetSessionToken(); tokV2 != nil { - sTok = new(sessionSDK.Object) - - err = sTok.ReadFromV2(*tokV2) - if err != nil { - return fmt.Errorf("invalid session token: %w", err) - } - - if sTok.AssertVerb(sessionSDK.VerbObjectDelete) { - // if session relates to object's removal, we don't check - // relation of the tombstone to the session here since user - // can't predict tomb's ID. - err = assertSessionRelation(*sTok, cnr, nil) - } else { - err = assertSessionRelation(*sTok, cnr, obj) - } - - if err != nil { - return err - } + sTok, err = p.readSessionToken(cnr, obj, request) + if err != nil { + return err } bTok, err := originalBearerToken(request.GetMetaHeader()) @@ -534,6 +515,34 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe return p.next.Send(ctx, request) } +func (p putStreamBasicChecker) readSessionToken(cnr cid.ID, obj *oid.ID, request *objectV2.PutRequest) (*sessionSDK.Object, error) { + var sTok *sessionSDK.Object + + if tokV2 := request.GetMetaHeader().GetSessionToken(); tokV2 != nil { + sTok = new(sessionSDK.Object) + + err := sTok.ReadFromV2(*tokV2) + if err != nil { + return nil, fmt.Errorf("invalid session token: %w", err) + } + + if sTok.AssertVerb(sessionSDK.VerbObjectDelete) { + // if session relates to object's removal, we don't check + // relation of the tombstone to the session here since user + // can't predict tomb's ID. + err = assertSessionRelation(*sTok, cnr, nil) + } else { + err = assertSessionRelation(*sTok, cnr, obj) + } + + if err != nil { + return nil, err + } + } + + return sTok, nil +} + func (p putStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PutResponse, error) { return p.next.CloseAndRecv(ctx) } From 4941926c9d5d653d49fe8a50f1c6c685342841e8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 12:25:11 +0300 Subject: [PATCH 0030/1943] [#207] aclsvc: Drop outdated tag Signed-off-by: Dmitrii Stepanov --- pkg/services/object/acl/v2/util.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/services/object/acl/v2/util.go b/pkg/services/object/acl/v2/util.go index 5e3be6e52..aa5d67584 100644 --- a/pkg/services/object/acl/v2/util.go +++ b/pkg/services/object/acl/v2/util.go @@ -166,7 +166,6 @@ func isOwnerFromKey(id user.ID, key *keys.PublicKey) bool { // assertVerb checks that token verb corresponds to op. func assertVerb(tok sessionSDK.Object, op acl.Op) bool { - //nolint:exhaustive switch op { case acl.OpObjectPut: return tok.AssertVerb(sessionSDK.VerbObjectPut, sessionSDK.VerbObjectDelete) From 5f2a1531fef53be54179734f992696f5a8fa5252 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 12:45:59 +0300 Subject: [PATCH 0031/1943] [#208] deletesvc: Resolve containedctx linter Signed-off-by: Dmitrii Stepanov --- pkg/services/object/delete/delete.go | 7 +++--- pkg/services/object/delete/exec.go | 35 +++++++++++---------------- pkg/services/object/delete/local.go | 14 ++++++----- pkg/services/object/delete/service.go | 12 +++++---- pkg/services/object/delete/util.go | 27 +++++++++++---------- 5 files changed, 46 insertions(+), 49 deletions(-) diff --git a/pkg/services/object/delete/delete.go b/pkg/services/object/delete/delete.go index 4a9c476d0..a959b53cb 100644 --- a/pkg/services/object/delete/delete.go +++ b/pkg/services/object/delete/delete.go @@ -23,22 +23,21 @@ func (s *Service) Delete(ctx context.Context, prm Prm) error { exec := &execCtx{ svc: s, - ctx: ctx, prm: prm, } exec.setLogger(s.log) - exec.execute() + exec.execute(ctx) return exec.statusError.err } -func (exec *execCtx) execute() { +func (exec *execCtx) execute(ctx context.Context) { exec.log.Debug("serving request...") // perform local operation - exec.executeLocal() + exec.executeLocal(ctx) exec.analyzeStatus(true) } diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index 4da4c8083..782cad71b 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -18,12 +18,9 @@ type statusError struct { err error } -// nolint: containedctx type execCtx struct { svc *Service - ctx context.Context - prm Prm statusError @@ -52,10 +49,6 @@ func (exec *execCtx) setLogger(l *logger.Logger) { )} } -func (exec execCtx) context() context.Context { - return exec.ctx -} - func (exec execCtx) isLocal() bool { return exec.prm.common.LocalOnly() } @@ -80,10 +73,10 @@ func (exec *execCtx) newAddress(id oid.ID) oid.Address { return a } -func (exec *execCtx) formSplitInfo() bool { +func (exec *execCtx) formSplitInfo(ctx context.Context) bool { var err error - exec.splitInfo, err = exec.svc.header.splitInfo(exec) + exec.splitInfo, err = exec.svc.header.splitInfo(ctx, exec) switch { default: @@ -101,29 +94,29 @@ func (exec *execCtx) formSplitInfo() bool { return err == nil } -func (exec *execCtx) collectMembers() (ok bool) { +func (exec *execCtx) collectMembers(ctx context.Context) (ok bool) { if exec.splitInfo == nil { exec.log.Debug("no split info, object is PHY") return true } if _, withLink := exec.splitInfo.Link(); withLink { - ok = exec.collectChildren() + ok = exec.collectChildren(ctx) } if !ok { if _, withLast := exec.splitInfo.LastPart(); withLast { - ok = exec.collectChain() + ok = exec.collectChain(ctx) if !ok { return } } } // may be fail if neither right nor linking ID is set? - return exec.supplementBySplitID() + return exec.supplementBySplitID(ctx) } -func (exec *execCtx) collectChain() bool { +func (exec *execCtx) collectChain(ctx context.Context) bool { var chain []oid.ID exec.log.Debug("assembling chain...") @@ -131,7 +124,7 @@ func (exec *execCtx) collectChain() bool { for prev, withPrev := exec.splitInfo.LastPart(); withPrev; { chain = append(chain, prev) - p, err := exec.svc.header.previous(exec, prev) + p, err := exec.svc.header.previous(ctx, exec, prev) switch { default: @@ -160,10 +153,10 @@ func (exec *execCtx) collectChain() bool { return true } -func (exec *execCtx) collectChildren() bool { +func (exec *execCtx) collectChildren(ctx context.Context) bool { exec.log.Debug("collecting children...") - children, err := exec.svc.header.children(exec) + children, err := exec.svc.header.children(ctx, exec) switch { default: @@ -187,10 +180,10 @@ func (exec *execCtx) collectChildren() bool { } } -func (exec *execCtx) supplementBySplitID() bool { +func (exec *execCtx) supplementBySplitID(ctx context.Context) bool { exec.log.Debug("supplement by split ID") - chain, err := exec.svc.searcher.splitMembers(exec) + chain, err := exec.svc.searcher.splitMembers(ctx, exec) switch { default: @@ -264,8 +257,8 @@ func (exec *execCtx) initTombstoneObject() bool { return true } -func (exec *execCtx) saveTombstone() bool { - id, err := exec.svc.placer.put(exec) +func (exec *execCtx) saveTombstone(ctx context.Context) bool { + id, err := exec.svc.placer.put(ctx, exec) switch { default: diff --git a/pkg/services/object/delete/local.go b/pkg/services/object/delete/local.go index 36af96448..17eb0e4e1 100644 --- a/pkg/services/object/delete/local.go +++ b/pkg/services/object/delete/local.go @@ -1,25 +1,27 @@ package deletesvc import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) -func (exec *execCtx) executeLocal() { +func (exec *execCtx) executeLocal(ctx context.Context) { exec.log.Debug("forming tombstone structure...") - ok := exec.formTombstone() + ok := exec.formTombstone(ctx) if !ok { return } exec.log.Debug("tombstone structure successfully formed, saving...") - exec.saveTombstone() + exec.saveTombstone(ctx) } -func (exec *execCtx) formTombstone() (ok bool) { +func (exec *execCtx) formTombstone(ctx context.Context) (ok bool) { tsLifetime, err := exec.svc.netInfo.TombstoneLifetime() if err != nil { exec.status = statusUndefined @@ -40,7 +42,7 @@ func (exec *execCtx) formTombstone() (ok bool) { exec.log.Debug("forming split info...") - ok = exec.formSplitInfo() + ok = exec.formSplitInfo(ctx) if !ok { return } @@ -49,7 +51,7 @@ func (exec *execCtx) formTombstone() (ok bool) { exec.tombstone.SetSplitID(exec.splitInfo.SplitID()) - ok = exec.collectMembers() + ok = exec.collectMembers(ctx) if !ok { return } diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index f2ea384de..11ff13b45 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -1,6 +1,8 @@ package deletesvc import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" @@ -39,20 +41,20 @@ type cfg struct { header interface { // must return (nil, nil) for PHY objects - splitInfo(*execCtx) (*object.SplitInfo, error) + splitInfo(context.Context, *execCtx) (*object.SplitInfo, error) - children(*execCtx) ([]oid.ID, error) + children(context.Context, *execCtx) ([]oid.ID, error) // must return (nil, nil) for 1st object in chain - previous(*execCtx, oid.ID) (*oid.ID, error) + previous(context.Context, *execCtx, oid.ID) (*oid.ID, error) } searcher interface { - splitMembers(*execCtx) ([]oid.ID, error) + splitMembers(context.Context, *execCtx) ([]oid.ID, error) } placer interface { - put(*execCtx) (*oid.ID, error) + put(context.Context, *execCtx) (*oid.ID, error) } netInfo NetworkInfo diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index cc5433740..f9870f7e0 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -1,6 +1,7 @@ package deletesvc import ( + "context" "errors" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" @@ -20,7 +21,7 @@ type simpleIDWriter struct { ids []oid.ID } -func (w *headSvcWrapper) headAddress(exec *execCtx, addr oid.Address) (*object.Object, error) { +func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oid.Address) (*object.Object, error) { wr := getsvc.NewSimpleObjectWriter() p := getsvc.HeadPrm{} @@ -29,7 +30,7 @@ func (w *headSvcWrapper) headAddress(exec *execCtx, addr oid.Address) (*object.O p.WithRawFlag(true) p.WithAddress(addr) - err := (*getsvc.Service)(w).Head(exec.context(), p) + err := (*getsvc.Service)(w).Head(ctx, p) if err != nil { return nil, err } @@ -37,8 +38,8 @@ func (w *headSvcWrapper) headAddress(exec *execCtx, addr oid.Address) (*object.O return wr.Object(), nil } -func (w *headSvcWrapper) splitInfo(exec *execCtx) (*object.SplitInfo, error) { - _, err := w.headAddress(exec, exec.address()) +func (w *headSvcWrapper) splitInfo(ctx context.Context, exec *execCtx) (*object.SplitInfo, error) { + _, err := w.headAddress(ctx, exec, exec.address()) var errSplitInfo *object.SplitInfoError @@ -52,12 +53,12 @@ func (w *headSvcWrapper) splitInfo(exec *execCtx) (*object.SplitInfo, error) { } } -func (w *headSvcWrapper) children(exec *execCtx) ([]oid.ID, error) { +func (w *headSvcWrapper) children(ctx context.Context, exec *execCtx) ([]oid.ID, error) { link, _ := exec.splitInfo.Link() a := exec.newAddress(link) - linking, err := w.headAddress(exec, a) + linking, err := w.headAddress(ctx, exec, a) if err != nil { return nil, err } @@ -65,10 +66,10 @@ func (w *headSvcWrapper) children(exec *execCtx) ([]oid.ID, error) { return linking.Children(), nil } -func (w *headSvcWrapper) previous(exec *execCtx, id oid.ID) (*oid.ID, error) { +func (w *headSvcWrapper) previous(ctx context.Context, exec *execCtx, id oid.ID) (*oid.ID, error) { a := exec.newAddress(id) - h, err := w.headAddress(exec, a) + h, err := w.headAddress(ctx, exec, a) if err != nil { return nil, err } @@ -81,7 +82,7 @@ func (w *headSvcWrapper) previous(exec *execCtx, id oid.ID) (*oid.ID, error) { return nil, nil } -func (w *searchSvcWrapper) splitMembers(exec *execCtx) ([]oid.ID, error) { +func (w *searchSvcWrapper) splitMembers(ctx context.Context, exec *execCtx) ([]oid.ID, error) { fs := object.SearchFilters{} fs.AddSplitIDFilter(object.MatchStringEqual, exec.splitInfo.SplitID()) @@ -93,7 +94,7 @@ func (w *searchSvcWrapper) splitMembers(exec *execCtx) ([]oid.ID, error) { p.WithContainerID(exec.containerID()) p.WithSearchFilters(fs) - err := (*searchsvc.Service)(w).Search(exec.context(), p) + err := (*searchsvc.Service)(w).Search(ctx, p) if err != nil { return nil, err } @@ -107,7 +108,7 @@ func (s *simpleIDWriter) WriteIDs(ids []oid.ID) error { return nil } -func (w *putSvcWrapper) put(exec *execCtx) (*oid.ID, error) { +func (w *putSvcWrapper) put(ctx context.Context, exec *execCtx) (*oid.ID, error) { streamer, err := (*putsvc.Service)(w).Put() if err != nil { return nil, err @@ -124,12 +125,12 @@ func (w *putSvcWrapper) put(exec *execCtx) (*oid.ID, error) { return nil, err } - err = streamer.SendChunk(exec.context(), new(putsvc.PutChunkPrm).WithChunk(payload)) + err = streamer.SendChunk(ctx, new(putsvc.PutChunkPrm).WithChunk(payload)) if err != nil { return nil, err } - r, err := streamer.Close(exec.context()) + r, err := streamer.Close(ctx) if err != nil { return nil, err } From 0b38419fbf494b7577583034ea83e7d4ec2716a1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 12:54:21 +0300 Subject: [PATCH 0032/1943] [#208] searchsvc: Resolve context linters Resolve containedctx and contextcheck linters. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/search/container.go | 10 +++++----- pkg/services/object/search/exec.go | 9 --------- pkg/services/object/search/search.go | 14 ++++++-------- pkg/services/object/search/search_test.go | 2 +- pkg/services/object/search/service.go | 4 +++- pkg/services/object/search/util.go | 5 +++-- 6 files changed, 18 insertions(+), 26 deletions(-) diff --git a/pkg/services/object/search/container.go b/pkg/services/object/search/container.go index fffcba42a..b158bc23e 100644 --- a/pkg/services/object/search/container.go +++ b/pkg/services/object/search/container.go @@ -9,7 +9,7 @@ import ( "go.uber.org/zap" ) -func (exec *execCtx) executeOnContainer() { +func (exec *execCtx) executeOnContainer(ctx context.Context) { if exec.isLocal() { exec.log.Debug("return result directly") return @@ -28,7 +28,7 @@ func (exec *execCtx) executeOnContainer() { } for { - if exec.processCurrentEpoch() { + if exec.processCurrentEpoch(ctx) { break } @@ -47,7 +47,7 @@ func (exec *execCtx) executeOnContainer() { exec.err = nil } -func (exec *execCtx) processCurrentEpoch() bool { +func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { exec.log.Debug("process epoch", zap.Uint64("number", exec.curProcEpoch), ) @@ -57,7 +57,7 @@ func (exec *execCtx) processCurrentEpoch() bool { return true } - ctx, cancel := context.WithCancel(exec.context()) + ctx, cancel := context.WithCancel(ctx) defer cancel() for { @@ -99,7 +99,7 @@ func (exec *execCtx) processCurrentEpoch() bool { return } - ids, err := c.searchObjects(exec, info) + ids, err := c.searchObjects(ctx, exec, info) if err != nil { exec.log.Debug("remote operation failed", zap.String("error", err.Error())) diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index 8cc9a9a26..f815270d9 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -1,8 +1,6 @@ package searchsvc import ( - "context" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -16,12 +14,9 @@ type statusError struct { err error } -// nolint: containedctx type execCtx struct { svc *Service - ctx context.Context - prm Prm statusError @@ -52,10 +47,6 @@ func (exec *execCtx) setLogger(l *logger.Logger) { )} } -func (exec execCtx) context() context.Context { - return exec.ctx -} - func (exec execCtx) isLocal() bool { return exec.prm.common.LocalOnly() } diff --git a/pkg/services/object/search/search.go b/pkg/services/object/search/search.go index 5a6880922..325b42a54 100644 --- a/pkg/services/object/search/search.go +++ b/pkg/services/object/search/search.go @@ -10,7 +10,6 @@ import ( func (s *Service) Search(ctx context.Context, prm Prm) error { exec := &execCtx{ svc: s, - ctx: ctx, prm: prm, } @@ -18,22 +17,21 @@ func (s *Service) Search(ctx context.Context, prm Prm) error { exec.setLogger(s.log) - //nolint: contextcheck - exec.execute() + exec.execute(ctx) return exec.statusError.err } -func (exec *execCtx) execute() { +func (exec *execCtx) execute(ctx context.Context) { exec.log.Debug("serving request...") // perform local operation exec.executeLocal() - exec.analyzeStatus(true) + exec.analyzeStatus(ctx, true) } -func (exec *execCtx) analyzeStatus(execCnr bool) { +func (exec *execCtx) analyzeStatus(ctx context.Context, execCnr bool) { // analyze local result switch exec.status { default: @@ -45,7 +43,7 @@ func (exec *execCtx) analyzeStatus(execCnr bool) { } if execCnr { - exec.executeOnContainer() - exec.analyzeStatus(false) + exec.executeOnContainer(ctx) + exec.analyzeStatus(ctx, false) } } diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index 1d902e7aa..e95970955 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -116,7 +116,7 @@ func (s *testStorage) search(exec *execCtx) ([]oid.ID, error) { return v.ids, v.err } -func (c *testStorage) searchObjects(exec *execCtx, _ clientcore.NodeInfo) ([]oid.ID, error) { +func (c *testStorage) searchObjects(_ context.Context, exec *execCtx, _ clientcore.NodeInfo) ([]oid.ID, error) { v, ok := c.items[exec.containerID().EncodeToString()] if !ok { return nil, nil diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index aebcfca0f..b858e2219 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -1,6 +1,8 @@ package searchsvc import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" @@ -24,7 +26,7 @@ type Option func(*cfg) type searchClient interface { // searchObjects searches objects on the specified node. // MUST NOT modify execCtx as it can be accessed concurrently. - searchObjects(*execCtx, client.NodeInfo) ([]oid.ID, error) + searchObjects(context.Context, *execCtx, client.NodeInfo) ([]oid.ID, error) } type ClientConstructor interface { diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index 741a224af..610dd77ff 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -1,6 +1,7 @@ package searchsvc import ( + "context" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" @@ -77,7 +78,7 @@ func (c *clientConstructorWrapper) get(info client.NodeInfo) (searchClient, erro }, nil } -func (c *clientWrapper) searchObjects(exec *execCtx, info client.NodeInfo) ([]oid.ID, error) { +func (c *clientWrapper) searchObjects(ctx context.Context, exec *execCtx, info client.NodeInfo) ([]oid.ID, error) { if exec.prm.forwarder != nil { return exec.prm.forwarder(info, c.client) } @@ -98,7 +99,7 @@ func (c *clientWrapper) searchObjects(exec *execCtx, info client.NodeInfo) ([]oi var prm internalclient.SearchObjectsPrm - prm.SetContext(exec.context()) + prm.SetContext(ctx) prm.SetClient(c.client) prm.SetPrivateKey(key) prm.SetSessionToken(exec.prm.common.SessionToken()) From d85703a963937d52eed191adf446748846a2ea16 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 13:05:40 +0300 Subject: [PATCH 0033/1943] [#208] searchsvc: Refactor request forwarding Resolve funlen & gocognit linters for toPrm method. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/search/prm.go | 4 +- pkg/services/object/search/util.go | 2 +- .../object/search/v2/request_forwarder.go | 99 +++++++++++++++++++ pkg/services/object/search/v2/util.go | 95 ++---------------- 4 files changed, 113 insertions(+), 87 deletions(-) create mode 100644 pkg/services/object/search/v2/request_forwarder.go diff --git a/pkg/services/object/search/prm.go b/pkg/services/object/search/prm.go index c80257bd3..da46dfeb6 100644 --- a/pkg/services/object/search/prm.go +++ b/pkg/services/object/search/prm.go @@ -1,6 +1,8 @@ package searchsvc import ( + "context" + coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -29,7 +31,7 @@ type IDListWriter interface { // RequestForwarder is a callback for forwarding of the // original Search requests. -type RequestForwarder func(coreclient.NodeInfo, coreclient.MultiAddressClient) ([]oid.ID, error) +type RequestForwarder func(context.Context, coreclient.NodeInfo, coreclient.MultiAddressClient) ([]oid.ID, error) // SetCommonParameters sets common parameters of the operation. func (p *Prm) SetCommonParameters(common *util.CommonPrm) { diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index 610dd77ff..c12ed2c9f 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -80,7 +80,7 @@ func (c *clientConstructorWrapper) get(info client.NodeInfo) (searchClient, erro func (c *clientWrapper) searchObjects(ctx context.Context, exec *execCtx, info client.NodeInfo) ([]oid.ID, error) { if exec.prm.forwarder != nil { - return exec.prm.forwarder(info, c.client) + return exec.prm.forwarder(ctx, info, c.client) } var sessionInfo *util.SessionInfo diff --git a/pkg/services/object/search/v2/request_forwarder.go b/pkg/services/object/search/v2/request_forwarder.go new file mode 100644 index 000000000..8023f2f0f --- /dev/null +++ b/pkg/services/object/search/v2/request_forwarder.go @@ -0,0 +1,99 @@ +package searchsvc + +import ( + "context" + "crypto/ecdsa" + "errors" + "fmt" + "io" + "sync" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" + rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +type requestForwarder struct { + OnceResign *sync.Once + Request *objectV2.SearchRequest + Key *ecdsa.PrivateKey +} + +func (f *requestForwarder) forwardRequest(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) ([]oid.ID, error) { + var err error + + // once compose and resign forwarding request + f.OnceResign.Do(func() { + // compose meta header of the local server + metaHdr := new(session.RequestMetaHeader) + metaHdr.SetTTL(f.Request.GetMetaHeader().GetTTL() - 1) + // TODO: #1165 think how to set the other fields + metaHdr.SetOrigin(f.Request.GetMetaHeader()) + + f.Request.SetMetaHeader(metaHdr) + + err = signature.SignServiceMessage(f.Key, f.Request) + }) + + if err != nil { + return nil, err + } + + var searchStream *rpc.SearchResponseReader + err = c.RawForAddress(addr, func(cli *rpcclient.Client) error { + searchStream, err = rpc.SearchObjects(cli, f.Request, rpcclient.WithContext(ctx)) + return err + }) + if err != nil { + return nil, err + } + + // code below is copy-pasted from c.SearchObjects implementation, + // perhaps it is worth highlighting the utility function in frostfs-api-go + var ( + searchResult []oid.ID + resp = new(objectV2.SearchResponse) + ) + + for { + // receive message from server stream + err := searchStream.Read(resp) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + + return nil, fmt.Errorf("reading the response failed: %w", err) + } + + // verify response key + if err = internal.VerifyResponseKeyV2(pubkey, resp); err != nil { + return nil, err + } + + // verify response structure + if err := signature.VerifyServiceMessage(resp); err != nil { + return nil, fmt.Errorf("could not verify %T: %w", resp, err) + } + + chunk := resp.GetBody().GetIDList() + var id oid.ID + + for i := range chunk { + err = id.ReadFromV2(chunk[i]) + if err != nil { + return nil, fmt.Errorf("invalid object ID: %w", err) + } + + searchResult = append(searchResult, id) + } + } + + return searchResult, nil +} diff --git a/pkg/services/object/search/v2/util.go b/pkg/services/object/search/v2/util.go index 2bde6b2f5..12158a820 100644 --- a/pkg/services/object/search/v2/util.go +++ b/pkg/services/object/search/v2/util.go @@ -1,20 +1,15 @@ package searchsvc import ( + "context" "errors" "fmt" - "io" "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" - rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -22,7 +17,6 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -// nolint: funlen, gocognit func (s *Service) toPrm(req *objectV2.SearchRequest, stream objectSvc.SearchStream) (*searchsvc.Prm, error) { body := req.GetBody() @@ -38,8 +32,6 @@ func (s *Service) toPrm(req *objectV2.SearchRequest, stream objectSvc.SearchStre return nil, fmt.Errorf("invalid container ID: %w", err) } - meta := req.GetMetaHeader() - commonPrm, err := util.CommonPrmFromV2(req) if err != nil { return nil, err @@ -53,85 +45,18 @@ func (s *Service) toPrm(req *objectV2.SearchRequest, stream objectSvc.SearchStre }) if !commonPrm.LocalOnly() { - var onceResign sync.Once - key, err := s.keyStorage.GetKey(nil) if err != nil { return nil, err } - p.SetRequestForwarder(groupAddressRequestForwarder(func(addr network.Address, c client.MultiAddressClient, pubkey []byte) ([]oid.ID, error) { - var err error + forwarder := &requestForwarder{ + OnceResign: &sync.Once{}, + Request: req, + Key: key, + } - // once compose and resign forwarding request - onceResign.Do(func() { - // compose meta header of the local server - metaHdr := new(session.RequestMetaHeader) - metaHdr.SetTTL(meta.GetTTL() - 1) - // TODO: #1165 think how to set the other fields - metaHdr.SetOrigin(meta) - - req.SetMetaHeader(metaHdr) - - err = signature.SignServiceMessage(key, req) - }) - - if err != nil { - return nil, err - } - - var searchStream *rpc.SearchResponseReader - err = c.RawForAddress(addr, func(cli *rpcclient.Client) error { - searchStream, err = rpc.SearchObjects(cli, req, rpcclient.WithContext(stream.Context())) - return err - }) - if err != nil { - return nil, err - } - - // code below is copy-pasted from c.SearchObjects implementation, - // perhaps it is worth highlighting the utility function in frostfs-api-go - var ( - searchResult []oid.ID - resp = new(objectV2.SearchResponse) - ) - - for { - // receive message from server stream - err := searchStream.Read(resp) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - - return nil, fmt.Errorf("reading the response failed: %w", err) - } - - // verify response key - if err = internal.VerifyResponseKeyV2(pubkey, resp); err != nil { - return nil, err - } - - // verify response structure - if err := signature.VerifyServiceMessage(resp); err != nil { - return nil, fmt.Errorf("could not verify %T: %w", resp, err) - } - - chunk := resp.GetBody().GetIDList() - var id oid.ID - - for i := range chunk { - err = id.ReadFromV2(chunk[i]) - if err != nil { - return nil, fmt.Errorf("invalid object ID: %w", err) - } - - searchResult = append(searchResult, id) - } - } - - return searchResult, nil - })) + p.SetRequestForwarder(groupAddressRequestForwarder(forwarder.forwardRequest)) } p.WithContainerID(id) @@ -140,8 +65,8 @@ func (s *Service) toPrm(req *objectV2.SearchRequest, stream objectSvc.SearchStre return p, nil } -func groupAddressRequestForwarder(f func(network.Address, client.MultiAddressClient, []byte) ([]oid.ID, error)) searchsvc.RequestForwarder { - return func(info client.NodeInfo, c client.MultiAddressClient) ([]oid.ID, error) { +func groupAddressRequestForwarder(f func(context.Context, network.Address, client.MultiAddressClient, []byte) ([]oid.ID, error)) searchsvc.RequestForwarder { + return func(ctx context.Context, info client.NodeInfo, c client.MultiAddressClient) ([]oid.ID, error) { var ( firstErr error res []oid.ID @@ -162,7 +87,7 @@ func groupAddressRequestForwarder(f func(network.Address, client.MultiAddressCli // would be nice to log otherwise }() - res, err = f(addr, c, key) + res, err = f(ctx, addr, c, key) return }) From 760af6b912d6e0c77ee44a56f08552c8b0618a56 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 5 Apr 2023 10:43:48 +0300 Subject: [PATCH 0034/1943] [#211] fstree: Consider ENOSPC a logical error We already do this for file writing, however directory creation can also fail. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobstor/fstree/fstree.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 1a1247001..99484860a 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -228,6 +228,9 @@ func (t *FSTree) Put(prm common.PutPrm) (common.PutRes, error) { p := t.treePath(prm.Address) if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { + if errors.Is(err, syscall.ENOSPC) { + return common.PutRes{}, common.ErrNoSpace + } return common.PutRes{}, err } if !prm.DontCompress { From 9098d0eec0bf9e2da3a51bf4d723173093bdb271 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 5 Apr 2023 10:46:59 +0300 Subject: [PATCH 0035/1943] [#211] engine: Unify shard mode checks for tree operations All operations must ensure the shard is not in a degraded mode. Write operations must also ensure the shard is not in a read-only mode. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/tree.go | 75 ++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 684c92e66..db07c001e 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -23,6 +23,9 @@ func (s *Shard) TreeMove(d pilorama.CIDDescriptor, treeID string, m *pilorama.Mo if s.info.Mode.ReadOnly() { return nil, ErrReadOnlyMode } + if s.info.Mode.NoMetabase() { + return nil, ErrDegradedMode + } return s.pilorama.TreeMove(d, treeID, m) } @@ -38,6 +41,9 @@ func (s *Shard) TreeAddByPath(d pilorama.CIDDescriptor, treeID string, attr stri if s.info.Mode.ReadOnly() { return nil, ErrReadOnlyMode } + if s.info.Mode.NoMetabase() { + return nil, ErrDegradedMode + } return s.pilorama.TreeAddByPath(d, treeID, attr, path, meta) } @@ -53,6 +59,9 @@ func (s *Shard) TreeApply(cnr cidSDK.ID, treeID string, m *pilorama.Move, backgr if s.info.Mode.ReadOnly() { return ErrReadOnlyMode } + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } return s.pilorama.TreeApply(cnr, treeID, m, backgroundSync) } @@ -61,6 +70,13 @@ func (s *Shard) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path [] if s.pilorama == nil { return nil, ErrPiloramaDisabled } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return nil, ErrDegradedMode + } return s.pilorama.TreeGetByPath(cid, treeID, attr, path, latest) } @@ -69,6 +85,13 @@ func (s *Shard) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) if s.pilorama == nil { return pilorama.Meta{}, 0, ErrPiloramaDisabled } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return pilorama.Meta{}, 0, ErrDegradedMode + } return s.pilorama.TreeGetMeta(cid, treeID, nodeID) } @@ -77,6 +100,13 @@ func (s *Shard) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID pilorama.No if s.pilorama == nil { return nil, ErrPiloramaDisabled } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return nil, ErrDegradedMode + } return s.pilorama.TreeGetChildren(cid, treeID, nodeID) } @@ -85,6 +115,13 @@ func (s *Shard) TreeGetOpLog(cid cidSDK.ID, treeID string, height uint64) (pilor if s.pilorama == nil { return pilorama.Move{}, ErrPiloramaDisabled } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return pilorama.Move{}, ErrDegradedMode + } return s.pilorama.TreeGetOpLog(cid, treeID, height) } @@ -93,6 +130,13 @@ func (s *Shard) TreeDrop(cid cidSDK.ID, treeID string) error { if s.pilorama == nil { return ErrPiloramaDisabled } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } return s.pilorama.TreeDrop(cid, treeID) } @@ -101,6 +145,13 @@ func (s *Shard) TreeList(cid cidSDK.ID) ([]string, error) { if s.pilorama == nil { return nil, ErrPiloramaDisabled } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return nil, ErrDegradedMode + } return s.pilorama.TreeList(cid) } @@ -109,6 +160,13 @@ func (s *Shard) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { if s.pilorama == nil { return false, ErrPiloramaDisabled } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return false, ErrDegradedMode + } return s.pilorama.TreeExists(cid, treeID) } @@ -117,6 +175,16 @@ func (s *Shard) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height ui if s.pilorama == nil { return ErrPiloramaDisabled } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.ReadOnly() { + return ErrReadOnlyMode + } + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } return s.pilorama.TreeUpdateLastSyncHeight(cid, treeID, height) } @@ -125,5 +193,12 @@ func (s *Shard) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) if s.pilorama == nil { return 0, ErrPiloramaDisabled } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return 0, ErrDegradedMode + } return s.pilorama.TreeLastSyncHeight(cid, treeID) } From 23575e1ac0bd0f92debd10abed427d2f184767dc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 14:40:01 +0300 Subject: [PATCH 0036/1943] [#210] policier: Resolve contextcheck linter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 12 ++++++------ pkg/local_object_storage/engine/delete.go | 15 ++++++++------- pkg/local_object_storage/engine/delete_test.go | 3 ++- pkg/local_object_storage/engine/inhume.go | 14 +++++++------- pkg/local_object_storage/engine/inhume_test.go | 5 +++-- pkg/local_object_storage/engine/lock_test.go | 18 +++++++++--------- pkg/local_object_storage/shard/control_test.go | 4 ++-- pkg/local_object_storage/shard/inhume.go | 4 ++-- pkg/local_object_storage/shard/inhume_test.go | 3 ++- pkg/local_object_storage/shard/lock_test.go | 12 ++++++------ pkg/local_object_storage/shard/metrics_test.go | 4 ++-- pkg/services/control/server/gc.go | 4 ++-- pkg/services/object/put/local.go | 6 +++--- pkg/services/policer/check.go | 5 ++--- pkg/services/policer/policer.go | 3 ++- 15 files changed, 58 insertions(+), 54 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 3b6bdcc7d..ff4335ff9 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -253,11 +253,11 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *reputati policerconfig.HeadTimeout(c.appCfg), ), policer.WithReplicator(c.replicator), - policer.WithRedundantCopyCallback(func(addr oid.Address) { + policer.WithRedundantCopyCallback(func(ctx context.Context, addr oid.Address) { var inhumePrm engine.InhumePrm inhumePrm.MarkAsGarbage(addr) - _, err := ls.Inhume(inhumePrm) + _, err := ls.Inhume(ctx, inhumePrm) if err != nil { c.log.Warn("could not inhume mark redundant copy as garbage", zap.String("error", err.Error()), @@ -620,8 +620,8 @@ func (e engineWithNotifications) IsLocked(address oid.Address) (bool, error) { return e.base.IsLocked(address) } -func (e engineWithNotifications) Delete(tombstone oid.Address, toDelete []oid.ID) error { - return e.base.Delete(tombstone, toDelete) +func (e engineWithNotifications) Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error { + return e.base.Delete(ctx, tombstone, toDelete) } func (e engineWithNotifications) Lock(locker oid.Address, toLock []oid.ID) error { @@ -657,7 +657,7 @@ func (e engineWithoutNotifications) IsLocked(address oid.Address) (bool, error) return e.engine.IsLocked(address) } -func (e engineWithoutNotifications) Delete(tombstone oid.Address, toDelete []oid.ID) error { +func (e engineWithoutNotifications) Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error { var prm engine.InhumePrm addrs := make([]oid.Address, len(toDelete)) @@ -668,7 +668,7 @@ func (e engineWithoutNotifications) Delete(tombstone oid.Address, toDelete []oid prm.WithTarget(tombstone, addrs...) - _, err := e.engine.Inhume(prm) + _, err := e.engine.Inhume(ctx, prm) return err } diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index a49b1e8fa..6ea5728bb 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -1,6 +1,7 @@ package engine import ( + "context" "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -44,16 +45,16 @@ func (p *DeletePrm) WithForceRemoval() { // NOTE: Marks any object to be deleted (despite any prohibitions // on operations with that object) if WithForceRemoval option has // been provided. -func (e *StorageEngine) Delete(prm DeletePrm) (res DeleteRes, err error) { +func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRes, err error) { err = e.execIfNotBlocked(func() error { - res, err = e.delete(prm) + res, err = e.delete(ctx, prm) return err }) return } -func (e *StorageEngine) delete(prm DeletePrm) (DeleteRes, error) { +func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { if e.metrics != nil { defer elapsed(e.metrics.AddDeleteDuration)() } @@ -95,7 +96,7 @@ func (e *StorageEngine) delete(prm DeletePrm) (DeleteRes, error) { shPrm.ForceRemoval() } - _, err = sh.Inhume(shPrm) + _, err = sh.Inhume(ctx, shPrm) if err != nil { e.reportShardError(sh, "could not inhume object in shard", err) @@ -113,13 +114,13 @@ func (e *StorageEngine) delete(prm DeletePrm) (DeleteRes, error) { } if splitInfo != nil { - e.deleteChildren(prm.addr, prm.forceRemoval, splitInfo.SplitID()) + e.deleteChildren(ctx, prm.addr, prm.forceRemoval, splitInfo.SplitID()) } return DeleteRes{}, nil } -func (e *StorageEngine) deleteChildren(addr oid.Address, force bool, splitID *objectSDK.SplitID) { +func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, force bool, splitID *objectSDK.SplitID) { var fs objectSDK.SearchFilters fs.AddSplitIDFilter(objectSDK.MatchStringEqual, splitID) @@ -144,7 +145,7 @@ func (e *StorageEngine) deleteChildren(addr oid.Address, force bool, splitID *ob for _, addr := range res.AddressList() { inhumePrm.MarkAsGarbage(addr) - _, err = sh.Inhume(inhumePrm) + _, err = sh.Inhume(ctx, inhumePrm) if err != nil { e.log.Debug("could not inhume object in shard", zap.Stringer("addr", addr), diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index ccba3d903..8a4e6a7fa 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -1,6 +1,7 @@ package engine import ( + "context" "os" "testing" @@ -78,7 +79,7 @@ func TestDeleteBigObject(t *testing.T) { deletePrm.WithForceRemoval() deletePrm.WithAddress(addrParent) - _, err := e.Delete(deletePrm) + _, err := e.Delete(context.Background(), deletePrm) require.NoError(t, err) checkGetError(t, e, addrParent, &apistatus.ObjectNotFound{}) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 680e773c6..2ecca5256 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -60,16 +60,16 @@ var errInhumeFailure = errors.New("inhume operation failed") // with that object) if WithForceRemoval option has been provided. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) Inhume(prm InhumePrm) (res InhumeRes, err error) { +func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err error) { err = e.execIfNotBlocked(func() error { - res, err = e.inhume(prm) + res, err = e.inhume(ctx, prm) return err }) return } -func (e *StorageEngine) inhume(prm InhumePrm) (InhumeRes, error) { +func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { if e.metrics != nil { defer elapsed(e.metrics.AddInhumeDuration)() } @@ -98,12 +98,12 @@ func (e *StorageEngine) inhume(prm InhumePrm) (InhumeRes, error) { shPrm.MarkAsGarbage(prm.addrs[i]) } - ok, err := e.inhumeAddr(prm.addrs[i], shPrm, true) + ok, err := e.inhumeAddr(ctx, prm.addrs[i], shPrm, true) if err != nil { return InhumeRes{}, err } if !ok { - ok, err := e.inhumeAddr(prm.addrs[i], shPrm, false) + ok, err := e.inhumeAddr(ctx, prm.addrs[i], shPrm, false) if err != nil { return InhumeRes{}, err } else if !ok { @@ -116,7 +116,7 @@ func (e *StorageEngine) inhume(prm InhumePrm) (InhumeRes, error) { } // Returns ok if object was inhumed during this invocation or before. -func (e *StorageEngine) inhumeAddr(addr oid.Address, prm shard.InhumePrm, checkExists bool) (bool, error) { +func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm shard.InhumePrm, checkExists bool) (bool, error) { root := false var errLocked apistatus.ObjectLocked var existPrm shard.ExistsPrm @@ -154,7 +154,7 @@ func (e *StorageEngine) inhumeAddr(addr oid.Address, prm shard.InhumePrm, checkE } } - _, err := sh.Inhume(prm) + _, err := sh.Inhume(ctx, prm) if err != nil { switch { case errors.As(err, &errLocked): diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index b001c6df9..e3150c17f 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -1,6 +1,7 @@ package engine import ( + "context" "os" "testing" @@ -47,7 +48,7 @@ func TestStorageEngine_Inhume(t *testing.T) { var inhumePrm InhumePrm inhumePrm.WithTarget(tombstoneID, object.AddressOf(parent)) - _, err = e.Inhume(inhumePrm) + _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) addrs, err := Select(e, cnr, fs) @@ -75,7 +76,7 @@ func TestStorageEngine_Inhume(t *testing.T) { var inhumePrm InhumePrm inhumePrm.WithTarget(tombstoneID, object.AddressOf(parent)) - _, err = e.Inhume(inhumePrm) + _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) addrs, err := Select(e, cnr, fs) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index f222ffe62..1014b2146 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -111,7 +111,7 @@ func TestLockUserScenario(t *testing.T) { var inhumePrm InhumePrm inhumePrm.WithTarget(tombAddr, objAddr) - _, err = e.Inhume(inhumePrm) + _, err = e.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) // 4. @@ -124,7 +124,7 @@ func TestLockUserScenario(t *testing.T) { inhumePrm.WithTarget(tombForLockAddr, lockerAddr) - _, err = e.Inhume(inhumePrm) + _, err = e.Inhume(context.Background(), inhumePrm) require.ErrorIs(t, err, meta.ErrLockObjectRemoval) // 5. @@ -135,7 +135,7 @@ func TestLockUserScenario(t *testing.T) { inhumePrm.WithTarget(tombAddr, objAddr) - _, err = e.Inhume(inhumePrm) + _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) } @@ -192,7 +192,7 @@ func TestLockExpiration(t *testing.T) { var inhumePrm InhumePrm inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj)) - _, err = e.Inhume(inhumePrm) + _, err = e.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) // 3. @@ -205,7 +205,7 @@ func TestLockExpiration(t *testing.T) { // 4. inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj)) - _, err = e.Inhume(inhumePrm) + _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) } @@ -259,12 +259,12 @@ func TestLockForceRemoval(t *testing.T) { var inhumePrm InhumePrm inhumePrm.MarkAsGarbage(objectcore.AddressOf(obj)) - _, err = e.Inhume(inhumePrm) + _, err = e.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj)) - _, err = e.Inhume(inhumePrm) + _, err = e.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) // 4. @@ -272,12 +272,12 @@ func TestLockForceRemoval(t *testing.T) { deletePrm.WithAddress(objectcore.AddressOf(lock)) deletePrm.WithForceRemoval() - _, err = e.Delete(deletePrm) + _, err = e.Delete(context.Background(), deletePrm) require.NoError(t, err) // 5. inhumePrm.MarkAsGarbage(objectcore.AddressOf(obj)) - _, err = e.Inhume(inhumePrm) + _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) } diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 6886438e0..fec268350 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -269,7 +269,7 @@ func TestRefillMetabase(t *testing.T) { var inhumePrm InhumePrm inhumePrm.SetTarget(object.AddressOf(tombObj), tombMembers...) - _, err = sh.Inhume(inhumePrm) + _, err = sh.Inhume(context.Background(), inhumePrm) require.NoError(t, err) var headPrm HeadPrm @@ -322,7 +322,7 @@ func TestRefillMetabase(t *testing.T) { var prm InhumePrm prm.MarkAsGarbage(addr) - _, err := sh.Inhume(prm) + _, err := sh.Inhume(context.Background(), prm) require.ErrorAs(t, err, new(apistatus.ObjectLocked), "object %s should be locked", locked[i]) } diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 199bb8b3f..40a5bf22e 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -60,7 +60,7 @@ var ErrLockObjectRemoval = meta.ErrLockObjectRemoval // if at least one object is locked. // // Returns ErrReadOnlyMode error if shard is in "read-only" mode. -func (s *Shard) Inhume(prm InhumePrm) (InhumeRes, error) { +func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { s.m.RLock() if s.info.Mode.ReadOnly() { @@ -119,7 +119,7 @@ func (s *Shard) Inhume(prm InhumePrm) (InhumeRes, error) { } if deletedLockObjs := res.DeletedLockObjects(); len(deletedLockObjs) != 0 { - s.deletedLockCallBack(context.Background(), deletedLockObjs) + s.deletedLockCallBack(ctx, deletedLockObjs) } return InhumeRes{}, nil diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index fca613941..191afab01 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -1,6 +1,7 @@ package shard_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -47,7 +48,7 @@ func testShardInhume(t *testing.T, hasWriteCache bool) { _, err = testGet(t, sh, getPrm, hasWriteCache) require.NoError(t, err) - _, err = sh.Inhume(inhPrm) + _, err = sh.Inhume(context.Background(), inhPrm) require.NoError(t, err) _, err = sh.Get(getPrm) diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 4e23e8c37..995aa1473 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -93,11 +93,11 @@ func TestShard_Lock(t *testing.T) { var inhumePrm shard.InhumePrm inhumePrm.SetTarget(objectcore.AddressOf(ts), objectcore.AddressOf(obj)) - _, err = sh.Inhume(inhumePrm) + _, err = sh.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) inhumePrm.MarkAsGarbage(objectcore.AddressOf(obj)) - _, err = sh.Inhume(inhumePrm) + _, err = sh.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) }) @@ -107,11 +107,11 @@ func TestShard_Lock(t *testing.T) { var inhumePrm shard.InhumePrm inhumePrm.SetTarget(objectcore.AddressOf(ts), objectcore.AddressOf(lock)) - _, err = sh.Inhume(inhumePrm) + _, err = sh.Inhume(context.Background(), inhumePrm) require.Error(t, err) inhumePrm.MarkAsGarbage(objectcore.AddressOf(lock)) - _, err = sh.Inhume(inhumePrm) + _, err = sh.Inhume(context.Background(), inhumePrm) require.Error(t, err) }) @@ -120,7 +120,7 @@ func TestShard_Lock(t *testing.T) { inhumePrm.MarkAsGarbage(objectcore.AddressOf(lock)) inhumePrm.ForceRemoval() - _, err = sh.Inhume(inhumePrm) + _, err = sh.Inhume(context.Background(), inhumePrm) require.NoError(t, err) // it should be possible to remove @@ -129,7 +129,7 @@ func TestShard_Lock(t *testing.T) { inhumePrm = shard.InhumePrm{} inhumePrm.MarkAsGarbage(objectcore.AddressOf(obj)) - _, err = sh.Inhume(inhumePrm) + _, err = sh.Inhume(context.Background(), inhumePrm) require.NoError(t, err) // check that object has been removed diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 959aebf8d..ba46881e2 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -127,7 +127,7 @@ func TestCounters(t *testing.T) { for i := 0; i < inhumedNumber; i++ { prm.MarkAsGarbage(objectcore.AddressOf(oo[i])) - _, err := sh.Inhume(prm) + _, err := sh.Inhume(context.Background(), prm) require.NoError(t, err) } @@ -149,7 +149,7 @@ func TestCounters(t *testing.T) { inhumedNumber := int(phy / 4) prm.SetTarget(ts, addrFromObjs(oo[:inhumedNumber])...) - _, err := sh.Inhume(prm) + _, err := sh.Inhume(context.Background(), prm) require.NoError(t, err) require.Equal(t, phy, mm.objCounters[physical]) diff --git a/pkg/services/control/server/gc.go b/pkg/services/control/server/gc.go index 7fa0a060f..7912d4e3e 100644 --- a/pkg/services/control/server/gc.go +++ b/pkg/services/control/server/gc.go @@ -17,7 +17,7 @@ import ( // // If some address is not a valid object address in a binary format, an error returns. // If request is unsigned or signed by disallowed key, permission error returns. -func (s *Server) DropObjects(_ context.Context, req *control.DropObjectsRequest) (*control.DropObjectsResponse, error) { +func (s *Server) DropObjects(ctx context.Context, req *control.DropObjectsRequest) (*control.DropObjectsResponse, error) { // verify request if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) @@ -42,7 +42,7 @@ func (s *Server) DropObjects(_ context.Context, req *control.DropObjectsRequest) prm.WithAddress(addrList[i]) //nolint: contextcheck - _, err := s.s.Delete(prm) + _, err := s.s.Delete(ctx, prm) if err != nil && firstErr == nil { firstErr = err } diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index 12e3a2eee..2e6a496f3 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -17,7 +17,7 @@ type ObjectStorage interface { Put(*object.Object) error // Delete must delete passed objects // and return any appeared error. - Delete(tombstone oid.Address, toDelete []oid.ID) error + Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error // Lock must lock passed objects // and return any appeared error. Lock(locker oid.Address, toLock []oid.ID) error @@ -39,10 +39,10 @@ func (t *localTarget) WriteObject(obj *object.Object, meta objectCore.ContentMet return nil } -func (t *localTarget) Close(_ context.Context) (*transformer.AccessIdentifiers, error) { +func (t *localTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { switch t.meta.Type() { case object.TypeTombstone: - err := t.storage.Delete(objectCore.AddressOf(t.obj), t.meta.Objects()) + err := t.storage.Delete(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) if err != nil { return nil, fmt.Errorf("could not delete objects from tombstone locally: %w", err) } diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 7d4b714f3..f393eb570 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -81,8 +81,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add prm.MarkAsGarbage(addrWithType.Address) prm.WithForceRemoval() - //nolint: contextcheck - _, err := p.jobQueue.localStorage.Inhume(prm) + _, err := p.jobQueue.localStorage.Inhume(ctx, prm) if err != nil { p.log.Error("could not inhume object with missing container", zap.Stringer("cid", idCnr), @@ -134,7 +133,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add zap.Stringer("object", addr), ) - p.cbRedundantCopy(addr) + p.cbRedundantCopy(ctx, addr) } } diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index dba0c1cba..541ab599c 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -1,6 +1,7 @@ package policer import ( + "context" "sync" "time" @@ -63,7 +64,7 @@ type Option func(*cfg) // RedundantCopyCallback is a callback to pass // the redundant local copy of the object. -type RedundantCopyCallback func(oid.Address) +type RedundantCopyCallback func(context.Context, oid.Address) type cfg struct { headTimeout time.Duration From 080be5cfcdc27149c56eef4ed201b4a0dac93943 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 14:57:44 +0300 Subject: [PATCH 0037/1943] [#210] policier: Refactor object placement Resolve containedctx and contextcheck linters. Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/check.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index f393eb570..6a2d9d327 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -105,9 +105,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add return } - c := &processPlacementContext{ - Context: ctx, - } + c := &placementRequirements{} var numOfContainerNodes int for i := range nn { @@ -124,8 +122,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add default: } - //nolint: contextcheck - p.processNodes(c, addrWithType, nn[i], policy.ReplicaNumberByIndex(i), checkedNodes) + p.processNodes(ctx, c, addrWithType, nn[i], policy.ReplicaNumberByIndex(i), checkedNodes) } if !c.needLocalCopy && c.removeLocalCopy { @@ -137,10 +134,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add } } -// nolint: containedctx -type processPlacementContext struct { - context.Context - +type placementRequirements struct { // needLocalCopy is true if the current node must store an object according to the storage policy. needLocalCopy bool // removeLocalCopy is true if all copies are stored according to the storage policy @@ -149,7 +143,7 @@ type processPlacementContext struct { } // nolint: funlen -func (p *Policer) processNodes(ctx *processPlacementContext, addrWithType objectcore.AddressWithType, +func (p *Policer) processNodes(ctx context.Context, requirements *placementRequirements, addrWithType objectcore.AddressWithType, nodes []netmap.NodeInfo, shortage uint32, checkedNodes *nodeCache) { addr := addrWithType.Address typ := addrWithType.Type @@ -189,7 +183,7 @@ func (p *Policer) processNodes(ctx *processPlacementContext, addrWithType object } if p.netmapKeys.IsLocalKey(nodes[i].PublicKey()) { - ctx.needLocalCopy = true + requirements.needLocalCopy = true shortage-- } else if nodes[i].IsMaintenance() { @@ -253,7 +247,7 @@ func (p *Policer) processNodes(ctx *processPlacementContext, addrWithType object zap.Int("count", uncheckedCopies)) } else if uncheckedCopies == 0 { // Safe to remove: checked all copies, shortage == 0. - ctx.removeLocalCopy = true + requirements.removeLocalCopy = true } } From d6486d172ec4f58004054b96de6a659d5c45cf3d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 15:29:12 +0300 Subject: [PATCH 0038/1943] [#210] policier: Refactor nodes processing Resolve funlen linter for processNodes method. Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/check.go | 44 +++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 6a2d9d327..9cdc4d813 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -13,6 +13,7 @@ import ( apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -142,7 +143,6 @@ type placementRequirements struct { removeLocalCopy bool } -// nolint: funlen func (p *Policer) processNodes(ctx context.Context, requirements *placementRequirements, addrWithType objectcore.AddressWithType, nodes []netmap.NodeInfo, shortage uint32, checkedNodes *nodeCache) { addr := addrWithType.Address @@ -152,21 +152,6 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi // Number of copies that are stored on maintenance nodes. var uncheckedCopies int - handleMaintenance := func(node netmap.NodeInfo) { - // consider remote nodes under maintenance as problem OK. Such - // nodes MAY not respond with object, however, this is how we - // prevent spam with new replicas. - // However, additional copies should not be removed in this case, - // because we can remove the only copy this way. - checkedNodes.submitReplicaHolder(node) - shortage-- - uncheckedCopies++ - - p.log.Debug("consider node under maintenance as OK", - zap.String("node", netmap.StringifyPublicKey(node)), - ) - } - if typ == object.TypeLock { // all nodes of a container must store the `LOCK` objects // for correct object removal protection: @@ -187,7 +172,7 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi shortage-- } else if nodes[i].IsMaintenance() { - handleMaintenance(nodes[i]) + shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) } else { if status := checkedNodes.processStatus(nodes[i]); status >= 0 { if status == 0 { @@ -212,7 +197,7 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi } if isClientErrMaintenance(err) { - handleMaintenance(nodes[i]) + shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) } else if err != nil { p.log.Error("receive object header to check policy compliance", zap.Stringer("object", addr), @@ -228,6 +213,29 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi i-- } + p.handleProcessNodesResult(ctx, addr, requirements, nodes, checkedNodes, shortage, uncheckedCopies) +} + +// handleMaintenance handles node in maintenance mode and returns new shortage and uncheckedCopies values +// +// consider remote nodes under maintenance as problem OK. Such +// nodes MAY not respond with object, however, this is how we +// prevent spam with new replicas. +// However, additional copies should not be removed in this case, +// because we can remove the only copy this way. +func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes *nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { + checkedNodes.submitReplicaHolder(node) + shortage-- + uncheckedCopies++ + + p.log.Debug("consider node under maintenance as OK", + zap.String("node", netmap.StringifyPublicKey(node)), + ) + return shortage, uncheckedCopies +} + +func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address, requirements *placementRequirements, + nodes []netmap.NodeInfo, checkedNodes *nodeCache, shortage uint32, uncheckedCopies int) { if shortage > 0 { p.log.Debug("shortage of object copies detected", zap.Stringer("object", addr), From 6f7b6a8813e10c8516c29efa0eb71a48af14c3e7 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 30 Mar 2023 14:58:20 +0300 Subject: [PATCH 0039/1943] [#116] node: Improve shard/engine construction in tests * Introduce testEngineWrapper that can be constructed with different options Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- .../engine/control_test.go | 30 ++--- .../engine/delete_test.go | 2 +- .../engine/engine_test.go | 113 ++++++++++-------- pkg/local_object_storage/engine/error_test.go | 54 +++++---- .../engine/evacuate_test.go | 39 +++--- pkg/local_object_storage/engine/head_test.go | 2 +- .../engine/inhume_test.go | 4 +- pkg/local_object_storage/engine/list_test.go | 2 +- pkg/local_object_storage/engine/lock_test.go | 64 ++++++---- .../engine/shards_test.go | 7 +- 10 files changed, 173 insertions(+), 144 deletions(-) diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index f954d906a..2c44eb169 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -192,7 +192,8 @@ func testEngineFailInitAndReload(t *testing.T, errOnAdd bool, opts []shard.Optio } func TestExecBlocks(t *testing.T) { - e := testNewEngineWithShardNum(t, 2) // number doesn't matter in this test, 2 is several but not many + e := testNewEngine(t).setShardsNum(t, 2).engine // number doesn't matter in this test, 2 is several but not many + t.Cleanup(func() { os.RemoveAll(t.Name()) }) @@ -314,25 +315,26 @@ func TestReload(t *testing.T) { // engineWithShards creates engine with specified number of shards. Returns // slice of paths to their metabase and the engine. -// TODO: #1776 unify engine construction in tests func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []string) { addPath := filepath.Join(path, "add") currShards := make([]string, 0, num) - e := New() - for i := 0; i < num; i++ { - id, err := e.AddShard( - shard.WithBlobStorOptions( - blobstor.WithStorages(newStorages(filepath.Join(addPath, strconv.Itoa(i)), errSmallSize))), - shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(addPath, fmt.Sprintf("%d.metabase", i))), - meta.WithPermissions(0700), - meta.WithEpochState(epochState{}), - ), - ) - require.NoError(t, err) + te := testNewEngine(t). + setShardsNumAdditionalOpts(t, num, func(id int) []shard.Option { + return []shard.Option{ + shard.WithBlobStorOptions( + blobstor.WithStorages(newStorages(filepath.Join(addPath, strconv.Itoa(id)), errSmallSize))), + shard.WithMetaBaseOptions( + meta.WithPath(filepath.Join(addPath, fmt.Sprintf("%d.metabase", id))), + meta.WithPermissions(0700), + meta.WithEpochState(epochState{}), + ), + } + }) + e, ids := te.engine, te.shardIDs + for _, id := range ids { currShards = append(currShards, calculateShardID(e.shards[id.String()].DumpInfo())) } diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 8a4e6a7fa..54d73cee8 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -54,7 +54,7 @@ func TestDeleteBigObject(t *testing.T) { s2 := testNewShard(t, 2) s3 := testNewShard(t, 3) - e := testNewEngineWithShards(t, s1, s2, s3) + e := testNewEngine(t).setInitializedShards(t, s1, s2, s3).engine e.log = &logger.Logger{Logger: zaptest.NewLogger(t)} defer e.Close() diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 83dbcd093..ddaf88d18 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/atomic" "go.uber.org/zap" + "go.uber.org/zap/zaptest" ) type epochState struct{} @@ -50,7 +51,7 @@ func benchmarkExists(b *testing.B, shardNum int) { shards[i] = testNewShard(b, i) } - e := testNewEngineWithShards(b, shards...) + e := testNewEngine(b).setInitializedShards(b, shards...).engine b.Cleanup(func() { _ = e.Close() _ = os.RemoveAll(b.Name()) @@ -75,24 +76,68 @@ func benchmarkExists(b *testing.B, shardNum int) { } } -func testNewEngineWithShards(t testing.TB, shards ...*shard.Shard) *StorageEngine { - engine := New() +type testEngineWrapper struct { + engine *StorageEngine + shardIDs []*shard.ID +} +func testNewEngine(t testing.TB, opts ...Option) *testEngineWrapper { + engine := New(WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)})) + for _, opt := range opts { + opt(engine.cfg) + } + return &testEngineWrapper{ + engine: engine, + } +} + +func (te *testEngineWrapper) setInitializedShards(t testing.TB, shards ...*shard.Shard) *testEngineWrapper { for _, s := range shards { pool, err := ants.NewPool(10, ants.WithNonblocking(true)) require.NoError(t, err) - engine.shards[s.ID().String()] = hashedShard{ + te.engine.shards[s.ID().String()] = hashedShard{ shardWrapper: shardWrapper{ errorCount: atomic.NewUint32(0), Shard: s, }, hash: hrw.Hash([]byte(s.ID().String())), } - engine.shardPools[s.ID().String()] = pool + te.engine.shardPools[s.ID().String()] = pool + te.shardIDs = append(te.shardIDs, s.ID()) + } + return te +} + +func (te *testEngineWrapper) setShardsNum(t testing.TB, num int) *testEngineWrapper { + shards := make([]*shard.Shard, 0, num) + + for i := 0; i < num; i++ { + shards = append(shards, testNewShard(t, i)) } - return engine + return te.setInitializedShards(t, shards...) +} + +func (te *testEngineWrapper) setShardsNumOpts(t testing.TB, num int, shardOpts func(id int) []shard.Option) *testEngineWrapper { + for i := 0; i < num; i++ { + opts := shardOpts(i) + id, err := te.engine.AddShard(opts...) + require.NoError(t, err) + te.shardIDs = append(te.shardIDs, id) + } + return te +} + +func (te *testEngineWrapper) setShardsNumAdditionalOpts(t testing.TB, num int, shardOpts func(id int) []shard.Option) *testEngineWrapper { + for i := 0; i < num; i++ { + defaultOpts := testDefaultShardOptions(t, i) + opts := append(defaultOpts, shardOpts(i)...) + id, err := te.engine.AddShard(opts...) + require.NoError(t, err) + te.shardIDs = append(te.shardIDs, id) + } + return te } func newStorages(root string, smallSize uint64) []blobstor.SubStorage { @@ -145,8 +190,17 @@ func testNewShard(t testing.TB, id int) *shard.Shard { sid, err := generateShardID() require.NoError(t, err) - s := shard.New( - shard.WithID(sid), + shardOpts := append([]shard.Option{shard.WithID(sid)}, testDefaultShardOptions(t, id)...) + s := shard.New(shardOpts...) + + require.NoError(t, s.Open()) + require.NoError(t, s.Init(context.Background())) + + return s +} + +func testDefaultShardOptions(t testing.TB, id int) []shard.Option { + return []shard.Option{ shard.WithLogger(&logger.Logger{Logger: zap.L()}), shard.WithBlobStorOptions( blobstor.WithStorages( @@ -157,46 +211,5 @@ func testNewShard(t testing.TB, id int) *shard.Shard { meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.metabase", id))), meta.WithPermissions(0700), meta.WithEpochState(epochState{}), - )) - - require.NoError(t, s.Open()) - require.NoError(t, s.Init(context.Background())) - - return s -} - -func testEngineFromShardOpts(t *testing.T, num int, extraOpts []shard.Option) *StorageEngine { - engine := New() - for i := 0; i < num; i++ { - _, err := engine.AddShard(append([]shard.Option{ - shard.WithBlobStorOptions( - blobstor.WithStorages( - newStorages(filepath.Join(t.Name(), fmt.Sprintf("blobstor%d", i)), - 1<<20)), - ), - shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("metabase%d", i))), - meta.WithPermissions(0700), - meta.WithEpochState(epochState{}), - ), - shard.WithPiloramaOptions( - pilorama.WithPath(filepath.Join(t.Name(), fmt.Sprintf("pilorama%d", i)))), - }, extraOpts...)...) - require.NoError(t, err) - } - - require.NoError(t, engine.Open()) - require.NoError(t, engine.Init(context.Background())) - - return engine -} - -func testNewEngineWithShardNum(t *testing.T, num int) *StorageEngine { - shards := make([]*shard.Shard, 0, num) - - for i := 0; i < num; i++ { - shards = append(shards, testNewShard(t, i)) - } - - return testNewEngineWithShards(t, shards...) + )} } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 8a32c8b69..c9b194f6f 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -48,37 +48,39 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) t.Cleanup(func() { _ = os.RemoveAll(dir) }) } - e := New( - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), - WithShardPoolSize(1), - WithErrorThreshold(errThreshold)) - var testShards [2]*testShard - for i := range testShards { - storages, smallFileStorage, largeFileStorage := newTestStorages(filepath.Join(dir, strconv.Itoa(i)), errSmallSize) - id, err := e.AddShard( - shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), - shard.WithBlobStorOptions(blobstor.WithStorages(storages)), - shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", i))), - meta.WithPermissions(0700), - meta.WithEpochState(epochState{}), - ), - shard.WithPiloramaOptions( - pilorama.WithPath(filepath.Join(dir, fmt.Sprintf("%d.pilorama", i))), - pilorama.WithPerm(0700))) - require.NoError(t, err) - - testShards[i] = &testShard{ - id: id, - smallFileStorage: smallFileStorage, - largeFileStorage: largeFileStorage, - } - } + te := testNewEngine(t, + WithShardPoolSize(1), + WithErrorThreshold(errThreshold), + ). + setShardsNumOpts(t, 2, func(id int) []shard.Option { + storages, smallFileStorage, largeFileStorage := newTestStorages(filepath.Join(dir, strconv.Itoa(id)), errSmallSize) + testShards[id] = &testShard{ + smallFileStorage: smallFileStorage, + largeFileStorage: largeFileStorage, + } + return []shard.Option{ + shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + shard.WithBlobStorOptions(blobstor.WithStorages(storages)), + shard.WithMetaBaseOptions( + meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", id))), + meta.WithPermissions(0700), + meta.WithEpochState(epochState{}), + ), + shard.WithPiloramaOptions( + pilorama.WithPath(filepath.Join(dir, fmt.Sprintf("%d.pilorama", id))), + pilorama.WithPerm(0700)), + } + }) + e := te.engine require.NoError(t, e.Open()) require.NoError(t, e.Init(context.Background())) + for i, id := range te.shardIDs { + testShards[i].id = id + } + return &testEngine{ ng: e, dir: dir, diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 04d68d2e4..51abc4b1c 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -29,28 +29,23 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng require.NoError(t, err) t.Cleanup(func() { _ = os.RemoveAll(dir) }) - e := New( - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), - WithShardPoolSize(1)) - - ids := make([]*shard.ID, shardNum) - - for i := range ids { - ids[i], err = e.AddShard( - shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), - shard.WithBlobStorOptions( - blobstor.WithStorages([]blobstor.SubStorage{{ - Storage: fstree.New( - fstree.WithPath(filepath.Join(dir, strconv.Itoa(i))), - fstree.WithDepth(1)), - }})), - shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", i))), - meta.WithPermissions(0700), - meta.WithEpochState(epochState{}), - )) - require.NoError(t, err) - } + te := testNewEngine(t, WithShardPoolSize(1)). + setShardsNumOpts(t, shardNum, func(id int) []shard.Option { + return []shard.Option{ + shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + shard.WithBlobStorOptions( + blobstor.WithStorages([]blobstor.SubStorage{{ + Storage: fstree.New( + fstree.WithPath(filepath.Join(dir, strconv.Itoa(id))), + fstree.WithDepth(1)), + }})), + shard.WithMetaBaseOptions( + meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", id))), + meta.WithPermissions(0700), + meta.WithEpochState(epochState{})), + } + }) + e, ids := te.engine, te.shardIDs require.NoError(t, e.Open()) require.NoError(t, e.Init(context.Background())) diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go index 1ddfedc5c..e2a1edc98 100644 --- a/pkg/local_object_storage/engine/head_test.go +++ b/pkg/local_object_storage/engine/head_test.go @@ -44,7 +44,7 @@ func TestHeadRaw(t *testing.T) { s1 := testNewShard(t, 1) s2 := testNewShard(t, 2) - e := testNewEngineWithShards(t, s1, s2) + e := testNewEngine(t).setInitializedShards(t, s1, s2).engine defer e.Close() var putPrmLeft shard.PutPrm diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index e3150c17f..4f8c96b99 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -39,7 +39,7 @@ func TestStorageEngine_Inhume(t *testing.T) { link.SetSplitID(splitID) t.Run("delete small object", func(t *testing.T) { - e := testNewEngineWithShardNum(t, 1) + e := testNewEngine(t).setShardsNum(t, 1).engine defer e.Close() err := Put(e, parent) @@ -60,7 +60,7 @@ func TestStorageEngine_Inhume(t *testing.T) { s1 := testNewShard(t, 1) s2 := testNewShard(t, 2) - e := testNewEngineWithShards(t, s1, s2) + e := testNewEngine(t).setInitializedShards(t, s1, s2).engine defer e.Close() var putChild shard.PutPrm diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index ad0eb1911..1261de9d4 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -16,7 +16,7 @@ import ( func TestListWithCursor(t *testing.T) { s1 := testNewShard(t, 1) s2 := testNewShard(t, 2) - e := testNewEngineWithShards(t, s1, s2) + e := testNewEngine(t).setInitializedShards(t, s1, s2).engine t.Cleanup(func() { e.Close() diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 1014b2146..fd3b04ef0 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -45,15 +45,21 @@ func TestLockUserScenario(t *testing.T) { tombForLockID := oidtest.ID() tombObj.SetID(tombForLockID) - e := testEngineFromShardOpts(t, 2, []shard.Option{ - shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { - pool, err := ants.NewPool(sz) - require.NoError(t, err) + testEngine := testNewEngine(t). + setShardsNumAdditionalOpts(t, 2, func(id int) []shard.Option { + return []shard.Option{ + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) - return pool - }), - shard.WithTombstoneSource(tss{lockerExpiresAfter}), - }) + return pool + }), + shard.WithTombstoneSource(tss{lockerExpiresAfter}), + } + }) + e := testEngine.engine + require.NoError(t, e.Open()) + require.NoError(t, e.Init(context.Background())) t.Cleanup(func() { _ = e.Close() @@ -146,14 +152,20 @@ func TestLockExpiration(t *testing.T) { // 3. lock expiration epoch is coming // 4. after some delay the object is not locked anymore - e := testEngineFromShardOpts(t, 2, []shard.Option{ - shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { - pool, err := ants.NewPool(sz) - require.NoError(t, err) + testEngine := testNewEngine(t). + setShardsNumAdditionalOpts(t, 2, func(id int) []shard.Option { + return []shard.Option{ + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) - return pool - }), - }) + return pool + }), + } + }) + e := testEngine.engine + require.NoError(t, e.Open()) + require.NoError(t, e.Init(context.Background())) t.Cleanup(func() { _ = e.Close() @@ -218,16 +230,20 @@ func TestLockForceRemoval(t *testing.T) { // 5. the object is not locked anymore var e *StorageEngine - e = testEngineFromShardOpts(t, 2, []shard.Option{ - shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { - pool, err := ants.NewPool(sz) - require.NoError(t, err) - - return pool - }), - shard.WithDeletedLockCallback(e.processDeletedLocks), - }) + e = testNewEngine(t). + setShardsNumAdditionalOpts(t, 2, func(id int) []shard.Option { + return []shard.Option{ + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) + return pool + }), + shard.WithDeletedLockCallback(e.processDeletedLocks), + } + }).engine + require.NoError(t, e.Open()) + require.NoError(t, e.Init(context.Background())) t.Cleanup(func() { _ = e.Close() _ = os.RemoveAll(t.Name()) diff --git a/pkg/local_object_storage/engine/shards_test.go b/pkg/local_object_storage/engine/shards_test.go index 67a006b5a..1bc0b880c 100644 --- a/pkg/local_object_storage/engine/shards_test.go +++ b/pkg/local_object_storage/engine/shards_test.go @@ -10,7 +10,8 @@ import ( func TestRemoveShard(t *testing.T) { const numOfShards = 6 - e := testNewEngineWithShardNum(t, numOfShards) + te := testNewEngine(t).setShardsNum(t, numOfShards) + e, ids := te.engine, te.shardIDs t.Cleanup(func() { e.Close() os.RemoveAll(t.Name()) @@ -22,12 +23,12 @@ func TestRemoveShard(t *testing.T) { removedNum := numOfShards / 2 mSh := make(map[string]bool, numOfShards) - for i, sh := range e.DumpInfo().Shards { + for i, id := range ids { if i == removedNum { break } - mSh[sh.ID.String()] = true + mSh[id.String()] = true } for id, remove := range mSh { From 279261ace36ce24efbac0cd509b185cea06318d4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Apr 2023 14:52:13 +0300 Subject: [PATCH 0040/1943] [#217] containersvc: Refactor route passing Resolve containedctx for routeCtx. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/container.go | 10 ++--- .../announcement/load/controller/calls.go | 8 ++-- .../load/controller/calls_test.go | 4 +- .../announcement/load/controller/deps.go | 26 ++++++++--- .../announcement/load/controller/util.go | 6 +-- .../announcement/load/route/calls.go | 45 +++++-------------- .../container/announcement/load/route/deps.go | 23 +--------- .../load/route/placement/calls.go | 6 +-- .../announcement/load/route/router.go | 5 ++- .../container/announcement/load/route/util.go | 3 +- 10 files changed, 54 insertions(+), 82 deletions(-) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 45c6e390c..7a88497eb 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -335,7 +335,7 @@ type remoteLoadAnnounceProvider struct { deadEndProvider loadcontroller.WriterProvider } -func (r *remoteLoadAnnounceProvider) InitRemote(srv loadroute.ServerInfo) (loadcontroller.WriterProvider, error) { +func (r *remoteLoadAnnounceProvider) InitRemote(srv loadcontroller.ServerInfo) (loadcontroller.WriterProvider, error) { if srv == nil { return r.deadEndProvider, nil } @@ -366,7 +366,7 @@ type remoteLoadAnnounceWriterProvider struct { client client.Client } -func (p *remoteLoadAnnounceWriterProvider) InitWriter(ctx context.Context) (loadcontroller.Writer, error) { +func (p *remoteLoadAnnounceWriterProvider) InitWriter([]loadcontroller.ServerInfo) (loadcontroller.Writer, error) { return &remoteLoadAnnounceWriter{ client: p.client, }, nil @@ -536,7 +536,7 @@ func (c *usedSpaceService) ExternalAddresses() []string { } func (c *usedSpaceService) AnnounceUsedSpace(ctx context.Context, req *containerV2.AnnounceUsedSpaceRequest) (*containerV2.AnnounceUsedSpaceResponse, error) { - var passedRoute []loadroute.ServerInfo + var passedRoute []loadcontroller.ServerInfo for hdr := req.GetVerificationHeader(); hdr != nil; hdr = hdr.GetOrigin() { passedRoute = append(passedRoute, &containerOnlyKeyRemoteServerInfo{ @@ -550,7 +550,7 @@ func (c *usedSpaceService) AnnounceUsedSpace(ctx context.Context, req *container passedRoute = append(passedRoute, c) - w, err := c.loadWriterProvider.InitWriter(loadroute.NewRouteContext(ctx, passedRoute)) + w, err := c.loadWriterProvider.InitWriter(passedRoute) if err != nil { return nil, fmt.Errorf("could not initialize container's used space writer: %w", err) } @@ -615,7 +615,7 @@ func (l *loadPlacementBuilder) isNodeFromContainerKey(epoch uint64, cnr cid.ID, } func (c *usedSpaceService) processLoadValue(_ context.Context, a containerSDK.SizeEstimation, - route []loadroute.ServerInfo, w loadcontroller.Writer) error { + route []loadcontroller.ServerInfo, w loadcontroller.Writer) error { fromCnr, err := c.loadPlacementBuilder.isNodeFromContainerKey(a.Epoch(), a.Container(), route[0].PublicKey()) if err != nil { return fmt.Errorf("could not verify that the sender belongs to the container: %w", err) diff --git a/pkg/services/container/announcement/load/controller/calls.go b/pkg/services/container/announcement/load/controller/calls.go index fde6913a7..8c5fbeacb 100644 --- a/pkg/services/container/announcement/load/controller/calls.go +++ b/pkg/services/container/announcement/load/controller/calls.go @@ -62,7 +62,7 @@ func (c *announceContext) announce() { ) // initialize iterator over locally collected metrics - metricsIterator, err = c.ctrl.prm.LocalMetrics.InitIterator(c.ctx) + metricsIterator, err = c.ctrl.prm.LocalMetrics.InitIterator() if err != nil { c.log.Debug("could not initialize iterator over locally collected metrics", zap.String("error", err.Error()), @@ -72,7 +72,7 @@ func (c *announceContext) announce() { } // initialize target of local announcements - targetWriter, err := c.ctrl.prm.LocalAnnouncementTarget.InitWriter(c.ctx) + targetWriter, err := c.ctrl.prm.LocalAnnouncementTarget.InitWriter(nil) if err != nil { c.log.Debug("could not initialize announcement accumulator", zap.String("error", err.Error()), @@ -268,7 +268,7 @@ func (c *stopContext) report() { ) // initialize iterator over locally accumulated announcements - localIterator, err = c.ctrl.prm.AnnouncementAccumulator.InitIterator(c.ctx) + localIterator, err = c.ctrl.prm.AnnouncementAccumulator.InitIterator() if err != nil { c.log.Debug("could not initialize iterator over locally accumulated announcements", zap.String("error", err.Error()), @@ -278,7 +278,7 @@ func (c *stopContext) report() { } // initialize final destination of load estimations - resultWriter, err := c.ctrl.prm.ResultReceiver.InitWriter(c.ctx) + resultWriter, err := c.ctrl.prm.ResultReceiver.InitWriter(nil) if err != nil { c.log.Debug("could not initialize result target", zap.String("error", err.Error()), diff --git a/pkg/services/container/announcement/load/controller/calls_test.go b/pkg/services/container/announcement/load/controller/calls_test.go index 8e4a3cedd..4a791f4bd 100644 --- a/pkg/services/container/announcement/load/controller/calls_test.go +++ b/pkg/services/container/announcement/load/controller/calls_test.go @@ -28,7 +28,7 @@ func newTestStorage() *testAnnouncementStorage { } } -func (s *testAnnouncementStorage) InitIterator(context.Context) (loadcontroller.Iterator, error) { +func (s *testAnnouncementStorage) InitIterator() (loadcontroller.Iterator, error) { if s.i != nil { return s.i, nil } @@ -53,7 +53,7 @@ func (s *testAnnouncementStorage) Iterate(f loadcontroller.UsedSpaceFilter, h lo return nil } -func (s *testAnnouncementStorage) InitWriter(context.Context) (loadcontroller.Writer, error) { +func (s *testAnnouncementStorage) InitWriter([]loadcontroller.ServerInfo) (loadcontroller.Writer, error) { if s.w != nil { return s.w, nil } diff --git a/pkg/services/container/announcement/load/controller/deps.go b/pkg/services/container/announcement/load/controller/deps.go index 7f7a270b9..99da8594f 100644 --- a/pkg/services/container/announcement/load/controller/deps.go +++ b/pkg/services/container/announcement/load/controller/deps.go @@ -45,7 +45,7 @@ type IteratorProvider interface { // // Implementations can have different logic for different // contexts, so specific ones may document their own behavior. - InitIterator(context.Context) (Iterator, error) + InitIterator() (Iterator, error) } // Writer describes the interface for storing container.SizeEstimation values. @@ -80,8 +80,24 @@ type WriterProvider interface { // // Initialization problems are reported via error. // If no error was returned, then the Writer must not be nil. - // - // Implementations can have different logic for different - // contexts, so specific ones may document their own behavior. - InitWriter(context.Context) (Writer, error) + InitWriter(route []ServerInfo) (Writer, error) +} + +// ServerInfo describes a set of +// characteristics of a point in a route. +type ServerInfo interface { + // PublicKey returns public key of the node + // from the route in a binary representation. + PublicKey() []byte + + // Iterates over network addresses of the node + // in the route. Breaks iterating on true return + // of the handler. + IterateAddresses(func(string) bool) + + // Returns number of server's network addresses. + NumberOfAddresses() int + + // ExternalAddresses returns external node's addresses. + ExternalAddresses() []string } diff --git a/pkg/services/container/announcement/load/controller/util.go b/pkg/services/container/announcement/load/controller/util.go index fb356393d..223de13ba 100644 --- a/pkg/services/container/announcement/load/controller/util.go +++ b/pkg/services/container/announcement/load/controller/util.go @@ -1,8 +1,6 @@ package loadcontroller import ( - "context" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" ) @@ -17,11 +15,11 @@ type storageWrapper struct { i Iterator } -func (s storageWrapper) InitIterator(context.Context) (Iterator, error) { +func (s storageWrapper) InitIterator() (Iterator, error) { return s.i, nil } -func (s storageWrapper) InitWriter(context.Context) (Writer, error) { +func (s storageWrapper) InitWriter([]ServerInfo) (Writer, error) { return s.w, nil } diff --git a/pkg/services/container/announcement/load/route/calls.go b/pkg/services/container/announcement/load/route/calls.go index 1cdd65911..83c368f57 100644 --- a/pkg/services/container/announcement/load/route/calls.go +++ b/pkg/services/container/announcement/load/route/calls.go @@ -10,26 +10,9 @@ import ( "go.uber.org/zap" ) -// nolint: containedctx -type routeContext struct { - context.Context - - passedRoute []ServerInfo -} - -// NewRouteContext wraps the main context of value passing with its traversal route. -// -// Passing the result to Router.InitWriter method will allow you to continue this route. -func NewRouteContext(ctx context.Context, passed []ServerInfo) context.Context { - return &routeContext{ - Context: ctx, - passedRoute: passed, - } -} - // InitWriter initializes and returns Writer that sends each value to its next route point. // -// If ctx was created by NewRouteContext, then the traversed route is taken into account, +// If route is present, then it is taken into account, // and the value will be sent to its continuation. Otherwise, the route will be laid // from scratch and the value will be sent to its primary point. // @@ -41,22 +24,14 @@ func NewRouteContext(ctx context.Context, passed []ServerInfo) context.Context { // runtime and never returns an error. // // Always returns nil error. -func (r *Router) InitWriter(ctx context.Context) (loadcontroller.Writer, error) { - var ( - routeCtx *routeContext - ok bool - ) - - if routeCtx, ok = ctx.(*routeContext); !ok { - routeCtx = &routeContext{ - Context: ctx, - passedRoute: []ServerInfo{r.localSrvInfo}, - } +func (r *Router) InitWriter(route []loadcontroller.ServerInfo) (loadcontroller.Writer, error) { + if len(route) == 0 { + route = []loadcontroller.ServerInfo{r.localSrvInfo} } return &loadWriter{ router: r, - ctx: routeCtx, + route: route, mRoute: make(map[routeKey]*valuesRoute), mServers: make(map[string]loadcontroller.Writer), }, nil @@ -69,7 +44,7 @@ type routeKey struct { } type valuesRoute struct { - route []ServerInfo + route []loadcontroller.ServerInfo values []container.SizeEstimation } @@ -77,7 +52,7 @@ type valuesRoute struct { type loadWriter struct { router *Router - ctx *routeContext + route []loadcontroller.ServerInfo routeMtx sync.RWMutex mRoute map[routeKey]*valuesRoute @@ -96,11 +71,11 @@ func (w *loadWriter) Put(a container.SizeEstimation) error { routeValues, ok := w.mRoute[key] if !ok { - route, err := w.router.routeBuilder.NextStage(a, w.ctx.passedRoute) + route, err := w.router.routeBuilder.NextStage(a, w.route) if err != nil { return err } else if len(route) == 0 { - route = []ServerInfo{nil} + route = []loadcontroller.ServerInfo{nil} } routeValues = &valuesRoute{ @@ -129,7 +104,7 @@ func (w *loadWriter) Put(a container.SizeEstimation) error { continue // best effort } - remoteWriter, err = provider.InitWriter(w.ctx) + remoteWriter, err = provider.InitWriter(w.route) if err != nil { w.router.log.Debug("could not initialize writer", zap.String("error", err.Error()), diff --git a/pkg/services/container/announcement/load/route/deps.go b/pkg/services/container/announcement/load/route/deps.go index 429cda3eb..b255900f7 100644 --- a/pkg/services/container/announcement/load/route/deps.go +++ b/pkg/services/container/announcement/load/route/deps.go @@ -5,25 +5,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" ) -// ServerInfo describes a set of -// characteristics of a point in a route. -type ServerInfo interface { - // PublicKey returns public key of the node - // from the route in a binary representation. - PublicKey() []byte - - // Iterates over network addresses of the node - // in the route. Breaks iterating on true return - // of the handler. - IterateAddresses(func(string) bool) - - // Returns number of server's network addresses. - NumberOfAddresses() int - - // ExternalAddresses returns external node's addresses. - ExternalAddresses() []string -} - // Builder groups methods to route values in the network. type Builder interface { // NextStage must return next group of route points for the value a @@ -36,7 +17,7 @@ type Builder interface { // in that list (means that point is the last point in one of the route groups), // returned route must contain nil point that should be interpreted as signal to, // among sending to other route points, save the announcement in that point. - NextStage(a container.SizeEstimation, passed []ServerInfo) ([]ServerInfo, error) + NextStage(a container.SizeEstimation, passed []loadcontroller.ServerInfo) ([]loadcontroller.ServerInfo, error) } // RemoteWriterProvider describes the component @@ -46,5 +27,5 @@ type RemoteWriterProvider interface { // corresponding to info. // // Nil info matches the end of the route. - InitRemote(info ServerInfo) (loadcontroller.WriterProvider, error) + InitRemote(info loadcontroller.ServerInfo) (loadcontroller.WriterProvider, error) } diff --git a/pkg/services/container/announcement/load/route/placement/calls.go b/pkg/services/container/announcement/load/route/placement/calls.go index 3db0d967c..68bdb43a7 100644 --- a/pkg/services/container/announcement/load/route/placement/calls.go +++ b/pkg/services/container/announcement/load/route/placement/calls.go @@ -5,7 +5,7 @@ import ( "fmt" netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - loadroute "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/route" + loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" ) @@ -15,7 +15,7 @@ import ( // If passed route has more than one point, then endpoint of the route is reached. // // The traversed route is not checked, it is assumed to be correct. -func (b *Builder) NextStage(a container.SizeEstimation, passed []loadroute.ServerInfo) ([]loadroute.ServerInfo, error) { +func (b *Builder) NextStage(a container.SizeEstimation, passed []loadcontroller.ServerInfo) ([]loadcontroller.ServerInfo, error) { if len(passed) > 1 { return nil, nil } @@ -27,7 +27,7 @@ func (b *Builder) NextStage(a container.SizeEstimation, passed []loadroute.Serve return nil, fmt.Errorf("could not build placement %s: %w", cnr, err) } - res := make([]loadroute.ServerInfo, 0, len(placement)) + res := make([]loadcontroller.ServerInfo, 0, len(placement)) for i := range placement { if len(placement[i]) == 0 { diff --git a/pkg/services/container/announcement/load/route/router.go b/pkg/services/container/announcement/load/route/router.go index 6169a2aee..c8f784b16 100644 --- a/pkg/services/container/announcement/load/route/router.go +++ b/pkg/services/container/announcement/load/route/router.go @@ -3,6 +3,7 @@ package loadroute import ( "fmt" + loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" ) @@ -15,7 +16,7 @@ type Prm struct { // Characteristics of the local node's server. // // Must not be nil. - LocalServerInfo ServerInfo + LocalServerInfo loadcontroller.ServerInfo // Component for sending values to a fixed route point. // @@ -46,7 +47,7 @@ type Router struct { routeBuilder Builder - localSrvInfo ServerInfo + localSrvInfo loadcontroller.ServerInfo } const invalidPrmValFmt = "invalid parameter %s (%T):%v" diff --git a/pkg/services/container/announcement/load/route/util.go b/pkg/services/container/announcement/load/route/util.go index fca1e5796..ea0f51aad 100644 --- a/pkg/services/container/announcement/load/route/util.go +++ b/pkg/services/container/announcement/load/route/util.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" + loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" ) @@ -12,7 +13,7 @@ var errWrongRoute = errors.New("wrong route") // CheckRoute checks if the route is a route correctly constructed by the builder for value a. // // Returns nil if route is correct, otherwise an error clarifying the inconsistency. -func CheckRoute(builder Builder, a container.SizeEstimation, route []ServerInfo) error { +func CheckRoute(builder Builder, a container.SizeEstimation, route []loadcontroller.ServerInfo) error { for i := 1; i < len(route); i++ { servers, err := builder.NextStage(a, route[:i]) if err != nil { From 206458c841c74555b11685002e8e4826cd167621 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Apr 2023 16:28:09 +0300 Subject: [PATCH 0041/1943] [#217] containersvc: Resolve containedctx linter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/container.go | 10 +-- cmd/frostfs-node/main.go | 2 +- .../announcement/load/controller/calls.go | 70 +++++++++---------- .../load/controller/calls_test.go | 8 +-- 4 files changed, 43 insertions(+), 47 deletions(-) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 7a88497eb..6c864431d 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -39,7 +39,7 @@ const ( stopEstimationNotifyEvent = "StopEstimation" ) -func initContainerService(c *cfg) { +func initContainerService(ctx context.Context, c *cfg) { // container wrapper that tries to invoke notary // requests if chain is configured so wrap, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0, cntClient.TryNotary()) @@ -77,7 +77,7 @@ func initContainerService(c *cfg) { loadroute.WithLogger(c.log), ) - setLoadController(c, loadRouter, loadAccumulator) + setLoadController(ctx, c, loadRouter, loadAccumulator) server := containerTransportGRPC.New( containerService.NewSignService( @@ -180,7 +180,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c return cnrRdr, cnrWrt } -func setLoadController(c *cfg, loadRouter *loadroute.Router, loadAccumulator *loadstorage.Storage) { +func setLoadController(ctx context.Context, c *cfg, loadRouter *loadroute.Router, loadAccumulator *loadstorage.Storage) { pubKey := c.key.PublicKey().Bytes() // container wrapper that always sends non-notary @@ -211,14 +211,14 @@ func setLoadController(c *cfg, loadRouter *loadroute.Router, loadAccumulator *lo setContainerNotificationParser(c, startEstimationNotifyEvent, containerEvent.ParseStartEstimation) addContainerAsyncNotificationHandler(c, startEstimationNotifyEvent, func(ev event.Event) { - ctrl.Start(loadcontroller.StartPrm{ + ctrl.Start(ctx, loadcontroller.StartPrm{ Epoch: ev.(containerEvent.StartEstimation).Epoch(), }) }) setContainerNotificationParser(c, stopEstimationNotifyEvent, containerEvent.ParseStopEstimation) addContainerAsyncNotificationHandler(c, stopEstimationNotifyEvent, func(ev event.Event) { - ctrl.Stop(loadcontroller.StopPrm{ + ctrl.Stop(ctx, loadcontroller.StopPrm{ Epoch: ev.(containerEvent.StopEstimation).Epoch(), }) }) diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 7768409b0..fdb003220 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -97,7 +97,7 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(c, "gRPC", initGRPC) initAndLog(c, "netmap", initNetmapService) initAndLog(c, "accounting", initAccountingService) - initAndLog(c, "container", initContainerService) + initAndLog(c, "container", func(c *cfg) { initContainerService(ctx, c) }) initAndLog(c, "session", initSessionService) initAndLog(c, "reputation", initReputationService) initAndLog(c, "notification", initNotifications) diff --git a/pkg/services/container/announcement/load/controller/calls.go b/pkg/services/container/announcement/load/controller/calls.go index 8c5fbeacb..f5d5d1a3d 100644 --- a/pkg/services/container/announcement/load/controller/calls.go +++ b/pkg/services/container/announcement/load/controller/calls.go @@ -15,18 +15,15 @@ type StartPrm struct { Epoch uint64 } -// nolint: containedctx type commonContext struct { epoch uint64 ctrl *Controller log *logger.Logger - - ctx context.Context } -type announceContext struct { +type announcer struct { commonContext } @@ -39,21 +36,22 @@ type announceContext struct { // // Each call acquires an announcement context for an Epoch parameter. // At the very end of the operation, the context is released. -func (c *Controller) Start(prm StartPrm) { +func (c *Controller) Start(ctx context.Context, prm StartPrm) { + var announcer *announcer // acquire announcement - execCtx := c.acquireAnnouncement(prm) - if execCtx == nil { + ctx, announcer = c.acquireAnnouncement(ctx, prm) + if announcer == nil { return } // finally stop and free the announcement - defer execCtx.freeAnnouncement() + defer announcer.freeAnnouncement() // announce local values - execCtx.announce() + announcer.announce(ctx) } -func (c *announceContext) announce() { +func (c *announcer) announce(ctx context.Context) { c.log.Debug("starting to announce the values of the metrics") var ( @@ -100,7 +98,7 @@ func (c *announceContext) announce() { } // finish writing - err = targetWriter.Close(c.ctx) + err = targetWriter.Close(ctx) if err != nil { c.log.Debug("could not finish writing local announcements", zap.String("error", err.Error()), @@ -112,35 +110,32 @@ func (c *announceContext) announce() { c.log.Debug("trust announcement successfully finished") } -func (c *Controller) acquireAnnouncement(prm StartPrm) *announceContext { - var ctx context.Context - +func (c *Controller) acquireAnnouncement(ctx context.Context, prm StartPrm) (context.Context, *announcer) { + started := true c.announceMtx.Lock() - { if cancel := c.mAnnounceCtx[prm.Epoch]; cancel == nil { - ctx, cancel = context.WithCancel(context.Background()) + ctx, cancel = context.WithCancel(ctx) c.mAnnounceCtx[prm.Epoch] = cancel + started = false } } - c.announceMtx.Unlock() log := &logger.Logger{Logger: c.opts.log.With( zap.Uint64("epoch", prm.Epoch), )} - if ctx == nil { + if started { log.Debug("announcement is already started") - return nil + return ctx, nil } - return &announceContext{ + return ctx, &announcer{ commonContext: commonContext{ epoch: prm.Epoch, ctrl: c, log: log, - ctx: ctx, }, } } @@ -176,7 +171,7 @@ type StopPrm struct { Epoch uint64 } -type stopContext struct { +type reporter struct { commonContext } @@ -188,31 +183,32 @@ type stopContext struct { // // Each call acquires a report context for an Epoch parameter. // At the very end of the operation, the context is released. -func (c *Controller) Stop(prm StopPrm) { - execCtx := c.acquireReport(prm) - if execCtx == nil { +func (c *Controller) Stop(ctx context.Context, prm StopPrm) { + var reporter *reporter + ctx, reporter = c.acquireReport(ctx, prm) + if reporter == nil { return } // finally stop and free reporting - defer execCtx.freeReport() + defer reporter.freeReport() // interrupt announcement - execCtx.freeAnnouncement() + reporter.freeAnnouncement() // report the estimations - execCtx.report() + reporter.report(ctx) } -func (c *Controller) acquireReport(prm StopPrm) *stopContext { - var ctx context.Context +func (c *Controller) acquireReport(ctx context.Context, prm StopPrm) (context.Context, *reporter) { + started := true c.reportMtx.Lock() - { if cancel := c.mReportCtx[prm.Epoch]; cancel == nil { - ctx, cancel = context.WithCancel(context.Background()) + ctx, cancel = context.WithCancel(ctx) c.mReportCtx[prm.Epoch] = cancel + started = false } } @@ -222,12 +218,12 @@ func (c *Controller) acquireReport(prm StopPrm) *stopContext { zap.Uint64("epoch", prm.Epoch), )} - if ctx == nil { + if started { log.Debug("report is already started") - return nil + return ctx, nil } - return &stopContext{ + return ctx, &reporter{ commonContext: commonContext{ epoch: prm.Epoch, ctrl: c, @@ -261,7 +257,7 @@ func (c *commonContext) freeReport() { } } -func (c *stopContext) report() { +func (c *reporter) report(ctx context.Context) { var ( localIterator Iterator err error @@ -301,7 +297,7 @@ func (c *stopContext) report() { } // finish writing - err = resultWriter.Close(c.ctx) + err = resultWriter.Close(ctx) if err != nil { c.log.Debug("could not finish writing load estimations", zap.String("error", err.Error()), diff --git a/pkg/services/container/announcement/load/controller/calls_test.go b/pkg/services/container/announcement/load/controller/calls_test.go index 4a791f4bd..6ca24e869 100644 --- a/pkg/services/container/announcement/load/controller/calls_test.go +++ b/pkg/services/container/announcement/load/controller/calls_test.go @@ -143,12 +143,12 @@ func TestSimpleScenario(t *testing.T) { // start both controllers go func() { - ctrlN1.Start(startPrm) + ctrlN1.Start(context.Background(), startPrm) wg.Done() }() go func() { - ctrlN2.Start(startPrm) + ctrlN2.Start(context.Background(), startPrm) wg.Done() }() @@ -161,12 +161,12 @@ func TestSimpleScenario(t *testing.T) { // stop both controllers go func() { - ctrlN1.Stop(stopPrm) + ctrlN1.Stop(context.Background(), stopPrm) wg.Done() }() go func() { - ctrlN2.Stop(stopPrm) + ctrlN2.Stop(context.Background(), stopPrm) wg.Done() }() From 1bf21dbb473d319308cf999a5fe5d2688f27fee6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 31 Mar 2023 14:49:40 +0300 Subject: [PATCH 0042/1943] [#193] getsvc: Resolve context linters Resolve containedctx and contextcheck linters. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assemble.go | 10 ++++------ pkg/services/object/get/container.go | 8 ++++---- pkg/services/object/get/exec.go | 21 +++++++-------------- pkg/services/object/get/get.go | 18 ++++++++---------- pkg/services/object/get/get_test.go | 2 +- pkg/services/object/get/local.go | 5 +++-- pkg/services/object/get/remote.go | 5 ++--- pkg/services/object/get/service.go | 4 +++- pkg/services/object/get/util.go | 14 +++++++------- 9 files changed, 39 insertions(+), 48 deletions(-) diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index ebae18eb5..db71df6a4 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -10,7 +10,7 @@ import ( "go.uber.org/zap" ) -func (exec *execCtx) assemble() { +func (exec *execCtx) assemble(ctx context.Context) { if !exec.canAssemble() { exec.log.Debug("can not assemble the object") return @@ -49,7 +49,7 @@ func (exec *execCtx) assemble() { zap.Uint64("range_length", exec.ctxRange().GetLength()), ) - obj, err := assembler.Assemble(exec.context(), exec.prm.objWriter) + obj, err := assembler.Assemble(ctx, exec.prm.objWriter) if err != nil { exec.log.Warn("failed to assemble splitted object", zap.Error(err), @@ -107,8 +107,7 @@ func (exec *execCtx) HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Obje w := NewSimpleObjectWriter() prm.SetHeaderWriter(w) - //nolint: contextcheck - err := exec.svc.Head(exec.context(), prm) + err := exec.svc.Head(ctx, prm) if err != nil { return nil, err @@ -128,8 +127,7 @@ func (exec *execCtx) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Ra p.addr.SetContainer(exec.containerID()) p.addr.SetObject(id) - //nolint: contextcheck - statusError := exec.svc.get(exec.context(), p.commonPrm, withPayloadRange(rng)) + statusError := exec.svc.get(ctx, p.commonPrm, withPayloadRange(rng)) if statusError.err != nil { return nil, statusError.err diff --git a/pkg/services/object/get/container.go b/pkg/services/object/get/container.go index 882861129..cfb538d38 100644 --- a/pkg/services/object/get/container.go +++ b/pkg/services/object/get/container.go @@ -7,7 +7,7 @@ import ( "go.uber.org/zap" ) -func (exec *execCtx) executeOnContainer() { +func (exec *execCtx) executeOnContainer(ctx context.Context) { if exec.isLocal() { exec.log.Debug("return result directly") return @@ -26,7 +26,7 @@ func (exec *execCtx) executeOnContainer() { } for { - if exec.processCurrentEpoch() { + if exec.processCurrentEpoch(ctx) { break } @@ -42,7 +42,7 @@ func (exec *execCtx) executeOnContainer() { } } -func (exec *execCtx) processCurrentEpoch() bool { +func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { exec.log.Debug("process epoch", zap.Uint64("number", exec.curProcEpoch), ) @@ -52,7 +52,7 @@ func (exec *execCtx) processCurrentEpoch() bool { return true } - ctx, cancel := context.WithCancel(exec.context()) + ctx, cancel := context.WithCancel(ctx) defer cancel() exec.status = statusUndefined diff --git a/pkg/services/object/get/exec.go b/pkg/services/object/get/exec.go index 9858b32b2..2ba014574 100644 --- a/pkg/services/object/get/exec.go +++ b/pkg/services/object/get/exec.go @@ -19,12 +19,9 @@ type statusError struct { err error } -// nolint: containedctx type execCtx struct { svc *Service - ctx context.Context - prm RangePrm statusError @@ -80,10 +77,6 @@ func (exec *execCtx) setLogger(l *logger.Logger) { )} } -func (exec execCtx) context() context.Context { - return exec.ctx -} - func (exec execCtx) isLocal() bool { return exec.prm.common.LocalOnly() } @@ -217,13 +210,13 @@ func mergeSplitInfo(dst, src *objectSDK.SplitInfo) { } } -func (exec *execCtx) writeCollectedHeader() bool { +func (exec *execCtx) writeCollectedHeader(ctx context.Context) bool { if exec.ctxRange() != nil { return true } err := exec.prm.objWriter.WriteHeader( - exec.context(), + ctx, exec.collectedObject.CutPayload(), ) @@ -243,12 +236,12 @@ func (exec *execCtx) writeCollectedHeader() bool { return exec.status == statusOK } -func (exec *execCtx) writeObjectPayload(obj *objectSDK.Object) bool { +func (exec *execCtx) writeObjectPayload(ctx context.Context, obj *objectSDK.Object) bool { if exec.headOnly() { return true } - err := exec.prm.objWriter.WriteChunk(exec.context(), obj.Payload()) + err := exec.prm.objWriter.WriteChunk(ctx, obj.Payload()) switch { default: @@ -266,9 +259,9 @@ func (exec *execCtx) writeObjectPayload(obj *objectSDK.Object) bool { return err == nil } -func (exec *execCtx) writeCollectedObject() { - if ok := exec.writeCollectedHeader(); ok { - exec.writeObjectPayload(exec.collectedObject) +func (exec *execCtx) writeCollectedObject(ctx context.Context) { + if ok := exec.writeCollectedHeader(ctx); ok { + exec.writeObjectPayload(ctx, exec.collectedObject) } } diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index cdb2d96fd..0f5983e99 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -65,7 +65,6 @@ func (s *Service) Head(ctx context.Context, prm HeadPrm) error { func (s *Service) get(ctx context.Context, prm commonPrm, opts ...execOption) statusError { exec := &execCtx{ svc: s, - ctx: ctx, prm: RangePrm{ commonPrm: prm, }, @@ -78,22 +77,21 @@ func (s *Service) get(ctx context.Context, prm commonPrm, opts ...execOption) st exec.setLogger(s.log) - //nolint: contextcheck - exec.execute() + exec.execute(ctx) return exec.statusError } -func (exec *execCtx) execute() { +func (exec *execCtx) execute(ctx context.Context) { exec.log.Debug("serving request...") // perform local operation - exec.executeLocal() + exec.executeLocal(ctx) - exec.analyzeStatus(true) + exec.analyzeStatus(ctx, true) } -func (exec *execCtx) analyzeStatus(execCnr bool) { +func (exec *execCtx) analyzeStatus(ctx context.Context, execCnr bool) { // analyze local result switch exec.status { case statusOK: @@ -102,7 +100,7 @@ func (exec *execCtx) analyzeStatus(execCnr bool) { exec.log.Debug("requested object was marked as removed") case statusVIRTUAL: exec.log.Debug("requested object is virtual") - exec.assemble() + exec.assemble(ctx) case statusOutOfRange: exec.log.Debug("requested range is out of object bounds") default: @@ -111,8 +109,8 @@ func (exec *execCtx) analyzeStatus(execCnr bool) { ) if execCnr { - exec.executeOnContainer() - exec.analyzeStatus(false) + exec.executeOnContainer(ctx) + exec.analyzeStatus(ctx, false) } } } diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 36a0e4976..3d1a95cbb 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -117,7 +117,7 @@ func newTestClient() *testClient { } } -func (c *testClient) getObject(exec *execCtx, _ client.NodeInfo) (*objectSDK.Object, error) { +func (c *testClient) getObject(ctx context.Context, exec *execCtx, _ client.NodeInfo) (*objectSDK.Object, error) { v, ok := c.results[exec.address().EncodeToString()] if !ok { var errNotFound apistatus.ObjectNotFound diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index f526af4e6..a6a77729c 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -1,6 +1,7 @@ package getsvc import ( + "context" "errors" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -8,7 +9,7 @@ import ( "go.uber.org/zap" ) -func (exec *execCtx) executeLocal() { +func (exec *execCtx) executeLocal(ctx context.Context) { var err error exec.collectedObject, err = exec.svc.localStorage.get(exec) @@ -28,7 +29,7 @@ func (exec *execCtx) executeLocal() { case err == nil: exec.status = statusOK exec.err = nil - exec.writeCollectedObject() + exec.writeCollectedObject(ctx) case errors.As(err, &errRemoved): exec.status = statusINHUMED exec.err = errRemoved diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index fbfb01bcd..1532bade0 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -18,7 +18,7 @@ func (exec *execCtx) processNode(ctx context.Context, info client.NodeInfo) bool return true } - obj, err := client.getObject(exec, info) + obj, err := client.getObject(ctx, exec, info) var errSplitInfo *objectSDK.SplitInfoError var errRemoved *apistatus.ObjectAlreadyRemoved @@ -43,8 +43,7 @@ func (exec *execCtx) processNode(ctx context.Context, info client.NodeInfo) bool // has already been streamed to the requesting party if obj != nil { exec.collectedObject = obj - //nolint: contextcheck - exec.writeCollectedObject() + exec.writeCollectedObject(ctx) } case errors.As(err, &errRemoved): exec.status = statusINHUMED diff --git a/pkg/services/object/get/service.go b/pkg/services/object/get/service.go index e69ab4f0f..dfa3b48ac 100644 --- a/pkg/services/object/get/service.go +++ b/pkg/services/object/get/service.go @@ -1,6 +1,8 @@ package getsvc import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" @@ -22,7 +24,7 @@ type Service struct { type Option func(*cfg) type getClient interface { - getObject(*execCtx, client.NodeInfo) (*object.Object, error) + getObject(context.Context, *execCtx, client.NodeInfo) (*object.Object, error) } type cfg struct { diff --git a/pkg/services/object/get/util.go b/pkg/services/object/get/util.go index d647b07f6..08c738280 100644 --- a/pkg/services/object/get/util.go +++ b/pkg/services/object/get/util.go @@ -88,7 +88,7 @@ func (c *clientCacheWrapper) get(info coreclient.NodeInfo) (getClient, error) { } // nolint: funlen -func (c *clientWrapper) getObject(exec *execCtx, info coreclient.NodeInfo) (*object.Object, error) { +func (c *clientWrapper) getObject(ctx context.Context, exec *execCtx, info coreclient.NodeInfo) (*object.Object, error) { if exec.isForwardingEnabled() { return exec.prm.forwarder(info, c.client) } @@ -101,7 +101,7 @@ func (c *clientWrapper) getObject(exec *execCtx, info coreclient.NodeInfo) (*obj if exec.headOnly() { var prm internalclient.HeadObjectPrm - prm.SetContext(exec.context()) + prm.SetContext(ctx) prm.SetClient(c.client) prm.SetTTL(exec.prm.common.TTL()) prm.SetNetmapEpoch(exec.curProcEpoch) @@ -127,7 +127,7 @@ func (c *clientWrapper) getObject(exec *execCtx, info coreclient.NodeInfo) (*obj if rng := exec.ctxRange(); rng != nil { var prm internalclient.PayloadRangePrm - prm.SetContext(exec.context()) + prm.SetContext(ctx) prm.SetClient(c.client) prm.SetTTL(exec.prm.common.TTL()) prm.SetNetmapEpoch(exec.curProcEpoch) @@ -148,7 +148,7 @@ func (c *clientWrapper) getObject(exec *execCtx, info coreclient.NodeInfo) (*obj if errors.As(err, &errAccessDenied) { // Current spec allows other storage node to deny access, // fallback to GET here. - obj, err := c.get(exec, key) + obj, err := c.get(ctx, exec, key) if err != nil { return nil, err } @@ -169,13 +169,13 @@ func (c *clientWrapper) getObject(exec *execCtx, info coreclient.NodeInfo) (*obj return payloadOnlyObject(res.PayloadRange()), nil } - return c.get(exec, key) + return c.get(ctx, exec, key) } -func (c *clientWrapper) get(exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { +func (c *clientWrapper) get(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { var prm internalclient.GetObjectPrm - prm.SetContext(exec.context()) + prm.SetContext(ctx) prm.SetClient(c.client) prm.SetTTL(exec.prm.common.TTL()) prm.SetNetmapEpoch(exec.curProcEpoch) From 91ead04fa47ca2827e5f234b037766eb81b149ec Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 31 Mar 2023 15:44:18 +0300 Subject: [PATCH 0043/1943] [#193] getsvc: Resolve funlen linter Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/util.go | 137 +++++++++++++++++--------------- 1 file changed, 72 insertions(+), 65 deletions(-) diff --git a/pkg/services/object/get/util.go b/pkg/services/object/get/util.go index 08c738280..a68232c9a 100644 --- a/pkg/services/object/get/util.go +++ b/pkg/services/object/get/util.go @@ -87,7 +87,6 @@ func (c *clientCacheWrapper) get(info coreclient.NodeInfo) (getClient, error) { }, nil } -// nolint: funlen func (c *clientWrapper) getObject(ctx context.Context, exec *execCtx, info coreclient.NodeInfo) (*object.Object, error) { if exec.isForwardingEnabled() { return exec.prm.forwarder(info, c.client) @@ -99,79 +98,87 @@ func (c *clientWrapper) getObject(ctx context.Context, exec *execCtx, info corec } if exec.headOnly() { - var prm internalclient.HeadObjectPrm - - prm.SetContext(ctx) - prm.SetClient(c.client) - prm.SetTTL(exec.prm.common.TTL()) - prm.SetNetmapEpoch(exec.curProcEpoch) - prm.SetAddress(exec.address()) - prm.SetPrivateKey(key) - prm.SetSessionToken(exec.prm.common.SessionToken()) - prm.SetBearerToken(exec.prm.common.BearerToken()) - prm.SetXHeaders(exec.prm.common.XHeaders()) - - if exec.isRaw() { - prm.SetRawFlag() - } - - res, err := internalclient.HeadObject(prm) - if err != nil { - return nil, err - } - - return res.Header(), nil + return c.getHeadOnly(ctx, exec, key) } // we don't specify payload writer because we accumulate // the object locally (even huge). if rng := exec.ctxRange(); rng != nil { - var prm internalclient.PayloadRangePrm - - prm.SetContext(ctx) - prm.SetClient(c.client) - prm.SetTTL(exec.prm.common.TTL()) - prm.SetNetmapEpoch(exec.curProcEpoch) - prm.SetAddress(exec.address()) - prm.SetPrivateKey(key) - prm.SetSessionToken(exec.prm.common.SessionToken()) - prm.SetBearerToken(exec.prm.common.BearerToken()) - prm.SetXHeaders(exec.prm.common.XHeaders()) - prm.SetRange(rng) - - if exec.isRaw() { - prm.SetRawFlag() - } - - res, err := internalclient.PayloadRange(prm) - if err != nil { - var errAccessDenied *apistatus.ObjectAccessDenied - if errors.As(err, &errAccessDenied) { - // Current spec allows other storage node to deny access, - // fallback to GET here. - obj, err := c.get(ctx, exec, key) - if err != nil { - return nil, err - } - - payload := obj.Payload() - from := rng.GetOffset() - to := from + rng.GetLength() - - if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to { - return nil, new(apistatus.ObjectOutOfRange) - } - - return payloadOnlyObject(payload[from:to]), nil - } - return nil, err - } - - return payloadOnlyObject(res.PayloadRange()), nil + // Current spec allows other storage node to deny access, + // fallback to GET here. + return c.getRange(ctx, exec, key, rng) } return c.get(ctx, exec, key) } +func (c *clientWrapper) getRange(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey, rng *object.Range) (*object.Object, error) { + var prm internalclient.PayloadRangePrm + + prm.SetContext(ctx) + prm.SetClient(c.client) + prm.SetTTL(exec.prm.common.TTL()) + prm.SetNetmapEpoch(exec.curProcEpoch) + prm.SetAddress(exec.address()) + prm.SetPrivateKey(key) + prm.SetSessionToken(exec.prm.common.SessionToken()) + prm.SetBearerToken(exec.prm.common.BearerToken()) + prm.SetXHeaders(exec.prm.common.XHeaders()) + prm.SetRange(rng) + + if exec.isRaw() { + prm.SetRawFlag() + } + + res, err := internalclient.PayloadRange(prm) + if err != nil { + var errAccessDenied *apistatus.ObjectAccessDenied + if errors.As(err, &errAccessDenied) { + obj, err := c.get(ctx, exec, key) + if err != nil { + return nil, err + } + + payload := obj.Payload() + from := rng.GetOffset() + to := from + rng.GetLength() + + if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to { + return nil, new(apistatus.ObjectOutOfRange) + } + + return payloadOnlyObject(payload[from:to]), nil + } + return nil, err + } + + return payloadOnlyObject(res.PayloadRange()), nil +} + +func (c *clientWrapper) getHeadOnly(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { + var prm internalclient.HeadObjectPrm + + prm.SetContext(ctx) + prm.SetClient(c.client) + prm.SetTTL(exec.prm.common.TTL()) + prm.SetNetmapEpoch(exec.curProcEpoch) + prm.SetAddress(exec.address()) + prm.SetPrivateKey(key) + prm.SetSessionToken(exec.prm.common.SessionToken()) + prm.SetBearerToken(exec.prm.common.BearerToken()) + prm.SetXHeaders(exec.prm.common.XHeaders()) + + if exec.isRaw() { + prm.SetRawFlag() + } + + res, err := internalclient.HeadObject(prm) + if err != nil { + return nil, err + } + + return res.Header(), nil +} + func (c *clientWrapper) get(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { var prm internalclient.GetObjectPrm From f8898932164b3b1affa5178f11b3a11f521b621c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 31 Mar 2023 16:30:46 +0300 Subject: [PATCH 0044/1943] [#193] getsvc: Refactor head param creation Resolve funlen linter for toHeadPrm method. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/v2/head_forwarder.go | 180 +++++++++++++++++++ pkg/services/object/get/v2/util.go | 138 +------------- 2 files changed, 188 insertions(+), 130 deletions(-) create mode 100644 pkg/services/object/get/v2/head_forwarder.go diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go new file mode 100644 index 000000000..b38da7131 --- /dev/null +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -0,0 +1,180 @@ +package getsvc + +import ( + "context" + "errors" + "fmt" + "sync" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" + rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +type headRequestForwarder struct { + Request *objectV2.HeadRequest + Response *objectV2.HeadResponse + OnceResign *sync.Once + ObjectAddr oid.Address + KeyStorage *util.KeyStorage +} + +func (f *headRequestForwarder) forward(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { + var err error + + key, err := f.KeyStorage.GetKey(nil) + if err != nil { + return nil, err + } + + // once compose and resign forwarding request + f.OnceResign.Do(func() { + // compose meta header of the local server + metaHdr := new(session.RequestMetaHeader) + metaHdr.SetTTL(f.Request.GetMetaHeader().GetTTL() - 1) + // TODO: #1165 think how to set the other fields + metaHdr.SetOrigin(f.Request.GetMetaHeader()) + writeCurrentVersion(metaHdr) + + f.Request.SetMetaHeader(metaHdr) + + err = signature.SignServiceMessage(key, f.Request) + }) + + if err != nil { + return nil, err + } + + headResp, err := f.sendHeadRequest(ctx, addr, c) + if err != nil { + return nil, err + } + + if err := f.verifyResponse(headResp, pubkey); err != nil { + return nil, err + } + + var ( + hdr *objectV2.Header + idSig *refs.Signature + ) + + switch v := headResp.GetBody().GetHeaderPart().(type) { + case nil: + return nil, fmt.Errorf("unexpected header type %T", v) + case *objectV2.ShortHeader: + if hdr, err = f.getHeaderFromShortHeader(v); err != nil { + return nil, err + } + case *objectV2.HeaderWithSignature: + if hdr, idSig, err = f.getHeaderAndSignature(v); err != nil { + return nil, err + } + case *objectV2.SplitInfo: + si := object.NewSplitInfoFromV2(v) + return nil, object.NewSplitInfoError(si) + } + + objv2 := new(objectV2.Object) + objv2.SetHeader(hdr) + objv2.SetSignature(idSig) + + obj := object.NewFromV2(objv2) + obj.SetID(f.ObjectAddr.Object()) + + return obj, nil +} + +func (f *headRequestForwarder) getHeaderFromShortHeader(sh *objectV2.ShortHeader) (*objectV2.Header, error) { + if !f.Request.GetBody().GetMainOnly() { + return nil, fmt.Errorf("wrong header part type: expected %T, received %T", + (*objectV2.ShortHeader)(nil), (*objectV2.HeaderWithSignature)(nil), + ) + } + + hdr := new(objectV2.Header) + hdr.SetPayloadLength(sh.GetPayloadLength()) + hdr.SetVersion(sh.GetVersion()) + hdr.SetOwnerID(sh.GetOwnerID()) + hdr.SetObjectType(sh.GetObjectType()) + hdr.SetCreationEpoch(sh.GetCreationEpoch()) + hdr.SetPayloadHash(sh.GetPayloadHash()) + hdr.SetHomomorphicHash(sh.GetHomomorphicHash()) + return hdr, nil +} + +func (f *headRequestForwarder) getHeaderAndSignature(hdrWithSig *objectV2.HeaderWithSignature) (*objectV2.Header, *refs.Signature, error) { + if f.Request.GetBody().GetMainOnly() { + return nil, nil, fmt.Errorf("wrong header part type: expected %T, received %T", + (*objectV2.HeaderWithSignature)(nil), (*objectV2.ShortHeader)(nil), + ) + } + + if hdrWithSig == nil { + return nil, nil, errors.New("nil object part") + } + + hdr := hdrWithSig.GetHeader() + idSig := hdrWithSig.GetSignature() + + if idSig == nil { + // TODO(@cthulhu-rider): #1387 use "const" error + return nil, nil, errors.New("missing signature") + } + + binID, err := f.ObjectAddr.Object().Marshal() + if err != nil { + return nil, nil, fmt.Errorf("marshal ID: %w", err) + } + + var sig frostfscrypto.Signature + if err := sig.ReadFromV2(*idSig); err != nil { + return nil, nil, fmt.Errorf("can't read signature: %w", err) + } + + if !sig.Verify(binID) { + return nil, nil, errors.New("invalid object ID signature") + } + + return hdr, idSig, nil +} + +func (f *headRequestForwarder) sendHeadRequest(ctx context.Context, addr network.Address, c client.MultiAddressClient) (*objectV2.HeadResponse, error) { + var headResp *objectV2.HeadResponse + err := c.RawForAddress(addr, func(cli *rpcclient.Client) error { + var e error + headResp, e = rpc.HeadObject(cli, f.Request, rpcclient.WithContext(ctx)) + return e + }) + if err != nil { + return nil, fmt.Errorf("sending the request failed: %w", err) + } + return headResp, nil +} + +func (f *headRequestForwarder) verifyResponse(headResp *objectV2.HeadResponse, pubkey []byte) error { + // verify response key + if err := internal.VerifyResponseKeyV2(pubkey, headResp); err != nil { + return err + } + + // verify response structure + if err := signature.VerifyServiceMessage(headResp); err != nil { + return fmt.Errorf("response verification failed: %w", err) + } + + if err := checkStatus(f.Response.GetMetaHeader().GetStatus()); err != nil { + return err + } + return nil +} diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index a871714a1..c659f4e7c 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -24,7 +24,6 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" versionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" @@ -426,7 +425,6 @@ func (w *headResponseWriter) WriteHeader(_ context.Context, hdr *object.Object) return nil } -// nolint: funlen func (s *Service) toHeadPrm(ctx context.Context, req *objectV2.HeadRequest, resp *objectV2.HeadResponse) (*getsvc.HeadPrm, error) { body := req.GetBody() @@ -442,8 +440,6 @@ func (s *Service) toHeadPrm(ctx context.Context, req *objectV2.HeadRequest, resp return nil, fmt.Errorf("invalid object address: %w", err) } - meta := req.GetMetaHeader() - commonPrm, err := util.CommonPrmFromV2(req) if err != nil { return nil, err @@ -463,134 +459,16 @@ func (s *Service) toHeadPrm(ctx context.Context, req *objectV2.HeadRequest, resp return p, nil } - var onceResign sync.Once + forwarder := &headRequestForwarder{ + Request: req, + Response: resp, + OnceResign: &sync.Once{}, + ObjectAddr: objAddr, + KeyStorage: s.keyStorage, + } p.SetRequestForwarder(groupAddressRequestForwarder(func(addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { - var err error - - key, err := s.keyStorage.GetKey(nil) - if err != nil { - return nil, err - } - - // once compose and resign forwarding request - onceResign.Do(func() { - // compose meta header of the local server - metaHdr := new(session.RequestMetaHeader) - metaHdr.SetTTL(meta.GetTTL() - 1) - // TODO: #1165 think how to set the other fields - metaHdr.SetOrigin(meta) - writeCurrentVersion(metaHdr) - - req.SetMetaHeader(metaHdr) - - err = signature.SignServiceMessage(key, req) - }) - - if err != nil { - return nil, err - } - - // code below is copy-pasted from c.GetObjectHeader implementation, - // perhaps it is worth highlighting the utility function in frostfs-api-go - - // send Head request - var headResp *objectV2.HeadResponse - err = c.RawForAddress(addr, func(cli *rpcclient.Client) error { - headResp, err = rpc.HeadObject(cli, req, rpcclient.WithContext(ctx)) - return err - }) - if err != nil { - return nil, fmt.Errorf("sending the request failed: %w", err) - } - - // verify response key - if err = internal.VerifyResponseKeyV2(pubkey, headResp); err != nil { - return nil, err - } - - // verify response structure - if err := signature.VerifyServiceMessage(headResp); err != nil { - return nil, fmt.Errorf("response verification failed: %w", err) - } - - if err = checkStatus(resp.GetMetaHeader().GetStatus()); err != nil { - return nil, err - } - - var ( - hdr *objectV2.Header - idSig *refs.Signature - ) - - switch v := headResp.GetBody().GetHeaderPart().(type) { - case nil: - return nil, fmt.Errorf("unexpected header type %T", v) - case *objectV2.ShortHeader: - if !body.GetMainOnly() { - return nil, fmt.Errorf("wrong header part type: expected %T, received %T", - (*objectV2.ShortHeader)(nil), (*objectV2.HeaderWithSignature)(nil), - ) - } - - h := v - - hdr = new(objectV2.Header) - hdr.SetPayloadLength(h.GetPayloadLength()) - hdr.SetVersion(h.GetVersion()) - hdr.SetOwnerID(h.GetOwnerID()) - hdr.SetObjectType(h.GetObjectType()) - hdr.SetCreationEpoch(h.GetCreationEpoch()) - hdr.SetPayloadHash(h.GetPayloadHash()) - hdr.SetHomomorphicHash(h.GetHomomorphicHash()) - case *objectV2.HeaderWithSignature: - if body.GetMainOnly() { - return nil, fmt.Errorf("wrong header part type: expected %T, received %T", - (*objectV2.HeaderWithSignature)(nil), (*objectV2.ShortHeader)(nil), - ) - } - - hdrWithSig := v - if hdrWithSig == nil { - return nil, errors.New("nil object part") - } - - hdr = hdrWithSig.GetHeader() - idSig = hdrWithSig.GetSignature() - - if idSig == nil { - // TODO(@cthulhu-rider): #1387 use "const" error - return nil, errors.New("missing signature") - } - - binID, err := objAddr.Object().Marshal() - if err != nil { - return nil, fmt.Errorf("marshal ID: %w", err) - } - - var sig frostfscrypto.Signature - if err := sig.ReadFromV2(*idSig); err != nil { - return nil, fmt.Errorf("can't read signature: %w", err) - } - - if !sig.Verify(binID) { - return nil, errors.New("invalid object ID signature") - } - case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - - return nil, object.NewSplitInfoError(si) - } - - objv2 := new(objectV2.Object) - objv2.SetHeader(hdr) - objv2.SetSignature(idSig) - - obj := object.NewFromV2(objv2) - obj.SetID(objAddr.Object()) - - // convert the object - return obj, nil + return forwarder.forward(ctx, addr, c, pubkey) })) return p, nil From b0786d2e5c5eeadf0a3859f17ef6c6c7a70d05be Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 31 Mar 2023 17:38:38 +0300 Subject: [PATCH 0045/1943] [#193] getsvc: Refactor get params creation Resolve funlen linter for toPrm function. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/v2/get_forwarder.go | 171 ++++++++++++++++++++ pkg/services/object/get/v2/util.go | 138 ++-------------- 2 files changed, 180 insertions(+), 129 deletions(-) create mode 100644 pkg/services/object/get/v2/get_forwarder.go diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go new file mode 100644 index 000000000..b0ba47523 --- /dev/null +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -0,0 +1,171 @@ +package getsvc + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" + rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" +) + +type getRequestForwarder struct { + OnceResign *sync.Once + OnceHeaderSending *sync.Once + GlobalProgress int + KeyStorage *util.KeyStorage + Request *objectV2.GetRequest + Stream *streamObjectWriter +} + +func (f *getRequestForwarder) forward(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { + key, err := f.KeyStorage.GetKey(nil) + if err != nil { + return nil, err + } + + // once compose and resign forwarding request + f.OnceResign.Do(func() { + // compose meta header of the local server + metaHdr := new(session.RequestMetaHeader) + metaHdr.SetTTL(f.Request.GetMetaHeader().GetTTL() - 1) + // TODO: #1165 think how to set the other fields + metaHdr.SetOrigin(f.Request.GetMetaHeader()) + writeCurrentVersion(metaHdr) + f.Request.SetMetaHeader(metaHdr) + err = signature.SignServiceMessage(key, f.Request) + }) + + if err != nil { + return nil, err + } + + getStream, err := f.openStream(ctx, addr, c) + if err != nil { + return nil, err + } + return nil, f.readStream(ctx, c, getStream, pubkey) +} + +func (f *getRequestForwarder) verifyResponse(resp *objectV2.GetResponse, pubkey []byte) error { + // verify response key + if err := internal.VerifyResponseKeyV2(pubkey, resp); err != nil { + return err + } + + // verify response structure + if err := signature.VerifyServiceMessage(resp); err != nil { + return fmt.Errorf("response verification failed: %w", err) + } + + if err := checkStatus(resp.GetMetaHeader().GetStatus()); err != nil { + return err + } + return nil +} + +func (f *getRequestForwarder) writeHeader(ctx context.Context, v *objectV2.GetObjectPartInit) error { + obj := new(objectV2.Object) + + obj.SetObjectID(v.GetObjectID()) + obj.SetSignature(v.GetSignature()) + obj.SetHeader(v.GetHeader()) + + var err error + f.OnceHeaderSending.Do(func() { + err = f.Stream.WriteHeader(ctx, object.NewFromV2(obj)) + }) + if err != nil { + return fmt.Errorf("could not write object header in Get forwarder: %w", err) + } + return nil +} + +func (f *getRequestForwarder) openStream(ctx context.Context, addr network.Address, c client.MultiAddressClient) (*rpc.GetResponseReader, error) { + var getStream *rpc.GetResponseReader + err := c.RawForAddress(addr, func(cli *rpcclient.Client) error { + var e error + getStream, e = rpc.GetObject(cli, f.Request, rpcclient.WithContext(ctx)) + return e + }) + if err != nil { + return nil, fmt.Errorf("stream opening failed: %w", err) + } + return getStream, nil +} + +func (f *getRequestForwarder) readStream(ctx context.Context, c client.MultiAddressClient, getStream *rpc.GetResponseReader, pubkey []byte) error { + var ( + headWas bool + resp = new(objectV2.GetResponse) + localProgress int + ) + + for { + // receive message from server stream + err := getStream.Read(resp) + if err != nil { + if errors.Is(err, io.EOF) { + if !headWas { + return io.ErrUnexpectedEOF + } + + break + } + + internalclient.ReportError(c, err) + return fmt.Errorf("reading the response failed: %w", err) + } + + if err := f.verifyResponse(resp, pubkey); err != nil { + return err + } + + switch v := resp.GetBody().GetObjectPart().(type) { + default: + return fmt.Errorf("unexpected object part %T", v) + case *objectV2.GetObjectPartInit: + if headWas { + return errWrongMessageSeq + } + headWas = true + if err := f.writeHeader(ctx, v); err != nil { + return err + } + case *objectV2.GetObjectPartChunk: + if !headWas { + return errWrongMessageSeq + } + + origChunk := v.GetChunk() + + chunk := chunkToSend(f.GlobalProgress, localProgress, origChunk) + if len(chunk) == 0 { + localProgress += len(origChunk) + continue + } + + if err = f.Stream.WriteChunk(ctx, chunk); err != nil { + return fmt.Errorf("could not write object chunk in Get forwarder: %w", err) + } + + localProgress += len(origChunk) + f.GlobalProgress += len(chunk) + case *objectV2.SplitInfo: + si := object.NewSplitInfoFromV2(v) + return object.NewSplitInfoError(si) + } + } + return nil +} diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index c659f4e7c..3a9cf3b07 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -32,7 +32,6 @@ import ( var errWrongMessageSeq = errors.New("incorrect message sequence") -// nolint: funlen, gocognit func (s *Service) toPrm(req *objectV2.GetRequest, stream objectSvc.GetObjectStream) (*getsvc.Prm, error) { body := req.GetBody() @@ -48,8 +47,6 @@ func (s *Service) toPrm(req *objectV2.GetRequest, stream objectSvc.GetObjectStre return nil, fmt.Errorf("invalid object address: %w", err) } - meta := req.GetMetaHeader() - commonPrm, err := util.CommonPrmFromV2(req) if err != nil { return nil, err @@ -65,134 +62,17 @@ func (s *Service) toPrm(req *objectV2.GetRequest, stream objectSvc.GetObjectStre p.SetObjectWriter(streamWrapper) if !commonPrm.LocalOnly() { - var onceResign sync.Once - - var onceHeaderSending sync.Once - var globalProgress int + forwarder := &getRequestForwarder{ + OnceResign: &sync.Once{}, + OnceHeaderSending: &sync.Once{}, + GlobalProgress: 0, + KeyStorage: s.keyStorage, + Request: req, + Stream: streamWrapper, + } p.SetRequestForwarder(groupAddressRequestForwarder(func(addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { - var err error - - key, err := s.keyStorage.GetKey(nil) - if err != nil { - return nil, err - } - - // once compose and resign forwarding request - onceResign.Do(func() { - // compose meta header of the local server - metaHdr := new(session.RequestMetaHeader) - metaHdr.SetTTL(meta.GetTTL() - 1) - // TODO: #1165 think how to set the other fields - metaHdr.SetOrigin(meta) - writeCurrentVersion(metaHdr) - - req.SetMetaHeader(metaHdr) - - err = signature.SignServiceMessage(key, req) - }) - - if err != nil { - return nil, err - } - - // code below is copy-pasted from c.GetObject implementation, - // perhaps it is worth highlighting the utility function in frostfs-api-go - - // open stream - var getStream *rpc.GetResponseReader - err = c.RawForAddress(addr, func(cli *rpcclient.Client) error { - getStream, err = rpc.GetObject(cli, req, rpcclient.WithContext(stream.Context())) - return err - }) - if err != nil { - return nil, fmt.Errorf("stream opening failed: %w", err) - } - - var ( - headWas bool - resp = new(objectV2.GetResponse) - localProgress int - ) - - for { - // receive message from server stream - err := getStream.Read(resp) - if err != nil { - if errors.Is(err, io.EOF) { - if !headWas { - return nil, io.ErrUnexpectedEOF - } - - break - } - - internalclient.ReportError(c, err) - return nil, fmt.Errorf("reading the response failed: %w", err) - } - - // verify response key - if err = internal.VerifyResponseKeyV2(pubkey, resp); err != nil { - return nil, err - } - - // verify response structure - if err := signature.VerifyServiceMessage(resp); err != nil { - return nil, fmt.Errorf("response verification failed: %w", err) - } - - if err = checkStatus(resp.GetMetaHeader().GetStatus()); err != nil { - return nil, err - } - - switch v := resp.GetBody().GetObjectPart().(type) { - default: - return nil, fmt.Errorf("unexpected object part %T", v) - case *objectV2.GetObjectPartInit: - if headWas { - return nil, errWrongMessageSeq - } - - headWas = true - - obj := new(objectV2.Object) - - obj.SetObjectID(v.GetObjectID()) - obj.SetSignature(v.GetSignature()) - obj.SetHeader(v.GetHeader()) - - onceHeaderSending.Do(func() { - err = streamWrapper.WriteHeader(stream.Context(), object.NewFromV2(obj)) - }) - if err != nil { - return nil, fmt.Errorf("could not write object header in Get forwarder: %w", err) - } - case *objectV2.GetObjectPartChunk: - if !headWas { - return nil, errWrongMessageSeq - } - - origChunk := v.GetChunk() - - chunk := chunkToSend(globalProgress, localProgress, origChunk) - if len(chunk) == 0 { - localProgress += len(origChunk) - continue - } - - if err = streamWrapper.WriteChunk(stream.Context(), chunk); err != nil { - return nil, fmt.Errorf("could not write object chunk in Get forwarder: %w", err) - } - - localProgress += len(origChunk) - globalProgress += len(chunk) - case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - return nil, object.NewSplitInfoError(si) - } - } - - return nil, nil + return forwarder.forward(stream.Context(), addr, c, pubkey) })) } From 6c7b708a98a7e95fbe8777a1abda75eedf589254 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 31 Mar 2023 17:41:42 +0300 Subject: [PATCH 0046/1943] [#193] getsvc: Refactor get range params creation Resolve funlen linter for toRangePrm function. Signed-off-by: Dmitrii Stepanov --- .../object/get/v2/get_range_forwarder.go | 137 ++++++++++++++++++ pkg/services/object/get/v2/util.go | 110 +------------- 2 files changed, 144 insertions(+), 103 deletions(-) create mode 100644 pkg/services/object/get/v2/get_range_forwarder.go diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go new file mode 100644 index 000000000..a9526f714 --- /dev/null +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -0,0 +1,137 @@ +package getsvc + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" + rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" +) + +type getRangeRequestForwarder struct { + OnceResign *sync.Once + GlobalProgress int + KeyStorage *util.KeyStorage + Request *objectV2.GetRangeRequest + Stream *streamObjectRangeWriter +} + +func (f *getRangeRequestForwarder) forward(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { + key, err := f.KeyStorage.GetKey(nil) + if err != nil { + return nil, err + } + + // once compose and resign forwarding request + f.OnceResign.Do(func() { + // compose meta header of the local server + metaHdr := new(session.RequestMetaHeader) + metaHdr.SetTTL(f.Request.GetMetaHeader().GetTTL() - 1) + // TODO: #1165 think how to set the other fields + metaHdr.SetOrigin(f.Request.GetMetaHeader()) + writeCurrentVersion(metaHdr) + + f.Request.SetMetaHeader(metaHdr) + + err = signature.SignServiceMessage(key, f.Request) + }) + + if err != nil { + return nil, err + } + + rangeStream, err := f.openStream(ctx, addr, c) + if err != nil { + return nil, err + } + + return nil, f.readStream(ctx, rangeStream, c, pubkey) +} + +func (f *getRangeRequestForwarder) verifyResponse(resp *objectV2.GetRangeResponse, pubkey []byte) error { + // verify response key + if err := internal.VerifyResponseKeyV2(pubkey, resp); err != nil { + return err + } + + // verify response structure + if err := signature.VerifyServiceMessage(resp); err != nil { + return fmt.Errorf("could not verify %T: %w", resp, err) + } + + if err := checkStatus(resp.GetMetaHeader().GetStatus()); err != nil { + return err + } + return nil +} + +func (f *getRangeRequestForwarder) openStream(ctx context.Context, addr network.Address, c client.MultiAddressClient) (*rpc.ObjectRangeResponseReader, error) { + // open stream + var rangeStream *rpc.ObjectRangeResponseReader + err := c.RawForAddress(addr, func(cli *rpcclient.Client) error { + var e error + rangeStream, e = rpc.GetObjectRange(cli, f.Request, rpcclient.WithContext(ctx)) + return e + }) + if err != nil { + return nil, fmt.Errorf("could not create Get payload range stream: %w", err) + } + return rangeStream, nil +} + +func (f *getRangeRequestForwarder) readStream(ctx context.Context, rangeStream *rpc.ObjectRangeResponseReader, c client.MultiAddressClient, pubkey []byte) error { + resp := new(objectV2.GetRangeResponse) + var localProgress int + + for { + // receive message from server stream + err := rangeStream.Read(resp) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + internalclient.ReportError(c, err) + return fmt.Errorf("reading the response failed: %w", err) + } + + if err := f.verifyResponse(resp, pubkey); err != nil { + return err + } + + switch v := resp.GetBody().GetRangePart().(type) { + case nil: + return fmt.Errorf("unexpected range type %T", v) + case *objectV2.GetRangePartChunk: + origChunk := v.GetChunk() + + chunk := chunkToSend(f.GlobalProgress, localProgress, origChunk) + if len(chunk) == 0 { + localProgress += len(origChunk) + continue + } + + if err = f.Stream.WriteChunk(ctx, chunk); err != nil { + return fmt.Errorf("could not write object chunk in GetRange forwarder: %w", err) + } + + localProgress += len(origChunk) + f.GlobalProgress += len(chunk) + case *objectV2.SplitInfo: + si := object.NewSplitInfoFromV2(v) + return object.NewSplitInfoError(si) + } + } + return nil +} diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 3a9cf3b07..dffa0d9b1 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -6,22 +6,16 @@ import ( "errors" "fmt" "hash" - "io" "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" - rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/status" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -79,7 +73,6 @@ func (s *Service) toPrm(req *objectV2.GetRequest, stream objectSvc.GetObjectStre return p, nil } -// nolint: funlen, gocognit func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.GetObjectRangeStream) (*getsvc.RangePrm, error) { body := req.GetBody() @@ -95,8 +88,6 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get return nil, fmt.Errorf("invalid object address: %w", err) } - meta := req.GetMetaHeader() - commonPrm, err := util.CommonPrmFromV2(req) if err != nil { return nil, err @@ -118,103 +109,16 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get } if !commonPrm.LocalOnly() { - var onceResign sync.Once - var globalProgress int - - key, err := s.keyStorage.GetKey(nil) - if err != nil { - return nil, err + forwarder := &getRangeRequestForwarder{ + OnceResign: &sync.Once{}, + GlobalProgress: 0, + KeyStorage: s.keyStorage, + Request: req, + Stream: streamWrapper, } p.SetRequestForwarder(groupAddressRequestForwarder(func(addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { - var err error - - // once compose and resign forwarding request - onceResign.Do(func() { - // compose meta header of the local server - metaHdr := new(session.RequestMetaHeader) - metaHdr.SetTTL(meta.GetTTL() - 1) - // TODO: #1165 think how to set the other fields - metaHdr.SetOrigin(meta) - writeCurrentVersion(metaHdr) - - req.SetMetaHeader(metaHdr) - - err = signature.SignServiceMessage(key, req) - }) - - if err != nil { - return nil, err - } - - // code below is copy-pasted from c.ObjectPayloadRangeData implementation, - // perhaps it is worth highlighting the utility function in frostfs-api-go - - // open stream - var rangeStream *rpc.ObjectRangeResponseReader - err = c.RawForAddress(addr, func(cli *rpcclient.Client) error { - rangeStream, err = rpc.GetObjectRange(cli, req, rpcclient.WithContext(stream.Context())) - return err - }) - if err != nil { - return nil, fmt.Errorf("could not create Get payload range stream: %w", err) - } - - resp := new(objectV2.GetRangeResponse) - var localProgress int - - for { - // receive message from server stream - err := rangeStream.Read(resp) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - - internalclient.ReportError(c, err) - return nil, fmt.Errorf("reading the response failed: %w", err) - } - - // verify response key - if err = internal.VerifyResponseKeyV2(pubkey, resp); err != nil { - return nil, err - } - - // verify response structure - if err := signature.VerifyServiceMessage(resp); err != nil { - return nil, fmt.Errorf("could not verify %T: %w", resp, err) - } - - if err = checkStatus(resp.GetMetaHeader().GetStatus()); err != nil { - return nil, err - } - - switch v := resp.GetBody().GetRangePart().(type) { - case nil: - return nil, fmt.Errorf("unexpected range type %T", v) - case *objectV2.GetRangePartChunk: - origChunk := v.GetChunk() - - chunk := chunkToSend(globalProgress, localProgress, origChunk) - if len(chunk) == 0 { - localProgress += len(origChunk) - continue - } - - if err = streamWrapper.WriteChunk(stream.Context(), chunk); err != nil { - return nil, fmt.Errorf("could not write object chunk in GetRange forwarder: %w", err) - } - - localProgress += len(origChunk) - globalProgress += len(chunk) - case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - - return nil, object.NewSplitInfoError(si) - } - } - - return nil, nil + return forwarder.forward(stream.Context(), addr, c, pubkey) })) } From 89924071cd2f06631af0bfac88f32fcfab10a34d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 11:17:59 +0300 Subject: [PATCH 0047/1943] [#193] getsvc: Edit request forwarder signature Pass context to forwarder direct, without closure. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/prm.go | 2 +- pkg/services/object/get/util.go | 2 +- pkg/services/object/get/v2/get_forwarder.go | 2 +- .../object/get/v2/get_range_forwarder.go | 2 +- pkg/services/object/get/v2/head_forwarder.go | 2 +- pkg/services/object/get/v2/util.go | 18 ++++++------------ 6 files changed, 11 insertions(+), 17 deletions(-) diff --git a/pkg/services/object/get/prm.go b/pkg/services/object/get/prm.go index 88848264e..7a0f1e062 100644 --- a/pkg/services/object/get/prm.go +++ b/pkg/services/object/get/prm.go @@ -59,7 +59,7 @@ type RangeHashPrm struct { salt []byte } -type RequestForwarder func(coreclient.NodeInfo, coreclient.MultiAddressClient) (*object.Object, error) +type RequestForwarder func(context.Context, coreclient.NodeInfo, coreclient.MultiAddressClient) (*object.Object, error) // HeadPrm groups parameters of Head service call. type HeadPrm struct { diff --git a/pkg/services/object/get/util.go b/pkg/services/object/get/util.go index a68232c9a..7986d05c0 100644 --- a/pkg/services/object/get/util.go +++ b/pkg/services/object/get/util.go @@ -89,7 +89,7 @@ func (c *clientCacheWrapper) get(info coreclient.NodeInfo) (getClient, error) { func (c *clientWrapper) getObject(ctx context.Context, exec *execCtx, info coreclient.NodeInfo) (*object.Object, error) { if exec.isForwardingEnabled() { - return exec.prm.forwarder(info, c.client) + return exec.prm.forwarder(ctx, info, c.client) } key, err := exec.key() diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index b0ba47523..7314cceb5 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -29,7 +29,7 @@ type getRequestForwarder struct { Stream *streamObjectWriter } -func (f *getRequestForwarder) forward(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *getRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { key, err := f.KeyStorage.GetKey(nil) if err != nil { return nil, err diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index a9526f714..8fa4351d4 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -28,7 +28,7 @@ type getRangeRequestForwarder struct { Stream *streamObjectRangeWriter } -func (f *getRangeRequestForwarder) forward(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *getRangeRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { key, err := f.KeyStorage.GetKey(nil) if err != nil { return nil, err diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index b38da7131..e0b58a35d 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -29,7 +29,7 @@ type headRequestForwarder struct { KeyStorage *util.KeyStorage } -func (f *headRequestForwarder) forward(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { var err error key, err := f.KeyStorage.GetKey(nil) diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index dffa0d9b1..75228e2b1 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -65,9 +65,7 @@ func (s *Service) toPrm(req *objectV2.GetRequest, stream objectSvc.GetObjectStre Stream: streamWrapper, } - p.SetRequestForwarder(groupAddressRequestForwarder(func(addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { - return forwarder.forward(stream.Context(), addr, c, pubkey) - })) + p.SetRequestForwarder(groupAddressRequestForwarder(forwarder.forwardRequestToNode)) } return p, nil @@ -117,9 +115,7 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get Stream: streamWrapper, } - p.SetRequestForwarder(groupAddressRequestForwarder(func(addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { - return forwarder.forward(stream.Context(), addr, c, pubkey) - })) + p.SetRequestForwarder(groupAddressRequestForwarder(forwarder.forwardRequestToNode)) } return p, nil @@ -251,9 +247,7 @@ func (s *Service) toHeadPrm(ctx context.Context, req *objectV2.HeadRequest, resp KeyStorage: s.keyStorage, } - p.SetRequestForwarder(groupAddressRequestForwarder(func(addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { - return forwarder.forward(ctx, addr, c, pubkey) - })) + p.SetRequestForwarder(groupAddressRequestForwarder(forwarder.forwardRequestToNode)) return p, nil } @@ -321,8 +315,8 @@ func toShortObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { return sh } -func groupAddressRequestForwarder(f func(network.Address, client.MultiAddressClient, []byte) (*object.Object, error)) getsvc.RequestForwarder { - return func(info client.NodeInfo, c client.MultiAddressClient) (*object.Object, error) { +func groupAddressRequestForwarder(f func(context.Context, network.Address, client.MultiAddressClient, []byte) (*object.Object, error)) getsvc.RequestForwarder { + return func(ctx context.Context, info client.NodeInfo, c client.MultiAddressClient) (*object.Object, error) { var ( firstErr error res *object.Object @@ -343,7 +337,7 @@ func groupAddressRequestForwarder(f func(network.Address, client.MultiAddressCli // would be nice to log otherwise }() - res, err = f(addr, c, key) + res, err = f(ctx, addr, c, key) return }) From c58ab0c3693078ecf527abf0b2955b0e5e021a6b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 13:20:57 +0300 Subject: [PATCH 0048/1943] [#193] getsvc: Reduce private key requests Get private key only once for request forwaring. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/v2/get_forwarder.go | 11 ++++------ .../object/get/v2/get_range_forwarder.go | 11 ++++------ pkg/services/object/get/v2/head_forwarder.go | 11 +++------- pkg/services/object/get/v2/util.go | 21 ++++++++++++++++--- 4 files changed, 29 insertions(+), 25 deletions(-) diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index 7314cceb5..330a0642f 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -2,6 +2,7 @@ package getsvc import ( "context" + "crypto/ecdsa" "errors" "fmt" "io" @@ -16,7 +17,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) @@ -24,16 +24,13 @@ type getRequestForwarder struct { OnceResign *sync.Once OnceHeaderSending *sync.Once GlobalProgress int - KeyStorage *util.KeyStorage + Key *ecdsa.PrivateKey Request *objectV2.GetRequest Stream *streamObjectWriter } func (f *getRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { - key, err := f.KeyStorage.GetKey(nil) - if err != nil { - return nil, err - } + var err error // once compose and resign forwarding request f.OnceResign.Do(func() { @@ -44,7 +41,7 @@ func (f *getRequestForwarder) forwardRequestToNode(ctx context.Context, addr net metaHdr.SetOrigin(f.Request.GetMetaHeader()) writeCurrentVersion(metaHdr) f.Request.SetMetaHeader(metaHdr) - err = signature.SignServiceMessage(key, f.Request) + err = signature.SignServiceMessage(f.Key, f.Request) }) if err != nil { diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 8fa4351d4..5893f8de3 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -2,6 +2,7 @@ package getsvc import ( "context" + "crypto/ecdsa" "errors" "fmt" "io" @@ -16,23 +17,19 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) type getRangeRequestForwarder struct { OnceResign *sync.Once GlobalProgress int - KeyStorage *util.KeyStorage + Key *ecdsa.PrivateKey Request *objectV2.GetRangeRequest Stream *streamObjectRangeWriter } func (f *getRangeRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { - key, err := f.KeyStorage.GetKey(nil) - if err != nil { - return nil, err - } + var err error // once compose and resign forwarding request f.OnceResign.Do(func() { @@ -45,7 +42,7 @@ func (f *getRangeRequestForwarder) forwardRequestToNode(ctx context.Context, add f.Request.SetMetaHeader(metaHdr) - err = signature.SignServiceMessage(key, f.Request) + err = signature.SignServiceMessage(f.Key, f.Request) }) if err != nil { diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index e0b58a35d..45c0174fd 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -2,6 +2,7 @@ package getsvc import ( "context" + "crypto/ecdsa" "errors" "fmt" "sync" @@ -15,7 +16,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -26,17 +26,12 @@ type headRequestForwarder struct { Response *objectV2.HeadResponse OnceResign *sync.Once ObjectAddr oid.Address - KeyStorage *util.KeyStorage + Key *ecdsa.PrivateKey } func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { var err error - key, err := f.KeyStorage.GetKey(nil) - if err != nil { - return nil, err - } - // once compose and resign forwarding request f.OnceResign.Do(func() { // compose meta header of the local server @@ -48,7 +43,7 @@ func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr ne f.Request.SetMetaHeader(metaHdr) - err = signature.SignServiceMessage(key, f.Request) + err = signature.SignServiceMessage(f.Key, f.Request) }) if err != nil { diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 75228e2b1..3a50a6ca5 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -56,11 +56,16 @@ func (s *Service) toPrm(req *objectV2.GetRequest, stream objectSvc.GetObjectStre p.SetObjectWriter(streamWrapper) if !commonPrm.LocalOnly() { + key, err := s.keyStorage.GetKey(nil) + if err != nil { + return nil, err + } + forwarder := &getRequestForwarder{ OnceResign: &sync.Once{}, OnceHeaderSending: &sync.Once{}, GlobalProgress: 0, - KeyStorage: s.keyStorage, + Key: key, Request: req, Stream: streamWrapper, } @@ -107,10 +112,15 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get } if !commonPrm.LocalOnly() { + key, err := s.keyStorage.GetKey(nil) + if err != nil { + return nil, err + } + forwarder := &getRangeRequestForwarder{ OnceResign: &sync.Once{}, GlobalProgress: 0, - KeyStorage: s.keyStorage, + Key: key, Request: req, Stream: streamWrapper, } @@ -239,12 +249,17 @@ func (s *Service) toHeadPrm(ctx context.Context, req *objectV2.HeadRequest, resp return p, nil } + key, err := s.keyStorage.GetKey(nil) + if err != nil { + return nil, err + } + forwarder := &headRequestForwarder{ Request: req, Response: resp, OnceResign: &sync.Once{}, ObjectAddr: objAddr, - KeyStorage: s.keyStorage, + Key: key, } p.SetRequestForwarder(groupAddressRequestForwarder(forwarder.forwardRequestToNode)) From ab891517de1a1a34c565bb3c59e0e1cf4eac6515 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 5 Apr 2023 18:47:11 +0300 Subject: [PATCH 0049/1943] [#116] node: Fix bug with extra generated files in TestReload * Create testNewEngine in engineWithShards without default opts Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/local_object_storage/engine/control_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 2c44eb169..12771340b 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" "go.etcd.io/bbolt" "go.uber.org/atomic" + "go.uber.org/zap" "go.uber.org/zap/zaptest" ) @@ -321,8 +322,9 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str currShards := make([]string, 0, num) te := testNewEngine(t). - setShardsNumAdditionalOpts(t, num, func(id int) []shard.Option { + setShardsNumOpts(t, num, func(id int) []shard.Option { return []shard.Option{ + shard.WithLogger(&logger.Logger{Logger: zap.L()}), shard.WithBlobStorOptions( blobstor.WithStorages(newStorages(filepath.Join(addPath, strconv.Itoa(id)), errSmallSize))), shard.WithMetaBaseOptions( From 9e2df4b7c73884368b2e6a06c5a55ff51d59f65a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 16:11:56 +0300 Subject: [PATCH 0050/1943] [#203] node: Fix double imports Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/head.go | 3 +-- pkg/innerring/initialization.go | 15 +++++++-------- pkg/local_object_storage/metabase/db_test.go | 5 ++--- pkg/local_object_storage/metabase/put.go | 9 ++++----- pkg/local_object_storage/pilorama/forest.go | 19 +++++++++---------- pkg/local_object_storage/shard/gc_test.go | 5 ++--- pkg/morph/client/container/get.go | 3 +-- pkg/services/object/get/util.go | 3 +-- .../object_manager/tombstone/source/source.go | 3 +-- pkg/services/tree/sync.go | 5 ++--- 10 files changed, 30 insertions(+), 40 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index fc2e7c9c6..139563e24 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -15,7 +15,6 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - oidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -113,7 +112,7 @@ func marshalHeader(cmd *cobra.Command, hdr *object.Object) ([]byte, error) { } } -func printObjectID(cmd *cobra.Command, recv func() (oidSDK.ID, bool)) { +func printObjectID(cmd *cobra.Command, recv func() (oid.ID, bool)) { var strID string id, ok := recv() diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 30b8d43a0..8db6328a2 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -25,7 +25,6 @@ import ( auditClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/audit" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" frostfsClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -229,7 +228,7 @@ func (s *Server) createSettlementProcessor(clientCache *ClientCache, cnrClient * // create settlement processor dependencies settlementDeps := settlementDeps{ log: s.log, - cnrSrc: cntClient.AsContainerSource(cnrClient), + cnrSrc: container.AsContainerSource(cnrClient), auditClient: s.auditClient, nmClient: s.netmapClient, clientCache: clientCache, @@ -545,7 +544,7 @@ func (s *Server) initGRPCServer(cfg *viper.Viper) error { } type serverMorphClients struct { - CnrClient *cntClient.Client + CnrClient *container.Client FrostFSIDClient *frostfsid.Client FrostFSClient *frostfsClient.Client MorphSubnetClient *morphsubnet.Client @@ -564,21 +563,21 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { } // form morph container client's options - morphCnrOpts := make([]cntClient.Option, 0, 3) + morphCnrOpts := make([]container.Option, 0, 3) morphCnrOpts = append(morphCnrOpts, - cntClient.TryNotary(), - cntClient.AsAlphabet(), + container.TryNotary(), + container.AsAlphabet(), ) if s.sideNotaryConfig.disabled { // in non-notary environments we customize fee for named container registration // because it takes much more additional GAS than other operations. morphCnrOpts = append(morphCnrOpts, - cntClient.WithCustomFeeForNamedPut(s.feeConfig.NamedContainerRegistrationFee()), + container.WithCustomFeeForNamedPut(s.feeConfig.NamedContainerRegistrationFee()), ) } - result.CnrClient, err = cntClient.NewFromMorph(s.morphClient, s.contracts.container, fee, morphCnrOpts...) + result.CnrClient, err = container.NewFromMorph(s.morphClient, s.contracts.container, fee, morphCnrOpts...) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index cdb90cc16..9ef7bf8bc 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" @@ -26,11 +25,11 @@ func (s epochState) CurrentEpoch() uint64 { } // saves "big" object in DB. -func putBig(db *meta.DB, obj *object.Object) error { +func putBig(db *meta.DB, obj *objectSDK.Object) error { return metaPut(db, obj, nil) } -func testSelect(t *testing.T, db *meta.DB, cnr cid.ID, fs object.SearchFilters, exp ...oid.Address) { +func testSelect(t *testing.T, db *meta.DB, cnr cid.ID, fs objectSDK.SearchFilters, exp ...oid.Address) { res, err := metaSelect(db, cnr, fs) require.NoError(t, err) require.Len(t, res, len(exp)) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 8e11c5d9c..b0fea6535 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -6,7 +6,6 @@ import ( "fmt" gio "io" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" @@ -89,7 +88,7 @@ func (db *DB) put(tx *bbolt.Tx, isParent := si != nil - exists, err := db.exists(tx, object.AddressOf(obj), currEpoch) + exists, err := db.exists(tx, objectCore.AddressOf(obj), currEpoch) if errors.As(err, &splitInfoError) { exists = true // object exists, however it is virtual @@ -111,14 +110,14 @@ func (db *DB) updateObj(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *obje // When storage engine moves objects between different sub-storages, // it calls metabase.Put method with new storage ID, thus triggering this code. if !isParent && id != nil { - return updateStorageID(tx, object.AddressOf(obj), id) + return updateStorageID(tx, objectCore.AddressOf(obj), id) } // when storage already has last object in split hierarchy and there is // a linking object to put (or vice versa), we should update split info // with object ids of these objects if isParent { - return updateSplitInfo(tx, object.AddressOf(obj), si) + return updateSplitInfo(tx, objectCore.AddressOf(obj), si) } return nil @@ -184,7 +183,7 @@ func putUniqueIndexes( id []byte, ) error { isParent := si != nil - addr := object.AddressOf(obj) + addr := objectCore.AddressOf(obj) cnr := addr.Container() objKey := objectKey(addr.Object(), make([]byte, objectKeySize)) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 21209420a..fa2f1dcd2 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -6,7 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) // memoryForest represents multiple replicating trees sharing a single storage. @@ -120,7 +119,7 @@ func (f *memoryForest) Close() error { } // TreeGetByPath implements the Forest interface. -func (f *memoryForest) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { +func (f *memoryForest) TreeGetByPath(cid cid.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { if !isAttributeInternal(attr) { return nil, ErrNotPathAttribute } @@ -135,7 +134,7 @@ func (f *memoryForest) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, } // TreeGetMeta implements the Forest interface. -func (f *memoryForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) { +func (f *memoryForest) TreeGetMeta(cid cid.ID, treeID string, nodeID Node) (Meta, Node, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -146,7 +145,7 @@ func (f *memoryForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (M } // TreeGetChildren implements the Forest interface. -func (f *memoryForest) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) { +func (f *memoryForest) TreeGetChildren(cid cid.ID, treeID string, nodeID Node) ([]uint64, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -164,7 +163,7 @@ func (f *memoryForest) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node } // TreeGetOpLog implements the pilorama.Forest interface. -func (f *memoryForest) TreeGetOpLog(cid cidSDK.ID, treeID string, height uint64) (Move, error) { +func (f *memoryForest) TreeGetOpLog(cid cid.ID, treeID string, height uint64) (Move, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -181,7 +180,7 @@ func (f *memoryForest) TreeGetOpLog(cid cidSDK.ID, treeID string, height uint64) } // TreeDrop implements the pilorama.Forest interface. -func (f *memoryForest) TreeDrop(cid cidSDK.ID, treeID string) error { +func (f *memoryForest) TreeDrop(cid cid.ID, treeID string) error { cidStr := cid.String() if treeID == "" { for k := range f.treeMap { @@ -201,7 +200,7 @@ func (f *memoryForest) TreeDrop(cid cidSDK.ID, treeID string) error { } // TreeList implements the pilorama.Forest interface. -func (f *memoryForest) TreeList(cid cidSDK.ID) ([]string, error) { +func (f *memoryForest) TreeList(cid cid.ID) ([]string, error) { var res []string cidStr := cid.EncodeToString() @@ -218,14 +217,14 @@ func (f *memoryForest) TreeList(cid cidSDK.ID) ([]string, error) { } // TreeExists implements the pilorama.Forest interface. -func (f *memoryForest) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { +func (f *memoryForest) TreeExists(cid cid.ID, treeID string) (bool, error) { fullID := cid.EncodeToString() + "/" + treeID _, ok := f.treeMap[fullID] return ok, nil } // TreeUpdateLastSyncHeight implements the pilorama.Forest interface. -func (f *memoryForest) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height uint64) error { +func (f *memoryForest) TreeUpdateLastSyncHeight(cid cid.ID, treeID string, height uint64) error { fullID := cid.EncodeToString() + "/" + treeID t, ok := f.treeMap[fullID] if !ok { @@ -236,7 +235,7 @@ func (f *memoryForest) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, he } // TreeLastSyncHeight implements the pilorama.Forest interface. -func (f *memoryForest) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) { +func (f *memoryForest) TreeLastSyncHeight(cid cid.ID, treeID string) (uint64, error) { fullID := cid.EncodeToString() + "/" + treeID t, ok := f.treeMap[fullID] if !ok { diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 245669ff1..e7aa3614e 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -17,7 +17,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/panjf2000/ants/v2" @@ -42,7 +41,7 @@ func Test_GCDropsLockedExpiredObject(t *testing.T) { blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(2), blobovniczatree.WithBlobovniczaShallowWidth(2)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return len(data) <= 1<<20 }, }, @@ -93,7 +92,7 @@ func Test_GCDropsLockedExpiredObject(t *testing.T) { lockExpirationAttr.SetValue("103") lock := testutil.GenerateObjectWithCID(cnr) - lock.SetType(object.TypeLock) + lock.SetType(objectSDK.TypeLock) lock.SetAttributes(lockExpirationAttr) lockID, _ := lock.ID() diff --git a/pkg/morph/client/container/get.go b/pkg/morph/client/container/get.go index 8d68141a5..4775cd281 100644 --- a/pkg/morph/client/container/get.go +++ b/pkg/morph/client/container/get.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" containerContract "git.frostfs.info/TrueCloudLab/frostfs-contract/container" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -23,7 +22,7 @@ func (x *containerSource) Get(cnr cid.ID) (*containercore.Container, error) { // AsContainerSource provides container Source interface // from Wrapper instance. -func AsContainerSource(w *Client) core.Source { +func AsContainerSource(w *Client) containercore.Source { return (*containerSource)(w) } diff --git a/pkg/services/object/get/util.go b/pkg/services/object/get/util.go index 7986d05c0..3bb68862b 100644 --- a/pkg/services/object/get/util.go +++ b/pkg/services/object/get/util.go @@ -9,7 +9,6 @@ import ( coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - internal "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -196,7 +195,7 @@ func (c *clientWrapper) get(ctx context.Context, exec *execCtx, key *ecdsa.Priva prm.SetRawFlag() } - res, err := internal.GetObject(prm) + res, err := internalclient.GetObject(prm) if err != nil { return nil, err } diff --git a/pkg/services/object_manager/tombstone/source/source.go b/pkg/services/object_manager/tombstone/source/source.go index 8dd0dcabb..4d0a81cde 100644 --- a/pkg/services/object_manager/tombstone/source/source.go +++ b/pkg/services/object_manager/tombstone/source/source.go @@ -8,7 +8,6 @@ import ( getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -59,7 +58,7 @@ func (h *headerWriter) WriteHeader(_ context.Context, o *objectSDK.Object) error // Tombstone checks if the engine stores tombstone. // Returns nil, nil if the tombstone has been removed // or marked for removal. -func (s Source) Tombstone(ctx context.Context, a oid.Address, _ uint64) (*object.Object, error) { +func (s Source) Tombstone(ctx context.Context, a oid.Address, _ uint64) (*objectSDK.Object, error) { var hr headerWriter var headPrm getsvc.HeadPrm diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index d4ef7df5d..2d5c104b0 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -14,7 +14,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" "go.uber.org/zap" @@ -123,7 +122,7 @@ func (s *Service) SynchronizeTree(ctx context.Context, cid cid.ID, treeID string return nil } -func (s *Service) synchronizeTree(ctx context.Context, cid cidSDK.ID, from uint64, +func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, treeID string, nodes []netmapSDK.NodeInfo) uint64 { s.log.Debug("synchronize tree", zap.Stringer("cid", cid), @@ -170,7 +169,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cidSDK.ID, from uint6 return newHeight } -func (s *Service) synchronizeSingle(ctx context.Context, cid cidSDK.ID, treeID string, height uint64, treeClient TreeServiceClient) (uint64, error) { +func (s *Service) synchronizeSingle(ctx context.Context, cid cid.ID, treeID string, height uint64, treeClient TreeServiceClient) (uint64, error) { rawCID := make([]byte, sha256.Size) cid.Encode(rawCID) From 68a2f36636fea51aaf62e699811641525fdc1e2c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 16:30:05 +0300 Subject: [PATCH 0051/1943] [#203] morph: Mark depracated methods Skip staticcheck for depracated methods. Will be fixed soon. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-adm/internal/modules/morph/initialize_nns.go | 1 + cmd/frostfs-adm/internal/modules/morph/initialize_register.go | 1 + cmd/frostfs-adm/internal/modules/morph/initialize_test.go | 4 ++-- pkg/morph/client/notary.go | 4 ++++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go index b43c2da33..edb7d6de5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go @@ -286,6 +286,7 @@ func parseNNSResolveResult(res stackitem.Item) (util.Uint160, error) { func nnsIsAvailable(c Client, nnsHash util.Uint160, name string) (bool, error) { switch ct := c.(type) { case *rpcclient.Client: + //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 return ct.NNSIsAvailable(nnsHash, name) default: b, err := unwrap.Bool(invokeFunction(c, nnsHash, "isAvailable", []any{name}, nil)) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index 1bfda7b54..27e1590cf 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -118,6 +118,7 @@ var errGetPriceInvalid = errors.New("`getRegisterPrice`: invalid response") func (c *initializeContext) getCandidateRegisterPrice() (int64, error) { switch ct := c.Client.(type) { case *rpcclient.Client: + //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 return ct.GetCandidateRegisterPrice() default: neoHash := neo.Hash diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index 3c6adfc3c..39a35b12e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -101,11 +101,11 @@ func generateTestData(t *testing.T, dir string, size int) { cfg := config.Config{} cfg.ProtocolConfiguration.Magic = 12345 cfg.ProtocolConfiguration.ValidatorsCount = size - cfg.ProtocolConfiguration.SecondsPerBlock = 1 + cfg.ProtocolConfiguration.SecondsPerBlock = 1 //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 cfg.ProtocolConfiguration.StandbyCommittee = pubs // sorted by glagolic letters cfg.ProtocolConfiguration.P2PSigExtensions = true cfg.ProtocolConfiguration.VerifyTransactions = true - cfg.ProtocolConfiguration.VerifyBlocks = true + cfg.ProtocolConfiguration.VerifyBlocks = true //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 data, err := yaml.Marshal(cfg) require.NoError(t, err) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 464d76d2a..9c7071e5a 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -423,6 +423,7 @@ func (c *Client) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { multiaddrAccount.PrivateKey().SignHashable(uint32(magicNumber), mainTx)..., ) + //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 resp, err := c.client.SignAndPushP2PNotaryRequest(mainTx, []byte{byte(opcode.RET)}, -1, @@ -518,6 +519,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint } // calculate notary fee + //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 notaryFee, err := c.client.CalculateNotaryFee(u8n) if err != nil { return err @@ -525,6 +527,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint // add network fee for cosigners //nolint:staticcheck // waits for neo-go v0.99.3 with notary actors + //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 err = c.client.AddNetworkFee( mainTx, notaryFee, @@ -537,6 +540,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint // define witnesses mainTx.Scripts = c.notaryWitnesses(invokedByAlpha, multiaddrAccount, mainTx) + //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 resp, err := c.client.SignAndPushP2PNotaryRequest(mainTx, []byte{byte(opcode.RET)}, -1, From e21c5bea21ae820433006fcd8d4dfe1e18ea754a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 16:39:29 +0300 Subject: [PATCH 0052/1943] [#203] cli: Fix error message Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/container/delete.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/delete.go b/cmd/frostfs-cli/modules/container/delete.go index 52ee5d4fb..308c7b942 100644 --- a/cmd/frostfs-cli/modules/container/delete.go +++ b/cmd/frostfs-cli/modules/container/delete.go @@ -77,8 +77,8 @@ Only owner of the container has a permission to remove container.`, if len(res.IDList()) != 0 { commonCmd.ExitOnErr(cmd, "", - fmt.Errorf("Container wasn't removed because LOCK objects were found.\n"+ - "Use --%s flag to remove anyway.", commonflags.ForceFlag)) + fmt.Errorf("container wasn't removed because LOCK objects were found, "+ + "use --%s flag to remove anyway", commonflags.ForceFlag)) } } } From bab11492adae90644d6b9ed2f1d440ed705fb449 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 16:53:55 +0300 Subject: [PATCH 0053/1943] [#203] node: Resolve never used errors Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/delete_test.go | 9 +++++++-- pkg/local_object_storage/metabase/exists_test.go | 1 + pkg/local_object_storage/pilorama/forest_test.go | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/metabase/delete_test.go b/pkg/local_object_storage/metabase/delete_test.go index eae8b1c66..ee161a881 100644 --- a/pkg/local_object_storage/metabase/delete_test.go +++ b/pkg/local_object_storage/metabase/delete_test.go @@ -50,6 +50,11 @@ func TestDB_Delete(t *testing.T) { err = metaInhume(db, object.AddressOf(child), object.AddressOf(ts)) require.NoError(t, err) + ts = testutil.GenerateObjectWithCID(cnr) + + err = metaInhume(db, object.AddressOf(parent), object.AddressOf(ts)) + require.NoError(t, err) + // delete object err = metaDelete(db, object.AddressOf(child)) require.NoError(t, err) @@ -62,11 +67,11 @@ func TestDB_Delete(t *testing.T) { // check if they marked as already removed ok, err := metaExists(db, object.AddressOf(child)) - require.Error(t, apistatus.ObjectAlreadyRemoved{}) + require.ErrorAs(t, err, &apistatus.ObjectAlreadyRemoved{}) require.False(t, ok) ok, err = metaExists(db, object.AddressOf(parent)) - require.Error(t, apistatus.ObjectAlreadyRemoved{}) + require.ErrorAs(t, err, &apistatus.ObjectAlreadyRemoved{}) require.False(t, ok) } diff --git a/pkg/local_object_storage/metabase/exists_test.go b/pkg/local_object_storage/metabase/exists_test.go index 66f8c2bb3..e344e9ee8 100644 --- a/pkg/local_object_storage/metabase/exists_test.go +++ b/pkg/local_object_storage/metabase/exists_test.go @@ -181,6 +181,7 @@ func TestDB_Exists(t *testing.T) { require.ErrorIs(t, err, meta.ErrObjectIsExpired) gotObj, err = metaExists(db, object.AddressOf(nonExp)) + require.NoError(t, err) require.True(t, gotObj) }) }) diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 8e6f12717..ebf801ef9 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -225,6 +225,7 @@ func testForestTreeDrop(t *testing.T, s Forest) { } } list, err := s.TreeList(cid) + require.NoError(t, err) require.NotEmpty(t, list) require.NoError(t, s.TreeDrop(cid, "")) From 8908798f59285d18cb52025dd5541dc29d8643db Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 16:54:57 +0300 Subject: [PATCH 0054/1943] [#203] node: Resolve unused vars Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/get_test.go | 2 -- pkg/local_object_storage/shard/metrics_test.go | 1 - 2 files changed, 3 deletions(-) diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index d647910d5..a242a099a 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -185,8 +185,6 @@ func BenchmarkGet(b *testing.B) { } } -var obj *objectSDK.Object - func benchmarkGet(b *testing.B, numOfObj int) { prepareDb := func(batchSize int) (*meta.DB, []oid.Address) { db := newDB(b, diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index ba46881e2..18e97e259 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -70,7 +70,6 @@ func (m *metricsStore) AddToPayloadSize(size int64) { const physical = "phy" const logical = "logic" -const readonly = "readonly" func TestCounters(t *testing.T) { dir := t.TempDir() From f32f61df8759670277818db1de3e8bdd559b3e73 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 16:56:06 +0300 Subject: [PATCH 0055/1943] [#203] pilorama: Refactor tests Do not pass 0 as channel capacity. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/pilorama/forest_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index ebf801ef9..bbd35246c 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -747,7 +747,7 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ actual := constructor(t, WithMaxBatchSize(batchSize)) wg := new(sync.WaitGroup) - ch := make(chan *Move, 0) + ch := make(chan *Move) for i := 0; i < batchSize; i++ { wg.Add(1) go func() { From 4ec69cbbf81e26cbb4fda91dbe3592b83d10f5dc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 16:58:37 +0300 Subject: [PATCH 0056/1943] [#203] control: Fix test Fix shard compare loop. Signed-off-by: Dmitrii Stepanov --- pkg/services/control/service_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/services/control/service_test.go b/pkg/services/control/service_test.go index f39a304f9..8c96e2b73 100644 --- a/pkg/services/control/service_test.go +++ b/pkg/services/control/service_test.go @@ -85,7 +85,9 @@ func equalListShardResponseBodies(b1, b2 *control.ListShardsResponse_Body) bool info1 := b1.Shards[i].GetBlobstor() info2 := b2.Shards[i].GetBlobstor() - return compareBlobstorInfo(info1, info2) + if !compareBlobstorInfo(info1, info2) { + return false + } } for i := range b1.Shards { From 90276953719d4cf74a94a2fc10d98f1fac0aa4d9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Apr 2023 17:05:10 +0300 Subject: [PATCH 0057/1943] [#203] node: Add staticcheck target and pre-commit Signed-off-by: Dmitrii Stepanov --- .pre-commit-config.yaml | 5 +++++ Makefile | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 169b7bf1c..db9930e91 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,3 +43,8 @@ repos: pass_filenames: false types: [go] language: system + + - repo: https://github.com/TekWizely/pre-commit-golang + rev: v1.0.0-rc.1 + hooks: + - id: go-staticcheck-repo-mod diff --git a/Makefile b/Makefile index 242a7ed1c..7d0f8d9e2 100755 --- a/Makefile +++ b/Makefile @@ -128,10 +128,17 @@ test: @echo "⇒ Running go test" @go test ./... +pre-commit-run: + @pre-commit run -a --hook-stage manual + # Run linters lint: @golangci-lint --timeout=5m run +# Run staticcheck +staticcheck: + @staticcheck ./... + # Run linters in Docker docker/lint: docker run --rm -t \ From 56282edf02ba3494c922fa6133e2671b3f8e59e2 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 5 Apr 2023 14:56:15 +0300 Subject: [PATCH 0058/1943] [#166] node: Parallelize background tree service sync * Run sync task for nodes in parallel within errgroup worker pool Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/services/tree/sync.go | 72 +++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 2d5c104b0..32d088c01 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -17,6 +17,7 @@ import ( netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" "go.uber.org/zap" + "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -129,37 +130,58 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, zap.String("tree", treeID), zap.Uint64("from", from)) - newHeight := uint64(math.MaxUint64) - for _, n := range nodes { - height := from - n.IterateNetworkEndpoints(func(addr string) bool { - var a network.Address - if err := a.FromString(addr); err != nil { - return false - } + errGroup, egCtx := errgroup.WithContext(ctx) + const workersCount = 4 + errGroup.SetLimit(workersCount) - cc, err := grpc.DialContext(ctx, a.URIAddr(), grpc.WithTransportCredentials(insecure.NewCredentials())) - if err != nil { - // Failed to connect, try the next address. - return false - } - defer cc.Close() + heights := make([]uint64, len(nodes)) + for i, n := range nodes { + i := i + n := n + errGroup.Go(func() error { + height := from + n.IterateNetworkEndpoints(func(addr string) bool { + var a network.Address + if err := a.FromString(addr); err != nil { + return false + } - treeClient := NewTreeServiceClient(cc) - for { - h, err := s.synchronizeSingle(ctx, cid, treeID, height, treeClient) - if height < h { - height = h + cc, err := grpc.DialContext(egCtx, a.URIAddr(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + // Failed to connect, try the next address. + return false } - if err != nil || h <= height { - // Error with the response, try the next node. - return true + defer cc.Close() + + treeClient := NewTreeServiceClient(cc) + for { + h, err := s.synchronizeSingle(egCtx, cid, treeID, height, treeClient) + if height < h { + height = h + } + if err != nil || h <= height { + // Error with the response, try the next node. + return true + } } + }) + + if height <= from { // do not increase starting height on fail + heights[i] = from + return nil } + heights[i] = height + return nil }) - if height <= from { // do not increase starting height on fail - newHeight = from - } else if height < newHeight { // take minimum across all clients + } + + if err := errGroup.Wait(); err != nil { + s.log.Warn("failed to run tree synchronization over all nodes", zap.Error(err)) + } + + newHeight := uint64(math.MaxUint64) + for _, height := range heights { // take minimum across all clients + if height < newHeight { newHeight = height } } From e815b19101d3619282e284acd7231b15f9357b0e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Apr 2023 16:58:32 +0300 Subject: [PATCH 0059/1943] [#219] morph: Resolve containedctx linter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/accounting.go | 6 +++-- cmd/frostfs-node/main.go | 4 ++-- cmd/frostfs-node/morph.go | 5 ++-- cmd/frostfs-node/netmap.go | 5 ++-- pkg/innerring/innerring.go | 2 +- pkg/morph/client/audit/result_test.go | 3 ++- pkg/morph/client/constructor.go | 29 ++++------------------- pkg/morph/client/multi.go | 33 ++++++++++++++------------- pkg/morph/client/notifications.go | 10 ++++---- 9 files changed, 43 insertions(+), 54 deletions(-) diff --git a/cmd/frostfs-node/accounting.go b/cmd/frostfs-node/accounting.go index 26acc0437..6a35f37d0 100644 --- a/cmd/frostfs-node/accounting.go +++ b/cmd/frostfs-node/accounting.go @@ -1,6 +1,8 @@ package main import ( + "context" + accountingGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" accountingTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/accounting/grpc" @@ -8,9 +10,9 @@ import ( accounting "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/accounting/morph" ) -func initAccountingService(c *cfg) { +func initAccountingService(ctx context.Context, c *cfg) { if c.cfgMorph.client == nil { - initMorphComponents(c) + initMorphComponents(ctx, c) } balanceMorphWrapper, err := balance.NewFromMorph(c.cfgMorph.client, c.cfgAccounting.scriptHash, 0) diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index fdb003220..d5f711a51 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -95,8 +95,8 @@ func initApp(ctx context.Context, c *cfg) { }) initAndLog(c, "gRPC", initGRPC) - initAndLog(c, "netmap", initNetmapService) - initAndLog(c, "accounting", initAccountingService) + initAndLog(c, "netmap", func(c *cfg) { initNetmapService(ctx, c) }) + initAndLog(c, "accounting", func(c *cfg) { initAccountingService(ctx, c) }) initAndLog(c, "container", func(c *cfg) { initContainerService(ctx, c) }) initAndLog(c, "session", initSessionService) initAndLog(c, "reputation", initReputationService) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 2dfbe5c18..2db865ca3 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -27,7 +27,7 @@ const ( notaryDepositRetriesAmount = 300 ) -func initMorphComponents(c *cfg) { +func initMorphComponents(ctx context.Context, c *cfg) { var err error addresses := morphconfig.RPCEndpoint(c.appCfg) @@ -38,7 +38,8 @@ func initMorphComponents(c *cfg) { addresses[i], addresses[j] = addresses[j], addresses[i] }) - cli, err := client.New(c.key, + cli, err := client.New(ctx, + c.key, client.WithDialTimeout(morphconfig.DialTimeout(c.appCfg)), client.WithLogger(c.log), client.WithEndpoints(addresses...), diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 845649dee..d9b1c9208 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "errors" "fmt" @@ -135,7 +136,7 @@ func (c *cfg) addressNum() int { return 0 } -func initNetmapService(c *cfg) { +func initNetmapService(ctx context.Context, c *cfg) { network.WriteToNodeInfo(c.localAddr, &c.cfgNodeInfo.localInfo) c.cfgNodeInfo.localInfo.SetPublicKey(c.key.PublicKey().Bytes()) parseAttributes(c) @@ -144,7 +145,7 @@ func initNetmapService(c *cfg) { readSubnetCfg(c) if c.cfgMorph.client == nil { - initMorphComponents(c) + initMorphComponents(ctx, c) } initNetmapState(c) diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 3b42a5853..a91d2fd0d 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -464,8 +464,8 @@ func createClient(ctx context.Context, p *chainParams, errChan chan<- error) (*c } return client.New( + ctx, p.key, - client.WithContext(ctx), client.WithLogger(p.log), client.WithDialTimeout(p.cfg.GetDuration(p.name+".dial_timeout")), client.WithSigner(p.sgn), diff --git a/pkg/morph/client/audit/result_test.go b/pkg/morph/client/audit/result_test.go index a0137e3b8..5ce1cc740 100644 --- a/pkg/morph/client/audit/result_test.go +++ b/pkg/morph/client/audit/result_test.go @@ -1,6 +1,7 @@ package audit import ( + "context" "testing" "time" @@ -26,7 +27,7 @@ func TestAuditResults(t *testing.T) { auditHash, err := util.Uint160DecodeStringLE(sAuditHash) require.NoError(t, err) - morphClient, err := client.New(key, client.WithEndpoints(client.Endpoint{Address: endpoint})) + morphClient, err := client.New(context.Background(), key, client.WithEndpoints(client.Endpoint{Address: endpoint})) require.NoError(t, err) auditClientWrapper, err := NewFromMorph(morphClient, auditHash, 0) diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index e4569ad0e..9ed275029 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -29,10 +29,7 @@ type Option func(*cfg) type Callback func() // groups the configurations with default values. -// nolint: containedctx type cfg struct { - ctx context.Context // neo-go client context - dialTimeout time.Duration // client dial timeout logger *logger.Logger // logging component @@ -57,7 +54,6 @@ const ( func defaultConfig() *cfg { return &cfg{ - ctx: context.Background(), dialTimeout: defaultDialTimeout, logger: &logger.Logger{Logger: zap.L()}, waitInterval: defaultWaitInterval, @@ -84,7 +80,7 @@ func defaultConfig() *cfg { // If desired option satisfies the default value, it can be omitted. // If multiple options of the same config value are supplied, // the option with the highest index in the arguments will be used. -func New(key *keys.PrivateKey, opts ...Option) (*Client, error) { +func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, error) { if key == nil { panic("empty private key") } @@ -142,20 +138,20 @@ func New(key *keys.PrivateKey, opts ...Option) (*Client, error) { return nil, fmt.Errorf("could not create RPC actor: %w", err) } } else { - cli.client, act, err = cli.newCli(cli.endpoints.list[0].Address) + cli.client, act, err = cli.newCli(ctx, cli.endpoints.list[0].Address) if err != nil { return nil, fmt.Errorf("could not create RPC client: %w", err) } } cli.setActor(act) - go cli.notificationLoop() + go cli.notificationLoop(ctx) return cli, nil } -func (c *Client) newCli(endpoint string) (*rpcclient.WSClient, *actor.Actor, error) { - cli, err := rpcclient.NewWS(c.cfg.ctx, endpoint, rpcclient.Options{ +func (c *Client) newCli(ctx context.Context, endpoint string) (*rpcclient.WSClient, *actor.Actor, error) { + cli, err := rpcclient.NewWS(ctx, endpoint, rpcclient.Options{ DialTimeout: c.cfg.dialTimeout, }) if err != nil { @@ -201,21 +197,6 @@ func newClientCache() cache { } } -// WithContext returns a client constructor option that -// specifies the neo-go client context. -// -// Ignores nil value. Has no effect if WithSingleClient -// is provided. -// -// If option not provided, context.Background() is used. -func WithContext(ctx context.Context) Option { - return func(c *cfg) { - if ctx != nil { - c.ctx = ctx - } - } -} - // WithDialTimeout returns a client constructor option // that specifies neo-go client dial timeout duration. // diff --git a/pkg/morph/client/multi.go b/pkg/morph/client/multi.go index 54af56b21..5d736839a 100644 --- a/pkg/morph/client/multi.go +++ b/pkg/morph/client/multi.go @@ -1,6 +1,7 @@ package client import ( + "context" "sort" "time" @@ -32,7 +33,7 @@ func (e *endpoints) init(ee []Endpoint) { e.list = ee } -func (c *Client) switchRPC() bool { +func (c *Client) switchRPC(ctx context.Context) bool { c.switchLock.Lock() defer c.switchLock.Unlock() @@ -41,7 +42,7 @@ func (c *Client) switchRPC() bool { // Iterate endpoints in the order of decreasing priority. for c.endpoints.curr = range c.endpoints.list { newEndpoint := c.endpoints.list[c.endpoints.curr].Address - cli, act, err := c.newCli(newEndpoint) + cli, act, err := c.newCli(ctx, newEndpoint) if err != nil { c.logger.Warn("could not establish connection to the switched RPC node", zap.String("endpoint", newEndpoint), @@ -56,7 +57,7 @@ func (c *Client) switchRPC() bool { c.logger.Info("connection to the new RPC node has been established", zap.String("endpoint", newEndpoint)) - subs, ok := c.restoreSubscriptions(cli, newEndpoint, false) + subs, ok := c.restoreSubscriptions(ctx, cli, newEndpoint, false) if !ok { // new WS client does not allow // restoring subscription, client @@ -74,7 +75,7 @@ func (c *Client) switchRPC() bool { if c.cfg.switchInterval != 0 && !c.switchIsActive.Load() && c.endpoints.list[c.endpoints.curr].Priority != c.endpoints.list[0].Priority { c.switchIsActive.Store(true) - go c.switchToMostPrioritized() + go c.switchToMostPrioritized(ctx) } return true @@ -83,7 +84,7 @@ func (c *Client) switchRPC() bool { return false } -func (c *Client) notificationLoop() { +func (c *Client) notificationLoop(ctx context.Context) { var e any var ok bool @@ -95,7 +96,7 @@ func (c *Client) notificationLoop() { c.switchLock.RUnlock() select { - case <-c.cfg.ctx.Done(): + case <-ctx.Done(): _ = c.UnsubscribeAll() c.close() @@ -111,17 +112,17 @@ func (c *Client) notificationLoop() { } if ok { - c.routeEvent(e) + c.routeEvent(ctx, e) continue } - if !c.reconnect() { + if !c.reconnect(ctx) { return } } } -func (c *Client) routeEvent(e any) { +func (c *Client) routeEvent(ctx context.Context, e any) { typedNotification := rpcclient.Notification{Value: e} switch e.(type) { @@ -135,7 +136,7 @@ func (c *Client) routeEvent(e any) { select { case c.notifications <- typedNotification: - case <-c.cfg.ctx.Done(): + case <-ctx.Done(): _ = c.UnsubscribeAll() c.close() case <-c.closeChan: @@ -144,7 +145,7 @@ func (c *Client) routeEvent(e any) { } } -func (c *Client) reconnect() bool { +func (c *Client) reconnect(ctx context.Context) bool { if closeErr := c.client.GetError(); closeErr != nil { c.logger.Warn("switching to the next RPC node", zap.String("reason", closeErr.Error()), @@ -156,7 +157,7 @@ func (c *Client) reconnect() bool { return true } - if !c.switchRPC() { + if !c.switchRPC(ctx) { c.logger.Error("could not establish connection to any RPC node") // could not connect to all endpoints => @@ -173,7 +174,7 @@ func (c *Client) reconnect() bool { return true } -func (c *Client) switchToMostPrioritized() { +func (c *Client) switchToMostPrioritized(ctx context.Context) { t := time.NewTicker(c.cfg.switchInterval) defer t.Stop() defer c.switchIsActive.Store(false) @@ -181,7 +182,7 @@ func (c *Client) switchToMostPrioritized() { mainLoop: for { select { - case <-c.cfg.ctx.Done(): + case <-ctx.Done(): return case <-t.C: c.switchLock.RLock() @@ -207,7 +208,7 @@ mainLoop: tryE := e.Address - cli, act, err := c.newCli(tryE) + cli, act, err := c.newCli(ctx, tryE) if err != nil { c.logger.Warn("could not create client to the higher priority node", zap.String("endpoint", tryE), @@ -216,7 +217,7 @@ mainLoop: continue } - if subs, ok := c.restoreSubscriptions(cli, tryE, true); ok { + if subs, ok := c.restoreSubscriptions(ctx, cli, tryE, true); ok { c.switchLock.Lock() // higher priority node could have been diff --git a/pkg/morph/client/notifications.go b/pkg/morph/client/notifications.go index ed0368700..300bab825 100644 --- a/pkg/morph/client/notifications.go +++ b/pkg/morph/client/notifications.go @@ -1,6 +1,8 @@ package client import ( + "context" + "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/neorpc" @@ -227,7 +229,7 @@ type subsInfo struct { // one contains subscription information applied to the passed cli // and receivers for the updated subscriptions. // Does not change Client instance. -func (c *Client) restoreSubscriptions(cli *rpcclient.WSClient, endpoint string, background bool) (si subsInfo, ok bool) { +func (c *Client) restoreSubscriptions(ctx context.Context, cli *rpcclient.WSClient, endpoint string, background bool) (si subsInfo, ok bool) { var ( err error id string @@ -240,7 +242,7 @@ func (c *Client) restoreSubscriptions(cli *rpcclient.WSClient, endpoint string, notificationRcv := make(chan *state.ContainedNotificationEvent) notaryReqRcv := make(chan *result.NotaryRequestEvent) - c.startListen(stopCh, blockRcv, notificationRcv, notaryReqRcv, background) + c.startListen(ctx, stopCh, blockRcv, notificationRcv, notaryReqRcv, background) if background { c.switchLock.RLock() @@ -304,7 +306,7 @@ func (c *Client) restoreSubscriptions(cli *rpcclient.WSClient, endpoint string, return si, true } -func (c *Client) startListen(stopCh <-chan struct{}, blockRcv <-chan *block.Block, +func (c *Client) startListen(ctx context.Context, stopCh <-chan struct{}, blockRcv <-chan *block.Block, notificationRcv <-chan *state.ContainedNotificationEvent, notaryReqRcv <-chan *result.NotaryRequestEvent, background bool) { // neo-go WS client says to _always_ read notifications // from its channel. Subscribing to any notification @@ -335,7 +337,7 @@ func (c *Client) startListen(stopCh <-chan struct{}, blockRcv <-chan *block.Bloc continue } - c.routeEvent(e) + c.routeEvent(ctx, e) } }() } From 775179f82354b8c5985df77b427c1c49814ebb62 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Apr 2023 17:52:32 +0300 Subject: [PATCH 0060/1943] [#219] morph: Refactor notary invoke Resolve funlen linter for Client.notaryInvoke method. Signed-off-by: Dmitrii Stepanov --- pkg/morph/client/notary.go | 114 +++++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 49 deletions(-) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 9c7071e5a..96dca0319 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -16,6 +16,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/neorpc" + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" "github.com/nspcc-dev/neo-go/pkg/rpcclient/notary" sc "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/util" @@ -446,19 +447,12 @@ func (c *Client) notaryInvokeAsCommittee(method string, nonce, vub uint32, args return c.notaryInvoke(true, true, designate, nonce, &vub, method, args...) } -// nolint: funlen func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint160, nonce uint32, vub *uint32, method string, args ...any) error { - alphabetList, err := c.notary.alphabetSource() // prepare arguments for test invocation + alphabetList, err := c.notary.alphabetSource() if err != nil { return err } - u8n := uint8(len(alphabetList)) - - if !invokedByAlpha { - u8n++ - } - cosigners, err := c.notaryCosigners(invokedByAlpha, alphabetList, committee) if err != nil { return err @@ -469,38 +463,77 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint return err } - // make test invocation of the method - test, err := c.client.InvokeFunction(contract, method, params, cosigners) + test, err := c.makeTestInvocation(contract, method, params, cosigners) if err != nil { return err } - // check invocation state - if test.State != HaltState { - return wrapFrostFSError(¬HaltStateError{state: test.State, exception: test.FaultException}) - } - - // if test invocation failed, then return error - if len(test.Script) == 0 { - return wrapFrostFSError(errEmptyInvocationScript) - } - - // after test invocation we build main multisig transaction - multiaddrAccount, err := c.notaryMultisigAccount(alphabetList, committee, invokedByAlpha) if err != nil { return err } - var until uint32 + until, err := c.getUntilValue(vub) + if err != nil { + return err + } + mainTx, err := c.buildMainTx(invokedByAlpha, nonce, alphabetList, test, cosigners, multiaddrAccount, until) + if err != nil { + return err + } + + //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 + resp, err := c.client.SignAndPushP2PNotaryRequest(mainTx, + []byte{byte(opcode.RET)}, + -1, + 0, + c.notary.fallbackTime, + c.acc) + if err != nil && !alreadyOnChainError(err) { + return err + } + + c.logger.Debug("notary request invoked", + zap.String("method", method), + zap.Uint32("valid_until_block", until), + zap.Uint32("fallback_valid_for", c.notary.fallbackTime), + zap.Stringer("tx_hash", resp.Hash().Reverse())) + + return nil +} + +func (c *Client) makeTestInvocation(contract util.Uint160, method string, params []sc.Parameter, cosigners []transaction.Signer) (*result.Invoke, error) { + test, err := c.client.InvokeFunction(contract, method, params, cosigners) + if err != nil { + return nil, err + } + + if test.State != HaltState { + return nil, wrapFrostFSError(¬HaltStateError{state: test.State, exception: test.FaultException}) + } + + if len(test.Script) == 0 { + return nil, wrapFrostFSError(errEmptyInvocationScript) + } + return test, nil +} + +func (c *Client) getUntilValue(vub *uint32) (uint32, error) { if vub != nil { - until = *vub - } else { - until, err = c.notaryTxValidationLimit() - if err != nil { - return err - } + return *vub, nil + } + return c.notaryTxValidationLimit() +} + +func (c *Client) buildMainTx(invokedByAlpha bool, nonce uint32, alphabetList keys.PublicKeys, test *result.Invoke, + cosigners []transaction.Signer, multiaddrAccount *wallet.Account, until uint32) (*transaction.Transaction, error) { + // after test invocation we build main multisig transaction + + u8n := uint8(len(alphabetList)) + + if !invokedByAlpha { + u8n++ } // prepare main tx @@ -522,7 +555,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 notaryFee, err := c.client.CalculateNotaryFee(u8n) if err != nil { - return err + return nil, err } // add network fee for cosigners @@ -534,30 +567,13 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint c.notaryAccounts(invokedByAlpha, multiaddrAccount)..., ) if err != nil { - return err + return nil, err } // define witnesses mainTx.Scripts = c.notaryWitnesses(invokedByAlpha, multiaddrAccount, mainTx) - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - resp, err := c.client.SignAndPushP2PNotaryRequest(mainTx, - []byte{byte(opcode.RET)}, - -1, - 0, - c.notary.fallbackTime, - c.acc) - if err != nil && !alreadyOnChainError(err) { - return err - } - - c.logger.Debug("notary request invoked", - zap.String("method", method), - zap.Uint32("valid_until_block", until), - zap.Uint32("fallback_valid_for", c.notary.fallbackTime), - zap.Stringer("tx_hash", resp.Hash().Reverse())) - - return nil + return mainTx, nil } func (c *Client) notaryCosigners(invokedByAlpha bool, ir []*keys.PublicKey, committee bool) ([]transaction.Signer, error) { From d07e40d6fe0ce96ba4a6b6e709991db84bce368c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Apr 2023 18:11:56 +0300 Subject: [PATCH 0061/1943] [#219] morph: Refactor moprh event listener Resolve funlen and gocognit linters for listener.listenLoop method. Signed-off-by: Dmitrii Stepanov --- pkg/morph/event/listener.go | 54 ++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 0bc7e89f8..3de199328 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/subscriber" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" "github.com/nspcc-dev/neo-go/pkg/util" @@ -210,7 +211,6 @@ func (l *listener) subscribe(errCh chan error) { } } -// nolint: funlen, gocognit func (l *listener) listenLoop(ctx context.Context, intErr chan<- error, subErrCh chan error) { chs := l.subscriber.NotificationChannels() @@ -243,12 +243,7 @@ loop: continue loop } - if err := l.pool.Submit(func() { - l.parseAndHandleNotification(notifyEvent) - }); err != nil { - l.log.Warn("listener worker pool drained", - zap.Int("capacity", l.pool.Cap())) - } + l.handleNotifyEvent(notifyEvent) case notaryEvent, ok := <-chs.NotaryRequestsCh: if !ok { l.log.Warn("stop event listener by notary channel") @@ -262,12 +257,7 @@ loop: continue loop } - if err := l.pool.Submit(func() { - l.parseAndHandleNotary(notaryEvent) - }); err != nil { - l.log.Warn("listener worker pool drained", - zap.Int("capacity", l.pool.Cap())) - } + l.handleNotaryEvent(notaryEvent) case b, ok := <-chs.BlockCh: if !ok { l.log.Warn("stop event listener by block channel") @@ -281,18 +271,40 @@ loop: continue loop } - if err := l.pool.Submit(func() { - for i := range l.blockHandlers { - l.blockHandlers[i](b) - } - }); err != nil { - l.log.Warn("listener worker pool drained", - zap.Int("capacity", l.pool.Cap())) - } + l.handleBlockEvent(b) } } } +func (l *listener) handleBlockEvent(b *block.Block) { + if err := l.pool.Submit(func() { + for i := range l.blockHandlers { + l.blockHandlers[i](b) + } + }); err != nil { + l.log.Warn("listener worker pool drained", + zap.Int("capacity", l.pool.Cap())) + } +} + +func (l *listener) handleNotaryEvent(notaryEvent *result.NotaryRequestEvent) { + if err := l.pool.Submit(func() { + l.parseAndHandleNotary(notaryEvent) + }); err != nil { + l.log.Warn("listener worker pool drained", + zap.Int("capacity", l.pool.Cap())) + } +} + +func (l *listener) handleNotifyEvent(notifyEvent *state.ContainedNotificationEvent) { + if err := l.pool.Submit(func() { + l.parseAndHandleNotification(notifyEvent) + }); err != nil { + l.log.Warn("listener worker pool drained", + zap.Int("capacity", l.pool.Cap())) + } +} + func (l *listener) parseAndHandleNotification(notifyEvent *state.ContainedNotificationEvent) { log := l.log.With( zap.String("script hash LE", notifyEvent.ScriptHash.StringLE()), From fe8773507342a753e475e4884fa40a592bfce128 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Apr 2023 18:21:14 +0300 Subject: [PATCH 0062/1943] [#219] morph: Refactor notary preparator Resolve funlen linter for Preparator.Prepare method. Signed-off-by: Dmitrii Stepanov --- pkg/morph/event/notary_preparator.go | 103 ++++++++++++++------------- 1 file changed, 55 insertions(+), 48 deletions(-) diff --git a/pkg/morph/event/notary_preparator.go b/pkg/morph/event/notary_preparator.go index f85b5b648..3d499fec5 100644 --- a/pkg/morph/event/notary_preparator.go +++ b/pkg/morph/event/notary_preparator.go @@ -103,55 +103,8 @@ func notaryPreparator(prm PreparatorPrm) NotaryPreparator { // transaction is expected to be received one more time // from the Notary service but already signed. This happens // since every notary call is a new notary request in fact. -// -// nolint: funlen func (p Preparator) Prepare(nr *payload.P2PNotaryRequest) (NotaryEvent, error) { - // notary request's main tx is expected to have - // three or four witnesses: one for proxy contract, - // one for alphabet multisignature, one optional for - // notary's invoker and one is for notary contract - ln := len(nr.MainTransaction.Scripts) - switch ln { - case 3, 4: - default: - return nil, errUnexpectedWitnessAmount - } - invokerWitness := ln == 4 - - // alphabet node should handle only notary requests - // that have been sent unsigned(by storage nodes) => - // such main TXs should have dummy scripts as an - // invocation script - // - // this check prevents notary flow recursion - if !bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript) { - return nil, ErrTXAlreadyHandled - } - - currentAlphabet, err := p.alphaKeys() - if err != nil { - return nil, fmt.Errorf("could not fetch Alphabet public keys: %w", err) - } - - err = p.validateCosigners(ln, nr.MainTransaction.Signers, currentAlphabet) - if err != nil { - return nil, err - } - - // validate main TX's notary attribute - err = p.validateAttributes(nr.MainTransaction.Attributes, currentAlphabet, invokerWitness) - if err != nil { - return nil, err - } - - // validate main TX's witnesses - err = p.validateWitnesses(nr.MainTransaction.Scripts, currentAlphabet, invokerWitness) - if err != nil { - return nil, err - } - - // validate main TX expiration - err = p.validateExpiration(nr.FallbackTransaction) + err := p.validateNotaryRequest(nr) if err != nil { return nil, err } @@ -219,6 +172,60 @@ func (p Preparator) Prepare(nr *payload.P2PNotaryRequest) (NotaryEvent, error) { }, nil } +func (p Preparator) validateNotaryRequest(nr *payload.P2PNotaryRequest) error { + // notary request's main tx is expected to have + // three or four witnesses: one for proxy contract, + // one for alphabet multisignature, one optional for + // notary's invoker and one is for notary contract + ln := len(nr.MainTransaction.Scripts) + switch ln { + case 3, 4: + default: + return errUnexpectedWitnessAmount + } + invokerWitness := ln == 4 + + // alphabet node should handle only notary requests + // that have been sent unsigned(by storage nodes) => + // such main TXs should have dummy scripts as an + // invocation script + // + // this check prevents notary flow recursion + if !bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript) { + return ErrTXAlreadyHandled + } + + currentAlphabet, err := p.alphaKeys() + if err != nil { + return fmt.Errorf("could not fetch Alphabet public keys: %w", err) + } + + err = p.validateCosigners(ln, nr.MainTransaction.Signers, currentAlphabet) + if err != nil { + return err + } + + // validate main TX's notary attribute + err = p.validateAttributes(nr.MainTransaction.Attributes, currentAlphabet, invokerWitness) + if err != nil { + return err + } + + // validate main TX's witnesses + err = p.validateWitnesses(nr.MainTransaction.Scripts, currentAlphabet, invokerWitness) + if err != nil { + return err + } + + // validate main TX expiration + err = p.validateExpiration(nr.FallbackTransaction) + if err != nil { + return err + } + + return nil +} + func (p Preparator) validateParameterOpcodes(ops []Op) error { l := len(ops) From 3dbff0a478d60423ccb5a5d84934bd977c50e913 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Apr 2023 09:45:10 +0300 Subject: [PATCH 0063/1943] [#222] auditsvc: Resolve containedctx linter Resolve containedctx linter for commonCommunicatorPrm type. Signed-off-by: Dmitrii Stepanov --- pkg/innerring/rpc.go | 8 ++++---- pkg/services/audit/auditor/context.go | 7 ++----- pkg/services/audit/auditor/pdp.go | 3 +-- pkg/services/audit/auditor/pop.go | 3 +-- pkg/services/audit/auditor/por.go | 3 +-- 5 files changed, 9 insertions(+), 15 deletions(-) diff --git a/pkg/innerring/rpc.go b/pkg/innerring/rpc.go index 665e94232..c58e0ddc4 100644 --- a/pkg/innerring/rpc.go +++ b/pkg/innerring/rpc.go @@ -131,7 +131,7 @@ func (c *ClientCache) getSG(ctx context.Context, addr oid.Address, nm *netmap.Ne } // GetHeader requests node from the container under audit to return object header by id. -func (c *ClientCache) GetHeader(prm auditor.GetHeaderPrm) (*object.Object, error) { +func (c *ClientCache) GetHeader(ctx context.Context, prm auditor.GetHeaderPrm) (*object.Object, error) { var objAddress oid.Address objAddress.SetContainer(prm.CID) objAddress.SetObject(prm.OID) @@ -148,7 +148,7 @@ func (c *ClientCache) GetHeader(prm auditor.GetHeaderPrm) (*object.Object, error return nil, fmt.Errorf("can't setup remote connection with %s: %w", info.AddressGroup(), err) } - cctx, cancel := context.WithTimeout(prm.Context, c.headTimeout) + cctx, cancel := context.WithTimeout(ctx, c.headTimeout) var obj *object.Object @@ -169,7 +169,7 @@ func (c *ClientCache) GetHeader(prm auditor.GetHeaderPrm) (*object.Object, error // GetRangeHash requests node from the container under audit to return Tillich-Zemor hash of the // payload range of the object with specified identifier. -func (c *ClientCache) GetRangeHash(prm auditor.GetRangeHashPrm) ([]byte, error) { +func (c *ClientCache) GetRangeHash(ctx context.Context, prm auditor.GetRangeHashPrm) ([]byte, error) { var objAddress oid.Address objAddress.SetContainer(prm.CID) objAddress.SetObject(prm.OID) @@ -186,7 +186,7 @@ func (c *ClientCache) GetRangeHash(prm auditor.GetRangeHashPrm) ([]byte, error) return nil, fmt.Errorf("can't setup remote connection with %s: %w", info.AddressGroup(), err) } - cctx, cancel := context.WithTimeout(prm.Context, c.rangeTimeout) + cctx, cancel := context.WithTimeout(ctx, c.rangeTimeout) h, err := frostfsapiclient.HashObjectRange(cctx, cli, objAddress, prm.Range) diff --git a/pkg/services/audit/auditor/context.go b/pkg/services/audit/auditor/context.go index 4a5cbb457..f2778fd65 100644 --- a/pkg/services/audit/auditor/context.go +++ b/pkg/services/audit/auditor/context.go @@ -81,10 +81,7 @@ type ContextPrm struct { pdpWorkerPool, porWorkerPool util.WorkerPool } -// nolint: containedctx type commonCommunicatorPrm struct { - Context context.Context - Node netmap.NodeInfo OID oid.ID @@ -109,11 +106,11 @@ type GetRangeHashPrm struct { // component of communication with container nodes. type ContainerCommunicator interface { // GetHeader must return object header from the container node. - GetHeader(GetHeaderPrm) (*object.Object, error) + GetHeader(context.Context, GetHeaderPrm) (*object.Object, error) // GetRangeHash must return homomorphic Tillich-Zemor hash of payload range of the // object stored in container node. - GetRangeHash(GetRangeHashPrm) ([]byte, error) + GetRangeHash(context.Context, GetRangeHashPrm) ([]byte, error) } // NewContext creates, initializes and returns Context. diff --git a/pkg/services/audit/auditor/pdp.go b/pkg/services/audit/auditor/pdp.go index beb2fdcf8..13b50e498 100644 --- a/pkg/services/audit/auditor/pdp.go +++ b/pkg/services/audit/auditor/pdp.go @@ -118,7 +118,6 @@ func (c *Context) collectHashes(p *gamePair) { rand.Shuffle(len(order), func(i, j int) { order[i], order[j] = order[j], order[i] }) var getRangeHashPrm GetRangeHashPrm - getRangeHashPrm.Context = c.task.AuditContext() getRangeHashPrm.CID = c.task.ContainerID() getRangeHashPrm.OID = p.id getRangeHashPrm.Node = n @@ -138,7 +137,7 @@ func (c *Context) collectHashes(p *gamePair) { getRangeHashPrm.Range = rngs[i] - h, err := c.cnrCom.GetRangeHash(getRangeHashPrm) + h, err := c.cnrCom.GetRangeHash(c.task.AuditContext(), getRangeHashPrm) if err != nil { c.log.Debug("could not get payload range hash", zap.Stringer("id", p.id), diff --git a/pkg/services/audit/auditor/pop.go b/pkg/services/audit/auditor/pop.go index f8a16cb0a..45afa7937 100644 --- a/pkg/services/audit/auditor/pop.go +++ b/pkg/services/audit/auditor/pop.go @@ -49,7 +49,6 @@ func (c *Context) processObjectPlacement(id oid.ID, nodes []netmap.NodeInfo, rep ) var getHeaderPrm GetHeaderPrm - getHeaderPrm.Context = c.task.AuditContext() getHeaderPrm.OID = id getHeaderPrm.CID = c.task.ContainerID() getHeaderPrm.NodeIsRelay = false @@ -58,7 +57,7 @@ func (c *Context) processObjectPlacement(id oid.ID, nodes []netmap.NodeInfo, rep getHeaderPrm.Node = nodes[i] // try to get object header from node - hdr, err := c.cnrCom.GetHeader(getHeaderPrm) + hdr, err := c.cnrCom.GetHeader(c.task.AuditContext(), getHeaderPrm) if err != nil { c.log.Debug("could not get object header from candidate", zap.Stringer("id", id), diff --git a/pkg/services/audit/auditor/por.go b/pkg/services/audit/auditor/por.go index 1922213e5..ff322d6e1 100644 --- a/pkg/services/audit/auditor/por.go +++ b/pkg/services/audit/auditor/por.go @@ -48,7 +48,6 @@ func (c *Context) checkStorageGroupPoR(sgID oid.ID, sg storagegroupSDK.StorageGr ) var getHeaderPrm GetHeaderPrm - getHeaderPrm.Context = c.task.AuditContext() getHeaderPrm.CID = c.task.ContainerID() getHeaderPrm.NodeIsRelay = true @@ -81,7 +80,7 @@ func (c *Context) checkStorageGroupPoR(sgID oid.ID, sg storagegroupSDK.StorageGr getHeaderPrm.Node = flat[j] - hdr, err := c.cnrCom.GetHeader(getHeaderPrm) + hdr, err := c.cnrCom.GetHeader(c.task.AuditContext(), getHeaderPrm) if err != nil { c.log.Debug("can't head object", zap.String("remote_node", netmap.StringifyPublicKey(flat[j])), From e8d340287f34073a08ba88a33a833ef290709bd9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Apr 2023 11:36:25 +0300 Subject: [PATCH 0064/1943] [#222] auditsvc: Refactor audit task Resolve containedctx linter. Cancel task by listen cancel. Signed-off-by: Dmitrii Stepanov --- pkg/innerring/processors/audit/process.go | 22 +++++++++++---------- pkg/innerring/processors/audit/processor.go | 2 +- pkg/services/audit/auditor/context.go | 4 +--- pkg/services/audit/auditor/exec.go | 10 ++++++---- pkg/services/audit/auditor/pdp.go | 17 ++++++++-------- pkg/services/audit/auditor/pop.go | 14 +++++++------ pkg/services/audit/auditor/por.go | 9 +++++---- pkg/services/audit/task.go | 20 +++++++------------ pkg/services/audit/taskmanager/listen.go | 21 +++++++++++++++----- pkg/services/audit/taskmanager/push.go | 5 +---- 10 files changed, 66 insertions(+), 58 deletions(-) diff --git a/pkg/innerring/processors/audit/process.go b/pkg/innerring/processors/audit/process.go index 7f148e57e..ecfc407be 100644 --- a/pkg/innerring/processors/audit/process.go +++ b/pkg/innerring/processors/audit/process.go @@ -46,15 +46,21 @@ func (ap *Processor) processStartAudit(epoch uint64) { return } - var auditCtx context.Context - auditCtx, ap.prevAuditCanceler = context.WithCancel(context.Background()) + cancelChannel := make(chan struct{}) + ap.prevAuditCanceler = func() { + select { + case <-cancelChannel: // already closed + default: + close(cancelChannel) + } + } pivot := make([]byte, sha256.Size) - ap.startAuditTasksOnContainers(auditCtx, containers, log, pivot, nm, epoch) + ap.startAuditTasksOnContainers(cancelChannel, containers, log, pivot, nm, epoch) } -func (ap *Processor) startAuditTasksOnContainers(ctx context.Context, containers []cid.ID, log *zap.Logger, pivot []byte, nm *netmap.NetMap, epoch uint64) { +func (ap *Processor) startAuditTasksOnContainers(cancelChannel <-chan struct{}, containers []cid.ID, log *zap.Logger, pivot []byte, nm *netmap.NetMap, epoch uint64) { for i := range containers { cnr, err := cntClient.Get(ap.containerClient, containers[i]) // get container structure if err != nil { @@ -107,18 +113,14 @@ func (ap *Processor) startAuditTasksOnContainers(ctx context.Context, containers epoch: epoch, rep: ap.reporter, }). - WithAuditContext(ctx). + WithCancelChannel(cancelChannel). WithContainerID(containers[i]). WithStorageGroupList(storageGroups). WithContainerStructure(cnr.Value). WithContainerNodes(nodes). WithNetworkMap(nm) - if err := ap.taskManager.PushTask(auditTask); err != nil { - ap.log.Error("could not push audit task", - zap.String("error", err.Error()), - ) - } + ap.taskManager.PushTask(auditTask) } } diff --git a/pkg/innerring/processors/audit/processor.go b/pkg/innerring/processors/audit/processor.go index cb514b165..31e8a8c55 100644 --- a/pkg/innerring/processors/audit/processor.go +++ b/pkg/innerring/processors/audit/processor.go @@ -24,7 +24,7 @@ type ( } TaskManager interface { - PushTask(*audit.Task) error + PushTask(*audit.Task) // Must skip all tasks planned for execution and // return their number. diff --git a/pkg/services/audit/auditor/context.go b/pkg/services/audit/auditor/context.go index f2778fd65..bf720c330 100644 --- a/pkg/services/audit/auditor/context.go +++ b/pkg/services/audit/auditor/context.go @@ -194,9 +194,7 @@ func (c *Context) init() { )} } -func (c *Context) expired() bool { - ctx := c.task.AuditContext() - +func (c *Context) expired(ctx context.Context) bool { select { case <-ctx.Done(): c.log.Debug("audit context is done", diff --git a/pkg/services/audit/auditor/exec.go b/pkg/services/audit/auditor/exec.go index ceb6556e2..e603818b8 100644 --- a/pkg/services/audit/auditor/exec.go +++ b/pkg/services/audit/auditor/exec.go @@ -1,16 +1,18 @@ package auditor import ( + "context" "fmt" ) // Execute audits container data. -func (c *Context) Execute() { +func (c *Context) Execute(ctx context.Context, onCompleted func()) { + defer onCompleted() c.init() checks := []struct { name string - exec func() + exec func(context.Context) }{ {name: "PoR", exec: c.executePoR}, {name: "PoP", exec: c.executePoP}, @@ -20,11 +22,11 @@ func (c *Context) Execute() { for i := range checks { c.log.Debug(fmt.Sprintf("executing %s check...", checks[i].name)) - if c.expired() { + if c.expired(ctx) { break } - checks[i].exec() + checks[i].exec(ctx) if i == len(checks)-1 { c.complete() diff --git a/pkg/services/audit/auditor/pdp.go b/pkg/services/audit/auditor/pdp.go index 13b50e498..8a184eb7e 100644 --- a/pkg/services/audit/auditor/pdp.go +++ b/pkg/services/audit/auditor/pdp.go @@ -2,6 +2,7 @@ package auditor import ( "bytes" + "context" "sync" "time" @@ -13,12 +14,12 @@ import ( "go.uber.org/zap" ) -func (c *Context) executePDP() { - c.processPairs() +func (c *Context) executePDP(ctx context.Context) { + c.processPairs(ctx) c.writePairsResult() } -func (c *Context) processPairs() { +func (c *Context) processPairs(ctx context.Context) { wg := new(sync.WaitGroup) for i := range c.pairs { @@ -26,7 +27,7 @@ func (c *Context) processPairs() { wg.Add(1) if err := c.pdpWorkerPool.Submit(func() { - c.processPair(p) + c.processPair(ctx, p) wg.Done() }); err != nil { wg.Done() @@ -37,9 +38,9 @@ func (c *Context) processPairs() { c.pdpWorkerPool.Release() } -func (c *Context) processPair(p *gamePair) { +func (c *Context) processPair(ctx context.Context, p *gamePair) { c.distributeRanges(p) - c.collectHashes(p) + c.collectHashes(ctx, p) c.analyzeHashes(p) } @@ -106,7 +107,7 @@ func (c *Context) splitPayload(id oid.ID) []uint64 { return notches } -func (c *Context) collectHashes(p *gamePair) { +func (c *Context) collectHashes(ctx context.Context, p *gamePair) { fn := func(n netmap.NodeInfo, rngs []*object.Range) [][]byte { // Here we randomize the order a bit: the hypothesis is that this // makes it harder for an unscrupulous node to come up with a @@ -137,7 +138,7 @@ func (c *Context) collectHashes(p *gamePair) { getRangeHashPrm.Range = rngs[i] - h, err := c.cnrCom.GetRangeHash(c.task.AuditContext(), getRangeHashPrm) + h, err := c.cnrCom.GetRangeHash(ctx, getRangeHashPrm) if err != nil { c.log.Debug("could not get payload range hash", zap.Stringer("id", p.id), diff --git a/pkg/services/audit/auditor/pop.go b/pkg/services/audit/auditor/pop.go index 45afa7937..32b837794 100644 --- a/pkg/services/audit/auditor/pop.go +++ b/pkg/services/audit/auditor/pop.go @@ -1,6 +1,8 @@ package auditor import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/tzhash/tz" @@ -12,8 +14,8 @@ const ( minGamePayloadSize = hashRangeNumber * tz.Size ) -func (c *Context) executePoP() { - c.buildCoverage() +func (c *Context) executePoP(ctx context.Context) { + c.buildCoverage(ctx) c.report.SetPlacementCounters( c.counters.hit, @@ -22,13 +24,13 @@ func (c *Context) executePoP() { ) } -func (c *Context) buildCoverage() { +func (c *Context) buildCoverage(ctx context.Context) { policy := c.task.ContainerStructure().PlacementPolicy() // select random member from another storage group // and process all placement vectors c.iterateSGMembersPlacementRand(func(id oid.ID, ind int, nodes []netmap.NodeInfo) bool { - c.processObjectPlacement(id, nodes, policy.ReplicaNumberByIndex(ind)) + c.processObjectPlacement(ctx, id, nodes, policy.ReplicaNumberByIndex(ind)) return c.containerCovered() }) } @@ -38,7 +40,7 @@ func (c *Context) containerCovered() bool { return c.cnrNodesNum <= len(c.pairedNodes) } -func (c *Context) processObjectPlacement(id oid.ID, nodes []netmap.NodeInfo, replicas uint32) { +func (c *Context) processObjectPlacement(ctx context.Context, id oid.ID, nodes []netmap.NodeInfo, replicas uint32) { var ( ok uint32 optimal bool @@ -57,7 +59,7 @@ func (c *Context) processObjectPlacement(id oid.ID, nodes []netmap.NodeInfo, rep getHeaderPrm.Node = nodes[i] // try to get object header from node - hdr, err := c.cnrCom.GetHeader(c.task.AuditContext(), getHeaderPrm) + hdr, err := c.cnrCom.GetHeader(ctx, getHeaderPrm) if err != nil { c.log.Debug("could not get object header from candidate", zap.Stringer("id", id), diff --git a/pkg/services/audit/auditor/por.go b/pkg/services/audit/auditor/por.go index ff322d6e1..6011217f8 100644 --- a/pkg/services/audit/auditor/por.go +++ b/pkg/services/audit/auditor/por.go @@ -2,6 +2,7 @@ package auditor import ( "bytes" + "context" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" @@ -14,7 +15,7 @@ import ( "go.uber.org/zap" ) -func (c *Context) executePoR() { +func (c *Context) executePoR(ctx context.Context) { wg := new(sync.WaitGroup) sgs := c.task.StorageGroupList() @@ -22,7 +23,7 @@ func (c *Context) executePoR() { wg.Add(1) if err := c.porWorkerPool.Submit(func() { - c.checkStorageGroupPoR(sg.ID(), sg.StorageGroup()) + c.checkStorageGroupPoR(ctx, sg.ID(), sg.StorageGroup()) wg.Done() }); err != nil { wg.Done() @@ -36,7 +37,7 @@ func (c *Context) executePoR() { } // nolint: funlen -func (c *Context) checkStorageGroupPoR(sgID oid.ID, sg storagegroupSDK.StorageGroup) { +func (c *Context) checkStorageGroupPoR(ctx context.Context, sgID oid.ID, sg storagegroupSDK.StorageGroup) { members := sg.Members() c.updateSGInfo(sgID, members) @@ -80,7 +81,7 @@ func (c *Context) checkStorageGroupPoR(sgID oid.ID, sg storagegroupSDK.StorageGr getHeaderPrm.Node = flat[j] - hdr, err := c.cnrCom.GetHeader(c.task.AuditContext(), getHeaderPrm) + hdr, err := c.cnrCom.GetHeader(ctx, getHeaderPrm) if err != nil { c.log.Debug("can't head object", zap.String("remote_node", netmap.StringifyPublicKey(flat[j])), diff --git a/pkg/services/audit/task.go b/pkg/services/audit/task.go index 35932a69e..3de5ac2c6 100644 --- a/pkg/services/audit/task.go +++ b/pkg/services/audit/task.go @@ -1,8 +1,6 @@ package audit import ( - "context" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/storagegroup" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -10,11 +8,10 @@ import ( ) // Task groups groups the container audit parameters. -// nolint: containedctx type Task struct { - reporter Reporter + cancelCh <-chan struct{} - auditContext context.Context + reporter Reporter idCnr cid.ID @@ -41,18 +38,15 @@ func (t *Task) Reporter() Reporter { return t.reporter } -// WithAuditContext sets context of the audit of the current epoch. -func (t *Task) WithAuditContext(ctx context.Context) *Task { - if t != nil { - t.auditContext = ctx +func (t *Task) WithCancelChannel(ch <-chan struct{}) *Task { + if ch != nil { + t.cancelCh = ch } - return t } -// AuditContext returns context of the audit of the current epoch. -func (t *Task) AuditContext() context.Context { - return t.auditContext +func (t *Task) CancelChannel() <-chan struct{} { + return t.cancelCh } // WithContainerID sets identifier of the container under audit. diff --git a/pkg/services/audit/taskmanager/listen.go b/pkg/services/audit/taskmanager/listen.go index 4e8a3df68..a16052e13 100644 --- a/pkg/services/audit/taskmanager/listen.go +++ b/pkg/services/audit/taskmanager/listen.go @@ -33,18 +33,28 @@ func (m *Manager) Listen(ctx context.Context) { return } - m.handleTask(task) + tCtx, tCancel := context.WithCancel(ctx) // cancel task in case of listen cancel + go func() { + select { + case <-tCtx.Done(): // listen cancelled or task completed + return + case <-task.CancelChannel(): // new epoch + tCancel() + } + }() + + m.handleTask(tCtx, task, tCancel) } } } -func (m *Manager) handleTask(task *audit.Task) { +func (m *Manager) handleTask(ctx context.Context, task *audit.Task, onCompleted func()) { pdpPool, err := m.pdpPoolGenerator() if err != nil { m.log.Error("could not generate PDP worker pool", zap.String("error", err.Error()), ) - + onCompleted() return } @@ -53,7 +63,7 @@ func (m *Manager) handleTask(task *audit.Task) { m.log.Error("could not generate PoR worker pool", zap.String("error", err.Error()), ) - + onCompleted() return } @@ -61,9 +71,10 @@ func (m *Manager) handleTask(task *audit.Task) { WithPDPWorkerPool(pdpPool). WithPoRWorkerPool(porPool) - if err := m.workerPool.Submit(auditContext.Execute); err != nil { + if err := m.workerPool.Submit(func() { auditContext.Execute(ctx, onCompleted) }); err != nil { // may be we should report it m.log.Warn("could not submit audit task") + onCompleted() } } diff --git a/pkg/services/audit/taskmanager/push.go b/pkg/services/audit/taskmanager/push.go index 13f8fd12d..805897dbf 100644 --- a/pkg/services/audit/taskmanager/push.go +++ b/pkg/services/audit/taskmanager/push.go @@ -5,9 +5,6 @@ import ( ) // PushTask adds a task to the queue for processing. -// -// Returns error if task was not added to the queue. -func (m *Manager) PushTask(t *audit.Task) error { +func (m *Manager) PushTask(t *audit.Task) { m.ch <- t - return nil } From e2f13d03d7c25b7399e817642a9532eff0bd8be2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Apr 2023 12:18:58 +0300 Subject: [PATCH 0065/1943] [#222] auditsvc: Refactor PoR audit Resolve funlen linter for Context.checkStorageGroupPoR method. Signed-off-by: Dmitrii Stepanov --- pkg/services/audit/auditor/por.go | 43 +++++++++++++++++++------------ 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/pkg/services/audit/auditor/por.go b/pkg/services/audit/auditor/por.go index 6011217f8..aebc25c68 100644 --- a/pkg/services/audit/auditor/por.go +++ b/pkg/services/audit/auditor/por.go @@ -36,7 +36,6 @@ func (c *Context) executePoR(ctx context.Context) { c.report.SetPoRCounters(c.porRequests.Load(), c.porRetries.Load()) } -// nolint: funlen func (c *Context) checkStorageGroupPoR(ctx context.Context, sgID oid.ID, sg storagegroupSDK.StorageGroup) { members := sg.Members() c.updateSGInfo(sgID, members) @@ -55,22 +54,11 @@ func (c *Context) checkStorageGroupPoR(ctx context.Context, sgID oid.ID, sg stor homomorphicHashingEnabled := !containerSDK.IsHomomorphicHashingDisabled(c.task.ContainerStructure()) for i := range members { - objectPlacement, err := c.buildPlacement(members[i]) - if err != nil { - c.log.Info("can't build placement for storage group member", - zap.Stringer("sg", sgID), - zap.String("member_id", members[i].String()), - ) - + flat, ok := c.getShuffledNodes(members[i], sgID) + if !ok { continue } - flat := placement.FlattenNodes(objectPlacement) - - rand.Shuffle(len(flat), func(i, j int) { - flat[i], flat[j] = flat[j], flat[i] - }) - getHeaderPrm.OID = members[i] for j := range flat { @@ -126,8 +114,12 @@ func (c *Context) checkStorageGroupPoR(ctx context.Context, sgID oid.ID, sg stor cs, _ := sg.ValidationDataHash() tzCheck := !homomorphicHashingEnabled || bytes.Equal(tzHash, cs.Value()) + c.writeCheckReport(sizeCheck, tzCheck, sgID, sg, totalSize) +} + +func (c *Context) writeCheckReport(sizeCheck, tzCheck bool, sgID oid.ID, sg storagegroupSDK.StorageGroup, totalSize uint64) { if sizeCheck && tzCheck { - c.report.PassedPoR(sgID) // write report + c.report.PassedPoR(sgID) } else { if !sizeCheck { c.log.Debug("storage group size check failed", @@ -139,6 +131,25 @@ func (c *Context) checkStorageGroupPoR(ctx context.Context, sgID oid.ID, sg stor c.log.Debug("storage group tz hash check failed") } - c.report.FailedPoR(sgID) // write report + c.report.FailedPoR(sgID) } } + +func (c *Context) getShuffledNodes(member oid.ID, sgID oid.ID) ([]netmap.NodeInfo, bool) { + objectPlacement, err := c.buildPlacement(member) + if err != nil { + c.log.Info("can't build placement for storage group member", + zap.Stringer("sg", sgID), + zap.String("member_id", member.String()), + ) + + return nil, false + } + + flat := placement.FlattenNodes(objectPlacement) + + rand.Shuffle(len(flat), func(i, j int) { + flat[i], flat[j] = flat[j], flat[i] + }) + return flat, true +} From 469e8a6e59a26a24614eefc8f50f641e452ad616 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Apr 2023 17:47:00 +0300 Subject: [PATCH 0066/1943] [#212] reputationsvc: Resolve containedctx linter Signed-off-by: Dmitrii Stepanov --- .../reputation/eigentrust/controller/calls.go | 18 ++---------------- .../eigentrust/controller/controller.go | 4 ++-- .../reputation/eigentrust/controller/deps.go | 6 ------ 3 files changed, 4 insertions(+), 24 deletions(-) diff --git a/pkg/services/reputation/eigentrust/controller/calls.go b/pkg/services/reputation/eigentrust/controller/calls.go index 28228706c..faf953aed 100644 --- a/pkg/services/reputation/eigentrust/controller/calls.go +++ b/pkg/services/reputation/eigentrust/controller/calls.go @@ -1,8 +1,6 @@ package eigentrustctrl import ( - "context" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" "go.uber.org/zap" ) @@ -12,10 +10,7 @@ type ContinuePrm struct { Epoch uint64 } -// nolint: containedctx type iterContext struct { - context.Context - eigentrust.EpochIteration iterationNumber uint32 @@ -26,12 +21,6 @@ func (x iterContext) Last() bool { return x.last } -type iterContextCancel struct { - iterContext - - cancel context.CancelFunc -} - // Continue moves the global reputation calculator to the next iteration. func (c *Controller) Continue(prm ContinuePrm) { c.mtx.Lock() @@ -39,10 +28,9 @@ func (c *Controller) Continue(prm ContinuePrm) { { iterCtx, ok := c.mCtx[prm.Epoch] if !ok { - iterCtx = new(iterContextCancel) + iterCtx = new(iterContext) c.mCtx[prm.Epoch] = iterCtx - iterCtx.Context, iterCtx.cancel = context.WithCancel(context.Background()) iterCtx.EpochIteration.SetEpoch(prm.Epoch) iterations, err := c.prm.IterationsProvider.EigenTrustIterations() @@ -53,14 +41,12 @@ func (c *Controller) Continue(prm ContinuePrm) { } else { iterCtx.iterationNumber = uint32(iterations) } - } else { - iterCtx.cancel() } iterCtx.last = iterCtx.I() == iterCtx.iterationNumber-1 err := c.prm.WorkerPool.Submit(func() { - c.prm.DaughtersTrustCalculator.Calculate(iterCtx.iterContext) + c.prm.DaughtersTrustCalculator.Calculate(iterCtx) // iteration++ iterCtx.Increment() diff --git a/pkg/services/reputation/eigentrust/controller/controller.go b/pkg/services/reputation/eigentrust/controller/controller.go index fe9150bcf..a6d0d4a82 100644 --- a/pkg/services/reputation/eigentrust/controller/controller.go +++ b/pkg/services/reputation/eigentrust/controller/controller.go @@ -47,7 +47,7 @@ type Controller struct { opts *options mtx sync.Mutex - mCtx map[uint64]*iterContextCancel + mCtx map[uint64]*iterContext } const invalidPrmValFmt = "invalid parameter %s (%T):%v" @@ -81,6 +81,6 @@ func New(prm Prm, opts ...Option) *Controller { return &Controller{ prm: prm, opts: o, - mCtx: make(map[uint64]*iterContextCancel), + mCtx: make(map[uint64]*iterContext), } } diff --git a/pkg/services/reputation/eigentrust/controller/deps.go b/pkg/services/reputation/eigentrust/controller/deps.go index 8c4752657..2aeafb61a 100644 --- a/pkg/services/reputation/eigentrust/controller/deps.go +++ b/pkg/services/reputation/eigentrust/controller/deps.go @@ -1,14 +1,8 @@ package eigentrustctrl -import ( - "context" -) - // IterationContext is a context of the i-th // stage of iterative EigenTrust algorithm. type IterationContext interface { - context.Context - // Must return epoch number to select the values // for global trust calculation. Epoch() uint64 From 7ebbfa3358b0c8e61a36548b29741ef0732be366 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Apr 2023 10:44:40 +0300 Subject: [PATCH 0067/1943] [#212] reputationsvc: Resolve linters and rename Resolved containedctx linters. Renamed context structs and interfaces to more understandble names. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/main.go | 2 +- cmd/frostfs-node/reputation.go | 37 ++++++------ cmd/frostfs-node/reputation/common/remote.go | 11 ++-- cmd/frostfs-node/reputation/common/util.go | 13 ++-- .../reputation/intermediate/calculator.go | 11 ++-- .../reputation/intermediate/consumers.go | 30 +++++----- .../reputation/intermediate/contract.go | 2 +- .../reputation/intermediate/daughters.go | 16 ++--- .../reputation/intermediate/remote.go | 32 +++++----- .../reputation/intermediate/storage.go | 6 +- .../reputation/internal/client/client.go | 18 ++---- cmd/frostfs-node/reputation/local/remote.go | 16 ++--- cmd/frostfs-node/reputation/local/storage.go | 10 ++-- pkg/services/reputation/common/deps.go | 13 ++-- .../reputation/common/router/calls.go | 53 +++++++++-------- .../reputation/eigentrust/calculator/calls.go | 59 +++++++++---------- .../reputation/eigentrust/calculator/deps.go | 22 +++---- .../reputation/eigentrust/controller/calls.go | 6 +- .../reputation/eigentrust/controller/deps.go | 4 +- .../reputation/eigentrust/iteration.go | 17 +----- .../reputation/local/controller/calls.go | 53 ++++++++--------- .../reputation/local/controller/controller.go | 4 +- .../reputation/local/controller/deps.go | 2 +- .../reputation/local/controller/util.go | 4 +- 24 files changed, 203 insertions(+), 238 deletions(-) diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index d5f711a51..cddedabe9 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -99,7 +99,7 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(c, "accounting", func(c *cfg) { initAccountingService(ctx, c) }) initAndLog(c, "container", func(c *cfg) { initContainerService(ctx, c) }) initAndLog(c, "session", initSessionService) - initAndLog(c, "reputation", initReputationService) + initAndLog(c, "reputation", func(c *cfg) { initReputationService(ctx, c) }) initAndLog(c, "notification", initNotifications) initAndLog(c, "object", initObjectService) initAndLog(c, "tree", initTreeService) diff --git a/cmd/frostfs-node/reputation.go b/cmd/frostfs-node/reputation.go index 7b43443c2..a96bd066e 100644 --- a/cmd/frostfs-node/reputation.go +++ b/cmd/frostfs-node/reputation.go @@ -33,7 +33,7 @@ import ( "go.uber.org/zap" ) -func initReputationService(c *cfg) { +func initReputationService(ctx context.Context, c *cfg) { wrap, err := repClient.NewFromMorph(c.cfgMorph.client, c.cfgReputation.scriptHash, 0, repClient.TryNotary()) fatalOnErr(err) @@ -73,7 +73,7 @@ func initReputationService(c *cfg) { c.cfgReputation.localTrustCtrl = createLocalTrustController(c, localTrustLogger, localKey, localTrustRouter) - addReputationReportHandler(c) + addReputationReportHandler(ctx, c) server := grpcreputation.New( reputationrpc.NewSignService( @@ -98,10 +98,10 @@ func initReputationService(c *cfg) { // initialize eigen trust block timer newEigenTrustIterTimer(c) - addEigenTrustEpochHandler(c, eigenTrustController) + addEigenTrustEpochHandler(ctx, c, eigenTrustController) } -func addReputationReportHandler(c *cfg) { +func addReputationReportHandler(ctx context.Context, c *cfg) { addNewEpochAsyncNotificationHandler( c, func(ev event.Event) { @@ -112,12 +112,12 @@ func addReputationReportHandler(c *cfg) { // report collected values from previous epoch reportPrm.SetEpoch(ev.(netmap.NewEpoch).EpochNumber() - 1) - c.cfgReputation.localTrustCtrl.Report(reportPrm) + c.cfgReputation.localTrustCtrl.Report(ctx, reportPrm) }, ) } -func addEigenTrustEpochHandler(c *cfg, eigenTrustController *eigentrustctrl.Controller) { +func addEigenTrustEpochHandler(ctx context.Context, c *cfg, eigenTrustController *eigentrustctrl.Controller) { addNewEpochAsyncNotificationHandler( c, func(e event.Event) { @@ -138,7 +138,7 @@ func addEigenTrustEpochHandler(c *cfg, eigenTrustController *eigentrustctrl.Cont } epochTimer, err := ticker.NewIterationsTicker(duration, iterations, func() { - eigenTrustController.Continue( + eigenTrustController.Continue(ctx, eigentrustctrl.ContinuePrm{ Epoch: epoch - 1, }, @@ -286,8 +286,8 @@ func createLocalTrustController(c *cfg, localTrustLogger *logger.Logger, localKe type reputationServer struct { *cfg log *logger.Logger - localRouter reputationcommon.WriterProvider - intermediateRouter reputationcommon.WriterProvider + localRouter *reputationrouter.Router + intermediateRouter *reputationrouter.Router routeBuilder reputationrouter.Builder } @@ -297,18 +297,17 @@ func (s *reputationServer) AnnounceLocalTrust(ctx context.Context, req *v2reputa body := req.GetBody() - eCtx := &common.EpochContext{ - Context: ctx, - E: body.GetEpoch(), + ep := &common.EpochProvider{ + E: body.GetEpoch(), } - w, err := s.localRouter.InitWriter(reputationrouter.NewRouteContext(eCtx, passedRoute)) + w, err := s.localRouter.InitWriter(reputationrouter.NewRouteInfo(ep, passedRoute)) if err != nil { return nil, fmt.Errorf("could not initialize local trust writer: %w", err) } for _, trust := range body.GetTrusts() { - err = s.processLocalTrust(body.GetEpoch(), apiToLocalTrust(&trust, passedRoute[0].PublicKey()), passedRoute, w) + err = s.processLocalTrust(ctx, body.GetEpoch(), apiToLocalTrust(&trust, passedRoute[0].PublicKey()), passedRoute, w) if err != nil { return nil, fmt.Errorf("could not write one of local trusts: %w", err) } @@ -326,9 +325,9 @@ func (s *reputationServer) AnnounceIntermediateResult(ctx context.Context, req * body := req.GetBody() - eiCtx := eigentrust.NewIterContext(ctx, body.GetEpoch(), body.GetIteration()) + ei := eigentrust.NewEpochIteration(body.GetEpoch(), body.GetIteration()) - w, err := s.intermediateRouter.InitWriter(reputationrouter.NewRouteContext(eiCtx, passedRoute)) + w, err := s.intermediateRouter.InitWriter(reputationrouter.NewRouteInfo(ei, passedRoute)) if err != nil { return nil, fmt.Errorf("could not initialize trust writer: %w", err) } @@ -337,7 +336,7 @@ func (s *reputationServer) AnnounceIntermediateResult(ctx context.Context, req * trust := apiToLocalTrust(v2Trust.GetTrust(), v2Trust.GetTrustingPeer().GetPublicKey()) - err = w.Write(trust) + err = w.Write(ctx, trust) if err != nil { return nil, fmt.Errorf("could not write trust: %w", err) } @@ -348,14 +347,14 @@ func (s *reputationServer) AnnounceIntermediateResult(ctx context.Context, req * return resp, nil } -func (s *reputationServer) processLocalTrust(epoch uint64, t reputation.Trust, +func (s *reputationServer) processLocalTrust(ctx context.Context, epoch uint64, t reputation.Trust, passedRoute []reputationcommon.ServerInfo, w reputationcommon.Writer) error { err := reputationrouter.CheckRoute(s.routeBuilder, epoch, t, passedRoute) if err != nil { return fmt.Errorf("wrong route of reputation trust value: %w", err) } - return w.Write(t) + return w.Write(ctx, t) } // apiToLocalTrust converts v2 Trust to local reputation.Trust, adding trustingPeer. diff --git a/cmd/frostfs-node/reputation/common/remote.go b/cmd/frostfs-node/reputation/common/remote.go index 0fe0a7fd7..cd0a024a9 100644 --- a/cmd/frostfs-node/reputation/common/remote.go +++ b/cmd/frostfs-node/reputation/common/remote.go @@ -6,7 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - reputationrouter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common/router" trustcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/local/controller" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" ) @@ -22,12 +21,12 @@ type clientKeyRemoteProvider interface { WithClient(client.Client) reputationcommon.WriterProvider } -// remoteTrustProvider is an implementation of reputation RemoteWriterProvider interface. +// RemoteTrustProvider is an implementation of reputation RemoteWriterProvider interface. // It caches clients, checks if it is the end of the route and checks either the current // node is a remote target or not. // // remoteTrustProvider requires to be provided with clientKeyRemoteProvider. -type remoteTrustProvider struct { +type RemoteTrustProvider struct { netmapKeys netmap.AnnouncedKeys deadEndProvider reputationcommon.WriterProvider clientCache clientCache @@ -48,7 +47,7 @@ type RemoteProviderPrm struct { Log *logger.Logger } -func NewRemoteTrustProvider(prm RemoteProviderPrm) reputationrouter.RemoteWriterProvider { +func NewRemoteTrustProvider(prm RemoteProviderPrm) *RemoteTrustProvider { switch { case prm.NetmapKeys == nil: PanicOnPrmValue("NetmapKeys", prm.NetmapKeys) @@ -62,7 +61,7 @@ func NewRemoteTrustProvider(prm RemoteProviderPrm) reputationrouter.RemoteWriter PanicOnPrmValue("Logger", prm.Log) } - return &remoteTrustProvider{ + return &RemoteTrustProvider{ netmapKeys: prm.NetmapKeys, deadEndProvider: prm.DeadEndProvider, clientCache: prm.ClientCache, @@ -71,7 +70,7 @@ func NewRemoteTrustProvider(prm RemoteProviderPrm) reputationrouter.RemoteWriter } } -func (rtp *remoteTrustProvider) InitRemote(srv reputationcommon.ServerInfo) (reputationcommon.WriterProvider, error) { +func (rtp *RemoteTrustProvider) InitRemote(srv reputationcommon.ServerInfo) (reputationcommon.WriterProvider, error) { rtp.log.Debug("initializing remote writer provider") if srv == nil { diff --git a/cmd/frostfs-node/reputation/common/util.go b/cmd/frostfs-node/reputation/common/util.go index 28351d0ce..443adb388 100644 --- a/cmd/frostfs-node/reputation/common/util.go +++ b/cmd/frostfs-node/reputation/common/util.go @@ -7,24 +7,21 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" ) -// EpochContext is a std context extended with epoch data. -// nolint: containedctx -type EpochContext struct { - context.Context +type EpochProvider struct { E uint64 } -func (ctx *EpochContext) Epoch() uint64 { - return ctx.E +func (ep *EpochProvider) Epoch() uint64 { + return ep.E } type NopReputationWriter struct{} -func (NopReputationWriter) Write(reputation.Trust) error { +func (NopReputationWriter) Write(context.Context, reputation.Trust) error { return nil } -func (NopReputationWriter) Close() error { +func (NopReputationWriter) Close(context.Context) error { return nil } diff --git a/cmd/frostfs-node/reputation/intermediate/calculator.go b/cmd/frostfs-node/reputation/intermediate/calculator.go index 8bc74324f..73dd12311 100644 --- a/cmd/frostfs-node/reputation/intermediate/calculator.go +++ b/cmd/frostfs-node/reputation/intermediate/calculator.go @@ -1,6 +1,7 @@ package intermediate import ( + "context" "errors" "fmt" @@ -42,15 +43,15 @@ type DaughtersTrustCalculator struct { } // Calculate converts and passes values to the wrapped calculator. -func (c *DaughtersTrustCalculator) Calculate(ctx eigentrustctrl.IterationContext) { +func (c *DaughtersTrustCalculator) Calculate(ctx context.Context, iterCtx eigentrustctrl.IterationContext) { calcPrm := eigencalc.CalculatePrm{} epochIteration := eigentrust.EpochIteration{} - epochIteration.SetEpoch(ctx.Epoch()) - epochIteration.SetI(ctx.I()) + epochIteration.SetEpoch(iterCtx.Epoch()) + epochIteration.SetI(iterCtx.I()) - calcPrm.SetLast(ctx.Last()) + calcPrm.SetLast(iterCtx.Last()) calcPrm.SetEpochIteration(epochIteration) - c.Calculator.Calculate(calcPrm) + c.Calculator.Calculate(ctx, calcPrm) } diff --git a/cmd/frostfs-node/reputation/intermediate/consumers.go b/cmd/frostfs-node/reputation/intermediate/consumers.go index fb89c4396..33eab605b 100644 --- a/cmd/frostfs-node/reputation/intermediate/consumers.go +++ b/cmd/frostfs-node/reputation/intermediate/consumers.go @@ -1,6 +1,8 @@ package intermediate import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" @@ -23,41 +25,41 @@ type ConsumerStorageWriterProvider struct { // that writes passed consumer's Trust values to the Consumer storage. After writing // that, values can be used in eigenTrust algorithm's iterations. type ConsumerTrustWriter struct { - log *logger.Logger - storage *consumerstorage.Storage - eiCtx eigencalc.Context + log *logger.Logger + storage *consumerstorage.Storage + iterInfo eigencalc.EpochIterationInfo } -func (w *ConsumerTrustWriter) Write(t reputation.Trust) error { +func (w *ConsumerTrustWriter) Write(_ context.Context, t reputation.Trust) error { w.log.Debug("writing received consumer's trusts", - zap.Uint64("epoch", w.eiCtx.Epoch()), - zap.Uint32("iteration", w.eiCtx.I()), + zap.Uint64("epoch", w.iterInfo.Epoch()), + zap.Uint32("iteration", w.iterInfo.I()), zap.Stringer("trusting_peer", t.TrustingPeer()), zap.Stringer("trusted_peer", t.Peer()), ) trust := eigentrust.IterationTrust{Trust: t} - trust.SetEpoch(w.eiCtx.Epoch()) - trust.SetI(w.eiCtx.I()) + trust.SetEpoch(w.iterInfo.Epoch()) + trust.SetI(w.iterInfo.I()) w.storage.Put(trust) return nil } -func (w *ConsumerTrustWriter) Close() error { +func (w *ConsumerTrustWriter) Close(context.Context) error { return nil } -func (s *ConsumerStorageWriterProvider) InitWriter(ctx reputationcommon.Context) (reputationcommon.Writer, error) { - eiCtx, ok := ctx.(eigencalc.Context) +func (s *ConsumerStorageWriterProvider) InitWriter(ep reputationcommon.EpochProvider) (reputationcommon.Writer, error) { + iterInfo, ok := ep.(eigencalc.EpochIterationInfo) if !ok { panic(ErrIncorrectContextPanicMsg) } return &ConsumerTrustWriter{ - log: s.Log, - storage: s.Storage, - eiCtx: eiCtx, + log: s.Log, + storage: s.Storage, + iterInfo: iterInfo, }, nil } diff --git a/cmd/frostfs-node/reputation/intermediate/contract.go b/cmd/frostfs-node/reputation/intermediate/contract.go index 7e6411329..6303b1219 100644 --- a/cmd/frostfs-node/reputation/intermediate/contract.go +++ b/cmd/frostfs-node/reputation/intermediate/contract.go @@ -51,7 +51,7 @@ type FinalWriterProvider struct { } func (fwp FinalWriterProvider) InitIntermediateWriter( - _ eigentrustcalc.Context) (eigentrustcalc.IntermediateWriter, error) { + _ eigentrustcalc.EpochIterationInfo) (eigentrustcalc.IntermediateWriter, error) { return &FinalWriter{ privatKey: fwp.prm.PrivatKey, pubKey: fwp.prm.PubKey, diff --git a/cmd/frostfs-node/reputation/intermediate/daughters.go b/cmd/frostfs-node/reputation/intermediate/daughters.go index 641a0afe2..d72eead43 100644 --- a/cmd/frostfs-node/reputation/intermediate/daughters.go +++ b/cmd/frostfs-node/reputation/intermediate/daughters.go @@ -1,6 +1,8 @@ package intermediate import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/storage/daughters" @@ -21,28 +23,28 @@ type DaughterStorageWriterProvider struct { type DaughterTrustWriter struct { log *logger.Logger storage *daughters.Storage - ctx reputationcommon.Context + ep reputationcommon.EpochProvider } -func (w *DaughterTrustWriter) Write(t reputation.Trust) error { +func (w *DaughterTrustWriter) Write(_ context.Context, t reputation.Trust) error { w.log.Debug("writing received daughter's trusts", - zap.Uint64("epoch", w.ctx.Epoch()), + zap.Uint64("epoch", w.ep.Epoch()), zap.Stringer("trusting_peer", t.TrustingPeer()), zap.Stringer("trusted_peer", t.Peer()), ) - w.storage.Put(w.ctx.Epoch(), t) + w.storage.Put(w.ep.Epoch(), t) return nil } -func (w *DaughterTrustWriter) Close() error { +func (w *DaughterTrustWriter) Close(context.Context) error { return nil } -func (s *DaughterStorageWriterProvider) InitWriter(ctx reputationcommon.Context) (reputationcommon.Writer, error) { +func (s *DaughterStorageWriterProvider) InitWriter(ep reputationcommon.EpochProvider) (reputationcommon.Writer, error) { return &DaughterTrustWriter{ log: s.Log, storage: s.Storage, - ctx: ctx, + ep: ep, }, nil } diff --git a/cmd/frostfs-node/reputation/intermediate/remote.go b/cmd/frostfs-node/reputation/intermediate/remote.go index 224da9439..b1a218b94 100644 --- a/cmd/frostfs-node/reputation/intermediate/remote.go +++ b/cmd/frostfs-node/reputation/intermediate/remote.go @@ -1,6 +1,7 @@ package intermediate import ( + "context" "crypto/ecdsa" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/common" @@ -64,32 +65,32 @@ type TrustWriterProvider struct { log *logger.Logger } -func (twp *TrustWriterProvider) InitWriter(ctx reputationcommon.Context) (reputationcommon.Writer, error) { - eiContext, ok := ctx.(eigentrustcalc.Context) +func (twp *TrustWriterProvider) InitWriter(ep reputationcommon.EpochProvider) (reputationcommon.Writer, error) { + iterInfo, ok := ep.(eigentrustcalc.EpochIterationInfo) if !ok { // TODO: #1164 think if this can be done without such limitation panic(ErrIncorrectContextPanicMsg) } return &RemoteTrustWriter{ - eiCtx: eiContext, - client: twp.client, - key: twp.key, - log: twp.log, + iterInfo: iterInfo, + client: twp.client, + key: twp.key, + log: twp.log, }, nil } type RemoteTrustWriter struct { - eiCtx eigentrustcalc.Context - client coreclient.Client - key *ecdsa.PrivateKey - log *logger.Logger + iterInfo eigentrustcalc.EpochIterationInfo + client coreclient.Client + key *ecdsa.PrivateKey + log *logger.Logger } // Write sends a trust value to a remote node via ReputationService.AnnounceIntermediateResult RPC. -func (rtp *RemoteTrustWriter) Write(t reputation.Trust) error { - epoch := rtp.eiCtx.Epoch() - i := rtp.eiCtx.I() +func (rtp *RemoteTrustWriter) Write(ctx context.Context, t reputation.Trust) error { + epoch := rtp.iterInfo.Epoch() + i := rtp.iterInfo.I() rtp.log.Debug("announcing trust", zap.Uint64("epoch", epoch), @@ -108,17 +109,16 @@ func (rtp *RemoteTrustWriter) Write(t reputation.Trust) error { var p internalclient.AnnounceIntermediatePrm - p.SetContext(rtp.eiCtx) p.SetClient(rtp.client) p.SetEpoch(epoch) p.SetIteration(i) p.SetTrust(apiPeerToPeerTrust) - _, err := internalclient.AnnounceIntermediate(p) + _, err := internalclient.AnnounceIntermediate(ctx, p) return err } -func (rtp *RemoteTrustWriter) Close() error { +func (rtp *RemoteTrustWriter) Close(context.Context) error { return nil } diff --git a/cmd/frostfs-node/reputation/intermediate/storage.go b/cmd/frostfs-node/reputation/intermediate/storage.go index 0f614d9f2..db29ff92b 100644 --- a/cmd/frostfs-node/reputation/intermediate/storage.go +++ b/cmd/frostfs-node/reputation/intermediate/storage.go @@ -18,7 +18,7 @@ type DaughterTrustIteratorProvider struct { // InitDaughterIterator returns an iterator over the received // local trusts for ctx.Epoch() epoch from daughter p. -func (ip *DaughterTrustIteratorProvider) InitDaughterIterator(ctx eigentrustcalc.Context, +func (ip *DaughterTrustIteratorProvider) InitDaughterIterator(ctx eigentrustcalc.EpochIterationInfo, p apireputation.PeerID) (eigentrustcalc.TrustIterator, error) { epoch := ctx.Epoch() @@ -34,7 +34,7 @@ func (ip *DaughterTrustIteratorProvider) InitDaughterIterator(ctx eigentrustcalc // daughters of the current node(manager) and all local // trusts received from them for ctx.Epoch() epoch. func (ip *DaughterTrustIteratorProvider) InitAllDaughtersIterator( - ctx eigentrustcalc.Context) (eigentrustcalc.PeerTrustsIterator, error) { + ctx eigentrustcalc.EpochIterationInfo) (eigentrustcalc.PeerTrustsIterator, error) { epoch := ctx.Epoch() iter, ok := ip.DaughterStorage.AllDaughterTrusts(epoch) @@ -49,7 +49,7 @@ func (ip *DaughterTrustIteratorProvider) InitAllDaughtersIterator( // of the current node(manager) and all their consumers' local // trusts for ctx.Epoch() epoch and ctx.I() iteration. func (ip *DaughterTrustIteratorProvider) InitConsumersIterator( - ctx eigentrustcalc.Context) (eigentrustcalc.PeerTrustsIterator, error) { + ctx eigentrustcalc.EpochIterationInfo) (eigentrustcalc.PeerTrustsIterator, error) { epoch, iter := ctx.Epoch(), ctx.I() consumerIterator, ok := ip.ConsumerStorage.Consumers(epoch, iter) diff --git a/cmd/frostfs-node/reputation/internal/client/client.go b/cmd/frostfs-node/reputation/internal/client/client.go index 22fd21d4a..ff5131262 100644 --- a/cmd/frostfs-node/reputation/internal/client/client.go +++ b/cmd/frostfs-node/reputation/internal/client/client.go @@ -9,11 +9,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" ) -// nolint: containedctx type commonPrm struct { cli coreclient.Client - - ctx context.Context } // SetClient sets the base client for FrostFS API communication. @@ -23,13 +20,6 @@ func (x *commonPrm) SetClient(cli coreclient.Client) { x.cli = cli } -// SetContext sets context.Context for network communication. -// -// Required parameter. -func (x *commonPrm) SetContext(ctx context.Context) { - x.ctx = ctx -} - // AnnounceLocalPrm groups parameters of AnnounceLocal operation. type AnnounceLocalPrm struct { commonPrm @@ -55,10 +45,10 @@ type AnnounceLocalRes struct{} // Client, context and key must be set. // // Returns any error which prevented the operation from completing correctly in error return. -func AnnounceLocal(prm AnnounceLocalPrm) (res AnnounceLocalRes, err error) { +func AnnounceLocal(ctx context.Context, prm AnnounceLocalPrm) (res AnnounceLocalRes, err error) { var cliRes *client.ResAnnounceLocalTrust - cliRes, err = prm.cli.AnnounceLocalTrust(prm.ctx, prm.cliPrm) + cliRes, err = prm.cli.AnnounceLocalTrust(ctx, prm.cliPrm) if err == nil { // pull out an error from status err = apistatus.ErrFromStatus(cliRes.Status()) @@ -98,10 +88,10 @@ type AnnounceIntermediateRes struct{} // Client, context and key must be set. // // Returns any error which prevented the operation from completing correctly in error return. -func AnnounceIntermediate(prm AnnounceIntermediatePrm) (res AnnounceIntermediateRes, err error) { +func AnnounceIntermediate(ctx context.Context, prm AnnounceIntermediatePrm) (res AnnounceIntermediateRes, err error) { var cliRes *client.ResAnnounceIntermediateTrust - cliRes, err = prm.cli.AnnounceIntermediateTrust(prm.ctx, prm.cliPrm) + cliRes, err = prm.cli.AnnounceIntermediateTrust(ctx, prm.cliPrm) if err == nil { // pull out an error from status err = apistatus.ErrFromStatus(cliRes.Status()) diff --git a/cmd/frostfs-node/reputation/local/remote.go b/cmd/frostfs-node/reputation/local/remote.go index 2fa93ff6f..3c929a9ca 100644 --- a/cmd/frostfs-node/reputation/local/remote.go +++ b/cmd/frostfs-node/reputation/local/remote.go @@ -1,6 +1,7 @@ package local import ( + "context" "crypto/ecdsa" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/common" @@ -63,9 +64,9 @@ type TrustWriterProvider struct { log *logger.Logger } -func (twp *TrustWriterProvider) InitWriter(ctx reputationcommon.Context) (reputationcommon.Writer, error) { +func (twp *TrustWriterProvider) InitWriter(ep reputationcommon.EpochProvider) (reputationcommon.Writer, error) { return &RemoteTrustWriter{ - ctx: ctx, + ep: ep, client: twp.client, key: twp.key, log: twp.log, @@ -73,7 +74,7 @@ func (twp *TrustWriterProvider) InitWriter(ctx reputationcommon.Context) (reputa } type RemoteTrustWriter struct { - ctx reputationcommon.Context + ep reputationcommon.EpochProvider client coreclient.Client key *ecdsa.PrivateKey log *logger.Logger @@ -81,7 +82,7 @@ type RemoteTrustWriter struct { buf []reputationapi.Trust } -func (rtp *RemoteTrustWriter) Write(t reputation.Trust) error { +func (rtp *RemoteTrustWriter) Write(_ context.Context, t reputation.Trust) error { var apiTrust reputationapi.Trust apiTrust.SetValue(t.Value().Float64()) @@ -92,8 +93,8 @@ func (rtp *RemoteTrustWriter) Write(t reputation.Trust) error { return nil } -func (rtp *RemoteTrustWriter) Close() error { - epoch := rtp.ctx.Epoch() +func (rtp *RemoteTrustWriter) Close(ctx context.Context) error { + epoch := rtp.ep.Epoch() rtp.log.Debug("announcing trusts", zap.Uint64("epoch", epoch), @@ -101,12 +102,11 @@ func (rtp *RemoteTrustWriter) Close() error { var prm internalclient.AnnounceLocalPrm - prm.SetContext(rtp.ctx) prm.SetClient(rtp.client) prm.SetEpoch(epoch) prm.SetTrusts(rtp.buf) - _, err := internalclient.AnnounceLocal(prm) + _, err := internalclient.AnnounceLocal(ctx, prm) return err } diff --git a/cmd/frostfs-node/reputation/local/storage.go b/cmd/frostfs-node/reputation/local/storage.go index 92d10dfee..861151871 100644 --- a/cmd/frostfs-node/reputation/local/storage.go +++ b/cmd/frostfs-node/reputation/local/storage.go @@ -24,8 +24,8 @@ type TrustStorage struct { LocalKey []byte } -func (s *TrustStorage) InitIterator(ctx reputationcommon.Context) (trustcontroller.Iterator, error) { - epoch := ctx.Epoch() +func (s *TrustStorage) InitIterator(ep reputationcommon.EpochProvider) (trustcontroller.Iterator, error) { + epoch := ep.Epoch() s.Log.Debug("initializing iterator over trusts", zap.Uint64("epoch", epoch), @@ -37,14 +37,14 @@ func (s *TrustStorage) InitIterator(ctx reputationcommon.Context) (trustcontroll } return &TrustIterator{ - ctx: ctx, + ep: ep, storage: s, epochStorage: epochStorage, }, nil } type TrustIterator struct { - ctx reputationcommon.Context + ep reputationcommon.EpochProvider storage *TrustStorage @@ -59,7 +59,7 @@ func (it *TrustIterator) Iterate(h reputation.TrustHandler) error { } } - nm, err := it.storage.NmSrc.GetNetMapByEpoch(it.ctx.Epoch()) + nm, err := it.storage.NmSrc.GetNetMapByEpoch(it.ep.Epoch()) if err != nil { return err } diff --git a/pkg/services/reputation/common/deps.go b/pkg/services/reputation/common/deps.go index ebb227b5e..3ea5aa88e 100644 --- a/pkg/services/reputation/common/deps.go +++ b/pkg/services/reputation/common/deps.go @@ -2,17 +2,12 @@ package common import ( "context" - "io" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" ) -// Context wraps stdlib context -// with accompanying meta values. -type Context interface { - context.Context - +type EpochProvider interface { // Must return epoch number to select the values. Epoch() uint64 } @@ -30,7 +25,7 @@ type Writer interface { // Close operation. // // Write must not be called after Close. - Write(reputation.Trust) error + Write(context.Context, reputation.Trust) error // Close exits with method-providing Writer. // @@ -38,7 +33,7 @@ type Writer interface { // the Close's return. // // Methods must not be called after Close. - io.Closer + Close(context.Context) error } // WriterProvider is a group of methods provided @@ -52,7 +47,7 @@ type WriterProvider interface { // // Implementations can have different logic for different // contexts, so specific ones may document their own behavior. - InitWriter(Context) (Writer, error) + InitWriter(EpochProvider) (Writer, error) } // ManagerBuilder defines an interface for providing a list diff --git a/pkg/services/reputation/common/router/calls.go b/pkg/services/reputation/common/router/calls.go index 75cdf56ea..a177f6a2b 100644 --- a/pkg/services/reputation/common/router/calls.go +++ b/pkg/services/reputation/common/router/calls.go @@ -1,6 +1,7 @@ package router import ( + "context" "encoding/hex" "sync" @@ -9,27 +10,27 @@ import ( "go.uber.org/zap" ) -// routeContext wraps context with additional passed +// RouteInfo wraps epoch provider with additional passed // route data. It is only used inside Router and is // not passed in any external methods. -type routeContext struct { - common.Context +type RouteInfo struct { + common.EpochProvider passedRoute []common.ServerInfo } -// NewRouteContext wraps the main context of value passing with its traversal route and epoch. -func NewRouteContext(ctx common.Context, passed []common.ServerInfo) common.Context { - return &routeContext{ - Context: ctx, - passedRoute: passed, +// NewRouteInfo wraps the main context of value passing with its traversal route and epoch. +func NewRouteInfo(ep common.EpochProvider, passed []common.ServerInfo) *RouteInfo { + return &RouteInfo{ + EpochProvider: ep, + passedRoute: passed, } } type trustWriter struct { router *Router - routeCtx *routeContext + routeInfo *RouteInfo routeMtx sync.RWMutex mServers map[string]common.Writer @@ -37,7 +38,7 @@ type trustWriter struct { // InitWriter initializes and returns Writer that sends each value to its next route point. // -// If ctx was created by NewRouteContext, then the traversed route is taken into account, +// If ep was created by NewRouteInfo, then the traversed route is taken into account, // and the value will be sent to its continuation. Otherwise, the route will be laid // from scratch and the value will be sent to its primary point. // @@ -49,31 +50,31 @@ type trustWriter struct { // runtime and never returns an error. // // Always returns nil error. -func (r *Router) InitWriter(ctx common.Context) (common.Writer, error) { +func (r *Router) InitWriter(ep common.EpochProvider) (common.Writer, error) { var ( - routeCtx *routeContext - ok bool + routeInfo *RouteInfo + ok bool ) - if routeCtx, ok = ctx.(*routeContext); !ok { - routeCtx = &routeContext{ - Context: ctx, - passedRoute: []common.ServerInfo{r.localSrvInfo}, + if routeInfo, ok = ep.(*RouteInfo); !ok { + routeInfo = &RouteInfo{ + EpochProvider: ep, + passedRoute: []common.ServerInfo{r.localSrvInfo}, } } return &trustWriter{ - router: r, - routeCtx: routeCtx, - mServers: make(map[string]common.Writer), + router: r, + routeInfo: routeInfo, + mServers: make(map[string]common.Writer), }, nil } -func (w *trustWriter) Write(t reputation.Trust) error { +func (w *trustWriter) Write(ctx context.Context, t reputation.Trust) error { w.routeMtx.Lock() defer w.routeMtx.Unlock() - route, err := w.router.routeBuilder.NextStage(w.routeCtx.Epoch(), t, w.routeCtx.passedRoute) + route, err := w.router.routeBuilder.NextStage(w.routeInfo.Epoch(), t, w.routeInfo.passedRoute) if err != nil { return err } else if len(route) == 0 { @@ -99,7 +100,7 @@ func (w *trustWriter) Write(t reputation.Trust) error { } // init writer with original context wrapped in routeContext - remoteWriter, err = provider.InitWriter(w.routeCtx.Context) + remoteWriter, err = provider.InitWriter(w.routeInfo.EpochProvider) if err != nil { w.router.log.Debug("could not initialize writer", zap.String("error", err.Error()), @@ -111,7 +112,7 @@ func (w *trustWriter) Write(t reputation.Trust) error { w.mServers[key] = remoteWriter } - err := remoteWriter.Write(t) + err := remoteWriter.Write(ctx, t) if err != nil { w.router.log.Debug("could not write the value", zap.String("error", err.Error()), @@ -122,9 +123,9 @@ func (w *trustWriter) Write(t reputation.Trust) error { return nil } -func (w *trustWriter) Close() error { +func (w *trustWriter) Close(ctx context.Context) error { for key, wRemote := range w.mServers { - err := wRemote.Close() + err := wRemote.Close(ctx) if err != nil { w.router.log.Debug("could not close remote server writer", zap.String("key", key), diff --git a/pkg/services/reputation/eigentrust/calculator/calls.go b/pkg/services/reputation/eigentrust/calculator/calls.go index 23e418729..2cc789780 100644 --- a/pkg/services/reputation/eigentrust/calculator/calls.go +++ b/pkg/services/reputation/eigentrust/calculator/calls.go @@ -23,7 +23,7 @@ func (p *CalculatePrm) SetEpochIteration(ei eigentrust.EpochIteration) { p.ei = ei } -func (c *Calculator) Calculate(prm CalculatePrm) { +func (c *Calculator) Calculate(ctx context.Context, prm CalculatePrm) { alpha, err := c.prm.AlphaProvider.EigenTrustAlpha() if err != nil { c.opts.log.Debug( @@ -36,28 +36,25 @@ func (c *Calculator) Calculate(prm CalculatePrm) { c.alpha = reputation.TrustValueFromFloat64(alpha) c.beta = reputation.TrustValueFromFloat64(1 - alpha) - ctx := eigentrust.IterContext{ - Context: context.Background(), - EpochIteration: prm.ei, - } + epochIteration := prm.ei - iter := ctx.I() + iter := epochIteration.I() log := c.opts.log.With( - zap.Uint64("epoch", ctx.Epoch()), + zap.Uint64("epoch", epochIteration.Epoch()), zap.Uint32("iteration", iter), ) if iter == 0 { - c.sendInitialValues(ctx) + c.sendInitialValues(ctx, epochIteration) return } // decrement iteration number to select the values collected // on the previous stage - ctx.SetI(iter - 1) + epochIteration.SetI(iter - 1) - consumersIter, err := c.prm.DaughterTrustSource.InitConsumersIterator(ctx) + consumersIter, err := c.prm.DaughterTrustSource.InitConsumersIterator(epochIteration) if err != nil { log.Debug("consumers trust iterator's init failure", zap.String("error", err.Error()), @@ -67,13 +64,13 @@ func (c *Calculator) Calculate(prm CalculatePrm) { } // continue with initial iteration number - ctx.SetI(iter) + epochIteration.SetI(iter) err = consumersIter.Iterate(func(daughter apireputation.PeerID, iter TrustIterator) error { err := c.prm.WorkerPool.Submit(func() { - c.iterateDaughter(iterDaughterPrm{ + c.iterateDaughter(ctx, iterDaughterPrm{ lastIter: prm.last, - ctx: ctx, + ei: epochIteration, id: daughter, consumersIter: iter, }) @@ -97,7 +94,7 @@ func (c *Calculator) Calculate(prm CalculatePrm) { type iterDaughterPrm struct { lastIter bool - ctx Context + ei EpochIterationInfo id apireputation.PeerID @@ -105,7 +102,7 @@ type iterDaughterPrm struct { } // nolint: funlen -func (c *Calculator) iterateDaughter(p iterDaughterPrm) { +func (c *Calculator) iterateDaughter(ctx context.Context, p iterDaughterPrm) { initTrust, err := c.prm.InitialTrustSource.InitialTrust(p.id) if err != nil { c.opts.log.Debug("get initial trust failure", @@ -116,7 +113,7 @@ func (c *Calculator) iterateDaughter(p iterDaughterPrm) { return } - daughterIter, err := c.prm.DaughterTrustSource.InitDaughterIterator(p.ctx, p.id) + daughterIter, err := c.prm.DaughterTrustSource.InitDaughterIterator(p.ei, p.id) if err != nil { c.opts.log.Debug("daughter trust iterator's init failure", zap.String("error", err.Error()), @@ -130,8 +127,8 @@ func (c *Calculator) iterateDaughter(p iterDaughterPrm) { err = p.consumersIter.Iterate(func(trust reputation.Trust) error { if !p.lastIter { select { - case <-p.ctx.Done(): - return p.ctx.Err() + case <-ctx.Done(): + return ctx.Err() default: } } @@ -155,12 +152,12 @@ func (c *Calculator) iterateDaughter(p iterDaughterPrm) { var intermediateTrust eigentrust.IterationTrust - intermediateTrust.SetEpoch(p.ctx.Epoch()) + intermediateTrust.SetEpoch(p.ei.Epoch()) intermediateTrust.SetPeer(p.id) - intermediateTrust.SetI(p.ctx.I()) + intermediateTrust.SetI(p.ei.I()) if p.lastIter { - finalWriter, err := c.prm.FinalResultTarget.InitIntermediateWriter(p.ctx) + finalWriter, err := c.prm.FinalResultTarget.InitIntermediateWriter(p.ei) if err != nil { c.opts.log.Debug("init writer failure", zap.String("error", err.Error()), @@ -180,7 +177,7 @@ func (c *Calculator) iterateDaughter(p iterDaughterPrm) { return } } else { - intermediateWriter, err := c.prm.IntermediateValueTarget.InitWriter(p.ctx) + intermediateWriter, err := c.prm.IntermediateValueTarget.InitWriter(p.ei) if err != nil { c.opts.log.Debug("init writer failure", zap.String("error", err.Error()), @@ -191,8 +188,8 @@ func (c *Calculator) iterateDaughter(p iterDaughterPrm) { err = daughterIter.Iterate(func(trust reputation.Trust) error { select { - case <-p.ctx.Done(): - return p.ctx.Err() + case <-ctx.Done(): + return ctx.Err() default: } @@ -201,7 +198,7 @@ func (c *Calculator) iterateDaughter(p iterDaughterPrm) { trust.SetValue(val) - err := intermediateWriter.Write(trust) + err := intermediateWriter.Write(ctx, trust) if err != nil { c.opts.log.Debug("write value failure", zap.String("error", err.Error()), @@ -216,7 +213,7 @@ func (c *Calculator) iterateDaughter(p iterDaughterPrm) { ) } - err = intermediateWriter.Close() + err = intermediateWriter.Close(ctx) if err != nil { c.opts.log.Error( "could not close writer", @@ -226,8 +223,8 @@ func (c *Calculator) iterateDaughter(p iterDaughterPrm) { } } -func (c *Calculator) sendInitialValues(ctx Context) { - daughterIter, err := c.prm.DaughterTrustSource.InitAllDaughtersIterator(ctx) +func (c *Calculator) sendInitialValues(ctx context.Context, epochInfo EpochIterationInfo) { + daughterIter, err := c.prm.DaughterTrustSource.InitAllDaughtersIterator(epochInfo) if err != nil { c.opts.log.Debug("all daughters trust iterator's init failure", zap.String("error", err.Error()), @@ -236,7 +233,7 @@ func (c *Calculator) sendInitialValues(ctx Context) { return } - intermediateWriter, err := c.prm.IntermediateValueTarget.InitWriter(ctx) + intermediateWriter, err := c.prm.IntermediateValueTarget.InitWriter(epochInfo) if err != nil { c.opts.log.Debug("init writer failure", zap.String("error", err.Error()), @@ -263,7 +260,7 @@ func (c *Calculator) sendInitialValues(ctx Context) { initTrust.Mul(trust.Value()) trust.SetValue(initTrust) - err = intermediateWriter.Write(trust) + err = intermediateWriter.Write(ctx, trust) if err != nil { c.opts.log.Debug("write value failure", zap.String("error", err.Error()), @@ -281,7 +278,7 @@ func (c *Calculator) sendInitialValues(ctx Context) { ) } - err = intermediateWriter.Close() + err = intermediateWriter.Close(ctx) if err != nil { c.opts.log.Debug("could not close writer", zap.String("error", err.Error()), diff --git a/pkg/services/reputation/eigentrust/calculator/deps.go b/pkg/services/reputation/eigentrust/calculator/deps.go index 66d3fd301..a22d1df76 100644 --- a/pkg/services/reputation/eigentrust/calculator/deps.go +++ b/pkg/services/reputation/eigentrust/calculator/deps.go @@ -1,16 +1,12 @@ package eigentrustcalc import ( - "context" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" ) -type Context interface { - context.Context - +type EpochIterationInfo interface { // Must return epoch number to select the values // for global trust calculation. Epoch() uint64 @@ -43,19 +39,19 @@ type PeerTrustsIterator interface { type DaughterTrustIteratorProvider interface { // InitDaughterIterator must init TrustIterator // that iterates over received local trusts from - // daughter p for ctx.Epoch() epoch. - InitDaughterIterator(ctx Context, p apireputation.PeerID) (TrustIterator, error) + // daughter p for epochInfo.Epoch() epoch. + InitDaughterIterator(epochInfo EpochIterationInfo, p apireputation.PeerID) (TrustIterator, error) // InitAllDaughtersIterator must init PeerTrustsIterator // that must iterate over all daughters of the current // node(manager) and all trusts received from them for - // ctx.Epoch() epoch. - InitAllDaughtersIterator(ctx Context) (PeerTrustsIterator, error) + // epochInfo.Epoch() epoch. + InitAllDaughtersIterator(epochInfo EpochIterationInfo) (PeerTrustsIterator, error) // InitConsumersIterator must init PeerTrustsIterator // that must iterate over all daughters of the current // node(manager) and their consumers' trusts received - // from other managers for ctx.Epoch() epoch and - // ctx.I() iteration. - InitConsumersIterator(Context) (PeerTrustsIterator, error) + // from other managers for epochInfo.Epoch() epoch and + // epochInfo.I() iteration. + InitConsumersIterator(EpochIterationInfo) (PeerTrustsIterator, error) } // IntermediateWriter must write intermediate result to contract. @@ -68,7 +64,7 @@ type IntermediateWriter interface { // IntermediateWriterProvider must provide ready-to-work // IntermediateWriter. type IntermediateWriterProvider interface { - InitIntermediateWriter(Context) (IntermediateWriter, error) + InitIntermediateWriter(EpochIterationInfo) (IntermediateWriter, error) } // AlphaProvider must provide information about required diff --git a/pkg/services/reputation/eigentrust/controller/calls.go b/pkg/services/reputation/eigentrust/controller/calls.go index faf953aed..1753a430b 100644 --- a/pkg/services/reputation/eigentrust/controller/calls.go +++ b/pkg/services/reputation/eigentrust/controller/calls.go @@ -1,6 +1,8 @@ package eigentrustctrl import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" "go.uber.org/zap" ) @@ -22,7 +24,7 @@ func (x iterContext) Last() bool { } // Continue moves the global reputation calculator to the next iteration. -func (c *Controller) Continue(prm ContinuePrm) { +func (c *Controller) Continue(ctx context.Context, prm ContinuePrm) { c.mtx.Lock() { @@ -46,7 +48,7 @@ func (c *Controller) Continue(prm ContinuePrm) { iterCtx.last = iterCtx.I() == iterCtx.iterationNumber-1 err := c.prm.WorkerPool.Submit(func() { - c.prm.DaughtersTrustCalculator.Calculate(iterCtx) + c.prm.DaughtersTrustCalculator.Calculate(ctx, iterCtx) // iteration++ iterCtx.Increment() diff --git a/pkg/services/reputation/eigentrust/controller/deps.go b/pkg/services/reputation/eigentrust/controller/deps.go index 2aeafb61a..c068f7cc4 100644 --- a/pkg/services/reputation/eigentrust/controller/deps.go +++ b/pkg/services/reputation/eigentrust/controller/deps.go @@ -1,5 +1,7 @@ package eigentrustctrl +import "context" + // IterationContext is a context of the i-th // stage of iterative EigenTrust algorithm. type IterationContext interface { @@ -25,7 +27,7 @@ type DaughtersTrustCalculator interface { // http://ilpubs.stanford.edu:8090/562/1/2002-56.pdf Ch.5.1. // // Execution should be interrupted if ctx.Last(). - Calculate(ctx IterationContext) + Calculate(ctx context.Context, iter IterationContext) } // IterationsProvider must provide information about numbers diff --git a/pkg/services/reputation/eigentrust/iteration.go b/pkg/services/reputation/eigentrust/iteration.go index b06064f78..e4793f044 100644 --- a/pkg/services/reputation/eigentrust/iteration.go +++ b/pkg/services/reputation/eigentrust/iteration.go @@ -1,8 +1,6 @@ package eigentrust import ( - "context" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" ) @@ -36,22 +34,11 @@ type IterationTrust struct { reputation.Trust } -// IterContext aggregates context and data required for -// iterations. -// nolint: containedctx -type IterContext struct { - context.Context - EpochIteration -} - -func NewIterContext(ctx context.Context, epoch uint64, iter uint32) *IterContext { +func NewEpochIteration(epoch uint64, iter uint32) *EpochIteration { ei := EpochIteration{} ei.SetI(iter) ei.SetEpoch(epoch) - return &IterContext{ - Context: ctx, - EpochIteration: ei, - } + return &ei } diff --git a/pkg/services/reputation/local/controller/calls.go b/pkg/services/reputation/local/controller/calls.go index 98815492c..80fa772d6 100644 --- a/pkg/services/reputation/local/controller/calls.go +++ b/pkg/services/reputation/local/controller/calls.go @@ -27,80 +27,75 @@ func (p *ReportPrm) SetEpoch(e uint64) { // // Each call acquires a report context for an Epoch parameter. // At the very end of the operation, the context is released. -func (c *Controller) Report(prm ReportPrm) { +func (c *Controller) Report(ctx context.Context, prm ReportPrm) { // acquire report - reportCtx := c.acquireReport(prm.epoch) - if reportCtx == nil { + rCtx, reporter := c.acquireReporter(ctx, prm.epoch) + if reporter == nil { return } // report local trust values - reportCtx.report() + reporter.report(rCtx) // finally stop and free the report - c.freeReport(prm.epoch, reportCtx.log) + c.freeReport(prm.epoch, reporter.log) } -type reportContext struct { +type reporter struct { epoch uint64 ctrl *Controller log *logger.Logger - ctx common.Context + ep common.EpochProvider } -// nolint: containedctx -type iteratorContext struct { - context.Context - +type epochProvider struct { epoch uint64 } -func (c iteratorContext) Epoch() uint64 { +func (c epochProvider) Epoch() uint64 { return c.epoch } -func (c *Controller) acquireReport(epoch uint64) *reportContext { - var ctx context.Context +func (c *Controller) acquireReporter(ctx context.Context, epoch uint64) (context.Context, *reporter) { + started := true c.mtx.Lock() - { if cancel := c.mCtx[epoch]; cancel == nil { - ctx, cancel = context.WithCancel(context.Background()) + ctx, cancel = context.WithCancel(ctx) c.mCtx[epoch] = cancel + started = false } } - c.mtx.Unlock() log := &logger.Logger{Logger: c.opts.log.With( zap.Uint64("epoch", epoch), )} - if ctx == nil { + if started { log.Debug("report is already started") - return nil + return ctx, nil } - return &reportContext{ + return ctx, &reporter{ epoch: epoch, ctrl: c, log: log, - ctx: &iteratorContext{ - Context: ctx, - epoch: epoch, + ep: &epochProvider{ + epoch: epoch, }, } } -func (c *reportContext) report() { +func (c *reporter) report(ctx context.Context) { c.log.Debug("starting to report local trust values") // initialize iterator over locally collected values - iterator, err := c.ctrl.prm.LocalTrustSource.InitIterator(c.ctx) + iterator, err := c.ctrl.prm.LocalTrustSource.InitIterator(c.ep) if err != nil { c.log.Debug("could not initialize iterator over local trust values", zap.String("error", err.Error()), @@ -110,7 +105,7 @@ func (c *reportContext) report() { } // initialize target of local trust values - targetWriter, err := c.ctrl.prm.LocalTrustTarget.InitWriter(c.ctx) + targetWriter, err := c.ctrl.prm.LocalTrustTarget.InitWriter(c.ep) if err != nil { c.log.Debug("could not initialize local trust target", zap.String("error", err.Error()), @@ -123,11 +118,11 @@ func (c *reportContext) report() { err = iterator.Iterate( func(t reputation.Trust) error { // check if context is done - if err := c.ctx.Err(); err != nil { + if err := ctx.Err(); err != nil { return err } - return targetWriter.Write(t) + return targetWriter.Write(ctx, t) }, ) if err != nil && !errors.Is(err, context.Canceled) { @@ -139,7 +134,7 @@ func (c *reportContext) report() { } // finish writing - err = targetWriter.Close() + err = targetWriter.Close(ctx) if err != nil { c.log.Debug("could not finish writing local trust values", zap.String("error", err.Error()), diff --git a/pkg/services/reputation/local/controller/controller.go b/pkg/services/reputation/local/controller/controller.go index 7bf56be89..373df36db 100644 --- a/pkg/services/reputation/local/controller/controller.go +++ b/pkg/services/reputation/local/controller/controller.go @@ -5,7 +5,7 @@ import ( "fmt" "sync" - reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" + reputationrouter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common/router" ) // Prm groups the required parameters of the Controller's constructor. @@ -24,7 +24,7 @@ type Prm struct { // trust to other nodes. // // Must not be nil. - LocalTrustTarget reputationcommon.WriterProvider + LocalTrustTarget *reputationrouter.Router } // Controller represents main handler for starting diff --git a/pkg/services/reputation/local/controller/deps.go b/pkg/services/reputation/local/controller/deps.go index 3ab72eb5c..6f4a29c99 100644 --- a/pkg/services/reputation/local/controller/deps.go +++ b/pkg/services/reputation/local/controller/deps.go @@ -30,5 +30,5 @@ type IteratorProvider interface { // // Implementations can have different logic for different // contexts, so specific ones may document their own behavior. - InitIterator(common.Context) (Iterator, error) + InitIterator(common.EpochProvider) (Iterator, error) } diff --git a/pkg/services/reputation/local/controller/util.go b/pkg/services/reputation/local/controller/util.go index 97b9e3a65..122550498 100644 --- a/pkg/services/reputation/local/controller/util.go +++ b/pkg/services/reputation/local/controller/util.go @@ -7,11 +7,11 @@ type storageWrapper struct { i Iterator } -func (s storageWrapper) InitIterator(common.Context) (Iterator, error) { +func (s storageWrapper) InitIterator(common.EpochProvider) (Iterator, error) { return s.i, nil } -func (s storageWrapper) InitWriter(common.Context) (common.Writer, error) { +func (s storageWrapper) InitWriter(common.EpochProvider) (common.Writer, error) { return s.w, nil } From c236b54a65fb8080951e65dfbb3ade45c16278c9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Apr 2023 11:46:56 +0300 Subject: [PATCH 0068/1943] [#212] reputationsvc: Resolve funlen linter Resolve funlen linter for Calculator.iterateDaughter method. Signed-off-by: Dmitrii Stepanov --- .../reputation/eigentrust/calculator/calls.go | 113 ++++++++++-------- 1 file changed, 60 insertions(+), 53 deletions(-) diff --git a/pkg/services/reputation/eigentrust/calculator/calls.go b/pkg/services/reputation/eigentrust/calculator/calls.go index 2cc789780..a8e5cf1da 100644 --- a/pkg/services/reputation/eigentrust/calculator/calls.go +++ b/pkg/services/reputation/eigentrust/calculator/calls.go @@ -101,7 +101,6 @@ type iterDaughterPrm struct { consumersIter TrustIterator } -// nolint: funlen func (c *Calculator) iterateDaughter(ctx context.Context, p iterDaughterPrm) { initTrust, err := c.prm.InitialTrustSource.InitialTrust(p.id) if err != nil { @@ -157,69 +156,77 @@ func (c *Calculator) iterateDaughter(ctx context.Context, p iterDaughterPrm) { intermediateTrust.SetI(p.ei.I()) if p.lastIter { - finalWriter, err := c.prm.FinalResultTarget.InitIntermediateWriter(p.ei) - if err != nil { - c.opts.log.Debug("init writer failure", - zap.String("error", err.Error()), - ) - - return - } - - intermediateTrust.SetValue(sum) - - err = finalWriter.WriteIntermediateTrust(intermediateTrust) - if err != nil { - c.opts.log.Debug("write final result failure", - zap.String("error", err.Error()), - ) - - return - } + c.processLastIteration(p, intermediateTrust, sum) } else { - intermediateWriter, err := c.prm.IntermediateValueTarget.InitWriter(p.ei) - if err != nil { - c.opts.log.Debug("init writer failure", - zap.String("error", err.Error()), - ) + c.processIntermediateIteration(ctx, p, daughterIter, sum) + } +} - return +func (c *Calculator) processLastIteration(p iterDaughterPrm, intermediateTrust eigentrust.IterationTrust, sum reputation.TrustValue) { + finalWriter, err := c.prm.FinalResultTarget.InitIntermediateWriter(p.ei) + if err != nil { + c.opts.log.Debug("init writer failure", + zap.String("error", err.Error()), + ) + + return + } + + intermediateTrust.SetValue(sum) + + err = finalWriter.WriteIntermediateTrust(intermediateTrust) + if err != nil { + c.opts.log.Debug("write final result failure", + zap.String("error", err.Error()), + ) + + return + } +} + +func (c *Calculator) processIntermediateIteration(ctx context.Context, p iterDaughterPrm, daughterIter TrustIterator, sum reputation.TrustValue) { + intermediateWriter, err := c.prm.IntermediateValueTarget.InitWriter(p.ei) + if err != nil { + c.opts.log.Debug("init writer failure", + zap.String("error", err.Error()), + ) + + return + } + + err = daughterIter.Iterate(func(trust reputation.Trust) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: } - err = daughterIter.Iterate(func(trust reputation.Trust) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } + val := trust.Value() + val.Mul(sum) - val := trust.Value() - val.Mul(sum) + trust.SetValue(val) - trust.SetValue(val) - - err := intermediateWriter.Write(ctx, trust) - if err != nil { - c.opts.log.Debug("write value failure", - zap.String("error", err.Error()), - ) - } - - return nil - }) + err := intermediateWriter.Write(ctx, trust) if err != nil { - c.opts.log.Debug("iterate daughter trusts failure", + c.opts.log.Debug("write value failure", zap.String("error", err.Error()), ) } - err = intermediateWriter.Close(ctx) - if err != nil { - c.opts.log.Error( - "could not close writer", - zap.String("error", err.Error()), - ) - } + return nil + }) + if err != nil { + c.opts.log.Debug("iterate daughter trusts failure", + zap.String("error", err.Error()), + ) + } + + err = intermediateWriter.Close(ctx) + if err != nil { + c.opts.log.Error( + "could not close writer", + zap.String("error", err.Error()), + ) } } From cb172e73a690735d192cdff465e5728ff86232ce Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 7 Apr 2023 17:12:23 +0300 Subject: [PATCH 0069/1943] [#228] node: Use uber atomic package instead standard Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/local_object_storage/shard/control_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index fec268350..df7e536cb 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -6,7 +6,6 @@ import ( "math" "os" "path/filepath" - "sync/atomic" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -28,6 +27,7 @@ import ( objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" + "go.uber.org/atomic" "go.uber.org/zap/zaptest" ) From dbc3811ff4653c9262b0b9c31b19ab4655eef046 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 30 Mar 2023 14:49:15 +0300 Subject: [PATCH 0070/1943] [#191] engine: Allow to remove redundant object copies RemoveDuplicates() removes all duplicate object copies stored on multiple shards. All shards are processed and the command tries to leave a copy on the best shard according to HRW. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/engine.go | 2 + .../engine/remove_copies.go | 138 ++++++++++++ .../engine/remove_copies_test.go | 208 ++++++++++++++++++ pkg/local_object_storage/engine/shards.go | 15 +- 4 files changed, 358 insertions(+), 5 deletions(-) create mode 100644 pkg/local_object_storage/engine/remove_copies.go create mode 100644 pkg/local_object_storage/engine/remove_copies_test.go diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 4d154d289..e0161bfe3 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -17,6 +17,8 @@ import ( type StorageEngine struct { *cfg + removeDuplicatesInProgress atomic.Bool + mtx *sync.RWMutex shards map[string]hashedShard diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go new file mode 100644 index 000000000..d881a52d1 --- /dev/null +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -0,0 +1,138 @@ +package engine + +import ( + "context" + "errors" + + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/hrw" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +// errRemoveDuplicatesInProgress is returned when another rebalancing is in progress. +// We need it because `Rebalance` removes objects and executing it concurrently +// on 2 shards can lead to data loss. In future this restriction could be relaxed. +var errRemoveDuplicatesInProgress = errors.New("redundant copies removal is already in progress") + +const defaultRemoveDuplicatesConcurrency = 256 + +type RemoveDuplicatesPrm struct { + Concurrency int +} + +// RemoveDuplicates iterates over all objects and removes duplicate object copies +// from shards which are worse as defined by HRW sort. +// Safety: +// 1. Concurrent execution is prohibited, thus 1 object copy should always be left. +// 2. If we delete an object from another thread, this is not a problem. Currently, +// we have 2 thread that can remove "valid" (non-expired and logically non-removed) objects: +// policer and rebalance. For rebalance see (1). +// If policer removes something, we do not care if both copies are removed or one of them is left, +// as the remaining copy will be removed during the next policer iteration. +func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicatesPrm) error { + if !e.removeDuplicatesInProgress.CompareAndSwap(false, true) { + return errRemoveDuplicatesInProgress + } + defer e.removeDuplicatesInProgress.Store(false) + + if prm.Concurrency <= 0 { + prm.Concurrency = defaultRemoveDuplicatesConcurrency + } + + e.log.Info("starting removal of locally-redundant copies", + zap.Int("concurrency", prm.Concurrency)) + + // The mutext must be taken for the whole duration to avoid target shard being removed + // concurrently: this can lead to data loss. + e.mtx.RLock() + defer e.mtx.RUnlock() + + // Iterate by shards to be sure that no objects from 2 different shards are removed simultaneously. + // This is not currently the case, because `FreeSpace` metric used by weight sorting is always 0. + // However we could change weights in future and easily forget this function. + for _, sh := range e.shards { + e.log.Debug("started duplicates removal routine", zap.String("shard_id", sh.ID().String())) + ch := make(chan oid.Address) + + errG, ctx := errgroup.WithContext(ctx) + errG.SetLimit(prm.Concurrency + 1) // +1 for the listing thread + + errG.Go(func() error { + defer close(ch) + + var cursor *meta.Cursor + for { + var listPrm shard.ListWithCursorPrm + listPrm.WithCount(uint32(prm.Concurrency)) + listPrm.WithCursor(cursor) + res, err := sh.ListWithCursor(listPrm) + if err != nil { + if errors.Is(err, meta.ErrEndOfListing) { + return nil + } + return err + } + for _, addr := range res.AddressList() { + select { + case <-ctx.Done(): + return ctx.Err() + case ch <- addr.Address: + } + } + cursor = res.Cursor() + } + }) + + for i := 0; i < prm.Concurrency; i++ { + errG.Go(func() error { + return e.removeObjects(ctx, ch) + }) + } + if err := errG.Wait(); err != nil { + e.log.Error("finished removal of locally-redundant copies", zap.Error(err)) + return err + } + } + + e.log.Info("finished removal of locally-redundant copies") + return nil +} + +// removeObjects reads addresses from ch and removes all objects from other shards, excluding excludeID. +func (e *StorageEngine) removeObjects(ctx context.Context, ch <-chan oid.Address) error { + shards := make([]hashedShard, 0, len(e.shards)) + for _, sh := range e.shards { + shards = append(shards, sh) + } + + for addr := range ch { + h := hrw.Hash([]byte(addr.EncodeToString())) + shards := sortShardsByWeight(shards, h) + found := false + for i := range shards { + var existsPrm shard.ExistsPrm + existsPrm.SetAddress(addr) + + res, err := shards[i].Exists(existsPrm) + if err != nil { + return err + } else if !res.Exists() { + continue + } else if !found { + found = true + continue + } + + var deletePrm shard.DeletePrm + deletePrm.SetAddresses(addr) + _, err = shards[i].Delete(deletePrm) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/local_object_storage/engine/remove_copies_test.go b/pkg/local_object_storage/engine/remove_copies_test.go new file mode 100644 index 000000000..4415d01c8 --- /dev/null +++ b/pkg/local_object_storage/engine/remove_copies_test.go @@ -0,0 +1,208 @@ +package engine + +import ( + "context" + "sync" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/stretchr/testify/require" +) + +func TestRebalance(t *testing.T) { + te := newEngineWithErrorThreshold(t, "", 0) + + const ( + objCount = 20 + copyCount = (objCount + 2) / 3 + ) + + type objectWithShard struct { + bestShard shard.ID + worstShard shard.ID + object *objectSDK.Object + } + + objects := make([]objectWithShard, objCount) + for i := range objects { + obj := testutil.GenerateObjectWithCID(cidtest.ID()) + obj.SetPayload(make([]byte, errSmallSize)) + objects[i].object = obj + + shards := te.ng.sortShardsByWeight(object.AddressOf(obj)) + objects[i].bestShard = *shards[0].Shard.ID() + objects[i].worstShard = *shards[1].Shard.ID() + } + + for i := range objects { + var prm shard.PutPrm + prm.SetObject(objects[i].object) + + var err1, err2 error + te.ng.mtx.RLock() + // Every 3rd object (i%3 == 0) is put to both shards, others are distributed. + if i%3 != 1 { + _, err1 = te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) + } + if i%3 != 2 { + _, err2 = te.ng.shards[te.shards[1].id.String()].Shard.Put(prm) + } + te.ng.mtx.RUnlock() + + require.NoError(t, err1) + require.NoError(t, err2) + } + + var removedMtx sync.Mutex + var removed []deleteEvent + for _, shard := range te.shards { + id := *shard.id + shard.largeFileStorage.SetOption(teststore.WithDelete(func(prm common.DeletePrm) (common.DeleteRes, error) { + removedMtx.Lock() + removed = append(removed, deleteEvent{shardID: id, addr: prm.Address}) + removedMtx.Unlock() + return common.DeleteRes{}, nil + })) + } + + err := te.ng.RemoveDuplicates(context.Background(), RemoveDuplicatesPrm{}) + require.NoError(t, err) + + require.Equal(t, copyCount, len(removed)) + + removedMask := make([]bool, len(objects)) +loop: + for i := range removed { + for j := range objects { + if removed[i].addr == object.AddressOf(objects[j].object) { + require.Equal(t, objects[j].worstShard, removed[i].shardID, + "object %d was expected to be removed from another shard", j) + removedMask[j] = true + continue loop + } + } + require.FailNow(t, "unexpected object was removed", removed[i].addr) + } + + for i := 0; i < copyCount; i++ { + if i%3 == 0 { + require.True(t, removedMask[i], "object %d was expected to be removed", i) + } else { + require.False(t, removedMask[i], "object %d was not expected to be removed", i) + } + } +} + +func TestRebalanceSingleThread(t *testing.T) { + te := newEngineWithErrorThreshold(t, "", 0) + + obj := testutil.GenerateObjectWithCID(cidtest.ID()) + obj.SetPayload(make([]byte, errSmallSize)) + + var prm shard.PutPrm + prm.SetObject(obj) + te.ng.mtx.RLock() + _, err1 := te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) + _, err2 := te.ng.shards[te.shards[1].id.String()].Shard.Put(prm) + te.ng.mtx.RUnlock() + require.NoError(t, err1) + require.NoError(t, err2) + + signal := make(chan struct{}) // unblock rebalance + started := make(chan struct{}) // make sure rebalance is started + for _, shard := range te.shards { + shard.largeFileStorage.SetOption(teststore.WithDelete(func(common.DeletePrm) (common.DeleteRes, error) { + close(started) + <-signal + return common.DeleteRes{}, nil + })) + } + + var firstErr error + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + firstErr = te.ng.RemoveDuplicates(context.Background(), RemoveDuplicatesPrm{}) + }() + + <-started + secondErr := te.ng.RemoveDuplicates(context.Background(), RemoveDuplicatesPrm{}) + require.ErrorIs(t, secondErr, errRemoveDuplicatesInProgress) + + close(signal) + wg.Wait() + require.NoError(t, firstErr) +} + +type deleteEvent struct { + shardID shard.ID + addr oid.Address +} + +func TestRebalanceExitByContext(t *testing.T) { + te := newEngineWithErrorThreshold(t, "", 0) + + objects := make([]*objectSDK.Object, 4) + for i := range objects { + obj := testutil.GenerateObjectWithCID(cidtest.ID()) + obj.SetPayload(make([]byte, errSmallSize)) + objects[i] = obj + } + + for i := range objects { + var prm shard.PutPrm + prm.SetObject(objects[i]) + + te.ng.mtx.RLock() + _, err1 := te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) + _, err2 := te.ng.shards[te.shards[1].id.String()].Shard.Put(prm) + te.ng.mtx.RUnlock() + + require.NoError(t, err1) + require.NoError(t, err2) + } + + var removed []deleteEvent + deleteCh := make(chan struct{}) + signal := make(chan struct{}) + for _, shard := range te.shards { + id := *shard.id + shard.largeFileStorage.SetOption(teststore.WithDelete(func(prm common.DeletePrm) (common.DeleteRes, error) { + deleteCh <- struct{}{} + <-signal + removed = append(removed, deleteEvent{shardID: id, addr: prm.Address}) + return common.DeleteRes{}, nil + })) + } + + ctx, cancel := context.WithCancel(context.Background()) + + var rebalanceErr error + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + rebalanceErr = te.ng.RemoveDuplicates(ctx, RemoveDuplicatesPrm{Concurrency: 1}) + }() + + const removeCount = 3 + for i := 0; i < removeCount-1; i++ { + <-deleteCh + signal <- struct{}{} + } + <-deleteCh + cancel() + close(signal) + + wg.Wait() + require.ErrorIs(t, rebalanceErr, context.Canceled) + require.Equal(t, removeCount, len(removed)) +} diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 34210d835..2b1146ff2 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -208,16 +208,21 @@ func (e *StorageEngine) sortShardsByWeight(objAddr interface{ EncodeToString() s e.mtx.RLock() defer e.mtx.RUnlock() + h := hrw.Hash([]byte(objAddr.EncodeToString())) shards := make([]hashedShard, 0, len(e.shards)) - weights := make([]float64, 0, len(e.shards)) - for _, sh := range e.shards { shards = append(shards, hashedShard(sh)) - weights = append(weights, e.shardWeight(sh.Shard)) + } + return sortShardsByWeight(shards, h) +} + +func sortShardsByWeight(shards []hashedShard, h uint64) []hashedShard { + weights := make([]float64, 0, len(shards)) + for _, sh := range shards { + weights = append(weights, float64(sh.Shard.WeightValues().FreeSpace)) } - hrw.SortHasherSliceByWeightValue(shards, weights, hrw.Hash([]byte(objAddr.EncodeToString()))) - + hrw.SortHasherSliceByWeightValue(shards, weights, h) return shards } From 0b9622c418d89d76e9b82ac1fa4969d2b2087a79 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 7 Apr 2023 14:21:05 +0300 Subject: [PATCH 0071/1943] [#191] control: Add Doctor RPC Doctor RPC performs complex operations on the storage engine. Currently only duplicate removal is supported. Signed-off-by: Evgenii Stratonikov --- pkg/services/control/convert.go | 18 + pkg/services/control/rpc.go | 14 + pkg/services/control/server/doctor.go | 37 ++ pkg/services/control/service.pb.go | 711 +++++++++++++++------ pkg/services/control/service.proto | 28 + pkg/services/control/service_frostfs.pb.go | 153 +++++ pkg/services/control/service_grpc.pb.go | 94 ++- 7 files changed, 824 insertions(+), 231 deletions(-) create mode 100644 pkg/services/control/server/doctor.go diff --git a/pkg/services/control/convert.go b/pkg/services/control/convert.go index 833288bb7..f7582dd68 100644 --- a/pkg/services/control/convert.go +++ b/pkg/services/control/convert.go @@ -200,3 +200,21 @@ func (w *flushCacheResponseWrapper) FromGRPCMessage(m grpc.Message) error { w.FlushCacheResponse = r return nil } + +type doctorResponseWrapper struct { + *DoctorResponse +} + +func (w *doctorResponseWrapper) ToGRPCMessage() grpc.Message { + return w.DoctorResponse +} + +func (w *doctorResponseWrapper) FromGRPCMessage(m grpc.Message) error { + r, ok := m.(*DoctorResponse) + if !ok { + return message.NewUnexpectedMessageType(m, (*DoctorResponse)(nil)) + } + + w.DoctorResponse = r + return nil +} diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 0779e177b..2676ea7a5 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -18,6 +18,7 @@ const ( rpcSynchronizeTree = "SynchronizeTree" rpcEvacuateShard = "EvacuateShard" rpcFlushCache = "FlushCache" + rpcDoctor = "Doctor" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -191,3 +192,16 @@ func FlushCache(cli *client.Client, req *FlushCacheRequest, opts ...client.CallO return wResp.FlushCacheResponse, nil } + +// Doctor executes ControlService.Doctor RPC. +func Doctor(cli *client.Client, req *DoctorRequest, opts ...client.CallOption) (*DoctorResponse, error) { + wResp := &doctorResponseWrapper{new(DoctorResponse)} + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcDoctor), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.DoctorResponse, nil +} diff --git a/pkg/services/control/server/doctor.go b/pkg/services/control/server/doctor.go new file mode 100644 index 000000000..2c91d4c2b --- /dev/null +++ b/pkg/services/control/server/doctor.go @@ -0,0 +1,37 @@ +package control + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *Server) Doctor(ctx context.Context, req *control.DoctorRequest) (*control.DoctorResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + if !req.Body.RemoveDuplicates { + return nil, status.Error(codes.InvalidArgument, "operation not specified") + } + + var prm engine.RemoveDuplicatesPrm + prm.Concurrency = int(req.Body.Concurrency) + + err = s.s.RemoveDuplicates(ctx, prm) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.DoctorResponse{Body: &control.DoctorResponse_Body{}} + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil +} diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 78e7b0598..ca3e2770e 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -1172,6 +1172,118 @@ func (x *FlushCacheResponse) GetSignature() *Signature { return nil } +// Doctor request. +type DoctorRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *DoctorRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *DoctorRequest) Reset() { + *x = DoctorRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DoctorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DoctorRequest) ProtoMessage() {} + +func (x *DoctorRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[20] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DoctorRequest.ProtoReflect.Descriptor instead. +func (*DoctorRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20} +} + +func (x *DoctorRequest) GetBody() *DoctorRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *DoctorRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// Doctor response. +type DoctorResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *DoctorResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *DoctorResponse) Reset() { + *x = DoctorResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DoctorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DoctorResponse) ProtoMessage() {} + +func (x *DoctorResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[21] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DoctorResponse.ProtoReflect.Descriptor instead. +func (*DoctorResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21} +} + +func (x *DoctorResponse) GetBody() *DoctorResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *DoctorResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + // Health check request body. type HealthCheckRequest_Body struct { state protoimpl.MessageState @@ -1182,7 +1294,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[20] + mi := &file_pkg_services_control_service_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1195,7 +1307,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[20] + mi := &file_pkg_services_control_service_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1226,7 +1338,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[21] + mi := &file_pkg_services_control_service_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1239,7 +1351,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[21] + mi := &file_pkg_services_control_service_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1290,7 +1402,7 @@ type SetNetmapStatusRequest_Body struct { func (x *SetNetmapStatusRequest_Body) Reset() { *x = SetNetmapStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[22] + mi := &file_pkg_services_control_service_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1303,7 +1415,7 @@ func (x *SetNetmapStatusRequest_Body) String() string { func (*SetNetmapStatusRequest_Body) ProtoMessage() {} func (x *SetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[22] + mi := &file_pkg_services_control_service_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1343,7 +1455,7 @@ type SetNetmapStatusResponse_Body struct { func (x *SetNetmapStatusResponse_Body) Reset() { *x = SetNetmapStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[23] + mi := &file_pkg_services_control_service_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1356,7 +1468,7 @@ func (x *SetNetmapStatusResponse_Body) String() string { func (*SetNetmapStatusResponse_Body) ProtoMessage() {} func (x *SetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[23] + mi := &file_pkg_services_control_service_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1386,7 +1498,7 @@ type DropObjectsRequest_Body struct { func (x *DropObjectsRequest_Body) Reset() { *x = DropObjectsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1399,7 +1511,7 @@ func (x *DropObjectsRequest_Body) String() string { func (*DropObjectsRequest_Body) ProtoMessage() {} func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1432,7 +1544,7 @@ type DropObjectsResponse_Body struct { func (x *DropObjectsResponse_Body) Reset() { *x = DropObjectsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1445,7 +1557,7 @@ func (x *DropObjectsResponse_Body) String() string { func (*DropObjectsResponse_Body) ProtoMessage() {} func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1471,7 +1583,7 @@ type ListShardsRequest_Body struct { func (x *ListShardsRequest_Body) Reset() { *x = ListShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1484,7 +1596,7 @@ func (x *ListShardsRequest_Body) String() string { func (*ListShardsRequest_Body) ProtoMessage() {} func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1513,7 +1625,7 @@ type ListShardsResponse_Body struct { func (x *ListShardsResponse_Body) Reset() { *x = ListShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1526,7 +1638,7 @@ func (x *ListShardsResponse_Body) String() string { func (*ListShardsResponse_Body) ProtoMessage() {} func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1566,7 +1678,7 @@ type SetShardModeRequest_Body struct { func (x *SetShardModeRequest_Body) Reset() { *x = SetShardModeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1579,7 +1691,7 @@ func (x *SetShardModeRequest_Body) String() string { func (*SetShardModeRequest_Body) ProtoMessage() {} func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1626,7 +1738,7 @@ type SetShardModeResponse_Body struct { func (x *SetShardModeResponse_Body) Reset() { *x = SetShardModeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1639,7 +1751,7 @@ func (x *SetShardModeResponse_Body) String() string { func (*SetShardModeResponse_Body) ProtoMessage() {} func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1672,7 +1784,7 @@ type DumpShardRequest_Body struct { func (x *DumpShardRequest_Body) Reset() { *x = DumpShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1685,7 +1797,7 @@ func (x *DumpShardRequest_Body) String() string { func (*DumpShardRequest_Body) ProtoMessage() {} func (x *DumpShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1732,7 +1844,7 @@ type DumpShardResponse_Body struct { func (x *DumpShardResponse_Body) Reset() { *x = DumpShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1745,7 +1857,7 @@ func (x *DumpShardResponse_Body) String() string { func (*DumpShardResponse_Body) ProtoMessage() {} func (x *DumpShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1778,7 +1890,7 @@ type RestoreShardRequest_Body struct { func (x *RestoreShardRequest_Body) Reset() { *x = RestoreShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1791,7 +1903,7 @@ func (x *RestoreShardRequest_Body) String() string { func (*RestoreShardRequest_Body) ProtoMessage() {} func (x *RestoreShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1838,7 +1950,7 @@ type RestoreShardResponse_Body struct { func (x *RestoreShardResponse_Body) Reset() { *x = RestoreShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1851,7 +1963,7 @@ func (x *RestoreShardResponse_Body) String() string { func (*RestoreShardResponse_Body) ProtoMessage() {} func (x *RestoreShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1882,7 +1994,7 @@ type SynchronizeTreeRequest_Body struct { func (x *SynchronizeTreeRequest_Body) Reset() { *x = SynchronizeTreeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1895,7 +2007,7 @@ func (x *SynchronizeTreeRequest_Body) String() string { func (*SynchronizeTreeRequest_Body) ProtoMessage() {} func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1942,7 +2054,7 @@ type SynchronizeTreeResponse_Body struct { func (x *SynchronizeTreeResponse_Body) Reset() { *x = SynchronizeTreeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1955,7 +2067,7 @@ func (x *SynchronizeTreeResponse_Body) String() string { func (*SynchronizeTreeResponse_Body) ProtoMessage() {} func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1986,7 +2098,7 @@ type EvacuateShardRequest_Body struct { func (x *EvacuateShardRequest_Body) Reset() { *x = EvacuateShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1999,7 +2111,7 @@ func (x *EvacuateShardRequest_Body) String() string { func (*EvacuateShardRequest_Body) ProtoMessage() {} func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2041,7 +2153,7 @@ type EvacuateShardResponse_Body struct { func (x *EvacuateShardResponse_Body) Reset() { *x = EvacuateShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2054,7 +2166,7 @@ func (x *EvacuateShardResponse_Body) String() string { func (*EvacuateShardResponse_Body) ProtoMessage() {} func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2090,7 +2202,7 @@ type FlushCacheRequest_Body struct { func (x *FlushCacheRequest_Body) Reset() { *x = FlushCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2103,7 +2215,7 @@ func (x *FlushCacheRequest_Body) String() string { func (*FlushCacheRequest_Body) ProtoMessage() {} func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2136,7 +2248,7 @@ type FlushCacheResponse_Body struct { func (x *FlushCacheResponse_Body) Reset() { *x = FlushCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2149,7 +2261,7 @@ func (x *FlushCacheResponse_Body) String() string { func (*FlushCacheResponse_Body) ProtoMessage() {} func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2165,6 +2277,103 @@ func (*FlushCacheResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{19, 0} } +// Request body structure. +type DoctorRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Number of threads to use for the operation. + Concurrency uint32 `protobuf:"varint,1,opt,name=concurrency,proto3" json:"concurrency,omitempty"` + // Flag to search engine for duplicate objects and leave only one copy. + RemoveDuplicates bool `protobuf:"varint,2,opt,name=remove_duplicates,json=removeDuplicates,proto3" json:"remove_duplicates,omitempty"` +} + +func (x *DoctorRequest_Body) Reset() { + *x = DoctorRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[42] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DoctorRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DoctorRequest_Body) ProtoMessage() {} + +func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[42] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DoctorRequest_Body.ProtoReflect.Descriptor instead. +func (*DoctorRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20, 0} +} + +func (x *DoctorRequest_Body) GetConcurrency() uint32 { + if x != nil { + return x.Concurrency + } + return 0 +} + +func (x *DoctorRequest_Body) GetRemoveDuplicates() bool { + if x != nil { + return x.RemoveDuplicates + } + return false +} + +// Response body structure. +type DoctorResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *DoctorResponse_Body) Reset() { + *x = DoctorResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[43] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DoctorResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DoctorResponse_Body) ProtoMessage() {} + +func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[43] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DoctorResponse_Body.ProtoReflect.Descriptor instead. +func (*DoctorResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0} +} + var File_pkg_services_control_service_proto protoreflect.FileDescriptor var file_pkg_services_control_service_proto_rawDesc = []byte{ @@ -2394,60 +2603,84 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x32, 0x8c, 0x06, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, - 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, - 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, - 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, - 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, - 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x42, 0x0a, 0x09, 0x44, 0x75, 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x19, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x75, 0x6d, 0x70, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x75, 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, - 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, - 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, - 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, - 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, - 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, - 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x22, 0xc9, 0x01, 0x0a, 0x0d, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x55, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x20, 0x0a, + 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, + 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x64, 0x75, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x44, 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x22, 0x7c, 0x0a, 0x0e, + 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xc7, 0x06, 0x0a, 0x0e, 0x43, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, + 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, + 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, + 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, + 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, + 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x09, 0x44, + 0x75, 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x19, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x44, 0x75, 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x75, + 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x4b, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, + 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, + 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, + 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, + 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, + 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, + 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, + 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, + 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, + 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, + 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2462,7 +2695,7 @@ func file_pkg_services_control_service_proto_rawDescGZIP() []byte { return file_pkg_services_control_service_proto_rawDescData } -var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 40) +var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 44) var file_pkg_services_control_service_proto_goTypes = []interface{}{ (*HealthCheckRequest)(nil), // 0: control.HealthCheckRequest (*HealthCheckResponse)(nil), // 1: control.HealthCheckResponse @@ -2484,103 +2717,113 @@ var file_pkg_services_control_service_proto_goTypes = []interface{}{ (*EvacuateShardResponse)(nil), // 17: control.EvacuateShardResponse (*FlushCacheRequest)(nil), // 18: control.FlushCacheRequest (*FlushCacheResponse)(nil), // 19: control.FlushCacheResponse - (*HealthCheckRequest_Body)(nil), // 20: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 21: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 22: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 23: control.SetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 24: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 25: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 26: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 27: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 28: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 29: control.SetShardModeResponse.Body - (*DumpShardRequest_Body)(nil), // 30: control.DumpShardRequest.Body - (*DumpShardResponse_Body)(nil), // 31: control.DumpShardResponse.Body - (*RestoreShardRequest_Body)(nil), // 32: control.RestoreShardRequest.Body - (*RestoreShardResponse_Body)(nil), // 33: control.RestoreShardResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 34: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 35: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 36: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 37: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 38: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 39: control.FlushCacheResponse.Body - (*Signature)(nil), // 40: control.Signature - (NetmapStatus)(0), // 41: control.NetmapStatus - (HealthStatus)(0), // 42: control.HealthStatus - (*ShardInfo)(nil), // 43: control.ShardInfo - (ShardMode)(0), // 44: control.ShardMode + (*DoctorRequest)(nil), // 20: control.DoctorRequest + (*DoctorResponse)(nil), // 21: control.DoctorResponse + (*HealthCheckRequest_Body)(nil), // 22: control.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 23: control.HealthCheckResponse.Body + (*SetNetmapStatusRequest_Body)(nil), // 24: control.SetNetmapStatusRequest.Body + (*SetNetmapStatusResponse_Body)(nil), // 25: control.SetNetmapStatusResponse.Body + (*DropObjectsRequest_Body)(nil), // 26: control.DropObjectsRequest.Body + (*DropObjectsResponse_Body)(nil), // 27: control.DropObjectsResponse.Body + (*ListShardsRequest_Body)(nil), // 28: control.ListShardsRequest.Body + (*ListShardsResponse_Body)(nil), // 29: control.ListShardsResponse.Body + (*SetShardModeRequest_Body)(nil), // 30: control.SetShardModeRequest.Body + (*SetShardModeResponse_Body)(nil), // 31: control.SetShardModeResponse.Body + (*DumpShardRequest_Body)(nil), // 32: control.DumpShardRequest.Body + (*DumpShardResponse_Body)(nil), // 33: control.DumpShardResponse.Body + (*RestoreShardRequest_Body)(nil), // 34: control.RestoreShardRequest.Body + (*RestoreShardResponse_Body)(nil), // 35: control.RestoreShardResponse.Body + (*SynchronizeTreeRequest_Body)(nil), // 36: control.SynchronizeTreeRequest.Body + (*SynchronizeTreeResponse_Body)(nil), // 37: control.SynchronizeTreeResponse.Body + (*EvacuateShardRequest_Body)(nil), // 38: control.EvacuateShardRequest.Body + (*EvacuateShardResponse_Body)(nil), // 39: control.EvacuateShardResponse.Body + (*FlushCacheRequest_Body)(nil), // 40: control.FlushCacheRequest.Body + (*FlushCacheResponse_Body)(nil), // 41: control.FlushCacheResponse.Body + (*DoctorRequest_Body)(nil), // 42: control.DoctorRequest.Body + (*DoctorResponse_Body)(nil), // 43: control.DoctorResponse.Body + (*Signature)(nil), // 44: control.Signature + (NetmapStatus)(0), // 45: control.NetmapStatus + (HealthStatus)(0), // 46: control.HealthStatus + (*ShardInfo)(nil), // 47: control.ShardInfo + (ShardMode)(0), // 48: control.ShardMode } var file_pkg_services_control_service_proto_depIdxs = []int32{ - 20, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 40, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 21, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 40, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 22, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 40, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 23, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 40, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 24, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 40, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature - 25, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 40, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature - 26, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 40, // 13: control.ListShardsRequest.signature:type_name -> control.Signature - 27, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 40, // 15: control.ListShardsResponse.signature:type_name -> control.Signature - 28, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 40, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature - 29, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 40, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature - 30, // 20: control.DumpShardRequest.body:type_name -> control.DumpShardRequest.Body - 40, // 21: control.DumpShardRequest.signature:type_name -> control.Signature - 31, // 22: control.DumpShardResponse.body:type_name -> control.DumpShardResponse.Body - 40, // 23: control.DumpShardResponse.signature:type_name -> control.Signature - 32, // 24: control.RestoreShardRequest.body:type_name -> control.RestoreShardRequest.Body - 40, // 25: control.RestoreShardRequest.signature:type_name -> control.Signature - 33, // 26: control.RestoreShardResponse.body:type_name -> control.RestoreShardResponse.Body - 40, // 27: control.RestoreShardResponse.signature:type_name -> control.Signature - 34, // 28: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 40, // 29: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 35, // 30: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 40, // 31: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 36, // 32: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 40, // 33: control.EvacuateShardRequest.signature:type_name -> control.Signature - 37, // 34: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 40, // 35: control.EvacuateShardResponse.signature:type_name -> control.Signature - 38, // 36: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 40, // 37: control.FlushCacheRequest.signature:type_name -> control.Signature - 39, // 38: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 40, // 39: control.FlushCacheResponse.signature:type_name -> control.Signature - 41, // 40: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 42, // 41: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 41, // 42: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 43, // 43: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 44, // 44: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 0, // 45: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 2, // 46: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 4, // 47: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 6, // 48: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 8, // 49: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 10, // 50: control.ControlService.DumpShard:input_type -> control.DumpShardRequest - 12, // 51: control.ControlService.RestoreShard:input_type -> control.RestoreShardRequest - 14, // 52: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 16, // 53: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 18, // 54: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 1, // 55: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 3, // 56: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 5, // 57: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 7, // 58: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 9, // 59: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 11, // 60: control.ControlService.DumpShard:output_type -> control.DumpShardResponse - 13, // 61: control.ControlService.RestoreShard:output_type -> control.RestoreShardResponse - 15, // 62: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 17, // 63: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 19, // 64: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 55, // [55:65] is the sub-list for method output_type - 45, // [45:55] is the sub-list for method input_type - 45, // [45:45] is the sub-list for extension type_name - 45, // [45:45] is the sub-list for extension extendee - 0, // [0:45] is the sub-list for field type_name + 22, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body + 44, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature + 23, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body + 44, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature + 24, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body + 44, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature + 25, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body + 44, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature + 26, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body + 44, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature + 27, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body + 44, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature + 28, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body + 44, // 13: control.ListShardsRequest.signature:type_name -> control.Signature + 29, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body + 44, // 15: control.ListShardsResponse.signature:type_name -> control.Signature + 30, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body + 44, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature + 31, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body + 44, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature + 32, // 20: control.DumpShardRequest.body:type_name -> control.DumpShardRequest.Body + 44, // 21: control.DumpShardRequest.signature:type_name -> control.Signature + 33, // 22: control.DumpShardResponse.body:type_name -> control.DumpShardResponse.Body + 44, // 23: control.DumpShardResponse.signature:type_name -> control.Signature + 34, // 24: control.RestoreShardRequest.body:type_name -> control.RestoreShardRequest.Body + 44, // 25: control.RestoreShardRequest.signature:type_name -> control.Signature + 35, // 26: control.RestoreShardResponse.body:type_name -> control.RestoreShardResponse.Body + 44, // 27: control.RestoreShardResponse.signature:type_name -> control.Signature + 36, // 28: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body + 44, // 29: control.SynchronizeTreeRequest.signature:type_name -> control.Signature + 37, // 30: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body + 44, // 31: control.SynchronizeTreeResponse.signature:type_name -> control.Signature + 38, // 32: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body + 44, // 33: control.EvacuateShardRequest.signature:type_name -> control.Signature + 39, // 34: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body + 44, // 35: control.EvacuateShardResponse.signature:type_name -> control.Signature + 40, // 36: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body + 44, // 37: control.FlushCacheRequest.signature:type_name -> control.Signature + 41, // 38: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body + 44, // 39: control.FlushCacheResponse.signature:type_name -> control.Signature + 42, // 40: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body + 44, // 41: control.DoctorRequest.signature:type_name -> control.Signature + 43, // 42: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body + 44, // 43: control.DoctorResponse.signature:type_name -> control.Signature + 45, // 44: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus + 46, // 45: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus + 45, // 46: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus + 47, // 47: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo + 48, // 48: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode + 0, // 49: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 2, // 50: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 4, // 51: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 6, // 52: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 8, // 53: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 10, // 54: control.ControlService.DumpShard:input_type -> control.DumpShardRequest + 12, // 55: control.ControlService.RestoreShard:input_type -> control.RestoreShardRequest + 14, // 56: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 16, // 57: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 18, // 58: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 20, // 59: control.ControlService.Doctor:input_type -> control.DoctorRequest + 1, // 60: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 3, // 61: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 5, // 62: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 7, // 63: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 9, // 64: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 11, // 65: control.ControlService.DumpShard:output_type -> control.DumpShardResponse + 13, // 66: control.ControlService.RestoreShard:output_type -> control.RestoreShardResponse + 15, // 67: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 17, // 68: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 19, // 69: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 21, // 70: control.ControlService.Doctor:output_type -> control.DoctorResponse + 60, // [60:71] is the sub-list for method output_type + 49, // [49:60] is the sub-list for method input_type + 49, // [49:49] is the sub-list for extension type_name + 49, // [49:49] is the sub-list for extension extendee + 0, // [0:49] is the sub-list for field type_name } func init() { file_pkg_services_control_service_proto_init() } @@ -2831,7 +3074,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*DoctorRequest); i { case 0: return &v.state case 1: @@ -2843,7 +3086,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { + switch v := v.(*DoctorResponse); i { case 0: return &v.state case 1: @@ -2855,7 +3098,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusRequest_Body); i { + switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state case 1: @@ -2867,7 +3110,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusResponse_Body); i { + switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state case 1: @@ -2879,7 +3122,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest_Body); i { + switch v := v.(*SetNetmapStatusRequest_Body); i { case 0: return &v.state case 1: @@ -2891,7 +3134,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse_Body); i { + switch v := v.(*SetNetmapStatusResponse_Body); i { case 0: return &v.state case 1: @@ -2903,7 +3146,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest_Body); i { + switch v := v.(*DropObjectsRequest_Body); i { case 0: return &v.state case 1: @@ -2915,7 +3158,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse_Body); i { + switch v := v.(*DropObjectsResponse_Body); i { case 0: return &v.state case 1: @@ -2927,7 +3170,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest_Body); i { + switch v := v.(*ListShardsRequest_Body); i { case 0: return &v.state case 1: @@ -2939,7 +3182,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse_Body); i { + switch v := v.(*ListShardsResponse_Body); i { case 0: return &v.state case 1: @@ -2951,7 +3194,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DumpShardRequest_Body); i { + switch v := v.(*SetShardModeRequest_Body); i { case 0: return &v.state case 1: @@ -2963,7 +3206,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DumpShardResponse_Body); i { + switch v := v.(*SetShardModeResponse_Body); i { case 0: return &v.state case 1: @@ -2975,7 +3218,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreShardRequest_Body); i { + switch v := v.(*DumpShardRequest_Body); i { case 0: return &v.state case 1: @@ -2987,7 +3230,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreShardResponse_Body); i { + switch v := v.(*DumpShardResponse_Body); i { case 0: return &v.state case 1: @@ -2999,7 +3242,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest_Body); i { + switch v := v.(*RestoreShardRequest_Body); i { case 0: return &v.state case 1: @@ -3011,7 +3254,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse_Body); i { + switch v := v.(*RestoreShardResponse_Body); i { case 0: return &v.state case 1: @@ -3023,7 +3266,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest_Body); i { + switch v := v.(*SynchronizeTreeRequest_Body); i { case 0: return &v.state case 1: @@ -3035,7 +3278,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse_Body); i { + switch v := v.(*SynchronizeTreeResponse_Body); i { case 0: return &v.state case 1: @@ -3047,7 +3290,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest_Body); i { + switch v := v.(*EvacuateShardRequest_Body); i { case 0: return &v.state case 1: @@ -3059,6 +3302,30 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EvacuateShardResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FlushCacheRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*FlushCacheResponse_Body); i { case 0: return &v.state @@ -3070,6 +3337,30 @@ func file_pkg_services_control_service_proto_init() { return nil } } + file_pkg_services_control_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DoctorRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DoctorResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -3077,7 +3368,7 @@ func file_pkg_services_control_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_service_proto_rawDesc, NumEnums: 0, - NumMessages: 40, + NumMessages: 44, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 5a09a74a5..7c661e661 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -37,6 +37,9 @@ service ControlService { // FlushCache moves all data from one shard to the others. rpc FlushCache (FlushCacheRequest) returns (FlushCacheResponse); + + // Doctor performs storage restructuring operations on engine. + rpc Doctor (DoctorRequest) returns (DoctorResponse); } // Health check request. @@ -345,3 +348,28 @@ message FlushCacheResponse { Body body = 1; Signature signature = 2; } + + +// Doctor request. +message DoctorRequest { + // Request body structure. + message Body { + // Number of threads to use for the operation. + uint32 concurrency = 1; + // Flag to search engine for duplicate objects and leave only one copy. + bool remove_duplicates = 2; + } + + Body body = 1; + Signature signature = 2; +} + +// Doctor response. +message DoctorResponse { + // Response body structure. + message Body { + } + + Body body = 1; + Signature signature = 2; +} diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 5bb119090..0f50d5893 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -1548,3 +1548,156 @@ func (x *FlushCacheResponse) ReadSignedData(buf []byte) ([]byte, error) { func (x *FlushCacheResponse) SetSignature(sig *Signature) { x.Signature = sig } + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *DoctorRequest_Body) StableSize() (size int) { + size += proto.UInt32Size(1, x.Concurrency) + size += proto.BoolSize(2, x.RemoveDuplicates) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *DoctorRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.UInt32Marshal(1, buf[offset:], x.Concurrency) + offset += proto.BoolMarshal(2, buf[offset:], x.RemoveDuplicates) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *DoctorRequest) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *DoctorRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *DoctorRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *DoctorRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *DoctorRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *DoctorResponse_Body) StableSize() (size int) { + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *DoctorResponse_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *DoctorResponse) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *DoctorResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *DoctorResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *DoctorResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *DoctorResponse) SetSignature(sig *Signature) { + x.Signature = sig +} diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 2cfddd7f5..4a4fbeac1 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.3.0 // - protoc v3.21.12 // source: pkg/services/control/service.proto @@ -18,6 +18,20 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + ControlService_HealthCheck_FullMethodName = "/control.ControlService/HealthCheck" + ControlService_SetNetmapStatus_FullMethodName = "/control.ControlService/SetNetmapStatus" + ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects" + ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" + ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" + ControlService_DumpShard_FullMethodName = "/control.ControlService/DumpShard" + ControlService_RestoreShard_FullMethodName = "/control.ControlService/RestoreShard" + ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree" + ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard" + ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache" + ControlService_Doctor_FullMethodName = "/control.ControlService/Doctor" +) + // ControlServiceClient is the client API for ControlService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -42,6 +56,8 @@ type ControlServiceClient interface { EvacuateShard(ctx context.Context, in *EvacuateShardRequest, opts ...grpc.CallOption) (*EvacuateShardResponse, error) // FlushCache moves all data from one shard to the others. FlushCache(ctx context.Context, in *FlushCacheRequest, opts ...grpc.CallOption) (*FlushCacheResponse, error) + // Doctor performs storage restructuring operations on engine. + Doctor(ctx context.Context, in *DoctorRequest, opts ...grpc.CallOption) (*DoctorResponse, error) } type controlServiceClient struct { @@ -54,7 +70,7 @@ func NewControlServiceClient(cc grpc.ClientConnInterface) ControlServiceClient { func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { out := new(HealthCheckResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/HealthCheck", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_HealthCheck_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -63,7 +79,7 @@ func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckR func (c *controlServiceClient) SetNetmapStatus(ctx context.Context, in *SetNetmapStatusRequest, opts ...grpc.CallOption) (*SetNetmapStatusResponse, error) { out := new(SetNetmapStatusResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/SetNetmapStatus", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_SetNetmapStatus_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -72,7 +88,7 @@ func (c *controlServiceClient) SetNetmapStatus(ctx context.Context, in *SetNetma func (c *controlServiceClient) DropObjects(ctx context.Context, in *DropObjectsRequest, opts ...grpc.CallOption) (*DropObjectsResponse, error) { out := new(DropObjectsResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/DropObjects", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_DropObjects_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -81,7 +97,7 @@ func (c *controlServiceClient) DropObjects(ctx context.Context, in *DropObjectsR func (c *controlServiceClient) ListShards(ctx context.Context, in *ListShardsRequest, opts ...grpc.CallOption) (*ListShardsResponse, error) { out := new(ListShardsResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/ListShards", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_ListShards_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -90,7 +106,7 @@ func (c *controlServiceClient) ListShards(ctx context.Context, in *ListShardsReq func (c *controlServiceClient) SetShardMode(ctx context.Context, in *SetShardModeRequest, opts ...grpc.CallOption) (*SetShardModeResponse, error) { out := new(SetShardModeResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/SetShardMode", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_SetShardMode_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -99,7 +115,7 @@ func (c *controlServiceClient) SetShardMode(ctx context.Context, in *SetShardMod func (c *controlServiceClient) DumpShard(ctx context.Context, in *DumpShardRequest, opts ...grpc.CallOption) (*DumpShardResponse, error) { out := new(DumpShardResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/DumpShard", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_DumpShard_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -108,7 +124,7 @@ func (c *controlServiceClient) DumpShard(ctx context.Context, in *DumpShardReque func (c *controlServiceClient) RestoreShard(ctx context.Context, in *RestoreShardRequest, opts ...grpc.CallOption) (*RestoreShardResponse, error) { out := new(RestoreShardResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/RestoreShard", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_RestoreShard_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -117,7 +133,7 @@ func (c *controlServiceClient) RestoreShard(ctx context.Context, in *RestoreShar func (c *controlServiceClient) SynchronizeTree(ctx context.Context, in *SynchronizeTreeRequest, opts ...grpc.CallOption) (*SynchronizeTreeResponse, error) { out := new(SynchronizeTreeResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/SynchronizeTree", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_SynchronizeTree_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -126,7 +142,7 @@ func (c *controlServiceClient) SynchronizeTree(ctx context.Context, in *Synchron func (c *controlServiceClient) EvacuateShard(ctx context.Context, in *EvacuateShardRequest, opts ...grpc.CallOption) (*EvacuateShardResponse, error) { out := new(EvacuateShardResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/EvacuateShard", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_EvacuateShard_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -135,7 +151,16 @@ func (c *controlServiceClient) EvacuateShard(ctx context.Context, in *EvacuateSh func (c *controlServiceClient) FlushCache(ctx context.Context, in *FlushCacheRequest, opts ...grpc.CallOption) (*FlushCacheResponse, error) { out := new(FlushCacheResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/FlushCache", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_FlushCache_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlServiceClient) Doctor(ctx context.Context, in *DoctorRequest, opts ...grpc.CallOption) (*DoctorResponse, error) { + out := new(DoctorResponse) + err := c.cc.Invoke(ctx, ControlService_Doctor_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -166,6 +191,8 @@ type ControlServiceServer interface { EvacuateShard(context.Context, *EvacuateShardRequest) (*EvacuateShardResponse, error) // FlushCache moves all data from one shard to the others. FlushCache(context.Context, *FlushCacheRequest) (*FlushCacheResponse, error) + // Doctor performs storage restructuring operations on engine. + Doctor(context.Context, *DoctorRequest) (*DoctorResponse, error) } // UnimplementedControlServiceServer should be embedded to have forward compatible implementations. @@ -202,6 +229,9 @@ func (UnimplementedControlServiceServer) EvacuateShard(context.Context, *Evacuat func (UnimplementedControlServiceServer) FlushCache(context.Context, *FlushCacheRequest) (*FlushCacheResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method FlushCache not implemented") } +func (UnimplementedControlServiceServer) Doctor(context.Context, *DoctorRequest) (*DoctorResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Doctor not implemented") +} // UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ControlServiceServer will @@ -224,7 +254,7 @@ func _ControlService_HealthCheck_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/HealthCheck", + FullMethod: ControlService_HealthCheck_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).HealthCheck(ctx, req.(*HealthCheckRequest)) @@ -242,7 +272,7 @@ func _ControlService_SetNetmapStatus_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/SetNetmapStatus", + FullMethod: ControlService_SetNetmapStatus_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).SetNetmapStatus(ctx, req.(*SetNetmapStatusRequest)) @@ -260,7 +290,7 @@ func _ControlService_DropObjects_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/DropObjects", + FullMethod: ControlService_DropObjects_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).DropObjects(ctx, req.(*DropObjectsRequest)) @@ -278,7 +308,7 @@ func _ControlService_ListShards_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/ListShards", + FullMethod: ControlService_ListShards_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).ListShards(ctx, req.(*ListShardsRequest)) @@ -296,7 +326,7 @@ func _ControlService_SetShardMode_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/SetShardMode", + FullMethod: ControlService_SetShardMode_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).SetShardMode(ctx, req.(*SetShardModeRequest)) @@ -314,7 +344,7 @@ func _ControlService_DumpShard_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/DumpShard", + FullMethod: ControlService_DumpShard_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).DumpShard(ctx, req.(*DumpShardRequest)) @@ -332,7 +362,7 @@ func _ControlService_RestoreShard_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/RestoreShard", + FullMethod: ControlService_RestoreShard_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).RestoreShard(ctx, req.(*RestoreShardRequest)) @@ -350,7 +380,7 @@ func _ControlService_SynchronizeTree_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/SynchronizeTree", + FullMethod: ControlService_SynchronizeTree_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).SynchronizeTree(ctx, req.(*SynchronizeTreeRequest)) @@ -368,7 +398,7 @@ func _ControlService_EvacuateShard_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/EvacuateShard", + FullMethod: ControlService_EvacuateShard_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).EvacuateShard(ctx, req.(*EvacuateShardRequest)) @@ -386,7 +416,7 @@ func _ControlService_FlushCache_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/FlushCache", + FullMethod: ControlService_FlushCache_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).FlushCache(ctx, req.(*FlushCacheRequest)) @@ -394,6 +424,24 @@ func _ControlService_FlushCache_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } +func _ControlService_Doctor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DoctorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).Doctor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_Doctor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).Doctor(ctx, req.(*DoctorRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -441,6 +489,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "FlushCache", Handler: _ControlService_FlushCache_Handler, }, + { + MethodName: "Doctor", + Handler: _ControlService_Doctor_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "pkg/services/control/service.proto", From b689027d573c7ea973786d320853b51173c96522 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 7 Apr 2023 14:36:00 +0300 Subject: [PATCH 0072/1943] [#191] cli: Add `control shards doctor` command Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/doctor.go | 53 +++++++++++++++++++++++ cmd/frostfs-cli/modules/control/shards.go | 2 + 2 files changed, 55 insertions(+) create mode 100644 cmd/frostfs-cli/modules/control/doctor.go diff --git a/cmd/frostfs-cli/modules/control/doctor.go b/cmd/frostfs-cli/modules/control/doctor.go new file mode 100644 index 000000000..13bb81a0a --- /dev/null +++ b/cmd/frostfs-cli/modules/control/doctor.go @@ -0,0 +1,53 @@ +package control + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "github.com/spf13/cobra" +) + +const ( + concurrencyFlag = "concurrency" + removeDuplicatesFlag = "remove-duplicates" +) + +var doctorCmd = &cobra.Command{ + Use: "doctor", + Short: "Restructure node's storage", + Long: "Restructure node's storage", + Run: doctor, +} + +func doctor(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + req := &control.DoctorRequest{Body: new(control.DoctorRequest_Body)} + req.Body.Concurrency, _ = cmd.Flags().GetUint32(concurrencyFlag) + req.Body.RemoveDuplicates, _ = cmd.Flags().GetBool(removeDuplicatesFlag) + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.DoctorResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.Doctor(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Operation has finished.") +} + +func initControlDoctorCmd() { + initControlFlags(doctorCmd) + + ff := doctorCmd.Flags() + ff.Uint32(concurrencyFlag, 0, "Number of parallel threads to use") + ff.Bool(removeDuplicatesFlag, false, "Remove duplicate objects") +} diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index 6719a4acf..9d3eb5c01 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -17,6 +17,7 @@ func initControlShardsCmd() { shardsCmd.AddCommand(restoreShardCmd) shardsCmd.AddCommand(evacuateShardCmd) shardsCmd.AddCommand(flushCacheCmd) + shardsCmd.AddCommand(doctorCmd) initControlShardsListCmd() initControlSetShardModeCmd() @@ -24,4 +25,5 @@ func initControlShardsCmd() { initControlRestoreShardCmd() initControlEvacuateShardCmd() initControlFlushCacheCmd() + initControlDoctorCmd() } From 38ae71cc7d4f2221a7771c98c836598c4f79d265 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Apr 2023 12:28:41 +0300 Subject: [PATCH 0073/1943] [#213] metrics: Refactor engine metrics Resolve funlen linter for newEngineMetrics function. Signed-off-by: Dmitrii Stepanov --- pkg/metrics/engine.go | 147 ++++++++++++------------------------------ 1 file changed, 40 insertions(+), 107 deletions(-) diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index fbc184832..4c51a55aa 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -1,6 +1,8 @@ package metrics import ( + "fmt" + "strings" "time" "github.com/prometheus/client_golang/prometheus" @@ -26,118 +28,49 @@ type ( const engineSubsystem = "engine" -// nolint: funlen func newEngineMetrics() engineMetrics { - var ( - listContainersDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "list_containers_duration", - Help: "Accumulated duration of engine list containers operations", - }) - - estimateContainerSizeDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "estimate_container_size_duration", - Help: "Accumulated duration of engine container size estimate operations", - }) - - deleteDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "delete_duration", - Help: "Accumulated duration of engine delete operations", - }) - - existsDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "exists_duration", - Help: "Accumulated duration of engine exists operations", - }) - - getDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "get_duration", - Help: "Accumulated duration of engine get operations", - }) - - headDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "head_duration", - Help: "Accumulated duration of engine head operations", - }) - - inhumeDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "inhume_duration", - Help: "Accumulated duration of engine inhume operations", - }) - - putDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "put_duration", - Help: "Accumulated duration of engine put operations", - }) - - rangeDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "range_duration", - Help: "Accumulated duration of engine range operations", - }) - - searchDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "search_duration", - Help: "Accumulated duration of engine search operations", - }) - - listObjectsDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "list_objects_duration", - Help: "Accumulated duration of engine list objects operations", - }) - - containerSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "container_size", - Help: "Accumulated size of all objects in a container", - }, []string{containerIDLabelKey}) - - payloadSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: "payload_size", - Help: "Accumulated size of all objects in a shard", - }, []string{shardIDLabelKey}) - ) - return engineMetrics{ - listContainersDuration: listContainersDuration, - estimateContainerSizeDuration: estimateContainerSizeDuration, - deleteDuration: deleteDuration, - existsDuration: existsDuration, - getDuration: getDuration, - headDuration: headDuration, - inhumeDuration: inhumeDuration, - putDuration: putDuration, - rangeDuration: rangeDuration, - searchDuration: searchDuration, - listObjectsDuration: listObjectsDuration, - containerSize: *containerSize, - payloadSize: *payloadSize, + listContainersDuration: newEngineMethodDurationCounter("list_containers_"), + estimateContainerSizeDuration: newEngineCounter("estimate_container_size_duration", "Accumulated duration of engine container size estimate operations"), + deleteDuration: newEngineMethodDurationCounter("delete"), + existsDuration: newEngineMethodDurationCounter("exists"), + getDuration: newEngineMethodDurationCounter("get"), + headDuration: newEngineMethodDurationCounter("head"), + inhumeDuration: newEngineMethodDurationCounter("inhume"), + putDuration: newEngineMethodDurationCounter("put"), + rangeDuration: newEngineMethodDurationCounter("range"), + searchDuration: newEngineMethodDurationCounter("search"), + listObjectsDuration: newEngineMethodDurationCounter("list_objects"), + containerSize: *newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), + payloadSize: *newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), } } +func newEngineCounter(name, help string) prometheus.Counter { + return prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: engineSubsystem, + Name: name, + Help: help, + }) +} + +func newEngineMethodDurationCounter(method string) prometheus.Counter { + return newEngineCounter( + fmt.Sprintf("%s_duration", method), + fmt.Sprintf("Accumulated duration of engine %s operations", strings.ReplaceAll(method, "_", " ")), + ) +} + +func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { + return prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: engineSubsystem, + Name: name, + Help: help, + }, labels) +} + func (m engineMetrics) register() { prometheus.MustRegister(m.listContainersDuration) prometheus.MustRegister(m.estimateContainerSizeDuration) From 02831d427b01ae0152e4238721c003c987e979e5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Apr 2023 12:43:32 +0300 Subject: [PATCH 0074/1943] [#213] metrics: Refactor object metrics Resolve funlen linter for newObjectServiceMetrics function. Signed-off-by: Dmitrii Stepanov --- pkg/metrics/object.go | 162 ++++++++++++------------------------------ 1 file changed, 47 insertions(+), 115 deletions(-) diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index ffa5d481d..fae86cb4a 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -2,6 +2,7 @@ package metrics import ( "fmt" + "strings" "time" "github.com/prometheus/client_golang/prometheus" @@ -46,7 +47,7 @@ const ( containerIDLabelKey = "cid" ) -func newMethodCallCounter(name string) methodCount { +func newObjectMethodCallCounter(name string) methodCount { return methodCount{ success: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: namespace, @@ -75,125 +76,56 @@ func (m methodCount) Inc(success bool) { } } -// nolint: funlen func newObjectServiceMetrics() objectServiceMetrics { - var ( // Request counter metrics. - getCounter = newMethodCallCounter("get") - putCounter = newMethodCallCounter("put") - headCounter = newMethodCallCounter("head") - searchCounter = newMethodCallCounter("search") - deleteCounter = newMethodCallCounter("delete") - rangeCounter = newMethodCallCounter("range") - rangeHashCounter = newMethodCallCounter("range_hash") - ) - - var ( // Request duration metrics. - getDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "get_req_duration", - Help: "Accumulated get request process duration", - }) - - putDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "put_req_duration", - Help: "Accumulated put request process duration", - }) - - headDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "head_req_duration", - Help: "Accumulated head request process duration", - }) - - searchDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "search_req_duration", - Help: "Accumulated search request process duration", - }) - - deleteDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "delete_req_duration", - Help: "Accumulated delete request process duration", - }) - - rangeDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "range_req_duration", - Help: "Accumulated range request process duration", - }) - - rangeHashDuration = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "range_hash_req_duration", - Help: "Accumulated range hash request process duration", - }) - ) - - var ( // Object payload metrics. - putPayload = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "put_payload", - Help: "Accumulated payload size at object put method", - }) - - getPayload = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "get_payload", - Help: "Accumulated payload size at object get method", - }) - - shardsMetrics = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "counter", - Help: "Objects counters per shards", - }, - []string{shardIDLabelKey, counterTypeLabelKey}, - ) - - shardsReadonly = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "readonly", - Help: "Shard state", - }, - []string{shardIDLabelKey}, - ) - ) - return objectServiceMetrics{ - getCounter: getCounter, - putCounter: putCounter, - headCounter: headCounter, - searchCounter: searchCounter, - deleteCounter: deleteCounter, - rangeCounter: rangeCounter, - rangeHashCounter: rangeHashCounter, - getDuration: getDuration, - putDuration: putDuration, - headDuration: headDuration, - searchDuration: searchDuration, - deleteDuration: deleteDuration, - rangeDuration: rangeDuration, - rangeHashDuration: rangeHashDuration, - putPayload: putPayload, - getPayload: getPayload, - shardMetrics: shardsMetrics, - shardsReadonly: shardsReadonly, + getCounter: newObjectMethodCallCounter("get"), + putCounter: newObjectMethodCallCounter("put"), + headCounter: newObjectMethodCallCounter("head"), + searchCounter: newObjectMethodCallCounter("search"), + deleteCounter: newObjectMethodCallCounter("delete"), + rangeCounter: newObjectMethodCallCounter("range"), + rangeHashCounter: newObjectMethodCallCounter("range_hash"), + getDuration: newObjectMethodDurationCounter("get"), + putDuration: newObjectMethodDurationCounter("put"), + headDuration: newObjectMethodDurationCounter("head"), + searchDuration: newObjectMethodDurationCounter("search"), + deleteDuration: newObjectMethodDurationCounter("delete"), + rangeDuration: newObjectMethodDurationCounter("range"), + rangeHashDuration: newObjectMethodDurationCounter("range_hash"), + putPayload: newObjectMethodPayloadCounter("put"), + getPayload: newObjectMethodPayloadCounter("get"), + shardMetrics: newObjectGaugeVector("counter", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), + shardsReadonly: newObjectGaugeVector("readonly", "Shard state", []string{shardIDLabelKey}), } } +func newObjectMethodPayloadCounter(method string) prometheus.Counter { + return prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: objectSubsystem, + Name: fmt.Sprintf("%s_payload", method), + Help: fmt.Sprintf("Accumulated payload size at object %s method", strings.ReplaceAll(method, "_", " ")), + }) +} + +func newObjectMethodDurationCounter(method string) prometheus.Counter { + return prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: objectSubsystem, + Name: fmt.Sprintf("%s_req_duration", method), + Help: fmt.Sprintf("Accumulated %s request process duration", strings.ReplaceAll(method, "_", " ")), + }) +} + +func newObjectGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { + return prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: objectSubsystem, + Name: name, + Help: help, + }, labels) +} + func (m objectServiceMetrics) register() { m.getCounter.mustRegister() m.putCounter.mustRegister() From 6016d78a45a0779463347af63dbefa7eb93aa889 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Apr 2023 13:02:37 +0300 Subject: [PATCH 0075/1943] [#223] core: Refactor object format validator Resolve funlen linter for FormatValidator.ValidateContent method. Signed-off-by: Dmitrii Stepanov --- pkg/core/object/fmt.go | 196 ++++++++++++++++++++++------------------- 1 file changed, 103 insertions(+), 93 deletions(-) diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index 053306356..33373b7cc 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -202,112 +202,24 @@ func (i ContentMeta) Objects() []oid.ID { } // ValidateContent validates payload content according to the object type. -// -// nolint: funlen func (v *FormatValidator) ValidateContent(o *object.Object) (ContentMeta, error) { meta := ContentMeta{ typ: o.Type(), } switch o.Type() { - case object.TypeRegular: - // ignore regular objects, they do not need payload formatting case object.TypeTombstone: - if len(o.Payload()) == 0 { - return ContentMeta{}, fmt.Errorf("(%T) empty payload in tombstone", v) - } - - tombstone := object.NewTombstone() - - if err := tombstone.Unmarshal(o.Payload()); err != nil { - return ContentMeta{}, fmt.Errorf("(%T) could not unmarshal tombstone content: %w", v, err) - } - - // check if the tombstone has the same expiration in the body and the header - exp, err := expirationEpochAttribute(o) - if err != nil { + if err := v.fillAndValidateTombstoneMeta(o, &meta); err != nil { return ContentMeta{}, err } - - if exp != tombstone.ExpirationEpoch() { - return ContentMeta{}, errTombstoneExpiration - } - - // mark all objects from the tombstone body as removed in the storage engine - _, ok := o.ContainerID() - if !ok { - return ContentMeta{}, errors.New("missing container ID") - } - - idList := tombstone.Members() - meta.objs = idList case object.TypeStorageGroup: - if len(o.Payload()) == 0 { - return ContentMeta{}, fmt.Errorf("(%T) empty payload in SG", v) - } - - var sg storagegroup.StorageGroup - - if err := sg.Unmarshal(o.Payload()); err != nil { - return ContentMeta{}, fmt.Errorf("(%T) could not unmarshal SG content: %w", v, err) - } - - mm := sg.Members() - meta.objs = mm - - lenMM := len(mm) - if lenMM == 0 { - return ContentMeta{}, errEmptySGMembers - } - - uniqueFilter := make(map[oid.ID]struct{}, lenMM) - - for i := 0; i < lenMM; i++ { - if _, alreadySeen := uniqueFilter[mm[i]]; alreadySeen { - return ContentMeta{}, fmt.Errorf("storage group contains non-unique member: %s", mm[i]) - } - - uniqueFilter[mm[i]] = struct{}{} + if err := v.fillAndValidateStorageGroupMeta(o, &meta); err != nil { + return ContentMeta{}, err } case object.TypeLock: - if len(o.Payload()) == 0 { - return ContentMeta{}, errors.New("empty payload in lock") + if err := v.fillAndValidateLockMeta(o, &meta); err != nil { + return ContentMeta{}, err } - - _, ok := o.ContainerID() - if !ok { - return ContentMeta{}, errors.New("missing container") - } - - _, ok = o.ID() - if !ok { - return ContentMeta{}, errors.New("missing ID") - } - - // check that LOCK object has correct expiration epoch - lockExp, err := expirationEpochAttribute(o) - if err != nil { - return ContentMeta{}, fmt.Errorf("lock object expiration epoch: %w", err) - } - - if currEpoch := v.netState.CurrentEpoch(); lockExp < currEpoch { - return ContentMeta{}, fmt.Errorf("lock object expiration: %d; current: %d", lockExp, currEpoch) - } - - var lock object.Lock - - err = lock.Unmarshal(o.Payload()) - if err != nil { - return ContentMeta{}, fmt.Errorf("decode lock payload: %w", err) - } - - num := lock.NumberOfMembers() - if num == 0 { - return ContentMeta{}, errors.New("missing locked members") - } - - meta.objs = make([]oid.ID, num) - lock.ReadMembers(meta.objs) default: // ignore all other object types, they do not need payload formatting } @@ -315,6 +227,104 @@ func (v *FormatValidator) ValidateContent(o *object.Object) (ContentMeta, error) return meta, nil } +func (v *FormatValidator) fillAndValidateLockMeta(o *object.Object, meta *ContentMeta) error { + if len(o.Payload()) == 0 { + return errors.New("empty payload in lock") + } + + if _, ok := o.ContainerID(); !ok { + return errors.New("missing container") + } + + if _, ok := o.ID(); !ok { + return errors.New("missing ID") + } + // check that LOCK object has correct expiration epoch + lockExp, err := expirationEpochAttribute(o) + if err != nil { + return fmt.Errorf("lock object expiration epoch: %w", err) + } + + if currEpoch := v.netState.CurrentEpoch(); lockExp < currEpoch { + return fmt.Errorf("lock object expiration: %d; current: %d", lockExp, currEpoch) + } + + var lock object.Lock + + if err = lock.Unmarshal(o.Payload()); err != nil { + return fmt.Errorf("decode lock payload: %w", err) + } + + num := lock.NumberOfMembers() + if num == 0 { + return errors.New("missing locked members") + } + + meta.objs = make([]oid.ID, num) + lock.ReadMembers(meta.objs) + return nil +} + +func (v *FormatValidator) fillAndValidateStorageGroupMeta(o *object.Object, meta *ContentMeta) error { + if len(o.Payload()) == 0 { + return fmt.Errorf("(%T) empty payload in storage group", v) + } + + var sg storagegroup.StorageGroup + + if err := sg.Unmarshal(o.Payload()); err != nil { + return fmt.Errorf("(%T) could not unmarshal storage group content: %w", v, err) + } + + mm := sg.Members() + meta.objs = mm + + lenMM := len(mm) + if lenMM == 0 { + return errEmptySGMembers + } + + uniqueFilter := make(map[oid.ID]struct{}, lenMM) + + for i := 0; i < lenMM; i++ { + if _, alreadySeen := uniqueFilter[mm[i]]; alreadySeen { + return fmt.Errorf("storage group contains non-unique member: %s", mm[i]) + } + + uniqueFilter[mm[i]] = struct{}{} + } + return nil +} + +func (v *FormatValidator) fillAndValidateTombstoneMeta(o *object.Object, meta *ContentMeta) error { + if len(o.Payload()) == 0 { + return fmt.Errorf("(%T) empty payload in tombstone", v) + } + + tombstone := object.NewTombstone() + + if err := tombstone.Unmarshal(o.Payload()); err != nil { + return fmt.Errorf("(%T) could not unmarshal tombstone content: %w", v, err) + } + // check if the tombstone has the same expiration in the body and the header + exp, err := expirationEpochAttribute(o) + if err != nil { + return err + } + + if exp != tombstone.ExpirationEpoch() { + return errTombstoneExpiration + } + + // mark all objects from the tombstone body as removed in the storage engine + if _, ok := o.ContainerID(); !ok { + return errors.New("missing container ID") + } + + meta.objs = tombstone.Members() + return nil +} + var errExpired = errors.New("object has expired") func (v *FormatValidator) checkExpiration(obj *object.Object) error { From ae86cda58c35d01092c895b9d053877ac0c02938 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Apr 2023 13:41:45 +0300 Subject: [PATCH 0076/1943] [#223] sg: Refactor storage group parameters Resolve containedctx linter for SearchSGPrm and GetSGPrm structs. Signed-off-by: Dmitrii Stepanov --- pkg/core/storagegroup/storagegroup.go | 10 ++-------- pkg/innerring/processors/audit/process.go | 6 ++---- pkg/innerring/rpc.go | 8 ++++---- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/pkg/core/storagegroup/storagegroup.go b/pkg/core/storagegroup/storagegroup.go index f4e584134..b16e5c61a 100644 --- a/pkg/core/storagegroup/storagegroup.go +++ b/pkg/core/storagegroup/storagegroup.go @@ -11,10 +11,7 @@ import ( ) // SearchSGPrm groups the parameters which are formed by Processor to search the storage group objects. -// nolint: containedctx type SearchSGPrm struct { - Context context.Context - Container cid.ID NodeInfo client.NodeInfo @@ -26,10 +23,7 @@ type SearchSGDst struct { } // GetSGPrm groups parameter of GetSG operation. -// nolint: containedctx type GetSGPrm struct { - Context context.Context - OID oid.ID CID cid.ID @@ -42,11 +36,11 @@ type SGSource interface { // ListSG must list storage group objects in the container. Formed list must be written to destination. // // Must return any error encountered which did not allow to form the list. - ListSG(*SearchSGDst, SearchSGPrm) error + ListSG(context.Context, *SearchSGDst, SearchSGPrm) error // GetSG must return storage group object for the provided CID, OID, // container and netmap state. - GetSG(GetSGPrm) (*storagegroup.StorageGroup, error) + GetSG(context.Context, GetSGPrm) (*storagegroup.StorageGroup, error) } // StorageGroup combines storage group object ID and its structure. diff --git a/pkg/innerring/processors/audit/process.go b/pkg/innerring/processors/audit/process.go index ecfc407be..656927816 100644 --- a/pkg/innerring/processors/audit/process.go +++ b/pkg/innerring/processors/audit/process.go @@ -153,12 +153,11 @@ func (ap *Processor) findStorageGroups(cnr cid.ID, shuffled netmapcore.Nodes) [] ctx, cancel := context.WithTimeout(context.Background(), ap.searchTimeout) - prm.Context = ctx prm.NodeInfo = info var dst storagegroup.SearchSGDst - err = ap.sgSrc.ListSG(&dst, prm) + err = ap.sgSrc.ListSG(ctx, &dst, prm) cancel() @@ -189,9 +188,8 @@ func (ap *Processor) filterExpiredSG(cid cid.ID, sgIDs []oid.ID, ctx, cancel := context.WithTimeout(context.Background(), ap.searchTimeout) getSGPrm.OID = sgID - getSGPrm.Context = ctx - sg, err := ap.sgSrc.GetSG(getSGPrm) + sg, err := ap.sgSrc.GetSG(ctx, getSGPrm) cancel() diff --git a/pkg/innerring/rpc.go b/pkg/innerring/rpc.go index c58e0ddc4..8e96deb7b 100644 --- a/pkg/innerring/rpc.go +++ b/pkg/innerring/rpc.go @@ -65,12 +65,12 @@ func (c *ClientCache) Get(info clientcore.NodeInfo) (clientcore.Client, error) { // Returns storage groups structure from received object. // // Returns an error of type apistatus.ObjectNotFound if storage group is missing. -func (c *ClientCache) GetSG(prm storagegroup2.GetSGPrm) (*storagegroup.StorageGroup, error) { +func (c *ClientCache) GetSG(ctx context.Context, prm storagegroup2.GetSGPrm) (*storagegroup.StorageGroup, error) { var sgAddress oid.Address sgAddress.SetContainer(prm.CID) sgAddress.SetObject(prm.OID) - return c.getSG(prm.Context, sgAddress, &prm.NetMap, prm.Container) + return c.getSG(ctx, sgAddress, &prm.NetMap, prm.Container) } func (c *ClientCache) getSG(ctx context.Context, addr oid.Address, nm *netmap.NetMap, cn [][]netmap.NodeInfo) (*storagegroup.StorageGroup, error) { @@ -214,7 +214,7 @@ func (c *ClientCache) getWrappedClient(info clientcore.NodeInfo) (frostfsapiclie return cInternal, nil } -func (c ClientCache) ListSG(dst *storagegroup2.SearchSGDst, prm storagegroup2.SearchSGPrm) error { +func (c ClientCache) ListSG(ctx context.Context, dst *storagegroup2.SearchSGDst, prm storagegroup2.SearchSGPrm) error { cli, err := c.getWrappedClient(prm.NodeInfo) if err != nil { return fmt.Errorf("could not get API client from cache") @@ -224,7 +224,7 @@ func (c ClientCache) ListSG(dst *storagegroup2.SearchSGDst, prm storagegroup2.Se cliPrm.SetContainerID(prm.Container) - res, err := cli.SearchSG(prm.Context, cliPrm) + res, err := cli.SearchSG(ctx, cliPrm) if err != nil { return err } From ccf8463e69e9abc308d3feecc24be1b9449d12e3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Apr 2023 15:18:56 +0300 Subject: [PATCH 0077/1943] [#223] controlsvc: Drop unnecessary nolint Signed-off-by: Dmitrii Stepanov --- pkg/services/control/server/gc.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/services/control/server/gc.go b/pkg/services/control/server/gc.go index 7912d4e3e..d382dd7e5 100644 --- a/pkg/services/control/server/gc.go +++ b/pkg/services/control/server/gc.go @@ -41,7 +41,6 @@ func (s *Server) DropObjects(ctx context.Context, req *control.DropObjectsReques prm.WithForceRemoval() prm.WithAddress(addrList[i]) - //nolint: contextcheck _, err := s.s.Delete(ctx, prm) if err != nil && firstErr == nil { firstErr = err From 2ed9fd3f9445ee651fe434680af7c567328ae2c1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Apr 2023 15:36:37 +0300 Subject: [PATCH 0078/1943] [#223] objectsvc: Refactor request parameters Resolve containedctx linter for commonPrm. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/util.go | 9 ++--- pkg/services/object/head/remote.go | 3 +- pkg/services/object/internal/client/client.go | 36 +++++++------------ pkg/services/object/put/remote.go | 3 +- pkg/services/object/search/util.go | 3 +- 5 files changed, 19 insertions(+), 35 deletions(-) diff --git a/pkg/services/object/get/util.go b/pkg/services/object/get/util.go index 3bb68862b..09d8c67af 100644 --- a/pkg/services/object/get/util.go +++ b/pkg/services/object/get/util.go @@ -113,7 +113,6 @@ func (c *clientWrapper) getObject(ctx context.Context, exec *execCtx, info corec func (c *clientWrapper) getRange(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey, rng *object.Range) (*object.Object, error) { var prm internalclient.PayloadRangePrm - prm.SetContext(ctx) prm.SetClient(c.client) prm.SetTTL(exec.prm.common.TTL()) prm.SetNetmapEpoch(exec.curProcEpoch) @@ -128,7 +127,7 @@ func (c *clientWrapper) getRange(ctx context.Context, exec *execCtx, key *ecdsa. prm.SetRawFlag() } - res, err := internalclient.PayloadRange(prm) + res, err := internalclient.PayloadRange(ctx, prm) if err != nil { var errAccessDenied *apistatus.ObjectAccessDenied if errors.As(err, &errAccessDenied) { @@ -156,7 +155,6 @@ func (c *clientWrapper) getRange(ctx context.Context, exec *execCtx, key *ecdsa. func (c *clientWrapper) getHeadOnly(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { var prm internalclient.HeadObjectPrm - prm.SetContext(ctx) prm.SetClient(c.client) prm.SetTTL(exec.prm.common.TTL()) prm.SetNetmapEpoch(exec.curProcEpoch) @@ -170,7 +168,7 @@ func (c *clientWrapper) getHeadOnly(ctx context.Context, exec *execCtx, key *ecd prm.SetRawFlag() } - res, err := internalclient.HeadObject(prm) + res, err := internalclient.HeadObject(ctx, prm) if err != nil { return nil, err } @@ -181,7 +179,6 @@ func (c *clientWrapper) getHeadOnly(ctx context.Context, exec *execCtx, key *ecd func (c *clientWrapper) get(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { var prm internalclient.GetObjectPrm - prm.SetContext(ctx) prm.SetClient(c.client) prm.SetTTL(exec.prm.common.TTL()) prm.SetNetmapEpoch(exec.curProcEpoch) @@ -195,7 +192,7 @@ func (c *clientWrapper) get(ctx context.Context, exec *execCtx, key *ecdsa.Priva prm.SetRawFlag() } - res, err := internalclient.GetObject(prm) + res, err := internalclient.GetObject(ctx, prm) if err != nil { return nil, err } diff --git a/pkg/services/object/head/remote.go b/pkg/services/object/head/remote.go index f50c3422a..85f076a76 100644 --- a/pkg/services/object/head/remote.go +++ b/pkg/services/object/head/remote.go @@ -84,13 +84,12 @@ func (h *RemoteHeader) Head(ctx context.Context, prm *RemoteHeadPrm) (*object.Ob var headPrm internalclient.HeadObjectPrm - headPrm.SetContext(ctx) headPrm.SetClient(c) headPrm.SetPrivateKey(key) headPrm.SetAddress(prm.commonHeadPrm.addr) headPrm.SetTTL(remoteOpTTL) - res, err := internalclient.HeadObject(headPrm) + res, err := internalclient.HeadObject(ctx, headPrm) if err != nil { return nil, fmt.Errorf("(%T) could not head object in %s: %w", h, info.AddressGroup(), err) } diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index ee6777a5d..10a6af271 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -18,12 +18,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" ) -// nolint: containedctx type commonPrm struct { cli coreclient.Client - ctx context.Context - key *ecdsa.PrivateKey tokenSession *session.Object @@ -42,13 +39,6 @@ func (x *commonPrm) SetClient(cli coreclient.Client) { x.cli = cli } -// SetContext sets context.Context for network communication. -// -// Required parameter. -func (x *commonPrm) SetContext(ctx context.Context) { - x.ctx = ctx -} - // SetPrivateKey sets private key to sign the request(s). // // Required parameter. @@ -138,7 +128,7 @@ func (x GetObjectRes) Object() *object.Object { // - error of type *apistatus.ObjectAlreadyRemoved if the requested object is marked to be removed. // // GetObject ignores the provided session if it is not related to the requested object. -func GetObject(prm GetObjectPrm) (*GetObjectRes, error) { +func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { // here we ignore session if it is opened for other object since such // request will almost definitely fail. The case can occur, for example, // when session is bound to the parent object and child object is requested. @@ -159,7 +149,7 @@ func GetObject(prm GetObjectPrm) (*GetObjectRes, error) { prm.cliPrm.UseKey(*prm.key) } - rdr, err := prm.cli.ObjectGetInit(prm.ctx, prm.cliPrm) + rdr, err := prm.cli.ObjectGetInit(ctx, prm.cliPrm) if err != nil { return nil, fmt.Errorf("init object reading: %w", err) } @@ -229,7 +219,7 @@ func (x HeadObjectRes) Header() *object.Object { // HeadObject reads object header by address. // -// Client, context and key must be set. +// Client and key must be set. // // Returns any error which prevented the operation from completing correctly in error return. // Returns: @@ -238,7 +228,7 @@ func (x HeadObjectRes) Header() *object.Object { // error of type *apistatus.ObjectAlreadyRemoved if the requested object is marked to be removed. // // HeadObject ignores the provided session if it is not related to the requested object. -func HeadObject(prm HeadObjectPrm) (*HeadObjectRes, error) { +func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) { if prm.local { prm.cliPrm.MarkLocal() } @@ -254,7 +244,7 @@ func HeadObject(prm HeadObjectPrm) (*HeadObjectRes, error) { prm.cliPrm.WithXHeaders(prm.xHeaders...) - cliRes, err := prm.cli.ObjectHead(prm.ctx, prm.cliPrm) + cliRes, err := prm.cli.ObjectHead(ctx, prm.cliPrm) if err == nil { // pull out an error from status err = apistatus.ErrFromStatus(cliRes.Status()) @@ -327,7 +317,7 @@ const maxInitialBufferSize = 1024 * 1024 // 1 MiB // PayloadRange reads object payload range by address. // -// Client, context and key must be set. +// Client and key must be set. // // Returns any error which prevented the operation from completing correctly in error return. // Returns: @@ -337,7 +327,7 @@ const maxInitialBufferSize = 1024 * 1024 // 1 MiB // error of type *apistatus.ObjectOutOfRange if the requested range is too big. // // PayloadRange ignores the provided session if it is not related to the requested object. -func PayloadRange(prm PayloadRangePrm) (*PayloadRangeRes, error) { +func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, error) { if prm.local { prm.cliPrm.MarkLocal() } @@ -354,7 +344,7 @@ func PayloadRange(prm PayloadRangePrm) (*PayloadRangeRes, error) { prm.cliPrm.SetLength(prm.ln) prm.cliPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectRangeInit(prm.ctx, prm.cliPrm) + rdr, err := prm.cli.ObjectRangeInit(ctx, prm.cliPrm) if err != nil { return nil, fmt.Errorf("init payload reading: %w", err) } @@ -408,10 +398,10 @@ func (x PutObjectRes) ID() oid.ID { // PutObject saves the object in local storage of the remote node. // -// Client, context and key must be set. +// Client and key must be set. // // Returns any error which prevented the operation from completing correctly in error return. -func PutObject(prm PutObjectPrm) (*PutObjectRes, error) { +func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { var prmCli client.PrmObjectPutInit prmCli.MarkLocal() @@ -430,7 +420,7 @@ func PutObject(prm PutObjectPrm) (*PutObjectRes, error) { prmCli.WithXHeaders(prm.xHeaders...) - w, err := prm.cli.ObjectPutInit(prm.ctx, prmCli) + w, err := prm.cli.ObjectPutInit(ctx, prmCli) if err != nil { return nil, fmt.Errorf("init object writing on client: %w", err) } @@ -487,7 +477,7 @@ func (x SearchObjectsRes) IDList() []oid.ID { // SearchObjects selects objects from container which match the filters. // // Returns any error which prevented the operation from completing correctly in error return. -func SearchObjects(prm SearchObjectsPrm) (*SearchObjectsRes, error) { +func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes, error) { if prm.local { prm.cliPrm.MarkLocal() } @@ -506,7 +496,7 @@ func SearchObjects(prm SearchObjectsPrm) (*SearchObjectsRes, error) { prm.cliPrm.UseKey(*prm.key) } - rdr, err := prm.cli.ObjectSearchInit(prm.ctx, prm.cliPrm) + rdr, err := prm.cli.ObjectSearchInit(ctx, prm.cliPrm) if err != nil { return nil, fmt.Errorf("init object searching in client: %w", err) } diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/put/remote.go index 6933abca6..e7fa124fa 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/put/remote.go @@ -56,7 +56,6 @@ func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifier var prm internalclient.PutObjectPrm - prm.SetContext(ctx) prm.SetClient(c) prm.SetPrivateKey(t.privateKey) prm.SetSessionToken(t.commonPrm.SessionToken()) @@ -64,7 +63,7 @@ func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifier prm.SetXHeaders(t.commonPrm.XHeaders()) prm.SetObject(t.obj) - res, err := internalclient.PutObject(prm) + res, err := internalclient.PutObject(ctx, prm) if err != nil { return nil, fmt.Errorf("(%T) could not put object to %s: %w", t, t.nodeInfo.AddressGroup(), err) } diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index c12ed2c9f..49f3e5efd 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -99,7 +99,6 @@ func (c *clientWrapper) searchObjects(ctx context.Context, exec *execCtx, info c var prm internalclient.SearchObjectsPrm - prm.SetContext(ctx) prm.SetClient(c.client) prm.SetPrivateKey(key) prm.SetSessionToken(exec.prm.common.SessionToken()) @@ -110,7 +109,7 @@ func (c *clientWrapper) searchObjects(ctx context.Context, exec *execCtx, info c prm.SetContainerID(exec.containerID()) prm.SetFilters(exec.searchFilters()) - res, err := internalclient.SearchObjects(prm) + res, err := internalclient.SearchObjects(ctx, prm) if err != nil { return nil, err } From 93eba19a8ee619bf9716cbd9fd26ef6adc87e693 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Apr 2023 16:03:00 +0300 Subject: [PATCH 0079/1943] [#223] objectsvc: Refactor split-tree traverse Resolve funlen & gocognit linters for traverseSplitChain method. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/util/chain.go | 141 ++++++++++++++++-------------- 1 file changed, 75 insertions(+), 66 deletions(-) diff --git a/pkg/services/object/util/chain.go b/pkg/services/object/util/chain.go index e6ec93630..96dafd10e 100644 --- a/pkg/services/object/util/chain.go +++ b/pkg/services/object/util/chain.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -72,7 +73,6 @@ func TraverseSplitChain(r HeadReceiver, addr oid.Address, h SplitMemberHandler) return err } -// nolint: funlen, gocognit func traverseSplitChain(r HeadReceiver, addr oid.Address, h SplitMemberHandler) (bool, error) { v, err := r.Head(addr) if err != nil { @@ -94,80 +94,89 @@ func traverseSplitChain(r HeadReceiver, addr oid.Address, h SplitMemberHandler) default: return false, errors.New("lack of split information") case withLink: - var addr oid.Address - addr.SetContainer(cnr) - addr.SetObject(link) - - chain := make([]oid.ID, 0) - - if _, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { - children := member.Children() - - if reverseDirection { - chain = append(children, chain...) - } else { - chain = append(chain, children...) - } - - return false - }); err != nil { - return false, err - } - - var reverseChain []*object.Object - - for i := range chain { - addr.SetObject(chain[i]) - - if stop, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { - if !reverseDirection { - return h(member, false) - } - - reverseChain = append(reverseChain, member) - return false - }); err != nil || stop { - return stop, err - } - } - - for i := len(reverseChain) - 1; i >= 0; i-- { - if h(reverseChain[i], false) { - return true, nil - } - } + return traverseByLink(cnr, link, r, h) case withLast: - var addr oid.Address - addr.SetContainer(cnr) + return traverseByLast(cnr, last, withLast, res, r, h) + } + } +} - for last, withLast = res.LastPart(); withLast; { - addr.SetObject(last) +func traverseByLink(cnr cid.ID, link oid.ID, r HeadReceiver, h SplitMemberHandler) (bool, error) { + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(link) - var directChain []*object.Object + chain := make([]oid.ID, 0) - if _, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { - if reverseDirection { - last, withLast = member.PreviousID() - return h(member, true) - } + if _, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + children := member.Children() - directChain = append(directChain, member) + if reverseDirection { + chain = append(children, chain...) + } else { + chain = append(chain, children...) + } - return false - }); err != nil { - return false, err - } + return false + }); err != nil { + return false, err + } - for i := len(directChain) - 1; i >= 0; i-- { - if h(directChain[i], true) { - return true, nil - } - } + var reverseChain []*object.Object - if len(directChain) > 0 { - last, withLast = directChain[len(directChain)-1].PreviousID() - } + for i := range chain { + addr.SetObject(chain[i]) + + if stop, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if !reverseDirection { + return h(member, false) } + + reverseChain = append(reverseChain, member) + return false + }); err != nil || stop { + return stop, err + } + } + + for i := len(reverseChain) - 1; i >= 0; i-- { + if h(reverseChain[i], false) { + return true, nil + } + } + return false, nil +} + +func traverseByLast(cnr cid.ID, last oid.ID, withLast bool, res *object.SplitInfo, r HeadReceiver, h SplitMemberHandler) (bool, error) { + var addr oid.Address + addr.SetContainer(cnr) + + for last, withLast = res.LastPart(); withLast; { + addr.SetObject(last) + + var directChain []*object.Object + + if _, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if reverseDirection { + last, withLast = member.PreviousID() + return h(member, true) + } + + directChain = append(directChain, member) + + return false + }); err != nil { + return false, err + } + + for i := len(directChain) - 1; i >= 0; i-- { + if h(directChain[i], true) { + return true, nil + } + } + + if len(directChain) > 0 { + last, withLast = directChain[len(directChain)-1].PreviousID() } } From 2c07f831c7b58246b5182f7aafb9813187005a9e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Apr 2023 16:24:43 +0300 Subject: [PATCH 0080/1943] [#223] node: Refactor cache usage Drop excess type args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/cache.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index eda691496..3d4fc7375 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -130,7 +130,7 @@ type ttlContainerStorage struct { func newCachedContainerStorage(v container.Source, ttl time.Duration) ttlContainerStorage { const containerCacheSize = 100 - lruCnrCache := newNetworkTTLCache[cid.ID, *container.Container](containerCacheSize, ttl, func(id cid.ID) (*container.Container, error) { + lruCnrCache := newNetworkTTLCache(containerCacheSize, ttl, func(id cid.ID) (*container.Container, error) { return v.Get(id) }) @@ -340,7 +340,7 @@ func newCachedIRFetcher(f interface{ InnerRingKeys() ([][]byte, error) }) cached irFetcherCacheTTL = 30 * time.Second ) - irFetcherCache := newNetworkTTLCache[struct{}, [][]byte](irFetcherCacheSize, irFetcherCacheTTL, + irFetcherCache := newNetworkTTLCache(irFetcherCacheSize, irFetcherCacheTTL, func(_ struct{}) ([][]byte, error) { return f.InnerRingKeys() }, From 6bf11f7cca5b2845b239b926b4c81fe56815c2fa Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 10 Apr 2023 10:44:37 +0300 Subject: [PATCH 0081/1943] [#230] CHANGELOG.md: Remove older entries Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1565 +------------------------------------------------- 1 file changed, 5 insertions(+), 1560 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6793ed340..db846936f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,7 +67,7 @@ Changelog for FrostFS Node - `github.com/TrueCloudLab/hrw` to `v.1.1.1` - Minimum go version to v1.18 -### Updating from v0.35.0 +### Updating from v0.35.0 (old NeoFS) You need to change configuration environment variables to `FROSTFS_*` if you use any. @@ -77,1564 +77,9 @@ more appropriate for a specific deployment. Use `__SYSTEM__` prefix for system attributes instead of `__NEOFS__` (existed objects with old attributes will be treated as before, but for new objects new attributes will be used). -## [0.35.0] - 2022-12-28 - Sindo (신도, 信島) +## Older versions -### Added -- `morph list-containers` in `neofs-adm` (#1689) -- `--binary` flag in `neofs-cli object put/get/delete` commands (#1338) -- `session` flag support to `neofs-cli object hash` (#2029) -- Shard can now change mode when encountering background disk errors (#2035) -- Background workers and object service now use separate client caches (#2048) -- `replicator.pool_size` config field to tune replicator pool size (#2049) -- Fix NNS hash parsing in morph client (#2063) -- `neofs-cli neofs-cli acl basic/extended print` commands (#2012) -- `neofs_node_object_*_req_count_success` prometheus metrics for tracking successfully executed requests (#1984) -- Metric 'readonly' to get shards mode (#2022) -- Tree service replication timeout (#2159) -- `apiclient.reconnect_timeout` setting allowing to ignore failed clients for some time (#2164) +This project is a fork of [NeoFS](https://github.com/nspcc-dev/neofs-node) from version v0.35.0. +To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs-node/blob/master/CHANGELOG.md. -### Changed -- `object lock` command reads CID and OID the same way other commands do (#1971) -- `LOCK` object are stored on every container node (#1502) -- `neofs-cli container get-eacl` print ACL table in json format only with arg `--json' (#2012) -- Side chain notary deposits use max uint32 as till parameter (#1486) -- Allow object removal without linking object (#2100) -- `neofs-cli container delete` command pre-checks container ownership (#2106) -- Policer cache size is now 1024 (#2158) -- Tree service now synchronizes with container nodes in a random order (#2127) -- Pilorama no longer tries to apply already applied operations (#2161) -- Use `sync.Pool` in Object.PUT service (#2139) -- Shard uses metabase for `HEAD` requests by default, not write-cache (#2167) -- Clarify help for `--expire-at` parameter for commands `object lock/put` and `bearer create` (#2097) -- Node spawns `GETRANGE` requests signed with the node's key if session key was not found for `RANGEHASH` (#2144) -- Full list of container is no longer cached (#2176) - -### Fixed -- Open FSTree in sync mode by default (#1992) -- `neofs-cli container nodes`'s output (#1991) -- Increase error counter for write-cache flush errors (#1818) -- Correctly select the shard for applying tree service operations (#1996) -- Do not panic and return correct errors for bad inputs in `GET_RANGE` (#2007, #2024) -- Physical child object removal by GC (#1699) -- Broadcasting helper objects (#1972) -- `neofs-cli lock object`'s `lifetime` flag handling (#1972) -- Do not move write-cache in read-only mode for flushing (#1906) -- Child object collection on CLI side with a bearer token (#2000) -- Fix concurrent map writes in `Object.Put` service (#2037) -- Malformed request errors' reasons in the responses (#2028) -- Session token's IAT and NBF checks in ACL service (#2028) -- Losing meta information on request forwarding (#2040) -- Assembly process triggered by a request with a bearer token (#2040) -- Losing locking context after metabase resync (#1502) -- Removing all trees by container ID if tree ID is empty in `pilorama.Forest.TreeDrop` (#1940) -- Concurrent mode changes in the metabase and blobstor (#2057) -- Panic in IR when performing HEAD requests (#2069) -- Write-cache flush duplication (#2074) -- Ignore error if a transaction already exists in a morph client (#2075) -- ObjectID signature output in the CLI (#2104) -- Pack arguments of `setPrice` invocation during contract update (#2078) -- `neofs-cli object hash` panic (#2079) -- Closing `neo-go` WS clients on shutdown and switch processes (#2080) -- Making notary deposits with a zero GAS balance (#2080) -- Notary requests on shutdown (#2075) -- `neofs-cli container create ` check the sufficiency of the number of nodes in the selector for replicas (#2038) -- Data duplication during request forwarding (#2047) -- Tree service panic on `TreeMove` operation (#2140) -- Panic in `GETRANGE` with zero length (#2095) -- Spawning useless `GETRANGE` with zero length for a big object (#2101) -- Incomplete object put errors do contain the deepest error's message (#2092) -- Prioritize internal addresses for clients (#2156) -- Force object removal via control service (#2145) -- Synchronizing a tree now longer reports an error for a single-node container (#2154) -- Prevent leaking goroutines in the tree service (#2162) -- Do not search for LOCK objects when delete container when session provided (#2152) -- Race conditions on shard's mode switch (#1956) -- Returning expired/removed objects from write-cache (#2016) - -### Removed -- `-g` option from `neofs-cli control ...` and `neofs-cli container create` commands (#2089) -- `--header` from `neofs-cli object get` (#2090) - -### Updated -- `neo-go` to `v0.100.0` -- `spf13/cobra` to `v1.6.1` -- `spf13/viper` to `v1.8.0` -- `google.golang.org/grpc` to `v1.50.1` - -### Updating from v0.34.0 -Pass CID and OID parameters via the `--cid` and `--oid` flags, not as the command arguments. - -Replicator pool size can now be fine-tuned with `replicator.pool_size` config field. -The default value is taken from `object.put.pool_size_remote` as in earlier versions. - -Added `neofs_node_object_*_req_count_success` metrics for tracking successfully executed requests. - -`neofs-cli container delete` command now requires given account or session issuer -to match the container owner. Use `--force` (`-f`) flag to bypass this requirement. - -Tree service network replication can now be fine-tuned with `tree.replication_timeout` config field. - -## [0.34.0] - 2022-10-31 - Marado (마라도, 馬羅島) - -### Added -- `--timeout` flag in `neofs-cli control` commands (#1917) -- Document shard modes of operation (#1909) -- `tree list` CLI command (#1332) -- `TreeService.GetTrees` RPC (#1902) -- All trees synchronization on bootstrap (#1902) -- `--force` flag to `neofs-cli control set-status` command (#1916) -- Logging `SessionService.Create` RPC on the server for debug (#1930) -- Debian packages can now be built with `make debpackage` (#409) - -### Changed -- Path to a metabase can now be reloaded with a SIGHUP (#1869) - -### Fixed -- `writecache.max_object_size` is now correctly handled (#1925) -- Correctly handle setting ONLINE netmap status after maintenance (#1922) -- Correctly reset shard errors in `ControlService.SetShardMode` RPC (#1931) -- Setting node's network state to `MAINTENANCE` while network settings forbid it (#1916) -- Do not panic during API client creation (#1936) -- Correctly sign new epoch transaction in neofs-adm for a committee of more than 4 nodes (#1949) -- Inability to provide session to NeoFS CLI in a NeoFS-binary format (#1933) -- `neofs-adm` now works correctly with a committee of more than 4 nodes (#1949, #1959) -- Closing a shard now waits until GC background workers stop (#1964) -- Make it possible to use `shard.ContainerSize` in read-only mode (#1975) -- Storage node now starts if at least one gRPC endpoint is available (#1893) -- Panic in API multy client (#1961) -- Blobstor object removal log messages (#1953) -- Missing object relatives in object removal session opened by NeoFS CLI (#1978) -- Bringing a node back online during maintenance (#1900) - -### Updated -- `neo-go` to `v0.99.4` -- `protoc` to `v3.21.7` -- `neofs-sdk` to `v1.0.0-rc.7` - -### Updating from v0.33.0 -Now storage node serves Control API `SetNemapStatus` request with `MAINTENANCE` -status only if the mode is allowed in the network settings. To force starting the local -maintenance on the node, provide `--force` flag to the `neofs-cli control set-status` -command. - -## [0.33.0] - 2022-10-17 - Anmado (안마도, 鞍馬島) - -### Added -- Serving `NetmapService.NetmapSnapshot` RPC (#1793) -- `netmap snapshot` command of NeoFS CLI (#1793) -- `apiclient.allow_external` config flag to fallback to node external addresses (#1817) -- Support `MAINTENANCE` state of the storage nodes (#1680, #1681) -- Changelog updates CI step (#1808) -- Validate storage node configuration before node startup (#1805) -- `neofs-node -check` command to check the configuration file (#1805) -- `flush-cache` control service command to flush write-cache (#1806) -- `wallet-address` flag in `neofs-adm morph refill-gas` command (#1820) -- Validate policy before container creation (#1704) -- `--timeout` flag in `neofs-cli` subcommands (#1837) -- `container nodes` command to output list of nodes for container, grouped by replica (#1704) -- Configuration flag to ignore shard in `neofs-node` (#1840) -- Add new RPC `TreeService.Healthcheck` -- Fallback to `GET` if `GET_RANGE` from one storage nodes to another is denied by basic ACL (#1884) -- List of shards and logger level runtime reconfiguration (#1770) -- `neofs-adm morph set-config` now supports well-known `MaintenanceModeAllowed` key (#1892) -- `add`, `get-by-path` and `add-by-path` tree service CLI commands (#1332) -- Tree synchronisation on startup (#1329) -- Morph client returns to the highest priority endpoint after the switch (#1615) - -### Changed -- Allow to evacuate shard data with `EvacuateShard` control RPC (#1800) -- Flush write-cache when moving shard to DEGRADED mode (#1825) -- Make `morph.cache_ttl` default value equal to morph block time (#1846) -- Policer marks nodes under maintenance as OK without requests (#1680) -- Unify help messages in CLI (#1854) -- `evacuate`, `set-mode` and `flush-cache` control subcommands now accept a list of shard ids (#1867) -- Reading `object` commands of NeoFS CLI don't open remote sessions (#1865) -- Use hex format to print storage node ID (#1765) - -### Fixed -- Description of command `netmap nodeinfo` (#1821) -- Proper status for object.Delete if session token is missing (#1697) -- Fail startup if metabase has an old version (#1809) -- Storage nodes could enter the network with any state (#1796) -- Missing check of new state value in `ControlService.SetNetmapStatus` (#1797) -- Correlation of object session to request (#1420) -- Do not increase error counter in `engine.Inhume` if shard is read-only (#1839) -- `control drop-objects` can remove split objects (#1830) -- Node's status in `neofs-cli netmap nodeinfo` command (#1833) -- Child check in object assembly process of `ObjectService.Get` handler (#1878) -- Shard ID in the object counter metrics (#1863) -- Metabase migration from the first version (#1860) - -### Removed -- Remove WIF and NEP2 support in `neofs-cli`'s --wallet flag (#1128) -- Remove --generate-key option in `neofs-cli container delete` (#1692) -- Serving `ControlService.NetmapSnapshot` RPC (#1793) -- `control netmap-snapshot` command of NeoFS CLI (#1793) - -### Updated - -- `neofs-contract` to `v0.16.0` -- `neofs-api-go` to `v2.14.0` - -### Updating from v0.32.0 -Replace using the `control netmap-snapshot` command with `netmap snapshot` one in NeoFS CLI. -Node can now specify additional addresses in `ExternalAddr` attribute. To allow a node to dial -other nodes external address, use `apiclient.allow_external` config setting. -Add `--force` option to skip placement validity check for container creation. - -Pass `maintenance` state to `neofs-cli control set-status` to enter maintenance mode. -If network allows maintenance state (*), it will be reflected in the network map. -Storage nodes under maintenance are not excluded from the network map, but don't -serve object operations. (*) can be fetched from network configuration via -`neofs-cli netmap netinfo` command. - -To allow maintenance mode during neofs-adm deployments, set -`network.maintenance_mode_allowed` parameter in config. - -When issuing an object session token for root (virtual, "big") objects, -additionally include all members of the split-chain. If session context -includes root object only, it is not spread to physical ("small") objects. - -`neofs-node` configuration now supports `mode: disabled` flag for a shard. -This can be used to temporarily ignore shards without completely removing them -from the config file. - -## [0.32.0] - 2022-09-14 - Pungdo (풍도, 楓島) - -### Added - -- Objects counter metric (#1712) -- `meta` subcommand to `neofs-lens` (#1714) -- Storage node metrics with global and per-shard object counters (#1658) -- Removal of trees on container removal (#1630) -- Logging new epoch events on storage node (#1763) -- Timeout for streaming RPC (#1746) -- `neofs-adm` is now able to dump hashes from a custom zone (#1748) -- Empty filename support in the Tree Service (#1698) -- Flag to `neofs-cli container list-objects` command for attribute printing (#1649) - -### Changed - -- `neofs-cli object put`'s object ID output has changed from "ID" to "OID" (#1296) -- `neofs-cli container set-eacl` command now pre-checks container ACL's extensibility (#1652) -- Access control in Tree service (#1628) -- Tree service doesn't restrict depth in `rpc GetSubTree` (#1753) -- `neofs-adm` registers contract hashes in both hex and string address formats (#1749) -- Container list cache synchronization with the Sidechain (#1632) -- Blobstor components are unified (#1584, #1686, #1523) - -### Fixed - -- Panic on write-cache's `Delete` operation (#1664) -- Payload duplication in `neofs-cli storagegroup put` (#1706) -- Contract calls in notary disabled environments (#1743) -- `Blobovnicza.Get` op now iterates over all size buckets (#1707) -- Object expiration time (#1670) -- Parser of the placement policy (#1775) -- Tree service timeout logs (#1759) -- Object flushing on writecache side (#1745) -- Active blobovniczas caching (#1691) -- `neofs-adm` TX waiting (#1738) -- `neofs-adm` registers contracts with a minimal GAS payment (#1683) -- Permissions of the file created by `neofs-cli` (#1719) -- `neofs-adm` creates TX with a high priority attribute (#1702) -- Storage node's restart after a hard reboot (#1647) - -### Updated - -- `neo-go` to `v0.99.2` -- `nspcc-dev/neofs-contract` to `v0.15.5` -- `prometheus/client_golang` to `v1.13.0` -- `google.golang.org/protobuf` to `v1.28.1` - -### Updating from v0.31.0 - -Storage Node now collects object count prometheus metrics: `neofs_node_object_counter`. - -Provide `--no-precheck` flag to `neofs-cli container set-eacl` for unconditional sending of a request -(previous default behavior). - -## [0.31.0] - 2022-08-04 - Baengnyeongdo (백령도, 白翎島) - -### Added - -- `neofs-adm` allows deploying arbitrary contracts (#1629) - -### Changed - -- Priority order in the Morph client (#1648) - -### Fixed - -- Losing request context in eACL response checks (#1595) -- Do not return expired objects that have not been handled by the GC yet (#1634) -- Setting CID field in `neofs-cli acl extended create` (#1650) -- `neofs-ir` no longer hangs if it cannot bind to the control endpoint (#1643) -- Do not require `lifetime` flag in `session create` CLI command (#1655) -- Using deprecated gRPC options (#1644) -- Increasing metabase error counter on disabled pilorama (#1642) -- Deadlock in the morph client related to synchronous notification handling (#1653) -- Slow metabase `COMMON_PREFIX` search for empty prefix (#1656) - -### Removed - -- Deprecated `profiler` and `metrics` configuration sections (#1654) - -### Updated - -- `chzyer/realine` to `v1.5.1` -- `google/uuid` to `v1.3.0` -- `nats-io/nats.go` to `v1.16.0` -- `prometheus/client_golang` to `v1.12.2` -- `spf13/cast` to `v1.5.0` -- `spf13/viper` to `v1.12.0` -- `go.uber.org/zap` to `v1.21.0` -- `google.golang.org/grpc` to `v1.48.0` - -### Updating from v0.30.0 -1. Change `morph.endpoint.client` priority values using the following rule: -the higher the priority the lower the value (non-specified or `0` values are -interpreted as the highest priority -- `1`). -2. Deprecated `profiler` and `metrics` configuration sections are dropped, -use `pprof` and `prometheus` instead. - -## [0.30.2] - 2022-08-01 - -### Added -- `EACL_NOT_FOUND` status code support (#1645). - -## [0.30.1] - 2022-07-29 - -### Fixed - -- `GetRange` operation now works correctly with objects stored in write-cache (#1638) -- Losing request context in eACL response checks (#1595) -- Wrong balance contract in innerring processor (#1636) -- `neofs-adm` now sets groups in manifest for all contracts properly (#1631) - -### Updated - -- `neo-go` to `v0.99.1` -- `neofs-sdk-go` to `v1.0.0-rc.6` - -## [0.30.0] - 2022-07-22 - Saengildo (생일도, 生日島) - -### Added - -- Profiler and metrics services now should be enabled with a separate flag -- Experimental support for the tree-service, disabled by default (#1607) -- Homomorphic hashes calculation can be disabled across the whole network (#1365) -- Improve `neofs-adm` auto-completion (#1594) - -### Changed - -- Require SG members to be unique (#1490) -- `neofs-cli` now doesn't remove container with LOCK objects without `--force` flag (#1500) -- LOCK objects are now required to have an expiration epoch (#1461) -- `morph` sections in IR and storage node configuration now accept an address and a priority of an endpoint (#1609) -- Morph client now retries connecting to the failed endpoint too (#1609) -- Redirecting `GET` and `GETRANGE` requests now does not store full object copy in memory (#1605) -- `neofs-adm` now registers candidates during initialization in a single transaction (#1608) - -### Fixed -- Invalid smart contract address in balance contract listener (#1636) - -- Shard now can start in degraded mode if the metabase is unavailable (#1559) -- Shard can now be disabled completely on init failure (#1559) -- Storage group members are now required to be unique (#1490) -- Print shard ID in component logs (#1611) - -### Updated -- `neofs-contract` to `v0.15.3` -- `neo-go` to the pre-release version -- `github.com/spf13/cobra` to v1.5.0 - -### Updating from v0.29.0 -1. Change morph endpoints from simple string to a pair of `address` and `priority`. The second can be omitted. -For inner ring node this resides in `morph.endpoint.client` section, -for storage node -- in `morph.rpc_endpoint` section. See `config/example` for an example. - -2. Move `storage.default` section to `storage.shard.default`. -3. Rename `metrics` and `profiler` sections to `prometheus` and `pprof` respectively, though old versions are supported. -In addition, these sections must now be explicitly enabled with `enabled: true` flag. - -## [0.29.0] - 2022-07-07 - Yeonpyeongdo (연평도, 延坪島) - -Support WalletConnect signature scheme. - -### Added -- Retrieve passwords for storage wallets from the configuration in neofs-adm (#1539) -- Metabase format versioning (#1483) -- `neofs-adm` generates wallets in a pretty JSON format -- `Makefile` supports building from sources without a git repo - -### Fixed -- Do not replicate object twice to the same node (#1410) -- Concurrent object handling by the Policer (#1411) -- Attaching API version to the forwarded requests (#1581) -- Node OOM panics on `GetRange` request with extremely huge range length (#1590) - -### Updated -- `neofs-sdk-go` to latest pre-release version -- `tzhash` to `v1.6.1` - -## [0.28.3] - 2022-06-08 - -### Updated -- Neo-go 0.98.3 => 0.99.0 (#1480) - -### Changed -- Replace pointers with raw structures in results for local storage (#1460) -- Move common CLI's functions in a separate package (#1452) - -### Fixed -- Confirmation of eACL tables by alphabet nodes when ACL extensibility is disabled (#1485) -- Do not use WS neo-go client in `neofs-adm` (#1378) -- Log more detailed network errors by the Alphabet (#1487) -- Fix container verification by the Alphabet (#1464) -- Include alphabet contracts to the base group in `neofs-adm` (#1489) - -## [0.28.2] - 2022-06-03 - -### Updated -- Neo-go 0.98.2 => 0.98.3 (#1430) -- NeoFS SDK v1.0.0-rc.3 => v1.0.0-rc.4 -- NeoFS API v2.12.1 => v2.12.2 -- NeoFS Contracts v0.14.2 => v0.15.1 - -### Added -- Config examples for Inner ring application (#1358) -- Command for documentation generation for `neofs-cli`, `neofs-adm` and `neofs-lens` (#1396) - -### Fixed -- Do not ask for contract wallet password twice (#1346) -- Do not update NNS group if the key is the same (#1375) -- Make LOCODE messages more descriptive (#1394) -- Basic income transfer's incorrect log message (#1374) -- Listen to subnet removal events in notary-enabled env (#1224) -- Update/remove nodes whose subnet has been removed (#1162) -- Potential removal of local object when policy isn't complied (#1335) -- Metabase `Select` is now slightly faster (#1433) -- Fix a number of bugs in writecache (#1462) -- Refactor eACL processing and fix bugs (#1471) -- Do not validate subnet removal by IR (#1441) -- Replace pointers with raw structures in parameters for local storage (#1418) - -#### Removed -- Remove `num` and `shard_num` parameters from the configuration (#1474) - -## [0.28.1] - 2022-05-05 - -### Fixed -- Loss of the connection scheme during address parsing in NeoFS CLI (#1351) - -## [0.28.0] - 2022-04-29 - Heuksando (흑산도, 黑山島) - -### Added - -- `morph dump-balances` command to NeoFS Adm (#1308) -- Ability to provide session token from file in NeoFS CLI (#1216) - -### Fixed - -- Panic in `netmap netinfo` command of NeoFS CLI (#1312) -- Container cache invalidation on DELETE op (#1313) -- Subscription to side-chain events in shards (#1321) -- Trusted object creation without session token (#1283) -- Storing invalid objects during trusted PUT op (#1286) -- RAM overhead when writing objects to local storage (#1343) - -### Changed - -- NeoFS Adm output from stderr to stdout (#1311) -- Node's object GC mechanism (#1318) - -### Updating from v0.28.0-rc.3 -Clean up all metabases and re-sync them using `resync_metabase` config flag. - -## [0.28.0-rc.3] - 2022-04-08 - -### Fixed -- Check expiration epoch of provided session token (#1168) -- Prevent corruption in `writecache.Head` (#1149) -- Use separate caches in N3 RPC multi client (#1213) -- `neofs-adm` fixes (#1288, #1294, #1295) -- Don't stop notification listener twice (#1291) -- Metabase panic (#1293) -- Disallow to tick block timer twice on the same height (#1208) - -### Added -- Persistent storage for session tokens (#1189) -- Cache for Inner Ring list fetcher (#1278) -- Degraded mode of storage engine (#1143) -- Command to change native Policy contract in `neofs-adm` (#1289) -- Single websocket endpoint pool for RPC and notifications (#1053) - -### Changed -- Cache NeoFS clients based only on public key (#1157) -- Make `blobovnicza.Put` idempotent (#1262) -- Optimize metabase list operations (#1262) -- PDP check ranges are now asked in random order (#1163) -- Update go version up to v1.17 (#1250) - -### Removed -- Reduced amount of slices with pointers (#1239) - -### Updating from v0.28.0-rc.2 -Remove `NEOFS_IR_MAINNET_ENDPOINT_NOTIFICATION`, -`NEOFS_IR_MORPH_ENDPOINT_NOTIFICATION`, and `NEOFS_MORPH_NOTIFICATION_ENDPOINT` -from Inner Ring and Storage configurations. - -Specify _WebSocket_ endpoints in `NEOFS_IR_MAINNET_ENDPOINT_CLIENT`, -`NEOFS_IR_MORPH_ENDPOINT_CLIENT`, and `NEOFS_MORPH_RPC_ENDPOINT` at Inner Ring -and Storage configurations. - -Specify path to persistent session token db in Storage configuration with -`NEOFS_NODE_PERSISTENT_SESSIONS_PATH`. - -## [0.28.0-rc.2] - 2022-03-24 - -### Fixed -- Respect format flags for `SplitInfo` output (#1233) -- Output errors in neofs-cli to stderr where possible (#1259) - -### Added -- Print details for AccessDenied errors in neofs-cli (#1252) -- Split client creation into 2 stages (#1244) -- Update morph client to work with v0.15.0 (#1253) - -## [0.28.0-rc.1] - 2022-03-18 - -Native RFC-6979 signatures of messages and tokens, LOCK object types, -experimental notifications over NATS with NeoFS API v2.12 support - -### Fixed -- Remove session tokens from local storage of storage node after expiration (#1133) -- Readme typos (#1167) -- LOCODE attribute and announced address are not mandatory for relay node config (#1114) -- Check session token verb (#1191) -- Fix data race leading to reputation data loss (#1210) - -### Added -- Look for `CustomGroup` scope in NNS contract before contract invocation (#749) -- Cache of notary transaction heights (#1151) -- NATS notifications (#1183) -- LOCK object type (#1175, #1176, #1181) -- Progress bar for object upload/download in neofs-cli (#1185) -- Support of new status codes (#1247) - -### Changed -- Update neofs-api-go and neofs-sdk-go (#1101, #1131, #1195, #1209, #1231) -- Use `path/filepath` package for OS path management (#1132) -- Shard sets mode to `read-only` if it hits threshold limit (#1118) -- Use request timeout in chain client of neofs-adm (#1115) -- Generate wallets with 0644 permissions in neofs-adm (#1115) -- Use cache of parsed addresses in GC (#1115) -- Determine config type based on file extension in neofs-ir (#1115) -- Reuse some error defined in contracts (#1115) -- Improved neofs-cli usability (#1103) -- Refactor v2 / SDK packages in eACL (#596) - -### Removed -- Remove some wrappers from `morph` package (#625) -- `GetRange` method in blobovnicza (#1115) -- Deprecated structures from SDK v1.0.0 rc (#1181) - -### Updating from neofs-node v0.27.5 -Set shard error threshold for read-only mode switch with -`NEOFS_STORAGE_SHARD_RO_ERROR_THRESHOLD` (default: 0, deactivated). - -Set NATS configuration for notifications in `NEOFS_NODE_NOTIFICATION` section. -See example config for more details. - -## [0.27.7] - 2022-03-30 - -### Fixed -- Shard ID is now consistent between restarts (#1204) - -### Added -- More N3 RPC caches in object service (#1278) - -## [0.27.6] - 2022-03-28 - -### Fixed -- Allow empty passwords in neofs-cli config (#1136) -- Set correct audit range hash type in neofs-ir (#1180) -- Read objects directly from blobstor in case of shard inconsistency (#1186) -- Fix `-w` flag in subnet commands of neofs-adm (#1223) -- Do not use explicit mutex lock in chain caches (#1236) -- Force gRPC server stop if it can't shut down gracefully in storage node (#1270) -- Return non-zero exit code in `acl extended create` command failures and fix - help message (#1259) - -### Added -- Interactive storage node configurator in neofs-adm (#1090) -- Logs in metabase operations (#1188) - -## [0.27.5] - 2022-01-31 - -### Fixed -- Flush small objects when persist write cache (#1088) -- Empty response body in object.Search request (#1098) -- Inner ring correctly checks session token in container.SetEACL request (#1110) -- Printing in verbose mode in CLI (#1120) -- Subnet removal event processing (#1123) - -### Added -- Password support in CLI config (#1103) -- Shard dump restore commands in CLI (#1085, #1086) -- `acl extended create` command in CLI (#1092) - -### Changed -- Adopt new `owner.ID` API from SDK (#1100) -- Use `go install` instead of `go get` in Makefile (#1102) -- Storage node returns Fixed12 decimal on accounting.Balance request. CLI - prints Fixed8 rounded value by default. (#1084) -- Support new update interface for NNS contract in NeoFS Adm (#1091) -- Rename `use_write_cache` to `writecache.enabled` in stoarge config (#1117) -- Preallocate slice in `headersFromObject` (#1115) -- Unify collection of expired objects (#1115) -- Calculate blobovnicza size at initialization properly (#1115) -- Process fast search filters outside bbolt transaction (#1115) -- Update TZHash library to v1.5.1 - -### Removed -- `--wif` and `--binary-key` keys from CLI (#1083) -- Extended ACL validator moved to SDK library (#1096) -- `--generate-key` flag in CLI control commands (#1103) -- Various unused code (#1123) - -### Upgrading from v0.27.4 -Use `--wallet` key in CLI to provide WIF or binary key file instead of `--wif` -and `--binary-key`. - -Replace `NEOFS_STORAGE_SHARD_N_USE_WRITE_CACHE` with -`NEOFS_STORAGE_SHARD_N_WRITECACHE_ENABLED` in Storage node config. - -Specify `password: xxx` in config file for NeoFS CLI to avoid password input. - -## [0.27.4] - 2022-01-13 - -### Fixed -- ACL check did not produce status code (#1062) -- Asset transfer wrapper used incorrect receiver (#1069) -- Empty search response missed meta header and body (#1063) -- IR node in single chain environment used incorrect source of IR list (#1025) -- Incorrect message sequence in object.Range request (#1077) - -### Added -- Option to disable compression of object based on their content-type attribute - (#1060) - -### Changed -- Factor out autocomplete command in CLI and Adm (#1041) -- Single crypto rand source (#851) - -### Upgrading from v0.27.3 -To disable compression for object with specific content-types, specify them -as a string array in blobstor section: -`NEOFS_STORAGE_SHARD_N_BLOBSTOR_COMPRESSION_EXCLUDE_CONTENT_TYPES`. Use -asterisk as wildcard, e.g. `video/*`. - -## [0.27.3] - 2021-12-30 - -### Added -- `SetShardMode` RPC in control API, available in CLI (#1044) -- Support of basic ACL constants without final flag in CLI (#1066) - -### Changed -- `neofs-adm` updates contracts in single tx (#1035) -- Proxy contract arguments for deployment in `neofs-adm` (#1056) - -## [0.27.2] - 2021-12-28 - -### Fixed -- Goroutine leak due to infinite response message await ([neofs-api-go#366](https://github.com/nspcc-dev/neofs-api-go/pull/366)) -- Inconsistency in placement function ([neofs-sdk-go#108](https://github.com/nspcc-dev/neofs-sdk-go/pull/108)) - -### Added -- `ListShards` RPC in control API, available in CLI (#1043) -- Epoch metric in Storage and Inner Ring applications (#1054) - -### Changed -- Some object replication related logs were moved to DEBUG level (#1052) - -## [0.27.1] - 2021-12-20 - -### Fixed -- Big objects now flushed from WriteCache after write (#1028) -- WriteCache big object counter (#1022) -- Panic in the container estimation routing (#1016) -- Shutdown freeze in policer component (#1047) - -### Added -- Shorthand `-g` for `--generate-key` in NeoFS CLI (#1034) -- Autocomplete generator command for neofs-adm (#1013) -- Max connection per host config value for neo-go client (#780) -- Sanity check of session token context in container service (#1045) - -### Changed -- CLI now checks NeoFS status code for exit code (#1039) -- New `Update` method signature for NNS contract in neofs-adm (#1038) - -## [0.27.0] - 2021-12-09 - Sinjido (신지도, 薪智島) - -NeoFS API v2.11.0 support with response status codes and storage subnetworks. - -### Fixed -- CLI now opens LOCODE database in read-only mode for listing command (#958) -- Tombstone owner now is always set (#842) -- Node in relay mode does not require shard config anymore (#969) -- Alphabet nodes now ignore notary notifications with non-HALT main tx (#976) -- neofs-adm now prints version of NNS contract (#1014) -- Possible NPE in blobovnicza (#1007) -- More precise calculation of blobovnicza size (#915) - -### Added -- Maintenance mode for Storage node (#922) -- Float values in Storage node config (#903) -- Status codes for NeoFS API Response messages (#961) -- Subnetwork support (#977, #973, #983, #974, #982, #979, #998, #995, #1001, #1004) -- Customized fee for named container registration (#1008) - -### Changed -- Alphabet contract number is not mandatory (#880) -- Alphabet nodes resign `AddPeer` request if it updates Storage node info (#938) -- All applications now use client from neofs-sdk-go library (#966) -- Some shard configuration records were renamed, see upgrading section (#859) -- `Nonce` and `VUB` values of notary transactions generated from notification - hash (#844) -- Non alphabet notary invocations now have 4 witnesses (#975) -- Object replication is now async and continuous (#965) -- NeoFS ADM updated for the neofs-contract v0.13.0 deploy (#984) -- Minimal TLS version is set to v1.2 (#878) -- Alphabet nodes now invoke `netmap.Register` to add node to the network map - candidates in notary enabled environment (#1008) - -### Upgrading from v0.26.1 -`NEOFS_IR_CONTRACTS_ALPHABET_AMOUNT` is not mandatory env anymore. If it -is not set, Inner Ring would try to read maximum from config and NNS contract. -However, that parameter still can be set in order to require the exact number -of contracts. - -Shard configuration records were renamed: -- `refill_metabase` -> `resync_metabase` -- `writecache.max_size` -> `writecache.max_object_size` -- `writecache.mem_size` -> `writecache.memcache_capacity` -- `writecache.size_limit` -> `writecache_capcity` -- `blobstor.blobovnicza.opened_cache_size` -> `blobstor.blobovnicza.opened_cache_capacity` -- `*.shallow_depth` -> `*.depth` -- `*.shallow_width` -> `*.width` -- `*.small_size_limit` -> `*.small_object_size` - -Specify storage subnetworks in `NEOFS_NODE_SUBNET_ENTRIES` as the list of -integer numbers. To exit default subnet, use `NEOFS_NODE_SUBNET_EXIT_ZERO=true` - -Specify fee for named container registration in notary disabled environment -with `NEOFS_IR_FEE_NAMED_CONTAINER_REGISTER`. - -## [0.26.1] - 2021-11-02 - -### Fixed -- Storage Node handles requests before its initialization is finished (#934) -- Release worker pools gracefully (#901) -- Metabase ignored containers of storage group and tombstone objects - in listing (#945) -- CLI missed endpoint flag in `control netmap-snapshot` command (#942) -- Write cache object persisting (#866) - -### Added -- Quote symbol support in `.env` example tests (#935) -- FSTree object counter (#821) -- neofs-adm prints contract version in `dump-hashes` command (#940) -- Default values section in shard configuration (#877) -- neofs-adm downloads contracts directly from GitHub (#733) - -### Changed -- Use FSTree counter in write cache (#821) -- Calculate notary deposit `till` parameter depending on available - deposit (#910) -- Storage node returns session token error if attached token's private key - is not available (#943) -- Refactor of NeoFS API client in inner ring (#946) -- LOCODE generator tries to find the closest continent if there are - no exact match (#955) - -### Upgrading from v0.26.0 -You can specify default section in storage engine configuration. -See [example](./config/example/node.yaml) for more details. - -## [0.26.0] - 2021-10-19 - Udo (우도, 牛島) - -NeoFS API v2.10 support - -### Fixed -- Check remote node public key in every response message (#645) -- Do not lose local container size estimations (#872) -- Compressed and uncompressed objects are always available for reading - regardless of compression configuration (#868) -- Use request session token in ACL check of object.Put (#881) -- Parse URI in neofs-cli properly (#883) -- Parse minutes in LOCODE DB properly (#902) -- Remove expired tombstones (#884) -- Close all opened blobovniczas properly (#896) -- Do not accept objects with empty OwnerID field (#841) - -### Added -- More logs in governance and policer components (#867, #882) -- Contract address getter in static blockchain clients (#627) -- Alphabet configuration option to disable governance sync (#869) -- neofs-lens app implementation (#791) -- Detailed comments in neofs-node config example (#858) -- Size suffixes support in neofs-node config (#857) -- Docs for neofs-adm (#906) -- Side chain block size duration and global NeoFS configuration in - NetworkConfig response (#833) -- Support native container names (#889) - -### Changed -- Updated grpc to v1.41.0 (#860) -- Updated neo-go to v0.97.3 (#833) -- Updated neofs-api-go to v1.30.0 -- Adopt neofs-adm for new contracts release (#835, #888) -- Adopt neofs-node for new contracts release (#905) -- SN and IR notary deposits are made dynamically depending on the Notary and - GAS balances (#771) -- VMagent port in testnet config is now 443 (#908) -- Use per-shard worker pools for object.Put operations (#674) -- Renamed `--rpc-endpoint` CLI flag for `control command` to `--endpoint` (#879) - -### Removed -- Global flags in CLI. Deleted useless flags from `accounting balance` - command (#810). -- Interactive mode in docker run command (#916) - -### Upgrading from v0.25.1 -Deleted `NEOFS_IR_NOTARY_SIDE_DEPOSIT_AMOUNT`, `NEOFS_IR_NOTARY_MAIN_DEPOSIT_AMOUNT` -and `NEOFS_IR_TIMERS_SIDE_NOTARY`, `NEOFS_IR_TIMERS_MAIN_NOTARY` Inner Ring envs. -Deleted `NEOFS_MORPH_NOTARY_DEPOSIT_AMOUNT` and `NEOFS_MORPH_NOTARY_DEPOSIT_DURATION` -Storage Node envs. -`control` CLI command does not have `--rpc-endpoint`/`r` flag, use `endpoint` -instead. - -## [0.25.1] - 2021-09-29 - -### Fixed -- Panic caused by missing Neo RPC endpoints in storage node's config (#863) - -### Added -- Support of multiple Neo RPC endpoints in Inner Ring node (#792) - -`mainchain` section of storage node config is left unused by the application. - -## [0.25.0] - 2021-09-27 - Mungapdo (문갑도, 文甲島) - -### Fixed -- Work of a storage node with one Neo RPC endpoint instead of a list (#746) -- Lack of support for HEAD operation on the object write cache (#762) -- Storage node attribute parsing is stable now (#787) -- Inner Ring node now logs transaction hashes of Deposit and Withdraw events - in LittleEndian encoding (#794) -- Storage node uses public keys of the remote nodes in placement traverser - checks (#645) -- Extended ACL `Target` check of role and public keys is mutual exclusive now - (#816) -- neofs-adm supports update and deploy of neofs-contract v0.11.0 (#834, #836) -- Possible NPE in public key conversion (#848) -- Object assembly routine do not forward existing request instead of creating - new one (#839) -- Shard now returns only physical stored objects for replication (#840) - -### Added -- Support events from P2P notary pool -- Smart contract address auto negotiation with NNS contract (#736) -- Detailed logs for all data writing operations in storage engine (#790) -- Docker build and release targets in Makefile (#785) -- Metabase restore option in the shard config (#789) -- Write cache used size limit in bytes (#776) - -### Changed -- Reduce container creation delay via listening P2P notary pool (#519) -- Extended ACL table is not limited to 1KiB (#731) -- Netmap side chain client wrapper now has `TryNotary` option (#793) -- Sticky bit is ignored in requests with `SYSTEM` role (#818) -- Incomplete object put error now contains last RPC error (#778) -- Container service invalidates container cache on writing operations (#803) -- Improved write cache size counters (#776) -- Metabase returns `NotFound` error instead of `AlreadyRemoved` on GCMarked - objects (#840) -- Object service uses two routine pools for remote and local GET requests (#845) - -### Removed -- Dockerfile for AllInOne image moved to a separate repository (#796) - -### Upgrading from v0.24.1 -Added `NEOFS_CONTRACTS_PROXY` env for Storage Node; mandatory in -notary enabled environments only. It should contain proxy contract's -scripthash in side chain. - -Added `NEOFS_MORPH_NOTARY_DEPOSIT_AMOUNT` and -`NEOFS_MORPH_NOTARY_DEPOSIT_DURATION` envs for Storage Node, that -have default values, not required. They should contain notary deposit -amount and frequency(in blocks) respectively. - -All side chain contract address config values are optional. If side chain -contract address is not specified, then value gathered from NNS contract. - -Added `NEOFS_STORAGE_SHARD__WRITECACHE_SIZE_LIMIT` where `` is shard ID. -This is the size limit for the all write cache storages combined in bytes. Default -size limit is 1 GiB. - -Added `NEOFS_STORAGE_SHARD__REFILL_METABASE` bool flag where `` is shard -ID. This flag purges metabase instance at the application start and reinitialize -it with available objects from the blobstor. - -Object service pool size now split into `NEOFS_OBJECT_PUT_POOL_SIZE_REMOTE` and -`NEOFS_OBJECT_PUT_POOL_SIZE_LOCAL` configuration records. - -## [0.24.1] - 2021-09-07 - -### Fixed -- Storage and Inner Ring will not start until Neo RPC node will have the height -of the latest processed block by the nodes (#795) - -### Upgrading from v0.24.0 -Specify path to the local state DB in Inner Ring node config with -`NEOFS_IR_NODE_PERSISTENT_STATE_PATH`. Specify path to the local state DB in -Storage node config with `NEOFS_NODE_PERSISTENT_STATE_PATH`. - -## [0.24.0] - 2021-08-30 Anmyeondo (안면도, 安眠島) - -### Fixed -- Linter warning messages (#766) -- Storage Node does not register itself in network in relay mode now (#761) - -### Changed -- `neofs-adm` fails when is called in a notary-disabled environment (#757) -- `neofs-adm` uses `neo-go` client's native NNS resolving method instead of the custom one (#756) -- Node selects pseudo-random list of objects from metabase for replication (#715) - -### Added -- Contract update support in `neofs-adm` utility (#748) -- Container transferring support in `neofs-adm` utility (#755) -- Storage Node's balance refilling support in `neofs-adm` utility (#758) -- Support `COMMON_PREFIX` filter for object attributes in storage engine and `neofs-cli` (#760) -- Node's and IR's notary status debug message on startup (#758) -- Go `1.17` unit tests in CI (#766) -- Supporting all eACL filter fields from the specification (#768) -- Cache for Container service's read operations (#676) - -### Updated -- `neofs-api-go` library to `v1.29.0` - -### Removed -- Unused `DB_SIZE` parameter of writecache (#773) - -### Upgrading from v0.23.1 -Storage Node does not read unused `NEOFS_STORAGE_SHARD_XXX_WRITECACHE_DB_SIZE` -config parameter anymore. - -## [0.23.1] - 2021-08-06 - -N3 Mainnet launch release with minor fixes. - -### Added -- Initial version of `neofs-adm` tool for fast side chain deployment and - management in private installations -- Notary support auto negotiation (#709) -- Option to disable side chain cache in Storage node (#704) -- Escape symbols in Storage node attributes (#700) - -### Changed -- Updated neo-go to v0.97.1 -- Updated multiaddr lib to v0.4.0 with native TLS protocol support (#661) -- Default file permission in storage engine is 660 (#646) - -### Fixed -- Container size estimation routine now aggregates values by cid-epoch tuple - (#723) -- Storage engine always creates executable dirs (#646) -- GC routines in storage engine shards shutdown gracefully (#745) -- Handle context shutdown at NeoFS multi client group address switching (#737) -- Scope for main chain invocations from Inner Ring nodes (#751) - -### Upgrading from v0.23.0 -Added `NEOFS_MORPH_DISABLE_CACHE` env. If `true`, none of -the `eACL`/`netmap`/`container` RPC responses cached. - -Remove `WITHOUT_NOTARY` and `WITHOUT_MAIN_NOTARY` records from Inner Ring node -config. Notary support is now auto negotiated. - -## [0.23.0] - 2021-07-23 - Wando (완도, 莞島) - -Improved stability for notary disabled environment. - -### Added -- Alphabet wallets generation command in neofs-adm (#684) -- Initial epoch timer tick synchronization at Inner Ring node startup (#679) - -### Changed -- `--address` flag is optional in NeoFS CLI (#656) -- Notary subsystem now logs `ValidUntilBlock` (#677) -- Updated neo-go to v0.96.1 -- Storage Node configuration example contains usable parameters (#699) - -### Fixed -- Do not use side chain RoleManagement contract as source of Inner Ring list - when notary disabled in side chain (#672) -- Alphabet list transition is even more effective (#697) -- Inner Ring node does not require proxy and processing contracts if notary - disabled (#701, #714) - -### Upgrading from v0.22.3 -To upgrade Storage node or Inner Ring node from v0.22.3, you don't need to -change configuration files. Make sure, that NEO RPC nodes, specified in config, -are connected to N3 RC4 (Testnet) network. - -## [0.22.3] - 2021-07-13 - -### Added -- Support binary eACL format in container CLI command ([#650](https://github.com/nspcc-dev/neofs-node/issues/650)). -- Dockerfile for neofs-adm utility ([#680](https://github.com/nspcc-dev/neofs-node/pull/680)). - -### Changed -- All docker files moved to `.docker` dir ([#682](https://github.com/nspcc-dev/neofs-node/pull/682)). - -### Fixed -- Do not require MainNet attributes in "Without MainNet" mode ([#663](https://github.com/nspcc-dev/neofs-node/issues/663)). -- Stable alphabet list merge in Inner Ring governance ([#670](https://github.com/nspcc-dev/neofs-node/issues/670)). -- User can specify only wallet section without node key ([#690](https://github.com/nspcc-dev/neofs-node/pull/690)). -- Log keys in hex format in reputation errors ([#693](https://github.com/nspcc-dev/neofs-node/pull/693)). -- Connections leak and reduced amount of connection overall ([#692](https://github.com/nspcc-dev/neofs-node/issues/692)). - -### Removed -- Debug output of public key in Inner Ring log ([#689](https://github.com/nspcc-dev/neofs-node/pull/689)). - -## [0.22.2] - 2021-07-07 - -Updated broken version of NeoFS API Go. - -### Updated -- NeoFS API Go: [v1.28.3](https://github.com/nspcc-dev/neofs-api-go/releases/tag/v1.28.3). - -## [0.22.1] - 2021-07-07 - -### Added -- `GetCandidates` method to morph client wrapper ([#647](https://github.com/nspcc-dev/neofs-node/pull/647)). -- All-in-One Docker image that contains all NeoFS related binaries ([#662](https://github.com/nspcc-dev/neofs-node/pull/662)). -- `--version` flag to Storage Node binary ([#664](https://github.com/nspcc-dev/neofs-node/issues/664)). - -### Changed -- Do not check NeoFS version in `LocalNodeInfo` requests and `Put` container operations; `v2.7.0` is genesis version of NeoFS ([#660](https://github.com/nspcc-dev/neofs-node/pull/660)). -- All error calls of CLI return `1` exit code ([#657](https://github.com/nspcc-dev/neofs-node/issues/657)). - -### Fixed -- Do not use multisignature for audit operations ([#658](https://github.com/nspcc-dev/neofs-node/pull/658)). -- Skip audit for containers without Storage Groups ([#659](https://github.com/nspcc-dev/neofs-node/issues/659)). - -### Updated -- NeoFS API Go: [v1.28.2](https://github.com/nspcc-dev/neofs-api-go/releases/tag/v1.28.2). - -## [0.22.0] - 2021-06-29 - Muuido (무의도, 舞衣島) - -Storage nodes with a group of network endpoints. - -### Added -- Support of Neo wallet credentials in CLI ([#610](https://github.com/nspcc-dev/neofs-node/issues/610)). -- More reliable approval of trust value by IR ([#500](https://github.com/nspcc-dev/neofs-node/issues/500)). -- Storage node's ability to announce and serve on multiple network addresses ([#607](https://github.com/nspcc-dev/neofs-node/issues/607)). -- Validation of network addresses of netmap candidates in IR ([#557](https://github.com/nspcc-dev/neofs-node/issues/557)). -- Control service with healthcheck RPC in IR and CLI support ([#414](https://github.com/nspcc-dev/neofs-node/issues/414)). - -### Fixed -- Approval of objects with with duplicate attribute keys or empty values ([#633](https://github.com/nspcc-dev/neofs-node/issues/633)). -- Approval of containers with with duplicate attribute keys or empty values ([#634](https://github.com/nspcc-dev/neofs-node/issues/634)). -- Default path for CLI config ([#626](https://github.com/nspcc-dev/neofs-node/issues/626)). - -### Changed -- `version` command replaced with `--version` flag in CLI ([#571](https://github.com/nspcc-dev/neofs-node/issues/571)). -- Command usage text is not printed on errors in CLI ([#623](https://github.com/nspcc-dev/neofs-node/issues/623)). -- `netmap snapshot` command replaced with `control netmap-snapshot` one in CLI ([#651](https://github.com/nspcc-dev/neofs-node/issues/651)). -- IR does not include nodes with LOCODE derived attributes to the network map ([#412](https://github.com/nspcc-dev/neofs-node/issues/412)). -- IR uses morph/client packages for contract invocations ([#496](https://github.com/nspcc-dev/neofs-node/issues/496)). -- Writecache decreases local size when objects are flushed ([#568](https://github.com/nspcc-dev/neofs-node/issues/568)). -- IR can override global configuration values only in debug build ([#363](https://github.com/nspcc-dev/neofs-node/issues/363)). - -### Updated -- Neo Go: [v0.95.3](https://github.com/nspcc-dev/neo-go/releases/tag/v0.95.3). -- NeoFS API Go: [v1.28.0](https://github.com/nspcc-dev/neofs-api-go/releases/tag/v1.28.0). -- protobuf: [v1.26.0](https://github.com/protocolbuffers/protobuf-go/releases/tag/v1.26.0). -- uuid: [v1.2.0](https://github.com/google/uuid/releases/tag/v1.2.0). -- compress: [v1.13.1](https://github.com/klauspost/compress/releases/tag/v1.13.1). -- base58: [v1.2.0](https://github.com/mr-tron/base58/releases/tag/v1.2.0). -- multiaddr: [v0.3.2](https://github.com/multiformats/go-multiaddr/releases/tag/v0.3.2). -- ants: [v2.4.0](https://github.com/panjf2000/ants/releases/tag/v2.4.0). -- orb: [v0.2.2](https://github.com/paulmach/orb/releases/tag/v0.2.2). -- prometheus: [v1.11.0](https://github.com/prometheus/client_golang/releases/tag/v1.11.0). -- testify: [v1.7.0](https://github.com/stretchr/testify/releases/tag/v1.7.0). -- atomic: [v1.8.0](https://github.com/uber-go/atomic/releases/tag/v1.8.0). -- zap: [v1.17.0](https://github.com/uber-go/zap/releases/tag/v1.17.0). -- grpc: [v1.38.0](https://github.com/grpc/grpc-go/releases/tag/v1.38.0). -- cast: [v1.3.1](https://github.com/spf13/cast/releases/tag/v1.3.1). -- cobra: [1.1.3](https://github.com/spf13/cobra/releases/tag/v1.1.3). -- viper: [v1.8.1](https://github.com/spf13/viper/releases/tag/v1.8.1). - -## [0.21.1] - 2021-06-10 - -### Fixed -- Session token lifetime check (#589). -- Payload size check on the relayed objects (#580). - -### Added -- VMagent to collect metrics from testnet storage image - -### Changed -- Updated neofs-api-go to v1.27.1 release. - -## [0.21.0] - 2021-06-03 - Seongmodo (석모도, 席毛島) - -Session token support in container service, refactored config in storage node, -TLS support on gRPC servers. - -### Fixed -- ACL service traverses over all RequestMetaHeader chain to find - bearer and session tokens (#548). -- Object service correctly resends complete objects without attached - session token (#501). -- Inner ring processes `neofs.Bind` and `neofs.Unbind` notifications (#556). -- Client cache now gracefully closes all available connections (#567). - -### Added -- Session token support in container service for `container.Put`, - `container.Delete` and `container.SetEACL` operations. -- Session token support in container and sign command of NeoFS CLI. -- TLS encryption support of gRPC service in storage node. - -### Changed -- Inner ring listens RoleManagement contract notifications to start governance - update earlier. -- Inner ring processes extended ACL changes. -- Inner ring makes signature checks of containers and extended ACLs. -- Refactored config of storage node. -- Static clients from `morph/client` do not process notary invocations - explicitly anymore. Now notary support specified at static client creation. -- Updated neo-go to v0.95.1 release. -- Updated neofs-api-go to v1.27.0 release. - -### Removed -- Container policy parser moved to neofs-sdk-go repository. -- Invoke package from inner ring. - -## [0.20.0] - 2021-05-21 - Dolsando (돌산도, 突山島) - -NeoFS is N3 RC2 compatible. - -### Fixed -- Calculations in EigenTrust algorithm (#527). -- NPE at object service request forwarding (#532, #543, #544). -- FSTree iterations in blobstor (#541). -- Inhume operation in storage engine (#546). - -### Added -- Optional endpoint to main chain in storage app. -- Client for NeoFSID contract. - -### Changed -- Reorganized and removed plenty of application configuration records - (#510, #511, #512, #514). -- Nodes do not resolve remote addresses manually. -- Presets for basic ACL in CLI are `private` ,`public-read` and - `public-read-write` now. -- Updated neo-go to v0.95.0 release. -- Updated neofs-api-go to v1.26.1 release. -- Updated go-multiaddr to v0.3.1 release. - -### Removed -- Unused external GC workers (GC is part of the shard in storage engine now). -- Unused worker pools for object service in storage app. -- `pkg/errors` dependency (stdlib errors used instead). - -## [0.19.0] - 2021-05-07 - Daecheongdo (대청도, 大靑島) - -Storage nodes exchange, calculate, aggregate and store reputation information -in reputation contract. Inner ring nodes support workflows with and without -notary subsystem in chains. - -### Fixed -- Build with go1.16. -- Notary deposits last more blocks. -- TX hashes now prints in little endian in logs. -- Metabase deletes graves regardless of the presence of objects. -- SplitInfo error created from all shards instead of first matched shard. -- Possible deadlock at cache eviction in blobovnicza. -- Storage node does not send rebootstrap messages after it went offline. - -### Added -- Reputation subsystem that includes reputation collection, exchange, -calculation and storage components. -- Notary and non notary workflows in inner ring. -- Audit fee transfer for inner ring nodes that performed audit. -- Unified encoding for all side chain payment details. -- New write cache implementation for storage engine. -- NEP-2 and NEP-6 key formats in CLI. - -### Changed -- Metabase puts data in batches. -- Network related new epoch handlers in storage node executed asynchronously. -- Storage node gets epoch duration from global config. -- Storage node resign and resend Search, Range, Head, Get requests of object -service without modification. -- Inner ring does not sync side chain validators in single chain deployment. -- neo-go updated to v0.94.1 -- neofs-api-go updated to v1.26.0 - -## [0.18.0] - 2021-03-26 - Yeongheungdo (영흥도, 靈興島) - -NeoFS operates with updated governance model. Alphabet keys and inner ring keys -are accessed from side chain committee and `RoleManagement` contract. Each epoch -alphabet keys are synchronized with main chain. - -### Fixed -- Metabase does not store object payloads anymore. -- TTLNetCache now always evict data after a timeout. -- NeoFS CLI keyer could misinterpret hex value as base58. - -### Added -- Local trust controller in storage node. -- Governance processor in inner ring that synchronizes list of alphabet keys. - -### Changed -- Inner ring keys and alphabet keys are managed separately by inner ring and - gathered from committee and `RoleManagement` contract. - -## [0.17.0] - 2021-03-22 - Jebudo (제부도, 濟扶島) - -Notary contract support, updated neofs-api-go with raw client, some performance -tweaks with extra caches and enhanced metrics. - -### Added -- Notary contract support. -- Cache for morph client. -- Metrics for object service and storage engine. -- Makefile target for fast and dirty docker images. -- GAS threshold value in inner ring GAS transfers. - -### Changed -- RPC client cache now re-used per address instead of (address+key) tuple. -- Updated neofs-api-go version to v1.25.0 with raw client support. -- Updated neo-go to testnet compatible v0.94.0 version. - -## [0.16.0] - 2021-02-26 - Ganghwado (강화도, 江華島) - -Garbage collector is now running inside storage engine. It is accessed -via Control API, from `policer` component and through object expiration -scrubbers. - -Inner ring configuration now supports single chain mode with any number of -alphabet contracts. - -Storage node now supports NetworkInfo method in netmap service. - -### Fixed -- Storage engine now inhumes object only in single shard. -- Metabase correctly removes parent data at batched children delete. -- Metabase does not accept tombstone on tombstone records in graveyard anymore. -- Object service now rejects expired objects. -- CLI now correctly attaches bearer token in storage group operations. -- Container policy parser now works with strings in filter key. -- Policer component now removes redundant objects locally. - -### Added -- GC job that monitors expired objects. -- GC job that removes marked objects from physical storage. -- Batch inhume operations in metabase, shard and engine. -- `control.DropObjects` RPC method. -- Support of `netmap.NetworkInfo` RPC method. -- Single chain inner ring configuration. - -### Changed -- `UN-LOCODE` node attribute now optional. -- `engine.Delete` method now marks object to be removed by GC. -- Inner ring node supports any number of alphabet contracts from 1 up to 40. - -## [0.15.0] - 2021-02-12 - Seonyudo (선유도, 仙遊島) - -NeoFS nodes are now preview5-compatible. - -IR nodes are now engaged in the distribution of funds to the storage nodes: -for the passed audit and for the amount of stored information. All timers -of the IR nodes related to the generation and processing of global system -events are decoupled from astronomical time, and are measured in the number -of blockchain blocks. - -For the geographic positioning of storage nodes, a global NeoFS location -database is now used, the key in which is a UN/LOCODE, and the base itself -is generated on the basis of the UN/LOCODE and OpenFlights databases. - -### Added -- Timers with time in blocks of the chain. -- Subscriptions to new blocks in blockchain event `Listener`. -- Tracking the volume of stored information by containers in the - storage engine and an external interface for obtaining this data. -- `TransferX` operation in sidechain client. -- Calculators of audit and basic settlements. -- Distribution of funds to storage nodes for audit and for the amount - of stored information (settlement processors of IR). -- NeoFS API `Container.AnnounceUsedSpace` RPC service. -- Exchange of information about container volumes between storage nodes - controlled by IR through sidechain notifications. -- Support of new search matchers (`STRING_NOT_EQUAL`, `NOT_PRESENT`). -- Functional for the formation of NeoFS location database. -- CLI commands for generating and reading the location database. -- Checking the locode attribute and generating geographic attributes - for candidates for a network map on IR side. -- Verification of the eACL signature when checking Object ACL rules. - -### Fixed -- Overwriting the local configuration of node attributes when updating - the network map. -- Ignoring the X-headers CLI `storagegroup` commands. -- Inability to attach bearer token in CLI `storagegroup` commands. - -### Changed -- Units of epoch and emit IR intervals. -- Query language in CLI `object search` command. - -### Updated -- neo-go v0.93.0. -- neofs-api-go v1.23.0. - -## [0.14.3] - 2021-01-27 - -### Fixed -- Upload of objects bigger than single gRPC message. -- Inconsistent placement issues (#347, #349). -- Bug when ACL request classifier failed to classify `RoleOthers` in - first epoch. - -### Added -- Debug section in readme file for testnet configuration. - -### Changed -- Docker images now based on alpine and contain shell. -- Node bootstraps with active state in node info structure. - -## [0.14.2] - 2021-01-20 - -Testnet4 related bugfixes. - -### Fixed -- Default values for blobovnicza object size limit and blobstor small object - size are not zero. -- Various storage engine log messages. -- Bug when inner ring node ignored bootstrap messages from restarted storage - nodes. - -### Added -- Timeout for reading boltDB files at storage node initialization. - -### Changed -- Increased default extra GAS fee for contract invocations at inner ring. - -## [0.14.1] - 2021-01-15 - -### Fixed - -- Inner ring node could not confirm `netmap.updateState` notification. -- `object.RangeHash` method ignored salt values. - -### Added - -- Control API service for storage node with health check, netmap and node state - relate methods. -- Object service now looks to previous epoch containers. -- Possibility to configure up multiple NEO RPC endpoints in storage node. - -### Changed - -- Storage node shuts down if event producer RPC node is down. - -## [0.14.0] - 2020-12-30 - Yeouido (여의도, 汝矣島) - -Preview4 compatible NeoFS nodes with data audit. - -### Added -- Data audit routines in inner ring nodes. -- Storage group operations in CLI (`neofs-cli storagegroup --help`). - -### Fixed -- Loss of request X-headers during the forwarding in Object service. - -### Changed -- Updated neo-go version for preview4 compatibility. - -### Updated -- neo-go v0.92.0. -- neofs-api-go v1.22.0. - -## [0.13.2] - 2020-12-24 - -Support changes from neofs-api-go v1.21.2 release. - -### Added - -- Support of request X-Headers in CLI commands. - -### Changed - -- Use updated API of container library. - -## [0.13.1] - 2020-12-18 - -Fixes based on Modo release testing results. - -### Added - -- Verification of chain element addresses during object assembling. - -### Changed - -- Processing of filters by non-address fields in Object Range/RangeHash/Delete. - -### Fixed - -- `Graveyard` and `ToMoveIt` bucket names in metabase. -- Double formation of the parent title when transforming an object. -- Loss of session token during Object Put. -- Potential generating Range requests inside Get request execution context. - -## [0.13.0] - 2020-12-15 - Modo (모도, 茅島) - -Implementation of a local object storage engine. -Adaptation of the object service work scheme for the engine. - -### Changed - -- Object format after transformations. -- Handling of object operations. - -### Added - -- Local storage components: `Engine`, `Shard`, `BlobStor`, - `Metabase`, `Blobovnicza`. -- Support of voting for sidechain governance in IR node. -- `Raw` flag support in Object Get/Head/GetRange CLI commands. - -### Fixed - -- Ignoring object address from session token in eACL validation. - -## [0.12.1] - 2020-11-25 - -Bugfixes and small performance improvements. - -### Fixed - -- Routine leak by adding SDK client cache. (#184) -- Variety of ACL bugs. (#180, #190, #209) -- Policer tried to replicate virtual objects. (#182) -- Search queries with object ID field. (#177) -- Bug with extended ACL signature check in neofs-cli (#206) - -### Added - -- More debug logs in object service. -- Dial timeouts in object service config (`NEOFS_OBJECT_PUT_DIAL_TIMEOUT=5s`) - -### Changed - -- Routine pools in object service are non-blocking now. -- Container service now returns error if extended ACL is not set. - -## [0.12.0] - 2020-11-17 - -NeoFS-API v2.0 support and updated brand-new storage node application. - -### Fixed - -- SetConfig method invocation of netmap contract. (#147) -- Balance response overflow. (#122) - -### Added - -- Gas emission routine in inner ring nodes. -- GRPC reflection service. (`NEOFS_GRPC_ENABLE_REFLECT_SERVICE=true`) -- New netmap query language parser. - -### Changed - -- Storage node application rebuilt from scratch. -- CLI supports accounting, object and container related operations. -- Inner ring node shutdowns on neo RPC node connection drop. -- Updated to preview4 compatible neo-go version. - -## [0.11.0] - 2020-07-23 - -### Added - -- Inner ring application to repository. -- Inner ring epoch processor. -- Inner ring asset processor for GAS deposit and withdraw. - -### Changed - -- The structure of source code tree. - -## [0.10.0] - 2020-07-10 - -First public review release. -[Unreleased]: https://github.com/nspcc-dev/neofs-node/compare/v0.35.0...master -[0.35.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.34.0...v0.35.0 -[0.34.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.33.0...v0.34.0 -[0.33.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.32.0...v0.33.0 -[0.32.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.31.0...v0.32.0 -[0.31.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.30.2...v0.31.0 -[0.30.2]: https://github.com/nspcc-dev/neofs-node/compare/v0.30.1...v0.30.2 -[0.30.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.30.0...v0.30.1 -[0.30.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.29.0...v0.30.0 -[0.29.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.28.3...v0.29.0 -[0.28.3]: https://github.com/nspcc-dev/neofs-node/compare/v0.28.2...v0.28.3 -[0.28.2]: https://github.com/nspcc-dev/neofs-node/compare/v0.28.1...v0.28.2 -[0.28.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.28.0...v0.28.1 -[0.28.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.28.0-rc.3...v0.28.0 -[0.28.0-rc.3]: https://github.com/nspcc-dev/neofs-node/compare/v0.28.0-rc.2...v0.28.0-rc.3 -[0.28.0-rc.2]: https://github.com/nspcc-dev/neofs-node/compare/v0.28.0-rc.1...v0.28.0-rc.2 -[0.28.0-rc.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.27.7...v0.28.0-rc.1 -[0.27.7]: https://github.com/nspcc-dev/neofs-node/compare/v0.27.6...v0.27.7 -[0.27.6]: https://github.com/nspcc-dev/neofs-node/compare/v0.27.5...v0.27.6 -[0.27.5]: https://github.com/nspcc-dev/neofs-node/compare/v0.27.4...v0.27.5 -[0.27.4]: https://github.com/nspcc-dev/neofs-node/compare/v0.27.3...v0.27.4 -[0.27.3]: https://github.com/nspcc-dev/neofs-node/compare/v0.27.2...v0.27.3 -[0.27.2]: https://github.com/nspcc-dev/neofs-node/compare/v0.27.1...v0.27.2 -[0.27.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.27.0...v0.27.1 -[0.27.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.26.1...v0.27.0 -[0.26.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.26.0...v0.26.1 -[0.26.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.25.1...v0.26.0 -[0.25.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.25.0...v0.25.1 -[0.25.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.24.1...v0.25.0 -[0.24.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.24.0...v0.24.1 -[0.24.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.23.1...v0.24.0 -[0.23.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.23.0...v0.23.1 -[0.23.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.22.3...v0.23.0 -[0.22.3]: https://github.com/nspcc-dev/neofs-node/compare/v0.22.2...v0.22.3 -[0.22.2]: https://github.com/nspcc-dev/neofs-node/compare/v0.22.1...v0.22.2 -[0.22.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.22.0...v0.22.1 -[0.22.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.21.1...v0.22.0 -[0.21.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.21.0...v0.21.1 -[0.21.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.20.0...v0.21.0 -[0.20.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.19.0...v0.20.0 -[0.19.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.18.0...v0.19.0 -[0.18.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.17.0...v0.18.0 -[0.17.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.16.0...v0.17.0 -[0.16.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.15.0...v0.16.0 -[0.15.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.14.3...v0.15.0 -[0.14.3]: https://github.com/nspcc-dev/neofs-node/compare/v0.14.2...v0.14.3 -[0.14.2]: https://github.com/nspcc-dev/neofs-node/compare/v0.14.1...v0.14.2 -[0.14.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.14.0...v0.14.1 -[0.14.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.13.2...v0.14.0 -[0.13.2]: https://github.com/nspcc-dev/neofs-node/compare/v0.13.1...v0.13.2 -[0.13.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.13.0...v0.13.1 -[0.13.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.12.1...v0.13.0 -[0.12.1]: https://github.com/nspcc-dev/neofs-node/compare/v0.12.0...v0.12.1 -[0.12.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.11.0...v0.12.0 -[0.11.0]: https://github.com/nspcc-dev/neofs-node/compare/v0.10.0...v0.11.0 +[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-node/compare/98e48b68514127afc291b8a8ff6b12838ed1cb5c...master From c85a0bc86601d4820bab4357f1396f7c066f26de Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Apr 2023 20:20:32 +0300 Subject: [PATCH 0082/1943] [#236] blobstor/test: Reduce test descriptions I tried to add 4 more tests and suddenly, it became harder to navigate in code. Move directory creation in a common function. Signed-off-by: Evgenii Stratonikov --- .../blobstor/perf_test.go | 84 ++++++++----------- 1 file changed, 35 insertions(+), 49 deletions(-) diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index 0fd3048d3..a593e1bf4 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -13,60 +13,65 @@ import ( "github.com/stretchr/testify/require" ) +type storage struct { + desc string + create func(string) common.Storage +} + +func (s storage) open(b *testing.B) common.Storage { + dir, err := os.MkdirTemp(os.TempDir(), s.desc) + if err != nil { + b.Fatalf("creating %s root path: %v", s.desc, err) + } + st := s.create(dir) + + require.NoError(b, st.Open(false)) + require.NoError(b, st.Init()) + + b.Cleanup(func() { + require.NoError(b, st.Close()) + require.NoError(b, os.RemoveAll(dir)) + }) + + return st +} + // The storages to benchmark. Each storage has a description and a function which returns the actual // storage along with a cleanup function. -var storages = []struct { - desc string - create func(*testing.B) (common.Storage, func()) -}{ +var storages = []storage{ { desc: "memstore", - create: func(*testing.B) (common.Storage, func()) { - return memstore.New(), func() {} + create: func(string) common.Storage { + return memstore.New() }, }, { desc: "fstree_nosync", - create: func(b *testing.B) (common.Storage, func()) { - dir, err := os.MkdirTemp(os.TempDir(), "fstree_nosync") - if err != nil { - b.Fatalf("creating fstree_nosync root path: %v", err) - } - cleanup := func() { os.RemoveAll(dir) } + create: func(dir string) common.Storage { return fstree.New( fstree.WithPath(dir), fstree.WithDepth(2), fstree.WithDirNameLen(2), fstree.WithNoSync(true), - ), cleanup + ) }, }, { desc: "fstree", - create: func(b *testing.B) (common.Storage, func()) { - dir, err := os.MkdirTemp(os.TempDir(), "fstree") - if err != nil { - b.Fatalf("creating fstree root path: %v", err) - } - cleanup := func() { os.RemoveAll(dir) } + create: func(dir string) common.Storage { return fstree.New( fstree.WithPath(dir), fstree.WithDepth(2), fstree.WithDirNameLen(2), - ), cleanup + ) }, }, { desc: "blobovniczatree", - create: func(b *testing.B) (common.Storage, func()) { - dir, err := os.MkdirTemp(os.TempDir(), "blobovniczatree") - if err != nil { - b.Fatalf("creating blobovniczatree root path: %v", err) - } - cleanup := func() { os.RemoveAll(dir) } + create: func(dir string) common.Storage { return blobovniczatree.NewBlobovniczaTree( blobovniczatree.WithRootPath(dir), - ), cleanup + ) }, }, } @@ -95,10 +100,7 @@ func BenchmarkSubstorageReadPerf(b *testing.B) { for _, stEntry := range storages { b.Run(fmt.Sprintf("%s-%s", stEntry.desc, tt.desc), func(b *testing.B) { objGen := tt.objGen() - st, cleanup := stEntry.create(b) - - require.NoError(b, st.Open(false)) - require.NoError(b, st.Init()) + st := stEntry.open(b) // Fill database for i := 0; i < tt.size; i++ { @@ -123,9 +125,6 @@ func BenchmarkSubstorageReadPerf(b *testing.B) { require.NoError(b, err) } }) - - require.NoError(b, st.Close()) - cleanup() }) } } @@ -150,10 +149,7 @@ func BenchmarkSubstorageWritePerf(b *testing.B) { for _, stEntry := range storages { b.Run(fmt.Sprintf("%s-%s", stEntry.desc, genEntry.desc), func(b *testing.B) { gen := genEntry.create() - st, cleanup := stEntry.create(b) - - require.NoError(b, st.Open(false)) - require.NoError(b, st.Init()) + st := stEntry.open(b) b.ResetTimer() b.RunParallel(func(pb *testing.PB) { @@ -170,9 +166,6 @@ func BenchmarkSubstorageWritePerf(b *testing.B) { } } }) - - require.NoError(b, st.Close()) - cleanup() }) } } @@ -194,10 +187,7 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) { for _, stEntry := range storages { b.Run(fmt.Sprintf("%s-%s", stEntry.desc, tt.desc), func(b *testing.B) { objGen := tt.objGen() - st, cleanup := stEntry.create(b) - - require.NoError(b, st.Open(false)) - require.NoError(b, st.Init()) + st := stEntry.open(b) // Fill database for i := 0; i < tt.size; i++ { @@ -224,10 +214,6 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) { }) require.NoError(b, err) require.Equal(b, tt.size, cnt) - b.StopTimer() - - require.NoError(b, st.Close()) - cleanup() }) } } From 6ad5c38225adba16946991a68eb12a79bd3cf7e3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Apr 2023 20:30:27 +0300 Subject: [PATCH 0083/1943] [#236] testutil: Use random object id in `RandObjGenerator` Before this commit it was like this: ``` BenchmarkSubstorageWritePerf/memstore-rand10-8 227425 4859 ns/op BenchmarkSubstorageWritePerf/fstree_nosync-rand10-8 --- FAIL: BenchmarkSubstorageWritePerf/fstree_nosync-rand10-8 perf_test.go:165: writing entry: file exists perf_test.go:165: writing entry: file exists perf_test.go:165: writing entry: file exists BenchmarkSubstorageWritePerf/fstree-rand10-8 --- FAIL: BenchmarkSubstorageWritePerf/fstree-rand10-8 perf_test.go:165: writing entry: file exists perf_test.go:165: writing entry: file exists perf_test.go:165: writing entry: file exists ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/internal/testutil/generators.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/internal/testutil/generators.go b/pkg/local_object_storage/internal/testutil/generators.go index a45ea72aa..4ee92db88 100644 --- a/pkg/local_object_storage/internal/testutil/generators.go +++ b/pkg/local_object_storage/internal/testutil/generators.go @@ -81,7 +81,9 @@ type RandObjGenerator struct { var _ ObjectGenerator = &RandObjGenerator{} func (g *RandObjGenerator) Next() *object.Object { - return generateObjectWithOIDWithCIDWithSize(oid.ID{}, cid.ID{}, g.ObjSize) + var id oid.ID + rand.Read(id[:]) + return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) } // OverwriteObjGenerator is an ObjectGenerator that generates entries with random payloads of size objSize and at most maxObjects distinct IDs. From c4865783fc272f4219c075a14b897ffc2845b59d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Apr 2023 20:51:58 +0300 Subject: [PATCH 0084/1943] [#236] blobstor/test: Prefill storage in parallel in read benchmark `blobovniczatree` takes a really long time to prefill, because each batch takes at least 10ms, so for 10k iterations we have at least 100s of prefill. Signed-off-by: Evgenii Stratonikov --- .../blobstor/perf_test.go | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index a593e1bf4..0351eb56e 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/memstore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) type storage struct { @@ -103,18 +104,23 @@ func BenchmarkSubstorageReadPerf(b *testing.B) { st := stEntry.open(b) // Fill database + var errG errgroup.Group for i := 0; i < tt.size; i++ { obj := objGen.Next() addr := testutil.AddressFromObject(b, obj) - raw, err := obj.Marshal() - require.NoError(b, err) - if _, err := st.Put(common.PutPrm{ - Address: addr, - RawData: raw, - }); err != nil { - b.Fatalf("writing entry: %v", err) - } + errG.Go(func() error { + raw, err := obj.Marshal() + if err != nil { + return fmt.Errorf("marshal: %v", err) + } + _, err = st.Put(common.PutPrm{ + Address: addr, + RawData: raw, + }) + return err + }) } + require.NoError(b, errG.Wait()) // Benchmark reading addrGen := tt.addrGen() From 72565a91ef67e093aff9200f97f70651c05da31f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 11 Apr 2023 11:59:14 +0300 Subject: [PATCH 0085/1943] [#135] node: Update api-go version Signed-off-by: Dmitrii Stepanov --- go.mod | 26 +++++++++++++++------ go.sum | 71 +++++++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 77 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 6d9f04030..af85e69d5 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.18 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.0 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85 git.frostfs.info/TrueCloudLab/hrw v1.2.0 @@ -29,14 +29,14 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 go.etcd.io/bbolt v1.3.6 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 golang.org/x/sync v0.1.0 - golang.org/x/term v0.3.0 - google.golang.org/grpc v1.52.0 + golang.org/x/term v0.5.0 + google.golang.org/grpc v1.53.0 google.golang.org/protobuf v1.28.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -47,15 +47,19 @@ require ( github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.3 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.0 // indirect @@ -90,12 +94,20 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect github.com/twmb/murmur3 v1.1.5 // indirect github.com/urfave/cli v1.22.5 // indirect + go.opentelemetry.io/otel v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect + go.opentelemetry.io/otel/sdk v1.14.0 // indirect + go.opentelemetry.io/otel/trace v1.14.0 // indirect + go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.4.0 // indirect - golang.org/x/net v0.4.0 // indirect + golang.org/x/net v0.7.0 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect - google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) diff --git a/go.sum b/go.sum index b6f1a1e0b..f65a7aeab 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703 h1:lxe0DtZq/uFZVZu9apx6OcIXCJskQBMd/GVeYGKA3wA= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.11.2-0.20230315095236-9dc375346703/go.mod h1:gRd5iE5A84viily6AcNBsSlTx2XgoWrwRDz7z0MayDQ= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.0 h1:oZ0/KiaFeveXRLi5VVEpuLSHczeFyWx4HDl9wTJUtsE= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.0/go.mod h1:sPyITTmQT662ZI38ud2aoE1SUCAr1mO5xV8P4nzLkKI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= @@ -56,6 +56,7 @@ github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1 github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/ishell/v2 v2.0.2/go.mod h1:E4oTCXfo6QjoCart0QYa5m9w4S+deXs/P/9jA77A9Bs= @@ -90,11 +91,15 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= +github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -110,7 +115,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= @@ -129,6 +138,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= @@ -156,12 +166,19 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -239,6 +256,8 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -445,6 +464,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= @@ -470,8 +490,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= @@ -500,14 +521,30 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= +go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 h1:/fXHZHGvro6MVqV34fJzDhi7sHGpX3Ej/Qjmfn003ho= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 h1:TKf2uAs2ueguzLaxOCBXNpHxfO/aC7PAdDsSH0IbeRQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 h1:sEL90JjOO/4yhquXl5zTAkLLsZ5+MycAgX99SDsxGc8= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0/go.mod h1:oCslUcizYdpKYyS9e8srZEqM6BB8fq41VJBjLAE6z1w= +go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= +go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= +go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= +go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= @@ -607,13 +644,14 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -624,6 +662,7 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -692,8 +731,10 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -708,14 +749,15 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= @@ -846,8 +888,9 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= -google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -866,9 +909,11 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= -google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 5af9f5846981b7262ce941c417e470059ada8937 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 13 Mar 2023 14:01:43 +0300 Subject: [PATCH 0086/1943] [#135] tracing: Add tracing to node gRPC endpoints Signed-off-by: Dmitrii Stepanov --- CHANGELOG.md | 1 + cmd/frostfs-cli/modules/tree/client.go | 12 +++++++-- cmd/frostfs-node/config.go | 9 +++++++ cmd/frostfs-node/config/tracing/config.go | 31 +++++++++++++++++++++++ cmd/frostfs-node/grpc.go | 7 +++++ cmd/frostfs-node/main.go | 2 ++ cmd/frostfs-node/tracing.go | 31 +++++++++++++++++++++++ config/example/node.env | 4 +++ config/example/node.json | 5 ++++ config/example/node.yaml | 6 +++++ pkg/services/tree/cache.go | 12 +++++++-- pkg/services/tree/sync.go | 10 +++++++- 12 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 cmd/frostfs-node/config/tracing/config.go create mode 100644 cmd/frostfs-node/tracing.go diff --git a/CHANGELOG.md b/CHANGELOG.md index db846936f..0e07bb2f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Changelog for FrostFS Node - Multiple configs support (#44) - Parameters `nns-name` and `nns-zone` for command `frostfs-cli container create` (#37) - Tree service now saves the last synchronization height which persists across restarts (#82) +- Add tracing support (#135) ### Changed - Change `frostfs_node_engine_container_size` to counting sizes of logical objects diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index f379de41b..f25bff166 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" @@ -22,8 +23,15 @@ func _client(ctx context.Context) (tree.TreeServiceClient, error) { return nil, err } - opts := make([]grpc.DialOption, 1, 2) - opts[0] = grpc.WithBlock() + opts := []grpc.DialOption{ + grpc.WithBlock(), + grpc.WithChainUnaryInterceptor( + tracing.NewGRPCUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + tracing.NewGRPCStreamClientInterceptor(), + ), + } if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index ab615d340..d81e47b17 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -16,6 +16,7 @@ import ( "time" netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient" contractsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/contracts" @@ -27,6 +28,7 @@ import ( nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" + tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -1055,6 +1057,13 @@ func (c *cfg) reloadConfig(ctx context.Context) { } components = append(components, dCmp{"logger", logPrm.Reload}) + components = append(components, dCmp{"tracing", func() error { + updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg)) + if updated { + c.log.Info("tracing configation updated") + } + return err + }}) if cmp, updated := metricsComponent(c); updated { if cmp.enabled { cmp.preReload = enableMetricsSvc diff --git a/cmd/frostfs-node/config/tracing/config.go b/cmd/frostfs-node/config/tracing/config.go new file mode 100644 index 000000000..76572cc31 --- /dev/null +++ b/cmd/frostfs-node/config/tracing/config.go @@ -0,0 +1,31 @@ +package tracing + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/misc" +) + +const ( + subsection = "tracing" +) + +// ToTracingConfig extracts tracing config. +func ToTracingConfig(c *config.Config) *tracing.Config { + return &tracing.Config{ + Enabled: config.BoolSafe(c.Sub(subsection), "enabled"), + Exporter: tracing.Exporter(config.StringSafe(c.Sub(subsection), "exporter")), + Endpoint: config.StringSafe(c.Sub(subsection), "endpoint"), + Service: "frostfs-node", + InstanceID: getInstanceIDOrDefault(c), + Version: misc.Version, + } +} + +func getInstanceIDOrDefault(c *config.Config) string { + s := config.StringSlice(c.Sub("node"), "addresses") + if len(s) > 0 { + return s[0] + } + return "" +} diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index a56c76606..f3943f3ff 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -7,6 +7,7 @@ import ( "net" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" @@ -19,6 +20,12 @@ func initGRPC(c *cfg) { grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) { serverOpts := []grpc.ServerOption{ grpc.MaxSendMsgSize(maxMsgSize), + grpc.ChainUnaryInterceptor( + tracing.NewGRPCUnaryServerInterceptor(), + ), + grpc.ChainStreamInterceptor( + tracing.NewGRPCStreamServerInterceptor(), + ), } tlsCfg := sc.TLS() diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index cddedabe9..2f4c9853f 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -87,6 +87,8 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(c, pprof.name, pprof.init) initAndLog(c, metrics.name, metrics.init) + initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) }) + initLocalStorage(c) initAndLog(c, "storage engine", func(c *cfg) { diff --git a/cmd/frostfs-node/tracing.go b/cmd/frostfs-node/tracing.go new file mode 100644 index 000000000..bbdb71c64 --- /dev/null +++ b/cmd/frostfs-node/tracing.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" + "go.uber.org/zap" +) + +func initTracing(ctx context.Context, c *cfg) { + conf := tracingconfig.ToTracingConfig(c.appCfg) + + _, err := tracing.Setup(ctx, *conf) + if err != nil { + c.log.Error("failed init tracing", zap.Error(err)) + } + + c.closers = append(c.closers, closer{ + name: "tracing", + fn: func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + err := tracing.Shutdown(ctx) //cfg context cancels before close + if err != nil { + c.log.Error("failed shutdown tracing", zap.Error(err)) + } + }, + }) +} diff --git a/config/example/node.env b/config/example/node.env index 8034fbb23..9a1a8b052 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -184,3 +184,7 @@ FROSTFS_STORAGE_SHARD_1_PILORAMA_MAX_BATCH_SIZE=100 FROSTFS_STORAGE_SHARD_1_GC_REMOVER_BATCH_SIZE=200 #### Sleep interval between data remover tacts FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m + +FROSTFS_TRACING_ENABLED=true +FROSTFS_TRACING_ENDPOINT="localhost" +FROSTFS_TRACING_EXPORTER="otlp_grpc" \ No newline at end of file diff --git a/config/example/node.json b/config/example/node.json index e7bb375a5..8cfb5bb69 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -243,5 +243,10 @@ } } } + }, + "tracing": { + "enabled": true, + "endpoint": "localhost:9090", + "exporter": "otlp_grpc" } } diff --git a/config/example/node.yaml b/config/example/node.yaml index 6a5ea5f03..e3b41d413 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -214,3 +214,9 @@ storage: path: tmp/1/blob/pilorama.db no_sync: true # USE WITH CAUTION. Return to user before pages have been persisted. perm: 0644 # permission to use for the database file and intermediate directories + +tracing: + enabled: true + exporter: "otlp_grpc" + endpoint: "localhost" + \ No newline at end of file diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 73745e1b1..ab9f509ac 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "github.com/hashicorp/golang-lru/v2/simplelru" "google.golang.org/grpc" @@ -84,8 +85,15 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, return nil, err } - opts := make([]grpc.DialOption, 1, 2) - opts[0] = grpc.WithBlock() + opts := []grpc.DialOption{ + grpc.WithBlock(), + grpc.WithChainUnaryInterceptor( + tracing.NewGRPCUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + tracing.NewGRPCStreamClientInterceptor(), + ), + } // FIXME(@fyrchik): ugly hack #1322 if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") { diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 32d088c01..47299d1c9 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -10,6 +10,7 @@ import ( "math/rand" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" @@ -146,7 +147,14 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return false } - cc, err := grpc.DialContext(egCtx, a.URIAddr(), grpc.WithTransportCredentials(insecure.NewCredentials())) + cc, err := grpc.DialContext(egCtx, a.URIAddr(), + grpc.WithChainUnaryInterceptor( + tracing.NewGRPCUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + tracing.NewGRPCStreamClientInterceptor(), + ), + grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { // Failed to connect, try the next address. return false From 0920d848d0ee8e069b95c8db5bf1043991221d8d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 13 Mar 2023 14:37:35 +0300 Subject: [PATCH 0087/1943] [#135] get-object: Add tracing spans Signed-off-by: Dmitrii Stepanov --- .../internal/blobovnicza/inspect.go | 2 +- cmd/frostfs-node/main.go | 2 +- cmd/frostfs-node/notificator.go | 11 +++-- go.mod | 6 +-- .../blobovnicza/blobovnicza_test.go | 3 +- pkg/local_object_storage/blobovnicza/get.go | 12 ++++- .../blobovnicza/get_test.go | 3 +- .../blobstor/blobovniczatree/exists.go | 16 ++++++- .../blobstor/blobovniczatree/exists_test.go | 5 +- .../blobstor/blobovniczatree/get.go | 31 +++++++++---- .../blobstor/blobovniczatree/get_range.go | 33 +++++++++---- .../blobstor/blobstor_test.go | 5 +- .../blobstor/common/storage.go | 12 +++-- pkg/local_object_storage/blobstor/exists.go | 21 +++++++-- .../blobstor/exists_test.go | 9 ++-- .../blobstor/fstree/fstree.go | 46 ++++++++++++++++--- pkg/local_object_storage/blobstor/get.go | 21 +++++++-- .../blobstor/get_range.go | 23 ++++++++-- .../blobstor/internal/blobstortest/control.go | 3 +- .../blobstor/internal/blobstortest/delete.go | 9 ++-- .../blobstor/internal/blobstortest/exists.go | 7 +-- .../blobstor/internal/blobstortest/get.go | 9 ++-- .../internal/blobstortest/get_range.go | 15 +++--- .../blobstor/memstore/memstore.go | 9 ++-- .../blobstor/memstore/memstore_test.go | 9 ++-- .../blobstor/perf_test.go | 3 +- .../blobstor/teststore/teststore.go | 13 +++--- .../engine/control_test.go | 6 +-- pkg/local_object_storage/engine/delete.go | 2 +- .../engine/delete_test.go | 2 +- pkg/local_object_storage/engine/error_test.go | 20 ++++---- pkg/local_object_storage/engine/evacuate.go | 19 ++++---- .../engine/evacuate_test.go | 20 ++++---- pkg/local_object_storage/engine/exists.go | 3 +- pkg/local_object_storage/engine/get.go | 32 ++++++++----- pkg/local_object_storage/engine/head.go | 21 +++++---- pkg/local_object_storage/engine/head_test.go | 3 +- pkg/local_object_storage/engine/inhume.go | 2 +- pkg/local_object_storage/engine/lock.go | 3 +- pkg/local_object_storage/engine/put.go | 7 +-- pkg/local_object_storage/engine/range.go | 35 +++++++++----- .../engine/remove_copies.go | 2 +- .../shard/control_test.go | 10 ++-- pkg/local_object_storage/shard/delete_test.go | 7 +-- pkg/local_object_storage/shard/dump_test.go | 3 +- pkg/local_object_storage/shard/exists.go | 6 ++- pkg/local_object_storage/shard/gc_test.go | 3 +- pkg/local_object_storage/shard/get.go | 18 ++++++-- pkg/local_object_storage/shard/get_test.go | 5 +- pkg/local_object_storage/shard/head.go | 17 ++++++- pkg/local_object_storage/shard/head_test.go | 7 +-- pkg/local_object_storage/shard/inhume_test.go | 2 +- pkg/local_object_storage/shard/lock_test.go | 3 +- pkg/local_object_storage/shard/range.go | 22 +++++++-- pkg/local_object_storage/shard/range_test.go | 3 +- pkg/local_object_storage/shard/reload_test.go | 2 +- pkg/local_object_storage/shard/shard_test.go | 1 + .../shard/shutdown_test.go | 3 +- .../writecache/flush_test.go | 11 +++-- pkg/local_object_storage/writecache/get.go | 25 ++++++++-- pkg/local_object_storage/writecache/init.go | 3 +- .../writecache/options.go | 3 +- .../writecache/writecache.go | 5 +- pkg/services/control/server/evacuate.go | 4 +- pkg/services/notificator/deps.go | 4 +- pkg/services/notificator/service.go | 5 +- pkg/services/object/acl/eacl/v2/eacl_test.go | 3 +- pkg/services/object/acl/eacl/v2/headers.go | 5 +- pkg/services/object/acl/eacl/v2/localstore.go | 5 +- pkg/services/object/get/get_test.go | 2 +- pkg/services/object/get/local.go | 8 +++- pkg/services/object/get/remote.go | 4 ++ pkg/services/object/get/service.go | 2 +- pkg/services/object/get/util.go | 8 ++-- pkg/services/object/get/v2/get_forwarder.go | 8 ++++ .../object/get/v2/get_range_forwarder.go | 8 ++++ pkg/services/object/get/v2/head_forwarder.go | 8 ++++ pkg/services/object/get/v2/service.go | 2 +- pkg/services/object/get/v2/util.go | 2 +- pkg/services/replicator/process.go | 2 +- 80 files changed, 523 insertions(+), 231 deletions(-) diff --git a/cmd/frostfs-lens/internal/blobovnicza/inspect.go b/cmd/frostfs-lens/internal/blobovnicza/inspect.go index 3f4e8cfe4..13442a4b8 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/inspect.go +++ b/cmd/frostfs-lens/internal/blobovnicza/inspect.go @@ -33,7 +33,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { var prm blobovnicza.GetPrm prm.SetAddress(addr) - res, err := blz.Get(prm) + res, err := blz.Get(cmd.Context(), prm) common.ExitOnErr(cmd, common.Errf("could not fetch object: %w", err)) data := res.Object() diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 2f4c9853f..a97ad3879 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -102,7 +102,7 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(c, "container", func(c *cfg) { initContainerService(ctx, c) }) initAndLog(c, "session", initSessionService) initAndLog(c, "reputation", func(c *cfg) { initReputationService(ctx, c) }) - initAndLog(c, "notification", initNotifications) + initAndLog(c, "notification", func(c *cfg) { initNotifications(ctx, c) }) initAndLog(c, "object", initObjectService) initAndLog(c, "tree", initTreeService) initAndLog(c, "control", initControlService) diff --git a/cmd/frostfs-node/notificator.go b/cmd/frostfs-node/notificator.go index d5cb1ded4..4a310e5b0 100644 --- a/cmd/frostfs-node/notificator.go +++ b/cmd/frostfs-node/notificator.go @@ -23,7 +23,7 @@ type notificationSource struct { defaultTopic string } -func (n *notificationSource) Iterate(epoch uint64, handler func(topic string, addr oid.Address)) { +func (n *notificationSource) Iterate(ctx context.Context, epoch uint64, handler func(topic string, addr oid.Address)) { log := n.l.With(zap.Uint64("epoch", epoch)) listRes, err := n.e.ListContainers(engine.ListContainersPrm{}) @@ -51,7 +51,7 @@ func (n *notificationSource) Iterate(epoch uint64, handler func(topic string, ad } for _, a := range selectRes.AddressList() { - err = n.processAddress(a, handler) + err = n.processAddress(ctx, a, handler) if err != nil { log.Error("notificator: could not process object", zap.Stringer("address", a), @@ -66,13 +66,14 @@ func (n *notificationSource) Iterate(epoch uint64, handler func(topic string, ad } func (n *notificationSource) processAddress( + ctx context.Context, a oid.Address, h func(topic string, addr oid.Address), ) error { var prm engine.HeadPrm prm.WithAddress(a) - res, err := n.e.Head(prm) + res, err := n.e.Head(ctx, prm) if err != nil { return err } @@ -108,7 +109,7 @@ func (n notificationWriter) Notify(topic string, address oid.Address) { } } -func initNotifications(c *cfg) { +func initNotifications(ctx context.Context, c *cfg) { if nodeconfig.Notification(c.appCfg).Enabled() { topic := nodeconfig.Notification(c.appCfg).DefaultTopic() pubKey := hex.EncodeToString(c.cfgNodeInfo.localInfo.PublicKey()) @@ -151,7 +152,7 @@ func initNotifications(c *cfg) { addNewEpochAsyncNotificationHandler(c, func(e event.Event) { ev := e.(netmap.NewEpoch) - n.ProcessEpoch(ev.EpochNumber()) + n.ProcessEpoch(ctx, ev.EpochNumber()) }) } } diff --git a/go.mod b/go.mod index af85e69d5..301be6934 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( github.com/multiformats/go-multiaddr v0.8.0 github.com/nats-io/nats.go v1.22.1 github.com/nspcc-dev/neo-go v0.100.1 - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.4.0 github.com/paulmach/orb v0.2.2 @@ -31,6 +30,8 @@ require ( github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.2 go.etcd.io/bbolt v1.3.6 + go.opentelemetry.io/otel v1.14.0 + go.opentelemetry.io/otel/trace v1.14.0 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 @@ -80,6 +81,7 @@ require ( github.com/nats-io/nkeys v0.3.0 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -94,13 +96,11 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect github.com/twmb/murmur3 v1.1.5 // indirect github.com/urfave/cli v1.22.5 // indirect - go.opentelemetry.io/otel v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect go.opentelemetry.io/otel/sdk v1.14.0 // indirect - go.opentelemetry.io/otel/trace v1.14.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.4.0 // indirect diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index 4499c5d14..853628fb4 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "context" "errors" "math/rand" "os" @@ -39,7 +40,7 @@ func testGet(t *testing.T, blz *Blobovnicza, addr oid.Address, expObj []byte, as pGet.SetAddress(addr) // try to read object from Blobovnicza - res, err := blz.Get(pGet) + res, err := blz.Get(context.Background(), pGet) if assertErr != nil { require.True(t, assertErr(err)) } else { diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index 776f08d2b..c1cd19e53 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -1,12 +1,16 @@ package blobovnicza import ( + "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // GetPrm groups the parameters of Get operation. @@ -39,7 +43,13 @@ var errInterruptForEach = errors.New("interrupt for-each") // // Returns an error of type apistatus.ObjectNotFound if the requested object is not // presented in Blobovnicza. -func (b *Blobovnicza) Get(prm GetPrm) (GetRes, error) { +func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Get", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + var ( data []byte addrKey = addressKey(prm.addr) diff --git a/pkg/local_object_storage/blobovnicza/get_test.go b/pkg/local_object_storage/blobovnicza/get_test.go index 98097e9c8..ad30e8d94 100644 --- a/pkg/local_object_storage/blobovnicza/get_test.go +++ b/pkg/local_object_storage/blobovnicza/get_test.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "context" "os" "path/filepath" "testing" @@ -56,7 +57,7 @@ func TestBlobovnicza_Get(t *testing.T) { prmGet.SetAddress(addr) checkObj := func() { - res, err := blz.Get(prmGet) + res, err := blz.Get(context.Background(), prmGet) require.NoError(t, err) require.Equal(t, obj, res.Object()) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index e13e49351..748843ee9 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -1,15 +1,27 @@ package blobovniczatree import ( + "context" + "encoding/hex" "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) // Exists implements common.Storage. -func (b *Blobovniczas) Exists(prm common.ExistsPrm) (common.ExistsRes, error) { +func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Exists", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + )) + defer span.End() + if prm.StorageID != nil { id := blobovnicza.NewIDFromBytes(prm.StorageID) blz, err := b.openBlobovnicza(id.String()) @@ -32,7 +44,7 @@ func (b *Blobovniczas) Exists(prm common.ExistsPrm) (common.ExistsRes, error) { _, ok := activeCache[dirPath] - _, err := b.getObjectFromLevel(gPrm, p, !ok) + _, err := b.getObjectFromLevel(ctx, gPrm, p, !ok) if err != nil { if !blobovnicza.IsErrNotFound(err) { b.log.Debug("could not get object from level", diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 4f466a81a..08fd2223f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "context" "os" "path/filepath" "testing" @@ -44,7 +45,7 @@ func TestExistsInvalidStorageID(t *testing.T) { storageID[0]-- } - res, err := b.Exists(common.ExistsPrm{Address: addr, StorageID: storageID}) + res, err := b.Exists(context.Background(), common.ExistsPrm{Address: addr, StorageID: storageID}) require.NoError(t, err) require.False(t, res.Exists) }) @@ -57,7 +58,7 @@ func TestExistsInvalidStorageID(t *testing.T) { require.NoError(t, os.Chmod(badDir, 0)) t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "9"), os.ModePerm) }) - res, err := b.Exists(common.ExistsPrm{Address: addr, StorageID: storageID}) + res, err := b.Exists(context.Background(), common.ExistsPrm{Address: addr, StorageID: storageID}) require.Error(t, err) require.False(t, res.Exists) }) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 89ea9b641..bb84db086 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -1,14 +1,19 @@ package blobovniczatree import ( + "context" + "encoding/hex" "fmt" "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -16,7 +21,15 @@ import ( // // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. -func (b *Blobovniczas) Get(prm common.GetPrm) (res common.GetRes, err error) { +func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.GetRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Get", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + attribute.Bool("raw", prm.Raw), + )) + defer span.End() + var bPrm blobovnicza.GetPrm bPrm.SetAddress(prm.Address) @@ -27,7 +40,7 @@ func (b *Blobovniczas) Get(prm common.GetPrm) (res common.GetRes, err error) { return res, err } - return b.getObject(blz, bPrm) + return b.getObject(ctx, blz, bPrm) } activeCache := make(map[string]struct{}) @@ -37,7 +50,7 @@ func (b *Blobovniczas) Get(prm common.GetPrm) (res common.GetRes, err error) { _, ok := activeCache[dirPath] - res, err = b.getObjectFromLevel(bPrm, p, !ok) + res, err = b.getObjectFromLevel(ctx, bPrm, p, !ok) if err != nil { if !blobovnicza.IsErrNotFound(err) { b.log.Debug("could not get object from level", @@ -64,7 +77,7 @@ func (b *Blobovniczas) Get(prm common.GetPrm) (res common.GetRes, err error) { // tries to read object from particular blobovnicza. // // returns error if object could not be read from any blobovnicza of the same level. -func (b *Blobovniczas) getObjectFromLevel(prm blobovnicza.GetPrm, blzPath string, tryActive bool) (common.GetRes, error) { +func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.GetPrm, blzPath string, tryActive bool) (common.GetRes, error) { lvlPath := filepath.Dir(blzPath) // try to read from blobovnicza if it is opened @@ -72,7 +85,7 @@ func (b *Blobovniczas) getObjectFromLevel(prm blobovnicza.GetPrm, blzPath string v, ok := b.opened.Get(blzPath) b.lruMtx.Unlock() if ok { - if res, err := b.getObject(v, prm); err == nil { + if res, err := b.getObject(ctx, v, prm); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { b.log.Debug("could not read object from opened blobovnicza", @@ -92,7 +105,7 @@ func (b *Blobovniczas) getObjectFromLevel(prm blobovnicza.GetPrm, blzPath string b.activeMtx.RUnlock() if ok && tryActive { - if res, err := b.getObject(active.blz, prm); err == nil { + if res, err := b.getObject(ctx, active.blz, prm); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { b.log.Debug("could not get object from active blobovnicza", @@ -117,12 +130,12 @@ func (b *Blobovniczas) getObjectFromLevel(prm blobovnicza.GetPrm, blzPath string return common.GetRes{}, err } - return b.getObject(blz, prm) + return b.getObject(ctx, blz, prm) } // reads object from blobovnicza and returns GetSmallRes. -func (b *Blobovniczas) getObject(blz *blobovnicza.Blobovnicza, prm blobovnicza.GetPrm) (common.GetRes, error) { - res, err := blz.Get(prm) +func (b *Blobovniczas) getObject(ctx context.Context, blz *blobovnicza.Blobovnicza, prm blobovnicza.GetPrm) (common.GetRes, error) { + res, err := blz.Get(ctx, prm) if err != nil { return common.GetRes{}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 29df23944..b12cb32d4 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -1,14 +1,20 @@ package blobovniczatree import ( + "context" + "encoding/hex" "fmt" "path/filepath" + "strconv" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -16,7 +22,16 @@ import ( // // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. -func (b *Blobovniczas) GetRange(prm common.GetRangePrm) (res common.GetRangeRes, err error) { +func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (res common.GetRangeRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.GetRange", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), + attribute.String("length", strconv.FormatUint(prm.Range.GetLength(), 10)), + )) + defer span.End() + if prm.StorageID != nil { id := blobovnicza.NewIDFromBytes(prm.StorageID) blz, err := b.openBlobovnicza(id.String()) @@ -24,7 +39,7 @@ func (b *Blobovniczas) GetRange(prm common.GetRangePrm) (res common.GetRangeRes, return common.GetRangeRes{}, err } - return b.getObjectRange(blz, prm) + return b.getObjectRange(ctx, blz, prm) } activeCache := make(map[string]struct{}) @@ -35,7 +50,7 @@ func (b *Blobovniczas) GetRange(prm common.GetRangePrm) (res common.GetRangeRes, _, ok := activeCache[dirPath] - res, err = b.getRangeFromLevel(prm, p, !ok) + res, err = b.getRangeFromLevel(ctx, prm, p, !ok) if err != nil { outOfBounds := isErrOutOfRange(err) if !outOfBounds && !blobovnicza.IsErrNotFound(err) { @@ -68,7 +83,7 @@ func (b *Blobovniczas) GetRange(prm common.GetRangePrm) (res common.GetRangeRes, // tries to read range of object payload data from particular blobovnicza. // // returns error if object could not be read from any blobovnicza of the same level. -func (b *Blobovniczas) getRangeFromLevel(prm common.GetRangePrm, blzPath string, tryActive bool) (common.GetRangeRes, error) { +func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRangePrm, blzPath string, tryActive bool) (common.GetRangeRes, error) { lvlPath := filepath.Dir(blzPath) // try to read from blobovnicza if it is opened @@ -76,7 +91,7 @@ func (b *Blobovniczas) getRangeFromLevel(prm common.GetRangePrm, blzPath string, v, ok := b.opened.Get(blzPath) b.lruMtx.Unlock() if ok { - res, err := b.getObjectRange(v, prm) + res, err := b.getObjectRange(ctx, v, prm) switch { case err == nil, isErrOutOfRange(err): @@ -101,7 +116,7 @@ func (b *Blobovniczas) getRangeFromLevel(prm common.GetRangePrm, blzPath string, b.activeMtx.RUnlock() if ok && tryActive { - res, err := b.getObjectRange(active.blz, prm) + res, err := b.getObjectRange(ctx, active.blz, prm) switch { case err == nil, isErrOutOfRange(err): @@ -131,11 +146,11 @@ func (b *Blobovniczas) getRangeFromLevel(prm common.GetRangePrm, blzPath string, return common.GetRangeRes{}, err } - return b.getObjectRange(blz, prm) + return b.getObjectRange(ctx, blz, prm) } // reads range of object payload data from blobovnicza and returns GetRangeSmallRes. -func (b *Blobovniczas) getObjectRange(blz *blobovnicza.Blobovnicza, prm common.GetRangePrm) (common.GetRangeRes, error) { +func (b *Blobovniczas) getObjectRange(ctx context.Context, blz *blobovnicza.Blobovnicza, prm common.GetRangePrm) (common.GetRangeRes, error) { var gPrm blobovnicza.GetPrm gPrm.SetAddress(prm.Address) @@ -143,7 +158,7 @@ func (b *Blobovniczas) getObjectRange(blz *blobovnicza.Blobovnicza, prm common.G // stores data that is compressed on BlobStor side. // If blobovnicza learns to do the compression itself, // we can start using GetRange. - res, err := blz.Get(gPrm) + res, err := blz.Get(ctx, gPrm) if err != nil { return common.GetRangeRes{}, err } diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index df001a365..738cd7eee 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -1,6 +1,7 @@ package blobstor import ( + "context" "path/filepath" "testing" @@ -62,11 +63,11 @@ func TestCompression(t *testing.T) { } testGet := func(t *testing.T, b *BlobStor, i int) { - res1, err := b.Get(common.GetPrm{Address: object.AddressOf(smallObj[i])}) + res1, err := b.Get(context.Background(), common.GetPrm{Address: object.AddressOf(smallObj[i])}) require.NoError(t, err) require.Equal(t, smallObj[i], res1.Object) - res2, err := b.Get(common.GetPrm{Address: object.AddressOf(bigObj[i])}) + res2, err := b.Get(context.Background(), common.GetPrm{Address: object.AddressOf(bigObj[i])}) require.NoError(t, err) require.Equal(t, bigObj[i], res2.Object) } diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 76dd6d96e..b5d186242 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -1,6 +1,10 @@ package common -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" +) // Storage represents key-value object storage. // It is used as a building block for a blobstor of a shard. @@ -16,9 +20,9 @@ type Storage interface { // This function MUST be called before Open. SetReportErrorFunc(f func(string, error)) - Get(GetPrm) (GetRes, error) - GetRange(GetRangePrm) (GetRangeRes, error) - Exists(ExistsPrm) (ExistsRes, error) + Get(context.Context, GetPrm) (GetRes, error) + GetRange(context.Context, GetRangePrm) (GetRangeRes, error) + Exists(context.Context, ExistsPrm) (ExistsRes, error) Put(PutPrm) (PutRes, error) Delete(DeletePrm) (DeleteRes, error) Iterate(IteratePrm) (IterateRes, error) diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 7a5a00957..5882c33e0 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -1,7 +1,13 @@ package blobstor import ( + "context" + "encoding/hex" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -9,15 +15,22 @@ import ( // // Returns any error encountered that did not allow // to completely check object existence. -func (b *BlobStor) Exists(prm common.ExistsPrm) (common.ExistsRes, error) { +func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Exists", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + )) + defer span.End() + b.modeMtx.RLock() defer b.modeMtx.RUnlock() if prm.StorageID != nil { if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.Exists(prm) + return b.storage[len(b.storage)-1].Storage.Exists(ctx, prm) } - return b.storage[0].Storage.Exists(prm) + return b.storage[0].Storage.Exists(ctx, prm) } // If there was an error during existence check below, @@ -31,7 +44,7 @@ func (b *BlobStor) Exists(prm common.ExistsPrm) (common.ExistsRes, error) { // error | error | log the first error, return the second var errors []error for i := range b.storage { - res, err := b.storage[i].Storage.Exists(prm) + res, err := b.storage[i].Storage.Exists(ctx, prm) if err == nil && res.Exists { return res, nil } else if err != nil { diff --git a/pkg/local_object_storage/blobstor/exists_test.go b/pkg/local_object_storage/blobstor/exists_test.go index 4bbc256ab..805d78297 100644 --- a/pkg/local_object_storage/blobstor/exists_test.go +++ b/pkg/local_object_storage/blobstor/exists_test.go @@ -1,6 +1,7 @@ package blobstor import ( + "context" "os" "testing" @@ -43,13 +44,13 @@ func TestExists(t *testing.T) { for i := range objects { prm.Address = objectCore.AddressOf(objects[i]) - res, err := b.Exists(prm) + res, err := b.Exists(context.Background(), prm) require.NoError(t, err) require.True(t, res.Exists) } prm.Address = oidtest.Address() - res, err := b.Exists(prm) + res, err := b.Exists(context.Background(), prm) require.NoError(t, err) require.False(t, res.Exists) @@ -60,13 +61,13 @@ func TestExists(t *testing.T) { // Object exists, first error is logged. prm.Address = objectCore.AddressOf(objects[0]) - res, err := b.Exists(prm) + res, err := b.Exists(context.Background(), prm) require.NoError(t, err) require.True(t, res.Exists) // Object doesn't exist, first error is returned. prm.Address = objectCore.AddressOf(objects[1]) - _, err = b.Exists(prm) + _, err = b.Exists(context.Background(), prm) require.Error(t, err) require.ErrorIs(t, err, teststore.ErrDiskExploded) }) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 99484860a..462fbd63f 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -1,6 +1,7 @@ package fstree import ( + "context" "crypto/sha256" "errors" "fmt" @@ -11,6 +12,7 @@ import ( "strings" "syscall" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -19,6 +21,8 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // FSTree represents an object storage as a filesystem tree. @@ -208,7 +212,13 @@ func (t *FSTree) Delete(prm common.DeletePrm) (common.DeleteRes, error) { // Exists returns the path to the file with object contents if it exists in the storage // and an error otherwise. -func (t *FSTree) Exists(prm common.ExistsPrm) (common.ExistsRes, error) { +func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Exists", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + )) + defer span.End() + p := t.treePath(prm.Address) _, err := os.Stat(p) @@ -336,16 +346,30 @@ func (t *FSTree) PutStream(addr oid.Address, handler func(*os.File) error) error } // Get returns an object from the storage by address. -func (t *FSTree) Get(prm common.GetPrm) (common.GetRes, error) { +func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "FSTree.Get", + trace.WithAttributes( + attribute.Bool("raw", prm.Raw), + attribute.String("address", prm.Address.EncodeToString()), + )) + defer span.End() + p := t.treePath(prm.Address) if _, err := os.Stat(p); os.IsNotExist(err) { return common.GetRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } - data, err := os.ReadFile(p) - if err != nil { - return common.GetRes{}, err + var data []byte + var err error + { + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Get.ReadFile") + defer span.End() + + data, err = os.ReadFile(p) + if err != nil { + return common.GetRes{}, err + } } data, err = t.Decompress(data) @@ -362,8 +386,16 @@ func (t *FSTree) Get(prm common.GetPrm) (common.GetRes, error) { } // GetRange implements common.Storage. -func (t *FSTree) GetRange(prm common.GetRangePrm) (common.GetRangeRes, error) { - res, err := t.Get(common.GetPrm{Address: prm.Address}) +func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.GetRangeRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "FSTree.GetRange", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), + attribute.String("length", strconv.FormatUint(prm.Range.GetLength(), 10)), + )) + defer span.End() + + res, err := t.Get(ctx, common.GetPrm{Address: prm.Address}) if err != nil { return common.GetRangeRes{}, err } diff --git a/pkg/local_object_storage/blobstor/get.go b/pkg/local_object_storage/blobstor/get.go index 6caa61b84..65bc87c07 100644 --- a/pkg/local_object_storage/blobstor/get.go +++ b/pkg/local_object_storage/blobstor/get.go @@ -1,23 +1,36 @@ package blobstor import ( + "context" + "encoding/hex" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Get reads the object from b. // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. -func (b *BlobStor) Get(prm common.GetPrm) (common.GetRes, error) { +func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Get", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.Bool("raw", prm.Raw), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + )) + defer span.End() + b.modeMtx.RLock() defer b.modeMtx.RUnlock() if prm.StorageID == nil { for i := range b.storage { - res, err := b.storage[i].Storage.Get(prm) + res, err := b.storage[i].Storage.Get(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { return res, err } @@ -26,7 +39,7 @@ func (b *BlobStor) Get(prm common.GetPrm) (common.GetRes, error) { return common.GetRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.Get(prm) + return b.storage[len(b.storage)-1].Storage.Get(ctx, prm) } - return b.storage[0].Storage.Get(prm) + return b.storage[0].Storage.Get(ctx, prm) } diff --git a/pkg/local_object_storage/blobstor/get_range.go b/pkg/local_object_storage/blobstor/get_range.go index 93939cabb..ff9e72e97 100644 --- a/pkg/local_object_storage/blobstor/get_range.go +++ b/pkg/local_object_storage/blobstor/get_range.go @@ -1,23 +1,38 @@ package blobstor import ( + "context" + "encoding/hex" "errors" + "strconv" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // GetRange reads object payload data from b. // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. -func (b *BlobStor) GetRange(prm common.GetRangePrm) (common.GetRangeRes, error) { +func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (common.GetRangeRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.GetRange", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), + attribute.String("length", strconv.FormatUint(prm.Range.GetLength(), 10)), + )) + defer span.End() + b.modeMtx.RLock() defer b.modeMtx.RUnlock() if prm.StorageID == nil { for i := range b.storage { - res, err := b.storage[i].Storage.GetRange(prm) + res, err := b.storage[i].Storage.GetRange(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { return res, err } @@ -26,7 +41,7 @@ func (b *BlobStor) GetRange(prm common.GetRangePrm) (common.GetRangeRes, error) return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.GetRange(prm) + return b.storage[len(b.storage)-1].Storage.GetRange(ctx, prm) } - return b.storage[0].Storage.GetRange(prm) + return b.storage[0].Storage.GetRange(ctx, prm) } diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go index 0a74495d7..350bea96a 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go @@ -1,6 +1,7 @@ package blobstortest import ( + "context" "math/rand" "testing" @@ -26,7 +27,7 @@ func TestControl(t *testing.T, cons Constructor, min, max uint64) { prm.StorageID = objects[i].storageID prm.Raw = true - _, err := s.Get(prm) + _, err := s.Get(context.Background(), prm) require.NoError(t, err) } diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go index f3bb4c3f2..ad0045316 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go @@ -1,6 +1,7 @@ package blobstortest import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -35,18 +36,18 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { t.Run("exists fail", func(t *testing.T) { prm := common.ExistsPrm{Address: oidtest.Address()} - res, err := s.Exists(prm) + res, err := s.Exists(context.Background(), prm) require.NoError(t, err) require.False(t, res.Exists) }) t.Run("get fail", func(t *testing.T) { prm := common.GetPrm{Address: oidtest.Address()} - _, err := s.Get(prm) + _, err := s.Get(context.Background(), prm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) }) t.Run("getrange fail", func(t *testing.T) { prm := common.GetRangePrm{Address: oidtest.Address()} - _, err := s.GetRange(prm) + _, err := s.GetRange(context.Background(), prm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) }) }) @@ -75,7 +76,7 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { prm.Address = objects[3].addr prm.Raw = true - res, err := s.Get(prm) + res, err := s.Get(context.Background(), prm) require.NoError(t, err) require.Equal(t, objects[3].raw, res.RawData) }) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go index ee16ddcb3..99f6a79e8 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go @@ -1,6 +1,7 @@ package blobstortest import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -18,7 +19,7 @@ func TestExists(t *testing.T, cons Constructor, min, max uint64) { t.Run("missing object", func(t *testing.T) { prm := common.ExistsPrm{Address: oidtest.Address()} - res, err := s.Exists(prm) + res, err := s.Exists(context.Background(), prm) require.NoError(t, err) require.False(t, res.Exists) }) @@ -29,7 +30,7 @@ func TestExists(t *testing.T, cons Constructor, min, max uint64) { t.Run("without storage ID", func(t *testing.T) { prm.StorageID = nil - res, err := s.Exists(prm) + res, err := s.Exists(context.Background(), prm) require.NoError(t, err) require.True(t, res.Exists) }) @@ -37,7 +38,7 @@ func TestExists(t *testing.T, cons Constructor, min, max uint64) { t.Run("with storage ID", func(t *testing.T) { prm.StorageID = objects[0].storageID - res, err := s.Exists(prm) + res, err := s.Exists(context.Background(), prm) require.NoError(t, err) require.True(t, res.Exists) }) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go index cc3da6b4b..c5755dfba 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go @@ -1,6 +1,7 @@ package blobstortest import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -19,7 +20,7 @@ func TestGet(t *testing.T, cons Constructor, min, max uint64) { t.Run("missing object", func(t *testing.T) { gPrm := common.GetPrm{Address: oidtest.Address()} - _, err := s.Get(gPrm) + _, err := s.Get(context.Background(), gPrm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) }) @@ -29,13 +30,13 @@ func TestGet(t *testing.T, cons Constructor, min, max uint64) { // With storage ID. gPrm.StorageID = objects[i].storageID - res, err := s.Get(gPrm) + res, err := s.Get(context.Background(), gPrm) require.NoError(t, err) require.Equal(t, objects[i].obj, res.Object) // Without storage ID. gPrm.StorageID = nil - res, err = s.Get(gPrm) + res, err = s.Get(context.Background(), gPrm) require.NoError(t, err) require.Equal(t, objects[i].obj, res.Object) @@ -43,7 +44,7 @@ func TestGet(t *testing.T, cons Constructor, min, max uint64) { gPrm.StorageID = objects[i].storageID gPrm.Raw = true - res, err = s.Get(gPrm) + res, err = s.Get(context.Background(), gPrm) require.NoError(t, err) require.Equal(t, objects[i].raw, res.RawData) } diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go index e105fe6e8..b0c8aa95a 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go @@ -1,6 +1,7 @@ package blobstortest import ( + "context" "math" "testing" @@ -20,7 +21,7 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { t.Run("missing object", func(t *testing.T) { gPrm := common.GetRangePrm{Address: oidtest.Address()} - _, err := s.GetRange(gPrm) + _, err := s.GetRange(context.Background(), gPrm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) }) @@ -38,14 +39,14 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { t.Run("without storage ID", func(t *testing.T) { // Without storage ID. - res, err := s.GetRange(gPrm) + res, err := s.GetRange(context.Background(), gPrm) require.NoError(t, err) require.Equal(t, payload[start:stop], res.Data) }) t.Run("with storage ID", func(t *testing.T) { gPrm.StorageID = objects[0].storageID - res, err := s.GetRange(gPrm) + res, err := s.GetRange(context.Background(), gPrm) require.NoError(t, err) require.Equal(t, payload[start:stop], res.Data) }) @@ -54,7 +55,7 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { gPrm.Range.SetOffset(uint64(len(payload) + 10)) gPrm.Range.SetLength(10) - _, err := s.GetRange(gPrm) + _, err := s.GetRange(context.Background(), gPrm) require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange)) }) @@ -62,7 +63,7 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { gPrm.Range.SetOffset(10) gPrm.Range.SetLength(uint64(len(payload))) - _, err := s.GetRange(gPrm) + _, err := s.GetRange(context.Background(), gPrm) require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange)) }) @@ -70,7 +71,7 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { gPrm.Range.SetOffset(0) gPrm.Range.SetLength(1 << 63) - _, err := s.GetRange(gPrm) + _, err := s.GetRange(context.Background(), gPrm) require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange)) }) @@ -78,7 +79,7 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { gPrm.Range.SetOffset(10) gPrm.Range.SetLength(math.MaxUint64 - 2) - _, err := s.GetRange(gPrm) + _, err := s.GetRange(context.Background(), gPrm) require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange)) }) } diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index 5f6238476..4068d742e 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -2,6 +2,7 @@ package memstore import ( + "context" "fmt" "sync" @@ -32,7 +33,7 @@ func New(opts ...Option) common.Storage { return st } -func (s *memstoreImpl) Get(req common.GetPrm) (common.GetRes, error) { +func (s *memstoreImpl) Get(_ context.Context, req common.GetPrm) (common.GetRes, error) { key := req.Address.EncodeToString() s.mu.RLock() @@ -58,8 +59,8 @@ func (s *memstoreImpl) Get(req common.GetPrm) (common.GetRes, error) { return common.GetRes{Object: obj, RawData: data}, nil } -func (s *memstoreImpl) GetRange(req common.GetRangePrm) (common.GetRangeRes, error) { - getResp, err := s.Get(common.GetPrm{ +func (s *memstoreImpl) GetRange(ctx context.Context, req common.GetRangePrm) (common.GetRangeRes, error) { + getResp, err := s.Get(ctx, common.GetPrm{ Address: req.Address, StorageID: req.StorageID, }) @@ -80,7 +81,7 @@ func (s *memstoreImpl) GetRange(req common.GetRangePrm) (common.GetRangeRes, err }, nil } -func (s *memstoreImpl) Exists(req common.ExistsPrm) (common.ExistsRes, error) { +func (s *memstoreImpl) Exists(_ context.Context, req common.ExistsPrm) (common.ExistsRes, error) { key := req.Address.EncodeToString() s.mu.RLock() diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index 531a7d9e7..6482b2cff 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -1,6 +1,7 @@ package memstore import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -32,13 +33,13 @@ func TestSimpleLifecycle(t *testing.T) { } { - resp, err := s.Exists(common.ExistsPrm{Address: addr}) + resp, err := s.Exists(context.Background(), common.ExistsPrm{Address: addr}) require.NoError(t, err) require.True(t, resp.Exists) } { - resp, err := s.Get(common.GetPrm{Address: addr}) + resp, err := s.Get(context.Background(), common.GetPrm{Address: addr}) require.NoError(t, err) require.Equal(t, obj.Payload(), resp.Object.Payload()) } @@ -47,7 +48,7 @@ func TestSimpleLifecycle(t *testing.T) { var objRange objectSDK.Range objRange.SetOffset(256) objRange.SetLength(512) - resp, err := s.GetRange(common.GetRangePrm{ + resp, err := s.GetRange(context.Background(), common.GetRangePrm{ Address: addr, Range: objRange, }) @@ -61,7 +62,7 @@ func TestSimpleLifecycle(t *testing.T) { } { - resp, err := s.Exists(common.ExistsPrm{Address: addr}) + resp, err := s.Exists(context.Background(), common.ExistsPrm{Address: addr}) require.NoError(t, err) require.False(t, resp.Exists) } diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index 0351eb56e..d2359335f 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -1,6 +1,7 @@ package blobstor import ( + "context" "fmt" "os" "testing" @@ -127,7 +128,7 @@ func BenchmarkSubstorageReadPerf(b *testing.B) { b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - _, err := st.Get(common.GetPrm{Address: addrGen.Next()}) + _, err := st.Get(context.Background(), common.GetPrm{Address: addrGen.Next()}) require.NoError(b, err) } }) diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index f6ab4607c..03f64f0f1 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -13,6 +13,7 @@ package teststore import ( + "context" "errors" "fmt" "sync" @@ -140,36 +141,36 @@ func (s *TestStore) SetReportErrorFunc(f func(string, error)) { } } -func (s *TestStore) Get(req common.GetPrm) (common.GetRes, error) { +func (s *TestStore) Get(ctx context.Context, req common.GetPrm) (common.GetRes, error) { switch { case s.overrides.Get != nil: return s.overrides.Get(req) case s.st != nil: - return s.st.Get(req) + return s.st.Get(ctx, req) default: panic(fmt.Sprintf("unexpected storage call: Get(%+v)", req)) } } -func (s *TestStore) GetRange(req common.GetRangePrm) (common.GetRangeRes, error) { +func (s *TestStore) GetRange(ctx context.Context, req common.GetRangePrm) (common.GetRangeRes, error) { s.mu.RLock() defer s.mu.RUnlock() switch { case s.overrides.GetRange != nil: return s.overrides.GetRange(req) case s.st != nil: - return s.st.GetRange(req) + return s.st.GetRange(ctx, req) default: panic(fmt.Sprintf("unexpected storage call: GetRange(%+v)", req)) } } -func (s *TestStore) Exists(req common.ExistsPrm) (common.ExistsRes, error) { +func (s *TestStore) Exists(ctx context.Context, req common.ExistsPrm) (common.ExistsRes, error) { switch { case s.overrides.Exists != nil: return s.overrides.Exists(req) case s.st != nil: - return s.st.Exists(req) + return s.st.Exists(ctx, req) default: panic(fmt.Sprintf("unexpected storage call: Exists(%+v)", req)) } diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 12771340b..91bec63a6 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -212,20 +212,20 @@ func TestExecBlocks(t *testing.T) { require.NoError(t, e.BlockExecution(errBlock)) // try to exec some op - _, err := Head(e, addr) + _, err := Head(context.Background(), e, addr) require.ErrorIs(t, err, errBlock) // resume executions require.NoError(t, e.ResumeExecution()) - _, err = Head(e, addr) // can be any data-related op + _, err = Head(context.Background(), e, addr) // can be any data-related op require.NoError(t, err) // close require.NoError(t, e.Close()) // try exec after close - _, err = Head(e, addr) + _, err = Head(context.Background(), e, addr) require.Error(t, err) // try to resume diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 6ea5728bb..2105c452f 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -72,7 +72,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e var existsPrm shard.ExistsPrm existsPrm.SetAddress(prm.addr) - resExists, err := sh.Exists(existsPrm) + resExists, err := sh.Exists(ctx, existsPrm) if err != nil { if shard.IsErrRemoved(err) || shard.IsErrObjectExpired(err) { return true diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 54d73cee8..259a40a7c 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -93,7 +93,7 @@ func checkGetError(t *testing.T, e *StorageEngine, addr oid.Address, expected an var getPrm GetPrm getPrm.WithAddress(addr) - _, err := e.Get(getPrm) + _, err := e.Get(context.Background(), getPrm) if expected != nil { require.ErrorAs(t, err, expected) } else { diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index c9b194f6f..4ff019e4d 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -102,7 +102,7 @@ func TestErrorReporting(t *testing.T) { te.ng.mtx.RUnlock() require.NoError(t, err) - _, err = te.ng.Get(GetPrm{addr: object.AddressOf(obj)}) + _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.NoError(t, err) checkShardState(t, te.ng, te.shards[0].id, 0, mode.ReadWrite) @@ -115,7 +115,7 @@ func TestErrorReporting(t *testing.T) { } for i := uint32(1); i < 3; i++ { - _, err = te.ng.Get(GetPrm{addr: object.AddressOf(obj)}) + _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) checkShardState(t, te.ng, te.shards[0].id, i, mode.ReadWrite) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) @@ -136,7 +136,7 @@ func TestErrorReporting(t *testing.T) { te.ng.mtx.RUnlock() require.NoError(t, err) - _, err = te.ng.Get(GetPrm{addr: object.AddressOf(obj)}) + _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.NoError(t, err) checkShardState(t, te.ng, te.shards[0].id, 0, mode.ReadWrite) @@ -149,14 +149,14 @@ func TestErrorReporting(t *testing.T) { } for i := uint32(1); i < errThreshold; i++ { - _, err = te.ng.Get(GetPrm{addr: object.AddressOf(obj)}) + _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) checkShardState(t, te.ng, te.shards[0].id, i, mode.ReadWrite) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } for i := uint32(0); i < 2; i++ { - _, err = te.ng.Get(GetPrm{addr: object.AddressOf(obj)}) + _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) @@ -193,9 +193,9 @@ func TestBlobstorFailback(t *testing.T) { for i := range objs { addr := object.AddressOf(objs[i]) - _, err = te.ng.Get(GetPrm{addr: addr}) + _, err = te.ng.Get(context.Background(), GetPrm{addr: addr}) require.NoError(t, err) - _, err = te.ng.GetRange(RngPrm{addr: addr}) + _, err = te.ng.GetRange(context.Background(), RngPrm{addr: addr}) require.NoError(t, err) } @@ -213,15 +213,15 @@ func TestBlobstorFailback(t *testing.T) { for i := range objs { addr := object.AddressOf(objs[i]) - getRes, err := te.ng.Get(GetPrm{addr: addr}) + getRes, err := te.ng.Get(context.Background(), GetPrm{addr: addr}) require.NoError(t, err) require.Equal(t, objs[i], getRes.Object()) - rngRes, err := te.ng.GetRange(RngPrm{addr: addr, off: 1, ln: 10}) + rngRes, err := te.ng.GetRange(context.Background(), RngPrm{addr: addr, off: 1, ln: 10}) require.NoError(t, err) require.Equal(t, objs[i].Payload()[1:11], rngRes.Object().Payload()) - _, err = te.ng.GetRange(RngPrm{addr: addr, off: errSmallSize + 10, ln: 1}) + _, err = te.ng.GetRange(context.Background(), RngPrm{addr: addr, off: errSmallSize + 10, ln: 1}) require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 457228bb2..f16413ea2 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -1,6 +1,7 @@ package engine import ( + "context" "errors" "fmt" @@ -58,7 +59,7 @@ var errMustHaveTwoShards = errors.New("must have at least 1 spare shard") // Evacuate moves data from one shard to the others. // The shard being moved must be in read-only mode. -func (e *StorageEngine) Evacuate(prm EvacuateShardPrm) (EvacuateShardRes, error) { +func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (EvacuateShardRes, error) { shardIDs := make([]string, len(prm.shardID)) for i := range prm.shardID { shardIDs[i] = prm.shardID[i].String() @@ -83,7 +84,7 @@ func (e *StorageEngine) Evacuate(prm EvacuateShardPrm) (EvacuateShardRes, error) var res EvacuateShardRes for _, shardID := range shardIDs { - if err = e.evacuateShard(shardID, prm, &res, shards, weights, shardsToEvacuate); err != nil { + if err = e.evacuateShard(ctx, shardID, prm, &res, shards, weights, shardsToEvacuate); err != nil { return res, err } } @@ -92,7 +93,7 @@ func (e *StorageEngine) Evacuate(prm EvacuateShardPrm) (EvacuateShardRes, error) return res, nil } -func (e *StorageEngine) evacuateShard(shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, +func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { var listPrm shard.ListWithCursorPrm listPrm.WithCount(defaultEvacuateBatchSize) @@ -113,7 +114,7 @@ func (e *StorageEngine) evacuateShard(shardID string, prm EvacuateShardPrm, res return err } - if err = e.evacuateObjects(sh, listRes.AddressList(), prm, res, shards, weights, shardsToEvacuate); err != nil { + if err = e.evacuateObjects(ctx, sh, listRes.AddressList(), prm, res, shards, weights, shardsToEvacuate); err != nil { return err } @@ -160,7 +161,7 @@ func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) return shards, weights, nil } -func (e *StorageEngine) evacuateObjects(sh *shard.Shard, toEvacuate []object.AddressWithType, prm EvacuateShardPrm, res *EvacuateShardRes, +func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, toEvacuate []object.AddressWithType, prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { for i := range toEvacuate { addr := toEvacuate[i].Address @@ -168,7 +169,7 @@ func (e *StorageEngine) evacuateObjects(sh *shard.Shard, toEvacuate []object.Add var getPrm shard.GetPrm getPrm.SetAddress(addr) - getRes, err := sh.Get(getPrm) + getRes, err := sh.Get(ctx, getPrm) if err != nil { if prm.ignoreErrors { continue @@ -176,7 +177,7 @@ func (e *StorageEngine) evacuateObjects(sh *shard.Shard, toEvacuate []object.Add return err } - if e.tryEvacuateObject(addr, getRes.Object(), sh, res, shards, weights, shardsToEvacuate) { + if e.tryEvacuateObject(ctx, addr, getRes.Object(), sh, res, shards, weights, shardsToEvacuate) { continue } @@ -195,14 +196,14 @@ func (e *StorageEngine) evacuateObjects(sh *shard.Shard, toEvacuate []object.Add return nil } -func (e *StorageEngine) tryEvacuateObject(addr oid.Address, object *objectSDK.Object, sh *shard.Shard, res *EvacuateShardRes, +func (e *StorageEngine) tryEvacuateObject(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, res *EvacuateShardRes, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) bool { hrw.SortHasherSliceByWeightValue(shards, weights, hrw.Hash([]byte(addr.EncodeToString()))) for j := range shards { if _, ok := shardsToEvacuate[shards[j].ID().String()]; ok { continue } - putDone, exists := e.putToShard(shards[j].hashedShard, j, shards[j].pool, addr, object) + putDone, exists := e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object) if putDone || exists { if putDone { e.log.Debug("object is moved to another shard", diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 51abc4b1c..c116aeff9 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -91,7 +91,7 @@ func TestEvacuateShard(t *testing.T) { var prm GetPrm prm.WithAddress(objectCore.AddressOf(objects[i])) - _, err := e.Get(prm) + _, err := e.Get(context.Background(), prm) require.NoError(t, err) } } @@ -102,14 +102,14 @@ func TestEvacuateShard(t *testing.T) { prm.WithShardIDList(ids[2:3]) t.Run("must be read-only", func(t *testing.T) { - res, err := e.Evacuate(prm) + res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, shard.ErrMustBeReadOnly) require.Equal(t, 0, res.Count()) }) require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly)) - res, err := e.Evacuate(prm) + res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) require.Equal(t, objPerShard, res.count) @@ -120,7 +120,7 @@ func TestEvacuateShard(t *testing.T) { checkHasObjects(t) // Calling it again is OK, but all objects are already moved, so no new PUTs should be done. - res, err = e.Evacuate(prm) + res, err = e.Evacuate(context.Background(), prm) require.NoError(t, err) require.Equal(t, 0, res.count) @@ -165,13 +165,13 @@ func TestEvacuateNetwork(t *testing.T) { var prm EvacuateShardPrm prm.shardID = ids[0:1] - res, err := e.Evacuate(prm) + res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errMustHaveTwoShards) require.Equal(t, 0, res.Count()) prm.handler = acceptOneOf(objects, 2) - res, err = e.Evacuate(prm) + res, err = e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) require.Equal(t, 2, res.Count()) }) @@ -185,14 +185,14 @@ func TestEvacuateNetwork(t *testing.T) { prm.shardID = ids[1:2] prm.handler = acceptOneOf(objects, 2) - res, err := e.Evacuate(prm) + res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) require.Equal(t, 2, res.Count()) t.Run("no errors", func(t *testing.T) { prm.handler = acceptOneOf(objects, 3) - res, err := e.Evacuate(prm) + res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) require.Equal(t, 3, res.Count()) }) @@ -217,14 +217,14 @@ func TestEvacuateNetwork(t *testing.T) { prm.shardID = evacuateIDs prm.handler = acceptOneOf(objects, totalCount-1) - res, err := e.Evacuate(prm) + res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) require.Equal(t, totalCount-1, res.Count()) t.Run("no errors", func(t *testing.T) { prm.handler = acceptOneOf(objects, totalCount) - res, err := e.Evacuate(prm) + res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) require.Equal(t, totalCount, res.Count()) }) diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index a43c7f23f..3a8e09a6d 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -1,6 +1,7 @@ package engine import ( + "context" "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -16,7 +17,7 @@ func (e *StorageEngine) exists(addr oid.Address) (bool, error) { exists := false e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { - res, err := sh.Exists(shPrm) + res, err := sh.Exists(context.TODO(), shPrm) if err != nil { if shard.IsErrRemoved(err) { alreadyRemoved = true diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 4d0a30bc8..7d17b50fa 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -1,14 +1,18 @@ package engine import ( + "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -43,16 +47,22 @@ func (r GetRes) Object() *objectSDK.Object { // Returns an error of type apistatus.ObjectAlreadyRemoved if the object has been marked as removed. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) Get(prm GetPrm) (res GetRes, err error) { +func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { err = e.execIfNotBlocked(func() error { - res, err = e.get(prm) + res, err = e.get(ctx, prm) return err }) return } -func (e *StorageEngine) get(prm GetPrm) (GetRes, error) { +func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.get", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + if e.metrics != nil { defer elapsed(e.metrics.AddGetDuration)() } @@ -69,7 +79,7 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) { Engine: e, } - it.tryGetWithMeta() + it.tryGetWithMeta(ctx) if it.SplitInfo != nil { return GetRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(it.SplitInfo)) @@ -84,7 +94,7 @@ func (e *StorageEngine) get(prm GetPrm) (GetRes, error) { return GetRes{}, it.OutError } - it.tryGetFromBlobstore() + it.tryGetFromBlobstore(ctx) if it.Object == nil { return GetRes{}, it.OutError @@ -116,14 +126,14 @@ type getShardIterator struct { splitInfoErr *objectSDK.SplitInfoError } -func (i *getShardIterator) tryGetWithMeta() { +func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { noMeta := sh.GetMode().NoMetabase() i.ShardPrm.SetIgnoreMeta(noMeta) i.HasDegraded = i.HasDegraded || noMeta - res, err := sh.Get(i.ShardPrm) + res, err := sh.Get(ctx, i.ShardPrm) if err == nil { i.Object = res.Object() return true @@ -162,7 +172,7 @@ func (i *getShardIterator) tryGetWithMeta() { }) } -func (i *getShardIterator) tryGetFromBlobstore() { +func (i *getShardIterator) tryGetFromBlobstore(ctx context.Context) { // If the object is not found but is present in metabase, // try to fetch it from blobstor directly. If it is found in any // blobstor, increase the error counter for the shard which contains the meta. @@ -174,18 +184,18 @@ func (i *getShardIterator) tryGetFromBlobstore() { return false } - res, err := sh.Get(i.ShardPrm) + res, err := sh.Get(ctx, i.ShardPrm) i.Object = res.Object() return err == nil }) } // Get reads object from local storage by provided address. -func Get(storage *StorageEngine, addr oid.Address) (*objectSDK.Object, error) { +func Get(ctx context.Context, storage *StorageEngine, addr oid.Address) (*objectSDK.Object, error) { var getPrm GetPrm getPrm.WithAddress(addr) - res, err := storage.Get(getPrm) + res, err := storage.Get(ctx, getPrm) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index 689b46de8..130e76c3d 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -1,8 +1,10 @@ package engine import ( + "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -52,16 +54,19 @@ func (r HeadRes) Header() *objectSDK.Object { // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object was inhumed. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) Head(prm HeadPrm) (res HeadRes, err error) { +func (e *StorageEngine) Head(ctx context.Context, prm HeadPrm) (res HeadRes, err error) { err = e.execIfNotBlocked(func() error { - res, err = e.head(prm) + res, err = e.head(ctx, prm) return err }) return } -func (e *StorageEngine) head(prm HeadPrm) (HeadRes, error) { +func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.head") + defer span.End() + if e.metrics != nil { defer elapsed(e.metrics.AddHeadDuration)() } @@ -81,7 +86,7 @@ func (e *StorageEngine) head(prm HeadPrm) (HeadRes, error) { shPrm.SetRaw(prm.raw) e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { - res, err := sh.Head(shPrm) + res, err := sh.Head(ctx, shPrm) if err != nil { switch { case shard.IsErrNotFound(err): @@ -139,11 +144,11 @@ func (e *StorageEngine) head(prm HeadPrm) (HeadRes, error) { } // Head reads object header from local storage by provided address. -func Head(storage *StorageEngine, addr oid.Address) (*objectSDK.Object, error) { +func Head(ctx context.Context, storage *StorageEngine, addr oid.Address) (*objectSDK.Object, error) { var headPrm HeadPrm headPrm.WithAddress(addr) - res, err := storage.Head(headPrm) + res, err := storage.Head(ctx, headPrm) if err != nil { return nil, err } @@ -153,12 +158,12 @@ func Head(storage *StorageEngine, addr oid.Address) (*objectSDK.Object, error) { // HeadRaw reads object header from local storage by provided address and raw // flag. -func HeadRaw(storage *StorageEngine, addr oid.Address, raw bool) (*objectSDK.Object, error) { +func HeadRaw(ctx context.Context, storage *StorageEngine, addr oid.Address, raw bool) (*objectSDK.Object, error) { var headPrm HeadPrm headPrm.WithAddress(addr) headPrm.WithRaw(raw) - res, err := storage.Head(headPrm) + res, err := storage.Head(ctx, headPrm) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go index e2a1edc98..e5fd4b04f 100644 --- a/pkg/local_object_storage/engine/head_test.go +++ b/pkg/local_object_storage/engine/head_test.go @@ -1,6 +1,7 @@ package engine import ( + "context" "os" "testing" @@ -66,7 +67,7 @@ func TestHeadRaw(t *testing.T) { headPrm.WithAddress(parentAddr) headPrm.WithRaw(true) - _, err = e.Head(headPrm) + _, err = e.Head(context.Background(), headPrm) require.Error(t, err) var si *object.SplitInfoError diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 2ecca5256..db9988338 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -134,7 +134,7 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh if checkExists { existPrm.SetAddress(addr) - exRes, err := sh.Exists(existPrm) + exRes, err := sh.Exists(ctx, existPrm) if err != nil { if shard.IsErrRemoved(err) || shard.IsErrObjectExpired(err) { // inhumed once - no need to be inhumed again diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 20a4d68e8..60a1d9c9f 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -1,6 +1,7 @@ package engine import ( + "context" "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -69,7 +70,7 @@ func (e *StorageEngine) lockSingle(idCnr cid.ID, locker, locked oid.ID, checkExi var existsPrm shard.ExistsPrm existsPrm.SetAddress(addrLocked) - exRes, err := sh.Exists(existsPrm) + exRes, err := sh.Exists(context.TODO(), existsPrm) if err != nil { var siErr *objectSDK.SplitInfoError if !errors.As(err, &siErr) { diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 68a4467f2..5f9105efc 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -1,6 +1,7 @@ package engine import ( + "context" "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -72,7 +73,7 @@ func (e *StorageEngine) put(prm PutPrm) (PutRes, error) { return false } - putDone, exists := e.putToShard(sh, ind, pool, addr, prm.obj) + putDone, exists := e.putToShard(context.TODO(), sh, ind, pool, addr, prm.obj) finished = putDone || exists return finished }) @@ -87,7 +88,7 @@ func (e *StorageEngine) put(prm PutPrm) (PutRes, error) { // putToShard puts object to sh. // First return value is true iff put has been successfully done. // Second return value is true iff object already exists. -func (e *StorageEngine) putToShard(sh hashedShard, ind int, pool util.WorkerPool, addr oid.Address, obj *objectSDK.Object) (bool, bool) { +func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, pool util.WorkerPool, addr oid.Address, obj *objectSDK.Object) (bool, bool) { var putSuccess, alreadyExists bool exitCh := make(chan struct{}) @@ -98,7 +99,7 @@ func (e *StorageEngine) putToShard(sh hashedShard, ind int, pool util.WorkerPool var existPrm shard.ExistsPrm existPrm.SetAddress(addr) - exists, err := sh.Exists(existPrm) + exists, err := sh.Exists(ctx, existPrm) if err != nil { if shard.IsErrObjectExpired(err) { // object is already found but diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 25b533bd4..3d119ac6f 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -1,14 +1,19 @@ package engine import ( + "context" "errors" + "strconv" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -56,16 +61,24 @@ func (r RngRes) Object() *objectSDK.Object { // Returns ErrRangeOutOfBounds if the requested object range is out of bounds. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) GetRange(prm RngPrm) (res RngRes, err error) { +func (e *StorageEngine) GetRange(ctx context.Context, prm RngPrm) (res RngRes, err error) { err = e.execIfNotBlocked(func() error { - res, err = e.getRange(prm) + res, err = e.getRange(ctx, prm) return err }) return } -func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) { +func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.getRange", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + attribute.String("offset", strconv.FormatUint(prm.off, 10)), + attribute.String("length", strconv.FormatUint(prm.ln, 10)), + )) + defer span.End() + if e.metrics != nil { defer elapsed(e.metrics.AddRangeDuration)() } @@ -83,7 +96,7 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) { Engine: e, } - it.tryGetWithMeta() + it.tryGetWithMeta(ctx) if it.SplitInfo != nil { return RngRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(it.SplitInfo)) @@ -96,7 +109,7 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) { return RngRes{}, it.OutError } - it.tryGetFromBlobstor() + it.tryGetFromBlobstor(ctx) if it.Object == nil { return RngRes{}, it.OutError @@ -114,12 +127,12 @@ func (e *StorageEngine) getRange(prm RngPrm) (RngRes, error) { } // GetRange reads object payload range from local storage by provided address. -func GetRange(storage *StorageEngine, addr oid.Address, rng *objectSDK.Range) ([]byte, error) { +func GetRange(ctx context.Context, storage *StorageEngine, addr oid.Address, rng *objectSDK.Range) ([]byte, error) { var rangePrm RngPrm rangePrm.WithAddress(addr) rangePrm.WithPayloadRange(rng) - res, err := storage.GetRange(rangePrm) + res, err := storage.GetRange(ctx, rangePrm) if err != nil { return nil, err } @@ -141,13 +154,13 @@ type getRangeShardIterator struct { Engine *StorageEngine } -func (i *getRangeShardIterator) tryGetWithMeta() { +func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) { i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { noMeta := sh.GetMode().NoMetabase() i.HasDegraded = i.HasDegraded || noMeta i.ShardPrm.SetIgnoreMeta(noMeta) - res, err := sh.GetRange(i.ShardPrm) + res, err := sh.GetRange(ctx, i.ShardPrm) if err == nil { i.Object = res.Object() return true @@ -185,7 +198,7 @@ func (i *getRangeShardIterator) tryGetWithMeta() { }) } -func (i *getRangeShardIterator) tryGetFromBlobstor() { +func (i *getRangeShardIterator) tryGetFromBlobstor(ctx context.Context) { // If the object is not found but is present in metabase, // try to fetch it from blobstor directly. If it is found in any // blobstor, increase the error counter for the shard which contains the meta. @@ -197,7 +210,7 @@ func (i *getRangeShardIterator) tryGetFromBlobstor() { return false } - res, err := sh.GetRange(i.ShardPrm) + res, err := sh.GetRange(ctx, i.ShardPrm) if shard.IsErrOutOfRange(err) { var errOutOfRange apistatus.ObjectOutOfRange diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index d881a52d1..c50c0844c 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -116,7 +116,7 @@ func (e *StorageEngine) removeObjects(ctx context.Context, ch <-chan oid.Address var existsPrm shard.ExistsPrm existsPrm.SetAddress(addr) - res, err := shards[i].Exists(existsPrm) + res, err := shards[i].Exists(ctx, existsPrm) if err != nil { return err } else if !res.Exists() { diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index df7e536cb..50ea20bb8 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -63,6 +63,7 @@ func TestShardOpen(t *testing.T) { newShard := func() *Shard { return New( + WithID(NewIDFromBytes([]byte{})), WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{ @@ -146,6 +147,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) { require.NoError(t, err) sh = New( + WithID(NewIDFromBytes([]byte{})), WithBlobStorOptions(blobOpts...), WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))), WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta_new")), meta.WithEpochState(epochState{})), @@ -155,7 +157,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) { var getPrm GetPrm getPrm.SetAddress(addr) - _, err = sh.Get(getPrm) + _, err = sh.Get(context.Background(), getPrm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) require.NoError(t, sh.Close()) } @@ -176,6 +178,7 @@ func TestRefillMetabase(t *testing.T) { } sh := New( + WithID(NewIDFromBytes([]byte{})), WithBlobStorOptions(blobOpts...), WithMetaBaseOptions( meta.WithPath(filepath.Join(p, "meta")), @@ -277,7 +280,7 @@ func TestRefillMetabase(t *testing.T) { checkObj := func(addr oid.Address, expObj *objectSDK.Object) { headPrm.SetAddress(addr) - res, err := sh.Head(headPrm) + res, err := sh.Head(context.Background(), headPrm) if expObj == nil { require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) @@ -302,7 +305,7 @@ func TestRefillMetabase(t *testing.T) { for _, member := range tombMembers { headPrm.SetAddress(member) - _, err := sh.Head(headPrm) + _, err := sh.Head(context.Background(), headPrm) if exists { require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) @@ -343,6 +346,7 @@ func TestRefillMetabase(t *testing.T) { require.NoError(t, err) sh = New( + WithID(NewIDFromBytes([]byte{})), WithBlobStorOptions(blobOpts...), WithMetaBaseOptions( meta.WithPath(filepath.Join(p, "meta_restored")), diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index 9115f3e0d..c37dfa285 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -1,6 +1,7 @@ package shard_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -51,7 +52,7 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { _, err = sh.Delete(delPrm) require.NoError(t, err) - _, err = sh.Get(getPrm) + _, err = sh.Get(context.Background(), getPrm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) }) @@ -69,13 +70,13 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { _, err := sh.Put(putPrm) require.NoError(t, err) - _, err = sh.Get(getPrm) + _, err = sh.Get(context.Background(), getPrm) require.NoError(t, err) _, err = sh.Delete(delPrm) require.NoError(t, err) - _, err = sh.Get(getPrm) + _, err = sh.Get(context.Background(), getPrm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) }) } diff --git a/pkg/local_object_storage/shard/dump_test.go b/pkg/local_object_storage/shard/dump_test.go index 65427dd5c..9d585cc06 100644 --- a/pkg/local_object_storage/shard/dump_test.go +++ b/pkg/local_object_storage/shard/dump_test.go @@ -2,6 +2,7 @@ package shard_test import ( "bytes" + "context" "io" "math/rand" "os" @@ -276,7 +277,7 @@ func checkRestore(t *testing.T, sh *shard.Shard, prm shard.RestorePrm, objects [ for i := range objects { getPrm.SetAddress(object.AddressOf(objects[i])) - res, err := sh.Get(getPrm) + res, err := sh.Get(context.Background(), getPrm) require.NoError(t, err) require.Equal(t, objects[i], res.Object()) } diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 35b9cba9b..76e4347d4 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -1,6 +1,8 @@ package shard import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -33,7 +35,7 @@ func (p ExistsRes) Exists() bool { // // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been marked as removed. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. -func (s *Shard) Exists(prm ExistsPrm) (ExistsRes, error) { +func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { var exists bool var err error @@ -45,7 +47,7 @@ func (s *Shard) Exists(prm ExistsPrm) (ExistsRes, error) { p.Address = prm.addr var res common.ExistsRes - res, err = s.blobStor.Exists(p) + res, err = s.blobStor.Exists(ctx, p) exists = res.Exists } else { var existsPrm meta.ExistsPrm diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index e7aa3614e..8012e60f8 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -33,6 +33,7 @@ func Test_GCDropsLockedExpiredObject(t *testing.T) { rootPath := t.TempDir() opts := []shard.Option{ + shard.WithID(shard.NewIDFromBytes([]byte{})), shard.WithLogger(&logger.Logger{Logger: zap.NewNop()}), shard.WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{ @@ -115,7 +116,7 @@ func Test_GCDropsLockedExpiredObject(t *testing.T) { var getPrm shard.GetPrm getPrm.SetAddress(objectCore.AddressOf(obj)) require.Eventually(t, func() bool { - _, err = sh.Get(getPrm) + _, err = sh.Get(context.Background(), getPrm) return shard.IsErrNotFound(err) }, 3*time.Second, 1*time.Second, "expired object must be deleted") } diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 4231c01db..3406b9338 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -1,8 +1,10 @@ package shard import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -11,6 +13,8 @@ import ( apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -61,7 +65,15 @@ func (r GetRes) HasMeta() bool { // Returns an error of type apistatus.ObjectNotFound if the requested object is missing in shard. // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. -func (s *Shard) Get(prm GetPrm) (GetRes, error) { +func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Get", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", prm.addr.EncodeToString()), + attribute.Bool("skip_meta", prm.skipMeta), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -70,7 +82,7 @@ func (s *Shard) Get(prm GetPrm) (GetRes, error) { getPrm.Address = prm.addr getPrm.StorageID = id - res, err := stor.Get(getPrm) + res, err := stor.Get(ctx, getPrm) if err != nil { return nil, err } @@ -79,7 +91,7 @@ func (s *Shard) Get(prm GetPrm) (GetRes, error) { } wc := func(c writecache.Cache) (*objectSDK.Object, error) { - return c.Get(prm.addr) + return c.Get(ctx, prm.addr) } skipMeta := prm.skipMeta || s.info.Mode.NoMetabase() diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index 9d1975331..f670b2864 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -2,6 +2,7 @@ package shard_test import ( "bytes" + "context" "errors" "testing" "time" @@ -111,11 +112,11 @@ func testShardGet(t *testing.T, hasWriteCache bool) { } func testGet(t *testing.T, sh *shard.Shard, getPrm shard.GetPrm, hasWriteCache bool) (shard.GetRes, error) { - res, err := sh.Get(getPrm) + res, err := sh.Get(context.Background(), getPrm) if hasWriteCache { require.Eventually(t, func() bool { if shard.IsErrNotFound(err) { - res, err = sh.Get(getPrm) + res, err = sh.Get(context.Background(), getPrm) } return !shard.IsErrNotFound(err) }, time.Second, time.Millisecond*100) diff --git a/pkg/local_object_storage/shard/head.go b/pkg/local_object_storage/shard/head.go index 6913d3162..8e8ff9433 100644 --- a/pkg/local_object_storage/shard/head.go +++ b/pkg/local_object_storage/shard/head.go @@ -1,9 +1,14 @@ package shard import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // HeadPrm groups the parameters of Head operation. @@ -43,7 +48,15 @@ func (r HeadRes) Object() *objectSDK.Object { // Returns an error of type apistatus.ObjectNotFound if object is missing in Shard. // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. -func (s *Shard) Head(prm HeadPrm) (HeadRes, error) { +func (s *Shard) Head(ctx context.Context, prm HeadPrm) (HeadRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Head", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", prm.addr.EncodeToString()), + attribute.Bool("raw", prm.raw), + )) + defer span.End() + var obj *objectSDK.Object var err error if s.GetMode().NoMetabase() { @@ -52,7 +65,7 @@ func (s *Shard) Head(prm HeadPrm) (HeadRes, error) { getPrm.SetIgnoreMeta(true) var res GetRes - res, err = s.Get(getPrm) + res, err = s.Get(ctx, getPrm) obj = res.Object() } else { var headParams meta.GetPrm diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index 36c8915b5..449626e93 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -1,6 +1,7 @@ package shard_test import ( + "context" "errors" "testing" "time" @@ -75,18 +76,18 @@ func testShardHead(t *testing.T, hasWriteCache bool) { headPrm.SetAddress(object.AddressOf(parent)) headPrm.SetRaw(false) - head, err := sh.Head(headPrm) + head, err := sh.Head(context.Background(), headPrm) require.NoError(t, err) require.Equal(t, parent.CutPayload(), head.Object()) }) } func testHead(t *testing.T, sh *shard.Shard, headPrm shard.HeadPrm, hasWriteCache bool) (shard.HeadRes, error) { - res, err := sh.Head(headPrm) + res, err := sh.Head(context.Background(), headPrm) if hasWriteCache { require.Eventually(t, func() bool { if shard.IsErrNotFound(err) { - res, err = sh.Head(headPrm) + res, err = sh.Head(context.Background(), headPrm) } return !shard.IsErrNotFound(err) }, time.Second, time.Millisecond*100) diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 191afab01..41845c414 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -51,6 +51,6 @@ func testShardInhume(t *testing.T, hasWriteCache bool) { _, err = sh.Inhume(context.Background(), inhPrm) require.NoError(t, err) - _, err = sh.Get(getPrm) + _, err = sh.Get(context.Background(), getPrm) require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) } diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 995aa1473..2bee66298 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -27,6 +27,7 @@ func TestShard_Lock(t *testing.T) { rootPath := t.TempDir() opts := []shard.Option{ + shard.WithID(shard.NewIDFromBytes([]byte{})), shard.WithLogger(&logger.Logger{Logger: zap.NewNop()}), shard.WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{ @@ -137,7 +138,7 @@ func TestShard_Lock(t *testing.T) { var getPrm shard.GetPrm getPrm.SetAddress(objectcore.AddressOf(obj)) - _, err = sh.Get(getPrm) + _, err = sh.Get(context.Background(), getPrm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) }) } diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index c9106c235..4355c31a3 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -1,6 +1,10 @@ package shard import ( + "context" + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -8,6 +12,8 @@ import ( apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // RngPrm groups the parameters of GetRange operation. @@ -66,7 +72,17 @@ func (r RngRes) HasMeta() bool { // Returns an error of type apistatus.ObjectNotFound if the requested object is missing. // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. -func (s *Shard) GetRange(prm RngPrm) (RngRes, error) { +func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.GetRange", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", prm.addr.EncodeToString()), + attribute.Bool("skip_meta", prm.skipMeta), + attribute.String("offset", strconv.FormatUint(prm.off, 10)), + attribute.String("length", strconv.FormatUint(prm.ln, 10)), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -77,7 +93,7 @@ func (s *Shard) GetRange(prm RngPrm) (RngRes, error) { getRngPrm.Range.SetLength(prm.ln) getRngPrm.StorageID = id - res, err := stor.GetRange(getRngPrm) + res, err := stor.GetRange(ctx, getRngPrm) if err != nil { return nil, err } @@ -89,7 +105,7 @@ func (s *Shard) GetRange(prm RngPrm) (RngRes, error) { } wc := func(c writecache.Cache) (*object.Object, error) { - res, err := c.Get(prm.addr) + res, err := c.Get(ctx, prm.addr) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 6782dca1e..164181214 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -1,6 +1,7 @@ package shard_test import ( + "context" "math" "path/filepath" "testing" @@ -105,7 +106,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { rngPrm.SetAddress(addr) rngPrm.SetRange(tc.rng.GetOffset(), tc.rng.GetLength()) - res, err := sh.GetRange(rngPrm) + res, err := sh.GetRange(context.Background(), rngPrm) if tc.hasErr { require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) } else { diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 7aa331c7f..1bfa33dd7 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -66,7 +66,7 @@ func TestShardReload(t *testing.T) { var prm ExistsPrm prm.SetAddress(objects[i].addr) - res, err := sh.Exists(prm) + res, err := sh.Exists(context.Background(), prm) require.NoError(t, err) require.Equal(t, exists, res.Exists(), "object #%d is missing", i) } diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 027e6ca7b..fea342766 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -63,6 +63,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts } opts := []shard.Option{ + shard.WithID(shard.NewIDFromBytes([]byte{})), shard.WithLogger(&logger.Logger{Logger: zap.L()}), shard.WithBlobStorOptions(bsOpts...), shard.WithMetaBaseOptions( diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 57a982684..5fd13221a 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -1,6 +1,7 @@ package shard_test import ( + "context" "math/rand" "testing" @@ -55,7 +56,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { for i := range objects { getPrm.SetAddress(object.AddressOf(objects[i])) - _, err := sh.Get(getPrm) + _, err := sh.Get(context.Background(), getPrm) require.NoError(t, err, i) } } diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index a6c2035db..9dc216fb3 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -1,6 +1,7 @@ package writecache import ( + "context" "os" "path/filepath" "testing" @@ -95,7 +96,7 @@ func TestFlush(t *testing.T) { prm.Address = objects[i].addr prm.StorageID = mRes.StorageID() - res, err := bs.Get(prm) + res, err := bs.Get(context.Background(), prm) require.NoError(t, err) require.Equal(t, objects[i].obj, res.Object) } @@ -119,7 +120,7 @@ func TestFlush(t *testing.T) { _, err := mb.Get(mPrm) require.Error(t, err) - _, err = bs.Get(common.GetPrm{Address: objects[i].addr}) + _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) require.Error(t, err) } @@ -149,7 +150,7 @@ func TestFlush(t *testing.T) { _, err := mb.Get(mPrm) require.Error(t, err) - _, err = bs.Get(common.GetPrm{Address: objects[i].addr}) + _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) require.Error(t, err) } @@ -266,7 +267,7 @@ func TestFlush(t *testing.T) { require.NoError(t, wc.Open(true)) initWC(t, wc) for i := range objects { - _, err := wc.Get(objects[i].addr) + _, err := wc.Get(context.Background(), objects[i].addr) require.NoError(t, err, i) } require.NoError(t, wc.Close()) @@ -275,7 +276,7 @@ func TestFlush(t *testing.T) { require.NoError(t, wc.Open(false)) initWC(t, wc) for i := range objects { - _, err := wc.Get(objects[i].addr) + _, err := wc.Get(context.Background(), objects[i].addr) if i < 2 { require.ErrorAs(t, err, new(apistatus.ObjectNotFound), i) } else { diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index a15f42e18..6af1bd181 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -1,6 +1,9 @@ package writecache import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -8,14 +11,22 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Get returns object from write-cache. // // Returns an error of type apistatus.ObjectNotFound if the requested object is missing in write-cache. -func (c *cache) Get(addr oid.Address) (*objectSDK.Object, error) { +func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { saddr := addr.EncodeToString() + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Get", + trace.WithAttributes( + attribute.String("address", saddr), + )) + defer span.End() + value, err := Get(c.db, []byte(saddr)) if err == nil { obj := objectSDK.New() @@ -23,7 +34,7 @@ func (c *cache) Get(addr oid.Address) (*objectSDK.Object, error) { return obj, obj.Unmarshal(value) } - res, err := c.fsTree.Get(common.GetPrm{Address: addr}) + res, err := c.fsTree.Get(ctx, common.GetPrm{Address: addr}) if err != nil { return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) } @@ -35,8 +46,14 @@ func (c *cache) Get(addr oid.Address) (*objectSDK.Object, error) { // Head returns object header from write-cache. // // Returns an error of type apistatus.ObjectNotFound if the requested object is missing in write-cache. -func (c *cache) Head(addr oid.Address) (*objectSDK.Object, error) { - obj, err := c.Get(addr) +func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Head", + trace.WithAttributes( + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + + obj, err := c.Get(ctx, addr) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/writecache/init.go b/pkg/local_object_storage/writecache/init.go index 56b27ec4b..ffe7a0129 100644 --- a/pkg/local_object_storage/writecache/init.go +++ b/pkg/local_object_storage/writecache/init.go @@ -1,6 +1,7 @@ package writecache import ( + "context" "errors" "sync" @@ -177,6 +178,6 @@ func (c *cache) flushStatus(addr oid.Address) (bool, bool) { prm.SetAddress(addr) mRes, _ := c.metabase.StorageID(prm) - res, err := c.blobstor.Exists(common.ExistsPrm{Address: addr, StorageID: mRes.StorageID()}) + res, err := c.blobstor.Exists(context.TODO(), common.ExistsPrm{Address: addr, StorageID: mRes.StorageID()}) return err == nil && res.Exists, false } diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index 59c7c9d47..cca8986b3 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -1,6 +1,7 @@ package writecache import ( + "context" "io/fs" "os" "time" @@ -27,7 +28,7 @@ type metabase interface { type blob interface { Put(common.PutPrm) (common.PutRes, error) NeedsCompression(obj *objectSDK.Object) bool - Exists(res common.ExistsPrm) (common.ExistsRes, error) + Exists(ctx context.Context, res common.ExistsPrm) (common.ExistsRes, error) } type options struct { diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index a5b8ff0a3..24070dbda 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -1,6 +1,7 @@ package writecache import ( + "context" "os" "sync" @@ -23,8 +24,8 @@ type Info struct { // Cache represents write-cache for objects. type Cache interface { - Get(address oid.Address) (*object.Object, error) - Head(oid.Address) (*object.Object, error) + Get(ctx context.Context, address oid.Address) (*object.Object, error) + Head(context.Context, oid.Address) (*object.Object, error) // Delete removes object referenced by the given oid.Address from the // Cache. Returns any error encountered that prevented the object to be // removed. diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index 6c064efa3..b64a91883 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -19,7 +19,7 @@ import ( "google.golang.org/grpc/status" ) -func (s *Server) EvacuateShard(_ context.Context, req *control.EvacuateShardRequest) (*control.EvacuateShardResponse, error) { +func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRequest) (*control.EvacuateShardResponse, error) { err := s.isValidRequest(req) if err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) @@ -30,7 +30,7 @@ func (s *Server) EvacuateShard(_ context.Context, req *control.EvacuateShardRequ prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) prm.WithFaultHandler(s.replicate) - res, err := s.s.Evacuate(prm) + res, err := s.s.Evacuate(ctx, prm) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/notificator/deps.go b/pkg/services/notificator/deps.go index ded4b4b7d..d6330f788 100644 --- a/pkg/services/notificator/deps.go +++ b/pkg/services/notificator/deps.go @@ -1,6 +1,8 @@ package notificator import ( + "context" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -8,7 +10,7 @@ import ( type NotificationSource interface { // Iterate must iterate over all notifications for the // provided epoch and call handler for all of them. - Iterate(epoch uint64, handler func(topic string, addr oid.Address)) + Iterate(ctx context.Context, epoch uint64, handler func(topic string, addr oid.Address)) } // NotificationWriter notifies all the subscribers diff --git a/pkg/services/notificator/service.go b/pkg/services/notificator/service.go index 096618300..0a8a5d96d 100644 --- a/pkg/services/notificator/service.go +++ b/pkg/services/notificator/service.go @@ -1,6 +1,7 @@ package notificator import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -71,11 +72,11 @@ func New(prm *Prm) *Notificator { // ProcessEpoch looks for all objects with defined epoch in the storage // and passes their addresses to the NotificationWriter. -func (n *Notificator) ProcessEpoch(epoch uint64) { +func (n *Notificator) ProcessEpoch(ctx context.Context, epoch uint64) { logger := n.l.With(zap.Uint64("epoch", epoch)) logger.Debug("notificator: start processing object notifications") - n.ns.Iterate(epoch, func(topic string, addr oid.Address) { + n.ns.Iterate(ctx, epoch, func(topic string, addr oid.Address) { n.l.Debug("notificator: processing object notification", zap.String("topic", topic), zap.Stringer("address", addr), diff --git a/pkg/services/object/acl/eacl/v2/eacl_test.go b/pkg/services/object/acl/eacl/v2/eacl_test.go index 4570e271a..ce5d98d5d 100644 --- a/pkg/services/object/acl/eacl/v2/eacl_test.go +++ b/pkg/services/object/acl/eacl/v2/eacl_test.go @@ -1,6 +1,7 @@ package v2 import ( + "context" "crypto/ecdsa" "errors" "testing" @@ -26,7 +27,7 @@ type testLocalStorage struct { err error } -func (s *testLocalStorage) Head(addr oid.Address) (*object.Object, error) { +func (s *testLocalStorage) Head(ctx context.Context, addr oid.Address) (*object.Object, error) { require.True(s.t, addr.Container().Equals(s.expAddr.Container())) require.True(s.t, addr.Object().Equals(s.expAddr.Object())) diff --git a/pkg/services/object/acl/eacl/v2/headers.go b/pkg/services/object/acl/eacl/v2/headers.go index 736c05763..095810848 100644 --- a/pkg/services/object/acl/eacl/v2/headers.go +++ b/pkg/services/object/acl/eacl/v2/headers.go @@ -1,6 +1,7 @@ package v2 import ( + "context" "errors" "fmt" @@ -27,7 +28,7 @@ type cfg struct { } type ObjectStorage interface { - Head(oid.Address) (*object.Object, error) + Head(context.Context, oid.Address) (*object.Object, error) } type Request interface { @@ -207,7 +208,7 @@ func (h *cfg) localObjectHeaders(cnr cid.ID, idObj *oid.ID) ([]eaclSDK.Header, b addr.SetContainer(cnr) addr.SetObject(*idObj) - obj, err := h.storage.Head(addr) + obj, err := h.storage.Head(context.TODO(), addr) if err == nil { return headersFromObject(obj, cnr, idObj), true } diff --git a/pkg/services/object/acl/eacl/v2/localstore.go b/pkg/services/object/acl/eacl/v2/localstore.go index 40271f1cd..0f23e9881 100644 --- a/pkg/services/object/acl/eacl/v2/localstore.go +++ b/pkg/services/object/acl/eacl/v2/localstore.go @@ -1,6 +1,7 @@ package v2 import ( + "context" "io" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" @@ -12,10 +13,10 @@ type localStorage struct { ls *engine.StorageEngine } -func (s *localStorage) Head(addr oid.Address) (*objectSDK.Object, error) { +func (s *localStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { if s.ls == nil { return nil, io.ErrUnexpectedEOF } - return engine.Head(s.ls, addr) + return engine.Head(ctx, s.ls, addr) } diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 3d1a95cbb..319bc6b58 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -139,7 +139,7 @@ func (c *testClient) addResult(addr oid.Address, obj *objectSDK.Object, err erro }{obj: obj, err: err} } -func (s *testStorage) get(exec *execCtx) (*objectSDK.Object, error) { +func (s *testStorage) get(_ context.Context, exec *execCtx) (*objectSDK.Object, error) { var ( ok bool obj *objectSDK.Object diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index a6a77729c..82ed911e4 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -4,15 +4,21 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" ) func (exec *execCtx) executeLocal(ctx context.Context) { + ctx, span := tracing.StartSpanFromContext(ctx, "getService.executeLocal") + defer func() { + span.End() + }() + var err error - exec.collectedObject, err = exec.svc.localStorage.get(exec) + exec.collectedObject, err = exec.svc.localStorage.get(ctx, exec) var errSplitInfo *objectSDK.SplitInfoError var errRemoved apistatus.ObjectAlreadyRemoved diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index 1532bade0..697e48ee2 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -11,6 +12,9 @@ import ( ) func (exec *execCtx) processNode(ctx context.Context, info client.NodeInfo) bool { + ctx, span := tracing.StartSpanFromContext(ctx, "getService.processNode") + defer span.End() + exec.log.Debug("processing node...") client, ok := exec.remoteClient(info) diff --git a/pkg/services/object/get/service.go b/pkg/services/object/get/service.go index dfa3b48ac..a9391d016 100644 --- a/pkg/services/object/get/service.go +++ b/pkg/services/object/get/service.go @@ -31,7 +31,7 @@ type cfg struct { log *logger.Logger localStorage interface { - get(*execCtx) (*object.Object, error) + get(context.Context, *execCtx) (*object.Object, error) } clientCache interface { diff --git a/pkg/services/object/get/util.go b/pkg/services/object/get/util.go index 09d8c67af..dd4ace407 100644 --- a/pkg/services/object/get/util.go +++ b/pkg/services/object/get/util.go @@ -200,13 +200,13 @@ func (c *clientWrapper) get(ctx context.Context, exec *execCtx, key *ecdsa.Priva return res.Object(), nil } -func (e *storageEngineWrapper) get(exec *execCtx) (*object.Object, error) { +func (e *storageEngineWrapper) get(ctx context.Context, exec *execCtx) (*object.Object, error) { if exec.headOnly() { var headPrm engine.HeadPrm headPrm.WithAddress(exec.address()) headPrm.WithRaw(exec.isRaw()) - r, err := e.engine.Head(headPrm) + r, err := e.engine.Head(ctx, headPrm) if err != nil { return nil, err } @@ -217,7 +217,7 @@ func (e *storageEngineWrapper) get(exec *execCtx) (*object.Object, error) { getRange.WithAddress(exec.address()) getRange.WithPayloadRange(rng) - r, err := e.engine.GetRange(getRange) + r, err := e.engine.GetRange(ctx, getRange) if err != nil { return nil, err } @@ -227,7 +227,7 @@ func (e *storageEngineWrapper) get(exec *execCtx) (*object.Object, error) { var getPrm engine.GetPrm getPrm.WithAddress(exec.address()) - r, err := e.engine.Get(getPrm) + r, err := e.engine.Get(ctx, getPrm) if err != nil { return nil, err } diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index 330a0642f..8163ae928 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -9,6 +9,7 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -18,6 +19,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) type getRequestForwarder struct { @@ -30,6 +33,11 @@ type getRequestForwarder struct { } func (f *getRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "getRequestForwarder.forwardRequestToNode", + trace.WithAttributes(attribute.String("address", addr.String())), + ) + defer span.End() + var err error // once compose and resign forwarding request diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 5893f8de3..9cf6384ed 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -9,6 +9,7 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -18,6 +19,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) type getRangeRequestForwarder struct { @@ -29,6 +32,11 @@ type getRangeRequestForwarder struct { } func (f *getRangeRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "getRangeRequestForwarder.forwardRequestToNode", + trace.WithAttributes(attribute.String("address", addr.String())), + ) + defer span.End() + var err error // once compose and resign forwarding request diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index 45c0174fd..e1d4c02db 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -8,6 +8,7 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -19,6 +20,8 @@ import ( frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) type headRequestForwarder struct { @@ -30,6 +33,11 @@ type headRequestForwarder struct { } func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "headRequestForwarder.forwardRequestToNode", + trace.WithAttributes(attribute.String("address", addr.String())), + ) + defer span.End() + var err error // once compose and resign forwarding request diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index 3fd8cd04a..1bd8befaf 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -96,7 +96,7 @@ func (s *Service) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV resp := new(objectV2.HeadResponse) resp.SetBody(new(objectV2.HeadResponseBody)) - p, err := s.toHeadPrm(ctx, req, resp) + p, err := s.toHeadPrm(req, resp) if err != nil { return nil, err } diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 3a50a6ca5..69bed23f4 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -215,7 +215,7 @@ func (w *headResponseWriter) WriteHeader(_ context.Context, hdr *object.Object) return nil } -func (s *Service) toHeadPrm(ctx context.Context, req *objectV2.HeadRequest, resp *objectV2.HeadResponse) (*getsvc.HeadPrm, error) { +func (s *Service) toHeadPrm(req *objectV2.HeadRequest, resp *objectV2.HeadResponse) (*getsvc.HeadPrm, error) { body := req.GetBody() addrV2 := body.GetAddress() diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 476a5bc0a..53df81b77 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -27,7 +27,7 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) if task.obj == nil { var err error - task.obj, err = engine.Get(p.localStorage, task.addr) + task.obj, err = engine.Get(ctx, p.localStorage, task.addr) if err != nil { p.log.Error("could not get object from local storage", zap.Stringer("object", task.addr), From b2ca73054722fcc4f73746d75b1599377c82040a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 11 Apr 2023 18:03:57 +0300 Subject: [PATCH 0088/1943] [#135] acl: Add tracing spans Signed-off-by: Dmitrii Stepanov --- pkg/services/object/acl/v2/service.go | 78 +++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 6544d78d7..93c1c65f8 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -6,6 +6,7 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" @@ -111,6 +112,14 @@ func New(opts ...Option) Service { // Get implements ServiceServer interface, makes ACL checks and calls // next Get method in the ServiceServer pipeline. func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream) error { + spanClosed := false + _, span := tracing.StartSpanFromContext(stream.Context(), "checkACL") + defer func() { + if !spanClosed { + span.End() + } + }() + cnr, err := getContainerIDFromRequest(request) if err != nil { return err @@ -158,6 +167,9 @@ func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream return eACLErr(reqInfo, err) } + span.End() + spanClosed = true + return b.next.Get(request, &getStreamBasicChecker{ GetObjectStream: stream, info: reqInfo, @@ -177,6 +189,14 @@ func (b Service) Put() (object.PutObjectStream, error) { func (b Service) Head( ctx context.Context, request *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { + spanClosed := false + _, span := tracing.StartSpanFromContext(ctx, "checkACL") + defer func() { + if !spanClosed { + span.End() + } + }() + cnr, err := getContainerIDFromRequest(request) if err != nil { return nil, err @@ -224,6 +244,9 @@ func (b Service) Head( return nil, eACLErr(reqInfo, err) } + span.End() + spanClosed = true + resp, err := b.next.Head(ctx, request) if err == nil { if err = b.checker.CheckEACL(resp, reqInfo); err != nil { @@ -235,6 +258,14 @@ func (b Service) Head( } func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStream) error { + spanClosed := false + _, span := tracing.StartSpanFromContext(stream.Context(), "checkACL") + defer func() { + if !spanClosed { + span.End() + } + }() + id, err := getContainerIDFromRequest(request) if err != nil { return err @@ -275,6 +306,9 @@ func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStr return eACLErr(reqInfo, err) } + span.End() + spanClosed = true + return b.next.Search(request, &searchStreamBasicChecker{ checker: b.checker, SearchStream: stream, @@ -285,6 +319,14 @@ func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStr func (b Service) Delete( ctx context.Context, request *objectV2.DeleteRequest) (*objectV2.DeleteResponse, error) { + spanClosed := false + _, span := tracing.StartSpanFromContext(ctx, "checkACL") + defer func() { + if !spanClosed { + span.End() + } + }() + cnr, err := getContainerIDFromRequest(request) if err != nil { return nil, err @@ -332,10 +374,21 @@ func (b Service) Delete( return nil, eACLErr(reqInfo, err) } + span.End() + spanClosed = true + return b.next.Delete(ctx, request) } func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetObjectRangeStream) error { + spanClosed := false + _, span := tracing.StartSpanFromContext(stream.Context(), "checkACL") + defer func() { + if !spanClosed { + span.End() + } + }() + cnr, err := getContainerIDFromRequest(request) if err != nil { return err @@ -383,6 +436,9 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb return eACLErr(reqInfo, err) } + span.End() + spanClosed = true + return b.next.GetRange(request, &rangeStreamBasicChecker{ checker: b.checker, GetObjectRangeStream: stream, @@ -393,6 +449,14 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb func (b Service) GetRangeHash( ctx context.Context, request *objectV2.GetRangeHashRequest) (*objectV2.GetRangeHashResponse, error) { + spanClosed := false + _, span := tracing.StartSpanFromContext(ctx, "checkACL") + defer func() { + if !spanClosed { + span.End() + } + }() + cnr, err := getContainerIDFromRequest(request) if err != nil { return nil, err @@ -440,10 +504,21 @@ func (b Service) GetRangeHash( return nil, eACLErr(reqInfo, err) } + span.End() + spanClosed = true + return b.next.GetRangeHash(ctx, request) } func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { + spanClosed := false + _, span := tracing.StartSpanFromContext(ctx, "checkACL") + defer func() { + if !spanClosed { + span.End() + } + }() + body := request.GetBody() if body == nil { return errEmptyBody @@ -512,6 +587,9 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe } } + span.End() + spanClosed = true + return p.next.Send(ctx, request) } From 57789802527e848577bea4e9567f6bb6587bef4d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 11 Apr 2023 18:21:13 +0300 Subject: [PATCH 0089/1943] [#135] signature: Add tracing Add tracing to verify request and sign response. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/sign.go | 6 +++--- pkg/services/util/sign.go | 31 ++++++++++++++++++++----------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 9d66c76ba..4eb5be365 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -54,7 +54,7 @@ func (s *getStreamSigner) Send(resp *object.GetResponse) error { } func (s *SignService) Get(req *object.GetRequest, stream GetObjectStream) error { - return s.sigSvc.HandleServerStreamRequest(req, + return s.sigSvc.HandleServerStreamRequest(stream.Context(), req, func(resp util.ResponseMessage) error { return stream.Send(resp.(*object.GetResponse)) }, @@ -126,7 +126,7 @@ func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { } func (s *SignService) Search(req *object.SearchRequest, stream SearchStream) error { - return s.sigSvc.HandleServerStreamRequest(req, + return s.sigSvc.HandleServerStreamRequest(stream.Context(), req, func(resp util.ResponseMessage) error { return stream.Send(resp.(*object.SearchResponse)) }, @@ -176,7 +176,7 @@ func (s *getRangeStreamSigner) Send(resp *object.GetRangeResponse) error { } func (s *SignService) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { - return s.sigSvc.HandleServerStreamRequest(req, + return s.sigSvc.HandleServerStreamRequest(stream.Context(), req, func(resp util.ResponseMessage) error { return stream.Send(resp.(*object.GetRangeResponse)) }, diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index cb4be3084..dbfde7051 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -67,8 +68,7 @@ func (s *RequestMessageStreamer) Send(ctx context.Context, req any) error { var err error - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { + if err = verifyRequestSignature(ctx, req); err != nil { err = fmt.Errorf("could not verify request: %w", err) } else { err = s.send(ctx, req) @@ -112,7 +112,7 @@ func (s *RequestMessageStreamer) CloseAndRecv(ctx context.Context) (ResponseMess setStatusV2(resp, err) } - if err = signResponse(s.key, resp, s.statusSupported); err != nil { + if err = signResponse(ctx, s.key, resp, s.statusSupported); err != nil { return nil, err } @@ -130,6 +130,7 @@ func (s *SignService) CreateRequestStreamer(sender RequestMessageWriter, closer } func (s *SignService) HandleServerStreamRequest( + ctx context.Context, req any, respWriter ResponseMessageWriter, blankResp ResponseConstructor, @@ -142,12 +143,11 @@ func (s *SignService) HandleServerStreamRequest( var err error - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { + if err = verifyRequestSignature(ctx, req); err != nil { err = fmt.Errorf("could not verify request: %w", err) } else { err = respWriterCaller(func(resp ResponseMessage) error { - if err := signResponse(s.key, resp, statusSupported); err != nil { + if err := signResponse(ctx, s.key, resp, statusSupported); err != nil { return err } @@ -164,7 +164,7 @@ func (s *SignService) HandleServerStreamRequest( setStatusV2(resp, err) - _ = signResponse(s.key, resp, false) // panics or returns nil with false arg + _ = signResponse(ctx, s.key, resp, false) // panics or returns nil with false arg return respWriter(resp) } @@ -183,8 +183,7 @@ func (s *SignService) HandleUnaryRequest(ctx context.Context, req any, handler U err error ) - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { + if err = verifyRequestSignature(ctx, req); err != nil { var sigErr apistatus.SignatureVerification sigErr.SetMessage(err.Error()) @@ -205,7 +204,7 @@ func (s *SignService) HandleUnaryRequest(ctx context.Context, req any, handler U } // sign the response - if err = signResponse(s.key, resp, statusSupported); err != nil { + if err = signResponse(ctx, s.key, resp, statusSupported); err != nil { return nil, err } @@ -233,7 +232,10 @@ func setStatusV2(resp ResponseMessage, err error) { // The signature error affects the result depending on the protocol version: // - if status return is supported, panics since we cannot return the failed status, because it will not be signed; // - otherwise, returns error in order to transport it directly. -func signResponse(key *ecdsa.PrivateKey, resp any, statusSupported bool) error { +func signResponse(ctx context.Context, key *ecdsa.PrivateKey, resp any, statusSupported bool) error { + _, span := tracing.StartSpanFromContext(ctx, "signResponse") + defer span.End() + err := signature.SignServiceMessage(key, resp) if err != nil { err = fmt.Errorf("could not sign response: %w", err) @@ -247,3 +249,10 @@ func signResponse(key *ecdsa.PrivateKey, resp any, statusSupported bool) error { return err } + +func verifyRequestSignature(ctx context.Context, req any) error { + _, span := tracing.StartSpanFromContext(ctx, "verifyRequestSignature") + defer span.End() + + return signature.VerifyServiceMessage(req) +} From 5d2affa5cd104104d266e2bbff9534e71125789e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 12 Apr 2023 10:59:57 +0300 Subject: [PATCH 0090/1943] testutil: Fix linter warning Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/internal/testutil/generators.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/internal/testutil/generators.go b/pkg/local_object_storage/internal/testutil/generators.go index 4ee92db88..1a1f3cf9e 100644 --- a/pkg/local_object_storage/internal/testutil/generators.go +++ b/pkg/local_object_storage/internal/testutil/generators.go @@ -82,7 +82,7 @@ var _ ObjectGenerator = &RandObjGenerator{} func (g *RandObjGenerator) Next() *object.Object { var id oid.ID - rand.Read(id[:]) + _, _ = rand.Read(id[:]) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) } From 08769f413f207f8df65f9a92f4ce877ee7acb36b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 12 Apr 2023 16:54:04 +0300 Subject: [PATCH 0091/1943] Revert "[#135] acl: Add tracing spans" This reverts commit b2ca73054722fcc4f73746d75b1599377c82040a. --- pkg/services/object/acl/v2/service.go | 78 --------------------------- 1 file changed, 78 deletions(-) diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 93c1c65f8..6544d78d7 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -6,7 +6,6 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" @@ -112,14 +111,6 @@ func New(opts ...Option) Service { // Get implements ServiceServer interface, makes ACL checks and calls // next Get method in the ServiceServer pipeline. func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream) error { - spanClosed := false - _, span := tracing.StartSpanFromContext(stream.Context(), "checkACL") - defer func() { - if !spanClosed { - span.End() - } - }() - cnr, err := getContainerIDFromRequest(request) if err != nil { return err @@ -167,9 +158,6 @@ func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream return eACLErr(reqInfo, err) } - span.End() - spanClosed = true - return b.next.Get(request, &getStreamBasicChecker{ GetObjectStream: stream, info: reqInfo, @@ -189,14 +177,6 @@ func (b Service) Put() (object.PutObjectStream, error) { func (b Service) Head( ctx context.Context, request *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { - spanClosed := false - _, span := tracing.StartSpanFromContext(ctx, "checkACL") - defer func() { - if !spanClosed { - span.End() - } - }() - cnr, err := getContainerIDFromRequest(request) if err != nil { return nil, err @@ -244,9 +224,6 @@ func (b Service) Head( return nil, eACLErr(reqInfo, err) } - span.End() - spanClosed = true - resp, err := b.next.Head(ctx, request) if err == nil { if err = b.checker.CheckEACL(resp, reqInfo); err != nil { @@ -258,14 +235,6 @@ func (b Service) Head( } func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStream) error { - spanClosed := false - _, span := tracing.StartSpanFromContext(stream.Context(), "checkACL") - defer func() { - if !spanClosed { - span.End() - } - }() - id, err := getContainerIDFromRequest(request) if err != nil { return err @@ -306,9 +275,6 @@ func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStr return eACLErr(reqInfo, err) } - span.End() - spanClosed = true - return b.next.Search(request, &searchStreamBasicChecker{ checker: b.checker, SearchStream: stream, @@ -319,14 +285,6 @@ func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStr func (b Service) Delete( ctx context.Context, request *objectV2.DeleteRequest) (*objectV2.DeleteResponse, error) { - spanClosed := false - _, span := tracing.StartSpanFromContext(ctx, "checkACL") - defer func() { - if !spanClosed { - span.End() - } - }() - cnr, err := getContainerIDFromRequest(request) if err != nil { return nil, err @@ -374,21 +332,10 @@ func (b Service) Delete( return nil, eACLErr(reqInfo, err) } - span.End() - spanClosed = true - return b.next.Delete(ctx, request) } func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetObjectRangeStream) error { - spanClosed := false - _, span := tracing.StartSpanFromContext(stream.Context(), "checkACL") - defer func() { - if !spanClosed { - span.End() - } - }() - cnr, err := getContainerIDFromRequest(request) if err != nil { return err @@ -436,9 +383,6 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb return eACLErr(reqInfo, err) } - span.End() - spanClosed = true - return b.next.GetRange(request, &rangeStreamBasicChecker{ checker: b.checker, GetObjectRangeStream: stream, @@ -449,14 +393,6 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb func (b Service) GetRangeHash( ctx context.Context, request *objectV2.GetRangeHashRequest) (*objectV2.GetRangeHashResponse, error) { - spanClosed := false - _, span := tracing.StartSpanFromContext(ctx, "checkACL") - defer func() { - if !spanClosed { - span.End() - } - }() - cnr, err := getContainerIDFromRequest(request) if err != nil { return nil, err @@ -504,21 +440,10 @@ func (b Service) GetRangeHash( return nil, eACLErr(reqInfo, err) } - span.End() - spanClosed = true - return b.next.GetRangeHash(ctx, request) } func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { - spanClosed := false - _, span := tracing.StartSpanFromContext(ctx, "checkACL") - defer func() { - if !spanClosed { - span.End() - } - }() - body := request.GetBody() if body == nil { return errEmptyBody @@ -587,9 +512,6 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe } } - span.End() - spanClosed = true - return p.next.Send(ctx, request) } From 04727ce1d6fd96646698a7c3357d1477ed24836b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 12 Apr 2023 16:56:56 +0300 Subject: [PATCH 0092/1943] Revert "[#135] signature: Add tracing" This reverts commit 57789802527e848577bea4e9567f6bb6587bef4d. --- pkg/services/object/sign.go | 6 +++--- pkg/services/util/sign.go | 31 +++++++++++-------------------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 4eb5be365..9d66c76ba 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -54,7 +54,7 @@ func (s *getStreamSigner) Send(resp *object.GetResponse) error { } func (s *SignService) Get(req *object.GetRequest, stream GetObjectStream) error { - return s.sigSvc.HandleServerStreamRequest(stream.Context(), req, + return s.sigSvc.HandleServerStreamRequest(req, func(resp util.ResponseMessage) error { return stream.Send(resp.(*object.GetResponse)) }, @@ -126,7 +126,7 @@ func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { } func (s *SignService) Search(req *object.SearchRequest, stream SearchStream) error { - return s.sigSvc.HandleServerStreamRequest(stream.Context(), req, + return s.sigSvc.HandleServerStreamRequest(req, func(resp util.ResponseMessage) error { return stream.Send(resp.(*object.SearchResponse)) }, @@ -176,7 +176,7 @@ func (s *getRangeStreamSigner) Send(resp *object.GetRangeResponse) error { } func (s *SignService) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { - return s.sigSvc.HandleServerStreamRequest(stream.Context(), req, + return s.sigSvc.HandleServerStreamRequest(req, func(resp util.ResponseMessage) error { return stream.Send(resp.(*object.GetRangeResponse)) }, diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index dbfde7051..cb4be3084 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -68,7 +67,8 @@ func (s *RequestMessageStreamer) Send(ctx context.Context, req any) error { var err error - if err = verifyRequestSignature(ctx, req); err != nil { + // verify request signatures + if err = signature.VerifyServiceMessage(req); err != nil { err = fmt.Errorf("could not verify request: %w", err) } else { err = s.send(ctx, req) @@ -112,7 +112,7 @@ func (s *RequestMessageStreamer) CloseAndRecv(ctx context.Context) (ResponseMess setStatusV2(resp, err) } - if err = signResponse(ctx, s.key, resp, s.statusSupported); err != nil { + if err = signResponse(s.key, resp, s.statusSupported); err != nil { return nil, err } @@ -130,7 +130,6 @@ func (s *SignService) CreateRequestStreamer(sender RequestMessageWriter, closer } func (s *SignService) HandleServerStreamRequest( - ctx context.Context, req any, respWriter ResponseMessageWriter, blankResp ResponseConstructor, @@ -143,11 +142,12 @@ func (s *SignService) HandleServerStreamRequest( var err error - if err = verifyRequestSignature(ctx, req); err != nil { + // verify request signatures + if err = signature.VerifyServiceMessage(req); err != nil { err = fmt.Errorf("could not verify request: %w", err) } else { err = respWriterCaller(func(resp ResponseMessage) error { - if err := signResponse(ctx, s.key, resp, statusSupported); err != nil { + if err := signResponse(s.key, resp, statusSupported); err != nil { return err } @@ -164,7 +164,7 @@ func (s *SignService) HandleServerStreamRequest( setStatusV2(resp, err) - _ = signResponse(ctx, s.key, resp, false) // panics or returns nil with false arg + _ = signResponse(s.key, resp, false) // panics or returns nil with false arg return respWriter(resp) } @@ -183,7 +183,8 @@ func (s *SignService) HandleUnaryRequest(ctx context.Context, req any, handler U err error ) - if err = verifyRequestSignature(ctx, req); err != nil { + // verify request signatures + if err = signature.VerifyServiceMessage(req); err != nil { var sigErr apistatus.SignatureVerification sigErr.SetMessage(err.Error()) @@ -204,7 +205,7 @@ func (s *SignService) HandleUnaryRequest(ctx context.Context, req any, handler U } // sign the response - if err = signResponse(ctx, s.key, resp, statusSupported); err != nil { + if err = signResponse(s.key, resp, statusSupported); err != nil { return nil, err } @@ -232,10 +233,7 @@ func setStatusV2(resp ResponseMessage, err error) { // The signature error affects the result depending on the protocol version: // - if status return is supported, panics since we cannot return the failed status, because it will not be signed; // - otherwise, returns error in order to transport it directly. -func signResponse(ctx context.Context, key *ecdsa.PrivateKey, resp any, statusSupported bool) error { - _, span := tracing.StartSpanFromContext(ctx, "signResponse") - defer span.End() - +func signResponse(key *ecdsa.PrivateKey, resp any, statusSupported bool) error { err := signature.SignServiceMessage(key, resp) if err != nil { err = fmt.Errorf("could not sign response: %w", err) @@ -249,10 +247,3 @@ func signResponse(ctx context.Context, key *ecdsa.PrivateKey, resp any, statusSu return err } - -func verifyRequestSignature(ctx context.Context, req any) error { - _, span := tracing.StartSpanFromContext(ctx, "verifyRequestSignature") - defer span.End() - - return signature.VerifyServiceMessage(req) -} From 7d39fecc6a84b19bb45af2b2cbf7298aedf8fa1f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 12 Apr 2023 10:45:43 +0300 Subject: [PATCH 0093/1943] Release v0.36.0 Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 9 +++++++++ VERSION | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e07bb2f2..b4d6e7ca0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,15 @@ Changelog for FrostFS Node ## [Unreleased] +### Added +### Changed +### Fixed +### Removed +### Updated +### Updating from v0.36.0 + +## [v0.36.0] - 2023-04-12 - Furtwängler + ### Added - Add GAS pouring mechanism for a configurable list of wallets (#128) - Separate batching for replicated operations over the same container in pilorama (#1621) diff --git a/VERSION b/VERSION index ab4e51c67..e1d6235d3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.35.0 +v0.36.0 From 01c0c90a869ef9f77ed412bd5a26c539579dacf7 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 6 Apr 2023 18:22:59 +0300 Subject: [PATCH 0094/1943] [#113] cli: add "name" option for "get container" command * Make get container command filter out the container by attribute name Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-cli/modules/container/list.go | 47 +++++++++++++++-------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 9565748c3..33dd17943 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/spf13/cobra" ) @@ -16,12 +17,14 @@ import ( const ( flagListPrintAttr = "with-attr" flagListContainerOwner = "owner" + flagListName = "name" ) // flag vars of list command. var ( flagVarListPrintAttr bool flagVarListContainerOwner string + flagVarListName string ) var listContainersCmd = &cobra.Command{ @@ -52,24 +55,33 @@ var listContainersCmd = &cobra.Command{ var prmGet internalclient.GetContainerPrm prmGet.SetClient(cli) - list := res.IDList() - for i := range list { - cmd.Println(list[i].String()) + containerIDs := res.IDList() + for _, cnrID := range containerIDs { + if flagVarListName == "" && !flagVarListPrintAttr { + cmd.Println(cnrID.String()) + continue + } + + prmGet.SetContainer(cnrID) + res, err := internalclient.GetContainer(prmGet) + if err != nil { + cmd.Printf(" failed to read attributes: %v\n", err) + continue + } + + cnr := res.Container() + if cnrName := containerSDK.Name(cnr); flagVarListName != "" && cnrName != flagVarListName { + continue + } + cmd.Println(cnrID.String()) if flagVarListPrintAttr { - prmGet.SetContainer(list[i]) - - res, err := internalclient.GetContainer(prmGet) - if err == nil { - res.Container().IterateAttributes(func(key, val string) { - if !strings.HasPrefix(key, container.SysAttributePrefix) && !strings.HasPrefix(key, container.SysAttributePrefixNeoFS) { - // FIXME(@cthulhu-rider): neofs-sdk-go#314 use dedicated method to skip system attributes - cmd.Printf(" %s: %s\n", key, val) - } - }) - } else { - cmd.Printf(" failed to read attributes: %v\n", err) - } + cnr.IterateAttributes(func(key, val string) { + if !strings.HasPrefix(key, container.SysAttributePrefix) && !strings.HasPrefix(key, container.SysAttributePrefixNeoFS) { + // FIXME(@cthulhu-rider): neofs-sdk-go#314 use dedicated method to skip system attributes + cmd.Printf(" %s: %s\n", key, val) + } + }) } } }, @@ -80,6 +92,9 @@ func initContainerListContainersCmd() { flags := listContainersCmd.Flags() + flags.StringVar(&flagVarListName, flagListName, "", + "List containers by the attribute name", + ) flags.StringVar(&flagVarListContainerOwner, flagListContainerOwner, "", "Owner of containers (omit to use owner from private key)", ) From 2f1beddfd3b9ef3d649321eda2d11b32abf3475e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 12 Apr 2023 13:52:14 +0300 Subject: [PATCH 0095/1943] [#202] adm: Remove deprecated warnings in tests `VerifyBlocks` is now `SkipBlockVerification` and is false by default. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/initialize_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index 39a35b12e..fb2dc3e3f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strconv" "testing" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/config" @@ -101,11 +102,10 @@ func generateTestData(t *testing.T, dir string, size int) { cfg := config.Config{} cfg.ProtocolConfiguration.Magic = 12345 cfg.ProtocolConfiguration.ValidatorsCount = size - cfg.ProtocolConfiguration.SecondsPerBlock = 1 //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 + cfg.ProtocolConfiguration.TimePerBlock = time.Second cfg.ProtocolConfiguration.StandbyCommittee = pubs // sorted by glagolic letters cfg.ProtocolConfiguration.P2PSigExtensions = true cfg.ProtocolConfiguration.VerifyTransactions = true - cfg.ProtocolConfiguration.VerifyBlocks = true //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 data, err := yaml.Marshal(cfg) require.NoError(t, err) From 96b38f7e86afbb8c9ad13fb174c34f76633b259f Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 17 Feb 2023 21:28:35 +0300 Subject: [PATCH 0096/1943] [#239] morph/client: Add CalledByEntry into the "grouped" scope Fixes #2230, fixes #2263. CustomGroups are nice while we're only calling NeoFS contracts, but it doesn't work at all for standard ones like GAS or Notary. Signed-off-by: Roman Khimov Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + pkg/morph/client/nns.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4d6e7ca0..b13469d06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ Changelog for FrostFS Node - Iterating over just removed files by FSTree (#98) - Parts of a locked object could not be removed anymore (#141) - Non-alphabet nodes do not try to handle alphabet events (#181) +- Failing SN and IR transactions because of incorrect scopes (#2230, #2263) ### Removed ### Updated diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index 0a23aa47a..c3b900ddb 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -208,7 +208,7 @@ func (c *Client) SetGroupSignerScope() error { return err } - c.signer.Scopes = transaction.CustomGroups + c.signer.Scopes = transaction.CustomGroups | transaction.CalledByEntry c.signer.AllowedGroups = []*keys.PublicKey{pub} return nil } From be4df989e5151b0031f64c09f26f9c49325fa199 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 17 Feb 2023 22:27:29 +0300 Subject: [PATCH 0097/1943] [#239] morph/client: Deduplicate signers in Client a bit One signer in the cfg is enough. Signed-off-by: Roman Khimov Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 2 -- pkg/morph/client/constructor.go | 1 - pkg/morph/client/nns.go | 6 ++++-- pkg/morph/client/notary.go | 12 ++++++------ 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index b93c5f75f..5e98211c4 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -57,8 +57,6 @@ type Client struct { acc *wallet.Account // neo account accAddr util.Uint160 // account's address - signer *transaction.Signer - notary *notaryInfo cfg cfg diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 9ed275029..c4ec70171 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -105,7 +105,6 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er logger: cfg.logger, acc: acc, accAddr: accAddr, - signer: cfg.signer, cfg: *cfg, switchLock: &sync.RWMutex{}, notifications: make(chan rpcclient.Notification), diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index c3b900ddb..e61cb8e15 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -208,8 +208,10 @@ func (c *Client) SetGroupSignerScope() error { return err } - c.signer.Scopes = transaction.CustomGroups | transaction.CalledByEntry - c.signer.AllowedGroups = []*keys.PublicKey{pub} + c.cfg.signer = &transaction.Signer{ + Scopes: transaction.CustomGroups | transaction.CalledByEntry, + AllowedGroups: []*keys.PublicKey{pub}, + } return nil } diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 96dca0319..069c35782 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -596,18 +596,18 @@ func (c *Client) notaryCosigners(invokedByAlpha bool, ir []*keys.PublicKey, comm s = append(s, transaction.Signer{ Account: hash.Hash160(multisigScript), - Scopes: c.signer.Scopes, - AllowedContracts: c.signer.AllowedContracts, - AllowedGroups: c.signer.AllowedGroups, + Scopes: c.cfg.signer.Scopes, + AllowedContracts: c.cfg.signer.AllowedContracts, + AllowedGroups: c.cfg.signer.AllowedGroups, }) if !invokedByAlpha { // then we have invoker signature s = append(s, transaction.Signer{ Account: hash.Hash160(c.acc.GetVerificationScript()), - Scopes: c.signer.Scopes, - AllowedContracts: c.signer.AllowedContracts, - AllowedGroups: c.signer.AllowedGroups, + Scopes: c.cfg.signer.Scopes, + AllowedContracts: c.cfg.signer.AllowedContracts, + AllowedGroups: c.cfg.signer.AllowedGroups, }) } From f41ad9d419d925195656ccf56ff764ad3ef57f75 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 17 Feb 2023 22:32:31 +0300 Subject: [PATCH 0098/1943] [#239] morph/client: Recreate actor/wrappers in SetGroupSignerScope That's the reason #2230 and #2263 were not detected earlier, we actually had Global scope being used before reconnection to RPC node. Signed-off-by: Roman Khimov Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + pkg/morph/client/nns.go | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b13469d06..270d0265e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,7 @@ Changelog for FrostFS Node - Parts of a locked object could not be removed anymore (#141) - Non-alphabet nodes do not try to handle alphabet events (#181) - Failing SN and IR transactions because of incorrect scopes (#2230, #2263) +- Global scope used for some transactions (#2230, #2263) ### Removed ### Updated diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index e61cb8e15..473b3500b 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -208,10 +208,18 @@ func (c *Client) SetGroupSignerScope() error { return err } - c.cfg.signer = &transaction.Signer{ + // Don't change c before everything is OK. + cfg := c.cfg + cfg.signer = &transaction.Signer{ Scopes: transaction.CustomGroups | transaction.CalledByEntry, AllowedGroups: []*keys.PublicKey{pub}, } + rpcActor, err := newActor(c.client, c.acc, cfg) + if err != nil { + return err + } + c.cfg = cfg + c.setActor(rpcActor) return nil } From d686ab49e80f33fa91b213ff9ceeb3caecff2597 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 12 Apr 2023 13:58:54 +0300 Subject: [PATCH 0099/1943] [#202] adm: Remove deprecated RPC client methods Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/initialize_nns.go | 8 +++++--- .../internal/modules/morph/initialize_register.go | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go index edb7d6de5..15657a6d9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go @@ -15,6 +15,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + nnsClient "github.com/nspcc-dev/neo-go/pkg/rpcclient/nns" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/util" @@ -284,10 +285,11 @@ func parseNNSResolveResult(res stackitem.Item) (util.Uint160, error) { } func nnsIsAvailable(c Client, nnsHash util.Uint160, name string) (bool, error) { - switch ct := c.(type) { + switch c.(type) { case *rpcclient.Client: - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - return ct.NNSIsAvailable(nnsHash, name) + inv := invoker.New(c, nil) + reader := nnsClient.NewReader(inv, nnsHash) + return reader.IsAvailable(name) default: b, err := unwrap.Bool(invokeFunction(c, nnsHash, "isAvailable", []any{name}, nil)) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index 27e1590cf..b1542cc92 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -9,6 +9,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/neo" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" @@ -116,10 +117,11 @@ func (c *initializeContext) transferNEOFinished(neoHash util.Uint160) (bool, err var errGetPriceInvalid = errors.New("`getRegisterPrice`: invalid response") func (c *initializeContext) getCandidateRegisterPrice() (int64, error) { - switch ct := c.Client.(type) { + switch c.Client.(type) { case *rpcclient.Client: - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - return ct.GetCandidateRegisterPrice() + inv := invoker.New(c.Client, nil) + reader := neo.NewReader(inv) + return reader.GetRegisterPrice() default: neoHash := neo.Hash res, err := invokeFunction(c.Client, neoHash, "getRegisterPrice", nil, nil) From d29b13454f1837202e597b5179626ceee50160bb Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Fri, 17 Feb 2023 23:43:22 +0300 Subject: [PATCH 0100/1943] [#239] morph/client: Simplify code interacting with magic numbers It can't be uint64 in fact, but this error is buried deeply in the NetworkInfo API structure, so we're not touching MagicNumber() for now. Signed-off-by: Roman Khimov Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/notary.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 069c35782..7399c19cd 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -412,16 +412,11 @@ func (c *Client) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { return err } - // error appears only if client - // is in inactive mode; that has - // been already checked above - magicNumber, _ := c.MagicNumber() - // mainTX is expected to be pre-validated: second witness must exist and be empty mainTx.Scripts[1].VerificationScript = multiaddrAccount.GetVerificationScript() mainTx.Scripts[1].InvocationScript = append( []byte{byte(opcode.PUSHDATA1), 64}, - multiaddrAccount.PrivateKey().SignHashable(uint32(magicNumber), mainTx)..., + multiaddrAccount.SignHashable(c.rpcActor.GetNetwork(), mainTx)..., ) //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 @@ -667,12 +662,12 @@ func (c *Client) notaryWitnesses(invokedByAlpha bool, multiaddr *wallet.Account, // to pass Notary module verification var invokeScript []byte - magicNumber, _ := c.MagicNumber() + magicNumber := c.rpcActor.GetNetwork() if invokedByAlpha { invokeScript = append( []byte{byte(opcode.PUSHDATA1), 64}, - multiaddr.PrivateKey().SignHashable(uint32(magicNumber), tx)..., + multiaddr.SignHashable(magicNumber, tx)..., ) } else { // we can't provide alphabet node signature @@ -694,7 +689,7 @@ func (c *Client) notaryWitnesses(invokedByAlpha bool, multiaddr *wallet.Account, // then we have invoker witness invokeScript = append( []byte{byte(opcode.PUSHDATA1), 64}, - c.acc.PrivateKey().SignHashable(uint32(magicNumber), tx)..., + c.acc.SignHashable(magicNumber, tx)..., ) w = append(w, transaction.Witness{ From 0e31c12e63d1082cca5775b661104c5b47825326 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 12 Apr 2023 17:35:10 +0300 Subject: [PATCH 0101/1943] [#240] logs: Move log messages to constants Drop duplicate entities. Format entities. Signed-off-by: Dmitrii Stepanov Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-ir/main.go | 9 +- cmd/frostfs-node/config.go | 41 +- cmd/frostfs-node/container.go | 13 +- cmd/frostfs-node/control.go | 3 +- cmd/frostfs-node/grpc.go | 15 +- cmd/frostfs-node/main.go | 5 +- cmd/frostfs-node/morph.go | 21 +- cmd/frostfs-node/netmap.go | 9 +- cmd/frostfs-node/notificator.go | 11 +- cmd/frostfs-node/object.go | 7 +- cmd/frostfs-node/reputation.go | 9 +- cmd/frostfs-node/reputation/common/remote.go | 7 +- .../reputation/intermediate/consumers.go | 3 +- .../reputation/intermediate/contract.go | 3 +- .../reputation/intermediate/daughters.go | 3 +- .../reputation/intermediate/remote.go | 3 +- cmd/frostfs-node/reputation/local/remote.go | 3 +- cmd/frostfs-node/reputation/local/storage.go | 3 +- cmd/frostfs-node/tracing.go | 5 +- cmd/frostfs-node/tree.go | 11 +- internal/logs/logs.go | 643 ++++++++++++++++++ pkg/innerring/blocktimer.go | 5 +- pkg/innerring/initialization.go | 13 +- pkg/innerring/innerring.go | 13 +- pkg/innerring/notary.go | 7 +- pkg/innerring/processors/alphabet/handlers.go | 5 +- .../processors/alphabet/process_emit.go | 19 +- .../processors/alphabet/processor.go | 3 +- pkg/innerring/processors/audit/handlers.go | 5 +- pkg/innerring/processors/audit/process.go | 21 +- pkg/innerring/processors/audit/scheduler.go | 3 +- pkg/innerring/processors/balance/handlers.go | 5 +- .../processors/balance/process_assets.go | 5 +- pkg/innerring/processors/balance/processor.go | 3 +- .../processors/container/handlers.go | 13 +- .../processors/container/process_container.go | 13 +- .../processors/container/process_eacl.go | 7 +- .../processors/container/processor.go | 3 +- pkg/innerring/processors/frostfs/handlers.go | 25 +- .../processors/frostfs/process_assets.go | 23 +- .../processors/frostfs/process_bind.go | 7 +- .../processors/frostfs/process_config.go | 5 +- pkg/innerring/processors/frostfs/processor.go | 3 +- .../processors/governance/handlers.go | 5 +- .../processors/governance/process_update.go | 29 +- pkg/innerring/processors/netmap/handlers.go | 27 +- .../processors/netmap/process_cleanup.go | 11 +- .../processors/netmap/process_epoch.go | 17 +- .../processors/netmap/process_peers.go | 35 +- pkg/innerring/processors/netmap/processor.go | 3 +- .../processors/reputation/handlers.go | 5 +- .../processors/reputation/process_put.go | 11 +- .../processors/reputation/processor.go | 3 +- .../processors/settlement/audit/calculate.go | 47 +- .../processors/settlement/basic/collect.go | 9 +- .../processors/settlement/basic/distribute.go | 5 +- pkg/innerring/processors/settlement/calls.go | 29 +- .../processors/settlement/handlers.go | 9 +- .../processors/settlement/processor.go | 3 +- pkg/innerring/rpc.go | 5 +- pkg/innerring/settlement.go | 5 +- pkg/innerring/state.go | 13 +- pkg/innerring/subnet.go | 23 +- .../blobovnicza/control.go | 13 +- .../blobovnicza/delete.go | 3 +- .../blobstor/blobovniczatree/blobovnicza.go | 11 +- .../blobstor/blobovniczatree/control.go | 11 +- .../blobstor/blobovniczatree/delete.go | 7 +- .../blobstor/blobovniczatree/exists.go | 3 +- .../blobstor/blobovniczatree/get.go | 7 +- .../blobstor/blobovniczatree/get_range.go | 7 +- .../blobstor/blobovniczatree/put.go | 15 +- pkg/local_object_storage/blobstor/control.go | 9 +- pkg/local_object_storage/blobstor/exists.go | 3 +- pkg/local_object_storage/blobstor/iterate.go | 3 +- pkg/local_object_storage/engine/control.go | 15 +- pkg/local_object_storage/engine/delete.go | 5 +- pkg/local_object_storage/engine/engine.go | 11 +- pkg/local_object_storage/engine/evacuate.go | 7 +- pkg/local_object_storage/engine/inhume.go | 7 +- pkg/local_object_storage/engine/put.go | 5 +- .../engine/remove_copies.go | 9 +- pkg/local_object_storage/engine/shards.go | 5 +- pkg/local_object_storage/metabase/control.go | 7 +- pkg/local_object_storage/metabase/select.go | 17 +- pkg/local_object_storage/shard/control.go | 15 +- pkg/local_object_storage/shard/delete.go | 7 +- pkg/local_object_storage/shard/gc.go | 43 +- pkg/local_object_storage/shard/get.go | 7 +- pkg/local_object_storage/shard/inhume.go | 3 +- pkg/local_object_storage/shard/list.go | 3 +- pkg/local_object_storage/shard/mode.go | 5 +- pkg/local_object_storage/shard/move.go | 3 +- pkg/local_object_storage/shard/put.go | 3 +- pkg/local_object_storage/shard/shard.go | 7 +- pkg/local_object_storage/writecache/flush.go | 3 +- pkg/local_object_storage/writecache/init.go | 9 +- pkg/local_object_storage/writecache/mode.go | 3 +- .../writecache/storage.go | 7 +- pkg/morph/client/client.go | 11 +- pkg/morph/client/multi.go | 15 +- pkg/morph/client/notary.go | 9 +- pkg/morph/client/notifications.go | 7 +- pkg/morph/event/listener.go | 69 +- pkg/morph/event/utils.go | 3 +- pkg/morph/subscriber/subscriber.go | 15 +- pkg/services/audit/auditor/context.go | 7 +- pkg/services/audit/auditor/pdp.go | 5 +- pkg/services/audit/auditor/pop.go | 5 +- pkg/services/audit/auditor/por.go | 11 +- pkg/services/audit/taskmanager/listen.go | 13 +- .../announcement/load/controller/calls.go | 33 +- .../announcement/load/route/calls.go | 9 +- pkg/services/notificator/nats/service.go | 7 +- pkg/services/notificator/service.go | 5 +- pkg/services/object/acl/v2/classifier.go | 5 +- pkg/services/object/delete/container.go | 4 +- pkg/services/object/delete/delete.go | 7 +- pkg/services/object/delete/exec.go | 21 +- pkg/services/object/delete/local.go | 13 +- pkg/services/object/get/assemble.go | 11 +- pkg/services/object/get/container.go | 13 +- pkg/services/object/get/exec.go | 11 +- pkg/services/object/get/get.go | 13 +- pkg/services/object/get/local.go | 3 +- pkg/services/object/get/remote.go | 5 +- pkg/services/object/put/distributed.go | 3 +- pkg/services/object/search/container.go | 17 +- pkg/services/object/search/exec.go | 7 +- pkg/services/object/search/local.go | 3 +- pkg/services/object/search/search.go | 7 +- pkg/services/object/util/log.go | 5 +- .../object_manager/tombstone/checker.go | 3 +- pkg/services/policer/check.go | 17 +- pkg/services/policer/process.go | 9 +- pkg/services/replicator/process.go | 9 +- pkg/services/reputation/common/managers.go | 3 +- .../reputation/common/router/calls.go | 9 +- .../reputation/eigentrust/calculator/calls.go | 37 +- .../reputation/eigentrust/controller/calls.go | 5 +- .../reputation/eigentrust/routes/calls.go | 3 +- .../reputation/local/controller/calls.go | 19 +- pkg/services/reputation/local/routes/calls.go | 3 +- pkg/services/session/executor.go | 3 +- .../session/storage/persistent/storage.go | 7 +- pkg/services/tree/redirect.go | 3 +- pkg/services/tree/replicator.go | 9 +- pkg/services/tree/signature.go | 3 +- pkg/services/tree/sync.go | 29 +- 149 files changed, 1481 insertions(+), 687 deletions(-) create mode 100644 internal/logs/logs.go diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index e4386a083..5db1db6b6 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -9,6 +9,7 @@ import ( "os/signal" "syscall" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" @@ -80,13 +81,13 @@ func main() { err = innerRing.Start(ctx, intErr) exitErr(err) - log.Info("application started", + log.Info(logs.CommonApplicationStarted, zap.String("version", misc.Version)) select { case <-ctx.Done(): case err := <-intErr: - log.Info("internal error", zap.String("msg", err.Error())) + log.Info(logs.FrostFSIRInternalError, zap.String("msg", err.Error())) } innerRing.Stop() @@ -98,14 +99,14 @@ func main() { go func() { err := srv.Shutdown() if err != nil { - log.Debug("could not shutdown HTTP server", + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error()), ) } }() } - log.Info("application stopped") + log.Info(logs.FrostFSIRApplicationStopped) } func initHTTPServers(cfg *viper.Viper, log *logger.Logger) []*httputil.Server { diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d81e47b17..d110665f5 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -29,6 +29,7 @@ import ( objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -342,13 +343,13 @@ type internals struct { func (c *cfg) startMaintenance() { c.isMaintenance.Store(true) c.cfgNetmap.state.setControlNetmapStatus(control.NetmapStatus_MAINTENANCE) - c.log.Info("started local node's maintenance") + c.log.Info(logs.FrostFSNodeStartedLocalNodesMaintenance) } // stops node's maintenance. func (c *internals) stopMaintenance() { c.isMaintenance.Store(false) - c.log.Info("stopped local node's maintenance") + c.log.Info(logs.FrostFSNodeStoppedLocalNodesMaintenance) } // IsMaintenance checks if storage node is under maintenance. @@ -881,10 +882,10 @@ func initLocalStorage(c *cfg) { for _, optsWithMeta := range c.shardOpts() { id, err := ls.AddShard(append(optsWithMeta.shOpts, shard.WithTombstoneSource(tombstoneSource))...) if err != nil { - c.log.Error("failed to attach shard to engine", zap.Error(err)) + c.log.Error(logs.FrostFSNodeFailedToAttachShardToEngine, zap.Error(err)) } else { shardsAttached++ - c.log.Info("shard attached to engine", zap.Stringer("id", id)) + c.log.Info(logs.FrostFSNodeShardAttachedToEngine, zap.Stringer("id", id)) } } if shardsAttached == 0 { @@ -894,15 +895,15 @@ func initLocalStorage(c *cfg) { c.cfgObject.cfgLocalStorage.localStorage = ls c.onShutdown(func() { - c.log.Info("closing components of the storage engine...") + c.log.Info(logs.FrostFSNodeClosingComponentsOfTheStorageEngine) err := ls.Close() if err != nil { - c.log.Info("storage engine closing failure", + c.log.Info(logs.FrostFSNodeStorageEngineClosingFailure, zap.String("error", err.Error()), ) } else { - c.log.Info("all components of the storage engine closed successfully") + c.log.Info(logs.FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully) } }) } @@ -976,11 +977,11 @@ func (c *cfg) bootstrap() error { // switch to online except when under maintenance st := c.cfgNetmap.state.controlNetmapStatus() if st == control.NetmapStatus_MAINTENANCE { - c.log.Info("bootstrapping with the maintenance state") + c.log.Info(logs.FrostFSNodeBootstrappingWithTheMaintenanceState) return c.bootstrapWithState((*netmap.NodeInfo).SetMaintenance) } - c.log.Info("bootstrapping with online state", + c.log.Info(logs.FrostFSNodeBootstrappingWithOnlineState, zap.Stringer("previous", st), ) @@ -1015,32 +1016,32 @@ func (c *cfg) signalWatcher(ctx context.Context) { case syscall.SIGHUP: c.reloadConfig(ctx) case syscall.SIGTERM, syscall.SIGINT: - c.log.Info("termination signal has been received, stopping...") + c.log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) // TODO (@acid-ant): #49 need to cover case when stuck at the middle(node health UNDEFINED or STARTING) c.shutdown() - c.log.Info("termination signal processing is complete") + c.log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) return } case err := <-c.internalErr: // internal application error - c.log.Warn("internal application error", + c.log.Warn(logs.FrostFSNodeInternalApplicationError, zap.String("message", err.Error())) c.shutdown() - c.log.Info("internal error processing is complete") + c.log.Info(logs.FrostFSNodeInternalErrorProcessingIsComplete) return } } } func (c *cfg) reloadConfig(ctx context.Context) { - c.log.Info("SIGHUP has been received, rereading configuration...") + c.log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) err := c.readConfig(c.appCfg) if err != nil { - c.log.Error("configuration reading", zap.Error(err)) + c.log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) return } @@ -1052,7 +1053,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { logPrm, err := c.loggerPrm() if err != nil { - c.log.Error("logger configuration preparation", zap.Error(err)) + c.log.Error(logs.FrostFSNodeLoggerConfigurationPreparation, zap.Error(err)) return } @@ -1060,7 +1061,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { components = append(components, dCmp{"tracing", func() error { updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg)) if updated { - c.log.Info("tracing configation updated") + c.log.Info(logs.FrostFSNodeTracingConfigationUpdated) } return err }}) @@ -1085,20 +1086,20 @@ func (c *cfg) reloadConfig(ctx context.Context) { err = c.cfgObject.cfgLocalStorage.localStorage.Reload(ctx, rcfg) if err != nil { - c.log.Error("storage engine configuration update", zap.Error(err)) + c.log.Error(logs.FrostFSNodeStorageEngineConfigurationUpdate, zap.Error(err)) return } for _, component := range components { err = component.reloadFunc() if err != nil { - c.log.Error("updated configuration applying", + c.log.Error(logs.FrostFSNodeUpdatedConfigurationApplying, zap.String("component", component.name), zap.Error(err)) } } - c.log.Info("configuration has been reloaded successfully") + c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } func (c *cfg) shutdown() { diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 6c864431d..d5d8601e3 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -11,6 +11,7 @@ import ( containerV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -136,13 +137,13 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c } else { // unlike removal, we expect successful receive of the container // after successful creation, so logging can be useful - c.log.Error("read newly created container after the notification", + c.log.Error(logs.FrostFSNodeReadNewlyCreatedContainerAfterTheNotification, zap.Stringer("id", ev.ID), zap.Error(err), ) } - c.log.Debug("container creation event's receipt", + c.log.Debug(logs.FrostFSNodeContainerCreationEventsReceipt, zap.Stringer("id", ev.ID), ) }) @@ -161,7 +162,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c cachedContainerStorage.handleRemoval(ev.ID) - c.log.Debug("container removal event's receipt", + c.log.Debug(logs.FrostFSNodeContainerRemovalEventsReceipt, zap.Stringer("id", ev.ID), ) }) @@ -295,7 +296,7 @@ type morphLoadWriter struct { } func (w *morphLoadWriter) Put(a containerSDK.SizeEstimation) error { - w.log.Debug("save used space announcement in contract", + w.log.Debug(logs.FrostFSNodeSaveUsedSpaceAnnouncementInContract, zap.Uint64("epoch", a.Epoch()), zap.Stringer("cid", a.Container()), zap.Uint64("size", a.Value()), @@ -458,7 +459,7 @@ func (d *localStorageLoad) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontr for i := range idList { sz, err := engine.ContainerSize(d.engine, idList[i]) if err != nil { - d.log.Debug("failed to calculate container size in storage engine", + d.log.Debug(logs.FrostFSNodeFailedToCalculateContainerSizeInStorageEngine, zap.Stringer("cid", idList[i]), zap.String("error", err.Error()), ) @@ -466,7 +467,7 @@ func (d *localStorageLoad) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontr continue } - d.log.Debug("container size in storage engine calculated successfully", + d.log.Debug(logs.FrostFSNodeContainerSizeInStorageEngineCalculatedSuccessfully, zap.Uint64("size", sz), zap.Stringer("cid", idList[i]), ) diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index 5492f585f..f4b068419 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -5,6 +5,7 @@ import ( "net" controlconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" controlSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" @@ -52,7 +53,7 @@ func initControlService(c *cfg) { lis, err := net.Listen("tcp", endpoint) if err != nil { - c.log.Error("can't listen gRPC endpoint (control)", zap.Error(err)) + c.log.Error(logs.FrostFSNodeCantListenGRPCEndpointControl, zap.Error(err)) return } diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index f3943f3ff..b0a587782 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" "google.golang.org/grpc" @@ -33,7 +34,7 @@ func initGRPC(c *cfg) { if tlsCfg != nil { cert, err := tls.LoadX509KeyPair(tlsCfg.CertificateFile(), tlsCfg.KeyFile()) if err != nil { - c.log.Error("could not read certificate from file", zap.Error(err)) + c.log.Error(logs.FrostFSNodeCouldNotReadCertificateFromFile, zap.Error(err)) return } @@ -63,7 +64,7 @@ func initGRPC(c *cfg) { lis, err := net.Listen("tcp", sc.Endpoint()) if err != nil { - c.log.Error("can't listen gRPC endpoint", zap.Error(err)) + c.log.Error(logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) return } @@ -93,14 +94,14 @@ func serveGRPC(c *cfg) { go func() { defer func() { - c.log.Info("stop listening gRPC endpoint", + c.log.Info(logs.FrostFSNodeStopListeningGRPCEndpoint, zap.String("endpoint", lis.Addr().String()), ) c.wg.Done() }() - c.log.Info("start listening gRPC endpoint", + c.log.Info(logs.FrostFSNodeStartListeningGRPCEndpoint, zap.String("endpoint", lis.Addr().String()), ) @@ -114,7 +115,7 @@ func serveGRPC(c *cfg) { func stopGRPC(name string, s *grpc.Server, l *logger.Logger) { l = &logger.Logger{Logger: l.With(zap.String("name", name))} - l.Info("stopping gRPC server...") + l.Info(logs.FrostFSNodeStoppingGRPCServer) // GracefulStop() may freeze forever, see #1270 done := make(chan struct{}) @@ -126,9 +127,9 @@ func stopGRPC(name string, s *grpc.Server, l *logger.Logger) { select { case <-done: case <-time.After(1 * time.Minute): - l.Info("gRPC cannot shutdown gracefully, forcing stop") + l.Info(logs.FrostFSNodeGRPCCannotShutdownGracefullyForcingStop) s.Stop() } - l.Info("gRPC server stopped successfully") + l.Info(logs.FrostFSNodeGRPCServerStoppedSuccessfully) } diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index a97ad3879..786843b0b 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -8,6 +8,7 @@ import ( "os" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "go.uber.org/zap" @@ -142,14 +143,14 @@ func bootUp(ctx context.Context, c *cfg) { } func wait(c *cfg, cancel func()) { - c.log.Info("application started", + c.log.Info(logs.CommonApplicationStarted, zap.String("version", misc.Version)) <-c.done // graceful shutdown cancel() - c.log.Debug("waiting for all processes to stop") + c.log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop) c.wg.Wait() } diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 2db865ca3..72378d8f3 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -7,6 +7,7 @@ import ( "time" morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -49,7 +50,7 @@ func initMorphComponents(ctx context.Context, c *cfg) { client.WithSwitchInterval(morphconfig.SwitchInterval(c.appCfg)), ) if err != nil { - c.log.Info("failed to create neo RPC client", + c.log.Info(logs.FrostFSNodeFailedToCreateNeoRPCClient, zap.Any("endpoints", addresses), zap.String("error", err.Error()), ) @@ -58,12 +59,12 @@ func initMorphComponents(ctx context.Context, c *cfg) { } c.onShutdown(func() { - c.log.Info("closing morph components...") + c.log.Info(logs.FrostFSNodeClosingMorphComponents) cli.Close() }) if err := cli.SetGroupSignerScope(); err != nil { - c.log.Info("failed to set group signer scope, continue with Global", zap.Error(err)) + c.log.Info(logs.FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal, zap.Error(err)) } c.cfgMorph.client = cli @@ -80,7 +81,7 @@ func initMorphComponents(ctx context.Context, c *cfg) { fatalOnErr(err) } - c.log.Info("notary support", + c.log.Info(logs.FrostFSNodeNotarySupport, zap.Bool("sidechain_enabled", c.cfgMorph.notaryEnabled), ) @@ -95,7 +96,7 @@ func initMorphComponents(ctx context.Context, c *cfg) { msPerBlock, err := c.cfgMorph.client.MsPerBlock() fatalOnErr(err) c.cfgMorph.cacheTTL = time.Duration(msPerBlock) * time.Millisecond - c.log.Debug("morph.cache_ttl fetched from network", zap.Duration("value", c.cfgMorph.cacheTTL)) + c.log.Debug(logs.FrostFSNodeMorphcacheTTLFetchedFromNetwork, zap.Duration("value", c.cfgMorph.cacheTTL)) } if c.cfgMorph.cacheTTL < 0 { @@ -122,7 +123,7 @@ func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) { // non-error deposit with an empty TX hash means // that the deposit has already been made; no // need to wait it. - c.log.Info("notary deposit has already been made") + c.log.Info(logs.FrostFSNodeNotaryDepositHasAlreadyBeenMade) return } @@ -190,7 +191,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { fromSideChainBlock, err := c.persistate.UInt32(persistateSideChainLastBlockKey) if err != nil { fromSideChainBlock = 0 - c.log.Warn("can't get last processed side chain block number", zap.String("error", err.Error())) + c.log.Warn(logs.FrostFSNodeCantGetLastProcessedSideChainBlockNumber, zap.String("error", err.Error())) } subs, err = subscriber.New(ctx, &subscriber.Params{ @@ -215,7 +216,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { setNetmapNotificationParser(c, newEpochNotification, func(src *state.ContainedNotificationEvent) (event.Event, error) { res, err := netmapEvent.ParseNewEpoch(src) if err == nil { - c.log.Info("new epoch event from sidechain", + c.log.Info(logs.FrostFSNodeNewEpochEventFromSidechain, zap.Uint64("number", res.(netmapEvent.NewEpoch).EpochNumber()), ) } @@ -226,11 +227,11 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { registerNotificationHandlers(c.cfgContainer.scriptHash, lis, c.cfgContainer.parsers, c.cfgContainer.subscribers) registerBlockHandler(lis, func(block *block.Block) { - c.log.Debug("new block", zap.Uint32("index", block.Index)) + c.log.Debug(logs.FrostFSNodeNewBlock, zap.Uint32("index", block.Index)) err = c.persistate.SetUInt32(persistateSideChainLastBlockKey, block.Index) if err != nil { - c.log.Warn("can't update persistent state", + c.log.Warn(logs.FrostFSNodeCantUpdatePersistentState, zap.String("chain", "side"), zap.Uint32("block_index", block.Index)) } diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index d9b1c9208..76cceeb6d 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -8,6 +8,7 @@ import ( netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -193,7 +194,7 @@ func addNewEpochNotificationHandlers(c *cfg) { if (n-c.cfgNetmap.startEpoch)%reBootstrapInterval == 0 { err := c.bootstrap() if err != nil { - c.log.Warn("can't send re-bootstrap tx", zap.Error(err)) + c.log.Warn(logs.FrostFSNodeCantSendRebootstrapTx, zap.Error(err)) } } }) @@ -203,7 +204,7 @@ func addNewEpochNotificationHandlers(c *cfg) { ni, err := c.netmapLocalNodeState(e) if err != nil { - c.log.Error("could not update node state on new epoch", + c.log.Error(logs.FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch, zap.Uint64("epoch", e), zap.String("error", err.Error()), ) @@ -218,7 +219,7 @@ func addNewEpochNotificationHandlers(c *cfg) { addNewEpochAsyncNotificationHandler(c, func(ev event.Event) { _, err := makeNotaryDeposit(c) if err != nil { - c.log.Error("could not make notary deposit", + c.log.Error(logs.FrostFSNodeCouldNotMakeNotaryDeposit, zap.String("error", err.Error()), ) } @@ -298,7 +299,7 @@ func initNetmapState(c *cfg) { } } - c.log.Info("initial network state", + c.log.Info(logs.FrostFSNodeInitialNetworkState, zap.Uint64("epoch", epoch), zap.String("state", stateWord), ) diff --git a/cmd/frostfs-node/notificator.go b/cmd/frostfs-node/notificator.go index 4a310e5b0..9c90e052c 100644 --- a/cmd/frostfs-node/notificator.go +++ b/cmd/frostfs-node/notificator.go @@ -6,6 +6,7 @@ import ( "fmt" nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" @@ -28,7 +29,7 @@ func (n *notificationSource) Iterate(ctx context.Context, epoch uint64, handler listRes, err := n.e.ListContainers(engine.ListContainersPrm{}) if err != nil { - log.Error("notificator: could not list containers", zap.Error(err)) + log.Error(logs.FrostFSNodeNotificatorCouldNotListContainers, zap.Error(err)) return } @@ -43,7 +44,7 @@ func (n *notificationSource) Iterate(ctx context.Context, epoch uint64, handler selectRes, err := n.e.Select(selectPrm) if err != nil { - log.Error("notificator: could not select objects from container", + log.Error(logs.FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer, zap.Stringer("cid", c), zap.Error(err), ) @@ -53,7 +54,7 @@ func (n *notificationSource) Iterate(ctx context.Context, epoch uint64, handler for _, a := range selectRes.AddressList() { err = n.processAddress(ctx, a, handler) if err != nil { - log.Error("notificator: could not process object", + log.Error(logs.FrostFSNodeNotificatorCouldNotProcessObject, zap.Stringer("address", a), zap.Error(err), ) @@ -62,7 +63,7 @@ func (n *notificationSource) Iterate(ctx context.Context, epoch uint64, handler } } - log.Debug("notificator: finished processing object notifications") + log.Debug(logs.FrostFSNodeNotificatorFinishedProcessingObjectNotifications) } func (n *notificationSource) processAddress( @@ -101,7 +102,7 @@ type notificationWriter struct { func (n notificationWriter) Notify(topic string, address oid.Address) { if err := n.w.Notify(topic, address); err != nil { - n.l.Warn("could not write object notification", + n.l.Warn(logs.FrostFSNodeCouldNotWriteObjectNotification, zap.Stringer("address", address), zap.String("topic", topic), zap.Error(err), diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index ff4335ff9..8f5a83eb0 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -11,6 +11,7 @@ import ( metricsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/metrics" policerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/policer" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -62,7 +63,7 @@ type objectSvc struct { func (c *cfg) MaxObjectSize() uint64 { sz, err := c.cfgNetmap.wrapper.MaxObjectSize() if err != nil { - c.log.Error("could not get max object size value", + c.log.Error(logs.FrostFSNodeCouldNotGetMaxObjectSizeValue, zap.String("error", err.Error()), ) } @@ -259,7 +260,7 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *reputati _, err := ls.Inhume(ctx, inhumePrm) if err != nil { - c.log.Warn("could not inhume mark redundant copy as garbage", + c.log.Warn(logs.FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage, zap.String("error", err.Error()), ) } @@ -600,7 +601,7 @@ func (c *reputationClientConstructor) Get(info coreclient.NodeInfo) (coreclient. } } } else { - c.log.Warn("could not get latest network map to overload the client", + c.log.Warn(logs.FrostFSNodeCouldNotGetLatestNetworkMapToOverloadTheClient, zap.String("error", err.Error()), ) } diff --git a/cmd/frostfs-node/reputation.go b/cmd/frostfs-node/reputation.go index a96bd066e..b3acf7eb0 100644 --- a/cmd/frostfs-node/reputation.go +++ b/cmd/frostfs-node/reputation.go @@ -11,6 +11,7 @@ import ( intermediatereputation "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/intermediate" localreputation "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/local" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/ticker" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" repClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/reputation" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" @@ -105,7 +106,7 @@ func addReputationReportHandler(ctx context.Context, c *cfg) { addNewEpochAsyncNotificationHandler( c, func(ev event.Event) { - c.log.Debug("start reporting reputation on new epoch event") + c.log.Debug(logs.FrostFSNodeStartReportingReputationOnNewEpochEvent) var reportPrm localtrustcontroller.ReportPrm @@ -127,13 +128,13 @@ func addEigenTrustEpochHandler(ctx context.Context, c *cfg, eigenTrustController duration, err := c.cfgNetmap.wrapper.EpochDuration() if err != nil { - log.Debug("could not fetch epoch duration", zap.Error(err)) + log.Debug(logs.FrostFSNodeCouldNotFetchEpochDuration, zap.Error(err)) return } iterations, err := c.cfgNetmap.wrapper.EigenTrustIterations() if err != nil { - log.Debug("could not fetch iteration number", zap.Error(err)) + log.Debug(logs.FrostFSNodeCouldNotFetchIterationNumber, zap.Error(err)) return } @@ -145,7 +146,7 @@ func addEigenTrustEpochHandler(ctx context.Context, c *cfg, eigenTrustController ) }) if err != nil { - log.Debug("could not create fixed epoch timer", zap.Error(err)) + log.Debug(logs.FrostFSNodeCouldNotCreateFixedEpochTimer, zap.Error(err)) return } diff --git a/cmd/frostfs-node/reputation/common/remote.go b/cmd/frostfs-node/reputation/common/remote.go index cd0a024a9..f1982301f 100644 --- a/cmd/frostfs-node/reputation/common/remote.go +++ b/cmd/frostfs-node/reputation/common/remote.go @@ -3,6 +3,7 @@ package common import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" @@ -71,16 +72,16 @@ func NewRemoteTrustProvider(prm RemoteProviderPrm) *RemoteTrustProvider { } func (rtp *RemoteTrustProvider) InitRemote(srv reputationcommon.ServerInfo) (reputationcommon.WriterProvider, error) { - rtp.log.Debug("initializing remote writer provider") + rtp.log.Debug(logs.CommonInitializingRemoteWriterProvider) if srv == nil { - rtp.log.Debug("route has reached dead-end provider") + rtp.log.Debug(logs.CommonRouteHasReachedDeadendProvider) return rtp.deadEndProvider, nil } if rtp.netmapKeys.IsLocalKey(srv.PublicKey()) { // if local => return no-op writer - rtp.log.Debug("initializing no-op writer provider") + rtp.log.Debug(logs.CommonInitializingNoopWriterProvider) return trustcontroller.SimpleWriterProvider(new(NopReputationWriter)), nil } diff --git a/cmd/frostfs-node/reputation/intermediate/consumers.go b/cmd/frostfs-node/reputation/intermediate/consumers.go index 33eab605b..02cdb2a2b 100644 --- a/cmd/frostfs-node/reputation/intermediate/consumers.go +++ b/cmd/frostfs-node/reputation/intermediate/consumers.go @@ -3,6 +3,7 @@ package intermediate import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" @@ -31,7 +32,7 @@ type ConsumerTrustWriter struct { } func (w *ConsumerTrustWriter) Write(_ context.Context, t reputation.Trust) error { - w.log.Debug("writing received consumer's trusts", + w.log.Debug(logs.IntermediateWritingReceivedConsumersTrusts, zap.Uint64("epoch", w.iterInfo.Epoch()), zap.Uint32("iteration", w.iterInfo.I()), zap.Stringer("trusting_peer", t.TrustingPeer()), diff --git a/cmd/frostfs-node/reputation/intermediate/contract.go b/cmd/frostfs-node/reputation/intermediate/contract.go index 6303b1219..2d83598bc 100644 --- a/cmd/frostfs-node/reputation/intermediate/contract.go +++ b/cmd/frostfs-node/reputation/intermediate/contract.go @@ -4,6 +4,7 @@ import ( "crypto/ecdsa" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" repClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/reputation" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" eigentrustcalc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/calculator" @@ -71,7 +72,7 @@ type FinalWriter struct { } func (fw FinalWriter) WriteIntermediateTrust(t eigentrust.IterationTrust) error { - fw.l.Debug("start writing global trusts to contract") + fw.l.Debug(logs.IntermediateStartWritingGlobalTrustsToContract) args := repClient.PutPrm{} diff --git a/cmd/frostfs-node/reputation/intermediate/daughters.go b/cmd/frostfs-node/reputation/intermediate/daughters.go index d72eead43..30237537c 100644 --- a/cmd/frostfs-node/reputation/intermediate/daughters.go +++ b/cmd/frostfs-node/reputation/intermediate/daughters.go @@ -3,6 +3,7 @@ package intermediate import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/storage/daughters" @@ -27,7 +28,7 @@ type DaughterTrustWriter struct { } func (w *DaughterTrustWriter) Write(_ context.Context, t reputation.Trust) error { - w.log.Debug("writing received daughter's trusts", + w.log.Debug(logs.IntermediateWritingReceivedDaughtersTrusts, zap.Uint64("epoch", w.ep.Epoch()), zap.Stringer("trusting_peer", t.TrustingPeer()), zap.Stringer("trusted_peer", t.Peer()), diff --git a/cmd/frostfs-node/reputation/intermediate/remote.go b/cmd/frostfs-node/reputation/intermediate/remote.go index b1a218b94..8087463b5 100644 --- a/cmd/frostfs-node/reputation/intermediate/remote.go +++ b/cmd/frostfs-node/reputation/intermediate/remote.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/common" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" @@ -92,7 +93,7 @@ func (rtp *RemoteTrustWriter) Write(ctx context.Context, t reputation.Trust) err epoch := rtp.iterInfo.Epoch() i := rtp.iterInfo.I() - rtp.log.Debug("announcing trust", + rtp.log.Debug(logs.IntermediateAnnouncingTrust, zap.Uint64("epoch", epoch), zap.Uint32("iteration", i), zap.Stringer("trusting_peer", t.TrustingPeer()), diff --git a/cmd/frostfs-node/reputation/local/remote.go b/cmd/frostfs-node/reputation/local/remote.go index 3c929a9ca..6197c6d69 100644 --- a/cmd/frostfs-node/reputation/local/remote.go +++ b/cmd/frostfs-node/reputation/local/remote.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/common" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" @@ -96,7 +97,7 @@ func (rtp *RemoteTrustWriter) Write(_ context.Context, t reputation.Trust) error func (rtp *RemoteTrustWriter) Close(ctx context.Context) error { epoch := rtp.ep.Epoch() - rtp.log.Debug("announcing trusts", + rtp.log.Debug(logs.LocalAnnouncingTrusts, zap.Uint64("epoch", epoch), ) diff --git a/cmd/frostfs-node/reputation/local/storage.go b/cmd/frostfs-node/reputation/local/storage.go index 861151871..a0dc3d4ce 100644 --- a/cmd/frostfs-node/reputation/local/storage.go +++ b/cmd/frostfs-node/reputation/local/storage.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" @@ -27,7 +28,7 @@ type TrustStorage struct { func (s *TrustStorage) InitIterator(ep reputationcommon.EpochProvider) (trustcontroller.Iterator, error) { epoch := ep.Epoch() - s.Log.Debug("initializing iterator over trusts", + s.Log.Debug(logs.LocalInitializingIteratorOverTrusts, zap.Uint64("epoch", epoch), ) diff --git a/cmd/frostfs-node/tracing.go b/cmd/frostfs-node/tracing.go index bbdb71c64..d963ba866 100644 --- a/cmd/frostfs-node/tracing.go +++ b/cmd/frostfs-node/tracing.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "go.uber.org/zap" ) @@ -14,7 +15,7 @@ func initTracing(ctx context.Context, c *cfg) { _, err := tracing.Setup(ctx, *conf) if err != nil { - c.log.Error("failed init tracing", zap.Error(err)) + c.log.Error(logs.FrostFSNodeFailedInitTracing, zap.Error(err)) } c.closers = append(c.closers, closer{ @@ -24,7 +25,7 @@ func initTracing(ctx context.Context, c *cfg) { defer cancel() err := tracing.Shutdown(ctx) //cfg context cancels before close if err != nil { - c.log.Error("failed shutdown tracing", zap.Error(err)) + c.log.Error(logs.FrostFSNodeFailedShutdownTracing, zap.Error(err)) } }, }) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index 93a364471..b4f43acac 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -6,6 +6,7 @@ import ( "time" treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" containerClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" @@ -37,7 +38,7 @@ func (c cnrSource) List() ([]cid.ID, error) { func initTreeService(c *cfg) { treeConfig := treeconfig.Tree(c.appCfg) if !treeConfig.Enabled() { - c.log.Info("tree service is not enabled, skip initialization") + c.log.Info(logs.FrostFSNodeTreeServiceIsNotEnabledSkipInitialization) return } @@ -68,7 +69,7 @@ func initTreeService(c *cfg) { addNewEpochNotificationHandler(c, func(_ event.Event) { err := c.treeService.SynchronizeAll() if err != nil { - c.log.Error("could not synchronize Tree Service", zap.Error(err)) + c.log.Error(logs.FrostFSNodeCouldNotSynchronizeTreeService, zap.Error(err)) } }) } else { @@ -79,7 +80,7 @@ func initTreeService(c *cfg) { for range tick.C { err := c.treeService.SynchronizeAll() if err != nil { - c.log.Error("could not synchronize Tree Service", zap.Error(err)) + c.log.Error(logs.FrostFSNodeCouldNotSynchronizeTreeService, zap.Error(err)) if errors.Is(err, tree.ErrShuttingDown) { return } @@ -92,11 +93,11 @@ func initTreeService(c *cfg) { ev := e.(containerEvent.DeleteSuccess) // This is executed asynchronously, so we don't care about the operation taking some time. - c.log.Debug("removing all trees for container", zap.Stringer("cid", ev.ID)) + c.log.Debug(logs.FrostFSNodeRemovingAllTreesForContainer, zap.Stringer("cid", ev.ID)) err := c.treeService.DropTree(context.Background(), ev.ID, "") if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { // Ignore pilorama.ErrTreeNotFound but other errors, including shard.ErrReadOnly, should be logged. - c.log.Error("container removal event received, but trees weren't removed", + c.log.Error(logs.FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved, zap.Stringer("cid", ev.ID), zap.String("error", err.Error())) } diff --git a/internal/logs/logs.go b/internal/logs/logs.go new file mode 100644 index 000000000..46ed8e867 --- /dev/null +++ b/internal/logs/logs.go @@ -0,0 +1,643 @@ +package logs + +const ( + InnerringAmountCanNotBeRepresentedAsAnInt64 = "amount can not be represented as an int64" // Error in ../node/pkg/innerring/settlement.go + InnerringCantGetUsedSpaceEstimation = "can't get used space estimation" // Warn in ../node/pkg/innerring/settlement.go + InnerringSubnetCreationQueueFailure = "subnet creation queue failure" // Error in ../node/pkg/innerring/subnet.go + InnerringDiscardSubnetCreation = "discard subnet creation" // Info in ../node/pkg/innerring/subnet.go + InnerringApproveSubnetCreation = "approve subnet creation" // Error in ../node/pkg/innerring/subnet.go + InnerringSubnetRemovalHandlingFailure = "subnet removal handling failure" // Error in ../node/pkg/innerring/subnet.go + InnerringGettingNetmapCandidates = "getting netmap candidates" // Error in ../node/pkg/innerring/subnet.go + InnerringUnmarshallingRemovedSubnetID = "unmarshalling removed subnet ID" // Error in ../node/pkg/innerring/subnet.go + InnerringIteratingNodesSubnets = "iterating node's subnets" // Error in ../node/pkg/innerring/subnet.go + InnerringRemovingNodeFromNetmapCandidates = "removing node from netmap candidates" // Debug in ../node/pkg/innerring/subnet.go + InnerringRemovingNodeFromCandidates = "removing node from candidates" // Error in ../node/pkg/innerring/subnet.go + InnerringRemovingSubnetFromTheNode = "removing subnet from the node" // Debug in ../node/pkg/innerring/subnet.go + InnerringUpdatingSubnetInfo = "updating subnet info" // Error in ../node/pkg/innerring/subnet.go + InnerringNonalphabetModeDoNotStopContainerEstimations = "non-alphabet mode, do not stop container estimations" // Debug in ../node/pkg/innerring/blocktimer.go + InnerringCantStopEpochEstimation = "can't stop epoch estimation" // Warn in ../node/pkg/innerring/blocktimer.go + InnerringCantMakeNotaryDepositInMainChain = "can't make notary deposit in main chain" // Error in ../node/pkg/innerring/notary.go + InnerringCantMakeNotaryDepositInSideChain = "can't make notary deposit in side chain" // Error in ../node/pkg/innerring/notary.go + InnerringNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" // Info in ../node/pkg/innerring/notary.go + InnerringCantGetInnerRingIndex = "can't get inner ring index" // Error in ../node/pkg/innerring/state.go + InnerringCantGetInnerRingSize = "can't get inner ring size" // Error in ../node/pkg/innerring/state.go + InnerringCantGetAlphabetIndex = "can't get alphabet index" // Error in ../node/pkg/innerring/state.go + InnerringIgnoreValidatorVoteNodeNotInAlphabetRange = "ignore validator vote: node not in alphabet range" // Info in ../node/pkg/innerring/state.go + InnerringIgnoreValidatorVoteEmptyValidatorsList = "ignore validator vote: empty validators list" // Info in ../node/pkg/innerring/state.go + InnerringCantInvokeVoteMethodInAlphabetContract = "can't invoke vote method in alphabet contract" // Warn in ../node/pkg/innerring/state.go + InnerringCantGetLastProcessedMainChainBlockNumber = "can't get last processed main chain block number" // Warn in ../node/pkg/innerring/initialization.go + InnerringNotarySupport = "notary support" // Info in ../node/pkg/innerring/initialization.go + InnerringAlphabetKeysSyncIsDisabled = "alphabet keys sync is disabled" // Debug in ../node/pkg/innerring/initialization.go + InnerringNoControlServerEndpointSpecified = "no Control server endpoint specified, service is disabled" // Info in ../node/pkg/innerring/initialization.go + InnerringCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" // Warn in ../node/pkg/innerring/initialization.go + InnerringFailedToSetGroupSignerScope = "failed to set group signer scope, continue with Global" // Info in ../node/pkg/innerring/initialization.go + InnerringCantVoteForPreparedValidators = "can't vote for prepared validators" // Warn in ../node/pkg/innerring/innerring.go + InnerringNewBlock = "new block" // Debug in ../node/pkg/innerring/innerring.go + InnerringCantUpdatePersistentState = "can't update persistent state" // Warn in ../node/pkg/innerring/innerring.go + InnerringCloserError = "closer error" // Warn in ../node/pkg/innerring/innerring.go + InnerringReadConfigFromBlockchain = "read config from blockchain" // Debug in ../node/pkg/innerring/innerring.go + InnerringCantSetupRemoteConnection = "can't setup remote connection" // Warn in ../node/pkg/innerring/rpc.go + InnerringCantGetStorageGroupObject = "can't get storage group object" // Warn in ../node/pkg/innerring/rpc.go + NotificatorNotificatorStartProcessingObjectNotifications = "notificator: start processing object notifications" // Debug in ../node/pkg/services/notificator/service.go + NotificatorNotificatorProcessingObjectNotification = "notificator: processing object notification" // Debug in ../node/pkg/services/notificator/service.go + PolicerCouldNotGetContainer = "could not get container" // Error in ../node/pkg/services/policer/check.go + PolicerCouldNotInhumeObjectWithMissingContainer = "could not inhume object with missing container" // Error in ../node/pkg/services/policer/check.go + PolicerCouldNotBuildPlacementVectorForObject = "could not build placement vector for object" // Error in ../node/pkg/services/policer/check.go + PolicerRedundantLocalObjectCopyDetected = "redundant local object copy detected" // Info in ../node/pkg/services/policer/check.go + PolicerReceiveObjectHeaderToCheckPolicyCompliance = "receive object header to check policy compliance" // Error in ../node/pkg/services/policer/check.go + PolicerConsiderNodeUnderMaintenanceAsOK = "consider node under maintenance as OK" // Debug in ../node/pkg/services/policer/check.go + PolicerShortageOfObjectCopiesDetected = "shortage of object copies detected" // Debug in ../node/pkg/services/policer/check.go + PolicerSomeOfTheCopiesAreStoredOnNodesUnderMaintenance = "some of the copies are stored on nodes under maintenance, save local copy" // Debug in ../node/pkg/services/policer/check.go + PolicerRoutineStopped = "routine stopped" // Info in ../node/pkg/services/policer/process.go + PolicerFailureAtObjectSelectForReplication = "failure at object select for replication" // Warn in ../node/pkg/services/policer/process.go + PolicerPoolSubmission = "pool submission" // Warn in ../node/pkg/services/policer/process.go + PolicerTuneReplicationCapacity = "tune replication capacity" // Debug in ../node/pkg/services/policer/process.go + ReplicatorFinishWork = "finish work" // Debug in ../node/pkg/services/replicator/process.go + ReplicatorCouldNotGetObjectFromLocalStorage = "could not get object from local storage" // Error in ../node/pkg/services/replicator/process.go + ReplicatorCouldNotReplicateObject = "could not replicate object" // Error in ../node/pkg/services/replicator/process.go + ReplicatorObjectSuccessfullyReplicated = "object successfully replicated" // Debug in ../node/pkg/services/replicator/process.go + SessionServingRequest = "serving request..." // Debug in ../node/pkg/services/session/executor.go + TreeRedirectingTreeServiceQuery = "redirecting tree service query" // Debug in ../node/pkg/services/tree/redirect.go + TreeBearerPresentedButNotAllowedByACL = "bearer presented but not allowed by ACL" // Debug in ../node/pkg/services/tree/signature.go + TreeCouldNotGetLastSynchronizedHeightForATree = "could not get last synchronized height for a tree" // Warn in ../node/pkg/services/tree/sync.go + TreeCouldNotUpdateLastSynchronizedHeightForATree = "could not update last synchronized height for a tree" // Warn in ../node/pkg/services/tree/sync.go + TreeSynchronizeTree = "synchronize tree" // Debug in ../node/pkg/services/tree/sync.go + TreeFailedToRunTreeSynchronizationOverAllNodes = "failed to run tree synchronization over all nodes" // Warn in ../node/pkg/services/tree/sync.go + TreeSyncingTrees = "syncing trees..." // Debug in ../node/pkg/services/tree/sync.go + TreeCouldNotFetchContainers = "could not fetch containers" // Error in ../node/pkg/services/tree/sync.go + TreeTreesHaveBeenSynchronized = "trees have been synchronized" // Debug in ../node/pkg/services/tree/sync.go + TreeSyncingContainerTrees = "syncing container trees..." // Debug in ../node/pkg/services/tree/sync.go + TreeCouldNotSyncTrees = "could not sync trees" // Error in ../node/pkg/services/tree/sync.go + TreeContainerTreesHaveBeenSynced = "container trees have been synced" // Debug in ../node/pkg/services/tree/sync.go + TreeCouldNotQueryTreesForSynchronization = "could not query trees for synchronization" // Error in ../node/pkg/services/tree/sync.go + TreeRemovingRedundantTrees = "removing redundant trees..." // Debug in ../node/pkg/services/tree/sync.go + TreeCouldNotRemoveRedundantTree = "could not remove redundant tree" // Error in ../node/pkg/services/tree/sync.go + TreeCouldNotCalculateContainerNodes = "could not calculate container nodes" // Error in ../node/pkg/services/tree/sync.go + TreeFailedToApplyReplicatedOperation = "failed to apply replicated operation" // Error in ../node/pkg/services/tree/replicator.go + TreeDoNotSendUpdateToTheNode = "do not send update to the node" // Debug in ../node/pkg/services/tree/replicator.go + TreeFailedToSentUpdateToTheNode = "failed to sent update to the node" // Warn in ../node/pkg/services/tree/replicator.go + TreeErrorDuringReplication = "error during replication" // Error in ../node/pkg/services/tree/replicator.go + PersistentCouldNotGetSessionFromPersistentStorage = "could not get session from persistent storage" // Error in ../node/pkg/services/session/storage/persistent/storage.go + PersistentCouldNotDeleteSToken = "could not delete token" // Error in ../node/pkg/services/session/storage/persistent/storage.go + PersistentCouldNotCleanUpExpiredTokens = "could not clean up expired tokens" // Error in ../node/pkg/services/session/storage/persistent/storage.go + CommonStartBuildingManagers = "start building managers" // Debug in ../node/pkg/services/reputation/common/managers.go + ControllerReportIsAlreadyStarted = "report is already started" // Debug in ../node/pkg/services/reputation/local/controller/calls.go + ControllerStartingToReportLocalTrustValues = "starting to report local trust values" // Debug in ../node/pkg/services/reputation/local/controller/calls.go + ControllerCouldNotInitializeIteratorOverLocalTrustValues = "could not initialize iterator over local trust values" // Debug in ../node/pkg/services/reputation/local/controller/calls.go + ControllerCouldNotInitializeLocalTrustTarget = "could not initialize local trust target" // Debug in ../node/pkg/services/reputation/local/controller/calls.go + ControllerIteratorOverLocalTrustFailed = "iterator over local trust failed" // Debug in ../node/pkg/services/reputation/local/controller/calls.go + ControllerCouldNotFinishWritingLocalTrustValues = "could not finish writing local trust values" // Debug in ../node/pkg/services/reputation/local/controller/calls.go + ControllerReportingSuccessfullyFinished = "reporting successfully finished" // Debug in ../node/pkg/services/reputation/local/controller/calls.go + ControllerReportingSuccessfullyInterrupted = "reporting successfully interrupted" // Debug in ../node/pkg/services/reputation/local/controller/calls.go + ControllerReportingIsNotStartedOrAlreadyInterrupted = "reporting is not started or already interrupted" // Debug in ../node/pkg/services/reputation/local/controller/calls.go + RoutesBuildingNextStageForLocalTrustRoute = "building next stage for local trust route" // Debug in ../node/pkg/services/reputation/local/routes/calls.go + CalculatorFailedToGetAlphaParam = "failed to get alpha param" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorConsumersTrustIteratorsInitFailure = "consumers trust iterator's init failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorWorkerPoolSubmitFailure = "worker pool submit failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorIterateDaughtersConsumersFailed = "iterate daughter's consumers failed" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorGetInitialTrustFailure = "get initial trust failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorDaughterTrustIteratorsInitFailure = "daughter trust iterator's init failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorIterateOverDaughtersTrustsFailure = "iterate over daughter's trusts failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorInitWriterFailure = "init writer failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorWriteFinalResultFailure = "write final result failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorWriteValueFailure = "write value failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorIterateDaughterTrustsFailure = "iterate daughter trusts failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorCouldNotCloseWriter = "could not close writer" // Error in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorAllDaughtersTrustIteratorsInitFailure = "all daughters trust iterator's init failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + CalculatorIterateOverAllDaughtersFailure = "iterate over all daughters failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go + ControllerCouldNotGetEigenTrustIterationNumber = "could not get EigenTrust iteration number" // Error in ../node/pkg/services/reputation/eigentrust/controller/calls.go + ControllerIterationSubmitFailure = "iteration submit failure" // Debug in ../node/pkg/services/reputation/eigentrust/controller/calls.go + RoutesBuildingNextStageForTrustRoute = "building next stage for trust route" // Debug in ../node/pkg/services/reputation/eigentrust/routes/calls.go + RouterCouldNotInitializeWriterProvider = "could not initialize writer provider" // Debug in ../node/pkg/services/reputation/common/router/calls.go + RouterCouldNotInitializeWriter = "could not initialize writer" // Debug in ../node/pkg/services/reputation/common/router/calls.go + RouterCouldNotWriteTheValue = "could not write the value" // Debug in ../node/pkg/services/reputation/common/router/calls.go + RouterCouldNotCloseRemoteServerWriter = "could not close remote server writer" // Debug in ../node/pkg/services/reputation/common/router/calls.go + TombstoneCouldNotGetTheTombstoneTheSource = "tombstone getter: could not get the tombstone the source" // Warn in ../node/pkg/services/object_manager/tombstone/checker.go + TombstoneCouldNotParseTombstoneExpirationEpoch = "tombstone getter: could not parse tombstone expiration epoch" // Warn in ../node/pkg/services/object_manager/tombstone/checker.go + DeleteRequestIsNotRolledOverToTheContainer = "request is not rolled over to the container" // Debug in ../node/pkg/services/object/delete/container.go + DeleteServingRequest = "serving request..." // Debug in ../node/pkg/services/object/delete/delete.go + DeleteOperationFinishedSuccessfully = "operation finished successfully" // Debug in ../node/pkg/services/object/delete/delete.go + DeleteOperationFinishedWithError = "operation finished with error" // Debug in ../node/pkg/services/object/delete/delete.go + DeleteCouldNotComposeSplitInfo = "could not compose split info" // Debug in ../node/pkg/services/object/delete/exec.go + DeleteNoSplitInfoObjectIsPHY = "no split info, object is PHY" // Debug in ../node/pkg/services/object/delete/exec.go + DeleteAssemblingChain = "assembling chain..." // Debug in ../node/pkg/services/object/delete/exec.go + DeleteCouldNotGetPreviousSplitElement = "could not get previous split element" // Debug in ../node/pkg/services/object/delete/exec.go + DeleteCollectingChildren = "collecting children..." // Debug in ../node/pkg/services/object/delete/exec.go + DeleteCouldNotCollectObjectChildren = "could not collect object children" // Debug in ../node/pkg/services/object/delete/exec.go + DeleteSupplementBySplitID = "supplement by split ID" // Debug in ../node/pkg/services/object/delete/exec.go + DeleteCouldNotSearchForSplitChainMembers = "could not search for split chain members" // Debug in ../node/pkg/services/object/delete/exec.go + DeleteCouldNotMarshalTombstoneStructure = "could not marshal tombstone structure" // Debug in ../node/pkg/services/object/delete/exec.go + DeleteCouldNotSaveTheTombstone = "could not save the tombstone" // Debug in ../node/pkg/services/object/delete/exec.go + DeleteFormingTombstoneStructure = "forming tombstone structure..." // Debug in ../node/pkg/services/object/delete/local.go + DeleteTombstoneStructureSuccessfullyFormedSaving = "tombstone structure successfully formed, saving..." // Debug in ../node/pkg/services/object/delete/local.go + DeleteCouldNotReadTombstoneLifetimeConfig = "could not read tombstone lifetime config" // Debug in ../node/pkg/services/object/delete/local.go + DeleteFormingSplitInfo = "forming split info..." // Debug in ../node/pkg/services/object/delete/local.go + DeleteSplitInfoSuccessfullyFormedCollectingMembers = "split info successfully formed, collecting members..." // Debug in ../node/pkg/services/object/delete/local.go + DeleteMembersSuccessfullyCollected = "members successfully collected" // Debug in ../node/pkg/services/object/delete/local.go + GetProcessingNode = "processing node..." // Debug in ../node/pkg/services/object/get/remote.go + GetRemoteCallFailed = "remote call failed" // Debug in ../node/pkg/services/object/get/remote.go + GetCanNotAssembleTheObject = "can not assemble the object" // Debug in ../node/pkg/services/object/get/assemble.go + GetTryingToAssembleTheObject = "trying to assemble the object..." // Debug in ../node/pkg/services/object/get/assemble.go + GetAssemblingSplittedObject = "assembling splitted object..." // Debug in ../node/pkg/services/object/get/assemble.go + GetAssemblingSplittedObjectCompleted = "assembling splitted object completed" // Debug in ../node/pkg/services/object/get/assemble.go + GetFailedToAssembleSplittedObject = "failed to assemble splitted object" // Warn in ../node/pkg/services/object/get/assemble.go + GetCouldNotGetCurrentEpochNumber = "could not get current epoch number" // Debug in ../node/pkg/services/object/get/exec.go + GetCouldNotGenerateContainerTraverser = "could not generate container traverser" // Debug in ../node/pkg/services/object/get/exec.go + GetCouldNotConstructRemoteNodeClient = "could not construct remote node client" // Debug in ../node/pkg/services/object/get/exec.go + GetCouldNotWriteHeader = "could not write header" // Debug in ../node/pkg/services/object/get/exec.go + GetCouldNotWritePayloadChunk = "could not write payload chunk" // Debug in ../node/pkg/services/object/get/exec.go + GetLocalGetFailed = "local get failed" // Debug in ../node/pkg/services/object/get/local.go + GetReturnResultDirectly = "return result directly" // Debug in ../node/pkg/services/object/get/container.go + GetTryingToExecuteInContainer = "trying to execute in container..." // Debug in ../node/pkg/services/object/get/container.go + GetProcessEpoch = "process epoch" // Debug in ../node/pkg/services/object/get/container.go + GetNoMoreNodesAbortPlacementIteration = "no more nodes, abort placement iteration" // Debug in ../node/pkg/services/object/get/container.go + GetInterruptPlacementIterationByContext = "interrupt placement iteration by context" // Debug in ../node/pkg/services/object/get/container.go + GetCompletingTheOperation = "completing the operation" // Debug in ../node/pkg/services/object/get/container.go + GetServingRequest = "serving request..." // Debug in ../node/pkg/services/object/get/get.go + GetOperationFinishedSuccessfully = "operation finished successfully" // Debug in ../node/pkg/services/object/get/get.go + GetRequestedObjectWasMarkedAsRemoved = "requested object was marked as removed" // Debug in ../node/pkg/services/object/get/get.go + GetRequestedObjectIsVirtual = "requested object is virtual" // Debug in ../node/pkg/services/object/get/get.go + GetRequestedRangeIsOutOfObjectBounds = "requested range is out of object bounds" // Debug in ../node/pkg/services/object/get/get.go + GetOperationFinishedWithError = "operation finished with error" // Debug in ../node/pkg/services/object/get/get.go + PutAdditionalContainerBroadcastFailure = "additional container broadcast failure" // Error in ../node/pkg/services/object/put/distributed.go + SearchReturnResultDirectly = "return result directly" // Debug in ../node/pkg/services/object/search/container.go + SearchTryingToExecuteInContainer = "trying to execute in container..." // Debug in ../node/pkg/services/object/search/container.go + SearchProcessEpoch = "process epoch" // Debug in ../node/pkg/services/object/search/container.go + SearchNoMoreNodesAbortPlacementIteration = "no more nodes, abort placement iteration" // Debug in ../node/pkg/services/object/search/container.go + SearchInterruptPlacementIterationByContext = "interrupt placement iteration by context" // Debug in ../node/pkg/services/object/search/container.go + SearchProcessingNode = "processing node..." // Debug in ../node/pkg/services/object/search/container.go + SearchCouldNotConstructRemoteNodeClient = "could not construct remote node client" // Debug in ../node/pkg/services/object/search/container.go + SearchRemoteOperationFailed = "remote operation failed" // Debug in ../node/pkg/services/object/search/container.go + SearchCouldNotGetCurrentEpochNumber = "could not get current epoch number" // Debug in ../node/pkg/services/object/search/exec.go + SearchCouldNotGenerateContainerTraverser = "could not generate container traverser" // Debug in ../node/pkg/services/object/search/exec.go + SearchCouldNotWriteObjectIdentifiers = "could not write object identifiers" // Debug in ../node/pkg/services/object/search/exec.go + SearchLocalOperationFailed = "local operation failed" // Debug in ../node/pkg/services/object/search/local.go + SearchServingRequest = "serving request..." // Debug in ../node/pkg/services/object/search/search.go + SearchOperationFinishedWithError = "operation finished with error" // Debug in ../node/pkg/services/object/search/search.go + SearchOperationFinishedSuccessfully = "operation finished successfully" // Debug in ../node/pkg/services/object/search/search.go + UtilObjectServiceError = "object service error" // Error in ../node/pkg/services/object/util/log.go + UtilCouldNotPushTaskToWorkerPool = "could not push task to worker pool" // Error in ../node/pkg/services/object/util/log.go + V2CantCheckIfRequestFromInnerRing = "can't check if request from inner ring" // Debug in ../node/pkg/services/object/acl/v2/classifier.go + V2CantCheckIfRequestFromContainerNode = "can't check if request from container node" // Debug in ../node/pkg/services/object/acl/v2/classifier.go + NatsNatsConnectionWasLost = "nats: connection was lost" // Error in ../node/pkg/services/notificator/nats/service.go + NatsNatsReconnectedToTheServer = "nats: reconnected to the server" // Warn in ../node/pkg/services/notificator/nats/service.go + NatsNatsClosingConnectionAsTheContextIsDone = "nats: closing connection as the context is done" // Info in ../node/pkg/services/notificator/nats/service.go + ControllerStartingToAnnounceTheValuesOfTheMetrics = "starting to announce the values of the metrics" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerCouldNotInitializeIteratorOverLocallyCollectedMetrics = "could not initialize iterator over locally collected metrics" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerCouldNotInitializeAnnouncementAccumulator = "could not initialize announcement accumulator" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerIteratorOverLocallyCollectedMetricsAborted = "iterator over locally collected metrics aborted" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerCouldNotFinishWritingLocalAnnouncements = "could not finish writing local announcements" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerTrustAnnouncementSuccessfullyFinished = "trust announcement successfully finished" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerAnnouncementIsAlreadyStarted = "announcement is already started" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerAnnouncementSuccessfullyInterrupted = "announcement successfully interrupted" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerAnnouncementIsNotStartedOrAlreadyInterrupted = "announcement is not started or already interrupted" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerCouldNotInitializeIteratorOverLocallyAccumulatedAnnouncements = "could not initialize iterator over locally accumulated announcements" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerCouldNotInitializeResultTarget = "could not initialize result target" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerIteratorOverLocalAnnouncementsAborted = "iterator over local announcements aborted" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + ControllerCouldNotFinishWritingLoadEstimations = "could not finish writing load estimations" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go + RouteCouldNotInitializeWriterProvider = "could not initialize writer provider" // Debug in ../node/pkg/services/container/announcement/load/route/calls.go + RouteCouldNotInitializeWriter = "could not initialize writer" // Debug in ../node/pkg/services/container/announcement/load/route/calls.go + RouteCouldNotPutTheValue = "could not put the value" // Debug in ../node/pkg/services/container/announcement/load/route/calls.go + RouteCouldNotCloseRemoteServerWriter = "could not close remote server writer" // Debug in ../node/pkg/services/container/announcement/load/route/calls.go + AuditorCouldNotGetObjectHeaderFromCandidate = "could not get object header from candidate" // Debug in ../node/pkg/services/audit/auditor/pop.go + AuditorCouldNotBuildPlacementForObject = "could not build placement for object" // Debug in ../node/pkg/services/audit/auditor/pop.go + AuditorCantHeadObject = "can't head object" // Debug in ../node/pkg/services/audit/auditor/por.go + AuditorCantConcatenateTzHash = "can't concatenate tz hash" // Debug in ../node/pkg/services/audit/auditor/por.go + AuditorStorageGroupSizeCheckFailed = "storage group size check failed" // Debug in ../node/pkg/services/audit/auditor/por.go + AuditorStorageGroupTzHashCheckFailed = "storage group tz hash check failed" // Debug in ../node/pkg/services/audit/auditor/por.go + AuditorCantBuildPlacementForStorageGroupMember = "can't build placement for storage group member" // Info in ../node/pkg/services/audit/auditor/por.go + AuditorAuditContextIsDone = "audit context is done" // Debug in ../node/pkg/services/audit/auditor/context.go + AuditorWritingAuditReport = "writing audit report..." // Debug in ../node/pkg/services/audit/auditor/context.go + AuditorCouldNotWriteAuditReport = "could not write audit report" // Error in ../node/pkg/services/audit/auditor/context.go + AuditorSleepBeforeGetRangeHash = "sleep before get range hash" // Debug in ../node/pkg/services/audit/auditor/pdp.go + AuditorCouldNotGetPayloadRangeHash = "could not get payload range hash" // Debug in ../node/pkg/services/audit/auditor/pdp.go + TaskmanagerProcessRoutine = "process routine" // Info in ../node/pkg/services/audit/taskmanager/listen.go + TaskmanagerStopListenerByContext = "stop listener by context" // Warn in ../node/pkg/services/audit/taskmanager/listen.go + TaskmanagerQueueChannelIsClosed = "queue channel is closed" // Warn in ../node/pkg/services/audit/taskmanager/listen.go + TaskmanagerCouldNotGeneratePDPWorkerPool = "could not generate PDP worker pool" // Error in ../node/pkg/services/audit/taskmanager/listen.go + TaskmanagerCouldNotGeneratePoRWorkerPool = "could not generate PoR worker pool" // Error in ../node/pkg/services/audit/taskmanager/listen.go + TaskmanagerCouldNotSubmitAuditTask = "could not submit audit task" // Warn in ../node/pkg/services/audit/taskmanager/listen.go + ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch = "could not restore block subscription after RPC switch" // Error in ../node/pkg/morph/client/notifications.go + ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch = "could not restore notification subscription after RPC switch" // Error in ../node/pkg/morph/client/notifications.go + ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch = "could not restore notary notification subscription after RPC switch" // Error in ../node/pkg/morph/client/notifications.go + ClientCouldNotEstablishConnectionToTheSwitchedRPCNode = "could not establish connection to the switched RPC node" // Warn in ../node/pkg/morph/client/multi.go + ClientConnectionToTheNewRPCNodeHasBeenEstablished = "connection to the new RPC node has been established" // Info in ../node/pkg/morph/client/multi.go + ClientSwitchingToTheNextRPCNode = "switching to the next RPC node" // Warn in ../node/pkg/morph/client/multi.go + ClientCouldNotEstablishConnectionToAnyRPCNode = "could not establish connection to any RPC node" // Error in ../node/pkg/morph/client/multi.go + ClientCouldNotCreateClientToTheHigherPriorityNode = "could not create client to the higher priority node" // Warn in ../node/pkg/morph/client/multi.go + ClientSwitchedToTheHigherPriorityRPC = "switched to the higher priority RPC" // Info in ../node/pkg/morph/client/multi.go + ClientCouldNotRestoreSideChainSubscriptionsUsingNode = "could not restore side chain subscriptions using node" // Warn in ../node/pkg/morph/client/multi.go + ClientNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" // Info in ../node/pkg/morph/client/notary.go + ClientNotaryDepositInvoke = "notary deposit invoke" // Info in ../node/pkg/morph/client/notary.go + ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked" // Debug in ../node/pkg/morph/client/notary.go + ClientNotaryRequestInvoked = "notary request invoked" // Debug in ../node/pkg/morph/client/notary.go + ClientNeoClientInvoke = "neo client invoke" // Debug in ../node/pkg/morph/client/client.go + ClientNativeGasTransferInvoke = "native gas transfer invoke" // Debug in ../node/pkg/morph/client/client.go + ClientBatchGasTransferInvoke = "batch gas transfer invoke" // Debug in ../node/pkg/morph/client/client.go + ClientCantGetBlockchainHeight = "can't get blockchain height" // Error in ../node/pkg/morph/client/client.go + ClientCantGetBlockchainHeight243 = "can't get blockchain height" // Error in ../node/pkg/morph/client/client.go + EventCouldNotSubmitHandlerToWorkerPool = "could not Submit handler to worker pool" // Warn in ../node/pkg/morph/event/utils.go + EventCouldNotStartListenToEvents = "could not start listen to events" // Error in ../node/pkg/morph/event/listener.go + EventStopEventListenerByError = "stop event listener by error" // Error in ../node/pkg/morph/event/listener.go + EventStopEventListenerByContext = "stop event listener by context" // Info in ../node/pkg/morph/event/listener.go + EventStopEventListenerByNotificationChannel = "stop event listener by notification channel" // Warn in ../node/pkg/morph/event/listener.go + EventNilNotificationEventWasCaught = "nil notification event was caught" // Warn in ../node/pkg/morph/event/listener.go + EventStopEventListenerByNotaryChannel = "stop event listener by notary channel" // Warn in ../node/pkg/morph/event/listener.go + EventNilNotaryEventWasCaught = "nil notary event was caught" // Warn in ../node/pkg/morph/event/listener.go + EventStopEventListenerByBlockChannel = "stop event listener by block channel" // Warn in ../node/pkg/morph/event/listener.go + EventNilBlockWasCaught = "nil block was caught" // Warn in ../node/pkg/morph/event/listener.go + EventListenerWorkerPoolDrained = "listener worker pool drained" // Warn in ../node/pkg/morph/event/listener.go + EventEventParserNotSet = "event parser not set" // Debug in ../node/pkg/morph/event/listener.go + EventCouldNotParseNotificationEvent = "could not parse notification event" // Warn in ../node/pkg/morph/event/listener.go + EventNotificationHandlersForParsedNotificationEventWereNotRegistered = "notification handlers for parsed notification event were not registered" // Info in ../node/pkg/morph/event/listener.go + EventSkipExpiredMainTXNotaryEvent = "skip expired main TX notary event" // Warn in ../node/pkg/morph/event/listener.go + EventCouldNotPrepareAndValidateNotaryEvent = "could not prepare and validate notary event" // Warn in ../node/pkg/morph/event/listener.go + EventNotaryParserNotSet = "notary parser not set" // Debug in ../node/pkg/morph/event/listener.go + EventCouldNotParseNotaryEvent = "could not parse notary event" // Warn in ../node/pkg/morph/event/listener.go + EventNotaryHandlersForParsedNotificationEventWereNotRegistered = "notary handlers for parsed notification event were not registered" // Info in ../node/pkg/morph/event/listener.go + EventIgnoreNilEventParser = "ignore nil event parser" // Info in ../node/pkg/morph/event/listener.go + EventListenerHasBeenAlreadyStartedIgnoreParser = "listener has been already started, ignore parser" // Warn in ../node/pkg/morph/event/listener.go + EventRegisteredNewEventParser = "registered new event parser" // Debug in ../node/pkg/morph/event/listener.go + EventIgnoreNilEventHandler = "ignore nil event handler" // Warn in ../node/pkg/morph/event/listener.go + EventIgnoreHandlerOfEventWoParser = "ignore handler of event w/o parser" // Warn in ../node/pkg/morph/event/listener.go + EventRegisteredNewEventHandler = "registered new event handler" // Debug in ../node/pkg/morph/event/listener.go + EventIgnoreNilNotaryEventParser = "ignore nil notary event parser" // Info in ../node/pkg/morph/event/listener.go + EventListenerHasBeenAlreadyStartedIgnoreNotaryParser = "listener has been already started, ignore notary parser" // Warn in ../node/pkg/morph/event/listener.go + EventIgnoreNilNotaryEventHandler = "ignore nil notary event handler" // Warn in ../node/pkg/morph/event/listener.go + EventIgnoreHandlerOfNotaryEventWoParser = "ignore handler of notary event w/o parser" // Warn in ../node/pkg/morph/event/listener.go + EventIgnoreNilBlockHandler = "ignore nil block handler" // Warn in ../node/pkg/morph/event/listener.go + SubscriberUnsubscribeForNotification = "unsubscribe for notification" // Error in ../node/pkg/morph/subscriber/subscriber.go + SubscriberRemoteNotificationChannelHasBeenClosed = "remote notification channel has been closed" // Warn in ../node/pkg/morph/subscriber/subscriber.go + SubscriberCantCastNotifyEventValueToTheNotifyStruct = "can't cast notify event value to the notify struct" // Error in ../node/pkg/morph/subscriber/subscriber.go + SubscriberNewNotificationEventFromSidechain = "new notification event from sidechain" // Debug in ../node/pkg/morph/subscriber/subscriber.go + SubscriberCantCastBlockEventValueToBlock = "can't cast block event value to block" // Error in ../node/pkg/morph/subscriber/subscriber.go + SubscriberCantCastNotifyEventValueToTheNotaryRequestStruct = "can't cast notify event value to the notary request struct" // Error in ../node/pkg/morph/subscriber/subscriber.go + SubscriberUnsupportedNotificationFromTheChain = "unsupported notification from the chain" // Debug in ../node/pkg/morph/subscriber/subscriber.go + BlobovniczaCreatingDirectoryForBoltDB = "creating directory for BoltDB" // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go + BlobovniczaOpeningBoltDB = "opening BoltDB" // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go + BlobovniczaInitializing = "initializing..." // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go + BlobovniczaAlreadyInitialized = "already initialized" // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go + BlobovniczaCreatingBucketForSizeRange = "creating bucket for size range" // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go + BlobovniczaClosingBoltDB = "closing BoltDB" // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go + BlobovniczaObjectWasRemovedFromBucket = "object was removed from bucket" // Debug in ../node/pkg/local_object_storage/blobovnicza/delete.go + BlobstorOpening = "opening..." // Debug in ../node/pkg/local_object_storage/blobstor/control.go + BlobstorInitializing = "initializing..." // Debug in ../node/pkg/local_object_storage/blobstor/control.go + BlobstorClosing = "closing..." // Debug in ../node/pkg/local_object_storage/blobstor/control.go + BlobstorCouldntCloseStorage = "couldn't close storage" // Info in ../node/pkg/local_object_storage/blobstor/control.go + BlobstorErrorOccurredDuringObjectExistenceChecking = "error occurred during object existence checking" // Warn in ../node/pkg/local_object_storage/blobstor/exists.go + BlobstorErrorOccurredDuringTheIteration = "error occurred during the iteration" // Warn in ../node/pkg/local_object_storage/blobstor/iterate.go + EngineShardHasBeenRemoved = "shard has been removed" // Info in ../node/pkg/local_object_storage/engine/shards.go + EngineCouldNotCloseRemovedShard = "could not close removed shard" // Error in ../node/pkg/local_object_storage/engine/shards.go + EngineCouldNotOpenShardClosingAndSkipping = "could not open shard, closing and skipping" // Error in ../node/pkg/local_object_storage/engine/control.go + EngineCouldNotClosePartiallyInitializedShard = "could not close partially initialized shard" // Error in ../node/pkg/local_object_storage/engine/control.go + EngineCouldNotInitializeShardClosingAndSkipping = "could not initialize shard, closing and skipping" // Error in ../node/pkg/local_object_storage/engine/control.go + EngineCouldNotCloseShard = "could not close shard" // Debug in ../node/pkg/local_object_storage/engine/control.go + EngineCouldNotReloadAShard = "could not reload a shard" // Error in ../node/pkg/local_object_storage/engine/control.go + EngineAddedNewShard = "added new shard" // Info in ../node/pkg/local_object_storage/engine/control.go + EngineCouldNotMarkObjectForShardRelocation = "could not mark object for shard relocation" // Warn in ../node/pkg/local_object_storage/engine/put.go + EngineCouldNotPutObjectToShard = "could not put object to shard" // Warn in ../node/pkg/local_object_storage/engine/put.go + EngineErrorDuringSearchingForObjectChildren = "error during searching for object children" // Warn in ../node/pkg/local_object_storage/engine/delete.go + EngineCouldNotInhumeObjectInShard = "could not inhume object in shard" // Debug in ../node/pkg/local_object_storage/engine/delete.go + EngineStartingRemovalOfLocallyredundantCopies = "starting removal of locally-redundant copies" // Info in ../node/pkg/local_object_storage/engine/remove_copies.go + EngineStartedDuplicatesRemovalRoutine = "started duplicates removal routine" // Debug in ../node/pkg/local_object_storage/engine/remove_copies.go + EngineFinishedRemovalOfLocallyredundantCopies = "finished removal of locally-redundant copies" // Error in ../node/pkg/local_object_storage/engine/remove_copies.go + EngineRemovingAnObjectWithoutFullLockingCheck = "removing an object without full locking check" // Warn in ../node/pkg/local_object_storage/engine/inhume.go + EngineInterruptProcessingTheExpiredLocks = "interrupt processing the expired locks" // Info in ../node/pkg/local_object_storage/engine/inhume.go + EngineInterruptProcessingTheDeletedLocks = "interrupt processing the deleted locks" // Info in ../node/pkg/local_object_storage/engine/inhume.go + EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly = "failed to move shard in degraded-read-only mode, moving to read-only" // Error in ../node/pkg/local_object_storage/engine/engine.go + EngineFailedToMoveShardInReadonlyMode = "failed to move shard in read-only mode" // Error in ../node/pkg/local_object_storage/engine/engine.go + EngineShardIsMovedInReadonlyModeDueToErrorThreshold = "shard is moved in read-only mode due to error threshold" // Info in ../node/pkg/local_object_storage/engine/engine.go + EngineShardIsMovedInDegradedModeDueToErrorThreshold = "shard is moved in degraded mode due to error threshold" // Info in ../node/pkg/local_object_storage/engine/engine.go + EngineModeChangeIsInProgressIgnoringSetmodeRequest = "mode change is in progress, ignoring set-mode request" // Debug in ../node/pkg/local_object_storage/engine/engine.go + EngineStartedShardsEvacuation = "started shards evacuation" // Info in ../node/pkg/local_object_storage/engine/evacuate.go + EngineFinishedShardsEvacuation = "finished shards evacuation" // Info in ../node/pkg/local_object_storage/engine/evacuate.go + EngineObjectIsMovedToAnotherShard = "object is moved to another shard" // Debug in ../node/pkg/local_object_storage/engine/evacuate.go + MetabaseMissingMatcher = "missing matcher" // Debug in ../node/pkg/local_object_storage/metabase/select.go + MetabaseErrorInFKBTSelection = "error in FKBT selection" // Debug in ../node/pkg/local_object_storage/metabase/select.go + MetabaseCantDecodeListBucketLeaf = "can't decode list bucket leaf" // Debug in ../node/pkg/local_object_storage/metabase/select.go + MetabaseUnknownOperation = "unknown operation" // Debug in ../node/pkg/local_object_storage/metabase/select.go + MetabaseCantIterateOverTheBucket = "can't iterate over the bucket" // Debug in ../node/pkg/local_object_storage/metabase/select.go + MetabaseCouldNotIterateOverTheBuckets = "could not iterate over the buckets" // Debug in ../node/pkg/local_object_storage/metabase/select.go + MetabaseCreatedDirectoryForMetabase = "created directory for Metabase" // Debug in ../node/pkg/local_object_storage/metabase/control.go + MetabaseOpenedBoltDBInstanceForMetabase = "opened boltDB instance for Metabase" // Debug in ../node/pkg/local_object_storage/metabase/control.go + MetabaseCheckingMetabaseVersion = "checking metabase version" // Debug in ../node/pkg/local_object_storage/metabase/control.go + ShardCantSelectAllObjects = "can't select all objects" // Debug in ../node/pkg/local_object_storage/shard/list.go + ShardSettingShardMode = "setting shard mode" // Info in ../node/pkg/local_object_storage/shard/mode.go + ShardShardModeSetSuccessfully = "shard mode set successfully" // Info in ../node/pkg/local_object_storage/shard/mode.go + ShardCouldNotMarkObjectForShardRelocationInMetabase = "could not mark object for shard relocation in metabase" // Debug in ../node/pkg/local_object_storage/shard/move.go + ShardCantDeleteObjectFromWriteCache = "can't delete object from write cache" // Warn in ../node/pkg/local_object_storage/shard/delete.go + ShardCantGetStorageIDFromMetabase = "can't get storage ID from metabase" // Debug in ../node/pkg/local_object_storage/shard/delete.go + ShardCantRemoveObjectFromBlobStor = "can't remove object from blobStor" // Debug in ../node/pkg/local_object_storage/shard/delete.go + ShardFetchingObjectWithoutMeta = "fetching object without meta" // Warn in ../node/pkg/local_object_storage/shard/get.go + ShardObjectIsMissingInWritecache = "object is missing in write-cache" // Debug in ../node/pkg/local_object_storage/shard/get.go + ShardFailedToFetchObjectFromWritecache = "failed to fetch object from write-cache" // Error in ../node/pkg/local_object_storage/shard/get.go + ShardCantPutObjectToTheWritecacheTryingBlobstor = "can't put object to the write-cache, trying blobstor" // Debug in ../node/pkg/local_object_storage/shard/put.go + ShardMetaObjectCounterRead = "meta: object counter read" // Warn in ../node/pkg/local_object_storage/shard/shard.go + ShardMetaCantReadContainerList = "meta: can't read container list" // Warn in ../node/pkg/local_object_storage/shard/shard.go + ShardMetaCantReadContainerSize = "meta: can't read container size" // Warn in ../node/pkg/local_object_storage/shard/shard.go + ShardMetabaseFailureSwitchingMode = "metabase failure, switching mode" // Error in ../node/pkg/local_object_storage/shard/control.go + ShardCantMoveShardToReadonlySwitchMode = "can't move shard to readonly, switch mode" // Error in ../node/pkg/local_object_storage/shard/control.go + ShardCouldNotUnmarshalObject = "could not unmarshal object" // Warn in ../node/pkg/local_object_storage/shard/control.go + ShardCouldNotCloseShardComponent = "could not close shard component" // Error in ../node/pkg/local_object_storage/shard/control.go + ShardCantOpenMetabaseMoveToADegradedMode = "can't open metabase, move to a degraded mode" // Error in ../node/pkg/local_object_storage/shard/control.go + ShardCantInitializeMetabaseMoveToADegradedreadonlyMode = "can't initialize metabase, move to a degraded-read-only mode" // Error in ../node/pkg/local_object_storage/shard/control.go + ShardTryingToRestoreReadwriteMode = "trying to restore read-write mode" // Info in ../node/pkg/local_object_storage/shard/control.go + ShardStopEventListenerByClosedChannel = "stop event listener by closed channel" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardCouldNotSubmitGCJobToWorkerPool = "could not submit GC job to worker pool" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardGCIsStopped = "GC is stopped" // Debug in ../node/pkg/local_object_storage/shard/gc.go + ShardWaitingForGCWorkersToStop = "waiting for GC workers to stop..." // Info in ../node/pkg/local_object_storage/shard/gc.go + ShardIteratorOverMetabaseGraveyardFailed = "iterator over metabase graveyard failed" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardCouldNotDeleteTheObjects = "could not delete the objects" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardIteratorOverExpiredObjectsFailed = "iterator over expired objects failed" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardCouldNotInhumeTheObjects = "could not inhume the objects" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardStartedExpiredTombstonesHandling = "started expired tombstones handling" // Debug in ../node/pkg/local_object_storage/shard/gc.go + ShardIteratingTombstones = "iterating tombstones" // Debug in ../node/pkg/local_object_storage/shard/gc.go + ShardShardIsInADegradedModeSkipCollectingExpiredTombstones = "shard is in a degraded mode, skip collecting expired tombstones" // Debug in ../node/pkg/local_object_storage/shard/gc.go + ShardIteratorOverGraveyardFailed = "iterator over graveyard failed" // Error in ../node/pkg/local_object_storage/shard/gc.go + ShardHandlingExpiredTombstonesBatch = "handling expired tombstones batch" // Debug in ../node/pkg/local_object_storage/shard/gc.go + ShardFinishedExpiredTombstonesHandling = "finished expired tombstones handling" // Debug in ../node/pkg/local_object_storage/shard/gc.go + ShardIteratorOverExpiredLocksFailed = "iterator over expired locks failed" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardCouldNotMarkTombstonesAsGarbage = "could not mark tombstones as garbage" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardCouldNotDropExpiredGraveRecords = "could not drop expired grave records" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardFailureToUnlockObjects = "failure to unlock objects" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardFailureToMarkLockersAsGarbage = "failure to mark lockers as garbage" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardFailureToGetExpiredUnlockedObjects = "failure to get expired unlocked objects" // Warn in ../node/pkg/local_object_storage/shard/gc.go + ShardCouldNotMarkObjectToDeleteInMetabase = "could not mark object to delete in metabase" // Debug in ../node/pkg/local_object_storage/shard/inhume.go + WritecacheTriedToFlushItemsFromWritecache = "tried to flush items from write-cache" // Debug in ../node/pkg/local_object_storage/writecache/flush.go + WritecacheWaitingForChannelsToFlush = "waiting for channels to flush" // Info in ../node/pkg/local_object_storage/writecache/mode.go + WritecacheFillingFlushMarksForObjectsInFSTree = "filling flush marks for objects in FSTree" // Info in ../node/pkg/local_object_storage/writecache/init.go + WritecacheFinishedUpdatingFSTreeFlushMarks = "finished updating FSTree flush marks" // Info in ../node/pkg/local_object_storage/writecache/init.go + WritecacheFillingFlushMarksForObjectsInDatabase = "filling flush marks for objects in database" // Info in ../node/pkg/local_object_storage/writecache/init.go + WritecacheFinishedUpdatingFlushMarks = "finished updating flush marks" // Info in ../node/pkg/local_object_storage/writecache/init.go + WritecacheCantRemoveObjectsFromTheDatabase = "can't remove objects from the database" // Error in ../node/pkg/local_object_storage/writecache/storage.go + WritecacheCantParseAddress = "can't parse address" // Error in ../node/pkg/local_object_storage/writecache/storage.go + WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" // Error in ../node/pkg/local_object_storage/writecache/storage.go + BlobovniczatreeCouldNotGetObjectFromLevel = "could not get object from level" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go + BlobovniczatreeCouldNotReadPayloadRangeFromOpenedBlobovnicza = "could not read payload range from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go + BlobovniczatreeCouldNotReadPayloadRangeFromActiveBlobovnicza = "could not read payload range from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go + BlobovniczatreeCouldNotCloseBlobovnicza = "could not close Blobovnicza" // Error in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go + BlobovniczatreeBlobovniczaSuccessfullyClosedOnEvict = "blobovnicza successfully closed on evict" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go + BlobovniczatreeUpdatingActiveBlobovnicza = "updating active blobovnicza..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go + BlobovniczatreeActiveBlobovniczaSuccessfullyUpdated = "active blobovnicza successfully updated" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go + BlobovniczatreeBlobovniczaSuccessfullyActivated = "blobovnicza successfully activated" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go + BlobovniczatreeCouldNotRemoveObjectFromLevel = "could not remove object from level" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go + BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza = "could not remove object from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go + BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza = "could not remove object from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go + BlobovniczatreeCouldNotGetActiveBlobovnicza = "could not get active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go + BlobovniczatreeBlobovniczaOverflowed = "blobovnicza overflowed" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go + BlobovniczatreeCouldNotUpdateActiveBlobovnicza = "could not update active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go + BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza = "could not put object to active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go + BlobovniczatreeCouldNotReadObjectFromOpenedBlobovnicza = "could not read object from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get.go + BlobovniczatreeCouldNotGetObjectFromActiveBlobovnicza = "could not get object from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get.go + BlobovniczatreeInitializingBlobovniczas = "initializing Blobovnicza's" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go + BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization = "read-only mode, skip blobovniczas initialization..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go + BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing = "blobovnicza successfully initialized, closing..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go + BlobovniczatreeCouldNotCloseActiveBlobovnicza = "could not close active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go + AlphabetTick = "tick" // Info in ../node/pkg/innerring/processors/alphabet/handlers.go + AlphabetAlphabetProcessorWorkerPoolDrained = "alphabet processor worker pool drained" // Warn in ../node/pkg/innerring/processors/alphabet/handlers.go + AlphabetNonAlphabetModeIgnoreGasEmissionEvent = "non alphabet mode, ignore gas emission event" // Info in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent = "node is out of alphabet range, ignore gas emission event" // Debug in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetCantInvokeAlphabetEmitMethod = "can't invoke alphabet emit method" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetStorageNodeEmissionIsOff = "storage node emission is off" // Info in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes = "can't get netmap snapshot to emit gas to storage nodes" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetGasEmission = "gas emission" // Debug in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetCantParseNodePublicKey = "can't parse node public key" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetCantTransferGas = "can't transfer gas" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetCantTransferGasToWallet = "can't transfer gas to wallet" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetAlphabetWorkerPool = "alphabet worker pool" // Debug in ../node/pkg/innerring/processors/alphabet/processor.go + AuditContainerListingFinished = "container listing finished" // Debug in ../node/pkg/innerring/processors/audit/scheduler.go + AuditNewRoundOfAudit = "new round of audit" // Info in ../node/pkg/innerring/processors/audit/handlers.go + AuditPreviousRoundOfAuditPrepareHasntFinishedYet = "previous round of audit prepare hasn't finished yet" // Warn in ../node/pkg/innerring/processors/audit/handlers.go + AuditSomeTasksFromPreviousEpochAreSkipped = "some tasks from previous epoch are skipped" // Info in ../node/pkg/innerring/processors/audit/process.go + AuditContainerSelectionFailure = "container selection failure" // Error in ../node/pkg/innerring/processors/audit/process.go + AuditSelectContainersForAudit = "select containers for audit" // Info in ../node/pkg/innerring/processors/audit/process.go + AuditCantFetchNetworkMap = "can't fetch network map" // Error in ../node/pkg/innerring/processors/audit/process.go + AuditCantGetContainerInfoIgnore = "can't get container info, ignore" // Error in ../node/pkg/innerring/processors/audit/process.go + AuditCantBuildPlacementForContainerIgnore = "can't build placement for container, ignore" // Info in ../node/pkg/innerring/processors/audit/process.go + AuditSelectStorageGroupsForAudit = "select storage groups for audit" // Info in ../node/pkg/innerring/processors/audit/process.go + AuditFilterExpiredStorageGroupsForAudit = "filter expired storage groups for audit" // Info in ../node/pkg/innerring/processors/audit/process.go + AuditParseClientNodeInfo = "parse client node info" // Warn in ../node/pkg/innerring/processors/audit/process.go + AuditErrorInStorageGroupSearch = "error in storage group search" // Warn in ../node/pkg/innerring/processors/audit/process.go + AuditCouldNotGetStorageGroupObjectForAuditSkipping = "could not get storage group object for audit, skipping" // Error in ../node/pkg/innerring/processors/audit/process.go + BalanceNotification = "notification" // Info in ../node/pkg/innerring/processors/balance/handlers.go + BalanceBalanceWorkerPoolDrained = "balance worker pool drained" // Warn in ../node/pkg/innerring/processors/balance/handlers.go + BalanceNonAlphabetModeIgnoreBalanceLock = "non alphabet mode, ignore balance lock" // Info in ../node/pkg/innerring/processors/balance/process_assets.go + BalanceCantSendLockAssetTx = "can't send lock asset tx" // Error in ../node/pkg/innerring/processors/balance/process_assets.go + BalanceBalanceWorkerPool = "balance worker pool" // Debug in ../node/pkg/innerring/processors/balance/processor.go + ContainerContainerWorkerPool = "container worker pool" // Debug in ../node/pkg/innerring/processors/container/processor.go + ContainerNotification = "notification" // Info in ../node/pkg/innerring/processors/container/handlers.go + ContainerContainerProcessorWorkerPoolDrained = "container processor worker pool drained" // Warn in ../node/pkg/innerring/processors/container/handlers.go + ContainerNonAlphabetModeIgnoreContainerPut = "non alphabet mode, ignore container put" // Info in ../node/pkg/innerring/processors/container/process_container.go + ContainerPutContainerCheckFailed = "put container check failed" // Error in ../node/pkg/innerring/processors/container/process_container.go + ContainerCouldNotApprovePutContainer = "could not approve put container" // Error in ../node/pkg/innerring/processors/container/process_container.go + ContainerNonAlphabetModeIgnoreContainerDelete = "non alphabet mode, ignore container delete" // Info in ../node/pkg/innerring/processors/container/process_container.go + ContainerDeleteContainerCheckFailed = "delete container check failed" // Error in ../node/pkg/innerring/processors/container/process_container.go + ContainerCouldNotApproveDeleteContainer = "could not approve delete container" // Error in ../node/pkg/innerring/processors/container/process_container.go + ContainerNonAlphabetModeIgnoreSetEACL = "non alphabet mode, ignore set EACL" // Info in ../node/pkg/innerring/processors/container/process_eacl.go + ContainerSetEACLCheckFailed = "set EACL check failed" // Error in ../node/pkg/innerring/processors/container/process_eacl.go + ContainerCouldNotApproveSetEACL = "could not approve set EACL" // Error in ../node/pkg/innerring/processors/container/process_eacl.go + FrostFSNonAlphabetModeIgnoreBind = "non alphabet mode, ignore bind" // Info in ../node/pkg/innerring/processors/frostfs/process_bind.go + FrostFSInvalidManageKeyEvent = "invalid manage key event" // Error in ../node/pkg/innerring/processors/frostfs/process_bind.go + FrostFSCouldNotDecodeScriptHashFromBytes = "could not decode script hash from bytes" // Error in ../node/pkg/innerring/processors/frostfs/process_bind.go + FrostFSNonAlphabetModeIgnoreConfig = "non alphabet mode, ignore config" // Info in ../node/pkg/innerring/processors/frostfs/process_config.go + FrostFSCantRelaySetConfigEvent = "can't relay set config event" // Error in ../node/pkg/innerring/processors/frostfs/process_config.go + FrostFSFrostfsWorkerPool = "frostfs worker pool" // Debug in ../node/pkg/innerring/processors/frostfs/processor.go + FrostFSNotification = "notification" // Info in ../node/pkg/innerring/processors/frostfs/handlers.go + FrostFSFrostfsProcessorWorkerPoolDrained = "frostfs processor worker pool drained" // Warn in ../node/pkg/innerring/processors/frostfs/handlers.go + FrostFSNonAlphabetModeIgnoreDeposit = "non alphabet mode, ignore deposit" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantTransferAssetsToBalanceContract = "can't transfer assets to balance contract" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSDoubleMintEmissionDeclined = "double mint emission declined" // Warn in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantGetGasBalanceOfTheNode = "can't get gas balance of the node" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSGasBalanceThresholdHasBeenReached = "gas balance threshold has been reached" // Warn in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantTransferNativeGasToReceiver = "can't transfer native gas to receiver" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSNonAlphabetModeIgnoreWithdraw = "non alphabet mode, ignore withdraw" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantCreateLockAccount = "can't create lock account" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantLockAssetsForWithdraw = "can't lock assets for withdraw" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSNonAlphabetModeIgnoreCheque = "non alphabet mode, ignore cheque" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantTransferAssetsToFedContract = "can't transfer assets to fed contract" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + GovernanceNewEvent = "new event" // Info in ../node/pkg/innerring/processors/governance/handlers.go + GovernanceGovernanceWorkerPoolDrained = "governance worker pool drained" // Warn in ../node/pkg/innerring/processors/governance/handlers.go + GovernanceNonAlphabetModeIgnoreAlphabetSync = "non alphabet mode, ignore alphabet sync" // Info in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantFetchAlphabetListFromMainNet = "can't fetch alphabet list from main net" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantFetchAlphabetListFromSideChain = "can't fetch alphabet list from side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantMergeAlphabetListsFromMainNetAndSideChain = "can't merge alphabet lists from main net and side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged = "no governance update, alphabet list has not been changed" // Info in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceAlphabetListHasBeenChangedStartingUpdate = "alphabet list has been changed, starting update" // Info in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantVoteForSideChainCommittee = "can't vote for side chain committee" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceFinishedAlphabetListUpdate = "finished alphabet list update" // Info in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantFetchInnerRingListFromSideChain = "can't fetch inner ring list from side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys = "can't create new inner ring list with new alphabet keys" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceUpdateOfTheInnerRingList = "update of the inner ring list" // Info in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantUpdateInnerRingListWithNewAlphabetKeys = "can't update inner ring list with new alphabet keys" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantUpdateListOfNotaryNodesInSideChain = "can't update list of notary nodes in side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract = "can't update list of alphabet nodes in frostfs contract" // Error in ../node/pkg/innerring/processors/governance/process_update.go + NetmapNetmapWorkerPool = "netmap worker pool" // Debug in ../node/pkg/innerring/processors/netmap/processor.go + NetmapTick = "tick" // Info in ../node/pkg/innerring/processors/netmap/handlers.go + NetmapNetmapWorkerPoolDrained = "netmap worker pool drained" // Warn in ../node/pkg/innerring/processors/netmap/handlers.go + NetmapNotification = "notification" // Info in ../node/pkg/innerring/processors/netmap/handlers.go + NetmapNetmapCleanUpRoutineIsDisabled518 = "netmap clean up routine is disabled" // Debug in ../node/pkg/innerring/processors/netmap/handlers.go + NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick = "non alphabet mode, ignore new netmap cleanup tick" // Info in ../node/pkg/innerring/processors/netmap/process_cleanup.go + NetmapCantDecodePublicKeyOfNetmapNode = "can't decode public key of netmap node" // Warn in ../node/pkg/innerring/processors/netmap/process_cleanup.go + NetmapVoteToRemoveNodeFromNetmap = "vote to remove node from netmap" // Info in ../node/pkg/innerring/processors/netmap/process_cleanup.go + NetmapCantInvokeNetmapUpdateState = "can't invoke netmap.UpdateState" // Error in ../node/pkg/innerring/processors/netmap/process_cleanup.go + NetmapCantIterateOnNetmapCleanerCache = "can't iterate on netmap cleaner cache" // Warn in ../node/pkg/innerring/processors/netmap/process_cleanup.go + NetmapCantGetEpochDuration = "can't get epoch duration" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapCantGetTransactionHeight = "can't get transaction height" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapCantResetEpochTimer = "can't reset epoch timer" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapCantGetNetmapSnapshotToPerformCleanup = "can't get netmap snapshot to perform cleanup" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapCantStartContainerSizeEstimation = "can't start container size estimation" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapNonAlphabetModeIgnoreNewEpochTick = "non alphabet mode, ignore new epoch tick" // Info in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapNextEpoch = "next epoch" // Debug in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapCantInvokeNetmapNewEpoch = "can't invoke netmap.NewEpoch" // Error in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapNonAlphabetModeIgnoreNewPeerNotification = "non alphabet mode, ignore new peer notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapNonhaltNotaryTransaction = "non-halt notary transaction" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCantParseNetworkMapCandidate = "can't parse network map candidate" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate = "could not verify and update information about network map candidate" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapApprovingNetworkMapCandidate = "approving network map candidate" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCantInvokeNetmapAddPeer = "can't invoke netmap.AddPeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapNonAlphabetModeIgnoreUpdatePeerNotification = "non alphabet mode, ignore update peer notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapPreventSwitchingNodeToMaintenanceState = "prevent switching node to maintenance state" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapNonAlphabetModeIgnoreRemoveNodeFromSubnetNotification = "non alphabet mode, ignore remove node from subnet notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCouldNotGetNetworkMapCandidates = "could not get network map candidates" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCouldNotUnmarshalSubnetId = "could not unmarshal subnet id" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapGotZeroSubnetInRemoveNodeNotification = "got zero subnet in remove node notification" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCouldNotIterateOverSubnetworksOfTheNode = "could not iterate over subnetworks of the node" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCouldNotInvokeNetmapUpdateState = "could not invoke netmap.UpdateState" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCouldNotInvokeNetmapAddPeer = "could not invoke netmap.AddPeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go + ReputationNotification = "notification" // Info in ../node/pkg/innerring/processors/reputation/handlers.go + ReputationReputationWorkerPoolDrained = "reputation worker pool drained" // Warn in ../node/pkg/innerring/processors/reputation/handlers.go + ReputationNonAlphabetModeIgnoreReputationPutNotification = "non alphabet mode, ignore reputation put notification" // Info in ../node/pkg/innerring/processors/reputation/process_put.go + ReputationIgnoreReputationValue = "ignore reputation value" // Info in ../node/pkg/innerring/processors/reputation/process_put.go + ReputationCantSendApprovalTxForReputationValue = "can't send approval tx for reputation value" // Warn in ../node/pkg/innerring/processors/reputation/process_put.go + ReputationReputationWorkerPool = "reputation worker pool" // Debug in ../node/pkg/innerring/processors/reputation/processor.go + SettlementNonAlphabetModeIgnoreAuditPayments = "non alphabet mode, ignore audit payments" // Info in ../node/pkg/innerring/processors/settlement/calls.go + SettlementNewAuditSettlementEvent = "new audit settlement event" // Info in ../node/pkg/innerring/processors/settlement/calls.go + SettlementIgnoreGenesisEpoch = "ignore genesis epoch" // Debug in ../node/pkg/innerring/processors/settlement/calls.go + SettlementCouldNotAddHandlerOfAuditEventToQueue = "could not add handler of AuditEvent to queue" // Warn in ../node/pkg/innerring/processors/settlement/calls.go + SettlementAuditEventHandlingSuccessfullyScheduled = "AuditEvent handling successfully scheduled" // Debug in ../node/pkg/innerring/processors/settlement/calls.go + SettlementNonAlphabetModeIgnoreIncomeCollectionEvent = "non alphabet mode, ignore income collection event" // Info in ../node/pkg/innerring/processors/settlement/calls.go + SettlementStartBasicIncomeCollection = "start basic income collection" // Info in ../node/pkg/innerring/processors/settlement/calls.go + SettlementIncomeContextAlreadyExists = "income context already exists" // Error in ../node/pkg/innerring/processors/settlement/calls.go + SettlementCantCreateIncomeContext = "can't create income context" // Error in ../node/pkg/innerring/processors/settlement/calls.go + SettlementCouldNotAddHandlerOfBasicIncomeCollectionToQueue = "could not add handler of basic income collection to queue" // Warn in ../node/pkg/innerring/processors/settlement/calls.go + SettlementNonAlphabetModeIgnoreIncomeDistributionEvent = "non alphabet mode, ignore income distribution event" // Info in ../node/pkg/innerring/processors/settlement/calls.go + SettlementStartBasicIncomeDistribution = "start basic income distribution" // Info in ../node/pkg/innerring/processors/settlement/calls.go + SettlementIncomeContextDistributionDoesNotExists = "income context distribution does not exists" // Warn in ../node/pkg/innerring/processors/settlement/calls.go + SettlementCouldNotAddHandlerOfBasicIncomeDistributionToQueue = "could not add handler of basic income distribution to queue" // Warn in ../node/pkg/innerring/processors/settlement/calls.go + SettlementProcessAuditSettlements = "process audit settlements" // Info in ../node/pkg/innerring/processors/settlement/handlers.go + SettlementAuditProcessingFinished = "audit processing finished" // Info in ../node/pkg/innerring/processors/settlement/handlers.go + SettlementWorkerPoolForSettlementProcessorSuccessfullyInitialized = "worker pool for settlement processor successfully initialized" // Debug in ../node/pkg/innerring/processors/settlement/processor.go + AuditSettlementsAreIgnoredForZeroEpoch = "settlements are ignored for zero epoch" // Info in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditCalculateAuditSettlements = "calculate audit settlements" // Info in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditGettingResultsForThePreviousEpoch = "getting results for the previous epoch" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditCouldNotCollectAuditResults = "could not collect audit results" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditNoAuditResultsInPreviousEpoch = "no audit results in previous epoch" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditCantFetchAuditFeeFromNetworkConfig = "can't fetch audit fee from network config" // Warn in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditProcessingAuditResults = "processing audit results" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditProcessingTransfers = "processing transfers" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditReadingInformationAboutTheContainer = "reading information about the container" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditBuildingPlacement = "building placement" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditCollectingPassedNodes = "collecting passed nodes" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditCalculatingSumOfTheSizesOfAllStorageGroups = "calculating sum of the sizes of all storage groups" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditFillingTransferTable = "filling transfer table" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditMissingContainerInAuditResult = "missing container in audit result" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditCouldNotGetContainerInfo = "could not get container info" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditCouldNotGetContainerNodes = "could not get container nodes" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditEmptyListOfContainerNodes = "empty list of container nodes" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditNoneOfTheContainerNodesPassedTheAudit = "none of the container nodes passed the audit" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditCouldNotGetSGInfo = "could not get storage group info" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditZeroSumSGSize = "zero sum storage group size" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditCouldNotResolvePublicKeyOfTheStorageNode = "could not resolve public key of the storage node" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditCalculatingStorageNodeSalaryForAudit = "calculating storage node salary for audit (GASe-12)" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go + AuditCouldNotParsePublicKeyOfTheInnerRingNode = "could not parse public key of the inner ring node" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go + BasicCantGetBasicIncomeRate = "can't get basic income rate" // Error in ../node/pkg/innerring/processors/settlement/basic/collect.go + BasicCantFetchContainerSizeEstimations = "can't fetch container size estimations" // Error in ../node/pkg/innerring/processors/settlement/basic/collect.go + BasicCantFetchContainerInfo = "can't fetch container info" // Warn in ../node/pkg/innerring/processors/settlement/basic/collect.go + BasicCantFetchBalanceOfBankingAccount = "can't fetch balance of banking account" // Error in ../node/pkg/innerring/processors/settlement/basic/distribute.go + BasicCantTransformPublicKeyToOwnerID = "can't transform public key to owner id" // Warn in ../node/pkg/innerring/processors/settlement/basic/distribute.go + FrostFSIRInternalError = "internal error" // Info in ../node/cmd/frostfs-ir/main.go + FrostFSIRCouldNotShutdownHTTPServer = "could not shutdown HTTP server" // Debug in ../node/cmd/frostfs-ir/main.go + FrostFSIRApplicationStopped = "application stopped" // Info in ../node/cmd/frostfs-ir/main.go + FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" // Error in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" // Error in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeStartListeningGRPCEndpoint = "start listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeStoppingGRPCServer = "stopping gRPC server..." // Info in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeGRPCCannotShutdownGracefullyForcingStop = "gRPC cannot shutdown gracefully, forcing stop" // Info in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeGRPCServerStoppedSuccessfully = "gRPC server stopped successfully" // Info in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeWaitingForAllProcessesToStop = "waiting for all processes to stop" // Debug in ../node/cmd/frostfs-node/main.go + FrostFSNodeStartReportingReputationOnNewEpochEvent = "start reporting reputation on new epoch event" // Debug in ../node/cmd/frostfs-node/reputation.go + FrostFSNodeCouldNotFetchEpochDuration = "could not fetch epoch duration" // Debug in ../node/cmd/frostfs-node/reputation.go + FrostFSNodeCouldNotFetchIterationNumber = "could not fetch iteration number" // Debug in ../node/cmd/frostfs-node/reputation.go + FrostFSNodeCouldNotCreateFixedEpochTimer = "could not create fixed epoch timer" // Debug in ../node/cmd/frostfs-node/reputation.go + FrostFSNodeStartedLocalNodesMaintenance = "started local node's maintenance" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeStoppedLocalNodesMaintenance = "stopped local node's maintenance" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeFailedToAttachShardToEngine = "failed to attach shard to engine" // Error in ../node/cmd/frostfs-node/config.go + FrostFSNodeShardAttachedToEngine = "shard attached to engine" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeClosingComponentsOfTheStorageEngine = "closing components of the storage engine..." // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeStorageEngineClosingFailure = "storage engine closing failure" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully = "all components of the storage engine closed successfully" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeBootstrappingWithTheMaintenanceState = "bootstrapping with the maintenance state" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeBootstrappingWithOnlineState = "bootstrapping with online state" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeTerminationSignalHasBeenReceivedStopping = "termination signal has been received, stopping..." // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeTerminationSignalProcessingIsComplete = "termination signal processing is complete" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeInternalApplicationError = "internal application error" // Warn in ../node/cmd/frostfs-node/config.go + FrostFSNodeInternalErrorProcessingIsComplete = "internal error processing is complete" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration = "SIGHUP has been received, rereading configuration..." // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeConfigurationReading = "configuration reading" // Error in ../node/cmd/frostfs-node/config.go + FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" // Error in ../node/cmd/frostfs-node/config.go + FrostFSNodeTracingConfigationUpdated = "tracing configation updated" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeStorageEngineConfigurationUpdate = "storage engine configuration update" // Error in ../node/cmd/frostfs-node/config.go + FrostFSNodeUpdatedConfigurationApplying = "updated configuration applying" // Error in ../node/cmd/frostfs-node/config.go + FrostFSNodeConfigurationHasBeenReloadedSuccessfully = "configuration has been reloaded successfully" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeReadNewlyCreatedContainerAfterTheNotification = "read newly created container after the notification" // Error in ../node/cmd/frostfs-node/container.go + FrostFSNodeContainerCreationEventsReceipt = "container creation event's receipt" // Debug in ../node/cmd/frostfs-node/container.go + FrostFSNodeContainerRemovalEventsReceipt = "container removal event's receipt" // Debug in ../node/cmd/frostfs-node/container.go + FrostFSNodeSaveUsedSpaceAnnouncementInContract = "save used space announcement in contract" // Debug in ../node/cmd/frostfs-node/container.go + FrostFSNodeFailedToCalculateContainerSizeInStorageEngine = "failed to calculate container size in storage engine" // Debug in ../node/cmd/frostfs-node/container.go + FrostFSNodeContainerSizeInStorageEngineCalculatedSuccessfully = "container size in storage engine calculated successfully" // Debug in ../node/cmd/frostfs-node/container.go + FrostFSNodeNotificatorCouldNotListContainers = "notificator: could not list containers" // Error in ../node/cmd/frostfs-node/notificator.go + FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer = "notificator: could not select objects from container" // Error in ../node/cmd/frostfs-node/notificator.go + FrostFSNodeNotificatorCouldNotProcessObject = "notificator: could not process object" // Error in ../node/cmd/frostfs-node/notificator.go + FrostFSNodeNotificatorFinishedProcessingObjectNotifications = "notificator: finished processing object notifications" // Debug in ../node/cmd/frostfs-node/notificator.go + FrostFSNodeCouldNotWriteObjectNotification = "could not write object notification" // Warn in ../node/cmd/frostfs-node/notificator.go + FrostFSNodeCouldNotGetMaxObjectSizeValue = "could not get max object size value" // Error in ../node/cmd/frostfs-node/object.go + FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage = "could not inhume mark redundant copy as garbage" // Warn in ../node/cmd/frostfs-node/object.go + FrostFSNodeWritingLocalReputationValues = "writing local reputation values" // Debug in ../node/cmd/frostfs-node/object.go + FrostFSNodeCouldNotGetLatestNetworkMapToOverloadTheClient = "could not get latest network map to overload the client" // Warn in ../node/cmd/frostfs-node/object.go + FrostFSNodeFailedInitTracing = "failed init tracing" // Error in ../node/cmd/frostfs-node/tracing.go + FrostFSNodeFailedShutdownTracing = "failed shutdown tracing" // Error in ../node/cmd/frostfs-node/tracing.go + FrostFSNodeFailedToCreateNeoRPCClient = "failed to create neo RPC client" // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeClosingMorphComponents = "closing morph components..." // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal = "failed to set group signer scope, continue with Global" // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeNotarySupport = "notary support" // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeMorphcacheTTLFetchedFromNetwork = "morph.cache_ttl fetched from network" // Debug in ../node/cmd/frostfs-node/morph.go + FrostFSNodeNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" // Warn in ../node/cmd/frostfs-node/morph.go + FrostFSNodeNewEpochEventFromSidechain = "new epoch event from sidechain" // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeNewBlock = "new block" // Debug in ../node/cmd/frostfs-node/morph.go + FrostFSNodeCantUpdatePersistentState = "can't update persistent state" // Warn in ../node/cmd/frostfs-node/morph.go + FrostFSNodeCantSendRebootstrapTx = "can't send re-bootstrap tx" // Warn in ../node/cmd/frostfs-node/netmap.go + FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch = "could not update node state on new epoch" // Error in ../node/cmd/frostfs-node/netmap.go + FrostFSNodeCouldNotMakeNotaryDeposit = "could not make notary deposit" // Error in ../node/cmd/frostfs-node/netmap.go + FrostFSNodeInitialNetworkState = "initial network state" // Info in ../node/cmd/frostfs-node/netmap.go + FrostFSNodeTreeServiceIsNotEnabledSkipInitialization = "tree service is not enabled, skip initialization" // Info in ../node/cmd/frostfs-node/tree.go + FrostFSNodeCouldNotSynchronizeTreeService = "could not synchronize Tree Service" // Error in ../node/cmd/frostfs-node/tree.go + FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" // Debug in ../node/cmd/frostfs-node/tree.go + FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" // Error in ../node/cmd/frostfs-node/tree.go + FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" // Error in ../node/cmd/frostfs-node/control.go + CommonApplicationStarted = "application started" // Info in ../node/cmd/frostfs-ir/main.go + CommonInitializingRemoteWriterProvider = "initializing remote writer provider" // Debug in ../node/cmd/frostfs-node/reputation/common/remote.go + CommonRouteHasReachedDeadendProvider = "route has reached dead-end provider" // Debug in ../node/cmd/frostfs-node/reputation/common/remote.go + CommonInitializingNoopWriterProvider = "initializing no-op writer provider" // Debug in ../node/cmd/frostfs-node/reputation/common/remote.go + IntermediateWritingReceivedConsumersTrusts = "writing received consumer's trusts" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/consumers.go + IntermediateStartWritingGlobalTrustsToContract = "start writing global trusts to contract" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/contract.go + IntermediateFailedToSignGlobalTrust = "failed to sign global trust" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/contract.go + IntermediateFailedToWriteGlobalTrustToContract = "failed to write global trust to contract" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/contract.go + IntermediateSentGlobalTrustToContract = "sent global trust to contract" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/contract.go + IntermediateWritingReceivedDaughtersTrusts = "writing received daughter's trusts" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/daughters.go + IntermediateAnnouncingTrust = "announcing trust" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/remote.go + LocalAnnouncingTrusts = "announcing trusts" // Debug in ../node/cmd/frostfs-node/reputation/local/remote.go + LocalInitializingIteratorOverTrusts = "initializing iterator over trusts" // Debug in ../node/cmd/frostfs-node/reputation/local/storage.go +) diff --git a/pkg/innerring/blocktimer.go b/pkg/innerring/blocktimer.go index eb74e44d4..94e262099 100644 --- a/pkg/innerring/blocktimer.go +++ b/pkg/innerring/blocktimer.go @@ -3,6 +3,7 @@ package innerring import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" @@ -98,7 +99,7 @@ func newEpochTimer(args *epochTimerArgs) *timer.BlockTimer { args.stopEstimationDDiv, func() { if !args.alphabetState.IsAlphabet() { - args.l.Debug("non-alphabet mode, do not stop container estimations") + args.l.Debug(logs.InnerringNonalphabetModeDoNotStopContainerEstimations) return } @@ -112,7 +113,7 @@ func newEpochTimer(args *epochTimerArgs) *timer.BlockTimer { err := args.cnrWrapper.StopEstimation(prm) if err != nil { - args.l.Warn("can't stop epoch estimation", + args.l.Warn(logs.InnerringCantStopEpochEstimation, zap.Uint64("epoch", epochN), zap.String("error", err.Error())) } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 8db6328a2..2f5e89e39 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -6,6 +6,7 @@ import ( "fmt" "net" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/balance" @@ -129,7 +130,7 @@ func (s *Server) initMainnet(ctx context.Context, cfg *viper.Viper, morphChain * fromMainChainBlock, err := s.persistate.UInt32(persistateMainChainLastBlockKey) if err != nil { fromMainChainBlock = 0 - s.log.Warn("can't get last processed main chain block number", zap.String("error", err.Error())) + s.log.Warn(logs.InnerringCantGetLastProcessedMainChainBlockNumber, zap.String("error", err.Error())) } mainnetChain.from = fromMainChainBlock @@ -177,7 +178,7 @@ func (s *Server) initNotaryConfig(cfg *viper.Viper) { !s.withoutMainNet && s.mainnetClient.ProbeNotary(), // if mainnet disabled then notary flag must be disabled too ) - s.log.Info("notary support", + s.log.Info(logs.InnerringNotarySupport, zap.Bool("sidechain_enabled", !s.sideNotaryConfig.disabled), zap.Bool("mainchain_enabled", !s.mainNotaryConfig.disabled), ) @@ -275,7 +276,7 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli if s.withoutMainNet || cfg.GetBool("governance.disable") { alphaSync = func(event.Event) { - s.log.Debug("alphabet keys sync is disabled") + s.log.Debug(logs.InnerringAlphabetKeysSyncIsDisabled) } } else { // create governance processor @@ -496,7 +497,7 @@ func (s *Server) initReputationProcessor(cfg *viper.Viper, sidechainFee fixedn.F func (s *Server) initGRPCServer(cfg *viper.Viper) error { controlSvcEndpoint := cfg.GetString("control.grpc.endpoint") if controlSvcEndpoint == "" { - s.log.Info("no Control server endpoint specified, service is disabled") + s.log.Info(logs.InnerringNoControlServerEndpointSpecified) return nil } @@ -692,7 +693,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- fromSideChainBlock, err := s.persistate.UInt32(persistateSideChainLastBlockKey) if err != nil { fromSideChainBlock = 0 - s.log.Warn("can't get last processed side chain block number", zap.String("error", err.Error())) + s.log.Warn(logs.InnerringCantGetLastProcessedSideChainBlockNumber, zap.String("error", err.Error())) } morphChain := &chainParams{ @@ -715,7 +716,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- return nil, err } if err := s.morphClient.SetGroupSignerScope(); err != nil { - morphChain.log.Info("failed to set group signer scope, continue with Global", zap.Error(err)) + morphChain.log.Info(logs.InnerringFailedToSetGroupSignerScope, zap.Error(err)) } return morphChain, nil diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index a91d2fd0d..b6c5ae2ac 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -6,6 +6,7 @@ import ( "fmt" "io" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap" @@ -168,7 +169,7 @@ func (s *Server) Start(ctx context.Context, intError chan<- error) (err error) { err = s.voteForSidechainValidator(prm) if err != nil { // we don't stop inner ring execution on this error - s.log.Warn("can't vote for prepared validators", + s.log.Warn(logs.InnerringCantVoteForPreparedValidators, zap.String("error", err.Error())) } @@ -210,13 +211,13 @@ func (s *Server) Start(ctx context.Context, intError chan<- error) (err error) { func (s *Server) registerMorphNewBlockEventHandler() { s.morphListener.RegisterBlockHandler(func(b *block.Block) { - s.log.Debug("new block", + s.log.Debug(logs.InnerringNewBlock, zap.Uint32("index", b.Index), ) err := s.persistate.SetUInt32(persistateSideChainLastBlockKey, b.Index) if err != nil { - s.log.Warn("can't update persistent state", + s.log.Warn(logs.InnerringCantUpdatePersistentState, zap.String("chain", "side"), zap.Uint32("block_index", b.Index)) } @@ -230,7 +231,7 @@ func (s *Server) registerMainnetNewBlockEventHandler() { s.mainnetListener.RegisterBlockHandler(func(b *block.Block) { err := s.persistate.SetUInt32(persistateMainChainLastBlockKey, b.Index) if err != nil { - s.log.Warn("can't update persistent state", + s.log.Warn(logs.InnerringCantUpdatePersistentState, zap.String("chain", "main"), zap.Uint32("block_index", b.Index)) } @@ -302,7 +303,7 @@ func (s *Server) Stop() { for _, c := range s.closers { if err := c(); err != nil { - s.log.Warn("closer error", + s.log.Warn(logs.InnerringCloserError, zap.String("error", err.Error()), ) } @@ -547,7 +548,7 @@ func (s *Server) initConfigFromBlockchain() error { return err } - s.log.Debug("read config from blockchain", + s.log.Debug(logs.InnerringReadConfigFromBlockchain, zap.Bool("active", s.IsActive()), zap.Bool("alphabet", s.IsAlphabet()), zap.Uint64("epoch", epoch), diff --git a/pkg/innerring/notary.go b/pkg/innerring/notary.go index 50353b574..30916cb99 100644 --- a/pkg/innerring/notary.go +++ b/pkg/innerring/notary.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/util" @@ -52,14 +53,14 @@ func (s *Server) notaryHandler(_ event.Event) { if !s.mainNotaryConfig.disabled { _, err := s.depositMainNotary() if err != nil { - s.log.Error("can't make notary deposit in main chain", zap.Error(err)) + s.log.Error(logs.InnerringCantMakeNotaryDepositInMainChain, zap.Error(err)) } } if !s.sideNotaryConfig.disabled { _, err := s.depositSideNotary() if err != nil { - s.log.Error("can't make notary deposit in side chain", zap.Error(err)) + s.log.Error(logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) } } } @@ -82,7 +83,7 @@ func (s *Server) initNotary(ctx context.Context, deposit depositor, await awaite // non-error deposit with an empty TX hash means // that the deposit has already been made; no // need to wait it. - s.log.Info("notary deposit has already been made") + s.log.Info(logs.InnerringNotaryDepositHasAlreadyBeenMade) return nil } diff --git a/pkg/innerring/processors/alphabet/handlers.go b/pkg/innerring/processors/alphabet/handlers.go index 9d61aa812..c0668a4f9 100644 --- a/pkg/innerring/processors/alphabet/handlers.go +++ b/pkg/innerring/processors/alphabet/handlers.go @@ -1,6 +1,7 @@ package alphabet import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "go.uber.org/zap" @@ -8,14 +9,14 @@ import ( func (ap *Processor) HandleGasEmission(ev event.Event) { _ = ev.(timers.NewAlphabetEmitTick) - ap.log.Info("tick", zap.String("type", "alphabet gas emit")) + ap.log.Info(logs.AlphabetTick, zap.String("type", "alphabet gas emit")) // send event to the worker pool err := ap.pool.Submit(func() { ap.processEmit() }) if err != nil { // there system can be moved into controlled degradation stage - ap.log.Warn("alphabet processor worker pool drained", + ap.log.Warn(logs.AlphabetAlphabetProcessorWorkerPoolDrained, zap.Int("capacity", ap.pool.Cap())) } } diff --git a/pkg/innerring/processors/alphabet/process_emit.go b/pkg/innerring/processors/alphabet/process_emit.go index 90c484b88..b8d65dbc5 100644 --- a/pkg/innerring/processors/alphabet/process_emit.go +++ b/pkg/innerring/processors/alphabet/process_emit.go @@ -3,6 +3,7 @@ package alphabet import ( "crypto/elliptic" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" @@ -14,14 +15,14 @@ const emitMethod = "emit" func (ap *Processor) processEmit() { index := ap.irList.AlphabetIndex() if index < 0 { - ap.log.Info("non alphabet mode, ignore gas emission event") + ap.log.Info(logs.AlphabetNonAlphabetModeIgnoreGasEmissionEvent) return } contract, ok := ap.alphabetContracts.GetByIndex(index) if !ok { - ap.log.Debug("node is out of alphabet range, ignore gas emission event", + ap.log.Debug(logs.AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent, zap.Int("index", index)) return @@ -30,20 +31,20 @@ func (ap *Processor) processEmit() { // there is no signature collecting, so we don't need extra fee err := ap.morphClient.Invoke(contract, 0, emitMethod) if err != nil { - ap.log.Warn("can't invoke alphabet emit method", zap.String("error", err.Error())) + ap.log.Warn(logs.AlphabetCantInvokeAlphabetEmitMethod, zap.String("error", err.Error())) return } if ap.storageEmission == 0 { - ap.log.Info("storage node emission is off") + ap.log.Info(logs.AlphabetStorageNodeEmissionIsOff) return } networkMap, err := ap.netmapClient.NetMap() if err != nil { - ap.log.Warn("can't get netmap snapshot to emit gas to storage nodes", + ap.log.Warn(logs.AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes, zap.String("error", err.Error())) return @@ -53,7 +54,7 @@ func (ap *Processor) processEmit() { nmLen := len(nmNodes) extraLen := len(ap.parsedWallets) - ap.log.Debug("gas emission", + ap.log.Debug(logs.AlphabetGasEmission, zap.Int("network_map", nmLen), zap.Int("extra_wallets", extraLen)) @@ -74,7 +75,7 @@ func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerN key, err := keys.NewPublicKeyFromBytes(keyBytes, elliptic.P256()) if err != nil { - ap.log.Warn("can't parse node public key", + ap.log.Warn(logs.AlphabetCantParseNodePublicKey, zap.String("error", err.Error())) continue @@ -82,7 +83,7 @@ func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerN err = ap.morphClient.TransferGas(key.GetScriptHash(), gasPerNode) if err != nil { - ap.log.Warn("can't transfer gas", + ap.log.Warn(logs.AlphabetCantTransferGas, zap.String("receiver", key.Address()), zap.Int64("amount", int64(gasPerNode)), zap.String("error", err.Error()), @@ -99,7 +100,7 @@ func (ap *Processor) transferGasToExtraNodes(extraLen int, gasPerNode fixedn.Fix for i, addr := range ap.parsedWallets { receiversLog[i] = addr.StringLE() } - ap.log.Warn("can't transfer gas to wallet", + ap.log.Warn(logs.AlphabetCantTransferGasToWallet, zap.Strings("receivers", receiversLog), zap.Int64("amount", int64(gasPerNode)), zap.String("error", err.Error()), diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index 980158132..79b61f14f 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -67,7 +68,7 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/alphabet: global state is not set") } - p.Log.Debug("alphabet worker pool", zap.Int("size", p.PoolSize)) + p.Log.Debug(logs.AlphabetAlphabetWorkerPool, zap.Int("size", p.PoolSize)) pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { diff --git a/pkg/innerring/processors/audit/handlers.go b/pkg/innerring/processors/audit/handlers.go index 8b2354bb8..06c656fa2 100644 --- a/pkg/innerring/processors/audit/handlers.go +++ b/pkg/innerring/processors/audit/handlers.go @@ -1,6 +1,7 @@ package audit import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "go.uber.org/zap" ) @@ -10,12 +11,12 @@ func (ap *Processor) handleNewAuditRound(ev event.Event) { epoch := auditEvent.Epoch() - ap.log.Info("new round of audit", zap.Uint64("epoch", epoch)) + ap.log.Info(logs.AuditNewRoundOfAudit, zap.Uint64("epoch", epoch)) // send an event to the worker pool err := ap.pool.Submit(func() { ap.processStartAudit(epoch) }) if err != nil { - ap.log.Warn("previous round of audit prepare hasn't finished yet") + ap.log.Warn(logs.AuditPreviousRoundOfAuditPrepareHasntFinishedYet) } } diff --git a/pkg/innerring/processors/audit/process.go b/pkg/innerring/processors/audit/process.go index 656927816..000279f01 100644 --- a/pkg/innerring/processors/audit/process.go +++ b/pkg/innerring/processors/audit/process.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/storagegroup" @@ -24,23 +25,23 @@ func (ap *Processor) processStartAudit(epoch uint64) { skipped := ap.taskManager.Reset() if skipped > 0 { - ap.log.Info("some tasks from previous epoch are skipped", + ap.log.Info(logs.AuditSomeTasksFromPreviousEpochAreSkipped, zap.Int("amount", skipped), ) } containers, err := ap.selectContainersToAudit(epoch) if err != nil { - log.Error("container selection failure", zap.String("error", err.Error())) + log.Error(logs.AuditContainerSelectionFailure, zap.String("error", err.Error())) return } - log.Info("select containers for audit", zap.Int("amount", len(containers))) + log.Info(logs.AuditSelectContainersForAudit, zap.Int("amount", len(containers))) nm, err := ap.netmapClient.GetNetMap(0) if err != nil { - ap.log.Error("can't fetch network map", + ap.log.Error(logs.AuditCantFetchNetworkMap, zap.String("error", err.Error())) return @@ -64,7 +65,7 @@ func (ap *Processor) startAuditTasksOnContainers(cancelChannel <-chan struct{}, for i := range containers { cnr, err := cntClient.Get(ap.containerClient, containers[i]) // get container structure if err != nil { - log.Error("can't get container info, ignore", + log.Error(logs.AuditCantGetContainerInfoIgnore, zap.Stringer("cid", containers[i]), zap.String("error", err.Error())) @@ -76,7 +77,7 @@ func (ap *Processor) startAuditTasksOnContainers(cancelChannel <-chan struct{}, // find all container nodes for current epoch nodes, err := nm.ContainerNodes(cnr.Value.PlacementPolicy(), pivot) if err != nil { - log.Info("can't build placement for container, ignore", + log.Info(logs.AuditCantBuildPlacementForContainerIgnore, zap.Stringer("cid", containers[i]), zap.String("error", err.Error())) @@ -92,13 +93,13 @@ func (ap *Processor) startAuditTasksOnContainers(cancelChannel <-chan struct{}, // search storage groups storageGroupsIDs := ap.findStorageGroups(containers[i], n) - log.Info("select storage groups for audit", + log.Info(logs.AuditSelectStorageGroupsForAudit, zap.Stringer("cid", containers[i]), zap.Int("amount", len(storageGroupsIDs))) // filter expired storage groups storageGroups := ap.filterExpiredSG(containers[i], storageGroupsIDs, nodes, *nm) - log.Info("filter expired storage groups for audit", + log.Info(logs.AuditFilterExpiredStorageGroupsForAudit, zap.Stringer("cid", containers[i]), zap.Int("amount", len(storageGroups))) @@ -146,7 +147,7 @@ func (ap *Processor) findStorageGroups(cnr cid.ID, shuffled netmapcore.Nodes) [] err := clientcore.NodeInfoFromRawNetmapElement(&info, netmapcore.Node(shuffled[i])) if err != nil { - log.Warn("parse client node info", zap.String("error", err.Error())) + log.Warn(logs.AuditParseClientNodeInfo, zap.String("error", err.Error())) continue } @@ -162,7 +163,7 @@ func (ap *Processor) findStorageGroups(cnr cid.ID, shuffled netmapcore.Nodes) [] cancel() if err != nil { - log.Warn("error in storage group search", zap.String("error", err.Error())) + log.Warn(logs.AuditErrorInStorageGroupSearch, zap.String("error", err.Error())) continue } diff --git a/pkg/innerring/processors/audit/scheduler.go b/pkg/innerring/processors/audit/scheduler.go index e1a521bad..fbc5fa927 100644 --- a/pkg/innerring/processors/audit/scheduler.go +++ b/pkg/innerring/processors/audit/scheduler.go @@ -6,6 +6,7 @@ import ( "sort" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" ) @@ -20,7 +21,7 @@ func (ap *Processor) selectContainersToAudit(epoch uint64) ([]cid.ID, error) { // consider getting extra information about container complexity from // audit contract there - ap.log.Debug("container listing finished", + ap.log.Debug(logs.AuditContainerListingFinished, zap.Int("total amount", len(containers)), ) diff --git a/pkg/innerring/processors/balance/handlers.go b/pkg/innerring/processors/balance/handlers.go index 4c5a2ddc6..3360af916 100644 --- a/pkg/innerring/processors/balance/handlers.go +++ b/pkg/innerring/processors/balance/handlers.go @@ -3,6 +3,7 @@ package balance import ( "encoding/hex" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" balanceEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/balance" "go.uber.org/zap" @@ -10,7 +11,7 @@ import ( func (bp *Processor) handleLock(ev event.Event) { lock := ev.(balanceEvent.Lock) - bp.log.Info("notification", + bp.log.Info(logs.BalanceNotification, zap.String("type", "lock"), zap.String("value", hex.EncodeToString(lock.ID()))) @@ -19,7 +20,7 @@ func (bp *Processor) handleLock(ev event.Event) { err := bp.pool.Submit(func() { bp.processLock(&lock) }) if err != nil { // there system can be moved into controlled degradation stage - bp.log.Warn("balance worker pool drained", + bp.log.Warn(logs.BalanceBalanceWorkerPoolDrained, zap.Int("capacity", bp.pool.Cap())) } } diff --git a/pkg/innerring/processors/balance/process_assets.go b/pkg/innerring/processors/balance/process_assets.go index 754dda34a..3f86a3cb7 100644 --- a/pkg/innerring/processors/balance/process_assets.go +++ b/pkg/innerring/processors/balance/process_assets.go @@ -1,6 +1,7 @@ package balance import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" frostfsContract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" balanceEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/balance" "go.uber.org/zap" @@ -10,7 +11,7 @@ import ( // back to the withdraw issuer. func (bp *Processor) processLock(lock *balanceEvent.Lock) { if !bp.alphabetState.IsAlphabet() { - bp.log.Info("non alphabet mode, ignore balance lock") + bp.log.Info(logs.BalanceNonAlphabetModeIgnoreBalanceLock) return } @@ -24,6 +25,6 @@ func (bp *Processor) processLock(lock *balanceEvent.Lock) { err := bp.frostfsClient.Cheque(prm) if err != nil { - bp.log.Error("can't send lock asset tx", zap.Error(err)) + bp.log.Error(logs.BalanceCantSendLockAssetTx, zap.Error(err)) } } diff --git a/pkg/innerring/processors/balance/processor.go b/pkg/innerring/processors/balance/processor.go index 2527b7ec3..370d06f44 100644 --- a/pkg/innerring/processors/balance/processor.go +++ b/pkg/innerring/processors/balance/processor.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" balanceEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/balance" @@ -60,7 +61,7 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/balance: balance precision converter is not set") } - p.Log.Debug("balance worker pool", zap.Int("size", p.PoolSize)) + p.Log.Debug(logs.BalanceBalanceWorkerPool, zap.Int("size", p.PoolSize)) pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { diff --git a/pkg/innerring/processors/container/handlers.go b/pkg/innerring/processors/container/handlers.go index f9f8b5841..3d1946b4f 100644 --- a/pkg/innerring/processors/container/handlers.go +++ b/pkg/innerring/processors/container/handlers.go @@ -3,6 +3,7 @@ package container import ( "crypto/sha256" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "github.com/mr-tron/base58" @@ -13,7 +14,7 @@ func (cp *Processor) handlePut(ev event.Event) { put := ev.(putEvent) id := sha256.Sum256(put.Container()) - cp.log.Info("notification", + cp.log.Info(logs.ContainerNotification, zap.String("type", "container put"), zap.String("id", base58.Encode(id[:]))) @@ -22,14 +23,14 @@ func (cp *Processor) handlePut(ev event.Event) { err := cp.pool.Submit(func() { cp.processContainerPut(put) }) if err != nil { // there system can be moved into controlled degradation stage - cp.log.Warn("container processor worker pool drained", + cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, zap.Int("capacity", cp.pool.Cap())) } } func (cp *Processor) handleDelete(ev event.Event) { del := ev.(containerEvent.Delete) - cp.log.Info("notification", + cp.log.Info(logs.ContainerNotification, zap.String("type", "container delete"), zap.String("id", base58.Encode(del.ContainerID()))) @@ -38,7 +39,7 @@ func (cp *Processor) handleDelete(ev event.Event) { err := cp.pool.Submit(func() { cp.processContainerDelete(&del) }) if err != nil { // there system can be moved into controlled degradation stage - cp.log.Warn("container processor worker pool drained", + cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, zap.Int("capacity", cp.pool.Cap())) } } @@ -46,7 +47,7 @@ func (cp *Processor) handleDelete(ev event.Event) { func (cp *Processor) handleSetEACL(ev event.Event) { e := ev.(containerEvent.SetEACL) - cp.log.Info("notification", + cp.log.Info(logs.ContainerNotification, zap.String("type", "set EACL"), ) @@ -57,7 +58,7 @@ func (cp *Processor) handleSetEACL(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - cp.log.Warn("container processor worker pool drained", + cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, zap.Int("capacity", cp.pool.Cap())) } } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 8b244aa5d..5ebe58375 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -3,6 +3,7 @@ package container import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" morphsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/subnet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -35,7 +36,7 @@ type putContainerContext struct { // and sending approve tx back to the morph. func (cp *Processor) processContainerPut(put putEvent) { if !cp.alphabetState.IsAlphabet() { - cp.log.Info("non alphabet mode, ignore container put") + cp.log.Info(logs.ContainerNonAlphabetModeIgnoreContainerPut) return } @@ -45,7 +46,7 @@ func (cp *Processor) processContainerPut(put putEvent) { err := cp.checkPutContainer(ctx) if err != nil { - cp.log.Error("put container check failed", + cp.log.Error(logs.ContainerPutContainerCheckFailed, zap.String("error", err.Error()), ) @@ -119,7 +120,7 @@ func (cp *Processor) approvePutContainer(ctx *putContainerContext) { err = cp.cnrClient.Put(prm) } if err != nil { - cp.log.Error("could not approve put container", + cp.log.Error(logs.ContainerCouldNotApprovePutContainer, zap.String("error", err.Error()), ) } @@ -129,13 +130,13 @@ func (cp *Processor) approvePutContainer(ctx *putContainerContext) { // and sending approve tx back to morph. func (cp *Processor) processContainerDelete(e *containerEvent.Delete) { if !cp.alphabetState.IsAlphabet() { - cp.log.Info("non alphabet mode, ignore container delete") + cp.log.Info(logs.ContainerNonAlphabetModeIgnoreContainerDelete) return } err := cp.checkDeleteContainer(e) if err != nil { - cp.log.Error("delete container check failed", + cp.log.Error(logs.ContainerDeleteContainerCheckFailed, zap.String("error", err.Error()), ) @@ -194,7 +195,7 @@ func (cp *Processor) approveDeleteContainer(e *containerEvent.Delete) { err = cp.cnrClient.Delete(prm) } if err != nil { - cp.log.Error("could not approve delete container", + cp.log.Error(logs.ContainerCouldNotApproveDeleteContainer, zap.String("error", err.Error()), ) } diff --git a/pkg/innerring/processors/container/process_eacl.go b/pkg/innerring/processors/container/process_eacl.go index e8bbb5db6..fce75c678 100644 --- a/pkg/innerring/processors/container/process_eacl.go +++ b/pkg/innerring/processors/container/process_eacl.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" @@ -13,13 +14,13 @@ import ( func (cp *Processor) processSetEACL(e container.SetEACL) { if !cp.alphabetState.IsAlphabet() { - cp.log.Info("non alphabet mode, ignore set EACL") + cp.log.Info(logs.ContainerNonAlphabetModeIgnoreSetEACL) return } err := cp.checkSetEACL(e) if err != nil { - cp.log.Error("set EACL check failed", + cp.log.Error(logs.ContainerSetEACLCheckFailed, zap.String("error", err.Error()), ) @@ -91,7 +92,7 @@ func (cp *Processor) approveSetEACL(e container.SetEACL) { err = cp.cnrClient.PutEACL(prm) } if err != nil { - cp.log.Error("could not approve set EACL", + cp.log.Error(logs.ContainerCouldNotApproveSetEACL, zap.String("error", err.Error()), ) } diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index ae0d28729..123ba77b8 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" morphsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/subnet" @@ -88,7 +89,7 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/container: subnet client is not set") } - p.Log.Debug("container worker pool", zap.Int("size", p.PoolSize)) + p.Log.Debug(logs.ContainerContainerWorkerPool, zap.Int("size", p.PoolSize)) pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index bc0dbec7f..4eff15abe 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -3,6 +3,7 @@ package frostfs import ( "encoding/hex" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -11,7 +12,7 @@ import ( func (np *Processor) handleDeposit(ev event.Event) { deposit := ev.(frostfsEvent.Deposit) - np.log.Info("notification", + np.log.Info(logs.FrostFSNotification, zap.String("type", "deposit"), zap.String("id", hex.EncodeToString(slice.CopyReverse(deposit.ID())))) @@ -20,14 +21,14 @@ func (np *Processor) handleDeposit(ev event.Event) { err := np.pool.Submit(func() { np.processDeposit(&deposit) }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("frostfs processor worker pool drained", + np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleWithdraw(ev event.Event) { withdraw := ev.(frostfsEvent.Withdraw) - np.log.Info("notification", + np.log.Info(logs.FrostFSNotification, zap.String("type", "withdraw"), zap.String("id", hex.EncodeToString(slice.CopyReverse(withdraw.ID())))) @@ -36,14 +37,14 @@ func (np *Processor) handleWithdraw(ev event.Event) { err := np.pool.Submit(func() { np.processWithdraw(&withdraw) }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("frostfs processor worker pool drained", + np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleCheque(ev event.Event) { cheque := ev.(frostfsEvent.Cheque) - np.log.Info("notification", + np.log.Info(logs.FrostFSNotification, zap.String("type", "cheque"), zap.String("id", hex.EncodeToString(cheque.ID()))) @@ -52,14 +53,14 @@ func (np *Processor) handleCheque(ev event.Event) { err := np.pool.Submit(func() { np.processCheque(&cheque) }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("frostfs processor worker pool drained", + np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleConfig(ev event.Event) { cfg := ev.(frostfsEvent.Config) - np.log.Info("notification", + np.log.Info(logs.FrostFSNotification, zap.String("type", "set config"), zap.String("key", hex.EncodeToString(cfg.Key())), zap.String("value", hex.EncodeToString(cfg.Value()))) @@ -69,14 +70,14 @@ func (np *Processor) handleConfig(ev event.Event) { err := np.pool.Submit(func() { np.processConfig(&cfg) }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("frostfs processor worker pool drained", + np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleBind(ev event.Event) { e := ev.(frostfsEvent.Bind) - np.log.Info("notification", + np.log.Info(logs.FrostFSNotification, zap.String("type", "bind"), ) @@ -85,14 +86,14 @@ func (np *Processor) handleBind(ev event.Event) { err := np.pool.Submit(func() { np.processBind(e) }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("frostfs processor worker pool drained", + np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleUnbind(ev event.Event) { e := ev.(frostfsEvent.Unbind) - np.log.Info("notification", + np.log.Info(logs.FrostFSNotification, zap.String("type", "unbind"), ) @@ -101,7 +102,7 @@ func (np *Processor) handleUnbind(ev event.Event) { err := np.pool.Submit(func() { np.processBind(e) }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("frostfs processor worker pool drained", + np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } diff --git a/pkg/innerring/processors/frostfs/process_assets.go b/pkg/innerring/processors/frostfs/process_assets.go index b28efaa33..e066975f7 100644 --- a/pkg/innerring/processors/frostfs/process_assets.go +++ b/pkg/innerring/processors/frostfs/process_assets.go @@ -1,6 +1,7 @@ package frostfs import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "github.com/nspcc-dev/neo-go/pkg/util" @@ -16,7 +17,7 @@ const ( // gas in the sidechain. func (np *Processor) processDeposit(deposit *frostfsEvent.Deposit) { if !np.alphabetState.IsAlphabet() { - np.log.Info("non alphabet mode, ignore deposit") + np.log.Info(logs.FrostFSNonAlphabetModeIgnoreDeposit) return } @@ -29,7 +30,7 @@ func (np *Processor) processDeposit(deposit *frostfsEvent.Deposit) { // send transferX to a balance contract err := np.balanceClient.Mint(prm) if err != nil { - np.log.Error("can't transfer assets to balance contract", zap.Error(err)) + np.log.Error(logs.FrostFSCantTransferAssetsToBalanceContract, zap.Error(err)) } curEpoch := np.epochState.EpochCounter() @@ -43,7 +44,7 @@ func (np *Processor) processDeposit(deposit *frostfsEvent.Deposit) { val, ok := np.mintEmitCache.Get(receiver.String()) if ok && val+np.mintEmitThreshold >= curEpoch { - np.log.Warn("double mint emission declined", + np.log.Warn(logs.FrostFSDoubleMintEmissionDeclined, zap.String("receiver", receiver.String()), zap.Uint64("last_emission", val), zap.Uint64("current_epoch", curEpoch)) @@ -55,12 +56,12 @@ func (np *Processor) processDeposit(deposit *frostfsEvent.Deposit) { // before gas transfer check if the balance is greater than the threshold balance, err := np.morphClient.GasBalance() if err != nil { - np.log.Error("can't get gas balance of the node", zap.Error(err)) + np.log.Error(logs.FrostFSCantGetGasBalanceOfTheNode, zap.Error(err)) return } if balance < np.gasBalanceThreshold { - np.log.Warn("gas balance threshold has been reached", + np.log.Warn(logs.FrostFSGasBalanceThresholdHasBeenReached, zap.Int64("balance", balance), zap.Int64("threshold", np.gasBalanceThreshold)) @@ -69,7 +70,7 @@ func (np *Processor) processDeposit(deposit *frostfsEvent.Deposit) { err = np.morphClient.TransferGas(receiver, np.mintEmitValue) if err != nil { - np.log.Error("can't transfer native gas to receiver", + np.log.Error(logs.FrostFSCantTransferNativeGasToReceiver, zap.String("error", err.Error())) return @@ -81,14 +82,14 @@ func (np *Processor) processDeposit(deposit *frostfsEvent.Deposit) { // Process withdraw event by locking assets in the balance account. func (np *Processor) processWithdraw(withdraw *frostfsEvent.Withdraw) { if !np.alphabetState.IsAlphabet() { - np.log.Info("non alphabet mode, ignore withdraw") + np.log.Info(logs.FrostFSNonAlphabetModeIgnoreWithdraw) return } // create lock account lock, err := util.Uint160DecodeBytesBE(withdraw.ID()[:util.Uint160Size]) if err != nil { - np.log.Error("can't create lock account", zap.Error(err)) + np.log.Error(logs.FrostFSCantCreateLockAccount, zap.Error(err)) return } @@ -104,7 +105,7 @@ func (np *Processor) processWithdraw(withdraw *frostfsEvent.Withdraw) { err = np.balanceClient.Lock(prm) if err != nil { - np.log.Error("can't lock assets for withdraw", zap.Error(err)) + np.log.Error(logs.FrostFSCantLockAssetsForWithdraw, zap.Error(err)) } } @@ -112,7 +113,7 @@ func (np *Processor) processWithdraw(withdraw *frostfsEvent.Withdraw) { // the reserve account. func (np *Processor) processCheque(cheque *frostfsEvent.Cheque) { if !np.alphabetState.IsAlphabet() { - np.log.Info("non alphabet mode, ignore cheque") + np.log.Info(logs.FrostFSNonAlphabetModeIgnoreCheque) return } @@ -124,6 +125,6 @@ func (np *Processor) processCheque(cheque *frostfsEvent.Cheque) { err := np.balanceClient.Burn(prm) if err != nil { - np.log.Error("can't transfer assets to fed contract", zap.Error(err)) + np.log.Error(logs.FrostFSCantTransferAssetsToFedContract, zap.Error(err)) } } diff --git a/pkg/innerring/processors/frostfs/process_bind.go b/pkg/innerring/processors/frostfs/process_bind.go index 0abce5827..c5f8a930e 100644 --- a/pkg/innerring/processors/frostfs/process_bind.go +++ b/pkg/innerring/processors/frostfs/process_bind.go @@ -4,6 +4,7 @@ import ( "crypto/elliptic" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -20,7 +21,7 @@ type bindCommon interface { func (np *Processor) processBind(e bindCommon) { if !np.alphabetState.IsAlphabet() { - np.log.Info("non alphabet mode, ignore bind") + np.log.Info(logs.FrostFSNonAlphabetModeIgnoreBind) return } @@ -32,7 +33,7 @@ func (np *Processor) processBind(e bindCommon) { err := np.checkBindCommon(c) if err != nil { - np.log.Error("invalid manage key event", + np.log.Error(logs.FrostFSInvalidManageKeyEvent, zap.Bool("bind", c.bind), zap.String("error", err.Error()), ) @@ -77,7 +78,7 @@ func (np *Processor) approveBindCommon(e *bindCommonContext) { u160, err := util.Uint160DecodeBytesBE(scriptHash) if err != nil { - np.log.Error("could not decode script hash from bytes", + np.log.Error(logs.FrostFSCouldNotDecodeScriptHashFromBytes, zap.String("error", err.Error()), ) diff --git a/pkg/innerring/processors/frostfs/process_config.go b/pkg/innerring/processors/frostfs/process_config.go index ecc90332f..471edb9b7 100644 --- a/pkg/innerring/processors/frostfs/process_config.go +++ b/pkg/innerring/processors/frostfs/process_config.go @@ -1,6 +1,7 @@ package frostfs import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "go.uber.org/zap" @@ -10,7 +11,7 @@ import ( // the sidechain. func (np *Processor) processConfig(config *frostfsEvent.Config) { if !np.alphabetState.IsAlphabet() { - np.log.Info("non alphabet mode, ignore config") + np.log.Info(logs.FrostFSNonAlphabetModeIgnoreConfig) return } @@ -23,6 +24,6 @@ func (np *Processor) processConfig(config *frostfsEvent.Config) { err := np.netmapClient.SetConfig(prm) if err != nil { - np.log.Error("can't relay set config event", zap.Error(err)) + np.log.Error(logs.FrostFSCantRelaySetConfigEvent, zap.Error(err)) } } diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index e9504cdb4..4d5bdee78 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -5,6 +5,7 @@ import ( "fmt" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" @@ -98,7 +99,7 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/frostfs: balance precision converter is not set") } - p.Log.Debug("frostfs worker pool", zap.Int("size", p.PoolSize)) + p.Log.Debug(logs.FrostFSFrostfsWorkerPool, zap.Int("size", p.PoolSize)) pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { diff --git a/pkg/innerring/processors/governance/handlers.go b/pkg/innerring/processors/governance/handlers.go index bfa88d3f0..727acc21a 100644 --- a/pkg/innerring/processors/governance/handlers.go +++ b/pkg/innerring/processors/governance/handlers.go @@ -1,6 +1,7 @@ package governance import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/rolemanagement" "github.com/nspcc-dev/neo-go/pkg/core/native" @@ -30,14 +31,14 @@ func (gp *Processor) HandleAlphabetSync(e event.Event) { return } - gp.log.Info("new event", zap.String("type", typ)) + gp.log.Info(logs.GovernanceNewEvent, zap.String("type", typ)) // send event to the worker pool err := gp.pool.Submit(func() { gp.processAlphabetSync(hash) }) if err != nil { // there system can be moved into controlled degradation stage - gp.log.Warn("governance worker pool drained", + gp.log.Warn(logs.GovernanceGovernanceWorkerPoolDrained, zap.Int("capacity", gp.pool.Cap())) } } diff --git a/pkg/innerring/processors/governance/process_update.go b/pkg/innerring/processors/governance/process_update.go index 3504e7a53..629d8741e 100644 --- a/pkg/innerring/processors/governance/process_update.go +++ b/pkg/innerring/processors/governance/process_update.go @@ -6,6 +6,7 @@ import ( "sort" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -20,37 +21,37 @@ const ( func (gp *Processor) processAlphabetSync(txHash util.Uint256) { if !gp.alphabetState.IsAlphabet() { - gp.log.Info("non alphabet mode, ignore alphabet sync") + gp.log.Info(logs.GovernanceNonAlphabetModeIgnoreAlphabetSync) return } mainnetAlphabet, err := gp.mainnetClient.NeoFSAlphabetList() if err != nil { - gp.log.Error("can't fetch alphabet list from main net", + gp.log.Error(logs.GovernanceCantFetchAlphabetListFromMainNet, zap.String("error", err.Error())) return } sidechainAlphabet, err := gp.morphClient.Committee() if err != nil { - gp.log.Error("can't fetch alphabet list from side chain", + gp.log.Error(logs.GovernanceCantFetchAlphabetListFromSideChain, zap.String("error", err.Error())) return } newAlphabet, err := newAlphabetList(sidechainAlphabet, mainnetAlphabet) if err != nil { - gp.log.Error("can't merge alphabet lists from main net and side chain", + gp.log.Error(logs.GovernanceCantMergeAlphabetListsFromMainNetAndSideChain, zap.String("error", err.Error())) return } if newAlphabet == nil { - gp.log.Info("no governance update, alphabet list has not been changed") + gp.log.Info(logs.GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged) return } - gp.log.Info("alphabet list has been changed, starting update", + gp.log.Info(logs.GovernanceAlphabetListHasBeenChangedStartingUpdate, zap.String("side_chain_alphabet", prettyKeys(sidechainAlphabet)), zap.String("new_alphabet", prettyKeys(newAlphabet)), ) @@ -63,7 +64,7 @@ func (gp *Processor) processAlphabetSync(txHash util.Uint256) { // 1. Vote to sidechain committee via alphabet contracts. err = gp.voter.VoteForSidechainValidator(votePrm) if err != nil { - gp.log.Error("can't vote for side chain committee", + gp.log.Error(logs.GovernanceCantVoteForSideChainCommittee, zap.String("error", err.Error())) } @@ -76,7 +77,7 @@ func (gp *Processor) processAlphabetSync(txHash util.Uint256) { // 4. Update FrostFS contract in the mainnet. gp.updateFrostFSContractInMainnet(newAlphabet) - gp.log.Info("finished alphabet list update") + gp.log.Info(logs.GovernanceFinishedAlphabetListUpdate) } func prettyKeys(keys keys.PublicKeys) string { @@ -94,21 +95,21 @@ func prettyKeys(keys keys.PublicKeys) string { func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(sidechainAlphabet, newAlphabet keys.PublicKeys, txHash util.Uint256) { innerRing, err := gp.irFetcher.InnerRingKeys() if err != nil { - gp.log.Error("can't fetch inner ring list from side chain", + gp.log.Error(logs.GovernanceCantFetchInnerRingListFromSideChain, zap.String("error", err.Error())) return } newInnerRing, err := updateInnerRing(innerRing, sidechainAlphabet, newAlphabet) if err != nil { - gp.log.Error("can't create new inner ring list with new alphabet keys", + gp.log.Error(logs.GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys, zap.String("error", err.Error())) return } sort.Sort(newInnerRing) - gp.log.Info("update of the inner ring list", + gp.log.Info(logs.GovernanceUpdateOfTheInnerRingList, zap.String("before", prettyKeys(innerRing)), zap.String("after", prettyKeys(newInnerRing)), ) @@ -130,7 +131,7 @@ func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(sidechainAlphabet, newAl } if err != nil { - gp.log.Error("can't update inner ring list with new alphabet keys", + gp.log.Error(logs.GovernanceCantUpdateInnerRingListWithNewAlphabetKeys, zap.String("error", err.Error())) } } @@ -147,7 +148,7 @@ func (gp *Processor) updateNotaryRoleInSidechain(newAlphabet keys.PublicKeys, tx err := gp.morphClient.UpdateNotaryList(updPrm) if err != nil { - gp.log.Error("can't update list of notary nodes in side chain", + gp.log.Error(logs.GovernanceCantUpdateListOfNotaryNodesInSideChain, zap.String("error", err.Error())) } } @@ -167,7 +168,7 @@ func (gp *Processor) updateFrostFSContractInMainnet(newAlphabet keys.PublicKeys) err := gp.frostfsClient.AlphabetUpdate(prm) if err != nil { - gp.log.Error("can't update list of alphabet nodes in frostfs contract", + gp.log.Error(logs.GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract, zap.String("error", err.Error())) } } diff --git a/pkg/innerring/processors/netmap/handlers.go b/pkg/innerring/processors/netmap/handlers.go index 54e4ea3ab..76b27c891 100644 --- a/pkg/innerring/processors/netmap/handlers.go +++ b/pkg/innerring/processors/netmap/handlers.go @@ -3,6 +3,7 @@ package netmap import ( "encoding/hex" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" @@ -12,21 +13,21 @@ import ( func (np *Processor) HandleNewEpochTick(ev event.Event) { _ = ev.(timerEvent.NewEpochTick) - np.log.Info("tick", zap.String("type", "epoch")) + np.log.Info(logs.NetmapTick, zap.String("type", "epoch")) // send an event to the worker pool err := np.pool.Submit(func() { np.processNewEpochTick() }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("netmap worker pool drained", + np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleNewEpoch(ev event.Event) { epochEvent := ev.(netmapEvent.NewEpoch) - np.log.Info("notification", + np.log.Info(logs.NetmapNotification, zap.String("type", "new epoch"), zap.Uint64("value", epochEvent.EpochNumber())) @@ -37,7 +38,7 @@ func (np *Processor) handleNewEpoch(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("netmap worker pool drained", + np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } @@ -45,7 +46,7 @@ func (np *Processor) handleNewEpoch(ev event.Event) { func (np *Processor) handleAddPeer(ev event.Event) { newPeer := ev.(netmapEvent.AddPeer) - np.log.Info("notification", + np.log.Info(logs.NetmapNotification, zap.String("type", "add peer"), ) @@ -56,14 +57,14 @@ func (np *Processor) handleAddPeer(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("netmap worker pool drained", + np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleUpdateState(ev event.Event) { updPeer := ev.(netmapEvent.UpdatePeer) - np.log.Info("notification", + np.log.Info(logs.NetmapNotification, zap.String("type", "update peer state"), zap.String("key", hex.EncodeToString(updPeer.PublicKey().Bytes()))) @@ -74,21 +75,21 @@ func (np *Processor) handleUpdateState(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("netmap worker pool drained", + np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleCleanupTick(ev event.Event) { if !np.netmapSnapshot.enabled { - np.log.Debug("netmap clean up routine is disabled") + np.log.Debug(logs.NetmapNetmapCleanUpRoutineIsDisabled518) return } cleanup := ev.(netmapCleanupTick) - np.log.Info("tick", zap.String("type", "netmap cleaner")) + np.log.Info(logs.NetmapTick, zap.String("type", "netmap cleaner")) // send event to the worker pool err := np.pool.Submit(func() { @@ -96,7 +97,7 @@ func (np *Processor) handleCleanupTick(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("netmap worker pool drained", + np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } @@ -104,7 +105,7 @@ func (np *Processor) handleCleanupTick(ev event.Event) { func (np *Processor) handleRemoveNode(ev event.Event) { removeNode := ev.(subnetevents.RemoveNode) - np.log.Info("notification", + np.log.Info(logs.NetmapNotification, zap.String("type", "remove node from subnet"), zap.String("subnetID", hex.EncodeToString(removeNode.SubnetworkID())), zap.String("key", hex.EncodeToString(removeNode.Node())), @@ -115,7 +116,7 @@ func (np *Processor) handleRemoveNode(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn("netmap worker pool drained", + np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } diff --git a/pkg/innerring/processors/netmap/process_cleanup.go b/pkg/innerring/processors/netmap/process_cleanup.go index e4425ef17..d50c69c78 100644 --- a/pkg/innerring/processors/netmap/process_cleanup.go +++ b/pkg/innerring/processors/netmap/process_cleanup.go @@ -2,6 +2,7 @@ package netmap import ( v2netmap "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.uber.org/zap" @@ -9,7 +10,7 @@ import ( func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { if !np.alphabetState.IsAlphabet() { - np.log.Info("non alphabet mode, ignore new netmap cleanup tick") + np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick) return } @@ -17,13 +18,13 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { err := np.netmapSnapshot.forEachRemoveCandidate(ev.epoch, func(s string) error { key, err := keys.NewPublicKeyFromString(s) if err != nil { - np.log.Warn("can't decode public key of netmap node", + np.log.Warn(logs.NetmapCantDecodePublicKeyOfNetmapNode, zap.String("key", s)) return nil } - np.log.Info("vote to remove node from netmap", zap.String("key", s)) + np.log.Info(logs.NetmapVoteToRemoveNodeFromNetmap, zap.String("key", s)) // In notary environments we call UpdateStateIR method instead of UpdateState. // It differs from UpdateState only by name, so we can do this in the same form. @@ -48,13 +49,13 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { ) } if err != nil { - np.log.Error("can't invoke netmap.UpdateState", zap.Error(err)) + np.log.Error(logs.NetmapCantInvokeNetmapUpdateState, zap.Error(err)) } return nil }) if err != nil { - np.log.Warn("can't iterate on netmap cleaner cache", + np.log.Warn(logs.NetmapCantIterateOnNetmapCleanerCache, zap.String("error", err.Error())) } } diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index ffcddc497..ebf128f82 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -1,6 +1,7 @@ package netmap import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement" @@ -16,7 +17,7 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { epochDuration, err := np.netmapClient.EpochDuration() if err != nil { - np.log.Warn("can't get epoch duration", + np.log.Warn(logs.NetmapCantGetEpochDuration, zap.String("error", err.Error())) } else { np.epochState.SetEpochDuration(epochDuration) @@ -26,20 +27,20 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { h, err := np.netmapClient.Morph().TxHeight(ev.TxHash()) if err != nil { - np.log.Warn("can't get transaction height", + np.log.Warn(logs.NetmapCantGetTransactionHeight, zap.String("hash", ev.TxHash().StringLE()), zap.String("error", err.Error())) } if err := np.epochTimer.ResetEpochTimer(h); err != nil { - np.log.Warn("can't reset epoch timer", + np.log.Warn(logs.NetmapCantResetEpochTimer, zap.String("error", err.Error())) } // get new netmap snapshot networkMap, err := np.netmapClient.NetMap() if err != nil { - np.log.Warn("can't get netmap snapshot to perform cleanup", + np.log.Warn(logs.NetmapCantGetNetmapSnapshotToPerformCleanup, zap.String("error", err.Error())) return @@ -54,7 +55,7 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { err = np.containerWrp.StartEstimation(prm) if err != nil { - np.log.Warn("can't start container size estimation", + np.log.Warn(logs.NetmapCantStartContainerSizeEstimation, zap.Uint64("epoch", epoch), zap.String("error", err.Error())) } @@ -71,15 +72,15 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { // Process new epoch tick by invoking new epoch method in network map contract. func (np *Processor) processNewEpochTick() { if !np.alphabetState.IsAlphabet() { - np.log.Info("non alphabet mode, ignore new epoch tick") + np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewEpochTick) return } nextEpoch := np.epochState.EpochCounter() + 1 - np.log.Debug("next epoch", zap.Uint64("value", nextEpoch)) + np.log.Debug(logs.NetmapNextEpoch, zap.Uint64("value", nextEpoch)) err := np.netmapClient.NewEpoch(nextEpoch) if err != nil { - np.log.Error("can't invoke netmap.NewEpoch", zap.Error(err)) + np.log.Error(logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) } } diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index 3734bae01..ffaad3b4e 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" subnetEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/subnet" @@ -16,7 +17,7 @@ import ( // local epoch timer. func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { if !np.alphabetState.IsAlphabet() { - np.log.Info("non alphabet mode, ignore new peer notification") + np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewPeerNotification) return } @@ -25,7 +26,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { tx := originalRequest.MainTransaction ok, err := np.netmapClient.Morph().IsValidScript(tx.Script, tx.Signers) if err != nil || !ok { - np.log.Warn("non-halt notary transaction", + np.log.Warn(logs.NetmapNonhaltNotaryTransaction, zap.String("method", "netmap.AddPeer"), zap.String("hash", tx.Hash().StringLE()), zap.Error(err)) @@ -37,14 +38,14 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { var nodeInfo netmap.NodeInfo if err := nodeInfo.Unmarshal(ev.Node()); err != nil { // it will be nice to have tx id at event structure to log it - np.log.Warn("can't parse network map candidate") + np.log.Warn(logs.NetmapCantParseNetworkMapCandidate) return } // validate and update node info err := np.nodeValidator.VerifyAndUpdate(&nodeInfo) if err != nil { - np.log.Warn("could not verify and update information about network map candidate", + np.log.Warn(logs.NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate, zap.String("error", err.Error()), ) @@ -62,7 +63,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { updated := np.netmapSnapshot.touch(keyString, np.epochState.EpochCounter(), nodeInfoBinary) if updated { - np.log.Info("approving network map candidate", + np.log.Info(logs.NetmapApprovingNetworkMapCandidate, zap.String("key", keyString)) prm := netmapclient.AddPeerPrm{} @@ -89,7 +90,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { } if err != nil { - np.log.Error("can't invoke netmap.AddPeer", zap.Error(err)) + np.log.Error(logs.NetmapCantInvokeNetmapAddPeer, zap.Error(err)) } } } @@ -97,7 +98,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { // Process update peer notification by sending approval tx to the smart contract. func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { if !np.alphabetState.IsAlphabet() { - np.log.Info("non alphabet mode, ignore update peer notification") + np.log.Info(logs.NetmapNonAlphabetModeIgnoreUpdatePeerNotification) return } @@ -110,7 +111,7 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { if ev.Maintenance() { err = np.nodeStateSettings.MaintenanceModeAllowed() if err != nil { - np.log.Info("prevent switching node to maintenance state", + np.log.Info(logs.NetmapPreventSwitchingNodeToMaintenanceState, zap.Error(err), ) @@ -135,19 +136,19 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { err = np.netmapClient.UpdatePeerState(prm) } if err != nil { - np.log.Error("can't invoke netmap.UpdatePeer", zap.Error(err)) + np.log.Error(logs.NetmapCantInvokeNetmapUpdatePeer, zap.Error(err)) } } func (np *Processor) processRemoveSubnetNode(ev subnetEvent.RemoveNode) { if !np.alphabetState.IsAlphabet() { - np.log.Info("non alphabet mode, ignore remove node from subnet notification") + np.log.Info(logs.NetmapNonAlphabetModeIgnoreRemoveNodeFromSubnetNotification) return } candidates, err := np.netmapClient.GetCandidates() if err != nil { - np.log.Warn("could not get network map candidates", + np.log.Warn(logs.NetmapCouldNotGetNetworkMapCandidates, zap.Error(err), ) return @@ -158,14 +159,14 @@ func (np *Processor) processRemoveSubnetNode(ev subnetEvent.RemoveNode) { err = subnetToRemoveFrom.Unmarshal(rawSubnet) if err != nil { - np.log.Warn("could not unmarshal subnet id", + np.log.Warn(logs.NetmapCouldNotUnmarshalSubnetId, zap.Error(err), ) return } if subnetid.IsZero(subnetToRemoveFrom) { - np.log.Warn("got zero subnet in remove node notification") + np.log.Warn(logs.NetmapGotZeroSubnetInRemoveNodeNotification) return } @@ -182,8 +183,8 @@ func (np *Processor) processRemoveSubnetNode(ev subnetEvent.RemoveNode) { return nil }) if err != nil { - np.log.Warn("could not iterate over subnetworks of the node", zap.Error(err)) - np.log.Info("vote to remove node from netmap", zap.String("key", hex.EncodeToString(ev.Node()))) + np.log.Warn(logs.NetmapCouldNotIterateOverSubnetworksOfTheNode, zap.Error(err)) + np.log.Info(logs.NetmapVoteToRemoveNodeFromNetmap, zap.String("key", hex.EncodeToString(ev.Node()))) prm := netmapclient.UpdatePeerPrm{} prm.SetKey(ev.Node()) @@ -191,7 +192,7 @@ func (np *Processor) processRemoveSubnetNode(ev subnetEvent.RemoveNode) { err = np.netmapClient.UpdatePeerState(prm) if err != nil { - np.log.Error("could not invoke netmap.UpdateState", zap.Error(err)) + np.log.Error(logs.NetmapCouldNotInvokeNetmapUpdateState, zap.Error(err)) return } } else { @@ -201,7 +202,7 @@ func (np *Processor) processRemoveSubnetNode(ev subnetEvent.RemoveNode) { err = np.netmapClient.AddPeer(prm) if err != nil { - np.log.Error("could not invoke netmap.AddPeer", zap.Error(err)) + np.log.Error(logs.NetmapCouldNotInvokeNetmapAddPeer, zap.Error(err)) return } } diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index de145d48c..85a123ef3 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -142,7 +143,7 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/netmap: node state settings is not set") } - p.Log.Debug("netmap worker pool", zap.Int("size", p.PoolSize)) + p.Log.Debug(logs.NetmapNetmapWorkerPool, zap.Int("size", p.PoolSize)) pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { diff --git a/pkg/innerring/processors/reputation/handlers.go b/pkg/innerring/processors/reputation/handlers.go index 36c9579e5..30e3e9503 100644 --- a/pkg/innerring/processors/reputation/handlers.go +++ b/pkg/innerring/processors/reputation/handlers.go @@ -3,6 +3,7 @@ package reputation import ( "encoding/hex" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" reputationEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/reputation" "go.uber.org/zap" @@ -13,7 +14,7 @@ func (rp *Processor) handlePutReputation(ev event.Event) { peerID := put.PeerID() // FIXME: #1147 do not use `ToV2` method outside frostfs-api-go library - rp.log.Info("notification", + rp.log.Info(logs.ReputationNotification, zap.String("type", "reputation put"), zap.String("peer_id", hex.EncodeToString(peerID.PublicKey()))) @@ -22,7 +23,7 @@ func (rp *Processor) handlePutReputation(ev event.Event) { err := rp.pool.Submit(func() { rp.processPut(&put) }) if err != nil { // there system can be moved into controlled degradation stage - rp.log.Warn("reputation worker pool drained", + rp.log.Warn(logs.ReputationReputationWorkerPoolDrained, zap.Int("capacity", rp.pool.Cap())) } } diff --git a/pkg/innerring/processors/reputation/process_put.go b/pkg/innerring/processors/reputation/process_put.go index 31e93763b..f8814dd06 100644 --- a/pkg/innerring/processors/reputation/process_put.go +++ b/pkg/innerring/processors/reputation/process_put.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" repClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/reputation" reputationEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/reputation" apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" @@ -16,7 +17,7 @@ var errWrongManager = errors.New("got manager that is incorrect for peer") func (rp *Processor) processPut(e *reputationEvent.Put) { if !rp.alphabetState.IsAlphabet() { - rp.log.Info("non alphabet mode, ignore reputation put notification") + rp.log.Info(logs.ReputationNonAlphabetModeIgnoreReputationPutNotification) return } @@ -27,7 +28,7 @@ func (rp *Processor) processPut(e *reputationEvent.Put) { // check if epoch is valid currentEpoch := rp.epochState.EpochCounter() if epoch >= currentEpoch { - rp.log.Info("ignore reputation value", + rp.log.Info(logs.ReputationIgnoreReputationValue, zap.String("reason", "invalid epoch number"), zap.Uint64("trust_epoch", epoch), zap.Uint64("local_epoch", currentEpoch)) @@ -37,7 +38,7 @@ func (rp *Processor) processPut(e *reputationEvent.Put) { // check signature if !value.VerifySignature() { - rp.log.Info("ignore reputation value", + rp.log.Info(logs.ReputationIgnoreReputationValue, zap.String("reason", "invalid signature"), ) @@ -46,7 +47,7 @@ func (rp *Processor) processPut(e *reputationEvent.Put) { // check if manager is correct if err := rp.checkManagers(epoch, value.Manager(), id); err != nil { - rp.log.Info("ignore reputation value", + rp.log.Info(logs.ReputationIgnoreReputationValue, zap.String("reason", "wrong manager"), zap.String("error", err.Error())) @@ -91,7 +92,7 @@ func (rp *Processor) approvePutReputation(e *reputationEvent.Put) { } if err != nil { // FIXME: #1147 do not use `ToV2` method outside frostfs-api-go library - rp.log.Warn("can't send approval tx for reputation value", + rp.log.Warn(logs.ReputationCantSendApprovalTxForReputationValue, zap.String("peer_id", hex.EncodeToString(id.PublicKey())), zap.String("error", err.Error())) } diff --git a/pkg/innerring/processors/reputation/processor.go b/pkg/innerring/processors/reputation/processor.go index 990358257..a248fa75f 100644 --- a/pkg/innerring/processors/reputation/processor.go +++ b/pkg/innerring/processors/reputation/processor.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" repClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/reputation" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" reputationEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/reputation" @@ -71,7 +72,7 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/reputation: manager builder is not set") } - p.Log.Debug("reputation worker pool", zap.Int("size", p.PoolSize)) + p.Log.Debug(logs.ReputationReputationWorkerPool, zap.Int("size", p.PoolSize)) pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { diff --git a/pkg/innerring/processors/settlement/audit/calculate.go b/pkg/innerring/processors/settlement/audit/calculate.go index d819865d8..75b8c56a4 100644 --- a/pkg/innerring/processors/settlement/audit/calculate.go +++ b/pkg/innerring/processors/settlement/audit/calculate.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "math/big" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/audit" @@ -58,32 +59,32 @@ func (c *Calculator) Calculate(p *CalculatePrm) { )} if p.Epoch == 0 { - log.Info("settlements are ignored for zero epoch") + log.Info(logs.AuditSettlementsAreIgnoredForZeroEpoch) return } - log.Info("calculate audit settlements") + log.Info(logs.AuditCalculateAuditSettlements) - log.Debug("getting results for the previous epoch") + log.Debug(logs.AuditGettingResultsForThePreviousEpoch) prevEpoch := p.Epoch - 1 auditResults, err := c.prm.ResultStorage.AuditResultsForEpoch(prevEpoch) if err != nil { - log.Error("could not collect audit results") + log.Error(logs.AuditCouldNotCollectAuditResults) return } else if len(auditResults) == 0 { - log.Debug("no audit results in previous epoch") + log.Debug(logs.AuditNoAuditResultsInPreviousEpoch) return } auditFee, err := c.prm.AuditFeeFetcher.AuditFee() if err != nil { - log.Warn("can't fetch audit fee from network config", + log.Warn(logs.AuditCantFetchAuditFeeFromNetworkConfig, zap.String("error", err.Error())) auditFee = 0 } - log.Debug("processing audit results", + log.Debug(logs.AuditProcessingAuditResults, zap.Int("number", len(auditResults)), ) @@ -98,7 +99,7 @@ func (c *Calculator) Calculate(p *CalculatePrm) { }) } - log.Debug("processing transfers") + log.Debug(logs.AuditProcessingTransfers) common.TransferAssets(c.prm.Exchanger, table, common.AuditSettlementDetails(prevEpoch)) } @@ -109,35 +110,35 @@ func (c *Calculator) processResult(ctx *singleResultCtx) { zap.Uint64("audit epoch", ctx.auditResult.Epoch()), )} - ctx.log.Debug("reading information about the container") + ctx.log.Debug(logs.AuditReadingInformationAboutTheContainer) ok := c.readContainerInfo(ctx) if !ok { return } - ctx.log.Debug("building placement") + ctx.log.Debug(logs.AuditBuildingPlacement) ok = c.buildPlacement(ctx) if !ok { return } - ctx.log.Debug("collecting passed nodes") + ctx.log.Debug(logs.AuditCollectingPassedNodes) ok = c.collectPassNodes(ctx) if !ok { return } - ctx.log.Debug("calculating sum of the sizes of all storage groups") + ctx.log.Debug(logs.AuditCalculatingSumOfTheSizesOfAllStorageGroups) ok = c.sumSGSizes(ctx) if !ok { return } - ctx.log.Debug("filling transfer table") + ctx.log.Debug(logs.AuditFillingTransferTable) c.fillTransferTable(ctx) } @@ -145,7 +146,7 @@ func (c *Calculator) processResult(ctx *singleResultCtx) { func (c *Calculator) readContainerInfo(ctx *singleResultCtx) bool { cnr, ok := ctx.auditResult.Container() if !ok { - ctx.log.Error("missing container in audit result") + ctx.log.Error(logs.AuditMissingContainerInAuditResult) return false } @@ -153,7 +154,7 @@ func (c *Calculator) readContainerInfo(ctx *singleResultCtx) bool { ctx.cnrInfo, err = c.prm.ContainerStorage.ContainerInfo(cnr) if err != nil { - ctx.log.Error("could not get container info", + ctx.log.Error(logs.AuditCouldNotGetContainerInfo, zap.String("error", err.Error()), ) } @@ -166,14 +167,14 @@ func (c *Calculator) buildPlacement(ctx *singleResultCtx) bool { ctx.cnrNodes, err = c.prm.PlacementCalculator.ContainerNodes(ctx.auditEpoch(), ctx.containerID()) if err != nil { - ctx.log.Error("could not get container nodes", + ctx.log.Error(logs.AuditCouldNotGetContainerNodes, zap.String("error", err.Error()), ) } empty := len(ctx.cnrNodes) == 0 if empty { - ctx.log.Debug("empty list of container nodes") + ctx.log.Debug(logs.AuditEmptyListOfContainerNodes) } return err == nil && !empty @@ -206,7 +207,7 @@ func (c *Calculator) collectPassNodes(ctx *singleResultCtx) bool { empty := len(ctx.passNodes) == 0 if empty { - ctx.log.Debug("none of the container nodes passed the audit") + ctx.log.Debug(logs.AuditNoneOfTheContainerNodesPassedTheAudit) } return !empty @@ -224,7 +225,7 @@ func (c *Calculator) sumSGSizes(ctx *singleResultCtx) bool { sgInfo, err := c.prm.SGStorage.SGInfo(addr) if err != nil { - ctx.log.Error("could not get SG info", + ctx.log.Error(logs.AuditCouldNotGetSGInfo, zap.String("id", id.String()), zap.String("error", err.Error()), ) @@ -244,7 +245,7 @@ func (c *Calculator) sumSGSizes(ctx *singleResultCtx) bool { } if sumPassSGSize == 0 { - ctx.log.Debug("zero sum SG size") + ctx.log.Debug(logs.AuditZeroSumSGSize) return false } @@ -260,7 +261,7 @@ func (c *Calculator) fillTransferTable(ctx *singleResultCtx) bool { for k, info := range ctx.passNodes { ownerID, err := c.prm.AccountStorage.ResolveKey(info) if err != nil { - ctx.log.Error("could not resolve public key of the storage node", + ctx.log.Error(logs.AuditCouldNotResolvePublicKeyOfTheStorageNode, zap.String("error", err.Error()), zap.String("key", k), ) @@ -270,7 +271,7 @@ func (c *Calculator) fillTransferTable(ctx *singleResultCtx) bool { price := info.Price() - ctx.log.Debug("calculating storage node salary for audit (GASe-12)", + ctx.log.Debug(logs.AuditCalculatingStorageNodeSalaryForAudit, zap.Stringer("sum SG size", ctx.sumSGSize), zap.Stringer("price", price), ) @@ -292,7 +293,7 @@ func (c *Calculator) fillTransferTable(ctx *singleResultCtx) bool { // add txs to pay inner ring node for audit result auditIR, err := ownerFromKey(ctx.auditResult.AuditorKey()) if err != nil { - ctx.log.Error("could not parse public key of the inner ring node", + ctx.log.Error(logs.AuditCouldNotParsePublicKeyOfTheInnerRingNode, zap.String("error", err.Error()), zap.String("key", hex.EncodeToString(ctx.auditResult.AuditorKey())), ) diff --git a/pkg/innerring/processors/settlement/basic/collect.go b/pkg/innerring/processors/settlement/basic/collect.go index ee7354c4f..024769c06 100644 --- a/pkg/innerring/processors/settlement/basic/collect.go +++ b/pkg/innerring/processors/settlement/basic/collect.go @@ -3,6 +3,7 @@ package basic import ( "math/big" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/common" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "go.uber.org/zap" @@ -20,7 +21,7 @@ func (inc *IncomeSettlementContext) Collect() { cachedRate, err := inc.rate.BasicRate() if err != nil { - inc.log.Error("can't get basic income rate", + inc.log.Error(logs.BasicCantGetBasicIncomeRate, zap.String("error", err.Error())) return @@ -33,7 +34,7 @@ func (inc *IncomeSettlementContext) Collect() { cnrEstimations, err := inc.estimations.Estimations(inc.epoch) if err != nil { - inc.log.Error("can't fetch container size estimations", + inc.log.Error(logs.BasicCantFetchContainerSizeEstimations, zap.Uint64("epoch", inc.epoch), zap.String("error", err.Error())) @@ -45,7 +46,7 @@ func (inc *IncomeSettlementContext) Collect() { for i := range cnrEstimations { owner, err := inc.container.ContainerInfo(cnrEstimations[i].ContainerID) if err != nil { - inc.log.Warn("can't fetch container info", + inc.log.Warn(logs.BasicCantFetchContainerInfo, zap.Uint64("epoch", inc.epoch), zap.Stringer("container_id", cnrEstimations[i].ContainerID), zap.String("error", err.Error())) @@ -55,7 +56,7 @@ func (inc *IncomeSettlementContext) Collect() { cnrNodes, err := inc.placement.ContainerNodes(inc.epoch, cnrEstimations[i].ContainerID) if err != nil { - inc.log.Debug("can't fetch container info", + inc.log.Debug(logs.BasicCantFetchContainerInfo, zap.Uint64("epoch", inc.epoch), zap.Stringer("container_id", cnrEstimations[i].ContainerID), zap.String("error", err.Error())) diff --git a/pkg/innerring/processors/settlement/basic/distribute.go b/pkg/innerring/processors/settlement/basic/distribute.go index e085f1e22..44a8ccea3 100644 --- a/pkg/innerring/processors/settlement/basic/distribute.go +++ b/pkg/innerring/processors/settlement/basic/distribute.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "math/big" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/common" "go.uber.org/zap" ) @@ -20,7 +21,7 @@ func (inc *IncomeSettlementContext) Distribute() { bankBalance, err := inc.balances.Balance(inc.bankOwner) if err != nil { - inc.log.Error("can't fetch balance of banking account", + inc.log.Error(logs.BasicCantFetchBalanceOfBankingAccount, zap.String("error", err.Error())) return @@ -31,7 +32,7 @@ func (inc *IncomeSettlementContext) Distribute() { inc.distributeTable.Iterate(func(key []byte, n *big.Int) { nodeOwner, err := inc.accounts.ResolveKey(nodeInfoWrapper(key)) if err != nil { - inc.log.Warn("can't transform public key to owner id", + inc.log.Warn(logs.BasicCantTransformPublicKeyToOwnerID, zap.String("public_key", hex.EncodeToString(key)), zap.String("error", err.Error())) diff --git a/pkg/innerring/processors/settlement/calls.go b/pkg/innerring/processors/settlement/calls.go index 33191662b..2687ad206 100644 --- a/pkg/innerring/processors/settlement/calls.go +++ b/pkg/innerring/processors/settlement/calls.go @@ -1,6 +1,7 @@ package settlement import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" @@ -14,7 +15,7 @@ func (p *Processor) HandleAuditEvent(e event.Event) { epoch := ev.Epoch() if !p.state.IsAlphabet() { - p.log.Info("non alphabet mode, ignore audit payments") + p.log.Info(logs.SettlementNonAlphabetModeIgnoreAuditPayments) return } @@ -23,10 +24,10 @@ func (p *Processor) HandleAuditEvent(e event.Event) { zap.Uint64("epoch", epoch), )} - log.Info("new audit settlement event") + log.Info(logs.SettlementNewAuditSettlementEvent) if epoch == 0 { - log.Debug("ignore genesis epoch") + log.Debug(logs.SettlementIgnoreGenesisEpoch) return } @@ -38,14 +39,14 @@ func (p *Processor) HandleAuditEvent(e event.Event) { err := p.pool.Submit(handler.handle) if err != nil { - log.Warn("could not add handler of AuditEvent to queue", + log.Warn(logs.SettlementCouldNotAddHandlerOfAuditEventToQueue, zap.String("error", err.Error()), ) return } - log.Debug("AuditEvent handling successfully scheduled") + log.Debug(logs.SettlementAuditEventHandlingSuccessfullyScheduled) } func (p *Processor) HandleIncomeCollectionEvent(e event.Event) { @@ -53,19 +54,19 @@ func (p *Processor) HandleIncomeCollectionEvent(e event.Event) { epoch := ev.Epoch() if !p.state.IsAlphabet() { - p.log.Info("non alphabet mode, ignore income collection event") + p.log.Info(logs.SettlementNonAlphabetModeIgnoreIncomeCollectionEvent) return } - p.log.Info("start basic income collection", + p.log.Info(logs.SettlementStartBasicIncomeCollection, zap.Uint64("epoch", epoch)) p.contextMu.Lock() defer p.contextMu.Unlock() if _, ok := p.incomeContexts[epoch]; ok { - p.log.Error("income context already exists", + p.log.Error(logs.SettlementIncomeContextAlreadyExists, zap.Uint64("epoch", epoch)) return @@ -73,7 +74,7 @@ func (p *Processor) HandleIncomeCollectionEvent(e event.Event) { incomeCtx, err := p.basicIncome.CreateContext(epoch) if err != nil { - p.log.Error("can't create income context", + p.log.Error(logs.SettlementCantCreateIncomeContext, zap.String("error", err.Error())) return @@ -85,7 +86,7 @@ func (p *Processor) HandleIncomeCollectionEvent(e event.Event) { incomeCtx.Collect() }) if err != nil { - p.log.Warn("could not add handler of basic income collection to queue", + p.log.Warn(logs.SettlementCouldNotAddHandlerOfBasicIncomeCollectionToQueue, zap.String("error", err.Error()), ) @@ -98,12 +99,12 @@ func (p *Processor) HandleIncomeDistributionEvent(e event.Event) { epoch := ev.Epoch() if !p.state.IsAlphabet() { - p.log.Info("non alphabet mode, ignore income distribution event") + p.log.Info(logs.SettlementNonAlphabetModeIgnoreIncomeDistributionEvent) return } - p.log.Info("start basic income distribution", + p.log.Info(logs.SettlementStartBasicIncomeDistribution, zap.Uint64("epoch", epoch)) p.contextMu.Lock() @@ -113,7 +114,7 @@ func (p *Processor) HandleIncomeDistributionEvent(e event.Event) { delete(p.incomeContexts, epoch) if !ok { - p.log.Warn("income context distribution does not exists", + p.log.Warn(logs.SettlementIncomeContextDistributionDoesNotExists, zap.Uint64("epoch", epoch)) return @@ -123,7 +124,7 @@ func (p *Processor) HandleIncomeDistributionEvent(e event.Event) { incomeCtx.Distribute() }) if err != nil { - p.log.Warn("could not add handler of basic income distribution to queue", + p.log.Warn(logs.SettlementCouldNotAddHandlerOfBasicIncomeDistributionToQueue, zap.String("error", err.Error()), ) diff --git a/pkg/innerring/processors/settlement/handlers.go b/pkg/innerring/processors/settlement/handlers.go index f73b61983..e69d829eb 100644 --- a/pkg/innerring/processors/settlement/handlers.go +++ b/pkg/innerring/processors/settlement/handlers.go @@ -1,6 +1,9 @@ package settlement -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" +) type auditEventHandler struct { log *logger.Logger @@ -11,9 +14,9 @@ type auditEventHandler struct { } func (p *auditEventHandler) handle() { - p.log.Info("process audit settlements") + p.log.Info(logs.SettlementProcessAuditSettlements) p.proc.ProcessAuditSettlements(p.epoch) - p.log.Info("audit processing finished") + p.log.Info(logs.SettlementAuditProcessingFinished) } diff --git a/pkg/innerring/processors/settlement/processor.go b/pkg/innerring/processors/settlement/processor.go index e86666d5c..1870a0351 100644 --- a/pkg/innerring/processors/settlement/processor.go +++ b/pkg/innerring/processors/settlement/processor.go @@ -4,6 +4,7 @@ import ( "fmt" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/basic" nodeutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -63,7 +64,7 @@ func New(prm Prm, opts ...Option) *Processor { panic(fmt.Errorf("could not create worker pool: %w", err)) } - o.log.Debug("worker pool for settlement processor successfully initialized", + o.log.Debug(logs.SettlementWorkerPoolForSettlementProcessorSuccessfullyInitialized, zap.Int("capacity", o.poolSize), ) diff --git a/pkg/innerring/rpc.go b/pkg/innerring/rpc.go index 8e96deb7b..9a89c4fcb 100644 --- a/pkg/innerring/rpc.go +++ b/pkg/innerring/rpc.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" storagegroup2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/storagegroup" @@ -94,7 +95,7 @@ func (c *ClientCache) getSG(ctx context.Context, addr oid.Address, nm *netmap.Ne cli, err := c.getWrappedClient(info) if err != nil { - c.log.Warn("can't setup remote connection", + c.log.Warn(logs.InnerringCantSetupRemoteConnection, zap.String("error", err.Error())) continue @@ -109,7 +110,7 @@ func (c *ClientCache) getSG(ctx context.Context, addr oid.Address, nm *netmap.Ne cancel() if err != nil { - c.log.Warn("can't get storage group object", + c.log.Warn(logs.InnerringCantGetStorageGroupObject, zap.String("error", err.Error())) continue diff --git a/pkg/innerring/settlement.go b/pkg/innerring/settlement.go index 08e7a9f4d..90255f5c1 100644 --- a/pkg/innerring/settlement.go +++ b/pkg/innerring/settlement.go @@ -9,6 +9,7 @@ import ( "fmt" "math/big" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/audit" @@ -223,7 +224,7 @@ func (s settlementDeps) Transfer(sender, recipient user.ID, amount *big.Int, det ) if !amount.IsInt64() { - s.log.Error("amount can not be represented as an int64") + s.log.Error(logs.InnerringAmountCanNotBeRepresentedAsAnInt64) return } @@ -262,7 +263,7 @@ func (b basicIncomeSettlementDeps) Estimations(epoch uint64) ([]*containerClient for i := range estimationIDs { estimation, err := b.cnrClient.GetUsedSpaceEstimations(estimationIDs[i]) if err != nil { - b.log.Warn("can't get used space estimation", + b.log.Warn(logs.InnerringCantGetUsedSpaceEstimation, zap.String("estimation_id", hex.EncodeToString(estimationIDs[i])), zap.String("error", err.Error())) diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 903d9c876..e3bf7886e 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" auditClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit" @@ -61,7 +62,7 @@ func (s *Server) IsAlphabet() bool { func (s *Server) InnerRingIndex() int { index, err := s.statusIndex.InnerRingIndex() if err != nil { - s.log.Error("can't get inner ring index", zap.String("error", err.Error())) + s.log.Error(logs.InnerringCantGetInnerRingIndex, zap.String("error", err.Error())) return -1 } @@ -73,7 +74,7 @@ func (s *Server) InnerRingIndex() int { func (s *Server) InnerRingSize() int { size, err := s.statusIndex.InnerRingSize() if err != nil { - s.log.Error("can't get inner ring size", zap.String("error", err.Error())) + s.log.Error(logs.InnerringCantGetInnerRingSize, zap.String("error", err.Error())) return 0 } @@ -85,7 +86,7 @@ func (s *Server) InnerRingSize() int { func (s *Server) AlphabetIndex() int { index, err := s.statusIndex.AlphabetIndex() if err != nil { - s.log.Error("can't get alphabet index", zap.String("error", err.Error())) + s.log.Error(logs.InnerringCantGetAlphabetIndex, zap.String("error", err.Error())) return -1 } @@ -97,13 +98,13 @@ func (s *Server) voteForSidechainValidator(prm governance.VoteValidatorPrm) erro index := s.InnerRingIndex() if s.contracts.alphabet.indexOutOfRange(index) { - s.log.Info("ignore validator vote: node not in alphabet range") + s.log.Info(logs.InnerringIgnoreValidatorVoteNodeNotInAlphabetRange) return nil } if len(validators) == 0 { - s.log.Info("ignore validator vote: empty validators list") + s.log.Info(logs.InnerringIgnoreValidatorVoteEmptyValidatorsList) return nil } @@ -128,7 +129,7 @@ func (s *Server) voteForSidechainValidator(prm governance.VoteValidatorPrm) erro s.contracts.alphabet.iterate(func(letter GlagoliticLetter, contract util.Uint160) { err := s.morphClient.NotaryInvoke(contract, s.feeConfig.SideChainFee(), nonce, vubP, voteMethod, epoch, validators) if err != nil { - s.log.Warn("can't invoke vote method in alphabet contract", + s.log.Warn(logs.InnerringCantInvokeVoteMethodInAlphabetContract, zap.Int8("alphabet_index", int8(letter)), zap.Uint64("epoch", epoch), zap.String("error", err.Error())) diff --git a/pkg/innerring/subnet.go b/pkg/innerring/subnet.go index 5375029d4..03108aac2 100644 --- a/pkg/innerring/subnet.go +++ b/pkg/innerring/subnet.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" irsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/subnet" netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" morphsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/subnet" @@ -173,7 +174,7 @@ func (s *Server) catchSubnetCreation(e event.Event) { s.handleSubnetCreation(e) }) if err != nil { - s.log.Error("subnet creation queue failure", + s.log.Error(logs.InnerringSubnetCreationQueueFailure, zap.String("error", err.Error()), ) } @@ -225,7 +226,7 @@ func (s *Server) handleSubnetCreation(e event.Event) { ev: putEv, }) if err != nil { - s.log.Info("discard subnet creation", + s.log.Info(logs.InnerringDiscardSubnetCreation, zap.String("reason", err.Error()), ) @@ -251,7 +252,7 @@ func (s *Server) handleSubnetCreation(e event.Event) { } if err != nil { - s.log.Error("approve subnet creation", + s.log.Error(logs.InnerringApproveSubnetCreation, zap.Bool("notary", isNotary), zap.String("error", err.Error()), ) @@ -266,7 +267,7 @@ func (s *Server) catchSubnetRemoval(e event.Event) { s.handleSubnetRemoval(e) }) if err != nil { - s.log.Error("subnet removal handling failure", + s.log.Error(logs.InnerringSubnetRemovalHandlingFailure, zap.String("error", err.Error()), ) } @@ -280,7 +281,7 @@ func (s *Server) handleSubnetRemoval(e event.Event) { candidates, err := s.netmapClient.GetCandidates() if err != nil { - s.log.Error("getting netmap candidates", + s.log.Error(logs.InnerringGettingNetmapCandidates, zap.Error(err), ) @@ -290,7 +291,7 @@ func (s *Server) handleSubnetRemoval(e event.Event) { var removedID subnetid.ID err = removedID.Unmarshal(delEv.ID()) if err != nil { - s.log.Error("unmarshalling removed subnet ID", + s.log.Error(logs.InnerringUnmarshallingRemovedSubnetID, zap.String("error", err.Error()), ) @@ -318,8 +319,8 @@ func (s *Server) processCandidate(txHash neogoutil.Uint256, removedID subnetid.I return nil }) if err != nil { - log.Error("iterating node's subnets", zap.Error(err)) - log.Debug("removing node from netmap candidates") + log.Error(logs.InnerringIteratingNodesSubnets, zap.Error(err)) + log.Debug(logs.InnerringRemovingNodeFromNetmapCandidates) var updateStatePrm netmapclient.UpdatePeerPrm updateStatePrm.SetKey(c.PublicKey()) @@ -327,7 +328,7 @@ func (s *Server) processCandidate(txHash neogoutil.Uint256, removedID subnetid.I err = s.netmapClient.UpdatePeerState(updateStatePrm) if err != nil { - log.Error("removing node from candidates", + log.Error(logs.InnerringRemovingNodeFromCandidates, zap.Error(err), ) } @@ -338,7 +339,7 @@ func (s *Server) processCandidate(txHash neogoutil.Uint256, removedID subnetid.I // remove subnet from node's information // if it contains removed subnet if removeSubnet { - log.Debug("removing subnet from the node") + log.Debug(logs.InnerringRemovingSubnetFromTheNode) var addPeerPrm netmapclient.AddPeerPrm addPeerPrm.SetNodeInfo(c) @@ -346,7 +347,7 @@ func (s *Server) processCandidate(txHash neogoutil.Uint256, removedID subnetid.I err = s.netmapClient.AddPeer(addPeerPrm) if err != nil { - log.Error("updating subnet info", + log.Error(logs.InnerringUpdatingSubnetInfo, zap.Error(err), ) } diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index 3912deac0..84274528a 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "go.etcd.io/bbolt" "go.uber.org/zap" @@ -14,7 +15,7 @@ import ( // // If the database file does not exist, it will be created automatically. func (b *Blobovnicza) Open() error { - b.log.Debug("creating directory for BoltDB", + b.log.Debug(logs.BlobovniczaCreatingDirectoryForBoltDB, zap.String("path", b.path), zap.Bool("ro", b.boltOptions.ReadOnly), ) @@ -28,7 +29,7 @@ func (b *Blobovnicza) Open() error { } } - b.log.Debug("opening BoltDB", + b.log.Debug(logs.BlobovniczaOpeningBoltDB, zap.String("path", b.path), zap.Stringer("permissions", b.perm), ) @@ -44,13 +45,13 @@ func (b *Blobovnicza) Open() error { // // Should not be called in read-only configuration. func (b *Blobovnicza) Init() error { - b.log.Debug("initializing...", + b.log.Debug(logs.BlobovniczaInitializing, zap.Uint64("object size limit", b.objSizeLimit), zap.Uint64("storage size limit", b.fullSizeLimit), ) if size := b.filled.Load(); size != 0 { - b.log.Debug("already initialized", zap.Uint64("size", size)) + b.log.Debug(logs.BlobovniczaAlreadyInitialized, zap.Uint64("size", size)) return nil } @@ -59,7 +60,7 @@ func (b *Blobovnicza) Init() error { // create size range bucket rangeStr := stringifyBounds(lower, upper) - b.log.Debug("creating bucket for size range", + b.log.Debug(logs.BlobovniczaCreatingBucketForSizeRange, zap.String("range", rangeStr)) _, err := tx.CreateBucketIfNotExists(key) @@ -86,7 +87,7 @@ func (b *Blobovnicza) Init() error { // Close releases all internal database resources. func (b *Blobovnicza) Close() error { - b.log.Debug("closing BoltDB", + b.log.Debug(logs.BlobovniczaClosingBoltDB, zap.String("path", b.path), ) diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 1f885bd8e..6ce6f349c 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -51,7 +52,7 @@ func (b *Blobovnicza) Delete(prm DeletePrm) (DeleteRes, error) { err := buck.Delete(addrKey) if err == nil { - b.log.Debug("object was removed from bucket", + b.log.Debug(logs.BlobovniczaObjectWasRemovedFromBucket, zap.String("binary size", stringifyByteSize(sz)), zap.String("range", stringifyBounds(lower, upper)), ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index c628c96be..af976f977 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -7,6 +7,7 @@ import ( "strconv" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" @@ -104,12 +105,12 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { // it from opened cache. return } else if err := value.Close(); err != nil { - blz.log.Error("could not close Blobovnicza", + blz.log.Error(logs.BlobovniczatreeCouldNotCloseBlobovnicza, zap.String("id", p), zap.String("error", err.Error()), ) } else { - blz.log.Debug("blobovnicza successfully closed on evict", + blz.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyClosedOnEvict, zap.String("id", p), ) } @@ -141,11 +142,11 @@ func (b *Blobovniczas) getActivated(lvlPath string) (blobovniczaWithIndex, error // // if current active blobovnicza's index is not old, it remains unchanged. func (b *Blobovniczas) updateActive(lvlPath string, old *uint64) error { - b.log.Debug("updating active blobovnicza...", zap.String("path", lvlPath)) + b.log.Debug(logs.BlobovniczatreeUpdatingActiveBlobovnicza, zap.String("path", lvlPath)) _, err := b.updateAndGet(lvlPath, old) - b.log.Debug("active blobovnicza successfully updated", zap.String("path", lvlPath)) + b.log.Debug(logs.BlobovniczatreeActiveBlobovniczaSuccessfullyUpdated, zap.String("path", lvlPath)) return err } @@ -201,7 +202,7 @@ func (b *Blobovniczas) updateAndGet(lvlPath string, old *uint64) (blobovniczaWit } b.lruMtx.Unlock() - b.log.Debug("blobovnicza successfully activated", + b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyActivated, zap.String("path", activePath)) return active, nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index e7e890e50..0240c7a97 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "go.uber.org/zap" ) @@ -18,10 +19,10 @@ func (b *Blobovniczas) Open(readOnly bool) error { // // Should be called exactly once. func (b *Blobovniczas) Init() error { - b.log.Debug("initializing Blobovnicza's") + b.log.Debug(logs.BlobovniczatreeInitializingBlobovniczas) if b.readOnly { - b.log.Debug("read-only mode, skip blobovniczas initialization...") + b.log.Debug(logs.BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization) return nil } @@ -36,7 +37,7 @@ func (b *Blobovniczas) Init() error { return true, fmt.Errorf("could not initialize blobovnicza structure %s: %w", p, err) } - b.log.Debug("blobovnicza successfully initialized, closing...", zap.String("id", p)) + b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) return false, nil }) } @@ -49,7 +50,7 @@ func (b *Blobovniczas) Close() error { for p, v := range b.active { if err := v.blz.Close(); err != nil { - b.log.Debug("could not close active blobovnicza", + b.log.Debug(logs.BlobovniczatreeCouldNotCloseActiveBlobovnicza, zap.String("path", p), zap.String("error", err.Error()), ) @@ -59,7 +60,7 @@ func (b *Blobovniczas) Close() error { for _, k := range b.opened.Keys() { blz, _ := b.opened.Get(k) if err := blz.Close(); err != nil { - b.log.Debug("could not close active blobovnicza", + b.log.Debug(logs.BlobovniczatreeCouldNotCloseActiveBlobovnicza, zap.String("path", k), zap.String("error", err.Error()), ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 7e14d6d8d..202807653 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -3,6 +3,7 @@ package blobovniczatree import ( "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -44,7 +45,7 @@ func (b *Blobovniczas) Delete(prm common.DeletePrm) (res common.DeleteRes, err e res, err = b.deleteObjectFromLevel(bPrm, p, !ok, prm) if err != nil { if !blobovnicza.IsErrNotFound(err) { - b.log.Debug("could not remove object from level", + b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), ) @@ -83,7 +84,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(prm blobovnicza.DeletePrm, blzPath if res, err := b.deleteObject(v, prm, dp); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { - b.log.Debug("could not remove object from opened blobovnicza", + b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), ) @@ -102,7 +103,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(prm blobovnicza.DeletePrm, blzPath if res, err := b.deleteObject(active.blz, prm, dp); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { - b.log.Debug("could not remove object from active blobovnicza", + b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index 748843ee9..9d9fd4cba 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -6,6 +6,7 @@ import ( "path/filepath" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "go.opentelemetry.io/otel/attribute" @@ -47,7 +48,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common _, err := b.getObjectFromLevel(ctx, gPrm, p, !ok) if err != nil { if !blobovnicza.IsErrNotFound(err) { - b.log.Debug("could not get object from level", + b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error())) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index bb84db086..0b8ccb64f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -7,6 +7,7 @@ import ( "path/filepath" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -53,7 +54,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G res, err = b.getObjectFromLevel(ctx, bPrm, p, !ok) if err != nil { if !blobovnicza.IsErrNotFound(err) { - b.log.Debug("could not get object from level", + b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), ) @@ -88,7 +89,7 @@ func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.G if res, err := b.getObject(ctx, v, prm); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { - b.log.Debug("could not read object from opened blobovnicza", + b.log.Debug(logs.BlobovniczatreeCouldNotReadObjectFromOpenedBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), ) @@ -108,7 +109,7 @@ func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.G if res, err := b.getObject(ctx, active.blz, prm); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { - b.log.Debug("could not get object from active blobovnicza", + b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromActiveBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index b12cb32d4..d6dfe51bd 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -8,6 +8,7 @@ import ( "strconv" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -54,7 +55,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re if err != nil { outOfBounds := isErrOutOfRange(err) if !outOfBounds && !blobovnicza.IsErrNotFound(err) { - b.log.Debug("could not get object from level", + b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), ) @@ -98,7 +99,7 @@ func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRang return res, err default: if !blobovnicza.IsErrNotFound(err) { - b.log.Debug("could not read payload range from opened blobovnicza", + b.log.Debug(logs.BlobovniczatreeCouldNotReadPayloadRangeFromOpenedBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), ) @@ -123,7 +124,7 @@ func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRang return res, err default: if !blobovnicza.IsErrNotFound(err) { - b.log.Debug("could not read payload range from active blobovnicza", + b.log.Debug(logs.BlobovniczatreeCouldNotReadPayloadRangeFromActiveBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index db7ca1082..8b29119c6 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -4,6 +4,7 @@ import ( "errors" "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "go.etcd.io/bbolt" @@ -56,9 +57,9 @@ func (i *putIterator) iterate(path string) (bool, error) { active, err := i.B.getActivated(path) if err != nil { if !isLogical(err) { - i.B.reportError("could not get active blobovnicza", err) + i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) } else { - i.B.log.Debug("could not get active blobovnicza", + i.B.log.Debug(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, zap.String("error", err.Error())) } @@ -71,15 +72,15 @@ func (i *putIterator) iterate(path string) (bool, error) { // and `updateActive` takes care of not updating the active blobovnicza twice. if isFull := errors.Is(err, blobovnicza.ErrFull); isFull || errors.Is(err, bbolt.ErrDatabaseNotOpen) { if isFull { - i.B.log.Debug("blobovnicza overflowed", + i.B.log.Debug(logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("path", filepath.Join(path, u64ToHexString(active.ind)))) } if err := i.B.updateActive(path, &active.ind); err != nil { if !isLogical(err) { - i.B.reportError("could not update active blobovnicza", err) + i.B.reportError(logs.BlobovniczatreeCouldNotUpdateActiveBlobovnicza, err) } else { - i.B.log.Debug("could not update active blobovnicza", + i.B.log.Debug(logs.BlobovniczatreeCouldNotUpdateActiveBlobovnicza, zap.String("level", path), zap.String("error", err.Error())) } @@ -92,9 +93,9 @@ func (i *putIterator) iterate(path string) (bool, error) { i.AllFull = false if !isLogical(err) { - i.B.reportError("could not put object to active blobovnicza", err) + i.B.reportError(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err) } else { - i.B.log.Debug("could not put object to active blobovnicza", + i.B.log.Debug(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", filepath.Join(path, u64ToHexString(active.ind))), zap.String("error", err.Error())) } diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 6ceb9cefa..abe39575b 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -4,12 +4,13 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "go.uber.org/zap" ) // Open opens BlobStor. func (b *BlobStor) Open(readOnly bool) error { - b.log.Debug("opening...") + b.log.Debug(logs.BlobstorOpening) for i := range b.storage { err := b.storage[i].Storage.Open(readOnly) @@ -29,7 +30,7 @@ var ErrInitBlobovniczas = errors.New("failure on blobovnicza initialization stag // // Returns wrapped ErrInitBlobovniczas on blobovnicza tree's initializaiton failure. func (b *BlobStor) Init() error { - b.log.Debug("initializing...") + b.log.Debug(logs.BlobstorInitializing) if err := b.compression.Init(); err != nil { return err @@ -46,13 +47,13 @@ func (b *BlobStor) Init() error { // Close releases all internal resources of BlobStor. func (b *BlobStor) Close() error { - b.log.Debug("closing...") + b.log.Debug(logs.BlobstorClosing) var firstErr error for i := range b.storage { err := b.storage[i].Storage.Close() if err != nil { - b.log.Info("couldn't close storage", zap.String("error", err.Error())) + b.log.Info(logs.BlobstorCouldntCloseStorage, zap.String("error", err.Error())) if firstErr == nil { firstErr = err } diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 5882c33e0..3c76764a9 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -57,7 +58,7 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi } for _, err := range errors[:len(errors)-1] { - b.log.Warn("error occurred during object existence checking", + b.log.Warn(logs.BlobstorErrorOccurredDuringObjectExistenceChecking, zap.Stringer("address", prm.Address), zap.String("error", err.Error())) } diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index 0461dd803..2c37ee776 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -3,6 +3,7 @@ package blobstor import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" @@ -38,7 +39,7 @@ func IterateBinaryObjects(blz *BlobStor, f func(addr oid.Address, data []byte, d } prm.IgnoreErrors = true prm.ErrorHandler = func(addr oid.Address, err error) error { - blz.log.Warn("error occurred during the iteration", + blz.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, zap.Stringer("address", addr), zap.String("err", err.Error())) return nil diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 3e176dc91..0c422ccc8 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "go.uber.org/zap" @@ -47,7 +48,7 @@ func (e *StorageEngine) open() error { for res := range errCh { if res.err != nil { - e.log.Error("could not open shard, closing and skipping", + e.log.Error(logs.EngineCouldNotOpenShardClosingAndSkipping, zap.String("id", res.id), zap.Error(res.err)) @@ -56,7 +57,7 @@ func (e *StorageEngine) open() error { err := sh.Close() if err != nil { - e.log.Error("could not close partially initialized shard", + e.log.Error(logs.EngineCouldNotClosePartiallyInitializedShard, zap.String("id", res.id), zap.Error(res.err)) } @@ -94,7 +95,7 @@ func (e *StorageEngine) Init(ctx context.Context) error { for res := range errCh { if res.err != nil { if errors.Is(res.err, blobstor.ErrInitBlobovniczas) { - e.log.Error("could not initialize shard, closing and skipping", + e.log.Error(logs.EngineCouldNotInitializeShardClosingAndSkipping, zap.String("id", res.id), zap.Error(res.err)) @@ -103,7 +104,7 @@ func (e *StorageEngine) Init(ctx context.Context) error { err := sh.Close() if err != nil { - e.log.Error("could not close partially initialized shard", + e.log.Error(logs.EngineCouldNotClosePartiallyInitializedShard, zap.String("id", res.id), zap.Error(res.err)) } @@ -149,7 +150,7 @@ func (e *StorageEngine) close(releasePools bool) error { for id, sh := range e.shards { if err := sh.Close(); err != nil { - e.log.Debug("could not close shard", + e.log.Debug(logs.EngineCouldNotCloseShard, zap.String("id", id), zap.String("error", err.Error()), ) @@ -309,7 +310,7 @@ loop: for _, p := range shardsToReload { err := p.sh.Reload(p.opts...) if err != nil { - e.log.Error("could not reload a shard", + e.log.Error(logs.EngineCouldNotReloadAShard, zap.Stringer("shard id", p.sh.ID()), zap.Error(err)) } @@ -338,7 +339,7 @@ loop: return fmt.Errorf("could not add %s shard: %w", idStr, err) } - e.log.Info("added new shard", zap.String("id", idStr)) + e.log.Info(logs.EngineAddedNewShard, zap.String("id", idStr)) } return nil diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 2105c452f..1f3c142a5 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -136,7 +137,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { res, err := sh.Select(selectPrm) if err != nil { - e.log.Warn("error during searching for object children", + e.log.Warn(logs.EngineErrorDuringSearchingForObjectChildren, zap.Stringer("addr", addr), zap.String("error", err.Error())) return false @@ -147,7 +148,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo _, err = sh.Inhume(ctx, inhumePrm) if err != nil { - e.log.Debug("could not inhume object in shard", + e.log.Debug(logs.EngineCouldNotInhumeObjectInShard, zap.Stringer("addr", addr), zap.String("err", err.Error())) continue diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index e0161bfe3..20c8a946b 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -4,6 +4,7 @@ import ( "errors" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -87,24 +88,24 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { sid := sh.ID() err := sh.SetMode(mode.DegradedReadOnly) if err != nil { - e.log.Error("failed to move shard in degraded-read-only mode, moving to read-only", + e.log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, zap.Stringer("shard_id", sid), zap.Uint32("error count", errCount), zap.Error(err)) err = sh.SetMode(mode.ReadOnly) if err != nil { - e.log.Error("failed to move shard in read-only mode", + e.log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Stringer("shard_id", sid), zap.Uint32("error count", errCount), zap.Error(err)) } else { - e.log.Info("shard is moved in read-only mode due to error threshold", + e.log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold, zap.Stringer("shard_id", sid), zap.Uint32("error count", errCount)) } } else { - e.log.Info("shard is moved in degraded mode due to error threshold", + e.log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold, zap.Stringer("shard_id", sid), zap.Uint32("error count", errCount)) } @@ -182,7 +183,7 @@ func (e *StorageEngine) reportShardErrorWithFlags( default: // For background workers we can have a lot of such errors, // thus logging is done with DEBUG level. - e.log.Debug("mode change is in progress, ignoring set-mode request", + e.log.Debug(logs.EngineModeChangeIsInProgressIgnoringSetmodeRequest, zap.Stringer("shard_id", sid), zap.Uint32("error_count", errCount)) } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index f16413ea2..2ec2c2b35 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -79,7 +80,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (Eva } } - e.log.Info("started shards evacuation", zap.Strings("shard_ids", shardIDs)) + e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs)) var res EvacuateShardRes @@ -89,7 +90,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (Eva } } - e.log.Info("finished shards evacuation", zap.Strings("shard_ids", shardIDs)) + e.log.Info(logs.EngineFinishedShardsEvacuation, zap.Strings("shard_ids", shardIDs)) return res, nil } @@ -206,7 +207,7 @@ func (e *StorageEngine) tryEvacuateObject(ctx context.Context, addr oid.Address, putDone, exists := e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object) if putDone || exists { if putDone { - e.log.Debug("object is moved to another shard", + e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), zap.Stringer("addr", addr)) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index db9988338..696e78742 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -83,7 +84,7 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e if !prm.forceRemoval { locked, err := e.IsLocked(prm.addrs[i]) if err != nil { - e.log.Warn("removing an object without full locking check", + e.log.Warn(logs.EngineRemovingAnObjectWithoutFullLockingCheck, zap.Error(err), zap.Stringer("addr", prm.addrs[i])) } else if locked { @@ -222,7 +223,7 @@ func (e *StorageEngine) processExpiredLocks(ctx context.Context, epoch uint64, l select { case <-ctx.Done(): - e.log.Info("interrupt processing the expired locks", zap.Error(ctx.Err())) + e.log.Info(logs.EngineInterruptProcessingTheExpiredLocks, zap.Error(ctx.Err())) return true default: return false @@ -236,7 +237,7 @@ func (e *StorageEngine) processDeletedLocks(ctx context.Context, lockers []oid.A select { case <-ctx.Done(): - e.log.Info("interrupt processing the deleted locks", zap.Error(ctx.Err())) + e.log.Info(logs.EngineInterruptProcessingTheDeletedLocks, zap.Error(ctx.Err())) return true default: return false diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 5f9105efc..aea296cc4 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -118,7 +119,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, _, err = sh.ToMoveIt(toMoveItPrm) if err != nil { - e.log.Warn("could not mark object for shard relocation", + e.log.Warn(logs.EngineCouldNotMarkObjectForShardRelocation, zap.Stringer("shard", sh.ID()), zap.String("error", err.Error()), ) @@ -135,7 +136,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, if err != nil { if errors.Is(err, shard.ErrReadOnlyMode) || errors.Is(err, blobstor.ErrNoPlaceFound) || errors.Is(err, common.ErrReadOnly) || errors.Is(err, common.ErrNoSpace) { - e.log.Warn("could not put object to shard", + e.log.Warn(logs.EngineCouldNotPutObjectToShard, zap.Stringer("shard_id", sh.ID()), zap.String("error", err.Error())) return diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index c50c0844c..d365fc7b4 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -42,7 +43,7 @@ func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicat prm.Concurrency = defaultRemoveDuplicatesConcurrency } - e.log.Info("starting removal of locally-redundant copies", + e.log.Info(logs.EngineStartingRemovalOfLocallyredundantCopies, zap.Int("concurrency", prm.Concurrency)) // The mutext must be taken for the whole duration to avoid target shard being removed @@ -54,7 +55,7 @@ func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicat // This is not currently the case, because `FreeSpace` metric used by weight sorting is always 0. // However we could change weights in future and easily forget this function. for _, sh := range e.shards { - e.log.Debug("started duplicates removal routine", zap.String("shard_id", sh.ID().String())) + e.log.Debug(logs.EngineStartedDuplicatesRemovalRoutine, zap.String("shard_id", sh.ID().String())) ch := make(chan oid.Address) errG, ctx := errgroup.WithContext(ctx) @@ -92,12 +93,12 @@ func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicat }) } if err := errG.Wait(); err != nil { - e.log.Error("finished removal of locally-redundant copies", zap.Error(err)) + e.log.Error(logs.EngineFinishedRemovalOfLocallyredundantCopies, zap.Error(err)) return err } } - e.log.Info("finished removal of locally-redundant copies") + e.log.Info(logs.EngineFinishedRemovalOfLocallyredundantCopies) return nil } diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 2b1146ff2..64546d9ef 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -3,6 +3,7 @@ package engine import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -168,7 +169,7 @@ func (e *StorageEngine) removeShards(ids ...string) { delete(e.shardPools, id) } - e.log.Info("shard has been removed", + e.log.Info(logs.EngineShardHasBeenRemoved, zap.String("id", id)) } e.mtx.Unlock() @@ -176,7 +177,7 @@ func (e *StorageEngine) removeShards(ids ...string) { for _, sh := range ss { err := sh.Close() if err != nil { - e.log.Error("could not close removed shard", + e.log.Error(logs.EngineCouldNotCloseRemovedShard, zap.Stringer("id", sh.ID()), zap.Error(err), ) diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 1a19c3e2a..4ae802aaa 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -5,6 +5,7 @@ import ( "fmt" "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -25,7 +26,7 @@ func (db *DB) Open(readOnly bool) error { return fmt.Errorf("can't create dir %s for metabase: %w", db.info.Path, err) } - db.log.Debug("created directory for Metabase", zap.String("path", db.info.Path)) + db.log.Debug(logs.MetabaseCreatedDirectoryForMetabase, zap.String("path", db.info.Path)) if db.boltOptions == nil { opts := *bbolt.DefaultOptions @@ -46,9 +47,9 @@ func (db *DB) openBolt() error { db.boltDB.MaxBatchDelay = db.boltBatchDelay db.boltDB.MaxBatchSize = db.boltBatchSize - db.log.Debug("opened boltDB instance for Metabase") + db.log.Debug(logs.MetabaseOpenedBoltDBInstanceForMetabase) - db.log.Debug("checking metabase version") + db.log.Debug(logs.MetabaseCheckingMetabaseVersion) return db.boltDB.View(func(tx *bbolt.Tx) error { // The safest way to check if the metabase is fresh is to check if it has no buckets. // However, shard info can be present. So here we check that the number of buckets is diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 20985f47a..74c261d35 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -7,6 +7,7 @@ import ( "strings" v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -267,7 +268,7 @@ func (db *DB) selectFromFKBT( ) { // matchFunc, ok := db.matchers[f.Operation()] if !ok { - db.log.Debug("missing matcher", zap.Uint32("operation", uint32(f.Operation()))) + db.log.Debug(logs.MetabaseMissingMatcher, zap.Uint32("operation", uint32(f.Operation()))) return } @@ -290,7 +291,7 @@ func (db *DB) selectFromFKBT( }) }) if err != nil { - db.log.Debug("error in FKBT selection", zap.String("error", err.Error())) + db.log.Debug(logs.MetabaseErrorInFKBTSelection, zap.String("error", err.Error())) } } @@ -360,13 +361,13 @@ func (db *DB) selectFromList( case object.MatchStringEqual: lst, err = decodeList(bkt.Get(bucketKeyHelper(f.Header(), f.Value()))) if err != nil { - db.log.Debug("can't decode list bucket leaf", zap.String("error", err.Error())) + db.log.Debug(logs.MetabaseCantDecodeListBucketLeaf, zap.String("error", err.Error())) return } default: fMatch, ok := db.matchers[op] if !ok { - db.log.Debug("unknown operation", zap.Uint32("operation", uint32(op))) + db.log.Debug(logs.MetabaseUnknownOperation, zap.Uint32("operation", uint32(op))) return } @@ -374,7 +375,7 @@ func (db *DB) selectFromList( if err = fMatch.matchBucket(bkt, f.Header(), f.Value(), func(key, val []byte) error { l, err := decodeList(val) if err != nil { - db.log.Debug("can't decode list bucket leaf", + db.log.Debug(logs.MetabaseCantDecodeListBucketLeaf, zap.String("error", err.Error()), ) @@ -385,7 +386,7 @@ func (db *DB) selectFromList( return nil }); err != nil { - db.log.Debug("can't iterate over the bucket", + db.log.Debug(logs.MetabaseCantIterateOverTheBucket, zap.String("error", err.Error()), ) @@ -429,7 +430,7 @@ func (db *DB) selectObjectID( default: fMatch, ok := db.matchers[op] if !ok { - db.log.Debug("unknown operation", + db.log.Debug(logs.MetabaseUnknownOperation, zap.Uint32("operation", uint32(f.Operation())), ) @@ -451,7 +452,7 @@ func (db *DB) selectObjectID( return nil }) if err != nil { - db.log.Debug("could not iterate over the buckets", + db.log.Debug(logs.MetabaseCouldNotIterateOverTheBuckets, zap.String("error", err.Error()), ) } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index d727d27a5..3d0f72922 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -15,7 +16,7 @@ import ( ) func (s *Shard) handleMetabaseFailure(stage string, err error) error { - s.log.Error("metabase failure, switching mode", + s.log.Error(logs.ShardMetabaseFailureSwitchingMode, zap.String("stage", stage), zap.Stringer("mode", mode.ReadOnly), zap.Error(err)) @@ -25,7 +26,7 @@ func (s *Shard) handleMetabaseFailure(stage string, err error) error { return nil } - s.log.Error("can't move shard to readonly, switch mode", + s.log.Error(logs.ShardCantMoveShardToReadonlySwitchMode, zap.String("stage", stage), zap.Stringer("mode", mode.DegradedReadOnly), zap.Error(err)) @@ -167,7 +168,7 @@ func (s *Shard) refillMetabase() error { err = blobstor.IterateBinaryObjects(s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { if err := obj.Unmarshal(data); err != nil { - s.log.Warn("could not unmarshal object", + s.log.Warn(logs.ShardCouldNotUnmarshalObject, zap.Stringer("address", addr), zap.String("err", err.Error())) return nil @@ -274,7 +275,7 @@ func (s *Shard) Close() error { for _, component := range components { if err := component.Close(); err != nil { lastErr = err - s.log.Error("could not close shard component", zap.Error(err)) + s.log.Error(logs.ShardCouldNotCloseShardComponent, zap.Error(err)) } } @@ -302,7 +303,7 @@ func (s *Shard) Reload(opts ...Option) error { ok, err := s.metaBase.Reload(c.metaOpts...) if err != nil { if errors.Is(err, meta.ErrDegradedMode) { - s.log.Error("can't open metabase, move to a degraded mode", zap.Error(err)) + s.log.Error(logs.ShardCantOpenMetabaseMoveToADegradedMode, zap.Error(err)) _ = s.setMode(mode.DegradedReadOnly) } return err @@ -318,12 +319,12 @@ func (s *Shard) Reload(opts ...Option) error { err = s.metaBase.Init() } if err != nil { - s.log.Error("can't initialize metabase, move to a degraded-read-only mode", zap.Error(err)) + s.log.Error(logs.ShardCantInitializeMetabaseMoveToADegradedreadonlyMode, zap.Error(err)) _ = s.setMode(mode.DegradedReadOnly) return err } } - s.log.Info("trying to restore read-write mode") + s.log.Info(logs.ShardTryingToRestoreReadwriteMode) return s.setMode(mode.ReadWrite) } diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 6ae3bf7dd..ed05f9982 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -3,6 +3,7 @@ package shard import ( "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" @@ -49,7 +50,7 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { if s.hasWriteCache() { err := s.writeCache.Delete(prm.addr[i]) if err != nil && !IsErrNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { - s.log.Warn("can't delete object from write cache", zap.String("error", err.Error())) + s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.String("error", err.Error())) } } @@ -58,7 +59,7 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { res, err := s.metaBase.StorageID(sPrm) if err != nil { - s.log.Debug("can't get storage ID from metabase", + s.log.Debug(logs.ShardCantGetStorageIDFromMetabase, zap.Stringer("object", prm.addr[i]), zap.String("error", err.Error())) @@ -100,7 +101,7 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { _, err = s.blobStor.Delete(delPrm) if err != nil { - s.log.Debug("can't remove object from blobStor", + s.log.Debug(logs.ShardCantRemoveObjectFromBlobStor, zap.Stringer("object_address", prm.addr[i]), zap.String("error", err.Error())) } diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 6f18e6c3a..5ea9ecedf 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -124,7 +125,7 @@ func (gc *gc) listenEvents(ctx context.Context) { for { event, ok := <-gc.eventChan if !ok { - gc.log.Warn("stop event listener by closed channel") + gc.log.Warn(logs.ShardStopEventListenerByClosedChannel) return } @@ -149,7 +150,7 @@ func (gc *gc) listenEvents(ctx context.Context) { v.prevGroup.Done() }) if err != nil { - gc.log.Warn("could not submit GC job to worker pool", + gc.log.Warn(logs.ShardCouldNotSubmitGCJobToWorkerPool, zap.String("error", err.Error()), ) @@ -174,7 +175,7 @@ func (gc *gc) tickRemover() { close(gc.eventChan) - gc.log.Debug("GC is stopped") + gc.log.Debug(logs.ShardGCIsStopped) return case <-timer.C: gc.remover() @@ -188,7 +189,7 @@ func (gc *gc) stop() { gc.stopChannel <- struct{}{} }) - gc.log.Info("waiting for GC workers to stop...") + gc.log.Info(logs.ShardWaitingForGCWorkersToStop) gc.wg.Wait() } @@ -220,7 +221,7 @@ func (s *Shard) removeGarbage() { // (no more than s.rmBatchSize objects) err := s.metaBase.IterateOverGarbage(iterPrm) if err != nil { - s.log.Warn("iterator over metabase graveyard failed", + s.log.Warn(logs.ShardIteratorOverMetabaseGraveyardFailed, zap.String("error", err.Error()), ) @@ -235,7 +236,7 @@ func (s *Shard) removeGarbage() { // delete accumulated objects _, err = s.delete(deletePrm) if err != nil { - s.log.Warn("could not delete the objects", + s.log.Warn(logs.ShardCouldNotDeleteTheObjects, zap.String("error", err.Error()), ) @@ -295,7 +296,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { }) if err := errGroup.Wait(); err != nil { - s.log.Warn("iterator over expired objects failed", zap.String("error", err.Error())) + s.log.Warn(logs.ShardIteratorOverExpiredObjectsFailed, zap.String("error", err.Error())) } } @@ -321,7 +322,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) // inhume the collected objects res, err := s.metaBase.Inhume(inhumePrm) if err != nil { - s.log.Warn("could not inhume the objects", + s.log.Warn(logs.ShardCouldNotInhumeTheObjects, zap.String("error", err.Error()), ) @@ -342,7 +343,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { epoch := e.(newEpoch).epoch log := s.log.With(zap.Uint64("epoch", epoch)) - log.Debug("started expired tombstones handling") + log.Debug(logs.ShardStartedExpiredTombstonesHandling) const tssDeleteBatch = 50 tss := make([]meta.TombstonedObject, 0, tssDeleteBatch) @@ -360,12 +361,12 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { }) for { - log.Debug("iterating tombstones") + log.Debug(logs.ShardIteratingTombstones) s.m.RLock() if s.info.Mode.NoMetabase() { - s.log.Debug("shard is in a degraded mode, skip collecting expired tombstones") + s.log.Debug(logs.ShardShardIsInADegradedModeSkipCollectingExpiredTombstones) s.m.RUnlock() return @@ -373,7 +374,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { err := s.metaBase.IterateOverGraveyard(iterPrm) if err != nil { - log.Error("iterator over graveyard failed", zap.Error(err)) + log.Error(logs.ShardIteratorOverGraveyardFailed, zap.Error(err)) s.m.RUnlock() return @@ -392,7 +393,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { } } - log.Debug("handling expired tombstones batch", zap.Int("number", len(tssExp))) + log.Debug(logs.ShardHandlingExpiredTombstonesBatch, zap.Int("number", len(tssExp))) s.expiredTombstonesCallback(ctx, tssExp) iterPrm.SetOffset(tss[tssLen-1].Address()) @@ -400,7 +401,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { tssExp = tssExp[:0] } - log.Debug("finished expired tombstones handling") + log.Debug(logs.ShardFinishedExpiredTombstonesHandling) } func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { @@ -442,7 +443,7 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { }) if err := errGroup.Wait(); err != nil { - s.log.Warn("iterator over expired locks failed", zap.String("error", err.Error())) + s.log.Warn(logs.ShardIteratorOverExpiredLocksFailed, zap.String("error", err.Error())) } } @@ -503,7 +504,7 @@ func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) { // inhume tombstones res, err := s.metaBase.Inhume(pInhume) if err != nil { - s.log.Warn("could not mark tombstones as garbage", + s.log.Warn(logs.ShardCouldNotMarkTombstonesAsGarbage, zap.String("error", err.Error()), ) @@ -523,7 +524,7 @@ func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) { // from graveyard err = s.metaBase.DropGraves(tss) if err != nil { - s.log.Warn("could not drop expired grave records", zap.Error(err)) + s.log.Warn(logs.ShardCouldNotDropExpiredGraveRecords, zap.Error(err)) } } @@ -535,7 +536,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] } unlocked, err := s.metaBase.FreeLockedBy(lockers) if err != nil { - s.log.Warn("failure to unlock objects", + s.log.Warn(logs.ShardFailureToUnlockObjects, zap.String("error", err.Error()), ) @@ -548,7 +549,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] res, err := s.metaBase.Inhume(pInhume) if err != nil { - s.log.Warn("failure to mark lockers as garbage", + s.log.Warn(logs.ShardFailureToMarkLockersAsGarbage, zap.String("error", err.Error()), ) @@ -570,7 +571,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] func (s *Shard) inhumeUnlockedIfExpired(ctx context.Context, epoch uint64, unlocked []oid.Address) { expiredUnlocked, err := s.selectExpired(ctx, epoch, unlocked) if err != nil { - s.log.Warn("failure to get expired unlocked objects", zap.Error(err)) + s.log.Warn(logs.ShardFailureToGetExpiredUnlockedObjects, zap.Error(err)) return } @@ -589,7 +590,7 @@ func (s *Shard) HandleDeletedLocks(lockers []oid.Address) { _, err := s.metaBase.FreeLockedBy(lockers) if err != nil { - s.log.Warn("failure to unlock objects", + s.log.Warn(logs.ShardFailureToUnlockObjects, zap.String("error", err.Error()), ) diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 3406b9338..8a0296ac6 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -126,7 +127,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, cb storFetcher, return nil, false, logicerr.Wrap(apistatus.ObjectNotFound{}) } } else { - s.log.Warn("fetching object without meta", zap.Stringer("addr", addr)) + s.log.Warn(logs.ShardFetchingObjectWithoutMeta, zap.Stringer("addr", addr)) } if s.hasWriteCache() { @@ -135,11 +136,11 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, cb storFetcher, return res, false, err } if IsErrNotFound(err) { - s.log.Debug("object is missing in write-cache", + s.log.Debug(logs.ShardObjectIsMissingInWritecache, zap.Stringer("addr", addr), zap.Bool("skip_meta", skipMeta)) } else { - s.log.Error("failed to fetch object from write-cache", + s.log.Error(logs.ShardFailedToFetchObjectFromWritecache, zap.Error(err), zap.Stringer("addr", addr), zap.Bool("skip_meta", skipMeta)) diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 40a5bf22e..3457188be 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" @@ -98,7 +99,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { return InhumeRes{}, ErrLockObjectRemoval } - s.log.Debug("could not mark object to delete in metabase", + s.log.Debug(logs.ShardCouldNotMarkObjectToDeleteInMetabase, zap.String("error", err.Error()), ) diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index 9efca8983..bab1090eb 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -3,6 +3,7 @@ package shard import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -86,7 +87,7 @@ func (s *Shard) List() (res SelectRes, err error) { sRes, err := s.metaBase.Select(sPrm) // consider making List in metabase if err != nil { - s.log.Debug("can't select all objects", + s.log.Debug(logs.ShardCantSelectAllObjects, zap.Stringer("cid", lst[i]), zap.String("error", err.Error())) diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index 17ed3f3c8..50c52accc 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -1,6 +1,7 @@ package shard import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "go.uber.org/zap" @@ -25,7 +26,7 @@ func (s *Shard) SetMode(m mode.Mode) error { } func (s *Shard) setMode(m mode.Mode) error { - s.log.Info("setting shard mode", + s.log.Info(logs.ShardSettingShardMode, zap.Stringer("old_mode", s.info.Mode), zap.Stringer("new_mode", m)) @@ -66,7 +67,7 @@ func (s *Shard) setMode(m mode.Mode) error { s.metricsWriter.SetReadonly(s.info.Mode != mode.ReadWrite) } - s.log.Info("shard mode set successfully", + s.log.Info(logs.ShardShardModeSetSuccessfully, zap.Stringer("mode", s.info.Mode)) return nil } diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go index c6bf8409e..f3199ac07 100644 --- a/pkg/local_object_storage/shard/move.go +++ b/pkg/local_object_storage/shard/move.go @@ -1,6 +1,7 @@ package shard import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" @@ -38,7 +39,7 @@ func (s *Shard) ToMoveIt(prm ToMoveItPrm) (ToMoveItRes, error) { _, err := s.metaBase.ToMoveIt(toMovePrm) if err != nil { - s.log.Debug("could not mark object for shard relocation in metabase", + s.log.Debug(logs.ShardCouldNotMarkObjectForShardRelocationInMetabase, zap.String("error", err.Error()), ) } diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index 48dbe1be2..a4cb2cb1f 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -3,6 +3,7 @@ package shard import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -58,7 +59,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) { } if err != nil || !tryCache { if err != nil { - s.log.Debug("can't put object to the write-cache, trying blobstor", + s.log.Debug(logs.ShardCantPutObjectToTheWritecacheTryingBlobstor, zap.String("err", err.Error())) } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 6d1fba141..44ec54645 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -349,7 +350,7 @@ func (s *Shard) updateMetrics() { if s.cfg.metricsWriter != nil && !s.GetMode().NoMetabase() { cc, err := s.metaBase.ObjectCounters() if err != nil { - s.log.Warn("meta: object counter read", + s.log.Warn(logs.ShardMetaObjectCounterRead, zap.Error(err), ) @@ -361,7 +362,7 @@ func (s *Shard) updateMetrics() { cnrList, err := s.metaBase.Containers() if err != nil { - s.log.Warn("meta: can't read container list", zap.Error(err)) + s.log.Warn(logs.ShardMetaCantReadContainerList, zap.Error(err)) return } @@ -370,7 +371,7 @@ func (s *Shard) updateMetrics() { for i := range cnrList { size, err := s.metaBase.ContainerSize(cnrList[i]) if err != nil { - s.log.Warn("meta: can't read container size", + s.log.Warn(logs.ShardMetaCantReadContainerSize, zap.String("cid", cnrList[i].EncodeToString()), zap.Error(err)) continue diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 0437367e7..3ca3aa905 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -5,6 +5,7 @@ import ( "errors" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -134,7 +135,7 @@ func (c *cache) flushDB() { c.modeMtx.RUnlock() - c.log.Debug("tried to flush items from write-cache", + c.log.Debug(logs.WritecacheTriedToFlushItemsFromWritecache, zap.Int("count", count), zap.String("start", base58.Encode(lastKey))) } diff --git a/pkg/local_object_storage/writecache/init.go b/pkg/local_object_storage/writecache/init.go index ffe7a0129..0ac8cea99 100644 --- a/pkg/local_object_storage/writecache/init.go +++ b/pkg/local_object_storage/writecache/init.go @@ -5,6 +5,7 @@ import ( "errors" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -54,7 +55,7 @@ func (c *cache) initFlushMarks() { var errStopIter = errors.New("stop iteration") func (c *cache) fsTreeFlushMarkUpdate() { - c.log.Info("filling flush marks for objects in FSTree") + c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInFSTree) var prm common.IteratePrm prm.LazyHandler = func(addr oid.Address, _ func() ([]byte, error)) error { @@ -86,11 +87,11 @@ func (c *cache) fsTreeFlushMarkUpdate() { return nil } _, _ = c.fsTree.Iterate(prm) - c.log.Info("finished updating FSTree flush marks") + c.log.Info(logs.WritecacheFinishedUpdatingFSTreeFlushMarks) } func (c *cache) dbFlushMarkUpdate() { - c.log.Info("filling flush marks for objects in database") + c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInDatabase) var m []string var indices []int @@ -158,7 +159,7 @@ func (c *cache) dbFlushMarkUpdate() { lastKey = append([]byte(m[len(m)-1]), 0) } - c.log.Info("finished updating flush marks") + c.log.Info(logs.WritecacheFinishedUpdatingFlushMarks) } // flushStatus returns info about the object state in the main storage. diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 997310d9e..939dc5b06 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" ) @@ -59,7 +60,7 @@ func (c *cache) setMode(m mode.Mode) error { // flushCh is populated by `flush` with `modeMtx` taken, thus waiting until it is empty // guarantees that there are no in-fly operations. for len(c.flushCh) != 0 { - c.log.Info("waiting for channels to flush") + c.log.Info(logs.WritecacheWaitingForChannelsToFlush) time.Sleep(time.Second) } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index 667d34cb9..ff7eb1d6a 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" @@ -124,7 +125,7 @@ func (c *cache) deleteFromDB(keys []string) []string { ) } if err != nil { - c.log.Error("can't remove objects from the database", zap.Error(err)) + c.log.Error(logs.WritecacheCantRemoveObjectsFromTheDatabase, zap.Error(err)) } copy(keys, keys[errorIndex:]) @@ -141,13 +142,13 @@ func (c *cache) deleteFromDisk(keys []string) []string { for i := range keys { if err := addr.DecodeString(keys[i]); err != nil { - c.log.Error("can't parse address", zap.String("address", keys[i])) + c.log.Error(logs.WritecacheCantParseAddress, zap.String("address", keys[i])) continue } _, err := c.fsTree.Delete(common.DeletePrm{Address: addr}) if err != nil && !errors.As(err, new(apistatus.ObjectNotFound)) { - c.log.Error("can't remove object from write-cache", zap.Error(err)) + c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) // Save the key for the next iteration. keys[copyIndex] = keys[i] diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 5e98211c4..1c33fa5e0 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" @@ -186,7 +187,7 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, return fmt.Errorf("could not invoke %s: %w", method, err) } - c.logger.Debug("neo client invoke", + c.logger.Debug(logs.ClientNeoClientInvoke, zap.String("method", method), zap.Uint32("vub", vub), zap.Stringer("tx_hash", txHash.Reverse())) @@ -269,7 +270,7 @@ func (c *Client) TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error return err } - c.logger.Debug("native gas transfer invoke", + c.logger.Debug(logs.ClientNativeGasTransferInvoke, zap.String("to", receiver.StringLE()), zap.Stringer("tx_hash", txHash.Reverse()), zap.Uint32("vub", vub)) @@ -303,7 +304,7 @@ func (c *Client) BatchTransferGas(receivers []util.Uint160, amount fixedn.Fixed8 return err } - c.logger.Debug("batch gas transfer invoke", + c.logger.Debug(logs.ClientBatchGasTransferInvoke, zap.Strings("to", receiversLog), zap.Stringer("tx_hash", txHash.Reverse()), zap.Uint32("vub", vub)) @@ -330,7 +331,7 @@ func (c *Client) Wait(ctx context.Context, n uint32) error { height, err = c.rpcActor.GetBlockCount() if err != nil { - c.logger.Error("can't get blockchain height", + c.logger.Error(logs.ClientCantGetBlockchainHeight, zap.String("error", err.Error())) return nil } @@ -344,7 +345,7 @@ func (c *Client) Wait(ctx context.Context, n uint32) error { newHeight, err = c.rpcActor.GetBlockCount() if err != nil { - c.logger.Error("can't get blockchain height", + c.logger.Error(logs.ClientCantGetBlockchainHeight243, zap.String("error", err.Error())) return nil } diff --git a/pkg/morph/client/multi.go b/pkg/morph/client/multi.go index 5d736839a..fab90b446 100644 --- a/pkg/morph/client/multi.go +++ b/pkg/morph/client/multi.go @@ -5,6 +5,7 @@ import ( "sort" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/neorpc" @@ -44,7 +45,7 @@ func (c *Client) switchRPC(ctx context.Context) bool { newEndpoint := c.endpoints.list[c.endpoints.curr].Address cli, act, err := c.newCli(ctx, newEndpoint) if err != nil { - c.logger.Warn("could not establish connection to the switched RPC node", + c.logger.Warn(logs.ClientCouldNotEstablishConnectionToTheSwitchedRPCNode, zap.String("endpoint", newEndpoint), zap.Error(err), ) @@ -54,7 +55,7 @@ func (c *Client) switchRPC(ctx context.Context) bool { c.cache.invalidate() - c.logger.Info("connection to the new RPC node has been established", + c.logger.Info(logs.ClientConnectionToTheNewRPCNodeHasBeenEstablished, zap.String("endpoint", newEndpoint)) subs, ok := c.restoreSubscriptions(ctx, cli, newEndpoint, false) @@ -147,7 +148,7 @@ func (c *Client) routeEvent(ctx context.Context, e any) { func (c *Client) reconnect(ctx context.Context) bool { if closeErr := c.client.GetError(); closeErr != nil { - c.logger.Warn("switching to the next RPC node", + c.logger.Warn(logs.ClientSwitchingToTheNextRPCNode, zap.String("reason", closeErr.Error()), ) } else { @@ -158,7 +159,7 @@ func (c *Client) reconnect(ctx context.Context) bool { } if !c.switchRPC(ctx) { - c.logger.Error("could not establish connection to any RPC node") + c.logger.Error(logs.ClientCouldNotEstablishConnectionToAnyRPCNode) // could not connect to all endpoints => // switch client to inactive mode @@ -210,7 +211,7 @@ mainLoop: cli, act, err := c.newCli(ctx, tryE) if err != nil { - c.logger.Warn("could not create client to the higher priority node", + c.logger.Warn(logs.ClientCouldNotCreateClientToTheHigherPriorityNode, zap.String("endpoint", tryE), zap.Error(err), ) @@ -237,13 +238,13 @@ mainLoop: c.switchLock.Unlock() - c.logger.Info("switched to the higher priority RPC", + c.logger.Info(logs.ClientSwitchedToTheHigherPriorityRPC, zap.String("endpoint", tryE)) return } - c.logger.Warn("could not restore side chain subscriptions using node", + c.logger.Warn(logs.ClientCouldNotRestoreSideChainSubscriptionsUsingNode, zap.String("endpoint", tryE), zap.Error(err), ) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 7399c19cd..427554372 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -8,6 +8,7 @@ import ( "math/big" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" "github.com/nspcc-dev/neo-go/pkg/core/native/nativenames" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" @@ -203,7 +204,7 @@ func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (res util.Uint2 // Transaction is already in mempool waiting to be processed. // This is an expected situation if we restart the service. - c.logger.Info("notary deposit has already been made", + c.logger.Info(logs.ClientNotaryDepositHasAlreadyBeenMade, zap.Int64("amount", int64(amount)), zap.Int64("expire_at", till), zap.Uint32("vub", vub), @@ -211,7 +212,7 @@ func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (res util.Uint2 return util.Uint256{}, nil } - c.logger.Info("notary deposit invoke", + c.logger.Info(logs.ClientNotaryDepositInvoke, zap.Int64("amount", int64(amount)), zap.Int64("expire_at", till), zap.Uint32("vub", vub), @@ -430,7 +431,7 @@ func (c *Client) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { return err } - c.logger.Debug("notary request with prepared main TX invoked", + c.logger.Debug(logs.ClientNotaryRequestWithPreparedMainTXInvoked, zap.Uint32("fallback_valid_for", c.notary.fallbackTime), zap.Stringer("tx_hash", resp.Hash().Reverse())) @@ -489,7 +490,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint return err } - c.logger.Debug("notary request invoked", + c.logger.Debug(logs.ClientNotaryRequestInvoked, zap.String("method", method), zap.Uint32("valid_until_block", until), zap.Uint32("fallback_valid_for", c.notary.fallbackTime), diff --git a/pkg/morph/client/notifications.go b/pkg/morph/client/notifications.go index 300bab825..69eafc659 100644 --- a/pkg/morph/client/notifications.go +++ b/pkg/morph/client/notifications.go @@ -3,6 +3,7 @@ package client import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/neorpc" @@ -260,7 +261,7 @@ func (c *Client) restoreSubscriptions(ctx context.Context, cli *rpcclient.WSClie if si.subscribedToBlocks { _, err = cli.ReceiveBlocks(nil, blockRcv) if err != nil { - c.logger.Error("could not restore block subscription after RPC switch", + c.logger.Error(logs.ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch, zap.String("endpoint", endpoint), zap.Error(err), ) @@ -274,7 +275,7 @@ func (c *Client) restoreSubscriptions(ctx context.Context, cli *rpcclient.WSClie contract := contract // See https://github.com/nspcc-dev/neo-go/issues/2890 id, err = cli.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, notificationRcv) if err != nil { - c.logger.Error("could not restore notification subscription after RPC switch", + c.logger.Error(logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, zap.String("endpoint", endpoint), zap.Error(err), ) @@ -291,7 +292,7 @@ func (c *Client) restoreSubscriptions(ctx context.Context, cli *rpcclient.WSClie signer := signer // See https://github.com/nspcc-dev/neo-go/issues/2890 id, err = cli.ReceiveNotaryRequests(&neorpc.TxFilter{Signer: &signer}, notaryReqRcv) if err != nil { - c.logger.Error("could not restore notary notification subscription after RPC switch", + c.logger.Error(logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, zap.String("endpoint", endpoint), zap.Error(err), ) diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 3de199328..405165702 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -6,6 +6,7 @@ import ( "fmt" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/subscriber" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -133,7 +134,7 @@ var ( func (l *listener) Listen(ctx context.Context) { l.startOnce.Do(func() { if err := l.listen(ctx, nil); err != nil { - l.log.Error("could not start listen to events", + l.log.Error(logs.EventCouldNotStartListenToEvents, zap.String("error", err.Error()), ) } @@ -149,7 +150,7 @@ func (l *listener) Listen(ctx context.Context) { func (l *listener) ListenWithError(ctx context.Context, intError chan<- error) { l.startOnce.Do(func() { if err := l.listen(ctx, intError); err != nil { - l.log.Error("could not start listen to events", + l.log.Error(logs.EventCouldNotStartListenToEvents, zap.String("error", err.Error()), ) intError <- err @@ -221,53 +222,53 @@ loop: if intErr != nil { intErr <- err } else { - l.log.Error("stop event listener by error", zap.Error(err)) + l.log.Error(logs.EventStopEventListenerByError, zap.Error(err)) } break loop case <-ctx.Done(): - l.log.Info("stop event listener by context", + l.log.Info(logs.EventStopEventListenerByContext, zap.String("reason", ctx.Err().Error()), ) break loop case notifyEvent, ok := <-chs.NotificationsCh: if !ok { - l.log.Warn("stop event listener by notification channel") + l.log.Warn(logs.EventStopEventListenerByNotificationChannel) if intErr != nil { intErr <- errors.New("event subscriber connection has been terminated") } break loop } else if notifyEvent == nil { - l.log.Warn("nil notification event was caught") + l.log.Warn(logs.EventNilNotificationEventWasCaught) continue loop } l.handleNotifyEvent(notifyEvent) case notaryEvent, ok := <-chs.NotaryRequestsCh: if !ok { - l.log.Warn("stop event listener by notary channel") + l.log.Warn(logs.EventStopEventListenerByNotaryChannel) if intErr != nil { intErr <- errors.New("notary event subscriber connection has been terminated") } break loop } else if notaryEvent == nil { - l.log.Warn("nil notary event was caught") + l.log.Warn(logs.EventNilNotaryEventWasCaught) continue loop } l.handleNotaryEvent(notaryEvent) case b, ok := <-chs.BlockCh: if !ok { - l.log.Warn("stop event listener by block channel") + l.log.Warn(logs.EventStopEventListenerByBlockChannel) if intErr != nil { intErr <- errors.New("new block notification channel is closed") } break loop } else if b == nil { - l.log.Warn("nil block was caught") + l.log.Warn(logs.EventNilBlockWasCaught) continue loop } @@ -282,7 +283,7 @@ func (l *listener) handleBlockEvent(b *block.Block) { l.blockHandlers[i](b) } }); err != nil { - l.log.Warn("listener worker pool drained", + l.log.Warn(logs.EventListenerWorkerPoolDrained, zap.Int("capacity", l.pool.Cap())) } } @@ -291,7 +292,7 @@ func (l *listener) handleNotaryEvent(notaryEvent *result.NotaryRequestEvent) { if err := l.pool.Submit(func() { l.parseAndHandleNotary(notaryEvent) }); err != nil { - l.log.Warn("listener worker pool drained", + l.log.Warn(logs.EventListenerWorkerPoolDrained, zap.Int("capacity", l.pool.Cap())) } } @@ -300,7 +301,7 @@ func (l *listener) handleNotifyEvent(notifyEvent *state.ContainedNotificationEve if err := l.pool.Submit(func() { l.parseAndHandleNotification(notifyEvent) }); err != nil { - l.log.Warn("listener worker pool drained", + l.log.Warn(logs.EventListenerWorkerPoolDrained, zap.Int("capacity", l.pool.Cap())) } } @@ -327,7 +328,7 @@ func (l *listener) parseAndHandleNotification(notifyEvent *state.ContainedNotifi l.mtx.RUnlock() if !ok { - log.Debug("event parser not set") + log.Debug(logs.EventEventParserNotSet) return } @@ -335,7 +336,7 @@ func (l *listener) parseAndHandleNotification(notifyEvent *state.ContainedNotifi // parse the notification event event, err := parser(notifyEvent) if err != nil { - log.Warn("could not parse notification event", + log.Warn(logs.EventCouldNotParseNotificationEvent, zap.String("error", err.Error()), ) @@ -348,7 +349,7 @@ func (l *listener) parseAndHandleNotification(notifyEvent *state.ContainedNotifi l.mtx.RUnlock() if len(handlers) == 0 { - log.Info("notification handlers for parsed notification event were not registered", + log.Info(logs.EventNotificationHandlersForParsedNotificationEventWereNotRegistered, zap.Any("event", event), ) @@ -367,11 +368,11 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { switch { case errors.Is(err, ErrTXAlreadyHandled): case errors.Is(err, ErrMainTXExpired): - l.log.Warn("skip expired main TX notary event", + l.log.Warn(logs.EventSkipExpiredMainTXNotaryEvent, zap.String("error", err.Error()), ) default: - l.log.Warn("could not prepare and validate notary event", + l.log.Warn(logs.EventCouldNotPrepareAndValidateNotaryEvent, zap.String("error", err.Error()), ) } @@ -395,7 +396,7 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { l.mtx.RUnlock() if !ok { - log.Debug("notary parser not set") + log.Debug(logs.EventNotaryParserNotSet) return } @@ -403,7 +404,7 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { // parse the notary event event, err := parser(notaryEvent) if err != nil { - log.Warn("could not parse notary event", + log.Warn(logs.EventCouldNotParseNotaryEvent, zap.String("error", err.Error()), ) @@ -416,7 +417,7 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { l.mtx.RUnlock() if !ok { - log.Info("notary handlers for parsed notification event were not registered", + log.Info(logs.EventNotaryHandlersForParsedNotificationEventWereNotRegistered, zap.Any("event", event), ) @@ -438,7 +439,7 @@ func (l *listener) SetNotificationParser(pi NotificationParserInfo) { parser := pi.parser() if parser == nil { - log.Info("ignore nil event parser") + log.Info(logs.EventIgnoreNilEventParser) return } @@ -447,7 +448,7 @@ func (l *listener) SetNotificationParser(pi NotificationParserInfo) { // check if the listener was started if l.started { - log.Warn("listener has been already started, ignore parser") + log.Warn(logs.EventListenerHasBeenAlreadyStartedIgnoreParser) return } @@ -456,7 +457,7 @@ func (l *listener) SetNotificationParser(pi NotificationParserInfo) { l.notificationParsers[pi.scriptHashWithType] = pi.parser() } - log.Debug("registered new event parser") + log.Debug(logs.EventRegisteredNewEventParser) } // RegisterNotificationHandler registers the handler for particular notification event of contract. @@ -471,7 +472,7 @@ func (l *listener) RegisterNotificationHandler(hi NotificationHandlerInfo) { handler := hi.Handler() if handler == nil { - log.Warn("ignore nil event handler") + log.Warn(logs.EventIgnoreNilEventHandler) return } @@ -481,7 +482,7 @@ func (l *listener) RegisterNotificationHandler(hi NotificationHandlerInfo) { l.mtx.RUnlock() if !ok { - log.Warn("ignore handler of event w/o parser") + log.Warn(logs.EventIgnoreHandlerOfEventWoParser) return } @@ -493,7 +494,7 @@ func (l *listener) RegisterNotificationHandler(hi NotificationHandlerInfo) { ) l.mtx.Unlock() - log.Debug("registered new event handler") + log.Debug(logs.EventRegisteredNewEventHandler) } // EnableNotarySupport enables notary request listening. Passed hash is @@ -534,7 +535,7 @@ func (l *listener) SetNotaryParser(pi NotaryParserInfo) { parser := pi.parser() if parser == nil { - log.Info("ignore nil notary event parser") + log.Info(logs.EventIgnoreNilNotaryEventParser) return } @@ -543,7 +544,7 @@ func (l *listener) SetNotaryParser(pi NotaryParserInfo) { // check if the listener was started if l.started { - log.Warn("listener has been already started, ignore notary parser") + log.Warn(logs.EventListenerHasBeenAlreadyStartedIgnoreNotaryParser) return } @@ -552,7 +553,7 @@ func (l *listener) SetNotaryParser(pi NotaryParserInfo) { l.notaryParsers[pi.notaryRequestTypes] = pi.parser() } - log.Info("registered new event parser") + log.Info(logs.EventRegisteredNewEventParser) } // RegisterNotaryHandler registers the handler for particular notification notary request event. @@ -572,7 +573,7 @@ func (l *listener) RegisterNotaryHandler(hi NotaryHandlerInfo) { handler := hi.Handler() if handler == nil { - log.Warn("ignore nil notary event handler") + log.Warn(logs.EventIgnoreNilNotaryEventHandler) return } @@ -582,7 +583,7 @@ func (l *listener) RegisterNotaryHandler(hi NotaryHandlerInfo) { l.mtx.RUnlock() if !ok { - log.Warn("ignore handler of notary event w/o parser") + log.Warn(logs.EventIgnoreHandlerOfNotaryEventWoParser) return } @@ -591,7 +592,7 @@ func (l *listener) RegisterNotaryHandler(hi NotaryHandlerInfo) { l.notaryHandlers[hi.notaryRequestTypes] = hi.Handler() l.mtx.Unlock() - log.Info("registered new event handler") + log.Info(logs.EventRegisteredNewEventHandler) } // Stop closes subscription channel with remote neo node. @@ -603,7 +604,7 @@ func (l *listener) Stop() { func (l *listener) RegisterBlockHandler(handler BlockHandler) { if handler == nil { - l.log.Warn("ignore nil block handler") + l.log.Warn(logs.EventIgnoreNilBlockHandler) return } diff --git a/pkg/morph/event/utils.go b/pkg/morph/event/utils.go index 355fd5b4d..2a7c6250d 100644 --- a/pkg/morph/event/utils.go +++ b/pkg/morph/event/utils.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" util2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -89,7 +90,7 @@ func WorkerPoolHandler(w util2.WorkerPool, h Handler, log *logger.Logger) Handle }) if err != nil { - log.Warn("could not Submit handler to worker pool", + log.Warn(logs.EventCouldNotSubmitHandlerToWorkerPool, zap.String("error", err.Error()), ) } diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 17bed5b2d..a2e1c32eb 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -6,6 +6,7 @@ import ( "fmt" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/core/block" @@ -99,7 +100,7 @@ func (s *subscriber) SubscribeForNotification(contracts ...util.Uint160) error { func (s *subscriber) UnsubscribeForNotification() { err := s.client.UnsubscribeAll() if err != nil { - s.log.Error("unsubscribe for notification", + s.log.Error(logs.SubscriberUnsubscribeForNotification, zap.Error(err)) } } @@ -133,7 +134,7 @@ func (s *subscriber) routeNotifications(ctx context.Context) { return case notification, ok := <-notificationChan: if !ok { - s.log.Warn("remote notification channel has been closed") + s.log.Warn(logs.SubscriberRemoteNotificationChannelHasBeenClosed) close(s.notifyChan) close(s.blockChan) close(s.notaryChan) @@ -145,13 +146,13 @@ func (s *subscriber) routeNotifications(ctx context.Context) { case neorpc.NotificationEventID: notifyEvent, ok := notification.Value.(*state.ContainedNotificationEvent) if !ok { - s.log.Error("can't cast notify event value to the notify struct", + s.log.Error(logs.SubscriberCantCastNotifyEventValueToTheNotifyStruct, zap.String("received type", fmt.Sprintf("%T", notification.Value)), ) continue } - s.log.Debug("new notification event from sidechain", + s.log.Debug(logs.SubscriberNewNotificationEventFromSidechain, zap.String("name", notifyEvent.Name), ) @@ -159,7 +160,7 @@ func (s *subscriber) routeNotifications(ctx context.Context) { case neorpc.BlockEventID: b, ok := notification.Value.(*block.Block) if !ok { - s.log.Error("can't cast block event value to block", + s.log.Error(logs.SubscriberCantCastBlockEventValueToBlock, zap.String("received type", fmt.Sprintf("%T", notification.Value)), ) continue @@ -169,7 +170,7 @@ func (s *subscriber) routeNotifications(ctx context.Context) { case neorpc.NotaryRequestEventID: notaryRequest, ok := notification.Value.(*result.NotaryRequestEvent) if !ok { - s.log.Error("can't cast notify event value to the notary request struct", + s.log.Error(logs.SubscriberCantCastNotifyEventValueToTheNotaryRequestStruct, zap.String("received type", fmt.Sprintf("%T", notification.Value)), ) continue @@ -177,7 +178,7 @@ func (s *subscriber) routeNotifications(ctx context.Context) { s.notaryChan <- notaryRequest default: - s.log.Debug("unsupported notification from the chain", + s.log.Debug(logs.SubscriberUnsupportedNotificationFromTheChain, zap.Uint8("type", uint8(notification.Type)), ) } diff --git a/pkg/services/audit/auditor/context.go b/pkg/services/audit/auditor/context.go index bf720c330..194c5188a 100644 --- a/pkg/services/audit/auditor/context.go +++ b/pkg/services/audit/auditor/context.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -197,7 +198,7 @@ func (c *Context) init() { func (c *Context) expired(ctx context.Context) bool { select { case <-ctx.Done(): - c.log.Debug("audit context is done", + c.log.Debug(logs.AuditorAuditContextIsDone, zap.String("error", ctx.Err().Error()), ) @@ -212,10 +213,10 @@ func (c *Context) complete() { } func (c *Context) writeReport() { - c.log.Debug("writing audit report...") + c.log.Debug(logs.AuditorWritingAuditReport) if err := c.task.Reporter().WriteReport(c.report); err != nil { - c.log.Error("could not write audit report") + c.log.Error(logs.AuditorCouldNotWriteAuditReport) } } diff --git a/pkg/services/audit/auditor/pdp.go b/pkg/services/audit/auditor/pdp.go index 8a184eb7e..d5ad0fea4 100644 --- a/pkg/services/audit/auditor/pdp.go +++ b/pkg/services/audit/auditor/pdp.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -130,7 +131,7 @@ func (c *Context) collectHashes(ctx context.Context, p *gamePair) { sleepDur = time.Duration(rand.Uint64() % c.maxPDPSleep) } - c.log.Debug("sleep before get range hash", + c.log.Debug(logs.AuditorSleepBeforeGetRangeHash, zap.Stringer("interval", sleepDur), ) @@ -140,7 +141,7 @@ func (c *Context) collectHashes(ctx context.Context, p *gamePair) { h, err := c.cnrCom.GetRangeHash(ctx, getRangeHashPrm) if err != nil { - c.log.Debug("could not get payload range hash", + c.log.Debug(logs.AuditorCouldNotGetPayloadRangeHash, zap.Stringer("id", p.id), zap.String("node", netmap.StringifyPublicKey(n)), zap.String("error", err.Error()), diff --git a/pkg/services/audit/auditor/pop.go b/pkg/services/audit/auditor/pop.go index 32b837794..b64004bbc 100644 --- a/pkg/services/audit/auditor/pop.go +++ b/pkg/services/audit/auditor/pop.go @@ -3,6 +3,7 @@ package auditor import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/tzhash/tz" @@ -61,7 +62,7 @@ func (c *Context) processObjectPlacement(ctx context.Context, id oid.ID, nodes [ // try to get object header from node hdr, err := c.cnrCom.GetHeader(ctx, getHeaderPrm) if err != nil { - c.log.Debug("could not get object header from candidate", + c.log.Debug(logs.AuditorCouldNotGetObjectHeaderFromCandidate, zap.Stringer("id", id), zap.String("error", err.Error()), ) @@ -133,7 +134,7 @@ func (c *Context) iterateSGMembersPlacementRand(f func(oid.ID, int, []netmap.Nod // build placement vector for the current object nn, err := c.buildPlacement(id) if err != nil { - c.log.Debug("could not build placement for object", + c.log.Debug(logs.AuditorCouldNotBuildPlacementForObject, zap.Stringer("id", id), zap.String("error", err.Error()), ) diff --git a/pkg/services/audit/auditor/por.go b/pkg/services/audit/auditor/por.go index aebc25c68..d579b3a7c 100644 --- a/pkg/services/audit/auditor/por.go +++ b/pkg/services/audit/auditor/por.go @@ -5,6 +5,7 @@ import ( "context" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" @@ -71,7 +72,7 @@ func (c *Context) checkStorageGroupPoR(ctx context.Context, sgID oid.ID, sg stor hdr, err := c.cnrCom.GetHeader(ctx, getHeaderPrm) if err != nil { - c.log.Debug("can't head object", + c.log.Debug(logs.AuditorCantHeadObject, zap.String("remote_node", netmap.StringifyPublicKey(flat[j])), zap.Stringer("oid", members[i]), ) @@ -92,7 +93,7 @@ func (c *Context) checkStorageGroupPoR(ctx context.Context, sgID oid.ID, sg stor cs.Value(), }) if err != nil { - c.log.Debug("can't concatenate tz hash", + c.log.Debug(logs.AuditorCantConcatenateTzHash, zap.String("oid", members[i].String()), zap.String("error", err.Error())) @@ -122,13 +123,13 @@ func (c *Context) writeCheckReport(sizeCheck, tzCheck bool, sgID oid.ID, sg stor c.report.PassedPoR(sgID) } else { if !sizeCheck { - c.log.Debug("storage group size check failed", + c.log.Debug(logs.AuditorStorageGroupSizeCheckFailed, zap.Uint64("expected", sg.ValidationDataSize()), zap.Uint64("got", totalSize)) } if !tzCheck { - c.log.Debug("storage group tz hash check failed") + c.log.Debug(logs.AuditorStorageGroupTzHashCheckFailed) } c.report.FailedPoR(sgID) @@ -138,7 +139,7 @@ func (c *Context) writeCheckReport(sizeCheck, tzCheck bool, sgID oid.ID, sg stor func (c *Context) getShuffledNodes(member oid.ID, sgID oid.ID) ([]netmap.NodeInfo, bool) { objectPlacement, err := c.buildPlacement(member) if err != nil { - c.log.Info("can't build placement for storage group member", + c.log.Info(logs.AuditorCantBuildPlacementForStorageGroupMember, zap.Stringer("sg", sgID), zap.String("member_id", member.String()), ) diff --git a/pkg/services/audit/taskmanager/listen.go b/pkg/services/audit/taskmanager/listen.go index a16052e13..bfc37c2a1 100644 --- a/pkg/services/audit/taskmanager/listen.go +++ b/pkg/services/audit/taskmanager/listen.go @@ -3,6 +3,7 @@ package audittask import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit/auditor" "go.uber.org/zap" @@ -12,7 +13,7 @@ import ( // // The listener is terminated by context. func (m *Manager) Listen(ctx context.Context) { - m.log.Info("process routine", + m.log.Info(logs.TaskmanagerProcessRoutine, zap.Uint32("queue_capacity", m.queueCap), ) @@ -21,7 +22,7 @@ func (m *Manager) Listen(ctx context.Context) { for { select { case <-ctx.Done(): - m.log.Warn("stop listener by context", + m.log.Warn(logs.TaskmanagerStopListenerByContext, zap.String("error", ctx.Err().Error()), ) m.workerPool.Release() @@ -29,7 +30,7 @@ func (m *Manager) Listen(ctx context.Context) { return case task, ok := <-m.ch: if !ok { - m.log.Warn("queue channel is closed") + m.log.Warn(logs.TaskmanagerQueueChannelIsClosed) return } @@ -51,7 +52,7 @@ func (m *Manager) Listen(ctx context.Context) { func (m *Manager) handleTask(ctx context.Context, task *audit.Task, onCompleted func()) { pdpPool, err := m.pdpPoolGenerator() if err != nil { - m.log.Error("could not generate PDP worker pool", + m.log.Error(logs.TaskmanagerCouldNotGeneratePDPWorkerPool, zap.String("error", err.Error()), ) onCompleted() @@ -60,7 +61,7 @@ func (m *Manager) handleTask(ctx context.Context, task *audit.Task, onCompleted porPool, err := m.pdpPoolGenerator() if err != nil { - m.log.Error("could not generate PoR worker pool", + m.log.Error(logs.TaskmanagerCouldNotGeneratePoRWorkerPool, zap.String("error", err.Error()), ) onCompleted() @@ -73,7 +74,7 @@ func (m *Manager) handleTask(ctx context.Context, task *audit.Task, onCompleted if err := m.workerPool.Submit(func() { auditContext.Execute(ctx, onCompleted) }); err != nil { // may be we should report it - m.log.Warn("could not submit audit task") + m.log.Warn(logs.TaskmanagerCouldNotSubmitAuditTask) onCompleted() } } diff --git a/pkg/services/container/announcement/load/controller/calls.go b/pkg/services/container/announcement/load/controller/calls.go index f5d5d1a3d..e1ed6e496 100644 --- a/pkg/services/container/announcement/load/controller/calls.go +++ b/pkg/services/container/announcement/load/controller/calls.go @@ -3,6 +3,7 @@ package loadcontroller import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "go.uber.org/zap" @@ -52,7 +53,7 @@ func (c *Controller) Start(ctx context.Context, prm StartPrm) { } func (c *announcer) announce(ctx context.Context) { - c.log.Debug("starting to announce the values of the metrics") + c.log.Debug(logs.ControllerStartingToAnnounceTheValuesOfTheMetrics) var ( metricsIterator Iterator @@ -62,7 +63,7 @@ func (c *announcer) announce(ctx context.Context) { // initialize iterator over locally collected metrics metricsIterator, err = c.ctrl.prm.LocalMetrics.InitIterator() if err != nil { - c.log.Debug("could not initialize iterator over locally collected metrics", + c.log.Debug(logs.ControllerCouldNotInitializeIteratorOverLocallyCollectedMetrics, zap.String("error", err.Error()), ) @@ -72,7 +73,7 @@ func (c *announcer) announce(ctx context.Context) { // initialize target of local announcements targetWriter, err := c.ctrl.prm.LocalAnnouncementTarget.InitWriter(nil) if err != nil { - c.log.Debug("could not initialize announcement accumulator", + c.log.Debug(logs.ControllerCouldNotInitializeAnnouncementAccumulator, zap.String("error", err.Error()), ) @@ -90,7 +91,7 @@ func (c *announcer) announce(ctx context.Context) { }, ) if err != nil { - c.log.Debug("iterator over locally collected metrics aborted", + c.log.Debug(logs.ControllerIteratorOverLocallyCollectedMetricsAborted, zap.String("error", err.Error()), ) @@ -100,14 +101,14 @@ func (c *announcer) announce(ctx context.Context) { // finish writing err = targetWriter.Close(ctx) if err != nil { - c.log.Debug("could not finish writing local announcements", + c.log.Debug(logs.ControllerCouldNotFinishWritingLocalAnnouncements, zap.String("error", err.Error()), ) return } - c.log.Debug("trust announcement successfully finished") + c.log.Debug(logs.ControllerTrustAnnouncementSuccessfullyFinished) } func (c *Controller) acquireAnnouncement(ctx context.Context, prm StartPrm) (context.Context, *announcer) { @@ -127,7 +128,7 @@ func (c *Controller) acquireAnnouncement(ctx context.Context, prm StartPrm) (con )} if started { - log.Debug("announcement is already started") + log.Debug(logs.ControllerAnnouncementIsAlreadyStarted) return ctx, nil } @@ -159,9 +160,9 @@ func (c *commonContext) freeAnnouncement() { c.ctrl.announceMtx.Unlock() if stopped { - c.log.Debug("announcement successfully interrupted") + c.log.Debug(logs.ControllerAnnouncementSuccessfullyInterrupted) } else { - c.log.Debug("announcement is not started or already interrupted") + c.log.Debug(logs.ControllerAnnouncementIsNotStartedOrAlreadyInterrupted) } } @@ -219,7 +220,7 @@ func (c *Controller) acquireReport(ctx context.Context, prm StopPrm) (context.Co )} if started { - log.Debug("report is already started") + log.Debug(logs.ControllerReportIsAlreadyStarted) return ctx, nil } @@ -251,9 +252,9 @@ func (c *commonContext) freeReport() { c.ctrl.reportMtx.Unlock() if stopped { - c.log.Debug("announcement successfully interrupted") + c.log.Debug(logs.ControllerAnnouncementSuccessfullyInterrupted) } else { - c.log.Debug("announcement is not started or already interrupted") + c.log.Debug(logs.ControllerAnnouncementIsNotStartedOrAlreadyInterrupted) } } @@ -266,7 +267,7 @@ func (c *reporter) report(ctx context.Context) { // initialize iterator over locally accumulated announcements localIterator, err = c.ctrl.prm.AnnouncementAccumulator.InitIterator() if err != nil { - c.log.Debug("could not initialize iterator over locally accumulated announcements", + c.log.Debug(logs.ControllerCouldNotInitializeIteratorOverLocallyAccumulatedAnnouncements, zap.String("error", err.Error()), ) @@ -276,7 +277,7 @@ func (c *reporter) report(ctx context.Context) { // initialize final destination of load estimations resultWriter, err := c.ctrl.prm.ResultReceiver.InitWriter(nil) if err != nil { - c.log.Debug("could not initialize result target", + c.log.Debug(logs.ControllerCouldNotInitializeResultTarget, zap.String("error", err.Error()), ) @@ -289,7 +290,7 @@ func (c *reporter) report(ctx context.Context) { resultWriter.Put, ) if err != nil { - c.log.Debug("iterator over local announcements aborted", + c.log.Debug(logs.ControllerIteratorOverLocalAnnouncementsAborted, zap.String("error", err.Error()), ) @@ -299,7 +300,7 @@ func (c *reporter) report(ctx context.Context) { // finish writing err = resultWriter.Close(ctx) if err != nil { - c.log.Debug("could not finish writing load estimations", + c.log.Debug(logs.ControllerCouldNotFinishWritingLoadEstimations, zap.String("error", err.Error()), ) } diff --git a/pkg/services/container/announcement/load/route/calls.go b/pkg/services/container/announcement/load/route/calls.go index 83c368f57..9a483aed0 100644 --- a/pkg/services/container/announcement/load/route/calls.go +++ b/pkg/services/container/announcement/load/route/calls.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "go.uber.org/zap" @@ -97,7 +98,7 @@ func (w *loadWriter) Put(a container.SizeEstimation) error { if !ok { provider, err := w.router.remoteProvider.InitRemote(remoteInfo) if err != nil { - w.router.log.Debug("could not initialize writer provider", + w.router.log.Debug(logs.RouteCouldNotInitializeWriterProvider, zap.String("error", err.Error()), ) @@ -106,7 +107,7 @@ func (w *loadWriter) Put(a container.SizeEstimation) error { remoteWriter, err = provider.InitWriter(w.route) if err != nil { - w.router.log.Debug("could not initialize writer", + w.router.log.Debug(logs.RouteCouldNotInitializeWriter, zap.String("error", err.Error()), ) @@ -118,7 +119,7 @@ func (w *loadWriter) Put(a container.SizeEstimation) error { err := remoteWriter.Put(a) if err != nil { - w.router.log.Debug("could not put the value", + w.router.log.Debug(logs.RouteCouldNotPutTheValue, zap.String("error", err.Error()), ) } @@ -133,7 +134,7 @@ func (w *loadWriter) Close(ctx context.Context) error { for key, wRemote := range w.mServers { err := wRemote.Close(ctx) if err != nil { - w.router.log.Debug("could not close remote server writer", + w.router.log.Debug(logs.RouteCouldNotCloseRemoteServerWriter, zap.String("key", key), zap.String("error", err.Error()), ) diff --git a/pkg/services/notificator/nats/service.go b/pkg/services/notificator/nats/service.go index 54eb373ec..6a7e80a53 100644 --- a/pkg/services/notificator/nats/service.go +++ b/pkg/services/notificator/nats/service.go @@ -6,6 +6,7 @@ import ( "fmt" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nats-io/nats.go" @@ -98,10 +99,10 @@ func New(oo ...Option) *Writer { w.opts.nOpts = append(w.opts.nOpts, nats.NoCallbacksAfterClientClose(), // do not call callbacks when it was planned writer stop nats.DisconnectErrHandler(func(conn *nats.Conn, err error) { - w.log.Error("nats: connection was lost", zap.Error(err)) + w.log.Error(logs.NatsNatsConnectionWasLost, zap.Error(err)) }), nats.ReconnectHandler(func(conn *nats.Conn) { - w.log.Warn("nats: reconnected to the server") + w.log.Warn(logs.NatsNatsReconnectedToTheServer) }), ) @@ -124,7 +125,7 @@ func (n *Writer) Connect(ctx context.Context, endpoint string) error { go func() { <-ctx.Done() - n.opts.log.Info("nats: closing connection as the context is done") + n.opts.log.Info(logs.NatsNatsClosingConnectionAsTheContextIsDone) nc.Close() }() diff --git a/pkg/services/notificator/service.go b/pkg/services/notificator/service.go index 0a8a5d96d..bbf4e4823 100644 --- a/pkg/services/notificator/service.go +++ b/pkg/services/notificator/service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" @@ -74,10 +75,10 @@ func New(prm *Prm) *Notificator { // and passes their addresses to the NotificationWriter. func (n *Notificator) ProcessEpoch(ctx context.Context, epoch uint64) { logger := n.l.With(zap.Uint64("epoch", epoch)) - logger.Debug("notificator: start processing object notifications") + logger.Debug(logs.NotificatorNotificatorStartProcessingObjectNotifications) n.ns.Iterate(ctx, epoch, func(topic string, addr oid.Address) { - n.l.Debug("notificator: processing object notification", + n.l.Debug(logs.NotificatorNotificatorProcessingObjectNotification, zap.String("topic", topic), zap.Stringer("address", addr), ) diff --git a/pkg/services/object/acl/v2/classifier.go b/pkg/services/object/acl/v2/classifier.go index 2bf5a3958..cdc5fb623 100644 --- a/pkg/services/object/acl/v2/classifier.go +++ b/pkg/services/object/acl/v2/classifier.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" @@ -48,7 +49,7 @@ func (c senderClassifier) classify( isInnerRingNode, err := c.isInnerRingKey(ownerKeyInBytes) if err != nil { // do not throw error, try best case matching - c.log.Debug("can't check if request from inner ring", + c.log.Debug(logs.V2CantCheckIfRequestFromInnerRing, zap.String("error", err.Error())) } else if isInnerRingNode { return &classifyResult{ @@ -65,7 +66,7 @@ func (c senderClassifier) classify( // error might happen if request has `RoleOther` key and placement // is not possible for previous epoch, so // do not throw error, try best case matching - c.log.Debug("can't check if request from container node", + c.log.Debug(logs.V2CantCheckIfRequestFromContainerNode, zap.String("error", err.Error())) } else if isContainerNode { return &classifyResult{ diff --git a/pkg/services/object/delete/container.go b/pkg/services/object/delete/container.go index a2f099d5b..3106d8efd 100644 --- a/pkg/services/object/delete/container.go +++ b/pkg/services/object/delete/container.go @@ -1,5 +1,7 @@ package deletesvc +import "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + func (exec *execCtx) executeOnContainer() { - exec.log.Debug("request is not rolled over to the container") + exec.log.Debug(logs.DeleteRequestIsNotRolledOverToTheContainer) } diff --git a/pkg/services/object/delete/delete.go b/pkg/services/object/delete/delete.go index a959b53cb..971f0a6f5 100644 --- a/pkg/services/object/delete/delete.go +++ b/pkg/services/object/delete/delete.go @@ -3,6 +3,7 @@ package deletesvc import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "go.uber.org/zap" ) @@ -34,7 +35,7 @@ func (s *Service) Delete(ctx context.Context, prm Prm) error { } func (exec *execCtx) execute(ctx context.Context) { - exec.log.Debug("serving request...") + exec.log.Debug(logs.DeleteServingRequest) // perform local operation exec.executeLocal(ctx) @@ -46,9 +47,9 @@ func (exec *execCtx) analyzeStatus(execCnr bool) { // analyze local result switch exec.status { case statusOK: - exec.log.Debug("operation finished successfully") + exec.log.Debug(logs.DeleteOperationFinishedSuccessfully) default: - exec.log.Debug("operation finished with error", + exec.log.Debug(logs.DeleteOperationFinishedWithError, zap.String("error", exec.err.Error()), ) diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index 782cad71b..91bc6b3d7 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -5,6 +5,7 @@ import ( "strconv" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -83,7 +84,7 @@ func (exec *execCtx) formSplitInfo(ctx context.Context) bool { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not compose split info", + exec.log.Debug(logs.DeleteCouldNotComposeSplitInfo, zap.String("error", err.Error()), ) case err == nil: @@ -96,7 +97,7 @@ func (exec *execCtx) formSplitInfo(ctx context.Context) bool { func (exec *execCtx) collectMembers(ctx context.Context) (ok bool) { if exec.splitInfo == nil { - exec.log.Debug("no split info, object is PHY") + exec.log.Debug(logs.DeleteNoSplitInfoObjectIsPHY) return true } @@ -119,7 +120,7 @@ func (exec *execCtx) collectMembers(ctx context.Context) (ok bool) { func (exec *execCtx) collectChain(ctx context.Context) bool { var chain []oid.ID - exec.log.Debug("assembling chain...") + exec.log.Debug(logs.DeleteAssemblingChain) for prev, withPrev := exec.splitInfo.LastPart(); withPrev; { chain = append(chain, prev) @@ -131,7 +132,7 @@ func (exec *execCtx) collectChain(ctx context.Context) bool { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not get previous split element", + exec.log.Debug(logs.DeleteCouldNotGetPreviousSplitElement, zap.Stringer("id", prev), zap.String("error", err.Error()), ) @@ -154,7 +155,7 @@ func (exec *execCtx) collectChain(ctx context.Context) bool { } func (exec *execCtx) collectChildren(ctx context.Context) bool { - exec.log.Debug("collecting children...") + exec.log.Debug(logs.DeleteCollectingChildren) children, err := exec.svc.header.children(ctx, exec) @@ -163,7 +164,7 @@ func (exec *execCtx) collectChildren(ctx context.Context) bool { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not collect object children", + exec.log.Debug(logs.DeleteCouldNotCollectObjectChildren, zap.String("error", err.Error()), ) @@ -181,7 +182,7 @@ func (exec *execCtx) collectChildren(ctx context.Context) bool { } func (exec *execCtx) supplementBySplitID(ctx context.Context) bool { - exec.log.Debug("supplement by split ID") + exec.log.Debug(logs.DeleteSupplementBySplitID) chain, err := exec.svc.searcher.splitMembers(ctx, exec) @@ -190,7 +191,7 @@ func (exec *execCtx) supplementBySplitID(ctx context.Context) bool { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not search for split chain members", + exec.log.Debug(logs.DeleteCouldNotSearchForSplitChainMembers, zap.String("error", err.Error()), ) @@ -226,7 +227,7 @@ func (exec *execCtx) initTombstoneObject() bool { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not marshal tombstone structure", + exec.log.Debug(logs.DeleteCouldNotMarshalTombstoneStructure, zap.String("error", err.Error()), ) @@ -265,7 +266,7 @@ func (exec *execCtx) saveTombstone(ctx context.Context) bool { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not save the tombstone", + exec.log.Debug(logs.DeleteCouldNotSaveTheTombstone, zap.String("error", err.Error()), ) diff --git a/pkg/services/object/delete/local.go b/pkg/services/object/delete/local.go index 17eb0e4e1..34839b194 100644 --- a/pkg/services/object/delete/local.go +++ b/pkg/services/object/delete/local.go @@ -3,20 +3,21 @@ package deletesvc import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) func (exec *execCtx) executeLocal(ctx context.Context) { - exec.log.Debug("forming tombstone structure...") + exec.log.Debug(logs.DeleteFormingTombstoneStructure) ok := exec.formTombstone(ctx) if !ok { return } - exec.log.Debug("tombstone structure successfully formed, saving...") + exec.log.Debug(logs.DeleteTombstoneStructureSuccessfullyFormedSaving) exec.saveTombstone(ctx) } @@ -27,7 +28,7 @@ func (exec *execCtx) formTombstone(ctx context.Context) (ok bool) { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not read tombstone lifetime config", + exec.log.Debug(logs.DeleteCouldNotReadTombstoneLifetimeConfig, zap.String("error", err.Error()), ) @@ -40,14 +41,14 @@ func (exec *execCtx) formTombstone(ctx context.Context) (ok bool) { ) exec.addMembers([]oid.ID{exec.address().Object()}) - exec.log.Debug("forming split info...") + exec.log.Debug(logs.DeleteFormingSplitInfo) ok = exec.formSplitInfo(ctx) if !ok { return } - exec.log.Debug("split info successfully formed, collecting members...") + exec.log.Debug(logs.DeleteSplitInfoSuccessfullyFormedCollectingMembers) exec.tombstone.SetSplitID(exec.splitInfo.SplitID()) @@ -56,7 +57,7 @@ func (exec *execCtx) formTombstone(ctx context.Context) (ok bool) { return } - exec.log.Debug("members successfully collected") + exec.log.Debug(logs.DeleteMembersSuccessfullyCollected) ok = exec.initTombstoneObject() if !ok { diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index db71df6a4..d2108b003 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -12,7 +13,7 @@ import ( func (exec *execCtx) assemble(ctx context.Context) { if !exec.canAssemble() { - exec.log.Debug("can not assemble the object") + exec.log.Debug(logs.GetCanNotAssembleTheObject) return } @@ -34,16 +35,16 @@ func (exec *execCtx) assemble(ctx context.Context) { // `execCtx` so it should be disabled there. exec.disableForwarding() - exec.log.Debug("trying to assemble the object...") + exec.log.Debug(logs.GetTryingToAssembleTheObject) assembler := newAssembler(exec.address(), exec.splitInfo(), exec.ctxRange(), exec) - exec.log.Debug("assembling splitted object...", + exec.log.Debug(logs.GetAssemblingSplittedObject, zap.Stringer("address", exec.address()), zap.Uint64("range_offset", exec.ctxRange().GetOffset()), zap.Uint64("range_length", exec.ctxRange().GetLength()), ) - defer exec.log.Debug("assembling splitted object completed", + defer exec.log.Debug(logs.GetAssemblingSplittedObjectCompleted, zap.Stringer("address", exec.address()), zap.Uint64("range_offset", exec.ctxRange().GetOffset()), zap.Uint64("range_length", exec.ctxRange().GetLength()), @@ -51,7 +52,7 @@ func (exec *execCtx) assemble(ctx context.Context) { obj, err := assembler.Assemble(ctx, exec.prm.objWriter) if err != nil { - exec.log.Warn("failed to assemble splitted object", + exec.log.Warn(logs.GetFailedToAssembleSplittedObject, zap.Error(err), zap.Stringer("address", exec.address()), zap.Uint64("range_offset", exec.ctxRange().GetOffset()), diff --git a/pkg/services/object/get/container.go b/pkg/services/object/get/container.go index cfb538d38..74d63966e 100644 --- a/pkg/services/object/get/container.go +++ b/pkg/services/object/get/container.go @@ -3,19 +3,20 @@ package getsvc import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "go.uber.org/zap" ) func (exec *execCtx) executeOnContainer(ctx context.Context) { if exec.isLocal() { - exec.log.Debug("return result directly") + exec.log.Debug(logs.GetReturnResultDirectly) return } lookupDepth := exec.netmapLookupDepth() - exec.log.Debug("trying to execute in container...", + exec.log.Debug(logs.GetTryingToExecuteInContainer, zap.Uint64("netmap lookup depth", lookupDepth), ) @@ -43,7 +44,7 @@ func (exec *execCtx) executeOnContainer(ctx context.Context) { } func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { - exec.log.Debug("process epoch", + exec.log.Debug(logs.GetProcessEpoch, zap.Uint64("number", exec.curProcEpoch), ) @@ -60,7 +61,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { for { addrs := traverser.Next() if len(addrs) == 0 { - exec.log.Debug("no more nodes, abort placement iteration") + exec.log.Debug(logs.GetNoMoreNodesAbortPlacementIteration) return false } @@ -68,7 +69,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { for i := range addrs { select { case <-ctx.Done(): - exec.log.Debug("interrupt placement iteration by context", + exec.log.Debug(logs.GetInterruptPlacementIterationByContext, zap.String("error", ctx.Err().Error()), ) @@ -84,7 +85,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { client.NodeInfoFromNetmapElement(&info, addrs[i]) if exec.processNode(ctx, info) { - exec.log.Debug("completing the operation") + exec.log.Debug(logs.GetCompletingTheOperation) return true } } diff --git a/pkg/services/object/get/exec.go b/pkg/services/object/get/exec.go index 2ba014574..7f090dd50 100644 --- a/pkg/services/object/get/exec.go +++ b/pkg/services/object/get/exec.go @@ -4,6 +4,7 @@ import ( "context" "crypto/ecdsa" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" @@ -149,7 +150,7 @@ func (exec *execCtx) initEpoch() bool { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not get current epoch number", + exec.log.Debug(logs.GetCouldNotGetCurrentEpochNumber, zap.String("error", err.Error()), ) @@ -170,7 +171,7 @@ func (exec *execCtx) generateTraverser(addr oid.Address) (*placement.Traverser, exec.status = statusUndefined exec.err = err - exec.log.Debug("could not generate container traverser", + exec.log.Debug(logs.GetCouldNotGenerateContainerTraverser, zap.String("error", err.Error()), ) @@ -188,7 +189,7 @@ func (exec execCtx) remoteClient(info clientcore.NodeInfo) (getClient, bool) { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not construct remote node client") + exec.log.Debug(logs.GetCouldNotConstructRemoteNodeClient) case err == nil: return c, true } @@ -225,7 +226,7 @@ func (exec *execCtx) writeCollectedHeader(ctx context.Context) bool { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not write header", + exec.log.Debug(logs.GetCouldNotWriteHeader, zap.String("error", err.Error()), ) case err == nil: @@ -248,7 +249,7 @@ func (exec *execCtx) writeObjectPayload(ctx context.Context, obj *objectSDK.Obje exec.status = statusUndefined exec.err = err - exec.log.Debug("could not write payload chunk", + exec.log.Debug(logs.GetCouldNotWritePayloadChunk, zap.String("error", err.Error()), ) case err == nil: diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 0f5983e99..bb0d669da 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -3,6 +3,7 @@ package getsvc import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" @@ -83,7 +84,7 @@ func (s *Service) get(ctx context.Context, prm commonPrm, opts ...execOption) st } func (exec *execCtx) execute(ctx context.Context) { - exec.log.Debug("serving request...") + exec.log.Debug(logs.GetServingRequest) // perform local operation exec.executeLocal(ctx) @@ -95,16 +96,16 @@ func (exec *execCtx) analyzeStatus(ctx context.Context, execCnr bool) { // analyze local result switch exec.status { case statusOK: - exec.log.Debug("operation finished successfully") + exec.log.Debug(logs.GetOperationFinishedSuccessfully) case statusINHUMED: - exec.log.Debug("requested object was marked as removed") + exec.log.Debug(logs.GetRequestedObjectWasMarkedAsRemoved) case statusVIRTUAL: - exec.log.Debug("requested object is virtual") + exec.log.Debug(logs.GetRequestedObjectIsVirtual) exec.assemble(ctx) case statusOutOfRange: - exec.log.Debug("requested range is out of object bounds") + exec.log.Debug(logs.GetRequestedRangeIsOutOfObjectBounds) default: - exec.log.Debug("operation finished with error", + exec.log.Debug(logs.GetOperationFinishedWithError, zap.String("error", exec.err.Error()), ) diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index 82ed911e4..8ac83d97a 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -5,6 +5,7 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" @@ -29,7 +30,7 @@ func (exec *execCtx) executeLocal(ctx context.Context) { exec.status = statusUndefined exec.err = err - exec.log.Debug("local get failed", + exec.log.Debug(logs.GetLocalGetFailed, zap.String("error", err.Error()), ) case err == nil: diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index 697e48ee2..f4f74083b 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -5,6 +5,7 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -15,7 +16,7 @@ func (exec *execCtx) processNode(ctx context.Context, info client.NodeInfo) bool ctx, span := tracing.StartSpanFromContext(ctx, "getService.processNode") defer span.End() - exec.log.Debug("processing node...") + exec.log.Debug(logs.GetProcessingNode) client, ok := exec.remoteClient(info) if !ok { @@ -35,7 +36,7 @@ func (exec *execCtx) processNode(ctx context.Context, info client.NodeInfo) bool exec.status = statusUndefined exec.err = errNotFound - exec.log.Debug("remote call failed", + exec.log.Debug(logs.GetRemoteCallFailed, zap.String("error", err.Error()), ) case err == nil: diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index d8b59487e..86dc3c2ca 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -6,6 +6,7 @@ import ( "sync" "sync/atomic" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" @@ -198,7 +199,7 @@ func (t *distributedTarget) iteratePlacement(ctx context.Context) (*transformer. if t.traversal.submitPrimaryPlacementFinish() { _, err = t.iteratePlacement(ctx) if err != nil { - t.log.Error("additional container broadcast failure", zap.Error(err)) + t.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) // we don't fail primary operation because of broadcast failure } } diff --git a/pkg/services/object/search/container.go b/pkg/services/object/search/container.go index b158bc23e..9df438e00 100644 --- a/pkg/services/object/search/container.go +++ b/pkg/services/object/search/container.go @@ -5,19 +5,20 @@ import ( "encoding/hex" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "go.uber.org/zap" ) func (exec *execCtx) executeOnContainer(ctx context.Context) { if exec.isLocal() { - exec.log.Debug("return result directly") + exec.log.Debug(logs.SearchReturnResultDirectly) return } lookupDepth := exec.netmapLookupDepth() - exec.log.Debug("trying to execute in container...", + exec.log.Debug(logs.SearchTryingToExecuteInContainer, zap.Uint64("netmap lookup depth", lookupDepth), ) @@ -48,7 +49,7 @@ func (exec *execCtx) executeOnContainer(ctx context.Context) { } func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { - exec.log.Debug("process epoch", + exec.log.Debug(logs.SearchProcessEpoch, zap.Uint64("number", exec.curProcEpoch), ) @@ -63,7 +64,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { for { addrs := traverser.Next() if len(addrs) == 0 { - exec.log.Debug("no more nodes, abort placement iteration") + exec.log.Debug(logs.SearchNoMoreNodesAbortPlacementIteration) break } @@ -76,7 +77,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { defer wg.Done() select { case <-ctx.Done(): - exec.log.Debug("interrupt placement iteration by context", + exec.log.Debug(logs.SearchInterruptPlacementIterationByContext, zap.String("error", ctx.Err().Error())) return default: @@ -86,7 +87,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { client.NodeInfoFromNetmapElement(&info, addrs[i]) - exec.log.Debug("processing node...", zap.String("key", hex.EncodeToString(addrs[i].PublicKey()))) + exec.log.Debug(logs.SearchProcessingNode, zap.String("key", hex.EncodeToString(addrs[i].PublicKey()))) c, err := exec.svc.clientConstructor.get(info) if err != nil { @@ -95,13 +96,13 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { exec.err = err mtx.Unlock() - exec.log.Debug("could not construct remote node client") + exec.log.Debug(logs.SearchCouldNotConstructRemoteNodeClient) return } ids, err := c.searchObjects(ctx, exec, info) if err != nil { - exec.log.Debug("remote operation failed", + exec.log.Debug(logs.SearchRemoteOperationFailed, zap.String("error", err.Error())) return diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index f815270d9..1733d7840 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -1,6 +1,7 @@ package searchsvc import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -80,7 +81,7 @@ func (exec *execCtx) initEpoch() bool { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not get current epoch number", + exec.log.Debug(logs.SearchCouldNotGetCurrentEpochNumber, zap.String("error", err.Error()), ) @@ -99,7 +100,7 @@ func (exec *execCtx) generateTraverser(cnr cid.ID) (*placement.Traverser, bool) exec.status = statusUndefined exec.err = err - exec.log.Debug("could not generate container traverser", + exec.log.Debug(logs.SearchCouldNotGenerateContainerTraverser, zap.String("error", err.Error()), ) @@ -118,7 +119,7 @@ func (exec *execCtx) writeIDList(ids []oid.ID) { exec.status = statusUndefined exec.err = err - exec.log.Debug("could not write object identifiers", + exec.log.Debug(logs.SearchCouldNotWriteObjectIdentifiers, zap.String("error", err.Error()), ) case err == nil: diff --git a/pkg/services/object/search/local.go b/pkg/services/object/search/local.go index 1e4776921..f768c8861 100644 --- a/pkg/services/object/search/local.go +++ b/pkg/services/object/search/local.go @@ -1,6 +1,7 @@ package searchsvc import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "go.uber.org/zap" ) @@ -11,7 +12,7 @@ func (exec *execCtx) executeLocal() { exec.status = statusUndefined exec.err = err - exec.log.Debug("local operation failed", + exec.log.Debug(logs.SearchLocalOperationFailed, zap.String("error", err.Error()), ) diff --git a/pkg/services/object/search/search.go b/pkg/services/object/search/search.go index 325b42a54..5bf0710ad 100644 --- a/pkg/services/object/search/search.go +++ b/pkg/services/object/search/search.go @@ -3,6 +3,7 @@ package searchsvc import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "go.uber.org/zap" ) @@ -23,7 +24,7 @@ func (s *Service) Search(ctx context.Context, prm Prm) error { } func (exec *execCtx) execute(ctx context.Context) { - exec.log.Debug("serving request...") + exec.log.Debug(logs.SearchServingRequest) // perform local operation exec.executeLocal() @@ -35,11 +36,11 @@ func (exec *execCtx) analyzeStatus(ctx context.Context, execCnr bool) { // analyze local result switch exec.status { default: - exec.log.Debug("operation finished with error", + exec.log.Debug(logs.SearchOperationFinishedWithError, zap.String("error", exec.err.Error()), ) case statusOK: - exec.log.Debug("operation finished successfully") + exec.log.Debug(logs.SearchOperationFinishedSuccessfully) } if execCnr { diff --git a/pkg/services/object/util/log.go b/pkg/services/object/util/log.go index beda45c0c..92beedaa7 100644 --- a/pkg/services/object/util/log.go +++ b/pkg/services/object/util/log.go @@ -1,6 +1,7 @@ package util import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" @@ -8,7 +9,7 @@ import ( // LogServiceError writes error message of object service to provided logger. func LogServiceError(l *logger.Logger, req string, node network.AddressGroup, err error) { - l.Error("object service error", + l.Error(logs.UtilObjectServiceError, zap.String("node", network.StringifyGroup(node)), zap.String("request", req), zap.String("error", err.Error()), @@ -17,7 +18,7 @@ func LogServiceError(l *logger.Logger, req string, node network.AddressGroup, er // LogWorkerPoolError writes debug error message of object worker pool to provided logger. func LogWorkerPoolError(l *logger.Logger, req string, err error) { - l.Error("could not push task to worker pool", + l.Error(logs.UtilCouldNotPushTaskToWorkerPool, zap.String("request", req), zap.String("error", err.Error()), ) diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go index 4097f22bf..46fcc9840 100644 --- a/pkg/services/object_manager/tombstone/checker.go +++ b/pkg/services/object_manager/tombstone/checker.go @@ -5,6 +5,7 @@ import ( "strconv" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -57,7 +58,7 @@ func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Addr ts, err := g.tsSource.Tombstone(ctx, a, epoch) if err != nil { log.Warn( - "tombstone getter: could not get the tombstone the source", + logs.TombstoneCouldNotGetTheTombstoneTheSource, zap.Error(err), ) } else { diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 9cdc4d813..e91b8871b 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" @@ -73,7 +74,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add cnr, err := p.cnrSrc.Get(idCnr) if err != nil { - p.log.Error("could not get container", + p.log.Error(logs.PolicerCouldNotGetContainer, zap.Stringer("cid", idCnr), zap.String("error", err.Error()), ) @@ -84,7 +85,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add _, err := p.jobQueue.localStorage.Inhume(ctx, prm) if err != nil { - p.log.Error("could not inhume object with missing container", + p.log.Error(logs.PolicerCouldNotInhumeObjectWithMissingContainer, zap.Stringer("cid", idCnr), zap.Stringer("oid", idObj), zap.String("error", err.Error())) @@ -98,7 +99,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add nn, err := p.placementBuilder.BuildPlacement(idCnr, &idObj, policy) if err != nil { - p.log.Error("could not build placement vector for object", + p.log.Error(logs.PolicerCouldNotBuildPlacementVectorForObject, zap.Stringer("cid", idCnr), zap.String("error", err.Error()), ) @@ -127,7 +128,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add } if !c.needLocalCopy && c.removeLocalCopy { - p.log.Info("redundant local object copy detected", + p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, zap.Stringer("object", addr), ) @@ -199,7 +200,7 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi if isClientErrMaintenance(err) { shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) } else if err != nil { - p.log.Error("receive object header to check policy compliance", + p.log.Error(logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, zap.Stringer("object", addr), zap.String("error", err.Error()), ) @@ -228,7 +229,7 @@ func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes *nodeCach shortage-- uncheckedCopies++ - p.log.Debug("consider node under maintenance as OK", + p.log.Debug(logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(node)), ) return shortage, uncheckedCopies @@ -237,7 +238,7 @@ func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes *nodeCach func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address, requirements *placementRequirements, nodes []netmap.NodeInfo, checkedNodes *nodeCache, shortage uint32, uncheckedCopies int) { if shortage > 0 { - p.log.Debug("shortage of object copies detected", + p.log.Debug(logs.PolicerShortageOfObjectCopiesDetected, zap.Stringer("object", addr), zap.Uint32("shortage", shortage), ) @@ -251,7 +252,7 @@ func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address } else if uncheckedCopies > 0 { // If we have more copies than needed, but some of them are from the maintenance nodes, // save the local copy. - p.log.Debug("some of the copies are stored on nodes under maintenance, save local copy", + p.log.Debug(logs.PolicerSomeOfTheCopiesAreStoredOnNodesUnderMaintenance, zap.Int("count", uncheckedCopies)) } else if uncheckedCopies == 0 { // Safe to remove: checked all copies, shortage == 0. diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 687216407..4a40f00ba 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -5,6 +5,7 @@ import ( "errors" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "go.uber.org/zap" @@ -12,7 +13,7 @@ import ( func (p *Policer) Run(ctx context.Context) { defer func() { - p.log.Info("routine stopped") + p.log.Info(logs.PolicerRoutineStopped) }() go p.poolCapacityWorker(ctx) @@ -39,7 +40,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { time.Sleep(time.Second) // finished whole cycle, sleep a bit continue } - p.log.Warn("failure at object select for replication", zap.Error(err)) + p.log.Warn(logs.PolicerFailureAtObjectSelectForReplication, zap.Error(err)) } for i := range addrs { @@ -68,7 +69,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { p.objsInWork.remove(addr.Address) }) if err != nil { - p.log.Warn("pool submission", zap.Error(err)) + p.log.Warn(logs.PolicerPoolSubmission, zap.Error(err)) } } } @@ -91,7 +92,7 @@ func (p *Policer) poolCapacityWorker(ctx context.Context) { if p.taskPool.Cap() != newCapacity { p.taskPool.Tune(newCapacity) - p.log.Debug("tune replication capacity", + p.log.Debug(logs.PolicerTuneReplicationCapacity, zap.Float64("system_load", frostfsSysLoad), zap.Int("new_capacity", newCapacity)) } diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 53df81b77..46e0c9468 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -3,6 +3,7 @@ package replicator import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -20,7 +21,7 @@ type TaskResult interface { // Passes all the nodes that accepted the replication to the TaskResult. func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) { defer func() { - p.log.Debug("finish work", + p.log.Debug(logs.ReplicatorFinishWork, zap.Uint32("amount of unfinished replicas", task.quantity), ) }() @@ -29,7 +30,7 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) var err error task.obj, err = engine.Get(ctx, p.localStorage, task.addr) if err != nil { - p.log.Error("could not get object from local storage", + p.log.Error(logs.ReplicatorCouldNotGetObjectFromLocalStorage, zap.Stringer("object", task.addr), zap.Error(err)) @@ -59,11 +60,11 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) cancel() if err != nil { - log.Error("could not replicate object", + log.Error(logs.ReplicatorCouldNotReplicateObject, zap.String("error", err.Error()), ) } else { - log.Debug("object successfully replicated") + log.Debug(logs.ReplicatorObjectSuccessfullyReplicated) task.quantity-- diff --git a/pkg/services/reputation/common/managers.go b/pkg/services/reputation/common/managers.go index ef11b8122..84201809f 100644 --- a/pkg/services/reputation/common/managers.go +++ b/pkg/services/reputation/common/managers.go @@ -3,6 +3,7 @@ package common import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apiNetmap "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -76,7 +77,7 @@ func (x nodeServer) ExternalAddresses() []string { // BuildManagers sorts nodes in NetMap with HRW algorithms and // takes the next node after the current one as the only manager. func (mb *managerBuilder) BuildManagers(epoch uint64, p apireputation.PeerID) ([]ServerInfo, error) { - mb.log.Debug("start building managers", + mb.log.Debug(logs.CommonStartBuildingManagers, zap.Uint64("epoch", epoch), zap.Stringer("peer", p), ) diff --git a/pkg/services/reputation/common/router/calls.go b/pkg/services/reputation/common/router/calls.go index a177f6a2b..4ed293beb 100644 --- a/pkg/services/reputation/common/router/calls.go +++ b/pkg/services/reputation/common/router/calls.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" "go.uber.org/zap" @@ -92,7 +93,7 @@ func (w *trustWriter) Write(ctx context.Context, t reputation.Trust) error { if !ok { provider, err := w.router.remoteProvider.InitRemote(remoteInfo) if err != nil { - w.router.log.Debug("could not initialize writer provider", + w.router.log.Debug(logs.RouterCouldNotInitializeWriterProvider, zap.String("error", err.Error()), ) @@ -102,7 +103,7 @@ func (w *trustWriter) Write(ctx context.Context, t reputation.Trust) error { // init writer with original context wrapped in routeContext remoteWriter, err = provider.InitWriter(w.routeInfo.EpochProvider) if err != nil { - w.router.log.Debug("could not initialize writer", + w.router.log.Debug(logs.RouterCouldNotInitializeWriter, zap.String("error", err.Error()), ) @@ -114,7 +115,7 @@ func (w *trustWriter) Write(ctx context.Context, t reputation.Trust) error { err := remoteWriter.Write(ctx, t) if err != nil { - w.router.log.Debug("could not write the value", + w.router.log.Debug(logs.RouterCouldNotWriteTheValue, zap.String("error", err.Error()), ) } @@ -127,7 +128,7 @@ func (w *trustWriter) Close(ctx context.Context) error { for key, wRemote := range w.mServers { err := wRemote.Close(ctx) if err != nil { - w.router.log.Debug("could not close remote server writer", + w.router.log.Debug(logs.RouterCouldNotCloseRemoteServerWriter, zap.String("key", key), zap.String("error", err.Error()), ) diff --git a/pkg/services/reputation/eigentrust/calculator/calls.go b/pkg/services/reputation/eigentrust/calculator/calls.go index a8e5cf1da..5e2e900ae 100644 --- a/pkg/services/reputation/eigentrust/calculator/calls.go +++ b/pkg/services/reputation/eigentrust/calculator/calls.go @@ -3,6 +3,7 @@ package eigentrustcalc import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" @@ -27,7 +28,7 @@ func (c *Calculator) Calculate(ctx context.Context, prm CalculatePrm) { alpha, err := c.prm.AlphaProvider.EigenTrustAlpha() if err != nil { c.opts.log.Debug( - "failed to get alpha param", + logs.CalculatorFailedToGetAlphaParam, zap.Error(err), ) return @@ -56,7 +57,7 @@ func (c *Calculator) Calculate(ctx context.Context, prm CalculatePrm) { consumersIter, err := c.prm.DaughterTrustSource.InitConsumersIterator(epochIteration) if err != nil { - log.Debug("consumers trust iterator's init failure", + log.Debug(logs.CalculatorConsumersTrustIteratorsInitFailure, zap.String("error", err.Error()), ) @@ -76,7 +77,7 @@ func (c *Calculator) Calculate(ctx context.Context, prm CalculatePrm) { }) }) if err != nil { - log.Debug("worker pool submit failure", + log.Debug(logs.CalculatorWorkerPoolSubmitFailure, zap.String("error", err.Error()), ) } @@ -85,7 +86,7 @@ func (c *Calculator) Calculate(ctx context.Context, prm CalculatePrm) { return nil }) if err != nil { - log.Debug("iterate daughter's consumers failed", + log.Debug(logs.CalculatorIterateDaughtersConsumersFailed, zap.String("error", err.Error()), ) } @@ -104,7 +105,7 @@ type iterDaughterPrm struct { func (c *Calculator) iterateDaughter(ctx context.Context, p iterDaughterPrm) { initTrust, err := c.prm.InitialTrustSource.InitialTrust(p.id) if err != nil { - c.opts.log.Debug("get initial trust failure", + c.opts.log.Debug(logs.CalculatorGetInitialTrustFailure, zap.Stringer("daughter", p.id), zap.String("error", err.Error()), ) @@ -114,7 +115,7 @@ func (c *Calculator) iterateDaughter(ctx context.Context, p iterDaughterPrm) { daughterIter, err := c.prm.DaughterTrustSource.InitDaughterIterator(p.ei, p.id) if err != nil { - c.opts.log.Debug("daughter trust iterator's init failure", + c.opts.log.Debug(logs.CalculatorDaughterTrustIteratorsInitFailure, zap.String("error", err.Error()), ) @@ -136,7 +137,7 @@ func (c *Calculator) iterateDaughter(ctx context.Context, p iterDaughterPrm) { return nil }) if err != nil { - c.opts.log.Debug("iterate over daughter's trusts failure", + c.opts.log.Debug(logs.CalculatorIterateOverDaughtersTrustsFailure, zap.String("error", err.Error()), ) @@ -165,7 +166,7 @@ func (c *Calculator) iterateDaughter(ctx context.Context, p iterDaughterPrm) { func (c *Calculator) processLastIteration(p iterDaughterPrm, intermediateTrust eigentrust.IterationTrust, sum reputation.TrustValue) { finalWriter, err := c.prm.FinalResultTarget.InitIntermediateWriter(p.ei) if err != nil { - c.opts.log.Debug("init writer failure", + c.opts.log.Debug(logs.CalculatorInitWriterFailure, zap.String("error", err.Error()), ) @@ -176,7 +177,7 @@ func (c *Calculator) processLastIteration(p iterDaughterPrm, intermediateTrust e err = finalWriter.WriteIntermediateTrust(intermediateTrust) if err != nil { - c.opts.log.Debug("write final result failure", + c.opts.log.Debug(logs.CalculatorWriteFinalResultFailure, zap.String("error", err.Error()), ) @@ -187,7 +188,7 @@ func (c *Calculator) processLastIteration(p iterDaughterPrm, intermediateTrust e func (c *Calculator) processIntermediateIteration(ctx context.Context, p iterDaughterPrm, daughterIter TrustIterator, sum reputation.TrustValue) { intermediateWriter, err := c.prm.IntermediateValueTarget.InitWriter(p.ei) if err != nil { - c.opts.log.Debug("init writer failure", + c.opts.log.Debug(logs.CalculatorInitWriterFailure, zap.String("error", err.Error()), ) @@ -208,7 +209,7 @@ func (c *Calculator) processIntermediateIteration(ctx context.Context, p iterDau err := intermediateWriter.Write(ctx, trust) if err != nil { - c.opts.log.Debug("write value failure", + c.opts.log.Debug(logs.CalculatorWriteValueFailure, zap.String("error", err.Error()), ) } @@ -216,7 +217,7 @@ func (c *Calculator) processIntermediateIteration(ctx context.Context, p iterDau return nil }) if err != nil { - c.opts.log.Debug("iterate daughter trusts failure", + c.opts.log.Debug(logs.CalculatorIterateDaughterTrustsFailure, zap.String("error", err.Error()), ) } @@ -233,7 +234,7 @@ func (c *Calculator) processIntermediateIteration(ctx context.Context, p iterDau func (c *Calculator) sendInitialValues(ctx context.Context, epochInfo EpochIterationInfo) { daughterIter, err := c.prm.DaughterTrustSource.InitAllDaughtersIterator(epochInfo) if err != nil { - c.opts.log.Debug("all daughters trust iterator's init failure", + c.opts.log.Debug(logs.CalculatorAllDaughtersTrustIteratorsInitFailure, zap.String("error", err.Error()), ) @@ -242,7 +243,7 @@ func (c *Calculator) sendInitialValues(ctx context.Context, epochInfo EpochItera intermediateWriter, err := c.prm.IntermediateValueTarget.InitWriter(epochInfo) if err != nil { - c.opts.log.Debug("init writer failure", + c.opts.log.Debug(logs.CalculatorInitWriterFailure, zap.String("error", err.Error()), ) @@ -255,7 +256,7 @@ func (c *Calculator) sendInitialValues(ctx context.Context, epochInfo EpochItera initTrust, err := c.prm.InitialTrustSource.InitialTrust(trusted) if err != nil { - c.opts.log.Debug("get initial trust failure", + c.opts.log.Debug(logs.CalculatorGetInitialTrustFailure, zap.Stringer("peer", trusted), zap.String("error", err.Error()), ) @@ -269,7 +270,7 @@ func (c *Calculator) sendInitialValues(ctx context.Context, epochInfo EpochItera err = intermediateWriter.Write(ctx, trust) if err != nil { - c.opts.log.Debug("write value failure", + c.opts.log.Debug(logs.CalculatorWriteValueFailure, zap.String("error", err.Error()), ) @@ -280,14 +281,14 @@ func (c *Calculator) sendInitialValues(ctx context.Context, epochInfo EpochItera }) }) if err != nil { - c.opts.log.Debug("iterate over all daughters failure", + c.opts.log.Debug(logs.CalculatorIterateOverAllDaughtersFailure, zap.String("error", err.Error()), ) } err = intermediateWriter.Close(ctx) if err != nil { - c.opts.log.Debug("could not close writer", + c.opts.log.Debug(logs.CalculatorCouldNotCloseWriter, zap.String("error", err.Error()), ) } diff --git a/pkg/services/reputation/eigentrust/controller/calls.go b/pkg/services/reputation/eigentrust/controller/calls.go index 1753a430b..886daf9be 100644 --- a/pkg/services/reputation/eigentrust/controller/calls.go +++ b/pkg/services/reputation/eigentrust/controller/calls.go @@ -3,6 +3,7 @@ package eigentrustctrl import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" "go.uber.org/zap" ) @@ -37,7 +38,7 @@ func (c *Controller) Continue(ctx context.Context, prm ContinuePrm) { iterations, err := c.prm.IterationsProvider.EigenTrustIterations() if err != nil { - c.opts.log.Error("could not get EigenTrust iteration number", + c.opts.log.Error(logs.ControllerCouldNotGetEigenTrustIterationNumber, zap.Error(err), ) } else { @@ -54,7 +55,7 @@ func (c *Controller) Continue(ctx context.Context, prm ContinuePrm) { iterCtx.Increment() }) if err != nil { - c.opts.log.Debug("iteration submit failure", + c.opts.log.Debug(logs.ControllerIterationSubmitFailure, zap.String("error", err.Error()), ) } diff --git a/pkg/services/reputation/eigentrust/routes/calls.go b/pkg/services/reputation/eigentrust/routes/calls.go index c4d9688a9..ccb2fe8ea 100644 --- a/pkg/services/reputation/eigentrust/routes/calls.go +++ b/pkg/services/reputation/eigentrust/routes/calls.go @@ -3,6 +3,7 @@ package routes import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" "go.uber.org/zap" @@ -14,7 +15,7 @@ import ( func (b *Builder) NextStage(epoch uint64, t reputation.Trust, passed []common.ServerInfo) ([]common.ServerInfo, error) { passedLen := len(passed) - b.log.Debug("building next stage for trust route", + b.log.Debug(logs.RoutesBuildingNextStageForTrustRoute, zap.Uint64("epoch", epoch), zap.Int("passed_length", passedLen), ) diff --git a/pkg/services/reputation/local/controller/calls.go b/pkg/services/reputation/local/controller/calls.go index 80fa772d6..1cad09313 100644 --- a/pkg/services/reputation/local/controller/calls.go +++ b/pkg/services/reputation/local/controller/calls.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -77,7 +78,7 @@ func (c *Controller) acquireReporter(ctx context.Context, epoch uint64) (context )} if started { - log.Debug("report is already started") + log.Debug(logs.ControllerReportIsAlreadyStarted) return ctx, nil } @@ -92,12 +93,12 @@ func (c *Controller) acquireReporter(ctx context.Context, epoch uint64) (context } func (c *reporter) report(ctx context.Context) { - c.log.Debug("starting to report local trust values") + c.log.Debug(logs.ControllerStartingToReportLocalTrustValues) // initialize iterator over locally collected values iterator, err := c.ctrl.prm.LocalTrustSource.InitIterator(c.ep) if err != nil { - c.log.Debug("could not initialize iterator over local trust values", + c.log.Debug(logs.ControllerCouldNotInitializeIteratorOverLocalTrustValues, zap.String("error", err.Error()), ) @@ -107,7 +108,7 @@ func (c *reporter) report(ctx context.Context) { // initialize target of local trust values targetWriter, err := c.ctrl.prm.LocalTrustTarget.InitWriter(c.ep) if err != nil { - c.log.Debug("could not initialize local trust target", + c.log.Debug(logs.ControllerCouldNotInitializeLocalTrustTarget, zap.String("error", err.Error()), ) @@ -126,7 +127,7 @@ func (c *reporter) report(ctx context.Context) { }, ) if err != nil && !errors.Is(err, context.Canceled) { - c.log.Debug("iterator over local trust failed", + c.log.Debug(logs.ControllerIteratorOverLocalTrustFailed, zap.String("error", err.Error()), ) @@ -136,14 +137,14 @@ func (c *reporter) report(ctx context.Context) { // finish writing err = targetWriter.Close(ctx) if err != nil { - c.log.Debug("could not finish writing local trust values", + c.log.Debug(logs.ControllerCouldNotFinishWritingLocalTrustValues, zap.String("error", err.Error()), ) return } - c.log.Debug("reporting successfully finished") + c.log.Debug(logs.ControllerReportingSuccessfullyFinished) } func (c *Controller) freeReport(epoch uint64, log *logger.Logger) { @@ -165,9 +166,9 @@ func (c *Controller) freeReport(epoch uint64, log *logger.Logger) { c.mtx.Unlock() if stopped { - log.Debug("reporting successfully interrupted") + log.Debug(logs.ControllerReportingSuccessfullyInterrupted) } else { - log.Debug("reporting is not started or already interrupted") + log.Debug(logs.ControllerReportingIsNotStartedOrAlreadyInterrupted) } } diff --git a/pkg/services/reputation/local/routes/calls.go b/pkg/services/reputation/local/routes/calls.go index f0eae16fe..2f99f0e10 100644 --- a/pkg/services/reputation/local/routes/calls.go +++ b/pkg/services/reputation/local/routes/calls.go @@ -3,6 +3,7 @@ package routes import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" "go.uber.org/zap" @@ -14,7 +15,7 @@ import ( func (b *Builder) NextStage(epoch uint64, t reputation.Trust, passed []common.ServerInfo) ([]common.ServerInfo, error) { passedLen := len(passed) - b.log.Debug("building next stage for local trust route", + b.log.Debug(logs.RoutesBuildingNextStageForLocalTrustRoute, zap.Uint64("epoch", epoch), zap.Int("passed_length", passedLen), ) diff --git a/pkg/services/session/executor.go b/pkg/services/session/executor.go index 237a13962..0be6497be 100644 --- a/pkg/services/session/executor.go +++ b/pkg/services/session/executor.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -28,7 +29,7 @@ func NewExecutionService(exec ServiceExecutor, l *logger.Logger) Server { } func (s *executorSvc) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - s.log.Debug("serving request...", + s.log.Debug(logs.SessionServingRequest, zap.String("component", "SessionService"), zap.String("request", "Create"), ) diff --git a/pkg/services/session/storage/persistent/storage.go b/pkg/services/session/storage/persistent/storage.go index ded33d1ec..25f067d62 100644 --- a/pkg/services/session/storage/persistent/storage.go +++ b/pkg/services/session/storage/persistent/storage.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -108,7 +109,7 @@ func (s *TokenStore) Get(ownerID user.ID, tokenID []byte) (t *storage.PrivateTok return nil }) if err != nil { - s.l.Error("could not get session from persistent storage", + s.l.Error(logs.PersistentCouldNotGetSessionFromPersistentStorage, zap.Error(err), zap.Stringer("ownerID", ownerID), zap.String("tokenID", hex.EncodeToString(tokenID)), @@ -133,7 +134,7 @@ func (s *TokenStore) RemoveOld(epoch uint64) { if epochFromToken(v) <= epoch { err = c.Delete() if err != nil { - s.l.Error("could not delete %s token", + s.l.Error(logs.PersistentCouldNotDeleteSToken, zap.String("token_id", hex.EncodeToString(k)), ) } @@ -144,7 +145,7 @@ func (s *TokenStore) RemoveOld(epoch uint64) { }) }) if err != nil { - s.l.Error("could not clean up expired tokens", + s.l.Error(logs.PersistentCouldNotCleanUpExpiredTokens, zap.Uint64("epoch", epoch), ) } diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index 1671d2511..9594514f1 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -5,6 +5,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.uber.org/zap" ) @@ -29,7 +30,7 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo return false } - s.log.Debug("redirecting tree service query", zap.String("endpoint", endpoint)) + s.log.Debug(logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint)) called = true stop = f(c) return true diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index bb20310b2..98ed3df39 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -8,6 +8,7 @@ import ( "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -45,7 +46,7 @@ func (s *Service) localReplicationWorker() { case op := <-s.replicateLocalCh: err := s.forest.TreeApply(op.cid, op.treeID, &op.Move, false) if err != nil { - s.log.Error("failed to apply replicated operation", + s.log.Error(logs.TreeFailedToApplyReplicatedOperation, zap.String("err", err.Error())) } } @@ -79,10 +80,10 @@ func (s *Service) replicationWorker(ctx context.Context) { if lastErr != nil { if errors.Is(lastErr, errRecentlyFailed) { - s.log.Debug("do not send update to the node", + s.log.Debug(logs.TreeDoNotSendUpdateToTheNode, zap.String("last_error", lastErr.Error())) } else { - s.log.Warn("failed to sent update to the node", + s.log.Warn(logs.TreeFailedToSentUpdateToTheNode, zap.String("last_error", lastErr.Error()), zap.String("address", lastAddr), zap.String("key", hex.EncodeToString(task.n.PublicKey()))) @@ -112,7 +113,7 @@ func (s *Service) replicateLoop(ctx context.Context) { case op := <-s.replicateCh: err := s.replicate(op) if err != nil { - s.log.Error("error during replication", + s.log.Error(logs.TreeErrorDuringReplication, zap.String("err", err.Error()), zap.Stringer("cid", op.cid), zap.String("treeID", op.treeID)) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 7a5a95c4c..e861541f4 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -8,6 +8,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -76,7 +77,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op var tableFromBearer bool if len(rawBearer) != 0 { if !basicACL.AllowedBearerRules(op) { - s.log.Debug("bearer presented but not allowed by ACL", + s.log.Debug(logs.TreeBearerPresentedButNotAllowedByACL, zap.String("cid", cid.EncodeToString()), zap.String("op", op.String()), ) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 47299d1c9..91f43900f 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -11,6 +11,7 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" @@ -86,7 +87,7 @@ func (s *Service) synchronizeAllTrees(ctx context.Context, cid cid.ID) error { for _, tid := range treesToSync { h, err := s.forest.TreeLastSyncHeight(cid, tid) if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { - s.log.Warn("could not get last synchronized height for a tree", + s.log.Warn(logs.TreeCouldNotGetLastSynchronizedHeightForATree, zap.Stringer("cid", cid), zap.String("tree", tid)) continue @@ -94,7 +95,7 @@ func (s *Service) synchronizeAllTrees(ctx context.Context, cid cid.ID) error { newHeight := s.synchronizeTree(ctx, cid, h, tid, nodes) if h < newHeight { if err := s.forest.TreeUpdateLastSyncHeight(cid, tid, newHeight); err != nil { - s.log.Warn("could not update last synchronized height for a tree", + s.log.Warn(logs.TreeCouldNotUpdateLastSynchronizedHeightForATree, zap.Stringer("cid", cid), zap.String("tree", tid)) } @@ -126,7 +127,7 @@ func (s *Service) SynchronizeTree(ctx context.Context, cid cid.ID, treeID string func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, treeID string, nodes []netmapSDK.NodeInfo) uint64 { - s.log.Debug("synchronize tree", + s.log.Debug(logs.TreeSynchronizeTree, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.Uint64("from", from)) @@ -184,7 +185,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, } if err := errGroup.Wait(); err != nil { - s.log.Warn("failed to run tree synchronization over all nodes", zap.Error(err)) + s.log.Warn(logs.TreeFailedToRunTreeSynchronizationOverAllNodes, zap.Error(err)) } newHeight := uint64(math.MaxUint64) @@ -283,11 +284,11 @@ func (s *Service) syncLoop(ctx context.Context) { case <-ctx.Done(): return case <-s.syncChan: - s.log.Debug("syncing trees...") + s.log.Debug(logs.TreeSyncingTrees) cnrs, err := s.cfg.cnrSource.List() if err != nil { - s.log.Error("could not fetch containers", zap.Error(err)) + s.log.Error(logs.TreeCouldNotFetchContainers, zap.Error(err)) continue } @@ -297,7 +298,7 @@ func (s *Service) syncLoop(ctx context.Context) { s.removeContainers(ctx, newMap) - s.log.Debug("trees have been synchronized") + s.log.Debug(logs.TreeTreesHaveBeenSynchronized) } } } @@ -310,19 +311,19 @@ func (s *Service) syncContainers(ctx context.Context, cnrs []cid.ID) { cnr := cnr err := s.syncPool.Submit(func() { defer wg.Done() - s.log.Debug("syncing container trees...", zap.Stringer("cid", cnr)) + s.log.Debug(logs.TreeSyncingContainerTrees, zap.Stringer("cid", cnr)) err := s.synchronizeAllTrees(ctx, cnr) if err != nil { - s.log.Error("could not sync trees", zap.Stringer("cid", cnr), zap.Error(err)) + s.log.Error(logs.TreeCouldNotSyncTrees, zap.Stringer("cid", cnr), zap.Error(err)) return } - s.log.Debug("container trees have been synced", zap.Stringer("cid", cnr)) + s.log.Debug(logs.TreeContainerTreesHaveBeenSynced, zap.Stringer("cid", cnr)) }) if err != nil { wg.Done() - s.log.Error("could not query trees for synchronization", + s.log.Error(logs.TreeCouldNotQueryTreesForSynchronization, zap.Stringer("cid", cnr), zap.Error(err)) if errors.Is(err, ants.ErrPoolClosed) { @@ -349,11 +350,11 @@ func (s *Service) removeContainers(ctx context.Context, newContainers map[cid.ID } for _, cnr := range removed { - s.log.Debug("removing redundant trees...", zap.Stringer("cid", cnr)) + s.log.Debug(logs.TreeRemovingRedundantTrees, zap.Stringer("cid", cnr)) err := s.DropTree(ctx, cnr, "") if err != nil { - s.log.Error("could not remove redundant tree", + s.log.Error(logs.TreeCouldNotRemoveRedundantTree, zap.Stringer("cid", cnr), zap.Error(err)) } @@ -367,7 +368,7 @@ func (s *Service) containersToSync(cnrs []cid.ID) (map[cid.ID]struct{}, []cid.ID for _, cnr := range cnrs { _, pos, err := s.getContainerNodes(cnr) if err != nil { - s.log.Error("could not calculate container nodes", + s.log.Error(logs.TreeCouldNotCalculateContainerNodes, zap.Stringer("cid", cnr), zap.Error(err)) continue From cffcc7745e99bb788f12f18e9f8d286239ac7eb9 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 13 Apr 2023 15:51:36 +0300 Subject: [PATCH 0102/1943] [#240] logs: Factor out common service log messages Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 43 +++++++------------ pkg/innerring/processors/balance/handlers.go | 2 +- .../processors/container/handlers.go | 6 +-- pkg/innerring/processors/frostfs/handlers.go | 12 +++--- pkg/innerring/processors/netmap/handlers.go | 8 ++-- .../processors/reputation/handlers.go | 2 +- pkg/services/object/delete/delete.go | 6 +-- pkg/services/object/get/container.go | 8 ++-- pkg/services/object/get/exec.go | 2 +- pkg/services/object/get/get.go | 6 +-- pkg/services/object/get/remote.go | 2 +- pkg/services/object/search/container.go | 10 ++--- pkg/services/object/search/exec.go | 2 +- pkg/services/object/search/search.go | 6 +-- pkg/services/session/executor.go | 2 +- 15 files changed, 53 insertions(+), 64 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 46ed8e867..83acedfb4 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -1,5 +1,21 @@ package logs +// Common service logs. +const ( + ServingRequest = "serving request..." + OperationFinishedSuccessfully = "operation finished successfully" + OperationFinishedWithError = "operation finished with error" + + TryingToExecuteInContainer = "trying to execute in container..." + CouldNotGetCurrentEpochNumber = "could not get current epoch number" + ProcessEpoch = "process epoch" + ProcessingNode = "processing node..." + NoMoreNodesAbortPlacementIteration = "no more nodes, abort placement iteration" + InterruptPlacementIterationByContext = "interrupt placement iteration by context" + + Notification = "notification" +) + const ( InnerringAmountCanNotBeRepresentedAsAnInt64 = "amount can not be represented as an int64" // Error in ../node/pkg/innerring/settlement.go InnerringCantGetUsedSpaceEstimation = "can't get used space estimation" // Warn in ../node/pkg/innerring/settlement.go @@ -56,7 +72,6 @@ const ( ReplicatorCouldNotGetObjectFromLocalStorage = "could not get object from local storage" // Error in ../node/pkg/services/replicator/process.go ReplicatorCouldNotReplicateObject = "could not replicate object" // Error in ../node/pkg/services/replicator/process.go ReplicatorObjectSuccessfullyReplicated = "object successfully replicated" // Debug in ../node/pkg/services/replicator/process.go - SessionServingRequest = "serving request..." // Debug in ../node/pkg/services/session/executor.go TreeRedirectingTreeServiceQuery = "redirecting tree service query" // Debug in ../node/pkg/services/tree/redirect.go TreeBearerPresentedButNotAllowedByACL = "bearer presented but not allowed by ACL" // Debug in ../node/pkg/services/tree/signature.go TreeCouldNotGetLastSynchronizedHeightForATree = "could not get last synchronized height for a tree" // Warn in ../node/pkg/services/tree/sync.go @@ -115,9 +130,6 @@ const ( TombstoneCouldNotGetTheTombstoneTheSource = "tombstone getter: could not get the tombstone the source" // Warn in ../node/pkg/services/object_manager/tombstone/checker.go TombstoneCouldNotParseTombstoneExpirationEpoch = "tombstone getter: could not parse tombstone expiration epoch" // Warn in ../node/pkg/services/object_manager/tombstone/checker.go DeleteRequestIsNotRolledOverToTheContainer = "request is not rolled over to the container" // Debug in ../node/pkg/services/object/delete/container.go - DeleteServingRequest = "serving request..." // Debug in ../node/pkg/services/object/delete/delete.go - DeleteOperationFinishedSuccessfully = "operation finished successfully" // Debug in ../node/pkg/services/object/delete/delete.go - DeleteOperationFinishedWithError = "operation finished with error" // Debug in ../node/pkg/services/object/delete/delete.go DeleteCouldNotComposeSplitInfo = "could not compose split info" // Debug in ../node/pkg/services/object/delete/exec.go DeleteNoSplitInfoObjectIsPHY = "no split info, object is PHY" // Debug in ../node/pkg/services/object/delete/exec.go DeleteAssemblingChain = "assembling chain..." // Debug in ../node/pkg/services/object/delete/exec.go @@ -134,47 +146,29 @@ const ( DeleteFormingSplitInfo = "forming split info..." // Debug in ../node/pkg/services/object/delete/local.go DeleteSplitInfoSuccessfullyFormedCollectingMembers = "split info successfully formed, collecting members..." // Debug in ../node/pkg/services/object/delete/local.go DeleteMembersSuccessfullyCollected = "members successfully collected" // Debug in ../node/pkg/services/object/delete/local.go - GetProcessingNode = "processing node..." // Debug in ../node/pkg/services/object/get/remote.go GetRemoteCallFailed = "remote call failed" // Debug in ../node/pkg/services/object/get/remote.go GetCanNotAssembleTheObject = "can not assemble the object" // Debug in ../node/pkg/services/object/get/assemble.go GetTryingToAssembleTheObject = "trying to assemble the object..." // Debug in ../node/pkg/services/object/get/assemble.go GetAssemblingSplittedObject = "assembling splitted object..." // Debug in ../node/pkg/services/object/get/assemble.go GetAssemblingSplittedObjectCompleted = "assembling splitted object completed" // Debug in ../node/pkg/services/object/get/assemble.go GetFailedToAssembleSplittedObject = "failed to assemble splitted object" // Warn in ../node/pkg/services/object/get/assemble.go - GetCouldNotGetCurrentEpochNumber = "could not get current epoch number" // Debug in ../node/pkg/services/object/get/exec.go GetCouldNotGenerateContainerTraverser = "could not generate container traverser" // Debug in ../node/pkg/services/object/get/exec.go GetCouldNotConstructRemoteNodeClient = "could not construct remote node client" // Debug in ../node/pkg/services/object/get/exec.go GetCouldNotWriteHeader = "could not write header" // Debug in ../node/pkg/services/object/get/exec.go GetCouldNotWritePayloadChunk = "could not write payload chunk" // Debug in ../node/pkg/services/object/get/exec.go GetLocalGetFailed = "local get failed" // Debug in ../node/pkg/services/object/get/local.go GetReturnResultDirectly = "return result directly" // Debug in ../node/pkg/services/object/get/container.go - GetTryingToExecuteInContainer = "trying to execute in container..." // Debug in ../node/pkg/services/object/get/container.go - GetProcessEpoch = "process epoch" // Debug in ../node/pkg/services/object/get/container.go - GetNoMoreNodesAbortPlacementIteration = "no more nodes, abort placement iteration" // Debug in ../node/pkg/services/object/get/container.go - GetInterruptPlacementIterationByContext = "interrupt placement iteration by context" // Debug in ../node/pkg/services/object/get/container.go GetCompletingTheOperation = "completing the operation" // Debug in ../node/pkg/services/object/get/container.go - GetServingRequest = "serving request..." // Debug in ../node/pkg/services/object/get/get.go - GetOperationFinishedSuccessfully = "operation finished successfully" // Debug in ../node/pkg/services/object/get/get.go GetRequestedObjectWasMarkedAsRemoved = "requested object was marked as removed" // Debug in ../node/pkg/services/object/get/get.go GetRequestedObjectIsVirtual = "requested object is virtual" // Debug in ../node/pkg/services/object/get/get.go GetRequestedRangeIsOutOfObjectBounds = "requested range is out of object bounds" // Debug in ../node/pkg/services/object/get/get.go - GetOperationFinishedWithError = "operation finished with error" // Debug in ../node/pkg/services/object/get/get.go PutAdditionalContainerBroadcastFailure = "additional container broadcast failure" // Error in ../node/pkg/services/object/put/distributed.go SearchReturnResultDirectly = "return result directly" // Debug in ../node/pkg/services/object/search/container.go - SearchTryingToExecuteInContainer = "trying to execute in container..." // Debug in ../node/pkg/services/object/search/container.go - SearchProcessEpoch = "process epoch" // Debug in ../node/pkg/services/object/search/container.go - SearchNoMoreNodesAbortPlacementIteration = "no more nodes, abort placement iteration" // Debug in ../node/pkg/services/object/search/container.go - SearchInterruptPlacementIterationByContext = "interrupt placement iteration by context" // Debug in ../node/pkg/services/object/search/container.go - SearchProcessingNode = "processing node..." // Debug in ../node/pkg/services/object/search/container.go SearchCouldNotConstructRemoteNodeClient = "could not construct remote node client" // Debug in ../node/pkg/services/object/search/container.go SearchRemoteOperationFailed = "remote operation failed" // Debug in ../node/pkg/services/object/search/container.go - SearchCouldNotGetCurrentEpochNumber = "could not get current epoch number" // Debug in ../node/pkg/services/object/search/exec.go SearchCouldNotGenerateContainerTraverser = "could not generate container traverser" // Debug in ../node/pkg/services/object/search/exec.go SearchCouldNotWriteObjectIdentifiers = "could not write object identifiers" // Debug in ../node/pkg/services/object/search/exec.go SearchLocalOperationFailed = "local operation failed" // Debug in ../node/pkg/services/object/search/local.go - SearchServingRequest = "serving request..." // Debug in ../node/pkg/services/object/search/search.go - SearchOperationFinishedWithError = "operation finished with error" // Debug in ../node/pkg/services/object/search/search.go - SearchOperationFinishedSuccessfully = "operation finished successfully" // Debug in ../node/pkg/services/object/search/search.go UtilObjectServiceError = "object service error" // Error in ../node/pkg/services/object/util/log.go UtilCouldNotPushTaskToWorkerPool = "could not push task to worker pool" // Error in ../node/pkg/services/object/util/log.go V2CantCheckIfRequestFromInnerRing = "can't check if request from inner ring" // Debug in ../node/pkg/services/object/acl/v2/classifier.go @@ -419,13 +413,11 @@ const ( AuditParseClientNodeInfo = "parse client node info" // Warn in ../node/pkg/innerring/processors/audit/process.go AuditErrorInStorageGroupSearch = "error in storage group search" // Warn in ../node/pkg/innerring/processors/audit/process.go AuditCouldNotGetStorageGroupObjectForAuditSkipping = "could not get storage group object for audit, skipping" // Error in ../node/pkg/innerring/processors/audit/process.go - BalanceNotification = "notification" // Info in ../node/pkg/innerring/processors/balance/handlers.go BalanceBalanceWorkerPoolDrained = "balance worker pool drained" // Warn in ../node/pkg/innerring/processors/balance/handlers.go BalanceNonAlphabetModeIgnoreBalanceLock = "non alphabet mode, ignore balance lock" // Info in ../node/pkg/innerring/processors/balance/process_assets.go BalanceCantSendLockAssetTx = "can't send lock asset tx" // Error in ../node/pkg/innerring/processors/balance/process_assets.go BalanceBalanceWorkerPool = "balance worker pool" // Debug in ../node/pkg/innerring/processors/balance/processor.go ContainerContainerWorkerPool = "container worker pool" // Debug in ../node/pkg/innerring/processors/container/processor.go - ContainerNotification = "notification" // Info in ../node/pkg/innerring/processors/container/handlers.go ContainerContainerProcessorWorkerPoolDrained = "container processor worker pool drained" // Warn in ../node/pkg/innerring/processors/container/handlers.go ContainerNonAlphabetModeIgnoreContainerPut = "non alphabet mode, ignore container put" // Info in ../node/pkg/innerring/processors/container/process_container.go ContainerPutContainerCheckFailed = "put container check failed" // Error in ../node/pkg/innerring/processors/container/process_container.go @@ -442,7 +434,6 @@ const ( FrostFSNonAlphabetModeIgnoreConfig = "non alphabet mode, ignore config" // Info in ../node/pkg/innerring/processors/frostfs/process_config.go FrostFSCantRelaySetConfigEvent = "can't relay set config event" // Error in ../node/pkg/innerring/processors/frostfs/process_config.go FrostFSFrostfsWorkerPool = "frostfs worker pool" // Debug in ../node/pkg/innerring/processors/frostfs/processor.go - FrostFSNotification = "notification" // Info in ../node/pkg/innerring/processors/frostfs/handlers.go FrostFSFrostfsProcessorWorkerPoolDrained = "frostfs processor worker pool drained" // Warn in ../node/pkg/innerring/processors/frostfs/handlers.go FrostFSNonAlphabetModeIgnoreDeposit = "non alphabet mode, ignore deposit" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go FrostFSCantTransferAssetsToBalanceContract = "can't transfer assets to balance contract" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go @@ -474,7 +465,6 @@ const ( NetmapNetmapWorkerPool = "netmap worker pool" // Debug in ../node/pkg/innerring/processors/netmap/processor.go NetmapTick = "tick" // Info in ../node/pkg/innerring/processors/netmap/handlers.go NetmapNetmapWorkerPoolDrained = "netmap worker pool drained" // Warn in ../node/pkg/innerring/processors/netmap/handlers.go - NetmapNotification = "notification" // Info in ../node/pkg/innerring/processors/netmap/handlers.go NetmapNetmapCleanUpRoutineIsDisabled518 = "netmap clean up routine is disabled" // Debug in ../node/pkg/innerring/processors/netmap/handlers.go NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick = "non alphabet mode, ignore new netmap cleanup tick" // Info in ../node/pkg/innerring/processors/netmap/process_cleanup.go NetmapCantDecodePublicKeyOfNetmapNode = "can't decode public key of netmap node" // Warn in ../node/pkg/innerring/processors/netmap/process_cleanup.go @@ -505,7 +495,6 @@ const ( NetmapCouldNotIterateOverSubnetworksOfTheNode = "could not iterate over subnetworks of the node" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapCouldNotInvokeNetmapUpdateState = "could not invoke netmap.UpdateState" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapCouldNotInvokeNetmapAddPeer = "could not invoke netmap.AddPeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go - ReputationNotification = "notification" // Info in ../node/pkg/innerring/processors/reputation/handlers.go ReputationReputationWorkerPoolDrained = "reputation worker pool drained" // Warn in ../node/pkg/innerring/processors/reputation/handlers.go ReputationNonAlphabetModeIgnoreReputationPutNotification = "non alphabet mode, ignore reputation put notification" // Info in ../node/pkg/innerring/processors/reputation/process_put.go ReputationIgnoreReputationValue = "ignore reputation value" // Info in ../node/pkg/innerring/processors/reputation/process_put.go diff --git a/pkg/innerring/processors/balance/handlers.go b/pkg/innerring/processors/balance/handlers.go index 3360af916..e325da1f9 100644 --- a/pkg/innerring/processors/balance/handlers.go +++ b/pkg/innerring/processors/balance/handlers.go @@ -11,7 +11,7 @@ import ( func (bp *Processor) handleLock(ev event.Event) { lock := ev.(balanceEvent.Lock) - bp.log.Info(logs.BalanceNotification, + bp.log.Info(logs.Notification, zap.String("type", "lock"), zap.String("value", hex.EncodeToString(lock.ID()))) diff --git a/pkg/innerring/processors/container/handlers.go b/pkg/innerring/processors/container/handlers.go index 3d1946b4f..8d260808b 100644 --- a/pkg/innerring/processors/container/handlers.go +++ b/pkg/innerring/processors/container/handlers.go @@ -14,7 +14,7 @@ func (cp *Processor) handlePut(ev event.Event) { put := ev.(putEvent) id := sha256.Sum256(put.Container()) - cp.log.Info(logs.ContainerNotification, + cp.log.Info(logs.Notification, zap.String("type", "container put"), zap.String("id", base58.Encode(id[:]))) @@ -30,7 +30,7 @@ func (cp *Processor) handlePut(ev event.Event) { func (cp *Processor) handleDelete(ev event.Event) { del := ev.(containerEvent.Delete) - cp.log.Info(logs.ContainerNotification, + cp.log.Info(logs.Notification, zap.String("type", "container delete"), zap.String("id", base58.Encode(del.ContainerID()))) @@ -47,7 +47,7 @@ func (cp *Processor) handleDelete(ev event.Event) { func (cp *Processor) handleSetEACL(ev event.Event) { e := ev.(containerEvent.SetEACL) - cp.log.Info(logs.ContainerNotification, + cp.log.Info(logs.Notification, zap.String("type", "set EACL"), ) diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index 4eff15abe..4822cac2c 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -12,7 +12,7 @@ import ( func (np *Processor) handleDeposit(ev event.Event) { deposit := ev.(frostfsEvent.Deposit) - np.log.Info(logs.FrostFSNotification, + np.log.Info(logs.Notification, zap.String("type", "deposit"), zap.String("id", hex.EncodeToString(slice.CopyReverse(deposit.ID())))) @@ -28,7 +28,7 @@ func (np *Processor) handleDeposit(ev event.Event) { func (np *Processor) handleWithdraw(ev event.Event) { withdraw := ev.(frostfsEvent.Withdraw) - np.log.Info(logs.FrostFSNotification, + np.log.Info(logs.Notification, zap.String("type", "withdraw"), zap.String("id", hex.EncodeToString(slice.CopyReverse(withdraw.ID())))) @@ -44,7 +44,7 @@ func (np *Processor) handleWithdraw(ev event.Event) { func (np *Processor) handleCheque(ev event.Event) { cheque := ev.(frostfsEvent.Cheque) - np.log.Info(logs.FrostFSNotification, + np.log.Info(logs.Notification, zap.String("type", "cheque"), zap.String("id", hex.EncodeToString(cheque.ID()))) @@ -60,7 +60,7 @@ func (np *Processor) handleCheque(ev event.Event) { func (np *Processor) handleConfig(ev event.Event) { cfg := ev.(frostfsEvent.Config) - np.log.Info(logs.FrostFSNotification, + np.log.Info(logs.Notification, zap.String("type", "set config"), zap.String("key", hex.EncodeToString(cfg.Key())), zap.String("value", hex.EncodeToString(cfg.Value()))) @@ -77,7 +77,7 @@ func (np *Processor) handleConfig(ev event.Event) { func (np *Processor) handleBind(ev event.Event) { e := ev.(frostfsEvent.Bind) - np.log.Info(logs.FrostFSNotification, + np.log.Info(logs.Notification, zap.String("type", "bind"), ) @@ -93,7 +93,7 @@ func (np *Processor) handleBind(ev event.Event) { func (np *Processor) handleUnbind(ev event.Event) { e := ev.(frostfsEvent.Unbind) - np.log.Info(logs.FrostFSNotification, + np.log.Info(logs.Notification, zap.String("type", "unbind"), ) diff --git a/pkg/innerring/processors/netmap/handlers.go b/pkg/innerring/processors/netmap/handlers.go index 76b27c891..60d279940 100644 --- a/pkg/innerring/processors/netmap/handlers.go +++ b/pkg/innerring/processors/netmap/handlers.go @@ -27,7 +27,7 @@ func (np *Processor) HandleNewEpochTick(ev event.Event) { func (np *Processor) handleNewEpoch(ev event.Event) { epochEvent := ev.(netmapEvent.NewEpoch) - np.log.Info(logs.NetmapNotification, + np.log.Info(logs.Notification, zap.String("type", "new epoch"), zap.Uint64("value", epochEvent.EpochNumber())) @@ -46,7 +46,7 @@ func (np *Processor) handleNewEpoch(ev event.Event) { func (np *Processor) handleAddPeer(ev event.Event) { newPeer := ev.(netmapEvent.AddPeer) - np.log.Info(logs.NetmapNotification, + np.log.Info(logs.Notification, zap.String("type", "add peer"), ) @@ -64,7 +64,7 @@ func (np *Processor) handleAddPeer(ev event.Event) { func (np *Processor) handleUpdateState(ev event.Event) { updPeer := ev.(netmapEvent.UpdatePeer) - np.log.Info(logs.NetmapNotification, + np.log.Info(logs.Notification, zap.String("type", "update peer state"), zap.String("key", hex.EncodeToString(updPeer.PublicKey().Bytes()))) @@ -105,7 +105,7 @@ func (np *Processor) handleCleanupTick(ev event.Event) { func (np *Processor) handleRemoveNode(ev event.Event) { removeNode := ev.(subnetevents.RemoveNode) - np.log.Info(logs.NetmapNotification, + np.log.Info(logs.Notification, zap.String("type", "remove node from subnet"), zap.String("subnetID", hex.EncodeToString(removeNode.SubnetworkID())), zap.String("key", hex.EncodeToString(removeNode.Node())), diff --git a/pkg/innerring/processors/reputation/handlers.go b/pkg/innerring/processors/reputation/handlers.go index 30e3e9503..9b8e7f66a 100644 --- a/pkg/innerring/processors/reputation/handlers.go +++ b/pkg/innerring/processors/reputation/handlers.go @@ -14,7 +14,7 @@ func (rp *Processor) handlePutReputation(ev event.Event) { peerID := put.PeerID() // FIXME: #1147 do not use `ToV2` method outside frostfs-api-go library - rp.log.Info(logs.ReputationNotification, + rp.log.Info(logs.Notification, zap.String("type", "reputation put"), zap.String("peer_id", hex.EncodeToString(peerID.PublicKey()))) diff --git a/pkg/services/object/delete/delete.go b/pkg/services/object/delete/delete.go index 971f0a6f5..ebc191538 100644 --- a/pkg/services/object/delete/delete.go +++ b/pkg/services/object/delete/delete.go @@ -35,7 +35,7 @@ func (s *Service) Delete(ctx context.Context, prm Prm) error { } func (exec *execCtx) execute(ctx context.Context) { - exec.log.Debug(logs.DeleteServingRequest) + exec.log.Debug(logs.ServingRequest) // perform local operation exec.executeLocal(ctx) @@ -47,9 +47,9 @@ func (exec *execCtx) analyzeStatus(execCnr bool) { // analyze local result switch exec.status { case statusOK: - exec.log.Debug(logs.DeleteOperationFinishedSuccessfully) + exec.log.Debug(logs.OperationFinishedSuccessfully) default: - exec.log.Debug(logs.DeleteOperationFinishedWithError, + exec.log.Debug(logs.OperationFinishedWithError, zap.String("error", exec.err.Error()), ) diff --git a/pkg/services/object/get/container.go b/pkg/services/object/get/container.go index 74d63966e..17628e577 100644 --- a/pkg/services/object/get/container.go +++ b/pkg/services/object/get/container.go @@ -16,7 +16,7 @@ func (exec *execCtx) executeOnContainer(ctx context.Context) { lookupDepth := exec.netmapLookupDepth() - exec.log.Debug(logs.GetTryingToExecuteInContainer, + exec.log.Debug(logs.TryingToExecuteInContainer, zap.Uint64("netmap lookup depth", lookupDepth), ) @@ -44,7 +44,7 @@ func (exec *execCtx) executeOnContainer(ctx context.Context) { } func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { - exec.log.Debug(logs.GetProcessEpoch, + exec.log.Debug(logs.ProcessEpoch, zap.Uint64("number", exec.curProcEpoch), ) @@ -61,7 +61,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { for { addrs := traverser.Next() if len(addrs) == 0 { - exec.log.Debug(logs.GetNoMoreNodesAbortPlacementIteration) + exec.log.Debug(logs.NoMoreNodesAbortPlacementIteration) return false } @@ -69,7 +69,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { for i := range addrs { select { case <-ctx.Done(): - exec.log.Debug(logs.GetInterruptPlacementIterationByContext, + exec.log.Debug(logs.InterruptPlacementIterationByContext, zap.String("error", ctx.Err().Error()), ) diff --git a/pkg/services/object/get/exec.go b/pkg/services/object/get/exec.go index 7f090dd50..1bd5aa7f8 100644 --- a/pkg/services/object/get/exec.go +++ b/pkg/services/object/get/exec.go @@ -150,7 +150,7 @@ func (exec *execCtx) initEpoch() bool { exec.status = statusUndefined exec.err = err - exec.log.Debug(logs.GetCouldNotGetCurrentEpochNumber, + exec.log.Debug(logs.CouldNotGetCurrentEpochNumber, zap.String("error", err.Error()), ) diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index bb0d669da..6295de9a9 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -84,7 +84,7 @@ func (s *Service) get(ctx context.Context, prm commonPrm, opts ...execOption) st } func (exec *execCtx) execute(ctx context.Context) { - exec.log.Debug(logs.GetServingRequest) + exec.log.Debug(logs.ServingRequest) // perform local operation exec.executeLocal(ctx) @@ -96,7 +96,7 @@ func (exec *execCtx) analyzeStatus(ctx context.Context, execCnr bool) { // analyze local result switch exec.status { case statusOK: - exec.log.Debug(logs.GetOperationFinishedSuccessfully) + exec.log.Debug(logs.OperationFinishedSuccessfully) case statusINHUMED: exec.log.Debug(logs.GetRequestedObjectWasMarkedAsRemoved) case statusVIRTUAL: @@ -105,7 +105,7 @@ func (exec *execCtx) analyzeStatus(ctx context.Context, execCnr bool) { case statusOutOfRange: exec.log.Debug(logs.GetRequestedRangeIsOutOfObjectBounds) default: - exec.log.Debug(logs.GetOperationFinishedWithError, + exec.log.Debug(logs.OperationFinishedWithError, zap.String("error", exec.err.Error()), ) diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index f4f74083b..ac8ec5105 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -16,7 +16,7 @@ func (exec *execCtx) processNode(ctx context.Context, info client.NodeInfo) bool ctx, span := tracing.StartSpanFromContext(ctx, "getService.processNode") defer span.End() - exec.log.Debug(logs.GetProcessingNode) + exec.log.Debug(logs.ProcessingNode) client, ok := exec.remoteClient(info) if !ok { diff --git a/pkg/services/object/search/container.go b/pkg/services/object/search/container.go index 9df438e00..2b6101a98 100644 --- a/pkg/services/object/search/container.go +++ b/pkg/services/object/search/container.go @@ -18,7 +18,7 @@ func (exec *execCtx) executeOnContainer(ctx context.Context) { lookupDepth := exec.netmapLookupDepth() - exec.log.Debug(logs.SearchTryingToExecuteInContainer, + exec.log.Debug(logs.TryingToExecuteInContainer, zap.Uint64("netmap lookup depth", lookupDepth), ) @@ -49,7 +49,7 @@ func (exec *execCtx) executeOnContainer(ctx context.Context) { } func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { - exec.log.Debug(logs.SearchProcessEpoch, + exec.log.Debug(logs.ProcessEpoch, zap.Uint64("number", exec.curProcEpoch), ) @@ -64,7 +64,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { for { addrs := traverser.Next() if len(addrs) == 0 { - exec.log.Debug(logs.SearchNoMoreNodesAbortPlacementIteration) + exec.log.Debug(logs.NoMoreNodesAbortPlacementIteration) break } @@ -77,7 +77,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { defer wg.Done() select { case <-ctx.Done(): - exec.log.Debug(logs.SearchInterruptPlacementIterationByContext, + exec.log.Debug(logs.InterruptPlacementIterationByContext, zap.String("error", ctx.Err().Error())) return default: @@ -87,7 +87,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { client.NodeInfoFromNetmapElement(&info, addrs[i]) - exec.log.Debug(logs.SearchProcessingNode, zap.String("key", hex.EncodeToString(addrs[i].PublicKey()))) + exec.log.Debug(logs.ProcessingNode, zap.String("key", hex.EncodeToString(addrs[i].PublicKey()))) c, err := exec.svc.clientConstructor.get(info) if err != nil { diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index 1733d7840..475a31b98 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -81,7 +81,7 @@ func (exec *execCtx) initEpoch() bool { exec.status = statusUndefined exec.err = err - exec.log.Debug(logs.SearchCouldNotGetCurrentEpochNumber, + exec.log.Debug(logs.CouldNotGetCurrentEpochNumber, zap.String("error", err.Error()), ) diff --git a/pkg/services/object/search/search.go b/pkg/services/object/search/search.go index 5bf0710ad..b192e1d04 100644 --- a/pkg/services/object/search/search.go +++ b/pkg/services/object/search/search.go @@ -24,7 +24,7 @@ func (s *Service) Search(ctx context.Context, prm Prm) error { } func (exec *execCtx) execute(ctx context.Context) { - exec.log.Debug(logs.SearchServingRequest) + exec.log.Debug(logs.ServingRequest) // perform local operation exec.executeLocal() @@ -36,11 +36,11 @@ func (exec *execCtx) analyzeStatus(ctx context.Context, execCnr bool) { // analyze local result switch exec.status { default: - exec.log.Debug(logs.SearchOperationFinishedWithError, + exec.log.Debug(logs.OperationFinishedWithError, zap.String("error", exec.err.Error()), ) case statusOK: - exec.log.Debug(logs.SearchOperationFinishedSuccessfully) + exec.log.Debug(logs.OperationFinishedSuccessfully) } if execCnr { diff --git a/pkg/services/session/executor.go b/pkg/services/session/executor.go index 0be6497be..5ad1d6518 100644 --- a/pkg/services/session/executor.go +++ b/pkg/services/session/executor.go @@ -29,7 +29,7 @@ func NewExecutionService(exec ServiceExecutor, l *logger.Logger) Server { } func (s *executorSvc) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - s.log.Debug(logs.SessionServingRequest, + s.log.Debug(logs.ServingRequest, zap.String("component", "SessionService"), zap.String("request", "Create"), ) From 4496999e525eee83cc9472f9bf573a6f7ad96b58 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 11 Apr 2023 15:04:05 +0300 Subject: [PATCH 0103/1943] [#100] Fix CHANGELOG Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 270d0265e..01a36c66d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,7 +86,7 @@ You need to change configuration environment variables to `FROSTFS_*` if you use New config field `object.delete.tombstone_lifetime` allows to set tombstone lifetime more appropriate for a specific deployment. -Use `__SYSTEM__` prefix for system attributes instead of `__NEOFS__` +Use `__SYSTEM__` prefix for system attributes instead of `__NEOFS__` (existed objects with old attributes will be treated as before, but for new objects new attributes will be used). ## Older versions From 0c6aeaaf18a708d051b72bb4a75c452611d1617d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 13 Apr 2023 17:42:59 +0300 Subject: [PATCH 0104/1943] [#100] adm: Take net settings into account during netmap contract update Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 2 + .../internal/modules/morph/config.go | 32 +++----- .../modules/morph/initialize_deploy.go | 77 +++++++++++++++---- .../internal/modules/morph/netmap_util.go | 47 +++++++++++ 4 files changed, 118 insertions(+), 40 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/netmap_util.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 01a36c66d..9d284f6fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ Changelog for FrostFS Node ### Added ### Changed ### Fixed +- Take network settings into account during netmap contract update (#100) + ### Removed ### Updated ### Updating from v0.36.0 diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index 8a888ab2c..11e1bd4f6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -15,7 +15,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/vm/emit" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -48,23 +47,12 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { buf := bytes.NewBuffer(nil) tw := tabwriter.NewWriter(buf, 0, 2, 2, ' ', 0) - for _, param := range arr { - tuple, ok := param.Value().([]stackitem.Item) - if !ok || len(tuple) != 2 { - return errors.New("invalid ListConfig response from netmap contract") - } - - k, err := tuple[0].TryBytes() - if err != nil { - return errors.New("invalid config key from netmap contract") - } - - v, err := tuple[1].TryBytes() - if err != nil { - return invalidConfigValueErr(k) - } - - switch string(k) { + m, err := parseConfigFromNetmapContract(arr) + if err != nil { + return err + } + for k, v := range m { + switch k { case netmapAuditFeeKey, netmapBasicIncomeRateKey, netmapContainerFeeKey, netmapContainerAliasFeeKey, netmapEigenTrustIterationsKey, @@ -77,12 +65,10 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { case netmapEigenTrustAlphaKey: _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%s (str)\n", k, v))) case netmapHomomorphicHashDisabledKey, netmapMaintenanceAllowedKey: - vBool, err := tuple[1].TryBool() - if err != nil { + if len(v) == 0 || len(v) > 1 { return invalidConfigValueErr(k) } - - _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%t (bool)\n", k, vBool))) + _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%t (bool)\n", k, v[0] == 1))) default: _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%s (hex)\n", k, hex.EncodeToString(v)))) } @@ -187,6 +173,6 @@ func parseConfigPair(kvStr string, force bool) (key string, val any, err error) return } -func invalidConfigValueErr(key []byte) error { +func invalidConfigValueErr(key string) error { return fmt.Errorf("invalid %s config value from netmap contract", key) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index ae80c2ffd..3446cd0b8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -23,6 +23,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest" @@ -30,8 +31,8 @@ import ( "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" - "github.com/spf13/viper" ) const ( @@ -85,6 +86,21 @@ var ( nnsContract, alphabetContract, }, contractList...) + + netmapConfigKeys = []string{ + netmapEpochKey, + netmapMaxObjectSizeKey, + netmapAuditFeeKey, + netmapContainerFeeKey, + netmapContainerAliasFeeKey, + netmapEigenTrustIterationsKey, + netmapEigenTrustAlphaKey, + netmapBasicIncomeRateKey, + netmapInnerRingCandidateFeeKey, + netmapWithdrawFeeKey, + netmapHomomorphicHashDisabledKey, + netmapMaintenanceAllowedKey, + } ) type contractState struct { @@ -239,7 +255,7 @@ func (c *initializeContext) deployOrUpdateContracts(w *io2.BufBinWriter, nnsHash invokeHash = ctrHash } - params := getContractDeployParameters(cs, c.getContractDeployData(ctrName, keysParam)) + params := getContractDeployParameters(cs, c.getContractDeployData(ctrName, keysParam, updateMethodName)) res, err := c.CommitteeAct.MakeCall(invokeHash, method, params...) if err != nil { if method != updateMethodName || !strings.Contains(err.Error(), common.ErrAlreadyUpdated) { @@ -362,7 +378,7 @@ func (c *initializeContext) deployContracts() error { return fmt.Errorf("can't sign manifest group: %v", err) } - params := getContractDeployParameters(cs, c.getContractDeployData(ctrName, keysParam)) + params := getContractDeployParameters(cs, c.getContractDeployData(ctrName, keysParam, deployMethodName)) res, err := c.CommitteeAct.MakeCall(management.Hash, deployMethodName, params...) if err != nil { return fmt.Errorf("can't deploy %s contract: %w", ctrName, err) @@ -529,7 +545,7 @@ func getContractDeployParameters(cs *contractState, deployData []any) []any { return []any{cs.RawNEF, cs.RawManifest, deployData} } -func (c *initializeContext) getContractDeployData(ctrName string, keysParam []any) []any { +func (c *initializeContext) getContractDeployData(ctrName string, keysParam []any, method string) []any { items := make([]any, 1, 6) items[0] = false // notaryDisabled is false @@ -566,20 +582,31 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an c.Contracts[netmapContract].Hash, c.Contracts[containerContract].Hash) case netmapContract: - configParam := []any{ - netmapEpochKey, viper.GetInt64(epochDurationInitFlag), - netmapMaxObjectSizeKey, viper.GetInt64(maxObjectSizeInitFlag), - netmapAuditFeeKey, viper.GetInt64(auditFeeInitFlag), - netmapContainerFeeKey, viper.GetInt64(containerFeeInitFlag), - netmapContainerAliasFeeKey, viper.GetInt64(containerAliasFeeInitFlag), - netmapEigenTrustIterationsKey, int64(defaultEigenTrustIterations), - netmapEigenTrustAlphaKey, defaultEigenTrustAlpha, - netmapBasicIncomeRateKey, viper.GetInt64(incomeRateInitFlag), - netmapInnerRingCandidateFeeKey, viper.GetInt64(candidateFeeInitFlag), - netmapWithdrawFeeKey, viper.GetInt64(withdrawFeeInitFlag), - netmapHomomorphicHashDisabledKey, viper.GetBool(homomorphicHashDisabledInitFlag), - netmapMaintenanceAllowedKey, viper.GetBool(maintenanceModeAllowedInitFlag), + md := getDefaultNetmapContractConfigMap() + if method == updateMethodName { + arr, err := c.getNetConfigFromNetmapContract() + if err != nil { + panic(err) + } + m, err := parseConfigFromNetmapContract(arr) + if err != nil { + panic(err) + } + for k, v := range m { + for _, key := range netmapConfigKeys { + if k == key { + md[k] = v + break + } + } + } } + + var configParam []any + for k, v := range md { + configParam = append(configParam, k, v) + } + items = append(items, c.Contracts[balanceContract].Hash, c.Contracts[containerContract].Hash, @@ -595,6 +622,22 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an return items } +func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, error) { + cs, err := c.Client.GetContractStateByID(1) + if err != nil { + return nil, fmt.Errorf("NNS is not yet deployed: %w", err) + } + nmHash, err := nnsResolveHash(c.ReadOnlyInvoker, cs.Hash, netmapContract+".frostfs") + if err != nil { + return nil, fmt.Errorf("can't get netmap contract hash: %w", err) + } + arr, err := unwrap.Array(c.ReadOnlyInvoker.Call(nmHash, "listConfig")) + if err != nil { + return nil, fmt.Errorf("can't fetch list of network config keys from the netmap contract") + } + return arr, err +} + func (c *initializeContext) getAlphabetDeployItems(i, n int) []any { items := make([]any, 6) items[0] = false diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_util.go b/cmd/frostfs-adm/internal/modules/morph/netmap_util.go new file mode 100644 index 000000000..2ba48b542 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/netmap_util.go @@ -0,0 +1,47 @@ +package morph + +import ( + "errors" + + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "github.com/spf13/viper" +) + +func getDefaultNetmapContractConfigMap() map[string]any { + m := make(map[string]any) + m[netmapEpochKey] = viper.GetInt64(epochDurationInitFlag) + m[netmapMaxObjectSizeKey] = viper.GetInt64(maxObjectSizeInitFlag) + m[netmapAuditFeeKey] = viper.GetInt64(auditFeeInitFlag) + m[netmapContainerFeeKey] = viper.GetInt64(containerFeeInitFlag) + m[netmapContainerAliasFeeKey] = viper.GetInt64(containerAliasFeeInitFlag) + m[netmapEigenTrustIterationsKey] = int64(defaultEigenTrustIterations) + m[netmapEigenTrustAlphaKey] = defaultEigenTrustAlpha + m[netmapBasicIncomeRateKey] = viper.GetInt64(incomeRateInitFlag) + m[netmapInnerRingCandidateFeeKey] = viper.GetInt64(candidateFeeInitFlag) + m[netmapWithdrawFeeKey] = viper.GetInt64(withdrawFeeInitFlag) + m[netmapHomomorphicHashDisabledKey] = viper.GetBool(homomorphicHashDisabledInitFlag) + m[netmapMaintenanceAllowedKey] = viper.GetBool(maintenanceModeAllowedInitFlag) + return m +} + +func parseConfigFromNetmapContract(arr []stackitem.Item) (map[string][]byte, error) { + m := make(map[string][]byte, len(arr)) + for _, param := range arr { + tuple, ok := param.Value().([]stackitem.Item) + if !ok || len(tuple) != 2 { + return nil, errors.New("invalid ListConfig response from netmap contract") + } + + k, err := tuple[0].TryBytes() + if err != nil { + return nil, errors.New("invalid config key from netmap contract") + } + + v, err := tuple[1].TryBytes() + if err != nil { + return nil, invalidConfigValueErr(string(k)) + } + m[string(k)] = v + } + return m, nil +} From 299b6a69389f6bb154ca9c6d442388234038929b Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 11 Apr 2023 14:59:24 +0300 Subject: [PATCH 0105/1943] [#100] adm: Use netmap constants from pkg Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/config.go | 29 ++++---- .../modules/morph/initialize_deploy.go | 38 ++++------ .../internal/modules/morph/netmap_util.go | 25 ++++--- pkg/morph/client/netmap/config.go | 74 +++++++++---------- 4 files changed, 78 insertions(+), 88 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index 11e1bd4f6..3a60e7197 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -10,6 +10,7 @@ import ( "strings" "text/tabwriter" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" @@ -53,18 +54,18 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { } for k, v := range m { switch k { - case netmapAuditFeeKey, netmapBasicIncomeRateKey, - netmapContainerFeeKey, netmapContainerAliasFeeKey, - netmapEigenTrustIterationsKey, - netmapEpochKey, netmapInnerRingCandidateFeeKey, - netmapMaxObjectSizeKey, netmapWithdrawFeeKey: + case netmap.AuditFeeConfig, netmap.BasicIncomeRateConfig, + netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, + netmap.EtIterationsConfig, + netmap.EpochDurationConfig, netmap.IrCandidateFeeConfig, + netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig: nbuf := make([]byte, 8) copy(nbuf[:], v) n := binary.LittleEndian.Uint64(nbuf) _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%d (int)\n", k, n))) - case netmapEigenTrustAlphaKey: + case netmap.EtAlphaConfig: _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%s (str)\n", k, v))) - case netmapHomomorphicHashDisabledKey, netmapMaintenanceAllowedKey: + case netmap.HomomorphicHashingDisabledKey, netmap.MaintenanceModeAllowedConfig: if len(v) == 0 || len(v) > 1 { return invalidConfigValueErr(k) } @@ -136,16 +137,16 @@ func parseConfigPair(kvStr string, force bool) (key string, val any, err error) valRaw := v switch key { - case netmapAuditFeeKey, netmapBasicIncomeRateKey, - netmapContainerFeeKey, netmapContainerAliasFeeKey, - netmapEigenTrustIterationsKey, - netmapEpochKey, netmapInnerRingCandidateFeeKey, - netmapMaxObjectSizeKey, netmapWithdrawFeeKey: + case netmap.AuditFeeConfig, netmap.BasicIncomeRateConfig, + netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, + netmap.EtIterationsConfig, + netmap.EpochDurationConfig, netmap.IrCandidateFeeConfig, + netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig: val, err = strconv.ParseInt(valRaw, 10, 64) if err != nil { err = fmt.Errorf("could not parse %s's value '%s' as int: %w", key, valRaw, err) } - case netmapEigenTrustAlphaKey: + case netmap.EtAlphaConfig: // just check that it could // be parsed correctly _, err = strconv.ParseFloat(v, 64) @@ -154,7 +155,7 @@ func parseConfigPair(kvStr string, force bool) (key string, val any, err error) } val = valRaw - case netmapHomomorphicHashDisabledKey, netmapMaintenanceAllowedKey: + case netmap.HomomorphicHashingDisabledKey, netmap.MaintenanceModeAllowedConfig: val, err = strconv.ParseBool(valRaw) if err != nil { err = fmt.Errorf("could not parse %s's value '%s' as bool: %w", key, valRaw, err) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 3446cd0b8..9e473463b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -16,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -51,19 +52,6 @@ const ( ) const ( - netmapEpochKey = "EpochDuration" - netmapMaxObjectSizeKey = "MaxObjectSize" - netmapAuditFeeKey = "AuditFee" - netmapContainerFeeKey = "ContainerFee" - netmapContainerAliasFeeKey = "ContainerAliasFee" - netmapEigenTrustIterationsKey = "EigenTrustIterations" - netmapEigenTrustAlphaKey = "EigenTrustAlpha" - netmapBasicIncomeRateKey = "BasicIncomeRate" - netmapInnerRingCandidateFeeKey = "InnerRingCandidateFee" - netmapWithdrawFeeKey = "WithdrawFee" - netmapHomomorphicHashDisabledKey = "HomomorphicHashingDisabled" - netmapMaintenanceAllowedKey = "MaintenanceModeAllowed" - defaultEigenTrustIterations = 4 defaultEigenTrustAlpha = "0.1" ) @@ -88,18 +76,18 @@ var ( }, contractList...) netmapConfigKeys = []string{ - netmapEpochKey, - netmapMaxObjectSizeKey, - netmapAuditFeeKey, - netmapContainerFeeKey, - netmapContainerAliasFeeKey, - netmapEigenTrustIterationsKey, - netmapEigenTrustAlphaKey, - netmapBasicIncomeRateKey, - netmapInnerRingCandidateFeeKey, - netmapWithdrawFeeKey, - netmapHomomorphicHashDisabledKey, - netmapMaintenanceAllowedKey, + netmap.EpochDurationConfig, + netmap.MaxObjectSizeConfig, + netmap.AuditFeeConfig, + netmap.ContainerFeeConfig, + netmap.ContainerAliasFeeConfig, + netmap.EtIterationsConfig, + netmap.EtAlphaConfig, + netmap.BasicIncomeRateConfig, + netmap.IrCandidateFeeConfig, + netmap.WithdrawFeeConfig, + netmap.HomomorphicHashingDisabledKey, + netmap.MaintenanceModeAllowedConfig, } ) diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_util.go b/cmd/frostfs-adm/internal/modules/morph/netmap_util.go index 2ba48b542..4b6de5bd2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap_util.go @@ -3,24 +3,25 @@ package morph import ( "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/spf13/viper" ) func getDefaultNetmapContractConfigMap() map[string]any { m := make(map[string]any) - m[netmapEpochKey] = viper.GetInt64(epochDurationInitFlag) - m[netmapMaxObjectSizeKey] = viper.GetInt64(maxObjectSizeInitFlag) - m[netmapAuditFeeKey] = viper.GetInt64(auditFeeInitFlag) - m[netmapContainerFeeKey] = viper.GetInt64(containerFeeInitFlag) - m[netmapContainerAliasFeeKey] = viper.GetInt64(containerAliasFeeInitFlag) - m[netmapEigenTrustIterationsKey] = int64(defaultEigenTrustIterations) - m[netmapEigenTrustAlphaKey] = defaultEigenTrustAlpha - m[netmapBasicIncomeRateKey] = viper.GetInt64(incomeRateInitFlag) - m[netmapInnerRingCandidateFeeKey] = viper.GetInt64(candidateFeeInitFlag) - m[netmapWithdrawFeeKey] = viper.GetInt64(withdrawFeeInitFlag) - m[netmapHomomorphicHashDisabledKey] = viper.GetBool(homomorphicHashDisabledInitFlag) - m[netmapMaintenanceAllowedKey] = viper.GetBool(maintenanceModeAllowedInitFlag) + m[netmap.EpochDurationConfig] = viper.GetInt64(epochDurationInitFlag) + m[netmap.MaxObjectSizeConfig] = viper.GetInt64(maxObjectSizeInitFlag) + m[netmap.AuditFeeConfig] = viper.GetInt64(auditFeeInitFlag) + m[netmap.ContainerFeeConfig] = viper.GetInt64(containerFeeInitFlag) + m[netmap.ContainerAliasFeeConfig] = viper.GetInt64(containerAliasFeeInitFlag) + m[netmap.EtIterationsConfig] = int64(defaultEigenTrustIterations) + m[netmap.EtAlphaConfig] = defaultEigenTrustAlpha + m[netmap.BasicIncomeRateConfig] = viper.GetInt64(incomeRateInitFlag) + m[netmap.IrCandidateFeeConfig] = viper.GetInt64(candidateFeeInitFlag) + m[netmap.WithdrawFeeConfig] = viper.GetInt64(withdrawFeeInitFlag) + m[netmap.HomomorphicHashingDisabledKey] = viper.GetBool(homomorphicHashDisabledInitFlag) + m[netmap.MaintenanceModeAllowedConfig] = viper.GetBool(maintenanceModeAllowedInitFlag) return m } diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 6b721cdfb..3011bd541 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -11,24 +11,24 @@ import ( ) const ( - maxObjectSizeConfig = "MaxObjectSize" - basicIncomeRateConfig = "BasicIncomeRate" - auditFeeConfig = "AuditFee" - epochDurationConfig = "EpochDuration" - containerFeeConfig = "ContainerFee" - containerAliasFeeConfig = "ContainerAliasFee" - etIterationsConfig = "EigenTrustIterations" - etAlphaConfig = "EigenTrustAlpha" - irCandidateFeeConfig = "InnerRingCandidateFee" - withdrawFeeConfig = "WithdrawFee" - homomorphicHashingDisabledKey = "HomomorphicHashingDisabled" - maintenanceModeAllowedConfig = "MaintenanceModeAllowed" + MaxObjectSizeConfig = "MaxObjectSize" + BasicIncomeRateConfig = "BasicIncomeRate" + AuditFeeConfig = "AuditFee" + EpochDurationConfig = "EpochDuration" + ContainerFeeConfig = "ContainerFee" + ContainerAliasFeeConfig = "ContainerAliasFee" + EtIterationsConfig = "EigenTrustIterations" + EtAlphaConfig = "EigenTrustAlpha" + IrCandidateFeeConfig = "InnerRingCandidateFee" + WithdrawFeeConfig = "WithdrawFee" + HomomorphicHashingDisabledKey = "HomomorphicHashingDisabled" + MaintenanceModeAllowedConfig = "MaintenanceModeAllowed" ) // MaxObjectSize receives max object size configuration // value through the Netmap contract call. func (c *Client) MaxObjectSize() (uint64, error) { - objectSize, err := c.readUInt64Config(maxObjectSizeConfig) + objectSize, err := c.readUInt64Config(MaxObjectSizeConfig) if err != nil { return 0, fmt.Errorf("(%T) could not get epoch number: %w", c, err) } @@ -39,7 +39,7 @@ func (c *Client) MaxObjectSize() (uint64, error) { // BasicIncomeRate returns basic income rate configuration value from network // config in netmap contract. func (c *Client) BasicIncomeRate() (uint64, error) { - rate, err := c.readUInt64Config(basicIncomeRateConfig) + rate, err := c.readUInt64Config(BasicIncomeRateConfig) if err != nil { return 0, fmt.Errorf("(%T) could not get basic income rate: %w", c, err) } @@ -50,7 +50,7 @@ func (c *Client) BasicIncomeRate() (uint64, error) { // AuditFee returns audit fee configuration value from network // config in netmap contract. func (c *Client) AuditFee() (uint64, error) { - fee, err := c.readUInt64Config(auditFeeConfig) + fee, err := c.readUInt64Config(AuditFeeConfig) if err != nil { return 0, fmt.Errorf("(%T) could not get audit fee: %w", c, err) } @@ -60,7 +60,7 @@ func (c *Client) AuditFee() (uint64, error) { // EpochDuration returns number of sidechain blocks per one FrostFS epoch. func (c *Client) EpochDuration() (uint64, error) { - epochDuration, err := c.readUInt64Config(epochDurationConfig) + epochDuration, err := c.readUInt64Config(EpochDurationConfig) if err != nil { return 0, fmt.Errorf("(%T) could not get epoch duration: %w", c, err) } @@ -71,7 +71,7 @@ func (c *Client) EpochDuration() (uint64, error) { // ContainerFee returns fee paid by container owner to each alphabet node // for container registration. func (c *Client) ContainerFee() (uint64, error) { - fee, err := c.readUInt64Config(containerFeeConfig) + fee, err := c.readUInt64Config(ContainerFeeConfig) if err != nil { return 0, fmt.Errorf("(%T) could not get container fee: %w", c, err) } @@ -82,7 +82,7 @@ func (c *Client) ContainerFee() (uint64, error) { // ContainerAliasFee returns additional fee paid by container owner to each // alphabet node for container nice name registration. func (c *Client) ContainerAliasFee() (uint64, error) { - fee, err := c.readUInt64Config(containerAliasFeeConfig) + fee, err := c.readUInt64Config(ContainerAliasFeeConfig) if err != nil { return 0, fmt.Errorf("(%T) could not get container alias fee: %w", c, err) } @@ -93,7 +93,7 @@ func (c *Client) ContainerAliasFee() (uint64, error) { // EigenTrustIterations returns global configuration value of iteration cycles // for EigenTrust algorithm per epoch. func (c *Client) EigenTrustIterations() (uint64, error) { - iterations, err := c.readUInt64Config(etIterationsConfig) + iterations, err := c.readUInt64Config(EtIterationsConfig) if err != nil { return 0, fmt.Errorf("(%T) could not get eigen trust iterations: %w", c, err) } @@ -104,7 +104,7 @@ func (c *Client) EigenTrustIterations() (uint64, error) { // EigenTrustAlpha returns global configuration value of alpha parameter. // It receives the alpha as a string and tries to convert it to float. func (c *Client) EigenTrustAlpha() (float64, error) { - strAlpha, err := c.readStringConfig(etAlphaConfig) + strAlpha, err := c.readStringConfig(EtAlphaConfig) if err != nil { return 0, fmt.Errorf("(%T) could not get eigen trust alpha: %w", c, err) } @@ -117,13 +117,13 @@ func (c *Client) EigenTrustAlpha() (float64, error) { // // Returns (false, nil) if config key is not found in the contract. func (c *Client) HomomorphicHashDisabled() (bool, error) { - return c.readBoolConfig(homomorphicHashingDisabledKey) + return c.readBoolConfig(HomomorphicHashingDisabledKey) } // InnerRingCandidateFee returns global configuration value of fee paid by // node to be in inner ring candidates list. func (c *Client) InnerRingCandidateFee() (uint64, error) { - fee, err := c.readUInt64Config(irCandidateFeeConfig) + fee, err := c.readUInt64Config(IrCandidateFeeConfig) if err != nil { return 0, fmt.Errorf("(%T) could not get inner ring candidate fee: %w", c, err) } @@ -134,7 +134,7 @@ func (c *Client) InnerRingCandidateFee() (uint64, error) { // WithdrawFee returns global configuration value of fee paid by user to // withdraw assets from FrostFS contract. func (c *Client) WithdrawFee() (uint64, error) { - fee, err := c.readUInt64Config(withdrawFeeConfig) + fee, err := c.readUInt64Config(WithdrawFeeConfig) if err != nil { return 0, fmt.Errorf("(%T) could not get withdraw fee: %w", c, err) } @@ -148,7 +148,7 @@ func (c *Client) WithdrawFee() (uint64, error) { // // By default, maintenance state is disallowed. func (c *Client) MaintenanceModeAllowed() (bool, error) { - return c.readBoolConfig(maintenanceModeAllowedConfig) + return c.readBoolConfig(MaintenanceModeAllowedConfig) } func (c *Client) readUInt64Config(key string) (uint64, error) { @@ -299,32 +299,32 @@ func (c *Client) ReadNetworkConfiguration() (NetworkConfiguration, error) { Name: name, Value: value, }) - case maxObjectSizeConfig: + case MaxObjectSizeConfig: res.MaxObjectSize = bytesToUint64(value) - case basicIncomeRateConfig: + case BasicIncomeRateConfig: res.StoragePrice = bytesToUint64(value) - case auditFeeConfig: + case AuditFeeConfig: res.AuditFee = bytesToUint64(value) - case epochDurationConfig: + case EpochDurationConfig: res.EpochDuration = bytesToUint64(value) - case containerFeeConfig: + case ContainerFeeConfig: res.ContainerFee = bytesToUint64(value) - case containerAliasFeeConfig: + case ContainerAliasFeeConfig: res.ContainerAliasFee = bytesToUint64(value) - case etIterationsConfig: + case EtIterationsConfig: res.EigenTrustIterations = bytesToUint64(value) - case etAlphaConfig: + case EtAlphaConfig: res.EigenTrustAlpha, err = strconv.ParseFloat(string(value), 64) if err != nil { - return fmt.Errorf("invalid prm %s: %v", etAlphaConfig, err) + return fmt.Errorf("invalid prm %s: %v", EtAlphaConfig, err) } - case irCandidateFeeConfig: + case IrCandidateFeeConfig: res.IRCandidateFee = bytesToUint64(value) - case withdrawFeeConfig: + case WithdrawFeeConfig: res.WithdrawalFee = bytesToUint64(value) - case homomorphicHashingDisabledKey: + case HomomorphicHashingDisabledKey: res.HomomorphicHashingDisabled = bytesToBool(value) - case maintenanceModeAllowedConfig: + case MaintenanceModeAllowedConfig: res.MaintenanceModeAllowed = bytesToBool(value) } From adcfce39cf707466dfa0cf571d496dd1ba779fa0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 13 Apr 2023 16:04:42 +0300 Subject: [PATCH 0106/1943] [#246] .gitattributes: Do not show diff for go.sum When we update dependencies it can be rather big. However it is generated automatically with `go mod tidy`, no need to review. Signed-off-by: Evgenii Stratonikov --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index c7a3f7a86..aa9391657 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ /**/*.pb.go -diff -merge /**/*.pb.go linguist-generated=true +/go.sum -diff From 41eb3129ae0fbe1dc177d9dbbd5ab6bb3917da97 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 14 Apr 2023 12:16:14 +0300 Subject: [PATCH 0107/1943] [#139] Refactor blobovnicza exist test to not use chmod Signed-off-by: Alejandro Lopez --- internal/logs/logs.go | 2 +- pkg/innerring/processors/netmap/process_peers.go | 2 +- .../blobstor/blobovniczatree/exists_test.go | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 83acedfb4..062538747 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -490,7 +490,7 @@ const ( NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapNonAlphabetModeIgnoreRemoveNodeFromSubnetNotification = "non alphabet mode, ignore remove node from subnet notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapCouldNotGetNetworkMapCandidates = "could not get network map candidates" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCouldNotUnmarshalSubnetId = "could not unmarshal subnet id" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCouldNotUnmarshalSubnetID = "could not unmarshal subnet id" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapGotZeroSubnetInRemoveNodeNotification = "got zero subnet in remove node notification" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapCouldNotIterateOverSubnetworksOfTheNode = "could not iterate over subnetworks of the node" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapCouldNotInvokeNetmapUpdateState = "could not invoke netmap.UpdateState" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index ffaad3b4e..130d08568 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -159,7 +159,7 @@ func (np *Processor) processRemoveSubnetNode(ev subnetEvent.RemoveNode) { err = subnetToRemoveFrom.Unmarshal(rawSubnet) if err != nil { - np.log.Warn(logs.NetmapCouldNotUnmarshalSubnetId, + np.log.Warn(logs.NetmapCouldNotUnmarshalSubnetID, zap.Error(err), ) return diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 08fd2223f..0c7c61d76 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -53,10 +53,11 @@ func TestExistsInvalidStorageID(t *testing.T) { t.Run("invalid storage id", func(t *testing.T) { storageID := slice.Copy(putRes.StorageID) storageID[0] = '9' - badDir := filepath.Join(dir, "9") - require.NoError(t, os.MkdirAll(badDir, os.ModePerm)) - require.NoError(t, os.Chmod(badDir, 0)) - t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "9"), os.ModePerm) }) + + // An invalid boltdb file is created so that it returns an error when opened + badFileDir := filepath.Join(dir, "9", "0") + require.NoError(t, os.MkdirAll(badFileDir, os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(badFileDir, "0"), []byte("not a boltdb file content"), 0777)) res, err := b.Exists(context.Background(), common.ExistsPrm{Address: addr, StorageID: storageID}) require.Error(t, err) From 8d2f443868ec600b020d5e1ff3c3b0ba866e3f9b Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 13 Apr 2023 12:02:29 +0300 Subject: [PATCH 0108/1943] [#238] Fix linter error Signed-off-by: Anton Nikiforov --- config/example/node.env | 2 +- config/example/node.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/config/example/node.env b/config/example/node.env index 9a1a8b052..a4088f75a 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -187,4 +187,4 @@ FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m FROSTFS_TRACING_ENABLED=true FROSTFS_TRACING_ENDPOINT="localhost" -FROSTFS_TRACING_EXPORTER="otlp_grpc" \ No newline at end of file +FROSTFS_TRACING_EXPORTER="otlp_grpc" diff --git a/config/example/node.yaml b/config/example/node.yaml index e3b41d413..0d71f0fd2 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -219,4 +219,3 @@ tracing: enabled: true exporter: "otlp_grpc" endpoint: "localhost" - \ No newline at end of file From 995db117d0cd6df0a7ab61d5fad271f0c1556621 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 14 Apr 2023 12:04:46 +0300 Subject: [PATCH 0109/1943] [#238] node: Read cfg from dir even if cfg file not set Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 1 + cmd/frostfs-node/config/configdir_test.go | 24 +++++++++++++++++++++++ pkg/util/config/dir.go | 24 +++++++++-------------- 3 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 cmd/frostfs-node/config/configdir_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d284f6fc..94018e357 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Changelog for FrostFS Node ### Changed ### Fixed - Take network settings into account during netmap contract update (#100) +- Read config files from dir even if config file not provided via `--config` for node (#238) ### Removed ### Updated diff --git a/cmd/frostfs-node/config/configdir_test.go b/cmd/frostfs-node/config/configdir_test.go new file mode 100644 index 000000000..2c3255670 --- /dev/null +++ b/cmd/frostfs-node/config/configdir_test.go @@ -0,0 +1,24 @@ +package config + +import ( + "os" + "path" + "testing" + + "github.com/spf13/cast" + "github.com/stretchr/testify/require" +) + +func TestConfigDir(t *testing.T) { + dir := t.TempDir() + + cfgFileName0 := path.Join(dir, "cfg_00.json") + cfgFileName1 := path.Join(dir, "cfg_01.yml") + + require.NoError(t, os.WriteFile(cfgFileName0, []byte(`{"storage":{"shard_pool_size":15}}`), 0777)) + require.NoError(t, os.WriteFile(cfgFileName1, []byte("logger:\n level: debug"), 0777)) + + c := New(Prm{}, WithConfigDir(dir)) + require.Equal(t, "debug", cast.ToString(c.Sub("logger").Value("level"))) + require.EqualValues(t, 15, cast.ToUint32(c.Sub("storage").Value("shard_pool_size"))) +} diff --git a/pkg/util/config/dir.go b/pkg/util/config/dir.go index a74992d19..0379fe268 100644 --- a/pkg/util/config/dir.go +++ b/pkg/util/config/dir.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "os" "path" @@ -20,7 +21,7 @@ func ReadConfigDir(v *viper.Viper, configDir string) error { continue } ext := path.Ext(entry.Name()) - if ext != ".yaml" && ext != ".yml" { + if ext != ".yaml" && ext != ".yml" && ext != ".json" { continue } @@ -33,22 +34,15 @@ func ReadConfigDir(v *viper.Viper, configDir string) error { } // mergeConfig reads config file and merge its content with current viper. -func mergeConfig(v *viper.Viper, fileName string) (err error) { - var cfgFile *os.File - cfgFile, err = os.Open(fileName) +func mergeConfig(v *viper.Viper, fileName string) error { + cv := viper.New() + cv.SetConfigFile(fileName) + err := cv.ReadInConfig() if err != nil { - return err + return fmt.Errorf("failed to read config: %w", err) } - - defer func() { - errClose := cfgFile.Close() - if err == nil { - err = errClose - } - }() - - if err = v.MergeConfig(cfgFile); err != nil { - return err + if err = v.MergeConfigMap(cv.AllSettings()); err != nil { + return fmt.Errorf("failed to merge config: %w", err) } return nil From 200fc8b8822f77597bb3ecc3f96fb0906dc07669 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 12 Apr 2023 11:02:25 +0300 Subject: [PATCH 0110/1943] [#242] put: Pass context to relay function Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/distributed.go | 4 ++-- pkg/services/object/put/prm.go | 6 ++++-- pkg/services/object/put/streamer.go | 8 ++++---- pkg/services/object/put/v2/streamer.go | 4 ++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 86dc3c2ca..15296f83f 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -36,7 +36,7 @@ type distributedTarget struct { isLocalKey func([]byte) bool - relay func(nodeDesc) error + relay func(context.Context, nodeDesc) error fmt *object.FormatValidator @@ -153,7 +153,7 @@ func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdent func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error { if !node.local && t.relay != nil { - return t.relay(node) + return t.relay(ctx, node) } target := t.nodeTargetInitializer(node) diff --git a/pkg/services/object/put/prm.go b/pkg/services/object/put/prm.go index aea5926f4..27d9c9c7a 100644 --- a/pkg/services/object/put/prm.go +++ b/pkg/services/object/put/prm.go @@ -1,6 +1,8 @@ package putsvc import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" @@ -17,7 +19,7 @@ type PutInitPrm struct { traverseOpts []placement.Option - relay func(client.NodeInfo, client.MultiAddressClient) error + relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error } type PutChunkPrm struct { @@ -40,7 +42,7 @@ func (p *PutInitPrm) WithObject(v *object.Object) *PutInitPrm { return p } -func (p *PutInitPrm) WithRelay(f func(client.NodeInfo, client.MultiAddressClient) error) *PutInitPrm { +func (p *PutInitPrm) WithRelay(f func(context.Context, client.NodeInfo, client.MultiAddressClient) error) *PutInitPrm { if p != nil { p.relay = f } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index fed161e03..e355990a3 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -23,7 +23,7 @@ type Streamer struct { target transformer.ObjectTarget - relay func(client.NodeInfo, client.MultiAddressClient) error + relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error maxPayloadSz uint64 // network config } @@ -197,9 +197,9 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error { } func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { - var relay func(nodeDesc) error + var relay func(context.Context, nodeDesc) error if p.relay != nil { - relay = func(node nodeDesc) error { + relay = func(ctx context.Context, node nodeDesc) error { var info client.NodeInfo client.NodeInfoFromNetmapElement(&info, node.info) @@ -209,7 +209,7 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) } - return p.relay(info, c) + return p.relay(ctx, info, c) } } diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 65846ea9f..3b8d7b88c 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -120,7 +120,7 @@ func (s *streamer) CloseAndRecv(ctx context.Context) (*object.PutResponse, error return fromPutResponse(resp), nil } -func (s *streamer) relayRequest(info client.NodeInfo, c client.MultiAddressClient) error { +func (s *streamer) relayRequest(ctx context.Context, info client.NodeInfo, c client.MultiAddressClient) error { // open stream resp := new(object.PutResponse) @@ -144,7 +144,7 @@ func (s *streamer) relayRequest(info client.NodeInfo, c client.MultiAddressClien var stream *rpc.PutRequestWriter err = c.RawForAddress(addr, func(cli *rawclient.Client) error { - stream, err = rpc.PutObject(cli, resp) + stream, err = rpc.PutObject(cli, resp, rawclient.WithContext(ctx)) return err }) if err != nil { From d62c6e4ce68f661ef7b677098a3734392e88686a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 12 Apr 2023 17:01:29 +0300 Subject: [PATCH 0111/1943] [#242] node: Add tracing spans Add tracing spans for PUT requests. Add tracing spans for DELETE requests. Add tracing spans for SELECT requests. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-lens/internal/meta/inspect.go | 4 +- cmd/frostfs-node/notificator.go | 2 +- cmd/frostfs-node/object.go | 24 ++++++------ pkg/core/object/fmt.go | 13 ++++--- pkg/core/object/fmt_test.go | 23 +++++------ .../blobovnicza/blobovnicza_test.go | 2 +- .../blobovnicza/delete.go | 13 ++++++- .../blobstor/blobovniczatree/delete.go | 30 +++++++++----- .../blobstor/blobovniczatree/exists_test.go | 2 +- .../blobstor/blobovniczatree/put.go | 13 ++++++- .../blobstor/blobstor_test.go | 4 +- .../blobstor/common/storage.go | 4 +- pkg/local_object_storage/blobstor/delete.go | 18 +++++++-- .../blobstor/exists_test.go | 2 +- .../blobstor/fstree/fstree.go | 17 +++++++- .../blobstor/internal/blobstortest/common.go | 3 +- .../blobstor/internal/blobstortest/control.go | 4 +- .../blobstor/internal/blobstortest/delete.go | 10 ++--- .../blobstor/internal/blobstortest/iterate.go | 3 +- .../blobstor/iterate_test.go | 3 +- .../blobstor/memstore/memstore.go | 4 +- .../blobstor/memstore/memstore_test.go | 4 +- .../blobstor/perf_test.go | 6 +-- pkg/local_object_storage/blobstor/put.go | 15 ++++++- .../blobstor/teststore/teststore.go | 8 ++-- pkg/local_object_storage/engine/control.go | 2 +- .../engine/control_test.go | 2 +- pkg/local_object_storage/engine/delete.go | 12 +++++- .../engine/delete_test.go | 4 +- .../engine/engine_test.go | 4 +- pkg/local_object_storage/engine/error_test.go | 6 +-- .../engine/evacuate_test.go | 4 +- pkg/local_object_storage/engine/exists.go | 4 +- pkg/local_object_storage/engine/get.go | 12 +++--- pkg/local_object_storage/engine/head_test.go | 4 +- pkg/local_object_storage/engine/inhume.go | 20 ++++++++-- .../engine/inhume_test.go | 10 ++--- pkg/local_object_storage/engine/list_test.go | 3 +- pkg/local_object_storage/engine/lock.go | 27 +++++++++---- pkg/local_object_storage/engine/lock_test.go | 20 +++++----- pkg/local_object_storage/engine/put.go | 36 +++++++++-------- .../engine/remove_copies.go | 2 +- .../engine/remove_copies_test.go | 12 +++--- pkg/local_object_storage/engine/restore.go | 19 +++++++-- pkg/local_object_storage/engine/select.go | 23 ++++++++--- pkg/local_object_storage/engine/tree_test.go | 5 ++- pkg/local_object_storage/engine/writecache.go | 16 +++++++- .../metabase/control_test.go | 3 +- .../metabase/counter_test.go | 17 ++++---- pkg/local_object_storage/metabase/delete.go | 12 +++++- .../metabase/delete_test.go | 3 +- pkg/local_object_storage/metabase/exists.go | 12 +++++- pkg/local_object_storage/metabase/get.go | 13 ++++++- pkg/local_object_storage/metabase/get_test.go | 9 +++-- .../metabase/graveyard_test.go | 13 ++++--- pkg/local_object_storage/metabase/inhume.go | 7 +++- .../metabase/inhume_test.go | 21 +++++----- .../metabase/iterators_test.go | 9 +++-- pkg/local_object_storage/metabase/lock.go | 22 ++++++++++- .../metabase/lock_test.go | 39 ++++++++++--------- pkg/local_object_storage/metabase/movable.go | 12 +++++- .../metabase/movable_test.go | 3 +- pkg/local_object_storage/metabase/put.go | 12 +++++- pkg/local_object_storage/metabase/put_test.go | 3 +- pkg/local_object_storage/metabase/select.go | 12 +++++- .../metabase/select_test.go | 5 ++- .../metabase/storage_id.go | 12 +++++- .../metabase/storage_id_test.go | 3 +- pkg/local_object_storage/shard/control.go | 29 ++++++++------ .../shard/control_test.go | 15 +++---- pkg/local_object_storage/shard/delete.go | 25 ++++++++---- pkg/local_object_storage/shard/delete_test.go | 8 ++-- pkg/local_object_storage/shard/dump_test.go | 24 ++++++------ pkg/local_object_storage/shard/exists.go | 12 +++++- pkg/local_object_storage/shard/gc.go | 10 ++--- pkg/local_object_storage/shard/gc_test.go | 6 +-- pkg/local_object_storage/shard/get.go | 8 ++-- pkg/local_object_storage/shard/get_test.go | 6 +-- pkg/local_object_storage/shard/head.go | 2 +- pkg/local_object_storage/shard/head_test.go | 4 +- pkg/local_object_storage/shard/inhume.go | 13 ++++++- pkg/local_object_storage/shard/inhume_test.go | 2 +- pkg/local_object_storage/shard/list.go | 3 +- pkg/local_object_storage/shard/list_test.go | 3 +- pkg/local_object_storage/shard/lock.go | 28 +++++++++++-- pkg/local_object_storage/shard/lock_test.go | 14 +++---- .../shard/metrics_test.go | 5 ++- pkg/local_object_storage/shard/move.go | 16 +++++++- pkg/local_object_storage/shard/put.go | 19 +++++++-- pkg/local_object_storage/shard/range.go | 2 +- pkg/local_object_storage/shard/range_test.go | 2 +- pkg/local_object_storage/shard/reload_test.go | 13 ++++--- pkg/local_object_storage/shard/restore.go | 17 ++++++-- pkg/local_object_storage/shard/select.go | 15 ++++++- .../shard/shutdown_test.go | 2 +- pkg/local_object_storage/shard/writecache.go | 16 +++++++- pkg/local_object_storage/writecache/delete.go | 15 ++++++- pkg/local_object_storage/writecache/flush.go | 36 ++++++++++------- .../writecache/flush_test.go | 24 ++++++------ pkg/local_object_storage/writecache/init.go | 24 ++++++------ pkg/local_object_storage/writecache/mode.go | 18 +++++++-- .../writecache/options.go | 6 +-- pkg/local_object_storage/writecache/put.go | 19 +++++++-- .../writecache/storage.go | 3 +- .../writecache/writecache.go | 14 ++++--- pkg/services/control/server/flush_cache.go | 4 +- pkg/services/control/server/restore.go | 4 +- pkg/services/object/delete/util.go | 2 +- pkg/services/object/internal/client/client.go | 4 ++ pkg/services/object/put/distributed.go | 2 +- pkg/services/object/put/local.go | 10 ++--- pkg/services/object/put/streamer.go | 4 +- pkg/services/object/put/v2/streamer.go | 20 +++++++++- pkg/services/object/put/validation.go | 6 +-- pkg/services/object/search/local.go | 6 ++- pkg/services/object/search/search.go | 2 +- pkg/services/object/search/search_test.go | 2 +- pkg/services/object/search/service.go | 2 +- pkg/services/object/search/util.go | 4 +- .../object_manager/transformer/fmt.go | 4 +- .../object_manager/transformer/transformer.go | 4 +- .../object_manager/transformer/types.go | 2 +- 122 files changed, 863 insertions(+), 417 deletions(-) diff --git a/cmd/frostfs-lens/internal/meta/inspect.go b/cmd/frostfs-lens/internal/meta/inspect.go index fb0065a62..bc7f28a3a 100644 --- a/cmd/frostfs-lens/internal/meta/inspect.go +++ b/cmd/frostfs-lens/internal/meta/inspect.go @@ -36,7 +36,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { storageID := meta.StorageIDPrm{} storageID.SetAddress(addr) - resStorageID, err := db.StorageID(storageID) + resStorageID, err := db.StorageID(cmd.Context(), storageID) common.ExitOnErr(cmd, common.Errf("could not check if the obj is small: %w", err)) if id := resStorageID.StorageID(); id != nil { @@ -51,7 +51,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { siErr := new(object.SplitInfoError) - res, err := db.Get(prm) + res, err := db.Get(cmd.Context(), prm) if errors.As(err, &siErr) { link, linkSet := siErr.SplitInfo().Link() last, lastSet := siErr.SplitInfo().LastPart() diff --git a/cmd/frostfs-node/notificator.go b/cmd/frostfs-node/notificator.go index 9c90e052c..358b39a72 100644 --- a/cmd/frostfs-node/notificator.go +++ b/cmd/frostfs-node/notificator.go @@ -42,7 +42,7 @@ func (n *notificationSource) Iterate(ctx context.Context, epoch uint64, handler for _, c := range listRes.Containers() { selectPrm.WithContainerID(c) - selectRes, err := n.e.Select(selectPrm) + selectRes, err := n.e.Select(ctx, selectPrm) if err != nil { log.Error(logs.FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer, zap.Stringer("cid", c), diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 8f5a83eb0..08a202df9 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -617,20 +617,20 @@ type engineWithNotifications struct { defaultTopic string } -func (e engineWithNotifications) IsLocked(address oid.Address) (bool, error) { - return e.base.IsLocked(address) +func (e engineWithNotifications) IsLocked(ctx context.Context, address oid.Address) (bool, error) { + return e.base.IsLocked(ctx, address) } func (e engineWithNotifications) Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error { return e.base.Delete(ctx, tombstone, toDelete) } -func (e engineWithNotifications) Lock(locker oid.Address, toLock []oid.ID) error { - return e.base.Lock(locker, toLock) +func (e engineWithNotifications) Lock(ctx context.Context, locker oid.Address, toLock []oid.ID) error { + return e.base.Lock(ctx, locker, toLock) } -func (e engineWithNotifications) Put(o *objectSDK.Object) error { - if err := e.base.Put(o); err != nil { +func (e engineWithNotifications) Put(ctx context.Context, o *objectSDK.Object) error { + if err := e.base.Put(ctx, o); err != nil { return err } @@ -654,8 +654,8 @@ type engineWithoutNotifications struct { engine *engine.StorageEngine } -func (e engineWithoutNotifications) IsLocked(address oid.Address) (bool, error) { - return e.engine.IsLocked(address) +func (e engineWithoutNotifications) IsLocked(ctx context.Context, address oid.Address) (bool, error) { + return e.engine.IsLocked(ctx, address) } func (e engineWithoutNotifications) Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error { @@ -673,10 +673,10 @@ func (e engineWithoutNotifications) Delete(ctx context.Context, tombstone oid.Ad return err } -func (e engineWithoutNotifications) Lock(locker oid.Address, toLock []oid.ID) error { - return e.engine.Lock(locker.Container(), locker.Object(), toLock) +func (e engineWithoutNotifications) Lock(ctx context.Context, locker oid.Address, toLock []oid.ID) error { + return e.engine.Lock(ctx, locker.Container(), locker.Object(), toLock) } -func (e engineWithoutNotifications) Put(o *objectSDK.Object) error { - return engine.Put(e.engine, o) +func (e engineWithoutNotifications) Put(ctx context.Context, o *objectSDK.Object) error { + return engine.Put(ctx, e.engine, o) } diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index 33373b7cc..ef99f3058 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -1,6 +1,7 @@ package object import ( + "context" "crypto/ecdsa" "errors" "fmt" @@ -42,7 +43,7 @@ type DeleteHandler interface { // LockSource is a source of lock relations between the objects. type LockSource interface { // IsLocked must clarify object's lock status. - IsLocked(address oid.Address) (bool, error) + IsLocked(ctx context.Context, address oid.Address) (bool, error) } // Locker is an object lock storage interface. @@ -89,7 +90,7 @@ func NewFormatValidator(opts ...FormatValidatorOption) *FormatValidator { // If unprepared is true, only fields set by user are validated. // // Returns nil error if the object has valid structure. -func (v *FormatValidator) Validate(obj *object.Object, unprepared bool) error { +func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unprepared bool) error { if obj == nil { return errNilObject } @@ -117,7 +118,7 @@ func (v *FormatValidator) Validate(obj *object.Object, unprepared bool) error { return fmt.Errorf("(%T) could not validate signature key: %w", v, err) } - if err := v.checkExpiration(obj); err != nil { + if err := v.checkExpiration(ctx, obj); err != nil { return fmt.Errorf("object did not pass expiration check: %w", err) } @@ -128,7 +129,7 @@ func (v *FormatValidator) Validate(obj *object.Object, unprepared bool) error { if obj = obj.Parent(); obj != nil { // Parent object already exists. - return v.Validate(obj, false) + return v.Validate(ctx, obj, false) } return nil @@ -327,7 +328,7 @@ func (v *FormatValidator) fillAndValidateTombstoneMeta(o *object.Object, meta *C var errExpired = errors.New("object has expired") -func (v *FormatValidator) checkExpiration(obj *object.Object) error { +func (v *FormatValidator) checkExpiration(ctx context.Context, obj *object.Object) error { exp, err := expirationEpochAttribute(obj) if err != nil { if errors.Is(err, errNoExpirationEpoch) { @@ -348,7 +349,7 @@ func (v *FormatValidator) checkExpiration(obj *object.Object) error { addr.SetContainer(cID) addr.SetObject(oID) - locked, err := v.e.IsLocked(addr) + locked, err := v.e.IsLocked(ctx, addr) if err != nil { return fmt.Errorf("locking status check for an expired object: %w", err) } diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index 563c7827d..be0602540 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -1,6 +1,7 @@ package object import ( + "context" "crypto/ecdsa" "strconv" "testing" @@ -40,7 +41,7 @@ type testLockSource struct { m map[oid.Address]bool } -func (t testLockSource) IsLocked(address oid.Address) (bool, error) { +func (t testLockSource) IsLocked(_ context.Context, address oid.Address) (bool, error) { return t.m[address], nil } @@ -62,20 +63,20 @@ func TestFormatValidator_Validate(t *testing.T) { require.NoError(t, err) t.Run("nil input", func(t *testing.T) { - require.Error(t, v.Validate(nil, true)) + require.Error(t, v.Validate(context.Background(), nil, true)) }) t.Run("nil identifier", func(t *testing.T) { obj := object.New() - require.ErrorIs(t, v.Validate(obj, false), errNilID) + require.ErrorIs(t, v.Validate(context.Background(), obj, false), errNilID) }) t.Run("nil container identifier", func(t *testing.T) { obj := object.New() obj.SetID(oidtest.ID()) - require.ErrorIs(t, v.Validate(obj, true), errNilCID) + require.ErrorIs(t, v.Validate(context.Background(), obj, true), errNilCID) }) t.Run("unsigned object", func(t *testing.T) { @@ -83,7 +84,7 @@ func TestFormatValidator_Validate(t *testing.T) { obj.SetContainerID(cidtest.ID()) obj.SetID(oidtest.ID()) - require.Error(t, v.Validate(obj, false)) + require.Error(t, v.Validate(context.Background(), obj, false)) }) t.Run("correct w/ session token", func(t *testing.T) { @@ -101,7 +102,7 @@ func TestFormatValidator_Validate(t *testing.T) { require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) - require.NoError(t, v.Validate(obj, false)) + require.NoError(t, v.Validate(context.Background(), obj, false)) }) t.Run("correct w/o session token", func(t *testing.T) { @@ -109,7 +110,7 @@ func TestFormatValidator_Validate(t *testing.T) { require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) - require.NoError(t, v.Validate(obj, false)) + require.NoError(t, v.Validate(context.Background(), obj, false)) }) t.Run("tombstone content", func(t *testing.T) { @@ -236,7 +237,7 @@ func TestFormatValidator_Validate(t *testing.T) { t.Run("invalid attribute value", func(t *testing.T) { val := "text" - err := v.Validate(fn(val), false) + err := v.Validate(context.Background(), fn(val), false) require.Error(t, err) }) @@ -245,7 +246,7 @@ func TestFormatValidator_Validate(t *testing.T) { obj := fn(val) t.Run("non-locked", func(t *testing.T) { - err := v.Validate(obj, false) + err := v.Validate(context.Background(), obj, false) require.ErrorIs(t, err, errExpired) }) @@ -258,14 +259,14 @@ func TestFormatValidator_Validate(t *testing.T) { addr.SetObject(oID) ls.m[addr] = true - err := v.Validate(obj, false) + err := v.Validate(context.Background(), obj, false) require.NoError(t, err) }) }) t.Run("alive object", func(t *testing.T) { val := strconv.FormatUint(curEpoch, 10) - err := v.Validate(fn(val), true) + err := v.Validate(context.Background(), fn(val), true) require.NoError(t, err) }) }) diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index 853628fb4..5deaf5e4a 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -88,7 +88,7 @@ func TestBlobovnicza(t *testing.T) { var dPrm DeletePrm dPrm.SetAddress(addr) - _, err := blz.Delete(dPrm) + _, err := blz.Delete(context.Background(), dPrm) require.NoError(t, err) // should return 404 diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 6ce6f349c..29a587cc9 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -1,10 +1,15 @@ package blobovnicza import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -30,7 +35,13 @@ func (p *DeletePrm) SetAddress(addr oid.Address) { // Returns an error of type apistatus.ObjectNotFound if the object to be deleted is not in blobovnicza. // // Should not be called in read-only configuration. -func (b *Blobovnicza) Delete(prm DeletePrm) (DeleteRes, error) { +func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Delete", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + addrKey := addressKey(prm.addr) removed := false diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 202807653..073e69b57 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -1,13 +1,18 @@ package blobovniczatree import ( + "context" + "encoding/hex" "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -15,7 +20,14 @@ import ( // // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. -func (b *Blobovniczas) Delete(prm common.DeletePrm) (res common.DeleteRes, err error) { +func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res common.DeleteRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Delete", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + )) + defer span.End() + if b.readOnly { return common.DeleteRes{}, common.ErrReadOnly } @@ -30,7 +42,7 @@ func (b *Blobovniczas) Delete(prm common.DeletePrm) (res common.DeleteRes, err e return res, err } - return b.deleteObject(blz, bPrm, prm) + return b.deleteObject(ctx, blz, bPrm, prm) } activeCache := make(map[string]struct{}) @@ -42,7 +54,7 @@ func (b *Blobovniczas) Delete(prm common.DeletePrm) (res common.DeleteRes, err e // don't process active blobovnicza of the level twice _, ok := activeCache[dirPath] - res, err = b.deleteObjectFromLevel(bPrm, p, !ok, prm) + res, err = b.deleteObjectFromLevel(ctx, bPrm, p, !ok, prm) if err != nil { if !blobovnicza.IsErrNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, @@ -73,7 +85,7 @@ func (b *Blobovniczas) Delete(prm common.DeletePrm) (res common.DeleteRes, err e // tries to delete object from particular blobovnicza. // // returns no error if object was removed from some blobovnicza of the same level. -func (b *Blobovniczas) deleteObjectFromLevel(prm blobovnicza.DeletePrm, blzPath string, tryActive bool, dp common.DeletePrm) (common.DeleteRes, error) { +func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicza.DeletePrm, blzPath string, tryActive bool, dp common.DeletePrm) (common.DeleteRes, error) { lvlPath := filepath.Dir(blzPath) // try to remove from blobovnicza if it is opened @@ -81,7 +93,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(prm blobovnicza.DeletePrm, blzPath v, ok := b.opened.Get(blzPath) b.lruMtx.Unlock() if ok { - if res, err := b.deleteObject(v, prm, dp); err == nil { + if res, err := b.deleteObject(ctx, v, prm, dp); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza, @@ -100,7 +112,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(prm blobovnicza.DeletePrm, blzPath b.activeMtx.RUnlock() if ok && tryActive { - if res, err := b.deleteObject(active.blz, prm, dp); err == nil { + if res, err := b.deleteObject(ctx, active.blz, prm, dp); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza, @@ -125,11 +137,11 @@ func (b *Blobovniczas) deleteObjectFromLevel(prm blobovnicza.DeletePrm, blzPath return common.DeleteRes{}, err } - return b.deleteObject(blz, prm, dp) + return b.deleteObject(ctx, blz, prm, dp) } // removes object from blobovnicza and returns common.DeleteRes. -func (b *Blobovniczas) deleteObject(blz *blobovnicza.Blobovnicza, prm blobovnicza.DeletePrm, dp common.DeletePrm) (common.DeleteRes, error) { - _, err := blz.Delete(prm) +func (b *Blobovniczas) deleteObject(ctx context.Context, blz *blobovnicza.Blobovnicza, prm blobovnicza.DeletePrm, dp common.DeletePrm) (common.DeleteRes, error) { + _, err := blz.Delete(ctx, prm) return common.DeleteRes{}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 0c7c61d76..ff927ccbb 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -33,7 +33,7 @@ func TestExistsInvalidStorageID(t *testing.T) { d, err := obj.Marshal() require.NoError(t, err) - putRes, err := b.Put(common.PutPrm{Address: addr, RawData: d, DontCompress: true}) + putRes, err := b.Put(context.Background(), common.PutPrm{Address: addr, RawData: d, DontCompress: true}) require.NoError(t, err) t.Run("valid but wrong storage id", func(t *testing.T) { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 8b29119c6..ec302d143 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -1,20 +1,31 @@ package blobovniczatree import ( + "context" "errors" "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) // Put saves object in the maximum weight blobobnicza. // // returns error if could not save object in any blobovnicza. -func (b *Blobovniczas) Put(prm common.PutPrm) (common.PutRes, error) { +func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Put", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.Bool("dont_compress", prm.DontCompress), + )) + defer span.End() + if b.readOnly { return common.PutRes{}, common.ErrReadOnly } diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index 738cd7eee..8c6766dc1 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -75,12 +75,12 @@ func TestCompression(t *testing.T) { testPut := func(t *testing.T, b *BlobStor, i int) { var prm common.PutPrm prm.Object = smallObj[i] - _, err := b.Put(prm) + _, err := b.Put(context.Background(), prm) require.NoError(t, err) prm = common.PutPrm{} prm.Object = bigObj[i] - _, err = b.Put(prm) + _, err = b.Put(context.Background(), prm) require.NoError(t, err) } diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index b5d186242..801d32c1e 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -23,7 +23,7 @@ type Storage interface { Get(context.Context, GetPrm) (GetRes, error) GetRange(context.Context, GetRangePrm) (GetRangeRes, error) Exists(context.Context, ExistsPrm) (ExistsRes, error) - Put(PutPrm) (PutRes, error) - Delete(DeletePrm) (DeleteRes, error) + Put(context.Context, PutPrm) (PutRes, error) + Delete(context.Context, DeletePrm) (DeleteRes, error) Iterate(IteratePrm) (IterateRes, error) } diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index 8c5a7aba6..377214fb8 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -1,19 +1,31 @@ package blobstor import ( + "context" + "encoding/hex" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) -func (b *BlobStor) Delete(prm common.DeletePrm) (common.DeleteRes, error) { +func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Delete", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + )) + defer span.End() + b.modeMtx.RLock() defer b.modeMtx.RUnlock() if prm.StorageID == nil { for i := range b.storage { - res, err := b.storage[i].Storage.Delete(prm) + res, err := b.storage[i].Storage.Delete(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { if err == nil { logOp(b.log, deleteOp, prm.Address, b.storage[i].Storage.Type(), prm.StorageID) @@ -31,7 +43,7 @@ func (b *BlobStor) Delete(prm common.DeletePrm) (common.DeleteRes, error) { st = b.storage[0].Storage } - res, err := st.Delete(prm) + res, err := st.Delete(ctx, prm) if err == nil { logOp(b.log, deleteOp, prm.Address, st.Type(), prm.StorageID) } diff --git a/pkg/local_object_storage/blobstor/exists_test.go b/pkg/local_object_storage/blobstor/exists_test.go index 805d78297..f5c5fbbef 100644 --- a/pkg/local_object_storage/blobstor/exists_test.go +++ b/pkg/local_object_storage/blobstor/exists_test.go @@ -36,7 +36,7 @@ func TestExists(t *testing.T) { for i := range objects { var prm common.PutPrm prm.Object = objects[i] - _, err := b.Put(prm) + _, err := b.Put(context.Background(), prm) require.NoError(t, err) } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 462fbd63f..8eb0d5be9 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -196,7 +196,13 @@ func (t *FSTree) treePath(addr oid.Address) string { } // Delete removes the object with the specified address from the storage. -func (t *FSTree) Delete(prm common.DeletePrm) (common.DeleteRes, error) { +func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Delete", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + )) + defer span.End() + if t.readOnly { return common.DeleteRes{}, common.ErrReadOnly } @@ -230,7 +236,14 @@ func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exist } // Put puts an object in the storage. -func (t *FSTree) Put(prm common.PutPrm) (common.PutRes, error) { +func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Put", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.Bool("dont_compress", prm.DontCompress), + )) + defer span.End() + if t.readOnly { return common.PutRes{}, common.ErrReadOnly } diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/common.go b/pkg/local_object_storage/blobstor/internal/blobstortest/common.go index b2663be21..e31f3280a 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/common.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/common.go @@ -1,6 +1,7 @@ package blobstortest import ( + "context" "math/rand" "testing" @@ -67,7 +68,7 @@ func prepare(t *testing.T, count int, s common.Storage, min, max uint64) []objec prm.Object = objects[i].obj prm.RawData = objects[i].raw - putRes, err := s.Put(prm) + putRes, err := s.Put(context.Background(), prm) require.NoError(t, err) objects[i].storageID = putRes.StorageID diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go index 350bea96a..96d54dec3 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go @@ -36,7 +36,7 @@ func TestControl(t *testing.T, cons Constructor, min, max uint64) { prm.Object = NewObject(min + uint64(rand.Intn(int(max-min+1)))) prm.Address = objectCore.AddressOf(prm.Object) - _, err := s.Put(prm) + _, err := s.Put(context.Background(), prm) require.ErrorIs(t, err, common.ErrReadOnly) }) t.Run("delete fails", func(t *testing.T) { @@ -44,7 +44,7 @@ func TestControl(t *testing.T, cons Constructor, min, max uint64) { prm.Address = objects[0].addr prm.StorageID = objects[0].storageID - _, err := s.Delete(prm) + _, err := s.Delete(context.Background(), prm) require.ErrorIs(t, err, common.ErrReadOnly) }) } diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go index ad0045316..7532a5b5f 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go @@ -22,7 +22,7 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { var prm common.DeletePrm prm.Address = oidtest.Address() - _, err := s.Delete(prm) + _, err := s.Delete(context.Background(), prm) require.Error(t, err, new(apistatus.ObjectNotFound)) }) @@ -31,7 +31,7 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { prm.Address = objects[0].addr prm.StorageID = objects[0].storageID - _, err := s.Delete(prm) + _, err := s.Delete(context.Background(), prm) require.NoError(t, err) t.Run("exists fail", func(t *testing.T) { @@ -55,7 +55,7 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { var prm common.DeletePrm prm.Address = objects[1].addr - _, err := s.Delete(prm) + _, err := s.Delete(context.Background(), prm) require.NoError(t, err) }) @@ -64,10 +64,10 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { prm.Address = objects[2].addr prm.StorageID = objects[2].storageID - _, err := s.Delete(prm) + _, err := s.Delete(context.Background(), prm) require.NoError(t, err) - _, err = s.Delete(prm) + _, err = s.Delete(context.Background(), prm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) }) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index f98cca638..83ada9607 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -1,6 +1,7 @@ package blobstortest import ( + "context" "errors" "testing" @@ -22,7 +23,7 @@ func TestIterate(t *testing.T, cons Constructor, min, max uint64) { var delPrm common.DeletePrm delPrm.Address = objects[2].addr delPrm.StorageID = objects[2].storageID - _, err := s.Delete(delPrm) + _, err := s.Delete(context.Background(), delPrm) require.NoError(t, err) objects = append(objects[:delID], objects[delID+1:]...) diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index b2a7ddfb9..6488ff5fc 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -1,6 +1,7 @@ package blobstor import ( + "context" "encoding/binary" "os" "testing" @@ -63,7 +64,7 @@ func TestIterateObjects(t *testing.T) { } for _, v := range mObjs { - _, err := blobStor.Put(common.PutPrm{Address: v.addr, RawData: v.data}) + _, err := blobStor.Put(context.Background(), common.PutPrm{Address: v.addr, RawData: v.data}) require.NoError(t, err) } diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index 4068d742e..e435cfef4 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -91,7 +91,7 @@ func (s *memstoreImpl) Exists(_ context.Context, req common.ExistsPrm) (common.E return common.ExistsRes{Exists: exists}, nil } -func (s *memstoreImpl) Put(req common.PutPrm) (common.PutRes, error) { +func (s *memstoreImpl) Put(_ context.Context, req common.PutPrm) (common.PutRes, error) { if s.readOnly { return common.PutRes{}, common.ErrReadOnly } @@ -108,7 +108,7 @@ func (s *memstoreImpl) Put(req common.PutPrm) (common.PutRes, error) { return common.PutRes{StorageID: []byte(s.rootPath)}, nil } -func (s *memstoreImpl) Delete(req common.DeletePrm) (common.DeleteRes, error) { +func (s *memstoreImpl) Delete(_ context.Context, req common.DeletePrm) (common.DeleteRes, error) { if s.readOnly { return common.DeleteRes{}, common.ErrReadOnly } diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index 6482b2cff..125276290 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -28,7 +28,7 @@ func TestSimpleLifecycle(t *testing.T) { require.NoError(t, err) { - _, err := s.Put(common.PutPrm{Address: addr, RawData: d, DontCompress: true}) + _, err := s.Put(context.Background(), common.PutPrm{Address: addr, RawData: d, DontCompress: true}) require.NoError(t, err) } @@ -57,7 +57,7 @@ func TestSimpleLifecycle(t *testing.T) { } { - _, err := s.Delete(common.DeletePrm{Address: addr}) + _, err := s.Delete(context.Background(), common.DeletePrm{Address: addr}) require.NoError(t, err) } diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index d2359335f..c88dc85ed 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -114,7 +114,7 @@ func BenchmarkSubstorageReadPerf(b *testing.B) { if err != nil { return fmt.Errorf("marshal: %v", err) } - _, err = st.Put(common.PutPrm{ + _, err = st.Put(context.Background(), common.PutPrm{ Address: addr, RawData: raw, }) @@ -165,7 +165,7 @@ func BenchmarkSubstorageWritePerf(b *testing.B) { addr := testutil.AddressFromObject(b, obj) raw, err := obj.Marshal() require.NoError(b, err) - if _, err := st.Put(common.PutPrm{ + if _, err := st.Put(context.Background(), common.PutPrm{ Address: addr, RawData: raw, }); err != nil { @@ -202,7 +202,7 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) { addr := testutil.AddressFromObject(b, obj) raw, err := obj.Marshal() require.NoError(b, err) - if _, err := st.Put(common.PutPrm{ + if _, err := st.Put(context.Background(), common.PutPrm{ Address: addr, RawData: raw, }); err != nil { diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index a4009ae43..2ae7f0fe6 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -1,12 +1,16 @@ package blobstor import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ErrNoPlaceFound is returned when object can't be saved to any sub-storage component @@ -21,7 +25,14 @@ var ErrNoPlaceFound = logicerr.New("couldn't find a place to store an object") // // Returns any error encountered that // did not allow to completely save the object. -func (b *BlobStor) Put(prm common.PutPrm) (common.PutRes, error) { +func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Put", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.Bool("dont_compress", prm.DontCompress), + )) + defer span.End() + b.modeMtx.RLock() defer b.modeMtx.RUnlock() @@ -39,7 +50,7 @@ func (b *BlobStor) Put(prm common.PutPrm) (common.PutRes, error) { for i := range b.storage { if b.storage[i].Policy == nil || b.storage[i].Policy(prm.Object, prm.RawData) { - res, err := b.storage[i].Storage.Put(prm) + res, err := b.storage[i].Storage.Put(ctx, prm) if err == nil { logOp(b.log, putOp, prm.Address, b.storage[i].Storage.Type(), res.StorageID) } diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 03f64f0f1..24d742fda 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -176,27 +176,27 @@ func (s *TestStore) Exists(ctx context.Context, req common.ExistsPrm) (common.Ex } } -func (s *TestStore) Put(req common.PutPrm) (common.PutRes, error) { +func (s *TestStore) Put(ctx context.Context, req common.PutPrm) (common.PutRes, error) { s.mu.RLock() defer s.mu.RUnlock() switch { case s.overrides.Put != nil: return s.overrides.Put(req) case s.st != nil: - return s.st.Put(req) + return s.st.Put(ctx, req) default: panic(fmt.Sprintf("unexpected storage call: Put(%+v)", req)) } } -func (s *TestStore) Delete(req common.DeletePrm) (common.DeleteRes, error) { +func (s *TestStore) Delete(ctx context.Context, req common.DeletePrm) (common.DeleteRes, error) { s.mu.RLock() defer s.mu.RUnlock() switch { case s.overrides.Delete != nil: return s.overrides.Delete(req) case s.st != nil: - return s.st.Delete(req) + return s.st.Delete(ctx, req) default: panic(fmt.Sprintf("unexpected storage call: Delete(%+v)", req)) } diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 0c422ccc8..9ad4fcf9c 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -308,7 +308,7 @@ loop: e.removeShards(shardsToRemove...) for _, p := range shardsToReload { - err := p.sh.Reload(p.opts...) + err := p.sh.Reload(ctx, p.opts...) if err != nil { e.log.Error(logs.EngineCouldNotReloadAShard, zap.Stringer("shard id", p.sh.ID()), diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 91bec63a6..046968dbe 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -204,7 +204,7 @@ func TestExecBlocks(t *testing.T) { addr := object.AddressOf(obj) - require.NoError(t, Put(e, obj)) + require.NoError(t, Put(context.Background(), e, obj)) // block executions errBlock := errors.New("block exec err") diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 1f3c142a5..f9b9c9a87 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -4,11 +4,14 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -47,6 +50,13 @@ func (p *DeletePrm) WithForceRemoval() { // on operations with that object) if WithForceRemoval option has // been provided. func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Delete", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + attribute.Bool("force_removal", prm.forceRemoval), + )) + defer span.End() + err = e.execIfNotBlocked(func() error { res, err = e.delete(ctx, prm) return err @@ -135,7 +145,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo } e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { - res, err := sh.Select(selectPrm) + res, err := sh.Select(ctx, selectPrm) if err != nil { e.log.Warn(logs.EngineErrorDuringSearchingForObjectChildren, zap.Stringer("addr", addr), diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 259a40a7c..53c62981c 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -59,9 +59,9 @@ func TestDeleteBigObject(t *testing.T) { defer e.Close() for i := range children { - require.NoError(t, Put(e, children[i])) + require.NoError(t, Put(context.Background(), e, children[i])) } - require.NoError(t, Put(e, link)) + require.NoError(t, Put(context.Background(), e, link)) var splitErr *objectSDK.SplitInfoError diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index ddaf88d18..4d2ddc100 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -60,7 +60,7 @@ func benchmarkExists(b *testing.B, shardNum int) { addr := oidtest.Address() for i := 0; i < 100; i++ { obj := testutil.GenerateObjectWithCID(cidtest.ID()) - err := Put(e, obj) + err := Put(context.Background(), e, obj) if err != nil { b.Fatal(err) } @@ -69,7 +69,7 @@ func benchmarkExists(b *testing.B, shardNum int) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - ok, err := e.exists(addr) + ok, err := e.exists(context.Background(), addr) if err != nil || ok { b.Fatalf("%t %v", ok, err) } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 4ff019e4d..017b635d4 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -98,7 +98,7 @@ func TestErrorReporting(t *testing.T) { var prm shard.PutPrm prm.SetObject(obj) te.ng.mtx.RLock() - _, err := te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) + _, err := te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) te.ng.mtx.RUnlock() require.NoError(t, err) @@ -132,7 +132,7 @@ func TestErrorReporting(t *testing.T) { var prm shard.PutPrm prm.SetObject(obj) te.ng.mtx.RLock() - _, err := te.ng.shards[te.shards[0].id.String()].Put(prm) + _, err := te.ng.shards[te.shards[0].id.String()].Put(context.Background(), prm) te.ng.mtx.RUnlock() require.NoError(t, err) @@ -185,7 +185,7 @@ func TestBlobstorFailback(t *testing.T) { var prm shard.PutPrm prm.SetObject(obj) te.ng.mtx.RLock() - _, err = te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) + _, err = te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) te.ng.mtx.RUnlock() require.NoError(t, err) objs = append(objs, obj) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index c116aeff9..291bc2b78 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -57,7 +57,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err := e.shards[sh.String()].Put(putPrm) + _, err := e.shards[sh.String()].Put(context.Background(), putPrm) require.NoError(t, err) } @@ -67,7 +67,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng var putPrm PutPrm putPrm.WithObject(objects[len(objects)-1]) - _, err := e.Put(putPrm) + err := e.Put(context.Background(), putPrm) require.NoError(t, err) res, err := e.shards[ids[len(ids)-1].String()].List() diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index 3a8e09a6d..6208461e9 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -10,14 +10,14 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -func (e *StorageEngine) exists(addr oid.Address) (bool, error) { +func (e *StorageEngine) exists(ctx context.Context, addr oid.Address) (bool, error) { var shPrm shard.ExistsPrm shPrm.SetAddress(addr) alreadyRemoved := false exists := false e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { - res, err := sh.Exists(context.TODO(), shPrm) + res, err := sh.Exists(ctx, shPrm) if err != nil { if shard.IsErrRemoved(err) { alreadyRemoved = true diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 7d17b50fa..683b7bde8 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -48,6 +48,12 @@ func (r GetRes) Object() *objectSDK.Object { // // Returns an error if executions are blocked (see BlockExecution). func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Get", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + err = e.execIfNotBlocked(func() error { res, err = e.get(ctx, prm) return err @@ -57,12 +63,6 @@ func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err er } func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.get", - trace.WithAttributes( - attribute.String("address", prm.addr.EncodeToString()), - )) - defer span.End() - if e.metrics != nil { defer elapsed(e.metrics.AddGetDuration)() } diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go index e5fd4b04f..bf00c4289 100644 --- a/pkg/local_object_storage/engine/head_test.go +++ b/pkg/local_object_storage/engine/head_test.go @@ -55,11 +55,11 @@ func TestHeadRaw(t *testing.T) { putPrmLink.SetObject(link) // put most left object in one shard - _, err := s1.Put(putPrmLeft) + _, err := s1.Put(context.Background(), putPrmLeft) require.NoError(t, err) // put link object in another shard - _, err = s2.Put(putPrmLink) + _, err = s2.Put(context.Background(), putPrmLink) require.NoError(t, err) // head with raw flag should return SplitInfoError diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 696e78742..b1204ed99 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -4,12 +4,15 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -62,6 +65,9 @@ var errInhumeFailure = errors.New("inhume operation failed") // // Returns an error if executions are blocked (see BlockExecution). func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Inhume") + defer span.End() + err = e.execIfNotBlocked(func() error { res, err = e.inhume(ctx, prm) return err @@ -82,7 +88,7 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e for i := range prm.addrs { if !prm.forceRemoval { - locked, err := e.IsLocked(prm.addrs[i]) + locked, err := e.IsLocked(ctx, prm.addrs[i]) if err != nil { e.log.Warn(logs.EngineRemovingAnObjectWithoutFullLockingCheck, zap.Error(err), @@ -181,13 +187,19 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh } // IsLocked checks whether an object is locked according to StorageEngine's state. -func (e *StorageEngine) IsLocked(addr oid.Address) (bool, error) { +func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.IsLocked", + trace.WithAttributes( + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + var locked bool var err error var outErr error e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { - locked, err = h.Shard.IsLocked(addr) + locked, err = h.Shard.IsLocked(ctx, addr) if err != nil { e.reportShardError(h, "can't check object's lockers", err, zap.Stringer("addr", addr)) outErr = err @@ -206,7 +218,7 @@ func (e *StorageEngine) IsLocked(addr oid.Address) (bool, error) { func (e *StorageEngine) processExpiredTombstones(ctx context.Context, addrs []meta.TombstonedObject) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - sh.HandleExpiredTombstones(addrs) + sh.HandleExpiredTombstones(ctx, addrs) select { case <-ctx.Done(): diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 4f8c96b99..924cf518b 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -42,7 +42,7 @@ func TestStorageEngine_Inhume(t *testing.T) { e := testNewEngine(t).setShardsNum(t, 1).engine defer e.Close() - err := Put(e, parent) + err := Put(context.Background(), e, parent) require.NoError(t, err) var inhumePrm InhumePrm @@ -51,7 +51,7 @@ func TestStorageEngine_Inhume(t *testing.T) { _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) - addrs, err := Select(e, cnr, fs) + addrs, err := Select(context.Background(), e, cnr, fs) require.NoError(t, err) require.Empty(t, addrs) }) @@ -65,12 +65,12 @@ func TestStorageEngine_Inhume(t *testing.T) { var putChild shard.PutPrm putChild.SetObject(child) - _, err := s1.Put(putChild) + _, err := s1.Put(context.Background(), putChild) require.NoError(t, err) var putLink shard.PutPrm putLink.SetObject(link) - _, err = s2.Put(putLink) + _, err = s2.Put(context.Background(), putLink) require.NoError(t, err) var inhumePrm InhumePrm @@ -79,7 +79,7 @@ func TestStorageEngine_Inhume(t *testing.T) { _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) - addrs, err := Select(e, cnr, fs) + addrs, err := Select(context.Background(), e, cnr, fs) require.NoError(t, err) require.Empty(t, addrs) }) diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 1261de9d4..fde799d05 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -1,6 +1,7 @@ package engine import ( + "context" "errors" "os" "sort" @@ -35,7 +36,7 @@ func TestListWithCursor(t *testing.T) { var prm PutPrm prm.WithObject(obj) - _, err := e.Put(prm) + err := e.Put(context.Background(), prm) require.NoError(t, err) expected = append(expected, object.AddressWithType{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)}) } diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 60a1d9c9f..4562c1a57 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -4,12 +4,15 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) var errLockFailed = errors.New("lock operation failed") @@ -20,19 +23,27 @@ var errLockFailed = errors.New("lock operation failed") // Allows locking regular objects only (otherwise returns apistatus.LockNonRegularObject). // // Locked list should be unique. Panics if it is empty. -func (e *StorageEngine) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { +func (e *StorageEngine) Lock(ctx context.Context, idCnr cid.ID, locker oid.ID, locked []oid.ID) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Lock", + trace.WithAttributes( + attribute.String("container_id", idCnr.EncodeToString()), + attribute.String("locker", locker.EncodeToString()), + attribute.Int("locked_count", len(locked)), + )) + defer span.End() + return e.execIfNotBlocked(func() error { - return e.lock(idCnr, locker, locked) + return e.lock(ctx, idCnr, locker, locked) }) } -func (e *StorageEngine) lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { +func (e *StorageEngine) lock(ctx context.Context, idCnr cid.ID, locker oid.ID, locked []oid.ID) error { for i := range locked { - switch e.lockSingle(idCnr, locker, locked[i], true) { + switch e.lockSingle(ctx, idCnr, locker, locked[i], true) { case 1: return logicerr.Wrap(apistatus.LockNonRegularObject{}) case 0: - switch e.lockSingle(idCnr, locker, locked[i], false) { + switch e.lockSingle(ctx, idCnr, locker, locked[i], false) { case 1: return logicerr.Wrap(apistatus.LockNonRegularObject{}) case 0: @@ -48,7 +59,7 @@ func (e *StorageEngine) lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error // - 0: fail // - 1: locking irregular object // - 2: ok -func (e *StorageEngine) lockSingle(idCnr cid.ID, locker, locked oid.ID, checkExists bool) (status uint8) { +func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, locked oid.ID, checkExists bool) (status uint8) { // code is pretty similar to inhumeAddr, maybe unify? root := false var errIrregular apistatus.LockNonRegularObject @@ -70,7 +81,7 @@ func (e *StorageEngine) lockSingle(idCnr cid.ID, locker, locked oid.ID, checkExi var existsPrm shard.ExistsPrm existsPrm.SetAddress(addrLocked) - exRes, err := sh.Exists(context.TODO(), existsPrm) + exRes, err := sh.Exists(ctx, existsPrm) if err != nil { var siErr *objectSDK.SplitInfoError if !errors.As(err, &siErr) { @@ -90,7 +101,7 @@ func (e *StorageEngine) lockSingle(idCnr cid.ID, locker, locked oid.ID, checkExi } } - err := sh.Lock(idCnr, locker, []oid.ID{locked}) + err := sh.Lock(ctx, idCnr, locker, []oid.ID{locked}) if err != nil { e.reportShardError(sh, "could not lock object in shard", err) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index fd3b04ef0..4c89b9226 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -99,7 +99,7 @@ func TestLockUserScenario(t *testing.T) { id, _ := obj.ID() objAddr.SetObject(id) - err = Put(e, obj) + err = Put(context.Background(), e, obj) require.NoError(t, err) // 2. @@ -107,10 +107,10 @@ func TestLockUserScenario(t *testing.T) { locker.WriteMembers([]oid.ID{id}) object.WriteLock(lockerObj, locker) - err = Put(e, lockerObj) + err = Put(context.Background(), e, lockerObj) require.NoError(t, err) - err = e.Lock(cnr, lockerID, []oid.ID{id}) + err = e.Lock(context.Background(), cnr, lockerID, []oid.ID{id}) require.NoError(t, err) // 3. @@ -125,7 +125,7 @@ func TestLockUserScenario(t *testing.T) { tombObj.SetID(tombForLockID) tombObj.SetAttributes(a) - err = Put(e, tombObj) + err = Put(context.Background(), e, tombObj) require.NoError(t, err) inhumePrm.WithTarget(tombForLockAddr, lockerAddr) @@ -180,7 +180,7 @@ func TestLockExpiration(t *testing.T) { // 1. obj := testutil.GenerateObjectWithCID(cnr) - err = Put(e, obj) + err = Put(context.Background(), e, obj) require.NoError(t, err) // 2. @@ -192,13 +192,13 @@ func TestLockExpiration(t *testing.T) { lock.SetType(object.TypeLock) lock.SetAttributes(a) - err = Put(e, lock) + err = Put(context.Background(), e, lock) require.NoError(t, err) id, _ := obj.ID() idLock, _ := lock.ID() - err = e.Lock(cnr, idLock, []oid.ID{id}) + err = e.Lock(context.Background(), cnr, idLock, []oid.ID{id}) require.NoError(t, err) var inhumePrm InhumePrm @@ -255,20 +255,20 @@ func TestLockForceRemoval(t *testing.T) { // 1. obj := testutil.GenerateObjectWithCID(cnr) - err = Put(e, obj) + err = Put(context.Background(), e, obj) require.NoError(t, err) // 2. lock := testutil.GenerateObjectWithCID(cnr) lock.SetType(object.TypeLock) - err = Put(e, lock) + err = Put(context.Background(), e, lock) require.NoError(t, err) id, _ := obj.ID() idLock, _ := lock.ID() - err = e.Lock(cnr, idLock, []oid.ID{id}) + err = e.Lock(context.Background(), cnr, idLock, []oid.ID{id}) require.NoError(t, err) // 3. diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index aea296cc4..0543f9f15 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -12,6 +13,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -20,9 +23,6 @@ type PutPrm struct { obj *objectSDK.Object } -// PutRes groups the resulting values of Put operation. -type PutRes struct{} - var errPutShard = errors.New("could not put object to any shard") // WithObject is a Put option to set object to save. @@ -40,16 +40,22 @@ func (p *PutPrm) WithObject(obj *objectSDK.Object) { // Returns an error if executions are blocked (see BlockExecution). // // Returns an error of type apistatus.ObjectAlreadyRemoved if the object has been marked as removed. -func (e *StorageEngine) Put(prm PutPrm) (res PutRes, err error) { +func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Put", + trace.WithAttributes( + attribute.String("address", object.AddressOf(prm.obj).EncodeToString()), + )) + defer span.End() + err = e.execIfNotBlocked(func() error { - res, err = e.put(prm) + err = e.put(ctx, prm) return err }) return } -func (e *StorageEngine) put(prm PutPrm) (PutRes, error) { +func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { if e.metrics != nil { defer elapsed(e.metrics.AddPutDuration)() } @@ -58,9 +64,9 @@ func (e *StorageEngine) put(prm PutPrm) (PutRes, error) { // In #1146 this check was parallelized, however, it became // much slower on fast machines for 4 shards. - _, err := e.exists(addr) + _, err := e.exists(ctx, addr) if err != nil { - return PutRes{}, err + return err } finished := false @@ -74,7 +80,7 @@ func (e *StorageEngine) put(prm PutPrm) (PutRes, error) { return false } - putDone, exists := e.putToShard(context.TODO(), sh, ind, pool, addr, prm.obj) + putDone, exists := e.putToShard(ctx, sh, ind, pool, addr, prm.obj) finished = putDone || exists return finished }) @@ -83,7 +89,7 @@ func (e *StorageEngine) put(prm PutPrm) (PutRes, error) { err = errPutShard } - return PutRes{}, err + return err } // putToShard puts object to sh. @@ -117,7 +123,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, var toMoveItPrm shard.ToMoveItPrm toMoveItPrm.SetAddress(addr) - _, err = sh.ToMoveIt(toMoveItPrm) + _, err = sh.ToMoveIt(ctx, toMoveItPrm) if err != nil { e.log.Warn(logs.EngineCouldNotMarkObjectForShardRelocation, zap.Stringer("shard", sh.ID()), @@ -132,7 +138,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err = sh.Put(putPrm) + _, err = sh.Put(ctx, putPrm) if err != nil { if errors.Is(err, shard.ErrReadOnlyMode) || errors.Is(err, blobstor.ErrNoPlaceFound) || errors.Is(err, common.ErrReadOnly) || errors.Is(err, common.ErrNoSpace) { @@ -157,11 +163,9 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, } // Put writes provided object to local storage. -func Put(storage *StorageEngine, obj *objectSDK.Object) error { +func Put(ctx context.Context, storage *StorageEngine, obj *objectSDK.Object) error { var putPrm PutPrm putPrm.WithObject(obj) - _, err := storage.Put(putPrm) - - return err + return storage.Put(ctx, putPrm) } diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index d365fc7b4..1ea569928 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -129,7 +129,7 @@ func (e *StorageEngine) removeObjects(ctx context.Context, ch <-chan oid.Address var deletePrm shard.DeletePrm deletePrm.SetAddresses(addr) - _, err = shards[i].Delete(deletePrm) + _, err = shards[i].Delete(ctx, deletePrm) if err != nil { return err } diff --git a/pkg/local_object_storage/engine/remove_copies_test.go b/pkg/local_object_storage/engine/remove_copies_test.go index 4415d01c8..c53e03bbf 100644 --- a/pkg/local_object_storage/engine/remove_copies_test.go +++ b/pkg/local_object_storage/engine/remove_copies_test.go @@ -49,10 +49,10 @@ func TestRebalance(t *testing.T) { te.ng.mtx.RLock() // Every 3rd object (i%3 == 0) is put to both shards, others are distributed. if i%3 != 1 { - _, err1 = te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) + _, err1 = te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) } if i%3 != 2 { - _, err2 = te.ng.shards[te.shards[1].id.String()].Shard.Put(prm) + _, err2 = te.ng.shards[te.shards[1].id.String()].Shard.Put(context.Background(), prm) } te.ng.mtx.RUnlock() @@ -109,8 +109,8 @@ func TestRebalanceSingleThread(t *testing.T) { var prm shard.PutPrm prm.SetObject(obj) te.ng.mtx.RLock() - _, err1 := te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) - _, err2 := te.ng.shards[te.shards[1].id.String()].Shard.Put(prm) + _, err1 := te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) + _, err2 := te.ng.shards[te.shards[1].id.String()].Shard.Put(context.Background(), prm) te.ng.mtx.RUnlock() require.NoError(t, err1) require.NoError(t, err2) @@ -162,8 +162,8 @@ func TestRebalanceExitByContext(t *testing.T) { prm.SetObject(objects[i]) te.ng.mtx.RLock() - _, err1 := te.ng.shards[te.shards[0].id.String()].Shard.Put(prm) - _, err2 := te.ng.shards[te.shards[1].id.String()].Shard.Put(prm) + _, err1 := te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) + _, err2 := te.ng.shards[te.shards[1].id.String()].Shard.Put(context.Background(), prm) te.ng.mtx.RUnlock() require.NoError(t, err1) diff --git a/pkg/local_object_storage/engine/restore.go b/pkg/local_object_storage/engine/restore.go index 84c750cd0..7cc2eaf6c 100644 --- a/pkg/local_object_storage/engine/restore.go +++ b/pkg/local_object_storage/engine/restore.go @@ -1,11 +1,24 @@ package engine -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) // RestoreShard restores objects from dump to the shard with provided identifier. // // Returns an error if shard is not read-only. -func (e *StorageEngine) RestoreShard(id *shard.ID, prm shard.RestorePrm) error { +func (e *StorageEngine) RestoreShard(ctx context.Context, id *shard.ID, prm shard.RestorePrm) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.RestoreShard", + trace.WithAttributes( + attribute.String("shard_id", id.String()), + )) + defer span.End() + e.mtx.RLock() defer e.mtx.RUnlock() @@ -14,6 +27,6 @@ func (e *StorageEngine) RestoreShard(id *shard.ID, prm shard.RestorePrm) error { return errShardNotFound } - _, err := sh.Restore(prm) + _, err := sh.Restore(ctx, prm) return err } diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 7b9b8be60..e1039ea23 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -1,10 +1,15 @@ package engine import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // SelectPrm groups the parameters of Select operation. @@ -38,16 +43,22 @@ func (r SelectRes) AddressList() []oid.Address { // Returns any error encountered that did not allow to completely select the objects. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) Select(prm SelectPrm) (res SelectRes, err error) { +func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Select", + trace.WithAttributes( + attribute.String("container_id", prm.cnr.EncodeToString()), + )) + defer span.End() + err = e.execIfNotBlocked(func() error { - res, err = e._select(prm) + res, err = e._select(ctx, prm) return err }) return } -func (e *StorageEngine) _select(prm SelectPrm) (SelectRes, error) { +func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) { if e.metrics != nil { defer elapsed(e.metrics.AddSearchDuration)() } @@ -62,7 +73,7 @@ func (e *StorageEngine) _select(prm SelectPrm) (SelectRes, error) { shPrm.SetFilters(prm.filters) e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - res, err := sh.Select(shPrm) + res, err := sh.Select(ctx, shPrm) if err != nil { e.reportShardError(sh, "could not select objects from shard", err) return false @@ -133,12 +144,12 @@ func (e *StorageEngine) list(limit uint64) (SelectRes, error) { } // Select selects objects from local storage using provided filters. -func Select(storage *StorageEngine, cnr cid.ID, fs object.SearchFilters) ([]oid.Address, error) { +func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs object.SearchFilters) ([]oid.Address, error) { var selectPrm SelectPrm selectPrm.WithContainerID(cnr) selectPrm.WithFilters(fs) - res, err := storage.Select(selectPrm) + res, err := storage.Select(ctx, selectPrm) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index 611c691f1..77573c9e6 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -1,6 +1,7 @@ package engine import ( + "context" "strconv" "testing" @@ -31,7 +32,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { for i := 0; i < objCount; i++ { obj := testutil.GenerateObjectWithCID(cid) testutil.AddAttribute(obj, pilorama.AttributeFilename, strconv.Itoa(i)) - err := Put(te.ng, obj) + err := Put(context.Background(), te.ng, obj) if err != nil { b.Fatal(err) } @@ -51,7 +52,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { prm.WithFilters(fs) for i := 0; i < b.N; i++ { - res, err := te.ng.Select(prm) + res, err := te.ng.Select(context.Background(), prm) if err != nil { b.Fatal(err) } diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 26600a3eb..4effb2b16 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -1,7 +1,12 @@ package engine import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // FlushWriteCachePrm groups the parameters of FlushWriteCache operation. @@ -26,7 +31,14 @@ func (p *FlushWriteCachePrm) SetIgnoreErrors(ignore bool) { type FlushWriteCacheRes struct{} // FlushWriteCache flushes write-cache on a single shard. -func (e *StorageEngine) FlushWriteCache(p FlushWriteCachePrm) (FlushWriteCacheRes, error) { +func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) (FlushWriteCacheRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.FlushWriteCache", + trace.WithAttributes( + attribute.String("shard)id", p.shardID.String()), + attribute.Bool("ignore_errors", p.ignoreErrors), + )) + defer span.End() + e.mtx.RLock() sh, ok := e.shards[p.shardID.String()] e.mtx.RUnlock() @@ -38,5 +50,5 @@ func (e *StorageEngine) FlushWriteCache(p FlushWriteCachePrm) (FlushWriteCacheRe var prm shard.FlushWriteCachePrm prm.SetIgnoreErrors(p.ignoreErrors) - return FlushWriteCacheRes{}, sh.FlushWriteCache(prm) + return FlushWriteCacheRes{}, sh.FlushWriteCache(ctx, prm) } diff --git a/pkg/local_object_storage/metabase/control_test.go b/pkg/local_object_storage/metabase/control_test.go index 17f3b3893..b67e748b3 100644 --- a/pkg/local_object_storage/metabase/control_test.go +++ b/pkg/local_object_storage/metabase/control_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -55,6 +56,6 @@ func metaExists(db *meta.DB, addr oid.Address) (bool, error) { var existsPrm meta.ExistsPrm existsPrm.SetAddress(addr) - res, err := db.Exists(existsPrm) + res, err := db.Exists(context.Background(), existsPrm) return res.Exists(), err } diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index d93bc436b..17a593b6d 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -38,7 +39,7 @@ func TestCounters(t *testing.T) { for i := 0; i < objCount; i++ { prm.SetObject(oo[i]) - _, err = db.Put(prm) + _, err = db.Put(context.Background(), prm) require.NoError(t, err) c, err = db.ObjectCounters() @@ -58,7 +59,7 @@ func TestCounters(t *testing.T) { for i := objCount - 1; i >= 0; i-- { prm.SetAddresses(objectcore.AddressOf(oo[i])) - res, err := db.Delete(prm) + res, err := db.Delete(context.Background(), prm) require.NoError(t, err) require.Equal(t, uint64(1), res.AvailableObjectsRemoved()) @@ -89,7 +90,7 @@ func TestCounters(t *testing.T) { prm.SetTombstoneAddress(oidtest.Address()) prm.SetAddresses(inhumedObjs...) - res, err := db.Inhume(prm) + res, err := db.Inhume(context.Background(), prm) require.NoError(t, err) require.Equal(t, uint64(len(inhumedObjs)), res.AvailableInhumed()) @@ -159,7 +160,7 @@ func TestCounters(t *testing.T) { prm.SetTombstoneAddress(oidtest.Address()) prm.SetAddresses(inhumedObjs...) - _, err = db.Inhume(prm) + _, err = db.Inhume(context.Background(), prm) require.NoError(t, err) c, err = db.ObjectCounters() @@ -223,7 +224,7 @@ func TestCounters_Expired(t *testing.T) { inhumePrm.SetGCMark() inhumePrm.SetAddresses(oo[0]) - inhumeRes, err := db.Inhume(inhumePrm) + inhumeRes, err := db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) require.Equal(t, uint64(1), inhumeRes.AvailableInhumed()) @@ -240,7 +241,7 @@ func TestCounters_Expired(t *testing.T) { var deletePrm meta.DeletePrm deletePrm.SetAddresses(oo[0]) - deleteRes, err := db.Delete(deletePrm) + deleteRes, err := db.Delete(context.Background(), deletePrm) require.NoError(t, err) require.Zero(t, deleteRes.AvailableObjectsRemoved()) @@ -257,7 +258,7 @@ func TestCounters_Expired(t *testing.T) { deletePrm.SetAddresses(oo[0]) - deleteRes, err = db.Delete(deletePrm) + deleteRes, err = db.Delete(context.Background(), deletePrm) require.NoError(t, err) require.Equal(t, uint64(1), deleteRes.AvailableObjectsRemoved()) @@ -284,7 +285,7 @@ func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*object.Ob oo = append(oo, o) prm.SetObject(o) - _, err = db.Put(prm) + _, err = db.Put(context.Background(), prm) require.NoError(t, err) c, err := db.ObjectCounters() diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 79f870372..990d3997b 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -2,15 +2,19 @@ package meta import ( "bytes" + "context" "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // DeletePrm groups the parameters of Delete operation. @@ -65,7 +69,13 @@ type referenceNumber struct { type referenceCounter map[string]*referenceNumber // Delete removed object records from metabase indexes. -func (db *DB) Delete(prm DeletePrm) (DeleteRes, error) { +func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Delete", + trace.WithAttributes( + attribute.Int("addr_count", len(prm.addrs)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/delete_test.go b/pkg/local_object_storage/metabase/delete_test.go index ee161a881..d2a4bfa7b 100644 --- a/pkg/local_object_storage/metabase/delete_test.go +++ b/pkg/local_object_storage/metabase/delete_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "errors" "testing" @@ -139,6 +140,6 @@ func metaDelete(db *meta.DB, addrs ...oid.Address) error { var deletePrm meta.DeletePrm deletePrm.SetAddresses(addrs...) - _, err := db.Delete(deletePrm) + _, err := db.Delete(context.Background(), deletePrm) return err } diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index 686b65880..cfd37b0d2 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -1,15 +1,19 @@ package meta import ( + "context" "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ExistsPrm groups the parameters of Exists operation. @@ -39,7 +43,13 @@ func (p ExistsRes) Exists() bool { // // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. -func (db *DB) Exists(prm ExistsPrm) (res ExistsRes, err error) { +func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Exists", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index c0feda06c..fff32d6ad 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -1,14 +1,18 @@ package meta import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // GetPrm groups the parameters of Get operation. @@ -46,7 +50,14 @@ func (r GetRes) Header() *objectSDK.Object { // Returns an error of type apistatus.ObjectNotFound if object is missing in DB. // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. -func (db *DB) Get(prm GetPrm) (res GetRes, err error) { +func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Get", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + attribute.Bool("raw", prm.raw), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index a242a099a..4d2a7682d 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -2,6 +2,7 @@ package meta_test import ( "bytes" + "context" "fmt" "os" "runtime" @@ -132,7 +133,7 @@ func TestDB_Get(t *testing.T) { var prm meta.InhumePrm prm.SetAddresses(obj) - _, err = db.Inhume(prm) + _, err = db.Inhume(context.Background(), prm) require.NoError(t, err) _, err = metaGet(db, obj, false) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) @@ -216,7 +217,7 @@ func benchmarkGet(b *testing.B, numOfObj int) { getPrm.SetAddress(addrs[counter%len(addrs)]) counter++ - _, err := db.Get(getPrm) + _, err := db.Get(context.Background(), getPrm) if err != nil { b.Fatal(err) } @@ -235,7 +236,7 @@ func benchmarkGet(b *testing.B, numOfObj int) { var getPrm meta.GetPrm getPrm.SetAddress(addrs[i%len(addrs)]) - _, err := db.Get(getPrm) + _, err := db.Get(context.Background(), getPrm) if err != nil { b.Fatal(err) } @@ -248,6 +249,6 @@ func metaGet(db *meta.DB, addr oid.Address, raw bool) (*objectSDK.Object, error) prm.SetAddress(addr) prm.SetRaw(raw) - res, err := db.Get(prm) + res, err := db.Get(context.Background(), prm) return res.Header(), err } diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index b8b665541..8cd09e3f7 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -68,7 +69,7 @@ func TestDB_Iterate_OffsetNotFound(t *testing.T) { inhumePrm.SetAddresses(object.AddressOf(obj1)) inhumePrm.SetGCMark() - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) var counter int @@ -138,14 +139,14 @@ func TestDB_IterateDeletedObjects(t *testing.T) { inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) inhumePrm.SetTombstoneAddress(addrTombstone) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) inhumePrm.SetAddresses(object.AddressOf(obj3), object.AddressOf(obj4)) inhumePrm.SetGCMark() // inhume with GC mark - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) var ( @@ -225,7 +226,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { object.AddressOf(obj3), object.AddressOf(obj4)) inhumePrm.SetTombstoneAddress(addrTombstone) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) expectedGraveyard := []oid.Address{ @@ -320,7 +321,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { object.AddressOf(obj3), object.AddressOf(obj4)) inhumePrm.SetGCMark() - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) expectedGarbage := []oid.Address{ @@ -404,7 +405,7 @@ func TestDB_DropGraves(t *testing.T) { inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) inhumePrm.SetTombstoneAddress(addrTombstone) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) buriedTS := make([]meta.TombstonedObject, 0) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index b6e6cadf1..a6887a33b 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -2,9 +2,11 @@ package meta import ( "bytes" + "context" "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -118,7 +120,10 @@ var ErrLockObjectRemoval = logicerr.New("lock object removal") // // NOTE: Marks any object with GC mark (despite any prohibitions on operations // with that object) if WithForceGCMark option has been provided. -func (db *DB) Inhume(prm InhumePrm) (res InhumeRes, err error) { +func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Inhume") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/inhume_test.go b/pkg/local_object_storage/metabase/inhume_test.go index b7ee5ef29..0f0774227 100644 --- a/pkg/local_object_storage/metabase/inhume_test.go +++ b/pkg/local_object_storage/metabase/inhume_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -50,40 +51,40 @@ func TestInhumeTombOnTomb(t *testing.T) { inhumePrm.SetTombstoneAddress(addr2) // inhume addr1 via addr2 - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) existsPrm.SetAddress(addr1) // addr1 should become inhumed {addr1:addr2} - _, err = db.Exists(existsPrm) + _, err = db.Exists(context.Background(), existsPrm) require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) inhumePrm.SetAddresses(addr3) inhumePrm.SetTombstoneAddress(addr1) // try to inhume addr3 via addr1 - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) // record with {addr1:addr2} should be removed from graveyard // as a tomb-on-tomb; metabase should return ObjectNotFound // NOT ObjectAlreadyRemoved since that record has been removed // from graveyard but addr1 is still marked with GC - _, err = db.Exists(existsPrm) + _, err = db.Exists(context.Background(), existsPrm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) existsPrm.SetAddress(addr3) // addr3 should be inhumed {addr3: addr1} - _, err = db.Exists(existsPrm) + _, err = db.Exists(context.Background(), existsPrm) require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) inhumePrm.SetAddresses(addr1) inhumePrm.SetTombstoneAddress(oidtest.Address()) // try to inhume addr1 (which is already a tombstone in graveyard) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) existsPrm.SetAddress(addr1) @@ -91,7 +92,7 @@ func TestInhumeTombOnTomb(t *testing.T) { // record with addr1 key should not appear in graveyard // (tomb can not be inhumed) but should be kept as object // with GC mark - _, err = db.Exists(existsPrm) + _, err = db.Exists(context.Background(), existsPrm) require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) } @@ -100,13 +101,13 @@ func TestInhumeLocked(t *testing.T) { locked := oidtest.Address() - err := db.Lock(locked.Container(), oidtest.ID(), []oid.ID{locked.Object()}) + err := db.Lock(context.Background(), locked.Container(), oidtest.ID(), []oid.ID{locked.Object()}) require.NoError(t, err) var prm meta.InhumePrm prm.SetAddresses(locked) - _, err = db.Inhume(prm) + _, err = db.Inhume(context.Background(), prm) var e apistatus.ObjectLocked require.ErrorAs(t, err, &e) @@ -117,6 +118,6 @@ func metaInhume(db *meta.DB, target, tomb oid.Address) error { inhumePrm.SetAddresses(target) inhumePrm.SetTombstoneAddress(tomb) - _, err := db.Inhume(inhumePrm) + _, err := db.Inhume(context.Background(), inhumePrm) return err } diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 6b3a3612d..e7d6ad04f 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "strconv" "testing" @@ -34,7 +35,7 @@ func TestDB_IterateExpired(t *testing.T) { expiredLocked := putWithExpiration(t, db, object.TypeRegular, epoch-1) - require.NoError(t, db.Lock(expiredLocked.Container(), oidtest.ID(), []oid.ID{expiredLocked.Object()})) + require.NoError(t, db.Lock(context.Background(), expiredLocked.Container(), oidtest.ID(), []oid.ID{expiredLocked.Object()})) err := db.IterateExpired(epoch, func(exp *meta.ExpiredObject) error { if addr, ok := mAlive[exp.Type()]; ok { @@ -81,13 +82,13 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { prm.SetAddresses(protected1, protected2, protectedLocked) prm.SetTombstoneAddress(ts) - _, err = db.Inhume(prm) + _, err = db.Inhume(context.Background(), prm) require.NoError(t, err) prm.SetAddresses(garbage) prm.SetGCMark() - _, err = db.Inhume(prm) + _, err = db.Inhume(context.Background(), prm) require.NoError(t, err) var handled []oid.Address @@ -107,7 +108,7 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { require.Contains(t, handled, protected2) require.Contains(t, handled, protectedLocked) - err = db.Lock(protectedLocked.Container(), oidtest.ID(), []oid.ID{protectedLocked.Object()}) + err = db.Lock(context.Background(), protectedLocked.Container(), oidtest.ID(), []oid.ID{protectedLocked.Object()}) require.NoError(t, err) handled = handled[:0] diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 2dcc85d40..5c3c9720d 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -2,14 +2,18 @@ package meta import ( "bytes" + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) var bucketNameLocked = []byte{lockedPrefix} @@ -30,7 +34,15 @@ func bucketNameLockers(idCnr cid.ID, key []byte) []byte { // Allows locking regular objects only (otherwise returns apistatus.LockNonRegularObject). // // Locked list should be unique. Panics if it is empty. -func (db *DB) Lock(cnr cid.ID, locker oid.ID, locked []oid.ID) error { +func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid.ID) error { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Lock", + trace.WithAttributes( + attribute.String("container_id", cnr.EncodeToString()), + attribute.String("locker", locker.EncodeToString()), + attribute.Int("locked_count", len(locked)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -266,7 +278,13 @@ func (i IsLockedRes) Locked() bool { // object is considered as non-locked. // // Returns only non-logical errors related to underlying database. -func (db *DB) IsLocked(prm IsLockedPrm) (res IsLockedRes, err error) { +func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.IsLocked", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index efa9fba06..1d6ea6ffb 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -20,8 +21,8 @@ func TestDB_Lock(t *testing.T) { db := newDB(t) t.Run("empty locked list", func(t *testing.T) { - require.Panics(t, func() { _ = db.Lock(cnr, oid.ID{}, nil) }) - require.Panics(t, func() { _ = db.Lock(cnr, oid.ID{}, []oid.ID{}) }) + require.Panics(t, func() { _ = db.Lock(context.Background(), cnr, oid.ID{}, nil) }) + require.Panics(t, func() { _ = db.Lock(context.Background(), cnr, oid.ID{}, []oid.ID{}) }) }) t.Run("(ir)regular", func(t *testing.T) { @@ -44,7 +45,7 @@ func TestDB_Lock(t *testing.T) { id, _ := obj.ID() // try to lock it - err = db.Lock(cnr, oidtest.ID(), []oid.ID{id}) + err = db.Lock(context.Background(), cnr, oidtest.ID(), []oid.ID{id}) if typ == object.TypeRegular { require.NoError(t, err, typ) } else { @@ -65,27 +66,27 @@ func TestDB_Lock(t *testing.T) { // check locking relation inhumePrm.SetAddresses(objAddr) - _, err := db.Inhume(inhumePrm) + _, err := db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) inhumePrm.SetTombstoneAddress(oidtest.Address()) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) // try to remove lock object inhumePrm.SetAddresses(lockAddr) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.Error(t, err) // check that locking relation has not been // dropped inhumePrm.SetAddresses(objAddr) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) inhumePrm.SetTombstoneAddress(oidtest.Address()) - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) }) @@ -105,7 +106,7 @@ func TestDB_Lock(t *testing.T) { inhumePrm.SetForceGCMark() inhumePrm.SetLockObjectHandling() - res, err := db.Inhume(inhumePrm) + res, err := db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) require.Len(t, res.DeletedLockObjects(), 1) require.Equal(t, objectcore.AddressOf(lockObj), res.DeletedLockObjects()[0]) @@ -117,7 +118,7 @@ func TestDB_Lock(t *testing.T) { inhumePrm.SetGCMark() // now we can inhume the object - _, err = db.Inhume(inhumePrm) + _, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) }) @@ -134,7 +135,7 @@ func TestDB_Lock(t *testing.T) { inhumePrm.SetAddresses(objectcore.AddressOf(lockObj)) inhumePrm.SetLockObjectHandling() - res, err := db.Inhume(inhumePrm) + res, err := db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) require.Len(t, res.DeletedLockObjects(), 1) require.Equal(t, objectcore.AddressOf(lockObj), res.DeletedLockObjects()[0]) @@ -151,7 +152,7 @@ func TestDB_Lock(t *testing.T) { for i := 0; i < objsNum; i++ { inhumePrm.SetAddresses(objectcore.AddressOf(objs[i])) - res, err = db.Inhume(inhumePrm) + res, err = db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) require.Len(t, res.DeletedLockObjects(), 0) } @@ -164,7 +165,7 @@ func TestDB_Lock(t *testing.T) { inhumePrm.SetForceGCMark() inhumePrm.SetAddresses(objectcore.AddressOf(lockObj)) - res, err := db.Inhume(inhumePrm) + res, err := db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) require.Len(t, res.DeletedLockObjects(), 0) }) @@ -184,7 +185,7 @@ func TestDB_Lock_Expired(t *testing.T) { require.ErrorIs(t, err, meta.ErrObjectIsExpired) // lock the obj - require.NoError(t, db.Lock(addr.Container(), oidtest.ID(), []oid.ID{addr.Object()})) + require.NoError(t, db.Lock(context.Background(), addr.Container(), oidtest.ID(), []oid.ID{addr.Object()})) // object is expired but locked, thus, must be available _, err = metaGet(db, addr, false) @@ -202,7 +203,7 @@ func TestDB_IsLocked(t *testing.T) { for _, obj := range objs { prm.SetAddress(objectcore.AddressOf(obj)) - res, err := db.IsLocked(prm) + res, err := db.IsLocked(context.Background(), prm) require.NoError(t, err) require.True(t, res.Locked()) @@ -212,7 +213,7 @@ func TestDB_IsLocked(t *testing.T) { prm.SetAddress(oidtest.Address()) - res, err := db.IsLocked(prm) + res, err := db.IsLocked(context.Background(), prm) require.NoError(t, err) require.False(t, res.Locked()) @@ -224,12 +225,12 @@ func TestDB_IsLocked(t *testing.T) { var putPrm meta.PutPrm putPrm.SetObject(obj) - _, err = db.Put(putPrm) + _, err = db.Put(context.Background(), putPrm) require.NoError(t, err) prm.SetAddress(objectcore.AddressOf(obj)) - res, err = db.IsLocked(prm) + res, err = db.IsLocked(context.Background(), prm) require.NoError(t, err) require.False(t, res.Locked()) @@ -260,7 +261,7 @@ func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*object.Ob err := putBig(db, lockObj) require.NoError(t, err) - err = db.Lock(cnr, lockID, lockedObjIDs) + err = db.Lock(context.Background(), cnr, lockID, lockedObjIDs) require.NoError(t, err) return lockedObjs, lockObj diff --git a/pkg/local_object_storage/metabase/movable.go b/pkg/local_object_storage/metabase/movable.go index e6990dc54..412c46393 100644 --- a/pkg/local_object_storage/metabase/movable.go +++ b/pkg/local_object_storage/metabase/movable.go @@ -1,10 +1,14 @@ package meta import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ToMoveItPrm groups the parameters of ToMoveIt operation. @@ -48,7 +52,13 @@ func (p MovableRes) AddressList() []oid.Address { // ToMoveIt marks objects to move it into another shard. This useful for // faster HRW fetching. -func (db *DB) ToMoveIt(prm ToMoveItPrm) (res ToMoveItRes, err error) { +func (db *DB) ToMoveIt(ctx context.Context, prm ToMoveItPrm) (res ToMoveItRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.ToMoveIt", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/movable_test.go b/pkg/local_object_storage/metabase/movable_test.go index 6918dec29..51e7e6d74 100644 --- a/pkg/local_object_storage/metabase/movable_test.go +++ b/pkg/local_object_storage/metabase/movable_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -61,7 +62,7 @@ func metaToMoveIt(db *meta.DB, addr oid.Address) error { var toMovePrm meta.ToMoveItPrm toMovePrm.SetAddress(addr) - _, err := db.ToMoveIt(toMovePrm) + _, err := db.ToMoveIt(context.Background(), toMovePrm) return err } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index b0fea6535..2c78bda0f 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -1,11 +1,13 @@ package meta import ( + "context" "encoding/binary" "errors" "fmt" gio "io" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" @@ -14,6 +16,8 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/io" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) type ( @@ -52,7 +56,13 @@ var ( // // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. -func (db *DB) Put(prm PutPrm) (res PutRes, err error) { +func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Put", + trace.WithAttributes( + attribute.String("address", objectCore.AddressOf(prm.obj).EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index 837d931ae..a3a071d19 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "runtime" "strconv" "testing" @@ -117,7 +118,7 @@ func metaPut(db *meta.DB, obj *objectSDK.Object, id []byte) error { putPrm.SetObject(obj) putPrm.SetStorageID(id) - _, err := db.Put(putPrm) + _, err := db.Put(context.Background(), putPrm) return err } diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 74c261d35..ecd83f863 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -1,17 +1,21 @@ package meta import ( + "context" "encoding/binary" "errors" "fmt" "strings" v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -56,7 +60,13 @@ func (r SelectRes) AddressList() []oid.Address { } // Select returns list of addresses of objects that match search filters. -func (db *DB) Select(prm SelectPrm) (res SelectRes, err error) { +func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.Select", + trace.WithAttributes( + attribute.String("container_id", prm.cnr.EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 5d4cc75e5..386797529 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "encoding/hex" "strconv" "testing" @@ -829,7 +830,7 @@ func benchmarkSelect(b *testing.B, db *meta.DB, cid cidSDK.ID, fs objectSDK.Sear prm.SetFilters(fs) for i := 0; i < b.N; i++ { - res, err := db.Select(prm) + res, err := db.Select(context.Background(), prm) if err != nil { b.Fatal(err) } @@ -844,6 +845,6 @@ func metaSelect(db *meta.DB, cnr cidSDK.ID, fs objectSDK.SearchFilters) ([]oid.A prm.SetFilters(fs) prm.SetContainerID(cnr) - res, err := db.Select(prm) + res, err := db.Select(context.Background(), prm) return res.AddressList(), err } diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index ae309d4b2..794879a3f 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -1,11 +1,15 @@ package meta import ( + "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // StorageIDPrm groups the parameters of StorageID operation. @@ -30,7 +34,13 @@ func (r StorageIDRes) StorageID() []byte { // StorageID returns storage descriptor for objects from the blobstor. // It is put together with the object can makes get/delete operation faster. -func (db *DB) StorageID(prm StorageIDPrm) (res StorageIDRes, err error) { +func (db *DB) StorageID(ctx context.Context, prm StorageIDPrm) (res StorageIDRes, err error) { + _, span := tracing.StartSpanFromContext(ctx, "metabase.StorageID", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go index f8185abee..5b27cdc87 100644 --- a/pkg/local_object_storage/metabase/storage_id_test.go +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -63,6 +64,6 @@ func metaStorageID(db *meta.DB, addr oid.Address) ([]byte, error) { var sidPrm meta.StorageIDPrm sidPrm.SetAddress(addr) - r, err := db.StorageID(sidPrm) + r, err := db.StorageID(context.Background(), sidPrm) return r.StorageID(), err } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 3d0f72922..e74f235f8 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -80,7 +81,10 @@ func (s *Shard) Open() error { type metabaseSynchronizer Shard func (x *metabaseSynchronizer) Init() error { - return (*Shard)(x).refillMetabase() + ctx, span := tracing.StartSpanFromContext(context.TODO(), "metabaseSynchronizer.Init") + defer span.End() + + return (*Shard)(x).refillMetabase(ctx) } // Init initializes all Shard's components. @@ -158,7 +162,7 @@ func (s *Shard) Init(ctx context.Context) error { return nil } -func (s *Shard) refillMetabase() error { +func (s *Shard) refillMetabase(ctx context.Context) error { err := s.metaBase.Reset() if err != nil { return fmt.Errorf("could not reset metabase: %w", err) @@ -177,9 +181,9 @@ func (s *Shard) refillMetabase() error { var err error switch obj.Type() { case objectSDK.TypeTombstone: - err = s.refillTombstoneObject(obj) + err = s.refillTombstoneObject(ctx, obj) case objectSDK.TypeLock: - err = s.refillLockObject(obj) + err = s.refillLockObject(ctx, obj) default: } if err != nil { @@ -190,7 +194,7 @@ func (s *Shard) refillMetabase() error { mPrm.SetObject(obj) mPrm.SetStorageID(descriptor) - _, err = s.metaBase.Put(mPrm) + _, err = s.metaBase.Put(ctx, mPrm) if err != nil && !meta.IsErrRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) { return err } @@ -209,7 +213,7 @@ func (s *Shard) refillMetabase() error { return nil } -func (s *Shard) refillLockObject(obj *objectSDK.Object) error { +func (s *Shard) refillLockObject(ctx context.Context, obj *objectSDK.Object) error { var lock objectSDK.Lock if err := lock.Unmarshal(obj.Payload()); err != nil { return fmt.Errorf("could not unmarshal lock content: %w", err) @@ -220,14 +224,14 @@ func (s *Shard) refillLockObject(obj *objectSDK.Object) error { cnr, _ := obj.ContainerID() id, _ := obj.ID() - err := s.metaBase.Lock(cnr, id, locked) + err := s.metaBase.Lock(ctx, cnr, id, locked) if err != nil { return fmt.Errorf("could not lock objects: %w", err) } return nil } -func (s *Shard) refillTombstoneObject(obj *objectSDK.Object) error { +func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object) error { tombstone := objectSDK.NewTombstone() if err := tombstone.Unmarshal(obj.Payload()); err != nil { @@ -250,7 +254,7 @@ func (s *Shard) refillTombstoneObject(obj *objectSDK.Object) error { inhumePrm.SetTombstoneAddress(tombAddr) inhumePrm.SetAddresses(tombMembers...) - _, err := s.metaBase.Inhume(inhumePrm) + _, err := s.metaBase.Inhume(ctx, inhumePrm) if err != nil { return fmt.Errorf("could not inhume objects: %w", err) } @@ -290,7 +294,10 @@ func (s *Shard) Close() error { // Reload reloads configuration portions that are necessary. // If a config option is invalid, it logs an error and returns nil. // If there was a problem with applying new configuration, an error is returned. -func (s *Shard) Reload(opts ...Option) error { +func (s *Shard) Reload(ctx context.Context, opts ...Option) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Reload") + defer span.End() + // Do not use defaultCfg here missing options need not be reloaded. var c cfg for i := range opts { @@ -314,7 +321,7 @@ func (s *Shard) Reload(opts ...Option) error { // Here we refill metabase only if a new instance was opened. This is a feature, // we don't want to hang for some time just because we forgot to change // config after the node was updated. - err = s.refillMetabase() + err = s.refillMetabase(ctx) } else { err = s.metaBase.Init() } diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 50ea20bb8..170052d63 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -126,6 +126,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) { } sh := New( + WithID(NewIDFromBytes([]byte{})), WithBlobStorOptions(blobOpts...), WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))), WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta")), meta.WithEpochState(epochState{}))) @@ -138,12 +139,12 @@ func TestRefillMetabaseCorrupted(t *testing.T) { var putPrm PutPrm putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) require.NoError(t, sh.Close()) addr := object.AddressOf(obj) - _, err = fsTree.Put(common.PutPrm{Address: addr, RawData: []byte("not an object")}) + _, err = fsTree.Put(context.Background(), common.PutPrm{Address: addr, RawData: []byte("not an object")}) require.NoError(t, err) sh = New( @@ -245,13 +246,13 @@ func TestRefillMetabase(t *testing.T) { for _, v := range mObjs { putPrm.SetObject(v.obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) } putPrm.SetObject(tombObj) - _, err = sh.Put(putPrm) + _, err = sh.Put(context.Background(), putPrm) require.NoError(t, err) // LOCK object handling @@ -263,11 +264,11 @@ func TestRefillMetabase(t *testing.T) { objectSDK.WriteLock(lockObj, lock) putPrm.SetObject(lockObj) - _, err = sh.Put(putPrm) + _, err = sh.Put(context.Background(), putPrm) require.NoError(t, err) lockID, _ := lockObj.ID() - require.NoError(t, sh.Lock(cnrLocked, lockID, locked)) + require.NoError(t, sh.Lock(context.Background(), cnrLocked, lockID, locked)) var inhumePrm InhumePrm inhumePrm.SetTarget(object.AddressOf(tombObj), tombMembers...) @@ -368,7 +369,7 @@ func TestRefillMetabase(t *testing.T) { checkObj(object.AddressOf(tombObj), nil) checkTombMembers(false) - err = sh.refillMetabase() + err = sh.refillMetabase(context.Background()) require.NoError(t, err) c, err = sh.metaBase.ObjectCounters() diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index ed05f9982..f086aa30f 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -1,13 +1,17 @@ package shard import ( + "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -28,14 +32,21 @@ func (p *DeletePrm) SetAddresses(addr ...oid.Address) { // Delete removes data from the shard's writeCache, metaBase and // blobStor. -func (s *Shard) Delete(prm DeletePrm) (DeleteRes, error) { +func (s *Shard) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Delete", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.Int("addr_count", len(prm.addr)), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() - return s.delete(prm) + return s.delete(ctx, prm) } -func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { +func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { if s.info.Mode.ReadOnly() { return DeleteRes{}, ErrReadOnlyMode } else if s.info.Mode.NoMetabase() { @@ -48,7 +59,7 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { for i := range prm.addr { if s.hasWriteCache() { - err := s.writeCache.Delete(prm.addr[i]) + err := s.writeCache.Delete(ctx, prm.addr[i]) if err != nil && !IsErrNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.String("error", err.Error())) } @@ -57,7 +68,7 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { var sPrm meta.StorageIDPrm sPrm.SetAddress(prm.addr[i]) - res, err := s.metaBase.StorageID(sPrm) + res, err := s.metaBase.StorageID(ctx, sPrm) if err != nil { s.log.Debug(logs.ShardCantGetStorageIDFromMetabase, zap.Stringer("object", prm.addr[i]), @@ -74,7 +85,7 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { var delPrm meta.DeletePrm delPrm.SetAddresses(prm.addr...) - res, err := s.metaBase.Delete(delPrm) + res, err := s.metaBase.Delete(ctx, delPrm) if err != nil { return DeleteRes{}, err // stop on metabase error ? } @@ -99,7 +110,7 @@ func (s *Shard) delete(prm DeletePrm) (DeleteRes, error) { id := smalls[prm.addr[i]] delPrm.StorageID = id - _, err = s.blobStor.Delete(delPrm) + _, err = s.blobStor.Delete(ctx, delPrm) if err != nil { s.log.Debug(logs.ShardCantRemoveObjectFromBlobStor, zap.Stringer("object_address", prm.addr[i]), diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index c37dfa285..9646e9aa0 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -43,13 +43,13 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { var delPrm shard.DeletePrm delPrm.SetAddresses(object.AddressOf(obj)) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) _, err = testGet(t, sh, getPrm, hasWriteCache) require.NoError(t, err) - _, err = sh.Delete(delPrm) + _, err = sh.Delete(context.TODO(), delPrm) require.NoError(t, err) _, err = sh.Get(context.Background(), getPrm) @@ -67,13 +67,13 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { var delPrm shard.DeletePrm delPrm.SetAddresses(object.AddressOf(obj)) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) _, err = sh.Get(context.Background(), getPrm) require.NoError(t, err) - _, err = sh.Delete(delPrm) + _, err = sh.Delete(context.Background(), delPrm) require.NoError(t, err) _, err = sh.Get(context.Background(), getPrm) diff --git a/pkg/local_object_storage/shard/dump_test.go b/pkg/local_object_storage/shard/dump_test.go index 9d585cc06..921717204 100644 --- a/pkg/local_object_storage/shard/dump_test.go +++ b/pkg/local_object_storage/shard/dump_test.go @@ -104,7 +104,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { var prm shard.PutPrm prm.SetObject(objects[i]) - _, err := sh.Put(prm) + _, err := sh.Put(context.Background(), prm) require.NoError(t, err) } @@ -129,13 +129,13 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { t.Run("empty dump", func(t *testing.T) { var restorePrm shard.RestorePrm restorePrm.WithPath(outEmpty) - res, err := sh.Restore(restorePrm) + res, err := sh.Restore(context.Background(), restorePrm) require.NoError(t, err) require.Equal(t, 0, res.Count()) }) t.Run("invalid path", func(t *testing.T) { - _, err := sh.Restore(*new(shard.RestorePrm)) + _, err := sh.Restore(context.Background(), *new(shard.RestorePrm)) require.ErrorIs(t, err, os.ErrNotExist) }) @@ -147,7 +147,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { var restorePrm shard.RestorePrm restorePrm.WithPath(out) - _, err := sh.Restore(restorePrm) + _, err := sh.Restore(context.Background(), restorePrm) require.ErrorIs(t, err, shard.ErrInvalidMagic) }) @@ -162,7 +162,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { var restorePrm shard.RestorePrm restorePrm.WithPath(out) - _, err := sh.Restore(restorePrm) + _, err := sh.Restore(context.Background(), restorePrm) require.ErrorIs(t, err, io.ErrUnexpectedEOF) }) t.Run("incomplete object data", func(t *testing.T) { @@ -173,7 +173,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { var restorePrm shard.RestorePrm restorePrm.WithPath(out) - _, err := sh.Restore(restorePrm) + _, err := sh.Restore(context.Background(), restorePrm) require.ErrorIs(t, err, io.EOF) }) t.Run("invalid object", func(t *testing.T) { @@ -184,7 +184,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { var restorePrm shard.RestorePrm restorePrm.WithPath(out) - _, err := sh.Restore(restorePrm) + _, err := sh.Restore(context.Background(), restorePrm) require.Error(t, err) t.Run("skip errors", func(t *testing.T) { @@ -195,7 +195,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { restorePrm.WithPath(out) restorePrm.WithIgnoreErrors(true) - res, err := sh.Restore(restorePrm) + res, err := sh.Restore(context.Background(), restorePrm) require.NoError(t, err) require.Equal(t, objCount, res.Count()) require.Equal(t, 2, res.FailCount()) @@ -208,7 +208,7 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) { t.Run("must allow write", func(t *testing.T) { require.NoError(t, sh.SetMode(mode.ReadOnly)) - _, err := sh.Restore(prm) + _, err := sh.Restore(context.Background(), prm) require.ErrorIs(t, err, shard.ErrReadOnlyMode) }) @@ -234,7 +234,7 @@ func TestStream(t *testing.T) { var prm shard.PutPrm prm.SetObject(objects[i]) - _, err := sh1.Put(prm) + _, err := sh1.Put(context.Background(), prm) require.NoError(t, err) } @@ -269,7 +269,7 @@ func TestStream(t *testing.T) { } func checkRestore(t *testing.T, sh *shard.Shard, prm shard.RestorePrm, objects []*objectSDK.Object) { - res, err := sh.Restore(prm) + res, err := sh.Restore(context.Background(), prm) require.NoError(t, err) require.Equal(t, len(objects), res.Count()) @@ -333,7 +333,7 @@ func TestDumpIgnoreErrors(t *testing.T) { var prm shard.PutPrm prm.SetObject(objects[i]) - _, err := sh.Put(prm) + _, err := sh.Put(context.Background(), prm) require.NoError(t, err) } diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 76e4347d4..66c61fccc 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -3,9 +3,12 @@ package shard import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ExistsPrm groups the parameters of Exists operation. @@ -36,6 +39,13 @@ func (p ExistsRes) Exists() bool { // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been marked as removed. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Exists", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + var exists bool var err error @@ -54,7 +64,7 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { existsPrm.SetAddress(prm.addr) var res meta.ExistsRes - res, err = s.metaBase.Exists(existsPrm) + res, err = s.metaBase.Exists(ctx, existsPrm) exists = res.Exists() } diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 5ea9ecedf..86995cd06 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -234,7 +234,7 @@ func (s *Shard) removeGarbage() { deletePrm.SetAddresses(buf...) // delete accumulated objects - _, err = s.delete(deletePrm) + _, err = s.delete(context.TODO(), deletePrm) if err != nil { s.log.Warn(logs.ShardCouldNotDeleteTheObjects, zap.String("error", err.Error()), @@ -320,7 +320,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) inhumePrm.SetGCMark() // inhume the collected objects - res, err := s.metaBase.Inhume(inhumePrm) + res, err := s.metaBase.Inhume(ctx, inhumePrm) if err != nil { s.log.Warn(logs.ShardCouldNotInhumeTheObjects, zap.String("error", err.Error()), @@ -485,7 +485,7 @@ func (s *Shard) selectExpired(ctx context.Context, epoch uint64, addresses []oid // and clears up corresponding graveyard records. // // Does not modify tss. -func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) { +func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.TombstonedObject) { if s.GetMode().NoMetabase() { return } @@ -502,7 +502,7 @@ func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) { pInhume.SetAddresses(tsAddrs...) // inhume tombstones - res, err := s.metaBase.Inhume(pInhume) + res, err := s.metaBase.Inhume(ctx, pInhume) if err != nil { s.log.Warn(logs.ShardCouldNotMarkTombstonesAsGarbage, zap.String("error", err.Error()), @@ -547,7 +547,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] pInhume.SetAddresses(lockers...) pInhume.SetForceGCMark() - res, err := s.metaBase.Inhume(pInhume) + res, err := s.metaBase.Inhume(ctx, pInhume) if err != nil { s.log.Warn(logs.ShardFailureToMarkLockersAsGarbage, zap.String("error", err.Error()), diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 8012e60f8..b0126fcd7 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -100,14 +100,14 @@ func Test_GCDropsLockedExpiredObject(t *testing.T) { var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) - err = sh.Lock(cnr, lockID, []oid.ID{objID}) + err = sh.Lock(context.Background(), cnr, lockID, []oid.ID{objID}) require.NoError(t, err) putPrm.SetObject(lock) - _, err = sh.Put(putPrm) + _, err = sh.Put(context.Background(), putPrm) require.NoError(t, err) epoch.Value = 105 diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 8a0296ac6..5268ac790 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -96,7 +96,7 @@ func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { } skipMeta := prm.skipMeta || s.info.Mode.NoMetabase() - obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, cb, wc) + obj, hasMeta, err := s.fetchObjectData(ctx, prm.addr, skipMeta, cb, wc) return GetRes{ obj: obj, @@ -109,7 +109,7 @@ func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { var emptyStorageID = make([]byte, 0) // fetchObjectData looks through writeCache and blobStor to find object. -func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, cb storFetcher, wc func(w writecache.Cache) (*objectSDK.Object, error)) (*objectSDK.Object, bool, error) { +func (s *Shard) fetchObjectData(ctx context.Context, addr oid.Address, skipMeta bool, cb storFetcher, wc func(w writecache.Cache) (*objectSDK.Object, error)) (*objectSDK.Object, bool, error) { var ( mErr error mRes meta.ExistsRes @@ -118,7 +118,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, cb storFetcher, if !skipMeta { var mPrm meta.ExistsPrm mPrm.SetAddress(addr) - mRes, mErr = s.metaBase.Exists(mPrm) + mRes, mErr = s.metaBase.Exists(ctx, mPrm) if mErr != nil && !s.info.Mode.NoMetabase() { return nil, false, mErr } @@ -154,7 +154,7 @@ func (s *Shard) fetchObjectData(addr oid.Address, skipMeta bool, cb storFetcher, var mPrm meta.StorageIDPrm mPrm.SetAddress(addr) - mExRes, err := s.metaBase.StorageID(mPrm) + mExRes, err := s.metaBase.StorageID(ctx, mPrm) if err != nil { return nil, true, fmt.Errorf("can't fetch blobovnicza id from metabase: %w", err) } diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index f670b2864..ea28c8e32 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -40,7 +40,7 @@ func testShardGet(t *testing.T, hasWriteCache bool) { putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) getPrm.SetAddress(object.AddressOf(obj)) @@ -58,7 +58,7 @@ func testShardGet(t *testing.T, hasWriteCache bool) { putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) getPrm.SetAddress(object.AddressOf(obj)) @@ -86,7 +86,7 @@ func testShardGet(t *testing.T, hasWriteCache bool) { putPrm.SetObject(child) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) getPrm.SetAddress(object.AddressOf(child)) diff --git a/pkg/local_object_storage/shard/head.go b/pkg/local_object_storage/shard/head.go index 8e8ff9433..a15cdfdca 100644 --- a/pkg/local_object_storage/shard/head.go +++ b/pkg/local_object_storage/shard/head.go @@ -73,7 +73,7 @@ func (s *Shard) Head(ctx context.Context, prm HeadPrm) (HeadRes, error) { headParams.SetRaw(prm.raw) var res meta.GetRes - res, err = s.metaBase.Get(headParams) + res, err = s.metaBase.Get(ctx, headParams) obj = res.Header() } diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index 449626e93..11e7a8b04 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -37,7 +37,7 @@ func testShardHead(t *testing.T, hasWriteCache bool) { putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) headPrm.SetAddress(object.AddressOf(obj)) @@ -62,7 +62,7 @@ func testShardHead(t *testing.T, hasWriteCache bool) { putPrm.SetObject(child) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) headPrm.SetAddress(object.AddressOf(parent)) diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 3457188be..12a2900ac 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -5,9 +5,12 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -62,6 +65,12 @@ var ErrLockObjectRemoval = meta.ErrLockObjectRemoval // // Returns ErrReadOnlyMode error if shard is in "read-only" mode. func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Inhume", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + )) + defer span.End() + s.m.RLock() if s.info.Mode.ReadOnly() { @@ -74,7 +83,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { if s.hasWriteCache() { for i := range prm.target { - _ = s.writeCache.Delete(prm.target[i]) + _ = s.writeCache.Delete(ctx, prm.target[i]) } } @@ -92,7 +101,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { metaPrm.SetForceGCMark() } - res, err := s.metaBase.Inhume(metaPrm) + res, err := s.metaBase.Inhume(ctx, metaPrm) if err != nil { if errors.Is(err, meta.ErrLockObjectRemoval) { s.m.RUnlock() diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 41845c414..0b4e51701 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -42,7 +42,7 @@ func testShardInhume(t *testing.T, hasWriteCache bool) { var getPrm shard.GetPrm getPrm.SetAddress(object.AddressOf(obj)) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) _, err = testGet(t, sh, getPrm, hasWriteCache) diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index bab1090eb..aaa1112cd 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -1,6 +1,7 @@ package shard import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -85,7 +86,7 @@ func (s *Shard) List() (res SelectRes, err error) { sPrm.SetContainerID(lst[i]) sPrm.SetFilters(filters) - sRes, err := s.metaBase.Select(sPrm) // consider making List in metabase + sRes, err := s.metaBase.Select(context.TODO(), sPrm) // consider making List in metabase if err != nil { s.log.Debug(logs.ShardCantSelectAllObjects, zap.Stringer("cid", lst[i]), diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index 33c9e489a..8fac41a0f 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -1,6 +1,7 @@ package shard_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -52,7 +53,7 @@ func testShardList(t *testing.T, sh *shard.Shard) { putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) } } diff --git a/pkg/local_object_storage/shard/lock.go b/pkg/local_object_storage/shard/lock.go index d8113cf30..cfbd94c5b 100644 --- a/pkg/local_object_storage/shard/lock.go +++ b/pkg/local_object_storage/shard/lock.go @@ -1,11 +1,15 @@ package shard import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Lock marks objects as locked with another object. All objects from the @@ -14,7 +18,16 @@ import ( // Allows locking regular objects only (otherwise returns apistatus.LockNonRegularObject). // // Locked list should be unique. Panics if it is empty. -func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { +func (s *Shard) Lock(ctx context.Context, idCnr cid.ID, locker oid.ID, locked []oid.ID) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Lock", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", idCnr.EncodeToString()), + attribute.String("locker", locker.EncodeToString()), + attribute.Int("locked_count", len(locked)), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -25,7 +38,7 @@ func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { return ErrDegradedMode } - err := s.metaBase.Lock(idCnr, locker, locked) + err := s.metaBase.Lock(ctx, idCnr, locker, locked) if err != nil { return fmt.Errorf("metabase lock: %w", err) } @@ -35,7 +48,14 @@ func (s *Shard) Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error { // IsLocked checks object locking relation of the provided object. Not found object is // considered as not locked. Requires healthy metabase, returns ErrDegradedMode otherwise. -func (s *Shard) IsLocked(addr oid.Address) (bool, error) { +func (s *Shard) IsLocked(ctx context.Context, addr oid.Address) (bool, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.IsLocked", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + m := s.GetMode() if m.NoMetabase() { return false, ErrDegradedMode @@ -44,7 +64,7 @@ func (s *Shard) IsLocked(addr oid.Address) (bool, error) { var prm meta.IsLockedPrm prm.SetAddress(addr) - res, err := s.metaBase.IsLocked(prm) + res, err := s.metaBase.IsLocked(ctx, prm) if err != nil { return false, err } diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 2bee66298..c577ae184 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -76,16 +76,16 @@ func TestShard_Lock(t *testing.T) { var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) // lock the object - err = sh.Lock(cnr, lockID, []oid.ID{objID}) + err = sh.Lock(context.Background(), cnr, lockID, []oid.ID{objID}) require.NoError(t, err) putPrm.SetObject(lock) - _, err = sh.Put(putPrm) + _, err = sh.Put(context.Background(), putPrm) require.NoError(t, err) t.Run("inhuming locked objects", func(t *testing.T) { @@ -158,21 +158,21 @@ func TestShard_IsLocked(t *testing.T) { var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) // not locked object is not locked - locked, err := sh.IsLocked(objectcore.AddressOf(obj)) + locked, err := sh.IsLocked(context.Background(), objectcore.AddressOf(obj)) require.NoError(t, err) require.False(t, locked) // locked object is locked - require.NoError(t, sh.Lock(cnrID, lockID, []oid.ID{objID})) + require.NoError(t, sh.Lock(context.Background(), cnrID, lockID, []oid.ID{objID})) - locked, err = sh.IsLocked(objectcore.AddressOf(obj)) + locked, err = sh.IsLocked(context.Background(), objectcore.AddressOf(obj)) require.NoError(t, err) require.True(t, locked) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 18e97e259..1578c662b 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -109,7 +109,7 @@ func TestCounters(t *testing.T) { for i := 0; i < objNumber; i++ { prm.SetObject(oo[i]) - _, err := sh.Put(prm) + _, err := sh.Put(context.Background(), prm) require.NoError(t, err) } @@ -168,7 +168,7 @@ func TestCounters(t *testing.T) { deletedNumber := int(phy / 4) prm.SetAddresses(addrFromObjs(oo[:deletedNumber])...) - _, err := sh.Delete(prm) + _, err := sh.Delete(context.Background(), prm) require.NoError(t, err) require.Equal(t, phy-uint64(deletedNumber), mm.objCounters[physical]) @@ -207,6 +207,7 @@ func shardWithMetrics(t *testing.T, path string) (*shard.Shard, *metricsStore) { } sh := shard.New( + shard.WithID(shard.NewIDFromBytes([]byte{})), shard.WithBlobStorOptions(blobOpts...), shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(path, "pilorama"))), shard.WithMetaBaseOptions( diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go index f3199ac07..119910623 100644 --- a/pkg/local_object_storage/shard/move.go +++ b/pkg/local_object_storage/shard/move.go @@ -1,9 +1,14 @@ package shard import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -23,7 +28,14 @@ func (p *ToMoveItPrm) SetAddress(addr oid.Address) { // ToMoveIt calls metabase.ToMoveIt method to mark object as relocatable to // another shard. -func (s *Shard) ToMoveIt(prm ToMoveItPrm) (ToMoveItRes, error) { +func (s *Shard) ToMoveIt(ctx context.Context, prm ToMoveItPrm) (ToMoveItRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.ToMoveIt", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", prm.addr.EncodeToString()), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -37,7 +49,7 @@ func (s *Shard) ToMoveIt(prm ToMoveItPrm) (ToMoveItRes, error) { var toMovePrm meta.ToMoveItPrm toMovePrm.SetAddress(prm.addr) - _, err := s.metaBase.ToMoveIt(toMovePrm) + _, err := s.metaBase.ToMoveIt(ctx, toMovePrm) if err != nil { s.log.Debug(logs.ShardCouldNotMarkObjectForShardRelocationInMetabase, zap.String("error", err.Error()), diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index a4cb2cb1f..d7d4ae538 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -1,13 +1,17 @@ package shard import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -30,7 +34,14 @@ func (p *PutPrm) SetObject(obj *object.Object) { // did not allow to completely save the object. // // Returns ErrReadOnlyMode error if shard is in "read-only" mode. -func (s *Shard) Put(prm PutPrm) (PutRes, error) { +func (s *Shard) Put(ctx context.Context, prm PutPrm) (PutRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Put", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", objectCore.AddressOf(prm.obj).EncodeToString()), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -55,7 +66,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) { // ahead of `Put` by storage engine tryCache := s.hasWriteCache() && !m.NoMetabase() if tryCache { - res, err = s.writeCache.Put(putPrm) + res, err = s.writeCache.Put(ctx, putPrm) } if err != nil || !tryCache { if err != nil { @@ -63,7 +74,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) { zap.String("err", err.Error())) } - res, err = s.blobStor.Put(putPrm) + res, err = s.blobStor.Put(ctx, putPrm) if err != nil { return PutRes{}, fmt.Errorf("could not put object to BLOB storage: %w", err) } @@ -73,7 +84,7 @@ func (s *Shard) Put(prm PutPrm) (PutRes, error) { var pPrm meta.PutPrm pPrm.SetObject(prm.obj) pPrm.SetStorageID(res.StorageID) - if _, err := s.metaBase.Put(pPrm); err != nil { + if _, err := s.metaBase.Put(ctx, pPrm); err != nil { // may we need to handle this case in a special way // since the object has been successfully written to BlobStor return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err) diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 4355c31a3..06aea2f8a 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -123,7 +123,7 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { } skipMeta := prm.skipMeta || s.info.Mode.NoMetabase() - obj, hasMeta, err := s.fetchObjectData(prm.addr, skipMeta, cb, wc) + obj, hasMeta, err := s.fetchObjectData(ctx, prm.addr, skipMeta, cb, wc) return RngRes{ obj: obj, diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 164181214..c95dbae98 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -99,7 +99,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { var putPrm shard.PutPrm putPrm.SetObject(obj) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) var rngPrm shard.RngPrm diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 1bfa33dd7..9ad05f525 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -44,6 +44,7 @@ func TestShardReload(t *testing.T) { meta.WithEpochState(epochState{})} opts := []Option{ + WithID(NewIDFromBytes([]byte{})), WithLogger(l), WithBlobStorOptions(blobOpts...), WithMetaBaseOptions(metaOpts...), @@ -75,7 +76,7 @@ func TestShardReload(t *testing.T) { checkHasObjects(t, true) t.Run("same config, no-op", func(t *testing.T) { - require.NoError(t, sh.Reload(opts...)) + require.NoError(t, sh.Reload(context.Background(), opts...)) checkHasObjects(t, true) }) @@ -86,7 +87,7 @@ func TestShardReload(t *testing.T) { } newOpts := newShardOpts(filepath.Join(p, "meta1"), false) - require.NoError(t, sh.Reload(newOpts...)) + require.NoError(t, sh.Reload(context.Background(), newOpts...)) checkHasObjects(t, false) // new path, but no resync @@ -97,7 +98,7 @@ func TestShardReload(t *testing.T) { }) newOpts = newShardOpts(filepath.Join(p, "meta2"), true) - require.NoError(t, sh.Reload(newOpts...)) + require.NoError(t, sh.Reload(context.Background(), newOpts...)) checkHasObjects(t, true) // all objects are restored, including the new one @@ -106,7 +107,7 @@ func TestShardReload(t *testing.T) { require.NoError(t, os.WriteFile(badPath, []byte{1}, 0)) newOpts = newShardOpts(badPath, true) - require.Error(t, sh.Reload(newOpts...)) + require.Error(t, sh.Reload(context.Background(), newOpts...)) // Cleanup is done, no panic. obj := newObject() @@ -117,7 +118,7 @@ func TestShardReload(t *testing.T) { // Successive reload produces no undesired effects. require.NoError(t, os.RemoveAll(badPath)) - require.NoError(t, sh.Reload(newOpts...)) + require.NoError(t, sh.Reload(context.Background(), newOpts...)) obj = newObject() require.NoError(t, putObject(sh, obj)) @@ -132,7 +133,7 @@ func putObject(sh *Shard, obj *objectSDK.Object) error { var prm PutPrm prm.SetObject(obj) - _, err := sh.Put(prm) + _, err := sh.Put(context.Background(), prm) return err } diff --git a/pkg/local_object_storage/shard/restore.go b/pkg/local_object_storage/shard/restore.go index 73dc1d178..2cb64a518 100644 --- a/pkg/local_object_storage/shard/restore.go +++ b/pkg/local_object_storage/shard/restore.go @@ -2,13 +2,17 @@ package shard import ( "bytes" + "context" "encoding/binary" "errors" "io" "os" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ErrInvalidMagic is returned when dump format is invalid. @@ -57,8 +61,15 @@ func (r RestoreRes) FailCount() int { // Restore restores objects from the dump prepared by Dump. // // Returns any error encountered. -func (s *Shard) Restore(prm RestorePrm) (RestoreRes, error) { - // Disallow changing mode during restore. +func (s *Shard) Restore(ctx context.Context, prm RestorePrm) (RestoreRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Restore", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("path", prm.path), + attribute.Bool("ignore_errors", prm.ignoreErrors), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -122,7 +133,7 @@ func (s *Shard) Restore(prm RestorePrm) (RestoreRes, error) { } putPrm.SetObject(obj) - _, err = s.Put(putPrm) + _, err = s.Put(ctx, putPrm) if err != nil && !IsErrObjectExpired(err) && !IsErrRemoved(err) { return RestoreRes{}, err } diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index 4bb467d48..7f776c18a 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -1,12 +1,16 @@ package shard import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // SelectPrm groups the parameters of Select operation. @@ -39,7 +43,14 @@ func (r SelectRes) AddressList() []oid.Address { // // Returns any error encountered that // did not allow to completely select the objects. -func (s *Shard) Select(prm SelectPrm) (SelectRes, error) { +func (s *Shard) Select(ctx context.Context, prm SelectPrm) (SelectRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Select", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", prm.cnr.EncodeToString()), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -51,7 +62,7 @@ func (s *Shard) Select(prm SelectPrm) (SelectRes, error) { selectPrm.SetFilters(prm.filters) selectPrm.SetContainerID(prm.cnr) - mRes, err := s.metaBase.Select(selectPrm) + mRes, err := s.metaBase.Select(ctx, selectPrm) if err != nil { return SelectRes{}, fmt.Errorf("could not select objects from metabase: %w", err) } diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 5fd13221a..714811b7e 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -43,7 +43,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { for i := range objects { putPrm.SetObject(objects[i]) - _, err := sh.Put(putPrm) + _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) } require.NoError(t, sh.Close()) diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index 7282f121c..245eb4c70 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -1,7 +1,12 @@ package shard import ( + "context" "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // FlushWriteCachePrm represents parameters of a `FlushWriteCache` operation. @@ -19,7 +24,14 @@ func (p *FlushWriteCachePrm) SetIgnoreErrors(ignore bool) { var errWriteCacheDisabled = errors.New("write-cache is disabled") // FlushWriteCache flushes all data from the write-cache. -func (s *Shard) FlushWriteCache(p FlushWriteCachePrm) error { +func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.FlushWriteCache", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.Bool("ignore_errors", p.ignoreErrors), + )) + defer span.End() + if !s.hasWriteCache() { return errWriteCacheDisabled } @@ -35,5 +47,5 @@ func (s *Shard) FlushWriteCache(p FlushWriteCachePrm) error { return ErrDegradedMode } - return s.writeCache.Flush(p.ignoreErrors) + return s.writeCache.Flush(ctx, p.ignoreErrors) } diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index dcfea8dd0..c1aab9e5a 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -1,16 +1,27 @@ package writecache import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Delete removes object from write-cache. // // Returns an error of type apistatus.ObjectNotFound if object is missing in write-cache. -func (c *cache) Delete(addr oid.Address) error { +func (c *cache) Delete(ctx context.Context, addr oid.Address) error { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Delete", + trace.WithAttributes( + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { @@ -45,7 +56,7 @@ func (c *cache) Delete(addr oid.Address) error { return nil } - _, err := c.fsTree.Delete(common.DeletePrm{Address: addr}) + _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err == nil { storagelog.Write(c.log, storagelog.AddressField(saddr), diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 3ca3aa905..9d0ffc98c 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -2,9 +2,11 @@ package writecache import ( "bytes" + "context" "errors" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -15,6 +17,8 @@ import ( "github.com/mr-tron/base58" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -37,7 +41,7 @@ func (c *cache) runFlushLoop() { } c.wg.Add(1) - go c.flushBigObjects() + go c.flushBigObjects(context.TODO()) c.wg.Add(1) go func() { @@ -141,7 +145,7 @@ func (c *cache) flushDB() { } } -func (c *cache) flushBigObjects() { +func (c *cache) flushBigObjects(ctx context.Context) { defer c.wg.Done() tick := time.NewTicker(defaultFlushInterval * 10) @@ -157,7 +161,7 @@ func (c *cache) flushBigObjects() { continue } - _ = c.flushFSTree(true) + _ = c.flushFSTree(ctx, true) c.modeMtx.RUnlock() case <-c.closeCh: @@ -176,7 +180,7 @@ func (c *cache) reportFlushError(msg string, addr string, err error) { } } -func (c *cache) flushFSTree(ignoreErrors bool) error { +func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { var prm common.IteratePrm prm.IgnoreErrors = ignoreErrors prm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { @@ -205,7 +209,7 @@ func (c *cache) flushFSTree(ignoreErrors bool) error { return err } - err = c.flushObject(&obj, data) + err = c.flushObject(ctx, &obj, data) if err != nil { if ignoreErrors { return nil @@ -236,7 +240,7 @@ func (c *cache) flushWorker(_ int) { return } - err := c.flushObject(obj, nil) + err := c.flushObject(context.TODO(), obj, nil) if err == nil { c.flushed.Add(objectCore.AddressOf(obj).EncodeToString(), true) } @@ -244,14 +248,14 @@ func (c *cache) flushWorker(_ int) { } // flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(obj *object.Object, data []byte) error { +func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte) error { addr := objectCore.AddressOf(obj) var prm common.PutPrm prm.Object = obj prm.RawData = data - res, err := c.blobstor.Put(prm) + res, err := c.blobstor.Put(ctx, prm) if err != nil { if !errors.Is(err, common.ErrNoSpace) && !errors.Is(err, common.ErrReadOnly) && !errors.Is(err, blobstor.ErrNoPlaceFound) { @@ -276,15 +280,21 @@ func (c *cache) flushObject(obj *object.Object, data []byte) error { // Flush flushes all objects from the write-cache to the main storage. // Write-cache must be in readonly mode to ensure correctness of an operation and // to prevent interference with background flush workers. -func (c *cache) Flush(ignoreErrors bool) error { +func (c *cache) Flush(ctx context.Context, ignoreErrors bool) error { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Flush", + trace.WithAttributes( + attribute.Bool("ignore_errors", ignoreErrors), + )) + defer span.End() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() - return c.flush(ignoreErrors) + return c.flush(ctx, ignoreErrors) } -func (c *cache) flush(ignoreErrors bool) error { - if err := c.flushFSTree(ignoreErrors); err != nil { +func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { + if err := c.flushFSTree(ctx, ignoreErrors); err != nil { return err } @@ -316,7 +326,7 @@ func (c *cache) flush(ignoreErrors bool) error { return err } - if err := c.flushObject(&obj, data); err != nil { + if err := c.flushObject(ctx, &obj, data); err != nil { return err } } diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 9dc216fb3..2cec07081 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -89,7 +89,7 @@ func TestFlush(t *testing.T) { var mPrm meta.StorageIDPrm mPrm.SetAddress(objects[i].addr) - mRes, err := mb.StorageID(mPrm) + mRes, err := mb.StorageID(context.Background(), mPrm) require.NoError(t, err) var prm common.GetPrm @@ -112,12 +112,12 @@ func TestFlush(t *testing.T) { wc.(*cache).flushed.Add(objects[0].addr.EncodeToString(), true) wc.(*cache).flushed.Add(objects[1].addr.EncodeToString(), false) - require.NoError(t, wc.Flush(false)) + require.NoError(t, wc.Flush(context.Background(), false)) for i := 0; i < 2; i++ { var mPrm meta.GetPrm mPrm.SetAddress(objects[i].addr) - _, err := mb.Get(mPrm) + _, err := mb.Get(context.Background(), mPrm) require.Error(t, err) _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) @@ -147,7 +147,7 @@ func TestFlush(t *testing.T) { for i := 0; i < 2; i++ { var mPrm meta.GetPrm mPrm.SetAddress(objects[i].addr) - _, err := mb.Get(mPrm) + _, err := mb.Get(context.Background(), mPrm) require.Error(t, err) _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) @@ -171,9 +171,9 @@ func TestFlush(t *testing.T) { require.NoError(t, mb.SetMode(mode.ReadWrite)) require.Equal(t, uint32(0), errCount.Load()) - require.Error(t, wc.Flush(false)) + require.Error(t, wc.Flush(context.Background(), false)) require.True(t, errCount.Load() > 0) - require.NoError(t, wc.Flush(true)) + require.NoError(t, wc.Flush(context.Background(), true)) check(t, mb, bs, objects) } @@ -202,7 +202,7 @@ func TestFlush(t *testing.T) { prm.Address = objectCore.AddressOf(obj) prm.RawData = data - _, err := c.fsTree.Put(prm) + _, err := c.fsTree.Put(context.Background(), prm) require.NoError(t, err) p := prm.Address.Object().EncodeToString() + "." + prm.Address.Container().EncodeToString() @@ -218,7 +218,7 @@ func TestFlush(t *testing.T) { var prm common.PutPrm prm.Address = oidtest.Address() prm.RawData = []byte{1, 2, 3} - _, err := c.fsTree.Put(prm) + _, err := c.fsTree.Put(context.Background(), prm) require.NoError(t, err) }) }) @@ -245,19 +245,19 @@ func TestFlush(t *testing.T) { for i := range objects { var prm meta.PutPrm prm.SetObject(objects[i].obj) - _, err := mb.Put(prm) + _, err := mb.Put(context.Background(), prm) require.NoError(t, err) } var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(objects[0].addr, objects[1].addr) inhumePrm.SetTombstoneAddress(oidtest.Address()) - _, err := mb.Inhume(inhumePrm) + _, err := mb.Inhume(context.Background(), inhumePrm) require.NoError(t, err) var deletePrm meta.DeletePrm deletePrm.SetAddresses(objects[2].addr, objects[3].addr) - _, err = mb.Delete(deletePrm) + _, err = mb.Delete(context.Background(), deletePrm) require.NoError(t, err) require.NoError(t, bs.SetMode(mode.ReadOnly)) @@ -294,7 +294,7 @@ func putObject(t *testing.T, c Cache, size int) objectPair { prm.Object = obj prm.RawData = data - _, err := c.Put(prm) + _, err := c.Put(context.Background(), prm) require.NoError(t, err) return objectPair{prm.Address, prm.Object} diff --git a/pkg/local_object_storage/writecache/init.go b/pkg/local_object_storage/writecache/init.go index 0ac8cea99..d92e9a2d9 100644 --- a/pkg/local_object_storage/writecache/init.go +++ b/pkg/local_object_storage/writecache/init.go @@ -15,21 +15,21 @@ import ( "go.uber.org/zap" ) -func (c *cache) initFlushMarks() { +func (c *cache) initFlushMarks(ctx context.Context) { var localWG sync.WaitGroup localWG.Add(1) go func() { defer localWG.Done() - c.fsTreeFlushMarkUpdate() + c.fsTreeFlushMarkUpdate(ctx) }() localWG.Add(1) go func() { defer localWG.Done() - c.dbFlushMarkUpdate() + c.dbFlushMarkUpdate(ctx) }() c.initWG.Add(1) @@ -54,7 +54,7 @@ func (c *cache) initFlushMarks() { var errStopIter = errors.New("stop iteration") -func (c *cache) fsTreeFlushMarkUpdate() { +func (c *cache) fsTreeFlushMarkUpdate(ctx context.Context) { c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInFSTree) var prm common.IteratePrm @@ -67,14 +67,14 @@ func (c *cache) fsTreeFlushMarkUpdate() { default: } - flushed, needRemove := c.flushStatus(addr) + flushed, needRemove := c.flushStatus(ctx, addr) if flushed { c.store.flushed.Add(addr.EncodeToString(), true) if needRemove { var prm common.DeletePrm prm.Address = addr - _, err := c.fsTree.Delete(prm) + _, err := c.fsTree.Delete(ctx, prm) if err == nil { storagelog.Write(c.log, storagelog.AddressField(addr), @@ -90,7 +90,7 @@ func (c *cache) fsTreeFlushMarkUpdate() { c.log.Info(logs.WritecacheFinishedUpdatingFSTreeFlushMarks) } -func (c *cache) dbFlushMarkUpdate() { +func (c *cache) dbFlushMarkUpdate(ctx context.Context) { c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInDatabase) var m []string @@ -125,7 +125,7 @@ func (c *cache) dbFlushMarkUpdate() { continue } - flushed, needRemove := c.flushStatus(addr) + flushed, needRemove := c.flushStatus(ctx, addr) if flushed { c.store.flushed.Add(addr.EncodeToString(), true) if needRemove { @@ -165,11 +165,11 @@ func (c *cache) dbFlushMarkUpdate() { // flushStatus returns info about the object state in the main storage. // First return value is true iff object exists. // Second return value is true iff object can be safely removed. -func (c *cache) flushStatus(addr oid.Address) (bool, bool) { +func (c *cache) flushStatus(ctx context.Context, addr oid.Address) (bool, bool) { var existsPrm meta.ExistsPrm existsPrm.SetAddress(addr) - _, err := c.metabase.Exists(existsPrm) + _, err := c.metabase.Exists(ctx, existsPrm) if err != nil { needRemove := errors.Is(err, meta.ErrObjectIsExpired) || errors.As(err, new(apistatus.ObjectAlreadyRemoved)) return needRemove, needRemove @@ -178,7 +178,7 @@ func (c *cache) flushStatus(addr oid.Address) (bool, bool) { var prm meta.StorageIDPrm prm.SetAddress(addr) - mRes, _ := c.metabase.StorageID(prm) - res, err := c.blobstor.Exists(context.TODO(), common.ExistsPrm{Address: addr, StorageID: mRes.StorageID()}) + mRes, _ := c.metabase.StorageID(ctx, prm) + res, err := c.blobstor.Exists(ctx, common.ExistsPrm{Address: addr, StorageID: mRes.StorageID()}) return err == nil && res.Exists, false } diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 939dc5b06..20b0cce29 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -1,12 +1,16 @@ package writecache import ( + "context" "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ErrReadOnly is returned when Put/Write is performed in a read-only mode. @@ -19,19 +23,25 @@ var ErrNotInitialized = logicerr.New("write-cache is not initialized yet") // When shard is put in read-only mode all objects in memory are flushed to disk // and all background jobs are suspended. func (c *cache) SetMode(m mode.Mode) error { + ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.SetMode", + trace.WithAttributes( + attribute.String("mode", m.String()), + )) + defer span.End() + c.modeMtx.Lock() defer c.modeMtx.Unlock() - return c.setMode(m) + return c.setMode(ctx, m) } // setMode applies new mode. Must be called with cache.modeMtx lock taken. -func (c *cache) setMode(m mode.Mode) error { +func (c *cache) setMode(ctx context.Context, m mode.Mode) error { var err error turnOffMeta := m.NoMetabase() if turnOffMeta && !c.mode.NoMetabase() { - err = c.flush(true) + err = c.flush(ctx, true) if err != nil { return err } @@ -45,7 +55,7 @@ func (c *cache) setMode(m mode.Mode) error { defer func() { if err == nil && !turnOffMeta { - c.initFlushMarks() + c.initFlushMarks(ctx) } }() } diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index cca8986b3..3434e9355 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -19,14 +19,14 @@ type Option func(*options) // meta is an interface for a metabase. type metabase interface { - Exists(meta.ExistsPrm) (meta.ExistsRes, error) - StorageID(meta.StorageIDPrm) (meta.StorageIDRes, error) + Exists(context.Context, meta.ExistsPrm) (meta.ExistsRes, error) + StorageID(context.Context, meta.StorageIDPrm) (meta.StorageIDRes, error) UpdateStorageID(meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error) } // blob is an interface for the blobstor. type blob interface { - Put(common.PutPrm) (common.PutRes, error) + Put(context.Context, common.PutPrm) (common.PutRes, error) NeedsCompression(obj *objectSDK.Object) bool Exists(ctx context.Context, res common.ExistsPrm) (common.ExistsRes, error) } diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 7791e93dc..e2535d9e2 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -1,11 +1,15 @@ package writecache import ( + "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) var ( @@ -21,7 +25,14 @@ var ( // Returns ErrNotInitialized if write-cache has not been initialized yet. // Returns ErrOutOfSpace if saving an object leads to WC's size overflow. // Returns ErrBigObject if an objects exceeds maximum object size. -func (c *cache) Put(prm common.PutPrm) (common.PutRes, error) { +func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Put", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.Bool("dont_compress", prm.DontCompress), + )) + defer span.End() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { @@ -44,7 +55,7 @@ func (c *cache) Put(prm common.PutPrm) (common.PutRes, error) { if sz <= c.smallObjectSize { return common.PutRes{}, c.putSmall(oi) } - return common.PutRes{}, c.putBig(oi.addr, prm) + return common.PutRes{}, c.putBig(ctx, oi.addr, prm) } // putSmall persists small objects to the write-cache database and @@ -71,13 +82,13 @@ func (c *cache) putSmall(obj objectInfo) error { } // putBig writes object to FSTree and pushes it to the flush workers queue. -func (c *cache) putBig(addr string, prm common.PutPrm) error { +func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) error { cacheSz := c.estimateCacheSize() if c.maxCacheSize < c.incSizeFS(cacheSz) { return ErrOutOfSpace } - _, err := c.fsTree.Put(prm) + _, err := c.fsTree.Put(ctx, prm) if err != nil { return err } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index ff7eb1d6a..aeae752e3 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -1,6 +1,7 @@ package writecache import ( + "context" "errors" "fmt" "os" @@ -146,7 +147,7 @@ func (c *cache) deleteFromDisk(keys []string) []string { continue } - _, err := c.fsTree.Delete(common.DeletePrm{Address: addr}) + _, err := c.fsTree.Delete(context.TODO(), common.DeletePrm{Address: addr}) if err != nil && !errors.As(err, new(apistatus.ObjectNotFound)) { c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 24070dbda..0fc2e601a 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -5,6 +5,7 @@ import ( "os" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -32,13 +33,13 @@ type Cache interface { // // Returns apistatus.ObjectNotFound if object is missing in the Cache. // Returns ErrReadOnly if the Cache is currently in the read-only mode. - Delete(oid.Address) error + Delete(context.Context, oid.Address) error Iterate(IterationPrm) error - Put(common.PutPrm) (common.PutRes, error) + Put(context.Context, common.PutPrm) (common.PutRes, error) SetMode(mode.Mode) error SetLogger(*logger.Logger) DumpInfo() Info - Flush(bool) error + Flush(context.Context, bool) error Init() error Open(readOnly bool) error @@ -152,7 +153,10 @@ func (c *cache) Open(readOnly bool) error { // Init runs necessary services. func (c *cache) Init() error { - c.initFlushMarks() + ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.Init") + defer span.End() + + c.initFlushMarks(ctx) c.runFlushLoop() return nil } @@ -163,7 +167,7 @@ func (c *cache) Close() error { defer c.modeMtx.Unlock() // Finish all in-progress operations. - if err := c.setMode(mode.ReadOnly); err != nil { + if err := c.setMode(context.TODO(), mode.ReadOnly); err != nil { return err } diff --git a/pkg/services/control/server/flush_cache.go b/pkg/services/control/server/flush_cache.go index fdfd136a6..9ead530db 100644 --- a/pkg/services/control/server/flush_cache.go +++ b/pkg/services/control/server/flush_cache.go @@ -9,7 +9,7 @@ import ( "google.golang.org/grpc/status" ) -func (s *Server) FlushCache(_ context.Context, req *control.FlushCacheRequest) (*control.FlushCacheResponse, error) { +func (s *Server) FlushCache(ctx context.Context, req *control.FlushCacheRequest) (*control.FlushCacheResponse, error) { err := s.isValidRequest(req) if err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) @@ -19,7 +19,7 @@ func (s *Server) FlushCache(_ context.Context, req *control.FlushCacheRequest) ( var prm engine.FlushWriteCachePrm prm.SetShardID(shardID) - _, err = s.s.FlushWriteCache(prm) + _, err = s.s.FlushWriteCache(ctx, prm) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/restore.go b/pkg/services/control/server/restore.go index 0e6367951..dba186f57 100644 --- a/pkg/services/control/server/restore.go +++ b/pkg/services/control/server/restore.go @@ -9,7 +9,7 @@ import ( "google.golang.org/grpc/status" ) -func (s *Server) RestoreShard(_ context.Context, req *control.RestoreShardRequest) (*control.RestoreShardResponse, error) { +func (s *Server) RestoreShard(ctx context.Context, req *control.RestoreShardRequest) (*control.RestoreShardResponse, error) { err := s.isValidRequest(req) if err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) @@ -21,7 +21,7 @@ func (s *Server) RestoreShard(_ context.Context, req *control.RestoreShardReques prm.WithPath(req.GetBody().GetFilepath()) prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) - err = s.s.RestoreShard(shardID, prm) + err = s.s.RestoreShard(ctx, shardID, prm) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index f9870f7e0..f6341f02a 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -120,7 +120,7 @@ func (w *putSvcWrapper) put(ctx context.Context, exec *execCtx) (*oid.ID, error) WithCommonPrm(exec.commonParameters()). WithObject(exec.tombstoneObj.CutPayload()) - err = streamer.Init(initPrm) + err = streamer.Init(ctx, initPrm) if err != nil { return nil, err } diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 10a6af271..6beb67476 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -8,6 +8,7 @@ import ( "fmt" "io" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -402,6 +403,9 @@ func (x PutObjectRes) ID() oid.ID { // // Returns any error which prevented the operation from completing correctly in error return. func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "client.PutObject") + defer span.End() + var prmCli client.PrmObjectPutInit prmCli.MarkLocal() diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 15296f83f..4b2056802 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -117,7 +117,7 @@ func (x errIncompletePut) Error() string { return commonMsg } -func (t *distributedTarget) WriteHeader(obj *objectSDK.Object) error { +func (t *distributedTarget) WriteHeader(_ context.Context, obj *objectSDK.Object) error { t.obj = obj return nil diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index 2e6a496f3..7aef9f065 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -14,15 +14,15 @@ import ( type ObjectStorage interface { // Put must save passed object // and return any appeared error. - Put(*object.Object) error + Put(context.Context, *object.Object) error // Delete must delete passed objects // and return any appeared error. Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error // Lock must lock passed objects // and return any appeared error. - Lock(locker oid.Address, toLock []oid.ID) error + Lock(ctx context.Context, locker oid.Address, toLock []oid.ID) error // IsLocked must clarify object's lock status. - IsLocked(oid.Address) (bool, error) + IsLocked(context.Context, oid.Address) (bool, error) } type localTarget struct { @@ -47,7 +47,7 @@ func (t *localTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers return nil, fmt.Errorf("could not delete objects from tombstone locally: %w", err) } case object.TypeLock: - err := t.storage.Lock(objectCore.AddressOf(t.obj), t.meta.Objects()) + err := t.storage.Lock(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) if err != nil { return nil, fmt.Errorf("could not lock object from lock objects locally: %w", err) } @@ -55,7 +55,7 @@ func (t *localTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers // objects that do not change meta storage } - if err := t.storage.Put(t.obj); err != nil { + if err := t.storage.Put(ctx, t.obj); err != nil { //TODO return nil, fmt.Errorf("(%T) could not put object to local storage: %w", t, err) } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index e355990a3..ea885366b 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -32,13 +32,13 @@ var errNotInit = errors.New("stream not initialized") var errInitRecall = errors.New("init recall") -func (p *Streamer) Init(prm *PutInitPrm) error { +func (p *Streamer) Init(ctx context.Context, prm *PutInitPrm) error { // initialize destination target if err := p.initTarget(prm); err != nil { return fmt.Errorf("(%T) could not initialize object target: %w", p, err) } - if err := p.target.WriteHeader(prm.hdr); err != nil { + if err := p.target.WriteHeader(ctx, prm.hdr); err != nil { return fmt.Errorf("(%T) could not write header to target: %w", p, err) } return nil diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 3b8d7b88c..f7a97a955 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -15,6 +16,8 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) type streamer struct { @@ -34,6 +37,9 @@ type sizes struct { } func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "putv2.streamer.Send") + defer span.End() + switch v := req.GetBody().GetObjectPart().(type) { case *object.PutObjectPartInit: var initPrm *putsvc.PutInitPrm @@ -43,7 +49,7 @@ func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) return err } - if err = s.stream.Init(initPrm); err != nil { + if err = s.stream.Init(ctx, initPrm); err != nil { err = fmt.Errorf("(%T) could not init object put stream: %w", s, err) } @@ -105,6 +111,9 @@ func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) } func (s *streamer) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "putv2.streamer.CloseAndRecv") + defer span.End() + if s.saveChunks { // check payload size correctness if s.writtenPayload != s.payloadSz { @@ -121,6 +130,9 @@ func (s *streamer) CloseAndRecv(ctx context.Context) (*object.PutResponse, error } func (s *streamer) relayRequest(ctx context.Context, info client.NodeInfo, c client.MultiAddressClient) error { + ctx, span := tracing.StartSpanFromContext(ctx, "putv2.streamer.relayRequest") + defer span.End() + // open stream resp := new(object.PutResponse) @@ -129,6 +141,12 @@ func (s *streamer) relayRequest(ctx context.Context, info client.NodeInfo, c cli var firstErr error info.AddressGroup().IterateAddresses(func(addr network.Address) (stop bool) { + ctx, span := tracing.StartSpanFromContext(ctx, "putv2.streamer.iterateAddress", + trace.WithAttributes( + attribute.String("address", addr.String()), + )) + defer span.End() + var err error defer func() { diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/put/validation.go index 70c6974d3..8c40d0677 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/put/validation.go @@ -41,7 +41,7 @@ var ( ErrWrongPayloadSize = errors.New("wrong payload size") ) -func (t *validatingTarget) WriteHeader(obj *objectSDK.Object) error { +func (t *validatingTarget) WriteHeader(ctx context.Context, obj *objectSDK.Object) error { t.payloadSz = obj.PayloadSize() chunkLn := uint64(len(obj.Payload())) @@ -73,11 +73,11 @@ func (t *validatingTarget) WriteHeader(obj *objectSDK.Object) error { t.checksum = cs.Value() } - if err := t.fmt.Validate(obj, t.unpreparedObject); err != nil { + if err := t.fmt.Validate(ctx, obj, t.unpreparedObject); err != nil { return fmt.Errorf("(%T) coult not validate object format: %w", t, err) } - err := t.nextTarget.WriteHeader(obj) + err := t.nextTarget.WriteHeader(ctx, obj) if err != nil { return err } diff --git a/pkg/services/object/search/local.go b/pkg/services/object/search/local.go index f768c8861..1af69caf1 100644 --- a/pkg/services/object/search/local.go +++ b/pkg/services/object/search/local.go @@ -1,12 +1,14 @@ package searchsvc import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "go.uber.org/zap" ) -func (exec *execCtx) executeLocal() { - ids, err := exec.svc.localStorage.search(exec) +func (exec *execCtx) executeLocal(ctx context.Context) { + ids, err := exec.svc.localStorage.search(ctx, exec) if err != nil { exec.status = statusUndefined diff --git a/pkg/services/object/search/search.go b/pkg/services/object/search/search.go index b192e1d04..7a7cbfc5b 100644 --- a/pkg/services/object/search/search.go +++ b/pkg/services/object/search/search.go @@ -27,7 +27,7 @@ func (exec *execCtx) execute(ctx context.Context) { exec.log.Debug(logs.ServingRequest) // perform local operation - exec.executeLocal() + exec.executeLocal(ctx) exec.analyzeStatus(ctx, true) } diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index e95970955..75059103f 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -107,7 +107,7 @@ func (c *testClientCache) get(info clientcore.NodeInfo) (searchClient, error) { return v, nil } -func (s *testStorage) search(exec *execCtx) ([]oid.ID, error) { +func (s *testStorage) search(_ context.Context, exec *execCtx) ([]oid.ID, error) { v, ok := s.items[exec.containerID().EncodeToString()] if !ok { return nil, nil diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index b858e2219..708979d79 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -37,7 +37,7 @@ type cfg struct { log *logger.Logger localStorage interface { - search(*execCtx) ([]oid.ID, error) + search(context.Context, *execCtx) ([]oid.ID, error) } clientConstructor interface { diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index 49f3e5efd..b5b351a3b 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -117,12 +117,12 @@ func (c *clientWrapper) searchObjects(ctx context.Context, exec *execCtx, info c return res.IDList(), nil } -func (e *storageEngineWrapper) search(exec *execCtx) ([]oid.ID, error) { +func (e *storageEngineWrapper) search(ctx context.Context, exec *execCtx) ([]oid.ID, error) { var selectPrm engine.SelectPrm selectPrm.WithFilters(exec.searchFilters()) selectPrm.WithContainerID(exec.containerID()) - r, err := e.storage.Select(selectPrm) + r, err := e.storage.Select(ctx, selectPrm) if err != nil { return nil, err } diff --git a/pkg/services/object_manager/transformer/fmt.go b/pkg/services/object_manager/transformer/fmt.go index 462cc7474..fbe8af2fb 100644 --- a/pkg/services/object_manager/transformer/fmt.go +++ b/pkg/services/object_manager/transformer/fmt.go @@ -48,7 +48,7 @@ func NewFormatTarget(p *FormatterParams) ObjectTarget { } } -func (f *formatter) WriteHeader(obj *object.Object) error { +func (f *formatter) WriteHeader(_ context.Context, obj *object.Object) error { f.obj = obj return nil @@ -97,7 +97,7 @@ func (f *formatter) Close(ctx context.Context) (*AccessIdentifiers, error) { return nil, fmt.Errorf("could not finalize object: %w", err) } - if err := f.prm.NextTarget.WriteHeader(f.obj); err != nil { + if err := f.prm.NextTarget.WriteHeader(ctx, f.obj); err != nil { return nil, fmt.Errorf("could not write header to next target: %w", err) } diff --git a/pkg/services/object_manager/transformer/transformer.go b/pkg/services/object_manager/transformer/transformer.go index 199f5d0c1..c23b4dca7 100644 --- a/pkg/services/object_manager/transformer/transformer.go +++ b/pkg/services/object_manager/transformer/transformer.go @@ -56,7 +56,7 @@ func NewPayloadSizeLimiter(maxSize uint64, withoutHomomorphicHash bool, targetIn } } -func (s *payloadSizeLimiter) WriteHeader(hdr *object.Object) error { +func (s *payloadSizeLimiter) WriteHeader(_ context.Context, hdr *object.Object) error { s.current = fromObject(hdr) s.initialize() @@ -190,7 +190,7 @@ func (s *payloadSizeLimiter) release(ctx context.Context, finalize bool) (*Acces writeHashes(s.currentHashers) // release current, get its id - if err := s.target.WriteHeader(s.current); err != nil { + if err := s.target.WriteHeader(ctx, s.current); err != nil { return nil, fmt.Errorf("could not write header: %w", err) } diff --git a/pkg/services/object_manager/transformer/types.go b/pkg/services/object_manager/transformer/types.go index 3e6e2feff..73cea5216 100644 --- a/pkg/services/object_manager/transformer/types.go +++ b/pkg/services/object_manager/transformer/types.go @@ -28,7 +28,7 @@ type ObjectTarget interface { // that depends on the implementation. // // Must not be called after Close call. - WriteHeader(*object.Object) error + WriteHeader(context.Context, *object.Object) error // Write writes object payload chunk. // From 6121b541b5ac79b58d3b7e9e891e47f258f8a6e5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Apr 2023 15:36:20 +0300 Subject: [PATCH 0112/1943] [#242] treesvc: Add tracing spans Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/tree.go | 175 ++++++++++++++--- pkg/local_object_storage/engine/tree_test.go | 4 +- pkg/local_object_storage/pilorama/boltdb.go | 138 ++++++++++++-- pkg/local_object_storage/pilorama/forest.go | 25 +-- .../pilorama/forest_test.go | 155 +++++++-------- .../pilorama/interface.go | 26 +-- pkg/local_object_storage/shard/tree.go | 176 +++++++++++++++--- pkg/services/tree/drop.go | 4 +- pkg/services/tree/getsubtree_test.go | 7 +- pkg/services/tree/redirect.go | 9 + pkg/services/tree/replicator.go | 32 +++- pkg/services/tree/service.go | 30 +-- pkg/services/tree/sync.go | 15 +- 13 files changed, 601 insertions(+), 195 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index b69ab4890..e7d66094c 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -1,24 +1,39 @@ package engine import ( + "context" "errors" + "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) var _ pilorama.Forest = (*StorageEngine)(nil) // TreeMove implements the pilorama.Forest interface. -func (e *StorageEngine) TreeMove(d pilorama.CIDDescriptor, treeID string, m *pilorama.Move) (*pilorama.Move, error) { - index, lst, err := e.getTreeShard(d.CID, treeID) +func (e *StorageEngine) TreeMove(ctx context.Context, d pilorama.CIDDescriptor, treeID string, m *pilorama.Move) (*pilorama.Move, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeMove", + trace.WithAttributes( + attribute.String("container_id", d.CID.EncodeToString()), + attribute.Int("position", d.Position), + attribute.Int("size", d.Size), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + index, lst, err := e.getTreeShard(ctx, d.CID, treeID) if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { return nil, err } - lm, err := lst[index].TreeMove(d, treeID, m) + lm, err := lst[index].TreeMove(ctx, d, treeID, m) if err != nil { if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't perform `TreeMove`", err, @@ -32,13 +47,26 @@ func (e *StorageEngine) TreeMove(d pilorama.CIDDescriptor, treeID string, m *pil } // TreeAddByPath implements the pilorama.Forest interface. -func (e *StorageEngine) TreeAddByPath(d pilorama.CIDDescriptor, treeID string, attr string, path []string, m []pilorama.KeyValue) ([]pilorama.Move, error) { - index, lst, err := e.getTreeShard(d.CID, treeID) +func (e *StorageEngine) TreeAddByPath(ctx context.Context, d pilorama.CIDDescriptor, treeID string, attr string, path []string, m []pilorama.KeyValue) ([]pilorama.Move, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeAddByPath", + trace.WithAttributes( + attribute.String("container_id", d.CID.EncodeToString()), + attribute.Int("position", d.Position), + attribute.Int("size", d.Size), + attribute.String("tree_id", treeID), + attribute.String("attr", attr), + attribute.Int("path_count", len(path)), + attribute.Int("meta_count", len(m)), + ), + ) + defer span.End() + + index, lst, err := e.getTreeShard(ctx, d.CID, treeID) if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { return nil, err } - lm, err := lst[index].TreeAddByPath(d, treeID, attr, path, m) + lm, err := lst[index].TreeAddByPath(ctx, d, treeID, attr, path, m) if err != nil { if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't perform `TreeAddByPath`", err, @@ -51,13 +79,22 @@ func (e *StorageEngine) TreeAddByPath(d pilorama.CIDDescriptor, treeID string, a } // TreeApply implements the pilorama.Forest interface. -func (e *StorageEngine) TreeApply(cnr cidSDK.ID, treeID string, m *pilorama.Move, backgroundSync bool) error { - index, lst, err := e.getTreeShard(cnr, treeID) +func (e *StorageEngine) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string, m *pilorama.Move, backgroundSync bool) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeApply", + trace.WithAttributes( + attribute.String("container_id", cnr.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.Bool("background", backgroundSync), + ), + ) + defer span.End() + + index, lst, err := e.getTreeShard(ctx, cnr, treeID) if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { return err } - err = lst[index].TreeApply(cnr, treeID, m, backgroundSync) + err = lst[index].TreeApply(ctx, cnr, treeID, m, backgroundSync) if err != nil { if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't perform `TreeApply`", err, @@ -70,11 +107,22 @@ func (e *StorageEngine) TreeApply(cnr cidSDK.ID, treeID string, m *pilorama.Move } // TreeGetByPath implements the pilorama.Forest interface. -func (e *StorageEngine) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]pilorama.Node, error) { +func (e *StorageEngine) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]pilorama.Node, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetByPath", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("attr", attr), + attribute.Int("path_count", len(path)), + attribute.Bool("latest", latest), + ), + ) + defer span.End() + var err error var nodes []pilorama.Node for _, sh := range e.sortShardsByWeight(cid) { - nodes, err = sh.TreeGetByPath(cid, treeID, attr, path, latest) + nodes, err = sh.TreeGetByPath(ctx, cid, treeID, attr, path, latest) if err != nil { if err == shard.ErrPiloramaDisabled { break @@ -92,12 +140,21 @@ func (e *StorageEngine) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, } // TreeGetMeta implements the pilorama.Forest interface. -func (e *StorageEngine) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, uint64, error) { +func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetMeta", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + ), + ) + defer span.End() + var err error var m pilorama.Meta var p uint64 for _, sh := range e.sortShardsByWeight(cid) { - m, p, err = sh.TreeGetMeta(cid, treeID, nodeID) + m, p, err = sh.TreeGetMeta(ctx, cid, treeID, nodeID) if err != nil { if err == shard.ErrPiloramaDisabled { break @@ -115,11 +172,20 @@ func (e *StorageEngine) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID piloram } // TreeGetChildren implements the pilorama.Forest interface. -func (e *StorageEngine) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { +func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetChildren", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + ), + ) + defer span.End() + var err error var nodes []uint64 for _, sh := range e.sortShardsByWeight(cid) { - nodes, err = sh.TreeGetChildren(cid, treeID, nodeID) + nodes, err = sh.TreeGetChildren(ctx, cid, treeID, nodeID) if err != nil { if err == shard.ErrPiloramaDisabled { break @@ -137,11 +203,20 @@ func (e *StorageEngine) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID pil } // TreeGetOpLog implements the pilorama.Forest interface. -func (e *StorageEngine) TreeGetOpLog(cid cidSDK.ID, treeID string, height uint64) (pilorama.Move, error) { +func (e *StorageEngine) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (pilorama.Move, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetOpLog", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("height", fmt.Sprintf("%d", height)), + ), + ) + defer span.End() + var err error var lm pilorama.Move for _, sh := range e.sortShardsByWeight(cid) { - lm, err = sh.TreeGetOpLog(cid, treeID, height) + lm, err = sh.TreeGetOpLog(ctx, cid, treeID, height) if err != nil { if err == shard.ErrPiloramaDisabled { break @@ -159,10 +234,18 @@ func (e *StorageEngine) TreeGetOpLog(cid cidSDK.ID, treeID string, height uint64 } // TreeDrop implements the pilorama.Forest interface. -func (e *StorageEngine) TreeDrop(cid cidSDK.ID, treeID string) error { +func (e *StorageEngine) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeDrop", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + var err error for _, sh := range e.sortShardsByWeight(cid) { - err = sh.TreeDrop(cid, treeID) + err = sh.TreeDrop(ctx, cid, treeID) if err != nil { if err == shard.ErrPiloramaDisabled { break @@ -180,11 +263,18 @@ func (e *StorageEngine) TreeDrop(cid cidSDK.ID, treeID string) error { } // TreeList implements the pilorama.Forest interface. -func (e *StorageEngine) TreeList(cid cidSDK.ID) ([]string, error) { +func (e *StorageEngine) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeList", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + ), + ) + defer span.End() + var resIDs []string for _, sh := range e.unsortedShards() { - ids, err := sh.TreeList(cid) + ids, err := sh.TreeList(ctx, cid) if err != nil { if errors.Is(err, shard.ErrPiloramaDisabled) || errors.Is(err, shard.ErrReadOnlyMode) { return nil, err @@ -205,8 +295,16 @@ func (e *StorageEngine) TreeList(cid cidSDK.ID) ([]string, error) { } // TreeExists implements the pilorama.Forest interface. -func (e *StorageEngine) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { - _, _, err := e.getTreeShard(cid, treeID) +func (e *StorageEngine) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeExists", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + _, _, err := e.getTreeShard(ctx, cid, treeID) if errors.Is(err, pilorama.ErrTreeNotFound) { return false, nil } @@ -214,13 +312,22 @@ func (e *StorageEngine) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { } // TreeUpdateLastSyncHeight implements the pilorama.Forest interface. -func (e *StorageEngine) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height uint64) error { - index, lst, err := e.getTreeShard(cid, treeID) +func (e *StorageEngine) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeUpdateLastSyncHeight", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("height", fmt.Sprintf("%d", height)), + ), + ) + defer span.End() + + index, lst, err := e.getTreeShard(ctx, cid, treeID) if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { return err } - err = lst[index].TreeUpdateLastSyncHeight(cid, treeID, height) + err = lst[index].TreeUpdateLastSyncHeight(ctx, cid, treeID, height) if err != nil && !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't update tree synchronization height", err, zap.Stringer("cid", cid), @@ -230,11 +337,19 @@ func (e *StorageEngine) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, h } // TreeLastSyncHeight implements the pilorama.Forest interface. -func (e *StorageEngine) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) { +func (e *StorageEngine) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeLastSyncHeight", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + var err error var height uint64 for _, sh := range e.sortShardsByWeight(cid) { - height, err = sh.TreeLastSyncHeight(cid, treeID) + height, err = sh.TreeLastSyncHeight(ctx, cid, treeID) if err != nil { if err == shard.ErrPiloramaDisabled { break @@ -251,10 +366,10 @@ func (e *StorageEngine) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64 return height, err } -func (e *StorageEngine) getTreeShard(cid cidSDK.ID, treeID string) (int, []hashedShard, error) { +func (e *StorageEngine) getTreeShard(ctx context.Context, cid cidSDK.ID, treeID string) (int, []hashedShard, error) { lst := e.sortShardsByWeight(cid) for i, sh := range lst { - exists, err := sh.TreeExists(cid, treeID) + exists, err := sh.TreeExists(ctx, cid, treeID) if err != nil { return 0, nil, err } diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index 77573c9e6..c2bae9772 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -36,7 +36,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { if err != nil { b.Fatal(err) } - _, err = te.ng.TreeAddByPath(d, treeID, pilorama.AttributeFilename, nil, + _, err = te.ng.TreeAddByPath(context.Background(), d, treeID, pilorama.AttributeFilename, nil, []pilorama.KeyValue{{pilorama.AttributeFilename, []byte(strconv.Itoa(i))}}) if err != nil { b.Fatal(err) @@ -63,7 +63,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { }) b.Run("TreeGetByPath", func(b *testing.B) { for i := 0; i < b.N; i++ { - nodes, err := te.ng.TreeGetByPath(cid, treeID, pilorama.AttributeFilename, []string{strconv.Itoa(objCount / 2)}, true) + nodes, err := te.ng.TreeGetByPath(context.Background(), cid, treeID, pilorama.AttributeFilename, []string{strconv.Itoa(objCount / 2)}, true) if err != nil { b.Fatal(err) } diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 994c3d416..1ecc89cb5 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -2,6 +2,7 @@ package pilorama import ( "bytes" + "context" "encoding/binary" "errors" "fmt" @@ -11,12 +12,15 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/nspcc-dev/neo-go/pkg/io" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) type boltForest struct { @@ -144,7 +148,17 @@ func (t *boltForest) Close() error { } // TreeMove implements the Forest interface. -func (t *boltForest) TreeMove(d CIDDescriptor, treeID string, m *Move) (*Move, error) { +func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID string, m *Move) (*Move, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeMove", + trace.WithAttributes( + attribute.String("container_id", d.CID.EncodeToString()), + attribute.Int("position", d.Position), + attribute.Int("size", d.Size), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + if !d.checkValid() { return nil, ErrInvalidCIDDescriptor } @@ -175,7 +189,15 @@ func (t *boltForest) TreeMove(d CIDDescriptor, treeID string, m *Move) (*Move, e } // TreeExists implements the Forest interface. -func (t *boltForest) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { +func (t *boltForest) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeExists", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + t.modeMtx.RLock() defer t.modeMtx.RUnlock() @@ -197,7 +219,16 @@ func (t *boltForest) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { var syncHeightKey = []byte{'h'} // TreeUpdateLastSyncHeight implements the pilorama.Forest interface. -func (t *boltForest) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height uint64) error { +func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeUpdateLastSyncHeight", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("height", fmt.Sprintf("%d", height)), + ), + ) + defer span.End() + rawHeight := make([]byte, 8) binary.LittleEndian.PutUint64(rawHeight, height) @@ -214,7 +245,15 @@ func (t *boltForest) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, heig } // TreeLastSyncHeight implements the pilorama.Forest interface. -func (t *boltForest) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) { +func (t *boltForest) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeLastSyncHeight", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + var height uint64 buck := bucketName(cid, treeID) @@ -235,7 +274,20 @@ func (t *boltForest) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, e } // TreeAddByPath implements the Forest interface. -func (t *boltForest) TreeAddByPath(d CIDDescriptor, treeID string, attr string, path []string, meta []KeyValue) ([]Move, error) { +func (t *boltForest) TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID string, attr string, path []string, meta []KeyValue) ([]Move, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeAddByPath", + trace.WithAttributes( + attribute.String("container_id", d.CID.EncodeToString()), + attribute.Int("position", d.Position), + attribute.Int("size", d.Size), + attribute.String("tree_id", treeID), + attribute.String("attr", attr), + attribute.Int("path_count", len(path)), + attribute.Int("meta_count", len(meta)), + ), + ) + defer span.End() + if !d.checkValid() { return nil, ErrInvalidCIDDescriptor } @@ -329,7 +381,16 @@ func (t *boltForest) findSpareID(bTree *bbolt.Bucket) uint64 { } // TreeApply implements the Forest interface. -func (t *boltForest) TreeApply(cnr cidSDK.ID, treeID string, m *Move, backgroundSync bool) error { +func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string, m *Move, backgroundSync bool) error { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeApply", + trace.WithAttributes( + attribute.String("container_id", cnr.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.Bool("background", backgroundSync), + ), + ) + defer span.End() + t.modeMtx.RLock() defer t.modeMtx.RUnlock() @@ -627,7 +688,18 @@ func (t *boltForest) isAncestor(b *bbolt.Bucket, parent, child Node) bool { } // TreeGetByPath implements the Forest interface. -func (t *boltForest) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { +func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetByPath", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("attr", attr), + attribute.Int("path_count", len(path)), + attribute.Bool("latest", latest), + ), + ) + defer span.End() + if !isAttributeInternal(attr) { return nil, ErrNotPathAttribute } @@ -686,7 +758,16 @@ func (t *boltForest) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, pa } // TreeGetMeta implements the forest interface. -func (t *boltForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) { +func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetMeta", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + ), + ) + defer span.End() + t.modeMtx.RLock() defer t.modeMtx.RUnlock() @@ -717,7 +798,16 @@ func (t *boltForest) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Met } // TreeGetChildren implements the Forest interface. -func (t *boltForest) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) { +func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetChildren", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + ), + ) + defer span.End() + t.modeMtx.RLock() defer t.modeMtx.RUnlock() @@ -749,7 +839,14 @@ func (t *boltForest) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) } // TreeList implements the Forest interface. -func (t *boltForest) TreeList(cid cidSDK.ID) ([]string, error) { +func (t *boltForest) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeList", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + ), + ) + defer span.End() + t.modeMtx.RLock() defer t.modeMtx.RUnlock() @@ -783,7 +880,16 @@ func (t *boltForest) TreeList(cid cidSDK.ID) ([]string, error) { } // TreeGetOpLog implements the pilorama.Forest interface. -func (t *boltForest) TreeGetOpLog(cid cidSDK.ID, treeID string, height uint64) (Move, error) { +func (t *boltForest) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetOpLog", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("height", fmt.Sprintf("%d", height)), + ), + ) + defer span.End() + t.modeMtx.RLock() defer t.modeMtx.RUnlock() @@ -813,7 +919,15 @@ func (t *boltForest) TreeGetOpLog(cid cidSDK.ID, treeID string, height uint64) ( } // TreeDrop implements the pilorama.Forest interface. -func (t *boltForest) TreeDrop(cid cidSDK.ID, treeID string) error { +func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) error { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeDrop", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + t.modeMtx.RLock() defer t.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index fa2f1dcd2..84530977c 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -1,6 +1,7 @@ package pilorama import ( + "context" "sort" "strings" @@ -25,7 +26,7 @@ func NewMemoryForest() ForestStorage { } // TreeMove implements the Forest interface. -func (f *memoryForest) TreeMove(d CIDDescriptor, treeID string, op *Move) (*Move, error) { +func (f *memoryForest) TreeMove(_ context.Context, d CIDDescriptor, treeID string, op *Move) (*Move, error) { if !d.checkValid() { return nil, ErrInvalidCIDDescriptor } @@ -48,7 +49,7 @@ func (f *memoryForest) TreeMove(d CIDDescriptor, treeID string, op *Move) (*Move } // TreeAddByPath implements the Forest interface. -func (f *memoryForest) TreeAddByPath(d CIDDescriptor, treeID string, attr string, path []string, m []KeyValue) ([]Move, error) { +func (f *memoryForest) TreeAddByPath(_ context.Context, d CIDDescriptor, treeID string, attr string, path []string, m []KeyValue) ([]Move, error) { if !d.checkValid() { return nil, ErrInvalidCIDDescriptor } @@ -93,7 +94,7 @@ func (f *memoryForest) TreeAddByPath(d CIDDescriptor, treeID string, attr string } // TreeApply implements the Forest interface. -func (f *memoryForest) TreeApply(cnr cid.ID, treeID string, op *Move, _ bool) error { +func (f *memoryForest) TreeApply(_ context.Context, cnr cid.ID, treeID string, op *Move, _ bool) error { fullID := cnr.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -119,7 +120,7 @@ func (f *memoryForest) Close() error { } // TreeGetByPath implements the Forest interface. -func (f *memoryForest) TreeGetByPath(cid cid.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { +func (f *memoryForest) TreeGetByPath(_ context.Context, cid cid.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { if !isAttributeInternal(attr) { return nil, ErrNotPathAttribute } @@ -134,7 +135,7 @@ func (f *memoryForest) TreeGetByPath(cid cid.ID, treeID string, attr string, pat } // TreeGetMeta implements the Forest interface. -func (f *memoryForest) TreeGetMeta(cid cid.ID, treeID string, nodeID Node) (Meta, Node, error) { +func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string, nodeID Node) (Meta, Node, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -145,7 +146,7 @@ func (f *memoryForest) TreeGetMeta(cid cid.ID, treeID string, nodeID Node) (Meta } // TreeGetChildren implements the Forest interface. -func (f *memoryForest) TreeGetChildren(cid cid.ID, treeID string, nodeID Node) ([]uint64, error) { +func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]uint64, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -163,7 +164,7 @@ func (f *memoryForest) TreeGetChildren(cid cid.ID, treeID string, nodeID Node) ( } // TreeGetOpLog implements the pilorama.Forest interface. -func (f *memoryForest) TreeGetOpLog(cid cid.ID, treeID string, height uint64) (Move, error) { +func (f *memoryForest) TreeGetOpLog(_ context.Context, cid cid.ID, treeID string, height uint64) (Move, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -180,7 +181,7 @@ func (f *memoryForest) TreeGetOpLog(cid cid.ID, treeID string, height uint64) (M } // TreeDrop implements the pilorama.Forest interface. -func (f *memoryForest) TreeDrop(cid cid.ID, treeID string) error { +func (f *memoryForest) TreeDrop(_ context.Context, cid cid.ID, treeID string) error { cidStr := cid.String() if treeID == "" { for k := range f.treeMap { @@ -200,7 +201,7 @@ func (f *memoryForest) TreeDrop(cid cid.ID, treeID string) error { } // TreeList implements the pilorama.Forest interface. -func (f *memoryForest) TreeList(cid cid.ID) ([]string, error) { +func (f *memoryForest) TreeList(_ context.Context, cid cid.ID) ([]string, error) { var res []string cidStr := cid.EncodeToString() @@ -217,14 +218,14 @@ func (f *memoryForest) TreeList(cid cid.ID) ([]string, error) { } // TreeExists implements the pilorama.Forest interface. -func (f *memoryForest) TreeExists(cid cid.ID, treeID string) (bool, error) { +func (f *memoryForest) TreeExists(_ context.Context, cid cid.ID, treeID string) (bool, error) { fullID := cid.EncodeToString() + "/" + treeID _, ok := f.treeMap[fullID] return ok, nil } // TreeUpdateLastSyncHeight implements the pilorama.Forest interface. -func (f *memoryForest) TreeUpdateLastSyncHeight(cid cid.ID, treeID string, height uint64) error { +func (f *memoryForest) TreeUpdateLastSyncHeight(_ context.Context, cid cid.ID, treeID string, height uint64) error { fullID := cid.EncodeToString() + "/" + treeID t, ok := f.treeMap[fullID] if !ok { @@ -235,7 +236,7 @@ func (f *memoryForest) TreeUpdateLastSyncHeight(cid cid.ID, treeID string, heigh } // TreeLastSyncHeight implements the pilorama.Forest interface. -func (f *memoryForest) TreeLastSyncHeight(cid cid.ID, treeID string) (uint64, error) { +func (f *memoryForest) TreeLastSyncHeight(_ context.Context, cid cid.ID, treeID string) (uint64, error) { fullID := cid.EncodeToString() + "/" + treeID t, ok := f.treeMap[fullID] if !ok { diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index bbd35246c..be53b3fe2 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -1,6 +1,7 @@ package pilorama import ( + "context" "fmt" "math/rand" "os" @@ -49,7 +50,7 @@ var providers = []struct { } func testMeta(t *testing.T, f Forest, cid cidSDK.ID, treeID string, nodeID, parentID Node, expected Meta) { - actualMeta, actualParent, err := f.TreeGetMeta(cid, treeID, nodeID) + actualMeta, actualParent, err := f.TreeGetMeta(context.Background(), cid, treeID, nodeID) require.NoError(t, err) require.Equal(t, parentID, actualParent) require.Equal(t, expected, actualMeta) @@ -71,13 +72,13 @@ func testForestTreeMove(t *testing.T, s Forest) { meta := []KeyValue{ {Key: AttributeVersion, Value: []byte("XXX")}, {Key: AttributeFilename, Value: []byte("file.txt")}} - lm, err := s.TreeAddByPath(d, treeID, AttributeFilename, []string{"path", "to"}, meta) + lm, err := s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, []string{"path", "to"}, meta) require.NoError(t, err) require.Equal(t, 3, len(lm)) nodeID := lm[2].Child t.Run("invalid descriptor", func(t *testing.T) { - _, err = s.TreeMove(CIDDescriptor{cid, 0, 0}, treeID, &Move{ + _, err = s.TreeMove(context.Background(), CIDDescriptor{cid, 0, 0}, treeID, &Move{ Parent: lm[1].Child, Meta: Meta{Items: append(meta, KeyValue{Key: "NewKey", Value: []byte("NewValue")})}, Child: nodeID, @@ -85,7 +86,7 @@ func testForestTreeMove(t *testing.T, s Forest) { require.ErrorIs(t, err, ErrInvalidCIDDescriptor) }) t.Run("same parent, update meta", func(t *testing.T) { - res, err := s.TreeMove(d, treeID, &Move{ + res, err := s.TreeMove(context.Background(), d, treeID, &Move{ Parent: lm[1].Child, Meta: Meta{Items: append(meta, KeyValue{Key: "NewKey", Value: []byte("NewValue")})}, Child: nodeID, @@ -93,12 +94,12 @@ func testForestTreeMove(t *testing.T, s Forest) { require.NoError(t, err) require.Equal(t, res.Child, nodeID) - nodes, err := s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"path", "to", "file.txt"}, false) + nodes, err := s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"path", "to", "file.txt"}, false) require.NoError(t, err) require.ElementsMatch(t, []Node{nodeID}, nodes) }) t.Run("different parent", func(t *testing.T) { - res, err := s.TreeMove(d, treeID, &Move{ + res, err := s.TreeMove(context.Background(), d, treeID, &Move{ Parent: RootID, Meta: Meta{Items: append(meta, KeyValue{Key: "NewKey", Value: []byte("NewValue")})}, Child: nodeID, @@ -106,11 +107,11 @@ func testForestTreeMove(t *testing.T, s Forest) { require.NoError(t, err) require.Equal(t, res.Child, nodeID) - nodes, err := s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"path", "to", "file.txt"}, false) + nodes, err := s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"path", "to", "file.txt"}, false) require.NoError(t, err) require.True(t, len(nodes) == 0) - nodes, err = s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"file.txt"}, false) + nodes, err = s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"file.txt"}, false) require.NoError(t, err) require.ElementsMatch(t, []Node{nodeID}, nodes) }) @@ -130,7 +131,7 @@ func testForestTreeGetChildren(t *testing.T, s Forest) { treeID := "version" treeAdd := func(t *testing.T, child, parent Node) { - _, err := s.TreeMove(d, treeID, &Move{ + _, err := s.TreeMove(context.Background(), d, treeID, &Move{ Parent: parent, Child: child, }) @@ -152,7 +153,7 @@ func testForestTreeGetChildren(t *testing.T, s Forest) { treeAdd(t, 7, 0) testGetChildren := func(t *testing.T, nodeID Node, expected []Node) { - actual, err := s.TreeGetChildren(cid, treeID, nodeID) + actual, err := s.TreeGetChildren(context.Background(), cid, treeID, nodeID) require.NoError(t, err) require.ElementsMatch(t, expected, actual) } @@ -168,7 +169,7 @@ func testForestTreeGetChildren(t *testing.T, s Forest) { testGetChildren(t, 42, nil) }) t.Run("missing tree", func(t *testing.T) { - _, err := s.TreeGetChildren(cid, treeID+"123", 0) + _, err := s.TreeGetChildren(context.Background(), cid, treeID+"123", 0) require.ErrorIs(t, err, ErrTreeNotFound) }) } @@ -191,10 +192,10 @@ func testForestTreeDrop(t *testing.T, s Forest) { cid := cids[0] t.Run("return nil if not found", func(t *testing.T) { - require.ErrorIs(t, s.TreeDrop(cid, "123"), ErrTreeNotFound) + require.ErrorIs(t, s.TreeDrop(context.Background(), cid, "123"), ErrTreeNotFound) }) - require.NoError(t, s.TreeDrop(cid, "")) + require.NoError(t, s.TreeDrop(context.Background(), cid, "")) trees := []string{"tree1", "tree2"} var descs [cidsSize]CIDDescriptor @@ -203,39 +204,39 @@ func testForestTreeDrop(t *testing.T, s Forest) { } d := descs[0] for i := range trees { - _, err := s.TreeAddByPath(d, trees[i], AttributeFilename, []string{"path"}, + _, err := s.TreeAddByPath(context.Background(), d, trees[i], AttributeFilename, []string{"path"}, []KeyValue{{Key: "TreeName", Value: []byte(trees[i])}}) require.NoError(t, err) } - err := s.TreeDrop(cid, trees[0]) + err := s.TreeDrop(context.Background(), cid, trees[0]) require.NoError(t, err) - _, err = s.TreeGetByPath(cid, trees[0], AttributeFilename, []string{"path"}, true) + _, err = s.TreeGetByPath(context.Background(), cid, trees[0], AttributeFilename, []string{"path"}, true) require.ErrorIs(t, err, ErrTreeNotFound) - _, err = s.TreeGetByPath(cid, trees[1], AttributeFilename, []string{"path"}, true) + _, err = s.TreeGetByPath(context.Background(), cid, trees[1], AttributeFilename, []string{"path"}, true) require.NoError(t, err) for j := range descs { for i := range trees { - _, err := s.TreeAddByPath(descs[j], trees[i], AttributeFilename, []string{"path"}, + _, err := s.TreeAddByPath(context.Background(), descs[j], trees[i], AttributeFilename, []string{"path"}, []KeyValue{{Key: "TreeName", Value: []byte(trees[i])}}) require.NoError(t, err) } } - list, err := s.TreeList(cid) + list, err := s.TreeList(context.Background(), cid) require.NoError(t, err) require.NotEmpty(t, list) - require.NoError(t, s.TreeDrop(cid, "")) + require.NoError(t, s.TreeDrop(context.Background(), cid, "")) - list, err = s.TreeList(cid) + list, err = s.TreeList(context.Background(), cid) require.NoError(t, err) require.Empty(t, list) for j := 1; j < len(cids); j++ { - list, err = s.TreeList(cids[j]) + list, err = s.TreeList(context.Background(), cids[j]) require.NoError(t, err) require.Equal(t, len(list), len(trees)) } @@ -264,24 +265,24 @@ func testForestTreeAdd(t *testing.T, s Forest) { } t.Run("invalid descriptor", func(t *testing.T) { - _, err := s.TreeMove(CIDDescriptor{cid, 0, 0}, treeID, m) + _, err := s.TreeMove(context.Background(), CIDDescriptor{cid, 0, 0}, treeID, m) require.ErrorIs(t, err, ErrInvalidCIDDescriptor) }) - lm, err := s.TreeMove(d, treeID, m) + lm, err := s.TreeMove(context.Background(), d, treeID, m) require.NoError(t, err) testMeta(t, s, cid, treeID, lm.Child, lm.Parent, Meta{Time: lm.Time, Items: meta}) - nodes, err := s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"file.txt"}, false) + nodes, err := s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"file.txt"}, false) require.NoError(t, err) require.ElementsMatch(t, []Node{lm.Child}, nodes) t.Run("other trees are unaffected", func(t *testing.T) { - _, err := s.TreeGetByPath(cid, treeID+"123", AttributeFilename, []string{"file.txt"}, false) + _, err := s.TreeGetByPath(context.Background(), cid, treeID+"123", AttributeFilename, []string{"file.txt"}, false) require.ErrorIs(t, err, ErrTreeNotFound) - _, _, err = s.TreeGetMeta(cid, treeID+"123", 0) + _, _, err = s.TreeGetMeta(context.Background(), cid, treeID+"123", 0) require.ErrorIs(t, err, ErrTreeNotFound) }) } @@ -304,15 +305,15 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { {Key: AttributeFilename, Value: []byte("file.txt")}} t.Run("invalid descriptor", func(t *testing.T) { - _, err := s.TreeAddByPath(CIDDescriptor{cid, 0, 0}, treeID, AttributeFilename, []string{"yyy"}, meta) + _, err := s.TreeAddByPath(context.Background(), CIDDescriptor{cid, 0, 0}, treeID, AttributeFilename, []string{"yyy"}, meta) require.ErrorIs(t, err, ErrInvalidCIDDescriptor) }) t.Run("invalid attribute", func(t *testing.T) { - _, err := s.TreeAddByPath(d, treeID, AttributeVersion, []string{"yyy"}, meta) + _, err := s.TreeAddByPath(context.Background(), d, treeID, AttributeVersion, []string{"yyy"}, meta) require.ErrorIs(t, err, ErrNotPathAttribute) }) - lm, err := s.TreeAddByPath(d, treeID, AttributeFilename, []string{"path", "to"}, meta) + lm, err := s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, []string{"path", "to"}, meta) require.NoError(t, err) require.Equal(t, 3, len(lm)) testMeta(t, s, cid, treeID, lm[0].Child, lm[0].Parent, Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("path")}}}) @@ -322,7 +323,7 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { testMeta(t, s, cid, treeID, firstID, lm[2].Parent, Meta{Time: lm[2].Time, Items: meta}) meta[0].Value = []byte("YYY") - lm, err = s.TreeAddByPath(d, treeID, AttributeFilename, []string{"path", "to"}, meta) + lm, err = s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, []string{"path", "to"}, meta) require.NoError(t, err) require.Equal(t, 1, len(lm)) @@ -331,19 +332,19 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { t.Run("get versions", func(t *testing.T) { // All versions. - nodes, err := s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"path", "to", "file.txt"}, false) + nodes, err := s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"path", "to", "file.txt"}, false) require.NoError(t, err) require.ElementsMatch(t, []Node{firstID, secondID}, nodes) // Latest version. - nodes, err = s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"path", "to", "file.txt"}, true) + nodes, err = s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"path", "to", "file.txt"}, true) require.NoError(t, err) require.Equal(t, []Node{secondID}, nodes) }) meta[0].Value = []byte("ZZZ") meta[1].Value = []byte("cat.jpg") - lm, err = s.TreeAddByPath(d, treeID, AttributeFilename, []string{"path", "dir"}, meta) + lm, err = s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, []string{"path", "dir"}, meta) require.NoError(t, err) require.Equal(t, 2, len(lm)) testMeta(t, s, cid, treeID, lm[0].Child, lm[0].Parent, Meta{Time: lm[0].Time, Items: []KeyValue{{AttributeFilename, []byte("dir")}}}) @@ -352,7 +353,7 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { t.Run("create internal nodes", func(t *testing.T) { meta[0].Value = []byte("SomeValue") meta[1].Value = []byte("another") - lm, err = s.TreeAddByPath(d, treeID, AttributeFilename, []string{"path"}, meta) + lm, err = s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, []string{"path"}, meta) require.NoError(t, err) require.Equal(t, 1, len(lm)) @@ -360,7 +361,7 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { meta[0].Value = []byte("Leaf") meta[1].Value = []byte("file.txt") - lm, err = s.TreeAddByPath(d, treeID, AttributeFilename, []string{"path", "another"}, meta) + lm, err = s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, []string{"path", "another"}, meta) require.NoError(t, err) require.Equal(t, 2, len(lm)) @@ -375,12 +376,12 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { {AttributeFilename, []byte("another")}}}) t.Run("get by path", func(t *testing.T) { - nodes, err := s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"path", "another"}, false) + nodes, err := s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"path", "another"}, false) require.NoError(t, err) require.Equal(t, 2, len(nodes)) require.ElementsMatch(t, []Node{lm[0].Child, oldMove.Child}, nodes) - nodes, err = s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"path", "another", "file.txt"}, false) + nodes, err = s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"path", "another", "file.txt"}, false) require.NoError(t, err) require.Equal(t, 1, len(nodes)) require.Equal(t, lm[1].Child, nodes[0]) @@ -391,11 +392,11 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { meta := []KeyValue{ {Key: AttributeVersion, Value: []byte("XXX")}, {Key: AttributeFilename, Value: []byte{}}} - lm, err := s.TreeAddByPath(d, treeID, AttributeFilename, []string{"path", "to"}, meta) + lm, err := s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, []string{"path", "to"}, meta) require.NoError(t, err) require.Equal(t, 1, len(lm)) - nodes, err := s.TreeGetByPath(d.CID, treeID, AttributeFilename, []string{"path", "to", ""}, false) + nodes, err := s.TreeGetByPath(context.Background(), d.CID, treeID, AttributeFilename, []string{"path", "to", ""}, false) require.NoError(t, err) require.Equal(t, 1, len(nodes)) require.Equal(t, lm[0].Child, nodes[0]) @@ -415,7 +416,7 @@ func testForestTreeApply(t *testing.T, constructor func(t testing.TB, _ ...Optio treeID := "version" testApply := func(t *testing.T, s Forest, child, parent Node, meta Meta) { - require.NoError(t, s.TreeApply(cid, treeID, &Move{ + require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &Move{ Child: child, Parent: parent, Meta: meta, @@ -475,16 +476,16 @@ func testForestTreeGetOpLog(t *testing.T, constructor func(t testing.TB, _ ...Op s := constructor(t) t.Run("empty log, no panic", func(t *testing.T) { - _, err := s.TreeGetOpLog(cid, treeID, 0) + _, err := s.TreeGetOpLog(context.Background(), cid, treeID, 0) require.ErrorIs(t, err, ErrTreeNotFound) }) for i := range logs { - require.NoError(t, s.TreeApply(cid, treeID, &logs[i], false)) + require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &logs[i], false)) } testGetOpLog := func(t *testing.T, height uint64, m Move) { - lm, err := s.TreeGetOpLog(cid, treeID, height) + lm, err := s.TreeGetOpLog(context.Background(), cid, treeID, height) require.NoError(t, err) require.Equal(t, m, lm) } @@ -498,7 +499,7 @@ func testForestTreeGetOpLog(t *testing.T, constructor func(t testing.TB, _ ...Op testGetOpLog(t, 261, Move{}) }) t.Run("missing tree", func(t *testing.T) { - _, err := s.TreeGetOpLog(cid, treeID+"123", 4) + _, err := s.TreeGetOpLog(context.Background(), cid, treeID+"123", 4) require.ErrorIs(t, err, ErrTreeNotFound) }) } @@ -515,7 +516,7 @@ func testForestTreeExists(t *testing.T, constructor func(t testing.TB, opts ...O s := constructor(t) checkExists := func(t *testing.T, expected bool, cid cidSDK.ID, treeID string) { - actual, err := s.TreeExists(cid, treeID) + actual, err := s.TreeExists(context.Background(), cid, treeID) require.NoError(t, err) require.Equal(t, expected, actual) } @@ -527,13 +528,13 @@ func testForestTreeExists(t *testing.T, constructor func(t testing.TB, opts ...O checkExists(t, false, cid, treeID) }) - require.NoError(t, s.TreeApply(cid, treeID, &Move{Parent: 0, Child: 1}, false)) + require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &Move{Parent: 0, Child: 1}, false)) checkExists(t, true, cid, treeID) checkExists(t, false, cidtest.ID(), treeID) // different CID, same tree checkExists(t, false, cid, "another tree") // same CID, different tree t.Run("can be removed", func(t *testing.T) { - require.NoError(t, s.TreeDrop(cid, treeID)) + require.NoError(t, s.TreeDrop(context.Background(), cid, treeID)) checkExists(t, false, cid, treeID) }) } @@ -563,11 +564,11 @@ func TestApplyTricky1(t *testing.T) { t.Run(providers[i].name, func(t *testing.T) { s := providers[i].construct(t) for i := range ops { - require.NoError(t, s.TreeApply(cid, treeID, &ops[i], false)) + require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } for i := range expected { - _, parent, err := s.TreeGetMeta(cid, treeID, expected[i].child) + _, parent, err := s.TreeGetMeta(context.Background(), cid, treeID, expected[i].child) require.NoError(t, err) require.Equal(t, expected[i].parent, parent) } @@ -624,11 +625,11 @@ func TestApplyTricky2(t *testing.T) { t.Run(providers[i].name, func(t *testing.T) { s := providers[i].construct(t) for i := range ops { - require.NoError(t, s.TreeApply(cid, treeID, &ops[i], false)) + require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } for i := range expected { - _, parent, err := s.TreeGetMeta(cid, treeID, expected[i].child) + _, parent, err := s.TreeGetMeta(context.Background(), cid, treeID, expected[i].child) require.NoError(t, err) require.Equal(t, expected[i].parent, parent) } @@ -697,9 +698,9 @@ func prepareRandomTree(nodeCount, opCount int) []Move { func compareForests(t *testing.T, expected, actual Forest, cid cidSDK.ID, treeID string, nodeCount int) { for i := uint64(0); i < uint64(nodeCount); i++ { - expectedMeta, expectedParent, err := expected.TreeGetMeta(cid, treeID, i) + expectedMeta, expectedParent, err := expected.TreeGetMeta(context.Background(), cid, treeID, i) require.NoError(t, err) - actualMeta, actualParent, err := actual.TreeGetMeta(cid, treeID, i) + actualMeta, actualParent, err := actual.TreeGetMeta(context.Background(), cid, treeID, i) require.NoError(t, err) require.Equal(t, expectedParent, actualParent, "node id: %d", i) require.Equal(t, expectedMeta, actualMeta, "node id: %d", i) @@ -738,7 +739,7 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ expected := constructor(t) for i := range ops { - require.NoError(t, expected.TreeApply(cid, treeID, &ops[i], false)) + require.NoError(t, expected.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } for i := 0; i < iterCount; i++ { @@ -753,7 +754,7 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ go func() { defer wg.Done() for op := range ch { - require.NoError(t, actual.TreeApply(cid, treeID, op, false)) + require.NoError(t, actual.TreeApply(context.Background(), cid, treeID, op, false)) } }() } @@ -783,7 +784,7 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ .. expected := constructor(t) for i := range ops { - require.NoError(t, expected.TreeApply(cid, treeID, &ops[i], false)) + require.NoError(t, expected.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } const iterCount = 200 @@ -793,7 +794,7 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ .. actual := constructor(t) for i := range ops { - require.NoError(t, actual.TreeApply(cid, treeID, &ops[i], false)) + require.NoError(t, actual.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } compareForests(t, expected, actual, cid, treeID, nodeCount) } @@ -886,7 +887,7 @@ func benchmarkApply(b *testing.B, s Forest, genFunc func(int) []Move) { b.SetParallelism(10) b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := s.TreeApply(cid, treeID, &ops[<-ch], false); err != nil { + if err := s.TreeApply(context.Background(), cid, treeID, &ops[<-ch], false); err != nil { b.Fatalf("error in `Apply`: %v", err) } } @@ -929,27 +930,27 @@ func testTreeGetByPath(t *testing.T, s Forest) { } t.Run("invalid attribute", func(t *testing.T) { - _, err := s.TreeGetByPath(cid, treeID, AttributeVersion, []string{"", "TTT"}, false) + _, err := s.TreeGetByPath(context.Background(), cid, treeID, AttributeVersion, []string{"", "TTT"}, false) require.ErrorIs(t, err, ErrNotPathAttribute) }) - nodes, err := s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"b", "cat1.jpg"}, false) + nodes, err := s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"b", "cat1.jpg"}, false) require.NoError(t, err) require.Equal(t, []Node{4, 5}, nodes) - nodes, err = s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"a", "cat1.jpg"}, false) + nodes, err = s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"a", "cat1.jpg"}, false) require.Equal(t, []Node{3}, nodes) t.Run("missing child", func(t *testing.T) { - nodes, err = s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"a", "cat3.jpg"}, false) + nodes, err = s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"a", "cat3.jpg"}, false) require.True(t, len(nodes) == 0) }) t.Run("missing parent", func(t *testing.T) { - nodes, err = s.TreeGetByPath(cid, treeID, AttributeFilename, []string{"xyz", "cat1.jpg"}, false) + nodes, err = s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"xyz", "cat1.jpg"}, false) require.True(t, len(nodes) == 0) }) t.Run("empty path", func(t *testing.T) { - nodes, err = s.TreeGetByPath(cid, treeID, AttributeFilename, nil, false) + nodes, err = s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, nil, false) require.True(t, len(nodes) == 0) }) } @@ -961,7 +962,7 @@ func testMove(t *testing.T, s Forest, ts int, node, parent Node, cid cidSDK.ID, items = append(items, KeyValue{AttributeVersion, []byte(version)}) } - require.NoError(t, s.TreeApply(cid, treeID, &Move{ + require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &Move{ Parent: parent, Child: node, Meta: Meta{ @@ -1000,7 +1001,7 @@ func testTreeGetTrees(t *testing.T, s Forest) { d.CID = cid for _, treeID := range treeIDs[cid] { - _, err := s.TreeAddByPath(d, treeID, objectSDK.AttributeFileName, []string{"path"}, nil) + _, err := s.TreeAddByPath(context.Background(), d, treeID, objectSDK.AttributeFileName, []string{"path"}, nil) require.NoError(t, err) } } @@ -1008,7 +1009,7 @@ func testTreeGetTrees(t *testing.T, s Forest) { for _, cid := range cids { d.CID = cid - trees, err := s.TreeList(cid) + trees, err := s.TreeList(context.Background(), cid) require.NoError(t, err) require.ElementsMatch(t, treeIDs[cid], trees) @@ -1028,38 +1029,38 @@ func testTreeLastSyncHeight(t *testing.T, f Forest) { treeID := "someTree" t.Run("ErrNotFound if no log operations are stored for a tree", func(t *testing.T) { - _, err := f.TreeLastSyncHeight(cnr, treeID) + _, err := f.TreeLastSyncHeight(context.Background(), cnr, treeID) require.ErrorIs(t, err, ErrTreeNotFound) - err = f.TreeUpdateLastSyncHeight(cnr, treeID, 1) + err = f.TreeUpdateLastSyncHeight(context.Background(), cnr, treeID, 1) require.ErrorIs(t, err, ErrTreeNotFound) }) - _, err := f.TreeMove(CIDDescriptor{CID: cnr, Size: 1}, treeID, &Move{ + _, err := f.TreeMove(context.Background(), CIDDescriptor{CID: cnr, Size: 1}, treeID, &Move{ Parent: RootID, Child: 1, }) require.NoError(t, err) - h, err := f.TreeLastSyncHeight(cnr, treeID) + h, err := f.TreeLastSyncHeight(context.Background(), cnr, treeID) require.NoError(t, err) require.EqualValues(t, 0, h) t.Run("separate storages for separate containers", func(t *testing.T) { - _, err := f.TreeLastSyncHeight(cidtest.ID(), treeID) + _, err := f.TreeLastSyncHeight(context.Background(), cidtest.ID(), treeID) require.ErrorIs(t, err, ErrTreeNotFound) }) - require.NoError(t, f.TreeUpdateLastSyncHeight(cnr, treeID, 10)) + require.NoError(t, f.TreeUpdateLastSyncHeight(context.Background(), cnr, treeID, 10)) - h, err = f.TreeLastSyncHeight(cnr, treeID) + h, err = f.TreeLastSyncHeight(context.Background(), cnr, treeID) require.NoError(t, err) require.EqualValues(t, 10, h) t.Run("removed correctly", func(t *testing.T) { - require.NoError(t, f.TreeDrop(cnr, treeID)) + require.NoError(t, f.TreeDrop(context.Background(), cnr, treeID)) - _, err := f.TreeLastSyncHeight(cnr, treeID) + _, err := f.TreeLastSyncHeight(context.Background(), cnr, treeID) require.ErrorIs(t, err, ErrTreeNotFound) }) } diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 290f633a5..9ca721be8 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -1,6 +1,8 @@ package pilorama import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -11,43 +13,43 @@ type Forest interface { // TreeMove moves node in the tree. // If the parent of the move operation is TrashID, the node is removed. // If the child of the move operation is RootID, new ID is generated and added to a tree. - TreeMove(d CIDDescriptor, treeID string, m *Move) (*Move, error) + TreeMove(ctx context.Context, d CIDDescriptor, treeID string, m *Move) (*Move, error) // TreeAddByPath adds new node in the tree using provided path. // The path is constructed by descending from the root using the values of the attr in meta. // Internal nodes in path should have exactly one attribute, otherwise a new node is created. - TreeAddByPath(d CIDDescriptor, treeID string, attr string, path []string, meta []KeyValue) ([]Move, error) + TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID string, attr string, path []string, meta []KeyValue) ([]Move, error) // TreeApply applies replicated operation from another node. // If background is true, TreeApply will first check whether an operation exists. - TreeApply(cnr cidSDK.ID, treeID string, m *Move, backgroundSync bool) error + TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string, m *Move, backgroundSync bool) error // TreeGetByPath returns all nodes corresponding to the path. // The path is constructed by descending from the root using the values of the // AttributeFilename in meta. // The last argument determines whether only the node with the latest timestamp is returned. // Should return ErrTreeNotFound if the tree is not found, and empty result if the path is not in the tree. - TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) + TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) // TreeGetMeta returns meta information of the node with the specified ID. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. - TreeGetMeta(cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) + TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) // TreeGetChildren returns children of the node with the specified ID. The order is arbitrary. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. - TreeGetChildren(cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) + TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) // TreeGetOpLog returns first log operation stored at or above the height. // In case no such operation is found, empty Move and nil error should be returned. - TreeGetOpLog(cid cidSDK.ID, treeID string, height uint64) (Move, error) + TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) // TreeDrop drops a tree from the database. // If the tree is not found, ErrTreeNotFound should be returned. // In case of empty treeID drops all trees related to container. - TreeDrop(cid cidSDK.ID, treeID string) error + TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) error // TreeList returns all the tree IDs that have been added to the // passed container ID. Nil slice should be returned if no tree found. - TreeList(cid cidSDK.ID) ([]string, error) + TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) // TreeExists checks if a tree exists locally. // If the tree is not found, false and a nil error should be returned. - TreeExists(cid cidSDK.ID, treeID string) (bool, error) + TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) // TreeUpdateLastSyncHeight updates last log height synchronized with _all_ container nodes. - TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height uint64) error + TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error // TreeLastSyncHeight returns last log height synchronized with _all_ container nodes. - TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) + TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) } type ForestStorage interface { diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index db07c001e..d5b3b67bf 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -1,9 +1,15 @@ package shard import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) var _ pilorama.Forest = (*Shard)(nil) @@ -12,7 +18,18 @@ var _ pilorama.Forest = (*Shard)(nil) var ErrPiloramaDisabled = logicerr.New("pilorama is disabled") // TreeMove implements the pilorama.Forest interface. -func (s *Shard) TreeMove(d pilorama.CIDDescriptor, treeID string, m *pilorama.Move) (*pilorama.Move, error) { +func (s *Shard) TreeMove(ctx context.Context, d pilorama.CIDDescriptor, treeID string, m *pilorama.Move) (*pilorama.Move, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeMove", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", d.CID.EncodeToString()), + attribute.Int("position", d.Position), + attribute.Int("size", d.Size), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + if s.pilorama == nil { return nil, ErrPiloramaDisabled } @@ -26,11 +43,25 @@ func (s *Shard) TreeMove(d pilorama.CIDDescriptor, treeID string, m *pilorama.Mo if s.info.Mode.NoMetabase() { return nil, ErrDegradedMode } - return s.pilorama.TreeMove(d, treeID, m) + return s.pilorama.TreeMove(ctx, d, treeID, m) } // TreeAddByPath implements the pilorama.Forest interface. -func (s *Shard) TreeAddByPath(d pilorama.CIDDescriptor, treeID string, attr string, path []string, meta []pilorama.KeyValue) ([]pilorama.Move, error) { +func (s *Shard) TreeAddByPath(ctx context.Context, d pilorama.CIDDescriptor, treeID string, attr string, path []string, meta []pilorama.KeyValue) ([]pilorama.Move, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeAddByPath", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", d.CID.EncodeToString()), + attribute.Int("position", d.Position), + attribute.Int("size", d.Size), + attribute.String("tree_id", treeID), + attribute.String("attr", attr), + attribute.Int("path_count", len(path)), + attribute.Int("meta_count", len(meta)), + ), + ) + defer span.End() + if s.pilorama == nil { return nil, ErrPiloramaDisabled } @@ -44,11 +75,21 @@ func (s *Shard) TreeAddByPath(d pilorama.CIDDescriptor, treeID string, attr stri if s.info.Mode.NoMetabase() { return nil, ErrDegradedMode } - return s.pilorama.TreeAddByPath(d, treeID, attr, path, meta) + return s.pilorama.TreeAddByPath(ctx, d, treeID, attr, path, meta) } // TreeApply implements the pilorama.Forest interface. -func (s *Shard) TreeApply(cnr cidSDK.ID, treeID string, m *pilorama.Move, backgroundSync bool) error { +func (s *Shard) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string, m *pilorama.Move, backgroundSync bool) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeApply", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cnr.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.Bool("background", backgroundSync), + ), + ) + defer span.End() + if s.pilorama == nil { return ErrPiloramaDisabled } @@ -62,11 +103,23 @@ func (s *Shard) TreeApply(cnr cidSDK.ID, treeID string, m *pilorama.Move, backgr if s.info.Mode.NoMetabase() { return ErrDegradedMode } - return s.pilorama.TreeApply(cnr, treeID, m, backgroundSync) + return s.pilorama.TreeApply(ctx, cnr, treeID, m, backgroundSync) } // TreeGetByPath implements the pilorama.Forest interface. -func (s *Shard) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]pilorama.Node, error) { +func (s *Shard) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]pilorama.Node, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetByPath", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("attr", attr), + attribute.Int("path_count", len(path)), + attribute.Bool("latest", latest), + ), + ) + defer span.End() + if s.pilorama == nil { return nil, ErrPiloramaDisabled } @@ -77,11 +130,21 @@ func (s *Shard) TreeGetByPath(cid cidSDK.ID, treeID string, attr string, path [] if s.info.Mode.NoMetabase() { return nil, ErrDegradedMode } - return s.pilorama.TreeGetByPath(cid, treeID, attr, path, latest) + return s.pilorama.TreeGetByPath(ctx, cid, treeID, attr, path, latest) } // TreeGetMeta implements the pilorama.Forest interface. -func (s *Shard) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, uint64, error) { +func (s *Shard) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) (pilorama.Meta, uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetMeta", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + ), + ) + defer span.End() + if s.pilorama == nil { return pilorama.Meta{}, 0, ErrPiloramaDisabled } @@ -92,11 +155,21 @@ func (s *Shard) TreeGetMeta(cid cidSDK.ID, treeID string, nodeID pilorama.Node) if s.info.Mode.NoMetabase() { return pilorama.Meta{}, 0, ErrDegradedMode } - return s.pilorama.TreeGetMeta(cid, treeID, nodeID) + return s.pilorama.TreeGetMeta(ctx, cid, treeID, nodeID) } // TreeGetChildren implements the pilorama.Forest interface. -func (s *Shard) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { +func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetChildren", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + ), + ) + defer span.End() + if s.pilorama == nil { return nil, ErrPiloramaDisabled } @@ -107,11 +180,21 @@ func (s *Shard) TreeGetChildren(cid cidSDK.ID, treeID string, nodeID pilorama.No if s.info.Mode.NoMetabase() { return nil, ErrDegradedMode } - return s.pilorama.TreeGetChildren(cid, treeID, nodeID) + return s.pilorama.TreeGetChildren(ctx, cid, treeID, nodeID) } // TreeGetOpLog implements the pilorama.Forest interface. -func (s *Shard) TreeGetOpLog(cid cidSDK.ID, treeID string, height uint64) (pilorama.Move, error) { +func (s *Shard) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (pilorama.Move, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetOpLog", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("height", fmt.Sprintf("%d", height)), + ), + ) + defer span.End() + if s.pilorama == nil { return pilorama.Move{}, ErrPiloramaDisabled } @@ -122,11 +205,20 @@ func (s *Shard) TreeGetOpLog(cid cidSDK.ID, treeID string, height uint64) (pilor if s.info.Mode.NoMetabase() { return pilorama.Move{}, ErrDegradedMode } - return s.pilorama.TreeGetOpLog(cid, treeID, height) + return s.pilorama.TreeGetOpLog(ctx, cid, treeID, height) } // TreeDrop implements the pilorama.Forest interface. -func (s *Shard) TreeDrop(cid cidSDK.ID, treeID string) error { +func (s *Shard) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeDrop", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + if s.pilorama == nil { return ErrPiloramaDisabled } @@ -137,11 +229,19 @@ func (s *Shard) TreeDrop(cid cidSDK.ID, treeID string) error { if s.info.Mode.NoMetabase() { return ErrDegradedMode } - return s.pilorama.TreeDrop(cid, treeID) + return s.pilorama.TreeDrop(ctx, cid, treeID) } // TreeList implements the pilorama.Forest interface. -func (s *Shard) TreeList(cid cidSDK.ID) ([]string, error) { +func (s *Shard) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeList", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + ), + ) + defer span.End() + if s.pilorama == nil { return nil, ErrPiloramaDisabled } @@ -152,11 +252,20 @@ func (s *Shard) TreeList(cid cidSDK.ID) ([]string, error) { if s.info.Mode.NoMetabase() { return nil, ErrDegradedMode } - return s.pilorama.TreeList(cid) + return s.pilorama.TreeList(ctx, cid) } // TreeExists implements the pilorama.Forest interface. -func (s *Shard) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { +func (s *Shard) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeExists", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + if s.pilorama == nil { return false, ErrPiloramaDisabled } @@ -167,11 +276,21 @@ func (s *Shard) TreeExists(cid cidSDK.ID, treeID string) (bool, error) { if s.info.Mode.NoMetabase() { return false, ErrDegradedMode } - return s.pilorama.TreeExists(cid, treeID) + return s.pilorama.TreeExists(ctx, cid, treeID) } // TreeUpdateLastSyncHeight implements the pilorama.Forest interface. -func (s *Shard) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height uint64) error { +func (s *Shard) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeUpdateLastSyncHeight", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("height", fmt.Sprintf("%d", height)), + ), + ) + defer span.End() + if s.pilorama == nil { return ErrPiloramaDisabled } @@ -185,11 +304,20 @@ func (s *Shard) TreeUpdateLastSyncHeight(cid cidSDK.ID, treeID string, height ui if s.info.Mode.NoMetabase() { return ErrDegradedMode } - return s.pilorama.TreeUpdateLastSyncHeight(cid, treeID, height) + return s.pilorama.TreeUpdateLastSyncHeight(ctx, cid, treeID, height) } // TreeLastSyncHeight implements the pilorama.Forest interface. -func (s *Shard) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) { +func (s *Shard) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeLastSyncHeight", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + if s.pilorama == nil { return 0, ErrPiloramaDisabled } @@ -200,5 +328,5 @@ func (s *Shard) TreeLastSyncHeight(cid cidSDK.ID, treeID string) (uint64, error) if s.info.Mode.NoMetabase() { return 0, ErrDegradedMode } - return s.pilorama.TreeLastSyncHeight(cid, treeID) + return s.pilorama.TreeLastSyncHeight(ctx, cid, treeID) } diff --git a/pkg/services/tree/drop.go b/pkg/services/tree/drop.go index c0750cbdc..a9e4e2e71 100644 --- a/pkg/services/tree/drop.go +++ b/pkg/services/tree/drop.go @@ -7,8 +7,8 @@ import ( ) // DropTree drops a tree from the database. If treeID is empty, all the trees are dropped. -func (s *Service) DropTree(_ context.Context, cid cid.ID, treeID string) error { +func (s *Service) DropTree(ctx context.Context, cid cid.ID, treeID string) error { // The only current use-case is a container removal, where all trees should be removed. // Thus there is no need to replicate the operation on other node. - return s.forest.TreeDrop(cid, treeID) + return s.forest.TreeDrop(ctx, cid, treeID) } diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index fd65ac3f0..dc4ce29aa 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -1,6 +1,7 @@ package tree import ( + "context" "errors" "testing" @@ -32,7 +33,7 @@ func TestGetSubTree(t *testing.T) { meta := []pilorama.KeyValue{ {Key: pilorama.AttributeFilename, Value: []byte(path[len(path)-1])}} - lm, err := p.TreeAddByPath(d, treeID, pilorama.AttributeFilename, path[:len(path)-1], meta) + lm, err := p.TreeAddByPath(context.Background(), d, treeID, pilorama.AttributeFilename, path[:len(path)-1], meta) require.NoError(t, err) require.Equal(t, 1, len(lm)) @@ -41,7 +42,7 @@ func TestGetSubTree(t *testing.T) { testGetSubTree := func(t *testing.T, rootID uint64, depth uint32, errIndex int) []uint64 { acc := subTreeAcc{errIndex: errIndex} - err := getSubTree(&acc, d.CID, &GetSubTreeRequest_Body{ + err := getSubTree(context.Background(), &acc, d.CID, &GetSubTreeRequest_Body{ TreeId: treeID, RootId: rootID, Depth: depth, @@ -68,7 +69,7 @@ func TestGetSubTree(t *testing.T) { // GetSubTree must return valid meta. for i := range acc.seen { b := acc.seen[i].Body - meta, node, err := p.TreeGetMeta(d.CID, treeID, b.NodeId) + meta, node, err := p.TreeGetMeta(context.Background(), d.CID, treeID, b.NodeId) require.NoError(t, err) require.Equal(t, node, b.ParentId) require.Equal(t, meta.Time, b.Timestamp) diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index 9594514f1..3de71b554 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -5,8 +5,11 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -25,6 +28,12 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo for _, n := range cntNodes { var stop bool n.IterateNetworkEndpoints(func(endpoint string) bool { + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.IterateNetworkEndpoints", + trace.WithAttributes( + attribute.String("endpoint", endpoint), + )) + defer span.End() + c, err := s.cache.get(ctx, endpoint) if err != nil { return false diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 98ed3df39..60d0eff50 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -8,10 +8,13 @@ import ( "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -38,17 +41,25 @@ const ( defaultReplicatorSendTimeout = time.Second * 5 ) -func (s *Service) localReplicationWorker() { +func (s *Service) localReplicationWorker(ctx context.Context) { for { select { case <-s.closeCh: return case op := <-s.replicateLocalCh: - err := s.forest.TreeApply(op.cid, op.treeID, &op.Move, false) + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.HandleReplicationOperation", + trace.WithAttributes( + attribute.String("tree_id", op.treeID), + attribute.String("container_id", op.cid.EncodeToString()), + ), + ) + + err := s.forest.TreeApply(ctx, op.cid, op.treeID, &op.Move, false) if err != nil { s.log.Error(logs.TreeFailedToApplyReplicatedOperation, zap.String("err", err.Error())) } + span.End() } } } @@ -59,10 +70,24 @@ func (s *Service) replicationWorker(ctx context.Context) { case <-s.closeCh: return case task := <-s.replicationTasks: + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.HandleReplicationTask", + trace.WithAttributes( + attribute.String("public_key", hex.EncodeToString(task.n.PublicKey())), + ), + ) + var lastErr error var lastAddr string task.n.IterateNetworkEndpoints(func(addr string) bool { + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.HandleReplicationTaskOnEndpoint", + trace.WithAttributes( + attribute.String("public_key", hex.EncodeToString(task.n.PublicKey())), + attribute.String("address", addr), + ), + ) + defer span.End() + lastAddr = addr c, err := s.cache.get(ctx, addr) @@ -89,6 +114,7 @@ func (s *Service) replicationWorker(ctx context.Context) { zap.String("key", hex.EncodeToString(task.n.PublicKey()))) } } + span.End() } } } @@ -96,7 +122,7 @@ func (s *Service) replicationWorker(ctx context.Context) { func (s *Service) replicateLoop(ctx context.Context) { for i := 0; i < s.replicatorWorkerCount; i++ { go s.replicationWorker(ctx) - go s.localReplicationWorker() + go s.localReplicationWorker(ctx) } defer func() { for len(s.replicationTasks) != 0 { diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index edea450f1..35dfda3df 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -119,7 +119,7 @@ func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error } d := pilorama.CIDDescriptor{CID: cid, Position: pos, Size: len(ns)} - log, err := s.forest.TreeMove(d, b.GetTreeId(), &pilorama.Move{ + log, err := s.forest.TreeMove(ctx, d, b.GetTreeId(), &pilorama.Move{ Parent: b.GetParentId(), Child: pilorama.RootID, Meta: pilorama.Meta{Items: protoToMeta(b.GetMeta())}, @@ -174,7 +174,7 @@ func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByP } d := pilorama.CIDDescriptor{CID: cid, Position: pos, Size: len(ns)} - logs, err := s.forest.TreeAddByPath(d, b.GetTreeId(), attr, b.GetPath(), meta) + logs, err := s.forest.TreeAddByPath(ctx, d, b.GetTreeId(), attr, b.GetPath(), meta) if err != nil { return nil, err } @@ -231,7 +231,7 @@ func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveRespon } d := pilorama.CIDDescriptor{CID: cid, Position: pos, Size: len(ns)} - log, err := s.forest.TreeMove(d, b.GetTreeId(), &pilorama.Move{ + log, err := s.forest.TreeMove(ctx, d, b.GetTreeId(), &pilorama.Move{ Parent: pilorama.TrashID, Child: b.GetNodeId(), }) @@ -280,7 +280,7 @@ func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, er } d := pilorama.CIDDescriptor{CID: cid, Position: pos, Size: len(ns)} - log, err := s.forest.TreeMove(d, b.GetTreeId(), &pilorama.Move{ + log, err := s.forest.TreeMove(ctx, d, b.GetTreeId(), &pilorama.Move{ Parent: b.GetParentId(), Child: b.GetNodeId(), Meta: pilorama.Meta{Items: protoToMeta(b.GetMeta())}, @@ -328,14 +328,14 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) attr = pilorama.AttributeFilename } - nodes, err := s.forest.TreeGetByPath(cid, b.GetTreeId(), attr, b.GetPath(), b.GetLatestOnly()) + nodes, err := s.forest.TreeGetByPath(ctx, cid, b.GetTreeId(), attr, b.GetPath(), b.GetLatestOnly()) if err != nil { return nil, err } info := make([]*GetNodeByPathResponse_Info, 0, len(nodes)) for _, node := range nodes { - m, parent, err := s.forest.TreeGetMeta(cid, b.GetTreeId(), node) + m, parent, err := s.forest.TreeGetMeta(ctx, cid, b.GetTreeId(), node) if err != nil { return nil, err } @@ -406,10 +406,10 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS return nil } - return getSubTree(srv, cid, b, s.forest) + return getSubTree(srv.Context(), srv, cid, b, s.forest) } -func getSubTree(srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { +func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { // Traverse the tree in a DFS manner. Because we need to support arbitrary depth, // recursive implementation is not suitable here, so we maintain explicit stack. stack := [][]uint64{{b.GetRootId()}} @@ -425,7 +425,7 @@ func getSubTree(srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRe nodeID := stack[len(stack)-1][0] stack[len(stack)-1] = stack[len(stack)-1][1:] - m, p, err := forest.TreeGetMeta(cid, b.GetTreeId(), nodeID) + m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), nodeID) if err != nil { return err } @@ -442,7 +442,7 @@ func getSubTree(srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRe } if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() { - children, err := forest.TreeGetChildren(cid, b.GetTreeId(), nodeID) + children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), nodeID) if err != nil { return err } @@ -455,7 +455,7 @@ func getSubTree(srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRe } // Apply locally applies operation from the remote node to the tree. -func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) { +func (s *Service) Apply(ctx context.Context, req *ApplyRequest) (*ApplyResponse, error) { err := verifyMessage(req) if err != nil { return nil, err @@ -468,7 +468,7 @@ func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, e key := req.GetSignature().GetKey() - _, pos, _, err := s.getContainerInfo(cid, key) + _, pos, _, err := s.getContainerInfo(ctx, cid, key) if err != nil { return nil, err } @@ -532,7 +532,7 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) h := b.GetHeight() for { - lm, err := s.forest.TreeGetOpLog(cid, b.GetTreeId(), h) + lm, err := s.forest.TreeGetOpLog(srv.Context(), cid, b.GetTreeId(), h) if err != nil || lm.Time == 0 { return err } @@ -587,7 +587,7 @@ func (s *Service) TreeList(ctx context.Context, req *TreeListRequest) (*TreeList return resp, outErr } - ids, err := s.forest.TreeList(cid) + ids, err := s.forest.TreeList(ctx, cid) if err != nil { return nil, err } @@ -623,7 +623,7 @@ func metaToProto(arr []pilorama.KeyValue) []*KeyValue { // getContainerInfo returns the list of container nodes, position in the container for the node // with pub key and total amount of nodes in all replicas. -func (s *Service) getContainerInfo(cid cidSDK.ID, pub []byte) ([]netmapSDK.NodeInfo, int, int, error) { +func (s *Service) getContainerInfo(ctx context.Context, cid cidSDK.ID, pub []byte) ([]netmapSDK.NodeInfo, int, int, error) { cntNodes, _, err := s.getContainerNodes(cid) if err != nil { return nil, 0, 0, err diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 91f43900f..6c4f585a6 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -85,7 +85,7 @@ func (s *Service) synchronizeAllTrees(ctx context.Context, cid cid.ID) error { } for _, tid := range treesToSync { - h, err := s.forest.TreeLastSyncHeight(cid, tid) + h, err := s.forest.TreeLastSyncHeight(ctx, cid, tid) if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { s.log.Warn(logs.TreeCouldNotGetLastSynchronizedHeightForATree, zap.Stringer("cid", cid), @@ -94,7 +94,7 @@ func (s *Service) synchronizeAllTrees(ctx context.Context, cid cid.ID) error { } newHeight := s.synchronizeTree(ctx, cid, h, tid, nodes) if h < newHeight { - if err := s.forest.TreeUpdateLastSyncHeight(cid, tid, newHeight); err != nil { + if err := s.forest.TreeUpdateLastSyncHeight(ctx, cid, tid, newHeight); err != nil { s.log.Warn(logs.TreeCouldNotUpdateLastSynchronizedHeightForATree, zap.Stringer("cid", cid), zap.String("tree", tid)) @@ -232,7 +232,7 @@ func (s *Service) synchronizeSingle(ctx context.Context, cid cid.ID, treeID stri if err := m.Meta.FromBytes(lm.Meta); err != nil { return newHeight, err } - if err := s.forest.TreeApply(cid, treeID, m, true); err != nil { + if err := s.forest.TreeApply(ctx, cid, treeID, m, true); err != nil { return newHeight, err } if m.Time > newHeight { @@ -284,11 +284,13 @@ func (s *Service) syncLoop(ctx context.Context) { case <-ctx.Done(): return case <-s.syncChan: + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.sync") s.log.Debug(logs.TreeSyncingTrees) cnrs, err := s.cfg.cnrSource.List() if err != nil { s.log.Error(logs.TreeCouldNotFetchContainers, zap.Error(err)) + span.End() continue } @@ -299,11 +301,15 @@ func (s *Service) syncLoop(ctx context.Context) { s.removeContainers(ctx, newMap) s.log.Debug(logs.TreeTreesHaveBeenSynchronized) + span.End() } } } func (s *Service) syncContainers(ctx context.Context, cnrs []cid.ID) { + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.syncContainers") + defer span.End() + // sync new containers var wg sync.WaitGroup for _, cnr := range cnrs { @@ -335,6 +341,9 @@ func (s *Service) syncContainers(ctx context.Context, cnrs []cid.ID) { } func (s *Service) removeContainers(ctx context.Context, newContainers map[cid.ID]struct{}) { + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.removeContainers") + defer span.End() + s.cnrMapMtx.Lock() defer s.cnrMapMtx.Unlock() From 560f73ab7ea83086f964505957967e1472ccfd1d Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 13 Apr 2023 20:06:34 +0300 Subject: [PATCH 0113/1943] [#247] node, ir: Drop reputation related code Signed-off-by: Pavel Karpy --- cmd/frostfs-ir/defaults.go | 1 - cmd/frostfs-node/config.go | 27 -- cmd/frostfs-node/container.go | 2 +- cmd/frostfs-node/main.go | 1 - cmd/frostfs-node/morph.go | 3 - cmd/frostfs-node/object.go | 192 +-------- cmd/frostfs-node/reputation.go | 386 ------------------ cmd/frostfs-node/reputation/common/remote.go | 101 ----- cmd/frostfs-node/reputation/common/util.go | 53 --- .../reputation/intermediate/calculator.go | 57 --- .../reputation/intermediate/consumers.go | 66 --- .../reputation/intermediate/contract.go | 147 ------- .../reputation/intermediate/daughters.go | 51 --- .../reputation/intermediate/remote.go | 125 ------ .../reputation/intermediate/storage.go | 64 --- .../reputation/internal/client/client.go | 101 ----- .../reputation/internal/client/doc.go | 11 - cmd/frostfs-node/reputation/local/remote.go | 113 ----- cmd/frostfs-node/reputation/local/storage.go | 108 ----- cmd/frostfs-node/reputation/ticker/fixed.go | 90 ---- .../reputation/ticker/fixed_test.go | 118 ------ cmd/frostfs-node/timers.go | 43 -- pkg/innerring/initialization.go | 37 -- .../processors/reputation/handlers.go | 29 -- .../processors/reputation/process_put.go | 99 ----- .../processors/reputation/processor.go | 156 ------- pkg/innerring/rpc.go | 2 +- pkg/network/cache/client.go | 2 +- .../transport/reputation/grpc/service.go | 50 --- pkg/services/object/head/remote.go | 2 +- pkg/services/reputation/common/deps.go | 78 ---- pkg/services/reputation/common/managers.go | 133 ------ .../reputation/common/router/calls.go | 139 ------- pkg/services/reputation/common/router/deps.go | 28 -- pkg/services/reputation/common/router/opts.go | 28 -- .../reputation/common/router/router.go | 81 ---- pkg/services/reputation/common/router/util.go | 40 -- .../eigentrust/calculator/calculator.go | 90 ---- .../reputation/eigentrust/calculator/calls.go | 295 ------------- .../reputation/eigentrust/calculator/deps.go | 74 ---- .../reputation/eigentrust/calculator/opts.go | 30 -- .../reputation/eigentrust/controller/calls.go | 73 ---- .../eigentrust/controller/controller.go | 86 ---- .../reputation/eigentrust/controller/deps.go | 37 -- .../reputation/eigentrust/controller/opts.go | 30 -- .../reputation/eigentrust/iteration.go | 44 -- .../reputation/eigentrust/routes/builder.go | 59 --- .../reputation/eigentrust/routes/calls.go | 33 -- .../eigentrust/storage/consumers/calls.go | 201 --------- .../eigentrust/storage/consumers/storage.go | 40 -- .../eigentrust/storage/daughters/calls.go | 177 -------- .../eigentrust/storage/daughters/storage.go | 38 -- .../reputation/local/controller/calls.go | 193 --------- .../reputation/local/controller/controller.go | 84 ---- .../reputation/local/controller/deps.go | 34 -- .../reputation/local/controller/opts.go | 30 -- .../reputation/local/controller/util.go | 32 -- .../reputation/local/routes/builder.go | 59 --- pkg/services/reputation/local/routes/calls.go | 33 -- .../reputation/local/storage/calls.go | 175 -------- .../reputation/local/storage/storage.go | 41 -- pkg/services/reputation/rpc/response.go | 50 --- pkg/services/reputation/rpc/server.go | 13 - pkg/services/reputation/rpc/sign.go | 54 --- pkg/services/reputation/trust.go | 102 ----- 65 files changed, 15 insertions(+), 4956 deletions(-) delete mode 100644 cmd/frostfs-node/reputation.go delete mode 100644 cmd/frostfs-node/reputation/common/remote.go delete mode 100644 cmd/frostfs-node/reputation/common/util.go delete mode 100644 cmd/frostfs-node/reputation/intermediate/calculator.go delete mode 100644 cmd/frostfs-node/reputation/intermediate/consumers.go delete mode 100644 cmd/frostfs-node/reputation/intermediate/contract.go delete mode 100644 cmd/frostfs-node/reputation/intermediate/daughters.go delete mode 100644 cmd/frostfs-node/reputation/intermediate/remote.go delete mode 100644 cmd/frostfs-node/reputation/intermediate/storage.go delete mode 100644 cmd/frostfs-node/reputation/internal/client/client.go delete mode 100644 cmd/frostfs-node/reputation/internal/client/doc.go delete mode 100644 cmd/frostfs-node/reputation/local/remote.go delete mode 100644 cmd/frostfs-node/reputation/local/storage.go delete mode 100644 cmd/frostfs-node/reputation/ticker/fixed.go delete mode 100644 cmd/frostfs-node/reputation/ticker/fixed_test.go delete mode 100644 cmd/frostfs-node/timers.go delete mode 100644 pkg/innerring/processors/reputation/handlers.go delete mode 100644 pkg/innerring/processors/reputation/process_put.go delete mode 100644 pkg/innerring/processors/reputation/processor.go delete mode 100644 pkg/network/transport/reputation/grpc/service.go delete mode 100644 pkg/services/reputation/common/deps.go delete mode 100644 pkg/services/reputation/common/managers.go delete mode 100644 pkg/services/reputation/common/router/calls.go delete mode 100644 pkg/services/reputation/common/router/deps.go delete mode 100644 pkg/services/reputation/common/router/opts.go delete mode 100644 pkg/services/reputation/common/router/router.go delete mode 100644 pkg/services/reputation/common/router/util.go delete mode 100644 pkg/services/reputation/eigentrust/calculator/calculator.go delete mode 100644 pkg/services/reputation/eigentrust/calculator/calls.go delete mode 100644 pkg/services/reputation/eigentrust/calculator/deps.go delete mode 100644 pkg/services/reputation/eigentrust/calculator/opts.go delete mode 100644 pkg/services/reputation/eigentrust/controller/calls.go delete mode 100644 pkg/services/reputation/eigentrust/controller/controller.go delete mode 100644 pkg/services/reputation/eigentrust/controller/deps.go delete mode 100644 pkg/services/reputation/eigentrust/controller/opts.go delete mode 100644 pkg/services/reputation/eigentrust/iteration.go delete mode 100644 pkg/services/reputation/eigentrust/routes/builder.go delete mode 100644 pkg/services/reputation/eigentrust/routes/calls.go delete mode 100644 pkg/services/reputation/eigentrust/storage/consumers/calls.go delete mode 100644 pkg/services/reputation/eigentrust/storage/consumers/storage.go delete mode 100644 pkg/services/reputation/eigentrust/storage/daughters/calls.go delete mode 100644 pkg/services/reputation/eigentrust/storage/daughters/storage.go delete mode 100644 pkg/services/reputation/local/controller/calls.go delete mode 100644 pkg/services/reputation/local/controller/controller.go delete mode 100644 pkg/services/reputation/local/controller/deps.go delete mode 100644 pkg/services/reputation/local/controller/opts.go delete mode 100644 pkg/services/reputation/local/controller/util.go delete mode 100644 pkg/services/reputation/local/routes/builder.go delete mode 100644 pkg/services/reputation/local/routes/calls.go delete mode 100644 pkg/services/reputation/local/storage/calls.go delete mode 100644 pkg/services/reputation/local/storage/storage.go delete mode 100644 pkg/services/reputation/rpc/response.go delete mode 100644 pkg/services/reputation/rpc/server.go delete mode 100644 pkg/services/reputation/rpc/sign.go delete mode 100644 pkg/services/reputation/trust.go diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index bd374fcb5..57959c1cf 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -142,7 +142,6 @@ func setWorkersDefaults(cfg *viper.Viper) { cfg.SetDefault("workers.frostfs", "10") cfg.SetDefault("workers.container", "10") cfg.SetDefault("workers.alphabet", "10") - cfg.SetDefault("workers.reputation", "10") cfg.SetDefault("workers.subnet", "10") } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d110665f5..987d27fcd 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -55,8 +55,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/tombstone" tsourse "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/tombstone/source" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" - trustcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/local/controller" - truststorage "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/local/storage" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -414,7 +412,6 @@ type cfg struct { cfgNodeInfo cfgNodeInfo cfgNetmap cfgNetmap cfgControlService cfgControlService - cfgReputation cfgReputation cfgObject cfgObject cfgNotifications cfgNotifications } @@ -452,8 +449,6 @@ type cfgMorph struct { // TTL of Sidechain cached values. Non-positive value disables caching. cacheTTL time.Duration - eigenTrustTicker *eigenTrustTickers // timers for EigenTrust iterations - proxyScriptHash neogoutil.Uint160 } @@ -532,16 +527,6 @@ type cfgControlService struct { server *grpc.Server } -type cfgReputation struct { - workerPool util.WorkerPool // pool for EigenTrust algorithm's iterations - - localTrustStorage *truststorage.Storage - - localTrustCtrl *trustcontroller.Controller - - scriptHash neogoutil.Uint160 -} - var persistateSideChainLastBlockKey = []byte("side_chain_last_processed_block") func initCfg(appCfg *config.Config) *cfg { @@ -582,8 +567,6 @@ func initCfg(appCfg *config.Config) *cfg { } c.cfgObject = initCfgObject(appCfg) - c.cfgReputation = initReputation(appCfg) - user.IDFromKey(&c.ownerIDFromKey, key.PrivateKey.PublicKey) c.metricsCollector = metrics.NewNodeMetrics() @@ -662,16 +645,6 @@ func initContainer(appCfg *config.Config) cfgContainer { } } -func initReputation(appCfg *config.Config) cfgReputation { - reputationWorkerPool, err := ants.NewPool(notificationHandlerPoolSize) - fatalOnErr(err) - - return cfgReputation{ - scriptHash: contractsconfig.Reputation(appCfg), - workerPool: reputationWorkerPool, - } -} - func initCfgGRPC() cfgGRPC { maxChunkSize := uint64(maxMsgSize) * 3 / 4 // 25% to meta, 75% to payload maxAddrAmount := uint64(maxChunkSize) / addressSize // each address is about 72 bytes diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index d5d8601e3..633d4b261 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -330,7 +330,7 @@ type remoteLoadAnnounceProvider struct { netmapKeys netmapCore.AnnouncedKeys clientCache interface { - Get(client.NodeInfo) (client.Client, error) + Get(client.NodeInfo) (client.MultiAddressClient, error) } deadEndProvider loadcontroller.WriterProvider diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 786843b0b..a6ee52464 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -102,7 +102,6 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(c, "accounting", func(c *cfg) { initAccountingService(ctx, c) }) initAndLog(c, "container", func(c *cfg) { initContainerService(ctx, c) }) initAndLog(c, "session", initSessionService) - initAndLog(c, "reputation", func(c *cfg) { initReputationService(ctx, c) }) initAndLog(c, "notification", func(c *cfg) { initNotifications(ctx, c) }) initAndLog(c, "object", initObjectService) initAndLog(c, "tree", initTreeService) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 72378d8f3..2e086f994 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -235,8 +235,6 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { zap.String("chain", "side"), zap.Uint32("block_index", block.Index)) } - - tickBlockTimers(c) }) } @@ -285,7 +283,6 @@ func lookupScriptHashesInNNS(c *cfg) { {&c.cfgNetmap.scriptHash, client.NNSNetmapContractName}, {&c.cfgAccounting.scriptHash, client.NNSBalanceContractName}, {&c.cfgContainer.scriptHash, client.NNSContainerContractName}, - {&c.cfgReputation.scriptHash, client.NNSReputationContractName}, {&c.cfgMorph.proxyScriptHash, client.NNSProxyContractName}, } ) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 08a202df9..83025a44c 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "context" "errors" "fmt" @@ -12,7 +11,6 @@ import ( policerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/policer" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -20,6 +18,7 @@ import ( morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/cache" objectTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/object/grpc" objectService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl" @@ -37,15 +36,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" - truststorage "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/local/storage" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "go.uber.org/zap" ) @@ -153,39 +147,12 @@ func (f *innerRingFetcherWithoutNotary) InnerRingKeys() ([][]byte, error) { return result, nil } -type coreClientConstructor reputationClientConstructor - -func (x *coreClientConstructor) Get(info coreclient.NodeInfo) (coreclient.MultiAddressClient, error) { - c, err := (*reputationClientConstructor)(x).Get(info) - if err != nil { - return nil, err - } - - return c.(coreclient.MultiAddressClient), nil -} - func initObjectService(c *cfg) { keyStorage := util.NewKeyStorage(&c.key.PrivateKey, c.privateTokenStore, c.cfgNetmap.state) - clientConstructor := &reputationClientConstructor{ - log: c.log, - nmSrc: c.netMapSource, - netState: c.cfgNetmap.state, - trustStorage: c.cfgReputation.localTrustStorage, - basicConstructor: c.bgClientCache, - } + c.replicator = createReplicator(c, keyStorage, c.bgClientCache) - coreConstructor := &coreClientConstructor{ - log: c.log, - nmSrc: c.netMapSource, - netState: c.cfgNetmap.state, - trustStorage: c.cfgReputation.localTrustStorage, - basicConstructor: c.clientCache, - } - - c.replicator = createReplicator(c, keyStorage, clientConstructor) - - addPolicer(c, keyStorage, clientConstructor) + addPolicer(c, keyStorage, c.bgClientCache) traverseGen := util.NewTraverserGenerator(c.netMapSource, c.cfgObject.cnrSource, c) @@ -193,11 +160,11 @@ func initObjectService(c *cfg) { sPutV2 := createPutSvcV2(sPut, keyStorage) - sSearch := createSearchSvc(c, keyStorage, traverseGen, coreConstructor) + sSearch := createSearchSvc(c, keyStorage, traverseGen, c.clientCache) sSearchV2 := createSearchSvcV2(sSearch, keyStorage) - sGet := createGetService(c, keyStorage, traverseGen, coreConstructor) + sGet := createGetService(c, keyStorage, traverseGen, c.clientCache) *c.cfgObject.getSvc = *sGet // need smth better @@ -236,7 +203,7 @@ func initObjectService(c *cfg) { } } -func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *reputationClientConstructor) { +func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.ClientCache) { ls := c.cfgObject.cfgLocalStorage.localStorage pol := policer.New( @@ -288,7 +255,7 @@ func createInnerRingFetcher(c *cfg) v2.InnerRingFetcher { } } -func createReplicator(c *cfg, keyStorage *util.KeyStorage, clientConstructor *reputationClientConstructor) *replicator.Replicator { +func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCache) *replicator.Replicator { ls := c.cfgObject.cfgLocalStorage.localStorage return replicator.New( @@ -298,7 +265,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, clientConstructor *re ), replicator.WithLocalStorage(ls), replicator.WithRemoteSender( - putsvc.NewRemoteSender(keyStorage, (*coreClientConstructor)(clientConstructor)), + putsvc.NewRemoteSender(keyStorage, cache), ), ) } @@ -319,17 +286,9 @@ func createPutSvc(c *cfg, keyStorage *util.KeyStorage) *putsvc.Service { } } - putConstructor := &coreClientConstructor{ - log: c.log, - nmSrc: c.netMapSource, - netState: c.cfgNetmap.state, - trustStorage: c.cfgReputation.localTrustStorage, - basicConstructor: c.putClientCache, - } - return putsvc.NewService( putsvc.WithKeyStorage(keyStorage), - putsvc.WithClientConstructor(putConstructor), + putsvc.WithClientConstructor(c.putClientCache), putsvc.WithMaxSizeSource(newCachedMaxObjectSizeSource(c)), putsvc.WithObjectStorage(os), putsvc.WithContainerSource(c.cfgObject.cnrSource), @@ -348,7 +307,7 @@ func createPutSvcV2(sPut *putsvc.Service, keyStorage *util.KeyStorage) *putsvcV2 ) } -func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *coreClientConstructor) *searchsvc.Service { +func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache) *searchsvc.Service { ls := c.cfgObject.cfgLocalStorage.localStorage return searchsvc.New( @@ -373,7 +332,7 @@ func createSearchSvcV2(sSearch *searchsvc.Service, keyStorage *util.KeyStorage) } func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, - coreConstructor *coreClientConstructor) *getsvc.Service { + coreConstructor *cache.ClientCache) *getsvc.Service { ls := c.cfgObject.cfgLocalStorage.localStorage return getsvc.New( @@ -480,135 +439,6 @@ func (s *morphEACLFetcher) GetEACL(cnr cid.ID) (*containercore.EACL, error) { return eaclInfo, nil } -type reputationClientConstructor struct { - log *logger.Logger - - nmSrc netmap.Source - - netState netmap.State - - trustStorage *truststorage.Storage - - basicConstructor interface { - Get(coreclient.NodeInfo) (coreclient.Client, error) - } -} - -type reputationClient struct { - coreclient.MultiAddressClient - - prm truststorage.UpdatePrm - - cons *reputationClientConstructor -} - -func (c *reputationClient) submitResult(err error) { - currEpoch := c.cons.netState.CurrentEpoch() - sat := err == nil - - c.cons.log.Debug( - "writing local reputation values", - zap.Uint64("epoch", currEpoch), - zap.Bool("satisfactory", sat), - ) - - prm := c.prm - prm.SetSatisfactory(sat) - prm.SetEpoch(currEpoch) - - c.cons.trustStorage.Update(prm) -} - -func (c *reputationClient) ObjectPutInit(ctx context.Context, prm client.PrmObjectPutInit) (*client.ObjectWriter, error) { - res, err := c.MultiAddressClient.ObjectPutInit(ctx, prm) - - // FIXME: (neofs-node#1193) here we submit only initialization errors, writing errors are not processed - c.submitResult(err) - - return res, err -} - -func (c *reputationClient) ObjectDelete(ctx context.Context, prm client.PrmObjectDelete) (*client.ResObjectDelete, error) { - res, err := c.MultiAddressClient.ObjectDelete(ctx, prm) - if err != nil { - c.submitResult(err) - } else { - c.submitResult(apistatus.ErrFromStatus(res.Status())) - } - - return res, err -} - -func (c *reputationClient) GetObjectInit(ctx context.Context, prm client.PrmObjectGet) (*client.ObjectReader, error) { - res, err := c.MultiAddressClient.ObjectGetInit(ctx, prm) - - // FIXME: (neofs-node#1193) here we submit only initialization errors, reading errors are not processed - c.submitResult(err) - - return res, err -} - -func (c *reputationClient) ObjectHead(ctx context.Context, prm client.PrmObjectHead) (*client.ResObjectHead, error) { - res, err := c.MultiAddressClient.ObjectHead(ctx, prm) - - c.submitResult(err) - - return res, err -} - -func (c *reputationClient) ObjectHash(ctx context.Context, prm client.PrmObjectHash) (*client.ResObjectHash, error) { - res, err := c.MultiAddressClient.ObjectHash(ctx, prm) - - c.submitResult(err) - - return res, err -} - -func (c *reputationClient) ObjectSearchInit(ctx context.Context, prm client.PrmObjectSearch) (*client.ObjectListReader, error) { - res, err := c.MultiAddressClient.ObjectSearchInit(ctx, prm) - - // FIXME: (neofs-node#1193) here we submit only initialization errors, reading errors are not processed - c.submitResult(err) - - return res, err -} - -func (c *reputationClientConstructor) Get(info coreclient.NodeInfo) (coreclient.Client, error) { - cl, err := c.basicConstructor.Get(info) - if err != nil { - return nil, err - } - - nm, err := netmap.GetLatestNetworkMap(c.nmSrc) - if err == nil { - key := info.PublicKey() - - nmNodes := nm.Nodes() - var peer apireputation.PeerID - - for i := range nmNodes { - if bytes.Equal(nmNodes[i].PublicKey(), key) { - peer.SetPublicKey(nmNodes[i].PublicKey()) - - prm := truststorage.UpdatePrm{} - prm.SetPeer(peer) - - return &reputationClient{ - MultiAddressClient: cl.(coreclient.MultiAddressClient), - prm: prm, - cons: c, - }, nil - } - } - } else { - c.log.Warn(logs.FrostFSNodeCouldNotGetLatestNetworkMapToOverloadTheClient, - zap.String("error", err.Error()), - ) - } - - return cl, nil -} - type engineWithNotifications struct { base putsvc.ObjectStorage nw notificationWriter diff --git a/cmd/frostfs-node/reputation.go b/cmd/frostfs-node/reputation.go deleted file mode 100644 index b3acf7eb0..000000000 --- a/cmd/frostfs-node/reputation.go +++ /dev/null @@ -1,386 +0,0 @@ -package main - -import ( - "context" - "fmt" - - v2reputation "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/reputation" - v2reputationgrpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/reputation/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/common" - intermediatereputation "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/intermediate" - localreputation "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/local" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/ticker" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - repClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" - grpcreputation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/reputation/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - reputationrouter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common/router" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" - eigentrustcalc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/calculator" - eigentrustctrl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/controller" - intermediateroutes "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/routes" - consumerstorage "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/storage/consumers" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/storage/daughters" - localtrustcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/local/controller" - localroutes "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/local/routes" - truststorage "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/local/storage" - reputationrpc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/rpc" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" - "go.uber.org/zap" -) - -func initReputationService(ctx context.Context, c *cfg) { - wrap, err := repClient.NewFromMorph(c.cfgMorph.client, c.cfgReputation.scriptHash, 0, repClient.TryNotary()) - fatalOnErr(err) - - localKey := c.key.PublicKey().Bytes() - - nmSrc := c.netMapSource - - // storing calculated trusts as a daughter - c.cfgReputation.localTrustStorage = truststorage.New( - truststorage.Prm{}, - ) - - daughterStorage := daughters.New(daughters.Prm{}) - consumerStorage := consumerstorage.New(consumerstorage.Prm{}) - - localTrustLogger := &logger.Logger{Logger: c.log.With(zap.String("trust_type", "local"))} - - managerBuilder := reputationcommon.NewManagerBuilder( - reputationcommon.ManagersPrm{ - NetMapSource: nmSrc, - }, - reputationcommon.WithLogger(c.log), - ) - - localRouteBuilder := localroutes.New( - localroutes.Prm{ - ManagerBuilder: managerBuilder, - Log: localTrustLogger, - }, - ) - - localTrustRouter := createLocalTrustRouter(c, localRouteBuilder, localTrustLogger, daughterStorage) - - intermediateTrustRouter := createIntermediateTrustRouter(c, consumerStorage, managerBuilder) - - eigenTrustController := createEigenTrustController(c, intermediateTrustRouter, localKey, wrap, daughterStorage, consumerStorage) - - c.cfgReputation.localTrustCtrl = createLocalTrustController(c, localTrustLogger, localKey, localTrustRouter) - - addReputationReportHandler(ctx, c) - - server := grpcreputation.New( - reputationrpc.NewSignService( - &c.key.PrivateKey, - reputationrpc.NewResponseService( - &reputationServer{ - cfg: c, - log: c.log, - localRouter: localTrustRouter, - intermediateRouter: intermediateTrustRouter, - routeBuilder: localRouteBuilder, - }, - c.respSvc, - ), - ), - ) - - for _, srv := range c.cfgGRPC.servers { - v2reputationgrpc.RegisterReputationServiceServer(srv, server) - } - - // initialize eigen trust block timer - newEigenTrustIterTimer(c) - - addEigenTrustEpochHandler(ctx, c, eigenTrustController) -} - -func addReputationReportHandler(ctx context.Context, c *cfg) { - addNewEpochAsyncNotificationHandler( - c, - func(ev event.Event) { - c.log.Debug(logs.FrostFSNodeStartReportingReputationOnNewEpochEvent) - - var reportPrm localtrustcontroller.ReportPrm - - // report collected values from previous epoch - reportPrm.SetEpoch(ev.(netmap.NewEpoch).EpochNumber() - 1) - - c.cfgReputation.localTrustCtrl.Report(ctx, reportPrm) - }, - ) -} - -func addEigenTrustEpochHandler(ctx context.Context, c *cfg, eigenTrustController *eigentrustctrl.Controller) { - addNewEpochAsyncNotificationHandler( - c, - func(e event.Event) { - epoch := e.(netmap.NewEpoch).EpochNumber() - - log := c.log.With(zap.Uint64("epoch", epoch)) - - duration, err := c.cfgNetmap.wrapper.EpochDuration() - if err != nil { - log.Debug(logs.FrostFSNodeCouldNotFetchEpochDuration, zap.Error(err)) - return - } - - iterations, err := c.cfgNetmap.wrapper.EigenTrustIterations() - if err != nil { - log.Debug(logs.FrostFSNodeCouldNotFetchIterationNumber, zap.Error(err)) - return - } - - epochTimer, err := ticker.NewIterationsTicker(duration, iterations, func() { - eigenTrustController.Continue(ctx, - eigentrustctrl.ContinuePrm{ - Epoch: epoch - 1, - }, - ) - }) - if err != nil { - log.Debug(logs.FrostFSNodeCouldNotCreateFixedEpochTimer, zap.Error(err)) - return - } - - c.cfgMorph.eigenTrustTicker.addEpochTimer(epoch, epochTimer) - }, - ) -} - -func createLocalTrustRouter(c *cfg, localRouteBuilder *localroutes.Builder, localTrustLogger *logger.Logger, daughterStorage *daughters.Storage) *reputationrouter.Router { - // storing received daughter(of current node) trusts as a manager - daughterStorageWriterProvider := &intermediatereputation.DaughterStorageWriterProvider{ - Log: c.log, - Storage: daughterStorage, - } - - remoteLocalTrustProvider := common.NewRemoteTrustProvider( - common.RemoteProviderPrm{ - NetmapKeys: c, - DeadEndProvider: daughterStorageWriterProvider, - ClientCache: c.bgClientCache, - WriterProvider: localreputation.NewRemoteProvider( - localreputation.RemoteProviderPrm{ - Key: &c.key.PrivateKey, - Log: localTrustLogger, - }, - ), - Log: localTrustLogger, - }, - ) - - localTrustRouter := reputationrouter.New( - reputationrouter.Prm{ - LocalServerInfo: c, - RemoteWriterProvider: remoteLocalTrustProvider, - Builder: localRouteBuilder, - }, - reputationrouter.WithLogger(localTrustLogger)) - return localTrustRouter -} - -func createIntermediateTrustRouter(c *cfg, consumerStorage *consumerstorage.Storage, managerBuilder reputationcommon.ManagerBuilder) *reputationrouter.Router { - intermediateTrustLogger := &logger.Logger{Logger: c.log.With(zap.String("trust_type", "intermediate"))} - - consumerStorageWriterProvider := &intermediatereputation.ConsumerStorageWriterProvider{ - Log: c.log, - Storage: consumerStorage, - } - - remoteIntermediateTrustProvider := common.NewRemoteTrustProvider( - common.RemoteProviderPrm{ - NetmapKeys: c, - DeadEndProvider: consumerStorageWriterProvider, - ClientCache: c.bgClientCache, - WriterProvider: intermediatereputation.NewRemoteProvider( - intermediatereputation.RemoteProviderPrm{ - Key: &c.key.PrivateKey, - Log: intermediateTrustLogger, - }, - ), - Log: intermediateTrustLogger, - }, - ) - - intermediateRouteBuilder := intermediateroutes.New( - intermediateroutes.Prm{ - ManagerBuilder: managerBuilder, - Log: intermediateTrustLogger, - }, - ) - - intermediateTrustRouter := reputationrouter.New( - reputationrouter.Prm{ - LocalServerInfo: c, - RemoteWriterProvider: remoteIntermediateTrustProvider, - Builder: intermediateRouteBuilder, - }, - reputationrouter.WithLogger(intermediateTrustLogger), - ) - return intermediateTrustRouter -} - -func createEigenTrustController(c *cfg, intermediateTrustRouter *reputationrouter.Router, localKey []byte, wrap *repClient.Client, - daughterStorage *daughters.Storage, consumerStorage *consumerstorage.Storage) *eigentrustctrl.Controller { - eigenTrustCalculator := eigentrustcalc.New( - eigentrustcalc.Prm{ - AlphaProvider: c.cfgNetmap.wrapper, - InitialTrustSource: intermediatereputation.InitialTrustSource{ - NetMap: c.netMapSource, - }, - IntermediateValueTarget: intermediateTrustRouter, - WorkerPool: c.cfgReputation.workerPool, - FinalResultTarget: intermediatereputation.NewFinalWriterProvider( - intermediatereputation.FinalWriterProviderPrm{ - PrivatKey: &c.key.PrivateKey, - PubKey: localKey, - Client: wrap, - }, - intermediatereputation.FinalWriterWithLogger(c.log), - ), - DaughterTrustSource: &intermediatereputation.DaughterTrustIteratorProvider{ - DaughterStorage: daughterStorage, - ConsumerStorage: consumerStorage, - }, - }, - eigentrustcalc.WithLogger(c.log), - ) - - eigenTrustController := eigentrustctrl.New( - eigentrustctrl.Prm{ - DaughtersTrustCalculator: &intermediatereputation.DaughtersTrustCalculator{ - Calculator: eigenTrustCalculator, - }, - IterationsProvider: c.cfgNetmap.wrapper, - WorkerPool: c.cfgReputation.workerPool, - }, - eigentrustctrl.WithLogger(c.log), - ) - return eigenTrustController -} - -func createLocalTrustController(c *cfg, localTrustLogger *logger.Logger, localKey []byte, localTrustRouter *reputationrouter.Router) *localtrustcontroller.Controller { - localTrustStorage := &localreputation.TrustStorage{ - Log: localTrustLogger, - Storage: c.cfgReputation.localTrustStorage, - NmSrc: c.netMapSource, - LocalKey: localKey, - } - - return localtrustcontroller.New( - localtrustcontroller.Prm{ - LocalTrustSource: localTrustStorage, - LocalTrustTarget: localTrustRouter, - }, - localtrustcontroller.WithLogger(c.log), - ) -} - -type reputationServer struct { - *cfg - log *logger.Logger - localRouter *reputationrouter.Router - intermediateRouter *reputationrouter.Router - routeBuilder reputationrouter.Builder -} - -func (s *reputationServer) AnnounceLocalTrust(ctx context.Context, req *v2reputation.AnnounceLocalTrustRequest) (*v2reputation.AnnounceLocalTrustResponse, error) { - passedRoute := reverseRoute(req.GetVerificationHeader()) - passedRoute = append(passedRoute, s) - - body := req.GetBody() - - ep := &common.EpochProvider{ - E: body.GetEpoch(), - } - - w, err := s.localRouter.InitWriter(reputationrouter.NewRouteInfo(ep, passedRoute)) - if err != nil { - return nil, fmt.Errorf("could not initialize local trust writer: %w", err) - } - - for _, trust := range body.GetTrusts() { - err = s.processLocalTrust(ctx, body.GetEpoch(), apiToLocalTrust(&trust, passedRoute[0].PublicKey()), passedRoute, w) - if err != nil { - return nil, fmt.Errorf("could not write one of local trusts: %w", err) - } - } - - resp := new(v2reputation.AnnounceLocalTrustResponse) - resp.SetBody(new(v2reputation.AnnounceLocalTrustResponseBody)) - - return resp, nil -} - -func (s *reputationServer) AnnounceIntermediateResult(ctx context.Context, req *v2reputation.AnnounceIntermediateResultRequest) (*v2reputation.AnnounceIntermediateResultResponse, error) { - passedRoute := reverseRoute(req.GetVerificationHeader()) - passedRoute = append(passedRoute, s) - - body := req.GetBody() - - ei := eigentrust.NewEpochIteration(body.GetEpoch(), body.GetIteration()) - - w, err := s.intermediateRouter.InitWriter(reputationrouter.NewRouteInfo(ei, passedRoute)) - if err != nil { - return nil, fmt.Errorf("could not initialize trust writer: %w", err) - } - - v2Trust := body.GetTrust() - - trust := apiToLocalTrust(v2Trust.GetTrust(), v2Trust.GetTrustingPeer().GetPublicKey()) - - err = w.Write(ctx, trust) - if err != nil { - return nil, fmt.Errorf("could not write trust: %w", err) - } - - resp := new(v2reputation.AnnounceIntermediateResultResponse) - resp.SetBody(new(v2reputation.AnnounceIntermediateResultResponseBody)) - - return resp, nil -} - -func (s *reputationServer) processLocalTrust(ctx context.Context, epoch uint64, t reputation.Trust, - passedRoute []reputationcommon.ServerInfo, w reputationcommon.Writer) error { - err := reputationrouter.CheckRoute(s.routeBuilder, epoch, t, passedRoute) - if err != nil { - return fmt.Errorf("wrong route of reputation trust value: %w", err) - } - - return w.Write(ctx, t) -} - -// apiToLocalTrust converts v2 Trust to local reputation.Trust, adding trustingPeer. -func apiToLocalTrust(t *v2reputation.Trust, trustingPeer []byte) reputation.Trust { - var trusted, trusting apireputation.PeerID - trusted.SetPublicKey(t.GetPeer().GetPublicKey()) - trusting.SetPublicKey(trustingPeer) - - localTrust := reputation.Trust{} - - localTrust.SetValue(reputation.TrustValueFromFloat64(t.GetValue())) - localTrust.SetPeer(trusted) - localTrust.SetTrustingPeer(trusting) - - return localTrust -} - -func reverseRoute(hdr *session.RequestVerificationHeader) (passedRoute []reputationcommon.ServerInfo) { - for hdr != nil { - passedRoute = append(passedRoute, &common.OnlyKeyRemoteServerInfo{ - Key: hdr.GetBodySignature().GetKey(), - }) - - hdr = hdr.GetOrigin() - } - - return -} diff --git a/cmd/frostfs-node/reputation/common/remote.go b/cmd/frostfs-node/reputation/common/remote.go deleted file mode 100644 index f1982301f..000000000 --- a/cmd/frostfs-node/reputation/common/remote.go +++ /dev/null @@ -1,101 +0,0 @@ -package common - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - trustcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/local/controller" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" -) - -type clientCache interface { - Get(client.NodeInfo) (client.Client, error) -} - -// clientKeyRemoteProvider must provide a remote writer and take into account -// that requests must be sent via the passed api client and must be signed with -// the passed private key. -type clientKeyRemoteProvider interface { - WithClient(client.Client) reputationcommon.WriterProvider -} - -// RemoteTrustProvider is an implementation of reputation RemoteWriterProvider interface. -// It caches clients, checks if it is the end of the route and checks either the current -// node is a remote target or not. -// -// remoteTrustProvider requires to be provided with clientKeyRemoteProvider. -type RemoteTrustProvider struct { - netmapKeys netmap.AnnouncedKeys - deadEndProvider reputationcommon.WriterProvider - clientCache clientCache - remoteProvider clientKeyRemoteProvider - log *logger.Logger -} - -// RemoteProviderPrm groups the required parameters of the remoteTrustProvider's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type RemoteProviderPrm struct { - NetmapKeys netmap.AnnouncedKeys - DeadEndProvider reputationcommon.WriterProvider - ClientCache clientCache - WriterProvider clientKeyRemoteProvider - Log *logger.Logger -} - -func NewRemoteTrustProvider(prm RemoteProviderPrm) *RemoteTrustProvider { - switch { - case prm.NetmapKeys == nil: - PanicOnPrmValue("NetmapKeys", prm.NetmapKeys) - case prm.DeadEndProvider == nil: - PanicOnPrmValue("DeadEndProvider", prm.DeadEndProvider) - case prm.ClientCache == nil: - PanicOnPrmValue("ClientCache", prm.ClientCache) - case prm.WriterProvider == nil: - PanicOnPrmValue("WriterProvider", prm.WriterProvider) - case prm.Log == nil: - PanicOnPrmValue("Logger", prm.Log) - } - - return &RemoteTrustProvider{ - netmapKeys: prm.NetmapKeys, - deadEndProvider: prm.DeadEndProvider, - clientCache: prm.ClientCache, - remoteProvider: prm.WriterProvider, - log: prm.Log, - } -} - -func (rtp *RemoteTrustProvider) InitRemote(srv reputationcommon.ServerInfo) (reputationcommon.WriterProvider, error) { - rtp.log.Debug(logs.CommonInitializingRemoteWriterProvider) - - if srv == nil { - rtp.log.Debug(logs.CommonRouteHasReachedDeadendProvider) - return rtp.deadEndProvider, nil - } - - if rtp.netmapKeys.IsLocalKey(srv.PublicKey()) { - // if local => return no-op writer - rtp.log.Debug(logs.CommonInitializingNoopWriterProvider) - return trustcontroller.SimpleWriterProvider(new(NopReputationWriter)), nil - } - - var info client.NodeInfo - - err := client.NodeInfoFromRawNetmapElement(&info, srv) - if err != nil { - return nil, fmt.Errorf("parse client node info: %w", err) - } - - c, err := rtp.clientCache.Get(info) - if err != nil { - return nil, fmt.Errorf("could not initialize API client: %w", err) - } - - return rtp.remoteProvider.WithClient(c), nil -} diff --git a/cmd/frostfs-node/reputation/common/util.go b/cmd/frostfs-node/reputation/common/util.go deleted file mode 100644 index 443adb388..000000000 --- a/cmd/frostfs-node/reputation/common/util.go +++ /dev/null @@ -1,53 +0,0 @@ -package common - -import ( - "context" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" -) - -type EpochProvider struct { - E uint64 -} - -func (ep *EpochProvider) Epoch() uint64 { - return ep.E -} - -type NopReputationWriter struct{} - -func (NopReputationWriter) Write(context.Context, reputation.Trust) error { - return nil -} - -func (NopReputationWriter) Close(context.Context) error { - return nil -} - -// OnlyKeyRemoteServerInfo is an implementation of reputation.ServerInfo -// interface but with only public key data. -type OnlyKeyRemoteServerInfo struct { - Key []byte -} - -func (i *OnlyKeyRemoteServerInfo) PublicKey() []byte { - return i.Key -} - -func (*OnlyKeyRemoteServerInfo) IterateAddresses(func(string) bool) { -} - -func (*OnlyKeyRemoteServerInfo) NumberOfAddresses() int { - return 0 -} - -func (*OnlyKeyRemoteServerInfo) ExternalAddresses() []string { - return nil -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func PanicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} diff --git a/cmd/frostfs-node/reputation/intermediate/calculator.go b/cmd/frostfs-node/reputation/intermediate/calculator.go deleted file mode 100644 index 73dd12311..000000000 --- a/cmd/frostfs-node/reputation/intermediate/calculator.go +++ /dev/null @@ -1,57 +0,0 @@ -package intermediate - -import ( - "context" - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" - eigencalc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/calculator" - eigentrustctrl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/controller" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" -) - -// InitialTrustSource is an implementation of the -// reputation/eigentrust/calculator's InitialTrustSource interface. -type InitialTrustSource struct { - NetMap netmap.Source -} - -var ErrEmptyNetMap = errors.New("empty NepMap") - -// InitialTrust returns `initialTrust` as an initial trust value. -func (i InitialTrustSource) InitialTrust(apireputation.PeerID) (reputation.TrustValue, error) { - nm, err := i.NetMap.GetNetMap(1) - if err != nil { - return reputation.TrustZero, fmt.Errorf("failed to get NetMap: %w", err) - } - - nodeCount := reputation.TrustValueFromFloat64(float64(len(nm.Nodes()))) - if nodeCount == 0 { - return reputation.TrustZero, ErrEmptyNetMap - } - - return reputation.TrustOne.Div(nodeCount), nil -} - -// DaughtersTrustCalculator wraps EigenTrust calculator and implements the -// eigentrust/calculator's DaughtersTrustCalculator interface. -type DaughtersTrustCalculator struct { - Calculator *eigencalc.Calculator -} - -// Calculate converts and passes values to the wrapped calculator. -func (c *DaughtersTrustCalculator) Calculate(ctx context.Context, iterCtx eigentrustctrl.IterationContext) { - calcPrm := eigencalc.CalculatePrm{} - epochIteration := eigentrust.EpochIteration{} - - epochIteration.SetEpoch(iterCtx.Epoch()) - epochIteration.SetI(iterCtx.I()) - - calcPrm.SetLast(iterCtx.Last()) - calcPrm.SetEpochIteration(epochIteration) - - c.Calculator.Calculate(ctx, calcPrm) -} diff --git a/cmd/frostfs-node/reputation/intermediate/consumers.go b/cmd/frostfs-node/reputation/intermediate/consumers.go deleted file mode 100644 index 02cdb2a2b..000000000 --- a/cmd/frostfs-node/reputation/intermediate/consumers.go +++ /dev/null @@ -1,66 +0,0 @@ -package intermediate - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" - eigencalc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/calculator" - consumerstorage "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/storage/consumers" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -var ErrIncorrectContextPanicMsg = "could not write intermediate trust: passed context incorrect" - -// ConsumerStorageWriterProvider is an implementation of the reputation.WriterProvider -// interface that provides ConsumerTrustWriter writer. -type ConsumerStorageWriterProvider struct { - Log *logger.Logger - Storage *consumerstorage.Storage -} - -// ConsumerTrustWriter is an implementation of the reputation.Writer interface -// that writes passed consumer's Trust values to the Consumer storage. After writing -// that, values can be used in eigenTrust algorithm's iterations. -type ConsumerTrustWriter struct { - log *logger.Logger - storage *consumerstorage.Storage - iterInfo eigencalc.EpochIterationInfo -} - -func (w *ConsumerTrustWriter) Write(_ context.Context, t reputation.Trust) error { - w.log.Debug(logs.IntermediateWritingReceivedConsumersTrusts, - zap.Uint64("epoch", w.iterInfo.Epoch()), - zap.Uint32("iteration", w.iterInfo.I()), - zap.Stringer("trusting_peer", t.TrustingPeer()), - zap.Stringer("trusted_peer", t.Peer()), - ) - - trust := eigentrust.IterationTrust{Trust: t} - - trust.SetEpoch(w.iterInfo.Epoch()) - trust.SetI(w.iterInfo.I()) - - w.storage.Put(trust) - return nil -} - -func (w *ConsumerTrustWriter) Close(context.Context) error { - return nil -} - -func (s *ConsumerStorageWriterProvider) InitWriter(ep reputationcommon.EpochProvider) (reputationcommon.Writer, error) { - iterInfo, ok := ep.(eigencalc.EpochIterationInfo) - if !ok { - panic(ErrIncorrectContextPanicMsg) - } - - return &ConsumerTrustWriter{ - log: s.Log, - storage: s.Storage, - iterInfo: iterInfo, - }, nil -} diff --git a/cmd/frostfs-node/reputation/intermediate/contract.go b/cmd/frostfs-node/reputation/intermediate/contract.go deleted file mode 100644 index 2d83598bc..000000000 --- a/cmd/frostfs-node/reputation/intermediate/contract.go +++ /dev/null @@ -1,147 +0,0 @@ -package intermediate - -import ( - "crypto/ecdsa" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - repClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" - eigentrustcalc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/calculator" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" - "go.uber.org/zap" -) - -// FinalWriterProviderPrm groups the required parameters of the FinalWriterProvider's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type FinalWriterProviderPrm struct { - PrivatKey *ecdsa.PrivateKey - PubKey []byte - Client *repClient.Client -} - -// NewFinalWriterProvider creates a new instance of the FinalWriterProvider. -// -// Panics if at least one value of the parameters is invalid. -// -// The created FinalWriterProvider does not require additional -// initialization and is completely ready for work. -func NewFinalWriterProvider(prm FinalWriterProviderPrm, opts ...FinalWriterOption) *FinalWriterProvider { - o := defaultFinalWriterOptionsOpts() - - for i := range opts { - opts[i](o) - } - - return &FinalWriterProvider{ - prm: prm, - opts: o, - } -} - -// FinalWriterProvider is an implementation of the reputation.eigentrust.calculator -// IntermediateWriterProvider interface. It inits FinalWriter. -type FinalWriterProvider struct { - prm FinalWriterProviderPrm - opts *finalWriterOptions -} - -func (fwp FinalWriterProvider) InitIntermediateWriter( - _ eigentrustcalc.EpochIterationInfo) (eigentrustcalc.IntermediateWriter, error) { - return &FinalWriter{ - privatKey: fwp.prm.PrivatKey, - pubKey: fwp.prm.PubKey, - client: fwp.prm.Client, - l: fwp.opts.log, - }, nil -} - -// FinalWriter is an implementation of the reputation.eigentrust.calculator IntermediateWriter -// interface that writes GlobalTrust to contract directly. -type FinalWriter struct { - privatKey *ecdsa.PrivateKey - pubKey []byte - client *repClient.Client - - l *logger.Logger -} - -func (fw FinalWriter) WriteIntermediateTrust(t eigentrust.IterationTrust) error { - fw.l.Debug(logs.IntermediateStartWritingGlobalTrustsToContract) - - args := repClient.PutPrm{} - - apiTrustedPeerID := t.Peer() - - var apiTrust apireputation.Trust - apiTrust.SetValue(t.Value().Float64()) - apiTrust.SetPeer(t.Peer()) - - var managerPublicKey [33]byte - copy(managerPublicKey[:], fw.pubKey) - - var apiMangerPeerID apireputation.PeerID - apiMangerPeerID.SetPublicKey(managerPublicKey[:]) - - var gTrust apireputation.GlobalTrust - gTrust.SetTrust(apiTrust) - gTrust.SetManager(apiMangerPeerID) - - err := gTrust.Sign(frostfsecdsa.Signer(*fw.privatKey)) - if err != nil { - fw.l.Debug( - "failed to sign global trust", - zap.Error(err), - ) - return fmt.Errorf("failed to sign global trust: %w", err) - } - - args.SetEpoch(t.Epoch()) - args.SetValue(gTrust) - args.SetPeerID(apiTrustedPeerID) - - err = fw.client.Put( - args, - ) - if err != nil { - fw.l.Debug( - "failed to write global trust to contract", - zap.Error(err), - ) - return fmt.Errorf("failed to write global trust to contract: %w", err) - } - - fw.l.Debug( - "sent global trust to contract", - zap.Uint64("epoch", t.Epoch()), - zap.Float64("value", t.Value().Float64()), - zap.Stringer("peer", t.Peer()), - ) - - return nil -} - -type finalWriterOptions struct { - log *logger.Logger -} - -type FinalWriterOption func(*finalWriterOptions) - -func defaultFinalWriterOptionsOpts() *finalWriterOptions { - return &finalWriterOptions{ - log: &logger.Logger{Logger: zap.L()}, - } -} - -func FinalWriterWithLogger(l *logger.Logger) FinalWriterOption { - return func(o *finalWriterOptions) { - if l != nil { - o.log = l - } - } -} diff --git a/cmd/frostfs-node/reputation/intermediate/daughters.go b/cmd/frostfs-node/reputation/intermediate/daughters.go deleted file mode 100644 index 30237537c..000000000 --- a/cmd/frostfs-node/reputation/intermediate/daughters.go +++ /dev/null @@ -1,51 +0,0 @@ -package intermediate - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/storage/daughters" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// DaughterStorageWriterProvider is an implementation of the reputation.WriterProvider -// interface that provides DaughterTrustWriter writer. -type DaughterStorageWriterProvider struct { - Log *logger.Logger - Storage *daughters.Storage -} - -// DaughterTrustWriter is an implementation of the reputation.Writer interface -// that writes passed daughter's Trust values to Daughter storage. After writing -// that, values can be used in eigenTrust algorithm's iterations. -type DaughterTrustWriter struct { - log *logger.Logger - storage *daughters.Storage - ep reputationcommon.EpochProvider -} - -func (w *DaughterTrustWriter) Write(_ context.Context, t reputation.Trust) error { - w.log.Debug(logs.IntermediateWritingReceivedDaughtersTrusts, - zap.Uint64("epoch", w.ep.Epoch()), - zap.Stringer("trusting_peer", t.TrustingPeer()), - zap.Stringer("trusted_peer", t.Peer()), - ) - - w.storage.Put(w.ep.Epoch(), t) - return nil -} - -func (w *DaughterTrustWriter) Close(context.Context) error { - return nil -} - -func (s *DaughterStorageWriterProvider) InitWriter(ep reputationcommon.EpochProvider) (reputationcommon.Writer, error) { - return &DaughterTrustWriter{ - log: s.Log, - storage: s.Storage, - ep: ep, - }, nil -} diff --git a/cmd/frostfs-node/reputation/intermediate/remote.go b/cmd/frostfs-node/reputation/intermediate/remote.go deleted file mode 100644 index 8087463b5..000000000 --- a/cmd/frostfs-node/reputation/intermediate/remote.go +++ /dev/null @@ -1,125 +0,0 @@ -package intermediate - -import ( - "context" - "crypto/ecdsa" - - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/common" - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - eigentrustcalc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/calculator" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - reputationapi "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" - "go.uber.org/zap" -) - -// RemoteProviderPrm groups the required parameters of the RemoteProvider's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type RemoteProviderPrm struct { - Key *ecdsa.PrivateKey - Log *logger.Logger -} - -// NewRemoteProvider creates a new instance of the RemoteProvider. -// -// Panics if at least one value of the parameters is invalid. -// -// The created RemoteProvider does not require additional -// initialization and is completely ready for work. -func NewRemoteProvider(prm RemoteProviderPrm) *RemoteProvider { - switch { - case prm.Key == nil: - common.PanicOnPrmValue("NetMapSource", prm.Key) - case prm.Log == nil: - common.PanicOnPrmValue("Logger", prm.Log) - } - - return &RemoteProvider{ - key: prm.Key, - log: prm.Log, - } -} - -// RemoteProvider is an implementation of the clientKeyRemoteProvider interface. -type RemoteProvider struct { - key *ecdsa.PrivateKey - log *logger.Logger -} - -func (rp RemoteProvider) WithClient(c coreclient.Client) reputationcommon.WriterProvider { - return &TrustWriterProvider{ - client: c, - key: rp.key, - log: rp.log, - } -} - -type TrustWriterProvider struct { - client coreclient.Client - key *ecdsa.PrivateKey - log *logger.Logger -} - -func (twp *TrustWriterProvider) InitWriter(ep reputationcommon.EpochProvider) (reputationcommon.Writer, error) { - iterInfo, ok := ep.(eigentrustcalc.EpochIterationInfo) - if !ok { - // TODO: #1164 think if this can be done without such limitation - panic(ErrIncorrectContextPanicMsg) - } - - return &RemoteTrustWriter{ - iterInfo: iterInfo, - client: twp.client, - key: twp.key, - log: twp.log, - }, nil -} - -type RemoteTrustWriter struct { - iterInfo eigentrustcalc.EpochIterationInfo - client coreclient.Client - key *ecdsa.PrivateKey - log *logger.Logger -} - -// Write sends a trust value to a remote node via ReputationService.AnnounceIntermediateResult RPC. -func (rtp *RemoteTrustWriter) Write(ctx context.Context, t reputation.Trust) error { - epoch := rtp.iterInfo.Epoch() - i := rtp.iterInfo.I() - - rtp.log.Debug(logs.IntermediateAnnouncingTrust, - zap.Uint64("epoch", epoch), - zap.Uint32("iteration", i), - zap.Stringer("trusting_peer", t.TrustingPeer()), - zap.Stringer("trusted_peer", t.Peer()), - ) - - var apiTrust reputationapi.Trust - apiTrust.SetValue(t.Value().Float64()) - apiTrust.SetPeer(t.Peer()) - - var apiPeerToPeerTrust reputationapi.PeerToPeerTrust - apiPeerToPeerTrust.SetTrustingPeer(t.TrustingPeer()) - apiPeerToPeerTrust.SetTrust(apiTrust) - - var p internalclient.AnnounceIntermediatePrm - - p.SetClient(rtp.client) - p.SetEpoch(epoch) - p.SetIteration(i) - p.SetTrust(apiPeerToPeerTrust) - - _, err := internalclient.AnnounceIntermediate(ctx, p) - - return err -} - -func (rtp *RemoteTrustWriter) Close(context.Context) error { - return nil -} diff --git a/cmd/frostfs-node/reputation/intermediate/storage.go b/cmd/frostfs-node/reputation/intermediate/storage.go deleted file mode 100644 index db29ff92b..000000000 --- a/cmd/frostfs-node/reputation/intermediate/storage.go +++ /dev/null @@ -1,64 +0,0 @@ -package intermediate - -import ( - "fmt" - - eigentrustcalc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/calculator" - consumerstorage "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/storage/consumers" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/storage/daughters" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" -) - -// DaughterTrustIteratorProvider is an implementation of the -// reputation/eigentrust/calculator's DaughterTrustIteratorProvider interface. -type DaughterTrustIteratorProvider struct { - DaughterStorage *daughters.Storage - ConsumerStorage *consumerstorage.Storage -} - -// InitDaughterIterator returns an iterator over the received -// local trusts for ctx.Epoch() epoch from daughter p. -func (ip *DaughterTrustIteratorProvider) InitDaughterIterator(ctx eigentrustcalc.EpochIterationInfo, - p apireputation.PeerID) (eigentrustcalc.TrustIterator, error) { - epoch := ctx.Epoch() - - daughterIterator, ok := ip.DaughterStorage.DaughterTrusts(epoch, p) - if !ok { - return nil, fmt.Errorf("no data in %d epoch for daughter: %s", epoch, p) - } - - return daughterIterator, nil -} - -// InitAllDaughtersIterator returns an iterator over all -// daughters of the current node(manager) and all local -// trusts received from them for ctx.Epoch() epoch. -func (ip *DaughterTrustIteratorProvider) InitAllDaughtersIterator( - ctx eigentrustcalc.EpochIterationInfo) (eigentrustcalc.PeerTrustsIterator, error) { - epoch := ctx.Epoch() - - iter, ok := ip.DaughterStorage.AllDaughterTrusts(epoch) - if !ok { - return nil, fmt.Errorf("no data in %d epoch for daughters", epoch) - } - - return iter, nil -} - -// InitConsumersIterator returns an iterator over all daughters -// of the current node(manager) and all their consumers' local -// trusts for ctx.Epoch() epoch and ctx.I() iteration. -func (ip *DaughterTrustIteratorProvider) InitConsumersIterator( - ctx eigentrustcalc.EpochIterationInfo) (eigentrustcalc.PeerTrustsIterator, error) { - epoch, iter := ctx.Epoch(), ctx.I() - - consumerIterator, ok := ip.ConsumerStorage.Consumers(epoch, iter) - if !ok { - return nil, fmt.Errorf("no data for %d iteration in %d epoch for consumers's trusts", - iter, - epoch, - ) - } - - return consumerIterator, nil -} diff --git a/cmd/frostfs-node/reputation/internal/client/client.go b/cmd/frostfs-node/reputation/internal/client/client.go deleted file mode 100644 index ff5131262..000000000 --- a/cmd/frostfs-node/reputation/internal/client/client.go +++ /dev/null @@ -1,101 +0,0 @@ -package internal - -import ( - "context" - - coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" -) - -type commonPrm struct { - cli coreclient.Client -} - -// SetClient sets the base client for FrostFS API communication. -// -// Required parameter. -func (x *commonPrm) SetClient(cli coreclient.Client) { - x.cli = cli -} - -// AnnounceLocalPrm groups parameters of AnnounceLocal operation. -type AnnounceLocalPrm struct { - commonPrm - - cliPrm client.PrmAnnounceLocalTrust -} - -// SetEpoch sets the epoch in which the trust was assessed. -func (x *AnnounceLocalPrm) SetEpoch(epoch uint64) { - x.cliPrm.SetEpoch(epoch) -} - -// SetTrusts sets a list of local trust values. -func (x *AnnounceLocalPrm) SetTrusts(ts []reputation.Trust) { - x.cliPrm.SetValues(ts) -} - -// AnnounceLocalRes groups the resulting values of AnnounceLocal operation. -type AnnounceLocalRes struct{} - -// AnnounceLocal sends estimations of local trust to the remote node. -// -// Client, context and key must be set. -// -// Returns any error which prevented the operation from completing correctly in error return. -func AnnounceLocal(ctx context.Context, prm AnnounceLocalPrm) (res AnnounceLocalRes, err error) { - var cliRes *client.ResAnnounceLocalTrust - - cliRes, err = prm.cli.AnnounceLocalTrust(ctx, prm.cliPrm) - if err == nil { - // pull out an error from status - err = apistatus.ErrFromStatus(cliRes.Status()) - } - - return -} - -// AnnounceIntermediatePrm groups parameters of AnnounceIntermediate operation. -type AnnounceIntermediatePrm struct { - commonPrm - - cliPrm client.PrmAnnounceIntermediateTrust -} - -// SetEpoch sets the number of the epoch when the trust calculation's iteration was executed. -func (x *AnnounceIntermediatePrm) SetEpoch(epoch uint64) { - x.cliPrm.SetEpoch(epoch) -} - -// SetIteration sets the number of the iteration of the trust calculation algorithm. -func (x *AnnounceIntermediatePrm) SetIteration(iter uint32) { - x.cliPrm.SetIteration(iter) -} - -// SetTrust sets the current global trust value computed at the iteration. -func (x *AnnounceIntermediatePrm) SetTrust(t reputation.PeerToPeerTrust) { - x.cliPrm.SetCurrentValue(t) -} - -// AnnounceIntermediateRes groups the resulting values of AnnounceIntermediate operation. -type AnnounceIntermediateRes struct{} - -// AnnounceIntermediate sends the global trust value calculated at the specified iteration -// and epoch to to the remote node. -// -// Client, context and key must be set. -// -// Returns any error which prevented the operation from completing correctly in error return. -func AnnounceIntermediate(ctx context.Context, prm AnnounceIntermediatePrm) (res AnnounceIntermediateRes, err error) { - var cliRes *client.ResAnnounceIntermediateTrust - - cliRes, err = prm.cli.AnnounceIntermediateTrust(ctx, prm.cliPrm) - if err == nil { - // pull out an error from status - err = apistatus.ErrFromStatus(cliRes.Status()) - } - - return -} diff --git a/cmd/frostfs-node/reputation/internal/client/doc.go b/cmd/frostfs-node/reputation/internal/client/doc.go deleted file mode 100644 index 1dc66cee6..000000000 --- a/cmd/frostfs-node/reputation/internal/client/doc.go +++ /dev/null @@ -1,11 +0,0 @@ -// Package internal provides functionality for FrostFS Node Reputation system communication with FrostFS network. -// The base client for accessing remote nodes via FrostFS API is a FrostFS SDK Go API client. -// However, although it encapsulates a useful piece of business logic (e.g. the signature mechanism), -// the Reputation service does not fully use the client's flexible interface. -// -// In this regard, this package provides functions over base API client necessary for the application. -// This allows you to concentrate the entire spectrum of the client's use in one place (this will be convenient -// both when updating the base client and for evaluating the UX of SDK library). So, it is expected that all -// Reputation service packages will be limited to this package for the development of functionality requiring -// FrostFS API communication. -package internal diff --git a/cmd/frostfs-node/reputation/local/remote.go b/cmd/frostfs-node/reputation/local/remote.go deleted file mode 100644 index 6197c6d69..000000000 --- a/cmd/frostfs-node/reputation/local/remote.go +++ /dev/null @@ -1,113 +0,0 @@ -package local - -import ( - "context" - "crypto/ecdsa" - - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/common" - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - reputationapi "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" - "go.uber.org/zap" -) - -// RemoteProviderPrm groups the required parameters of the RemoteProvider's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type RemoteProviderPrm struct { - Key *ecdsa.PrivateKey - Log *logger.Logger -} - -// NewRemoteProvider creates a new instance of the RemoteProvider. -// -// Panics if at least one value of the parameters is invalid. -// -// The created RemoteProvider does not require additional -// initialization and is completely ready for work. -func NewRemoteProvider(prm RemoteProviderPrm) *RemoteProvider { - switch { - case prm.Key == nil: - common.PanicOnPrmValue("NetMapSource", prm.Key) - case prm.Log == nil: - common.PanicOnPrmValue("Logger", prm.Log) - } - - return &RemoteProvider{ - key: prm.Key, - log: prm.Log, - } -} - -// RemoteProvider is an implementation of the clientKeyRemoteProvider interface. -type RemoteProvider struct { - key *ecdsa.PrivateKey - log *logger.Logger -} - -func (rp RemoteProvider) WithClient(c coreclient.Client) reputationcommon.WriterProvider { - return &TrustWriterProvider{ - client: c, - key: rp.key, - log: rp.log, - } -} - -type TrustWriterProvider struct { - client coreclient.Client - key *ecdsa.PrivateKey - log *logger.Logger -} - -func (twp *TrustWriterProvider) InitWriter(ep reputationcommon.EpochProvider) (reputationcommon.Writer, error) { - return &RemoteTrustWriter{ - ep: ep, - client: twp.client, - key: twp.key, - log: twp.log, - }, nil -} - -type RemoteTrustWriter struct { - ep reputationcommon.EpochProvider - client coreclient.Client - key *ecdsa.PrivateKey - log *logger.Logger - - buf []reputationapi.Trust -} - -func (rtp *RemoteTrustWriter) Write(_ context.Context, t reputation.Trust) error { - var apiTrust reputationapi.Trust - - apiTrust.SetValue(t.Value().Float64()) - apiTrust.SetPeer(t.Peer()) - - rtp.buf = append(rtp.buf, apiTrust) - - return nil -} - -func (rtp *RemoteTrustWriter) Close(ctx context.Context) error { - epoch := rtp.ep.Epoch() - - rtp.log.Debug(logs.LocalAnnouncingTrusts, - zap.Uint64("epoch", epoch), - ) - - var prm internalclient.AnnounceLocalPrm - - prm.SetClient(rtp.client) - prm.SetEpoch(epoch) - prm.SetTrusts(rtp.buf) - - _, err := internalclient.AnnounceLocal(ctx, prm) - - return err -} diff --git a/cmd/frostfs-node/reputation/local/storage.go b/cmd/frostfs-node/reputation/local/storage.go deleted file mode 100644 index a0dc3d4ce..000000000 --- a/cmd/frostfs-node/reputation/local/storage.go +++ /dev/null @@ -1,108 +0,0 @@ -package local - -import ( - "bytes" - "errors" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - trustcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/local/controller" - truststorage "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/local/storage" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" - "go.uber.org/zap" -) - -type TrustStorage struct { - Log *logger.Logger - - Storage *truststorage.Storage - - NmSrc netmapcore.Source - - LocalKey []byte -} - -func (s *TrustStorage) InitIterator(ep reputationcommon.EpochProvider) (trustcontroller.Iterator, error) { - epoch := ep.Epoch() - - s.Log.Debug(logs.LocalInitializingIteratorOverTrusts, - zap.Uint64("epoch", epoch), - ) - - epochStorage, err := s.Storage.DataForEpoch(epoch) - if err != nil && !errors.Is(err, truststorage.ErrNoPositiveTrust) { - return nil, err - } - - return &TrustIterator{ - ep: ep, - storage: s, - epochStorage: epochStorage, - }, nil -} - -type TrustIterator struct { - ep reputationcommon.EpochProvider - - storage *TrustStorage - - epochStorage *truststorage.EpochTrustValueStorage -} - -func (it *TrustIterator) Iterate(h reputation.TrustHandler) error { - if it.epochStorage != nil { - err := it.epochStorage.Iterate(h) - if !errors.Is(err, truststorage.ErrNoPositiveTrust) { - return err - } - } - - nm, err := it.storage.NmSrc.GetNetMapByEpoch(it.ep.Epoch()) - if err != nil { - return err - } - - // find out if local node is presented in netmap - localIndex := -1 - - nmNodes := nm.Nodes() - for i := range nmNodes { - if bytes.Equal(nmNodes[i].PublicKey(), it.storage.LocalKey) { - localIndex = i - break - } - } - - ln := len(nmNodes) - if localIndex >= 0 && ln > 0 { - ln-- - } - - // calculate Pj http://ilpubs.stanford.edu:8090/562/1/2002-56.pdf Chapter 4.5. - p := reputation.TrustOne.Div(reputation.TrustValueFromInt(ln)) - - for i := range nmNodes { - if i == localIndex { - continue - } - - var trusted, trusting apireputation.PeerID - - trusted.SetPublicKey(nmNodes[i].PublicKey()) - trusting.SetPublicKey(it.storage.LocalKey) - - trust := reputation.Trust{} - trust.SetPeer(trusted) - trust.SetValue(p) - trust.SetTrustingPeer(trusting) - - if err := h(trust); err != nil { - return err - } - } - - return nil -} diff --git a/cmd/frostfs-node/reputation/ticker/fixed.go b/cmd/frostfs-node/reputation/ticker/fixed.go deleted file mode 100644 index 5403882d5..000000000 --- a/cmd/frostfs-node/reputation/ticker/fixed.go +++ /dev/null @@ -1,90 +0,0 @@ -package ticker - -import ( - "fmt" - "sync" -) - -// IterationHandler is a callback of a certain block advance. -type IterationHandler func() - -// IterationsTicker represents a fixed tick number block timer. -// -// It can tick the blocks and perform certain actions -// on block time intervals. -type IterationsTicker struct { - m sync.Mutex - - curr uint64 - period uint64 - - times uint64 - - h IterationHandler -} - -// NewIterationsTicker creates a new IterationsTicker. -// -// It guaranties that a handler would be called the -// specified amount of times in the specified amount -// of blocks. After the last meaningful Tick, IterationsTicker -// becomes no-op timer. -// -// Returns an error only if times is greater than totalBlocks. -func NewIterationsTicker(totalBlocks uint64, times uint64, h IterationHandler) (*IterationsTicker, error) { - period := totalBlocks / times - - if period == 0 { - return nil, fmt.Errorf("impossible to tick %d times in %d blocks", - times, totalBlocks, - ) - } - - var curr uint64 - - // try to make handler calls as rare as possible - if totalBlocks%times != 0 { - extraBlocks := (period+1)*times - totalBlocks - - if period >= extraBlocks { - curr = extraBlocks + (period-extraBlocks)/2 - period++ - } - } - - return &IterationsTicker{ - curr: curr, - period: period, - times: times, - h: h, - }, nil -} - -// Tick ticks one block in the IterationsTicker. -// -// Returns `false` if the timer has finished its operations -// and there will be no more handler calls. -// Calling Tick after the returned `false` is safe, no-op -// and also returns `false`. -func (ft *IterationsTicker) Tick() bool { - ft.m.Lock() - defer ft.m.Unlock() - - if ft.times == 0 { - return false - } - - ft.curr++ - - if ft.curr%ft.period == 0 { - ft.h() - - ft.times-- - - if ft.times == 0 { - return false - } - } - - return true -} diff --git a/cmd/frostfs-node/reputation/ticker/fixed_test.go b/cmd/frostfs-node/reputation/ticker/fixed_test.go deleted file mode 100644 index 25e9bd08f..000000000 --- a/cmd/frostfs-node/reputation/ticker/fixed_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package ticker - -import ( - "errors" - "fmt" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestFixedTimer_Tick(t *testing.T) { - tests := [...]struct { - duration uint64 - times uint64 - err error - }{ - { - duration: 20, - times: 4, - err: nil, - }, - { - duration: 6, - times: 6, - err: nil, - }, - { - duration: 10, - times: 6, - err: nil, - }, - { - duration: 5, - times: 6, - err: errors.New("impossible to tick 6 times in 5 blocks"), - }, - } - - for _, test := range tests { - t.Run(fmt.Sprintf("duration:%d,times:%d", test.duration, test.times), func(t *testing.T) { - counter := uint64(0) - - timer, err := NewIterationsTicker(test.duration, test.times, func() { - counter++ - }) - if test.err != nil { - require.EqualError(t, err, test.err.Error()) - return - } - - require.NoError(t, err) - - for i := 0; i < int(test.duration); i++ { - if !timer.Tick() { - break - } - } - - require.Equal(t, false, timer.Tick()) - require.Equal(t, test.times, counter) - }) - } -} - -func TestFixedTimer_RareCalls(t *testing.T) { - tests := [...]struct { - duration uint64 - times uint64 - firstCall uint64 - period uint64 - }{ - { - duration: 11, - times: 6, - firstCall: 1, - period: 2, - }, - { - duration: 11, - times: 4, - firstCall: 2, - period: 3, - }, - { - duration: 20, - times: 3, - firstCall: 4, - period: 7, - }, - } - - for _, test := range tests { - t.Run(fmt.Sprintf("duration:%d,times:%d", test.duration, test.times), func(t *testing.T) { - var counter uint64 - - timer, err := NewIterationsTicker(test.duration, test.times, func() { - counter++ - }) - require.NoError(t, err) - - checked := false - - for i := 1; i <= int(test.duration); i++ { - if !timer.Tick() { - break - } - - if !checked && counter == 1 { - require.Equal(t, test.firstCall, uint64(i)) - checked = true - } - } - - require.Equal(t, false, timer.Tick()) - require.Equal(t, test.times, counter) - }) - } -} diff --git a/cmd/frostfs-node/timers.go b/cmd/frostfs-node/timers.go deleted file mode 100644 index 2ee2e8656..000000000 --- a/cmd/frostfs-node/timers.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/reputation/ticker" -) - -type eigenTrustTickers struct { - m sync.Mutex - - timers map[uint64]*ticker.IterationsTicker -} - -func (e *eigenTrustTickers) addEpochTimer(epoch uint64, timer *ticker.IterationsTicker) { - e.m.Lock() - defer e.m.Unlock() - - e.timers[epoch] = timer -} - -func (e *eigenTrustTickers) tick() { - e.m.Lock() - defer e.m.Unlock() - - for epoch, t := range e.timers { - if !t.Tick() { - delete(e.timers, epoch) - } - } -} - -func tickBlockTimers(c *cfg) { - c.cfgMorph.eigenTrustTicker.tick() -} - -func newEigenTrustIterTimer(c *cfg) { - c.cfgMorph.eigenTrustTicker = &eigenTrustTickers{ - // it is expected to have max 2 concurrent epoch - // in normal mode work - timers: make(map[uint64]*ticker.IterationsTicker, 2), - } -} diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 2f5e89e39..5ac3154bc 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -18,7 +18,6 @@ import ( addrvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/maddress" statevalidation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" subnetvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/subnet" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/reputation" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement" auditSettlement "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" @@ -29,13 +28,11 @@ import ( frostfsClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" - repClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/reputation" morphsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/subnet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" audittask "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit/taskmanager" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" controlsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" - reputationcommon "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" util2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" utilConfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -467,33 +464,6 @@ func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper, frostfsIDClient * return bindMainnetProcessor(frostfsProcessor, s) } -func (s *Server) initReputationProcessor(cfg *viper.Viper, sidechainFee fixedn.Fixed8) error { - repClient, err := repClient.NewFromMorph(s.morphClient, s.contracts.reputation, sidechainFee, repClient.TryNotary(), repClient.AsAlphabet()) - if err != nil { - return err - } - - // create reputation processor - reputationProcessor, err := reputation.New(&reputation.Params{ - Log: s.log, - PoolSize: cfg.GetInt("workers.reputation"), - EpochState: s, - AlphabetState: s, - ReputationWrapper: repClient, - ManagerBuilder: reputationcommon.NewManagerBuilder( - reputationcommon.ManagersPrm{ - NetMapSource: s.netmapClient, - }, - ), - NotaryDisabled: s.sideNotaryConfig.disabled, - }) - if err != nil { - return err - } - - return bindMorphProcessor(reputationProcessor, s) -} - func (s *Server) initGRPCServer(cfg *viper.Viper) error { controlSvcEndpoint := cfg.GetString("control.grpc.endpoint") if controlSvcEndpoint == "" { @@ -620,8 +590,6 @@ type serverProcessors struct { func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClients) (*serverProcessors, error) { result := &serverProcessors{} - fee := s.feeConfig.SideChainFee() - irf := s.createIRFetcher() s.statusIndex = newInnerRingIndexer( @@ -681,11 +649,6 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien return nil, err } - err = s.initReputationProcessor(cfg, fee) - if err != nil { - return nil, err - } - return result, nil } diff --git a/pkg/innerring/processors/reputation/handlers.go b/pkg/innerring/processors/reputation/handlers.go deleted file mode 100644 index 9b8e7f66a..000000000 --- a/pkg/innerring/processors/reputation/handlers.go +++ /dev/null @@ -1,29 +0,0 @@ -package reputation - -import ( - "encoding/hex" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - reputationEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/reputation" - "go.uber.org/zap" -) - -func (rp *Processor) handlePutReputation(ev event.Event) { - put := ev.(reputationEvent.Put) - peerID := put.PeerID() - - // FIXME: #1147 do not use `ToV2` method outside frostfs-api-go library - rp.log.Info(logs.Notification, - zap.String("type", "reputation put"), - zap.String("peer_id", hex.EncodeToString(peerID.PublicKey()))) - - // send event to the worker pool - - err := rp.pool.Submit(func() { rp.processPut(&put) }) - if err != nil { - // there system can be moved into controlled degradation stage - rp.log.Warn(logs.ReputationReputationWorkerPoolDrained, - zap.Int("capacity", rp.pool.Cap())) - } -} diff --git a/pkg/innerring/processors/reputation/process_put.go b/pkg/innerring/processors/reputation/process_put.go deleted file mode 100644 index f8814dd06..000000000 --- a/pkg/innerring/processors/reputation/process_put.go +++ /dev/null @@ -1,99 +0,0 @@ -package reputation - -import ( - "bytes" - "encoding/hex" - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - repClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/reputation" - reputationEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/reputation" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" - "go.uber.org/zap" -) - -var errWrongManager = errors.New("got manager that is incorrect for peer") - -func (rp *Processor) processPut(e *reputationEvent.Put) { - if !rp.alphabetState.IsAlphabet() { - rp.log.Info(logs.ReputationNonAlphabetModeIgnoreReputationPutNotification) - return - } - - epoch := e.Epoch() - id := e.PeerID() - value := e.Value() - - // check if epoch is valid - currentEpoch := rp.epochState.EpochCounter() - if epoch >= currentEpoch { - rp.log.Info(logs.ReputationIgnoreReputationValue, - zap.String("reason", "invalid epoch number"), - zap.Uint64("trust_epoch", epoch), - zap.Uint64("local_epoch", currentEpoch)) - - return - } - - // check signature - if !value.VerifySignature() { - rp.log.Info(logs.ReputationIgnoreReputationValue, - zap.String("reason", "invalid signature"), - ) - - return - } - - // check if manager is correct - if err := rp.checkManagers(epoch, value.Manager(), id); err != nil { - rp.log.Info(logs.ReputationIgnoreReputationValue, - zap.String("reason", "wrong manager"), - zap.String("error", err.Error())) - - return - } - - rp.approvePutReputation(e) -} - -func (rp *Processor) checkManagers(e uint64, mng apireputation.PeerID, peer apireputation.PeerID) error { - mm, err := rp.mngBuilder.BuildManagers(e, peer) - if err != nil { - return fmt.Errorf("could not build managers: %w", err) - } - - for _, m := range mm { - // FIXME: #1147 do not use `ToV2` method outside frostfs-api-go library - if bytes.Equal(mng.PublicKey(), m.PublicKey()) { - return nil - } - } - - return errWrongManager -} - -func (rp *Processor) approvePutReputation(e *reputationEvent.Put) { - var ( - id = e.PeerID() - err error - ) - - if nr := e.NotaryRequest(); nr != nil { - // put event was received via Notary service - err = rp.reputationWrp.Morph().NotarySignAndInvokeTX(nr.MainTransaction) - } else { - args := repClient.PutPrm{} - args.SetEpoch(e.Epoch()) - args.SetPeerID(id) - args.SetValue(e.Value()) - - err = rp.reputationWrp.Put(args) - } - if err != nil { - // FIXME: #1147 do not use `ToV2` method outside frostfs-api-go library - rp.log.Warn(logs.ReputationCantSendApprovalTxForReputationValue, - zap.String("peer_id", hex.EncodeToString(id.PublicKey())), - zap.String("error", err.Error())) - } -} diff --git a/pkg/innerring/processors/reputation/processor.go b/pkg/innerring/processors/reputation/processor.go deleted file mode 100644 index a248fa75f..000000000 --- a/pkg/innerring/processors/reputation/processor.go +++ /dev/null @@ -1,156 +0,0 @@ -package reputation - -import ( - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - repClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - reputationEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/nspcc-dev/neo-go/pkg/core/mempoolevent" - "github.com/panjf2000/ants/v2" - "go.uber.org/zap" -) - -type ( - // EpochState is a callback interface for inner ring global state. - EpochState interface { - EpochCounter() uint64 - } - - // AlphabetState is a callback interface for inner ring global state. - AlphabetState interface { - IsAlphabet() bool - } - - // Processor of events produced by reputation contract. - Processor struct { - log *logger.Logger - pool *ants.Pool - - epochState EpochState - alphabetState AlphabetState - - reputationWrp *repClient.Client - - mngBuilder common.ManagerBuilder - - notaryDisabled bool - } - - // Params of the processor constructor. - Params struct { - Log *logger.Logger - PoolSize int - EpochState EpochState - AlphabetState AlphabetState - ReputationWrapper *repClient.Client - ManagerBuilder common.ManagerBuilder - NotaryDisabled bool - } -) - -const ( - putReputationNotification = "reputationPut" -) - -// New creates reputation contract processor instance. -func New(p *Params) (*Processor, error) { - switch { - case p.Log == nil: - return nil, errors.New("ir/reputation: logger is not set") - case p.EpochState == nil: - return nil, errors.New("ir/reputation: global state is not set") - case p.AlphabetState == nil: - return nil, errors.New("ir/reputation: global state is not set") - case p.ReputationWrapper == nil: - return nil, errors.New("ir/reputation: reputation contract wrapper is not set") - case p.ManagerBuilder == nil: - return nil, errors.New("ir/reputation: manager builder is not set") - } - - p.Log.Debug(logs.ReputationReputationWorkerPool, zap.Int("size", p.PoolSize)) - - pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) - if err != nil { - return nil, fmt.Errorf("ir/reputation: can't create worker pool: %w", err) - } - - return &Processor{ - log: p.Log, - pool: pool, - epochState: p.EpochState, - alphabetState: p.AlphabetState, - reputationWrp: p.ReputationWrapper, - mngBuilder: p.ManagerBuilder, - notaryDisabled: p.NotaryDisabled, - }, nil -} - -// ListenerNotificationParsers for the 'event.Listener' event producer. -func (rp *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { - if !rp.notaryDisabled { - return nil - } - - var parsers []event.NotificationParserInfo - - // put reputation event - put := event.NotificationParserInfo{} - put.SetType(putReputationNotification) - put.SetScriptHash(rp.reputationWrp.ContractAddress()) - put.SetParser(reputationEvent.ParsePut) - parsers = append(parsers, put) - - return parsers -} - -// ListenerNotificationHandlers for the 'event.Listener' event producer. -func (rp *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { - if !rp.notaryDisabled { - return nil - } - - var handlers []event.NotificationHandlerInfo - - // put reputation handler - put := event.NotificationHandlerInfo{} - put.SetType(putReputationNotification) - put.SetScriptHash(rp.reputationWrp.ContractAddress()) - put.SetHandler(rp.handlePutReputation) - handlers = append(handlers, put) - - return handlers -} - -// ListenerNotaryParsers for the 'event.Listener' notary event producer. -func (rp *Processor) ListenerNotaryParsers() []event.NotaryParserInfo { - var p event.NotaryParserInfo - - p.SetMempoolType(mempoolevent.TransactionAdded) - p.SetRequestType(reputationEvent.PutNotaryEvent) - p.SetScriptHash(rp.reputationWrp.ContractAddress()) - p.SetParser(reputationEvent.ParsePutNotary) - - return []event.NotaryParserInfo{p} -} - -// ListenerNotaryHandlers for the 'event.Listener' notary event producer. -func (rp *Processor) ListenerNotaryHandlers() []event.NotaryHandlerInfo { - var h event.NotaryHandlerInfo - - h.SetMempoolType(mempoolevent.TransactionAdded) - h.SetRequestType(reputationEvent.PutNotaryEvent) - h.SetScriptHash(rp.reputationWrp.ContractAddress()) - h.SetHandler(rp.handlePutReputation) - - return []event.NotaryHandlerInfo{h} -} - -// TimersHandlers for the 'Timers' event producer. -func (rp *Processor) TimersHandlers() []event.NotificationHandlerInfo { - return nil -} diff --git a/pkg/innerring/rpc.go b/pkg/innerring/rpc.go index 9a89c4fcb..27338eaa4 100644 --- a/pkg/innerring/rpc.go +++ b/pkg/innerring/rpc.go @@ -27,7 +27,7 @@ type ( ClientCache struct { log *logger.Logger cache interface { - Get(clientcore.NodeInfo) (clientcore.Client, error) + Get(clientcore.NodeInfo) (clientcore.MultiAddressClient, error) CloseAll() } key *ecdsa.PrivateKey diff --git a/pkg/network/cache/client.go b/pkg/network/cache/client.go index 549e98b65..371d3c76f 100644 --- a/pkg/network/cache/client.go +++ b/pkg/network/cache/client.go @@ -38,7 +38,7 @@ func NewSDKClientCache(opts ClientCacheOpts) *ClientCache { } // Get function returns existing client or creates a new one. -func (c *ClientCache) Get(info clientcore.NodeInfo) (clientcore.Client, error) { +func (c *ClientCache) Get(info clientcore.NodeInfo) (clientcore.MultiAddressClient, error) { netAddr := info.AddressGroup() if c.opts.AllowExternal { netAddr = append(netAddr, info.ExternalAddressGroup()...) diff --git a/pkg/network/transport/reputation/grpc/service.go b/pkg/network/transport/reputation/grpc/service.go deleted file mode 100644 index bb9074324..000000000 --- a/pkg/network/transport/reputation/grpc/service.go +++ /dev/null @@ -1,50 +0,0 @@ -package grpcreputation - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/reputation" - reputation2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/reputation/grpc" - reputationrpc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/rpc" -) - -// Server wraps FrostFS API v2 Reputation service server -// and provides gRPC Reputation service server interface. -type Server struct { - srv reputationrpc.Server -} - -// New creates, initializes and returns Server instance. -func New(srv reputationrpc.Server) *Server { - return &Server{ - srv: srv, - } -} - -func (s *Server) AnnounceLocalTrust(ctx context.Context, r *reputation2.AnnounceLocalTrustRequest) (*reputation2.AnnounceLocalTrustResponse, error) { - req := new(reputation.AnnounceLocalTrustRequest) - if err := req.FromGRPCMessage(r); err != nil { - return nil, err - } - - resp, err := s.srv.AnnounceLocalTrust(ctx, req) - if err != nil { - return nil, err - } - - return resp.ToGRPCMessage().(*reputation2.AnnounceLocalTrustResponse), nil -} - -func (s *Server) AnnounceIntermediateResult(ctx context.Context, r *reputation2.AnnounceIntermediateResultRequest) (*reputation2.AnnounceIntermediateResultResponse, error) { - req := new(reputation.AnnounceIntermediateResultRequest) - if err := req.FromGRPCMessage(r); err != nil { - return nil, err - } - - resp, err := s.srv.AnnounceIntermediateResult(ctx, req) - if err != nil { - return nil, err - } - - return resp.ToGRPCMessage().(*reputation2.AnnounceIntermediateResultResponse), nil -} diff --git a/pkg/services/object/head/remote.go b/pkg/services/object/head/remote.go index 85f076a76..bcba181f2 100644 --- a/pkg/services/object/head/remote.go +++ b/pkg/services/object/head/remote.go @@ -15,7 +15,7 @@ import ( ) type ClientConstructor interface { - Get(clientcore.NodeInfo) (clientcore.Client, error) + Get(clientcore.NodeInfo) (clientcore.MultiAddressClient, error) } // RemoteHeader represents utility for getting diff --git a/pkg/services/reputation/common/deps.go b/pkg/services/reputation/common/deps.go deleted file mode 100644 index 3ea5aa88e..000000000 --- a/pkg/services/reputation/common/deps.go +++ /dev/null @@ -1,78 +0,0 @@ -package common - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" -) - -type EpochProvider interface { - // Must return epoch number to select the values. - Epoch() uint64 -} - -// Writer describes the interface for storing reputation.Trust values. -// -// This interface is provided by both local storage -// of values and remote (wrappers over the RPC). -type Writer interface { - // Write performs a write operation of reputation.Trust value - // and returns any error encountered. - // - // All values after the Close call must be flushed to the - // physical target. Implementations can cache values before - // Close operation. - // - // Write must not be called after Close. - Write(context.Context, reputation.Trust) error - - // Close exits with method-providing Writer. - // - // All cached values must be flushed before - // the Close's return. - // - // Methods must not be called after Close. - Close(context.Context) error -} - -// WriterProvider is a group of methods provided -// by entity which generates keepers of -// reputation.Trust values. -type WriterProvider interface { - // InitWriter should return an initialized Writer. - // - // Initialization problems are reported via error. - // If no error was returned, then the Writer must not be nil. - // - // Implementations can have different logic for different - // contexts, so specific ones may document their own behavior. - InitWriter(EpochProvider) (Writer, error) -} - -// ManagerBuilder defines an interface for providing a list -// of Managers for specific epoch. Implementation depends on trust value. -type ManagerBuilder interface { - // BuildManagers must compose list of managers. It depends on - // particular epoch and PeerID of the current route point. - BuildManagers(epoch uint64, p apireputation.PeerID) ([]ServerInfo, error) -} - -// ServerInfo describes a set of -// characteristics of a point in a route. -type ServerInfo interface { - // PublicKey returns public key of the node - // from the route in a binary representation. - PublicKey() []byte - - // Iterates over network addresses of the node - // in the route. Breaks iterating on true return - // of the handler. - IterateAddresses(func(string) bool) - - // Returns number of server's network addresses. - NumberOfAddresses() int - - // ExternalAddresses returns external addresses of a node. - ExternalAddresses() []string -} diff --git a/pkg/services/reputation/common/managers.go b/pkg/services/reputation/common/managers.go deleted file mode 100644 index 84201809f..000000000 --- a/pkg/services/reputation/common/managers.go +++ /dev/null @@ -1,133 +0,0 @@ -package common - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - apiNetmap "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" - "git.frostfs.info/TrueCloudLab/hrw" - "go.uber.org/zap" -) - -// managerBuilder is implementation of reputation ManagerBuilder interface. -// It sorts nodes in NetMap with HRW algorithms and -// takes the next node after the current one as the only manager. -type managerBuilder struct { - log *logger.Logger - nmSrc netmapcore.Source - opts *mngOptions -} - -// ManagersPrm groups the required parameters of the managerBuilder's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type ManagersPrm struct { - NetMapSource netmapcore.Source -} - -// NewManagerBuilder creates a new instance of the managerBuilder. -// -// Panics if at least one value of the parameters is invalid. -// -// The created managerBuilder does not require additional -// initialization and is completely ready for work. -func NewManagerBuilder(prm ManagersPrm, opts ...MngOption) ManagerBuilder { - switch { - case prm.NetMapSource == nil: - panic(fmt.Sprintf("invalid NetMapSource (%T):%v", prm.NetMapSource, prm.NetMapSource)) - } - - o := defaultMngOpts() - - for i := range opts { - opts[i](o) - } - - return &managerBuilder{ - log: o.log, - nmSrc: prm.NetMapSource, - opts: o, - } -} - -// implements Server on apiNetmap.NodeInfo. -type nodeServer apiNetmap.NodeInfo - -func (x nodeServer) PublicKey() []byte { - return (apiNetmap.NodeInfo)(x).PublicKey() -} - -func (x nodeServer) IterateAddresses(f func(string) bool) { - (apiNetmap.NodeInfo)(x).IterateNetworkEndpoints(f) -} - -func (x nodeServer) NumberOfAddresses() int { - return (apiNetmap.NodeInfo)(x).NumberOfNetworkEndpoints() -} - -func (x nodeServer) ExternalAddresses() []string { - return (apiNetmap.NodeInfo)(x).ExternalAddresses() -} - -// BuildManagers sorts nodes in NetMap with HRW algorithms and -// takes the next node after the current one as the only manager. -func (mb *managerBuilder) BuildManagers(epoch uint64, p apireputation.PeerID) ([]ServerInfo, error) { - mb.log.Debug(logs.CommonStartBuildingManagers, - zap.Uint64("epoch", epoch), - zap.Stringer("peer", p), - ) - - nm, err := mb.nmSrc.GetNetMapByEpoch(epoch) - if err != nil { - return nil, err - } - - nmNodes := nm.Nodes() - - // make a copy to keep order consistency of the origin netmap after sorting - nodes := make([]apiNetmap.NodeInfo, len(nmNodes)) - - copy(nodes, nmNodes) - - hrw.SortHasherSliceByValue(nodes, epoch) - - for i := range nodes { - if apireputation.ComparePeerKey(p, nodes[i].PublicKey()) { - managerIndex := i + 1 - - if managerIndex == len(nodes) { - managerIndex = 0 - } - - return []ServerInfo{nodeServer(nodes[managerIndex])}, nil - } - } - - return nil, nil -} - -type mngOptions struct { - log *logger.Logger -} - -type MngOption func(*mngOptions) - -func defaultMngOpts() *mngOptions { - return &mngOptions{ - log: &logger.Logger{Logger: zap.L()}, - } -} - -// WithLogger returns MngOption to specify logging component. -func WithLogger(l *logger.Logger) MngOption { - return func(o *mngOptions) { - if l != nil { - o.log = l - } - } -} diff --git a/pkg/services/reputation/common/router/calls.go b/pkg/services/reputation/common/router/calls.go deleted file mode 100644 index 4ed293beb..000000000 --- a/pkg/services/reputation/common/router/calls.go +++ /dev/null @@ -1,139 +0,0 @@ -package router - -import ( - "context" - "encoding/hex" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "go.uber.org/zap" -) - -// RouteInfo wraps epoch provider with additional passed -// route data. It is only used inside Router and is -// not passed in any external methods. -type RouteInfo struct { - common.EpochProvider - - passedRoute []common.ServerInfo -} - -// NewRouteInfo wraps the main context of value passing with its traversal route and epoch. -func NewRouteInfo(ep common.EpochProvider, passed []common.ServerInfo) *RouteInfo { - return &RouteInfo{ - EpochProvider: ep, - passedRoute: passed, - } -} - -type trustWriter struct { - router *Router - - routeInfo *RouteInfo - - routeMtx sync.RWMutex - mServers map[string]common.Writer -} - -// InitWriter initializes and returns Writer that sends each value to its next route point. -// -// If ep was created by NewRouteInfo, then the traversed route is taken into account, -// and the value will be sent to its continuation. Otherwise, the route will be laid -// from scratch and the value will be sent to its primary point. -// -// After building a list of remote points of the next leg of the route, the value is sent -// sequentially to all of them. If any transmissions (even all) fail, an error will not -// be returned. -// -// Close of the composed Writer calls Close method on each internal Writer generated in -// runtime and never returns an error. -// -// Always returns nil error. -func (r *Router) InitWriter(ep common.EpochProvider) (common.Writer, error) { - var ( - routeInfo *RouteInfo - ok bool - ) - - if routeInfo, ok = ep.(*RouteInfo); !ok { - routeInfo = &RouteInfo{ - EpochProvider: ep, - passedRoute: []common.ServerInfo{r.localSrvInfo}, - } - } - - return &trustWriter{ - router: r, - routeInfo: routeInfo, - mServers: make(map[string]common.Writer), - }, nil -} - -func (w *trustWriter) Write(ctx context.Context, t reputation.Trust) error { - w.routeMtx.Lock() - defer w.routeMtx.Unlock() - - route, err := w.router.routeBuilder.NextStage(w.routeInfo.Epoch(), t, w.routeInfo.passedRoute) - if err != nil { - return err - } else if len(route) == 0 { - route = []common.ServerInfo{nil} - } - - for _, remoteInfo := range route { - var key string - - if remoteInfo != nil { - key = hex.EncodeToString(remoteInfo.PublicKey()) - } - - remoteWriter, ok := w.mServers[key] - if !ok { - provider, err := w.router.remoteProvider.InitRemote(remoteInfo) - if err != nil { - w.router.log.Debug(logs.RouterCouldNotInitializeWriterProvider, - zap.String("error", err.Error()), - ) - - continue - } - - // init writer with original context wrapped in routeContext - remoteWriter, err = provider.InitWriter(w.routeInfo.EpochProvider) - if err != nil { - w.router.log.Debug(logs.RouterCouldNotInitializeWriter, - zap.String("error", err.Error()), - ) - - continue - } - - w.mServers[key] = remoteWriter - } - - err := remoteWriter.Write(ctx, t) - if err != nil { - w.router.log.Debug(logs.RouterCouldNotWriteTheValue, - zap.String("error", err.Error()), - ) - } - } - - return nil -} - -func (w *trustWriter) Close(ctx context.Context) error { - for key, wRemote := range w.mServers { - err := wRemote.Close(ctx) - if err != nil { - w.router.log.Debug(logs.RouterCouldNotCloseRemoteServerWriter, - zap.String("key", key), - zap.String("error", err.Error()), - ) - } - } - - return nil -} diff --git a/pkg/services/reputation/common/router/deps.go b/pkg/services/reputation/common/router/deps.go deleted file mode 100644 index 36aecb59f..000000000 --- a/pkg/services/reputation/common/router/deps.go +++ /dev/null @@ -1,28 +0,0 @@ -package router - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" -) - -// Builder groups methods to route values in the network. -type Builder interface { - // NextStage must return next group of route points - // for passed epoch and trust values. - // Implementation must take into account already passed route points. - // - // Empty passed list means being at the starting point of the route. - // - // Must return empty list and no error if the endpoint of the route is reached. - NextStage(epoch uint64, t reputation.Trust, passed []common.ServerInfo) ([]common.ServerInfo, error) -} - -// RemoteWriterProvider describes the component -// for sending values to a fixed route point. -type RemoteWriterProvider interface { - // InitRemote must return WriterProvider to the route point - // corresponding to info. - // - // Nil info matches the end of the route. - InitRemote(info common.ServerInfo) (common.WriterProvider, error) -} diff --git a/pkg/services/reputation/common/router/opts.go b/pkg/services/reputation/common/router/opts.go deleted file mode 100644 index 1b3454412..000000000 --- a/pkg/services/reputation/common/router/opts.go +++ /dev/null @@ -1,28 +0,0 @@ -package router - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// Option sets an optional parameter of Router. -type Option func(*options) - -type options struct { - log *logger.Logger -} - -func defaultOpts() *options { - return &options{ - log: &logger.Logger{Logger: zap.L()}, - } -} - -// WithLogger returns Option to specify logging component. -func WithLogger(l *logger.Logger) Option { - return func(o *options) { - if l != nil { - o.log = l - } - } -} diff --git a/pkg/services/reputation/common/router/router.go b/pkg/services/reputation/common/router/router.go deleted file mode 100644 index b80f6ce52..000000000 --- a/pkg/services/reputation/common/router/router.go +++ /dev/null @@ -1,81 +0,0 @@ -package router - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" -) - -// Prm groups the required parameters of the Router's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Characteristics of the local node's server. - // - // Must not be nil. - LocalServerInfo common.ServerInfo - - // Component for sending values to a fixed route point. - // - // Must not be nil. - RemoteWriterProvider RemoteWriterProvider - - // Route planner. - // - // Must not be nil. - Builder Builder -} - -// Router represents component responsible for routing -// local trust values over the network. -// -// For each fixed pair (node peer, epoch) there is a -// single value route on the network. Router provides the -// interface for writing values to the next point of the route. -// -// For correct operation, Router must be created using -// the constructor (New) based on the required parameters -// and optional components. After successful creation, -// the Router is immediately ready to work through API. -type Router struct { - log *logger.Logger - - remoteProvider RemoteWriterProvider - - routeBuilder Builder - - localSrvInfo common.ServerInfo -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -func New(prm Prm, opts ...Option) *Router { - switch { - case prm.RemoteWriterProvider == nil: - panicOnPrmValue("RemoteWriterProvider", prm.RemoteWriterProvider) - case prm.Builder == nil: - panicOnPrmValue("Builder", prm.Builder) - case prm.LocalServerInfo == nil: - panicOnPrmValue("LocalServerInfo", prm.LocalServerInfo) - } - - o := defaultOpts() - - for i := range opts { - opts[i](o) - } - - return &Router{ - log: o.log, - remoteProvider: prm.RemoteWriterProvider, - routeBuilder: prm.Builder, - localSrvInfo: prm.LocalServerInfo, - } -} diff --git a/pkg/services/reputation/common/router/util.go b/pkg/services/reputation/common/router/util.go deleted file mode 100644 index aa3190d2b..000000000 --- a/pkg/services/reputation/common/router/util.go +++ /dev/null @@ -1,40 +0,0 @@ -package router - -import ( - "bytes" - "errors" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" -) - -var errWrongRoute = errors.New("wrong route") - -// CheckRoute checks if the route is a route correctly constructed by the builder for value a. -// -// Returns nil if route is correct, otherwise an error clarifying the inconsistency. -func CheckRoute(builder Builder, epoch uint64, t reputation.Trust, route []common.ServerInfo) error { - for i := 1; i < len(route); i++ { - servers, err := builder.NextStage(epoch, t, route[:i]) - if err != nil { - return err - } else if len(servers) == 0 { - break - } - - found := false - - for j := range servers { - if bytes.Equal(servers[j].PublicKey(), route[i].PublicKey()) { - found = true - break - } - } - - if !found { - return errWrongRoute - } - } - - return nil -} diff --git a/pkg/services/reputation/eigentrust/calculator/calculator.go b/pkg/services/reputation/eigentrust/calculator/calculator.go deleted file mode 100644 index bfa274fea..000000000 --- a/pkg/services/reputation/eigentrust/calculator/calculator.go +++ /dev/null @@ -1,90 +0,0 @@ -package eigentrustcalc - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" -) - -// Prm groups the required parameters of the Calculator's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Alpha parameter from origin EigenTrust algorithm - // http://ilpubs.stanford.edu:8090/562/1/2002-56.pdf Ch.5.1. - // - // Must be in range (0, 1). - AlphaProvider AlphaProvider - - // Source of initial node trust values - // - // Must not be nil. - InitialTrustSource InitialTrustSource - - DaughterTrustSource DaughterTrustIteratorProvider - - IntermediateValueTarget common.WriterProvider - - FinalResultTarget IntermediateWriterProvider - - WorkerPool util.WorkerPool -} - -// Calculator is a processor of a single iteration of EigenTrust algorithm. -// -// For correct operation, the Calculator must be created -// using the constructor (New) based on the required parameters -// and optional components. After successful creation, -// the Calculator is immediately ready to work through -// API of external control of calculations and data transfer. -type Calculator struct { - alpha, beta reputation.TrustValue // beta = 1 - alpha - - prm Prm - - opts *options -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the Calculator. -// -// Panics if at least one value of the parameters is invalid. -// -// The created Calculator does not require additional -// initialization and is completely ready for work. -func New(prm Prm, opts ...Option) *Calculator { - switch { - case prm.AlphaProvider == nil: - panicOnPrmValue("AlphaProvider", prm.AlphaProvider) - case prm.InitialTrustSource == nil: - panicOnPrmValue("InitialTrustSource", prm.InitialTrustSource) - case prm.DaughterTrustSource == nil: - panicOnPrmValue("DaughterTrustSource", prm.DaughterTrustSource) - case prm.IntermediateValueTarget == nil: - panicOnPrmValue("IntermediateValueTarget", prm.IntermediateValueTarget) - case prm.FinalResultTarget == nil: - panicOnPrmValue("FinalResultTarget", prm.FinalResultTarget) - case prm.WorkerPool == nil: - panicOnPrmValue("WorkerPool", prm.WorkerPool) - } - - o := defaultOpts() - - for _, opt := range opts { - opt(o) - } - - return &Calculator{ - prm: prm, - opts: o, - } -} diff --git a/pkg/services/reputation/eigentrust/calculator/calls.go b/pkg/services/reputation/eigentrust/calculator/calls.go deleted file mode 100644 index 5e2e900ae..000000000 --- a/pkg/services/reputation/eigentrust/calculator/calls.go +++ /dev/null @@ -1,295 +0,0 @@ -package eigentrustcalc - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" - "go.uber.org/zap" -) - -type CalculatePrm struct { - last bool - - ei eigentrust.EpochIteration -} - -func (p *CalculatePrm) SetLast(last bool) { - p.last = last -} - -func (p *CalculatePrm) SetEpochIteration(ei eigentrust.EpochIteration) { - p.ei = ei -} - -func (c *Calculator) Calculate(ctx context.Context, prm CalculatePrm) { - alpha, err := c.prm.AlphaProvider.EigenTrustAlpha() - if err != nil { - c.opts.log.Debug( - logs.CalculatorFailedToGetAlphaParam, - zap.Error(err), - ) - return - } - - c.alpha = reputation.TrustValueFromFloat64(alpha) - c.beta = reputation.TrustValueFromFloat64(1 - alpha) - - epochIteration := prm.ei - - iter := epochIteration.I() - - log := c.opts.log.With( - zap.Uint64("epoch", epochIteration.Epoch()), - zap.Uint32("iteration", iter), - ) - - if iter == 0 { - c.sendInitialValues(ctx, epochIteration) - return - } - - // decrement iteration number to select the values collected - // on the previous stage - epochIteration.SetI(iter - 1) - - consumersIter, err := c.prm.DaughterTrustSource.InitConsumersIterator(epochIteration) - if err != nil { - log.Debug(logs.CalculatorConsumersTrustIteratorsInitFailure, - zap.String("error", err.Error()), - ) - - return - } - - // continue with initial iteration number - epochIteration.SetI(iter) - - err = consumersIter.Iterate(func(daughter apireputation.PeerID, iter TrustIterator) error { - err := c.prm.WorkerPool.Submit(func() { - c.iterateDaughter(ctx, iterDaughterPrm{ - lastIter: prm.last, - ei: epochIteration, - id: daughter, - consumersIter: iter, - }) - }) - if err != nil { - log.Debug(logs.CalculatorWorkerPoolSubmitFailure, - zap.String("error", err.Error()), - ) - } - - // don't stop trying - return nil - }) - if err != nil { - log.Debug(logs.CalculatorIterateDaughtersConsumersFailed, - zap.String("error", err.Error()), - ) - } -} - -type iterDaughterPrm struct { - lastIter bool - - ei EpochIterationInfo - - id apireputation.PeerID - - consumersIter TrustIterator -} - -func (c *Calculator) iterateDaughter(ctx context.Context, p iterDaughterPrm) { - initTrust, err := c.prm.InitialTrustSource.InitialTrust(p.id) - if err != nil { - c.opts.log.Debug(logs.CalculatorGetInitialTrustFailure, - zap.Stringer("daughter", p.id), - zap.String("error", err.Error()), - ) - - return - } - - daughterIter, err := c.prm.DaughterTrustSource.InitDaughterIterator(p.ei, p.id) - if err != nil { - c.opts.log.Debug(logs.CalculatorDaughterTrustIteratorsInitFailure, - zap.String("error", err.Error()), - ) - - return - } - - sum := reputation.TrustZero - - err = p.consumersIter.Iterate(func(trust reputation.Trust) error { - if !p.lastIter { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - } - - sum.Add(trust.Value()) - return nil - }) - if err != nil { - c.opts.log.Debug(logs.CalculatorIterateOverDaughtersTrustsFailure, - zap.String("error", err.Error()), - ) - - return - } - - // Alpha * Pd - initTrust.Mul(c.alpha) - - sum.Mul(c.beta) - sum.Add(initTrust) - - var intermediateTrust eigentrust.IterationTrust - - intermediateTrust.SetEpoch(p.ei.Epoch()) - intermediateTrust.SetPeer(p.id) - intermediateTrust.SetI(p.ei.I()) - - if p.lastIter { - c.processLastIteration(p, intermediateTrust, sum) - } else { - c.processIntermediateIteration(ctx, p, daughterIter, sum) - } -} - -func (c *Calculator) processLastIteration(p iterDaughterPrm, intermediateTrust eigentrust.IterationTrust, sum reputation.TrustValue) { - finalWriter, err := c.prm.FinalResultTarget.InitIntermediateWriter(p.ei) - if err != nil { - c.opts.log.Debug(logs.CalculatorInitWriterFailure, - zap.String("error", err.Error()), - ) - - return - } - - intermediateTrust.SetValue(sum) - - err = finalWriter.WriteIntermediateTrust(intermediateTrust) - if err != nil { - c.opts.log.Debug(logs.CalculatorWriteFinalResultFailure, - zap.String("error", err.Error()), - ) - - return - } -} - -func (c *Calculator) processIntermediateIteration(ctx context.Context, p iterDaughterPrm, daughterIter TrustIterator, sum reputation.TrustValue) { - intermediateWriter, err := c.prm.IntermediateValueTarget.InitWriter(p.ei) - if err != nil { - c.opts.log.Debug(logs.CalculatorInitWriterFailure, - zap.String("error", err.Error()), - ) - - return - } - - err = daughterIter.Iterate(func(trust reputation.Trust) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - val := trust.Value() - val.Mul(sum) - - trust.SetValue(val) - - err := intermediateWriter.Write(ctx, trust) - if err != nil { - c.opts.log.Debug(logs.CalculatorWriteValueFailure, - zap.String("error", err.Error()), - ) - } - - return nil - }) - if err != nil { - c.opts.log.Debug(logs.CalculatorIterateDaughterTrustsFailure, - zap.String("error", err.Error()), - ) - } - - err = intermediateWriter.Close(ctx) - if err != nil { - c.opts.log.Error( - "could not close writer", - zap.String("error", err.Error()), - ) - } -} - -func (c *Calculator) sendInitialValues(ctx context.Context, epochInfo EpochIterationInfo) { - daughterIter, err := c.prm.DaughterTrustSource.InitAllDaughtersIterator(epochInfo) - if err != nil { - c.opts.log.Debug(logs.CalculatorAllDaughtersTrustIteratorsInitFailure, - zap.String("error", err.Error()), - ) - - return - } - - intermediateWriter, err := c.prm.IntermediateValueTarget.InitWriter(epochInfo) - if err != nil { - c.opts.log.Debug(logs.CalculatorInitWriterFailure, - zap.String("error", err.Error()), - ) - - return - } - - err = daughterIter.Iterate(func(daughter apireputation.PeerID, iterator TrustIterator) error { - return iterator.Iterate(func(trust reputation.Trust) error { - trusted := trust.Peer() - - initTrust, err := c.prm.InitialTrustSource.InitialTrust(trusted) - if err != nil { - c.opts.log.Debug(logs.CalculatorGetInitialTrustFailure, - zap.Stringer("peer", trusted), - zap.String("error", err.Error()), - ) - - // don't stop on single failure - return nil - } - - initTrust.Mul(trust.Value()) - trust.SetValue(initTrust) - - err = intermediateWriter.Write(ctx, trust) - if err != nil { - c.opts.log.Debug(logs.CalculatorWriteValueFailure, - zap.String("error", err.Error()), - ) - - // don't stop on single failure - } - - return nil - }) - }) - if err != nil { - c.opts.log.Debug(logs.CalculatorIterateOverAllDaughtersFailure, - zap.String("error", err.Error()), - ) - } - - err = intermediateWriter.Close(ctx) - if err != nil { - c.opts.log.Debug(logs.CalculatorCouldNotCloseWriter, - zap.String("error", err.Error()), - ) - } -} diff --git a/pkg/services/reputation/eigentrust/calculator/deps.go b/pkg/services/reputation/eigentrust/calculator/deps.go deleted file mode 100644 index a22d1df76..000000000 --- a/pkg/services/reputation/eigentrust/calculator/deps.go +++ /dev/null @@ -1,74 +0,0 @@ -package eigentrustcalc - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" -) - -type EpochIterationInfo interface { - // Must return epoch number to select the values - // for global trust calculation. - Epoch() uint64 - - // Must return the sequence number of the iteration. - I() uint32 -} - -// InitialTrustSource must provide initial(non-calculated) -// trusts to current node's daughter. Realization may depends -// on daughter. -type InitialTrustSource interface { - InitialTrust(apireputation.PeerID) (reputation.TrustValue, error) -} - -// TrustIterator must iterate over all retrieved(or calculated) trusts -// and call passed TrustHandler on them. -type TrustIterator interface { - Iterate(reputation.TrustHandler) error -} - -type PeerTrustsHandler func(apireputation.PeerID, TrustIterator) error - -// PeerTrustsIterator must iterate over all nodes(PeerIDs) and provide -// TrustIterator for iteration over node's Trusts to others peers. -type PeerTrustsIterator interface { - Iterate(PeerTrustsHandler) error -} - -type DaughterTrustIteratorProvider interface { - // InitDaughterIterator must init TrustIterator - // that iterates over received local trusts from - // daughter p for epochInfo.Epoch() epoch. - InitDaughterIterator(epochInfo EpochIterationInfo, p apireputation.PeerID) (TrustIterator, error) - // InitAllDaughtersIterator must init PeerTrustsIterator - // that must iterate over all daughters of the current - // node(manager) and all trusts received from them for - // epochInfo.Epoch() epoch. - InitAllDaughtersIterator(epochInfo EpochIterationInfo) (PeerTrustsIterator, error) - // InitConsumersIterator must init PeerTrustsIterator - // that must iterate over all daughters of the current - // node(manager) and their consumers' trusts received - // from other managers for epochInfo.Epoch() epoch and - // epochInfo.I() iteration. - InitConsumersIterator(EpochIterationInfo) (PeerTrustsIterator, error) -} - -// IntermediateWriter must write intermediate result to contract. -// It may depends on realization either trust is sent directly to contract -// or via redirecting to other node. -type IntermediateWriter interface { - WriteIntermediateTrust(eigentrust.IterationTrust) error -} - -// IntermediateWriterProvider must provide ready-to-work -// IntermediateWriter. -type IntermediateWriterProvider interface { - InitIntermediateWriter(EpochIterationInfo) (IntermediateWriter, error) -} - -// AlphaProvider must provide information about required -// alpha parameter for eigen trust algorithm. -type AlphaProvider interface { - EigenTrustAlpha() (float64, error) -} diff --git a/pkg/services/reputation/eigentrust/calculator/opts.go b/pkg/services/reputation/eigentrust/calculator/opts.go deleted file mode 100644 index e1e572361..000000000 --- a/pkg/services/reputation/eigentrust/calculator/opts.go +++ /dev/null @@ -1,30 +0,0 @@ -package eigentrustcalc - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// Option sets an optional parameter of Controller. -type Option func(*options) - -type options struct { - log *logger.Logger -} - -func defaultOpts() *options { - return &options{ - log: &logger.Logger{Logger: zap.L()}, - } -} - -// WithLogger returns option to specify logging component. -// -// Ignores nil values. -func WithLogger(l *logger.Logger) Option { - return func(o *options) { - if l != nil { - o.log = l - } - } -} diff --git a/pkg/services/reputation/eigentrust/controller/calls.go b/pkg/services/reputation/eigentrust/controller/calls.go deleted file mode 100644 index 886daf9be..000000000 --- a/pkg/services/reputation/eigentrust/controller/calls.go +++ /dev/null @@ -1,73 +0,0 @@ -package eigentrustctrl - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" - "go.uber.org/zap" -) - -// ContinuePrm groups the required parameters of Continue operation. -type ContinuePrm struct { - Epoch uint64 -} - -type iterContext struct { - eigentrust.EpochIteration - - iterationNumber uint32 - last bool -} - -func (x iterContext) Last() bool { - return x.last -} - -// Continue moves the global reputation calculator to the next iteration. -func (c *Controller) Continue(ctx context.Context, prm ContinuePrm) { - c.mtx.Lock() - - { - iterCtx, ok := c.mCtx[prm.Epoch] - if !ok { - iterCtx = new(iterContext) - c.mCtx[prm.Epoch] = iterCtx - - iterCtx.EpochIteration.SetEpoch(prm.Epoch) - - iterations, err := c.prm.IterationsProvider.EigenTrustIterations() - if err != nil { - c.opts.log.Error(logs.ControllerCouldNotGetEigenTrustIterationNumber, - zap.Error(err), - ) - } else { - iterCtx.iterationNumber = uint32(iterations) - } - } - - iterCtx.last = iterCtx.I() == iterCtx.iterationNumber-1 - - err := c.prm.WorkerPool.Submit(func() { - c.prm.DaughtersTrustCalculator.Calculate(ctx, iterCtx) - - // iteration++ - iterCtx.Increment() - }) - if err != nil { - c.opts.log.Debug(logs.ControllerIterationSubmitFailure, - zap.String("error", err.Error()), - ) - } - - if iterCtx.last { - // will only live while the application is alive. - // during normal operation of the system. Also, such information - // number as already processed, but in any case it grows up - // In this case and worker pool failure we can mark epoch - delete(c.mCtx, prm.Epoch) - } - } - - c.mtx.Unlock() -} diff --git a/pkg/services/reputation/eigentrust/controller/controller.go b/pkg/services/reputation/eigentrust/controller/controller.go deleted file mode 100644 index a6d0d4a82..000000000 --- a/pkg/services/reputation/eigentrust/controller/controller.go +++ /dev/null @@ -1,86 +0,0 @@ -package eigentrustctrl - -import ( - "fmt" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" -) - -// Prm groups the required parameters of the Controller's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Component of computing iteration of EigenTrust algorithm. - // - // Must not be nil. - DaughtersTrustCalculator DaughtersTrustCalculator - - // IterationsProvider provides information about numbers - // of iterations for algorithm. - IterationsProvider IterationsProvider - - // Routine execution pool for single epoch iteration. - WorkerPool util.WorkerPool -} - -// Controller represents EigenTrust algorithm transient controller. -// -// Controller's main goal is to separate the two main stages of -// the calculation: -// 1. reporting local values to manager nodes -// 2. calculating global trusts of child nodes -// -// Calculation stages are controlled based on external signals -// that come from the application through the Controller's API. -// -// For correct operation, the controller must be created -// using the constructor (New) based on the required parameters -// and optional components. After successful creation, -// the constructor is immediately ready to work through -// API of external control of calculations and data transfer. -type Controller struct { - prm Prm - - opts *options - - mtx sync.Mutex - mCtx map[uint64]*iterContext -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the Controller. -// -// Panics if at least one value of the parameters is invalid. -// -// The created Controller does not require additional -// initialization and is completely ready for work. -func New(prm Prm, opts ...Option) *Controller { - switch { - case prm.IterationsProvider == nil: - panicOnPrmValue("IterationNumber", prm.IterationsProvider) - case prm.WorkerPool == nil: - panicOnPrmValue("WorkerPool", prm.WorkerPool) - case prm.DaughtersTrustCalculator == nil: - panicOnPrmValue("DaughtersTrustCalculator", prm.DaughtersTrustCalculator) - } - - o := defaultOpts() - - for _, opt := range opts { - opt(o) - } - - return &Controller{ - prm: prm, - opts: o, - mCtx: make(map[uint64]*iterContext), - } -} diff --git a/pkg/services/reputation/eigentrust/controller/deps.go b/pkg/services/reputation/eigentrust/controller/deps.go deleted file mode 100644 index c068f7cc4..000000000 --- a/pkg/services/reputation/eigentrust/controller/deps.go +++ /dev/null @@ -1,37 +0,0 @@ -package eigentrustctrl - -import "context" - -// IterationContext is a context of the i-th -// stage of iterative EigenTrust algorithm. -type IterationContext interface { - // Must return epoch number to select the values - // for global trust calculation. - Epoch() uint64 - - // Must return the sequence number of the iteration. - I() uint32 - - // Must return true if I() is the last iteration. - Last() bool -} - -// DaughtersTrustCalculator is an interface of entity -// responsible for calculating the global trust of -// daughter nodes in terms of EigenTrust algorithm. -type DaughtersTrustCalculator interface { - // Must perform the iteration step of the loop - // for computing the global trust of all daughter - // nodes and sending intermediate values - // according to EigenTrust description - // http://ilpubs.stanford.edu:8090/562/1/2002-56.pdf Ch.5.1. - // - // Execution should be interrupted if ctx.Last(). - Calculate(ctx context.Context, iter IterationContext) -} - -// IterationsProvider must provide information about numbers -// of iterations for algorithm. -type IterationsProvider interface { - EigenTrustIterations() (uint64, error) -} diff --git a/pkg/services/reputation/eigentrust/controller/opts.go b/pkg/services/reputation/eigentrust/controller/opts.go deleted file mode 100644 index 16bc61c2f..000000000 --- a/pkg/services/reputation/eigentrust/controller/opts.go +++ /dev/null @@ -1,30 +0,0 @@ -package eigentrustctrl - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// Option sets an optional parameter of Controller. -type Option func(*options) - -type options struct { - log *logger.Logger -} - -func defaultOpts() *options { - return &options{ - log: &logger.Logger{Logger: zap.L()}, - } -} - -// WithLogger returns option to specify logging component. -// -// Ignores nil values. -func WithLogger(l *logger.Logger) Option { - return func(o *options) { - if l != nil { - o.log = l - } - } -} diff --git a/pkg/services/reputation/eigentrust/iteration.go b/pkg/services/reputation/eigentrust/iteration.go deleted file mode 100644 index e4793f044..000000000 --- a/pkg/services/reputation/eigentrust/iteration.go +++ /dev/null @@ -1,44 +0,0 @@ -package eigentrust - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" -) - -type EpochIteration struct { - e uint64 - i uint32 -} - -func (x EpochIteration) Epoch() uint64 { - return x.e -} - -func (x *EpochIteration) SetEpoch(e uint64) { - x.e = e -} - -func (x EpochIteration) I() uint32 { - return x.i -} - -func (x *EpochIteration) SetI(i uint32) { - x.i = i -} - -func (x *EpochIteration) Increment() { - x.i++ -} - -type IterationTrust struct { - EpochIteration - reputation.Trust -} - -func NewEpochIteration(epoch uint64, iter uint32) *EpochIteration { - ei := EpochIteration{} - - ei.SetI(iter) - ei.SetEpoch(epoch) - - return &ei -} diff --git a/pkg/services/reputation/eigentrust/routes/builder.go b/pkg/services/reputation/eigentrust/routes/builder.go deleted file mode 100644 index ddd5a2ae0..000000000 --- a/pkg/services/reputation/eigentrust/routes/builder.go +++ /dev/null @@ -1,59 +0,0 @@ -package routes - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" -) - -// Prm groups the required parameters of the Builder's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Manager builder for current node. - // - // Must not be nil. - ManagerBuilder common.ManagerBuilder - - Log *logger.Logger -} - -// Builder represents component that routes node to its managers. -// -// For correct operation, Builder must be created using -// the constructor (New) based on the required parameters -// and optional components. After successful creation, -// the Builder is immediately ready to work through API. -type Builder struct { - managerBuilder common.ManagerBuilder - log *logger.Logger -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the Builder. -// -// Panics if at least one value of the parameters is invalid. -// -// The created Builder does not require additional -// initialization and is completely ready for work. -func New(prm Prm) *Builder { - switch { - case prm.ManagerBuilder == nil: - panicOnPrmValue("ManagerBuilder", prm.ManagerBuilder) - case prm.Log == nil: - panicOnPrmValue("Logger", prm.Log) - } - - return &Builder{ - managerBuilder: prm.ManagerBuilder, - log: prm.Log, - } -} diff --git a/pkg/services/reputation/eigentrust/routes/calls.go b/pkg/services/reputation/eigentrust/routes/calls.go deleted file mode 100644 index ccb2fe8ea..000000000 --- a/pkg/services/reputation/eigentrust/routes/calls.go +++ /dev/null @@ -1,33 +0,0 @@ -package routes - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "go.uber.org/zap" -) - -// NextStage builds Manager list for trusted node and returns it directly. -// -// If passed route has more than one point, then endpoint of the route is reached. -func (b *Builder) NextStage(epoch uint64, t reputation.Trust, passed []common.ServerInfo) ([]common.ServerInfo, error) { - passedLen := len(passed) - - b.log.Debug(logs.RoutesBuildingNextStageForTrustRoute, - zap.Uint64("epoch", epoch), - zap.Int("passed_length", passedLen), - ) - - if passedLen > 1 { - return nil, nil - } - - route, err := b.managerBuilder.BuildManagers(epoch, t.Peer()) - if err != nil { - return nil, fmt.Errorf("could not build managers for epoch: %d: %w", epoch, err) - } - - return route, nil -} diff --git a/pkg/services/reputation/eigentrust/storage/consumers/calls.go b/pkg/services/reputation/eigentrust/storage/consumers/calls.go deleted file mode 100644 index 55a4d6f3d..000000000 --- a/pkg/services/reputation/eigentrust/storage/consumers/calls.go +++ /dev/null @@ -1,201 +0,0 @@ -package consumerstorage - -import ( - "fmt" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust" - eigentrustcalc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/calculator" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" -) - -// Put saves intermediate trust of the consumer to daughter peer. -func (x *Storage) Put(trust eigentrust.IterationTrust) { - var s *iterationConsumersStorage - - x.mtx.Lock() - - { - epoch := trust.Epoch() - - s = x.mItems[epoch] - if s == nil { - s = &iterationConsumersStorage{ - mItems: make(map[uint32]*ConsumersStorage, 1), - } - - x.mItems[epoch] = s - } - } - - x.mtx.Unlock() - - s.put(trust) -} - -// Consumers returns the storage of trusts of the consumers of the daughter peers -// for particular iteration of EigenTrust calculation for particular epoch. -// -// Returns false if there is no data for the epoch and iter. -func (x *Storage) Consumers(epoch uint64, iter uint32) (*ConsumersStorage, bool) { - var ( - s *iterationConsumersStorage - ok bool - ) - - x.mtx.Lock() - - { - s, ok = x.mItems[epoch] - } - - x.mtx.Unlock() - - if !ok { - return nil, false - } - - return s.consumers(iter) -} - -// maps iteration numbers of EigenTrust algorithm to repositories -// of the trusts of the consumers of the daughter peers. -type iterationConsumersStorage struct { - mtx sync.RWMutex - - mItems map[uint32]*ConsumersStorage -} - -func (x *iterationConsumersStorage) put(trust eigentrust.IterationTrust) { - var s *ConsumersStorage - - x.mtx.Lock() - - { - iter := trust.I() - - s = x.mItems[iter] - if s == nil { - s = &ConsumersStorage{ - mItems: make(map[string]*ConsumersTrusts, 1), - } - - x.mItems[iter] = s - } - } - - x.mtx.Unlock() - - s.put(trust) -} - -func (x *iterationConsumersStorage) consumers(iter uint32) (s *ConsumersStorage, ok bool) { - x.mtx.Lock() - - { - s, ok = x.mItems[iter] - } - - x.mtx.Unlock() - - return -} - -// ConsumersStorage represents in-memory storage of intermediate trusts -// of the peer consumers. -// -// Maps daughter peers to repositories of the trusts of their consumers. -type ConsumersStorage struct { - mtx sync.RWMutex - - mItems map[string]*ConsumersTrusts -} - -func (x *ConsumersStorage) put(trust eigentrust.IterationTrust) { - var s *ConsumersTrusts - - x.mtx.Lock() - - { - daughter := trust.Peer().EncodeToString() - - s = x.mItems[daughter] - if s == nil { - s = &ConsumersTrusts{ - mItems: make(map[string]reputation.Trust, 1), - } - - x.mItems[daughter] = s - } - } - - x.mtx.Unlock() - - s.put(trust) -} - -// Iterate passes IDs of the daughter peers with the trusts of their consumers to h. -// -// Returns errors from h directly. -func (x *ConsumersStorage) Iterate(h eigentrustcalc.PeerTrustsHandler) (err error) { - x.mtx.RLock() - - { - for strTrusted, trusts := range x.mItems { - var trusted apireputation.PeerID - - if strTrusted != "" { - err = trusted.DecodeString(strTrusted) - if err != nil { - panic(fmt.Sprintf("decode peer ID string %s: %v", strTrusted, err)) - } - } - - if err = h(trusted, trusts); err != nil { - break - } - } - } - - x.mtx.RUnlock() - - return -} - -// ConsumersTrusts represents in-memory storage of the trusts -// of the consumer peers to some other peer. -type ConsumersTrusts struct { - mtx sync.RWMutex - - mItems map[string]reputation.Trust -} - -func (x *ConsumersTrusts) put(trust eigentrust.IterationTrust) { - x.mtx.Lock() - - { - x.mItems[trust.TrustingPeer().EncodeToString()] = trust.Trust - } - - x.mtx.Unlock() -} - -// Iterate passes all stored trusts to h. -// -// Returns errors from h directly. -func (x *ConsumersTrusts) Iterate(h reputation.TrustHandler) (err error) { - x.mtx.RLock() - - { - for _, trust := range x.mItems { - if err = h(trust); err != nil { - break - } - } - } - - x.mtx.RUnlock() - - return -} diff --git a/pkg/services/reputation/eigentrust/storage/consumers/storage.go b/pkg/services/reputation/eigentrust/storage/consumers/storage.go deleted file mode 100644 index ee811d84b..000000000 --- a/pkg/services/reputation/eigentrust/storage/consumers/storage.go +++ /dev/null @@ -1,40 +0,0 @@ -package consumerstorage - -import ( - "sync" -) - -// Prm groups the required parameters of the Storage's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -// -// The component is not parameterizable at the moment. -type Prm struct{} - -// Storage represents in-memory storage of the trusts -// of the consumer peers. -// -// It maps epoch numbers to the repositories of intermediate -// trusts of the consumers of the daughter peers. -// -// For correct operation, Storage must be created -// using the constructor (New) based on the required parameters -// and optional components. After successful creation, -// Storage is immediately ready to work through API. -type Storage struct { - mtx sync.RWMutex - - mItems map[uint64]*iterationConsumersStorage -} - -// New creates a new instance of the Storage. -// -// The created Storage does not require additional -// initialization and is completely ready for work. -func New(_ Prm) *Storage { - return &Storage{ - mItems: make(map[uint64]*iterationConsumersStorage), - } -} diff --git a/pkg/services/reputation/eigentrust/storage/daughters/calls.go b/pkg/services/reputation/eigentrust/storage/daughters/calls.go deleted file mode 100644 index eb229365e..000000000 --- a/pkg/services/reputation/eigentrust/storage/daughters/calls.go +++ /dev/null @@ -1,177 +0,0 @@ -package daughters - -import ( - "fmt" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - eigentrustcalc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/eigentrust/calculator" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" -) - -// Put saves daughter peer's trust to its provider for the epoch. -func (x *Storage) Put(epoch uint64, trust reputation.Trust) { - var s *DaughterStorage - - x.mtx.Lock() - - { - s = x.mItems[epoch] - if s == nil { - s = &DaughterStorage{ - mItems: make(map[string]*DaughterTrusts, 1), - } - - x.mItems[epoch] = s - } - } - - x.mtx.Unlock() - - s.put(trust) -} - -// DaughterTrusts returns daughter trusts for the epoch. -// -// Returns false if there is no data for the epoch and daughter. -func (x *Storage) DaughterTrusts(epoch uint64, daughter apireputation.PeerID) (*DaughterTrusts, bool) { - var ( - s *DaughterStorage - ok bool - ) - - x.mtx.RLock() - - { - s, ok = x.mItems[epoch] - } - - x.mtx.RUnlock() - - if !ok { - return nil, false - } - - return s.daughterTrusts(daughter) -} - -// AllDaughterTrusts returns daughter iterator for the epoch. -// -// Returns false if there is no data for the epoch and daughter. -func (x *Storage) AllDaughterTrusts(epoch uint64) (*DaughterStorage, bool) { - x.mtx.RLock() - defer x.mtx.RUnlock() - - s, ok := x.mItems[epoch] - - return s, ok -} - -// DaughterStorage maps IDs of daughter peers to repositories of the local trusts to their providers. -type DaughterStorage struct { - mtx sync.RWMutex - - mItems map[string]*DaughterTrusts -} - -// Iterate passes IDs of the daughter peers with their trusts to h. -// -// Returns errors from h directly. -func (x *DaughterStorage) Iterate(h eigentrustcalc.PeerTrustsHandler) (err error) { - x.mtx.RLock() - - { - for strDaughter, daughterTrusts := range x.mItems { - var daughter apireputation.PeerID - - if strDaughter != "" { - err = daughter.DecodeString(strDaughter) - if err != nil { - panic(fmt.Sprintf("decode peer ID string %s: %v", strDaughter, err)) - } - } - - if err = h(daughter, daughterTrusts); err != nil { - break - } - } - } - - x.mtx.RUnlock() - - return -} - -func (x *DaughterStorage) put(trust reputation.Trust) { - var dt *DaughterTrusts - - x.mtx.Lock() - - { - trusting := trust.TrustingPeer().EncodeToString() - - dt = x.mItems[trusting] - if dt == nil { - dt = &DaughterTrusts{ - mItems: make(map[string]reputation.Trust, 1), - } - - x.mItems[trusting] = dt - } - } - - x.mtx.Unlock() - - dt.put(trust) -} - -func (x *DaughterStorage) daughterTrusts(id apireputation.PeerID) (dt *DaughterTrusts, ok bool) { - x.mtx.RLock() - - { - dt, ok = x.mItems[id.EncodeToString()] - } - - x.mtx.RUnlock() - - return -} - -// DaughterTrusts represents in-memory storage of local trusts -// of the daughter peer to its providers. -// -// Maps IDs of daughter's providers to the local trusts to them. -type DaughterTrusts struct { - mtx sync.RWMutex - - mItems map[string]reputation.Trust -} - -func (x *DaughterTrusts) put(trust reputation.Trust) { - x.mtx.Lock() - - { - x.mItems[trust.Peer().EncodeToString()] = trust - } - - x.mtx.Unlock() -} - -// Iterate passes all stored trusts to h. -// -// Returns errors from h directly. -func (x *DaughterTrusts) Iterate(h reputation.TrustHandler) (err error) { - x.mtx.RLock() - - { - for _, trust := range x.mItems { - if err = h(trust); err != nil { - break - } - } - } - - x.mtx.RUnlock() - - return -} diff --git a/pkg/services/reputation/eigentrust/storage/daughters/storage.go b/pkg/services/reputation/eigentrust/storage/daughters/storage.go deleted file mode 100644 index 26399fce4..000000000 --- a/pkg/services/reputation/eigentrust/storage/daughters/storage.go +++ /dev/null @@ -1,38 +0,0 @@ -package daughters - -import "sync" - -// Prm groups the required parameters of the Storage's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -// -// The component is not parameterizable at the moment. -type Prm struct{} - -// Storage represents in-memory storage of local trust -// values of the daughter peers. -// -// It maps epoch numbers to the repositories of local trusts -// of the daughter peers. -// -// For correct operation, Storage must be created -// using the constructor (New) based on the required parameters -// and optional components. After successful creation, -// Storage is immediately ready to work through API. -type Storage struct { - mtx sync.RWMutex - - mItems map[uint64]*DaughterStorage -} - -// New creates a new instance of the Storage. -// -// The created Storage does not require additional -// initialization and is completely ready for work. -func New(_ Prm) *Storage { - return &Storage{ - mItems: make(map[uint64]*DaughterStorage), - } -} diff --git a/pkg/services/reputation/local/controller/calls.go b/pkg/services/reputation/local/controller/calls.go deleted file mode 100644 index 1cad09313..000000000 --- a/pkg/services/reputation/local/controller/calls.go +++ /dev/null @@ -1,193 +0,0 @@ -package trustcontroller - -import ( - "context" - "errors" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// ReportPrm groups the required parameters of the Controller.Report method. -type ReportPrm struct { - epoch uint64 -} - -// SetEpoch sets epoch number to select reputation values. -func (p *ReportPrm) SetEpoch(e uint64) { - p.epoch = e -} - -// Report reports local reputation values. -// -// Single Report operation overtakes all data from LocalTrustSource -// to LocalTrustTarget (Controller's parameters). -// -// Each call acquires a report context for an Epoch parameter. -// At the very end of the operation, the context is released. -func (c *Controller) Report(ctx context.Context, prm ReportPrm) { - // acquire report - rCtx, reporter := c.acquireReporter(ctx, prm.epoch) - if reporter == nil { - return - } - - // report local trust values - reporter.report(rCtx) - - // finally stop and free the report - c.freeReport(prm.epoch, reporter.log) -} - -type reporter struct { - epoch uint64 - - ctrl *Controller - - log *logger.Logger - - ep common.EpochProvider -} - -type epochProvider struct { - epoch uint64 -} - -func (c epochProvider) Epoch() uint64 { - return c.epoch -} - -func (c *Controller) acquireReporter(ctx context.Context, epoch uint64) (context.Context, *reporter) { - started := true - - c.mtx.Lock() - { - if cancel := c.mCtx[epoch]; cancel == nil { - ctx, cancel = context.WithCancel(ctx) - c.mCtx[epoch] = cancel - started = false - } - } - c.mtx.Unlock() - - log := &logger.Logger{Logger: c.opts.log.With( - zap.Uint64("epoch", epoch), - )} - - if started { - log.Debug(logs.ControllerReportIsAlreadyStarted) - return ctx, nil - } - - return ctx, &reporter{ - epoch: epoch, - ctrl: c, - log: log, - ep: &epochProvider{ - epoch: epoch, - }, - } -} - -func (c *reporter) report(ctx context.Context) { - c.log.Debug(logs.ControllerStartingToReportLocalTrustValues) - - // initialize iterator over locally collected values - iterator, err := c.ctrl.prm.LocalTrustSource.InitIterator(c.ep) - if err != nil { - c.log.Debug(logs.ControllerCouldNotInitializeIteratorOverLocalTrustValues, - zap.String("error", err.Error()), - ) - - return - } - - // initialize target of local trust values - targetWriter, err := c.ctrl.prm.LocalTrustTarget.InitWriter(c.ep) - if err != nil { - c.log.Debug(logs.ControllerCouldNotInitializeLocalTrustTarget, - zap.String("error", err.Error()), - ) - - return - } - - // iterate over all values and write them to the target - err = iterator.Iterate( - func(t reputation.Trust) error { - // check if context is done - if err := ctx.Err(); err != nil { - return err - } - - return targetWriter.Write(ctx, t) - }, - ) - if err != nil && !errors.Is(err, context.Canceled) { - c.log.Debug(logs.ControllerIteratorOverLocalTrustFailed, - zap.String("error", err.Error()), - ) - - return - } - - // finish writing - err = targetWriter.Close(ctx) - if err != nil { - c.log.Debug(logs.ControllerCouldNotFinishWritingLocalTrustValues, - zap.String("error", err.Error()), - ) - - return - } - - c.log.Debug(logs.ControllerReportingSuccessfullyFinished) -} - -func (c *Controller) freeReport(epoch uint64, log *logger.Logger) { - var stopped bool - - c.mtx.Lock() - - { - var cancel context.CancelFunc - - cancel, stopped = c.mCtx[epoch] - - if stopped { - cancel() - delete(c.mCtx, epoch) - } - } - - c.mtx.Unlock() - - if stopped { - log.Debug(logs.ControllerReportingSuccessfullyInterrupted) - } else { - log.Debug(logs.ControllerReportingIsNotStartedOrAlreadyInterrupted) - } -} - -// StopPrm groups the required parameters of the Controller.Stop method. -type StopPrm struct { - epoch uint64 -} - -// SetEpoch sets epoch number the processing of the values of which must be interrupted. -func (p *StopPrm) SetEpoch(e uint64) { - p.epoch = e -} - -// Stop interrupts the processing of local trust values. -// -// Releases acquired report context. -func (c *Controller) Stop(prm StopPrm) { - c.freeReport( - prm.epoch, - &logger.Logger{Logger: c.opts.log.With(zap.Uint64("epoch", prm.epoch))}, - ) -} diff --git a/pkg/services/reputation/local/controller/controller.go b/pkg/services/reputation/local/controller/controller.go deleted file mode 100644 index 373df36db..000000000 --- a/pkg/services/reputation/local/controller/controller.go +++ /dev/null @@ -1,84 +0,0 @@ -package trustcontroller - -import ( - "context" - "fmt" - "sync" - - reputationrouter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common/router" -) - -// Prm groups the required parameters of the Controller's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Iterator over the reputation values - // collected by the node locally. - // - // Must not be nil. - LocalTrustSource IteratorProvider - - // Place of recording the local values of - // trust to other nodes. - // - // Must not be nil. - LocalTrustTarget *reputationrouter.Router -} - -// Controller represents main handler for starting -// and interrupting the reporting local trust values. -// -// It binds the interfaces of the local value stores -// to the target storage points. Controller is abstracted -// from the internal storage device and the network location -// of the connecting components. At its core, it is a -// high-level start-stop trigger for reporting. -// -// For correct operation, the controller must be created -// using the constructor (New) based on the required parameters -// and optional components. After successful creation, -// the constructor is immediately ready to work through -// API of external control of calculations and data transfer. -type Controller struct { - prm Prm - - opts *options - - mtx sync.Mutex - mCtx map[uint64]context.CancelFunc -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the Controller. -// -// Panics if at least one value of the parameters is invalid. -// -// The created Controller does not require additional -// initialization and is completely ready for work. -func New(prm Prm, opts ...Option) *Controller { - switch { - case prm.LocalTrustSource == nil: - panicOnPrmValue("LocalTrustSource", prm.LocalTrustSource) - case prm.LocalTrustTarget == nil: - panicOnPrmValue("LocalTrustTarget", prm.LocalTrustTarget) - } - - o := defaultOpts() - - for _, opt := range opts { - opt(o) - } - - return &Controller{ - prm: prm, - opts: o, - mCtx: make(map[uint64]context.CancelFunc), - } -} diff --git a/pkg/services/reputation/local/controller/deps.go b/pkg/services/reputation/local/controller/deps.go deleted file mode 100644 index 6f4a29c99..000000000 --- a/pkg/services/reputation/local/controller/deps.go +++ /dev/null @@ -1,34 +0,0 @@ -package trustcontroller - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" -) - -// Iterator is a group of methods provided by entity -// which can iterate over a group of reputation.Trust values. -type Iterator interface { - // Iterate must start an iterator over all trust values. - // For each value should call a handler, the error - // of which should be directly returned from the method. - // - // Internal failures of the iterator are also signaled via - // an error. After a successful call to the last value - // handler, nil should be returned. - Iterate(reputation.TrustHandler) error -} - -// IteratorProvider is a group of methods provided -// by entity which generates iterators over -// reputation.Trust values. -type IteratorProvider interface { - // InitIterator should return an initialized Iterator - // that iterates over values from IteratorContext.Epoch() epoch. - // - // Initialization problems are reported via error. - // If no error was returned, then the Iterator must not be nil. - // - // Implementations can have different logic for different - // contexts, so specific ones may document their own behavior. - InitIterator(common.EpochProvider) (Iterator, error) -} diff --git a/pkg/services/reputation/local/controller/opts.go b/pkg/services/reputation/local/controller/opts.go deleted file mode 100644 index 385a4243b..000000000 --- a/pkg/services/reputation/local/controller/opts.go +++ /dev/null @@ -1,30 +0,0 @@ -package trustcontroller - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// Option sets an optional parameter of Controller. -type Option func(*options) - -type options struct { - log *logger.Logger -} - -func defaultOpts() *options { - return &options{ - log: &logger.Logger{Logger: zap.L()}, - } -} - -// WithLogger returns option to specify logging component. -// -// Ignores nil values. -func WithLogger(l *logger.Logger) Option { - return func(o *options) { - if l != nil { - o.log = l - } - } -} diff --git a/pkg/services/reputation/local/controller/util.go b/pkg/services/reputation/local/controller/util.go deleted file mode 100644 index 122550498..000000000 --- a/pkg/services/reputation/local/controller/util.go +++ /dev/null @@ -1,32 +0,0 @@ -package trustcontroller - -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - -type storageWrapper struct { - w common.Writer - i Iterator -} - -func (s storageWrapper) InitIterator(common.EpochProvider) (Iterator, error) { - return s.i, nil -} - -func (s storageWrapper) InitWriter(common.EpochProvider) (common.Writer, error) { - return s.w, nil -} - -// SimpleIteratorProvider returns IteratorProvider that provides -// static context-independent Iterator. -func SimpleIteratorProvider(i Iterator) IteratorProvider { - return &storageWrapper{ - i: i, - } -} - -// SimpleWriterProvider returns WriterProvider that provides -// static context-independent Writer. -func SimpleWriterProvider(w common.Writer) common.WriterProvider { - return &storageWrapper{ - w: w, - } -} diff --git a/pkg/services/reputation/local/routes/builder.go b/pkg/services/reputation/local/routes/builder.go deleted file mode 100644 index ddd5a2ae0..000000000 --- a/pkg/services/reputation/local/routes/builder.go +++ /dev/null @@ -1,59 +0,0 @@ -package routes - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" -) - -// Prm groups the required parameters of the Builder's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Manager builder for current node. - // - // Must not be nil. - ManagerBuilder common.ManagerBuilder - - Log *logger.Logger -} - -// Builder represents component that routes node to its managers. -// -// For correct operation, Builder must be created using -// the constructor (New) based on the required parameters -// and optional components. After successful creation, -// the Builder is immediately ready to work through API. -type Builder struct { - managerBuilder common.ManagerBuilder - log *logger.Logger -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the Builder. -// -// Panics if at least one value of the parameters is invalid. -// -// The created Builder does not require additional -// initialization and is completely ready for work. -func New(prm Prm) *Builder { - switch { - case prm.ManagerBuilder == nil: - panicOnPrmValue("ManagerBuilder", prm.ManagerBuilder) - case prm.Log == nil: - panicOnPrmValue("Logger", prm.Log) - } - - return &Builder{ - managerBuilder: prm.ManagerBuilder, - log: prm.Log, - } -} diff --git a/pkg/services/reputation/local/routes/calls.go b/pkg/services/reputation/local/routes/calls.go deleted file mode 100644 index 2f99f0e10..000000000 --- a/pkg/services/reputation/local/routes/calls.go +++ /dev/null @@ -1,33 +0,0 @@ -package routes - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation/common" - "go.uber.org/zap" -) - -// NextStage builds Manager list for trusting node and returns it directly. -// -// If passed route has more than one point, then endpoint of the route is reached. -func (b *Builder) NextStage(epoch uint64, t reputation.Trust, passed []common.ServerInfo) ([]common.ServerInfo, error) { - passedLen := len(passed) - - b.log.Debug(logs.RoutesBuildingNextStageForLocalTrustRoute, - zap.Uint64("epoch", epoch), - zap.Int("passed_length", passedLen), - ) - - if passedLen > 1 { - return nil, nil - } - - route, err := b.managerBuilder.BuildManagers(epoch, t.TrustingPeer()) - if err != nil { - return nil, fmt.Errorf("could not build managers for epoch: %d: %w", epoch, err) - } - - return route, nil -} diff --git a/pkg/services/reputation/local/storage/calls.go b/pkg/services/reputation/local/storage/calls.go deleted file mode 100644 index 14acbb64f..000000000 --- a/pkg/services/reputation/local/storage/calls.go +++ /dev/null @@ -1,175 +0,0 @@ -package truststorage - -import ( - "errors" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/reputation" - apireputation "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" -) - -// UpdatePrm groups the parameters of Storage's Update operation. -type UpdatePrm struct { - sat bool - - epoch uint64 - - peer apireputation.PeerID -} - -// SetEpoch sets number of the epoch -// when the interaction happened. -func (p *UpdatePrm) SetEpoch(e uint64) { - p.epoch = e -} - -// SetPeer sets identifier of the peer -// with which the local node interacted. -func (p *UpdatePrm) SetPeer(id apireputation.PeerID) { - p.peer = id -} - -// SetSatisfactory sets successful completion status. -func (p *UpdatePrm) SetSatisfactory(sat bool) { - p.sat = sat -} - -type trustValue struct { - sat, all int -} - -// EpochTrustValueStorage represents storage of -// the trust values by particular epoch. -type EpochTrustValueStorage struct { - mtx sync.RWMutex - - mItems map[string]*trustValue -} - -func newTrustValueStorage() *EpochTrustValueStorage { - return &EpochTrustValueStorage{ - mItems: make(map[string]*trustValue, 1), - } -} - -func stringifyPeerID(id apireputation.PeerID) string { - return string(id.PublicKey()) -} - -func peerIDFromString(str string) (res apireputation.PeerID) { - res.SetPublicKey([]byte(str)) - return -} - -func (s *EpochTrustValueStorage) update(prm UpdatePrm) { - s.mtx.Lock() - - { - strID := stringifyPeerID(prm.peer) - - val, ok := s.mItems[strID] - if !ok { - val = new(trustValue) - s.mItems[strID] = val - } - - if prm.sat { - val.sat++ - } - - val.all++ - } - - s.mtx.Unlock() -} - -// Update updates the number of satisfactory transactions with peer. -func (s *Storage) Update(prm UpdatePrm) { - var trustStorage *EpochTrustValueStorage - - s.mtx.Lock() - - { - var ( - ok bool - epoch = prm.epoch - ) - - trustStorage, ok = s.mItems[epoch] - if !ok { - trustStorage = newTrustValueStorage() - s.mItems[epoch] = trustStorage - } - } - - s.mtx.Unlock() - - trustStorage.update(prm) -} - -// ErrNoPositiveTrust is returned by iterator when -// there is no positive number of successful transactions. -var ErrNoPositiveTrust = errors.New("no positive trust") - -// DataForEpoch returns EpochValueStorage for epoch. -// -// If there is no data for the epoch, ErrNoPositiveTrust returns. -func (s *Storage) DataForEpoch(epoch uint64) (*EpochTrustValueStorage, error) { - s.mtx.RLock() - trustStorage, ok := s.mItems[epoch] - s.mtx.RUnlock() - - if !ok { - return nil, ErrNoPositiveTrust - } - - return trustStorage, nil -} - -// Iterate iterates over normalized trust values and passes them to parameterized handler. -// -// Values are normalized according to http://ilpubs.stanford.edu:8090/562/1/2002-56.pdf Chapter 4.5. -// If divisor in formula is zero, ErrNoPositiveTrust returns. -func (s *EpochTrustValueStorage) Iterate(h reputation.TrustHandler) (err error) { - s.mtx.RLock() - - { - var ( - sum reputation.TrustValue - mVals = make(map[string]reputation.TrustValue, len(s.mItems)) - ) - - // iterate first time to calculate normalizing divisor - for strID, val := range s.mItems { - if val.all > 0 { - num := reputation.TrustValueFromInt(val.sat) - denom := reputation.TrustValueFromInt(val.all) - - v := num.Div(denom) - - mVals[strID] = v - - sum.Add(v) - } - } - - err = ErrNoPositiveTrust - - if !sum.IsZero() { - for strID, val := range mVals { - t := reputation.Trust{} - - t.SetPeer(peerIDFromString(strID)) - t.SetValue(val.Div(sum)) - - if err = h(t); err != nil { - break - } - } - } - } - - s.mtx.RUnlock() - - return -} diff --git a/pkg/services/reputation/local/storage/storage.go b/pkg/services/reputation/local/storage/storage.go deleted file mode 100644 index d7e54a3fc..000000000 --- a/pkg/services/reputation/local/storage/storage.go +++ /dev/null @@ -1,41 +0,0 @@ -package truststorage - -import ( - "sync" -) - -// Prm groups the required parameters of the Storage's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct{} - -// Storage represents in-memory storage of -// local reputation values. -// -// Storage provides access to normalized local trust -// values through iterator interface. -// -// For correct operation, Storage must be created -// using the constructor (New) based on the required parameters -// and optional components. After successful creation, -// Storage is immediately ready to work through API. -type Storage struct { - prm Prm - - mtx sync.RWMutex - - mItems map[uint64]*EpochTrustValueStorage -} - -// New creates a new instance of the Storage. -// -// The created Storage does not require additional -// initialization and is completely ready for work. -func New(prm Prm) *Storage { - return &Storage{ - prm: prm, - mItems: make(map[uint64]*EpochTrustValueStorage), - } -} diff --git a/pkg/services/reputation/rpc/response.go b/pkg/services/reputation/rpc/response.go deleted file mode 100644 index 808a0a476..000000000 --- a/pkg/services/reputation/rpc/response.go +++ /dev/null @@ -1,50 +0,0 @@ -package reputationrpc - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns reputation service server instance that passes -// internal service call to response service. -func NewResponseService(cnrSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: cnrSvc, - } -} - -func (s *responseService) AnnounceLocalTrust(ctx context.Context, req *reputation.AnnounceLocalTrustRequest) (*reputation.AnnounceLocalTrustResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.AnnounceLocalTrust(ctx, req.(*reputation.AnnounceLocalTrustRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*reputation.AnnounceLocalTrustResponse), nil -} - -func (s *responseService) AnnounceIntermediateResult(ctx context.Context, req *reputation.AnnounceIntermediateResultRequest) (*reputation.AnnounceIntermediateResultResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.AnnounceIntermediateResult(ctx, req.(*reputation.AnnounceIntermediateResultRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*reputation.AnnounceIntermediateResultResponse), nil -} diff --git a/pkg/services/reputation/rpc/server.go b/pkg/services/reputation/rpc/server.go deleted file mode 100644 index 78af30ea7..000000000 --- a/pkg/services/reputation/rpc/server.go +++ /dev/null @@ -1,13 +0,0 @@ -package reputationrpc - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/reputation" -) - -// Server is an interface of the FrostFS API v2 Reputation service server. -type Server interface { - AnnounceLocalTrust(context.Context, *reputation.AnnounceLocalTrustRequest) (*reputation.AnnounceLocalTrustResponse, error) - AnnounceIntermediateResult(context.Context, *reputation.AnnounceIntermediateResultRequest) (*reputation.AnnounceIntermediateResultResponse, error) -} diff --git a/pkg/services/reputation/rpc/sign.go b/pkg/services/reputation/rpc/sign.go deleted file mode 100644 index 9db06ff1e..000000000 --- a/pkg/services/reputation/rpc/sign.go +++ /dev/null @@ -1,54 +0,0 @@ -package reputationrpc - -import ( - "context" - "crypto/ecdsa" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/reputation" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" -) - -type signService struct { - sigSvc *util.SignService - - svc Server -} - -func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { - return &signService{ - sigSvc: util.NewUnarySignService(key), - svc: svc, - } -} - -func (s *signService) AnnounceLocalTrust(ctx context.Context, req *reputation.AnnounceLocalTrustRequest) (*reputation.AnnounceLocalTrustResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.AnnounceLocalTrust(ctx, req.(*reputation.AnnounceLocalTrustRequest)) - }, - func() util.ResponseMessage { - return new(reputation.AnnounceLocalTrustResponse) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*reputation.AnnounceLocalTrustResponse), nil -} - -func (s *signService) AnnounceIntermediateResult(ctx context.Context, req *reputation.AnnounceIntermediateResultRequest) (*reputation.AnnounceIntermediateResultResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.AnnounceIntermediateResult(ctx, req.(*reputation.AnnounceIntermediateResultRequest)) - }, - func() util.ResponseMessage { - return new(reputation.AnnounceIntermediateResultResponse) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*reputation.AnnounceIntermediateResultResponse), nil -} diff --git a/pkg/services/reputation/trust.go b/pkg/services/reputation/trust.go deleted file mode 100644 index 8c5d9091a..000000000 --- a/pkg/services/reputation/trust.go +++ /dev/null @@ -1,102 +0,0 @@ -package reputation - -import ( - "strconv" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" -) - -// TrustValue represents the numeric value of the node's trust. -type TrustValue float64 - -const ( - // TrustOne is a trust value equal to one. - TrustOne = TrustValue(1) - - // TrustZero is a trust value equal to zero. - TrustZero = TrustValue(0) -) - -// TrustValueFromFloat64 converts float64 to TrustValue. -func TrustValueFromFloat64(v float64) TrustValue { - return TrustValue(v) -} - -// TrustValueFromInt converts int to TrustValue. -func TrustValueFromInt(v int) TrustValue { - return TrustValue(v) -} - -func (v TrustValue) String() string { - return strconv.FormatFloat(float64(v), 'f', -1, 64) -} - -// Float64 converts TrustValue to float64. -func (v TrustValue) Float64() float64 { - return float64(v) -} - -// Add adds v2 to v. -func (v *TrustValue) Add(v2 TrustValue) { - *v = *v + v2 -} - -// Div returns the result of dividing v by v2. -func (v TrustValue) Div(v2 TrustValue) TrustValue { - return v / v2 -} - -// Mul multiplies v by v2. -func (v *TrustValue) Mul(v2 TrustValue) { - *v *= v2 -} - -// IsZero returns true if v equal to zero. -func (v TrustValue) IsZero() bool { - return v == 0 -} - -// Trust represents peer's trust (reputation). -type Trust struct { - trusting, peer reputation.PeerID - - val TrustValue -} - -// TrustHandler describes the signature of the reputation.Trust -// value handling function. -// -// Termination of processing without failures is usually signaled -// with a zero error, while a specific value may describe the reason -// for failure. -type TrustHandler func(Trust) error - -// Value returns peer's trust value. -func (t Trust) Value() TrustValue { - return t.val -} - -// SetValue sets peer's trust value. -func (t *Trust) SetValue(val TrustValue) { - t.val = val -} - -// Peer returns trusted peer ID. -func (t Trust) Peer() reputation.PeerID { - return t.peer -} - -// SetPeer sets trusted peer ID. -func (t *Trust) SetPeer(id reputation.PeerID) { - t.peer = id -} - -// TrustingPeer returns trusting peer ID. -func (t Trust) TrustingPeer() reputation.PeerID { - return t.trusting -} - -// SetTrustingPeer sets trusting peer ID. -func (t *Trust) SetTrustingPeer(id reputation.PeerID) { - t.trusting = id -} From 960e3c219e99254288d73b60404015b1bbe778a8 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 14 Apr 2023 13:41:40 +0300 Subject: [PATCH 0114/1943] [#247] config, doc: Drop reputation references Signed-off-by: Pavel Karpy --- cmd/frostfs-adm/docs/deploy.md | 1 - .../internal/modules/morph/initialize_deploy.go | 5 ----- cmd/frostfs-ir/defaults.go | 1 - cmd/frostfs-node/config/contracts/config.go | 9 --------- cmd/frostfs-node/config/contracts/config_test.go | 6 ------ config/example/ir.env | 2 -- config/example/ir.yaml | 2 -- config/example/node.env | 1 - config/example/node.json | 1 - config/example/node.yaml | 1 - config/mainnet/config.yml | 1 - config/testnet/config.yml | 1 - docs/storage-node-configuration.md | 2 -- pkg/innerring/contracts.go | 2 -- 14 files changed, 35 deletions(-) diff --git a/cmd/frostfs-adm/docs/deploy.md b/cmd/frostfs-adm/docs/deploy.md index a1923bb2c..aead65fe0 100644 --- a/cmd/frostfs-adm/docs/deploy.md +++ b/cmd/frostfs-adm/docs/deploy.md @@ -147,7 +147,6 @@ NNS: Set container.frostfs -> cae60bdd689d185901e495352d0247752ce50846 NNS: Set frostfsid.frostfs -> c421fb60a3895865a8f24d197d6a80ef686041d2 NNS: Set netmap.frostfs -> 894eb854632f50fb124412ce7951ebc00763525e NNS: Set proxy.frostfs -> ac6e6fe4b373d0ca0ca4969d1e58fa0988724e7d -NNS: Set reputation.frostfs -> 6eda57c9d93d990573646762d1fea327ce41191f Waiting for transactions to persist... ``` diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 9e473463b..156b1f788 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -47,7 +47,6 @@ const ( frostfsIDContract = "frostfsid" netmapContract = "netmap" proxyContract = "proxy" - reputationContract = "reputation" subnetContract = "subnet" ) @@ -64,7 +63,6 @@ var ( frostfsIDContract, netmapContract, proxyContract, - reputationContract, subnetContract, } @@ -81,8 +79,6 @@ var ( netmap.AuditFeeConfig, netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, - netmap.EtIterationsConfig, - netmap.EtAlphaConfig, netmap.BasicIncomeRateConfig, netmap.IrCandidateFeeConfig, netmap.WithdrawFeeConfig, @@ -602,7 +598,6 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an configParam) case proxyContract: items = nil - case reputationContract: case subnetContract: default: panic(fmt.Sprintf("invalid contract name: %s", ctrName)) diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index 57959c1cf..837c1f170 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -163,7 +163,6 @@ func setContractsDefaults(cfg *viper.Viper) { cfg.SetDefault("contracts.audit", "") cfg.SetDefault("contracts.proxy", "") cfg.SetDefault("contracts.processing", "") - cfg.SetDefault("contracts.reputation", "") cfg.SetDefault("contracts.subnet", "") cfg.SetDefault("contracts.proxy", "") } diff --git a/cmd/frostfs-node/config/contracts/config.go b/cmd/frostfs-node/config/contracts/config.go index 0450d0d57..c5f14f3ca 100644 --- a/cmd/frostfs-node/config/contracts/config.go +++ b/cmd/frostfs-node/config/contracts/config.go @@ -38,15 +38,6 @@ func Container(c *config.Config) util.Uint160 { return contractAddress(c, "container") } -// Reputation returnsthe value of "reputation" config parameter -// from "contracts" section. -// -// Returns zero filled script hash if the value is not set. -// Throws panic if the value is not a 20-byte LE hex-encoded string. -func Reputation(c *config.Config) util.Uint160 { - return contractAddress(c, "reputation") -} - // Proxy returnsthe value of "proxy" config parameter // from "contracts" section. // diff --git a/cmd/frostfs-node/config/contracts/config_test.go b/cmd/frostfs-node/config/contracts/config_test.go index 93956d6bb..d816ea1e4 100644 --- a/cmd/frostfs-node/config/contracts/config_test.go +++ b/cmd/frostfs-node/config/contracts/config_test.go @@ -18,7 +18,6 @@ func TestContractsSection(t *testing.T) { require.Equal(t, emptyHash, contractsconfig.Balance(empty)) require.Equal(t, emptyHash, contractsconfig.Container(empty)) require.Equal(t, emptyHash, contractsconfig.Netmap(empty)) - require.Equal(t, emptyHash, contractsconfig.Reputation(empty)) require.Equal(t, emptyHash, contractsconfig.Proxy(empty)) }) @@ -33,9 +32,6 @@ func TestContractsSection(t *testing.T) { expNetmap, err := util.Uint160DecodeStringLE("0cce9e948dca43a6b592efe59ddb4ecb89bdd9ca") require.NoError(t, err) - expReputation, err := util.Uint160DecodeStringLE("441995f631c1da2b133462b71859494a5cd45e90") - require.NoError(t, err) - expProxy, err := util.Uint160DecodeStringLE("ad7c6b55b737b696e5c82c85445040964a03e97f") require.NoError(t, err) @@ -43,13 +39,11 @@ func TestContractsSection(t *testing.T) { balance := contractsconfig.Balance(c) container := contractsconfig.Container(c) netmap := contractsconfig.Netmap(c) - reputation := contractsconfig.Reputation(c) proxy := contractsconfig.Proxy(c) require.Equal(t, expBalance, balance) require.Equal(t, expConatiner, container) require.Equal(t, expNetmap, netmap) - require.Equal(t, expReputation, reputation) require.Equal(t, expProxy, proxy) } diff --git a/config/example/ir.env b/config/example/ir.env index 2ec821932..b7816dd77 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -49,7 +49,6 @@ FROSTFS_IR_WORKERS_BALANCE=10 FROSTFS_IR_WORKERS_CONTAINER=10 FROSTFS_IR_WORKERS_NEOFS=10 FROSTFS_IR_WORKERS_NETMAP=10 -FROSTFS_IR_WORKERS_REPUTATION=10 FROSTFS_IR_WORKERS_SUBNET=10 FROSTFS_IR_AUDIT_TIMEOUT_GET=5s @@ -75,7 +74,6 @@ FROSTFS_IR_CONTRACTS_CONTAINER=ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6 FROSTFS_IR_CONTRACTS_NEOFSID=9f5866decbc751a099e74c7c7bc89f609201755a FROSTFS_IR_CONTRACTS_NETMAP=83c600c81d47a1b1b7cf58eb49ae7ee7240dc742 FROSTFS_IR_CONTRACTS_PROXY=abc8794bb40a21f2db5f21ae62741eb46c8cad1c -FROSTFS_IR_CONTRACTS_REPUTATION=d793b842ff0c103fe89e385069e82a27602135ff FROSTFS_IR_CONTRACTS_SUBNET=e9266864d3c562c6e17f2bb9cb1392aaa293d93a FROSTFS_IR_CONTRACTS_ALPHABET_AMOUNT=7 FROSTFS_IR_CONTRACTS_ALPHABET_AZ=c1d211fceeb4b1dc76b8e4054d11fdf887e418ea diff --git a/config/example/ir.yaml b/config/example/ir.yaml index 3dca0017a..0e1b12bf6 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -83,7 +83,6 @@ workers: container: 10 # Number of workers to process events from container contract in parallel frostfs: 10 # Number of workers to process events from frostfs contracts in parallel netmap: 10 # Number of workers to process events from netmap contract in parallel - reputation: 10 # Number of workers to process events from reputation contract in parallel subnet: 10 # Number of workers to process events from subnet contract in parallel audit: @@ -117,7 +116,6 @@ contracts: frostfsid: 9f5866decbc751a099e74c7c7bc89f609201755a # Optional: override address of frostfsid contract in sidechain netmap: 83c600c81d47a1b1b7cf58eb49ae7ee7240dc742 # Optional: override address of netmap contract in sidechain proxy: abc8794bb40a21f2db5f21ae62741eb46c8cad1c # Optional: override address of proxy contract in sidechain; ignore if notary is disabled in sidechain - reputation: d793b842ff0c103fe89e385069e82a27602135ff # Optional: override address of reputation contract in sidechain subnet: e9266864d3c562c6e17f2bb9cb1392aaa293d93a # Optional: override address of subnet contract in sidechain alphabet: amount: 7 # Optional: override amount of alphabet contracts diff --git a/config/example/node.env b/config/example/node.env index a4088f75a..b3fb4f9f9 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -58,7 +58,6 @@ FROSTFS_CONTROL_GRPC_ENDPOINT=localhost:8090 FROSTFS_CONTRACTS_BALANCE=5263abba1abedbf79bb57f3e40b50b4425d2d6cd FROSTFS_CONTRACTS_CONTAINER=5d084790d7aa36cea7b53fe897380dab11d2cd3c FROSTFS_CONTRACTS_NETMAP=0cce9e948dca43a6b592efe59ddb4ecb89bdd9ca -FROSTFS_CONTRACTS_REPUTATION=441995f631c1da2b133462b71859494a5cd45e90 FROSTFS_CONTRACTS_PROXY=ad7c6b55b737b696e5c82c85445040964a03e97f # Morph chain section diff --git a/config/example/node.json b/config/example/node.json index 8cfb5bb69..9e2a0a487 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -96,7 +96,6 @@ "balance": "5263abba1abedbf79bb57f3e40b50b4425d2d6cd", "container": "5d084790d7aa36cea7b53fe897380dab11d2cd3c", "netmap": "0cce9e948dca43a6b592efe59ddb4ecb89bdd9ca", - "reputation": "441995f631c1da2b133462b71859494a5cd45e90", "proxy": "ad7c6b55b737b696e5c82c85445040964a03e97f" }, "morph": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 0d71f0fd2..54d774174 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -78,7 +78,6 @@ contracts: # side chain NEOFS contract script hashes; optional, override values balance: 5263abba1abedbf79bb57f3e40b50b4425d2d6cd container: 5d084790d7aa36cea7b53fe897380dab11d2cd3c netmap: 0cce9e948dca43a6b592efe59ddb4ecb89bdd9ca - reputation: 441995f631c1da2b133462b71859494a5cd45e90 proxy: ad7c6b55b737b696e5c82c85445040964a03e97f morph: diff --git a/config/mainnet/config.yml b/config/mainnet/config.yml index 2b240e0b2..7db476e55 100644 --- a/config/mainnet/config.yml +++ b/config/mainnet/config.yml @@ -68,4 +68,3 @@ contracts: balance: dc1ec98d9d0c5f9dfade16144defe08cffc5ca55 container: 1b6e68d299b570e1cb7e86eadfdc06aa2e8e0cc5 netmap: 7c5bdb23e36cc7cce95bf42f3ab9e452c2501df1 - reputation: 7ad824fd1eeb1565be2cee3889214b9aa605d2fc diff --git a/config/testnet/config.yml b/config/testnet/config.yml index 416ba35e4..76b36cdf6 100644 --- a/config/testnet/config.yml +++ b/config/testnet/config.yml @@ -16,7 +16,6 @@ contracts: balance: e0420c216003747626670d1424569c17c79015bf container: 9dbd2b5e67568ed285c3d6f96bac4edf5e1efba0 netmap: d4b331639799e2958d4bc5b711b469d79de94e01 - reputation: 376c23a2ae1fad088c82046abb59984e3c4519d9 node: key: /node.key diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 6ecd15907..306577307 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -113,7 +113,6 @@ contracts: balance: 5263abba1abedbf79bb57f3e40b50b4425d2d6cd container: 5d084790d7aa36cea7b53fe897380dab11d2cd3c netmap: 0cce9e948dca43a6b592efe59ddb4ecb89bdd9ca - reputation: 441995f631c1da2b133462b71859494a5cd45e90 proxy: ad7c6b55b737b696e5c82c85445040964a03e97f ``` @@ -123,7 +122,6 @@ contracts: | `balance` | `hash160` | | Balance contract hash. | | `container` | `hash160` | | Container contract hash. | | `netmap` | `hash160` | | Netmap contract hash. | -| `reputation` | `hash160` | | Reputation contract hash. | | `subnet` | `hash160` | | Subnet contract hash. | # `morph` section diff --git a/pkg/innerring/contracts.go b/pkg/innerring/contracts.go index c280eb4bf..f723d3507 100644 --- a/pkg/innerring/contracts.go +++ b/pkg/innerring/contracts.go @@ -18,7 +18,6 @@ type contracts struct { audit util.Uint160 // in morph proxy util.Uint160 // in morph processing util.Uint160 // in mainnet - reputation util.Uint160 // in morph subnet util.Uint160 // in morph frostfsID util.Uint160 // in morph @@ -61,7 +60,6 @@ func parseContracts(cfg *viper.Viper, morph *client.Client, withoutMainNet, with {"contracts.balance", client.NNSBalanceContractName, &result.balance}, {"contracts.container", client.NNSContainerContractName, &result.container}, {"contracts.audit", client.NNSAuditContractName, &result.audit}, - {"contracts.reputation", client.NNSReputationContractName, &result.reputation}, {"contracts.subnet", client.NNSSubnetworkContractName, &result.subnet}, {"contracts.frostfsid", client.NNSFrostFSIDContractName, &result.frostfsID}, } From 8799138fcb67f21d2cc41749463ad6146bbedacd Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 14 Apr 2023 14:05:31 +0300 Subject: [PATCH 0115/1943] [#247] morph: Drop reputation contract Signed-off-by: Pavel Karpy --- pkg/morph/client/nns.go | 2 - pkg/morph/client/reputation/client.go | 83 ----------------- pkg/morph/client/reputation/get.go | 108 ----------------------- pkg/morph/client/reputation/list.go | 55 ------------ pkg/morph/client/reputation/put.go | 47 ---------- pkg/morph/event/reputation/put.go | 99 --------------------- pkg/morph/event/reputation/put_notary.go | 74 ---------------- pkg/morph/event/reputation/put_test.go | 92 ------------------- 8 files changed, 560 deletions(-) delete mode 100644 pkg/morph/client/reputation/client.go delete mode 100644 pkg/morph/client/reputation/get.go delete mode 100644 pkg/morph/client/reputation/list.go delete mode 100644 pkg/morph/client/reputation/put.go delete mode 100644 pkg/morph/event/reputation/put.go delete mode 100644 pkg/morph/event/reputation/put_notary.go delete mode 100644 pkg/morph/event/reputation/put_test.go diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index 473b3500b..236cf1ba0 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -32,8 +32,6 @@ const ( NNSNetmapContractName = "netmap.frostfs" // NNSProxyContractName is a name of the proxy contract in NNS. NNSProxyContractName = "proxy.frostfs" - // NNSReputationContractName is a name of the reputation contract in NNS. - NNSReputationContractName = "reputation.frostfs" // NNSSubnetworkContractName is a name of the subnet contract in NNS. NNSSubnetworkContractName = "subnet.frostfs" // NNSGroupKeyName is a name for the FrostFS group key record in NNS. diff --git a/pkg/morph/client/reputation/client.go b/pkg/morph/client/reputation/client.go deleted file mode 100644 index cdaf191ad..000000000 --- a/pkg/morph/client/reputation/client.go +++ /dev/null @@ -1,83 +0,0 @@ -package reputation - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - "github.com/nspcc-dev/neo-go/pkg/util" -) - -// Client is a wrapper over StaticClient -// which makes calls with the names and arguments -// of the FrostFS reputation contract. -// -// Working client must be created via constructor New. -// Using the Client that has been created with new(Client) -// expression (or just declaring a Client variable) is unsafe -// and can lead to panic. -type Client struct { - client *client.StaticClient // static reputation contract client -} - -const ( - putMethod = "put" - getMethod = "get" - getByIDMethod = "getByID" - listByEpochMethod = "listByEpoch" -) - -// NewFromMorph returns the wrapper instance from the raw morph client. -func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8, opts ...Option) (*Client, error) { - o := defaultOpts() - - for i := range opts { - opts[i](o) - } - - sc, err := client.NewStatic(cli, contract, fee, ([]client.StaticClientOption)(*o)...) - if err != nil { - return nil, fmt.Errorf("could not create static client of reputation contract: %w", err) - } - - return &Client{client: sc}, nil -} - -// Morph returns raw morph client. -func (c Client) Morph() *client.Client { - return c.client.Morph() -} - -// ContractAddress returns the address of the associated contract. -func (c Client) ContractAddress() util.Uint160 { - return c.client.ContractAddress() -} - -// Option allows to set an optional -// parameter of ClientWrapper. -type Option func(*opts) - -type opts []client.StaticClientOption - -func defaultOpts() *opts { - return new(opts) -} - -// TryNotary returns option to enable -// notary invocation tries. -func TryNotary() Option { - return func(o *opts) { - *o = append(*o, client.TryNotary()) - } -} - -// AsAlphabet returns option to sign main TX -// of notary requests with client's private -// key. -// -// Considered to be used by IR nodes only. -func AsAlphabet() Option { - return func(o *opts) { - *o = append(*o, client.AsAlphabet()) - } -} diff --git a/pkg/morph/client/reputation/get.go b/pkg/morph/client/reputation/get.go deleted file mode 100644 index 8f1d24176..000000000 --- a/pkg/morph/client/reputation/get.go +++ /dev/null @@ -1,108 +0,0 @@ -package reputation - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" -) - -type ( - // GetPrm groups the arguments of "get reputation value" test invocation. - GetPrm struct { - epoch uint64 - peerID reputation.PeerID - } - - // GetByIDPrm groups the arguments of "get reputation value by - // reputation id" test invocation. - GetByIDPrm struct { - id ID - } -) - -// SetEpoch sets epoch of expected reputation value. -func (g *GetPrm) SetEpoch(v uint64) { - g.epoch = v -} - -// SetPeerID sets peer id of expected reputation value. -func (g *GetPrm) SetPeerID(v reputation.PeerID) { - g.peerID = v -} - -// SetID sets id of expected reputation value in reputation contract. -func (g *GetByIDPrm) SetID(v ID) { - g.id = v -} - -// Get invokes the call of "get reputation value" method of reputation contract. -func (c *Client) Get(p GetPrm) ([]reputation.GlobalTrust, error) { - invokePrm := client.TestInvokePrm{} - invokePrm.SetMethod(getMethod) - invokePrm.SetArgs(p.epoch, p.peerID.PublicKey()) - - res, err := c.client.TestInvoke(invokePrm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", getMethod, err) - } - - return parseReputations(res, getMethod) -} - -// GetByID invokes the call of "get reputation value by reputation id" method -// of reputation contract. -func (c *Client) GetByID(p GetByIDPrm) ([]reputation.GlobalTrust, error) { - invokePrm := client.TestInvokePrm{} - invokePrm.SetMethod(getByIDMethod) - invokePrm.SetArgs([]byte(p.id)) - - prms, err := c.client.TestInvoke(invokePrm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", getByIDMethod, err) - } - - return parseReputations(prms, getByIDMethod) -} - -func parseGetResult(rawReputations [][]byte, method string) ([]reputation.GlobalTrust, error) { - reputations := make([]reputation.GlobalTrust, 0, len(rawReputations)) - - for i := range rawReputations { - r := reputation.GlobalTrust{} - - err := r.Unmarshal(rawReputations[i]) - if err != nil { - return nil, fmt.Errorf("can't unmarshal global trust value (%s): %w", method, err) - } - - reputations = append(reputations, r) - } - - return reputations, nil -} - -func parseReputations(items []stackitem.Item, method string) ([]reputation.GlobalTrust, error) { - if ln := len(items); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", method, ln) - } - - items, err := client.ArrayFromStackItem(items[0]) - if err != nil { - return nil, fmt.Errorf("could not get stack item array from stack item (%s): %w", method, err) - } - - res := make([][]byte, 0, len(items)) - - for i := range items { - rawReputation, err := client.BytesFromStackItem(items[i]) - if err != nil { - return nil, fmt.Errorf("could not get byte array from stack item (%s): %w", method, err) - } - - res = append(res, rawReputation) - } - - return parseGetResult(res, method) -} diff --git a/pkg/morph/client/reputation/list.go b/pkg/morph/client/reputation/list.go deleted file mode 100644 index 0090efb10..000000000 --- a/pkg/morph/client/reputation/list.go +++ /dev/null @@ -1,55 +0,0 @@ -package reputation - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" -) - -type ( - // ID is an ID of the reputation record in reputation contract. - ID []byte - - // ListByEpochArgs groups the arguments of - // "list reputation ids by epoch" test invoke call. - ListByEpochArgs struct { - epoch uint64 - } -) - -// SetEpoch sets epoch of expected reputation ids. -func (l *ListByEpochArgs) SetEpoch(v uint64) { - l.epoch = v -} - -// ListByEpoch invokes the call of "list reputation ids by epoch" method of -// reputation contract. -func (c *Client) ListByEpoch(p ListByEpochArgs) ([]ID, error) { - invokePrm := client.TestInvokePrm{} - invokePrm.SetMethod(listByEpochMethod) - invokePrm.SetArgs(p.epoch) - - prms, err := c.client.TestInvoke(invokePrm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", listByEpochMethod, err) - } else if ln := len(prms); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", listByEpochMethod, ln) - } - - items, err := client.ArrayFromStackItem(prms[0]) - if err != nil { - return nil, fmt.Errorf("could not get stack item array from stack item (%s): %w", listByEpochMethod, err) - } - - result := make([]ID, 0, len(items)) - for i := range items { - rawReputation, err := client.BytesFromStackItem(items[i]) - if err != nil { - return nil, fmt.Errorf("could not get byte array from stack item (%s): %w", listByEpochMethod, err) - } - - result = append(result, rawReputation) - } - - return result, nil -} diff --git a/pkg/morph/client/reputation/put.go b/pkg/morph/client/reputation/put.go deleted file mode 100644 index 02b47defe..000000000 --- a/pkg/morph/client/reputation/put.go +++ /dev/null @@ -1,47 +0,0 @@ -package reputation - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" -) - -type ( - // PutPrm groups the arguments of "put reputation value" invocation call. - PutPrm struct { - epoch uint64 - peerID reputation.PeerID - value reputation.GlobalTrust - } -) - -// SetEpoch sets epoch of reputation value. -func (p *PutPrm) SetEpoch(v uint64) { - p.epoch = v -} - -// SetPeerID sets peer id of reputation value. -func (p *PutPrm) SetPeerID(v reputation.PeerID) { - p.peerID = v -} - -// SetValue sets reputation value. -func (p *PutPrm) SetValue(v reputation.GlobalTrust) { - p.value = v -} - -// Put invokes direct call of "put reputation value" method of reputation contract. -// -// If TryNotary is provided, calls notary contract. -func (c *Client) Put(p PutPrm) error { - prm := client.InvokePrm{} - prm.SetMethod(putMethod) - prm.SetArgs(p.epoch, p.peerID.PublicKey(), p.value.Marshal()) - - err := c.client.Invoke(prm) - if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", putMethod, err) - } - return nil -} diff --git a/pkg/morph/event/reputation/put.go b/pkg/morph/event/reputation/put.go deleted file mode 100644 index a182bf26c..000000000 --- a/pkg/morph/event/reputation/put.go +++ /dev/null @@ -1,99 +0,0 @@ -package reputation - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/network/payload" -) - -// Put structure of reputation.reputationPut notification from -// morph chain. -type Put struct { - epoch uint64 - peerID reputation.PeerID - value reputation.GlobalTrust - - // For notary notifications only. - // Contains raw transactions of notary request. - notaryRequest *payload.P2PNotaryRequest -} - -const peerIDLength = 33 // compressed public key - -// MorphEvent implements Neo:Morph Event interface. -func (Put) MorphEvent() {} - -// Epoch returns epoch value of reputation data. -func (p Put) Epoch() uint64 { - return p.epoch -} - -// PeerID returns peer id of reputation data. -func (p Put) PeerID() reputation.PeerID { - return p.peerID -} - -// Value returns reputation structure. -func (p Put) Value() reputation.GlobalTrust { - return p.value -} - -// NotaryRequest returns raw notary request if notification -// was received via notary service. Otherwise, returns nil. -func (p Put) NotaryRequest() *payload.P2PNotaryRequest { - return p.notaryRequest -} - -// ParsePut from notification into reputation event structure. -func ParsePut(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Put - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != 3 { - return nil, event.WrongNumberOfParameters(3, ln) - } - - // parse epoch number - epoch, err := client.IntFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get integer epoch number: %w", err) - } - - ev.epoch = uint64(epoch) - - // parse peer ID value - peerID, err := client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get peer ID value: %w", err) - } - - if ln := len(peerID); ln != peerIDLength { - return nil, fmt.Errorf("peer ID is %d byte long, expected %d", ln, peerIDLength) - } - - ev.peerID.SetPublicKey(peerID) - - // parse global trust value - rawValue, err := client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get global trust value: %w", err) - } - - err = ev.value.Unmarshal(rawValue) - if err != nil { - return nil, fmt.Errorf("could not parse global trust value: %w", err) - } - - return ev, nil -} diff --git a/pkg/morph/event/reputation/put_notary.go b/pkg/morph/event/reputation/put_notary.go deleted file mode 100644 index f3cd749fc..000000000 --- a/pkg/morph/event/reputation/put_notary.go +++ /dev/null @@ -1,74 +0,0 @@ -package reputation - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" -) - -func (p *Put) setEpoch(v uint64) { - p.epoch = v -} - -func (p *Put) setPeerID(v []byte) error { - if ln := len(v); ln != peerIDLength { - return fmt.Errorf("peer ID is %d byte long, expected %d", ln, peerIDLength) - } - - p.peerID.SetPublicKey(v) - - return nil -} - -func (p *Put) setValue(v []byte) error { - return p.value.Unmarshal(v) -} - -var fieldSetters = []func(*Put, []byte) error{ - // order on stack is reversed - (*Put).setValue, - (*Put).setPeerID, -} - -const ( - // PutNotaryEvent is method name for reputation put operations - // in `Reputation` contract. Is used as identifier for notary - // put reputation requests. - PutNotaryEvent = "put" -) - -// ParsePutNotary from NotaryEvent into reputation event structure. -func ParsePutNotary(ne event.NotaryEvent) (event.Event, error) { - var ev Put - - fieldNum := 0 - - for _, op := range ne.Params() { - switch fieldNum { - case 0, 1: - data, err := event.BytesFromOpcode(op) - if err != nil { - return nil, err - } - - err = fieldSetters[fieldNum](&ev, data) - if err != nil { - return nil, fmt.Errorf("can't parse field num %d: %w", fieldNum, err) - } - case 2: - n, err := event.IntFromOpcode(op) - if err != nil { - return nil, err - } - - ev.setEpoch(uint64(n)) - default: - return nil, event.UnexpectedArgNumErr(PutNotaryEvent) - } - fieldNum++ - } - - ev.notaryRequest = ne.Raw() - - return ev, nil -} diff --git a/pkg/morph/event/reputation/put_test.go b/pkg/morph/event/reputation/put_test.go deleted file mode 100644 index 46356b317..000000000 --- a/pkg/morph/event/reputation/put_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package reputation - -import ( - "math/big" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation" - reputationtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/reputation/test" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" -) - -func TestParsePut(t *testing.T) { - var ( - peerID = reputationtest.PeerID() - - value reputation.GlobalTrust - trust reputation.Trust - trustValue float64 = 0.64 - - epoch uint64 = 42 - ) - - trust.SetValue(trustValue) - trust.SetPeer(peerID) - - value.SetTrust(trust) - - rawValue := value.Marshal() - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParsePut(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(3, len(prms)).Error()) - }) - - t.Run("wrong epoch parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong peerID parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewBigInteger(new(big.Int).SetUint64(epoch)), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong value parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewBigInteger(new(big.Int).SetUint64(epoch)), - stackitem.NewByteArray(peerID.PublicKey()), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewBigInteger(new(big.Int).SetUint64(epoch)), - stackitem.NewByteArray(peerID.PublicKey()), - stackitem.NewByteArray(rawValue), - })) - require.NoError(t, err) - - require.Equal(t, Put{ - epoch: epoch, - peerID: peerID, - value: value, - }, ev) - }) -} - -func createNotifyEventFromItems(items []stackitem.Item) *state.ContainedNotificationEvent { - return &state.ContainedNotificationEvent{ - NotificationEvent: state.NotificationEvent{ - Item: stackitem.NewArray(items), - }, - } -} From b453bb754c3bbeed9694ae8f128e4bcd59ee7677 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 14 Apr 2023 14:12:39 +0300 Subject: [PATCH 0116/1943] [#247] logs: Drop reputation log messages Signed-off-by: Pavel Karpy --- internal/logs/logs.go | 56 +------------------------------------------ 1 file changed, 1 insertion(+), 55 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 062538747..a6488dcc9 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -95,38 +95,7 @@ const ( PersistentCouldNotGetSessionFromPersistentStorage = "could not get session from persistent storage" // Error in ../node/pkg/services/session/storage/persistent/storage.go PersistentCouldNotDeleteSToken = "could not delete token" // Error in ../node/pkg/services/session/storage/persistent/storage.go PersistentCouldNotCleanUpExpiredTokens = "could not clean up expired tokens" // Error in ../node/pkg/services/session/storage/persistent/storage.go - CommonStartBuildingManagers = "start building managers" // Debug in ../node/pkg/services/reputation/common/managers.go - ControllerReportIsAlreadyStarted = "report is already started" // Debug in ../node/pkg/services/reputation/local/controller/calls.go - ControllerStartingToReportLocalTrustValues = "starting to report local trust values" // Debug in ../node/pkg/services/reputation/local/controller/calls.go - ControllerCouldNotInitializeIteratorOverLocalTrustValues = "could not initialize iterator over local trust values" // Debug in ../node/pkg/services/reputation/local/controller/calls.go - ControllerCouldNotInitializeLocalTrustTarget = "could not initialize local trust target" // Debug in ../node/pkg/services/reputation/local/controller/calls.go - ControllerIteratorOverLocalTrustFailed = "iterator over local trust failed" // Debug in ../node/pkg/services/reputation/local/controller/calls.go - ControllerCouldNotFinishWritingLocalTrustValues = "could not finish writing local trust values" // Debug in ../node/pkg/services/reputation/local/controller/calls.go - ControllerReportingSuccessfullyFinished = "reporting successfully finished" // Debug in ../node/pkg/services/reputation/local/controller/calls.go - ControllerReportingSuccessfullyInterrupted = "reporting successfully interrupted" // Debug in ../node/pkg/services/reputation/local/controller/calls.go - ControllerReportingIsNotStartedOrAlreadyInterrupted = "reporting is not started or already interrupted" // Debug in ../node/pkg/services/reputation/local/controller/calls.go - RoutesBuildingNextStageForLocalTrustRoute = "building next stage for local trust route" // Debug in ../node/pkg/services/reputation/local/routes/calls.go - CalculatorFailedToGetAlphaParam = "failed to get alpha param" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorConsumersTrustIteratorsInitFailure = "consumers trust iterator's init failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorWorkerPoolSubmitFailure = "worker pool submit failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorIterateDaughtersConsumersFailed = "iterate daughter's consumers failed" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorGetInitialTrustFailure = "get initial trust failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorDaughterTrustIteratorsInitFailure = "daughter trust iterator's init failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorIterateOverDaughtersTrustsFailure = "iterate over daughter's trusts failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorInitWriterFailure = "init writer failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorWriteFinalResultFailure = "write final result failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorWriteValueFailure = "write value failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorIterateDaughterTrustsFailure = "iterate daughter trusts failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorCouldNotCloseWriter = "could not close writer" // Error in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorAllDaughtersTrustIteratorsInitFailure = "all daughters trust iterator's init failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - CalculatorIterateOverAllDaughtersFailure = "iterate over all daughters failure" // Debug in ../node/pkg/services/reputation/eigentrust/calculator/calls.go - ControllerCouldNotGetEigenTrustIterationNumber = "could not get EigenTrust iteration number" // Error in ../node/pkg/services/reputation/eigentrust/controller/calls.go - ControllerIterationSubmitFailure = "iteration submit failure" // Debug in ../node/pkg/services/reputation/eigentrust/controller/calls.go - RoutesBuildingNextStageForTrustRoute = "building next stage for trust route" // Debug in ../node/pkg/services/reputation/eigentrust/routes/calls.go - RouterCouldNotInitializeWriterProvider = "could not initialize writer provider" // Debug in ../node/pkg/services/reputation/common/router/calls.go - RouterCouldNotInitializeWriter = "could not initialize writer" // Debug in ../node/pkg/services/reputation/common/router/calls.go - RouterCouldNotWriteTheValue = "could not write the value" // Debug in ../node/pkg/services/reputation/common/router/calls.go - RouterCouldNotCloseRemoteServerWriter = "could not close remote server writer" // Debug in ../node/pkg/services/reputation/common/router/calls.go + ControllerReportIsAlreadyStarted = "report is already started" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go TombstoneCouldNotGetTheTombstoneTheSource = "tombstone getter: could not get the tombstone the source" // Warn in ../node/pkg/services/object_manager/tombstone/checker.go TombstoneCouldNotParseTombstoneExpirationEpoch = "tombstone getter: could not parse tombstone expiration epoch" // Warn in ../node/pkg/services/object_manager/tombstone/checker.go DeleteRequestIsNotRolledOverToTheContainer = "request is not rolled over to the container" // Debug in ../node/pkg/services/object/delete/container.go @@ -495,11 +464,6 @@ const ( NetmapCouldNotIterateOverSubnetworksOfTheNode = "could not iterate over subnetworks of the node" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapCouldNotInvokeNetmapUpdateState = "could not invoke netmap.UpdateState" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapCouldNotInvokeNetmapAddPeer = "could not invoke netmap.AddPeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go - ReputationReputationWorkerPoolDrained = "reputation worker pool drained" // Warn in ../node/pkg/innerring/processors/reputation/handlers.go - ReputationNonAlphabetModeIgnoreReputationPutNotification = "non alphabet mode, ignore reputation put notification" // Info in ../node/pkg/innerring/processors/reputation/process_put.go - ReputationIgnoreReputationValue = "ignore reputation value" // Info in ../node/pkg/innerring/processors/reputation/process_put.go - ReputationCantSendApprovalTxForReputationValue = "can't send approval tx for reputation value" // Warn in ../node/pkg/innerring/processors/reputation/process_put.go - ReputationReputationWorkerPool = "reputation worker pool" // Debug in ../node/pkg/innerring/processors/reputation/processor.go SettlementNonAlphabetModeIgnoreAuditPayments = "non alphabet mode, ignore audit payments" // Info in ../node/pkg/innerring/processors/settlement/calls.go SettlementNewAuditSettlementEvent = "new audit settlement event" // Info in ../node/pkg/innerring/processors/settlement/calls.go SettlementIgnoreGenesisEpoch = "ignore genesis epoch" // Debug in ../node/pkg/innerring/processors/settlement/calls.go @@ -556,10 +520,6 @@ const ( FrostFSNodeGRPCCannotShutdownGracefullyForcingStop = "gRPC cannot shutdown gracefully, forcing stop" // Info in ../node/cmd/frostfs-node/grpc.go FrostFSNodeGRPCServerStoppedSuccessfully = "gRPC server stopped successfully" // Info in ../node/cmd/frostfs-node/grpc.go FrostFSNodeWaitingForAllProcessesToStop = "waiting for all processes to stop" // Debug in ../node/cmd/frostfs-node/main.go - FrostFSNodeStartReportingReputationOnNewEpochEvent = "start reporting reputation on new epoch event" // Debug in ../node/cmd/frostfs-node/reputation.go - FrostFSNodeCouldNotFetchEpochDuration = "could not fetch epoch duration" // Debug in ../node/cmd/frostfs-node/reputation.go - FrostFSNodeCouldNotFetchIterationNumber = "could not fetch iteration number" // Debug in ../node/cmd/frostfs-node/reputation.go - FrostFSNodeCouldNotCreateFixedEpochTimer = "could not create fixed epoch timer" // Debug in ../node/cmd/frostfs-node/reputation.go FrostFSNodeStartedLocalNodesMaintenance = "started local node's maintenance" // Info in ../node/cmd/frostfs-node/config.go FrostFSNodeStoppedLocalNodesMaintenance = "stopped local node's maintenance" // Info in ../node/cmd/frostfs-node/config.go FrostFSNodeFailedToAttachShardToEngine = "failed to attach shard to engine" // Error in ../node/cmd/frostfs-node/config.go @@ -593,8 +553,6 @@ const ( FrostFSNodeCouldNotWriteObjectNotification = "could not write object notification" // Warn in ../node/cmd/frostfs-node/notificator.go FrostFSNodeCouldNotGetMaxObjectSizeValue = "could not get max object size value" // Error in ../node/cmd/frostfs-node/object.go FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage = "could not inhume mark redundant copy as garbage" // Warn in ../node/cmd/frostfs-node/object.go - FrostFSNodeWritingLocalReputationValues = "writing local reputation values" // Debug in ../node/cmd/frostfs-node/object.go - FrostFSNodeCouldNotGetLatestNetworkMapToOverloadTheClient = "could not get latest network map to overload the client" // Warn in ../node/cmd/frostfs-node/object.go FrostFSNodeFailedInitTracing = "failed init tracing" // Error in ../node/cmd/frostfs-node/tracing.go FrostFSNodeFailedShutdownTracing = "failed shutdown tracing" // Error in ../node/cmd/frostfs-node/tracing.go FrostFSNodeFailedToCreateNeoRPCClient = "failed to create neo RPC client" // Info in ../node/cmd/frostfs-node/morph.go @@ -617,16 +575,4 @@ const ( FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" // Error in ../node/cmd/frostfs-node/tree.go FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" // Error in ../node/cmd/frostfs-node/control.go CommonApplicationStarted = "application started" // Info in ../node/cmd/frostfs-ir/main.go - CommonInitializingRemoteWriterProvider = "initializing remote writer provider" // Debug in ../node/cmd/frostfs-node/reputation/common/remote.go - CommonRouteHasReachedDeadendProvider = "route has reached dead-end provider" // Debug in ../node/cmd/frostfs-node/reputation/common/remote.go - CommonInitializingNoopWriterProvider = "initializing no-op writer provider" // Debug in ../node/cmd/frostfs-node/reputation/common/remote.go - IntermediateWritingReceivedConsumersTrusts = "writing received consumer's trusts" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/consumers.go - IntermediateStartWritingGlobalTrustsToContract = "start writing global trusts to contract" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/contract.go - IntermediateFailedToSignGlobalTrust = "failed to sign global trust" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/contract.go - IntermediateFailedToWriteGlobalTrustToContract = "failed to write global trust to contract" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/contract.go - IntermediateSentGlobalTrustToContract = "sent global trust to contract" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/contract.go - IntermediateWritingReceivedDaughtersTrusts = "writing received daughter's trusts" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/daughters.go - IntermediateAnnouncingTrust = "announcing trust" // Debug in ../node/cmd/frostfs-node/reputation/intermediate/remote.go - LocalAnnouncingTrusts = "announcing trusts" // Debug in ../node/cmd/frostfs-node/reputation/local/remote.go - LocalInitializingIteratorOverTrusts = "initializing iterator over trusts" // Debug in ../node/cmd/frostfs-node/reputation/local/storage.go ) From beabed788c72fe473cafe54b5afac110f7f82532 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 14 Apr 2023 14:51:13 +0300 Subject: [PATCH 0117/1943] [#247] network_config: Drop reputation Drop the code that was expected to work with global reputation network parameters. Signed-off-by: Pavel Karpy --- .../internal/modules/morph/config.go | 13 ------- .../modules/morph/initialize_deploy.go | 5 --- .../internal/modules/morph/netmap_util.go | 2 -- cmd/frostfs-cli/modules/netmap/netinfo.go | 2 -- cmd/frostfs-node/netmap.go | 2 -- pkg/morph/client/netmap/config.go | 36 ------------------- 6 files changed, 60 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index 3a60e7197..09d071b53 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -56,15 +56,12 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { switch k { case netmap.AuditFeeConfig, netmap.BasicIncomeRateConfig, netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, - netmap.EtIterationsConfig, netmap.EpochDurationConfig, netmap.IrCandidateFeeConfig, netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig: nbuf := make([]byte, 8) copy(nbuf[:], v) n := binary.LittleEndian.Uint64(nbuf) _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%d (int)\n", k, n))) - case netmap.EtAlphaConfig: - _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%s (str)\n", k, v))) case netmap.HomomorphicHashingDisabledKey, netmap.MaintenanceModeAllowedConfig: if len(v) == 0 || len(v) > 1 { return invalidConfigValueErr(k) @@ -139,22 +136,12 @@ func parseConfigPair(kvStr string, force bool) (key string, val any, err error) switch key { case netmap.AuditFeeConfig, netmap.BasicIncomeRateConfig, netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, - netmap.EtIterationsConfig, netmap.EpochDurationConfig, netmap.IrCandidateFeeConfig, netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig: val, err = strconv.ParseInt(valRaw, 10, 64) if err != nil { err = fmt.Errorf("could not parse %s's value '%s' as int: %w", key, valRaw, err) } - case netmap.EtAlphaConfig: - // just check that it could - // be parsed correctly - _, err = strconv.ParseFloat(v, 64) - if err != nil { - err = fmt.Errorf("could not parse %s's value '%s' as float: %w", key, valRaw, err) - } - - val = valRaw case netmap.HomomorphicHashingDisabledKey, netmap.MaintenanceModeAllowedConfig: val, err = strconv.ParseBool(valRaw) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 156b1f788..9a15b69ba 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -50,11 +50,6 @@ const ( subnetContract = "subnet" ) -const ( - defaultEigenTrustIterations = 4 - defaultEigenTrustAlpha = "0.1" -) - var ( contractList = []string{ auditContract, diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_util.go b/cmd/frostfs-adm/internal/modules/morph/netmap_util.go index 4b6de5bd2..23cfd120c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap_util.go @@ -15,8 +15,6 @@ func getDefaultNetmapContractConfigMap() map[string]any { m[netmap.AuditFeeConfig] = viper.GetInt64(auditFeeInitFlag) m[netmap.ContainerFeeConfig] = viper.GetInt64(containerFeeInitFlag) m[netmap.ContainerAliasFeeConfig] = viper.GetInt64(containerAliasFeeInitFlag) - m[netmap.EtIterationsConfig] = int64(defaultEigenTrustIterations) - m[netmap.EtAlphaConfig] = defaultEigenTrustAlpha m[netmap.BasicIncomeRateConfig] = viper.GetInt64(incomeRateInitFlag) m[netmap.IrCandidateFeeConfig] = viper.GetInt64(candidateFeeInitFlag) m[netmap.WithdrawFeeConfig] = viper.GetInt64(withdrawFeeInitFlag) diff --git a/cmd/frostfs-cli/modules/netmap/netinfo.go b/cmd/frostfs-cli/modules/netmap/netinfo.go index f34456c71..17acfd59c 100644 --- a/cmd/frostfs-cli/modules/netmap/netinfo.go +++ b/cmd/frostfs-cli/modules/netmap/netinfo.go @@ -41,8 +41,6 @@ var netInfoCmd = &cobra.Command{ cmd.Printf(format, "Audit fee", netInfo.AuditFee()) cmd.Printf(format, "Storage price", netInfo.StoragePrice()) cmd.Printf(format, "Container fee", netInfo.ContainerFee()) - cmd.Printf(format, "EigenTrust alpha", netInfo.EigenTrustAlpha()) - cmd.Printf(format, "Number of EigenTrust iterations", netInfo.NumberOfEigenTrustIterations()) cmd.Printf(format, "Epoch duration", netInfo.EpochDuration()) cmd.Printf(format, "Inner Ring candidate fee", netInfo.IRCandidateFee()) cmd.Printf(format, "Maximum object size", netInfo.MaxObjectSize()) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 76cceeb6d..b8a25cb8c 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -451,8 +451,6 @@ func (n *netInfo) Dump(ver version.Version) (*netmapSDK.NetworkInfo, error) { ni.SetEpochDuration(netInfoMorph.EpochDuration) ni.SetContainerFee(netInfoMorph.ContainerFee) ni.SetNamedContainerFee(netInfoMorph.ContainerAliasFee) - ni.SetNumberOfEigenTrustIterations(netInfoMorph.EigenTrustIterations) - ni.SetEigenTrustAlpha(netInfoMorph.EigenTrustAlpha) ni.SetIRCandidateFee(netInfoMorph.IRCandidateFee) ni.SetWithdrawalFee(netInfoMorph.WithdrawalFee) diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 3011bd541..71ffa2a7a 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -3,7 +3,6 @@ package netmap import ( "errors" "fmt" - "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/encoding/bigint" @@ -17,8 +16,6 @@ const ( EpochDurationConfig = "EpochDuration" ContainerFeeConfig = "ContainerFee" ContainerAliasFeeConfig = "ContainerAliasFee" - EtIterationsConfig = "EigenTrustIterations" - EtAlphaConfig = "EigenTrustAlpha" IrCandidateFeeConfig = "InnerRingCandidateFee" WithdrawFeeConfig = "WithdrawFee" HomomorphicHashingDisabledKey = "HomomorphicHashingDisabled" @@ -90,28 +87,6 @@ func (c *Client) ContainerAliasFee() (uint64, error) { return fee, nil } -// EigenTrustIterations returns global configuration value of iteration cycles -// for EigenTrust algorithm per epoch. -func (c *Client) EigenTrustIterations() (uint64, error) { - iterations, err := c.readUInt64Config(EtIterationsConfig) - if err != nil { - return 0, fmt.Errorf("(%T) could not get eigen trust iterations: %w", c, err) - } - - return iterations, nil -} - -// EigenTrustAlpha returns global configuration value of alpha parameter. -// It receives the alpha as a string and tries to convert it to float. -func (c *Client) EigenTrustAlpha() (float64, error) { - strAlpha, err := c.readStringConfig(EtAlphaConfig) - if err != nil { - return 0, fmt.Errorf("(%T) could not get eigen trust alpha: %w", c, err) - } - - return strconv.ParseFloat(strAlpha, 64) -} - // HomomorphicHashDisabled returns global configuration value of homomorphic hashing // settings. // @@ -246,10 +221,6 @@ type NetworkConfiguration struct { ContainerAliasFee uint64 - EigenTrustIterations uint64 - - EigenTrustAlpha float64 - IRCandidateFee uint64 WithdrawalFee uint64 @@ -311,13 +282,6 @@ func (c *Client) ReadNetworkConfiguration() (NetworkConfiguration, error) { res.ContainerFee = bytesToUint64(value) case ContainerAliasFeeConfig: res.ContainerAliasFee = bytesToUint64(value) - case EtIterationsConfig: - res.EigenTrustIterations = bytesToUint64(value) - case EtAlphaConfig: - res.EigenTrustAlpha, err = strconv.ParseFloat(string(value), 64) - if err != nil { - return fmt.Errorf("invalid prm %s: %v", EtAlphaConfig, err) - } case IrCandidateFeeConfig: res.IRCandidateFee = bytesToUint64(value) case WithdrawFeeConfig: From 070154d506238f6712e0a327918b1e797106d134 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 14 Apr 2023 14:54:28 +0300 Subject: [PATCH 0118/1943] [#247] client: Drop reputation related RPCs Signed-off-by: Pavel Karpy --- pkg/core/client/client.go | 2 -- pkg/network/cache/multi.go | 18 ------------------ 2 files changed, 20 deletions(-) diff --git a/pkg/core/client/client.go b/pkg/core/client/client.go index d74adddcc..98e9d6367 100644 --- a/pkg/core/client/client.go +++ b/pkg/core/client/client.go @@ -19,8 +19,6 @@ type Client interface { ObjectSearchInit(context.Context, client.PrmObjectSearch) (*client.ObjectListReader, error) ObjectRangeInit(context.Context, client.PrmObjectRange) (*client.ObjectRangeReader, error) ObjectHash(context.Context, client.PrmObjectHash) (*client.ResObjectHash, error) - AnnounceLocalTrust(context.Context, client.PrmAnnounceLocalTrust) (*client.ResAnnounceLocalTrust, error) - AnnounceIntermediateTrust(context.Context, client.PrmAnnounceIntermediateTrust) (*client.ResAnnounceIntermediateTrust, error) ExecRaw(f func(client *rawclient.Client) error) error Close() error } diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 39c191b78..2f698912c 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -277,24 +277,6 @@ func (x *multiClient) ObjectSearchInit(ctx context.Context, p client.PrmObjectSe return } -func (x *multiClient) AnnounceLocalTrust(ctx context.Context, prm client.PrmAnnounceLocalTrust) (res *client.ResAnnounceLocalTrust, err error) { - err = x.iterateClients(ctx, func(c clientcore.Client) error { - res, err = c.AnnounceLocalTrust(ctx, prm) - return err - }) - - return -} - -func (x *multiClient) AnnounceIntermediateTrust(ctx context.Context, prm client.PrmAnnounceIntermediateTrust) (res *client.ResAnnounceIntermediateTrust, err error) { - err = x.iterateClients(ctx, func(c clientcore.Client) error { - res, err = c.AnnounceIntermediateTrust(ctx, prm) - return err - }) - - return -} - func (x *multiClient) ExecRaw(f func(client *rawclient.Client) error) error { panic("multiClient.ExecRaw() must not be called") } From 8466894fdff20a6ac1b8b9b9b972f8c0cb436862 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 14 Apr 2023 09:38:29 +0300 Subject: [PATCH 0119/1943] [#250] control: remove `DumpShard` and `RestoreShard` RPC We have `Evacuate` with a cleaner interface. Also, remove them from CLI and engine. Signed-off-by: Evgenii Stratonikov --- .../modules/control/evacuate_shard.go | 6 +- cmd/frostfs-cli/modules/control/shards.go | 4 - .../modules/control/shards_dump.go | 66 - .../modules/control/shards_restore.go | 66 - pkg/local_object_storage/engine/dump.go | 19 - pkg/local_object_storage/engine/evacuate.go | 5 +- .../engine/evacuate_test.go | 2 +- pkg/local_object_storage/engine/restore.go | 32 - pkg/local_object_storage/shard/dump.go | 129 -- pkg/local_object_storage/shard/dump_test.go | 412 ------ pkg/local_object_storage/shard/restore.go | 145 -- pkg/services/control/convert.go | 36 - pkg/services/control/rpc.go | 28 - pkg/services/control/server/dump.go | 37 - pkg/services/control/server/restore.go | 37 - pkg/services/control/service.go | 58 - pkg/services/control/service.pb.go | 1246 +++++------------ pkg/services/control/service.proto | 75 - pkg/services/control/service_frostfs.pb.go | 310 ---- pkg/services/control/service_grpc.pb.go | 78 -- 20 files changed, 325 insertions(+), 2466 deletions(-) delete mode 100644 cmd/frostfs-cli/modules/control/shards_dump.go delete mode 100644 cmd/frostfs-cli/modules/control/shards_restore.go delete mode 100644 pkg/local_object_storage/engine/dump.go delete mode 100644 pkg/local_object_storage/engine/restore.go delete mode 100644 pkg/local_object_storage/shard/dump.go delete mode 100644 pkg/local_object_storage/shard/dump_test.go delete mode 100644 pkg/local_object_storage/shard/restore.go delete mode 100644 pkg/services/control/server/dump.go delete mode 100644 pkg/services/control/server/restore.go diff --git a/cmd/frostfs-cli/modules/control/evacuate_shard.go b/cmd/frostfs-cli/modules/control/evacuate_shard.go index 02ee88ce0..b72ff6301 100644 --- a/cmd/frostfs-cli/modules/control/evacuate_shard.go +++ b/cmd/frostfs-cli/modules/control/evacuate_shard.go @@ -8,6 +8,8 @@ import ( "github.com/spf13/cobra" ) +const ignoreErrorsFlag = "no-errors" + var evacuateShardCmd = &cobra.Command{ Use: "evacuate", Short: "Evacuate objects from shard", @@ -20,7 +22,7 @@ func evacuateShard(cmd *cobra.Command, _ []string) { req := &control.EvacuateShardRequest{Body: new(control.EvacuateShardRequest_Body)} req.Body.Shard_ID = getShardIDList(cmd) - req.Body.IgnoreErrors, _ = cmd.Flags().GetBool(dumpIgnoreErrorsFlag) + req.Body.IgnoreErrors, _ = cmd.Flags().GetBool(ignoreErrorsFlag) signRequest(cmd, pk, req) @@ -47,7 +49,7 @@ func initControlEvacuateShardCmd() { flags := evacuateShardCmd.Flags() flags.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") flags.Bool(shardAllFlag, false, "Process all shards") - flags.Bool(dumpIgnoreErrorsFlag, false, "Skip invalid/unreadable objects") + flags.Bool(ignoreErrorsFlag, false, "Skip invalid/unreadable objects") evacuateShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) } diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index 9d3eb5c01..8e7ecff8c 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -13,16 +13,12 @@ var shardsCmd = &cobra.Command{ func initControlShardsCmd() { shardsCmd.AddCommand(listShardsCmd) shardsCmd.AddCommand(setShardModeCmd) - shardsCmd.AddCommand(dumpShardCmd) - shardsCmd.AddCommand(restoreShardCmd) shardsCmd.AddCommand(evacuateShardCmd) shardsCmd.AddCommand(flushCacheCmd) shardsCmd.AddCommand(doctorCmd) initControlShardsListCmd() initControlSetShardModeCmd() - initControlDumpShardCmd() - initControlRestoreShardCmd() initControlEvacuateShardCmd() initControlFlushCacheCmd() initControlDoctorCmd() diff --git a/cmd/frostfs-cli/modules/control/shards_dump.go b/cmd/frostfs-cli/modules/control/shards_dump.go deleted file mode 100644 index c0d0aca95..000000000 --- a/cmd/frostfs-cli/modules/control/shards_dump.go +++ /dev/null @@ -1,66 +0,0 @@ -package control - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "github.com/spf13/cobra" -) - -const ( - dumpFilepathFlag = "path" - dumpIgnoreErrorsFlag = "no-errors" -) - -var dumpShardCmd = &cobra.Command{ - Use: "dump", - Short: "Dump objects from shard", - Long: "Dump objects from shard to a file", - Run: dumpShard, -} - -func dumpShard(cmd *cobra.Command, _ []string) { - pk := key.Get(cmd) - - body := new(control.DumpShardRequest_Body) - body.SetShardID(getShardID(cmd)) - - p, _ := cmd.Flags().GetString(dumpFilepathFlag) - body.SetFilepath(p) - - ignore, _ := cmd.Flags().GetBool(dumpIgnoreErrorsFlag) - body.SetIgnoreErrors(ignore) - - req := new(control.DumpShardRequest) - req.SetBody(body) - - signRequest(cmd, pk, req) - - cli := getClient(cmd, pk) - - var resp *control.DumpShardResponse - var err error - err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.DumpShard(client, req) - return err - }) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - - cmd.Println("Shard has been dumped successfully.") -} - -func initControlDumpShardCmd() { - initControlFlags(dumpShardCmd) - - flags := dumpShardCmd.Flags() - flags.String(shardIDFlag, "", "Shard ID in base58 encoding") - flags.String(dumpFilepathFlag, "", "File to write objects to") - flags.Bool(dumpIgnoreErrorsFlag, false, "Skip invalid/unreadable objects") - - _ = dumpShardCmd.MarkFlagRequired(shardIDFlag) - _ = dumpShardCmd.MarkFlagRequired(dumpFilepathFlag) - _ = dumpShardCmd.MarkFlagRequired(controlRPC) -} diff --git a/cmd/frostfs-cli/modules/control/shards_restore.go b/cmd/frostfs-cli/modules/control/shards_restore.go deleted file mode 100644 index edf97a731..000000000 --- a/cmd/frostfs-cli/modules/control/shards_restore.go +++ /dev/null @@ -1,66 +0,0 @@ -package control - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "github.com/spf13/cobra" -) - -const ( - restoreFilepathFlag = "path" - restoreIgnoreErrorsFlag = "no-errors" -) - -var restoreShardCmd = &cobra.Command{ - Use: "restore", - Short: "Restore objects from shard", - Long: "Restore objects from shard to a file", - Run: restoreShard, -} - -func restoreShard(cmd *cobra.Command, _ []string) { - pk := key.Get(cmd) - - body := new(control.RestoreShardRequest_Body) - body.SetShardID(getShardID(cmd)) - - p, _ := cmd.Flags().GetString(restoreFilepathFlag) - body.SetFilepath(p) - - ignore, _ := cmd.Flags().GetBool(restoreIgnoreErrorsFlag) - body.SetIgnoreErrors(ignore) - - req := new(control.RestoreShardRequest) - req.SetBody(body) - - signRequest(cmd, pk, req) - - cli := getClient(cmd, pk) - - var resp *control.RestoreShardResponse - var err error - err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.RestoreShard(client, req) - return err - }) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - - cmd.Println("Shard has been restored successfully.") -} - -func initControlRestoreShardCmd() { - initControlFlags(restoreShardCmd) - - flags := restoreShardCmd.Flags() - flags.String(shardIDFlag, "", "Shard ID in base58 encoding") - flags.String(restoreFilepathFlag, "", "File to read objects from") - flags.Bool(restoreIgnoreErrorsFlag, false, "Skip invalid/unreadable objects") - - _ = restoreShardCmd.MarkFlagRequired(shardIDFlag) - _ = restoreShardCmd.MarkFlagRequired(restoreFilepathFlag) - _ = restoreShardCmd.MarkFlagRequired(controlRPC) -} diff --git a/pkg/local_object_storage/engine/dump.go b/pkg/local_object_storage/engine/dump.go deleted file mode 100644 index f5cf8c32e..000000000 --- a/pkg/local_object_storage/engine/dump.go +++ /dev/null @@ -1,19 +0,0 @@ -package engine - -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - -// DumpShard dumps objects from the shard with provided identifier. -// -// Returns an error if shard is not read-only. -func (e *StorageEngine) DumpShard(id *shard.ID, prm shard.DumpPrm) error { - e.mtx.RLock() - defer e.mtx.RUnlock() - - sh, ok := e.shards[id.String()] - if !ok { - return errShardNotFound - } - - _, err := sh.Dump(prm) - return err -} diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 2ec2c2b35..e212784a3 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -16,6 +17,8 @@ import ( "go.uber.org/zap" ) +var ErrMustBeReadOnly = logicerr.New("shard must be in read-only mode") + // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { shardID []*shard.ID @@ -135,7 +138,7 @@ func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) } if !sh.GetMode().ReadOnly() { - return nil, nil, shard.ErrMustBeReadOnly + return nil, nil, ErrMustBeReadOnly } } diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 291bc2b78..fc9da5e3f 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -103,7 +103,7 @@ func TestEvacuateShard(t *testing.T) { t.Run("must be read-only", func(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) - require.ErrorIs(t, err, shard.ErrMustBeReadOnly) + require.ErrorIs(t, err, ErrMustBeReadOnly) require.Equal(t, 0, res.Count()) }) diff --git a/pkg/local_object_storage/engine/restore.go b/pkg/local_object_storage/engine/restore.go deleted file mode 100644 index 7cc2eaf6c..000000000 --- a/pkg/local_object_storage/engine/restore.go +++ /dev/null @@ -1,32 +0,0 @@ -package engine - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -// RestoreShard restores objects from dump to the shard with provided identifier. -// -// Returns an error if shard is not read-only. -func (e *StorageEngine) RestoreShard(ctx context.Context, id *shard.ID, prm shard.RestorePrm) error { - ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.RestoreShard", - trace.WithAttributes( - attribute.String("shard_id", id.String()), - )) - defer span.End() - - e.mtx.RLock() - defer e.mtx.RUnlock() - - sh, ok := e.shards[id.String()] - if !ok { - return errShardNotFound - } - - _, err := sh.Restore(ctx, prm) - return err -} diff --git a/pkg/local_object_storage/shard/dump.go b/pkg/local_object_storage/shard/dump.go deleted file mode 100644 index 8d9fe0f71..000000000 --- a/pkg/local_object_storage/shard/dump.go +++ /dev/null @@ -1,129 +0,0 @@ -package shard - -import ( - "encoding/binary" - "io" - "os" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" -) - -var dumpMagic = []byte("NEOF") - -// DumpPrm groups the parameters of Dump operation. -type DumpPrm struct { - path string - stream io.Writer - ignoreErrors bool -} - -// WithPath is an Dump option to set the destination path. -func (p *DumpPrm) WithPath(path string) { - p.path = path -} - -// WithStream is an Dump option to set the destination stream. -// It takes priority over `path` option. -func (p *DumpPrm) WithStream(r io.Writer) { - p.stream = r -} - -// WithIgnoreErrors is an Dump option to allow ignore all errors during iteration. -// This includes invalid blobovniczas as well as corrupted objects. -func (p *DumpPrm) WithIgnoreErrors(ignore bool) { - p.ignoreErrors = ignore -} - -// DumpRes groups the result fields of Dump operation. -type DumpRes struct { - count int -} - -// Count return amount of object written. -func (r DumpRes) Count() int { - return r.count -} - -var ErrMustBeReadOnly = logicerr.New("shard must be in read-only mode") - -// Dump dumps all objects from the shard to a file or stream. -// -// Returns any error encountered. -func (s *Shard) Dump(prm DumpPrm) (DumpRes, error) { - s.m.RLock() - defer s.m.RUnlock() - - if !s.info.Mode.ReadOnly() { - return DumpRes{}, ErrMustBeReadOnly - } - - w := prm.stream - if w == nil { - f, err := os.OpenFile(prm.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640) - if err != nil { - return DumpRes{}, err - } - defer f.Close() - - w = f - } - - _, err := w.Write(dumpMagic) - if err != nil { - return DumpRes{}, err - } - - var count int - - if s.hasWriteCache() { - var iterPrm writecache.IterationPrm - - iterPrm.WithIgnoreErrors(prm.ignoreErrors) - iterPrm.WithHandler(func(data []byte) error { - var size [4]byte - binary.LittleEndian.PutUint32(size[:], uint32(len(data))) - if _, err := w.Write(size[:]); err != nil { - return err - } - - if _, err := w.Write(data); err != nil { - return err - } - - count++ - return nil - }) - - err := s.writeCache.Iterate(iterPrm) - if err != nil { - return DumpRes{}, err - } - } - - var pi common.IteratePrm - pi.IgnoreErrors = prm.ignoreErrors - pi.Handler = func(elem common.IterationElement) error { - data := elem.ObjectData - - var size [4]byte - binary.LittleEndian.PutUint32(size[:], uint32(len(data))) - if _, err := w.Write(size[:]); err != nil { - return err - } - - if _, err := w.Write(data); err != nil { - return err - } - - count++ - return nil - } - - if _, err := s.blobStor.Iterate(pi); err != nil { - return DumpRes{}, err - } - - return DumpRes{count: count}, nil -} diff --git a/pkg/local_object_storage/shard/dump_test.go b/pkg/local_object_storage/shard/dump_test.go deleted file mode 100644 index 921717204..000000000 --- a/pkg/local_object_storage/shard/dump_test.go +++ /dev/null @@ -1,412 +0,0 @@ -package shard_test - -import ( - "bytes" - "context" - "io" - "math/rand" - "os" - "path/filepath" - "testing" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - "github.com/klauspost/compress/zstd" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" -) - -func TestDump(t *testing.T) { - t.Run("without write-cache", func(t *testing.T) { - testDump(t, 10, false) - }) - t.Run("with write-cache", func(t *testing.T) { - // Put a bit more objects to write-cache to facilitate race-conditions. - testDump(t, 100, true) - }) -} - -func testDump(t *testing.T, objCount int, hasWriteCache bool) { - const ( - wcSmallObjectSize = 1024 // 1 KiB, goes to write-cache memory - wcBigObjectSize = 4 * 1024 // 4 KiB, goes to write-cache FSTree - bsSmallObjectSize = 10 * 1024 // 10 KiB, goes to blobovnicza DB - bsBigObjectSize = 1024*1024 + 1 // > 1 MiB, goes to blobovnicza FSTree - ) - - var sh *shard.Shard - if !hasWriteCache { - sh = newShard(t, false) - } else { - sh = newCustomShard(t, t.TempDir(), true, - []writecache.Option{ - writecache.WithSmallObjectSize(wcSmallObjectSize), - writecache.WithMaxObjectSize(wcBigObjectSize), - writecache.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), - }, - nil) - } - defer releaseShard(sh, t) - - out := filepath.Join(t.TempDir(), "dump") - var prm shard.DumpPrm - prm.WithPath(out) - - t.Run("must be read-only", func(t *testing.T) { - _, err := sh.Dump(prm) - require.ErrorIs(t, err, shard.ErrMustBeReadOnly) - }) - - require.NoError(t, sh.SetMode(mode.ReadOnly)) - outEmpty := out + ".empty" - var dumpPrm shard.DumpPrm - dumpPrm.WithPath(outEmpty) - - res, err := sh.Dump(dumpPrm) - require.NoError(t, err) - require.Equal(t, 0, res.Count()) - require.NoError(t, sh.SetMode(mode.ReadWrite)) - - // Approximate object header size. - const headerSize = 400 - - objects := make([]*objectSDK.Object, objCount) - for i := 0; i < objCount; i++ { - cnr := cidtest.ID() - var size int - switch i % 6 { - case 0, 1: - size = wcSmallObjectSize - headerSize - case 2, 3: - size = bsSmallObjectSize - headerSize - case 4: - size = wcBigObjectSize - headerSize - default: - size = bsBigObjectSize - headerSize - } - data := make([]byte, size) - rand.Read(data) - obj := testutil.GenerateObjectWithCIDWithPayload(cnr, data) - objects[i] = obj - - var prm shard.PutPrm - prm.SetObject(objects[i]) - _, err := sh.Put(context.Background(), prm) - require.NoError(t, err) - } - - require.NoError(t, sh.SetMode(mode.ReadOnly)) - - t.Run("invalid path", func(t *testing.T) { - var dumpPrm shard.DumpPrm - dumpPrm.WithPath("\x00") - - _, err := sh.Dump(dumpPrm) - require.Error(t, err) - }) - - res, err = sh.Dump(prm) - require.NoError(t, err) - require.Equal(t, objCount, res.Count()) - - t.Run("restore", func(t *testing.T) { - sh := newShard(t, false) - defer releaseShard(sh, t) - - t.Run("empty dump", func(t *testing.T) { - var restorePrm shard.RestorePrm - restorePrm.WithPath(outEmpty) - res, err := sh.Restore(context.Background(), restorePrm) - require.NoError(t, err) - require.Equal(t, 0, res.Count()) - }) - - t.Run("invalid path", func(t *testing.T) { - _, err := sh.Restore(context.Background(), *new(shard.RestorePrm)) - require.ErrorIs(t, err, os.ErrNotExist) - }) - - t.Run("invalid file", func(t *testing.T) { - t.Run("invalid magic", func(t *testing.T) { - out := out + ".wrongmagic" - require.NoError(t, os.WriteFile(out, []byte{0, 0, 0, 0}, os.ModePerm)) - - var restorePrm shard.RestorePrm - restorePrm.WithPath(out) - - _, err := sh.Restore(context.Background(), restorePrm) - require.ErrorIs(t, err, shard.ErrInvalidMagic) - }) - - fileData, err := os.ReadFile(out) - require.NoError(t, err) - - t.Run("incomplete size", func(t *testing.T) { - out := out + ".wrongsize" - fileData := append(fileData, 1) - require.NoError(t, os.WriteFile(out, fileData, os.ModePerm)) - - var restorePrm shard.RestorePrm - restorePrm.WithPath(out) - - _, err := sh.Restore(context.Background(), restorePrm) - require.ErrorIs(t, err, io.ErrUnexpectedEOF) - }) - t.Run("incomplete object data", func(t *testing.T) { - out := out + ".wrongsize" - fileData := append(fileData, 1, 0, 0, 0) - require.NoError(t, os.WriteFile(out, fileData, os.ModePerm)) - - var restorePrm shard.RestorePrm - restorePrm.WithPath(out) - - _, err := sh.Restore(context.Background(), restorePrm) - require.ErrorIs(t, err, io.EOF) - }) - t.Run("invalid object", func(t *testing.T) { - out := out + ".wrongobj" - fileData := append(fileData, 1, 0, 0, 0, 0xFF, 4, 0, 0, 0, 1, 2, 3, 4) - require.NoError(t, os.WriteFile(out, fileData, os.ModePerm)) - - var restorePrm shard.RestorePrm - restorePrm.WithPath(out) - - _, err := sh.Restore(context.Background(), restorePrm) - require.Error(t, err) - - t.Run("skip errors", func(t *testing.T) { - sh := newCustomShard(t, filepath.Join(t.TempDir(), "ignore"), false, nil, nil) - t.Cleanup(func() { require.NoError(t, sh.Close()) }) - - var restorePrm shard.RestorePrm - restorePrm.WithPath(out) - restorePrm.WithIgnoreErrors(true) - - res, err := sh.Restore(context.Background(), restorePrm) - require.NoError(t, err) - require.Equal(t, objCount, res.Count()) - require.Equal(t, 2, res.FailCount()) - }) - }) - }) - - var prm shard.RestorePrm - prm.WithPath(out) - t.Run("must allow write", func(t *testing.T) { - require.NoError(t, sh.SetMode(mode.ReadOnly)) - - _, err := sh.Restore(context.Background(), prm) - require.ErrorIs(t, err, shard.ErrReadOnlyMode) - }) - - require.NoError(t, sh.SetMode(mode.ReadWrite)) - - checkRestore(t, sh, prm, objects) - }) -} - -func TestStream(t *testing.T) { - sh1 := newCustomShard(t, filepath.Join(t.TempDir(), "shard1"), false, nil, nil) - defer releaseShard(sh1, t) - - sh2 := newCustomShard(t, filepath.Join(t.TempDir(), "shard2"), false, nil, nil) - defer releaseShard(sh2, t) - - const objCount = 5 - objects := make([]*objectSDK.Object, objCount) - for i := 0; i < objCount; i++ { - cnr := cidtest.ID() - obj := testutil.GenerateObjectWithCID(cnr) - objects[i] = obj - - var prm shard.PutPrm - prm.SetObject(objects[i]) - _, err := sh1.Put(context.Background(), prm) - require.NoError(t, err) - } - - require.NoError(t, sh1.SetMode(mode.ReadOnly)) - - r, w := io.Pipe() - finish := make(chan struct{}) - - go func() { - var dumpPrm shard.DumpPrm - dumpPrm.WithStream(w) - - res, err := sh1.Dump(dumpPrm) - require.NoError(t, err) - require.Equal(t, objCount, res.Count()) - require.NoError(t, w.Close()) - close(finish) - }() - - var restorePrm shard.RestorePrm - restorePrm.WithStream(r) - - checkRestore(t, sh2, restorePrm, objects) - require.Eventually(t, func() bool { - select { - case <-finish: - return true - default: - return false - } - }, time.Second, time.Millisecond) -} - -func checkRestore(t *testing.T, sh *shard.Shard, prm shard.RestorePrm, objects []*objectSDK.Object) { - res, err := sh.Restore(context.Background(), prm) - require.NoError(t, err) - require.Equal(t, len(objects), res.Count()) - - var getPrm shard.GetPrm - - for i := range objects { - getPrm.SetAddress(object.AddressOf(objects[i])) - res, err := sh.Get(context.Background(), getPrm) - require.NoError(t, err) - require.Equal(t, objects[i], res.Object()) - } -} - -func TestDumpIgnoreErrors(t *testing.T) { - const ( - wcSmallObjectSize = 512 // goes to write-cache memory - wcBigObjectSize = wcSmallObjectSize << 1 // goes to write-cache FSTree - bsSmallObjectSize = wcSmallObjectSize << 2 // goes to blobovnicza DB - - objCount = 10 - headerSize = 400 - ) - - dir := t.TempDir() - bsPath := filepath.Join(dir, "blob") - bsOpts := func(sw uint64) []blobstor.Option { - return []blobstor.Option{ - blobstor.WithCompressObjects(true), - blobstor.WithStorages([]blobstor.SubStorage{ - { - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(filepath.Join(bsPath, "blobovnicza")), - blobovniczatree.WithBlobovniczaShallowDepth(1), - blobovniczatree.WithBlobovniczaShallowWidth(sw), - blobovniczatree.WithOpenedCacheSize(1)), - Policy: func(_ *objectSDK.Object, data []byte) bool { - return len(data) < bsSmallObjectSize - }, - }, - { - Storage: fstree.New( - fstree.WithPath(bsPath), - fstree.WithDepth(1)), - }, - }), - } - } - wcPath := filepath.Join(dir, "writecache") - wcOpts := []writecache.Option{ - writecache.WithPath(wcPath), - writecache.WithSmallObjectSize(wcSmallObjectSize), - writecache.WithMaxObjectSize(wcBigObjectSize), - } - sh := newCustomShard(t, dir, true, wcOpts, bsOpts(2)) - - objects := make([]*objectSDK.Object, objCount) - for i := 0; i < objCount; i++ { - size := (wcSmallObjectSize << (i % 4)) - headerSize - obj := testutil.GenerateObjectWithCIDWithPayload(cidtest.ID(), make([]byte, size)) - objects[i] = obj - - var prm shard.PutPrm - prm.SetObject(objects[i]) - _, err := sh.Put(context.Background(), prm) - require.NoError(t, err) - } - - releaseShard(sh, t) - - b := bytes.NewBuffer(nil) - badObject := make([]byte, 1000) - enc, err := zstd.NewWriter(b) - require.NoError(t, err) - corruptedData := enc.EncodeAll(badObject, nil) - for i := 4; i < len(corruptedData); i++ { - corruptedData[i] ^= 0xFF - } - - // There are 3 different types of errors to consider. - // To setup envirionment we use implementation details so this test must be updated - // if any of them are changed. - { - // 1. Invalid object in fs tree. - // 1.1. Invalid compressed data. - addr := cidtest.ID().EncodeToString() + "." + objecttest.ID().EncodeToString() - dirName := filepath.Join(bsPath, addr[:2]) - require.NoError(t, os.MkdirAll(dirName, os.ModePerm)) - require.NoError(t, os.WriteFile(filepath.Join(dirName, addr[2:]), corruptedData, os.ModePerm)) - - // 1.2. Unreadable file. - addr = cidtest.ID().EncodeToString() + "." + objecttest.ID().EncodeToString() - dirName = filepath.Join(bsPath, addr[:2]) - require.NoError(t, os.MkdirAll(dirName, os.ModePerm)) - - fname := filepath.Join(dirName, addr[2:]) - require.NoError(t, os.WriteFile(fname, []byte{}, 0)) - - // 1.3. Unreadable dir. - require.NoError(t, os.MkdirAll(filepath.Join(bsPath, "ZZ"), 0)) - } - - sh = newCustomShard(t, dir, true, wcOpts, bsOpts(3)) - require.NoError(t, sh.SetMode(mode.ReadOnly)) - - { - // 2. Invalid object in blobovnicza. - // 2.1. Invalid blobovnicza. - bTree := filepath.Join(bsPath, "blobovnicza") - data := make([]byte, 1024) - rand.Read(data) - require.NoError(t, os.WriteFile(filepath.Join(bTree, "0", "2"), data, 0)) - - // 2.2. Invalid object in valid blobovnicza. - var prm blobovnicza.PutPrm - prm.SetAddress(oid.Address{}) - prm.SetMarshaledObject(corruptedData) - b := blobovnicza.New(blobovnicza.WithPath(filepath.Join(bTree, "1", "2"))) - require.NoError(t, b.Open()) - _, err := b.Put(prm) - require.NoError(t, err) - require.NoError(t, b.Close()) - } - - { - // 3. Invalid object in write-cache. Note that because shard is read-only - // the object won't be flushed. - addr := cidtest.ID().EncodeToString() + "." + objecttest.ID().EncodeToString() - dir := filepath.Join(wcPath, addr[:1]) - require.NoError(t, os.MkdirAll(dir, os.ModePerm)) - require.NoError(t, os.WriteFile(filepath.Join(dir, addr[1:]), nil, 0)) - } - - out := filepath.Join(t.TempDir(), "out.dump") - var dumpPrm shard.DumpPrm - dumpPrm.WithPath(out) - dumpPrm.WithIgnoreErrors(true) - res, err := sh.Dump(dumpPrm) - require.NoError(t, err) - require.Equal(t, objCount, res.Count()) -} diff --git a/pkg/local_object_storage/shard/restore.go b/pkg/local_object_storage/shard/restore.go deleted file mode 100644 index 2cb64a518..000000000 --- a/pkg/local_object_storage/shard/restore.go +++ /dev/null @@ -1,145 +0,0 @@ -package shard - -import ( - "bytes" - "context" - "encoding/binary" - "errors" - "io" - "os" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -// ErrInvalidMagic is returned when dump format is invalid. -var ErrInvalidMagic = logicerr.New("invalid magic") - -// RestorePrm groups the parameters of Restore operation. -type RestorePrm struct { - path string - stream io.Reader - ignoreErrors bool -} - -// WithPath is a Restore option to set the destination path. -func (p *RestorePrm) WithPath(path string) { - p.path = path -} - -// WithStream is a Restore option to set the stream to read objects from. -// It takes priority over `WithPath` option. -func (p *RestorePrm) WithStream(r io.Reader) { - p.stream = r -} - -// WithIgnoreErrors is a Restore option which allows to ignore errors encountered during restore. -// Corrupted objects will not be processed. -func (p *RestorePrm) WithIgnoreErrors(ignore bool) { - p.ignoreErrors = ignore -} - -// RestoreRes groups the result fields of Restore operation. -type RestoreRes struct { - count int - failed int -} - -// Count return amount of object written. -func (r RestoreRes) Count() int { - return r.count -} - -// FailCount return amount of object skipped. -func (r RestoreRes) FailCount() int { - return r.failed -} - -// Restore restores objects from the dump prepared by Dump. -// -// Returns any error encountered. -func (s *Shard) Restore(ctx context.Context, prm RestorePrm) (RestoreRes, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Restore", - trace.WithAttributes( - attribute.String("shard_id", s.ID().String()), - attribute.String("path", prm.path), - attribute.Bool("ignore_errors", prm.ignoreErrors), - )) - defer span.End() - - s.m.RLock() - defer s.m.RUnlock() - - if s.info.Mode.ReadOnly() { - return RestoreRes{}, ErrReadOnlyMode - } - - r := prm.stream - if r == nil { - f, err := os.OpenFile(prm.path, os.O_RDONLY, os.ModeExclusive) - if err != nil { - return RestoreRes{}, err - } - defer f.Close() - - r = f - } - - var m [4]byte - _, _ = io.ReadFull(r, m[:]) - if !bytes.Equal(m[:], dumpMagic) { - return RestoreRes{}, ErrInvalidMagic - } - - var putPrm PutPrm - - var count, failCount int - var data []byte - var size [4]byte - for { - // If there are less than 4 bytes left, `Read` returns nil error instead of - // io.ErrUnexpectedEOF, thus `ReadFull` is used. - _, err := io.ReadFull(r, size[:]) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return RestoreRes{}, err - } - - sz := binary.LittleEndian.Uint32(size[:]) - if uint32(cap(data)) < sz { - data = make([]byte, sz) - } else { - data = data[:sz] - } - - _, err = r.Read(data) - if err != nil { - return RestoreRes{}, err - } - - obj := object.New() - err = obj.Unmarshal(data) - if err != nil { - if prm.ignoreErrors { - failCount++ - continue - } - return RestoreRes{}, err - } - - putPrm.SetObject(obj) - _, err = s.Put(ctx, putPrm) - if err != nil && !IsErrObjectExpired(err) && !IsErrRemoved(err) { - return RestoreRes{}, err - } - - count++ - } - - return RestoreRes{count: count, failed: failCount}, nil -} diff --git a/pkg/services/control/convert.go b/pkg/services/control/convert.go index f7582dd68..84bde31d6 100644 --- a/pkg/services/control/convert.go +++ b/pkg/services/control/convert.go @@ -111,42 +111,6 @@ func (w *setShardModeResponseWrapper) FromGRPCMessage(m grpc.Message) error { return nil } -type dumpShardResponseWrapper struct { - *DumpShardResponse -} - -func (w *dumpShardResponseWrapper) ToGRPCMessage() grpc.Message { - return w.DumpShardResponse -} - -func (w *dumpShardResponseWrapper) FromGRPCMessage(m grpc.Message) error { - r, ok := m.(*DumpShardResponse) - if !ok { - return message.NewUnexpectedMessageType(m, (*DumpShardResponse)(nil)) - } - - w.DumpShardResponse = r - return nil -} - -type restoreShardResponseWrapper struct { - *RestoreShardResponse -} - -func (w *restoreShardResponseWrapper) ToGRPCMessage() grpc.Message { - return w.RestoreShardResponse -} - -func (w *restoreShardResponseWrapper) FromGRPCMessage(m grpc.Message) error { - r, ok := m.(*RestoreShardResponse) - if !ok { - return message.NewUnexpectedMessageType(m, (*RestoreShardResponse)(nil)) - } - - w.RestoreShardResponse = r - return nil -} - type synchronizeTreeResponseWrapper struct { *SynchronizeTreeResponse } diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 2676ea7a5..625f485c9 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -13,8 +13,6 @@ const ( rpcDropObjects = "DropObjects" rpcListShards = "ListShards" rpcSetShardMode = "SetShardMode" - rpcDumpShard = "DumpShard" - rpcRestoreShard = "RestoreShard" rpcSynchronizeTree = "SynchronizeTree" rpcEvacuateShard = "EvacuateShard" rpcFlushCache = "FlushCache" @@ -128,32 +126,6 @@ func SetShardMode( return wResp.m, nil } -// DumpShard executes ControlService.DumpShard RPC. -func DumpShard(cli *client.Client, req *DumpShardRequest, opts ...client.CallOption) (*DumpShardResponse, error) { - wResp := &dumpShardResponseWrapper{new(DumpShardResponse)} - wReq := &requestWrapper{m: req} - - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcDumpShard), wReq, wResp, opts...) - if err != nil { - return nil, err - } - - return wResp.DumpShardResponse, nil -} - -// RestoreShard executes ControlService.DumpShard RPC. -func RestoreShard(cli *client.Client, req *RestoreShardRequest, opts ...client.CallOption) (*RestoreShardResponse, error) { - wResp := &restoreShardResponseWrapper{new(RestoreShardResponse)} - wReq := &requestWrapper{m: req} - - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcRestoreShard), wReq, wResp, opts...) - if err != nil { - return nil, err - } - - return wResp.RestoreShardResponse, nil -} - // SynchronizeTree executes ControlService.SynchronizeTree RPC. func SynchronizeTree(cli *client.Client, req *SynchronizeTreeRequest, opts ...client.CallOption) (*SynchronizeTreeResponse, error) { wResp := &synchronizeTreeResponseWrapper{new(SynchronizeTreeResponse)} diff --git a/pkg/services/control/server/dump.go b/pkg/services/control/server/dump.go deleted file mode 100644 index 28be02aa4..000000000 --- a/pkg/services/control/server/dump.go +++ /dev/null @@ -1,37 +0,0 @@ -package control - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func (s *Server) DumpShard(_ context.Context, req *control.DumpShardRequest) (*control.DumpShardResponse, error) { - err := s.isValidRequest(req) - if err != nil { - return nil, status.Error(codes.PermissionDenied, err.Error()) - } - - shardID := shard.NewIDFromBytes(req.GetBody().GetShard_ID()) - - var prm shard.DumpPrm - prm.WithPath(req.GetBody().GetFilepath()) - prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) - - err = s.s.DumpShard(shardID, prm) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - resp := new(control.DumpShardResponse) - resp.SetBody(new(control.DumpShardResponse_Body)) - - err = SignMessage(s.key, resp) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - return resp, nil -} diff --git a/pkg/services/control/server/restore.go b/pkg/services/control/server/restore.go deleted file mode 100644 index dba186f57..000000000 --- a/pkg/services/control/server/restore.go +++ /dev/null @@ -1,37 +0,0 @@ -package control - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func (s *Server) RestoreShard(ctx context.Context, req *control.RestoreShardRequest) (*control.RestoreShardResponse, error) { - err := s.isValidRequest(req) - if err != nil { - return nil, status.Error(codes.PermissionDenied, err.Error()) - } - - shardID := shard.NewIDFromBytes(req.GetBody().GetShard_ID()) - - var prm shard.RestorePrm - prm.WithPath(req.GetBody().GetFilepath()) - prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) - - err = s.s.RestoreShard(ctx, shardID, prm) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - resp := new(control.RestoreShardResponse) - resp.SetBody(new(control.RestoreShardResponse_Body)) - - err = SignMessage(s.key, resp) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - return resp, nil -} diff --git a/pkg/services/control/service.go b/pkg/services/control/service.go index dd349dc57..ef0c0a8d2 100644 --- a/pkg/services/control/service.go +++ b/pkg/services/control/service.go @@ -127,64 +127,6 @@ func (x *SetShardModeResponse) SetBody(v *SetShardModeResponse_Body) { } } -// SetShardID sets shard ID for the dump shard request. -func (x *DumpShardRequest_Body) SetShardID(id []byte) { - x.Shard_ID = id -} - -// SetFilepath sets filepath for the dump shard request. -func (x *DumpShardRequest_Body) SetFilepath(p string) { - x.Filepath = p -} - -// SetIgnoreErrors sets ignore errors flag for the dump shard request. -func (x *DumpShardRequest_Body) SetIgnoreErrors(ignore bool) { - x.IgnoreErrors = ignore -} - -// SetBody sets request body. -func (x *DumpShardRequest) SetBody(v *DumpShardRequest_Body) { - if x != nil { - x.Body = v - } -} - -// SetBody sets response body. -func (x *DumpShardResponse) SetBody(v *DumpShardResponse_Body) { - if x != nil { - x.Body = v - } -} - -// SetShardID sets shard ID for the restore shard request. -func (x *RestoreShardRequest_Body) SetShardID(id []byte) { - x.Shard_ID = id -} - -// SetFilepath sets filepath for the restore shard request. -func (x *RestoreShardRequest_Body) SetFilepath(p string) { - x.Filepath = p -} - -// SetIgnoreErrors sets ignore errors flag for the restore shard request. -func (x *RestoreShardRequest_Body) SetIgnoreErrors(ignore bool) { - x.IgnoreErrors = ignore -} - -// SetBody sets request body. -func (x *RestoreShardRequest) SetBody(v *RestoreShardRequest_Body) { - if x != nil { - x.Body = v - } -} - -// SetBody sets response body. -func (x *RestoreShardResponse) SetBody(v *RestoreShardResponse_Body) { - if x != nil { - x.Body = v - } -} - // SetBody sets list shards request body. func (x *SynchronizeTreeRequest) SetBody(v *SynchronizeTreeRequest_Body) { if x != nil { diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index ca3e2770e..d713bb38d 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -600,238 +600,6 @@ func (x *SetShardModeResponse) GetSignature() *Signature { return nil } -// DumpShard request. -type DumpShardRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of dump shard request message. - Body *DumpShardRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *DumpShardRequest) Reset() { - *x = DumpShardRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DumpShardRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DumpShardRequest) ProtoMessage() {} - -func (x *DumpShardRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DumpShardRequest.ProtoReflect.Descriptor instead. -func (*DumpShardRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{10} -} - -func (x *DumpShardRequest) GetBody() *DumpShardRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *DumpShardRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// DumpShard response. -type DumpShardResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of dump shard response message. - Body *DumpShardResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *DumpShardResponse) Reset() { - *x = DumpShardResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DumpShardResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DumpShardResponse) ProtoMessage() {} - -func (x *DumpShardResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DumpShardResponse.ProtoReflect.Descriptor instead. -func (*DumpShardResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{11} -} - -func (x *DumpShardResponse) GetBody() *DumpShardResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *DumpShardResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// RestoreShard request. -type RestoreShardRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of restore shard request message. - Body *RestoreShardRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RestoreShardRequest) Reset() { - *x = RestoreShardRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RestoreShardRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RestoreShardRequest) ProtoMessage() {} - -func (x *RestoreShardRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RestoreShardRequest.ProtoReflect.Descriptor instead. -func (*RestoreShardRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{12} -} - -func (x *RestoreShardRequest) GetBody() *RestoreShardRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RestoreShardRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// RestoreShard response. -type RestoreShardResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of restore shard response message. - Body *RestoreShardResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RestoreShardResponse) Reset() { - *x = RestoreShardResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RestoreShardResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RestoreShardResponse) ProtoMessage() {} - -func (x *RestoreShardResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RestoreShardResponse.ProtoReflect.Descriptor instead. -func (*RestoreShardResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{13} -} - -func (x *RestoreShardResponse) GetBody() *RestoreShardResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RestoreShardResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - // SynchronizeTree request. type SynchronizeTreeRequest struct { state protoimpl.MessageState @@ -847,7 +615,7 @@ type SynchronizeTreeRequest struct { func (x *SynchronizeTreeRequest) Reset() { *x = SynchronizeTreeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[14] + mi := &file_pkg_services_control_service_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -860,7 +628,7 @@ func (x *SynchronizeTreeRequest) String() string { func (*SynchronizeTreeRequest) ProtoMessage() {} func (x *SynchronizeTreeRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[14] + mi := &file_pkg_services_control_service_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -873,7 +641,7 @@ func (x *SynchronizeTreeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SynchronizeTreeRequest.ProtoReflect.Descriptor instead. func (*SynchronizeTreeRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{14} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{10} } func (x *SynchronizeTreeRequest) GetBody() *SynchronizeTreeRequest_Body { @@ -905,7 +673,7 @@ type SynchronizeTreeResponse struct { func (x *SynchronizeTreeResponse) Reset() { *x = SynchronizeTreeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[15] + mi := &file_pkg_services_control_service_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -918,7 +686,7 @@ func (x *SynchronizeTreeResponse) String() string { func (*SynchronizeTreeResponse) ProtoMessage() {} func (x *SynchronizeTreeResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[15] + mi := &file_pkg_services_control_service_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -931,7 +699,7 @@ func (x *SynchronizeTreeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SynchronizeTreeResponse.ProtoReflect.Descriptor instead. func (*SynchronizeTreeResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{15} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{11} } func (x *SynchronizeTreeResponse) GetBody() *SynchronizeTreeResponse_Body { @@ -961,7 +729,7 @@ type EvacuateShardRequest struct { func (x *EvacuateShardRequest) Reset() { *x = EvacuateShardRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[16] + mi := &file_pkg_services_control_service_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -974,7 +742,7 @@ func (x *EvacuateShardRequest) String() string { func (*EvacuateShardRequest) ProtoMessage() {} func (x *EvacuateShardRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[16] + mi := &file_pkg_services_control_service_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -987,7 +755,7 @@ func (x *EvacuateShardRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use EvacuateShardRequest.ProtoReflect.Descriptor instead. func (*EvacuateShardRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{16} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{12} } func (x *EvacuateShardRequest) GetBody() *EvacuateShardRequest_Body { @@ -1017,7 +785,7 @@ type EvacuateShardResponse struct { func (x *EvacuateShardResponse) Reset() { *x = EvacuateShardResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[17] + mi := &file_pkg_services_control_service_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1030,7 +798,7 @@ func (x *EvacuateShardResponse) String() string { func (*EvacuateShardResponse) ProtoMessage() {} func (x *EvacuateShardResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[17] + mi := &file_pkg_services_control_service_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1043,7 +811,7 @@ func (x *EvacuateShardResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use EvacuateShardResponse.ProtoReflect.Descriptor instead. func (*EvacuateShardResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{17} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{13} } func (x *EvacuateShardResponse) GetBody() *EvacuateShardResponse_Body { @@ -1073,7 +841,7 @@ type FlushCacheRequest struct { func (x *FlushCacheRequest) Reset() { *x = FlushCacheRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[18] + mi := &file_pkg_services_control_service_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1086,7 +854,7 @@ func (x *FlushCacheRequest) String() string { func (*FlushCacheRequest) ProtoMessage() {} func (x *FlushCacheRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[18] + mi := &file_pkg_services_control_service_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1099,7 +867,7 @@ func (x *FlushCacheRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FlushCacheRequest.ProtoReflect.Descriptor instead. func (*FlushCacheRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{14} } func (x *FlushCacheRequest) GetBody() *FlushCacheRequest_Body { @@ -1129,7 +897,7 @@ type FlushCacheResponse struct { func (x *FlushCacheResponse) Reset() { *x = FlushCacheResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[19] + mi := &file_pkg_services_control_service_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1142,7 +910,7 @@ func (x *FlushCacheResponse) String() string { func (*FlushCacheResponse) ProtoMessage() {} func (x *FlushCacheResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[19] + mi := &file_pkg_services_control_service_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1155,7 +923,7 @@ func (x *FlushCacheResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FlushCacheResponse.ProtoReflect.Descriptor instead. func (*FlushCacheResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{19} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{15} } func (x *FlushCacheResponse) GetBody() *FlushCacheResponse_Body { @@ -1185,7 +953,7 @@ type DoctorRequest struct { func (x *DoctorRequest) Reset() { *x = DoctorRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[20] + mi := &file_pkg_services_control_service_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1198,7 +966,7 @@ func (x *DoctorRequest) String() string { func (*DoctorRequest) ProtoMessage() {} func (x *DoctorRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[20] + mi := &file_pkg_services_control_service_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1211,7 +979,7 @@ func (x *DoctorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DoctorRequest.ProtoReflect.Descriptor instead. func (*DoctorRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{16} } func (x *DoctorRequest) GetBody() *DoctorRequest_Body { @@ -1241,7 +1009,7 @@ type DoctorResponse struct { func (x *DoctorResponse) Reset() { *x = DoctorResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[21] + mi := &file_pkg_services_control_service_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1254,7 +1022,7 @@ func (x *DoctorResponse) String() string { func (*DoctorResponse) ProtoMessage() {} func (x *DoctorResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[21] + mi := &file_pkg_services_control_service_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1267,7 +1035,7 @@ func (x *DoctorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DoctorResponse.ProtoReflect.Descriptor instead. func (*DoctorResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{17} } func (x *DoctorResponse) GetBody() *DoctorResponse_Body { @@ -1294,7 +1062,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[22] + mi := &file_pkg_services_control_service_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1307,7 +1075,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[22] + mi := &file_pkg_services_control_service_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1338,7 +1106,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[23] + mi := &file_pkg_services_control_service_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1351,7 +1119,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[23] + mi := &file_pkg_services_control_service_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1402,7 +1170,7 @@ type SetNetmapStatusRequest_Body struct { func (x *SetNetmapStatusRequest_Body) Reset() { *x = SetNetmapStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1415,7 +1183,7 @@ func (x *SetNetmapStatusRequest_Body) String() string { func (*SetNetmapStatusRequest_Body) ProtoMessage() {} func (x *SetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1455,7 +1223,7 @@ type SetNetmapStatusResponse_Body struct { func (x *SetNetmapStatusResponse_Body) Reset() { *x = SetNetmapStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1468,7 +1236,7 @@ func (x *SetNetmapStatusResponse_Body) String() string { func (*SetNetmapStatusResponse_Body) ProtoMessage() {} func (x *SetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1498,7 +1266,7 @@ type DropObjectsRequest_Body struct { func (x *DropObjectsRequest_Body) Reset() { *x = DropObjectsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1511,7 +1279,7 @@ func (x *DropObjectsRequest_Body) String() string { func (*DropObjectsRequest_Body) ProtoMessage() {} func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1544,7 +1312,7 @@ type DropObjectsResponse_Body struct { func (x *DropObjectsResponse_Body) Reset() { *x = DropObjectsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1557,7 +1325,7 @@ func (x *DropObjectsResponse_Body) String() string { func (*DropObjectsResponse_Body) ProtoMessage() {} func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1583,7 +1351,7 @@ type ListShardsRequest_Body struct { func (x *ListShardsRequest_Body) Reset() { *x = ListShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1596,7 +1364,7 @@ func (x *ListShardsRequest_Body) String() string { func (*ListShardsRequest_Body) ProtoMessage() {} func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1625,7 +1393,7 @@ type ListShardsResponse_Body struct { func (x *ListShardsResponse_Body) Reset() { *x = ListShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1638,7 +1406,7 @@ func (x *ListShardsResponse_Body) String() string { func (*ListShardsResponse_Body) ProtoMessage() {} func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1678,7 +1446,7 @@ type SetShardModeRequest_Body struct { func (x *SetShardModeRequest_Body) Reset() { *x = SetShardModeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1691,7 +1459,7 @@ func (x *SetShardModeRequest_Body) String() string { func (*SetShardModeRequest_Body) ProtoMessage() {} func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1738,7 +1506,7 @@ type SetShardModeResponse_Body struct { func (x *SetShardModeResponse_Body) Reset() { *x = SetShardModeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1751,7 +1519,7 @@ func (x *SetShardModeResponse_Body) String() string { func (*SetShardModeResponse_Body) ProtoMessage() {} func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1767,218 +1535,6 @@ func (*SetShardModeResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{9, 0} } -// Request body structure. -type DumpShardRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // ID of the shard. - Shard_ID []byte `protobuf:"bytes,1,opt,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` - // Path to the output. - Filepath string `protobuf:"bytes,2,opt,name=filepath,proto3" json:"filepath,omitempty"` - // Flag indicating whether object read errors should be ignored. - IgnoreErrors bool `protobuf:"varint,3,opt,name=ignore_errors,json=ignoreErrors,proto3" json:"ignore_errors,omitempty"` -} - -func (x *DumpShardRequest_Body) Reset() { - *x = DumpShardRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[32] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DumpShardRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DumpShardRequest_Body) ProtoMessage() {} - -func (x *DumpShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[32] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DumpShardRequest_Body.ProtoReflect.Descriptor instead. -func (*DumpShardRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{10, 0} -} - -func (x *DumpShardRequest_Body) GetShard_ID() []byte { - if x != nil { - return x.Shard_ID - } - return nil -} - -func (x *DumpShardRequest_Body) GetFilepath() string { - if x != nil { - return x.Filepath - } - return "" -} - -func (x *DumpShardRequest_Body) GetIgnoreErrors() bool { - if x != nil { - return x.IgnoreErrors - } - return false -} - -// Response body structure. -type DumpShardResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *DumpShardResponse_Body) Reset() { - *x = DumpShardResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[33] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DumpShardResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DumpShardResponse_Body) ProtoMessage() {} - -func (x *DumpShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[33] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DumpShardResponse_Body.ProtoReflect.Descriptor instead. -func (*DumpShardResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{11, 0} -} - -// Request body structure. -type RestoreShardRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // ID of the shard. - Shard_ID []byte `protobuf:"bytes,1,opt,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` - // Path to the output. - Filepath string `protobuf:"bytes,2,opt,name=filepath,proto3" json:"filepath,omitempty"` - // Flag indicating whether object read errors should be ignored. - IgnoreErrors bool `protobuf:"varint,3,opt,name=ignore_errors,json=ignoreErrors,proto3" json:"ignore_errors,omitempty"` -} - -func (x *RestoreShardRequest_Body) Reset() { - *x = RestoreShardRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[34] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RestoreShardRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RestoreShardRequest_Body) ProtoMessage() {} - -func (x *RestoreShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[34] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RestoreShardRequest_Body.ProtoReflect.Descriptor instead. -func (*RestoreShardRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{12, 0} -} - -func (x *RestoreShardRequest_Body) GetShard_ID() []byte { - if x != nil { - return x.Shard_ID - } - return nil -} - -func (x *RestoreShardRequest_Body) GetFilepath() string { - if x != nil { - return x.Filepath - } - return "" -} - -func (x *RestoreShardRequest_Body) GetIgnoreErrors() bool { - if x != nil { - return x.IgnoreErrors - } - return false -} - -// Response body structure. -type RestoreShardResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *RestoreShardResponse_Body) Reset() { - *x = RestoreShardResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[35] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RestoreShardResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RestoreShardResponse_Body) ProtoMessage() {} - -func (x *RestoreShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[35] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RestoreShardResponse_Body.ProtoReflect.Descriptor instead. -func (*RestoreShardResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{13, 0} -} - // Request body structure. type SynchronizeTreeRequest_Body struct { state protoimpl.MessageState @@ -1994,7 +1550,7 @@ type SynchronizeTreeRequest_Body struct { func (x *SynchronizeTreeRequest_Body) Reset() { *x = SynchronizeTreeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2007,7 +1563,7 @@ func (x *SynchronizeTreeRequest_Body) String() string { func (*SynchronizeTreeRequest_Body) ProtoMessage() {} func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2020,7 +1576,7 @@ func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SynchronizeTreeRequest_Body.ProtoReflect.Descriptor instead. func (*SynchronizeTreeRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{14, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{10, 0} } func (x *SynchronizeTreeRequest_Body) GetContainerId() []byte { @@ -2054,7 +1610,7 @@ type SynchronizeTreeResponse_Body struct { func (x *SynchronizeTreeResponse_Body) Reset() { *x = SynchronizeTreeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2067,7 +1623,7 @@ func (x *SynchronizeTreeResponse_Body) String() string { func (*SynchronizeTreeResponse_Body) ProtoMessage() {} func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2080,7 +1636,7 @@ func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SynchronizeTreeResponse_Body.ProtoReflect.Descriptor instead. func (*SynchronizeTreeResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{15, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{11, 0} } // Request body structure. @@ -2098,7 +1654,7 @@ type EvacuateShardRequest_Body struct { func (x *EvacuateShardRequest_Body) Reset() { *x = EvacuateShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2111,7 +1667,7 @@ func (x *EvacuateShardRequest_Body) String() string { func (*EvacuateShardRequest_Body) ProtoMessage() {} func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2124,7 +1680,7 @@ func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use EvacuateShardRequest_Body.ProtoReflect.Descriptor instead. func (*EvacuateShardRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{16, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{12, 0} } func (x *EvacuateShardRequest_Body) GetShard_ID() [][]byte { @@ -2153,7 +1709,7 @@ type EvacuateShardResponse_Body struct { func (x *EvacuateShardResponse_Body) Reset() { *x = EvacuateShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2166,7 +1722,7 @@ func (x *EvacuateShardResponse_Body) String() string { func (*EvacuateShardResponse_Body) ProtoMessage() {} func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2179,7 +1735,7 @@ func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use EvacuateShardResponse_Body.ProtoReflect.Descriptor instead. func (*EvacuateShardResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{17, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{13, 0} } func (x *EvacuateShardResponse_Body) GetCount() uint32 { @@ -2202,7 +1758,7 @@ type FlushCacheRequest_Body struct { func (x *FlushCacheRequest_Body) Reset() { *x = FlushCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2215,7 +1771,7 @@ func (x *FlushCacheRequest_Body) String() string { func (*FlushCacheRequest_Body) ProtoMessage() {} func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2228,7 +1784,7 @@ func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use FlushCacheRequest_Body.ProtoReflect.Descriptor instead. func (*FlushCacheRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{14, 0} } func (x *FlushCacheRequest_Body) GetShard_ID() [][]byte { @@ -2248,7 +1804,7 @@ type FlushCacheResponse_Body struct { func (x *FlushCacheResponse_Body) Reset() { *x = FlushCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2261,7 +1817,7 @@ func (x *FlushCacheResponse_Body) String() string { func (*FlushCacheResponse_Body) ProtoMessage() {} func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2274,7 +1830,7 @@ func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use FlushCacheResponse_Body.ProtoReflect.Descriptor instead. func (*FlushCacheResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{19, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{15, 0} } // Request body structure. @@ -2292,7 +1848,7 @@ type DoctorRequest_Body struct { func (x *DoctorRequest_Body) Reset() { *x = DoctorRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2305,7 +1861,7 @@ func (x *DoctorRequest_Body) String() string { func (*DoctorRequest_Body) ProtoMessage() {} func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2318,7 +1874,7 @@ func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DoctorRequest_Body.ProtoReflect.Descriptor instead. func (*DoctorRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{16, 0} } func (x *DoctorRequest_Body) GetConcurrency() uint32 { @@ -2345,7 +1901,7 @@ type DoctorResponse_Body struct { func (x *DoctorResponse_Body) Reset() { *x = DoctorResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2358,7 +1914,7 @@ func (x *DoctorResponse_Body) String() string { func (*DoctorResponse_Body) ProtoMessage() {} func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2371,7 +1927,7 @@ func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DoctorResponse_Body.ProtoReflect.Descriptor instead. func (*DoctorResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{17, 0} } var File_pkg_services_control_service_proto protoreflect.FileDescriptor @@ -2493,194 +2049,140 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x22, 0xdc, 0x01, 0x0a, 0x10, 0x44, 0x75, 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x32, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x44, 0x75, 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x62, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, - 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, - 0x44, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x70, 0x61, 0x74, 0x68, 0x12, 0x23, 0x0a, - 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, - 0x72, 0x73, 0x22, 0x82, 0x01, 0x0a, 0x11, 0x44, 0x75, 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x44, 0x75, 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x6f, 0x64, 0x79, 0x22, 0xe0, 0x01, 0x0a, 0x16, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, + 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, + 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x5a, 0x0a, 0x04, 0x42, 0x6f, + 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x16, + 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, + 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x8e, 0x01, 0x0a, 0x17, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x39, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xe2, 0x01, 0x0a, 0x13, 0x52, 0x65, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x62, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x1a, 0x0a, 0x08, 0x66, - 0x69, 0x6c, 0x65, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, - 0x69, 0x6c, 0x65, 0x70, 0x61, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, - 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, - 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0x88, 0x01, 0x0a, - 0x14, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xe0, 0x01, 0x0a, 0x16, 0x53, 0x79, 0x6e, 0x63, - 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x5a, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, - 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, - 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x8e, 0x01, 0x0a, 0x17, 0x53, - 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xc8, 0x01, 0x0a, 0x14, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, - 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, - 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, - 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0xa0, 0x01, 0x0a, 0x15, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, + 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xc8, 0x01, 0x0a, 0x14, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x9d, 0x01, 0x0a, 0x11, 0x46, 0x6c, - 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x33, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, - 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, - 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, - 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x84, 0x01, 0x0a, 0x12, 0x46, 0x6c, - 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x22, 0xc9, 0x01, 0x0a, 0x0d, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, - 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x55, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x20, 0x0a, - 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, - 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x64, 0x75, 0x70, 0x6c, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x44, 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x22, 0x7c, 0x0a, 0x0e, - 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, + 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, + 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, + 0x72, 0x73, 0x22, 0xa0, 0x01, 0x0a, 0x15, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x9d, 0x01, 0x0a, 0x11, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, + 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x33, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xc7, 0x06, 0x0a, 0x0e, 0x43, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, - 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, - 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, - 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, - 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, - 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x09, 0x44, - 0x75, 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x19, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x44, 0x75, 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x75, - 0x6d, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4b, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, - 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, - 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, - 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, - 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, - 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, - 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, - 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, - 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, - 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, + 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, + 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x84, 0x01, 0x0a, 0x12, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, + 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xc9, 0x01, 0x0a, + 0x0d, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x55, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, + 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, + 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x72, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x64, 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x44, 0x75, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x22, 0x7c, 0x0a, 0x0e, 0x44, 0x6f, 0x63, 0x74, + 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb6, 0x05, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, + 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, + 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, + 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, + 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, + 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, + 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, + 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, + 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, + 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2695,7 +2197,7 @@ func file_pkg_services_control_service_proto_rawDescGZIP() []byte { return file_pkg_services_control_service_proto_rawDescData } -var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 44) +var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 36) var file_pkg_services_control_service_proto_goTypes = []interface{}{ (*HealthCheckRequest)(nil), // 0: control.HealthCheckRequest (*HealthCheckResponse)(nil), // 1: control.HealthCheckResponse @@ -2707,123 +2209,103 @@ var file_pkg_services_control_service_proto_goTypes = []interface{}{ (*ListShardsResponse)(nil), // 7: control.ListShardsResponse (*SetShardModeRequest)(nil), // 8: control.SetShardModeRequest (*SetShardModeResponse)(nil), // 9: control.SetShardModeResponse - (*DumpShardRequest)(nil), // 10: control.DumpShardRequest - (*DumpShardResponse)(nil), // 11: control.DumpShardResponse - (*RestoreShardRequest)(nil), // 12: control.RestoreShardRequest - (*RestoreShardResponse)(nil), // 13: control.RestoreShardResponse - (*SynchronizeTreeRequest)(nil), // 14: control.SynchronizeTreeRequest - (*SynchronizeTreeResponse)(nil), // 15: control.SynchronizeTreeResponse - (*EvacuateShardRequest)(nil), // 16: control.EvacuateShardRequest - (*EvacuateShardResponse)(nil), // 17: control.EvacuateShardResponse - (*FlushCacheRequest)(nil), // 18: control.FlushCacheRequest - (*FlushCacheResponse)(nil), // 19: control.FlushCacheResponse - (*DoctorRequest)(nil), // 20: control.DoctorRequest - (*DoctorResponse)(nil), // 21: control.DoctorResponse - (*HealthCheckRequest_Body)(nil), // 22: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 23: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 24: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 25: control.SetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 26: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 27: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 28: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 29: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 30: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 31: control.SetShardModeResponse.Body - (*DumpShardRequest_Body)(nil), // 32: control.DumpShardRequest.Body - (*DumpShardResponse_Body)(nil), // 33: control.DumpShardResponse.Body - (*RestoreShardRequest_Body)(nil), // 34: control.RestoreShardRequest.Body - (*RestoreShardResponse_Body)(nil), // 35: control.RestoreShardResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 36: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 37: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 38: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 39: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 40: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 41: control.FlushCacheResponse.Body - (*DoctorRequest_Body)(nil), // 42: control.DoctorRequest.Body - (*DoctorResponse_Body)(nil), // 43: control.DoctorResponse.Body - (*Signature)(nil), // 44: control.Signature - (NetmapStatus)(0), // 45: control.NetmapStatus - (HealthStatus)(0), // 46: control.HealthStatus - (*ShardInfo)(nil), // 47: control.ShardInfo - (ShardMode)(0), // 48: control.ShardMode + (*SynchronizeTreeRequest)(nil), // 10: control.SynchronizeTreeRequest + (*SynchronizeTreeResponse)(nil), // 11: control.SynchronizeTreeResponse + (*EvacuateShardRequest)(nil), // 12: control.EvacuateShardRequest + (*EvacuateShardResponse)(nil), // 13: control.EvacuateShardResponse + (*FlushCacheRequest)(nil), // 14: control.FlushCacheRequest + (*FlushCacheResponse)(nil), // 15: control.FlushCacheResponse + (*DoctorRequest)(nil), // 16: control.DoctorRequest + (*DoctorResponse)(nil), // 17: control.DoctorResponse + (*HealthCheckRequest_Body)(nil), // 18: control.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 19: control.HealthCheckResponse.Body + (*SetNetmapStatusRequest_Body)(nil), // 20: control.SetNetmapStatusRequest.Body + (*SetNetmapStatusResponse_Body)(nil), // 21: control.SetNetmapStatusResponse.Body + (*DropObjectsRequest_Body)(nil), // 22: control.DropObjectsRequest.Body + (*DropObjectsResponse_Body)(nil), // 23: control.DropObjectsResponse.Body + (*ListShardsRequest_Body)(nil), // 24: control.ListShardsRequest.Body + (*ListShardsResponse_Body)(nil), // 25: control.ListShardsResponse.Body + (*SetShardModeRequest_Body)(nil), // 26: control.SetShardModeRequest.Body + (*SetShardModeResponse_Body)(nil), // 27: control.SetShardModeResponse.Body + (*SynchronizeTreeRequest_Body)(nil), // 28: control.SynchronizeTreeRequest.Body + (*SynchronizeTreeResponse_Body)(nil), // 29: control.SynchronizeTreeResponse.Body + (*EvacuateShardRequest_Body)(nil), // 30: control.EvacuateShardRequest.Body + (*EvacuateShardResponse_Body)(nil), // 31: control.EvacuateShardResponse.Body + (*FlushCacheRequest_Body)(nil), // 32: control.FlushCacheRequest.Body + (*FlushCacheResponse_Body)(nil), // 33: control.FlushCacheResponse.Body + (*DoctorRequest_Body)(nil), // 34: control.DoctorRequest.Body + (*DoctorResponse_Body)(nil), // 35: control.DoctorResponse.Body + (*Signature)(nil), // 36: control.Signature + (NetmapStatus)(0), // 37: control.NetmapStatus + (HealthStatus)(0), // 38: control.HealthStatus + (*ShardInfo)(nil), // 39: control.ShardInfo + (ShardMode)(0), // 40: control.ShardMode } var file_pkg_services_control_service_proto_depIdxs = []int32{ - 22, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 44, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 23, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 44, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 24, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 44, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 25, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 44, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 26, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 44, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature - 27, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 44, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature - 28, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 44, // 13: control.ListShardsRequest.signature:type_name -> control.Signature - 29, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 44, // 15: control.ListShardsResponse.signature:type_name -> control.Signature - 30, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 44, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature - 31, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 44, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature - 32, // 20: control.DumpShardRequest.body:type_name -> control.DumpShardRequest.Body - 44, // 21: control.DumpShardRequest.signature:type_name -> control.Signature - 33, // 22: control.DumpShardResponse.body:type_name -> control.DumpShardResponse.Body - 44, // 23: control.DumpShardResponse.signature:type_name -> control.Signature - 34, // 24: control.RestoreShardRequest.body:type_name -> control.RestoreShardRequest.Body - 44, // 25: control.RestoreShardRequest.signature:type_name -> control.Signature - 35, // 26: control.RestoreShardResponse.body:type_name -> control.RestoreShardResponse.Body - 44, // 27: control.RestoreShardResponse.signature:type_name -> control.Signature - 36, // 28: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 44, // 29: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 37, // 30: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 44, // 31: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 38, // 32: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 44, // 33: control.EvacuateShardRequest.signature:type_name -> control.Signature - 39, // 34: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 44, // 35: control.EvacuateShardResponse.signature:type_name -> control.Signature - 40, // 36: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 44, // 37: control.FlushCacheRequest.signature:type_name -> control.Signature - 41, // 38: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 44, // 39: control.FlushCacheResponse.signature:type_name -> control.Signature - 42, // 40: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body - 44, // 41: control.DoctorRequest.signature:type_name -> control.Signature - 43, // 42: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body - 44, // 43: control.DoctorResponse.signature:type_name -> control.Signature - 45, // 44: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 46, // 45: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 45, // 46: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 47, // 47: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 48, // 48: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 0, // 49: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 2, // 50: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 4, // 51: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 6, // 52: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 8, // 53: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 10, // 54: control.ControlService.DumpShard:input_type -> control.DumpShardRequest - 12, // 55: control.ControlService.RestoreShard:input_type -> control.RestoreShardRequest - 14, // 56: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 16, // 57: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 18, // 58: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 20, // 59: control.ControlService.Doctor:input_type -> control.DoctorRequest - 1, // 60: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 3, // 61: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 5, // 62: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 7, // 63: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 9, // 64: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 11, // 65: control.ControlService.DumpShard:output_type -> control.DumpShardResponse - 13, // 66: control.ControlService.RestoreShard:output_type -> control.RestoreShardResponse - 15, // 67: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 17, // 68: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 19, // 69: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 21, // 70: control.ControlService.Doctor:output_type -> control.DoctorResponse - 60, // [60:71] is the sub-list for method output_type - 49, // [49:60] is the sub-list for method input_type - 49, // [49:49] is the sub-list for extension type_name - 49, // [49:49] is the sub-list for extension extendee - 0, // [0:49] is the sub-list for field type_name + 18, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body + 36, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature + 19, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body + 36, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature + 20, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body + 36, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature + 21, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body + 36, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature + 22, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body + 36, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature + 23, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body + 36, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature + 24, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body + 36, // 13: control.ListShardsRequest.signature:type_name -> control.Signature + 25, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body + 36, // 15: control.ListShardsResponse.signature:type_name -> control.Signature + 26, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body + 36, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature + 27, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body + 36, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature + 28, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body + 36, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature + 29, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body + 36, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature + 30, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body + 36, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature + 31, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body + 36, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature + 32, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body + 36, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature + 33, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body + 36, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature + 34, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body + 36, // 33: control.DoctorRequest.signature:type_name -> control.Signature + 35, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body + 36, // 35: control.DoctorResponse.signature:type_name -> control.Signature + 37, // 36: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus + 38, // 37: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus + 37, // 38: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus + 39, // 39: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo + 40, // 40: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode + 0, // 41: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 2, // 42: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 4, // 43: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 6, // 44: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 8, // 45: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 10, // 46: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 12, // 47: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 14, // 48: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 16, // 49: control.ControlService.Doctor:input_type -> control.DoctorRequest + 1, // 50: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 3, // 51: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 5, // 52: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 7, // 53: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 9, // 54: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 11, // 55: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 13, // 56: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 15, // 57: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 17, // 58: control.ControlService.Doctor:output_type -> control.DoctorResponse + 50, // [50:59] is the sub-list for method output_type + 41, // [41:50] is the sub-list for method input_type + 41, // [41:41] is the sub-list for extension type_name + 41, // [41:41] is the sub-list for extension extendee + 0, // [0:41] is the sub-list for field type_name } func init() { file_pkg_services_control_service_proto_init() } @@ -2954,54 +2436,6 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DumpShardRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DumpShardResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreShardRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreShardResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SynchronizeTreeRequest); i { case 0: return &v.state @@ -3013,7 +2447,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SynchronizeTreeResponse); i { case 0: return &v.state @@ -3025,7 +2459,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EvacuateShardRequest); i { case 0: return &v.state @@ -3037,7 +2471,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EvacuateShardResponse); i { case 0: return &v.state @@ -3049,7 +2483,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*FlushCacheRequest); i { case 0: return &v.state @@ -3061,7 +2495,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*FlushCacheResponse); i { case 0: return &v.state @@ -3073,7 +2507,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DoctorRequest); i { case 0: return &v.state @@ -3085,7 +2519,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DoctorResponse); i { case 0: return &v.state @@ -3097,7 +2531,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state @@ -3109,7 +2543,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state @@ -3121,7 +2555,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SetNetmapStatusRequest_Body); i { case 0: return &v.state @@ -3133,7 +2567,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SetNetmapStatusResponse_Body); i { case 0: return &v.state @@ -3145,7 +2579,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DropObjectsRequest_Body); i { case 0: return &v.state @@ -3157,7 +2591,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DropObjectsResponse_Body); i { case 0: return &v.state @@ -3169,7 +2603,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListShardsRequest_Body); i { case 0: return &v.state @@ -3181,7 +2615,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListShardsResponse_Body); i { case 0: return &v.state @@ -3193,7 +2627,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SetShardModeRequest_Body); i { case 0: return &v.state @@ -3205,7 +2639,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SetShardModeResponse_Body); i { case 0: return &v.state @@ -3217,55 +2651,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DumpShardRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DumpShardResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreShardRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestoreShardResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SynchronizeTreeRequest_Body); i { case 0: return &v.state @@ -3277,7 +2663,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SynchronizeTreeResponse_Body); i { case 0: return &v.state @@ -3289,7 +2675,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EvacuateShardRequest_Body); i { case 0: return &v.state @@ -3301,7 +2687,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*EvacuateShardResponse_Body); i { case 0: return &v.state @@ -3313,7 +2699,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*FlushCacheRequest_Body); i { case 0: return &v.state @@ -3325,7 +2711,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*FlushCacheResponse_Body); i { case 0: return &v.state @@ -3337,7 +2723,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DoctorRequest_Body); i { case 0: return &v.state @@ -3349,7 +2735,7 @@ func file_pkg_services_control_service_proto_init() { return nil } } - file_pkg_services_control_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { + file_pkg_services_control_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DoctorResponse_Body); i { case 0: return &v.state @@ -3368,7 +2754,7 @@ func file_pkg_services_control_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_service_proto_rawDesc, NumEnums: 0, - NumMessages: 44, + NumMessages: 36, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 7c661e661..32a87c744 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -23,12 +23,6 @@ service ControlService { // Sets mode of the shard. rpc SetShardMode (SetShardModeRequest) returns (SetShardModeResponse); - // Dump objects from the shard. - rpc DumpShard (DumpShardRequest) returns (DumpShardResponse); - - // Restore objects from dump. - rpc RestoreShard (RestoreShardRequest) returns (RestoreShardResponse); - // Synchronizes all log operations for the specified tree. rpc SynchronizeTree (SynchronizeTreeRequest) returns (SynchronizeTreeResponse); @@ -201,75 +195,6 @@ message SetShardModeResponse { Signature signature = 2; } -// DumpShard request. -message DumpShardRequest { - // Request body structure. - message Body { - // ID of the shard. - bytes shard_ID = 1; - - // Path to the output. - string filepath = 2; - - // Flag indicating whether object read errors should be ignored. - bool ignore_errors = 3; - } - - // Body of dump shard request message. - Body body = 1; - - // Body signature. - Signature signature = 2; -} - -// DumpShard response. -message DumpShardResponse { - // Response body structure. - message Body { - } - - // Body of dump shard response message. - Body body = 1; - - // Body signature. - Signature signature = 2; -} - - -// RestoreShard request. -message RestoreShardRequest { - // Request body structure. - message Body { - // ID of the shard. - bytes shard_ID = 1; - - // Path to the output. - string filepath = 2; - - // Flag indicating whether object read errors should be ignored. - bool ignore_errors = 3; - } - - // Body of restore shard request message. - Body body = 1; - - // Body signature. - Signature signature = 2; -} - -// RestoreShard response. -message RestoreShardResponse { - // Response body structure. - message Body { - } - - // Body of restore shard response message. - Body body = 1; - - // Body signature. - Signature signature = 2; -} - // SynchronizeTree request. message SynchronizeTreeRequest { // Request body structure. diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 0f50d5893..b9b865a90 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -771,316 +771,6 @@ func (x *SetShardModeResponse) SetSignature(sig *Signature) { x.Signature = sig } -// StableSize returns the size of x in protobuf format. -// -// Structures with the same field values have the same binary size. -func (x *DumpShardRequest_Body) StableSize() (size int) { - size += proto.BytesSize(1, x.Shard_ID) - size += proto.StringSize(2, x.Filepath) - size += proto.BoolSize(3, x.IgnoreErrors) - return size -} - -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DumpShardRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.Shard_ID) - offset += proto.StringMarshal(2, buf[offset:], x.Filepath) - offset += proto.BoolMarshal(3, buf[offset:], x.IgnoreErrors) - return buf -} - -// StableSize returns the size of x in protobuf format. -// -// Structures with the same field values have the same binary size. -func (x *DumpShardRequest) StableSize() (size int) { - size += proto.NestedStructureSize(1, x.Body) - size += proto.NestedStructureSize(2, x.Signature) - return size -} - -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DumpShardRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - -// ReadSignedData fills buf with signed data of x. -// If buffer length is less than x.SignedDataSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same signed data. -func (x *DumpShardRequest) SignedDataSize() int { - return x.GetBody().StableSize() -} - -// SignedDataSize returns size of the request signed data in bytes. -// -// Structures with the same field values have the same signed data size. -func (x *DumpShardRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil -} - -func (x *DumpShardRequest) SetSignature(sig *Signature) { - x.Signature = sig -} - -// StableSize returns the size of x in protobuf format. -// -// Structures with the same field values have the same binary size. -func (x *DumpShardResponse_Body) StableSize() (size int) { - return size -} - -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DumpShardResponse_Body) StableMarshal(buf []byte) []byte { - return buf -} - -// StableSize returns the size of x in protobuf format. -// -// Structures with the same field values have the same binary size. -func (x *DumpShardResponse) StableSize() (size int) { - size += proto.NestedStructureSize(1, x.Body) - size += proto.NestedStructureSize(2, x.Signature) - return size -} - -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DumpShardResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - -// ReadSignedData fills buf with signed data of x. -// If buffer length is less than x.SignedDataSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same signed data. -func (x *DumpShardResponse) SignedDataSize() int { - return x.GetBody().StableSize() -} - -// SignedDataSize returns size of the request signed data in bytes. -// -// Structures with the same field values have the same signed data size. -func (x *DumpShardResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil -} - -func (x *DumpShardResponse) SetSignature(sig *Signature) { - x.Signature = sig -} - -// StableSize returns the size of x in protobuf format. -// -// Structures with the same field values have the same binary size. -func (x *RestoreShardRequest_Body) StableSize() (size int) { - size += proto.BytesSize(1, x.Shard_ID) - size += proto.StringSize(2, x.Filepath) - size += proto.BoolSize(3, x.IgnoreErrors) - return size -} - -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RestoreShardRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.Shard_ID) - offset += proto.StringMarshal(2, buf[offset:], x.Filepath) - offset += proto.BoolMarshal(3, buf[offset:], x.IgnoreErrors) - return buf -} - -// StableSize returns the size of x in protobuf format. -// -// Structures with the same field values have the same binary size. -func (x *RestoreShardRequest) StableSize() (size int) { - size += proto.NestedStructureSize(1, x.Body) - size += proto.NestedStructureSize(2, x.Signature) - return size -} - -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RestoreShardRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - -// ReadSignedData fills buf with signed data of x. -// If buffer length is less than x.SignedDataSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same signed data. -func (x *RestoreShardRequest) SignedDataSize() int { - return x.GetBody().StableSize() -} - -// SignedDataSize returns size of the request signed data in bytes. -// -// Structures with the same field values have the same signed data size. -func (x *RestoreShardRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil -} - -func (x *RestoreShardRequest) SetSignature(sig *Signature) { - x.Signature = sig -} - -// StableSize returns the size of x in protobuf format. -// -// Structures with the same field values have the same binary size. -func (x *RestoreShardResponse_Body) StableSize() (size int) { - return size -} - -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RestoreShardResponse_Body) StableMarshal(buf []byte) []byte { - return buf -} - -// StableSize returns the size of x in protobuf format. -// -// Structures with the same field values have the same binary size. -func (x *RestoreShardResponse) StableSize() (size int) { - size += proto.NestedStructureSize(1, x.Body) - size += proto.NestedStructureSize(2, x.Signature) - return size -} - -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RestoreShardResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - -// ReadSignedData fills buf with signed data of x. -// If buffer length is less than x.SignedDataSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same signed data. -func (x *RestoreShardResponse) SignedDataSize() int { - return x.GetBody().StableSize() -} - -// SignedDataSize returns size of the request signed data in bytes. -// -// Structures with the same field values have the same signed data size. -func (x *RestoreShardResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil -} - -func (x *RestoreShardResponse) SetSignature(sig *Signature) { - x.Signature = sig -} - // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 4a4fbeac1..1e8dd9e3c 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -24,8 +24,6 @@ const ( ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects" ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" - ControlService_DumpShard_FullMethodName = "/control.ControlService/DumpShard" - ControlService_RestoreShard_FullMethodName = "/control.ControlService/RestoreShard" ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree" ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard" ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache" @@ -46,10 +44,6 @@ type ControlServiceClient interface { ListShards(ctx context.Context, in *ListShardsRequest, opts ...grpc.CallOption) (*ListShardsResponse, error) // Sets mode of the shard. SetShardMode(ctx context.Context, in *SetShardModeRequest, opts ...grpc.CallOption) (*SetShardModeResponse, error) - // Dump objects from the shard. - DumpShard(ctx context.Context, in *DumpShardRequest, opts ...grpc.CallOption) (*DumpShardResponse, error) - // Restore objects from dump. - RestoreShard(ctx context.Context, in *RestoreShardRequest, opts ...grpc.CallOption) (*RestoreShardResponse, error) // Synchronizes all log operations for the specified tree. SynchronizeTree(ctx context.Context, in *SynchronizeTreeRequest, opts ...grpc.CallOption) (*SynchronizeTreeResponse, error) // EvacuateShard moves all data from one shard to the others. @@ -113,24 +107,6 @@ func (c *controlServiceClient) SetShardMode(ctx context.Context, in *SetShardMod return out, nil } -func (c *controlServiceClient) DumpShard(ctx context.Context, in *DumpShardRequest, opts ...grpc.CallOption) (*DumpShardResponse, error) { - out := new(DumpShardResponse) - err := c.cc.Invoke(ctx, ControlService_DumpShard_FullMethodName, in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *controlServiceClient) RestoreShard(ctx context.Context, in *RestoreShardRequest, opts ...grpc.CallOption) (*RestoreShardResponse, error) { - out := new(RestoreShardResponse) - err := c.cc.Invoke(ctx, ControlService_RestoreShard_FullMethodName, in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *controlServiceClient) SynchronizeTree(ctx context.Context, in *SynchronizeTreeRequest, opts ...grpc.CallOption) (*SynchronizeTreeResponse, error) { out := new(SynchronizeTreeResponse) err := c.cc.Invoke(ctx, ControlService_SynchronizeTree_FullMethodName, in, out, opts...) @@ -181,10 +157,6 @@ type ControlServiceServer interface { ListShards(context.Context, *ListShardsRequest) (*ListShardsResponse, error) // Sets mode of the shard. SetShardMode(context.Context, *SetShardModeRequest) (*SetShardModeResponse, error) - // Dump objects from the shard. - DumpShard(context.Context, *DumpShardRequest) (*DumpShardResponse, error) - // Restore objects from dump. - RestoreShard(context.Context, *RestoreShardRequest) (*RestoreShardResponse, error) // Synchronizes all log operations for the specified tree. SynchronizeTree(context.Context, *SynchronizeTreeRequest) (*SynchronizeTreeResponse, error) // EvacuateShard moves all data from one shard to the others. @@ -214,12 +186,6 @@ func (UnimplementedControlServiceServer) ListShards(context.Context, *ListShards func (UnimplementedControlServiceServer) SetShardMode(context.Context, *SetShardModeRequest) (*SetShardModeResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method SetShardMode not implemented") } -func (UnimplementedControlServiceServer) DumpShard(context.Context, *DumpShardRequest) (*DumpShardResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DumpShard not implemented") -} -func (UnimplementedControlServiceServer) RestoreShard(context.Context, *RestoreShardRequest) (*RestoreShardResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method RestoreShard not implemented") -} func (UnimplementedControlServiceServer) SynchronizeTree(context.Context, *SynchronizeTreeRequest) (*SynchronizeTreeResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method SynchronizeTree not implemented") } @@ -334,42 +300,6 @@ func _ControlService_SetShardMode_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } -func _ControlService_DumpShard_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DumpShardRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ControlServiceServer).DumpShard(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: ControlService_DumpShard_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ControlServiceServer).DumpShard(ctx, req.(*DumpShardRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _ControlService_RestoreShard_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(RestoreShardRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ControlServiceServer).RestoreShard(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: ControlService_RestoreShard_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ControlServiceServer).RestoreShard(ctx, req.(*RestoreShardRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _ControlService_SynchronizeTree_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(SynchronizeTreeRequest) if err := dec(in); err != nil { @@ -469,14 +399,6 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "SetShardMode", Handler: _ControlService_SetShardMode_Handler, }, - { - MethodName: "DumpShard", - Handler: _ControlService_DumpShard_Handler, - }, - { - MethodName: "RestoreShard", - Handler: _ControlService_RestoreShard_Handler, - }, { MethodName: "SynchronizeTree", Handler: _ControlService_SynchronizeTree_Handler, From 0b42a00a60a5d5e1fd8da386c910296f1ae19647 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 14 Apr 2023 14:22:23 +0300 Subject: [PATCH 0120/1943] [#254] innerring: Remove unused TimersHandlers() method from processors Signed-off-by: Evgenii Stratonikov --- pkg/innerring/bindings.go | 1 - pkg/innerring/processors/alphabet/processor.go | 5 ----- pkg/innerring/processors/audit/processor.go | 5 ----- pkg/innerring/processors/balance/processor.go | 5 ----- pkg/innerring/processors/container/processor.go | 5 ----- pkg/innerring/processors/frostfs/processor.go | 5 ----- pkg/innerring/processors/governance/processor.go | 5 ----- pkg/innerring/processors/netmap/processor.go | 5 ----- 8 files changed, 36 deletions(-) diff --git a/pkg/innerring/bindings.go b/pkg/innerring/bindings.go index 0e10125c3..c4de07a5f 100644 --- a/pkg/innerring/bindings.go +++ b/pkg/innerring/bindings.go @@ -12,7 +12,6 @@ type ( ListenerNotificationHandlers() []event.NotificationHandlerInfo ListenerNotaryParsers() []event.NotaryParserInfo ListenerNotaryHandlers() []event.NotaryHandlerInfo - TimersHandlers() []event.NotificationHandlerInfo } ) diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index 79b61f14f..169bfb3e2 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -106,8 +106,3 @@ func (ap *Processor) ListenerNotaryParsers() []event.NotaryParserInfo { func (ap *Processor) ListenerNotaryHandlers() []event.NotaryHandlerInfo { return nil } - -// TimersHandlers for the 'Timers' event producer. -func (ap *Processor) TimersHandlers() []event.NotificationHandlerInfo { - return nil -} diff --git a/pkg/innerring/processors/audit/processor.go b/pkg/innerring/processors/audit/processor.go index 31e8a8c55..6e0a98209 100644 --- a/pkg/innerring/processors/audit/processor.go +++ b/pkg/innerring/processors/audit/processor.go @@ -130,11 +130,6 @@ func (ap *Processor) ListenerNotificationHandlers() []event.NotificationHandlerI return nil } -// TimersHandlers for the 'Timers' event producer. -func (ap *Processor) TimersHandlers() []event.NotificationHandlerInfo { - return nil -} - // StartAuditHandler for the internal event producer. func (ap *Processor) StartAuditHandler() event.Handler { return ap.handleNewAuditRound diff --git a/pkg/innerring/processors/balance/processor.go b/pkg/innerring/processors/balance/processor.go index 370d06f44..7ae639e89 100644 --- a/pkg/innerring/processors/balance/processor.go +++ b/pkg/innerring/processors/balance/processor.go @@ -115,8 +115,3 @@ func (bp *Processor) ListenerNotaryParsers() []event.NotaryParserInfo { func (bp *Processor) ListenerNotaryHandlers() []event.NotaryHandlerInfo { return nil } - -// TimersHandlers for the 'Timers' event producer. -func (bp *Processor) TimersHandlers() []event.NotificationHandlerInfo { - return nil -} diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index 123ba77b8..56d2eee96 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -238,8 +238,3 @@ func (cp *Processor) ListenerNotaryHandlers() []event.NotaryHandlerInfo { return hh } - -// TimersHandlers for the 'Timers' event producer. -func (cp *Processor) TimersHandlers() []event.NotificationHandlerInfo { - return nil -} diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index 4d5bdee78..42362eeed 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -226,8 +226,3 @@ func (np *Processor) ListenerNotaryParsers() []event.NotaryParserInfo { func (np *Processor) ListenerNotaryHandlers() []event.NotaryHandlerInfo { return nil } - -// TimersHandlers for the 'Timers' event producer. -func (np *Processor) TimersHandlers() []event.NotificationHandlerInfo { - return nil -} diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index 9397186ee..b65dd17b7 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -161,8 +161,3 @@ func (gp *Processor) ListenerNotaryParsers() []event.NotaryParserInfo { func (gp *Processor) ListenerNotaryHandlers() []event.NotaryHandlerInfo { return nil } - -// TimersHandlers for the 'Timers' event producer. -func (gp *Processor) TimersHandlers() []event.NotificationHandlerInfo { - return nil -} diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index 85a123ef3..035ccfafd 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -297,8 +297,3 @@ func (np *Processor) ListenerNotaryHandlers() []event.NotaryHandlerInfo { return hh } - -// TimersHandlers for the 'Timers' event producer. -func (np *Processor) TimersHandlers() []event.NotificationHandlerInfo { - return nil -} From 262c9c2b93dbb38c3353800224fd9bdaa6c0fa05 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 14 Apr 2023 16:42:20 +0300 Subject: [PATCH 0121/1943] [#256] blobovniczaTree: Make `Exists` test stable Corrupt and request _the same_ file. Signed-off-by: Pavel Karpy --- .../blobstor/blobovniczatree/exists_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index ff927ccbb..8d9fe526e 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -50,16 +50,15 @@ func TestExistsInvalidStorageID(t *testing.T) { require.False(t, res.Exists) }) - t.Run("invalid storage id", func(t *testing.T) { - storageID := slice.Copy(putRes.StorageID) - storageID[0] = '9' + t.Run("valid id but corrupted file", func(t *testing.T) { + relBadFileDir := filepath.Join("9", "0") + badFileName := "0" // An invalid boltdb file is created so that it returns an error when opened - badFileDir := filepath.Join(dir, "9", "0") - require.NoError(t, os.MkdirAll(badFileDir, os.ModePerm)) - require.NoError(t, os.WriteFile(filepath.Join(badFileDir, "0"), []byte("not a boltdb file content"), 0777)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, relBadFileDir), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(dir, relBadFileDir, badFileName), []byte("not a boltdb file content"), 0777)) - res, err := b.Exists(context.Background(), common.ExistsPrm{Address: addr, StorageID: storageID}) + res, err := b.Exists(context.Background(), common.ExistsPrm{Address: addr, StorageID: []byte(filepath.Join(relBadFileDir, badFileName))}) require.Error(t, err) require.False(t, res.Exists) }) From 160147b05d4a51a6fdf29392f67e3282bda9da14 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Apr 2023 10:10:22 +0300 Subject: [PATCH 0122/1943] [#249] adm: Drop subnet support Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-adm/docs/subnetwork-creation.md | 39 - cmd/frostfs-adm/docs/subnetwork-usage.md | 137 -- .../modules/morph/initialize_deploy.go | 3 - .../internal/modules/morph/internal/types.go | 65 - .../modules/morph/internal/types.pb.go | 156 --- .../modules/morph/internal/types.proto | 15 - .../internal/modules/morph/root.go | 5 - .../internal/modules/morph/subnet.go | 1101 ----------------- .../internal/modules/storagecfg/config.go | 3 - 9 files changed, 1524 deletions(-) delete mode 100644 cmd/frostfs-adm/docs/subnetwork-creation.md delete mode 100644 cmd/frostfs-adm/docs/subnetwork-usage.md delete mode 100644 cmd/frostfs-adm/internal/modules/morph/internal/types.go delete mode 100644 cmd/frostfs-adm/internal/modules/morph/internal/types.pb.go delete mode 100644 cmd/frostfs-adm/internal/modules/morph/internal/types.proto delete mode 100644 cmd/frostfs-adm/internal/modules/morph/subnet.go diff --git a/cmd/frostfs-adm/docs/subnetwork-creation.md b/cmd/frostfs-adm/docs/subnetwork-creation.md deleted file mode 100644 index 5ada94387..000000000 --- a/cmd/frostfs-adm/docs/subnetwork-creation.md +++ /dev/null @@ -1,39 +0,0 @@ -# FrostFS subnetwork creation - -This is a short guide on how to create FrostFS subnetworks. This guide -considers that the sidechain and the inner ring (alphabet nodes) have already been -deployed and the sidechain contains a deployed `subnet` contract. - -## Prerequisites - -To follow this guide, you need: -- neo-go sidechain RPC endpoint; -- latest released version of [frostfs-adm](https://github.com/TrueCloudLab/frostfs-node/releases); -- wallet with FrostFS account. - -## Creation - -```shell -$ frostfs-adm morph subnet create \ - -r \ - -w \ - --notary -Create subnet request sent successfully. ID: 4223489767. -``` - -**NOTE:** in notary-enabled environment you should have a sufficient -notary deposit (not expired, with enough GAS balance). Your subnet ID -will differ from the example. - -The default account in the wallet that has been passed with `-w` flag is the owner -of the just created subnetwork. - -You can check if your subnetwork was created successfully: - -```shell -$ frostfs-adm morph subnet get \ - -r \ - --subnet -Owner: NUc734PMJXiqa2J9jRtvskU3kCdyyuSN8Q -``` -Your owner will differ from the example. diff --git a/cmd/frostfs-adm/docs/subnetwork-usage.md b/cmd/frostfs-adm/docs/subnetwork-usage.md deleted file mode 100644 index 0d505b3a4..000000000 --- a/cmd/frostfs-adm/docs/subnetwork-usage.md +++ /dev/null @@ -1,137 +0,0 @@ -# Managing Subnetworks - -This is a short guide on how to manage FrostFS subnetworks. This guide -considers that the sidechain and the inner ring (alphabet nodes) have already been -deployed, and the sidechain contains a deployed `subnet` contract. - -## Prerequisites - -- neo-go sidechain RPC endpoint; -- latest released version of [frostfs-adm](https://github.com/TrueCloudLab/frostfs-node/releases); -- [created](subnetwork-creation.md) subnetwork; -- wallet with the account that owns the subnetwork; -- public key of the Storage Node; -- public keys of the node and client administrators; -- owner IDs of the FrostFS users. - -## Add node administrator - -Node administrators are accounts that can manage (add and delete nodes) -the whitelist of the nodes which can be included to a subnetwork. Only the subnet -owner is allowed to add and remove node administrators from the subnetwork. - -```shell -$ frostfs-adm morph subnet admin add \ - -r \ - -w \ - --admin \ - --subnet -Add admin request sent successfully. -``` - -## Add node - -Adding a node to a subnetwork means that the node becomes able to service -containers that have been created in that subnetwork. Addition only changes -the list of the allowed nodes. Node is not required to be bootstrapped at the -moment of its inclusion. - -```shell -$ frostfs-adm morph subnet node add \ - -r \ - -w \ - --node \ - --subnet -Add node request sent successfully. -``` - -**NOTE:** the owner of the subnetwork is also allowed to add nodes. - -## Add client administrator - -Client administrators are accounts that can manage (add and delete -nodes) the whitelist of the clients that can create containers in the -subnetwork. Only the subnet owner is allowed to add and remove client -administrators from the subnetwork. - -```shell -$ frostfs-adm morph subnet admin add \ - -r \ - -w \ - --admin \ - --subnet \ - --client \ - --group -Add admin request sent successfully. -``` - -**NOTE:** you do not need to create a group explicitly, it will be created -right after the first client admin is added. Group ID is a 4-byte -positive integer number. - -## Add client - -```shell -$ frostfs-adm morph subnet client add \ - -r \ - -w \ - --client \ - --subnet \ - --group -Add client request sent successfully. -``` - -**NOTE:** the owner of the subnetwork is also allowed to add clients. This is -the only one command that accepts `ownerID`, not the public key. -Administrator can manage only their group (a group where that administrator -has been added by the subnet owner). - -# Bootstrapping Storage Node - -After a subnetwork [is created](subnetwork-creation.md) and a node is included into it, the -node could be bootstrapped and service subnetwork containers. - -For bootstrapping, you need to specify the ID of the subnetwork in the node's -configuration: - -```yaml -... -node: - ... - subnet: - entries: # list of IDs of subnets to enter in a text format of FrostFS API protocol (overrides corresponding attributes) - - - ... -... -``` - -**NOTE:** specifying subnetwork that is denied for the node is not an error: -that configuration value would be ignored. You do not need to specify zero -(with 0 ID) subnetwork: its inclusion is implicit. On the contrary, to exclude -a node from the default zero subnetwork, you need to specify it explicitly: - -```yaml -... -node: - ... - subnet: - exit_zero: true # toggle entrance to zero subnet (overrides corresponding attribute and occurrence in `entries`) - ... -... -``` - -# Creating container in non-zero subnetwork - -Creating containers without using `--subnet` flag is equivalent to -creating container in the zero subnetwork. - -To create a container in a private network, your wallet must be added to -the client whitelist by the client admins or the subnet owners: - -```shell -$ frostfs-cli container create \ - --policy 'REP 1' \ - -w \ - -r s01.frostfs.devenv:8080 \ - --subnet -``` diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 9a15b69ba..1e35fb79f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -47,7 +47,6 @@ const ( frostfsIDContract = "frostfsid" netmapContract = "netmap" proxyContract = "proxy" - subnetContract = "subnet" ) var ( @@ -58,7 +57,6 @@ var ( frostfsIDContract, netmapContract, proxyContract, - subnetContract, } fullContractList = append([]string{ @@ -593,7 +591,6 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an configParam) case proxyContract: items = nil - case subnetContract: default: panic(fmt.Sprintf("invalid contract name: %s", ctrName)) } diff --git a/cmd/frostfs-adm/internal/modules/morph/internal/types.go b/cmd/frostfs-adm/internal/modules/morph/internal/types.go deleted file mode 100644 index 93d2cb00e..000000000 --- a/cmd/frostfs-adm/internal/modules/morph/internal/types.go +++ /dev/null @@ -1,65 +0,0 @@ -package internal - -import ( - "fmt" - "strconv" - - "google.golang.org/protobuf/proto" -) - -// StringifySubnetClientGroupID returns string representation of SubnetClientGroupID using MarshalText. -// Returns a string with a message on error. -func StringifySubnetClientGroupID(id *SubnetClientGroupID) string { - text, err := id.MarshalText() - if err != nil { - return fmt.Sprintf(" %v", err) - } - - return string(text) -} - -// MarshalText encodes SubnetClientGroupID into text format according to FrostFS API V2 protocol: -// value in base-10 integer string format. -// -// It implements encoding.TextMarshaler. -func (x *SubnetClientGroupID) MarshalText() ([]byte, error) { - num := x.GetValue() // NPE safe, returns zero on nil - - return []byte(strconv.FormatUint(uint64(num), 10)), nil -} - -// UnmarshalText decodes the SubnetID from the text according to FrostFS API V2 protocol: -// should be base-10 integer string format with bitsize = 32. -// -// Returns strconv.ErrRange if integer overflows uint32. -// -// Must not be called on nil. -// -// Implements encoding.TextUnmarshaler. -func (x *SubnetClientGroupID) UnmarshalText(txt []byte) error { - num, err := strconv.ParseUint(string(txt), 10, 32) - if err != nil { - return fmt.Errorf("invalid numeric value: %w", err) - } - - x.SetNumber(uint32(num)) - - return nil -} - -// Marshal encodes the SubnetClientGroupID into a binary format of FrostFS API V2 protocol -// (Protocol Buffers with direct field order). -func (x *SubnetClientGroupID) Marshal() ([]byte, error) { - return proto.Marshal(x) -} - -// Unmarshal decodes the SubnetClientGroupID from FrostFS API V2 binary format (see Marshal). Must not be called on nil. -func (x *SubnetClientGroupID) Unmarshal(data []byte) error { - return proto.Unmarshal(data, x) -} - -// SetNumber sets SubnetClientGroupID value in uint32 format. Must not be called on nil. -// By default, number is 0. -func (x *SubnetClientGroupID) SetNumber(num uint32) { - x.Value = num -} diff --git a/cmd/frostfs-adm/internal/modules/morph/internal/types.pb.go b/cmd/frostfs-adm/internal/modules/morph/internal/types.pb.go deleted file mode 100644 index 6754bf66e..000000000 --- a/cmd/frostfs-adm/internal/modules/morph/internal/types.pb.go +++ /dev/null @@ -1,156 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.26.0 -// protoc v3.21.12 -// source: cmd/frostfs-adm/internal/modules/morph/internal/types.proto - -package internal - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// Client group identifier in the FrostFS subnet. -// -// String representation of a value is base-10 integer. -// -// JSON representation is an object containing single `value` number field. -type SubnetClientGroupID struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // 4-byte integer identifier of the subnet client group. - Value uint32 `protobuf:"fixed32,1,opt,name=value,proto3" json:"value,omitempty"` -} - -func (x *SubnetClientGroupID) Reset() { - *x = SubnetClientGroupID{} - if protoimpl.UnsafeEnabled { - mi := &file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SubnetClientGroupID) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SubnetClientGroupID) ProtoMessage() {} - -func (x *SubnetClientGroupID) ProtoReflect() protoreflect.Message { - mi := &file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SubnetClientGroupID.ProtoReflect.Descriptor instead. -func (*SubnetClientGroupID) Descriptor() ([]byte, []int) { - return file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDescGZIP(), []int{0} -} - -func (x *SubnetClientGroupID) GetValue() uint32 { - if x != nil { - return x.Value - } - return 0 -} - -var File_cmd_frostfs_adm_internal_modules_morph_internal_types_proto protoreflect.FileDescriptor - -var file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDesc = []byte{ - 0x0a, 0x3b, 0x63, 0x6d, 0x64, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x61, 0x64, - 0x6d, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, - 0x65, 0x73, 0x2f, 0x6d, 0x6f, 0x72, 0x70, 0x68, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 0x6e, - 0x65, 0x6f, 0x2e, 0x66, 0x73, 0x2e, 0x76, 0x32, 0x2e, 0x72, 0x65, 0x66, 0x73, 0x22, 0x2b, 0x0a, - 0x13, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x49, 0x44, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x07, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x5c, 0x5a, 0x5a, 0x67, 0x69, - 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, - 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, - 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x63, 0x6d, 0x64, 0x2f, 0x66, 0x72, 0x6f, - 0x73, 0x74, 0x66, 0x73, 0x2d, 0x61, 0x64, 0x6d, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x2f, 0x6d, 0x6f, 0x72, 0x70, 0x68, 0x2f, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDescOnce sync.Once - file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDescData = file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDesc -) - -func file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDescGZIP() []byte { - file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDescOnce.Do(func() { - file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDescData = protoimpl.X.CompressGZIP(file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDescData) - }) - return file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDescData -} - -var file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_goTypes = []interface{}{ - (*SubnetClientGroupID)(nil), // 0: neo.fs.v2.refs.SubnetClientGroupID -} -var file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_init() } -func file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_init() { - if File_cmd_frostfs_adm_internal_modules_morph_internal_types_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SubnetClientGroupID); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDesc, - NumEnums: 0, - NumMessages: 1, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_goTypes, - DependencyIndexes: file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_depIdxs, - MessageInfos: file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_msgTypes, - }.Build() - File_cmd_frostfs_adm_internal_modules_morph_internal_types_proto = out.File - file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_rawDesc = nil - file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_goTypes = nil - file_cmd_frostfs_adm_internal_modules_morph_internal_types_proto_depIdxs = nil -} diff --git a/cmd/frostfs-adm/internal/modules/morph/internal/types.proto b/cmd/frostfs-adm/internal/modules/morph/internal/types.proto deleted file mode 100644 index 2ce61b3c0..000000000 --- a/cmd/frostfs-adm/internal/modules/morph/internal/types.proto +++ /dev/null @@ -1,15 +0,0 @@ -syntax = "proto3"; - -package neo.fs.v2.refs; - -option go_package = "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/internal"; - -// Client group identifier in the FrostFS subnet. -// -// String representation of a value is base-10 integer. -// -// JSON representation is an object containing single `value` number field. -message SubnetClientGroupID { - // 4-byte integer identifier of the subnet client group. - fixed32 value = 1 [json_name = "value"]; -} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index e92fd2160..1361fe158 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -255,7 +255,6 @@ func init() { initRestoreContainersCmd() initListContainersCmd() initRefillGasCmd() - initSubnetCmd() initDepositoryNotaryCmd() initNetmapCandidatesCmd() } @@ -274,10 +273,6 @@ func initDepositoryNotaryCmd() { depositNotaryCmd.Flags().String(notaryDepositTillFlag, "", "Notary deposit duration in blocks") } -func initSubnetCmd() { - RootCmd.AddCommand(cmdSubnet) -} - func initRefillGasCmd() { RootCmd.AddCommand(refillGasCmd) refillGasCmd.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") diff --git a/cmd/frostfs-adm/internal/modules/morph/subnet.go b/cmd/frostfs-adm/internal/modules/morph/subnet.go deleted file mode 100644 index bdead7730..000000000 --- a/cmd/frostfs-adm/internal/modules/morph/subnet.go +++ /dev/null @@ -1,1101 +0,0 @@ -package morph - -import ( - "encoding/hex" - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/internal" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet" - subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/cli/flags" - "github.com/nspcc-dev/neo-go/cli/input" - "github.com/nspcc-dev/neo-go/pkg/core/native/nativenames" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/core/transaction" - "github.com/nspcc-dev/neo-go/pkg/crypto/hash" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" - "github.com/nspcc-dev/neo-go/pkg/smartcontract" - "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm/opcode" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/nspcc-dev/neo-go/pkg/wallet" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -func viperBindFlags(cmd *cobra.Command, flags ...string) { - for i := range flags { - _ = viper.BindPFlag(flags[i], cmd.Flags().Lookup(flags[i])) - } -} - -// subnet command section. -var cmdSubnet = &cobra.Command{ - Use: "subnet", - Short: "FrostFS subnet management", - PreRun: func(cmd *cobra.Command, _ []string) { - viperBindFlags(cmd, - endpointFlag, - ) - }, -} - -// shared flags of cmdSubnet sub-commands. -const ( - flagSubnet = "subnet" // subnet identifier - flagSubnetGroup = "group" // subnet client group ID - flagSubnetWallet = "wallet" // filepath to wallet - flagSubnetAddress = "address" // address in the wallet, optional -) - -// reads wallet from the filepath configured in flagSubnetWallet flag, -// looks for address specified in flagSubnetAddress flag (uses default -// address if flag is empty) and decrypts private key. -func readSubnetKey(key *keys.PrivateKey) error { - // read wallet from file - - walletPath := viper.GetString(flagSubnetWallet) - if walletPath == "" { - return errors.New("missing path to wallet") - } - - w, err := wallet.NewWalletFromFile(walletPath) - if err != nil { - return fmt.Errorf("read wallet from file: %w", err) - } - - // read account from the wallet - - var ( - addr util.Uint160 - addrStr = viper.GetString(flagSubnetAddress) - ) - - if addrStr == "" { - addr = w.GetChangeAddress() - } else { - addr, err = flags.ParseAddress(addrStr) - if err != nil { - return fmt.Errorf("read wallet address: %w", err) - } - } - - acc := w.GetAccount(addr) - if acc == nil { - return fmt.Errorf("address %s not found in %s", addrStr, walletPath) - } - - // read password - pass, err := input.ReadPassword("Enter password > ") - if err != nil { - return fmt.Errorf("read password: %w", err) - } - - // decrypt with just read password - err = acc.Decrypt(pass, keys.NEP2ScryptParams()) - if err != nil { - return fmt.Errorf("decrypt wallet: %w", err) - } - - *key = *acc.PrivateKey() - - return nil -} - -// create subnet command. -var cmdSubnetCreate = &cobra.Command{ - Use: "create", - Short: "Create FrostFS subnet", - PreRun: func(cmd *cobra.Command, _ []string) { - viperBindFlags(cmd, - flagSubnetWallet, - flagSubnetAddress, - ) - }, - RunE: func(cmd *cobra.Command, _ []string) error { - // read private key - var key keys.PrivateKey - - err := readSubnetKey(&key) - if err != nil { - return fmt.Errorf("read private key: %w", err) - } - - // generate subnet ID and marshal it - var ( - id subnetid.ID - num uint32 - ) - - for { - num = rand.Uint32() - - id.SetNumeric(num) - - if !subnetid.IsZero(id) { - break - } - } - - // declare creator ID and encode it - var creator user.ID - user.IDFromKey(&creator, key.PrivateKey.PublicKey) - - // fill subnet info and encode it - var info subnet.Info - - info.SetID(id) - info.SetOwner(creator) - - err = invokeMethod(key, true, "put", id.Marshal(), key.PublicKey().Bytes(), info.Marshal()) - if err != nil { - return fmt.Errorf("morph invocation: %w", err) - } - - cmd.Printf("Create subnet request sent successfully. ID: %s.\n", &id) - - return nil - }, -} - -// cmdSubnetRemove flags. -const ( - // subnet ID to be removed. - flagSubnetRemoveID = flagSubnet -) - -// errZeroSubnet is returned on attempts to work with zero subnet which is virtual. -var errZeroSubnet = errors.New("zero subnet") - -// remove subnet command. -var cmdSubnetRemove = &cobra.Command{ - Use: "remove", - Short: "Remove FrostFS subnet", - PreRun: func(cmd *cobra.Command, _ []string) { - viperBindFlags(cmd, - flagSubnetWallet, - flagSubnetAddress, - flagSubnetRemoveID, - ) - }, - RunE: func(cmd *cobra.Command, _ []string) error { - // read private key - var key keys.PrivateKey - - err := readSubnetKey(&key) - if err != nil { - return fmt.Errorf("read private key: %w", err) - } - - // read ID and encode it - var id subnetid.ID - - err = id.DecodeString(viper.GetString(flagSubnetRemoveID)) - if err != nil { - return fmt.Errorf("decode ID text: %w", err) - } - - if subnetid.IsZero(id) { - return errZeroSubnet - } - - err = invokeMethod(key, false, "delete", id.Marshal()) - if err != nil { - return fmt.Errorf("morph invocation: %w", err) - } - - cmd.Println("Remove subnet request sent successfully") - - return nil - }, -} - -// cmdSubnetGet flags. -const ( - // subnet ID to be read. - flagSubnetGetID = flagSubnet -) - -// get subnet command. -var cmdSubnetGet = &cobra.Command{ - Use: "get", - Short: "Read information about the FrostFS subnet", - PreRun: func(cmd *cobra.Command, _ []string) { - viperBindFlags(cmd, - flagSubnetGetID, - ) - }, - RunE: func(cmd *cobra.Command, _ []string) error { - // read ID and encode it - var id subnetid.ID - - err := id.DecodeString(viper.GetString(flagSubnetGetID)) - if err != nil { - return fmt.Errorf("decode ID text: %w", err) - } - - if subnetid.IsZero(id) { - return errZeroSubnet - } - - // use random key to fetch the data - // we could use raw neo-go client to perform testInvoke - // without keys, as it is done in other commands - key, err := keys.NewPrivateKey() - if err != nil { - return fmt.Errorf("init subnet client: %w", err) - } - - res, err := testInvokeMethod(*key, "get", id.Marshal()) - if err != nil { - return fmt.Errorf("morph invocation: %w", err) - } - - if len(res) == 0 { - return errors.New("subnet does not exist") - } - - data, err := client.BytesFromStackItem(res[0]) - if err != nil { - return fmt.Errorf("decoding contract response: %w", err) - } - - // decode info - var info subnet.Info - if err = info.Unmarshal(data); err != nil { - return fmt.Errorf("decode subnet info: %w", err) - } - - // print information - cmd.Printf("Owner: %s\n", info.Owner()) - - return nil - }, -} - -// cmdSubnetAdmin subnet flags. -const ( - flagSubnetAdminSubnet = flagSubnet // subnet ID to be managed - flagSubnetAdminID = "admin" // admin public key - flagSubnetAdminClient = "client" // manage client admins instead of node ones -) - -// command to manage subnet admins. -var cmdSubnetAdmin = &cobra.Command{ - Use: "admin", - Short: "Manage administrators of the FrostFS subnet", - PreRun: func(cmd *cobra.Command, args []string) { - viperBindFlags(cmd, - flagSubnetWallet, - flagSubnetAddress, - flagSubnetAdminSubnet, - flagSubnetAdminID, - ) - }, -} - -// cmdSubnetAdminAdd flags. -const ( - flagSubnetAdminAddGroup = flagSubnetGroup // client group ID -) - -// common executor cmdSubnetAdminAdd and cmdSubnetAdminRemove commands. -func manageSubnetAdmins(cmd *cobra.Command, rm bool) error { - // read private key - var key keys.PrivateKey - - err := readSubnetKey(&key) - if err != nil { - return fmt.Errorf("read private key: %w", err) - } - - // read ID and encode it - var id subnetid.ID - - err = id.DecodeString(viper.GetString(flagSubnetAdminSubnet)) - if err != nil { - return fmt.Errorf("decode ID text: %w", err) - } - - if subnetid.IsZero(id) { - return errZeroSubnet - } - - // read admin key and decode it - binAdminKey, err := hex.DecodeString(viper.GetString(flagSubnetAdminID)) - if err != nil { - return fmt.Errorf("decode admin key text: %w", err) - } - - var pubkey keys.PublicKey - if err = pubkey.DecodeBytes(binAdminKey); err != nil { - return fmt.Errorf("admin key format: %w", err) - } - - return invokeMethodWithParams(cmd, id, rm, binAdminKey, key) -} - -func invokeMethodWithParams(cmd *cobra.Command, id subnetid.ID, rm bool, binAdminKey []byte, key keys.PrivateKey) error { - prm := make([]any, 0, 3) - prm = append(prm, id.Marshal()) - - var method string - - if viper.GetBool(flagSubnetAdminClient) { - var groupID internal.SubnetClientGroupID - - err := groupID.UnmarshalText([]byte(viper.GetString(flagSubnetAdminAddGroup))) - if err != nil { - return fmt.Errorf("decode group ID text: %w", err) - } - - binGroupID, err := groupID.Marshal() - if err != nil { - return fmt.Errorf("marshal group ID: %w", err) - } - - if rm { - method = "removeClientAdmin" - } else { - method = "addClientAdmin" - } - - prm = append(prm, binGroupID) - } else { - if rm { - method = "removeNodeAdmin" - } else { - method = "addNodeAdmin" - } - } - - prm = append(prm, binAdminKey) - - err := invokeMethod(key, false, method, prm...) - if err != nil { - return fmt.Errorf("morph invocation: %w", err) - } - - var op string - - if rm { - op = "Remove" - } else { - op = "Add" - } - - cmd.Printf("%s admin request sent successfully.\n", op) - - return nil -} - -// command to add subnet admin. -var cmdSubnetAdminAdd = &cobra.Command{ - Use: "add", - Short: "Add admin to the FrostFS subnet", - PreRun: func(cmd *cobra.Command, _ []string) { - viperBindFlags(cmd, - flagSubnetAdminAddGroup, - flagSubnetAdminClient, - ) - }, - RunE: func(cmd *cobra.Command, _ []string) error { - return manageSubnetAdmins(cmd, false) - }, -} - -// command to remove subnet admin. -var cmdSubnetAdminRemove = &cobra.Command{ - Use: "remove", - Short: "Remove admin of the FrostFS subnet", - PreRun: func(cmd *cobra.Command, _ []string) { - viperBindFlags(cmd, - flagSubnetAdminClient, - ) - }, - RunE: func(cmd *cobra.Command, _ []string) error { - return manageSubnetAdmins(cmd, true) - }, -} - -// cmdSubnetClient flags. -const ( - flagSubnetClientSubnet = flagSubnet // ID of the subnet to be managed - flagSubnetClientID = flagSubnetAdminClient // client's NeoFS ID - flagSubnetClientGroup = flagSubnetGroup // ID of the subnet client group -) - -// command to manage subnet clients. -var cmdSubnetClient = &cobra.Command{ - Use: "client", - Short: "Manage clients of the FrostFS subnet", - PreRun: func(cmd *cobra.Command, _ []string) { - viperBindFlags(cmd, - flagSubnetWallet, - flagSubnetAddress, - flagSubnetClientSubnet, - flagSubnetClientID, - flagSubnetClientGroup, - ) - }, -} - -// common executor cmdSubnetClientAdd and cmdSubnetClientRemove commands. -func manageSubnetClients(cmd *cobra.Command, rm bool) error { - // read private key - var key keys.PrivateKey - - err := readSubnetKey(&key) - if err != nil { - return fmt.Errorf("read private key: %w", err) - } - - // read ID and encode it - var id subnetid.ID - - err = id.DecodeString(viper.GetString(flagSubnetClientSubnet)) - if err != nil { - return fmt.Errorf("decode ID text: %w", err) - } - - if subnetid.IsZero(id) { - return errZeroSubnet - } - - // read client ID and encode it - var clientID user.ID - - err = clientID.DecodeString(viper.GetString(flagSubnetClientID)) - if err != nil { - return fmt.Errorf("decode client ID text: %w", err) - } - - // read group ID and encode it - var groupID internal.SubnetClientGroupID - - err = groupID.UnmarshalText([]byte(viper.GetString(flagSubnetAdminAddGroup))) - if err != nil { - return fmt.Errorf("decode group ID text: %w", err) - } - - binGroupID, err := groupID.Marshal() - if err != nil { - return fmt.Errorf("marshal group ID: %w", err) - } - - var method string - if rm { - method = "removeUser" - } else { - method = "addUser" - } - - err = invokeMethod(key, false, method, id.Marshal(), binGroupID, clientID.WalletBytes()) - if err != nil { - return fmt.Errorf("morph invocation: %w", err) - } - - var op string - - if rm { - op = "Remove" - } else { - op = "Add" - } - - cmd.Printf("%s client request sent successfully.\n", op) - - return nil -} - -// command to add subnet client. -var cmdSubnetClientAdd = &cobra.Command{ - Use: "add", - Short: "Add client to the FrostFS subnet", - RunE: func(cmd *cobra.Command, _ []string) error { - return manageSubnetClients(cmd, false) - }, -} - -// command to remove subnet client. -var cmdSubnetClientRemove = &cobra.Command{ - Use: "remove", - Short: "Remove client of the FrostFS subnet", - RunE: func(cmd *cobra.Command, _ []string) error { - return manageSubnetClients(cmd, true) - }, -} - -// cmdSubnetNode flags. -const ( - flagSubnetNode = "node" // node ID - flagSubnetNodeSubnet = flagSubnet // ID of the subnet to be managed -) - -// common executor cmdSubnetNodeAdd and cmdSubnetNodeRemove commands. -func manageSubnetNodes(cmd *cobra.Command, rm bool) error { - // read private key - var key keys.PrivateKey - - err := readSubnetKey(&key) - if err != nil { - return fmt.Errorf("read private key: %w", err) - } - - // read ID and encode it - var id subnetid.ID - - err = id.DecodeString(viper.GetString(flagSubnetNodeSubnet)) - if err != nil { - return fmt.Errorf("decode ID text: %w", err) - } - - if subnetid.IsZero(id) { - return errZeroSubnet - } - - // read node ID and encode it - binNodeID, err := hex.DecodeString(viper.GetString(flagSubnetNode)) - if err != nil { - return fmt.Errorf("decode node ID text: %w", err) - } - - var pubkey keys.PublicKey - if err = pubkey.DecodeBytes(binNodeID); err != nil { - return fmt.Errorf("node ID format: %w", err) - } - - var method string - if rm { - method = "removeNode" - } else { - method = "addNode" - } - - err = invokeMethod(key, false, method, id.Marshal(), binNodeID) - if err != nil { - return fmt.Errorf("morph invocation: %w", err) - } - - var op string - - if rm { - op = "Remove" - } else { - op = "Add" - } - - cmd.Printf("%s node request sent successfully.\n", op) - - return nil -} - -// command to manage subnet nodes. -var cmdSubnetNode = &cobra.Command{ - Use: "node", - Short: "Manage nodes of the FrostFS subnet", - PreRun: func(cmd *cobra.Command, _ []string) { - viperBindFlags(cmd, - flagSubnetWallet, - flagSubnetNode, - flagSubnetNodeSubnet, - ) - }, -} - -// command to add subnet node. -var cmdSubnetNodeAdd = &cobra.Command{ - Use: "add", - Short: "Add node to the FrostFS subnet", - RunE: func(cmd *cobra.Command, _ []string) error { - return manageSubnetNodes(cmd, false) - }, -} - -// command to remove subnet node. -var cmdSubnetNodeRemove = &cobra.Command{ - Use: "remove", - Short: "Remove node from the FrostFS subnet", - RunE: func(cmd *cobra.Command, _ []string) error { - return manageSubnetNodes(cmd, true) - }, -} - -// returns function which calls PreRun on parent if it exists. -func inheritPreRun(preRun func(*cobra.Command, []string)) func(*cobra.Command, []string) { - return func(cmd *cobra.Command, args []string) { - par := cmd.Parent() - if par != nil && par.PreRun != nil { - par.PreRun(par, args) - } - - if preRun != nil { - preRun(cmd, args) - } - } -} - -// inherits PreRun function of parent command in all sub-commands and -// adds them to the parent. -func addCommandInheritPreRun(par *cobra.Command, subs ...*cobra.Command) { - for _, sub := range subs { - sub.PreRun = inheritPreRun(sub.PreRun) - } - - par.AddCommand(subs...) -} - -// registers flags and binds sub-commands for subnet commands. -func init() { - initCreateSubnetFlags() - initGetSubnetFlags() - initRemoveSubnetFlags() - initSubnetAdminFlags() - initSubnetAdminAddFlags() - initSubnetAdminRemoveFlags() - initClientManagementFlags() - - // add all admin managing commands to corresponding command section - addCommandInheritPreRun(cmdSubnetAdmin, - cmdSubnetAdminAdd, - cmdSubnetAdminRemove, - ) - - // add all client managing commands to corresponding command section - addCommandInheritPreRun(cmdSubnetClient, - cmdSubnetClientAdd, - cmdSubnetClientRemove, - ) - - initSubnetNodeFlags() - - // add all node managing commands to corresponding command section - addCommandInheritPreRun(cmdSubnetNode, - cmdSubnetNodeAdd, - cmdSubnetNodeRemove, - ) - - initSubnetGlobalFlags() - - // add all subnet commands to corresponding command section - addCommandInheritPreRun(cmdSubnet, - cmdSubnetCreate, - cmdSubnetRemove, - cmdSubnetGet, - cmdSubnetAdmin, - cmdSubnetClient, - cmdSubnetNode, - ) -} - -func initSubnetGlobalFlags() { - cmdSubnetFlags := cmdSubnet.PersistentFlags() - cmdSubnetFlags.StringP(endpointFlag, "r", "", "N3 RPC node endpoint") - _ = cmdSubnet.MarkFlagRequired(endpointFlag) -} - -func initSubnetNodeFlags() { - nodeFlags := cmdSubnetNode.PersistentFlags() - nodeFlags.StringP(flagSubnetWallet, "w", "", "Path to file with wallet") - _ = cmdSubnetNode.MarkFlagRequired(flagSubnetWallet) - nodeFlags.String(flagSubnetNode, "", "Hex-encoded public key of the node") - _ = cmdSubnetNode.MarkFlagRequired(flagSubnetNode) - nodeFlags.String(flagSubnetNodeSubnet, "", "ID of the subnet to manage nodes") - _ = cmdSubnetNode.MarkFlagRequired(flagSubnetNodeSubnet) -} - -func initClientManagementFlags() { - clientFlags := cmdSubnetClient.PersistentFlags() - clientFlags.String(flagSubnetClientSubnet, "", "ID of the subnet to be managed") - _ = cmdSubnetClient.MarkFlagRequired(flagSubnetClientSubnet) - clientFlags.String(flagSubnetClientGroup, "", "ID of the client group to work with") - _ = cmdSubnetClient.MarkFlagRequired(flagSubnetClientGroup) - clientFlags.String(flagSubnetClientID, "", "Client's user ID in FrostFS system in text format") - _ = cmdSubnetClient.MarkFlagRequired(flagSubnetClientID) - clientFlags.StringP(flagSubnetWallet, "w", "", "Path to file with wallet") - _ = cmdSubnetClient.MarkFlagRequired(flagSubnetWallet) - clientFlags.StringP(flagSubnetAddress, "a", "", "Address in the wallet, optional") -} - -func initSubnetAdminRemoveFlags() { - cmdSubnetAdminRemoveFlags := cmdSubnetAdminRemove.Flags() - cmdSubnetAdminRemoveFlags.Bool(flagSubnetAdminClient, false, "Remove client admin instead of node one") -} - -func initSubnetAdminAddFlags() { - cmdSubnetAdminAddFlags := cmdSubnetAdminAdd.Flags() - cmdSubnetAdminAddFlags.String(flagSubnetAdminAddGroup, "", fmt.Sprintf( - "Client group ID in text format (needed with --%s only)", flagSubnetAdminClient)) - cmdSubnetAdminAddFlags.Bool(flagSubnetAdminClient, false, "Add client admin instead of node one") -} - -func initSubnetAdminFlags() { - adminFlags := cmdSubnetAdmin.PersistentFlags() - adminFlags.String(flagSubnetAdminSubnet, "", "ID of the subnet to manage administrators") - _ = cmdSubnetAdmin.MarkFlagRequired(flagSubnetAdminSubnet) - adminFlags.String(flagSubnetAdminID, "", "Hex-encoded public key of the admin") - _ = cmdSubnetAdmin.MarkFlagRequired(flagSubnetAdminID) - adminFlags.StringP(flagSubnetWallet, "w", "", "Path to file with wallet") - _ = cmdSubnetAdmin.MarkFlagRequired(flagSubnetWallet) - adminFlags.StringP(flagSubnetAddress, "a", "", "Address in the wallet, optional") -} - -func initRemoveSubnetFlags() { - cmdSubnetRemove.Flags().String(flagSubnetRemoveID, "", "ID of the subnet to remove") - _ = cmdSubnetRemove.MarkFlagRequired(flagSubnetRemoveID) - cmdSubnetRemove.Flags().StringP(flagSubnetWallet, "w", "", "Path to file with wallet") - _ = cmdSubnetRemove.MarkFlagRequired(flagSubnetWallet) - cmdSubnetRemove.Flags().StringP(flagSubnetAddress, "a", "", "Address in the wallet, optional") -} - -func initGetSubnetFlags() { - cmdSubnetGet.Flags().String(flagSubnetGetID, "", "ID of the subnet to read") - _ = cmdSubnetAdminAdd.MarkFlagRequired(flagSubnetGetID) -} - -func initCreateSubnetFlags() { - cmdSubnetCreate.Flags().StringP(flagSubnetWallet, "w", "", "Path to file with wallet") - _ = cmdSubnetCreate.MarkFlagRequired(flagSubnetWallet) - cmdSubnetCreate.Flags().StringP(flagSubnetAddress, "a", "", "Address in the wallet, optional") -} - -func testInvokeMethod(key keys.PrivateKey, method string, args ...any) ([]stackitem.Item, error) { - c, err := getN3Client(viper.GetViper()) - if err != nil { - return nil, fmt.Errorf("morph client creation: %w", err) - } - - nnsCs, err := c.GetContractStateByID(1) - if err != nil { - return nil, fmt.Errorf("NNS contract resolving: %w", err) - } - - cosigner := []transaction.Signer{ - { - Account: key.PublicKey().GetScriptHash(), - Scopes: transaction.Global, - }, - } - - inv := invoker.New(c, cosigner) - - subnetHash, err := nnsResolveHash(inv, nnsCs.Hash, subnetContract+".frostfs") - if err != nil { - return nil, fmt.Errorf("subnet hash resolving: %w", err) - } - - res, err := inv.Call(subnetHash, method, args...) - if err != nil { - return nil, fmt.Errorf("invocation parameters prepararion: %w", err) - } - - err = checkInvocationResults(res) - if err != nil { - return nil, err - } - - return res.Stack, nil -} - -func invokeMethod(key keys.PrivateKey, tryNotary bool, method string, args ...any) error { - c, err := getN3Client(viper.GetViper()) - if err != nil { - return fmt.Errorf("morph client creation: %w", err) - } - - if tryNotary { - cc, err := c.GetNativeContracts() - if err != nil { - return fmt.Errorf("native hashes: %w", err) - } - - var notary bool - var notaryHash util.Uint160 - for _, c := range cc { - if c.Manifest.Name == nativenames.Notary { - notary = len(c.UpdateHistory) > 0 - notaryHash = c.Hash - - break - } - } - - if notary { - err = invokeNotary(c, key, method, notaryHash, args...) - if err != nil { - return fmt.Errorf("notary invocation: %w", err) - } - - return nil - } - } - - err = invokeNonNotary(c, key, method, args...) - if err != nil { - return fmt.Errorf("non-notary invocation: %w", err) - } - - return nil -} - -func invokeNonNotary(c Client, key keys.PrivateKey, method string, args ...any) error { - nnsCs, err := c.GetContractStateByID(1) - if err != nil { - return fmt.Errorf("NNS contract resolving: %w", err) - } - - acc := wallet.NewAccountFromPrivateKey(&key) - - cosigner := []transaction.Signer{ - { - Account: key.PublicKey().GetScriptHash(), - Scopes: transaction.Global, - }, - } - - cosignerAcc := []rpcclient.SignerAccount{ - { - Signer: cosigner[0], - Account: acc, - }, - } - - inv := invoker.New(c, cosigner) - - subnetHash, err := nnsResolveHash(inv, nnsCs.Hash, subnetContract+".frostfs") - if err != nil { - return fmt.Errorf("subnet hash resolving: %w", err) - } - - test, err := inv.Call(subnetHash, method, args...) - if err != nil { - return fmt.Errorf("test invocation: %w", err) - } - - err = checkInvocationResults(test) - if err != nil { - return err - } - - _, err = c.SignAndPushInvocationTx(test.Script, acc, test.GasConsumed, 0, cosignerAcc) - if err != nil { - return fmt.Errorf("sending transaction: %w", err) - } - - return nil -} - -func invokeNotary(c Client, key keys.PrivateKey, method string, notaryHash util.Uint160, args ...any) error { - nnsCs, err := c.GetContractStateByID(1) - if err != nil { - return fmt.Errorf("NNS contract resolving: %w", err) - } - - alphabet, err := c.GetCommittee() - if err != nil { - return fmt.Errorf("alphabet list: %w", err) - } - - multisigScript, err := smartcontract.CreateDefaultMultiSigRedeemScript(alphabet) - if err != nil { - return fmt.Errorf("alphabet multi-signature script: %w", err) - } - - cosigners, err := notaryCosigners(c, notaryHash, nnsCs, key, hash.Hash160(multisigScript)) - if err != nil { - return fmt.Errorf("cosigners collecting: %w", err) - } - - inv := invoker.New(c, cosigners) - - subnetHash, err := nnsResolveHash(inv, nnsCs.Hash, subnetContract+".frostfs") - if err != nil { - return fmt.Errorf("subnet hash resolving: %w", err) - } - - test, err := makeTestInvocation(inv, subnetHash, method, args) - if err != nil { - return err - } - - multisigAccount := &wallet.Account{ - Contract: &wallet.Contract{ - Script: multisigScript, - }, - } - - bc, err := c.GetBlockCount() - if err != nil { - return fmt.Errorf("blockchain height: %w", err) - } - - return createAndPushTransaction(alphabet, test, bc, cosigners, c, key, multisigAccount) -} - -func makeTestInvocation(inv *invoker.Invoker, subnetHash util.Uint160, method string, args []any) (*result.Invoke, error) { - test, err := inv.Call(subnetHash, method, args...) - if err != nil { - return nil, fmt.Errorf("test invocation: %w", err) - } - - err = checkInvocationResults(test) - if err != nil { - return nil, err - } - return test, nil -} - -func createAndPushTransaction(alphabet keys.PublicKeys, test *result.Invoke, blockCount uint32, cosigners []transaction.Signer, - client Client, key keys.PrivateKey, multisigAccount *wallet.Account) error { - // alphabet multisig + key signature - signersNumber := uint8(smartcontract.GetDefaultHonestNodeCount(len(alphabet)) + 1) - - // notaryRequestValidity is number of blocks during - // witch notary request is considered valid - const notaryRequestValidity = 100 - - mainTx := &transaction.Transaction{ - Nonce: rand.Uint32(), - SystemFee: test.GasConsumed, - ValidUntilBlock: blockCount + notaryRequestValidity, - Script: test.Script, - Attributes: []transaction.Attribute{ - { - Type: transaction.NotaryAssistedT, - Value: &transaction.NotaryAssisted{NKeys: signersNumber}, - }, - }, - Signers: cosigners, - } - - notaryFee, err := client.CalculateNotaryFee(signersNumber) - if err != nil { - return err - } - - acc := wallet.NewAccountFromPrivateKey(&key) - aa := notaryAccounts(multisigAccount, acc) - - err = client.AddNetworkFee(mainTx, notaryFee, aa...) - if err != nil { - return fmt.Errorf("notary network fee adding: %w", err) - } - - mainTx.Scripts = notaryWitnesses(client, multisigAccount, acc, mainTx) - - _, err = client.SignAndPushP2PNotaryRequest(mainTx, - []byte{byte(opcode.RET)}, - -1, - 0, - 40, - acc) - if err != nil { - return fmt.Errorf("sending notary request: %w", err) - } - - return nil -} - -func notaryCosigners(c Client, notaryHash util.Uint160, nnsCs *state.Contract, - key keys.PrivateKey, alphabetAccount util.Uint160) ([]transaction.Signer, error) { - proxyHash, err := nnsResolveHash(invoker.New(c, nil), nnsCs.Hash, proxyContract+".frostfs") - if err != nil { - return nil, fmt.Errorf("proxy hash resolving: %w", err) - } - - return []transaction.Signer{ - { - Account: proxyHash, - Scopes: transaction.None, - }, - { - Account: alphabetAccount, - Scopes: transaction.Global, - }, - { - Account: hash.Hash160(key.PublicKey().GetVerificationScript()), - Scopes: transaction.Global, - }, - { - Account: notaryHash, - Scopes: transaction.None, - }, - }, nil -} - -func notaryAccounts(alphabet, acc *wallet.Account) []*wallet.Account { - return []*wallet.Account{ - // proxy - { - Contract: &wallet.Contract{ - Deployed: true, - }, - }, - alphabet, - // caller's account - acc, - // last one is a placeholder for notary contract account - { - Contract: &wallet.Contract{}, - }, - } -} - -func notaryWitnesses(c Client, alphabet, acc *wallet.Account, tx *transaction.Transaction) []transaction.Witness { - ww := make([]transaction.Witness, 0, 4) - - // empty proxy contract witness - ww = append(ww, transaction.Witness{ - InvocationScript: []byte{}, - VerificationScript: []byte{}, - }) - - // alphabet multi-address witness - ww = append(ww, transaction.Witness{ - InvocationScript: append( - []byte{byte(opcode.PUSHDATA1), 64}, - make([]byte, 64)..., - ), - VerificationScript: alphabet.GetVerificationScript(), - }) - - magicNumber, _ := c.GetNetwork() - - // caller's witness - ww = append(ww, transaction.Witness{ - InvocationScript: append( - []byte{byte(opcode.PUSHDATA1), 64}, - acc.PrivateKey().SignHashable(uint32(magicNumber), tx)...), - VerificationScript: acc.GetVerificationScript(), - }) - - // notary contract witness - ww = append(ww, transaction.Witness{ - InvocationScript: append( - []byte{byte(opcode.PUSHDATA1), 64}, - make([]byte, 64)..., - ), - VerificationScript: []byte{}, - }) - - return ww -} - -func checkInvocationResults(res *result.Invoke) error { - if res.State != "HALT" { - return fmt.Errorf("test invocation state: %s, exception %s: ", res.State, res.FaultException) - } - - if len(res.Script) == 0 { - return errors.New("empty invocation script") - } - - return nil -} diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/config.go b/cmd/frostfs-adm/internal/modules/storagecfg/config.go index 435f79dfb..a07ce32c6 100644 --- a/cmd/frostfs-adm/internal/modules/storagecfg/config.go +++ b/cmd/frostfs-adm/internal/modules/storagecfg/config.go @@ -12,9 +12,6 @@ node: - {{ .AnnouncedAddress }} attribute_0: UN-LOCODE:{{ .Attribute.Locode }} relay: {{ .Relay }} # start Storage node in relay mode without bootstrapping into the Network map - subnet: - exit_zero: false # toggle entrance to zero subnet (overrides corresponding attribute and occurrence in entries) - entries: [] # list of IDs of subnets to enter in a text format of FrostFS API protocol (overrides corresponding attributes) grpc: num: 1 # total number of listener endpoints From 05c870f39a579a138a55183cf9970e67f2823acf Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Apr 2023 12:05:56 +0300 Subject: [PATCH 0123/1943] [#249] cli: Drop subnet support Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/container/create.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 410663e84..873ef3235 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -16,7 +16,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/spf13/cobra" ) @@ -30,7 +29,6 @@ var ( containerNnsName string containerNnsZone string containerNoTimestamp bool - containerSubnet string force bool ) @@ -70,15 +68,6 @@ It will be stored in sidechain when inner ring will accepts it.`, } } - if containerSubnet != "" { - var subnetID subnetid.ID - - err = subnetID.DecodeString(containerSubnet) - commonCmd.ExitOnErr(cmd, "could not parse subnetID: %w", err) - - placementPolicy.RestrictSubnet(subnetID) - } - var cnr container.Container cnr.Init() @@ -166,7 +155,6 @@ func initContainerCreateCmd() { flags.StringVar(&containerNnsName, "nns-name", "", "Container nns name attribute") flags.StringVar(&containerNnsZone, "nns-zone", "", "Container nns zone attribute") flags.BoolVar(&containerNoTimestamp, "disable-timestamp", false, "Disable timestamp container attribute") - flags.StringVar(&containerSubnet, "subnet", "", "String representation of container subnetwork") flags.BoolVarP(&force, commonflags.ForceFlag, commonflags.ForceFlagShorthand, false, "Skip placement validity check") } From d757d881d0d270861cdd191e7ce4deb52545fcbf Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Apr 2023 12:11:27 +0300 Subject: [PATCH 0124/1943] [#249] node: Drop subnet from config Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config/node/config.go | 27 -------------------- cmd/frostfs-node/config/node/config_test.go | 28 --------------------- cmd/frostfs-node/netmap.go | 27 -------------------- config/example/node.env | 2 -- config/example/node.json | 8 ------ config/example/node.yaml | 6 ----- docs/storage-node-configuration.md | 14 ----------- 7 files changed, 112 deletions(-) diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index a2794422e..9dfe8ddf4 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -177,33 +177,6 @@ func (p PersistentStateConfig) Path() string { return PersistentStatePathDefault } -// SubnetConfig represents node configuration related to subnets. -type SubnetConfig config.Config - -// Init initializes SubnetConfig from "subnet" sub-section of "node" section -// of the root config. -func (x *SubnetConfig) Init(root config.Config) { - *x = SubnetConfig(*root.Sub(subsection).Sub("subnet")) -} - -// ExitZero returns the value of "exit_zero" config parameter as bool. -// Returns false if the value can not be cast. -func (x SubnetConfig) ExitZero() bool { - return config.BoolSafe((*config.Config)(&x), "exit_zero") -} - -// IterateSubnets casts the value of "entries" config parameter to string slice, -// iterates over all of its elements and passes them to f. -// -// Does nothing if the value can not be cast to string slice. -func (x SubnetConfig) IterateSubnets(f func(string)) { - ids := config.StringSliceSafe((*config.Config)(&x), "entries") - - for i := range ids { - f(ids[i]) - } -} - // Notification returns structure that provides access to "notification" // subsection of "node" section. func Notification(c *config.Config) NotificationConfig { diff --git a/cmd/frostfs-node/config/node/config_test.go b/cmd/frostfs-node/config/node/config_test.go index ff085c506..3a1120491 100644 --- a/cmd/frostfs-node/config/node/config_test.go +++ b/cmd/frostfs-node/config/node/config_test.go @@ -52,20 +52,6 @@ func TestNodeSection(t *testing.T) { require.Equal(t, "", notificationDefaultCertPath) require.Equal(t, "", notificationDefaultKeyPath) require.Equal(t, "", notificationDefaultCAPath) - - var subnetCfg SubnetConfig - - subnetCfg.Init(*empty) - - require.False(t, subnetCfg.ExitZero()) - - called := false - - subnetCfg.IterateSubnets(func(string) { - called = true - }) - - require.False(t, called) }) const path = "../../../../config/example/node" @@ -143,20 +129,6 @@ func TestNodeSection(t *testing.T) { require.Equal(t, "/cert/path", notificationCertPath) require.Equal(t, "/key/path", notificationKeyPath) require.Equal(t, "/ca/path", notificationCAPath) - - var subnetCfg SubnetConfig - - subnetCfg.Init(*c) - - require.True(t, subnetCfg.ExitZero()) - - var ids []string - - subnetCfg.IterateSubnets(func(id string) { - ids = append(ids, id) - }) - - require.Equal(t, []string{"123", "456", "789"}, ids) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index b8a25cb8c..58e3cb2f2 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -7,7 +7,6 @@ import ( "fmt" netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" - nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" @@ -19,7 +18,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" netmapService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/netmap" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "go.uber.org/atomic" "go.uber.org/zap" @@ -143,8 +141,6 @@ func initNetmapService(ctx context.Context, c *cfg) { parseAttributes(c) c.cfgNodeInfo.localInfo.SetOffline() - readSubnetCfg(c) - if c.cfgMorph.client == nil { initMorphComponents(ctx, c) } @@ -227,29 +223,6 @@ func addNewEpochNotificationHandlers(c *cfg) { } } -func readSubnetCfg(c *cfg) { - var subnetCfg nodeconfig.SubnetConfig - - subnetCfg.Init(*c.appCfg) - - var ( - id subnetid.ID - err error - ) - - subnetCfg.IterateSubnets(func(idTxt string) { - err = id.DecodeString(idTxt) - fatalOnErrDetails("parse subnet entry", err) - - c.cfgNodeInfo.localInfo.EnterSubnet(id) - }) - - if subnetCfg.ExitZero() { - subnetid.MakeZero(&id) - c.cfgNodeInfo.localInfo.ExitSubnet(id) - } -} - // bootstrapNode adds current node to the Network map. // Must be called after initNetmapService. func bootstrapNode(c *cfg) { diff --git a/config/example/node.env b/config/example/node.env index b3fb4f9f9..77992d995 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -19,8 +19,6 @@ FROSTFS_NODE_ATTRIBUTE_1="UN-LOCODE:RU MSK" FROSTFS_NODE_RELAY=true FROSTFS_NODE_PERSISTENT_SESSIONS_PATH=/sessions FROSTFS_NODE_PERSISTENT_STATE_PATH=/state -FROSTFS_NODE_SUBNET_EXIT_ZERO=true -FROSTFS_NODE_SUBNET_ENTRIES=123 456 789 FROSTFS_NODE_NOTIFICATION_ENABLED=true FROSTFS_NODE_NOTIFICATION_ENDPOINT=tls://localhost:4222 FROSTFS_NODE_NOTIFICATION_TIMEOUT=6s diff --git a/config/example/node.json b/config/example/node.json index 9e2a0a487..b52eb6d96 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -34,14 +34,6 @@ "persistent_state": { "path": "/state" }, - "subnet": { - "exit_zero": true, - "entries": [ - "123", - "456", - "789" - ] - }, "notification": { "enabled": true, "endpoint": "tls://localhost:4222", diff --git a/config/example/node.yaml b/config/example/node.yaml index 54d774174..1669e0e86 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -29,12 +29,6 @@ node: path: /sessions # path to persistent session tokens file of Storage node (default: in-memory sessions) persistent_state: path: /state # path to persistent state file of Storage node - subnet: - exit_zero: true # toggle entrance to zero subnet (overrides corresponding attribute and occurrence in `entries`) - entries: # list of IDs of subnets to enter in a text format of FrostFS API protocol (overrides corresponding attributes) - - 123 - - 456 - - 789 notification: enabled: true # turn on object notification service endpoint: "tls://localhost:4222" # notification server endpoint diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 306577307..d21725668 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -122,7 +122,6 @@ contracts: | `balance` | `hash160` | | Balance contract hash. | | `container` | `hash160` | | Container contract hash. | | `netmap` | `hash160` | | Netmap contract hash. | -| `subnet` | `hash160` | | Subnet contract hash. | # `morph` section @@ -300,10 +299,6 @@ node: path: /sessions persistent_state: path: /state - subnet: - exit_zero: false - entries: - - 123 notification: enabled: true endpoint: tls://localhost:4222 @@ -323,7 +318,6 @@ node: | `relay` | `bool` | | Enable relay mode. | | `persistent_sessions` | [Persistent sessions config](#persistent_sessions-subsection) | | Persistent session token store configuration. | | `persistent_state` | [Persistent state config](#persistent_state-subsection) | | Persistent state configuration. | -| `subnet` | [Subnet config](#subnet-subsection) | | Subnet configuration. | | `notification` | [Notification config](#notification-subsection) | | NATS configuration. | @@ -352,14 +346,6 @@ It is used to correctly handle node restarts or crashes. |-----------|----------|------------------------|------------------------| | `path` | `string` | `.frostfs-storage-state` | Path to the database. | -## `subnet` subsection -This is an advanced section, use with caution. - -| Parameter | Type | Default value | Description | -|-------------|------------|---------------|------------------------------------------------------| -| `exit_zero` | `bool` | `false` | Exit from the default subnet. | -| `entries` | `[]uint32` | | List of non-default subnet ID this node belongs to. | - ## `notification` subsection This is an advanced section, use with caution. From f07d4158f50ed5c7f44cc0bc224c3d03edf27f3b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Apr 2023 13:47:08 +0300 Subject: [PATCH 0125/1943] [#249] node: Drop subnet from IR and morph Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/defaults.go | 2 - config/example/ir.env | 2 - config/example/ir.yaml | 2 - pkg/innerring/contracts.go | 2 - pkg/innerring/initialization.go | 50 +-- pkg/innerring/innerring.go | 6 - .../processors/container/process_container.go | 31 -- .../processors/container/processor.go | 6 - pkg/innerring/processors/netmap/handlers.go | 20 - .../netmap/nodevalidation/subnet/calls.go | 42 --- .../netmap/nodevalidation/subnet/validator.go | 41 -- .../processors/netmap/process_peers.go | 74 ---- pkg/innerring/processors/netmap/processor.go | 23 -- pkg/innerring/processors/subnet/common.go | 22 -- .../processors/subnet/common_test.go | 19 - pkg/innerring/processors/subnet/put.go | 82 ---- pkg/innerring/processors/subnet/put_test.go | 113 ------ pkg/innerring/subnet.go | 355 ------------------ pkg/morph/client/nns.go | 2 - pkg/morph/client/subnet/admin.go | 87 ----- pkg/morph/client/subnet/client.go | 108 ------ pkg/morph/client/subnet/clients.go | 114 ------ pkg/morph/client/subnet/delete.go | 40 -- pkg/morph/client/subnet/get.go | 55 --- pkg/morph/client/subnet/node.go | 58 --- pkg/morph/client/subnet/nodes.go | 54 --- pkg/morph/client/subnet/put.go | 50 --- pkg/morph/event/subnet/delete.go | 63 ---- pkg/morph/event/subnet/delete_test.go | 42 --- pkg/morph/event/subnet/put.go | 147 -------- pkg/morph/event/subnet/put_test.go | 69 ---- pkg/morph/event/subnet/remove_node.go | 69 ---- pkg/morph/event/subnet/remove_node_test.go | 56 --- pkg/services/control/ir/service.pb.go | 2 +- pkg/services/control/ir/service_grpc.pb.go | 12 +- pkg/services/control/ir/types.pb.go | 2 +- pkg/services/control/types.pb.go | 5 +- pkg/services/control/types.proto | 3 - pkg/services/tree/service.pb.go | 2 +- pkg/services/tree/service_grpc.pb.go | 56 ++- pkg/services/tree/types.pb.go | 2 +- 41 files changed, 55 insertions(+), 1935 deletions(-) delete mode 100644 pkg/innerring/processors/netmap/nodevalidation/subnet/calls.go delete mode 100644 pkg/innerring/processors/netmap/nodevalidation/subnet/validator.go delete mode 100644 pkg/innerring/processors/subnet/common.go delete mode 100644 pkg/innerring/processors/subnet/common_test.go delete mode 100644 pkg/innerring/processors/subnet/put.go delete mode 100644 pkg/innerring/processors/subnet/put_test.go delete mode 100644 pkg/innerring/subnet.go delete mode 100644 pkg/morph/client/subnet/admin.go delete mode 100644 pkg/morph/client/subnet/client.go delete mode 100644 pkg/morph/client/subnet/clients.go delete mode 100644 pkg/morph/client/subnet/delete.go delete mode 100644 pkg/morph/client/subnet/get.go delete mode 100644 pkg/morph/client/subnet/node.go delete mode 100644 pkg/morph/client/subnet/nodes.go delete mode 100644 pkg/morph/client/subnet/put.go delete mode 100644 pkg/morph/event/subnet/delete.go delete mode 100644 pkg/morph/event/subnet/delete_test.go delete mode 100644 pkg/morph/event/subnet/put.go delete mode 100644 pkg/morph/event/subnet/put_test.go delete mode 100644 pkg/morph/event/subnet/remove_node.go delete mode 100644 pkg/morph/event/subnet/remove_node_test.go diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index 837c1f170..693dd08ee 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -142,7 +142,6 @@ func setWorkersDefaults(cfg *viper.Viper) { cfg.SetDefault("workers.frostfs", "10") cfg.SetDefault("workers.container", "10") cfg.SetDefault("workers.alphabet", "10") - cfg.SetDefault("workers.subnet", "10") } func setTimersDefaults(cfg *viper.Viper) { @@ -163,7 +162,6 @@ func setContractsDefaults(cfg *viper.Viper) { cfg.SetDefault("contracts.audit", "") cfg.SetDefault("contracts.proxy", "") cfg.SetDefault("contracts.processing", "") - cfg.SetDefault("contracts.subnet", "") cfg.SetDefault("contracts.proxy", "") } diff --git a/config/example/ir.env b/config/example/ir.env index b7816dd77..1d306ebba 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -49,7 +49,6 @@ FROSTFS_IR_WORKERS_BALANCE=10 FROSTFS_IR_WORKERS_CONTAINER=10 FROSTFS_IR_WORKERS_NEOFS=10 FROSTFS_IR_WORKERS_NETMAP=10 -FROSTFS_IR_WORKERS_SUBNET=10 FROSTFS_IR_AUDIT_TIMEOUT_GET=5s FROSTFS_IR_AUDIT_TIMEOUT_HEAD=5s @@ -74,7 +73,6 @@ FROSTFS_IR_CONTRACTS_CONTAINER=ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6 FROSTFS_IR_CONTRACTS_NEOFSID=9f5866decbc751a099e74c7c7bc89f609201755a FROSTFS_IR_CONTRACTS_NETMAP=83c600c81d47a1b1b7cf58eb49ae7ee7240dc742 FROSTFS_IR_CONTRACTS_PROXY=abc8794bb40a21f2db5f21ae62741eb46c8cad1c -FROSTFS_IR_CONTRACTS_SUBNET=e9266864d3c562c6e17f2bb9cb1392aaa293d93a FROSTFS_IR_CONTRACTS_ALPHABET_AMOUNT=7 FROSTFS_IR_CONTRACTS_ALPHABET_AZ=c1d211fceeb4b1dc76b8e4054d11fdf887e418ea FROSTFS_IR_CONTRACTS_ALPHABET_BUKY=e2ba789320899658b100f331bdebb74474757920 diff --git a/config/example/ir.yaml b/config/example/ir.yaml index 0e1b12bf6..85c1b5d9a 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -83,7 +83,6 @@ workers: container: 10 # Number of workers to process events from container contract in parallel frostfs: 10 # Number of workers to process events from frostfs contracts in parallel netmap: 10 # Number of workers to process events from netmap contract in parallel - subnet: 10 # Number of workers to process events from subnet contract in parallel audit: timeout: @@ -116,7 +115,6 @@ contracts: frostfsid: 9f5866decbc751a099e74c7c7bc89f609201755a # Optional: override address of frostfsid contract in sidechain netmap: 83c600c81d47a1b1b7cf58eb49ae7ee7240dc742 # Optional: override address of netmap contract in sidechain proxy: abc8794bb40a21f2db5f21ae62741eb46c8cad1c # Optional: override address of proxy contract in sidechain; ignore if notary is disabled in sidechain - subnet: e9266864d3c562c6e17f2bb9cb1392aaa293d93a # Optional: override address of subnet contract in sidechain alphabet: amount: 7 # Optional: override amount of alphabet contracts az: c1d211fceeb4b1dc76b8e4054d11fdf887e418ea # Optional: override address of az alphabet contract in sidechain diff --git a/pkg/innerring/contracts.go b/pkg/innerring/contracts.go index f723d3507..def55f22a 100644 --- a/pkg/innerring/contracts.go +++ b/pkg/innerring/contracts.go @@ -18,7 +18,6 @@ type contracts struct { audit util.Uint160 // in morph proxy util.Uint160 // in morph processing util.Uint160 // in mainnet - subnet util.Uint160 // in morph frostfsID util.Uint160 // in morph alphabet alphabetContracts // in morph @@ -60,7 +59,6 @@ func parseContracts(cfg *viper.Viper, morph *client.Client, withoutMainNet, with {"contracts.balance", client.NNSBalanceContractName, &result.balance}, {"contracts.container", client.NNSContainerContractName, &result.container}, {"contracts.audit", client.NNSAuditContractName, &result.audit}, - {"contracts.subnet", client.NNSSubnetworkContractName, &result.subnet}, {"contracts.frostfsid", client.NNSFrostFSIDContractName, &result.frostfsID}, } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 5ac3154bc..31602a48e 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -17,7 +17,6 @@ import ( nodevalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation" addrvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/maddress" statevalidation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" - subnetvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/subnet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement" auditSettlement "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" @@ -28,7 +27,6 @@ import ( frostfsClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" - morphsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/subnet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" audittask "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit/taskmanager" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" @@ -46,7 +44,6 @@ import ( func (s *Server) initNetmapProcessor(cfg *viper.Viper, cnrClient *container.Client, alphaSync event.Handler, - subnetClient *morphsubnet.Client, auditProcessor *audit.Processor, settlementProcessor *settlement.Processor) error { locodeValidator, err := s.newLocodeValidator(cfg) @@ -54,11 +51,6 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, return err } - subnetValidator, err := subnetvalidator.New( - subnetvalidator.Prm{ - SubnetClient: subnetClient, - }, - ) if err != nil { return err } @@ -94,10 +86,8 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, &netMapCandidateStateValidator, addrvalidator.New(), locodeValidator, - subnetValidator, ), NotaryDisabled: s.sideNotaryConfig.disabled, - SubnetContract: &s.contracts.subnet, NodeStateSettings: netSettings, }) @@ -349,27 +339,6 @@ func (s *Server) initTimers(cfg *viper.Viper, processors *serverProcessors, morp s.addBlockTimer(emissionTimer) } -func (s *Server) createMorphSubnetClient() (*morphsubnet.Client, error) { - // initialize morph client of Subnet contract - clientMode := morphsubnet.NotaryAlphabet - - if s.sideNotaryConfig.disabled { - clientMode = morphsubnet.NonNotary - } - - subnetInitPrm := morphsubnet.InitPrm{} - subnetInitPrm.SetBaseClient(s.morphClient) - subnetInitPrm.SetContractAddress(s.contracts.subnet) - subnetInitPrm.SetMode(clientMode) - - subnetClient := &morphsubnet.Client{} - err := subnetClient.Init(subnetInitPrm) - if err != nil { - return nil, fmt.Errorf("could not initialize subnet client: %w", err) - } - return subnetClient, nil -} - func (s *Server) initAlphabetProcessor(cfg *viper.Viper) (*alphabet.Processor, error) { parsedWallets, err := parseWalletAddressesFromStrings(cfg.GetStringSlice("emit.extra_wallets")) if err != nil { @@ -400,7 +369,7 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) (*alphabet.Processor, e } func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.Client, - frostfsIDClient *frostfsid.Client, subnetClient *morphsubnet.Client) error { + frostfsIDClient *frostfsid.Client) error { // container processor containerProcessor, err := cont.New(&cont.Params{ Log: s.log, @@ -410,7 +379,6 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C FrostFSIDClient: frostfsIDClient, NetworkState: s.netmapClient, NotaryDisabled: s.sideNotaryConfig.disabled, - SubnetClient: subnetClient, }) if err != nil { return err @@ -515,10 +483,9 @@ func (s *Server) initGRPCServer(cfg *viper.Viper) error { } type serverMorphClients struct { - CnrClient *container.Client - FrostFSIDClient *frostfsid.Client - FrostFSClient *frostfsClient.Client - MorphSubnetClient *morphsubnet.Client + CnrClient *container.Client + FrostFSIDClient *frostfsid.Client + FrostFSClient *frostfsClient.Client } func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { @@ -574,11 +541,6 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { return nil, err } - result.MorphSubnetClient, err = s.createMorphSubnetClient() - if err != nil { - return nil, err - } - return result, nil } @@ -624,12 +586,12 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien return nil, err } - err = s.initNetmapProcessor(cfg, morphClients.CnrClient, alphaSync, morphClients.MorphSubnetClient, auditProcessor, result.SettlementProcessor) + err = s.initNetmapProcessor(cfg, morphClients.CnrClient, alphaSync, auditProcessor, result.SettlementProcessor) if err != nil { return nil, err } - err = s.initContainerProcessor(cfg, morphClients.CnrClient, morphClients.FrostFSIDClient, morphClients.MorphSubnetClient) + err = s.initContainerProcessor(cfg, morphClients.CnrClient, morphClients.FrostFSIDClient) if err != nil { return nil, err } diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index b6c5ae2ac..42917f1d6 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -100,8 +100,6 @@ type ( // should report start errors // to the application. runners []func(chan<- error) error - - subnetHandler } chainParams struct { @@ -400,10 +398,6 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - server.initSubnet(subnetConfig{ - queueSize: cfg.GetUint32("workers.subnet"), - }) - server.initMetrics(cfg) return server, nil diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 5ebe58375..c4421c453 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -5,13 +5,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - morphsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/subnet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" - subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" "github.com/nspcc-dev/neo-go/pkg/network/payload" "go.uber.org/zap" ) @@ -77,12 +75,6 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { return fmt.Errorf("auth container creation: %w", err) } - // check owner allowance in the subnetwork - err = checkSubnet(cp.subnetClient, cnr) - if err != nil { - return fmt.Errorf("incorrect subnetwork: %w", err) - } - // check homomorphic hashing setting err = checkHomomorphicHashing(cp.netState, cnr) if err != nil { @@ -222,29 +214,6 @@ func checkNNS(ctx *putContainerContext, cnr containerSDK.Container) error { return nil } -func checkSubnet(subCli *morphsubnet.Client, cnr containerSDK.Container) error { - prm := morphsubnet.UserAllowedPrm{} - - subID := cnr.PlacementPolicy().Subnet() - if subnetid.IsZero(subID) { - return nil - } - - prm.SetID(subID.Marshal()) - prm.SetClient(cnr.Owner().WalletBytes()) - - res, err := subCli.UserAllowed(prm) - if err != nil { - return fmt.Errorf("could not check user in contract: %w", err) - } - - if !res.Allowed() { - return fmt.Errorf("user is not allowed to create containers in %v subnetwork", subID) - } - - return nil -} - func checkHomomorphicHashing(ns NetworkState, cnr containerSDK.Container) error { netSetting, err := ns.HomomorphicHashDisabled() if err != nil { diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index 56d2eee96..321596eb4 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" - morphsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/subnet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -29,7 +28,6 @@ type ( alphabetState AlphabetState cnrClient *container.Client // notary must be enabled idClient *frostfsid.Client - subnetClient *morphsubnet.Client netState NetworkState notaryDisabled bool } @@ -41,7 +39,6 @@ type ( AlphabetState AlphabetState ContainerClient *container.Client FrostFSIDClient *frostfsid.Client - SubnetClient *morphsubnet.Client NetworkState NetworkState NotaryDisabled bool } @@ -85,8 +82,6 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/container: FrostFS ID client is not set") case p.NetworkState == nil: return nil, errors.New("ir/container: network state is not set") - case p.SubnetClient == nil: - return nil, errors.New("ir/container: subnet client is not set") } p.Log.Debug(logs.ContainerContainerWorkerPool, zap.Int("size", p.PoolSize)) @@ -104,7 +99,6 @@ func New(p *Params) (*Processor, error) { idClient: p.FrostFSIDClient, netState: p.NetworkState, notaryDisabled: p.NotaryDisabled, - subnetClient: p.SubnetClient, }, nil } diff --git a/pkg/innerring/processors/netmap/handlers.go b/pkg/innerring/processors/netmap/handlers.go index 60d279940..6adeac562 100644 --- a/pkg/innerring/processors/netmap/handlers.go +++ b/pkg/innerring/processors/netmap/handlers.go @@ -7,7 +7,6 @@ import ( timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" - subnetevents "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/subnet" "go.uber.org/zap" ) @@ -101,22 +100,3 @@ func (np *Processor) handleCleanupTick(ev event.Event) { zap.Int("capacity", np.pool.Cap())) } } - -func (np *Processor) handleRemoveNode(ev event.Event) { - removeNode := ev.(subnetevents.RemoveNode) - - np.log.Info(logs.Notification, - zap.String("type", "remove node from subnet"), - zap.String("subnetID", hex.EncodeToString(removeNode.SubnetworkID())), - zap.String("key", hex.EncodeToString(removeNode.Node())), - ) - - err := np.pool.Submit(func() { - np.processRemoveSubnetNode(removeNode) - }) - if err != nil { - // there system can be moved into controlled degradation stage - np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, - zap.Int("capacity", np.pool.Cap())) - } -} diff --git a/pkg/innerring/processors/netmap/nodevalidation/subnet/calls.go b/pkg/innerring/processors/netmap/nodevalidation/subnet/calls.go deleted file mode 100644 index 4c859703d..000000000 --- a/pkg/innerring/processors/netmap/nodevalidation/subnet/calls.go +++ /dev/null @@ -1,42 +0,0 @@ -package subnet - -import ( - "fmt" - - morphsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/subnet" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" -) - -// VerifyAndUpdate calls subnet contract's `NodeAllowed` method. -// Removes subnets that have not been approved by the contract. -func (v *Validator) VerifyAndUpdate(n *netmap.NodeInfo) error { - prm := morphsubnet.NodeAllowedPrm{} - - err := n.IterateSubnets(func(id subnetid.ID) error { - // every node can be bootstrapped - // to the zero subnetwork - if subnetid.IsZero(id) { - return nil - } - - prm.SetID(id.Marshal()) - prm.SetNode(n.PublicKey()) - - res, err := v.subnetClient.NodeAllowed(prm) - if err != nil { - return fmt.Errorf("could not call `NodeAllowed` contract method: %w", err) - } - - if !res.Allowed() { - return netmap.ErrRemoveSubnet - } - - return nil - }) - if err != nil { - return fmt.Errorf("could not verify subnet entrance of the node: %w", err) - } - - return nil -} diff --git a/pkg/innerring/processors/netmap/nodevalidation/subnet/validator.go b/pkg/innerring/processors/netmap/nodevalidation/subnet/validator.go deleted file mode 100644 index f9ae4e614..000000000 --- a/pkg/innerring/processors/netmap/nodevalidation/subnet/validator.go +++ /dev/null @@ -1,41 +0,0 @@ -package subnet - -import ( - "errors" - - morphsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/subnet" -) - -// Validator is an utility that verifies node subnet -// allowance. -// -// For correct operation, Validator must be created -// using the constructor (New). After successful creation, -// the Validator is immediately ready to work through API. -type Validator struct { - subnetClient *morphsubnet.Client -} - -// Prm groups the required parameters of the Validator's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - SubnetClient *morphsubnet.Client -} - -// New creates a new instance of the Validator. -// -// The created Validator does not require additional -// initialization and is completely ready for work. -func New(prm Prm) (*Validator, error) { - switch { - case prm.SubnetClient == nil: - return nil, errors.New("ir/nodeValidator: subnet client is not set") - } - - return &Validator{ - subnetClient: prm.SubnetClient, - }, nil -} diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index 130d08568..bf54ed341 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -1,15 +1,12 @@ package netmap import ( - "bytes" "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" - subnetEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/subnet" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" "go.uber.org/zap" ) @@ -139,74 +136,3 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { np.log.Error(logs.NetmapCantInvokeNetmapUpdatePeer, zap.Error(err)) } } - -func (np *Processor) processRemoveSubnetNode(ev subnetEvent.RemoveNode) { - if !np.alphabetState.IsAlphabet() { - np.log.Info(logs.NetmapNonAlphabetModeIgnoreRemoveNodeFromSubnetNotification) - return - } - - candidates, err := np.netmapClient.GetCandidates() - if err != nil { - np.log.Warn(logs.NetmapCouldNotGetNetworkMapCandidates, - zap.Error(err), - ) - return - } - - rawSubnet := ev.SubnetworkID() - var subnetToRemoveFrom subnetid.ID - - err = subnetToRemoveFrom.Unmarshal(rawSubnet) - if err != nil { - np.log.Warn(logs.NetmapCouldNotUnmarshalSubnetID, - zap.Error(err), - ) - return - } - - if subnetid.IsZero(subnetToRemoveFrom) { - np.log.Warn(logs.NetmapGotZeroSubnetInRemoveNodeNotification) - return - } - - for i := range candidates { - if !bytes.Equal(candidates[i].PublicKey(), ev.Node()) { - continue - } - - err = candidates[i].IterateSubnets(func(subNetID subnetid.ID) error { - if subNetID.Equals(subnetToRemoveFrom) { - return netmap.ErrRemoveSubnet - } - - return nil - }) - if err != nil { - np.log.Warn(logs.NetmapCouldNotIterateOverSubnetworksOfTheNode, zap.Error(err)) - np.log.Info(logs.NetmapVoteToRemoveNodeFromNetmap, zap.String("key", hex.EncodeToString(ev.Node()))) - - prm := netmapclient.UpdatePeerPrm{} - prm.SetKey(ev.Node()) - prm.SetHash(ev.TxHash()) - - err = np.netmapClient.UpdatePeerState(prm) - if err != nil { - np.log.Error(logs.NetmapCouldNotInvokeNetmapUpdateState, zap.Error(err)) - return - } - } else { - prm := netmapclient.AddPeerPrm{} - prm.SetNodeInfo(candidates[i]) - prm.SetHash(ev.TxHash()) - - err = np.netmapClient.AddPeer(prm) - if err != nil { - np.log.Error(logs.NetmapCouldNotInvokeNetmapAddPeer, zap.Error(err)) - return - } - } - - break - } -} diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index 035ccfafd..f6f490cec 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -10,11 +10,9 @@ import ( nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" - subnetEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/subnet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/nspcc-dev/neo-go/pkg/core/mempoolevent" - "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" "go.uber.org/zap" ) @@ -65,8 +63,6 @@ type ( netmapClient *nmClient.Client containerWrp *container.Client - subnetContract util.Uint160 - netmapSnapshot cleanupTable handleNewAudit event.Handler @@ -92,7 +88,6 @@ type ( CleanupEnabled bool CleanupThreshold uint64 // in epochs ContainerWrapper *container.Client - SubnetContract *util.Uint160 HandleAudit event.Handler AuditSettlementsHandler event.Handler @@ -111,7 +106,6 @@ const ( newEpochNotification = "NewEpoch" addPeerNotification = "AddPeer" updatePeerStateNotification = "UpdateState" - removeNodeNotification = "RemoveNode" ) // New creates network map contract processor instance. @@ -137,8 +131,6 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/netmap: container contract wrapper is not set") case p.NodeValidator == nil: return nil, errors.New("ir/netmap: node validator is not set") - case p.SubnetContract == nil: - return nil, errors.New("ir/netmap: subnet contract script hash is not set") case p.NodeStateSettings == nil: return nil, errors.New("ir/netmap: node state settings is not set") } @@ -160,7 +152,6 @@ func New(p *Params) (*Processor, error) { containerWrp: p.ContainerWrapper, netmapSnapshot: newCleanupTable(p.CleanupEnabled, p.CleanupThreshold), handleNewAudit: p.HandleAudit, - subnetContract: *p.SubnetContract, handleAuditSettlements: p.AuditSettlementsHandler, @@ -182,13 +173,6 @@ func (np *Processor) ListenerNotificationParsers() []event.NotificationParserInf var p event.NotificationParserInfo - // remove node from subnetwork event - p.SetScriptHash(np.subnetContract) - p.SetType(removeNodeNotification) - p.SetParser(subnetEvent.ParseRemoveNode) - - parsers = append(parsers, p) - p.SetScriptHash(np.netmapClient.ContractAddress()) // new epoch event @@ -219,13 +203,6 @@ func (np *Processor) ListenerNotificationHandlers() []event.NotificationHandlerI var i event.NotificationHandlerInfo - // remove node from subnetwork event - i.SetScriptHash(np.subnetContract) - i.SetType(removeNodeNotification) - i.SetHandler(np.handleRemoveNode) - - handlers = append(handlers, i) - i.SetScriptHash(np.netmapClient.ContractAddress()) // new epoch handler diff --git a/pkg/innerring/processors/subnet/common.go b/pkg/innerring/processors/subnet/common.go deleted file mode 100644 index 2026c8641..000000000 --- a/pkg/innerring/processors/subnet/common.go +++ /dev/null @@ -1,22 +0,0 @@ -package subnetevents - -import ( - "fmt" - - subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" -) - -// common interface of subnet notifications with subnet ID. -type eventWithID interface { - // ReadID reads identifier of the subnet. - ReadID(*subnetid.ID) error -} - -// an error which is returned on zero subnet operation attempt. -type zeroSubnetOp struct { - op string -} - -func (x zeroSubnetOp) Error() string { - return fmt.Sprintf("zero subnet %s", x.op) -} diff --git a/pkg/innerring/processors/subnet/common_test.go b/pkg/innerring/processors/subnet/common_test.go deleted file mode 100644 index 23e61a44a..000000000 --- a/pkg/innerring/processors/subnet/common_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package subnetevents - -import subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" - -type idEvent struct { - id subnetid.ID - - idErr error -} - -func (x idEvent) ReadID(id *subnetid.ID) error { - if x.idErr != nil { - return x.idErr - } - - *id = x.id - - return nil -} diff --git a/pkg/innerring/processors/subnet/put.go b/pkg/innerring/processors/subnet/put.go deleted file mode 100644 index ba1588756..000000000 --- a/pkg/innerring/processors/subnet/put.go +++ /dev/null @@ -1,82 +0,0 @@ -package subnetevents - -import ( - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet" - subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" -) - -// Put represents a notification about FrostFS subnet creation. -// Generated by a contract when intending to create a subnet. -type Put interface { - // Contains the ID of the subnet to be created. - eventWithID - - // ReadCreator reads the user ID of the subnet creator. - // Returns an error if the ID is missing. - ReadCreator(id *user.ID) error - - // ReadInfo reads information about a subnet to be created. - ReadInfo(info *subnet.Info) error -} - -// PutValidator asserts intent to create a subnet. -type PutValidator struct{} - -// errDiffOwner is returned when the subnet owners differ. -var errDiffOwner = errors.New("diff subnet owners") - -// errDiffID is returned when the subnet IDs differ. -var errDiffID = errors.New("diff subnet IDs") - -// Assert processes the attempt to create a subnet. It approves the creation through nil return. -// -// All read errors of Put are forwarded. -// -// It returns an error on: -// - zero subnet creation; -// - empty ID or different from the one wired into info; -// - empty owner ID or different from the one wired into info. -func (x PutValidator) Assert(event Put) error { - var err error - - // read ID - var id subnetid.ID - if err = event.ReadID(&id); err != nil { - return fmt.Errorf("read ID: %w", err) - } - - // prevent zero subnet creation - if subnetid.IsZero(id) { - return zeroSubnetOp{ - op: "creation", - } - } - - // read creator's user ID in FrostFS system - var creator user.ID - if err = event.ReadCreator(&creator); err != nil { - return fmt.Errorf("read creator: %w", err) - } - - // read information about the subnet - var info subnet.Info - if err = event.ReadInfo(&info); err != nil { - return fmt.Errorf("read info: %w", err) - } - - // check if the explicit ID equals to the one from info - if !subnet.AssertReference(info, id) { - return errDiffID - } - - // check if the explicit creator equals to the one from info - if !subnet.AssertOwnership(info, creator) { - return errDiffOwner - } - - return nil -} diff --git a/pkg/innerring/processors/subnet/put_test.go b/pkg/innerring/processors/subnet/put_test.go deleted file mode 100644 index dda6ee90a..000000000 --- a/pkg/innerring/processors/subnet/put_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package subnetevents - -import ( - "errors" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" - "github.com/stretchr/testify/require" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet" - subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" -) - -type put struct { - idEvent - - creator user.ID - - creatorErr error - - info subnet.Info - - infoErr error -} - -func (x put) ReadCreator(id *user.ID) error { - if x.creatorErr != nil { - return x.creatorErr - } - - *id = x.creator - - return nil -} - -func (x put) ReadInfo(info *subnet.Info) error { - if x.infoErr != nil { - return x.infoErr - } - - *info = x.info - - return nil -} - -func TestPutValidator_Assert(t *testing.T) { - var ( - v PutValidator - - e put - - err error - ) - - // read ID error - e.idErr = errors.New("id err") - - err = v.Assert(e) - require.ErrorIs(t, err, e.idErr) - - e.idErr = nil - - // zero subnet ID - subnetid.MakeZero(&e.id) - - err = v.Assert(e) - require.ErrorAs(t, err, new(zeroSubnetOp)) - - const idNum = 13 - e.id.SetNumeric(idNum) - - // read creator error - e.creatorErr = errors.New("creator err") - - err = v.Assert(e) - require.ErrorIs(t, err, e.creatorErr) - - e.creatorErr = nil - - // read info error - e.infoErr = errors.New("info err") - - err = v.Assert(e) - require.ErrorIs(t, err, e.infoErr) - - e.infoErr = nil - - // diff explicit ID and the one in info - var id2 subnetid.ID - - id2.SetNumeric(idNum + 1) - - e.info.SetID(id2) - - err = v.Assert(e) - require.ErrorIs(t, err, errDiffID) - - e.info.SetID(e.id) - - // diff explicit creator and the one in info - creator2 := *usertest.ID() - - e.info.SetOwner(creator2) - - err = v.Assert(e) - require.ErrorIs(t, err, errDiffOwner) - - e.info.SetOwner(e.creator) - - err = v.Assert(e) - require.NoError(t, err) -} diff --git a/pkg/innerring/subnet.go b/pkg/innerring/subnet.go deleted file mode 100644 index 03108aac2..000000000 --- a/pkg/innerring/subnet.go +++ /dev/null @@ -1,355 +0,0 @@ -package innerring - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - irsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/subnet" - netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" - morphsubnet "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/subnet" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - subnetevents "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/subnet" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet" - subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/core/mempoolevent" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - neogoutil "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/panjf2000/ants/v2" - "go.uber.org/zap" -) - -// IR server's component to handle Subnet contract notifications. -type subnetHandler struct { - workerPool util.WorkerPool - - morphClient morphsubnet.Client - - putValidator irsubnet.PutValidator -} - -// configuration of subnet component. -type subnetConfig struct { - queueSize uint32 -} - -// makes IR server to catch Subnet notifications from the sidechain listener, -// and to release the corresponding processing queue on stop. -func (s *Server) initSubnet(cfg subnetConfig) { - s.registerStarter(func() error { - var err error - - // initialize queue for processing of the events from Subnet contract - s.subnetHandler.workerPool, err = ants.NewPool(int(cfg.queueSize), ants.WithNonblocking(true)) - if err != nil { - return fmt.Errorf("subnet queue initialization: %w", err) - } - - // initialize morph client of Subnet contract - clientMode := morphsubnet.NotaryAlphabet - - if s.sideNotaryConfig.disabled { - clientMode = morphsubnet.NonNotary - } - - var initPrm morphsubnet.InitPrm - - initPrm.SetBaseClient(s.morphClient) - initPrm.SetContractAddress(s.contracts.subnet) - initPrm.SetMode(clientMode) - - err = s.subnetHandler.morphClient.Init(initPrm) - if err != nil { - return fmt.Errorf("init morph subnet client: %w", err) - } - - s.listenSubnet() - - return nil - }) - - s.registerCloser(func() error { - s.stopSubnet() - return nil - }) -} - -// releases the Subnet contract notification processing queue. -func (s *Server) stopSubnet() { - s.workerPool.Release() -} - -// names of listened notification events from Subnet contract. -const ( - subnetCreateEvName = "Put" - subnetRemoveEvName = "Delete" - notarySubnetCreateEvName = "put" -) - -// makes the IR server to listen to notifications of Subnet contract. -// All required resources must be initialized before (initSubnet). -// It works in one of two modes (configured): notary and non-notary. -// -// All handlers are executed only if the local node is an alphabet one. -// -// Events (notary): -// - put (parser: subnetevents.ParseNotaryPut, handler: catchSubnetCreation); -// - Delete (parser: subnetevents.ParseDelete, handler: catchSubnetCreation). -// -// Events (non-notary): -// - Put (parser: subnetevents.ParsePut, handler: catchSubnetCreation); -// - Delete (parser: subnetevents.ParseDelete, handler: catchSubnetCreation). -func (s *Server) listenSubnet() { - if s.sideNotaryConfig.disabled { - s.listenSubnetWithoutNotary() - return - } - - var ( - parserInfo event.NotaryParserInfo - handlerInfo event.NotaryHandlerInfo - ) - - parserInfo.SetScriptHash(s.contracts.subnet) - handlerInfo.SetScriptHash(s.contracts.subnet) - - listenNotaryEvent := func(notifyName string, parser event.NotaryParser, handler event.Handler) { - notifyTyp := event.NotaryTypeFromString(notifyName) - - parserInfo.SetMempoolType(mempoolevent.TransactionAdded) - handlerInfo.SetMempoolType(mempoolevent.TransactionAdded) - - parserInfo.SetParser(parser) - handlerInfo.SetHandler(handler) - - parserInfo.SetRequestType(notifyTyp) - handlerInfo.SetRequestType(notifyTyp) - - s.morphListener.SetNotaryParser(parserInfo) - s.morphListener.RegisterNotaryHandler(handlerInfo) - } - - // subnet creation - listenNotaryEvent(notarySubnetCreateEvName, subnetevents.ParseNotaryPut, s.onlyAlphabetEventHandler(s.catchSubnetCreation)) - // subnet removal - listenNotifySubnetEvent(s, subnetRemoveEvName, subnetevents.ParseDelete, s.onlyAlphabetEventHandler(s.catchSubnetRemoval)) -} - -func (s *Server) listenSubnetWithoutNotary() { - // subnet creation - listenNotifySubnetEvent(s, subnetCreateEvName, subnetevents.ParsePut, s.onlyAlphabetEventHandler(s.catchSubnetCreation)) - // subnet removal - listenNotifySubnetEvent(s, subnetRemoveEvName, subnetevents.ParseDelete, s.onlyAlphabetEventHandler(s.catchSubnetRemoval)) -} - -func listenNotifySubnetEvent(s *Server, notifyName string, parser event.NotificationParser, handler event.Handler) { - var ( - parserInfo event.NotificationParserInfo - handlerInfo event.NotificationHandlerInfo - ) - - parserInfo.SetScriptHash(s.contracts.subnet) - handlerInfo.SetScriptHash(s.contracts.subnet) - - notifyTyp := event.TypeFromString(notifyName) - - parserInfo.SetType(notifyTyp) - handlerInfo.SetType(notifyTyp) - - parserInfo.SetParser(parser) - handlerInfo.SetHandler(handler) - - s.morphListener.SetNotificationParser(parserInfo) - s.morphListener.RegisterNotificationHandler(handlerInfo) -} - -// catchSubnetCreation catches event of subnet creation from listener and queues the processing. -func (s *Server) catchSubnetCreation(e event.Event) { - err := s.subnetHandler.workerPool.Submit(func() { - s.handleSubnetCreation(e) - }) - if err != nil { - s.log.Error(logs.InnerringSubnetCreationQueueFailure, - zap.String("error", err.Error()), - ) - } -} - -// implements irsubnet.Put event interface required by irsubnet.PutValidator. -type putSubnetEvent struct { - ev subnetevents.Put -} - -// ReadID unmarshals the subnet ID from a binary FrostFS API protocol's format. -func (x putSubnetEvent) ReadID(id *subnetid.ID) error { - return id.Unmarshal(x.ev.ID()) -} - -var errMissingSubnetOwner = errors.New("missing subnet owner") - -// ReadCreator unmarshals the subnet creator from a binary FrostFS API protocol's format. -// Returns an error if the byte array is empty. -func (x putSubnetEvent) ReadCreator(id *user.ID) error { - data := x.ev.Owner() - - if len(data) == 0 { - return errMissingSubnetOwner - } - - key, err := keys.NewPublicKeyFromBytes(data, elliptic.P256()) - if err != nil { - return err - } - - user.IDFromKey(id, (ecdsa.PublicKey)(*key)) - - return nil -} - -// ReadInfo unmarshal the subnet info from a binary FrostFS API protocol's format. -func (x putSubnetEvent) ReadInfo(info *subnet.Info) error { - return info.Unmarshal(x.ev.Info()) -} - -// handleSubnetCreation handles an event of subnet creation parsed via subnetevents.ParsePut. -// -// Validates the event using irsubnet.PutValidator. Logs message about (dis)agreement. -func (s *Server) handleSubnetCreation(e event.Event) { - putEv := e.(subnetevents.Put) // panic occurs only if we registered handler incorrectly - - err := s.subnetHandler.putValidator.Assert(putSubnetEvent{ - ev: putEv, - }) - if err != nil { - s.log.Info(logs.InnerringDiscardSubnetCreation, - zap.String("reason", err.Error()), - ) - - return - } - - notaryMainTx := putEv.NotaryMainTx() - - isNotary := notaryMainTx != nil - if isNotary { - // re-sign notary request - err = s.morphClient.NotarySignAndInvokeTX(notaryMainTx) - } else { - // send new transaction - var prm morphsubnet.PutPrm - - prm.SetID(putEv.ID()) - prm.SetOwner(putEv.Owner()) - prm.SetInfo(putEv.Info()) - prm.SetTxHash(putEv.TxHash()) - - _, err = s.subnetHandler.morphClient.Put(prm) - } - - if err != nil { - s.log.Error(logs.InnerringApproveSubnetCreation, - zap.Bool("notary", isNotary), - zap.String("error", err.Error()), - ) - - return - } -} - -// catchSubnetRemoval catches an event of subnet removal from listener and queues the processing. -func (s *Server) catchSubnetRemoval(e event.Event) { - err := s.subnetHandler.workerPool.Submit(func() { - s.handleSubnetRemoval(e) - }) - if err != nil { - s.log.Error(logs.InnerringSubnetRemovalHandlingFailure, - zap.String("error", err.Error()), - ) - } -} - -// handleSubnetRemoval handles event of subnet removal parsed via subnetevents.ParseDelete. -func (s *Server) handleSubnetRemoval(e event.Event) { - delEv := e.(subnetevents.Delete) // panic occurs only if we registered handler incorrectly - - // handle subnet changes in netmap - - candidates, err := s.netmapClient.GetCandidates() - if err != nil { - s.log.Error(logs.InnerringGettingNetmapCandidates, - zap.Error(err), - ) - - return - } - - var removedID subnetid.ID - err = removedID.Unmarshal(delEv.ID()) - if err != nil { - s.log.Error(logs.InnerringUnmarshallingRemovedSubnetID, - zap.String("error", err.Error()), - ) - - return - } - - for i := range candidates { - s.processCandidate(delEv.TxHash(), removedID, candidates[i]) - } -} - -func (s *Server) processCandidate(txHash neogoutil.Uint256, removedID subnetid.ID, c netmap.NodeInfo) { - removeSubnet := false - log := s.log.With( - zap.String("public_key", netmap.StringifyPublicKey(c)), - zap.String("removed_subnet", removedID.String()), - ) - - err := c.IterateSubnets(func(id subnetid.ID) error { - if removedID.Equals(id) { - removeSubnet = true - return netmap.ErrRemoveSubnet - } - - return nil - }) - if err != nil { - log.Error(logs.InnerringIteratingNodesSubnets, zap.Error(err)) - log.Debug(logs.InnerringRemovingNodeFromNetmapCandidates) - - var updateStatePrm netmapclient.UpdatePeerPrm - updateStatePrm.SetKey(c.PublicKey()) - updateStatePrm.SetHash(txHash) - - err = s.netmapClient.UpdatePeerState(updateStatePrm) - if err != nil { - log.Error(logs.InnerringRemovingNodeFromCandidates, - zap.Error(err), - ) - } - - return - } - - // remove subnet from node's information - // if it contains removed subnet - if removeSubnet { - log.Debug(logs.InnerringRemovingSubnetFromTheNode) - - var addPeerPrm netmapclient.AddPeerPrm - addPeerPrm.SetNodeInfo(c) - addPeerPrm.SetHash(txHash) - - err = s.netmapClient.AddPeer(addPeerPrm) - if err != nil { - log.Error(logs.InnerringUpdatingSubnetInfo, - zap.Error(err), - ) - } - } -} diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index 236cf1ba0..2f7079dfe 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -32,8 +32,6 @@ const ( NNSNetmapContractName = "netmap.frostfs" // NNSProxyContractName is a name of the proxy contract in NNS. NNSProxyContractName = "proxy.frostfs" - // NNSSubnetworkContractName is a name of the subnet contract in NNS. - NNSSubnetworkContractName = "subnet.frostfs" // NNSGroupKeyName is a name for the FrostFS group key record in NNS. NNSGroupKeyName = "group.frostfs" ) diff --git a/pkg/morph/client/subnet/admin.go b/pkg/morph/client/subnet/admin.go deleted file mode 100644 index 387da656d..000000000 --- a/pkg/morph/client/subnet/admin.go +++ /dev/null @@ -1,87 +0,0 @@ -package morphsubnet - -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - -// ManageAdminsPrm groups parameters of administer methods of Subnet contract. -// -// Zero value adds node admin. Subnet, key and group must be specified via setters. -type ManageAdminsPrm struct { - // remove or add admin - rm bool - - // client or node admin - client bool - - subnet []byte - - admin []byte - - group []byte -} - -// SetRemove marks admin to be removed. By default, admin is added. -func (x *ManageAdminsPrm) SetRemove() { - x.rm = true -} - -// SetClient switches to client admin. By default, node admin is modified. -func (x *ManageAdminsPrm) SetClient() { - x.client = true -} - -// SetSubnet sets identifier of the subnet in a binary FrostFS API protocol format. -func (x *ManageAdminsPrm) SetSubnet(id []byte) { - x.subnet = id -} - -// SetAdmin sets admin's public key in a binary format. -func (x *ManageAdminsPrm) SetAdmin(key []byte) { - x.admin = key -} - -// SetGroup sets identifier of the client group in a binary FrostFS API protocol format. -// Makes sense only for client admins (see ManageAdminsPrm.SetClient). -func (x *ManageAdminsPrm) SetGroup(id []byte) { - x.group = id -} - -// ManageAdminsRes groups the resulting values of node administer methods of Subnet contract. -type ManageAdminsRes struct{} - -// ManageAdmins manages admin list of the FrostFS subnet through Subnet contract calls. -func (x Client) ManageAdmins(prm ManageAdminsPrm) (*ManageAdminsPrm, error) { - var method string - - args := make([]any, 1, 3) - args[0] = prm.subnet - - if prm.client { - args = append(args, prm.group, prm.admin) - - if prm.rm { - method = removeClientAdminMethod - } else { - method = addClientAdminMethod - } - } else { - args = append(args, prm.admin) - - if prm.rm { - method = removeNodeAdminMethod - } else { - method = addNodeAdminMethod - } - } - - var prmInvoke client.InvokePrm - - prmInvoke.SetMethod(method) - prmInvoke.SetArgs(args...) - - err := x.client.Invoke(prmInvoke) - if err != nil { - return nil, err - } - - return new(ManageAdminsPrm), nil -} diff --git a/pkg/morph/client/subnet/client.go b/pkg/morph/client/subnet/client.go deleted file mode 100644 index 8cbae8f95..000000000 --- a/pkg/morph/client/subnet/client.go +++ /dev/null @@ -1,108 +0,0 @@ -package morphsubnet - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "github.com/nspcc-dev/neo-go/pkg/util" -) - -// Client represents Subnet contract client. -// -// Client should be preliminary initialized (see Init method). -type Client struct { - client *client.StaticClient -} - -// InitPrm groups parameters of Client's initialization. -type InitPrm struct { - base *client.Client - - addr util.Uint160 - - modeSet bool - mode Mode -} - -const ( - deleteMethod = "delete" - getMethod = "get" - putMethod = "put" - - removeClientAdminMethod = "removeClientAdmin" - addClientAdminMethod = "addClientAdmin" - - userAllowedMethod = "userAllowed" - removeUserMethod = "removeUser" - addUserMethod = "addUser" - - removeNodeAdminMethod = "removeNodeAdmin" - addNodeAdminMethod = "addNodeAdmin" - nodeAllowedMethod = "nodeAllowed" - removeNodeMethod = "removeNode" - addNodeMethod = "addNode" -) - -// SetBaseClient sets basic morph client. -func (x *InitPrm) SetBaseClient(base *client.Client) { - x.base = base -} - -// SetContractAddress sets address of Subnet contract in FrostFS sidechain. -func (x *InitPrm) SetContractAddress(addr util.Uint160) { - x.addr = addr -} - -// Mode regulates client work mode. -type Mode uint8 - -const ( - _ Mode = iota - - // NonNotary makes client to work in non-notary environment. - NonNotary - - // NotaryAlphabet makes client to use its internal key for signing the notary requests. - NotaryAlphabet - - // NotaryNonAlphabet makes client to not use its internal key for signing the notary requests. - NotaryNonAlphabet -) - -// SetMode makes client to work with non-notary sidechain. -// By default, NonNotary is used. -func (x *InitPrm) SetMode(mode Mode) { - x.modeSet = true - x.mode = mode -} - -// Init initializes client with specified parameters. -// -// Base client must be set. -func (x *Client) Init(prm InitPrm) error { - if prm.base == nil { - panic("missing base morph client") - } - - if !prm.modeSet { - prm.mode = NonNotary - } - - var opts []client.StaticClientOption - - switch prm.mode { - default: - panic(fmt.Sprintf("invalid work mode %d", prm.mode)) - case NonNotary: - case NotaryNonAlphabet: - opts = []client.StaticClientOption{client.TryNotary()} - case NotaryAlphabet: - opts = []client.StaticClientOption{client.TryNotary(), client.AsAlphabet()} - } - - var err error - - x.client, err = client.NewStatic(prm.base, prm.addr, 0, opts...) - - return err -} diff --git a/pkg/morph/client/subnet/clients.go b/pkg/morph/client/subnet/clients.go deleted file mode 100644 index 1c855496e..000000000 --- a/pkg/morph/client/subnet/clients.go +++ /dev/null @@ -1,114 +0,0 @@ -package morphsubnet - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" -) - -// UserAllowedPrm groups parameters of UserAllowed method of Subnet contract. -type UserAllowedPrm struct { - args [2]any -} - -// SetID sets identifier of the subnet in a binary FrostFS API protocol format. -func (x *UserAllowedPrm) SetID(id []byte) { - x.args[0] = id -} - -// SetClient sets owner ID of the client that is being checked in a binary FrostFS API protocol format. -func (x *UserAllowedPrm) SetClient(id []byte) { - x.args[1] = id -} - -// UserAllowedRes groups the resulting values of UserAllowed method of Subnet contract. -type UserAllowedRes struct { - result bool -} - -// Allowed returns true iff the client is allowed to create containers in the subnet. -func (x UserAllowedRes) Allowed() bool { - return x.result -} - -// UserAllowed checks if the user has access to the subnetwork. -func (x *Client) UserAllowed(prm UserAllowedPrm) (*UserAllowedRes, error) { - args := client.TestInvokePrm{} - - args.SetMethod(userAllowedMethod) - args.SetArgs(prm.args[:]...) - - res, err := x.client.TestInvoke(args) - if err != nil { - return nil, fmt.Errorf("could not make test invoke: %w", err) - } - - if len(res) == 0 { - return nil, errEmptyResponse - } - - result, err := client.BoolFromStackItem(res[0]) - if err != nil { - return nil, err - } - - return &UserAllowedRes{ - result: result, - }, nil -} - -// ManageClientsPrm groups parameters of client management in Subnet contract. -// -// Zero value adds subnet client. Subnet, group and client ID must be specified via setters. -type ManageClientsPrm struct { - // remove or add client - rm bool - - args [3]any -} - -// SetRemove marks client to be removed. By default, client is added. -func (x *ManageClientsPrm) SetRemove() { - x.rm = true -} - -// SetSubnet sets identifier of the subnet in a binary FrostFS API protocol format. -func (x *ManageClientsPrm) SetSubnet(id []byte) { - x.args[0] = id -} - -// SetGroup sets identifier of the client group in a binary FrostFS API protocol format. -func (x *ManageClientsPrm) SetGroup(id []byte) { - x.args[1] = id -} - -// SetClient sets client's user ID in a binary FrostFS API protocol format. -func (x *ManageClientsPrm) SetClient(id []byte) { - x.args[2] = id -} - -// ManageClientsRes groups the resulting values of client management methods of Subnet contract. -type ManageClientsRes struct{} - -// ManageClients manages client list of the FrostFS subnet through Subnet contract calls. -func (x Client) ManageClients(prm ManageClientsPrm) (*ManageClientsRes, error) { - var method string - - if prm.rm { - method = removeUserMethod - } else { - method = addUserMethod - } - - var prmInvoke client.InvokePrm - - prmInvoke.SetMethod(method) - prmInvoke.SetArgs(prm.args[:]...) - - err := x.client.Invoke(prmInvoke) - if err != nil { - return nil, err - } - - return new(ManageClientsRes), nil -} diff --git a/pkg/morph/client/subnet/delete.go b/pkg/morph/client/subnet/delete.go deleted file mode 100644 index f7f8bb2c9..000000000 --- a/pkg/morph/client/subnet/delete.go +++ /dev/null @@ -1,40 +0,0 @@ -package morphsubnet - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "github.com/nspcc-dev/neo-go/pkg/util" -) - -// DeletePrm groups parameters of Delete method of Subnet contract. -type DeletePrm struct { - cliPrm client.InvokePrm - - args [1]any -} - -// SetTxHash sets hash of the transaction which spawned the notification. -// Ignore this parameter for new requests. -func (x *DeletePrm) SetTxHash(hash util.Uint256) { - x.cliPrm.SetHash(hash) -} - -// SetID sets identifier of the subnet to be removed in a binary FrostFS API protocol format. -func (x *DeletePrm) SetID(id []byte) { - x.args[0] = id -} - -// DeleteRes groups the resulting values of Delete method of Subnet contract. -type DeleteRes struct{} - -// Delete removes subnet though the call of the corresponding method of the Subnet contract. -func (x Client) Delete(prm DeletePrm) (*DeleteRes, error) { - prm.cliPrm.SetMethod(deleteMethod) - prm.cliPrm.SetArgs(prm.args[:]...) - - err := x.client.Invoke(prm.cliPrm) - if err != nil { - return nil, err - } - - return new(DeleteRes), nil -} diff --git a/pkg/morph/client/subnet/get.go b/pkg/morph/client/subnet/get.go deleted file mode 100644 index 5cd7c39a0..000000000 --- a/pkg/morph/client/subnet/get.go +++ /dev/null @@ -1,55 +0,0 @@ -package morphsubnet - -import ( - "errors" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" -) - -// GetPrm groups parameters of Get method of Subnet contract. -type GetPrm struct { - args [1]any -} - -// SetID sets identifier of the subnet to be read in a binary FrostFS API protocol format. -func (x *GetPrm) SetID(id []byte) { - x.args[0] = id -} - -// GetRes groups the resulting values of Get method of Subnet contract. -type GetRes struct { - info []byte -} - -// Info returns information about the subnet in a binary format of FrostFS API protocol. -func (x GetRes) Info() []byte { - return x.info -} - -var errEmptyResponse = errors.New("empty response") - -// Get reads the subnet through the call of the corresponding method of the Subnet contract. -func (x *Client) Get(prm GetPrm) (*GetRes, error) { - var prmGet client.TestInvokePrm - - prmGet.SetMethod(getMethod) - prmGet.SetArgs(prm.args[:]...) - - res, err := x.client.TestInvoke(prmGet) - if err != nil { - return nil, err - } - - if len(res) == 0 { - return nil, errEmptyResponse - } - - data, err := client.BytesFromStackItem(res[0]) - if err != nil { - return nil, err - } - - return &GetRes{ - info: data, - }, nil -} diff --git a/pkg/morph/client/subnet/node.go b/pkg/morph/client/subnet/node.go deleted file mode 100644 index 134b92943..000000000 --- a/pkg/morph/client/subnet/node.go +++ /dev/null @@ -1,58 +0,0 @@ -package morphsubnet - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" -) - -// NodeAllowedPrm groups parameters of NodeAllowed method of Subnet contract. -type NodeAllowedPrm struct { - cliPrm client.TestInvokePrm - - args [2]any -} - -// SetID sets identifier of the subnet of the node in a binary FrostFS API protocol format. -func (x *NodeAllowedPrm) SetID(id []byte) { - x.args[0] = id -} - -// SetNode sets public key of the node that is being checked. -func (x *NodeAllowedPrm) SetNode(id []byte) { - x.args[1] = id -} - -// NodeAllowedRes groups the resulting values of NodeAllowed method of Subnet contract. -type NodeAllowedRes struct { - result bool -} - -// Allowed returns true iff the node is allowed to enter the subnet. -func (x NodeAllowedRes) Allowed() bool { - return x.result -} - -// NodeAllowed checks if the node is included in the subnetwork. -func (x *Client) NodeAllowed(prm NodeAllowedPrm) (*NodeAllowedRes, error) { - prm.cliPrm.SetMethod(nodeAllowedMethod) - prm.cliPrm.SetArgs(prm.args[:]...) - - res, err := x.client.TestInvoke(prm.cliPrm) - if err != nil { - return nil, fmt.Errorf("could not make test invoke: %w", err) - } - - if len(res) == 0 { - return nil, errEmptyResponse - } - - result, err := client.BoolFromStackItem(res[0]) - if err != nil { - return nil, err - } - - return &NodeAllowedRes{ - result: result, - }, nil -} diff --git a/pkg/morph/client/subnet/nodes.go b/pkg/morph/client/subnet/nodes.go deleted file mode 100644 index 68725a016..000000000 --- a/pkg/morph/client/subnet/nodes.go +++ /dev/null @@ -1,54 +0,0 @@ -package morphsubnet - -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - -// ManageNodesPrm groups parameters of node management in Subnet contract. -// -// Zero value adds node to subnet. Subnet and node IDs must be specified via setters. -type ManageNodesPrm struct { - // remove or add node - rm bool - - args [2]any -} - -// SetRemove marks node to be removed. By default, node is added. -func (x *ManageNodesPrm) SetRemove() { - x.rm = true -} - -// SetSubnet sets identifier of the subnet in a binary NeoFS API protocol format. -func (x *ManageNodesPrm) SetSubnet(id []byte) { - x.args[0] = id -} - -// SetNode sets node's public key in a binary format. -func (x *ManageNodesPrm) SetNode(id []byte) { - x.args[1] = id -} - -// ManageNodesRes groups the resulting values of node management methods of Subnet contract. -type ManageNodesRes struct{} - -// ManageNodes manages node list of the NeoFS subnet through Subnet contract calls. -func (x Client) ManageNodes(prm ManageNodesPrm) (*ManageNodesRes, error) { - var method string - - if prm.rm { - method = removeNodeMethod - } else { - method = addNodeMethod - } - - var prmInvoke client.InvokePrm - - prmInvoke.SetMethod(method) - prmInvoke.SetArgs(prm.args[:]...) - - err := x.client.Invoke(prmInvoke) - if err != nil { - return nil, err - } - - return new(ManageNodesRes), nil -} diff --git a/pkg/morph/client/subnet/put.go b/pkg/morph/client/subnet/put.go deleted file mode 100644 index 2046e79c2..000000000 --- a/pkg/morph/client/subnet/put.go +++ /dev/null @@ -1,50 +0,0 @@ -package morphsubnet - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "github.com/nspcc-dev/neo-go/pkg/util" -) - -// PutPrm groups parameters of Put method of Subnet contract. -type PutPrm struct { - cliPrm client.InvokePrm - - args [3]any -} - -// SetTxHash sets hash of the transaction which spawned the notification. -// Ignore this parameter for new requests. -func (x *PutPrm) SetTxHash(hash util.Uint256) { - x.cliPrm.SetHash(hash) -} - -// SetID sets identifier of the created subnet in a binary FrostFS API protocol format. -func (x *PutPrm) SetID(id []byte) { - x.args[0] = id -} - -// SetOwner sets identifier of the subnet owner in a binary FrostFS API protocol format. -func (x *PutPrm) SetOwner(id []byte) { - x.args[1] = id -} - -// SetInfo sets information about the created subnet in a binary FrostFS API protocol format. -func (x *PutPrm) SetInfo(id []byte) { - x.args[2] = id -} - -// PutRes groups the resulting values of Put method of Subnet contract. -type PutRes struct{} - -// Put creates subnet though the call of the corresponding method of the Subnet contract. -func (x Client) Put(prm PutPrm) (*PutRes, error) { - prm.cliPrm.SetMethod(putMethod) - prm.cliPrm.SetArgs(prm.args[:]...) - - err := x.client.Invoke(prm.cliPrm) - if err != nil { - return nil, err - } - - return new(PutRes), nil -} diff --git a/pkg/morph/event/subnet/delete.go b/pkg/morph/event/subnet/delete.go deleted file mode 100644 index f46658b58..000000000 --- a/pkg/morph/event/subnet/delete.go +++ /dev/null @@ -1,63 +0,0 @@ -package subnetevents - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/util" -) - -// Delete structures information about the notification generated by Delete method of Subnet contract. -type Delete struct { - txHash util.Uint256 - - id []byte -} - -// MorphEvent implements Neo:Morph Event interface. -func (Delete) MorphEvent() {} - -// ID returns identifier of the removed subnet in a binary format of NeoFS API protocol. -func (x Delete) ID() []byte { - return x.id -} - -// TxHash returns hash of the transaction which thrown the notification event. -// Makes sense only in notary environments. -func (x Delete) TxHash() util.Uint256 { - return x.txHash -} - -// ParseDelete parses the notification about the removal of a subnet which has been thrown -// by the appropriate method of the Subnet contract. -// -// Resulting event is of Delete type. -func ParseDelete(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Delete - err error - ) - - items, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("parse stack array: %w", err) - } - - const itemNumDelete = 1 - - if ln := len(items); ln != itemNumDelete { - return nil, event.WrongNumberOfParameters(itemNumDelete, ln) - } - - // parse ID - ev.id, err = client.BytesFromStackItem(items[0]) - if err != nil { - return nil, fmt.Errorf("id item: %w", err) - } - - ev.txHash = e.Container - - return ev, nil -} diff --git a/pkg/morph/event/subnet/delete_test.go b/pkg/morph/event/subnet/delete_test.go deleted file mode 100644 index fc68bb227..000000000 --- a/pkg/morph/event/subnet/delete_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package subnetevents_test - -import ( - "testing" - - subnetevents "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/subnet" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" -) - -func TestParseDelete(t *testing.T) { - id := []byte("id") - - t.Run("wrong number of items", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewByteArray(nil), - stackitem.NewByteArray(nil), - } - - _, err := subnetevents.ParseDelete(createNotifyEventFromItems(prms)) - require.Error(t, err) - }) - - t.Run("wrong id item", func(t *testing.T) { - _, err := subnetevents.ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := subnetevents.ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(id), - })) - require.NoError(t, err) - - v := ev.(subnetevents.Delete) - - require.Equal(t, id, v.ID()) - }) -} diff --git a/pkg/morph/event/subnet/put.go b/pkg/morph/event/subnet/put.go deleted file mode 100644 index 3b1f5297d..000000000 --- a/pkg/morph/event/subnet/put.go +++ /dev/null @@ -1,147 +0,0 @@ -package subnetevents - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/core/transaction" - "github.com/nspcc-dev/neo-go/pkg/network/payload" - "github.com/nspcc-dev/neo-go/pkg/util" -) - -// Put structures information about the notification generated by Put method of Subnet contract. -type Put struct { - notaryRequest *payload.P2PNotaryRequest - - txHash util.Uint256 - - id []byte - - owner []byte - - info []byte -} - -// MorphEvent implements Neo:Morph Event interface. -func (Put) MorphEvent() {} - -// ID returns identifier of the creating subnet in a binary format of FrostFS API protocol. -func (x Put) ID() []byte { - return x.id -} - -// Owner returns subnet owner's public key in a binary format. -func (x Put) Owner() []byte { - return x.owner -} - -// Info returns information about the subnet in a binary format of FrostFS API protocol. -func (x Put) Info() []byte { - return x.info -} - -// TxHash returns hash of the transaction which thrown the notification event. -// Makes sense only in notary environments. -func (x Put) TxHash() util.Uint256 { - return x.txHash -} - -// NotaryMainTx returns main transaction of the request in the Notary service. -// Returns nil in non-notary environments. -func (x Put) NotaryMainTx() *transaction.Transaction { - if x.notaryRequest != nil { - return x.notaryRequest.MainTransaction - } - - return nil -} - -// number of items in notification about subnet creation. -const itemNumPut = 3 - -// ParsePut parses the notification about the creation of a subnet which has been thrown -// by the appropriate method of the subnet contract. -// -// Resulting event is of Put type. -func ParsePut(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - put Put - err error - ) - - items, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("parse stack array: %w", err) - } - - if ln := len(items); ln != itemNumPut { - return nil, event.WrongNumberOfParameters(itemNumPut, ln) - } - - // parse ID - put.id, err = client.BytesFromStackItem(items[0]) - if err != nil { - return nil, fmt.Errorf("id item: %w", err) - } - - // parse owner - put.owner, err = client.BytesFromStackItem(items[1]) - if err != nil { - return nil, fmt.Errorf("owner item: %w", err) - } - - // parse info about subnet - put.info, err = client.BytesFromStackItem(items[2]) - if err != nil { - return nil, fmt.Errorf("info item: %w", err) - } - - put.txHash = e.Container - - return put, nil -} - -// ParseNotaryPut parses the notary notification about the creation of a subnet which has been -// thrown by the appropriate method of the subnet contract. -// -// Resulting event is of Put type. -func ParseNotaryPut(e event.NotaryEvent) (event.Event, error) { - var put Put - - put.notaryRequest = e.Raw() - if put.notaryRequest == nil { - panic(fmt.Sprintf("nil %T in notary environment", put.notaryRequest)) - } - - var ( - err error - - prms = e.Params() - ) - - if ln := len(prms); ln != itemNumPut { - return nil, event.WrongNumberOfParameters(itemNumPut, ln) - } - - // parse info about subnet - put.info, err = event.BytesFromOpcode(prms[0]) - if err != nil { - return nil, fmt.Errorf("info param: %w", err) - } - - // parse owner - put.owner, err = event.BytesFromOpcode(prms[1]) - if err != nil { - return nil, fmt.Errorf("creator param: %w", err) - } - - // parse ID - put.id, err = event.BytesFromOpcode(prms[2]) - if err != nil { - return nil, fmt.Errorf("id param: %w", err) - } - - return put, nil -} diff --git a/pkg/morph/event/subnet/put_test.go b/pkg/morph/event/subnet/put_test.go deleted file mode 100644 index 8a75b62c8..000000000 --- a/pkg/morph/event/subnet/put_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package subnetevents_test - -import ( - "testing" - - subnetevents "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/subnet" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" -) - -func TestParsePut(t *testing.T) { - var ( - id = []byte("id") - owner = []byte("owner") - info = []byte("info") - ) - - t.Run("wrong number of items", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewByteArray(nil), - stackitem.NewByteArray(nil), - } - - _, err := subnetevents.ParsePut(createNotifyEventFromItems(prms)) - require.Error(t, err) - }) - - t.Run("wrong id item", func(t *testing.T) { - _, err := subnetevents.ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong owner item", func(t *testing.T) { - _, err := subnetevents.ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(id), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong info item", func(t *testing.T) { - _, err := subnetevents.ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(id), - stackitem.NewByteArray(owner), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := subnetevents.ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(id), - stackitem.NewByteArray(owner), - stackitem.NewByteArray(info), - })) - require.NoError(t, err) - - v := ev.(subnetevents.Put) - - require.Equal(t, id, v.ID()) - require.Equal(t, owner, v.Owner()) - require.Equal(t, info, v.Info()) - }) -} diff --git a/pkg/morph/event/subnet/remove_node.go b/pkg/morph/event/subnet/remove_node.go deleted file mode 100644 index 67bfb8918..000000000 --- a/pkg/morph/event/subnet/remove_node.go +++ /dev/null @@ -1,69 +0,0 @@ -package subnetevents - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/util" -) - -// RemoveNode structure of subnet.RemoveNode notification from morph chain. -type RemoveNode struct { - subnetID []byte - nodeKey []byte - - // txHash is used in notary environmental - // for calculating unique but same for - // all notification receivers values. - txHash util.Uint256 -} - -// MorphEvent implements Neo:Morph Event interface. -func (RemoveNode) MorphEvent() {} - -// SubnetworkID returns a marshalled subnetID structure, defined in API. -func (rn RemoveNode) SubnetworkID() []byte { return rn.subnetID } - -// Node is public key of the nodeKey that is being deleted. -func (rn RemoveNode) Node() []byte { return rn.nodeKey } - -// TxHash returns hash of the TX with RemoveNode -// notification. -func (rn RemoveNode) TxHash() util.Uint256 { return rn.txHash } - -const expectedItemNumRemoveNode = 2 - -// ParseRemoveNode parses notification into subnet event structure. -// -// Expects 2 stack items. -func ParseRemoveNode(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev RemoveNode - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumRemoveNode { - return nil, event.WrongNumberOfParameters(expectedItemNumRemoveNode, ln) - } - - ev.subnetID, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get raw subnetID: %w", err) - } - - ev.nodeKey, err = client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get raw public key of the node: %w", err) - } - - ev.txHash = e.Container - - return ev, nil -} diff --git a/pkg/morph/event/subnet/remove_node_test.go b/pkg/morph/event/subnet/remove_node_test.go deleted file mode 100644 index 70fff4dc8..000000000 --- a/pkg/morph/event/subnet/remove_node_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package subnetevents_test - -import ( - "testing" - - . "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/subnet" - subnetid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/subnet/id" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" -) - -func TestParseRemoveNode(t *testing.T) { - t.Run("wrong number of arguments", func(t *testing.T) { - _, err := ParseRemoveNode(createNotifyEventFromItems([]stackitem.Item{})) - require.Error(t, err) - }) - - t.Run("invalid item type", func(t *testing.T) { - args := []stackitem.Item{stackitem.NewMap(), stackitem.Make(123)} - _, err := ParseRemoveNode(createNotifyEventFromItems(args)) - require.Error(t, err) - }) - - subnetID := subnetid.ID{} - subnetID.SetNumeric(123) - - rawSubnetID := subnetID.Marshal() - - priv, err := keys.NewPrivateKey() - require.NoError(t, err) - - pub := priv.PublicKey() - - t.Run("good", func(t *testing.T) { - args := []stackitem.Item{stackitem.NewByteArray(rawSubnetID), stackitem.Make(pub.Bytes())} - - e, err := ParseRemoveNode(createNotifyEventFromItems(args)) - require.NoError(t, err) - - gotRaw := e.(RemoveNode).SubnetworkID() - require.NoError(t, err) - - require.Equal(t, rawSubnetID, gotRaw) - require.Equal(t, pub.Bytes(), e.(RemoveNode).Node()) - }) -} - -func createNotifyEventFromItems(items []stackitem.Item) *state.ContainedNotificationEvent { - return &state.ContainedNotificationEvent{ - NotificationEvent: state.NotificationEvent{ - Item: stackitem.NewArray(items), - }, - } -} diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index 9f2834706..44ea2dd66 100644 --- a/pkg/services/control/ir/service.pb.go +++ b/pkg/services/control/ir/service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.21.12 +// protoc v3.12.4 // source: pkg/services/control/ir/service.proto package control diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index b6bc6fdba..bdcac73e5 100644 --- a/pkg/services/control/ir/service_grpc.pb.go +++ b/pkg/services/control/ir/service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.12 +// - protoc-gen-go-grpc v1.3.0 +// - protoc v3.12.4 // source: pkg/services/control/ir/service.proto package control @@ -18,6 +18,10 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + ControlService_HealthCheck_FullMethodName = "/ircontrol.ControlService/HealthCheck" +) + // ControlServiceClient is the client API for ControlService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -36,7 +40,7 @@ func NewControlServiceClient(cc grpc.ClientConnInterface) ControlServiceClient { func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { out := new(HealthCheckResponse) - err := c.cc.Invoke(ctx, "/ircontrol.ControlService/HealthCheck", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_HealthCheck_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -80,7 +84,7 @@ func _ControlService_HealthCheck_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/ircontrol.ControlService/HealthCheck", + FullMethod: ControlService_HealthCheck_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).HealthCheck(ctx, req.(*HealthCheckRequest)) diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go index c89cd5a0d..8107b917e 100644 --- a/pkg/services/control/ir/types.pb.go +++ b/pkg/services/control/ir/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.21.12 +// protoc v3.12.4 // source: pkg/services/control/ir/types.proto package control diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index 735517d3d..9d6864759 100644 --- a/pkg/services/control/types.pb.go +++ b/pkg/services/control/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.21.12 +// protoc v3.12.4 // source: pkg/services/control/types.proto package control @@ -569,9 +569,6 @@ func (x *BlobstorInfo) GetType() string { // attributes it's a string presenting floating point number with comma or // point delimiter for decimal part. In the Network Map it will be saved as // 64-bit unsigned integer representing number of minimal token fractions. -// - Subnet \ -// String ID of Node's storage subnet. There can be only one subnet served -// by the Storage Node. // - Locode \ // Node's geographic location in // [UN/LOCODE](https://www.unece.org/cefact/codesfortrade/codes_index.html) diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto index c85d672c2..5b4844580 100644 --- a/pkg/services/control/types.proto +++ b/pkg/services/control/types.proto @@ -54,9 +54,6 @@ message NodeInfo { // attributes it's a string presenting floating point number with comma or // point delimiter for decimal part. In the Network Map it will be saved as // 64-bit unsigned integer representing number of minimal token fractions. - // * Subnet \ - // String ID of Node's storage subnet. There can be only one subnet served - // by the Storage Node. // * Locode \ // Node's geographic location in // [UN/LOCODE](https://www.unece.org/cefact/codesfortrade/codes_index.html) diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go index 6b5571c3a..08664a6d0 100644 --- a/pkg/services/tree/service.pb.go +++ b/pkg/services/tree/service.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.21.12 +// protoc v3.12.4 // source: pkg/services/tree/service.proto package tree diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index 722fd1705..f981746de 100644 --- a/pkg/services/tree/service_grpc.pb.go +++ b/pkg/services/tree/service_grpc.pb.go @@ -1,7 +1,10 @@ +//* +// Service for working with CRDT tree. + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.12 +// - protoc-gen-go-grpc v1.3.0 +// - protoc v3.12.4 // source: pkg/services/tree/service.proto package tree @@ -18,6 +21,19 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + TreeService_Add_FullMethodName = "/tree.TreeService/Add" + TreeService_AddByPath_FullMethodName = "/tree.TreeService/AddByPath" + TreeService_Remove_FullMethodName = "/tree.TreeService/Remove" + TreeService_Move_FullMethodName = "/tree.TreeService/Move" + TreeService_GetNodeByPath_FullMethodName = "/tree.TreeService/GetNodeByPath" + TreeService_GetSubTree_FullMethodName = "/tree.TreeService/GetSubTree" + TreeService_TreeList_FullMethodName = "/tree.TreeService/TreeList" + TreeService_Apply_FullMethodName = "/tree.TreeService/Apply" + TreeService_GetOpLog_FullMethodName = "/tree.TreeService/GetOpLog" + TreeService_Healthcheck_FullMethodName = "/tree.TreeService/Healthcheck" +) + // TreeServiceClient is the client API for TreeService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -55,7 +71,7 @@ func NewTreeServiceClient(cc grpc.ClientConnInterface) TreeServiceClient { func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*AddResponse, error) { out := new(AddResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Add", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Add_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -64,7 +80,7 @@ func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grp func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, opts ...grpc.CallOption) (*AddByPathResponse, error) { out := new(AddByPathResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/AddByPath", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_AddByPath_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -73,7 +89,7 @@ func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts ...grpc.CallOption) (*RemoveResponse, error) { out := new(RemoveResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Remove", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Remove_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -82,7 +98,7 @@ func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...grpc.CallOption) (*MoveResponse, error) { out := new(MoveResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Move", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Move_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -91,7 +107,7 @@ func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...g func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPathRequest, opts ...grpc.CallOption) (*GetNodeByPathResponse, error) { out := new(GetNodeByPathResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/GetNodeByPath", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_GetNodeByPath_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -99,7 +115,7 @@ func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPath } func (c *treeServiceClient) GetSubTree(ctx context.Context, in *GetSubTreeRequest, opts ...grpc.CallOption) (TreeService_GetSubTreeClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], "/tree.TreeService/GetSubTree", opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], TreeService_GetSubTree_FullMethodName, opts...) if err != nil { return nil, err } @@ -132,7 +148,7 @@ func (x *treeServiceGetSubTreeClient) Recv() (*GetSubTreeResponse, error) { func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, opts ...grpc.CallOption) (*TreeListResponse, error) { out := new(TreeListResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/TreeList", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_TreeList_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -141,7 +157,7 @@ func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, o func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts ...grpc.CallOption) (*ApplyResponse, error) { out := new(ApplyResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Apply", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Apply_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -149,7 +165,7 @@ func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts .. } func (c *treeServiceClient) GetOpLog(ctx context.Context, in *GetOpLogRequest, opts ...grpc.CallOption) (TreeService_GetOpLogClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], "/tree.TreeService/GetOpLog", opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], TreeService_GetOpLog_FullMethodName, opts...) if err != nil { return nil, err } @@ -182,7 +198,7 @@ func (x *treeServiceGetOpLogClient) Recv() (*GetOpLogResponse, error) { func (c *treeServiceClient) Healthcheck(ctx context.Context, in *HealthcheckRequest, opts ...grpc.CallOption) (*HealthcheckResponse, error) { out := new(HealthcheckResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Healthcheck", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Healthcheck_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -272,7 +288,7 @@ func _TreeService_Add_Handler(srv interface{}, ctx context.Context, dec func(int } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Add", + FullMethod: TreeService_Add_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Add(ctx, req.(*AddRequest)) @@ -290,7 +306,7 @@ func _TreeService_AddByPath_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/AddByPath", + FullMethod: TreeService_AddByPath_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).AddByPath(ctx, req.(*AddByPathRequest)) @@ -308,7 +324,7 @@ func _TreeService_Remove_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Remove", + FullMethod: TreeService_Remove_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Remove(ctx, req.(*RemoveRequest)) @@ -326,7 +342,7 @@ func _TreeService_Move_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Move", + FullMethod: TreeService_Move_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Move(ctx, req.(*MoveRequest)) @@ -344,7 +360,7 @@ func _TreeService_GetNodeByPath_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/GetNodeByPath", + FullMethod: TreeService_GetNodeByPath_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).GetNodeByPath(ctx, req.(*GetNodeByPathRequest)) @@ -383,7 +399,7 @@ func _TreeService_TreeList_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/TreeList", + FullMethod: TreeService_TreeList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).TreeList(ctx, req.(*TreeListRequest)) @@ -401,7 +417,7 @@ func _TreeService_Apply_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Apply", + FullMethod: TreeService_Apply_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Apply(ctx, req.(*ApplyRequest)) @@ -440,7 +456,7 @@ func _TreeService_Healthcheck_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Healthcheck", + FullMethod: TreeService_Healthcheck_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Healthcheck(ctx, req.(*HealthcheckRequest)) diff --git a/pkg/services/tree/types.pb.go b/pkg/services/tree/types.pb.go index f3f64180a..45d889177 100644 --- a/pkg/services/tree/types.pb.go +++ b/pkg/services/tree/types.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.21.12 +// protoc v3.12.4 // source: pkg/services/tree/types.proto package tree From 7b981bfe97144eba4d01f123696f53d13d8b15b9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Apr 2023 14:28:24 +0300 Subject: [PATCH 0126/1943] [#249] logs: Drop unused consts Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index a6488dcc9..717e3529a 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -19,17 +19,6 @@ const ( const ( InnerringAmountCanNotBeRepresentedAsAnInt64 = "amount can not be represented as an int64" // Error in ../node/pkg/innerring/settlement.go InnerringCantGetUsedSpaceEstimation = "can't get used space estimation" // Warn in ../node/pkg/innerring/settlement.go - InnerringSubnetCreationQueueFailure = "subnet creation queue failure" // Error in ../node/pkg/innerring/subnet.go - InnerringDiscardSubnetCreation = "discard subnet creation" // Info in ../node/pkg/innerring/subnet.go - InnerringApproveSubnetCreation = "approve subnet creation" // Error in ../node/pkg/innerring/subnet.go - InnerringSubnetRemovalHandlingFailure = "subnet removal handling failure" // Error in ../node/pkg/innerring/subnet.go - InnerringGettingNetmapCandidates = "getting netmap candidates" // Error in ../node/pkg/innerring/subnet.go - InnerringUnmarshallingRemovedSubnetID = "unmarshalling removed subnet ID" // Error in ../node/pkg/innerring/subnet.go - InnerringIteratingNodesSubnets = "iterating node's subnets" // Error in ../node/pkg/innerring/subnet.go - InnerringRemovingNodeFromNetmapCandidates = "removing node from netmap candidates" // Debug in ../node/pkg/innerring/subnet.go - InnerringRemovingNodeFromCandidates = "removing node from candidates" // Error in ../node/pkg/innerring/subnet.go - InnerringRemovingSubnetFromTheNode = "removing subnet from the node" // Debug in ../node/pkg/innerring/subnet.go - InnerringUpdatingSubnetInfo = "updating subnet info" // Error in ../node/pkg/innerring/subnet.go InnerringNonalphabetModeDoNotStopContainerEstimations = "non-alphabet mode, do not stop container estimations" // Debug in ../node/pkg/innerring/blocktimer.go InnerringCantStopEpochEstimation = "can't stop epoch estimation" // Warn in ../node/pkg/innerring/blocktimer.go InnerringCantMakeNotaryDepositInMainChain = "can't make notary deposit in main chain" // Error in ../node/pkg/innerring/notary.go @@ -97,7 +86,6 @@ const ( PersistentCouldNotCleanUpExpiredTokens = "could not clean up expired tokens" // Error in ../node/pkg/services/session/storage/persistent/storage.go ControllerReportIsAlreadyStarted = "report is already started" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go TombstoneCouldNotGetTheTombstoneTheSource = "tombstone getter: could not get the tombstone the source" // Warn in ../node/pkg/services/object_manager/tombstone/checker.go - TombstoneCouldNotParseTombstoneExpirationEpoch = "tombstone getter: could not parse tombstone expiration epoch" // Warn in ../node/pkg/services/object_manager/tombstone/checker.go DeleteRequestIsNotRolledOverToTheContainer = "request is not rolled over to the container" // Debug in ../node/pkg/services/object/delete/container.go DeleteCouldNotComposeSplitInfo = "could not compose split info" // Debug in ../node/pkg/services/object/delete/exec.go DeleteNoSplitInfoObjectIsPHY = "no split info, object is PHY" // Debug in ../node/pkg/services/object/delete/exec.go @@ -381,7 +369,6 @@ const ( AuditFilterExpiredStorageGroupsForAudit = "filter expired storage groups for audit" // Info in ../node/pkg/innerring/processors/audit/process.go AuditParseClientNodeInfo = "parse client node info" // Warn in ../node/pkg/innerring/processors/audit/process.go AuditErrorInStorageGroupSearch = "error in storage group search" // Warn in ../node/pkg/innerring/processors/audit/process.go - AuditCouldNotGetStorageGroupObjectForAuditSkipping = "could not get storage group object for audit, skipping" // Error in ../node/pkg/innerring/processors/audit/process.go BalanceBalanceWorkerPoolDrained = "balance worker pool drained" // Warn in ../node/pkg/innerring/processors/balance/handlers.go BalanceNonAlphabetModeIgnoreBalanceLock = "non alphabet mode, ignore balance lock" // Info in ../node/pkg/innerring/processors/balance/process_assets.go BalanceCantSendLockAssetTx = "can't send lock asset tx" // Error in ../node/pkg/innerring/processors/balance/process_assets.go @@ -456,14 +443,7 @@ const ( NetmapCantInvokeNetmapAddPeer = "can't invoke netmap.AddPeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapNonAlphabetModeIgnoreUpdatePeerNotification = "non alphabet mode, ignore update peer notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapPreventSwitchingNodeToMaintenanceState = "prevent switching node to maintenance state" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapNonAlphabetModeIgnoreRemoveNodeFromSubnetNotification = "non alphabet mode, ignore remove node from subnet notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCouldNotGetNetworkMapCandidates = "could not get network map candidates" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCouldNotUnmarshalSubnetID = "could not unmarshal subnet id" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapGotZeroSubnetInRemoveNodeNotification = "got zero subnet in remove node notification" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCouldNotIterateOverSubnetworksOfTheNode = "could not iterate over subnetworks of the node" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCouldNotInvokeNetmapUpdateState = "could not invoke netmap.UpdateState" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCouldNotInvokeNetmapAddPeer = "could not invoke netmap.AddPeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go // Debug in ../node/pkg/innerring/processors/reputation/processor.go SettlementNonAlphabetModeIgnoreAuditPayments = "non alphabet mode, ignore audit payments" // Info in ../node/pkg/innerring/processors/settlement/calls.go SettlementNewAuditSettlementEvent = "new audit settlement event" // Info in ../node/pkg/innerring/processors/settlement/calls.go SettlementIgnoreGenesisEpoch = "ignore genesis epoch" // Debug in ../node/pkg/innerring/processors/settlement/calls.go From b447ff99aa9c964e6bdee739c44e6503ab1ddaa8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 18 Apr 2023 12:01:35 +0300 Subject: [PATCH 0127/1943] [#265] node: Up SDK and API-Go versions Fix tracing panic. Signed-off-by: Dmitrii Stepanov --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 301be6934..6bc9a0e47 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.18 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.0 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418075216-d0c5d837d204 git.frostfs.info/TrueCloudLab/hrw v1.2.0 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index f65a7aeab..36546b3dd 100644 --- a/go.sum +++ b/go.sum @@ -36,14 +36,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.0 h1:oZ0/KiaFeveXRLi5VVEpuLSHczeFyWx4HDl9wTJUtsE= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.0/go.mod h1:sPyITTmQT662ZI38ud2aoE1SUCAr1mO5xV8P4nzLkKI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 h1:77lvdk0kMhnUgtnmqEcAPXPQaGlt24goMPu2+E5WRTk= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85/go.mod h1:sPyITTmQT662ZI38ud2aoE1SUCAr1mO5xV8P4nzLkKI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85 h1:TUcJ5A0C1gWi3bAhw4b+V+iVM3E9mbBOdJIWWkAPNxo= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230316081442-bec77f280a85/go.mod h1:23fUGlEv/ImaOi3vck6vZj0v0b4hteOhLLPnVWHSQeA= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418075216-d0c5d837d204 h1:oQk6Fns+51JPtawUR5cJyYPQ35yC8Gi6e6P/PKkbvIc= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418075216-d0c5d837d204/go.mod h1:qbeYz8Z/3fZ0M0jiJY/zycuXB3DQ/8xQL5xU2G78akQ= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From a358255c1b811172aff963144317314988d20e8f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 18 Apr 2023 12:04:31 +0300 Subject: [PATCH 0128/1943] [#265] node: Drop unused Resolve unused linter. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/shards_set_mode.go | 7 ------- pkg/morph/client/netmap/config.go | 10 ---------- 2 files changed, 17 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/shards_set_mode.go b/cmd/frostfs-cli/modules/control/shards_set_mode.go index 4f6e24082..135c0efa6 100644 --- a/cmd/frostfs-cli/modules/control/shards_set_mode.go +++ b/cmd/frostfs-cli/modules/control/shards_set_mode.go @@ -137,13 +137,6 @@ func setShardMode(cmd *cobra.Command, _ []string) { cmd.Println("Shard mode update request successfully sent.") } -func getShardID(cmd *cobra.Command) []byte { - sid, _ := cmd.Flags().GetString(shardIDFlag) - raw, err := base58.Decode(sid) - commonCmd.ExitOnErr(cmd, "incorrect shard ID encoding: %w", err) - return raw -} - func getShardIDList(cmd *cobra.Command) [][]byte { all, _ := cmd.Flags().GetBool(shardAllFlag) if all { diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 71ffa2a7a..c837cbf0c 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -136,16 +136,6 @@ func (c *Client) readUInt64Config(key string) (uint64, error) { return uint64(v.(int64)), nil } -func (c *Client) readStringConfig(key string) (string, error) { - v, err := c.config([]byte(key), StringAssert) - if err != nil { - return "", err - } - - // StringAssert is guaranteed to return string if the error is nil. - return v.(string), nil -} - // reads boolean value by the given key from the FrostFS network configuration // stored in the Sidechain. Returns false if key is not presented. func (c *Client) readBoolConfig(key string) (bool, error) { From 3d43b0f7f93d88aa5e985385a684019cb69f30c5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 18 Apr 2023 12:04:59 +0300 Subject: [PATCH 0129/1943] [#265] node: Fix after SDK & API-Go version up Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/internal/client/sdk.go | 8 ++++---- cmd/frostfs-cli/modules/session/create.go | 2 +- pkg/core/client/client.go | 2 +- pkg/network/cache/multi.go | 14 +++++++------- pkg/services/object/get/v2/get_forwarder.go | 2 +- pkg/services/object/get/v2/get_range_forwarder.go | 2 +- pkg/services/object/get/v2/head_forwarder.go | 2 +- pkg/services/object/put/v2/streamer.go | 2 +- pkg/services/object/search/v2/request_forwarder.go | 2 +- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/sdk.go b/cmd/frostfs-cli/internal/client/sdk.go index 4cf0b0a34..13dacc04c 100644 --- a/cmd/frostfs-cli/internal/client/sdk.go +++ b/cmd/frostfs-cli/internal/client/sdk.go @@ -36,11 +36,11 @@ func getSDKClientByFlag(cmd *cobra.Command, key *ecdsa.PrivateKey, endpointFlag if err != nil { return nil, fmt.Errorf("%v: %w", errInvalidEndpoint, err) } - return GetSDKClient(cmd, key, addr) + return GetSDKClient(cmd.Context(), cmd, key, addr) } // GetSDKClient returns default frostfs-sdk-go client. -func GetSDKClient(cmd *cobra.Command, key *ecdsa.PrivateKey, addr network.Address) (*client.Client, error) { +func GetSDKClient(ctx context.Context, cmd *cobra.Command, key *ecdsa.PrivateKey, addr network.Address) (*client.Client, error) { var ( c client.Client prmInit client.PrmInit @@ -62,7 +62,7 @@ func GetSDKClient(cmd *cobra.Command, key *ecdsa.PrivateKey, addr network.Addres c.Init(prmInit) - if err := c.Dial(prmDial); err != nil { + if err := c.Dial(ctx, prmDial); err != nil { return nil, fmt.Errorf("can't init SDK client: %w", err) } @@ -82,7 +82,7 @@ func GetCurrentEpoch(ctx context.Context, cmd *cobra.Command, endpoint string) ( return 0, fmt.Errorf("can't generate key to sign query: %w", err) } - c, err := GetSDKClient(cmd, key, addr) + c, err := GetSDKClient(ctx, cmd, key, addr) if err != nil { return 0, err } diff --git a/cmd/frostfs-cli/modules/session/create.go b/cmd/frostfs-cli/modules/session/create.go index e1c951b93..341681f5b 100644 --- a/cmd/frostfs-cli/modules/session/create.go +++ b/cmd/frostfs-cli/modules/session/create.go @@ -54,7 +54,7 @@ func createSession(cmd *cobra.Command, _ []string) { addrStr, _ := cmd.Flags().GetString(commonflags.RPC) commonCmd.ExitOnErr(cmd, "can't parse endpoint: %w", netAddr.FromString(addrStr)) - c, err := internalclient.GetSDKClient(cmd, privKey, netAddr) + c, err := internalclient.GetSDKClient(cmd.Context(), cmd, privKey, netAddr) commonCmd.ExitOnErr(cmd, "can't create client: %w", err) lifetime := uint64(defaultLifetime) diff --git a/pkg/core/client/client.go b/pkg/core/client/client.go index 98e9d6367..bd186006e 100644 --- a/pkg/core/client/client.go +++ b/pkg/core/client/client.go @@ -30,7 +30,7 @@ type MultiAddressClient interface { // RawForAddress must return rawclient.Client // for the passed network.Address. - RawForAddress(network.Address, func(cli *rawclient.Client) error) error + RawForAddress(context.Context, network.Address, func(cli *rawclient.Client) error) error ReportError(error) } diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 2f698912c..b1afaa90d 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -47,7 +47,7 @@ func newMultiClient(addr network.AddressGroup, opts ClientCacheOpts) *multiClien } } -func (x *multiClient) createForAddress(addr network.Address) (clientcore.Client, error) { +func (x *multiClient) createForAddress(ctx context.Context, addr network.Address) (clientcore.Client, error) { var ( c client.Client prmInit client.PrmInit @@ -73,7 +73,7 @@ func (x *multiClient) createForAddress(addr network.Address) (clientcore.Client, } c.Init(prmInit) - err := c.Dial(prmDial) + err := c.Dial(ctx, prmDial) if err != nil { return nil, fmt.Errorf("can't init SDK client: %w", err) } @@ -144,7 +144,7 @@ func (x *multiClient) iterateClients(ctx context.Context, f func(clientcore.Clie var err error - c, err := x.client(addr) + c, err := x.client(ctx, addr) if err == nil { err = f(c) } @@ -297,8 +297,8 @@ func (x *multiClient) Close() error { return nil } -func (x *multiClient) RawForAddress(addr network.Address, f func(client *rawclient.Client) error) error { - c, err := x.client(addr) +func (x *multiClient) RawForAddress(ctx context.Context, addr network.Address, f func(client *rawclient.Client) error) error { + c, err := x.client(ctx, addr) if err != nil { return err } @@ -310,7 +310,7 @@ func (x *multiClient) RawForAddress(addr network.Address, f func(client *rawclie return err } -func (x *multiClient) client(addr network.Address) (clientcore.Client, error) { +func (x *multiClient) client(ctx context.Context, addr network.Address) (clientcore.Client, error) { strAddr := addr.String() x.mtx.RLock() @@ -351,7 +351,7 @@ func (x *multiClient) client(addr network.Address) (clientcore.Client, error) { return nil, errRecentlyFailed } - cl, err := x.createForAddress(addr) + cl, err := x.createForAddress(ctx, addr) if err != nil { c.lastAttempt = time.Now() return nil, err diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index 8163ae928..3677327c0 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -99,7 +99,7 @@ func (f *getRequestForwarder) writeHeader(ctx context.Context, v *objectV2.GetOb func (f *getRequestForwarder) openStream(ctx context.Context, addr network.Address, c client.MultiAddressClient) (*rpc.GetResponseReader, error) { var getStream *rpc.GetResponseReader - err := c.RawForAddress(addr, func(cli *rpcclient.Client) error { + err := c.RawForAddress(ctx, addr, func(cli *rpcclient.Client) error { var e error getStream, e = rpc.GetObject(cli, f.Request, rpcclient.WithContext(ctx)) return e diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 9cf6384ed..06b65fb50 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -85,7 +85,7 @@ func (f *getRangeRequestForwarder) verifyResponse(resp *objectV2.GetRangeRespons func (f *getRangeRequestForwarder) openStream(ctx context.Context, addr network.Address, c client.MultiAddressClient) (*rpc.ObjectRangeResponseReader, error) { // open stream var rangeStream *rpc.ObjectRangeResponseReader - err := c.RawForAddress(addr, func(cli *rpcclient.Client) error { + err := c.RawForAddress(ctx, addr, func(cli *rpcclient.Client) error { var e error rangeStream, e = rpc.GetObjectRange(cli, f.Request, rpcclient.WithContext(ctx)) return e diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index e1d4c02db..f02ef81bb 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -154,7 +154,7 @@ func (f *headRequestForwarder) getHeaderAndSignature(hdrWithSig *objectV2.Header func (f *headRequestForwarder) sendHeadRequest(ctx context.Context, addr network.Address, c client.MultiAddressClient) (*objectV2.HeadResponse, error) { var headResp *objectV2.HeadResponse - err := c.RawForAddress(addr, func(cli *rpcclient.Client) error { + err := c.RawForAddress(ctx, addr, func(cli *rpcclient.Client) error { var e error headResp, e = rpc.HeadObject(cli, f.Request, rpcclient.WithContext(ctx)) return e diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index f7a97a955..65531dc66 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -161,7 +161,7 @@ func (s *streamer) relayRequest(ctx context.Context, info client.NodeInfo, c cli var stream *rpc.PutRequestWriter - err = c.RawForAddress(addr, func(cli *rawclient.Client) error { + err = c.RawForAddress(ctx, addr, func(cli *rawclient.Client) error { stream, err = rpc.PutObject(cli, resp, rawclient.WithContext(ctx)) return err }) diff --git a/pkg/services/object/search/v2/request_forwarder.go b/pkg/services/object/search/v2/request_forwarder.go index 8023f2f0f..d8719986f 100644 --- a/pkg/services/object/search/v2/request_forwarder.go +++ b/pkg/services/object/search/v2/request_forwarder.go @@ -46,7 +46,7 @@ func (f *requestForwarder) forwardRequest(ctx context.Context, addr network.Addr } var searchStream *rpc.SearchResponseReader - err = c.RawForAddress(addr, func(cli *rpcclient.Client) error { + err = c.RawForAddress(ctx, addr, func(cli *rpcclient.Client) error { searchStream, err = rpc.SearchObjects(cli, f.Request, rpcclient.WithContext(ctx)) return err }) From 20cd080323ef632ca09d34a7c5a2b3d4be5043df Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Mon, 17 Apr 2023 18:40:23 +0300 Subject: [PATCH 0130/1943] [#255] write-cache: Fix init race condition Do not use WC's internals in the initialization routines without mode protection. WC should be able to change its mode even if the initialization is not finished yet. Signed-off-by: Pavel Karpy --- pkg/local_object_storage/writecache/init.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/local_object_storage/writecache/init.go b/pkg/local_object_storage/writecache/init.go index d92e9a2d9..2ca8cceef 100644 --- a/pkg/local_object_storage/writecache/init.go +++ b/pkg/local_object_storage/writecache/init.go @@ -86,13 +86,21 @@ func (c *cache) fsTreeFlushMarkUpdate(ctx context.Context) { } return nil } + + c.modeMtx.RLock() + defer c.modeMtx.RUnlock() + _, _ = c.fsTree.Iterate(prm) + c.log.Info(logs.WritecacheFinishedUpdatingFSTreeFlushMarks) } func (c *cache) dbFlushMarkUpdate(ctx context.Context) { c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInDatabase) + c.modeMtx.RLock() + defer c.modeMtx.RUnlock() + var m []string var indices []int var lastKey []byte From 13c8afcb024910ee42ba1121cc753dd347c7c4ac Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 10 Apr 2023 21:46:03 +0300 Subject: [PATCH 0131/1943] [#118] node: add unit concurrent tests for blobstor Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- .../blobstor/blobstor_test.go | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index 8c6766dc1..e1dae41fe 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -3,6 +3,7 @@ package blobstor import ( "context" "path/filepath" + "sync" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -174,3 +175,156 @@ func TestBlobstor_needsCompression(t *testing.T) { require.False(t, b.NeedsCompression(obj)) }) } + +func TestConcurrentPut(t *testing.T) { + dir := t.TempDir() + + const ( + smallSizeLimit = 512 + + // concurrentPutCount is fstree implementation specific + concurrentPutCount = 5 + ) + + blobStor := New( + WithStorages(defaultStorages(dir, smallSizeLimit))) + require.NoError(t, blobStor.Open(false)) + require.NoError(t, blobStor.Init()) + + testGet := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { + res, err := b.Get(common.GetPrm{Address: object.AddressOf(obj)}) + require.NoError(t, err) + require.Equal(t, obj, res.Object) + } + + testPut := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { + var prm common.PutPrm + prm.Object = obj + _, err := b.Put(prm) + require.NoError(t, err) + } + + testPutFileExistsError := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { + var prm common.PutPrm + prm.Object = obj + if _, err := b.Put(prm); err != nil { + require.ErrorContains(t, err, "file exists") + } + } + + t.Run("put the same big object", func(t *testing.T) { + bigObj := testObject(smallSizeLimit * 2) + + var wg sync.WaitGroup + for i := 0; i < concurrentPutCount; i++ { + wg.Add(1) + go func() { + testPut(t, blobStor, bigObj) + wg.Done() + }() + } + wg.Wait() + + testGet(t, blobStor, bigObj) + }) + + t.Run("put the same big object with error", func(t *testing.T) { + bigObj := testObject(smallSizeLimit * 2) + + var wg sync.WaitGroup + for i := 0; i < concurrentPutCount+1; i++ { + wg.Add(1) + go func() { + testPutFileExistsError(t, blobStor, bigObj) + wg.Done() + }() + } + wg.Wait() + + testGet(t, blobStor, bigObj) + }) + + t.Run("put the same small object", func(t *testing.T) { + smallObj := testObject(smallSizeLimit / 2) + + var wg sync.WaitGroup + for i := 0; i < concurrentPutCount; i++ { + wg.Add(1) + go func() { + testPut(t, blobStor, smallObj) + wg.Done() + }() + } + wg.Wait() + + testGet(t, blobStor, smallObj) + }) +} + +func TestConcurrentDelete(t *testing.T) { + dir := t.TempDir() + + const smallSizeLimit = 512 + + blobStor := New( + WithStorages(defaultStorages(dir, smallSizeLimit))) + require.NoError(t, blobStor.Open(false)) + require.NoError(t, blobStor.Init()) + + testPut := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { + var prm common.PutPrm + prm.Object = obj + _, err := b.Put(prm) + require.NoError(t, err) + } + + testDelete := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { + var prm common.DeletePrm + prm.Address = object.AddressOf(obj) + if _, err := b.Delete(prm); err != nil { + require.ErrorContains(t, err, "object not found") + } + } + + testDeletedExists := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { + var prm common.ExistsPrm + prm.Address = object.AddressOf(obj) + res, err := b.Exists(prm) + require.NoError(t, err) + require.False(t, res.Exists) + } + + t.Run("delete the same big object", func(t *testing.T) { + bigObj := testObject(smallSizeLimit * 2) + testPut(t, blobStor, bigObj) + + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + testDelete(t, blobStor, bigObj) + wg.Done() + }() + } + wg.Wait() + + testDeletedExists(t, blobStor, bigObj) + }) + + t.Run("delete the same small object", func(t *testing.T) { + smallObj := testObject(smallSizeLimit / 2) + testPut(t, blobStor, smallObj) + + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + testDelete(t, blobStor, smallObj) + wg.Done() + }() + } + wg.Wait() + + testDeletedExists(t, blobStor, smallObj) + }) +} From 3d23b08773cdb440d9082a035934299af66ce293 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Mon, 17 Apr 2023 20:24:12 +0300 Subject: [PATCH 0132/1943] [#262] meta: Do not return old expired objects Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 + pkg/local_object_storage/metabase/db_test.go | 7 ++++++ pkg/local_object_storage/metabase/get.go | 13 +++++++++- pkg/local_object_storage/metabase/util.go | 25 ++++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94018e357..738fb38e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Changelog for FrostFS Node ### Fixed - Take network settings into account during netmap contract update (#100) - Read config files from dir even if config file not provided via `--config` for node (#238) +- Expired by more than 1 epoch objects could be returned (#262) ### Removed ### Updated diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index 9ef7bf8bc..a3b26da79 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -73,6 +73,13 @@ func checkExpiredObjects(t *testing.T, db *meta.DB, f func(exp, nonExp *objectSD require.NoError(t, metaPut(db, nonExpObj, nil)) f(expObj, nonExpObj) + + oldExpObj := testutil.GenerateObject() + setExpiration(oldExpObj, 1) + + require.NoError(t, metaPut(db, oldExpObj, nil)) + + f(oldExpObj, nonExpObj) } func setExpiration(o *objectSDK.Object, epoch uint64) { diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index fff32d6ad..c3a5880bf 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -66,13 +66,24 @@ func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { } currEpoch := db.epochState.CurrentEpoch() + var obj *objectSDK.Object err = db.boltDB.View(func(tx *bbolt.Tx) error { key := make([]byte, addressKeySize) - res.hdr, err = db.get(tx, prm.addr, key, true, prm.raw, currEpoch) + obj, err = db.get(tx, prm.addr, key, true, prm.raw, currEpoch) return err }) + if err != nil { + return + } + + err = validate(obj, currEpoch) + if err != nil { + return + } + + res.hdr = obj return } diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index b60c97fd7..d4c93c750 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -4,7 +4,9 @@ import ( "bytes" "crypto/sha256" "fmt" + "strconv" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -259,3 +261,26 @@ func isLockObject(tx *bbolt.Tx, idCnr cid.ID, obj oid.ID) bool { bucketNameLockers(idCnr, make([]byte, bucketKeySize)), objectKey(obj, make([]byte, objectKeySize))) } + +// performs object validity checks. +func validate(obj *object.Object, currEpoch uint64) error { + for _, a := range obj.Attributes() { + if key := a.Key(); key != objectV2.SysAttributeExpEpoch && key != objectV2.SysAttributeExpEpochNeoFS { + continue + } + + expEpoch, err := strconv.ParseUint(a.Value(), 10, 64) + if err != nil { + // unexpected for already stored and valudated objects + return fmt.Errorf("expiration epoch parsing: %w", err) + } + + if expEpoch < currEpoch { + return ErrObjectIsExpired + } + + break + } + + return nil +} From 6c90bb87f160931a2e82bf3d7c7c2c0b0d64d3e0 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 19 Apr 2023 18:09:33 +0300 Subject: [PATCH 0133/1943] [#118] node: add ctx for unit tests for blobstor Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/local_object_storage/blobstor/blobstor_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index e1dae41fe..ef85f0f1d 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -192,7 +192,7 @@ func TestConcurrentPut(t *testing.T) { require.NoError(t, blobStor.Init()) testGet := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { - res, err := b.Get(common.GetPrm{Address: object.AddressOf(obj)}) + res, err := b.Get(context.Background(), common.GetPrm{Address: object.AddressOf(obj)}) require.NoError(t, err) require.Equal(t, obj, res.Object) } @@ -200,14 +200,14 @@ func TestConcurrentPut(t *testing.T) { testPut := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { var prm common.PutPrm prm.Object = obj - _, err := b.Put(prm) + _, err := b.Put(context.Background(), prm) require.NoError(t, err) } testPutFileExistsError := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { var prm common.PutPrm prm.Object = obj - if _, err := b.Put(prm); err != nil { + if _, err := b.Put(context.Background(), prm); err != nil { require.ErrorContains(t, err, "file exists") } } @@ -274,14 +274,14 @@ func TestConcurrentDelete(t *testing.T) { testPut := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { var prm common.PutPrm prm.Object = obj - _, err := b.Put(prm) + _, err := b.Put(context.Background(), prm) require.NoError(t, err) } testDelete := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { var prm common.DeletePrm prm.Address = object.AddressOf(obj) - if _, err := b.Delete(prm); err != nil { + if _, err := b.Delete(context.Background(), prm); err != nil { require.ErrorContains(t, err, "object not found") } } @@ -289,7 +289,7 @@ func TestConcurrentDelete(t *testing.T) { testDeletedExists := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { var prm common.ExistsPrm prm.Address = object.AddressOf(obj) - res, err := b.Exists(prm) + res, err := b.Exists(context.Background(), prm) require.NoError(t, err) require.False(t, res.Exists) } From 4f5f832137d544f628337588573059ba72ccdcce Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 19 Apr 2023 17:11:32 +0300 Subject: [PATCH 0134/1943] [#268] notary_preparator: Actualize notary requests parsing After https://github.com/nspcc-dev/neo-go/commit/75d7891ca183f0e24b8fb13d21b7076a95ddebbd `neo-go` does claim that an empty invocation script is the only way to fill missing signature for unsigned notary requests. The new notary actor does it that way and, therefore, breaks notary request parsing by the Alphabet because of skipping any request that is not filled with a dummy (64 zeros) invocation script. Support both way. The "Dummy" approach will be dropped later. Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 + pkg/morph/event/notary_preparator.go | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 738fb38e1..954bad844 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Changelog for FrostFS Node - Take network settings into account during netmap contract update (#100) - Read config files from dir even if config file not provided via `--config` for node (#238) - Expired by more than 1 epoch objects could be returned (#262) +- Notary requests parsing according to `neo-go`'s updates (#268) ### Removed ### Updated diff --git a/pkg/morph/event/notary_preparator.go b/pkg/morph/event/notary_preparator.go index 3d499fec5..f7b10d906 100644 --- a/pkg/morph/event/notary_preparator.go +++ b/pkg/morph/event/notary_preparator.go @@ -185,13 +185,15 @@ func (p Preparator) validateNotaryRequest(nr *payload.P2PNotaryRequest) error { } invokerWitness := ln == 4 + multiInvScript := nr.MainTransaction.Scripts[1].InvocationScript + // alphabet node should handle only notary requests - // that have been sent unsigned(by storage nodes) => - // such main TXs should have dummy scripts as an - // invocation script + // that have been sent unsigned (by storage nodes) => + // such main TXs should have either a dummy or an + // empty script as an invocation script // // this check prevents notary flow recursion - if !bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript) { + if len(multiInvScript) > 0 && !bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript) { return ErrTXAlreadyHandled } From 6b6f33ed7182fa75716e90b35b709fe25f5841e0 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 20 Apr 2023 17:59:44 +0300 Subject: [PATCH 0135/1943] [#274] wc: Make wait groups work more explicit Do not run routine that calls `wg.Done()` inside, it is hard to read. Signed-off-by: Pavel Karpy --- pkg/local_object_storage/writecache/flush.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 9d0ffc98c..04fcccede 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -41,7 +41,10 @@ func (c *cache) runFlushLoop() { } c.wg.Add(1) - go c.flushBigObjects(context.TODO()) + go func() { + c.flushBigObjects(context.TODO()) + c.wg.Done() + }() c.wg.Add(1) go func() { @@ -146,8 +149,6 @@ func (c *cache) flushDB() { } func (c *cache) flushBigObjects(ctx context.Context) { - defer c.wg.Done() - tick := time.NewTicker(defaultFlushInterval * 10) for { select { From e9461686b87496c5d60e7ce8899865f267c3abc1 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 20 Apr 2023 18:51:16 +0300 Subject: [PATCH 0136/1943] [#274] wc: Resolve possible deadlock If operation with WC are _fast enough_ (e.g. `Init` failed and `Close` is called immediately) there is a race and a deadlock that do not allow finish (and start, in fact) an initialization routine because of taken `modeMtx` and also do not allow finish `Close` call because of awaiting initialization finish. So do stop initialization _before_ any mutex is taken. Signed-off-by: Pavel Karpy --- pkg/local_object_storage/writecache/mode.go | 20 +++++++++---------- .../writecache/writecache.go | 3 --- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 20b0cce29..14f8af49e 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -29,9 +29,6 @@ func (c *cache) SetMode(m mode.Mode) error { )) defer span.End() - c.modeMtx.Lock() - defer c.modeMtx.Unlock() - return c.setMode(ctx, m) } @@ -40,13 +37,6 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode) error { var err error turnOffMeta := m.NoMetabase() - if turnOffMeta && !c.mode.NoMetabase() { - err = c.flush(ctx, true) - if err != nil { - return err - } - } - if !c.initialized.Load() { close(c.stopInitCh) @@ -60,6 +50,16 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode) error { }() } + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + + if turnOffMeta && !c.mode.NoMetabase() { + err = c.flush(ctx, true) + if err != nil { + return err + } + } + if c.db != nil { if err = c.db.Close(); err != nil { return fmt.Errorf("can't close write-cache database: %w", err) diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 0fc2e601a..bdcc9bbf6 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -163,9 +163,6 @@ func (c *cache) Init() error { // Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. func (c *cache) Close() error { - c.modeMtx.Lock() - defer c.modeMtx.Unlock() - // Finish all in-progress operations. if err := c.setMode(context.TODO(), mode.ReadOnly); err != nil { return err From 09938a98411142872f9c0981c9d21a6a49ebb058 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 20 Apr 2023 19:17:08 +0300 Subject: [PATCH 0137/1943] Revert "[#262] meta: Do not return old expired objects" This reverts commit 3d23b087 Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 - pkg/local_object_storage/metabase/db_test.go | 7 ------ pkg/local_object_storage/metabase/get.go | 13 +--------- pkg/local_object_storage/metabase/util.go | 25 -------------------- 4 files changed, 1 insertion(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 954bad844..8a9bcfd41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ Changelog for FrostFS Node ### Fixed - Take network settings into account during netmap contract update (#100) - Read config files from dir even if config file not provided via `--config` for node (#238) -- Expired by more than 1 epoch objects could be returned (#262) - Notary requests parsing according to `neo-go`'s updates (#268) ### Removed diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index a3b26da79..9ef7bf8bc 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -73,13 +73,6 @@ func checkExpiredObjects(t *testing.T, db *meta.DB, f func(exp, nonExp *objectSD require.NoError(t, metaPut(db, nonExpObj, nil)) f(expObj, nonExpObj) - - oldExpObj := testutil.GenerateObject() - setExpiration(oldExpObj, 1) - - require.NoError(t, metaPut(db, oldExpObj, nil)) - - f(oldExpObj, nonExpObj) } func setExpiration(o *objectSDK.Object, epoch uint64) { diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index c3a5880bf..fff32d6ad 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -66,24 +66,13 @@ func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { } currEpoch := db.epochState.CurrentEpoch() - var obj *objectSDK.Object err = db.boltDB.View(func(tx *bbolt.Tx) error { key := make([]byte, addressKeySize) - obj, err = db.get(tx, prm.addr, key, true, prm.raw, currEpoch) + res.hdr, err = db.get(tx, prm.addr, key, true, prm.raw, currEpoch) return err }) - if err != nil { - return - } - - err = validate(obj, currEpoch) - if err != nil { - return - } - - res.hdr = obj return } diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index d4c93c750..b60c97fd7 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -4,9 +4,7 @@ import ( "bytes" "crypto/sha256" "fmt" - "strconv" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -261,26 +259,3 @@ func isLockObject(tx *bbolt.Tx, idCnr cid.ID, obj oid.ID) bool { bucketNameLockers(idCnr, make([]byte, bucketKeySize)), objectKey(obj, make([]byte, objectKeySize))) } - -// performs object validity checks. -func validate(obj *object.Object, currEpoch uint64) error { - for _, a := range obj.Attributes() { - if key := a.Key(); key != objectV2.SysAttributeExpEpoch && key != objectV2.SysAttributeExpEpochNeoFS { - continue - } - - expEpoch, err := strconv.ParseUint(a.Value(), 10, 64) - if err != nil { - // unexpected for already stored and valudated objects - return fmt.Errorf("expiration epoch parsing: %w", err) - } - - if expEpoch < currEpoch { - return ErrObjectIsExpired - } - - break - } - - return nil -} From dc2e9d70c7ae1891a3ea545f895ea7983a49f184 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 20 Apr 2023 13:37:24 +0300 Subject: [PATCH 0138/1943] [#271] go.mod: Update prometheus to v1.15.0 Signed-off-by: Evgenii Stratonikov --- go.mod | 14 +++++++------- go.sum | 25 ++++++++++++++++--------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 6bc9a0e47..494bac00a 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.4.0 github.com/paulmach/orb v0.2.2 - github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_golang v1.15.0 github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 @@ -38,7 +38,7 @@ require ( golang.org/x/sync v0.1.0 golang.org/x/term v0.5.0 google.golang.org/grpc v1.53.0 - google.golang.org/protobuf v1.28.1 + google.golang.org/protobuf v1.30.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -56,7 +56,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.3 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect @@ -69,7 +69,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.2 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect @@ -85,9 +85,9 @@ require ( github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.5 // indirect diff --git a/go.sum b/go.sum index 36546b3dd..a1616a7b5 100644 --- a/go.sum +++ b/go.sum @@ -205,8 +205,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= @@ -304,8 +305,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -325,8 +326,9 @@ github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= @@ -431,28 +433,32 @@ github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNk github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= +github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= @@ -927,14 +933,15 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= From 59822f7fb404fd567dcc1f506ddff1506b4bd899 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 22 Mar 2023 20:28:14 +0300 Subject: [PATCH 0139/1943] [#1248] node: Do not update cache twice Do not request morph values on morph cache misses concurrently. Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 + cmd/frostfs-node/cache.go | 64 +++++++++++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a9bcfd41..3c2fe6811 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ Changelog for FrostFS Node - Non-alphabet nodes do not try to handle alphabet events (#181) - Failing SN and IR transactions because of incorrect scopes (#2230, #2263) - Global scope used for some transactions (#2230, #2263) +- Concurrent morph cache misses (#30) ### Removed ### Updated diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index 3d4fc7375..2dc8b7f69 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -24,9 +24,19 @@ type valueWithTime[V any] struct { e error } +// valueInProgress is a struct that contains +// values that are being fetched/updated. +type valueInProgress[V any] struct { + m sync.RWMutex + v V + e error +} + // entity that provides TTL cache interface. type ttlNetCache[K comparable, V any] struct { - ttl time.Duration + m sync.RWMutex // protects progMap + progMap map[K]*valueInProgress[V] // contains fetch-in-progress keys + ttl time.Duration sz int @@ -41,32 +51,68 @@ func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr n fatalOnErr(err) return &ttlNetCache[K, V]{ - ttl: ttl, - sz: sz, - cache: cache, - netRdr: netRdr, + ttl: ttl, + sz: sz, + cache: cache, + netRdr: netRdr, + progMap: make(map[K]*valueInProgress[V]), } } +func waitForUpdate[V any](vip *valueInProgress[V]) (V, error) { + vip.m.RLock() + defer vip.m.RUnlock() + + return vip.v, vip.e +} + // reads value by the key. // // updates the value from the network on cache miss or by TTL. // // returned value should not be modified. func (c *ttlNetCache[K, V]) get(key K) (V, error) { - val, ok := c.cache.Peek(key) + valWithTime, ok := c.cache.Peek(key) if ok { - if time.Since(val.t) < c.ttl { - return val.v, val.e + if time.Since(valWithTime.t) < c.ttl { + return valWithTime.v, valWithTime.e } c.cache.Remove(key) } - v, err := c.netRdr(key) + c.m.RLock() + valInProg, ok := c.progMap[key] + c.m.RUnlock() + if ok { + return waitForUpdate(valInProg) + } + + c.m.Lock() + valInProg, ok = c.progMap[key] + if ok { + c.m.Unlock() + return waitForUpdate(valInProg) + } + + valInProg = &valueInProgress[V]{} + valInProg.m.Lock() + c.progMap[key] = valInProg + + c.m.Unlock() + + v, err := c.netRdr(key) c.set(key, v, err) + valInProg.v = v + valInProg.e = err + valInProg.m.Unlock() + + c.m.Lock() + delete(c.progMap, key) + c.m.Unlock() + return v, err } From 015d62425bc3dbe720948ab9d4f50e39ec4f2781 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 5 Apr 2023 11:44:13 +0300 Subject: [PATCH 0140/1943] [#164] metrics: Fill local registry explicitly Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/metrics.go | 4 ++-- pkg/metrics/engine.go | 26 ++++++++++++------------ pkg/metrics/innerring.go | 4 ++-- pkg/metrics/metrics.go | 2 +- pkg/metrics/object.go | 26 ++++++++++++------------ pkg/metrics/registry.go | 40 +++++++++++++++++++++++++++++++++++++ pkg/metrics/state.go | 2 +- 7 files changed, 72 insertions(+), 32 deletions(-) create mode 100644 pkg/metrics/registry.go diff --git a/cmd/frostfs-node/metrics.go b/cmd/frostfs-node/metrics.go index f18bbe65d..cf621086d 100644 --- a/cmd/frostfs-node/metrics.go +++ b/cmd/frostfs-node/metrics.go @@ -2,7 +2,7 @@ package main import ( metricsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/metrics" - "github.com/prometheus/client_golang/prometheus/promhttp" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) func metricsComponent(c *cfg) (*httpComponent, bool) { @@ -12,7 +12,7 @@ func metricsComponent(c *cfg) (*httpComponent, bool) { c.dynamicConfiguration.metrics = new(httpComponent) c.dynamicConfiguration.metrics.cfg = c c.dynamicConfiguration.metrics.name = "metrics" - c.dynamicConfiguration.metrics.handler = promhttp.Handler() + c.dynamicConfiguration.metrics.handler = metrics.Handler() updated = true } diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 4c51a55aa..73606fd23 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -72,19 +72,19 @@ func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeV } func (m engineMetrics) register() { - prometheus.MustRegister(m.listContainersDuration) - prometheus.MustRegister(m.estimateContainerSizeDuration) - prometheus.MustRegister(m.deleteDuration) - prometheus.MustRegister(m.existsDuration) - prometheus.MustRegister(m.getDuration) - prometheus.MustRegister(m.headDuration) - prometheus.MustRegister(m.inhumeDuration) - prometheus.MustRegister(m.putDuration) - prometheus.MustRegister(m.rangeDuration) - prometheus.MustRegister(m.searchDuration) - prometheus.MustRegister(m.listObjectsDuration) - prometheus.MustRegister(m.containerSize) - prometheus.MustRegister(m.payloadSize) + mustRegister(m.listContainersDuration) + mustRegister(m.estimateContainerSizeDuration) + mustRegister(m.deleteDuration) + mustRegister(m.existsDuration) + mustRegister(m.getDuration) + mustRegister(m.headDuration) + mustRegister(m.inhumeDuration) + mustRegister(m.putDuration) + mustRegister(m.rangeDuration) + mustRegister(m.searchDuration) + mustRegister(m.listObjectsDuration) + mustRegister(m.containerSize) + mustRegister(m.payloadSize) } func (m engineMetrics) AddListContainersDuration(d time.Duration) { diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 55b0aa089..4171b7423 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -27,8 +27,8 @@ func NewInnerRingMetrics() InnerRingServiceMetrics { }) ) - prometheus.MustRegister(epoch) - prometheus.MustRegister(health) + mustRegister(epoch) + mustRegister(health) return InnerRingServiceMetrics{ epoch: epoch, diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 640da2d5f..4ebd2afa1 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -27,7 +27,7 @@ func NewNodeMetrics() *NodeMetrics { Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }) - prometheus.MustRegister(epoch) + mustRegister(epoch) return &NodeMetrics{ objectServiceMetrics: objectService, diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index fae86cb4a..ccc7c129c 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -65,8 +65,8 @@ func newObjectMethodCallCounter(name string) methodCount { } func (m methodCount) mustRegister() { - prometheus.MustRegister(m.success) - prometheus.MustRegister(m.total) + mustRegister(m.success) + mustRegister(m.total) } func (m methodCount) Inc(success bool) { @@ -135,19 +135,19 @@ func (m objectServiceMetrics) register() { m.rangeCounter.mustRegister() m.rangeHashCounter.mustRegister() - prometheus.MustRegister(m.getDuration) - prometheus.MustRegister(m.putDuration) - prometheus.MustRegister(m.headDuration) - prometheus.MustRegister(m.searchDuration) - prometheus.MustRegister(m.deleteDuration) - prometheus.MustRegister(m.rangeDuration) - prometheus.MustRegister(m.rangeHashDuration) + mustRegister(m.getDuration) + mustRegister(m.putDuration) + mustRegister(m.headDuration) + mustRegister(m.searchDuration) + mustRegister(m.deleteDuration) + mustRegister(m.rangeDuration) + mustRegister(m.rangeHashDuration) - prometheus.MustRegister(m.putPayload) - prometheus.MustRegister(m.getPayload) + mustRegister(m.putPayload) + mustRegister(m.getPayload) - prometheus.MustRegister(m.shardMetrics) - prometheus.MustRegister(m.shardsReadonly) + mustRegister(m.shardMetrics) + mustRegister(m.shardsReadonly) } func (m objectServiceMetrics) IncGetReqCounter(success bool) { diff --git a/pkg/metrics/registry.go b/pkg/metrics/registry.go new file mode 100644 index 000000000..c21c80449 --- /dev/null +++ b/pkg/metrics/registry.go @@ -0,0 +1,40 @@ +package metrics + +import ( + "net/http" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// Handler returns an http.Handler for the local registry. +func Handler() http.Handler { + promhttp.Handler() + return promhttp.InstrumentMetricHandler( + registry, + promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) +} + +var ( + registry = prometheus.NewRegistry() + // registeredCollectorsMtx protects collectors slice. + // It should not be acessed concurrently, but we can easily forget this in future, thus this mutex. + registeredCollectorsMtx sync.Mutex + registeredCollectors []prometheus.Collector +) + +func init() { + mustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + mustRegister(collectors.NewGoCollector()) +} + +func mustRegister(cs ...prometheus.Collector) { + for i := range cs { + registry.MustRegister(cs[i]) + } + registeredCollectorsMtx.Lock() + registeredCollectors = append(registeredCollectors, cs...) + registeredCollectorsMtx.Unlock() +} diff --git a/pkg/metrics/state.go b/pkg/metrics/state.go index 94e28af38..9c96743ff 100644 --- a/pkg/metrics/state.go +++ b/pkg/metrics/state.go @@ -20,7 +20,7 @@ func newStateMetrics() stateMetrics { } func (m stateMetrics) register() { - prometheus.MustRegister(m.healthCheck) + mustRegister(m.healthCheck) } func (m stateMetrics) SetHealth(s int32) { From 4ade5339da6fd422fe0eabeb5a83503c97a582f1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 5 Apr 2023 11:45:22 +0300 Subject: [PATCH 0141/1943] [#164] metrics: Rename `metrics.go` -> `node.go` Signed-off-by: Evgenii Stratonikov --- pkg/metrics/{metrics.go => node.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/metrics/{metrics.go => node.go} (100%) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/node.go similarity index 100% rename from pkg/metrics/metrics.go rename to pkg/metrics/node.go From 6fef2726b865de241ddcb7eb2bdbbb818fb439de Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 5 Apr 2023 13:39:02 +0300 Subject: [PATCH 0142/1943] [#164] metrics: Allow to export metrics description Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- pkg/metrics/desc.go | 71 ++++++++++++++++++++++++++++++++++++++++ pkg/metrics/desc_test.go | 65 ++++++++++++++++++++++++++++++++++++ pkg/metrics/engine.go | 66 ++++++++++++++++++------------------- pkg/metrics/innerring.go | 12 +++---- pkg/metrics/node.go | 6 ++-- pkg/metrics/object.go | 70 +++++++++++++++++++-------------------- pkg/metrics/registry.go | 22 +++++++------ pkg/metrics/state.go | 6 ++-- 9 files changed, 229 insertions(+), 91 deletions(-) create mode 100644 pkg/metrics/desc.go create mode 100644 pkg/metrics/desc_test.go diff --git a/go.mod b/go.mod index 494bac00a..8cbb4ba38 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/panjf2000/ants/v2 v2.4.0 github.com/paulmach/orb v0.2.2 github.com/prometheus/client_golang v1.15.0 + github.com/prometheus/client_model v0.3.0 github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 @@ -85,7 +86,6 @@ require ( github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/pkg/metrics/desc.go b/pkg/metrics/desc.go new file mode 100644 index 000000000..74d2d4e6e --- /dev/null +++ b/pkg/metrics/desc.go @@ -0,0 +1,71 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +type metric[T prometheus.Collector] struct { + value T + desc Description +} + +// Descriptions contains metric description suitable for further processing. +// The only reason for it to exist is `prometheus.Desc` disallowing field access directly. +// https://github.com/prometheus/client_golang/pull/326 +// https://github.com/prometheus/client_golang/issues/516 +// https://github.com/prometheus/client_golang/issues/222 +type Description struct { + Name string `json:"name"` + Help string `json:"help"` + Type string `json:"type"` + ConstantLabels prometheus.Labels `json:"constant_labels,omitempty"` + VariableLabels []string `json:"variable_labels,omitempty"` +} + +func newGauge(opts prometheus.GaugeOpts) metric[prometheus.Gauge] { + return metric[prometheus.Gauge]{ + value: prometheus.NewGauge(opts), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_GAUGE.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + }, + } +} + +func newGaugeVec(opts prometheus.GaugeOpts, labelNames []string) metric[*prometheus.GaugeVec] { + return metric[*prometheus.GaugeVec]{ + value: prometheus.NewGaugeVec(opts, labelNames), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_GAUGE.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + VariableLabels: labelNames, + }, + } +} + +func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] { + return metric[prometheus.Counter]{ + value: prometheus.NewCounter(opts), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_COUNTER.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + }, + } +} + +// DescribeAll returns descriptions for all registered metrics. +func DescribeAll() ([]Description, error) { + registeredDescriptionsMtx.Lock() + defer registeredDescriptionsMtx.Unlock() + + ds := make([]Description, len(registeredDescriptions)) + copy(ds, registeredDescriptions) + return ds, nil +} diff --git a/pkg/metrics/desc_test.go b/pkg/metrics/desc_test.go new file mode 100644 index 000000000..28b5e2132 --- /dev/null +++ b/pkg/metrics/desc_test.go @@ -0,0 +1,65 @@ +package metrics + +import ( + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" +) + +func TestDescribeAll(t *testing.T) { + const ( + namespace = "my_ns" + subsystem = "mysub" + ) + mustRegister(newCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "my_counter", + })) + + labels := []string{"label1", "label2"} + mustRegister(newGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "my_gauge", + }, labels)) + + constLabels := prometheus.Labels{ + "const1": "abc", + "const2": "xyz", + } + mustRegister(newCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "with_const_labels", + ConstLabels: constLabels, + })) + + descriptions, err := DescribeAll() + require.NoError(t, err) + + seen := make(map[string]bool) + for i := range descriptions { + if !strings.HasPrefix(descriptions[i].Name, namespace) { + continue + } + + require.False(t, seen[descriptions[i].Name], "metric %s was seen twice", descriptions[i].Name) + seen[descriptions[i].Name] = true + + switch descriptions[i].Name { + case prometheus.BuildFQName(namespace, subsystem, "my_counter"): + require.True(t, len(descriptions[i].VariableLabels) == 0) + case prometheus.BuildFQName(namespace, subsystem, "my_gauge"): + require.Equal(t, labels, descriptions[i].VariableLabels) + case prometheus.BuildFQName(namespace, subsystem, "with_const_labels"): + require.Equal(t, len(constLabels), len(descriptions[i].ConstantLabels)) + require.Equal(t, constLabels, descriptions[i].ConstantLabels) + default: + require.FailNow(t, "unexpected metric name: %s", descriptions[i].Name) + } + } + require.Equal(t, 3, len(seen), "not all registered metrics were iterated over") +} diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 73606fd23..28fc1e028 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -10,19 +10,19 @@ import ( type ( engineMetrics struct { - listContainersDuration prometheus.Counter - estimateContainerSizeDuration prometheus.Counter - deleteDuration prometheus.Counter - existsDuration prometheus.Counter - getDuration prometheus.Counter - headDuration prometheus.Counter - inhumeDuration prometheus.Counter - putDuration prometheus.Counter - rangeDuration prometheus.Counter - searchDuration prometheus.Counter - listObjectsDuration prometheus.Counter - containerSize prometheus.GaugeVec - payloadSize prometheus.GaugeVec + listContainersDuration metric[prometheus.Counter] + estimateContainerSizeDuration metric[prometheus.Counter] + deleteDuration metric[prometheus.Counter] + existsDuration metric[prometheus.Counter] + getDuration metric[prometheus.Counter] + headDuration metric[prometheus.Counter] + inhumeDuration metric[prometheus.Counter] + putDuration metric[prometheus.Counter] + rangeDuration metric[prometheus.Counter] + searchDuration metric[prometheus.Counter] + listObjectsDuration metric[prometheus.Counter] + containerSize metric[*prometheus.GaugeVec] + payloadSize metric[*prometheus.GaugeVec] } ) @@ -41,13 +41,13 @@ func newEngineMetrics() engineMetrics { rangeDuration: newEngineMethodDurationCounter("range"), searchDuration: newEngineMethodDurationCounter("search"), listObjectsDuration: newEngineMethodDurationCounter("list_objects"), - containerSize: *newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), - payloadSize: *newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), + containerSize: newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), + payloadSize: newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), } } -func newEngineCounter(name, help string) prometheus.Counter { - return prometheus.NewCounter(prometheus.CounterOpts{ +func newEngineCounter(name, help string) metric[prometheus.Counter] { + return newCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: name, @@ -55,15 +55,15 @@ func newEngineCounter(name, help string) prometheus.Counter { }) } -func newEngineMethodDurationCounter(method string) prometheus.Counter { +func newEngineMethodDurationCounter(method string) metric[prometheus.Counter] { return newEngineCounter( fmt.Sprintf("%s_duration", method), fmt.Sprintf("Accumulated duration of engine %s operations", strings.ReplaceAll(method, "_", " ")), ) } -func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { - return prometheus.NewGaugeVec(prometheus.GaugeOpts{ +func newEngineGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] { + return newGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: name, @@ -88,53 +88,53 @@ func (m engineMetrics) register() { } func (m engineMetrics) AddListContainersDuration(d time.Duration) { - m.listObjectsDuration.Add(float64(d)) + m.listObjectsDuration.value.Add(float64(d)) } func (m engineMetrics) AddEstimateContainerSizeDuration(d time.Duration) { - m.estimateContainerSizeDuration.Add(float64(d)) + m.estimateContainerSizeDuration.value.Add(float64(d)) } func (m engineMetrics) AddDeleteDuration(d time.Duration) { - m.deleteDuration.Add(float64(d)) + m.deleteDuration.value.Add(float64(d)) } func (m engineMetrics) AddExistsDuration(d time.Duration) { - m.existsDuration.Add(float64(d)) + m.existsDuration.value.Add(float64(d)) } func (m engineMetrics) AddGetDuration(d time.Duration) { - m.getDuration.Add(float64(d)) + m.getDuration.value.Add(float64(d)) } func (m engineMetrics) AddHeadDuration(d time.Duration) { - m.headDuration.Add(float64(d)) + m.headDuration.value.Add(float64(d)) } func (m engineMetrics) AddInhumeDuration(d time.Duration) { - m.inhumeDuration.Add(float64(d)) + m.inhumeDuration.value.Add(float64(d)) } func (m engineMetrics) AddPutDuration(d time.Duration) { - m.putDuration.Add(float64(d)) + m.putDuration.value.Add(float64(d)) } func (m engineMetrics) AddRangeDuration(d time.Duration) { - m.rangeDuration.Add(float64(d)) + m.rangeDuration.value.Add(float64(d)) } func (m engineMetrics) AddSearchDuration(d time.Duration) { - m.searchDuration.Add(float64(d)) + m.searchDuration.value.Add(float64(d)) } func (m engineMetrics) AddListObjectsDuration(d time.Duration) { - m.listObjectsDuration.Add(float64(d)) + m.listObjectsDuration.value.Add(float64(d)) } func (m engineMetrics) AddToContainerSize(cnrID string, size int64) { - m.containerSize.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) + m.containerSize.value.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) } func (m engineMetrics) AddToPayloadCounter(shardID string, size int64) { - m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) + m.payloadSize.value.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 4171b7423..05b76f9c9 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -6,20 +6,20 @@ const innerRingSubsystem = "ir" // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { - epoch prometheus.Gauge - health prometheus.Gauge + epoch metric[prometheus.Gauge] + health metric[prometheus.Gauge] } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. func NewInnerRingMetrics() InnerRingServiceMetrics { var ( - epoch = prometheus.NewGauge(prometheus.GaugeOpts{ + epoch = newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }) - health = prometheus.NewGauge(prometheus.GaugeOpts{ + health = newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "health", @@ -38,10 +38,10 @@ func NewInnerRingMetrics() InnerRingServiceMetrics { // SetEpoch updates epoch metrics. func (m InnerRingServiceMetrics) SetEpoch(epoch uint64) { - m.epoch.Set(float64(epoch)) + m.epoch.value.Set(float64(epoch)) } // SetHealth updates health metrics. func (m InnerRingServiceMetrics) SetHealth(s int32) { - m.health.Set(float64(s)) + m.health.value.Set(float64(s)) } diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 4ebd2afa1..0f9c6183d 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -8,7 +8,7 @@ type NodeMetrics struct { objectServiceMetrics engineMetrics stateMetrics - epoch prometheus.Gauge + epoch metric[prometheus.Gauge] } func NewNodeMetrics() *NodeMetrics { @@ -21,7 +21,7 @@ func NewNodeMetrics() *NodeMetrics { state := newStateMetrics() state.register() - epoch := prometheus.NewGauge(prometheus.GaugeOpts{ + epoch := newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "epoch", @@ -39,5 +39,5 @@ func NewNodeMetrics() *NodeMetrics { // SetEpoch updates epoch metric. func (m *NodeMetrics) SetEpoch(epoch uint64) { - m.epoch.Set(float64(epoch)) + m.epoch.value.Set(float64(epoch)) } diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index ccc7c129c..5ec575749 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -12,8 +12,8 @@ const objectSubsystem = "object" type ( methodCount struct { - success prometheus.Counter - total prometheus.Counter + success metric[prometheus.Counter] + total metric[prometheus.Counter] } objectServiceMetrics struct { @@ -25,19 +25,19 @@ type ( rangeCounter methodCount rangeHashCounter methodCount - getDuration prometheus.Counter - putDuration prometheus.Counter - headDuration prometheus.Counter - searchDuration prometheus.Counter - deleteDuration prometheus.Counter - rangeDuration prometheus.Counter - rangeHashDuration prometheus.Counter + getDuration metric[prometheus.Counter] + putDuration metric[prometheus.Counter] + headDuration metric[prometheus.Counter] + searchDuration metric[prometheus.Counter] + deleteDuration metric[prometheus.Counter] + rangeDuration metric[prometheus.Counter] + rangeHashDuration metric[prometheus.Counter] - putPayload prometheus.Counter - getPayload prometheus.Counter + putPayload metric[prometheus.Counter] + getPayload metric[prometheus.Counter] - shardMetrics *prometheus.GaugeVec - shardsReadonly *prometheus.GaugeVec + shardMetrics metric[*prometheus.GaugeVec] + shardsReadonly metric[*prometheus.GaugeVec] } ) @@ -49,13 +49,13 @@ const ( func newObjectMethodCallCounter(name string) methodCount { return methodCount{ - success: prometheus.NewCounter(prometheus.CounterOpts{ + success: newCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_count_success", name), Help: fmt.Sprintf("The number of successful %s requests processed", name), }), - total: prometheus.NewCounter(prometheus.CounterOpts{ + total: newCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_count", name), @@ -70,9 +70,9 @@ func (m methodCount) mustRegister() { } func (m methodCount) Inc(success bool) { - m.total.Inc() + m.total.value.Inc() if success { - m.success.Inc() + m.success.value.Inc() } } @@ -99,8 +99,8 @@ func newObjectServiceMetrics() objectServiceMetrics { } } -func newObjectMethodPayloadCounter(method string) prometheus.Counter { - return prometheus.NewCounter(prometheus.CounterOpts{ +func newObjectMethodPayloadCounter(method string) metric[prometheus.Counter] { + return newCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_payload", method), @@ -108,8 +108,8 @@ func newObjectMethodPayloadCounter(method string) prometheus.Counter { }) } -func newObjectMethodDurationCounter(method string) prometheus.Counter { - return prometheus.NewCounter(prometheus.CounterOpts{ +func newObjectMethodDurationCounter(method string) metric[prometheus.Counter] { + return newCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_duration", method), @@ -117,8 +117,8 @@ func newObjectMethodDurationCounter(method string) prometheus.Counter { }) } -func newObjectGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { - return prometheus.NewGaugeVec(prometheus.GaugeOpts{ +func newObjectGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] { + return newGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: name, @@ -179,43 +179,43 @@ func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) { } func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) { - m.getDuration.Add(float64(d)) + m.getDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddPutReqDuration(d time.Duration) { - m.putDuration.Add(float64(d)) + m.putDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddHeadReqDuration(d time.Duration) { - m.headDuration.Add(float64(d)) + m.headDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddSearchReqDuration(d time.Duration) { - m.searchDuration.Add(float64(d)) + m.searchDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddDeleteReqDuration(d time.Duration) { - m.deleteDuration.Add(float64(d)) + m.deleteDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddRangeReqDuration(d time.Duration) { - m.rangeDuration.Add(float64(d)) + m.rangeDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddRangeHashReqDuration(d time.Duration) { - m.rangeHashDuration.Add(float64(d)) + m.rangeHashDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddPutPayload(ln int) { - m.putPayload.Add(float64(ln)) + m.putPayload.value.Add(float64(ln)) } func (m objectServiceMetrics) AddGetPayload(ln int) { - m.getPayload.Add(float64(ln)) + m.getPayload.value.Add(float64(ln)) } func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, delta int) { - m.shardMetrics.With( + m.shardMetrics.value.With( prometheus.Labels{ shardIDLabelKey: shardID, counterTypeLabelKey: objectType, @@ -224,7 +224,7 @@ func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, del } func (m objectServiceMetrics) SetObjectCounter(shardID, objectType string, v uint64) { - m.shardMetrics.With( + m.shardMetrics.value.With( prometheus.Labels{ shardIDLabelKey: shardID, counterTypeLabelKey: objectType, @@ -237,7 +237,7 @@ func (m objectServiceMetrics) SetReadonly(shardID string, readonly bool) { if readonly { flag = 1 } - m.shardsReadonly.With( + m.shardsReadonly.value.With( prometheus.Labels{ shardIDLabelKey: shardID, }, diff --git a/pkg/metrics/registry.go b/pkg/metrics/registry.go index c21c80449..eef613d04 100644 --- a/pkg/metrics/registry.go +++ b/pkg/metrics/registry.go @@ -19,22 +19,24 @@ func Handler() http.Handler { var ( registry = prometheus.NewRegistry() - // registeredCollectorsMtx protects collectors slice. + // registeredDescriptionsMtx protects collectors slice. // It should not be acessed concurrently, but we can easily forget this in future, thus this mutex. - registeredCollectorsMtx sync.Mutex - registeredCollectors []prometheus.Collector + registeredDescriptionsMtx sync.Mutex + registeredDescriptions []Description ) func init() { - mustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) - mustRegister(collectors.NewGoCollector()) + registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + registry.MustRegister(collectors.NewGoCollector()) } -func mustRegister(cs ...prometheus.Collector) { +func mustRegister[T prometheus.Collector](cs ...metric[T]) { for i := range cs { - registry.MustRegister(cs[i]) + registry.MustRegister(cs[i].value) } - registeredCollectorsMtx.Lock() - registeredCollectors = append(registeredCollectors, cs...) - registeredCollectorsMtx.Unlock() + registeredDescriptionsMtx.Lock() + for i := range cs { + registeredDescriptions = append(registeredDescriptions, cs[i].desc) + } + registeredDescriptionsMtx.Unlock() } diff --git a/pkg/metrics/state.go b/pkg/metrics/state.go index 9c96743ff..dce0402cd 100644 --- a/pkg/metrics/state.go +++ b/pkg/metrics/state.go @@ -5,12 +5,12 @@ import "github.com/prometheus/client_golang/prometheus" const stateSubsystem = "state" type stateMetrics struct { - healthCheck prometheus.Gauge + healthCheck metric[prometheus.Gauge] } func newStateMetrics() stateMetrics { return stateMetrics{ - healthCheck: prometheus.NewGauge(prometheus.GaugeOpts{ + healthCheck: newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: stateSubsystem, Name: "health", @@ -24,5 +24,5 @@ func (m stateMetrics) register() { } func (m stateMetrics) SetHealth(s int32) { - m.healthCheck.Set(float64(s)) + m.healthCheck.value.Set(float64(s)) } From c70306b3240bc265b11dddaf4c1f04d68970bad3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 5 Apr 2023 13:47:40 +0300 Subject: [PATCH 0143/1943] [#164] scripts: Add metrics description exporter Signed-off-by: Evgenii Stratonikov --- scripts/export-metrics/main.go | 56 ++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 scripts/export-metrics/main.go diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go new file mode 100644 index 000000000..ac6e786ab --- /dev/null +++ b/scripts/export-metrics/main.go @@ -0,0 +1,56 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "os" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +var ( + node = flag.String("node", "", "File to export storage node metrics to.") + ir = flag.String("ir", "", "File to export innerring node metrics to.") +) + +func main() { + flag.Parse() + + if *node != "" && *ir != "" { + fmt.Println("-node and -ir flags are mutually exclusive") + os.Exit(1) + } + + var filename string + switch { + case *node != "": + _ = metrics.NewNodeMetrics() + filename = *node + case *ir != "": + _ = metrics.NewInnerRingMetrics() + filename = *ir + + default: + flag.Usage() + os.Exit(1) + } + + ds, err := metrics.DescribeAll() + if err != nil { + fmt.Fprintf(os.Stderr, "Could not parse metric descriptions: %v\n", err) + os.Exit(1) + } + + data, err := json.Marshal(ds) + if err != nil { + fmt.Fprintf(os.Stderr, "Could not parse marshal: %v\n", err) + os.Exit(1) + } + + if err := ioutil.WriteFile(filename, data, 0644); err != nil { + fmt.Fprintf(os.Stderr, "Could write to file: %v\n", err) + os.Exit(1) + } +} From ddbc9e255f1a179a787179b5297a6011244e0879 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 20 Apr 2023 17:21:30 +0300 Subject: [PATCH 0144/1943] [#231] node: Invalidate container cache on PutSuccess event For example: frostfs-cli creates container and makes polling GetContainer requests. These requests go through container cache, so not found error stores in container cache. So container cache can contain not found error when PutSuccess event received. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/container.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 633d4b261..569e4a7ca 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -131,9 +131,10 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c // TODO: use owner directly from the event after neofs-contract#256 will become resolved // but don't forget about the profit of reading the new container and caching it: // creation success are most commonly tracked by polling GET op. - cnr, err := cachedContainerStorage.Get(ev.ID) + cnr, err := cnrSrc.Get(ev.ID) if err == nil { cachedContainerLister.update(cnr.Value.Owner(), ev.ID, true) + cachedContainerStorage.set(ev.ID, cnr, nil) } else { // unlike removal, we expect successful receive of the container // after successful creation, so logging can be useful From 04be9415d99f40479f2a40cc14757bec411e8f41 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 21 Apr 2023 13:49:05 +0300 Subject: [PATCH 0145/1943] [#231] node: Fix race condition in TTL cache Use key locker to lock by key. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/cache.go | 134 ++++++++++++++++++++------------- cmd/frostfs-node/cache_test.go | 32 ++++++++ 2 files changed, 112 insertions(+), 54 deletions(-) create mode 100644 cmd/frostfs-node/cache_test.go diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index 2dc8b7f69..dfbaf3525 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -24,25 +24,72 @@ type valueWithTime[V any] struct { e error } -// valueInProgress is a struct that contains -// values that are being fetched/updated. -type valueInProgress[V any] struct { - m sync.RWMutex - v V - e error +type locker struct { + mtx *sync.Mutex + waiters int // not protected by mtx, must used outer mutex to update concurrently +} + +type keyLocker[K comparable] struct { + lockers map[K]*locker + lockersMtx *sync.Mutex +} + +func newKeyLocker[K comparable]() *keyLocker[K] { + return &keyLocker[K]{ + lockers: make(map[K]*locker), + lockersMtx: &sync.Mutex{}, + } +} + +func (l *keyLocker[K]) LockKey(key K) { + l.lockersMtx.Lock() + + if locker, found := l.lockers[key]; found { + locker.waiters++ + l.lockersMtx.Unlock() + + locker.mtx.Lock() + return + } + + locker := &locker{ + mtx: &sync.Mutex{}, + waiters: 1, + } + locker.mtx.Lock() + + l.lockers[key] = locker + l.lockersMtx.Unlock() +} + +func (l *keyLocker[K]) UnlockKey(key K) { + l.lockersMtx.Lock() + defer l.lockersMtx.Unlock() + + locker, found := l.lockers[key] + if !found { + return + } + + if locker.waiters == 1 { + delete(l.lockers, key) + } + locker.waiters-- + + locker.mtx.Unlock() } // entity that provides TTL cache interface. type ttlNetCache[K comparable, V any] struct { - m sync.RWMutex // protects progMap - progMap map[K]*valueInProgress[V] // contains fetch-in-progress keys - ttl time.Duration + ttl time.Duration sz int cache *lru.Cache[K, *valueWithTime[V]] netRdr netValueReader[K, V] + + keyLocker *keyLocker[K] } // complicates netValueReader with TTL caching mechanism. @@ -51,72 +98,48 @@ func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr n fatalOnErr(err) return &ttlNetCache[K, V]{ - ttl: ttl, - sz: sz, - cache: cache, - netRdr: netRdr, - progMap: make(map[K]*valueInProgress[V]), + ttl: ttl, + sz: sz, + cache: cache, + netRdr: netRdr, + keyLocker: newKeyLocker[K](), } } -func waitForUpdate[V any](vip *valueInProgress[V]) (V, error) { - vip.m.RLock() - defer vip.m.RUnlock() - - return vip.v, vip.e -} - // reads value by the key. // // updates the value from the network on cache miss or by TTL. // // returned value should not be modified. func (c *ttlNetCache[K, V]) get(key K) (V, error) { - valWithTime, ok := c.cache.Peek(key) - if ok { - if time.Since(valWithTime.t) < c.ttl { - return valWithTime.v, valWithTime.e - } - - c.cache.Remove(key) + val, ok := c.cache.Peek(key) + if ok && time.Since(val.t) < c.ttl { + return val.v, val.e } - c.m.RLock() - valInProg, ok := c.progMap[key] - c.m.RUnlock() + c.keyLocker.LockKey(key) + defer c.keyLocker.UnlockKey(key) - if ok { - return waitForUpdate(valInProg) + val, ok = c.cache.Peek(key) + if ok && time.Since(val.t) < c.ttl { + return val.v, val.e } - c.m.Lock() - valInProg, ok = c.progMap[key] - if ok { - c.m.Unlock() - return waitForUpdate(valInProg) - } - - valInProg = &valueInProgress[V]{} - valInProg.m.Lock() - c.progMap[key] = valInProg - - c.m.Unlock() - v, err := c.netRdr(key) - c.set(key, v, err) - valInProg.v = v - valInProg.e = err - valInProg.m.Unlock() - - c.m.Lock() - delete(c.progMap, key) - c.m.Unlock() + c.cache.Add(key, &valueWithTime[V]{ + v: v, + t: time.Now(), + e: err, + }) return v, err } func (c *ttlNetCache[K, V]) set(k K, v V, e error) { + c.keyLocker.LockKey(k) + defer c.keyLocker.UnlockKey(k) + c.cache.Add(k, &valueWithTime[V]{ v: v, t: time.Now(), @@ -125,6 +148,9 @@ func (c *ttlNetCache[K, V]) set(k K, v V, e error) { } func (c *ttlNetCache[K, V]) remove(key K) { + c.keyLocker.LockKey(key) + defer c.keyLocker.UnlockKey(key) + c.cache.Remove(key) } diff --git a/cmd/frostfs-node/cache_test.go b/cmd/frostfs-node/cache_test.go new file mode 100644 index 000000000..a3e1c4ea6 --- /dev/null +++ b/cmd/frostfs-node/cache_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +func TestKeyLocker(t *testing.T) { + taken := false + eg, _ := errgroup.WithContext(context.Background()) + keyLocker := newKeyLocker[int]() + for i := 0; i < 100; i++ { + eg.Go(func() error { + keyLocker.LockKey(0) + defer keyLocker.UnlockKey(0) + + require.False(t, taken) + taken = true + require.True(t, taken) + time.Sleep(10 * time.Millisecond) + taken = false + require.False(t, taken) + + return nil + }) + } + require.NoError(t, eg.Wait()) +} From c04f6c5e593e013f42037745390e506b1730b065 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Mon, 24 Oct 2022 16:29:05 +0300 Subject: [PATCH 0146/1943] [#229] acl: Allow Impersonate Signed-off-by: Alex Vanin --- pkg/services/object/acl/acl.go | 6 ++++-- pkg/services/object/acl/v2/request.go | 12 ++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/services/object/acl/acl.go b/pkg/services/object/acl/acl.go index 87d2f9c82..351b4ad3b 100644 --- a/pkg/services/object/acl/acl.go +++ b/pkg/services/object/acl/acl.go @@ -125,15 +125,17 @@ func (c *Checker) CheckEACL(msg any, reqInfo v2.RequestInfo) error { return nil } + bearerTok := reqInfo.Bearer() + impersonate := bearerTok != nil && bearerTok.Impersonate() + // if bearer token is not allowed, then ignore it - if !basicACL.AllowedBearerRules(reqInfo.Operation()) { + if impersonate || !basicACL.AllowedBearerRules(reqInfo.Operation()) { reqInfo.CleanBearer() } var table eaclSDK.Table cnr := reqInfo.ContainerID() - bearerTok := reqInfo.Bearer() if bearerTok == nil { eaclInfo, err := c.eaclSrc.GetEACL(cnr) if err != nil { diff --git a/pkg/services/object/acl/v2/request.go b/pkg/services/object/acl/v2/request.go index 0cf734d7a..675768969 100644 --- a/pkg/services/object/acl/v2/request.go +++ b/pkg/services/object/acl/v2/request.go @@ -113,6 +113,10 @@ func (r MetaWithToken) RequestOwner() (*user.ID, *keys.PublicKey, error) { return nil, nil, errEmptyVerificationHeader } + if r.bearer != nil && r.bearer.Impersonate() { + return unmarshalPublicKeyWithOwner(r.bearer.SigningKeyBytes()) + } + // if session token is presented, use it as truth source if r.token != nil { // verify signature of session token @@ -125,9 +129,13 @@ func (r MetaWithToken) RequestOwner() (*user.ID, *keys.PublicKey, error) { return nil, nil, errEmptyBodySig } - key, err := unmarshalPublicKey(bodySignature.GetKey()) + return unmarshalPublicKeyWithOwner(bodySignature.GetKey()) +} + +func unmarshalPublicKeyWithOwner(rawKey []byte) (*user.ID, *keys.PublicKey, error) { + key, err := unmarshalPublicKey(rawKey) if err != nil { - return nil, nil, fmt.Errorf("invalid key in body signature: %w", err) + return nil, nil, fmt.Errorf("invalid signature key: %w", err) } var idSender user.ID From 89530534a133f470f4a3a1bb2e0d3d843a0c70fb Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Tue, 25 Oct 2022 12:52:41 +0300 Subject: [PATCH 0147/1943] [#229] service/tree: Disable container owner check in tree service Signed-off-by: Alex Vanin --- pkg/services/tree/signature.go | 68 ++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index e861541f4..4aacbc3b1 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -57,7 +57,23 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op return fmt.Errorf("can't get container %s: %w", cid, err) } - role, err := roleFromReq(cnr, req) + eaclOp := eACLOp(op) + + var bt *bearer.Token + if len(rawBearer) > 0 { + bt = new(bearer.Token) + if err = bt.Unmarshal(rawBearer); err != nil { + return eACLErr(eaclOp, fmt.Errorf("invalid bearer token: %w", err)) + } + if !bt.AssertContainer(cid) { + return eACLErr(eaclOp, errBearerWrongContainer) + } + if !bt.VerifySignature() { + return eACLErr(eaclOp, errBearerSignature) + } + } + + role, err := roleFromReq(cnr, req, bt) if err != nil { return fmt.Errorf("can't get request role: %w", err) } @@ -72,8 +88,6 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op return nil } - eaclOp := eACLOp(op) - var tableFromBearer bool if len(rawBearer) != 0 { if !basicACL.AllowedBearerRules(op) { @@ -88,29 +102,22 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op var tb eacl.Table if tableFromBearer { - var bt bearer.Token - if err = bt.Unmarshal(rawBearer); err != nil { - return eACLErr(eaclOp, fmt.Errorf("invalid bearer token: %w", err)) + if bt.Impersonate() { + tbCore, err := s.eaclSource.GetEACL(cid) + if err != nil { + return handleGetEACLError(err) + } + tb = *tbCore.Value + } else { + if !bearer.ResolveIssuer(*bt).Equals(cnr.Value.Owner()) { + return eACLErr(eaclOp, errBearerWrongOwner) + } + tb = bt.EACLTable() } - if !bearer.ResolveIssuer(bt).Equals(cnr.Value.Owner()) { - return eACLErr(eaclOp, errBearerWrongOwner) - } - if !bt.AssertContainer(cid) { - return eACLErr(eaclOp, errBearerWrongContainer) - } - if !bt.VerifySignature() { - return eACLErr(eaclOp, errBearerSignature) - } - - tb = bt.EACLTable() } else { tbCore, err := s.eaclSource.GetEACL(cid) if err != nil { - if client.IsErrEACLNotFound(err) { - return nil - } - - return fmt.Errorf("get eACL table: %w", err) + return handleGetEACLError(err) } tb = *tbCore.Value @@ -119,6 +126,14 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op return checkEACL(tb, req.GetSignature().GetKey(), eACLRole(role), eaclOp) } +func handleGetEACLError(err error) error { + if client.IsErrEACLNotFound(err) { + return nil + } + + return fmt.Errorf("get eACL table: %w", err) +} + func verifyMessage(m message) error { binBody, err := m.ReadSignedData(nil) if err != nil { @@ -169,11 +184,16 @@ func SignMessage(m message, key *ecdsa.PrivateKey) error { return nil } -func roleFromReq(cnr *core.Container, req message) (acl.Role, error) { +func roleFromReq(cnr *core.Container, req message, bt *bearer.Token) (acl.Role, error) { role := acl.RoleOthers owner := cnr.Value.Owner() - pub, err := keys.NewPublicKeyFromBytes(req.GetSignature().GetKey(), elliptic.P256()) + rawKey := req.GetSignature().GetKey() + if bt != nil && bt.Impersonate() { + rawKey = bt.SigningKeyBytes() + } + + pub, err := keys.NewPublicKeyFromBytes(rawKey, elliptic.P256()) if err != nil { return role, fmt.Errorf("invalid public key: %w", err) } From dce5924a89dd5058bf3b1c3edf78413b7a5ffce8 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 25 Oct 2022 15:24:06 +0300 Subject: [PATCH 0148/1943] [#229] services/tree: Use bearer owner as signer Signed-off-by: Denis Kirillov --- go.mod | 2 +- go.sum | 4 +-- pkg/services/tree/signature.go | 4 ++- pkg/services/tree/signature_test.go | 40 ++++++++++++++++++++++++++--- 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 8cbb4ba38..d54770ba2 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418075216-d0c5d837d204 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418145405-db5b89496d68 git.frostfs.info/TrueCloudLab/hrw v1.2.0 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index a1616a7b5..7b140a44d 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02f git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418075216-d0c5d837d204 h1:oQk6Fns+51JPtawUR5cJyYPQ35yC8Gi6e6P/PKkbvIc= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418075216-d0c5d837d204/go.mod h1:qbeYz8Z/3fZ0M0jiJY/zycuXB3DQ/8xQL5xU2G78akQ= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418145405-db5b89496d68 h1:m9HLrwRINt00cSQ07hKTPExOdAmmfO8m/3iGelnTo2o= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418145405-db5b89496d68/go.mod h1:TaJJOF3Uhuq8aqv2CrfuY2yhxePUinW35Xd3wfXLV/I= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 4aacbc3b1..439912969 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -101,6 +101,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op } var tb eacl.Table + signer := req.GetSignature().GetKey() if tableFromBearer { if bt.Impersonate() { tbCore, err := s.eaclSource.GetEACL(cid) @@ -108,6 +109,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op return handleGetEACLError(err) } tb = *tbCore.Value + signer = bt.SigningKeyBytes() } else { if !bearer.ResolveIssuer(*bt).Equals(cnr.Value.Owner()) { return eACLErr(eaclOp, errBearerWrongOwner) @@ -123,7 +125,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op tb = *tbCore.Value } - return checkEACL(tb, req.GetSignature().GetKey(), eACLRole(role), eaclOp) + return checkEACL(tb, signer, eACLRole(role), eaclOp) } func handleGetEACLError(err error) error { diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index b336e60a2..eaf9b8b79 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -53,6 +53,16 @@ func (s dummyContainerSource) Get(id cid.ID) (*containercore.Container, error) { return cnt, nil } +type dummyEACLSource map[string]*containercore.EACL + +func (s dummyEACLSource) GetEACL(id cid.ID) (*containercore.EACL, error) { + cntEACL, ok := s[id.String()] + if !ok { + return nil, errors.New("container not found") + } + return cntEACL, nil +} + func testContainer(owner user.ID) container.Container { var r netmapSDK.ReplicaDescriptor r.SetNumberOfObjects(1) @@ -93,6 +103,11 @@ func TestMessageSign(t *testing.T) { cnrSource: dummyContainerSource{ cid1.String(): cnr, }, + eaclSource: dummyEACLSource{ + cid1.String(): &containercore.EACL{ + Value: testTable(cid1, privs[0].PublicKey(), privs[1].PublicKey()), + }, + }, }, } @@ -178,6 +193,19 @@ func TestMessageSign(t *testing.T) { require.Error(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) + t.Run("impersonate", func(t *testing.T) { + cnr.Value.SetBasicACL(acl.PublicRWExtended) + var bt bearer.Token + bt.SetImpersonate(true) + + require.NoError(t, bt.Sign(privs[1].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + + require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) + require.Error(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + }) + bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) require.NoError(t, bt.Sign(privs[0].PrivateKey)) req.Body.BearerToken = bt.Marshal() @@ -202,6 +230,13 @@ func TestMessageSign(t *testing.T) { } func testBearerToken(cid cid.ID, forPutGet, forGet *keys.PublicKey) bearer.Token { + var b bearer.Token + b.SetEACLTable(*testTable(cid, forPutGet, forGet)) + + return b +} + +func testTable(cid cid.ID, forPutGet, forGet *keys.PublicKey) *eaclSDK.Table { tgtGet := eaclSDK.NewTarget() tgtGet.SetRole(eaclSDK.RoleUnknown) tgtGet.SetBinaryKeys([][]byte{forPutGet.Bytes(), forGet.Bytes()}) @@ -237,8 +272,5 @@ func testBearerToken(cid cid.ID, forPutGet, forGet *keys.PublicKey) bearer.Token tb.SetCID(cid) - var b bearer.Token - b.SetEACLTable(*tb) - - return b + return tb } From 700f39c3f8c6073dece256779f73a7149ca7f04e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 21 Apr 2023 16:57:34 +0300 Subject: [PATCH 0149/1943] [#229] Update CHANGELOG.md Signed-off-by: Denis Kirillov --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c2fe6811..e6792c5b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Changelog for FrostFS Node ## [Unreleased] ### Added +- Support impersonate bearer token (#229) ### Changed ### Fixed - Take network settings into account during netmap contract update (#100) From 299b24b9742756fa2ee65d3dcbdf65e4435673dd Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 11 Apr 2023 18:06:34 +0300 Subject: [PATCH 0150/1943] [#166] node: Parallelize background tree service sync by batching * Concurrently dispatch TreeApply operations for batching in forest Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/services/tree/sync.go | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 6c4f585a6..868d6ea4f 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -204,6 +204,11 @@ func (s *Service) synchronizeSingle(ctx context.Context, cid cid.ID, treeID stri rawCID := make([]byte, sha256.Size) cid.Encode(rawCID) + const treeApplyWorkersCount = 1024 + errGroup, egCtx := errgroup.WithContext(ctx) + errGroup.SetLimit(treeApplyWorkersCount) + var heightMtx sync.Mutex + for { newHeight := height req := &GetOpLogRequest{ @@ -217,7 +222,7 @@ func (s *Service) synchronizeSingle(ctx context.Context, cid cid.ID, treeID stri return newHeight, err } - c, err := treeClient.GetOpLog(ctx, req) + c, err := treeClient.GetOpLog(egCtx, req) if err != nil { return newHeight, fmt.Errorf("can't initialize client: %w", err) } @@ -230,16 +235,25 @@ func (s *Service) synchronizeSingle(ctx context.Context, cid cid.ID, treeID stri Child: lm.ChildId, } if err := m.Meta.FromBytes(lm.Meta); err != nil { + _ = errGroup.Wait() return newHeight, err } - if err := s.forest.TreeApply(ctx, cid, treeID, m, true); err != nil { - return newHeight, err - } - if m.Time > newHeight { - newHeight = m.Time + 1 - } else { - newHeight++ - } + errGroup.Go(func() error { + if err := s.forest.TreeApply(egCtx, cid, treeID, m, true); err != nil { + return err + } + heightMtx.Lock() + if m.Time > newHeight { + newHeight = m.Time + 1 + } else { + newHeight++ + } + heightMtx.Unlock() + return nil + }) + } + if errGroupErr := errGroup.Wait(); errGroupErr != nil { + return newHeight, err } if height == newHeight || err != nil && !errors.Is(err, io.EOF) { return newHeight, err From 9d010297331e490a41a55b4d782f385d40fdd9b5 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 20 Apr 2023 15:58:40 +0300 Subject: [PATCH 0151/1943] [#166] node: Parallelize background tree service sync by batching * Merge operations Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/services/tree/sync.go | 235 ++++++++++++++++++++++----------- pkg/services/tree/sync_test.go | 85 ++++++++++++ 2 files changed, 240 insertions(+), 80 deletions(-) create mode 100644 pkg/services/tree/sync_test.go diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 868d6ea4f..5c0300c14 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -125,6 +125,137 @@ func (s *Service) SynchronizeTree(ctx context.Context, cid cid.ID, treeID string return nil } +// mergeOperationStreams performs merge sort for node operation streams to one stream. +func mergeOperationStreams(ctx context.Context, streams []chan *pilorama.Move, merged chan<- *pilorama.Move) uint64 { + defer close(merged) + + ms := make([]*pilorama.Move, len(streams)) + for i := range streams { + ms[i] = <-streams[i] + } + + // Merging different node streams shuffles incoming operations like that: + // + // x - operation from the stream A + // o - operation from the stream B + // + // --o---o--x--x--x--o---x--x------> t + // ^ + // If all ops have been successfully applied, we must start from the last + // operation height from the stream B. This height is stored in minStreamedLastHeight. + var minStreamedLastHeight uint64 = math.MaxUint64 + + for { + var minTimeMoveTime uint64 = math.MaxUint64 + minTimeMoveIndex := -1 + for i, m := range ms { + if m != nil && minTimeMoveTime > m.Time { + minTimeMoveTime = m.Time + minTimeMoveIndex = i + } + } + + if minTimeMoveIndex == -1 { + break + } + + merged <- ms[minTimeMoveIndex] + height := ms[minTimeMoveIndex].Time + if ms[minTimeMoveIndex] = <-streams[minTimeMoveIndex]; ms[minTimeMoveIndex] == nil { + if minStreamedLastHeight > height { + minStreamedLastHeight = height + } + } + } + + return minStreamedLastHeight +} + +func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID string, + operationStream <-chan *pilorama.Move) uint64 { + errGroup, _ := errgroup.WithContext(ctx) + const workersCount = 1024 + errGroup.SetLimit(workersCount) + + // We run TreeApply concurrently for the operation batch. Let's consider two operations + // in the batch m1 and m2 such that m1.Time < m2.Time. The engine may apply m2 and fail + // on m1. That means the service must start sync from m1.Time in the next iteration and + // this height is stored in unappliedOperationHeight. + var unappliedOperationHeight uint64 = math.MaxUint64 + var heightMtx sync.Mutex + + var prev *pilorama.Move + for m := range operationStream { + m := m + + // skip already applied op + if prev != nil && prev.Time == m.Time { + continue + } + prev = m + + errGroup.Go(func() error { + if err := s.forest.TreeApply(ctx, cid, treeID, m, true); err != nil { + heightMtx.Lock() + if m.Time < unappliedOperationHeight { + unappliedOperationHeight = m.Time + } + heightMtx.Unlock() + return err + } + return nil + }) + } + _ = errGroup.Wait() + return unappliedOperationHeight +} + +func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, + height uint64, treeClient TreeServiceClient, opsCh chan<- *pilorama.Move) (uint64, error) { + rawCID := make([]byte, sha256.Size) + cid.Encode(rawCID) + + for { + newHeight := height + req := &GetOpLogRequest{ + Body: &GetOpLogRequest_Body{ + ContainerId: rawCID, + TreeId: treeID, + Height: newHeight, + }, + } + if err := SignMessage(req, s.key); err != nil { + return 0, err + } + + c, err := treeClient.GetOpLog(ctx, req) + if err != nil { + return 0, fmt.Errorf("can't initialize client: %w", err) + } + res, err := c.Recv() + for ; err == nil; res, err = c.Recv() { + lm := res.GetBody().GetOperation() + m := &pilorama.Move{ + Parent: lm.ParentId, + Child: lm.ChildId, + } + if err := m.Meta.FromBytes(lm.Meta); err != nil { + return 0, err + } + opsCh <- m + } + if height == newHeight || err != nil && !errors.Is(err, io.EOF) { + return newHeight, err + } + height = newHeight + } +} + +// synchronizeTree synchronizes operations getting them from different nodes. +// Each available node does stream operations to a separate stream. These streams +// are merged into one big stream ordered by operation time. This way allows to skip +// already applied operation and keep good batching. +// The method returns a height that service should start sync from in the next time. func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, treeID string, nodes []netmapSDK.NodeInfo) uint64 { s.log.Debug(logs.TreeSynchronizeTree, @@ -133,10 +264,25 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, zap.Uint64("from", from)) errGroup, egCtx := errgroup.WithContext(ctx) - const workersCount = 4 + const workersCount = 1024 errGroup.SetLimit(workersCount) - heights := make([]uint64, len(nodes)) + nodeOperationStreams := make([]chan *pilorama.Move, len(nodes)) + for i := range nodeOperationStreams { + nodeOperationStreams[i] = make(chan *pilorama.Move) + } + merged := make(chan *pilorama.Move) + var minStreamedLastHeight uint64 + errGroup.Go(func() error { + minStreamedLastHeight = mergeOperationStreams(ctx, nodeOperationStreams, merged) + return nil + }) + var minUnappliedHeight uint64 + errGroup.Go(func() error { + minUnappliedHeight = s.applyOperationStream(ctx, cid, treeID, merged) + return nil + }) + for i, n := range nodes { i := i n := n @@ -164,7 +310,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, treeClient := NewTreeServiceClient(cc) for { - h, err := s.synchronizeSingle(egCtx, cid, treeID, height, treeClient) + h, err := s.startStream(egCtx, cid, treeID, from, treeClient, nodeOperationStreams[i]) if height < h { height = h } @@ -174,94 +320,23 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, } } }) - - if height <= from { // do not increase starting height on fail - heights[i] = from - return nil - } - heights[i] = height + close(nodeOperationStreams[i]) return nil }) } - if err := errGroup.Wait(); err != nil { s.log.Warn(logs.TreeFailedToRunTreeSynchronizationOverAllNodes, zap.Error(err)) } - newHeight := uint64(math.MaxUint64) - for _, height := range heights { // take minimum across all clients - if height < newHeight { - newHeight = height - } - } - if newHeight == math.MaxUint64 { - newHeight = from + newHeight := minStreamedLastHeight + if newHeight > minUnappliedHeight { + newHeight = minUnappliedHeight + } else { + newHeight++ } return newHeight } -func (s *Service) synchronizeSingle(ctx context.Context, cid cid.ID, treeID string, height uint64, treeClient TreeServiceClient) (uint64, error) { - rawCID := make([]byte, sha256.Size) - cid.Encode(rawCID) - - const treeApplyWorkersCount = 1024 - errGroup, egCtx := errgroup.WithContext(ctx) - errGroup.SetLimit(treeApplyWorkersCount) - var heightMtx sync.Mutex - - for { - newHeight := height - req := &GetOpLogRequest{ - Body: &GetOpLogRequest_Body{ - ContainerId: rawCID, - TreeId: treeID, - Height: newHeight, - }, - } - if err := SignMessage(req, s.key); err != nil { - return newHeight, err - } - - c, err := treeClient.GetOpLog(egCtx, req) - if err != nil { - return newHeight, fmt.Errorf("can't initialize client: %w", err) - } - - res, err := c.Recv() - for ; err == nil; res, err = c.Recv() { - lm := res.GetBody().GetOperation() - m := &pilorama.Move{ - Parent: lm.ParentId, - Child: lm.ChildId, - } - if err := m.Meta.FromBytes(lm.Meta); err != nil { - _ = errGroup.Wait() - return newHeight, err - } - errGroup.Go(func() error { - if err := s.forest.TreeApply(egCtx, cid, treeID, m, true); err != nil { - return err - } - heightMtx.Lock() - if m.Time > newHeight { - newHeight = m.Time + 1 - } else { - newHeight++ - } - heightMtx.Unlock() - return nil - }) - } - if errGroupErr := errGroup.Wait(); errGroupErr != nil { - return newHeight, err - } - if height == newHeight || err != nil && !errors.Is(err, io.EOF) { - return newHeight, err - } - height = newHeight - } -} - // ErrAlreadySyncing is returned when a service synchronization has already // been started. var ErrAlreadySyncing = errors.New("service is being synchronized") diff --git a/pkg/services/tree/sync_test.go b/pkg/services/tree/sync_test.go new file mode 100644 index 000000000..eb51bc3f8 --- /dev/null +++ b/pkg/services/tree/sync_test.go @@ -0,0 +1,85 @@ +package tree + +import ( + "context" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "github.com/stretchr/testify/require" +) + +func Test_mergeOperationStreams(t *testing.T) { + tests := []struct { + name string + ctx context.Context + opTimes [][]uint64 + wantValues []uint64 + wantMinHeight uint64 + }{ + { + name: "1", + ctx: context.Background(), + opTimes: [][]uint64{ + {250, 251, 255}, + {252, 253, 254, 256, 257}, + }, + wantValues: []uint64{250, 251, 252, 253, 254, 255, 256, 257}, + wantMinHeight: 255, + }, + { + name: "2", + ctx: context.Background(), + opTimes: [][]uint64{ + {250, 251, 255, 259}, + {252, 253, 254, 256, 257}, + }, + wantValues: []uint64{250, 251, 252, 253, 254, 255, 256, 257, 259}, + wantMinHeight: 257, + }, + { + name: "3", + ctx: context.Background(), + opTimes: [][]uint64{ + {250, 251, 255}, + {249, 250, 251, 253, 254, 256, 257}, + }, + wantValues: []uint64{249, 250, 250, 251, 251, 253, 254, 255, 256, 257}, + wantMinHeight: 255, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nodeOpChans := make([]chan *pilorama.Move, len(tt.opTimes)) + for i := range nodeOpChans { + nodeOpChans[i] = make(chan *pilorama.Move) + } + + // generate and put values to all chans + for i, ch := range nodeOpChans { + i := i + ch := ch + go func() { + for _, tm := range tt.opTimes[i] { + op := &pilorama.Move{} + op.Time = tm + ch <- op + } + close(nodeOpChans[i]) + }() + } + + merged := make(chan *pilorama.Move, 1) + min := make(chan uint64) + go func() { + min <- mergeOperationStreams(tt.ctx, nodeOpChans, merged) + }() + + var res []uint64 + for op := range merged { + res = append(res, op.Time) + } + require.Equal(t, tt.wantValues, res) + require.Equal(t, tt.wantMinHeight, <-min) + }) + } +} From ee01275d25fe5b703625ea5684bfcf9b348a8d96 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 20 Apr 2023 11:38:10 +0300 Subject: [PATCH 0152/1943] [#125] node: Remove redundant `env` from `config/internal` Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config/calls_test.go | 6 ++++-- cmd/frostfs-node/config/config.go | 16 ++++++++++---- cmd/frostfs-node/config/internal/env.go | 22 -------------------- cmd/frostfs-node/config/internal/env_test.go | 15 ------------- 4 files changed, 16 insertions(+), 43 deletions(-) delete mode 100644 cmd/frostfs-node/config/internal/env.go delete mode 100644 cmd/frostfs-node/config/internal/env_test.go diff --git a/cmd/frostfs-node/config/calls_test.go b/cmd/frostfs-node/config/calls_test.go index 37ffd2e88..9b283c8f6 100644 --- a/cmd/frostfs-node/config/calls_test.go +++ b/cmd/frostfs-node/config/calls_test.go @@ -2,10 +2,10 @@ package config_test import ( "os" + "strings" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/internal" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" "github.com/stretchr/testify/require" ) @@ -35,7 +35,9 @@ func TestConfigEnv(t *testing.T) { value = "some value" ) - err := os.Setenv(internal.Env(section, name), value) + envName := strings.ToUpper( + strings.Join([]string{config.EnvPrefix, section, name}, config.EnvSeparator)) + err := os.Setenv(envName, value) require.NoError(t, err) c := configtest.EmptyConfig() diff --git a/cmd/frostfs-node/config/config.go b/cmd/frostfs-node/config/config.go index 12c92306b..dba3d12e7 100644 --- a/cmd/frostfs-node/config/config.go +++ b/cmd/frostfs-node/config/config.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config" "github.com/spf13/viper" ) @@ -24,7 +23,16 @@ type Config struct { path []string } -const separator = "." +const ( + separator = "." + + // EnvPrefix is a prefix of ENV variables related + // to storage node configuration. + EnvPrefix = "FROSTFS" + + // EnvSeparator is a section separator in ENV variables. + EnvSeparator = "_" +) // Prm groups required parameters of the Config. type Prm struct{} @@ -37,9 +45,9 @@ type Prm struct{} func New(_ Prm, opts ...Option) *Config { v := viper.New() - v.SetEnvPrefix(internal.EnvPrefix) + v.SetEnvPrefix(EnvPrefix) v.AutomaticEnv() - v.SetEnvKeyReplacer(strings.NewReplacer(separator, internal.EnvSeparator)) + v.SetEnvKeyReplacer(strings.NewReplacer(separator, EnvSeparator)) o := defaultOpts() for i := range opts { diff --git a/cmd/frostfs-node/config/internal/env.go b/cmd/frostfs-node/config/internal/env.go deleted file mode 100644 index 6bf260ac1..000000000 --- a/cmd/frostfs-node/config/internal/env.go +++ /dev/null @@ -1,22 +0,0 @@ -package internal - -import ( - "strings" -) - -// EnvPrefix is a prefix of ENV variables related -// to storage node configuration. -const EnvPrefix = "FROSTFS" - -// EnvSeparator is a section separator in ENV variables. -const EnvSeparator = "_" - -// Env returns ENV variable key for a particular config parameter. -func Env(path ...string) string { - return strings.ToUpper( - strings.Join( - append([]string{EnvPrefix}, path...), - EnvSeparator, - ), - ) -} diff --git a/cmd/frostfs-node/config/internal/env_test.go b/cmd/frostfs-node/config/internal/env_test.go deleted file mode 100644 index 3d1af0715..000000000 --- a/cmd/frostfs-node/config/internal/env_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package internal_test - -import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/internal" - "github.com/stretchr/testify/require" -) - -func TestEnv(t *testing.T) { - require.Equal(t, - "FROSTFS_SECTION_PARAMETER", - internal.Env("section", "parameter"), - ) -} From b2123bfd1a0487d41d9b1d0868242d2d6f4e36dd Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 20 Apr 2023 12:17:03 +0300 Subject: [PATCH 0153/1943] [#125] node: Add new option `WithEnvPrefix` Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config/config.go | 10 ++++++---- cmd/frostfs-node/config/opts.go | 9 +++++++++ cmd/frostfs-node/config/test/config.go | 4 ++-- cmd/frostfs-node/main.go | 4 +++- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-node/config/config.go b/cmd/frostfs-node/config/config.go index dba3d12e7..cee6a1ea6 100644 --- a/cmd/frostfs-node/config/config.go +++ b/cmd/frostfs-node/config/config.go @@ -45,15 +45,17 @@ type Prm struct{} func New(_ Prm, opts ...Option) *Config { v := viper.New() - v.SetEnvPrefix(EnvPrefix) - v.AutomaticEnv() - v.SetEnvKeyReplacer(strings.NewReplacer(separator, EnvSeparator)) - o := defaultOpts() for i := range opts { opts[i](o) } + if o.envPrefix != "" { + v.SetEnvPrefix(o.envPrefix) + v.AutomaticEnv() + v.SetEnvKeyReplacer(strings.NewReplacer(separator, EnvSeparator)) + } + if o.path != "" { v.SetConfigFile(o.path) diff --git a/cmd/frostfs-node/config/opts.go b/cmd/frostfs-node/config/opts.go index 009c6efef..f49e1db69 100644 --- a/cmd/frostfs-node/config/opts.go +++ b/cmd/frostfs-node/config/opts.go @@ -3,6 +3,7 @@ package config type opts struct { path string configDir string + envPrefix string } func defaultOpts() *opts { @@ -27,3 +28,11 @@ func WithConfigDir(path string) Option { o.configDir = path } } + +// WithEnvPrefix returns an option to defines +// a prefix that ENVIRONMENT variables will use. +func WithEnvPrefix(envPrefix string) Option { + return func(o *opts) { + o.envPrefix = envPrefix + } +} diff --git a/cmd/frostfs-node/config/test/config.go b/cmd/frostfs-node/config/test/config.go index cd10edaec..808a481f9 100644 --- a/cmd/frostfs-node/config/test/config.go +++ b/cmd/frostfs-node/config/test/config.go @@ -25,7 +25,7 @@ func fromEnvFile(t testing.TB, path string) *config.Config { loadEnv(t, path) // github.com/joho/godotenv can do that as well - return config.New(p) + return config.New(p, config.WithEnvPrefix(config.EnvPrefix)) } func forEachFile(paths []string, f func(*config.Config)) { @@ -53,7 +53,7 @@ func ForEnvFileType(t testing.TB, pref string, f func(*config.Config)) { func EmptyConfig() *config.Config { var p config.Prm - return config.New(p) + return config.New(p, config.WithEnvPrefix(config.EnvPrefix)) } // loadEnv reads .env file, parses `X=Y` records and sets OS ENVs. diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index a6ee52464..b8bec87b5 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -46,7 +46,9 @@ func main() { os.Exit(SuccessReturnCode) } - appCfg := config.New(config.Prm{}, config.WithConfigFile(*configFile), config.WithConfigDir(*configDir)) + appCfg := config.New(config.Prm{}, + config.WithConfigFile(*configFile), config.WithConfigDir(*configDir), + config.WithEnvPrefix(config.EnvPrefix)) err := validateConfig(appCfg) fatalOnErr(err) From d390f093e06f7153813c2adedde21c5cb1f7dd6b Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 20 Apr 2023 12:44:39 +0300 Subject: [PATCH 0154/1943] [#125] node: Move viper creation to `internal/common/config` Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config/calls.go | 6 +- cmd/frostfs-node/config/calls_test.go | 3 +- cmd/frostfs-node/config/config.go | 57 ++-------------- cmd/frostfs-node/config/configdir_test.go | 3 +- cmd/frostfs-node/config/test/config.go | 7 +- cmd/frostfs-node/main.go | 5 +- cmd/frostfs-node/validate_test.go | 5 +- .../common}/config/opts.go | 26 ++++---- cmd/internal/common/config/viper.go | 65 +++++++++++++++++++ 9 files changed, 101 insertions(+), 76 deletions(-) rename cmd/{frostfs-node => internal/common}/config/opts.go (65%) create mode 100644 cmd/internal/common/config/viper.go diff --git a/cmd/frostfs-node/config/calls.go b/cmd/frostfs-node/config/calls.go index 6cf3baa03..36e53ea7c 100644 --- a/cmd/frostfs-node/config/calls.go +++ b/cmd/frostfs-node/config/calls.go @@ -2,6 +2,8 @@ package config import ( "strings" + + configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" ) // Sub returns a subsection of the Config by name. @@ -38,11 +40,11 @@ func (x *Config) Sub(name string) *Config { // // Returns nil if config is nil. func (x *Config) Value(name string) any { - value := x.v.Get(strings.Join(append(x.path, name), separator)) + value := x.v.Get(strings.Join(append(x.path, name), configViper.Separator)) if value != nil || x.defaultPath == nil { return value } - return x.v.Get(strings.Join(append(x.defaultPath, name), separator)) + return x.v.Get(strings.Join(append(x.defaultPath, name), configViper.Separator)) } // SetDefault sets fallback config for missing values. diff --git a/cmd/frostfs-node/config/calls_test.go b/cmd/frostfs-node/config/calls_test.go index 9b283c8f6..68bf1c679 100644 --- a/cmd/frostfs-node/config/calls_test.go +++ b/cmd/frostfs-node/config/calls_test.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "github.com/stretchr/testify/require" ) @@ -36,7 +37,7 @@ func TestConfigEnv(t *testing.T) { ) envName := strings.ToUpper( - strings.Join([]string{config.EnvPrefix, section, name}, config.EnvSeparator)) + strings.Join([]string{config.EnvPrefix, section, name}, configViper.EnvSeparator)) err := os.Setenv(envName, value) require.NoError(t, err) diff --git a/cmd/frostfs-node/config/config.go b/cmd/frostfs-node/config/config.go index cee6a1ea6..cf8ca0a4d 100644 --- a/cmd/frostfs-node/config/config.go +++ b/cmd/frostfs-node/config/config.go @@ -1,10 +1,7 @@ package config import ( - "fmt" - "strings" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config" + configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "github.com/spf13/viper" ) @@ -17,21 +14,16 @@ import ( type Config struct { v *viper.Viper - opts opts + opts configViper.Opts defaultPath []string path []string } const ( - separator = "." - // EnvPrefix is a prefix of ENV variables related // to storage node configuration. EnvPrefix = "FROSTFS" - - // EnvSeparator is a section separator in ENV variables. - EnvSeparator = "_" ) // Prm groups required parameters of the Config. @@ -42,34 +34,8 @@ type Prm struct{} // If file option is provided (WithConfigFile), // configuration values are read from it. // Otherwise, Config is a degenerate tree. -func New(_ Prm, opts ...Option) *Config { - v := viper.New() - - o := defaultOpts() - for i := range opts { - opts[i](o) - } - - if o.envPrefix != "" { - v.SetEnvPrefix(o.envPrefix) - v.AutomaticEnv() - v.SetEnvKeyReplacer(strings.NewReplacer(separator, EnvSeparator)) - } - - if o.path != "" { - v.SetConfigFile(o.path) - - err := v.ReadInConfig() - if err != nil { - panic(fmt.Errorf("failed to read config: %w", err)) - } - } - - if o.configDir != "" { - if err := config.ReadConfigDir(v, o.configDir); err != nil { - panic(fmt.Errorf("failed to read config dir: %w", err)) - } - } +func New(_ Prm, opts ...configViper.Option) *Config { + v, o := configViper.CreateViper(opts...) return &Config{ v: v, @@ -79,18 +45,5 @@ func New(_ Prm, opts ...Option) *Config { // Reload reads configuration path if it was provided to New. func (x *Config) Reload() error { - if x.opts.path != "" { - err := x.v.ReadInConfig() - if err != nil { - return fmt.Errorf("rereading configuration file: %w", err) - } - } - - if x.opts.configDir != "" { - if err := config.ReadConfigDir(x.v, x.opts.configDir); err != nil { - return fmt.Errorf("rereading configuration dir: %w", err) - } - } - - return nil + return configViper.ReloadViper(x.v, x.opts) } diff --git a/cmd/frostfs-node/config/configdir_test.go b/cmd/frostfs-node/config/configdir_test.go index 2c3255670..2e2e831a1 100644 --- a/cmd/frostfs-node/config/configdir_test.go +++ b/cmd/frostfs-node/config/configdir_test.go @@ -5,6 +5,7 @@ import ( "path" "testing" + configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "github.com/spf13/cast" "github.com/stretchr/testify/require" ) @@ -18,7 +19,7 @@ func TestConfigDir(t *testing.T) { require.NoError(t, os.WriteFile(cfgFileName0, []byte(`{"storage":{"shard_pool_size":15}}`), 0777)) require.NoError(t, os.WriteFile(cfgFileName1, []byte("logger:\n level: debug"), 0777)) - c := New(Prm{}, WithConfigDir(dir)) + c := New(Prm{}, configViper.WithConfigDir(dir)) require.Equal(t, "debug", cast.ToString(c.Sub("logger").Value("level"))) require.EqualValues(t, 15, cast.ToUint32(c.Sub("storage").Value("shard_pool_size"))) } diff --git a/cmd/frostfs-node/config/test/config.go b/cmd/frostfs-node/config/test/config.go index 808a481f9..b3f659030 100644 --- a/cmd/frostfs-node/config/test/config.go +++ b/cmd/frostfs-node/config/test/config.go @@ -7,6 +7,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "github.com/stretchr/testify/require" ) @@ -16,7 +17,7 @@ func fromFile(path string) *config.Config { os.Clearenv() // ENVs have priority over config files, so we do this in tests return config.New(p, - config.WithConfigFile(path), + configViper.WithConfigFile(path), ) } @@ -25,7 +26,7 @@ func fromEnvFile(t testing.TB, path string) *config.Config { loadEnv(t, path) // github.com/joho/godotenv can do that as well - return config.New(p, config.WithEnvPrefix(config.EnvPrefix)) + return config.New(p, configViper.WithEnvPrefix(config.EnvPrefix)) } func forEachFile(paths []string, f func(*config.Config)) { @@ -53,7 +54,7 @@ func ForEnvFileType(t testing.TB, pref string, f func(*config.Config)) { func EmptyConfig() *config.Config { var p config.Prm - return config.New(p, config.WithEnvPrefix(config.EnvPrefix)) + return config.New(p, configViper.WithEnvPrefix(config.EnvPrefix)) } // loadEnv reads .env file, parses `X=Y` records and sets OS ENVs. diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index b8bec87b5..877a80f6a 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -8,6 +8,7 @@ import ( "os" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" @@ -47,8 +48,8 @@ func main() { } appCfg := config.New(config.Prm{}, - config.WithConfigFile(*configFile), config.WithConfigDir(*configDir), - config.WithEnvPrefix(config.EnvPrefix)) + configViper.WithConfigFile(*configFile), configViper.WithConfigDir(*configDir), + configViper.WithEnvPrefix(config.EnvPrefix)) err := validateConfig(appCfg) fatalOnErr(err) diff --git a/cmd/frostfs-node/validate_test.go b/cmd/frostfs-node/validate_test.go index e35fc1afd..74903884b 100644 --- a/cmd/frostfs-node/validate_test.go +++ b/cmd/frostfs-node/validate_test.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "github.com/stretchr/testify/require" ) @@ -26,13 +27,13 @@ func TestValidate(t *testing.T) { t.Run("mainnet", func(t *testing.T) { os.Clearenv() // ENVs have priority over config files, so we do this in tests p := filepath.Join(exampleConfigPrefix, "mainnet/config.yml") - c := config.New(config.Prm{}, config.WithConfigFile(p)) + c := config.New(config.Prm{}, configViper.WithConfigFile(p)) require.NoError(t, validateConfig(c)) }) t.Run("testnet", func(t *testing.T) { os.Clearenv() // ENVs have priority over config files, so we do this in tests p := filepath.Join(exampleConfigPrefix, "testnet/config.yml") - c := config.New(config.Prm{}, config.WithConfigFile(p)) + c := config.New(config.Prm{}, configViper.WithConfigFile(p)) require.NoError(t, validateConfig(c)) }) } diff --git a/cmd/frostfs-node/config/opts.go b/cmd/internal/common/config/opts.go similarity index 65% rename from cmd/frostfs-node/config/opts.go rename to cmd/internal/common/config/opts.go index f49e1db69..a28b2b522 100644 --- a/cmd/frostfs-node/config/opts.go +++ b/cmd/internal/common/config/opts.go @@ -1,38 +1,38 @@ package config -type opts struct { - path string - configDir string - envPrefix string +type Opts struct { + Path string + ConfigDir string + EnvPrefix string } -func defaultOpts() *opts { - return new(opts) +func DefaultOpts() *Opts { + return new(Opts) } // Option allows to set an optional parameter of the Config. -type Option func(*opts) +type Option func(*Opts) // WithConfigFile returns an option to set the system path // to the configuration file. func WithConfigFile(path string) Option { - return func(o *opts) { - o.path = path + return func(o *Opts) { + o.Path = path } } // WithConfigDir returns an option to set the system path // to the directory with configuration files. func WithConfigDir(path string) Option { - return func(o *opts) { - o.configDir = path + return func(o *Opts) { + o.ConfigDir = path } } // WithEnvPrefix returns an option to defines // a prefix that ENVIRONMENT variables will use. func WithEnvPrefix(envPrefix string) Option { - return func(o *opts) { - o.envPrefix = envPrefix + return func(o *Opts) { + o.EnvPrefix = envPrefix } } diff --git a/cmd/internal/common/config/viper.go b/cmd/internal/common/config/viper.go new file mode 100644 index 000000000..63e8ba05c --- /dev/null +++ b/cmd/internal/common/config/viper.go @@ -0,0 +1,65 @@ +package config + +import ( + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config" + "github.com/spf13/viper" +) + +const ( + Separator = "." + + // EnvSeparator is a section separator in ENV variables. + EnvSeparator = "_" +) + +func CreateViper(opts ...Option) (*viper.Viper, *Opts) { + v := viper.New() + + o := DefaultOpts() + for i := range opts { + opts[i](o) + } + + if o.EnvPrefix != "" { + v.SetEnvPrefix(o.EnvPrefix) + v.AutomaticEnv() + v.SetEnvKeyReplacer(strings.NewReplacer(Separator, EnvSeparator)) + } + + if o.Path != "" { + v.SetConfigFile(o.Path) + + err := v.ReadInConfig() + if err != nil { + panic(fmt.Errorf("failed to read config: %w", err)) + } + } + + if o.ConfigDir != "" { + if err := config.ReadConfigDir(v, o.ConfigDir); err != nil { + panic(fmt.Errorf("failed to read config dir: %w", err)) + } + } + + return v, o +} + +func ReloadViper(v *viper.Viper, o Opts) error { + if o.Path != "" { + err := v.ReadInConfig() + if err != nil { + return fmt.Errorf("rereading configuration file: %w", err) + } + } + + if o.ConfigDir != "" { + if err := config.ReadConfigDir(v, o.ConfigDir); err != nil { + return fmt.Errorf("rereading configuration dir: %w", err) + } + } + + return nil +} From eb7be82e87d9da5bdeaae1b6c598f1056dc06d54 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 24 Apr 2023 10:09:04 +0300 Subject: [PATCH 0155/1943] [#125] node: Avoid panic when reading config Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config/config.go | 6 +++++- cmd/internal/common/config/viper.go | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-node/config/config.go b/cmd/frostfs-node/config/config.go index cf8ca0a4d..e1b142aed 100644 --- a/cmd/frostfs-node/config/config.go +++ b/cmd/frostfs-node/config/config.go @@ -35,7 +35,11 @@ type Prm struct{} // configuration values are read from it. // Otherwise, Config is a degenerate tree. func New(_ Prm, opts ...configViper.Option) *Config { - v, o := configViper.CreateViper(opts...) + v, o, err := configViper.CreateViper(opts...) + + if err != nil { + panic(err) + } return &Config{ v: v, diff --git a/cmd/internal/common/config/viper.go b/cmd/internal/common/config/viper.go index 63e8ba05c..b0af9ae28 100644 --- a/cmd/internal/common/config/viper.go +++ b/cmd/internal/common/config/viper.go @@ -15,7 +15,7 @@ const ( EnvSeparator = "_" ) -func CreateViper(opts ...Option) (*viper.Viper, *Opts) { +func CreateViper(opts ...Option) (*viper.Viper, *Opts, error) { v := viper.New() o := DefaultOpts() @@ -34,17 +34,17 @@ func CreateViper(opts ...Option) (*viper.Viper, *Opts) { err := v.ReadInConfig() if err != nil { - panic(fmt.Errorf("failed to read config: %w", err)) + return nil, nil, fmt.Errorf("failed to read config: %w", err) } } if o.ConfigDir != "" { if err := config.ReadConfigDir(v, o.ConfigDir); err != nil { - panic(fmt.Errorf("failed to read config dir: %w", err)) + return nil, nil, fmt.Errorf("failed to read config dir: %w", err) } } - return v, o + return v, o, nil } func ReloadViper(v *viper.Viper, o Opts) error { From e61aec4a7ddda2443a98657ff1b7bb705d2b2914 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 25 Apr 2023 08:55:42 +0300 Subject: [PATCH 0156/1943] [#125] node: Remove unused config.Prm Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config/config.go | 5 +---- cmd/frostfs-node/config/configdir_test.go | 2 +- cmd/frostfs-node/config/test/config.go | 12 +++--------- cmd/frostfs-node/main.go | 2 +- cmd/frostfs-node/validate_test.go | 4 ++-- 5 files changed, 8 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-node/config/config.go b/cmd/frostfs-node/config/config.go index e1b142aed..383897fc8 100644 --- a/cmd/frostfs-node/config/config.go +++ b/cmd/frostfs-node/config/config.go @@ -26,15 +26,12 @@ const ( EnvPrefix = "FROSTFS" ) -// Prm groups required parameters of the Config. -type Prm struct{} - // New creates a new Config instance. // // If file option is provided (WithConfigFile), // configuration values are read from it. // Otherwise, Config is a degenerate tree. -func New(_ Prm, opts ...configViper.Option) *Config { +func New(opts ...configViper.Option) *Config { v, o, err := configViper.CreateViper(opts...) if err != nil { diff --git a/cmd/frostfs-node/config/configdir_test.go b/cmd/frostfs-node/config/configdir_test.go index 2e2e831a1..f41e6edd9 100644 --- a/cmd/frostfs-node/config/configdir_test.go +++ b/cmd/frostfs-node/config/configdir_test.go @@ -19,7 +19,7 @@ func TestConfigDir(t *testing.T) { require.NoError(t, os.WriteFile(cfgFileName0, []byte(`{"storage":{"shard_pool_size":15}}`), 0777)) require.NoError(t, os.WriteFile(cfgFileName1, []byte("logger:\n level: debug"), 0777)) - c := New(Prm{}, configViper.WithConfigDir(dir)) + c := New(configViper.WithConfigDir(dir)) require.Equal(t, "debug", cast.ToString(c.Sub("logger").Value("level"))) require.EqualValues(t, 15, cast.ToUint32(c.Sub("storage").Value("shard_pool_size"))) } diff --git a/cmd/frostfs-node/config/test/config.go b/cmd/frostfs-node/config/test/config.go index b3f659030..2bd5ade21 100644 --- a/cmd/frostfs-node/config/test/config.go +++ b/cmd/frostfs-node/config/test/config.go @@ -12,21 +12,17 @@ import ( ) func fromFile(path string) *config.Config { - var p config.Prm - os.Clearenv() // ENVs have priority over config files, so we do this in tests - return config.New(p, + return config.New( configViper.WithConfigFile(path), ) } func fromEnvFile(t testing.TB, path string) *config.Config { - var p config.Prm - loadEnv(t, path) // github.com/joho/godotenv can do that as well - return config.New(p, configViper.WithEnvPrefix(config.EnvPrefix)) + return config.New(configViper.WithEnvPrefix(config.EnvPrefix)) } func forEachFile(paths []string, f func(*config.Config)) { @@ -52,9 +48,7 @@ func ForEnvFileType(t testing.TB, pref string, f func(*config.Config)) { // EmptyConfig returns config without any values and sections. func EmptyConfig() *config.Config { - var p config.Prm - - return config.New(p, configViper.WithEnvPrefix(config.EnvPrefix)) + return config.New(configViper.WithEnvPrefix(config.EnvPrefix)) } // loadEnv reads .env file, parses `X=Y` records and sets OS ENVs. diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 877a80f6a..5a66b210b 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -47,7 +47,7 @@ func main() { os.Exit(SuccessReturnCode) } - appCfg := config.New(config.Prm{}, + appCfg := config.New( configViper.WithConfigFile(*configFile), configViper.WithConfigDir(*configDir), configViper.WithEnvPrefix(config.EnvPrefix)) diff --git a/cmd/frostfs-node/validate_test.go b/cmd/frostfs-node/validate_test.go index 74903884b..75b694cec 100644 --- a/cmd/frostfs-node/validate_test.go +++ b/cmd/frostfs-node/validate_test.go @@ -27,13 +27,13 @@ func TestValidate(t *testing.T) { t.Run("mainnet", func(t *testing.T) { os.Clearenv() // ENVs have priority over config files, so we do this in tests p := filepath.Join(exampleConfigPrefix, "mainnet/config.yml") - c := config.New(config.Prm{}, configViper.WithConfigFile(p)) + c := config.New(configViper.WithConfigFile(p)) require.NoError(t, validateConfig(c)) }) t.Run("testnet", func(t *testing.T) { os.Clearenv() // ENVs have priority over config files, so we do this in tests p := filepath.Join(exampleConfigPrefix, "testnet/config.yml") - c := config.New(config.Prm{}, configViper.WithConfigFile(p)) + c := config.New(configViper.WithConfigFile(p)) require.NoError(t, validateConfig(c)) }) } From ef222e24876b36299e246c0b239768d45b2da6d1 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 25 Apr 2023 10:04:26 +0300 Subject: [PATCH 0157/1943] [#125] cmd: Refactor `internal/common/viper` Add `opts.WithViper`, set opts struct as private. Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config/config.go | 24 +++++++---- cmd/frostfs-node/config/configdir_test.go | 3 +- cmd/frostfs-node/config/test/config.go | 9 ++-- cmd/frostfs-node/main.go | 5 +-- cmd/frostfs-node/validate_test.go | 5 +-- cmd/internal/common/config/opts.go | 37 +++++++++++------ cmd/internal/common/config/viper.go | 50 +++++++++++++++-------- 7 files changed, 80 insertions(+), 53 deletions(-) diff --git a/cmd/frostfs-node/config/config.go b/cmd/frostfs-node/config/config.go index 383897fc8..77e34d613 100644 --- a/cmd/frostfs-node/config/config.go +++ b/cmd/frostfs-node/config/config.go @@ -14,7 +14,9 @@ import ( type Config struct { v *viper.Viper - opts configViper.Opts + configFile string + configDir string + envPrefix string defaultPath []string path []string @@ -28,23 +30,31 @@ const ( // New creates a new Config instance. // -// If file option is provided (WithConfigFile), +// If file option is provided, // configuration values are read from it. // Otherwise, Config is a degenerate tree. -func New(opts ...configViper.Option) *Config { - v, o, err := configViper.CreateViper(opts...) +func New(configFile, configDir, envPrefix string) *Config { + v, err := configViper.CreateViper( + configViper.WithConfigFile(configFile), + configViper.WithConfigDir(configDir), + configViper.WithEnvPrefix(envPrefix)) if err != nil { panic(err) } return &Config{ - v: v, - opts: *o, + v: v, + configFile: configFile, + configDir: configDir, + envPrefix: envPrefix, } } // Reload reads configuration path if it was provided to New. func (x *Config) Reload() error { - return configViper.ReloadViper(x.v, x.opts) + return configViper.ReloadViper( + configViper.WithViper(x.v), configViper.WithConfigFile(x.configFile), + configViper.WithConfigDir(x.configDir), + configViper.WithEnvPrefix(x.envPrefix)) } diff --git a/cmd/frostfs-node/config/configdir_test.go b/cmd/frostfs-node/config/configdir_test.go index f41e6edd9..ede15a522 100644 --- a/cmd/frostfs-node/config/configdir_test.go +++ b/cmd/frostfs-node/config/configdir_test.go @@ -5,7 +5,6 @@ import ( "path" "testing" - configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "github.com/spf13/cast" "github.com/stretchr/testify/require" ) @@ -19,7 +18,7 @@ func TestConfigDir(t *testing.T) { require.NoError(t, os.WriteFile(cfgFileName0, []byte(`{"storage":{"shard_pool_size":15}}`), 0777)) require.NoError(t, os.WriteFile(cfgFileName1, []byte("logger:\n level: debug"), 0777)) - c := New(configViper.WithConfigDir(dir)) + c := New("", dir, "") require.Equal(t, "debug", cast.ToString(c.Sub("logger").Value("level"))) require.EqualValues(t, 15, cast.ToUint32(c.Sub("storage").Value("shard_pool_size"))) } diff --git a/cmd/frostfs-node/config/test/config.go b/cmd/frostfs-node/config/test/config.go index 2bd5ade21..28ec65291 100644 --- a/cmd/frostfs-node/config/test/config.go +++ b/cmd/frostfs-node/config/test/config.go @@ -7,22 +7,19 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" - configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "github.com/stretchr/testify/require" ) func fromFile(path string) *config.Config { os.Clearenv() // ENVs have priority over config files, so we do this in tests - return config.New( - configViper.WithConfigFile(path), - ) + return config.New(path, "", "") } func fromEnvFile(t testing.TB, path string) *config.Config { loadEnv(t, path) // github.com/joho/godotenv can do that as well - return config.New(configViper.WithEnvPrefix(config.EnvPrefix)) + return config.New("", "", config.EnvPrefix) } func forEachFile(paths []string, f func(*config.Config)) { @@ -48,7 +45,7 @@ func ForEnvFileType(t testing.TB, pref string, f func(*config.Config)) { // EmptyConfig returns config without any values and sections. func EmptyConfig() *config.Config { - return config.New(configViper.WithEnvPrefix(config.EnvPrefix)) + return config.New("", "", config.EnvPrefix) } // loadEnv reads .env file, parses `X=Y` records and sets OS ENVs. diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 5a66b210b..4fcc38e08 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -8,7 +8,6 @@ import ( "os" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" - configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" @@ -47,9 +46,7 @@ func main() { os.Exit(SuccessReturnCode) } - appCfg := config.New( - configViper.WithConfigFile(*configFile), configViper.WithConfigDir(*configDir), - configViper.WithEnvPrefix(config.EnvPrefix)) + appCfg := config.New(*configFile, *configDir, config.EnvPrefix) err := validateConfig(appCfg) fatalOnErr(err) diff --git a/cmd/frostfs-node/validate_test.go b/cmd/frostfs-node/validate_test.go index 75b694cec..d9c0f167f 100644 --- a/cmd/frostfs-node/validate_test.go +++ b/cmd/frostfs-node/validate_test.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" - configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "github.com/stretchr/testify/require" ) @@ -27,13 +26,13 @@ func TestValidate(t *testing.T) { t.Run("mainnet", func(t *testing.T) { os.Clearenv() // ENVs have priority over config files, so we do this in tests p := filepath.Join(exampleConfigPrefix, "mainnet/config.yml") - c := config.New(configViper.WithConfigFile(p)) + c := config.New(p, "", config.EnvPrefix) require.NoError(t, validateConfig(c)) }) t.Run("testnet", func(t *testing.T) { os.Clearenv() // ENVs have priority over config files, so we do this in tests p := filepath.Join(exampleConfigPrefix, "testnet/config.yml") - c := config.New(configViper.WithConfigFile(p)) + c := config.New(p, "", config.EnvPrefix) require.NoError(t, validateConfig(c)) }) } diff --git a/cmd/internal/common/config/opts.go b/cmd/internal/common/config/opts.go index a28b2b522..46e565639 100644 --- a/cmd/internal/common/config/opts.go +++ b/cmd/internal/common/config/opts.go @@ -1,38 +1,49 @@ package config -type Opts struct { - Path string - ConfigDir string - EnvPrefix string +import "github.com/spf13/viper" + +type opts struct { + path string + configDir string + envPrefix string + v *viper.Viper } -func DefaultOpts() *Opts { - return new(Opts) +func defaultOpts() *opts { + return new(opts) } // Option allows to set an optional parameter of the Config. -type Option func(*Opts) +type Option func(*opts) // WithConfigFile returns an option to set the system path // to the configuration file. func WithConfigFile(path string) Option { - return func(o *Opts) { - o.Path = path + return func(o *opts) { + o.path = path } } // WithConfigDir returns an option to set the system path // to the directory with configuration files. func WithConfigDir(path string) Option { - return func(o *Opts) { - o.ConfigDir = path + return func(o *opts) { + o.configDir = path } } // WithEnvPrefix returns an option to defines // a prefix that ENVIRONMENT variables will use. func WithEnvPrefix(envPrefix string) Option { - return func(o *Opts) { - o.EnvPrefix = envPrefix + return func(o *opts) { + o.envPrefix = envPrefix + } +} + +// WithViper returns an option to defines +// a predefined viper.Viper. +func WithViper(v *viper.Viper) Option { + return func(o *opts) { + o.v = v } } diff --git a/cmd/internal/common/config/viper.go b/cmd/internal/common/config/viper.go index b0af9ae28..41b8831ff 100644 --- a/cmd/internal/common/config/viper.go +++ b/cmd/internal/common/config/viper.go @@ -15,48 +15,62 @@ const ( EnvSeparator = "_" ) -func CreateViper(opts ...Option) (*viper.Viper, *Opts, error) { - v := viper.New() - - o := DefaultOpts() +func CreateViper(opts ...Option) (*viper.Viper, error) { + o := defaultOpts() for i := range opts { opts[i](o) } - if o.EnvPrefix != "" { - v.SetEnvPrefix(o.EnvPrefix) + var v *viper.Viper + if o.v != nil { + v = o.v + } else { + v = viper.New() + } + + if o.envPrefix != "" { + v.SetEnvPrefix(o.envPrefix) v.AutomaticEnv() v.SetEnvKeyReplacer(strings.NewReplacer(Separator, EnvSeparator)) } - if o.Path != "" { - v.SetConfigFile(o.Path) + if o.path != "" { + v.SetConfigFile(o.path) err := v.ReadInConfig() if err != nil { - return nil, nil, fmt.Errorf("failed to read config: %w", err) + return nil, fmt.Errorf("failed to read config: %w", err) } } - if o.ConfigDir != "" { - if err := config.ReadConfigDir(v, o.ConfigDir); err != nil { - return nil, nil, fmt.Errorf("failed to read config dir: %w", err) + if o.configDir != "" { + if err := config.ReadConfigDir(v, o.configDir); err != nil { + return nil, fmt.Errorf("failed to read config dir: %w", err) } } - return v, o, nil + return v, nil } -func ReloadViper(v *viper.Viper, o Opts) error { - if o.Path != "" { - err := v.ReadInConfig() +func ReloadViper(opts ...Option) error { + o := defaultOpts() + for i := range opts { + opts[i](o) + } + + if o.v == nil { + return fmt.Errorf("provide viper in opts") + } + + if o.path != "" { + err := o.v.ReadInConfig() if err != nil { return fmt.Errorf("rereading configuration file: %w", err) } } - if o.ConfigDir != "" { - if err := config.ReadConfigDir(v, o.ConfigDir); err != nil { + if o.configDir != "" { + if err := config.ReadConfigDir(o.v, o.configDir); err != nil { return fmt.Errorf("rereading configuration dir: %w", err) } } From 563780057d69d320134975e3e1a0664350963783 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 20 Apr 2023 14:19:48 +0300 Subject: [PATCH 0158/1943] [#125] ir: Use `internal/common/config` for reading config Signed-off-by: Anton Nikiforov --- cmd/frostfs-ir/defaults.go | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index 693dd08ee..a3bd867d6 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -1,44 +1,28 @@ package main import ( - "strings" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config" + configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "github.com/spf13/viper" ) func newConfig(path, directory string) (*viper.Viper, error) { const envPrefix = "FROSTFS_IR" - var ( - err error - v = viper.New() - ) + var err error + var dv = viper.New() - v.SetEnvPrefix(envPrefix) - v.AutomaticEnv() - v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + defaultConfiguration(dv) - defaultConfiguration(v) - - if path != "" { - v.SetConfigFile(path) - if strings.HasSuffix(path, ".json") { - v.SetConfigType("json") - } else { - v.SetConfigType("yml") - } - if err = v.ReadInConfig(); err != nil { - return v, err - } + _, _, err = configViper.CreateViper(configViper.WithConfigFile(path), + configViper.WithConfigDir(directory), configViper.WithEnvPrefix(envPrefix), + configViper.WithViper(dv)) + if err != nil { + return nil, err } - if directory != "" { - err = config.ReadConfigDir(v, directory) - } - - return v, err + return dv, err } func defaultConfiguration(cfg *viper.Viper) { From 14f83b8aa9211ca8968e3a48a279693a7d5261dd Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 25 Apr 2023 13:10:24 +0300 Subject: [PATCH 0159/1943] [#125] ir: Change log level on SIGHUP Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 2 + cmd/frostfs-ir/config.go | 73 +++++++++++++++++++++++++++++ cmd/frostfs-ir/defaults.go | 19 -------- cmd/frostfs-ir/main.go | 94 +++++++++++++++++++++----------------- 4 files changed, 127 insertions(+), 61 deletions(-) create mode 100644 cmd/frostfs-ir/config.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e6792c5b3..e4c86fa37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Changelog for FrostFS Node ### Added - Support impersonate bearer token (#229) +- Change log level on SIGHUP for ir (#125) + ### Changed ### Fixed - Take network settings into account during netmap contract update (#100) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go new file mode 100644 index 000000000..c4787f965 --- /dev/null +++ b/cmd/frostfs-ir/config.go @@ -0,0 +1,73 @@ +package main + +import ( + "os" + "os/signal" + "syscall" + + configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "github.com/spf13/viper" + "go.uber.org/zap" +) + +func newConfig() (*viper.Viper, error) { + var err error + var dv = viper.New() + + defaultConfiguration(dv) + + _, err = configViper.CreateViper(configViper.WithConfigFile(*configFile), + configViper.WithConfigDir(*configDir), configViper.WithEnvPrefix(EnvPrefix), + configViper.WithViper(dv)) + if err != nil { + return nil, err + } + + return dv, err +} + +func reloadConfig() error { + err := configViper.ReloadViper(configViper.WithConfigFile(*configFile), + configViper.WithConfigDir(*configDir), configViper.WithEnvPrefix(EnvPrefix), + configViper.WithViper(cfg)) + if err != nil { + return err + } + err = logPrm.SetLevelString(cfg.GetString("logger.level")) + if err != nil { + return err + } + return logPrm.Reload() +} + +func watchForSignal(cancel func()) { + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) + + for { + select { + case err := <-intErr: + log.Info(logs.FrostFSIRInternalError, zap.String("msg", err.Error())) + cancel() + shutdown() + return + case sig := <-ch: + switch sig { + case syscall.SIGHUP: + log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) + err := reloadConfig() + if err != nil { + log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) + } + log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) + case syscall.SIGTERM, syscall.SIGINT: + log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + cancel() + shutdown() + log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) + return + } + } + } +} diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index a3bd867d6..eaad05f2e 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -3,28 +3,9 @@ package main import ( "time" - configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "github.com/spf13/viper" ) -func newConfig(path, directory string) (*viper.Viper, error) { - const envPrefix = "FROSTFS_IR" - - var err error - var dv = viper.New() - - defaultConfiguration(dv) - - _, _, err = configViper.CreateViper(configViper.WithConfigFile(path), - configViper.WithConfigDir(directory), configViper.WithEnvPrefix(envPrefix), - configViper.WithViper(dv)) - if err != nil { - return nil, err - } - - return dv, err -} - func defaultConfiguration(cfg *viper.Viper) { cfg.SetDefault("logger.level", "info") diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 5db1db6b6..932b90404 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -6,8 +6,7 @@ import ( "fmt" "net/http" "os" - "os/signal" - "syscall" + "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" @@ -25,6 +24,20 @@ const ( // SuccessReturnCode returns when application closed without panic. SuccessReturnCode = 0 + + EnvPrefix = "FROSTFS_IR" +) + +var ( + wg = new(sync.WaitGroup) + intErr = make(chan error) // internal inner ring errors + logPrm = new(logger.Prm) + innerRing *innerring.Server + httpServers []*httputil.Server + log *logger.Logger + cfg *viper.Viper + configFile *string + configDir *string ) func exitErr(err error) { @@ -35,8 +48,8 @@ func exitErr(err error) { } func main() { - configFile := flag.String("config", "", "path to config") - configDir := flag.String("config-dir", "", "path to config directory") + configFile = flag.String("config", "", "path to config") + configDir = flag.String("config-dir", "", "path to config directory") versionFlag := flag.Bool("version", false, "frostfs-ir node version") flag.Parse() @@ -46,35 +59,32 @@ func main() { os.Exit(SuccessReturnCode) } - cfg, err := newConfig(*configFile, *configDir) + var err error + cfg, err = newConfig() exitErr(err) - var logPrm logger.Prm - err = logPrm.SetLevelString( cfg.GetString("logger.level"), ) exitErr(err) - log, err := logger.NewLogger(&logPrm) + log, err = logger.NewLogger(logPrm) exitErr(err) - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) - defer cancel() + ctx, cancel := context.WithCancel(context.Background()) - intErr := make(chan error) // internal inner ring errors + initHTTPServers(cfg) - httpServers := initHTTPServers(cfg, log) - - innerRing, err := innerring.New(ctx, log, cfg, intErr) + innerRing, err = innerring.New(ctx, log, cfg, intErr) exitErr(err) // start HTTP servers - for i := range httpServers { - srv := httpServers[i] - go func() { + for _, srv := range httpServers { + wg.Add(1) + go func(srv *httputil.Server) { exitErr(srv.Serve()) - }() + wg.Done() + }(srv) } // start inner ring @@ -84,32 +94,16 @@ func main() { log.Info(logs.CommonApplicationStarted, zap.String("version", misc.Version)) - select { - case <-ctx.Done(): - case err := <-intErr: - log.Info(logs.FrostFSIRInternalError, zap.String("msg", err.Error())) - } + watchForSignal(cancel) - innerRing.Stop() - - // shut down HTTP servers - for i := range httpServers { - srv := httpServers[i] - - go func() { - err := srv.Shutdown() - if err != nil { - log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, - zap.String("error", err.Error()), - ) - } - }() - } + <-ctx.Done() // graceful shutdown + log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop) + wg.Wait() log.Info(logs.FrostFSIRApplicationStopped) } -func initHTTPServers(cfg *viper.Viper, log *logger.Logger) []*httputil.Server { +func initHTTPServers(cfg *viper.Viper) { items := []struct { cfgPrefix string handler func() http.Handler @@ -118,7 +112,7 @@ func initHTTPServers(cfg *viper.Viper, log *logger.Logger) []*httputil.Server { {"prometheus", promhttp.Handler}, } - httpServers := make([]*httputil.Server, 0, len(items)) + httpServers = make([]*httputil.Server, 0, len(items)) for _, item := range items { if !cfg.GetBool(item.cfgPrefix + ".enabled") { @@ -141,6 +135,22 @@ func initHTTPServers(cfg *viper.Viper, log *logger.Logger) []*httputil.Server { ), ) } - - return httpServers +} + +func shutdown() { + innerRing.Stop() + + // shut down HTTP servers + for _, srv := range httpServers { + wg.Add(1) + go func(srv *httputil.Server) { + err := srv.Shutdown() + if err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), + ) + } + wg.Done() + }(srv) + } } From 57718bd6b47a4a1b3ffdeabdfdc8c460587ecb70 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 10:48:28 +0300 Subject: [PATCH 0160/1943] [#285] *: Update golangci-lint to v1.52.2 Signed-off-by: Evgenii Stratonikov --- .pre-commit-config.yaml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db9930e91..c7273e1ee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: - id: shellcheck - repo: https://github.com/golangci/golangci-lint - rev: v1.51.2 + rev: v1.52.2 hooks: - id: golangci-lint diff --git a/Makefile b/Makefile index 7d0f8d9e2..29625e657 100755 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ HUB_IMAGE ?= truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.19 -LINT_VERSION ?= 1.50.0 +LINT_VERSION ?= 1.52.2 ARCH = amd64 BIN = bin From f07e2d481209da5b2794b8eb1886fba7036ae9f4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 11:24:40 +0300 Subject: [PATCH 0161/1943] [#285] lint: Fix revive/unused-parameter Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/config/config.go | 2 +- cmd/frostfs-adm/internal/modules/morph/epoch.go | 2 +- cmd/frostfs-adm/internal/modules/morph/generate.go | 2 +- .../internal/modules/morph/initialize.go | 6 +++--- .../internal/modules/morph/local_client.go | 2 +- cmd/frostfs-adm/internal/modules/root.go | 2 +- pkg/innerring/initialization.go | 2 +- pkg/innerring/innerring.go | 2 +- .../blobstor/blobovniczatree/delete.go | 14 +++++++------- pkg/local_object_storage/blobstor/fstree/fstree.go | 2 +- pkg/network/cache/multi.go | 2 +- pkg/services/accounting/morph/executor.go | 2 +- pkg/services/container/morph/executor.go | 8 ++++---- pkg/services/control/server/set_netmap_status.go | 2 +- pkg/services/netmap/executor.go | 2 +- pkg/services/object_manager/placement/netmap.go | 2 +- .../session/storage/persistent/executor.go | 2 +- pkg/services/session/storage/temporary/executor.go | 2 +- pkg/services/tree/service.go | 6 +++--- 19 files changed, 32 insertions(+), 32 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/config/config.go b/cmd/frostfs-adm/internal/modules/config/config.go index 13e575933..ebefe1d2c 100644 --- a/cmd/frostfs-adm/internal/modules/config/config.go +++ b/cmd/frostfs-adm/internal/modules/config/config.go @@ -47,7 +47,7 @@ credentials: {{.}}: password{{end}} ` -func initConfig(cmd *cobra.Command, args []string) error { +func initConfig(cmd *cobra.Command, _ []string) error { configPath, err := readConfigPathFromArgs(cmd) if err != nil { return nil diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/epoch.go index 608ff6b43..1f622fa89 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/epoch.go @@ -13,7 +13,7 @@ import ( "github.com/spf13/viper" ) -func forceNewEpochCmd(cmd *cobra.Command, args []string) error { +func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { wCtx, err := newInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index 23637fed3..e714482dd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -29,7 +29,7 @@ const ( consensusAccountName = "consensus" ) -func generateAlphabetCreds(cmd *cobra.Command, args []string) error { +func generateAlphabetCreds(cmd *cobra.Command, _ []string) error { // alphabet size is not part of the config size, err := cmd.Flags().GetUint(alphabetSizeFlag) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index a99a5faff..2f4721653 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -45,7 +45,7 @@ type initializeContext struct { ContractPath string } -func initializeSideChainCmd(cmd *cobra.Command, args []string) error { +func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { initCtx, err := newInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("initialization error: %w", err) @@ -142,7 +142,7 @@ func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContex return nil, err } - ctrPath, err := getContractsPath(cmd, v, needContracts) + ctrPath, err := getContractsPath(cmd, needContracts) if err != nil { return nil, err } @@ -222,7 +222,7 @@ func getWallet(cmd *cobra.Command, v *viper.Viper, needContracts bool, walletDir return openContractWallet(v, cmd, walletDir) } -func getContractsPath(cmd *cobra.Command, v *viper.Viper, needContracts bool) (string, error) { +func getContractsPath(cmd *cobra.Command, needContracts bool) (string, error) { if !needContracts { return "", nil } diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index 94ccedb37..816f9da4c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -248,7 +248,7 @@ func (l *localClient) GetVersion() (*result.Version, error) { }, nil } -func (l *localClient) InvokeContractVerify(contract util.Uint160, params []smartcontract.Parameter, signers []transaction.Signer, witnesses ...transaction.Witness) (*result.Invoke, error) { +func (l *localClient) InvokeContractVerify(util.Uint160, []smartcontract.Parameter, []transaction.Signer, ...transaction.Witness) (*result.Invoke, error) { // not used by `morph init` command panic("unexpected call") } diff --git a/cmd/frostfs-adm/internal/modules/root.go b/cmd/frostfs-adm/internal/modules/root.go index 4e30d5e62..fd1517f90 100644 --- a/cmd/frostfs-adm/internal/modules/root.go +++ b/cmd/frostfs-adm/internal/modules/root.go @@ -52,7 +52,7 @@ func Execute() error { return rootCmd.Execute() } -func entryPoint(cmd *cobra.Command, args []string) error { +func entryPoint(cmd *cobra.Command, _ []string) error { printVersion, _ := cmd.Flags().GetBool("version") if printVersion { cmd.Print(misc.BuildInfo("FrostFS Adm")) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 31602a48e..7e0c310fa 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -159,7 +159,7 @@ func (s *Server) enableNotarySupport() error { return nil } -func (s *Server) initNotaryConfig(cfg *viper.Viper) { +func (s *Server) initNotaryConfig() { s.mainNotaryConfig, s.sideNotaryConfig = notaryConfigs( s.morphClient.ProbeNotary(), !s.withoutMainNet && s.mainnetClient.ProbeNotary(), // if mainnet disabled then notary flag must be disabled too diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 42917f1d6..2cb29b90a 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -359,7 +359,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - server.initNotaryConfig(cfg) + server.initNotaryConfig() err = server.initContracts(cfg) if err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 073e69b57..f84d8fbe8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -42,7 +42,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co return res, err } - return b.deleteObject(ctx, blz, bPrm, prm) + return b.deleteObject(ctx, blz, bPrm) } activeCache := make(map[string]struct{}) @@ -54,7 +54,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co // don't process active blobovnicza of the level twice _, ok := activeCache[dirPath] - res, err = b.deleteObjectFromLevel(ctx, bPrm, p, !ok, prm) + res, err = b.deleteObjectFromLevel(ctx, bPrm, p, !ok) if err != nil { if !blobovnicza.IsErrNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, @@ -85,7 +85,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co // tries to delete object from particular blobovnicza. // // returns no error if object was removed from some blobovnicza of the same level. -func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicza.DeletePrm, blzPath string, tryActive bool, dp common.DeletePrm) (common.DeleteRes, error) { +func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicza.DeletePrm, blzPath string, tryActive bool) (common.DeleteRes, error) { lvlPath := filepath.Dir(blzPath) // try to remove from blobovnicza if it is opened @@ -93,7 +93,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicz v, ok := b.opened.Get(blzPath) b.lruMtx.Unlock() if ok { - if res, err := b.deleteObject(ctx, v, prm, dp); err == nil { + if res, err := b.deleteObject(ctx, v, prm); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza, @@ -112,7 +112,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicz b.activeMtx.RUnlock() if ok && tryActive { - if res, err := b.deleteObject(ctx, active.blz, prm, dp); err == nil { + if res, err := b.deleteObject(ctx, active.blz, prm); err == nil { return res, err } else if !blobovnicza.IsErrNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza, @@ -137,11 +137,11 @@ func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicz return common.DeleteRes{}, err } - return b.deleteObject(ctx, blz, prm, dp) + return b.deleteObject(ctx, blz, prm) } // removes object from blobovnicza and returns common.DeleteRes. -func (b *Blobovniczas) deleteObject(ctx context.Context, blz *blobovnicza.Blobovnicza, prm blobovnicza.DeletePrm, dp common.DeletePrm) (common.DeleteRes, error) { +func (b *Blobovniczas) deleteObject(ctx context.Context, blz *blobovnicza.Blobovnicza, prm blobovnicza.DeletePrm) (common.DeleteRes, error) { _, err := blz.Delete(ctx, prm) return common.DeleteRes{}, err } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 8eb0d5be9..75f63193a 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -468,6 +468,6 @@ func (t *FSTree) SetCompressor(cc *compression.Config) { } // SetReportErrorFunc implements common.Storage. -func (t *FSTree) SetReportErrorFunc(f func(string, error)) { +func (t *FSTree) SetReportErrorFunc(_ func(string, error)) { // Do nothing, FSTree can encounter only one error which is returned. } diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index b1afaa90d..b8a0aa4bc 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -277,7 +277,7 @@ func (x *multiClient) ObjectSearchInit(ctx context.Context, p client.PrmObjectSe return } -func (x *multiClient) ExecRaw(f func(client *rawclient.Client) error) error { +func (x *multiClient) ExecRaw(func(client *rawclient.Client) error) error { panic("multiClient.ExecRaw() must not be called") } diff --git a/pkg/services/accounting/morph/executor.go b/pkg/services/accounting/morph/executor.go index 434c89508..ac836b71d 100644 --- a/pkg/services/accounting/morph/executor.go +++ b/pkg/services/accounting/morph/executor.go @@ -21,7 +21,7 @@ func NewExecutor(client *balance.Client) accountingSvc.ServiceExecutor { } } -func (s *morphExecutor) Balance(ctx context.Context, body *accounting.BalanceRequestBody) (*accounting.BalanceResponseBody, error) { +func (s *morphExecutor) Balance(_ context.Context, body *accounting.BalanceRequestBody) (*accounting.BalanceResponseBody, error) { idV2 := body.GetOwnerID() if idV2 == nil { return nil, errors.New("missing account") diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index 42035c8d0..8e6b30856 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -136,7 +136,7 @@ func (s *morphExecutor) Delete(_ context.Context, tokV2 *sessionV2.Token, body * return new(container.DeleteResponseBody), nil } -func (s *morphExecutor) Get(ctx context.Context, body *container.GetRequestBody) (*container.GetResponseBody, error) { +func (s *morphExecutor) Get(_ context.Context, body *container.GetRequestBody) (*container.GetResponseBody, error) { idV2 := body.GetContainerID() if idV2 == nil { return nil, errors.New("missing container ID") @@ -176,7 +176,7 @@ func (s *morphExecutor) Get(ctx context.Context, body *container.GetRequestBody) return res, nil } -func (s *morphExecutor) List(ctx context.Context, body *container.ListRequestBody) (*container.ListResponseBody, error) { +func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) (*container.ListResponseBody, error) { idV2 := body.GetOwnerID() if idV2 == nil { return nil, fmt.Errorf("missing user ID") @@ -205,7 +205,7 @@ func (s *morphExecutor) List(ctx context.Context, body *container.ListRequestBod return res, nil } -func (s *morphExecutor) SetExtendedACL(ctx context.Context, tokV2 *sessionV2.Token, body *container.SetExtendedACLRequestBody) (*container.SetExtendedACLResponseBody, error) { +func (s *morphExecutor) SetExtendedACL(_ context.Context, tokV2 *sessionV2.Token, body *container.SetExtendedACLRequestBody) (*container.SetExtendedACLResponseBody, error) { sigV2 := body.GetSignature() if sigV2 == nil { // TODO(@cthulhu-rider): #1387 use "const" error @@ -238,7 +238,7 @@ func (s *morphExecutor) SetExtendedACL(ctx context.Context, tokV2 *sessionV2.Tok return new(container.SetExtendedACLResponseBody), nil } -func (s *morphExecutor) GetExtendedACL(ctx context.Context, body *container.GetExtendedACLRequestBody) (*container.GetExtendedACLResponseBody, error) { +func (s *morphExecutor) GetExtendedACL(_ context.Context, body *container.GetExtendedACLRequestBody) (*container.GetExtendedACLResponseBody, error) { idV2 := body.GetContainerID() if idV2 == nil { return nil, errors.New("missing container ID") diff --git a/pkg/services/control/server/set_netmap_status.go b/pkg/services/control/server/set_netmap_status.go index ba7e682c6..d4a856952 100644 --- a/pkg/services/control/server/set_netmap_status.go +++ b/pkg/services/control/server/set_netmap_status.go @@ -11,7 +11,7 @@ import ( // SetNetmapStatus sets node status in FrostFS network. // // If request is unsigned or signed by disallowed key, permission error returns. -func (s *Server) SetNetmapStatus(ctx context.Context, req *control.SetNetmapStatusRequest) (*control.SetNetmapStatusResponse, error) { +func (s *Server) SetNetmapStatus(_ context.Context, req *control.SetNetmapStatusRequest) (*control.SetNetmapStatusResponse, error) { // verify request if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) diff --git a/pkg/services/netmap/executor.go b/pkg/services/netmap/executor.go index 9fa3d767f..d77c69d4d 100644 --- a/pkg/services/netmap/executor.go +++ b/pkg/services/netmap/executor.go @@ -129,7 +129,7 @@ func (s *executorSvc) NetworkInfo( return resp, nil } -func (s *executorSvc) Snapshot(_ context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { +func (s *executorSvc) Snapshot(_ context.Context, _ *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { var nm netmap.NetMap err := s.state.ReadCurrentNetMap(&nm) diff --git a/pkg/services/object_manager/placement/netmap.go b/pkg/services/object_manager/placement/netmap.go index 01d607020..787c04421 100644 --- a/pkg/services/object_manager/placement/netmap.go +++ b/pkg/services/object_manager/placement/netmap.go @@ -47,7 +47,7 @@ func NewNetworkMapSourceBuilder(nmSrc netmap.Source) Builder { } } -func (s *netMapSrc) GetNetMap(diff uint64) (*netmapSDK.NetMap, error) { +func (s *netMapSrc) GetNetMap(_ uint64) (*netmapSDK.NetMap, error) { return s.nm, nil } diff --git a/pkg/services/session/storage/persistent/executor.go b/pkg/services/session/storage/persistent/executor.go index f59e312c4..21f55a7d1 100644 --- a/pkg/services/session/storage/persistent/executor.go +++ b/pkg/services/session/storage/persistent/executor.go @@ -17,7 +17,7 @@ import ( // encrypts private keys if storage has been configured so). // Returns response that is filled with just created token's // ID and public key for it. -func (s *TokenStore) Create(ctx context.Context, body *session.CreateRequestBody) (*session.CreateResponseBody, error) { +func (s *TokenStore) Create(_ context.Context, body *session.CreateRequestBody) (*session.CreateResponseBody, error) { idV2 := body.GetOwnerID() if idV2 == nil { return nil, errors.New("missing owner") diff --git a/pkg/services/session/storage/temporary/executor.go b/pkg/services/session/storage/temporary/executor.go index aa64d796b..cd498709c 100644 --- a/pkg/services/session/storage/temporary/executor.go +++ b/pkg/services/session/storage/temporary/executor.go @@ -12,7 +12,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) -func (s *TokenStore) Create(ctx context.Context, body *session.CreateRequestBody) (*session.CreateResponseBody, error) { +func (s *TokenStore) Create(_ context.Context, body *session.CreateRequestBody) (*session.CreateResponseBody, error) { idV2 := body.GetOwnerID() if idV2 == nil { return nil, errors.New("missing owner") diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 35dfda3df..546b7a207 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -455,7 +455,7 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD } // Apply locally applies operation from the remote node to the tree. -func (s *Service) Apply(ctx context.Context, req *ApplyRequest) (*ApplyResponse, error) { +func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) { err := verifyMessage(req) if err != nil { return nil, err @@ -468,7 +468,7 @@ func (s *Service) Apply(ctx context.Context, req *ApplyRequest) (*ApplyResponse, key := req.GetSignature().GetKey() - _, pos, _, err := s.getContainerInfo(ctx, cid, key) + _, pos, _, err := s.getContainerInfo(cid, key) if err != nil { return nil, err } @@ -623,7 +623,7 @@ func metaToProto(arr []pilorama.KeyValue) []*KeyValue { // getContainerInfo returns the list of container nodes, position in the container for the node // with pub key and total amount of nodes in all replicas. -func (s *Service) getContainerInfo(ctx context.Context, cid cidSDK.ID, pub []byte) ([]netmapSDK.NodeInfo, int, int, error) { +func (s *Service) getContainerInfo(cid cidSDK.ID, pub []byte) ([]netmapSDK.NodeInfo, int, int, error) { cntNodes, _, err := s.getContainerNodes(cid) if err != nil { return nil, 0, 0, err From dfe4ada8388037b1f847edb66df1ee0f7ba60cba Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 11:31:41 +0300 Subject: [PATCH 0162/1943] [#285] lint: Resolve revive/if-return Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/initialize.go | 6 +----- pkg/services/object/get/assembler.go | 5 +---- pkg/services/object/get/v2/get_forwarder.go | 5 +---- pkg/services/object/get/v2/get_range_forwarder.go | 5 +---- pkg/services/object/get/v2/head_forwarder.go | 5 +---- 5 files changed, 5 insertions(+), 21 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 2f4721653..494ad5296 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -91,11 +91,7 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { } cmd.Println("Stage 7: set addresses in NNS.") - if err := initCtx.setNNS(); err != nil { - return err - } - - return nil + return initCtx.setNNS() } func (c *initializeContext) close() { diff --git a/pkg/services/object/get/assembler.go b/pkg/services/object/get/assembler.go index 4ae1981b1..7a3023c74 100644 --- a/pkg/services/object/get/assembler.go +++ b/pkg/services/object/get/assembler.go @@ -159,10 +159,7 @@ func (a *assembler) assembleObjectByChildrenList(ctx context.Context, childrenID if err := a.assemblePayloadInReverse(ctx, writer, childrenIDs[len(childrenIDs)-1]); err != nil { return err } - if err := writer.WriteChunk(ctx, a.parentObject.Payload()); err != nil { - return err - } - return nil + return writer.WriteChunk(ctx, a.parentObject.Payload()) } func (a *assembler) assemleObjectByPreviousIDInReverse(ctx context.Context, prevID oid.ID, writer ObjectWriter) error { diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index 3677327c0..b0de3a55b 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -74,10 +74,7 @@ func (f *getRequestForwarder) verifyResponse(resp *objectV2.GetResponse, pubkey return fmt.Errorf("response verification failed: %w", err) } - if err := checkStatus(resp.GetMetaHeader().GetStatus()); err != nil { - return err - } - return nil + return checkStatus(resp.GetMetaHeader().GetStatus()) } func (f *getRequestForwarder) writeHeader(ctx context.Context, v *objectV2.GetObjectPartInit) error { diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 06b65fb50..91d52a593 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -76,10 +76,7 @@ func (f *getRangeRequestForwarder) verifyResponse(resp *objectV2.GetRangeRespons return fmt.Errorf("could not verify %T: %w", resp, err) } - if err := checkStatus(resp.GetMetaHeader().GetStatus()); err != nil { - return err - } - return nil + return checkStatus(resp.GetMetaHeader().GetStatus()) } func (f *getRangeRequestForwarder) openStream(ctx context.Context, addr network.Address, c client.MultiAddressClient) (*rpc.ObjectRangeResponseReader, error) { diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index f02ef81bb..aab5046e4 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -176,8 +176,5 @@ func (f *headRequestForwarder) verifyResponse(headResp *objectV2.HeadResponse, p return fmt.Errorf("response verification failed: %w", err) } - if err := checkStatus(f.Response.GetMetaHeader().GetStatus()); err != nil { - return err - } - return nil + return checkStatus(f.Response.GetMetaHeader().GetStatus()) } From 1f4061c0e2d13a0852f170749a1aeacc60646241 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 11:29:33 +0300 Subject: [PATCH 0163/1943] [#285] blobonicza: Optimize upperPowerOfTwo() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The real reason is this: ``` pkg/local_object_storage/blobovnicza/sizes.go:36:69 revive empty-block: this block is empty, you can remove it ``` Didn't want to make this function longer or to add `nolint`, thus this change. To justify: ``` UpperBound/size=1-8 0.4924n ± 1% 0.2472n ± 2% -49.80% (p=0.000 n=10) UpperBound/size=1023-8 0.4936n ± 3% 0.2442n ± 1% -50.52% (p=0.000 n=10) UpperBound/size=66560-8 0.8201n ± 2% 0.2436n ± 1% -70.29% (p=0.000 n=10) UpperBound/size=41943040-8 6.6900n ± 5% 0.2432n ± 0% -96.36% (p=0.000 n=10) geomean 1.075n 0.2446n -77.24% ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobovnicza/sizes.go | 9 +++++---- pkg/local_object_storage/blobovnicza/sizes_test.go | 11 +++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index 5b89760c6..82454fa28 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -3,6 +3,7 @@ package blobovnicza import ( "encoding/binary" "fmt" + "math/bits" "strconv" ) @@ -31,11 +32,11 @@ func bucketForSize(sz uint64) []byte { return bucketKeyFromBounds(upperPowerOfTwo(sz)) } -func upperPowerOfTwo(v uint64) (upperBound uint64) { - for upperBound = firstBucketBound; upperBound < v; upperBound *= 2 { +func upperPowerOfTwo(v uint64) uint64 { + if v <= firstBucketBound { + return firstBucketBound } - - return + return 1 << bits.Len64(v-1) } func (b *Blobovnicza) incSize(sz uint64) { diff --git a/pkg/local_object_storage/blobovnicza/sizes_test.go b/pkg/local_object_storage/blobovnicza/sizes_test.go index c218dc638..2bf451f12 100644 --- a/pkg/local_object_storage/blobovnicza/sizes_test.go +++ b/pkg/local_object_storage/blobovnicza/sizes_test.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "fmt" "testing" "github.com/stretchr/testify/require" @@ -36,3 +37,13 @@ func TestSizes(t *testing.T) { require.Equal(t, bucketKeyFromBounds(item.upperBound), bucketForSize(item.sz)) } } + +func BenchmarkUpperBound(b *testing.B) { + for _, size := range []uint64{1, 1023, 65 * 1024, 40 * 1024 * 1024} { + b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = upperPowerOfTwo(size) + } + }) + } +} From ada081dfd566341d38dc5633dc63783095e2c20c Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Sun, 16 Apr 2023 17:03:42 +0300 Subject: [PATCH 0164/1943] [#19] node: Make policier read shards concurrently * Introduce ListWithMultiCursor that simultaneously reads objects from different shards Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/local_object_storage/engine/list.go | 123 ++++++++++----- pkg/local_object_storage/engine/list_test.go | 156 ++++++++++++------- 2 files changed, 187 insertions(+), 92 deletions(-) diff --git a/pkg/local_object_storage/engine/list.go b/pkg/local_object_storage/engine/list.go index 8644a7f7e..8781416f4 100644 --- a/pkg/local_object_storage/engine/list.go +++ b/pkg/local_object_storage/engine/list.go @@ -1,6 +1,7 @@ package engine import ( + "math/rand" "sort" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -12,10 +13,38 @@ import ( // cursor. Use nil cursor object to start listing again. var ErrEndOfListing = shard.ErrEndOfListing -// Cursor is a type for continuous object listing. +// Cursor is a type for continuous object listing. Cursor contains shard IDs to read +// and shard cursors that contain state from previous read. type Cursor struct { - shardID string - shardCursor *shard.Cursor + current string + shardIDs map[string]bool + shardIDToCursor map[string]*shard.Cursor +} + +func (c *Cursor) getCurrentShardCursor() *shard.Cursor { + return c.shardIDToCursor[c.current] +} + +func (c *Cursor) setCurrentShardCursor(sc *shard.Cursor) { + c.shardIDToCursor[c.current] = sc +} + +func (c *Cursor) nextShard() bool { + var shardsToRead []string + for shardID, read := range c.shardIDs { + if !read { + shardsToRead = append(shardsToRead, shardID) + } + } + if len(shardsToRead) == 0 { + return false + } + c.current = shardsToRead[rand.Intn(len(shardsToRead))] + return true +} + +func (c *Cursor) setShardRead(shardID string) { + c.shardIDs[shardID] = true } // ListWithCursorPrm contains parameters for ListWithCursor operation. @@ -57,65 +86,69 @@ func (l ListWithCursorRes) Cursor() *Cursor { // Does not include inhumed objects. Use cursor value from the response // for consecutive requests. // +// If count param is big enough, then the method reads objects from different shards +// by portions. In this case shards are chosen randomly, if they're not read out yet. +// +// Adding a shard between ListWithCursor does not invalidate the cursor but new shard +// won't be listed. +// Removing a shard between ListWithCursor leads to the undefined behavior +// (e.g. usage of the objects from the removed shard). +// // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. func (e *StorageEngine) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) { result := make([]objectcore.AddressWithType, 0, prm.count) - // 1. Get available shards and sort them. - e.mtx.RLock() - shardIDs := make([]string, 0, len(e.shards)) - for id := range e.shards { - shardIDs = append(shardIDs, id) - } - e.mtx.RUnlock() - - if len(shardIDs) == 0 { - return ListWithCursorRes{}, ErrEndOfListing - } - - sort.Slice(shardIDs, func(i, j int) bool { - return shardIDs[i] < shardIDs[j] - }) - - // 2. Prepare cursor object. + // Set initial cursors cursor := prm.cursor if cursor == nil { - cursor = &Cursor{shardID: shardIDs[0]} + shardIDs := getSortedShardIDs(e) + if len(shardIDs) == 0 { + return ListWithCursorRes{}, ErrEndOfListing + } + cursor = newCursor(shardIDs) } - // 3. Iterate over available shards. Skip unavailable shards. - for i := range shardIDs { + const ( + splitShardCountLimit = 100 + shardsNum = 4 + ) + + batchSize := prm.count + if batchSize >= splitShardCountLimit { + batchSize /= shardsNum + } + + for cursor.nextShard() { if len(result) >= int(prm.count) { break } - - if shardIDs[i] < cursor.shardID { - continue - } + curr := cursor.current e.mtx.RLock() - shardInstance, ok := e.shards[shardIDs[i]] + shardInstance, ok := e.shards[curr] e.mtx.RUnlock() if !ok { + cursor.setShardRead(curr) continue } - count := uint32(int(prm.count) - len(result)) + count := prm.count - uint32(len(result)) + if count > batchSize { + count = batchSize + } + var shardPrm shard.ListWithCursorPrm shardPrm.WithCount(count) - if shardIDs[i] == cursor.shardID { - shardPrm.WithCursor(cursor.shardCursor) - } + shardPrm.WithCursor(cursor.getCurrentShardCursor()) res, err := shardInstance.ListWithCursor(shardPrm) if err != nil { + cursor.setShardRead(curr) continue } - result = append(result, res.AddressList()...) - cursor.shardCursor = res.Cursor() - cursor.shardID = shardIDs[i] + cursor.setCurrentShardCursor(res.Cursor()) } if len(result) == 0 { @@ -127,3 +160,23 @@ func (e *StorageEngine) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes cursor: cursor, }, nil } + +func getSortedShardIDs(e *StorageEngine) []string { + e.mtx.RLock() + shardIDs := make([]string, 0, len(e.shards)) + for id := range e.shards { + shardIDs = append(shardIDs, id) + } + e.mtx.RUnlock() + sort.Strings(shardIDs) + return shardIDs +} + +func newCursor(shardIDs []string) *Cursor { + shardIDsMap := make(map[string]bool) + shardIDToCursor := make(map[string]*shard.Cursor) + for _, shardID := range shardIDs { + shardIDsMap[shardID] = false + } + return &Cursor{shardIDs: shardIDsMap, shardIDToCursor: shardIDToCursor} +} diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index fde799d05..44062be68 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -2,77 +2,119 @@ package engine import ( "context" - "errors" + "fmt" "os" + "path/filepath" "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) -func TestListWithCursor(t *testing.T) { - s1 := testNewShard(t, 1) - s2 := testNewShard(t, 2) - e := testNewEngine(t).setInitializedShards(t, s1, s2).engine - - t.Cleanup(func() { - e.Close() - os.RemoveAll(t.Name()) - }) - - const total = 20 - - expected := make([]object.AddressWithType, 0, total) - got := make([]object.AddressWithType, 0, total) - - for i := 0; i < total; i++ { - containerID := cidtest.ID() - obj := testutil.GenerateObjectWithCID(containerID) - - var prm PutPrm - prm.WithObject(obj) - - err := e.Put(context.Background(), prm) - require.NoError(t, err) - expected = append(expected, object.AddressWithType{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)}) - } - - expected = sortAddresses(expected) - - var prm ListWithCursorPrm - prm.WithCount(1) - - res, err := e.ListWithCursor(prm) - require.NoError(t, err) - require.NotEmpty(t, res.AddressList()) - got = append(got, res.AddressList()...) - - for i := 0; i < total-1; i++ { - prm.WithCursor(res.Cursor()) - - res, err = e.ListWithCursor(prm) - if errors.Is(err, ErrEndOfListing) { - break - } - got = append(got, res.AddressList()...) - } - - prm.WithCursor(res.Cursor()) - - _, err = e.ListWithCursor(prm) - require.ErrorIs(t, err, ErrEndOfListing) - - got = sortAddresses(got) - require.Equal(t, expected, got) -} - func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithType { sort.Slice(addrWithType, func(i, j int) bool { return addrWithType[i].Address.EncodeToString() < addrWithType[j].Address.EncodeToString() }) return addrWithType } + +func TestListWithCursor(t *testing.T) { + tests := []struct { + name string + shardNum int + objectNum int + batchSize uint32 + }{ + { + name: "one shard, few objects, small batch size", + shardNum: 1, + objectNum: 2, + batchSize: 1, + }, + { + name: "one shard, many objects, big batch size", + shardNum: 1, + objectNum: 53, + batchSize: 100, + }, + { + name: "many shards, many objects, small batch size", + shardNum: 6, + objectNum: 66, + batchSize: 1, + }, + { + name: "many shards, many objects, big batch size", + shardNum: 6, + objectNum: 99, + batchSize: 100, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := testNewEngine(t).setShardsNumOpts(t, tt.shardNum, func(id int) []shard.Option { + return []shard.Option{ + shard.WithLogger(&logger.Logger{Logger: zap.L()}), + shard.WithBlobStorOptions( + blobstor.WithStorages( + newStorages(filepath.Join(t.Name(), tt.name, fmt.Sprintf("%d.blobstor", id)), + 1<<20))), + shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.Name(), tt.name, fmt.Sprintf("%d.pilorama", id)))), + shard.WithMetaBaseOptions( + meta.WithPath(filepath.Join(t.Name(), tt.name, fmt.Sprintf("%d.metabase", id))), + meta.WithPermissions(0700), + meta.WithEpochState(epochState{}), + )} + }).engine + require.NoError(t, e.Open()) + require.NoError(t, e.Init(context.Background())) + + t.Cleanup(func() { + e.Close() + os.RemoveAll(t.Name()) + }) + + expected := make([]object.AddressWithType, 0, tt.objectNum) + got := make([]object.AddressWithType, 0, tt.objectNum) + + for i := 0; i < tt.objectNum; i++ { + containerID := cidtest.ID() + obj := testutil.GenerateObjectWithCIDWithPayload(containerID, []byte{'a'}) + + var prm PutPrm + prm.WithObject(obj) + + err := e.Put(context.Background(), prm) + require.NoError(t, err) + expected = append(expected, object.AddressWithType{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)}) + } + expected = sortAddresses(expected) + + var prm ListWithCursorPrm + prm.count = tt.batchSize + for { + res, err := e.ListWithCursor(prm) + if err == ErrEndOfListing { + require.Empty(t, res.AddressList()) + break + } + require.NotEmpty(t, res.AddressList()) + got = append(got, res.AddressList()...) + prm.cursor = res.Cursor() + } + + got = sortAddresses(got) + require.Equal(t, expected, got) + }) + } +} From f856ad74802d62ef487d18e624856da4fcde5665 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 15:23:07 +0300 Subject: [PATCH 0165/1943] [#288] pilorama: Remove `childMap` from memory forest Memory forest is here to check the correctness of boltdb optimized implementation. Let's keep it simple. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/forest.go | 6 +- .../pilorama/forest_test.go | 7 --- pkg/local_object_storage/pilorama/inmemory.go | 58 +++++++------------ 3 files changed, 22 insertions(+), 49 deletions(-) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 84530977c..0db919351 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -153,11 +153,7 @@ func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID str return nil, ErrTreeNotFound } - children, ok := s.childMap[nodeID] - if !ok { - return nil, nil - } - + children := s.tree.getChildren(nodeID) res := make([]Node, len(children)) copy(res, children) return res, nil diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index be53b3fe2..94b14457a 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -714,13 +714,6 @@ func compareForests(t *testing.T, expected, actual Forest, cid cidSDK.ID, treeID require.True(t, ok) require.Equal(t, se.operations, sa.operations) require.Equal(t, se.infoMap, sa.infoMap) - - require.Equal(t, len(se.childMap), len(sa.childMap)) - for ck, la := range sa.childMap { - le, ok := se.childMap[ck] - require.True(t, ok) - require.ElementsMatch(t, le, la) - } } require.Equal(t, expected, actual, i) } diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index 92dc9b6aa..bfc2a5b26 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -1,5 +1,7 @@ package pilorama +import "sort" + // nodeInfo couples parent and metadata. type nodeInfo struct { Parent Node @@ -27,27 +29,8 @@ func newState() *state { // undo un-does op and changes s in-place. func (s *state) undo(op *move) { - children := s.tree.childMap[op.Parent] - for i := range children { - if children[i] == op.Child { - if len(children) > 1 { - s.tree.childMap[op.Parent] = append(children[:i], children[i+1:]...) - } else { - delete(s.tree.childMap, op.Parent) - } - break - } - } - if op.HasOld { s.tree.infoMap[op.Child] = op.Old - oldChildren := s.tree.childMap[op.Old.Parent] - for i := range oldChildren { - if oldChildren[i] == op.Child { - return - } - } - s.tree.childMap[op.Old.Parent] = append(oldChildren, op.Child) } else { delete(s.tree.infoMap, op.Child) } @@ -104,28 +87,15 @@ func (s *state) do(op *Move) move { if !ok { p.Meta.Time = op.Time - } else { - s.removeChild(op.Child, p.Parent) } p.Meta = op.Meta p.Parent = op.Parent s.tree.infoMap[op.Child] = p - s.tree.childMap[op.Parent] = append(s.tree.childMap[op.Parent], op.Child) return lm } -func (s *state) removeChild(child, parent Node) { - oldChildren := s.tree.childMap[parent] - for i := range oldChildren { - if oldChildren[i] == child { - s.tree.childMap[parent] = append(oldChildren[:i], oldChildren[i+1:]...) - break - } - } -} - func (s *state) timestamp(pos, size int) Timestamp { if len(s.operations) == 0 { return nextTimestamp(0, uint64(pos), uint64(size)) @@ -145,16 +115,30 @@ func (s *state) findSpareID() Node { type tree struct { syncHeight uint64 infoMap map[Node]nodeInfo - childMap map[Node][]Node } func newTree() *tree { return &tree{ - childMap: make(map[Node][]Node), - infoMap: make(map[Node]nodeInfo), + infoMap: make(map[Node]nodeInfo), } } +func (t tree) getChildren(parent Node) []Node { + var children []Node + for c, info := range t.infoMap { + if info.Parent == parent { + children = append(children, c) + } + } + + sort.Slice(children, func(i, j int) bool { + a := t.infoMap[children[i]] + b := t.infoMap[children[j]] + return a.Meta.Time < b.Meta.Time + }) + return children +} + // isAncestor returns true if parent is an ancestor of a child. // For convenience, also return true if parent == child. func (t tree) isAncestor(parent, child Node) bool { @@ -176,7 +160,7 @@ func (t tree) getPathPrefix(attr string, path []string) (int, Node) { loop: for i := range path { - children := t.childMap[curNode] + children := t.getChildren(curNode) for j := range children { meta := t.infoMap[children[j]].Meta f := meta.GetAttr(attr) @@ -206,7 +190,7 @@ func (t tree) get(attr string, path []string, latest bool) []Node { var nodes []Node var lastTs Timestamp - children := t.childMap[curNode] + children := t.getChildren(curNode) for i := range children { info := t.infoMap[children[i]] fileName := string(info.Meta.GetAttr(attr)) From 0045f1bcd4f338fa575e5a6f437c7196c5fa3a9c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 17:07:51 +0300 Subject: [PATCH 0166/1943] [#288] pilorama: Use more descriptive names for memory tree Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/forest.go | 12 +++---- pkg/local_object_storage/pilorama/inmemory.go | 35 +++++++++---------- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 0db919351..1110927f9 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -12,7 +12,7 @@ import ( // memoryForest represents multiple replicating trees sharing a single storage. type memoryForest struct { // treeMap maps tree identifier (container ID + name) to the replicated log. - treeMap map[string]*state + treeMap map[string]*memoryTree } var _ Forest = (*memoryForest)(nil) @@ -21,7 +21,7 @@ var _ Forest = (*memoryForest)(nil) // TODO: this function will eventually be removed and is here for debugging. func NewMemoryForest() ForestStorage { return &memoryForest{ - treeMap: make(map[string]*state), + treeMap: make(map[string]*memoryTree), } } @@ -34,7 +34,7 @@ func (f *memoryForest) TreeMove(_ context.Context, d CIDDescriptor, treeID strin fullID := d.CID.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { - s = newState() + s = newMemoryTree() f.treeMap[fullID] = s } @@ -60,7 +60,7 @@ func (f *memoryForest) TreeAddByPath(_ context.Context, d CIDDescriptor, treeID fullID := d.CID.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { - s = newState() + s = newMemoryTree() f.treeMap[fullID] = s } @@ -98,7 +98,7 @@ func (f *memoryForest) TreeApply(_ context.Context, cnr cid.ID, treeID string, o fullID := cnr.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { - s = newState() + s = newMemoryTree() f.treeMap[fullID] = s } @@ -131,7 +131,7 @@ func (f *memoryForest) TreeGetByPath(_ context.Context, cid cid.ID, treeID strin return nil, ErrTreeNotFound } - return s.get(attr, path, latest), nil + return s.getByPath(attr, path, latest), nil } // TreeGetMeta implements the Forest interface. diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index bfc2a5b26..b836d4fb8 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -14,21 +14,23 @@ type move struct { Old nodeInfo } -// state represents state being replicated. -type state struct { +// memoryTree represents memoryTree being replicated. +type memoryTree struct { operations []move tree } -// newState constructs new empty tree. -func newState() *state { - return &state{ - tree: *newTree(), +// newMemoryTree constructs new empty tree. +func newMemoryTree() *memoryTree { + return &memoryTree{ + tree: tree{ + infoMap: make(map[Node]nodeInfo), + }, } } // undo un-does op and changes s in-place. -func (s *state) undo(op *move) { +func (s *memoryTree) undo(op *move) { if op.HasOld { s.tree.infoMap[op.Child] = op.Old } else { @@ -38,7 +40,7 @@ func (s *state) undo(op *move) { // Apply puts op in log at a proper position, re-applies all subsequent operations // from log and changes s in-place. -func (s *state) Apply(op *Move) error { +func (s *memoryTree) Apply(op *Move) error { var index int for index = len(s.operations); index > 0; index-- { if s.operations[index-1].Time <= op.Time { @@ -65,7 +67,7 @@ func (s *state) Apply(op *Move) error { } // do performs a single move operation on a tree. -func (s *state) do(op *Move) move { +func (s *memoryTree) do(op *Move) move { lm := move{ Move: Move{ Parent: op.Parent, @@ -96,14 +98,14 @@ func (s *state) do(op *Move) move { return lm } -func (s *state) timestamp(pos, size int) Timestamp { +func (s *memoryTree) timestamp(pos, size int) Timestamp { if len(s.operations) == 0 { return nextTimestamp(0, uint64(pos), uint64(size)) } return nextTimestamp(s.operations[len(s.operations)-1].Time, uint64(pos), uint64(size)) } -func (s *state) findSpareID() Node { +func (s *memoryTree) findSpareID() Node { id := uint64(1) for _, ok := s.infoMap[id]; ok; _, ok = s.infoMap[id] { id++ @@ -117,12 +119,6 @@ type tree struct { infoMap map[Node]nodeInfo } -func newTree() *tree { - return &tree{ - infoMap: make(map[Node]nodeInfo), - } -} - func (t tree) getChildren(parent Node) []Node { var children []Node for c, info := range t.infoMap { @@ -175,9 +171,10 @@ loop: return len(path), curNode } -// get returns list of nodes which have the specified path from root +// getByPath returns list of nodes which have the specified path from root // descending by values of attr from meta. -func (t tree) get(attr string, path []string, latest bool) []Node { +// If latest is true, only the latest node is returned. +func (t tree) getByPath(attr string, path []string, latest bool) []Node { if len(path) == 0 { return nil } From 291f9e809a28346d296af6a7514b5b32d80810d5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 17:12:04 +0300 Subject: [PATCH 0167/1943] [#288] pilorama: Remove getMeta() wrapper Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/forest.go | 2 +- pkg/local_object_storage/pilorama/inmemory.go | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 1110927f9..4286bdef5 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -142,7 +142,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string, return Meta{}, 0, ErrTreeNotFound } - return s.getMeta(nodeID), s.infoMap[nodeID].Parent, nil + return s.infoMap[nodeID].Meta, s.infoMap[nodeID].Parent, nil } // TreeGetChildren implements the Forest interface. diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index b836d4fb8..1bde312ac 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -204,8 +204,3 @@ func (t tree) getByPath(attr string, path []string, latest bool) []Node { return nodes } - -// getMeta returns meta information of node n. -func (t tree) getMeta(n Node) Meta { - return t.infoMap[n].Meta -} From daa26f6e9b6920fce3a0b7730a53bf8ee97d0fd3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 17:30:26 +0300 Subject: [PATCH 0168/1943] [#288] pilorama/test: Check operation order for TreeGetByPath() Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/forest_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 94b14457a..caa999f94 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -322,6 +322,10 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { firstID := lm[2].Child testMeta(t, s, cid, treeID, firstID, lm[2].Parent, Meta{Time: lm[2].Time, Items: meta}) + // TreeAddByPath must return operations in increasing time order. + require.True(t, lm[0].Time < lm[1].Time) + require.True(t, lm[1].Time < lm[2].Time) + meta[0].Value = []byte("YYY") lm, err = s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, []string{"path", "to"}, meta) require.NoError(t, err) From 58f1ba4b518644232463c4e1be26f8dcc77da436 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 17:31:43 +0300 Subject: [PATCH 0169/1943] [#288] pilorama: Add missing operation in log Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/forest.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 4286bdef5..672a38edd 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -89,6 +89,7 @@ func (f *memoryForest) TreeAddByPath(_ context.Context, d CIDDescriptor, treeID }, Child: s.findSpareID(), }) + s.operations = append(s.operations, op) lm[len(lm)-1] = op.Move return lm, nil } From ff255212046f508faa637b04d61418bc5bb39fd4 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 25 Apr 2023 15:09:20 +0300 Subject: [PATCH 0170/1943] [#270] Add IR epoch tick control call Signed-off-by: Alejandro Lopez --- cmd/frostfs-cli/modules/control/ir.go | 15 + .../modules/control/ir_tick_epoch.go | 43 +++ cmd/frostfs-cli/modules/control/root.go | 2 + pkg/innerring/initialization.go | 2 +- .../processors/netmap/process_epoch.go | 2 +- pkg/morph/client/netmap/new_epoch.go | 6 +- pkg/morph/client/notary.go | 18 + pkg/morph/client/static.go | 17 +- pkg/services/control/ir/convert.go | 10 +- pkg/services/control/ir/rpc.go | 22 +- pkg/services/control/ir/server/calls.go | 31 +- pkg/services/control/ir/server/server.go | 10 +- pkg/services/control/ir/service.go | 12 + pkg/services/control/ir/service.pb.go | 334 ++++++++++++++++-- pkg/services/control/ir/service.proto | 16 + pkg/services/control/ir/service_frostfs.pb.go | 142 ++++++++ pkg/services/control/ir/service_grpc.pb.go | 48 ++- pkg/services/control/service.pb.go | 2 +- pkg/services/control/service_grpc.pb.go | 52 ++- pkg/services/tree/service_grpc.pb.go | 54 +-- 20 files changed, 711 insertions(+), 127 deletions(-) create mode 100644 cmd/frostfs-cli/modules/control/ir.go create mode 100644 cmd/frostfs-cli/modules/control/ir_tick_epoch.go diff --git a/cmd/frostfs-cli/modules/control/ir.go b/cmd/frostfs-cli/modules/control/ir.go new file mode 100644 index 000000000..e89dda076 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir.go @@ -0,0 +1,15 @@ +package control + +import "github.com/spf13/cobra" + +var irCmd = &cobra.Command{ + Use: "ir", + Short: "Operations with inner ring nodes", + Long: "Operations with inner ring nodes", +} + +func initControlIRCmd() { + irCmd.AddCommand(tickEpochCmd) + + initControlIRTickEpochCmd() +} diff --git a/cmd/frostfs-cli/modules/control/ir_tick_epoch.go b/cmd/frostfs-cli/modules/control/ir_tick_epoch.go new file mode 100644 index 000000000..3e6af0081 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir_tick_epoch.go @@ -0,0 +1,43 @@ +package control + +import ( + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + "github.com/spf13/cobra" +) + +var tickEpochCmd = &cobra.Command{ + Use: "tick-epoch", + Short: "Forces a new epoch", + Long: "Forces a new epoch via a notary request. It should be executed on other IR nodes as well.", + Run: tickEpoch, +} + +func initControlIRTickEpochCmd() { + initControlFlags(tickEpochCmd) +} + +func tickEpoch(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + c := getClient(cmd, pk) + + req := new(ircontrol.TickEpochRequest) + req.SetBody(new(ircontrol.TickEpochRequest_Body)) + + err := ircontrolsrv.SignMessage(pk, req) + commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) + + var resp *ircontrol.TickEpochResponse + err = c.ExecRaw(func(client *rawclient.Client) error { + resp, err = ircontrol.TickEpoch(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Epoch tick requested") +} diff --git a/cmd/frostfs-cli/modules/control/root.go b/cmd/frostfs-cli/modules/control/root.go index d3b656392..015676185 100644 --- a/cmd/frostfs-cli/modules/control/root.go +++ b/cmd/frostfs-cli/modules/control/root.go @@ -33,6 +33,7 @@ func init() { dropObjectsCmd, shardsCmd, synchronizeTreeCmd, + irCmd, ) initControlHealthCheckCmd() @@ -40,4 +41,5 @@ func init() { initControlDropObjectsCmd() initControlShardsCmd() initControlSynchronizeTreeCmd() + initControlIRCmd() } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 7e0c310fa..38bde1412 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -459,7 +459,7 @@ func (s *Server) initGRPCServer(cfg *viper.Viper) error { p.SetPrivateKey(*s.key) p.SetHealthChecker(s) - controlSvc := controlsrv.New(p, + controlSvc := controlsrv.New(p, s.netmapClient, controlsrv.WithAllowedKeys(authKeys), ) diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index ebf128f82..17e445b13 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -79,7 +79,7 @@ func (np *Processor) processNewEpochTick() { nextEpoch := np.epochState.EpochCounter() + 1 np.log.Debug(logs.NetmapNextEpoch, zap.Uint64("value", nextEpoch)) - err := np.netmapClient.NewEpoch(nextEpoch) + err := np.netmapClient.NewEpoch(nextEpoch, false) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) } diff --git a/pkg/morph/client/netmap/new_epoch.go b/pkg/morph/client/netmap/new_epoch.go index 0b4d31b1d..7a63f14d7 100644 --- a/pkg/morph/client/netmap/new_epoch.go +++ b/pkg/morph/client/netmap/new_epoch.go @@ -8,10 +8,14 @@ import ( // NewEpoch updates FrostFS epoch number through // Netmap contract call. -func (c *Client) NewEpoch(epoch uint64) error { +// If `force` is true, this call is normally initiated by a control +// service command and uses a control notary transaction internally +// to ensure all nodes produce the same transaction with high probability. +func (c *Client) NewEpoch(epoch uint64, force bool) error { prm := client.InvokePrm{} prm.SetMethod(newEpochMethod) prm.SetArgs(epoch) + prm.SetControlTX(force) if err := c.client.Invoke(prm); err != nil { return fmt.Errorf("could not invoke method (%s): %w", newEpochMethod, err) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 427554372..3e21911e1 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -886,6 +886,16 @@ func CalculateNotaryDepositAmount(c *Client, gasMul, gasDiv int64) (fixedn.Fixed // CalculateNonceAndVUB calculates nonce and ValidUntilBlock values // based on transaction hash. func (c *Client) CalculateNonceAndVUB(hash util.Uint256) (nonce uint32, vub uint32, err error) { + return c.calculateNonceAndVUB(hash, false) +} + +// CalculateNonceAndVUBControl calculates nonce and rounded ValidUntilBlock values +// based on transaction hash for use in control transactions. +func (c *Client) CalculateNonceAndVUBControl(hash util.Uint256) (nonce uint32, vub uint32, err error) { + return c.calculateNonceAndVUB(hash, true) +} + +func (c *Client) calculateNonceAndVUB(hash util.Uint256, roundBlockHeight bool) (nonce uint32, vub uint32, err error) { c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -904,6 +914,14 @@ func (c *Client) CalculateNonceAndVUB(hash util.Uint256) (nonce uint32, vub uint return 0, 0, fmt.Errorf("could not get transaction height: %w", err) } + // For control transactions, we round down the block height to control the + // probability of all nodes producing the same transaction, since it depends + // on this value. + if roundBlockHeight { + inc := c.rpcActor.GetVersion().Protocol.MaxValidUntilBlockIncrement + height = height / inc * inc + } + return nonce, height + c.notary.txValidTime, nil } diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index afaf49f3d..910f78537 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -94,6 +94,12 @@ type InvokePrmOptional struct { // `validUntilBlock` values by all notification // receivers. hash *util.Uint256 + // controlTX controls whether the invoke method will use a rounded + // block height value, which is useful for control transactions which + // are required to be produced by all nodes with very high probability. + // It's only used by notary transactions and it affects only the + // computation of `validUntilBlock` values. + controlTX bool } // SetHash sets optional hash of the transaction. @@ -104,6 +110,11 @@ func (i *InvokePrmOptional) SetHash(hash util.Uint256) { i.hash = &hash } +// SetControlTX sets whether a control transaction will be used. +func (i *InvokePrmOptional) SetControlTX(b bool) { + i.controlTX = b +} + // Invoke calls Invoke method of Client with static internal script hash and fee. // Supported args types are the same as in Client. // @@ -126,7 +137,11 @@ func (s StaticClient) Invoke(prm InvokePrm) error { ) if prm.hash != nil { - nonce, vub, err = s.client.CalculateNonceAndVUB(*prm.hash) + if prm.controlTX { + nonce, vub, err = s.client.CalculateNonceAndVUBControl(*prm.hash) + } else { + nonce, vub, err = s.client.CalculateNonceAndVUB(*prm.hash) + } if err != nil { return fmt.Errorf("could not calculate nonce and VUB for notary alphabet invoke: %w", err) } diff --git a/pkg/services/control/ir/convert.go b/pkg/services/control/ir/convert.go index 01bc48724..c892c5b6c 100644 --- a/pkg/services/control/ir/convert.go +++ b/pkg/services/control/ir/convert.go @@ -14,18 +14,18 @@ func (w *requestWrapper) ToGRPCMessage() grpc.Message { return w.m } -type healthCheckResponseWrapper struct { - m *HealthCheckResponse +type responseWrapper[M grpc.Message] struct { + m M } -func (w *healthCheckResponseWrapper) ToGRPCMessage() grpc.Message { +func (w *responseWrapper[M]) ToGRPCMessage() grpc.Message { return w.m } -func (w *healthCheckResponseWrapper) FromGRPCMessage(m grpc.Message) error { +func (w *responseWrapper[M]) FromGRPCMessage(m grpc.Message) error { var ok bool - w.m, ok = m.(*HealthCheckResponse) + w.m, ok = m.(M) if !ok { return message.NewUnexpectedMessageType(m, w.m) } diff --git a/pkg/services/control/ir/rpc.go b/pkg/services/control/ir/rpc.go index a8b16b607..6b2234954 100644 --- a/pkg/services/control/ir/rpc.go +++ b/pkg/services/control/ir/rpc.go @@ -3,12 +3,14 @@ package control import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/common" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/grpc" ) const serviceName = "ircontrol.ControlService" const ( rpcHealthCheck = "HealthCheck" + rpcTickEpoch = "TickEpoch" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -17,15 +19,29 @@ func HealthCheck( req *HealthCheckRequest, opts ...client.CallOption, ) (*HealthCheckResponse, error) { - wResp := &healthCheckResponseWrapper{ - m: new(HealthCheckResponse), + return sendUnary[HealthCheckRequest, HealthCheckResponse](cli, rpcHealthCheck, req, opts...) +} + +// TickEpoch executes ControlService.TickEpoch RPC. +func TickEpoch( + cli *client.Client, + req *TickEpochRequest, + opts ...client.CallOption, +) (*TickEpochResponse, error) { + return sendUnary[TickEpochRequest, TickEpochResponse](cli, rpcTickEpoch, req, opts...) +} + +func sendUnary[I, O grpc.Message](cli *client.Client, rpcName string, req *I, opts ...client.CallOption) (*O, error) { + var resp O + wResp := &responseWrapper[*O]{ + m: &resp, } wReq := &requestWrapper{ m: req, } - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcHealthCheck), wReq, wResp, opts...) + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcName), wReq, wResp, opts...) if err != nil { return nil, err } diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 986da90f1..56e2e3f79 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -2,6 +2,7 @@ package control import ( "context" + "fmt" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "google.golang.org/grpc/codes" @@ -12,12 +13,10 @@ import ( // // If request is not signed with a key from white list, permission error returns. func (s *Server) HealthCheck(_ context.Context, req *control.HealthCheckRequest) (*control.HealthCheckResponse, error) { - // verify request if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) } - // create and fill response resp := new(control.HealthCheckResponse) body := new(control.HealthCheckResponse_Body) @@ -25,7 +24,33 @@ func (s *Server) HealthCheck(_ context.Context, req *control.HealthCheckRequest) body.SetHealthStatus(s.prm.healthChecker.HealthStatus()) - // sign the response + if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return resp, nil +} + +// TickEpoch forces a new epoch. +// +// If request is not signed with a key from white list, permission error returns. +func (s *Server) TickEpoch(_ context.Context, req *control.TickEpochRequest) (*control.TickEpochResponse, error) { + if err := s.isValidRequest(req); err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + resp := new(control.TickEpochResponse) + resp.SetBody(new(control.TickEpochResponse_Body)) + + epoch, err := s.netmapClient.Epoch() + if err != nil { + return nil, fmt.Errorf("getting current epoch: %w", err) + } + + if err := s.netmapClient.NewEpoch(epoch+1, true); err != nil { + return nil, fmt.Errorf("forcing new epoch: %w", err) + } + if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/ir/server/server.go b/pkg/services/control/ir/server/server.go index c75c1504e..dc00809a6 100644 --- a/pkg/services/control/ir/server/server.go +++ b/pkg/services/control/ir/server/server.go @@ -2,6 +2,8 @@ package control import ( "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" ) // Server is an entity that serves @@ -10,7 +12,8 @@ import ( // To gain access to the service, any request must be // signed with a key from the white list. type Server struct { - prm Prm + prm Prm + netmapClient *netmap.Client allowedKeys [][]byte } @@ -29,7 +32,7 @@ func panicOnPrmValue(n string, v any) { // Forms white list from all keys specified via // WithAllowedKeys option and a public key of // the parameterized private key. -func New(prm Prm, opts ...Option) *Server { +func New(prm Prm, netmapClient *netmap.Client, opts ...Option) *Server { // verify required parameters switch { case prm.healthChecker == nil: @@ -44,7 +47,8 @@ func New(prm Prm, opts ...Option) *Server { } return &Server{ - prm: prm, + prm: prm, + netmapClient: netmapClient, allowedKeys: append(o.allowedKeys, prm.key.PublicKey().Bytes()), } diff --git a/pkg/services/control/ir/service.go b/pkg/services/control/ir/service.go index dc04e4904..1aaec2c87 100644 --- a/pkg/services/control/ir/service.go +++ b/pkg/services/control/ir/service.go @@ -20,3 +20,15 @@ func (x *HealthCheckResponse) SetBody(v *HealthCheckResponse_Body) { x.Body = v } } + +func (x *TickEpochRequest) SetBody(v *TickEpochRequest_Body) { + if x != nil { + x.Body = v + } +} + +func (x *TickEpochResponse) SetBody(v *TickEpochResponse_Body) { + if x != nil { + x.Body = v + } +} diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index 44ea2dd66..84acdfc82 100644 --- a/pkg/services/control/ir/service.pb.go +++ b/pkg/services/control/ir/service.pb.go @@ -138,6 +138,116 @@ func (x *HealthCheckResponse) GetSignature() *Signature { return nil } +type TickEpochRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *TickEpochRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *TickEpochRequest) Reset() { + *x = TickEpochRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TickEpochRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TickEpochRequest) ProtoMessage() {} + +func (x *TickEpochRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TickEpochRequest.ProtoReflect.Descriptor instead. +func (*TickEpochRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{2} +} + +func (x *TickEpochRequest) GetBody() *TickEpochRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *TickEpochRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +type TickEpochResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *TickEpochResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *TickEpochResponse) Reset() { + *x = TickEpochResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TickEpochResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TickEpochResponse) ProtoMessage() {} + +func (x *TickEpochResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TickEpochResponse.ProtoReflect.Descriptor instead. +func (*TickEpochResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{3} +} + +func (x *TickEpochResponse) GetBody() *TickEpochResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *TickEpochResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + // Health check request body. type HealthCheckRequest_Body struct { state protoimpl.MessageState @@ -148,7 +258,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[2] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -161,7 +271,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[2] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -190,7 +300,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[3] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -203,7 +313,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[3] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -226,6 +336,82 @@ func (x *HealthCheckResponse_Body) GetHealthStatus() HealthStatus { return HealthStatus_HEALTH_STATUS_UNDEFINED } +type TickEpochRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *TickEpochRequest_Body) Reset() { + *x = TickEpochRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TickEpochRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TickEpochRequest_Body) ProtoMessage() {} + +func (x *TickEpochRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TickEpochRequest_Body.ProtoReflect.Descriptor instead. +func (*TickEpochRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{2, 0} +} + +type TickEpochResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *TickEpochResponse_Body) Reset() { + *x = TickEpochResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TickEpochResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TickEpochResponse_Body) ProtoMessage() {} + +func (x *TickEpochResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TickEpochResponse_Body.ProtoReflect.Descriptor instead. +func (*TickEpochResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{3, 0} +} + var File_pkg_services_control_ir_service_proto protoreflect.FileDescriptor var file_pkg_services_control_ir_service_proto_rawDesc = []byte{ @@ -255,18 +441,40 @@ var file_pkg_services_control_ir_service_proto_rawDesc = []byte{ 0x3c, 0x0a, 0x0d, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x0c, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x32, 0x5e, 0x0a, - 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, - 0x4c, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1d, + 0x0c, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x84, 0x01, + 0x0a, 0x10, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x20, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, + 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x22, 0x86, 0x01, 0x0a, 0x11, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, + 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xa6, 0x01, + 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x4c, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, + 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, - 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x44, 0x5a, - 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, - 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, - 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x69, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, + 0x0a, 0x09, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x1b, 0x2e, 0x69, 0x72, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, + 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, + 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, + 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, + 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2f, 0x69, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -281,28 +489,38 @@ func file_pkg_services_control_ir_service_proto_rawDescGZIP() []byte { return file_pkg_services_control_ir_service_proto_rawDescData } -var file_pkg_services_control_ir_service_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_pkg_services_control_ir_service_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_pkg_services_control_ir_service_proto_goTypes = []interface{}{ (*HealthCheckRequest)(nil), // 0: ircontrol.HealthCheckRequest (*HealthCheckResponse)(nil), // 1: ircontrol.HealthCheckResponse - (*HealthCheckRequest_Body)(nil), // 2: ircontrol.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 3: ircontrol.HealthCheckResponse.Body - (*Signature)(nil), // 4: ircontrol.Signature - (HealthStatus)(0), // 5: ircontrol.HealthStatus + (*TickEpochRequest)(nil), // 2: ircontrol.TickEpochRequest + (*TickEpochResponse)(nil), // 3: ircontrol.TickEpochResponse + (*HealthCheckRequest_Body)(nil), // 4: ircontrol.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 5: ircontrol.HealthCheckResponse.Body + (*TickEpochRequest_Body)(nil), // 6: ircontrol.TickEpochRequest.Body + (*TickEpochResponse_Body)(nil), // 7: ircontrol.TickEpochResponse.Body + (*Signature)(nil), // 8: ircontrol.Signature + (HealthStatus)(0), // 9: ircontrol.HealthStatus } var file_pkg_services_control_ir_service_proto_depIdxs = []int32{ - 2, // 0: ircontrol.HealthCheckRequest.body:type_name -> ircontrol.HealthCheckRequest.Body - 4, // 1: ircontrol.HealthCheckRequest.signature:type_name -> ircontrol.Signature - 3, // 2: ircontrol.HealthCheckResponse.body:type_name -> ircontrol.HealthCheckResponse.Body - 4, // 3: ircontrol.HealthCheckResponse.signature:type_name -> ircontrol.Signature - 5, // 4: ircontrol.HealthCheckResponse.Body.health_status:type_name -> ircontrol.HealthStatus - 0, // 5: ircontrol.ControlService.HealthCheck:input_type -> ircontrol.HealthCheckRequest - 1, // 6: ircontrol.ControlService.HealthCheck:output_type -> ircontrol.HealthCheckResponse - 6, // [6:7] is the sub-list for method output_type - 5, // [5:6] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 4, // 0: ircontrol.HealthCheckRequest.body:type_name -> ircontrol.HealthCheckRequest.Body + 8, // 1: ircontrol.HealthCheckRequest.signature:type_name -> ircontrol.Signature + 5, // 2: ircontrol.HealthCheckResponse.body:type_name -> ircontrol.HealthCheckResponse.Body + 8, // 3: ircontrol.HealthCheckResponse.signature:type_name -> ircontrol.Signature + 6, // 4: ircontrol.TickEpochRequest.body:type_name -> ircontrol.TickEpochRequest.Body + 8, // 5: ircontrol.TickEpochRequest.signature:type_name -> ircontrol.Signature + 7, // 6: ircontrol.TickEpochResponse.body:type_name -> ircontrol.TickEpochResponse.Body + 8, // 7: ircontrol.TickEpochResponse.signature:type_name -> ircontrol.Signature + 9, // 8: ircontrol.HealthCheckResponse.Body.health_status:type_name -> ircontrol.HealthStatus + 0, // 9: ircontrol.ControlService.HealthCheck:input_type -> ircontrol.HealthCheckRequest + 2, // 10: ircontrol.ControlService.TickEpoch:input_type -> ircontrol.TickEpochRequest + 1, // 11: ircontrol.ControlService.HealthCheck:output_type -> ircontrol.HealthCheckResponse + 3, // 12: ircontrol.ControlService.TickEpoch:output_type -> ircontrol.TickEpochResponse + 11, // [11:13] is the sub-list for method output_type + 9, // [9:11] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_pkg_services_control_ir_service_proto_init() } @@ -337,7 +555,7 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*TickEpochRequest); i { case 0: return &v.state case 1: @@ -349,6 +567,30 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TickEpochResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_ir_service_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HealthCheckRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_ir_service_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state @@ -360,6 +602,30 @@ func file_pkg_services_control_ir_service_proto_init() { return nil } } + file_pkg_services_control_ir_service_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TickEpochRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_ir_service_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TickEpochResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -367,7 +633,7 @@ func file_pkg_services_control_ir_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_ir_service_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 8, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/ir/service.proto b/pkg/services/control/ir/service.proto index 5f99be16b..5862e8fbd 100644 --- a/pkg/services/control/ir/service.proto +++ b/pkg/services/control/ir/service.proto @@ -10,6 +10,8 @@ option go_package = "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/ir/ service ControlService { // Performs health check of the IR node. rpc HealthCheck (HealthCheckRequest) returns (HealthCheckResponse); + // Forces a new epoch to be signaled by the IR node with high probability. + rpc TickEpoch (TickEpochRequest) returns (TickEpochResponse); } // Health check request. @@ -41,3 +43,17 @@ message HealthCheckResponse { // Body signature. Signature signature = 2; } + +message TickEpochRequest { + message Body{} + + Body body = 1; + Signature signature = 2; +} + +message TickEpochResponse { + message Body{} + + Body body = 1; + Signature signature = 2; +} diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index f6dd94b3a..d480f0b50 100644 --- a/pkg/services/control/ir/service_frostfs.pb.go +++ b/pkg/services/control/ir/service_frostfs.pb.go @@ -154,3 +154,145 @@ func (x *HealthCheckResponse) ReadSignedData(buf []byte) ([]byte, error) { func (x *HealthCheckResponse) SetSignature(sig *Signature) { x.Signature = sig } + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *TickEpochRequest_Body) StableSize() (size int) { + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *TickEpochRequest_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *TickEpochRequest) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *TickEpochRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *TickEpochRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *TickEpochRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *TickEpochRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *TickEpochResponse_Body) StableSize() (size int) { + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *TickEpochResponse_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *TickEpochResponse) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *TickEpochResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *TickEpochResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *TickEpochResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *TickEpochResponse) SetSignature(sig *Signature) { + x.Signature = sig +} diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index bdcac73e5..700d340ca 100644 --- a/pkg/services/control/ir/service_grpc.pb.go +++ b/pkg/services/control/ir/service_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.3.0 +// - protoc-gen-go-grpc v1.2.0 // - protoc v3.12.4 // source: pkg/services/control/ir/service.proto @@ -18,16 +18,14 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -const ( - ControlService_HealthCheck_FullMethodName = "/ircontrol.ControlService/HealthCheck" -) - // ControlServiceClient is the client API for ControlService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type ControlServiceClient interface { // Performs health check of the IR node. HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) + // Forces a new epoch to be signaled by the IR node with high probability. + TickEpoch(ctx context.Context, in *TickEpochRequest, opts ...grpc.CallOption) (*TickEpochResponse, error) } type controlServiceClient struct { @@ -40,7 +38,16 @@ func NewControlServiceClient(cc grpc.ClientConnInterface) ControlServiceClient { func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { out := new(HealthCheckResponse) - err := c.cc.Invoke(ctx, ControlService_HealthCheck_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/ircontrol.ControlService/HealthCheck", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlServiceClient) TickEpoch(ctx context.Context, in *TickEpochRequest, opts ...grpc.CallOption) (*TickEpochResponse, error) { + out := new(TickEpochResponse) + err := c.cc.Invoke(ctx, "/ircontrol.ControlService/TickEpoch", in, out, opts...) if err != nil { return nil, err } @@ -53,6 +60,8 @@ func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckR type ControlServiceServer interface { // Performs health check of the IR node. HealthCheck(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) + // Forces a new epoch to be signaled by the IR node with high probability. + TickEpoch(context.Context, *TickEpochRequest) (*TickEpochResponse, error) } // UnimplementedControlServiceServer should be embedded to have forward compatible implementations. @@ -62,6 +71,9 @@ type UnimplementedControlServiceServer struct { func (UnimplementedControlServiceServer) HealthCheck(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method HealthCheck not implemented") } +func (UnimplementedControlServiceServer) TickEpoch(context.Context, *TickEpochRequest) (*TickEpochResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method TickEpoch not implemented") +} // UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ControlServiceServer will @@ -84,7 +96,7 @@ func _ControlService_HealthCheck_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_HealthCheck_FullMethodName, + FullMethod: "/ircontrol.ControlService/HealthCheck", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).HealthCheck(ctx, req.(*HealthCheckRequest)) @@ -92,6 +104,24 @@ func _ControlService_HealthCheck_Handler(srv interface{}, ctx context.Context, d return interceptor(ctx, in, info, handler) } +func _ControlService_TickEpoch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(TickEpochRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).TickEpoch(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ircontrol.ControlService/TickEpoch", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).TickEpoch(ctx, req.(*TickEpochRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -103,6 +133,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "HealthCheck", Handler: _ControlService_HealthCheck_Handler, }, + { + MethodName: "TickEpoch", + Handler: _ControlService_TickEpoch_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "pkg/services/control/ir/service.proto", diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index d713bb38d..a126ce16d 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.21.12 +// protoc v3.12.4 // source: pkg/services/control/service.proto package control diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 1e8dd9e3c..3fa1e54de 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.12 +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.12.4 // source: pkg/services/control/service.proto package control @@ -18,18 +18,6 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -const ( - ControlService_HealthCheck_FullMethodName = "/control.ControlService/HealthCheck" - ControlService_SetNetmapStatus_FullMethodName = "/control.ControlService/SetNetmapStatus" - ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects" - ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" - ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" - ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree" - ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard" - ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache" - ControlService_Doctor_FullMethodName = "/control.ControlService/Doctor" -) - // ControlServiceClient is the client API for ControlService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -64,7 +52,7 @@ func NewControlServiceClient(cc grpc.ClientConnInterface) ControlServiceClient { func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { out := new(HealthCheckResponse) - err := c.cc.Invoke(ctx, ControlService_HealthCheck_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/control.ControlService/HealthCheck", in, out, opts...) if err != nil { return nil, err } @@ -73,7 +61,7 @@ func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckR func (c *controlServiceClient) SetNetmapStatus(ctx context.Context, in *SetNetmapStatusRequest, opts ...grpc.CallOption) (*SetNetmapStatusResponse, error) { out := new(SetNetmapStatusResponse) - err := c.cc.Invoke(ctx, ControlService_SetNetmapStatus_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/control.ControlService/SetNetmapStatus", in, out, opts...) if err != nil { return nil, err } @@ -82,7 +70,7 @@ func (c *controlServiceClient) SetNetmapStatus(ctx context.Context, in *SetNetma func (c *controlServiceClient) DropObjects(ctx context.Context, in *DropObjectsRequest, opts ...grpc.CallOption) (*DropObjectsResponse, error) { out := new(DropObjectsResponse) - err := c.cc.Invoke(ctx, ControlService_DropObjects_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/control.ControlService/DropObjects", in, out, opts...) if err != nil { return nil, err } @@ -91,7 +79,7 @@ func (c *controlServiceClient) DropObjects(ctx context.Context, in *DropObjectsR func (c *controlServiceClient) ListShards(ctx context.Context, in *ListShardsRequest, opts ...grpc.CallOption) (*ListShardsResponse, error) { out := new(ListShardsResponse) - err := c.cc.Invoke(ctx, ControlService_ListShards_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/control.ControlService/ListShards", in, out, opts...) if err != nil { return nil, err } @@ -100,7 +88,7 @@ func (c *controlServiceClient) ListShards(ctx context.Context, in *ListShardsReq func (c *controlServiceClient) SetShardMode(ctx context.Context, in *SetShardModeRequest, opts ...grpc.CallOption) (*SetShardModeResponse, error) { out := new(SetShardModeResponse) - err := c.cc.Invoke(ctx, ControlService_SetShardMode_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/control.ControlService/SetShardMode", in, out, opts...) if err != nil { return nil, err } @@ -109,7 +97,7 @@ func (c *controlServiceClient) SetShardMode(ctx context.Context, in *SetShardMod func (c *controlServiceClient) SynchronizeTree(ctx context.Context, in *SynchronizeTreeRequest, opts ...grpc.CallOption) (*SynchronizeTreeResponse, error) { out := new(SynchronizeTreeResponse) - err := c.cc.Invoke(ctx, ControlService_SynchronizeTree_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/control.ControlService/SynchronizeTree", in, out, opts...) if err != nil { return nil, err } @@ -118,7 +106,7 @@ func (c *controlServiceClient) SynchronizeTree(ctx context.Context, in *Synchron func (c *controlServiceClient) EvacuateShard(ctx context.Context, in *EvacuateShardRequest, opts ...grpc.CallOption) (*EvacuateShardResponse, error) { out := new(EvacuateShardResponse) - err := c.cc.Invoke(ctx, ControlService_EvacuateShard_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/control.ControlService/EvacuateShard", in, out, opts...) if err != nil { return nil, err } @@ -127,7 +115,7 @@ func (c *controlServiceClient) EvacuateShard(ctx context.Context, in *EvacuateSh func (c *controlServiceClient) FlushCache(ctx context.Context, in *FlushCacheRequest, opts ...grpc.CallOption) (*FlushCacheResponse, error) { out := new(FlushCacheResponse) - err := c.cc.Invoke(ctx, ControlService_FlushCache_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/control.ControlService/FlushCache", in, out, opts...) if err != nil { return nil, err } @@ -136,7 +124,7 @@ func (c *controlServiceClient) FlushCache(ctx context.Context, in *FlushCacheReq func (c *controlServiceClient) Doctor(ctx context.Context, in *DoctorRequest, opts ...grpc.CallOption) (*DoctorResponse, error) { out := new(DoctorResponse) - err := c.cc.Invoke(ctx, ControlService_Doctor_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/control.ControlService/Doctor", in, out, opts...) if err != nil { return nil, err } @@ -220,7 +208,7 @@ func _ControlService_HealthCheck_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_HealthCheck_FullMethodName, + FullMethod: "/control.ControlService/HealthCheck", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).HealthCheck(ctx, req.(*HealthCheckRequest)) @@ -238,7 +226,7 @@ func _ControlService_SetNetmapStatus_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_SetNetmapStatus_FullMethodName, + FullMethod: "/control.ControlService/SetNetmapStatus", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).SetNetmapStatus(ctx, req.(*SetNetmapStatusRequest)) @@ -256,7 +244,7 @@ func _ControlService_DropObjects_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_DropObjects_FullMethodName, + FullMethod: "/control.ControlService/DropObjects", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).DropObjects(ctx, req.(*DropObjectsRequest)) @@ -274,7 +262,7 @@ func _ControlService_ListShards_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_ListShards_FullMethodName, + FullMethod: "/control.ControlService/ListShards", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).ListShards(ctx, req.(*ListShardsRequest)) @@ -292,7 +280,7 @@ func _ControlService_SetShardMode_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_SetShardMode_FullMethodName, + FullMethod: "/control.ControlService/SetShardMode", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).SetShardMode(ctx, req.(*SetShardModeRequest)) @@ -310,7 +298,7 @@ func _ControlService_SynchronizeTree_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_SynchronizeTree_FullMethodName, + FullMethod: "/control.ControlService/SynchronizeTree", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).SynchronizeTree(ctx, req.(*SynchronizeTreeRequest)) @@ -328,7 +316,7 @@ func _ControlService_EvacuateShard_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_EvacuateShard_FullMethodName, + FullMethod: "/control.ControlService/EvacuateShard", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).EvacuateShard(ctx, req.(*EvacuateShardRequest)) @@ -346,7 +334,7 @@ func _ControlService_FlushCache_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_FlushCache_FullMethodName, + FullMethod: "/control.ControlService/FlushCache", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).FlushCache(ctx, req.(*FlushCacheRequest)) @@ -364,7 +352,7 @@ func _ControlService_Doctor_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_Doctor_FullMethodName, + FullMethod: "/control.ControlService/Doctor", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).Doctor(ctx, req.(*DoctorRequest)) diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index f981746de..fa259e804 100644 --- a/pkg/services/tree/service_grpc.pb.go +++ b/pkg/services/tree/service_grpc.pb.go @@ -1,9 +1,6 @@ -//* -// Service for working with CRDT tree. - // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.3.0 +// - protoc-gen-go-grpc v1.2.0 // - protoc v3.12.4 // source: pkg/services/tree/service.proto @@ -21,19 +18,6 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -const ( - TreeService_Add_FullMethodName = "/tree.TreeService/Add" - TreeService_AddByPath_FullMethodName = "/tree.TreeService/AddByPath" - TreeService_Remove_FullMethodName = "/tree.TreeService/Remove" - TreeService_Move_FullMethodName = "/tree.TreeService/Move" - TreeService_GetNodeByPath_FullMethodName = "/tree.TreeService/GetNodeByPath" - TreeService_GetSubTree_FullMethodName = "/tree.TreeService/GetSubTree" - TreeService_TreeList_FullMethodName = "/tree.TreeService/TreeList" - TreeService_Apply_FullMethodName = "/tree.TreeService/Apply" - TreeService_GetOpLog_FullMethodName = "/tree.TreeService/GetOpLog" - TreeService_Healthcheck_FullMethodName = "/tree.TreeService/Healthcheck" -) - // TreeServiceClient is the client API for TreeService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -71,7 +55,7 @@ func NewTreeServiceClient(cc grpc.ClientConnInterface) TreeServiceClient { func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*AddResponse, error) { out := new(AddResponse) - err := c.cc.Invoke(ctx, TreeService_Add_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/Add", in, out, opts...) if err != nil { return nil, err } @@ -80,7 +64,7 @@ func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grp func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, opts ...grpc.CallOption) (*AddByPathResponse, error) { out := new(AddByPathResponse) - err := c.cc.Invoke(ctx, TreeService_AddByPath_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/AddByPath", in, out, opts...) if err != nil { return nil, err } @@ -89,7 +73,7 @@ func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts ...grpc.CallOption) (*RemoveResponse, error) { out := new(RemoveResponse) - err := c.cc.Invoke(ctx, TreeService_Remove_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/Remove", in, out, opts...) if err != nil { return nil, err } @@ -98,7 +82,7 @@ func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...grpc.CallOption) (*MoveResponse, error) { out := new(MoveResponse) - err := c.cc.Invoke(ctx, TreeService_Move_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/Move", in, out, opts...) if err != nil { return nil, err } @@ -107,7 +91,7 @@ func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...g func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPathRequest, opts ...grpc.CallOption) (*GetNodeByPathResponse, error) { out := new(GetNodeByPathResponse) - err := c.cc.Invoke(ctx, TreeService_GetNodeByPath_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/GetNodeByPath", in, out, opts...) if err != nil { return nil, err } @@ -115,7 +99,7 @@ func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPath } func (c *treeServiceClient) GetSubTree(ctx context.Context, in *GetSubTreeRequest, opts ...grpc.CallOption) (TreeService_GetSubTreeClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], TreeService_GetSubTree_FullMethodName, opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], "/tree.TreeService/GetSubTree", opts...) if err != nil { return nil, err } @@ -148,7 +132,7 @@ func (x *treeServiceGetSubTreeClient) Recv() (*GetSubTreeResponse, error) { func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, opts ...grpc.CallOption) (*TreeListResponse, error) { out := new(TreeListResponse) - err := c.cc.Invoke(ctx, TreeService_TreeList_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/TreeList", in, out, opts...) if err != nil { return nil, err } @@ -157,7 +141,7 @@ func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, o func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts ...grpc.CallOption) (*ApplyResponse, error) { out := new(ApplyResponse) - err := c.cc.Invoke(ctx, TreeService_Apply_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/Apply", in, out, opts...) if err != nil { return nil, err } @@ -165,7 +149,7 @@ func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts .. } func (c *treeServiceClient) GetOpLog(ctx context.Context, in *GetOpLogRequest, opts ...grpc.CallOption) (TreeService_GetOpLogClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], TreeService_GetOpLog_FullMethodName, opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], "/tree.TreeService/GetOpLog", opts...) if err != nil { return nil, err } @@ -198,7 +182,7 @@ func (x *treeServiceGetOpLogClient) Recv() (*GetOpLogResponse, error) { func (c *treeServiceClient) Healthcheck(ctx context.Context, in *HealthcheckRequest, opts ...grpc.CallOption) (*HealthcheckResponse, error) { out := new(HealthcheckResponse) - err := c.cc.Invoke(ctx, TreeService_Healthcheck_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/Healthcheck", in, out, opts...) if err != nil { return nil, err } @@ -288,7 +272,7 @@ func _TreeService_Add_Handler(srv interface{}, ctx context.Context, dec func(int } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_Add_FullMethodName, + FullMethod: "/tree.TreeService/Add", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Add(ctx, req.(*AddRequest)) @@ -306,7 +290,7 @@ func _TreeService_AddByPath_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_AddByPath_FullMethodName, + FullMethod: "/tree.TreeService/AddByPath", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).AddByPath(ctx, req.(*AddByPathRequest)) @@ -324,7 +308,7 @@ func _TreeService_Remove_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_Remove_FullMethodName, + FullMethod: "/tree.TreeService/Remove", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Remove(ctx, req.(*RemoveRequest)) @@ -342,7 +326,7 @@ func _TreeService_Move_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_Move_FullMethodName, + FullMethod: "/tree.TreeService/Move", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Move(ctx, req.(*MoveRequest)) @@ -360,7 +344,7 @@ func _TreeService_GetNodeByPath_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_GetNodeByPath_FullMethodName, + FullMethod: "/tree.TreeService/GetNodeByPath", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).GetNodeByPath(ctx, req.(*GetNodeByPathRequest)) @@ -399,7 +383,7 @@ func _TreeService_TreeList_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_TreeList_FullMethodName, + FullMethod: "/tree.TreeService/TreeList", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).TreeList(ctx, req.(*TreeListRequest)) @@ -417,7 +401,7 @@ func _TreeService_Apply_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_Apply_FullMethodName, + FullMethod: "/tree.TreeService/Apply", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Apply(ctx, req.(*ApplyRequest)) @@ -456,7 +440,7 @@ func _TreeService_Healthcheck_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_Healthcheck_FullMethodName, + FullMethod: "/tree.TreeService/Healthcheck", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Healthcheck(ctx, req.(*HealthcheckRequest)) From f73ac6e02d06594dae206817fcd38635be08d166 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 27 Apr 2023 13:38:25 +0300 Subject: [PATCH 0171/1943] [#290] control: Use generics for response wrappers Signed-off-by: Evgenii Stratonikov --- pkg/services/control/convert.go | 171 +++----------------------------- pkg/services/control/rpc.go | 47 ++++----- 2 files changed, 31 insertions(+), 187 deletions(-) diff --git a/pkg/services/control/convert.go b/pkg/services/control/convert.go index 84bde31d6..fd6f020d1 100644 --- a/pkg/services/control/convert.go +++ b/pkg/services/control/convert.go @@ -14,171 +14,26 @@ func (w *requestWrapper) ToGRPCMessage() grpc.Message { return w.m } -type healthCheckResponseWrapper struct { - m *HealthCheckResponse +type responseWrapper[T grpc.Message] struct { + message *T } -func (w *healthCheckResponseWrapper) ToGRPCMessage() grpc.Message { - return w.m +func newResponseWrapper[T grpc.Message]() *responseWrapper[T] { + return &responseWrapper[T]{ + message: new(T), + } } -func (w *healthCheckResponseWrapper) FromGRPCMessage(m grpc.Message) error { - var ok bool +func (w *responseWrapper[T]) ToGRPCMessage() grpc.Message { + return w.message +} - w.m, ok = m.(*HealthCheckResponse) +func (w *responseWrapper[T]) FromGRPCMessage(m grpc.Message) error { + response, ok := m.(*T) if !ok { - return message.NewUnexpectedMessageType(m, w.m) + return message.NewUnexpectedMessageType(m, w.message) } - return nil -} - -type setNetmapStatusResponseWrapper struct { - message.Message - m *SetNetmapStatusResponse -} - -func (w *setNetmapStatusResponseWrapper) ToGRPCMessage() grpc.Message { - return w.m -} - -func (w *setNetmapStatusResponseWrapper) FromGRPCMessage(m grpc.Message) error { - var ok bool - - w.m, ok = m.(*SetNetmapStatusResponse) - if !ok { - return message.NewUnexpectedMessageType(m, w.m) - } - - return nil -} - -type dropObjectsResponseWrapper struct { - message.Message - m *DropObjectsResponse -} - -func (w *dropObjectsResponseWrapper) ToGRPCMessage() grpc.Message { - return w.m -} - -func (w *dropObjectsResponseWrapper) FromGRPCMessage(m grpc.Message) error { - var ok bool - - w.m, ok = m.(*DropObjectsResponse) - if !ok { - return message.NewUnexpectedMessageType(m, w.m) - } - - return nil -} - -type listShardsResponseWrapper struct { - m *ListShardsResponse -} - -func (w *listShardsResponseWrapper) ToGRPCMessage() grpc.Message { - return w.m -} - -func (w *listShardsResponseWrapper) FromGRPCMessage(m grpc.Message) error { - var ok bool - - w.m, ok = m.(*ListShardsResponse) - if !ok { - return message.NewUnexpectedMessageType(m, w.m) - } - - return nil -} - -type setShardModeResponseWrapper struct { - m *SetShardModeResponse -} - -func (w *setShardModeResponseWrapper) ToGRPCMessage() grpc.Message { - return w.m -} - -func (w *setShardModeResponseWrapper) FromGRPCMessage(m grpc.Message) error { - var ok bool - - w.m, ok = m.(*SetShardModeResponse) - if !ok { - return message.NewUnexpectedMessageType(m, w.m) - } - - return nil -} - -type synchronizeTreeResponseWrapper struct { - *SynchronizeTreeResponse -} - -func (w *synchronizeTreeResponseWrapper) ToGRPCMessage() grpc.Message { - return w.SynchronizeTreeResponse -} - -func (w *synchronizeTreeResponseWrapper) FromGRPCMessage(m grpc.Message) error { - r, ok := m.(*SynchronizeTreeResponse) - if !ok { - return message.NewUnexpectedMessageType(m, (*SynchronizeTreeResponse)(nil)) - } - - w.SynchronizeTreeResponse = r - return nil -} - -type evacuateShardResponseWrapper struct { - *EvacuateShardResponse -} - -func (w *evacuateShardResponseWrapper) ToGRPCMessage() grpc.Message { - return w.EvacuateShardResponse -} - -func (w *evacuateShardResponseWrapper) FromGRPCMessage(m grpc.Message) error { - r, ok := m.(*EvacuateShardResponse) - if !ok { - return message.NewUnexpectedMessageType(m, (*EvacuateShardResponse)(nil)) - } - - w.EvacuateShardResponse = r - return nil -} - -type flushCacheResponseWrapper struct { - *FlushCacheResponse -} - -func (w *flushCacheResponseWrapper) ToGRPCMessage() grpc.Message { - return w.FlushCacheResponse -} - -func (w *flushCacheResponseWrapper) FromGRPCMessage(m grpc.Message) error { - r, ok := m.(*FlushCacheResponse) - if !ok { - return message.NewUnexpectedMessageType(m, (*FlushCacheResponse)(nil)) - } - - w.FlushCacheResponse = r - return nil -} - -type doctorResponseWrapper struct { - *DoctorResponse -} - -func (w *doctorResponseWrapper) ToGRPCMessage() grpc.Message { - return w.DoctorResponse -} - -func (w *doctorResponseWrapper) FromGRPCMessage(m grpc.Message) error { - r, ok := m.(*DoctorResponse) - if !ok { - return message.NewUnexpectedMessageType(m, (*DoctorResponse)(nil)) - } - - w.DoctorResponse = r + w.message = response return nil } diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 625f485c9..31ebfa760 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -25,10 +25,7 @@ func HealthCheck( req *HealthCheckRequest, opts ...client.CallOption, ) (*HealthCheckResponse, error) { - wResp := &healthCheckResponseWrapper{ - m: new(HealthCheckResponse), - } - + wResp := newResponseWrapper[HealthCheckResponse]() wReq := &requestWrapper{ m: req, } @@ -38,7 +35,7 @@ func HealthCheck( return nil, err } - return wResp.m, nil + return wResp.message, nil } // SetNetmapStatus executes ControlService.SetNetmapStatus RPC. @@ -47,9 +44,7 @@ func SetNetmapStatus( req *SetNetmapStatusRequest, opts ...client.CallOption, ) (*SetNetmapStatusResponse, error) { - wResp := &setNetmapStatusResponseWrapper{ - m: new(SetNetmapStatusResponse), - } + wResp := newResponseWrapper[SetNetmapStatusResponse]() wReq := &requestWrapper{ m: req, @@ -60,7 +55,7 @@ func SetNetmapStatus( return nil, err } - return wResp.m, nil + return wResp.message, nil } // DropObjects executes ControlService.DropObjects RPC. @@ -69,9 +64,7 @@ func DropObjects( req *DropObjectsRequest, opts ...client.CallOption, ) (*DropObjectsResponse, error) { - wResp := &dropObjectsResponseWrapper{ - m: new(DropObjectsResponse), - } + wResp := newResponseWrapper[DropObjectsResponse]() wReq := &requestWrapper{ m: req, @@ -81,7 +74,7 @@ func DropObjects( return nil, err } - return wResp.m, nil + return wResp.message, nil } // ListShards executes ControlService.ListShards RPC. @@ -90,9 +83,7 @@ func ListShards( req *ListShardsRequest, opts ...client.CallOption, ) (*ListShardsResponse, error) { - wResp := &listShardsResponseWrapper{ - m: new(ListShardsResponse), - } + wResp := newResponseWrapper[ListShardsResponse]() wReq := &requestWrapper{ m: req, @@ -102,7 +93,7 @@ func ListShards( return nil, err } - return wResp.m, nil + return wResp.message, nil } // SetShardMode executes ControlService.SetShardMode RPC. @@ -111,9 +102,7 @@ func SetShardMode( req *SetShardModeRequest, opts ...client.CallOption, ) (*SetShardModeResponse, error) { - wResp := &setShardModeResponseWrapper{ - m: new(SetShardModeResponse), - } + wResp := newResponseWrapper[SetShardModeResponse]() wReq := &requestWrapper{ m: req, @@ -123,12 +112,12 @@ func SetShardMode( return nil, err } - return wResp.m, nil + return wResp.message, nil } // SynchronizeTree executes ControlService.SynchronizeTree RPC. func SynchronizeTree(cli *client.Client, req *SynchronizeTreeRequest, opts ...client.CallOption) (*SynchronizeTreeResponse, error) { - wResp := &synchronizeTreeResponseWrapper{new(SynchronizeTreeResponse)} + wResp := newResponseWrapper[SynchronizeTreeResponse]() wReq := &requestWrapper{m: req} err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcSynchronizeTree), wReq, wResp, opts...) @@ -136,12 +125,12 @@ func SynchronizeTree(cli *client.Client, req *SynchronizeTreeRequest, opts ...cl return nil, err } - return wResp.SynchronizeTreeResponse, nil + return wResp.message, nil } // EvacuateShard executes ControlService.EvacuateShard RPC. func EvacuateShard(cli *client.Client, req *EvacuateShardRequest, opts ...client.CallOption) (*EvacuateShardResponse, error) { - wResp := &evacuateShardResponseWrapper{new(EvacuateShardResponse)} + wResp := newResponseWrapper[EvacuateShardResponse]() wReq := &requestWrapper{m: req} err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcEvacuateShard), wReq, wResp, opts...) @@ -149,12 +138,12 @@ func EvacuateShard(cli *client.Client, req *EvacuateShardRequest, opts ...client return nil, err } - return wResp.EvacuateShardResponse, nil + return wResp.message, nil } // FlushCache executes ControlService.FlushCache RPC. func FlushCache(cli *client.Client, req *FlushCacheRequest, opts ...client.CallOption) (*FlushCacheResponse, error) { - wResp := &flushCacheResponseWrapper{new(FlushCacheResponse)} + wResp := newResponseWrapper[FlushCacheResponse]() wReq := &requestWrapper{m: req} err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcFlushCache), wReq, wResp, opts...) @@ -162,12 +151,12 @@ func FlushCache(cli *client.Client, req *FlushCacheRequest, opts ...client.CallO return nil, err } - return wResp.FlushCacheResponse, nil + return wResp.message, nil } // Doctor executes ControlService.Doctor RPC. func Doctor(cli *client.Client, req *DoctorRequest, opts ...client.CallOption) (*DoctorResponse, error) { - wResp := &doctorResponseWrapper{new(DoctorResponse)} + wResp := newResponseWrapper[DoctorResponse]() wReq := &requestWrapper{m: req} err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcDoctor), wReq, wResp, opts...) @@ -175,5 +164,5 @@ func Doctor(cli *client.Client, req *DoctorRequest, opts ...client.CallOption) ( return nil, err } - return wResp.DoctorResponse, nil + return wResp.message, nil } From 8b2aae73c61fd1466ae7fbdf30009fa663aff38f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 25 Apr 2023 09:06:40 +0300 Subject: [PATCH 0172/1943] [#248] cli: Remove storagegroup commands Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/docs/storage-node-xheaders.md | 1 - cmd/frostfs-cli/modules/root.go | 2 - .../modules/storagegroup/delete.go | 53 ------- cmd/frostfs-cli/modules/storagegroup/get.go | 82 ---------- cmd/frostfs-cli/modules/storagegroup/list.go | 52 ------- cmd/frostfs-cli/modules/storagegroup/put.go | 145 ------------------ cmd/frostfs-cli/modules/storagegroup/root.go | 46 ------ cmd/frostfs-cli/modules/storagegroup/util.go | 45 ------ 8 files changed, 426 deletions(-) delete mode 100644 cmd/frostfs-cli/modules/storagegroup/delete.go delete mode 100644 cmd/frostfs-cli/modules/storagegroup/get.go delete mode 100644 cmd/frostfs-cli/modules/storagegroup/list.go delete mode 100644 cmd/frostfs-cli/modules/storagegroup/put.go delete mode 100644 cmd/frostfs-cli/modules/storagegroup/root.go delete mode 100644 cmd/frostfs-cli/modules/storagegroup/util.go diff --git a/cmd/frostfs-cli/docs/storage-node-xheaders.md b/cmd/frostfs-cli/docs/storage-node-xheaders.md index 15bce457a..f86b97ec5 100644 --- a/cmd/frostfs-cli/docs/storage-node-xheaders.md +++ b/cmd/frostfs-cli/docs/storage-node-xheaders.md @@ -26,7 +26,6 @@ If set to '0' or not set, only the current epoch is used. List of commands with support of extended headers: * `container list-objects` * `object delete/get/hash/head/lock/put/range/search` -* `storagegroup delete/get/list/put` Example: ```shell diff --git a/cmd/frostfs-cli/modules/root.go b/cmd/frostfs-cli/modules/root.go index cf6d1d40b..93513479a 100644 --- a/cmd/frostfs-cli/modules/root.go +++ b/cmd/frostfs-cli/modules/root.go @@ -14,7 +14,6 @@ import ( netmapCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/netmap" objectCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/object" sessionCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/session" - sgCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/storagegroup" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/tree" utilCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -84,7 +83,6 @@ func init() { rootCmd.AddCommand(utilCli.Cmd) rootCmd.AddCommand(netmapCli.Cmd) rootCmd.AddCommand(objectCli.Cmd) - rootCmd.AddCommand(sgCli.Cmd) rootCmd.AddCommand(containerCli.Cmd) rootCmd.AddCommand(tree.Cmd) rootCmd.AddCommand(gendoc.Command(rootCmd)) diff --git a/cmd/frostfs-cli/modules/storagegroup/delete.go b/cmd/frostfs-cli/modules/storagegroup/delete.go deleted file mode 100644 index 83751de7d..000000000 --- a/cmd/frostfs-cli/modules/storagegroup/delete.go +++ /dev/null @@ -1,53 +0,0 @@ -package storagegroup - -import ( - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - objectCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/object" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/spf13/cobra" -) - -var sgDelCmd = &cobra.Command{ - Use: "delete", - Short: "Delete storage group from FrostFS", - Long: "Delete storage group from FrostFS", - Run: delSG, -} - -func initSGDeleteCmd() { - commonflags.Init(sgDelCmd) - - flags := sgDelCmd.Flags() - - flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) - _ = sgDelCmd.MarkFlagRequired(commonflags.CIDFlag) - - flags.StringVarP(&sgID, sgIDFlag, "", "", "Storage group identifier") - _ = sgDelCmd.MarkFlagRequired(sgIDFlag) -} - -func delSG(cmd *cobra.Command, _ []string) { - pk := key.GetOrGenerate(cmd) - - var cnr cid.ID - var obj oid.ID - - addr := readObjectAddress(cmd, &cnr, &obj) - - var prm internalclient.DeleteObjectPrm - objectCli.OpenSession(cmd, &prm, pk, cnr, &obj) - objectCli.Prepare(cmd, &prm) - prm.SetAddress(addr) - - res, err := internalclient.DeleteObject(prm) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - tombstone := res.Tombstone() - - cmd.Println("Storage group removed successfully.") - cmd.Printf(" Tombstone: %s\n", tombstone) -} diff --git a/cmd/frostfs-cli/modules/storagegroup/get.go b/cmd/frostfs-cli/modules/storagegroup/get.go deleted file mode 100644 index c59d3c51c..000000000 --- a/cmd/frostfs-cli/modules/storagegroup/get.go +++ /dev/null @@ -1,82 +0,0 @@ -package storagegroup - -import ( - "bytes" - - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - objectCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/object" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - storagegroupSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/storagegroup" - "github.com/spf13/cobra" -) - -var sgID string - -var sgGetCmd = &cobra.Command{ - Use: "get", - Short: "Get storage group from FrostFS", - Long: "Get storage group from FrostFS", - Run: getSG, -} - -func initSGGetCmd() { - commonflags.Init(sgGetCmd) - - flags := sgGetCmd.Flags() - - flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) - _ = sgGetCmd.MarkFlagRequired(commonflags.CIDFlag) - - flags.StringVarP(&sgID, sgIDFlag, "", "", "Storage group identifier") - _ = sgGetCmd.MarkFlagRequired(sgIDFlag) - - flags.Bool(sgRawFlag, false, "Set raw request option") -} - -func getSG(cmd *cobra.Command, _ []string) { - var cnr cid.ID - var obj oid.ID - - addr := readObjectAddress(cmd, &cnr, &obj) - pk := key.GetOrGenerate(cmd) - buf := bytes.NewBuffer(nil) - - cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - - var prm internalclient.GetObjectPrm - objectCli.Prepare(cmd, &prm) - prm.SetClient(cli) - - raw, _ := cmd.Flags().GetBool(sgRawFlag) - prm.SetRawFlag(raw) - prm.SetAddress(addr) - prm.SetPayloadWriter(buf) - - res, err := internalclient.GetObject(prm) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - rawObj := res.Header() - rawObj.SetPayload(buf.Bytes()) - - var sg storagegroupSDK.StorageGroup - - err = storagegroupSDK.ReadFromObject(&sg, *rawObj) - commonCmd.ExitOnErr(cmd, "could not read storage group from the obj: %w", err) - - cmd.Printf("The last active epoch: %d\n", sg.ExpirationEpoch()) - cmd.Printf("Group size: %d\n", sg.ValidationDataSize()) - common.PrintChecksum(cmd, "Group hash", sg.ValidationDataHash) - - if members := sg.Members(); len(members) > 0 { - cmd.Println("Members:") - - for i := range members { - cmd.Printf("\t%s\n", members[i].String()) - } - } -} diff --git a/cmd/frostfs-cli/modules/storagegroup/list.go b/cmd/frostfs-cli/modules/storagegroup/list.go deleted file mode 100644 index 4c12453d5..000000000 --- a/cmd/frostfs-cli/modules/storagegroup/list.go +++ /dev/null @@ -1,52 +0,0 @@ -package storagegroup - -import ( - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - objectCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/object" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/storagegroup" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "github.com/spf13/cobra" -) - -var sgListCmd = &cobra.Command{ - Use: "list", - Short: "List storage groups in FrostFS container", - Long: "List storage groups in FrostFS container", - Run: listSG, -} - -func initSGListCmd() { - commonflags.Init(sgListCmd) - - sgListCmd.Flags().String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) - _ = sgListCmd.MarkFlagRequired(commonflags.CIDFlag) -} - -func listSG(cmd *cobra.Command, _ []string) { - var cnr cid.ID - readCID(cmd, &cnr) - - pk := key.GetOrGenerate(cmd) - - cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - - var prm internalclient.SearchObjectsPrm - objectCli.Prepare(cmd, &prm) - prm.SetClient(cli) - prm.SetContainerID(cnr) - prm.SetFilters(storagegroup.SearchQuery()) - - res, err := internalclient.SearchObjects(prm) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - ids := res.IDList() - - cmd.Printf("Found %d storage groups.\n", len(ids)) - - for i := range ids { - cmd.Println(ids[i].String()) - } -} diff --git a/cmd/frostfs-cli/modules/storagegroup/put.go b/cmd/frostfs-cli/modules/storagegroup/put.go deleted file mode 100644 index f4024a833..000000000 --- a/cmd/frostfs-cli/modules/storagegroup/put.go +++ /dev/null @@ -1,145 +0,0 @@ -package storagegroup - -import ( - "crypto/ecdsa" - "errors" - "fmt" - - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - objectCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/object" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/storagegroup" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - storagegroupSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/storagegroup" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/spf13/cobra" -) - -const sgMembersFlag = "members" - -var sgMembers []string - -var sgPutCmd = &cobra.Command{ - Use: "put", - Short: "Put storage group to FrostFS", - Long: "Put storage group to FrostFS", - Run: putSG, -} - -func initSGPutCmd() { - commonflags.Init(sgPutCmd) - - flags := sgPutCmd.Flags() - - flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) - _ = sgPutCmd.MarkFlagRequired(commonflags.CIDFlag) - - flags.StringSliceVarP(&sgMembers, sgMembersFlag, "m", nil, "ID list of storage group members") - _ = sgPutCmd.MarkFlagRequired(sgMembersFlag) - - flags.Uint64(commonflags.Lifetime, 0, "Storage group lifetime in epochs") - _ = sgPutCmd.MarkFlagRequired(commonflags.Lifetime) -} - -func putSG(cmd *cobra.Command, _ []string) { - pk := key.GetOrGenerate(cmd) - - var ownerID user.ID - user.IDFromKey(&ownerID, pk.PublicKey) - - var cnr cid.ID - readCID(cmd, &cnr) - - members := make([]oid.ID, len(sgMembers)) - uniqueFilter := make(map[oid.ID]struct{}, len(sgMembers)) - - for i := range sgMembers { - err := members[i].DecodeString(sgMembers[i]) - commonCmd.ExitOnErr(cmd, "could not parse object ID: %w", err) - - if _, alreadyExists := uniqueFilter[members[i]]; alreadyExists { - commonCmd.ExitOnErr(cmd, "", fmt.Errorf("%s member in not unique", members[i])) - } - - uniqueFilter[members[i]] = struct{}{} - } - - var ( - headPrm internalclient.HeadObjectPrm - putPrm internalclient.PutObjectPrm - getCnrPrm internalclient.GetContainerPrm - ) - - cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - getCnrPrm.SetClient(cli) - getCnrPrm.SetContainer(cnr) - - resGetCnr, err := internalclient.GetContainer(getCnrPrm) - commonCmd.ExitOnErr(cmd, "get container RPC call: %w", err) - - objectCli.OpenSessionViaClient(cmd, &putPrm, cli, pk, cnr, nil) - objectCli.Prepare(cmd, &headPrm, &putPrm) - - headPrm.SetRawFlag(true) - headPrm.SetClient(cli) - - sg, err := storagegroup.CollectMembers(sgHeadReceiver{ - cmd: cmd, - key: pk, - ownerID: &ownerID, - prm: headPrm, - }, cnr, members, !container.IsHomomorphicHashingDisabled(resGetCnr.Container())) - commonCmd.ExitOnErr(cmd, "could not collect storage group members: %w", err) - - var netInfoPrm internalclient.NetworkInfoPrm - netInfoPrm.SetClient(cli) - - ni, err := internalclient.NetworkInfo(netInfoPrm) - commonCmd.ExitOnErr(cmd, "can't fetch network info: %w", err) - - lifetime, _ := cmd.Flags().GetUint64(commonflags.Lifetime) - sg.SetExpirationEpoch(ni.NetworkInfo().CurrentEpoch() + lifetime) - - obj := object.New() - obj.SetContainerID(cnr) - obj.SetOwnerID(&ownerID) - - storagegroupSDK.WriteToObject(*sg, obj) - - putPrm.SetHeader(obj) - - res, err := internalclient.PutObject(putPrm) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - cmd.Println("Storage group successfully stored") - cmd.Printf(" ID: %s\n CID: %s\n", res.ID(), cnr) -} - -type sgHeadReceiver struct { - cmd *cobra.Command - key *ecdsa.PrivateKey - ownerID *user.ID - prm internalclient.HeadObjectPrm -} - -func (c sgHeadReceiver) Head(addr oid.Address) (any, error) { - c.prm.SetAddress(addr) - - res, err := internalclient.HeadObject(c.prm) - - var errSplitInfo *object.SplitInfoError - - switch { - default: - return nil, err - case err == nil: - return res.Header(), nil - case errors.As(err, &errSplitInfo): - return errSplitInfo.SplitInfo(), nil - } -} diff --git a/cmd/frostfs-cli/modules/storagegroup/root.go b/cmd/frostfs-cli/modules/storagegroup/root.go deleted file mode 100644 index 413310420..000000000 --- a/cmd/frostfs-cli/modules/storagegroup/root.go +++ /dev/null @@ -1,46 +0,0 @@ -package storagegroup - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" - objectCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/object" - "github.com/spf13/cobra" -) - -// Cmd represents the storagegroup command. -var Cmd = &cobra.Command{ - Use: "storagegroup", - Short: "Operations with Storage Groups", - Long: `Operations with Storage Groups`, - PersistentPreRun: func(cmd *cobra.Command, args []string) { - // bind exactly that cmd's flags to - // the viper before execution - commonflags.Bind(cmd) - commonflags.BindAPI(cmd) - }, -} - -const ( - sgIDFlag = "id" - sgRawFlag = "raw" -) - -func init() { - storageGroupChildCommands := []*cobra.Command{ - sgPutCmd, - sgGetCmd, - sgListCmd, - sgDelCmd, - } - - Cmd.AddCommand(storageGroupChildCommands...) - - for _, sgCommand := range storageGroupChildCommands { - objectCli.InitBearer(sgCommand) - commonflags.InitAPI(sgCommand) - } - - initSGPutCmd() - initSGGetCmd() - initSGListCmd() - initSGDeleteCmd() -} diff --git a/cmd/frostfs-cli/modules/storagegroup/util.go b/cmd/frostfs-cli/modules/storagegroup/util.go deleted file mode 100644 index 5ca61a58e..000000000 --- a/cmd/frostfs-cli/modules/storagegroup/util.go +++ /dev/null @@ -1,45 +0,0 @@ -package storagegroup - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/spf13/cobra" -) - -func readObjectAddress(cmd *cobra.Command, cnr *cid.ID, obj *oid.ID) oid.Address { - readCID(cmd, cnr) - readSGID(cmd, obj) - - var addr oid.Address - addr.SetContainer(*cnr) - addr.SetObject(*obj) - return addr -} - -func readCID(cmd *cobra.Command, id *cid.ID) { - f := cmd.Flag(commonflags.CIDFlag) - if f == nil { - commonCmd.ExitOnErr(cmd, "", fmt.Errorf("missing container flag (%s)", commonflags.CIDFlag)) - return - } - - err := id.DecodeString(f.Value.String()) - commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) -} - -func readSGID(cmd *cobra.Command, id *oid.ID) { - const flag = "id" - - f := cmd.Flag(flag) - if f == nil { - commonCmd.ExitOnErr(cmd, "", fmt.Errorf("missing storage group flag (%s)", flag)) - return - } - - err := id.DecodeString(f.Value.String()) - commonCmd.ExitOnErr(cmd, "decode storage group ID string: %w", err) -} From 8879c6ea4aa5431bdfb25e6fca7c15b9e339c6b1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 25 Apr 2023 09:16:14 +0300 Subject: [PATCH 0173/1943] [#248] innerring: Remove audit and settlement code Signed-off-by: Evgenii Stratonikov --- pkg/innerring/blocktimer.go | 39 -- pkg/innerring/contracts.go | 2 - pkg/innerring/initialization.go | 147 +------- pkg/innerring/innerring.go | 12 - pkg/innerring/internal/client/client.go | 339 ------------------ pkg/innerring/internal/client/doc.go | 12 - pkg/innerring/internal/client/prm.go | 18 - pkg/innerring/processors/audit/events.go | 19 - pkg/innerring/processors/audit/handlers.go | 22 -- pkg/innerring/processors/audit/process.go | 217 ----------- pkg/innerring/processors/audit/processor.go | 143 -------- pkg/innerring/processors/audit/scheduler.go | 65 ---- .../processors/audit/scheduler_test.go | 106 ------ .../processors/netmap/process_epoch.go | 4 - pkg/innerring/processors/netmap/processor.go | 19 +- .../processors/settlement/audit/calculate.go | 336 ----------------- .../processors/settlement/audit/calculator.go | 48 --- .../processors/settlement/audit/prm.go | 49 --- .../processors/settlement/basic/collect.go | 114 ------ .../processors/settlement/basic/context.go | 80 ----- .../processors/settlement/basic/distribute.go | 59 --- .../settlement/basic/distribute_test.go | 54 --- .../processors/settlement/basic/util.go | 44 --- pkg/innerring/processors/settlement/calls.go | 133 ------- .../processors/settlement/common/details.go | 33 -- .../settlement/common/details_test.go | 28 -- .../processors/settlement/common/types.go | 54 --- .../processors/settlement/common/util.go | 74 ---- pkg/innerring/processors/settlement/deps.go | 17 - pkg/innerring/processors/settlement/events.go | 46 --- .../processors/settlement/handlers.go | 22 -- pkg/innerring/processors/settlement/opts.go | 31 -- .../processors/settlement/processor.go | 79 ---- pkg/innerring/rpc.go | 236 ------------ pkg/innerring/settlement.go | 301 ---------------- pkg/innerring/state.go | 14 - 36 files changed, 8 insertions(+), 3008 deletions(-) delete mode 100644 pkg/innerring/internal/client/client.go delete mode 100644 pkg/innerring/internal/client/doc.go delete mode 100644 pkg/innerring/internal/client/prm.go delete mode 100644 pkg/innerring/processors/audit/events.go delete mode 100644 pkg/innerring/processors/audit/handlers.go delete mode 100644 pkg/innerring/processors/audit/process.go delete mode 100644 pkg/innerring/processors/audit/processor.go delete mode 100644 pkg/innerring/processors/audit/scheduler.go delete mode 100644 pkg/innerring/processors/audit/scheduler_test.go delete mode 100644 pkg/innerring/processors/settlement/audit/calculate.go delete mode 100644 pkg/innerring/processors/settlement/audit/calculator.go delete mode 100644 pkg/innerring/processors/settlement/audit/prm.go delete mode 100644 pkg/innerring/processors/settlement/basic/collect.go delete mode 100644 pkg/innerring/processors/settlement/basic/context.go delete mode 100644 pkg/innerring/processors/settlement/basic/distribute.go delete mode 100644 pkg/innerring/processors/settlement/basic/distribute_test.go delete mode 100644 pkg/innerring/processors/settlement/basic/util.go delete mode 100644 pkg/innerring/processors/settlement/calls.go delete mode 100644 pkg/innerring/processors/settlement/common/details.go delete mode 100644 pkg/innerring/processors/settlement/common/details_test.go delete mode 100644 pkg/innerring/processors/settlement/common/types.go delete mode 100644 pkg/innerring/processors/settlement/common/util.go delete mode 100644 pkg/innerring/processors/settlement/deps.go delete mode 100644 pkg/innerring/processors/settlement/events.go delete mode 100644 pkg/innerring/processors/settlement/handlers.go delete mode 100644 pkg/innerring/processors/settlement/opts.go delete mode 100644 pkg/innerring/processors/settlement/processor.go delete mode 100644 pkg/innerring/rpc.go delete mode 100644 pkg/innerring/settlement.go diff --git a/pkg/innerring/blocktimer.go b/pkg/innerring/blocktimer.go index 94e262099..6a4dee801 100644 --- a/pkg/innerring/blocktimer.go +++ b/pkg/innerring/blocktimer.go @@ -5,10 +5,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/timer" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/util" @@ -25,12 +23,6 @@ type ( IsAlphabet() bool } - subEpochEventHandler struct { - handler event.Handler // handle to execute - durationMul uint32 // X: X/Y of epoch in blocks - durationDiv uint32 // Y: X/Y of epoch in blocks - } - newEpochHandler func() epochTimerArgs struct { @@ -45,9 +37,6 @@ type ( stopEstimationDMul uint32 // X: X/Y of epoch in blocks stopEstimationDDiv uint32 // Y: X/Y of epoch in blocks - - collectBasicIncome subEpochEventHandler - distributeBasicIncome subEpochEventHandler } emitTimerArgs struct { @@ -119,34 +108,6 @@ func newEpochTimer(args *epochTimerArgs) *timer.BlockTimer { } }) - epochTimer.OnDelta( - args.collectBasicIncome.durationMul, - args.collectBasicIncome.durationDiv, - func() { - epochN := args.epoch.EpochCounter() - if epochN == 0 { // estimates are invalid in genesis epoch - return - } - - args.collectBasicIncome.handler( - settlement.NewBasicIncomeCollectEvent(epochN - 1), - ) - }) - - epochTimer.OnDelta( - args.distributeBasicIncome.durationMul, - args.distributeBasicIncome.durationDiv, - func() { - epochN := args.epoch.EpochCounter() - if epochN == 0 { // estimates are invalid in genesis epoch - return - } - - args.distributeBasicIncome.handler( - settlement.NewBasicIncomeDistributeEvent(epochN - 1), - ) - }) - return epochTimer } diff --git a/pkg/innerring/contracts.go b/pkg/innerring/contracts.go index def55f22a..85a4bf946 100644 --- a/pkg/innerring/contracts.go +++ b/pkg/innerring/contracts.go @@ -15,7 +15,6 @@ type contracts struct { netmap util.Uint160 // in morph balance util.Uint160 // in morph container util.Uint160 // in morph - audit util.Uint160 // in morph proxy util.Uint160 // in morph processing util.Uint160 // in mainnet frostfsID util.Uint160 // in morph @@ -58,7 +57,6 @@ func parseContracts(cfg *viper.Viper, morph *client.Client, withoutMainNet, with {"contracts.netmap", client.NNSNetmapContractName, &result.netmap}, {"contracts.balance", client.NNSBalanceContractName, &result.balance}, {"contracts.container", client.NNSContainerContractName, &result.container}, - {"contracts.audit", client.NNSAuditContractName, &result.audit}, {"contracts.frostfsid", client.NNSFrostFSIDContractName, &result.frostfsID}, } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 38bde1412..cb9b10648 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/balance" cont "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/frostfs" @@ -17,25 +16,19 @@ import ( nodevalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation" addrvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/maddress" statevalidation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement" - auditSettlement "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - auditClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/audit" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" frostfsClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - audittask "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit/taskmanager" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" controlsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" - util2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" utilConfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - "github.com/panjf2000/ants/v2" "github.com/spf13/viper" "go.uber.org/zap" "google.golang.org/grpc" @@ -43,9 +36,7 @@ import ( func (s *Server) initNetmapProcessor(cfg *viper.Viper, cnrClient *container.Client, - alphaSync event.Handler, - auditProcessor *audit.Processor, - settlementProcessor *settlement.Processor) error { + alphaSync event.Handler) error { locodeValidator, err := s.newLocodeValidator(cfg) if err != nil { return err @@ -70,15 +61,9 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, CleanupEnabled: cfg.GetBool("netmap_cleaner.enabled"), CleanupThreshold: cfg.GetUint64("netmap_cleaner.threshold"), ContainerWrapper: cnrClient, - HandleAudit: s.onlyActiveEventHandler( - auditProcessor.StartAuditHandler(), - ), NotaryDepositHandler: s.onlyAlphabetEventHandler( s.notaryHandler, ), - AuditSettlementsHandler: s.onlyAlphabetEventHandler( - settlementProcessor.HandleAuditEvent, - ), AlphabetSyncHandler: s.onlyAlphabetEventHandler( alphaSync, ), @@ -171,93 +156,6 @@ func (s *Server) initNotaryConfig() { ) } -func (s *Server) createAuditProcessor(cfg *viper.Viper, clientCache *ClientCache, cnrClient *container.Client) (*audit.Processor, error) { - auditPool, err := ants.NewPool(cfg.GetInt("audit.task.exec_pool_size")) - if err != nil { - return nil, err - } - - pdpPoolSize := cfg.GetInt("audit.pdp.pairs_pool_size") - porPoolSize := cfg.GetInt("audit.por.pool_size") - - // create audit processor dependencies - auditTaskManager := audittask.New( - audittask.WithQueueCapacity(cfg.GetUint32("audit.task.queue_capacity")), - audittask.WithWorkerPool(auditPool), - audittask.WithLogger(s.log), - audittask.WithContainerCommunicator(clientCache), - audittask.WithMaxPDPSleepInterval(cfg.GetDuration("audit.pdp.max_sleep_interval")), - audittask.WithPDPWorkerPoolGenerator(func() (util2.WorkerPool, error) { - return ants.NewPool(pdpPoolSize) - }), - audittask.WithPoRWorkerPoolGenerator(func() (util2.WorkerPool, error) { - return ants.NewPool(porPoolSize) - }), - ) - - s.workers = append(s.workers, auditTaskManager.Listen) - - // create audit processor - return audit.New(&audit.Params{ - Log: s.log, - NetmapClient: s.netmapClient, - ContainerClient: cnrClient, - IRList: s, - EpochSource: s, - SGSource: clientCache, - Key: &s.key.PrivateKey, - RPCSearchTimeout: cfg.GetDuration("audit.timeout.search"), - TaskManager: auditTaskManager, - Reporter: s, - }) -} - -func (s *Server) createSettlementProcessor(clientCache *ClientCache, cnrClient *container.Client) *settlement.Processor { - // create settlement processor dependencies - settlementDeps := settlementDeps{ - log: s.log, - cnrSrc: container.AsContainerSource(cnrClient), - auditClient: s.auditClient, - nmClient: s.netmapClient, - clientCache: clientCache, - balanceClient: s.balanceClient, - } - - settlementDeps.settlementCtx = auditSettlementContext - auditCalcDeps := &auditSettlementDeps{ - settlementDeps: settlementDeps, - } - - settlementDeps.settlementCtx = basicIncomeSettlementContext - basicSettlementDeps := &basicIncomeSettlementDeps{ - settlementDeps: settlementDeps, - cnrClient: cnrClient, - } - - auditSettlementCalc := auditSettlement.NewCalculator( - &auditSettlement.CalculatorPrm{ - ResultStorage: auditCalcDeps, - ContainerStorage: auditCalcDeps, - PlacementCalculator: auditCalcDeps, - SGStorage: auditCalcDeps, - AccountStorage: auditCalcDeps, - Exchanger: auditCalcDeps, - AuditFeeFetcher: s.netmapClient, - }, - auditSettlement.WithLogger(s.log), - ) - - // create settlement processor - return settlement.New( - settlement.Prm{ - AuditProcessor: (*auditSettlementCalculator)(auditSettlementCalc), - BasicIncome: &basicSettlementConstructor{dep: basicSettlementDeps}, - State: s, - }, - settlement.WithLogger(s.log), - ) -} - func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Client, irf irFetcher) (event.Handler, error) { var alphaSync event.Handler @@ -316,16 +214,6 @@ func (s *Server) initTimers(cfg *viper.Viper, processors *serverProcessors, morp epoch: s, stopEstimationDMul: cfg.GetUint32("timers.stop_estimation.mul"), stopEstimationDDiv: cfg.GetUint32("timers.stop_estimation.div"), - collectBasicIncome: subEpochEventHandler{ - handler: processors.SettlementProcessor.HandleIncomeCollectionEvent, - durationMul: cfg.GetUint32("timers.collect_basic_income.mul"), - durationDiv: cfg.GetUint32("timers.collect_basic_income.div"), - }, - distributeBasicIncome: subEpochEventHandler{ - handler: processors.SettlementProcessor.HandleIncomeDistributionEvent, - durationMul: cfg.GetUint32("timers.distribute_basic_income.mul"), - durationDiv: cfg.GetUint32("timers.distribute_basic_income.div"), - }, }) s.addBlockTimer(s.epochTimer) @@ -493,12 +381,6 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { var err error fee := s.feeConfig.SideChainFee() - // do not use TryNotary() in audit wrapper - // audit operations do not require multisignatures - s.auditClient, err = auditClient.NewFromMorph(s.morphClient, s.contracts.audit, fee) - if err != nil { - return nil, err - } // form morph container client's options morphCnrOpts := make([]container.Option, 0, 3) @@ -545,8 +427,7 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { } type serverProcessors struct { - AlphabetProcessor *alphabet.Processor - SettlementProcessor *settlement.Processor + AlphabetProcessor *alphabet.Processor } func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClients) (*serverProcessors, error) { @@ -561,32 +442,12 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien cfg.GetDuration("indexer.cache_timeout"), ) - clientCache := newClientCache(&clientCacheParams{ - Log: s.log, - Key: &s.key.PrivateKey, - SGTimeout: cfg.GetDuration("audit.timeout.get"), - HeadTimeout: cfg.GetDuration("audit.timeout.head"), - RangeTimeout: cfg.GetDuration("audit.timeout.rangehash"), - AllowExternal: cfg.GetBool("audit.allow_external"), - }) - - s.registerNoErrCloser(clientCache.cache.CloseAll) - - // create audit processor - auditProcessor, err := s.createAuditProcessor(cfg, clientCache, morphClients.CnrClient) + alphaSync, err := s.createAlphaSync(cfg, morphClients.FrostFSClient, irf) if err != nil { return nil, err } - result.SettlementProcessor = s.createSettlementProcessor(clientCache, morphClients.CnrClient) - - var alphaSync event.Handler - alphaSync, err = s.createAlphaSync(cfg, morphClients.FrostFSClient, irf) - if err != nil { - return nil, err - } - - err = s.initNetmapProcessor(cfg, morphClients.CnrClient, alphaSync, auditProcessor, result.SettlementProcessor) + err = s.initNetmapProcessor(cfg, morphClients.CnrClient, alphaSync) if err != nil { return nil, err } diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 2cb29b90a..deb546f08 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -13,7 +13,6 @@ import ( timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - auditClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/audit" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -52,7 +51,6 @@ type ( epochDuration atomic.Uint64 statusIndex *innerRingIndexer precision precision.Fixed8Converter - auditClient *auditClient.Client healthStatus atomic.Value balanceClient *balanceClient.Client netmapClient *nmClient.Client @@ -572,16 +570,6 @@ func (s *Server) nextEpochBlockDelta() (uint32, error) { return delta - blockHeight, nil } -// onlyActiveHandler wrapper around event handler that executes it -// only if inner ring node state is active. -func (s *Server) onlyActiveEventHandler(f event.Handler) event.Handler { - return func(ev event.Event) { - if s.IsActive() { - f(ev) - } - } -} - // onlyAlphabet wrapper around event handler that executes it // only if inner ring node is alphabet node. func (s *Server) onlyAlphabetEventHandler(f event.Handler) event.Handler { diff --git a/pkg/innerring/internal/client/client.go b/pkg/innerring/internal/client/client.go deleted file mode 100644 index 3e95e9766..000000000 --- a/pkg/innerring/internal/client/client.go +++ /dev/null @@ -1,339 +0,0 @@ -package frostfsapiclient - -import ( - "context" - "crypto/ecdsa" - "errors" - "fmt" - "io" - - clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/storagegroup" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -// Client represents FrostFS API client cut down to the needs of a purely IR application. -type Client struct { - key *ecdsa.PrivateKey - - c clientcore.Client -} - -// WrapBasicClient wraps a client.Client instance to use it for FrostFS API RPC. -func (x *Client) WrapBasicClient(c clientcore.Client) { - x.c = c -} - -// SetPrivateKey sets a private key to sign RPC requests. -func (x *Client) SetPrivateKey(key *ecdsa.PrivateKey) { - x.key = key -} - -// SearchSGPrm groups parameters of SearchSG operation. -type SearchSGPrm struct { - cnrID cid.ID -} - -// SetContainerID sets the ID of the container to search for storage groups. -func (x *SearchSGPrm) SetContainerID(id cid.ID) { - x.cnrID = id -} - -// SearchSGRes groups the resulting values of SearchSG operation. -type SearchSGRes struct { - cliRes []oid.ID -} - -// IDList returns a list of IDs of storage groups in the container. -func (x SearchSGRes) IDList() []oid.ID { - return x.cliRes -} - -var sgFilter = storagegroup.SearchQuery() - -// SearchSG lists objects of storage group type in the container. -// -// Returns any error which prevented the operation from completing correctly in error return. -func (x Client) SearchSG(ctx context.Context, prm SearchSGPrm) (*SearchSGRes, error) { - var cliPrm client.PrmObjectSearch - cliPrm.InContainer(prm.cnrID) - cliPrm.SetFilters(sgFilter) - cliPrm.UseKey(*x.key) - - rdr, err := x.c.ObjectSearchInit(ctx, cliPrm) - if err != nil { - return nil, fmt.Errorf("init object search: %w", err) - } - - buf := make([]oid.ID, 10) - var list []oid.ID - var n int - var ok bool - - for { - n, ok = rdr.Read(buf) - for i := 0; i < n; i++ { - list = append(list, buf[i]) - } - if !ok { - break - } - } - - res, err := rdr.Close() - if err == nil { - // pull out an error from status - err = apistatus.ErrFromStatus(res.Status()) - } - - if err != nil { - return nil, fmt.Errorf("read object list: %w", err) - } - - return &SearchSGRes{ - cliRes: list, - }, nil -} - -// GetObjectPrm groups parameters of GetObject operation. -type GetObjectPrm struct { - getObjectPrm -} - -// GetObjectRes groups the resulting values of GetObject operation. -type GetObjectRes struct { - obj *object.Object -} - -// Object returns the received object. -func (x GetObjectRes) Object() *object.Object { - return x.obj -} - -// GetObject reads the object by address. -// -// Returns any error which prevented the operation from completing correctly in error return. -func (x Client) GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { - var cliPrm client.PrmObjectGet - cliPrm.FromContainer(prm.objAddr.Container()) - cliPrm.ByID(prm.objAddr.Object()) - cliPrm.UseKey(*x.key) - - rdr, err := x.c.ObjectGetInit(ctx, cliPrm) - if err != nil { - return nil, fmt.Errorf("init object search: %w", err) - } - - var obj object.Object - - if !rdr.ReadHeader(&obj) { - res, err := rdr.Close() - if err == nil { - // pull out an error from status - err = apistatus.ErrFromStatus(res.Status()) - } - - return nil, fmt.Errorf("read object header: %w", err) - } - - buf := make([]byte, obj.PayloadSize()) - - _, err = rdr.Read(buf) - if err != nil && !errors.Is(err, io.EOF) { - return nil, fmt.Errorf("read payload: %w", err) - } - - obj.SetPayload(buf) - - return &GetObjectRes{ - obj: &obj, - }, nil -} - -// HeadObjectPrm groups parameters of HeadObject operation. -type HeadObjectPrm struct { - getObjectPrm - - raw bool - - local bool -} - -// SetRawFlag sets flag of raw request. -func (x *HeadObjectPrm) SetRawFlag() { - x.raw = true -} - -// SetTTL sets request TTL value. -func (x *HeadObjectPrm) SetTTL(ttl uint32) { - x.local = ttl < 2 -} - -// HeadObjectRes groups the resulting values of HeadObject operation. -type HeadObjectRes struct { - hdr *object.Object -} - -// Header returns the received object header. -func (x HeadObjectRes) Header() *object.Object { - return x.hdr -} - -// HeadObject reads short object header by address. -// -// Returns any error which prevented the operation from completing correctly in error return. -// For raw requests, returns *object.SplitInfoError error if the requested object is virtual. -func (x Client) HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) { - var cliPrm client.PrmObjectHead - - if prm.raw { - cliPrm.MarkRaw() - } - - if prm.local { - cliPrm.MarkLocal() - } - - cliPrm.FromContainer(prm.objAddr.Container()) - cliPrm.ByID(prm.objAddr.Object()) - cliPrm.UseKey(*x.key) - - cliRes, err := x.c.ObjectHead(ctx, cliPrm) - if err == nil { - // pull out an error from status - err = apistatus.ErrFromStatus(cliRes.Status()) - } - - if err != nil { - return nil, fmt.Errorf("read object header from FrostFS: %w", err) - } - - var hdr object.Object - - if !cliRes.ReadHeader(&hdr) { - return nil, errors.New("missing object header in the response") - } - - return &HeadObjectRes{ - hdr: &hdr, - }, nil -} - -// GetObjectPayload reads an object by address from FrostFS via Client and returns its payload. -// -// Returns any error which prevented the operation from completing correctly in error return. -func GetObjectPayload(ctx context.Context, c Client, addr oid.Address) ([]byte, error) { - var prm GetObjectPrm - - prm.SetAddress(addr) - - obj, err := c.GetObject(ctx, prm) - if err != nil { - return nil, err - } - - return obj.Object().Payload(), nil -} - -func headObject(ctx context.Context, c Client, addr oid.Address, raw bool, ttl uint32) (*object.Object, error) { - var prm HeadObjectPrm - - prm.SetAddress(addr) - prm.SetTTL(ttl) - - if raw { - prm.SetRawFlag() - } - - obj, err := c.HeadObject(ctx, prm) - if err != nil { - return nil, err - } - - return obj.Header(), nil -} - -// GetRawObjectHeaderLocally reads the raw short object header from the server's local storage by address via Client. -func GetRawObjectHeaderLocally(ctx context.Context, c Client, addr oid.Address) (*object.Object, error) { - return headObject(ctx, c, addr, true, 1) -} - -// GetObjectHeaderFromContainer reads the short object header by address via Client with TTL = 10 -// for deep traversal of the container. -func GetObjectHeaderFromContainer(ctx context.Context, c Client, addr oid.Address) (*object.Object, error) { - return headObject(ctx, c, addr, false, 10) -} - -// HashPayloadRangePrm groups parameters of HashPayloadRange operation. -type HashPayloadRangePrm struct { - getObjectPrm - - rng *object.Range -} - -// SetRange sets payload range to calculate the hash. -func (x *HashPayloadRangePrm) SetRange(rng *object.Range) { - x.rng = rng -} - -// HashPayloadRangeRes groups the resulting values of HashPayloadRange operation. -type HashPayloadRangeRes struct { - h []byte -} - -// Hash returns the hash of the object payload range. -func (x HashPayloadRangeRes) Hash() []byte { - return x.h -} - -// HashPayloadRange requests to calculate Tillich-Zemor hash of the payload range of the object -// from the remote server's local storage. -// -// Returns any error which prevented the operation from completing correctly in error return. -func (x Client) HashPayloadRange(ctx context.Context, prm HashPayloadRangePrm) (res HashPayloadRangeRes, err error) { - var cliPrm client.PrmObjectHash - cliPrm.FromContainer(prm.objAddr.Container()) - cliPrm.ByID(prm.objAddr.Object()) - cliPrm.SetRangeList(prm.rng.GetOffset(), prm.rng.GetLength()) - cliPrm.TillichZemorAlgo() - - cliRes, err := x.c.ObjectHash(ctx, cliPrm) - if err == nil { - // pull out an error from status - err = apistatus.ErrFromStatus(cliRes.Status()) - if err != nil { - return - } - - hs := cliRes.Checksums() - if ln := len(hs); ln != 1 { - err = fmt.Errorf("wrong number of checksums %d", ln) - } else { - res.h = hs[0] - } - } - - return -} - -// HashObjectRange reads Tillich-Zemor hash of the object payload range by address -// from the remote server's local storage via Client. -// -// Returns any error which prevented the operation from completing correctly in error return. -func HashObjectRange(ctx context.Context, c Client, addr oid.Address, rng *object.Range) ([]byte, error) { - var prm HashPayloadRangePrm - - prm.SetAddress(addr) - prm.SetRange(rng) - - res, err := c.HashPayloadRange(ctx, prm) - if err != nil { - return nil, err - } - - return res.Hash(), nil -} diff --git a/pkg/innerring/internal/client/doc.go b/pkg/innerring/internal/client/doc.go deleted file mode 100644 index a04b0627b..000000000 --- a/pkg/innerring/internal/client/doc.go +++ /dev/null @@ -1,12 +0,0 @@ -// Package frostfsapiclient provides functionality for IR application communication with FrostFS network. -// -// The basic client for accessing remote nodes via FrostFS API is a FrostFS SDK Go API client. -// However, although it encapsulates a useful piece of business logic (e.g. the signature mechanism), -// the IR application does not fully use the client's flexible interface. -// -// In this regard, this package represents an abstraction -- a type-wrapper over the base client. -// The type provides the minimum interface necessary for the application and also allows you to concentrate -// the entire spectrum of the client's use in one place (this will be convenient both when updating the base client -// and for evaluating the UX of SDK library). So, it is expected that all application packages will be limited -// to this package for the development of functionality requiring FrostFS API communication. -package frostfsapiclient diff --git a/pkg/innerring/internal/client/prm.go b/pkg/innerring/internal/client/prm.go deleted file mode 100644 index 9b5872434..000000000 --- a/pkg/innerring/internal/client/prm.go +++ /dev/null @@ -1,18 +0,0 @@ -package frostfsapiclient - -import ( - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -type objectAddressPrm struct { - objAddr oid.Address -} - -// SetAddress sets address of the object. -func (x *objectAddressPrm) SetAddress(addr oid.Address) { - x.objAddr = addr -} - -type getObjectPrm struct { - objectAddressPrm -} diff --git a/pkg/innerring/processors/audit/events.go b/pkg/innerring/processors/audit/events.go deleted file mode 100644 index 4fb106122..000000000 --- a/pkg/innerring/processors/audit/events.go +++ /dev/null @@ -1,19 +0,0 @@ -package audit - -// Start is an event to start a new round of data audit. -type Start struct { - epoch uint64 -} - -// MorphEvent implements the Event interface. -func (a Start) MorphEvent() {} - -func NewAuditStartEvent(epoch uint64) Start { - return Start{ - epoch: epoch, - } -} - -func (a Start) Epoch() uint64 { - return a.epoch -} diff --git a/pkg/innerring/processors/audit/handlers.go b/pkg/innerring/processors/audit/handlers.go deleted file mode 100644 index 06c656fa2..000000000 --- a/pkg/innerring/processors/audit/handlers.go +++ /dev/null @@ -1,22 +0,0 @@ -package audit - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "go.uber.org/zap" -) - -func (ap *Processor) handleNewAuditRound(ev event.Event) { - auditEvent := ev.(Start) - - epoch := auditEvent.Epoch() - - ap.log.Info(logs.AuditNewRoundOfAudit, zap.Uint64("epoch", epoch)) - - // send an event to the worker pool - - err := ap.pool.Submit(func() { ap.processStartAudit(epoch) }) - if err != nil { - ap.log.Warn(logs.AuditPreviousRoundOfAuditPrepareHasntFinishedYet) - } -} diff --git a/pkg/innerring/processors/audit/process.go b/pkg/innerring/processors/audit/process.go deleted file mode 100644 index 000279f01..000000000 --- a/pkg/innerring/processors/audit/process.go +++ /dev/null @@ -1,217 +0,0 @@ -package audit - -import ( - "context" - "crypto/sha256" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/storagegroup" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.uber.org/zap" -) - -func (ap *Processor) processStartAudit(epoch uint64) { - log := ap.log.With(zap.Uint64("epoch", epoch)) - - ap.prevAuditCanceler() - - skipped := ap.taskManager.Reset() - if skipped > 0 { - ap.log.Info(logs.AuditSomeTasksFromPreviousEpochAreSkipped, - zap.Int("amount", skipped), - ) - } - - containers, err := ap.selectContainersToAudit(epoch) - if err != nil { - log.Error(logs.AuditContainerSelectionFailure, zap.String("error", err.Error())) - - return - } - - log.Info(logs.AuditSelectContainersForAudit, zap.Int("amount", len(containers))) - - nm, err := ap.netmapClient.GetNetMap(0) - if err != nil { - ap.log.Error(logs.AuditCantFetchNetworkMap, - zap.String("error", err.Error())) - - return - } - - cancelChannel := make(chan struct{}) - ap.prevAuditCanceler = func() { - select { - case <-cancelChannel: // already closed - default: - close(cancelChannel) - } - } - - pivot := make([]byte, sha256.Size) - - ap.startAuditTasksOnContainers(cancelChannel, containers, log, pivot, nm, epoch) -} - -func (ap *Processor) startAuditTasksOnContainers(cancelChannel <-chan struct{}, containers []cid.ID, log *zap.Logger, pivot []byte, nm *netmap.NetMap, epoch uint64) { - for i := range containers { - cnr, err := cntClient.Get(ap.containerClient, containers[i]) // get container structure - if err != nil { - log.Error(logs.AuditCantGetContainerInfoIgnore, - zap.Stringer("cid", containers[i]), - zap.String("error", err.Error())) - - continue - } - - containers[i].Encode(pivot) - - // find all container nodes for current epoch - nodes, err := nm.ContainerNodes(cnr.Value.PlacementPolicy(), pivot) - if err != nil { - log.Info(logs.AuditCantBuildPlacementForContainerIgnore, - zap.Stringer("cid", containers[i]), - zap.String("error", err.Error())) - - continue - } - - n := placement.FlattenNodes(nodes) - - // shuffle nodes to ask a random one - rand.Shuffle(len(n), func(i, j int) { - n[i], n[j] = n[j], n[i] - }) - - // search storage groups - storageGroupsIDs := ap.findStorageGroups(containers[i], n) - log.Info(logs.AuditSelectStorageGroupsForAudit, - zap.Stringer("cid", containers[i]), - zap.Int("amount", len(storageGroupsIDs))) - - // filter expired storage groups - storageGroups := ap.filterExpiredSG(containers[i], storageGroupsIDs, nodes, *nm) - log.Info(logs.AuditFilterExpiredStorageGroupsForAudit, - zap.Stringer("cid", containers[i]), - zap.Int("amount", len(storageGroups))) - - // skip audit for containers without - // non-expired storage groups - if len(storageGroupsIDs) == 0 { - continue - } - - auditTask := new(audit.Task). - WithReporter(&epochAuditReporter{ - epoch: epoch, - rep: ap.reporter, - }). - WithCancelChannel(cancelChannel). - WithContainerID(containers[i]). - WithStorageGroupList(storageGroups). - WithContainerStructure(cnr.Value). - WithContainerNodes(nodes). - WithNetworkMap(nm) - - ap.taskManager.PushTask(auditTask) - } -} - -func (ap *Processor) findStorageGroups(cnr cid.ID, shuffled netmapcore.Nodes) []oid.ID { - var sg []oid.ID - - ln := len(shuffled) - - var ( - info clientcore.NodeInfo - prm storagegroup.SearchSGPrm - ) - - prm.Container = cnr - - for i := range shuffled { // consider iterating over some part of container - log := ap.log.With( - zap.Stringer("cid", cnr), - zap.String("key", netmap.StringifyPublicKey(shuffled[0])), - zap.Int("try", i), - zap.Int("total_tries", ln), - ) - - err := clientcore.NodeInfoFromRawNetmapElement(&info, netmapcore.Node(shuffled[i])) - if err != nil { - log.Warn(logs.AuditParseClientNodeInfo, zap.String("error", err.Error())) - - continue - } - - ctx, cancel := context.WithTimeout(context.Background(), ap.searchTimeout) - - prm.NodeInfo = info - - var dst storagegroup.SearchSGDst - - err = ap.sgSrc.ListSG(ctx, &dst, prm) - - cancel() - - if err != nil { - log.Warn(logs.AuditErrorInStorageGroupSearch, zap.String("error", err.Error())) - continue - } - - sg = append(sg, dst.Objects...) - - break // we found storage groups, so break loop - } - - return sg -} - -func (ap *Processor) filterExpiredSG(cid cid.ID, sgIDs []oid.ID, - cnr [][]netmap.NodeInfo, nm netmap.NetMap) []storagegroup.StorageGroup { - sgs := make([]storagegroup.StorageGroup, 0, len(sgIDs)) - var coreSG storagegroup.StorageGroup - - var getSGPrm storagegroup.GetSGPrm - getSGPrm.CID = cid - getSGPrm.Container = cnr - getSGPrm.NetMap = nm - - for _, sgID := range sgIDs { - ctx, cancel := context.WithTimeout(context.Background(), ap.searchTimeout) - - getSGPrm.OID = sgID - - sg, err := ap.sgSrc.GetSG(ctx, getSGPrm) - - cancel() - - if err != nil { - ap.log.Error( - "could not get storage group object for audit, skipping", - zap.Stringer("cid", cid), - zap.Stringer("oid", sgID), - zap.Error(err), - ) - continue - } - - // filter expired epochs - if sg.ExpirationEpoch() >= ap.epochSrc.EpochCounter() { - coreSG.SetID(sgID) - coreSG.SetStorageGroup(*sg) - - sgs = append(sgs, coreSG) - } - } - - return sgs -} diff --git a/pkg/innerring/processors/audit/processor.go b/pkg/innerring/processors/audit/processor.go deleted file mode 100644 index 6e0a98209..000000000 --- a/pkg/innerring/processors/audit/processor.go +++ /dev/null @@ -1,143 +0,0 @@ -package audit - -import ( - "context" - "crypto/ecdsa" - "errors" - "fmt" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/storagegroup" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/panjf2000/ants/v2" -) - -type ( - // Indexer is a callback interface for inner ring global state. - Indexer interface { - InnerRingIndex() int - InnerRingSize() int - } - - TaskManager interface { - PushTask(*audit.Task) - - // Must skip all tasks planned for execution and - // return their number. - Reset() int - } - - // EpochSource is an interface that provides actual - // epoch information. - EpochSource interface { - // EpochCounter must return current epoch number. - EpochCounter() uint64 - } - - // Processor of events related to data audit. - Processor struct { - log *logger.Logger - pool *ants.Pool - irList Indexer - sgSrc storagegroup.SGSource - epochSrc EpochSource - searchTimeout time.Duration - - containerClient *cntClient.Client - netmapClient *nmClient.Client - - taskManager TaskManager - reporter audit.Reporter - prevAuditCanceler context.CancelFunc - } - - // Params of the processor constructor. - Params struct { - Log *logger.Logger - NetmapClient *nmClient.Client - ContainerClient *cntClient.Client - IRList Indexer - SGSource storagegroup.SGSource - RPCSearchTimeout time.Duration - TaskManager TaskManager - Reporter audit.Reporter - Key *ecdsa.PrivateKey - EpochSource EpochSource - } -) - -type epochAuditReporter struct { - epoch uint64 - - rep audit.Reporter -} - -// ProcessorPoolSize limits pool size for audit Processor. Processor manages -// audit tasks and fills queue for the next epoch. This process must not be interrupted -// by a new audit epoch, so we limit the pool size for the processor to one. -const ProcessorPoolSize = 1 - -// New creates audit processor instance. -func New(p *Params) (*Processor, error) { - switch { - case p.Log == nil: - return nil, errors.New("ir/audit: logger is not set") - case p.IRList == nil: - return nil, errors.New("ir/audit: global state is not set") - case p.SGSource == nil: - return nil, errors.New("ir/audit: SG source is not set") - case p.TaskManager == nil: - return nil, errors.New("ir/audit: audit task manager is not set") - case p.Reporter == nil: - return nil, errors.New("ir/audit: audit result reporter is not set") - case p.Key == nil: - return nil, errors.New("ir/audit: signing key is not set") - case p.EpochSource == nil: - return nil, errors.New("ir/audit: epoch source is not set") - } - - pool, err := ants.NewPool(ProcessorPoolSize, ants.WithNonblocking(true)) - if err != nil { - return nil, fmt.Errorf("ir/audit: can't create worker pool: %w", err) - } - - return &Processor{ - log: p.Log, - pool: pool, - containerClient: p.ContainerClient, - irList: p.IRList, - sgSrc: p.SGSource, - epochSrc: p.EpochSource, - searchTimeout: p.RPCSearchTimeout, - netmapClient: p.NetmapClient, - taskManager: p.TaskManager, - reporter: p.Reporter, - prevAuditCanceler: func() {}, - }, nil -} - -// ListenerNotificationParsers for the 'event.Listener' event producer. -func (ap *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { - return nil -} - -// ListenerNotificationHandlers for the 'event.Listener' event producer. -func (ap *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { - return nil -} - -// StartAuditHandler for the internal event producer. -func (ap *Processor) StartAuditHandler() event.Handler { - return ap.handleNewAuditRound -} - -func (r *epochAuditReporter) WriteReport(rep *audit.Report) error { - res := rep.Result() - res.ForEpoch(r.epoch) - - return r.rep.WriteReport(rep) -} diff --git a/pkg/innerring/processors/audit/scheduler.go b/pkg/innerring/processors/audit/scheduler.go deleted file mode 100644 index fbc5fa927..000000000 --- a/pkg/innerring/processors/audit/scheduler.go +++ /dev/null @@ -1,65 +0,0 @@ -package audit - -import ( - "errors" - "fmt" - "sort" - "strings" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "go.uber.org/zap" -) - -var ErrInvalidIRNode = errors.New("node is not in the inner ring list") - -func (ap *Processor) selectContainersToAudit(epoch uint64) ([]cid.ID, error) { - containers, err := ap.containerClient.ContainersOf(nil) - if err != nil { - return nil, fmt.Errorf("can't get list of containers to start audit: %w", err) - } - - // consider getting extra information about container complexity from - // audit contract there - ap.log.Debug(logs.AuditContainerListingFinished, - zap.Int("total amount", len(containers)), - ) - - sort.Slice(containers, func(i, j int) bool { - return strings.Compare(containers[i].EncodeToString(), containers[j].EncodeToString()) < 0 - }) - - ind := ap.irList.InnerRingIndex() - irSize := ap.irList.InnerRingSize() - - if ind < 0 || ind >= irSize { - return nil, ErrInvalidIRNode - } - - return Select(containers, epoch, uint64(ind), uint64(irSize)), nil -} - -func Select(ids []cid.ID, epoch, index, size uint64) []cid.ID { - if index >= size { - return nil - } - - var a, b uint64 - - ln := uint64(len(ids)) - pivot := ln % size - delta := ln / size - - index = (index + epoch) % size - if index < pivot { - a = delta + 1 - } else { - a = delta - b = pivot - } - - from := a*index + b - to := a*(index+1) + b - - return ids[from:to] -} diff --git a/pkg/innerring/processors/audit/scheduler_test.go b/pkg/innerring/processors/audit/scheduler_test.go deleted file mode 100644 index 51461beb7..000000000 --- a/pkg/innerring/processors/audit/scheduler_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package audit_test - -import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/audit" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "github.com/stretchr/testify/require" -) - -func TestSelect(t *testing.T) { - cids := generateContainers(10) - - t.Run("invalid input", func(t *testing.T) { - require.Empty(t, audit.Select(cids, 0, 0, 0)) - }) - - t.Run("even split", func(t *testing.T) { - const irSize = 5 // every node takes two audit nodes - - m := hitMap(cids) - - for i := 0; i < irSize; i++ { - s := audit.Select(cids, 0, uint64(i), irSize) - require.Equal(t, len(cids)/irSize, len(s)) - - for _, id := range s { - n, ok := m[id.EncodeToString()] - require.True(t, ok) - require.Equal(t, 0, n) - m[id.EncodeToString()] = 1 - } - } - - require.True(t, allHit(m)) - }) - - t.Run("odd split", func(t *testing.T) { - const irSize = 3 - - m := hitMap(cids) - - for i := 0; i < irSize; i++ { - s := audit.Select(cids, 0, uint64(i), irSize) - - for _, id := range s { - n, ok := m[id.EncodeToString()] - require.True(t, ok) - require.Equal(t, 0, n) - m[id.EncodeToString()] = 1 - } - } - - require.True(t, allHit(m)) - }) - - t.Run("epoch shift", func(t *testing.T) { - const irSize = 4 - - m := hitMap(cids) - - for i := 0; i < irSize; i++ { - s := audit.Select(cids, uint64(i), 0, irSize) - - for _, id := range s { - n, ok := m[id.EncodeToString()] - require.True(t, ok) - require.Equal(t, 0, n) - m[id.EncodeToString()] = 1 - } - } - - require.True(t, allHit(m)) - }) -} - -func generateContainers(n int) []cid.ID { - result := make([]cid.ID, n) - - for i := 0; i < n; i++ { - result[i] = cidtest.ID() - } - - return result -} - -func hitMap(ids []cid.ID) map[string]int { - result := make(map[string]int, len(ids)) - - for _, id := range ids { - result[id.EncodeToString()] = 0 - } - - return result -} - -func allHit(m map[string]int) bool { - for _, v := range m { - if v == 0 { - return false - } - } - - return true -} diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index 17e445b13..96ea4a01a 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -2,9 +2,7 @@ package netmap import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" "go.uber.org/zap" @@ -63,8 +61,6 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { np.netmapSnapshot.update(*networkMap, epoch) np.handleCleanupTick(netmapCleanupTick{epoch: epoch, txHash: ev.TxHash()}) - np.handleNewAudit(audit.NewAuditStartEvent(epoch)) - np.handleAuditSettlements(settlement.NewAuditEvent(epoch)) np.handleAlphabetSync(governance.NewSyncEvent(ev.TxHash())) np.handleNotaryDeposit(ev) } diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index f6f490cec..39069a963 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -65,10 +65,8 @@ type ( netmapSnapshot cleanupTable - handleNewAudit event.Handler - handleAuditSettlements event.Handler - handleAlphabetSync event.Handler - handleNotaryDeposit event.Handler + handleAlphabetSync event.Handler + handleNotaryDeposit event.Handler nodeValidator NodeValidator @@ -89,10 +87,8 @@ type ( CleanupThreshold uint64 // in epochs ContainerWrapper *container.Client - HandleAudit event.Handler - AuditSettlementsHandler event.Handler - AlphabetSyncHandler event.Handler - NotaryDepositHandler event.Handler + AlphabetSyncHandler event.Handler + NotaryDepositHandler event.Handler NodeValidator NodeValidator @@ -119,10 +115,6 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/netmap: global state is not set") case p.AlphabetState == nil: return nil, errors.New("ir/netmap: global state is not set") - case p.HandleAudit == nil: - return nil, errors.New("ir/netmap: audit handler is not set") - case p.AuditSettlementsHandler == nil: - return nil, errors.New("ir/netmap: audit settlement handler is not set") case p.AlphabetSyncHandler == nil: return nil, errors.New("ir/netmap: alphabet sync handler is not set") case p.NotaryDepositHandler == nil: @@ -151,9 +143,6 @@ func New(p *Params) (*Processor, error) { netmapClient: p.NetmapClient, containerWrp: p.ContainerWrapper, netmapSnapshot: newCleanupTable(p.CleanupEnabled, p.CleanupThreshold), - handleNewAudit: p.HandleAudit, - - handleAuditSettlements: p.AuditSettlementsHandler, handleAlphabetSync: p.AlphabetSyncHandler, diff --git a/pkg/innerring/processors/settlement/audit/calculate.go b/pkg/innerring/processors/settlement/audit/calculate.go deleted file mode 100644 index 75b8c56a4..000000000 --- a/pkg/innerring/processors/settlement/audit/calculate.go +++ /dev/null @@ -1,336 +0,0 @@ -package audit - -import ( - "bytes" - "crypto/ecdsa" - "crypto/elliptic" - "encoding/hex" - "math/big" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/audit" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "go.uber.org/zap" -) - -// CalculatePrm groups the required parameters of -// Calculator.CalculateForEpoch call. -type CalculatePrm struct { - // Number of epoch to perform the calculation. - Epoch uint64 -} - -type singleResultCtx struct { - eAudit uint64 - - auditResult *audit.Result - - log *logger.Logger - - txTable *common.TransferTable - - cnrInfo common.ContainerInfo - - cnrNodes []common.NodeInfo - - passNodes map[string]common.NodeInfo - - sumSGSize *big.Int - - auditFee *big.Int -} - -var ( - bigGB = big.NewInt(1 << 30) - bigZero = big.NewInt(0) - bigOne = big.NewInt(1) -) - -// Calculate calculates payments for audit results in a specific epoch of the network. -// Wraps the results in a money transfer transaction and sends it to the network. -func (c *Calculator) Calculate(p *CalculatePrm) { - log := &logger.Logger{Logger: c.opts.log.With( - zap.Uint64("current epoch", p.Epoch), - )} - - if p.Epoch == 0 { - log.Info(logs.AuditSettlementsAreIgnoredForZeroEpoch) - return - } - - log.Info(logs.AuditCalculateAuditSettlements) - - log.Debug(logs.AuditGettingResultsForThePreviousEpoch) - prevEpoch := p.Epoch - 1 - - auditResults, err := c.prm.ResultStorage.AuditResultsForEpoch(prevEpoch) - if err != nil { - log.Error(logs.AuditCouldNotCollectAuditResults) - return - } else if len(auditResults) == 0 { - log.Debug(logs.AuditNoAuditResultsInPreviousEpoch) - return - } - - auditFee, err := c.prm.AuditFeeFetcher.AuditFee() - if err != nil { - log.Warn(logs.AuditCantFetchAuditFeeFromNetworkConfig, - zap.String("error", err.Error())) - auditFee = 0 - } - - log.Debug(logs.AuditProcessingAuditResults, - zap.Int("number", len(auditResults)), - ) - - table := common.NewTransferTable() - - for i := range auditResults { - c.processResult(&singleResultCtx{ - log: log, - auditResult: auditResults[i], - txTable: table, - auditFee: big.NewInt(0).SetUint64(auditFee), - }) - } - - log.Debug(logs.AuditProcessingTransfers) - - common.TransferAssets(c.prm.Exchanger, table, common.AuditSettlementDetails(prevEpoch)) -} - -func (c *Calculator) processResult(ctx *singleResultCtx) { - ctx.log = &logger.Logger{Logger: ctx.log.With( - zap.Stringer("cid", ctx.containerID()), - zap.Uint64("audit epoch", ctx.auditResult.Epoch()), - )} - - ctx.log.Debug(logs.AuditReadingInformationAboutTheContainer) - - ok := c.readContainerInfo(ctx) - if !ok { - return - } - - ctx.log.Debug(logs.AuditBuildingPlacement) - - ok = c.buildPlacement(ctx) - if !ok { - return - } - - ctx.log.Debug(logs.AuditCollectingPassedNodes) - - ok = c.collectPassNodes(ctx) - if !ok { - return - } - - ctx.log.Debug(logs.AuditCalculatingSumOfTheSizesOfAllStorageGroups) - - ok = c.sumSGSizes(ctx) - if !ok { - return - } - - ctx.log.Debug(logs.AuditFillingTransferTable) - - c.fillTransferTable(ctx) -} - -func (c *Calculator) readContainerInfo(ctx *singleResultCtx) bool { - cnr, ok := ctx.auditResult.Container() - if !ok { - ctx.log.Error(logs.AuditMissingContainerInAuditResult) - return false - } - - var err error - - ctx.cnrInfo, err = c.prm.ContainerStorage.ContainerInfo(cnr) - if err != nil { - ctx.log.Error(logs.AuditCouldNotGetContainerInfo, - zap.String("error", err.Error()), - ) - } - - return err == nil -} - -func (c *Calculator) buildPlacement(ctx *singleResultCtx) bool { - var err error - - ctx.cnrNodes, err = c.prm.PlacementCalculator.ContainerNodes(ctx.auditEpoch(), ctx.containerID()) - if err != nil { - ctx.log.Error(logs.AuditCouldNotGetContainerNodes, - zap.String("error", err.Error()), - ) - } - - empty := len(ctx.cnrNodes) == 0 - if empty { - ctx.log.Debug(logs.AuditEmptyListOfContainerNodes) - } - - return err == nil && !empty -} - -func (c *Calculator) collectPassNodes(ctx *singleResultCtx) bool { - ctx.passNodes = make(map[string]common.NodeInfo) - - for _, cnrNode := range ctx.cnrNodes { - // TODO(@cthulhu-rider): neofs-sdk-go#241 use dedicated method - ctx.auditResult.IteratePassedStorageNodes(func(passNode []byte) bool { - if !bytes.Equal(cnrNode.PublicKey(), passNode) { - return true - } - - failed := false - - ctx.auditResult.IterateFailedStorageNodes(func(failNode []byte) bool { - failed = bytes.Equal(cnrNode.PublicKey(), failNode) - return !failed - }) - - if !failed { - ctx.passNodes[hex.EncodeToString(passNode)] = cnrNode - } - - return false - }) - } - - empty := len(ctx.passNodes) == 0 - if empty { - ctx.log.Debug(logs.AuditNoneOfTheContainerNodesPassedTheAudit) - } - - return !empty -} - -func (c *Calculator) sumSGSizes(ctx *singleResultCtx) bool { - sumPassSGSize := uint64(0) - fail := false - - var addr oid.Address - addr.SetContainer(ctx.containerID()) - - ctx.auditResult.IteratePassedStorageGroups(func(id oid.ID) bool { - addr.SetObject(id) - - sgInfo, err := c.prm.SGStorage.SGInfo(addr) - if err != nil { - ctx.log.Error(logs.AuditCouldNotGetSGInfo, - zap.String("id", id.String()), - zap.String("error", err.Error()), - ) - - fail = true - - return false // we also can continue and calculate at least some part - } - - sumPassSGSize += sgInfo.Size() - - return true - }) - - if fail { - return false - } - - if sumPassSGSize == 0 { - ctx.log.Debug(logs.AuditZeroSumSGSize) - return false - } - - ctx.sumSGSize = big.NewInt(int64(sumPassSGSize)) - - return true -} - -func (c *Calculator) fillTransferTable(ctx *singleResultCtx) bool { - cnrOwner := ctx.cnrInfo.Owner() - - // add txs to pay for storage node - for k, info := range ctx.passNodes { - ownerID, err := c.prm.AccountStorage.ResolveKey(info) - if err != nil { - ctx.log.Error(logs.AuditCouldNotResolvePublicKeyOfTheStorageNode, - zap.String("error", err.Error()), - zap.String("key", k), - ) - - return false // we also can continue and calculate at least some part - } - - price := info.Price() - - ctx.log.Debug(logs.AuditCalculatingStorageNodeSalaryForAudit, - zap.Stringer("sum SG size", ctx.sumSGSize), - zap.Stringer("price", price), - ) - - fee := big.NewInt(0).Mul(price, ctx.sumSGSize) - fee.Div(fee, bigGB) - - if fee.Cmp(bigZero) == 0 { - fee.Add(fee, bigOne) - } - - ctx.txTable.Transfer(&common.TransferTx{ - From: cnrOwner, - To: *ownerID, - Amount: fee, - }) - } - - // add txs to pay inner ring node for audit result - auditIR, err := ownerFromKey(ctx.auditResult.AuditorKey()) - if err != nil { - ctx.log.Error(logs.AuditCouldNotParsePublicKeyOfTheInnerRingNode, - zap.String("error", err.Error()), - zap.String("key", hex.EncodeToString(ctx.auditResult.AuditorKey())), - ) - - return false - } - - ctx.txTable.Transfer(&common.TransferTx{ - From: cnrOwner, - To: *auditIR, - Amount: ctx.auditFee, - }) - - return false -} - -func (c *singleResultCtx) containerID() cid.ID { - cnr, _ := c.auditResult.Container() - return cnr -} - -func (c *singleResultCtx) auditEpoch() uint64 { - if c.eAudit == 0 { - c.eAudit = c.auditResult.Epoch() - } - - return c.eAudit -} - -func ownerFromKey(key []byte) (*user.ID, error) { - pubKey, err := keys.NewPublicKeyFromBytes(key, elliptic.P256()) - if err != nil { - return nil, err - } - - var id user.ID - user.IDFromKey(&id, (ecdsa.PublicKey)(*pubKey)) - - return &id, nil -} diff --git a/pkg/innerring/processors/settlement/audit/calculator.go b/pkg/innerring/processors/settlement/audit/calculator.go deleted file mode 100644 index fb8d82071..000000000 --- a/pkg/innerring/processors/settlement/audit/calculator.go +++ /dev/null @@ -1,48 +0,0 @@ -package audit - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// Calculator represents a component for calculating payments -// based on data audit results and sending remittances to the chain. -type Calculator struct { - prm *CalculatorPrm - - opts *options -} - -// CalculatorOption is a Calculator constructor's option. -type CalculatorOption func(*options) - -type options struct { - log *logger.Logger -} - -func defaultOptions() *options { - return &options{ - log: &logger.Logger{Logger: zap.L()}, - } -} - -// NewCalculator creates, initializes and returns a new Calculator instance. -func NewCalculator(p *CalculatorPrm, opts ...CalculatorOption) *Calculator { - o := defaultOptions() - - for i := range opts { - opts[i](o) - } - - return &Calculator{ - prm: p, - opts: o, - } -} - -// WithLogger returns an option to specify the logging component. -func WithLogger(l *logger.Logger) CalculatorOption { - return func(o *options) { - o.log = l - } -} diff --git a/pkg/innerring/processors/settlement/audit/prm.go b/pkg/innerring/processors/settlement/audit/prm.go deleted file mode 100644 index d357f0d4f..000000000 --- a/pkg/innerring/processors/settlement/audit/prm.go +++ /dev/null @@ -1,49 +0,0 @@ -package audit - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/common" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/audit" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -// CalculatorPrm groups the parameters of Calculator's constructor. -type CalculatorPrm struct { - ResultStorage ResultStorage - - ContainerStorage common.ContainerStorage - - PlacementCalculator common.PlacementCalculator - - SGStorage SGStorage - - AccountStorage common.AccountStorage - - Exchanger common.Exchanger - - AuditFeeFetcher FeeFetcher -} - -// ResultStorage is an interface of storage of the audit results. -type ResultStorage interface { - // Must return all audit results by epoch number. - AuditResultsForEpoch(epoch uint64) ([]*audit.Result, error) -} - -// SGInfo groups the data about FrostFS storage group -// necessary for calculating audit fee. -type SGInfo interface { - // Must return sum size of the all group members. - Size() uint64 -} - -// SGStorage is an interface of storage of the storage groups. -type SGStorage interface { - // Must return information about the storage group by address. - SGInfo(oid.Address) (SGInfo, error) -} - -// FeeFetcher wraps AuditFee method that returns audit fee price from -// the network configuration. -type FeeFetcher interface { - AuditFee() (uint64, error) -} diff --git a/pkg/innerring/processors/settlement/basic/collect.go b/pkg/innerring/processors/settlement/basic/collect.go deleted file mode 100644 index 024769c06..000000000 --- a/pkg/innerring/processors/settlement/basic/collect.go +++ /dev/null @@ -1,114 +0,0 @@ -package basic - -import ( - "math/big" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/common" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - "go.uber.org/zap" -) - -var ( - bigGB = big.NewInt(1 << 30) - bigZero = big.NewInt(0) - bigOne = big.NewInt(1) -) - -func (inc *IncomeSettlementContext) Collect() { - inc.mu.Lock() - defer inc.mu.Unlock() - - cachedRate, err := inc.rate.BasicRate() - if err != nil { - inc.log.Error(logs.BasicCantGetBasicIncomeRate, - zap.String("error", err.Error())) - - return - } - - if cachedRate == 0 { - inc.noop = true - return - } - - cnrEstimations, err := inc.estimations.Estimations(inc.epoch) - if err != nil { - inc.log.Error(logs.BasicCantFetchContainerSizeEstimations, - zap.Uint64("epoch", inc.epoch), - zap.String("error", err.Error())) - - return - } - - txTable := common.NewTransferTable() - - for i := range cnrEstimations { - owner, err := inc.container.ContainerInfo(cnrEstimations[i].ContainerID) - if err != nil { - inc.log.Warn(logs.BasicCantFetchContainerInfo, - zap.Uint64("epoch", inc.epoch), - zap.Stringer("container_id", cnrEstimations[i].ContainerID), - zap.String("error", err.Error())) - - continue - } - - cnrNodes, err := inc.placement.ContainerNodes(inc.epoch, cnrEstimations[i].ContainerID) - if err != nil { - inc.log.Debug(logs.BasicCantFetchContainerInfo, - zap.Uint64("epoch", inc.epoch), - zap.Stringer("container_id", cnrEstimations[i].ContainerID), - zap.String("error", err.Error())) - - continue - } - - avg := inc.avgEstimation(cnrEstimations[i]) // average container size per node - total := calculateBasicSum(avg, cachedRate, len(cnrNodes)) - - // fill distribute asset table - for i := range cnrNodes { - inc.distributeTable.Put(cnrNodes[i].PublicKey(), avg) - } - - txTable.Transfer(&common.TransferTx{ - From: owner.Owner(), - To: inc.bankOwner, - Amount: total, - }) - } - - common.TransferAssets(inc.exchange, txTable, common.BasicIncomeCollectionDetails(inc.epoch)) -} - -// avgEstimation returns estimation value for a single container. Right now it -// simply calculates an average of all announcements, however it can be smarter and -// base the result on reputation of the announcers and clever math. -func (inc *IncomeSettlementContext) avgEstimation(e *cntClient.Estimations) (avg uint64) { - if len(e.Values) == 0 { - return 0 - } - - for i := range e.Values { - avg += e.Values[i].Size - } - - return avg / uint64(len(e.Values)) -} - -func calculateBasicSum(size, rate uint64, ln int) *big.Int { - bigRate := big.NewInt(int64(rate)) - - total := size * uint64(ln) - - price := big.NewInt(0).SetUint64(total) - price.Mul(price, bigRate) - price.Div(price, bigGB) - - if price.Cmp(bigZero) == 0 { - price.Add(price, bigOne) - } - - return price -} diff --git a/pkg/innerring/processors/settlement/basic/context.go b/pkg/innerring/processors/settlement/basic/context.go deleted file mode 100644 index 59bedf2e4..000000000 --- a/pkg/innerring/processors/settlement/basic/context.go +++ /dev/null @@ -1,80 +0,0 @@ -package basic - -import ( - "math/big" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/util" -) - -type ( - EstimationFetcher interface { - Estimations(uint64) ([]*container.Estimations, error) - } - - RateFetcher interface { - BasicRate() (uint64, error) - } - - // BalanceFetcher uses NEP-17 compatible balance contract. - BalanceFetcher interface { - Balance(id user.ID) (*big.Int, error) - } - - IncomeSettlementContext struct { - mu sync.Mutex // lock to prevent collection and distribution in the same time - - noop bool - - log *logger.Logger - epoch uint64 - - rate RateFetcher - estimations EstimationFetcher - balances BalanceFetcher - container common.ContainerStorage - placement common.PlacementCalculator - exchange common.Exchanger - accounts common.AccountStorage - - bankOwner user.ID - - // this table is not thread safe, make sure you use it with mu.Lock() - distributeTable *NodeSizeTable - } - - IncomeSettlementContextPrms struct { - Log *logger.Logger - Epoch uint64 - Rate RateFetcher - Estimations EstimationFetcher - Balances BalanceFetcher - Container common.ContainerStorage - Placement common.PlacementCalculator - Exchange common.Exchanger - Accounts common.AccountStorage - } -) - -func NewIncomeSettlementContext(p *IncomeSettlementContextPrms) *IncomeSettlementContext { - res := &IncomeSettlementContext{ - log: p.Log, - epoch: p.Epoch, - rate: p.Rate, - estimations: p.Estimations, - balances: p.Balances, - container: p.Container, - placement: p.Placement, - exchange: p.Exchange, - accounts: p.Accounts, - distributeTable: NewNodeSizeTable(), - } - - res.bankOwner.SetScriptHash(util.Uint160{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) - - return res -} diff --git a/pkg/innerring/processors/settlement/basic/distribute.go b/pkg/innerring/processors/settlement/basic/distribute.go deleted file mode 100644 index 44a8ccea3..000000000 --- a/pkg/innerring/processors/settlement/basic/distribute.go +++ /dev/null @@ -1,59 +0,0 @@ -package basic - -import ( - "encoding/hex" - "math/big" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/common" - "go.uber.org/zap" -) - -func (inc *IncomeSettlementContext) Distribute() { - inc.mu.Lock() - defer inc.mu.Unlock() - - if inc.noop { - return - } - - txTable := common.NewTransferTable() - - bankBalance, err := inc.balances.Balance(inc.bankOwner) - if err != nil { - inc.log.Error(logs.BasicCantFetchBalanceOfBankingAccount, - zap.String("error", err.Error())) - - return - } - - total := inc.distributeTable.Total() - - inc.distributeTable.Iterate(func(key []byte, n *big.Int) { - nodeOwner, err := inc.accounts.ResolveKey(nodeInfoWrapper(key)) - if err != nil { - inc.log.Warn(logs.BasicCantTransformPublicKeyToOwnerID, - zap.String("public_key", hex.EncodeToString(key)), - zap.String("error", err.Error())) - - return - } - - txTable.Transfer(&common.TransferTx{ - From: inc.bankOwner, - To: *nodeOwner, - Amount: normalizedValue(n, total, bankBalance), - }) - }) - - common.TransferAssets(inc.exchange, txTable, common.BasicIncomeDistributionDetails(inc.epoch)) -} - -func normalizedValue(n, total, limit *big.Int) *big.Int { - if limit.Cmp(bigZero) == 0 { - return big.NewInt(0) - } - - n.Mul(n, limit) - return n.Div(n, total) -} diff --git a/pkg/innerring/processors/settlement/basic/distribute_test.go b/pkg/innerring/processors/settlement/basic/distribute_test.go deleted file mode 100644 index 24eb0db3d..000000000 --- a/pkg/innerring/processors/settlement/basic/distribute_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package basic - -import ( - "math/big" - "testing" - - "github.com/stretchr/testify/require" -) - -type normalizedValueCase struct { - name string - n, total, limit uint64 - expected uint64 -} - -func TestNormalizedValues(t *testing.T) { - testCases := []normalizedValueCase{ - { - name: "zero limit", - n: 50, - total: 100, - limit: 0, - expected: 0, - }, - { - name: "scale down", - n: 50, - total: 100, - limit: 10, - expected: 5, - }, - { - name: "scale up", - n: 50, - total: 100, - limit: 1000, - expected: 500, - }, - } - - for _, testCase := range testCases { - testNormalizedValues(t, testCase) - } -} - -func testNormalizedValues(t *testing.T, c normalizedValueCase) { - n := big.NewInt(0).SetUint64(c.n) - total := big.NewInt(0).SetUint64(c.total) - limit := big.NewInt(0).SetUint64(c.limit) - exp := big.NewInt(0).SetUint64(c.expected) - - got := normalizedValue(n, total, limit) - require.Zero(t, exp.Cmp(got), c.name) -} diff --git a/pkg/innerring/processors/settlement/basic/util.go b/pkg/innerring/processors/settlement/basic/util.go deleted file mode 100644 index 258bae46f..000000000 --- a/pkg/innerring/processors/settlement/basic/util.go +++ /dev/null @@ -1,44 +0,0 @@ -package basic - -import ( - "math/big" -) - -// NodeSizeTable is not thread safe, make sure it is accessed with external -// locks or in single routine. -type NodeSizeTable struct { - prices map[string]uint64 - total uint64 -} - -func (t *NodeSizeTable) Put(id []byte, avg uint64) { - t.prices[string(id)] += avg - t.total += avg -} - -func (t *NodeSizeTable) Total() *big.Int { - return big.NewInt(0).SetUint64(t.total) -} - -func (t *NodeSizeTable) Iterate(f func([]byte, *big.Int)) { - for k, v := range t.prices { - n := big.NewInt(0).SetUint64(v) - f([]byte(k), n) - } -} - -func NewNodeSizeTable() *NodeSizeTable { - return &NodeSizeTable{ - prices: make(map[string]uint64), - } -} - -type nodeInfoWrapper []byte - -func (nodeInfoWrapper) Price() *big.Int { - panic("should not be used") -} - -func (n nodeInfoWrapper) PublicKey() []byte { - return n -} diff --git a/pkg/innerring/processors/settlement/calls.go b/pkg/innerring/processors/settlement/calls.go deleted file mode 100644 index 2687ad206..000000000 --- a/pkg/innerring/processors/settlement/calls.go +++ /dev/null @@ -1,133 +0,0 @@ -package settlement - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// HandleAuditEvent catches a new AuditEvent and -// adds AuditProcessor call to the execution queue. -func (p *Processor) HandleAuditEvent(e event.Event) { - ev := e.(AuditEvent) - - epoch := ev.Epoch() - - if !p.state.IsAlphabet() { - p.log.Info(logs.SettlementNonAlphabetModeIgnoreAuditPayments) - - return - } - - log := &logger.Logger{Logger: p.log.With( - zap.Uint64("epoch", epoch), - )} - - log.Info(logs.SettlementNewAuditSettlementEvent) - - if epoch == 0 { - log.Debug(logs.SettlementIgnoreGenesisEpoch) - return - } - - handler := &auditEventHandler{ - log: log, - epoch: epoch, - proc: p.auditProc, - } - - err := p.pool.Submit(handler.handle) - if err != nil { - log.Warn(logs.SettlementCouldNotAddHandlerOfAuditEventToQueue, - zap.String("error", err.Error()), - ) - - return - } - - log.Debug(logs.SettlementAuditEventHandlingSuccessfullyScheduled) -} - -func (p *Processor) HandleIncomeCollectionEvent(e event.Event) { - ev := e.(BasicIncomeCollectEvent) - epoch := ev.Epoch() - - if !p.state.IsAlphabet() { - p.log.Info(logs.SettlementNonAlphabetModeIgnoreIncomeCollectionEvent) - - return - } - - p.log.Info(logs.SettlementStartBasicIncomeCollection, - zap.Uint64("epoch", epoch)) - - p.contextMu.Lock() - defer p.contextMu.Unlock() - - if _, ok := p.incomeContexts[epoch]; ok { - p.log.Error(logs.SettlementIncomeContextAlreadyExists, - zap.Uint64("epoch", epoch)) - - return - } - - incomeCtx, err := p.basicIncome.CreateContext(epoch) - if err != nil { - p.log.Error(logs.SettlementCantCreateIncomeContext, - zap.String("error", err.Error())) - - return - } - - p.incomeContexts[epoch] = incomeCtx - - err = p.pool.Submit(func() { - incomeCtx.Collect() - }) - if err != nil { - p.log.Warn(logs.SettlementCouldNotAddHandlerOfBasicIncomeCollectionToQueue, - zap.String("error", err.Error()), - ) - - return - } -} - -func (p *Processor) HandleIncomeDistributionEvent(e event.Event) { - ev := e.(BasicIncomeDistributeEvent) - epoch := ev.Epoch() - - if !p.state.IsAlphabet() { - p.log.Info(logs.SettlementNonAlphabetModeIgnoreIncomeDistributionEvent) - - return - } - - p.log.Info(logs.SettlementStartBasicIncomeDistribution, - zap.Uint64("epoch", epoch)) - - p.contextMu.Lock() - defer p.contextMu.Unlock() - - incomeCtx, ok := p.incomeContexts[epoch] - delete(p.incomeContexts, epoch) - - if !ok { - p.log.Warn(logs.SettlementIncomeContextDistributionDoesNotExists, - zap.Uint64("epoch", epoch)) - - return - } - - err := p.pool.Submit(func() { - incomeCtx.Distribute() - }) - if err != nil { - p.log.Warn(logs.SettlementCouldNotAddHandlerOfBasicIncomeDistributionToQueue, - zap.String("error", err.Error()), - ) - - return - } -} diff --git a/pkg/innerring/processors/settlement/common/details.go b/pkg/innerring/processors/settlement/common/details.go deleted file mode 100644 index 1cf719f63..000000000 --- a/pkg/innerring/processors/settlement/common/details.go +++ /dev/null @@ -1,33 +0,0 @@ -package common - -import ( - "encoding/binary" -) - -var ( - auditPrefix = []byte{0x40} - basicIncomeCollectionPrefix = []byte{0x41} - basicIncomeDistributionPrefix = []byte{0x42} -) - -func AuditSettlementDetails(epoch uint64) []byte { - return details(auditPrefix, epoch) -} - -func BasicIncomeCollectionDetails(epoch uint64) []byte { - return details(basicIncomeCollectionPrefix, epoch) -} - -func BasicIncomeDistributionDetails(epoch uint64) []byte { - return details(basicIncomeDistributionPrefix, epoch) -} - -func details(prefix []byte, epoch uint64) []byte { - prefixLen := len(prefix) - buf := make([]byte, prefixLen+8) - - copy(buf, prefix) - binary.LittleEndian.PutUint64(buf[prefixLen:], epoch) - - return buf -} diff --git a/pkg/innerring/processors/settlement/common/details_test.go b/pkg/innerring/processors/settlement/common/details_test.go deleted file mode 100644 index 9755e6aef..000000000 --- a/pkg/innerring/processors/settlement/common/details_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package common - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestAuditSettlementDetails(t *testing.T) { - var n uint64 = 1994 // 0x7CA - exp := []byte{0x40, 0xCA, 0x07, 0, 0, 0, 0, 0, 0} - got := AuditSettlementDetails(n) - require.Equal(t, exp, got) -} - -func TestBasicIncomeCollectionDetails(t *testing.T) { - var n uint64 = 1994 // 0x7CA - exp := []byte{0x41, 0xCA, 0x07, 0, 0, 0, 0, 0, 0} - got := BasicIncomeCollectionDetails(n) - require.Equal(t, exp, got) -} - -func TestBasicIncomeDistributionDetails(t *testing.T) { - var n uint64 = 1994 // 0x7CA - exp := []byte{0x42, 0xCA, 0x07, 0, 0, 0, 0, 0, 0} - got := BasicIncomeDistributionDetails(n) - require.Equal(t, exp, got) -} diff --git a/pkg/innerring/processors/settlement/common/types.go b/pkg/innerring/processors/settlement/common/types.go deleted file mode 100644 index 9dca0fd0d..000000000 --- a/pkg/innerring/processors/settlement/common/types.go +++ /dev/null @@ -1,54 +0,0 @@ -package common - -import ( - "math/big" - - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" -) - -// NodeInfo groups the data about the storage node -// necessary for calculating audit fees. -type NodeInfo interface { - // Must return storage price of the node for one epoch in GASe-12. - Price() *big.Int - - // Must return public key of the node. - PublicKey() []byte -} - -// ContainerInfo groups the data about FrostFS container -// necessary for calculating audit fee. -type ContainerInfo interface { - // Must return identifier of the container owner. - Owner() user.ID -} - -// ContainerStorage is an interface of -// storage of the FrostFS containers. -type ContainerStorage interface { - // Must return information about the container by ID. - ContainerInfo(cid.ID) (ContainerInfo, error) -} - -// PlacementCalculator is a component interface -// that builds placement vectors. -type PlacementCalculator interface { - // Must return information about the nodes from container by its ID of the given epoch. - ContainerNodes(uint64, cid.ID) ([]NodeInfo, error) -} - -// AccountStorage is an network member accounts interface. -type AccountStorage interface { - // Must resolve information about the storage node - // to its ID in system. - ResolveKey(NodeInfo) (*user.ID, error) -} - -// Exchanger is an interface of monetary component. -type Exchanger interface { - // Must transfer amount of GASe-12 from sender to recipient. - // - // Amount must be positive. - Transfer(sender, recipient user.ID, amount *big.Int, details []byte) -} diff --git a/pkg/innerring/processors/settlement/common/util.go b/pkg/innerring/processors/settlement/common/util.go deleted file mode 100644 index 6f40fb577..000000000 --- a/pkg/innerring/processors/settlement/common/util.go +++ /dev/null @@ -1,74 +0,0 @@ -package common - -import ( - "math/big" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" -) - -type TransferTable struct { - txs map[string]map[string]*TransferTx -} - -type TransferTx struct { - From, To user.ID - - Amount *big.Int -} - -func NewTransferTable() *TransferTable { - return &TransferTable{ - txs: make(map[string]map[string]*TransferTx), - } -} - -func (t *TransferTable) Transfer(tx *TransferTx) { - if tx.From.Equals(tx.To) { - return - } - - from, to := tx.From.EncodeToString(), tx.To.EncodeToString() - - m, ok := t.txs[from] - if !ok { - if m, ok = t.txs[to]; ok { - to = from // ignore `From = To` swap because `From` doesn't require - tx.Amount.Neg(tx.Amount) - } else { - m = make(map[string]*TransferTx, 1) - t.txs[from] = m - } - } - - tgt, ok := m[to] - if !ok { - m[to] = tx - return - } - - tgt.Amount.Add(tgt.Amount, tx.Amount) -} - -func (t *TransferTable) Iterate(f func(*TransferTx)) { - for _, m := range t.txs { - for _, tx := range m { - f(tx) - } - } -} - -func TransferAssets(e Exchanger, t *TransferTable, details []byte) { - t.Iterate(func(tx *TransferTx) { - sign := tx.Amount.Sign() - if sign == 0 { - return - } - - if sign < 0 { - tx.From, tx.To = tx.To, tx.From - tx.Amount.Neg(tx.Amount) - } - - e.Transfer(tx.From, tx.To, tx.Amount, details) - }) -} diff --git a/pkg/innerring/processors/settlement/deps.go b/pkg/innerring/processors/settlement/deps.go deleted file mode 100644 index 37d7955ad..000000000 --- a/pkg/innerring/processors/settlement/deps.go +++ /dev/null @@ -1,17 +0,0 @@ -package settlement - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/basic" -) - -// AuditProcessor is an interface of data audit fee processor. -type AuditProcessor interface { - // Must process data audit conducted in epoch. - ProcessAuditSettlements(epoch uint64) -} - -// BasicIncomeInitializer is an interface of basic income context creator. -type BasicIncomeInitializer interface { - // Creates context that processes basic income for provided epoch. - CreateContext(epoch uint64) (*basic.IncomeSettlementContext, error) -} diff --git a/pkg/innerring/processors/settlement/events.go b/pkg/innerring/processors/settlement/events.go deleted file mode 100644 index a47a3e89b..000000000 --- a/pkg/innerring/processors/settlement/events.go +++ /dev/null @@ -1,46 +0,0 @@ -package settlement - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" -) - -// AuditEvent is an event of the start of -// cash settlements for data audit. -type AuditEvent struct { - epoch uint64 -} - -type ( - BasicIncomeCollectEvent = AuditEvent - BasicIncomeDistributeEvent = AuditEvent -) - -// MorphEvent implements Neo:Morph event. -func (e AuditEvent) MorphEvent() {} - -// NewAuditEvent creates new AuditEvent for epoch. -func NewAuditEvent(epoch uint64) event.Event { - return AuditEvent{ - epoch: epoch, - } -} - -// Epoch returns the number of the epoch -// in which the event was generated. -func (e AuditEvent) Epoch() uint64 { - return e.epoch -} - -// NewBasicIncomeCollectEvent for epoch. -func NewBasicIncomeCollectEvent(epoch uint64) event.Event { - return BasicIncomeCollectEvent{ - epoch: epoch, - } -} - -// NewBasicIncomeDistributeEvent for epoch. -func NewBasicIncomeDistributeEvent(epoch uint64) event.Event { - return BasicIncomeDistributeEvent{ - epoch: epoch, - } -} diff --git a/pkg/innerring/processors/settlement/handlers.go b/pkg/innerring/processors/settlement/handlers.go deleted file mode 100644 index e69d829eb..000000000 --- a/pkg/innerring/processors/settlement/handlers.go +++ /dev/null @@ -1,22 +0,0 @@ -package settlement - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" -) - -type auditEventHandler struct { - log *logger.Logger - - epoch uint64 - - proc AuditProcessor -} - -func (p *auditEventHandler) handle() { - p.log.Info(logs.SettlementProcessAuditSettlements) - - p.proc.ProcessAuditSettlements(p.epoch) - - p.log.Info(logs.SettlementAuditProcessingFinished) -} diff --git a/pkg/innerring/processors/settlement/opts.go b/pkg/innerring/processors/settlement/opts.go deleted file mode 100644 index b344f98d6..000000000 --- a/pkg/innerring/processors/settlement/opts.go +++ /dev/null @@ -1,31 +0,0 @@ -package settlement - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// Option is a Processor constructor's option. -type Option func(*options) - -type options struct { - poolSize int - - log *logger.Logger -} - -func defaultOptions() *options { - const poolSize = 10 - - return &options{ - poolSize: poolSize, - log: &logger.Logger{Logger: zap.L()}, - } -} - -// WithLogger returns option to override the component for logging. -func WithLogger(l *logger.Logger) Option { - return func(o *options) { - o.log = l - } -} diff --git a/pkg/innerring/processors/settlement/processor.go b/pkg/innerring/processors/settlement/processor.go deleted file mode 100644 index 1870a0351..000000000 --- a/pkg/innerring/processors/settlement/processor.go +++ /dev/null @@ -1,79 +0,0 @@ -package settlement - -import ( - "fmt" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/basic" - nodeutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/panjf2000/ants/v2" - "go.uber.org/zap" -) - -type ( - // AlphabetState is a callback interface for inner ring global state. - AlphabetState interface { - IsAlphabet() bool - } - - // Processor is an event handler for payments in the system. - Processor struct { - log *logger.Logger - - state AlphabetState - - pool nodeutil.WorkerPool - - auditProc AuditProcessor - - basicIncome BasicIncomeInitializer - - contextMu sync.Mutex - incomeContexts map[uint64]*basic.IncomeSettlementContext - } - - // Prm groups the required parameters of Processor's constructor. - Prm struct { - AuditProcessor AuditProcessor - BasicIncome BasicIncomeInitializer - State AlphabetState - } -) - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf("invalid parameter %s (%T):%v", n, v, v)) -} - -// New creates and returns a new Processor instance. -func New(prm Prm, opts ...Option) *Processor { - switch { - case prm.AuditProcessor == nil: - panicOnPrmValue("AuditProcessor", prm.AuditProcessor) - } - - o := defaultOptions() - - for i := range opts { - opts[i](o) - } - - pool, err := ants.NewPool(o.poolSize, ants.WithNonblocking(true)) - if err != nil { - panic(fmt.Errorf("could not create worker pool: %w", err)) - } - - o.log.Debug(logs.SettlementWorkerPoolForSettlementProcessorSuccessfullyInitialized, - zap.Int("capacity", o.poolSize), - ) - - return &Processor{ - log: o.log, - state: prm.State, - pool: pool, - auditProc: prm.AuditProcessor, - basicIncome: prm.BasicIncome, - incomeContexts: make(map[uint64]*basic.IncomeSettlementContext), - } -} diff --git a/pkg/innerring/rpc.go b/pkg/innerring/rpc.go deleted file mode 100644 index 27338eaa4..000000000 --- a/pkg/innerring/rpc.go +++ /dev/null @@ -1,236 +0,0 @@ -package innerring - -import ( - "context" - "crypto/ecdsa" - "fmt" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - storagegroup2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/storagegroup" - frostfsapiclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/cache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit/auditor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/storagegroup" - "go.uber.org/zap" -) - -type ( - ClientCache struct { - log *logger.Logger - cache interface { - Get(clientcore.NodeInfo) (clientcore.MultiAddressClient, error) - CloseAll() - } - key *ecdsa.PrivateKey - - sgTimeout, headTimeout, rangeTimeout time.Duration - } - - clientCacheParams struct { - Log *logger.Logger - Key *ecdsa.PrivateKey - - AllowExternal bool - - SGTimeout, HeadTimeout, RangeTimeout time.Duration - } -) - -func newClientCache(p *clientCacheParams) *ClientCache { - return &ClientCache{ - log: p.Log, - cache: cache.NewSDKClientCache(cache.ClientCacheOpts{AllowExternal: p.AllowExternal, Key: p.Key}), - key: p.Key, - sgTimeout: p.SGTimeout, - headTimeout: p.HeadTimeout, - rangeTimeout: p.RangeTimeout, - } -} - -func (c *ClientCache) Get(info clientcore.NodeInfo) (clientcore.Client, error) { - // Because cache is used by `ClientCache` exclusively, - // client will always have valid key. - return c.cache.Get(info) -} - -// GetSG polls the container to get the object by id. -// Returns storage groups structure from received object. -// -// Returns an error of type apistatus.ObjectNotFound if storage group is missing. -func (c *ClientCache) GetSG(ctx context.Context, prm storagegroup2.GetSGPrm) (*storagegroup.StorageGroup, error) { - var sgAddress oid.Address - sgAddress.SetContainer(prm.CID) - sgAddress.SetObject(prm.OID) - - return c.getSG(ctx, sgAddress, &prm.NetMap, prm.Container) -} - -func (c *ClientCache) getSG(ctx context.Context, addr oid.Address, nm *netmap.NetMap, cn [][]netmap.NodeInfo) (*storagegroup.StorageGroup, error) { - obj := addr.Object() - - nodes, err := placement.BuildObjectPlacement(nm, cn, &obj) - if err != nil { - return nil, fmt.Errorf("can't build object placement: %w", err) - } - - var info clientcore.NodeInfo - - var getObjPrm frostfsapiclient.GetObjectPrm - getObjPrm.SetAddress(addr) - - for _, node := range placement.FlattenNodes(nodes) { - err := clientcore.NodeInfoFromRawNetmapElement(&info, netmapcore.Node(node)) - if err != nil { - return nil, fmt.Errorf("parse client node info: %w", err) - } - - cli, err := c.getWrappedClient(info) - if err != nil { - c.log.Warn(logs.InnerringCantSetupRemoteConnection, - zap.String("error", err.Error())) - - continue - } - - ctx, cancel := context.WithTimeout(ctx, c.sgTimeout) - - // NOTE: we use the function which does not verify object integrity (checksums, signature), - // but it would be useful to do as part of a data audit. - res, err := cli.GetObject(ctx, getObjPrm) - - cancel() - - if err != nil { - c.log.Warn(logs.InnerringCantGetStorageGroupObject, - zap.String("error", err.Error())) - - continue - } - - var sg storagegroup.StorageGroup - - err = storagegroup.ReadFromObject(&sg, *res.Object()) - if err != nil { - return nil, fmt.Errorf("can't parse storage group from a object: %w", err) - } - - return &sg, nil - } - - var errNotFound apistatus.ObjectNotFound - - return nil, errNotFound -} - -// GetHeader requests node from the container under audit to return object header by id. -func (c *ClientCache) GetHeader(ctx context.Context, prm auditor.GetHeaderPrm) (*object.Object, error) { - var objAddress oid.Address - objAddress.SetContainer(prm.CID) - objAddress.SetObject(prm.OID) - - var info clientcore.NodeInfo - - err := clientcore.NodeInfoFromRawNetmapElement(&info, netmapcore.Node(prm.Node)) - if err != nil { - return nil, fmt.Errorf("parse client node info: %w", err) - } - - cli, err := c.getWrappedClient(info) - if err != nil { - return nil, fmt.Errorf("can't setup remote connection with %s: %w", info.AddressGroup(), err) - } - - cctx, cancel := context.WithTimeout(ctx, c.headTimeout) - - var obj *object.Object - - if prm.NodeIsRelay { - obj, err = frostfsapiclient.GetObjectHeaderFromContainer(cctx, cli, objAddress) - } else { - obj, err = frostfsapiclient.GetRawObjectHeaderLocally(cctx, cli, objAddress) - } - - cancel() - - if err != nil { - return nil, fmt.Errorf("object head error: %w", err) - } - - return obj, nil -} - -// GetRangeHash requests node from the container under audit to return Tillich-Zemor hash of the -// payload range of the object with specified identifier. -func (c *ClientCache) GetRangeHash(ctx context.Context, prm auditor.GetRangeHashPrm) ([]byte, error) { - var objAddress oid.Address - objAddress.SetContainer(prm.CID) - objAddress.SetObject(prm.OID) - - var info clientcore.NodeInfo - - err := clientcore.NodeInfoFromRawNetmapElement(&info, netmapcore.Node(prm.Node)) - if err != nil { - return nil, fmt.Errorf("parse client node info: %w", err) - } - - cli, err := c.getWrappedClient(info) - if err != nil { - return nil, fmt.Errorf("can't setup remote connection with %s: %w", info.AddressGroup(), err) - } - - cctx, cancel := context.WithTimeout(ctx, c.rangeTimeout) - - h, err := frostfsapiclient.HashObjectRange(cctx, cli, objAddress, prm.Range) - - cancel() - - if err != nil { - return nil, fmt.Errorf("object rangehash error: %w", err) - } - - return h, nil -} - -func (c *ClientCache) getWrappedClient(info clientcore.NodeInfo) (frostfsapiclient.Client, error) { - // can be also cached - var cInternal frostfsapiclient.Client - - cli, err := c.Get(info) - if err != nil { - return cInternal, fmt.Errorf("could not get API client from cache") - } - - cInternal.WrapBasicClient(cli) - cInternal.SetPrivateKey(c.key) - - return cInternal, nil -} - -func (c ClientCache) ListSG(ctx context.Context, dst *storagegroup2.SearchSGDst, prm storagegroup2.SearchSGPrm) error { - cli, err := c.getWrappedClient(prm.NodeInfo) - if err != nil { - return fmt.Errorf("could not get API client from cache") - } - - var cliPrm frostfsapiclient.SearchSGPrm - - cliPrm.SetContainerID(prm.Container) - - res, err := cli.SearchSG(ctx, cliPrm) - if err != nil { - return err - } - - dst.Objects = res.IDList() - - return nil -} diff --git a/pkg/innerring/settlement.go b/pkg/innerring/settlement.go deleted file mode 100644 index 90255f5c1..000000000 --- a/pkg/innerring/settlement.go +++ /dev/null @@ -1,301 +0,0 @@ -package innerring - -import ( - "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/sha256" - "encoding/hex" - "fmt" - "math/big" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/audit" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/basic" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement/common" - auditClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/audit" - balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" - containerClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - netmapClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - auditAPI "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/audit" - containerAPI "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - netmapAPI "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/storagegroup" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "go.uber.org/zap" -) - -const ( - auditSettlementContext = "audit" - basicIncomeSettlementContext = "basic income" -) - -type settlementDeps struct { - log *logger.Logger - - cnrSrc container.Source - - auditClient *auditClient.Client - - nmClient *netmapClient.Client - - clientCache *ClientCache - - balanceClient *balanceClient.Client - - settlementCtx string -} - -type auditSettlementDeps struct { - settlementDeps -} - -type basicIncomeSettlementDeps struct { - settlementDeps - cnrClient *containerClient.Client -} - -type basicSettlementConstructor struct { - dep *basicIncomeSettlementDeps -} - -type auditSettlementCalculator audit.Calculator - -type containerWrapper containerAPI.Container - -type nodeInfoWrapper struct { - ni netmapAPI.NodeInfo -} - -type sgWrapper storagegroup.StorageGroup - -func (s *sgWrapper) Size() uint64 { - return (*storagegroup.StorageGroup)(s).ValidationDataSize() -} - -func (n nodeInfoWrapper) PublicKey() []byte { - return n.ni.PublicKey() -} - -func (n nodeInfoWrapper) Price() *big.Int { - return big.NewInt(int64(n.ni.Price())) -} - -func (c containerWrapper) Owner() user.ID { - return (containerAPI.Container)(c).Owner() -} - -func (s settlementDeps) AuditResultsForEpoch(epoch uint64) ([]*auditAPI.Result, error) { - idList, err := s.auditClient.ListAuditResultIDByEpoch(epoch) - if err != nil { - return nil, fmt.Errorf("could not list audit results in sidechain: %w", err) - } - - res := make([]*auditAPI.Result, 0, len(idList)) - - for i := range idList { - r, err := s.auditClient.GetAuditResult(idList[i]) - if err != nil { - return nil, fmt.Errorf("could not get audit result: %w", err) - } - - res = append(res, r) - } - - return res, nil -} - -func (s settlementDeps) ContainerInfo(cid cid.ID) (common.ContainerInfo, error) { - cnr, err := s.cnrSrc.Get(cid) - if err != nil { - return nil, fmt.Errorf("could not get container from storage: %w", err) - } - - return (containerWrapper)(cnr.Value), nil -} - -func (s settlementDeps) buildContainer(e uint64, cid cid.ID) ([][]netmapAPI.NodeInfo, *netmapAPI.NetMap, error) { - var ( - nm *netmapAPI.NetMap - err error - ) - - if e > 0 { - nm, err = s.nmClient.GetNetMapByEpoch(e) - } else { - nm, err = netmap.GetLatestNetworkMap(s.nmClient) - } - - if err != nil { - return nil, nil, fmt.Errorf("could not get network map from storage: %w", err) - } - - cnr, err := s.cnrSrc.Get(cid) - if err != nil { - return nil, nil, fmt.Errorf("could not get container from sidechain: %w", err) - } - - binCnr := make([]byte, sha256.Size) - cid.Encode(binCnr) - - cn, err := nm.ContainerNodes( - cnr.Value.PlacementPolicy(), - binCnr, // may be replace pivot calculation to frostfs-api-go - ) - if err != nil { - return nil, nil, fmt.Errorf("could not calculate container nodes: %w", err) - } - - return cn, nm, nil -} - -func (s settlementDeps) ContainerNodes(e uint64, cid cid.ID) ([]common.NodeInfo, error) { - cn, _, err := s.buildContainer(e, cid) - if err != nil { - return nil, err - } - - var sz int - - for i := range cn { - sz += len(cn[i]) - } - - res := make([]common.NodeInfo, 0, sz) - - for i := range cn { - for j := range cn[i] { - res = append(res, nodeInfoWrapper{ - ni: cn[i][j], - }) - } - } - - return res, nil -} - -// SGInfo returns audit.SGInfo by object address. -// -// Returns an error of type apistatus.ObjectNotFound if storage group is missing. -func (s settlementDeps) SGInfo(addr oid.Address) (audit.SGInfo, error) { - cnr := addr.Container() - - cn, nm, err := s.buildContainer(0, cnr) - if err != nil { - return nil, err - } - - sg, err := s.clientCache.getSG(context.Background(), addr, nm, cn) - if err != nil { - return nil, err - } - - return (*sgWrapper)(sg), nil -} - -func (s settlementDeps) ResolveKey(ni common.NodeInfo) (*user.ID, error) { - pub, err := keys.NewPublicKeyFromBytes(ni.PublicKey(), elliptic.P256()) - if err != nil { - return nil, err - } - - var id user.ID - user.IDFromKey(&id, (ecdsa.PublicKey)(*pub)) - - return &id, nil -} - -func (s settlementDeps) Transfer(sender, recipient user.ID, amount *big.Int, details []byte) { - if s.settlementCtx == "" { - panic("unknown settlement deps context") - } - - log := s.log.With( - zap.Stringer("sender", sender), - zap.Stringer("recipient", recipient), - zap.Stringer("amount (GASe-12)", amount), - zap.String("details", hex.EncodeToString(details)), - ) - - if !amount.IsInt64() { - s.log.Error(logs.InnerringAmountCanNotBeRepresentedAsAnInt64) - - return - } - - params := balanceClient.TransferPrm{ - Amount: amount.Int64(), - From: sender, - To: recipient, - Details: details, - } - - err := s.balanceClient.TransferX(params) - if err != nil { - log.Error(fmt.Sprintf("%s: could not send transfer", s.settlementCtx), - zap.String("error", err.Error()), - ) - - return - } - - log.Debug(fmt.Sprintf("%s: transfer was successfully sent", s.settlementCtx)) -} - -func (b basicIncomeSettlementDeps) BasicRate() (uint64, error) { - return b.nmClient.BasicIncomeRate() -} - -func (b basicIncomeSettlementDeps) Estimations(epoch uint64) ([]*containerClient.Estimations, error) { - estimationIDs, err := b.cnrClient.ListLoadEstimationsByEpoch(epoch) - if err != nil { - return nil, err - } - - result := make([]*containerClient.Estimations, 0, len(estimationIDs)) - - for i := range estimationIDs { - estimation, err := b.cnrClient.GetUsedSpaceEstimations(estimationIDs[i]) - if err != nil { - b.log.Warn(logs.InnerringCantGetUsedSpaceEstimation, - zap.String("estimation_id", hex.EncodeToString(estimationIDs[i])), - zap.String("error", err.Error())) - - continue - } - - result = append(result, estimation) - } - - return result, nil -} - -func (b basicIncomeSettlementDeps) Balance(id user.ID) (*big.Int, error) { - return b.balanceClient.BalanceOf(id) -} - -func (s *auditSettlementCalculator) ProcessAuditSettlements(epoch uint64) { - (*audit.Calculator)(s).Calculate(&audit.CalculatePrm{ - Epoch: epoch, - }) -} - -func (b *basicSettlementConstructor) CreateContext(epoch uint64) (*basic.IncomeSettlementContext, error) { - return basic.NewIncomeSettlementContext(&basic.IncomeSettlementContextPrms{ - Log: b.dep.log, - Epoch: epoch, - Rate: b.dep, - Estimations: b.dep, - Balances: b.dep, - Container: b.dep, - Placement: b.dep, - Exchange: b.dep, - Accounts: b.dep, - }), nil -} diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index e3bf7886e..6a6ca0ade 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -6,8 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" - auditClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/audit" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state" "github.com/nspcc-dev/neo-go/pkg/util" @@ -146,18 +144,6 @@ func (s *Server) VoteForSidechainValidator(prm governance.VoteValidatorPrm) erro return s.voteForSidechainValidator(prm) } -// WriteReport composes the audit result structure from the audit report -// and sends it to Audit contract. -func (s *Server) WriteReport(r *audit.Report) error { - res := r.Result() - res.SetAuditorKey(s.pubKey) - - prm := auditClient.PutPrm{} - prm.SetResult(res) - - return s.auditClient.PutAuditResult(prm) -} - // ResetEpochTimer resets the block timer that produces events to update epoch // counter in the netmap contract. It is used to synchronize this even production // based on the block with a notification of the last epoch. From 7f49f07255d4fb53c49ca4fa55f8d9ba87731e7b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 25 Apr 2023 09:32:14 +0300 Subject: [PATCH 0174/1943] [#248] services: Remove audit service Signed-off-by: Evgenii Stratonikov --- pkg/services/audit/auditor/context.go | 298 ---------------------- pkg/services/audit/auditor/exec.go | 37 --- pkg/services/audit/auditor/pdp.go | 240 ----------------- pkg/services/audit/auditor/pop.go | 187 -------------- pkg/services/audit/auditor/por.go | 156 ----------- pkg/services/audit/auditor/util.go | 18 -- pkg/services/audit/report.go | 89 ------- pkg/services/audit/task.go | 120 --------- pkg/services/audit/taskmanager/listen.go | 85 ------ pkg/services/audit/taskmanager/manager.go | 107 -------- pkg/services/audit/taskmanager/push.go | 10 - pkg/services/audit/taskmanager/reset.go | 11 - 12 files changed, 1358 deletions(-) delete mode 100644 pkg/services/audit/auditor/context.go delete mode 100644 pkg/services/audit/auditor/exec.go delete mode 100644 pkg/services/audit/auditor/pdp.go delete mode 100644 pkg/services/audit/auditor/pop.go delete mode 100644 pkg/services/audit/auditor/por.go delete mode 100644 pkg/services/audit/auditor/util.go delete mode 100644 pkg/services/audit/report.go delete mode 100644 pkg/services/audit/task.go delete mode 100644 pkg/services/audit/taskmanager/listen.go delete mode 100644 pkg/services/audit/taskmanager/manager.go delete mode 100644 pkg/services/audit/taskmanager/push.go delete mode 100644 pkg/services/audit/taskmanager/reset.go diff --git a/pkg/services/audit/auditor/context.go b/pkg/services/audit/auditor/context.go deleted file mode 100644 index 194c5188a..000000000 --- a/pkg/services/audit/auditor/context.go +++ /dev/null @@ -1,298 +0,0 @@ -package auditor - -import ( - "context" - "sync" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.uber.org/atomic" - "go.uber.org/zap" -) - -// Context represents container data audit execution context. -type Context struct { - ContextPrm - - task *audit.Task - - report *audit.Report - - sgMembersMtx sync.RWMutex - sgMembersCache map[oid.ID][]oid.ID - - placementMtx sync.Mutex - placementCache map[string][][]netmap.NodeInfo - - porRequests, porRetries atomic.Uint32 - - pairs []gamePair - - pairedMtx sync.Mutex - pairedNodes map[uint64]*pairMemberInfo - - counters struct { - hit, miss, fail uint32 - } - - cnrNodesNum int - - headMtx sync.RWMutex - headResponses map[string]shortHeader -} - -type pairMemberInfo struct { - failedPDP, passedPDP bool // at least one - - node netmap.NodeInfo -} - -type gamePair struct { - n1, n2 netmap.NodeInfo - - id oid.ID - - rn1, rn2 []*object.Range - - hh1, hh2 [][]byte -} - -type shortHeader struct { - tzhash []byte - - objectSize uint64 -} - -// ContextPrm groups components required to conduct data audit checks. -type ContextPrm struct { - maxPDPSleep uint64 - - log *logger.Logger - - cnrCom ContainerCommunicator - - pdpWorkerPool, porWorkerPool util.WorkerPool -} - -type commonCommunicatorPrm struct { - Node netmap.NodeInfo - - OID oid.ID - CID cid.ID -} - -// GetHeaderPrm groups parameter of GetHeader operation. -type GetHeaderPrm struct { - commonCommunicatorPrm - - NodeIsRelay bool -} - -// GetRangeHashPrm groups parameter of GetRangeHash operation. -type GetRangeHashPrm struct { - commonCommunicatorPrm - - Range *object.Range -} - -// ContainerCommunicator is an interface of -// component of communication with container nodes. -type ContainerCommunicator interface { - // GetHeader must return object header from the container node. - GetHeader(context.Context, GetHeaderPrm) (*object.Object, error) - - // GetRangeHash must return homomorphic Tillich-Zemor hash of payload range of the - // object stored in container node. - GetRangeHash(context.Context, GetRangeHashPrm) ([]byte, error) -} - -// NewContext creates, initializes and returns Context. -func NewContext(prm ContextPrm) *Context { - return &Context{ - ContextPrm: prm, - } -} - -// SetLogger sets logging component. -func (p *ContextPrm) SetLogger(l *logger.Logger) { - if p != nil { - p.log = l - } -} - -// SetContainerCommunicator sets component of communication with container nodes. -func (p *ContextPrm) SetContainerCommunicator(cnrCom ContainerCommunicator) { - if p != nil { - p.cnrCom = cnrCom - } -} - -// SetMaxPDPSleep sets maximum sleep interval between range hash requests. -// as part of PDP check. -func (p *ContextPrm) SetMaxPDPSleep(dur time.Duration) { - if p != nil { - p.maxPDPSleep = uint64(dur) - } -} - -// WithTask sets container audit parameters. -func (c *Context) WithTask(t *audit.Task) *Context { - if c != nil { - c.task = t - } - - return c -} - -// WithPDPWorkerPool sets worker pool for PDP pairs processing. -func (c *Context) WithPDPWorkerPool(pool util.WorkerPool) *Context { - if c != nil { - c.pdpWorkerPool = pool - } - - return c -} - -// WithPoRWorkerPool sets worker pool for PoR SG processing. -func (c *Context) WithPoRWorkerPool(pool util.WorkerPool) *Context { - if c != nil { - c.porWorkerPool = pool - } - - return c -} - -func (c *Context) containerID() cid.ID { - return c.task.ContainerID() -} - -func (c *Context) init() { - c.report = audit.NewReport(c.containerID()) - - c.sgMembersCache = make(map[oid.ID][]oid.ID) - - c.placementCache = make(map[string][][]netmap.NodeInfo) - - cnrVectors := c.task.ContainerNodes() - for i := range cnrVectors { - c.cnrNodesNum += len(cnrVectors[i]) - } - - c.pairedNodes = make(map[uint64]*pairMemberInfo) - - c.headResponses = make(map[string]shortHeader) - - c.log = &logger.Logger{Logger: c.log.With( - zap.Stringer("container ID", c.task.ContainerID()), - )} -} - -func (c *Context) expired(ctx context.Context) bool { - select { - case <-ctx.Done(): - c.log.Debug(logs.AuditorAuditContextIsDone, - zap.String("error", ctx.Err().Error()), - ) - - return true - default: - return false - } -} - -func (c *Context) complete() { - c.report.Complete() -} - -func (c *Context) writeReport() { - c.log.Debug(logs.AuditorWritingAuditReport) - - if err := c.task.Reporter().WriteReport(c.report); err != nil { - c.log.Error(logs.AuditorCouldNotWriteAuditReport) - } -} - -func (c *Context) buildPlacement(id oid.ID) ([][]netmap.NodeInfo, error) { - c.placementMtx.Lock() - defer c.placementMtx.Unlock() - - strID := id.EncodeToString() - - if nn, ok := c.placementCache[strID]; ok { - return nn, nil - } - - nn, err := placement.BuildObjectPlacement( - c.task.NetworkMap(), - c.task.ContainerNodes(), - &id, - ) - if err != nil { - return nil, err - } - - c.placementCache[strID] = nn - - return nn, nil -} - -func (c *Context) objectSize(id oid.ID) uint64 { - c.headMtx.RLock() - defer c.headMtx.RUnlock() - - strID := id.EncodeToString() - - if hdr, ok := c.headResponses[strID]; ok { - return hdr.objectSize - } - - return 0 -} - -func (c *Context) objectHomoHash(id oid.ID) []byte { - c.headMtx.RLock() - defer c.headMtx.RUnlock() - - strID := id.EncodeToString() - - if hdr, ok := c.headResponses[strID]; ok { - return hdr.tzhash - } - - return nil -} - -func (c *Context) updateHeadResponses(hdr *object.Object) { - id, ok := hdr.ID() - if !ok { - return - } - - strID := id.EncodeToString() - cs, _ := hdr.PayloadHomomorphicHash() - - c.headMtx.Lock() - defer c.headMtx.Unlock() - - if _, ok := c.headResponses[strID]; !ok { - c.headResponses[strID] = shortHeader{ - tzhash: cs.Value(), - objectSize: hdr.PayloadSize(), - } - } -} - -func (c *Context) updateSGInfo(id oid.ID, members []oid.ID) { - c.sgMembersMtx.Lock() - defer c.sgMembersMtx.Unlock() - - c.sgMembersCache[id] = members -} diff --git a/pkg/services/audit/auditor/exec.go b/pkg/services/audit/auditor/exec.go deleted file mode 100644 index e603818b8..000000000 --- a/pkg/services/audit/auditor/exec.go +++ /dev/null @@ -1,37 +0,0 @@ -package auditor - -import ( - "context" - "fmt" -) - -// Execute audits container data. -func (c *Context) Execute(ctx context.Context, onCompleted func()) { - defer onCompleted() - c.init() - - checks := []struct { - name string - exec func(context.Context) - }{ - {name: "PoR", exec: c.executePoR}, - {name: "PoP", exec: c.executePoP}, - {name: "PDP", exec: c.executePDP}, - } - - for i := range checks { - c.log.Debug(fmt.Sprintf("executing %s check...", checks[i].name)) - - if c.expired(ctx) { - break - } - - checks[i].exec(ctx) - - if i == len(checks)-1 { - c.complete() - } - } - - c.writeReport() -} diff --git a/pkg/services/audit/auditor/pdp.go b/pkg/services/audit/auditor/pdp.go deleted file mode 100644 index d5ad0fea4..000000000 --- a/pkg/services/audit/auditor/pdp.go +++ /dev/null @@ -1,240 +0,0 @@ -package auditor - -import ( - "bytes" - "context" - "sync" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/tzhash/tz" - "go.uber.org/zap" -) - -func (c *Context) executePDP(ctx context.Context) { - c.processPairs(ctx) - c.writePairsResult() -} - -func (c *Context) processPairs(ctx context.Context) { - wg := new(sync.WaitGroup) - - for i := range c.pairs { - p := &c.pairs[i] - wg.Add(1) - - if err := c.pdpWorkerPool.Submit(func() { - c.processPair(ctx, p) - wg.Done() - }); err != nil { - wg.Done() - } - } - - wg.Wait() - c.pdpWorkerPool.Release() -} - -func (c *Context) processPair(ctx context.Context, p *gamePair) { - c.distributeRanges(p) - c.collectHashes(ctx, p) - c.analyzeHashes(p) -} - -func (c *Context) distributeRanges(p *gamePair) { - p.rn1 = make([]*object.Range, hashRangeNumber-1) - p.rn2 = make([]*object.Range, hashRangeNumber-1) - - for i := 0; i < hashRangeNumber-1; i++ { - p.rn1[i] = object.NewRange() - p.rn2[i] = object.NewRange() - } - - notches := c.splitPayload(p.id) - - { // node 1 - // [0:n2] - p.rn1[0].SetLength(notches[1]) - - // [n2:n3] - p.rn1[1].SetOffset(notches[1]) - p.rn1[1].SetLength(notches[2] - notches[1]) - - // [n3:full] - p.rn1[2].SetOffset(notches[2]) - p.rn1[2].SetLength(notches[3] - notches[2]) - } - - { // node 2 - // [0:n1] - p.rn2[0].SetLength(notches[0]) - - // [n1:n2] - p.rn2[1].SetOffset(notches[0]) - p.rn2[1].SetLength(notches[1] - notches[0]) - - // [n2:full] - p.rn2[2].SetOffset(notches[1]) - p.rn2[2].SetLength(notches[3] - notches[1]) - } -} - -func (c *Context) splitPayload(id oid.ID) []uint64 { - var ( - prev uint64 - size = c.objectSize(id) - notches = make([]uint64, 0, hashRangeNumber) - ) - - for i := uint64(0); i < hashRangeNumber; i++ { - if i < hashRangeNumber-1 { - max := size - prev - (hashRangeNumber - i) - if max == 0 { - prev++ - } else { - prev += rand.Uint64()%max + 1 - } - } else { - prev = size - } - - notches = append(notches, prev) - } - - return notches -} - -func (c *Context) collectHashes(ctx context.Context, p *gamePair) { - fn := func(n netmap.NodeInfo, rngs []*object.Range) [][]byte { - // Here we randomize the order a bit: the hypothesis is that this - // makes it harder for an unscrupulous node to come up with a - // reliable cheating strategy. - order := make([]int, len(rngs)) - for i := range order { - order[i] = i - } - rand.Shuffle(len(order), func(i, j int) { order[i], order[j] = order[j], order[i] }) - - var getRangeHashPrm GetRangeHashPrm - getRangeHashPrm.CID = c.task.ContainerID() - getRangeHashPrm.OID = p.id - getRangeHashPrm.Node = n - - res := make([][]byte, len(rngs)) - for _, i := range order { - var sleepDur time.Duration - if c.maxPDPSleep > 0 { - sleepDur = time.Duration(rand.Uint64() % c.maxPDPSleep) - } - - c.log.Debug(logs.AuditorSleepBeforeGetRangeHash, - zap.Stringer("interval", sleepDur), - ) - - time.Sleep(sleepDur) - - getRangeHashPrm.Range = rngs[i] - - h, err := c.cnrCom.GetRangeHash(ctx, getRangeHashPrm) - if err != nil { - c.log.Debug(logs.AuditorCouldNotGetPayloadRangeHash, - zap.Stringer("id", p.id), - zap.String("node", netmap.StringifyPublicKey(n)), - zap.String("error", err.Error()), - ) - return res - } - res[i] = h - } - return res - } - - p.hh1 = fn(p.n1, p.rn1) - p.hh2 = fn(p.n2, p.rn2) -} - -func (c *Context) analyzeHashes(p *gamePair) { - if len(p.hh1) != hashRangeNumber-1 || len(p.hh2) != hashRangeNumber-1 { - c.failNodesPDP(p.n1, p.n2) - return - } - - h1, err := tz.Concat([][]byte{p.hh2[0], p.hh2[1]}) - if err != nil || !bytes.Equal(p.hh1[0], h1) { - c.failNodesPDP(p.n1, p.n2) - return - } - - h2, err := tz.Concat([][]byte{p.hh1[1], p.hh1[2]}) - if err != nil || !bytes.Equal(p.hh2[2], h2) { - c.failNodesPDP(p.n1, p.n2) - return - } - - fh, err := tz.Concat([][]byte{h1, h2}) - if err != nil || !bytes.Equal(fh, c.objectHomoHash(p.id)) { - c.failNodesPDP(p.n1, p.n2) - return - } - - c.passNodesPDP(p.n1, p.n2) -} - -func (c *Context) failNodesPDP(ns ...netmap.NodeInfo) { - c.pairedMtx.Lock() - - for i := range ns { - c.pairedNodes[ns[i].Hash()].failedPDP = true - } - - c.pairedMtx.Unlock() -} - -func (c *Context) passNodesPDP(ns ...netmap.NodeInfo) { - c.pairedMtx.Lock() - - for i := range ns { - c.pairedNodes[ns[i].Hash()].passedPDP = true - } - - c.pairedMtx.Unlock() -} - -func (c *Context) writePairsResult() { - var failCount, okCount int - - c.iteratePairedNodes( - func(netmap.NodeInfo) { failCount++ }, - func(netmap.NodeInfo) { okCount++ }, - ) - - failedNodes := make([][]byte, 0, failCount) - passedNodes := make([][]byte, 0, okCount) - - c.iteratePairedNodes( - func(n netmap.NodeInfo) { - failedNodes = append(failedNodes, n.PublicKey()) - }, - func(n netmap.NodeInfo) { - passedNodes = append(passedNodes, n.PublicKey()) - }, - ) - - c.report.SetPDPResults(passedNodes, failedNodes) -} - -func (c *Context) iteratePairedNodes(onFail, onPass func(netmap.NodeInfo)) { - for _, pairedNode := range c.pairedNodes { - if pairedNode.failedPDP { - onFail(pairedNode.node) - } - - if pairedNode.passedPDP { - onPass(pairedNode.node) - } - } -} diff --git a/pkg/services/audit/auditor/pop.go b/pkg/services/audit/auditor/pop.go deleted file mode 100644 index b64004bbc..000000000 --- a/pkg/services/audit/auditor/pop.go +++ /dev/null @@ -1,187 +0,0 @@ -package auditor - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/tzhash/tz" - "go.uber.org/zap" -) - -const ( - hashRangeNumber = 4 - minGamePayloadSize = hashRangeNumber * tz.Size -) - -func (c *Context) executePoP(ctx context.Context) { - c.buildCoverage(ctx) - - c.report.SetPlacementCounters( - c.counters.hit, - c.counters.miss, - c.counters.fail, - ) -} - -func (c *Context) buildCoverage(ctx context.Context) { - policy := c.task.ContainerStructure().PlacementPolicy() - - // select random member from another storage group - // and process all placement vectors - c.iterateSGMembersPlacementRand(func(id oid.ID, ind int, nodes []netmap.NodeInfo) bool { - c.processObjectPlacement(ctx, id, nodes, policy.ReplicaNumberByIndex(ind)) - return c.containerCovered() - }) -} - -func (c *Context) containerCovered() bool { - // number of container nodes can be calculated once - return c.cnrNodesNum <= len(c.pairedNodes) -} - -func (c *Context) processObjectPlacement(ctx context.Context, id oid.ID, nodes []netmap.NodeInfo, replicas uint32) { - var ( - ok uint32 - optimal bool - - unpairedCandidate1, unpairedCandidate2 = -1, -1 - - pairedCandidate = -1 - ) - - var getHeaderPrm GetHeaderPrm - getHeaderPrm.OID = id - getHeaderPrm.CID = c.task.ContainerID() - getHeaderPrm.NodeIsRelay = false - - for i := 0; ok < replicas && i < len(nodes); i++ { - getHeaderPrm.Node = nodes[i] - - // try to get object header from node - hdr, err := c.cnrCom.GetHeader(ctx, getHeaderPrm) - if err != nil { - c.log.Debug(logs.AuditorCouldNotGetObjectHeaderFromCandidate, - zap.Stringer("id", id), - zap.String("error", err.Error()), - ) - - continue - } - - c.updateHeadResponses(hdr) - - // increment success counter - ok++ - - // update optimal flag - optimal = ok == replicas && uint32(i) < replicas - - // exclude small objects from coverage - if c.objectSize(id) < minGamePayloadSize { - continue - } - - // update potential candidates to be paired - if _, ok := c.pairedNodes[nodes[i].Hash()]; !ok { - if unpairedCandidate1 < 0 { - unpairedCandidate1 = i - } else if unpairedCandidate2 < 0 { - unpairedCandidate2 = i - } - } else if pairedCandidate < 0 { - pairedCandidate = i - } - } - - if optimal { - c.counters.hit++ - } else if ok == replicas { - c.counters.miss++ - } else { - c.counters.fail++ - } - - if unpairedCandidate1 >= 0 { - if unpairedCandidate2 >= 0 { - c.composePair(id, nodes[unpairedCandidate1], nodes[unpairedCandidate2]) - } else if pairedCandidate >= 0 { - c.composePair(id, nodes[unpairedCandidate1], nodes[pairedCandidate]) - } - } -} - -func (c *Context) composePair(id oid.ID, n1, n2 netmap.NodeInfo) { - c.pairs = append(c.pairs, gamePair{ - n1: n1, - n2: n2, - id: id, - }) - - c.pairedNodes[n1.Hash()] = &pairMemberInfo{ - node: n1, - } - c.pairedNodes[n2.Hash()] = &pairMemberInfo{ - node: n2, - } -} - -func (c *Context) iterateSGMembersPlacementRand(f func(oid.ID, int, []netmap.NodeInfo) bool) { - // iterate over storage groups members for all storage groups (one by one) - // with randomly shuffled members - c.iterateSGMembersRand(func(id oid.ID) bool { - // build placement vector for the current object - nn, err := c.buildPlacement(id) - if err != nil { - c.log.Debug(logs.AuditorCouldNotBuildPlacementForObject, - zap.Stringer("id", id), - zap.String("error", err.Error()), - ) - - return false - } - - for i, nodes := range nn { - if f(id, i, nodes) { - return true - } - } - - return false - }) -} - -func (c *Context) iterateSGMembersRand(f func(oid.ID) bool) { - c.iterateSGInfo(func(members []oid.ID) bool { - ln := len(members) - - processed := make(map[uint64]struct{}, ln-1) - - for len(processed) < ln { - ind := nextRandUint64(uint64(ln), processed) - processed[ind] = struct{}{} - - if f(members[ind]) { - return true - } - } - - return false - }) -} - -func (c *Context) iterateSGInfo(f func([]oid.ID) bool) { - c.sgMembersMtx.RLock() - defer c.sgMembersMtx.RUnlock() - - // we can add randomization like for SG members, - // but list of storage groups is already expected - // to be shuffled since it is a Search response - // with unpredictable order - for i := range c.sgMembersCache { - if f(c.sgMembersCache[i]) { - return - } - } -} diff --git a/pkg/services/audit/auditor/por.go b/pkg/services/audit/auditor/por.go deleted file mode 100644 index d579b3a7c..000000000 --- a/pkg/services/audit/auditor/por.go +++ /dev/null @@ -1,156 +0,0 @@ -package auditor - -import ( - "bytes" - "context" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" - containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - storagegroupSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/storagegroup" - "git.frostfs.info/TrueCloudLab/tzhash/tz" - "go.uber.org/zap" -) - -func (c *Context) executePoR(ctx context.Context) { - wg := new(sync.WaitGroup) - sgs := c.task.StorageGroupList() - - for _, sg := range sgs { - wg.Add(1) - - if err := c.porWorkerPool.Submit(func() { - c.checkStorageGroupPoR(ctx, sg.ID(), sg.StorageGroup()) - wg.Done() - }); err != nil { - wg.Done() - } - } - - wg.Wait() - c.porWorkerPool.Release() - - c.report.SetPoRCounters(c.porRequests.Load(), c.porRetries.Load()) -} - -func (c *Context) checkStorageGroupPoR(ctx context.Context, sgID oid.ID, sg storagegroupSDK.StorageGroup) { - members := sg.Members() - c.updateSGInfo(sgID, members) - - var ( - tzHash []byte - totalSize uint64 - - accRequests, accRetries uint32 - ) - - var getHeaderPrm GetHeaderPrm - getHeaderPrm.CID = c.task.ContainerID() - getHeaderPrm.NodeIsRelay = true - - homomorphicHashingEnabled := !containerSDK.IsHomomorphicHashingDisabled(c.task.ContainerStructure()) - - for i := range members { - flat, ok := c.getShuffledNodes(members[i], sgID) - if !ok { - continue - } - - getHeaderPrm.OID = members[i] - - for j := range flat { - accRequests++ - if j > 0 { // in best case audit get object header on first iteration - accRetries++ - } - - getHeaderPrm.Node = flat[j] - - hdr, err := c.cnrCom.GetHeader(ctx, getHeaderPrm) - if err != nil { - c.log.Debug(logs.AuditorCantHeadObject, - zap.String("remote_node", netmap.StringifyPublicKey(flat[j])), - zap.Stringer("oid", members[i]), - ) - - continue - } - - // update cache for PoR and PDP audit checks - c.updateHeadResponses(hdr) - - if homomorphicHashingEnabled { - cs, _ := hdr.PayloadHomomorphicHash() - if len(tzHash) == 0 { - tzHash = cs.Value() - } else { - tzHash, err = tz.Concat([][]byte{ - tzHash, - cs.Value(), - }) - if err != nil { - c.log.Debug(logs.AuditorCantConcatenateTzHash, - zap.String("oid", members[i].String()), - zap.String("error", err.Error())) - - break - } - } - } - - totalSize += hdr.PayloadSize() - - break - } - } - - c.porRequests.Add(accRequests) - c.porRetries.Add(accRetries) - - sizeCheck := sg.ValidationDataSize() == totalSize - cs, _ := sg.ValidationDataHash() - tzCheck := !homomorphicHashingEnabled || bytes.Equal(tzHash, cs.Value()) - - c.writeCheckReport(sizeCheck, tzCheck, sgID, sg, totalSize) -} - -func (c *Context) writeCheckReport(sizeCheck, tzCheck bool, sgID oid.ID, sg storagegroupSDK.StorageGroup, totalSize uint64) { - if sizeCheck && tzCheck { - c.report.PassedPoR(sgID) - } else { - if !sizeCheck { - c.log.Debug(logs.AuditorStorageGroupSizeCheckFailed, - zap.Uint64("expected", sg.ValidationDataSize()), - zap.Uint64("got", totalSize)) - } - - if !tzCheck { - c.log.Debug(logs.AuditorStorageGroupTzHashCheckFailed) - } - - c.report.FailedPoR(sgID) - } -} - -func (c *Context) getShuffledNodes(member oid.ID, sgID oid.ID) ([]netmap.NodeInfo, bool) { - objectPlacement, err := c.buildPlacement(member) - if err != nil { - c.log.Info(logs.AuditorCantBuildPlacementForStorageGroupMember, - zap.Stringer("sg", sgID), - zap.String("member_id", member.String()), - ) - - return nil, false - } - - flat := placement.FlattenNodes(objectPlacement) - - rand.Shuffle(len(flat), func(i, j int) { - flat[i], flat[j] = flat[j], flat[i] - }) - return flat, true -} diff --git a/pkg/services/audit/auditor/util.go b/pkg/services/audit/auditor/util.go deleted file mode 100644 index 5f8685534..000000000 --- a/pkg/services/audit/auditor/util.go +++ /dev/null @@ -1,18 +0,0 @@ -package auditor - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" -) - -// nextRandUint64 returns random uint64 number [0; n) outside exclude map. -// Panics if len(exclude) >= n. -func nextRandUint64(n uint64, exclude map[uint64]struct{}) uint64 { - ln := uint64(len(exclude)) - ind := rand.Uint64() % (n - ln) - - for i := ind; ; i++ { - if _, ok := exclude[i]; !ok { - return i - } - } -} diff --git a/pkg/services/audit/report.go b/pkg/services/audit/report.go deleted file mode 100644 index f16f97384..000000000 --- a/pkg/services/audit/report.go +++ /dev/null @@ -1,89 +0,0 @@ -package audit - -import ( - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/audit" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -// Report tracks the progress of auditing container data. -type Report struct { - mu sync.RWMutex - res audit.Result -} - -// Reporter is an interface of the entity that records -// the data audit report. -type Reporter interface { - WriteReport(r *Report) error -} - -// NewReport creates and returns blank Report instance. -func NewReport(cnr cid.ID) *Report { - var rep Report - rep.res.ForContainer(cnr) - - return &rep -} - -// Result forms the structure of the data audit result. -func (r *Report) Result() *audit.Result { - r.mu.RLock() - defer r.mu.RUnlock() - - return &r.res -} - -// Complete completes audit report. -func (r *Report) Complete() { - r.mu.Lock() - defer r.mu.Unlock() - - r.res.Complete() -} - -// PassedPoR updates list of passed storage groups. -func (r *Report) PassedPoR(sg oid.ID) { - r.mu.Lock() - defer r.mu.Unlock() - - r.res.SubmitPassedStorageGroup(sg) -} - -// FailedPoR updates list of failed storage groups. -func (r *Report) FailedPoR(sg oid.ID) { - r.mu.Lock() - defer r.mu.Unlock() - - r.res.SubmitFailedStorageGroup(sg) -} - -// SetPlacementCounters sets counters of compliance with placement. -func (r *Report) SetPlacementCounters(hit, miss, fail uint32) { - r.mu.Lock() - defer r.mu.Unlock() - - r.res.SetHits(hit) - r.res.SetMisses(miss) - r.res.SetFailures(fail) -} - -// SetPDPResults sets lists of nodes according to their PDP results. -func (r *Report) SetPDPResults(passed, failed [][]byte) { - r.mu.Lock() - defer r.mu.Unlock() - - r.res.SubmitPassedStorageNodes(passed) - r.res.SubmitFailedStorageNodes(failed) -} - -// SetPoRCounters sets amounts of head requests and retries at PoR audit stage. -func (r *Report) SetPoRCounters(requests, retries uint32) { - r.mu.Lock() - defer r.mu.Unlock() - - r.res.SetRequestsPoR(requests) - r.res.SetRetriesPoR(retries) -} diff --git a/pkg/services/audit/task.go b/pkg/services/audit/task.go deleted file mode 100644 index 3de5ac2c6..000000000 --- a/pkg/services/audit/task.go +++ /dev/null @@ -1,120 +0,0 @@ -package audit - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/storagegroup" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" -) - -// Task groups groups the container audit parameters. -type Task struct { - cancelCh <-chan struct{} - - reporter Reporter - - idCnr cid.ID - - cnr container.Container - - nm *netmap.NetMap - - cnrNodes [][]netmap.NodeInfo - - sgList []storagegroup.StorageGroup -} - -// WithReporter sets audit report writer. -func (t *Task) WithReporter(r Reporter) *Task { - if t != nil { - t.reporter = r - } - - return t -} - -// Reporter returns audit report writer. -func (t *Task) Reporter() Reporter { - return t.reporter -} - -func (t *Task) WithCancelChannel(ch <-chan struct{}) *Task { - if ch != nil { - t.cancelCh = ch - } - return t -} - -func (t *Task) CancelChannel() <-chan struct{} { - return t.cancelCh -} - -// WithContainerID sets identifier of the container under audit. -func (t *Task) WithContainerID(cnr cid.ID) *Task { - if t != nil { - t.idCnr = cnr - } - - return t -} - -// ContainerID returns identifier of the container under audit. -func (t *Task) ContainerID() cid.ID { - return t.idCnr -} - -// WithContainerStructure sets structure of the container under audit. -func (t *Task) WithContainerStructure(cnr container.Container) *Task { - if t != nil { - t.cnr = cnr - } - - return t -} - -// ContainerStructure returns structure of the container under audit. -func (t *Task) ContainerStructure() container.Container { - return t.cnr -} - -// WithContainerNodes sets nodes in the container under audit. -func (t *Task) WithContainerNodes(cnrNodes [][]netmap.NodeInfo) *Task { - if t != nil { - t.cnrNodes = cnrNodes - } - - return t -} - -// NetworkMap returns network map of audit epoch. -func (t *Task) NetworkMap() *netmap.NetMap { - return t.nm -} - -// WithNetworkMap sets network map of audit epoch. -func (t *Task) WithNetworkMap(nm *netmap.NetMap) *Task { - if t != nil { - t.nm = nm - } - - return t -} - -// ContainerNodes returns nodes in the container under audit. -func (t *Task) ContainerNodes() [][]netmap.NodeInfo { - return t.cnrNodes -} - -// WithStorageGroupList sets a list of storage groups from container under audit. -func (t *Task) WithStorageGroupList(sgList []storagegroup.StorageGroup) *Task { - if t != nil { - t.sgList = sgList - } - - return t -} - -// StorageGroupList returns list of storage groups from container under audit. -func (t *Task) StorageGroupList() []storagegroup.StorageGroup { - return t.sgList -} diff --git a/pkg/services/audit/taskmanager/listen.go b/pkg/services/audit/taskmanager/listen.go deleted file mode 100644 index bfc37c2a1..000000000 --- a/pkg/services/audit/taskmanager/listen.go +++ /dev/null @@ -1,85 +0,0 @@ -package audittask - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit/auditor" - "go.uber.org/zap" -) - -// Listen starts the process of processing tasks from the queue. -// -// The listener is terminated by context. -func (m *Manager) Listen(ctx context.Context) { - m.log.Info(logs.TaskmanagerProcessRoutine, - zap.Uint32("queue_capacity", m.queueCap), - ) - - m.ch = make(chan *audit.Task, m.queueCap) - - for { - select { - case <-ctx.Done(): - m.log.Warn(logs.TaskmanagerStopListenerByContext, - zap.String("error", ctx.Err().Error()), - ) - m.workerPool.Release() - - return - case task, ok := <-m.ch: - if !ok { - m.log.Warn(logs.TaskmanagerQueueChannelIsClosed) - return - } - - tCtx, tCancel := context.WithCancel(ctx) // cancel task in case of listen cancel - go func() { - select { - case <-tCtx.Done(): // listen cancelled or task completed - return - case <-task.CancelChannel(): // new epoch - tCancel() - } - }() - - m.handleTask(tCtx, task, tCancel) - } - } -} - -func (m *Manager) handleTask(ctx context.Context, task *audit.Task, onCompleted func()) { - pdpPool, err := m.pdpPoolGenerator() - if err != nil { - m.log.Error(logs.TaskmanagerCouldNotGeneratePDPWorkerPool, - zap.String("error", err.Error()), - ) - onCompleted() - return - } - - porPool, err := m.pdpPoolGenerator() - if err != nil { - m.log.Error(logs.TaskmanagerCouldNotGeneratePoRWorkerPool, - zap.String("error", err.Error()), - ) - onCompleted() - return - } - - auditContext := m.generateContext(task). - WithPDPWorkerPool(pdpPool). - WithPoRWorkerPool(porPool) - - if err := m.workerPool.Submit(func() { auditContext.Execute(ctx, onCompleted) }); err != nil { - // may be we should report it - m.log.Warn(logs.TaskmanagerCouldNotSubmitAuditTask) - onCompleted() - } -} - -func (m *Manager) generateContext(task *audit.Task) *auditor.Context { - return auditor.NewContext(m.ctxPrm). - WithTask(task) -} diff --git a/pkg/services/audit/taskmanager/manager.go b/pkg/services/audit/taskmanager/manager.go deleted file mode 100644 index bf7698799..000000000 --- a/pkg/services/audit/taskmanager/manager.go +++ /dev/null @@ -1,107 +0,0 @@ -package audittask - -import ( - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit/auditor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// Manager represents an entity performing data audit tasks. -type Manager struct { - *cfg - - ch chan *audit.Task -} - -// Option is a Manager's constructor option. -type Option func(*cfg) - -type cfg struct { - queueCap uint32 - - log *logger.Logger - - ctxPrm auditor.ContextPrm - - workerPool util.WorkerPool - - pdpPoolGenerator, porPoolGenerator func() (util.WorkerPool, error) -} - -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - } -} - -// New creates, initializes and returns new Manager instance. -func New(opts ...Option) *Manager { - c := defaultCfg() - - for i := range opts { - opts[i](c) - } - - return &Manager{ - cfg: c, - } -} - -// WithLogger returns option to specify Manager's logger. -func WithLogger(l *logger.Logger) Option { - return func(c *cfg) { - c.log = &logger.Logger{Logger: l.With(zap.String("component", "Audit task manager"))} - c.ctxPrm.SetLogger(l) - } -} - -// WithWorkerPool returns option to set worker pool -// for task execution. -func WithWorkerPool(p util.WorkerPool) Option { - return func(c *cfg) { - c.workerPool = p - } -} - -// WithQueueCapacity returns option to set task queue capacity. -func WithQueueCapacity(capacity uint32) Option { - return func(c *cfg) { - c.queueCap = capacity - } -} - -// WithContainerCommunicator returns option to set component of communication -// with container nodes. -func WithContainerCommunicator(cnrCom auditor.ContainerCommunicator) Option { - return func(c *cfg) { - c.ctxPrm.SetContainerCommunicator(cnrCom) - } -} - -// WithMaxPDPSleepInterval returns option to set maximum sleep interval -// between range hash requests as part of PDP check. -func WithMaxPDPSleepInterval(dur time.Duration) Option { - return func(c *cfg) { - c.ctxPrm.SetMaxPDPSleep(dur) - } -} - -// WithPDPWorkerPoolGenerator returns option to set worker pool for PDP pairs processing. -// Callback caller owns returned pool and must release it appropriately. -func WithPDPWorkerPoolGenerator(f func() (util.WorkerPool, error)) Option { - return func(c *cfg) { - c.pdpPoolGenerator = f - } -} - -// WithPoRWorkerPoolGenerator returns option to set worker pool for PoR SG processing. -// Callback caller owns returned pool and must release it appropriately. -func WithPoRWorkerPoolGenerator(f func() (util.WorkerPool, error)) Option { - return func(c *cfg) { - c.porPoolGenerator = f - } -} diff --git a/pkg/services/audit/taskmanager/push.go b/pkg/services/audit/taskmanager/push.go deleted file mode 100644 index 805897dbf..000000000 --- a/pkg/services/audit/taskmanager/push.go +++ /dev/null @@ -1,10 +0,0 @@ -package audittask - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/audit" -) - -// PushTask adds a task to the queue for processing. -func (m *Manager) PushTask(t *audit.Task) { - m.ch <- t -} diff --git a/pkg/services/audit/taskmanager/reset.go b/pkg/services/audit/taskmanager/reset.go deleted file mode 100644 index 86f2538cf..000000000 --- a/pkg/services/audit/taskmanager/reset.go +++ /dev/null @@ -1,11 +0,0 @@ -package audittask - -// Reset pops all tasks from the queue. -// Returns amount of popped elements. -func (m *Manager) Reset() (popped int) { - for ; len(m.ch) > 0; popped++ { - <-m.ch - } - - return -} From f1ea8fec9302d8779f485a485de15601c5cd49c8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 25 Apr 2023 09:32:48 +0300 Subject: [PATCH 0175/1943] [#248] object_manager: Remove storage group Signed-off-by: Evgenii Stratonikov --- .../object_manager/storagegroup/collect.go | 67 ------------------- .../object_manager/storagegroup/search.go | 14 ---- 2 files changed, 81 deletions(-) delete mode 100644 pkg/services/object_manager/storagegroup/collect.go delete mode 100644 pkg/services/object_manager/storagegroup/search.go diff --git a/pkg/services/object_manager/storagegroup/collect.go b/pkg/services/object_manager/storagegroup/collect.go deleted file mode 100644 index d9578dea3..000000000 --- a/pkg/services/object_manager/storagegroup/collect.go +++ /dev/null @@ -1,67 +0,0 @@ -package storagegroup - -import ( - objutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/storagegroup" - "git.frostfs.info/TrueCloudLab/tzhash/tz" -) - -// CollectMembers creates new storage group structure and fills it -// with information about members collected via HeadReceiver. -// -// Resulting storage group consists of physically stored objects only. -func CollectMembers(r objutil.HeadReceiver, cnr cid.ID, members []oid.ID, calcHomoHash bool) (*storagegroup.StorageGroup, error) { - var ( - sumPhySize uint64 - phyMembers []oid.ID - phyHashes [][]byte - addr oid.Address - sg storagegroup.StorageGroup - ) - - addr.SetContainer(cnr) - - for i := range members { - addr.SetObject(members[i]) - - if err := objutil.IterateAllSplitLeaves(r, addr, func(leaf *object.Object) { - id, ok := leaf.ID() - if !ok { - return - } - - phyMembers = append(phyMembers, id) - sumPhySize += leaf.PayloadSize() - cs, _ := leaf.PayloadHomomorphicHash() - - if calcHomoHash { - phyHashes = append(phyHashes, cs.Value()) - } - }); err != nil { - return nil, err - } - } - - sg.SetMembers(phyMembers) - sg.SetValidationDataSize(sumPhySize) - - if calcHomoHash { - sumHash, err := tz.Concat(phyHashes) - if err != nil { - return nil, err - } - - var cs checksum.Checksum - tzHash := [64]byte{} - copy(tzHash[:], sumHash) - cs.SetTillichZemor(tzHash) - - sg.SetValidationDataHash(cs) - } - - return &sg, nil -} diff --git a/pkg/services/object_manager/storagegroup/search.go b/pkg/services/object_manager/storagegroup/search.go deleted file mode 100644 index 39019aa6d..000000000 --- a/pkg/services/object_manager/storagegroup/search.go +++ /dev/null @@ -1,14 +0,0 @@ -package storagegroup - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" -) - -// SearchQuery returns search query to filter -// objects with storage group content. -func SearchQuery() object.SearchFilters { - fs := object.SearchFilters{} - fs.AddTypeFilter(object.MatchStringEqual, object.TypeStorageGroup) - - return fs -} From 586f5986bc1401d46a41ef27657c21b47f036649 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 25 Apr 2023 09:35:58 +0300 Subject: [PATCH 0176/1943] [#248] core: Remove storage group Signed-off-by: Evgenii Stratonikov --- pkg/core/object/fmt.go | 40 --------------- pkg/core/object/fmt_test.go | 54 --------------------- pkg/core/storagegroup/storagegroup.go | 70 --------------------------- 3 files changed, 164 deletions(-) delete mode 100644 pkg/core/storagegroup/storagegroup.go diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index ef99f3058..946cfc462 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -14,7 +14,6 @@ import ( frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/storagegroup" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -65,8 +64,6 @@ var errNoExpirationEpoch = errors.New("missing expiration epoch attribute") var errTombstoneExpiration = errors.New("tombstone body and header contain different expiration values") -var errEmptySGMembers = errors.New("storage group with empty members list") - func defaultCfg() *cfg { return new(cfg) } @@ -180,7 +177,6 @@ func (v *FormatValidator) checkOwnerKey(id user.ID, key frostfsecdsa.PublicKey) // ContentMeta describes FrostFS meta information that brings object's payload if the object // is one of: // - object.TypeTombstone; -// - object.TypeStorageGroup; // - object.TypeLock. type ContentMeta struct { typ object.Type @@ -196,7 +192,6 @@ func (i ContentMeta) Type() object.Type { // Objects returns objects that the original object's payload affects: // - inhumed objects, if the original object is a Tombstone; // - locked objects, if the original object is a Lock; -// - members of a storage group, if the original object is a Storage group; // - nil, if the original object is a Regular object. func (i ContentMeta) Objects() []oid.ID { return i.objs @@ -213,10 +208,6 @@ func (v *FormatValidator) ValidateContent(o *object.Object) (ContentMeta, error) if err := v.fillAndValidateTombstoneMeta(o, &meta); err != nil { return ContentMeta{}, err } - case object.TypeStorageGroup: - if err := v.fillAndValidateStorageGroupMeta(o, &meta); err != nil { - return ContentMeta{}, err - } case object.TypeLock: if err := v.fillAndValidateLockMeta(o, &meta); err != nil { return ContentMeta{}, err @@ -266,37 +257,6 @@ func (v *FormatValidator) fillAndValidateLockMeta(o *object.Object, meta *Conten return nil } -func (v *FormatValidator) fillAndValidateStorageGroupMeta(o *object.Object, meta *ContentMeta) error { - if len(o.Payload()) == 0 { - return fmt.Errorf("(%T) empty payload in storage group", v) - } - - var sg storagegroup.StorageGroup - - if err := sg.Unmarshal(o.Payload()); err != nil { - return fmt.Errorf("(%T) could not unmarshal storage group content: %w", v, err) - } - - mm := sg.Members() - meta.objs = mm - - lenMM := len(mm) - if lenMM == 0 { - return errEmptySGMembers - } - - uniqueFilter := make(map[oid.ID]struct{}, lenMM) - - for i := 0; i < lenMM; i++ { - if _, alreadySeen := uniqueFilter[mm[i]]; alreadySeen { - return fmt.Errorf("storage group contains non-unique member: %s", mm[i]) - } - - uniqueFilter[mm[i]] = struct{}{} - } - return nil -} - func (v *FormatValidator) fillAndValidateTombstoneMeta(o *object.Object, meta *ContentMeta) error { if len(o.Payload()) == 0 { return fmt.Errorf("(%T) empty payload in tombstone", v) diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index be0602540..2cf5099ba 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -12,7 +12,6 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" sessiontest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/storagegroup" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" @@ -167,59 +166,6 @@ func TestFormatValidator_Validate(t *testing.T) { require.Equal(t, object.TypeTombstone, contentGot.Type()) }) - t.Run("storage group content", func(t *testing.T) { - obj := object.New() - obj.SetType(object.TypeStorageGroup) - - t.Run("empty payload", func(t *testing.T) { - _, err := v.ValidateContent(obj) - require.Error(t, err) - }) - - var content storagegroup.StorageGroup - content.SetExpirationEpoch(1) // some non-default value - - t.Run("empty members", func(t *testing.T) { - data, err := content.Marshal() - require.NoError(t, err) - - obj.SetPayload(data) - - _, err = v.ValidateContent(obj) - require.ErrorIs(t, err, errEmptySGMembers) - }) - - t.Run("non-unique members", func(t *testing.T) { - id := oidtest.ID() - - content.SetMembers([]oid.ID{id, id}) - - data, err := content.Marshal() - require.NoError(t, err) - - obj.SetPayload(data) - - _, err = v.ValidateContent(obj) - require.Error(t, err) - }) - - t.Run("correct SG", func(t *testing.T) { - ids := []oid.ID{oidtest.ID(), oidtest.ID()} - content.SetMembers(ids) - - data, err := content.Marshal() - require.NoError(t, err) - - obj.SetPayload(data) - - content, err := v.ValidateContent(obj) - require.NoError(t, err) - - require.EqualValues(t, ids, content.Objects()) - require.Equal(t, object.TypeStorageGroup, content.Type()) - }) - }) - t.Run("expiration", func(t *testing.T) { fn := func(val string) *object.Object { obj := blankValidObject(&ownerKey.PrivateKey) diff --git a/pkg/core/storagegroup/storagegroup.go b/pkg/core/storagegroup/storagegroup.go deleted file mode 100644 index b16e5c61a..000000000 --- a/pkg/core/storagegroup/storagegroup.go +++ /dev/null @@ -1,70 +0,0 @@ -package storagegroup - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/storagegroup" -) - -// SearchSGPrm groups the parameters which are formed by Processor to search the storage group objects. -type SearchSGPrm struct { - Container cid.ID - - NodeInfo client.NodeInfo -} - -// SearchSGDst groups the target values which Processor expects from SG searching to process. -type SearchSGDst struct { - Objects []oid.ID -} - -// GetSGPrm groups parameter of GetSG operation. -type GetSGPrm struct { - OID oid.ID - CID cid.ID - - NetMap netmap.NetMap - Container [][]netmap.NodeInfo -} - -// SGSource is a storage group information source interface. -type SGSource interface { - // ListSG must list storage group objects in the container. Formed list must be written to destination. - // - // Must return any error encountered which did not allow to form the list. - ListSG(context.Context, *SearchSGDst, SearchSGPrm) error - - // GetSG must return storage group object for the provided CID, OID, - // container and netmap state. - GetSG(context.Context, GetSGPrm) (*storagegroup.StorageGroup, error) -} - -// StorageGroup combines storage group object ID and its structure. -type StorageGroup struct { - id oid.ID - sg storagegroup.StorageGroup -} - -// ID returns object ID of the storage group. -func (s StorageGroup) ID() oid.ID { - return s.id -} - -// SetID sets an object ID of the storage group. -func (s *StorageGroup) SetID(id oid.ID) { - s.id = id -} - -// StorageGroup returns the storage group descriptor. -func (s StorageGroup) StorageGroup() storagegroup.StorageGroup { - return s.sg -} - -// SetStorageGroup sets a storage group descriptor. -func (s *StorageGroup) SetStorageGroup(sg storagegroup.StorageGroup) { - s.sg = sg -} From a9c4ba62c348ed3a47d91b92697012d4b442869b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 25 Apr 2023 09:48:24 +0300 Subject: [PATCH 0177/1943] [#248] metabase: Remove storage group bucket Backwards compatible change, so no version increase. Signed-off-by: Evgenii Stratonikov --- .../metabase/containers_test.go | 1 - pkg/local_object_storage/metabase/delete.go | 2 - .../metabase/exists_test.go | 12 ---- pkg/local_object_storage/metabase/get.go | 6 -- pkg/local_object_storage/metabase/get_test.go | 12 ---- .../metabase/iterators.go | 1 - .../metabase/iterators_test.go | 1 - pkg/local_object_storage/metabase/list.go | 2 - .../metabase/list_test.go | 9 +-- .../metabase/lock_test.go | 1 - pkg/local_object_storage/metabase/put.go | 2 - pkg/local_object_storage/metabase/select.go | 9 +-- .../metabase/select_test.go | 58 ------------------- pkg/local_object_storage/metabase/util.go | 14 ++--- 14 files changed, 8 insertions(+), 122 deletions(-) diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index ef2bba638..a0be2c743 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -100,7 +100,6 @@ func TestDB_ContainersCount(t *testing.T) { }{ {R, objectSDK.TypeRegular}, {T, objectSDK.TypeTombstone}, - {SG, objectSDK.TypeStorageGroup}, {L, objectSDK.TypeLock}, } diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 990d3997b..5340f5d08 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -345,8 +345,6 @@ func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error bucketName = primaryBucketName(cnr, bucketName) case objectSDK.TypeTombstone: bucketName = tombstoneBucketName(cnr, bucketName) - case objectSDK.TypeStorageGroup: - bucketName = storageGroupBucketName(cnr, bucketName) case objectSDK.TypeLock: bucketName = bucketNameLockers(cnr, bucketName) default: diff --git a/pkg/local_object_storage/metabase/exists_test.go b/pkg/local_object_storage/metabase/exists_test.go index e344e9ee8..034bc0005 100644 --- a/pkg/local_object_storage/metabase/exists_test.go +++ b/pkg/local_object_storage/metabase/exists_test.go @@ -57,18 +57,6 @@ func TestDB_Exists(t *testing.T) { require.True(t, exists) }) - t.Run("storage group object", func(t *testing.T) { - sg := testutil.GenerateObject() - sg.SetType(objectSDK.TypeStorageGroup) - - err := putBig(db, sg) - require.NoError(t, err) - - exists, err := metaExists(db, object.AddressOf(sg)) - require.NoError(t, err) - require.True(t, exists) - }) - t.Run("lock object", func(t *testing.T) { lock := testutil.GenerateObject() lock.SetType(objectSDK.TypeLock) diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index fff32d6ad..e76b9d4a7 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -106,12 +106,6 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b return obj, obj.Unmarshal(data) } - // if not found then check in storage group index - data = getFromBucket(tx, storageGroupBucketName(cnr, bucketName), key) - if len(data) != 0 { - return obj, obj.Unmarshal(data) - } - // if not found then check in locker index data = getFromBucket(tx, bucketNameLockers(cnr, bucketName), key) if len(data) != 0 { diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index 4d2a7682d..c19fea8d8 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -55,18 +55,6 @@ func TestDB_Get(t *testing.T) { require.Equal(t, raw.CutPayload(), newObj) }) - t.Run("put storage group object", func(t *testing.T) { - raw.SetType(objectSDK.TypeStorageGroup) - raw.SetID(oidtest.ID()) - - err := putBig(db, raw) - require.NoError(t, err) - - newObj, err := metaGet(db, object.AddressOf(raw), false) - require.NoError(t, err) - require.Equal(t, raw.CutPayload(), newObj) - }) - t.Run("put lock object", func(t *testing.T) { raw.SetType(objectSDK.TypeLock) raw.SetID(oidtest.ID()) diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index 3c5888e1c..4c9dc782c 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -186,7 +186,6 @@ func iteratePhyObjects(tx *bbolt.Tx, f func(cid.ID, oid.ID) error) error { switch postfix { case primaryPrefix, - storageGroupPrefix, lockersPrefix, tombstonePrefix: default: diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index e7d6ad04f..69bf2bee5 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -26,7 +26,6 @@ func TestDB_IterateExpired(t *testing.T) { for _, typ := range []object.Type{ object.TypeRegular, object.TypeTombstone, - object.TypeStorageGroup, object.TypeLock, } { mAlive[typ] = putWithExpiration(t, db, typ, epoch) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 93b7efb99..7fe1040bb 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -108,8 +108,6 @@ loop: switch prefix { case primaryPrefix: objType = object.TypeRegular - case storageGroupPrefix: - objType = object.TypeStorageGroup case lockersPrefix: objType = object.TypeLock case tombstonePrefix: diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index ab2d9d75d..07a0d80f8 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -70,7 +70,7 @@ func TestLisObjectsWithCursor(t *testing.T) { const ( containers = 5 - total = containers * 5 // regular + ts + sg + child + lock + total = containers * 4 // regular + ts + child + lock ) expected := make([]object.AddressWithType, 0, total) @@ -93,13 +93,6 @@ func TestLisObjectsWithCursor(t *testing.T) { require.NoError(t, err) expected = append(expected, object.AddressWithType{Address: object.AddressOf(obj), Type: objectSDK.TypeTombstone}) - // add one storage group - obj = testutil.GenerateObjectWithCID(containerID) - obj.SetType(objectSDK.TypeStorageGroup) - err = putBig(db, obj) - require.NoError(t, err) - expected = append(expected, object.AddressWithType{Address: object.AddressOf(obj), Type: objectSDK.TypeStorageGroup}) - // add one lock obj = testutil.GenerateObjectWithCID(containerID) obj.SetType(objectSDK.TypeLock) diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 1d6ea6ffb..7b62841dc 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -28,7 +28,6 @@ func TestDB_Lock(t *testing.T) { t.Run("(ir)regular", func(t *testing.T) { for _, typ := range [...]object.Type{ object.TypeTombstone, - object.TypeStorageGroup, object.TypeLock, object.TypeRegular, } { diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 2c78bda0f..bc6520a05 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -205,8 +205,6 @@ func putUniqueIndexes( bucketName = primaryBucketName(cnr, bucketName) case objectSDK.TypeTombstone: bucketName = tombstoneBucketName(cnr, bucketName) - case objectSDK.TypeStorageGroup: - bucketName = storageGroupBucketName(cnr, bucketName) case objectSDK.TypeLock: bucketName = bucketNameLockers(cnr, bucketName) default: diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index ecd83f863..a4b14f77c 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -151,7 +151,6 @@ func (db *DB) selectAll(tx *bbolt.Tx, cnr cid.ID, to map[string]int) { bucketName := make([]byte, bucketKeySize) selectAllFromBucket(tx, primaryBucketName(cnr, bucketName), to, 0) selectAllFromBucket(tx, tombstoneBucketName(cnr, bucketName), to, 0) - selectAllFromBucket(tx, storageGroupBucketName(cnr, bucketName), to, 0) selectAllFromBucket(tx, parentBucketName(cnr, bucketName), to, 0) selectAllFromBucket(tx, bucketNameLockers(cnr, bucketName), to, 0) } @@ -206,7 +205,6 @@ func (db *DB) selectFastFilter( case v2object.FilterPropertyPhy: selectAllFromBucket(tx, primaryBucketName(cnr, bucketName), to, fNum) selectAllFromBucket(tx, tombstoneBucketName(cnr, bucketName), to, fNum) - selectAllFromBucket(tx, storageGroupBucketName(cnr, bucketName), to, fNum) selectAllFromBucket(tx, bucketNameLockers(cnr, bucketName), to, fNum) default: // user attribute bucketName := attributeBucketName(cnr, f.Header(), bucketName) @@ -220,10 +218,9 @@ func (db *DB) selectFastFilter( } var mBucketNaming = map[string][]func(cid.ID, []byte) []byte{ - v2object.TypeRegular.String(): {primaryBucketName, parentBucketName}, - v2object.TypeTombstone.String(): {tombstoneBucketName}, - v2object.TypeStorageGroup.String(): {storageGroupBucketName}, - v2object.TypeLock.String(): {bucketNameLockers}, + v2object.TypeRegular.String(): {primaryBucketName, parentBucketName}, + v2object.TypeTombstone.String(): {tombstoneBucketName}, + v2object.TypeLock.String(): {bucketNameLockers}, } func allBucketNames(cnr cid.ID) (names [][]byte) { diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 386797529..dab4c028d 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -157,11 +157,6 @@ func TestDB_SelectRootPhyParent(t *testing.T) { err = putBig(db, ts) require.NoError(t, err) - sg := testutil.GenerateObjectWithCID(cnr) - sg.SetType(objectSDK.TypeStorageGroup) - err = putBig(db, sg) - require.NoError(t, err) - leftChild := testutil.GenerateObjectWithCID(cnr) leftChild.InitRelations() err = putBig(db, leftChild) @@ -210,7 +205,6 @@ func TestDB_SelectRootPhyParent(t *testing.T) { testSelect(t, db, cnr, fs, object.AddressOf(small), object.AddressOf(ts), - object.AddressOf(sg), object.AddressOf(leftChild), object.AddressOf(rightChild), object.AddressOf(link), @@ -237,7 +231,6 @@ func TestDB_SelectRootPhyParent(t *testing.T) { fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeRegular.String(), objectSDK.MatchStringNotEqual) testSelect(t, db, cnr, fs, object.AddressOf(ts), - object.AddressOf(sg), object.AddressOf(lock), ) @@ -259,29 +252,6 @@ func TestDB_SelectRootPhyParent(t *testing.T) { object.AddressOf(rightChild), object.AddressOf(link), object.AddressOf(parent), - object.AddressOf(sg), - object.AddressOf(lock), - ) - - fs = objectSDK.SearchFilters{} - fs.AddFilter(v2object.FilterHeaderObjectType, "", objectSDK.MatchNotPresent) - testSelect(t, db, cnr, fs) - }) - - t.Run("storage group objects", func(t *testing.T) { - fs := objectSDK.SearchFilters{} - fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeStorageGroup.String(), objectSDK.MatchStringEqual) - testSelect(t, db, cnr, fs, object.AddressOf(sg)) - - fs = objectSDK.SearchFilters{} - fs.AddFilter(v2object.FilterHeaderObjectType, v2object.TypeStorageGroup.String(), objectSDK.MatchStringNotEqual) - testSelect(t, db, cnr, fs, - object.AddressOf(small), - object.AddressOf(leftChild), - object.AddressOf(rightChild), - object.AddressOf(link), - object.AddressOf(parent), - object.AddressOf(ts), object.AddressOf(lock), ) @@ -313,7 +283,6 @@ func TestDB_SelectRootPhyParent(t *testing.T) { testSelect(t, db, cnr, fs, object.AddressOf(small), object.AddressOf(ts), - object.AddressOf(sg), object.AddressOf(leftChild), object.AddressOf(rightChild), object.AddressOf(link), @@ -550,11 +519,6 @@ func TestDB_SelectObjectID(t *testing.T) { err = putBig(db, ts) require.NoError(t, err) - sg := testutil.GenerateObjectWithCID(cnr) - sg.SetType(objectSDK.TypeStorageGroup) - err = putBig(db, sg) - require.NoError(t, err) - lock := testutil.GenerateObjectWithCID(cnr) lock.SetType(objectSDK.TypeLock) err = putBig(db, lock) @@ -576,7 +540,6 @@ func TestDB_SelectObjectID(t *testing.T) { testSelect(t, db, cnr, fs, object.AddressOf(regular), object.AddressOf(parent), - object.AddressOf(sg), object.AddressOf(ts), object.AddressOf(lock), ) @@ -593,7 +556,6 @@ func TestDB_SelectObjectID(t *testing.T) { fs.AddObjectIDFilter(objectSDK.MatchStringNotEqual, id) testSelect(t, db, cnr, fs, object.AddressOf(parent), - object.AddressOf(sg), object.AddressOf(ts), object.AddressOf(lock), ) @@ -611,24 +573,6 @@ func TestDB_SelectObjectID(t *testing.T) { testSelect(t, db, cnr, fs, object.AddressOf(regular), object.AddressOf(parent), - object.AddressOf(sg), - object.AddressOf(lock), - ) - }) - - t.Run("storage group objects", func(t *testing.T) { - id, _ := sg.ID() - - fs := objectSDK.SearchFilters{} - fs.AddObjectIDFilter(objectSDK.MatchStringEqual, id) - testSelect(t, db, cnr, fs, object.AddressOf(sg)) - - fs = objectSDK.SearchFilters{} - fs.AddObjectIDFilter(objectSDK.MatchStringNotEqual, id) - testSelect(t, db, cnr, fs, - object.AddressOf(regular), - object.AddressOf(parent), - object.AddressOf(ts), object.AddressOf(lock), ) }) @@ -644,7 +588,6 @@ func TestDB_SelectObjectID(t *testing.T) { fs.AddObjectIDFilter(objectSDK.MatchStringNotEqual, id) testSelect(t, db, cnr, fs, object.AddressOf(regular), - object.AddressOf(sg), object.AddressOf(ts), object.AddressOf(lock), ) @@ -662,7 +605,6 @@ func TestDB_SelectObjectID(t *testing.T) { testSelect(t, db, cnr, fs, object.AddressOf(regular), object.AddressOf(parent), - object.AddressOf(sg), object.AddressOf(ts), ) }) diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index b60c97fd7..c50fd051f 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -65,10 +65,10 @@ const ( // Key: object ID // Value: marshalled object lockersPrefix - // storageGroupPrefix is used for prefixing buckets containing objects of STORAGEGROUP type. + // _ is unused. Previous usage was for prefixing buckets containing objects of STORAGEGROUP type. // Key: object ID // Value: marshaled object - storageGroupPrefix + _ // tombstonePrefix is used for prefixing buckets containing objects of TOMBSTONE type. // Key: object ID // Value: marshaled object @@ -138,11 +138,6 @@ func tombstoneBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, tombstonePrefix, key) } -// storageGroupBucketName returns _SG. -func storageGroupBucketName(cnr cid.ID, key []byte) []byte { - return bucketName(cnr, storageGroupPrefix, key) -} - // smallBucketName returns _small. func smallBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, smallPrefix, key) @@ -231,15 +226,14 @@ func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) object panic("empty object list in firstIrregularObjectType") } - var keys [3][1 + cidSize]byte + var keys [2][1 + cidSize]byte irregularTypeBuckets := [...]struct { typ object.Type name []byte }{ {object.TypeTombstone, tombstoneBucketName(idCnr, keys[0][:])}, - {object.TypeStorageGroup, storageGroupBucketName(idCnr, keys[1][:])}, - {object.TypeLock, bucketNameLockers(idCnr, keys[2][:])}, + {object.TypeLock, bucketNameLockers(idCnr, keys[1][:])}, } for i := range objs { From b2bc1fccbc3811a35526083ced7e31ac2a13df17 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 25 Apr 2023 10:00:40 +0300 Subject: [PATCH 0178/1943] [#248] logs: Remove unused messages Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 80 ------------------------------------------- 1 file changed, 80 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 717e3529a..6c1c931fb 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -17,8 +17,6 @@ const ( ) const ( - InnerringAmountCanNotBeRepresentedAsAnInt64 = "amount can not be represented as an int64" // Error in ../node/pkg/innerring/settlement.go - InnerringCantGetUsedSpaceEstimation = "can't get used space estimation" // Warn in ../node/pkg/innerring/settlement.go InnerringNonalphabetModeDoNotStopContainerEstimations = "non-alphabet mode, do not stop container estimations" // Debug in ../node/pkg/innerring/blocktimer.go InnerringCantStopEpochEstimation = "can't stop epoch estimation" // Warn in ../node/pkg/innerring/blocktimer.go InnerringCantMakeNotaryDepositInMainChain = "can't make notary deposit in main chain" // Error in ../node/pkg/innerring/notary.go @@ -41,8 +39,6 @@ const ( InnerringCantUpdatePersistentState = "can't update persistent state" // Warn in ../node/pkg/innerring/innerring.go InnerringCloserError = "closer error" // Warn in ../node/pkg/innerring/innerring.go InnerringReadConfigFromBlockchain = "read config from blockchain" // Debug in ../node/pkg/innerring/innerring.go - InnerringCantSetupRemoteConnection = "can't setup remote connection" // Warn in ../node/pkg/innerring/rpc.go - InnerringCantGetStorageGroupObject = "can't get storage group object" // Warn in ../node/pkg/innerring/rpc.go NotificatorNotificatorStartProcessingObjectNotifications = "notificator: start processing object notifications" // Debug in ../node/pkg/services/notificator/service.go NotificatorNotificatorProcessingObjectNotification = "notificator: processing object notification" // Debug in ../node/pkg/services/notificator/service.go PolicerCouldNotGetContainer = "could not get container" // Error in ../node/pkg/services/policer/check.go @@ -150,24 +146,6 @@ const ( RouteCouldNotInitializeWriter = "could not initialize writer" // Debug in ../node/pkg/services/container/announcement/load/route/calls.go RouteCouldNotPutTheValue = "could not put the value" // Debug in ../node/pkg/services/container/announcement/load/route/calls.go RouteCouldNotCloseRemoteServerWriter = "could not close remote server writer" // Debug in ../node/pkg/services/container/announcement/load/route/calls.go - AuditorCouldNotGetObjectHeaderFromCandidate = "could not get object header from candidate" // Debug in ../node/pkg/services/audit/auditor/pop.go - AuditorCouldNotBuildPlacementForObject = "could not build placement for object" // Debug in ../node/pkg/services/audit/auditor/pop.go - AuditorCantHeadObject = "can't head object" // Debug in ../node/pkg/services/audit/auditor/por.go - AuditorCantConcatenateTzHash = "can't concatenate tz hash" // Debug in ../node/pkg/services/audit/auditor/por.go - AuditorStorageGroupSizeCheckFailed = "storage group size check failed" // Debug in ../node/pkg/services/audit/auditor/por.go - AuditorStorageGroupTzHashCheckFailed = "storage group tz hash check failed" // Debug in ../node/pkg/services/audit/auditor/por.go - AuditorCantBuildPlacementForStorageGroupMember = "can't build placement for storage group member" // Info in ../node/pkg/services/audit/auditor/por.go - AuditorAuditContextIsDone = "audit context is done" // Debug in ../node/pkg/services/audit/auditor/context.go - AuditorWritingAuditReport = "writing audit report..." // Debug in ../node/pkg/services/audit/auditor/context.go - AuditorCouldNotWriteAuditReport = "could not write audit report" // Error in ../node/pkg/services/audit/auditor/context.go - AuditorSleepBeforeGetRangeHash = "sleep before get range hash" // Debug in ../node/pkg/services/audit/auditor/pdp.go - AuditorCouldNotGetPayloadRangeHash = "could not get payload range hash" // Debug in ../node/pkg/services/audit/auditor/pdp.go - TaskmanagerProcessRoutine = "process routine" // Info in ../node/pkg/services/audit/taskmanager/listen.go - TaskmanagerStopListenerByContext = "stop listener by context" // Warn in ../node/pkg/services/audit/taskmanager/listen.go - TaskmanagerQueueChannelIsClosed = "queue channel is closed" // Warn in ../node/pkg/services/audit/taskmanager/listen.go - TaskmanagerCouldNotGeneratePDPWorkerPool = "could not generate PDP worker pool" // Error in ../node/pkg/services/audit/taskmanager/listen.go - TaskmanagerCouldNotGeneratePoRWorkerPool = "could not generate PoR worker pool" // Error in ../node/pkg/services/audit/taskmanager/listen.go - TaskmanagerCouldNotSubmitAuditTask = "could not submit audit task" // Warn in ../node/pkg/services/audit/taskmanager/listen.go ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch = "could not restore block subscription after RPC switch" // Error in ../node/pkg/morph/client/notifications.go ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch = "could not restore notification subscription after RPC switch" // Error in ../node/pkg/morph/client/notifications.go ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch = "could not restore notary notification subscription after RPC switch" // Error in ../node/pkg/morph/client/notifications.go @@ -356,19 +334,6 @@ const ( AlphabetCantTransferGas = "can't transfer gas" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go AlphabetCantTransferGasToWallet = "can't transfer gas to wallet" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go AlphabetAlphabetWorkerPool = "alphabet worker pool" // Debug in ../node/pkg/innerring/processors/alphabet/processor.go - AuditContainerListingFinished = "container listing finished" // Debug in ../node/pkg/innerring/processors/audit/scheduler.go - AuditNewRoundOfAudit = "new round of audit" // Info in ../node/pkg/innerring/processors/audit/handlers.go - AuditPreviousRoundOfAuditPrepareHasntFinishedYet = "previous round of audit prepare hasn't finished yet" // Warn in ../node/pkg/innerring/processors/audit/handlers.go - AuditSomeTasksFromPreviousEpochAreSkipped = "some tasks from previous epoch are skipped" // Info in ../node/pkg/innerring/processors/audit/process.go - AuditContainerSelectionFailure = "container selection failure" // Error in ../node/pkg/innerring/processors/audit/process.go - AuditSelectContainersForAudit = "select containers for audit" // Info in ../node/pkg/innerring/processors/audit/process.go - AuditCantFetchNetworkMap = "can't fetch network map" // Error in ../node/pkg/innerring/processors/audit/process.go - AuditCantGetContainerInfoIgnore = "can't get container info, ignore" // Error in ../node/pkg/innerring/processors/audit/process.go - AuditCantBuildPlacementForContainerIgnore = "can't build placement for container, ignore" // Info in ../node/pkg/innerring/processors/audit/process.go - AuditSelectStorageGroupsForAudit = "select storage groups for audit" // Info in ../node/pkg/innerring/processors/audit/process.go - AuditFilterExpiredStorageGroupsForAudit = "filter expired storage groups for audit" // Info in ../node/pkg/innerring/processors/audit/process.go - AuditParseClientNodeInfo = "parse client node info" // Warn in ../node/pkg/innerring/processors/audit/process.go - AuditErrorInStorageGroupSearch = "error in storage group search" // Warn in ../node/pkg/innerring/processors/audit/process.go BalanceBalanceWorkerPoolDrained = "balance worker pool drained" // Warn in ../node/pkg/innerring/processors/balance/handlers.go BalanceNonAlphabetModeIgnoreBalanceLock = "non alphabet mode, ignore balance lock" // Info in ../node/pkg/innerring/processors/balance/process_assets.go BalanceCantSendLockAssetTx = "can't send lock asset tx" // Error in ../node/pkg/innerring/processors/balance/process_assets.go @@ -444,51 +409,6 @@ const ( NetmapNonAlphabetModeIgnoreUpdatePeerNotification = "non alphabet mode, ignore update peer notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapPreventSwitchingNodeToMaintenanceState = "prevent switching node to maintenance state" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go // Debug in ../node/pkg/innerring/processors/reputation/processor.go - SettlementNonAlphabetModeIgnoreAuditPayments = "non alphabet mode, ignore audit payments" // Info in ../node/pkg/innerring/processors/settlement/calls.go - SettlementNewAuditSettlementEvent = "new audit settlement event" // Info in ../node/pkg/innerring/processors/settlement/calls.go - SettlementIgnoreGenesisEpoch = "ignore genesis epoch" // Debug in ../node/pkg/innerring/processors/settlement/calls.go - SettlementCouldNotAddHandlerOfAuditEventToQueue = "could not add handler of AuditEvent to queue" // Warn in ../node/pkg/innerring/processors/settlement/calls.go - SettlementAuditEventHandlingSuccessfullyScheduled = "AuditEvent handling successfully scheduled" // Debug in ../node/pkg/innerring/processors/settlement/calls.go - SettlementNonAlphabetModeIgnoreIncomeCollectionEvent = "non alphabet mode, ignore income collection event" // Info in ../node/pkg/innerring/processors/settlement/calls.go - SettlementStartBasicIncomeCollection = "start basic income collection" // Info in ../node/pkg/innerring/processors/settlement/calls.go - SettlementIncomeContextAlreadyExists = "income context already exists" // Error in ../node/pkg/innerring/processors/settlement/calls.go - SettlementCantCreateIncomeContext = "can't create income context" // Error in ../node/pkg/innerring/processors/settlement/calls.go - SettlementCouldNotAddHandlerOfBasicIncomeCollectionToQueue = "could not add handler of basic income collection to queue" // Warn in ../node/pkg/innerring/processors/settlement/calls.go - SettlementNonAlphabetModeIgnoreIncomeDistributionEvent = "non alphabet mode, ignore income distribution event" // Info in ../node/pkg/innerring/processors/settlement/calls.go - SettlementStartBasicIncomeDistribution = "start basic income distribution" // Info in ../node/pkg/innerring/processors/settlement/calls.go - SettlementIncomeContextDistributionDoesNotExists = "income context distribution does not exists" // Warn in ../node/pkg/innerring/processors/settlement/calls.go - SettlementCouldNotAddHandlerOfBasicIncomeDistributionToQueue = "could not add handler of basic income distribution to queue" // Warn in ../node/pkg/innerring/processors/settlement/calls.go - SettlementProcessAuditSettlements = "process audit settlements" // Info in ../node/pkg/innerring/processors/settlement/handlers.go - SettlementAuditProcessingFinished = "audit processing finished" // Info in ../node/pkg/innerring/processors/settlement/handlers.go - SettlementWorkerPoolForSettlementProcessorSuccessfullyInitialized = "worker pool for settlement processor successfully initialized" // Debug in ../node/pkg/innerring/processors/settlement/processor.go - AuditSettlementsAreIgnoredForZeroEpoch = "settlements are ignored for zero epoch" // Info in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditCalculateAuditSettlements = "calculate audit settlements" // Info in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditGettingResultsForThePreviousEpoch = "getting results for the previous epoch" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditCouldNotCollectAuditResults = "could not collect audit results" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditNoAuditResultsInPreviousEpoch = "no audit results in previous epoch" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditCantFetchAuditFeeFromNetworkConfig = "can't fetch audit fee from network config" // Warn in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditProcessingAuditResults = "processing audit results" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditProcessingTransfers = "processing transfers" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditReadingInformationAboutTheContainer = "reading information about the container" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditBuildingPlacement = "building placement" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditCollectingPassedNodes = "collecting passed nodes" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditCalculatingSumOfTheSizesOfAllStorageGroups = "calculating sum of the sizes of all storage groups" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditFillingTransferTable = "filling transfer table" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditMissingContainerInAuditResult = "missing container in audit result" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditCouldNotGetContainerInfo = "could not get container info" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditCouldNotGetContainerNodes = "could not get container nodes" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditEmptyListOfContainerNodes = "empty list of container nodes" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditNoneOfTheContainerNodesPassedTheAudit = "none of the container nodes passed the audit" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditCouldNotGetSGInfo = "could not get storage group info" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditZeroSumSGSize = "zero sum storage group size" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditCouldNotResolvePublicKeyOfTheStorageNode = "could not resolve public key of the storage node" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditCalculatingStorageNodeSalaryForAudit = "calculating storage node salary for audit (GASe-12)" // Debug in ../node/pkg/innerring/processors/settlement/audit/calculate.go - AuditCouldNotParsePublicKeyOfTheInnerRingNode = "could not parse public key of the inner ring node" // Error in ../node/pkg/innerring/processors/settlement/audit/calculate.go - BasicCantGetBasicIncomeRate = "can't get basic income rate" // Error in ../node/pkg/innerring/processors/settlement/basic/collect.go - BasicCantFetchContainerSizeEstimations = "can't fetch container size estimations" // Error in ../node/pkg/innerring/processors/settlement/basic/collect.go - BasicCantFetchContainerInfo = "can't fetch container info" // Warn in ../node/pkg/innerring/processors/settlement/basic/collect.go - BasicCantFetchBalanceOfBankingAccount = "can't fetch balance of banking account" // Error in ../node/pkg/innerring/processors/settlement/basic/distribute.go - BasicCantTransformPublicKeyToOwnerID = "can't transform public key to owner id" // Warn in ../node/pkg/innerring/processors/settlement/basic/distribute.go FrostFSIRInternalError = "internal error" // Info in ../node/cmd/frostfs-ir/main.go FrostFSIRCouldNotShutdownHTTPServer = "could not shutdown HTTP server" // Debug in ../node/cmd/frostfs-ir/main.go FrostFSIRApplicationStopped = "application stopped" // Info in ../node/cmd/frostfs-ir/main.go From f368ccbdf0ea7454801ddd50393092893f588530 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 25 Apr 2023 10:04:55 +0300 Subject: [PATCH 0179/1943] [#248] morph: Remove audit client Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/audit/client.go | 40 ----------- pkg/morph/client/audit/get_result.go | 34 ---------- pkg/morph/client/audit/list_results.go | 93 -------------------------- pkg/morph/client/audit/put_result.go | 40 ----------- pkg/morph/client/audit/result_test.go | 58 ---------------- 5 files changed, 265 deletions(-) delete mode 100644 pkg/morph/client/audit/client.go delete mode 100644 pkg/morph/client/audit/get_result.go delete mode 100644 pkg/morph/client/audit/list_results.go delete mode 100644 pkg/morph/client/audit/put_result.go delete mode 100644 pkg/morph/client/audit/result_test.go diff --git a/pkg/morph/client/audit/client.go b/pkg/morph/client/audit/client.go deleted file mode 100644 index b922fc792..000000000 --- a/pkg/morph/client/audit/client.go +++ /dev/null @@ -1,40 +0,0 @@ -package audit - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - "github.com/nspcc-dev/neo-go/pkg/util" -) - -// Client is a wrapper over StaticClient -// which makes calls with the names and arguments -// of the FrostFS Audit contract. -// -// Working client must be created via constructor New. -// Using the Client that has been created with new(Client) -// expression (or just declaring a Client variable) is unsafe -// and can lead to panic. -type Client struct { - client *client.StaticClient // static Audit contract client -} - -const ( - putResultMethod = "put" - getResultMethod = "get" - listResultsMethod = "list" - listByEpochResultsMethod = "listByEpoch" - listByCIDResultsMethod = "listByCID" - listByNodeResultsMethod = "listByNode" -) - -// NewFromMorph returns the wrapper instance from the raw morph client. -func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8, opts ...client.StaticClientOption) (*Client, error) { - sc, err := client.NewStatic(cli, contract, fee, opts...) - if err != nil { - return nil, fmt.Errorf("could not create static client of audit contract: %w", err) - } - - return &Client{client: sc}, nil -} diff --git a/pkg/morph/client/audit/get_result.go b/pkg/morph/client/audit/get_result.go deleted file mode 100644 index 9e1b75db4..000000000 --- a/pkg/morph/client/audit/get_result.go +++ /dev/null @@ -1,34 +0,0 @@ -package audit - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - auditAPI "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/audit" -) - -// GetAuditResult returns audit result structure stored in audit contract. -func (c *Client) GetAuditResult(id ResultID) (*auditAPI.Result, error) { - prm := client.TestInvokePrm{} - prm.SetMethod(getResultMethod) - prm.SetArgs([]byte(id)) - - prms, err := c.client.TestInvoke(prm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", getResultMethod, err) - } else if ln := len(prms); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", getResultMethod, ln) - } - - value, err := client.BytesFromStackItem(prms[0]) - if err != nil { - return nil, fmt.Errorf("could not get byte array from stack item (%s): %w", getResultMethod, err) - } - - var auditRes auditAPI.Result - if err := auditRes.Unmarshal(value); err != nil { - return nil, fmt.Errorf("could not unmarshal audit result structure: %w", err) - } - - return &auditRes, nil -} diff --git a/pkg/morph/client/audit/list_results.go b/pkg/morph/client/audit/list_results.go deleted file mode 100644 index ace01d15b..000000000 --- a/pkg/morph/client/audit/list_results.go +++ /dev/null @@ -1,93 +0,0 @@ -package audit - -import ( - "crypto/sha256" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" -) - -// ListAllAuditResultID returns a list of all audit result IDs inside audit contract. -func (c *Client) ListAllAuditResultID() ([]ResultID, error) { - invokePrm := client.TestInvokePrm{} - invokePrm.SetMethod(listResultsMethod) - - items, err := c.client.TestInvoke(invokePrm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", listResultsMethod, err) - } - return parseAuditResults(items, listResultsMethod) -} - -// ListAuditResultIDByEpoch returns a list of audit result IDs inside audit -// contract for specific epoch number. -func (c *Client) ListAuditResultIDByEpoch(epoch uint64) ([]ResultID, error) { - prm := client.TestInvokePrm{} - prm.SetMethod(listByEpochResultsMethod) - prm.SetArgs(epoch) - - items, err := c.client.TestInvoke(prm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", listByEpochResultsMethod, err) - } - return parseAuditResults(items, listByEpochResultsMethod) -} - -// ListAuditResultIDByCID returns a list of audit result IDs inside audit -// contract for specific epoch number and container ID. -func (c *Client) ListAuditResultIDByCID(epoch uint64, cnr cid.ID) ([]ResultID, error) { - binCnr := make([]byte, sha256.Size) - cnr.Encode(binCnr) - - prm := client.TestInvokePrm{} - prm.SetMethod(listByCIDResultsMethod) - prm.SetArgs(epoch, binCnr) - - items, err := c.client.TestInvoke(prm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", listByCIDResultsMethod, err) - } - return parseAuditResults(items, listByCIDResultsMethod) -} - -// ListAuditResultIDByNode returns a list of audit result IDs inside audit -// contract for specific epoch number, container ID and inner ring public key. -func (c *Client) ListAuditResultIDByNode(epoch uint64, cnr cid.ID, nodeKey []byte) ([]ResultID, error) { - binCnr := make([]byte, sha256.Size) - cnr.Encode(binCnr) - - prm := client.TestInvokePrm{} - prm.SetMethod(listByNodeResultsMethod) - prm.SetArgs(epoch, binCnr, nodeKey) - - items, err := c.client.TestInvoke(prm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", listByNodeResultsMethod, err) - } - return parseAuditResults(items, listByNodeResultsMethod) -} - -func parseAuditResults(items []stackitem.Item, method string) ([]ResultID, error) { - if ln := len(items); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", method, ln) - } - - items, err := client.ArrayFromStackItem(items[0]) - if err != nil { - return nil, fmt.Errorf("could not get stack item array from stack item (%s): %w", method, err) - } - - res := make([]ResultID, 0, len(items)) - for i := range items { - rawRes, err := client.BytesFromStackItem(items[i]) - if err != nil { - return nil, fmt.Errorf("could not get byte array from stack item (%s): %w", method, err) - } - - res = append(res, rawRes) - } - - return res, nil -} diff --git a/pkg/morph/client/audit/put_result.go b/pkg/morph/client/audit/put_result.go deleted file mode 100644 index f8e233b26..000000000 --- a/pkg/morph/client/audit/put_result.go +++ /dev/null @@ -1,40 +0,0 @@ -package audit - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - auditAPI "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/audit" -) - -// ResultID is an identity of audit result inside audit contract. -type ResultID []byte - -// PutPrm groups parameters of PutAuditResult operation. -type PutPrm struct { - result *auditAPI.Result - - client.InvokePrmOptional -} - -// SetResult sets audit result. -func (p *PutPrm) SetResult(result *auditAPI.Result) { - p.result = result -} - -// PutAuditResult saves passed audit result structure in FrostFS system -// through Audit contract call. -// -// Returns encountered error that caused the saving to interrupt. -func (c *Client) PutAuditResult(p PutPrm) error { - prm := client.InvokePrm{} - prm.SetMethod(putResultMethod) - prm.SetArgs(p.result.Marshal()) - prm.InvokePrmOptional = p.InvokePrmOptional - - err := c.client.Invoke(prm) - if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", putResultMethod, err) - } - return nil -} diff --git a/pkg/morph/client/audit/result_test.go b/pkg/morph/client/audit/result_test.go deleted file mode 100644 index 5ce1cc740..000000000 --- a/pkg/morph/client/audit/result_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package audit - -import ( - "context" - "testing" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - auditAPI "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/audit" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/stretchr/testify/require" -) - -func TestAuditResults(t *testing.T) { - t.Skip() - const epoch = 11 - - endpoint := "http://morph_chain.frostfs.devenv:30333" - sAuditHash := "cdfb3dab86e6d60e8a143d9e2ecb0b188f3dc2eb" - irKeyWIF := "L3o221BojgcCPYgdbXsm6jn7ayTZ72xwREvBHXKknR8VJ3G4WmjB" - - key, err := keys.NewPrivateKeyFromWIF(irKeyWIF) - require.NoError(t, err) - - auditHash, err := util.Uint160DecodeStringLE(sAuditHash) - require.NoError(t, err) - - morphClient, err := client.New(context.Background(), key, client.WithEndpoints(client.Endpoint{Address: endpoint})) - require.NoError(t, err) - - auditClientWrapper, err := NewFromMorph(morphClient, auditHash, 0) - require.NoError(t, err) - - id := cidtest.ID() - - var auditRes auditAPI.Result - auditRes.ForEpoch(epoch) - auditRes.SetAuditorKey(key.PublicKey().Bytes()) - auditRes.ForContainer(id) - - prm := PutPrm{} - prm.SetResult(&auditRes) - - require.NoError(t, auditClientWrapper.PutAuditResult(prm)) - - time.Sleep(5 * time.Second) - - list, err := auditClientWrapper.ListAuditResultIDByCID(epoch, id) - require.NoError(t, err) - require.Len(t, list, 1) - - savedAuditRes, err := auditClientWrapper.GetAuditResult(list[0]) - require.NoError(t, err) - - require.Equal(t, auditRes, savedAuditRes) -} From 3bac5a485dd7b30459c8cc340d49b8a6b5c2e57a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 25 Apr 2023 10:42:44 +0300 Subject: [PATCH 0180/1943] [#248] config: Remove audit-related parameters Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-ir/defaults.go | 22 ---------------------- config/example/ir.env | 14 -------------- config/example/ir.yaml | 20 -------------------- docs/storage-node-configuration.md | 1 - 4 files changed, 57 deletions(-) diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index eaad05f2e..007bb8964 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -33,10 +33,6 @@ func defaultConfiguration(cfg *viper.Viper) { setEmitDefaults(cfg) - setAuditDefaults(cfg) - - setSettlementDefaults(cfg) - cfg.SetDefault("indexer.cache_timeout", 15*time.Second) cfg.SetDefault("locode.db.path", "") @@ -60,23 +56,6 @@ func setFeeDefaults(cfg *viper.Viper) { cfg.SetDefault("fee.named_container_register", 25_0000_0000) // 25.0 Fixed8 } -func setSettlementDefaults(cfg *viper.Viper) { - cfg.SetDefault("settlement.basic_income_rate", 0) - cfg.SetDefault("settlement.audit_fee", 0) -} - -func setAuditDefaults(cfg *viper.Viper) { - cfg.SetDefault("audit.task.exec_pool_size", 10) - cfg.SetDefault("audit.task.queue_capacity", 100) - cfg.SetDefault("audit.timeout.get", "5s") - cfg.SetDefault("audit.timeout.head", "5s") - cfg.SetDefault("audit.timeout.rangehash", "5s") - cfg.SetDefault("audit.timeout.search", "10s") - cfg.SetDefault("audit.pdp.max_sleep_interval", "5s") - cfg.SetDefault("audit.pdp.pairs_pool_size", "10") - cfg.SetDefault("audit.por.pool_size", "10") -} - func setEmitDefaults(cfg *viper.Viper) { cfg.SetDefault("emit.storage.amount", 0) cfg.SetDefault("emit.mint.cache_size", 1000) @@ -124,7 +103,6 @@ func setContractsDefaults(cfg *viper.Viper) { cfg.SetDefault("contracts.frostfs", "") cfg.SetDefault("contracts.balance", "") cfg.SetDefault("contracts.container", "") - cfg.SetDefault("contracts.audit", "") cfg.SetDefault("contracts.proxy", "") cfg.SetDefault("contracts.processing", "") cfg.SetDefault("contracts.proxy", "") diff --git a/config/example/ir.env b/config/example/ir.env index 1d306ebba..e3de23ac1 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -50,16 +50,6 @@ FROSTFS_IR_WORKERS_CONTAINER=10 FROSTFS_IR_WORKERS_NEOFS=10 FROSTFS_IR_WORKERS_NETMAP=10 -FROSTFS_IR_AUDIT_TIMEOUT_GET=5s -FROSTFS_IR_AUDIT_TIMEOUT_HEAD=5s -FROSTFS_IR_AUDIT_TIMEOUT_RANGEHASH=5s -FROSTFS_IR_AUDIT_TIMEOUT_SEARCH=10s -FROSTFS_IR_AUDIT_TASK_EXEC_POOL_SIZE=10 -FROSTFS_IR_AUDIT_TASK_QUEUE_CAPACITY=100 -FROSTFS_IR_AUDIT_PDP_PAIRS_POOL_SIZE=10 -FROSTFS_IR_AUDIT_PDP_MAX_SLEEP_INTERVAL=5s -FROSTFS_IR_AUDIT_POR_POOL_SIZE=10 - FROSTFS_IR_INDEXER_CACHE_TIMEOUT=15s FROSTFS_IR_NETMAP_CLEANER_ENABLED=true @@ -67,7 +57,6 @@ FROSTFS_IR_NETMAP_CLEANER_THRESHOLD=3 FROSTFS_IR_CONTRACTS_NEOFS=ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62 FROSTFS_IR_CONTRACTS_PROCESSING=597f5894867113a41e192801709c02497f611de8 -FROSTFS_IR_CONTRACTS_AUDIT=219e37aed2180b87e7fe945dbf97d67125e8d73f FROSTFS_IR_CONTRACTS_BALANCE=d2aa48d14b17b11bc4c68205027884a96706dd16 FROSTFS_IR_CONTRACTS_CONTAINER=ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6 FROSTFS_IR_CONTRACTS_NEOFSID=9f5866decbc751a099e74c7c7bc89f609201755a @@ -89,6 +78,3 @@ FROSTFS_IR_PPROF_SHUTDOWN_TIMEOUT=30s FROSTFS_IR_PROMETHEUS_ENABLED=true FROSTFS_IR_PROMETHEUS_ADDRESS=localhost:9090 FROSTFS_IR_PROMETHEUS_SHUTDOWN_TIMEOUT=30s - -FROSTFS_IR_SETTLEMENT_BASIC_INCOME_RATE=100 -FROSTFS_IR_SETTLEMENT_AUDIT_FEE=100 diff --git a/config/example/ir.yaml b/config/example/ir.yaml index 85c1b5d9a..bd56ec74b 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -84,21 +84,6 @@ workers: frostfs: 10 # Number of workers to process events from frostfs contracts in parallel netmap: 10 # Number of workers to process events from netmap contract in parallel -audit: - timeout: - get: 5s # Timeout for object.Get operation during data audit - head: 5s # Timeout for object.Head operation during data audit - rangehash: 5s # Timeout for object.RangeHash operation during data audit - search: 10s # Timeout for object.Search operation during data audit - task: - exec_pool_size: 10 # Number of workers to process audit routine in parallel - queue_capacity: 100 # Maximum amount of simultaneous audit jobs - pdp: - pairs_pool_size: 10 # Number of workers to process PDP part of data audit in parallel - max_sleep_interval: 5s # Maximum timeout between object.RangeHash requests to the storage node - por: - pool_size: 10 # Number of workers to process PoR part of data audit in parallel - indexer: cache_timeout: 15s # Duration between internal state update about current list of inner ring nodes @@ -109,7 +94,6 @@ netmap_cleaner: contracts: frostfs: ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62 # Address of FrostFS contract in mainchain; ignore if mainchain is disabled processing: 597f5894867113a41e192801709c02497f611de8 # Address of processing contract in mainchain; ignore if mainchain is disabled or notary is disabled in mainchain - audit: 219e37aed2180b87e7fe945dbf97d67125e8d73f # Optional: override address of audit contract in sidechain balance: d2aa48d14b17b11bc4c68205027884a96706dd16 # Optional: override address of balance contract in sidechain container: ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6 # Optional: override address of container contract in sidechain frostfsid: 9f5866decbc751a099e74c7c7bc89f609201755a # Optional: override address of frostfsid contract in sidechain @@ -134,7 +118,3 @@ prometheus: enabled: true address: localhost:9090 # Endpoint for application prometheus metrics; disabled by default shutdown_timeout: 30s # Timeout for metrics HTTP server graceful shutdown - -settlement: - basic_income_rate: 100 # Optional: override basic income rate value from network config; applied only in debug mode - audit_fee: 100 # Optional: override audit fee value from network config; applied only in debug mode diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index d21725668..366c263a0 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -118,7 +118,6 @@ contracts: | Parameter | Type | Default value | Description | |--------------|-----------|---------------|---------------------------| -| `audit` | `hash160` | | Audit contract hash. | | `balance` | `hash160` | | Balance contract hash. | | `container` | `hash160` | | Container contract hash. | | `netmap` | `hash160` | | Netmap contract hash. | From 8fc082b68810e269a049b0767396d5018e06a9d8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 20 Apr 2023 10:58:43 +0300 Subject: [PATCH 0181/1943] [#277] getsvc: Do not return status error Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assemble.go | 6 ++---- pkg/services/object/get/get.go | 12 ++++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index d2108b003..4ca55188c 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -128,10 +128,8 @@ func (exec *execCtx) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Ra p.addr.SetContainer(exec.containerID()) p.addr.SetObject(id) - statusError := exec.svc.get(ctx, p.commonPrm, withPayloadRange(rng)) - - if statusError.err != nil { - return nil, statusError.err + if err := exec.svc.get(ctx, p.commonPrm, withPayloadRange(rng)); err != nil { + return nil, err } return w.Object(), nil } diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 6295de9a9..4e65fb528 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -11,7 +11,7 @@ import ( // Get serves a request to get an object by address, and returns Streamer instance. func (s *Service) Get(ctx context.Context, prm Prm) error { - return s.get(ctx, prm.commonPrm).err + return s.get(ctx, prm.commonPrm) } // GetRange serves a request to get an object by address, and returns Streamer instance. @@ -19,8 +19,8 @@ func (s *Service) GetRange(ctx context.Context, prm RangePrm) error { return s.getRange(ctx, prm) } -func (s *Service) getRange(ctx context.Context, prm RangePrm, opts ...execOption) error { - return s.get(ctx, prm.commonPrm, append(opts, withPayloadRange(prm.rng))...).err +func (s *Service) getRange(ctx context.Context, prm RangePrm) error { + return s.get(ctx, prm.commonPrm, withPayloadRange(prm.rng)) } func (s *Service) GetRangeHash(ctx context.Context, prm RangeHashPrm) (*RangeHashRes, error) { @@ -60,10 +60,10 @@ func (s *Service) GetRangeHash(ctx context.Context, prm RangeHashPrm) (*RangeHas // Returns ErrNotFound if the header was not received for the call. // Returns SplitInfoError if object is virtual and raw flag is set. func (s *Service) Head(ctx context.Context, prm HeadPrm) error { - return s.get(ctx, prm.commonPrm, headOnly()).err + return s.get(ctx, prm.commonPrm, headOnly()) } -func (s *Service) get(ctx context.Context, prm commonPrm, opts ...execOption) statusError { +func (s *Service) get(ctx context.Context, prm commonPrm, opts ...execOption) error { exec := &execCtx{ svc: s, prm: RangePrm{ @@ -80,7 +80,7 @@ func (s *Service) get(ctx context.Context, prm commonPrm, opts ...execOption) st exec.execute(ctx) - return exec.statusError + return exec.statusError.err } func (exec *execCtx) execute(ctx context.Context) { From 30e1b62b67276baf51a59e09d3d05e09605f5a46 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 20 Apr 2023 13:57:45 +0300 Subject: [PATCH 0182/1943] [#277] getsvc: Fix service deps Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 17 +- pkg/services/object/get/exec.go | 16 +- pkg/services/object/get/get_test.go | 97 ++++++------ pkg/services/object/get/local.go | 12 +- pkg/services/object/get/remote.go | 2 +- pkg/services/object/get/service.go | 136 ++++------------ pkg/services/object/get/types.go | 230 ++++++++++++++++++++++++++++ pkg/services/object/get/util.go | 188 ----------------------- 8 files changed, 339 insertions(+), 359 deletions(-) create mode 100644 pkg/services/object/get/types.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 83025a44c..4ff9b8522 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -336,17 +336,14 @@ func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Tra ls := c.cfgObject.cfgLocalStorage.localStorage return getsvc.New( - getsvc.WithLogger(c.log), - getsvc.WithLocalStorageEngine(ls), - getsvc.WithClientConstructor(coreConstructor), - getsvc.WithTraverserGenerator( - traverseGen.WithTraverseOptions( - placement.SuccessAfter(1), - ), + keyStorage, + c.netMapSource, + ls, + traverseGen.WithTraverseOptions( + placement.SuccessAfter(1), ), - getsvc.WithNetMapSource(c.netMapSource), - getsvc.WithKeyStorage(keyStorage), - ) + coreConstructor, + getsvc.WithLogger(c.log)) } func createGetServiceV2(sGet *getsvc.Service, keyStorage *util.KeyStorage) *getsvcV2.Service { diff --git a/pkg/services/object/get/exec.go b/pkg/services/object/get/exec.go index 1bd5aa7f8..e57decb67 100644 --- a/pkg/services/object/get/exec.go +++ b/pkg/services/object/get/exec.go @@ -143,7 +143,7 @@ func (exec *execCtx) initEpoch() bool { return true } - e, err := exec.svc.currentEpochReceiver.currentEpoch() + e, err := exec.svc.epochSource.Epoch() switch { default: @@ -181,20 +181,18 @@ func (exec *execCtx) generateTraverser(addr oid.Address) (*placement.Traverser, } } -func (exec execCtx) remoteClient(info clientcore.NodeInfo) (getClient, bool) { - c, err := exec.svc.clientCache.get(info) - - switch { - default: +func (exec execCtx) remoteClient(info clientcore.NodeInfo) (remoteStorage, bool) { + rs, err := exec.svc.remoteStorageConstructor.Get(info) + if err != nil { exec.status = statusUndefined exec.err = err exec.log.Debug(logs.GetCouldNotConstructRemoteNodeClient) - case err == nil: - return c, true + + return nil, false } - return nil, false + return rs, true } func mergeSplitInfo(dst, src *objectSDK.SplitInfo) { diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 319bc6b58..15a14ac18 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -56,7 +56,7 @@ type testClient struct { type testEpochReceiver uint64 -func (e testEpochReceiver) currentEpoch() (uint64, error) { +func (e testEpochReceiver) Epoch() (uint64, error) { return uint64(e), nil } @@ -99,7 +99,7 @@ func (p *testPlacementBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, _ netmap. return vs, nil } -func (c *testClientCache) get(info client.NodeInfo) (getClient, error) { +func (c *testClientCache) Get(info client.NodeInfo) (remoteStorage, error) { v, ok := c.clients[network.StringifyGroup(info.AddressGroup())] if !ok { return nil, errors.New("could not construct client") @@ -117,7 +117,7 @@ func newTestClient() *testClient { } } -func (c *testClient) getObject(ctx context.Context, exec *execCtx, _ client.NodeInfo) (*objectSDK.Object, error) { +func (c *testClient) GetObject(ctx context.Context, exec *execCtx, _ client.NodeInfo) (*objectSDK.Object, error) { v, ok := c.results[exec.address().EncodeToString()] if !ok { var errNotFound apistatus.ObjectNotFound @@ -139,11 +139,19 @@ func (c *testClient) addResult(addr oid.Address, obj *objectSDK.Object, err erro }{obj: obj, err: err} } -func (s *testStorage) get(_ context.Context, exec *execCtx) (*objectSDK.Object, error) { +func (s *testStorage) Get(ctx context.Context, address oid.Address) (*objectSDK.Object, error) { + return s.Range(ctx, address, nil) +} + +func (s *testStorage) Head(ctx context.Context, address oid.Address, isRaw bool) (*objectSDK.Object, error) { + return s.Range(ctx, address, nil) +} + +func (s *testStorage) Range(_ context.Context, address oid.Address, rng *objectSDK.Range) (*objectSDK.Object, error) { var ( ok bool obj *objectSDK.Object - sAddr = exec.address().EncodeToString() + sAddr = address.EncodeToString() ) if _, ok = s.inhumed[sAddr]; ok { @@ -157,7 +165,7 @@ func (s *testStorage) get(_ context.Context, exec *execCtx) (*objectSDK.Object, } if obj, ok = s.phy[sAddr]; ok { - return cutToRange(obj, exec.ctxRange()), nil + return cutToRange(obj, rng), nil } var errNotFound apistatus.ObjectNotFound @@ -245,11 +253,10 @@ func TestGetLocalOnly(t *testing.T) { ctx := context.Background() newSvc := func(storage *testStorage) *Service { - svc := &Service{cfg: new(cfg)} - svc.log = test.NewLogger(t, false) - svc.localStorage = storage - - return svc + return &Service{ + log: test.NewLogger(t, false), + localStorage: storage, + } } newPrm := func(raw bool, w ObjectWriter) Prm { @@ -506,22 +513,20 @@ func TestGetRemoteSmall(t *testing.T) { container.CalculateID(&idCnr, cnr) newSvc := func(b *testPlacementBuilder, c *testClientCache) *Service { - svc := &Service{cfg: new(cfg)} - svc.log = test.NewLogger(t, false) - svc.localStorage = newTestStorage() - const curEpoch = 13 - svc.traverserGenerator = &testTraverserGenerator{ - c: cnr, - b: map[uint64]placement.Builder{ - curEpoch: b, + return &Service{ + log: test.NewLogger(t, false), + localStorage: newTestStorage(), + traverserGenerator: &testTraverserGenerator{ + c: cnr, + b: map[uint64]placement.Builder{ + curEpoch: b, + }, }, + epochSource: testEpochReceiver(curEpoch), + remoteStorageConstructor: c, } - svc.clientCache = c - svc.currentEpochReceiver = testEpochReceiver(curEpoch) - - return svc } newPrm := func(raw bool, w ObjectWriter) Prm { @@ -1639,39 +1644,37 @@ func TestGetFromPastEpoch(t *testing.T) { c22 := newTestClient() c22.addResult(addr, obj, nil) - svc := &Service{cfg: new(cfg)} - svc.log = test.NewLogger(t, false) - svc.localStorage = newTestStorage() - const curEpoch = 13 - svc.traverserGenerator = &testTraverserGenerator{ - c: cnr, - b: map[uint64]placement.Builder{ - curEpoch: &testPlacementBuilder{ - vectors: map[string][][]netmap.NodeInfo{ - addr.EncodeToString(): ns[:1], + svc := &Service{ + log: test.NewLogger(t, false), + localStorage: newTestStorage(), + epochSource: testEpochReceiver(curEpoch), + traverserGenerator: &testTraverserGenerator{ + c: cnr, + b: map[uint64]placement.Builder{ + curEpoch: &testPlacementBuilder{ + vectors: map[string][][]netmap.NodeInfo{ + addr.EncodeToString(): ns[:1], + }, }, - }, - curEpoch - 1: &testPlacementBuilder{ - vectors: map[string][][]netmap.NodeInfo{ - addr.EncodeToString(): ns[1:], + curEpoch - 1: &testPlacementBuilder{ + vectors: map[string][][]netmap.NodeInfo{ + addr.EncodeToString(): ns[1:], + }, }, }, }, - } - - svc.clientCache = &testClientCache{ - clients: map[string]*testClient{ - as[0][0]: c11, - as[0][1]: c12, - as[1][0]: c21, - as[1][1]: c22, + remoteStorageConstructor: &testClientCache{ + clients: map[string]*testClient{ + as[0][0]: c11, + as[0][1]: c12, + as[1][0]: c21, + as[1][1]: c22, + }, }, } - svc.currentEpochReceiver = testEpochReceiver(curEpoch) - w := NewSimpleObjectWriter() commonPrm := new(util.CommonPrm) diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index 8ac83d97a..97fda6ce7 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -19,7 +19,7 @@ func (exec *execCtx) executeLocal(ctx context.Context) { var err error - exec.collectedObject, err = exec.svc.localStorage.get(ctx, exec) + exec.collectedObject, err = exec.get(ctx) var errSplitInfo *objectSDK.SplitInfoError var errRemoved apistatus.ObjectAlreadyRemoved @@ -49,3 +49,13 @@ func (exec *execCtx) executeLocal(ctx context.Context) { exec.err = errOutOfRange } } + +func (exec *execCtx) get(ctx context.Context) (*objectSDK.Object, error) { + if exec.headOnly() { + return exec.svc.localStorage.Head(ctx, exec.address(), exec.isRaw()) + } + if rng := exec.ctxRange(); rng != nil { + return exec.svc.localStorage.Range(ctx, exec.address(), rng) + } + return exec.svc.localStorage.Get(ctx, exec.address()) +} diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index ac8ec5105..56be476f2 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -23,7 +23,7 @@ func (exec *execCtx) processNode(ctx context.Context, info client.NodeInfo) bool return true } - obj, err := client.getObject(ctx, exec, info) + obj, err := client.GetObject(ctx, exec, info) var errSplitInfo *objectSDK.SplitInfoError var errRemoved *apistatus.ObjectAlreadyRemoved diff --git a/pkg/services/object/get/service.go b/pkg/services/object/get/service.go index a9391d016..bdf01a977 100644 --- a/pkg/services/object/get/service.go +++ b/pkg/services/object/get/service.go @@ -1,124 +1,54 @@ package getsvc import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) +// Option is a Service's constructor option. +type Option func(*Service) + // Service utility serving requests of Object.Get service. type Service struct { - *cfg -} - -// Option is a Service's constructor option. -type Option func(*cfg) - -type getClient interface { - getObject(context.Context, *execCtx, client.NodeInfo) (*object.Object, error) -} - -type cfg struct { - log *logger.Logger - - localStorage interface { - get(context.Context, *execCtx) (*object.Object, error) - } - - clientCache interface { - get(client.NodeInfo) (getClient, error) - } - - traverserGenerator interface { - GenerateTraverser(cid.ID, *oid.ID, uint64) (*placement.Traverser, error) - } - - currentEpochReceiver interface { - currentEpoch() (uint64, error) - } - - keyStore *util.KeyStorage -} - -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - localStorage: new(storageEngineWrapper), - clientCache: new(clientCacheWrapper), - } + log *logger.Logger + localStorage localStorage + traverserGenerator traverserGenerator + epochSource epochSource + keyStore keyStorage + remoteStorageConstructor remoteStorageConstructor } // New creates, initializes and returns utility serving // Object.Get service requests. -func New(opts ...Option) *Service { - c := defaultCfg() - - for i := range opts { - opts[i](c) +func New( + ks keyStorage, + es epochSource, + e localStorageEngine, + tg traverserGenerator, + cc clientConstructor, + opts ...Option, +) *Service { + result := &Service{ + keyStore: ks, + epochSource: es, + log: &logger.Logger{Logger: zap.L()}, + localStorage: &engineLocalStorage{ + engine: e, + }, + traverserGenerator: tg, + remoteStorageConstructor: &multiclientRemoteStorageConstructor{ + clientConstructor: cc, + }, } - - return &Service{ - cfg: c, + for _, option := range opts { + option(result) } + return result } // WithLogger returns option to specify Get service's logger. func WithLogger(l *logger.Logger) Option { - return func(c *cfg) { - c.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Get service"))} - } -} - -// WithLocalStorageEngine returns option to set local storage -// instance. -func WithLocalStorageEngine(e *engine.StorageEngine) Option { - return func(c *cfg) { - c.localStorage.(*storageEngineWrapper).engine = e - } -} - -type ClientConstructor interface { - Get(client.NodeInfo) (client.MultiAddressClient, error) -} - -// WithClientConstructor returns option to set constructor of remote node clients. -func WithClientConstructor(v ClientConstructor) Option { - return func(c *cfg) { - c.clientCache.(*clientCacheWrapper).cache = v - } -} - -// WithTraverserGenerator returns option to set generator of -// placement traverser to get the objects from containers. -func WithTraverserGenerator(t *util.TraverserGenerator) Option { - return func(c *cfg) { - c.traverserGenerator = t - } -} - -// WithNetMapSource returns option to set network -// map storage to receive current network state. -func WithNetMapSource(nmSrc netmap.Source) Option { - return func(c *cfg) { - c.currentEpochReceiver = &nmSrcWrapper{ - nmSrc: nmSrc, - } - } -} - -// WithKeyStorage returns option to set private -// key storage for session tokens and node key. -func WithKeyStorage(store *util.KeyStorage) Option { - return func(c *cfg) { - c.keyStore = store + return func(s *Service) { + s.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Get service"))} } } diff --git a/pkg/services/object/get/types.go b/pkg/services/object/get/types.go new file mode 100644 index 000000000..47f27c0f0 --- /dev/null +++ b/pkg/services/object/get/types.go @@ -0,0 +1,230 @@ +package getsvc + +import ( + "context" + "crypto/ecdsa" + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +type epochSource interface { + Epoch() (uint64, error) +} + +type traverserGenerator interface { + GenerateTraverser(cid.ID, *oid.ID, uint64) (*placement.Traverser, error) +} + +type keyStorage interface { + GetKey(info *util.SessionInfo) (*ecdsa.PrivateKey, error) +} + +type localStorageEngine interface { + Head(ctx context.Context, p engine.HeadPrm) (engine.HeadRes, error) + GetRange(ctx context.Context, p engine.RngPrm) (engine.RngRes, error) + Get(ctx context.Context, p engine.GetPrm) (engine.GetRes, error) +} + +type clientConstructor interface { + Get(client.NodeInfo) (client.MultiAddressClient, error) +} + +type remoteStorageConstructor interface { + Get(client.NodeInfo) (remoteStorage, error) +} + +type multiclientRemoteStorageConstructor struct { + clientConstructor clientConstructor +} + +func (c *multiclientRemoteStorageConstructor) Get(info client.NodeInfo) (remoteStorage, error) { + clt, err := c.clientConstructor.Get(info) + if err != nil { + return nil, err + } + + return &multiaddressRemoteStorage{ + client: clt, + }, nil +} + +type remoteStorage interface { + GetObject(context.Context, *execCtx, client.NodeInfo) (*objectSDK.Object, error) +} + +type localStorage interface { + Head(ctx context.Context, address oid.Address, isRaw bool) (*objectSDK.Object, error) + Range(ctx context.Context, address oid.Address, rng *objectSDK.Range) (*objectSDK.Object, error) + Get(ctx context.Context, address oid.Address) (*objectSDK.Object, error) +} + +type engineLocalStorage struct { + engine localStorageEngine +} + +func (s *engineLocalStorage) Head(ctx context.Context, address oid.Address, isRaw bool) (*objectSDK.Object, error) { + var headPrm engine.HeadPrm + headPrm.WithAddress(address) + headPrm.WithRaw(isRaw) + + r, err := s.engine.Head(ctx, headPrm) + if err != nil { + return nil, err + } + + return r.Header(), nil +} + +func (s *engineLocalStorage) Range(ctx context.Context, address oid.Address, rng *objectSDK.Range) (*objectSDK.Object, error) { + var getRange engine.RngPrm + getRange.WithAddress(address) + getRange.WithPayloadRange(rng) + + r, err := s.engine.GetRange(ctx, getRange) + if err != nil { + return nil, err + } + + return r.Object(), nil +} + +func (s *engineLocalStorage) Get(ctx context.Context, address oid.Address) (*objectSDK.Object, error) { + var getPrm engine.GetPrm + getPrm.WithAddress(address) + + r, err := s.engine.Get(ctx, getPrm) + if err != nil { + return nil, err + } + + return r.Object(), nil +} + +type multiaddressRemoteStorage struct { + client coreclient.MultiAddressClient +} + +func (s *multiaddressRemoteStorage) GetObject(ctx context.Context, exec *execCtx, info coreclient.NodeInfo) (*objectSDK.Object, error) { + if exec.isForwardingEnabled() { + return exec.prm.forwarder(ctx, info, s.client) + } + + key, err := exec.key() + if err != nil { + return nil, err + } + + if exec.headOnly() { + return s.getHeadOnly(ctx, exec, key) + } + // we don't specify payload writer because we accumulate + // the object locally (even huge). + if rng := exec.ctxRange(); rng != nil { + // Current spec allows other storage node to deny access, + // fallback to GET here. + return s.getRange(ctx, exec, key, rng) + } + + return s.get(ctx, exec, key) +} + +func (s *multiaddressRemoteStorage) getRange(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey, rng *object.Range) (*object.Object, error) { + var prm internalclient.PayloadRangePrm + + prm.SetClient(s.client) + prm.SetTTL(exec.prm.common.TTL()) + prm.SetNetmapEpoch(exec.curProcEpoch) + prm.SetAddress(exec.address()) + prm.SetPrivateKey(key) + prm.SetSessionToken(exec.prm.common.SessionToken()) + prm.SetBearerToken(exec.prm.common.BearerToken()) + prm.SetXHeaders(exec.prm.common.XHeaders()) + prm.SetRange(rng) + + if exec.isRaw() { + prm.SetRawFlag() + } + + res, err := internalclient.PayloadRange(ctx, prm) + if err != nil { + var errAccessDenied *apistatus.ObjectAccessDenied + if errors.As(err, &errAccessDenied) { + obj, err := s.get(ctx, exec, key) + if err != nil { + return nil, err + } + + payload := obj.Payload() + from := rng.GetOffset() + to := from + rng.GetLength() + + if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to { + return nil, new(apistatus.ObjectOutOfRange) + } + + return payloadOnlyObject(payload[from:to]), nil + } + return nil, err + } + + return payloadOnlyObject(res.PayloadRange()), nil +} + +func (s *multiaddressRemoteStorage) getHeadOnly(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { + var prm internalclient.HeadObjectPrm + + prm.SetClient(s.client) + prm.SetTTL(exec.prm.common.TTL()) + prm.SetNetmapEpoch(exec.curProcEpoch) + prm.SetAddress(exec.address()) + prm.SetPrivateKey(key) + prm.SetSessionToken(exec.prm.common.SessionToken()) + prm.SetBearerToken(exec.prm.common.BearerToken()) + prm.SetXHeaders(exec.prm.common.XHeaders()) + + if exec.isRaw() { + prm.SetRawFlag() + } + + res, err := internalclient.HeadObject(ctx, prm) + if err != nil { + return nil, err + } + + return res.Header(), nil +} + +func (s *multiaddressRemoteStorage) get(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { + var prm internalclient.GetObjectPrm + + prm.SetClient(s.client) + prm.SetTTL(exec.prm.common.TTL()) + prm.SetNetmapEpoch(exec.curProcEpoch) + prm.SetAddress(exec.address()) + prm.SetPrivateKey(key) + prm.SetSessionToken(exec.prm.common.SessionToken()) + prm.SetBearerToken(exec.prm.common.BearerToken()) + prm.SetXHeaders(exec.prm.common.XHeaders()) + + if exec.isRaw() { + prm.SetRawFlag() + } + + res, err := internalclient.GetObject(ctx, prm) + if err != nil { + return nil, err + } + + return res.Object(), nil +} diff --git a/pkg/services/object/get/util.go b/pkg/services/object/get/util.go index dd4ace407..3038d0d5e 100644 --- a/pkg/services/object/get/util.go +++ b/pkg/services/object/get/util.go @@ -2,15 +2,8 @@ package getsvc import ( "context" - "crypto/ecdsa" - "errors" "io" - coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) @@ -20,18 +13,6 @@ type SimpleObjectWriter struct { pld []byte } -type clientCacheWrapper struct { - cache ClientConstructor -} - -type clientWrapper struct { - client coreclient.MultiAddressClient -} - -type storageEngineWrapper struct { - engine *engine.StorageEngine -} - type partWriter struct { ObjectWriter @@ -44,10 +25,6 @@ type hasherWrapper struct { hash io.Writer } -type nmSrcWrapper struct { - nmSrc netmap.Source -} - func NewSimpleObjectWriter() *SimpleObjectWriter { return &SimpleObjectWriter{ obj: object.New(), @@ -75,167 +52,6 @@ func (s *SimpleObjectWriter) Object() *object.Object { return s.obj } -func (c *clientCacheWrapper) get(info coreclient.NodeInfo) (getClient, error) { - clt, err := c.cache.Get(info) - if err != nil { - return nil, err - } - - return &clientWrapper{ - client: clt, - }, nil -} - -func (c *clientWrapper) getObject(ctx context.Context, exec *execCtx, info coreclient.NodeInfo) (*object.Object, error) { - if exec.isForwardingEnabled() { - return exec.prm.forwarder(ctx, info, c.client) - } - - key, err := exec.key() - if err != nil { - return nil, err - } - - if exec.headOnly() { - return c.getHeadOnly(ctx, exec, key) - } - // we don't specify payload writer because we accumulate - // the object locally (even huge). - if rng := exec.ctxRange(); rng != nil { - // Current spec allows other storage node to deny access, - // fallback to GET here. - return c.getRange(ctx, exec, key, rng) - } - - return c.get(ctx, exec, key) -} - -func (c *clientWrapper) getRange(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey, rng *object.Range) (*object.Object, error) { - var prm internalclient.PayloadRangePrm - - prm.SetClient(c.client) - prm.SetTTL(exec.prm.common.TTL()) - prm.SetNetmapEpoch(exec.curProcEpoch) - prm.SetAddress(exec.address()) - prm.SetPrivateKey(key) - prm.SetSessionToken(exec.prm.common.SessionToken()) - prm.SetBearerToken(exec.prm.common.BearerToken()) - prm.SetXHeaders(exec.prm.common.XHeaders()) - prm.SetRange(rng) - - if exec.isRaw() { - prm.SetRawFlag() - } - - res, err := internalclient.PayloadRange(ctx, prm) - if err != nil { - var errAccessDenied *apistatus.ObjectAccessDenied - if errors.As(err, &errAccessDenied) { - obj, err := c.get(ctx, exec, key) - if err != nil { - return nil, err - } - - payload := obj.Payload() - from := rng.GetOffset() - to := from + rng.GetLength() - - if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to { - return nil, new(apistatus.ObjectOutOfRange) - } - - return payloadOnlyObject(payload[from:to]), nil - } - return nil, err - } - - return payloadOnlyObject(res.PayloadRange()), nil -} - -func (c *clientWrapper) getHeadOnly(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { - var prm internalclient.HeadObjectPrm - - prm.SetClient(c.client) - prm.SetTTL(exec.prm.common.TTL()) - prm.SetNetmapEpoch(exec.curProcEpoch) - prm.SetAddress(exec.address()) - prm.SetPrivateKey(key) - prm.SetSessionToken(exec.prm.common.SessionToken()) - prm.SetBearerToken(exec.prm.common.BearerToken()) - prm.SetXHeaders(exec.prm.common.XHeaders()) - - if exec.isRaw() { - prm.SetRawFlag() - } - - res, err := internalclient.HeadObject(ctx, prm) - if err != nil { - return nil, err - } - - return res.Header(), nil -} - -func (c *clientWrapper) get(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { - var prm internalclient.GetObjectPrm - - prm.SetClient(c.client) - prm.SetTTL(exec.prm.common.TTL()) - prm.SetNetmapEpoch(exec.curProcEpoch) - prm.SetAddress(exec.address()) - prm.SetPrivateKey(key) - prm.SetSessionToken(exec.prm.common.SessionToken()) - prm.SetBearerToken(exec.prm.common.BearerToken()) - prm.SetXHeaders(exec.prm.common.XHeaders()) - - if exec.isRaw() { - prm.SetRawFlag() - } - - res, err := internalclient.GetObject(ctx, prm) - if err != nil { - return nil, err - } - - return res.Object(), nil -} - -func (e *storageEngineWrapper) get(ctx context.Context, exec *execCtx) (*object.Object, error) { - if exec.headOnly() { - var headPrm engine.HeadPrm - headPrm.WithAddress(exec.address()) - headPrm.WithRaw(exec.isRaw()) - - r, err := e.engine.Head(ctx, headPrm) - if err != nil { - return nil, err - } - - return r.Header(), nil - } else if rng := exec.ctxRange(); rng != nil { - var getRange engine.RngPrm - getRange.WithAddress(exec.address()) - getRange.WithPayloadRange(rng) - - r, err := e.engine.GetRange(ctx, getRange) - if err != nil { - return nil, err - } - - return r.Object(), nil - } else { - var getPrm engine.GetPrm - getPrm.WithAddress(exec.address()) - - r, err := e.engine.Get(ctx, getPrm) - if err != nil { - return nil, err - } - - return r.Object(), nil - } -} - func (w *partWriter) WriteChunk(ctx context.Context, p []byte) error { return w.chunkWriter.WriteChunk(ctx, p) } @@ -255,7 +71,3 @@ func (h *hasherWrapper) WriteChunk(_ context.Context, p []byte) error { _, err := h.hash.Write(p) return err } - -func (n *nmSrcWrapper) currentEpoch() (uint64, error) { - return n.nmSrc.Epoch() -} From 265d2326a0a2ab7dd3b3862e279993a40d439ee5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 21 Apr 2023 18:21:46 +0300 Subject: [PATCH 0183/1943] [#277] getsvc: Extract remote storage Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/exec.go | 2 +- pkg/services/object/get/get_test.go | 42 ++++++++-- pkg/services/object/get/remote.go | 38 ++++++++- pkg/services/object/get/types.go | 124 ++++++++++++++-------------- pkg/services/object/get/util.go | 7 -- 5 files changed, 133 insertions(+), 80 deletions(-) diff --git a/pkg/services/object/get/exec.go b/pkg/services/object/get/exec.go index e57decb67..6b8e0e997 100644 --- a/pkg/services/object/get/exec.go +++ b/pkg/services/object/get/exec.go @@ -181,7 +181,7 @@ func (exec *execCtx) generateTraverser(addr oid.Address) (*placement.Traverser, } } -func (exec execCtx) remoteClient(info clientcore.NodeInfo) (remoteStorage, bool) { +func (exec execCtx) getRemoteStorage(info clientcore.NodeInfo) (remoteStorage, bool) { rs, err := exec.svc.remoteStorageConstructor.Get(info) if err != nil { exec.status = statusUndefined diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 15a14ac18..d6ff00880 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -2,6 +2,7 @@ package getsvc import ( "context" + "crypto/ecdsa" "crypto/rand" "errors" "fmt" @@ -117,8 +118,15 @@ func newTestClient() *testClient { } } -func (c *testClient) GetObject(ctx context.Context, exec *execCtx, _ client.NodeInfo) (*objectSDK.Object, error) { - v, ok := c.results[exec.address().EncodeToString()] +func (c *testClient) addResult(addr oid.Address, obj *objectSDK.Object, err error) { + c.results[addr.EncodeToString()] = struct { + obj *objectSDK.Object + err error + }{obj: obj, err: err} +} + +func (c *testClient) Get(ctx context.Context, address oid.Address, requestParams RemoteRequestParams) (*objectSDK.Object, error) { + v, ok := c.results[address.EncodeToString()] if !ok { var errNotFound apistatus.ObjectNotFound @@ -129,14 +137,23 @@ func (c *testClient) GetObject(ctx context.Context, exec *execCtx, _ client.Node return nil, v.err } - return cutToRange(v.obj, exec.ctxRange()), nil + return v.obj, nil } -func (c *testClient) addResult(addr oid.Address, obj *objectSDK.Object, err error) { - c.results[addr.EncodeToString()] = struct { - obj *objectSDK.Object - err error - }{obj: obj, err: err} +func (c *testClient) Head(ctx context.Context, address oid.Address, requestParams RemoteRequestParams) (*objectSDK.Object, error) { + return c.Get(ctx, address, requestParams) +} + +func (c *testClient) Range(ctx context.Context, address oid.Address, rng *objectSDK.Range, requestParams RemoteRequestParams) (*objectSDK.Object, error) { + obj, err := c.Get(ctx, address, requestParams) + if err != nil { + return nil, err + } + return cutToRange(obj, rng), nil +} + +func (c *testClient) ForwardRequest(ctx context.Context, info client.NodeInfo, forwarder RequestForwarder) (*objectSDK.Object, error) { + return nil, fmt.Errorf("not implemented") } func (s *testStorage) Get(ctx context.Context, address oid.Address) (*objectSDK.Object, error) { @@ -249,6 +266,13 @@ func (w *writePayloadErrorObjectWriter) WriteChunk(_ context.Context, _ []byte) return &writePayloadError{} } +type testKeyStorage struct { +} + +func (ks *testKeyStorage) GetKey(_ *util.SessionInfo) (*ecdsa.PrivateKey, error) { + return &ecdsa.PrivateKey{}, nil +} + func TestGetLocalOnly(t *testing.T) { ctx := context.Background() @@ -526,6 +550,7 @@ func TestGetRemoteSmall(t *testing.T) { }, epochSource: testEpochReceiver(curEpoch), remoteStorageConstructor: c, + keyStore: &testKeyStorage{}, } } @@ -1673,6 +1698,7 @@ func TestGetFromPastEpoch(t *testing.T) { as[1][1]: c22, }, }, + keyStore: &testKeyStorage{}, } w := NewSimpleObjectWriter() diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index 56be476f2..d4c5d7a6c 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -18,12 +18,12 @@ func (exec *execCtx) processNode(ctx context.Context, info client.NodeInfo) bool exec.log.Debug(logs.ProcessingNode) - client, ok := exec.remoteClient(info) + rs, ok := exec.getRemoteStorage(info) if !ok { return true } - obj, err := client.GetObject(ctx, exec, info) + obj, err := exec.getRemote(ctx, rs, info) var errSplitInfo *objectSDK.SplitInfoError var errRemoved *apistatus.ObjectAlreadyRemoved @@ -64,3 +64,37 @@ func (exec *execCtx) processNode(ctx context.Context, info client.NodeInfo) bool return exec.status != statusUndefined } + +func (exec *execCtx) getRemote(ctx context.Context, rs remoteStorage, info client.NodeInfo) (*objectSDK.Object, error) { + if exec.isForwardingEnabled() { + return rs.ForwardRequest(ctx, info, exec.prm.forwarder) + } + + key, err := exec.key() + if err != nil { + return nil, err + } + + prm := RemoteRequestParams{ + Epoch: exec.curProcEpoch, + TTL: exec.prm.common.TTL(), + PrivateKey: key, + SessionToken: exec.prm.common.SessionToken(), + BearerToken: exec.prm.common.BearerToken(), + XHeaders: exec.prm.common.XHeaders(), + IsRaw: exec.isRaw(), + } + + if exec.headOnly() { + return rs.Head(ctx, exec.address(), prm) + } + // we don't specify payload writer because we accumulate + // the object locally (even huge). + if rng := exec.ctxRange(); rng != nil { + // Current spec allows other storage node to deny access, + // fallback to GET here. + return rs.Range(ctx, exec.address(), rng, prm) + } + + return rs.Get(ctx, exec.address(), prm) +} diff --git a/pkg/services/object/get/types.go b/pkg/services/object/get/types.go index 47f27c0f0..800d37996 100644 --- a/pkg/services/object/get/types.go +++ b/pkg/services/object/get/types.go @@ -5,17 +5,17 @@ import ( "crypto/ecdsa" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" ) type epochSource interface { @@ -37,18 +37,18 @@ type localStorageEngine interface { } type clientConstructor interface { - Get(client.NodeInfo) (client.MultiAddressClient, error) + Get(coreclient.NodeInfo) (coreclient.MultiAddressClient, error) } type remoteStorageConstructor interface { - Get(client.NodeInfo) (remoteStorage, error) + Get(coreclient.NodeInfo) (remoteStorage, error) } type multiclientRemoteStorageConstructor struct { clientConstructor clientConstructor } -func (c *multiclientRemoteStorageConstructor) Get(info client.NodeInfo) (remoteStorage, error) { +func (c *multiclientRemoteStorageConstructor) Get(info coreclient.NodeInfo) (remoteStorage, error) { clt, err := c.clientConstructor.Get(info) if err != nil { return nil, err @@ -59,10 +59,6 @@ func (c *multiclientRemoteStorageConstructor) Get(info client.NodeInfo) (remoteS }, nil } -type remoteStorage interface { - GetObject(context.Context, *execCtx, client.NodeInfo) (*objectSDK.Object, error) -} - type localStorage interface { Head(ctx context.Context, address oid.Address, isRaw bool) (*objectSDK.Object, error) Range(ctx context.Context, address oid.Address, rng *objectSDK.Range) (*objectSDK.Object, error) @@ -111,48 +107,45 @@ func (s *engineLocalStorage) Get(ctx context.Context, address oid.Address) (*obj return r.Object(), nil } +type RemoteRequestParams struct { + Epoch uint64 + TTL uint32 + PrivateKey *ecdsa.PrivateKey + SessionToken *session.Object + BearerToken *bearer.Token + XHeaders []string + IsRaw bool +} + +type remoteStorage interface { + Get(ctx context.Context, address oid.Address, requestParams RemoteRequestParams) (*objectSDK.Object, error) + Head(ctx context.Context, address oid.Address, requestParams RemoteRequestParams) (*objectSDK.Object, error) + Range(ctx context.Context, address oid.Address, rng *objectSDK.Range, requestParams RemoteRequestParams) (*objectSDK.Object, error) + + ForwardRequest(ctx context.Context, info coreclient.NodeInfo, forwarder RequestForwarder) (*objectSDK.Object, error) +} + type multiaddressRemoteStorage struct { client coreclient.MultiAddressClient } -func (s *multiaddressRemoteStorage) GetObject(ctx context.Context, exec *execCtx, info coreclient.NodeInfo) (*objectSDK.Object, error) { - if exec.isForwardingEnabled() { - return exec.prm.forwarder(ctx, info, s.client) - } - - key, err := exec.key() - if err != nil { - return nil, err - } - - if exec.headOnly() { - return s.getHeadOnly(ctx, exec, key) - } - // we don't specify payload writer because we accumulate - // the object locally (even huge). - if rng := exec.ctxRange(); rng != nil { - // Current spec allows other storage node to deny access, - // fallback to GET here. - return s.getRange(ctx, exec, key, rng) - } - - return s.get(ctx, exec, key) +func (s *multiaddressRemoteStorage) ForwardRequest(ctx context.Context, info coreclient.NodeInfo, forwarder RequestForwarder) (*objectSDK.Object, error) { + return forwarder(ctx, info, s.client) } -func (s *multiaddressRemoteStorage) getRange(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey, rng *object.Range) (*object.Object, error) { +func (s *multiaddressRemoteStorage) Range(ctx context.Context, address oid.Address, rng *objectSDK.Range, requestParams RemoteRequestParams) (*objectSDK.Object, error) { var prm internalclient.PayloadRangePrm prm.SetClient(s.client) - prm.SetTTL(exec.prm.common.TTL()) - prm.SetNetmapEpoch(exec.curProcEpoch) - prm.SetAddress(exec.address()) - prm.SetPrivateKey(key) - prm.SetSessionToken(exec.prm.common.SessionToken()) - prm.SetBearerToken(exec.prm.common.BearerToken()) - prm.SetXHeaders(exec.prm.common.XHeaders()) + prm.SetTTL(requestParams.TTL) + prm.SetNetmapEpoch(requestParams.Epoch) + prm.SetAddress(address) + prm.SetPrivateKey(requestParams.PrivateKey) + prm.SetSessionToken(requestParams.SessionToken) + prm.SetBearerToken(requestParams.BearerToken) + prm.SetXHeaders(requestParams.XHeaders) prm.SetRange(rng) - - if exec.isRaw() { + if requestParams.IsRaw { prm.SetRawFlag() } @@ -160,7 +153,7 @@ func (s *multiaddressRemoteStorage) getRange(ctx context.Context, exec *execCtx, if err != nil { var errAccessDenied *apistatus.ObjectAccessDenied if errors.As(err, &errAccessDenied) { - obj, err := s.get(ctx, exec, key) + obj, err := s.Get(ctx, address, requestParams) if err != nil { return nil, err } @@ -173,27 +166,27 @@ func (s *multiaddressRemoteStorage) getRange(ctx context.Context, exec *execCtx, return nil, new(apistatus.ObjectOutOfRange) } - return payloadOnlyObject(payload[from:to]), nil + return s.payloadOnlyObject(payload[from:to]), nil } return nil, err } - return payloadOnlyObject(res.PayloadRange()), nil + return s.payloadOnlyObject(res.PayloadRange()), nil } -func (s *multiaddressRemoteStorage) getHeadOnly(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { +func (s *multiaddressRemoteStorage) Head(ctx context.Context, address oid.Address, requestParams RemoteRequestParams) (*objectSDK.Object, error) { var prm internalclient.HeadObjectPrm prm.SetClient(s.client) - prm.SetTTL(exec.prm.common.TTL()) - prm.SetNetmapEpoch(exec.curProcEpoch) - prm.SetAddress(exec.address()) - prm.SetPrivateKey(key) - prm.SetSessionToken(exec.prm.common.SessionToken()) - prm.SetBearerToken(exec.prm.common.BearerToken()) - prm.SetXHeaders(exec.prm.common.XHeaders()) + prm.SetTTL(requestParams.TTL) + prm.SetNetmapEpoch(requestParams.Epoch) + prm.SetAddress(address) + prm.SetPrivateKey(requestParams.PrivateKey) + prm.SetSessionToken(requestParams.SessionToken) + prm.SetBearerToken(requestParams.BearerToken) + prm.SetXHeaders(requestParams.XHeaders) - if exec.isRaw() { + if requestParams.IsRaw { prm.SetRawFlag() } @@ -205,19 +198,19 @@ func (s *multiaddressRemoteStorage) getHeadOnly(ctx context.Context, exec *execC return res.Header(), nil } -func (s *multiaddressRemoteStorage) get(ctx context.Context, exec *execCtx, key *ecdsa.PrivateKey) (*object.Object, error) { +func (s *multiaddressRemoteStorage) Get(ctx context.Context, address oid.Address, requestParams RemoteRequestParams) (*objectSDK.Object, error) { var prm internalclient.GetObjectPrm prm.SetClient(s.client) - prm.SetTTL(exec.prm.common.TTL()) - prm.SetNetmapEpoch(exec.curProcEpoch) - prm.SetAddress(exec.address()) - prm.SetPrivateKey(key) - prm.SetSessionToken(exec.prm.common.SessionToken()) - prm.SetBearerToken(exec.prm.common.BearerToken()) - prm.SetXHeaders(exec.prm.common.XHeaders()) + prm.SetTTL(requestParams.TTL) + prm.SetNetmapEpoch(requestParams.Epoch) + prm.SetAddress(address) + prm.SetPrivateKey(requestParams.PrivateKey) + prm.SetSessionToken(requestParams.SessionToken) + prm.SetBearerToken(requestParams.BearerToken) + prm.SetXHeaders(requestParams.XHeaders) - if exec.isRaw() { + if requestParams.IsRaw { prm.SetRawFlag() } @@ -228,3 +221,10 @@ func (s *multiaddressRemoteStorage) get(ctx context.Context, exec *execCtx, key return res.Object(), nil } + +func (s *multiaddressRemoteStorage) payloadOnlyObject(payload []byte) *objectSDK.Object { + obj := objectSDK.New() + obj.SetPayload(payload) + + return obj +} diff --git a/pkg/services/object/get/util.go b/pkg/services/object/get/util.go index 3038d0d5e..2e01809d3 100644 --- a/pkg/services/object/get/util.go +++ b/pkg/services/object/get/util.go @@ -60,13 +60,6 @@ func (w *partWriter) WriteHeader(ctx context.Context, o *object.Object) error { return w.headWriter.WriteHeader(ctx, o) } -func payloadOnlyObject(payload []byte) *object.Object { - obj := object.New() - obj.SetPayload(payload) - - return obj -} - func (h *hasherWrapper) WriteChunk(_ context.Context, p []byte) error { _, err := h.hash.Write(p) return err From 591c4e7d50e0897c92407729d5013c2f314a0861 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Apr 2023 10:33:12 +0300 Subject: [PATCH 0184/1943] [#277] getsvc: Move headOnly to request params Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assemble.go | 4 +-- pkg/services/object/get/exec.go | 26 ++++++------------- pkg/services/object/get/get.go | 40 +++++++++++++---------------- pkg/services/object/get/prm.go | 2 +- 4 files changed, 29 insertions(+), 43 deletions(-) diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index 4ca55188c..09063ca50 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -123,12 +123,12 @@ func (exec *execCtx) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Ra p := exec.prm p.common = p.common.WithLocalOnly(false) p.objWriter = w - p.SetRange(rng) + p.rng = rng p.addr.SetContainer(exec.containerID()) p.addr.SetObject(id) - if err := exec.svc.get(ctx, p.commonPrm, withPayloadRange(rng)); err != nil { + if err := exec.svc.get(ctx, p); err != nil { return nil, err } return w.Object(), nil diff --git a/pkg/services/object/get/exec.go b/pkg/services/object/get/exec.go index 6b8e0e997..e4283bc6c 100644 --- a/pkg/services/object/get/exec.go +++ b/pkg/services/object/get/exec.go @@ -20,10 +20,16 @@ type statusError struct { err error } +type RequestParameters struct { + commonPrm + head bool + rng *objectSDK.Range +} + type execCtx struct { svc *Service - prm RangePrm + prm RequestParameters statusError @@ -33,13 +39,9 @@ type execCtx struct { collectedObject *objectSDK.Object - head bool - curProcEpoch uint64 } -type execOption func(*execCtx) - const ( statusUndefined int = iota statusOK @@ -48,18 +50,6 @@ const ( statusOutOfRange ) -func headOnly() execOption { - return func(c *execCtx) { - c.head = true - } -} - -func withPayloadRange(r *objectSDK.Range) execOption { - return func(c *execCtx) { - c.prm.rng = r - } -} - func (exec *execCtx) setLogger(l *logger.Logger) { req := "GET" if exec.headOnly() { @@ -126,7 +116,7 @@ func (exec *execCtx) ctxRange() *objectSDK.Range { } func (exec *execCtx) headOnly() bool { - return exec.head + return exec.prm.head } func (exec *execCtx) netmapEpoch() uint64 { diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 4e65fb528..fe3429990 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -11,18 +11,18 @@ import ( // Get serves a request to get an object by address, and returns Streamer instance. func (s *Service) Get(ctx context.Context, prm Prm) error { - return s.get(ctx, prm.commonPrm) + return s.get(ctx, RequestParameters{ + commonPrm: prm.commonPrm, + }) } // GetRange serves a request to get an object by address, and returns Streamer instance. func (s *Service) GetRange(ctx context.Context, prm RangePrm) error { - return s.getRange(ctx, prm) + return s.get(ctx, RequestParameters{ + commonPrm: prm.commonPrm, + rng: prm.rng, + }) } - -func (s *Service) getRange(ctx context.Context, prm RangePrm) error { - return s.get(ctx, prm.commonPrm, withPayloadRange(prm.rng)) -} - func (s *Service) GetRangeHash(ctx context.Context, prm RangeHashPrm) (*RangeHashRes, error) { hashes := make([][]byte, 0, len(prm.rngs)) @@ -34,16 +34,15 @@ func (s *Service) GetRangeHash(ctx context.Context, prm RangeHashPrm) (*RangeHas // 1. Potential gains are insignificant when operating in the Internet given typical latencies and losses. // 2. Parallel solution is more complex in terms of code. // 3. TZ-hash is likely to be disabled in private installations. - rngPrm := RangePrm{ + reqPrm := RequestParameters{ commonPrm: prm.commonPrm, + rng: &rng, } - - rngPrm.SetRange(&rng) - rngPrm.SetChunkWriter(&hasherWrapper{ + reqPrm.SetChunkWriter(&hasherWrapper{ hash: util.NewSaltingWriter(h, prm.salt), }) - if err := s.getRange(ctx, rngPrm); err != nil { + if err := s.get(ctx, reqPrm); err != nil { return nil, err } @@ -60,22 +59,19 @@ func (s *Service) GetRangeHash(ctx context.Context, prm RangeHashPrm) (*RangeHas // Returns ErrNotFound if the header was not received for the call. // Returns SplitInfoError if object is virtual and raw flag is set. func (s *Service) Head(ctx context.Context, prm HeadPrm) error { - return s.get(ctx, prm.commonPrm, headOnly()) + return s.get(ctx, RequestParameters{ + head: true, + commonPrm: prm.commonPrm, + }) } -func (s *Service) get(ctx context.Context, prm commonPrm, opts ...execOption) error { +func (s *Service) get(ctx context.Context, prm RequestParameters) error { exec := &execCtx{ - svc: s, - prm: RangePrm{ - commonPrm: prm, - }, + svc: s, + prm: prm, infoSplit: object.NewSplitInfo(), } - for i := range opts { - opts[i](exec) - } - exec.setLogger(s.log) exec.execute(ctx) diff --git a/pkg/services/object/get/prm.go b/pkg/services/object/get/prm.go index 7a0f1e062..c7db415c6 100644 --- a/pkg/services/object/get/prm.go +++ b/pkg/services/object/get/prm.go @@ -107,7 +107,7 @@ func (p *Prm) SetObjectWriter(w ObjectWriter) { } // SetChunkWriter sets target component to write the object payload range. -func (p *RangePrm) SetChunkWriter(w ChunkWriter) { +func (p *commonPrm) SetChunkWriter(w ChunkWriter) { p.objWriter = &partWriter{ chunkWriter: w, } From 144045060615fe96cbbd1d652695e84770845d69 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Apr 2023 11:36:15 +0300 Subject: [PATCH 0185/1943] [#277] getsvc: Drop cyclic struct dependency Drop cyclic dependency between execCtx and Service. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assemble.go | 34 +++++++++++++++++++++-------- pkg/services/object/get/exec.go | 16 +++++++++----- pkg/services/object/get/get.go | 7 +++++- pkg/services/object/get/local.go | 6 ++--- pkg/services/object/get/prm.go | 2 +- 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index 09063ca50..0c323a6f3 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -97,18 +97,16 @@ func equalAddresses(a, b oid.Address) bool { } func (exec *execCtx) HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Object, error) { - p := exec.prm + w := NewSimpleObjectWriter() + + p := RequestParameters{} p.common = p.common.WithLocalOnly(false) p.addr.SetContainer(exec.containerID()) p.addr.SetObject(id) + p.head = true + p.SetHeaderWriter(w) - prm := HeadPrm{ - commonPrm: p.commonPrm, - } - - w := NewSimpleObjectWriter() - prm.SetHeaderWriter(w) - err := exec.svc.Head(ctx, prm) + err := exec.getDetached(ctx, p) if err != nil { return nil, err @@ -128,8 +126,26 @@ func (exec *execCtx) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Ra p.addr.SetContainer(exec.containerID()) p.addr.SetObject(id) - if err := exec.svc.get(ctx, p); err != nil { + if err := exec.getDetached(ctx, p); err != nil { return nil, err } return w.Object(), nil } + +func (exec *execCtx) getDetached(ctx context.Context, prm RequestParameters) error { + detachedExecutor := &execCtx{ + keyStore: exec.keyStore, + traverserGenerator: exec.traverserGenerator, + remoteStorageConstructor: exec.remoteStorageConstructor, + epochSource: exec.epochSource, + localStorage: exec.localStorage, + + prm: prm, + infoSplit: objectSDK.NewSplitInfo(), + log: exec.log, + } + + detachedExecutor.execute(ctx) + + return detachedExecutor.statusError.err +} diff --git a/pkg/services/object/get/exec.go b/pkg/services/object/get/exec.go index e4283bc6c..45bd601e1 100644 --- a/pkg/services/object/get/exec.go +++ b/pkg/services/object/get/exec.go @@ -27,8 +27,6 @@ type RequestParameters struct { } type execCtx struct { - svc *Service - prm RequestParameters statusError @@ -40,6 +38,12 @@ type execCtx struct { collectedObject *objectSDK.Object curProcEpoch uint64 + + keyStore keyStorage + epochSource epochSource + traverserGenerator traverserGenerator + remoteStorageConstructor remoteStorageConstructor + localStorage localStorage } const ( @@ -96,7 +100,7 @@ func (exec execCtx) key() (*ecdsa.PrivateKey, error) { } } - return exec.svc.keyStore.GetKey(sessionInfo) + return exec.keyStore.GetKey(sessionInfo) } func (exec *execCtx) canAssemble() bool { @@ -133,7 +137,7 @@ func (exec *execCtx) initEpoch() bool { return true } - e, err := exec.svc.epochSource.Epoch() + e, err := exec.epochSource.Epoch() switch { default: @@ -154,7 +158,7 @@ func (exec *execCtx) initEpoch() bool { func (exec *execCtx) generateTraverser(addr oid.Address) (*placement.Traverser, bool) { obj := addr.Object() - t, err := exec.svc.traverserGenerator.GenerateTraverser(addr.Container(), &obj, exec.curProcEpoch) + t, err := exec.traverserGenerator.GenerateTraverser(addr.Container(), &obj, exec.curProcEpoch) switch { default: @@ -172,7 +176,7 @@ func (exec *execCtx) generateTraverser(addr oid.Address) (*placement.Traverser, } func (exec execCtx) getRemoteStorage(info clientcore.NodeInfo) (remoteStorage, bool) { - rs, err := exec.svc.remoteStorageConstructor.Get(info) + rs, err := exec.remoteStorageConstructor.Get(info) if err != nil { exec.status = statusUndefined exec.err = err diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index fe3429990..a562d9a1d 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -67,7 +67,12 @@ func (s *Service) Head(ctx context.Context, prm HeadPrm) error { func (s *Service) get(ctx context.Context, prm RequestParameters) error { exec := &execCtx{ - svc: s, + keyStore: s.keyStore, + traverserGenerator: s.traverserGenerator, + remoteStorageConstructor: s.remoteStorageConstructor, + epochSource: s.epochSource, + localStorage: s.localStorage, + prm: prm, infoSplit: object.NewSplitInfo(), } diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index 97fda6ce7..135fc3cd5 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -52,10 +52,10 @@ func (exec *execCtx) executeLocal(ctx context.Context) { func (exec *execCtx) get(ctx context.Context) (*objectSDK.Object, error) { if exec.headOnly() { - return exec.svc.localStorage.Head(ctx, exec.address(), exec.isRaw()) + return exec.localStorage.Head(ctx, exec.address(), exec.isRaw()) } if rng := exec.ctxRange(); rng != nil { - return exec.svc.localStorage.Range(ctx, exec.address(), rng) + return exec.localStorage.Range(ctx, exec.address(), rng) } - return exec.svc.localStorage.Get(ctx, exec.address()) + return exec.localStorage.Get(ctx, exec.address()) } diff --git a/pkg/services/object/get/prm.go b/pkg/services/object/get/prm.go index c7db415c6..7278eb2e9 100644 --- a/pkg/services/object/get/prm.go +++ b/pkg/services/object/get/prm.go @@ -158,7 +158,7 @@ func (p *commonPrm) WithCachedSignerKey(signerKey *ecdsa.PrivateKey) { } // SetHeaderWriter sets target component to write the object header. -func (p *HeadPrm) SetHeaderWriter(w HeaderWriter) { +func (p *commonPrm) SetHeaderWriter(w HeaderWriter) { p.objWriter = &partWriter{ headWriter: w, } From 45438e7b060b1504d95d7e9b104f940b2ed6aae7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Apr 2023 12:09:43 +0300 Subject: [PATCH 0186/1943] [#277] getsvc: Rename and reorder code Rename execCtx to request. Move code to appropriate files. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assemble.go | 102 ++++--- pkg/services/object/get/assembler.go | 7 +- pkg/services/object/get/container.go | 34 +-- pkg/services/object/get/errors.go | 10 + pkg/services/object/get/exec.go | 271 ------------------ pkg/services/object/get/get.go | 6 +- pkg/services/object/get/get_test.go | 4 +- pkg/services/object/get/local.go | 42 +-- pkg/services/object/get/prm.go | 42 +-- pkg/services/object/get/remote.go | 68 ++--- pkg/services/object/get/request.go | 252 ++++++++++++++++ pkg/services/object/get/res.go | 9 - pkg/services/object/get/status.go | 14 + pkg/services/object/get/types.go | 8 + .../object/get/{util.go => writer.go} | 18 ++ 15 files changed, 442 insertions(+), 445 deletions(-) create mode 100644 pkg/services/object/get/errors.go delete mode 100644 pkg/services/object/get/exec.go create mode 100644 pkg/services/object/get/request.go delete mode 100644 pkg/services/object/get/res.go create mode 100644 pkg/services/object/get/status.go rename pkg/services/object/get/{util.go => writer.go} (72%) diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index 0c323a6f3..777822ac3 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -11,9 +11,9 @@ import ( "go.uber.org/zap" ) -func (exec *execCtx) assemble(ctx context.Context) { - if !exec.canAssemble() { - exec.log.Debug(logs.GetCanNotAssembleTheObject) +func (r *request) assemble(ctx context.Context) { + if !r.canAssemble() { + r.log.Debug(logs.GetCanNotAssembleTheObject) return } @@ -28,35 +28,35 @@ func (exec *execCtx) assemble(ctx context.Context) { // - the assembly process is expected to be handled on a container node // only since the requests forwarding mechanism presentation; such the // node should have enough rights for getting any child object by design. - exec.prm.common.ForgetTokens() + r.prm.common.ForgetTokens() // Do not use forwarding during assembly stage. // Request forwarding closure inherited in produced // `execCtx` so it should be disabled there. - exec.disableForwarding() + r.disableForwarding() - exec.log.Debug(logs.GetTryingToAssembleTheObject) + r.log.Debug(logs.GetTryingToAssembleTheObject) - assembler := newAssembler(exec.address(), exec.splitInfo(), exec.ctxRange(), exec) + assembler := newAssembler(r.address(), r.splitInfo(), r.ctxRange(), r) - exec.log.Debug(logs.GetAssemblingSplittedObject, - zap.Stringer("address", exec.address()), - zap.Uint64("range_offset", exec.ctxRange().GetOffset()), - zap.Uint64("range_length", exec.ctxRange().GetLength()), + r.log.Debug(logs.GetAssemblingSplittedObject, + zap.Stringer("address", r.address()), + zap.Uint64("range_offset", r.ctxRange().GetOffset()), + zap.Uint64("range_length", r.ctxRange().GetLength()), ) - defer exec.log.Debug(logs.GetAssemblingSplittedObjectCompleted, - zap.Stringer("address", exec.address()), - zap.Uint64("range_offset", exec.ctxRange().GetOffset()), - zap.Uint64("range_length", exec.ctxRange().GetLength()), + defer r.log.Debug(logs.GetAssemblingSplittedObjectCompleted, + zap.Stringer("address", r.address()), + zap.Uint64("range_offset", r.ctxRange().GetOffset()), + zap.Uint64("range_length", r.ctxRange().GetLength()), ) - obj, err := assembler.Assemble(ctx, exec.prm.objWriter) + obj, err := assembler.Assemble(ctx, r.prm.objWriter) if err != nil { - exec.log.Warn(logs.GetFailedToAssembleSplittedObject, + r.log.Warn(logs.GetFailedToAssembleSplittedObject, zap.Error(err), - zap.Stringer("address", exec.address()), - zap.Uint64("range_offset", exec.ctxRange().GetOffset()), - zap.Uint64("range_length", exec.ctxRange().GetLength()), + zap.Stringer("address", r.address()), + zap.Uint64("range_offset", r.ctxRange().GetOffset()), + zap.Uint64("range_length", r.ctxRange().GetLength()), ) } @@ -68,27 +68,27 @@ func (exec *execCtx) assemble(ctx context.Context) { switch { default: - exec.status = statusUndefined - exec.err = err + r.status = statusUndefined + r.err = err case err == nil: - exec.status = statusOK - exec.err = nil - exec.collectedObject = obj + r.status = statusOK + r.err = nil + r.collectedObject = obj case errors.As(err, &errRemovedRemote): - exec.status = statusINHUMED - exec.err = errRemovedRemote + r.status = statusINHUMED + r.err = errRemovedRemote case errors.As(err, &errRemovedLocal): - exec.status = statusINHUMED - exec.err = errRemovedLocal + r.status = statusINHUMED + r.err = errRemovedLocal case errors.As(err, &errSplitInfo): - exec.status = statusVIRTUAL - exec.err = errSplitInfo + r.status = statusVIRTUAL + r.err = errSplitInfo case errors.As(err, &errOutOfRangeRemote): - exec.status = statusOutOfRange - exec.err = errOutOfRangeRemote + r.status = statusOutOfRange + r.err = errOutOfRangeRemote case errors.As(err, &errOutOfRangeLocal): - exec.status = statusOutOfRange - exec.err = errOutOfRangeLocal + r.status = statusOutOfRange + r.err = errOutOfRangeLocal } } @@ -96,53 +96,51 @@ func equalAddresses(a, b oid.Address) bool { return a.Container().Equals(b.Container()) && a.Object().Equals(b.Object()) } -func (exec *execCtx) HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Object, error) { +func (r *request) HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Object, error) { w := NewSimpleObjectWriter() p := RequestParameters{} p.common = p.common.WithLocalOnly(false) - p.addr.SetContainer(exec.containerID()) + p.addr.SetContainer(r.containerID()) p.addr.SetObject(id) p.head = true p.SetHeaderWriter(w) - err := exec.getDetached(ctx, p) - - if err != nil { + if err := r.getObjectWithIndependentRequest(ctx, p); err != nil { return nil, err } return w.Object(), nil } -func (exec *execCtx) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range) (*objectSDK.Object, error) { +func (r *request) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range) (*objectSDK.Object, error) { w := NewSimpleObjectWriter() - p := exec.prm + p := r.prm p.common = p.common.WithLocalOnly(false) p.objWriter = w p.rng = rng - p.addr.SetContainer(exec.containerID()) + p.addr.SetContainer(r.containerID()) p.addr.SetObject(id) - if err := exec.getDetached(ctx, p); err != nil { + if err := r.getObjectWithIndependentRequest(ctx, p); err != nil { return nil, err } return w.Object(), nil } -func (exec *execCtx) getDetached(ctx context.Context, prm RequestParameters) error { - detachedExecutor := &execCtx{ - keyStore: exec.keyStore, - traverserGenerator: exec.traverserGenerator, - remoteStorageConstructor: exec.remoteStorageConstructor, - epochSource: exec.epochSource, - localStorage: exec.localStorage, +func (r *request) getObjectWithIndependentRequest(ctx context.Context, prm RequestParameters) error { + detachedExecutor := &request{ + keyStore: r.keyStore, + traverserGenerator: r.traverserGenerator, + remoteStorageConstructor: r.remoteStorageConstructor, + epochSource: r.epochSource, + localStorage: r.localStorage, prm: prm, infoSplit: objectSDK.NewSplitInfo(), - log: exec.log, + log: r.log, } detachedExecutor.execute(ctx) diff --git a/pkg/services/object/get/assembler.go b/pkg/services/object/get/assembler.go index 7a3023c74..321d4b16d 100644 --- a/pkg/services/object/get/assembler.go +++ b/pkg/services/object/get/assembler.go @@ -2,7 +2,6 @@ package getsvc import ( "context" - "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -15,10 +14,6 @@ type objectGetter interface { HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Object, error) } -var ( - errParentAddressDiffers = errors.New("parent address in child object differs") -) - type assembler struct { addr oid.Address splitInfo *objectSDK.SplitInfo @@ -89,7 +84,7 @@ func (a *assembler) initializeFromSourceObjectID(ctx context.Context, id oid.ID) parentObject := sourceObject.Parent() if parentObject == nil { - return nil, nil, errors.New("received child with empty parent") + return nil, nil, errChildWithEmptyParent } a.parentObject = parentObject diff --git a/pkg/services/object/get/container.go b/pkg/services/object/get/container.go index 17628e577..689d1a16d 100644 --- a/pkg/services/object/get/container.go +++ b/pkg/services/object/get/container.go @@ -8,26 +8,26 @@ import ( "go.uber.org/zap" ) -func (exec *execCtx) executeOnContainer(ctx context.Context) { - if exec.isLocal() { - exec.log.Debug(logs.GetReturnResultDirectly) +func (r *request) executeOnContainer(ctx context.Context) { + if r.isLocal() { + r.log.Debug(logs.GetReturnResultDirectly) return } - lookupDepth := exec.netmapLookupDepth() + lookupDepth := r.netmapLookupDepth() - exec.log.Debug(logs.TryingToExecuteInContainer, + r.log.Debug(logs.TryingToExecuteInContainer, zap.Uint64("netmap lookup depth", lookupDepth), ) // initialize epoch number - ok := exec.initEpoch() + ok := r.initEpoch() if !ok { return } for { - if exec.processCurrentEpoch(ctx) { + if r.processCurrentEpoch(ctx) { break } @@ -39,16 +39,16 @@ func (exec *execCtx) executeOnContainer(ctx context.Context) { lookupDepth-- // go to the previous epoch - exec.curProcEpoch-- + r.curProcEpoch-- } } -func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { - exec.log.Debug(logs.ProcessEpoch, - zap.Uint64("number", exec.curProcEpoch), +func (r *request) processCurrentEpoch(ctx context.Context) bool { + r.log.Debug(logs.ProcessEpoch, + zap.Uint64("number", r.curProcEpoch), ) - traverser, ok := exec.generateTraverser(exec.address()) + traverser, ok := r.generateTraverser(r.address()) if !ok { return true } @@ -56,12 +56,12 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { ctx, cancel := context.WithCancel(ctx) defer cancel() - exec.status = statusUndefined + r.status = statusUndefined for { addrs := traverser.Next() if len(addrs) == 0 { - exec.log.Debug(logs.NoMoreNodesAbortPlacementIteration) + r.log.Debug(logs.NoMoreNodesAbortPlacementIteration) return false } @@ -69,7 +69,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { for i := range addrs { select { case <-ctx.Done(): - exec.log.Debug(logs.InterruptPlacementIterationByContext, + r.log.Debug(logs.InterruptPlacementIterationByContext, zap.String("error", ctx.Err().Error()), ) @@ -84,8 +84,8 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { client.NodeInfoFromNetmapElement(&info, addrs[i]) - if exec.processNode(ctx, info) { - exec.log.Debug(logs.GetCompletingTheOperation) + if r.processNode(ctx, info) { + r.log.Debug(logs.GetCompletingTheOperation) return true } } diff --git a/pkg/services/object/get/errors.go b/pkg/services/object/get/errors.go new file mode 100644 index 000000000..6ea16a144 --- /dev/null +++ b/pkg/services/object/get/errors.go @@ -0,0 +1,10 @@ +package getsvc + +import "errors" + +var ( + errRangeZeroLength = errors.New("zero range length") + errRangeOverflow = errors.New("range overflow") + errChildWithEmptyParent = errors.New("received child with empty parent") + errParentAddressDiffers = errors.New("parent address in child object differs") +) diff --git a/pkg/services/object/get/exec.go b/pkg/services/object/get/exec.go deleted file mode 100644 index 45bd601e1..000000000 --- a/pkg/services/object/get/exec.go +++ /dev/null @@ -1,271 +0,0 @@ -package getsvc - -import ( - "context" - "crypto/ecdsa" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.uber.org/zap" -) - -type statusError struct { - status int - err error -} - -type RequestParameters struct { - commonPrm - head bool - rng *objectSDK.Range -} - -type execCtx struct { - prm RequestParameters - - statusError - - infoSplit *objectSDK.SplitInfo - - log *logger.Logger - - collectedObject *objectSDK.Object - - curProcEpoch uint64 - - keyStore keyStorage - epochSource epochSource - traverserGenerator traverserGenerator - remoteStorageConstructor remoteStorageConstructor - localStorage localStorage -} - -const ( - statusUndefined int = iota - statusOK - statusINHUMED - statusVIRTUAL - statusOutOfRange -) - -func (exec *execCtx) setLogger(l *logger.Logger) { - req := "GET" - if exec.headOnly() { - req = "HEAD" - } else if exec.ctxRange() != nil { - req = "GET_RANGE" - } - - exec.log = &logger.Logger{Logger: l.With( - zap.String("request", req), - zap.Stringer("address", exec.address()), - zap.Bool("raw", exec.isRaw()), - zap.Bool("local", exec.isLocal()), - zap.Bool("with session", exec.prm.common.SessionToken() != nil), - zap.Bool("with bearer", exec.prm.common.BearerToken() != nil), - )} -} - -func (exec execCtx) isLocal() bool { - return exec.prm.common.LocalOnly() -} - -func (exec execCtx) isRaw() bool { - return exec.prm.raw -} - -func (exec execCtx) address() oid.Address { - return exec.prm.addr -} - -func (exec execCtx) key() (*ecdsa.PrivateKey, error) { - if exec.prm.signerKey != nil { - // the key has already been requested and - // cached in the previous operations - return exec.prm.signerKey, nil - } - - var sessionInfo *util.SessionInfo - - if tok := exec.prm.common.SessionToken(); tok != nil { - sessionInfo = &util.SessionInfo{ - ID: tok.ID(), - Owner: tok.Issuer(), - } - } - - return exec.keyStore.GetKey(sessionInfo) -} - -func (exec *execCtx) canAssemble() bool { - return !exec.isRaw() && !exec.headOnly() -} - -func (exec *execCtx) splitInfo() *objectSDK.SplitInfo { - return exec.infoSplit -} - -func (exec *execCtx) containerID() cid.ID { - return exec.address().Container() -} - -func (exec *execCtx) ctxRange() *objectSDK.Range { - return exec.prm.rng -} - -func (exec *execCtx) headOnly() bool { - return exec.prm.head -} - -func (exec *execCtx) netmapEpoch() uint64 { - return exec.prm.common.NetmapEpoch() -} - -func (exec *execCtx) netmapLookupDepth() uint64 { - return exec.prm.common.NetmapLookupDepth() -} - -func (exec *execCtx) initEpoch() bool { - exec.curProcEpoch = exec.netmapEpoch() - if exec.curProcEpoch > 0 { - return true - } - - e, err := exec.epochSource.Epoch() - - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.CouldNotGetCurrentEpochNumber, - zap.String("error", err.Error()), - ) - - return false - case err == nil: - exec.curProcEpoch = e - return true - } -} - -func (exec *execCtx) generateTraverser(addr oid.Address) (*placement.Traverser, bool) { - obj := addr.Object() - - t, err := exec.traverserGenerator.GenerateTraverser(addr.Container(), &obj, exec.curProcEpoch) - - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.GetCouldNotGenerateContainerTraverser, - zap.String("error", err.Error()), - ) - - return nil, false - case err == nil: - return t, true - } -} - -func (exec execCtx) getRemoteStorage(info clientcore.NodeInfo) (remoteStorage, bool) { - rs, err := exec.remoteStorageConstructor.Get(info) - if err != nil { - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.GetCouldNotConstructRemoteNodeClient) - - return nil, false - } - - return rs, true -} - -func mergeSplitInfo(dst, src *objectSDK.SplitInfo) { - if last, ok := src.LastPart(); ok { - dst.SetLastPart(last) - } - - if link, ok := src.Link(); ok { - dst.SetLink(link) - } - - if splitID := src.SplitID(); splitID != nil { - dst.SetSplitID(splitID) - } -} - -func (exec *execCtx) writeCollectedHeader(ctx context.Context) bool { - if exec.ctxRange() != nil { - return true - } - - err := exec.prm.objWriter.WriteHeader( - ctx, - exec.collectedObject.CutPayload(), - ) - - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.GetCouldNotWriteHeader, - zap.String("error", err.Error()), - ) - case err == nil: - exec.status = statusOK - exec.err = nil - } - - return exec.status == statusOK -} - -func (exec *execCtx) writeObjectPayload(ctx context.Context, obj *objectSDK.Object) bool { - if exec.headOnly() { - return true - } - - err := exec.prm.objWriter.WriteChunk(ctx, obj.Payload()) - - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.GetCouldNotWritePayloadChunk, - zap.String("error", err.Error()), - ) - case err == nil: - exec.status = statusOK - exec.err = nil - } - - return err == nil -} - -func (exec *execCtx) writeCollectedObject(ctx context.Context) { - if ok := exec.writeCollectedHeader(ctx); ok { - exec.writeObjectPayload(ctx, exec.collectedObject) - } -} - -// isForwardingEnabled returns true if common execution -// parameters has request forwarding closure set. -func (exec execCtx) isForwardingEnabled() bool { - return exec.prm.forwarder != nil -} - -// disableForwarding removes request forwarding closure from common -// parameters, so it won't be inherited in new execution contexts. -func (exec *execCtx) disableForwarding() { - exec.prm.SetRequestForwarder(nil) -} diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index a562d9a1d..2d24456d7 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -66,7 +66,7 @@ func (s *Service) Head(ctx context.Context, prm HeadPrm) error { } func (s *Service) get(ctx context.Context, prm RequestParameters) error { - exec := &execCtx{ + exec := &request{ keyStore: s.keyStore, traverserGenerator: s.traverserGenerator, remoteStorageConstructor: s.remoteStorageConstructor, @@ -84,7 +84,7 @@ func (s *Service) get(ctx context.Context, prm RequestParameters) error { return exec.statusError.err } -func (exec *execCtx) execute(ctx context.Context) { +func (exec *request) execute(ctx context.Context) { exec.log.Debug(logs.ServingRequest) // perform local operation @@ -93,7 +93,7 @@ func (exec *execCtx) execute(ctx context.Context) { exec.analyzeStatus(ctx, true) } -func (exec *execCtx) analyzeStatus(ctx context.Context, execCnr bool) { +func (exec *request) analyzeStatus(ctx context.Context, execCnr bool) { // analyze local result switch exec.status { case statusOK: diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index d6ff00880..9c5506064 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -1206,7 +1206,7 @@ func TestGetRemoteSmall(t *testing.T) { err := svc.Get(ctx, p) require.Error(t, err) - require.Equal(t, err.Error(), "received child with empty parent") + require.ErrorIs(t, err, errChildWithEmptyParent) w = NewSimpleObjectWriter() payloadSz := srcObj.PayloadSize() @@ -1219,7 +1219,7 @@ func TestGetRemoteSmall(t *testing.T) { err = svc.GetRange(ctx, rngPrm) require.Error(t, err) - require.Equal(t, err.Error(), "received child with empty parent") + require.ErrorIs(t, err, errChildWithEmptyParent) }) t.Run("out of range", func(t *testing.T) { diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index 135fc3cd5..8affa71c2 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -11,7 +11,7 @@ import ( "go.uber.org/zap" ) -func (exec *execCtx) executeLocal(ctx context.Context) { +func (r *request) executeLocal(ctx context.Context) { ctx, span := tracing.StartSpanFromContext(ctx, "getService.executeLocal") defer func() { span.End() @@ -19,7 +19,7 @@ func (exec *execCtx) executeLocal(ctx context.Context) { var err error - exec.collectedObject, err = exec.get(ctx) + r.collectedObject, err = r.get(ctx) var errSplitInfo *objectSDK.SplitInfoError var errRemoved apistatus.ObjectAlreadyRemoved @@ -27,35 +27,35 @@ func (exec *execCtx) executeLocal(ctx context.Context) { switch { default: - exec.status = statusUndefined - exec.err = err + r.status = statusUndefined + r.err = err - exec.log.Debug(logs.GetLocalGetFailed, + r.log.Debug(logs.GetLocalGetFailed, zap.String("error", err.Error()), ) case err == nil: - exec.status = statusOK - exec.err = nil - exec.writeCollectedObject(ctx) + r.status = statusOK + r.err = nil + r.writeCollectedObject(ctx) case errors.As(err, &errRemoved): - exec.status = statusINHUMED - exec.err = errRemoved + r.status = statusINHUMED + r.err = errRemoved case errors.As(err, &errSplitInfo): - exec.status = statusVIRTUAL - mergeSplitInfo(exec.splitInfo(), errSplitInfo.SplitInfo()) - exec.err = objectSDK.NewSplitInfoError(exec.infoSplit) + r.status = statusVIRTUAL + mergeSplitInfo(r.splitInfo(), errSplitInfo.SplitInfo()) + r.err = objectSDK.NewSplitInfoError(r.infoSplit) case errors.As(err, &errOutOfRange): - exec.status = statusOutOfRange - exec.err = errOutOfRange + r.status = statusOutOfRange + r.err = errOutOfRange } } -func (exec *execCtx) get(ctx context.Context) (*objectSDK.Object, error) { - if exec.headOnly() { - return exec.localStorage.Head(ctx, exec.address(), exec.isRaw()) +func (r *request) get(ctx context.Context) (*objectSDK.Object, error) { + if r.headOnly() { + return r.localStorage.Head(ctx, r.address(), r.isRaw()) } - if rng := exec.ctxRange(); rng != nil { - return exec.localStorage.Range(ctx, exec.address(), rng) + if rng := r.ctxRange(); rng != nil { + return r.localStorage.Range(ctx, r.address(), rng) } - return exec.localStorage.Get(ctx, exec.address()) + return r.localStorage.Get(ctx, r.address()) } diff --git a/pkg/services/object/get/prm.go b/pkg/services/object/get/prm.go index 7278eb2e9..cbdb7a3e2 100644 --- a/pkg/services/object/get/prm.go +++ b/pkg/services/object/get/prm.go @@ -3,12 +3,11 @@ package getsvc import ( "context" "crypto/ecdsa" - "errors" "hash" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -21,14 +20,9 @@ type Prm struct { type RangePrm struct { commonPrm - rng *object.Range + rng *objectSDK.Range } -var ( - errRangeZeroLength = errors.New("zero range length") - errRangeOverflow = errors.New("range overflow") -) - // Validate pre-validates `OBJECTRANGE` request's parameters content // without access to the requested object's payload. func (p RangePrm) Validate() error { @@ -54,12 +48,18 @@ type RangeHashPrm struct { hashGen func() hash.Hash - rngs []object.Range + rngs []objectSDK.Range salt []byte } -type RequestForwarder func(context.Context, coreclient.NodeInfo, coreclient.MultiAddressClient) (*object.Object, error) +type RequestParameters struct { + commonPrm + head bool + rng *objectSDK.Range +} + +type RequestForwarder func(context.Context, coreclient.NodeInfo, coreclient.MultiAddressClient) (*objectSDK.Object, error) // HeadPrm groups parameters of Head service call. type HeadPrm struct { @@ -83,24 +83,6 @@ type commonPrm struct { signerKey *ecdsa.PrivateKey } -// ChunkWriter is an interface of target component -// to write payload chunk. -type ChunkWriter interface { - WriteChunk(context.Context, []byte) error -} - -// HeaderWriter is an interface of target component -// to write object header. -type HeaderWriter interface { - WriteHeader(context.Context, *object.Object) error -} - -// ObjectWriter is an interface of target component to write object. -type ObjectWriter interface { - HeaderWriter - ChunkWriter -} - // SetObjectWriter sets target component to write the object. func (p *Prm) SetObjectWriter(w ObjectWriter) { p.objWriter = w @@ -114,12 +96,12 @@ func (p *commonPrm) SetChunkWriter(w ChunkWriter) { } // SetRange sets range of the requested payload data. -func (p *RangePrm) SetRange(rng *object.Range) { +func (p *RangePrm) SetRange(rng *objectSDK.Range) { p.rng = rng } // SetRangeList sets a list of object payload ranges. -func (p *RangeHashPrm) SetRangeList(rngs []object.Range) { +func (p *RangeHashPrm) SetRangeList(rngs []objectSDK.Range) { p.rngs = rngs } diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index d4c5d7a6c..ac662b3bb 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -12,18 +12,18 @@ import ( "go.uber.org/zap" ) -func (exec *execCtx) processNode(ctx context.Context, info client.NodeInfo) bool { +func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { ctx, span := tracing.StartSpanFromContext(ctx, "getService.processNode") defer span.End() - exec.log.Debug(logs.ProcessingNode) + r.log.Debug(logs.ProcessingNode) - rs, ok := exec.getRemoteStorage(info) + rs, ok := r.getRemoteStorage(info) if !ok { return true } - obj, err := exec.getRemote(ctx, rs, info) + obj, err := r.getRemote(ctx, rs, info) var errSplitInfo *objectSDK.SplitInfoError var errRemoved *apistatus.ObjectAlreadyRemoved @@ -33,68 +33,68 @@ func (exec *execCtx) processNode(ctx context.Context, info client.NodeInfo) bool default: var errNotFound apistatus.ObjectNotFound - exec.status = statusUndefined - exec.err = errNotFound + r.status = statusUndefined + r.err = errNotFound - exec.log.Debug(logs.GetRemoteCallFailed, + r.log.Debug(logs.GetRemoteCallFailed, zap.String("error", err.Error()), ) case err == nil: - exec.status = statusOK - exec.err = nil + r.status = statusOK + r.err = nil // both object and err are nil only if the original // request was forwarded to another node and the object // has already been streamed to the requesting party if obj != nil { - exec.collectedObject = obj - exec.writeCollectedObject(ctx) + r.collectedObject = obj + r.writeCollectedObject(ctx) } case errors.As(err, &errRemoved): - exec.status = statusINHUMED - exec.err = errRemoved + r.status = statusINHUMED + r.err = errRemoved case errors.As(err, &errOutOfRange): - exec.status = statusOutOfRange - exec.err = errOutOfRange + r.status = statusOutOfRange + r.err = errOutOfRange case errors.As(err, &errSplitInfo): - exec.status = statusVIRTUAL - mergeSplitInfo(exec.splitInfo(), errSplitInfo.SplitInfo()) - exec.err = objectSDK.NewSplitInfoError(exec.infoSplit) + r.status = statusVIRTUAL + mergeSplitInfo(r.splitInfo(), errSplitInfo.SplitInfo()) + r.err = objectSDK.NewSplitInfoError(r.infoSplit) } - return exec.status != statusUndefined + return r.status != statusUndefined } -func (exec *execCtx) getRemote(ctx context.Context, rs remoteStorage, info client.NodeInfo) (*objectSDK.Object, error) { - if exec.isForwardingEnabled() { - return rs.ForwardRequest(ctx, info, exec.prm.forwarder) +func (r *request) getRemote(ctx context.Context, rs remoteStorage, info client.NodeInfo) (*objectSDK.Object, error) { + if r.isForwardingEnabled() { + return rs.ForwardRequest(ctx, info, r.prm.forwarder) } - key, err := exec.key() + key, err := r.key() if err != nil { return nil, err } prm := RemoteRequestParams{ - Epoch: exec.curProcEpoch, - TTL: exec.prm.common.TTL(), + Epoch: r.curProcEpoch, + TTL: r.prm.common.TTL(), PrivateKey: key, - SessionToken: exec.prm.common.SessionToken(), - BearerToken: exec.prm.common.BearerToken(), - XHeaders: exec.prm.common.XHeaders(), - IsRaw: exec.isRaw(), + SessionToken: r.prm.common.SessionToken(), + BearerToken: r.prm.common.BearerToken(), + XHeaders: r.prm.common.XHeaders(), + IsRaw: r.isRaw(), } - if exec.headOnly() { - return rs.Head(ctx, exec.address(), prm) + if r.headOnly() { + return rs.Head(ctx, r.address(), prm) } // we don't specify payload writer because we accumulate // the object locally (even huge). - if rng := exec.ctxRange(); rng != nil { + if rng := r.ctxRange(); rng != nil { // Current spec allows other storage node to deny access, // fallback to GET here. - return rs.Range(ctx, exec.address(), rng, prm) + return rs.Range(ctx, r.address(), rng, prm) } - return rs.Get(ctx, exec.address(), prm) + return rs.Get(ctx, r.address(), prm) } diff --git a/pkg/services/object/get/request.go b/pkg/services/object/get/request.go new file mode 100644 index 000000000..db77d619a --- /dev/null +++ b/pkg/services/object/get/request.go @@ -0,0 +1,252 @@ +package getsvc + +import ( + "context" + "crypto/ecdsa" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" +) + +type request struct { + prm RequestParameters + + statusError + + infoSplit *objectSDK.SplitInfo + + log *logger.Logger + + collectedObject *objectSDK.Object + + curProcEpoch uint64 + + keyStore keyStorage + epochSource epochSource + traverserGenerator traverserGenerator + remoteStorageConstructor remoteStorageConstructor + localStorage localStorage +} + +func (r *request) setLogger(l *logger.Logger) { + req := "GET" + if r.headOnly() { + req = "HEAD" + } else if r.ctxRange() != nil { + req = "GET_RANGE" + } + + r.log = &logger.Logger{Logger: l.With( + zap.String("request", req), + zap.Stringer("address", r.address()), + zap.Bool("raw", r.isRaw()), + zap.Bool("local", r.isLocal()), + zap.Bool("with session", r.prm.common.SessionToken() != nil), + zap.Bool("with bearer", r.prm.common.BearerToken() != nil), + )} +} + +func (r *request) isLocal() bool { + return r.prm.common.LocalOnly() +} + +func (r *request) isRaw() bool { + return r.prm.raw +} + +func (r *request) address() oid.Address { + return r.prm.addr +} + +func (r *request) key() (*ecdsa.PrivateKey, error) { + if r.prm.signerKey != nil { + // the key has already been requested and + // cached in the previous operations + return r.prm.signerKey, nil + } + + var sessionInfo *util.SessionInfo + + if tok := r.prm.common.SessionToken(); tok != nil { + sessionInfo = &util.SessionInfo{ + ID: tok.ID(), + Owner: tok.Issuer(), + } + } + + return r.keyStore.GetKey(sessionInfo) +} + +func (r *request) canAssemble() bool { + return !r.isRaw() && !r.headOnly() +} + +func (r *request) splitInfo() *objectSDK.SplitInfo { + return r.infoSplit +} + +func (r *request) containerID() cid.ID { + return r.address().Container() +} + +func (r *request) ctxRange() *objectSDK.Range { + return r.prm.rng +} + +func (r *request) headOnly() bool { + return r.prm.head +} + +func (r *request) netmapEpoch() uint64 { + return r.prm.common.NetmapEpoch() +} + +func (r *request) netmapLookupDepth() uint64 { + return r.prm.common.NetmapLookupDepth() +} + +func (r *request) initEpoch() bool { + r.curProcEpoch = r.netmapEpoch() + if r.curProcEpoch > 0 { + return true + } + + e, err := r.epochSource.Epoch() + + switch { + default: + r.status = statusUndefined + r.err = err + + r.log.Debug(logs.CouldNotGetCurrentEpochNumber, + zap.String("error", err.Error()), + ) + + return false + case err == nil: + r.curProcEpoch = e + return true + } +} + +func (r *request) generateTraverser(addr oid.Address) (*placement.Traverser, bool) { + obj := addr.Object() + + t, err := r.traverserGenerator.GenerateTraverser(addr.Container(), &obj, r.curProcEpoch) + + switch { + default: + r.status = statusUndefined + r.err = err + + r.log.Debug(logs.GetCouldNotGenerateContainerTraverser, + zap.String("error", err.Error()), + ) + + return nil, false + case err == nil: + return t, true + } +} + +func (r *request) getRemoteStorage(info clientcore.NodeInfo) (remoteStorage, bool) { + rs, err := r.remoteStorageConstructor.Get(info) + if err != nil { + r.status = statusUndefined + r.err = err + + r.log.Debug(logs.GetCouldNotConstructRemoteNodeClient) + + return nil, false + } + + return rs, true +} + +func (r *request) writeCollectedHeader(ctx context.Context) bool { + if r.ctxRange() != nil { + return true + } + + err := r.prm.objWriter.WriteHeader( + ctx, + r.collectedObject.CutPayload(), + ) + + switch { + default: + r.status = statusUndefined + r.err = err + + r.log.Debug(logs.GetCouldNotWriteHeader, + zap.String("error", err.Error()), + ) + case err == nil: + r.status = statusOK + r.err = nil + } + + return r.status == statusOK +} + +func (r *request) writeObjectPayload(ctx context.Context, obj *objectSDK.Object) bool { + if r.headOnly() { + return true + } + + err := r.prm.objWriter.WriteChunk(ctx, obj.Payload()) + + switch { + default: + r.status = statusUndefined + r.err = err + + r.log.Debug(logs.GetCouldNotWritePayloadChunk, + zap.String("error", err.Error()), + ) + case err == nil: + r.status = statusOK + r.err = nil + } + + return err == nil +} + +func (r *request) writeCollectedObject(ctx context.Context) { + if ok := r.writeCollectedHeader(ctx); ok { + r.writeObjectPayload(ctx, r.collectedObject) + } +} + +// isForwardingEnabled returns true if common execution +// parameters has request forwarding closure set. +func (r request) isForwardingEnabled() bool { + return r.prm.forwarder != nil +} + +// disableForwarding removes request forwarding closure from common +// parameters, so it won't be inherited in new execution contexts. +func (r *request) disableForwarding() { + r.prm.SetRequestForwarder(nil) +} + +func mergeSplitInfo(dst, src *objectSDK.SplitInfo) { + if last, ok := src.LastPart(); ok { + dst.SetLastPart(last) + } + + if link, ok := src.Link(); ok { + dst.SetLink(link) + } + + if splitID := src.SplitID(); splitID != nil { + dst.SetSplitID(splitID) + } +} diff --git a/pkg/services/object/get/res.go b/pkg/services/object/get/res.go deleted file mode 100644 index 75a5aaedd..000000000 --- a/pkg/services/object/get/res.go +++ /dev/null @@ -1,9 +0,0 @@ -package getsvc - -type RangeHashRes struct { - hashes [][]byte -} - -func (r *RangeHashRes) Hashes() [][]byte { - return r.hashes -} diff --git a/pkg/services/object/get/status.go b/pkg/services/object/get/status.go new file mode 100644 index 000000000..3a5eebe32 --- /dev/null +++ b/pkg/services/object/get/status.go @@ -0,0 +1,14 @@ +package getsvc + +const ( + statusUndefined int = iota + statusOK + statusINHUMED + statusVIRTUAL + statusOutOfRange +) + +type statusError struct { + status int + err error +} diff --git a/pkg/services/object/get/types.go b/pkg/services/object/get/types.go index 800d37996..a866132cc 100644 --- a/pkg/services/object/get/types.go +++ b/pkg/services/object/get/types.go @@ -228,3 +228,11 @@ func (s *multiaddressRemoteStorage) payloadOnlyObject(payload []byte) *objectSDK return obj } + +type RangeHashRes struct { + hashes [][]byte +} + +func (r *RangeHashRes) Hashes() [][]byte { + return r.hashes +} diff --git a/pkg/services/object/get/util.go b/pkg/services/object/get/writer.go similarity index 72% rename from pkg/services/object/get/util.go rename to pkg/services/object/get/writer.go index 2e01809d3..78af5db41 100644 --- a/pkg/services/object/get/util.go +++ b/pkg/services/object/get/writer.go @@ -7,6 +7,24 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) +// ChunkWriter is an interface of target component +// to write payload chunk. +type ChunkWriter interface { + WriteChunk(context.Context, []byte) error +} + +// HeaderWriter is an interface of target component +// to write object header. +type HeaderWriter interface { + WriteHeader(context.Context, *object.Object) error +} + +// ObjectWriter is an interface of target component to write object. +type ObjectWriter interface { + HeaderWriter + ChunkWriter +} + type SimpleObjectWriter struct { obj *object.Object From 22d47376a60b59b5503413d2f8e7d3745fa88fff Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Apr 2023 13:11:44 +0300 Subject: [PATCH 0187/1943] [#277] getsvc: Refactor errors Move errors to separate files. Use zap.Error for error logging. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/container.go | 2 +- pkg/services/object/get/get.go | 2 +- pkg/services/object/get/local.go | 4 +- pkg/services/object/get/remote.go | 4 +- pkg/services/object/get/request.go | 16 +--- pkg/services/object/get/v2/errors.go | 92 +++++++++++++++++++ pkg/services/object/get/v2/get_forwarder.go | 13 ++- .../object/get/v2/get_range_forwarder.go | 11 +-- pkg/services/object/get/v2/head_forwarder.go | 27 ++---- pkg/services/object/get/v2/util.go | 25 +++-- 10 files changed, 132 insertions(+), 64 deletions(-) create mode 100644 pkg/services/object/get/v2/errors.go diff --git a/pkg/services/object/get/container.go b/pkg/services/object/get/container.go index 689d1a16d..d22b14192 100644 --- a/pkg/services/object/get/container.go +++ b/pkg/services/object/get/container.go @@ -70,7 +70,7 @@ func (r *request) processCurrentEpoch(ctx context.Context) bool { select { case <-ctx.Done(): r.log.Debug(logs.InterruptPlacementIterationByContext, - zap.String("error", ctx.Err().Error()), + zap.Error(ctx.Err()), ) return true diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 2d24456d7..457193a59 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -107,7 +107,7 @@ func (exec *request) analyzeStatus(ctx context.Context, execCnr bool) { exec.log.Debug(logs.GetRequestedRangeIsOutOfObjectBounds) default: exec.log.Debug(logs.OperationFinishedWithError, - zap.String("error", exec.err.Error()), + zap.Error(exec.err), ) if execCnr { diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index 8affa71c2..62dde3281 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -30,9 +30,7 @@ func (r *request) executeLocal(ctx context.Context) { r.status = statusUndefined r.err = err - r.log.Debug(logs.GetLocalGetFailed, - zap.String("error", err.Error()), - ) + r.log.Debug(logs.GetLocalGetFailed, zap.Error(err)) case err == nil: r.status = statusOK r.err = nil diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index ac662b3bb..69bdbf271 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -36,9 +36,7 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { r.status = statusUndefined r.err = errNotFound - r.log.Debug(logs.GetRemoteCallFailed, - zap.String("error", err.Error()), - ) + r.log.Debug(logs.GetRemoteCallFailed, zap.Error(err)) case err == nil: r.status = statusOK r.err = nil diff --git a/pkg/services/object/get/request.go b/pkg/services/object/get/request.go index db77d619a..b9223a637 100644 --- a/pkg/services/object/get/request.go +++ b/pkg/services/object/get/request.go @@ -125,9 +125,7 @@ func (r *request) initEpoch() bool { r.status = statusUndefined r.err = err - r.log.Debug(logs.CouldNotGetCurrentEpochNumber, - zap.String("error", err.Error()), - ) + r.log.Debug(logs.CouldNotGetCurrentEpochNumber, zap.Error(err)) return false case err == nil: @@ -146,9 +144,7 @@ func (r *request) generateTraverser(addr oid.Address) (*placement.Traverser, boo r.status = statusUndefined r.err = err - r.log.Debug(logs.GetCouldNotGenerateContainerTraverser, - zap.String("error", err.Error()), - ) + r.log.Debug(logs.GetCouldNotGenerateContainerTraverser, zap.Error(err)) return nil, false case err == nil: @@ -185,9 +181,7 @@ func (r *request) writeCollectedHeader(ctx context.Context) bool { r.status = statusUndefined r.err = err - r.log.Debug(logs.GetCouldNotWriteHeader, - zap.String("error", err.Error()), - ) + r.log.Debug(logs.GetCouldNotWriteHeader, zap.Error(err)) case err == nil: r.status = statusOK r.err = nil @@ -208,9 +202,7 @@ func (r *request) writeObjectPayload(ctx context.Context, obj *objectSDK.Object) r.status = statusUndefined r.err = err - r.log.Debug(logs.GetCouldNotWritePayloadChunk, - zap.String("error", err.Error()), - ) + r.log.Debug(logs.GetCouldNotWritePayloadChunk, zap.Error(err)) case err == nil: r.status = statusOK r.err = nil diff --git a/pkg/services/object/get/v2/errors.go b/pkg/services/object/get/v2/errors.go new file mode 100644 index 000000000..01b57f1f2 --- /dev/null +++ b/pkg/services/object/get/v2/errors.go @@ -0,0 +1,92 @@ +package getsvc + +import ( + "errors" + "fmt" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + refs "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" +) + +var ( + errMissingObjAddress = errors.New("missing object address") + errWrongMessageSeq = errors.New("incorrect message sequence") + errNilObjectPart = errors.New("nil object part") + errMissingSignature = errors.New("missing signature") + errInvalidObjectIDSign = errors.New("invalid object ID signature") + + errWrongHeaderPartTypeExpShortRecvWithSignature = fmt.Errorf("wrong header part type: expected %T, received %T", + (*objectV2.ShortHeader)(nil), (*objectV2.HeaderWithSignature)(nil), + ) + errWrongHeaderPartTypeExpWithSignRecvShort = fmt.Errorf("wrong header part type: expected %T, received %T", + (*objectV2.HeaderWithSignature)(nil), (*objectV2.ShortHeader)(nil), + ) +) + +func errInvalidObjAddress(err error) error { + return fmt.Errorf("invalid object address: %w", err) +} + +func errRequestParamsValidation(err error) error { + return fmt.Errorf("request params validation: %w", err) +} + +func errFetchingSessionKey(err error) error { + return fmt.Errorf("fetching session key: %w", err) +} + +func errUnknownChechsumType(t refs.ChecksumType) error { + return fmt.Errorf("unknown checksum type %v", t) +} + +func errResponseVerificationFailed(err error) error { + return fmt.Errorf("response verification failed: %w", err) +} + +func errCouldNotWriteObjHeader(err error) error { + return fmt.Errorf("could not write object header in Get forwarder: %w", err) +} + +func errStreamOpenningFailed(err error) error { + return fmt.Errorf("stream opening failed: %w", err) +} + +func errReadingResponseFailed(err error) error { + return fmt.Errorf("reading the response failed: %w", err) +} + +func errUnexpectedObjectPart(v objectV2.GetObjectPart) error { + return fmt.Errorf("unexpected object part %T", v) +} + +func errCouldNotWriteObjChunk(forwarder string, err error) error { + return fmt.Errorf("could not write object chunk in %s forwarder: %w", forwarder, err) +} + +func errCouldNotVerifyRangeResponse(resp *objectV2.GetRangeResponse, err error) error { + return fmt.Errorf("could not verify %T: %w", resp, err) +} + +func errCouldNotCreateGetRangeStream(err error) error { + return fmt.Errorf("could not create Get payload range stream: %w", err) +} + +func errUnexpectedRangePart(v objectV2.GetRangePart) error { + return fmt.Errorf("unexpected range type %T", v) +} + +func errUnexpectedHeaderPart(v objectV2.GetHeaderPart) error { + return fmt.Errorf("unexpected header type %T", v) +} + +func errMarshalID(err error) error { + return fmt.Errorf("marshal ID: %w", err) +} + +func errCantReadSignature(err error) error { + return fmt.Errorf("can't read signature: %w", err) +} + +func errSendingRequestFailed(err error) error { + return fmt.Errorf("sending the request failed: %w", err) +} diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index b0de3a55b..d11f94b26 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -4,7 +4,6 @@ import ( "context" "crypto/ecdsa" "errors" - "fmt" "io" "sync" @@ -71,7 +70,7 @@ func (f *getRequestForwarder) verifyResponse(resp *objectV2.GetResponse, pubkey // verify response structure if err := signature.VerifyServiceMessage(resp); err != nil { - return fmt.Errorf("response verification failed: %w", err) + return errResponseVerificationFailed(err) } return checkStatus(resp.GetMetaHeader().GetStatus()) @@ -89,7 +88,7 @@ func (f *getRequestForwarder) writeHeader(ctx context.Context, v *objectV2.GetOb err = f.Stream.WriteHeader(ctx, object.NewFromV2(obj)) }) if err != nil { - return fmt.Errorf("could not write object header in Get forwarder: %w", err) + return errCouldNotWriteObjHeader(err) } return nil } @@ -102,7 +101,7 @@ func (f *getRequestForwarder) openStream(ctx context.Context, addr network.Addre return e }) if err != nil { - return nil, fmt.Errorf("stream opening failed: %w", err) + return nil, errStreamOpenningFailed(err) } return getStream, nil } @@ -127,7 +126,7 @@ func (f *getRequestForwarder) readStream(ctx context.Context, c client.MultiAddr } internalclient.ReportError(c, err) - return fmt.Errorf("reading the response failed: %w", err) + return errReadingResponseFailed(err) } if err := f.verifyResponse(resp, pubkey); err != nil { @@ -136,7 +135,7 @@ func (f *getRequestForwarder) readStream(ctx context.Context, c client.MultiAddr switch v := resp.GetBody().GetObjectPart().(type) { default: - return fmt.Errorf("unexpected object part %T", v) + return errUnexpectedObjectPart(v) case *objectV2.GetObjectPartInit: if headWas { return errWrongMessageSeq @@ -159,7 +158,7 @@ func (f *getRequestForwarder) readStream(ctx context.Context, c client.MultiAddr } if err = f.Stream.WriteChunk(ctx, chunk); err != nil { - return fmt.Errorf("could not write object chunk in Get forwarder: %w", err) + return errCouldNotWriteObjChunk("Get", err) } localProgress += len(origChunk) diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 91d52a593..6c744b23a 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -4,7 +4,6 @@ import ( "context" "crypto/ecdsa" "errors" - "fmt" "io" "sync" @@ -73,7 +72,7 @@ func (f *getRangeRequestForwarder) verifyResponse(resp *objectV2.GetRangeRespons // verify response structure if err := signature.VerifyServiceMessage(resp); err != nil { - return fmt.Errorf("could not verify %T: %w", resp, err) + return errCouldNotVerifyRangeResponse(resp, err) } return checkStatus(resp.GetMetaHeader().GetStatus()) @@ -88,7 +87,7 @@ func (f *getRangeRequestForwarder) openStream(ctx context.Context, addr network. return e }) if err != nil { - return nil, fmt.Errorf("could not create Get payload range stream: %w", err) + return nil, errCouldNotCreateGetRangeStream(err) } return rangeStream, nil } @@ -105,7 +104,7 @@ func (f *getRangeRequestForwarder) readStream(ctx context.Context, rangeStream * break } internalclient.ReportError(c, err) - return fmt.Errorf("reading the response failed: %w", err) + return errReadingResponseFailed(err) } if err := f.verifyResponse(resp, pubkey); err != nil { @@ -114,7 +113,7 @@ func (f *getRangeRequestForwarder) readStream(ctx context.Context, rangeStream * switch v := resp.GetBody().GetRangePart().(type) { case nil: - return fmt.Errorf("unexpected range type %T", v) + return errUnexpectedRangePart(v) case *objectV2.GetRangePartChunk: origChunk := v.GetChunk() @@ -125,7 +124,7 @@ func (f *getRangeRequestForwarder) readStream(ctx context.Context, rangeStream * } if err = f.Stream.WriteChunk(ctx, chunk); err != nil { - return fmt.Errorf("could not write object chunk in GetRange forwarder: %w", err) + return errCouldNotWriteObjChunk("GetRange", err) } localProgress += len(origChunk) diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index aab5046e4..0c91ec5d8 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -3,8 +3,6 @@ package getsvc import ( "context" "crypto/ecdsa" - "errors" - "fmt" "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" @@ -74,7 +72,7 @@ func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr ne switch v := headResp.GetBody().GetHeaderPart().(type) { case nil: - return nil, fmt.Errorf("unexpected header type %T", v) + return nil, errUnexpectedHeaderPart(v) case *objectV2.ShortHeader: if hdr, err = f.getHeaderFromShortHeader(v); err != nil { return nil, err @@ -100,9 +98,7 @@ func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr ne func (f *headRequestForwarder) getHeaderFromShortHeader(sh *objectV2.ShortHeader) (*objectV2.Header, error) { if !f.Request.GetBody().GetMainOnly() { - return nil, fmt.Errorf("wrong header part type: expected %T, received %T", - (*objectV2.ShortHeader)(nil), (*objectV2.HeaderWithSignature)(nil), - ) + return nil, errWrongHeaderPartTypeExpShortRecvWithSignature } hdr := new(objectV2.Header) @@ -118,35 +114,32 @@ func (f *headRequestForwarder) getHeaderFromShortHeader(sh *objectV2.ShortHeader func (f *headRequestForwarder) getHeaderAndSignature(hdrWithSig *objectV2.HeaderWithSignature) (*objectV2.Header, *refs.Signature, error) { if f.Request.GetBody().GetMainOnly() { - return nil, nil, fmt.Errorf("wrong header part type: expected %T, received %T", - (*objectV2.HeaderWithSignature)(nil), (*objectV2.ShortHeader)(nil), - ) + return nil, nil, errWrongHeaderPartTypeExpWithSignRecvShort } if hdrWithSig == nil { - return nil, nil, errors.New("nil object part") + return nil, nil, errNilObjectPart } hdr := hdrWithSig.GetHeader() idSig := hdrWithSig.GetSignature() if idSig == nil { - // TODO(@cthulhu-rider): #1387 use "const" error - return nil, nil, errors.New("missing signature") + return nil, nil, errMissingSignature } binID, err := f.ObjectAddr.Object().Marshal() if err != nil { - return nil, nil, fmt.Errorf("marshal ID: %w", err) + return nil, nil, errMarshalID(err) } var sig frostfscrypto.Signature if err := sig.ReadFromV2(*idSig); err != nil { - return nil, nil, fmt.Errorf("can't read signature: %w", err) + return nil, nil, errCantReadSignature(err) } if !sig.Verify(binID) { - return nil, nil, errors.New("invalid object ID signature") + return nil, nil, errInvalidObjectIDSign } return hdr, idSig, nil @@ -160,7 +153,7 @@ func (f *headRequestForwarder) sendHeadRequest(ctx context.Context, addr network return e }) if err != nil { - return nil, fmt.Errorf("sending the request failed: %w", err) + return nil, errSendingRequestFailed(err) } return headResp, nil } @@ -173,7 +166,7 @@ func (f *headRequestForwarder) verifyResponse(headResp *objectV2.HeadResponse, p // verify response structure if err := signature.VerifyServiceMessage(headResp); err != nil { - return fmt.Errorf("response verification failed: %w", err) + return errResponseVerificationFailed(err) } return checkStatus(f.Response.GetMetaHeader().GetStatus()) diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 69bed23f4..91e7a96a2 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "errors" - "fmt" "hash" "sync" @@ -24,21 +23,19 @@ import ( "git.frostfs.info/TrueCloudLab/tzhash/tz" ) -var errWrongMessageSeq = errors.New("incorrect message sequence") - func (s *Service) toPrm(req *objectV2.GetRequest, stream objectSvc.GetObjectStream) (*getsvc.Prm, error) { body := req.GetBody() addrV2 := body.GetAddress() if addrV2 == nil { - return nil, errors.New("missing object address") + return nil, errMissingObjAddress } var addr oid.Address err := addr.ReadFromV2(*addrV2) if err != nil { - return nil, fmt.Errorf("invalid object address: %w", err) + return nil, errInvalidObjAddress(err) } commonPrm, err := util.CommonPrmFromV2(req) @@ -81,14 +78,14 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get addrV2 := body.GetAddress() if addrV2 == nil { - return nil, errors.New("missing object address") + return nil, errMissingObjAddress } var addr oid.Address err := addr.ReadFromV2(*addrV2) if err != nil { - return nil, fmt.Errorf("invalid object address: %w", err) + return nil, errInvalidObjAddress(err) } commonPrm, err := util.CommonPrmFromV2(req) @@ -108,7 +105,7 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get err = p.Validate() if err != nil { - return nil, fmt.Errorf("request params validation: %w", err) + return nil, errRequestParamsValidation(err) } if !commonPrm.LocalOnly() { @@ -136,14 +133,14 @@ func (s *Service) toHashRangePrm(req *objectV2.GetRangeHashRequest) (*getsvc.Ran addrV2 := body.GetAddress() if addrV2 == nil { - return nil, errors.New("missing object address") + return nil, errMissingObjAddress } var addr oid.Address err := addr.ReadFromV2(*addrV2) if err != nil { - return nil, fmt.Errorf("invalid object address: %w", err) + return nil, errInvalidObjAddress(err) } commonPrm, err := util.CommonPrmFromV2(req) @@ -167,7 +164,7 @@ func (s *Service) toHashRangePrm(req *objectV2.GetRangeHashRequest) (*getsvc.Ran } if err != nil { - return nil, fmt.Errorf("fetching session key: %w", err) + return nil, errFetchingSessionKey(err) } p.WithCachedSignerKey(signerKey) @@ -185,7 +182,7 @@ func (s *Service) toHashRangePrm(req *objectV2.GetRangeHashRequest) (*getsvc.Ran switch t := body.GetType(); t { default: - return nil, fmt.Errorf("unknown checksum type %v", t) + return nil, errUnknownChechsumType(t) case refs.SHA256: p.SetHashGenerator(func() hash.Hash { return sha256.New() @@ -220,14 +217,14 @@ func (s *Service) toHeadPrm(req *objectV2.HeadRequest, resp *objectV2.HeadRespon addrV2 := body.GetAddress() if addrV2 == nil { - return nil, errors.New("missing object address") + return nil, errMissingObjAddress } var objAddr oid.Address err := objAddr.ReadFromV2(*addrV2) if err != nil { - return nil, fmt.Errorf("invalid object address: %w", err) + return nil, errInvalidObjAddress(err) } commonPrm, err := util.CommonPrmFromV2(req) From 3b64dffda2745d13e57c01b31c69d79ed63ffe0f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 25 Apr 2023 11:57:37 +0300 Subject: [PATCH 0188/1943] [#280] ir: Add fee config tests Signed-off-by: Dmitrii Stepanov --- pkg/innerring/config/fee_test.go | 68 ++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 pkg/innerring/config/fee_test.go diff --git a/pkg/innerring/config/fee_test.go b/pkg/innerring/config/fee_test.go new file mode 100644 index 000000000..a0c56aac1 --- /dev/null +++ b/pkg/innerring/config/fee_test.go @@ -0,0 +1,68 @@ +package config + +import ( + "strings" + "testing" + + "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestConfig(t *testing.T) { + t.Parallel() + t.Run("all set", func(t *testing.T) { + t.Parallel() + file := strings.NewReader( + ` +fee: + main_chain: 50000000 + side_chain: 200000000 + named_container_register: 2500000000 +`, + ) + v := viper.New() + v.SetConfigType("yaml") + err := v.ReadConfig(file) + require.NoError(t, err, "read config file failed") + + config := NewFeeConfig(v) + require.Equal(t, fixedn.Fixed8(50000000), config.MainChainFee(), "main chain fee invalid") + require.Equal(t, fixedn.Fixed8(200000000), config.SideChainFee(), "side chain fee invalid") + require.Equal(t, fixedn.Fixed8(2500000000), config.NamedContainerRegistrationFee(), "named container register fee invalid") + }) + + t.Run("nothing set", func(t *testing.T) { + t.Parallel() + file := strings.NewReader("") + v := viper.New() + v.SetConfigType("yaml") + err := v.ReadConfig(file) + require.NoError(t, err, "read config file failed") + + config := NewFeeConfig(v) + require.Equal(t, fixedn.Fixed8(0), config.MainChainFee(), "main chain fee invalid") + require.Equal(t, fixedn.Fixed8(0), config.SideChainFee(), "side chain fee invalid") + require.Equal(t, fixedn.Fixed8(0), config.NamedContainerRegistrationFee(), "named container register fee invalid") + }) + + t.Run("partially set", func(t *testing.T) { + t.Parallel() + file := strings.NewReader( + ` +fee: + main_chain: 10 +`, + ) + v := viper.New() + v.SetConfigType("yaml") + err := v.ReadConfig(file) + require.NoError(t, err, "read config file failed") + + config := NewFeeConfig(v) + require.Equal(t, fixedn.Fixed8(10), config.MainChainFee(), "main chain fee invalid") + require.Equal(t, fixedn.Fixed8(0), config.SideChainFee(), "side chain fee invalid") + require.Equal(t, fixedn.Fixed8(0), config.NamedContainerRegistrationFee(), "named container register fee invalid") + }) + +} From 500611f3c8eec50b3f3afffb8a2078b6213e5ebe Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 25 Apr 2023 13:58:46 +0300 Subject: [PATCH 0189/1943] [#280] ir: Add indexer tests Signed-off-by: Dmitrii Stepanov --- pkg/innerring/indexer_test.go | 225 ++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 pkg/innerring/indexer_test.go diff --git a/pkg/innerring/indexer_test.go b/pkg/innerring/indexer_test.go new file mode 100644 index 000000000..493ae92de --- /dev/null +++ b/pkg/innerring/indexer_test.go @@ -0,0 +1,225 @@ +package innerring + +import ( + "fmt" + "testing" + "time" + + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" +) + +func TestIndexerReturnsIndexes(t *testing.T) { + t.Parallel() + commiteeKeys, err := keys.NewPublicKeysFromStrings([]string{ + "03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae", + "022bb4041c50d607ff871dec7e4cd7778388e0ea6849d84ccbd9aa8f32e16a8131", + }) + require.NoError(t, err, "convert string to commitee public keys failed") + cf := &testCommiteeFetcher{ + keys: commiteeKeys, + } + + irKeys, err := keys.NewPublicKeysFromStrings([]string{ + "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35", + "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3", + "022bb4041c50d607ff871dec7e4cd7778388e0ea6849d84ccbd9aa8f32e16a8131", + }) + require.NoError(t, err, "convert string to IR public keys failed") + irf := &testIRFetcher{ + keys: irKeys, + } + + t.Run("success", func(t *testing.T) { + t.Parallel() + key := irKeys[2] + + indexer := newInnerRingIndexer(cf, irf, key, time.Second) + + idx, err := indexer.AlphabetIndex() + require.NoError(t, err, "failed to get alphabet index") + require.Equal(t, int32(1), idx, "invalid alphabet index") + + idx, err = indexer.InnerRingIndex() + require.NoError(t, err, "failed to get IR index") + require.Equal(t, int32(2), idx, "invalid IR index") + + size, err := indexer.InnerRingSize() + require.NoError(t, err, "failed to get IR size") + require.Equal(t, int32(3), size, "invalid IR size") + }) + + t.Run("not found alphabet", func(t *testing.T) { + t.Parallel() + key := irKeys[0] + + indexer := newInnerRingIndexer(cf, irf, key, time.Second) + + idx, err := indexer.AlphabetIndex() + require.NoError(t, err, "failed to get alphabet index") + require.Equal(t, int32(-1), idx, "invalid alphabet index") + + idx, err = indexer.InnerRingIndex() + require.NoError(t, err, "failed to get IR index") + require.Equal(t, int32(0), idx, "invalid IR index") + }) + + t.Run("not found IR", func(t *testing.T) { + t.Parallel() + key := commiteeKeys[0] + + indexer := newInnerRingIndexer(cf, irf, key, time.Second) + + idx, err := indexer.AlphabetIndex() + require.NoError(t, err, "failed to get alphabet index") + require.Equal(t, int32(0), idx, "invalid alphabet index") + + idx, err = indexer.InnerRingIndex() + require.NoError(t, err, "failed to get IR index") + require.Equal(t, int32(-1), idx, "invalid IR index") + }) +} + +func TestIndexerCachesIndexes(t *testing.T) { + t.Parallel() + commiteeKeys, err := keys.NewPublicKeysFromStrings([]string{}) + require.NoError(t, err, "convert string to commitee public keys failed") + cf := &testCommiteeFetcher{ + keys: commiteeKeys, + } + + irKeys, err := keys.NewPublicKeysFromStrings([]string{}) + require.NoError(t, err, "convert string to IR public keys failed") + irf := &testIRFetcher{ + keys: irKeys, + } + + key, err := keys.NewPublicKeyFromString("022bb4041c50d607ff871dec7e4cd7778388e0ea6849d84ccbd9aa8f32e16a8131") + require.NoError(t, err, "convert string to public key failed") + + indexer := newInnerRingIndexer(cf, irf, key, time.Second) + + idx, err := indexer.AlphabetIndex() + require.NoError(t, err, "failed to get alphabet index") + require.Equal(t, int32(-1), idx, "invalid alphabet index") + + idx, err = indexer.InnerRingIndex() + require.NoError(t, err, "failed to get IR index") + require.Equal(t, int32(-1), idx, "invalid IR index") + + size, err := indexer.InnerRingSize() + require.NoError(t, err, "failed to get IR size") + require.Equal(t, int32(0), size, "invalid IR size") + + require.Equal(t, int32(1), cf.calls.Load(), "invalid commitee calls count") + require.Equal(t, int32(1), irf.calls.Load(), "invalid IR calls count") + + idx, err = indexer.AlphabetIndex() + require.NoError(t, err, "failed to get alphabet index") + require.Equal(t, int32(-1), idx, "invalid alphabet index") + + idx, err = indexer.InnerRingIndex() + require.NoError(t, err, "failed to get IR index") + require.Equal(t, int32(-1), idx, "invalid IR index") + + size, err = indexer.InnerRingSize() + require.NoError(t, err, "failed to get IR size") + require.Equal(t, int32(0), size, "invalid IR size") + + require.Equal(t, int32(1), cf.calls.Load(), "invalid commitee calls count") + require.Equal(t, int32(1), irf.calls.Load(), "invalid IR calls count") + + time.Sleep(2 * time.Second) + + idx, err = indexer.AlphabetIndex() + require.NoError(t, err, "failed to get alphabet index") + require.Equal(t, int32(-1), idx, "invalid alphabet index") + + idx, err = indexer.InnerRingIndex() + require.NoError(t, err, "failed to get IR index") + require.Equal(t, int32(-1), idx, "invalid IR index") + + size, err = indexer.InnerRingSize() + require.NoError(t, err, "failed to get IR size") + require.Equal(t, int32(0), size, "invalid IR size") + + require.Equal(t, int32(2), cf.calls.Load(), "invalid commitee calls count") + require.Equal(t, int32(2), irf.calls.Load(), "invalid IR calls count") +} + +func TestIndexerThrowsErrors(t *testing.T) { + t.Parallel() + cf := &testCommiteeFetcher{ + err: fmt.Errorf("test commitee error"), + } + + irKeys, err := keys.NewPublicKeysFromStrings([]string{}) + require.NoError(t, err, "convert string to IR public keys failed") + irf := &testIRFetcher{ + keys: irKeys, + } + + key, err := keys.NewPublicKeyFromString("022bb4041c50d607ff871dec7e4cd7778388e0ea6849d84ccbd9aa8f32e16a8131") + require.NoError(t, err, "convert string to public key failed") + + indexer := newInnerRingIndexer(cf, irf, key, time.Second) + + idx, err := indexer.AlphabetIndex() + require.ErrorContains(t, err, "test commitee error", "error from commitee not throwed") + require.Equal(t, int32(0), idx, "invalid alphabet index") + + idx, err = indexer.InnerRingIndex() + require.ErrorContains(t, err, "test commitee error", "error from IR not throwed") + require.Equal(t, int32(0), idx, "invalid IR index") + + size, err := indexer.InnerRingSize() + require.ErrorContains(t, err, "test commitee error", "error from IR not throwed") + require.Equal(t, int32(0), size, "invalid IR size") + + commiteeKeys, err := keys.NewPublicKeysFromStrings([]string{}) + require.NoError(t, err, "convert string to commitee public keys failed") + cf = &testCommiteeFetcher{ + keys: commiteeKeys, + } + + irf = &testIRFetcher{ + err: fmt.Errorf("test IR error"), + } + + indexer = newInnerRingIndexer(cf, irf, key, time.Second) + + idx, err = indexer.AlphabetIndex() + require.ErrorContains(t, err, "test IR error", "error from commitee not throwed") + require.Equal(t, int32(0), idx, "invalid alphabet index") + + idx, err = indexer.InnerRingIndex() + require.ErrorContains(t, err, "test IR error", "error from IR not throwed") + require.Equal(t, int32(0), idx, "invalid IR index") + + size, err = indexer.InnerRingSize() + require.ErrorContains(t, err, "test IR error", "error from IR not throwed") + require.Equal(t, int32(0), size, "invalid IR size") +} + +type testCommiteeFetcher struct { + keys keys.PublicKeys + err error + calls atomic.Int32 +} + +func (f *testCommiteeFetcher) Committee() (keys.PublicKeys, error) { + f.calls.Inc() + return f.keys, f.err +} + +type testIRFetcher struct { + keys keys.PublicKeys + err error + calls atomic.Int32 +} + +func (f *testIRFetcher) InnerRingKeys() (keys.PublicKeys, error) { + f.calls.Inc() + return f.keys, f.err +} From 2dbe382b5f19c0c46115bd40f8d472c0e0cc6076 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 25 Apr 2023 14:54:01 +0300 Subject: [PATCH 0190/1943] [#280] ir: Add state unit tests Signed-off-by: Dmitrii Stepanov --- pkg/innerring/state_test.go | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 pkg/innerring/state_test.go diff --git a/pkg/innerring/state_test.go b/pkg/innerring/state_test.go new file mode 100644 index 000000000..fe09f8f2d --- /dev/null +++ b/pkg/innerring/state_test.go @@ -0,0 +1,53 @@ +package innerring + +import ( + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestServerState(t *testing.T) { + keyStr := "03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae" + commiteeKeys, err := keys.NewPublicKeysFromStrings([]string{keyStr}) + require.NoError(t, err, "convert string to commitee public keys failed") + cf := &testCommiteeFetcher{ + keys: commiteeKeys, + } + + irKeys, err := keys.NewPublicKeysFromStrings([]string{keyStr}) + require.NoError(t, err, "convert string to IR public keys failed") + irf := &testIRFetcher{ + keys: irKeys, + } + + key, err := keys.NewPublicKeyFromString(keyStr) + require.NoError(t, err, "convert string to public key failed") + + require.NoError(t, err, "failed to create morph client") + srv := &Server{ + statusIndex: newInnerRingIndexer(cf, irf, key, time.Second), + morphClient: &client.Client{}, + } + + var epoch uint64 = 100 + srv.SetEpochCounter(epoch) + require.Equal(t, epoch, srv.EpochCounter(), "invalid epoch counter") + + var epochDuration uint64 = 15 + srv.SetEpochDuration(epochDuration) + require.Equal(t, epochDuration, srv.EpochDuration(), "invalid epoch duration") + + var healthStatus control.HealthStatus = control.HealthStatus_READY + srv.setHealthStatus(healthStatus) + require.Equal(t, healthStatus, srv.HealthStatus(), "invalid health status") + + require.True(t, srv.IsActive(), "invalid IsActive result") + require.True(t, srv.IsAlphabet(), "invalid IsAlphabet result") + require.Equal(t, 0, srv.InnerRingIndex(), "invalid IR index") + require.Equal(t, 1, srv.InnerRingSize(), "invalid IR index") + require.Equal(t, 0, srv.AlphabetIndex(), "invalid alphabet index") +} From e70f808dc3ea64c5e00392e7044418ed87e2d653 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 25 Apr 2023 16:34:15 +0300 Subject: [PATCH 0191/1943] [#280] ir: Add parser unit tests Signed-off-by: Dmitrii Stepanov --- pkg/innerring/contracts.go | 10 +- pkg/innerring/contracts_test.go | 222 ++++++++++++++++++++++++++++++++ 2 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 pkg/innerring/contracts_test.go diff --git a/pkg/innerring/contracts.go b/pkg/innerring/contracts.go index 85a4bf946..32e817787 100644 --- a/pkg/innerring/contracts.go +++ b/pkg/innerring/contracts.go @@ -22,7 +22,7 @@ type contracts struct { alphabet alphabetContracts // in morph } -func parseContracts(cfg *viper.Viper, morph *client.Client, withoutMainNet, withoutMainNotary, withoutSideNotary bool) (*contracts, error) { +func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, withoutMainNotary, withoutSideNotary bool) (*contracts, error) { var ( result = new(contracts) err error @@ -76,7 +76,7 @@ func parseContracts(cfg *viper.Viper, morph *client.Client, withoutMainNet, with return result, nil } -func parseAlphabetContracts(cfg *viper.Viper, morph *client.Client) (alphabetContracts, error) { +func parseAlphabetContracts(cfg *viper.Viper, morph nnsResolver) (alphabetContracts, error) { num := GlagoliticLetter(cfg.GetUint("contracts.alphabet.amount")) alpha := newAlphabetContracts() @@ -115,7 +115,7 @@ func parseAlphabetContracts(cfg *viper.Viper, morph *client.Client) (alphabetCon return alpha, nil } -func parseContract(cfg *viper.Viper, morph *client.Client, cfgName, nnsName string) (res util.Uint160, err error) { +func parseContract(cfg *viper.Viper, morph nnsResolver, cfgName, nnsName string) (res util.Uint160, err error) { contractStr := cfg.GetString(cfgName) if len(contractStr) == 0 { return morph.NNSContractAddress(nnsName) @@ -123,3 +123,7 @@ func parseContract(cfg *viper.Viper, morph *client.Client, cfgName, nnsName stri return util.Uint160DecodeStringLE(contractStr) } + +type nnsResolver interface { + NNSContractAddress(name string) (sh util.Uint160, err error) +} diff --git a/pkg/innerring/contracts_test.go b/pkg/innerring/contracts_test.go new file mode 100644 index 000000000..2d6e69c05 --- /dev/null +++ b/pkg/innerring/contracts_test.go @@ -0,0 +1,222 @@ +package innerring + +import ( + "strings" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestParseContractsSuccess(t *testing.T) { + t.Parallel() + file := strings.NewReader(` +contracts: + frostfs: ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62 + processing: 597f5894867113a41e192801709c02497f611de8 + audit: 219e37aed2180b87e7fe945dbf97d67125e8d73f + balance: d2aa48d14b17b11bc4c68205027884a96706dd16 + container: ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6 + frostfsid: 9f5866decbc751a099e74c7c7bc89f609201755a + netmap: 83c600c81d47a1b1b7cf58eb49ae7ee7240dc742 + proxy: abc8794bb40a21f2db5f21ae62741eb46c8cad1c + alphabet: + amount: 2 + az: c1d211fceeb4b1dc76b8e4054d11fdf887e418ea + buky: e2ba789320899658b100f331bdebb74474757920 +`) + + v := viper.New() + v.SetConfigType("yaml") + err := v.ReadConfig(file) + require.NoError(t, err, "read config file failed") + + t.Run("all enabled", func(t *testing.T) { + t.Parallel() + c, err := parseContracts(v, nil, false, false, false) + require.NoError(t, err, "failed to parse contracts") + + frostfsExp, _ := util.Uint160DecodeStringLE("ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62") + require.Equal(t, frostfsExp, c.frostfs, "invalid frostfs") + + processingExp, _ := util.Uint160DecodeStringLE("597f5894867113a41e192801709c02497f611de8") + require.Equal(t, processingExp, c.processing, "invalid processing") + + auditExp, _ := util.Uint160DecodeStringLE("219e37aed2180b87e7fe945dbf97d67125e8d73f") + require.Equal(t, auditExp, c.audit, "invalid audit") + + balanceExp, _ := util.Uint160DecodeStringLE("d2aa48d14b17b11bc4c68205027884a96706dd16") + require.Equal(t, balanceExp, c.balance, "invalid balance") + + containerExp, _ := util.Uint160DecodeStringLE("ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6") + require.Equal(t, containerExp, c.container, "invalid container") + + frostfsIDExp, _ := util.Uint160DecodeStringLE("9f5866decbc751a099e74c7c7bc89f609201755a") + require.Equal(t, frostfsIDExp, c.frostfsID, "invalid frostfsID") + + netmapIDExp, _ := util.Uint160DecodeStringLE("83c600c81d47a1b1b7cf58eb49ae7ee7240dc742") + require.Equal(t, netmapIDExp, c.netmap, "invalid netmap") + + proxyExp, _ := util.Uint160DecodeStringLE("abc8794bb40a21f2db5f21ae62741eb46c8cad1c") + require.Equal(t, proxyExp, c.proxy, "invalid proxy") + + require.Equal(t, 2, len(c.alphabet), "invalid alphabet contracts length") + + azExp, _ := util.Uint160DecodeStringLE("c1d211fceeb4b1dc76b8e4054d11fdf887e418ea") + require.Equal(t, azExp, c.alphabet[az], "invalid az") + + bukyExp, _ := util.Uint160DecodeStringLE("e2ba789320899658b100f331bdebb74474757920") + require.Equal(t, bukyExp, c.alphabet[buky], "invalid buky") + }) + + t.Run("all disabled", func(t *testing.T) { + t.Parallel() + c, err := parseContracts(v, nil, true, true, true) + require.NoError(t, err, "failed to parse contracts") + + require.Equal(t, util.Uint160{}, c.frostfs, "invalid frostfs") + + require.Equal(t, util.Uint160{}, c.processing, "invalid processing") + + auditExp, _ := util.Uint160DecodeStringLE("219e37aed2180b87e7fe945dbf97d67125e8d73f") + require.Equal(t, auditExp, c.audit, "invalid audit") + + balanceExp, _ := util.Uint160DecodeStringLE("d2aa48d14b17b11bc4c68205027884a96706dd16") + require.Equal(t, balanceExp, c.balance, "invalid balance") + + containerExp, _ := util.Uint160DecodeStringLE("ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6") + require.Equal(t, containerExp, c.container, "invalid container") + + frostfsIDExp, _ := util.Uint160DecodeStringLE("9f5866decbc751a099e74c7c7bc89f609201755a") + require.Equal(t, frostfsIDExp, c.frostfsID, "invalid frostfsID") + + netmapIDExp, _ := util.Uint160DecodeStringLE("83c600c81d47a1b1b7cf58eb49ae7ee7240dc742") + require.Equal(t, netmapIDExp, c.netmap, "invalid netmap") + + require.Equal(t, util.Uint160{}, c.proxy, "invalid proxy") + + require.Equal(t, 2, len(c.alphabet), "invalid alphabet contracts length") + + azExp, _ := util.Uint160DecodeStringLE("c1d211fceeb4b1dc76b8e4054d11fdf887e418ea") + require.Equal(t, azExp, c.alphabet[az], "invalid az") + + bukyExp, _ := util.Uint160DecodeStringLE("e2ba789320899658b100f331bdebb74474757920") + require.Equal(t, bukyExp, c.alphabet[buky], "invalid buky") + }) + + t.Run("main notary & side notary disabled", func(t *testing.T) { + t.Parallel() + c, err := parseContracts(v, nil, false, true, true) + require.NoError(t, err, "failed to parse contracts") + + frostfsExp, _ := util.Uint160DecodeStringLE("ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62") + require.Equal(t, frostfsExp, c.frostfs, "invalid frostfs") + + require.Equal(t, util.Uint160{}, c.processing, "invalid processing") + + auditExp, _ := util.Uint160DecodeStringLE("219e37aed2180b87e7fe945dbf97d67125e8d73f") + require.Equal(t, auditExp, c.audit, "invalid audit") + + balanceExp, _ := util.Uint160DecodeStringLE("d2aa48d14b17b11bc4c68205027884a96706dd16") + require.Equal(t, balanceExp, c.balance, "invalid balance") + + containerExp, _ := util.Uint160DecodeStringLE("ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6") + require.Equal(t, containerExp, c.container, "invalid container") + + frostfsIDExp, _ := util.Uint160DecodeStringLE("9f5866decbc751a099e74c7c7bc89f609201755a") + require.Equal(t, frostfsIDExp, c.frostfsID, "invalid frostfsID") + + netmapIDExp, _ := util.Uint160DecodeStringLE("83c600c81d47a1b1b7cf58eb49ae7ee7240dc742") + require.Equal(t, netmapIDExp, c.netmap, "invalid netmap") + + require.Equal(t, util.Uint160{}, c.proxy, "invalid proxy") + + require.Equal(t, 2, len(c.alphabet), "invalid alphabet contracts length") + + azExp, _ := util.Uint160DecodeStringLE("c1d211fceeb4b1dc76b8e4054d11fdf887e418ea") + require.Equal(t, azExp, c.alphabet[az], "invalid az") + + bukyExp, _ := util.Uint160DecodeStringLE("e2ba789320899658b100f331bdebb74474757920") + require.Equal(t, bukyExp, c.alphabet[buky], "invalid buky") + }) +} + +func TestParseContractsInvalid(t *testing.T) { + t.Parallel() + t.Run("invalid audit contract", func(t *testing.T) { + t.Parallel() + file := strings.NewReader(` +contracts: + frostfs: invalid_data + processing: 597f5894867113a41e192801709c02497f611de8 + audit: 219e37aed2180b87e7fe945dbf97d67125e8d73f + balance: d2aa48d14b17b11bc4c68205027884a96706dd16 + container: ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6 + frostfsid: 9f5866decbc751a099e74c7c7bc89f609201755a + netmap: 83c600c81d47a1b1b7cf58eb49ae7ee7240dc742 + proxy: abc8794bb40a21f2db5f21ae62741eb46c8cad1c + alphabet: + amount: 2 + az: c1d211fceeb4b1dc76b8e4054d11fdf887e418ea + buky: e2ba789320899658b100f331bdebb74474757920 +`) + + v := viper.New() + v.SetConfigType("yaml") + err := v.ReadConfig(file) + require.NoError(t, err, "read config file failed") + + _, err = parseContracts(v, nil, false, false, false) + require.Error(t, err, "unexpected success") + }) + + t.Run("invalid alphabet count", func(t *testing.T) { + t.Parallel() + file := strings.NewReader(` +contracts: + frostfs: ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62 + processing: 597f5894867113a41e192801709c02497f611de8 + audit: 219e37aed2180b87e7fe945dbf97d67125e8d73f + balance: d2aa48d14b17b11bc4c68205027884a96706dd16 + container: ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6 + frostfsid: 9f5866decbc751a099e74c7c7bc89f609201755a + netmap: 83c600c81d47a1b1b7cf58eb49ae7ee7240dc742 + proxy: abc8794bb40a21f2db5f21ae62741eb46c8cad1c + alphabet: + amount: 3 + az: c1d211fceeb4b1dc76b8e4054d11fdf887e418ea + buky: e2ba789320899658b100f331bdebb74474757920 +`) + + v := viper.New() + v.SetConfigType("yaml") + err := v.ReadConfig(file) + require.NoError(t, err, "read config file failed") + + azExp, _ := util.Uint160DecodeStringLE("c1d211fceeb4b1dc76b8e4054d11fdf887e418ea") + bukyExp, _ := util.Uint160DecodeStringLE("e2ba789320899658b100f331bdebb74474757920") + + morph := &testParserMorph{ + values: map[string]util.Uint160{ + "az": azExp, + "buky": bukyExp, + }, + } + + _, err = parseContracts(v, morph, false, false, false) + require.ErrorContains(t, err, "could not read all contracts: required 3, read 2", "unexpected success") + }) +} + +type testParserMorph struct { + values map[string]util.Uint160 +} + +func (m *testParserMorph) NNSContractAddress(name string) (sh util.Uint160, err error) { + if value, found := m.values[name]; found { + return value, nil + } + return util.Uint160{}, client.ErrNNSRecordNotFound +} From 53693071de2018a162728d961b0f85098038aadc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 25 Apr 2023 18:13:00 +0300 Subject: [PATCH 0192/1943] [#280] ir: Add block timer unit tests Signed-off-by: Dmitrii Stepanov --- pkg/innerring/blocktimer.go | 8 +- pkg/innerring/blocktimer_test.go | 167 +++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 pkg/innerring/blocktimer_test.go diff --git a/pkg/innerring/blocktimer.go b/pkg/innerring/blocktimer.go index 6a4dee801..ad69f207b 100644 --- a/pkg/innerring/blocktimer.go +++ b/pkg/innerring/blocktimer.go @@ -25,6 +25,10 @@ type ( newEpochHandler func() + containerEstimationStopper interface { + StopEstimation(p container.StopEstimationPrm) error + } + epochTimerArgs struct { l *logger.Logger @@ -32,8 +36,8 @@ type ( newEpochHandlers []newEpochHandler - cnrWrapper *container.Client // to invoke stop container estimation - epoch epochState // to specify which epoch to stop, and epoch duration + cnrWrapper containerEstimationStopper // to invoke stop container estimation + epoch epochState // to specify which epoch to stop, and epoch duration stopEstimationDMul uint32 // X: X/Y of epoch in blocks stopEstimationDDiv uint32 // Y: X/Y of epoch in blocks diff --git a/pkg/innerring/blocktimer_test.go b/pkg/innerring/blocktimer_test.go new file mode 100644 index 000000000..e1a79c2a8 --- /dev/null +++ b/pkg/innerring/blocktimer_test.go @@ -0,0 +1,167 @@ +package innerring + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "github.com/stretchr/testify/require" +) + +func TestEpochTimer(t *testing.T) { + t.Parallel() + alphaState := &testAlphabetState{isAlphabet: true} + neh := &testNewEpochHandler{} + cnrStopper := &testContainerEstStopper{} + epochState := &testEpochState{ + counter: 99, + duration: 10, + } + collectHandler := &testEventHandler{} + distributeHandler := &testEventHandler{} + + args := &epochTimerArgs{ + l: test.NewLogger(t, true), + alphabetState: alphaState, + newEpochHandlers: []newEpochHandler{neh.Handle}, + cnrWrapper: cnrStopper, + epoch: epochState, + stopEstimationDMul: 2, + stopEstimationDDiv: 10, + collectBasicIncome: subEpochEventHandler{ + handler: collectHandler.Handle, + durationMul: 3, + durationDiv: 10, + }, + distributeBasicIncome: subEpochEventHandler{ + handler: distributeHandler.Handle, + durationMul: 4, + durationDiv: 10, + }, + } + et := newEpochTimer(args) + err := et.Reset() + require.NoError(t, err, "failed to reset timer") + + et.Tick(100) + require.Equal(t, 0, neh.called, "invalid new epoch handler calls") + require.Equal(t, 0, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 0, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 0, distributeHandler.called, "invalid distribute basic income calls") + + et.Tick(101) + require.Equal(t, 0, neh.called, "invalid new epoch handler calls") + require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 0, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 0, distributeHandler.called, "invalid distribute basic income calls") + + et.Tick(102) + require.Equal(t, 0, neh.called, "invalid new epoch handler calls") + require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 0, distributeHandler.called, "invalid distribute basic income calls") + + et.Tick(103) + require.Equal(t, 0, neh.called, "invalid new epoch handler calls") + require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") + + var h uint32 + for h = 104; h < 109; h++ { + et.Tick(h) + require.Equal(t, 0, neh.called, "invalid new epoch handler calls") + require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") + } + + et.Tick(109) + require.Equal(t, 1, neh.called, "invalid new epoch handler calls") + require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") + + et.Tick(110) + require.Equal(t, 1, neh.called, "invalid new epoch handler calls") + require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") + + et.Tick(111) + require.Equal(t, 1, neh.called, "invalid new epoch handler calls") + require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") + + et.Tick(112) + require.Equal(t, 1, neh.called, "invalid new epoch handler calls") + require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 2, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") + + et.Tick(113) + require.Equal(t, 1, neh.called, "invalid new epoch handler calls") + require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 2, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 2, distributeHandler.called, "invalid distribute basic income calls") + + for h = 114; h < 119; h++ { + et.Tick(h) + require.Equal(t, 1, neh.called, "invalid new epoch handler calls") + require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 2, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 2, distributeHandler.called, "invalid distribute basic income calls") + } + et.Tick(120) + require.Equal(t, 2, neh.called, "invalid new epoch handler calls") + require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") + require.Equal(t, 2, collectHandler.called, "invalid collect basic income calls") + require.Equal(t, 2, distributeHandler.called, "invalid distribute basic income calls") +} + +type testAlphabetState struct { + isAlphabet bool +} + +func (s *testAlphabetState) IsAlphabet() bool { + return s.isAlphabet +} + +type testNewEpochHandler struct { + called int +} + +func (h *testNewEpochHandler) Handle() { + h.called++ +} + +type testContainerEstStopper struct { + called int +} + +func (s *testContainerEstStopper) StopEstimation(_ container.StopEstimationPrm) error { + s.called++ + return nil +} + +type testEpochState struct { + counter uint64 + duration uint64 +} + +func (s *testEpochState) EpochCounter() uint64 { + return s.counter +} +func (s *testEpochState) EpochDuration() uint64 { + return s.duration +} + +type testEventHandler struct { + called int +} + +func (h *testEventHandler) Handle(e event.Event) { + h.called++ +} From e89fa7f69f26e962d6cf40f437f5af6f8c74450c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 26 Apr 2023 10:51:30 +0300 Subject: [PATCH 0193/1943] [#280] ir: Add alphabet processor unit tests Signed-off-by: Dmitrii Stepanov --- pkg/innerring/alphabet.go | 12 +- pkg/innerring/contracts.go | 6 +- .../processors/alphabet/handlers_test.go | 282 ++++++++++++++++++ .../processors/alphabet/processor.go | 31 +- 4 files changed, 316 insertions(+), 15 deletions(-) create mode 100644 pkg/innerring/processors/alphabet/handlers_test.go diff --git a/pkg/innerring/alphabet.go b/pkg/innerring/alphabet.go index 78930db7d..ddb344403 100644 --- a/pkg/innerring/alphabet.go +++ b/pkg/innerring/alphabet.go @@ -104,13 +104,13 @@ func (l GlagoliticLetter) String() string { return "unknown" } -type alphabetContracts map[GlagoliticLetter]util.Uint160 +type AlphabetContracts map[GlagoliticLetter]util.Uint160 -func newAlphabetContracts() alphabetContracts { +func NewAlphabetContracts() AlphabetContracts { return make(map[GlagoliticLetter]util.Uint160, lastLetterNum) } -func (a alphabetContracts) GetByIndex(ind int) (util.Uint160, bool) { +func (a AlphabetContracts) GetByIndex(ind int) (util.Uint160, bool) { if ind < 0 || ind >= int(lastLetterNum) { return util.Uint160{}, false } @@ -120,16 +120,16 @@ func (a alphabetContracts) GetByIndex(ind int) (util.Uint160, bool) { return contract, ok } -func (a alphabetContracts) indexOutOfRange(ind int) bool { +func (a AlphabetContracts) indexOutOfRange(ind int) bool { return ind < 0 && ind >= len(a) } -func (a alphabetContracts) iterate(f func(GlagoliticLetter, util.Uint160)) { +func (a AlphabetContracts) iterate(f func(GlagoliticLetter, util.Uint160)) { for letter, contract := range a { f(letter, contract) } } -func (a *alphabetContracts) set(l GlagoliticLetter, h util.Uint160) { +func (a *AlphabetContracts) set(l GlagoliticLetter, h util.Uint160) { (*a)[l] = h } diff --git a/pkg/innerring/contracts.go b/pkg/innerring/contracts.go index 32e817787..55c2ff582 100644 --- a/pkg/innerring/contracts.go +++ b/pkg/innerring/contracts.go @@ -19,7 +19,7 @@ type contracts struct { processing util.Uint160 // in mainnet frostfsID util.Uint160 // in morph - alphabet alphabetContracts // in morph + alphabet AlphabetContracts // in morph } func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, withoutMainNotary, withoutSideNotary bool) (*contracts, error) { @@ -76,9 +76,9 @@ func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, without return result, nil } -func parseAlphabetContracts(cfg *viper.Viper, morph nnsResolver) (alphabetContracts, error) { +func parseAlphabetContracts(cfg *viper.Viper, morph nnsResolver) (AlphabetContracts, error) { num := GlagoliticLetter(cfg.GetUint("contracts.alphabet.amount")) - alpha := newAlphabetContracts() + alpha := NewAlphabetContracts() if num > lastLetterNum { return nil, fmt.Errorf("amount of alphabet contracts overflows glagolitsa %d > %d", num, lastLetterNum) diff --git a/pkg/innerring/processors/alphabet/handlers_test.go b/pkg/innerring/processors/alphabet/handlers_test.go new file mode 100644 index 000000000..5e71cbc90 --- /dev/null +++ b/pkg/innerring/processors/alphabet/handlers_test.go @@ -0,0 +1,282 @@ +package alphabet_test + +import ( + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/stretchr/testify/require" +) + +func TestProcessorEmitsGasToNetmapAndAlphabet(t *testing.T) { + t.Parallel() + var emission uint64 = 100_000 + var index int = 5 + var parsedWallets []util.Uint160 = []util.Uint160{{20}, {25}} + + alphabetContracts := innerring.NewAlphabetContracts() + for i := 0; i <= index; i++ { + alphabetContracts[innerring.GlagoliticLetter(i)] = util.Uint160{uint8(i)} + } + + morphClient := &testMorphClient{} + + var node1 netmap.NodeInfo + key1, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key1") + node1.SetPublicKey(key1.Bytes()) + + var node2 netmap.NodeInfo + key2, err := keys.NewPublicKeyFromString("02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3") + require.NoError(t, err, "failed to parse key2") + node2.SetPublicKey(key2.Bytes()) + + nodes := []netmap.NodeInfo{node1, node2} + + network := &netmap.NetMap{} + network.SetNodes(nodes) + + netmapClient := &testNetmapClient{ + netmap: network, + } + + params := &alphabet.Params{ + ParsedWallets: parsedWallets, + Log: test.NewLogger(t, true), + PoolSize: 2, + StorageEmission: emission, + IRList: &testIndexer{index: index}, + AlphabetContracts: alphabetContracts, + MorphClient: morphClient, + NetmapClient: netmapClient, + } + + processor, err := alphabet.New(params) + require.NoError(t, err, "failed to create processor instance") + + processor.HandleGasEmission(timers.NewAlphabetEmitTick{}) + + processor.WaitPoolRunning() + + require.EqualValues(t, []invokedMethod{ + { + contract: alphabetContracts[innerring.GlagoliticLetter(index)], + fee: 0, + method: "emit", + }, + }, morphClient.invokedMethods, "invalid invoked morph methods") + + require.EqualValues(t, []transferGas{ + { + receiver: key1.GetScriptHash(), + amount: fixedn.Fixed8(25_000), + }, + { + receiver: key2.GetScriptHash(), + amount: fixedn.Fixed8(25_000), + }, + }, morphClient.transferedGas, "invalid transfered Gas") + + require.EqualValues(t, []batchTransferGas{ + { + receivers: parsedWallets, + amount: fixedn.Fixed8(25_000), + }, + }, morphClient.batchTransferedGas, "invalid batch transfered Gas") +} + +func TestProcessorEmitsGasToNetmapIfNoParsedWallets(t *testing.T) { + t.Parallel() + var emission uint64 = 100_000 + var index int = 5 + var parsedWallets []util.Uint160 = []util.Uint160{} + + alphabetContracts := innerring.NewAlphabetContracts() + for i := 0; i <= index; i++ { + alphabetContracts[innerring.GlagoliticLetter(i)] = util.Uint160{uint8(i)} + } + + morphClient := &testMorphClient{} + + var node1 netmap.NodeInfo + key1, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key1") + node1.SetPublicKey(key1.Bytes()) + + var node2 netmap.NodeInfo + key2, err := keys.NewPublicKeyFromString("02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3") + require.NoError(t, err, "failed to parse key2") + node2.SetPublicKey(key2.Bytes()) + + nodes := []netmap.NodeInfo{node1, node2} + + network := &netmap.NetMap{} + network.SetNodes(nodes) + + netmapClient := &testNetmapClient{ + netmap: network, + } + + params := &alphabet.Params{ + ParsedWallets: parsedWallets, + Log: test.NewLogger(t, true), + PoolSize: 2, + StorageEmission: emission, + IRList: &testIndexer{index: index}, + AlphabetContracts: alphabetContracts, + MorphClient: morphClient, + NetmapClient: netmapClient, + } + + processor, err := alphabet.New(params) + require.NoError(t, err, "failed to create processor instance") + + processor.HandleGasEmission(timers.NewAlphabetEmitTick{}) + + time.Sleep(time.Second) + + require.EqualValues(t, []invokedMethod{ + { + contract: alphabetContracts[innerring.GlagoliticLetter(index)], + fee: 0, + method: "emit", + }, + }, morphClient.invokedMethods, "invalid invoked morph methods") + + require.EqualValues(t, []transferGas{ + { + receiver: key1.GetScriptHash(), + amount: fixedn.Fixed8(50_000), + }, + { + receiver: key2.GetScriptHash(), + amount: fixedn.Fixed8(50_000), + }, + }, morphClient.transferedGas, "invalid transfered Gas") + + require.Equal(t, 0, len(morphClient.batchTransferedGas), "invalid batch transfered Gas") +} + +func TestProcessorDoesntEmitGasIfNoNetmapOrParsedWallets(t *testing.T) { + t.Parallel() + var emission uint64 = 100_000 + var index int = 5 + var parsedWallets []util.Uint160 = []util.Uint160{} + + alphabetContracts := innerring.NewAlphabetContracts() + for i := 0; i <= index; i++ { + alphabetContracts[innerring.GlagoliticLetter(i)] = util.Uint160{uint8(i)} + } + + morphClient := &testMorphClient{} + + nodes := []netmap.NodeInfo{} + network := &netmap.NetMap{} + network.SetNodes(nodes) + + netmapClient := &testNetmapClient{ + netmap: network, + } + + params := &alphabet.Params{ + ParsedWallets: parsedWallets, + Log: test.NewLogger(t, true), + PoolSize: 2, + StorageEmission: emission, + IRList: &testIndexer{index: index}, + AlphabetContracts: alphabetContracts, + MorphClient: morphClient, + NetmapClient: netmapClient, + } + + processor, err := alphabet.New(params) + require.NoError(t, err, "failed to create processor instance") + + processor.HandleGasEmission(timers.NewAlphabetEmitTick{}) + + time.Sleep(time.Second) + + require.EqualValues(t, []invokedMethod{ + { + contract: alphabetContracts[innerring.GlagoliticLetter(index)], + fee: 0, + method: "emit", + }, + }, morphClient.invokedMethods, "invalid invoked morph methods") + + require.Equal(t, 0, len(morphClient.transferedGas), "invalid transfered Gas") + + require.Equal(t, 0, len(morphClient.batchTransferedGas), "invalid batch transfered Gas") +} + +type testIndexer struct { + index int +} + +func (i *testIndexer) AlphabetIndex() int { + return i.index +} + +type invokedMethod struct { + contract util.Uint160 + fee fixedn.Fixed8 + method string + args []any +} + +type transferGas struct { + receiver util.Uint160 + amount fixedn.Fixed8 +} + +type batchTransferGas struct { + receivers []util.Uint160 + amount fixedn.Fixed8 +} + +type testMorphClient struct { + invokedMethods []invokedMethod + transferedGas []transferGas + batchTransferedGas []batchTransferGas +} + +func (c *testMorphClient) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) error { + c.invokedMethods = append(c.invokedMethods, + invokedMethod{ + contract: contract, + fee: fee, + method: method, + args: args, + }) + return nil +} +func (c *testMorphClient) TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error { + c.transferedGas = append(c.transferedGas, transferGas{ + receiver: receiver, + amount: amount, + }) + return nil +} + +func (c *testMorphClient) BatchTransferGas(receivers []util.Uint160, amount fixedn.Fixed8) error { + c.batchTransferedGas = append(c.batchTransferedGas, batchTransferGas{ + receivers: receivers, + amount: amount, + }) + return nil +} + +type testNetmapClient struct { + netmap *netmap.NetMap +} + +func (c *testNetmapClient) NetMap() (*netmap.NetMap, error) { + return c.netmap, nil +} diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index 169bfb3e2..c2d7c1164 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -3,12 +3,13 @@ package alphabet import ( "errors" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" "go.uber.org/zap" @@ -32,14 +33,24 @@ type ( GetByIndex(int) (util.Uint160, bool) } + netmapClient interface { + NetMap() (*netmap.NetMap, error) + } + + morphClient interface { + Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) error + TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error + BatchTransferGas(receivers []util.Uint160, amount fixedn.Fixed8) error + } + // Processor of events produced for alphabet contracts in the sidechain. Processor struct { parsedWallets []util.Uint160 log *logger.Logger pool *ants.Pool alphabetContracts Contracts - netmapClient *nmClient.Client - morphClient *client.Client + netmapClient netmapClient + morphClient morphClient irList Indexer storageEmission uint64 } @@ -50,8 +61,8 @@ type ( Log *logger.Logger PoolSize int AlphabetContracts Contracts - NetmapClient *nmClient.Client - MorphClient *client.Client + NetmapClient netmapClient + MorphClient morphClient IRList Indexer StorageEmission uint64 } @@ -106,3 +117,11 @@ func (ap *Processor) ListenerNotaryParsers() []event.NotaryParserInfo { func (ap *Processor) ListenerNotaryHandlers() []event.NotaryHandlerInfo { return nil } + +// WaitPoolRunning waits while pool has running tasks +// For use in test only. +func (ap *Processor) WaitPoolRunning() { + for ap.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } +} From 686f01bce5649c8056d52a824041f759a83f50d8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 26 Apr 2023 11:42:02 +0300 Subject: [PATCH 0194/1943] [#280] ir: Add balance processor unit tests Signed-off-by: Dmitrii Stepanov --- .../processors/balance/handlers_test.go | 90 +++++++++++++++++++ pkg/innerring/processors/balance/processor.go | 8 +- 2 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 pkg/innerring/processors/balance/handlers_test.go diff --git a/pkg/innerring/processors/balance/handlers_test.go b/pkg/innerring/processors/balance/handlers_test.go new file mode 100644 index 000000000..3470fba2d --- /dev/null +++ b/pkg/innerring/processors/balance/handlers_test.go @@ -0,0 +1,90 @@ +package balance + +import ( + "testing" + "time" + + frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" + balanceEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/balance" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/stretchr/testify/require" +) + +func TestProcessorCallsFrostFSContractForLockEvent(t *testing.T) { + t.Parallel() + as := &testAlphabetState{ + isAlphabet: true, + } + conv := &testPresicionConverter{} + cl := &testFrostFSContractClient{} + bsc := util.Uint160{100} + + processor, err := New(&Params{ + Log: test.NewLogger(t, true), + PoolSize: 2, + FrostFSClient: cl, + BalanceSC: bsc, + AlphabetState: as, + Converter: conv, + }) + require.NoError(t, err, "failed to create processor") + + processor.handleLock(balanceEvent.Lock{}) + + for processor.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + require.Equal(t, 1, cl.chequeCalls, "invalid Cheque calls") +} + +func TestProcessorDoesntCallFrostFSContractIfNotAlphabet(t *testing.T) { + t.Parallel() + as := &testAlphabetState{} + conv := &testPresicionConverter{} + cl := &testFrostFSContractClient{} + bsc := util.Uint160{100} + + processor, err := New(&Params{ + Log: test.NewLogger(t, true), + PoolSize: 2, + FrostFSClient: cl, + BalanceSC: bsc, + AlphabetState: as, + Converter: conv, + }) + require.NoError(t, err, "failed to create processor") + + processor.handleLock(balanceEvent.Lock{}) + + for processor.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + require.Equal(t, 0, cl.chequeCalls, "invalid Cheque calls") +} + +type testAlphabetState struct { + isAlphabet bool +} + +func (s *testAlphabetState) IsAlphabet() bool { + return s.isAlphabet +} + +type testPresicionConverter struct { +} + +func (c *testPresicionConverter) ToFixed8(v int64) int64 { + return v +} + +type testFrostFSContractClient struct { + chequeCalls int +} + +func (c *testFrostFSContractClient) Cheque(p frostfscontract.ChequePrm) error { + c.chequeCalls++ + return nil +} diff --git a/pkg/innerring/processors/balance/processor.go b/pkg/innerring/processors/balance/processor.go index 7ae639e89..356754cfb 100644 --- a/pkg/innerring/processors/balance/processor.go +++ b/pkg/innerring/processors/balance/processor.go @@ -25,11 +25,15 @@ type ( ToFixed8(int64) int64 } + FrostFSClient interface { + Cheque(p frostfscontract.ChequePrm) error + } + // Processor of events produced by balance contract in the morphchain. Processor struct { log *logger.Logger pool *ants.Pool - frostfsClient *frostfscontract.Client + frostfsClient FrostFSClient balanceSC util.Uint160 alphabetState AlphabetState converter PrecisionConverter @@ -39,7 +43,7 @@ type ( Params struct { Log *logger.Logger PoolSize int - FrostFSClient *frostfscontract.Client + FrostFSClient FrostFSClient BalanceSC util.Uint160 AlphabetState AlphabetState Converter PrecisionConverter From 5010b35466ffedb7e143bc9ee6952808241dc9b4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 26 Apr 2023 12:05:33 +0300 Subject: [PATCH 0195/1943] [#280] ir: Add container processor unit tests Signed-off-by: Dmitrii Stepanov --- .../processors/container/handlers.go | 2 +- .../processors/container/handlers_test.go | 337 ++++++++++++++++++ .../processors/container/process_container.go | 6 +- .../processors/container/process_eacl.go | 8 +- .../processors/container/processor.go | 27 +- pkg/morph/client/container/get.go | 6 +- pkg/morph/event/container/delete.go | 22 +- pkg/morph/event/container/delete_notary.go | 8 +- pkg/morph/event/container/delete_test.go | 6 +- pkg/morph/event/container/eacl.go | 28 +- pkg/morph/event/container/eacl_notary.go | 10 +- 11 files changed, 409 insertions(+), 51 deletions(-) create mode 100644 pkg/innerring/processors/container/handlers_test.go diff --git a/pkg/innerring/processors/container/handlers.go b/pkg/innerring/processors/container/handlers.go index 8d260808b..2ab1147c8 100644 --- a/pkg/innerring/processors/container/handlers.go +++ b/pkg/innerring/processors/container/handlers.go @@ -36,7 +36,7 @@ func (cp *Processor) handleDelete(ev event.Event) { // send an event to the worker pool - err := cp.pool.Submit(func() { cp.processContainerDelete(&del) }) + err := cp.pool.Submit(func() { cp.processContainerDelete(del) }) if err != nil { // there system can be moved into controlled degradation stage cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go new file mode 100644 index 000000000..6075ff577 --- /dev/null +++ b/pkg/innerring/processors/container/handlers_test.go @@ -0,0 +1,337 @@ +package container + +import ( + "crypto/ecdsa" + "encoding/hex" + "testing" + "time" + + containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" + containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" + frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/network/payload" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/stretchr/testify/require" +) + +func TestPutEvent(t *testing.T) { + t.Parallel() + nst := &testNetworkState{ + homHashDisabled: true, + epoch: 100, + } + cc := &testContainerClient{ + get: make(map[string]*containercore.Container), + } + + proc, err := New(&Params{ + Log: test.NewLogger(t, true), + PoolSize: 2, + AlphabetState: &testAlphabetState{isAlphabet: true}, + FrostFSIDClient: &testIDClient{}, + NotaryDisabled: true, + NetworkState: nst, + ContainerClient: cc, + }) + require.NoError(t, err, "failed to create processor") + + p, err := keys.NewPrivateKey() + require.NoError(t, err) + var usr user.ID + user.IDFromKey(&usr, (ecdsa.PublicKey)(*p.PublicKey())) + + var pp netmap.PlacementPolicy + pp.AddReplicas(netmap.ReplicaDescriptor{}) + + var cnr containerSDK.Container + cnr.Init() + cnr.SetOwner(usr) + cnr.SetPlacementPolicy(pp) + cnr.SetBasicACL(acl.Private) + containerSDK.DisableHomomorphicHashing(&cnr) + + event := &testPutEvent{ + cnr: &cnr, + pk: p, + st: nil, + } + + proc.handlePut(event) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + var expectedPut cntClient.PutPrm + expectedPut.SetContainer(cnr.Marshal()) + expectedPut.SetKey(p.PublicKey().Bytes()) + expectedPut.SetSignature(p.Sign(cnr.Marshal())) + expectedPut.SetZone("container") + + require.EqualValues(t, []cntClient.PutPrm{expectedPut}, cc.put, "invalid put requests") +} + +func TestDeleteEvent(t *testing.T) { + t.Parallel() + nst := &testNetworkState{ + homHashDisabled: true, + epoch: 100, + } + cc := &testContainerClient{ + get: make(map[string]*containercore.Container), + } + + p, err := keys.NewPrivateKey() + require.NoError(t, err) + + idc := &testIDClient{ + publicKeys: []*keys.PublicKey{ + p.PublicKey(), + }, + } + + proc, err := New(&Params{ + Log: test.NewLogger(t, true), + PoolSize: 2, + AlphabetState: &testAlphabetState{isAlphabet: true}, + FrostFSIDClient: idc, + NotaryDisabled: true, + NetworkState: nst, + ContainerClient: cc, + }) + require.NoError(t, err, "failed to create processor") + + var usr user.ID + user.IDFromKey(&usr, (ecdsa.PublicKey)(*p.PublicKey())) + + var pp netmap.PlacementPolicy + pp.AddReplicas(netmap.ReplicaDescriptor{}) + + var cnr containerSDK.Container + cnr.Init() + cnr.SetOwner(usr) + cnr.SetPlacementPolicy(pp) + cnr.SetBasicACL(acl.Private) + containerSDK.DisableHomomorphicHashing(&cnr) + + var cid cid.ID + containerSDK.CalculateID(&cid, cnr) + cidBin := make([]byte, 32) + cid.Encode(cidBin) + + ev := containerEvent.Delete{ + ContainerIDValue: cidBin, + SignatureValue: p.Sign(cidBin), + } + + var signature frostfscrypto.Signature + signer := frostfsecdsa.Signer(p.PrivateKey) + require.NoError(t, signature.Calculate(signer, ev.ContainerID()), "failed to calculate signature") + cc.get[hex.EncodeToString(ev.ContainerID())] = &containercore.Container{ + Value: cnr, + Signature: signature, + } + + proc.handleDelete(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + var expectedDelete cntClient.DeletePrm + expectedDelete.SetCID(ev.ContainerID()) + expectedDelete.SetSignature(ev.Signature()) + + require.EqualValues(t, []cntClient.DeletePrm{expectedDelete}, cc.delete, "invalid delete requests") +} + +func TestSetEACLEvent(t *testing.T) { + t.Parallel() + nst := &testNetworkState{ + homHashDisabled: true, + epoch: 100, + } + cc := &testContainerClient{ + get: make(map[string]*containercore.Container), + } + + proc, err := New(&Params{ + Log: test.NewLogger(t, true), + PoolSize: 2, + AlphabetState: &testAlphabetState{isAlphabet: true}, + FrostFSIDClient: &testIDClient{}, + NotaryDisabled: true, + NetworkState: nst, + ContainerClient: cc, + }) + require.NoError(t, err, "failed to create processor") + + p, err := keys.NewPrivateKey() + require.NoError(t, err) + + var usr user.ID + user.IDFromKey(&usr, (ecdsa.PublicKey)(*p.PublicKey())) + + var pp netmap.PlacementPolicy + pp.AddReplicas(netmap.ReplicaDescriptor{}) + + var cnr containerSDK.Container + cnr.Init() + cnr.SetOwner(usr) + cnr.SetPlacementPolicy(pp) + cnr.SetBasicACL(acl.PrivateExtended) + containerSDK.DisableHomomorphicHashing(&cnr) + + var cid cid.ID + containerSDK.CalculateID(&cid, cnr) + cidBytes := make([]byte, 32) + cid.Encode(cidBytes) + + var signature frostfscrypto.Signature + signer := frostfsecdsa.Signer(p.PrivateKey) + require.NoError(t, signature.Calculate(signer, cidBytes), "failed to calculate signature") + + cc.get[hex.EncodeToString(cidBytes)] = &containercore.Container{ + Value: cnr, + Signature: signature, + } + + table := eacl.NewTable() + table.SetCID(cid) + table.SetVersion(version.Current()) + + r := &eacl.Record{} + r.AddObjectContainerIDFilter(eacl.MatchStringEqual, cid) + + table.AddRecord(r) + + event := containerEvent.SetEACL{ + TableValue: table.ToV2().StableMarshal(nil), + PublicKeyValue: p.PublicKey().Bytes(), + SignatureValue: p.Sign(table.ToV2().StableMarshal(nil)), + } + + proc.handleSetEACL(event) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + var expectedPutEACL cntClient.PutEACLPrm + expectedPutEACL.SetTable(table.ToV2().StableMarshal(nil)) + expectedPutEACL.SetKey(p.PublicKey().Bytes()) + expectedPutEACL.SetSignature(p.Sign(table.ToV2().StableMarshal(nil))) + + require.EqualValues(t, []cntClient.PutEACLPrm{expectedPutEACL}, cc.putEACL, "invalid set EACL requests") +} + +type testAlphabetState struct { + isAlphabet bool +} + +func (s *testAlphabetState) IsAlphabet() bool { + return s.isAlphabet +} + +type testNetworkState struct { + homHashDisabled bool + epoch uint64 +} + +func (s *testNetworkState) HomomorphicHashDisabled() (bool, error) { + return s.homHashDisabled, nil +} + +func (s *testNetworkState) Epoch() (uint64, error) { + return s.epoch, nil +} + +type testContainerClient struct { + contractAddress util.Uint160 + put []cntClient.PutPrm + get map[string]*containercore.Container + delete []cntClient.DeletePrm + putEACL []cntClient.PutEACLPrm +} + +func (c *testContainerClient) ContractAddress() util.Uint160 { + return c.contractAddress +} + +func (c *testContainerClient) Morph() *client.Client { + return nil +} + +func (c *testContainerClient) Put(p cntClient.PutPrm) error { + c.put = append(c.put, p) + return nil +} + +func (c *testContainerClient) Get(cid []byte) (*containercore.Container, error) { + key := hex.EncodeToString(cid) + if cont, found := c.get[key]; found { + return cont, nil + } + return nil, apistatus.ContainerNotFound{} +} + +func (c *testContainerClient) Delete(p cntClient.DeletePrm) error { + c.delete = append(c.delete, p) + return nil +} + +func (c *testContainerClient) PutEACL(p cntClient.PutEACLPrm) error { + c.putEACL = append(c.putEACL, p) + return nil +} + +type testIDClient struct { + publicKeys keys.PublicKeys +} + +func (c *testIDClient) AccountKeys(p frostfsid.AccountKeysPrm) (keys.PublicKeys, error) { + return c.publicKeys, nil +} + +var _ putEvent = &testPutEvent{} + +type testPutEvent struct { + cnr *containerSDK.Container + pk *keys.PrivateKey + st []byte +} + +func (e *testPutEvent) MorphEvent() {} + +func (e *testPutEvent) Container() []byte { + return e.cnr.Marshal() +} + +func (e *testPutEvent) PublicKey() []byte { + return e.pk.PublicKey().Bytes() +} + +func (e *testPutEvent) Signature() []byte { + return e.pk.Sign(e.cnr.Marshal()) +} + +func (e *testPutEvent) SessionToken() []byte { + return e.st +} +func (e *testPutEvent) NotaryRequest() *payload.P2PNotaryRequest { + return nil +} diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index c4421c453..603ab86d1 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -120,7 +120,7 @@ func (cp *Processor) approvePutContainer(ctx *putContainerContext) { // Process delete container operation from the user by checking container sanity // and sending approve tx back to morph. -func (cp *Processor) processContainerDelete(e *containerEvent.Delete) { +func (cp *Processor) processContainerDelete(e containerEvent.Delete) { if !cp.alphabetState.IsAlphabet() { cp.log.Info(logs.ContainerNonAlphabetModeIgnoreContainerDelete) return @@ -138,7 +138,7 @@ func (cp *Processor) processContainerDelete(e *containerEvent.Delete) { cp.approveDeleteContainer(e) } -func (cp *Processor) checkDeleteContainer(e *containerEvent.Delete) error { +func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { binCnr := e.ContainerID() var idCnr cid.ID @@ -170,7 +170,7 @@ func (cp *Processor) checkDeleteContainer(e *containerEvent.Delete) error { return nil } -func (cp *Processor) approveDeleteContainer(e *containerEvent.Delete) { +func (cp *Processor) approveDeleteContainer(e containerEvent.Delete) { var err error prm := cntClient.DeletePrm{} diff --git a/pkg/innerring/processors/container/process_eacl.go b/pkg/innerring/processors/container/process_eacl.go index fce75c678..a8d880c8f 100644 --- a/pkg/innerring/processors/container/process_eacl.go +++ b/pkg/innerring/processors/container/process_eacl.go @@ -6,13 +6,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" + containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "go.uber.org/zap" ) -func (cp *Processor) processSetEACL(e container.SetEACL) { +func (cp *Processor) processSetEACL(e containerEvent.SetEACL) { if !cp.alphabetState.IsAlphabet() { cp.log.Info(logs.ContainerNonAlphabetModeIgnoreSetEACL) return @@ -30,7 +30,7 @@ func (cp *Processor) processSetEACL(e container.SetEACL) { cp.approveSetEACL(e) } -func (cp *Processor) checkSetEACL(e container.SetEACL) error { +func (cp *Processor) checkSetEACL(e containerEvent.SetEACL) error { binTable := e.Table() // unmarshal table @@ -74,7 +74,7 @@ func (cp *Processor) checkSetEACL(e container.SetEACL) error { return nil } -func (cp *Processor) approveSetEACL(e container.SetEACL) { +func (cp *Processor) approveSetEACL(e containerEvent.SetEACL) { var err error prm := cntClient.PutEACLPrm{} diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index 321596eb4..d5af5e394 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -5,12 +5,16 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/core/mempoolevent" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" "go.uber.org/zap" ) @@ -21,13 +25,26 @@ type ( IsAlphabet() bool } + ContClient interface { + ContractAddress() util.Uint160 + Morph() *client.Client + Put(p cntClient.PutPrm) error + Get(cid []byte) (*containercore.Container, error) + Delete(p cntClient.DeletePrm) error + PutEACL(p cntClient.PutEACLPrm) error + } + + IDClient interface { + AccountKeys(p frostfsid.AccountKeysPrm) (keys.PublicKeys, error) + } + // Processor of events produced by container contract in the sidechain. Processor struct { log *logger.Logger pool *ants.Pool alphabetState AlphabetState - cnrClient *container.Client // notary must be enabled - idClient *frostfsid.Client + cnrClient ContClient // notary must be enabled + idClient IDClient netState NetworkState notaryDisabled bool } @@ -37,8 +54,8 @@ type ( Log *logger.Logger PoolSize int AlphabetState AlphabetState - ContainerClient *container.Client - FrostFSIDClient *frostfsid.Client + ContainerClient ContClient + FrostFSIDClient IDClient NetworkState NetworkState NotaryDisabled bool } diff --git a/pkg/morph/client/container/get.go b/pkg/morph/client/container/get.go index 4775cd281..009b22f3c 100644 --- a/pkg/morph/client/container/get.go +++ b/pkg/morph/client/container/get.go @@ -26,8 +26,12 @@ func AsContainerSource(w *Client) containercore.Source { return (*containerSource)(w) } +type getContainer interface { + Get(cid []byte) (*containercore.Container, error) +} + // Get marshals container ID, and passes it to Wrapper's Get method. -func Get(c *Client, cnr cid.ID) (*containercore.Container, error) { +func Get(c getContainer, cnr cid.ID) (*containercore.Container, error) { binCnr := make([]byte, sha256.Size) cnr.Encode(binCnr) diff --git a/pkg/morph/event/container/delete.go b/pkg/morph/event/container/delete.go index 398466f51..7286ddcfc 100644 --- a/pkg/morph/event/container/delete.go +++ b/pkg/morph/event/container/delete.go @@ -12,34 +12,34 @@ import ( // Delete structure of container.Delete notification from morph chain. type Delete struct { - containerID []byte - signature []byte - token []byte + ContainerIDValue []byte + SignatureValue []byte + TokenValue []byte // For notary notifications only. // Contains raw transactions of notary request. - notaryRequest *payload.P2PNotaryRequest + NotaryRequestValue *payload.P2PNotaryRequest } // MorphEvent implements Neo:Morph Event interface. func (Delete) MorphEvent() {} // ContainerID is a marshalled container structure, defined in API. -func (d Delete) ContainerID() []byte { return d.containerID } +func (d Delete) ContainerID() []byte { return d.ContainerIDValue } // Signature of marshalled container by container owner. -func (d Delete) Signature() []byte { return d.signature } +func (d Delete) Signature() []byte { return d.SignatureValue } // SessionToken returns binary token of the session // within which the eACL was set. func (d Delete) SessionToken() []byte { - return d.token + return d.TokenValue } // NotaryRequest returns raw notary request if notification // was received via notary service. Otherwise, returns nil. func (d Delete) NotaryRequest() *payload.P2PNotaryRequest { - return d.notaryRequest + return d.NotaryRequestValue } const expectedItemNumDelete = 3 @@ -63,19 +63,19 @@ func ParseDelete(e *state.ContainedNotificationEvent) (event.Event, error) { } // parse container - ev.containerID, err = client.BytesFromStackItem(params[0]) + ev.ContainerIDValue, err = client.BytesFromStackItem(params[0]) if err != nil { return nil, fmt.Errorf("could not get container: %w", err) } // parse signature - ev.signature, err = client.BytesFromStackItem(params[1]) + ev.SignatureValue, err = client.BytesFromStackItem(params[1]) if err != nil { return nil, fmt.Errorf("could not get signature: %w", err) } // parse session token - ev.token, err = client.BytesFromStackItem(params[2]) + ev.TokenValue, err = client.BytesFromStackItem(params[2]) if err != nil { return nil, fmt.Errorf("could not get session token: %w", err) } diff --git a/pkg/morph/event/container/delete_notary.go b/pkg/morph/event/container/delete_notary.go index 371f18733..23f13acbb 100644 --- a/pkg/morph/event/container/delete_notary.go +++ b/pkg/morph/event/container/delete_notary.go @@ -7,19 +7,19 @@ import ( func (d *Delete) setContainerID(v []byte) { if v != nil { - d.containerID = v + d.ContainerIDValue = v } } func (d *Delete) setSignature(v []byte) { if v != nil { - d.signature = v + d.SignatureValue = v } } func (d *Delete) setToken(v []byte) { if v != nil { - d.token = v + d.TokenValue = v } } @@ -62,7 +62,7 @@ func ParseDeleteNotary(ne event.NotaryEvent) (event.Event, error) { } } - ev.notaryRequest = ne.Raw() + ev.NotaryRequestValue = ne.Raw() return ev, nil } diff --git a/pkg/morph/event/container/delete_test.go b/pkg/morph/event/container/delete_test.go index 8bf894791..782f4aade 100644 --- a/pkg/morph/event/container/delete_test.go +++ b/pkg/morph/event/container/delete_test.go @@ -63,9 +63,9 @@ func TestParseDelete(t *testing.T) { require.NoError(t, err) require.Equal(t, Delete{ - containerID: containerID, - signature: signature, - token: token, + ContainerIDValue: containerID, + SignatureValue: signature, + TokenValue: token, }, ev) }) } diff --git a/pkg/morph/event/container/eacl.go b/pkg/morph/event/container/eacl.go index 8ef6a71af..41058ea43 100644 --- a/pkg/morph/event/container/eacl.go +++ b/pkg/morph/event/container/eacl.go @@ -12,14 +12,14 @@ import ( // SetEACL represents structure of notification about // modified eACL table coming from FrostFS Container contract. type SetEACL struct { - table []byte - signature []byte - publicKey []byte - token []byte + TableValue []byte + SignatureValue []byte + PublicKeyValue []byte + TokenValue []byte // For notary notifications only. // Contains raw transactions of notary request. - notaryRequest *payload.P2PNotaryRequest + NotaryRequestValue *payload.P2PNotaryRequest } // MorphEvent implements Neo:Morph Event interface. @@ -27,30 +27,30 @@ func (SetEACL) MorphEvent() {} // Table returns returns eACL table in a binary FrostFS API format. func (x SetEACL) Table() []byte { - return x.table + return x.TableValue } // Signature returns signature of the binary table. func (x SetEACL) Signature() []byte { - return x.signature + return x.SignatureValue } // PublicKey returns public keys of container // owner in a binary format. func (x SetEACL) PublicKey() []byte { - return x.publicKey + return x.PublicKeyValue } // SessionToken returns binary token of the session // within which the eACL was set. func (x SetEACL) SessionToken() []byte { - return x.token + return x.TokenValue } // NotaryRequest returns raw notary request if notification // was received via notary service. Otherwise, returns nil. func (x SetEACL) NotaryRequest() *payload.P2PNotaryRequest { - return x.notaryRequest + return x.NotaryRequestValue } const expectedItemNumEACL = 4 @@ -74,25 +74,25 @@ func ParseSetEACL(e *state.ContainedNotificationEvent) (event.Event, error) { } // parse table - ev.table, err = client.BytesFromStackItem(params[0]) + ev.TableValue, err = client.BytesFromStackItem(params[0]) if err != nil { return nil, fmt.Errorf("could not parse binary table: %w", err) } // parse signature - ev.signature, err = client.BytesFromStackItem(params[1]) + ev.SignatureValue, err = client.BytesFromStackItem(params[1]) if err != nil { return nil, fmt.Errorf("could not parse table signature: %w", err) } // parse public key - ev.publicKey, err = client.BytesFromStackItem(params[2]) + ev.PublicKeyValue, err = client.BytesFromStackItem(params[2]) if err != nil { return nil, fmt.Errorf("could not parse binary public key: %w", err) } // parse session token - ev.token, err = client.BytesFromStackItem(params[3]) + ev.TokenValue, err = client.BytesFromStackItem(params[3]) if err != nil { return nil, fmt.Errorf("could not get session token: %w", err) } diff --git a/pkg/morph/event/container/eacl_notary.go b/pkg/morph/event/container/eacl_notary.go index 112065b42..a4fe7c966 100644 --- a/pkg/morph/event/container/eacl_notary.go +++ b/pkg/morph/event/container/eacl_notary.go @@ -7,25 +7,25 @@ import ( func (x *SetEACL) setTable(v []byte) { if v != nil { - x.table = v + x.TableValue = v } } func (x *SetEACL) setSignature(v []byte) { if v != nil { - x.signature = v + x.SignatureValue = v } } func (x *SetEACL) setPublicKey(v []byte) { if v != nil { - x.publicKey = v + x.PublicKeyValue = v } } func (x *SetEACL) setToken(v []byte) { if v != nil { - x.token = v + x.TokenValue = v } } @@ -69,7 +69,7 @@ func ParseSetEACLNotary(ne event.NotaryEvent) (event.Event, error) { } } - ev.notaryRequest = ne.Raw() + ev.NotaryRequestValue = ne.Raw() return ev, nil } From 31b4da225a63f60e3ad3f7604bf0138dda9ac502 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 26 Apr 2023 16:25:50 +0300 Subject: [PATCH 0196/1943] [#280] ir: Add frostfs processor unit tests Signed-off-by: Dmitrii Stepanov --- pkg/innerring/processors/frostfs/handlers.go | 12 +- .../processors/frostfs/handlers_test.go | 371 ++++++++++++++++++ .../processors/frostfs/process_assets.go | 6 +- .../processors/frostfs/process_bind.go | 6 +- .../processors/frostfs/process_config.go | 2 +- pkg/innerring/processors/frostfs/processor.go | 38 +- pkg/morph/event/frostfs/bind.go | 34 +- pkg/morph/event/frostfs/cheque.go | 24 +- pkg/morph/event/frostfs/cheque_test.go | 8 +- pkg/morph/event/frostfs/config.go | 26 +- pkg/morph/event/frostfs/config_test.go | 6 +- pkg/morph/event/frostfs/deposit.go | 24 +- pkg/morph/event/frostfs/deposit_test.go | 8 +- pkg/morph/event/frostfs/unbind.go | 6 +- pkg/morph/event/frostfs/withdraw.go | 18 +- pkg/morph/event/frostfs/withdraw_test.go | 6 +- 16 files changed, 491 insertions(+), 104 deletions(-) create mode 100644 pkg/innerring/processors/frostfs/handlers_test.go diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index 4822cac2c..574cf057a 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -18,7 +18,7 @@ func (np *Processor) handleDeposit(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processDeposit(&deposit) }) + err := np.pool.Submit(func() { np.processDeposit(deposit) }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -34,7 +34,7 @@ func (np *Processor) handleWithdraw(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processWithdraw(&withdraw) }) + err := np.pool.Submit(func() { np.processWithdraw(withdraw) }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -50,7 +50,7 @@ func (np *Processor) handleCheque(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processCheque(&cheque) }) + err := np.pool.Submit(func() { np.processCheque(cheque) }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -67,7 +67,7 @@ func (np *Processor) handleConfig(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processConfig(&cfg) }) + err := np.pool.Submit(func() { np.processConfig(cfg) }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -83,7 +83,7 @@ func (np *Processor) handleBind(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processBind(e) }) + err := np.pool.Submit(func() { np.processBind(e, true) }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -99,7 +99,7 @@ func (np *Processor) handleUnbind(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processBind(e) }) + err := np.pool.Submit(func() { np.processBind(e, false) }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, diff --git a/pkg/innerring/processors/frostfs/handlers_test.go b/pkg/innerring/processors/frostfs/handlers_test.go new file mode 100644 index 000000000..ee2a00427 --- /dev/null +++ b/pkg/innerring/processors/frostfs/handlers_test.go @@ -0,0 +1,371 @@ +package frostfs + +import ( + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" + nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/stretchr/testify/require" +) + +func TestHandleDeposit(t *testing.T) { + t.Parallel() + es := &testEpochState{ + epochCounter: 100, + } + b := &testBalaceClient{} + m := &testMorphClient{ + balance: 150, + } + proc, err := newTestProc(t, func(p *Params) { + p.EpochState = es + p.BalanceClient = b + p.MorphClient = m + }) + require.NoError(t, err, "failed to create processor") + + ev := frostfsEvent.Deposit{ + IDValue: []byte{1, 2, 3, 4, 5}, + FromValue: util.Uint160{100}, + ToValue: util.Uint160{200}, + AmountValue: 1000, + } + + proc.handleDeposit(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + var expMint balance.MintPrm + expMint.SetAmount(ev.AmountValue) + expMint.SetID(ev.IDValue) + expMint.SetTo(ev.ToValue) + + require.EqualValues(t, []balance.MintPrm{expMint}, b.mint, "invalid mint value") + require.EqualValues(t, []transferGas{ + { + receiver: ev.ToValue, + amount: fixedn.Fixed8(50), + }, + }, m.transferGas, "invalid transfer gas") + + es.epochCounter = 109 + + proc.handleDeposit(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + expMint.SetAmount(ev.AmountValue) + expMint.SetID(ev.IDValue) + expMint.SetTo(ev.ToValue) + + require.EqualValues(t, []balance.MintPrm{expMint, expMint}, b.mint, "invalid mint value") + require.EqualValues(t, []transferGas{ + { + receiver: ev.ToValue, + amount: fixedn.Fixed8(50), + }, + }, m.transferGas, "invalid transfer gas") +} + +func TestHandleWithdraw(t *testing.T) { + t.Parallel() + es := &testEpochState{ + epochCounter: 100, + } + b := &testBalaceClient{} + m := &testMorphClient{ + balance: 150, + } + proc, err := newTestProc(t, func(p *Params) { + p.EpochState = es + p.BalanceClient = b + p.MorphClient = m + }) + require.NoError(t, err, "failed to create processor") + + ev := frostfsEvent.Withdraw{ + IDValue: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, + UserValue: util.Uint160{100}, + AmountValue: 1000, + } + + proc.handleWithdraw(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + lock, err := util.Uint160DecodeBytesBE(ev.ID()[:util.Uint160Size]) + require.NoError(t, err, "failed to decode ID") + var expLock balance.LockPrm + expLock.SetAmount(ev.AmountValue) + expLock.SetID(ev.IDValue) + expLock.SetDueEpoch(int64(es.epochCounter) + int64(lockAccountLifetime)) + expLock.SetLock(lock) + expLock.SetUser(ev.UserValue) + + require.EqualValues(t, []balance.LockPrm{expLock}, b.lock, "invalid lock value") +} + +func TestHandleCheque(t *testing.T) { + t.Parallel() + es := &testEpochState{ + epochCounter: 100, + } + b := &testBalaceClient{} + m := &testMorphClient{ + balance: 150, + } + proc, err := newTestProc(t, func(p *Params) { + p.BalanceClient = b + p.MorphClient = m + p.EpochState = es + }) + require.NoError(t, err, "failed to create processor") + + ev := frostfsEvent.Cheque{ + IDValue: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, + UserValue: util.Uint160{100}, + AmountValue: 1000, + LockValue: util.Uint160{200}, + } + + proc.handleCheque(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + var expBurn balance.BurnPrm + expBurn.SetAmount(ev.AmountValue) + expBurn.SetID(ev.IDValue) + expBurn.SetTo(util.Uint160{200}) + + require.EqualValues(t, []balance.BurnPrm{expBurn}, b.burn, "invalid burn value") +} + +func TestHandleConfig(t *testing.T) { + t.Parallel() + es := &testEpochState{ + epochCounter: 100, + } + nm := &testNetmapClient{} + m := &testMorphClient{ + balance: 150, + } + proc, err := newTestProc(t, func(p *Params) { + p.NetmapClient = nm + p.MorphClient = m + p.EpochState = es + }) + require.NoError(t, err, "failed to create processor") + + ev := frostfsEvent.Config{ + IDValue: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, + KeyValue: []byte{1, 2, 3, 4, 5}, + ValueValue: []byte{6, 7, 8, 9, 0}, + TxHashValue: util.Uint256{100}, + } + + proc.handleConfig(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + var expConfig nmClient.SetConfigPrm + expConfig.SetHash(ev.TxHashValue) + expConfig.SetID(ev.IDValue) + expConfig.SetKey(ev.KeyValue) + expConfig.SetValue(ev.ValueValue) + + require.EqualValues(t, []nmClient.SetConfigPrm{expConfig}, nm.config, "invalid config value") +} + +func TestHandleUnbind(t *testing.T) { + t.Parallel() + es := &testEpochState{ + epochCounter: 100, + } + m := &testMorphClient{ + balance: 150, + } + id := &testIDClient{} + proc, err := newTestProc(t, func(p *Params) { + p.EpochState = es + p.MorphClient = m + p.FrostFSIDClient = id + }) + require.NoError(t, err, "failed to create processor") + + p, err := keys.NewPrivateKey() + require.NoError(t, err) + + evUnbind := frostfsEvent.Unbind{ + BindCommon: frostfsEvent.BindCommon{ + UserValue: util.Uint160{49}.BytesBE(), + KeysValue: [][]byte{ + p.PublicKey().Bytes(), + }, + TxHashValue: util.Uint256{100}, + }, + } + + proc.handleUnbind(evUnbind) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + var userID user.ID + userID.SetScriptHash(util.Uint160{49}) + + var expBind frostfsid.CommonBindPrm + expBind.SetOwnerID(userID.WalletBytes()) + expBind.SetKeys(evUnbind.BindCommon.KeysValue) + expBind.SetHash(evUnbind.BindCommon.TxHashValue) + + var expNilSlice []frostfsid.CommonBindPrm + + require.EqualValues(t, []frostfsid.CommonBindPrm{expBind}, id.remove, "invalid remove keys value") + require.EqualValues(t, expNilSlice, id.add, "invalid add keys value") + + evBind := frostfsEvent.Bind{ + BindCommon: frostfsEvent.BindCommon{ + UserValue: util.Uint160{49}.BytesBE(), + KeysValue: [][]byte{ + p.PublicKey().Bytes(), + }, + TxHashValue: util.Uint256{100}, + }, + } + + proc.handleBind(evBind) + + time.Sleep(time.Second) + + require.EqualValues(t, []frostfsid.CommonBindPrm{expBind}, id.remove, "invalid remove keys value") + require.EqualValues(t, []frostfsid.CommonBindPrm{expBind}, id.add, "invalid add keys value") +} + +func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { + p := &Params{ + Log: test.NewLogger(t, true), + PoolSize: 1, + FrostFSContract: util.Uint160{0}, + FrostFSIDClient: &testIDClient{}, + BalanceClient: &testBalaceClient{}, + NetmapClient: &testNetmapClient{}, + MorphClient: &testMorphClient{}, + EpochState: &testEpochState{}, + AlphabetState: &testAlphabetState{isAlphabet: true}, + Converter: &testPrecisionConverter{}, + MintEmitCacheSize: 100, + MintEmitThreshold: 10, + MintEmitValue: fixedn.Fixed8(50), + GasBalanceThreshold: 50, + } + + nonDefault(p) + + return New(p) +} + +type testEpochState struct { + epochCounter uint64 +} + +func (s *testEpochState) EpochCounter() uint64 { + return s.epochCounter +} + +type testAlphabetState struct { + isAlphabet bool +} + +func (s *testAlphabetState) IsAlphabet() bool { + return s.isAlphabet +} + +type testPrecisionConverter struct { +} + +func (c *testPrecisionConverter) ToBalancePrecision(v int64) int64 { + return v +} + +type testBalaceClient struct { + mint []balance.MintPrm + lock []balance.LockPrm + burn []balance.BurnPrm +} + +func (c *testBalaceClient) Mint(p balance.MintPrm) error { + c.mint = append(c.mint, p) + return nil +} +func (c *testBalaceClient) Lock(p balance.LockPrm) error { + c.lock = append(c.lock, p) + return nil +} +func (c *testBalaceClient) Burn(p balance.BurnPrm) error { + c.burn = append(c.burn, p) + return nil +} + +type testNetmapClient struct { + config []nmClient.SetConfigPrm +} + +func (c *testNetmapClient) SetConfig(p nmClient.SetConfigPrm) error { + c.config = append(c.config, p) + return nil +} + +type transferGas struct { + receiver util.Uint160 + amount fixedn.Fixed8 +} + +type testMorphClient struct { + balance int64 + transferGas []transferGas +} + +func (c *testMorphClient) GasBalance() (res int64, err error) { + return c.balance, nil +} +func (c *testMorphClient) TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error { + c.transferGas = append(c.transferGas, transferGas{ + receiver: receiver, + amount: amount, + }) + return nil +} + +type testIDClient struct { + add []frostfsid.CommonBindPrm + remove []frostfsid.CommonBindPrm +} + +func (c *testIDClient) AddKeys(p frostfsid.CommonBindPrm) error { + c.add = append(c.add, p) + return nil +} + +func (c *testIDClient) RemoveKeys(args frostfsid.CommonBindPrm) error { + c.remove = append(c.remove, args) + return nil +} diff --git a/pkg/innerring/processors/frostfs/process_assets.go b/pkg/innerring/processors/frostfs/process_assets.go index e066975f7..cfbf21b08 100644 --- a/pkg/innerring/processors/frostfs/process_assets.go +++ b/pkg/innerring/processors/frostfs/process_assets.go @@ -15,7 +15,7 @@ const ( // Process deposit event by invoking a balance contract and sending native // gas in the sidechain. -func (np *Processor) processDeposit(deposit *frostfsEvent.Deposit) { +func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreDeposit) return @@ -80,7 +80,7 @@ func (np *Processor) processDeposit(deposit *frostfsEvent.Deposit) { } // Process withdraw event by locking assets in the balance account. -func (np *Processor) processWithdraw(withdraw *frostfsEvent.Withdraw) { +func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreWithdraw) return @@ -111,7 +111,7 @@ func (np *Processor) processWithdraw(withdraw *frostfsEvent.Withdraw) { // Process cheque event by transferring assets from the lock account back to // the reserve account. -func (np *Processor) processCheque(cheque *frostfsEvent.Cheque) { +func (np *Processor) processCheque(cheque frostfsEvent.Cheque) { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreCheque) return diff --git a/pkg/innerring/processors/frostfs/process_bind.go b/pkg/innerring/processors/frostfs/process_bind.go index c5f8a930e..a9b523a77 100644 --- a/pkg/innerring/processors/frostfs/process_bind.go +++ b/pkg/innerring/processors/frostfs/process_bind.go @@ -6,7 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" @@ -19,7 +18,7 @@ type bindCommon interface { TxHash() util.Uint256 } -func (np *Processor) processBind(e bindCommon) { +func (np *Processor) processBind(e bindCommon, bind bool) { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreBind) return @@ -27,10 +26,9 @@ func (np *Processor) processBind(e bindCommon) { c := &bindCommonContext{ bindCommon: e, + bind: bind, } - _, c.bind = e.(frostfs.Bind) - err := np.checkBindCommon(c) if err != nil { np.log.Error(logs.FrostFSInvalidManageKeyEvent, diff --git a/pkg/innerring/processors/frostfs/process_config.go b/pkg/innerring/processors/frostfs/process_config.go index 471edb9b7..ce2dabfdd 100644 --- a/pkg/innerring/processors/frostfs/process_config.go +++ b/pkg/innerring/processors/frostfs/process_config.go @@ -9,7 +9,7 @@ import ( // Process config event by setting configuration value from the mainchain in // the sidechain. -func (np *Processor) processConfig(config *frostfsEvent.Config) { +func (np *Processor) processConfig(config frostfsEvent.Config) { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreConfig) return diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index 42362eeed..2af15a814 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -6,7 +6,6 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -36,14 +35,34 @@ type ( ToBalancePrecision(int64) int64 } + BalanceClient interface { + Mint(p balance.MintPrm) error + Lock(p balance.LockPrm) error + Burn(p balance.BurnPrm) error + } + + NetmapClient interface { + SetConfig(p nmClient.SetConfigPrm) error + } + + MorphClient interface { + GasBalance() (res int64, err error) + TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error + } + + IDClient interface { + AddKeys(p frostfsid.CommonBindPrm) error + RemoveKeys(args frostfsid.CommonBindPrm) error + } + // Processor of events produced by frostfs contract in main net. Processor struct { log *logger.Logger pool *ants.Pool frostfsContract util.Uint160 - balanceClient *balance.Client - netmapClient *nmClient.Client - morphClient *client.Client + balanceClient BalanceClient + netmapClient NetmapClient + morphClient MorphClient epochState EpochState alphabetState AlphabetState converter PrecisionConverter @@ -52,8 +71,7 @@ type ( mintEmitThreshold uint64 mintEmitValue fixedn.Fixed8 gasBalanceThreshold int64 - - frostfsIDClient *frostfsid.Client + frostfsIDClient IDClient } // Params of the processor constructor. @@ -61,10 +79,10 @@ type ( Log *logger.Logger PoolSize int FrostFSContract util.Uint160 - FrostFSIDClient *frostfsid.Client - BalanceClient *balance.Client - NetmapClient *nmClient.Client - MorphClient *client.Client + FrostFSIDClient IDClient + BalanceClient BalanceClient + NetmapClient NetmapClient + MorphClient MorphClient EpochState EpochState AlphabetState AlphabetState Converter PrecisionConverter diff --git a/pkg/morph/event/frostfs/bind.go b/pkg/morph/event/frostfs/bind.go index 49d10d3c3..8655b1222 100644 --- a/pkg/morph/event/frostfs/bind.go +++ b/pkg/morph/event/frostfs/bind.go @@ -11,31 +11,31 @@ import ( ) type Bind struct { - bindCommon + BindCommon } -type bindCommon struct { - user []byte - keys [][]byte +type BindCommon struct { + UserValue []byte + KeysValue [][]byte - // txHash is used in notary environmental + // TxHashValue is used in notary environmental // for calculating unique but same for // all notification receivers values. - txHash util.Uint256 + TxHashValue util.Uint256 } // TxHash returns hash of the TX with new epoch // notification. -func (b bindCommon) TxHash() util.Uint256 { - return b.txHash +func (b BindCommon) TxHash() util.Uint256 { + return b.TxHashValue } // MorphEvent implements Neo:Morph Event interface. -func (bindCommon) MorphEvent() {} +func (BindCommon) MorphEvent() {} -func (b bindCommon) Keys() [][]byte { return b.keys } +func (b BindCommon) Keys() [][]byte { return b.KeysValue } -func (b bindCommon) User() []byte { return b.user } +func (b BindCommon) User() []byte { return b.UserValue } func ParseBind(e *state.ContainedNotificationEvent) (event.Event, error) { var ( @@ -48,17 +48,17 @@ func ParseBind(e *state.ContainedNotificationEvent) (event.Event, error) { return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) } - err = parseBind(&ev.bindCommon, params) + err = parseBind(&ev.BindCommon, params) if err != nil { return nil, err } - ev.txHash = e.Container + ev.TxHashValue = e.Container return ev, nil } -func parseBind(dst *bindCommon, params []stackitem.Item) error { +func parseBind(dst *BindCommon, params []stackitem.Item) error { if ln := len(params); ln != 2 { return event.WrongNumberOfParameters(2, ln) } @@ -66,7 +66,7 @@ func parseBind(dst *bindCommon, params []stackitem.Item) error { var err error // parse user - dst.user, err = client.BytesFromStackItem(params[0]) + dst.UserValue, err = client.BytesFromStackItem(params[0]) if err != nil { return fmt.Errorf("could not get bind user: %w", err) } @@ -77,7 +77,7 @@ func parseBind(dst *bindCommon, params []stackitem.Item) error { return fmt.Errorf("could not get bind keys: %w", err) } - dst.keys = make([][]byte, 0, len(bindKeys)) + dst.KeysValue = make([][]byte, 0, len(bindKeys)) for i := range bindKeys { rawKey, err := client.BytesFromStackItem(bindKeys[i]) @@ -85,7 +85,7 @@ func parseBind(dst *bindCommon, params []stackitem.Item) error { return fmt.Errorf("could not get bind public key: %w", err) } - dst.keys = append(dst.keys, rawKey) + dst.KeysValue = append(dst.KeysValue, rawKey) } return nil diff --git a/pkg/morph/event/frostfs/cheque.go b/pkg/morph/event/frostfs/cheque.go index 239ddb1a4..eae2a23f5 100644 --- a/pkg/morph/event/frostfs/cheque.go +++ b/pkg/morph/event/frostfs/cheque.go @@ -11,26 +11,26 @@ import ( // Cheque structure of frostfs.Cheque notification from mainnet chain. type Cheque struct { - id []byte - amount int64 // Fixed8 - user util.Uint160 - lock util.Uint160 + IDValue []byte + AmountValue int64 // Fixed8 + UserValue util.Uint160 + LockValue util.Uint160 } // MorphEvent implements Neo:Morph Event interface. func (Cheque) MorphEvent() {} // ID is a withdraw transaction hash. -func (c Cheque) ID() []byte { return c.id } +func (c Cheque) ID() []byte { return c.IDValue } // User returns withdraw receiver script hash from main net. -func (c Cheque) User() util.Uint160 { return c.user } +func (c Cheque) User() util.Uint160 { return c.UserValue } // Amount of the sent assets. -func (c Cheque) Amount() int64 { return c.amount } +func (c Cheque) Amount() int64 { return c.AmountValue } // LockAccount return script hash for balance contract wallet. -func (c Cheque) LockAccount() util.Uint160 { return c.lock } +func (c Cheque) LockAccount() util.Uint160 { return c.LockValue } // ParseCheque from notification into cheque structure. func ParseCheque(e *state.ContainedNotificationEvent) (event.Event, error) { @@ -49,7 +49,7 @@ func ParseCheque(e *state.ContainedNotificationEvent) (event.Event, error) { } // parse id - ev.id, err = client.BytesFromStackItem(params[0]) + ev.IDValue, err = client.BytesFromStackItem(params[0]) if err != nil { return nil, fmt.Errorf("could not get cheque id: %w", err) } @@ -60,13 +60,13 @@ func ParseCheque(e *state.ContainedNotificationEvent) (event.Event, error) { return nil, fmt.Errorf("could not get cheque user: %w", err) } - ev.user, err = util.Uint160DecodeBytesBE(user) + ev.UserValue, err = util.Uint160DecodeBytesBE(user) if err != nil { return nil, fmt.Errorf("could not convert cheque user to uint160: %w", err) } // parse amount - ev.amount, err = client.IntFromStackItem(params[2]) + ev.AmountValue, err = client.IntFromStackItem(params[2]) if err != nil { return nil, fmt.Errorf("could not get cheque amount: %w", err) } @@ -77,7 +77,7 @@ func ParseCheque(e *state.ContainedNotificationEvent) (event.Event, error) { return nil, fmt.Errorf("could not get cheque lock account: %w", err) } - ev.lock, err = util.Uint160DecodeBytesBE(lock) + ev.LockValue, err = util.Uint160DecodeBytesBE(lock) if err != nil { return nil, fmt.Errorf("could not convert cheque lock account to uint160: %w", err) } diff --git a/pkg/morph/event/frostfs/cheque_test.go b/pkg/morph/event/frostfs/cheque_test.go index 861f05a68..be53592ca 100644 --- a/pkg/morph/event/frostfs/cheque_test.go +++ b/pkg/morph/event/frostfs/cheque_test.go @@ -77,10 +77,10 @@ func TestParseCheque(t *testing.T) { require.NoError(t, err) require.Equal(t, Cheque{ - id: id, - amount: amount, - user: user, - lock: lock, + IDValue: id, + AmountValue: amount, + UserValue: user, + LockValue: lock, }, ev) }) } diff --git a/pkg/morph/event/frostfs/config.go b/pkg/morph/event/frostfs/config.go index 1b9824b39..4c87634c2 100644 --- a/pkg/morph/event/frostfs/config.go +++ b/pkg/morph/event/frostfs/config.go @@ -10,30 +10,30 @@ import ( ) type Config struct { - key []byte - value []byte - id []byte + KeyValue []byte + ValueValue []byte + IDValue []byte - // txHash is used in notary environmental + // TxHashValue is used in notary environmental // for calculating unique but same for // all notification receivers values. - txHash util.Uint256 + TxHashValue util.Uint256 } // TxHash returns hash of the TX with new epoch // notification. func (u Config) TxHash() util.Uint256 { - return u.txHash + return u.TxHashValue } // MorphEvent implements Neo:Morph Event interface. func (Config) MorphEvent() {} -func (u Config) ID() []byte { return u.id } +func (u Config) ID() []byte { return u.IDValue } -func (u Config) Key() []byte { return u.key } +func (u Config) Key() []byte { return u.KeyValue } -func (u Config) Value() []byte { return u.value } +func (u Config) Value() []byte { return u.ValueValue } func ParseConfig(e *state.ContainedNotificationEvent) (event.Event, error) { var ( @@ -51,24 +51,24 @@ func ParseConfig(e *state.ContainedNotificationEvent) (event.Event, error) { } // parse id - ev.id, err = client.BytesFromStackItem(params[0]) + ev.IDValue, err = client.BytesFromStackItem(params[0]) if err != nil { return nil, fmt.Errorf("could not get config update id: %w", err) } // parse key - ev.key, err = client.BytesFromStackItem(params[1]) + ev.KeyValue, err = client.BytesFromStackItem(params[1]) if err != nil { return nil, fmt.Errorf("could not get config key: %w", err) } // parse value - ev.value, err = client.BytesFromStackItem(params[2]) + ev.ValueValue, err = client.BytesFromStackItem(params[2]) if err != nil { return nil, fmt.Errorf("could not get config value: %w", err) } - ev.txHash = e.Container + ev.TxHashValue = e.Container return ev, nil } diff --git a/pkg/morph/event/frostfs/config_test.go b/pkg/morph/event/frostfs/config_test.go index b56c8ecb2..dcd4201e4 100644 --- a/pkg/morph/event/frostfs/config_test.go +++ b/pkg/morph/event/frostfs/config_test.go @@ -60,9 +60,9 @@ func TestParseConfig(t *testing.T) { require.NoError(t, err) require.Equal(t, Config{ - id: id, - key: key, - value: value, + IDValue: id, + KeyValue: key, + ValueValue: value, }, ev) }) } diff --git a/pkg/morph/event/frostfs/deposit.go b/pkg/morph/event/frostfs/deposit.go index b9467d112..d8a3b82f0 100644 --- a/pkg/morph/event/frostfs/deposit.go +++ b/pkg/morph/event/frostfs/deposit.go @@ -11,26 +11,26 @@ import ( // Deposit structure of frostfs.Deposit notification from mainnet chain. type Deposit struct { - id []byte - amount int64 // Fixed8 - from util.Uint160 - to util.Uint160 + IDValue []byte + AmountValue int64 // Fixed8 + FromValue util.Uint160 + ToValue util.Uint160 } // MorphEvent implements Neo:Morph Event interface. func (Deposit) MorphEvent() {} // ID is a deposit transaction hash. -func (d Deposit) ID() []byte { return d.id } +func (d Deposit) ID() []byte { return d.IDValue } // From is a script hash of asset sender in main net. -func (d Deposit) From() util.Uint160 { return d.from } +func (d Deposit) From() util.Uint160 { return d.FromValue } // To is a script hash of asset receiver in balance contract. -func (d Deposit) To() util.Uint160 { return d.to } +func (d Deposit) To() util.Uint160 { return d.ToValue } // Amount of transferred assets. -func (d Deposit) Amount() int64 { return d.amount } +func (d Deposit) Amount() int64 { return d.AmountValue } // ParseDeposit notification into deposit structure. func ParseDeposit(e *state.ContainedNotificationEvent) (event.Event, error) { @@ -51,13 +51,13 @@ func ParseDeposit(e *state.ContainedNotificationEvent) (event.Event, error) { return nil, fmt.Errorf("could not get deposit sender: %w", err) } - ev.from, err = util.Uint160DecodeBytesBE(from) + ev.FromValue, err = util.Uint160DecodeBytesBE(from) if err != nil { return nil, fmt.Errorf("could not convert deposit sender to uint160: %w", err) } // parse amount - ev.amount, err = client.IntFromStackItem(params[1]) + ev.AmountValue, err = client.IntFromStackItem(params[1]) if err != nil { return nil, fmt.Errorf("could not get deposit amount: %w", err) } @@ -68,13 +68,13 @@ func ParseDeposit(e *state.ContainedNotificationEvent) (event.Event, error) { return nil, fmt.Errorf("could not get deposit receiver: %w", err) } - ev.to, err = util.Uint160DecodeBytesBE(to) + ev.ToValue, err = util.Uint160DecodeBytesBE(to) if err != nil { return nil, fmt.Errorf("could not convert deposit receiver to uint160: %w", err) } // parse id - ev.id, err = client.BytesFromStackItem(params[3]) + ev.IDValue, err = client.BytesFromStackItem(params[3]) if err != nil { return nil, fmt.Errorf("could not get deposit id: %w", err) } diff --git a/pkg/morph/event/frostfs/deposit_test.go b/pkg/morph/event/frostfs/deposit_test.go index 0f52e2119..f279a7f9c 100644 --- a/pkg/morph/event/frostfs/deposit_test.go +++ b/pkg/morph/event/frostfs/deposit_test.go @@ -77,10 +77,10 @@ func TestParseDeposit(t *testing.T) { require.NoError(t, err) require.Equal(t, Deposit{ - id: id, - amount: amount, - from: from, - to: to, + IDValue: id, + AmountValue: amount, + FromValue: from, + ToValue: to, }, ev) }) } diff --git a/pkg/morph/event/frostfs/unbind.go b/pkg/morph/event/frostfs/unbind.go index f88d67994..5a6a8dad9 100644 --- a/pkg/morph/event/frostfs/unbind.go +++ b/pkg/morph/event/frostfs/unbind.go @@ -8,7 +8,7 @@ import ( ) type Unbind struct { - bindCommon + BindCommon } func ParseUnbind(e *state.ContainedNotificationEvent) (event.Event, error) { @@ -22,12 +22,12 @@ func ParseUnbind(e *state.ContainedNotificationEvent) (event.Event, error) { return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) } - err = parseBind(&ev.bindCommon, params) + err = parseBind(&ev.BindCommon, params) if err != nil { return nil, err } - ev.txHash = e.Container + ev.TxHashValue = e.Container return ev, nil } diff --git a/pkg/morph/event/frostfs/withdraw.go b/pkg/morph/event/frostfs/withdraw.go index 3bbf76c2c..f48067f86 100644 --- a/pkg/morph/event/frostfs/withdraw.go +++ b/pkg/morph/event/frostfs/withdraw.go @@ -11,22 +11,22 @@ import ( // Withdraw structure of frostfs.Withdraw notification from mainnet chain. type Withdraw struct { - id []byte - amount int64 // Fixed8 - user util.Uint160 + IDValue []byte + AmountValue int64 // Fixed8 + UserValue util.Uint160 } // MorphEvent implements Neo:Morph Event interface. func (Withdraw) MorphEvent() {} // ID is a withdraw transaction hash. -func (w Withdraw) ID() []byte { return w.id } +func (w Withdraw) ID() []byte { return w.IDValue } // User returns withdraw receiver script hash from main net. -func (w Withdraw) User() util.Uint160 { return w.user } +func (w Withdraw) User() util.Uint160 { return w.UserValue } // Amount of the withdraw assets. -func (w Withdraw) Amount() int64 { return w.amount } +func (w Withdraw) Amount() int64 { return w.AmountValue } // ParseWithdraw notification into withdraw structure. func ParseWithdraw(e *state.ContainedNotificationEvent) (event.Event, error) { @@ -47,19 +47,19 @@ func ParseWithdraw(e *state.ContainedNotificationEvent) (event.Event, error) { return nil, fmt.Errorf("could not get withdraw user: %w", err) } - ev.user, err = util.Uint160DecodeBytesBE(user) + ev.UserValue, err = util.Uint160DecodeBytesBE(user) if err != nil { return nil, fmt.Errorf("could not convert withdraw user to uint160: %w", err) } // parse amount - ev.amount, err = client.IntFromStackItem(params[1]) + ev.AmountValue, err = client.IntFromStackItem(params[1]) if err != nil { return nil, fmt.Errorf("could not get withdraw amount: %w", err) } // parse id - ev.id, err = client.BytesFromStackItem(params[2]) + ev.IDValue, err = client.BytesFromStackItem(params[2]) if err != nil { return nil, fmt.Errorf("could not get withdraw id: %w", err) } diff --git a/pkg/morph/event/frostfs/withdraw_test.go b/pkg/morph/event/frostfs/withdraw_test.go index 5544283ef..33435d19a 100644 --- a/pkg/morph/event/frostfs/withdraw_test.go +++ b/pkg/morph/event/frostfs/withdraw_test.go @@ -64,9 +64,9 @@ func TestParseWithdraw(t *testing.T) { require.NoError(t, err) require.Equal(t, Withdraw{ - id: id, - amount: amount, - user: user, + IDValue: id, + AmountValue: amount, + UserValue: user, }, ev) }) } From fb5dcc15d2bdb0be70a3979d04d96e28798f9207 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 27 Apr 2023 11:25:17 +0300 Subject: [PATCH 0197/1943] [#280] ir: Add governance processor unit tests Signed-off-by: Dmitrii Stepanov --- .../processors/governance/handlers_test.go | 304 ++++++++++++++++++ .../processors/governance/processor.go | 35 +- 2 files changed, 331 insertions(+), 8 deletions(-) create mode 100644 pkg/innerring/processors/governance/handlers_test.go diff --git a/pkg/innerring/processors/governance/handlers_test.go b/pkg/innerring/processors/governance/handlers_test.go new file mode 100644 index 000000000..23f91869a --- /dev/null +++ b/pkg/innerring/processors/governance/handlers_test.go @@ -0,0 +1,304 @@ +package governance + +import ( + "encoding/binary" + "sort" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" + nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/rolemanagement" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/stretchr/testify/require" +) + +func TestHandleAlphabetSyncEvent(t *testing.T) { + t.Parallel() + testKeys := generateTestKeys(t) + + es := &testEpochState{ + epoch: 100, + } + as := &testAlphabetState{ + isAlphabet: true, + } + v := &testVoter{} + irf := &testIRFetcher{ + publicKeys: testKeys.sidechainKeys, + } + m := &testMorphClient{ + commiteeKeys: testKeys.sidechainKeys, + } + mn := &testMainnetClient{ + alphabetKeys: testKeys.mainnetKeys, + } + f := &testFrostFSClient{} + nm := &testNetmapClient{} + + proc, err := New( + &Params{ + Log: test.NewLogger(t, true), + EpochState: es, + AlphabetState: as, + Voter: v, + IRFetcher: irf, + NotaryDisabled: true, + MorphClient: m, + MainnetClient: mn, + FrostFSClient: f, + NetmapClient: nm, + }, + ) + + require.NoError(t, err, "failed to create processor") + + ev := Sync{ + txHash: util.Uint256{100}, + } + + proc.HandleAlphabetSync(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + require.EqualValues(t, []VoteValidatorPrm{ + { + Validators: testKeys.newAlphabetExp, + Hash: &ev.txHash, + }, + }, v.votes, "invalid vote calls") + + var irUpdateExp nmClient.UpdateIRPrm + irUpdateExp.SetKeys(testKeys.newInnerRingExp) + irUpdateExp.SetHash(ev.txHash) + + require.EqualValues(t, []nmClient.UpdateIRPrm{irUpdateExp}, nm.updates, "invalid IR updates") + + var expAlphabetUpdates []client.UpdateAlphabetListPrm + require.EqualValues(t, expAlphabetUpdates, m.alphabetUpdates, "invalid alphabet updates") + + var expNotaryUpdates []client.UpdateNotaryListPrm + require.EqualValues(t, expNotaryUpdates, m.notaryUpdates, "invalid notary list updates") + + buf := make([]byte, 8) + binary.LittleEndian.PutUint64(buf, es.epoch) + + id := append([]byte(alphabetUpdateIDPrefix), buf...) + var expFrostFSAlphabetUpd frostfscontract.AlphabetUpdatePrm + expFrostFSAlphabetUpd.SetID(id) + expFrostFSAlphabetUpd.SetPubs(testKeys.newAlphabetExp) + + require.EqualValues(t, []frostfscontract.AlphabetUpdatePrm{expFrostFSAlphabetUpd}, f.updates, "invalid FrostFS alphabet updates") +} + +func TestHandleAlphabetDesignateEvent(t *testing.T) { + t.Parallel() + testKeys := generateTestKeys(t) + + es := &testEpochState{ + epoch: 100, + } + as := &testAlphabetState{ + isAlphabet: true, + } + v := &testVoter{} + irf := &testIRFetcher{ + publicKeys: testKeys.sidechainKeys, + } + m := &testMorphClient{ + commiteeKeys: testKeys.sidechainKeys, + } + mn := &testMainnetClient{ + alphabetKeys: testKeys.mainnetKeys, + } + f := &testFrostFSClient{} + nm := &testNetmapClient{} + + proc, err := New( + &Params{ + Log: test.NewLogger(t, true), + EpochState: es, + AlphabetState: as, + Voter: v, + IRFetcher: irf, + NotaryDisabled: false, + MorphClient: m, + MainnetClient: mn, + FrostFSClient: f, + NetmapClient: nm, + }, + ) + + require.NoError(t, err, "failed to create processor") + + ev := rolemanagement.Designate{ + TxHash: util.Uint256{100}, + Role: noderoles.NeoFSAlphabet, + } + + proc.HandleAlphabetSync(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + require.EqualValues(t, []VoteValidatorPrm{ + { + Validators: testKeys.newAlphabetExp, + Hash: &ev.TxHash, + }, + }, v.votes, "invalid vote calls") + + var irUpdatesExp []nmClient.UpdateIRPrm + require.EqualValues(t, irUpdatesExp, nm.updates, "invalid IR updates") + + var alpabetUpdExp client.UpdateAlphabetListPrm + alpabetUpdExp.SetList(testKeys.newInnerRingExp) + alpabetUpdExp.SetHash(ev.TxHash) + require.EqualValues(t, []client.UpdateAlphabetListPrm{alpabetUpdExp}, m.alphabetUpdates, "invalid alphabet updates") + + var expNotaryUpdate client.UpdateNotaryListPrm + expNotaryUpdate.SetList(testKeys.newAlphabetExp) + expNotaryUpdate.SetHash(ev.TxHash) + require.EqualValues(t, []client.UpdateNotaryListPrm{expNotaryUpdate}, m.notaryUpdates, "invalid notary list updates") + + buf := make([]byte, 8) + binary.LittleEndian.PutUint64(buf, es.epoch) + + id := append([]byte(alphabetUpdateIDPrefix), buf...) + var expFrostFSAlphabetUpd frostfscontract.AlphabetUpdatePrm + expFrostFSAlphabetUpd.SetID(id) + expFrostFSAlphabetUpd.SetPubs(testKeys.newAlphabetExp) + + require.EqualValues(t, []frostfscontract.AlphabetUpdatePrm{expFrostFSAlphabetUpd}, f.updates, "invalid FrostFS alphabet updates") +} + +type testKeys struct { + sidechainKeys keys.PublicKeys + mainnetKeys keys.PublicKeys + newAlphabetExp keys.PublicKeys + newInnerRingExp keys.PublicKeys +} + +func generateTestKeys(t *testing.T) testKeys { + for { + var result testKeys + + for i := 0; i < 4; i++ { + pk, err := keys.NewPrivateKey() + require.NoError(t, err, "failed to create private key") + result.sidechainKeys = append(result.sidechainKeys, pk.PublicKey()) + } + + result.mainnetKeys = append(result.mainnetKeys, result.sidechainKeys...) + pk, err := keys.NewPrivateKey() + require.NoError(t, err, "failed to create private key") + result.mainnetKeys = append(result.mainnetKeys, pk.PublicKey()) + + result.newAlphabetExp, err = newAlphabetList(result.sidechainKeys, result.mainnetKeys) + require.NoError(t, err, "failed to create expected new alphabet") + + if len(result.newAlphabetExp) == 0 { + continue //can be happen because of random and sort + } + + var irKeys keys.PublicKeys + irKeys = append(irKeys, result.sidechainKeys...) + result.newInnerRingExp, err = updateInnerRing(irKeys, result.sidechainKeys, result.newAlphabetExp) + require.NoError(t, err, "failed to create expected new IR") + sort.Sort(result.newInnerRingExp) + + return result + } +} + +type testEpochState struct { + epoch uint64 +} + +func (s *testEpochState) EpochCounter() uint64 { + return s.epoch +} + +type testAlphabetState struct { + isAlphabet bool +} + +func (s *testAlphabetState) IsAlphabet() bool { + return s.isAlphabet +} + +type testVoter struct { + votes []VoteValidatorPrm +} + +func (v *testVoter) VoteForSidechainValidator(prm VoteValidatorPrm) error { + v.votes = append(v.votes, prm) + return nil +} + +type testIRFetcher struct { + publicKeys keys.PublicKeys +} + +func (f *testIRFetcher) InnerRingKeys() (keys.PublicKeys, error) { + return f.publicKeys, nil +} + +type testMorphClient struct { + commiteeKeys keys.PublicKeys + + alphabetUpdates []client.UpdateAlphabetListPrm + notaryUpdates []client.UpdateNotaryListPrm +} + +func (c *testMorphClient) Committee() (res keys.PublicKeys, err error) { + return c.commiteeKeys, nil +} + +func (c *testMorphClient) UpdateNeoFSAlphabetList(prm client.UpdateAlphabetListPrm) error { + c.alphabetUpdates = append(c.alphabetUpdates, prm) + return nil +} + +func (c *testMorphClient) UpdateNotaryList(prm client.UpdateNotaryListPrm) error { + c.notaryUpdates = append(c.notaryUpdates, prm) + return nil +} + +type testMainnetClient struct { + alphabetKeys keys.PublicKeys + designateHash util.Uint160 +} + +func (c *testMainnetClient) NeoFSAlphabetList() (res keys.PublicKeys, err error) { + return c.alphabetKeys, nil +} + +func (c *testMainnetClient) GetDesignateHash() util.Uint160 { + return c.designateHash +} + +type testFrostFSClient struct { + updates []frostfscontract.AlphabetUpdatePrm +} + +func (c *testFrostFSClient) AlphabetUpdate(p frostfscontract.AlphabetUpdatePrm) error { + c.updates = append(c.updates, p) + return nil +} + +type testNetmapClient struct { + updates []nmClient.UpdateIRPrm +} + +func (c *testNetmapClient) UpdateInnerRing(p nmClient.UpdateIRPrm) error { + c.updates = append(c.updates, p) + return nil +} diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index b65dd17b7..e08bd3809 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -53,20 +53,39 @@ type ( InnerRingKeys() (keys.PublicKeys, error) } + FrostFSClient interface { + AlphabetUpdate(p frostfscontract.AlphabetUpdatePrm) error + } + + NetmapClient interface { + UpdateInnerRing(p nmClient.UpdateIRPrm) error + } + + MainnetClient interface { + NeoFSAlphabetList() (res keys.PublicKeys, err error) + GetDesignateHash() util.Uint160 + } + + MorphClient interface { + Committee() (res keys.PublicKeys, err error) + UpdateNeoFSAlphabetList(prm client.UpdateAlphabetListPrm) error + UpdateNotaryList(prm client.UpdateNotaryListPrm) error + } + // Processor of events related to governance in the network. Processor struct { log *logger.Logger pool *ants.Pool - frostfsClient *frostfscontract.Client - netmapClient *nmClient.Client + frostfsClient FrostFSClient + netmapClient NetmapClient alphabetState AlphabetState epochState EpochState voter Voter irFetcher IRFetcher - mainnetClient *client.Client - morphClient *client.Client + mainnetClient MainnetClient + morphClient MorphClient notaryDisabled bool @@ -82,10 +101,10 @@ type ( Voter Voter IRFetcher IRFetcher - MorphClient *client.Client - MainnetClient *client.Client - FrostFSClient *frostfscontract.Client - NetmapClient *nmClient.Client + MorphClient MorphClient + MainnetClient MainnetClient + FrostFSClient FrostFSClient + NetmapClient NetmapClient NotaryDisabled bool } From d00b1c0d2988403df3e0b7485d58abcab7052a8c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 27 Apr 2023 17:57:27 +0300 Subject: [PATCH 0198/1943] [#280] ir: Add netmap processor unit tests Signed-off-by: Dmitrii Stepanov --- pkg/innerring/initialization.go | 2 +- .../processors/netmap/handlers_test.go | 570 ++++++++++++++++++ .../processors/netmap/process_cleanup.go | 2 +- .../processors/netmap/process_epoch.go | 2 +- .../processors/netmap/process_peers.go | 6 +- pkg/innerring/processors/netmap/processor.go | 32 +- pkg/innerring/processors/netmap/wrappers.go | 59 ++ pkg/morph/event/netmap/add_peer.go | 10 +- pkg/morph/event/netmap/add_peer_notary.go | 4 +- pkg/morph/event/netmap/add_peer_test.go | 2 +- pkg/morph/event/netmap/epoch.go | 14 +- pkg/morph/event/netmap/epoch_test.go | 2 +- pkg/morph/event/netmap/update_peer.go | 18 +- pkg/morph/event/netmap/update_peer_notary.go | 4 +- pkg/morph/event/netmap/update_peer_test.go | 4 +- 15 files changed, 690 insertions(+), 41 deletions(-) create mode 100644 pkg/innerring/processors/netmap/handlers_test.go create mode 100644 pkg/innerring/processors/netmap/wrappers.go diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index cb9b10648..84d08c4c6 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -54,7 +54,7 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, s.netmapProcessor, err = netmap.New(&netmap.Params{ Log: s.log, PoolSize: cfg.GetInt("workers.netmap"), - NetmapClient: s.netmapClient, + NetmapClient: netmap.NewNetmapClient(s.netmapClient), EpochTimer: s, EpochState: s, AlphabetState: s, diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go new file mode 100644 index 000000000..576d6c5dd --- /dev/null +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -0,0 +1,570 @@ +package netmap + +import ( + "fmt" + "testing" + "time" + + v2netmap "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" + netmapContract "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/audit" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement" + timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" + cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" + netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" + "github.com/nspcc-dev/neo-go/pkg/network/payload" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/stretchr/testify/require" +) + +func TestNewEpochTick(t *testing.T) { + t.Parallel() + es := &testEpochState{ + counter: 100, + } + nc := &testNetmapClient{} + + proc, err := newTestProc(t, func(p *Params) { + p.NotaryDisabled = true + p.CleanupEnabled = true + p.EpochState = es + p.NetmapClient = nc + }) + + require.NoError(t, err, "failed to create processor") + + ev := timerEvent.NewEpochTick{} + proc.HandleNewEpochTick(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + require.EqualValues(t, []uint64{101}, nc.newEpochs, "invalid epochs") +} + +func TestNewEpoch(t *testing.T) { + t.Parallel() + var node1 netmap.NodeInfo + key1, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key1") + node1.SetPublicKey(key1.Bytes()) + + var node2 netmap.NodeInfo + key2, err := keys.NewPublicKeyFromString("02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3") + require.NoError(t, err, "failed to parse key2") + node2.SetPublicKey(key2.Bytes()) + + network := &netmap.NetMap{} + network.SetNodes([]netmap.NodeInfo{node1, node2}) + + es := &testEpochState{ + counter: 100, + duration: 10, + } + r := &testEpochResetter{} + cc := &testContainerClient{} + nc := &testNetmapClient{ + epochDuration: 20, + txHeights: map[util.Uint256]uint32{ + {101}: 10_000, + }, + netmap: network, + } + eh := &testEventHandler{} + + proc, err := newTestProc(t, func(p *Params) { + p.NotaryDisabled = true + p.NotaryDepositHandler = eh.Handle + p.HandleAudit = eh.Handle + p.AuditSettlementsHandler = eh.Handle + p.AlphabetSyncHandler = eh.Handle + p.NetmapClient = nc + p.ContainerWrapper = cc + p.EpochTimer = r + p.EpochState = es + }) + + require.NoError(t, err, "failed to create processor") + + ev := netmapEvent.NewEpoch{ + Num: 101, + Hash: util.Uint256{101}, + } + proc.handleNewEpoch(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + require.Equal(t, nc.epochDuration, es.duration, "invalid epoch duration") + require.Equal(t, ev.Num, es.counter, "invalid epoch counter") + require.EqualValues(t, []uint32{nc.txHeights[ev.Hash]}, r.timers, "invalid epoch timer resets") + + var expEstimation cntClient.StartEstimationPrm + expEstimation.SetEpoch(ev.Num - 1) + expEstimation.SetHash(ev.Hash) + require.EqualValues(t, []cntClient.StartEstimationPrm{expEstimation}, cc.estimations, "invalid estimations") + + require.EqualValues(t, []event.Event{ + audit.NewAuditStartEvent(ev.Num), + settlement.NewAuditEvent(ev.Num), + governance.NewSyncEvent(ev.TxHash()), + ev, + }, eh.handledEvents, "invalid handled events") +} + +func TestAddPeer(t *testing.T) { + t.Parallel() + + t.Run("with notary", func(t *testing.T) { + t.Parallel() + nc := &testNetmapClient{ + contractAddress: util.Uint160{47}, + } + + proc, err := newTestProc(t, func(p *Params) { + p.NotaryDisabled = true + p.NetmapClient = nc + }) + + require.NoError(t, err, "failed to create processor") + + var node netmap.NodeInfo + key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key1") + node.SetPublicKey(key.Bytes()) + + ev := netmapEvent.AddPeer{ + NodeBytes: node.Marshal(), + Request: &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{ + Nonce: 100, + }, + }, + } + proc.handleAddPeer(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + require.EqualValues(t, []notaryInvoke{ + { + contract: nc.contractAddress, + fee: 0, + nonce: ev.Request.MainTransaction.Nonce, + vub: nil, + method: "addPeerIR", + args: []any{ev.Node()}, + }, + }, nc.notaryInvokes, "invalid notary invokes") + }) + + t.Run("without notary", func(t *testing.T) { + t.Parallel() + + nc := &testNetmapClient{ + contractAddress: util.Uint160{47}, + } + + proc, err := newTestProc(t, func(p *Params) { + p.NotaryDisabled = true + p.NetmapClient = nc + }) + + require.NoError(t, err, "failed to create processor") + + var node netmap.NodeInfo + key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key") + node.SetPublicKey(key.Bytes()) + + ev := netmapEvent.AddPeer{ + NodeBytes: node.Marshal(), + } + proc.handleAddPeer(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + var addPeerExp netmapclient.AddPeerPrm + addPeerExp.SetNodeInfo(node) + require.EqualValues(t, []netmapclient.AddPeerPrm{addPeerExp}, nc.addPeers, "invalid peers") + }) +} + +func TestUpdateState(t *testing.T) { + t.Parallel() + + t.Run("with notary", func(t *testing.T) { + t.Parallel() + ns := &testNodeStateSettings{ + maintAllowed: true, + } + nc := &testNetmapClient{} + + proc, err := newTestProc(t, func(p *Params) { + p.NotaryDisabled = true + p.NodeStateSettings = ns + p.NetmapClient = nc + }) + + require.NoError(t, err, "failed to create processor") + + key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key") + + ev := netmapEvent.UpdatePeer{ + State: netmapContract.NodeStateOnline, + PubKey: key, + Request: &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{ + Nonce: 100, + }, + }, + } + proc.handleUpdateState(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + require.EqualValues(t, []*transaction.Transaction{ + ev.Request.MainTransaction, + }, nc.invokedTxs, "invalid invoked transactions") + }) + + t.Run("without notary", func(t *testing.T) { + t.Parallel() + ns := &testNodeStateSettings{ + maintAllowed: true, + } + nc := &testNetmapClient{} + + proc, err := newTestProc(t, func(p *Params) { + p.NetmapClient = nc + p.NodeStateSettings = ns + }) + + require.NoError(t, err, "failed to create processor") + + key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key") + + ev := netmapEvent.UpdatePeer{ + State: netmapContract.NodeStateOnline, + PubKey: key, + } + proc.handleUpdateState(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + var expUpdPeer netmapclient.UpdatePeerPrm + expUpdPeer.SetMaintenance() + expUpdPeer.SetOnline() + expUpdPeer.SetKey(ev.PubKey.Bytes()) + + require.EqualValues(t, []netmapclient.UpdatePeerPrm{expUpdPeer}, nc.peerStateUpdates, "invalid peer state updates") + }) +} + +func TestCleanupTick(t *testing.T) { + t.Parallel() + + t.Run("notary disabled", func(t *testing.T) { + t.Parallel() + + nc := &testNetmapClient{} + + proc, err := newTestProc(t, func(p *Params) { + p.NetmapClient = nc + p.NotaryDisabled = true + p.CleanupEnabled = true + }) + + require.NoError(t, err, "failed to create processor") + + key1Str := "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35" + proc.netmapSnapshot.lastAccess[key1Str] = epochStampWithNodeInfo{ + epochStamp: epochStamp{ + epoch: 95, + removeFlag: false, + }, + } + key2Str := "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3" + proc.netmapSnapshot.lastAccess[key2Str] = epochStampWithNodeInfo{ + epochStamp: epochStamp{ + epoch: 98, + removeFlag: false, + }, + } + + ev := netmapCleanupTick{ + epoch: 100, + txHash: util.Uint256{123}, + } + + proc.handleCleanupTick(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + keyExp, err := keys.NewPublicKeyFromString(key1Str) + require.NoError(t, err, "failed to parse expired key") + + updExp := netmapclient.UpdatePeerPrm{} + updExp.SetKey(keyExp.Bytes()) + updExp.SetHash(ev.TxHash()) + + require.EqualValues(t, []netmapclient.UpdatePeerPrm{updExp}, nc.peerStateUpdates, "invalid peer updates") + require.True(t, proc.netmapSnapshot.lastAccess[key1Str].removeFlag, "invalid expired removed flag") + require.False(t, proc.netmapSnapshot.lastAccess[key2Str].removeFlag, "invalid non expired removed flag") + }) + + t.Run("notary enabled", func(t *testing.T) { + t.Parallel() + + nc := &testNetmapClient{ + contractAddress: util.Uint160{111}, + } + proc, err := newTestProc(t, + func(p *Params) { + p.NetmapClient = nc + p.CleanupEnabled = true + }, + ) + + require.NoError(t, err, "failed to create processor") + + key1Str := "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35" + proc.netmapSnapshot.lastAccess[key1Str] = epochStampWithNodeInfo{ + epochStamp: epochStamp{ + epoch: 95, + removeFlag: false, + }, + } + key2Str := "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3" + proc.netmapSnapshot.lastAccess[key2Str] = epochStampWithNodeInfo{ + epochStamp: epochStamp{ + epoch: 98, + removeFlag: false, + }, + } + + ev := netmapCleanupTick{ + epoch: 100, + txHash: util.Uint256{123}, + } + + proc.handleCleanupTick(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + + keyExp, err := keys.NewPublicKeyFromString(key1Str) + require.NoError(t, err, "failed to parse expired key") + + updExp := netmapclient.UpdatePeerPrm{} + updExp.SetKey(keyExp.Bytes()) + updExp.SetHash(ev.TxHash()) + + require.EqualValues(t, []notaryInvoke{ + { + contract: nc.contractAddress, + fee: 0, + nonce: uint32(ev.epoch), + vub: nil, + method: "updateStateIR", + args: []any{int64(v2netmap.Offline), keyExp.Bytes()}, + }, + }, nc.notaryInvokes, "invalid notary invokes") + require.True(t, proc.netmapSnapshot.lastAccess[key1Str].removeFlag, "invalid expired removed flag") + require.False(t, proc.netmapSnapshot.lastAccess[key2Str].removeFlag, "invalid non expired removed flag") + }) +} + +func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { + ns := &testNodeStateSettings{} + es := &testEpochState{} + r := &testEpochResetter{} + as := &testAlphabetState{ + isAlphabet: true, + } + cc := &testContainerClient{} + nc := &testNetmapClient{} + eh := &testEventHandler{} + + p := &Params{ + Log: test.NewLogger(t, true), + PoolSize: 1, + CleanupEnabled: false, + CleanupThreshold: 3, + NotaryDisabled: false, + NodeStateSettings: ns, + NodeValidator: &testValidator{}, + EpochState: es, + EpochTimer: r, + AlphabetState: as, + ContainerWrapper: cc, + NetmapClient: nc, + NotaryDepositHandler: eh.Handle, + HandleAudit: eh.Handle, + AuditSettlementsHandler: eh.Handle, + AlphabetSyncHandler: eh.Handle, + } + + nonDefault(p) + + return New(p) +} + +type testNodeStateSettings struct { + maintAllowed bool +} + +func (s *testNodeStateSettings) MaintenanceModeAllowed() error { + if s.maintAllowed { + return nil + } + return fmt.Errorf("maintenance mode not allowed") +} + +type testValidator struct{} + +func (v *testValidator) VerifyAndUpdate(*netmap.NodeInfo) error { + return nil +} + +type testEpochState struct { + counter uint64 + duration uint64 +} + +func (s *testEpochState) SetEpochCounter(c uint64) { + s.counter = c +} +func (s *testEpochState) EpochCounter() uint64 { + return s.counter +} +func (s *testEpochState) SetEpochDuration(d uint64) { + s.duration = d +} +func (s *testEpochState) EpochDuration() uint64 { + return s.duration +} + +type testEpochResetter struct { + timers []uint32 +} + +func (r *testEpochResetter) ResetEpochTimer(t uint32) error { + r.timers = append(r.timers, t) + return nil +} + +type testAlphabetState struct { + isAlphabet bool +} + +func (s *testAlphabetState) IsAlphabet() bool { + return s.isAlphabet +} + +type testContainerClient struct { + estimations []cntClient.StartEstimationPrm +} + +func (c *testContainerClient) StartEstimation(p cntClient.StartEstimationPrm) error { + c.estimations = append(c.estimations, p) + return nil +} + +type notaryInvoke struct { + contract util.Uint160 + fee fixedn.Fixed8 + nonce uint32 + vub *uint32 + method string + args []any +} + +type testNetmapClient struct { + contractAddress util.Uint160 + epochDuration uint64 + netmap *netmap.NetMap + txHeights map[util.Uint256]uint32 + + peerStateUpdates []netmapclient.UpdatePeerPrm + notaryInvokes []notaryInvoke + newEpochs []uint64 + addPeers []netmapclient.AddPeerPrm + invokedTxs []*transaction.Transaction +} + +func (c *testNetmapClient) UpdatePeerState(p netmapclient.UpdatePeerPrm) error { + c.peerStateUpdates = append(c.peerStateUpdates, p) + return nil +} +func (c *testNetmapClient) MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { + c.notaryInvokes = append(c.notaryInvokes, notaryInvoke{ + contract: contract, + fee: fee, + nonce: nonce, + vub: vub, + method: method, + args: args, + }) + return nil +} +func (c *testNetmapClient) ContractAddress() util.Uint160 { + return c.contractAddress +} +func (c *testNetmapClient) EpochDuration() (uint64, error) { + return c.epochDuration, nil +} +func (c *testNetmapClient) MorphTxHeight(h util.Uint256) (uint32, error) { + if res, found := c.txHeights[h]; found { + return res, nil + } + return 0, fmt.Errorf("not found") +} +func (c *testNetmapClient) NetMap() (*netmap.NetMap, error) { + return c.netmap, nil +} +func (c *testNetmapClient) NewEpoch(epoch uint64, force bool) error { + c.newEpochs = append(c.newEpochs, epoch) + return nil +} +func (c *testNetmapClient) MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) { + return true, nil +} +func (c *testNetmapClient) AddPeer(p netmapclient.AddPeerPrm) error { + c.addPeers = append(c.addPeers, p) + return nil +} +func (c *testNetmapClient) MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error { + c.invokedTxs = append(c.invokedTxs, mainTx) + return nil +} + +type testEventHandler struct { + handledEvents []event.Event +} + +func (h *testEventHandler) Handle(e event.Event) { + h.handledEvents = append(h.handledEvents, e) +} diff --git a/pkg/innerring/processors/netmap/process_cleanup.go b/pkg/innerring/processors/netmap/process_cleanup.go index d50c69c78..45a08b377 100644 --- a/pkg/innerring/processors/netmap/process_cleanup.go +++ b/pkg/innerring/processors/netmap/process_cleanup.go @@ -39,7 +39,7 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { err = np.netmapClient.UpdatePeerState(prm) } else { - err = np.netmapClient.Morph().NotaryInvoke( + err = np.netmapClient.MorphNotaryInvoke( np.netmapClient.ContractAddress(), 0, uint32(ev.epoch), diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index 96ea4a01a..b655db9aa 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -23,7 +23,7 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { np.epochState.SetEpochCounter(epoch) - h, err := np.netmapClient.Morph().TxHeight(ev.TxHash()) + h, err := np.netmapClient.MorphTxHeight(ev.TxHash()) if err != nil { np.log.Warn(logs.NetmapCantGetTransactionHeight, zap.String("hash", ev.TxHash().StringLE()), diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index bf54ed341..9e6eeb53e 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -21,7 +21,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { // check if notary transaction is valid, see #976 if originalRequest := ev.NotaryRequest(); originalRequest != nil { tx := originalRequest.MainTransaction - ok, err := np.netmapClient.Morph().IsValidScript(tx.Script, tx.Signers) + ok, err := np.netmapClient.MorphIsValidScript(tx.Script, tx.Signers) if err != nil || !ok { np.log.Warn(logs.NetmapNonhaltNotaryTransaction, zap.String("method", "netmap.AddPeer"), @@ -73,7 +73,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { if nr := ev.NotaryRequest(); nr != nil { // create new notary request with the original nonce - err = np.netmapClient.Morph().NotaryInvoke( + err = np.netmapClient.MorphNotaryInvoke( np.netmapClient.ContractAddress(), 0, nr.MainTransaction.Nonce, @@ -117,7 +117,7 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { } if nr := ev.NotaryRequest(); nr != nil { - err = np.netmapClient.Morph().NotarySignAndInvokeTX(nr.MainTransaction) + err = np.netmapClient.MorphNotarySignAndInvokeTX(nr.MainTransaction) } else { prm := netmapclient.UpdatePeerPrm{} diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index 39069a963..522fa8b86 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -6,13 +6,16 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/nspcc-dev/neo-go/pkg/core/mempoolevent" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" "go.uber.org/zap" ) @@ -51,6 +54,23 @@ type ( VerifyAndUpdate(*netmap.NodeInfo) error } + Client interface { + UpdatePeerState(p netmapclient.UpdatePeerPrm) error + MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error + ContractAddress() util.Uint160 + EpochDuration() (uint64, error) + MorphTxHeight(h util.Uint256) (res uint32, err error) + NetMap() (*netmap.NetMap, error) + NewEpoch(epoch uint64, force bool) error + MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) + AddPeer(p netmapclient.AddPeerPrm) error + MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error + } + + ContainerClient interface { + StartEstimation(p cntClient.StartEstimationPrm) error + } + // Processor of events produced by network map contract // and new epoch ticker, because it is related to contract. Processor struct { @@ -60,8 +80,8 @@ type ( epochState EpochState alphabetState AlphabetState - netmapClient *nmClient.Client - containerWrp *container.Client + netmapClient Client + containerWrp ContainerClient netmapSnapshot cleanupTable @@ -79,13 +99,13 @@ type ( Params struct { Log *logger.Logger PoolSize int - NetmapClient *nmClient.Client + NetmapClient Client EpochTimer EpochTimerReseter EpochState EpochState AlphabetState AlphabetState CleanupEnabled bool CleanupThreshold uint64 // in epochs - ContainerWrapper *container.Client + ContainerWrapper ContainerClient AlphabetSyncHandler event.Handler NotaryDepositHandler event.Handler diff --git a/pkg/innerring/processors/netmap/wrappers.go b/pkg/innerring/processors/netmap/wrappers.go new file mode 100644 index 000000000..255d498d3 --- /dev/null +++ b/pkg/innerring/processors/netmap/wrappers.go @@ -0,0 +1,59 @@ +package netmap + +import ( + netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" + "github.com/nspcc-dev/neo-go/pkg/util" +) + +func NewNetmapClient(netmapClient *netmapclient.Client) Client { + return &netmapClientWrapper{ + netmapClient: netmapClient, + } +} + +type netmapClientWrapper struct { + netmapClient *netmapclient.Client +} + +func (w *netmapClientWrapper) UpdatePeerState(p netmapclient.UpdatePeerPrm) error { + return w.netmapClient.UpdatePeerState(p) +} + +func (w *netmapClientWrapper) MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { + return w.netmapClient.Morph().NotaryInvoke(contract, fee, nonce, vub, method, args...) +} + +func (w *netmapClientWrapper) ContractAddress() util.Uint160 { + return w.netmapClient.ContractAddress() +} + +func (w *netmapClientWrapper) EpochDuration() (uint64, error) { + return w.netmapClient.EpochDuration() +} + +func (w *netmapClientWrapper) MorphTxHeight(h util.Uint256) (res uint32, err error) { + return w.netmapClient.Morph().TxHeight(h) +} + +func (w *netmapClientWrapper) NetMap() (*netmap.NetMap, error) { + return w.netmapClient.NetMap() +} + +func (w *netmapClientWrapper) NewEpoch(epoch uint64, force bool) error { + return w.netmapClient.NewEpoch(epoch, force) +} + +func (w *netmapClientWrapper) MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) { + return w.netmapClient.Morph().IsValidScript(script, signers) +} + +func (w *netmapClientWrapper) AddPeer(p netmapclient.AddPeerPrm) error { + return w.netmapClient.AddPeer(p) +} + +func (w *netmapClientWrapper) MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error { + return w.netmapClient.Morph().NotarySignAndInvokeTX(mainTx) +} diff --git a/pkg/morph/event/netmap/add_peer.go b/pkg/morph/event/netmap/add_peer.go index 87cf94082..6f839bada 100644 --- a/pkg/morph/event/netmap/add_peer.go +++ b/pkg/morph/event/netmap/add_peer.go @@ -10,24 +10,24 @@ import ( ) type AddPeer struct { - node []byte + NodeBytes []byte // For notary notifications only. // Contains raw transactions of notary request. - notaryRequest *payload.P2PNotaryRequest + Request *payload.P2PNotaryRequest } // MorphEvent implements Neo:Morph Event interface. func (AddPeer) MorphEvent() {} func (s AddPeer) Node() []byte { - return s.node + return s.NodeBytes } // NotaryRequest returns raw notary request if notification // was received via notary service. Otherwise, returns nil. func (s AddPeer) NotaryRequest() *payload.P2PNotaryRequest { - return s.notaryRequest + return s.Request } const expectedItemNumAddPeer = 1 @@ -47,7 +47,7 @@ func ParseAddPeer(e *state.ContainedNotificationEvent) (event.Event, error) { return nil, event.WrongNumberOfParameters(expectedItemNumAddPeer, ln) } - ev.node, err = client.BytesFromStackItem(params[0]) + ev.NodeBytes, err = client.BytesFromStackItem(params[0]) if err != nil { return nil, fmt.Errorf("could not get raw nodeinfo: %w", err) } diff --git a/pkg/morph/event/netmap/add_peer_notary.go b/pkg/morph/event/netmap/add_peer_notary.go index a506b052d..a24722a97 100644 --- a/pkg/morph/event/netmap/add_peer_notary.go +++ b/pkg/morph/event/netmap/add_peer_notary.go @@ -7,7 +7,7 @@ import ( func (s *AddPeer) setNode(v []byte) { if v != nil { - s.node = v + s.NodeBytes = v } } @@ -43,7 +43,7 @@ func ParseAddPeerNotary(ne event.NotaryEvent) (event.Event, error) { } } - ev.notaryRequest = ne.Raw() + ev.Request = ne.Raw() return ev, nil } diff --git a/pkg/morph/event/netmap/add_peer_test.go b/pkg/morph/event/netmap/add_peer_test.go index 1b8bcf40a..0574c4048 100644 --- a/pkg/morph/event/netmap/add_peer_test.go +++ b/pkg/morph/event/netmap/add_peer_test.go @@ -37,7 +37,7 @@ func TestParseAddPeer(t *testing.T) { require.NoError(t, err) require.Equal(t, AddPeer{ - node: info, + NodeBytes: info, }, ev) }) } diff --git a/pkg/morph/event/netmap/epoch.go b/pkg/morph/event/netmap/epoch.go index 0eaa9f285..e454e2a6a 100644 --- a/pkg/morph/event/netmap/epoch.go +++ b/pkg/morph/event/netmap/epoch.go @@ -11,12 +11,12 @@ import ( // NewEpoch is a new epoch Neo:Morph event. type NewEpoch struct { - num uint64 + Num uint64 - // txHash is used in notary environmental + // Hash is used in notary environmental // for calculating unique but same for // all notification receivers values. - txHash util.Uint256 + Hash util.Uint256 } // MorphEvent implements Neo:Morph Event interface. @@ -24,13 +24,13 @@ func (NewEpoch) MorphEvent() {} // EpochNumber returns new epoch number. func (s NewEpoch) EpochNumber() uint64 { - return s.num + return s.Num } // TxHash returns hash of the TX with new epoch // notification. func (s NewEpoch) TxHash() util.Uint256 { - return s.txHash + return s.Hash } // ParseNewEpoch is a parser of new epoch notification event. @@ -52,7 +52,7 @@ func ParseNewEpoch(e *state.ContainedNotificationEvent) (event.Event, error) { } return NewEpoch{ - num: uint64(prmEpochNum), - txHash: e.Container, + Num: uint64(prmEpochNum), + Hash: e.Container, }, nil } diff --git a/pkg/morph/event/netmap/epoch_test.go b/pkg/morph/event/netmap/epoch_test.go index b175b5275..bc267ecb6 100644 --- a/pkg/morph/event/netmap/epoch_test.go +++ b/pkg/morph/event/netmap/epoch_test.go @@ -37,7 +37,7 @@ func TestParseNewEpoch(t *testing.T) { require.NoError(t, err) require.Equal(t, NewEpoch{ - num: epochNum, + Num: epochNum, }, ev) }) } diff --git a/pkg/morph/event/netmap/update_peer.go b/pkg/morph/event/netmap/update_peer.go index 535d57e4d..f02ca408d 100644 --- a/pkg/morph/event/netmap/update_peer.go +++ b/pkg/morph/event/netmap/update_peer.go @@ -13,13 +13,13 @@ import ( ) type UpdatePeer struct { - publicKey *keys.PublicKey + PubKey *keys.PublicKey - state netmap.NodeState + State netmap.NodeState // For notary notifications only. // Contains raw transactions of notary request. - notaryRequest *payload.P2PNotaryRequest + Request *payload.P2PNotaryRequest } // MorphEvent implements Neo:Morph Event interface. @@ -28,27 +28,27 @@ func (UpdatePeer) MorphEvent() {} // Online returns true if node's state is requested to be switched // to "online". func (s UpdatePeer) Online() bool { - return s.state == netmap.NodeStateOnline + return s.State == netmap.NodeStateOnline } // Maintenance returns true if node's state is requested to be switched // to "maintenance". func (s UpdatePeer) Maintenance() bool { - return s.state == netmap.NodeStateMaintenance + return s.State == netmap.NodeStateMaintenance } func (s UpdatePeer) PublicKey() *keys.PublicKey { - return s.publicKey + return s.PubKey } // NotaryRequest returns raw notary request if notification // was received via notary service. Otherwise, returns nil. func (s UpdatePeer) NotaryRequest() *payload.P2PNotaryRequest { - return s.notaryRequest + return s.Request } func (s *UpdatePeer) decodeState(state int64) error { - switch s.state = netmap.NodeState(state); s.state { + switch s.State = netmap.NodeState(state); s.State { default: return fmt.Errorf("unsupported node state %d", state) case @@ -82,7 +82,7 @@ func ParseUpdatePeer(e *state.ContainedNotificationEvent) (event.Event, error) { return nil, fmt.Errorf("could not get public key: %w", err) } - ev.publicKey, err = keys.NewPublicKeyFromBytes(key, elliptic.P256()) + ev.PubKey, err = keys.NewPublicKeyFromBytes(key, elliptic.P256()) if err != nil { return nil, fmt.Errorf("could not parse public key: %w", err) } diff --git a/pkg/morph/event/netmap/update_peer_notary.go b/pkg/morph/event/netmap/update_peer_notary.go index b7a251f98..0260810b8 100644 --- a/pkg/morph/event/netmap/update_peer_notary.go +++ b/pkg/morph/event/netmap/update_peer_notary.go @@ -17,7 +17,7 @@ func (s *UpdatePeer) setPublicKey(v []byte) (err error) { return errNilPubKey } - s.publicKey, err = keys.NewPublicKeyFromBytes(v, elliptic.P256()) + s.PubKey, err = keys.NewPublicKeyFromBytes(v, elliptic.P256()) if err != nil { return fmt.Errorf("could not parse public key: %w", err) } @@ -73,7 +73,7 @@ func ParseUpdatePeerNotary(ne event.NotaryEvent) (event.Event, error) { } } - ev.notaryRequest = ne.Raw() + ev.Request = ne.Raw() return ev, nil } diff --git a/pkg/morph/event/netmap/update_peer_test.go b/pkg/morph/event/netmap/update_peer_test.go index 1772c88a7..b79dd6385 100644 --- a/pkg/morph/event/netmap/update_peer_test.go +++ b/pkg/morph/event/netmap/update_peer_test.go @@ -52,8 +52,8 @@ func TestParseUpdatePeer(t *testing.T) { require.NoError(t, err) require.Equal(t, UpdatePeer{ - publicKey: publicKey, - state: state, + PubKey: publicKey, + State: state, }, ev) }) } From 235fe84ea3e30a6932bb05fc850c79843d149e05 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 2 May 2023 11:51:30 +0300 Subject: [PATCH 0199/1943] [#298] innerring: Fix broken tests Signed-off-by: Evgenii Stratonikov --- pkg/innerring/blocktimer_test.go | 36 ------------------- pkg/innerring/contracts_test.go | 9 ----- .../processors/netmap/handlers_test.go | 36 ++++++++----------- 3 files changed, 14 insertions(+), 67 deletions(-) diff --git a/pkg/innerring/blocktimer_test.go b/pkg/innerring/blocktimer_test.go index e1a79c2a8..f1cf620a8 100644 --- a/pkg/innerring/blocktimer_test.go +++ b/pkg/innerring/blocktimer_test.go @@ -18,8 +18,6 @@ func TestEpochTimer(t *testing.T) { counter: 99, duration: 10, } - collectHandler := &testEventHandler{} - distributeHandler := &testEventHandler{} args := &epochTimerArgs{ l: test.NewLogger(t, true), @@ -29,16 +27,6 @@ func TestEpochTimer(t *testing.T) { epoch: epochState, stopEstimationDMul: 2, stopEstimationDDiv: 10, - collectBasicIncome: subEpochEventHandler{ - handler: collectHandler.Handle, - durationMul: 3, - durationDiv: 10, - }, - distributeBasicIncome: subEpochEventHandler{ - handler: distributeHandler.Handle, - durationMul: 4, - durationDiv: 10, - }, } et := newEpochTimer(args) err := et.Reset() @@ -47,78 +35,54 @@ func TestEpochTimer(t *testing.T) { et.Tick(100) require.Equal(t, 0, neh.called, "invalid new epoch handler calls") require.Equal(t, 0, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 0, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 0, distributeHandler.called, "invalid distribute basic income calls") et.Tick(101) require.Equal(t, 0, neh.called, "invalid new epoch handler calls") require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 0, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 0, distributeHandler.called, "invalid distribute basic income calls") et.Tick(102) require.Equal(t, 0, neh.called, "invalid new epoch handler calls") require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 0, distributeHandler.called, "invalid distribute basic income calls") et.Tick(103) require.Equal(t, 0, neh.called, "invalid new epoch handler calls") require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") var h uint32 for h = 104; h < 109; h++ { et.Tick(h) require.Equal(t, 0, neh.called, "invalid new epoch handler calls") require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") } et.Tick(109) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") et.Tick(110) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") et.Tick(111) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 1, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") et.Tick(112) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 2, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 1, distributeHandler.called, "invalid distribute basic income calls") et.Tick(113) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 2, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 2, distributeHandler.called, "invalid distribute basic income calls") for h = 114; h < 119; h++ { et.Tick(h) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 2, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 2, distributeHandler.called, "invalid distribute basic income calls") } et.Tick(120) require.Equal(t, 2, neh.called, "invalid new epoch handler calls") require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") - require.Equal(t, 2, collectHandler.called, "invalid collect basic income calls") - require.Equal(t, 2, distributeHandler.called, "invalid distribute basic income calls") } type testAlphabetState struct { diff --git a/pkg/innerring/contracts_test.go b/pkg/innerring/contracts_test.go index 2d6e69c05..e04e22f06 100644 --- a/pkg/innerring/contracts_test.go +++ b/pkg/innerring/contracts_test.go @@ -44,9 +44,6 @@ contracts: processingExp, _ := util.Uint160DecodeStringLE("597f5894867113a41e192801709c02497f611de8") require.Equal(t, processingExp, c.processing, "invalid processing") - auditExp, _ := util.Uint160DecodeStringLE("219e37aed2180b87e7fe945dbf97d67125e8d73f") - require.Equal(t, auditExp, c.audit, "invalid audit") - balanceExp, _ := util.Uint160DecodeStringLE("d2aa48d14b17b11bc4c68205027884a96706dd16") require.Equal(t, balanceExp, c.balance, "invalid balance") @@ -80,9 +77,6 @@ contracts: require.Equal(t, util.Uint160{}, c.processing, "invalid processing") - auditExp, _ := util.Uint160DecodeStringLE("219e37aed2180b87e7fe945dbf97d67125e8d73f") - require.Equal(t, auditExp, c.audit, "invalid audit") - balanceExp, _ := util.Uint160DecodeStringLE("d2aa48d14b17b11bc4c68205027884a96706dd16") require.Equal(t, balanceExp, c.balance, "invalid balance") @@ -116,9 +110,6 @@ contracts: require.Equal(t, util.Uint160{}, c.processing, "invalid processing") - auditExp, _ := util.Uint160DecodeStringLE("219e37aed2180b87e7fe945dbf97d67125e8d73f") - require.Equal(t, auditExp, c.audit, "invalid audit") - balanceExp, _ := util.Uint160DecodeStringLE("d2aa48d14b17b11bc4c68205027884a96706dd16") require.Equal(t, balanceExp, c.balance, "invalid balance") diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index 576d6c5dd..4905b45d2 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -7,9 +7,7 @@ import ( v2netmap "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" netmapContract "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/settlement" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -84,8 +82,6 @@ func TestNewEpoch(t *testing.T) { proc, err := newTestProc(t, func(p *Params) { p.NotaryDisabled = true p.NotaryDepositHandler = eh.Handle - p.HandleAudit = eh.Handle - p.AuditSettlementsHandler = eh.Handle p.AlphabetSyncHandler = eh.Handle p.NetmapClient = nc p.ContainerWrapper = cc @@ -115,8 +111,6 @@ func TestNewEpoch(t *testing.T) { require.EqualValues(t, []cntClient.StartEstimationPrm{expEstimation}, cc.estimations, "invalid estimations") require.EqualValues(t, []event.Event{ - audit.NewAuditStartEvent(ev.Num), - settlement.NewAuditEvent(ev.Num), governance.NewSyncEvent(ev.TxHash()), ev, }, eh.handledEvents, "invalid handled events") @@ -409,22 +403,20 @@ func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { eh := &testEventHandler{} p := &Params{ - Log: test.NewLogger(t, true), - PoolSize: 1, - CleanupEnabled: false, - CleanupThreshold: 3, - NotaryDisabled: false, - NodeStateSettings: ns, - NodeValidator: &testValidator{}, - EpochState: es, - EpochTimer: r, - AlphabetState: as, - ContainerWrapper: cc, - NetmapClient: nc, - NotaryDepositHandler: eh.Handle, - HandleAudit: eh.Handle, - AuditSettlementsHandler: eh.Handle, - AlphabetSyncHandler: eh.Handle, + Log: test.NewLogger(t, true), + PoolSize: 1, + CleanupEnabled: false, + CleanupThreshold: 3, + NotaryDisabled: false, + NodeStateSettings: ns, + NodeValidator: &testValidator{}, + EpochState: es, + EpochTimer: r, + AlphabetState: as, + ContainerWrapper: cc, + NetmapClient: nc, + NotaryDepositHandler: eh.Handle, + AlphabetSyncHandler: eh.Handle, } nonDefault(p) From b939e4e5c5d0d761b231737619806446dee78d0d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 2 May 2023 14:05:35 +0300 Subject: [PATCH 0200/1943] [#299] ir: Drop unused structs Signed-off-by: Dmitrii Stepanov --- pkg/innerring/blocktimer_test.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pkg/innerring/blocktimer_test.go b/pkg/innerring/blocktimer_test.go index f1cf620a8..224aa5c58 100644 --- a/pkg/innerring/blocktimer_test.go +++ b/pkg/innerring/blocktimer_test.go @@ -4,7 +4,6 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "github.com/stretchr/testify/require" ) @@ -121,11 +120,3 @@ func (s *testEpochState) EpochCounter() uint64 { func (s *testEpochState) EpochDuration() uint64 { return s.duration } - -type testEventHandler struct { - called int -} - -func (h *testEventHandler) Handle(e event.Event) { - h.called++ -} From eca5c210dd21c5c6f30c5817c45b808589a2c651 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 2 May 2023 14:16:13 +0300 Subject: [PATCH 0201/1943] [#299] evacuate: Add context cancel checks Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 3 +- pkg/local_object_storage/engine/evacuate.go | 35 ++++++++++++++----- .../engine/evacuate_test.go | 29 +++++++++++++-- pkg/services/control/server/evacuate.go | 4 +-- 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 6c1c931fb..ab7b8f63c 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -239,7 +239,8 @@ const ( EngineShardIsMovedInDegradedModeDueToErrorThreshold = "shard is moved in degraded mode due to error threshold" // Info in ../node/pkg/local_object_storage/engine/engine.go EngineModeChangeIsInProgressIgnoringSetmodeRequest = "mode change is in progress, ignoring set-mode request" // Debug in ../node/pkg/local_object_storage/engine/engine.go EngineStartedShardsEvacuation = "started shards evacuation" // Info in ../node/pkg/local_object_storage/engine/evacuate.go - EngineFinishedShardsEvacuation = "finished shards evacuation" // Info in ../node/pkg/local_object_storage/engine/evacuate.go + EngineFinishedSuccessfullyShardsEvacuation = "shards evacuation finished successfully" // Info in ../node/pkg/local_object_storage/engine/evacuate.go + EngineFinishedWithErrorShardsEvacuation = "shards evacuation finished with error" // Error in ../node/pkg/local_object_storage/engine/evacuate.go EngineObjectIsMovedToAnotherShard = "object is moved to another shard" // Debug in ../node/pkg/local_object_storage/engine/evacuate.go MetabaseMissingMatcher = "missing matcher" // Debug in ../node/pkg/local_object_storage/metabase/select.go MetabaseErrorInFKBTSelection = "error in FKBT selection" // Debug in ../node/pkg/local_object_storage/metabase/select.go diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index e212784a3..761ed24b9 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -22,7 +22,7 @@ var ErrMustBeReadOnly = logicerr.New("shard must be in read-only mode") // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { shardID []*shard.ID - handler func(oid.Address, *objectSDK.Object) error + handler func(context.Context, oid.Address, *objectSDK.Object) error ignoreErrors bool } @@ -42,7 +42,7 @@ func (p *EvacuateShardPrm) WithIgnoreErrors(ignore bool) { } // WithFaultHandler sets handler to call for objects which cannot be saved on other shards. -func (p *EvacuateShardPrm) WithFaultHandler(f func(oid.Address, *objectSDK.Object) error) { +func (p *EvacuateShardPrm) WithFaultHandler(f func(context.Context, oid.Address, *objectSDK.Object) error) { p.handler = f } @@ -89,11 +89,12 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (Eva for _, shardID := range shardIDs { if err = e.evacuateShard(ctx, shardID, prm, &res, shards, weights, shardsToEvacuate); err != nil { + e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs)) return res, err } } - e.log.Info(logs.EngineFinishedShardsEvacuation, zap.Strings("shard_ids", shardIDs)) + e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, zap.Strings("shard_ids", shardIDs)) return res, nil } @@ -168,6 +169,11 @@ func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, toEvacuate []object.AddressWithType, prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { for i := range toEvacuate { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } addr := toEvacuate[i].Address var getPrm shard.GetPrm @@ -181,7 +187,12 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to return err } - if e.tryEvacuateObject(ctx, addr, getRes.Object(), sh, res, shards, weights, shardsToEvacuate) { + evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), sh, res, shards, weights, shardsToEvacuate) + if err != nil { + return err + } + + if evacuatedLocal { continue } @@ -191,7 +202,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to return fmt.Errorf("%w: %s", errPutShard, toEvacuate[i]) } - err = prm.handler(addr, getRes.Object()) + err = prm.handler(ctx, addr, getRes.Object()) if err != nil { return err } @@ -200,10 +211,16 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to return nil } -func (e *StorageEngine) tryEvacuateObject(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) bool { +func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, res *EvacuateShardRes, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) (bool, error) { hrw.SortHasherSliceByWeightValue(shards, weights, hrw.Hash([]byte(addr.EncodeToString()))) for j := range shards { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + if _, ok := shardsToEvacuate[shards[j].ID().String()]; ok { continue } @@ -216,9 +233,9 @@ func (e *StorageEngine) tryEvacuateObject(ctx context.Context, addr oid.Address, zap.Stringer("addr", addr)) res.count++ } - return true + return true, nil } } - return false + return false, nil } diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index fc9da5e3f..11cb362e0 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -137,9 +137,9 @@ func TestEvacuateShard(t *testing.T) { func TestEvacuateNetwork(t *testing.T) { var errReplication = errors.New("handler error") - acceptOneOf := func(objects []*objectSDK.Object, max int) func(oid.Address, *objectSDK.Object) error { + acceptOneOf := func(objects []*objectSDK.Object, max int) func(context.Context, oid.Address, *objectSDK.Object) error { var n int - return func(addr oid.Address, obj *objectSDK.Object) error { + return func(_ context.Context, addr oid.Address, obj *objectSDK.Object) error { if n == max { return errReplication } @@ -230,3 +230,28 @@ func TestEvacuateNetwork(t *testing.T) { }) }) } + +func TestEvacuateCancellation(t *testing.T) { + e, ids, _ := newEngineEvacuate(t, 2, 3) + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + + var prm EvacuateShardPrm + prm.shardID = ids[1:2] + prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return nil + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + res, err := e.Evacuate(ctx, prm) + require.ErrorContains(t, err, "context canceled") + require.Equal(t, 0, res.Count()) +} diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index b64a91883..afa4011b9 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -48,7 +48,7 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe return resp, nil } -func (s *Server) replicate(addr oid.Address, obj *objectSDK.Object) error { +func (s *Server) replicate(ctx context.Context, addr oid.Address, obj *objectSDK.Object) error { cid, ok := obj.ContainerID() if !ok { // Return nil to prevent situations where a shard can't be evacuated @@ -89,7 +89,7 @@ func (s *Server) replicate(addr oid.Address, obj *objectSDK.Object) error { task.SetObjectAddress(addr) task.SetCopiesNumber(1) task.SetNodes(nodes) - s.replicator.HandleTask(context.TODO(), task, &res) + s.replicator.HandleTask(ctx, task, &res) if res.count == 0 { return errors.New("object was not replicated") From 529d0bc710e72baed1e5523117d061e2b01ffe16 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 2 May 2023 16:50:03 +0300 Subject: [PATCH 0202/1943] [#302] tree: Drop unused ctx Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/sync.go | 4 ++-- pkg/services/tree/sync_test.go | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 5c0300c14..ed2455194 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -126,7 +126,7 @@ func (s *Service) SynchronizeTree(ctx context.Context, cid cid.ID, treeID string } // mergeOperationStreams performs merge sort for node operation streams to one stream. -func mergeOperationStreams(ctx context.Context, streams []chan *pilorama.Move, merged chan<- *pilorama.Move) uint64 { +func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *pilorama.Move) uint64 { defer close(merged) ms := make([]*pilorama.Move, len(streams)) @@ -274,7 +274,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, merged := make(chan *pilorama.Move) var minStreamedLastHeight uint64 errGroup.Go(func() error { - minStreamedLastHeight = mergeOperationStreams(ctx, nodeOperationStreams, merged) + minStreamedLastHeight = mergeOperationStreams(nodeOperationStreams, merged) return nil }) var minUnappliedHeight uint64 diff --git a/pkg/services/tree/sync_test.go b/pkg/services/tree/sync_test.go index eb51bc3f8..190b4ccbb 100644 --- a/pkg/services/tree/sync_test.go +++ b/pkg/services/tree/sync_test.go @@ -1,7 +1,6 @@ package tree import ( - "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -11,14 +10,12 @@ import ( func Test_mergeOperationStreams(t *testing.T) { tests := []struct { name string - ctx context.Context opTimes [][]uint64 wantValues []uint64 wantMinHeight uint64 }{ { name: "1", - ctx: context.Background(), opTimes: [][]uint64{ {250, 251, 255}, {252, 253, 254, 256, 257}, @@ -28,7 +25,6 @@ func Test_mergeOperationStreams(t *testing.T) { }, { name: "2", - ctx: context.Background(), opTimes: [][]uint64{ {250, 251, 255, 259}, {252, 253, 254, 256, 257}, @@ -38,7 +34,6 @@ func Test_mergeOperationStreams(t *testing.T) { }, { name: "3", - ctx: context.Background(), opTimes: [][]uint64{ {250, 251, 255}, {249, 250, 251, 253, 254, 256, 257}, @@ -71,7 +66,7 @@ func Test_mergeOperationStreams(t *testing.T) { merged := make(chan *pilorama.Move, 1) min := make(chan uint64) go func() { - min <- mergeOperationStreams(tt.ctx, nodeOpChans, merged) + min <- mergeOperationStreams(nodeOpChans, merged) }() var res []uint64 From a1b4ba9980bfe2dd5fdb67a5095ea814169755b6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Apr 2023 13:50:25 +0300 Subject: [PATCH 0203/1943] [#209] compression: Do not store uncompressible data Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobstor/compression/compress.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobstor/compression/compress.go b/pkg/local_object_storage/blobstor/compression/compress.go index 79e37f8ae..4ebf057d3 100644 --- a/pkg/local_object_storage/blobstor/compression/compress.go +++ b/pkg/local_object_storage/blobstor/compression/compress.go @@ -87,7 +87,11 @@ func (c *Config) Compress(data []byte) []byte { return data } maxSize := c.encoder.MaxEncodedSize(len(data)) - return c.encoder.EncodeAll(data, make([]byte, 0, maxSize)) + compressed := c.encoder.EncodeAll(data, make([]byte, 0, maxSize)) + if len(data) < len(compressed) { + return data + } + return compressed } // Close closes encoder and decoder, returns any error occurred. From 5f1af84587f234dc3f96d8da77729adda024c0fa Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 28 Apr 2023 14:08:14 +0300 Subject: [PATCH 0204/1943] [#297] go.mod: Update compress to v1.16.5 Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d54770ba2..551fb0fb7 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/go-github/v39 v39.2.0 github.com/google/uuid v1.3.0 github.com/hashicorp/golang-lru/v2 v2.0.1 - github.com/klauspost/compress v1.15.13 + github.com/klauspost/compress v1.16.5 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.8.0 diff --git a/go.sum b/go.sum index 7b140a44d..185e3e00d 100644 --- a/go.sum +++ b/go.sum @@ -294,8 +294,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= -github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.2 h1:xPMwiykqNK9VK0NYC3+jTMYv9I6Vl3YdjZgPZKG3zO0= From d1661ae7dc3eaa3fa13b98e2934c7ea300b0b1dd Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 4 May 2023 11:00:45 +0300 Subject: [PATCH 0205/1943] [#308] Remove downloadContractsFromGithub from the frostfs-adm Signed-off-by: Alejandro Lopez --- .../internal/modules/morph/download.go | 40 ------------------- .../modules/morph/initialize_deploy.go | 3 +- .../internal/modules/morph/root.go | 6 ++- go.mod | 2 - go.sum | 5 --- 5 files changed, 5 insertions(+), 51 deletions(-) delete mode 100644 cmd/frostfs-adm/internal/modules/morph/download.go diff --git a/cmd/frostfs-adm/internal/modules/morph/download.go b/cmd/frostfs-adm/internal/modules/morph/download.go deleted file mode 100644 index 3c50c0b34..000000000 --- a/cmd/frostfs-adm/internal/modules/morph/download.go +++ /dev/null @@ -1,40 +0,0 @@ -package morph - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "strings" - - "github.com/google/go-github/v39/github" - "github.com/spf13/cobra" -) - -func downloadContractsFromGithub(cmd *cobra.Command) (io.ReadCloser, error) { - gcl := github.NewClient(nil) - release, _, err := gcl.Repositories.GetLatestRelease(context.Background(), "nspcc-dev", "frostfs-contract") - if err != nil { - return nil, fmt.Errorf("can't fetch release info: %w", err) - } - - cmd.Printf("Found %s (%s), downloading...\n", release.GetTagName(), release.GetName()) - - var url string - for _, a := range release.Assets { - if strings.HasPrefix(a.GetName(), "frostfs-contract") { - url = a.GetBrowserDownloadURL() - break - } - } - if url == "" { - return nil, errors.New("can't find contracts archive in release assets") - } - - resp, err := http.Get(url) - if err != nil { - return nil, fmt.Errorf("can't fetch contracts archive: %w", err) - } - return resp.Body, nil -} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 1e35fb79f..7e03eae20 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -401,8 +401,7 @@ func (c *initializeContext) readContracts(names []string) error { } else { var r io.ReadCloser if c.ContractPath == "" { - c.Command.Println("Contracts flag is missing, latest release will be fetched from Github.") - r, err = downloadContractsFromGithub(c.Command) + return errors.New("contracts flag is missing") } else { r, err = os.Open(c.ContractPath) } diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 1361fe158..431be125c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -309,7 +309,8 @@ func initUpdateContractsCmd() { RootCmd.AddCommand(updateContractsCmd) updateContractsCmd.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") updateContractsCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") - updateContractsCmd.Flags().String(contractsInitFlag, "", "Path to archive with compiled FrostFS contracts (default fetched from latest github release)") + updateContractsCmd.Flags().String(contractsInitFlag, "", "Path to archive with compiled FrostFS contracts") + _ = updateContractsCmd.MarkFlagRequired(contractsInitFlag) } func initDumpBalancesCmd() { @@ -370,7 +371,8 @@ func initInitCmd() { RootCmd.AddCommand(initCmd) initCmd.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") initCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") - initCmd.Flags().String(contractsInitFlag, "", "Path to archive with compiled FrostFS contracts (default fetched from latest github release)") + initCmd.Flags().String(contractsInitFlag, "", "Path to archive with compiled FrostFS contracts") + _ = initCmd.MarkFlagRequired(contractsInitFlag) initCmd.Flags().Uint(epochDurationCLIFlag, 240, "Amount of side chain blocks in one FrostFS epoch") initCmd.Flags().Uint(maxObjectSizeCLIFlag, 67108864, "Max single object size in bytes") initCmd.Flags().Bool(homomorphicHashDisabledCLIFlag, false, "Disable object homomorphic hashing") diff --git a/go.mod b/go.mod index 551fb0fb7..2f0244c6e 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 - github.com/google/go-github/v39 v39.2.0 github.com/google/uuid v1.3.0 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/klauspost/compress v1.16.5 @@ -59,7 +58,6 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.3 // indirect - github.com/google/go-querystring v1.1.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect diff --git a/go.sum b/go.sum index 185e3e00d..3e5bcd570 100644 --- a/go.sum +++ b/go.sum @@ -228,10 +228,6 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ= -github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -572,7 +568,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= From 872fe90c40396d4b85db8da01c7aa5ef203c075e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 4 May 2023 13:47:52 +0300 Subject: [PATCH 0206/1943] [#317] pre-commit: Add go-mod-tidy hook Signed-off-by: Evgenii Stratonikov --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7273e1ee..3d9b2b8f2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,3 +48,4 @@ repos: rev: v1.0.0-rc.1 hooks: - id: go-staticcheck-repo-mod + - id: go-mod-tidy From 2f6757c828acf9235577175a4cbc8aa08fb36e98 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 4 May 2023 14:13:25 +0300 Subject: [PATCH 0207/1943] [#311] ir: Fix data race in unit tests Signed-off-by: Dmitrii Stepanov --- pkg/innerring/processors/alphabet/handlers_test.go | 5 ++--- pkg/innerring/processors/frostfs/handlers_test.go | 4 +++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/innerring/processors/alphabet/handlers_test.go b/pkg/innerring/processors/alphabet/handlers_test.go index 5e71cbc90..c098ca27d 100644 --- a/pkg/innerring/processors/alphabet/handlers_test.go +++ b/pkg/innerring/processors/alphabet/handlers_test.go @@ -2,7 +2,6 @@ package alphabet_test import ( "testing" - "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" @@ -140,7 +139,7 @@ func TestProcessorEmitsGasToNetmapIfNoParsedWallets(t *testing.T) { processor.HandleGasEmission(timers.NewAlphabetEmitTick{}) - time.Sleep(time.Second) + processor.WaitPoolRunning() require.EqualValues(t, []invokedMethod{ { @@ -201,7 +200,7 @@ func TestProcessorDoesntEmitGasIfNoNetmapOrParsedWallets(t *testing.T) { processor.HandleGasEmission(timers.NewAlphabetEmitTick{}) - time.Sleep(time.Second) + processor.WaitPoolRunning() require.EqualValues(t, []invokedMethod{ { diff --git a/pkg/innerring/processors/frostfs/handlers_test.go b/pkg/innerring/processors/frostfs/handlers_test.go index ee2a00427..db7835811 100644 --- a/pkg/innerring/processors/frostfs/handlers_test.go +++ b/pkg/innerring/processors/frostfs/handlers_test.go @@ -254,7 +254,9 @@ func TestHandleUnbind(t *testing.T) { proc.handleBind(evBind) - time.Sleep(time.Second) + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } require.EqualValues(t, []frostfsid.CommonBindPrm{expBind}, id.remove, "invalid remove keys value") require.EqualValues(t, []frostfsid.CommonBindPrm{expBind}, id.add, "invalid add keys value") From 479c5a65e141f5418251124deb20de0b0f2c2424 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 4 May 2023 19:44:49 +0300 Subject: [PATCH 0208/1943] [#322] node: Fix tree svc panic If a connection has not been established earlier, it stores `nil` in LRU cache. Cache eviction tries to close every connection (even a `nil` one) and panics but not crash the app because we are using pools. That ugly bug also leads to a deadlock where `Unlock` is not called via `defer` func (and that is the way I found it). Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 + pkg/services/tree/cache.go | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4c86fa37..60a5d7d92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Changelog for FrostFS Node - Take network settings into account during netmap contract update (#100) - Read config files from dir even if config file not provided via `--config` for node (#238) - Notary requests parsing according to `neo-go`'s updates (#268) +- Tree service panic in its internal client cache (#322) ### Removed ### Updated diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index ab9f509ac..56b97e687 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -36,7 +36,9 @@ var errRecentlyFailed = errors.New("client has recently failed") func (c *clientCache) init() { l, _ := simplelru.NewLRU[string, cacheItem](defaultClientCacheSize, func(_ string, value cacheItem) { - _ = value.cc.Close() + if conn := value.cc; conn != nil { + _ = conn.Close() + } }) c.LRU = *l } From cedd07bbc80fd6f1e1ed1fc4238e7c5320fb45d1 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 3 May 2023 15:58:21 +0300 Subject: [PATCH 0209/1943] [#304] morph: Iterate endpoints when create ws client in constructor Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 1 + internal/logs/logs.go | 2 ++ pkg/morph/client/constructor.go | 24 +++++++++++++++++++++--- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60a5d7d92..281386256 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Changelog for FrostFS Node - Read config files from dir even if config file not provided via `--config` for node (#238) - Notary requests parsing according to `neo-go`'s updates (#268) - Tree service panic in its internal client cache (#322) +- Iterate over endpoints when create ws client in morph's constructor (#304) ### Removed ### Updated diff --git a/internal/logs/logs.go b/internal/logs/logs.go index ab7b8f63c..9b6e03499 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -413,6 +413,8 @@ const ( FrostFSIRInternalError = "internal error" // Info in ../node/cmd/frostfs-ir/main.go FrostFSIRCouldNotShutdownHTTPServer = "could not shutdown HTTP server" // Debug in ../node/cmd/frostfs-ir/main.go FrostFSIRApplicationStopped = "application stopped" // Info in ../node/cmd/frostfs-ir/main.go + FrostFSIRCouldntCreateRPCClientForEndpoint = "could not create RPC client for endpoint" // Debug in ../node/pkg/morph/client/constructor.go + FrostFSIRCreatedRPCClientForEndpoint = "created RPC client for endpoint" // Info in ../node/pkg/morph/client/constructor.go FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" // Error in ../node/cmd/frostfs-node/grpc.go FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" // Error in ../node/cmd/frostfs-node/grpc.go FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index c4ec70171..4232b349d 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/block" @@ -52,6 +53,10 @@ const ( defaultWaitInterval = 500 * time.Millisecond ) +var ( + ErrNoHealthyEndpoint = errors.New("no healthy endpoint") +) + func defaultConfig() *cfg { return &cfg{ dialTimeout: defaultDialTimeout, @@ -80,6 +85,8 @@ func defaultConfig() *cfg { // If desired option satisfies the default value, it can be omitted. // If multiple options of the same config value are supplied, // the option with the highest index in the arguments will be used. +// If the list of endpoints provided - uses first alive. +// If there are no healthy endpoint - returns ErrNoHealthyEndpoint. func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, error) { if key == nil { panic("empty private key") @@ -137,9 +144,20 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er return nil, fmt.Errorf("could not create RPC actor: %w", err) } } else { - cli.client, act, err = cli.newCli(ctx, cli.endpoints.list[0].Address) - if err != nil { - return nil, fmt.Errorf("could not create RPC client: %w", err) + var endpoint Endpoint + for cli.endpoints.curr, endpoint = range cli.endpoints.list { + cli.client, act, err = cli.newCli(ctx, endpoint.Address) + if err != nil { + cli.logger.Warn(logs.FrostFSIRCouldntCreateRPCClientForEndpoint, + zap.Error(err), zap.String("endpoint", endpoint.Address)) + } else { + cli.logger.Info(logs.FrostFSIRCreatedRPCClientForEndpoint, + zap.String("endpoint", endpoint.Address)) + break + } + } + if cli.client == nil { + return nil, ErrNoHealthyEndpoint } } cli.setActor(act) From 973af12854e876c48861bc82e832408b7f2d89cb Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 5 May 2023 10:38:59 +0300 Subject: [PATCH 0210/1943] [#327] tests: replace os.MkdirTemp with t.TempDir Signed-off-by: Alejandro Lopez --- pkg/local_object_storage/blobstor/exists_test.go | 7 +------ pkg/local_object_storage/blobstor/perf_test.go | 8 +------- pkg/local_object_storage/engine/control_test.go | 3 +-- pkg/local_object_storage/engine/error_test.go | 14 ++++---------- pkg/local_object_storage/engine/evacuate_test.go | 5 +---- pkg/local_object_storage/pilorama/forest_test.go | 8 +------- 6 files changed, 9 insertions(+), 36 deletions(-) diff --git a/pkg/local_object_storage/blobstor/exists_test.go b/pkg/local_object_storage/blobstor/exists_test.go index f5c5fbbef..7a3dcd617 100644 --- a/pkg/local_object_storage/blobstor/exists_test.go +++ b/pkg/local_object_storage/blobstor/exists_test.go @@ -2,7 +2,6 @@ package blobstor import ( "context" - "os" "testing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -15,13 +14,9 @@ import ( ) func TestExists(t *testing.T) { - dir, err := os.MkdirTemp("", "frostfs*") - require.NoError(t, err) - t.Cleanup(func() { _ = os.RemoveAll(dir) }) - const smallSizeLimit = 512 - storages, _, largeFileStorage := defaultTestStorages(dir, smallSizeLimit) + storages, _, largeFileStorage := defaultTestStorages(t.TempDir(), smallSizeLimit) b := New(WithStorages(storages)) diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index c88dc85ed..f21982530 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -3,7 +3,6 @@ package blobstor import ( "context" "fmt" - "os" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" @@ -21,18 +20,13 @@ type storage struct { } func (s storage) open(b *testing.B) common.Storage { - dir, err := os.MkdirTemp(os.TempDir(), s.desc) - if err != nil { - b.Fatalf("creating %s root path: %v", s.desc, err) - } - st := s.create(dir) + st := s.create(b.TempDir()) require.NoError(b, st.Open(false)) require.NoError(b, st.Init()) b.Cleanup(func() { require.NoError(b, st.Close()) - require.NoError(b, os.RemoveAll(dir)) }) return st diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 046968dbe..d7eaae1d8 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -233,8 +233,7 @@ func TestExecBlocks(t *testing.T) { } func TestPersistentShardID(t *testing.T) { - dir, err := os.MkdirTemp("", "*") - require.NoError(t, err) + dir := t.TempDir() te := newEngineWithErrorThreshold(t, dir, 1) diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 017b635d4..dc28d35fa 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -41,11 +41,7 @@ type testShard struct { func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) *testEngine { if dir == "" { - var err error - - dir, err = os.MkdirTemp("", "*") - require.NoError(t, err) - t.Cleanup(func() { _ = os.RemoveAll(dir) }) + dir = t.TempDir() } var testShards [2]*testShard @@ -171,9 +167,7 @@ func TestErrorReporting(t *testing.T) { } func TestBlobstorFailback(t *testing.T) { - dir, err := os.MkdirTemp("", "*") - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + dir := t.TempDir() te := newEngineWithErrorThreshold(t, dir, 1) @@ -185,7 +179,7 @@ func TestBlobstorFailback(t *testing.T) { var prm shard.PutPrm prm.SetObject(obj) te.ng.mtx.RLock() - _, err = te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) + _, err := te.ng.shards[te.shards[0].id.String()].Shard.Put(context.Background(), prm) te.ng.mtx.RUnlock() require.NoError(t, err) objs = append(objs, obj) @@ -193,7 +187,7 @@ func TestBlobstorFailback(t *testing.T) { for i := range objs { addr := object.AddressOf(objs[i]) - _, err = te.ng.Get(context.Background(), GetPrm{addr: addr}) + _, err := te.ng.Get(context.Background(), GetPrm{addr: addr}) require.NoError(t, err) _, err = te.ng.GetRange(context.Background(), RngPrm{addr: addr}) require.NoError(t, err) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 11cb362e0..bea6d4ff5 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "os" "path/filepath" "strconv" "testing" @@ -25,9 +24,7 @@ import ( ) func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEngine, []*shard.ID, []*objectSDK.Object) { - dir, err := os.MkdirTemp("", "*") - require.NoError(t, err) - t.Cleanup(func() { _ = os.RemoveAll(dir) }) + dir := t.TempDir() te := testNewEngine(t, WithShardPoolSize(1)). setShardsNumOpts(t, shardNum, func(id int) []shard.Option { diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index caa999f94..0cff28c3f 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math/rand" - "os" "path/filepath" "strconv" "sync" @@ -31,19 +30,14 @@ var providers = []struct { return f }}, {"bbolt", func(t testing.TB, opts ...Option) Forest { - // Use `os.TempDir` because we construct multiple times in the same test. - tmpDir, err := os.MkdirTemp(os.TempDir(), "*") - require.NoError(t, err) - f := NewBoltForest( append([]Option{ - WithPath(filepath.Join(tmpDir, "test.db")), + WithPath(filepath.Join(t.TempDir(), "test.db")), WithMaxBatchSize(1)}, opts...)...) require.NoError(t, f.Open(false)) require.NoError(t, f.Init()) t.Cleanup(func() { require.NoError(t, f.Close()) - require.NoError(t, os.RemoveAll(tmpDir)) }) return f }}, From d02950ad637764bf6a267757459406d89270c0c2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 5 May 2023 17:59:51 +0300 Subject: [PATCH 0211/1943] [#330] morph: Fix linter issue Revive became smarter after go 1.20.4 upgrade. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 7e03eae20..8a0a88899 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -402,9 +402,8 @@ func (c *initializeContext) readContracts(names []string) error { var r io.ReadCloser if c.ContractPath == "" { return errors.New("contracts flag is missing") - } else { - r, err = os.Open(c.ContractPath) } + r, err = os.Open(c.ContractPath) if err != nil { return fmt.Errorf("can't open contracts archive: %w", err) } From ee58b390bb8954c7631188dec17cc3ae8ae59701 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Tue, 11 Apr 2023 21:01:00 +0300 Subject: [PATCH 0212/1943] [#221] node: Allow using vector `copies_number` Also, take into account that value in general (it was not used before at all). Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 + pkg/services/object/put/prm.go | 8 +++++ pkg/services/object/put/v2/util.go | 3 +- .../object_manager/placement/traverser.go | 36 +++++++++++++++---- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 281386256..6ed434da5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ Changelog for FrostFS Node - Parameters `nns-name` and `nns-zone` for command `frostfs-cli container create` (#37) - Tree service now saves the last synchronization height which persists across restarts (#82) - Add tracing support (#135) +- Multiple (and a fix for single) copies number support for `PUT` requests (#221) ### Changed - Change `frostfs_node_engine_container_size` to counting sizes of logical objects diff --git a/pkg/services/object/put/prm.go b/pkg/services/object/put/prm.go index 27d9c9c7a..c8d1b29a2 100644 --- a/pkg/services/object/put/prm.go +++ b/pkg/services/object/put/prm.go @@ -42,6 +42,14 @@ func (p *PutInitPrm) WithObject(v *object.Object) *PutInitPrm { return p } +func (p *PutInitPrm) WithCopyNumbers(v []uint32) *PutInitPrm { + if p != nil && len(v) > 0 { + p.traverseOpts = append(p.traverseOpts, placement.WithCopyNumbers(v)) + } + + return p +} + func (p *PutInitPrm) WithRelay(f func(context.Context, client.NodeInfo, client.MultiAddressClient) error) *PutInitPrm { if p != nil { p.relay = f diff --git a/pkg/services/object/put/v2/util.go b/pkg/services/object/put/v2/util.go index 790f061f1..758470f6c 100644 --- a/pkg/services/object/put/v2/util.go +++ b/pkg/services/object/put/v2/util.go @@ -24,7 +24,8 @@ func (s *streamer) toInitPrm(part *objectV2.PutObjectPartInit, req *objectV2.Put object.NewFromV2(oV2), ). WithRelay(s.relayRequest). - WithCommonPrm(commonPrm), nil + WithCommonPrm(commonPrm). + WithCopyNumbers(part.GetCopiesNumber()), nil } func toChunkPrm(req *objectV2.PutObjectPartChunk) *putsvc.PutChunkPrm { diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 75d5fbfd1..e46240a86 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -38,6 +38,7 @@ type Traverser struct { type cfg struct { trackCopies bool + copyNumbers []uint32 flatSuccess *uint32 @@ -84,19 +85,23 @@ func NewTraverser(opts ...Option) (*Traverser, error) { return nil, fmt.Errorf("could not build placement: %w", err) } + // backward compatibility for scalar `copies_number` + if len(cfg.copyNumbers) == 1 { + cfg.flatSuccess = &cfg.copyNumbers[0] + } + var rem []int if cfg.flatSuccess != nil { ns = flatNodes(ns) rem = []int{int(*cfg.flatSuccess)} } else { - replNum := cfg.policy.NumberOfReplicas() - rem = make([]int, 0, replNum) + rem = defaultCopiesVector(cfg.policy) - for i := 0; i < replNum; i++ { - if cfg.trackCopies { - rem = append(rem, int(cfg.policy.ReplicaNumberByIndex(i))) - } else { - rem = append(rem, -1) + for i := range rem { + if !cfg.trackCopies { + rem[i] = -1 + } else if len(cfg.copyNumbers) > i { + rem[i] = int(cfg.copyNumbers[i]) } } } @@ -108,6 +113,17 @@ func NewTraverser(opts ...Option) (*Traverser, error) { }, nil } +func defaultCopiesVector(policy netmap.PlacementPolicy) []int { + replNum := policy.NumberOfReplicas() + copyVector := make([]int, 0, replNum) + + for i := 0; i < replNum; i++ { + copyVector = append(copyVector, int(policy.ReplicaNumberByIndex(i))) + } + + return copyVector +} + func flatNodes(ns [][]netmap.NodeInfo) [][]netmap.NodeInfo { sz := 0 for i := range ns { @@ -265,3 +281,9 @@ func WithoutSuccessTracking() Option { c.trackCopies = false } } + +func WithCopyNumbers(v []uint32) Option { + return func(c *cfg) { + c.copyNumbers = v + } +} From 35ea207df64d7bff230554469aeae68dc7fdad88 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 27 Apr 2023 18:42:06 +0300 Subject: [PATCH 0213/1943] [#291] object: Split validating target in two Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/streamer.go | 5 +- pkg/services/object/put/validation.go | 123 ++++++++++++++------------ 2 files changed, 70 insertions(+), 58 deletions(-) diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index ea885366b..f1ecd4df1 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -77,7 +77,7 @@ func (p *Streamer) initUntrustedTarget(prm *PutInitPrm) error { p.relay = prm.relay // prepare untrusted-Put object target - p.target = &validatingTarget{ + p.target = &validatingPreparedTarget{ nextTarget: p.newCommonTarget(prm), fmt: p.fmtValidator, @@ -125,8 +125,7 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { p.sessionKey = sessionKey p.target = &validatingTarget{ - fmt: p.fmtValidator, - unpreparedObject: true, + fmt: p.fmtValidator, nextTarget: transformer.NewPayloadSizeLimiter( p.maxPayloadSz, containerSDK.IsHomomorphicHashingDisabled(prm.cnr), diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/put/validation.go index 8c40d0677..a4790071a 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/put/validation.go @@ -15,13 +15,18 @@ import ( "git.frostfs.info/TrueCloudLab/tzhash/tz" ) -// validatingTarget validates object format and content. +// validatingTarget validates unprepared object format and content (streaming PUT case). type validatingTarget struct { nextTarget transformer.ObjectTarget fmt *object.FormatValidator +} - unpreparedObject bool +// validatingPreparedTarget validates prepared object format and content. +type validatingPreparedTarget struct { + nextTarget transformer.ObjectTarget + + fmt *object.FormatValidator hash hash.Hash @@ -42,38 +47,52 @@ var ( ) func (t *validatingTarget) WriteHeader(ctx context.Context, obj *objectSDK.Object) error { + if err := t.fmt.Validate(ctx, obj, true); err != nil { + return fmt.Errorf("(%T) coult not validate object format: %w", t, err) + } + + return t.nextTarget.WriteHeader(ctx, obj) +} + +func (t *validatingTarget) Write(ctx context.Context, p []byte) (n int, err error) { + return t.nextTarget.Write(ctx, p) +} + +func (t *validatingTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { + return t.nextTarget.Close(ctx) +} + +func (t *validatingPreparedTarget) WriteHeader(ctx context.Context, obj *objectSDK.Object) error { t.payloadSz = obj.PayloadSize() chunkLn := uint64(len(obj.Payload())) - if !t.unpreparedObject { - // check chunk size - if chunkLn > t.payloadSz { - return ErrWrongPayloadSize - } - - // check payload size limit - if t.payloadSz > t.maxPayloadSz { - return ErrExceedingMaxSize - } - - cs, csSet := obj.PayloadChecksum() - if !csSet { - return errors.New("missing payload checksum") - } - - switch typ := cs.Type(); typ { - default: - return fmt.Errorf("(%T) unsupported payload checksum type %v", t, typ) - case checksum.SHA256: - t.hash = sha256.New() - case checksum.TZ: - t.hash = tz.New() - } - - t.checksum = cs.Value() + // check chunk size + if chunkLn > t.payloadSz { + return ErrWrongPayloadSize } - if err := t.fmt.Validate(ctx, obj, t.unpreparedObject); err != nil { + // check payload size limit + if t.payloadSz > t.maxPayloadSz { + return ErrExceedingMaxSize + } + + cs, csSet := obj.PayloadChecksum() + if !csSet { + return errors.New("missing payload checksum") + } + + switch typ := cs.Type(); typ { + default: + return fmt.Errorf("(%T) unsupported payload checksum type %v", t, typ) + case checksum.SHA256: + t.hash = sha256.New() + case checksum.TZ: + t.hash = tz.New() + } + + t.checksum = cs.Value() + + if err := t.fmt.Validate(ctx, obj, false); err != nil { return fmt.Errorf("(%T) coult not validate object format: %w", t, err) } @@ -82,30 +101,26 @@ func (t *validatingTarget) WriteHeader(ctx context.Context, obj *objectSDK.Objec return err } - if !t.unpreparedObject { - // update written bytes - // - // Note: we MUST NOT add obj.PayloadSize() since obj - // can carry only the chunk of the full payload - t.writtenPayload += chunkLn - } + // update written bytes + // + // Note: we MUST NOT add obj.PayloadSize() since obj + // can carry only the chunk of the full payload + t.writtenPayload += chunkLn return nil } -func (t *validatingTarget) Write(ctx context.Context, p []byte) (n int, err error) { +func (t *validatingPreparedTarget) Write(ctx context.Context, p []byte) (n int, err error) { chunkLn := uint64(len(p)) - if !t.unpreparedObject { - // check if new chunk will overflow payload size - if t.writtenPayload+chunkLn > t.payloadSz { - return 0, ErrWrongPayloadSize - } + // check if new chunk will overflow payload size + if t.writtenPayload+chunkLn > t.payloadSz { + return 0, ErrWrongPayloadSize + } - _, err = t.hash.Write(p) - if err != nil { - return - } + _, err = t.hash.Write(p) + if err != nil { + return } n, err = t.nextTarget.Write(ctx, p) @@ -116,16 +131,14 @@ func (t *validatingTarget) Write(ctx context.Context, p []byte) (n int, err erro return } -func (t *validatingTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { - if !t.unpreparedObject { - // check payload size correctness - if t.payloadSz != t.writtenPayload { - return nil, ErrWrongPayloadSize - } +func (t *validatingPreparedTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { + // check payload size correctness + if t.payloadSz != t.writtenPayload { + return nil, ErrWrongPayloadSize + } - if !bytes.Equal(t.hash.Sum(nil), t.checksum) { - return nil, fmt.Errorf("(%T) incorrect payload checksum", t) - } + if !bytes.Equal(t.hash.Sum(nil), t.checksum) { + return nil, fmt.Errorf("(%T) incorrect payload checksum", t) } return t.nextTarget.Close(ctx) From b2ffd7df537ac6ff85c71a017da07781eefd5192 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 27 Apr 2023 18:46:42 +0300 Subject: [PATCH 0214/1943] [#291] object: Use PayloadSizeLimiter from SDK Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 +- pkg/services/object/put/distributed.go | 7 +- pkg/services/object/put/local.go | 7 +- pkg/services/object/put/remote.go | 5 +- pkg/services/object/put/streamer.go | 26 +- pkg/services/object/put/validation.go | 2 +- .../object_manager/transformer/fmt.go | 114 ------- .../object_manager/transformer/transformer.go | 294 ------------------ .../object_manager/transformer/types.go | 111 ------- .../object_manager/transformer/writer.go | 52 ---- 11 files changed, 25 insertions(+), 599 deletions(-) delete mode 100644 pkg/services/object_manager/transformer/fmt.go delete mode 100644 pkg/services/object_manager/transformer/transformer.go delete mode 100644 pkg/services/object_manager/transformer/types.go delete mode 100644 pkg/services/object_manager/transformer/writer.go diff --git a/go.mod b/go.mod index 2f0244c6e..ae0bc88c1 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418145405-db5b89496d68 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230503082209-d4fe9a193d1a git.frostfs.info/TrueCloudLab/hrw v1.2.0 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 3e5bcd570..633ac9a18 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02f git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418145405-db5b89496d68 h1:m9HLrwRINt00cSQ07hKTPExOdAmmfO8m/3iGelnTo2o= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230418145405-db5b89496d68/go.mod h1:TaJJOF3Uhuq8aqv2CrfuY2yhxePUinW35Xd3wfXLV/I= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230503082209-d4fe9a193d1a h1:I8cMZUthFTDGbQ5CYpThYEkypJmfLOBYVsC0metlpFI= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230503082209-d4fe9a193d1a/go.mod h1:TaJJOF3Uhuq8aqv2CrfuY2yhxePUinW35Xd3wfXLV/I= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 4b2056802..b24218621 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -10,10 +10,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/transformer" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "go.uber.org/zap" ) @@ -206,8 +206,9 @@ func (t *distributedTarget) iteratePlacement(ctx context.Context) (*transformer. id, _ = t.obj.ID() - return new(transformer.AccessIdentifiers). - WithSelfID(id), nil + return &transformer.AccessIdentifiers{ + SelfID: id, + }, nil } func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *placement.Traverser, addrs []placement.Node, resErr *atomic.Value) bool { diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index 7aef9f065..f07122729 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -5,9 +5,9 @@ import ( "fmt" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/transformer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" ) // ObjectStorage is an object storage interface. @@ -61,6 +61,7 @@ func (t *localTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers id, _ := t.obj.ID() - return new(transformer.AccessIdentifiers). - WithSelfID(id), nil + return &transformer.AccessIdentifiers{ + SelfID: id, + }, nil } diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/put/remote.go index e7fa124fa..bcc566b74 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/put/remote.go @@ -10,9 +10,9 @@ import ( objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/transformer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" ) type remoteTarget struct { @@ -68,8 +68,7 @@ func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifier return nil, fmt.Errorf("(%T) could not put object to %s: %w", t, t.nodeInfo.AddressGroup(), err) } - return new(transformer.AccessIdentifiers). - WithSelfID(res.ID()), nil + return &transformer.AccessIdentifiers{SelfID: res.ID()}, nil } // NewRemoteSender creates, initializes and returns new RemoteSender instance. diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index f1ecd4df1..6d0d8062e 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -10,9 +10,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/transformer" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -126,18 +126,14 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { p.sessionKey = sessionKey p.target = &validatingTarget{ fmt: p.fmtValidator, - nextTarget: transformer.NewPayloadSizeLimiter( - p.maxPayloadSz, - containerSDK.IsHomomorphicHashingDisabled(prm.cnr), - func() transformer.ObjectTarget { - return transformer.NewFormatTarget(&transformer.FormatterParams{ - Key: sessionKey, - NextTarget: p.newCommonTarget(prm), - SessionToken: sToken, - NetworkState: p.networkState, - }) - }, - ), + nextTarget: transformer.NewPayloadSizeLimiter(transformer.Params{ + Key: sessionKey, + NextTargetInit: func() transformer.ObjectTarget { return p.newCommonTarget(prm) }, + NetworkState: p.networkState, + MaxSize: p.maxPayloadSz, + WithoutHomomorphicHash: containerSDK.IsHomomorphicHashingDisabled(prm.cnr), + SessionToken: sToken, + }), } return nil @@ -273,7 +269,7 @@ func (p *Streamer) Close(ctx context.Context) (*PutResponse, error) { return nil, fmt.Errorf("(%T) could not close object target: %w", p, err) } - id := ids.ParentID() + id := ids.ParentID if id != nil { return &PutResponse{ id: *id, @@ -281,6 +277,6 @@ func (p *Streamer) Close(ctx context.Context) (*PutResponse, error) { } return &PutResponse{ - id: ids.SelfID(), + id: ids.SelfID, }, nil } diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/put/validation.go index a4790071a..406304422 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/put/validation.go @@ -9,9 +9,9 @@ import ( "hash" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/transformer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "git.frostfs.info/TrueCloudLab/tzhash/tz" ) diff --git a/pkg/services/object_manager/transformer/fmt.go b/pkg/services/object_manager/transformer/fmt.go deleted file mode 100644 index fbe8af2fb..000000000 --- a/pkg/services/object_manager/transformer/fmt.go +++ /dev/null @@ -1,114 +0,0 @@ -package transformer - -import ( - "context" - "crypto/ecdsa" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" -) - -type formatter struct { - prm *FormatterParams - - obj *object.Object - - sz uint64 -} - -// FormatterParams groups NewFormatTarget parameters. -type FormatterParams struct { - Key *ecdsa.PrivateKey - - NextTarget ObjectTarget - - SessionToken *session.Object - - NetworkState netmap.State -} - -// NewFormatTarget returns ObjectTarget instance that finalizes object structure -// and writes it to the next target. -// -// Chunks must be written before the WriteHeader call. -// -// Object changes: -// - sets version to current SDK version; -// - sets payload size to the total length of all written chunks; -// - sets session token; -// - sets number of creation epoch; -// - calculates and sets verification fields (ID, Signature). -func NewFormatTarget(p *FormatterParams) ObjectTarget { - return &formatter{ - prm: p, - } -} - -func (f *formatter) WriteHeader(_ context.Context, obj *object.Object) error { - f.obj = obj - - return nil -} - -func (f *formatter) Write(ctx context.Context, p []byte) (n int, err error) { - n, err = f.prm.NextTarget.Write(ctx, p) - - f.sz += uint64(n) - - return -} - -func (f *formatter) Close(ctx context.Context) (*AccessIdentifiers, error) { - curEpoch := f.prm.NetworkState.CurrentEpoch() - ver := version.Current() - - f.obj.SetVersion(&ver) - f.obj.SetPayloadSize(f.sz) - f.obj.SetSessionToken(f.prm.SessionToken) - f.obj.SetCreationEpoch(curEpoch) - - var ( - parID *oid.ID - parHdr *object.Object - ) - - if par := f.obj.Parent(); par != nil && par.Signature() == nil { - rawPar := object.NewFromV2(par.ToV2()) - - rawPar.SetSessionToken(f.prm.SessionToken) - rawPar.SetCreationEpoch(curEpoch) - - if err := object.SetIDWithSignature(*f.prm.Key, rawPar); err != nil { - return nil, fmt.Errorf("could not finalize parent object: %w", err) - } - - id, _ := rawPar.ID() - parID = &id - parHdr = rawPar - - f.obj.SetParent(parHdr) - } - - if err := object.SetIDWithSignature(*f.prm.Key, f.obj); err != nil { - return nil, fmt.Errorf("could not finalize object: %w", err) - } - - if err := f.prm.NextTarget.WriteHeader(ctx, f.obj); err != nil { - return nil, fmt.Errorf("could not write header to next target: %w", err) - } - - if _, err := f.prm.NextTarget.Close(ctx); err != nil { - return nil, fmt.Errorf("could not close next target: %w", err) - } - - id, _ := f.obj.ID() - - return new(AccessIdentifiers). - WithSelfID(id). - WithParentID(parID). - WithParent(parHdr), nil -} diff --git a/pkg/services/object_manager/transformer/transformer.go b/pkg/services/object_manager/transformer/transformer.go deleted file mode 100644 index c23b4dca7..000000000 --- a/pkg/services/object_manager/transformer/transformer.go +++ /dev/null @@ -1,294 +0,0 @@ -package transformer - -import ( - "context" - "crypto/sha256" - "fmt" - "hash" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/tzhash/tz" -) - -type payloadSizeLimiter struct { - maxSize, written uint64 - - withoutHomomorphicHash bool - - targetInit func() ObjectTarget - - target ObjectTarget - - current, parent *object.Object - - currentHashers, parentHashers []*payloadChecksumHasher - - previous []oid.ID - - chunkWriter writer - - splitID *object.SplitID - - parAttrs []object.Attribute -} - -type payloadChecksumHasher struct { - hasher hash.Hash - - checksumWriter func([]byte) -} - -// NewPayloadSizeLimiter returns ObjectTarget instance that restricts payload length -// of the writing object and writes generated objects to targets from initializer. -// -// Calculates and adds homomorphic hash to resulting objects only if withoutHomomorphicHash -// is false. -// -// Objects w/ payload size less or equal than max size remain untouched. -func NewPayloadSizeLimiter(maxSize uint64, withoutHomomorphicHash bool, targetInit TargetInitializer) ObjectTarget { - return &payloadSizeLimiter{ - maxSize: maxSize, - withoutHomomorphicHash: withoutHomomorphicHash, - targetInit: targetInit, - splitID: object.NewSplitID(), - } -} - -func (s *payloadSizeLimiter) WriteHeader(_ context.Context, hdr *object.Object) error { - s.current = fromObject(hdr) - - s.initialize() - - return nil -} - -func (s *payloadSizeLimiter) Write(ctx context.Context, p []byte) (int, error) { - if err := s.writeChunk(ctx, p); err != nil { - return 0, err - } - - return len(p), nil -} - -func (s *payloadSizeLimiter) Close(ctx context.Context) (*AccessIdentifiers, error) { - return s.release(ctx, true) -} - -func (s *payloadSizeLimiter) initialize() { - // if it is an object after the 1st - if ln := len(s.previous); ln > 0 { - // initialize parent object once (after 1st object) - if ln == 1 { - s.detachParent() - } - - // set previous object to the last previous identifier - s.current.SetPreviousID(s.previous[ln-1]) - } - - s.initializeCurrent() -} - -func fromObject(obj *object.Object) *object.Object { - cnr, _ := obj.ContainerID() - - res := object.New() - res.SetContainerID(cnr) - res.SetOwnerID(obj.OwnerID()) - res.SetAttributes(obj.Attributes()...) - res.SetType(obj.Type()) - - // obj.SetSplitID creates splitHeader but we don't need to do it in case - // of small objects, so we should make nil check. - if obj.SplitID() != nil { - res.SetSplitID(obj.SplitID()) - } - - return res -} - -func (s *payloadSizeLimiter) initializeCurrent() { - // initialize current object target - s.target = s.targetInit() - - // create payload hashers - s.currentHashers = payloadHashersForObject(s.current, s.withoutHomomorphicHash) - - // compose multi-writer from target and all payload hashers - ws := make([]writer, 0, 1+len(s.currentHashers)+len(s.parentHashers)) - - ws = append(ws, s.target) - - for i := range s.currentHashers { - ws = append(ws, newWriter(s.currentHashers[i].hasher)) - } - - for i := range s.parentHashers { - ws = append(ws, newWriter(s.parentHashers[i].hasher)) - } - - s.chunkWriter = newMultiWriter(ws...) -} - -func payloadHashersForObject(obj *object.Object, withoutHomomorphicHash bool) []*payloadChecksumHasher { - hashers := make([]*payloadChecksumHasher, 0, 2) - - hashers = append(hashers, &payloadChecksumHasher{ - hasher: sha256.New(), - checksumWriter: func(binChecksum []byte) { - if ln := len(binChecksum); ln != sha256.Size { - panic(fmt.Sprintf("wrong checksum length: expected %d, has %d", sha256.Size, ln)) - } - - csSHA := [sha256.Size]byte{} - copy(csSHA[:], binChecksum) - - var cs checksum.Checksum - cs.SetSHA256(csSHA) - - obj.SetPayloadChecksum(cs) - }, - }) - - if !withoutHomomorphicHash { - hashers = append(hashers, &payloadChecksumHasher{ - hasher: tz.New(), - checksumWriter: func(binChecksum []byte) { - if ln := len(binChecksum); ln != tz.Size { - panic(fmt.Sprintf("wrong checksum length: expected %d, has %d", tz.Size, ln)) - } - - csTZ := [tz.Size]byte{} - copy(csTZ[:], binChecksum) - - var cs checksum.Checksum - cs.SetTillichZemor(csTZ) - - obj.SetPayloadHomomorphicHash(cs) - }, - }) - } - - return hashers -} - -func (s *payloadSizeLimiter) release(ctx context.Context, finalize bool) (*AccessIdentifiers, error) { - // Arg finalize is true only when called from Close method. - // We finalize parent and generate linking objects only if it is more - // than 1 object in split-chain. - withParent := finalize && len(s.previous) > 0 - - if withParent { - writeHashes(s.parentHashers) - s.parent.SetPayloadSize(s.written) - s.current.SetParent(s.parent) - } - - // release current object - writeHashes(s.currentHashers) - - // release current, get its id - if err := s.target.WriteHeader(ctx, s.current); err != nil { - return nil, fmt.Errorf("could not write header: %w", err) - } - - ids, err := s.target.Close(ctx) - if err != nil { - return nil, fmt.Errorf("could not close target: %w", err) - } - - // save identifier of the released object - s.previous = append(s.previous, ids.SelfID()) - - if withParent { - // generate and release linking object - s.initializeLinking(ids.Parent()) - s.initializeCurrent() - - if _, err := s.release(ctx, false); err != nil { - return nil, fmt.Errorf("could not release linking object: %w", err) - } - } - - return ids, nil -} - -func writeHashes(hashers []*payloadChecksumHasher) { - for i := range hashers { - hashers[i].checksumWriter(hashers[i].hasher.Sum(nil)) - } -} - -func (s *payloadSizeLimiter) initializeLinking(parHdr *object.Object) { - s.current = fromObject(s.current) - s.current.SetParent(parHdr) - s.current.SetChildren(s.previous...) - s.current.SetSplitID(s.splitID) -} - -func (s *payloadSizeLimiter) writeChunk(ctx context.Context, chunk []byte) error { - // statement is true if the previous write of bytes reached exactly the boundary. - if s.written > 0 && s.written%s.maxSize == 0 { - if s.written == s.maxSize { - s.prepareFirstChild() - } - - // we need to release current object - if _, err := s.release(ctx, false); err != nil { - return fmt.Errorf("could not release object: %w", err) - } - - // initialize another object - s.initialize() - } - - var ( - ln = uint64(len(chunk)) - cut = ln - leftToEdge = s.maxSize - s.written%s.maxSize - ) - - // write bytes no further than the boundary of the current object - if ln > leftToEdge { - cut = leftToEdge - } - - if _, err := s.chunkWriter.Write(ctx, chunk[:cut]); err != nil { - return fmt.Errorf("could not write chunk to target: %w", err) - } - - // increase written bytes counter - s.written += cut - - // if there are more bytes in buffer we call method again to start filling another object - if ln > leftToEdge { - return s.writeChunk(ctx, chunk[cut:]) - } - - return nil -} - -func (s *payloadSizeLimiter) prepareFirstChild() { - // initialize split header with split ID on first object in chain - s.current.InitRelations() - s.current.SetSplitID(s.splitID) - - // cut source attributes - s.parAttrs = s.current.Attributes() - s.current.SetAttributes() - - // attributes will be added to parent in detachParent -} - -func (s *payloadSizeLimiter) detachParent() { - s.parent = s.current - s.current = fromObject(s.parent) - s.parent.ResetRelations() - s.parent.SetSignature(nil) - s.parentHashers = s.currentHashers - - // return source attributes - s.parent.SetAttributes(s.parAttrs...) -} diff --git a/pkg/services/object_manager/transformer/types.go b/pkg/services/object_manager/transformer/types.go deleted file mode 100644 index 73cea5216..000000000 --- a/pkg/services/object_manager/transformer/types.go +++ /dev/null @@ -1,111 +0,0 @@ -package transformer - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -// AccessIdentifiers represents group of the object identifiers -// that are returned after writing the object. -// Consists of the ID of the stored object and the ID of the parent object. -type AccessIdentifiers struct { - par *oid.ID - - self oid.ID - - parHdr *object.Object -} - -// ObjectTarget is an interface of the object writer. -type ObjectTarget interface { - // WriteHeader writes object header w/ payload part. - // The payload of the object may be incomplete. - // - // Must be called exactly once. Control remains with the caller. - // Missing a call or re-calling can lead to undefined behavior - // that depends on the implementation. - // - // Must not be called after Close call. - WriteHeader(context.Context, *object.Object) error - - // Write writes object payload chunk. - // - // Can be called multiple times. - // - // Must not be called after Close call. - Write(ctx context.Context, p []byte) (n int, err error) - - // Close is used to finish object writing. - // - // Close must return access identifiers of the object - // that has been written. - // - // Must be called no more than once. Control remains with the caller. - // Re-calling can lead to undefined behavior - // that depends on the implementation. - Close(ctx context.Context) (*AccessIdentifiers, error) -} - -// TargetInitializer represents ObjectTarget constructor. -type TargetInitializer func() ObjectTarget - -// SelfID returns identifier of the written object. -func (a AccessIdentifiers) SelfID() oid.ID { - return a.self -} - -// WithSelfID returns AccessIdentifiers with passed self identifier. -func (a *AccessIdentifiers) WithSelfID(v oid.ID) *AccessIdentifiers { - res := a - if res == nil { - res = new(AccessIdentifiers) - } - - res.self = v - - return res -} - -// ParentID return identifier of the parent of the written object. -func (a *AccessIdentifiers) ParentID() *oid.ID { - if a != nil { - return a.par - } - - return nil -} - -// WithParentID returns AccessIdentifiers with passed parent identifier. -func (a *AccessIdentifiers) WithParentID(v *oid.ID) *AccessIdentifiers { - res := a - if res == nil { - res = new(AccessIdentifiers) - } - - res.par = v - - return res -} - -// Parent return identifier of the parent of the written object. -func (a *AccessIdentifiers) Parent() *object.Object { - if a != nil { - return a.parHdr - } - - return nil -} - -// WithParent returns AccessIdentifiers with passed parent identifier. -func (a *AccessIdentifiers) WithParent(v *object.Object) *AccessIdentifiers { - res := a - if res == nil { - res = new(AccessIdentifiers) - } - - res.parHdr = v - - return res -} diff --git a/pkg/services/object_manager/transformer/writer.go b/pkg/services/object_manager/transformer/writer.go deleted file mode 100644 index 27aed16ff..000000000 --- a/pkg/services/object_manager/transformer/writer.go +++ /dev/null @@ -1,52 +0,0 @@ -package transformer - -import ( - "context" - "io" -) - -type writer interface { - Write(ctx context.Context, p []byte) (n int, err error) -} - -type multiWriter struct { - writers []writer -} - -func (t *multiWriter) Write(ctx context.Context, p []byte) (n int, err error) { - for _, w := range t.writers { - n, err = w.Write(ctx, p) - if err != nil { - return - } - if n != len(p) { - err = io.ErrShortWrite - return - } - } - return len(p), nil -} - -func newMultiWriter(writers ...writer) writer { - allWriters := make([]writer, 0, len(writers)) - for _, w := range writers { - if mw, ok := w.(*multiWriter); ok { - allWriters = append(allWriters, mw.writers...) - } else { - allWriters = append(allWriters, w) - } - } - return &multiWriter{allWriters} -} - -type writerWrapper struct { - Writer io.Writer -} - -func (w *writerWrapper) Write(_ context.Context, p []byte) (n int, err error) { - return w.Writer.Write(p) -} - -func newWriter(w io.Writer) writer { - return &writerWrapper{Writer: w} -} From 1309622b20818ed6e4c37ff3348de010db339872 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 2 May 2023 11:37:04 +0300 Subject: [PATCH 0215/1943] [#296] morph: Add listener unit tests Signed-off-by: Dmitrii Stepanov --- pkg/morph/event/listener_test.go | 187 +++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 pkg/morph/event/listener_test.go diff --git a/pkg/morph/event/listener_test.go b/pkg/morph/event/listener_test.go new file mode 100644 index 000000000..dc7ac3b81 --- /dev/null +++ b/pkg/morph/event/listener_test.go @@ -0,0 +1,187 @@ +package event + +import ( + "context" + "fmt" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/subscriber" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "github.com/nspcc-dev/neo-go/pkg/core/block" + "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/stretchr/testify/require" +) + +func TestEventHandling(t *testing.T) { + blockCh := make(chan *block.Block) + notificationCh := make(chan *state.ContainedNotificationEvent) + notaryRequestsCh := make(chan *result.NotaryRequestEvent) + + l, err := NewListener(ListenerParams{ + Logger: test.NewLogger(t, true), + Subscriber: &testSubscriber{ + blockCh: blockCh, + notificationCh: notificationCh, + notaryRequestsCh: notaryRequestsCh, + }, + WorkerPoolCapacity: 10, + }) + require.NoError(t, err, "failed to create listener") + + list := l.(*listener) + + blockHandled := make(chan bool) + handledBlocks := make([]*block.Block, 0) + l.RegisterBlockHandler(func(b *block.Block) { + handledBlocks = append(handledBlocks, b) + blockHandled <- true + }) + + key := scriptHashWithType{ + scriptHashValue: scriptHashValue{ + hash: util.Uint160{100}, + }, + typeValue: typeValue{ + typ: TypeFromString("notification type"), + }, + } + + l.SetNotificationParser(NotificationParserInfo{ + scriptHashWithType: key, + p: func(cne *state.ContainedNotificationEvent) (Event, error) { + return testNotificationEvent{source: cne}, nil + }, + }) + + notificationHandled := make(chan bool) + handledNotifications := make([]Event, 0) + l.RegisterNotificationHandler(NotificationHandlerInfo{ + scriptHashWithType: key, + h: func(e Event) { + handledNotifications = append(handledNotifications, e) + notificationHandled <- true + }, + }) + + go list.Listen(context.Background()) + + t.Run("handles block events", func(t *testing.T) { + block := &block.Block{} + + blockCh <- block + + <-blockHandled + + require.Equal(t, 1, len(handledBlocks), "invalid handled blocks length") + require.Equal(t, block, handledBlocks[0], "invalid handled block") + }) + + t.Run("handles notifications", func(t *testing.T) { + notification := &state.ContainedNotificationEvent{ + Container: util.Uint256{49}, + NotificationEvent: state.NotificationEvent{ + ScriptHash: util.Uint160{100}, + Name: "notification type", + }, + } + + notificationCh <- notification + + <-notificationHandled + require.EqualValues(t, []Event{testNotificationEvent{source: notification}}, handledNotifications, "invalid handled notifications") + }) +} + +func TestErrorPassing(t *testing.T) { + blockCh := make(chan *block.Block) + notificationCh := make(chan *state.ContainedNotificationEvent) + notaryRequestsCh := make(chan *result.NotaryRequestEvent) + + t.Run("notification error", func(t *testing.T) { + nErr := fmt.Errorf("notification error") + l, err := NewListener(ListenerParams{ + Logger: test.NewLogger(t, true), + Subscriber: &testSubscriber{ + blockCh: blockCh, + notificationCh: notificationCh, + notaryRequestsCh: notaryRequestsCh, + + notificationErr: nErr, + }, + WorkerPoolCapacity: 10, + }) + require.NoError(t, err, "failed to create listener") + + errCh := make(chan error) + + go l.ListenWithError(context.Background(), errCh) + + err = <-errCh + + require.ErrorIs(t, err, nErr, "invalid notification error") + }) + + t.Run("block error", func(t *testing.T) { + bErr := fmt.Errorf("notification error") + l, err := NewListener(ListenerParams{ + Logger: test.NewLogger(t, true), + Subscriber: &testSubscriber{ + blockCh: blockCh, + notificationCh: notificationCh, + notaryRequestsCh: notaryRequestsCh, + + blockErr: bErr, + }, + WorkerPoolCapacity: 10, + }) + require.NoError(t, err, "failed to create listener") + l.RegisterBlockHandler(func(b *block.Block) {}) + + errCh := make(chan error) + + go l.ListenWithError(context.Background(), errCh) + + err = <-errCh + + require.ErrorIs(t, err, bErr, "invalid block error") + }) + +} + +type testSubscriber struct { + blockCh chan *block.Block + notificationCh chan *state.ContainedNotificationEvent + notaryRequestsCh chan *result.NotaryRequestEvent + + blockErr error + notificationErr error +} + +func (s *testSubscriber) SubscribeForNotification(...util.Uint160) error { + return s.notificationErr +} +func (s *testSubscriber) UnsubscribeForNotification() {} +func (s *testSubscriber) BlockNotifications() error { + return s.blockErr +} +func (s *testSubscriber) SubscribeForNotaryRequests(mainTXSigner util.Uint160) error { + return nil +} + +func (s *testSubscriber) NotificationChannels() subscriber.NotificationChannels { + return subscriber.NotificationChannels{ + BlockCh: s.blockCh, + NotificationsCh: s.notificationCh, + NotaryRequestsCh: s.notaryRequestsCh, + } +} + +func (s *testSubscriber) Close() {} + +type testNotificationEvent struct { + source *state.ContainedNotificationEvent +} + +func (e testNotificationEvent) MorphEvent() {} From ea10abb42a987f09a2afb6f41100154e9007bc53 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 2 May 2023 11:58:29 +0300 Subject: [PATCH 0216/1943] [#296] morph: Add parser unit tests Signed-off-by: Dmitrii Stepanov --- pkg/morph/event/utils_test.go | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 pkg/morph/event/utils_test.go diff --git a/pkg/morph/event/utils_test.go b/pkg/morph/event/utils_test.go new file mode 100644 index 000000000..83facc653 --- /dev/null +++ b/pkg/morph/event/utils_test.go @@ -0,0 +1,48 @@ +package event + +import ( + "math/big" + "testing" + + "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "github.com/stretchr/testify/require" +) + +func TestParseStackArray(t *testing.T) { + t.Run("success", func(t *testing.T) { + arr := &stackitem.Array{} + arr.Append(stackitem.NewBigInteger(big.NewInt(1))) + arr.Append(stackitem.NewBigInteger(big.NewInt(2))) + ev := &state.ContainedNotificationEvent{ + Container: util.Uint256{67}, + NotificationEvent: state.NotificationEvent{ + ScriptHash: util.Uint160{69}, + Name: "name", + Item: arr, + }, + } + + items, err := ParseStackArray(ev) + require.NoError(t, err, "failed to parse event items") + require.Equal(t, 2, len(items), "invalid length") + require.Equal(t, stackitem.NewBigInteger(big.NewInt(1)), items[0], "invalid item 0") + require.Equal(t, stackitem.NewBigInteger(big.NewInt(2)), items[1], "invalid item 0") + }) + t.Run("empty stack error", func(t *testing.T) { + arr := &stackitem.Array{} + ev := &state.ContainedNotificationEvent{ + Container: util.Uint256{67}, + NotificationEvent: state.NotificationEvent{ + ScriptHash: util.Uint160{69}, + Name: "name", + Item: arr, + }, + } + + items, err := ParseStackArray(ev) + require.ErrorIs(t, err, errEmptyStackArray, "invalid empty array error") + require.Equal(t, 0, len(items), "items was returned") + }) +} From 90799497d35a944e2268de274de536f37b3c46c7 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Wed, 3 May 2023 16:19:46 +0300 Subject: [PATCH 0217/1943] [#114] Add remove-node IR control command Signed-off-by: Alejandro Lopez --- cmd/frostfs-cli/modules/control/ir.go | 2 + .../modules/control/ir_remove_node.go | 58 +++ .../client/netmap/{add_peer.go => peer.go} | 16 + pkg/services/control/ir/rpc.go | 9 + pkg/services/control/ir/server/calls.go | 42 ++ pkg/services/control/ir/service.go | 12 + pkg/services/control/ir/service.pb.go | 384 +++++++++++++++--- pkg/services/control/ir/service.proto | 18 + pkg/services/control/ir/service_frostfs.pb.go | 151 +++++++ pkg/services/control/ir/service_grpc.pb.go | 38 ++ 10 files changed, 676 insertions(+), 54 deletions(-) create mode 100644 cmd/frostfs-cli/modules/control/ir_remove_node.go rename pkg/morph/client/netmap/{add_peer.go => peer.go} (70%) diff --git a/cmd/frostfs-cli/modules/control/ir.go b/cmd/frostfs-cli/modules/control/ir.go index e89dda076..ae3f8502c 100644 --- a/cmd/frostfs-cli/modules/control/ir.go +++ b/cmd/frostfs-cli/modules/control/ir.go @@ -10,6 +10,8 @@ var irCmd = &cobra.Command{ func initControlIRCmd() { irCmd.AddCommand(tickEpochCmd) + irCmd.AddCommand(removeNodeCmd) initControlIRTickEpochCmd() + initControlIRRemoveNodeCmd() } diff --git a/cmd/frostfs-cli/modules/control/ir_remove_node.go b/cmd/frostfs-cli/modules/control/ir_remove_node.go new file mode 100644 index 000000000..f5b968b7f --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir_remove_node.go @@ -0,0 +1,58 @@ +package control + +import ( + "encoding/hex" + "errors" + + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + "github.com/spf13/cobra" +) + +var removeNodeCmd = &cobra.Command{ + Use: "remove-node", + Short: "Forces a node removal from netmap", + Long: "Forces a node removal from netmap via a notary request. It should be executed on other IR nodes as well.", + Run: removeNode, +} + +func initControlIRRemoveNodeCmd() { + initControlFlags(removeNodeCmd) + + flags := removeNodeCmd.Flags() + flags.String("node", "", "Node public key as a hex string") + _ = removeNodeCmd.MarkFlagRequired("node") +} + +func removeNode(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + c := getClient(cmd, pk) + + nodeKeyStr, _ := cmd.Flags().GetString("node") + if len(nodeKeyStr) == 0 { + commonCmd.ExitOnErr(cmd, "parsing node public key: ", errors.New("key cannot be empty")) + } + nodeKey, err := hex.DecodeString(nodeKeyStr) + commonCmd.ExitOnErr(cmd, "can't decode node public key: %w", err) + + req := new(ircontrol.RemoveNodeRequest) + req.SetBody(&ircontrol.RemoveNodeRequest_Body{ + Key: nodeKey, + }) + + commonCmd.ExitOnErr(cmd, "could not sign request: %w", ircontrolsrv.SignMessage(pk, req)) + + var resp *ircontrol.RemoveNodeResponse + err = c.ExecRaw(func(client *rawclient.Client) error { + resp, err = ircontrol.RemoveNode(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Node removed") +} diff --git a/pkg/morph/client/netmap/add_peer.go b/pkg/morph/client/netmap/peer.go similarity index 70% rename from pkg/morph/client/netmap/add_peer.go rename to pkg/morph/client/netmap/peer.go index dc6c25540..7ceaa0250 100644 --- a/pkg/morph/client/netmap/add_peer.go +++ b/pkg/morph/client/netmap/peer.go @@ -41,3 +41,19 @@ func (c *Client) AddPeer(p AddPeerPrm) error { } return nil } + +// ForceRemovePeer marks the given peer as offline via a notary control transaction. +func (c *Client) ForceRemovePeer(nodeInfo netmap.NodeInfo) error { + if !c.client.WithNotary() { + return fmt.Errorf("peer can be forcefully removed only in notary environment") + } + + prm := UpdatePeerPrm{} + prm.SetKey(nodeInfo.PublicKey()) + prm.SetControlTX(true) + + if err := c.UpdatePeerState(prm); err != nil { + return fmt.Errorf("updating peer state: %v", err) + } + return nil +} diff --git a/pkg/services/control/ir/rpc.go b/pkg/services/control/ir/rpc.go index 6b2234954..1b635c149 100644 --- a/pkg/services/control/ir/rpc.go +++ b/pkg/services/control/ir/rpc.go @@ -11,6 +11,7 @@ const serviceName = "ircontrol.ControlService" const ( rpcHealthCheck = "HealthCheck" rpcTickEpoch = "TickEpoch" + rpcRemoveNode = "RemoveNode" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -31,6 +32,14 @@ func TickEpoch( return sendUnary[TickEpochRequest, TickEpochResponse](cli, rpcTickEpoch, req, opts...) } +func RemoveNode( + cli *client.Client, + req *RemoveNodeRequest, + opts ...client.CallOption, +) (*RemoveNodeResponse, error) { + return sendUnary[RemoveNodeRequest, RemoveNodeResponse](cli, rpcRemoveNode, req, opts...) +} + func sendUnary[I, O grpc.Message](cli *client.Client, rpcName string, req *I, opts ...client.CallOption) (*O, error) { var resp O wResp := &responseWrapper[*O]{ diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 56e2e3f79..680d1e606 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -1,9 +1,11 @@ package control import ( + "bytes" "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -57,3 +59,43 @@ func (s *Server) TickEpoch(_ context.Context, req *control.TickEpochRequest) (*c return resp, nil } + +// RemoveNode forces a node removal. +// +// If request is not signed with a key from white list, permission error returns. +func (s *Server) RemoveNode(_ context.Context, req *control.RemoveNodeRequest) (*control.RemoveNodeResponse, error) { + if err := s.isValidRequest(req); err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + resp := new(control.RemoveNodeResponse) + resp.SetBody(new(control.RemoveNodeResponse_Body)) + + nm, err := s.netmapClient.NetMap() + if err != nil { + return nil, fmt.Errorf("getting netmap: %w", err) + } + var nodeInfo netmap.NodeInfo + for _, info := range nm.Nodes() { + if bytes.Equal(info.PublicKey(), req.GetBody().GetKey()) { + nodeInfo = info + break + } + } + if len(nodeInfo.PublicKey()) == 0 { + return nil, status.Error(codes.NotFound, "no such node") + } + if nodeInfo.IsOffline() { + return nil, status.Error(codes.FailedPrecondition, "node is already offline") + } + + if err := s.netmapClient.ForceRemovePeer(nodeInfo); err != nil { + return nil, fmt.Errorf("forcing node removal: %w", err) + } + + if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return resp, nil +} diff --git a/pkg/services/control/ir/service.go b/pkg/services/control/ir/service.go index 1aaec2c87..b2db2b43a 100644 --- a/pkg/services/control/ir/service.go +++ b/pkg/services/control/ir/service.go @@ -32,3 +32,15 @@ func (x *TickEpochResponse) SetBody(v *TickEpochResponse_Body) { x.Body = v } } + +func (x *RemoveNodeRequest) SetBody(v *RemoveNodeRequest_Body) { + if x != nil { + x.Body = v + } +} + +func (x *RemoveNodeResponse) SetBody(v *RemoveNodeResponse_Body) { + if x != nil { + x.Body = v + } +} diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index 84acdfc82..bec74a3be 100644 --- a/pkg/services/control/ir/service.pb.go +++ b/pkg/services/control/ir/service.pb.go @@ -248,6 +248,116 @@ func (x *TickEpochResponse) GetSignature() *Signature { return nil } +type RemoveNodeRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *RemoveNodeRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *RemoveNodeRequest) Reset() { + *x = RemoveNodeRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveNodeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveNodeRequest) ProtoMessage() {} + +func (x *RemoveNodeRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveNodeRequest.ProtoReflect.Descriptor instead. +func (*RemoveNodeRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{4} +} + +func (x *RemoveNodeRequest) GetBody() *RemoveNodeRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *RemoveNodeRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +type RemoveNodeResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *RemoveNodeResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *RemoveNodeResponse) Reset() { + *x = RemoveNodeResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveNodeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveNodeResponse) ProtoMessage() {} + +func (x *RemoveNodeResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveNodeResponse.ProtoReflect.Descriptor instead. +func (*RemoveNodeResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{5} +} + +func (x *RemoveNodeResponse) GetBody() *RemoveNodeResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *RemoveNodeResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + // Health check request body. type HealthCheckRequest_Body struct { state protoimpl.MessageState @@ -258,7 +368,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[4] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -271,7 +381,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[4] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -300,7 +410,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[5] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -313,7 +423,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[5] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -345,7 +455,7 @@ type TickEpochRequest_Body struct { func (x *TickEpochRequest_Body) Reset() { *x = TickEpochRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -358,7 +468,7 @@ func (x *TickEpochRequest_Body) String() string { func (*TickEpochRequest_Body) ProtoMessage() {} func (x *TickEpochRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -383,7 +493,7 @@ type TickEpochResponse_Body struct { func (x *TickEpochResponse_Body) Reset() { *x = TickEpochResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -396,7 +506,7 @@ func (x *TickEpochResponse_Body) String() string { func (*TickEpochResponse_Body) ProtoMessage() {} func (x *TickEpochResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -412,6 +522,91 @@ func (*TickEpochResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{3, 0} } +type RemoveNodeRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` +} + +func (x *RemoveNodeRequest_Body) Reset() { + *x = RemoveNodeRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveNodeRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveNodeRequest_Body) ProtoMessage() {} + +func (x *RemoveNodeRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveNodeRequest_Body.ProtoReflect.Descriptor instead. +func (*RemoveNodeRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{4, 0} +} + +func (x *RemoveNodeRequest_Body) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} + +type RemoveNodeResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *RemoveNodeResponse_Body) Reset() { + *x = RemoveNodeResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveNodeResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveNodeResponse_Body) ProtoMessage() {} + +func (x *RemoveNodeResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveNodeResponse_Body.ProtoReflect.Descriptor instead. +func (*RemoveNodeResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{5, 0} +} + var File_pkg_services_control_ir_service_proto protoreflect.FileDescriptor var file_pkg_services_control_ir_service_proto_rawDesc = []byte{ @@ -458,23 +653,46 @@ var file_pkg_services_control_ir_service_proto_rawDesc = []byte{ 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xa6, 0x01, - 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x12, 0x4c, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, - 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, - 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, - 0x0a, 0x09, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x1b, 0x2e, 0x69, 0x72, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, - 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, - 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, - 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, - 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2f, 0x69, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x98, 0x01, + 0x0a, 0x11, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x21, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x18, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x88, 0x01, 0x0a, 0x12, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, + 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, + 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, + 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x32, 0xf1, 0x01, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x09, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, + 0x68, 0x12, 0x1b, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, + 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, + 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, + 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x0a, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x69, 0x72, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, + 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, + 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, + 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x73, 0x2f, 0x69, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -489,38 +707,48 @@ func file_pkg_services_control_ir_service_proto_rawDescGZIP() []byte { return file_pkg_services_control_ir_service_proto_rawDescData } -var file_pkg_services_control_ir_service_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_pkg_services_control_ir_service_proto_msgTypes = make([]protoimpl.MessageInfo, 12) var file_pkg_services_control_ir_service_proto_goTypes = []interface{}{ (*HealthCheckRequest)(nil), // 0: ircontrol.HealthCheckRequest (*HealthCheckResponse)(nil), // 1: ircontrol.HealthCheckResponse (*TickEpochRequest)(nil), // 2: ircontrol.TickEpochRequest (*TickEpochResponse)(nil), // 3: ircontrol.TickEpochResponse - (*HealthCheckRequest_Body)(nil), // 4: ircontrol.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 5: ircontrol.HealthCheckResponse.Body - (*TickEpochRequest_Body)(nil), // 6: ircontrol.TickEpochRequest.Body - (*TickEpochResponse_Body)(nil), // 7: ircontrol.TickEpochResponse.Body - (*Signature)(nil), // 8: ircontrol.Signature - (HealthStatus)(0), // 9: ircontrol.HealthStatus + (*RemoveNodeRequest)(nil), // 4: ircontrol.RemoveNodeRequest + (*RemoveNodeResponse)(nil), // 5: ircontrol.RemoveNodeResponse + (*HealthCheckRequest_Body)(nil), // 6: ircontrol.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 7: ircontrol.HealthCheckResponse.Body + (*TickEpochRequest_Body)(nil), // 8: ircontrol.TickEpochRequest.Body + (*TickEpochResponse_Body)(nil), // 9: ircontrol.TickEpochResponse.Body + (*RemoveNodeRequest_Body)(nil), // 10: ircontrol.RemoveNodeRequest.Body + (*RemoveNodeResponse_Body)(nil), // 11: ircontrol.RemoveNodeResponse.Body + (*Signature)(nil), // 12: ircontrol.Signature + (HealthStatus)(0), // 13: ircontrol.HealthStatus } var file_pkg_services_control_ir_service_proto_depIdxs = []int32{ - 4, // 0: ircontrol.HealthCheckRequest.body:type_name -> ircontrol.HealthCheckRequest.Body - 8, // 1: ircontrol.HealthCheckRequest.signature:type_name -> ircontrol.Signature - 5, // 2: ircontrol.HealthCheckResponse.body:type_name -> ircontrol.HealthCheckResponse.Body - 8, // 3: ircontrol.HealthCheckResponse.signature:type_name -> ircontrol.Signature - 6, // 4: ircontrol.TickEpochRequest.body:type_name -> ircontrol.TickEpochRequest.Body - 8, // 5: ircontrol.TickEpochRequest.signature:type_name -> ircontrol.Signature - 7, // 6: ircontrol.TickEpochResponse.body:type_name -> ircontrol.TickEpochResponse.Body - 8, // 7: ircontrol.TickEpochResponse.signature:type_name -> ircontrol.Signature - 9, // 8: ircontrol.HealthCheckResponse.Body.health_status:type_name -> ircontrol.HealthStatus - 0, // 9: ircontrol.ControlService.HealthCheck:input_type -> ircontrol.HealthCheckRequest - 2, // 10: ircontrol.ControlService.TickEpoch:input_type -> ircontrol.TickEpochRequest - 1, // 11: ircontrol.ControlService.HealthCheck:output_type -> ircontrol.HealthCheckResponse - 3, // 12: ircontrol.ControlService.TickEpoch:output_type -> ircontrol.TickEpochResponse - 11, // [11:13] is the sub-list for method output_type - 9, // [9:11] is the sub-list for method input_type - 9, // [9:9] is the sub-list for extension type_name - 9, // [9:9] is the sub-list for extension extendee - 0, // [0:9] is the sub-list for field type_name + 6, // 0: ircontrol.HealthCheckRequest.body:type_name -> ircontrol.HealthCheckRequest.Body + 12, // 1: ircontrol.HealthCheckRequest.signature:type_name -> ircontrol.Signature + 7, // 2: ircontrol.HealthCheckResponse.body:type_name -> ircontrol.HealthCheckResponse.Body + 12, // 3: ircontrol.HealthCheckResponse.signature:type_name -> ircontrol.Signature + 8, // 4: ircontrol.TickEpochRequest.body:type_name -> ircontrol.TickEpochRequest.Body + 12, // 5: ircontrol.TickEpochRequest.signature:type_name -> ircontrol.Signature + 9, // 6: ircontrol.TickEpochResponse.body:type_name -> ircontrol.TickEpochResponse.Body + 12, // 7: ircontrol.TickEpochResponse.signature:type_name -> ircontrol.Signature + 10, // 8: ircontrol.RemoveNodeRequest.body:type_name -> ircontrol.RemoveNodeRequest.Body + 12, // 9: ircontrol.RemoveNodeRequest.signature:type_name -> ircontrol.Signature + 11, // 10: ircontrol.RemoveNodeResponse.body:type_name -> ircontrol.RemoveNodeResponse.Body + 12, // 11: ircontrol.RemoveNodeResponse.signature:type_name -> ircontrol.Signature + 13, // 12: ircontrol.HealthCheckResponse.Body.health_status:type_name -> ircontrol.HealthStatus + 0, // 13: ircontrol.ControlService.HealthCheck:input_type -> ircontrol.HealthCheckRequest + 2, // 14: ircontrol.ControlService.TickEpoch:input_type -> ircontrol.TickEpochRequest + 4, // 15: ircontrol.ControlService.RemoveNode:input_type -> ircontrol.RemoveNodeRequest + 1, // 16: ircontrol.ControlService.HealthCheck:output_type -> ircontrol.HealthCheckResponse + 3, // 17: ircontrol.ControlService.TickEpoch:output_type -> ircontrol.TickEpochResponse + 5, // 18: ircontrol.ControlService.RemoveNode:output_type -> ircontrol.RemoveNodeResponse + 16, // [16:19] is the sub-list for method output_type + 13, // [13:16] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name } func init() { file_pkg_services_control_ir_service_proto_init() } @@ -579,7 +807,7 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*RemoveNodeRequest); i { case 0: return &v.state case 1: @@ -591,7 +819,7 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { + switch v := v.(*RemoveNodeResponse); i { case 0: return &v.state case 1: @@ -603,7 +831,7 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TickEpochRequest_Body); i { + switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state case 1: @@ -615,6 +843,30 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HealthCheckResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_ir_service_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TickEpochRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_ir_service_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TickEpochResponse_Body); i { case 0: return &v.state @@ -626,6 +878,30 @@ func file_pkg_services_control_ir_service_proto_init() { return nil } } + file_pkg_services_control_ir_service_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoveNodeRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_ir_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoveNodeResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -633,7 +909,7 @@ func file_pkg_services_control_ir_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_ir_service_proto_rawDesc, NumEnums: 0, - NumMessages: 8, + NumMessages: 12, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/ir/service.proto b/pkg/services/control/ir/service.proto index 5862e8fbd..d647db0df 100644 --- a/pkg/services/control/ir/service.proto +++ b/pkg/services/control/ir/service.proto @@ -12,6 +12,8 @@ service ControlService { rpc HealthCheck (HealthCheckRequest) returns (HealthCheckResponse); // Forces a new epoch to be signaled by the IR node with high probability. rpc TickEpoch (TickEpochRequest) returns (TickEpochResponse); + // Forces a node removal to be signaled by the IR node with high probability. + rpc RemoveNode (RemoveNodeRequest) returns (RemoveNodeResponse); } // Health check request. @@ -57,3 +59,19 @@ message TickEpochResponse { Body body = 1; Signature signature = 2; } + +message RemoveNodeRequest { + message Body{ + bytes key = 1; + } + + Body body = 1; + Signature signature = 2; +} + +message RemoveNodeResponse { + message Body{} + + Body body = 1; + Signature signature = 2; +} diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index d480f0b50..c93253105 100644 --- a/pkg/services/control/ir/service_frostfs.pb.go +++ b/pkg/services/control/ir/service_frostfs.pb.go @@ -296,3 +296,154 @@ func (x *TickEpochResponse) ReadSignedData(buf []byte) ([]byte, error) { func (x *TickEpochResponse) SetSignature(sig *Signature) { x.Signature = sig } + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveNodeRequest_Body) StableSize() (size int) { + size += proto.BytesSize(1, x.Key) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveNodeRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.BytesMarshal(1, buf[offset:], x.Key) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveNodeRequest) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveNodeRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *RemoveNodeRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *RemoveNodeRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *RemoveNodeRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveNodeResponse_Body) StableSize() (size int) { + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveNodeResponse_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveNodeResponse) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveNodeResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *RemoveNodeResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *RemoveNodeResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *RemoveNodeResponse) SetSignature(sig *Signature) { + x.Signature = sig +} diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index 700d340ca..6ba214da0 100644 --- a/pkg/services/control/ir/service_grpc.pb.go +++ b/pkg/services/control/ir/service_grpc.pb.go @@ -26,6 +26,8 @@ type ControlServiceClient interface { HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) // Forces a new epoch to be signaled by the IR node with high probability. TickEpoch(ctx context.Context, in *TickEpochRequest, opts ...grpc.CallOption) (*TickEpochResponse, error) + // Forces a node removal to be signaled by the IR node with high probability. + RemoveNode(ctx context.Context, in *RemoveNodeRequest, opts ...grpc.CallOption) (*RemoveNodeResponse, error) } type controlServiceClient struct { @@ -54,6 +56,15 @@ func (c *controlServiceClient) TickEpoch(ctx context.Context, in *TickEpochReque return out, nil } +func (c *controlServiceClient) RemoveNode(ctx context.Context, in *RemoveNodeRequest, opts ...grpc.CallOption) (*RemoveNodeResponse, error) { + out := new(RemoveNodeResponse) + err := c.cc.Invoke(ctx, "/ircontrol.ControlService/RemoveNode", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // ControlServiceServer is the server API for ControlService service. // All implementations should embed UnimplementedControlServiceServer // for forward compatibility @@ -62,6 +73,8 @@ type ControlServiceServer interface { HealthCheck(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) // Forces a new epoch to be signaled by the IR node with high probability. TickEpoch(context.Context, *TickEpochRequest) (*TickEpochResponse, error) + // Forces a node removal to be signaled by the IR node with high probability. + RemoveNode(context.Context, *RemoveNodeRequest) (*RemoveNodeResponse, error) } // UnimplementedControlServiceServer should be embedded to have forward compatible implementations. @@ -74,6 +87,9 @@ func (UnimplementedControlServiceServer) HealthCheck(context.Context, *HealthChe func (UnimplementedControlServiceServer) TickEpoch(context.Context, *TickEpochRequest) (*TickEpochResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method TickEpoch not implemented") } +func (UnimplementedControlServiceServer) RemoveNode(context.Context, *RemoveNodeRequest) (*RemoveNodeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RemoveNode not implemented") +} // UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ControlServiceServer will @@ -122,6 +138,24 @@ func _ControlService_TickEpoch_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _ControlService_RemoveNode_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RemoveNodeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).RemoveNode(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ircontrol.ControlService/RemoveNode", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).RemoveNode(ctx, req.(*RemoveNodeRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -137,6 +171,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "TickEpoch", Handler: _ControlService_TickEpoch_Handler, }, + { + MethodName: "RemoveNode", + Handler: _ControlService_RemoveNode_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "pkg/services/control/ir/service.proto", From a181c9e434636aff4d75f12fcf7b5448314ea646 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 10 May 2023 17:16:15 +0300 Subject: [PATCH 0218/1943] [#332] gc: Add additional logging Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 6 ++++++ pkg/local_object_storage/shard/gc.go | 12 ++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 9b6e03499..69a3b229a 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -478,4 +478,10 @@ const ( FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" // Error in ../node/cmd/frostfs-node/tree.go FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" // Error in ../node/cmd/frostfs-node/control.go CommonApplicationStarted = "application started" // Info in ../node/cmd/frostfs-ir/main.go + ShardGCCollectingExpiredObjectsStarted = "collecting expired objects started" + ShardGCCollectingExpiredObjectsCompleted = "collecting expired objects completed" + ShardGCCollectingExpiredLocksStarted = "collecting expired locks started" + ShardGCCollectingExpiredLocksCompleted = "collecting expired locks completed" + ShardGCRemoveGarbageStarted = "garbage remove started" + ShardGCRemoveGarbageCompleted = "garbage remove completed" ) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 86995cd06..82876e675 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -204,6 +204,9 @@ func (s *Shard) removeGarbage() { return } + s.log.Debug(logs.ShardGCRemoveGarbageStarted) + defer s.log.Debug(logs.ShardGCRemoveGarbageCompleted) + buf := make([]oid.Address, 0, s.rmBatchSize) var iterPrm meta.GarbageIterationPrm @@ -259,6 +262,9 @@ func (s *Shard) getExpiredObjectsParameters() (workersCount, batchSize int) { } func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { + s.log.Debug(logs.ShardGCCollectingExpiredObjectsStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) + defer s.log.Debug(logs.ShardGCCollectingExpiredObjectsCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) + workersCount, batchSize := s.getExpiredObjectsParameters() errGroup, egCtx := errgroup.WithContext(ctx) @@ -344,6 +350,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { log := s.log.With(zap.Uint64("epoch", epoch)) log.Debug(logs.ShardStartedExpiredTombstonesHandling) + defer log.Debug(logs.ShardFinishedExpiredTombstonesHandling) const tssDeleteBatch = 50 tss := make([]meta.TombstonedObject, 0, tssDeleteBatch) @@ -400,11 +407,12 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { tss = tss[:0] tssExp = tssExp[:0] } - - log.Debug(logs.ShardFinishedExpiredTombstonesHandling) } func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { + s.log.Debug(logs.ShardGCCollectingExpiredLocksStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) + defer s.log.Debug(logs.ShardGCCollectingExpiredLocksCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) + workersCount, batchSize := s.getExpiredObjectsParameters() errGroup, egCtx := errgroup.WithContext(ctx) From 800eb5e983fba1b9dc4e5048df1f71229486c5bb Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 26 Apr 2023 15:45:57 +0300 Subject: [PATCH 0219/1943] [#125] ir: Reconfigure pprof and metrics on SIGHUP Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 1 + cmd/frostfs-ir/config.go | 2 + cmd/frostfs-ir/httpcomponent.go | 80 ++++++++++++++++++++++++++++ cmd/frostfs-ir/main.go | 94 +++++++++------------------------ cmd/frostfs-ir/metrics.go | 21 ++++++++ cmd/frostfs-ir/pprof.go | 21 ++++++++ pkg/innerring/initialization.go | 9 ---- pkg/innerring/innerring.go | 2 +- pkg/metrics/innerring.go | 4 +- 9 files changed, 154 insertions(+), 80 deletions(-) create mode 100644 cmd/frostfs-ir/httpcomponent.go create mode 100644 cmd/frostfs-ir/metrics.go create mode 100644 cmd/frostfs-ir/pprof.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ed434da5..58fe9c3ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Changelog for FrostFS Node ### Added - Support impersonate bearer token (#229) - Change log level on SIGHUP for ir (#125) +- Reload pprof and metrics on SIGHUP for ir (#125) ### Changed ### Fixed diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index c4787f965..2e2aa1613 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -60,6 +60,8 @@ func watchForSignal(cancel func()) { if err != nil { log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) } + pprofCmp.reload() + metricsCmp.reload() log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) case syscall.SIGTERM, syscall.SIGINT: log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go new file mode 100644 index 000000000..d3cda8930 --- /dev/null +++ b/cmd/frostfs-ir/httpcomponent.go @@ -0,0 +1,80 @@ +package main + +import ( + "fmt" + "net/http" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" + "go.uber.org/zap" +) + +type httpComponent struct { + srv *httputil.Server + address string + addressKey string + name string + handler http.Handler + shutdownDur time.Duration + shutdownTimeoutKey string + enabled bool + enabledKey string +} + +func (c *httpComponent) init() { + log.Info(fmt.Sprintf("init %s", c.name)) + c.enabled = cfg.GetBool(c.enabledKey) + c.address = cfg.GetString(c.addressKey) + c.shutdownDur = cfg.GetDuration(c.shutdownTimeoutKey) + + if c.enabled { + c.srv = httputil.New( + httputil.HTTPSrvPrm{ + Address: c.address, + Handler: c.handler, + }, + httputil.WithShutdownTimeout(c.shutdownDur), + ) + } else { + log.Info(fmt.Sprintf("%s is disabled, skip", c.name)) + c.srv = nil + } +} + +func (c *httpComponent) start() { + if c.srv != nil { + log.Info(fmt.Sprintf("start %s", c.name)) + wg.Add(1) + go func() { + exitErr(c.srv.Serve()) + wg.Done() + }() + } +} + +func (c *httpComponent) shutdown() error { + if c.srv != nil { + log.Info(fmt.Sprintf("shutdown %s", c.name)) + return c.srv.Shutdown() + } + return nil +} + +func (c *httpComponent) reload() { + log.Info(fmt.Sprintf("reload %s", c.name)) + enabled := cfg.GetBool(c.enabledKey) + address := cfg.GetString(c.addressKey) + dur := cfg.GetDuration(c.shutdownTimeoutKey) + if enabled != c.enabled || address != c.address || dur != c.shutdownDur { + log.Info(fmt.Sprintf("%s config updated", c.name)) + if err := c.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), + ) + } else { + c.init() + c.start() + } + } +} diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 932b90404..84dff9101 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -4,16 +4,13 @@ import ( "context" "flag" "fmt" - "net/http" "os" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" - httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -29,15 +26,16 @@ const ( ) var ( - wg = new(sync.WaitGroup) - intErr = make(chan error) // internal inner ring errors - logPrm = new(logger.Prm) - innerRing *innerring.Server - httpServers []*httputil.Server - log *logger.Logger - cfg *viper.Viper - configFile *string - configDir *string + wg = new(sync.WaitGroup) + intErr = make(chan error) // internal inner ring errors + logPrm = new(logger.Prm) + innerRing *innerring.Server + pprofCmp *httpComponent + metricsCmp *httpComponent + log *logger.Logger + cfg *viper.Viper + configFile *string + configDir *string ) func exitErr(err error) { @@ -73,19 +71,17 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) - initHTTPServers(cfg) + pprofCmp = newPprofComponent() + pprofCmp.init() + + metricsCmp = newMetricsComponent() + metricsCmp.init() innerRing, err = innerring.New(ctx, log, cfg, intErr) exitErr(err) - // start HTTP servers - for _, srv := range httpServers { - wg.Add(1) - go func(srv *httputil.Server) { - exitErr(srv.Serve()) - wg.Done() - }(srv) - } + pprofCmp.start() + metricsCmp.start() // start inner ring err = innerRing.Start(ctx, intErr) @@ -103,54 +99,16 @@ func main() { log.Info(logs.FrostFSIRApplicationStopped) } -func initHTTPServers(cfg *viper.Viper) { - items := []struct { - cfgPrefix string - handler func() http.Handler - }{ - {"pprof", httputil.Handler}, - {"prometheus", promhttp.Handler}, +func shutdown() { + innerRing.Stop() + if err := metricsCmp.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), + ) } - - httpServers = make([]*httputil.Server, 0, len(items)) - - for _, item := range items { - if !cfg.GetBool(item.cfgPrefix + ".enabled") { - log.Info(item.cfgPrefix + " is disabled, skip") - continue - } - - addr := cfg.GetString(item.cfgPrefix + ".address") - - var prm httputil.HTTPSrvPrm - - prm.Address = addr - prm.Handler = item.handler() - - httpServers = append(httpServers, - httputil.New(prm, - httputil.WithShutdownTimeout( - cfg.GetDuration(item.cfgPrefix+".shutdown_timeout"), - ), - ), + if err := pprofCmp.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), ) } } - -func shutdown() { - innerRing.Stop() - - // shut down HTTP servers - for _, srv := range httpServers { - wg.Add(1) - go func(srv *httputil.Server) { - err := srv.Shutdown() - if err != nil { - log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, - zap.String("error", err.Error()), - ) - } - wg.Done() - }(srv) - } -} diff --git a/cmd/frostfs-ir/metrics.go b/cmd/frostfs-ir/metrics.go new file mode 100644 index 000000000..debeb7d60 --- /dev/null +++ b/cmd/frostfs-ir/metrics.go @@ -0,0 +1,21 @@ +package main + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +const ( + prometheusEnabledKey = "prometheus.enabled" + prometheusAddressKey = "prometheus.address" + prometheusShutdownTimeoutKey = "prometheus.shutdown_timeout" +) + +func newMetricsComponent() *httpComponent { + return &httpComponent{ + name: "prometheus", + enabledKey: prometheusEnabledKey, + addressKey: prometheusAddressKey, + shutdownTimeoutKey: prometheusShutdownTimeoutKey, + handler: metrics.Handler(), + } +} diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go new file mode 100644 index 000000000..0d48c11de --- /dev/null +++ b/cmd/frostfs-ir/pprof.go @@ -0,0 +1,21 @@ +package main + +import ( + httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" +) + +const ( + pprofEnabledKey = "pprof.enabled" + pprofAddressKey = "pprof.address" + pprofShutdownTimeoutKey = "pprof.shutdown_timeout" +) + +func newPprofComponent() *httpComponent { + return &httpComponent{ + name: "pprof", + enabledKey: pprofEnabledKey, + addressKey: pprofAddressKey, + shutdownTimeoutKey: pprofShutdownTimeoutKey, + handler: httputil.Handler(), + } +} diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 84d08c4c6..1dc1d40ea 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -16,7 +16,6 @@ import ( nodevalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation" addrvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/maddress" statevalidation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" @@ -535,11 +534,3 @@ func (s *Server) initKey(cfg *viper.Viper) error { s.key = acc.PrivateKey() return nil } - -func (s *Server) initMetrics(cfg *viper.Viper) { - if cfg.GetString("prometheus.address") == "" { - return - } - m := metrics.NewInnerRingMetrics() - s.metrics = &m -} diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index deb546f08..9119ff201 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -396,7 +396,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - server.initMetrics(cfg) + server.metrics = metrics.NewInnerRingMetrics() return server, nil } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 05b76f9c9..79db424b8 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -11,7 +11,7 @@ type InnerRingServiceMetrics struct { } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. -func NewInnerRingMetrics() InnerRingServiceMetrics { +func NewInnerRingMetrics() *InnerRingServiceMetrics { var ( epoch = newGauge(prometheus.GaugeOpts{ Namespace: namespace, @@ -30,7 +30,7 @@ func NewInnerRingMetrics() InnerRingServiceMetrics { mustRegister(epoch) mustRegister(health) - return InnerRingServiceMetrics{ + return &InnerRingServiceMetrics{ epoch: epoch, health: health, } From a5f118a987c6df52a3dbda19c0a0f7fe151516eb Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 17 Apr 2023 19:00:54 +0300 Subject: [PATCH 0220/1943] [#337] subscriber: Drop unused UnsubscribeForNotification It's not really needed, closing the connection works fine when exiting and normally the app doesn't need to unsubscribe at all. Signed-off-by: Roman Khimov Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 1 - pkg/morph/subscriber/subscriber.go | 9 --------- 2 files changed, 10 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 69a3b229a..936042d2d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -195,7 +195,6 @@ const ( EventIgnoreNilNotaryEventHandler = "ignore nil notary event handler" // Warn in ../node/pkg/morph/event/listener.go EventIgnoreHandlerOfNotaryEventWoParser = "ignore handler of notary event w/o parser" // Warn in ../node/pkg/morph/event/listener.go EventIgnoreNilBlockHandler = "ignore nil block handler" // Warn in ../node/pkg/morph/event/listener.go - SubscriberUnsubscribeForNotification = "unsubscribe for notification" // Error in ../node/pkg/morph/subscriber/subscriber.go SubscriberRemoteNotificationChannelHasBeenClosed = "remote notification channel has been closed" // Warn in ../node/pkg/morph/subscriber/subscriber.go SubscriberCantCastNotifyEventValueToTheNotifyStruct = "can't cast notify event value to the notify struct" // Error in ../node/pkg/morph/subscriber/subscriber.go SubscriberNewNotificationEventFromSidechain = "new notification event from sidechain" // Debug in ../node/pkg/morph/subscriber/subscriber.go diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index a2e1c32eb..2d478c9c3 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -27,7 +27,6 @@ type ( // Subscriber is an interface of the NotificationEvent listener. Subscriber interface { SubscribeForNotification(...util.Uint160) error - UnsubscribeForNotification() BlockNotifications() error SubscribeForNotaryRequests(mainTXSigner util.Uint160) error @@ -97,14 +96,6 @@ func (s *subscriber) SubscribeForNotification(contracts ...util.Uint160) error { return nil } -func (s *subscriber) UnsubscribeForNotification() { - err := s.client.UnsubscribeAll() - if err != nil { - s.log.Error(logs.SubscriberUnsubscribeForNotification, - zap.Error(err)) - } -} - func (s *subscriber) Close() { s.client.Close() } From 35fdf6f315dfc58c1bfcc8855aea9a08cc2ba461 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 May 2023 17:05:24 +0300 Subject: [PATCH 0221/1943] [#337] morph: Move subscription logic to subscriber Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 26 +-- pkg/morph/client/constructor.go | 27 +-- pkg/morph/client/multi.go | 157 +++------------- pkg/morph/client/notifications.go | 280 ++--------------------------- pkg/morph/subscriber/subscriber.go | 275 ++++++++++++++++++++-------- 5 files changed, 258 insertions(+), 507 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 1c33fa5e0..832315018 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -69,9 +69,6 @@ type Client struct { // on every normal call. switchLock *sync.RWMutex - notifications chan rpcclient.Notification - subsInfo // protected with switchLock - // channel for internal stop closeChan chan struct{} @@ -566,26 +563,11 @@ func (c *Client) IsValidScript(script []byte, signers []transaction.Signer) (val // NotificationChannel returns channel than receives subscribed // notification from the connected RPC node. -// Channel is closed when connection to the RPC node has been -// lost without the possibility of recovery. +// Channel is closed when connection to the RPC node is lost. func (c *Client) NotificationChannel() <-chan rpcclient.Notification { - return c.notifications -} - -// inactiveMode switches Client to an inactive mode: -// - notification channel is closed; -// - all the new RPC request would return ErrConnectionLost; -// - inactiveModeCb is called if not nil. -func (c *Client) inactiveMode() { - c.switchLock.Lock() - defer c.switchLock.Unlock() - - close(c.notifications) - c.inactive = true - - if c.cfg.inactiveModeCb != nil { - c.cfg.inactiveModeCb() - } + c.switchLock.RLock() + defer c.switchLock.RUnlock() + return c.client.Notifications //lint:ignore SA1019 waits for neo-go v0.102.0 https://github.com/nspcc-dev/neo-go/pull/2980 } func (c *Client) setActor(act *actor.Actor) { diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 4232b349d..1f2a1eb8d 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -10,11 +10,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" - "github.com/nspcc-dev/neo-go/pkg/core/block" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/util" @@ -108,21 +105,13 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } cli := &Client{ - cache: newClientCache(), - logger: cfg.logger, - acc: acc, - accAddr: accAddr, - cfg: *cfg, - switchLock: &sync.RWMutex{}, - notifications: make(chan rpcclient.Notification), - subsInfo: subsInfo{ - blockRcv: make(chan *block.Block), - notificationRcv: make(chan *state.ContainedNotificationEvent), - notaryReqRcv: make(chan *result.NotaryRequestEvent), - subscribedEvents: make(map[util.Uint160]string), - subscribedNotaryEvents: make(map[util.Uint160]string), - }, - closeChan: make(chan struct{}), + cache: newClientCache(), + logger: cfg.logger, + acc: acc, + accAddr: accAddr, + cfg: *cfg, + switchLock: &sync.RWMutex{}, + closeChan: make(chan struct{}), } cli.endpoints.init(cfg.endpoints) @@ -162,7 +151,7 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } cli.setActor(act) - go cli.notificationLoop(ctx) + go cli.closeWaiter(ctx) return cli, nil } diff --git a/pkg/morph/client/multi.go b/pkg/morph/client/multi.go index fab90b446..e006ca69a 100644 --- a/pkg/morph/client/multi.go +++ b/pkg/morph/client/multi.go @@ -6,11 +6,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "github.com/nspcc-dev/neo-go/pkg/core/block" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/neorpc" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "go.uber.org/zap" ) @@ -34,7 +29,8 @@ func (e *endpoints) init(ee []Endpoint) { e.list = ee } -func (c *Client) switchRPC(ctx context.Context) bool { +// SwitchRPC performs reconnection and returns true if it was successful. +func (c *Client) SwitchRPC(ctx context.Context) bool { c.switchLock.Lock() defer c.switchLock.Unlock() @@ -58,20 +54,8 @@ func (c *Client) switchRPC(ctx context.Context) bool { c.logger.Info(logs.ClientConnectionToTheNewRPCNodeHasBeenEstablished, zap.String("endpoint", newEndpoint)) - subs, ok := c.restoreSubscriptions(ctx, cli, newEndpoint, false) - if !ok { - // new WS client does not allow - // restoring subscription, client - // could not work correctly => - // closing connection to RPC node - // to switch to another one - cli.Close() - continue - } - c.client = cli c.setActor(act) - c.subsInfo = subs if c.cfg.switchInterval != 0 && !c.switchIsActive.Load() && c.endpoints.list[c.endpoints.curr].Priority != c.endpoints.list[0].Priority { @@ -82,97 +66,21 @@ func (c *Client) switchRPC(ctx context.Context) bool { return true } + c.inactive = true + + if c.cfg.inactiveModeCb != nil { + c.cfg.inactiveModeCb() + } return false } -func (c *Client) notificationLoop(ctx context.Context) { - var e any - var ok bool - - for { - c.switchLock.RLock() - bChan := c.blockRcv - nChan := c.notificationRcv - nrChan := c.notaryReqRcv - c.switchLock.RUnlock() - - select { - case <-ctx.Done(): - _ = c.UnsubscribeAll() - c.close() - - return - case <-c.closeChan: - _ = c.UnsubscribeAll() - c.close() - - return - case e, ok = <-bChan: - case e, ok = <-nChan: - case e, ok = <-nrChan: - } - - if ok { - c.routeEvent(ctx, e) - continue - } - - if !c.reconnect(ctx) { - return - } - } -} - -func (c *Client) routeEvent(ctx context.Context, e any) { - typedNotification := rpcclient.Notification{Value: e} - - switch e.(type) { - case *block.Block: - typedNotification.Type = neorpc.BlockEventID - case *state.ContainedNotificationEvent: - typedNotification.Type = neorpc.NotificationEventID - case *result.NotaryRequestEvent: - typedNotification.Type = neorpc.NotaryRequestEventID - } - +func (c *Client) closeWaiter(ctx context.Context) { select { - case c.notifications <- typedNotification: case <-ctx.Done(): - _ = c.UnsubscribeAll() - c.close() case <-c.closeChan: - _ = c.UnsubscribeAll() - c.close() } -} - -func (c *Client) reconnect(ctx context.Context) bool { - if closeErr := c.client.GetError(); closeErr != nil { - c.logger.Warn(logs.ClientSwitchingToTheNextRPCNode, - zap.String("reason", closeErr.Error()), - ) - } else { - // neo-go client was closed by calling `Close` - // method, that happens only when a client has - // switched to the more prioritized RPC - return true - } - - if !c.switchRPC(ctx) { - c.logger.Error(logs.ClientCouldNotEstablishConnectionToAnyRPCNode) - - // could not connect to all endpoints => - // switch client to inactive mode - c.inactiveMode() - - return false - } - - // TODO(@carpawell): call here some callback retrieved in constructor - // of the client to allow checking chain state since during switch - // process some notification could be lost - - return true + _ = c.UnsubscribeAll() + c.close() } func (c *Client) switchToMostPrioritized(ctx context.Context) { @@ -218,36 +126,28 @@ mainLoop: continue } - if subs, ok := c.restoreSubscriptions(ctx, cli, tryE, true); ok { - c.switchLock.Lock() - - // higher priority node could have been - // connected in the other goroutine - if e.Priority >= c.endpoints.list[c.endpoints.curr].Priority { - cli.Close() - c.switchLock.Unlock() - return - } - - c.client.Close() - c.cache.invalidate() - c.client = cli - c.setActor(act) - c.subsInfo = subs - c.endpoints.curr = i + c.switchLock.Lock() + // higher priority node could have been + // connected in the other goroutine + if e.Priority >= c.endpoints.list[c.endpoints.curr].Priority { + cli.Close() c.switchLock.Unlock() - - c.logger.Info(logs.ClientSwitchedToTheHigherPriorityRPC, - zap.String("endpoint", tryE)) - return } - c.logger.Warn(logs.ClientCouldNotRestoreSideChainSubscriptionsUsingNode, - zap.String("endpoint", tryE), - zap.Error(err), - ) + c.client.Close() + c.cache.invalidate() + c.client = cli + c.setActor(act) + c.endpoints.curr = i + + c.switchLock.Unlock() + + c.logger.Info(logs.ClientSwitchedToTheHigherPriorityRPC, + zap.String("endpoint", tryE)) + + return } } } @@ -255,6 +155,7 @@ mainLoop: // close closes notification channel and wrapped WS client. func (c *Client) close() { - close(c.notifications) + c.switchLock.RLock() + defer c.switchLock.RUnlock() c.client.Close() } diff --git a/pkg/morph/client/notifications.go b/pkg/morph/client/notifications.go index 69eafc659..dbca00d7c 100644 --- a/pkg/morph/client/notifications.go +++ b/pkg/morph/client/notifications.go @@ -1,16 +1,11 @@ package client import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/neorpc" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/util" - "go.uber.org/zap" ) // Close closes connection to the remote side making @@ -24,71 +19,46 @@ func (c *Client) Close() { close(c.closeChan) } -// SubscribeForExecutionNotifications adds subscription for notifications -// generated during contract transaction execution to this instance of client. +// ReceiveExecutionNotifications performs subscription for notifications +// generated during contract execution. Events are sent to the specified channel. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) SubscribeForExecutionNotifications(contract util.Uint160) error { +func (c *Client) ReceiveExecutionNotifications(contract util.Uint160, ch chan<- *state.ContainedNotificationEvent) (string, error) { c.switchLock.Lock() defer c.switchLock.Unlock() if c.inactive { - return ErrConnectionLost + return "", ErrConnectionLost } - _, subscribed := c.subscribedEvents[contract] - if subscribed { - // no need to subscribe one more time - return nil - } - - id, err := c.client.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, c.notificationRcv) - if err != nil { - return err - } - - c.subscribedEvents[contract] = id - - return nil + return c.client.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, ch) } -// SubscribeForNewBlocks adds subscription for new block events to this -// instance of client. +// ReceiveBlocks performs subscription for new block events. Events are sent +// to the specified channel. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) SubscribeForNewBlocks() error { +func (c *Client) ReceiveBlocks(ch chan<- *block.Block) (string, error) { c.switchLock.Lock() defer c.switchLock.Unlock() if c.inactive { - return ErrConnectionLost + return "", ErrConnectionLost } - if c.subscribedToBlocks { - // no need to subscribe one more time - return nil - } - - _, err := c.client.ReceiveBlocks(nil, c.blockRcv) - if err != nil { - return err - } - - c.subscribedToBlocks = true - - return nil + return c.client.ReceiveBlocks(nil, ch) } -// SubscribeForNotaryRequests adds subscription for notary request payloads +// ReceiveNotaryRequests performsn subscription for notary request payloads // addition or removal events to this instance of client. Passed txSigner is // used as filter: subscription is only for the notary requests that must be -// signed by txSigner. +// signed by txSigner. Events are sent to the specified channel. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) SubscribeForNotaryRequests(txSigner util.Uint160) error { +func (c *Client) ReceiveNotaryRequests(txSigner util.Uint160, ch chan<- *result.NotaryRequestEvent) (string, error) { if c.notary == nil { panic(notaryNotEnabledPanicMsg) } @@ -97,30 +67,17 @@ func (c *Client) SubscribeForNotaryRequests(txSigner util.Uint160) error { defer c.switchLock.Unlock() if c.inactive { - return ErrConnectionLost + return "", ErrConnectionLost } - _, subscribed := c.subscribedNotaryEvents[txSigner] - if subscribed { - // no need to subscribe one more time - return nil - } - - id, err := c.client.ReceiveNotaryRequests(&neorpc.TxFilter{Signer: &txSigner}, c.notaryReqRcv) - if err != nil { - return err - } - - c.subscribedNotaryEvents[txSigner] = id - - return nil + return c.client.ReceiveNotaryRequests(&neorpc.TxFilter{Signer: &txSigner}, ch) } -// UnsubscribeContract removes subscription for given contract event stream. +// Unsubscribe performs unsubscription for the given subscription ID. // // Returns ErrConnectionLost if client has not been able to establish // connection to any of passed RPC endpoints. -func (c *Client) UnsubscribeContract(contract util.Uint160) error { +func (c *Client) Unsubscribe(subID string) error { c.switchLock.Lock() defer c.switchLock.Unlock() @@ -128,55 +85,7 @@ func (c *Client) UnsubscribeContract(contract util.Uint160) error { return ErrConnectionLost } - _, subscribed := c.subscribedEvents[contract] - if !subscribed { - // no need to unsubscribe contract - // without subscription - return nil - } - - err := c.client.Unsubscribe(c.subscribedEvents[contract]) - if err != nil { - return err - } - - delete(c.subscribedEvents, contract) - - return nil -} - -// UnsubscribeNotaryRequest removes subscription for given notary requests -// signer. -// -// Returns ErrConnectionLost if client has not been able to establish -// connection to any of passed RPC endpoints. -func (c *Client) UnsubscribeNotaryRequest(signer util.Uint160) error { - if c.notary == nil { - panic(notaryNotEnabledPanicMsg) - } - - c.switchLock.Lock() - defer c.switchLock.Unlock() - - if c.inactive { - return ErrConnectionLost - } - - _, subscribed := c.subscribedNotaryEvents[signer] - if !subscribed { - // no need to unsubscribe signer's - // requests without subscription - return nil - } - - err := c.client.Unsubscribe(c.subscribedNotaryEvents[signer]) - if err != nil { - return err - } - - delete(c.subscribedNotaryEvents, signer) - - return nil + return c.client.Unsubscribe(subID) } // UnsubscribeAll removes all active subscriptions of current client. @@ -191,163 +100,10 @@ func (c *Client) UnsubscribeAll() error { return ErrConnectionLost } - // no need to unsubscribe if there are - // no active subscriptions - if len(c.subscribedEvents) == 0 && len(c.subscribedNotaryEvents) == 0 && - !c.subscribedToBlocks { - return nil - } - err := c.client.UnsubscribeAll() if err != nil { return err } - c.subscribedEvents = make(map[util.Uint160]string) - c.subscribedNotaryEvents = make(map[util.Uint160]string) - c.subscribedToBlocks = false - return nil } - -// subsInfo includes channels for ws notifications; -// cached subscription information. -type subsInfo struct { - blockRcv chan *block.Block - notificationRcv chan *state.ContainedNotificationEvent - notaryReqRcv chan *result.NotaryRequestEvent - - subscribedToBlocks bool - subscribedEvents map[util.Uint160]string - subscribedNotaryEvents map[util.Uint160]string -} - -// restoreSubscriptions restores subscriptions according to cached -// information about them. -// -// If it is NOT a background operation switchLock MUST be held. -// Returns a pair: the second is a restoration status and the first -// one contains subscription information applied to the passed cli -// and receivers for the updated subscriptions. -// Does not change Client instance. -func (c *Client) restoreSubscriptions(ctx context.Context, cli *rpcclient.WSClient, endpoint string, background bool) (si subsInfo, ok bool) { - var ( - err error - id string - ) - - stopCh := make(chan struct{}) - defer close(stopCh) - - blockRcv := make(chan *block.Block) - notificationRcv := make(chan *state.ContainedNotificationEvent) - notaryReqRcv := make(chan *result.NotaryRequestEvent) - - c.startListen(ctx, stopCh, blockRcv, notificationRcv, notaryReqRcv, background) - - if background { - c.switchLock.RLock() - defer c.switchLock.RUnlock() - } - - si.subscribedToBlocks = c.subscribedToBlocks - si.subscribedEvents = copySubsMap(c.subscribedEvents) - si.subscribedNotaryEvents = copySubsMap(c.subscribedNotaryEvents) - si.blockRcv = blockRcv - si.notificationRcv = notificationRcv - si.notaryReqRcv = notaryReqRcv - - // new block events restoration - if si.subscribedToBlocks { - _, err = cli.ReceiveBlocks(nil, blockRcv) - if err != nil { - c.logger.Error(logs.ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch, - zap.String("endpoint", endpoint), - zap.Error(err), - ) - - return - } - } - - // notification events restoration - for contract := range si.subscribedEvents { - contract := contract // See https://github.com/nspcc-dev/neo-go/issues/2890 - id, err = cli.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, notificationRcv) - if err != nil { - c.logger.Error(logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, - zap.String("endpoint", endpoint), - zap.Error(err), - ) - - return - } - - si.subscribedEvents[contract] = id - } - - // notary notification events restoration - if c.notary != nil { - for signer := range si.subscribedNotaryEvents { - signer := signer // See https://github.com/nspcc-dev/neo-go/issues/2890 - id, err = cli.ReceiveNotaryRequests(&neorpc.TxFilter{Signer: &signer}, notaryReqRcv) - if err != nil { - c.logger.Error(logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, - zap.String("endpoint", endpoint), - zap.Error(err), - ) - - return - } - - si.subscribedNotaryEvents[signer] = id - } - } - - return si, true -} - -func (c *Client) startListen(ctx context.Context, stopCh <-chan struct{}, blockRcv <-chan *block.Block, - notificationRcv <-chan *state.ContainedNotificationEvent, notaryReqRcv <-chan *result.NotaryRequestEvent, background bool) { - // neo-go WS client says to _always_ read notifications - // from its channel. Subscribing to any notification - // while not reading them in another goroutine may - // lead to a dead-lock, thus that async side notification - // listening while restoring subscriptions - - go func() { - var e any - var ok bool - - for { - select { - case <-stopCh: - return - case e, ok = <-blockRcv: - case e, ok = <-notificationRcv: - case e, ok = <-notaryReqRcv: - } - - if !ok { - return - } - - if background { - // background client (test) switch, no need to send - // any notification, just preventing dead-lock - continue - } - - c.routeEvent(ctx, e) - } - }() -} - -func copySubsMap(m map[util.Uint160]string) map[util.Uint160]string { - newM := make(map[util.Uint160]string, len(m)) - for k, v := range m { - newM[k] = v - } - - return newM -} diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 2d478c9c3..383d58407 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -11,8 +11,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/neorpc" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" ) @@ -35,16 +35,27 @@ type ( Close() } + subChannels struct { + NotifyChan chan *state.ContainedNotificationEvent + BlockChan chan *block.Block + NotaryChan chan *result.NotaryRequestEvent + } + subscriber struct { *sync.RWMutex log *logger.Logger client *client.Client notifyChan chan *state.ContainedNotificationEvent - - blockChan chan *block.Block - + blockChan chan *block.Block notaryChan chan *result.NotaryRequestEvent + + current subChannels + + // cached subscription information + subscribedEvents map[util.Uint160]bool + subscribedNotaryEvents map[util.Uint160]bool + subscribedToNewBlocks bool } // Params is a group of Subscriber constructor parameters. @@ -75,22 +86,28 @@ func (s *subscriber) SubscribeForNotification(contracts ...util.Uint160) error { s.Lock() defer s.Unlock() - notifyIDs := make(map[util.Uint160]struct{}, len(contracts)) + notifyIDs := make([]string, 0, len(contracts)) for i := range contracts { + if s.subscribedEvents[contracts[i]] { + continue + } // subscribe to contract notifications - err := s.client.SubscribeForExecutionNotifications(contracts[i]) + id, err := s.client.ReceiveExecutionNotifications(contracts[i], s.current.NotifyChan) if err != nil { // if there is some error, undo all subscriptions and return error - for hash := range notifyIDs { - _ = s.client.UnsubscribeContract(hash) + for _, id := range notifyIDs { + _ = s.client.Unsubscribe(id) } return err } // save notification id - notifyIDs[contracts[i]] = struct{}{} + notifyIDs = append(notifyIDs, id) + } + for i := range contracts { + s.subscribedEvents[contracts[i]] = true } return nil @@ -101,82 +118,34 @@ func (s *subscriber) Close() { } func (s *subscriber) BlockNotifications() error { - if err := s.client.SubscribeForNewBlocks(); err != nil { + s.Lock() + defer s.Unlock() + if s.subscribedToNewBlocks { + return nil + } + if _, err := s.client.ReceiveBlocks(s.current.BlockChan); err != nil { return fmt.Errorf("could not subscribe for new block events: %w", err) } + s.subscribedToNewBlocks = true + return nil } func (s *subscriber) SubscribeForNotaryRequests(mainTXSigner util.Uint160) error { - if err := s.client.SubscribeForNotaryRequests(mainTXSigner); err != nil { + s.Lock() + defer s.Unlock() + if s.subscribedNotaryEvents[mainTXSigner] { + return nil + } + if _, err := s.client.ReceiveNotaryRequests(mainTXSigner, s.current.NotaryChan); err != nil { return fmt.Errorf("could not subscribe for notary request events: %w", err) } + s.subscribedNotaryEvents[mainTXSigner] = true return nil } -func (s *subscriber) routeNotifications(ctx context.Context) { - notificationChan := s.client.NotificationChannel() - - for { - select { - case <-ctx.Done(): - return - case notification, ok := <-notificationChan: - if !ok { - s.log.Warn(logs.SubscriberRemoteNotificationChannelHasBeenClosed) - close(s.notifyChan) - close(s.blockChan) - close(s.notaryChan) - - return - } - - switch notification.Type { - case neorpc.NotificationEventID: - notifyEvent, ok := notification.Value.(*state.ContainedNotificationEvent) - if !ok { - s.log.Error(logs.SubscriberCantCastNotifyEventValueToTheNotifyStruct, - zap.String("received type", fmt.Sprintf("%T", notification.Value)), - ) - continue - } - - s.log.Debug(logs.SubscriberNewNotificationEventFromSidechain, - zap.String("name", notifyEvent.Name), - ) - - s.notifyChan <- notifyEvent - case neorpc.BlockEventID: - b, ok := notification.Value.(*block.Block) - if !ok { - s.log.Error(logs.SubscriberCantCastBlockEventValueToBlock, - zap.String("received type", fmt.Sprintf("%T", notification.Value)), - ) - continue - } - - s.blockChan <- b - case neorpc.NotaryRequestEventID: - notaryRequest, ok := notification.Value.(*result.NotaryRequestEvent) - if !ok { - s.log.Error(logs.SubscriberCantCastNotifyEventValueToTheNotaryRequestStruct, - zap.String("received type", fmt.Sprintf("%T", notification.Value)), - ) - continue - } - - s.notaryChan <- notaryRequest - default: - s.log.Debug(logs.SubscriberUnsupportedNotificationFromTheChain, - zap.Uint8("type", uint8(notification.Type)), - ) - } - } - } -} - // New is a constructs Neo:Morph event listener and returns Subscriber interface. func New(ctx context.Context, p *Params) (Subscriber, error) { switch { @@ -200,16 +169,170 @@ func New(ctx context.Context, p *Params) (Subscriber, error) { notifyChan: make(chan *state.ContainedNotificationEvent), blockChan: make(chan *block.Block), notaryChan: make(chan *result.NotaryRequestEvent), - } - // Worker listens all events from neo-go websocket and puts them - // into corresponding channel. It may be notifications, transactions, - // new blocks. For now only notifications. + current: newSubChannels(), + + subscribedEvents: make(map[util.Uint160]bool), + subscribedNotaryEvents: make(map[util.Uint160]bool), + } + // Worker listens all events from temporary NeoGo channel and puts them + // into corresponding permanent channels. go sub.routeNotifications(ctx) return sub, nil } +func (s *subscriber) routeNotifications(ctx context.Context) { + var ( + // TODO: not needed after nspcc-dev/neo-go#2980. + cliCh = s.client.NotificationChannel() + restoreCh = make(chan bool) + restoreInProgress bool + ) + +routeloop: + for { + var connLost bool + s.RLock() + curr := s.current + s.RUnlock() + select { + case <-ctx.Done(): + break routeloop + case ev, ok := <-curr.NotifyChan: + if ok { + s.notifyChan <- ev + } else { + connLost = true + } + case ev, ok := <-curr.BlockChan: + if ok { + s.blockChan <- ev + } else { + connLost = true + } + case ev, ok := <-curr.NotaryChan: + if ok { + s.notaryChan <- ev + } else { + connLost = true + } + case _, ok := <-cliCh: + connLost = !ok + case ok := <-restoreCh: + restoreInProgress = false + if !ok { + connLost = true + } + } + if connLost { + if !restoreInProgress { + restoreInProgress, cliCh = s.switchEndpoint(ctx, restoreCh) + if !restoreInProgress { + break routeloop + } + curr.drain() + } else { // Avoid getting additional !ok events. + s.Lock() + s.current.NotifyChan = nil + s.current.BlockChan = nil + s.current.NotaryChan = nil + s.Unlock() + } + } + } + close(s.notifyChan) + close(s.blockChan) + close(s.notaryChan) +} + +func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) (bool, <-chan rpcclient.Notification) { + s.log.Info("RPC connection lost, attempting reconnect") + if !s.client.SwitchRPC(ctx) { + s.log.Error("can't switch RPC node") + return false, nil + } + + cliCh := s.client.NotificationChannel() + + s.Lock() + chs := newSubChannels() + go func() { + finishCh <- s.restoreSubscriptions(chs.NotifyChan, chs.BlockChan, chs.NotaryChan) + }() + s.current = chs + s.Unlock() + + return true, cliCh +} + +func newSubChannels() subChannels { + return subChannels{ + NotifyChan: make(chan *state.ContainedNotificationEvent), + BlockChan: make(chan *block.Block), + NotaryChan: make(chan *result.NotaryRequestEvent), + } +} + +func (s *subChannels) drain() { +drainloop: + for { + select { + case _, ok := <-s.NotifyChan: + if !ok { + s.NotifyChan = nil + } + case _, ok := <-s.BlockChan: + if !ok { + s.BlockChan = nil + } + case _, ok := <-s.NotaryChan: + if !ok { + s.NotaryChan = nil + } + default: + break drainloop + } + } +} + +// restoreSubscriptions restores subscriptions according to +// cached information about them. +func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotificationEvent, + blCh chan<- *block.Block, notaryCh chan<- *result.NotaryRequestEvent) bool { + var err error + + // new block events restoration + if s.subscribedToNewBlocks { + _, err = s.client.ReceiveBlocks(blCh) + if err != nil { + s.log.Error(logs.ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch, zap.Error(err)) + return false + } + } + + // notification events restoration + for contract := range s.subscribedEvents { + contract := contract // See https://github.com/nspcc-dev/neo-go/issues/2890 + _, err = s.client.ReceiveExecutionNotifications(contract, notifCh) + if err != nil { + s.log.Error(logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) + return false + } + } + + // notary notification events restoration + for signer := range s.subscribedNotaryEvents { + signer := signer // See https://github.com/nspcc-dev/neo-go/issues/2890 + _, err = s.client.ReceiveNotaryRequests(signer, notaryCh) + if err != nil { + s.log.Error(logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) + return false + } + } + return true +} + // awaitHeight checks if remote client has least expected block height and // returns error if it is not reached that height after timeout duration. // This function is required to avoid connections to unsynced RPC nodes, because From ddcc156eccc3e82dc50738bc5f3aef9bb50896a2 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 18 Apr 2023 18:40:20 +0300 Subject: [PATCH 0222/1943] [#337] morph: Use Notary Actor for notary requests Signed-off-by: Anna Shaleva Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 2 - pkg/morph/client/notary.go | 394 ++++++++-------------- pkg/morph/event/notary_preparator.go | 25 +- pkg/morph/event/notary_preparator_test.go | 196 +++++++---- 4 files changed, 281 insertions(+), 336 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 832315018..284e065fb 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -153,8 +153,6 @@ func (e *notHaltStateError) Error() string { ) } -var errEmptyInvocationScript = errors.New("got empty invocation script from neo node") - // implementation of error interface for FrostFS-specific errors. type frostfsError struct { err error diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 3e21911e1..e478a5118 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -1,6 +1,7 @@ package client import ( + "crypto/elliptic" "encoding/binary" "errors" "fmt" @@ -18,10 +19,12 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/neorpc" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/notary" sc "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm/opcode" + "github.com/nspcc-dev/neo-go/pkg/vm" + "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" "go.uber.org/zap" ) @@ -408,32 +411,32 @@ func (c *Client) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { return fmt.Errorf("could not fetch current alphabet keys: %w", err) } - multiaddrAccount, err := c.notaryMultisigAccount(alphabetList, false, true) + cosigners, err := c.notaryCosignersFromTx(mainTx, alphabetList) if err != nil { return err } - // mainTX is expected to be pre-validated: second witness must exist and be empty - mainTx.Scripts[1].VerificationScript = multiaddrAccount.GetVerificationScript() - mainTx.Scripts[1].InvocationScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - multiaddrAccount.SignHashable(c.rpcActor.GetNetwork(), mainTx)..., - ) + nAct, err := notary.NewActor(c.client, cosigners, c.acc) + if err != nil { + return err + } + + // Sign exactly the same transaction we've got from the received Notary request. + err = nAct.Sign(mainTx) + if err != nil { + return fmt.Errorf("faield to sign notary request: %w", err) + } + + mainH, fbH, untilActual, err := nAct.Notarize(mainTx, nil) - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - resp, err := c.client.SignAndPushP2PNotaryRequest(mainTx, - []byte{byte(opcode.RET)}, - -1, - 0, - c.notary.fallbackTime, - c.acc) if err != nil && !alreadyOnChainError(err) { return err } c.logger.Debug(logs.ClientNotaryRequestWithPreparedMainTXInvoked, - zap.Uint32("fallback_valid_for", c.notary.fallbackTime), - zap.Stringer("tx_hash", resp.Hash().Reverse())) + zap.String("tx_hash", mainH.StringLE()), + zap.Uint32("valid_until_block", untilActual), + zap.String("fallback_hash", fbH.StringLE())) return nil } @@ -449,70 +452,147 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint return err } - cosigners, err := c.notaryCosigners(invokedByAlpha, alphabetList, committee) - if err != nil { - return err - } - - params, err := invocationParams(args...) - if err != nil { - return err - } - - test, err := c.makeTestInvocation(contract, method, params, cosigners) - if err != nil { - return err - } - - multiaddrAccount, err := c.notaryMultisigAccount(alphabetList, committee, invokedByAlpha) - if err != nil { - return err - } - until, err := c.getUntilValue(vub) if err != nil { return err } - mainTx, err := c.buildMainTx(invokedByAlpha, nonce, alphabetList, test, cosigners, multiaddrAccount, until) + cosigners, err := c.notaryCosigners(invokedByAlpha, alphabetList, committee) if err != nil { return err } - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - resp, err := c.client.SignAndPushP2PNotaryRequest(mainTx, - []byte{byte(opcode.RET)}, - -1, - 0, - c.notary.fallbackTime, - c.acc) + nAct, err := notary.NewActor(c.client, cosigners, c.acc) + if err != nil { + return err + } + + mainH, fbH, untilActual, err := nAct.Notarize(nAct.MakeTunedCall(contract, method, nil, func(r *result.Invoke, t *transaction.Transaction) error { + if r.State != vmstate.Halt.String() { + return wrapFrostFSError(¬HaltStateError{state: r.State, exception: r.FaultException}) + } + + t.ValidUntilBlock = until + t.Nonce = nonce + + return nil + }, args...)) + if err != nil && !alreadyOnChainError(err) { return err } c.logger.Debug(logs.ClientNotaryRequestInvoked, zap.String("method", method), - zap.Uint32("valid_until_block", until), - zap.Uint32("fallback_valid_for", c.notary.fallbackTime), - zap.Stringer("tx_hash", resp.Hash().Reverse())) + zap.Uint32("valid_until_block", untilActual), + zap.String("tx_hash", mainH.StringLE()), + zap.String("fallback_hash", fbH.StringLE())) return nil } -func (c *Client) makeTestInvocation(contract util.Uint160, method string, params []sc.Parameter, cosigners []transaction.Signer) (*result.Invoke, error) { - test, err := c.client.InvokeFunction(contract, method, params, cosigners) +func (c *Client) notaryCosignersFromTx(mainTx *transaction.Transaction, alphabetList keys.PublicKeys) ([]actor.SignerAccount, error) { + multiaddrAccount, err := c.notaryMultisigAccount(alphabetList, false, true) if err != nil { return nil, err } - if test.State != HaltState { - return nil, wrapFrostFSError(¬HaltStateError{state: test.State, exception: test.FaultException}) + // Here we need to add a committee signature (second witness) to the pre-validated + // main transaction without creating a new one. However, Notary actor demands the + // proper set of signers for constructor, thus, fill it from the main transaction's signers list. + s := make([]actor.SignerAccount, 2, 3) + s[0] = actor.SignerAccount{ + // Proxy contract that will pay for the execution. + Signer: mainTx.Signers[0], + Account: notary.FakeContractAccount(mainTx.Signers[0].Account), + } + s[1] = actor.SignerAccount{ + // Inner ring multisignature. + Signer: mainTx.Signers[1], + Account: multiaddrAccount, + } + if len(mainTx.Signers) > 3 { + // Invoker signature (simple signature account of storage node is expected). + var acc *wallet.Account + script := mainTx.Scripts[2].VerificationScript + if len(script) == 0 { + acc = notary.FakeContractAccount(mainTx.Signers[2].Account) + } else { + pubBytes, ok := vm.ParseSignatureContract(script) + if ok { + pub, err := keys.NewPublicKeyFromBytes(pubBytes, elliptic.P256()) + if err != nil { + return nil, fmt.Errorf("failed to parse verification script of signer #2: invalid public key: %w", err) + } + acc = notary.FakeSimpleAccount(pub) + } else { + m, pubsBytes, ok := vm.ParseMultiSigContract(script) + if !ok { + return nil, errors.New("failed to parse verification script of signer #2: unknown witness type") + } + pubs := make(keys.PublicKeys, len(pubsBytes)) + for i := range pubs { + pubs[i], err = keys.NewPublicKeyFromBytes(pubsBytes[i], elliptic.P256()) + if err != nil { + return nil, fmt.Errorf("failed to parse verification script of signer #2: invalid public key #%d: %w", i, err) + } + } + acc, err = notary.FakeMultisigAccount(m, pubs) + if err != nil { + return nil, fmt.Errorf("failed to create fake account for signer #2: %w", err) + } + } + } + s = append(s, actor.SignerAccount{ + Signer: mainTx.Signers[2], + Account: acc, + }) } - if len(test.Script) == 0 { - return nil, wrapFrostFSError(errEmptyInvocationScript) + return s, nil +} + +func (c *Client) notaryCosigners(invokedByAlpha bool, ir []*keys.PublicKey, committee bool) ([]actor.SignerAccount, error) { + multiaddrAccount, err := c.notaryMultisigAccount(ir, committee, invokedByAlpha) + if err != nil { + return nil, err } - return test, nil + s := make([]actor.SignerAccount, 2, 3) + // Proxy contract that will pay for the execution. + s[0] = actor.SignerAccount{ + Signer: transaction.Signer{ + Account: c.notary.proxy, + Scopes: transaction.None, + }, + Account: notary.FakeContractAccount(c.notary.proxy), + } + // Inner ring multisignature. + s[1] = actor.SignerAccount{ + Signer: transaction.Signer{ + Account: multiaddrAccount.ScriptHash(), + Scopes: c.cfg.signer.Scopes, + AllowedContracts: c.cfg.signer.AllowedContracts, + AllowedGroups: c.cfg.signer.AllowedGroups, + }, + Account: multiaddrAccount, + } + + if !invokedByAlpha { + // Invoker signature. + s = append(s, actor.SignerAccount{ + Signer: transaction.Signer{ + Account: hash.Hash160(c.acc.GetVerificationScript()), + Scopes: c.cfg.signer.Scopes, + AllowedContracts: c.cfg.signer.AllowedContracts, + AllowedGroups: c.cfg.signer.AllowedGroups, + }, + Account: c.acc, + }) + } + + // The last one is Notary contract that will be added to the signers list + // by Notary actor automatically. + return s, nil } func (c *Client) getUntilValue(vub *uint32) (uint32, error) { @@ -522,195 +602,6 @@ func (c *Client) getUntilValue(vub *uint32) (uint32, error) { return c.notaryTxValidationLimit() } -func (c *Client) buildMainTx(invokedByAlpha bool, nonce uint32, alphabetList keys.PublicKeys, test *result.Invoke, - cosigners []transaction.Signer, multiaddrAccount *wallet.Account, until uint32) (*transaction.Transaction, error) { - // after test invocation we build main multisig transaction - - u8n := uint8(len(alphabetList)) - - if !invokedByAlpha { - u8n++ - } - - // prepare main tx - mainTx := &transaction.Transaction{ - Nonce: nonce, - SystemFee: test.GasConsumed, - ValidUntilBlock: until, - Script: test.Script, - Attributes: []transaction.Attribute{ - { - Type: transaction.NotaryAssistedT, - Value: &transaction.NotaryAssisted{NKeys: u8n}, - }, - }, - Signers: cosigners, - } - - // calculate notary fee - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - notaryFee, err := c.client.CalculateNotaryFee(u8n) - if err != nil { - return nil, err - } - - // add network fee for cosigners - //nolint:staticcheck // waits for neo-go v0.99.3 with notary actors - //lint:ignore SA1019 https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/202 - err = c.client.AddNetworkFee( - mainTx, - notaryFee, - c.notaryAccounts(invokedByAlpha, multiaddrAccount)..., - ) - if err != nil { - return nil, err - } - - // define witnesses - mainTx.Scripts = c.notaryWitnesses(invokedByAlpha, multiaddrAccount, mainTx) - - return mainTx, nil -} - -func (c *Client) notaryCosigners(invokedByAlpha bool, ir []*keys.PublicKey, committee bool) ([]transaction.Signer, error) { - s := make([]transaction.Signer, 0, 4) - - // first we have proxy contract signature, as it will pay for the execution - s = append(s, transaction.Signer{ - Account: c.notary.proxy, - Scopes: transaction.None, - }) - - // then we have inner ring multiaddress signature - m := sigCount(ir, committee) - - multisigScript, err := sc.CreateMultiSigRedeemScript(m, ir) - if err != nil { - // wrap error as FrostFS-specific since the call is not related to any client - return nil, wrapFrostFSError(fmt.Errorf("can't create ir multisig redeem script: %w", err)) - } - - s = append(s, transaction.Signer{ - Account: hash.Hash160(multisigScript), - Scopes: c.cfg.signer.Scopes, - AllowedContracts: c.cfg.signer.AllowedContracts, - AllowedGroups: c.cfg.signer.AllowedGroups, - }) - - if !invokedByAlpha { - // then we have invoker signature - s = append(s, transaction.Signer{ - Account: hash.Hash160(c.acc.GetVerificationScript()), - Scopes: c.cfg.signer.Scopes, - AllowedContracts: c.cfg.signer.AllowedContracts, - AllowedGroups: c.cfg.signer.AllowedGroups, - }) - } - - // last one is a placeholder for notary contract signature - s = append(s, transaction.Signer{ - Account: c.notary.notary, - Scopes: transaction.None, - }) - - return s, nil -} - -func (c *Client) notaryAccounts(invokedByAlpha bool, multiaddr *wallet.Account) []*wallet.Account { - if multiaddr == nil { - return nil - } - - a := make([]*wallet.Account, 0, 4) - - // first we have proxy account, as it will pay for the execution - a = append(a, notary.FakeContractAccount(c.notary.proxy)) - - // then we have inner ring multiaddress account - a = append(a, multiaddr) - - if !invokedByAlpha { - // then we have invoker account - a = append(a, c.acc) - } - - // last one is a placeholder for notary contract account - a = append(a, &wallet.Account{ - Contract: &wallet.Contract{}, - }) - - return a -} - -func (c *Client) notaryWitnesses(invokedByAlpha bool, multiaddr *wallet.Account, tx *transaction.Transaction) []transaction.Witness { - if multiaddr == nil || tx == nil { - return nil - } - - w := make([]transaction.Witness, 0, 4) - - // first we have empty proxy witness, because notary will execute `Verify` - // method on the proxy contract to check witness - w = append(w, transaction.Witness{ - InvocationScript: []byte{}, - VerificationScript: []byte{}, - }) - - // then we have inner ring multiaddress witness - - // invocation script should be of the form: - // { PUSHDATA1, 64, signatureBytes... } - // to pass Notary module verification - var invokeScript []byte - - magicNumber := c.rpcActor.GetNetwork() - - if invokedByAlpha { - invokeScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - multiaddr.SignHashable(magicNumber, tx)..., - ) - } else { - // we can't provide alphabet node signature - // because Storage Node doesn't own alphabet's - // private key. Thus, add dummy witness with - // empty bytes instead of signature - invokeScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - make([]byte, 64)..., - ) - } - - w = append(w, transaction.Witness{ - InvocationScript: invokeScript, - VerificationScript: multiaddr.GetVerificationScript(), - }) - - if !invokedByAlpha { - // then we have invoker witness - invokeScript = append( - []byte{byte(opcode.PUSHDATA1), 64}, - c.acc.SignHashable(magicNumber, tx)..., - ) - - w = append(w, transaction.Witness{ - InvocationScript: invokeScript, - VerificationScript: c.acc.GetVerificationScript(), - }) - } - - // last one is a placeholder for notary contract witness - w = append(w, transaction.Witness{ - InvocationScript: append( - []byte{byte(opcode.PUSHDATA1), 64}, - make([]byte, 64)..., - ), - VerificationScript: []byte{}, - }) - - return w -} - func (c *Client) notaryMultisigAccount(ir []*keys.PublicKey, committee, invokedByAlpha bool) (*wallet.Account, error) { m := sigCount(ir, committee) @@ -767,21 +658,6 @@ func (c *Client) depositExpirationOf() (int64, error) { return currentTillBig.Int64(), nil } -func invocationParams(args ...any) ([]sc.Parameter, error) { - params := make([]sc.Parameter, 0, len(args)) - - for i := range args { - param, err := toStackParameter(args[i]) - if err != nil { - return nil, err - } - - params = append(params, param) - } - - return params, nil -} - // sigCount returns the number of required signature. // For FrostFS Alphabet M is a 2/3+1 of it (like in dBFT). // If committee is true, returns M as N/2+1. diff --git a/pkg/morph/event/notary_preparator.go b/pkg/morph/event/notary_preparator.go index f7b10d906..a8b7376fa 100644 --- a/pkg/morph/event/notary_preparator.go +++ b/pkg/morph/event/notary_preparator.go @@ -185,15 +185,15 @@ func (p Preparator) validateNotaryRequest(nr *payload.P2PNotaryRequest) error { } invokerWitness := ln == 4 - multiInvScript := nr.MainTransaction.Scripts[1].InvocationScript - - // alphabet node should handle only notary requests - // that have been sent unsigned (by storage nodes) => - // such main TXs should have either a dummy or an - // empty script as an invocation script + // alphabet node should handle only notary requests that do not yet have inner + // ring multisignature filled => such main TXs either have empty invocation script + // of the inner ring witness (in case if Notary Actor is used to create request) + // or have it filled with dummy bytes (if request was created manually with the old + // neo-go API) // // this check prevents notary flow recursion - if len(multiInvScript) > 0 && !bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript) { + if !(len(nr.MainTransaction.Scripts[1].InvocationScript) == 0 || + bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript)) { // compatibility with old version return ErrTXAlreadyHandled } @@ -220,12 +220,7 @@ func (p Preparator) validateNotaryRequest(nr *payload.P2PNotaryRequest) error { } // validate main TX expiration - err = p.validateExpiration(nr.FallbackTransaction) - if err != nil { - return err - } - - return nil + return p.validateExpiration(nr.FallbackTransaction) } func (p Preparator) validateParameterOpcodes(ops []Op) error { @@ -363,7 +358,9 @@ func (p Preparator) validateWitnesses(w []transaction.Witness, alphaKeys keys.Pu // the last one must be a placeholder for notary contract witness last := len(w) - 1 - if !bytes.Equal(w[last].InvocationScript, p.dummyInvocationScript) || len(w[last].VerificationScript) != 0 { + if !(len(w[last].InvocationScript) == 0 || // https://github.com/nspcc-dev/neo-go/pull/2981 + bytes.Equal(w[last].InvocationScript, p.dummyInvocationScript)) || // compatibility with old version + len(w[last].VerificationScript) != 0 { return errIncorrectNotaryPlaceholder } diff --git a/pkg/morph/event/notary_preparator_test.go b/pkg/morph/event/notary_preparator_test.go index d0463348d..b2e46890b 100644 --- a/pkg/morph/event/notary_preparator_test.go +++ b/pkg/morph/event/notary_preparator_test.go @@ -1,6 +1,7 @@ package event import ( + "fmt" "testing" "github.com/nspcc-dev/neo-go/pkg/vm" @@ -24,8 +25,9 @@ var ( alphaKeys keys.PublicKeys wrongAlphaKeys keys.PublicKeys - dummyInvocationScript = append([]byte{byte(opcode.PUSHDATA1), 64}, make([]byte, 64)...) - wrongDummyInvocationScript = append([]byte{byte(opcode.PUSHDATA1), 64, 1}, make([]byte, 63)...) + dummyAlphabetInvocationScript = []byte{} // expected to be empty if generated by Notary Actor, as requester can't fill it in + dummyAlphabetInvocationScriptOld = append([]byte{byte(opcode.PUSHDATA1), 64}, make([]byte, 64)...) // expected to be dummy if generated manually + wrongDummyInvocationScript = append([]byte{byte(opcode.PUSHDATA1), 64, 1}, make([]byte, 63)...) scriptHash util.Uint160 ) @@ -61,35 +63,37 @@ func TestPrepare_IncorrectScript(t *testing.T) { }, ) - t.Run("not contract call", func(t *testing.T) { - bw := io.NewBufBinWriter() + for _, dummyMultisig := range []bool{true, false} { // try both empty and dummy multisig/Notary invocation witness script + t.Run(fmt.Sprintf("not contract call, compat: %t", dummyMultisig), func(t *testing.T) { + bw := io.NewBufBinWriter() - emit.Int(bw.BinWriter, 4) - emit.String(bw.BinWriter, "test") - emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) - emit.Syscall(bw.BinWriter, interopnames.SystemContractCallNative) // any != interopnames.SystemContractCall + emit.Int(bw.BinWriter, 4) + emit.String(bw.BinWriter, "test") + emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) + emit.Syscall(bw.BinWriter, interopnames.SystemContractCallNative) // any != interopnames.SystemContractCall - nr := correctNR(bw.Bytes(), false) + nr := correctNR(bw.Bytes(), dummyMultisig, false) - _, err := preparator.Prepare(nr) + _, err := preparator.Prepare(nr) - require.EqualError(t, err, errNotContractCall.Error()) - }) + require.EqualError(t, err, errNotContractCall.Error()) + }) - t.Run("incorrect ", func(t *testing.T) { - bw := io.NewBufBinWriter() + t.Run(fmt.Sprintf("incorrect, compat: %t", dummyMultisig), func(t *testing.T) { + bw := io.NewBufBinWriter() - emit.Int(bw.BinWriter, -1) - emit.String(bw.BinWriter, "test") - emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) - emit.Syscall(bw.BinWriter, interopnames.SystemContractCall) + emit.Int(bw.BinWriter, -1) + emit.String(bw.BinWriter, "test") + emit.Bytes(bw.BinWriter, scriptHash.BytesBE()) + emit.Syscall(bw.BinWriter, interopnames.SystemContractCall) - nr := correctNR(bw.Bytes(), false) + nr := correctNR(bw.Bytes(), dummyMultisig, false) - _, err := preparator.Prepare(nr) + _, err := preparator.Prepare(nr) - require.EqualError(t, err, errIncorrectCallFlag.Error()) - }) + require.EqualError(t, err, errIncorrectCallFlag.Error()) + }) + } } func TestPrepare_IncorrectNR(t *testing.T) { @@ -209,7 +213,23 @@ func TestPrepare_IncorrectNR(t *testing.T) { InvocationScript: make([]byte, 1), }, { - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + {}, + }, + }, + expErr: errIncorrectProxyWitnesses, + }, + { + name: "incorrect main TX proxy witness compat", + addW: false, + mTX: mTX{ + scripts: []transaction.Witness{ + { + InvocationScript: make([]byte, 1), + }, + { + InvocationScript: dummyAlphabetInvocationScriptOld, }, {}, }, @@ -224,7 +244,22 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, { VerificationScript: wrongAlphaVerificationScript, - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + {}, + }, + }, + expErr: errIncorrectAlphabet, + }, + { + name: "incorrect main TX Alphabet witness compat", + addW: false, + mTX: mTX{ + scripts: []transaction.Witness{ + {}, + { + VerificationScript: wrongAlphaVerificationScript, + InvocationScript: dummyAlphabetInvocationScriptOld, }, {}, }, @@ -239,7 +274,24 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, { VerificationScript: alphaVerificationScript, - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + { + InvocationScript: wrongDummyInvocationScript, + }, + }, + }, + expErr: errIncorrectNotaryPlaceholder, + }, + { + name: "incorrect main TX Notary witness compat", + addW: false, + mTX: mTX{ + scripts: []transaction.Witness{ + {}, + { + VerificationScript: alphaVerificationScript, + InvocationScript: dummyAlphabetInvocationScriptOld, }, { InvocationScript: wrongDummyInvocationScript, @@ -289,7 +341,23 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, { VerificationScript: alphaVerificationScript, - InvocationScript: dummyInvocationScript, + InvocationScript: dummyAlphabetInvocationScript, + }, + {}, + {}, + }, + }, + expErr: errIncorrectInvokerWitnesses, + }, + { + name: "incorrect invoker TX Alphabet witness compat", + addW: true, + mTX: mTX{ + scripts: []transaction.Witness{ + {}, + { + VerificationScript: alphaVerificationScript, + InvocationScript: dummyAlphabetInvocationScriptOld, }, {}, {}, @@ -327,7 +395,7 @@ func TestPrepare_IncorrectNR(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - correctNR := correctNR(nil, test.addW) + correctNR := correctNR(nil, false, test.addW) incorrectNR = setIncorrectFields(*correctNR, test.mTX, test.fbTX) _, err = preparator.Prepare(&incorrectNR) @@ -372,40 +440,42 @@ func TestPrepare_CorrectNR(t *testing.T) { for _, test := range tests { for i := 0; i < 1; i++ { // run tests against 3 and 4 witness NR - additionalWitness := i == 0 - nr := correctNR(script(test.hash, test.method, test.args...), additionalWitness) + for _, dummyMultisig := range []bool{true, false} { // run tests against empty and dummy multisig/Notary witness + additionalWitness := i == 0 + nr := correctNR(script(test.hash, test.method, test.args...), dummyMultisig, additionalWitness) - event, err := preparator.Prepare(nr) + event, err := preparator.Prepare(nr) - require.NoError(t, err) - require.Equal(t, test.method, event.Type().String()) - require.Equal(t, test.hash.StringLE(), event.ScriptHash().StringLE()) - - // check args parsing - bw := io.NewBufBinWriter() - emit.Array(bw.BinWriter, test.args...) - - ctx := vm.NewContext(bw.Bytes()) - - opCode, param, err := ctx.Next() - require.NoError(t, err) - - for _, opGot := range event.Params() { - require.Equal(t, opCode, opGot.code) - require.Equal(t, param, opGot.param) - - opCode, param, err = ctx.Next() require.NoError(t, err) + require.Equal(t, test.method, event.Type().String()) + require.Equal(t, test.hash.StringLE(), event.ScriptHash().StringLE()) + + // check args parsing + bw := io.NewBufBinWriter() + emit.Array(bw.BinWriter, test.args...) + + ctx := vm.NewContext(bw.Bytes()) + + opCode, param, err := ctx.Next() + require.NoError(t, err) + + for _, opGot := range event.Params() { + require.Equal(t, opCode, opGot.code) + require.Equal(t, param, opGot.param) + + opCode, param, err = ctx.Next() + require.NoError(t, err) + } + + _, _, err = ctx.Next() // PACK opcode + require.NoError(t, err) + _, _, err = ctx.Next() // packing len opcode + require.NoError(t, err) + + opCode, _, err = ctx.Next() + require.NoError(t, err) + require.Equal(t, opcode.RET, opCode) } - - _, _, err = ctx.Next() // PACK opcode - require.NoError(t, err) - _, _, err = ctx.Next() // packing len opcode - require.NoError(t, err) - - opCode, _, err = ctx.Next() - require.NoError(t, err) - require.Equal(t, opcode.RET, opCode) } } } @@ -428,7 +498,7 @@ func script(hash util.Uint160, method string, args ...any) []byte { return bw.Bytes() } -func correctNR(script []byte, additionalWitness bool) *payload.P2PNotaryRequest { +func correctNR(script []byte, dummyMultisig, additionalWitness bool) *payload.P2PNotaryRequest { alphaVerificationScript, _ := smartcontract.CreateMultiSigRedeemScript(len(alphaKeys)*2/3+1, alphaKeys) signers := []transaction.Signer{ @@ -443,20 +513,24 @@ func correctNR(script []byte, additionalWitness bool) *payload.P2PNotaryRequest signers[2] = transaction.Signer{Account: hash.Hash160(alphaVerificationScript)} } + multisigInv := dummyAlphabetInvocationScript + if dummyMultisig { + multisigInv = dummyAlphabetInvocationScriptOld + } scripts := []transaction.Witness{ {}, { - InvocationScript: dummyInvocationScript, + InvocationScript: multisigInv, VerificationScript: alphaVerificationScript, }, { - InvocationScript: dummyInvocationScript, + InvocationScript: multisigInv, }, } if additionalWitness { // insert on element with index 2 scripts = append(scripts[:2+1], scripts[2:]...) scripts[2] = transaction.Witness{ - InvocationScript: dummyInvocationScript, + InvocationScript: multisigInv, VerificationScript: alphaVerificationScript, } } From bcdb0f330dd0d3b51ddbe0d02cccfe1af562d6c7 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 20 Apr 2023 18:53:48 +0300 Subject: [PATCH 0223/1943] [#337] morph: Completely remove fallbackTime from client cfg It's unused and not needed, default fallback lifetime is set by Notary actor. Signed-off-by: Anna Shaleva Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/notary.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index e478a5118..1ed1ca912 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -31,9 +31,8 @@ import ( type ( notaryInfo struct { - txValidTime uint32 // minimum amount of blocks when mainTx will be valid - roundTime uint32 // extra amount of blocks to synchronize sidechain height diff of inner ring nodes - fallbackTime uint32 // mainTx's ValidUntilBlock - fallbackTime + 1 is when fallbackTx is sent + txValidTime uint32 // minimum amount of blocks when mainTx will be valid + roundTime uint32 // extra amount of blocks to synchronize sidechain height diff of inner ring nodes alphabetSource AlphabetKeys // source of alphabet node keys to prepare witness @@ -44,7 +43,7 @@ type ( notaryCfg struct { proxy util.Uint160 - txValidTime, roundTime, fallbackTime uint32 + txValidTime, roundTime uint32 alphabetSource AlphabetKeys } @@ -54,9 +53,8 @@ type ( ) const ( - defaultNotaryValidTime = 50 - defaultNotaryRoundTime = 100 - defaultNotaryFallbackTime = 40 + defaultNotaryValidTime = 50 + defaultNotaryRoundTime = 100 notaryBalanceOfMethod = "balanceOf" notaryExpirationOfMethod = "expirationOf" @@ -72,7 +70,6 @@ func defaultNotaryConfig(c *Client) *notaryCfg { return ¬aryCfg{ txValidTime: defaultNotaryValidTime, roundTime: defaultNotaryRoundTime, - fallbackTime: defaultNotaryFallbackTime, alphabetSource: c.Committee, } } @@ -107,7 +104,6 @@ func (c *Client) EnableNotarySupport(opts ...NotaryOption) error { proxy: cfg.proxy, txValidTime: cfg.txValidTime, roundTime: cfg.roundTime, - fallbackTime: cfg.fallbackTime, alphabetSource: cfg.alphabetSource, notary: notary.Hash, } @@ -685,15 +681,6 @@ func WithRoundTime(t uint32) NotaryOption { } } -// WithFallbackTime returns a notary support option for client -// that specifies amount of blocks before fallbackTx will be sent. -// Should be less than TxValidTime. -func WithFallbackTime(t uint32) NotaryOption { - return func(c *notaryCfg) { - c.fallbackTime = t - } -} - // WithAlphabetSource returns a notary support option for client // that specifies function to return list of alphabet node keys. // By default notary subsystem uses committee as a source. This is From b480df4985a1be3cf7914638783a369a1f8f6f00 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 May 2023 18:14:00 +0300 Subject: [PATCH 0224/1943] [#337] container: Remove unused events Done in TrueCloudLab/frostfs-contract#16. Signed-off-by: Evgenii Stratonikov --- .../processors/container/processor.go | 65 +-------------- pkg/morph/event/container/delete.go | 39 --------- pkg/morph/event/container/delete_test.go | 60 -------------- pkg/morph/event/container/eacl.go | 50 ------------ pkg/morph/event/container/eacl_test.go | 80 ------------------- pkg/morph/event/container/put.go | 43 ---------- pkg/morph/event/container/put_test.go | 74 ----------------- 7 files changed, 2 insertions(+), 409 deletions(-) diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index d5af5e394..2141b0764 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -79,13 +79,6 @@ type NetworkState interface { HomomorphicHashDisabled() (bool, error) } -const ( - putNotification = "containerPut" - deleteNotification = "containerDelete" - - setEACLNotification = "setEACL" -) - // New creates a container contract processor instance. func New(p *Params) (*Processor, error) { switch { @@ -121,66 +114,12 @@ func New(p *Params) (*Processor, error) { // ListenerNotificationParsers for the 'event.Listener' event producer. func (cp *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { - if !cp.notaryDisabled { - return nil - } - - var ( - parsers = make([]event.NotificationParserInfo, 0, 3) - - p event.NotificationParserInfo - ) - - p.SetScriptHash(cp.cnrClient.ContractAddress()) - - // container put - p.SetType(event.TypeFromString(putNotification)) - p.SetParser(containerEvent.ParsePut) - parsers = append(parsers, p) - - // container delete - p.SetType(event.TypeFromString(deleteNotification)) - p.SetParser(containerEvent.ParseDelete) - parsers = append(parsers, p) - - // set eACL - p.SetType(event.TypeFromString(setEACLNotification)) - p.SetParser(containerEvent.ParseSetEACL) - parsers = append(parsers, p) - - return parsers + return nil } // ListenerNotificationHandlers for the 'event.Listener' event producer. func (cp *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { - if !cp.notaryDisabled { - return nil - } - - var ( - handlers = make([]event.NotificationHandlerInfo, 0, 3) - - h event.NotificationHandlerInfo - ) - - h.SetScriptHash(cp.cnrClient.ContractAddress()) - - // container put - h.SetType(event.TypeFromString(putNotification)) - h.SetHandler(cp.handlePut) - handlers = append(handlers, h) - - // container delete - h.SetType(event.TypeFromString(deleteNotification)) - h.SetHandler(cp.handleDelete) - handlers = append(handlers, h) - - // set eACL - h.SetType(event.TypeFromString(setEACLNotification)) - h.SetHandler(cp.handleSetEACL) - handlers = append(handlers, h) - - return handlers + return nil } // ListenerNotaryParsers for the 'event.Listener' notary event producer. diff --git a/pkg/morph/event/container/delete.go b/pkg/morph/event/container/delete.go index 7286ddcfc..4926af27d 100644 --- a/pkg/morph/event/container/delete.go +++ b/pkg/morph/event/container/delete.go @@ -44,45 +44,6 @@ func (d Delete) NotaryRequest() *payload.P2PNotaryRequest { const expectedItemNumDelete = 3 -// ParseDelete from notification into container event structure. -// -// Expects 3 stack items. -func ParseDelete(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Delete - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumDelete { - return nil, event.WrongNumberOfParameters(expectedItemNumDelete, ln) - } - - // parse container - ev.ContainerIDValue, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get container: %w", err) - } - - // parse signature - ev.SignatureValue, err = client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get signature: %w", err) - } - - // parse session token - ev.TokenValue, err = client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get session token: %w", err) - } - - return ev, nil -} - // DeleteSuccess structures notification event of successful container removal // thrown by Container contract. type DeleteSuccess struct { diff --git a/pkg/morph/event/container/delete_test.go b/pkg/morph/event/container/delete_test.go index 782f4aade..627c5fcf5 100644 --- a/pkg/morph/event/container/delete_test.go +++ b/pkg/morph/event/container/delete_test.go @@ -10,66 +10,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestParseDelete(t *testing.T) { - var ( - containerID = []byte("containreID") - signature = []byte("signature") - token = []byte("token") - ) - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - } - - _, err := ParseDelete(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(3, len(prms)).Error()) - }) - - t.Run("wrong container parameter", func(t *testing.T) { - _, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong signature parameter", func(t *testing.T) { - _, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerID), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong session token parameter", func(t *testing.T) { - _, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerID), - stackitem.NewByteArray(signature), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParseDelete(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerID), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(token), - })) - - require.NoError(t, err) - - require.Equal(t, Delete{ - ContainerIDValue: containerID, - SignatureValue: signature, - TokenValue: token, - }, ev) - }) -} - func TestParseDeleteSuccess(t *testing.T) { t.Run("wrong number of parameters", func(t *testing.T) { prms := []stackitem.Item{ diff --git a/pkg/morph/event/container/eacl.go b/pkg/morph/event/container/eacl.go index 41058ea43..4168d8842 100644 --- a/pkg/morph/event/container/eacl.go +++ b/pkg/morph/event/container/eacl.go @@ -1,11 +1,6 @@ package container import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/network/payload" ) @@ -54,48 +49,3 @@ func (x SetEACL) NotaryRequest() *payload.P2PNotaryRequest { } const expectedItemNumEACL = 4 - -// ParseSetEACL parses SetEACL notification event from list of stack items. -// -// Expects 4 stack items. -func ParseSetEACL(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev SetEACL - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumEACL { - return nil, event.WrongNumberOfParameters(expectedItemNumEACL, ln) - } - - // parse table - ev.TableValue, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not parse binary table: %w", err) - } - - // parse signature - ev.SignatureValue, err = client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not parse table signature: %w", err) - } - - // parse public key - ev.PublicKeyValue, err = client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not parse binary public key: %w", err) - } - - // parse session token - ev.TokenValue, err = client.BytesFromStackItem(params[3]) - if err != nil { - return nil, fmt.Errorf("could not get session token: %w", err) - } - - return ev, nil -} diff --git a/pkg/morph/event/container/eacl_test.go b/pkg/morph/event/container/eacl_test.go index 2f0598597..159f6cd9f 100644 --- a/pkg/morph/event/container/eacl_test.go +++ b/pkg/morph/event/container/eacl_test.go @@ -1,90 +1,10 @@ package container import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" ) -func TestParseEACL(t *testing.T) { - var ( - binaryTable = []byte("table") - signature = []byte("signature") - publicKey = []byte("pubkey") - token = []byte("token") - ) - - t.Run("wrong number of parameters", func(t *testing.T) { - items := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParseSetEACL(createNotifyEventFromItems(items)) - require.EqualError(t, err, event.WrongNumberOfParameters(4, len(items)).Error()) - }) - - t.Run("wrong container parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong signature parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong key parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewByteArray(signature), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong session token parameter", func(t *testing.T) { - _, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParseSetEACL(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binaryTable), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewByteArray(token), - })) - require.NoError(t, err) - - e := ev.(SetEACL) - - require.Equal(t, binaryTable, e.Table()) - require.Equal(t, signature, e.Signature()) - require.Equal(t, publicKey, e.PublicKey()) - require.Equal(t, token, e.SessionToken()) - }) -} - func createNotifyEventFromItems(items []stackitem.Item) *state.ContainedNotificationEvent { return &state.ContainedNotificationEvent{ NotificationEvent: state.NotificationEvent{ diff --git a/pkg/morph/event/container/put.go b/pkg/morph/event/container/put.go index d163c6836..335034bf3 100644 --- a/pkg/morph/event/container/put.go +++ b/pkg/morph/event/container/put.go @@ -65,49 +65,6 @@ func (x PutNamed) Zone() string { return x.zone } -// ParsePut from notification into container event structure. -func ParsePut(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Put - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumPut { - return nil, event.WrongNumberOfParameters(expectedItemNumPut, ln) - } - - // parse container - ev.rawContainer, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get container: %w", err) - } - - // parse signature - ev.signature, err = client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get signature: %w", err) - } - - // parse public key - ev.publicKey, err = client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get public key: %w", err) - } - - // parse session token - ev.token, err = client.BytesFromStackItem(params[3]) - if err != nil { - return nil, fmt.Errorf("could not get sesison token: %w", err) - } - - return ev, nil -} - // PutSuccess structures notification event of successful container creation // thrown by Container contract. type PutSuccess struct { diff --git a/pkg/morph/event/container/put_test.go b/pkg/morph/event/container/put_test.go index 2ccea296f..3622f9943 100644 --- a/pkg/morph/event/container/put_test.go +++ b/pkg/morph/event/container/put_test.go @@ -10,80 +10,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestParsePut(t *testing.T) { - var ( - containerData = []byte("containerData") - signature = []byte("signature") - publicKey = []byte("pubkey") - token = []byte("token") - ) - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParsePut(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(expectedItemNumPut, len(prms)).Error()) - }) - - t.Run("wrong container parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong signature parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong key parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewByteArray(signature), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong session token parameter", func(t *testing.T) { - _, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParsePut(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(containerData), - stackitem.NewByteArray(signature), - stackitem.NewByteArray(publicKey), - stackitem.NewByteArray(token), - })) - require.NoError(t, err) - - require.Equal(t, Put{ - rawContainer: containerData, - signature: signature, - publicKey: publicKey, - token: token, - }, ev) - }) -} - func TestParsePutSuccess(t *testing.T) { t.Run("wrong number of parameters", func(t *testing.T) { prms := []stackitem.Item{ From 47b0ec33c354413116e48a2111e7bb71ca699b31 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 May 2023 18:17:06 +0300 Subject: [PATCH 0225/1943] [#337] netmap: Remove unused events Done in TrueCloudLab/frostfs-contract#16. Signed-off-by: Evgenii Stratonikov --- pkg/innerring/processors/netmap/processor.go | 32 +---------- pkg/morph/event/netmap/add_peer.go | 28 ---------- pkg/morph/event/netmap/add_peer_test.go | 37 ------------ pkg/morph/event/netmap/update_peer.go | 44 --------------- pkg/morph/event/netmap/update_peer_test.go | 59 -------------------- 5 files changed, 1 insertion(+), 199 deletions(-) delete mode 100644 pkg/morph/event/netmap/update_peer_test.go diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index 522fa8b86..c466cfb1b 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -119,9 +119,7 @@ type ( ) const ( - newEpochNotification = "NewEpoch" - addPeerNotification = "AddPeer" - updatePeerStateNotification = "UpdateState" + newEpochNotification = "NewEpoch" ) // New creates network map contract processor instance. @@ -189,20 +187,6 @@ func (np *Processor) ListenerNotificationParsers() []event.NotificationParserInf p.SetParser(netmapEvent.ParseNewEpoch) parsers = append(parsers, p) - if !np.notaryDisabled { - return parsers - } - - // new peer event - p.SetType(addPeerNotification) - p.SetParser(netmapEvent.ParseAddPeer) - parsers = append(parsers, p) - - // update peer event - p.SetType(updatePeerStateNotification) - p.SetParser(netmapEvent.ParseUpdatePeer) - parsers = append(parsers, p) - return parsers } @@ -219,20 +203,6 @@ func (np *Processor) ListenerNotificationHandlers() []event.NotificationHandlerI i.SetHandler(np.handleNewEpoch) handlers = append(handlers, i) - if !np.notaryDisabled { - return handlers - } - - // new peer handler - i.SetType(addPeerNotification) - i.SetHandler(np.handleAddPeer) - handlers = append(handlers, i) - - // update peer handler - i.SetType(updatePeerStateNotification) - i.SetHandler(np.handleUpdateState) - handlers = append(handlers, i) - return handlers } diff --git a/pkg/morph/event/netmap/add_peer.go b/pkg/morph/event/netmap/add_peer.go index 6f839bada..80c5559fc 100644 --- a/pkg/morph/event/netmap/add_peer.go +++ b/pkg/morph/event/netmap/add_peer.go @@ -1,11 +1,6 @@ package netmap import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/network/payload" ) @@ -31,26 +26,3 @@ func (s AddPeer) NotaryRequest() *payload.P2PNotaryRequest { } const expectedItemNumAddPeer = 1 - -func ParseAddPeer(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev AddPeer - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumAddPeer { - return nil, event.WrongNumberOfParameters(expectedItemNumAddPeer, ln) - } - - ev.NodeBytes, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get raw nodeinfo: %w", err) - } - - return ev, nil -} diff --git a/pkg/morph/event/netmap/add_peer_test.go b/pkg/morph/event/netmap/add_peer_test.go index 0574c4048..4118bb8c8 100644 --- a/pkg/morph/event/netmap/add_peer_test.go +++ b/pkg/morph/event/netmap/add_peer_test.go @@ -1,47 +1,10 @@ package netmap import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" ) -func TestParseAddPeer(t *testing.T) { - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParseAddPeer(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(1, len(prms)).Error()) - }) - - t.Run("wrong first parameter type", func(t *testing.T) { - _, err := ParseAddPeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - info := []byte{1, 2, 3} - - ev, err := ParseAddPeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(info), - })) - - require.NoError(t, err) - require.Equal(t, AddPeer{ - NodeBytes: info, - }, ev) - }) -} - func createNotifyEventFromItems(items []stackitem.Item) *state.ContainedNotificationEvent { return &state.ContainedNotificationEvent{ NotificationEvent: state.NotificationEvent{ diff --git a/pkg/morph/event/netmap/update_peer.go b/pkg/morph/event/netmap/update_peer.go index f02ca408d..e29671131 100644 --- a/pkg/morph/event/netmap/update_peer.go +++ b/pkg/morph/event/netmap/update_peer.go @@ -1,13 +1,9 @@ package netmap import ( - "crypto/elliptic" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/network/payload" ) @@ -60,43 +56,3 @@ func (s *UpdatePeer) decodeState(state int64) error { } const expectedItemNumUpdatePeer = 2 - -func ParseUpdatePeer(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev UpdatePeer - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != expectedItemNumUpdatePeer { - return nil, event.WrongNumberOfParameters(expectedItemNumUpdatePeer, ln) - } - - // parse public key - key, err := client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get public key: %w", err) - } - - ev.PubKey, err = keys.NewPublicKeyFromBytes(key, elliptic.P256()) - if err != nil { - return nil, fmt.Errorf("could not parse public key: %w", err) - } - - // parse node status - st, err := client.IntFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get node status: %w", err) - } - - err = ev.decodeState(st) - if err != nil { - return nil, err - } - - return ev, nil -} diff --git a/pkg/morph/event/netmap/update_peer_test.go b/pkg/morph/event/netmap/update_peer_test.go deleted file mode 100644 index b79dd6385..000000000 --- a/pkg/morph/event/netmap/update_peer_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package netmap - -import ( - "math/big" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" -) - -func TestParseUpdatePeer(t *testing.T) { - priv, err := keys.NewPrivateKey() - require.NoError(t, err) - - publicKey := priv.PublicKey() - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - } - - _, err := ParseUpdatePeer(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(2, len(prms)).Error()) - }) - - t.Run("wrong first parameter type", func(t *testing.T) { - _, err := ParseUpdatePeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong second parameter type", func(t *testing.T) { - _, err := ParseUpdatePeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(publicKey.Bytes()), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - const state = netmap.NodeStateMaintenance - ev, err := ParseUpdatePeer(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewBigInteger(big.NewInt(int64(state))), - stackitem.NewByteArray(publicKey.Bytes()), - })) - require.NoError(t, err) - - require.Equal(t, UpdatePeer{ - PubKey: publicKey, - State: state, - }, ev) - }) -} From 969bfb603f7877eacb2d7c8140ebe001f7785264 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 13:53:33 +0300 Subject: [PATCH 0226/1943] [#321] shard/test: Parallelize TestShard_List ``` go test -count=1 -run TestShard_List -race . Before: 2.492s After: 0.109s ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/list_test.go | 57 ++++++++++++--------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index 8fac41a0f..e0759f668 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -2,6 +2,7 @@ package shard_test import ( "context" + "sync" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -9,22 +10,21 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) func TestShard_List(t *testing.T) { - sh := newShard(t, false) - shWC := newShard(t, true) - - defer func() { - releaseShard(sh, t) - releaseShard(shWC, t) - }() - t.Run("without write cache", func(t *testing.T) { + t.Parallel() + sh := newShard(t, false) + defer releaseShard(sh, t) testShardList(t, sh) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() + shWC := newShard(t, true) + defer releaseShard(shWC, t) testShardList(t, shWC) }) } @@ -33,30 +33,41 @@ func testShardList(t *testing.T, sh *shard.Shard) { const C = 10 const N = 5 + var mtx sync.Mutex objs := make(map[string]int) - var putPrm shard.PutPrm + var errG errgroup.Group + errG.SetLimit(C * N) for i := 0; i < C; i++ { - cnr := cidtest.ID() + errG.Go(func() error { + cnr := cidtest.ID() - for j := 0; j < N; j++ { - obj := testutil.GenerateObjectWithCID(cnr) - testutil.AddPayload(obj, 1<<2) + for j := 0; j < N; j++ { + errG.Go(func() error { + obj := testutil.GenerateObjectWithCID(cnr) + testutil.AddPayload(obj, 1<<2) - // add parent as virtual object, it must be ignored in List() - parent := testutil.GenerateObjectWithCID(cnr) - idParent, _ := parent.ID() - obj.SetParentID(idParent) - obj.SetParent(parent) + // add parent as virtual object, it must be ignored in List() + parent := testutil.GenerateObjectWithCID(cnr) + idParent, _ := parent.ID() + obj.SetParentID(idParent) + obj.SetParent(parent) - objs[object.AddressOf(obj).EncodeToString()] = 0 + mtx.Lock() + objs[object.AddressOf(obj).EncodeToString()] = 0 + mtx.Unlock() - putPrm.SetObject(obj) + var putPrm shard.PutPrm + putPrm.SetObject(obj) - _, err := sh.Put(context.Background(), putPrm) - require.NoError(t, err) - } + _, err := sh.Put(context.Background(), putPrm) + return err + }) + } + return nil + }) } + require.NoError(t, errG.Wait()) res, err := sh.List() require.NoError(t, err) From d35e4c389f560552f3868573e082be4d99dd3ab5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 14:07:11 +0300 Subject: [PATCH 0227/1943] [#321] shard/test: Parallelize TestWriteCacheObjectLoss Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/shutdown_test.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 714811b7e..947bd73fd 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -12,6 +12,7 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) func TestWriteCacheObjectLoss(t *testing.T) { @@ -39,13 +40,17 @@ func TestWriteCacheObjectLoss(t *testing.T) { sh := newCustomShard(t, dir, true, wcOpts, nil) - var putPrm shard.PutPrm - + var errG errgroup.Group for i := range objects { - putPrm.SetObject(objects[i]) - _, err := sh.Put(context.Background(), putPrm) - require.NoError(t, err) + obj := objects[i] + errG.Go(func() error { + var putPrm shard.PutPrm + putPrm.SetObject(obj) + _, err := sh.Put(context.Background(), putPrm) + return err + }) } + require.NoError(t, errG.Wait()) require.NoError(t, sh.Close()) sh = newCustomShard(t, dir, true, wcOpts, nil) From 4578d0061980ba8843ef807c0bc907e803a73fe7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 14:08:10 +0300 Subject: [PATCH 0228/1943] [#321] shard/test: Execute tests in parallel Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/control_test.go | 6 ++++++ pkg/local_object_storage/shard/delete_test.go | 4 ++++ pkg/local_object_storage/shard/gc_test.go | 2 ++ pkg/local_object_storage/shard/get_test.go | 4 ++++ pkg/local_object_storage/shard/head_test.go | 4 ++++ pkg/local_object_storage/shard/inhume_test.go | 4 ++++ pkg/local_object_storage/shard/list_test.go | 2 ++ pkg/local_object_storage/shard/lock_test.go | 2 ++ pkg/local_object_storage/shard/metrics_test.go | 2 ++ pkg/local_object_storage/shard/range_test.go | 3 +++ pkg/local_object_storage/shard/reload_test.go | 2 ++ pkg/local_object_storage/shard/shutdown_test.go | 2 ++ 12 files changed, 37 insertions(+) diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 170052d63..3f0a19cd4 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -43,6 +43,8 @@ type objAddr struct { } func TestShardOpen(t *testing.T) { + t.Parallel() + dir := t.TempDir() metaPath := filepath.Join(dir, "meta") @@ -111,6 +113,8 @@ func TestShardOpen(t *testing.T) { } func TestRefillMetabaseCorrupted(t *testing.T) { + t.Parallel() + dir := t.TempDir() fsTree := fstree.New( @@ -164,6 +168,8 @@ func TestRefillMetabaseCorrupted(t *testing.T) { } func TestRefillMetabase(t *testing.T) { + t.Parallel() + p := t.Name() defer os.RemoveAll(p) diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index 9646e9aa0..441e1c455 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -13,11 +13,15 @@ import ( ) func TestShard_Delete(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardDelete(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardDelete(t, true) }) } diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index b0126fcd7..acc039cd2 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -25,6 +25,8 @@ import ( ) func Test_GCDropsLockedExpiredObject(t *testing.T) { + t.Parallel() + var sh *shard.Shard epoch := &epochState{ diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index ea28c8e32..2db86c48a 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -17,11 +17,15 @@ import ( ) func TestShard_Get(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardGet(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardGet(t, true) }) } diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index 11e7a8b04..7e336ea06 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -15,11 +15,15 @@ import ( ) func TestShard_Head(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardHead(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardHead(t, true) }) } diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 0b4e51701..4151d6218 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -13,11 +13,15 @@ import ( ) func TestShard_Inhume(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardInhume(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardInhume(t, true) }) } diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index e0759f668..bbce28430 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -14,6 +14,8 @@ import ( ) func TestShard_List(t *testing.T) { + t.Parallel() + t.Run("without write cache", func(t *testing.T) { t.Parallel() sh := newShard(t, false) diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index c577ae184..75010179a 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -23,6 +23,8 @@ import ( ) func TestShard_Lock(t *testing.T) { + t.Parallel() + var sh *shard.Shard rootPath := t.TempDir() diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 1578c662b..16f6989c4 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -72,6 +72,8 @@ const physical = "phy" const logical = "logic" func TestCounters(t *testing.T) { + t.Parallel() + dir := t.TempDir() sh, mm := shardWithMetrics(t, dir) diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index c95dbae98..4574ce415 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -22,11 +22,14 @@ import ( ) func TestShard_GetRange(t *testing.T) { + t.Parallel() t.Run("without write cache", func(t *testing.T) { + t.Parallel() testShardGetRange(t, false) }) t.Run("with write cache", func(t *testing.T) { + t.Parallel() testShardGetRange(t, true) }) } diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 9ad05f525..0b964ba2e 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -24,6 +24,8 @@ import ( ) func TestShardReload(t *testing.T) { + t.Parallel() + p := t.Name() defer os.RemoveAll(p) diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 947bd73fd..76b20d659 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -16,6 +16,8 @@ import ( ) func TestWriteCacheObjectLoss(t *testing.T) { + t.Parallel() + const ( smallSize = 1024 objCount = 100 From 945454f60c95ce424cb9eb77fccd2f75ea1d98d6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 16:17:29 +0300 Subject: [PATCH 0229/1943] [#321] engine/test: Execute tests in parallel Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/delete_test.go | 2 ++ pkg/local_object_storage/engine/evacuate_test.go | 8 ++++++++ pkg/local_object_storage/engine/list_test.go | 6 +++++- pkg/local_object_storage/engine/lock_test.go | 6 ++++++ pkg/local_object_storage/engine/remove_copies_test.go | 4 ++++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 53c62981c..bbc27615a 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -18,6 +18,8 @@ import ( ) func TestDeleteBigObject(t *testing.T) { + t.Parallel() + defer os.RemoveAll(t.Name()) cnr := cidtest.ID() diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index bea6d4ff5..bc5b05ef0 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -77,6 +77,8 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng } func TestEvacuateShard(t *testing.T) { + t.Parallel() + const objPerShard = 3 e, ids, objects := newEngineEvacuate(t, 3, objPerShard) @@ -132,6 +134,8 @@ func TestEvacuateShard(t *testing.T) { } func TestEvacuateNetwork(t *testing.T) { + t.Parallel() + var errReplication = errors.New("handler error") acceptOneOf := func(objects []*objectSDK.Object, max int) func(context.Context, oid.Address, *objectSDK.Object) error { @@ -154,6 +158,7 @@ func TestEvacuateNetwork(t *testing.T) { } t.Run("single shard", func(t *testing.T) { + t.Parallel() e, ids, objects := newEngineEvacuate(t, 1, 3) evacuateShardID := ids[0].String() @@ -173,6 +178,7 @@ func TestEvacuateNetwork(t *testing.T) { require.Equal(t, 2, res.Count()) }) t.Run("multiple shards, evacuate one", func(t *testing.T) { + t.Parallel() e, ids, objects := newEngineEvacuate(t, 2, 3) require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) @@ -195,6 +201,7 @@ func TestEvacuateNetwork(t *testing.T) { }) }) t.Run("multiple shards, evacuate many", func(t *testing.T) { + t.Parallel() e, ids, objects := newEngineEvacuate(t, 4, 5) evacuateIDs := ids[0:3] @@ -229,6 +236,7 @@ func TestEvacuateNetwork(t *testing.T) { } func TestEvacuateCancellation(t *testing.T) { + t.Parallel() e, ids, _ := newEngineEvacuate(t, 2, 3) require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 44062be68..5b927cf11 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -29,6 +29,8 @@ func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithTy } func TestListWithCursor(t *testing.T) { + t.Parallel() + tests := []struct { name string shardNum int @@ -60,8 +62,10 @@ func TestListWithCursor(t *testing.T) { batchSize: 100, }, } - for _, tt := range tests { + for i := range tests { + tt := tests[i] t.Run(tt.name, func(t *testing.T) { + t.Parallel() e := testNewEngine(t).setShardsNumOpts(t, tt.shardNum, func(id int) []shard.Option { return []shard.Option{ shard.WithLogger(&logger.Logger{Logger: zap.L()}), diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 4c89b9226..7796913ea 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -31,6 +31,8 @@ func (t tss) IsTombstoneAvailable(ctx context.Context, _ oid.Address, epoch uint } func TestLockUserScenario(t *testing.T) { + t.Parallel() + // Tested user actions: // 1. stores some object // 2. locks the object @@ -146,6 +148,8 @@ func TestLockUserScenario(t *testing.T) { } func TestLockExpiration(t *testing.T) { + t.Parallel() + // Tested scenario: // 1. some object is stored // 2. lock object for it is stored, and the object is locked @@ -222,6 +226,8 @@ func TestLockExpiration(t *testing.T) { } func TestLockForceRemoval(t *testing.T) { + t.Parallel() + // Tested scenario: // 1. some object is stored // 2. lock object for it is stored, and the object is locked diff --git a/pkg/local_object_storage/engine/remove_copies_test.go b/pkg/local_object_storage/engine/remove_copies_test.go index c53e03bbf..8131fcf0d 100644 --- a/pkg/local_object_storage/engine/remove_copies_test.go +++ b/pkg/local_object_storage/engine/remove_copies_test.go @@ -17,6 +17,8 @@ import ( ) func TestRebalance(t *testing.T) { + t.Parallel() + te := newEngineWithErrorThreshold(t, "", 0) const ( @@ -101,6 +103,8 @@ loop: } func TestRebalanceSingleThread(t *testing.T) { + t.Parallel() + te := newEngineWithErrorThreshold(t, "", 0) obj := testutil.GenerateObjectWithCID(cidtest.ID()) From fe4082799a106a720240f685d3f5c5d84b64fabb Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 16:44:02 +0300 Subject: [PATCH 0230/1943] [#321] adm: Create multisig accounts in parallel Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/generate.go | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index e714482dd..8975a6d76 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -21,6 +21,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" "github.com/spf13/viper" + "golang.org/x/sync/errgroup" ) const ( @@ -92,28 +93,30 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er pubs[i] = w.Accounts[0].PrivateKey().PublicKey() } + var errG errgroup.Group + // Create committee account with N/2+1 multi-signature. majCount := smartcontract.GetMajorityHonestNodeCount(size) - for i, w := range wallets { - if err := addMultisigAccount(w, majCount, committeeAccountName, passwords[i], pubs); err != nil { - return nil, fmt.Errorf("can't create committee account: %w", err) - } - } - // Create consensus account with 2*N/3+1 multi-signature. bftCount := smartcontract.GetDefaultHonestNodeCount(size) - for i, w := range wallets { - if err := addMultisigAccount(w, bftCount, consensusAccountName, passwords[i], pubs); err != nil { - return nil, fmt.Errorf("can't create consensus account: %w", err) - } + for i := range wallets { + i := i + errG.Go(func() error { + if err := addMultisigAccount(wallets[i], majCount, committeeAccountName, passwords[i], pubs); err != nil { + return fmt.Errorf("can't create committee account: %w", err) + } + if err := addMultisigAccount(wallets[i], bftCount, consensusAccountName, passwords[i], pubs); err != nil { + return fmt.Errorf("can't create consentus account: %w", err) + } + if err := wallets[i].SavePretty(); err != nil { + return fmt.Errorf("can't save wallet: %w", err) + } + return nil + }) } - - for _, w := range wallets { - if err := w.SavePretty(); err != nil { - return nil, fmt.Errorf("can't save wallet: %w", err) - } + if err := errG.Wait(); err != nil { + return nil, err } - return passwords, nil } From c8c5f14e2e4fcc22291307cffa650e7472cb6002 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 16:45:43 +0300 Subject: [PATCH 0231/1943] [#321] adm/test: Check wallet correctness in parallel Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/generate_test.go | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate_test.go index 39cfc5718..457813df0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strconv" + "sync" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" @@ -71,24 +72,31 @@ func TestGenerateAlphabet(t *testing.T) { buf.WriteString(testContractPassword + "\r") require.NoError(t, generateAlphabetCreds(generateAlphabetCmd, nil)) + var wg sync.WaitGroup for i := uint64(0); i < size; i++ { - p := filepath.Join(walletDir, innerring.GlagoliticLetter(i).String()+".json") - w, err := wallet.NewWalletFromFile(p) - require.NoError(t, err, "wallet doesn't exist") - require.Equal(t, 3, len(w.Accounts), "not all accounts were created") - for _, a := range w.Accounts { - err := a.Decrypt(strconv.FormatUint(i, 10), keys.NEP2ScryptParams()) - require.NoError(t, err, "can't decrypt account") - switch a.Label { - case consensusAccountName: - require.Equal(t, smartcontract.GetDefaultHonestNodeCount(size), len(a.Contract.Parameters)) - case committeeAccountName: - require.Equal(t, smartcontract.GetMajorityHonestNodeCount(size), len(a.Contract.Parameters)) - default: - require.Equal(t, singleAccountName, a.Label) + i := i + go func() { + defer wg.Done() + p := filepath.Join(walletDir, innerring.GlagoliticLetter(i).String()+".json") + w, err := wallet.NewWalletFromFile(p) + require.NoError(t, err, "wallet doesn't exist") + require.Equal(t, 3, len(w.Accounts), "not all accounts were created") + + for _, a := range w.Accounts { + err := a.Decrypt(strconv.FormatUint(i, 10), keys.NEP2ScryptParams()) + require.NoError(t, err, "can't decrypt account") + switch a.Label { + case consensusAccountName: + require.Equal(t, smartcontract.GetDefaultHonestNodeCount(size), len(a.Contract.Parameters)) + case committeeAccountName: + require.Equal(t, smartcontract.GetMajorityHonestNodeCount(size), len(a.Contract.Parameters)) + default: + require.Equal(t, singleAccountName, a.Label) + } } - } + }() } + wg.Wait() t.Run("check contract group wallet", func(t *testing.T) { p := filepath.Join(walletDir, contractWalletFilename) From c62025c8365126537964df0ac536756400063bbd Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 May 2023 17:02:27 +0300 Subject: [PATCH 0232/1943] [#321] metabase/test: execute tests in parallel Signed-off-by: Evgenii Stratonikov --- .../metabase/containers_test.go | 6 +++ .../metabase/counter_test.go | 49 ++++++++++--------- .../metabase/generic_test.go | 2 + .../metabase/list_test.go | 4 ++ .../metabase/lock_test.go | 6 +++ .../metabase/select_test.go | 18 +++++++ .../metabase/storage_id_test.go | 2 + 7 files changed, 65 insertions(+), 22 deletions(-) diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index a0be2c743..c0565b35a 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -15,6 +15,8 @@ import ( ) func TestDB_Containers(t *testing.T) { + t.Parallel() + db := newDB(t) const N = 10 @@ -90,6 +92,8 @@ func TestDB_Containers(t *testing.T) { } func TestDB_ContainersCount(t *testing.T) { + t.Parallel() + db := newDB(t) const R, T, SG, L = 10, 11, 12, 13 // amount of object per type @@ -133,6 +137,8 @@ func TestDB_ContainersCount(t *testing.T) { } func TestDB_ContainerSize(t *testing.T) { + t.Parallel() + db := newDB(t) const ( diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 17a593b6d..507bfcd89 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -2,6 +2,7 @@ package meta_test import ( "context" + "os" "testing" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -16,19 +17,23 @@ import ( const objCount = 10 func TestCounters(t *testing.T) { - db := newDB(t) - - var c meta.ObjectCounters - var err error + t.Parallel() + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(t.Name())) + }) t.Run("defaults", func(t *testing.T) { - c, err = db.ObjectCounters() + t.Parallel() + db := newDB(t) + c, err := db.ObjectCounters() require.NoError(t, err) require.Zero(t, c.Phy()) require.Zero(t, c.Logic()) }) t.Run("put", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := make([]*object.Object, 0, objCount) for i := 0; i < objCount; i++ { oo = append(oo, testutil.GenerateObject()) @@ -39,10 +44,10 @@ func TestCounters(t *testing.T) { for i := 0; i < objCount; i++ { prm.SetObject(oo[i]) - _, err = db.Put(context.Background(), prm) + _, err := db.Put(context.Background(), prm) require.NoError(t, err) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(i+1), c.Phy()) @@ -50,9 +55,9 @@ func TestCounters(t *testing.T) { } }) - require.NoError(t, db.Reset()) - t.Run("delete", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, false) var prm meta.DeletePrm @@ -63,7 +68,7 @@ func TestCounters(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(1), res.AvailableObjectsRemoved()) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(i), c.Phy()) @@ -71,9 +76,9 @@ func TestCounters(t *testing.T) { } }) - require.NoError(t, db.Reset()) - t.Run("inhume", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, false) inhumedObjs := make([]oid.Address, objCount/2) @@ -94,16 +99,16 @@ func TestCounters(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(len(inhumedObjs)), res.AvailableInhumed()) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(objCount), c.Phy()) require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic()) }) - require.NoError(t, db.Reset()) - t.Run("put_split", func(t *testing.T) { + t.Parallel() + db := newDB(t) parObj := testutil.GenerateObject() // put objects and check that parent info @@ -116,16 +121,16 @@ func TestCounters(t *testing.T) { require.NoError(t, putBig(db, o)) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(i+1), c.Phy()) require.Equal(t, uint64(i+1), c.Logic()) } }) - require.NoError(t, db.Reset()) - t.Run("delete_split", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, true) // delete objects that have parent info @@ -141,9 +146,9 @@ func TestCounters(t *testing.T) { } }) - require.NoError(t, db.Reset()) - t.Run("inhume_split", func(t *testing.T) { + t.Parallel() + db := newDB(t) oo := putObjs(t, db, objCount, true) inhumedObjs := make([]oid.Address, objCount/2) @@ -160,10 +165,10 @@ func TestCounters(t *testing.T) { prm.SetTombstoneAddress(oidtest.Address()) prm.SetAddresses(inhumedObjs...) - _, err = db.Inhume(context.Background(), prm) + _, err := db.Inhume(context.Background(), prm) require.NoError(t, err) - c, err = db.ObjectCounters() + c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(objCount), c.Phy()) diff --git a/pkg/local_object_storage/metabase/generic_test.go b/pkg/local_object_storage/metabase/generic_test.go index 227aa9f8d..9d15b6f7a 100644 --- a/pkg/local_object_storage/metabase/generic_test.go +++ b/pkg/local_object_storage/metabase/generic_test.go @@ -10,6 +10,8 @@ import ( ) func TestGeneric(t *testing.T) { + t.Parallel() + defer func() { _ = os.RemoveAll(t.Name()) }() var n int diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 07a0d80f8..4bf3ca827 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -66,6 +66,8 @@ func benchmarkListWithCursor(b *testing.B, db *meta.DB, batchSize int) { } func TestLisObjectsWithCursor(t *testing.T) { + t.Parallel() + db := newDB(t) const ( @@ -159,6 +161,8 @@ func TestLisObjectsWithCursor(t *testing.T) { } func TestAddObjectDuringListingWithCursor(t *testing.T) { + t.Parallel() + db := newDB(t) const total = 5 diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 7b62841dc..d5e063431 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -17,6 +17,8 @@ import ( ) func TestDB_Lock(t *testing.T) { + t.Parallel() + cnr := cidtest.ID() db := newDB(t) @@ -171,6 +173,8 @@ func TestDB_Lock(t *testing.T) { } func TestDB_Lock_Expired(t *testing.T) { + t.Parallel() + es := &epochState{e: 123} db := newDB(t, meta.WithEpochState(es)) @@ -192,6 +196,8 @@ func TestDB_Lock_Expired(t *testing.T) { } func TestDB_IsLocked(t *testing.T) { + t.Parallel() + db := newDB(t) // existing and locked objs diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index dab4c028d..e107085ab 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -20,6 +20,8 @@ import ( ) func TestDB_SelectUserAttributes(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -142,6 +144,8 @@ func TestDB_SelectUserAttributes(t *testing.T) { } func TestDB_SelectRootPhyParent(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -293,6 +297,8 @@ func TestDB_SelectRootPhyParent(t *testing.T) { } func TestDB_SelectInhume(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -325,6 +331,8 @@ func TestDB_SelectInhume(t *testing.T) { } func TestDB_SelectPayloadHash(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -393,6 +401,8 @@ func TestDB_SelectPayloadHash(t *testing.T) { } func TestDB_SelectWithSlowFilters(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -498,6 +508,8 @@ func TestDB_SelectWithSlowFilters(t *testing.T) { } func TestDB_SelectObjectID(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -611,6 +623,8 @@ func TestDB_SelectObjectID(t *testing.T) { } func TestDB_SelectSplitID(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -665,6 +679,8 @@ func TestDB_SelectSplitID(t *testing.T) { } func TestDB_SelectContainerID(t *testing.T) { + t.Parallel() + db := newDB(t) cnr := cidtest.ID() @@ -750,6 +766,8 @@ func BenchmarkSelect(b *testing.B) { } func TestExpiredObjects(t *testing.T) { + t.Parallel() + db := newDB(t, meta.WithEpochState(epochState{currEpoch})) checkExpiredObjects(t, db, func(exp, nonExp *objectSDK.Object) { diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go index 5b27cdc87..b3652a680 100644 --- a/pkg/local_object_storage/metabase/storage_id_test.go +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -12,6 +12,8 @@ import ( ) func TestDB_StorageID(t *testing.T) { + t.Parallel() + db := newDB(t) raw1 := testutil.GenerateObject() From 147ae8728a250a4cce9ad8da19a6f8c76a46a33d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 3 May 2023 15:39:35 +0300 Subject: [PATCH 0233/1943] [#309] go.mod: Update paulmach/orb to v0.9.1 Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + go.mod | 3 ++- go.sum | 18 +++++++++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58fe9c3ed..07e7b043f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Changelog for FrostFS Node ### Removed ### Updated +- `paulmach/orb` to v0.9.1 ### Updating from v0.36.0 ## [v0.36.0] - 2023-04-12 - Furtwängler diff --git a/go.mod b/go.mod index ae0bc88c1..0193937f3 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/nspcc-dev/neo-go v0.100.1 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.4.0 - github.com/paulmach/orb v0.2.2 + github.com/paulmach/orb v0.9.1 github.com/prometheus/client_golang v1.15.0 github.com/prometheus/client_model v0.3.0 github.com/spf13/cast v1.5.0 @@ -94,6 +94,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect github.com/twmb/murmur3 v1.1.5 // indirect github.com/urfave/cli v1.22.5 // indirect + go.mongodb.org/mongo-driver v1.11.4 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect diff --git a/go.sum b/go.sum index 633ac9a18..68526591d 100644 --- a/go.sum +++ b/go.sum @@ -290,6 +290,7 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -337,6 +338,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -410,9 +412,9 @@ github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/panjf2000/ants/v2 v2.4.0 h1:embKPQeNWMRbnrRKURv4TXJwjQRWMEAfqZT6Pe5hZNc= github.com/panjf2000/ants/v2 v2.4.0/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= -github.com/paulmach/orb v0.2.2 h1:PblToKAbU0xHVypex/GdZfibA1CeCfN5s0UjxyWExdo= -github.com/paulmach/orb v0.2.2/go.mod h1:FkcWtplUAIVqAuhAOV2d3rpbnQyliDOjOcLW9dUrfdU= -github.com/paulmach/protoscan v0.2.1-0.20210522164731-4e53c6875432/go.mod h1:2sV+uZ/oQh66m4XJVZm5iqUZ62BN88Ex1E+TTS0nLzI= +github.com/paulmach/orb v0.9.1 h1:NAL1AHClSzudQdZZA1kpjpNhcCEP1SXX+rQviOw0e/g= +github.com/paulmach/orb v0.9.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -489,6 +491,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -500,6 +503,8 @@ github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNG github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/twmb/murmur3 v1.1.5 h1:i9OLS9fkuLzBXjt6dptlAEyk58fJsSTXbRg3SgVyqgk= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -507,6 +512,10 @@ github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -517,6 +526,8 @@ github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBU go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -568,6 +579,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= From 02c02974b3967d6968b207f13c65ba8adea8f702 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 3 May 2023 16:23:58 +0300 Subject: [PATCH 0234/1943] [#309] locode: Parallelize DB generation For v0.4.0 release: Before: ``` Executed in 571.64 secs fish external usr time 283.07 secs 744.00 micros 283.07 secs sys time 8.41 secs 179.00 micros 8.41 secs ``` After: ``` Executed in 54.23 secs fish external usr time 418.65 secs 1.01 millis 418.65 secs sys time 0.61 secs 0.25 millis 0.60 secs ``` Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + pkg/util/locode/db/boltdb/calls.go | 2 +- pkg/util/locode/db/db.go | 130 ++++++++++++++++------------- 3 files changed, 74 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07e7b043f..316518e60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Changelog for FrostFS Node - Reload pprof and metrics on SIGHUP for ir (#125) ### Changed +- `frostfs-cli util locode generate` is now much faster (#309) ### Fixed - Take network settings into account during netmap contract update (#100) - Read config files from dir even if config file not provided via `--config` for node (#238) diff --git a/pkg/util/locode/db/boltdb/calls.go b/pkg/util/locode/db/boltdb/calls.go index 171808af2..6a80def3a 100644 --- a/pkg/util/locode/db/boltdb/calls.go +++ b/pkg/util/locode/db/boltdb/calls.go @@ -103,7 +103,7 @@ func recordFromValue(data []byte) (*locodedb.Record, error) { // Must not be called before successful Open call. // Must not be called in read-only mode: behavior is undefined. func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error { - return db.bolt.Update(func(tx *bbolt.Tx) error { + return db.bolt.Batch(func(tx *bbolt.Tx) error { countryKey, err := countryBucketKey(key.CountryCode()) if err != nil { return err diff --git a/pkg/util/locode/db/db.go b/pkg/util/locode/db/db.go index 2a0f26689..8c71ea794 100644 --- a/pkg/util/locode/db/db.go +++ b/pkg/util/locode/db/db.go @@ -3,8 +3,10 @@ package locodedb import ( "errors" "fmt" + "runtime" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" + "golang.org/x/sync/errgroup" ) // SourceTable is an interface of the UN/LOCODE table. @@ -75,81 +77,93 @@ type NamesDB interface { // FillDatabase generates the FrostFS location database based on the UN/LOCODE table. func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error { - return table.IterateAll(func(tableRecord locode.Record) error { - if tableRecord.LOCODE.LocationCode() == "" { + var errG errgroup.Group + + // Pick some sane default, after this the performance stopped increasing. + errG.SetLimit(runtime.NumCPU() * 4) + _ = table.IterateAll(func(tableRecord locode.Record) error { + errG.Go(func() error { + return processTableRecord(tableRecord, airports, continents, names, db) + }) + return nil + }) + return errG.Wait() +} + +func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error { + if tableRecord.LOCODE.LocationCode() == "" { + return nil + } + + dbKey, err := NewKey(tableRecord.LOCODE) + if err != nil { + return err + } + + dbRecord, err := NewRecord(tableRecord) + if err != nil { + if errors.Is(err, errParseCoordinates) { return nil } - dbKey, err := NewKey(tableRecord.LOCODE) - if err != nil { - return err - } + return err + } - dbRecord, err := NewRecord(tableRecord) + geoPoint := dbRecord.GeoPoint() + countryName := "" + + if geoPoint == nil { + airportRecord, err := airports.Get(tableRecord) if err != nil { - if errors.Is(err, errParseCoordinates) { + if errors.Is(err, ErrAirportNotFound) { return nil } return err } - geoPoint := dbRecord.GeoPoint() - countryName := "" + geoPoint = airportRecord.Point + countryName = airportRecord.CountryName + } - if geoPoint == nil { - airportRecord, err := airports.Get(tableRecord) - if err != nil { - if errors.Is(err, ErrAirportNotFound) { - return nil - } + dbRecord.SetGeoPoint(geoPoint) - return err - } - - geoPoint = airportRecord.Point - countryName = airportRecord.CountryName - } - - dbRecord.SetGeoPoint(geoPoint) - - if countryName == "" { - countryName, err = names.CountryName(dbKey.CountryCode()) - if err != nil { - if errors.Is(err, ErrCountryNotFound) { - return nil - } - - return err - } - } - - dbRecord.SetCountryName(countryName) - - if subDivCode := dbRecord.SubDivCode(); subDivCode != "" { - subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode) - if err != nil { - if errors.Is(err, ErrSubDivNotFound) { - return nil - } - - return err - } - - dbRecord.SetSubDivName(subDivName) - } - - continent, err := continents.PointContinent(geoPoint) + if countryName == "" { + countryName, err = names.CountryName(dbKey.CountryCode()) if err != nil { - return fmt.Errorf("could not calculate continent geo point: %w", err) - } else if continent.Is(ContinentUnknown) { - return nil + if errors.Is(err, ErrCountryNotFound) { + return nil + } + + return err + } + } + + dbRecord.SetCountryName(countryName) + + if subDivCode := dbRecord.SubDivCode(); subDivCode != "" { + subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode) + if err != nil { + if errors.Is(err, ErrSubDivNotFound) { + return nil + } + + return err } - dbRecord.SetContinent(continent) + dbRecord.SetSubDivName(subDivName) + } - return db.Put(*dbKey, *dbRecord) - }) + continent, err := continents.PointContinent(geoPoint) + if err != nil { + return fmt.Errorf("could not calculate continent geo point: %w", err) + } else if continent.Is(ContinentUnknown) { + return nil + } + + dbRecord.SetContinent(continent) + + return db.Put(*dbKey, *dbRecord) } // LocodeRecord returns the record from the FrostFS location database From 6055b183620da920d821053f69b02e7dc55e4215 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 May 2023 13:24:08 +0300 Subject: [PATCH 0235/1943] [#342] morph: Remove unused toStackParameter() Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 59 --------------------------------- pkg/morph/client/client_test.go | 57 ------------------------------- 2 files changed, 116 deletions(-) delete mode 100644 pkg/morph/client/client_test.go diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 284e065fb..c9ff14a66 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -21,7 +21,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/rpcclient/rolemgmt" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" - sc "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" @@ -444,64 +443,6 @@ func (c *Client) roleList(r noderoles.Role) (keys.PublicKeys, error) { return c.rolemgmt.GetDesignatedByRole(r, height) } -// tries to resolve sc.Parameter from the arg. -// -// Wraps any error to frostfsError. -func toStackParameter(value any) (sc.Parameter, error) { - var res = sc.Parameter{ - Value: value, - } - - switch v := value.(type) { - case []byte: - res.Type = sc.ByteArrayType - case int: - res.Type = sc.IntegerType - res.Value = big.NewInt(int64(v)) - case int64: - res.Type = sc.IntegerType - res.Value = big.NewInt(v) - case uint64: - res.Type = sc.IntegerType - res.Value = new(big.Int).SetUint64(v) - case [][]byte: - arr := make([]sc.Parameter, 0, len(v)) - for i := range v { - elem, err := toStackParameter(v[i]) - if err != nil { - return res, err - } - - arr = append(arr, elem) - } - - res.Type = sc.ArrayType - res.Value = arr - case string: - res.Type = sc.StringType - case util.Uint160: - res.Type = sc.ByteArrayType - res.Value = v.BytesBE() - case noderoles.Role: - res.Type = sc.IntegerType - res.Value = big.NewInt(int64(v)) - case keys.PublicKeys: - arr := make([][]byte, 0, len(v)) - for i := range v { - arr = append(arr, v[i].Bytes()) - } - - return toStackParameter(arr) - case bool: - res.Type = sc.BoolType - res.Value = v - default: - return res, wrapFrostFSError(fmt.Errorf("chain/client: unsupported parameter %v", value)) - } - - return res, nil -} - // MagicNumber returns the magic number of the network // to which the underlying RPC node client is connected. func (c *Client) MagicNumber() (uint64, error) { diff --git a/pkg/morph/client/client_test.go b/pkg/morph/client/client_test.go deleted file mode 100644 index a448c2cf4..000000000 --- a/pkg/morph/client/client_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package client - -import ( - "math/big" - "testing" - - sc "github.com/nspcc-dev/neo-go/pkg/smartcontract" - "github.com/stretchr/testify/require" -) - -func TestToStackParameter(t *testing.T) { - items := []struct { - value any - expType sc.ParamType - expVal any - }{ - { - value: []byte{1, 2, 3}, - expType: sc.ByteArrayType, - }, - { - value: int64(100), - expType: sc.IntegerType, - expVal: big.NewInt(100), - }, - { - value: uint64(100), - expType: sc.IntegerType, - expVal: big.NewInt(100), - }, - { - value: "hello world", - expType: sc.StringType, - }, - { - value: false, - expType: sc.BoolType, - }, - { - value: true, - expType: sc.BoolType, - }, - } - - for _, item := range items { - t.Run(item.expType.String()+" to stack parameter", func(t *testing.T) { - res, err := toStackParameter(item.value) - require.NoError(t, err) - require.Equal(t, item.expType, res.Type) - if item.expVal != nil { - require.Equal(t, item.expVal, res.Value) - } else { - require.Equal(t, item.value, res.Value) - } - }) - } -} From a6ee7a3087b162b2c67e9f3e2a8d1580a86b59bf Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 12 May 2023 11:44:21 +0300 Subject: [PATCH 0236/1943] [#324] Add replicator metrics Signed-off-by: Alejandro Lopez --- cmd/frostfs-node/object.go | 1 + pkg/metrics/node.go | 5 +++ pkg/metrics/replicator.go | 59 +++++++++++++++++++++++++++ pkg/services/replicator/metrics.go | 8 ++++ pkg/services/replicator/process.go | 5 +++ pkg/services/replicator/replicator.go | 8 ++++ 6 files changed, 86 insertions(+) create mode 100644 pkg/metrics/replicator.go create mode 100644 pkg/services/replicator/metrics.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 4ff9b8522..1ce330cb5 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -267,6 +267,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa replicator.WithRemoteSender( putsvc.NewRemoteSender(keyStorage, cache), ), + replicator.WithMetrics(c.metricsCollector), ) } diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 0f9c6183d..bf12e610f 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -8,6 +8,7 @@ type NodeMetrics struct { objectServiceMetrics engineMetrics stateMetrics + replicatorMetrics epoch metric[prometheus.Gauge] } @@ -21,6 +22,9 @@ func NewNodeMetrics() *NodeMetrics { state := newStateMetrics() state.register() + replicator := newReplicatorMetrics() + replicator.register() + epoch := newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, @@ -33,6 +37,7 @@ func NewNodeMetrics() *NodeMetrics { objectServiceMetrics: objectService, engineMetrics: engine, stateMetrics: state, + replicatorMetrics: replicator, epoch: epoch, } } diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go new file mode 100644 index 000000000..55f736c66 --- /dev/null +++ b/pkg/metrics/replicator.go @@ -0,0 +1,59 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +const replicatorSubsystem = "replicator" + +type replicatorMetrics struct { + inFlightRequests metric[prometheus.Gauge] + processedObjects metric[prometheus.Counter] + totalReplicatedPayloadSize metric[prometheus.Counter] +} + +func (m replicatorMetrics) IncInFlightRequest() { + m.inFlightRequests.value.Inc() +} + +func (m replicatorMetrics) DecInFlightRequest() { + m.inFlightRequests.value.Dec() +} + +func (m replicatorMetrics) IncProcessedObjects() { + m.processedObjects.value.Inc() +} + +func (m replicatorMetrics) AddPayloadSize(size int64) { + m.totalReplicatedPayloadSize.value.Add(float64(size)) +} + +func newReplicatorMetrics() replicatorMetrics { + return replicatorMetrics{ + inFlightRequests: newReplicatorGauge("in_flight_requests", "Number of in-flight requests"), + processedObjects: newReplicatorCounter("processed_objects", "Number of objects processed since the node startup"), + totalReplicatedPayloadSize: newReplicatorCounter("total_replicated_payload_size", "Total size of payloads replicated"), + } +} + +func (m replicatorMetrics) register() { + mustRegister(m.inFlightRequests) + mustRegister(m.processedObjects) + mustRegister(m.totalReplicatedPayloadSize) +} + +func newReplicatorCounter(name, help string) metric[prometheus.Counter] { + return newCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: replicatorSubsystem, + Name: name, + Help: help, + }) +} + +func newReplicatorGauge(name, help string) metric[prometheus.Gauge] { + return newGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: replicatorSubsystem, + Name: name, + Help: help, + }) +} diff --git a/pkg/services/replicator/metrics.go b/pkg/services/replicator/metrics.go new file mode 100644 index 000000000..3fc062926 --- /dev/null +++ b/pkg/services/replicator/metrics.go @@ -0,0 +1,8 @@ +package replicator + +type MetricsRegister interface { + IncInFlightRequest() + DecInFlightRequest() + IncProcessedObjects() + AddPayloadSize(size int64) +} diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 46e0c9468..0f82ff232 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -20,6 +20,8 @@ type TaskResult interface { // HandleTask executes replication task inside invoking goroutine. // Passes all the nodes that accepted the replication to the TaskResult. func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) { + p.metrics.IncInFlightRequest() + defer p.metrics.DecInFlightRequest() defer func() { p.log.Debug(logs.ReplicatorFinishWork, zap.Uint32("amount of unfinished replicas", task.quantity), @@ -69,6 +71,9 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) task.quantity-- res.SubmitSuccessfulReplication(task.nodes[i]) + + p.metrics.IncProcessedObjects() + p.metrics.AddPayloadSize(int64(task.obj.PayloadSize())) } } } diff --git a/pkg/services/replicator/replicator.go b/pkg/services/replicator/replicator.go index 493982100..bb817cb32 100644 --- a/pkg/services/replicator/replicator.go +++ b/pkg/services/replicator/replicator.go @@ -26,6 +26,8 @@ type cfg struct { remoteSender *putsvc.RemoteSender localStorage *engine.StorageEngine + + metrics MetricsRegister } func defaultCfg() *cfg { @@ -74,3 +76,9 @@ func WithLocalStorage(v *engine.StorageEngine) Option { c.localStorage = v } } + +func WithMetrics(v MetricsRegister) Option { + return func(c *cfg) { + c.metrics = v + } +} From 14c35d776edcbe5c069052ad3d7bb3f4ecd845fc Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 1 Feb 2023 10:21:17 +0300 Subject: [PATCH 0237/1943] [#39] node: Add optional profilers Include settings for block and mutex profilers. They are disabled by default, as in Go runtime itself. Signed-off-by: Pavel Karpy Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config/profiler/config.go | 24 ++++++++++++++++++ .../config/profiler/config_test.go | 6 +++++ cmd/frostfs-node/main.go | 3 +-- cmd/frostfs-node/pprof.go | 25 +++++++++++++++++++ config/example/node.env | 2 ++ config/example/node.json | 4 ++- config/example/node.yaml | 2 ++ docs/storage-node-configuration.md | 20 +++++++++++---- 8 files changed, 78 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-node/config/profiler/config.go b/cmd/frostfs-node/config/profiler/config.go index f891833a6..191694970 100644 --- a/cmd/frostfs-node/config/profiler/config.go +++ b/cmd/frostfs-node/config/profiler/config.go @@ -51,3 +51,27 @@ func Address(c *config.Config) string { return AddressDefault } + +// BlockRates returns the value of "block_rate" config parameter +// from "pprof" section. +func BlockRate(c *config.Config) int { + s := c.Sub(subsection) + + v := int(config.IntSafe(s, "block_rate")) + if v <= 0 { + return 0 + } + return v +} + +// MutexRate returns the value of "mutex_rate" config parameter +// from "pprof" section. +func MutexRate(c *config.Config) int { + s := c.Sub(subsection) + + v := int(config.IntSafe(s, "mutex_rate")) + if v <= 0 { + return 0 + } + return v +} diff --git a/cmd/frostfs-node/config/profiler/config_test.go b/cmd/frostfs-node/config/profiler/config_test.go index bb1b20eb8..355874387 100644 --- a/cmd/frostfs-node/config/profiler/config_test.go +++ b/cmd/frostfs-node/config/profiler/config_test.go @@ -18,6 +18,9 @@ func TestProfilerSection(t *testing.T) { require.Equal(t, profilerconfig.ShutdownTimeoutDefault, to) require.Equal(t, profilerconfig.AddressDefault, addr) require.False(t, profilerconfig.Enabled(configtest.EmptyConfig())) + + require.Zero(t, profilerconfig.BlockRate(configtest.EmptyConfig())) + require.Zero(t, profilerconfig.MutexRate(configtest.EmptyConfig())) }) const path = "../../../../config/example/node" @@ -29,6 +32,9 @@ func TestProfilerSection(t *testing.T) { require.Equal(t, 15*time.Second, to) require.Equal(t, "localhost:6060", addr) require.True(t, profilerconfig.Enabled(c)) + + require.Equal(t, 10_000, profilerconfig.BlockRate(c)) + require.Equal(t, 10_000, profilerconfig.MutexRate(c)) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 4fcc38e08..a2024706b 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -83,9 +83,8 @@ func initApp(ctx context.Context, c *cfg) { c.wg.Done() }() - pprof, _ := pprofComponent(c) metrics, _ := metricsComponent(c) - initAndLog(c, pprof.name, pprof.init) + initAndLog(c, "profiler", initProfilerService) initAndLog(c, metrics.name, metrics.init) initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) }) diff --git a/cmd/frostfs-node/pprof.go b/cmd/frostfs-node/pprof.go index 9be2dd9df..dcd320146 100644 --- a/cmd/frostfs-node/pprof.go +++ b/cmd/frostfs-node/pprof.go @@ -1,10 +1,19 @@ package main import ( + "runtime" + profilerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/profiler" httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" ) +func initProfilerService(c *cfg) { + tuneProfilers(c) + + pprof, _ := pprofComponent(c) + pprof.init(c) +} + func pprofComponent(c *cfg) (*httpComponent, bool) { var updated bool // check if it has been inited before @@ -13,6 +22,7 @@ func pprofComponent(c *cfg) (*httpComponent, bool) { c.dynamicConfiguration.pprof.cfg = c c.dynamicConfiguration.pprof.name = "pprof" c.dynamicConfiguration.pprof.handler = httputil.Handler() + c.dynamicConfiguration.pprof.preReload = tuneProfilers updated = true } @@ -35,3 +45,18 @@ func pprofComponent(c *cfg) (*httpComponent, bool) { return c.dynamicConfiguration.pprof, updated } + +func tuneProfilers(c *cfg) { + // Disabled by default, see documentation for + // runtime.SetBlockProfileRate() and runtime.SetMutexProfileFraction(). + blockRate := 0 + mutexRate := 0 + + if profilerconfig.Enabled(c.appCfg) { + blockRate = profilerconfig.BlockRate(c.appCfg) + mutexRate = profilerconfig.MutexRate(c.appCfg) + } + + runtime.SetBlockProfileRate(blockRate) + runtime.SetMutexProfileFraction(mutexRate) +} diff --git a/config/example/node.env b/config/example/node.env index 77992d995..143bf0388 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -3,6 +3,8 @@ FROSTFS_LOGGER_LEVEL=debug FROSTFS_PPROF_ENABLED=true FROSTFS_PPROF_ADDRESS=localhost:6060 FROSTFS_PPROF_SHUTDOWN_TIMEOUT=15s +FROSTFS_PPROF_BLOCK_RATE=10000 +FROSTFS_PPROF_MUTEX_RATE=10000 FROSTFS_PROMETHEUS_ENABLED=true FROSTFS_PROMETHEUS_ADDRESS=localhost:9090 diff --git a/config/example/node.json b/config/example/node.json index b52eb6d96..04aabdd42 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -5,7 +5,9 @@ "pprof": { "enabled": true, "address": "localhost:6060", - "shutdown_timeout": "15s" + "shutdown_timeout": "15s", + "block_rate": 10000, + "mutex_rate": 10000 }, "prometheus": { "enabled": true, diff --git a/config/example/node.yaml b/config/example/node.yaml index 1669e0e86..bc665a688 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -5,6 +5,8 @@ pprof: enabled: true address: localhost:6060 # endpoint for Node profiling shutdown_timeout: 15s # timeout for profiling HTTP server graceful shutdown + block_rate: 10000 # sampling rate: an average of one blocking event per rate nanoseconds spent blocked is reported; "1" reports every blocking event; "0" disables profiler + mutex_rate: 10000 # sampling rate: on average 1/rate events are reported; "0" disables profiler prometheus: enabled: true diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 366c263a0..2c78cf6b1 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -75,13 +75,23 @@ element. Contains configuration for the `pprof` profiler. -| Parameter | Type | Default value | Description | -|--------------------|------------|---------------|-----------------------------------------| -| `enabled` | `bool` | `false` | Flag to enable the service. | -| `address` | `string` | | Address that service listener binds to. | -| `shutdown_timeout` | `duration` | `30s` | Time to wait for a graceful shutdown. | +| Parameter | Type | Default value | Description | +|--------------------|-----------------------------------|---------------|-----------------------------------------| +| `enabled` | `bool` | `false` | Flag to enable the service. | +| `address` | `string` | | Address that service listener binds to. | +| `shutdown_timeout` | `duration` | `30s` | Time to wait for a graceful shutdown. | +| `debug` | [Debug config](#debug-subsection) | | Optional profiles configuration | +## `debug` subsection + +Contains optional profiles configuration. + +| Parameter | Type | Default value | Description | +|--------------|-------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `block_rate` | `int` | `0` | Controls the block profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetBlockProfileRate. | +| `mutex_rate` | `int` | `0` | Controls the mutex profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetMutexProfileFraction. | + # `prometheus` section Contains configuration for the `prometheus` metrics service. From 61776033c2df233c520fe55783058ee724c936a1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 May 2023 10:58:52 +0300 Subject: [PATCH 0238/1943] [#39] ir: Do not reload services if they are disabled Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-ir/httpcomponent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index d3cda8930..d73160a1d 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -66,7 +66,7 @@ func (c *httpComponent) reload() { enabled := cfg.GetBool(c.enabledKey) address := cfg.GetString(c.addressKey) dur := cfg.GetDuration(c.shutdownTimeoutKey) - if enabled != c.enabled || address != c.address || dur != c.shutdownDur { + if enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) { log.Info(fmt.Sprintf("%s config updated", c.name)) if err := c.shutdown(); err != nil { log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, From f989bc52be968a4ce7f96adb4da0bd550dae33f1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 May 2023 11:04:52 +0300 Subject: [PATCH 0239/1943] [#39] ir: Do not store config keys in `httpComponent` Pprof will have specific options, it seems wrong to have them in a generic struct. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-ir/httpcomponent.go | 33 ++++++++++++++++++--------------- cmd/frostfs-ir/metrics.go | 13 ++----------- cmd/frostfs-ir/pprof.go | 13 ++----------- 3 files changed, 22 insertions(+), 37 deletions(-) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index d73160a1d..38e936471 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -11,22 +11,25 @@ import ( ) type httpComponent struct { - srv *httputil.Server - address string - addressKey string - name string - handler http.Handler - shutdownDur time.Duration - shutdownTimeoutKey string - enabled bool - enabledKey string + srv *httputil.Server + address string + name string + handler http.Handler + shutdownDur time.Duration + enabled bool } +const ( + enabledKeyPostfix = ".enabled" + addressKeyPostfix = ".address" + shutdownTimeoutKeyPostfix = ".shutdown_timeout" +) + func (c *httpComponent) init() { log.Info(fmt.Sprintf("init %s", c.name)) - c.enabled = cfg.GetBool(c.enabledKey) - c.address = cfg.GetString(c.addressKey) - c.shutdownDur = cfg.GetDuration(c.shutdownTimeoutKey) + c.enabled = cfg.GetBool(c.name + enabledKeyPostfix) + c.address = cfg.GetString(c.name + addressKeyPostfix) + c.shutdownDur = cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) if c.enabled { c.srv = httputil.New( @@ -63,9 +66,9 @@ func (c *httpComponent) shutdown() error { func (c *httpComponent) reload() { log.Info(fmt.Sprintf("reload %s", c.name)) - enabled := cfg.GetBool(c.enabledKey) - address := cfg.GetString(c.addressKey) - dur := cfg.GetDuration(c.shutdownTimeoutKey) + enabled := cfg.GetBool(c.name + enabledKeyPostfix) + address := cfg.GetString(c.name + addressKeyPostfix) + dur := cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) if enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) { log.Info(fmt.Sprintf("%s config updated", c.name)) if err := c.shutdown(); err != nil { diff --git a/cmd/frostfs-ir/metrics.go b/cmd/frostfs-ir/metrics.go index debeb7d60..39b432c74 100644 --- a/cmd/frostfs-ir/metrics.go +++ b/cmd/frostfs-ir/metrics.go @@ -4,18 +4,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) -const ( - prometheusEnabledKey = "prometheus.enabled" - prometheusAddressKey = "prometheus.address" - prometheusShutdownTimeoutKey = "prometheus.shutdown_timeout" -) - func newMetricsComponent() *httpComponent { return &httpComponent{ - name: "prometheus", - enabledKey: prometheusEnabledKey, - addressKey: prometheusAddressKey, - shutdownTimeoutKey: prometheusShutdownTimeoutKey, - handler: metrics.Handler(), + name: "prometheus", + handler: metrics.Handler(), } } diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go index 0d48c11de..8228a0f53 100644 --- a/cmd/frostfs-ir/pprof.go +++ b/cmd/frostfs-ir/pprof.go @@ -4,18 +4,9 @@ import ( httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" ) -const ( - pprofEnabledKey = "pprof.enabled" - pprofAddressKey = "pprof.address" - pprofShutdownTimeoutKey = "pprof.shutdown_timeout" -) - func newPprofComponent() *httpComponent { return &httpComponent{ - name: "pprof", - enabledKey: pprofEnabledKey, - addressKey: pprofAddressKey, - shutdownTimeoutKey: pprofShutdownTimeoutKey, - handler: httputil.Handler(), + name: "pprof", + handler: httputil.Handler(), } } From f604d6bbdce327a3560020864b9f37857d0ebbf0 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 19 Apr 2023 20:02:02 +0300 Subject: [PATCH 0240/1943] [#39] ir: Add optional profilers Includes `block` and `mutex` profiles configuration. Signed-off-by: Pavel Karpy Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-ir/httpcomponent.go | 10 ++++-- cmd/frostfs-ir/main.go | 2 +- cmd/frostfs-ir/pprof.go | 64 ++++++++++++++++++++++++++++++--- config/example/ir.env | 2 ++ config/example/ir.yaml | 2 ++ 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index 38e936471..8f5e3753b 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -64,12 +64,16 @@ func (c *httpComponent) shutdown() error { return nil } -func (c *httpComponent) reload() { - log.Info(fmt.Sprintf("reload %s", c.name)) +func (c *httpComponent) needReload() bool { enabled := cfg.GetBool(c.name + enabledKeyPostfix) address := cfg.GetString(c.name + addressKeyPostfix) dur := cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) - if enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) { + return enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) +} + +func (c *httpComponent) reload() { + log.Info(fmt.Sprintf("reload %s", c.name)) + if c.needReload() { log.Info(fmt.Sprintf("%s config updated", c.name)) if err := c.shutdown(); err != nil { log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 84dff9101..1718a46ab 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -30,7 +30,7 @@ var ( intErr = make(chan error) // internal inner ring errors logPrm = new(logger.Prm) innerRing *innerring.Server - pprofCmp *httpComponent + pprofCmp *pprofComponent metricsCmp *httpComponent log *logger.Logger cfg *viper.Viper diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go index 8228a0f53..d67c463fc 100644 --- a/cmd/frostfs-ir/pprof.go +++ b/cmd/frostfs-ir/pprof.go @@ -1,12 +1,68 @@ package main import ( + "fmt" + "runtime" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" + "go.uber.org/zap" ) -func newPprofComponent() *httpComponent { - return &httpComponent{ - name: "pprof", - handler: httputil.Handler(), +type pprofComponent struct { + httpComponent + blockRate int + mutexRate int +} + +const ( + pprofBlockRateKey = "pprof.block_rate" + pprofMutexRateKey = "pprof.mutex_rate" +) + +func newPprofComponent() *pprofComponent { + return &pprofComponent{ + httpComponent: httpComponent{ + name: "pprof", + handler: httputil.Handler(), + }, + } +} + +func (c *pprofComponent) init() { + c.httpComponent.init() + + if c.enabled { + c.blockRate = cfg.GetInt(pprofBlockRateKey) + c.mutexRate = cfg.GetInt(pprofMutexRateKey) + runtime.SetBlockProfileRate(c.blockRate) + runtime.SetMutexProfileFraction(c.mutexRate) + } else { + c.blockRate = 0 + c.mutexRate = 0 + runtime.SetBlockProfileRate(0) + runtime.SetMutexProfileFraction(0) + } +} + +func (c *pprofComponent) needReload() bool { + blockRate := cfg.GetInt(pprofBlockRateKey) + mutexRate := cfg.GetInt(pprofMutexRateKey) + return c.httpComponent.needReload() || + c.enabled && (c.blockRate != blockRate || c.mutexRate != mutexRate) +} + +func (c *pprofComponent) reload() { + log.Info(fmt.Sprintf("reload %s", c.name)) + if c.needReload() { + log.Info(fmt.Sprintf("%s config updated", c.name)) + if err := c.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error())) + return + } + + c.init() + c.start() } } diff --git a/config/example/ir.env b/config/example/ir.env index e3de23ac1..7b8f8a89d 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -74,6 +74,8 @@ FROSTFS_IR_CONTRACTS_ALPHABET_ZHIVETE=f584699bc2ff457d339fb09f16217042c1a42101 FROSTFS_IR_PPROF_ENABLED=true FROSTFS_IR_PPROF_ADDRESS=localhost:6060 FROSTFS_IR_PPROF_SHUTDOWN_TIMEOUT=30s +FROSTFS_IR_PPROF_BLOCK_RATE=10000 +FROSTFS_IR_PPROF_MUTEX_RATE=10000 FROSTFS_IR_PROMETHEUS_ENABLED=true FROSTFS_IR_PROMETHEUS_ADDRESS=localhost:9090 diff --git a/config/example/ir.yaml b/config/example/ir.yaml index bd56ec74b..1130a840b 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -113,6 +113,8 @@ pprof: enabled: true address: localhost:6060 # Endpoint for application pprof profiling; disabled by default shutdown_timeout: 30s # Timeout for profiling HTTP server graceful shutdown + block_rate: 10000 # sampling rate: an average of one blocking event per rate nanoseconds spent blocked is reported; "1" reports every blocking event; "0" disables profiler + mutex_rate: 10000 # sampling rate: on average 1/rate events are reported; "0" disables profiler prometheus: enabled: true From 429a87e83bf98d4e3a3df8a4977e03856ed30450 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 May 2023 11:38:43 +0300 Subject: [PATCH 0241/1943] [#39] ir: Use defer for wg.Done() Not important, but `exitOnErr` can alter control flow, let's be explicit. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-ir/httpcomponent.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index 8f5e3753b..3a6d77d84 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -50,8 +50,8 @@ func (c *httpComponent) start() { log.Info(fmt.Sprintf("start %s", c.name)) wg.Add(1) go func() { + defer wg.Done() exitErr(c.srv.Serve()) - wg.Done() }() } } From d4d921dcaf389303844e1e19413e969539c9cadd Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 15 May 2023 11:50:20 +0300 Subject: [PATCH 0242/1943] [#346] adm: fix race in wallet generation Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/generate.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index 8975a6d76..dd327a4b1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -101,11 +101,13 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er bftCount := smartcontract.GetDefaultHonestNodeCount(size) for i := range wallets { i := i + ps := make(keys.PublicKeys, len(pubs)) + copy(ps, pubs) errG.Go(func() error { - if err := addMultisigAccount(wallets[i], majCount, committeeAccountName, passwords[i], pubs); err != nil { + if err := addMultisigAccount(wallets[i], majCount, committeeAccountName, passwords[i], ps); err != nil { return fmt.Errorf("can't create committee account: %w", err) } - if err := addMultisigAccount(wallets[i], bftCount, consensusAccountName, passwords[i], pubs); err != nil { + if err := addMultisigAccount(wallets[i], bftCount, consensusAccountName, passwords[i], ps); err != nil { return fmt.Errorf("can't create consentus account: %w", err) } if err := wallets[i].SavePretty(); err != nil { From 079b28fa0f51fe918c7857bdbef4ba9fb5d91cc4 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Mon, 15 May 2023 14:46:17 +0300 Subject: [PATCH 0243/1943] [#341] Register candidates in separate transactions Signed-off-by: Alejandro Lopez --- .../modules/morph/initialize_register.go | 72 ++++++++++++------- .../internal/modules/morph/initialize_test.go | 6 ++ 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index b1542cc92..469b269de 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -19,33 +19,24 @@ import ( ) // initialAlphabetNEOAmount represents the total amount of GAS distributed between alphabet nodes. -const initialAlphabetNEOAmount = native.NEOTotalSupply - -func (c *initializeContext) registerCandidates() error { - neoHash := neo.Hash - - cc, err := unwrap.Array(c.ReadOnlyInvoker.Call(neoHash, "getCandidates")) - if err != nil { - return fmt.Errorf("`getCandidates`: %w", err) - } - - if len(cc) > 0 { - c.Command.Println("Candidates are already registered.") - return nil - } +const ( + initialAlphabetNEOAmount = native.NEOTotalSupply + registerBatchSize = transaction.MaxAttributes - 1 +) +func (c *initializeContext) registerCandidateRange(start, end int) error { regPrice, err := c.getCandidateRegisterPrice() if err != nil { return fmt.Errorf("can't fetch registration price: %w", err) } w := io.NewBufBinWriter() - emit.AppCall(w.BinWriter, neoHash, "setRegisterPrice", callflag.States, 1) - for _, acc := range c.Accounts { - emit.AppCall(w.BinWriter, neoHash, "registerCandidate", callflag.States, acc.PrivateKey().PublicKey().Bytes()) + emit.AppCall(w.BinWriter, neo.Hash, "setRegisterPrice", callflag.States, 1) + for _, acc := range c.Accounts[start:end] { + emit.AppCall(w.BinWriter, neo.Hash, "registerCandidate", callflag.States, acc.PrivateKey().PublicKey().Bytes()) emit.Opcodes(w.BinWriter, opcode.ASSERT) } - emit.AppCall(w.BinWriter, neoHash, "setRegisterPrice", callflag.States, regPrice) + emit.AppCall(w.BinWriter, neo.Hash, "setRegisterPrice", callflag.States, regPrice) if w.Err != nil { panic(fmt.Sprintf("BUG: %v", w.Err)) } @@ -54,14 +45,14 @@ func (c *initializeContext) registerCandidates() error { Signer: c.getSigner(false, c.CommitteeAcc), Account: c.CommitteeAcc, }} - for i := range c.Accounts { + for _, acc := range c.Accounts[start:end] { signers = append(signers, rpcclient.SignerAccount{ Signer: transaction.Signer{ - Account: c.Accounts[i].Contract.ScriptHash(), + Account: acc.Contract.ScriptHash(), Scopes: transaction.CustomContracts, - AllowedContracts: []util.Uint160{neoHash}, + AllowedContracts: []util.Uint160{neo.Hash}, }, - Account: c.Accounts[i], + Account: acc, }) } @@ -74,8 +65,8 @@ func (c *initializeContext) registerCandidates() error { } network := c.CommitteeAct.GetNetwork() - for i := range c.Accounts { - if err := c.Accounts[i].SignTx(network, tx); err != nil { + for _, acc := range c.Accounts[start:end] { + if err := acc.SignTx(network, tx); err != nil { return fmt.Errorf("can't sign a transaction: %w", err) } } @@ -83,6 +74,39 @@ func (c *initializeContext) registerCandidates() error { return c.sendTx(tx, c.Command, true) } +func (c *initializeContext) registerCandidates() error { + cc, err := unwrap.Array(c.ReadOnlyInvoker.Call(neo.Hash, "getCandidates")) + if err != nil { + return fmt.Errorf("`getCandidates`: %w", err) + } + + need := len(c.Accounts) + have := len(cc) + + if need == have { + c.Command.Println("Candidates are already registered.") + return nil + } + + // Register candidates in batches in order to overcome the signers amount limit. + // See: https://github.com/nspcc-dev/neo-go/blob/master/pkg/core/transaction/transaction.go#L27 + for i := 0; i < need; i += registerBatchSize { + start, end := i, i+registerBatchSize + if end > need { + end = need + } + // This check is sound because transactions are accepted/rejected atomically. + if have >= end { + continue + } + if err := c.registerCandidateRange(start, end); err != nil { + return fmt.Errorf("registering candidates %d..%d: %q", start, end-1, err) + } + } + + return nil +} + func (c *initializeContext) transferNEOToAlphabetContracts() error { neoHash := neo.Hash diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index fb2dc3e3f..e2e5aa0ad 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -37,6 +37,12 @@ func TestInitialize(t *testing.T) { t.Run("7 nodes", func(t *testing.T) { testInitialize(t, 7) }) + t.Run("16 nodes", func(t *testing.T) { + testInitialize(t, 16) + }) + t.Run("22 nodes", func(t *testing.T) { + testInitialize(t, 22) + }) } func testInitialize(t *testing.T, committeeSize int) { From 8da6530f412d73f5ff3641782e889f47052fb54d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 15 May 2023 15:24:29 +0300 Subject: [PATCH 0244/1943] [#351] cli: Support copies number parameter in `object put` Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 1 + cmd/frostfs-cli/internal/client/client.go | 9 +++++++++ cmd/frostfs-cli/modules/object/put.go | 15 +++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 316518e60..e87a2ca0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Changelog for FrostFS Node - Support impersonate bearer token (#229) - Change log level on SIGHUP for ir (#125) - Reload pprof and metrics on SIGHUP for ir (#125) +- Support copies number parameter in `frostfs-cli object put` (#351) ### Changed - `frostfs-cli util locode generate` is now much faster (#309) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index cbf19eb4b..875ccf904 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -329,6 +329,8 @@ func CreateSession(prm CreateSessionPrm) (res CreateSessionRes, err error) { type PutObjectPrm struct { commonObjectPrm + copyNum []uint32 + hdr *object.Object rdr io.Reader @@ -352,6 +354,12 @@ func (x *PutObjectPrm) SetHeaderCallback(f func(*object.Object)) { x.headerCallback = f } +// SetCopiesNumberByVectors sets ordered list of minimal required object copies numbers +// per placement vector. +func (x *PutObjectPrm) SetCopiesNumberByVectors(copiesNumbers []uint32) { + x.copyNum = copiesNumbers +} + // PutObjectRes groups the resulting values of PutObject operation. type PutObjectRes struct { id oid.ID @@ -381,6 +389,7 @@ func PutObject(prm PutObjectPrm) (*PutObjectRes, error) { } putPrm.WithXHeaders(prm.xHeaders...) + putPrm.SetCopiesNumberByVectors(prm.copyNum) wrt, err := prm.cli.ObjectPutInit(context.Background(), putPrm) if err != nil { diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index fe8e9dda9..6fd91ca4b 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -25,6 +25,7 @@ import ( const ( noProgressFlag = "no-progress" notificationFlag = "notify" + copiesNumberFlag = "copies-number" ) var putExpiredOn uint64 @@ -56,6 +57,8 @@ func initObjectPutCmd() { flags.String(notificationFlag, "", "Object notification in the form of *epoch*:*topic*; '-' topic means using default") flags.Bool(binaryFlag, false, "Deserialize object structure from given file.") + + flags.String(copiesNumberFlag, "", "Number of copies of the object to store within the RPC call") } func putObject(cmd *cobra.Command, _ []string) { @@ -116,6 +119,18 @@ func putObject(cmd *cobra.Command, _ []string) { } } + copyNum, err := cmd.Flags().GetString(copiesNumberFlag) + commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) + if len(copyNum) > 0 { + var cn []uint32 + for _, num := range strings.Split(copyNum, ",") { + val, err := strconv.ParseUint(num, 10, 32) + commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) + cn = append(cn, uint32(val)) + } + prm.SetCopiesNumberByVectors(cn) + } + res, err := internalclient.PutObject(prm) if p != nil { p.Finish() From ab07bad33de15a1fadc9ffad15ca8989e8fc5a63 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 15 May 2023 11:43:44 +0300 Subject: [PATCH 0245/1943] [#332] gc: Add complex object unit test Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc_test.go | 195 +++++++++++++++++----- 1 file changed, 150 insertions(+), 45 deletions(-) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index acc039cd2..0a494fd52 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -2,6 +2,8 @@ package shard_test import ( "context" + "encoding/binary" + "errors" "path/filepath" "testing" "time" @@ -19,62 +21,20 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" "go.uber.org/zap" ) -func Test_GCDropsLockedExpiredObject(t *testing.T) { +func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { t.Parallel() - var sh *shard.Shard - epoch := &epochState{ Value: 100, } - rootPath := t.TempDir() - opts := []shard.Option{ - shard.WithID(shard.NewIDFromBytes([]byte{})), - shard.WithLogger(&logger.Logger{Logger: zap.NewNop()}), - shard.WithBlobStorOptions( - blobstor.WithStorages([]blobstor.SubStorage{ - { - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), - blobovniczatree.WithBlobovniczaShallowDepth(2), - blobovniczatree.WithBlobovniczaShallowWidth(2)), - Policy: func(_ *objectSDK.Object, data []byte) bool { - return len(data) <= 1<<20 - }, - }, - { - Storage: fstree.New( - fstree.WithPath(filepath.Join(rootPath, "blob"))), - }, - }), - ), - shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(rootPath, "meta")), - meta.WithEpochState(epoch), - ), - shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { - sh.HandleDeletedLocks(addresses) - }), - shard.WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { - sh.HandleExpiredLocks(ctx, epoch, a) - }), - shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { - pool, err := ants.NewPool(sz) - require.NoError(t, err) - - return pool - }), - } - - sh = shard.New(opts...) - require.NoError(t, sh.Open()) - require.NoError(t, sh.Init(context.Background())) + sh := createAndInitGCTestShard(t, epoch) t.Cleanup(func() { releaseShard(sh, t) @@ -122,3 +82,148 @@ func Test_GCDropsLockedExpiredObject(t *testing.T) { return shard.IsErrNotFound(err) }, 3*time.Second, 1*time.Second, "expired object must be deleted") } + +func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { + t.Parallel() + + epoch := &epochState{ + Value: 100, + } + + cnr := cidtest.ID() + parentID := oidtest.ID() + splitID := objectSDK.NewSplitID() + + var objExpirationAttr objectSDK.Attribute + objExpirationAttr.SetKey(objectV2.SysAttributeExpEpoch) + objExpirationAttr.SetValue("101") + + var lockExpirationAttr objectSDK.Attribute + lockExpirationAttr.SetKey(objectV2.SysAttributeExpEpoch) + lockExpirationAttr.SetValue("103") + + parent := testutil.GenerateObjectWithCID(cnr) + parent.SetID(parentID) + parent.SetPayload(nil) + parent.SetAttributes(objExpirationAttr) + + const childCount = 10 + children := make([]*objectSDK.Object, childCount) + childIDs := make([]oid.ID, childCount) + for i := range children { + children[i] = testutil.GenerateObjectWithCID(cnr) + if i != 0 { + children[i].SetPreviousID(childIDs[i-1]) + } + if i == len(children)-1 { + children[i].SetParent(parent) + } + children[i].SetSplitID(splitID) + children[i].SetPayload([]byte{byte(i), byte(i + 1), byte(i + 2)}) + childIDs[i], _ = children[i].ID() + } + + link := testutil.GenerateObjectWithCID(cnr) + link.SetParent(parent) + link.SetParentID(parentID) + link.SetSplitID(splitID) + link.SetChildren(childIDs...) + + linkID, _ := link.ID() + + sh := createAndInitGCTestShard(t, epoch) + + t.Cleanup(func() { + releaseShard(sh, t) + }) + + lock := testutil.GenerateObjectWithCID(cnr) + lock.SetType(objectSDK.TypeLock) + lock.SetAttributes(lockExpirationAttr) + lockID, _ := lock.ID() + + var putPrm shard.PutPrm + + for _, child := range children { + putPrm.SetObject(child) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + } + + putPrm.SetObject(link) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + err = sh.Lock(context.Background(), cnr, lockID, append(childIDs, parentID, linkID)) + require.NoError(t, err) + + putPrm.SetObject(lock) + _, err = sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + var getPrm shard.GetPrm + getPrm.SetAddress(objectCore.AddressOf(parent)) + + _, err = sh.Get(context.Background(), getPrm) + var splitInfoError *objectSDK.SplitInfoError + require.True(t, errors.As(err, &splitInfoError), "split info must be provided") + + epoch.Value = 105 + sh.NotificationChannel() <- shard.EventNewEpoch(epoch.Value) + + require.Eventually(t, func() bool { + _, err = sh.Get(context.Background(), getPrm) + return shard.IsErrNotFound(err) + }, 3*time.Second, 1*time.Second, "expired complex object must be deleted on epoch after lock expires") +} + +func createAndInitGCTestShard(t *testing.T, epoch *epochState) *shard.Shard { + var sh *shard.Shard + + rootPath := t.TempDir() + opts := []shard.Option{ + shard.WithID(shard.NewIDFromBytes(binary.AppendVarint([]byte{}, int64(48)))), + shard.WithLogger(&logger.Logger{Logger: zap.NewNop()}), + shard.WithBlobStorOptions( + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(2), + blobovniczatree.WithBlobovniczaShallowWidth(2)), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return len(data) <= 1<<20 + }, + }, + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(rootPath, "blob"))), + }, + }), + ), + shard.WithMetaBaseOptions( + meta.WithPath(filepath.Join(rootPath, "meta")), + meta.WithEpochState(epoch), + ), + shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(addresses) + }), + shard.WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { + sh.HandleExpiredLocks(ctx, epoch, a) + }), + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) + + return pool + }), + shard.WithGCRemoverSleepInterval(1 * time.Millisecond), + } + + sh = shard.New(opts...) + + require.NoError(t, sh.Open()) + require.NoError(t, sh.Init(context.Background())) + + return sh +} From 869fcbf59166732d70517ec731fe8ed3c9c9ff5c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 15 May 2023 12:06:38 +0300 Subject: [PATCH 0246/1943] [#332] gc: Fix expired complex object deletion Signed-off-by: Dmitrii Stepanov --- CHANGELOG.md | 1 + internal/logs/logs.go | 1 + pkg/local_object_storage/metabase/children.go | 57 +++++++++++++++++ pkg/local_object_storage/shard/gc.go | 20 ++++++ pkg/local_object_storage/shard/gc_test.go | 64 +------------------ pkg/local_object_storage/shard/range_test.go | 3 +- pkg/local_object_storage/shard/shard_test.go | 27 ++++++-- .../shard/shutdown_test.go | 4 +- 8 files changed, 108 insertions(+), 69 deletions(-) create mode 100644 pkg/local_object_storage/metabase/children.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e87a2ca0c..6a4c48097 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Changelog for FrostFS Node - Notary requests parsing according to `neo-go`'s updates (#268) - Tree service panic in its internal client cache (#322) - Iterate over endpoints when create ws client in morph's constructor (#304) +- Delete complex objects with GC (#332) ### Removed ### Updated diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 936042d2d..5cd1b8dba 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -483,4 +483,5 @@ const ( ShardGCCollectingExpiredLocksCompleted = "collecting expired locks completed" ShardGCRemoveGarbageStarted = "garbage remove started" ShardGCRemoveGarbageCompleted = "garbage remove completed" + ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" ) diff --git a/pkg/local_object_storage/metabase/children.go b/pkg/local_object_storage/metabase/children.go new file mode 100644 index 000000000..f0591b43c --- /dev/null +++ b/pkg/local_object_storage/metabase/children.go @@ -0,0 +1,57 @@ +package meta + +import ( + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.etcd.io/bbolt" +) + +// GetChildren returns parent -> children map. +// If an object has no children, then map will contain addr -> empty slice value. +func (db *DB) GetChildren(addresses []oid.Address) (map[oid.Address][]oid.Address, error) { + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return nil, ErrDegradedMode + } + + result := make(map[oid.Address][]oid.Address, len(addresses)) + + buffer := make([]byte, bucketKeySize) + err := db.boltDB.View(func(tx *bbolt.Tx) error { + for _, addr := range addresses { + if _, found := result[addr]; found { + continue + } + + result[addr] = []oid.Address{} + bkt := tx.Bucket(parentBucketName(addr.Container(), buffer)) + if bkt == nil { + continue + } + + binObjIDs, err := decodeList(bkt.Get(objectKey(addr.Object(), buffer))) + if err != nil { + return err + } + + for _, binObjID := range binObjIDs { + var id oid.ID + if err = id.Decode(binObjID); err != nil { + return err + } + var resultAddress oid.Address + resultAddress.SetContainer(addr.Container()) + resultAddress.SetObject(id) + result[addr] = append(result[addr], resultAddress) + } + } + return nil + }) + + if err != nil { + return nil, err + } + + return result, nil +} diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 82876e675..f4f7a21e2 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -320,6 +320,12 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } + expired, err := s.getExpiredWithLinked(expired) + if err != nil { + s.log.Warn(logs.ShardGCFailedToGetExpiredWithLinked, zap.Error(err)) + return + } + var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(expired...) @@ -345,6 +351,20 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) } } +func (s *Shard) getExpiredWithLinked(source []oid.Address) ([]oid.Address, error) { + result := make([]oid.Address, 0, len(source)) + parentToChildren, err := s.metaBase.GetChildren(source) + if err != nil { + return nil, err + } + for parent, children := range parentToChildren { + result = append(result, parent) + result = append(result, children...) + } + + return result, nil +} + func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { epoch := e.(newEpoch).epoch log := s.log.With(zap.Uint64("epoch", epoch)) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 0a494fd52..263a0ea4d 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -2,29 +2,20 @@ package shard_test import ( "context" - "encoding/binary" "errors" - "path/filepath" "testing" "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { @@ -34,7 +25,7 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { Value: 100, } - sh := createAndInitGCTestShard(t, epoch) + sh := newCustomShard(t, t.TempDir(), false, nil, nil, []meta.Option{meta.WithEpochState(epoch)}) t.Cleanup(func() { releaseShard(sh, t) @@ -131,7 +122,7 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { linkID, _ := link.ID() - sh := createAndInitGCTestShard(t, epoch) + sh := newCustomShard(t, t.TempDir(), false, nil, nil, []meta.Option{meta.WithEpochState(epoch)}) t.Cleanup(func() { releaseShard(sh, t) @@ -176,54 +167,3 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { return shard.IsErrNotFound(err) }, 3*time.Second, 1*time.Second, "expired complex object must be deleted on epoch after lock expires") } - -func createAndInitGCTestShard(t *testing.T, epoch *epochState) *shard.Shard { - var sh *shard.Shard - - rootPath := t.TempDir() - opts := []shard.Option{ - shard.WithID(shard.NewIDFromBytes(binary.AppendVarint([]byte{}, int64(48)))), - shard.WithLogger(&logger.Logger{Logger: zap.NewNop()}), - shard.WithBlobStorOptions( - blobstor.WithStorages([]blobstor.SubStorage{ - { - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), - blobovniczatree.WithBlobovniczaShallowDepth(2), - blobovniczatree.WithBlobovniczaShallowWidth(2)), - Policy: func(_ *objectSDK.Object, data []byte) bool { - return len(data) <= 1<<20 - }, - }, - { - Storage: fstree.New( - fstree.WithPath(filepath.Join(rootPath, "blob"))), - }, - }), - ), - shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(rootPath, "meta")), - meta.WithEpochState(epoch), - ), - shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { - sh.HandleDeletedLocks(addresses) - }), - shard.WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { - sh.HandleExpiredLocks(ctx, epoch, a) - }), - shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { - pool, err := ants.NewPool(sz) - require.NoError(t, err) - - return pool - }), - shard.WithGCRemoverSleepInterval(1 * time.Millisecond), - } - - sh = shard.New(opts...) - - require.NoError(t, sh.Open()) - require.NoError(t, sh.Init(context.Background())) - - return sh -} diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 4574ce415..9ef2106b0 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -87,7 +87,8 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { Storage: fstree.New( fstree.WithPath(filepath.Join(t.TempDir(), "blob"))), }, - })}) + })}, + nil) defer releaseShard(sh, t) for _, tc := range testCases { diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index fea342766..7b2fdb5d1 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -4,6 +4,7 @@ import ( "context" "path/filepath" "testing" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" @@ -12,8 +13,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -29,11 +33,13 @@ func (s epochState) CurrentEpoch() uint64 { func newShard(t testing.TB, enableWriteCache bool) *shard.Shard { return newCustomShard(t, t.TempDir(), enableWriteCache, + nil, nil, nil) } -func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts []writecache.Option, bsOpts []blobstor.Option) *shard.Shard { +func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts []writecache.Option, bsOpts []blobstor.Option, metaOptions []meta.Option) *shard.Shard { + var sh *shard.Shard if enableWriteCache { rootPath = filepath.Join(rootPath, "wc") } else { @@ -67,8 +73,9 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts shard.WithLogger(&logger.Logger{Logger: zap.L()}), shard.WithBlobStorOptions(bsOpts...), shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(rootPath, "meta")), - meta.WithEpochState(epochState{}), + append([]meta.Option{ + meta.WithPath(filepath.Join(rootPath, "meta")), meta.WithEpochState(epochState{})}, + metaOptions...)..., ), shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))), shard.WithWriteCache(enableWriteCache), @@ -77,9 +84,21 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts []writecache.Option{writecache.WithPath(filepath.Join(rootPath, "wcache"))}, wcOpts...)..., ), + shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(addresses) + }), + shard.WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { + sh.HandleExpiredLocks(ctx, epoch, a) + }), + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) + return pool + }), + shard.WithGCRemoverSleepInterval(1 * time.Millisecond), } - sh := shard.New(opts...) + sh = shard.New(opts...) require.NoError(t, sh.Open()) require.NoError(t, sh.Init(context.Background())) diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 76b20d659..5fe9fd7e9 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -40,7 +40,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { writecache.WithSmallObjectSize(smallSize), writecache.WithMaxObjectSize(smallSize * 2)} - sh := newCustomShard(t, dir, true, wcOpts, nil) + sh := newCustomShard(t, dir, true, wcOpts, nil, nil) var errG errgroup.Group for i := range objects { @@ -55,7 +55,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { require.NoError(t, errG.Wait()) require.NoError(t, sh.Close()) - sh = newCustomShard(t, dir, true, wcOpts, nil) + sh = newCustomShard(t, dir, true, wcOpts, nil, nil) defer releaseShard(sh, t) var getPrm shard.GetPrm From 0624820909ac344eb0038a2fc3557137e1497af0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 May 2023 14:49:04 +0300 Subject: [PATCH 0247/1943] [#336] go.mod: Update dependencies Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 7 ++++++ go.mod | 34 +++++++++++++------------- go.sum | 69 ++++++++++++++++++++++++++-------------------------- 3 files changed, 58 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a4c48097..c05d065fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,13 @@ Changelog for FrostFS Node ### Removed ### Updated - `paulmach/orb` to v0.9.1 +- `github.com/nats-io/nats.go` to `v1.25.0` +- `golang.org/x/sync` to `v0.2.0` +- `golang.org/x/term` to `v0.8.0` +- `github.com/spf13/cobra` to `v1.7.0` +- `github.com/multiformats/go-multiaddr` to `v0.9.0` +- `go.uber.org/atomic` to `v1.11.0` + ### Updating from v0.36.0 ## [v0.36.0] - 2023-04-12 - Furtwängler diff --git a/go.mod b/go.mod index 0193937f3..2e9286a05 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230503082209-d4fe9a193d1a + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd git.frostfs.info/TrueCloudLab/hrw v1.2.0 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 @@ -16,8 +16,8 @@ require ( github.com/klauspost/compress v1.16.5 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 - github.com/multiformats/go-multiaddr v0.8.0 - github.com/nats-io/nats.go v1.22.1 + github.com/multiformats/go-multiaddr v0.9.0 + github.com/nats-io/nats.go v1.25.0 github.com/nspcc-dev/neo-go v0.100.1 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.4.0 @@ -25,18 +25,18 @@ require ( github.com/prometheus/client_golang v1.15.0 github.com/prometheus/client_model v0.3.0 github.com/spf13/cast v1.5.0 - github.com/spf13/cobra v1.6.1 + github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.2 go.etcd.io/bbolt v1.3.6 go.opentelemetry.io/otel v1.14.0 go.opentelemetry.io/otel/trace v1.14.0 - go.uber.org/atomic v1.10.0 + go.uber.org/atomic v1.11.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 - golang.org/x/sync v0.1.0 - golang.org/x/term v0.5.0 + golang.org/x/sync v0.2.0 + golang.org/x/term v0.8.0 google.golang.org/grpc v1.53.0 google.golang.org/protobuf v1.30.0 gopkg.in/yaml.v3 v3.0.1 @@ -63,9 +63,9 @@ require ( github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.0 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect - github.com/ipfs/go-cid v0.3.2 // indirect - github.com/klauspost/cpuid/v2 v2.2.2 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -73,11 +73,11 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multibase v0.1.1 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multihash v0.2.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/nats-io/nats-server/v2 v2.7.4 // indirect - github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect @@ -101,11 +101,11 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect go.opentelemetry.io/otel/sdk v1.14.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.4.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.9.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.1.7 // indirect diff --git a/go.sum b/go.sum index 68526591d..29fef2509 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02f git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230503082209-d4fe9a193d1a h1:I8cMZUthFTDGbQ5CYpThYEkypJmfLOBYVsC0metlpFI= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230503082209-d4fe9a193d1a/go.mod h1:TaJJOF3Uhuq8aqv2CrfuY2yhxePUinW35Xd3wfXLV/I= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd h1:HxacVl1Lc2RrfxAE13AGkp1tR/Mf4DDP6TgrgbLP5fQ= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd/go.mod h1:TaJJOF3Uhuq8aqv2CrfuY2yhxePUinW35Xd3wfXLV/I= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= @@ -269,10 +269,10 @@ github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= -github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -295,8 +295,8 @@ github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/d github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.2 h1:xPMwiykqNK9VK0NYC3+jTMYv9I6Vl3YdjZgPZKG3zO0= -github.com/klauspost/cpuid/v2 v2.2.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -346,10 +346,10 @@ github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aG github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= -github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= -github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= +github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= +github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= @@ -359,10 +359,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 h1:vU9tpM3apjYlLLeY23zRWJ9Zktr5jp+mloR942LEOpY= github.com/nats-io/nats-server/v2 v2.7.4 h1:c+BZJ3rGzUKCBIM4IXO8uNT2u1vajGbD1kPA6wqCEaM= github.com/nats-io/nats-server/v2 v2.7.4/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc= -github.com/nats-io/nats.go v1.22.1 h1:XzfqDspY0RNufzdrB8c4hFR+R3dahkxlpWe5+IWJzbE= -github.com/nats-io/nats.go v1.22.1/go.mod h1:tLqubohF7t4z3du1QDPYJIQQyhb4wl6DhjxEajSI7UA= -github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= -github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= +github.com/nats-io/nats.go v1.25.0 h1:t5/wCPGciR7X3Mu8QOi4jiJaXaWM8qtkLu4lzGZvYHE= +github.com/nats-io/nats.go v1.25.0/go.mod h1:D2WALIhz7V8M0pH8Scx8JZXlg6Oqz5VG+nQkK8nJdvg= +github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= +github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= @@ -475,8 +475,8 @@ github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -554,14 +554,14 @@ go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -576,13 +576,12 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -663,8 +662,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -689,8 +688,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -757,13 +756,13 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -773,8 +772,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 6f47c75e434143d39058ceeae2a1f028e6633c0a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 12 May 2023 10:41:04 +0300 Subject: [PATCH 0248/1943] [#125] ir: Set extra wallets on SIGHUP Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 1 + cmd/frostfs-ir/config.go | 5 +++ internal/logs/logs.go | 1 + pkg/innerring/initialization.go | 42 ++++++++----------- pkg/innerring/innerring.go | 18 ++++++-- .../processors/alphabet/process_emit.go | 18 ++++---- .../processors/alphabet/processor.go | 12 +++++- 7 files changed, 61 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c05d065fe..97feb2245 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Changelog for FrostFS Node - Change log level on SIGHUP for ir (#125) - Reload pprof and metrics on SIGHUP for ir (#125) - Support copies number parameter in `frostfs-cli object put` (#351) +- Set extra wallets on SIGHUP for ir (#125) ### Changed - `frostfs-cli util locode generate` is now much faster (#309) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 2e2aa1613..54c7d18e3 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -62,6 +62,11 @@ func watchForSignal(cancel func()) { } pprofCmp.reload() metricsCmp.reload() + log.Info(logs.FrostFSIRReloadExtraWallets) + err = innerRing.SetExtraWallets(cfg) + if err != nil { + log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) + } log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) case syscall.SIGTERM, syscall.SIGINT: log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 5cd1b8dba..742f6a8f7 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -414,6 +414,7 @@ const ( FrostFSIRApplicationStopped = "application stopped" // Info in ../node/cmd/frostfs-ir/main.go FrostFSIRCouldntCreateRPCClientForEndpoint = "could not create RPC client for endpoint" // Debug in ../node/pkg/morph/client/constructor.go FrostFSIRCreatedRPCClientForEndpoint = "created RPC client for endpoint" // Info in ../node/pkg/morph/client/constructor.go + FrostFSIRReloadExtraWallets = "reload extra wallets" // Info in ../node/cmd/frostfs-ir/config.go FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" // Error in ../node/cmd/frostfs-node/grpc.go FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" // Error in ../node/cmd/frostfs-node/grpc.go FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 1dc1d40ea..89269d50d 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -204,7 +204,7 @@ func (s *Server) createIRFetcher() irFetcher { return irf } -func (s *Server) initTimers(cfg *viper.Viper, processors *serverProcessors, morphClients *serverMorphClients) { +func (s *Server) initTimers(cfg *viper.Viper, morphClients *serverMorphClients) { s.epochTimer = newEpochTimer(&epochTimerArgs{ l: s.log, alphabetState: s, @@ -219,21 +219,21 @@ func (s *Server) initTimers(cfg *viper.Viper, processors *serverProcessors, morp // initialize emission timer emissionTimer := newEmissionTimer(&emitTimerArgs{ - ap: processors.AlphabetProcessor, + ap: s.alphabetProcessor, emitDuration: cfg.GetUint32("timers.emit"), }) s.addBlockTimer(emissionTimer) } -func (s *Server) initAlphabetProcessor(cfg *viper.Viper) (*alphabet.Processor, error) { +func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { parsedWallets, err := parseWalletAddressesFromStrings(cfg.GetStringSlice("emit.extra_wallets")) if err != nil { - return nil, err + return err } // create alphabet processor - alphabetProcessor, err := alphabet.New(&alphabet.Params{ + s.alphabetProcessor, err = alphabet.New(&alphabet.Params{ ParsedWallets: parsedWallets, Log: s.log, PoolSize: cfg.GetInt("workers.alphabet"), @@ -244,15 +244,15 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) (*alphabet.Processor, e StorageEmission: cfg.GetUint64("emit.storage.amount"), }) if err != nil { - return nil, err + return err } - err = bindMorphProcessor(alphabetProcessor, s) + err = bindMorphProcessor(s.alphabetProcessor, s) if err != nil { - return nil, err + return err } - return alphabetProcessor, nil + return nil } func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.Client, @@ -425,13 +425,7 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { return result, nil } -type serverProcessors struct { - AlphabetProcessor *alphabet.Processor -} - -func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClients) (*serverProcessors, error) { - result := &serverProcessors{} - +func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClients) error { irf := s.createIRFetcher() s.statusIndex = newInnerRingIndexer( @@ -443,35 +437,35 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien alphaSync, err := s.createAlphaSync(cfg, morphClients.FrostFSClient, irf) if err != nil { - return nil, err + return err } err = s.initNetmapProcessor(cfg, morphClients.CnrClient, alphaSync) if err != nil { - return nil, err + return err } err = s.initContainerProcessor(cfg, morphClients.CnrClient, morphClients.FrostFSIDClient) if err != nil { - return nil, err + return err } err = s.initBalanceProcessor(cfg, morphClients.FrostFSClient) if err != nil { - return nil, err + return err } err = s.initFrostFSMainnetProcessor(cfg, morphClients.FrostFSIDClient) if err != nil { - return nil, err + return err } - result.AlphabetProcessor, err = s.initAlphabetProcessor(cfg) + err = s.initAlphabetProcessor(cfg) if err != nil { - return nil, err + return err } - return result, nil + return nil } func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- error) (*chainParams, error) { diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 9119ff201..8c8c13dc3 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" @@ -73,7 +74,8 @@ type ( withoutMainNet bool // runtime processors - netmapProcessor *netmap.Processor + netmapProcessor *netmap.Processor + alphabetProcessor *alphabet.Processor workers []func(context.Context) @@ -383,13 +385,12 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - var processors *serverProcessors - processors, err = server.initProcessors(cfg, morphClients) + err = server.initProcessors(cfg, morphClients) if err != nil { return nil, err } - server.initTimers(cfg, processors, morphClients) + server.initTimers(cfg, morphClients) err = server.initGRPCServer(cfg) if err != nil { @@ -589,3 +590,12 @@ func (s *Server) newEpochTickHandlers() []newEpochHandler { return newEpochHandlers } + +func (s *Server) SetExtraWallets(cfg *viper.Viper) error { + parsedWallets, err := parseWalletAddressesFromStrings(cfg.GetStringSlice("emit.extra_wallets")) + if err != nil { + return err + } + s.alphabetProcessor.SetParsedWallets(parsedWallets) + return nil +} diff --git a/pkg/innerring/processors/alphabet/process_emit.go b/pkg/innerring/processors/alphabet/process_emit.go index b8d65dbc5..7a268ac52 100644 --- a/pkg/innerring/processors/alphabet/process_emit.go +++ b/pkg/innerring/processors/alphabet/process_emit.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" + "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" ) @@ -52,7 +53,10 @@ func (ap *Processor) processEmit() { nmNodes := networkMap.Nodes() nmLen := len(nmNodes) - extraLen := len(ap.parsedWallets) + ap.pwLock.RLock() + pw := ap.parsedWallets + ap.pwLock.RUnlock() + extraLen := len(pw) ap.log.Debug(logs.AlphabetGasEmission, zap.Int("network_map", nmLen), @@ -66,7 +70,7 @@ func (ap *Processor) processEmit() { ap.transferGasToNetmapNodes(nmNodes, gasPerNode) - ap.transferGasToExtraNodes(extraLen, gasPerNode) + ap.transferGasToExtraNodes(pw, gasPerNode) } func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerNode fixedn.Fixed8) { @@ -92,12 +96,12 @@ func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerN } } -func (ap *Processor) transferGasToExtraNodes(extraLen int, gasPerNode fixedn.Fixed8) { - if extraLen != 0 { - err := ap.morphClient.BatchTransferGas(ap.parsedWallets, gasPerNode) +func (ap *Processor) transferGasToExtraNodes(pw []util.Uint160, gasPerNode fixedn.Fixed8) { + if len(pw) > 0 { + err := ap.morphClient.BatchTransferGas(pw, gasPerNode) if err != nil { - receiversLog := make([]string, extraLen) - for i, addr := range ap.parsedWallets { + receiversLog := make([]string, len(pw)) + for i, addr := range pw { receiversLog[i] = addr.StringLE() } ap.log.Warn(logs.AlphabetCantTransferGasToWallet, diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index c2d7c1164..cd9088e03 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -3,6 +3,7 @@ package alphabet import ( "errors" "fmt" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -45,7 +46,9 @@ type ( // Processor of events produced for alphabet contracts in the sidechain. Processor struct { - parsedWallets []util.Uint160 + parsedWallets []util.Uint160 + // protects parsedWallets from concurrent change + pwLock *sync.RWMutex log *logger.Logger pool *ants.Pool alphabetContracts Contracts @@ -88,6 +91,7 @@ func New(p *Params) (*Processor, error) { return &Processor{ parsedWallets: p.ParsedWallets, + pwLock: new(sync.RWMutex), log: p.Log, pool: pool, alphabetContracts: p.AlphabetContracts, @@ -98,6 +102,12 @@ func New(p *Params) (*Processor, error) { }, nil } +func (ap *Processor) SetParsedWallets(parsedWallets []util.Uint160) { + ap.pwLock.Lock() + ap.parsedWallets = parsedWallets + ap.pwLock.Unlock() +} + // ListenerNotificationParsers for the 'event.Listener' event producer. func (ap *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { return nil From 13a7a90101d2b84917e7f281aae1de6804aa941a Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Wed, 17 May 2023 14:11:46 +0300 Subject: [PATCH 0249/1943] [#355] Increase tree svc client cache size to test hypotheses Signed-off-by: Alejandro Lopez --- pkg/services/tree/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 56b97e687..3288083cc 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -27,7 +27,7 @@ type cacheItem struct { } const ( - defaultClientCacheSize = 10 + defaultClientCacheSize = 32 defaultClientConnectTimeout = time.Second * 2 defaultReconnectInterval = time.Second * 15 ) From df9e099fa7af00e0cce242cf768b68ba5dcfb9e4 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 16 May 2023 11:22:20 +0300 Subject: [PATCH 0250/1943] [#352] Add explicit max number of alphabet nodes Signed-off-by: Alejandro Lopez --- cmd/frostfs-adm/docs/deploy.md | 1 + .../internal/modules/morph/generate.go | 3 ++ .../internal/modules/morph/initialize.go | 13 +++++++ .../internal/modules/morph/initialize_test.go | 34 +++++++++++++------ 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-adm/docs/deploy.md b/cmd/frostfs-adm/docs/deploy.md index aead65fe0..974c2a93c 100644 --- a/cmd/frostfs-adm/docs/deploy.md +++ b/cmd/frostfs-adm/docs/deploy.md @@ -18,6 +18,7 @@ To start a network, you need a set of consensus nodes, the same number of Alphabet nodes and any number of Storage nodes. While the number of Storage nodes can be scaled almost infinitely, the number of consensus and Alphabet nodes can't be changed so easily right now. Consider this before going any further. +Note also that there is an upper limit on the number of alphabet nodes (currently 22). It is easier to use`frostfs-adm` with a predefined configuration. First, create a network configuration file. In this example, there is going to be only one diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index dd327a4b1..ccdc4519f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -39,6 +39,9 @@ func generateAlphabetCreds(cmd *cobra.Command, _ []string) error { if size == 0 { return errors.New("size must be > 0") } + if size > maxAlphabetNodes { + return ErrTooManyAlphabetNodes + } v := viper.GetViper() walletDir := config.ResolveHomePath(viper.GetString(alphabetWalletsFlag)) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 494ad5296..9eb867faa 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -23,6 +23,13 @@ import ( "github.com/spf13/viper" ) +const ( + // maxAlphabetNodes is the maximum number of candidates allowed, which is currently limited by the size + // of the invocation script. + // See: https://github.com/nspcc-dev/neo-go/blob/740488f7f35e367eaa99a71c0a609c315fe2b0fc/pkg/core/transaction/witness.go#L10 + maxAlphabetNodes = 22 +) + type cache struct { nnsCs *state.Contract groupKey *keys.PublicKey @@ -45,6 +52,8 @@ type initializeContext struct { ContractPath string } +var ErrTooManyAlphabetNodes = fmt.Errorf("too many alphabet nodes (maximum allowed is %d)", maxAlphabetNodes) + func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { initCtx, err := newInitializeContext(cmd, viper.GetViper()) if err != nil { @@ -111,6 +120,10 @@ func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContex return nil, err } + if len(wallets) > maxAlphabetNodes { + return nil, ErrTooManyAlphabetNodes + } + needContracts := cmd.Name() == "update-contracts" || cmd.Name() == "init" var w *wallet.Wallet diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index e2e5aa0ad..07d2da8cc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -2,6 +2,7 @@ package morph import ( "encoding/hex" + "fmt" "os" "path/filepath" "strconv" @@ -40,8 +41,11 @@ func TestInitialize(t *testing.T) { t.Run("16 nodes", func(t *testing.T) { testInitialize(t, 16) }) - t.Run("22 nodes", func(t *testing.T) { - testInitialize(t, 22) + t.Run("max nodes", func(t *testing.T) { + testInitialize(t, maxAlphabetNodes) + }) + t.Run("too many nodes", func(t *testing.T) { + require.ErrorIs(t, generateTestData(t, t.TempDir(), maxAlphabetNodes+1), ErrTooManyAlphabetNodes) }) } @@ -49,7 +53,7 @@ func testInitialize(t *testing.T, committeeSize int) { testdataDir := t.TempDir() v := viper.GetViper() - generateTestData(t, testdataDir, committeeSize) + require.NoError(t, generateTestData(t, testdataDir, committeeSize)) v.Set(protoConfigPath, filepath.Join(testdataDir, protoFileName)) // Set to the path or remove the next statement to download from the network. @@ -80,25 +84,33 @@ func testInitialize(t *testing.T, committeeSize int) { }) } -func generateTestData(t *testing.T, dir string, size int) { +func generateTestData(t *testing.T, dir string, size int) error { v := viper.GetViper() v.Set(alphabetWalletsFlag, dir) sizeStr := strconv.FormatUint(uint64(size), 10) - require.NoError(t, generateAlphabetCmd.Flags().Set(alphabetSizeFlag, sizeStr)) + if err := generateAlphabetCmd.Flags().Set(alphabetSizeFlag, sizeStr); err != nil { + return err + } setTestCredentials(v, size) - require.NoError(t, generateAlphabetCreds(generateAlphabetCmd, nil)) + if err := generateAlphabetCreds(generateAlphabetCmd, nil); err != nil { + return err + } var pubs []string for i := 0; i < size; i++ { p := filepath.Join(dir, innerring.GlagoliticLetter(i).String()+".json") w, err := wallet.NewWalletFromFile(p) - require.NoError(t, err, "wallet doesn't exist") + if err != nil { + return fmt.Errorf("wallet doesn't exist: %w", err) + } for _, acc := range w.Accounts { if acc.Label == singleAccountName { pub, ok := vm.ParseSignatureContract(acc.Contract.Script) - require.True(t, ok) + if !ok { + return fmt.Errorf("could not parse signature script for %s", acc.Address) + } pubs = append(pubs, hex.EncodeToString(pub)) continue } @@ -113,10 +125,12 @@ func generateTestData(t *testing.T, dir string, size int) { cfg.ProtocolConfiguration.P2PSigExtensions = true cfg.ProtocolConfiguration.VerifyTransactions = true data, err := yaml.Marshal(cfg) - require.NoError(t, err) + if err != nil { + return err + } protoPath := filepath.Join(dir, protoFileName) - require.NoError(t, os.WriteFile(protoPath, data, os.ModePerm)) + return os.WriteFile(protoPath, data, os.ModePerm) } func setTestCredentials(v *viper.Viper, size int) { From aabcd8d3e42ad7bd8338f48d1e3c8ef0447c344f Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 18 May 2023 09:28:10 +0300 Subject: [PATCH 0251/1943] [#362] node: Cancel ctx in the same routine as for signal watcher Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config.go | 2 ++ cmd/frostfs-node/main.go | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 987d27fcd..c45d44fd2 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -321,6 +321,7 @@ func (a *applicationConfiguration) setGCConfig(newConfig *shardCfg, oldConfig *s // helpers and fields. type internals struct { done chan struct{} + ctxCancel func() internalErr chan error // channel for internal application errors at runtime appCfg *config.Config @@ -1078,6 +1079,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { func (c *cfg) shutdown() { c.setHealthStatus(control.HealthStatus_SHUTTING_DOWN) + c.ctxCancel() c.done <- struct{}{} for i := range c.closers { c.closers[len(c.closers)-1-i].fn() diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index a2024706b..425cf25a0 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -57,7 +57,8 @@ func main() { c := initCfg(appCfg) - ctx, cancel := context.WithCancel(context.Background()) + var ctx context.Context + ctx, c.ctxCancel = context.WithCancel(context.Background()) initApp(ctx, c) @@ -67,7 +68,7 @@ func main() { c.setHealthStatus(control.HealthStatus_READY) - wait(c, cancel) + wait(c) } func initAndLog(c *cfg, name string, initializer func(*cfg)) { @@ -140,14 +141,12 @@ func bootUp(ctx context.Context, c *cfg) { startWorkers(ctx, c) } -func wait(c *cfg, cancel func()) { +func wait(c *cfg) { c.log.Info(logs.CommonApplicationStarted, zap.String("version", misc.Version)) <-c.done // graceful shutdown - cancel() - c.log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop) c.wg.Wait() From 0fc494637f83ec51382c2faf460ef44147bd8859 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 18 May 2023 12:21:12 +0300 Subject: [PATCH 0252/1943] [#367] node: Disable unit tests cache Signed-off-by: Dmitrii Stepanov --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 29625e657..cf2095ec0 100755 --- a/Makefile +++ b/Makefile @@ -126,7 +126,7 @@ imports: # Run Unit Test with go test test: @echo "⇒ Running go test" - @go test ./... + @go test ./... -count=1 pre-commit-run: @pre-commit run -a --hook-stage manual From 100b1b5128f83eff490406e44f8af2b5905e2281 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 2 May 2023 18:40:54 +0300 Subject: [PATCH 0253/1943] [#329] node: Add async evacuate proto methods Signed-off-by: Dmitrii Stepanov --- pkg/services/control/server/evacuate_async.go | 20 + pkg/services/control/service.pb.go | 1550 ++++++++++++++--- pkg/services/control/service.proto | 103 ++ pkg/services/control/service_frostfs.pb.go | 516 ++++++ pkg/services/control/service_grpc.pb.go | 169 +- pkg/services/control/types.pb.go | 2 +- 6 files changed, 2129 insertions(+), 231 deletions(-) create mode 100644 pkg/services/control/server/evacuate_async.go diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go new file mode 100644 index 000000000..94ddc73d1 --- /dev/null +++ b/pkg/services/control/server/evacuate_async.go @@ -0,0 +1,20 @@ +package control + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" +) + +func (s *Server) StartShardEvacuation(context.Context, *control.StartShardEvacuationRequest) (*control.StartShardEvacuationResponse, error) { + return nil, fmt.Errorf("not implemented") +} + +func (s *Server) GetShardEvacuationStatus(context.Context, *control.GetShardEvacuationStatusRequest) (*control.GetShardEvacuationStatusResponse, error) { + return nil, fmt.Errorf("not implemented") +} + +func (s *Server) StopShardEvacuation(context.Context, *control.StopShardEvacuationRequest) (*control.StopShardEvacuationResponse, error) { + return nil, fmt.Errorf("not implemented") +} diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index a126ce16d..b1bebb1e2 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.12.4 +// protoc v3.21.9 // source: pkg/services/control/service.proto package control @@ -20,6 +20,56 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// Evacuate status enum. +type GetShardEvacuationStatusResponse_Body_Status int32 + +const ( + GetShardEvacuationStatusResponse_Body_EVACUATE_SHARD_STATUS_UNDEFINED GetShardEvacuationStatusResponse_Body_Status = 0 + GetShardEvacuationStatusResponse_Body_RUNNING GetShardEvacuationStatusResponse_Body_Status = 1 + GetShardEvacuationStatusResponse_Body_COMPLETED GetShardEvacuationStatusResponse_Body_Status = 2 +) + +// Enum value maps for GetShardEvacuationStatusResponse_Body_Status. +var ( + GetShardEvacuationStatusResponse_Body_Status_name = map[int32]string{ + 0: "EVACUATE_SHARD_STATUS_UNDEFINED", + 1: "RUNNING", + 2: "COMPLETED", + } + GetShardEvacuationStatusResponse_Body_Status_value = map[string]int32{ + "EVACUATE_SHARD_STATUS_UNDEFINED": 0, + "RUNNING": 1, + "COMPLETED": 2, + } +) + +func (x GetShardEvacuationStatusResponse_Body_Status) Enum() *GetShardEvacuationStatusResponse_Body_Status { + p := new(GetShardEvacuationStatusResponse_Body_Status) + *p = x + return p +} + +func (x GetShardEvacuationStatusResponse_Body_Status) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GetShardEvacuationStatusResponse_Body_Status) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_services_control_service_proto_enumTypes[0].Descriptor() +} + +func (GetShardEvacuationStatusResponse_Body_Status) Type() protoreflect.EnumType { + return &file_pkg_services_control_service_proto_enumTypes[0] +} + +func (x GetShardEvacuationStatusResponse_Body_Status) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GetShardEvacuationStatusResponse_Body_Status.Descriptor instead. +func (GetShardEvacuationStatusResponse_Body_Status) EnumDescriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0, 0} +} + // Health check request. type HealthCheckRequest struct { state protoimpl.MessageState @@ -1052,6 +1102,342 @@ func (x *DoctorResponse) GetSignature() *Signature { return nil } +// StartShardEvacuation request. +type StartShardEvacuationRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *StartShardEvacuationRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *StartShardEvacuationRequest) Reset() { + *x = StartShardEvacuationRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StartShardEvacuationRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartShardEvacuationRequest) ProtoMessage() {} + +func (x *StartShardEvacuationRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[18] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartShardEvacuationRequest.ProtoReflect.Descriptor instead. +func (*StartShardEvacuationRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18} +} + +func (x *StartShardEvacuationRequest) GetBody() *StartShardEvacuationRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *StartShardEvacuationRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// StartShardEvacuation response. +type StartShardEvacuationResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *StartShardEvacuationResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *StartShardEvacuationResponse) Reset() { + *x = StartShardEvacuationResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StartShardEvacuationResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartShardEvacuationResponse) ProtoMessage() {} + +func (x *StartShardEvacuationResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[19] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartShardEvacuationResponse.ProtoReflect.Descriptor instead. +func (*StartShardEvacuationResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{19} +} + +func (x *StartShardEvacuationResponse) GetBody() *StartShardEvacuationResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *StartShardEvacuationResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// GetShardEvacuationStatus request. +type GetShardEvacuationStatusRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *GetShardEvacuationStatusRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *GetShardEvacuationStatusRequest) Reset() { + *x = GetShardEvacuationStatusRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetShardEvacuationStatusRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetShardEvacuationStatusRequest) ProtoMessage() {} + +func (x *GetShardEvacuationStatusRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[20] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetShardEvacuationStatusRequest.ProtoReflect.Descriptor instead. +func (*GetShardEvacuationStatusRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20} +} + +func (x *GetShardEvacuationStatusRequest) GetBody() *GetShardEvacuationStatusRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *GetShardEvacuationStatusRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// GetShardEvacuationStatus response. +type GetShardEvacuationStatusResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *GetShardEvacuationStatusResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *GetShardEvacuationStatusResponse) Reset() { + *x = GetShardEvacuationStatusResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetShardEvacuationStatusResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetShardEvacuationStatusResponse) ProtoMessage() {} + +func (x *GetShardEvacuationStatusResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[21] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetShardEvacuationStatusResponse.ProtoReflect.Descriptor instead. +func (*GetShardEvacuationStatusResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21} +} + +func (x *GetShardEvacuationStatusResponse) GetBody() *GetShardEvacuationStatusResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *GetShardEvacuationStatusResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// StopShardEvacuation request. +type StopShardEvacuationRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *StopShardEvacuationRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *StopShardEvacuationRequest) Reset() { + *x = StopShardEvacuationRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StopShardEvacuationRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StopShardEvacuationRequest) ProtoMessage() {} + +func (x *StopShardEvacuationRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[22] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StopShardEvacuationRequest.ProtoReflect.Descriptor instead. +func (*StopShardEvacuationRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22} +} + +func (x *StopShardEvacuationRequest) GetBody() *StopShardEvacuationRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *StopShardEvacuationRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// StopShardEvacuation response. +type StopShardEvacuationResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *StopShardEvacuationResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *StopShardEvacuationResponse) Reset() { + *x = StopShardEvacuationResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StopShardEvacuationResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StopShardEvacuationResponse) ProtoMessage() {} + +func (x *StopShardEvacuationResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[23] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StopShardEvacuationResponse.ProtoReflect.Descriptor instead. +func (*StopShardEvacuationResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23} +} + +func (x *StopShardEvacuationResponse) GetBody() *StopShardEvacuationResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *StopShardEvacuationResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + // Health check request body. type HealthCheckRequest_Body struct { state protoimpl.MessageState @@ -1062,7 +1448,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[18] + mi := &file_pkg_services_control_service_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1075,7 +1461,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[18] + mi := &file_pkg_services_control_service_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1106,7 +1492,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[19] + mi := &file_pkg_services_control_service_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1119,7 +1505,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[19] + mi := &file_pkg_services_control_service_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1170,7 +1556,7 @@ type SetNetmapStatusRequest_Body struct { func (x *SetNetmapStatusRequest_Body) Reset() { *x = SetNetmapStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[20] + mi := &file_pkg_services_control_service_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1183,7 +1569,7 @@ func (x *SetNetmapStatusRequest_Body) String() string { func (*SetNetmapStatusRequest_Body) ProtoMessage() {} func (x *SetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[20] + mi := &file_pkg_services_control_service_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1223,7 +1609,7 @@ type SetNetmapStatusResponse_Body struct { func (x *SetNetmapStatusResponse_Body) Reset() { *x = SetNetmapStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[21] + mi := &file_pkg_services_control_service_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1236,7 +1622,7 @@ func (x *SetNetmapStatusResponse_Body) String() string { func (*SetNetmapStatusResponse_Body) ProtoMessage() {} func (x *SetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[21] + mi := &file_pkg_services_control_service_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1266,7 +1652,7 @@ type DropObjectsRequest_Body struct { func (x *DropObjectsRequest_Body) Reset() { *x = DropObjectsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[22] + mi := &file_pkg_services_control_service_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1279,7 +1665,7 @@ func (x *DropObjectsRequest_Body) String() string { func (*DropObjectsRequest_Body) ProtoMessage() {} func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[22] + mi := &file_pkg_services_control_service_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1312,7 +1698,7 @@ type DropObjectsResponse_Body struct { func (x *DropObjectsResponse_Body) Reset() { *x = DropObjectsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[23] + mi := &file_pkg_services_control_service_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1325,7 +1711,7 @@ func (x *DropObjectsResponse_Body) String() string { func (*DropObjectsResponse_Body) ProtoMessage() {} func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[23] + mi := &file_pkg_services_control_service_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1351,7 +1737,7 @@ type ListShardsRequest_Body struct { func (x *ListShardsRequest_Body) Reset() { *x = ListShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1364,7 +1750,7 @@ func (x *ListShardsRequest_Body) String() string { func (*ListShardsRequest_Body) ProtoMessage() {} func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1393,7 +1779,7 @@ type ListShardsResponse_Body struct { func (x *ListShardsResponse_Body) Reset() { *x = ListShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1406,7 +1792,7 @@ func (x *ListShardsResponse_Body) String() string { func (*ListShardsResponse_Body) ProtoMessage() {} func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1446,7 +1832,7 @@ type SetShardModeRequest_Body struct { func (x *SetShardModeRequest_Body) Reset() { *x = SetShardModeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1459,7 +1845,7 @@ func (x *SetShardModeRequest_Body) String() string { func (*SetShardModeRequest_Body) ProtoMessage() {} func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1506,7 +1892,7 @@ type SetShardModeResponse_Body struct { func (x *SetShardModeResponse_Body) Reset() { *x = SetShardModeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1519,7 +1905,7 @@ func (x *SetShardModeResponse_Body) String() string { func (*SetShardModeResponse_Body) ProtoMessage() {} func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1550,7 +1936,7 @@ type SynchronizeTreeRequest_Body struct { func (x *SynchronizeTreeRequest_Body) Reset() { *x = SynchronizeTreeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1563,7 +1949,7 @@ func (x *SynchronizeTreeRequest_Body) String() string { func (*SynchronizeTreeRequest_Body) ProtoMessage() {} func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1610,7 +1996,7 @@ type SynchronizeTreeResponse_Body struct { func (x *SynchronizeTreeResponse_Body) Reset() { *x = SynchronizeTreeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1623,7 +2009,7 @@ func (x *SynchronizeTreeResponse_Body) String() string { func (*SynchronizeTreeResponse_Body) ProtoMessage() {} func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1654,7 +2040,7 @@ type EvacuateShardRequest_Body struct { func (x *EvacuateShardRequest_Body) Reset() { *x = EvacuateShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1667,7 +2053,7 @@ func (x *EvacuateShardRequest_Body) String() string { func (*EvacuateShardRequest_Body) ProtoMessage() {} func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1709,7 +2095,7 @@ type EvacuateShardResponse_Body struct { func (x *EvacuateShardResponse_Body) Reset() { *x = EvacuateShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1722,7 +2108,7 @@ func (x *EvacuateShardResponse_Body) String() string { func (*EvacuateShardResponse_Body) ProtoMessage() {} func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1758,7 +2144,7 @@ type FlushCacheRequest_Body struct { func (x *FlushCacheRequest_Body) Reset() { *x = FlushCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1771,7 +2157,7 @@ func (x *FlushCacheRequest_Body) String() string { func (*FlushCacheRequest_Body) ProtoMessage() {} func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1804,7 +2190,7 @@ type FlushCacheResponse_Body struct { func (x *FlushCacheResponse_Body) Reset() { *x = FlushCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1817,7 +2203,7 @@ func (x *FlushCacheResponse_Body) String() string { func (*FlushCacheResponse_Body) ProtoMessage() {} func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1848,7 +2234,7 @@ type DoctorRequest_Body struct { func (x *DoctorRequest_Body) Reset() { *x = DoctorRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1861,7 +2247,7 @@ func (x *DoctorRequest_Body) String() string { func (*DoctorRequest_Body) ProtoMessage() {} func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1901,7 +2287,7 @@ type DoctorResponse_Body struct { func (x *DoctorResponse_Body) Reset() { *x = DoctorResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1914,7 +2300,7 @@ func (x *DoctorResponse_Body) String() string { func (*DoctorResponse_Body) ProtoMessage() {} func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1930,6 +2316,428 @@ func (*DoctorResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{17, 0} } +// Request body structure. +type StartShardEvacuationRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // IDs of the shards. + Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` + // Flag indicating whether object read errors should be ignored. + IgnoreErrors bool `protobuf:"varint,2,opt,name=ignore_errors,json=ignoreErrors,proto3" json:"ignore_errors,omitempty"` +} + +func (x *StartShardEvacuationRequest_Body) Reset() { + *x = StartShardEvacuationRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[42] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StartShardEvacuationRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartShardEvacuationRequest_Body) ProtoMessage() {} + +func (x *StartShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[42] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartShardEvacuationRequest_Body.ProtoReflect.Descriptor instead. +func (*StartShardEvacuationRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18, 0} +} + +func (x *StartShardEvacuationRequest_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} + +func (x *StartShardEvacuationRequest_Body) GetIgnoreErrors() bool { + if x != nil { + return x.IgnoreErrors + } + return false +} + +// Response body structure. +type StartShardEvacuationResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *StartShardEvacuationResponse_Body) Reset() { + *x = StartShardEvacuationResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[43] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StartShardEvacuationResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartShardEvacuationResponse_Body) ProtoMessage() {} + +func (x *StartShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[43] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartShardEvacuationResponse_Body.ProtoReflect.Descriptor instead. +func (*StartShardEvacuationResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{19, 0} +} + +// Request body structure. +type GetShardEvacuationStatusRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *GetShardEvacuationStatusRequest_Body) Reset() { + *x = GetShardEvacuationStatusRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[44] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetShardEvacuationStatusRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetShardEvacuationStatusRequest_Body) ProtoMessage() {} + +func (x *GetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[44] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetShardEvacuationStatusRequest_Body.ProtoReflect.Descriptor instead. +func (*GetShardEvacuationStatusRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20, 0} +} + +// Response body structure. +type GetShardEvacuationStatusResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Total objects to evacuate count. The value is approximate, so evacuated + failed == total is not guaranteed after completion. + Total uint64 `protobuf:"varint,1,opt,name=total,proto3" json:"total,omitempty"` + // Evacuated objects count. + Evacuated uint64 `protobuf:"varint,2,opt,name=evacuated,proto3" json:"evacuated,omitempty"` + // Failed objects count. + Failed uint64 `protobuf:"varint,3,opt,name=failed,proto3" json:"failed,omitempty"` + // Shard IDs. + Shard_ID [][]byte `protobuf:"bytes,4,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` + // Evacuation process status. + Status GetShardEvacuationStatusResponse_Body_Status `protobuf:"varint,5,opt,name=status,proto3,enum=control.GetShardEvacuationStatusResponse_Body_Status" json:"status,omitempty"` + // Evacuation process duration. + Duration *GetShardEvacuationStatusResponse_Body_Duration `protobuf:"bytes,6,opt,name=duration,proto3" json:"duration,omitempty"` + // Evacuation process started at timestamp. + StartedAt *GetShardEvacuationStatusResponse_Body_UnixTimestamp `protobuf:"bytes,7,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + // Error message if evacuation failed. + ErrorMessage string `protobuf:"bytes,8,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` +} + +func (x *GetShardEvacuationStatusResponse_Body) Reset() { + *x = GetShardEvacuationStatusResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[45] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetShardEvacuationStatusResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetShardEvacuationStatusResponse_Body) ProtoMessage() {} + +func (x *GetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[45] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetShardEvacuationStatusResponse_Body.ProtoReflect.Descriptor instead. +func (*GetShardEvacuationStatusResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0} +} + +func (x *GetShardEvacuationStatusResponse_Body) GetTotal() uint64 { + if x != nil { + return x.Total + } + return 0 +} + +func (x *GetShardEvacuationStatusResponse_Body) GetEvacuated() uint64 { + if x != nil { + return x.Evacuated + } + return 0 +} + +func (x *GetShardEvacuationStatusResponse_Body) GetFailed() uint64 { + if x != nil { + return x.Failed + } + return 0 +} + +func (x *GetShardEvacuationStatusResponse_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} + +func (x *GetShardEvacuationStatusResponse_Body) GetStatus() GetShardEvacuationStatusResponse_Body_Status { + if x != nil { + return x.Status + } + return GetShardEvacuationStatusResponse_Body_EVACUATE_SHARD_STATUS_UNDEFINED +} + +func (x *GetShardEvacuationStatusResponse_Body) GetDuration() *GetShardEvacuationStatusResponse_Body_Duration { + if x != nil { + return x.Duration + } + return nil +} + +func (x *GetShardEvacuationStatusResponse_Body) GetStartedAt() *GetShardEvacuationStatusResponse_Body_UnixTimestamp { + if x != nil { + return x.StartedAt + } + return nil +} + +func (x *GetShardEvacuationStatusResponse_Body) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" +} + +// Unix timestamp value. +type GetShardEvacuationStatusResponse_Body_UnixTimestamp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Value int64 `protobuf:"varint,1,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) Reset() { + *x = GetShardEvacuationStatusResponse_Body_UnixTimestamp{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[46] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoMessage() {} + +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[46] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetShardEvacuationStatusResponse_Body_UnixTimestamp.ProtoReflect.Descriptor instead. +func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0, 0} +} + +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) GetValue() int64 { + if x != nil { + return x.Value + } + return 0 +} + +// Duration in seconds. +type GetShardEvacuationStatusResponse_Body_Duration struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Seconds int64 `protobuf:"varint,1,opt,name=seconds,proto3" json:"seconds,omitempty"` +} + +func (x *GetShardEvacuationStatusResponse_Body_Duration) Reset() { + *x = GetShardEvacuationStatusResponse_Body_Duration{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[47] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetShardEvacuationStatusResponse_Body_Duration) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetShardEvacuationStatusResponse_Body_Duration) ProtoMessage() {} + +func (x *GetShardEvacuationStatusResponse_Body_Duration) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[47] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetShardEvacuationStatusResponse_Body_Duration.ProtoReflect.Descriptor instead. +func (*GetShardEvacuationStatusResponse_Body_Duration) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0, 1} +} + +func (x *GetShardEvacuationStatusResponse_Body_Duration) GetSeconds() int64 { + if x != nil { + return x.Seconds + } + return 0 +} + +// Request body structure. +type StopShardEvacuationRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *StopShardEvacuationRequest_Body) Reset() { + *x = StopShardEvacuationRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[48] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StopShardEvacuationRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StopShardEvacuationRequest_Body) ProtoMessage() {} + +func (x *StopShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[48] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StopShardEvacuationRequest_Body.ProtoReflect.Descriptor instead. +func (*StopShardEvacuationRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22, 0} +} + +// Response body structure. +type StopShardEvacuationResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *StopShardEvacuationResponse_Body) Reset() { + *x = StopShardEvacuationResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[49] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StopShardEvacuationResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StopShardEvacuationResponse_Body) ProtoMessage() {} + +func (x *StopShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[49] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StopShardEvacuationResponse_Body.ProtoReflect.Descriptor instead. +func (*StopShardEvacuationResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0} +} + var File_pkg_services_control_service_proto protoreflect.FileDescriptor var file_pkg_services_control_service_proto_rawDesc = []byte{ @@ -2134,55 +2942,171 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb6, 0x05, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, - 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xd6, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, + 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, + 0x98, 0x01, 0x0a, 0x1c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x9e, 0x01, 0x0a, 0x1f, 0x47, + 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xc6, 0x05, 0x0a, 0x20, + 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xab, 0x04, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, + 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x4d, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x35, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5b, 0x0a, 0x0a, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x55, + 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x25, 0x0a, 0x0d, + 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x1a, 0x24, 0x0a, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x18, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x23, 0x0a, 0x1f, 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, + 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, + 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, + 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, + 0x45, 0x44, 0x10, 0x02, 0x22, 0x94, 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, + 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x32, 0xee, 0x07, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, + 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, + 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, + 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, - 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, - 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, - 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, - 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, - 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, - 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, - 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, + 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, + 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, + 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, + 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, + 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, + 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, + 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, + 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, + 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, + 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2197,115 +3121,152 @@ func file_pkg_services_control_service_proto_rawDescGZIP() []byte { return file_pkg_services_control_service_proto_rawDescData } -var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 36) +var file_pkg_services_control_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 50) var file_pkg_services_control_service_proto_goTypes = []interface{}{ - (*HealthCheckRequest)(nil), // 0: control.HealthCheckRequest - (*HealthCheckResponse)(nil), // 1: control.HealthCheckResponse - (*SetNetmapStatusRequest)(nil), // 2: control.SetNetmapStatusRequest - (*SetNetmapStatusResponse)(nil), // 3: control.SetNetmapStatusResponse - (*DropObjectsRequest)(nil), // 4: control.DropObjectsRequest - (*DropObjectsResponse)(nil), // 5: control.DropObjectsResponse - (*ListShardsRequest)(nil), // 6: control.ListShardsRequest - (*ListShardsResponse)(nil), // 7: control.ListShardsResponse - (*SetShardModeRequest)(nil), // 8: control.SetShardModeRequest - (*SetShardModeResponse)(nil), // 9: control.SetShardModeResponse - (*SynchronizeTreeRequest)(nil), // 10: control.SynchronizeTreeRequest - (*SynchronizeTreeResponse)(nil), // 11: control.SynchronizeTreeResponse - (*EvacuateShardRequest)(nil), // 12: control.EvacuateShardRequest - (*EvacuateShardResponse)(nil), // 13: control.EvacuateShardResponse - (*FlushCacheRequest)(nil), // 14: control.FlushCacheRequest - (*FlushCacheResponse)(nil), // 15: control.FlushCacheResponse - (*DoctorRequest)(nil), // 16: control.DoctorRequest - (*DoctorResponse)(nil), // 17: control.DoctorResponse - (*HealthCheckRequest_Body)(nil), // 18: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 19: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 20: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 21: control.SetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 22: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 23: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 24: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 25: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 26: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 27: control.SetShardModeResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 28: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 29: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 30: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 31: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 32: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 33: control.FlushCacheResponse.Body - (*DoctorRequest_Body)(nil), // 34: control.DoctorRequest.Body - (*DoctorResponse_Body)(nil), // 35: control.DoctorResponse.Body - (*Signature)(nil), // 36: control.Signature - (NetmapStatus)(0), // 37: control.NetmapStatus - (HealthStatus)(0), // 38: control.HealthStatus - (*ShardInfo)(nil), // 39: control.ShardInfo - (ShardMode)(0), // 40: control.ShardMode + (GetShardEvacuationStatusResponse_Body_Status)(0), // 0: control.GetShardEvacuationStatusResponse.Body.Status + (*HealthCheckRequest)(nil), // 1: control.HealthCheckRequest + (*HealthCheckResponse)(nil), // 2: control.HealthCheckResponse + (*SetNetmapStatusRequest)(nil), // 3: control.SetNetmapStatusRequest + (*SetNetmapStatusResponse)(nil), // 4: control.SetNetmapStatusResponse + (*DropObjectsRequest)(nil), // 5: control.DropObjectsRequest + (*DropObjectsResponse)(nil), // 6: control.DropObjectsResponse + (*ListShardsRequest)(nil), // 7: control.ListShardsRequest + (*ListShardsResponse)(nil), // 8: control.ListShardsResponse + (*SetShardModeRequest)(nil), // 9: control.SetShardModeRequest + (*SetShardModeResponse)(nil), // 10: control.SetShardModeResponse + (*SynchronizeTreeRequest)(nil), // 11: control.SynchronizeTreeRequest + (*SynchronizeTreeResponse)(nil), // 12: control.SynchronizeTreeResponse + (*EvacuateShardRequest)(nil), // 13: control.EvacuateShardRequest + (*EvacuateShardResponse)(nil), // 14: control.EvacuateShardResponse + (*FlushCacheRequest)(nil), // 15: control.FlushCacheRequest + (*FlushCacheResponse)(nil), // 16: control.FlushCacheResponse + (*DoctorRequest)(nil), // 17: control.DoctorRequest + (*DoctorResponse)(nil), // 18: control.DoctorResponse + (*StartShardEvacuationRequest)(nil), // 19: control.StartShardEvacuationRequest + (*StartShardEvacuationResponse)(nil), // 20: control.StartShardEvacuationResponse + (*GetShardEvacuationStatusRequest)(nil), // 21: control.GetShardEvacuationStatusRequest + (*GetShardEvacuationStatusResponse)(nil), // 22: control.GetShardEvacuationStatusResponse + (*StopShardEvacuationRequest)(nil), // 23: control.StopShardEvacuationRequest + (*StopShardEvacuationResponse)(nil), // 24: control.StopShardEvacuationResponse + (*HealthCheckRequest_Body)(nil), // 25: control.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 26: control.HealthCheckResponse.Body + (*SetNetmapStatusRequest_Body)(nil), // 27: control.SetNetmapStatusRequest.Body + (*SetNetmapStatusResponse_Body)(nil), // 28: control.SetNetmapStatusResponse.Body + (*DropObjectsRequest_Body)(nil), // 29: control.DropObjectsRequest.Body + (*DropObjectsResponse_Body)(nil), // 30: control.DropObjectsResponse.Body + (*ListShardsRequest_Body)(nil), // 31: control.ListShardsRequest.Body + (*ListShardsResponse_Body)(nil), // 32: control.ListShardsResponse.Body + (*SetShardModeRequest_Body)(nil), // 33: control.SetShardModeRequest.Body + (*SetShardModeResponse_Body)(nil), // 34: control.SetShardModeResponse.Body + (*SynchronizeTreeRequest_Body)(nil), // 35: control.SynchronizeTreeRequest.Body + (*SynchronizeTreeResponse_Body)(nil), // 36: control.SynchronizeTreeResponse.Body + (*EvacuateShardRequest_Body)(nil), // 37: control.EvacuateShardRequest.Body + (*EvacuateShardResponse_Body)(nil), // 38: control.EvacuateShardResponse.Body + (*FlushCacheRequest_Body)(nil), // 39: control.FlushCacheRequest.Body + (*FlushCacheResponse_Body)(nil), // 40: control.FlushCacheResponse.Body + (*DoctorRequest_Body)(nil), // 41: control.DoctorRequest.Body + (*DoctorResponse_Body)(nil), // 42: control.DoctorResponse.Body + (*StartShardEvacuationRequest_Body)(nil), // 43: control.StartShardEvacuationRequest.Body + (*StartShardEvacuationResponse_Body)(nil), // 44: control.StartShardEvacuationResponse.Body + (*GetShardEvacuationStatusRequest_Body)(nil), // 45: control.GetShardEvacuationStatusRequest.Body + (*GetShardEvacuationStatusResponse_Body)(nil), // 46: control.GetShardEvacuationStatusResponse.Body + (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 47: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 48: control.GetShardEvacuationStatusResponse.Body.Duration + (*StopShardEvacuationRequest_Body)(nil), // 49: control.StopShardEvacuationRequest.Body + (*StopShardEvacuationResponse_Body)(nil), // 50: control.StopShardEvacuationResponse.Body + (*Signature)(nil), // 51: control.Signature + (NetmapStatus)(0), // 52: control.NetmapStatus + (HealthStatus)(0), // 53: control.HealthStatus + (*ShardInfo)(nil), // 54: control.ShardInfo + (ShardMode)(0), // 55: control.ShardMode } var file_pkg_services_control_service_proto_depIdxs = []int32{ - 18, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 36, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 19, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 36, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 20, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 36, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 21, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 36, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 22, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 36, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature - 23, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 36, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature - 24, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 36, // 13: control.ListShardsRequest.signature:type_name -> control.Signature - 25, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 36, // 15: control.ListShardsResponse.signature:type_name -> control.Signature - 26, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 36, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature - 27, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 36, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature - 28, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 36, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 29, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 36, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 30, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 36, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature - 31, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 36, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature - 32, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 36, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature - 33, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 36, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature - 34, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body - 36, // 33: control.DoctorRequest.signature:type_name -> control.Signature - 35, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body - 36, // 35: control.DoctorResponse.signature:type_name -> control.Signature - 37, // 36: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 38, // 37: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 37, // 38: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 39, // 39: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 40, // 40: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 0, // 41: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 2, // 42: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 4, // 43: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 6, // 44: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 8, // 45: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 10, // 46: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 12, // 47: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 14, // 48: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 16, // 49: control.ControlService.Doctor:input_type -> control.DoctorRequest - 1, // 50: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 3, // 51: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 5, // 52: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 7, // 53: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 9, // 54: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 11, // 55: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 13, // 56: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 15, // 57: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 17, // 58: control.ControlService.Doctor:output_type -> control.DoctorResponse - 50, // [50:59] is the sub-list for method output_type - 41, // [41:50] is the sub-list for method input_type - 41, // [41:41] is the sub-list for extension type_name - 41, // [41:41] is the sub-list for extension extendee - 0, // [0:41] is the sub-list for field type_name + 25, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body + 51, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature + 26, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body + 51, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature + 27, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body + 51, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature + 28, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body + 51, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature + 29, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body + 51, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature + 30, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body + 51, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature + 31, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body + 51, // 13: control.ListShardsRequest.signature:type_name -> control.Signature + 32, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body + 51, // 15: control.ListShardsResponse.signature:type_name -> control.Signature + 33, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body + 51, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature + 34, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body + 51, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature + 35, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body + 51, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature + 36, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body + 51, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature + 37, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body + 51, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature + 38, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body + 51, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature + 39, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body + 51, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature + 40, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body + 51, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature + 41, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body + 51, // 33: control.DoctorRequest.signature:type_name -> control.Signature + 42, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body + 51, // 35: control.DoctorResponse.signature:type_name -> control.Signature + 43, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body + 51, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature + 44, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body + 51, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature + 45, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body + 51, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 46, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body + 51, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 49, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body + 51, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature + 50, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body + 51, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature + 52, // 48: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus + 53, // 49: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus + 52, // 50: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus + 54, // 51: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo + 55, // 52: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode + 0, // 53: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status + 48, // 54: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration + 47, // 55: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + 1, // 56: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 3, // 57: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 5, // 58: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 7, // 59: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 9, // 60: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 11, // 61: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 13, // 62: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 19, // 63: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest + 21, // 64: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest + 23, // 65: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest + 15, // 66: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 17, // 67: control.ControlService.Doctor:input_type -> control.DoctorRequest + 2, // 68: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 4, // 69: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 6, // 70: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 8, // 71: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 10, // 72: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 12, // 73: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 14, // 74: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 20, // 75: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse + 22, // 76: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse + 24, // 77: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse + 16, // 78: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 18, // 79: control.ControlService.Doctor:output_type -> control.DoctorResponse + 68, // [68:80] is the sub-list for method output_type + 56, // [56:68] is the sub-list for method input_type + 56, // [56:56] is the sub-list for extension type_name + 56, // [56:56] is the sub-list for extension extendee + 0, // [0:56] is the sub-list for field type_name } func init() { file_pkg_services_control_service_proto_init() } @@ -2532,7 +3493,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*StartShardEvacuationRequest); i { case 0: return &v.state case 1: @@ -2544,7 +3505,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { + switch v := v.(*StartShardEvacuationResponse); i { case 0: return &v.state case 1: @@ -2556,7 +3517,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusRequest_Body); i { + switch v := v.(*GetShardEvacuationStatusRequest); i { case 0: return &v.state case 1: @@ -2568,7 +3529,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusResponse_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse); i { case 0: return &v.state case 1: @@ -2580,7 +3541,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest_Body); i { + switch v := v.(*StopShardEvacuationRequest); i { case 0: return &v.state case 1: @@ -2592,7 +3553,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse_Body); i { + switch v := v.(*StopShardEvacuationResponse); i { case 0: return &v.state case 1: @@ -2604,7 +3565,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest_Body); i { + switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state case 1: @@ -2616,7 +3577,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse_Body); i { + switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state case 1: @@ -2628,7 +3589,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest_Body); i { + switch v := v.(*SetNetmapStatusRequest_Body); i { case 0: return &v.state case 1: @@ -2640,7 +3601,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse_Body); i { + switch v := v.(*SetNetmapStatusResponse_Body); i { case 0: return &v.state case 1: @@ -2652,7 +3613,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest_Body); i { + switch v := v.(*DropObjectsRequest_Body); i { case 0: return &v.state case 1: @@ -2664,7 +3625,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse_Body); i { + switch v := v.(*DropObjectsResponse_Body); i { case 0: return &v.state case 1: @@ -2676,7 +3637,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest_Body); i { + switch v := v.(*ListShardsRequest_Body); i { case 0: return &v.state case 1: @@ -2688,7 +3649,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse_Body); i { + switch v := v.(*ListShardsResponse_Body); i { case 0: return &v.state case 1: @@ -2700,7 +3661,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest_Body); i { + switch v := v.(*SetShardModeRequest_Body); i { case 0: return &v.state case 1: @@ -2712,7 +3673,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheResponse_Body); i { + switch v := v.(*SetShardModeResponse_Body); i { case 0: return &v.state case 1: @@ -2724,7 +3685,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorRequest_Body); i { + switch v := v.(*SynchronizeTreeRequest_Body); i { case 0: return &v.state case 1: @@ -2736,6 +3697,78 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SynchronizeTreeResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EvacuateShardRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EvacuateShardResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FlushCacheRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FlushCacheResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DoctorRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DoctorResponse_Body); i { case 0: return &v.state @@ -2747,19 +3780,116 @@ func file_pkg_services_control_service_proto_init() { return nil } } + file_pkg_services_control_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StartShardEvacuationRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StartShardEvacuationResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetShardEvacuationStatusRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetShardEvacuationStatusResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StopShardEvacuationRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StopShardEvacuationResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_service_proto_rawDesc, - NumEnums: 0, - NumMessages: 36, + NumEnums: 1, + NumMessages: 50, NumExtensions: 0, NumServices: 1, }, GoTypes: file_pkg_services_control_service_proto_goTypes, DependencyIndexes: file_pkg_services_control_service_proto_depIdxs, + EnumInfos: file_pkg_services_control_service_proto_enumTypes, MessageInfos: file_pkg_services_control_service_proto_msgTypes, }.Build() File_pkg_services_control_service_proto = out.File diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 32a87c744..ca35ae043 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -29,6 +29,15 @@ service ControlService { // EvacuateShard moves all data from one shard to the others. rpc EvacuateShard (EvacuateShardRequest) returns (EvacuateShardResponse); + // StartShardEvacuation starts moving all data from one shard to the others. + rpc StartShardEvacuation (StartShardEvacuationRequest) returns (StartShardEvacuationResponse); + + // GetShardEvacuationStatus returns evacuation status. + rpc GetShardEvacuationStatus (GetShardEvacuationStatusRequest) returns (GetShardEvacuationStatusResponse); + + // StopShardEvacuation stops moving all data from one shard to the others. + rpc StopShardEvacuation (StopShardEvacuationRequest) returns (StopShardEvacuationResponse); + // FlushCache moves all data from one shard to the others. rpc FlushCache (FlushCacheRequest) returns (FlushCacheResponse); @@ -298,3 +307,97 @@ message DoctorResponse { Body body = 1; Signature signature = 2; } + +// StartShardEvacuation request. +message StartShardEvacuationRequest { + // Request body structure. + message Body { + // IDs of the shards. + repeated bytes shard_ID = 1; + // Flag indicating whether object read errors should be ignored. + bool ignore_errors = 2; + } + + Body body = 1; + Signature signature = 2; +} + +// StartShardEvacuation response. +message StartShardEvacuationResponse { + // Response body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} + +// GetShardEvacuationStatus request. +message GetShardEvacuationStatusRequest { + // Request body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} + +// GetShardEvacuationStatus response. +message GetShardEvacuationStatusResponse { + // Response body structure. + message Body { + // Evacuate status enum. + enum Status { + EVACUATE_SHARD_STATUS_UNDEFINED = 0; + RUNNING = 1; + COMPLETED = 2; + } + + // Unix timestamp value. + message UnixTimestamp { + int64 value = 1; + } + + // Duration in seconds. + message Duration { + int64 seconds = 1; + } + + // Total objects to evacuate count. The value is approximate, so evacuated + failed == total is not guaranteed after completion. + uint64 total = 1; + // Evacuated objects count. + uint64 evacuated = 2; + // Failed objects count. + uint64 failed = 3; + + // Shard IDs. + repeated bytes shard_ID = 4; + // Evacuation process status. + Status status = 5; + // Evacuation process duration. + Duration duration = 6; + // Evacuation process started at timestamp. + UnixTimestamp started_at = 7; + // Error message if evacuation failed. + string error_message = 8; + } + + Body body = 1; + Signature signature = 2; +} + +// StopShardEvacuation request. +message StopShardEvacuationRequest { + // Request body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} + +// StopShardEvacuation response. +message StopShardEvacuationResponse { + // Response body structure. + message Body {} + + Body body = 1; + Signature signature = 2; +} diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index b9b865a90..979c22685 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -1391,3 +1391,519 @@ func (x *DoctorResponse) ReadSignedData(buf []byte) ([]byte, error) { func (x *DoctorResponse) SetSignature(sig *Signature) { x.Signature = sig } + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StartShardEvacuationRequest_Body) StableSize() (size int) { + size += proto.RepeatedBytesSize(1, x.Shard_ID) + size += proto.BoolSize(2, x.IgnoreErrors) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *StartShardEvacuationRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) + offset += proto.BoolMarshal(2, buf[offset:], x.IgnoreErrors) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StartShardEvacuationRequest) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *StartShardEvacuationRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *StartShardEvacuationRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *StartShardEvacuationRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *StartShardEvacuationRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StartShardEvacuationResponse_Body) StableSize() (size int) { + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *StartShardEvacuationResponse_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StartShardEvacuationResponse) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *StartShardEvacuationResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *StartShardEvacuationResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *StartShardEvacuationResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *StartShardEvacuationResponse) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetShardEvacuationStatusRequest_Body) StableSize() (size int) { + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetShardEvacuationStatusRequest_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetShardEvacuationStatusRequest) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetShardEvacuationStatusRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *GetShardEvacuationStatusRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *GetShardEvacuationStatusRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *GetShardEvacuationStatusRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) StableSize() (size int) { + size += proto.Int64Size(1, x.Value) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.Int64Marshal(1, buf[offset:], x.Value) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetShardEvacuationStatusResponse_Body_Duration) StableSize() (size int) { + size += proto.Int64Size(1, x.Seconds) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetShardEvacuationStatusResponse_Body_Duration) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.Int64Marshal(1, buf[offset:], x.Seconds) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetShardEvacuationStatusResponse_Body) StableSize() (size int) { + size += proto.UInt64Size(1, x.Total) + size += proto.UInt64Size(2, x.Evacuated) + size += proto.UInt64Size(3, x.Failed) + size += proto.RepeatedBytesSize(4, x.Shard_ID) + size += proto.EnumSize(5, int32(x.Status)) + size += proto.NestedStructureSize(6, x.Duration) + size += proto.NestedStructureSize(7, x.StartedAt) + size += proto.StringSize(8, x.ErrorMessage) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetShardEvacuationStatusResponse_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.UInt64Marshal(1, buf[offset:], x.Total) + offset += proto.UInt64Marshal(2, buf[offset:], x.Evacuated) + offset += proto.UInt64Marshal(3, buf[offset:], x.Failed) + offset += proto.RepeatedBytesMarshal(4, buf[offset:], x.Shard_ID) + offset += proto.EnumMarshal(5, buf[offset:], int32(x.Status)) + offset += proto.NestedStructureMarshal(6, buf[offset:], x.Duration) + offset += proto.NestedStructureMarshal(7, buf[offset:], x.StartedAt) + offset += proto.StringMarshal(8, buf[offset:], x.ErrorMessage) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetShardEvacuationStatusResponse) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetShardEvacuationStatusResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *GetShardEvacuationStatusResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *GetShardEvacuationStatusResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *GetShardEvacuationStatusResponse) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StopShardEvacuationRequest_Body) StableSize() (size int) { + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *StopShardEvacuationRequest_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StopShardEvacuationRequest) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *StopShardEvacuationRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *StopShardEvacuationRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *StopShardEvacuationRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *StopShardEvacuationRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StopShardEvacuationResponse_Body) StableSize() (size int) { + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *StopShardEvacuationResponse_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StopShardEvacuationResponse) StableSize() (size int) { + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *StopShardEvacuationResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *StopShardEvacuationResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *StopShardEvacuationResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *StopShardEvacuationResponse) SetSignature(sig *Signature) { + x.Signature = sig +} diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 3fa1e54de..966421533 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.3.0 +// - protoc v3.21.9 // source: pkg/services/control/service.proto package control @@ -18,6 +18,21 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + ControlService_HealthCheck_FullMethodName = "/control.ControlService/HealthCheck" + ControlService_SetNetmapStatus_FullMethodName = "/control.ControlService/SetNetmapStatus" + ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects" + ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" + ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" + ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree" + ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard" + ControlService_StartShardEvacuation_FullMethodName = "/control.ControlService/StartShardEvacuation" + ControlService_GetShardEvacuationStatus_FullMethodName = "/control.ControlService/GetShardEvacuationStatus" + ControlService_StopShardEvacuation_FullMethodName = "/control.ControlService/StopShardEvacuation" + ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache" + ControlService_Doctor_FullMethodName = "/control.ControlService/Doctor" +) + // ControlServiceClient is the client API for ControlService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -36,6 +51,12 @@ type ControlServiceClient interface { SynchronizeTree(ctx context.Context, in *SynchronizeTreeRequest, opts ...grpc.CallOption) (*SynchronizeTreeResponse, error) // EvacuateShard moves all data from one shard to the others. EvacuateShard(ctx context.Context, in *EvacuateShardRequest, opts ...grpc.CallOption) (*EvacuateShardResponse, error) + // StartShardEvacuation starts moving all data from one shard to the others. + StartShardEvacuation(ctx context.Context, in *StartShardEvacuationRequest, opts ...grpc.CallOption) (*StartShardEvacuationResponse, error) + // GetShardEvacuationStatus returns evacuation status. + GetShardEvacuationStatus(ctx context.Context, in *GetShardEvacuationStatusRequest, opts ...grpc.CallOption) (*GetShardEvacuationStatusResponse, error) + // StopShardEvacuation stops moving all data from one shard to the others. + StopShardEvacuation(ctx context.Context, in *StopShardEvacuationRequest, opts ...grpc.CallOption) (*StopShardEvacuationResponse, error) // FlushCache moves all data from one shard to the others. FlushCache(ctx context.Context, in *FlushCacheRequest, opts ...grpc.CallOption) (*FlushCacheResponse, error) // Doctor performs storage restructuring operations on engine. @@ -52,7 +73,7 @@ func NewControlServiceClient(cc grpc.ClientConnInterface) ControlServiceClient { func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { out := new(HealthCheckResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/HealthCheck", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_HealthCheck_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -61,7 +82,7 @@ func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckR func (c *controlServiceClient) SetNetmapStatus(ctx context.Context, in *SetNetmapStatusRequest, opts ...grpc.CallOption) (*SetNetmapStatusResponse, error) { out := new(SetNetmapStatusResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/SetNetmapStatus", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_SetNetmapStatus_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -70,7 +91,7 @@ func (c *controlServiceClient) SetNetmapStatus(ctx context.Context, in *SetNetma func (c *controlServiceClient) DropObjects(ctx context.Context, in *DropObjectsRequest, opts ...grpc.CallOption) (*DropObjectsResponse, error) { out := new(DropObjectsResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/DropObjects", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_DropObjects_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -79,7 +100,7 @@ func (c *controlServiceClient) DropObjects(ctx context.Context, in *DropObjectsR func (c *controlServiceClient) ListShards(ctx context.Context, in *ListShardsRequest, opts ...grpc.CallOption) (*ListShardsResponse, error) { out := new(ListShardsResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/ListShards", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_ListShards_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -88,7 +109,7 @@ func (c *controlServiceClient) ListShards(ctx context.Context, in *ListShardsReq func (c *controlServiceClient) SetShardMode(ctx context.Context, in *SetShardModeRequest, opts ...grpc.CallOption) (*SetShardModeResponse, error) { out := new(SetShardModeResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/SetShardMode", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_SetShardMode_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -97,7 +118,7 @@ func (c *controlServiceClient) SetShardMode(ctx context.Context, in *SetShardMod func (c *controlServiceClient) SynchronizeTree(ctx context.Context, in *SynchronizeTreeRequest, opts ...grpc.CallOption) (*SynchronizeTreeResponse, error) { out := new(SynchronizeTreeResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/SynchronizeTree", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_SynchronizeTree_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -106,7 +127,34 @@ func (c *controlServiceClient) SynchronizeTree(ctx context.Context, in *Synchron func (c *controlServiceClient) EvacuateShard(ctx context.Context, in *EvacuateShardRequest, opts ...grpc.CallOption) (*EvacuateShardResponse, error) { out := new(EvacuateShardResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/EvacuateShard", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_EvacuateShard_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlServiceClient) StartShardEvacuation(ctx context.Context, in *StartShardEvacuationRequest, opts ...grpc.CallOption) (*StartShardEvacuationResponse, error) { + out := new(StartShardEvacuationResponse) + err := c.cc.Invoke(ctx, ControlService_StartShardEvacuation_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlServiceClient) GetShardEvacuationStatus(ctx context.Context, in *GetShardEvacuationStatusRequest, opts ...grpc.CallOption) (*GetShardEvacuationStatusResponse, error) { + out := new(GetShardEvacuationStatusResponse) + err := c.cc.Invoke(ctx, ControlService_GetShardEvacuationStatus_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlServiceClient) StopShardEvacuation(ctx context.Context, in *StopShardEvacuationRequest, opts ...grpc.CallOption) (*StopShardEvacuationResponse, error) { + out := new(StopShardEvacuationResponse) + err := c.cc.Invoke(ctx, ControlService_StopShardEvacuation_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -115,7 +163,7 @@ func (c *controlServiceClient) EvacuateShard(ctx context.Context, in *EvacuateSh func (c *controlServiceClient) FlushCache(ctx context.Context, in *FlushCacheRequest, opts ...grpc.CallOption) (*FlushCacheResponse, error) { out := new(FlushCacheResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/FlushCache", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_FlushCache_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -124,7 +172,7 @@ func (c *controlServiceClient) FlushCache(ctx context.Context, in *FlushCacheReq func (c *controlServiceClient) Doctor(ctx context.Context, in *DoctorRequest, opts ...grpc.CallOption) (*DoctorResponse, error) { out := new(DoctorResponse) - err := c.cc.Invoke(ctx, "/control.ControlService/Doctor", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_Doctor_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -149,6 +197,12 @@ type ControlServiceServer interface { SynchronizeTree(context.Context, *SynchronizeTreeRequest) (*SynchronizeTreeResponse, error) // EvacuateShard moves all data from one shard to the others. EvacuateShard(context.Context, *EvacuateShardRequest) (*EvacuateShardResponse, error) + // StartShardEvacuation starts moving all data from one shard to the others. + StartShardEvacuation(context.Context, *StartShardEvacuationRequest) (*StartShardEvacuationResponse, error) + // GetShardEvacuationStatus returns evacuation status. + GetShardEvacuationStatus(context.Context, *GetShardEvacuationStatusRequest) (*GetShardEvacuationStatusResponse, error) + // StopShardEvacuation stops moving all data from one shard to the others. + StopShardEvacuation(context.Context, *StopShardEvacuationRequest) (*StopShardEvacuationResponse, error) // FlushCache moves all data from one shard to the others. FlushCache(context.Context, *FlushCacheRequest) (*FlushCacheResponse, error) // Doctor performs storage restructuring operations on engine. @@ -180,6 +234,15 @@ func (UnimplementedControlServiceServer) SynchronizeTree(context.Context, *Synch func (UnimplementedControlServiceServer) EvacuateShard(context.Context, *EvacuateShardRequest) (*EvacuateShardResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method EvacuateShard not implemented") } +func (UnimplementedControlServiceServer) StartShardEvacuation(context.Context, *StartShardEvacuationRequest) (*StartShardEvacuationResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method StartShardEvacuation not implemented") +} +func (UnimplementedControlServiceServer) GetShardEvacuationStatus(context.Context, *GetShardEvacuationStatusRequest) (*GetShardEvacuationStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetShardEvacuationStatus not implemented") +} +func (UnimplementedControlServiceServer) StopShardEvacuation(context.Context, *StopShardEvacuationRequest) (*StopShardEvacuationResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method StopShardEvacuation not implemented") +} func (UnimplementedControlServiceServer) FlushCache(context.Context, *FlushCacheRequest) (*FlushCacheResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method FlushCache not implemented") } @@ -208,7 +271,7 @@ func _ControlService_HealthCheck_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/HealthCheck", + FullMethod: ControlService_HealthCheck_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).HealthCheck(ctx, req.(*HealthCheckRequest)) @@ -226,7 +289,7 @@ func _ControlService_SetNetmapStatus_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/SetNetmapStatus", + FullMethod: ControlService_SetNetmapStatus_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).SetNetmapStatus(ctx, req.(*SetNetmapStatusRequest)) @@ -244,7 +307,7 @@ func _ControlService_DropObjects_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/DropObjects", + FullMethod: ControlService_DropObjects_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).DropObjects(ctx, req.(*DropObjectsRequest)) @@ -262,7 +325,7 @@ func _ControlService_ListShards_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/ListShards", + FullMethod: ControlService_ListShards_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).ListShards(ctx, req.(*ListShardsRequest)) @@ -280,7 +343,7 @@ func _ControlService_SetShardMode_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/SetShardMode", + FullMethod: ControlService_SetShardMode_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).SetShardMode(ctx, req.(*SetShardModeRequest)) @@ -298,7 +361,7 @@ func _ControlService_SynchronizeTree_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/SynchronizeTree", + FullMethod: ControlService_SynchronizeTree_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).SynchronizeTree(ctx, req.(*SynchronizeTreeRequest)) @@ -316,7 +379,7 @@ func _ControlService_EvacuateShard_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/EvacuateShard", + FullMethod: ControlService_EvacuateShard_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).EvacuateShard(ctx, req.(*EvacuateShardRequest)) @@ -324,6 +387,60 @@ func _ControlService_EvacuateShard_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _ControlService_StartShardEvacuation_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StartShardEvacuationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).StartShardEvacuation(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_StartShardEvacuation_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).StartShardEvacuation(ctx, req.(*StartShardEvacuationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ControlService_GetShardEvacuationStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetShardEvacuationStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).GetShardEvacuationStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_GetShardEvacuationStatus_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).GetShardEvacuationStatus(ctx, req.(*GetShardEvacuationStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ControlService_StopShardEvacuation_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StopShardEvacuationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).StopShardEvacuation(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_StopShardEvacuation_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).StopShardEvacuation(ctx, req.(*StopShardEvacuationRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _ControlService_FlushCache_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(FlushCacheRequest) if err := dec(in); err != nil { @@ -334,7 +451,7 @@ func _ControlService_FlushCache_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/FlushCache", + FullMethod: ControlService_FlushCache_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).FlushCache(ctx, req.(*FlushCacheRequest)) @@ -352,7 +469,7 @@ func _ControlService_Doctor_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/control.ControlService/Doctor", + FullMethod: ControlService_Doctor_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).Doctor(ctx, req.(*DoctorRequest)) @@ -395,6 +512,18 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "EvacuateShard", Handler: _ControlService_EvacuateShard_Handler, }, + { + MethodName: "StartShardEvacuation", + Handler: _ControlService_StartShardEvacuation_Handler, + }, + { + MethodName: "GetShardEvacuationStatus", + Handler: _ControlService_GetShardEvacuationStatus_Handler, + }, + { + MethodName: "StopShardEvacuation", + Handler: _ControlService_StopShardEvacuation_Handler, + }, { MethodName: "FlushCache", Handler: _ControlService_FlushCache_Handler, diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index 9d6864759..d2ee50770 100644 --- a/pkg/services/control/types.pb.go +++ b/pkg/services/control/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.12.4 +// protoc v3.21.9 // source: pkg/services/control/types.proto package control From e4889e06ba07a32241c4eb4cd723f8aec189eb23 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 4 May 2023 13:58:26 +0300 Subject: [PATCH 0254/1943] [#329] node: Make evacuate async Now it's possible to run evacuate shard in async. Also only one evacuate process can be in progress. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 4 + pkg/local_object_storage/engine/engine.go | 4 + pkg/local_object_storage/engine/evacuate.go | 194 ++++++++++++++++-- .../engine/evacuate_limiter.go | 178 ++++++++++++++++ .../engine/evacuate_test.go | 139 +++++++++++-- pkg/local_object_storage/shard/count.go | 31 +++ pkg/services/control/server/convert.go | 60 ++++++ pkg/services/control/server/evacuate.go | 2 +- pkg/services/control/server/evacuate_async.go | 91 +++++++- pkg/services/control/service.proto | 1 + pkg/services/control/service_grpc.pb.go | 2 + 11 files changed, 669 insertions(+), 37 deletions(-) create mode 100644 pkg/local_object_storage/engine/evacuate_limiter.go create mode 100644 pkg/local_object_storage/shard/count.go create mode 100644 pkg/services/control/server/convert.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 742f6a8f7..b84746a72 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -484,5 +484,9 @@ const ( ShardGCCollectingExpiredLocksCompleted = "collecting expired locks completed" ShardGCRemoveGarbageStarted = "garbage remove started" ShardGCRemoveGarbageCompleted = "garbage remove completed" + EngineShardsEvacuationFailedToCount = "failed to get total objects count to evacuate" + EngineShardsEvacuationFailedToListObjects = "failed to list objects to evacuate" + EngineShardsEvacuationFailedToReadObject = "failed to read object to evacuate" + EngineShardsEvacuationFailedToMoveObject = "failed to evacuate object to other node" ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" ) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 20c8a946b..b7be4756d 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -35,6 +35,7 @@ type StorageEngine struct { err error } + evacuateLimiter *evacuationLimiter } type shardWrapper struct { @@ -230,6 +231,9 @@ func New(opts ...Option) *StorageEngine { shardPools: make(map[string]util.WorkerPool), closeCh: make(chan struct{}), setModeCh: make(chan setModeRequest), + evacuateLimiter: &evacuationLimiter{ + guard: &sync.RWMutex{}, + }, } } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 761ed24b9..4693b2618 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -14,6 +15,9 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/atomic" "go.uber.org/zap" ) @@ -24,11 +28,23 @@ type EvacuateShardPrm struct { shardID []*shard.ID handler func(context.Context, oid.Address, *objectSDK.Object) error ignoreErrors bool + async bool } // EvacuateShardRes represents result of the EvacuateShard operation. type EvacuateShardRes struct { - count int + evacuated *atomic.Uint64 + total *atomic.Uint64 + failed *atomic.Uint64 +} + +// NewEvacuateShardRes creates new EvacuateShardRes instance. +func NewEvacuateShardRes() *EvacuateShardRes { + return &EvacuateShardRes{ + evacuated: atomic.NewUint64(0), + total: atomic.NewUint64(0), + failed: atomic.NewUint64(0), + } } // WithShardIDList sets shard ID. @@ -46,10 +62,46 @@ func (p *EvacuateShardPrm) WithFaultHandler(f func(context.Context, oid.Address, p.handler = f } -// Count returns amount of evacuated objects. +// WithAsync sets flag to run evacuate async. +func (p *EvacuateShardPrm) WithAsync(async bool) { + p.async = async +} + +// Evacuated returns amount of evacuated objects. // Objects for which handler returned no error are also assumed evacuated. -func (p EvacuateShardRes) Count() int { - return p.count +func (p *EvacuateShardRes) Evacuated() uint64 { + if p == nil { + return 0 + } + return p.evacuated.Load() +} + +// Total returns total count objects to evacuate. +func (p *EvacuateShardRes) Total() uint64 { + if p == nil { + return 0 + } + return p.total.Load() +} + +// Failed returns count of failed objects to evacuate. +func (p *EvacuateShardRes) Failed() uint64 { + if p == nil { + return 0 + } + return p.failed.Load() +} + +// DeepCopy returns deep copy of result instance. +func (p *EvacuateShardRes) DeepCopy() *EvacuateShardRes { + if p == nil { + return nil + } + return &EvacuateShardRes{ + evacuated: atomic.NewUint64(p.evacuated.Load()), + total: atomic.NewUint64(p.total.Load()), + failed: atomic.NewUint64(p.failed.Load()), + } } const defaultEvacuateBatchSize = 100 @@ -63,15 +115,29 @@ var errMustHaveTwoShards = errors.New("must have at least 1 spare shard") // Evacuate moves data from one shard to the others. // The shard being moved must be in read-only mode. -func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (EvacuateShardRes, error) { +func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*EvacuateShardRes, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + shardIDs := make([]string, len(prm.shardID)) for i := range prm.shardID { shardIDs[i] = prm.shardID[i].String() } + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Evacuate", + trace.WithAttributes( + attribute.StringSlice("shardIDs", shardIDs), + attribute.Bool("async", prm.async), + attribute.Bool("ignoreErrors", prm.ignoreErrors), + )) + defer span.End() + shards, weights, err := e.getActualShards(shardIDs, prm.handler != nil) if err != nil { - return EvacuateShardRes{}, err + return nil, err } shardsToEvacuate := make(map[string]*shard.Shard) @@ -83,23 +149,91 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (Eva } } + res := NewEvacuateShardRes() + ctx = ctxOrBackground(ctx, prm.async) + eg, egCtx, err := e.evacuateLimiter.TryStart(ctx, shardIDs, res) + + if err != nil { + return nil, err + } + + eg.Go(func() error { + return e.evacuateShards(egCtx, shardIDs, prm, res, shards, weights, shardsToEvacuate) + }) + + if prm.async { + return nil, nil + } + + return res, eg.Wait() +} + +func ctxOrBackground(ctx context.Context, background bool) context.Context { + if background { + return context.Background() + } + return ctx +} + +func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, prm EvacuateShardPrm, res *EvacuateShardRes, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + var err error + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShards", + trace.WithAttributes( + attribute.StringSlice("shardIDs", shardIDs), + attribute.Bool("async", prm.async), + attribute.Bool("ignoreErrors", prm.ignoreErrors), + )) + + defer func() { + span.End() + e.evacuateLimiter.Complete(err) + }() + e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs)) - var res EvacuateShardRes + err = e.getTotalObjectsCount(ctx, shardsToEvacuate, res) + if err != nil { + e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err)) + return err + } for _, shardID := range shardIDs { - if err = e.evacuateShard(ctx, shardID, prm, &res, shards, weights, shardsToEvacuate); err != nil { + if err = e.evacuateShard(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs)) - return res, err + return err } } e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, zap.Strings("shard_ids", shardIDs)) - return res, nil + return nil +} + +func (e *StorageEngine) getTotalObjectsCount(ctx context.Context, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.getTotalObjectsCount") + defer span.End() + + for _, sh := range shardsToEvacuate { + cnt, err := sh.LogicalObjectsCount(ctx) + if err != nil { + if errors.Is(err, shard.ErrDegradedMode) { + continue + } + return err + } + res.total.Add(cnt) + } + return nil } func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShard", + trace.WithAttributes( + attribute.String("shardID", shardID), + )) + defer span.End() + var listPrm shard.ListWithCursorPrm listPrm.WithCount(defaultEvacuateBatchSize) @@ -116,6 +250,7 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { break } + e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err)) return err } @@ -168,6 +303,12 @@ func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, toEvacuate []object.AddressWithType, prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateObjects", + trace.WithAttributes( + attribute.Int("objects_count", len(toEvacuate)), + )) + defer span.End() + for i := range toEvacuate { select { case <-ctx.Done(): @@ -182,12 +323,14 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to getRes, err := sh.Get(ctx, getPrm) if err != nil { if prm.ignoreErrors { + res.failed.Inc() continue } + e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) return err } - evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), sh, res, shards, weights, shardsToEvacuate) + evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), sh, shards, weights, shardsToEvacuate, res) if err != nil { return err } @@ -204,15 +347,16 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to err = prm.handler(ctx, addr, getRes.Object()) if err != nil { + e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) return err } - res.count++ + res.evacuated.Inc() } return nil } -func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) (bool, error) { +func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) (bool, error) { hrw.SortHasherSliceByWeightValue(shards, weights, hrw.Hash([]byte(addr.EncodeToString()))) for j := range shards { select { @@ -227,11 +371,11 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add putDone, exists := e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object) if putDone || exists { if putDone { + res.evacuated.Inc() e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), zap.Stringer("addr", addr)) - res.count++ } return true, nil } @@ -239,3 +383,23 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add return false, nil } + +func (e *StorageEngine) GetEvacuationState(ctx context.Context) (*EvacuationState, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + return e.evacuateLimiter.GetState(), nil +} + +func (e *StorageEngine) EnqueRunningEvacuationStop(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + return e.evacuateLimiter.CancelIfRunning() +} diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go new file mode 100644 index 000000000..425fdc775 --- /dev/null +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -0,0 +1,178 @@ +package engine + +import ( + "context" + "fmt" + "sync" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "golang.org/x/sync/errgroup" +) + +type EvacuateProcessState int + +const ( + EvacuateProcessStateUndefined EvacuateProcessState = iota + EvacuateProcessStateRunning + EvacuateProcessStateCompleted +) + +type EvacuationState struct { + shardIDs []string + processState EvacuateProcessState + startedAt time.Time + finishedAt time.Time + result *EvacuateShardRes + errMessage string +} + +func (s *EvacuationState) ShardIDs() []string { + if s == nil { + return nil + } + return s.shardIDs +} + +func (s *EvacuationState) Evacuated() uint64 { + if s == nil { + return 0 + } + return s.result.Evacuated() +} + +func (s *EvacuationState) Total() uint64 { + if s == nil { + return 0 + } + return s.result.Total() +} + +func (s *EvacuationState) Failed() uint64 { + if s == nil { + return 0 + } + return s.result.Failed() +} + +func (s *EvacuationState) ProcessingStatus() EvacuateProcessState { + if s == nil { + return EvacuateProcessStateUndefined + } + return s.processState +} + +func (s *EvacuationState) StartedAt() *time.Time { + if s == nil { + return nil + } + defaultTime := time.Time{} + if s.startedAt == defaultTime { + return nil + } + return &s.startedAt +} + +func (s *EvacuationState) FinishedAt() *time.Time { + if s == nil { + return nil + } + defaultTime := time.Time{} + if s.finishedAt == defaultTime { + return nil + } + return &s.finishedAt +} + +func (s *EvacuationState) ErrorMessage() string { + if s == nil { + return "" + } + return s.errMessage +} + +func (s *EvacuationState) DeepCopy() *EvacuationState { + if s == nil { + return nil + } + shardIDs := make([]string, len(s.shardIDs)) + copy(shardIDs, s.shardIDs) + + return &EvacuationState{ + shardIDs: shardIDs, + processState: s.processState, + startedAt: s.startedAt, + finishedAt: s.finishedAt, + errMessage: s.errMessage, + result: s.result.DeepCopy(), + } +} + +type evacuationLimiter struct { + state EvacuationState + eg *errgroup.Group + cancel context.CancelFunc + + guard *sync.RWMutex +} + +func (l *evacuationLimiter) TryStart(ctx context.Context, shardIDs []string, result *EvacuateShardRes) (*errgroup.Group, context.Context, error) { + l.guard.Lock() + defer l.guard.Unlock() + + select { + case <-ctx.Done(): + return nil, nil, ctx.Err() + default: + } + + if l.state.processState == EvacuateProcessStateRunning { + return nil, nil, logicerr.New(fmt.Sprintf("evacuate is already running for shard ids %v", l.state.shardIDs)) + } + + var egCtx context.Context + egCtx, l.cancel = context.WithCancel(ctx) + l.eg, egCtx = errgroup.WithContext(egCtx) + l.state = EvacuationState{ + shardIDs: shardIDs, + processState: EvacuateProcessStateRunning, + startedAt: time.Now().UTC(), + result: result, + } + + return l.eg, egCtx, nil +} + +func (l *evacuationLimiter) Complete(err error) { + l.guard.Lock() + defer l.guard.Unlock() + + errMsq := "" + if err != nil { + errMsq = err.Error() + } + l.state.processState = EvacuateProcessStateCompleted + l.state.errMessage = errMsq + l.state.finishedAt = time.Now().UTC() + + l.eg = nil +} + +func (l *evacuationLimiter) GetState() *EvacuationState { + l.guard.RLock() + defer l.guard.RUnlock() + + return l.state.DeepCopy() +} + +func (l *evacuationLimiter) CancelIfRunning() error { + l.guard.Lock() + defer l.guard.Unlock() + + if l.state.processState != EvacuateProcessStateRunning { + return logicerr.New("there is no running evacuation task") + } + + l.cancel() + return nil +} diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index bc5b05ef0..43737e7f7 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strconv" "testing" + "time" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -21,6 +22,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" + "golang.org/x/sync/errgroup" ) func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEngine, []*shard.ID, []*objectSDK.Object) { @@ -103,14 +105,14 @@ func TestEvacuateShard(t *testing.T) { t.Run("must be read-only", func(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, ErrMustBeReadOnly) - require.Equal(t, 0, res.Count()) + require.Equal(t, uint64(0), res.Evacuated()) }) require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly)) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, objPerShard, res.count) + require.Equal(t, uint64(objPerShard), res.Evacuated()) // We check that all objects are available both before and after shard removal. // First case is a real-world use-case. It ensures that an object can be put in presense @@ -121,7 +123,7 @@ func TestEvacuateShard(t *testing.T) { // Calling it again is OK, but all objects are already moved, so no new PUTs should be done. res, err = e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, 0, res.count) + require.Equal(t, uint64(0), res.Evacuated()) checkHasObjects(t) @@ -138,8 +140,8 @@ func TestEvacuateNetwork(t *testing.T) { var errReplication = errors.New("handler error") - acceptOneOf := func(objects []*objectSDK.Object, max int) func(context.Context, oid.Address, *objectSDK.Object) error { - var n int + acceptOneOf := func(objects []*objectSDK.Object, max uint64) func(context.Context, oid.Address, *objectSDK.Object) error { + var n uint64 return func(_ context.Context, addr oid.Address, obj *objectSDK.Object) error { if n == max { return errReplication @@ -169,13 +171,13 @@ func TestEvacuateNetwork(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errMustHaveTwoShards) - require.Equal(t, 0, res.Count()) + require.Equal(t, uint64(0), res.Evacuated()) prm.handler = acceptOneOf(objects, 2) res, err = e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, 2, res.Count()) + require.Equal(t, uint64(2), res.Evacuated()) }) t.Run("multiple shards, evacuate one", func(t *testing.T) { t.Parallel() @@ -190,14 +192,14 @@ func TestEvacuateNetwork(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, 2, res.Count()) + require.Equal(t, uint64(2), res.Evacuated()) t.Run("no errors", func(t *testing.T) { prm.handler = acceptOneOf(objects, 3) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, 3, res.Count()) + require.Equal(t, uint64(3), res.Evacuated()) }) }) t.Run("multiple shards, evacuate many", func(t *testing.T) { @@ -205,12 +207,12 @@ func TestEvacuateNetwork(t *testing.T) { e, ids, objects := newEngineEvacuate(t, 4, 5) evacuateIDs := ids[0:3] - var totalCount int + var totalCount uint64 for i := range evacuateIDs { res, err := e.shards[ids[i].String()].List() require.NoError(t, err) - totalCount += len(res.AddressList()) + totalCount += uint64(len(res.AddressList())) } for i := range ids { @@ -223,14 +225,14 @@ func TestEvacuateNetwork(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, totalCount-1, res.Count()) + require.Equal(t, totalCount-1, res.Evacuated()) t.Run("no errors", func(t *testing.T) { prm.handler = acceptOneOf(objects, totalCount) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, totalCount, res.Count()) + require.Equal(t, totalCount, res.Evacuated()) }) }) } @@ -258,5 +260,114 @@ func TestEvacuateCancellation(t *testing.T) { res, err := e.Evacuate(ctx, prm) require.ErrorContains(t, err, "context canceled") - require.Equal(t, 0, res.Count()) + require.Equal(t, uint64(0), res.Evacuated()) +} + +func TestEvacuateSingleProcess(t *testing.T) { + e, ids, _ := newEngineEvacuate(t, 2, 3) + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + + blocker := make(chan interface{}) + running := make(chan interface{}) + + var prm EvacuateShardPrm + prm.shardID = ids[1:2] + prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + select { + case <-running: + default: + close(running) + } + <-blocker + return nil + } + + eg, egCtx := errgroup.WithContext(context.Background()) + eg.Go(func() error { + res, err := e.Evacuate(egCtx, prm) + require.NoError(t, err, "first evacuation failed") + require.Equal(t, uint64(3), res.Evacuated()) + return nil + }) + eg.Go(func() error { + <-running + res, err := e.Evacuate(egCtx, prm) + require.ErrorContains(t, err, "evacuate is already running for shard ids", "second evacuation not failed") + require.Equal(t, uint64(0), res.Evacuated()) + close(blocker) + return nil + }) + require.NoError(t, eg.Wait()) +} + +func TestEvacuateAsync(t *testing.T) { + e, ids, _ := newEngineEvacuate(t, 2, 3) + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + + blocker := make(chan interface{}) + running := make(chan interface{}) + + var prm EvacuateShardPrm + prm.shardID = ids[1:2] + prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + select { + case <-running: + default: + close(running) + } + <-blocker + return nil + } + + st, err := e.GetEvacuationState(context.Background()) + require.NoError(t, err, "get init state failed") + require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state") + require.Equal(t, uint64(0), st.Evacuated(), "invalid init count") + require.Nil(t, st.StartedAt(), "invalid init started at") + require.Nil(t, st.FinishedAt(), "invalid init finished at") + require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid init error message") + + eg, egCtx := errgroup.WithContext(context.Background()) + eg.Go(func() error { + res, err := e.Evacuate(egCtx, prm) + require.NoError(t, err, "first evacuation failed") + require.Equal(t, uint64(3), res.Evacuated()) + return nil + }) + + <-running + + st, err = e.GetEvacuationState(context.Background()) + require.NoError(t, err, "get running state failed") + require.Equal(t, EvacuateProcessStateRunning, st.ProcessingStatus(), "invalid running state") + require.Equal(t, uint64(0), st.Evacuated(), "invalid running count") + require.NotNil(t, st.StartedAt(), "invalid running started at") + require.Nil(t, st.FinishedAt(), "invalid init finished at") + expectedShardIDs := make([]string, 0, 2) + for _, id := range ids[1:2] { + expectedShardIDs = append(expectedShardIDs, id.String()) + } + require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid running shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid init error message") + + close(blocker) + + require.Eventually(t, func() bool { + st, err = e.GetEvacuationState(context.Background()) + return st.ProcessingStatus() == EvacuateProcessStateCompleted + }, 3*time.Second, 10*time.Millisecond, "invalid final state") + + require.NoError(t, err, "get final state failed") + require.Equal(t, uint64(3), st.Evacuated(), "invalid final count") + require.NotNil(t, st.StartedAt(), "invalid final started at") + require.NotNil(t, st.FinishedAt(), "invalid final finished at") + require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid final error message") + + require.NoError(t, eg.Wait()) } diff --git a/pkg/local_object_storage/shard/count.go b/pkg/local_object_storage/shard/count.go new file mode 100644 index 000000000..b68c2f43e --- /dev/null +++ b/pkg/local_object_storage/shard/count.go @@ -0,0 +1,31 @@ +package shard + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// LogicalObjectsCount returns logical objects count. +func (s *Shard) LogicalObjectsCount(ctx context.Context) (uint64, error) { + _, span := tracing.StartSpanFromContext(ctx, "Shard.LogicalObjectsCount", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + )) + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.GetMode().NoMetabase() { + return 0, ErrDegradedMode + } + + cc, err := s.metaBase.ObjectCounters() + if err != nil { + return 0, err + } + return cc.Logic(), nil +} diff --git a/pkg/services/control/server/convert.go b/pkg/services/control/server/convert.go new file mode 100644 index 000000000..1d29ed406 --- /dev/null +++ b/pkg/services/control/server/convert.go @@ -0,0 +1,60 @@ +package control + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "github.com/mr-tron/base58" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func stateToResponse(state *engine.EvacuationState) (*control.GetShardEvacuationStatusResponse, error) { + shardIDs := make([][]byte, 0, len(state.ShardIDs())) + for _, shID := range state.ShardIDs() { + id, err := base58.Decode(shID) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("invalid shard id format: %s", shID)) + } + shardIDs = append(shardIDs, id) + } + var evacStatus control.GetShardEvacuationStatusResponse_Body_Status + switch state.ProcessingStatus() { + case engine.EvacuateProcessStateRunning: + evacStatus = control.GetShardEvacuationStatusResponse_Body_RUNNING + case engine.EvacuateProcessStateCompleted: + evacStatus = control.GetShardEvacuationStatusResponse_Body_COMPLETED + default: + evacStatus = control.GetShardEvacuationStatusResponse_Body_EVACUATE_SHARD_STATUS_UNDEFINED + } + var startedAt *control.GetShardEvacuationStatusResponse_Body_UnixTimestamp + if state.StartedAt() != nil { + startedAt = &control.GetShardEvacuationStatusResponse_Body_UnixTimestamp{ + Value: state.StartedAt().Unix(), + } + } + var duration *control.GetShardEvacuationStatusResponse_Body_Duration + if state.StartedAt() != nil { + end := time.Now().UTC() + if state.FinishedAt() != nil { + end = *state.FinishedAt() + } + duration = &control.GetShardEvacuationStatusResponse_Body_Duration{ + Seconds: int64(end.Sub(*state.StartedAt()).Seconds()), + } + } + return &control.GetShardEvacuationStatusResponse{ + Body: &control.GetShardEvacuationStatusResponse_Body{ + Shard_ID: shardIDs, + Evacuated: state.Evacuated(), + Total: state.Total(), + Failed: state.Failed(), + Status: evacStatus, + StartedAt: startedAt, + Duration: duration, + ErrorMessage: state.ErrorMessage(), + }, + }, nil +} diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index afa4011b9..fc6dd3f60 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -37,7 +37,7 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe resp := &control.EvacuateShardResponse{ Body: &control.EvacuateShardResponse_Body{ - Count: uint32(res.Count()), + Count: uint32(res.Evacuated()), }, } diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index 94ddc73d1..cdf3656e2 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -2,19 +2,96 @@ package control import ( "context" - "fmt" + "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -func (s *Server) StartShardEvacuation(context.Context, *control.StartShardEvacuationRequest) (*control.StartShardEvacuationResponse, error) { - return nil, fmt.Errorf("not implemented") +func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartShardEvacuationRequest) (*control.StartShardEvacuationResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + var prm engine.EvacuateShardPrm + prm.WithShardIDList(s.getShardIDList(req.GetBody().GetShard_ID())) + prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) + prm.WithFaultHandler(s.replicate) + prm.WithAsync(true) + + _, err = s.s.Evacuate(ctx, prm) + if err != nil { + var logicalErr logicerr.Logical + if errors.As(err, &logicalErr) { + return nil, status.Error(codes.Aborted, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.StartShardEvacuationResponse{ + Body: &control.StartShardEvacuationResponse_Body{}, + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil } -func (s *Server) GetShardEvacuationStatus(context.Context, *control.GetShardEvacuationStatusRequest) (*control.GetShardEvacuationStatusResponse, error) { - return nil, fmt.Errorf("not implemented") +func (s *Server) GetShardEvacuationStatus(ctx context.Context, req *control.GetShardEvacuationStatusRequest) (*control.GetShardEvacuationStatusResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + state, err := s.s.GetEvacuationState(ctx) + if err != nil { + var logicalErr logicerr.Logical + if errors.As(err, &logicalErr) { + return nil, status.Error(codes.Aborted, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + resp, err := stateToResponse(state) + if err != nil { + return nil, err + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil } -func (s *Server) StopShardEvacuation(context.Context, *control.StopShardEvacuationRequest) (*control.StopShardEvacuationResponse, error) { - return nil, fmt.Errorf("not implemented") +func (s *Server) StopShardEvacuation(ctx context.Context, req *control.StopShardEvacuationRequest) (*control.StopShardEvacuationResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + err = s.s.EnqueRunningEvacuationStop(ctx) + if err != nil { + var logicalErr logicerr.Logical + if errors.As(err, &logicalErr) { + return nil, status.Error(codes.Aborted, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.StopShardEvacuationResponse{ + Body: &control.StopShardEvacuationResponse_Body{}, + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil } diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index ca35ae043..a80deb2da 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -27,6 +27,7 @@ service ControlService { rpc SynchronizeTree (SynchronizeTreeRequest) returns (SynchronizeTreeResponse); // EvacuateShard moves all data from one shard to the others. + // Deprecated: Use StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation rpc EvacuateShard (EvacuateShardRequest) returns (EvacuateShardResponse); // StartShardEvacuation starts moving all data from one shard to the others. diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 966421533..8afc6086a 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -50,6 +50,7 @@ type ControlServiceClient interface { // Synchronizes all log operations for the specified tree. SynchronizeTree(ctx context.Context, in *SynchronizeTreeRequest, opts ...grpc.CallOption) (*SynchronizeTreeResponse, error) // EvacuateShard moves all data from one shard to the others. + // Deprecated: Use StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation EvacuateShard(ctx context.Context, in *EvacuateShardRequest, opts ...grpc.CallOption) (*EvacuateShardResponse, error) // StartShardEvacuation starts moving all data from one shard to the others. StartShardEvacuation(ctx context.Context, in *StartShardEvacuationRequest, opts ...grpc.CallOption) (*StartShardEvacuationResponse, error) @@ -196,6 +197,7 @@ type ControlServiceServer interface { // Synchronizes all log operations for the specified tree. SynchronizeTree(context.Context, *SynchronizeTreeRequest) (*SynchronizeTreeResponse, error) // EvacuateShard moves all data from one shard to the others. + // Deprecated: Use StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation EvacuateShard(context.Context, *EvacuateShardRequest) (*EvacuateShardResponse, error) // StartShardEvacuation starts moving all data from one shard to the others. StartShardEvacuation(context.Context, *StartShardEvacuationRequest) (*StartShardEvacuationResponse, error) From f7c0b50d70f2d5795c8fe7194bfe6d1d1585659d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 5 May 2023 10:57:07 +0300 Subject: [PATCH 0255/1943] [#329] cli: Add async evacuate commands Add start, stop evacuate and evacuate status commands. Signed-off-by: Dmitrii Stepanov --- .../modules/control/evacuate_shard.go | 9 +- cmd/frostfs-cli/modules/control/evacuation.go | 296 ++++++++++++++++++ cmd/frostfs-cli/modules/control/shards.go | 3 +- pkg/services/control/rpc.go | 60 +++- 4 files changed, 354 insertions(+), 14 deletions(-) create mode 100644 cmd/frostfs-cli/modules/control/evacuation.go diff --git a/cmd/frostfs-cli/modules/control/evacuate_shard.go b/cmd/frostfs-cli/modules/control/evacuate_shard.go index b72ff6301..458e4cc0b 100644 --- a/cmd/frostfs-cli/modules/control/evacuate_shard.go +++ b/cmd/frostfs-cli/modules/control/evacuate_shard.go @@ -11,10 +11,11 @@ import ( const ignoreErrorsFlag = "no-errors" var evacuateShardCmd = &cobra.Command{ - Use: "evacuate", - Short: "Evacuate objects from shard", - Long: "Evacuate objects from shard to other shards", - Run: evacuateShard, + Use: "evacuate", + Short: "Evacuate objects from shard", + Long: "Evacuate objects from shard to other shards", + Run: evacuateShard, + Deprecated: "use frostfs-cli control shards evacuation start", } func evacuateShard(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go new file mode 100644 index 000000000..69d2dd8d4 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -0,0 +1,296 @@ +package control + +import ( + "crypto/ecdsa" + "fmt" + "strings" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + "github.com/spf13/cobra" + "go.uber.org/atomic" +) + +const ( + awaitFlag = "await" + noProgressFlag = "no-progress" +) + +var evacuationShardCmd = &cobra.Command{ + Use: "evacuation", + Short: "Objects evacuation from shard", + Long: "Objects evacuation from shard to other shards", +} + +var startEvacuationShardCmd = &cobra.Command{ + Use: "start", + Short: "Start evacuate objects from shard", + Long: "Start evacuate objects from shard to other shards", + Run: startEvacuateShard, +} + +var getEvacuationShardStatusCmd = &cobra.Command{ + Use: "status", + Short: "Get evacuate objects from shard status", + Long: "Get evacuate objects from shard to other shards status", + Run: getEvacuateShardStatus, +} + +var stopEvacuationShardCmd = &cobra.Command{ + Use: "stop", + Short: "Stop running evacuate process", + Long: "Stop running evacuate process from shard to other shards", + Run: stopEvacuateShardStatus, +} + +func startEvacuateShard(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag) + + req := &control.StartShardEvacuationRequest{ + Body: &control.StartShardEvacuationRequest_Body{ + Shard_ID: getShardIDList(cmd), + IgnoreErrors: ignoreErrors, + }, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.StartShardEvacuationResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.StartEvacuateShard(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "Start evacuate shards failed, rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Shard evacuation has been successfully started.") + + if awaitCompletion, _ := cmd.Flags().GetBool(awaitFlag); awaitCompletion { + noProgress, _ := cmd.Flags().GetBool(noProgressFlag) + waitEvacuateCompletion(cmd, pk, cli, !noProgress, true) + } +} + +func getEvacuateShardStatus(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + req := &control.GetShardEvacuationStatusRequest{ + Body: &control.GetShardEvacuationStatusRequest_Body{}, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.GetShardEvacuationStatusResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.GetEvacuateShardStatus(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "Get evacuate shards status failed, rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + printStatus(cmd, resp) +} + +func stopEvacuateShardStatus(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + req := &control.StopShardEvacuationRequest{ + Body: &control.StopShardEvacuationRequest_Body{}, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.StopShardEvacuationResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.StopEvacuateShard(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "Stop evacuate shards failed, rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + waitEvacuateCompletion(cmd, pk, cli, false, false) + + cmd.Println("Evacuation stopped.") +} + +func waitEvacuateCompletion(cmd *cobra.Command, pk *ecdsa.PrivateKey, cli *clientSDK.Client, printProgress, printCompleted bool) { + const statusPollingInterval = 1 * time.Second + const reportIntervalSeconds = 5 + var resp *control.GetShardEvacuationStatusResponse + reportResponse := atomic.NewPointer(resp) + pollingCompleted := make(chan struct{}) + progressReportCompleted := make(chan struct{}) + + go func() { + defer close(progressReportCompleted) + if !printProgress { + return + } + cmd.Printf("Progress will be reported every %d seconds.\n", reportIntervalSeconds) + for { + select { + case <-pollingCompleted: + return + case <-time.After(reportIntervalSeconds * time.Second): + r := reportResponse.Load() + if r == nil || r.GetBody().GetStatus() == control.GetShardEvacuationStatusResponse_Body_COMPLETED { + continue + } + printStatus(cmd, r) + } + } + }() + + for { + req := &control.GetShardEvacuationStatusRequest{ + Body: &control.GetShardEvacuationStatusRequest_Body{}, + } + signRequest(cmd, pk, req) + + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.GetEvacuateShardStatus(client, req) + return err + }) + + reportResponse.Store(resp) + + if err != nil { + commonCmd.ExitOnErr(cmd, "Failed to get evacuate status, rpc error: %w", err) + return + } + if resp.GetBody().GetStatus() != control.GetShardEvacuationStatusResponse_Body_RUNNING { + break + } + + time.Sleep(statusPollingInterval) + } + close(pollingCompleted) + <-progressReportCompleted + if printCompleted { + printCompletedStatusMessage(cmd, resp) + } +} + +func printCompletedStatusMessage(cmd *cobra.Command, resp *control.GetShardEvacuationStatusResponse) { + cmd.Println("Shard evacuation has been completed.") + sb := &strings.Builder{} + appendShardIDs(sb, resp) + appendCounts(sb, resp) + appendError(sb, resp) + appendStartedAt(sb, resp) + appendDuration(sb, resp) + cmd.Println(sb.String()) +} + +func printStatus(cmd *cobra.Command, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetStatus() == control.GetShardEvacuationStatusResponse_Body_EVACUATE_SHARD_STATUS_UNDEFINED { + cmd.Println("There is no running or completed evacuation.") + return + } + sb := &strings.Builder{} + appendShardIDs(sb, resp) + appendStatus(sb, resp) + appendCounts(sb, resp) + appendError(sb, resp) + appendStartedAt(sb, resp) + appendDuration(sb, resp) + cmd.Println(sb.String()) +} + +func appendDuration(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetDuration() != nil { + duration := time.Second * time.Duration(resp.GetBody().GetDuration().GetSeconds()) + hour := int(duration.Seconds() / 3600) + minute := int(duration.Seconds()/60) % 60 + second := int(duration.Seconds()) % 60 + sb.WriteString(fmt.Sprintf(" Duration: %02d:%02d:%02d.", hour, minute, second)) + } +} + +func appendStartedAt(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetStartedAt() != nil { + startedAt := time.Unix(resp.GetBody().GetStartedAt().GetValue(), 0).UTC() + sb.WriteString(fmt.Sprintf(" Started at: %s UTC.", startedAt.Format(time.RFC3339))) + } +} + +func appendError(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if len(resp.Body.GetErrorMessage()) > 0 { + sb.WriteString(fmt.Sprintf(" Error: %s.", resp.Body.GetErrorMessage())) + } +} + +func appendStatus(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + var status string + switch resp.GetBody().GetStatus() { + case control.GetShardEvacuationStatusResponse_Body_COMPLETED: + status = "completed" + case control.GetShardEvacuationStatusResponse_Body_RUNNING: + status = "running" + default: + status = "undefined" + } + sb.WriteString(fmt.Sprintf(" Status: %s.", status)) +} + +func appendShardIDs(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + sb.WriteString("Shard IDs: ") + for idx, shardID := range resp.GetBody().GetShard_ID() { + shardIDStr := shard.NewIDFromBytes(shardID).String() + if idx > 0 { + sb.WriteString(", ") + } + sb.WriteString(shardIDStr) + if idx == len(resp.GetBody().GetShard_ID())-1 { + sb.WriteString(".") + } + } +} + +func appendCounts(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + sb.WriteString(fmt.Sprintf(" Evacuated %d object out of %d, failed to evacuate %d objects.", + resp.GetBody().GetEvacuated(), + resp.Body.GetTotal(), + resp.Body.GetFailed())) +} + +func initControlEvacuationShardCmd() { + evacuationShardCmd.AddCommand(startEvacuationShardCmd) + evacuationShardCmd.AddCommand(getEvacuationShardStatusCmd) + evacuationShardCmd.AddCommand(stopEvacuationShardCmd) + + initControlStartEvacuationShardCmd() + initControlFlags(getEvacuationShardStatusCmd) + initControlFlags(stopEvacuationShardCmd) +} + +func initControlStartEvacuationShardCmd() { + initControlFlags(startEvacuationShardCmd) + + flags := startEvacuationShardCmd.Flags() + flags.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") + flags.Bool(shardAllFlag, false, "Process all shards") + flags.Bool(ignoreErrorsFlag, true, "Skip invalid/unreadable objects") + flags.Bool(awaitFlag, false, "Block execution until evacuation is completed") + flags.Bool(noProgressFlag, false, fmt.Sprintf("Print progress if %s provided", awaitFlag)) + + startEvacuationShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) +} diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index 8e7ecff8c..742109673 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -13,13 +13,14 @@ var shardsCmd = &cobra.Command{ func initControlShardsCmd() { shardsCmd.AddCommand(listShardsCmd) shardsCmd.AddCommand(setShardModeCmd) - shardsCmd.AddCommand(evacuateShardCmd) + shardsCmd.AddCommand(evacuationShardCmd) shardsCmd.AddCommand(flushCacheCmd) shardsCmd.AddCommand(doctorCmd) initControlShardsListCmd() initControlSetShardModeCmd() initControlEvacuateShardCmd() + initControlEvacuationShardCmd() initControlFlushCacheCmd() initControlDoctorCmd() } diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 31ebfa760..3822c9f70 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -8,15 +8,18 @@ import ( const serviceName = "control.ControlService" const ( - rpcHealthCheck = "HealthCheck" - rpcSetNetmapStatus = "SetNetmapStatus" - rpcDropObjects = "DropObjects" - rpcListShards = "ListShards" - rpcSetShardMode = "SetShardMode" - rpcSynchronizeTree = "SynchronizeTree" - rpcEvacuateShard = "EvacuateShard" - rpcFlushCache = "FlushCache" - rpcDoctor = "Doctor" + rpcHealthCheck = "HealthCheck" + rpcSetNetmapStatus = "SetNetmapStatus" + rpcDropObjects = "DropObjects" + rpcListShards = "ListShards" + rpcSetShardMode = "SetShardMode" + rpcSynchronizeTree = "SynchronizeTree" + rpcEvacuateShard = "EvacuateShard" + rpcStartEvacuateShard = "StartEvacuateShard" + rpcGetEvacuateShardStatus = "GetEvacuateShardStatus" + rpcStopEvacuateShardStatus = "StopEvacuateShard" + rpcFlushCache = "FlushCache" + rpcDoctor = "Doctor" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -141,6 +144,45 @@ func EvacuateShard(cli *client.Client, req *EvacuateShardRequest, opts ...client return wResp.message, nil } +// StartEvacuateShard executes ControlService.StartEvacuateShard RPC. +func StartEvacuateShard(cli *client.Client, req *StartShardEvacuationRequest, opts ...client.CallOption) (*StartShardEvacuationResponse, error) { + wResp := newResponseWrapper[StartShardEvacuationResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStartEvacuateShard), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + +// GetEvacuateShardStatus executes ControlService.GetEvacuateShardStatus RPC. +func GetEvacuateShardStatus(cli *client.Client, req *GetShardEvacuationStatusRequest, opts ...client.CallOption) (*GetShardEvacuationStatusResponse, error) { + wResp := newResponseWrapper[GetShardEvacuationStatusResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcGetEvacuateShardStatus), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + +// StopEvacuateShard executes ControlService.StopEvacuateShard RPC. +func StopEvacuateShard(cli *client.Client, req *StopShardEvacuationRequest, opts ...client.CallOption) (*StopShardEvacuationResponse, error) { + wResp := newResponseWrapper[StopShardEvacuationResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStopEvacuateShardStatus), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + // FlushCache executes ControlService.FlushCache RPC. func FlushCache(cli *client.Client, req *FlushCacheRequest, opts ...client.CallOption) (*FlushCacheResponse, error) { wResp := newResponseWrapper[FlushCacheResponse]() From 483fac03d65ba5b12e76e0d983bd887874fe1c00 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 10 May 2023 14:03:49 +0300 Subject: [PATCH 0256/1943] [#329] docs: Add shard evacuation description Signed-off-by: Dmitrii Stepanov --- docs/evacuation.md | 92 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/evacuation.md diff --git a/docs/evacuation.md b/docs/evacuation.md new file mode 100644 index 000000000..7c8beb790 --- /dev/null +++ b/docs/evacuation.md @@ -0,0 +1,92 @@ +# Shard data evacuation + +## Overview + +Evacuation is the process of transferring data from one shard to another. Evacuation is used in case of problems with the shard in order to save data. + +To start the evacuation, it is necessary that the shard is in read-only mode (read more [here](./shard-modes.md)). + +First of all, by the evacuation the data is transferred to other shards of the same node; if it is not possible, then the data is transferred to other nodes. + +Only one running evacuation process is allowed on the node at a time. + +`frostfs-cli` utility is used to manage evacuation. + +## Commands + +`frostfs-cli control shards evacuation start` starts evacuation process for shards specified. To start evacuating all node shards, use the `--all` flag. + +`frostfs-cli control shards evacuation stop` stops running evacuation process. + +`frostfs-cli control shards evacuation status` prints evacuation process status. + +See commands `--help` output for detailed description. + +## Examples + +### Set shard mode to read only +```bash +frostfs-cli control shards set-mode --mode read-only --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 8kEBwtvKLU3Hva3PaaodUi +Enter password > +Shard mode update request successfully sent. +``` + +### Start evacuation and get status +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 8kEBwtvKLU3Hva3PaaodUi +Enter password > +Shard evacuation has been successfully started. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 14 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:03. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 23 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:05. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: completed. Evacuated 61 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:13. +``` + +### Stop running evacuation process +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 54Y8aot9uc7BSadw2XtYr3 +Enter password > +Shard evacuation has been successfully started. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 15 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:03. + +frostfs-cli control shards evacuation stop --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Evacuation stopped. + +frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json +Enter password > +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: completed. Evacuated 31 object out of 73, failed to evacuate 0 objects. Error: context canceled. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:07. +``` + +### Start evacuation and await it completes +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 54Y8aot9uc7BSadw2XtYr3 --await +Enter password > +Shard evacuation has been successfully started. +Progress will be reported every 5 seconds. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 18 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:04. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 43 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:09. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 68 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. +Shard evacuation has been completed. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. +``` + +### Start evacuation and await it completes without progress notifications +```bash +frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 54Y8aot9uc7BSadw2XtYr3 --await --no-progress +Enter password > +Shard evacuation has been successfully started. +Shard evacuation has been completed. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:20:00Z UTC. Duration: 00:00:14. +``` \ No newline at end of file From 9e56592be3794ccdc8d3724cc42dcc5feacdfcc3 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 5 May 2023 18:40:57 +0300 Subject: [PATCH 0257/1943] [#314] writecache: Simplify background workers naming Also, drop not used arg. Signed-off-by: Pavel Karpy --- pkg/local_object_storage/writecache/flush.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 04fcccede..1e24a42ee 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -37,12 +37,12 @@ const ( func (c *cache) runFlushLoop() { for i := 0; i < c.workersCount; i++ { c.wg.Add(1) - go c.flushWorker(i) + go c.workerFlushSmall() } c.wg.Add(1) go func() { - c.flushBigObjects(context.TODO()) + c.workerFlushBig(context.TODO()) c.wg.Done() }() @@ -56,7 +56,7 @@ func (c *cache) runFlushLoop() { for { select { case <-tt.C: - c.flushDB() + c.flushSmallObjects() tt.Reset(defaultFlushInterval) case <-c.closeCh: return @@ -65,7 +65,7 @@ func (c *cache) runFlushLoop() { }() } -func (c *cache) flushDB() { +func (c *cache) flushSmallObjects() { var lastKey []byte var m []objectInfo for { @@ -148,7 +148,7 @@ func (c *cache) flushDB() { } } -func (c *cache) flushBigObjects(ctx context.Context) { +func (c *cache) workerFlushBig(ctx context.Context) { tick := time.NewTicker(defaultFlushInterval * 10) for { select { @@ -228,8 +228,8 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } -// flushWorker writes objects to the main storage. -func (c *cache) flushWorker(_ int) { +// workerFlushSmall writes small objects to the main storage. +func (c *cache) workerFlushSmall() { defer c.wg.Done() var obj *object.Object From bf79d06f039cd1e5110b725636667b57a50dc2c2 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Fri, 5 May 2023 18:59:49 +0300 Subject: [PATCH 0258/1943] [#314] writecache: Do not lose small objects on disk errors Do return error if an object could not been stored on WC's disk. Signed-off-by: Pavel Karpy --- pkg/local_object_storage/writecache/put.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index e2535d9e2..223a73bef 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -78,7 +78,7 @@ func (c *cache) putSmall(obj objectInfo) error { ) c.objCounters.IncDB() } - return nil + return err } // putBig writes object to FSTree and pushes it to the flush workers queue. From 35c9b6b26dcbb8fe1e03f69f0f8f10262db07e29 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 May 2023 17:43:49 +0300 Subject: [PATCH 0259/1943] [#314] writecache: remove objects right after they are flushed Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/writecache/flush.go | 28 +-- .../writecache/flush_test.go | 99 +-------- pkg/local_object_storage/writecache/get.go | 2 - pkg/local_object_storage/writecache/init.go | 192 ------------------ .../writecache/iterate.go | 6 - pkg/local_object_storage/writecache/mode.go | 16 -- pkg/local_object_storage/writecache/put.go | 2 - .../writecache/storage.go | 46 +---- .../writecache/writecache.go | 42 ++-- 9 files changed, 25 insertions(+), 408 deletions(-) delete mode 100644 pkg/local_object_storage/writecache/init.go diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 1e24a42ee..c6c8a9465 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -78,7 +78,7 @@ func (c *cache) flushSmallObjects() { m = m[:0] c.modeMtx.RLock() - if c.readOnly() || !c.initialized.Load() { + if c.readOnly() { c.modeMtx.RUnlock() time.Sleep(time.Second) continue @@ -117,10 +117,6 @@ func (c *cache) flushSmallObjects() { var count int for i := range m { - if c.flushed.Contains(m[i].addr) { - continue - } - obj := object.New() if err := obj.Unmarshal(m[i].data); err != nil { continue @@ -157,9 +153,6 @@ func (c *cache) workerFlushBig(ctx context.Context) { if c.readOnly() { c.modeMtx.RUnlock() break - } else if !c.initialized.Load() { - c.modeMtx.RUnlock() - continue } _ = c.flushFSTree(ctx, true) @@ -187,10 +180,6 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { prm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { sAddr := addr.EncodeToString() - if _, ok := c.store.flushed.Peek(sAddr); ok { - return nil - } - data, err := f() if err != nil { c.reportFlushError("can't read a file", sAddr, err) @@ -218,9 +207,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - // mark object as flushed - c.flushed.Add(sAddr, false) - + c.deleteFromDisk(ctx, []string{sAddr}) return nil } @@ -242,9 +229,12 @@ func (c *cache) workerFlushSmall() { } err := c.flushObject(context.TODO(), obj, nil) - if err == nil { - c.flushed.Add(objectCore.AddressOf(obj).EncodeToString(), true) + if err != nil { + // Error is handled in flushObject. + continue } + + c.deleteFromDB([]string{objectCore.AddressOf(obj).EncodeToString()}) } } @@ -306,10 +296,6 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { cs := b.Cursor() for k, data := cs.Seek(nil); k != nil; k, data = cs.Next() { sa := string(k) - if _, ok := c.flushed.Peek(sa); ok { - continue - } - if err := addr.DecodeString(sa); err != nil { c.reportFlushError("can't decode object address from the DB", sa, err) if ignoreErrors { diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 2cec07081..e5ca85735 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -5,7 +5,6 @@ import ( "os" "path/filepath" "testing" - "time" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -15,7 +14,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -109,22 +107,9 @@ func TestFlush(t *testing.T) { require.NoError(t, bs.SetMode(mode.ReadWrite)) require.NoError(t, mb.SetMode(mode.ReadWrite)) - wc.(*cache).flushed.Add(objects[0].addr.EncodeToString(), true) - wc.(*cache).flushed.Add(objects[1].addr.EncodeToString(), false) - require.NoError(t, wc.Flush(context.Background(), false)) - for i := 0; i < 2; i++ { - var mPrm meta.GetPrm - mPrm.SetAddress(objects[i].addr) - _, err := mb.Get(context.Background(), mPrm) - require.Error(t, err) - - _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) - require.Error(t, err) - } - - check(t, mb, bs, objects[2:]) + check(t, mb, bs, objects) }) t.Run("flush on moving to degraded mode", func(t *testing.T) { @@ -138,23 +123,9 @@ func TestFlush(t *testing.T) { require.NoError(t, wc.SetMode(mode.ReadOnly)) require.NoError(t, bs.SetMode(mode.ReadWrite)) require.NoError(t, mb.SetMode(mode.ReadWrite)) - - wc.(*cache).flushed.Add(objects[0].addr.EncodeToString(), true) - wc.(*cache).flushed.Add(objects[1].addr.EncodeToString(), false) - require.NoError(t, wc.SetMode(mode.Degraded)) - for i := 0; i < 2; i++ { - var mPrm meta.GetPrm - mPrm.SetAddress(objects[i].addr) - _, err := mb.Get(context.Background(), mPrm) - require.Error(t, err) - - _, err = bs.Get(context.Background(), common.GetPrm{Address: objects[i].addr}) - require.Error(t, err) - } - - check(t, mb, bs, objects[2:]) + check(t, mb, bs, objects) }) t.Run("ignore errors", func(t *testing.T) { @@ -223,67 +194,6 @@ func TestFlush(t *testing.T) { }) }) }) - - t.Run("on init", func(t *testing.T) { - wc, bs, mb := newCache(t) - objects := []objectPair{ - // removed - putObject(t, wc, 1), - putObject(t, wc, smallSize+1), - // not found - putObject(t, wc, 1), - putObject(t, wc, smallSize+1), - // ok - putObject(t, wc, 1), - putObject(t, wc, smallSize+1), - } - - require.NoError(t, wc.Close()) - require.NoError(t, bs.SetMode(mode.ReadWrite)) - require.NoError(t, mb.SetMode(mode.ReadWrite)) - - for i := range objects { - var prm meta.PutPrm - prm.SetObject(objects[i].obj) - _, err := mb.Put(context.Background(), prm) - require.NoError(t, err) - } - - var inhumePrm meta.InhumePrm - inhumePrm.SetAddresses(objects[0].addr, objects[1].addr) - inhumePrm.SetTombstoneAddress(oidtest.Address()) - _, err := mb.Inhume(context.Background(), inhumePrm) - require.NoError(t, err) - - var deletePrm meta.DeletePrm - deletePrm.SetAddresses(objects[2].addr, objects[3].addr) - _, err = mb.Delete(context.Background(), deletePrm) - require.NoError(t, err) - - require.NoError(t, bs.SetMode(mode.ReadOnly)) - require.NoError(t, mb.SetMode(mode.ReadOnly)) - - // Open in read-only: no error, nothing is removed. - require.NoError(t, wc.Open(true)) - initWC(t, wc) - for i := range objects { - _, err := wc.Get(context.Background(), objects[i].addr) - require.NoError(t, err, i) - } - require.NoError(t, wc.Close()) - - // Open in read-write: no error, something is removed. - require.NoError(t, wc.Open(false)) - initWC(t, wc) - for i := range objects { - _, err := wc.Get(context.Background(), objects[i].addr) - if i < 2 { - require.ErrorAs(t, err, new(apistatus.ObjectNotFound), i) - } else { - require.NoError(t, err, i) - } - } - }) } func putObject(t *testing.T, c Cache, size int) objectPair { @@ -321,11 +231,6 @@ func newObject(t *testing.T, size int) (*object.Object, []byte) { func initWC(t *testing.T, wc Cache) { require.NoError(t, wc.Init()) - - require.Eventually(t, func() bool { - rawWc := wc.(*cache) - return rawWc.initialized.Load() - }, 100*time.Second, 1*time.Millisecond) } type dummyEpoch struct{} diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index 6af1bd181..030f9b413 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -30,7 +30,6 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e value, err := Get(c.db, []byte(saddr)) if err == nil { obj := objectSDK.New() - c.flushed.Get(saddr) return obj, obj.Unmarshal(value) } @@ -39,7 +38,6 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) } - c.flushed.Get(saddr) return res.Object, nil } diff --git a/pkg/local_object_storage/writecache/init.go b/pkg/local_object_storage/writecache/init.go deleted file mode 100644 index 2ca8cceef..000000000 --- a/pkg/local_object_storage/writecache/init.go +++ /dev/null @@ -1,192 +0,0 @@ -package writecache - -import ( - "context" - "errors" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.etcd.io/bbolt" - "go.uber.org/zap" -) - -func (c *cache) initFlushMarks(ctx context.Context) { - var localWG sync.WaitGroup - - localWG.Add(1) - go func() { - defer localWG.Done() - - c.fsTreeFlushMarkUpdate(ctx) - }() - - localWG.Add(1) - go func() { - defer localWG.Done() - - c.dbFlushMarkUpdate(ctx) - }() - - c.initWG.Add(1) - c.wg.Add(1) - go func() { - defer c.wg.Done() - defer c.initWG.Done() - - localWG.Wait() - - select { - case <-c.stopInitCh: - return - case <-c.closeCh: - return - default: - } - - c.initialized.Store(true) - }() -} - -var errStopIter = errors.New("stop iteration") - -func (c *cache) fsTreeFlushMarkUpdate(ctx context.Context) { - c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInFSTree) - - var prm common.IteratePrm - prm.LazyHandler = func(addr oid.Address, _ func() ([]byte, error)) error { - select { - case <-c.closeCh: - return errStopIter - case <-c.stopInitCh: - return errStopIter - default: - } - - flushed, needRemove := c.flushStatus(ctx, addr) - if flushed { - c.store.flushed.Add(addr.EncodeToString(), true) - if needRemove { - var prm common.DeletePrm - prm.Address = addr - - _, err := c.fsTree.Delete(ctx, prm) - if err == nil { - storagelog.Write(c.log, - storagelog.AddressField(addr), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("fstree DELETE"), - ) - } - } - } - return nil - } - - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - - _, _ = c.fsTree.Iterate(prm) - - c.log.Info(logs.WritecacheFinishedUpdatingFSTreeFlushMarks) -} - -func (c *cache) dbFlushMarkUpdate(ctx context.Context) { - c.log.Info(logs.WritecacheFillingFlushMarksForObjectsInDatabase) - - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - - var m []string - var indices []int - var lastKey []byte - var batchSize = flushBatchSize - for { - select { - case <-c.closeCh: - return - case <-c.stopInitCh: - return - default: - } - - m = m[:0] - indices = indices[:0] - - // We put objects in batches of fixed size to not interfere with main put cycle a lot. - _ = c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - cs := b.Cursor() - for k, _ := cs.Seek(lastKey); k != nil && len(m) < batchSize; k, _ = cs.Next() { - m = append(m, string(k)) - } - return nil - }) - - var addr oid.Address - for i := range m { - if err := addr.DecodeString(m[i]); err != nil { - continue - } - - flushed, needRemove := c.flushStatus(ctx, addr) - if flushed { - c.store.flushed.Add(addr.EncodeToString(), true) - if needRemove { - indices = append(indices, i) - } - } - } - - if len(m) == 0 { - break - } - - err := c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - for _, j := range indices { - if err := b.Delete([]byte(m[j])); err != nil { - return err - } - } - return nil - }) - if err == nil { - for _, j := range indices { - storagelog.Write(c.log, - zap.String("address", m[j]), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("db DELETE"), - ) - } - } - lastKey = append([]byte(m[len(m)-1]), 0) - } - - c.log.Info(logs.WritecacheFinishedUpdatingFlushMarks) -} - -// flushStatus returns info about the object state in the main storage. -// First return value is true iff object exists. -// Second return value is true iff object can be safely removed. -func (c *cache) flushStatus(ctx context.Context, addr oid.Address) (bool, bool) { - var existsPrm meta.ExistsPrm - existsPrm.SetAddress(addr) - - _, err := c.metabase.Exists(ctx, existsPrm) - if err != nil { - needRemove := errors.Is(err, meta.ErrObjectIsExpired) || errors.As(err, new(apistatus.ObjectAlreadyRemoved)) - return needRemove, needRemove - } - - var prm meta.StorageIDPrm - prm.SetAddress(addr) - - mRes, _ := c.metabase.StorageID(ctx, prm) - res, err := c.blobstor.Exists(ctx, common.ExistsPrm{Address: addr, StorageID: mRes.StorageID()}) - return err == nil && res.Exists, false -} diff --git a/pkg/local_object_storage/writecache/iterate.go b/pkg/local_object_storage/writecache/iterate.go index 228dd2597..ebe979520 100644 --- a/pkg/local_object_storage/writecache/iterate.go +++ b/pkg/local_object_storage/writecache/iterate.go @@ -41,9 +41,6 @@ func (c *cache) Iterate(prm IterationPrm) error { err := c.db.View(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) return b.ForEach(func(k, data []byte) error { - if _, ok := c.flushed.Peek(string(k)); ok { - return nil - } return prm.handler(data) }) }) @@ -54,9 +51,6 @@ func (c *cache) Iterate(prm IterationPrm) error { var fsPrm common.IteratePrm fsPrm.IgnoreErrors = prm.ignoreErrors fsPrm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { - if _, ok := c.flushed.Peek(addr.EncodeToString()); ok { - return nil - } data, err := f() if err != nil { if prm.ignoreErrors { diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 14f8af49e..ca6faff4c 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -37,22 +37,6 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode) error { var err error turnOffMeta := m.NoMetabase() - if !c.initialized.Load() { - close(c.stopInitCh) - - c.initWG.Wait() - c.stopInitCh = make(chan struct{}) - - defer func() { - if err == nil && !turnOffMeta { - c.initFlushMarks(ctx) - } - }() - } - - c.modeMtx.Lock() - defer c.modeMtx.Unlock() - if turnOffMeta && !c.mode.NoMetabase() { err = c.flush(ctx, true) if err != nil { diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 223a73bef..04d818b31 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -37,8 +37,6 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro defer c.modeMtx.RUnlock() if c.readOnly() { return common.PutRes{}, ErrReadOnly - } else if !c.initialized.Load() { - return common.PutRes{}, ErrNotInitialized } sz := uint64(len(prm.RawData)) diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index aeae752e3..c06d16c0b 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -13,8 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - lru "github.com/hashicorp/golang-lru/v2" - "github.com/hashicorp/golang-lru/v2/simplelru" "go.etcd.io/bbolt" "go.uber.org/zap" ) @@ -22,19 +20,7 @@ import ( // store represents persistent storage with in-memory LRU cache // for flushed items on top of it. type store struct { - maxFlushedMarksCount int - maxRemoveBatchSize int - - // flushed contains addresses of objects that were already flushed to the main storage. - // We use LRU cache instead of map here to facilitate removing of unused object in favour of - // frequently read ones. - // MUST NOT be used inside bolt db transaction because it's eviction handler - // removes untracked items from the database. - flushed simplelru.LRUCache[string, bool] - db *bbolt.DB - - dbKeysToRemove []string - fsKeysToRemove []string + db *bbolt.DB } const dbName = "small.bolt" @@ -73,35 +59,9 @@ func (c *cache) openStore(readOnly bool) error { return fmt.Errorf("could not open FSTree: %w", err) } - // Write-cache can be opened multiple times during `SetMode`. - // flushed map must not be re-created in this case. - if c.flushed == nil { - c.flushed, _ = lru.NewWithEvict[string, bool](c.maxFlushedMarksCount, c.removeFlushed) - } - - c.initialized.Store(false) - return nil } -// removeFlushed removes an object from the writecache. -// To minimize interference with the client operations, the actual removal -// is done in batches. -// It is not thread-safe and is used only as an evict callback to LRU cache. -func (c *cache) removeFlushed(key string, value bool) { - fromDatabase := value - if fromDatabase { - c.dbKeysToRemove = append(c.dbKeysToRemove, key) - } else { - c.fsKeysToRemove = append(c.fsKeysToRemove, key) - } - - if len(c.dbKeysToRemove)+len(c.fsKeysToRemove) >= c.maxRemoveBatchSize { - c.dbKeysToRemove = c.deleteFromDB(c.dbKeysToRemove) - c.fsKeysToRemove = c.deleteFromDisk(c.fsKeysToRemove) - } -} - func (c *cache) deleteFromDB(keys []string) []string { if len(keys) == 0 { return keys @@ -133,7 +93,7 @@ func (c *cache) deleteFromDB(keys []string) []string { return keys[:len(keys)-errorIndex] } -func (c *cache) deleteFromDisk(keys []string) []string { +func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { if len(keys) == 0 { return keys } @@ -147,7 +107,7 @@ func (c *cache) deleteFromDisk(keys []string) []string { continue } - _, err := c.fsTree.Delete(context.TODO(), common.DeletePrm{Address: addr}) + _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err != nil && !errors.As(err, new(apistatus.ObjectNotFound)) { c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index bdcc9bbf6..83ecf219c 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -5,7 +5,6 @@ import ( "os" "sync" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -13,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -52,11 +50,8 @@ type cache struct { // mtx protects statistics, counters and compressFlags. mtx sync.RWMutex - mode mode.Mode - initialized atomic.Bool - stopInitCh chan struct{} // used to sync initWG initialisation routines and _only_ them - initWG sync.WaitGroup // for initialisation routines only - modeMtx sync.RWMutex + mode mode.Mode + modeMtx sync.RWMutex // compressFlags maps address of a big object to boolean value indicating // whether object should be compressed. @@ -96,9 +91,8 @@ var ( // New creates new writecache instance. func New(opts ...Option) Cache { c := &cache{ - flushCh: make(chan *object.Object), - mode: mode.ReadWrite, - stopInitCh: make(chan struct{}), + flushCh: make(chan *object.Object), + mode: mode.ReadWrite, compressFlags: make(map[string]struct{}), options: options{ @@ -117,12 +111,6 @@ func New(opts ...Option) Cache { opts[i](&c.options) } - // Make the LRU cache contain which take approximately 3/4 of the maximum space. - // Assume small and big objects are stored in 50-50 proportion. - c.maxFlushedMarksCount = int(c.maxCacheSize/c.maxObjectSize+c.maxCacheSize/c.smallObjectSize) / 2 * 3 / 4 - // Trigger the removal when the cache is 7/8 full, so that new items can still arrive. - c.maxRemoveBatchSize = c.maxFlushedMarksCount / 8 - return c } @@ -153,31 +141,27 @@ func (c *cache) Open(readOnly bool) error { // Init runs necessary services. func (c *cache) Init() error { - ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.Init") - defer span.End() - - c.initFlushMarks(ctx) c.runFlushLoop() return nil } // Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. func (c *cache) Close() error { - // Finish all in-progress operations. - if err := c.setMode(context.TODO(), mode.ReadOnly); err != nil { - return err - } - + // We cannot lock mutex for the whole operation duration + // because it is taken by some background workers, so `wg.Wait()` is done without modeMtx. + c.modeMtx.Lock() if c.closeCh != nil { close(c.closeCh) } + c.mode = mode.DegradedReadOnly // prevent new operations from being processed + c.modeMtx.Unlock() + c.wg.Wait() - if c.closeCh != nil { - c.closeCh = nil - } - c.initialized.Store(false) + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + c.closeCh = nil var err error if c.db != nil { err = c.db.Close() From e3ad3c296510ab08132c73d372de9173a852eb52 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 18 May 2023 13:08:05 +0300 Subject: [PATCH 0260/1943] [#283] cli: Move control healthcheck command under control ir Signed-off-by: Alejandro Lopez --- .../modules/control/healthcheck.go | 41 ++++------------- cmd/frostfs-cli/modules/control/ir.go | 2 + .../modules/control/ir_healthcheck.go | 44 +++++++++++++++++++ 3 files changed, 54 insertions(+), 33 deletions(-) create mode 100644 cmd/frostfs-cli/modules/control/ir_healthcheck.go diff --git a/cmd/frostfs-cli/modules/control/healthcheck.go b/cmd/frostfs-cli/modules/control/healthcheck.go index 8d18a5c11..097fba540 100644 --- a/cmd/frostfs-cli/modules/control/healthcheck.go +++ b/cmd/frostfs-cli/modules/control/healthcheck.go @@ -1,15 +1,10 @@ package control import ( - "crypto/ecdsa" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" - ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" ) @@ -19,8 +14,8 @@ const ( var healthCheckCmd = &cobra.Command{ Use: "healthcheck", - Short: "Health check of the FrostFS node", - Long: "Health check of the FrostFS node. Checks storage node by default, use --ir flag to work with Inner Ring.", + Short: "Health check for FrostFS storage nodes", + Long: "Health check for FrostFS storage nodes.", Run: healthCheck, } @@ -29,18 +24,18 @@ func initControlHealthCheckCmd() { flags := healthCheckCmd.Flags() flags.Bool(healthcheckIRFlag, false, "Communicate with IR node") + _ = flags.MarkDeprecated(healthcheckIRFlag, "for health check of inner ring nodes, use the 'control ir healthcheck' command instead.") } -func healthCheck(cmd *cobra.Command, _ []string) { - pk := key.Get(cmd) - - cli := getClient(cmd, pk) - +func healthCheck(cmd *cobra.Command, args []string) { if isIR, _ := cmd.Flags().GetBool(healthcheckIRFlag); isIR { - healthCheckIR(cmd, pk, cli) + irHealthCheck(cmd, args) return } + pk := key.Get(cmd) + cli := getClient(cmd, pk) + req := new(control.HealthCheckRequest) req.SetBody(new(control.HealthCheckRequest_Body)) @@ -59,23 +54,3 @@ func healthCheck(cmd *cobra.Command, _ []string) { cmd.Printf("Network status: %s\n", resp.GetBody().GetNetmapStatus()) cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus()) } - -func healthCheckIR(cmd *cobra.Command, key *ecdsa.PrivateKey, c *client.Client) { - req := new(ircontrol.HealthCheckRequest) - - req.SetBody(new(ircontrol.HealthCheckRequest_Body)) - - err := ircontrolsrv.SignMessage(key, req) - commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) - - var resp *ircontrol.HealthCheckResponse - err = c.ExecRaw(func(client *rawclient.Client) error { - resp, err = ircontrol.HealthCheck(client, req) - return err - }) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - - cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus()) -} diff --git a/cmd/frostfs-cli/modules/control/ir.go b/cmd/frostfs-cli/modules/control/ir.go index ae3f8502c..396d5d0a5 100644 --- a/cmd/frostfs-cli/modules/control/ir.go +++ b/cmd/frostfs-cli/modules/control/ir.go @@ -11,7 +11,9 @@ var irCmd = &cobra.Command{ func initControlIRCmd() { irCmd.AddCommand(tickEpochCmd) irCmd.AddCommand(removeNodeCmd) + irCmd.AddCommand(irHealthCheckCmd) initControlIRTickEpochCmd() initControlIRRemoveNodeCmd() + initControlIRHealthCheckCmd() } diff --git a/cmd/frostfs-cli/modules/control/ir_healthcheck.go b/cmd/frostfs-cli/modules/control/ir_healthcheck.go new file mode 100644 index 000000000..e70538ce2 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir_healthcheck.go @@ -0,0 +1,44 @@ +package control + +import ( + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + "github.com/spf13/cobra" +) + +var irHealthCheckCmd = &cobra.Command{ + Use: "healthcheck", + Short: "Health check for FrostFS inner ring nodes", + Long: "Health check for FrostFS inner ring nodes.", + Run: irHealthCheck, +} + +func initControlIRHealthCheckCmd() { + initControlFlags(irHealthCheckCmd) +} + +func irHealthCheck(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + cli := getClient(cmd, pk) + + req := new(ircontrol.HealthCheckRequest) + + req.SetBody(new(ircontrol.HealthCheckRequest_Body)) + + err := ircontrolsrv.SignMessage(pk, req) + commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) + + var resp *ircontrol.HealthCheckResponse + err = cli.ExecRaw(func(client *rawclient.Client) error { + resp, err = ircontrol.HealthCheck(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus()) +} From 731bf5d0ee6683e4bf624813356039b96e915d12 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 19 May 2023 16:46:01 +0300 Subject: [PATCH 0261/1943] [#380] node: go version up Signed-off-by: Dmitrii Stepanov --- .docker/Dockerfile.adm | 2 +- .docker/Dockerfile.ci | 2 +- .docker/Dockerfile.cli | 2 +- .docker/Dockerfile.ir | 2 +- .docker/Dockerfile.storage | 2 +- .docker/Dockerfile.storage-testnet | 2 +- CHANGELOG.md | 1 + README.md | 2 +- go.mod | 2 +- scripts/export-metrics/main.go | 3 +-- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.docker/Dockerfile.adm b/.docker/Dockerfile.adm index 09d66e642..f0075c7ab 100644 --- a/.docker/Dockerfile.adm +++ b/.docker/Dockerfile.adm @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ci b/.docker/Dockerfile.ci index 010616a17..e22762b7b 100644 --- a/.docker/Dockerfile.ci +++ b/.docker/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM golang:1.19 +FROM golang:1.20 WORKDIR /tmp diff --git a/.docker/Dockerfile.cli b/.docker/Dockerfile.cli index c706359b3..8fad863be 100644 --- a/.docker/Dockerfile.cli +++ b/.docker/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ir b/.docker/Dockerfile.ir index 9f8e72386..0b01ea7e4 100644 --- a/.docker/Dockerfile.ir +++ b/.docker/Dockerfile.ir @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage b/.docker/Dockerfile.storage index 39eb19559..dd9ff2684 100644 --- a/.docker/Dockerfile.storage +++ b/.docker/Dockerfile.storage @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage-testnet b/.docker/Dockerfile.storage-testnet index 908ff0aad..28bbf7a1b 100644 --- a/.docker/Dockerfile.storage-testnet +++ b/.docker/Dockerfile.storage-testnet @@ -1,4 +1,4 @@ -FROM golang:1.18 as builder +FROM golang:1.20 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/CHANGELOG.md b/CHANGELOG.md index 97feb2245..337a03794 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ Changelog for FrostFS Node - `github.com/spf13/cobra` to `v1.7.0` - `github.com/multiformats/go-multiaddr` to `v0.9.0` - `go.uber.org/atomic` to `v1.11.0` +- Minimum go version to v1.19 ### Updating from v0.36.0 diff --git a/README.md b/README.md index 81701c441..f228cd426 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The latest version of frostfs-node works with frostfs-contract # Building -To make all binaries you need Go 1.18+ and `make`: +To make all binaries you need Go 1.19+ and `make`: ``` make all ``` diff --git a/go.mod b/go.mod index 2e9286a05..5c6972f4b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-node -go 1.18 +go 1.19 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go index ac6e786ab..694eea38b 100644 --- a/scripts/export-metrics/main.go +++ b/scripts/export-metrics/main.go @@ -4,7 +4,6 @@ import ( "encoding/json" "flag" "fmt" - "io/ioutil" "os" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" @@ -49,7 +48,7 @@ func main() { os.Exit(1) } - if err := ioutil.WriteFile(filename, data, 0644); err != nil { + if err := os.WriteFile(filename, data, 0644); err != nil { fmt.Fprintf(os.Stderr, "Could write to file: %v\n", err) os.Exit(1) } From 7eb8fa6350b42fb8e762a61807a583c9fafe81d3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 19 May 2023 13:00:27 +0300 Subject: [PATCH 0262/1943] [#381] go.mod: Update dependencies Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 11 ++++- go.mod | 65 +++++++++++++------------ go.sum | 134 +++++++++++++++++++++++++++------------------------ 3 files changed, 115 insertions(+), 95 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 337a03794..71f1a98b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,14 +22,23 @@ Changelog for FrostFS Node ### Removed ### Updated -- `paulmach/orb` to v0.9.1 +- `neo-go` to `v0.101.1` +- `google.golang.org/grpc` to `v1.55.0` +- `paulmach/orb` to `v0.9.2` - `github.com/nats-io/nats.go` to `v1.25.0` - `golang.org/x/sync` to `v0.2.0` - `golang.org/x/term` to `v0.8.0` - `github.com/spf13/cobra` to `v1.7.0` +- `github.com/panjf2000/ants/v2` `v2.7.4` - `github.com/multiformats/go-multiaddr` to `v0.9.0` +- `github.com/hashicorp/golang-lru/v2` to `v2.0.2` - `go.uber.org/atomic` to `v1.11.0` - Minimum go version to v1.19 +- `github.com/prometheus/client_golang` to `v1.15.1` +- `github.com/prometheus/client_model` to `v0.4.0` +- `go.opentelemetry.io/otel` to `v1.15.1` +- `go.opentelemetry.io/otel/trace` to `v1.15.1` +- `github.com/spf13/cast` to `v1.5.1` ### Updating from v0.36.0 diff --git a/go.mod b/go.mod index 5c6972f4b..ed92cc7b6 100644 --- a/go.mod +++ b/go.mod @@ -12,32 +12,32 @@ require ( github.com/chzyer/readline v1.5.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/uuid v1.3.0 - github.com/hashicorp/golang-lru/v2 v2.0.1 + github.com/hashicorp/golang-lru/v2 v2.0.2 github.com/klauspost/compress v1.16.5 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.9.0 github.com/nats-io/nats.go v1.25.0 - github.com/nspcc-dev/neo-go v0.100.1 + github.com/nspcc-dev/neo-go v0.101.1 github.com/olekukonko/tablewriter v0.0.5 - github.com/panjf2000/ants/v2 v2.4.0 - github.com/paulmach/orb v0.9.1 - github.com/prometheus/client_golang v1.15.0 - github.com/prometheus/client_model v0.3.0 - github.com/spf13/cast v1.5.0 + github.com/panjf2000/ants/v2 v2.7.4 + github.com/paulmach/orb v0.9.2 + github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_model v0.4.0 + github.com/spf13/cast v1.5.1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 go.etcd.io/bbolt v1.3.6 - go.opentelemetry.io/otel v1.14.0 - go.opentelemetry.io/otel/trace v1.14.0 + go.opentelemetry.io/otel v1.15.1 + go.opentelemetry.io/otel/trace v1.15.1 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc golang.org/x/sync v0.2.0 golang.org/x/term v0.8.0 - google.golang.org/grpc v1.53.0 + google.golang.org/grpc v1.55.0 google.golang.org/protobuf v1.30.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -45,29 +45,29 @@ require ( require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect - github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/golang/snappy v0.0.3 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/holiman/uint256 v1.2.0 // indirect + github.com/holiman/uint256 v1.2.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -84,31 +84,32 @@ require ( github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/common v0.43.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect - github.com/twmb/murmur3 v1.1.5 // indirect - github.com/urfave/cli v1.22.5 // indirect - go.mongodb.org/mongo-driver v1.11.4 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect - go.opentelemetry.io/otel/sdk v1.14.0 // indirect + github.com/twmb/murmur3 v1.1.7 // indirect + github.com/urfave/cli v1.22.13 // indirect + go.mongodb.org/mongo-driver v1.11.6 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.15.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.15.1 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect + go.opentelemetry.io/otel/sdk v1.15.1 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.9.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - lukechampine.com/blake3 v1.1.7 // indirect + lukechampine.com/blake3 v1.2.1 // indirect ) retract ( diff --git a/go.sum b/go.sum index 29fef2509..913a57613 100644 --- a/go.sum +++ b/go.sum @@ -51,6 +51,7 @@ git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+ git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1:MJCkWUBhi9pn/CrYO1Q3P687y2KeahrOPS9BD9LDGb0= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= @@ -71,8 +72,8 @@ github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGn github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 h1:npHgfD4Tl2WJS3AJaMUi5ynGDPUBfkg3U3fCzDyXZ+4= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -91,8 +92,8 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= @@ -129,9 +130,8 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -149,7 +149,7 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -167,8 +167,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= @@ -177,8 +177,8 @@ github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaL github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -210,8 +210,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -250,22 +250,25 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAszqCJMbfTISJ7oMftp8+UGV08= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= -github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= +github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/holiman/uint256 v1.2.2 h1:TXKcSGc2WaxPD2+bmzAsVthL4+pEN0YwXcL5qED83vk= +github.com/holiman/uint256 v1.2.2/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -294,7 +297,6 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -321,8 +323,9 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -378,8 +381,8 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= github.com/nspcc-dev/neo-go v0.99.4/go.mod h1:mKTolfRUfKjFso5HPvGSQtUZc70n0VKBMs16eGuC5gA= -github.com/nspcc-dev/neo-go v0.100.1 h1:yugxbQRdzM+ObVa5mtr9/n4rYjxSIrryne8MVr9NBwU= -github.com/nspcc-dev/neo-go v0.100.1/go.mod h1:Nnp7F4e9IBccsgtCeLtUWV+0T6gk1PtP5HRtA13hUfc= +github.com/nspcc-dev/neo-go v0.101.1 h1:TVdcIpH/+bxQBTLRwWE3+Pw3j6j/JwguENbBSGAGid0= +github.com/nspcc-dev/neo-go v0.101.1/go.mod h1:J4tspxWw7jknX06F+VSMsKvIiNpYGfVTb2IxVC005YU= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb h1:GFxfkpXEYAbMIr69JpKOsQWeLOaGrd49HNAor8uDW+A= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= @@ -410,10 +413,10 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/panjf2000/ants/v2 v2.4.0 h1:embKPQeNWMRbnrRKURv4TXJwjQRWMEAfqZT6Pe5hZNc= -github.com/panjf2000/ants/v2 v2.4.0/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= -github.com/paulmach/orb v0.9.1 h1:NAL1AHClSzudQdZZA1kpjpNhcCEP1SXX+rQviOw0e/g= -github.com/paulmach/orb v0.9.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/panjf2000/ants/v2 v2.7.4 h1:mJqMDtMckZltyL458pq81IGNfiDhEgzX5s/lhjwPWIM= +github.com/panjf2000/ants/v2 v2.7.4/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8= +github.com/paulmach/orb v0.9.2 h1:p/YWV2uJwamAynnDOJGNbPBVtDHj3vG51k9tR1rFwJE= +github.com/paulmach/orb v0.9.2/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= @@ -432,22 +435,22 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.43.0 h1:iq+BVjvYLei5f27wiuNiB1DN6DYQkp1c8Bx0Vykh5us= +github.com/prometheus/common v0.43.0/go.mod h1:NCvr5cQIh3Y/gy73/RdVtC9r8xxrxwJnB+2lB3BxrFc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= @@ -457,10 +460,13 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -473,8 +479,8 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= @@ -496,8 +502,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= @@ -505,11 +512,13 @@ github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkk github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/twmb/murmur3 v1.1.5 h1:i9OLS9fkuLzBXjt6dptlAEyk58fJsSTXbRg3SgVyqgk= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/twmb/murmur3 v1.1.7 h1:ULWBiM04n/XoN3YMSJ6Z2pHDFLf+MeIVQU71ZPrvbWg= +github.com/twmb/murmur3 v1.1.7/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.13 h1:wsLILXG8qCJNse/qAgLNf23737Cx05GflHg/PJGe1Ok= +github.com/urfave/cli v1.22.13/go.mod h1:VufqObjsMTF2BBwKawpx9R8eAneNEWhoO0yx8Vd+FkE= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -526,28 +535,29 @@ github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBU go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.mongodb.org/mongo-driver v1.11.6 h1:XM7G6PjiGAO5betLF13BIa5TlLUUE3uJ/2Ox3Lz1K+o= +go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 h1:/fXHZHGvro6MVqV34fJzDhi7sHGpX3Ej/Qjmfn003ho= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 h1:TKf2uAs2ueguzLaxOCBXNpHxfO/aC7PAdDsSH0IbeRQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 h1:sEL90JjOO/4yhquXl5zTAkLLsZ5+MycAgX99SDsxGc8= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0/go.mod h1:oCslUcizYdpKYyS9e8srZEqM6BB8fq41VJBjLAE6z1w= -go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= -go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +go.opentelemetry.io/otel v1.15.1 h1:3Iwq3lfRByPaws0f6bU3naAqOR1n5IeDWd9390kWHa8= +go.opentelemetry.io/otel v1.15.1/go.mod h1:mHHGEHVDLal6YrKMmk9LqC4a3sF5g+fHfrttQIB1NTc= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.1 h1:XYDQtNzdb2T4uM1pku2m76eSMDJgqhJ+6KzkqgQBALc= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.1/go.mod h1:uOTV75+LOzV+ODmL8ahRLWkFA3eQcSC2aAsbxIu4duk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.15.1 h1:tyoeaUh8REKay72DVYsSEBYV18+fGONe+YYPaOxgLoE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.15.1/go.mod h1:HUSnrjQQ19KX9ECjpQxufsF+3ioD3zISPMlauTPZu2g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.15.1 h1:pIfoG5IAZFzp9EUlJzdSkpUwpaUAAnD+Ru1nBLTACIQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.15.1/go.mod h1:poNKBqF5+nR/6ke2oGTDjHfksrsHDOHXAl2g4+9ONsY= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= +go.opentelemetry.io/otel/sdk v1.15.1 h1:5FKR+skgpzvhPQHIEfcwMYjCBr14LWzs3uSqKiQzETI= +go.opentelemetry.io/otel/sdk v1.15.1/go.mod h1:8rVtxQfrbmbHKfqzpQkT5EzZMcbMBwTzNAggbEAM0KA= +go.opentelemetry.io/otel/trace v1.15.1 h1:uXLo6iHJEzDfrNC0L0mNjItIp06SyaBQxu5t3xMlngY= +go.opentelemetry.io/otel/trace v1.15.1/go.mod h1:IWdQG/5N1x7f6YUlmdLeJvH9yxtuJAfc4VW5Agv9r/8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -592,8 +602,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 h1:Jvc7gsqn21cJHCmAWx0LiimpP18LZmUxkT5Mp7EZ1mI= -golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -688,6 +698,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -901,8 +912,8 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -924,8 +935,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -959,7 +970,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -975,8 +985,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= -lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From ff570847a4dd18540a0fc3ffd19acff4c409225c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 19 May 2023 13:08:13 +0300 Subject: [PATCH 0263/1943] [#381] go.mod: Update bbolt Adopt new `ForEachBucket` function where possible. Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 3 ++- pkg/local_object_storage/metabase/iterators.go | 6 +----- pkg/local_object_storage/metabase/select.go | 6 +----- 5 files changed, 6 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71f1a98b4..f79940f76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ Changelog for FrostFS Node - `neo-go` to `v0.101.1` - `google.golang.org/grpc` to `v1.55.0` - `paulmach/orb` to `v0.9.2` +- `go.etcd.io/bbolt` to `v1.3.7` - `github.com/nats-io/nats.go` to `v1.25.0` - `golang.org/x/sync` to `v0.2.0` - `golang.org/x/term` to `v0.8.0` diff --git a/go.mod b/go.mod index ed92cc7b6..d90f45766 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.3 - go.etcd.io/bbolt v1.3.6 + go.etcd.io/bbolt v1.3.7 go.opentelemetry.io/otel v1.15.1 go.opentelemetry.io/otel/trace v1.15.1 go.uber.org/atomic v1.11.0 diff --git a/go.sum b/go.sum index 913a57613..648afba0e 100644 --- a/go.sum +++ b/go.sum @@ -533,8 +533,9 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver v1.11.6 h1:XM7G6PjiGAO5betLF13BIa5TlLUUE3uJ/2Ox3Lz1K+o= go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY= diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index 4c9dc782c..ba22566a3 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -72,12 +72,8 @@ func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) return fmt.Errorf("could not parse container ID of expired bucket: %w", err) } - return b.ForEach(func(expKey, _ []byte) error { + return b.ForEachBucket(func(expKey []byte) error { bktExpired := b.Bucket(expKey) - if bktExpired == nil { - return nil - } - expiresAfter, err := strconv.ParseUint(string(expKey), 10, 64) if err != nil { return fmt.Errorf("could not parse expiration epoch: %w", err) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index a4b14f77c..2ced6c4b7 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -315,12 +315,8 @@ func selectOutsideFKBT( bktExcl := tx.Bucket(name) if bktExcl != nil { - _ = bktExcl.ForEach(func(k, _ []byte) error { + _ = bktExcl.ForEachBucket(func(k []byte) error { exclBktLeaf := bktExcl.Bucket(k) - if exclBktLeaf == nil { - return nil - } - return exclBktLeaf.ForEach(func(k, _ []byte) error { mExcl[string(k)] = struct{}{} From 4b768fd11595d3e7f5350afd57a2d712f26fcf33 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 19 May 2023 18:06:20 +0300 Subject: [PATCH 0264/1943] [#381] *: Move to sync/atomic Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/evacuation.go | 4 +-- cmd/frostfs-node/config.go | 14 ++++++---- cmd/frostfs-node/netmap.go | 18 ++++++------ go.mod | 2 +- pkg/innerring/indexer_test.go | 6 ++-- pkg/innerring/innerring.go | 4 +-- pkg/innerring/state.go | 4 +-- .../blobovnicza/blobovnicza.go | 2 +- pkg/local_object_storage/blobovnicza/sizes.go | 2 +- .../engine/control_test.go | 2 +- pkg/local_object_storage/engine/engine.go | 6 ++-- .../engine/engine_test.go | 4 +-- pkg/local_object_storage/engine/evacuate.go | 28 +++++++++++-------- pkg/local_object_storage/engine/shards.go | 4 +-- .../internal/testutil/generators.go | 6 ++-- pkg/local_object_storage/metabase/put_test.go | 13 +++++---- .../shard/control_test.go | 2 +- .../writecache/flush_test.go | 4 +-- pkg/local_object_storage/writecache/state.go | 11 ++++---- pkg/morph/client/client.go | 2 +- pkg/util/worker_pool.go | 3 +- 21 files changed, 77 insertions(+), 64 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 69d2dd8d4..665bcf85f 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -4,6 +4,7 @@ import ( "crypto/ecdsa" "fmt" "strings" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -13,7 +14,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" - "go.uber.org/atomic" ) const ( @@ -134,7 +134,7 @@ func waitEvacuateCompletion(cmd *cobra.Command, pk *ecdsa.PrivateKey, cli *clien const statusPollingInterval = 1 * time.Second const reportIntervalSeconds = 5 var resp *control.GetShardEvacuationStatusResponse - reportResponse := atomic.NewPointer(resp) + reportResponse := new(atomic.Pointer[control.GetShardEvacuationStatusResponse]) pollingCompleted := make(chan struct{}) progressReportCompleted := make(chan struct{}) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c45d44fd2..fe2b5ad35 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -11,7 +11,7 @@ import ( "path/filepath" "strings" "sync" - atomicstd "sync/atomic" + "sync/atomic" "syscall" "time" @@ -68,7 +68,6 @@ import ( neogoutil "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -374,7 +373,7 @@ type shared struct { ownerIDFromKey user.ID // user ID calculated from key // current network map - netMap atomicstd.Value // type netmap.NetMap + netMap atomic.Value // type netmap.NetMap netMapSource netmapCore.Source cnrClient *containerClient.Client @@ -582,6 +581,9 @@ func initCfg(appCfg *config.Config) *cfg { } func initInternals(appCfg *config.Config, log *logger.Logger) internals { + var healthStatus atomic.Int32 + healthStatus.Store(int32(control.HealthStatus_HEALTH_STATUS_UNDEFINED)) + return internals{ done: make(chan struct{}), appCfg: appCfg, @@ -589,7 +591,7 @@ func initInternals(appCfg *config.Config, log *logger.Logger) internals { log: log, wg: new(sync.WaitGroup), apiVersion: version.Current(), - healthStatus: atomic.NewInt32(int32(control.HealthStatus_HEALTH_STATUS_UNDEFINED)), + healthStatus: &healthStatus, } } @@ -627,12 +629,14 @@ func initNetmap(appCfg *config.Config, netState *networkState, relayOnly bool) c netmapWorkerPool, err := ants.NewPool(notificationHandlerPoolSize) fatalOnErr(err) + var reBootstrapTurnedOff atomic.Bool + reBootstrapTurnedOff.Store(relayOnly) return cfgNetmap{ scriptHash: contractsconfig.Netmap(appCfg), state: netState, workerPool: netmapWorkerPool, needBootstrap: !relayOnly, - reBoostrapTurnedOff: atomic.NewBool(relayOnly), + reBoostrapTurnedOff: &reBootstrapTurnedOff, } } diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 58e3cb2f2..c6623a385 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "sync/atomic" netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -19,7 +20,6 @@ import ( netmapService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/netmap" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -27,7 +27,7 @@ import ( type networkState struct { epoch *atomic.Uint64 - controlNetStatus atomic.Value // control.NetmapStatus + controlNetStatus atomic.Int32 // control.NetmapStatus nodeInfo atomic.Value // *netmapSDK.NodeInfo @@ -35,13 +35,11 @@ type networkState struct { } func newNetworkState() *networkState { - var nmStatus atomic.Value - nmStatus.Store(control.NetmapStatus_STATUS_UNDEFINED) - - return &networkState{ - epoch: atomic.NewUint64(0), - controlNetStatus: nmStatus, + ns := &networkState{ + epoch: new(atomic.Uint64), } + ns.controlNetStatus.Store(int32(control.NetmapStatus_STATUS_UNDEFINED)) + return ns } func (s *networkState) CurrentEpoch() uint64 { @@ -91,11 +89,11 @@ func (s *networkState) setNodeInfo(ni *netmapSDK.NodeInfo) { // calls will process this value to decide what status node should set in the // network. func (s *networkState) setControlNetmapStatus(st control.NetmapStatus) { - s.controlNetStatus.Store(st) + s.controlNetStatus.Store(int32(st)) } func (s *networkState) controlNetmapStatus() (res control.NetmapStatus) { - return s.controlNetStatus.Load().(control.NetmapStatus) + return control.NetmapStatus(s.controlNetStatus.Load()) } func (s *networkState) getNodeInfo() (res netmapSDK.NodeInfo, ok bool) { diff --git a/go.mod b/go.mod index d90f45766..015ad5c2d 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,6 @@ require ( go.etcd.io/bbolt v1.3.7 go.opentelemetry.io/otel v1.15.1 go.opentelemetry.io/otel/trace v1.15.1 - go.uber.org/atomic v1.11.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc golang.org/x/sync v0.2.0 @@ -102,6 +101,7 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect go.opentelemetry.io/otel/sdk v1.15.1 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.9.0 // indirect golang.org/x/net v0.10.0 // indirect diff --git a/pkg/innerring/indexer_test.go b/pkg/innerring/indexer_test.go index 493ae92de..1937f7a49 100644 --- a/pkg/innerring/indexer_test.go +++ b/pkg/innerring/indexer_test.go @@ -2,12 +2,12 @@ package innerring import ( "fmt" + "sync/atomic" "testing" "time" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" - "go.uber.org/atomic" ) func TestIndexerReturnsIndexes(t *testing.T) { @@ -209,7 +209,7 @@ type testCommiteeFetcher struct { } func (f *testCommiteeFetcher) Committee() (keys.PublicKeys, error) { - f.calls.Inc() + f.calls.Add(1) return f.keys, f.err } @@ -220,6 +220,6 @@ type testIRFetcher struct { } func (f *testIRFetcher) InnerRingKeys() (keys.PublicKeys, error) { - f.calls.Inc() + f.calls.Add(1) return f.keys, f.err } diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 8c8c13dc3..17acdedda 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/config" @@ -29,7 +30,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/viper" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -52,7 +52,7 @@ type ( epochDuration atomic.Uint64 statusIndex *innerRingIndexer precision precision.Fixed8Converter - healthStatus atomic.Value + healthStatus atomic.Int32 balanceClient *balanceClient.Client netmapClient *nmClient.Client persistate *state.PersistentStorage diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 6a6ca0ade..27f265ae2 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -153,7 +153,7 @@ func (s *Server) ResetEpochTimer(h uint32) error { } func (s *Server) setHealthStatus(hs control.HealthStatus) { - s.healthStatus.Store(hs) + s.healthStatus.Store(int32(hs)) if s.metrics != nil { s.metrics.SetHealth(int32(hs)) } @@ -161,7 +161,7 @@ func (s *Server) setHealthStatus(hs control.HealthStatus) { // HealthStatus returns the current health status of the IR application. func (s *Server) HealthStatus() control.HealthStatus { - return s.healthStatus.Load().(control.HealthStatus) + return control.HealthStatus(s.healthStatus.Load()) } func initPersistentStateStorage(cfg *viper.Viper) (*state.PersistentStorage, error) { diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index a49324406..21c9f9e58 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -3,11 +3,11 @@ package blobovnicza import ( "io/fs" "os" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" ) diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index 82454fa28..1cc100d19 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -44,7 +44,7 @@ func (b *Blobovnicza) incSize(sz uint64) { } func (b *Blobovnicza) decSize(sz uint64) { - b.filled.Sub(sz) + b.filled.Add(^(sz - 1)) } func (b *Blobovnicza) full() bool { diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index d7eaae1d8..0c433f226 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strconv" + "sync/atomic" "testing" "time" @@ -24,7 +25,6 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zaptest" ) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index b7be4756d..19e7b5237 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -3,6 +3,7 @@ package engine import ( "errors" "sync" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -10,7 +11,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -131,7 +131,7 @@ func (e *StorageEngine) reportShardErrorBackground(id string, msg string, err er return } - errCount := sh.errorCount.Inc() + errCount := sh.errorCount.Add(1) e.reportShardErrorWithFlags(sh.Shard, errCount, false, msg, err) } @@ -149,7 +149,7 @@ func (e *StorageEngine) reportShardError( return } - errCount := sh.errorCount.Inc() + errCount := sh.errorCount.Add(1) e.reportShardErrorWithFlags(sh.Shard, errCount, true, msg, err, fields...) } diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 4d2ddc100..9c239739f 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "sync/atomic" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -22,7 +23,6 @@ import ( "git.frostfs.info/TrueCloudLab/hrw" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "go.uber.org/zap" "go.uber.org/zap/zaptest" ) @@ -98,7 +98,7 @@ func (te *testEngineWrapper) setInitializedShards(t testing.TB, shards ...*shard te.engine.shards[s.ID().String()] = hashedShard{ shardWrapper: shardWrapper{ - errorCount: atomic.NewUint32(0), + errorCount: new(atomic.Uint32), Shard: s, }, hash: hrw.Hash([]byte(s.ID().String())), diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 4693b2618..d698fca74 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -17,7 +18,6 @@ import ( "git.frostfs.info/TrueCloudLab/hrw" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -41,9 +41,9 @@ type EvacuateShardRes struct { // NewEvacuateShardRes creates new EvacuateShardRes instance. func NewEvacuateShardRes() *EvacuateShardRes { return &EvacuateShardRes{ - evacuated: atomic.NewUint64(0), - total: atomic.NewUint64(0), - failed: atomic.NewUint64(0), + evacuated: new(atomic.Uint64), + total: new(atomic.Uint64), + failed: new(atomic.Uint64), } } @@ -97,11 +97,17 @@ func (p *EvacuateShardRes) DeepCopy() *EvacuateShardRes { if p == nil { return nil } - return &EvacuateShardRes{ - evacuated: atomic.NewUint64(p.evacuated.Load()), - total: atomic.NewUint64(p.total.Load()), - failed: atomic.NewUint64(p.failed.Load()), + + res := &EvacuateShardRes{ + evacuated: new(atomic.Uint64), + total: new(atomic.Uint64), + failed: new(atomic.Uint64), } + + res.evacuated.Store(p.evacuated.Load()) + res.total.Store(p.total.Load()) + res.failed.Store(p.failed.Load()) + return res } const defaultEvacuateBatchSize = 100 @@ -323,7 +329,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to getRes, err := sh.Get(ctx, getPrm) if err != nil { if prm.ignoreErrors { - res.failed.Inc() + res.failed.Add(1) continue } e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) @@ -350,7 +356,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) return err } - res.evacuated.Inc() + res.evacuated.Add(1) } return nil } @@ -371,7 +377,7 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add putDone, exists := e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object) if putDone || exists { if putDone { - res.evacuated.Inc() + res.evacuated.Add(1) e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 64546d9ef..e16d2c498 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -2,6 +2,7 @@ package engine import ( "fmt" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -11,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/hrw" "github.com/google/uuid" "github.com/panjf2000/ants/v2" - "go.uber.org/atomic" "go.uber.org/zap" ) @@ -133,7 +133,7 @@ func (e *StorageEngine) addShard(sh *shard.Shard) error { e.shards[strID] = hashedShard{ shardWrapper: shardWrapper{ - errorCount: atomic.NewUint32(0), + errorCount: new(atomic.Uint32), Shard: sh, }, hash: hrw.Hash([]byte(strID)), diff --git a/pkg/local_object_storage/internal/testutil/generators.go b/pkg/local_object_storage/internal/testutil/generators.go index 1a1f3cf9e..059f00b3a 100644 --- a/pkg/local_object_storage/internal/testutil/generators.go +++ b/pkg/local_object_storage/internal/testutil/generators.go @@ -2,13 +2,13 @@ package testutil import ( "encoding/binary" + "sync/atomic" "testing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "golang.org/x/exp/rand" ) @@ -27,7 +27,7 @@ var _ AddressGenerator = &SeqAddrGenerator{} func (g *SeqAddrGenerator) Next() oid.Address { var id oid.ID - binary.LittleEndian.PutUint64(id[:], ((g.cnt.Inc()-1)%g.MaxID)+1) + binary.LittleEndian.PutUint64(id[:], ((g.cnt.Add(1)-1)%g.MaxID)+1) var addr oid.Address addr.SetContainer(cid.ID{}) addr.SetObject(id) @@ -69,7 +69,7 @@ func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *ob func (g *SeqObjGenerator) Next() *object.Object { var id oid.ID - binary.LittleEndian.PutUint64(id[:], g.cnt.Inc()) + binary.LittleEndian.PutUint64(id[:], g.cnt.Add(1)) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) } diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index a3a071d19..dbf89c9c8 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -4,6 +4,7 @@ import ( "context" "runtime" "strconv" + "sync/atomic" "testing" "time" @@ -15,7 +16,6 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" - "go.uber.org/atomic" ) func prepareObjects(t testing.TB, n int) []*objectSDK.Object { @@ -49,13 +49,15 @@ func BenchmarkPut(b *testing.B) { // Ensure the benchmark is bound by CPU and not waiting batch-delay time. b.SetParallelism(1) - index := atomic.NewInt64(-1) + var index atomic.Int64 + index.Store(-1) + objs := prepareObjects(b, b.N) b.ResetTimer() b.ReportAllocs() b.RunParallel(func(pb *testing.PB) { for pb.Next() { - if err := metaPut(db, objs[index.Inc()], nil); err != nil { + if err := metaPut(db, objs[index.Add(1)], nil); err != nil { b.Fatal(err) } } @@ -65,12 +67,13 @@ func BenchmarkPut(b *testing.B) { db := newDB(b, meta.WithMaxBatchDelay(time.Millisecond*10), meta.WithMaxBatchSize(1)) - index := atomic.NewInt64(-1) + var index atomic.Int64 + index.Store(-1) objs := prepareObjects(b, b.N) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - if err := metaPut(db, objs[index.Inc()], nil); err != nil { + if err := metaPut(db, objs[index.Add(1)], nil); err != nil { b.Fatal(err) } } diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 3f0a19cd4..82b107196 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -6,6 +6,7 @@ import ( "math" "os" "path/filepath" + "sync/atomic" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -27,7 +28,6 @@ import ( objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap/zaptest" ) diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index e5ca85735..e8224ce5d 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "sync/atomic" "testing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -22,7 +23,6 @@ import ( versionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/atomic" "go.uber.org/zap/zaptest" ) @@ -132,7 +132,7 @@ func TestFlush(t *testing.T) { testIgnoreErrors := func(t *testing.T, f func(*cache)) { var errCount atomic.Uint32 wc, bs, mb := newCache(t, WithReportErrorFunc(func(message string, err error) { - errCount.Inc() + errCount.Add(1) })) objects := putObjects(t, wc) f(wc.(*cache)) diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index 1ba5a4bd3..9c1c562b0 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -2,9 +2,10 @@ package writecache import ( "fmt" + "math" + "sync/atomic" "go.etcd.io/bbolt" - "go.uber.org/atomic" ) func (c *cache) estimateCacheSize() uint64 { @@ -24,11 +25,11 @@ type counters struct { } func (x *counters) IncDB() { - x.cDB.Inc() + x.cDB.Add(1) } func (x *counters) DecDB() { - x.cDB.Dec() + x.cDB.Add(math.MaxUint64) } func (x *counters) DB() uint64 { @@ -36,11 +37,11 @@ func (x *counters) DB() uint64 { } func (x *counters) IncFS() { - x.cFS.Inc() + x.cFS.Add(1) } func (x *counters) DecFS() { - x.cFS.Dec() + x.cFS.Add(math.MaxUint64) } func (x *counters) FS() uint64 { diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index c9ff14a66..6b6e1284b 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -6,6 +6,7 @@ import ( "fmt" "math/big" "sync" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -26,7 +27,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" - "go.uber.org/atomic" "go.uber.org/zap" ) diff --git a/pkg/util/worker_pool.go b/pkg/util/worker_pool.go index 145fd1a5a..97d76c492 100644 --- a/pkg/util/worker_pool.go +++ b/pkg/util/worker_pool.go @@ -1,8 +1,9 @@ package util import ( + "sync/atomic" + "github.com/panjf2000/ants/v2" - "go.uber.org/atomic" ) // WorkerPool represents a tool to control From cf8531ccd74ea79747f1ac23c4f852ed426a4e7f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 22 May 2023 11:30:03 +0300 Subject: [PATCH 0265/1943] [#381] go.mod: Update api-go and sdk-go Signed-off-by: Evgenii Stratonikov --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 015ad5c2d..2c95bd41b 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230519114017-0c67b8fefa41 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230519144724-f5b23eb22569 git.frostfs.info/TrueCloudLab/hrw v1.2.0 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 648afba0e..d728851b9 100644 --- a/go.sum +++ b/go.sum @@ -36,14 +36,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85 h1:77lvdk0kMhnUgtnmqEcAPXPQaGlt24goMPu2+E5WRTk= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230418080822-bd44a3f47b85/go.mod h1:sPyITTmQT662ZI38ud2aoE1SUCAr1mO5xV8P4nzLkKI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230519114017-0c67b8fefa41 h1:xtGsOUX8Rz0hwWIFa148URysWuD4nRHspPNbYAUc1tg= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230519114017-0c67b8fefa41/go.mod h1:6wEpMfSwD5xNtQYYVHWWTHwpYuvyumyntZEzILBIXUo= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd h1:HxacVl1Lc2RrfxAE13AGkp1tR/Mf4DDP6TgrgbLP5fQ= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230505094539-15b4287092bd/go.mod h1:TaJJOF3Uhuq8aqv2CrfuY2yhxePUinW35Xd3wfXLV/I= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230519144724-f5b23eb22569 h1:JDySbbi0MyU1fSuew0jNyWwDhrQREetHWJiSzKzXXXI= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230519144724-f5b23eb22569/go.mod h1:0n2gQYkqCgiXkDzk0RLUUxoF2xX79Ke85bAiGV8cBl8= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 03ab0ca30f31304a566056c6338ad528d0a0d5f6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 15 May 2023 15:14:56 +0300 Subject: [PATCH 0266/1943] [#338] adm: Drop notaryless code Signed-off-by: Dmitrii Stepanov --- .../internal/modules/morph/balance.go | 51 +++++-------------- 1 file changed, 14 insertions(+), 37 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance.go index f97250c38..6debc50b9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance.go @@ -37,11 +37,6 @@ const ( dumpBalancesAlphabetFlag = "alphabet" dumpBalancesProxyFlag = "proxy" dumpBalancesUseScriptHashFlag = "script-hash" - - // notaryEnabled signifies whether contracts were deployed in a notary-enabled environment. - // The setting is here to simplify testing and building the command for testnet (notary currently disabled). - // It will be removed eventually. - notaryEnabled = true ) func dumpBalances(cmd *cobra.Command, _ []string) error { @@ -60,7 +55,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { inv := invoker.New(c, nil) - if !notaryEnabled || dumpStorage || dumpAlphabet || dumpProxy { + if dumpStorage || dumpAlphabet || dumpProxy { nnsCs, err = c.GetContractStateByID(1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) @@ -72,7 +67,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { } } - irList, err := fetchIRNodes(c, nmHash, rolemgmt.Hash) + irList, err := fetchIRNodes(c, rolemgmt.Hash) if err != nil { return err } @@ -187,40 +182,22 @@ func printAlphabetContractBalances(cmd *cobra.Command, c Client, inv *invoker.In return nil } -func fetchIRNodes(c Client, nmHash, desigHash util.Uint160) ([]accBalancePair, error) { - var irList []accBalancePair - +func fetchIRNodes(c Client, desigHash util.Uint160) ([]accBalancePair, error) { inv := invoker.New(c, nil) - if notaryEnabled { - height, err := c.GetBlockCount() - if err != nil { - return nil, fmt.Errorf("can't get block height: %w", err) - } + height, err := c.GetBlockCount() + if err != nil { + return nil, fmt.Errorf("can't get block height: %w", err) + } - arr, err := getDesignatedByRole(inv, desigHash, noderoles.NeoFSAlphabet, height) - if err != nil { - return nil, errors.New("can't fetch list of IR nodes from the netmap contract") - } + arr, err := getDesignatedByRole(inv, desigHash, noderoles.NeoFSAlphabet, height) + if err != nil { + return nil, errors.New("can't fetch list of IR nodes from the netmap contract") + } - irList = make([]accBalancePair, len(arr)) - for i := range arr { - irList[i].scriptHash = arr[i].GetScriptHash() - } - } else { - arr, err := unwrap.ArrayOfBytes(inv.Call(nmHash, "innerRingList")) - if err != nil { - return nil, errors.New("can't fetch list of IR nodes from the netmap contract") - } - - irList = make([]accBalancePair, len(arr)) - for i := range arr { - pub, err := keys.NewPublicKeyFromBytes(arr[i], elliptic.P256()) - if err != nil { - return nil, fmt.Errorf("can't parse IR node public key: %w", err) - } - irList[i].scriptHash = pub.GetScriptHash() - } + irList := make([]accBalancePair, len(arr)) + for i := range arr { + irList[i].scriptHash = arr[i].GetScriptHash() } return irList, nil } From fb708b3a2d5d3744a1b4b49e163e70a011b05f82 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 17 May 2023 15:43:56 +0300 Subject: [PATCH 0267/1943] [#338] ir: Drop container notaryless code Signed-off-by: Dmitrii Stepanov --- pkg/innerring/initialization.go | 2 +- .../processors/container/handlers_test.go | 84 +++++++++---------- .../processors/container/process_container.go | 40 +-------- .../processors/container/process_eacl.go | 18 +--- .../processors/container/processor.go | 43 +++++----- 5 files changed, 66 insertions(+), 121 deletions(-) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 89269d50d..6eb67ae7d 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -263,9 +263,9 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C PoolSize: cfg.GetInt("workers.container"), AlphabetState: s, ContainerClient: cnrClient, + MorphClient: cnrClient.Morph(), FrostFSIDClient: frostfsIDClient, NetworkState: s.netmapClient, - NotaryDisabled: s.sideNotaryConfig.disabled, }) if err != nil { return err diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index 6075ff577..1e518f474 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -7,7 +7,6 @@ import ( "time" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" @@ -22,6 +21,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/network/payload" "github.com/nspcc-dev/neo-go/pkg/util" @@ -34,18 +34,16 @@ func TestPutEvent(t *testing.T) { homHashDisabled: true, epoch: 100, } - cc := &testContainerClient{ - get: make(map[string]*containercore.Container), - } + mc := &testMorphClient{} proc, err := New(&Params{ Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, FrostFSIDClient: &testIDClient{}, - NotaryDisabled: true, NetworkState: nst, - ContainerClient: cc, + ContainerClient: &testContainerClient{}, + MorphClient: mc, }) require.NoError(t, err, "failed to create processor") @@ -64,10 +62,15 @@ func TestPutEvent(t *testing.T) { cnr.SetBasicACL(acl.Private) containerSDK.DisableHomomorphicHashing(&cnr) + nr := &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + } + event := &testPutEvent{ cnr: &cnr, pk: p, st: nil, + nr: nr, } proc.handlePut(event) @@ -76,13 +79,7 @@ func TestPutEvent(t *testing.T) { time.Sleep(10 * time.Millisecond) } - var expectedPut cntClient.PutPrm - expectedPut.SetContainer(cnr.Marshal()) - expectedPut.SetKey(p.PublicKey().Bytes()) - expectedPut.SetSignature(p.Sign(cnr.Marshal())) - expectedPut.SetZone("container") - - require.EqualValues(t, []cntClient.PutPrm{expectedPut}, cc.put, "invalid put requests") + require.EqualValues(t, []*transaction.Transaction{nr.MainTransaction}, mc.transactions, "invalid notary requests") } func TestDeleteEvent(t *testing.T) { @@ -103,15 +100,16 @@ func TestDeleteEvent(t *testing.T) { p.PublicKey(), }, } + mc := &testMorphClient{} proc, err := New(&Params{ Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, FrostFSIDClient: idc, - NotaryDisabled: true, NetworkState: nst, ContainerClient: cc, + MorphClient: mc, }) require.NoError(t, err, "failed to create processor") @@ -133,9 +131,14 @@ func TestDeleteEvent(t *testing.T) { cidBin := make([]byte, 32) cid.Encode(cidBin) + nr := &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + } + ev := containerEvent.Delete{ - ContainerIDValue: cidBin, - SignatureValue: p.Sign(cidBin), + ContainerIDValue: cidBin, + SignatureValue: p.Sign(cidBin), + NotaryRequestValue: nr, } var signature frostfscrypto.Signature @@ -156,7 +159,7 @@ func TestDeleteEvent(t *testing.T) { expectedDelete.SetCID(ev.ContainerID()) expectedDelete.SetSignature(ev.Signature()) - require.EqualValues(t, []cntClient.DeletePrm{expectedDelete}, cc.delete, "invalid delete requests") + require.EqualValues(t, []*transaction.Transaction{nr.MainTransaction}, mc.transactions, "invalid notary requests") } func TestSetEACLEvent(t *testing.T) { @@ -168,15 +171,16 @@ func TestSetEACLEvent(t *testing.T) { cc := &testContainerClient{ get: make(map[string]*containercore.Container), } + mc := &testMorphClient{} proc, err := New(&Params{ Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, FrostFSIDClient: &testIDClient{}, - NotaryDisabled: true, NetworkState: nst, ContainerClient: cc, + MorphClient: mc, }) require.NoError(t, err, "failed to create processor") @@ -219,10 +223,14 @@ func TestSetEACLEvent(t *testing.T) { table.AddRecord(r) + nr := &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + } event := containerEvent.SetEACL{ - TableValue: table.ToV2().StableMarshal(nil), - PublicKeyValue: p.PublicKey().Bytes(), - SignatureValue: p.Sign(table.ToV2().StableMarshal(nil)), + TableValue: table.ToV2().StableMarshal(nil), + PublicKeyValue: p.PublicKey().Bytes(), + SignatureValue: p.Sign(table.ToV2().StableMarshal(nil)), + NotaryRequestValue: nr, } proc.handleSetEACL(event) @@ -236,7 +244,7 @@ func TestSetEACLEvent(t *testing.T) { expectedPutEACL.SetKey(p.PublicKey().Bytes()) expectedPutEACL.SetSignature(p.Sign(table.ToV2().StableMarshal(nil))) - require.EqualValues(t, []cntClient.PutEACLPrm{expectedPutEACL}, cc.putEACL, "invalid set EACL requests") + require.EqualValues(t, []*transaction.Transaction{nr.MainTransaction}, mc.transactions, "invalid notary requests") } type testAlphabetState struct { @@ -262,25 +270,13 @@ func (s *testNetworkState) Epoch() (uint64, error) { type testContainerClient struct { contractAddress util.Uint160 - put []cntClient.PutPrm get map[string]*containercore.Container - delete []cntClient.DeletePrm - putEACL []cntClient.PutEACLPrm } func (c *testContainerClient) ContractAddress() util.Uint160 { return c.contractAddress } -func (c *testContainerClient) Morph() *client.Client { - return nil -} - -func (c *testContainerClient) Put(p cntClient.PutPrm) error { - c.put = append(c.put, p) - return nil -} - func (c *testContainerClient) Get(cid []byte) (*containercore.Container, error) { key := hex.EncodeToString(cid) if cont, found := c.get[key]; found { @@ -289,16 +285,6 @@ func (c *testContainerClient) Get(cid []byte) (*containercore.Container, error) return nil, apistatus.ContainerNotFound{} } -func (c *testContainerClient) Delete(p cntClient.DeletePrm) error { - c.delete = append(c.delete, p) - return nil -} - -func (c *testContainerClient) PutEACL(p cntClient.PutEACLPrm) error { - c.putEACL = append(c.putEACL, p) - return nil -} - type testIDClient struct { publicKeys keys.PublicKeys } @@ -313,6 +299,7 @@ type testPutEvent struct { cnr *containerSDK.Container pk *keys.PrivateKey st []byte + nr *payload.P2PNotaryRequest } func (e *testPutEvent) MorphEvent() {} @@ -333,5 +320,14 @@ func (e *testPutEvent) SessionToken() []byte { return e.st } func (e *testPutEvent) NotaryRequest() *payload.P2PNotaryRequest { + return e.nr +} + +type testMorphClient struct { + transactions []*transaction.Transaction +} + +func (c *testMorphClient) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { + c.transactions = append(c.transactions, mainTx) return nil } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 603ab86d1..3bee1c4d5 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -4,7 +4,6 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" @@ -91,27 +90,7 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { } func (cp *Processor) approvePutContainer(ctx *putContainerContext) { - e := ctx.e - - var err error - - prm := cntClient.PutPrm{} - - prm.SetContainer(e.Container()) - prm.SetKey(e.PublicKey()) - prm.SetSignature(e.Signature()) - prm.SetToken(e.SessionToken()) - prm.SetName(ctx.d.Name()) - prm.SetZone(ctx.d.Zone()) - - if nr := e.NotaryRequest(); nr != nil { - // put event was received via Notary service - err = cp.cnrClient.Morph().NotarySignAndInvokeTX(nr.MainTransaction) - } else { - // put event was received via notification service - err = cp.cnrClient.Put(prm) - } - if err != nil { + if err := cp.morphClient.NotarySignAndInvokeTX(ctx.e.NotaryRequest().MainTransaction); err != nil { cp.log.Error(logs.ContainerCouldNotApprovePutContainer, zap.String("error", err.Error()), ) @@ -171,22 +150,7 @@ func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { } func (cp *Processor) approveDeleteContainer(e containerEvent.Delete) { - var err error - - prm := cntClient.DeletePrm{} - - prm.SetCID(e.ContainerID()) - prm.SetSignature(e.Signature()) - prm.SetToken(e.SessionToken()) - - if nr := e.NotaryRequest(); nr != nil { - // delete event was received via Notary service - err = cp.cnrClient.Morph().NotarySignAndInvokeTX(nr.MainTransaction) - } else { - // delete event was received via notification service - err = cp.cnrClient.Delete(prm) - } - if err != nil { + if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { cp.log.Error(logs.ContainerCouldNotApproveDeleteContainer, zap.String("error", err.Error()), ) diff --git a/pkg/innerring/processors/container/process_eacl.go b/pkg/innerring/processors/container/process_eacl.go index a8d880c8f..43a59e223 100644 --- a/pkg/innerring/processors/container/process_eacl.go +++ b/pkg/innerring/processors/container/process_eacl.go @@ -75,23 +75,7 @@ func (cp *Processor) checkSetEACL(e containerEvent.SetEACL) error { } func (cp *Processor) approveSetEACL(e containerEvent.SetEACL) { - var err error - - prm := cntClient.PutEACLPrm{} - - prm.SetTable(e.Table()) - prm.SetKey(e.PublicKey()) - prm.SetSignature(e.Signature()) - prm.SetToken(e.SessionToken()) - - if nr := e.NotaryRequest(); nr != nil { - // setEACL event was received via Notary service - err = cp.cnrClient.Morph().NotarySignAndInvokeTX(nr.MainTransaction) - } else { - // setEACL event was received via notification service - err = cp.cnrClient.PutEACL(prm) - } - if err != nil { + if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { cp.log.Error(logs.ContainerCouldNotApproveSetEACL, zap.String("error", err.Error()), ) diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index 2141b0764..ec82ace70 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -6,13 +6,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/core/mempoolevent" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" @@ -27,11 +26,11 @@ type ( ContClient interface { ContractAddress() util.Uint160 - Morph() *client.Client - Put(p cntClient.PutPrm) error Get(cid []byte) (*containercore.Container, error) - Delete(p cntClient.DeletePrm) error - PutEACL(p cntClient.PutEACLPrm) error + } + + MorphClient interface { + NotarySignAndInvokeTX(mainTx *transaction.Transaction) error } IDClient interface { @@ -40,13 +39,13 @@ type ( // Processor of events produced by container contract in the sidechain. Processor struct { - log *logger.Logger - pool *ants.Pool - alphabetState AlphabetState - cnrClient ContClient // notary must be enabled - idClient IDClient - netState NetworkState - notaryDisabled bool + log *logger.Logger + pool *ants.Pool + alphabetState AlphabetState + cnrClient ContClient // notary must be enabled + morphClient MorphClient + idClient IDClient + netState NetworkState } // Params of the processor constructor. @@ -55,9 +54,9 @@ type ( PoolSize int AlphabetState AlphabetState ContainerClient ContClient + MorphClient MorphClient FrostFSIDClient IDClient NetworkState NetworkState - NotaryDisabled bool } ) @@ -88,6 +87,8 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/container: global state is not set") case p.ContainerClient == nil: return nil, errors.New("ir/container: Container client is not set") + case p.MorphClient == nil: + return nil, errors.New("ir/container: Morph client is not set") case p.FrostFSIDClient == nil: return nil, errors.New("ir/container: FrostFS ID client is not set") case p.NetworkState == nil: @@ -102,13 +103,13 @@ func New(p *Params) (*Processor, error) { } return &Processor{ - log: p.Log, - pool: pool, - alphabetState: p.AlphabetState, - cnrClient: p.ContainerClient, - idClient: p.FrostFSIDClient, - netState: p.NetworkState, - notaryDisabled: p.NotaryDisabled, + log: p.Log, + pool: pool, + alphabetState: p.AlphabetState, + cnrClient: p.ContainerClient, + idClient: p.FrostFSIDClient, + netState: p.NetworkState, + morphClient: p.MorphClient, }, nil } From 656fd7f376db7ad97f33184ca1b4cb576303a23b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 17 May 2023 16:56:47 +0300 Subject: [PATCH 0268/1943] [#338] ir: Drop notaryless code from netmap Signed-off-by: Dmitrii Stepanov --- pkg/innerring/initialization.go | 1 - .../processors/netmap/handlers_test.go | 361 ++++++------------ .../processors/netmap/process_cleanup.go | 26 +- .../processors/netmap/process_peers.go | 61 +-- pkg/innerring/processors/netmap/processor.go | 9 - 5 files changed, 136 insertions(+), 322 deletions(-) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 6eb67ae7d..50ba67124 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -71,7 +71,6 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, addrvalidator.New(), locodeValidator, ), - NotaryDisabled: s.sideNotaryConfig.disabled, NodeStateSettings: netSettings, }) diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index 4905b45d2..6c9e265cc 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -31,7 +31,6 @@ func TestNewEpochTick(t *testing.T) { nc := &testNetmapClient{} proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true p.CleanupEnabled = true p.EpochState = es p.NetmapClient = nc @@ -80,7 +79,6 @@ func TestNewEpoch(t *testing.T) { eh := &testEventHandler{} proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true p.NotaryDepositHandler = eh.Handle p.AlphabetSyncHandler = eh.Handle p.NetmapClient = nc @@ -119,276 +117,139 @@ func TestNewEpoch(t *testing.T) { func TestAddPeer(t *testing.T) { t.Parallel() - t.Run("with notary", func(t *testing.T) { - t.Parallel() - nc := &testNetmapClient{ - contractAddress: util.Uint160{47}, - } + nc := &testNetmapClient{ + contractAddress: util.Uint160{47}, + } - proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true - p.NetmapClient = nc - }) - - require.NoError(t, err, "failed to create processor") - - var node netmap.NodeInfo - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key1") - node.SetPublicKey(key.Bytes()) - - ev := netmapEvent.AddPeer{ - NodeBytes: node.Marshal(), - Request: &payload.P2PNotaryRequest{ - MainTransaction: &transaction.Transaction{ - Nonce: 100, - }, - }, - } - proc.handleAddPeer(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - require.EqualValues(t, []notaryInvoke{ - { - contract: nc.contractAddress, - fee: 0, - nonce: ev.Request.MainTransaction.Nonce, - vub: nil, - method: "addPeerIR", - args: []any{ev.Node()}, - }, - }, nc.notaryInvokes, "invalid notary invokes") + proc, err := newTestProc(t, func(p *Params) { + p.NetmapClient = nc }) - t.Run("without notary", func(t *testing.T) { - t.Parallel() + require.NoError(t, err, "failed to create processor") - nc := &testNetmapClient{ - contractAddress: util.Uint160{47}, - } + var node netmap.NodeInfo + key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key") + node.SetPublicKey(key.Bytes()) - proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true - p.NetmapClient = nc - }) + ev := netmapEvent.AddPeer{ + NodeBytes: node.Marshal(), + Request: &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + }, + } + proc.handleAddPeer(ev) - require.NoError(t, err, "failed to create processor") + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } - var node netmap.NodeInfo - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key") - node.SetPublicKey(key.Bytes()) - - ev := netmapEvent.AddPeer{ - NodeBytes: node.Marshal(), - } - proc.handleAddPeer(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - var addPeerExp netmapclient.AddPeerPrm - addPeerExp.SetNodeInfo(node) - require.EqualValues(t, []netmapclient.AddPeerPrm{addPeerExp}, nc.addPeers, "invalid peers") - }) + require.EqualValues(t, []notaryInvoke{ + { + contract: nc.contractAddress, + fee: 0, + nonce: ev.NotaryRequest().MainTransaction.Nonce, + vub: nil, + method: "addPeerIR", + args: []any{node.Marshal()}, + }, + }, nc.notaryInvokes, "invalid notary invokes") } func TestUpdateState(t *testing.T) { t.Parallel() - t.Run("with notary", func(t *testing.T) { - t.Parallel() - ns := &testNodeStateSettings{ - maintAllowed: true, - } - nc := &testNetmapClient{} + ns := &testNodeStateSettings{ + maintAllowed: true, + } + nc := &testNetmapClient{} - proc, err := newTestProc(t, func(p *Params) { - p.NotaryDisabled = true - p.NodeStateSettings = ns - p.NetmapClient = nc - }) - - require.NoError(t, err, "failed to create processor") - - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key") - - ev := netmapEvent.UpdatePeer{ - State: netmapContract.NodeStateOnline, - PubKey: key, - Request: &payload.P2PNotaryRequest{ - MainTransaction: &transaction.Transaction{ - Nonce: 100, - }, - }, - } - proc.handleUpdateState(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - require.EqualValues(t, []*transaction.Transaction{ - ev.Request.MainTransaction, - }, nc.invokedTxs, "invalid invoked transactions") + proc, err := newTestProc(t, func(p *Params) { + p.NetmapClient = nc + p.NodeStateSettings = ns }) - t.Run("without notary", func(t *testing.T) { - t.Parallel() - ns := &testNodeStateSettings{ - maintAllowed: true, - } - nc := &testNetmapClient{} + require.NoError(t, err, "failed to create processor") - proc, err := newTestProc(t, func(p *Params) { - p.NetmapClient = nc - p.NodeStateSettings = ns - }) + key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") + require.NoError(t, err, "failed to parse key") - require.NoError(t, err, "failed to create processor") + ev := netmapEvent.UpdatePeer{ + State: netmapContract.NodeStateOnline, + PubKey: key, + Request: &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + }, + } + proc.handleUpdateState(ev) - key, err := keys.NewPublicKeyFromString("038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35") - require.NoError(t, err, "failed to parse key") + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } - ev := netmapEvent.UpdatePeer{ - State: netmapContract.NodeStateOnline, - PubKey: key, - } - proc.handleUpdateState(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - var expUpdPeer netmapclient.UpdatePeerPrm - expUpdPeer.SetMaintenance() - expUpdPeer.SetOnline() - expUpdPeer.SetKey(ev.PubKey.Bytes()) - - require.EqualValues(t, []netmapclient.UpdatePeerPrm{expUpdPeer}, nc.peerStateUpdates, "invalid peer state updates") - }) + require.EqualValues(t, []*transaction.Transaction{ev.Request.MainTransaction}, nc.invokedTxs, "invalid transactions") } func TestCleanupTick(t *testing.T) { t.Parallel() - t.Run("notary disabled", func(t *testing.T) { - t.Parallel() - - nc := &testNetmapClient{} - - proc, err := newTestProc(t, func(p *Params) { + nc := &testNetmapClient{ + contractAddress: util.Uint160{111}, + } + proc, err := newTestProc(t, + func(p *Params) { p.NetmapClient = nc - p.NotaryDisabled = true p.CleanupEnabled = true - }) + }, + ) - require.NoError(t, err, "failed to create processor") + require.NoError(t, err, "failed to create processor") - key1Str := "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35" - proc.netmapSnapshot.lastAccess[key1Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 95, - removeFlag: false, - }, - } - key2Str := "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3" - proc.netmapSnapshot.lastAccess[key2Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 98, - removeFlag: false, - }, - } + key1Str := "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35" + proc.netmapSnapshot.lastAccess[key1Str] = epochStampWithNodeInfo{ + epochStamp: epochStamp{ + epoch: 95, + removeFlag: false, + }, + } + key2Str := "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3" + proc.netmapSnapshot.lastAccess[key2Str] = epochStampWithNodeInfo{ + epochStamp: epochStamp{ + epoch: 98, + removeFlag: false, + }, + } - ev := netmapCleanupTick{ - epoch: 100, - txHash: util.Uint256{123}, - } + ev := netmapCleanupTick{ + epoch: 100, + txHash: util.Uint256{123}, + } - proc.handleCleanupTick(ev) + proc.handleCleanupTick(ev) - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } - keyExp, err := keys.NewPublicKeyFromString(key1Str) - require.NoError(t, err, "failed to parse expired key") + keyExp, err := keys.NewPublicKeyFromString(key1Str) + require.NoError(t, err, "failed to parse expired key") - updExp := netmapclient.UpdatePeerPrm{} - updExp.SetKey(keyExp.Bytes()) - updExp.SetHash(ev.TxHash()) + updExp := netmapclient.UpdatePeerPrm{} + updExp.SetKey(keyExp.Bytes()) + updExp.SetHash(ev.TxHash()) - require.EqualValues(t, []netmapclient.UpdatePeerPrm{updExp}, nc.peerStateUpdates, "invalid peer updates") - require.True(t, proc.netmapSnapshot.lastAccess[key1Str].removeFlag, "invalid expired removed flag") - require.False(t, proc.netmapSnapshot.lastAccess[key2Str].removeFlag, "invalid non expired removed flag") - }) - - t.Run("notary enabled", func(t *testing.T) { - t.Parallel() - - nc := &testNetmapClient{ - contractAddress: util.Uint160{111}, - } - proc, err := newTestProc(t, - func(p *Params) { - p.NetmapClient = nc - p.CleanupEnabled = true - }, - ) - - require.NoError(t, err, "failed to create processor") - - key1Str := "038c862959e56b43e20f79187c4fe9e0bc7c8c66c1603e6cf0ec7f87ab6b08dc35" - proc.netmapSnapshot.lastAccess[key1Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 95, - removeFlag: false, - }, - } - key2Str := "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3" - proc.netmapSnapshot.lastAccess[key2Str] = epochStampWithNodeInfo{ - epochStamp: epochStamp{ - epoch: 98, - removeFlag: false, - }, - } - - ev := netmapCleanupTick{ - epoch: 100, - txHash: util.Uint256{123}, - } - - proc.handleCleanupTick(ev) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - keyExp, err := keys.NewPublicKeyFromString(key1Str) - require.NoError(t, err, "failed to parse expired key") - - updExp := netmapclient.UpdatePeerPrm{} - updExp.SetKey(keyExp.Bytes()) - updExp.SetHash(ev.TxHash()) - - require.EqualValues(t, []notaryInvoke{ - { - contract: nc.contractAddress, - fee: 0, - nonce: uint32(ev.epoch), - vub: nil, - method: "updateStateIR", - args: []any{int64(v2netmap.Offline), keyExp.Bytes()}, - }, - }, nc.notaryInvokes, "invalid notary invokes") - require.True(t, proc.netmapSnapshot.lastAccess[key1Str].removeFlag, "invalid expired removed flag") - require.False(t, proc.netmapSnapshot.lastAccess[key2Str].removeFlag, "invalid non expired removed flag") - }) + require.EqualValues(t, []notaryInvoke{ + { + contract: nc.contractAddress, + fee: 0, + nonce: uint32(ev.epoch), + vub: nil, + method: "updateStateIR", + args: []any{int64(v2netmap.Offline), keyExp.Bytes()}, + }, + }, nc.notaryInvokes, "invalid notary invokes") + require.True(t, proc.netmapSnapshot.lastAccess[key1Str].removeFlag, "invalid expired removed flag") + require.False(t, proc.netmapSnapshot.lastAccess[key2Str].removeFlag, "invalid non expired removed flag") } func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { @@ -407,7 +268,6 @@ func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { PoolSize: 1, CleanupEnabled: false, CleanupThreshold: 3, - NotaryDisabled: false, NodeStateSettings: ns, NodeValidator: &testValidator{}, EpochState: es, @@ -500,17 +360,11 @@ type testNetmapClient struct { netmap *netmap.NetMap txHeights map[util.Uint256]uint32 - peerStateUpdates []netmapclient.UpdatePeerPrm - notaryInvokes []notaryInvoke - newEpochs []uint64 - addPeers []netmapclient.AddPeerPrm - invokedTxs []*transaction.Transaction + notaryInvokes []notaryInvoke + newEpochs []uint64 + invokedTxs []*transaction.Transaction } -func (c *testNetmapClient) UpdatePeerState(p netmapclient.UpdatePeerPrm) error { - c.peerStateUpdates = append(c.peerStateUpdates, p) - return nil -} func (c *testNetmapClient) MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { c.notaryInvokes = append(c.notaryInvokes, notaryInvoke{ contract: contract, @@ -522,32 +376,35 @@ func (c *testNetmapClient) MorphNotaryInvoke(contract util.Uint160, fee fixedn.F }) return nil } + func (c *testNetmapClient) ContractAddress() util.Uint160 { return c.contractAddress } + func (c *testNetmapClient) EpochDuration() (uint64, error) { return c.epochDuration, nil } + func (c *testNetmapClient) MorphTxHeight(h util.Uint256) (uint32, error) { if res, found := c.txHeights[h]; found { return res, nil } return 0, fmt.Errorf("not found") } + func (c *testNetmapClient) NetMap() (*netmap.NetMap, error) { return c.netmap, nil } + func (c *testNetmapClient) NewEpoch(epoch uint64, force bool) error { c.newEpochs = append(c.newEpochs, epoch) return nil } + func (c *testNetmapClient) MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) { return true, nil } -func (c *testNetmapClient) AddPeer(p netmapclient.AddPeerPrm) error { - c.addPeers = append(c.addPeers, p) - return nil -} + func (c *testNetmapClient) MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error { c.invokedTxs = append(c.invokedTxs, mainTx) return nil diff --git a/pkg/innerring/processors/netmap/process_cleanup.go b/pkg/innerring/processors/netmap/process_cleanup.go index 45a08b377..287844a6a 100644 --- a/pkg/innerring/processors/netmap/process_cleanup.go +++ b/pkg/innerring/processors/netmap/process_cleanup.go @@ -3,7 +3,6 @@ package netmap import ( v2netmap "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.uber.org/zap" ) @@ -31,23 +30,14 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { // See https://github.com/nspcc-dev/frostfs-contract/issues/225 const methodUpdateStateNotary = "updateStateIR" - if np.notaryDisabled { - prm := netmapclient.UpdatePeerPrm{} - - prm.SetKey(key.Bytes()) - prm.SetHash(ev.TxHash()) - - err = np.netmapClient.UpdatePeerState(prm) - } else { - err = np.netmapClient.MorphNotaryInvoke( - np.netmapClient.ContractAddress(), - 0, - uint32(ev.epoch), - nil, - methodUpdateStateNotary, - int64(v2netmap.Offline), key.Bytes(), - ) - } + err = np.netmapClient.MorphNotaryInvoke( + np.netmapClient.ContractAddress(), + 0, + uint32(ev.epoch), + nil, + methodUpdateStateNotary, + int64(v2netmap.Offline), key.Bytes(), + ) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapUpdateState, zap.Error(err)) } diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index 9e6eeb53e..e4f1a4d6c 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -19,16 +19,14 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { } // check if notary transaction is valid, see #976 - if originalRequest := ev.NotaryRequest(); originalRequest != nil { - tx := originalRequest.MainTransaction - ok, err := np.netmapClient.MorphIsValidScript(tx.Script, tx.Signers) - if err != nil || !ok { - np.log.Warn(logs.NetmapNonhaltNotaryTransaction, - zap.String("method", "netmap.AddPeer"), - zap.String("hash", tx.Hash().StringLE()), - zap.Error(err)) - return - } + tx := ev.NotaryRequest().MainTransaction + ok, err := np.netmapClient.MorphIsValidScript(tx.Script, tx.Signers) + if err != nil || !ok { + np.log.Warn(logs.NetmapNonhaltNotaryTransaction, + zap.String("method", "netmap.AddPeer"), + zap.String("hash", tx.Hash().StringLE()), + zap.Error(err)) + return } // unmarshal node info @@ -40,7 +38,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { } // validate and update node info - err := np.nodeValidator.VerifyAndUpdate(&nodeInfo) + err = np.nodeValidator.VerifyAndUpdate(&nodeInfo) if err != nil { np.log.Warn(logs.NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate, zap.String("error", err.Error()), @@ -71,20 +69,15 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { // See https://github.com/nspcc-dev/frostfs-contract/issues/154. const methodAddPeerNotary = "addPeerIR" - if nr := ev.NotaryRequest(); nr != nil { - // create new notary request with the original nonce - err = np.netmapClient.MorphNotaryInvoke( - np.netmapClient.ContractAddress(), - 0, - nr.MainTransaction.Nonce, - nil, - methodAddPeerNotary, - nodeInfoBinary, - ) - } else { - // notification event case - err = np.netmapClient.AddPeer(prm) - } + // create new notary request with the original nonce + err = np.netmapClient.MorphNotaryInvoke( + np.netmapClient.ContractAddress(), + 0, + ev.NotaryRequest().MainTransaction.Nonce, + nil, + methodAddPeerNotary, + nodeInfoBinary, + ) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapAddPeer, zap.Error(err)) @@ -116,23 +109,7 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { } } - if nr := ev.NotaryRequest(); nr != nil { - err = np.netmapClient.MorphNotarySignAndInvokeTX(nr.MainTransaction) - } else { - prm := netmapclient.UpdatePeerPrm{} - - switch { - case ev.Online(): - prm.SetOnline() - case ev.Maintenance(): - prm.SetMaintenance() - } - - prm.SetKey(ev.PublicKey().Bytes()) - - err = np.netmapClient.UpdatePeerState(prm) - } - if err != nil { + if err = np.netmapClient.MorphNotarySignAndInvokeTX(ev.NotaryRequest().MainTransaction); err != nil { np.log.Error(logs.NetmapCantInvokeNetmapUpdatePeer, zap.Error(err)) } } diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index c466cfb1b..5984cbbe9 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -55,7 +54,6 @@ type ( } Client interface { - UpdatePeerState(p netmapclient.UpdatePeerPrm) error MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error ContractAddress() util.Uint160 EpochDuration() (uint64, error) @@ -63,7 +61,6 @@ type ( NetMap() (*netmap.NetMap, error) NewEpoch(epoch uint64, force bool) error MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) - AddPeer(p netmapclient.AddPeerPrm) error MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error } @@ -90,8 +87,6 @@ type ( nodeValidator NodeValidator - notaryDisabled bool - nodeStateSettings state.NetworkSettings } @@ -112,8 +107,6 @@ type ( NodeValidator NodeValidator - NotaryDisabled bool - NodeStateSettings state.NetworkSettings } ) @@ -168,8 +161,6 @@ func New(p *Params) (*Processor, error) { nodeValidator: p.NodeValidator, - notaryDisabled: p.NotaryDisabled, - nodeStateSettings: p.NodeStateSettings, }, nil } From 81718afb397c8598bb43eb28d0d4cd7e9522b03a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 17 May 2023 17:41:28 +0300 Subject: [PATCH 0269/1943] [#338] ir: Drop named named put fee Named put fee value used only when notary disabled. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/defaults.go | 5 ++-- config/example/ir.env | 1 - config/example/ir.yaml | 1 - pkg/innerring/config/fee.go | 13 ++--------- pkg/innerring/config/fee_test.go | 4 ---- pkg/innerring/initialization.go | 8 ------- pkg/morph/client/container/client.go | 15 ------------ pkg/morph/client/fee.go | 35 ---------------------------- pkg/morph/client/fee_test.go | 32 ------------------------- pkg/morph/client/static.go | 20 ++++------------ 10 files changed, 9 insertions(+), 125 deletions(-) delete mode 100644 pkg/morph/client/fee.go delete mode 100644 pkg/morph/client/fee_test.go diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index 007bb8964..a7fe8d563 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -51,9 +51,8 @@ func setControlDefaults(cfg *viper.Viper) { func setFeeDefaults(cfg *viper.Viper) { // extra fee values for working mode without notary contract - cfg.SetDefault("fee.main_chain", 5000_0000) // 0.5 Fixed8 - cfg.SetDefault("fee.side_chain", 2_0000_0000) // 2.0 Fixed8 - cfg.SetDefault("fee.named_container_register", 25_0000_0000) // 25.0 Fixed8 + cfg.SetDefault("fee.main_chain", 5000_0000) // 0.5 Fixed8 + cfg.SetDefault("fee.side_chain", 2_0000_0000) // 2.0 Fixed8 } func setEmitDefaults(cfg *viper.Viper) { diff --git a/config/example/ir.env b/config/example/ir.env index 7b8f8a89d..3f9530ab6 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -28,7 +28,6 @@ FROSTFS_IR_LOCODE_DB_PATH=/path/to/locode.db FROSTFS_IR_FEE_MAIN_CHAIN=50000000 FROSTFS_IR_FEE_SIDE_CHAIN=200000000 -FROSTFS_IR_FEE_NAMED_CONTAINER_REGISTER=2500000000 FROSTFS_IR_TIMERS_EMIT=240 FROSTFS_IR_TIMERS_STOP_ESTIMATION_MUL=1 diff --git a/config/example/ir.yaml b/config/example/ir.yaml index 1130a840b..a01f3d0bb 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -49,7 +49,6 @@ locode: fee: main_chain: 50000000 # Fixed8 value of extra GAS fee for mainchain contract invocation; ignore if notary is enabled in mainchain side_chain: 200000000 # Fixed8 value of extra GAS fee for sidechain contract invocation; ignore if notary is enabled in sidechain - named_container_register: 2500000000 # Fixed8 value of extra GAS fee for named conatiner registration in container contract; ignore if notary is enabled in sidechain timers: emit: 240 # Number of sidechain blocks between GAS emission cycles; disabled by default diff --git a/pkg/innerring/config/fee.go b/pkg/innerring/config/fee.go index d77685643..a26a7bcc6 100644 --- a/pkg/innerring/config/fee.go +++ b/pkg/innerring/config/fee.go @@ -8,19 +8,15 @@ import ( // FeeConfig is an instance that returns extra fee values for contract // invocations without notary support. type FeeConfig struct { - registerNamedCnr, mainchain, sidechain fixedn.Fixed8 } // NewFeeConfig constructs FeeConfig from viper.Viper instance. Latter must not be nil. -// -// Fee for named container registration is taken from "fee.named_container_register" value. func NewFeeConfig(v *viper.Viper) *FeeConfig { return &FeeConfig{ - registerNamedCnr: fixedn.Fixed8(v.GetInt64("fee.named_container_register")), - mainchain: fixedn.Fixed8(v.GetInt64("fee.main_chain")), - sidechain: fixedn.Fixed8(v.GetInt64("fee.side_chain")), + mainchain: fixedn.Fixed8(v.GetInt64("fee.main_chain")), + sidechain: fixedn.Fixed8(v.GetInt64("fee.side_chain")), } } @@ -31,8 +27,3 @@ func (f FeeConfig) MainChainFee() fixedn.Fixed8 { func (f FeeConfig) SideChainFee() fixedn.Fixed8 { return f.sidechain } - -// NamedContainerRegistrationFee returns additional GAS fee for named container registration in FrostFS network. -func (f FeeConfig) NamedContainerRegistrationFee() fixedn.Fixed8 { - return f.registerNamedCnr -} diff --git a/pkg/innerring/config/fee_test.go b/pkg/innerring/config/fee_test.go index a0c56aac1..f7330c6ca 100644 --- a/pkg/innerring/config/fee_test.go +++ b/pkg/innerring/config/fee_test.go @@ -18,7 +18,6 @@ func TestConfig(t *testing.T) { fee: main_chain: 50000000 side_chain: 200000000 - named_container_register: 2500000000 `, ) v := viper.New() @@ -29,7 +28,6 @@ fee: config := NewFeeConfig(v) require.Equal(t, fixedn.Fixed8(50000000), config.MainChainFee(), "main chain fee invalid") require.Equal(t, fixedn.Fixed8(200000000), config.SideChainFee(), "side chain fee invalid") - require.Equal(t, fixedn.Fixed8(2500000000), config.NamedContainerRegistrationFee(), "named container register fee invalid") }) t.Run("nothing set", func(t *testing.T) { @@ -43,7 +41,6 @@ fee: config := NewFeeConfig(v) require.Equal(t, fixedn.Fixed8(0), config.MainChainFee(), "main chain fee invalid") require.Equal(t, fixedn.Fixed8(0), config.SideChainFee(), "side chain fee invalid") - require.Equal(t, fixedn.Fixed8(0), config.NamedContainerRegistrationFee(), "named container register fee invalid") }) t.Run("partially set", func(t *testing.T) { @@ -62,7 +59,6 @@ fee: config := NewFeeConfig(v) require.Equal(t, fixedn.Fixed8(10), config.MainChainFee(), "main chain fee invalid") require.Equal(t, fixedn.Fixed8(0), config.SideChainFee(), "side chain fee invalid") - require.Equal(t, fixedn.Fixed8(0), config.NamedContainerRegistrationFee(), "named container register fee invalid") }) } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 50ba67124..9b5634382 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -387,14 +387,6 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { container.AsAlphabet(), ) - if s.sideNotaryConfig.disabled { - // in non-notary environments we customize fee for named container registration - // because it takes much more additional GAS than other operations. - morphCnrOpts = append(morphCnrOpts, - container.WithCustomFeeForNamedPut(s.feeConfig.NamedContainerRegistrationFee()), - ) - } - result.CnrClient, err = container.NewFromMorph(s.morphClient, s.contracts.container, fee, morphCnrOpts...) if err != nil { return nil, err diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index 85d742328..2b5996cd7 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -56,10 +56,6 @@ func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8, opts[i](o) } - if o.feePutNamedSet { - o.staticOpts = append(o.staticOpts, client.WithCustomFee(putNamedMethod, o.feePutNamed)) - } - sc, err := client.NewStatic(cli, contract, fee, o.staticOpts...) if err != nil { return nil, fmt.Errorf("can't create container static client: %w", err) @@ -83,9 +79,6 @@ func (c Client) ContractAddress() util.Uint160 { type Option func(*opts) type opts struct { - feePutNamedSet bool - feePutNamed fixedn.Fixed8 - staticOpts []client.StaticClientOption } @@ -111,11 +104,3 @@ func AsAlphabet() Option { o.staticOpts = append(o.staticOpts, client.AsAlphabet()) } } - -// WithCustomFeeForNamedPut returns option to specify custom fee for each Put operation with named container. -func WithCustomFeeForNamedPut(fee fixedn.Fixed8) Option { - return func(o *opts) { - o.feePutNamed = fee - o.feePutNamedSet = true - } -} diff --git a/pkg/morph/client/fee.go b/pkg/morph/client/fee.go deleted file mode 100644 index 8a38c4f55..000000000 --- a/pkg/morph/client/fee.go +++ /dev/null @@ -1,35 +0,0 @@ -package client - -import "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - -// setFeeForMethod sets fee for the operation executed using specified contract method. -func (x *fees) setFeeForMethod(method string, fee fixedn.Fixed8) { - if x.customFees == nil { - x.customFees = make(map[string]fixedn.Fixed8, 1) - } - - x.customFees[method] = fee -} - -// fees represents source of per-operation fees. -// Can be initialized using var declaration. -// -// Instances are not thread-safe, so they mean initially filling, and then only reading. -type fees struct { - defaultFee fixedn.Fixed8 - - // customFees represents source of customized per-operation fees. - customFees map[string]fixedn.Fixed8 -} - -// returns fee for the operation executed using specified contract method. -// Returns customized value if it is set. Otherwise, returns default value. -func (x fees) feeForMethod(method string) fixedn.Fixed8 { - if x.customFees != nil { - if fee, ok := x.customFees[method]; ok { - return fee - } - } - - return x.defaultFee -} diff --git a/pkg/morph/client/fee_test.go b/pkg/morph/client/fee_test.go deleted file mode 100644 index 963d64ce4..000000000 --- a/pkg/morph/client/fee_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package client - -import ( - "testing" - - "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - "github.com/stretchr/testify/require" -) - -func TestFees(t *testing.T) { - var v fees - - const method = "some method" - - var ( - fee fixedn.Fixed8 - def = fixedn.Fixed8(13) - ) - - v.defaultFee = def - - fee = v.feeForMethod(method) - require.True(t, fee.Equal(def)) - - const customFee = fixedn.Fixed8(10) - - v.setFeeForMethod(method, customFee) - - fee = v.feeForMethod(method) - - require.Equal(t, customFee, fee) -} diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index 910f78537..7aa17a70f 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -27,7 +27,7 @@ type staticOpts struct { tryNotary bool alpha bool // use client's key to sign notary request's main TX - fees fees + fee fixedn.Fixed8 } // WithNotary returns notary status of the client. @@ -63,7 +63,7 @@ func NewStatic(client *Client, scriptHash util.Uint160, fee fixedn.Fixed8, opts scScriptHash: scriptHash, } - c.fees.defaultFee = fee + c.fee = fee for i := range opts { opts[i](&c.staticOpts) @@ -125,8 +125,6 @@ func (i *InvokePrmOptional) SetControlTX(b bool) { // If fee for the operation executed using specified method is customized, then StaticClient uses it. // Otherwise, default fee is used. func (s StaticClient) Invoke(prm InvokePrm) error { - fee := s.fees.feeForMethod(prm.method) - if s.tryNotary { if s.alpha { var ( @@ -149,15 +147,15 @@ func (s StaticClient) Invoke(prm InvokePrm) error { vubP = &vub } - return s.client.NotaryInvoke(s.scScriptHash, fee, nonce, vubP, prm.method, prm.args...) + return s.client.NotaryInvoke(s.scScriptHash, s.fee, nonce, vubP, prm.method, prm.args...) } - return s.client.NotaryInvokeNotAlpha(s.scScriptHash, fee, prm.method, prm.args...) + return s.client.NotaryInvokeNotAlpha(s.scScriptHash, s.fee, prm.method, prm.args...) } return s.client.Invoke( s.scScriptHash, - fee, + s.fee, prm.method, prm.args..., ) @@ -211,11 +209,3 @@ func AsAlphabet() StaticClientOption { o.alpha = true } } - -// WithCustomFee returns option to specify custom fee for the operation executed using -// specified contract method. -func WithCustomFee(method string, fee fixedn.Fixed8) StaticClientOption { - return func(o *staticOpts) { - o.fees.setFeeForMethod(method, fee) - } -} From 0ab589dd52e80f0fd3ab562b77b82761e1527397 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 17 May 2023 17:56:11 +0300 Subject: [PATCH 0270/1943] [#338] ir: Always enable notary on sidechain Signed-off-by: Dmitrii Stepanov --- pkg/innerring/contracts.go | 10 ++++------ pkg/innerring/contracts_test.go | 18 ++++++++++-------- pkg/innerring/initialization.go | 24 ++++++++++-------------- pkg/innerring/innerring.go | 14 +++++--------- pkg/innerring/notary.go | 17 +++-------------- 5 files changed, 32 insertions(+), 51 deletions(-) diff --git a/pkg/innerring/contracts.go b/pkg/innerring/contracts.go index 55c2ff582..4a80296f4 100644 --- a/pkg/innerring/contracts.go +++ b/pkg/innerring/contracts.go @@ -22,7 +22,7 @@ type contracts struct { alphabet AlphabetContracts // in morph } -func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, withoutMainNotary, withoutSideNotary bool) (*contracts, error) { +func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, withoutMainNotary bool) (*contracts, error) { var ( result = new(contracts) err error @@ -42,11 +42,9 @@ func parseContracts(cfg *viper.Viper, morph nnsResolver, withoutMainNet, without } } - if !withoutSideNotary { - result.proxy, err = parseContract(cfg, morph, "contracts.proxy", client.NNSProxyContractName) - if err != nil { - return nil, fmt.Errorf("can't get proxy script hash: %w", err) - } + result.proxy, err = parseContract(cfg, morph, "contracts.proxy", client.NNSProxyContractName) + if err != nil { + return nil, fmt.Errorf("can't get proxy script hash: %w", err) } targets := [...]struct { diff --git a/pkg/innerring/contracts_test.go b/pkg/innerring/contracts_test.go index e04e22f06..5f877133c 100644 --- a/pkg/innerring/contracts_test.go +++ b/pkg/innerring/contracts_test.go @@ -35,7 +35,7 @@ contracts: t.Run("all enabled", func(t *testing.T) { t.Parallel() - c, err := parseContracts(v, nil, false, false, false) + c, err := parseContracts(v, nil, false, false) require.NoError(t, err, "failed to parse contracts") frostfsExp, _ := util.Uint160DecodeStringLE("ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62") @@ -70,7 +70,7 @@ contracts: t.Run("all disabled", func(t *testing.T) { t.Parallel() - c, err := parseContracts(v, nil, true, true, true) + c, err := parseContracts(v, nil, true, true) require.NoError(t, err, "failed to parse contracts") require.Equal(t, util.Uint160{}, c.frostfs, "invalid frostfs") @@ -89,7 +89,8 @@ contracts: netmapIDExp, _ := util.Uint160DecodeStringLE("83c600c81d47a1b1b7cf58eb49ae7ee7240dc742") require.Equal(t, netmapIDExp, c.netmap, "invalid netmap") - require.Equal(t, util.Uint160{}, c.proxy, "invalid proxy") + proxyExp, _ := util.Uint160DecodeStringLE("abc8794bb40a21f2db5f21ae62741eb46c8cad1c") + require.Equal(t, proxyExp, c.proxy, "invalid proxy") require.Equal(t, 2, len(c.alphabet), "invalid alphabet contracts length") @@ -100,9 +101,9 @@ contracts: require.Equal(t, bukyExp, c.alphabet[buky], "invalid buky") }) - t.Run("main notary & side notary disabled", func(t *testing.T) { + t.Run("main notary disabled", func(t *testing.T) { t.Parallel() - c, err := parseContracts(v, nil, false, true, true) + c, err := parseContracts(v, nil, false, true) require.NoError(t, err, "failed to parse contracts") frostfsExp, _ := util.Uint160DecodeStringLE("ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62") @@ -122,7 +123,8 @@ contracts: netmapIDExp, _ := util.Uint160DecodeStringLE("83c600c81d47a1b1b7cf58eb49ae7ee7240dc742") require.Equal(t, netmapIDExp, c.netmap, "invalid netmap") - require.Equal(t, util.Uint160{}, c.proxy, "invalid proxy") + proxyExp, _ := util.Uint160DecodeStringLE("abc8794bb40a21f2db5f21ae62741eb46c8cad1c") + require.Equal(t, proxyExp, c.proxy, "invalid proxy") require.Equal(t, 2, len(c.alphabet), "invalid alphabet contracts length") @@ -159,7 +161,7 @@ contracts: err := v.ReadConfig(file) require.NoError(t, err, "read config file failed") - _, err = parseContracts(v, nil, false, false, false) + _, err = parseContracts(v, nil, false, false) require.Error(t, err, "unexpected success") }) @@ -196,7 +198,7 @@ contracts: }, } - _, err = parseContracts(v, morph, false, false, false) + _, err = parseContracts(v, morph, false, false) require.ErrorContains(t, err, "could not read all contracts: required 3, read 2", "unexpected success") }) } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 9b5634382..076d7be43 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -116,18 +116,16 @@ func (s *Server) initMainnet(ctx context.Context, cfg *viper.Viper, morphChain * } func (s *Server) enableNotarySupport() error { - if !s.sideNotaryConfig.disabled { - // enable notary support in the side client - err := s.morphClient.EnableNotarySupport( - client.WithProxyContract(s.contracts.proxy), - ) - if err != nil { - return fmt.Errorf("could not enable side chain notary support: %w", err) - } - - s.morphListener.EnableNotarySupport(s.contracts.proxy, s.morphClient.Committee, s.morphClient) + // enable notary support in the side client + err := s.morphClient.EnableNotarySupport( + client.WithProxyContract(s.contracts.proxy), + ) + if err != nil { + return fmt.Errorf("could not enable side chain notary support: %w", err) } + s.morphListener.EnableNotarySupport(s.contracts.proxy, s.morphClient.Committee, s.morphClient) + if !s.mainNotaryConfig.disabled { // enable notary support in the main client err := s.mainnetClient.EnableNotarySupport( @@ -143,13 +141,12 @@ func (s *Server) enableNotarySupport() error { } func (s *Server) initNotaryConfig() { - s.mainNotaryConfig, s.sideNotaryConfig = notaryConfigs( - s.morphClient.ProbeNotary(), + s.mainNotaryConfig = notaryConfigs( !s.withoutMainNet && s.mainnetClient.ProbeNotary(), // if mainnet disabled then notary flag must be disabled too ) s.log.Info(logs.InnerringNotarySupport, - zap.Bool("sidechain_enabled", !s.sideNotaryConfig.disabled), + zap.Bool("sidechain_enabled", true), zap.Bool("mainchain_enabled", !s.mainNotaryConfig.disabled), ) } @@ -500,7 +497,6 @@ func (s *Server) initContracts(cfg *viper.Viper) error { s.morphClient, s.withoutMainNet, s.mainNotaryConfig.disabled, - s.sideNotaryConfig.disabled, ) return err diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 17acdedda..3a690bbbe 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -63,7 +63,6 @@ type ( // notary configuration feeConfig *config.FeeConfig mainNotaryConfig *notaryConfig - sideNotaryConfig *notaryConfig // internal variables key *keys.PrivateKey @@ -267,14 +266,11 @@ func (s *Server) initMainNotary(ctx context.Context) error { } func (s *Server) initSideNotary(ctx context.Context) error { - if !s.sideNotaryConfig.disabled { - return s.initNotary(ctx, - s.depositSideNotary, - s.awaitSideNotaryDeposit, - "waiting to accept side notary deposit", - ) - } - return nil + return s.initNotary(ctx, + s.depositSideNotary, + s.awaitSideNotaryDeposit, + "waiting to accept side notary deposit", + ) } func (s *Server) tickInitialExpoch() { diff --git a/pkg/innerring/notary.go b/pkg/innerring/notary.go index 30916cb99..c601f5587 100644 --- a/pkg/innerring/notary.go +++ b/pkg/innerring/notary.go @@ -57,11 +57,8 @@ func (s *Server) notaryHandler(_ event.Event) { } } - if !s.sideNotaryConfig.disabled { - _, err := s.depositSideNotary() - if err != nil { - s.log.Error(logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) - } + if _, err := s.depositSideNotary(); err != nil { + s.log.Error(logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) } } @@ -115,16 +112,8 @@ func awaitNotaryDepositInClient(ctx context.Context, cli *client.Client, txHash return errDepositTimeout } -func notaryConfigs(withSideNotary, withMainNotary bool) (main, side *notaryConfig) { +func notaryConfigs(withMainNotary bool) (main *notaryConfig) { main = new(notaryConfig) - side = new(notaryConfig) - - if !withSideNotary { - main.disabled = true - side.disabled = true - - return - } main.disabled = !withMainNotary From 53a1b15693e3a14ec6343c2ef02dd080c6b752d8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 18 May 2023 11:19:08 +0300 Subject: [PATCH 0271/1943] [#338] ir: Drop notaryless code from governance Signed-off-by: Dmitrii Stepanov --- pkg/innerring/initialization.go | 19 +++---- .../processors/governance/handlers_test.go | 56 +++++++++---------- .../processors/governance/process_update.go | 25 ++------- .../processors/governance/processor.go | 27 ++++----- 4 files changed, 52 insertions(+), 75 deletions(-) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 076d7be43..8e5aef951 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -161,16 +161,15 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli } else { // create governance processor governanceProcessor, err := governance.New(&governance.Params{ - Log: s.log, - FrostFSClient: frostfsCli, - NetmapClient: s.netmapClient, - AlphabetState: s, - EpochState: s, - Voter: s, - IRFetcher: irf, - MorphClient: s.morphClient, - MainnetClient: s.mainnetClient, - NotaryDisabled: s.sideNotaryConfig.disabled, + Log: s.log, + FrostFSClient: frostfsCli, + NetmapClient: s.netmapClient, + AlphabetState: s, + EpochState: s, + Voter: s, + IRFetcher: irf, + MorphClient: s.morphClient, + MainnetClient: s.mainnetClient, }) if err != nil { return nil, err diff --git a/pkg/innerring/processors/governance/handlers_test.go b/pkg/innerring/processors/governance/handlers_test.go index 23f91869a..63d156dac 100644 --- a/pkg/innerring/processors/governance/handlers_test.go +++ b/pkg/innerring/processors/governance/handlers_test.go @@ -42,16 +42,15 @@ func TestHandleAlphabetSyncEvent(t *testing.T) { proc, err := New( &Params{ - Log: test.NewLogger(t, true), - EpochState: es, - AlphabetState: as, - Voter: v, - IRFetcher: irf, - NotaryDisabled: true, - MorphClient: m, - MainnetClient: mn, - FrostFSClient: f, - NetmapClient: nm, + Log: test.NewLogger(t, true), + EpochState: es, + AlphabetState: as, + Voter: v, + IRFetcher: irf, + MorphClient: m, + MainnetClient: mn, + FrostFSClient: f, + NetmapClient: nm, }, ) @@ -74,17 +73,19 @@ func TestHandleAlphabetSyncEvent(t *testing.T) { }, }, v.votes, "invalid vote calls") - var irUpdateExp nmClient.UpdateIRPrm - irUpdateExp.SetKeys(testKeys.newInnerRingExp) - irUpdateExp.SetHash(ev.txHash) + var irUpdateExp []nmClient.UpdateIRPrm - require.EqualValues(t, []nmClient.UpdateIRPrm{irUpdateExp}, nm.updates, "invalid IR updates") + require.EqualValues(t, irUpdateExp, nm.updates, "invalid IR updates") - var expAlphabetUpdates []client.UpdateAlphabetListPrm - require.EqualValues(t, expAlphabetUpdates, m.alphabetUpdates, "invalid alphabet updates") + var expAlphabetUpdate client.UpdateAlphabetListPrm + expAlphabetUpdate.SetHash(ev.txHash) + expAlphabetUpdate.SetList(testKeys.newInnerRingExp) + require.EqualValues(t, []client.UpdateAlphabetListPrm{expAlphabetUpdate}, m.alphabetUpdates, "invalid alphabet updates") - var expNotaryUpdates []client.UpdateNotaryListPrm - require.EqualValues(t, expNotaryUpdates, m.notaryUpdates, "invalid notary list updates") + var expNotaryUpdate client.UpdateNotaryListPrm + expNotaryUpdate.SetHash(ev.txHash) + expNotaryUpdate.SetList(testKeys.newAlphabetExp) + require.EqualValues(t, []client.UpdateNotaryListPrm{expNotaryUpdate}, m.notaryUpdates, "invalid notary list updates") buf := make([]byte, 8) binary.LittleEndian.PutUint64(buf, es.epoch) @@ -122,16 +123,15 @@ func TestHandleAlphabetDesignateEvent(t *testing.T) { proc, err := New( &Params{ - Log: test.NewLogger(t, true), - EpochState: es, - AlphabetState: as, - Voter: v, - IRFetcher: irf, - NotaryDisabled: false, - MorphClient: m, - MainnetClient: mn, - FrostFSClient: f, - NetmapClient: nm, + Log: test.NewLogger(t, true), + EpochState: es, + AlphabetState: as, + Voter: v, + IRFetcher: irf, + MorphClient: m, + MainnetClient: mn, + FrostFSClient: f, + NetmapClient: nm, }, ) diff --git a/pkg/innerring/processors/governance/process_update.go b/pkg/innerring/processors/governance/process_update.go index 629d8741e..3eae676d4 100644 --- a/pkg/innerring/processors/governance/process_update.go +++ b/pkg/innerring/processors/governance/process_update.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" - nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" @@ -114,33 +113,17 @@ func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(sidechainAlphabet, newAl zap.String("after", prettyKeys(newInnerRing)), ) - if gp.notaryDisabled { - updPrm := nmClient.UpdateIRPrm{} + updPrm := client.UpdateAlphabetListPrm{} + updPrm.SetList(newInnerRing) + updPrm.SetHash(txHash) - updPrm.SetKeys(newInnerRing) - updPrm.SetHash(txHash) - - err = gp.netmapClient.UpdateInnerRing(updPrm) - } else { - updPrm := client.UpdateAlphabetListPrm{} - - updPrm.SetList(newInnerRing) - updPrm.SetHash(txHash) - - err = gp.morphClient.UpdateNeoFSAlphabetList(updPrm) - } - - if err != nil { + if err = gp.morphClient.UpdateNeoFSAlphabetList(updPrm); err != nil { gp.log.Error(logs.GovernanceCantUpdateInnerRingListWithNewAlphabetKeys, zap.String("error", err.Error())) } } func (gp *Processor) updateNotaryRoleInSidechain(newAlphabet keys.PublicKeys, txHash util.Uint256) { - if gp.notaryDisabled { - return - } - updPrm := client.UpdateNotaryListPrm{} updPrm.SetList(newAlphabet) diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index e08bd3809..07b5b5cef 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -87,8 +87,6 @@ type ( mainnetClient MainnetClient morphClient MorphClient - notaryDisabled bool - designate util.Uint160 } @@ -105,8 +103,6 @@ type ( MainnetClient MainnetClient FrostFSClient FrostFSClient NetmapClient NetmapClient - - NotaryDisabled bool } ) @@ -138,18 +134,17 @@ func New(p *Params) (*Processor, error) { designate := p.MainnetClient.GetDesignateHash() return &Processor{ - log: p.Log, - pool: pool, - frostfsClient: p.FrostFSClient, - netmapClient: p.NetmapClient, - alphabetState: p.AlphabetState, - epochState: p.EpochState, - voter: p.Voter, - irFetcher: p.IRFetcher, - mainnetClient: p.MainnetClient, - morphClient: p.MorphClient, - notaryDisabled: p.NotaryDisabled, - designate: designate, + log: p.Log, + pool: pool, + frostfsClient: p.FrostFSClient, + netmapClient: p.NetmapClient, + alphabetState: p.AlphabetState, + epochState: p.EpochState, + voter: p.Voter, + irFetcher: p.IRFetcher, + mainnetClient: p.MainnetClient, + morphClient: p.MorphClient, + designate: designate, }, nil } From 4503a61997aadaefa4bb3988b5d3cf42b53d95ff Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 18 May 2023 15:38:41 +0300 Subject: [PATCH 0272/1943] [#312] wc: Delete unused Iterate method Signed-off-by: Dmitrii Stepanov --- .../writecache/iterate.go | 54 ------------------- .../writecache/writecache.go | 1 - 2 files changed, 55 deletions(-) diff --git a/pkg/local_object_storage/writecache/iterate.go b/pkg/local_object_storage/writecache/iterate.go index ebe979520..59ace93bd 100644 --- a/pkg/local_object_storage/writecache/iterate.go +++ b/pkg/local_object_storage/writecache/iterate.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -12,59 +11,6 @@ import ( // ErrNoDefaultBucket is returned by IterateDB when default bucket for objects is missing. var ErrNoDefaultBucket = errors.New("no default bucket") -// IterationPrm contains iteration parameters. -type IterationPrm struct { - handler func([]byte) error - ignoreErrors bool -} - -// WithHandler sets a callback to be executed on every object. -func (p *IterationPrm) WithHandler(f func([]byte) error) { - p.handler = f -} - -// WithIgnoreErrors sets a flag indicating that errors should be ignored. -func (p *IterationPrm) WithIgnoreErrors(ignore bool) { - p.ignoreErrors = ignore -} - -// Iterate iterates over all objects present in write cache. -// This is very difficult to do correctly unless write-cache is put in read-only mode. -// Thus we silently fail if shard is not in read-only mode to avoid reporting misleading results. -func (c *cache) Iterate(prm IterationPrm) error { - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - if !c.readOnly() { - return nil - } - - err := c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - return b.ForEach(func(k, data []byte) error { - return prm.handler(data) - }) - }) - if err != nil { - return err - } - - var fsPrm common.IteratePrm - fsPrm.IgnoreErrors = prm.ignoreErrors - fsPrm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { - data, err := f() - if err != nil { - if prm.ignoreErrors { - return nil - } - return err - } - return prm.handler(data) - } - - _, err = c.fsTree.Iterate(fsPrm) - return err -} - // IterateDB iterates over all objects stored in bbolt.DB instance and passes them to f until error return. // It is assumed that db is an underlying database of some WriteCache instance. // diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 83ecf219c..0edf4b9be 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -32,7 +32,6 @@ type Cache interface { // Returns apistatus.ObjectNotFound if object is missing in the Cache. // Returns ErrReadOnly if the Cache is currently in the read-only mode. Delete(context.Context, oid.Address) error - Iterate(IterationPrm) error Put(context.Context, common.PutPrm) (common.PutRes, error) SetMode(mode.Mode) error SetLogger(*logger.Logger) From d212d908b5303b56e8b7eabe82560b4a8f58cf4c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 18 May 2023 17:19:41 +0300 Subject: [PATCH 0273/1943] [#312] wc: Add metrics Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/writecache/delete.go | 19 +++++++-- pkg/local_object_storage/writecache/flush.go | 14 +++++-- pkg/local_object_storage/writecache/get.go | 22 +++++++++- .../writecache/metrics.go | 42 +++++++++++++++++++ pkg/local_object_storage/writecache/mode.go | 6 ++- .../writecache/options.go | 9 ++++ pkg/local_object_storage/writecache/put.go | 23 +++++++++- pkg/local_object_storage/writecache/state.go | 5 ++- .../writecache/storage.go | 2 + .../writecache/writecache.go | 2 + 10 files changed, 130 insertions(+), 14 deletions(-) create mode 100644 pkg/local_object_storage/writecache/metrics.go diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index c1aab9e5a..ed44a8ad8 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -2,6 +2,7 @@ package writecache import ( "context" + "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -22,6 +23,13 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { )) defer span.End() + deleted := false + storageType := storageTypeUndefined + startedAt := time.Now() + defer func() { + c.metrics.Delete(time.Since(startedAt), deleted, storageType) + }() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { @@ -30,15 +38,15 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { saddr := addr.EncodeToString() - // Check disk cache. - var has int + var dataSize int _ = c.db.View(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) - has = len(b.Get([]byte(saddr))) + dataSize = len(b.Get([]byte(saddr))) return nil }) - if 0 < has { + if dataSize > 0 { + storageType = storageTypeDB err := c.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) err := b.Delete([]byte(saddr)) @@ -52,10 +60,12 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("db DELETE"), ) + deleted = true c.objCounters.DecDB() return nil } + storageType = storageTypeFSTree _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err == nil { storagelog.Write(c.log, @@ -64,6 +74,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.OpField("fstree DELETE"), ) c.objCounters.DecFS() + deleted = true } return err diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index c6c8a9465..09c5451ad 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -199,7 +199,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - err = c.flushObject(ctx, &obj, data) + err = c.flushObject(ctx, &obj, data, storageTypeFSTree) if err != nil { if ignoreErrors { return nil @@ -228,7 +228,7 @@ func (c *cache) workerFlushSmall() { return } - err := c.flushObject(context.TODO(), obj, nil) + err := c.flushObject(context.TODO(), obj, nil, storageTypeDB) if err != nil { // Error is handled in flushObject. continue @@ -239,7 +239,13 @@ func (c *cache) workerFlushSmall() { } // flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte) error { +func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte, st storageType) error { + var err error + + defer func() { + c.metrics.Flush(err == nil, st) + }() + addr := objectCore.AddressOf(obj) var prm common.PutPrm @@ -313,7 +319,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { return err } - if err := c.flushObject(ctx, &obj, data); err != nil { + if err := c.flushObject(ctx, &obj, data, storageTypeDB); err != nil { return err } } diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index 030f9b413..f8eb01091 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -2,6 +2,7 @@ package writecache import ( "context" + "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -27,9 +28,22 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e )) defer span.End() + return c.getInternal(ctx, saddr, addr) +} + +func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) (*objectSDK.Object, error) { + found := false + storageType := storageTypeUndefined + startedAt := time.Now() + defer func() { + c.metrics.Get(time.Since(startedAt), found, storageType) + }() + value, err := Get(c.db, []byte(saddr)) if err == nil { obj := objectSDK.New() + found = true + storageType = storageTypeDB return obj, obj.Unmarshal(value) } @@ -38,6 +52,8 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) } + found = true + storageType = storageTypeFSTree return res.Object, nil } @@ -45,13 +61,15 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e // // Returns an error of type apistatus.ObjectNotFound if the requested object is missing in write-cache. func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { + saddr := addr.EncodeToString() + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Head", trace.WithAttributes( - attribute.String("address", addr.EncodeToString()), + attribute.String("address", saddr), )) defer span.End() - obj, err := c.Get(ctx, addr) + obj, err := c.getInternal(ctx, saddr, addr) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go new file mode 100644 index 000000000..afa6f69f1 --- /dev/null +++ b/pkg/local_object_storage/writecache/metrics.go @@ -0,0 +1,42 @@ +package writecache + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) + +type storageType string + +const ( + storageTypeUndefined storageType = "null" + storageTypeDB storageType = "db" + storageTypeFSTree storageType = "fstree" +) + +type Metrics interface { + Get(d time.Duration, success bool, st storageType) + Delete(d time.Duration, success bool, st storageType) + Put(d time.Duration, success bool, st storageType) + Flush(success bool, st storageType) + Evict(st storageType) + + Estimate(db, fstree uint64) + SetMode(m mode.Mode) +} + +type metricsStub struct{} + +func (s *metricsStub) Get(time.Duration, bool, storageType) {} + +func (s *metricsStub) Delete(time.Duration, bool, storageType) {} + +func (s *metricsStub) Put(time.Duration, bool, storageType) {} + +func (s *metricsStub) Estimate(uint64, uint64) {} + +func (s *metricsStub) SetMode(mode.Mode) {} + +func (s *metricsStub) Flush(bool, storageType) {} + +func (s *metricsStub) Evict(storageType) {} diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index ca6faff4c..7e9373a42 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -29,7 +29,11 @@ func (c *cache) SetMode(m mode.Mode) error { )) defer span.End() - return c.setMode(ctx, m) + err := c.setMode(ctx, m) + if err == nil { + c.metrics.SetMode(m) + } + return err } // setMode applies new mode. Must be called with cache.modeMtx lock taken. diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index 3434e9355..bea40aa36 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -60,6 +60,8 @@ type options struct { reportError func(string, error) // openFile is the function called internally by bbolt to open database files. Useful for hermetic testing. openFile func(string, int, fs.FileMode) (*os.File, error) + // metrics is metrics implementation + metrics Metrics } // WithLogger sets logger. @@ -164,3 +166,10 @@ func WithOpenFile(f func(string, int, fs.FileMode) (*os.File, error)) Option { o.openFile = f } } + +// WithMetrics sets metrics implementation. +func WithMetrics(metrics Metrics) Option { + return func(o *options) { + o.metrics = metrics + } +} diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 04d818b31..c0001f926 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -3,6 +3,7 @@ package writecache import ( "context" "errors" + "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -33,6 +34,13 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro )) defer span.End() + startedAt := time.Now() + added := false + storageType := storageTypeUndefined + defer func() { + c.metrics.Put(time.Since(startedAt), added, storageType) + }() + c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { @@ -51,9 +59,20 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro } if sz <= c.smallObjectSize { - return common.PutRes{}, c.putSmall(oi) + storageType = storageTypeDB + err := c.putSmall(oi) + if err == nil { + added = true + } + return common.PutRes{}, err } - return common.PutRes{}, c.putBig(ctx, oi.addr, prm) + + storageType = storageTypeFSTree + err := c.putBig(ctx, oi.addr, prm) + if err == nil { + added = true + } + return common.PutRes{}, err } // putSmall persists small objects to the write-cache database and diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index 9c1c562b0..5f3092526 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -9,7 +9,10 @@ import ( ) func (c *cache) estimateCacheSize() uint64 { - return c.objCounters.DB()*c.smallObjectSize + c.objCounters.FS()*c.maxObjectSize + db := c.objCounters.DB() * c.smallObjectSize + fstree := c.objCounters.FS() * c.maxObjectSize + c.metrics.Estimate(db, fstree) + return db + fstree } func (c *cache) incSizeDB(sz uint64) uint64 { diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index c06d16c0b..50c110c5c 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -79,6 +79,7 @@ func (c *cache) deleteFromDB(keys []string) []string { }) for i := 0; i < errorIndex; i++ { c.objCounters.DecDB() + c.metrics.Evict(storageTypeDB) storagelog.Write(c.log, storagelog.AddressField(keys[i]), storagelog.StorageTypeField(wcStorageType), @@ -121,6 +122,7 @@ func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree DELETE"), ) + c.metrics.Evict(storageTypeFSTree) c.objCounters.DecFS() } } diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 0edf4b9be..664beff80 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -103,6 +103,7 @@ func New(opts ...Option) Cache { maxBatchSize: bbolt.DefaultMaxBatchSize, maxBatchDelay: bbolt.DefaultMaxBatchDelay, openFile: os.OpenFile, + metrics: &metricsStub{}, }, } @@ -140,6 +141,7 @@ func (c *cache) Open(readOnly bool) error { // Init runs necessary services. func (c *cache) Init() error { + c.metrics.SetMode(c.mode) c.runFlushLoop() return nil } From 2ce43935f95e1e6f665affeae9bf4cdcf81e8fff Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 19 May 2023 11:17:19 +0300 Subject: [PATCH 0274/1943] [#312] metrics: Add writecache metrcis Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/metrics.go | 4 + pkg/local_object_storage/engine/shards.go | 7 + pkg/local_object_storage/engine/writecache.go | 53 ++++ pkg/local_object_storage/shard/shard.go | 7 + pkg/local_object_storage/writecache/delete.go | 6 +- pkg/local_object_storage/writecache/flush.go | 8 +- pkg/local_object_storage/writecache/get.go | 6 +- .../writecache/metrics.go | 39 +-- pkg/local_object_storage/writecache/put.go | 6 +- pkg/local_object_storage/writecache/state.go | 3 +- .../writecache/storage.go | 4 +- pkg/metrics/desc.go | 38 +++ pkg/metrics/node.go | 14 + pkg/metrics/writecache.go | 252 ++++++++++++++++++ 14 files changed, 415 insertions(+), 32 deletions(-) create mode 100644 pkg/metrics/writecache.go diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 13dcdfe02..7a11888c5 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -2,6 +2,8 @@ package engine import ( "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) type MetricRegister interface { @@ -24,6 +26,8 @@ type MetricRegister interface { AddToContainerSize(cnrID string, size int64) AddToPayloadCounter(shardID string, size int64) + + WriteCache() metrics.WriteCacheMetrics } func elapsed(addFunc func(d time.Duration)) func() { diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index e16d2c498..6c49c8312 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" "github.com/google/uuid" @@ -98,6 +99,12 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { mw: e.metrics, }, )) + opts = append(opts, shard.WithExtraWriteCacheOptions(writecache.WithMetrics( + &writeCacheMetrics{ + shardID: id.String(), + metrics: e.metrics.WriteCache(), + }, + ))) } e.mtx.RUnlock() diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 4effb2b16..ea2299b8b 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -2,9 +2,13 @@ package engine import ( "context" + "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -52,3 +56,52 @@ func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePr return FlushWriteCacheRes{}, sh.FlushWriteCache(ctx, prm) } + +type writeCacheMetrics struct { + shardID string + metrics metrics.WriteCacheMetrics +} + +func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.StorageType) { + m.metrics.AddGetDuration(m.shardID, success, d) + m.metrics.IncGetCounter(m.shardID, success, st.String()) +} + +func (m *writeCacheMetrics) Delete(d time.Duration, success bool, st writecache.StorageType) { + m.metrics.AddDeleteDuration(m.shardID, success, d) + m.metrics.IncDeleteCounter(m.shardID, success, st.String()) + if success { + m.metrics.DecActualCount(m.shardID, st.String()) + } +} + +func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.StorageType) { + m.metrics.AddPutDuration(m.shardID, success, d) + m.metrics.IncPutCounter(m.shardID, success, st.String()) + if success { + m.metrics.IncActualCount(m.shardID, st.String()) + } +} + +func (m *writeCacheMetrics) SetEstimateSize(db, fstree uint64) { + m.metrics.SetEstimateSize(m.shardID, db, writecache.StorageTypeDB.String()) + m.metrics.SetEstimateSize(m.shardID, fstree, writecache.StorageTypeFSTree.String()) +} + +func (m *writeCacheMetrics) SetMode(mode mode.Mode) { + m.metrics.SetMode(m.shardID, mode.String()) +} + +func (m *writeCacheMetrics) SetActualCounters(db, fstree uint64) { + m.metrics.SetActualCount(m.shardID, db, writecache.StorageTypeDB.String()) + m.metrics.SetActualCount(m.shardID, fstree, writecache.StorageTypeFSTree.String()) +} + +func (m *writeCacheMetrics) Flush(success bool, st writecache.StorageType) { + m.metrics.IncFlushCounter(m.shardID, success, st.String()) +} + +func (m *writeCacheMetrics) Evict(st writecache.StorageType) { + m.metrics.DecActualCount(m.shardID, st.String()) + m.metrics.IncEvictCounter(m.shardID, st.String()) +} diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 44ec54645..e0059105f 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -187,6 +187,13 @@ func WithWriteCacheOptions(opts ...writecache.Option) Option { } } +// WithExtraWriteCacheOptions returns option to add extra write cache options. +func WithExtraWriteCacheOptions(opts ...writecache.Option) Option { + return func(c *cfg) { + c.writeCacheOpts = append(c.writeCacheOpts, opts...) + } +} + // WithPiloramaOptions returns option to set internal write cache options. func WithPiloramaOptions(opts ...pilorama.Option) Option { return func(c *cfg) { diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index ed44a8ad8..f5a292ed4 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -24,7 +24,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { defer span.End() deleted := false - storageType := storageTypeUndefined + storageType := StorageTypeUndefined startedAt := time.Now() defer func() { c.metrics.Delete(time.Since(startedAt), deleted, storageType) @@ -46,7 +46,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { }) if dataSize > 0 { - storageType = storageTypeDB + storageType = StorageTypeDB err := c.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) err := b.Delete([]byte(saddr)) @@ -65,7 +65,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { return nil } - storageType = storageTypeFSTree + storageType = StorageTypeFSTree _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err == nil { storagelog.Write(c.log, diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 09c5451ad..28409f609 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -199,7 +199,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - err = c.flushObject(ctx, &obj, data, storageTypeFSTree) + err = c.flushObject(ctx, &obj, data, StorageTypeFSTree) if err != nil { if ignoreErrors { return nil @@ -228,7 +228,7 @@ func (c *cache) workerFlushSmall() { return } - err := c.flushObject(context.TODO(), obj, nil, storageTypeDB) + err := c.flushObject(context.TODO(), obj, nil, StorageTypeDB) if err != nil { // Error is handled in flushObject. continue @@ -239,7 +239,7 @@ func (c *cache) workerFlushSmall() { } // flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte, st storageType) error { +func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte, st StorageType) error { var err error defer func() { @@ -319,7 +319,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { return err } - if err := c.flushObject(ctx, &obj, data, storageTypeDB); err != nil { + if err := c.flushObject(ctx, &obj, data, StorageTypeDB); err != nil { return err } } diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index f8eb01091..f8f6de9b0 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -33,7 +33,7 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) (*objectSDK.Object, error) { found := false - storageType := storageTypeUndefined + storageType := StorageTypeUndefined startedAt := time.Now() defer func() { c.metrics.Get(time.Since(startedAt), found, storageType) @@ -43,7 +43,7 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) if err == nil { obj := objectSDK.New() found = true - storageType = storageTypeDB + storageType = StorageTypeDB return obj, obj.Unmarshal(value) } @@ -53,7 +53,7 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) } found = true - storageType = storageTypeFSTree + storageType = StorageTypeFSTree return res.Object, nil } diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go index afa6f69f1..e20c7e65e 100644 --- a/pkg/local_object_storage/writecache/metrics.go +++ b/pkg/local_object_storage/writecache/metrics.go @@ -6,37 +6,44 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) -type storageType string +type StorageType string + +func (t StorageType) String() string { + return string(t) +} const ( - storageTypeUndefined storageType = "null" - storageTypeDB storageType = "db" - storageTypeFSTree storageType = "fstree" + StorageTypeUndefined StorageType = "null" + StorageTypeDB StorageType = "db" + StorageTypeFSTree StorageType = "fstree" ) type Metrics interface { - Get(d time.Duration, success bool, st storageType) - Delete(d time.Duration, success bool, st storageType) - Put(d time.Duration, success bool, st storageType) - Flush(success bool, st storageType) - Evict(st storageType) + Get(d time.Duration, success bool, st StorageType) + Delete(d time.Duration, success bool, st StorageType) + Put(d time.Duration, success bool, st StorageType) + Flush(success bool, st StorageType) + Evict(st StorageType) - Estimate(db, fstree uint64) + SetEstimateSize(db, fstree uint64) SetMode(m mode.Mode) + SetActualCounters(db, fstree uint64) } type metricsStub struct{} -func (s *metricsStub) Get(time.Duration, bool, storageType) {} +func (s *metricsStub) Get(time.Duration, bool, StorageType) {} -func (s *metricsStub) Delete(time.Duration, bool, storageType) {} +func (s *metricsStub) Delete(time.Duration, bool, StorageType) {} -func (s *metricsStub) Put(time.Duration, bool, storageType) {} +func (s *metricsStub) Put(time.Duration, bool, StorageType) {} -func (s *metricsStub) Estimate(uint64, uint64) {} +func (s *metricsStub) SetEstimateSize(uint64, uint64) {} func (s *metricsStub) SetMode(mode.Mode) {} -func (s *metricsStub) Flush(bool, storageType) {} +func (s *metricsStub) SetActualCounters(uint64, uint64) {} -func (s *metricsStub) Evict(storageType) {} +func (s *metricsStub) Flush(bool, StorageType) {} + +func (s *metricsStub) Evict(StorageType) {} diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index c0001f926..1e99e4a28 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -36,7 +36,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro startedAt := time.Now() added := false - storageType := storageTypeUndefined + storageType := StorageTypeUndefined defer func() { c.metrics.Put(time.Since(startedAt), added, storageType) }() @@ -59,7 +59,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro } if sz <= c.smallObjectSize { - storageType = storageTypeDB + storageType = StorageTypeDB err := c.putSmall(oi) if err == nil { added = true @@ -67,7 +67,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro return common.PutRes{}, err } - storageType = storageTypeFSTree + storageType = StorageTypeFSTree err := c.putBig(ctx, oi.addr, prm) if err == nil { added = true diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index 5f3092526..14103e626 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -11,7 +11,7 @@ import ( func (c *cache) estimateCacheSize() uint64 { db := c.objCounters.DB() * c.smallObjectSize fstree := c.objCounters.FS() * c.maxObjectSize - c.metrics.Estimate(db, fstree) + c.metrics.SetEstimateSize(db, fstree) return db + fstree } @@ -71,6 +71,7 @@ func (c *cache) initCounters() error { c.objCounters.cDB.Store(inDB) c.objCounters.cFS.Store(inFS) + c.metrics.SetActualCounters(inDB, inFS) return nil } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index 50c110c5c..3bd3813d1 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -79,7 +79,7 @@ func (c *cache) deleteFromDB(keys []string) []string { }) for i := 0; i < errorIndex; i++ { c.objCounters.DecDB() - c.metrics.Evict(storageTypeDB) + c.metrics.Evict(StorageTypeDB) storagelog.Write(c.log, storagelog.AddressField(keys[i]), storagelog.StorageTypeField(wcStorageType), @@ -122,7 +122,7 @@ func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree DELETE"), ) - c.metrics.Evict(storageTypeFSTree) + c.metrics.Evict(StorageTypeFSTree) c.objCounters.DecFS() } } diff --git a/pkg/metrics/desc.go b/pkg/metrics/desc.go index 74d2d4e6e..612435b2f 100644 --- a/pkg/metrics/desc.go +++ b/pkg/metrics/desc.go @@ -48,6 +48,18 @@ func newGaugeVec(opts prometheus.GaugeOpts, labelNames []string) metric[*prometh } } +func newGaugeFunc(opts prometheus.GaugeOpts, f func() float64) metric[prometheus.GaugeFunc] { + return metric[prometheus.GaugeFunc]{ + value: prometheus.NewGaugeFunc(opts, f), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_GAUGE.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + }, + } +} + func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] { return metric[prometheus.Counter]{ value: prometheus.NewCounter(opts), @@ -60,6 +72,32 @@ func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] { } } +func newCounterVec(opts prometheus.CounterOpts, labels []string) metric[*prometheus.CounterVec] { + return metric[*prometheus.CounterVec]{ + value: prometheus.NewCounterVec(opts, labels), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_COUNTER.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + VariableLabels: labels, + }, + } +} + +func newHistogramVec(opts prometheus.HistogramOpts, labelNames []string) metric[*prometheus.HistogramVec] { + return metric[*prometheus.HistogramVec]{ + value: prometheus.NewHistogramVec(opts, labelNames), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_HISTOGRAM.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + VariableLabels: labelNames, + }, + } +} + // DescribeAll returns descriptions for all registered metrics. func DescribeAll() ([]Description, error) { registeredDescriptionsMtx.Lock() diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index bf12e610f..b8041eec8 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -10,6 +10,8 @@ type NodeMetrics struct { stateMetrics replicatorMetrics epoch metric[prometheus.Gauge] + + writeCacheMetrics *writeCacheMetrics } func NewNodeMetrics() *NodeMetrics { @@ -33,12 +35,16 @@ func NewNodeMetrics() *NodeMetrics { }) mustRegister(epoch) + writeCacheMetrics := newWriteCacheMetrics() + writeCacheMetrics.register() + return &NodeMetrics{ objectServiceMetrics: objectService, engineMetrics: engine, stateMetrics: state, replicatorMetrics: replicator, epoch: epoch, + writeCacheMetrics: writeCacheMetrics, } } @@ -46,3 +52,11 @@ func NewNodeMetrics() *NodeMetrics { func (m *NodeMetrics) SetEpoch(epoch uint64) { m.epoch.value.Set(float64(epoch)) } + +// WriteCache returns WriteCache metrics. +func (m *NodeMetrics) WriteCache() WriteCacheMetrics { + if m == nil { + return nil + } + return m.writeCacheMetrics +} diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go new file mode 100644 index 000000000..890903546 --- /dev/null +++ b/pkg/metrics/writecache.go @@ -0,0 +1,252 @@ +package metrics + +import ( + "fmt" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + wcSubsystem = "writecache" + wcShardID = "shard_id" + wcSuccess = "success" + wcStorage = "storage" + wcMode = "mode" +) + +type shardIDMode struct { + shardID, mode string +} + +type WriteCacheMetrics interface { + AddGetDuration(shardID string, success bool, d time.Duration) + IncGetCounter(shardID string, success bool, storageType string) + + AddDeleteDuration(shardID string, success bool, d time.Duration) + IncDeleteCounter(shardID string, success bool, storageType string) + + AddPutDuration(shardID string, success bool, d time.Duration) + IncPutCounter(shardID string, success bool, storageType string) + + IncActualCount(shardID string, storageType string) + DecActualCount(shardID string, storageType string) + SetActualCount(shardID string, count uint64, storageType string) + + SetEstimateSize(shardID string, size uint64, storageType string) + SetMode(shardID string, mode string) + + IncFlushCounter(shardID string, success bool, storageType string) + IncEvictCounter(shardID string, storageType string) +} + +type writeCacheMetrics struct { + getDuration metric[*prometheus.HistogramVec] + getCounter metric[*prometheus.CounterVec] + + putDuration metric[*prometheus.HistogramVec] + putCounter metric[*prometheus.CounterVec] + + deleteDuration metric[*prometheus.HistogramVec] + deleteCounter metric[*prometheus.CounterVec] + + flushCounter metric[*prometheus.CounterVec] + evictCounter metric[*prometheus.CounterVec] + + actualCount metric[*prometheus.GaugeVec] + + estimatedSize metric[*prometheus.GaugeVec] + + modeMetrics map[shardIDMode]metric[prometheus.GaugeFunc] + modeValues map[string]string + modeMtx sync.RWMutex +} + +func newWriteCacheMetrics() *writeCacheMetrics { + return &writeCacheMetrics{ + getDuration: newWCMethodDurationCounter("get"), + getCounter: newWCMethodCounterVec("get"), + putDuration: newWCMethodDurationCounter("put"), + putCounter: newWCMethodCounterVec("put"), + deleteDuration: newWCMethodDurationCounter("delete"), + deleteCounter: newWCMethodCounterVec("delete"), + flushCounter: newWCOperationCounterVec("flush", []string{wcShardID, wcStorage, wcSuccess}), + evictCounter: newWCOperationCounterVec("evict", []string{wcShardID, wcStorage}), + actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), + estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), + modeMtx: sync.RWMutex{}, + modeMetrics: make(map[shardIDMode]metric[prometheus.GaugeFunc]), + modeValues: make(map[string]string), + } +} + +func (m *writeCacheMetrics) AddGetDuration(shardID string, success bool, d time.Duration) { + setWriteCacheDuration(m.getDuration.value, shardID, success, d) +} + +func (m *writeCacheMetrics) IncGetCounter(shardID string, success bool, storageType string) { + incWriteCacheCounter(m.getCounter.value, shardID, success, storageType) +} + +func (m *writeCacheMetrics) AddDeleteDuration(shardID string, success bool, d time.Duration) { + setWriteCacheDuration(m.deleteDuration.value, shardID, success, d) +} + +func (m *writeCacheMetrics) IncDeleteCounter(shardID string, success bool, storageType string) { + incWriteCacheCounter(m.deleteCounter.value, shardID, success, storageType) +} + +func (m *writeCacheMetrics) AddPutDuration(shardID string, success bool, d time.Duration) { + setWriteCacheDuration(m.putDuration.value, shardID, success, d) +} + +func (m *writeCacheMetrics) IncPutCounter(shardID string, success bool, storageType string) { + incWriteCacheCounter(m.putCounter.value, shardID, success, storageType) +} + +func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { + m.actualCount.value.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + }).Inc() +} + +func (m *writeCacheMetrics) DecActualCount(shardID string, storageType string) { + m.actualCount.value.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + }).Dec() +} + +func (m *writeCacheMetrics) SetActualCount(shardID string, count uint64, storageType string) { + m.actualCount.value.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + }).Set(float64(count)) +} + +func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storageType string) { + m.estimatedSize.value.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + }).Set(float64(size)) +} + +func (m *writeCacheMetrics) SetMode(shardID string, mode string) { + m.modeMtx.Lock() + defer m.modeMtx.Unlock() + + m.modeValues[shardID] = mode + key := shardIDMode{ + shardID: shardID, + mode: mode, + } + if _, found := m.modeMetrics[key]; found { + return + } + + metric := newGaugeFunc( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: "writecache_mode", + Help: "Writecache mode value", + ConstLabels: prometheus.Labels{ + wcShardID: shardID, + wcMode: mode, + }, + }, func() float64 { + m.modeMtx.RLock() + defer m.modeMtx.RUnlock() + + value := m.modeValues[shardID] + if value == mode { + return 1 + } + return 0 + }) + mustRegister(metric) + m.modeMetrics[key] = metric +} + +func (m *writeCacheMetrics) IncFlushCounter(shardID string, success bool, storageType string) { + m.flushCounter.value.With(prometheus.Labels{ + wcShardID: shardID, + wcSuccess: fmt.Sprintf("%v", success), + wcStorage: storageType, + }).Inc() +} + +func (m *writeCacheMetrics) IncEvictCounter(shardID string, storageType string) { + m.evictCounter.value.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + }).Inc() +} + +func (m *writeCacheMetrics) register() { + mustRegister(m.getDuration) + mustRegister(m.getCounter) + mustRegister(m.putDuration) + mustRegister(m.putCounter) + mustRegister(m.deleteDuration) + mustRegister(m.deleteCounter) + mustRegister(m.actualCount) + mustRegister(m.estimatedSize) + mustRegister(m.flushCounter) + mustRegister(m.evictCounter) +} + +func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success bool, d time.Duration) { + m.With( + prometheus.Labels{ + wcShardID: shardID, + wcSuccess: fmt.Sprintf("%v", success), + }, + ).Observe(float64(d)) +} + +func incWriteCacheCounter(m *prometheus.CounterVec, shardID string, success bool, storageType string) { + m.With(prometheus.Labels{ + wcShardID: shardID, + wcSuccess: fmt.Sprintf("%v", success), + wcStorage: storageType, + }).Inc() +} + +func newWCMethodDurationCounter(method string) metric[*prometheus.HistogramVec] { + return newHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: fmt.Sprintf("%s_req_duration_seconds", method), + Help: fmt.Sprintf("Accumulated %s request process duration", method), + }, []string{wcShardID, wcSuccess}) +} + +func newWCMethodCounterVec(method string) metric[*prometheus.CounterVec] { + return newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: fmt.Sprintf("%s_req_count", method), + Help: fmt.Sprintf("The number of %s requests processed", method), + }, []string{wcShardID, wcSuccess, wcStorage}) +} + +func newWCOperationCounterVec(operation string, labels []string) metric[*prometheus.CounterVec] { + return newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: fmt.Sprintf("%s_operation_count", operation), + Help: fmt.Sprintf("The number of %s operations processed", operation), + }, labels) +} + +func newWCGaugeVec(name, help string, labels []string) metric[*prometheus.GaugeVec] { + return newGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: name, + Help: help, + }, labels) +} From a1823423cf238d9c8634111fd9203e8c1f8aa940 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 22 May 2023 10:10:25 +0300 Subject: [PATCH 0275/1943] [#312] node: Add WC metrics info to CHANGELOG Signed-off-by: Dmitrii Stepanov --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f79940f76..b5924b781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Changelog for FrostFS Node - Reload pprof and metrics on SIGHUP for ir (#125) - Support copies number parameter in `frostfs-cli object put` (#351) - Set extra wallets on SIGHUP for ir (#125) +- Writecache metrics (#312) ### Changed - `frostfs-cli util locode generate` is now much faster (#309) From 261335100845817f99133eac2bb5d658c3cd3d9e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 23 May 2023 15:02:38 +0300 Subject: [PATCH 0276/1943] [#387] gc: Cancel GC is change mode requested Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + pkg/local_object_storage/shard/control.go | 16 +- pkg/local_object_storage/shard/delete.go | 120 ++++++++------- pkg/local_object_storage/shard/gc.go | 16 +- .../shard/gc_internal_test.go | 144 ++++++++++++++++++ pkg/local_object_storage/shard/mode.go | 4 +- pkg/local_object_storage/shard/shard.go | 8 +- 7 files changed, 244 insertions(+), 65 deletions(-) create mode 100644 pkg/local_object_storage/shard/gc_internal_test.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index b84746a72..a862e745a 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -489,4 +489,5 @@ const ( EngineShardsEvacuationFailedToReadObject = "failed to read object to evacuate" EngineShardsEvacuationFailedToMoveObject = "failed to evacuate object to other node" ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" + ShardDeleteCantDeleteFromWriteCache = "can't delete object from write cache" ) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index e74f235f8..f885451af 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -304,8 +304,8 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { opts[i](&c) } - s.m.Lock() - defer s.m.Unlock() + unlock := s.lockExclusive() + defer unlock() ok, err := s.metaBase.Reload(c.metaOpts...) if err != nil { @@ -335,3 +335,15 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { s.log.Info(logs.ShardTryingToRestoreReadwriteMode) return s.setMode(mode.ReadWrite) } + +func (s *Shard) lockExclusive() func() { + s.setModeRequested.Store(true) + val := s.gcCancel.Load() + if val != nil { + cancelGC := val.(context.CancelFunc) + cancelGC() + } + s.m.Lock() + s.setModeRequested.Store(false) + return s.m.Unlock +} diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index f086aa30f..4eb7ad6af 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -53,70 +53,74 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { return DeleteRes{}, ErrDegradedMode } - ln := len(prm.addr) - - smalls := make(map[oid.Address][]byte, ln) - - for i := range prm.addr { - if s.hasWriteCache() { - err := s.writeCache.Delete(ctx, prm.addr[i]) - if err != nil && !IsErrNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { - s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.String("error", err.Error())) - } + for _, addr := range prm.addr { + select { + case <-ctx.Done(): + return DeleteRes{}, ctx.Err() + default: } - var sPrm meta.StorageIDPrm - sPrm.SetAddress(prm.addr[i]) + s.deleteObjectFromWriteCacheSafe(ctx, addr) - res, err := s.metaBase.StorageID(ctx, sPrm) - if err != nil { - s.log.Debug(logs.ShardCantGetStorageIDFromMetabase, - zap.Stringer("object", prm.addr[i]), - zap.String("error", err.Error())) + s.deleteFromBlobstorSafe(ctx, addr) - continue - } - - if res.StorageID() != nil { - smalls[prm.addr[i]] = res.StorageID() - } - } - - var delPrm meta.DeletePrm - delPrm.SetAddresses(prm.addr...) - - res, err := s.metaBase.Delete(ctx, delPrm) - if err != nil { - return DeleteRes{}, err // stop on metabase error ? - } - - var totalRemovedPayload uint64 - - s.decObjectCounterBy(physical, res.RawObjectsRemoved()) - s.decObjectCounterBy(logical, res.AvailableObjectsRemoved()) - for i := range prm.addr { - removedPayload := res.RemovedPhysicalObjectSizes()[i] - totalRemovedPayload += removedPayload - logicalRemovedPayload := res.RemovedLogicalObjectSizes()[i] - if logicalRemovedPayload > 0 { - s.addToContainerSize(prm.addr[i].Container().EncodeToString(), -int64(logicalRemovedPayload)) - } - } - s.addToPayloadSize(-int64(totalRemovedPayload)) - - for i := range prm.addr { - var delPrm common.DeletePrm - delPrm.Address = prm.addr[i] - id := smalls[prm.addr[i]] - delPrm.StorageID = id - - _, err = s.blobStor.Delete(ctx, delPrm) - if err != nil { - s.log.Debug(logs.ShardCantRemoveObjectFromBlobStor, - zap.Stringer("object_address", prm.addr[i]), - zap.String("error", err.Error())) + if err := s.deleteFromMetabase(ctx, addr); err != nil { + return DeleteRes{}, err // stop on metabase error ? } } return DeleteRes{}, nil } + +func (s *Shard) deleteObjectFromWriteCacheSafe(ctx context.Context, addr oid.Address) { + if s.hasWriteCache() { + err := s.writeCache.Delete(ctx, addr) + if err != nil && !IsErrNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { + s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.Error(err)) + } + } +} + +func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { + var sPrm meta.StorageIDPrm + sPrm.SetAddress(addr) + + res, err := s.metaBase.StorageID(ctx, sPrm) + if err != nil { + s.log.Debug("can't get storage ID from metabase", + zap.Stringer("object", addr), + zap.String("error", err.Error())) + } + storageID := res.StorageID() + + var delPrm common.DeletePrm + delPrm.Address = addr + delPrm.StorageID = storageID + + _, err = s.blobStor.Delete(ctx, delPrm) + if err != nil { + s.log.Debug("can't remove object from blobStor", + zap.Stringer("object_address", addr), + zap.String("error", err.Error())) + } +} + +func (s *Shard) deleteFromMetabase(ctx context.Context, addr oid.Address) error { + var delPrm meta.DeletePrm + delPrm.SetAddresses(addr) + + res, err := s.metaBase.Delete(ctx, delPrm) + if err != nil { + return err + } + s.decObjectCounterBy(physical, res.RawObjectsRemoved()) + s.decObjectCounterBy(logical, res.AvailableObjectsRemoved()) + removedPayload := res.RemovedPhysicalObjectSizes()[0] + logicalRemovedPayload := res.RemovedLogicalObjectSizes()[0] + if logicalRemovedPayload > 0 { + s.addToContainerSize(addr.Container().EncodeToString(), -int64(logicalRemovedPayload)) + } + s.addToPayloadSize(-int64(removedPayload)) + + return nil +} diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index f4f7a21e2..97899acc2 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -197,6 +197,14 @@ func (gc *gc) stop() { // with GC-marked graves. // Does nothing if shard is in "read-only" mode. func (s *Shard) removeGarbage() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s.gcCancel.Store(cancel) + if s.setModeRequested.Load() { + return + } + s.m.RLock() defer s.m.RUnlock() @@ -211,6 +219,12 @@ func (s *Shard) removeGarbage() { var iterPrm meta.GarbageIterationPrm iterPrm.SetHandler(func(g meta.GarbageObject) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + buf = append(buf, g.Address()) if len(buf) == s.rmBatchSize { @@ -237,7 +251,7 @@ func (s *Shard) removeGarbage() { deletePrm.SetAddresses(buf...) // delete accumulated objects - _, err = s.delete(context.TODO(), deletePrm) + _, err = s.delete(ctx, deletePrm) if err != nil { s.log.Warn(logs.ShardCouldNotDeleteTheObjects, zap.String("error", err.Error()), diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go new file mode 100644 index 000000000..d94029976 --- /dev/null +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -0,0 +1,144 @@ +package shard + +import ( + "context" + "path/filepath" + "testing" + "time" + + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/panjf2000/ants/v2" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" +) + +func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { + t.Parallel() + + rootPath := t.TempDir() + + var sh *Shard + + l := &logger.Logger{Logger: zaptest.NewLogger(t)} + blobOpts := []blobstor.Option{ + blobstor.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(1), + blobovniczatree.WithBlobovniczaShallowWidth(1)), + Policy: func(_ *object.Object, data []byte) bool { + return len(data) <= 1<<20 + }, + }, + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(rootPath, "blob"))), + }, + }), + } + + opts := []Option{ + WithID(NewIDFromBytes([]byte{})), + WithLogger(l), + WithBlobStorOptions(blobOpts...), + WithMetaBaseOptions( + meta.WithPath(filepath.Join(rootPath, "meta")), + meta.WithEpochState(epochState{}), + ), + WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))), + WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(addresses) + }), + WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { + sh.HandleExpiredLocks(ctx, epoch, a) + }), + WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) + return pool + }), + WithGCRemoverSleepInterval(1 * time.Second), + } + + sh = New(opts...) + + require.NoError(t, sh.Open()) + require.NoError(t, sh.Init(context.Background())) + + t.Cleanup(func() { + require.NoError(t, sh.Close()) + }) + + cnr := cidtest.ID() + obj := testutil.GenerateObjectWithCID(cnr) + objID, _ := obj.ID() + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(objID) + + var putPrm PutPrm + putPrm.SetObject(obj) + + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + var getPrm GetPrm + getPrm.SetAddress(objectCore.AddressOf(obj)) + _, err = sh.Get(context.Background(), getPrm) + require.NoError(t, err, "failed to get") + + //inhume + var inhumePrm InhumePrm + inhumePrm.MarkAsGarbage(addr) + _, err = sh.Inhume(context.Background(), inhumePrm) + require.NoError(t, err, "failed to inhume") + _, err = sh.Get(context.Background(), getPrm) + require.Error(t, err, "get returned error") + require.True(t, IsErrNotFound(err), "invalid error type") + + //storageID + var metaStIDPrm meta.StorageIDPrm + metaStIDPrm.SetAddress(addr) + storageID, err := sh.metaBase.StorageID(context.Background(), metaStIDPrm) + require.NoError(t, err, "failed to get storage ID") + + //check existance in blobstore + var bsExisted common.ExistsPrm + bsExisted.Address = addr + bsExisted.StorageID = storageID.StorageID() + exRes, err := sh.blobStor.Exists(context.Background(), bsExisted) + require.NoError(t, err, "failed to check blobstore existance") + require.True(t, exRes.Exists, "invalid blobstore existance result") + + //drop from blobstor + var bsDeletePrm common.DeletePrm + bsDeletePrm.Address = addr + bsDeletePrm.StorageID = storageID.StorageID() + _, err = sh.blobStor.Delete(context.Background(), bsDeletePrm) + require.NoError(t, err, "failed to delete from blobstore") + + //check existance in blobstore + exRes, err = sh.blobStor.Exists(context.Background(), bsExisted) + require.NoError(t, err, "failed to check blobstore existance") + require.False(t, exRes.Exists, "invalid blobstore existance result") + + //get should return object not found + _, err = sh.Get(context.Background(), getPrm) + require.Error(t, err, "get returned no error") + require.True(t, IsErrNotFound(err), "invalid error type") +} diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index 50c52accc..efd41863b 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -19,8 +19,8 @@ var ErrDegradedMode = logicerr.New("shard is in degraded mode") // Returns any error encountered that did not allow // setting shard mode. func (s *Shard) SetMode(m mode.Mode) error { - s.m.Lock() - defer s.m.Unlock() + unlock := s.lockExclusive() + defer unlock() return s.setMode(m) } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index e0059105f..65cc1ef55 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -3,6 +3,7 @@ package shard import ( "context" "sync" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -32,6 +33,9 @@ type Shard struct { metaBase *meta.DB tsSource TombstoneSource + + gcCancel atomic.Value + setModeRequested atomic.Bool } // Option represents Shard's constructor option. @@ -217,12 +221,12 @@ func WithWriteCache(use bool) Option { } // hasWriteCache returns bool if write cache exists on shards. -func (s Shard) hasWriteCache() bool { +func (s *Shard) hasWriteCache() bool { return s.cfg.useWriteCache } // needRefillMetabase returns true if metabase is needed to be refilled. -func (s Shard) needRefillMetabase() bool { +func (s *Shard) needRefillMetabase() bool { return s.cfg.refillMetabase } From 9119199f6ed0ad7c8059132b5b9c506df1f8d0b7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 26 May 2023 11:26:54 +0300 Subject: [PATCH 0277/1943] [#397] cli: Fix evacuation method names Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/evacuation.go | 8 ++-- pkg/services/control/rpc.go | 42 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 665bcf85f..45152fa0a 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -67,7 +67,7 @@ func startEvacuateShard(cmd *cobra.Command, _ []string) { var resp *control.StartShardEvacuationResponse var err error err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.StartEvacuateShard(client, req) + resp, err = control.StartShardEvacuation(client, req) return err }) commonCmd.ExitOnErr(cmd, "Start evacuate shards failed, rpc error: %w", err) @@ -95,7 +95,7 @@ func getEvacuateShardStatus(cmd *cobra.Command, _ []string) { var resp *control.GetShardEvacuationStatusResponse var err error err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.GetEvacuateShardStatus(client, req) + resp, err = control.GetShardEvacuationStatus(client, req) return err }) commonCmd.ExitOnErr(cmd, "Get evacuate shards status failed, rpc error: %w", err) @@ -118,7 +118,7 @@ func stopEvacuateShardStatus(cmd *cobra.Command, _ []string) { var resp *control.StopShardEvacuationResponse var err error err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.StopEvacuateShard(client, req) + resp, err = control.StopShardEvacuation(client, req) return err }) commonCmd.ExitOnErr(cmd, "Stop evacuate shards failed, rpc error: %w", err) @@ -166,7 +166,7 @@ func waitEvacuateCompletion(cmd *cobra.Command, pk *ecdsa.PrivateKey, cli *clien var err error err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.GetEvacuateShardStatus(client, req) + resp, err = control.GetShardEvacuationStatus(client, req) return err }) diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 3822c9f70..a2e7c411a 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -8,18 +8,18 @@ import ( const serviceName = "control.ControlService" const ( - rpcHealthCheck = "HealthCheck" - rpcSetNetmapStatus = "SetNetmapStatus" - rpcDropObjects = "DropObjects" - rpcListShards = "ListShards" - rpcSetShardMode = "SetShardMode" - rpcSynchronizeTree = "SynchronizeTree" - rpcEvacuateShard = "EvacuateShard" - rpcStartEvacuateShard = "StartEvacuateShard" - rpcGetEvacuateShardStatus = "GetEvacuateShardStatus" - rpcStopEvacuateShardStatus = "StopEvacuateShard" - rpcFlushCache = "FlushCache" - rpcDoctor = "Doctor" + rpcHealthCheck = "HealthCheck" + rpcSetNetmapStatus = "SetNetmapStatus" + rpcDropObjects = "DropObjects" + rpcListShards = "ListShards" + rpcSetShardMode = "SetShardMode" + rpcSynchronizeTree = "SynchronizeTree" + rpcEvacuateShard = "EvacuateShard" + rpcStartShardEvacuation = "StartShardEvacuation" + rpcGetShardEvacuationStatus = "GetShardEvacuationStatus" + rpcStopShardEvacuation = "StopShardEvacuation" + rpcFlushCache = "FlushCache" + rpcDoctor = "Doctor" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -144,12 +144,12 @@ func EvacuateShard(cli *client.Client, req *EvacuateShardRequest, opts ...client return wResp.message, nil } -// StartEvacuateShard executes ControlService.StartEvacuateShard RPC. -func StartEvacuateShard(cli *client.Client, req *StartShardEvacuationRequest, opts ...client.CallOption) (*StartShardEvacuationResponse, error) { +// StartShardEvacuation executes ControlService.StartShardEvacuation RPC. +func StartShardEvacuation(cli *client.Client, req *StartShardEvacuationRequest, opts ...client.CallOption) (*StartShardEvacuationResponse, error) { wResp := newResponseWrapper[StartShardEvacuationResponse]() wReq := &requestWrapper{m: req} - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStartEvacuateShard), wReq, wResp, opts...) + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStartShardEvacuation), wReq, wResp, opts...) if err != nil { return nil, err } @@ -157,12 +157,12 @@ func StartEvacuateShard(cli *client.Client, req *StartShardEvacuationRequest, op return wResp.message, nil } -// GetEvacuateShardStatus executes ControlService.GetEvacuateShardStatus RPC. -func GetEvacuateShardStatus(cli *client.Client, req *GetShardEvacuationStatusRequest, opts ...client.CallOption) (*GetShardEvacuationStatusResponse, error) { +// GetShardEvacuationStatus executes ControlService.GetShardEvacuationStatus RPC. +func GetShardEvacuationStatus(cli *client.Client, req *GetShardEvacuationStatusRequest, opts ...client.CallOption) (*GetShardEvacuationStatusResponse, error) { wResp := newResponseWrapper[GetShardEvacuationStatusResponse]() wReq := &requestWrapper{m: req} - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcGetEvacuateShardStatus), wReq, wResp, opts...) + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcGetShardEvacuationStatus), wReq, wResp, opts...) if err != nil { return nil, err } @@ -170,12 +170,12 @@ func GetEvacuateShardStatus(cli *client.Client, req *GetShardEvacuationStatusReq return wResp.message, nil } -// StopEvacuateShard executes ControlService.StopEvacuateShard RPC. -func StopEvacuateShard(cli *client.Client, req *StopShardEvacuationRequest, opts ...client.CallOption) (*StopShardEvacuationResponse, error) { +// StopShardEvacuation executes ControlService.StopShardEvacuation RPC. +func StopShardEvacuation(cli *client.Client, req *StopShardEvacuationRequest, opts ...client.CallOption) (*StopShardEvacuationResponse, error) { wResp := newResponseWrapper[StopShardEvacuationResponse]() wReq := &requestWrapper{m: req} - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStopEvacuateShardStatus), wReq, wResp, opts...) + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStopShardEvacuation), wReq, wResp, opts...) if err != nil { return nil, err } From 20a489bdb58b595084be6240de8967205e09eca8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 25 May 2023 16:12:46 +0300 Subject: [PATCH 0278/1943] [#393] gc: Use defer to mark handler done Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 97899acc2..78c41cb64 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -146,8 +146,8 @@ func (gc *gc) listenEvents(ctx context.Context) { h := v.handlers[i] err := gc.workerPool.Submit(func() { + defer v.prevGroup.Done() h(runCtx, event) - v.prevGroup.Done() }) if err != nil { gc.log.Warn(logs.ShardCouldNotSubmitGCJobToWorkerPool, From 598361706969c697c18826fd6d85005e5b905cea Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 25 May 2023 16:20:39 +0300 Subject: [PATCH 0279/1943] [#393] shard: Create tombstone source when reload Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index fe2b5ad35..30d457967 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -847,18 +847,9 @@ func initLocalStorage(c *cfg) { // service will be created later c.cfgObject.getSvc = new(getsvc.Service) - var tssPrm tsourse.TombstoneSourcePrm - tssPrm.SetGetService(c.cfgObject.getSvc) - tombstoneSrc := tsourse.NewSource(tssPrm) - - tombstoneSource := tombstone.NewChecker( - tombstone.WithLogger(c.log), - tombstone.WithTombstoneSource(tombstoneSrc), - ) - var shardsAttached int for _, optsWithMeta := range c.shardOpts() { - id, err := ls.AddShard(append(optsWithMeta.shOpts, shard.WithTombstoneSource(tombstoneSource))...) + id, err := ls.AddShard(append(optsWithMeta.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))...) if err != nil { c.log.Error(logs.FrostFSNodeFailedToAttachShardToEngine, zap.Error(err)) } else { @@ -1059,7 +1050,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { var rcfg engine.ReConfiguration for _, optsWithID := range c.shardOpts() { - rcfg.AddShard(optsWithID.configID, optsWithID.shOpts) + rcfg.AddShard(optsWithID.configID, append(optsWithID.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))) } err = c.cfgObject.cfgLocalStorage.localStorage.Reload(ctx, rcfg) @@ -1080,6 +1071,18 @@ func (c *cfg) reloadConfig(ctx context.Context) { c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } +func (c *cfg) createTombstoneSource() *tombstone.ExpirationChecker { + var tssPrm tsourse.TombstoneSourcePrm + tssPrm.SetGetService(c.cfgObject.getSvc) + tombstoneSrc := tsourse.NewSource(tssPrm) + + tombstoneSource := tombstone.NewChecker( + tombstone.WithLogger(c.log), + tombstone.WithTombstoneSource(tombstoneSrc), + ) + return tombstoneSource +} + func (c *cfg) shutdown() { c.setHealthStatus(control.HealthStatus_SHUTTING_DOWN) From f2e5dead7e9f893fe4a7de807480e999aed74a57 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 25 May 2023 18:40:27 +0300 Subject: [PATCH 0280/1943] [#398] pilorama: Disallow applying same operations 1. In redo() we save the old state. 2. If we do redo() for the same operation twice, the old state will be overritten with the new one. 3. This in turn affects undo() and subsequent isAncestor() check. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/batch.go | 15 +++- .../pilorama/batch_test.go | 70 +++++++++++++++++ .../pilorama/forest_test.go | 78 +++++++++++++++++++ 3 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 pkg/local_object_storage/pilorama/batch_test.go diff --git a/pkg/local_object_storage/pilorama/batch.go b/pkg/local_object_storage/pilorama/batch.go index 3065c8370..5722c68aa 100644 --- a/pkg/local_object_storage/pilorama/batch.go +++ b/pkg/local_object_storage/pilorama/batch.go @@ -50,10 +50,23 @@ func (b *batch) run() { return b.operations[i].Time < b.operations[j].Time }) + b.operations = removeDuplicatesInPlace(b.operations) var lm Move return b.forest.applyOperation(bLog, bTree, b.operations, &lm) }) - for i := range b.operations { + for i := range b.results { b.results[i] <- err } } + +func removeDuplicatesInPlace(a []*Move) []*Move { + equalCount := 0 + for i := 1; i < len(a); i++ { + if a[i].Time == a[i-1].Time { + equalCount++ + } else { + a[i-equalCount] = a[i] + } + } + return a[:len(a)-equalCount] +} diff --git a/pkg/local_object_storage/pilorama/batch_test.go b/pkg/local_object_storage/pilorama/batch_test.go new file mode 100644 index 000000000..931fce18c --- /dev/null +++ b/pkg/local_object_storage/pilorama/batch_test.go @@ -0,0 +1,70 @@ +package pilorama + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_removeDuplicatesInPlace(t *testing.T) { + testCases := []struct { + before []int + after []int + }{ + { + before: []int{}, + after: []int{}, + }, + { + before: []int{1}, + after: []int{1}, + }, + { + before: []int{1, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 3}, + after: []int{1, 2, 3}, + }, + { + before: []int{1, 1, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 2, 3}, + after: []int{1, 2, 3}, + }, + { + before: []int{1, 1, 1}, + after: []int{1}, + }, + { + before: []int{1, 1, 2, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 1, 1, 2, 3, 3, 3}, + after: []int{1, 2, 3}, + }, + } + + for _, tc := range testCases { + ops := make([]*Move, len(tc.before)) + for i := range ops { + ops[i] = &Move{Meta: Meta{Time: Timestamp(tc.before[i])}} + } + + expected := make([]*Move, len(tc.after)) + for i := range expected { + expected[i] = &Move{Meta: Meta{Time: Timestamp(tc.after[i])}} + } + + actual := removeDuplicatesInPlace(ops) + require.Equal(t, expected, actual, "%d", tc.before) + } +} diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 0cff28c3f..ebb4667f5 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -13,6 +13,7 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) var providers = []struct { @@ -444,6 +445,83 @@ func testForestTreeApply(t *testing.T, constructor func(t testing.TB, _ ...Optio }) } +func TestForest_ApplySameOperation(t *testing.T) { + for i := range providers { + t.Run(providers[i].name, func(t *testing.T) { + parallel := providers[i].name != "inmemory" + testForestApplySameOperation(t, providers[i].construct, parallel) + }) + } +} + +func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ ...Option) Forest, parallel bool) { + cid := cidtest.ID() + treeID := "version" + + batchSize := 3 + ctx := context.Background() + errG, _ := errgroup.WithContext(ctx) + if !parallel { + batchSize = 1 + errG.SetLimit(1) + } + + meta := []Meta{ + {Time: 1, Items: []KeyValue{{AttributeFilename, []byte("1")}, {"attr", []byte{1}}}}, + {Time: 2, Items: []KeyValue{{AttributeFilename, []byte("2")}, {"attr", []byte{1}}}}, + {Time: 3, Items: []KeyValue{{AttributeFilename, []byte("3")}, {"attr", []byte{1}}}}, + } + logs := []Move{ + { + Child: 1, + Parent: RootID, + Meta: meta[0], + }, + { + Child: 2, + Parent: 1, + Meta: meta[1], + }, + { + Child: 1, + Parent: 2, + Meta: meta[2], + }, + } + + check := func(t *testing.T, s Forest) { + testMeta(t, s, cid, treeID, 1, RootID, meta[0]) + testMeta(t, s, cid, treeID, 2, 1, meta[1]) + + nodes, err := s.TreeGetChildren(ctx, cid, treeID, RootID) + require.NoError(t, err) + require.Equal(t, []Node{1}, nodes) + + nodes, err = s.TreeGetChildren(ctx, cid, treeID, 1) + require.NoError(t, err) + require.Equal(t, []Node{2}, nodes) + } + + t.Run("expected", func(t *testing.T) { + s := constructor(t) + for i := range logs { + require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[i], false)) + } + check(t, s) + }) + + s := constructor(t, WithMaxBatchSize(batchSize)) + require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[0], false)) + for i := 0; i < batchSize; i++ { + errG.Go(func() error { + return s.TreeApply(ctx, cid, treeID, &logs[2], false) + }) + } + require.NoError(t, errG.Wait()) + require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[1], false)) + check(t, s) +} + func TestForest_GetOpLog(t *testing.T) { for i := range providers { t.Run(providers[i].name, func(t *testing.T) { From bc34fee6a78dfe916b1407d67ccad5416a1aa141 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Wed, 24 May 2023 10:01:50 +0300 Subject: [PATCH 0281/1943] [#370] Add tree service metrics Signed-off-by: Alejandro Lopez --- CHANGELOG.md | 1 + cmd/frostfs-node/tree.go | 3 +- pkg/metrics/node.go | 16 +++++++-- pkg/metrics/treeservice.go | 64 +++++++++++++++++++++++++++++++++ pkg/services/tree/metrics.go | 15 ++++++++ pkg/services/tree/options.go | 8 +++++ pkg/services/tree/replicator.go | 6 ++++ pkg/services/tree/service.go | 1 + pkg/services/tree/sync.go | 6 ++++ 9 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 pkg/metrics/treeservice.go create mode 100644 pkg/services/tree/metrics.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b5924b781..83ec8e41a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Changelog for FrostFS Node - Support copies number parameter in `frostfs-cli object put` (#351) - Set extra wallets on SIGHUP for ir (#125) - Writecache metrics (#312) +- Add tree service metrics (#370) ### Changed - `frostfs-cli util locode generate` is now much faster (#309) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index b4f43acac..fffaa01d1 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -55,7 +55,8 @@ func initTreeService(c *cfg) { tree.WithContainerCacheSize(treeConfig.CacheSize()), tree.WithReplicationTimeout(treeConfig.ReplicationTimeout()), tree.WithReplicationChannelCapacity(treeConfig.ReplicationChannelCapacity()), - tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount())) + tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount()), + tree.WithMetrics(c.metricsCollector.TreeService())) for _, srv := range c.cfgGRPC.servers { tree.RegisterTreeServiceServer(srv, c.treeService) diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index b8041eec8..cca82b5fe 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -1,6 +1,9 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + "github.com/prometheus/client_golang/prometheus" +) const namespace = "frostfs_node" @@ -9,9 +12,10 @@ type NodeMetrics struct { engineMetrics stateMetrics replicatorMetrics - epoch metric[prometheus.Gauge] writeCacheMetrics *writeCacheMetrics + treeService *treeServiceMetrics + epoch metric[prometheus.Gauge] } func NewNodeMetrics() *NodeMetrics { @@ -27,6 +31,9 @@ func NewNodeMetrics() *NodeMetrics { replicator := newReplicatorMetrics() replicator.register() + treeService := newTreeServiceMetrics() + treeService.register() + epoch := newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, @@ -43,6 +50,7 @@ func NewNodeMetrics() *NodeMetrics { engineMetrics: engine, stateMetrics: state, replicatorMetrics: replicator, + treeService: treeService, epoch: epoch, writeCacheMetrics: writeCacheMetrics, } @@ -60,3 +68,7 @@ func (m *NodeMetrics) WriteCache() WriteCacheMetrics { } return m.writeCacheMetrics } + +func (m *NodeMetrics) TreeService() tree.MetricsRegister { + return m.treeService +} diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go new file mode 100644 index 000000000..135f6e6d2 --- /dev/null +++ b/pkg/metrics/treeservice.go @@ -0,0 +1,64 @@ +package metrics + +import ( + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const treeServiceLabelSuccess = "success" + +type treeServiceMetrics struct { + replicateTaskDuration metric[*prometheus.HistogramVec] + replicateWaitDuration metric[*prometheus.HistogramVec] + syncOpDuration metric[*prometheus.HistogramVec] +} + +func newTreeServiceMetrics() *treeServiceMetrics { + const treeServiceSubsystem = "treeservice" + return &treeServiceMetrics{ + replicateTaskDuration: newHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: treeServiceSubsystem, + Name: "replicate_task_duration_seconds", + Help: "Duration of individual replication tasks executed as part of replication loops", + }, []string{treeServiceLabelSuccess}), + replicateWaitDuration: newHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: treeServiceSubsystem, + Name: "replicate_wait_duration_seconds", + Help: "Duration of overall waiting time for replication loops", + }, []string{treeServiceLabelSuccess}), + syncOpDuration: newHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: treeServiceSubsystem, + Name: "sync_duration_seconds", + Help: "Duration of synchronization operations", + }, []string{treeServiceLabelSuccess}), + } +} + +func (m *treeServiceMetrics) register() { + mustRegister(m.replicateTaskDuration) + mustRegister(m.replicateWaitDuration) + mustRegister(m.syncOpDuration) +} + +func (m *treeServiceMetrics) AddReplicateTaskDuration(d time.Duration, success bool) { + m.replicateTaskDuration.value.With(prometheus.Labels{ + treeServiceLabelSuccess: fmt.Sprintf("%v", success), + }).Observe(d.Seconds()) +} + +func (m *treeServiceMetrics) AddReplicateWaitDuration(d time.Duration, success bool) { + m.replicateWaitDuration.value.With(prometheus.Labels{ + treeServiceLabelSuccess: fmt.Sprintf("%v", success), + }).Observe(d.Seconds()) +} + +func (m *treeServiceMetrics) AddSyncDuration(d time.Duration, success bool) { + m.syncOpDuration.value.With(prometheus.Labels{ + treeServiceLabelSuccess: fmt.Sprintf("%v", success), + }).Observe(d.Seconds()) +} diff --git a/pkg/services/tree/metrics.go b/pkg/services/tree/metrics.go new file mode 100644 index 000000000..0f0e4ee57 --- /dev/null +++ b/pkg/services/tree/metrics.go @@ -0,0 +1,15 @@ +package tree + +import "time" + +type MetricsRegister interface { + AddReplicateTaskDuration(time.Duration, bool) + AddReplicateWaitDuration(time.Duration, bool) + AddSyncDuration(time.Duration, bool) +} + +type defaultMetricsRegister struct{} + +func (defaultMetricsRegister) AddReplicateTaskDuration(time.Duration, bool) {} +func (defaultMetricsRegister) AddReplicateWaitDuration(time.Duration, bool) {} +func (defaultMetricsRegister) AddSyncDuration(time.Duration, bool) {} diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index d60bc14c5..bcaf21f92 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -33,6 +33,8 @@ type cfg struct { replicatorWorkerCount int replicatorTimeout time.Duration containerCacheSize int + + metrics MetricsRegister } // Option represents configuration option for a tree service. @@ -116,3 +118,9 @@ func WithReplicationTimeout(t time.Duration) Option { } } } + +func WithMetrics(v MetricsRegister) Option { + return func(c *cfg) { + c.metrics = v + } +} diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 60d0eff50..7199dc40e 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -75,6 +75,7 @@ func (s *Service) replicationWorker(ctx context.Context) { attribute.String("public_key", hex.EncodeToString(task.n.PublicKey())), ), ) + start := time.Now() var lastErr error var lastAddr string @@ -113,6 +114,9 @@ func (s *Service) replicationWorker(ctx context.Context) { zap.String("address", lastAddr), zap.String("key", hex.EncodeToString(task.n.PublicKey()))) } + s.metrics.AddReplicateTaskDuration(time.Since(start), false) + } else { + s.metrics.AddReplicateTaskDuration(time.Since(start), true) } span.End() } @@ -137,6 +141,7 @@ func (s *Service) replicateLoop(ctx context.Context) { case <-ctx.Done(): return case op := <-s.replicateCh: + start := time.Now() err := s.replicate(op) if err != nil { s.log.Error(logs.TreeErrorDuringReplication, @@ -144,6 +149,7 @@ func (s *Service) replicateLoop(ctx context.Context) { zap.Stringer("cid", op.cid), zap.String("treeID", op.treeID)) } + s.metrics.AddReplicateWaitDuration(time.Since(start), err == nil) } } } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 546b7a207..96e547f36 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -46,6 +46,7 @@ func New(opts ...Option) *Service { s.replicatorChannelCapacity = defaultReplicatorCapacity s.replicatorWorkerCount = defaultReplicatorWorkerCount s.replicatorTimeout = defaultReplicatorSendTimeout + s.metrics = defaultMetricsRegister{} for i := range opts { opts[i](&s.cfg) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index ed2455194..ec51c6bc6 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -9,6 +9,7 @@ import ( "math" "math/rand" "sync" + "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -376,9 +377,12 @@ func (s *Service) syncLoop(ctx context.Context) { ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.sync") s.log.Debug(logs.TreeSyncingTrees) + start := time.Now() + cnrs, err := s.cfg.cnrSource.List() if err != nil { s.log.Error(logs.TreeCouldNotFetchContainers, zap.Error(err)) + s.metrics.AddSyncDuration(time.Since(start), false) span.End() continue } @@ -390,6 +394,8 @@ func (s *Service) syncLoop(ctx context.Context) { s.removeContainers(ctx, newMap) s.log.Debug(logs.TreeTreesHaveBeenSynchronized) + + s.metrics.AddSyncDuration(time.Since(start), true) span.End() } } From 271a56c2ab6ef3796041b32ece3d6507e8dad022 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 25 May 2023 17:46:24 +0300 Subject: [PATCH 0282/1943] [#395] metrics: Drop redundant metrics HistogramVec already has labeled counter. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/writecache.go | 9 +-- pkg/metrics/writecache.go | 70 ++++--------------- 2 files changed, 16 insertions(+), 63 deletions(-) diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index ea2299b8b..692fa4be5 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -63,21 +63,18 @@ type writeCacheMetrics struct { } func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddGetDuration(m.shardID, success, d) - m.metrics.IncGetCounter(m.shardID, success, st.String()) + m.metrics.AddGetDuration(m.shardID, success, d, st.String()) } func (m *writeCacheMetrics) Delete(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddDeleteDuration(m.shardID, success, d) - m.metrics.IncDeleteCounter(m.shardID, success, st.String()) + m.metrics.AddDeleteDuration(m.shardID, success, d, st.String()) if success { m.metrics.DecActualCount(m.shardID, st.String()) } } func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddPutDuration(m.shardID, success, d) - m.metrics.IncPutCounter(m.shardID, success, st.String()) + m.metrics.AddPutDuration(m.shardID, success, d, st.String()) if success { m.metrics.IncActualCount(m.shardID, st.String()) } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 890903546..74c330842 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -21,14 +21,9 @@ type shardIDMode struct { } type WriteCacheMetrics interface { - AddGetDuration(shardID string, success bool, d time.Duration) - IncGetCounter(shardID string, success bool, storageType string) - - AddDeleteDuration(shardID string, success bool, d time.Duration) - IncDeleteCounter(shardID string, success bool, storageType string) - - AddPutDuration(shardID string, success bool, d time.Duration) - IncPutCounter(shardID string, success bool, storageType string) + AddGetDuration(shardID string, success bool, d time.Duration, storageType string) + AddDeleteDuration(shardID string, success bool, d time.Duration, storageType string) + AddPutDuration(shardID string, success bool, d time.Duration, storageType string) IncActualCount(shardID string, storageType string) DecActualCount(shardID string, storageType string) @@ -42,14 +37,9 @@ type WriteCacheMetrics interface { } type writeCacheMetrics struct { - getDuration metric[*prometheus.HistogramVec] - getCounter metric[*prometheus.CounterVec] - - putDuration metric[*prometheus.HistogramVec] - putCounter metric[*prometheus.CounterVec] - + getDuration metric[*prometheus.HistogramVec] + putDuration metric[*prometheus.HistogramVec] deleteDuration metric[*prometheus.HistogramVec] - deleteCounter metric[*prometheus.CounterVec] flushCounter metric[*prometheus.CounterVec] evictCounter metric[*prometheus.CounterVec] @@ -66,11 +56,8 @@ type writeCacheMetrics struct { func newWriteCacheMetrics() *writeCacheMetrics { return &writeCacheMetrics{ getDuration: newWCMethodDurationCounter("get"), - getCounter: newWCMethodCounterVec("get"), putDuration: newWCMethodDurationCounter("put"), - putCounter: newWCMethodCounterVec("put"), deleteDuration: newWCMethodDurationCounter("delete"), - deleteCounter: newWCMethodCounterVec("delete"), flushCounter: newWCOperationCounterVec("flush", []string{wcShardID, wcStorage, wcSuccess}), evictCounter: newWCOperationCounterVec("evict", []string{wcShardID, wcStorage}), actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), @@ -81,28 +68,16 @@ func newWriteCacheMetrics() *writeCacheMetrics { } } -func (m *writeCacheMetrics) AddGetDuration(shardID string, success bool, d time.Duration) { - setWriteCacheDuration(m.getDuration.value, shardID, success, d) +func (m *writeCacheMetrics) AddGetDuration(shardID string, success bool, d time.Duration, storageType string) { + setWriteCacheDuration(m.getDuration.value, shardID, success, d, storageType) } -func (m *writeCacheMetrics) IncGetCounter(shardID string, success bool, storageType string) { - incWriteCacheCounter(m.getCounter.value, shardID, success, storageType) +func (m *writeCacheMetrics) AddDeleteDuration(shardID string, success bool, d time.Duration, storageType string) { + setWriteCacheDuration(m.deleteDuration.value, shardID, success, d, storageType) } -func (m *writeCacheMetrics) AddDeleteDuration(shardID string, success bool, d time.Duration) { - setWriteCacheDuration(m.deleteDuration.value, shardID, success, d) -} - -func (m *writeCacheMetrics) IncDeleteCounter(shardID string, success bool, storageType string) { - incWriteCacheCounter(m.deleteCounter.value, shardID, success, storageType) -} - -func (m *writeCacheMetrics) AddPutDuration(shardID string, success bool, d time.Duration) { - setWriteCacheDuration(m.putDuration.value, shardID, success, d) -} - -func (m *writeCacheMetrics) IncPutCounter(shardID string, success bool, storageType string) { - incWriteCacheCounter(m.putCounter.value, shardID, success, storageType) +func (m *writeCacheMetrics) AddPutDuration(shardID string, success bool, d time.Duration, storageType string) { + setWriteCacheDuration(m.putDuration.value, shardID, success, d, storageType) } func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { @@ -187,49 +162,30 @@ func (m *writeCacheMetrics) IncEvictCounter(shardID string, storageType string) func (m *writeCacheMetrics) register() { mustRegister(m.getDuration) - mustRegister(m.getCounter) mustRegister(m.putDuration) - mustRegister(m.putCounter) mustRegister(m.deleteDuration) - mustRegister(m.deleteCounter) mustRegister(m.actualCount) mustRegister(m.estimatedSize) mustRegister(m.flushCounter) mustRegister(m.evictCounter) } -func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success bool, d time.Duration) { +func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success bool, d time.Duration, storageType string) { m.With( prometheus.Labels{ wcShardID: shardID, wcSuccess: fmt.Sprintf("%v", success), + wcStorage: storageType, }, ).Observe(float64(d)) } -func incWriteCacheCounter(m *prometheus.CounterVec, shardID string, success bool, storageType string) { - m.With(prometheus.Labels{ - wcShardID: shardID, - wcSuccess: fmt.Sprintf("%v", success), - wcStorage: storageType, - }).Inc() -} - func newWCMethodDurationCounter(method string) metric[*prometheus.HistogramVec] { return newHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: wcSubsystem, Name: fmt.Sprintf("%s_req_duration_seconds", method), Help: fmt.Sprintf("Accumulated %s request process duration", method), - }, []string{wcShardID, wcSuccess}) -} - -func newWCMethodCounterVec(method string) metric[*prometheus.CounterVec] { - return newCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: wcSubsystem, - Name: fmt.Sprintf("%s_req_count", method), - Help: fmt.Sprintf("The number of %s requests processed", method), }, []string{wcShardID, wcSuccess, wcStorage}) } From 9c54a2410111087022a5e5ab9d268d0ab85aaeae Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 22 May 2023 09:59:38 +0300 Subject: [PATCH 0283/1943] [#364] Fix trailing whitespace Signed-off-by: Anton Nikiforov --- docs/evacuation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/evacuation.md b/docs/evacuation.md index 7c8beb790..5befad164 100644 --- a/docs/evacuation.md +++ b/docs/evacuation.md @@ -85,8 +85,8 @@ Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evac ### Start evacuation and await it completes without progress notifications ```bash frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --id 54Y8aot9uc7BSadw2XtYr3 --await --no-progress -Enter password > +Enter password > Shard evacuation has been successfully started. Shard evacuation has been completed. Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:20:00Z UTC. Duration: 00:00:14. -``` \ No newline at end of file +``` From 802168c0c66b532409e4964cf032fd4e6528006a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 24 May 2023 14:09:11 +0300 Subject: [PATCH 0284/1943] [#364] node: Stop flushing big object when termination signal received Signed-off-by: Anton Nikiforov --- .../blobstor/blobovniczatree/iterate.go | 3 ++- .../blobstor/common/storage.go | 2 +- .../blobstor/fstree/fstree.go | 13 +++++++++---- .../blobstor/internal/blobstortest/iterate.go | 6 +++--- pkg/local_object_storage/blobstor/iterate.go | 9 +++++---- pkg/local_object_storage/blobstor/iterate_test.go | 2 +- .../blobstor/memstore/memstore.go | 2 +- pkg/local_object_storage/blobstor/perf_test.go | 2 +- .../blobstor/teststore/teststore.go | 4 ++-- pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/writecache/flush.go | 15 +++++++++++++-- 11 files changed, 39 insertions(+), 21 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 9918801b9..ad933da0b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "context" "fmt" "path/filepath" @@ -11,7 +12,7 @@ import ( ) // Iterate iterates over all objects in b. -func (b *Blobovniczas) Iterate(prm common.IteratePrm) (common.IterateRes, error) { +func (b *Blobovniczas) Iterate(_ context.Context, prm common.IteratePrm) (common.IterateRes, error) { return common.IterateRes{}, b.iterateBlobovniczas(prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { var subPrm blobovnicza.IteratePrm subPrm.SetHandler(func(elem blobovnicza.IterationElement) error { diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 801d32c1e..c5d187f30 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -25,5 +25,5 @@ type Storage interface { Exists(context.Context, ExistsPrm) (ExistsRes, error) Put(context.Context, PutPrm) (PutRes, error) Delete(context.Context, DeletePrm) (DeleteRes, error) - Iterate(IteratePrm) (IterateRes, error) + Iterate(context.Context, IteratePrm) (IterateRes, error) } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 75f63193a..b0879f68e 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -100,11 +100,11 @@ func addressFromString(s string) (oid.Address, error) { } // Iterate iterates over all stored objects. -func (t *FSTree) Iterate(prm common.IteratePrm) (common.IterateRes, error) { - return common.IterateRes{}, t.iterate(0, []string{t.RootPath}, prm) +func (t *FSTree) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { + return common.IterateRes{}, t.iterate(ctx, 0, []string{t.RootPath}, prm) } -func (t *FSTree) iterate(depth uint64, curPath []string, prm common.IteratePrm) error { +func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, prm common.IteratePrm) error { curName := strings.Join(curPath[1:], "") des, err := os.ReadDir(filepath.Join(curPath...)) if err != nil { @@ -119,10 +119,15 @@ func (t *FSTree) iterate(depth uint64, curPath []string, prm common.IteratePrm) curPath = append(curPath, "") for i := range des { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } curPath[l] = des[i].Name() if !isLast && des[i].IsDir() { - err := t.iterate(depth+1, curPath, prm) + err := t.iterate(ctx, depth+1, curPath, prm) if err != nil { // Must be error from handler in case errors are ignored. // Need to report. diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index 83ada9607..34622c857 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -49,7 +49,7 @@ func runTestNormalHandler(t *testing.T, s common.Storage, objects []objectDesc) return nil } - _, err := s.Iterate(iterPrm) + _, err := s.Iterate(context.Background(), iterPrm) require.NoError(t, err) require.Equal(t, len(objects), len(seen)) for i := range objects { @@ -72,7 +72,7 @@ func runTestLazyHandler(t *testing.T, s common.Storage, objects []objectDesc) { return nil } - _, err := s.Iterate(iterPrm) + _, err := s.Iterate(context.Background(), iterPrm) require.NoError(t, err) require.Equal(t, len(objects), len(seen)) for i := range objects { @@ -107,7 +107,7 @@ func runTestIgnoreLogicalErrors(t *testing.T, s common.Storage, objects []object return nil } - _, err := s.Iterate(iterPrm) + _, err := s.Iterate(context.Background(), iterPrm) require.Equal(t, err, logicErr) require.Equal(t, len(objects)/2, len(seen)) for i := range objects { diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index 2c37ee776..4e52f0abf 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -1,6 +1,7 @@ package blobstor import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -16,12 +17,12 @@ import ( // did not allow to completely iterate over the storage. // // If handler returns an error, method wraps and returns it immediately. -func (b *BlobStor) Iterate(prm common.IteratePrm) (common.IterateRes, error) { +func (b *BlobStor) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { b.modeMtx.RLock() defer b.modeMtx.RUnlock() for i := range b.storage { - _, err := b.storage[i].Storage.Iterate(prm) + _, err := b.storage[i].Storage.Iterate(ctx, prm) if err != nil && !prm.IgnoreErrors { return common.IterateRes{}, fmt.Errorf("blobstor iterator failure: %w", err) } @@ -31,7 +32,7 @@ func (b *BlobStor) Iterate(prm common.IteratePrm) (common.IterateRes, error) { // IterateBinaryObjects is a helper function which iterates over BlobStor and passes binary objects to f. // Errors related to object reading and unmarshaling are logged and skipped. -func IterateBinaryObjects(blz *BlobStor, f func(addr oid.Address, data []byte, descriptor []byte) error) error { +func IterateBinaryObjects(ctx context.Context, blz *BlobStor, f func(addr oid.Address, data []byte, descriptor []byte) error) error { var prm common.IteratePrm prm.Handler = func(elem common.IterationElement) error { @@ -45,7 +46,7 @@ func IterateBinaryObjects(blz *BlobStor, f func(addr oid.Address, data []byte, d return nil } - _, err := blz.Iterate(prm) + _, err := blz.Iterate(ctx, prm) return err } diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index 6488ff5fc..c35869655 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -68,7 +68,7 @@ func TestIterateObjects(t *testing.T) { require.NoError(t, err) } - err := IterateBinaryObjects(blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { + err := IterateBinaryObjects(context.Background(), blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { v, ok := mObjs[string(data)] require.True(t, ok) diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index e435cfef4..b6cca2551 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -126,7 +126,7 @@ func (s *memstoreImpl) Delete(_ context.Context, req common.DeletePrm) (common.D return common.DeleteRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } -func (s *memstoreImpl) Iterate(req common.IteratePrm) (common.IterateRes, error) { +func (s *memstoreImpl) Iterate(_ context.Context, req common.IteratePrm) (common.IterateRes, error) { s.mu.RLock() defer s.mu.RUnlock() for k, v := range s.objs { diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index f21982530..5245146cb 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -207,7 +207,7 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) { // Benchmark iterate cnt := 0 b.ResetTimer() - _, err := st.Iterate(common.IteratePrm{ + _, err := st.Iterate(context.Background(), common.IteratePrm{ Handler: func(elem common.IterationElement) error { cnt++ return nil diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 24d742fda..c7179eb45 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -202,14 +202,14 @@ func (s *TestStore) Delete(ctx context.Context, req common.DeletePrm) (common.De } } -func (s *TestStore) Iterate(req common.IteratePrm) (common.IterateRes, error) { +func (s *TestStore) Iterate(ctx context.Context, req common.IteratePrm) (common.IterateRes, error) { s.mu.RLock() defer s.mu.RUnlock() switch { case s.overrides.Iterate != nil: return s.overrides.Iterate(req) case s.st != nil: - return s.st.Iterate(req) + return s.st.Iterate(ctx, req) default: panic(fmt.Sprintf("unexpected storage call: Iterate(%+v)", req)) } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index f885451af..e8e2bd4d7 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -170,7 +170,7 @@ func (s *Shard) refillMetabase(ctx context.Context) error { obj := objectSDK.New() - err = blobstor.IterateBinaryObjects(s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { + err = blobstor.IterateBinaryObjects(ctx, s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { if err := obj.Unmarshal(data); err != nil { s.log.Warn(logs.ShardCouldNotUnmarshalObject, zap.Stringer("address", addr), diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 28409f609..767435ebf 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -35,6 +35,17 @@ const ( // runFlushLoop starts background workers which periodically flush objects to the blobstor. func (c *cache) runFlushLoop() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ch := c.closeCh + c.wg.Add(1) + go func() { + <-ch + cancel() + c.wg.Done() + }() + for i := 0; i < c.workersCount; i++ { c.wg.Add(1) go c.workerFlushSmall() @@ -42,7 +53,7 @@ func (c *cache) runFlushLoop() { c.wg.Add(1) go func() { - c.workerFlushBig(context.TODO()) + c.workerFlushBig(ctx) c.wg.Done() }() @@ -211,7 +222,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return nil } - _, err := c.fsTree.Iterate(prm) + _, err := c.fsTree.Iterate(ctx, prm) return err } From 365a7ca0f424f85a76b91d4878812bdf34b16729 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 29 May 2023 09:35:08 +0300 Subject: [PATCH 0285/1943] [#366] node: Stop GC once termination signal received Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/shard/gc.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 78c41cb64..34a48d44a 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -76,7 +76,7 @@ type gc struct { workerPool util.WorkerPool - remover func() + remover func(context.Context) eventChan chan Event mEventHandler map[eventType]*eventHandlers @@ -115,7 +115,7 @@ func (gc *gc) init(ctx context.Context) { } gc.wg.Add(2) - go gc.tickRemover() + go gc.tickRemover(ctx) go gc.listenEvents(ctx) } @@ -160,7 +160,7 @@ func (gc *gc) listenEvents(ctx context.Context) { } } -func (gc *gc) tickRemover() { +func (gc *gc) tickRemover(ctx context.Context) { defer gc.wg.Done() timer := time.NewTimer(gc.removerInterval) @@ -178,7 +178,7 @@ func (gc *gc) tickRemover() { gc.log.Debug(logs.ShardGCIsStopped) return case <-timer.C: - gc.remover() + gc.remover(ctx) timer.Reset(gc.removerInterval) } } @@ -196,8 +196,8 @@ func (gc *gc) stop() { // iterates over metabase and deletes objects // with GC-marked graves. // Does nothing if shard is in "read-only" mode. -func (s *Shard) removeGarbage() { - ctx, cancel := context.WithCancel(context.Background()) +func (s *Shard) removeGarbage(pctx context.Context) { + ctx, cancel := context.WithCancel(pctx) defer cancel() s.gcCancel.Store(cancel) From a3e30062dfa3446e3772ac4ba32bbdc76a200195 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 22 May 2023 13:00:24 +0300 Subject: [PATCH 0286/1943] [#325] node: Introduce unsafe_disable param to disable policer Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-node/config/policer/config.go | 6 ++++++ cmd/frostfs-node/object.go | 5 +++++ internal/logs/logs.go | 3 ++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config/policer/config.go b/cmd/frostfs-node/config/policer/config.go index 51e55c341..487e42be8 100644 --- a/cmd/frostfs-node/config/policer/config.go +++ b/cmd/frostfs-node/config/policer/config.go @@ -25,3 +25,9 @@ func HeadTimeout(c *config.Config) time.Duration { return HeadTimeoutDefault } + +// UnsafeDisable returns the value of "unsafe_disable" config parameter +// from "policer" section. +func UnsafeDisable(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "unsafe_disable") +} diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 1ce330cb5..7a409da03 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -204,6 +204,11 @@ func initObjectService(c *cfg) { } func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.ClientCache) { + if policerconfig.UnsafeDisable(c.appCfg) { + c.log.Warn(logs.FrostFSNodePolicerIsDisabled) + return + } + ls := c.cfgObject.cfgLocalStorage.localStorage pol := policer.New( diff --git a/internal/logs/logs.go b/internal/logs/logs.go index a862e745a..9b800adfc 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -477,7 +477,8 @@ const ( FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" // Debug in ../node/cmd/frostfs-node/tree.go FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" // Error in ../node/cmd/frostfs-node/tree.go FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" // Error in ../node/cmd/frostfs-node/control.go - CommonApplicationStarted = "application started" // Info in ../node/cmd/frostfs-ir/main.go + FrostFSNodePolicerIsDisabled = "policer is disabled" + CommonApplicationStarted = "application started" ShardGCCollectingExpiredObjectsStarted = "collecting expired objects started" ShardGCCollectingExpiredObjectsCompleted = "collecting expired objects completed" ShardGCCollectingExpiredLocksStarted = "collecting expired locks started" From 8dcd06c5871f98581251d5db666af6970b7ce0a6 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Sun, 28 May 2023 22:37:37 +0300 Subject: [PATCH 0287/1943] [#394] node: Use `Context` in `Blobovniczas.Iterate()` Signed-off-by: Anton Nikiforov --- cmd/frostfs-lens/internal/blobovnicza/list.go | 3 +- .../blobovnicza/iterate.go | 12 ++++++-- .../blobovnicza/iterate_test.go | 9 +++--- .../blobstor/blobovniczatree/control.go | 3 +- .../blobstor/blobovniczatree/delete.go | 2 +- .../blobstor/blobovniczatree/exists.go | 2 +- .../blobstor/blobovniczatree/get.go | 2 +- .../blobstor/blobovniczatree/get_range.go | 2 +- .../blobstor/blobovniczatree/iterate.go | 29 ++++++++++++------- .../blobstor/blobovniczatree/put.go | 2 +- 10 files changed, 41 insertions(+), 25 deletions(-) diff --git a/cmd/frostfs-lens/internal/blobovnicza/list.go b/cmd/frostfs-lens/internal/blobovnicza/list.go index 67242a7d1..d327dbc41 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/list.go +++ b/cmd/frostfs-lens/internal/blobovnicza/list.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "context" "fmt" "io" @@ -33,6 +34,6 @@ func listFunc(cmd *cobra.Command, _ []string) { blz := openBlobovnicza(cmd) defer blz.Close() - err := blobovnicza.IterateAddresses(blz, wAddr) + err := blobovnicza.IterateAddresses(context.Background(), blz, wAddr) common.ExitOnErr(cmd, common.Errf("blobovnicza iterator failure: %w", err)) } diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index 1adfacbc0..8f8357053 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "context" "fmt" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -117,12 +118,17 @@ type IterateRes struct { // Returns handler's errors directly. Returns nil after iterating finish. // // Handler should not retain object data. Handler must not be nil. -func (b *Blobovnicza) Iterate(prm IteratePrm) (IterateRes, error) { +func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes, error) { var elem IterationElement if err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(name []byte, buck *bbolt.Bucket) error { return buck.ForEach(func(k, v []byte) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } if prm.decodeAddresses { if err := addressFromKey(&elem.addr, k); err != nil { if prm.ignoreErrors { @@ -147,7 +153,7 @@ func (b *Blobovnicza) Iterate(prm IteratePrm) (IterateRes, error) { } // IterateAddresses is a helper function which iterates over Blobovnicza and passes addresses of the objects to f. -func IterateAddresses(blz *Blobovnicza, f func(oid.Address) error) error { +func IterateAddresses(ctx context.Context, blz *Blobovnicza, f func(oid.Address) error) error { var prm IteratePrm prm.DecodeAddresses() @@ -157,7 +163,7 @@ func IterateAddresses(blz *Blobovnicza, f func(oid.Address) error) error { return f(elem.Address()) }) - _, err := blz.Iterate(prm) + _, err := blz.Iterate(ctx, prm) return err } diff --git a/pkg/local_object_storage/blobovnicza/iterate_test.go b/pkg/local_object_storage/blobovnicza/iterate_test.go index 6ecb20c77..505685ced 100644 --- a/pkg/local_object_storage/blobovnicza/iterate_test.go +++ b/pkg/local_object_storage/blobovnicza/iterate_test.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "context" "errors" "path/filepath" "testing" @@ -33,22 +34,22 @@ func TestBlobovniczaIterate(t *testing.T) { return nil } - _, err = b.Iterate(IteratePrm{handler: inc}) + _, err = b.Iterate(context.Background(), IteratePrm{handler: inc}) require.NoError(t, err) require.ElementsMatch(t, seen, data) seen = seen[:0] - _, err = b.Iterate(IteratePrm{handler: inc, decodeAddresses: true}) + _, err = b.Iterate(context.Background(), IteratePrm{handler: inc, decodeAddresses: true}) require.Error(t, err) seen = seen[:0] - _, err = b.Iterate(IteratePrm{handler: inc, decodeAddresses: true, ignoreErrors: true}) + _, err = b.Iterate(context.Background(), IteratePrm{handler: inc, decodeAddresses: true, ignoreErrors: true}) require.NoError(t, err) require.ElementsMatch(t, seen, data[:1]) seen = seen[:0] expectedErr := errors.New("stop iteration") - _, err = b.Iterate(IteratePrm{ + _, err = b.Iterate(context.Background(), IteratePrm{ decodeAddresses: true, handler: func(IterationElement) error { return expectedErr }, ignoreErrors: true, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 0240c7a97..0b2135631 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "context" "fmt" "path/filepath" @@ -26,7 +27,7 @@ func (b *Blobovniczas) Init() error { return nil } - return b.iterateLeaves(func(p string) (bool, error) { + return b.iterateLeaves(context.TODO(), func(p string) (bool, error) { blz, err := b.openBlobovniczaNoCache(p) if err != nil { return true, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index f84d8fbe8..5aa9062a6 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -48,7 +48,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co activeCache := make(map[string]struct{}) objectFound := false - err = b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) // don't process active blobovnicza of the level twice diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index 9d9fd4cba..e7852612b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -40,7 +40,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common gPrm.SetAddress(prm.Address) var found bool - err := b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err := b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) _, ok := activeCache[dirPath] diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 0b8ccb64f..8955eb148 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -46,7 +46,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G activeCache := make(map[string]struct{}) - err = b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) _, ok := activeCache[dirPath] diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index d6dfe51bd..fb23a9671 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -46,7 +46,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re activeCache := make(map[string]struct{}) objectFound := false - err = b.iterateSortedLeaves(&prm.Address, func(p string) (bool, error) { + err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) _, ok := activeCache[dirPath] diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index ad933da0b..140716690 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -12,8 +12,8 @@ import ( ) // Iterate iterates over all objects in b. -func (b *Blobovniczas) Iterate(_ context.Context, prm common.IteratePrm) (common.IterateRes, error) { - return common.IterateRes{}, b.iterateBlobovniczas(prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { +func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { + return common.IterateRes{}, b.iterateBlobovniczas(ctx, prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { var subPrm blobovnicza.IteratePrm subPrm.SetHandler(func(elem blobovnicza.IterationElement) error { data, err := b.compression.Decompress(elem.ObjectData()) @@ -40,14 +40,14 @@ func (b *Blobovniczas) Iterate(_ context.Context, prm common.IteratePrm) (common }) subPrm.DecodeAddresses() - _, err := blz.Iterate(subPrm) + _, err := blz.Iterate(ctx, subPrm) return err }) } // iterator over all Blobovniczas in unsorted order. Break on f's error return. -func (b *Blobovniczas) iterateBlobovniczas(ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error { - return b.iterateLeaves(func(p string) (bool, error) { +func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error { + return b.iterateLeaves(ctx, func(p string) (bool, error) { blz, err := b.openBlobovnicza(p) if err != nil { if ignoreErrors { @@ -63,8 +63,9 @@ func (b *Blobovniczas) iterateBlobovniczas(ignoreErrors bool, f func(string, *bl } // iterator over the paths of Blobovniczas sorted by weight. -func (b *Blobovniczas) iterateSortedLeaves(addr *oid.Address, f func(string) (bool, error)) error { +func (b *Blobovniczas) iterateSortedLeaves(ctx context.Context, addr *oid.Address, f func(string) (bool, error)) error { _, err := b.iterateSorted( + ctx, addr, make([]string, 0, b.blzShallowDepth), b.blzShallowDepth, @@ -75,13 +76,14 @@ func (b *Blobovniczas) iterateSortedLeaves(addr *oid.Address, f func(string) (bo } // iterator over directories with Blobovniczas sorted by weight. -func (b *Blobovniczas) iterateDeepest(addr oid.Address, f func(string) (bool, error)) error { +func (b *Blobovniczas) iterateDeepest(ctx context.Context, addr oid.Address, f func(string) (bool, error)) error { depth := b.blzShallowDepth if depth > 0 { depth-- } _, err := b.iterateSorted( + ctx, &addr, make([]string, 0, depth), depth, @@ -92,7 +94,7 @@ func (b *Blobovniczas) iterateDeepest(addr oid.Address, f func(string) (bool, er } // iterator over particular level of directories. -func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) { +func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) { indices := indexSlice(b.blzShallowWidth) hrw.SortSliceByValue(indices, addressHash(addr, filepath.Join(curPath...))) @@ -100,6 +102,11 @@ func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDe exec := uint64(len(curPath)) == execDepth for i := range indices { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } if i == 0 { curPath = append(curPath, u64ToHexString(indices[i])) } else { @@ -112,7 +119,7 @@ func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDe } else if stop { return true, nil } - } else if stop, err := b.iterateSorted(addr, curPath, execDepth, f); err != nil { + } else if stop, err := b.iterateSorted(ctx, addr, curPath, execDepth, f); err != nil { return false, err } else if stop { return true, nil @@ -123,8 +130,8 @@ func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDe } // iterator over the paths of Blobovniczas in random order. -func (b *Blobovniczas) iterateLeaves(f func(string) (bool, error)) error { - return b.iterateSortedLeaves(nil, f) +func (b *Blobovniczas) iterateLeaves(ctx context.Context, f func(string) (bool, error)) error { + return b.iterateSortedLeaves(ctx, nil, f) } // makes slice of uint64 values from 0 to number-1. diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index ec302d143..4e1d6621f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -45,7 +45,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe PutPrm: putPrm, } - if err := b.iterateDeepest(prm.Address, it.iterate); err != nil { + if err := b.iterateDeepest(ctx, prm.Address, it.iterate); err != nil { return common.PutRes{}, err } else if it.ID == nil { if it.AllFull { From ebcc8afbeeeb97c66cdfefab3cde227081b6c59e Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 26 May 2023 13:24:41 +0300 Subject: [PATCH 0288/1943] [#374] Add inner-ring event metrics Signed-off-by: Alejandro Lopez --- pkg/innerring/initialization.go | 6 +++ pkg/innerring/metrics/metrics.go | 15 ++++++ pkg/innerring/processors/alphabet/handlers.go | 3 +- .../processors/alphabet/process_emit.go | 16 ++++--- .../processors/alphabet/processor.go | 9 ++++ pkg/innerring/processors/balance/handlers.go | 5 +- .../processors/balance/process_assets.go | 7 ++- pkg/innerring/processors/balance/processor.go | 9 ++++ .../processors/container/handlers.go | 13 +++-- .../processors/container/process_container.go | 47 +++++++++---------- .../processors/container/process_eacl.go | 23 +++++---- .../processors/container/processor.go | 9 ++++ pkg/innerring/processors/frostfs/handlers.go | 25 +++++++--- .../processors/frostfs/process_assets.go | 30 +++++++----- .../processors/frostfs/process_bind.go | 14 +++--- .../processors/frostfs/process_config.go | 7 ++- pkg/innerring/processors/frostfs/processor.go | 9 ++++ .../processors/governance/handlers.go | 5 +- .../processors/governance/process_update.go | 14 +++--- .../processors/governance/processor.go | 11 ++++- pkg/innerring/processors/netmap/handlers.go | 19 ++++---- .../processors/netmap/process_cleanup.go | 7 ++- .../processors/netmap/process_epoch.go | 13 +++-- .../processors/netmap/process_peers.go | 22 +++++---- pkg/innerring/processors/netmap/processor.go | 9 ++++ pkg/innerring/processors/util.go | 16 +++++++ pkg/metrics/innerring.go | 37 ++++++++++++--- 27 files changed, 287 insertions(+), 113 deletions(-) create mode 100644 pkg/innerring/metrics/metrics.go create mode 100644 pkg/innerring/processors/util.go diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 8e5aef951..c49d22509 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -52,6 +52,7 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, s.netmapProcessor, err = netmap.New(&netmap.Params{ Log: s.log, + Metrics: s.metrics, PoolSize: cfg.GetInt("workers.netmap"), NetmapClient: netmap.NewNetmapClient(s.netmapClient), EpochTimer: s, @@ -162,6 +163,7 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli // create governance processor governanceProcessor, err := governance.New(&governance.Params{ Log: s.log, + Metrics: s.metrics, FrostFSClient: frostfsCli, NetmapClient: s.netmapClient, AlphabetState: s, @@ -231,6 +233,7 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { s.alphabetProcessor, err = alphabet.New(&alphabet.Params{ ParsedWallets: parsedWallets, Log: s.log, + Metrics: s.metrics, PoolSize: cfg.GetInt("workers.alphabet"), AlphabetContracts: s.contracts.alphabet, NetmapClient: s.netmapClient, @@ -255,6 +258,7 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C // container processor containerProcessor, err := cont.New(&cont.Params{ Log: s.log, + Metrics: s.metrics, PoolSize: cfg.GetInt("workers.container"), AlphabetState: s, ContainerClient: cnrClient, @@ -273,6 +277,7 @@ func (s *Server) initBalanceProcessor(cfg *viper.Viper, frostfsCli *frostfsClien // create balance processor balanceProcessor, err := balance.New(&balance.Params{ Log: s.log, + Metrics: s.metrics, PoolSize: cfg.GetInt("workers.balance"), FrostFSClient: frostfsCli, BalanceSC: s.contracts.balance, @@ -293,6 +298,7 @@ func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper, frostfsIDClient * frostfsProcessor, err := frostfs.New(&frostfs.Params{ Log: s.log, + Metrics: s.metrics, PoolSize: cfg.GetInt("workers.frostfs"), FrostFSContract: s.contracts.frostfs, FrostFSIDClient: frostfsIDClient, diff --git a/pkg/innerring/metrics/metrics.go b/pkg/innerring/metrics/metrics.go new file mode 100644 index 000000000..002f3afe1 --- /dev/null +++ b/pkg/innerring/metrics/metrics.go @@ -0,0 +1,15 @@ +package metrics + +import "time" + +type Register interface { + SetEpoch(epoch uint64) + SetHealth(s int32) + AddEvent(d time.Duration, typ string, success bool) +} + +type DefaultRegister struct{} + +func (DefaultRegister) SetEpoch(uint64) {} +func (DefaultRegister) SetHealth(int32) {} +func (DefaultRegister) AddEvent(time.Duration, string, bool) {} diff --git a/pkg/innerring/processors/alphabet/handlers.go b/pkg/innerring/processors/alphabet/handlers.go index c0668a4f9..9de075f17 100644 --- a/pkg/innerring/processors/alphabet/handlers.go +++ b/pkg/innerring/processors/alphabet/handlers.go @@ -2,6 +2,7 @@ package alphabet import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "go.uber.org/zap" @@ -13,7 +14,7 @@ func (ap *Processor) HandleGasEmission(ev event.Event) { // send event to the worker pool - err := ap.pool.Submit(func() { ap.processEmit() }) + err := processors.SubmitEvent(ap.pool, ap.metrics, "alphabet_emit_gas", ap.processEmit) if err != nil { // there system can be moved into controlled degradation stage ap.log.Warn(logs.AlphabetAlphabetProcessorWorkerPoolDrained, diff --git a/pkg/innerring/processors/alphabet/process_emit.go b/pkg/innerring/processors/alphabet/process_emit.go index 7a268ac52..8a2336011 100644 --- a/pkg/innerring/processors/alphabet/process_emit.go +++ b/pkg/innerring/processors/alphabet/process_emit.go @@ -13,12 +13,12 @@ import ( const emitMethod = "emit" -func (ap *Processor) processEmit() { +func (ap *Processor) processEmit() bool { index := ap.irList.AlphabetIndex() if index < 0 { ap.log.Info(logs.AlphabetNonAlphabetModeIgnoreGasEmissionEvent) - return + return true } contract, ok := ap.alphabetContracts.GetByIndex(index) @@ -26,7 +26,7 @@ func (ap *Processor) processEmit() { ap.log.Debug(logs.AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent, zap.Int("index", index)) - return + return false } // there is no signature collecting, so we don't need extra fee @@ -34,13 +34,13 @@ func (ap *Processor) processEmit() { if err != nil { ap.log.Warn(logs.AlphabetCantInvokeAlphabetEmitMethod, zap.String("error", err.Error())) - return + return false } if ap.storageEmission == 0 { ap.log.Info(logs.AlphabetStorageNodeEmissionIsOff) - return + return true } networkMap, err := ap.netmapClient.NetMap() @@ -48,7 +48,7 @@ func (ap *Processor) processEmit() { ap.log.Warn(logs.AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes, zap.String("error", err.Error())) - return + return false } nmNodes := networkMap.Nodes() @@ -63,7 +63,7 @@ func (ap *Processor) processEmit() { zap.Int("extra_wallets", extraLen)) if nmLen+extraLen == 0 { - return + return true } gasPerNode := fixedn.Fixed8(ap.storageEmission / uint64(nmLen+extraLen)) @@ -71,6 +71,8 @@ func (ap *Processor) processEmit() { ap.transferGasToNetmapNodes(nmNodes, gasPerNode) ap.transferGasToExtraNodes(pw, gasPerNode) + + return true } func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerNode fixedn.Fixed8) { diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index cd9088e03..972f84281 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -7,6 +7,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -50,6 +51,7 @@ type ( // protects parsedWallets from concurrent change pwLock *sync.RWMutex log *logger.Logger + metrics metrics.Register pool *ants.Pool alphabetContracts Contracts netmapClient netmapClient @@ -62,6 +64,7 @@ type ( Params struct { ParsedWallets []util.Uint160 Log *logger.Logger + Metrics metrics.Register PoolSize int AlphabetContracts Contracts NetmapClient netmapClient @@ -89,10 +92,16 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/frostfs: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ parsedWallets: p.ParsedWallets, pwLock: new(sync.RWMutex), log: p.Log, + metrics: metricsRegister, pool: pool, alphabetContracts: p.AlphabetContracts, netmapClient: p.NetmapClient, diff --git a/pkg/innerring/processors/balance/handlers.go b/pkg/innerring/processors/balance/handlers.go index e325da1f9..e39f3abbd 100644 --- a/pkg/innerring/processors/balance/handlers.go +++ b/pkg/innerring/processors/balance/handlers.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" balanceEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/balance" "go.uber.org/zap" @@ -17,7 +18,9 @@ func (bp *Processor) handleLock(ev event.Event) { // send an event to the worker pool - err := bp.pool.Submit(func() { bp.processLock(&lock) }) + err := processors.SubmitEvent(bp.pool, bp.metrics, "lock", func() bool { + return bp.processLock(&lock) + }) if err != nil { // there system can be moved into controlled degradation stage bp.log.Warn(logs.BalanceBalanceWorkerPoolDrained, diff --git a/pkg/innerring/processors/balance/process_assets.go b/pkg/innerring/processors/balance/process_assets.go index 3f86a3cb7..1d94fa454 100644 --- a/pkg/innerring/processors/balance/process_assets.go +++ b/pkg/innerring/processors/balance/process_assets.go @@ -9,10 +9,10 @@ import ( // Process lock event by invoking Cheque method in main net to send assets // back to the withdraw issuer. -func (bp *Processor) processLock(lock *balanceEvent.Lock) { +func (bp *Processor) processLock(lock *balanceEvent.Lock) bool { if !bp.alphabetState.IsAlphabet() { bp.log.Info(logs.BalanceNonAlphabetModeIgnoreBalanceLock) - return + return true } prm := frostfsContract.ChequePrm{} @@ -26,5 +26,8 @@ func (bp *Processor) processLock(lock *balanceEvent.Lock) { err := bp.frostfsClient.Cheque(prm) if err != nil { bp.log.Error(logs.BalanceCantSendLockAssetTx, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/balance/processor.go b/pkg/innerring/processors/balance/processor.go index 356754cfb..5cc849b5c 100644 --- a/pkg/innerring/processors/balance/processor.go +++ b/pkg/innerring/processors/balance/processor.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" balanceEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/balance" @@ -32,6 +33,7 @@ type ( // Processor of events produced by balance contract in the morphchain. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool frostfsClient FrostFSClient balanceSC util.Uint160 @@ -42,6 +44,7 @@ type ( // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int FrostFSClient FrostFSClient BalanceSC util.Uint160 @@ -72,8 +75,14 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/balance: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, frostfsClient: p.FrostFSClient, balanceSC: p.BalanceSC, diff --git a/pkg/innerring/processors/container/handlers.go b/pkg/innerring/processors/container/handlers.go index 2ab1147c8..3ec10b889 100644 --- a/pkg/innerring/processors/container/handlers.go +++ b/pkg/innerring/processors/container/handlers.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "github.com/mr-tron/base58" @@ -20,7 +21,9 @@ func (cp *Processor) handlePut(ev event.Event) { // send an event to the worker pool - err := cp.pool.Submit(func() { cp.processContainerPut(put) }) + err := processors.SubmitEvent(cp.pool, cp.metrics, "container_put", func() bool { + return cp.processContainerPut(put) + }) if err != nil { // there system can be moved into controlled degradation stage cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, @@ -36,7 +39,9 @@ func (cp *Processor) handleDelete(ev event.Event) { // send an event to the worker pool - err := cp.pool.Submit(func() { cp.processContainerDelete(del) }) + err := processors.SubmitEvent(cp.pool, cp.metrics, "container_delete", func() bool { + return cp.processContainerDelete(del) + }) if err != nil { // there system can be moved into controlled degradation stage cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, @@ -53,8 +58,8 @@ func (cp *Processor) handleSetEACL(ev event.Event) { // send an event to the worker pool - err := cp.pool.Submit(func() { - cp.processSetEACL(e) + err := processors.SubmitEvent(cp.pool, cp.metrics, "container_set_eacl", func() bool { + return cp.processSetEACL(e) }) if err != nil { // there system can be moved into controlled degradation stage diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 3bee1c4d5..33ef90034 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -31,10 +31,10 @@ type putContainerContext struct { // Process a new container from the user by checking the container sanity // and sending approve tx back to the morph. -func (cp *Processor) processContainerPut(put putEvent) { +func (cp *Processor) processContainerPut(put putEvent) bool { if !cp.alphabetState.IsAlphabet() { cp.log.Info(logs.ContainerNonAlphabetModeIgnoreContainerPut) - return + return true } ctx := &putContainerContext{ @@ -47,10 +47,17 @@ func (cp *Processor) processContainerPut(put putEvent) { zap.String("error", err.Error()), ) - return + return false } - cp.approvePutContainer(ctx) + if err := cp.morphClient.NotarySignAndInvokeTX(ctx.e.NotaryRequest().MainTransaction); err != nil { + cp.log.Error(logs.ContainerCouldNotApprovePutContainer, + zap.String("error", err.Error()), + ) + return false + } + + return true } func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { @@ -89,20 +96,12 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { return nil } -func (cp *Processor) approvePutContainer(ctx *putContainerContext) { - if err := cp.morphClient.NotarySignAndInvokeTX(ctx.e.NotaryRequest().MainTransaction); err != nil { - cp.log.Error(logs.ContainerCouldNotApprovePutContainer, - zap.String("error", err.Error()), - ) - } -} - // Process delete container operation from the user by checking container sanity // and sending approve tx back to morph. -func (cp *Processor) processContainerDelete(e containerEvent.Delete) { +func (cp *Processor) processContainerDelete(e containerEvent.Delete) bool { if !cp.alphabetState.IsAlphabet() { cp.log.Info(logs.ContainerNonAlphabetModeIgnoreContainerDelete) - return + return true } err := cp.checkDeleteContainer(e) @@ -111,10 +110,18 @@ func (cp *Processor) processContainerDelete(e containerEvent.Delete) { zap.String("error", err.Error()), ) - return + return false } - cp.approveDeleteContainer(e) + if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { + cp.log.Error(logs.ContainerCouldNotApproveDeleteContainer, + zap.String("error", err.Error()), + ) + + return false + } + + return true } func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { @@ -149,14 +156,6 @@ func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { return nil } -func (cp *Processor) approveDeleteContainer(e containerEvent.Delete) { - if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { - cp.log.Error(logs.ContainerCouldNotApproveDeleteContainer, - zap.String("error", err.Error()), - ) - } -} - func checkNNS(ctx *putContainerContext, cnr containerSDK.Container) error { // fetch domain info ctx.d = containerSDK.ReadDomain(cnr) diff --git a/pkg/innerring/processors/container/process_eacl.go b/pkg/innerring/processors/container/process_eacl.go index 43a59e223..8ab0d5c39 100644 --- a/pkg/innerring/processors/container/process_eacl.go +++ b/pkg/innerring/processors/container/process_eacl.go @@ -12,10 +12,10 @@ import ( "go.uber.org/zap" ) -func (cp *Processor) processSetEACL(e containerEvent.SetEACL) { +func (cp *Processor) processSetEACL(e containerEvent.SetEACL) bool { if !cp.alphabetState.IsAlphabet() { cp.log.Info(logs.ContainerNonAlphabetModeIgnoreSetEACL) - return + return true } err := cp.checkSetEACL(e) @@ -24,10 +24,17 @@ func (cp *Processor) processSetEACL(e containerEvent.SetEACL) { zap.String("error", err.Error()), ) - return + return false } - cp.approveSetEACL(e) + if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { + cp.log.Error(logs.ContainerCouldNotApproveSetEACL, + zap.String("error", err.Error()), + ) + return false + } + + return true } func (cp *Processor) checkSetEACL(e containerEvent.SetEACL) error { @@ -73,11 +80,3 @@ func (cp *Processor) checkSetEACL(e containerEvent.SetEACL) error { return nil } - -func (cp *Processor) approveSetEACL(e containerEvent.SetEACL) { - if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { - cp.log.Error(logs.ContainerCouldNotApproveSetEACL, - zap.String("error", err.Error()), - ) - } -} diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index ec82ace70..fd5348c6f 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" @@ -40,6 +41,7 @@ type ( // Processor of events produced by container contract in the sidechain. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool alphabetState AlphabetState cnrClient ContClient // notary must be enabled @@ -51,6 +53,7 @@ type ( // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int AlphabetState AlphabetState ContainerClient ContClient @@ -102,8 +105,14 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/container: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, alphabetState: p.AlphabetState, cnrClient: p.ContainerClient, diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index 574cf057a..ab53d5c48 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -18,7 +19,9 @@ func (np *Processor) handleDeposit(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processDeposit(deposit) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_deposit", func() bool { + return np.processDeposit(deposit) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -34,7 +37,9 @@ func (np *Processor) handleWithdraw(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processWithdraw(withdraw) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_withdraw", func() bool { + return np.processWithdraw(withdraw) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -50,7 +55,9 @@ func (np *Processor) handleCheque(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processCheque(cheque) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_cheque", func() bool { + return np.processCheque(cheque) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -67,7 +74,9 @@ func (np *Processor) handleConfig(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processConfig(cfg) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_config", func() bool { + return np.processConfig(cfg) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -83,7 +92,9 @@ func (np *Processor) handleBind(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processBind(e, true) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_bind", func() bool { + return np.processBind(e, true) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, @@ -99,7 +110,9 @@ func (np *Processor) handleUnbind(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { np.processBind(e, false) }) + err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_unbind", func() bool { + return np.processBind(e, false) + }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, diff --git a/pkg/innerring/processors/frostfs/process_assets.go b/pkg/innerring/processors/frostfs/process_assets.go index cfbf21b08..327a4a3aa 100644 --- a/pkg/innerring/processors/frostfs/process_assets.go +++ b/pkg/innerring/processors/frostfs/process_assets.go @@ -15,10 +15,10 @@ const ( // Process deposit event by invoking a balance contract and sending native // gas in the sidechain. -func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { +func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreDeposit) - return + return true } prm := balance.MintPrm{} @@ -49,7 +49,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { zap.Uint64("last_emission", val), zap.Uint64("current_epoch", curEpoch)) - return + return false } // get gas balance of the node @@ -57,7 +57,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { balance, err := np.morphClient.GasBalance() if err != nil { np.log.Error(logs.FrostFSCantGetGasBalanceOfTheNode, zap.Error(err)) - return + return false } if balance < np.gasBalanceThreshold { @@ -65,7 +65,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { zap.Int64("balance", balance), zap.Int64("threshold", np.gasBalanceThreshold)) - return + return false } err = np.morphClient.TransferGas(receiver, np.mintEmitValue) @@ -73,24 +73,26 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) { np.log.Error(logs.FrostFSCantTransferNativeGasToReceiver, zap.String("error", err.Error())) - return + return false } np.mintEmitCache.Add(receiver.String(), curEpoch) + + return true } // Process withdraw event by locking assets in the balance account. -func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) { +func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreWithdraw) - return + return true } // create lock account lock, err := util.Uint160DecodeBytesBE(withdraw.ID()[:util.Uint160Size]) if err != nil { np.log.Error(logs.FrostFSCantCreateLockAccount, zap.Error(err)) - return + return false } curEpoch := np.epochState.EpochCounter() @@ -106,15 +108,18 @@ func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) { err = np.balanceClient.Lock(prm) if err != nil { np.log.Error(logs.FrostFSCantLockAssetsForWithdraw, zap.Error(err)) + return false } + + return true } // Process cheque event by transferring assets from the lock account back to // the reserve account. -func (np *Processor) processCheque(cheque frostfsEvent.Cheque) { +func (np *Processor) processCheque(cheque frostfsEvent.Cheque) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreCheque) - return + return true } prm := balance.BurnPrm{} @@ -126,5 +131,8 @@ func (np *Processor) processCheque(cheque frostfsEvent.Cheque) { err := np.balanceClient.Burn(prm) if err != nil { np.log.Error(logs.FrostFSCantTransferAssetsToFedContract, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/frostfs/process_bind.go b/pkg/innerring/processors/frostfs/process_bind.go index a9b523a77..50c6bf5f5 100644 --- a/pkg/innerring/processors/frostfs/process_bind.go +++ b/pkg/innerring/processors/frostfs/process_bind.go @@ -18,10 +18,10 @@ type bindCommon interface { TxHash() util.Uint256 } -func (np *Processor) processBind(e bindCommon, bind bool) { +func (np *Processor) processBind(e bindCommon, bind bool) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreBind) - return + return true } c := &bindCommonContext{ @@ -36,10 +36,10 @@ func (np *Processor) processBind(e bindCommon, bind bool) { zap.String("error", err.Error()), ) - return + return false } - np.approveBindCommon(c) + return np.approveBindCommon(c) == nil } type bindCommonContext struct { @@ -70,7 +70,7 @@ func (np *Processor) checkBindCommon(e *bindCommonContext) error { return nil } -func (np *Processor) approveBindCommon(e *bindCommonContext) { +func (np *Processor) approveBindCommon(e *bindCommonContext) error { // calculate wallet address scriptHash := e.User() @@ -80,7 +80,7 @@ func (np *Processor) approveBindCommon(e *bindCommonContext) { zap.String("error", err.Error()), ) - return + return err } var id user.ID @@ -104,4 +104,6 @@ func (np *Processor) approveBindCommon(e *bindCommonContext) { np.log.Error(fmt.Sprintf("could not approve %s", typ), zap.String("error", err.Error())) } + + return err } diff --git a/pkg/innerring/processors/frostfs/process_config.go b/pkg/innerring/processors/frostfs/process_config.go index ce2dabfdd..2ae3e6ced 100644 --- a/pkg/innerring/processors/frostfs/process_config.go +++ b/pkg/innerring/processors/frostfs/process_config.go @@ -9,10 +9,10 @@ import ( // Process config event by setting configuration value from the mainchain in // the sidechain. -func (np *Processor) processConfig(config frostfsEvent.Config) { +func (np *Processor) processConfig(config frostfsEvent.Config) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.FrostFSNonAlphabetModeIgnoreConfig) - return + return true } prm := nmClient.SetConfigPrm{} @@ -25,5 +25,8 @@ func (np *Processor) processConfig(config frostfsEvent.Config) { err := np.netmapClient.SetConfig(prm) if err != nil { np.log.Error(logs.FrostFSCantRelaySetConfigEvent, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index 2af15a814..e6ee78837 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -6,6 +6,7 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -58,6 +59,7 @@ type ( // Processor of events produced by frostfs contract in main net. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool frostfsContract util.Uint160 balanceClient BalanceClient @@ -77,6 +79,7 @@ type ( // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int FrostFSContract util.Uint160 FrostFSIDClient IDClient @@ -129,8 +132,14 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/frostfs: can't create LRU cache for gas emission: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, frostfsContract: p.FrostFSContract, balanceClient: p.BalanceClient, diff --git a/pkg/innerring/processors/governance/handlers.go b/pkg/innerring/processors/governance/handlers.go index 727acc21a..fd7f539c3 100644 --- a/pkg/innerring/processors/governance/handlers.go +++ b/pkg/innerring/processors/governance/handlers.go @@ -2,6 +2,7 @@ package governance import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/rolemanagement" "github.com/nspcc-dev/neo-go/pkg/core/native" @@ -35,7 +36,9 @@ func (gp *Processor) HandleAlphabetSync(e event.Event) { // send event to the worker pool - err := gp.pool.Submit(func() { gp.processAlphabetSync(hash) }) + err := processors.SubmitEvent(gp.pool, gp.metrics, "alphabet_sync", func() bool { + return gp.processAlphabetSync(hash) + }) if err != nil { // there system can be moved into controlled degradation stage gp.log.Warn(logs.GovernanceGovernanceWorkerPoolDrained, diff --git a/pkg/innerring/processors/governance/process_update.go b/pkg/innerring/processors/governance/process_update.go index 3eae676d4..50ba58e77 100644 --- a/pkg/innerring/processors/governance/process_update.go +++ b/pkg/innerring/processors/governance/process_update.go @@ -18,36 +18,36 @@ const ( alphabetUpdateIDPrefix = "AlphabetUpdate" ) -func (gp *Processor) processAlphabetSync(txHash util.Uint256) { +func (gp *Processor) processAlphabetSync(txHash util.Uint256) bool { if !gp.alphabetState.IsAlphabet() { gp.log.Info(logs.GovernanceNonAlphabetModeIgnoreAlphabetSync) - return + return true } mainnetAlphabet, err := gp.mainnetClient.NeoFSAlphabetList() if err != nil { gp.log.Error(logs.GovernanceCantFetchAlphabetListFromMainNet, zap.String("error", err.Error())) - return + return false } sidechainAlphabet, err := gp.morphClient.Committee() if err != nil { gp.log.Error(logs.GovernanceCantFetchAlphabetListFromSideChain, zap.String("error", err.Error())) - return + return false } newAlphabet, err := newAlphabetList(sidechainAlphabet, mainnetAlphabet) if err != nil { gp.log.Error(logs.GovernanceCantMergeAlphabetListsFromMainNetAndSideChain, zap.String("error", err.Error())) - return + return false } if newAlphabet == nil { gp.log.Info(logs.GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged) - return + return true } gp.log.Info(logs.GovernanceAlphabetListHasBeenChangedStartingUpdate, @@ -77,6 +77,8 @@ func (gp *Processor) processAlphabetSync(txHash util.Uint256) { gp.updateFrostFSContractInMainnet(newAlphabet) gp.log.Info(logs.GovernanceFinishedAlphabetListUpdate) + + return true } func prettyKeys(keys keys.PublicKeys) string { diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index 07b5b5cef..fa267eade 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -75,6 +76,7 @@ type ( // Processor of events related to governance in the network. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool frostfsClient FrostFSClient netmapClient NetmapClient @@ -92,7 +94,8 @@ type ( // Params of the processor constructor. Params struct { - Log *logger.Logger + Log *logger.Logger + Metrics metrics.Register AlphabetState AlphabetState EpochState EpochState @@ -130,11 +133,17 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/governance: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + // result is cached by neo-go, so we can pre-calc it designate := p.MainnetClient.GetDesignateHash() return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, frostfsClient: p.FrostFSClient, netmapClient: p.NetmapClient, diff --git a/pkg/innerring/processors/netmap/handlers.go b/pkg/innerring/processors/netmap/handlers.go index 6adeac562..c6053e281 100644 --- a/pkg/innerring/processors/netmap/handlers.go +++ b/pkg/innerring/processors/netmap/handlers.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" @@ -16,7 +17,7 @@ func (np *Processor) HandleNewEpochTick(ev event.Event) { // send an event to the worker pool - err := np.pool.Submit(func() { np.processNewEpochTick() }) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_new_epoch_tick", np.processNewEpochTick) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, @@ -32,8 +33,8 @@ func (np *Processor) handleNewEpoch(ev event.Event) { // send an event to the worker pool - err := np.pool.Submit(func() { - np.processNewEpoch(epochEvent) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_new_epoch", func() bool { + return np.processNewEpoch(epochEvent) }) if err != nil { // there system can be moved into controlled degradation stage @@ -51,8 +52,8 @@ func (np *Processor) handleAddPeer(ev event.Event) { // send an event to the worker pool - err := np.pool.Submit(func() { - np.processAddPeer(newPeer) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_add_peer", func() bool { + return np.processAddPeer(newPeer) }) if err != nil { // there system can be moved into controlled degradation stage @@ -69,8 +70,8 @@ func (np *Processor) handleUpdateState(ev event.Event) { // send event to the worker pool - err := np.pool.Submit(func() { - np.processUpdatePeer(updPeer) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_update_peer", func() bool { + return np.processUpdatePeer(updPeer) }) if err != nil { // there system can be moved into controlled degradation stage @@ -91,8 +92,8 @@ func (np *Processor) handleCleanupTick(ev event.Event) { np.log.Info(logs.NetmapTick, zap.String("type", "netmap cleaner")) // send event to the worker pool - err := np.pool.Submit(func() { - np.processNetmapCleanupTick(cleanup) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_cleanup_tick", func() bool { + return np.processNetmapCleanupTick(cleanup) }) if err != nil { // there system can be moved into controlled degradation stage diff --git a/pkg/innerring/processors/netmap/process_cleanup.go b/pkg/innerring/processors/netmap/process_cleanup.go index 287844a6a..170c39e2c 100644 --- a/pkg/innerring/processors/netmap/process_cleanup.go +++ b/pkg/innerring/processors/netmap/process_cleanup.go @@ -7,11 +7,11 @@ import ( "go.uber.org/zap" ) -func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { +func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick) - return + return true } err := np.netmapSnapshot.forEachRemoveCandidate(ev.epoch, func(s string) error { @@ -47,5 +47,8 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) { if err != nil { np.log.Warn(logs.NetmapCantIterateOnNetmapCleanerCache, zap.String("error", err.Error())) + return false } + + return true } diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index b655db9aa..01bfbae67 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -10,7 +10,7 @@ import ( // Process new epoch notification by setting global epoch value and resetting // local epoch timer. -func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { +func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) bool { epoch := ev.EpochNumber() epochDuration, err := np.netmapClient.EpochDuration() @@ -41,7 +41,7 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { np.log.Warn(logs.NetmapCantGetNetmapSnapshotToPerformCleanup, zap.String("error", err.Error())) - return + return false } prm := cntClient.StartEstimationPrm{} @@ -63,13 +63,15 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) { np.handleCleanupTick(netmapCleanupTick{epoch: epoch, txHash: ev.TxHash()}) np.handleAlphabetSync(governance.NewSyncEvent(ev.TxHash())) np.handleNotaryDeposit(ev) + + return true } // Process new epoch tick by invoking new epoch method in network map contract. -func (np *Processor) processNewEpochTick() { +func (np *Processor) processNewEpochTick() bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewEpochTick) - return + return true } nextEpoch := np.epochState.EpochCounter() + 1 @@ -78,5 +80,8 @@ func (np *Processor) processNewEpochTick() { err := np.netmapClient.NewEpoch(nextEpoch, false) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index e4f1a4d6c..96b8c8e97 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -12,10 +12,10 @@ import ( // Process add peer notification by sanity check of new node // local epoch timer. -func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { +func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewPeerNotification) - return + return true } // check if notary transaction is valid, see #976 @@ -26,7 +26,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { zap.String("method", "netmap.AddPeer"), zap.String("hash", tx.Hash().StringLE()), zap.Error(err)) - return + return false } // unmarshal node info @@ -34,7 +34,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { if err := nodeInfo.Unmarshal(ev.Node()); err != nil { // it will be nice to have tx id at event structure to log it np.log.Warn(logs.NetmapCantParseNetworkMapCandidate) - return + return false } // validate and update node info @@ -44,7 +44,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { zap.String("error", err.Error()), ) - return + return false } // sort attributes to make it consistent @@ -81,15 +81,18 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) { if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapAddPeer, zap.Error(err)) + return false } } + + return true } // Process update peer notification by sending approval tx to the smart contract. -func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { +func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) bool { if !np.alphabetState.IsAlphabet() { np.log.Info(logs.NetmapNonAlphabetModeIgnoreUpdatePeerNotification) - return + return true } // flag node to remove from local view, so it can be re-bootstrapped @@ -105,11 +108,14 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) { zap.Error(err), ) - return + return false } } if err = np.netmapClient.MorphNotarySignAndInvokeTX(ev.NotaryRequest().MainTransaction); err != nil { np.log.Error(logs.NetmapCantInvokeNetmapUpdatePeer, zap.Error(err)) + return false } + + return true } diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index 5984cbbe9..6b8a24a62 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -72,6 +73,7 @@ type ( // and new epoch ticker, because it is related to contract. Processor struct { log *logger.Logger + metrics metrics.Register pool *ants.Pool epochTimer EpochTimerReseter epochState EpochState @@ -93,6 +95,7 @@ type ( // Params of the processor constructor. Params struct { Log *logger.Logger + Metrics metrics.Register PoolSize int NetmapClient Client EpochTimer EpochTimerReseter @@ -145,8 +148,14 @@ func New(p *Params) (*Processor, error) { return nil, fmt.Errorf("ir/netmap: can't create worker pool: %w", err) } + metricsRegister := p.Metrics + if metricsRegister == nil { + metricsRegister = metrics.DefaultRegister{} + } + return &Processor{ log: p.Log, + metrics: metricsRegister, pool: pool, epochTimer: p.EpochTimer, epochState: p.EpochState, diff --git a/pkg/innerring/processors/util.go b/pkg/innerring/processors/util.go new file mode 100644 index 000000000..364ffe25e --- /dev/null +++ b/pkg/innerring/processors/util.go @@ -0,0 +1,16 @@ +package processors + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" + "github.com/panjf2000/ants/v2" +) + +func SubmitEvent(pool *ants.Pool, metrics metrics.Register, eventLabel string, eventProcessor func() bool) error { + return pool.Submit(func() { + start := time.Now() + success := eventProcessor() + metrics.AddEvent(time.Since(start), eventLabel, success) + }) +} diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 79db424b8..bff9184ec 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -1,13 +1,23 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "strconv" + "time" -const innerRingSubsystem = "ir" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + innerRingSubsystem = "ir" + innerRingLabelSuccess = "success" + innerRingLabelType = "type" +) // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { - epoch metric[prometheus.Gauge] - health metric[prometheus.Gauge] + epoch metric[prometheus.Gauge] + health metric[prometheus.Gauge] + eventDuration metric[*prometheus.HistogramVec] } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. @@ -25,14 +35,22 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { Name: "health", Help: "Current inner-ring node state.", }) + eventDuration = newHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: innerRingSubsystem, + Name: "event_duration_seconds", + Help: "Duration of processing of inner-ring events", + }, []string{innerRingLabelType, innerRingLabelSuccess}) ) mustRegister(epoch) mustRegister(health) + mustRegister(eventDuration) return &InnerRingServiceMetrics{ - epoch: epoch, - health: health, + epoch: epoch, + health: health, + eventDuration: eventDuration, } } @@ -45,3 +63,10 @@ func (m InnerRingServiceMetrics) SetEpoch(epoch uint64) { func (m InnerRingServiceMetrics) SetHealth(s int32) { m.health.value.Set(float64(s)) } + +func (m InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) { + m.eventDuration.value.With(prometheus.Labels{ + innerRingLabelType: typ, + innerRingLabelSuccess: strconv.FormatBool(success), + }).Observe(d.Seconds()) +} From f64322576a86c5b35489bf1adfca73e03aaba4ef Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 29 May 2023 18:44:46 +0300 Subject: [PATCH 0289/1943] [#402] cli: Add estimated evacuation time left Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/evacuation.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 45152fa0a..4eb6505cf 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -212,9 +212,28 @@ func printStatus(cmd *cobra.Command, resp *control.GetShardEvacuationStatusRespo appendError(sb, resp) appendStartedAt(sb, resp) appendDuration(sb, resp) + appendEstimation(sb, resp) cmd.Println(sb.String()) } +func appendEstimation(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { + if resp.GetBody().GetStatus() != control.GetShardEvacuationStatusResponse_Body_RUNNING || + resp.GetBody().GetDuration() == nil || + resp.GetBody().GetTotal() == 0 || + resp.GetBody().GetEvacuated()+resp.GetBody().GetFailed() == 0 { + return + } + + durationSeconds := float64(resp.GetBody().GetDuration().GetSeconds()) + evacuated := float64(resp.GetBody().GetEvacuated() + resp.GetBody().GetFailed()) + avgObjEvacuationTimeSeconds := durationSeconds / evacuated + objectsLeft := float64(resp.GetBody().GetTotal()) - evacuated + leftSeconds := avgObjEvacuationTimeSeconds * objectsLeft + leftMinutes := int(leftSeconds / 60) + + sb.WriteString(fmt.Sprintf(" Estimated time left: %d minutes.", leftMinutes)) +} + func appendDuration(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { if resp.GetBody().GetDuration() != nil { duration := time.Second * time.Duration(resp.GetBody().GetDuration().GetSeconds()) From f934abed8f2a1e6b6870ef8cc190fad987cc1845 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 30 May 2023 10:11:08 +0300 Subject: [PATCH 0290/1943] [#402] doc: Update evacuation docs Add estimated time left output to examples. Signed-off-by: Dmitrii Stepanov --- docs/evacuation.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/evacuation.md b/docs/evacuation.md index 5befad164..9bfa0e214 100644 --- a/docs/evacuation.md +++ b/docs/evacuation.md @@ -39,15 +39,15 @@ Shard evacuation has been successfully started. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 14 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:03. +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 14 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:03. Estimated time left: 2 minutes. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 23 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:05. +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 23 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:01:05. Estimated time left: 1 minutes. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: completed. Evacuated 61 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:13. +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: completed. Evacuated 61 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:02:13. ``` ### Stop running evacuation process @@ -58,7 +58,7 @@ Shard evacuation has been successfully started. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 15 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:03. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 15 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:03. Estimated time left: 0 minutes. frostfs-cli control shards evacuation stop --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > @@ -75,9 +75,9 @@ frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 - Enter password > Shard evacuation has been successfully started. Progress will be reported every 5 seconds. -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 18 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:04. -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 43 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:09. -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 68 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 18 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:04. Estimated time left: 0 minutes. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 43 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:09. Estimated time left: 0 minutes. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 68 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. Estimated time left: 0 minutes. Shard evacuation has been completed. Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. ``` From faca8614513a8fab594152b146d89270a0cd4576 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 30 May 2023 10:14:37 +0300 Subject: [PATCH 0291/1943] [#411] Remove unnecessary pointers for sync objects Signed-off-by: Alejandro Lopez --- cmd/frostfs-node/cache.go | 8 +++----- cmd/frostfs-node/config.go | 3 +-- pkg/innerring/processors/alphabet/processor.go | 3 +-- pkg/innerring/processors/frostfs/processor.go | 3 +-- .../processors/netmap/cleanup_table.go | 3 +-- pkg/local_object_storage/engine/engine.go | 17 +++++++---------- .../engine/evacuate_limiter.go | 2 +- pkg/morph/client/client.go | 4 ++-- pkg/morph/client/constructor.go | 15 ++++++--------- pkg/morph/subscriber/subscriber.go | 3 +-- pkg/morph/timer/block.go | 4 +--- pkg/services/notificator/nats/service.go | 3 +-- pkg/services/object/get/v2/get_forwarder.go | 4 ++-- .../object/get/v2/get_range_forwarder.go | 2 +- pkg/services/object/get/v2/head_forwarder.go | 2 +- pkg/services/object/get/v2/util.go | 13 ++++--------- .../object/search/v2/request_forwarder.go | 2 +- pkg/services/object/search/v2/util.go | 6 ++---- .../object_manager/placement/traverser.go | 3 +-- .../session/storage/temporary/storage.go | 3 +-- 20 files changed, 39 insertions(+), 64 deletions(-) diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index dfbaf3525..bae38b299 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -25,19 +25,18 @@ type valueWithTime[V any] struct { } type locker struct { - mtx *sync.Mutex + mtx sync.Mutex waiters int // not protected by mtx, must used outer mutex to update concurrently } type keyLocker[K comparable] struct { lockers map[K]*locker - lockersMtx *sync.Mutex + lockersMtx sync.Mutex } func newKeyLocker[K comparable]() *keyLocker[K] { return &keyLocker[K]{ - lockers: make(map[K]*locker), - lockersMtx: &sync.Mutex{}, + lockers: make(map[K]*locker), } } @@ -53,7 +52,6 @@ func (l *keyLocker[K]) LockKey(key K) { } locker := &locker{ - mtx: &sync.Mutex{}, waiters: 1, } locker.mtx.Lock() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 30d457967..00ade7ce2 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -327,7 +327,7 @@ type internals struct { log *logger.Logger - wg *sync.WaitGroup + wg sync.WaitGroup workers []worker closers []closer @@ -589,7 +589,6 @@ func initInternals(appCfg *config.Config, log *logger.Logger) internals { appCfg: appCfg, internalErr: make(chan error), log: log, - wg: new(sync.WaitGroup), apiVersion: version.Current(), healthStatus: &healthStatus, } diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index 972f84281..04dde80f7 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -49,7 +49,7 @@ type ( Processor struct { parsedWallets []util.Uint160 // protects parsedWallets from concurrent change - pwLock *sync.RWMutex + pwLock sync.RWMutex log *logger.Logger metrics metrics.Register pool *ants.Pool @@ -99,7 +99,6 @@ func New(p *Params) (*Processor, error) { return &Processor{ parsedWallets: p.ParsedWallets, - pwLock: new(sync.RWMutex), log: p.Log, metrics: metricsRegister, pool: pool, diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index e6ee78837..20f44adcd 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -68,7 +68,7 @@ type ( epochState EpochState alphabetState AlphabetState converter PrecisionConverter - mintEmitLock *sync.Mutex + mintEmitLock sync.Mutex mintEmitCache *lru.Cache[string, uint64] mintEmitThreshold uint64 mintEmitValue fixedn.Fixed8 @@ -148,7 +148,6 @@ func New(p *Params) (*Processor, error) { epochState: p.EpochState, alphabetState: p.AlphabetState, converter: p.Converter, - mintEmitLock: new(sync.Mutex), mintEmitCache: lruCache, mintEmitThreshold: p.MintEmitThreshold, mintEmitValue: p.MintEmitValue, diff --git a/pkg/innerring/processors/netmap/cleanup_table.go b/pkg/innerring/processors/netmap/cleanup_table.go index e4024e95f..80117247d 100644 --- a/pkg/innerring/processors/netmap/cleanup_table.go +++ b/pkg/innerring/processors/netmap/cleanup_table.go @@ -9,7 +9,7 @@ import ( type ( cleanupTable struct { - *sync.RWMutex + sync.RWMutex enabled bool threshold uint64 lastAccess map[string]epochStampWithNodeInfo @@ -29,7 +29,6 @@ type ( func newCleanupTable(enabled bool, threshold uint64) cleanupTable { return cleanupTable{ - RWMutex: new(sync.RWMutex), enabled: enabled, threshold: threshold, lastAccess: make(map[string]epochStampWithNodeInfo), diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 19e7b5237..21e863005 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -20,7 +20,7 @@ type StorageEngine struct { removeDuplicatesInProgress atomic.Bool - mtx *sync.RWMutex + mtx sync.RWMutex shards map[string]hashedShard @@ -225,15 +225,12 @@ func New(opts ...Option) *StorageEngine { } return &StorageEngine{ - cfg: c, - mtx: new(sync.RWMutex), - shards: make(map[string]hashedShard), - shardPools: make(map[string]util.WorkerPool), - closeCh: make(chan struct{}), - setModeCh: make(chan setModeRequest), - evacuateLimiter: &evacuationLimiter{ - guard: &sync.RWMutex{}, - }, + cfg: c, + shards: make(map[string]hashedShard), + shardPools: make(map[string]util.WorkerPool), + closeCh: make(chan struct{}), + setModeCh: make(chan setModeRequest), + evacuateLimiter: &evacuationLimiter{}, } } diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go index 425fdc775..62795fa1a 100644 --- a/pkg/local_object_storage/engine/evacuate_limiter.go +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -113,7 +113,7 @@ type evacuationLimiter struct { eg *errgroup.Group cancel context.CancelFunc - guard *sync.RWMutex + guard sync.RWMutex } func (l *evacuationLimiter) TryStart(ctx context.Context, shardIDs []string, result *EvacuateShardRes) (*errgroup.Group, context.Context, error) { diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 6b6e1284b..de29d0c03 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -66,7 +66,7 @@ type Client struct { // switchLock protects endpoints, inactive, and subscription-related fields. // It is taken exclusively during endpoint switch and locked in shared mode // on every normal call. - switchLock *sync.RWMutex + switchLock sync.RWMutex // channel for internal stop closeChan chan struct{} @@ -83,7 +83,7 @@ type Client struct { } type cache struct { - m *sync.RWMutex + m sync.RWMutex nnsHash *util.Uint160 gKey *keys.PublicKey diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 1f2a1eb8d..ff37da728 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -105,13 +104,12 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } cli := &Client{ - cache: newClientCache(), - logger: cfg.logger, - acc: acc, - accAddr: accAddr, - cfg: *cfg, - switchLock: &sync.RWMutex{}, - closeChan: make(chan struct{}), + cache: newClientCache(), + logger: cfg.logger, + acc: acc, + accAddr: accAddr, + cfg: *cfg, + closeChan: make(chan struct{}), } cli.endpoints.init(cfg.endpoints) @@ -198,7 +196,6 @@ func newActor(ws *rpcclient.WSClient, acc *wallet.Account, cfg cfg) (*actor.Acto func newClientCache() cache { c, _ := lru.New[util.Uint256, uint32](100) // returns error only if size is negative return cache{ - m: &sync.RWMutex{}, txHeights: c, } } diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 383d58407..608872dec 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -42,7 +42,7 @@ type ( } subscriber struct { - *sync.RWMutex + sync.RWMutex log *logger.Logger client *client.Client @@ -163,7 +163,6 @@ func New(ctx context.Context, p *Params) (Subscriber, error) { } sub := &subscriber{ - RWMutex: new(sync.RWMutex), log: p.Log, client: p.Client, notifyChan: make(chan *state.ContainedNotificationEvent), diff --git a/pkg/morph/timer/block.go b/pkg/morph/timer/block.go index 31c28e2ff..be20d3571 100644 --- a/pkg/morph/timer/block.go +++ b/pkg/morph/timer/block.go @@ -17,7 +17,7 @@ type BlockTickHandler func() type BlockTimer struct { rolledBack bool - mtx *sync.Mutex + mtx sync.Mutex dur BlockMeter @@ -64,7 +64,6 @@ func StaticBlockMeter(d uint32) BlockMeter { // Reset should be called before timer ticking. func NewBlockTimer(dur BlockMeter, h BlockTickHandler) *BlockTimer { return &BlockTimer{ - mtx: new(sync.Mutex), dur: dur, mul: 1, div: 1, @@ -80,7 +79,6 @@ func NewBlockTimer(dur BlockMeter, h BlockTickHandler) *BlockTimer { // Do not use delta handlers with pulse in this timer. func NewOneTickTimer(dur BlockMeter, h BlockTickHandler) *BlockTimer { return &BlockTimer{ - mtx: new(sync.Mutex), dur: dur, mul: 1, div: 1, diff --git a/pkg/services/notificator/nats/service.go b/pkg/services/notificator/nats/service.go index 6a7e80a53..19f0cbde1 100644 --- a/pkg/services/notificator/nats/service.go +++ b/pkg/services/notificator/nats/service.go @@ -25,7 +25,7 @@ type Writer struct { js nats.JetStreamContext nc *nats.Conn - m *sync.RWMutex + m sync.RWMutex createdStreams map[string]struct{} opts } @@ -84,7 +84,6 @@ func (n *Writer) Notify(topic string, address oid.Address) error { // New creates new Writer. func New(oo ...Option) *Writer { w := &Writer{ - m: &sync.RWMutex{}, createdStreams: make(map[string]struct{}), opts: opts{ log: &logger.Logger{Logger: zap.L()}, diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index d11f94b26..bc27f69a2 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -23,8 +23,8 @@ import ( ) type getRequestForwarder struct { - OnceResign *sync.Once - OnceHeaderSending *sync.Once + OnceResign sync.Once + OnceHeaderSending sync.Once GlobalProgress int Key *ecdsa.PrivateKey Request *objectV2.GetRequest diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 6c744b23a..1137ba33c 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -23,7 +23,7 @@ import ( ) type getRangeRequestForwarder struct { - OnceResign *sync.Once + OnceResign sync.Once GlobalProgress int Key *ecdsa.PrivateKey Request *objectV2.GetRangeRequest diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index 0c91ec5d8..1b0374aea 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -25,7 +25,7 @@ import ( type headRequestForwarder struct { Request *objectV2.HeadRequest Response *objectV2.HeadResponse - OnceResign *sync.Once + OnceResign sync.Once ObjectAddr oid.Address Key *ecdsa.PrivateKey } diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 91e7a96a2..519d9afa0 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -5,7 +5,6 @@ import ( "crypto/sha256" "errors" "hash" - "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" @@ -59,12 +58,10 @@ func (s *Service) toPrm(req *objectV2.GetRequest, stream objectSvc.GetObjectStre } forwarder := &getRequestForwarder{ - OnceResign: &sync.Once{}, - OnceHeaderSending: &sync.Once{}, - GlobalProgress: 0, - Key: key, - Request: req, - Stream: streamWrapper, + GlobalProgress: 0, + Key: key, + Request: req, + Stream: streamWrapper, } p.SetRequestForwarder(groupAddressRequestForwarder(forwarder.forwardRequestToNode)) @@ -115,7 +112,6 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get } forwarder := &getRangeRequestForwarder{ - OnceResign: &sync.Once{}, GlobalProgress: 0, Key: key, Request: req, @@ -254,7 +250,6 @@ func (s *Service) toHeadPrm(req *objectV2.HeadRequest, resp *objectV2.HeadRespon forwarder := &headRequestForwarder{ Request: req, Response: resp, - OnceResign: &sync.Once{}, ObjectAddr: objAddr, Key: key, } diff --git a/pkg/services/object/search/v2/request_forwarder.go b/pkg/services/object/search/v2/request_forwarder.go index d8719986f..5a2e9b936 100644 --- a/pkg/services/object/search/v2/request_forwarder.go +++ b/pkg/services/object/search/v2/request_forwarder.go @@ -20,7 +20,7 @@ import ( ) type requestForwarder struct { - OnceResign *sync.Once + OnceResign sync.Once Request *objectV2.SearchRequest Key *ecdsa.PrivateKey } diff --git a/pkg/services/object/search/v2/util.go b/pkg/services/object/search/v2/util.go index 12158a820..cfccaede6 100644 --- a/pkg/services/object/search/v2/util.go +++ b/pkg/services/object/search/v2/util.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" @@ -51,9 +50,8 @@ func (s *Service) toPrm(req *objectV2.SearchRequest, stream objectSvc.SearchStre } forwarder := &requestForwarder{ - OnceResign: &sync.Once{}, - Request: req, - Key: key, + Request: req, + Key: key, } p.SetRequestForwarder(groupAddressRequestForwarder(forwarder.forwardRequest)) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index e46240a86..7b9b26a0b 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -29,7 +29,7 @@ type Option func(*cfg) // Traverser represents utility for controlling // traversal of object placement vectors. type Traverser struct { - mtx *sync.RWMutex + mtx sync.RWMutex vectors [][]netmap.NodeInfo @@ -107,7 +107,6 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } return &Traverser{ - mtx: new(sync.RWMutex), rem: rem, vectors: ns, }, nil diff --git a/pkg/services/session/storage/temporary/storage.go b/pkg/services/session/storage/temporary/storage.go index 370499e06..ee93dee71 100644 --- a/pkg/services/session/storage/temporary/storage.go +++ b/pkg/services/session/storage/temporary/storage.go @@ -18,7 +18,7 @@ type key struct { // expiring (removing) session tokens. // Must be created only via calling NewTokenStore. type TokenStore struct { - mtx *sync.RWMutex + mtx sync.RWMutex tokens map[key]*storage.PrivateToken } @@ -28,7 +28,6 @@ type TokenStore struct { // The elements of the instance are stored in the map. func NewTokenStore() *TokenStore { return &TokenStore{ - mtx: new(sync.RWMutex), tokens: make(map[key]*storage.PrivateToken), } } From 3220c4df9fb513d5f6cd3b15e056cc9ea5e8f49e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 29 May 2023 17:32:13 +0300 Subject: [PATCH 0292/1943] [#376] metrics: Add GC metrics Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/metrics.go | 22 +++++ pkg/local_object_storage/engine/shards.go | 21 +++-- pkg/local_object_storage/shard/delete.go | 12 ++- pkg/local_object_storage/shard/gc.go | 95 ++++++++++++++++--- pkg/local_object_storage/shard/shard.go | 7 ++ pkg/metrics/gc.go | 104 +++++++++++++++++++++ pkg/metrics/node.go | 9 ++ 7 files changed, 244 insertions(+), 26 deletions(-) create mode 100644 pkg/metrics/gc.go diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 7a11888c5..1be888eae 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -28,6 +28,7 @@ type MetricRegister interface { AddToPayloadCounter(shardID string, size int64) WriteCache() metrics.WriteCacheMetrics + GC() metrics.GCMetrics } func elapsed(addFunc func(d time.Duration)) func() { @@ -37,3 +38,24 @@ func elapsed(addFunc func(d time.Duration)) func() { addFunc(time.Since(t)) } } + +type gcMetrics struct { + storage metrics.GCMetrics + shardID string +} + +func (m *gcMetrics) AddRunDuration(d time.Duration, success bool) { + m.storage.AddRunDuration(m.shardID, d, success) +} + +func (m *gcMetrics) AddDeletedCount(deleted, failed uint64) { + m.storage.AddDeletedCount(m.shardID, deleted, failed) +} + +func (m *gcMetrics) AddExpiredObjectCollectionDuration(d time.Duration, success bool, objectType string) { + m.storage.AddExpiredObjectCollectionDuration(m.shardID, d, success, objectType) +} + +func (m *gcMetrics) AddInhumedObjectCount(count uint64, objectType string) { + m.storage.AddInhumedObjectCount(m.shardID, count, objectType) +} diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 6c49c8312..07d22d3fe 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -98,13 +98,20 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { id: id.String(), mw: e.metrics, }, - )) - opts = append(opts, shard.WithExtraWriteCacheOptions(writecache.WithMetrics( - &writeCacheMetrics{ - shardID: id.String(), - metrics: e.metrics.WriteCache(), - }, - ))) + ), + shard.WithExtraWriteCacheOptions(writecache.WithMetrics( + &writeCacheMetrics{ + shardID: id.String(), + metrics: e.metrics.WriteCache(), + }), + ), + shard.WithGCMetrics( + &gcMetrics{ + storage: e.metrics.GC(), + shardID: id.String(), + }, + ), + ) } e.mtx.RUnlock() diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 4eb7ad6af..4843314de 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -21,7 +21,9 @@ type DeletePrm struct { } // DeleteRes groups the resulting values of Delete operation. -type DeleteRes struct{} +type DeleteRes struct { + deleted uint64 +} // SetAddresses is a Delete option to set the addresses of the objects to delete. // @@ -53,10 +55,11 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { return DeleteRes{}, ErrDegradedMode } + result := DeleteRes{} for _, addr := range prm.addr { select { case <-ctx.Done(): - return DeleteRes{}, ctx.Err() + return result, ctx.Err() default: } @@ -65,11 +68,12 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { s.deleteFromBlobstorSafe(ctx, addr) if err := s.deleteFromMetabase(ctx, addr); err != nil { - return DeleteRes{}, err // stop on metabase error ? + return result, err // stop on metabase error ? } + result.deleted++ } - return DeleteRes{}, nil + return result, nil } func (s *Shard) deleteObjectFromWriteCacheSafe(ctx context.Context, addr oid.Address) { diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 34a48d44a..2580173c2 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -67,6 +67,32 @@ type eventHandlers struct { handlers []eventHandler } +type gcRunResult struct { + success bool + deleted uint64 + failedToDelete uint64 +} + +const ( + objectTypeLock = "lock" + objectTypeTombstone = "tombstone" + objectTypeRegular = "regular" +) + +type GCMectrics interface { + AddRunDuration(d time.Duration, success bool) + AddDeletedCount(deleted, failed uint64) + AddExpiredObjectCollectionDuration(d time.Duration, success bool, objectType string) + AddInhumedObjectCount(count uint64, objectType string) +} + +type noopGCMetrics struct{} + +func (m *noopGCMetrics) AddRunDuration(time.Duration, bool) {} +func (m *noopGCMetrics) AddDeletedCount(uint64, uint64) {} +func (m *noopGCMetrics) AddExpiredObjectCollectionDuration(time.Duration, bool, string) {} +func (m *noopGCMetrics) AddInhumedObjectCount(uint64, string) {} + type gc struct { *gcCfg @@ -76,7 +102,7 @@ type gc struct { workerPool util.WorkerPool - remover func(context.Context) + remover func(context.Context) gcRunResult eventChan chan Event mEventHandler map[eventType]*eventHandlers @@ -91,6 +117,8 @@ type gcCfg struct { expiredCollectorWorkersCount int expiredCollectorBatchSize int + + metrics GCMectrics } func defaultGCCfg() gcCfg { @@ -100,6 +128,7 @@ func defaultGCCfg() gcCfg { workerPoolInit: func(int) util.WorkerPool { return nil }, + metrics: &noopGCMetrics{}, } } @@ -178,8 +207,13 @@ func (gc *gc) tickRemover(ctx context.Context) { gc.log.Debug(logs.ShardGCIsStopped) return case <-timer.C: - gc.remover(ctx) + startedAt := time.Now() + + result := gc.remover(ctx) timer.Reset(gc.removerInterval) + + gc.metrics.AddRunDuration(time.Since(startedAt), result.success) + gc.metrics.AddDeletedCount(result.deleted, result.failedToDelete) } } } @@ -196,7 +230,7 @@ func (gc *gc) stop() { // iterates over metabase and deletes objects // with GC-marked graves. // Does nothing if shard is in "read-only" mode. -func (s *Shard) removeGarbage(pctx context.Context) { +func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { ctx, cancel := context.WithCancel(pctx) defer cancel() @@ -244,6 +278,7 @@ func (s *Shard) removeGarbage(pctx context.Context) { return } else if len(buf) == 0 { + result.success = true return } @@ -251,14 +286,20 @@ func (s *Shard) removeGarbage(pctx context.Context) { deletePrm.SetAddresses(buf...) // delete accumulated objects - _, err = s.delete(ctx, deletePrm) + res, err := s.delete(ctx, deletePrm) + + result.deleted = res.deleted + result.failedToDelete = uint64(len(buf)) - res.deleted + result.success = true + if err != nil { s.log.Warn(logs.ShardCouldNotDeleteTheObjects, zap.String("error", err.Error()), ) - - return + result.success = false } + + return } func (s *Shard) getExpiredObjectsParameters() (workersCount, batchSize int) { @@ -276,6 +317,13 @@ func (s *Shard) getExpiredObjectsParameters() (workersCount, batchSize int) { } func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { + var err error + startedAt := time.Now() + + defer func() { + s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeRegular) + }() + s.log.Debug(logs.ShardGCCollectingExpiredObjectsStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) defer s.log.Debug(logs.ShardGCCollectingExpiredObjectsCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) @@ -286,7 +334,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { errGroup.Go(func() error { batch := make([]oid.Address, 0, batchSize) - err := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { + expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { if o.Type() != object.TypeTombstone && o.Type() != object.TypeLock { batch = append(batch, o.Address()) @@ -300,8 +348,8 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { } } }) - if err != nil { - return err + if expErr != nil { + return expErr } if len(batch) > 0 { @@ -315,7 +363,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { return nil }) - if err := errGroup.Wait(); err != nil { + if err = errGroup.Wait(); err != nil { s.log.Warn(logs.ShardIteratorOverExpiredObjectsFailed, zap.String("error", err.Error())) } } @@ -355,6 +403,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } + s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeRegular) s.decObjectCounterBy(logical, res.AvailableInhumed()) i := 0 @@ -380,6 +429,13 @@ func (s *Shard) getExpiredWithLinked(source []oid.Address) ([]oid.Address, error } func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { + var err error + startedAt := time.Now() + + defer func() { + s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeTombstone) + }() + epoch := e.(newEpoch).epoch log := s.log.With(zap.Uint64("epoch", epoch)) @@ -413,7 +469,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { return } - err := s.metaBase.IterateOverGraveyard(iterPrm) + err = s.metaBase.IterateOverGraveyard(iterPrm) if err != nil { log.Error(logs.ShardIteratorOverGraveyardFailed, zap.Error(err)) s.m.RUnlock() @@ -444,6 +500,13 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { } func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { + var err error + startedAt := time.Now() + + defer func() { + s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeLock) + }() + s.log.Debug(logs.ShardGCCollectingExpiredLocksStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) defer s.log.Debug(logs.ShardGCCollectingExpiredLocksCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) @@ -455,7 +518,7 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { errGroup.Go(func() error { batch := make([]oid.Address, 0, batchSize) - err := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { + expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { if o.Type() == object.TypeLock { batch = append(batch, o.Address()) @@ -469,8 +532,8 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { } } }) - if err != nil { - return err + if expErr != nil { + return expErr } if len(batch) > 0 { @@ -484,7 +547,7 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { return nil }) - if err := errGroup.Wait(); err != nil { + if err = errGroup.Wait(); err != nil { s.log.Warn(logs.ShardIteratorOverExpiredLocksFailed, zap.String("error", err.Error())) } } @@ -553,6 +616,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston return } + s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeTombstone) s.decObjectCounterBy(logical, res.AvailableInhumed()) i := 0 @@ -598,6 +662,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] return } + s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeLock) s.decObjectCounterBy(logical, res.AvailableInhumed()) i := 0 diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 65cc1ef55..2123bca1f 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -309,6 +309,13 @@ func WithMetricsWriter(v MetricsWriter) Option { } } +// WithGCMetrics returns option to specify storage of the GC metrics. +func WithGCMetrics(v GCMectrics) Option { + return func(c *cfg) { + c.gcCfg.metrics = v + } +} + // WithReportErrorFunc returns option to specify callback for handling storage-related errors // in the background workers. func WithReportErrorFunc(f func(selfID string, message string, err error)) Option { diff --git a/pkg/metrics/gc.go b/pkg/metrics/gc.go new file mode 100644 index 000000000..c4d5ecb50 --- /dev/null +++ b/pkg/metrics/gc.go @@ -0,0 +1,104 @@ +package metrics + +import ( + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + gcSubsystem = "garbage_collector" + gcShardID = "shard_id" + gcSuccess = "success" + gcStatus = "status" + gcDeleted = "deleted" + gcFailed = "failed_to_delete" + gcObjectType = "object_type" +) + +type GCMetrics interface { + AddRunDuration(shardID string, d time.Duration, success bool) + AddDeletedCount(shardID string, deleted, failed uint64) + AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) + AddInhumedObjectCount(shardID string, count uint64, objectType string) +} + +type gcMetrics struct { + runDuration metric[*prometheus.CounterVec] + deletedCounter metric[*prometheus.CounterVec] + expCollectDuration metric[*prometheus.CounterVec] + inhumedCounter metric[*prometheus.CounterVec] +} + +func (m *gcMetrics) register() { + mustRegister(m.runDuration) + mustRegister(m.deletedCounter) + mustRegister(m.expCollectDuration) + mustRegister(m.inhumedCounter) +} + +func newGCMetrics() *gcMetrics { + return &gcMetrics{ + runDuration: newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "delete_duration_seconds", + Help: "The total time of GC runs to delete objects from disk", + }, []string{gcShardID, gcSuccess}), + deletedCounter: newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "deleted_objects_count", + Help: "Total count of objects GC deleted or failed to delete from disk", + }, []string{gcShardID, gcStatus}), + expCollectDuration: newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "marking_duration_seconds", + Help: "The total time of GC runs to mark expired objects as removed", + }, []string{gcShardID, gcSuccess, gcObjectType}), + inhumedCounter: newCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: gcSubsystem, + Name: "marked_for_removal_objects_count", + Help: "Total count of expired objects GC marked to remove", + }, []string{gcShardID, gcObjectType}), + } +} + +func (m *gcMetrics) AddRunDuration(shardID string, d time.Duration, success bool) { + m.runDuration.value.With(prometheus.Labels{ + gcShardID: shardID, + gcSuccess: fmt.Sprintf("%v", success), + }).Add(d.Seconds()) +} + +func (m *gcMetrics) AddDeletedCount(shardID string, deleted, failed uint64) { + m.deletedCounter.value.With( + prometheus.Labels{ + gcShardID: shardID, + gcStatus: gcDeleted, + }).Add(float64(deleted)) + m.deletedCounter.value.With( + prometheus.Labels{ + gcShardID: shardID, + gcStatus: gcFailed, + }).Add(float64(failed)) +} + +func (m *gcMetrics) AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) { + m.expCollectDuration.value.With(prometheus.Labels{ + gcShardID: shardID, + gcSuccess: fmt.Sprintf("%v", success), + gcObjectType: objectType, + }).Add(d.Seconds()) +} + +func (m *gcMetrics) AddInhumedObjectCount(shardID string, count uint64, objectType string) { + m.inhumedCounter.value.With( + prometheus.Labels{ + gcShardID: shardID, + gcObjectType: objectType, + }).Add(float64(count)) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index cca82b5fe..526e460c5 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -16,6 +16,7 @@ type NodeMetrics struct { writeCacheMetrics *writeCacheMetrics treeService *treeServiceMetrics epoch metric[prometheus.Gauge] + gc *gcMetrics } func NewNodeMetrics() *NodeMetrics { @@ -45,6 +46,9 @@ func NewNodeMetrics() *NodeMetrics { writeCacheMetrics := newWriteCacheMetrics() writeCacheMetrics.register() + gc := newGCMetrics() + gc.register() + return &NodeMetrics{ objectServiceMetrics: objectService, engineMetrics: engine, @@ -53,6 +57,7 @@ func NewNodeMetrics() *NodeMetrics { treeService: treeService, epoch: epoch, writeCacheMetrics: writeCacheMetrics, + gc: gc, } } @@ -72,3 +77,7 @@ func (m *NodeMetrics) WriteCache() WriteCacheMetrics { func (m *NodeMetrics) TreeService() tree.MetricsRegister { return m.treeService } + +func (m *NodeMetrics) GC() GCMetrics { + return m.gc +} From dbf41391b55992decff4e55b9c3480874af3cde3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 13:24:30 +0300 Subject: [PATCH 0293/1943] [#401] engine: Extend evacuation logs Add operation-tag to logger. Log evacuation results. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 29 ++++++++++++++------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index d698fca74..b0a7e77b4 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -21,7 +21,11 @@ import ( "go.uber.org/zap" ) -var ErrMustBeReadOnly = logicerr.New("shard must be in read-only mode") +var ( + ErrMustBeReadOnly = logicerr.New("shard must be in read-only mode") + + evacuationOperationLogField = zap.String("operation", "evacuation") +) // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { @@ -196,22 +200,28 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p e.evacuateLimiter.Complete(err) }() - e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs)) + e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField) err = e.getTotalObjectsCount(ctx, shardsToEvacuate, res) if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err)) + e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField) return err } for _, shardID := range shardIDs { if err = e.evacuateShard(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { - e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs)) + e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField) return err } } - e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, zap.Strings("shard_ids", shardIDs)) + e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, + zap.Strings("shard_ids", shardIDs), + evacuationOperationLogField, + zap.Uint64("total", res.Total()), + zap.Uint64("evacuated", res.Evacuated()), + zap.Uint64("failed", res.Failed()), + ) return nil } @@ -256,7 +266,7 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { break } - e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err)) + e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField) return err } @@ -332,7 +342,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to res.failed.Add(1) continue } - e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) + e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) return err } @@ -353,7 +363,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to err = prm.handler(ctx, addr, getRes.Object()) if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err)) + e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) return err } res.evacuated.Add(1) @@ -381,7 +391,8 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), - zap.Stringer("addr", addr)) + zap.Stringer("addr", addr), + evacuationOperationLogField) } return true, nil } From 4476a1dbaf20bc244ea81e22865ae6898c83c6c7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 31 May 2023 15:37:18 +0300 Subject: [PATCH 0294/1943] [#413] morph/client: Fix govet warnings Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index de29d0c03..6a7a5b51a 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -90,7 +90,7 @@ type cache struct { txHeights *lru.Cache[util.Uint256, uint32] } -func (c cache) nns() *util.Uint160 { +func (c *cache) nns() *util.Uint160 { c.m.RLock() defer c.m.RUnlock() @@ -104,7 +104,7 @@ func (c *cache) setNNSHash(nnsHash util.Uint160) { c.nnsHash = &nnsHash } -func (c cache) groupKey() *keys.PublicKey { +func (c *cache) groupKey() *keys.PublicKey { c.m.RLock() defer c.m.RUnlock() From f7c4c0745318837e5104ed9a06b3a5572d4223eb Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 1 Jun 2023 10:07:48 +0300 Subject: [PATCH 0295/1943] [#xx] Create innerring metrics before metrics usage Signed-off-by: Alejandro Lopez --- pkg/innerring/innerring.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 3a690bbbe..cbcb4699a 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -326,7 +326,10 @@ func (s *Server) registerStarter(f func() error) { // New creates instance of inner ring sever structure. func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error) (*Server, error) { var err error - server := &Server{log: log} + server := &Server{ + log: log, + metrics: metrics.NewInnerRingMetrics(), + } server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED) @@ -393,8 +396,6 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - server.metrics = metrics.NewInnerRingMetrics() - return server, nil } From 7e9a1f394a62b4e274ea5b93cd67ce3a89c43124 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 12:19:59 +0300 Subject: [PATCH 0296/1943] [#412] node: Update deps Signed-off-by: Dmitrii Stepanov --- go.mod | 24 ++++++++++++++---------- go.sum | 50 ++++++++++++++++++++++++++++++++------------------ 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 2c95bd41b..6c49c02b4 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230519114017-0c67b8fefa41 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230519144724-f5b23eb22569 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230530135122-10482ffbed3b git.frostfs.info/TrueCloudLab/hrw v1.2.0 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 @@ -23,15 +24,14 @@ require ( github.com/panjf2000/ants/v2 v2.7.4 github.com/paulmach/orb v0.9.2 github.com/prometheus/client_golang v1.15.1 - github.com/prometheus/client_model v0.4.0 github.com/spf13/cast v1.5.1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.3 go.etcd.io/bbolt v1.3.7 - go.opentelemetry.io/otel v1.15.1 - go.opentelemetry.io/otel/trace v1.15.1 + go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc golang.org/x/sync v0.2.0 @@ -58,6 +58,8 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -83,6 +85,7 @@ require ( github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.43.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect @@ -95,11 +98,12 @@ require ( github.com/twmb/murmur3 v1.1.7 // indirect github.com/urfave/cli v1.22.13 // indirect go.mongodb.org/mongo-driver v1.11.6 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.15.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.15.1 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect - go.opentelemetry.io/otel/sdk v1.15.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/sdk v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index d728851b9..b9b3fde41 100644 --- a/go.sum +++ b/go.sum @@ -36,14 +36,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230519114017-0c67b8fefa41 h1:xtGsOUX8Rz0hwWIFa148URysWuD4nRHspPNbYAUc1tg= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230519114017-0c67b8fefa41/go.mod h1:6wEpMfSwD5xNtQYYVHWWTHwpYuvyumyntZEzILBIXUo= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac h1:a6/Zc5BejflmguShwbllgJdEehnM9gshkLrLbKQHCU0= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230519144724-f5b23eb22569 h1:JDySbbi0MyU1fSuew0jNyWwDhrQREetHWJiSzKzXXXI= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230519144724-f5b23eb22569/go.mod h1:0n2gQYkqCgiXkDzk0RLUUxoF2xX79Ke85bAiGV8cBl8= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230530135122-10482ffbed3b h1:zxrMDBKALGeEI0SuxNxRlGW/JpZb/2RWywU78L5lLis= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230530135122-10482ffbed3b/go.mod h1:0n2gQYkqCgiXkDzk0RLUUxoF2xX79Ke85bAiGV8cBl8= git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= @@ -138,6 +140,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= @@ -253,6 +256,10 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 h1:mdLirNAJBxnGgyB6pjZLcs6ue/6eZGBui6gXspfq4ks= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0/go.mod h1:kdXbOySqcQeTxiqglW7aahTmWZy3Pgi6SYL36yvKeyA= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 h1:o95KDiV/b1xdkumY5YbLR0/n2+wBxUpgf3HgfKgTyLI= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3/go.mod h1:hTxjzRcX49ogbTGVJ1sM5mz5s+SSgiGIyL3jjPxl32E= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE= @@ -545,20 +552,22 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opentelemetry.io/otel v1.15.1 h1:3Iwq3lfRByPaws0f6bU3naAqOR1n5IeDWd9390kWHa8= -go.opentelemetry.io/otel v1.15.1/go.mod h1:mHHGEHVDLal6YrKMmk9LqC4a3sF5g+fHfrttQIB1NTc= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.1 h1:XYDQtNzdb2T4uM1pku2m76eSMDJgqhJ+6KzkqgQBALc= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.15.1/go.mod h1:uOTV75+LOzV+ODmL8ahRLWkFA3eQcSC2aAsbxIu4duk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.15.1 h1:tyoeaUh8REKay72DVYsSEBYV18+fGONe+YYPaOxgLoE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.15.1/go.mod h1:HUSnrjQQ19KX9ECjpQxufsF+3ioD3zISPMlauTPZu2g= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.15.1 h1:pIfoG5IAZFzp9EUlJzdSkpUwpaUAAnD+Ru1nBLTACIQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.15.1/go.mod h1:poNKBqF5+nR/6ke2oGTDjHfksrsHDOHXAl2g4+9ONsY= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= -go.opentelemetry.io/otel/sdk v1.15.1 h1:5FKR+skgpzvhPQHIEfcwMYjCBr14LWzs3uSqKiQzETI= -go.opentelemetry.io/otel/sdk v1.15.1/go.mod h1:8rVtxQfrbmbHKfqzpQkT5EzZMcbMBwTzNAggbEAM0KA= -go.opentelemetry.io/otel/trace v1.15.1 h1:uXLo6iHJEzDfrNC0L0mNjItIp06SyaBQxu5t3xMlngY= -go.opentelemetry.io/otel/trace v1.15.1/go.mod h1:IWdQG/5N1x7f6YUlmdLeJvH9yxtuJAfc4VW5Agv9r/8= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 h1:+XWJd3jf75RXJq29mxbuXhCXFDG3S3R4vBUeSI2P7tE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0/go.mod h1:hqgzBPTf4yONMFgdZvL/bK42R/iinTyVQtiWihs3SZc= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -668,6 +677,7 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -684,6 +694,7 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= @@ -904,6 +915,7 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -933,11 +945,13 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc/examples v0.0.0-20210424002626-9572fd6faeae/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 74578052f9ac750c3f66b2ff4197d50096464f4c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 12:24:04 +0300 Subject: [PATCH 0297/1943] [#412] node: Replace tracing package Use observability module. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/config/tracing/config.go | 2 +- cmd/frostfs-node/tracing.go | 2 +- pkg/local_object_storage/blobovnicza/delete.go | 2 +- pkg/local_object_storage/blobovnicza/get.go | 2 +- pkg/local_object_storage/blobstor/blobovniczatree/delete.go | 2 +- pkg/local_object_storage/blobstor/blobovniczatree/exists.go | 2 +- pkg/local_object_storage/blobstor/blobovniczatree/get.go | 2 +- pkg/local_object_storage/blobstor/blobovniczatree/get_range.go | 2 +- pkg/local_object_storage/blobstor/blobovniczatree/put.go | 2 +- pkg/local_object_storage/blobstor/delete.go | 2 +- pkg/local_object_storage/blobstor/exists.go | 2 +- pkg/local_object_storage/blobstor/fstree/fstree.go | 2 +- pkg/local_object_storage/blobstor/get.go | 2 +- pkg/local_object_storage/blobstor/get_range.go | 2 +- pkg/local_object_storage/blobstor/put.go | 2 +- pkg/local_object_storage/engine/delete.go | 2 +- pkg/local_object_storage/engine/evacuate.go | 2 +- pkg/local_object_storage/engine/get.go | 2 +- pkg/local_object_storage/engine/head.go | 2 +- pkg/local_object_storage/engine/inhume.go | 2 +- pkg/local_object_storage/engine/lock.go | 2 +- pkg/local_object_storage/engine/put.go | 2 +- pkg/local_object_storage/engine/range.go | 2 +- pkg/local_object_storage/engine/select.go | 2 +- pkg/local_object_storage/engine/tree.go | 2 +- pkg/local_object_storage/engine/writecache.go | 2 +- pkg/local_object_storage/metabase/delete.go | 2 +- pkg/local_object_storage/metabase/exists.go | 2 +- pkg/local_object_storage/metabase/get.go | 2 +- pkg/local_object_storage/metabase/inhume.go | 2 +- pkg/local_object_storage/metabase/lock.go | 2 +- pkg/local_object_storage/metabase/movable.go | 2 +- pkg/local_object_storage/metabase/put.go | 2 +- pkg/local_object_storage/metabase/select.go | 2 +- pkg/local_object_storage/metabase/storage_id.go | 2 +- pkg/local_object_storage/pilorama/boltdb.go | 2 +- pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/shard/count.go | 2 +- pkg/local_object_storage/shard/delete.go | 2 +- pkg/local_object_storage/shard/exists.go | 2 +- pkg/local_object_storage/shard/get.go | 2 +- pkg/local_object_storage/shard/head.go | 2 +- pkg/local_object_storage/shard/inhume.go | 2 +- pkg/local_object_storage/shard/lock.go | 2 +- pkg/local_object_storage/shard/move.go | 2 +- pkg/local_object_storage/shard/put.go | 2 +- pkg/local_object_storage/shard/range.go | 2 +- pkg/local_object_storage/shard/select.go | 2 +- pkg/local_object_storage/shard/tree.go | 2 +- pkg/local_object_storage/shard/writecache.go | 2 +- pkg/local_object_storage/writecache/delete.go | 2 +- pkg/local_object_storage/writecache/flush.go | 2 +- pkg/local_object_storage/writecache/get.go | 2 +- pkg/local_object_storage/writecache/mode.go | 2 +- pkg/local_object_storage/writecache/put.go | 2 +- pkg/services/object/get/local.go | 2 +- pkg/services/object/get/remote.go | 2 +- pkg/services/object/get/v2/get_forwarder.go | 2 +- pkg/services/object/get/v2/get_range_forwarder.go | 2 +- pkg/services/object/get/v2/head_forwarder.go | 2 +- pkg/services/object/internal/client/client.go | 2 +- pkg/services/object/put/v2/streamer.go | 2 +- pkg/services/tree/redirect.go | 2 +- pkg/services/tree/replicator.go | 2 +- 65 files changed, 65 insertions(+), 65 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 00ade7ce2..f8605c21e 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -16,7 +16,6 @@ import ( "time" netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient" contractsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/contracts" @@ -60,6 +59,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" diff --git a/cmd/frostfs-node/config/tracing/config.go b/cmd/frostfs-node/config/tracing/config.go index 76572cc31..e846be158 100644 --- a/cmd/frostfs-node/config/tracing/config.go +++ b/cmd/frostfs-node/config/tracing/config.go @@ -1,9 +1,9 @@ package tracing import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" ) const ( diff --git a/cmd/frostfs-node/tracing.go b/cmd/frostfs-node/tracing.go index d963ba866..08dc049da 100644 --- a/cmd/frostfs-node/tracing.go +++ b/cmd/frostfs-node/tracing.go @@ -4,9 +4,9 @@ import ( "context" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.uber.org/zap" ) diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 29a587cc9..e880815a6 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -3,8 +3,8 @@ package blobovnicza import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index c1cd19e53..ff29358b0 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 5aa9062a6..1e38c6ea1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -5,11 +5,11 @@ import ( "encoding/hex" "path/filepath" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index e7852612b..3324507cf 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -5,10 +5,10 @@ import ( "encoding/hex" "path/filepath" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 8955eb148..3c8e288d4 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -6,11 +6,11 @@ import ( "fmt" "path/filepath" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index fb23a9671..6e9620adf 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -7,11 +7,11 @@ import ( "path/filepath" "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 4e1d6621f..95ed15540 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -5,10 +5,10 @@ import ( "errors" "path/filepath" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index 377214fb8..f1b14481c 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -5,8 +5,8 @@ import ( "encoding/hex" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 3c76764a9..760e7b2a4 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -4,9 +4,9 @@ import ( "context" "encoding/hex" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index b0879f68e..76487813e 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -12,11 +12,11 @@ import ( "strings" "syscall" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/blobstor/get.go b/pkg/local_object_storage/blobstor/get.go index 65bc87c07..eadb990c8 100644 --- a/pkg/local_object_storage/blobstor/get.go +++ b/pkg/local_object_storage/blobstor/get.go @@ -5,9 +5,9 @@ import ( "encoding/hex" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/blobstor/get_range.go b/pkg/local_object_storage/blobstor/get_range.go index ff9e72e97..ca4e41f33 100644 --- a/pkg/local_object_storage/blobstor/get_range.go +++ b/pkg/local_object_storage/blobstor/get_range.go @@ -6,9 +6,9 @@ import ( "errors" "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index 2ae7f0fe6..125b445b6 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -4,10 +4,10 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index f9b9c9a87..2125fad72 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -4,9 +4,9 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index b0a7e77b4..c103ede73 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -6,13 +6,13 @@ import ( "fmt" "sync/atomic" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 683b7bde8..d376198ba 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -4,10 +4,10 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index 130e76c3d..5da97bab5 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -4,10 +4,10 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index b1204ed99..57ac52751 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -4,10 +4,10 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 4562c1a57..61a5a0dc9 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -4,9 +4,9 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 0543f9f15..4ac7f90f9 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -4,13 +4,13 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 3d119ac6f..29ec8b2bc 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -5,10 +5,10 @@ import ( "errors" "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index e1039ea23..48d2be674 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -3,8 +3,8 @@ package engine import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index e7d66094c..6b8f83f31 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 692fa4be5..cd8278272 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -4,11 +4,11 @@ import ( "context" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 5340f5d08..6ddf96593 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -6,9 +6,9 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index cfd37b0d2..74faf1634 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -5,8 +5,8 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index e76b9d4a7..2f94c72d0 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index a6887a33b..e4af1d1b9 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -6,8 +6,8 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 5c3c9720d..2b5fcd775 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -5,8 +5,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/metabase/movable.go b/pkg/local_object_storage/metabase/movable.go index 412c46393..033b9c718 100644 --- a/pkg/local_object_storage/metabase/movable.go +++ b/pkg/local_object_storage/metabase/movable.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index bc6520a05..86830f7da 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -7,10 +7,10 @@ import ( "fmt" gio "io" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 2ced6c4b7..b6b8f5b10 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -8,8 +8,8 @@ import ( "strings" v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index 794879a3f..b53010a1e 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 1ecc89cb5..9b62f0649 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -12,10 +12,10 @@ import ( "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/nspcc-dev/neo-go/pkg/io" "go.etcd.io/bbolt" diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index e8e2bd4d7..84efa1d31 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -5,12 +5,12 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" diff --git a/pkg/local_object_storage/shard/count.go b/pkg/local_object_storage/shard/count.go index b68c2f43e..abed5278e 100644 --- a/pkg/local_object_storage/shard/count.go +++ b/pkg/local_object_storage/shard/count.go @@ -3,7 +3,7 @@ package shard import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 4843314de..2c7e0af27 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -4,11 +4,11 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 66c61fccc..60809da6a 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -3,9 +3,9 @@ package shard import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 5268ac790..589ec53c9 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -4,13 +4,13 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/shard/head.go b/pkg/local_object_storage/shard/head.go index a15cdfdca..a0ec231af 100644 --- a/pkg/local_object_storage/shard/head.go +++ b/pkg/local_object_storage/shard/head.go @@ -3,8 +3,8 @@ package shard import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 12a2900ac..6a2f9311d 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/lock.go b/pkg/local_object_storage/shard/lock.go index cfbd94c5b..52186cbfd 100644 --- a/pkg/local_object_storage/shard/lock.go +++ b/pkg/local_object_storage/shard/lock.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go index 119910623..9832c9c84 100644 --- a/pkg/local_object_storage/shard/move.go +++ b/pkg/local_object_storage/shard/move.go @@ -3,9 +3,9 @@ package shard import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index d7d4ae538..79dc4846e 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -4,11 +4,11 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 06aea2f8a..895dc2fe0 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -4,11 +4,11 @@ import ( "context" "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index 7f776c18a..2d4d473b4 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index d5b3b67bf..ad89fa633 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -4,9 +4,9 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index 245eb4c70..7ce279c54 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index f5a292ed4..796fda623 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -4,9 +4,9 @@ import ( "context" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 767435ebf..da8b3a0fa 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -6,12 +6,12 @@ import ( "errors" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/mr-tron/base58" diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index f8f6de9b0..aac5759ae 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -4,9 +4,9 @@ import ( "context" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 7e9373a42..bdbbec7c9 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -5,10 +5,10 @@ import ( "fmt" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 1e99e4a28..61e81ec30 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -5,9 +5,9 @@ import ( "errors" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index 62dde3281..03ede58cc 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -4,8 +4,8 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index 69bdbf271..e3464f941 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -4,9 +4,9 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index bc27f69a2..580c0b58c 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -8,7 +8,6 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -17,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 1137ba33c..7858e2b53 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -8,7 +8,6 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -17,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index 1b0374aea..fa1506435 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -6,7 +6,6 @@ import ( "sync" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -15,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 6beb67476..8214c784d 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -8,8 +8,8 @@ import ( "fmt" "io" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 65531dc66..9c6de4ca8 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -5,7 +5,6 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -16,6 +15,7 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index 3de71b554..0afd3439a 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -5,8 +5,8 @@ import ( "context" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 7199dc40e..0ca30273e 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -8,9 +8,9 @@ import ( "fmt" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" From c09144ecf155758ec65d92fa8d4506ce0db68ee4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 12:25:32 +0300 Subject: [PATCH 0298/1943] [#412] node: Replace metrics package Use observability module. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/metrics.go | 2 +- cmd/frostfs-node/metrics.go | 2 +- pkg/metrics/desc.go | 109 --------------------------------- pkg/metrics/desc_test.go | 65 -------------------- pkg/metrics/engine.go | 79 ++++++++++-------------- pkg/metrics/gc.go | 34 +++++----- pkg/metrics/innerring.go | 23 +++---- pkg/metrics/node.go | 15 ++--- pkg/metrics/object.go | 100 +++++++++++------------------- pkg/metrics/registry.go | 42 ------------- pkg/metrics/replicator.go | 33 +++++----- pkg/metrics/state.go | 15 +++-- pkg/metrics/treeservice.go | 25 +++----- pkg/metrics/writecache.go | 64 ++++++++----------- scripts/export-metrics/main.go | 13 ++-- 15 files changed, 162 insertions(+), 459 deletions(-) delete mode 100644 pkg/metrics/desc.go delete mode 100644 pkg/metrics/desc_test.go delete mode 100644 pkg/metrics/registry.go diff --git a/cmd/frostfs-ir/metrics.go b/cmd/frostfs-ir/metrics.go index 39b432c74..dd982b780 100644 --- a/cmd/frostfs-ir/metrics.go +++ b/cmd/frostfs-ir/metrics.go @@ -1,7 +1,7 @@ package main import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) func newMetricsComponent() *httpComponent { diff --git a/cmd/frostfs-node/metrics.go b/cmd/frostfs-node/metrics.go index cf621086d..19b4af51f 100644 --- a/cmd/frostfs-node/metrics.go +++ b/cmd/frostfs-node/metrics.go @@ -2,7 +2,7 @@ package main import ( metricsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/metrics" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) func metricsComponent(c *cfg) (*httpComponent, bool) { diff --git a/pkg/metrics/desc.go b/pkg/metrics/desc.go deleted file mode 100644 index 612435b2f..000000000 --- a/pkg/metrics/desc.go +++ /dev/null @@ -1,109 +0,0 @@ -package metrics - -import ( - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" -) - -type metric[T prometheus.Collector] struct { - value T - desc Description -} - -// Descriptions contains metric description suitable for further processing. -// The only reason for it to exist is `prometheus.Desc` disallowing field access directly. -// https://github.com/prometheus/client_golang/pull/326 -// https://github.com/prometheus/client_golang/issues/516 -// https://github.com/prometheus/client_golang/issues/222 -type Description struct { - Name string `json:"name"` - Help string `json:"help"` - Type string `json:"type"` - ConstantLabels prometheus.Labels `json:"constant_labels,omitempty"` - VariableLabels []string `json:"variable_labels,omitempty"` -} - -func newGauge(opts prometheus.GaugeOpts) metric[prometheus.Gauge] { - return metric[prometheus.Gauge]{ - value: prometheus.NewGauge(opts), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_GAUGE.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - }, - } -} - -func newGaugeVec(opts prometheus.GaugeOpts, labelNames []string) metric[*prometheus.GaugeVec] { - return metric[*prometheus.GaugeVec]{ - value: prometheus.NewGaugeVec(opts, labelNames), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_GAUGE.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - VariableLabels: labelNames, - }, - } -} - -func newGaugeFunc(opts prometheus.GaugeOpts, f func() float64) metric[prometheus.GaugeFunc] { - return metric[prometheus.GaugeFunc]{ - value: prometheus.NewGaugeFunc(opts, f), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_GAUGE.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - }, - } -} - -func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] { - return metric[prometheus.Counter]{ - value: prometheus.NewCounter(opts), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_COUNTER.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - }, - } -} - -func newCounterVec(opts prometheus.CounterOpts, labels []string) metric[*prometheus.CounterVec] { - return metric[*prometheus.CounterVec]{ - value: prometheus.NewCounterVec(opts, labels), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_COUNTER.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - VariableLabels: labels, - }, - } -} - -func newHistogramVec(opts prometheus.HistogramOpts, labelNames []string) metric[*prometheus.HistogramVec] { - return metric[*prometheus.HistogramVec]{ - value: prometheus.NewHistogramVec(opts, labelNames), - desc: Description{ - Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), - Type: dto.MetricType_HISTOGRAM.String(), - Help: opts.Help, - ConstantLabels: opts.ConstLabels, - VariableLabels: labelNames, - }, - } -} - -// DescribeAll returns descriptions for all registered metrics. -func DescribeAll() ([]Description, error) { - registeredDescriptionsMtx.Lock() - defer registeredDescriptionsMtx.Unlock() - - ds := make([]Description, len(registeredDescriptions)) - copy(ds, registeredDescriptions) - return ds, nil -} diff --git a/pkg/metrics/desc_test.go b/pkg/metrics/desc_test.go deleted file mode 100644 index 28b5e2132..000000000 --- a/pkg/metrics/desc_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package metrics - -import ( - "strings" - "testing" - - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/require" -) - -func TestDescribeAll(t *testing.T) { - const ( - namespace = "my_ns" - subsystem = "mysub" - ) - mustRegister(newCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "my_counter", - })) - - labels := []string{"label1", "label2"} - mustRegister(newGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "my_gauge", - }, labels)) - - constLabels := prometheus.Labels{ - "const1": "abc", - "const2": "xyz", - } - mustRegister(newCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: subsystem, - Name: "with_const_labels", - ConstLabels: constLabels, - })) - - descriptions, err := DescribeAll() - require.NoError(t, err) - - seen := make(map[string]bool) - for i := range descriptions { - if !strings.HasPrefix(descriptions[i].Name, namespace) { - continue - } - - require.False(t, seen[descriptions[i].Name], "metric %s was seen twice", descriptions[i].Name) - seen[descriptions[i].Name] = true - - switch descriptions[i].Name { - case prometheus.BuildFQName(namespace, subsystem, "my_counter"): - require.True(t, len(descriptions[i].VariableLabels) == 0) - case prometheus.BuildFQName(namespace, subsystem, "my_gauge"): - require.Equal(t, labels, descriptions[i].VariableLabels) - case prometheus.BuildFQName(namespace, subsystem, "with_const_labels"): - require.Equal(t, len(constLabels), len(descriptions[i].ConstantLabels)) - require.Equal(t, constLabels, descriptions[i].ConstantLabels) - default: - require.FailNow(t, "unexpected metric name: %s", descriptions[i].Name) - } - } - require.Equal(t, 3, len(seen), "not all registered metrics were iterated over") -} diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 28fc1e028..4e78f4ac2 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -5,24 +5,25 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) type ( engineMetrics struct { - listContainersDuration metric[prometheus.Counter] - estimateContainerSizeDuration metric[prometheus.Counter] - deleteDuration metric[prometheus.Counter] - existsDuration metric[prometheus.Counter] - getDuration metric[prometheus.Counter] - headDuration metric[prometheus.Counter] - inhumeDuration metric[prometheus.Counter] - putDuration metric[prometheus.Counter] - rangeDuration metric[prometheus.Counter] - searchDuration metric[prometheus.Counter] - listObjectsDuration metric[prometheus.Counter] - containerSize metric[*prometheus.GaugeVec] - payloadSize metric[*prometheus.GaugeVec] + listContainersDuration prometheus.Counter + estimateContainerSizeDuration prometheus.Counter + deleteDuration prometheus.Counter + existsDuration prometheus.Counter + getDuration prometheus.Counter + headDuration prometheus.Counter + inhumeDuration prometheus.Counter + putDuration prometheus.Counter + rangeDuration prometheus.Counter + searchDuration prometheus.Counter + listObjectsDuration prometheus.Counter + containerSize *prometheus.GaugeVec + payloadSize *prometheus.GaugeVec } ) @@ -46,8 +47,8 @@ func newEngineMetrics() engineMetrics { } } -func newEngineCounter(name, help string) metric[prometheus.Counter] { - return newCounter(prometheus.CounterOpts{ +func newEngineCounter(name, help string) prometheus.Counter { + return metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: name, @@ -55,15 +56,15 @@ func newEngineCounter(name, help string) metric[prometheus.Counter] { }) } -func newEngineMethodDurationCounter(method string) metric[prometheus.Counter] { +func newEngineMethodDurationCounter(method string) prometheus.Counter { return newEngineCounter( fmt.Sprintf("%s_duration", method), fmt.Sprintf("Accumulated duration of engine %s operations", strings.ReplaceAll(method, "_", " ")), ) } -func newEngineGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] { - return newGaugeVec(prometheus.GaugeOpts{ +func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { + return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: name, @@ -71,70 +72,54 @@ func newEngineGaugeVector(name, help string, labels []string) metric[*prometheus }, labels) } -func (m engineMetrics) register() { - mustRegister(m.listContainersDuration) - mustRegister(m.estimateContainerSizeDuration) - mustRegister(m.deleteDuration) - mustRegister(m.existsDuration) - mustRegister(m.getDuration) - mustRegister(m.headDuration) - mustRegister(m.inhumeDuration) - mustRegister(m.putDuration) - mustRegister(m.rangeDuration) - mustRegister(m.searchDuration) - mustRegister(m.listObjectsDuration) - mustRegister(m.containerSize) - mustRegister(m.payloadSize) -} - func (m engineMetrics) AddListContainersDuration(d time.Duration) { - m.listObjectsDuration.value.Add(float64(d)) + m.listObjectsDuration.Add(float64(d)) } func (m engineMetrics) AddEstimateContainerSizeDuration(d time.Duration) { - m.estimateContainerSizeDuration.value.Add(float64(d)) + m.estimateContainerSizeDuration.Add(float64(d)) } func (m engineMetrics) AddDeleteDuration(d time.Duration) { - m.deleteDuration.value.Add(float64(d)) + m.deleteDuration.Add(float64(d)) } func (m engineMetrics) AddExistsDuration(d time.Duration) { - m.existsDuration.value.Add(float64(d)) + m.existsDuration.Add(float64(d)) } func (m engineMetrics) AddGetDuration(d time.Duration) { - m.getDuration.value.Add(float64(d)) + m.getDuration.Add(float64(d)) } func (m engineMetrics) AddHeadDuration(d time.Duration) { - m.headDuration.value.Add(float64(d)) + m.headDuration.Add(float64(d)) } func (m engineMetrics) AddInhumeDuration(d time.Duration) { - m.inhumeDuration.value.Add(float64(d)) + m.inhumeDuration.Add(float64(d)) } func (m engineMetrics) AddPutDuration(d time.Duration) { - m.putDuration.value.Add(float64(d)) + m.putDuration.Add(float64(d)) } func (m engineMetrics) AddRangeDuration(d time.Duration) { - m.rangeDuration.value.Add(float64(d)) + m.rangeDuration.Add(float64(d)) } func (m engineMetrics) AddSearchDuration(d time.Duration) { - m.searchDuration.value.Add(float64(d)) + m.searchDuration.Add(float64(d)) } func (m engineMetrics) AddListObjectsDuration(d time.Duration) { - m.listObjectsDuration.value.Add(float64(d)) + m.listObjectsDuration.Add(float64(d)) } func (m engineMetrics) AddToContainerSize(cnrID string, size int64) { - m.containerSize.value.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) + m.containerSize.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) } func (m engineMetrics) AddToPayloadCounter(shardID string, size int64) { - m.payloadSize.value.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) + m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) } diff --git a/pkg/metrics/gc.go b/pkg/metrics/gc.go index c4d5ecb50..2457c0c6b 100644 --- a/pkg/metrics/gc.go +++ b/pkg/metrics/gc.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -25,40 +26,33 @@ type GCMetrics interface { } type gcMetrics struct { - runDuration metric[*prometheus.CounterVec] - deletedCounter metric[*prometheus.CounterVec] - expCollectDuration metric[*prometheus.CounterVec] - inhumedCounter metric[*prometheus.CounterVec] -} - -func (m *gcMetrics) register() { - mustRegister(m.runDuration) - mustRegister(m.deletedCounter) - mustRegister(m.expCollectDuration) - mustRegister(m.inhumedCounter) + runDuration *prometheus.CounterVec + deletedCounter *prometheus.CounterVec + expCollectDuration *prometheus.CounterVec + inhumedCounter *prometheus.CounterVec } func newGCMetrics() *gcMetrics { return &gcMetrics{ - runDuration: newCounterVec(prometheus.CounterOpts{ + runDuration: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "delete_duration_seconds", Help: "The total time of GC runs to delete objects from disk", }, []string{gcShardID, gcSuccess}), - deletedCounter: newCounterVec(prometheus.CounterOpts{ + deletedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "deleted_objects_count", Help: "Total count of objects GC deleted or failed to delete from disk", }, []string{gcShardID, gcStatus}), - expCollectDuration: newCounterVec(prometheus.CounterOpts{ + expCollectDuration: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "marking_duration_seconds", Help: "The total time of GC runs to mark expired objects as removed", }, []string{gcShardID, gcSuccess, gcObjectType}), - inhumedCounter: newCounterVec(prometheus.CounterOpts{ + inhumedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "marked_for_removal_objects_count", @@ -68,19 +62,19 @@ func newGCMetrics() *gcMetrics { } func (m *gcMetrics) AddRunDuration(shardID string, d time.Duration, success bool) { - m.runDuration.value.With(prometheus.Labels{ + m.runDuration.With(prometheus.Labels{ gcShardID: shardID, gcSuccess: fmt.Sprintf("%v", success), }).Add(d.Seconds()) } func (m *gcMetrics) AddDeletedCount(shardID string, deleted, failed uint64) { - m.deletedCounter.value.With( + m.deletedCounter.With( prometheus.Labels{ gcShardID: shardID, gcStatus: gcDeleted, }).Add(float64(deleted)) - m.deletedCounter.value.With( + m.deletedCounter.With( prometheus.Labels{ gcShardID: shardID, gcStatus: gcFailed, @@ -88,7 +82,7 @@ func (m *gcMetrics) AddDeletedCount(shardID string, deleted, failed uint64) { } func (m *gcMetrics) AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) { - m.expCollectDuration.value.With(prometheus.Labels{ + m.expCollectDuration.With(prometheus.Labels{ gcShardID: shardID, gcSuccess: fmt.Sprintf("%v", success), gcObjectType: objectType, @@ -96,7 +90,7 @@ func (m *gcMetrics) AddExpiredObjectCollectionDuration(shardID string, d time.Du } func (m *gcMetrics) AddInhumedObjectCount(shardID string, count uint64, objectType string) { - m.inhumedCounter.value.With( + m.inhumedCounter.With( prometheus.Labels{ gcShardID: shardID, gcObjectType: objectType, diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index bff9184ec..9d8b76bf9 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -4,6 +4,7 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -15,27 +16,27 @@ const ( // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { - epoch metric[prometheus.Gauge] - health metric[prometheus.Gauge] - eventDuration metric[*prometheus.HistogramVec] + epoch prometheus.Gauge + health prometheus.Gauge + eventDuration *prometheus.HistogramVec } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. func NewInnerRingMetrics() *InnerRingServiceMetrics { var ( - epoch = newGauge(prometheus.GaugeOpts{ + epoch = metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }) - health = newGauge(prometheus.GaugeOpts{ + health = metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "health", Help: "Current inner-ring node state.", }) - eventDuration = newHistogramVec(prometheus.HistogramOpts{ + eventDuration = metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "event_duration_seconds", @@ -43,10 +44,6 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { }, []string{innerRingLabelType, innerRingLabelSuccess}) ) - mustRegister(epoch) - mustRegister(health) - mustRegister(eventDuration) - return &InnerRingServiceMetrics{ epoch: epoch, health: health, @@ -56,16 +53,16 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { // SetEpoch updates epoch metrics. func (m InnerRingServiceMetrics) SetEpoch(epoch uint64) { - m.epoch.value.Set(float64(epoch)) + m.epoch.Set(float64(epoch)) } // SetHealth updates health metrics. func (m InnerRingServiceMetrics) SetHealth(s int32) { - m.health.value.Set(float64(s)) + m.health.Set(float64(s)) } func (m InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) { - m.eventDuration.value.With(prometheus.Labels{ + m.eventDuration.With(prometheus.Labels{ innerRingLabelType: typ, innerRingLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 526e460c5..8819ba15b 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -2,6 +2,7 @@ package metrics import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -15,39 +16,31 @@ type NodeMetrics struct { writeCacheMetrics *writeCacheMetrics treeService *treeServiceMetrics - epoch metric[prometheus.Gauge] + epoch prometheus.Gauge gc *gcMetrics } func NewNodeMetrics() *NodeMetrics { objectService := newObjectServiceMetrics() - objectService.register() engine := newEngineMetrics() - engine.register() state := newStateMetrics() - state.register() replicator := newReplicatorMetrics() - replicator.register() treeService := newTreeServiceMetrics() - treeService.register() - epoch := newGauge(prometheus.GaugeOpts{ + epoch := metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }) - mustRegister(epoch) writeCacheMetrics := newWriteCacheMetrics() - writeCacheMetrics.register() gc := newGCMetrics() - gc.register() return &NodeMetrics{ objectServiceMetrics: objectService, @@ -63,7 +56,7 @@ func NewNodeMetrics() *NodeMetrics { // SetEpoch updates epoch metric. func (m *NodeMetrics) SetEpoch(epoch uint64) { - m.epoch.value.Set(float64(epoch)) + m.epoch.Set(float64(epoch)) } // WriteCache returns WriteCache metrics. diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index 5ec575749..87916414d 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -12,8 +13,8 @@ const objectSubsystem = "object" type ( methodCount struct { - success metric[prometheus.Counter] - total metric[prometheus.Counter] + success prometheus.Counter + total prometheus.Counter } objectServiceMetrics struct { @@ -25,19 +26,19 @@ type ( rangeCounter methodCount rangeHashCounter methodCount - getDuration metric[prometheus.Counter] - putDuration metric[prometheus.Counter] - headDuration metric[prometheus.Counter] - searchDuration metric[prometheus.Counter] - deleteDuration metric[prometheus.Counter] - rangeDuration metric[prometheus.Counter] - rangeHashDuration metric[prometheus.Counter] + getDuration prometheus.Counter + putDuration prometheus.Counter + headDuration prometheus.Counter + searchDuration prometheus.Counter + deleteDuration prometheus.Counter + rangeDuration prometheus.Counter + rangeHashDuration prometheus.Counter - putPayload metric[prometheus.Counter] - getPayload metric[prometheus.Counter] + putPayload prometheus.Counter + getPayload prometheus.Counter - shardMetrics metric[*prometheus.GaugeVec] - shardsReadonly metric[*prometheus.GaugeVec] + shardMetrics *prometheus.GaugeVec + shardsReadonly *prometheus.GaugeVec } ) @@ -49,13 +50,13 @@ const ( func newObjectMethodCallCounter(name string) methodCount { return methodCount{ - success: newCounter(prometheus.CounterOpts{ + success: metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_count_success", name), Help: fmt.Sprintf("The number of successful %s requests processed", name), }), - total: newCounter(prometheus.CounterOpts{ + total: metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_count", name), @@ -64,15 +65,10 @@ func newObjectMethodCallCounter(name string) methodCount { } } -func (m methodCount) mustRegister() { - mustRegister(m.success) - mustRegister(m.total) -} - func (m methodCount) Inc(success bool) { - m.total.value.Inc() + m.total.Inc() if success { - m.success.value.Inc() + m.success.Inc() } } @@ -99,8 +95,8 @@ func newObjectServiceMetrics() objectServiceMetrics { } } -func newObjectMethodPayloadCounter(method string) metric[prometheus.Counter] { - return newCounter(prometheus.CounterOpts{ +func newObjectMethodPayloadCounter(method string) prometheus.Counter { + return metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_payload", method), @@ -108,8 +104,8 @@ func newObjectMethodPayloadCounter(method string) metric[prometheus.Counter] { }) } -func newObjectMethodDurationCounter(method string) metric[prometheus.Counter] { - return newCounter(prometheus.CounterOpts{ +func newObjectMethodDurationCounter(method string) prometheus.Counter { + return metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_duration", method), @@ -117,8 +113,8 @@ func newObjectMethodDurationCounter(method string) metric[prometheus.Counter] { }) } -func newObjectGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] { - return newGaugeVec(prometheus.GaugeOpts{ +func newObjectGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { + return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: name, @@ -126,30 +122,6 @@ func newObjectGaugeVector(name, help string, labels []string) metric[*prometheus }, labels) } -func (m objectServiceMetrics) register() { - m.getCounter.mustRegister() - m.putCounter.mustRegister() - m.headCounter.mustRegister() - m.searchCounter.mustRegister() - m.deleteCounter.mustRegister() - m.rangeCounter.mustRegister() - m.rangeHashCounter.mustRegister() - - mustRegister(m.getDuration) - mustRegister(m.putDuration) - mustRegister(m.headDuration) - mustRegister(m.searchDuration) - mustRegister(m.deleteDuration) - mustRegister(m.rangeDuration) - mustRegister(m.rangeHashDuration) - - mustRegister(m.putPayload) - mustRegister(m.getPayload) - - mustRegister(m.shardMetrics) - mustRegister(m.shardsReadonly) -} - func (m objectServiceMetrics) IncGetReqCounter(success bool) { m.getCounter.Inc(success) } @@ -179,43 +151,43 @@ func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) { } func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) { - m.getDuration.value.Add(float64(d)) + m.getDuration.Add(float64(d)) } func (m objectServiceMetrics) AddPutReqDuration(d time.Duration) { - m.putDuration.value.Add(float64(d)) + m.putDuration.Add(float64(d)) } func (m objectServiceMetrics) AddHeadReqDuration(d time.Duration) { - m.headDuration.value.Add(float64(d)) + m.headDuration.Add(float64(d)) } func (m objectServiceMetrics) AddSearchReqDuration(d time.Duration) { - m.searchDuration.value.Add(float64(d)) + m.searchDuration.Add(float64(d)) } func (m objectServiceMetrics) AddDeleteReqDuration(d time.Duration) { - m.deleteDuration.value.Add(float64(d)) + m.deleteDuration.Add(float64(d)) } func (m objectServiceMetrics) AddRangeReqDuration(d time.Duration) { - m.rangeDuration.value.Add(float64(d)) + m.rangeDuration.Add(float64(d)) } func (m objectServiceMetrics) AddRangeHashReqDuration(d time.Duration) { - m.rangeHashDuration.value.Add(float64(d)) + m.rangeHashDuration.Add(float64(d)) } func (m objectServiceMetrics) AddPutPayload(ln int) { - m.putPayload.value.Add(float64(ln)) + m.putPayload.Add(float64(ln)) } func (m objectServiceMetrics) AddGetPayload(ln int) { - m.getPayload.value.Add(float64(ln)) + m.getPayload.Add(float64(ln)) } func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, delta int) { - m.shardMetrics.value.With( + m.shardMetrics.With( prometheus.Labels{ shardIDLabelKey: shardID, counterTypeLabelKey: objectType, @@ -224,7 +196,7 @@ func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, del } func (m objectServiceMetrics) SetObjectCounter(shardID, objectType string, v uint64) { - m.shardMetrics.value.With( + m.shardMetrics.With( prometheus.Labels{ shardIDLabelKey: shardID, counterTypeLabelKey: objectType, @@ -237,7 +209,7 @@ func (m objectServiceMetrics) SetReadonly(shardID string, readonly bool) { if readonly { flag = 1 } - m.shardsReadonly.value.With( + m.shardsReadonly.With( prometheus.Labels{ shardIDLabelKey: shardID, }, diff --git a/pkg/metrics/registry.go b/pkg/metrics/registry.go deleted file mode 100644 index eef613d04..000000000 --- a/pkg/metrics/registry.go +++ /dev/null @@ -1,42 +0,0 @@ -package metrics - -import ( - "net/http" - "sync" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -// Handler returns an http.Handler for the local registry. -func Handler() http.Handler { - promhttp.Handler() - return promhttp.InstrumentMetricHandler( - registry, - promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) -} - -var ( - registry = prometheus.NewRegistry() - // registeredDescriptionsMtx protects collectors slice. - // It should not be acessed concurrently, but we can easily forget this in future, thus this mutex. - registeredDescriptionsMtx sync.Mutex - registeredDescriptions []Description -) - -func init() { - registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) - registry.MustRegister(collectors.NewGoCollector()) -} - -func mustRegister[T prometheus.Collector](cs ...metric[T]) { - for i := range cs { - registry.MustRegister(cs[i].value) - } - registeredDescriptionsMtx.Lock() - for i := range cs { - registeredDescriptions = append(registeredDescriptions, cs[i].desc) - } - registeredDescriptionsMtx.Unlock() -} diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go index 55f736c66..0deafe916 100644 --- a/pkg/metrics/replicator.go +++ b/pkg/metrics/replicator.go @@ -1,29 +1,32 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) const replicatorSubsystem = "replicator" type replicatorMetrics struct { - inFlightRequests metric[prometheus.Gauge] - processedObjects metric[prometheus.Counter] - totalReplicatedPayloadSize metric[prometheus.Counter] + inFlightRequests prometheus.Gauge + processedObjects prometheus.Counter + totalReplicatedPayloadSize prometheus.Counter } func (m replicatorMetrics) IncInFlightRequest() { - m.inFlightRequests.value.Inc() + m.inFlightRequests.Inc() } func (m replicatorMetrics) DecInFlightRequest() { - m.inFlightRequests.value.Dec() + m.inFlightRequests.Dec() } func (m replicatorMetrics) IncProcessedObjects() { - m.processedObjects.value.Inc() + m.processedObjects.Inc() } func (m replicatorMetrics) AddPayloadSize(size int64) { - m.totalReplicatedPayloadSize.value.Add(float64(size)) + m.totalReplicatedPayloadSize.Add(float64(size)) } func newReplicatorMetrics() replicatorMetrics { @@ -34,14 +37,8 @@ func newReplicatorMetrics() replicatorMetrics { } } -func (m replicatorMetrics) register() { - mustRegister(m.inFlightRequests) - mustRegister(m.processedObjects) - mustRegister(m.totalReplicatedPayloadSize) -} - -func newReplicatorCounter(name, help string) metric[prometheus.Counter] { - return newCounter(prometheus.CounterOpts{ +func newReplicatorCounter(name, help string) prometheus.Counter { + return metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: replicatorSubsystem, Name: name, @@ -49,8 +46,8 @@ func newReplicatorCounter(name, help string) metric[prometheus.Counter] { }) } -func newReplicatorGauge(name, help string) metric[prometheus.Gauge] { - return newGauge(prometheus.GaugeOpts{ +func newReplicatorGauge(name, help string) prometheus.Gauge { + return metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: replicatorSubsystem, Name: name, diff --git a/pkg/metrics/state.go b/pkg/metrics/state.go index dce0402cd..893849911 100644 --- a/pkg/metrics/state.go +++ b/pkg/metrics/state.go @@ -1,16 +1,19 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) const stateSubsystem = "state" type stateMetrics struct { - healthCheck metric[prometheus.Gauge] + healthCheck prometheus.Gauge } func newStateMetrics() stateMetrics { return stateMetrics{ - healthCheck: newGauge(prometheus.GaugeOpts{ + healthCheck: metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: stateSubsystem, Name: "health", @@ -19,10 +22,6 @@ func newStateMetrics() stateMetrics { } } -func (m stateMetrics) register() { - mustRegister(m.healthCheck) -} - func (m stateMetrics) SetHealth(s int32) { - m.healthCheck.value.Set(float64(s)) + m.healthCheck.Set(float64(s)) } diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go index 135f6e6d2..903ef3496 100644 --- a/pkg/metrics/treeservice.go +++ b/pkg/metrics/treeservice.go @@ -4,33 +4,34 @@ import ( "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) const treeServiceLabelSuccess = "success" type treeServiceMetrics struct { - replicateTaskDuration metric[*prometheus.HistogramVec] - replicateWaitDuration metric[*prometheus.HistogramVec] - syncOpDuration metric[*prometheus.HistogramVec] + replicateTaskDuration *prometheus.HistogramVec + replicateWaitDuration *prometheus.HistogramVec + syncOpDuration *prometheus.HistogramVec } func newTreeServiceMetrics() *treeServiceMetrics { const treeServiceSubsystem = "treeservice" return &treeServiceMetrics{ - replicateTaskDuration: newHistogramVec(prometheus.HistogramOpts{ + replicateTaskDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "replicate_task_duration_seconds", Help: "Duration of individual replication tasks executed as part of replication loops", }, []string{treeServiceLabelSuccess}), - replicateWaitDuration: newHistogramVec(prometheus.HistogramOpts{ + replicateWaitDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "replicate_wait_duration_seconds", Help: "Duration of overall waiting time for replication loops", }, []string{treeServiceLabelSuccess}), - syncOpDuration: newHistogramVec(prometheus.HistogramOpts{ + syncOpDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "sync_duration_seconds", @@ -39,26 +40,20 @@ func newTreeServiceMetrics() *treeServiceMetrics { } } -func (m *treeServiceMetrics) register() { - mustRegister(m.replicateTaskDuration) - mustRegister(m.replicateWaitDuration) - mustRegister(m.syncOpDuration) -} - func (m *treeServiceMetrics) AddReplicateTaskDuration(d time.Duration, success bool) { - m.replicateTaskDuration.value.With(prometheus.Labels{ + m.replicateTaskDuration.With(prometheus.Labels{ treeServiceLabelSuccess: fmt.Sprintf("%v", success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddReplicateWaitDuration(d time.Duration, success bool) { - m.replicateWaitDuration.value.With(prometheus.Labels{ + m.replicateWaitDuration.With(prometheus.Labels{ treeServiceLabelSuccess: fmt.Sprintf("%v", success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddSyncDuration(d time.Duration, success bool) { - m.syncOpDuration.value.With(prometheus.Labels{ + m.syncOpDuration.With(prometheus.Labels{ treeServiceLabelSuccess: fmt.Sprintf("%v", success), }).Observe(d.Seconds()) } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 74c330842..3c56aa2ba 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -37,18 +38,18 @@ type WriteCacheMetrics interface { } type writeCacheMetrics struct { - getDuration metric[*prometheus.HistogramVec] - putDuration metric[*prometheus.HistogramVec] - deleteDuration metric[*prometheus.HistogramVec] + getDuration *prometheus.HistogramVec + putDuration *prometheus.HistogramVec + deleteDuration *prometheus.HistogramVec - flushCounter metric[*prometheus.CounterVec] - evictCounter metric[*prometheus.CounterVec] + flushCounter *prometheus.CounterVec + evictCounter *prometheus.CounterVec - actualCount metric[*prometheus.GaugeVec] + actualCount *prometheus.GaugeVec - estimatedSize metric[*prometheus.GaugeVec] + estimatedSize *prometheus.GaugeVec - modeMetrics map[shardIDMode]metric[prometheus.GaugeFunc] + modeMetrics map[shardIDMode]prometheus.GaugeFunc modeValues map[string]string modeMtx sync.RWMutex } @@ -63,46 +64,46 @@ func newWriteCacheMetrics() *writeCacheMetrics { actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), modeMtx: sync.RWMutex{}, - modeMetrics: make(map[shardIDMode]metric[prometheus.GaugeFunc]), + modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc), modeValues: make(map[string]string), } } func (m *writeCacheMetrics) AddGetDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.getDuration.value, shardID, success, d, storageType) + setWriteCacheDuration(m.getDuration, shardID, success, d, storageType) } func (m *writeCacheMetrics) AddDeleteDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.deleteDuration.value, shardID, success, d, storageType) + setWriteCacheDuration(m.deleteDuration, shardID, success, d, storageType) } func (m *writeCacheMetrics) AddPutDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.putDuration.value, shardID, success, d, storageType) + setWriteCacheDuration(m.putDuration, shardID, success, d, storageType) } func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { - m.actualCount.value.With(prometheus.Labels{ + m.actualCount.With(prometheus.Labels{ wcShardID: shardID, wcStorage: storageType, }).Inc() } func (m *writeCacheMetrics) DecActualCount(shardID string, storageType string) { - m.actualCount.value.With(prometheus.Labels{ + m.actualCount.With(prometheus.Labels{ wcShardID: shardID, wcStorage: storageType, }).Dec() } func (m *writeCacheMetrics) SetActualCount(shardID string, count uint64, storageType string) { - m.actualCount.value.With(prometheus.Labels{ + m.actualCount.With(prometheus.Labels{ wcShardID: shardID, wcStorage: storageType, }).Set(float64(count)) } func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storageType string) { - m.estimatedSize.value.With(prometheus.Labels{ + m.estimatedSize.With(prometheus.Labels{ wcShardID: shardID, wcStorage: storageType, }).Set(float64(size)) @@ -121,7 +122,7 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { return } - metric := newGaugeFunc( + metric := metrics.NewGaugeFunc( prometheus.GaugeOpts{ Namespace: namespace, Subsystem: wcSubsystem, @@ -141,12 +142,11 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { } return 0 }) - mustRegister(metric) m.modeMetrics[key] = metric } func (m *writeCacheMetrics) IncFlushCounter(shardID string, success bool, storageType string) { - m.flushCounter.value.With(prometheus.Labels{ + m.flushCounter.With(prometheus.Labels{ wcShardID: shardID, wcSuccess: fmt.Sprintf("%v", success), wcStorage: storageType, @@ -154,22 +154,12 @@ func (m *writeCacheMetrics) IncFlushCounter(shardID string, success bool, storag } func (m *writeCacheMetrics) IncEvictCounter(shardID string, storageType string) { - m.evictCounter.value.With(prometheus.Labels{ + m.evictCounter.With(prometheus.Labels{ wcShardID: shardID, wcStorage: storageType, }).Inc() } -func (m *writeCacheMetrics) register() { - mustRegister(m.getDuration) - mustRegister(m.putDuration) - mustRegister(m.deleteDuration) - mustRegister(m.actualCount) - mustRegister(m.estimatedSize) - mustRegister(m.flushCounter) - mustRegister(m.evictCounter) -} - func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success bool, d time.Duration, storageType string) { m.With( prometheus.Labels{ @@ -180,17 +170,17 @@ func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success b ).Observe(float64(d)) } -func newWCMethodDurationCounter(method string) metric[*prometheus.HistogramVec] { - return newHistogramVec(prometheus.HistogramOpts{ +func newWCMethodDurationCounter(method string) *prometheus.HistogramVec { + return metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: wcSubsystem, Name: fmt.Sprintf("%s_req_duration_seconds", method), Help: fmt.Sprintf("Accumulated %s request process duration", method), - }, []string{wcShardID, wcSuccess, wcStorage}) + }, []string{wcShardID, wcSuccess}) } -func newWCOperationCounterVec(operation string, labels []string) metric[*prometheus.CounterVec] { - return newCounterVec(prometheus.CounterOpts{ +func newWCOperationCounterVec(operation string, labels []string) *prometheus.CounterVec { + return metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: wcSubsystem, Name: fmt.Sprintf("%s_operation_count", operation), @@ -198,8 +188,8 @@ func newWCOperationCounterVec(operation string, labels []string) metric[*prometh }, labels) } -func newWCGaugeVec(name, help string, labels []string) metric[*prometheus.GaugeVec] { - return newGaugeVec(prometheus.GaugeOpts{ +func newWCGaugeVec(name, help string, labels []string) *prometheus.GaugeVec { + return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: wcSubsystem, Name: name, diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go index 694eea38b..f29eca37c 100644 --- a/scripts/export-metrics/main.go +++ b/scripts/export-metrics/main.go @@ -6,7 +6,8 @@ import ( "fmt" "os" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + local_metrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) var ( @@ -25,10 +26,10 @@ func main() { var filename string switch { case *node != "": - _ = metrics.NewNodeMetrics() + _ = local_metrics.NewNodeMetrics() filename = *node case *ir != "": - _ = metrics.NewInnerRingMetrics() + _ = local_metrics.NewInnerRingMetrics() filename = *ir default: @@ -36,11 +37,7 @@ func main() { os.Exit(1) } - ds, err := metrics.DescribeAll() - if err != nil { - fmt.Fprintf(os.Stderr, "Could not parse metric descriptions: %v\n", err) - os.Exit(1) - } + ds := metrics.DescribeAll() data, err := json.Marshal(ds) if err != nil { From dcdfb6ed41ba5f8edb104d563c0c0828cd8173c6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 12:26:54 +0300 Subject: [PATCH 0299/1943] [#412] node: Use observability interceptors Use metrics and tracing interceptors. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/tree/client.go | 9 ++++++--- cmd/frostfs-node/grpc.go | 9 ++++++--- pkg/services/tree/cache.go | 9 ++++++--- pkg/services/tree/sync.go | 10 +++++++--- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index f25bff166..4f4f54657 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -5,10 +5,11 @@ import ( "strings" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "github.com/spf13/viper" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -26,10 +27,12 @@ func _client(ctx context.Context) (tree.TreeServiceClient, error) { opts := []grpc.DialOption{ grpc.WithBlock(), grpc.WithChainUnaryInterceptor( - tracing.NewGRPCUnaryClientInteceptor(), + metrics.NewUnaryClientInterceptor(), + tracing.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( - tracing.NewGRPCStreamClientInterceptor(), + metrics.NewStreamClientInterceptor(), + tracing.NewStreamClientInterceptor(), ), } diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index b0a587782..b62ae9c45 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -7,10 +7,11 @@ import ( "net" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -22,10 +23,12 @@ func initGRPC(c *cfg) { serverOpts := []grpc.ServerOption{ grpc.MaxSendMsgSize(maxMsgSize), grpc.ChainUnaryInterceptor( - tracing.NewGRPCUnaryServerInterceptor(), + metrics.NewUnaryServerInterceptor(), + tracing.NewUnaryServerInterceptor(), ), grpc.ChainStreamInterceptor( - tracing.NewGRPCStreamServerInterceptor(), + metrics.NewStreamServerInterceptor(), + tracing.NewStreamServerInterceptor(), ), } diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 3288083cc..97218da08 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -8,8 +8,9 @@ import ( "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "github.com/hashicorp/golang-lru/v2/simplelru" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" @@ -90,10 +91,12 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, opts := []grpc.DialOption{ grpc.WithBlock(), grpc.WithChainUnaryInterceptor( - tracing.NewGRPCUnaryClientInteceptor(), + metrics.NewUnaryClientInterceptor(), + tracing.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( - tracing.NewGRPCStreamClientInterceptor(), + metrics.NewStreamClientInterceptor(), + tracing.NewStreamClientInterceptor(), ), } diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index ec51c6bc6..d132faf6e 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -11,11 +11,13 @@ import ( "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + tracing_grpc "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" @@ -297,10 +299,12 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, cc, err := grpc.DialContext(egCtx, a.URIAddr(), grpc.WithChainUnaryInterceptor( - tracing.NewGRPCUnaryClientInteceptor(), + metrics.NewUnaryClientInterceptor(), + tracing_grpc.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( - tracing.NewGRPCStreamClientInterceptor(), + metrics.NewStreamClientInterceptor(), + tracing_grpc.NewStreamClientInterceptor(), ), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { From 55c28fd5f439a38e2ae70d5e12d91dc0f04ec3e6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 May 2023 12:30:46 +0300 Subject: [PATCH 0300/1943] [#412] cache: Pass DialOptions to create new conn Signed-off-by: Dmitrii Stepanov --- pkg/network/cache/multi.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index b8a0aa4bc..ba81df16c 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -10,8 +10,11 @@ import ( rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -72,6 +75,17 @@ func (x *multiClient) createForAddress(ctx context.Context, addr network.Address prmInit.SetResponseInfoCallback(x.opts.ResponseCallback) } + prmDial.SetGRPCDialOptions( + grpc.WithChainUnaryInterceptor( + metrics.NewUnaryClientInterceptor(), + tracing.NewUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + metrics.NewStreamClientInterceptor(), + tracing.NewStreamClientInterceptor(), + ), + ) + c.Init(prmInit) err := c.Dial(ctx, prmDial) if err != nil { From 2dd3fc8b7e7c4e00d1b808488f31a4afd94d5516 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 1 Jun 2023 12:56:51 +0300 Subject: [PATCH 0301/1943] [#xx] cli: Add policy-playground command Signed-off-by: Alejandro Lopez --- .../modules/container/policy_playground.go | 178 ++++++++++++++++++ cmd/frostfs-cli/modules/container/root.go | 2 + 2 files changed, 180 insertions(+) create mode 100644 cmd/frostfs-cli/modules/container/policy_playground.go diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go new file mode 100644 index 000000000..fa10dc10e --- /dev/null +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -0,0 +1,178 @@ +package container + +import ( + "bufio" + "encoding/hex" + "fmt" + "io" + "os" + "strings" + + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +type policyPlaygroundREPL struct { + cmd *cobra.Command + args []string + nodes map[string]netmap.NodeInfo +} + +func newPolicyPlaygroundREPL(cmd *cobra.Command, args []string) (*policyPlaygroundREPL, error) { + return &policyPlaygroundREPL{ + cmd: cmd, + args: args, + nodes: map[string]netmap.NodeInfo{}, + }, nil +} + +func (repl *policyPlaygroundREPL) handleLs(args []string) error { + if len(args) > 0 { + return fmt.Errorf("too many arguments for command 'ls': got %d, want 0", len(args)) + } + i := 1 + for id, node := range repl.nodes { + var attrs []string + node.IterateAttributes(func(k, v string) { + attrs = append(attrs, fmt.Sprintf("%s:%s", k, v)) + }) + fmt.Printf("\t%2d: id=%s attrs={%v}\n", i, id, strings.Join(attrs, " ")) + i++ + } + return nil +} + +func (repl *policyPlaygroundREPL) handleAdd(args []string) error { + if len(args) == 0 { + return fmt.Errorf("too few arguments for command 'add': got %d, want >0", len(args)) + } + id := args[0] + key, err := hex.DecodeString(id) + if err != nil { + return fmt.Errorf("node id must be a hex string: got %q: %v", id, err) + } + node := repl.nodes[id] + node.SetPublicKey(key) + for _, attr := range args[1:] { + kv := strings.Split(attr, ":") + if len(kv) != 2 { + return fmt.Errorf("node attributes must be in the format 'KEY:VALUE': got %q", attr) + } + node.SetAttribute(kv[0], kv[1]) + } + repl.nodes[id] = node + return nil +} + +func (repl *policyPlaygroundREPL) handleRemove(args []string) error { + if len(args) == 0 { + return fmt.Errorf("too few arguments for command 'remove': got %d, want >0", len(args)) + } + id := args[0] + if _, exists := repl.nodes[id]; exists { + delete(repl.nodes, id) + return nil + } + return fmt.Errorf("node not found: id=%q", id) +} + +func (repl *policyPlaygroundREPL) handleEval(args []string) error { + policyStr := strings.Join(args, " ") + placementPolicy, err := parseContainerPolicy(repl.cmd, policyStr) + if err != nil { + return fmt.Errorf("parsing placement policy: %v", err) + } + nm := repl.netMap() + nodes, err := nm.ContainerNodes(*placementPolicy, nil) + if err != nil { + return fmt.Errorf("building container nodes: %v", err) + } + for i, ns := range nodes { + var ids []string + for _, node := range ns { + ids = append(ids, hex.EncodeToString(node.PublicKey())) + } + fmt.Printf("\t%2d: %v\n", i+1, ids) + } + + return nil +} + +func (repl *policyPlaygroundREPL) netMap() netmap.NetMap { + var nm netmap.NetMap + var nodes []netmap.NodeInfo + for _, node := range repl.nodes { + nodes = append(nodes, node) + } + nm.SetNodes(nodes) + return nm +} + +func (repl *policyPlaygroundREPL) run() error { + if len(viper.GetString(commonflags.RPC)) > 0 { + key := key.GetOrGenerate(repl.cmd) + cli := internalclient.GetSDKClientByFlag(repl.cmd, key, commonflags.RPC) + + var prm internalclient.NetMapSnapshotPrm + prm.SetClient(cli) + + resp, err := internalclient.NetMapSnapshot(prm) + commonCmd.ExitOnErr(repl.cmd, "unable to get netmap snapshot to populate initial netmap: %w", err) + + for _, node := range resp.NetMap().Nodes() { + id := hex.EncodeToString(node.PublicKey()) + repl.nodes[id] = node + } + } + + cmdHandlers := map[string]func([]string) error{ + "ls": repl.handleLs, + "add": repl.handleAdd, + "remove": repl.handleRemove, + "eval": repl.handleEval, + } + for reader := bufio.NewReader(os.Stdin); ; { + fmt.Print("> ") + line, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + return nil + } + return fmt.Errorf("reading line: %v", err) + } + parts := strings.Fields(line) + if len(parts) == 0 { + continue + } + cmd := parts[0] + handler, exists := cmdHandlers[cmd] + if exists { + if err := handler(parts[1:]); err != nil { + fmt.Printf("error: %v\n", err) + } + } else { + fmt.Printf("error: unknown command %q\n", cmd) + } + } +} + +var policyPlaygroundCmd = &cobra.Command{ + Use: "policy-playground", + Short: "A REPL for testing placement policies", + Long: `A REPL for testing placement policies. +If a wallet and endpoint is provided, the initial netmap data will be loaded from the snapshot of the node. Otherwise, an empty playground is created.`, + Run: func(cmd *cobra.Command, args []string) { + repl, err := newPolicyPlaygroundREPL(cmd, args) + commonCmd.ExitOnErr(cmd, "could not create policy playground: %w", err) + commonCmd.ExitOnErr(cmd, "policy playground failed: %w", repl.run()) + }, +} + +func initContainerPolicyPlaygroundCmd() { + commonflags.Init(policyPlaygroundCmd) +} diff --git a/cmd/frostfs-cli/modules/container/root.go b/cmd/frostfs-cli/modules/container/root.go index 30a82954a..ab7f5fb90 100644 --- a/cmd/frostfs-cli/modules/container/root.go +++ b/cmd/frostfs-cli/modules/container/root.go @@ -28,6 +28,7 @@ func init() { getExtendedACLCmd, setExtendedACLCmd, containerNodesCmd, + policyPlaygroundCmd, } Cmd.AddCommand(containerChildCommand...) @@ -40,6 +41,7 @@ func init() { initContainerGetEACLCmd() initContainerSetEACLCmd() initContainerNodesCmd() + initContainerPolicyPlaygroundCmd() for _, containerCommand := range containerChildCommand { commonflags.InitAPI(containerCommand) From 63473d0806691c3f7cdd84a98710611338d637c6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 1 Jun 2023 17:21:03 +0300 Subject: [PATCH 0302/1943] [#417] go.mod: Update neo-go Update to master, because after the API update notary signer is being incorrectly formed. Signed-off-by: Evgenii Stratonikov --- go.mod | 8 ++++++-- go.sum | 21 ++++++++++++++++----- pkg/morph/client/constructor.go | 6 ++++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 6c49c02b4..d5b3826b2 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.9.0 github.com/nats-io/nats.go v1.25.0 - github.com/nspcc-dev/neo-go v0.101.1 + github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.7.4 github.com/paulmach/orb v0.9.2 @@ -49,6 +49,8 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.9.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -72,6 +74,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect @@ -81,7 +84,7 @@ require ( github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230601131642-a0117042e8fc // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -114,6 +117,7 @@ require ( google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect + rsc.io/tmplfunc v0.0.3 // indirect ) retract ( diff --git a/go.sum b/go.sum index b9b3fde41..242614781 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,10 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.9.1 h1:mru55qKdWl3E035hAoh1jj9d7hVnYY5pfb6tmovSmII= +github.com/consensys/gnark-crypto v0.9.1/go.mod h1:a2DQL4+5ywF6safEeZFEPGRiiGbjzGFRUN2sg06VuU4= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= @@ -152,7 +156,7 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -246,6 +250,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -317,6 +322,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -343,6 +349,9 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -388,11 +397,11 @@ github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkP github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= github.com/nspcc-dev/neo-go v0.99.4/go.mod h1:mKTolfRUfKjFso5HPvGSQtUZc70n0VKBMs16eGuC5gA= -github.com/nspcc-dev/neo-go v0.101.1 h1:TVdcIpH/+bxQBTLRwWE3+Pw3j6j/JwguENbBSGAGid0= -github.com/nspcc-dev/neo-go v0.101.1/go.mod h1:J4tspxWw7jknX06F+VSMsKvIiNpYGfVTb2IxVC005YU= +github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc h1:fySIWvUQsitK5e5qYIHnTDCXuPpwzz89SEUEIyY11sg= +github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc/go.mod h1:s9QhjMC784MWqTURovMbyYduIJc86mnCruxcMiAebpc= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb h1:GFxfkpXEYAbMIr69JpKOsQWeLOaGrd49HNAor8uDW+A= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230601131642-a0117042e8fc h1:yJM7a9iW4ZIvos2R5jn/Z7I4iZCqWdBVPclRs3UebzY= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230601131642-a0117042e8fc/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= @@ -1005,3 +1014,5 @@ lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index ff37da728..1d55db71c 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -155,8 +155,10 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } func (c *Client) newCli(ctx context.Context, endpoint string) (*rpcclient.WSClient, *actor.Actor, error) { - cli, err := rpcclient.NewWS(ctx, endpoint, rpcclient.Options{ - DialTimeout: c.cfg.dialTimeout, + cli, err := rpcclient.NewWS(ctx, endpoint, rpcclient.WSOptions{ + Options: rpcclient.Options{ + DialTimeout: c.cfg.dialTimeout, + }, }) if err != nil { return nil, nil, fmt.Errorf("WS client creation: %w", err) From 9562123c49d43c7f84cc47d6606fdcefc17f2085 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 24 May 2023 16:51:57 +0300 Subject: [PATCH 0303/1943] [#406] cli: Pass context to internal client Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/client/client.go | 80 +++++++++---------- cmd/frostfs-cli/modules/accounting/balance.go | 2 +- cmd/frostfs-cli/modules/container/create.go | 8 +- cmd/frostfs-cli/modules/container/delete.go | 8 +- cmd/frostfs-cli/modules/container/get.go | 2 +- cmd/frostfs-cli/modules/container/get_eacl.go | 2 +- cmd/frostfs-cli/modules/container/list.go | 4 +- .../modules/container/list_objects.go | 4 +- cmd/frostfs-cli/modules/container/nodes.go | 2 +- cmd/frostfs-cli/modules/container/set_eacl.go | 6 +- cmd/frostfs-cli/modules/netmap/get_epoch.go | 2 +- cmd/frostfs-cli/modules/netmap/netinfo.go | 2 +- cmd/frostfs-cli/modules/netmap/nodeinfo.go | 2 +- cmd/frostfs-cli/modules/netmap/snapshot.go | 2 +- cmd/frostfs-cli/modules/object/delete.go | 2 +- cmd/frostfs-cli/modules/object/get.go | 2 +- cmd/frostfs-cli/modules/object/hash.go | 4 +- cmd/frostfs-cli/modules/object/head.go | 2 +- cmd/frostfs-cli/modules/object/lock.go | 2 +- cmd/frostfs-cli/modules/object/put.go | 2 +- cmd/frostfs-cli/modules/object/range.go | 2 +- cmd/frostfs-cli/modules/object/search.go | 2 +- cmd/frostfs-cli/modules/object/util.go | 12 +-- cmd/frostfs-cli/modules/session/create.go | 9 ++- 24 files changed, 83 insertions(+), 82 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 875ccf904..0e0aadbb3 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -37,8 +37,8 @@ func (x BalanceOfRes) Balance() accounting.Decimal { // BalanceOf requests the current balance of a FrostFS user. // // Returns any error which prevented the operation from completing correctly in error return. -func BalanceOf(prm BalanceOfPrm) (res BalanceOfRes, err error) { - res.cliRes, err = prm.cli.BalanceGet(context.Background(), prm.PrmBalanceGet) +func BalanceOf(ctx context.Context, prm BalanceOfPrm) (res BalanceOfRes, err error) { + res.cliRes, err = prm.cli.BalanceGet(ctx, prm.PrmBalanceGet) return } @@ -62,8 +62,8 @@ func (x ListContainersRes) IDList() []cid.ID { // ListContainers requests a list of FrostFS user's containers. // // Returns any error which prevented the operation from completing correctly in error return. -func ListContainers(prm ListContainersPrm) (res ListContainersRes, err error) { - res.cliRes, err = prm.cli.ContainerList(context.Background(), prm.PrmContainerList) +func ListContainers(ctx context.Context, prm ListContainersPrm) (res ListContainersRes, err error) { + res.cliRes, err = prm.cli.ContainerList(ctx, prm.PrmContainerList) return } @@ -92,8 +92,8 @@ func (x PutContainerRes) ID() cid.ID { // Success can be verified by reading by identifier. // // Returns any error which prevented the operation from completing correctly in error return. -func PutContainer(prm PutContainerPrm) (res PutContainerRes, err error) { - cliRes, err := prm.cli.ContainerPut(context.Background(), prm.PrmContainerPut) +func PutContainer(ctx context.Context, prm PutContainerPrm) (res PutContainerRes, err error) { + cliRes, err := prm.cli.ContainerPut(ctx, prm.PrmContainerPut) if err == nil { res.cnr = cliRes.ID() } @@ -125,20 +125,20 @@ func (x GetContainerRes) Container() containerSDK.Container { // GetContainer reads a container from FrostFS by ID. // // Returns any error which prevented the operation from completing correctly in error return. -func GetContainer(prm GetContainerPrm) (res GetContainerRes, err error) { - res.cliRes, err = prm.cli.ContainerGet(context.Background(), prm.cliPrm) +func GetContainer(ctx context.Context, prm GetContainerPrm) (res GetContainerRes, err error) { + res.cliRes, err = prm.cli.ContainerGet(ctx, prm.cliPrm) return } // IsACLExtendable checks if ACL of the container referenced by the given identifier // can be extended. Client connection MUST BE correctly established in advance. -func IsACLExtendable(c *client.Client, cnr cid.ID) (bool, error) { +func IsACLExtendable(ctx context.Context, c *client.Client, cnr cid.ID) (bool, error) { var prm GetContainerPrm prm.SetClient(c) prm.SetContainer(cnr) - res, err := GetContainer(prm) + res, err := GetContainer(ctx, prm) if err != nil { return false, fmt.Errorf("get container from the FrostFS: %w", err) } @@ -163,8 +163,8 @@ type DeleteContainerRes struct{} // Success can be verified by reading by identifier. // // Returns any error which prevented the operation from completing correctly in error return. -func DeleteContainer(prm DeleteContainerPrm) (res DeleteContainerRes, err error) { - _, err = prm.cli.ContainerDelete(context.Background(), prm.PrmContainerDelete) +func DeleteContainer(ctx context.Context, prm DeleteContainerPrm) (res DeleteContainerRes, err error) { + _, err = prm.cli.ContainerDelete(ctx, prm.PrmContainerDelete) return } @@ -188,8 +188,8 @@ func (x EACLRes) EACL() eacl.Table { // EACL reads eACL table from FrostFS by container ID. // // Returns any error which prevented the operation from completing correctly in error return. -func EACL(prm EACLPrm) (res EACLRes, err error) { - res.cliRes, err = prm.cli.ContainerEACL(context.Background(), prm.PrmContainerEACL) +func EACL(ctx context.Context, prm EACLPrm) (res EACLRes, err error) { + res.cliRes, err = prm.cli.ContainerEACL(ctx, prm.PrmContainerEACL) return } @@ -211,8 +211,8 @@ type SetEACLRes struct{} // Success can be verified by reading by container identifier. // // Returns any error which prevented the operation from completing correctly in error return. -func SetEACL(prm SetEACLPrm) (res SetEACLRes, err error) { - _, err = prm.cli.ContainerSetEACL(context.Background(), prm.PrmContainerSetEACL) +func SetEACL(ctx context.Context, prm SetEACLPrm) (res SetEACLRes, err error) { + _, err = prm.cli.ContainerSetEACL(ctx, prm.PrmContainerSetEACL) return } @@ -236,8 +236,8 @@ func (x NetworkInfoRes) NetworkInfo() netmap.NetworkInfo { // NetworkInfo reads information about the FrostFS network. // // Returns any error which prevented the operation from completing correctly in error return. -func NetworkInfo(prm NetworkInfoPrm) (res NetworkInfoRes, err error) { - res.cliRes, err = prm.cli.NetworkInfo(context.Background(), prm.PrmNetworkInfo) +func NetworkInfo(ctx context.Context, prm NetworkInfoPrm) (res NetworkInfoRes, err error) { + res.cliRes, err = prm.cli.NetworkInfo(ctx, prm.PrmNetworkInfo) return } @@ -266,8 +266,8 @@ func (x NodeInfoRes) LatestVersion() version.Version { // NodeInfo requests information about the remote server from FrostFS netmap. // // Returns any error which prevented the operation from completing correctly in error return. -func NodeInfo(prm NodeInfoPrm) (res NodeInfoRes, err error) { - res.cliRes, err = prm.cli.EndpointInfo(context.Background(), prm.PrmEndpointInfo) +func NodeInfo(ctx context.Context, prm NodeInfoPrm) (res NodeInfoRes, err error) { + res.cliRes, err = prm.cli.EndpointInfo(ctx, prm.PrmEndpointInfo) return } @@ -290,8 +290,8 @@ func (x NetMapSnapshotRes) NetMap() netmap.NetMap { // NetMapSnapshot requests current network view of the remote server. // // Returns any error which prevented the operation from completing correctly in error return. -func NetMapSnapshot(prm NetMapSnapshotPrm) (res NetMapSnapshotRes, err error) { - res.cliRes, err = prm.cli.NetMapSnapshot(context.Background(), client.PrmNetMapSnapshot{}) +func NetMapSnapshot(ctx context.Context, prm NetMapSnapshotPrm) (res NetMapSnapshotRes, err error) { + res.cliRes, err = prm.cli.NetMapSnapshot(ctx, client.PrmNetMapSnapshot{}) return } @@ -319,8 +319,8 @@ func (x CreateSessionRes) SessionKey() []byte { // CreateSession opens a new unlimited session with the remote node. // // Returns any error which prevented the operation from completing correctly in error return. -func CreateSession(prm CreateSessionPrm) (res CreateSessionRes, err error) { - res.cliRes, err = prm.cli.SessionCreate(context.Background(), prm.PrmSessionCreate) +func CreateSession(ctx context.Context, prm CreateSessionPrm) (res CreateSessionRes, err error) { + res.cliRes, err = prm.cli.SessionCreate(ctx, prm.PrmSessionCreate) return } @@ -373,7 +373,7 @@ func (x PutObjectRes) ID() oid.ID { // PutObject saves the object in FrostFS network. // // Returns any error which prevented the operation from completing correctly in error return. -func PutObject(prm PutObjectPrm) (*PutObjectRes, error) { +func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { var putPrm client.PrmObjectPutInit if prm.sessionToken != nil { @@ -391,7 +391,7 @@ func PutObject(prm PutObjectPrm) (*PutObjectRes, error) { putPrm.WithXHeaders(prm.xHeaders...) putPrm.SetCopiesNumberByVectors(prm.copyNum) - wrt, err := prm.cli.ObjectPutInit(context.Background(), putPrm) + wrt, err := prm.cli.ObjectPutInit(ctx, putPrm) if err != nil { return nil, fmt.Errorf("init object writing: %w", err) } @@ -471,7 +471,7 @@ func (x DeleteObjectRes) Tombstone() oid.ID { // DeleteObject marks an object to be removed from FrostFS through tombstone placement. // // Returns any error which prevented the operation from completing correctly in error return. -func DeleteObject(prm DeleteObjectPrm) (*DeleteObjectRes, error) { +func DeleteObject(ctx context.Context, prm DeleteObjectPrm) (*DeleteObjectRes, error) { var delPrm client.PrmObjectDelete delPrm.FromContainer(prm.objAddr.Container()) delPrm.ByID(prm.objAddr.Object()) @@ -486,7 +486,7 @@ func DeleteObject(prm DeleteObjectPrm) (*DeleteObjectRes, error) { delPrm.WithXHeaders(prm.xHeaders...) - cliRes, err := prm.cli.ObjectDelete(context.Background(), delPrm) + cliRes, err := prm.cli.ObjectDelete(ctx, delPrm) if err != nil { return nil, fmt.Errorf("remove object via client: %w", err) } @@ -527,7 +527,7 @@ func (x GetObjectRes) Header() *object.Object { // // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. -func GetObject(prm GetObjectPrm) (*GetObjectRes, error) { +func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { var getPrm client.PrmObjectGet getPrm.FromContainer(prm.objAddr.Container()) getPrm.ByID(prm.objAddr.Object()) @@ -550,7 +550,7 @@ func GetObject(prm GetObjectPrm) (*GetObjectRes, error) { getPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectGetInit(context.Background(), getPrm) + rdr, err := prm.cli.ObjectGetInit(ctx, getPrm) if err != nil { return nil, fmt.Errorf("init object reading on client: %w", err) } @@ -603,7 +603,7 @@ func (x HeadObjectRes) Header() *object.Object { // // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. -func HeadObject(prm HeadObjectPrm) (*HeadObjectRes, error) { +func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) { var cliPrm client.PrmObjectHead cliPrm.FromContainer(prm.objAddr.Container()) cliPrm.ByID(prm.objAddr.Object()) @@ -626,7 +626,7 @@ func HeadObject(prm HeadObjectPrm) (*HeadObjectRes, error) { cliPrm.WithXHeaders(prm.xHeaders...) - res, err := prm.cli.ObjectHead(context.Background(), cliPrm) + res, err := prm.cli.ObjectHead(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("read object header via client: %w", err) } @@ -668,7 +668,7 @@ func (x SearchObjectsRes) IDList() []oid.ID { // SearchObjects selects objects from the container which match the filters. // // Returns any error which prevented the operation from completing correctly in error return. -func SearchObjects(prm SearchObjectsPrm) (*SearchObjectsRes, error) { +func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes, error) { var cliPrm client.PrmObjectSearch cliPrm.InContainer(prm.cnrID) cliPrm.SetFilters(prm.filters) @@ -687,7 +687,7 @@ func SearchObjects(prm SearchObjectsPrm) (*SearchObjectsRes, error) { cliPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectSearchInit(context.Background(), cliPrm) + rdr, err := prm.cli.ObjectSearchInit(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("init object search: %w", err) } @@ -758,7 +758,7 @@ func (x HashPayloadRangesRes) HashList() [][]byte { // // Returns any error which prevented the operation from completing correctly in error return. // Returns an error if number of received hashes differs with the number of requested ranges. -func HashPayloadRanges(prm HashPayloadRangesPrm) (*HashPayloadRangesRes, error) { +func HashPayloadRanges(ctx context.Context, prm HashPayloadRangesPrm) (*HashPayloadRangesRes, error) { var cliPrm client.PrmObjectHash cliPrm.FromContainer(prm.objAddr.Container()) cliPrm.ByID(prm.objAddr.Object()) @@ -792,7 +792,7 @@ func HashPayloadRanges(prm HashPayloadRangesPrm) (*HashPayloadRangesRes, error) cliPrm.WithXHeaders(prm.xHeaders...) - res, err := prm.cli.ObjectHash(context.Background(), cliPrm) + res, err := prm.cli.ObjectHash(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("read payload hashes via client: %w", err) } @@ -826,7 +826,7 @@ type PayloadRangeRes struct{} // // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. -func PayloadRange(prm PayloadRangePrm) (*PayloadRangeRes, error) { +func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, error) { var cliPrm client.PrmObjectRange cliPrm.FromContainer(prm.objAddr.Container()) cliPrm.ByID(prm.objAddr.Object()) @@ -852,7 +852,7 @@ func PayloadRange(prm PayloadRangePrm) (*PayloadRangeRes, error) { cliPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectRangeInit(context.Background(), cliPrm) + rdr, err := prm.cli.ObjectRangeInit(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("init payload reading: %w", err) } @@ -886,12 +886,12 @@ type SyncContainerRes struct{} // Interrupts on any writer error. // // Panics if a container passed as a parameter is nil. -func SyncContainerSettings(prm SyncContainerPrm) (*SyncContainerRes, error) { +func SyncContainerSettings(ctx context.Context, prm SyncContainerPrm) (*SyncContainerRes, error) { if prm.c == nil { panic("sync container settings with the network: nil container") } - err := client.SyncContainerWithNetwork(context.Background(), prm.c, prm.cli) + err := client.SyncContainerWithNetwork(ctx, prm.c, prm.cli) if err != nil { return nil, err } diff --git a/cmd/frostfs-cli/modules/accounting/balance.go b/cmd/frostfs-cli/modules/accounting/balance.go index bec40f1ff..5ed8f9403 100644 --- a/cmd/frostfs-cli/modules/accounting/balance.go +++ b/cmd/frostfs-cli/modules/accounting/balance.go @@ -41,7 +41,7 @@ var accountingBalanceCmd = &cobra.Command{ prm.SetClient(cli) prm.SetAccount(idUser) - res, err := internalclient.BalanceOf(prm) + res, err := internalclient.BalanceOf(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) // print to stdout diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 873ef3235..2acdfc16f 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -48,7 +48,7 @@ It will be stored in sidechain when inner ring will accepts it.`, var prm internalclient.NetMapSnapshotPrm prm.SetClient(cli) - resmap, err := internalclient.NetMapSnapshot(prm) + resmap, err := internalclient.NetMapSnapshot(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "unable to get netmap snapshot to validate container placement, "+ "use --force option to skip this check: %w", err) @@ -96,7 +96,7 @@ It will be stored in sidechain when inner ring will accepts it.`, syncContainerPrm.SetClient(cli) syncContainerPrm.SetContainer(&cnr) - _, err = internalclient.SyncContainerSettings(syncContainerPrm) + _, err = internalclient.SyncContainerSettings(cmd.Context(), syncContainerPrm) commonCmd.ExitOnErr(cmd, "syncing container's settings rpc error: %w", err) var putPrm internalclient.PutContainerPrm @@ -107,7 +107,7 @@ It will be stored in sidechain when inner ring will accepts it.`, putPrm.WithinSession(*tok) } - res, err := internalclient.PutContainer(putPrm) + res, err := internalclient.PutContainer(cmd.Context(), putPrm) commonCmd.ExitOnErr(cmd, "put container rpc error: %w", err) id := res.ID() @@ -124,7 +124,7 @@ It will be stored in sidechain when inner ring will accepts it.`, for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) - _, err := internalclient.GetContainer(getPrm) + _, err := internalclient.GetContainer(cmd.Context(), getPrm) if err == nil { cmd.Println("container has been persisted on sidechain") return diff --git a/cmd/frostfs-cli/modules/container/delete.go b/cmd/frostfs-cli/modules/container/delete.go index 308c7b942..54b49a32d 100644 --- a/cmd/frostfs-cli/modules/container/delete.go +++ b/cmd/frostfs-cli/modules/container/delete.go @@ -34,7 +34,7 @@ Only owner of the container has a permission to remove container.`, getPrm.SetClient(cli) getPrm.SetContainer(id) - resGet, err := internalclient.GetContainer(getPrm) + resGet, err := internalclient.GetContainer(cmd.Context(), getPrm) commonCmd.ExitOnErr(cmd, "can't get the container: %w", err) owner := resGet.Container().Owner() @@ -72,7 +72,7 @@ Only owner of the container has a permission to remove container.`, common.PrintVerbose(cmd, "Searching for LOCK objects...") - res, err := internalclient.SearchObjects(searchPrm) + res, err := internalclient.SearchObjects(cmd.Context(), searchPrm) commonCmd.ExitOnErr(cmd, "can't search for LOCK objects: %w", err) if len(res.IDList()) != 0 { @@ -91,7 +91,7 @@ Only owner of the container has a permission to remove container.`, delPrm.WithinSession(*tok) } - _, err := internalclient.DeleteContainer(delPrm) + _, err := internalclient.DeleteContainer(cmd.Context(), delPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) cmd.Println("container delete method invoked") @@ -106,7 +106,7 @@ Only owner of the container has a permission to remove container.`, for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) - _, err := internalclient.GetContainer(getPrm) + _, err := internalclient.GetContainer(cmd.Context(), getPrm) if err != nil { cmd.Println("container has been removed:", containerID) return diff --git a/cmd/frostfs-cli/modules/container/get.go b/cmd/frostfs-cli/modules/container/get.go index 2db1f7c8d..90bcc190a 100644 --- a/cmd/frostfs-cli/modules/container/get.go +++ b/cmd/frostfs-cli/modules/container/get.go @@ -151,7 +151,7 @@ func getContainer(cmd *cobra.Command) (container.Container, *ecdsa.PrivateKey) { prm.SetClient(cli) prm.SetContainer(id) - res, err := internalclient.GetContainer(prm) + res, err := internalclient.GetContainer(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) cnr = res.Container() diff --git a/cmd/frostfs-cli/modules/container/get_eacl.go b/cmd/frostfs-cli/modules/container/get_eacl.go index bc04a84fc..21ea5b5bc 100644 --- a/cmd/frostfs-cli/modules/container/get_eacl.go +++ b/cmd/frostfs-cli/modules/container/get_eacl.go @@ -24,7 +24,7 @@ var getExtendedACLCmd = &cobra.Command{ eaclPrm.SetClient(cli) eaclPrm.SetContainer(id) - res, err := internalclient.EACL(eaclPrm) + res, err := internalclient.EACL(cmd.Context(), eaclPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) eaclTable := res.EACL() diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 33dd17943..1c9e4767f 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -49,7 +49,7 @@ var listContainersCmd = &cobra.Command{ prm.SetClient(cli) prm.SetAccount(idUser) - res, err := internalclient.ListContainers(prm) + res, err := internalclient.ListContainers(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) var prmGet internalclient.GetContainerPrm @@ -63,7 +63,7 @@ var listContainersCmd = &cobra.Command{ } prmGet.SetContainer(cnrID) - res, err := internalclient.GetContainer(prmGet) + res, err := internalclient.GetContainer(cmd.Context(), prmGet) if err != nil { cmd.Printf(" failed to read attributes: %v\n", err) continue diff --git a/cmd/frostfs-cli/modules/container/list_objects.go b/cmd/frostfs-cli/modules/container/list_objects.go index aef4a1f80..e417560e8 100644 --- a/cmd/frostfs-cli/modules/container/list_objects.go +++ b/cmd/frostfs-cli/modules/container/list_objects.go @@ -51,7 +51,7 @@ var listContainerObjectsCmd = &cobra.Command{ prmSearch.SetContainerID(id) prmSearch.SetFilters(*filters) - res, err := internalclient.SearchObjects(prmSearch) + res, err := internalclient.SearchObjects(cmd.Context(), prmSearch) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) objectIDs := res.IDList() @@ -65,7 +65,7 @@ var listContainerObjectsCmd = &cobra.Command{ addr.SetObject(objectIDs[i]) prmHead.SetAddress(addr) - resHead, err := internalclient.HeadObject(prmHead) + resHead, err := internalclient.HeadObject(cmd.Context(), prmHead) if err == nil { attrs := resHead.Header().Attributes() for i := range attrs { diff --git a/cmd/frostfs-cli/modules/container/nodes.go b/cmd/frostfs-cli/modules/container/nodes.go index d89772fcc..8b0f266a7 100644 --- a/cmd/frostfs-cli/modules/container/nodes.go +++ b/cmd/frostfs-cli/modules/container/nodes.go @@ -31,7 +31,7 @@ var containerNodesCmd = &cobra.Command{ var prm internalclient.NetMapSnapshotPrm prm.SetClient(cli) - resmap, err := internalclient.NetMapSnapshot(prm) + resmap, err := internalclient.NetMapSnapshot(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "unable to get netmap snapshot", err) var id cid.ID diff --git a/cmd/frostfs-cli/modules/container/set_eacl.go b/cmd/frostfs-cli/modules/container/set_eacl.go index 0b781589f..c88d5767b 100644 --- a/cmd/frostfs-cli/modules/container/set_eacl.go +++ b/cmd/frostfs-cli/modules/container/set_eacl.go @@ -38,7 +38,7 @@ Container ID in EACL table will be substituted with ID from the CLI.`, if !flagVarsSetEACL.noPreCheck { cmd.Println("Checking the ability to modify access rights in the container...") - extendable, err := internalclient.IsACLExtendable(cli, id) + extendable, err := internalclient.IsACLExtendable(cmd.Context(), cli, id) commonCmd.ExitOnErr(cmd, "Extensibility check failure: %w", err) if !extendable { @@ -56,7 +56,7 @@ Container ID in EACL table will be substituted with ID from the CLI.`, setEACLPrm.WithinSession(*tok) } - _, err := internalclient.SetEACL(setEACLPrm) + _, err := internalclient.SetEACL(cmd.Context(), setEACLPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) if containerAwait { @@ -72,7 +72,7 @@ Container ID in EACL table will be substituted with ID from the CLI.`, for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) - res, err := internalclient.EACL(getEACLPrm) + res, err := internalclient.EACL(cmd.Context(), getEACLPrm) if err == nil { // compare binary values because EACL could have been set already table := res.EACL() diff --git a/cmd/frostfs-cli/modules/netmap/get_epoch.go b/cmd/frostfs-cli/modules/netmap/get_epoch.go index 6e05721ff..a9c2e1f19 100644 --- a/cmd/frostfs-cli/modules/netmap/get_epoch.go +++ b/cmd/frostfs-cli/modules/netmap/get_epoch.go @@ -19,7 +19,7 @@ var getEpochCmd = &cobra.Command{ var prm internalclient.NetworkInfoPrm prm.SetClient(cli) - res, err := internalclient.NetworkInfo(prm) + res, err := internalclient.NetworkInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) netInfo := res.NetworkInfo() diff --git a/cmd/frostfs-cli/modules/netmap/netinfo.go b/cmd/frostfs-cli/modules/netmap/netinfo.go index 17acfd59c..c6ab0b6f6 100644 --- a/cmd/frostfs-cli/modules/netmap/netinfo.go +++ b/cmd/frostfs-cli/modules/netmap/netinfo.go @@ -23,7 +23,7 @@ var netInfoCmd = &cobra.Command{ var prm internalclient.NetworkInfoPrm prm.SetClient(cli) - res, err := internalclient.NetworkInfo(prm) + res, err := internalclient.NetworkInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) netInfo := res.NetworkInfo() diff --git a/cmd/frostfs-cli/modules/netmap/nodeinfo.go b/cmd/frostfs-cli/modules/netmap/nodeinfo.go index 4a94d9e70..3b2113efb 100644 --- a/cmd/frostfs-cli/modules/netmap/nodeinfo.go +++ b/cmd/frostfs-cli/modules/netmap/nodeinfo.go @@ -25,7 +25,7 @@ var nodeInfoCmd = &cobra.Command{ var prm internalclient.NodeInfoPrm prm.SetClient(cli) - res, err := internalclient.NodeInfo(prm) + res, err := internalclient.NodeInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) prettyPrintNodeInfo(cmd, res.NodeInfo()) diff --git a/cmd/frostfs-cli/modules/netmap/snapshot.go b/cmd/frostfs-cli/modules/netmap/snapshot.go index 0878f5ceb..eaaf598b9 100644 --- a/cmd/frostfs-cli/modules/netmap/snapshot.go +++ b/cmd/frostfs-cli/modules/netmap/snapshot.go @@ -19,7 +19,7 @@ var snapshotCmd = &cobra.Command{ var prm internalclient.NetMapSnapshotPrm prm.SetClient(cli) - res, err := internalclient.NetMapSnapshot(prm) + res, err := internalclient.NetMapSnapshot(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) commonCmd.PrettyPrintNetMap(cmd, res.NetMap(), false) diff --git a/cmd/frostfs-cli/modules/object/delete.go b/cmd/frostfs-cli/modules/object/delete.go index 25d5703de..e4e9cddb8 100644 --- a/cmd/frostfs-cli/modules/object/delete.go +++ b/cmd/frostfs-cli/modules/object/delete.go @@ -65,7 +65,7 @@ func deleteObject(cmd *cobra.Command, _ []string) { Prepare(cmd, &prm) prm.SetAddress(objAddr) - res, err := internalclient.DeleteObject(prm) + res, err := internalclient.DeleteObject(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) tomb := res.Tombstone() diff --git a/cmd/frostfs-cli/modules/object/get.go b/cmd/frostfs-cli/modules/object/get.go index 68e47da6f..c72a26239 100644 --- a/cmd/frostfs-cli/modules/object/get.go +++ b/cmd/frostfs-cli/modules/object/get.go @@ -90,7 +90,7 @@ func getObject(cmd *cobra.Command, _ []string) { }) } - res, err := internalclient.GetObject(prm) + res, err := internalclient.GetObject(cmd.Context(), prm) if p != nil { p.Finish() } diff --git a/cmd/frostfs-cli/modules/object/hash.go b/cmd/frostfs-cli/modules/object/hash.go index c7d734e67..26243e7e7 100644 --- a/cmd/frostfs-cli/modules/object/hash.go +++ b/cmd/frostfs-cli/modules/object/hash.go @@ -75,7 +75,7 @@ func getObjectHash(cmd *cobra.Command, _ []string) { headPrm.SetAddress(objAddr) // get hash of full payload through HEAD (may be user can do it through dedicated command?) - res, err := internalclient.HeadObject(headPrm) + res, err := internalclient.HeadObject(cmd.Context(), headPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) var cs checksum.Checksum @@ -108,7 +108,7 @@ func getObjectHash(cmd *cobra.Command, _ []string) { hashPrm.TZ() } - res, err := internalclient.HashPayloadRanges(hashPrm) + res, err := internalclient.HashPayloadRanges(cmd.Context(), hashPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) hs := res.HashList() diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index 139563e24..6fce04490 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -64,7 +64,7 @@ func getObjectHeader(cmd *cobra.Command, _ []string) { prm.SetAddress(objAddr) prm.SetMainOnlyFlag(mainOnly) - res, err := internalclient.HeadObject(prm) + res, err := internalclient.HeadObject(cmd.Context(), prm) if err != nil { if ok := printSplitInfoErr(cmd, err); ok { return diff --git a/cmd/frostfs-cli/modules/object/lock.go b/cmd/frostfs-cli/modules/object/lock.go index e6fbedd2c..fa1898586 100644 --- a/cmd/frostfs-cli/modules/object/lock.go +++ b/cmd/frostfs-cli/modules/object/lock.go @@ -104,7 +104,7 @@ var objectLockCmd = &cobra.Command{ Prepare(cmd, &prm) prm.SetHeader(obj) - res, err := internalclient.PutObject(prm) + res, err := internalclient.PutObject(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "Store lock object in FrostFS: %w", err) cmd.Printf("Lock object ID: %s\n", res.ID()) diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index 6fd91ca4b..af9e9fd0b 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -131,7 +131,7 @@ func putObject(cmd *cobra.Command, _ []string) { prm.SetCopiesNumberByVectors(cn) } - res, err := internalclient.PutObject(prm) + res, err := internalclient.PutObject(cmd.Context(), prm) if p != nil { p.Finish() } diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index a594204f0..76425d948 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -87,7 +87,7 @@ func getObjectRange(cmd *cobra.Command, _ []string) { prm.SetRange(ranges[0]) prm.SetPayloadWriter(out) - _, err = internalclient.PayloadRange(prm) + _, err = internalclient.PayloadRange(cmd.Context(), prm) if err != nil { if ok := printSplitInfoErr(cmd, err); ok { return diff --git a/cmd/frostfs-cli/modules/object/search.go b/cmd/frostfs-cli/modules/object/search.go index de4a8a3b4..b603e5fe8 100644 --- a/cmd/frostfs-cli/modules/object/search.go +++ b/cmd/frostfs-cli/modules/object/search.go @@ -61,7 +61,7 @@ func searchObject(cmd *cobra.Command, _ []string) { prm.SetContainerID(cnr) prm.SetFilters(sf) - res, err := internalclient.SearchObjects(prm) + res, err := internalclient.SearchObjects(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) ids := res.IDList() diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index 204409df3..094b62314 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -278,7 +278,7 @@ func OpenSessionViaClient(cmd *cobra.Command, dst SessionPrm, cli *client.Client common.PrintVerbose(cmd, "Opening remote session with the node...") - err := sessionCli.CreateSession(&tok, cli, sessionLifetime) + err := sessionCli.CreateSession(cmd.Context(), &tok, cli, sessionLifetime) commonCmd.ExitOnErr(cmd, "open remote session: %w", err) common.PrintVerbose(cmd, "Session successfully opened.") @@ -354,7 +354,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, Prepare(cmd, &prmHead) - _, err := internal.HeadObject(prmHead) + _, err := internal.HeadObject(cmd.Context(), prmHead) var errSplit *object.SplitInfoError @@ -396,7 +396,7 @@ func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *object.Spl prmHead.SetRawFlag(false) // client is already set - res, err := internal.HeadObject(prmHead) + res, err := internal.HeadObject(cmd.Context(), prmHead) if err == nil { children := res.Header().Children() @@ -425,7 +425,7 @@ func tryGetSplitMembersBySplitID(cmd *cobra.Command, splitInfo *object.SplitInfo prm.SetClient(cli) prm.SetFilters(query) - res, err := internal.SearchObjects(prm) + res, err := internal.SearchObjects(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "failed to search objects by split ID: %w", err) parts := res.IDList() @@ -463,7 +463,7 @@ func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *object.SplitInfo, p addrObj.SetObject(idMember) prmHead.SetAddress(addrObj) - res, err = internal.HeadObject(prmHead) + res, err = internal.HeadObject(cmd.Context(), prmHead) commonCmd.ExitOnErr(cmd, "failed to read split chain member's header: %w", err) idMember, ok = res.Header().PreviousID() @@ -490,7 +490,7 @@ func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *object.SplitInfo, p prmSearch.SetContainerID(cnr) prmSearch.SetFilters(query) - resSearch, err := internal.SearchObjects(prmSearch) + resSearch, err := internal.SearchObjects(cmd.Context(), prmSearch) commonCmd.ExitOnErr(cmd, "failed to find object children: %w", err) list := resSearch.IDList() diff --git a/cmd/frostfs-cli/modules/session/create.go b/cmd/frostfs-cli/modules/session/create.go index 341681f5b..0cc12606e 100644 --- a/cmd/frostfs-cli/modules/session/create.go +++ b/cmd/frostfs-cli/modules/session/create.go @@ -1,6 +1,7 @@ package session import ( + "context" "fmt" "os" @@ -64,7 +65,7 @@ func createSession(cmd *cobra.Command, _ []string) { var tok session.Object - err = CreateSession(&tok, c, lifetime) + err = CreateSession(cmd.Context(), &tok, c, lifetime) commonCmd.ExitOnErr(cmd, "can't create session: %w", err) var data []byte @@ -86,11 +87,11 @@ func createSession(cmd *cobra.Command, _ []string) { // number of epochs. // // Fills ID, lifetime and session key. -func CreateSession(dst *session.Object, c *client.Client, lifetime uint64) error { +func CreateSession(ctx context.Context, dst *session.Object, c *client.Client, lifetime uint64) error { var netInfoPrm internalclient.NetworkInfoPrm netInfoPrm.SetClient(c) - ni, err := internalclient.NetworkInfo(netInfoPrm) + ni, err := internalclient.NetworkInfo(ctx, netInfoPrm) if err != nil { return fmt.Errorf("can't fetch network info: %w", err) } @@ -102,7 +103,7 @@ func CreateSession(dst *session.Object, c *client.Client, lifetime uint64) error sessionPrm.SetClient(c) sessionPrm.SetExp(exp) - sessionRes, err := internalclient.CreateSession(sessionPrm) + sessionRes, err := internalclient.CreateSession(ctx, sessionPrm) if err != nil { return fmt.Errorf("can't open session: %w", err) } From 96c98435913a8208c323e81c7e4ac7f9e9601868 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 1 Jun 2023 16:30:54 +0300 Subject: [PATCH 0304/1943] [#406] cmd/common: Execute PersistentPostRun() on errors Signed-off-by: Evgenii Stratonikov --- cmd/internal/common/exit.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/internal/common/exit.go b/cmd/internal/common/exit.go index 9b912ddc0..d38f27e02 100644 --- a/cmd/internal/common/exit.go +++ b/cmd/internal/common/exit.go @@ -46,5 +46,6 @@ func ExitOnErr(cmd *cobra.Command, errFmt string, err error) { } cmd.PrintErrln(err) + cmd.PersistentPostRun(cmd, nil) os.Exit(code) } From f1f56ef8712cfe37af377c86e491d3388ec7500e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 24 May 2023 14:50:19 +0300 Subject: [PATCH 0305/1943] [#406] cli: Add --trace flag Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/client/sdk.go | 5 ++ cmd/frostfs-cli/internal/common/tracing.go | 64 +++++++++++++++++++ cmd/frostfs-cli/internal/commonflags/flags.go | 5 ++ cmd/frostfs-cli/modules/accounting/root.go | 3 + cmd/frostfs-cli/modules/container/create.go | 1 + cmd/frostfs-cli/modules/container/delete.go | 1 + cmd/frostfs-cli/modules/container/root.go | 3 + cmd/frostfs-cli/modules/netmap/root.go | 3 + cmd/frostfs-cli/modules/object/root.go | 3 + cmd/frostfs-cli/modules/session/create.go | 5 +- 10 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-cli/internal/common/tracing.go diff --git a/cmd/frostfs-cli/internal/client/sdk.go b/cmd/frostfs-cli/internal/client/sdk.go index 13dacc04c..79d3dcb0d 100644 --- a/cmd/frostfs-cli/internal/client/sdk.go +++ b/cmd/frostfs-cli/internal/client/sdk.go @@ -12,9 +12,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" "github.com/spf13/viper" + "google.golang.org/grpc" ) var errInvalidEndpoint = errors.New("provided RPC endpoint is incorrect") @@ -59,6 +61,9 @@ func GetSDKClient(ctx context.Context, cmd *cobra.Command, key *ecdsa.PrivateKey common.PrintVerbose(cmd, "Set request timeout to %s.", timeout) } + prmDial.SetGRPCDialOptions( + grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInteceptor()), + grpc.WithChainStreamInterceptor(tracing.NewStreamClientInterceptor())) c.Init(prmInit) diff --git a/cmd/frostfs-cli/internal/common/tracing.go b/cmd/frostfs-cli/internal/common/tracing.go new file mode 100644 index 000000000..7a9684667 --- /dev/null +++ b/cmd/frostfs-cli/internal/common/tracing.go @@ -0,0 +1,64 @@ +package common + +import ( + "context" + "fmt" + "sort" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/misc" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "github.com/spf13/cobra" + "go.opentelemetry.io/otel/trace" +) + +type spanKey struct{} + +// StopClientCommandSpan stops tracing span for the command and prints trace ID on the standard output. +func StopClientCommandSpan(cmd *cobra.Command, _ []string) { + span, ok := cmd.Context().Value(spanKey{}).(trace.Span) + if !ok { + return + } + + span.End() + + // Noop provider cannot fail on flush. + _ = tracing.Shutdown(cmd.Context()) + + cmd.PrintErrf("Trace ID: %s\n", span.SpanContext().TraceID()) +} + +// StartClientCommandSpan starts tracing span for the command. +func StartClientCommandSpan(cmd *cobra.Command) { + enableTracing, err := cmd.Flags().GetBool(commonflags.TracingFlag) + if err != nil || !enableTracing { + return + } + + _, err = tracing.Setup(cmd.Context(), tracing.Config{ + Enabled: true, + Exporter: tracing.NoOpExporter, + Service: "frostfs-cli", + Version: misc.Version, + }) + commonCmd.ExitOnErr(cmd, "init tracing: %w", err) + + var components sort.StringSlice + for c := cmd; c != nil; c = c.Parent() { + fmt.Println(c.Name()) + components = append(components, c.Name()) + } + for i, j := 0, len(components)-1; i < j; { + components.Swap(i, j) + i++ + j-- + } + + operation := strings.Join(components, ".") + ctx, span := tracing.StartSpanFromContext(cmd.Context(), operation) + ctx = context.WithValue(ctx, spanKey{}, span) + cmd.SetContext(ctx) +} diff --git a/cmd/frostfs-cli/internal/commonflags/flags.go b/cmd/frostfs-cli/internal/commonflags/flags.go index 810e62107..5049dc3b1 100644 --- a/cmd/frostfs-cli/internal/commonflags/flags.go +++ b/cmd/frostfs-cli/internal/commonflags/flags.go @@ -47,6 +47,9 @@ const ( OIDFlag = "oid" OIDFlagUsage = "Object ID." + + TracingFlag = "trace" + TracingFlagUsage = "Generate trace ID and print it." ) // Init adds common flags to the command: @@ -54,12 +57,14 @@ const ( // - WalletPath, // - Account, // - RPC, +// - Tracing, // - Timeout. func Init(cmd *cobra.Command) { InitWithoutRPC(cmd) ff := cmd.Flags() ff.StringP(RPC, RPCShorthand, RPCDefault, RPCUsage) + ff.Bool(TracingFlag, false, TracingFlagUsage) ff.DurationP(Timeout, TimeoutShorthand, TimeoutDefault, TimeoutUsage) } diff --git a/cmd/frostfs-cli/modules/accounting/root.go b/cmd/frostfs-cli/modules/accounting/root.go index 8ab8aa125..f94488b6f 100644 --- a/cmd/frostfs-cli/modules/accounting/root.go +++ b/cmd/frostfs-cli/modules/accounting/root.go @@ -1,6 +1,7 @@ package accounting import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -17,7 +18,9 @@ var Cmd = &cobra.Command{ _ = viper.BindPFlag(commonflags.WalletPath, flags.Lookup(commonflags.WalletPath)) _ = viper.BindPFlag(commonflags.Account, flags.Lookup(commonflags.Account)) _ = viper.BindPFlag(commonflags.RPC, flags.Lookup(commonflags.RPC)) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 2acdfc16f..e5bbeacd4 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -141,6 +141,7 @@ func initContainerCreateCmd() { // Init common flags flags.StringP(commonflags.RPC, commonflags.RPCShorthand, commonflags.RPCDefault, commonflags.RPCUsage) + flags.Bool(commonflags.TracingFlag, false, commonflags.TracingFlagUsage) flags.DurationP(commonflags.Timeout, commonflags.TimeoutShorthand, commonflags.TimeoutDefault, commonflags.TimeoutUsage) flags.StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, commonflags.WalletPathDefault, commonflags.WalletPathUsage) flags.StringP(commonflags.Account, commonflags.AccountShorthand, commonflags.AccountDefault, commonflags.AccountUsage) diff --git a/cmd/frostfs-cli/modules/container/delete.go b/cmd/frostfs-cli/modules/container/delete.go index 54b49a32d..f5b69edfd 100644 --- a/cmd/frostfs-cli/modules/container/delete.go +++ b/cmd/frostfs-cli/modules/container/delete.go @@ -124,6 +124,7 @@ func initContainerDeleteCmd() { flags.StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, commonflags.WalletPathDefault, commonflags.WalletPathUsage) flags.StringP(commonflags.Account, commonflags.AccountShorthand, commonflags.AccountDefault, commonflags.AccountUsage) flags.StringP(commonflags.RPC, commonflags.RPCShorthand, commonflags.RPCDefault, commonflags.RPCUsage) + flags.Bool(commonflags.TracingFlag, false, commonflags.TracingFlagUsage) flags.StringVar(&containerID, commonflags.CIDFlag, "", commonflags.CIDFlagUsage) flags.BoolVar(&containerAwait, "await", false, "Block execution until container is removed") diff --git a/cmd/frostfs-cli/modules/container/root.go b/cmd/frostfs-cli/modules/container/root.go index ab7f5fb90..f3c3e0e3a 100644 --- a/cmd/frostfs-cli/modules/container/root.go +++ b/cmd/frostfs-cli/modules/container/root.go @@ -1,6 +1,7 @@ package container import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -15,7 +16,9 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/netmap/root.go b/cmd/frostfs-cli/modules/netmap/root.go index aaa83f12f..006ac6d9f 100644 --- a/cmd/frostfs-cli/modules/netmap/root.go +++ b/cmd/frostfs-cli/modules/netmap/root.go @@ -1,6 +1,7 @@ package netmap import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -14,7 +15,9 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/object/root.go b/cmd/frostfs-cli/modules/object/root.go index 886153075..23badc576 100644 --- a/cmd/frostfs-cli/modules/object/root.go +++ b/cmd/frostfs-cli/modules/object/root.go @@ -1,6 +1,7 @@ package object import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -15,7 +16,9 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/session/create.go b/cmd/frostfs-cli/modules/session/create.go index 0cc12606e..53f6e8bc4 100644 --- a/cmd/frostfs-cli/modules/session/create.go +++ b/cmd/frostfs-cli/modules/session/create.go @@ -6,6 +6,7 @@ import ( "os" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -29,10 +30,12 @@ var createCmd = &cobra.Command{ Use: "create", Short: "Create session token", Run: createSession, - PersistentPreRun: func(cmd *cobra.Command, _ []string) { + PersistentPreRun: func(cmd *cobra.Command, args []string) { _ = viper.BindPFlag(commonflags.WalletPath, cmd.Flags().Lookup(commonflags.WalletPath)) _ = viper.BindPFlag(commonflags.Account, cmd.Flags().Lookup(commonflags.Account)) + common.StartClientCommandSpan(cmd) }, + PersistentPostRun: common.StopClientCommandSpan, } func init() { From f8c1e0639d93af814793580dc9f50c0f1dd3a321 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Jun 2023 14:45:30 +0300 Subject: [PATCH 0306/1943] [#422] cli: Provide context to NetmapSnapshot() Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/container/policy_playground.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index fa10dc10e..65255f9c2 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -121,7 +121,7 @@ func (repl *policyPlaygroundREPL) run() error { var prm internalclient.NetMapSnapshotPrm prm.SetClient(cli) - resp, err := internalclient.NetMapSnapshot(prm) + resp, err := internalclient.NetMapSnapshot(repl.cmd.Context(), prm) commonCmd.ExitOnErr(repl.cmd, "unable to get netmap snapshot to populate initial netmap: %w", err) for _, node := range resp.NetMap().Nodes() { From a770b89fd8fe726078fa9f20d5a35e22f844b2b5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Jun 2023 14:48:07 +0300 Subject: [PATCH 0307/1943] [#422] adm: Fix ValidatorCount problems after neo-go update Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/initialize_test.go | 2 +- cmd/frostfs-adm/internal/modules/morph/local_client.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index 07d2da8cc..30a7168dd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -119,7 +119,7 @@ func generateTestData(t *testing.T, dir string, size int) error { cfg := config.Config{} cfg.ProtocolConfiguration.Magic = 12345 - cfg.ProtocolConfiguration.ValidatorsCount = size + cfg.ProtocolConfiguration.ValidatorsCount = uint32(size) cfg.ProtocolConfiguration.TimePerBlock = time.Second cfg.ProtocolConfiguration.StandbyCommittee = pubs // sorted by glagolic letters cfg.ProtocolConfiguration.P2PSigExtensions = true diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index 816f9da4c..45d09c387 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -62,7 +62,7 @@ func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet return nil, err } - m := smartcontract.GetDefaultHonestNodeCount(cfg.ProtocolConfiguration.ValidatorsCount) + m := smartcontract.GetDefaultHonestNodeCount(int(cfg.ProtocolConfiguration.ValidatorsCount)) accounts := make([]*wallet.Account, len(wallets)) for i := range accounts { accounts[i], err = getWalletAccount(wallets[i], consensusAccountName) From 189a367ef24304260372e6201038ad2e06bdb049 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 5 Jun 2023 18:54:59 +0300 Subject: [PATCH 0308/1943] [#390] frostfs-cli: Pass bearer token to Tree srv * Add --bearer flag for "tree" subcommand Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-cli/modules/tree/add.go | 3 ++- cmd/frostfs-cli/modules/tree/add_by_path.go | 2 +- cmd/frostfs-cli/modules/tree/get_by_path.go | 2 +- cmd/frostfs-cli/modules/tree/root.go | 4 ++++ cmd/internal/common/exit.go | 4 +++- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index 707a4d8ee..ea9245271 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -60,7 +61,7 @@ func add(cmd *cobra.Command, _ []string) { TreeId: tid, ParentId: pid, Meta: meta, - BearerToken: nil, // TODO: #1891 add token handling + BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), } commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index e83408a89..6b182e1e4 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -72,7 +72,7 @@ func addByPath(cmd *cobra.Command, _ []string) { // PathAttribute: pAttr, Path: strings.Split(path, "/"), Meta: meta, - BearerToken: nil, // TODO: #1891 add token handling + BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), } commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index 75acbaedf..6f6fc7ff9 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -71,7 +71,7 @@ func getByPath(cmd *cobra.Command, _ []string) { Path: strings.Split(path, "/"), LatestOnly: latestOnly, AllAttributes: true, - BearerToken: nil, // TODO: #1891 add token handling + BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), } commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) diff --git a/cmd/frostfs-cli/modules/tree/root.go b/cmd/frostfs-cli/modules/tree/root.go index c70e6f5e2..b5de0dd6f 100644 --- a/cmd/frostfs-cli/modules/tree/root.go +++ b/cmd/frostfs-cli/modules/tree/root.go @@ -32,6 +32,8 @@ const ( pathAttributeFlagKey = "pattr" latestOnlyFlagKey = "latest" + + bearerFlagKey = "bearer" ) func initCTID(cmd *cobra.Command) { @@ -42,4 +44,6 @@ func initCTID(cmd *cobra.Command) { ff.String(treeIDFlagKey, "", "Tree ID") _ = cmd.MarkFlagRequired(treeIDFlagKey) + + ff.StringP(bearerFlagKey, "", "", "Path to bearer token") } diff --git a/cmd/internal/common/exit.go b/cmd/internal/common/exit.go index d38f27e02..9e4fa3098 100644 --- a/cmd/internal/common/exit.go +++ b/cmd/internal/common/exit.go @@ -46,6 +46,8 @@ func ExitOnErr(cmd *cobra.Command, errFmt string, err error) { } cmd.PrintErrln(err) - cmd.PersistentPostRun(cmd, nil) + if cmd.PersistentPostRun != nil { + cmd.PersistentPostRun(cmd, nil) + } os.Exit(code) } From 263c6fdc50693e5df0b453c593e2a9a7808a7909 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Jun 2023 17:28:04 +0300 Subject: [PATCH 0309/1943] [#372] node: Add metrics for the error counter in the engine Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/engine/engine.go | 2 ++ pkg/local_object_storage/engine/metrics.go | 3 +++ pkg/local_object_storage/engine/shards.go | 15 ++++++++++++ .../shard/metrics_test.go | 13 ++++++++++ pkg/local_object_storage/shard/shard.go | 24 +++++++++++++++++++ pkg/metrics/engine.go | 14 +++++++++++ 6 files changed, 71 insertions(+) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 21e863005..7c36811e7 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -132,6 +132,7 @@ func (e *StorageEngine) reportShardErrorBackground(id string, msg string, err er } errCount := sh.errorCount.Add(1) + sh.Shard.IncErrorCounter() e.reportShardErrorWithFlags(sh.Shard, errCount, false, msg, err) } @@ -150,6 +151,7 @@ func (e *StorageEngine) reportShardError( } errCount := sh.errorCount.Add(1) + sh.Shard.IncErrorCounter() e.reportShardErrorWithFlags(sh.Shard, errCount, true, msg, err, fields...) } diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 1be888eae..f9e9191cd 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -26,6 +26,9 @@ type MetricRegister interface { AddToContainerSize(cnrID string, size int64) AddToPayloadCounter(shardID string, size int64) + IncErrorCounter(shardID string) + ClearErrorCounter(shardID string) + DeleteErrorCounter(shardID string) WriteCache() metrics.WriteCacheMetrics GC() metrics.GCMetrics diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 07d22d3fe..c4c356a75 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -62,6 +62,18 @@ func (m *metricsWithID) AddToPayloadSize(size int64) { m.mw.AddToPayloadCounter(m.id, size) } +func (m *metricsWithID) IncErrorCounter() { + m.mw.IncErrorCounter(m.id) +} + +func (m *metricsWithID) ClearErrorCounter() { + m.mw.ClearErrorCounter(m.id) +} + +func (m *metricsWithID) DeleteErrorCounter() { + m.mw.DeleteErrorCounter(m.id) +} + // AddShard adds a new shard to the storage engine. // // Returns any error encountered that did not allow adding a shard. @@ -174,6 +186,8 @@ func (e *StorageEngine) removeShards(ids ...string) { continue } + sh.DeleteErrorCounter() + ss = append(ss, sh) delete(e.shards, id) @@ -281,6 +295,7 @@ func (e *StorageEngine) SetShardMode(id *shard.ID, m mode.Mode, resetErrorCounte if id.String() == shID { if resetErrorCounter { sh.errorCount.Store(0) + sh.Shard.ClearErrorCounter() } return sh.SetMode(m) } diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 16f6989c4..f1581b6d4 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -23,6 +23,7 @@ type metricsStore struct { cnrSize map[string]int64 pldSize int64 readOnly bool + errCounter int64 } func (m metricsStore) SetShardID(_ string) {} @@ -68,6 +69,18 @@ func (m *metricsStore) AddToPayloadSize(size int64) { m.pldSize += size } +func (m *metricsStore) IncErrorCounter() { + m.errCounter += 1 +} + +func (m *metricsStore) ClearErrorCounter() { + m.errCounter = 0 +} + +func (m *metricsStore) DeleteErrorCounter() { + m.errCounter = 0 +} + const physical = "phy" const logical = "logic" diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 2123bca1f..b740fc572 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -75,6 +75,12 @@ type MetricsWriter interface { SetShardID(id string) // SetReadonly must set shard readonly state. SetReadonly(readonly bool) + // IncErrorCounter increment error counter. + IncErrorCounter() + // ClearErrorCounter clear error counter. + ClearErrorCounter() + // DeleteErrorCounter delete error counter. + DeleteErrorCounter() } type cfg struct { @@ -428,3 +434,21 @@ func (s *Shard) addToPayloadSize(size int64) { s.cfg.metricsWriter.AddToPayloadSize(size) } } + +func (s *Shard) IncErrorCounter() { + if s.cfg.metricsWriter != nil { + s.cfg.metricsWriter.IncErrorCounter() + } +} + +func (s *Shard) ClearErrorCounter() { + if s.cfg.metricsWriter != nil { + s.cfg.metricsWriter.ClearErrorCounter() + } +} + +func (s *Shard) DeleteErrorCounter() { + if s.cfg.metricsWriter != nil { + s.cfg.metricsWriter.DeleteErrorCounter() + } +} diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 4e78f4ac2..7992da9f8 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -24,6 +24,7 @@ type ( listObjectsDuration prometheus.Counter containerSize *prometheus.GaugeVec payloadSize *prometheus.GaugeVec + errorCounter *prometheus.GaugeVec } ) @@ -44,6 +45,7 @@ func newEngineMetrics() engineMetrics { listObjectsDuration: newEngineMethodDurationCounter("list_objects"), containerSize: newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), payloadSize: newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), + errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}), } } @@ -123,3 +125,15 @@ func (m engineMetrics) AddToContainerSize(cnrID string, size int64) { func (m engineMetrics) AddToPayloadCounter(shardID string, size int64) { m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) } + +func (m engineMetrics) IncErrorCounter(shardID string) { + m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Inc() +} + +func (m engineMetrics) ClearErrorCounter(shardID string) { + m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0) +} + +func (m engineMetrics) DeleteErrorCounter(shardID string) { + m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID}) +} From 5b75432ca2a74dc518f93bbc5655dbb5d1e44eea Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 09:35:19 +0300 Subject: [PATCH 0310/1943] [#431] metrics: Add missed label Signed-off-by: Dmitrii Stepanov --- pkg/metrics/writecache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 3c56aa2ba..a20c3ca6a 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -176,7 +176,7 @@ func newWCMethodDurationCounter(method string) *prometheus.HistogramVec { Subsystem: wcSubsystem, Name: fmt.Sprintf("%s_req_duration_seconds", method), Help: fmt.Sprintf("Accumulated %s request process duration", method), - }, []string{wcShardID, wcSuccess}) + }, []string{wcShardID, wcSuccess, wcStorage}) } func newWCOperationCounterVec(operation string, labels []string) *prometheus.CounterVec { From 508e2064ebfb2e3b7dce2921a7f0791f2757129c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Jun 2023 14:08:28 +0300 Subject: [PATCH 0311/1943] [#438] cli: Drop tracing debug print Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/internal/common/tracing.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/frostfs-cli/internal/common/tracing.go b/cmd/frostfs-cli/internal/common/tracing.go index 7a9684667..30c2f2b1a 100644 --- a/cmd/frostfs-cli/internal/common/tracing.go +++ b/cmd/frostfs-cli/internal/common/tracing.go @@ -2,7 +2,6 @@ package common import ( "context" - "fmt" "sort" "strings" @@ -48,7 +47,6 @@ func StartClientCommandSpan(cmd *cobra.Command) { var components sort.StringSlice for c := cmd; c != nil; c = c.Parent() { - fmt.Println(c.Name()) components = append(components, c.Name()) } for i, j := 0, len(components)-1; i < j; { From 9bc1a25c0766ca04ef0d02161bcafebe7a403f67 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 17:10:25 +0300 Subject: [PATCH 0312/1943] [#434] metrics: Fix writecache duration Signed-off-by: Dmitrii Stepanov --- pkg/metrics/writecache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index a20c3ca6a..67b9d8caf 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -167,7 +167,7 @@ func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success b wcSuccess: fmt.Sprintf("%v", success), wcStorage: storageType, }, - ).Observe(float64(d)) + ).Observe(d.Seconds()) } func newWCMethodDurationCounter(method string) *prometheus.HistogramVec { From cd92d8a9e74556b53060fcf097bfd87ab96707bc Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Jun 2023 15:36:05 +0300 Subject: [PATCH 0313/1943] [#423] go.mod: Update sdk-go and hrw Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 1 + go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83ec8e41a..6afb8f700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ Changelog for FrostFS Node - `go.opentelemetry.io/otel` to `v1.15.1` - `go.opentelemetry.io/otel/trace` to `v1.15.1` - `github.com/spf13/cast` to `v1.5.1` +- `git.frostfs.info/TrueCloudLab/hrw` to `v1.2.1` ### Updating from v0.36.0 diff --git a/go.mod b/go.mod index d5b3826b2..8bb31686e 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230530135122-10482ffbed3b - git.frostfs.info/TrueCloudLab/hrw v1.2.0 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230602115440-ec59ebfd8826 + git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 @@ -44,7 +44,7 @@ require ( require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect - github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect @@ -98,7 +98,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect - github.com/twmb/murmur3 v1.1.7 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/urfave/cli v1.22.13 // indirect go.mongodb.org/mongo-driver v1.11.6 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect diff --git a/go.sum b/go.sum index 242614781..c64597f65 100644 --- a/go.sum +++ b/go.sum @@ -44,10 +44,10 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230530135122-10482ffbed3b h1:zxrMDBKALGeEI0SuxNxRlGW/JpZb/2RWywU78L5lLis= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230530135122-10482ffbed3b/go.mod h1:0n2gQYkqCgiXkDzk0RLUUxoF2xX79Ke85bAiGV8cBl8= -git.frostfs.info/TrueCloudLab/hrw v1.2.0 h1:KvAES7xIqmQBGd2q8KanNosD9+4BhU/zqD5Kt5KSflk= -git.frostfs.info/TrueCloudLab/hrw v1.2.0/go.mod h1:mq2sbvYfO+BB6iFZwYBkgC0yc6mJNx+qZi4jW918m+Y= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230602115440-ec59ebfd8826 h1:dIk/pUsCME/nbhDdGKe1n7XOdrfvLCCTzJbnuI5zkOk= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230602115440-ec59ebfd8826/go.mod h1:xpCauXjq+v7mgbjr+2Ebs6/Xx8gxKxXHFfHEZUH3N4E= +git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= +git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= @@ -74,8 +74,8 @@ github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGn github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -529,8 +529,8 @@ github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKk github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/twmb/murmur3 v1.1.7 h1:ULWBiM04n/XoN3YMSJ6Z2pHDFLf+MeIVQU71ZPrvbWg= -github.com/twmb/murmur3 v1.1.7/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.13 h1:wsLILXG8qCJNse/qAgLNf23737Cx05GflHg/PJGe1Ok= From 41ab4d070e6b700491ae741cf4d9ffbb0845105f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Jun 2023 15:39:16 +0300 Subject: [PATCH 0314/1943] [#423] *: Use hrw.StringHash() where possible Signed-off-by: Evgenii Stratonikov --- .../blobstor/blobovniczatree/blobovnicza.go | 2 +- pkg/local_object_storage/engine/engine_test.go | 2 +- pkg/local_object_storage/engine/evacuate.go | 2 +- pkg/local_object_storage/engine/remove_copies.go | 2 +- pkg/local_object_storage/engine/shards.go | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index af976f977..d99e9aa89 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -216,7 +216,7 @@ func addressHash(addr *oid.Address, path string) uint64 { a = addr.EncodeToString() } - return hrw.Hash([]byte(a + path)) + return hrw.StringHash(a + path) } // converts uint64 to hex string. diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 9c239739f..11c0848fe 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -101,7 +101,7 @@ func (te *testEngineWrapper) setInitializedShards(t testing.TB, shards ...*shard errorCount: new(atomic.Uint32), Shard: s, }, - hash: hrw.Hash([]byte(s.ID().String())), + hash: hrw.StringHash(s.ID().String()), } te.engine.shardPools[s.ID().String()] = pool te.shardIDs = append(te.shardIDs, s.ID()) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index c103ede73..73b7a7830 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -373,7 +373,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) (bool, error) { - hrw.SortHasherSliceByWeightValue(shards, weights, hrw.Hash([]byte(addr.EncodeToString()))) + hrw.SortHasherSliceByWeightValue(shards, weights, hrw.StringHash(addr.EncodeToString())) for j := range shards { select { case <-ctx.Done(): diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index 1ea569928..7681e0e50 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -110,7 +110,7 @@ func (e *StorageEngine) removeObjects(ctx context.Context, ch <-chan oid.Address } for addr := range ch { - h := hrw.Hash([]byte(addr.EncodeToString())) + h := hrw.StringHash(addr.EncodeToString()) shards := sortShardsByWeight(shards, h) found := false for i := range shards { diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index c4c356a75..fd9605e9b 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -162,7 +162,7 @@ func (e *StorageEngine) addShard(sh *shard.Shard) error { errorCount: new(atomic.Uint32), Shard: sh, }, - hash: hrw.Hash([]byte(strID)), + hash: hrw.StringHash(strID), } e.shardPools[strID] = pool @@ -237,7 +237,7 @@ func (e *StorageEngine) sortShardsByWeight(objAddr interface{ EncodeToString() s e.mtx.RLock() defer e.mtx.RUnlock() - h := hrw.Hash([]byte(objAddr.EncodeToString())) + h := hrw.StringHash(objAddr.EncodeToString()) shards := make([]hashedShard, 0, len(e.shards)) for _, sh := range e.shards { shards = append(shards, hashedShard(sh)) From 0400153b7d9974b80bb1d57f4f4ed243ffb6d4cd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 2 Jun 2023 17:33:51 +0300 Subject: [PATCH 0315/1943] [#424] object: Do not store large slices in pool Dynamically growing an unbounded buffers can cause a large amount of memory to be pinned and never be freed. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/pool.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/services/object/put/pool.go b/pkg/services/object/put/pool.go index 5726856e5..ebe214caf 100644 --- a/pkg/services/object/put/pool.go +++ b/pkg/services/object/put/pool.go @@ -4,7 +4,10 @@ import ( "sync" ) -const defaultAllocSize = 1024 +const ( + defaultAllocSize = 1024 + poolSliceMaxSize = 128 * 1024 +) type payload struct { Data []byte @@ -19,6 +22,9 @@ func getPayload() *payload { } func putPayload(p *payload) { + if cap(p.Data) > poolSliceMaxSize { + return + } p.Data = p.Data[:0] putBytesPool.Put(p) } From 83d600ed77cbabd9db925e025b3eaae5de92b46a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Jun 2023 18:08:32 +0300 Subject: [PATCH 0316/1943] [#424] node: Update api-go version Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8bb31686e..0fdcb4409 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230602115440-ec59ebfd8826 diff --git a/go.sum b/go.sum index c64597f65..5da5a9a9e 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac h1:a6/Zc5BejflmguShwbllgJdEehnM9gshkLrLbKQHCU0= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230531114046-62edd68f47ac/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb h1:gaq4wgk7b+i9+rkDa+goPkv5FzqYlMxEuCKdZ57mSc0= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= From 0c40d98f7a7ad6f39755c4c0a7e298e419d2aafd Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 9 Jun 2023 11:47:39 +0300 Subject: [PATCH 0317/1943] [#375] Add log metrics Signed-off-by: Alejandro Lopez --- cmd/frostfs-ir/main.go | 1 + cmd/frostfs-node/config.go | 2 ++ pkg/util/logger/logger.go | 8 ++++++++ pkg/util/logger/metrics.go | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+) create mode 100644 pkg/util/logger/metrics.go diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 1718a46ab..81fcaa489 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -61,6 +61,7 @@ func main() { cfg, err = newConfig() exitErr(err) + logPrm.MetricsNamespace = "frostfs-ir" err = logPrm.SetLevelString( cfg.GetString("logger.level"), ) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index f8605c21e..e4547364d 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -542,6 +542,8 @@ func initCfg(appCfg *config.Config) *cfg { logPrm, err := c.loggerPrm() fatalOnErr(err) + logPrm.MetricsNamespace = "frostfs-node" + log, err := logger.NewLogger(logPrm) fatalOnErr(err) diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index 4a536368a..fcac09321 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -31,6 +31,9 @@ type Prm struct { // support runtime rereading level zapcore.Level + // MetricsNamespace is the namespace string used for log counter metrics + MetricsNamespace string + // do not support runtime rereading } @@ -79,10 +82,15 @@ func NewLogger(prm *Prm) (*Logger, error) { lvl := zap.NewAtomicLevelAt(prm.level) + m := newLogMetrics(prm.MetricsNamespace) + c := zap.NewProductionConfig() c.Level = lvl c.Encoding = "console" c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + c.Sampling.Hook = func(e zapcore.Entry, sd zapcore.SamplingDecision) { + m.Inc(e.Level, sd == zapcore.LogDropped) + } lZap, err := c.Build( zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), diff --git a/pkg/util/logger/metrics.go b/pkg/util/logger/metrics.go new file mode 100644 index 000000000..10be4a8f0 --- /dev/null +++ b/pkg/util/logger/metrics.go @@ -0,0 +1,37 @@ +package logger + +import ( + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap/zapcore" +) + +const ( + logSubsystem = "log" + logLevelLabel = "level" + logDroppedLabel = "dropped" +) + +type logMetrics struct { + logCount *prometheus.CounterVec +} + +func newLogMetrics(namespace string) *logMetrics { + return &logMetrics{ + logCount: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: logSubsystem, + Name: "log_count", + Help: "Total log entries emitted or dropped by severity level", + }, []string{logLevelLabel, logDroppedLabel}), + } +} + +func (m *logMetrics) Inc(level zapcore.Level, dropped bool) { + m.logCount.With(prometheus.Labels{ + logLevelLabel: level.String(), + logDroppedLabel: strconv.FormatBool(dropped), + }).Inc() +} From 2541d319de64d6b111830993188bdd4b597e39d4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 11:26:59 +0300 Subject: [PATCH 0318/1943] [#266] pilorama: Allow to get current tree height Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/tree.go | 16 +++++++++ pkg/local_object_storage/pilorama/boltdb.go | 34 +++++++++++++++++++ pkg/local_object_storage/pilorama/forest.go | 9 +++++ .../pilorama/forest_test.go | 13 +++++-- .../pilorama/interface.go | 2 ++ pkg/local_object_storage/shard/tree.go | 16 +++++++++ 6 files changed, 88 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 6b8f83f31..08c6d26b0 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -311,6 +311,22 @@ func (e *StorageEngine) TreeExists(ctx context.Context, cid cidSDK.ID, treeID st return err == nil, err } +func (e *StorageEngine) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeHeight", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + index, lst, err := e.getTreeShard(ctx, cid, treeID) + if err != nil { + return 0, nil + } + return lst[index].TreeHeight(ctx, cid, treeID) +} + // TreeUpdateLastSyncHeight implements the pilorama.Forest interface. func (e *StorageEngine) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeUpdateLastSyncHeight", diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 9b62f0649..5b2c97f10 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -188,6 +188,40 @@ func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID strin }) } +func (t *boltForest) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeHeight", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + t.modeMtx.RLock() + defer t.modeMtx.RUnlock() + + if t.mode.NoMetabase() { + return 0, ErrDegradedMode + } + + var height uint64 + var retErr error + err := t.db.View(func(tx *bbolt.Tx) error { + treeRoot := tx.Bucket(bucketName(cid, treeID)) + if treeRoot != nil { + k, _ := treeRoot.Bucket(logBucket).Cursor().Last() + height = binary.BigEndian.Uint64(k) + } else { + retErr = ErrTreeNotFound + } + return nil + }) + if err == nil { + err = retErr + } + return height, err +} + // TreeExists implements the Forest interface. func (t *boltForest) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) { _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeExists", diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 672a38edd..7c5897ed0 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -214,6 +214,15 @@ func (f *memoryForest) TreeList(_ context.Context, cid cid.ID) ([]string, error) return res, nil } +func (f *memoryForest) TreeHeight(_ context.Context, cid cid.ID, treeID string) (uint64, error) { + fullID := cid.EncodeToString() + "/" + treeID + tree, ok := f.treeMap[fullID] + if !ok { + return 0, ErrTreeNotFound + } + return tree.operations[len(tree.operations)-1].Time, nil +} + // TreeExists implements the pilorama.Forest interface. func (f *memoryForest) TreeExists(_ context.Context, cid cid.ID, treeID string) (bool, error) { fullID := cid.EncodeToString() + "/" + treeID diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index ebb4667f5..9e8e98863 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -604,10 +604,19 @@ func testForestTreeExists(t *testing.T, constructor func(t testing.TB, opts ...O checkExists(t, false, cid, treeID) }) - require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &Move{Parent: 0, Child: 1}, false)) + require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &Move{Meta: Meta{Time: 11}, Parent: 0, Child: 1}, false)) checkExists(t, true, cid, treeID) + + height, err := s.TreeHeight(context.Background(), cid, treeID) + require.NoError(t, err) + require.EqualValues(t, 11, height) + checkExists(t, false, cidtest.ID(), treeID) // different CID, same tree - checkExists(t, false, cid, "another tree") // same CID, different tree + + _, err = s.TreeHeight(context.Background(), cidtest.ID(), treeID) + require.ErrorIs(t, err, ErrTreeNotFound) + + checkExists(t, false, cid, "another tree") // same CID, different tree t.Run("can be removed", func(t *testing.T) { require.NoError(t, s.TreeDrop(context.Background(), cid, treeID)) diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 9ca721be8..c8287c6d4 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -50,6 +50,8 @@ type Forest interface { TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error // TreeLastSyncHeight returns last log height synchronized with _all_ container nodes. TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) + // TreeHeight returns current tree height. + TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) } type ForestStorage interface { diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index ad89fa633..2331d37a8 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -255,6 +255,22 @@ func (s *Shard) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) { return s.pilorama.TreeList(ctx, cid) } +func (s *Shard) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeHeight", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + if s.pilorama == nil { + return 0, ErrPiloramaDisabled + } + return s.pilorama.TreeHeight(ctx, cid, treeID) +} + // TreeExists implements the pilorama.Forest interface. func (s *Shard) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeExists", From e69a1e84827a1839e6c1d63955985aeb24eee985 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 11:32:01 +0300 Subject: [PATCH 0319/1943] [#266] services/tree: Return operation log up to some height Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/service.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 96e547f36..4364095e2 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -532,9 +532,13 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) } h := b.GetHeight() + lastHeight, err := s.forest.TreeHeight(srv.Context(), cid, b.GetTreeId()) + if err != nil { + return err + } for { lm, err := s.forest.TreeGetOpLog(srv.Context(), cid, b.GetTreeId(), h) - if err != nil || lm.Time == 0 { + if err != nil || lm.Time == 0 || lastHeight < lm.Time { return err } From 957a43a12408fd9e4751d8b72d29bd08f4103ac0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 11:43:25 +0300 Subject: [PATCH 0320/1943] [#266] services/tree: Add sync check Do not accept requests until initial sync is finished. `Apply` is deliberately left out -- we don't want to miss anything new. Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/service.go | 39 ++++++++++++++++++++++++++++++++++++ pkg/services/tree/sync.go | 3 ++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 4364095e2..12d970c42 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -31,6 +32,8 @@ type Service struct { syncChan chan struct{} syncPool *ants.Pool + initialSyncDone atomic.Bool + // cnrMap contains existing (used) container IDs. cnrMap map[cidSDK.ID]struct{} // cnrMapMtx protects cnrMap @@ -90,6 +93,10 @@ func (s *Service) Shutdown() { } func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -138,6 +145,10 @@ func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error } func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByPathResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -198,6 +209,10 @@ func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByP } func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -247,6 +262,10 @@ func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveRespon // Move applies client operation to the specified tree and pushes in queue // for replication on other nodes. func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -295,6 +314,10 @@ func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, er } func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) (*GetNodeByPathResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -371,6 +394,10 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) } func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeServer) error { + if !s.initialSyncDone.Load() { + return ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -500,6 +527,10 @@ func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, e } func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) error { + if !s.initialSyncDone.Load() { + return ErrAlreadySyncing + } + b := req.GetBody() var cid cidSDK.ID @@ -560,6 +591,10 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) } func (s *Service) TreeList(ctx context.Context, req *TreeListRequest) (*TreeListResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + var cid cidSDK.ID err := cid.Decode(req.GetBody().GetContainerId()) @@ -643,5 +678,9 @@ func (s *Service) getContainerInfo(cid cidSDK.ID, pub []byte) ([]netmapSDK.NodeI } func (s *Service) Healthcheck(context.Context, *HealthcheckRequest) (*HealthcheckResponse, error) { + if !s.initialSyncDone.Load() { + return nil, ErrAlreadySyncing + } + return new(HealthcheckResponse), nil } diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index d132faf6e..e44e8dbbf 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -388,7 +388,7 @@ func (s *Service) syncLoop(ctx context.Context) { s.log.Error(logs.TreeCouldNotFetchContainers, zap.Error(err)) s.metrics.AddSyncDuration(time.Since(start), false) span.End() - continue + break } newMap, cnrsToSync := s.containersToSync(cnrs) @@ -402,6 +402,7 @@ func (s *Service) syncLoop(ctx context.Context) { s.metrics.AddSyncDuration(time.Since(start), true) span.End() } + s.initialSyncDone.Store(true) } } From 898f0686b1197caf60e57cd95074ac9cc73a2239 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 12:06:04 +0300 Subject: [PATCH 0321/1943] [#409] node: Log maintenance state on startup Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/netmap.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index c6623a385..9d8ad6a9a 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -267,6 +267,8 @@ func initNetmapState(c *cfg) { stateWord = "online" case ni.IsOffline(): stateWord = "offline" + case ni.IsMaintenance(): + stateWord = "maintenance" } } From e68384d4e3b494e50b1c82990c312a5fa4af923e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 12:08:56 +0300 Subject: [PATCH 0322/1943] [#409] node: Fetch last bootstrap info on startup Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/netmap.go | 64 ++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 9d8ad6a9a..c86e3a2e7 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -256,21 +256,10 @@ func initNetmapState(c *cfg) { epoch, err := c.cfgNetmap.wrapper.Epoch() fatalOnErrDetails("could not initialize current epoch number", err) - ni, err := c.netmapLocalNodeState(epoch) + ni, err := c.netmapInitLocalNodeState(epoch) fatalOnErrDetails("could not init network state", err) - stateWord := "undefined" - - if ni != nil { - switch { - case ni.IsOnline(): - stateWord = "online" - case ni.IsOffline(): - stateWord = "offline" - case ni.IsMaintenance(): - stateWord = "maintenance" - } - } + stateWord := nodeState(ni) c.log.Info(logs.FrostFSNodeInitialNetworkState, zap.Uint64("epoch", epoch), @@ -282,6 +271,55 @@ func initNetmapState(c *cfg) { c.handleLocalNodeInfo(ni) } +func nodeState(ni *netmapSDK.NodeInfo) string { + if ni != nil { + switch { + case ni.IsOnline(): + return "online" + case ni.IsOffline(): + return "offline" + case ni.IsMaintenance(): + return "maintenance" + } + } + return "undefined" +} + +func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { + nmNodes, err := c.cfgNetmap.wrapper.GetCandidates() + if err != nil { + return nil, err + } + + var candidate *netmapSDK.NodeInfo + for i := range nmNodes { + if bytes.Equal(nmNodes[i].PublicKey(), c.binPublicKey) { + candidate = &nmNodes[i] + break + } + } + + node, err := c.netmapLocalNodeState(epoch) + if err != nil { + return nil, err + } + + if candidate == nil { + return node, nil + } + + nmState := nodeState(node) + candidateState := nodeState(candidate) + if nmState != candidateState { + // This happens when the node was switched to maintenance without epoch tick. + // We expect it to continue staying in maintenance. + c.log.Info("candidate status is different from the netmap status, the former takes priority", + zap.String("netmap", nmState), + zap.String("candidate", candidateState)) + } + return candidate, nil +} + func (c *cfg) netmapLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { // calculate current network state nm, err := c.cfgNetmap.wrapper.GetNetMapByEpoch(epoch) From 4f83ab0fb43d969e1f8b81aa555b7894678aef88 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 12:11:59 +0300 Subject: [PATCH 0323/1943] [#409] node: Do not sent initial bootstrap under maintenance Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/netmap.go | 8 ++++++++ internal/logs/logs.go | 1 + 2 files changed, 9 insertions(+) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index c86e3a2e7..f1ea8b40e 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -225,6 +225,10 @@ func addNewEpochNotificationHandlers(c *cfg) { // Must be called after initNetmapService. func bootstrapNode(c *cfg) { if c.needBootstrap() { + if c.IsMaintenance() { + c.log.Info(logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) + return + } err := c.bootstrap() fatalOnErrDetails("bootstrap error", err) } @@ -266,6 +270,10 @@ func initNetmapState(c *cfg) { zap.String("state", stateWord), ) + if ni != nil && ni.IsMaintenance() { + c.isMaintenance.Store(true) + } + c.cfgNetmap.state.setCurrentEpoch(epoch) c.cfgNetmap.startEpoch = epoch c.handleLocalNodeInfo(ni) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 9b800adfc..dc54f2d2a 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -491,4 +491,5 @@ const ( EngineShardsEvacuationFailedToMoveObject = "failed to evacuate object to other node" ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" ShardDeleteCantDeleteFromWriteCache = "can't delete object from write cache" + FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" ) From 90e9247b69955a04a1e3aa97b042d52bdfb5b03b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 26 May 2023 12:15:50 +0300 Subject: [PATCH 0324/1943] [#371] ir: Add morph cache metrics Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/innerring/initialization.go | 11 ++--- pkg/innerring/innerring.go | 14 +++--- pkg/metrics/innerring.go | 18 +++++--- pkg/metrics/morphcache.go | 77 +++++++++++++++++++++++++++++++++ pkg/metrics/node.go | 3 +- pkg/metrics/treeservice.go | 8 ++++ pkg/morph/client/client.go | 3 ++ pkg/morph/client/constructor.go | 14 +++++- pkg/morph/client/nns.go | 18 ++++++++ pkg/morph/client/notary.go | 9 ++++ pkg/services/tree/metrics.go | 6 --- 11 files changed, 154 insertions(+), 27 deletions(-) create mode 100644 pkg/metrics/morphcache.go diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index c49d22509..05e503f2f 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -469,11 +469,12 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- } morphChain := &chainParams{ - log: s.log, - cfg: cfg, - key: s.key, - name: morphPrefix, - from: fromSideChainBlock, + log: s.log, + cfg: cfg, + key: s.key, + name: morphPrefix, + from: fromSideChainBlock, + morphCacheMetric: s.metrics.MorphCacheMetrics(), } // create morph client diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index cbcb4699a..335d3d179 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -102,12 +102,13 @@ type ( } chainParams struct { - log *logger.Logger - cfg *viper.Viper - key *keys.PrivateKey - name string - sgn *transaction.Signer - from uint32 // block height + log *logger.Logger + cfg *viper.Viper + key *keys.PrivateKey + name string + sgn *transaction.Signer + from uint32 // block height + morphCacheMetric metrics.MorphCacheMetrics } ) @@ -465,6 +466,7 @@ func createClient(ctx context.Context, p *chainParams, errChan chan<- error) (*c errChan <- fmt.Errorf("%s chain connection has been lost", p.name) }), client.WithSwitchInterval(p.cfg.GetDuration(p.name+".switch_interval")), + client.WithMorphCacheMetrics(p.morphCacheMetric), ) } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 9d8b76bf9..d756a539f 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -16,9 +16,10 @@ const ( // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { - epoch prometheus.Gauge - health prometheus.Gauge - eventDuration *prometheus.HistogramVec + epoch prometheus.Gauge + health prometheus.Gauge + eventDuration *prometheus.HistogramVec + morphCacheMetrics *morphCacheMetrics } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. @@ -45,9 +46,10 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { ) return &InnerRingServiceMetrics{ - epoch: epoch, - health: health, - eventDuration: eventDuration, + epoch: epoch, + health: health, + eventDuration: eventDuration, + morphCacheMetrics: newMorphCacheMetrics(), } } @@ -67,3 +69,7 @@ func (m InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success b innerRingLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) } + +func (m InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { + return m.morphCacheMetrics +} diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go new file mode 100644 index 000000000..7f6f2d7d5 --- /dev/null +++ b/pkg/metrics/morphcache.go @@ -0,0 +1,77 @@ +package metrics + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + mcSubsystem = "morphcache" + mcSuccess = "success" +) + +type MorphCacheMetrics interface { + AddNNSContractHashDuration(success bool, d time.Duration) + + AddGroupKeyDuration(success bool, d time.Duration) + + AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) +} + +type morphCacheMetrics struct { + // Duration of processing get nns contract hash request + nnsContractHashDuration *prometheus.HistogramVec + + // Duration of processing get group key request + groupKeyDuration *prometheus.HistogramVec + + // Duration of processing get tx height request + txHeightDuration *prometheus.HistogramVec +} + +var _ MorphCacheMetrics = (*morphCacheMetrics)(nil) + +func newMorphCacheMetrics() *morphCacheMetrics { + return &morphCacheMetrics{ + nnsContractHashDuration: newMCMethodDurationCounter("nns_contract_hash"), + groupKeyDuration: newMCMethodDurationCounter("group_key"), + txHeightDuration: newMCMethodDurationCounter("tx_height"), + } +} + +func (m *morphCacheMetrics) AddNNSContractHashDuration(success bool, d time.Duration) { + m.nnsContractHashDuration.With( + prometheus.Labels{ + mcSuccess: fmt.Sprintf("%v", success), + }, + ).Observe(float64(d)) +} + +func (m *morphCacheMetrics) AddGroupKeyDuration(success bool, d time.Duration) { + m.groupKeyDuration.With( + prometheus.Labels{ + mcSuccess: fmt.Sprintf("%v", success), + }, + ).Observe(float64(d)) +} + +func (m *morphCacheMetrics) AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) { + m.txHeightDuration.With( + prometheus.Labels{ + mcSuccess: fmt.Sprintf("%v", success), + }, + ).Observe(float64(d)) +} + +func newMCMethodDurationCounter(method string) *prometheus.HistogramVec { + return metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: mcSubsystem, + Name: fmt.Sprintf("%s_req_duration_seconds", method), + Help: fmt.Sprintf("Accumulated %s request process duration", method), + }, []string{mcSuccess}) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 8819ba15b..16a4c2b5b 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -1,7 +1,6 @@ package metrics import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -67,7 +66,7 @@ func (m *NodeMetrics) WriteCache() WriteCacheMetrics { return m.writeCacheMetrics } -func (m *NodeMetrics) TreeService() tree.MetricsRegister { +func (m *NodeMetrics) TreeService() TreeMetricsRegister { return m.treeService } diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go index 903ef3496..48a450c76 100644 --- a/pkg/metrics/treeservice.go +++ b/pkg/metrics/treeservice.go @@ -10,12 +10,20 @@ import ( const treeServiceLabelSuccess = "success" +type TreeMetricsRegister interface { + AddReplicateTaskDuration(time.Duration, bool) + AddReplicateWaitDuration(time.Duration, bool) + AddSyncDuration(time.Duration, bool) +} + type treeServiceMetrics struct { replicateTaskDuration *prometheus.HistogramVec replicateWaitDuration *prometheus.HistogramVec syncOpDuration *prometheus.HistogramVec } +var _ TreeMetricsRegister = (*treeServiceMetrics)(nil) + func newTreeServiceMetrics() *treeServiceMetrics { const treeServiceSubsystem = "treeservice" return &treeServiceMetrics{ diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 6a7a5b51a..f03ee1dbf 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -10,6 +10,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" @@ -88,6 +89,8 @@ type cache struct { nnsHash *util.Uint160 gKey *keys.PublicKey txHeights *lru.Cache[util.Uint256, uint32] + + metrics metrics.MorphCacheMetrics } func (c *cache) nns() *util.Uint160 { diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 1d55db71c..8b5fb3ff0 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -7,6 +7,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -42,6 +43,8 @@ type cfg struct { inactiveModeCb Callback switchInterval time.Duration + + morphCacheMetrics metrics.MorphCacheMetrics } const ( @@ -104,7 +107,7 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } cli := &Client{ - cache: newClientCache(), + cache: newClientCache(cfg.morphCacheMetrics), logger: cfg.logger, acc: acc, accAddr: accAddr, @@ -195,10 +198,11 @@ func newActor(ws *rpcclient.WSClient, acc *wallet.Account, cfg cfg) (*actor.Acto }}) } -func newClientCache() cache { +func newClientCache(morphCacheMetrics metrics.MorphCacheMetrics) cache { c, _ := lru.New[util.Uint256, uint32](100) // returns error only if size is negative return cache{ txHeights: c, + metrics: morphCacheMetrics, } } @@ -282,3 +286,9 @@ func WithSwitchInterval(i time.Duration) Option { c.switchInterval = i } } + +func WithMorphCacheMetrics(morphCacheMetrics metrics.MorphCacheMetrics) Option { + return func(c *cfg) { + c.morphCacheMetrics = morphCacheMetrics + } +} diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index 2f7079dfe..d8d798b07 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -5,6 +5,7 @@ import ( "fmt" "math/big" "strconv" + "time" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -81,6 +82,13 @@ func (c *Client) NNSHash() (util.Uint160, error) { return util.Uint160{}, ErrConnectionLost } + success := false + startedAt := time.Now() + + defer func() { + c.cache.metrics.AddNNSContractHashDuration(success, time.Since(startedAt)) + }() + nnsHash := c.cache.nns() if nnsHash == nil { @@ -92,6 +100,7 @@ func (c *Client) NNSHash() (util.Uint160, error) { c.cache.setNNSHash(cs.Hash) nnsHash = &cs.Hash } + success = true return *nnsHash, nil } @@ -221,7 +230,14 @@ func (c *Client) SetGroupSignerScope() error { // contractGroupKey returns public key designating FrostFS contract group. func (c *Client) contractGroupKey() (*keys.PublicKey, error) { + success := false + startedAt := time.Now() + defer func() { + c.cache.metrics.AddGroupKeyDuration(success, time.Since(startedAt)) + }() + if gKey := c.cache.groupKey(); gKey != nil { + success = true return gKey, nil } @@ -251,5 +267,7 @@ func (c *Client) contractGroupKey() (*keys.PublicKey, error) { } c.cache.setGroupKey(pub) + + success = true return pub, nil } diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 1ed1ca912..3a3ff0b46 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -8,6 +8,7 @@ import ( "math" "math/big" "strings" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" @@ -789,7 +790,14 @@ func (c *Client) calculateNonceAndVUB(hash util.Uint256, roundBlockHeight bool) } func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { + success := false + startedAt := time.Now() + defer func() { + c.cache.metrics.AddTxHeightDuration(h, success, time.Since(startedAt)) + }() + if rh, ok := c.cache.txHeights.Get(h); ok { + success = true return rh, nil } height, err := c.client.GetTransactionHeight(h) @@ -797,5 +805,6 @@ func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { return 0, err } c.cache.txHeights.Add(h, height) + success = true return height, nil } diff --git a/pkg/services/tree/metrics.go b/pkg/services/tree/metrics.go index 0f0e4ee57..53708c4f1 100644 --- a/pkg/services/tree/metrics.go +++ b/pkg/services/tree/metrics.go @@ -2,12 +2,6 @@ package tree import "time" -type MetricsRegister interface { - AddReplicateTaskDuration(time.Duration, bool) - AddReplicateWaitDuration(time.Duration, bool) - AddSyncDuration(time.Duration, bool) -} - type defaultMetricsRegister struct{} func (defaultMetricsRegister) AddReplicateTaskDuration(time.Duration, bool) {} From 4887f489a195418340992eca63d609b293c26e6f Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 8 Jun 2023 16:37:46 +0300 Subject: [PATCH 0325/1943] [#17] Add morph client metrics Signed-off-by: Alejandro Lopez --- cmd/frostfs-node/morph.go | 2 + pkg/innerring/initialization.go | 14 ++--- pkg/innerring/innerring.go | 6 +-- pkg/innerring/state.go | 8 +-- pkg/metrics/morph.go | 82 ++++++++++++++++++++++++++++++ pkg/morph/client/client.go | 30 ++++++++++- pkg/morph/client/constructor.go | 20 ++++++++ pkg/morph/client/notary.go | 7 +++ pkg/morph/metrics/metrics.go | 17 +++++++ pkg/morph/subscriber/subscriber.go | 5 ++ pkg/services/tree/metrics.go | 6 +++ 11 files changed, 182 insertions(+), 15 deletions(-) create mode 100644 pkg/metrics/morph.go create mode 100644 pkg/morph/metrics/metrics.go diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 2e086f994..ae50b8a8a 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -9,6 +9,7 @@ import ( morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -43,6 +44,7 @@ func initMorphComponents(ctx context.Context, c *cfg) { c.key, client.WithDialTimeout(morphconfig.DialTimeout(c.appCfg)), client.WithLogger(c.log), + client.WithMetrics(metrics.NewMorphClientMetrics()), client.WithEndpoints(addresses...), client.WithConnLostCallback(func() { c.internalErr <- errors.New("morph connection has been lost") diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 05e503f2f..52ffb10a1 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -52,7 +52,7 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, s.netmapProcessor, err = netmap.New(&netmap.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.netmap"), NetmapClient: netmap.NewNetmapClient(s.netmapClient), EpochTimer: s, @@ -163,7 +163,7 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli // create governance processor governanceProcessor, err := governance.New(&governance.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, FrostFSClient: frostfsCli, NetmapClient: s.netmapClient, AlphabetState: s, @@ -233,7 +233,7 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { s.alphabetProcessor, err = alphabet.New(&alphabet.Params{ ParsedWallets: parsedWallets, Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.alphabet"), AlphabetContracts: s.contracts.alphabet, NetmapClient: s.netmapClient, @@ -258,7 +258,7 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C // container processor containerProcessor, err := cont.New(&cont.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.container"), AlphabetState: s, ContainerClient: cnrClient, @@ -277,7 +277,7 @@ func (s *Server) initBalanceProcessor(cfg *viper.Viper, frostfsCli *frostfsClien // create balance processor balanceProcessor, err := balance.New(&balance.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.balance"), FrostFSClient: frostfsCli, BalanceSC: s.contracts.balance, @@ -298,7 +298,7 @@ func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper, frostfsIDClient * frostfsProcessor, err := frostfs.New(&frostfs.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.frostfs"), FrostFSContract: s.contracts.frostfs, FrostFSIDClient: frostfsIDClient, @@ -474,7 +474,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- key: s.key, name: morphPrefix, from: fromSideChainBlock, - morphCacheMetric: s.metrics.MorphCacheMetrics(), + morphCacheMetric: s.irMetrics.MorphCacheMetrics(), } // create morph client diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 335d3d179..1567e40d3 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -58,7 +58,7 @@ type ( persistate *state.PersistentStorage // metrics - metrics *metrics.InnerRingServiceMetrics + irMetrics *metrics.InnerRingServiceMetrics // notary configuration feeConfig *config.FeeConfig @@ -328,8 +328,8 @@ func (s *Server) registerStarter(f func() error) { func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error) (*Server, error) { var err error server := &Server{ - log: log, - metrics: metrics.NewInnerRingMetrics(), + log: log, + irMetrics: metrics.NewInnerRingMetrics(), } server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED) diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 27f265ae2..c5adb71eb 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -29,8 +29,8 @@ func (s *Server) EpochCounter() uint64 { // epoch counter. func (s *Server) SetEpochCounter(val uint64) { s.epochCounter.Store(val) - if s.metrics != nil { - s.metrics.SetEpoch(val) + if s.irMetrics != nil { + s.irMetrics.SetEpoch(val) } } @@ -154,8 +154,8 @@ func (s *Server) ResetEpochTimer(h uint32) error { func (s *Server) setHealthStatus(hs control.HealthStatus) { s.healthStatus.Store(int32(hs)) - if s.metrics != nil { - s.metrics.SetHealth(int32(hs)) + if s.irMetrics != nil { + s.irMetrics.SetHealth(int32(hs)) } } diff --git a/pkg/metrics/morph.go b/pkg/metrics/morph.go new file mode 100644 index 000000000..cc851d3db --- /dev/null +++ b/pkg/metrics/morph.go @@ -0,0 +1,82 @@ +package metrics + +import ( + "strconv" + "time" + + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + morphSubsystem = "morph" + morphNotificationTypeLabel = "notification_type" + morphInvokeTypeLabel = "invoke_type" + morphContractLabel = "contract" + morphMethodLabel = "method" + morphSuccessLabel = "success" +) + +type morphClientMetrics struct { + switchCount prometheus.Counter + lastBlock prometheus.Gauge + notificationCount *prometheus.CounterVec + invokeDuration *prometheus.HistogramVec +} + +func NewMorphClientMetrics() morphmetrics.Register { + return &morphClientMetrics{ + switchCount: metrics.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "switch_count", + Help: "Number of endpoint switches", + }), + lastBlock: metrics.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "last_block", + Help: "Index of the last received block", + }), + notificationCount: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "notification_count", + Help: "Number of notifications received by notification type", + }, []string{morphNotificationTypeLabel}), + invokeDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "invoke_duration_seconds", + Help: "Cummulative duration of contract invocations", + }, []string{morphInvokeTypeLabel, morphContractLabel, morphMethodLabel, morphSuccessLabel}), + } +} + +func (m *morphClientMetrics) IncSwitchCount() { + m.switchCount.Inc() +} + +func (m *morphClientMetrics) SetLastBlock(index uint32) { + m.lastBlock.Set(float64(index)) +} + +func (m *morphClientMetrics) IncNotificationCount(typ string) { + m.notificationCount.With( + prometheus.Labels{ + morphNotificationTypeLabel: typ, + }, + ).Inc() +} + +func (m *morphClientMetrics) ObserveInvoke(typ string, contract string, method string, success bool, d time.Duration) { + m.invokeDuration.With( + prometheus.Labels{ + morphInvokeTypeLabel: typ, + morphContractLabel: contract, + morphMethodLabel: method, + morphSuccessLabel: strconv.FormatBool(success), + }, + ).Observe(d.Seconds()) +} diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index f03ee1dbf..606f3bd66 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" @@ -48,7 +49,8 @@ import ( type Client struct { cache cache - logger *logger.Logger // logging component + logger *logger.Logger // logging component + metrics morphmetrics.Register client *rpcclient.WSClient // neo-go websocket client rpcActor *actor.Actor // neo-go RPC actor @@ -172,6 +174,12 @@ func wrapFrostFSError(err error) error { // Invoke invokes contract method by sending transaction into blockchain. // Supported args types: int64, string, util.Uint160, []byte and bool. func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("Invoke", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -189,6 +197,7 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, zap.Uint32("vub", vub), zap.Stringer("tx_hash", txHash.Reverse())) + success = true return nil } @@ -196,6 +205,12 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, // If cb returns an error, the session is closed and this error is returned as-is. // If the remove neo-go node does not support sessions, `unwrap.ErrNoSessionID` is returned. func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, contract util.Uint160, method string, args ...interface{}) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("TestInvokeIterator", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -228,12 +243,20 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, contract util } items, err = c.rpcActor.TraverseIterator(sid, &r, 0) } + + success = err == nil return err } // TestInvoke invokes contract method locally in neo-go node. This method should // be used to read data from smart-contract. func (c *Client) TestInvoke(contract util.Uint160, method string, args ...any) (res []stackitem.Item, err error) { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("TestInvoke", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -250,6 +273,7 @@ func (c *Client) TestInvoke(contract util.Uint160, method string, args ...any) ( return nil, wrapFrostFSError(¬HaltStateError{state: val.State, exception: val.FaultException}) } + success = true return val.Stack, nil } @@ -512,6 +536,10 @@ func (c *Client) NotificationChannel() <-chan rpcclient.Notification { return c.client.Notifications //lint:ignore SA1019 waits for neo-go v0.102.0 https://github.com/nspcc-dev/neo-go/pull/2980 } +func (c *Client) Metrics() morphmetrics.Register { + return c.metrics +} + func (c *Client) setActor(act *actor.Actor) { c.rpcActor = act c.gasToken = nep17.New(act, gas.Hash) diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 8b5fb3ff0..6cc66c5cc 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -32,6 +33,8 @@ type cfg struct { logger *logger.Logger // logging component + metrics morphmetrics.Register + waitInterval time.Duration signer *transaction.Signer @@ -60,6 +63,7 @@ func defaultConfig() *cfg { return &cfg{ dialTimeout: defaultDialTimeout, logger: &logger.Logger{Logger: zap.L()}, + metrics: morphmetrics.NoopRegister{}, waitInterval: defaultWaitInterval, signer: &transaction.Signer{ Scopes: transaction.Global, @@ -80,6 +84,7 @@ func defaultConfig() *cfg { // - signer with the global scope; // - wait interval: 500ms; // - logger: &logger.Logger{Logger: zap.L()}. +// - metrics: metrics.NoopRegister // // If desired option satisfies the default value, it can be omitted. // If multiple options of the same config value are supplied, @@ -109,6 +114,7 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er cli := &Client{ cache: newClientCache(cfg.morphCacheMetrics), logger: cfg.logger, + metrics: cfg.metrics, acc: acc, accAddr: accAddr, cfg: *cfg, @@ -235,6 +241,20 @@ func WithLogger(logger *logger.Logger) Option { } } +// WithMetrics returns a client constructor option +// that specifies the component for reporting metrics. +// +// Ignores nil value. +// +// If option not provided, NoopMetrics is used. +func WithMetrics(metrics morphmetrics.Register) Option { + return func(c *cfg) { + if metrics != nil { + c.metrics = metrics + } + } +} + // WithSigner returns a client constructor option // that specifies the signer and the scope of the transaction. // diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 3a3ff0b46..680ba6a68 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -444,6 +444,12 @@ func (c *Client) notaryInvokeAsCommittee(method string, nonce, vub uint32, args } func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint160, nonce uint32, vub *uint32, method string, args ...any) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("notaryInvoke", contract.String(), method, success, time.Since(start)) + }() + alphabetList, err := c.notary.alphabetSource() if err != nil { return err @@ -485,6 +491,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint zap.String("tx_hash", mainH.StringLE()), zap.String("fallback_hash", fbH.StringLE())) + success = true return nil } diff --git a/pkg/morph/metrics/metrics.go b/pkg/morph/metrics/metrics.go new file mode 100644 index 000000000..9e41a0b86 --- /dev/null +++ b/pkg/morph/metrics/metrics.go @@ -0,0 +1,17 @@ +package metrics + +import "time" + +type Register interface { + IncSwitchCount() + SetLastBlock(uint32) + IncNotificationCount(notificationType string) + ObserveInvoke(typ string, contract string, method string, success bool, d time.Duration) +} + +type NoopRegister struct{} + +func (NoopRegister) IncSwitchCount() {} +func (NoopRegister) SetLastBlock(uint32) {} +func (NoopRegister) IncNotificationCount(string) {} +func (NoopRegister) ObserveInvoke(string, string, string, bool, time.Duration) {} diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 608872dec..4076111f0 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -200,18 +200,22 @@ routeloop: break routeloop case ev, ok := <-curr.NotifyChan: if ok { + s.client.Metrics().IncNotificationCount("notify") s.notifyChan <- ev } else { connLost = true } case ev, ok := <-curr.BlockChan: if ok { + s.client.Metrics().IncNotificationCount("block") + s.client.Metrics().SetLastBlock(ev.Index) s.blockChan <- ev } else { connLost = true } case ev, ok := <-curr.NotaryChan: if ok { + s.client.Metrics().IncNotificationCount("notary") s.notaryChan <- ev } else { connLost = true @@ -262,6 +266,7 @@ func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) ( s.current = chs s.Unlock() + s.client.Metrics().IncSwitchCount() return true, cliCh } diff --git a/pkg/services/tree/metrics.go b/pkg/services/tree/metrics.go index 53708c4f1..0f0e4ee57 100644 --- a/pkg/services/tree/metrics.go +++ b/pkg/services/tree/metrics.go @@ -2,6 +2,12 @@ package tree import "time" +type MetricsRegister interface { + AddReplicateTaskDuration(time.Duration, bool) + AddReplicateWaitDuration(time.Duration, bool) + AddSyncDuration(time.Duration, bool) +} + type defaultMetricsRegister struct{} func (defaultMetricsRegister) AddReplicateTaskDuration(time.Duration, bool) {} From 344d6b2ae1f241a43bdeacc88ff2b5772d0da508 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 13 Jun 2023 15:27:38 +0300 Subject: [PATCH 0326/1943] [#xx] Fix invalid log metric namespace identifier Signed-off-by: Alejandro Lopez --- cmd/frostfs-ir/main.go | 2 +- cmd/frostfs-node/config.go | 2 +- pkg/util/logger/metrics.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 81fcaa489..70199b094 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -61,7 +61,7 @@ func main() { cfg, err = newConfig() exitErr(err) - logPrm.MetricsNamespace = "frostfs-ir" + logPrm.MetricsNamespace = "frostfs_ir" err = logPrm.SetLevelString( cfg.GetString("logger.level"), ) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e4547364d..758803291 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -542,7 +542,7 @@ func initCfg(appCfg *config.Config) *cfg { logPrm, err := c.loggerPrm() fatalOnErr(err) - logPrm.MetricsNamespace = "frostfs-node" + logPrm.MetricsNamespace = "frostfs_node" log, err := logger.NewLogger(logPrm) fatalOnErr(err) diff --git a/pkg/util/logger/metrics.go b/pkg/util/logger/metrics.go index 10be4a8f0..708583473 100644 --- a/pkg/util/logger/metrics.go +++ b/pkg/util/logger/metrics.go @@ -9,7 +9,7 @@ import ( ) const ( - logSubsystem = "log" + logSubsystem = "logger" logLevelLabel = "level" logDroppedLabel = "dropped" ) @@ -23,7 +23,7 @@ func newLogMetrics(namespace string) *logMetrics { logCount: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: logSubsystem, - Name: "log_count", + Name: "entry_count", Help: "Total log entries emitted or dropped by severity level", }, []string{logLevelLabel, logDroppedLabel}), } From fb8fee0c8edd3cca1d9aaa40bf7334df46400964 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 13 Jun 2023 14:19:33 +0300 Subject: [PATCH 0327/1943] [#442] Use strconv.FormatBool instead of untyped conversion Signed-off-by: Alejandro Lopez --- pkg/metrics/gc.go | 6 +++--- pkg/metrics/morphcache.go | 7 ++++--- pkg/metrics/treeservice.go | 8 ++++---- pkg/metrics/writecache.go | 5 +++-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/metrics/gc.go b/pkg/metrics/gc.go index 2457c0c6b..9c00d8722 100644 --- a/pkg/metrics/gc.go +++ b/pkg/metrics/gc.go @@ -1,7 +1,7 @@ package metrics import ( - "fmt" + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -64,7 +64,7 @@ func newGCMetrics() *gcMetrics { func (m *gcMetrics) AddRunDuration(shardID string, d time.Duration, success bool) { m.runDuration.With(prometheus.Labels{ gcShardID: shardID, - gcSuccess: fmt.Sprintf("%v", success), + gcSuccess: strconv.FormatBool(success), }).Add(d.Seconds()) } @@ -84,7 +84,7 @@ func (m *gcMetrics) AddDeletedCount(shardID string, deleted, failed uint64) { func (m *gcMetrics) AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) { m.expCollectDuration.With(prometheus.Labels{ gcShardID: shardID, - gcSuccess: fmt.Sprintf("%v", success), + gcSuccess: strconv.FormatBool(success), gcObjectType: objectType, }).Add(d.Seconds()) } diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go index 7f6f2d7d5..2440adcfb 100644 --- a/pkg/metrics/morphcache.go +++ b/pkg/metrics/morphcache.go @@ -2,6 +2,7 @@ package metrics import ( "fmt" + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -46,7 +47,7 @@ func newMorphCacheMetrics() *morphCacheMetrics { func (m *morphCacheMetrics) AddNNSContractHashDuration(success bool, d time.Duration) { m.nnsContractHashDuration.With( prometheus.Labels{ - mcSuccess: fmt.Sprintf("%v", success), + mcSuccess: strconv.FormatBool(success), }, ).Observe(float64(d)) } @@ -54,7 +55,7 @@ func (m *morphCacheMetrics) AddNNSContractHashDuration(success bool, d time.Dura func (m *morphCacheMetrics) AddGroupKeyDuration(success bool, d time.Duration) { m.groupKeyDuration.With( prometheus.Labels{ - mcSuccess: fmt.Sprintf("%v", success), + mcSuccess: strconv.FormatBool(success), }, ).Observe(float64(d)) } @@ -62,7 +63,7 @@ func (m *morphCacheMetrics) AddGroupKeyDuration(success bool, d time.Duration) { func (m *morphCacheMetrics) AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) { m.txHeightDuration.With( prometheus.Labels{ - mcSuccess: fmt.Sprintf("%v", success), + mcSuccess: strconv.FormatBool(success), }, ).Observe(float64(d)) } diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go index 48a450c76..ae24c41b2 100644 --- a/pkg/metrics/treeservice.go +++ b/pkg/metrics/treeservice.go @@ -1,7 +1,7 @@ package metrics import ( - "fmt" + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -50,18 +50,18 @@ func newTreeServiceMetrics() *treeServiceMetrics { func (m *treeServiceMetrics) AddReplicateTaskDuration(d time.Duration, success bool) { m.replicateTaskDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: fmt.Sprintf("%v", success), + treeServiceLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddReplicateWaitDuration(d time.Duration, success bool) { m.replicateWaitDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: fmt.Sprintf("%v", success), + treeServiceLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddSyncDuration(d time.Duration, success bool) { m.syncOpDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: fmt.Sprintf("%v", success), + treeServiceLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 67b9d8caf..b27e684d5 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -2,6 +2,7 @@ package metrics import ( "fmt" + "strconv" "sync" "time" @@ -148,7 +149,7 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { func (m *writeCacheMetrics) IncFlushCounter(shardID string, success bool, storageType string) { m.flushCounter.With(prometheus.Labels{ wcShardID: shardID, - wcSuccess: fmt.Sprintf("%v", success), + wcSuccess: strconv.FormatBool(success), wcStorage: storageType, }).Inc() } @@ -164,7 +165,7 @@ func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success b m.With( prometheus.Labels{ wcShardID: shardID, - wcSuccess: fmt.Sprintf("%v", success), + wcSuccess: strconv.FormatBool(success), wcStorage: storageType, }, ).Observe(d.Seconds()) From 71bbeddb64793cc5e7be7d4d144293dc1d38efcc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 15:03:37 +0300 Subject: [PATCH 0328/1943] [#424] metrics: Drop unused arg Signed-off-by: Dmitrii Stepanov --- pkg/metrics/morphcache.go | 5 ++--- pkg/morph/client/notary.go | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go index 2440adcfb..000ec0d80 100644 --- a/pkg/metrics/morphcache.go +++ b/pkg/metrics/morphcache.go @@ -6,7 +6,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" - "github.com/nspcc-dev/neo-go/pkg/util" "github.com/prometheus/client_golang/prometheus" ) @@ -20,7 +19,7 @@ type MorphCacheMetrics interface { AddGroupKeyDuration(success bool, d time.Duration) - AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) + AddTxHeightDuration(success bool, d time.Duration) } type morphCacheMetrics struct { @@ -60,7 +59,7 @@ func (m *morphCacheMetrics) AddGroupKeyDuration(success bool, d time.Duration) { ).Observe(float64(d)) } -func (m *morphCacheMetrics) AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) { +func (m *morphCacheMetrics) AddTxHeightDuration(success bool, d time.Duration) { m.txHeightDuration.With( prometheus.Labels{ mcSuccess: strconv.FormatBool(success), diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 680ba6a68..e567b6df0 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -800,7 +800,7 @@ func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { success := false startedAt := time.Now() defer func() { - c.cache.metrics.AddTxHeightDuration(h, success, time.Since(startedAt)) + c.cache.metrics.AddTxHeightDuration(success, time.Since(startedAt)) }() if rh, ok := c.cache.txHeights.Get(h); ok { From 07f155ac77e3af09d849f271caf123eb0e71826d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 15:05:45 +0300 Subject: [PATCH 0329/1943] [#424] metrics: Use labels for writecache methods and operations Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/writecache.go | 10 +- pkg/metrics/types.go | 17 +++ pkg/metrics/writecache.go | 110 ++++++------------ 3 files changed, 60 insertions(+), 77 deletions(-) create mode 100644 pkg/metrics/types.go diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index cd8278272..b947f12f4 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -63,18 +63,18 @@ type writeCacheMetrics struct { } func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddGetDuration(m.shardID, success, d, st.String()) + m.metrics.AddMethodDuration(m.shardID, "Get", success, d, st.String()) } func (m *writeCacheMetrics) Delete(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddDeleteDuration(m.shardID, success, d, st.String()) + m.metrics.AddMethodDuration(m.shardID, "Delete", success, d, st.String()) if success { m.metrics.DecActualCount(m.shardID, st.String()) } } func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddPutDuration(m.shardID, success, d, st.String()) + m.metrics.AddMethodDuration(m.shardID, "Put", success, d, st.String()) if success { m.metrics.IncActualCount(m.shardID, st.String()) } @@ -95,10 +95,10 @@ func (m *writeCacheMetrics) SetActualCounters(db, fstree uint64) { } func (m *writeCacheMetrics) Flush(success bool, st writecache.StorageType) { - m.metrics.IncFlushCounter(m.shardID, success, st.String()) + m.metrics.IncOperationCounter(m.shardID, "Flush", metrics.NullBool{Bool: success, Valid: true}, st.String()) } func (m *writeCacheMetrics) Evict(st writecache.StorageType) { m.metrics.DecActualCount(m.shardID, st.String()) - m.metrics.IncEvictCounter(m.shardID, st.String()) + m.metrics.IncOperationCounter(m.shardID, "Evict", metrics.NullBool{}, st.String()) } diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go new file mode 100644 index 000000000..6a76248bf --- /dev/null +++ b/pkg/metrics/types.go @@ -0,0 +1,17 @@ +package metrics + +import ( + "strconv" +) + +type NullBool struct { + Bool bool + Valid bool // Valid is true if Bool is not NULL +} + +func (v NullBool) String() string { + if !v.Valid { + return "" + } + return strconv.FormatBool(v.Bool) +} diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index b27e684d5..33c58fc65 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -2,7 +2,6 @@ package metrics import ( "fmt" - "strconv" "sync" "time" @@ -16,6 +15,8 @@ const ( wcSuccess = "success" wcStorage = "storage" wcMode = "mode" + wcMethod = "method" + wcOperation = "operation" ) type shardIDMode struct { @@ -23,9 +24,7 @@ type shardIDMode struct { } type WriteCacheMetrics interface { - AddGetDuration(shardID string, success bool, d time.Duration, storageType string) - AddDeleteDuration(shardID string, success bool, d time.Duration, storageType string) - AddPutDuration(shardID string, success bool, d time.Duration, storageType string) + AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) IncActualCount(shardID string, storageType string) DecActualCount(shardID string, storageType string) @@ -34,17 +33,12 @@ type WriteCacheMetrics interface { SetEstimateSize(shardID string, size uint64, storageType string) SetMode(shardID string, mode string) - IncFlushCounter(shardID string, success bool, storageType string) - IncEvictCounter(shardID string, storageType string) + IncOperationCounter(shardID string, operation string, success NullBool, storageType string) } type writeCacheMetrics struct { - getDuration *prometheus.HistogramVec - putDuration *prometheus.HistogramVec - deleteDuration *prometheus.HistogramVec - - flushCounter *prometheus.CounterVec - evictCounter *prometheus.CounterVec + methodDuration *prometheus.HistogramVec + operationCounter *prometheus.CounterVec actualCount *prometheus.GaugeVec @@ -57,29 +51,35 @@ type writeCacheMetrics struct { func newWriteCacheMetrics() *writeCacheMetrics { return &writeCacheMetrics{ - getDuration: newWCMethodDurationCounter("get"), - putDuration: newWCMethodDurationCounter("put"), - deleteDuration: newWCMethodDurationCounter("delete"), - flushCounter: newWCOperationCounterVec("flush", []string{wcShardID, wcStorage, wcSuccess}), - evictCounter: newWCOperationCounterVec("evict", []string{wcShardID, wcStorage}), - actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), - estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), - modeMtx: sync.RWMutex{}, - modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc), - modeValues: make(map[string]string), + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: "request_duration_seconds", + Help: "Writecache request process duration", + }, []string{wcShardID, wcSuccess, wcStorage, wcMethod}), + operationCounter: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: wcSubsystem, + Name: "operation_count", + Help: "The number of writecache operations processed", + }, []string{wcShardID, wcStorage, wcSuccess, wcOperation}), + actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), + estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), + modeMtx: sync.RWMutex{}, + modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc), + modeValues: make(map[string]string), } } -func (m *writeCacheMetrics) AddGetDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.getDuration, shardID, success, d, storageType) -} - -func (m *writeCacheMetrics) AddDeleteDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.deleteDuration, shardID, success, d, storageType) -} - -func (m *writeCacheMetrics) AddPutDuration(shardID string, success bool, d time.Duration, storageType string) { - setWriteCacheDuration(m.putDuration, shardID, success, d, storageType) +func (m *writeCacheMetrics) AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) { + m.methodDuration.With( + prometheus.Labels{ + wcShardID: shardID, + wcSuccess: fmt.Sprintf("%v", success), + wcStorage: storageType, + wcMethod: method, + }, + ).Observe(d.Seconds()) } func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { @@ -146,49 +146,15 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { m.modeMetrics[key] = metric } -func (m *writeCacheMetrics) IncFlushCounter(shardID string, success bool, storageType string) { - m.flushCounter.With(prometheus.Labels{ - wcShardID: shardID, - wcSuccess: strconv.FormatBool(success), - wcStorage: storageType, +func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) { + m.operationCounter.With(prometheus.Labels{ + wcShardID: shardID, + wcStorage: storageType, + wcOperation: operation, + wcSuccess: success.String(), }).Inc() } -func (m *writeCacheMetrics) IncEvictCounter(shardID string, storageType string) { - m.evictCounter.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, - }).Inc() -} - -func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success bool, d time.Duration, storageType string) { - m.With( - prometheus.Labels{ - wcShardID: shardID, - wcSuccess: strconv.FormatBool(success), - wcStorage: storageType, - }, - ).Observe(d.Seconds()) -} - -func newWCMethodDurationCounter(method string) *prometheus.HistogramVec { - return metrics.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: wcSubsystem, - Name: fmt.Sprintf("%s_req_duration_seconds", method), - Help: fmt.Sprintf("Accumulated %s request process duration", method), - }, []string{wcShardID, wcSuccess, wcStorage}) -} - -func newWCOperationCounterVec(operation string, labels []string) *prometheus.CounterVec { - return metrics.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: wcSubsystem, - Name: fmt.Sprintf("%s_operation_count", operation), - Help: fmt.Sprintf("The number of %s operations processed", operation), - }, labels) -} - func newWCGaugeVec(name, help string, labels []string) *prometheus.GaugeVec { return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, From 85deb12f4d078b5b3dcdd490a1b4c49ec6c9c0ce Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 15:14:28 +0300 Subject: [PATCH 0330/1943] [#424] writecache: Drop metrics when close Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/writecache.go | 4 ++++ pkg/local_object_storage/writecache/metrics.go | 3 +++ pkg/local_object_storage/writecache/writecache.go | 1 + pkg/metrics/writecache.go | 10 ++++++++++ 4 files changed, 18 insertions(+) diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index b947f12f4..2e518c6ff 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -102,3 +102,7 @@ func (m *writeCacheMetrics) Evict(st writecache.StorageType) { m.metrics.DecActualCount(m.shardID, st.String()) m.metrics.IncOperationCounter(m.shardID, "Evict", metrics.NullBool{}, st.String()) } + +func (m *writeCacheMetrics) Close() { + m.metrics.Close(m.shardID) +} diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go index e20c7e65e..957bf2770 100644 --- a/pkg/local_object_storage/writecache/metrics.go +++ b/pkg/local_object_storage/writecache/metrics.go @@ -28,6 +28,7 @@ type Metrics interface { SetEstimateSize(db, fstree uint64) SetMode(m mode.Mode) SetActualCounters(db, fstree uint64) + Close() } type metricsStub struct{} @@ -47,3 +48,5 @@ func (s *metricsStub) SetActualCounters(uint64, uint64) {} func (s *metricsStub) Flush(bool, StorageType) {} func (s *metricsStub) Evict(StorageType) {} + +func (s *metricsStub) Close() {} diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 664beff80..6fa0e36de 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -170,5 +170,6 @@ func (c *cache) Close() error { c.db = nil } } + c.metrics.Close() return nil } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 33c58fc65..31636a295 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -34,6 +34,8 @@ type WriteCacheMetrics interface { SetMode(shardID string, mode string) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) + + Close(shardID string) } type writeCacheMetrics struct { @@ -155,6 +157,14 @@ func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string }).Inc() } +func (m *writeCacheMetrics) Close(shardID string) { + m.SetMode(shardID, "CLOSED") + m.methodDuration.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) + m.operationCounter.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) + m.actualCount.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) + m.estimatedSize.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) +} + func newWCGaugeVec(name, help string, labels []string) *prometheus.GaugeVec { return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, From c8023a9c8d2c45ac03a53a62daaf218e44e399fc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 16:15:59 +0300 Subject: [PATCH 0331/1943] [#424] morphcache: Use labels for method duration Signed-off-by: Dmitrii Stepanov --- pkg/metrics/morphcache.go | 58 +++++++++----------------------------- pkg/morph/client/nns.go | 4 +-- pkg/morph/client/notary.go | 2 +- 3 files changed, 16 insertions(+), 48 deletions(-) diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go index 000ec0d80..9cf2c8a0c 100644 --- a/pkg/metrics/morphcache.go +++ b/pkg/metrics/morphcache.go @@ -1,7 +1,6 @@ package metrics import ( - "fmt" "strconv" "time" @@ -12,66 +11,35 @@ import ( const ( mcSubsystem = "morphcache" mcSuccess = "success" + mcMethod = "method" ) type MorphCacheMetrics interface { - AddNNSContractHashDuration(success bool, d time.Duration) - - AddGroupKeyDuration(success bool, d time.Duration) - - AddTxHeightDuration(success bool, d time.Duration) + AddMethodDuration(method string, success bool, d time.Duration) } type morphCacheMetrics struct { - // Duration of processing get nns contract hash request - nnsContractHashDuration *prometheus.HistogramVec - - // Duration of processing get group key request - groupKeyDuration *prometheus.HistogramVec - - // Duration of processing get tx height request - txHeightDuration *prometheus.HistogramVec + methodDuration *prometheus.HistogramVec } var _ MorphCacheMetrics = (*morphCacheMetrics)(nil) func newMorphCacheMetrics() *morphCacheMetrics { return &morphCacheMetrics{ - nnsContractHashDuration: newMCMethodDurationCounter("nns_contract_hash"), - groupKeyDuration: newMCMethodDurationCounter("group_key"), - txHeightDuration: newMCMethodDurationCounter("tx_height"), + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: mcSubsystem, + Name: "request_duration_seconds", + Help: "Morph cache request process duration", + }, []string{mcSuccess, mcMethod}), } } -func (m *morphCacheMetrics) AddNNSContractHashDuration(success bool, d time.Duration) { - m.nnsContractHashDuration.With( +func (m *morphCacheMetrics) AddMethodDuration(method string, success bool, d time.Duration) { + m.methodDuration.With( prometheus.Labels{ mcSuccess: strconv.FormatBool(success), + mcMethod: method, }, - ).Observe(float64(d)) -} - -func (m *morphCacheMetrics) AddGroupKeyDuration(success bool, d time.Duration) { - m.groupKeyDuration.With( - prometheus.Labels{ - mcSuccess: strconv.FormatBool(success), - }, - ).Observe(float64(d)) -} - -func (m *morphCacheMetrics) AddTxHeightDuration(success bool, d time.Duration) { - m.txHeightDuration.With( - prometheus.Labels{ - mcSuccess: strconv.FormatBool(success), - }, - ).Observe(float64(d)) -} - -func newMCMethodDurationCounter(method string) *prometheus.HistogramVec { - return metrics.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: mcSubsystem, - Name: fmt.Sprintf("%s_req_duration_seconds", method), - Help: fmt.Sprintf("Accumulated %s request process duration", method), - }, []string{mcSuccess}) + ).Observe(d.Seconds()) } diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index d8d798b07..53dbe180e 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -86,7 +86,7 @@ func (c *Client) NNSHash() (util.Uint160, error) { startedAt := time.Now() defer func() { - c.cache.metrics.AddNNSContractHashDuration(success, time.Since(startedAt)) + c.cache.metrics.AddMethodDuration("NNSContractHash", success, time.Since(startedAt)) }() nnsHash := c.cache.nns() @@ -233,7 +233,7 @@ func (c *Client) contractGroupKey() (*keys.PublicKey, error) { success := false startedAt := time.Now() defer func() { - c.cache.metrics.AddGroupKeyDuration(success, time.Since(startedAt)) + c.cache.metrics.AddMethodDuration("GroupKey", success, time.Since(startedAt)) }() if gKey := c.cache.groupKey(); gKey != nil { diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index e567b6df0..17644361a 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -800,7 +800,7 @@ func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { success := false startedAt := time.Now() defer func() { - c.cache.metrics.AddTxHeightDuration(success, time.Since(startedAt)) + c.cache.metrics.AddMethodDuration("TxHeight", success, time.Since(startedAt)) }() if rh, ok := c.cache.txHeights.Get(h); ok { From 1b364d8cf4db87b07909331584c86ee0347e1bdf Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 13 Jun 2023 19:48:15 +0300 Subject: [PATCH 0332/1943] [#424] metrics: Refactor engine metrics Use histogram vector to measure request duration. Fix naming like in Prometheus best practice. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/container.go | 4 +- pkg/local_object_storage/engine/delete.go | 2 +- pkg/local_object_storage/engine/get.go | 2 +- pkg/local_object_storage/engine/head.go | 2 +- pkg/local_object_storage/engine/inhume.go | 2 +- pkg/local_object_storage/engine/metrics.go | 18 +-- pkg/local_object_storage/engine/put.go | 2 +- pkg/local_object_storage/engine/range.go | 2 +- pkg/local_object_storage/engine/select.go | 4 +- pkg/local_object_storage/engine/shards.go | 6 +- .../shard/metrics_test.go | 2 +- pkg/local_object_storage/shard/shard.go | 8 +- pkg/metrics/engine.go | 114 ++++-------------- 13 files changed, 46 insertions(+), 122 deletions(-) diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 034837110..061e2fea0 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -67,7 +67,7 @@ func ContainerSize(e *StorageEngine, id cid.ID) (uint64, error) { func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRes, err error) { if e.metrics != nil { - defer elapsed(e.metrics.AddEstimateContainerSizeDuration)() + defer elapsed("EstimateContainerSize", e.metrics.AddMethodDuration)() } e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { @@ -115,7 +115,7 @@ func ListContainers(e *StorageEngine) ([]cid.ID, error) { func (e *StorageEngine) listContainers() (ListContainersRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddListContainersDuration)() + defer elapsed("ListContainers", e.metrics.AddMethodDuration)() } uniqueIDs := make(map[string]cid.ID) diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 2125fad72..4d6d838bc 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -67,7 +67,7 @@ func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRe func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddDeleteDuration)() + defer elapsed("Delete", e.metrics.AddMethodDuration)() } var locked struct { diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index d376198ba..bd094770a 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -64,7 +64,7 @@ func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err er func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddGetDuration)() + defer elapsed("Get", e.metrics.AddMethodDuration)() } var errNotFound apistatus.ObjectNotFound diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index 5da97bab5..ca51015db 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -68,7 +68,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) defer span.End() if e.metrics != nil { - defer elapsed(e.metrics.AddHeadDuration)() + defer elapsed("Head", e.metrics.AddMethodDuration)() } var ( diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 57ac52751..0b9ae602b 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -78,7 +78,7 @@ func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRe func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddInhumeDuration)() + defer elapsed("Inhume", e.metrics.AddMethodDuration)() } var shPrm shard.InhumePrm diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index f9e9191cd..dd1240ea7 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -7,17 +7,7 @@ import ( ) type MetricRegister interface { - AddListContainersDuration(d time.Duration) - AddEstimateContainerSizeDuration(d time.Duration) - AddDeleteDuration(d time.Duration) - AddExistsDuration(d time.Duration) - AddGetDuration(d time.Duration) - AddHeadDuration(d time.Duration) - AddInhumeDuration(d time.Duration) - AddPutDuration(d time.Duration) - AddRangeDuration(d time.Duration) - AddSearchDuration(d time.Duration) - AddListObjectsDuration(d time.Duration) + AddMethodDuration(method string, d time.Duration) SetObjectCounter(shardID, objectType string, v uint64) AddToObjectCounter(shardID, objectType string, delta int) @@ -28,17 +18,17 @@ type MetricRegister interface { AddToPayloadCounter(shardID string, size int64) IncErrorCounter(shardID string) ClearErrorCounter(shardID string) - DeleteErrorCounter(shardID string) + DeleteShardMetrics(shardID string) WriteCache() metrics.WriteCacheMetrics GC() metrics.GCMetrics } -func elapsed(addFunc func(d time.Duration)) func() { +func elapsed(method string, addFunc func(method string, d time.Duration)) func() { t := time.Now() return func() { - addFunc(time.Since(t)) + addFunc(method, time.Since(t)) } } diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 4ac7f90f9..98c4504e6 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -57,7 +57,7 @@ func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) { func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { if e.metrics != nil { - defer elapsed(e.metrics.AddPutDuration)() + defer elapsed("Put", e.metrics.AddMethodDuration)() } addr := object.AddressOf(prm.obj) diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 29ec8b2bc..328df4587 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -80,7 +80,7 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error defer span.End() if e.metrics != nil { - defer elapsed(e.metrics.AddRangeDuration)() + defer elapsed("GetRange", e.metrics.AddMethodDuration)() } var errNotFound apistatus.ObjectNotFound diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 48d2be674..9f651845f 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -60,7 +60,7 @@ func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRe func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddSearchDuration)() + defer elapsed("Search", e.metrics.AddMethodDuration)() } addrList := make([]oid.Address, 0) @@ -109,7 +109,7 @@ func (e *StorageEngine) List(limit uint64) (res SelectRes, err error) { func (e *StorageEngine) list(limit uint64) (SelectRes, error) { if e.metrics != nil { - defer elapsed(e.metrics.AddListObjectsDuration)() + defer elapsed("ListObjects", e.metrics.AddMethodDuration)() } addrList := make([]oid.Address, 0, limit) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index fd9605e9b..5f4f13635 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -70,8 +70,8 @@ func (m *metricsWithID) ClearErrorCounter() { m.mw.ClearErrorCounter(m.id) } -func (m *metricsWithID) DeleteErrorCounter() { - m.mw.DeleteErrorCounter(m.id) +func (m *metricsWithID) DeleteShardMetrics() { + m.mw.DeleteShardMetrics(m.id) } // AddShard adds a new shard to the storage engine. @@ -186,7 +186,7 @@ func (e *StorageEngine) removeShards(ids ...string) { continue } - sh.DeleteErrorCounter() + sh.DeleteShardMetrics() ss = append(ss, sh) delete(e.shards, id) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index f1581b6d4..f9e1bc760 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -77,7 +77,7 @@ func (m *metricsStore) ClearErrorCounter() { m.errCounter = 0 } -func (m *metricsStore) DeleteErrorCounter() { +func (m *metricsStore) DeleteShardMetrics() { m.errCounter = 0 } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index b740fc572..8c4db87b5 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -79,8 +79,8 @@ type MetricsWriter interface { IncErrorCounter() // ClearErrorCounter clear error counter. ClearErrorCounter() - // DeleteErrorCounter delete error counter. - DeleteErrorCounter() + // DeleteShardMetrics deletes shard metrics from registry. + DeleteShardMetrics() } type cfg struct { @@ -447,8 +447,8 @@ func (s *Shard) ClearErrorCounter() { } } -func (s *Shard) DeleteErrorCounter() { +func (s *Shard) DeleteShardMetrics() { if s.cfg.metricsWriter != nil { - s.cfg.metricsWriter.DeleteErrorCounter() + s.cfg.metricsWriter.DeleteShardMetrics() } } diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 7992da9f8..4b32344e4 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -1,8 +1,6 @@ package metrics import ( - "fmt" - "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -11,60 +9,33 @@ import ( type ( engineMetrics struct { - listContainersDuration prometheus.Counter - estimateContainerSizeDuration prometheus.Counter - deleteDuration prometheus.Counter - existsDuration prometheus.Counter - getDuration prometheus.Counter - headDuration prometheus.Counter - inhumeDuration prometheus.Counter - putDuration prometheus.Counter - rangeDuration prometheus.Counter - searchDuration prometheus.Counter - listObjectsDuration prometheus.Counter - containerSize *prometheus.GaugeVec - payloadSize *prometheus.GaugeVec - errorCounter *prometheus.GaugeVec + methodDuration *prometheus.HistogramVec + + containerSize *prometheus.GaugeVec + payloadSize *prometheus.GaugeVec + errorCounter *prometheus.GaugeVec } ) -const engineSubsystem = "engine" +const ( + engineSubsystem = "engine" + engineMethod = "method" +) func newEngineMetrics() engineMetrics { return engineMetrics{ - listContainersDuration: newEngineMethodDurationCounter("list_containers_"), - estimateContainerSizeDuration: newEngineCounter("estimate_container_size_duration", "Accumulated duration of engine container size estimate operations"), - deleteDuration: newEngineMethodDurationCounter("delete"), - existsDuration: newEngineMethodDurationCounter("exists"), - getDuration: newEngineMethodDurationCounter("get"), - headDuration: newEngineMethodDurationCounter("head"), - inhumeDuration: newEngineMethodDurationCounter("inhume"), - putDuration: newEngineMethodDurationCounter("put"), - rangeDuration: newEngineMethodDurationCounter("range"), - searchDuration: newEngineMethodDurationCounter("search"), - listObjectsDuration: newEngineMethodDurationCounter("list_objects"), - containerSize: newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), - payloadSize: newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), - errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}), + containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), + payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), + errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}), + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: engineSubsystem, + Name: "request_duration_seconds", + Help: "Duration of Engine requests", + }, []string{engineMethod}), } } -func newEngineCounter(name, help string) prometheus.Counter { - return metrics.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: engineSubsystem, - Name: name, - Help: help, - }) -} - -func newEngineMethodDurationCounter(method string) prometheus.Counter { - return newEngineCounter( - fmt.Sprintf("%s_duration", method), - fmt.Sprintf("Accumulated duration of engine %s operations", strings.ReplaceAll(method, "_", " ")), - ) -} - func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, @@ -74,48 +45,10 @@ func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeV }, labels) } -func (m engineMetrics) AddListContainersDuration(d time.Duration) { - m.listObjectsDuration.Add(float64(d)) -} - -func (m engineMetrics) AddEstimateContainerSizeDuration(d time.Duration) { - m.estimateContainerSizeDuration.Add(float64(d)) -} - -func (m engineMetrics) AddDeleteDuration(d time.Duration) { - m.deleteDuration.Add(float64(d)) -} - -func (m engineMetrics) AddExistsDuration(d time.Duration) { - m.existsDuration.Add(float64(d)) -} - -func (m engineMetrics) AddGetDuration(d time.Duration) { - m.getDuration.Add(float64(d)) -} - -func (m engineMetrics) AddHeadDuration(d time.Duration) { - m.headDuration.Add(float64(d)) -} - -func (m engineMetrics) AddInhumeDuration(d time.Duration) { - m.inhumeDuration.Add(float64(d)) -} - -func (m engineMetrics) AddPutDuration(d time.Duration) { - m.putDuration.Add(float64(d)) -} - -func (m engineMetrics) AddRangeDuration(d time.Duration) { - m.rangeDuration.Add(float64(d)) -} - -func (m engineMetrics) AddSearchDuration(d time.Duration) { - m.searchDuration.Add(float64(d)) -} - -func (m engineMetrics) AddListObjectsDuration(d time.Duration) { - m.listObjectsDuration.Add(float64(d)) +func (m *engineMetrics) AddMethodDuration(method string, d time.Duration) { + m.methodDuration.With(prometheus.Labels{ + engineMethod: method, + }).Observe(d.Seconds()) } func (m engineMetrics) AddToContainerSize(cnrID string, size int64) { @@ -134,6 +67,7 @@ func (m engineMetrics) ClearErrorCounter(shardID string) { m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0) } -func (m engineMetrics) DeleteErrorCounter(shardID string) { +func (m engineMetrics) DeleteShardMetrics(shardID string) { m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID}) + m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID}) } From c348ae35b0b7a856219e9ab98f2fc5e1ca31db72 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 10:05:51 +0300 Subject: [PATCH 0333/1943] [#424] metrics: Drop embedded metrics It was not obvious where metrics are used. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/control.go | 2 +- cmd/frostfs-node/object.go | 4 +- pkg/metrics/engine.go | 91 +++++++++++--- pkg/metrics/innerring.go | 8 +- pkg/metrics/node.go | 57 ++++----- pkg/metrics/object.go | 218 +++++---------------------------- pkg/metrics/replicator.go | 21 +++- pkg/metrics/state.go | 10 +- pkg/services/object/metrics.go | 45 ++----- 10 files changed, 173 insertions(+), 285 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 758803291..c88b7224e 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -679,7 +679,7 @@ func (c *cfg) engineOpts() []engine.Option { ) if c.metricsCollector != nil { - opts = append(opts, engine.WithMetrics(c.metricsCollector)) + opts = append(opts, engine.WithMetrics(c.metricsCollector.Engine())) } return opts diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index f4b068419..3ed6bc54a 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -80,7 +80,7 @@ func (c *cfg) setHealthStatus(st control.HealthStatus) { c.healthStatus.Store(int32(st)) if c.metricsCollector != nil { - c.metricsCollector.SetHealth(int32(st)) + c.metricsCollector.State().SetHealth(int32(st)) } } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 7a409da03..0476dbcac 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -195,7 +195,7 @@ func initObjectService(c *cfg) { ) c.shared.metricsSvc = objectService.NewMetricCollector( - signSvc, c.metricsCollector, metricsconfig.Enabled(c.appCfg)) + signSvc, c.metricsCollector.ObjectService(), metricsconfig.Enabled(c.appCfg)) server := objectTransportGRPC.New(c.shared.metricsSvc) for _, srv := range c.cfgGRPC.servers { @@ -272,7 +272,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa replicator.WithRemoteSender( putsvc.NewRemoteSender(keyStorage, cache), ), - replicator.WithMetrics(c.metricsCollector), + replicator.WithMetrics(c.metricsCollector.Replicator()), ) } diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 4b32344e4..19e3591a6 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -7,23 +7,40 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -type ( - engineMetrics struct { - methodDuration *prometheus.HistogramVec +type EngineMetrics interface { + AddMethodDuration(method string, d time.Duration) + AddToContainerSize(cnrID string, size int64) + IncErrorCounter(shardID string) + ClearErrorCounter(shardID string) + DeleteShardMetrics(shardID string) + AddToObjectCounter(shardID, objectType string, delta int) + SetObjectCounter(shardID, objectType string, v uint64) + AddToPayloadCounter(shardID string, size int64) + SetReadonly(shardID string, readonly bool) - containerSize *prometheus.GaugeVec - payloadSize *prometheus.GaugeVec - errorCounter *prometheus.GaugeVec - } -) + WriteCache() WriteCacheMetrics + GC() GCMetrics +} + +type engineMetrics struct { + methodDuration *prometheus.HistogramVec + objectCounter *prometheus.GaugeVec + containerSize *prometheus.GaugeVec + payloadSize *prometheus.GaugeVec + errorCounter *prometheus.GaugeVec + shardsReadonly *prometheus.GaugeVec + + gc *gcMetrics + writeCache *writeCacheMetrics +} const ( engineSubsystem = "engine" engineMethod = "method" ) -func newEngineMetrics() engineMetrics { - return engineMetrics{ +func newEngineMetrics() *engineMetrics { + return &engineMetrics{ containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}), @@ -33,6 +50,10 @@ func newEngineMetrics() engineMetrics { Name: "request_duration_seconds", Help: "Duration of Engine requests", }, []string{engineMethod}), + objectCounter: newEngineGaugeVector("object_counter", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), + shardsReadonly: newEngineGaugeVector("mode", "Shard mode", []string{shardIDLabelKey}), + gc: newGCMetrics(), + writeCache: newWriteCacheMetrics(), } } @@ -51,23 +72,63 @@ func (m *engineMetrics) AddMethodDuration(method string, d time.Duration) { }).Observe(d.Seconds()) } -func (m engineMetrics) AddToContainerSize(cnrID string, size int64) { +func (m *engineMetrics) AddToContainerSize(cnrID string, size int64) { m.containerSize.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) } -func (m engineMetrics) AddToPayloadCounter(shardID string, size int64) { +func (m *engineMetrics) AddToPayloadCounter(shardID string, size int64) { m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) } -func (m engineMetrics) IncErrorCounter(shardID string) { +func (m *engineMetrics) IncErrorCounter(shardID string) { m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Inc() } -func (m engineMetrics) ClearErrorCounter(shardID string) { +func (m *engineMetrics) ClearErrorCounter(shardID string) { m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0) } -func (m engineMetrics) DeleteShardMetrics(shardID string) { +func (m *engineMetrics) DeleteShardMetrics(shardID string) { m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID}) m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID}) + m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabelKey: shardID}) + m.shardsReadonly.Delete(prometheus.Labels{shardIDLabelKey: shardID}) +} + +func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) { + m.objectCounter.With( + prometheus.Labels{ + shardIDLabelKey: shardID, + counterTypeLabelKey: objectType, + }, + ).Add(float64(delta)) +} + +func (m *engineMetrics) SetObjectCounter(shardID, objectType string, v uint64) { + m.objectCounter.With( + prometheus.Labels{ + shardIDLabelKey: shardID, + counterTypeLabelKey: objectType, + }, + ).Set(float64(v)) +} + +func (m *engineMetrics) SetReadonly(shardID string, readonly bool) { + var flag float64 + if readonly { + flag = 1 + } + m.shardsReadonly.With( + prometheus.Labels{ + shardIDLabelKey: shardID, + }, + ).Set(flag) +} + +func (m *engineMetrics) WriteCache() WriteCacheMetrics { + return m.writeCache +} + +func (m *engineMetrics) GC() GCMetrics { + return m.gc } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index d756a539f..b3cb3e285 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -54,22 +54,22 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { } // SetEpoch updates epoch metrics. -func (m InnerRingServiceMetrics) SetEpoch(epoch uint64) { +func (m *InnerRingServiceMetrics) SetEpoch(epoch uint64) { m.epoch.Set(float64(epoch)) } // SetHealth updates health metrics. -func (m InnerRingServiceMetrics) SetHealth(s int32) { +func (m *InnerRingServiceMetrics) SetHealth(s int32) { m.health.Set(float64(s)) } -func (m InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) { +func (m *InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) { m.eventDuration.With(prometheus.Labels{ innerRingLabelType: typ, innerRingLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) } -func (m InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { +func (m *InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { return m.morphCacheMetrics } diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 16a4c2b5b..232c178d8 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -8,15 +8,12 @@ import ( const namespace = "frostfs_node" type NodeMetrics struct { - objectServiceMetrics - engineMetrics - stateMetrics - replicatorMetrics - - writeCacheMetrics *writeCacheMetrics - treeService *treeServiceMetrics - epoch prometheus.Gauge - gc *gcMetrics + engine *engineMetrics + state *stateMetrics + replicator *replicatorMetrics + objectService *objectServiceMetrics + treeService *treeServiceMetrics + epoch prometheus.Gauge } func NewNodeMetrics() *NodeMetrics { @@ -37,19 +34,13 @@ func NewNodeMetrics() *NodeMetrics { Help: "Current epoch as seen by inner-ring node.", }) - writeCacheMetrics := newWriteCacheMetrics() - - gc := newGCMetrics() - return &NodeMetrics{ - objectServiceMetrics: objectService, - engineMetrics: engine, - stateMetrics: state, - replicatorMetrics: replicator, - treeService: treeService, - epoch: epoch, - writeCacheMetrics: writeCacheMetrics, - gc: gc, + objectService: objectService, + engine: engine, + state: state, + replicator: replicator, + treeService: treeService, + epoch: epoch, } } @@ -58,18 +49,22 @@ func (m *NodeMetrics) SetEpoch(epoch uint64) { m.epoch.Set(float64(epoch)) } -// WriteCache returns WriteCache metrics. -func (m *NodeMetrics) WriteCache() WriteCacheMetrics { - if m == nil { - return nil - } - return m.writeCacheMetrics -} - func (m *NodeMetrics) TreeService() TreeMetricsRegister { return m.treeService } -func (m *NodeMetrics) GC() GCMetrics { - return m.gc +func (m *NodeMetrics) Replicator() ReplicatorMetrics { + return m.replicator +} + +func (m *NodeMetrics) ObjectService() ObjectServiceMetrics { + return m.objectService +} + +func (m *NodeMetrics) Engine() EngineMetrics { + return m.engine +} + +func (m *NodeMetrics) State() StateMetrics { + return m.state } diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index 87916414d..6e1b3a5c4 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -1,8 +1,7 @@ package metrics import ( - "fmt" - "strings" + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -11,207 +10,50 @@ import ( const objectSubsystem = "object" -type ( - methodCount struct { - success prometheus.Counter - total prometheus.Counter - } +type ObjectServiceMetrics interface { + AddRequestDuration(method string, d time.Duration, success bool) + AddPayloadSize(method string, size int) +} - objectServiceMetrics struct { - getCounter methodCount - putCounter methodCount - headCounter methodCount - searchCounter methodCount - deleteCounter methodCount - rangeCounter methodCount - rangeHashCounter methodCount - - getDuration prometheus.Counter - putDuration prometheus.Counter - headDuration prometheus.Counter - searchDuration prometheus.Counter - deleteDuration prometheus.Counter - rangeDuration prometheus.Counter - rangeHashDuration prometheus.Counter - - putPayload prometheus.Counter - getPayload prometheus.Counter - - shardMetrics *prometheus.GaugeVec - shardsReadonly *prometheus.GaugeVec - } -) +type objectServiceMetrics struct { + methodDuration *prometheus.HistogramVec + payloadCounter *prometheus.CounterVec +} const ( shardIDLabelKey = "shard" counterTypeLabelKey = "type" containerIDLabelKey = "cid" + methodLabelKey = "method" + successLabelKey = "success" ) -func newObjectMethodCallCounter(name string) methodCount { - return methodCount{ - success: metrics.NewCounter(prometheus.CounterOpts{ +func newObjectServiceMetrics() *objectServiceMetrics { + return &objectServiceMetrics{ + methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_req_count_success", name), - Help: fmt.Sprintf("The number of successful %s requests processed", name), - }), - total: metrics.NewCounter(prometheus.CounterOpts{ + Name: "request_duration_seconds", + Help: "Object Service request process duration", + }, []string{methodLabelKey, successLabelKey}), + payloadCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_req_count", name), - Help: fmt.Sprintf("Total number of %s requests processed", name), - }), + Name: "request_payload_bytes", + Help: "Object Service request payload", + }, []string{methodLabelKey}), } } -func (m methodCount) Inc(success bool) { - m.total.Inc() - if success { - m.success.Inc() - } +func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool) { + m.methodDuration.With(prometheus.Labels{ + methodLabelKey: method, + successLabelKey: strconv.FormatBool(success), + }).Observe(d.Seconds()) } -func newObjectServiceMetrics() objectServiceMetrics { - return objectServiceMetrics{ - getCounter: newObjectMethodCallCounter("get"), - putCounter: newObjectMethodCallCounter("put"), - headCounter: newObjectMethodCallCounter("head"), - searchCounter: newObjectMethodCallCounter("search"), - deleteCounter: newObjectMethodCallCounter("delete"), - rangeCounter: newObjectMethodCallCounter("range"), - rangeHashCounter: newObjectMethodCallCounter("range_hash"), - getDuration: newObjectMethodDurationCounter("get"), - putDuration: newObjectMethodDurationCounter("put"), - headDuration: newObjectMethodDurationCounter("head"), - searchDuration: newObjectMethodDurationCounter("search"), - deleteDuration: newObjectMethodDurationCounter("delete"), - rangeDuration: newObjectMethodDurationCounter("range"), - rangeHashDuration: newObjectMethodDurationCounter("range_hash"), - putPayload: newObjectMethodPayloadCounter("put"), - getPayload: newObjectMethodPayloadCounter("get"), - shardMetrics: newObjectGaugeVector("counter", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), - shardsReadonly: newObjectGaugeVector("readonly", "Shard state", []string{shardIDLabelKey}), - } -} - -func newObjectMethodPayloadCounter(method string) prometheus.Counter { - return metrics.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_payload", method), - Help: fmt.Sprintf("Accumulated payload size at object %s method", strings.ReplaceAll(method, "_", " ")), - }) -} - -func newObjectMethodDurationCounter(method string) prometheus.Counter { - return metrics.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: fmt.Sprintf("%s_req_duration", method), - Help: fmt.Sprintf("Accumulated %s request process duration", strings.ReplaceAll(method, "_", " ")), - }) -} - -func newObjectGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { - return metrics.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: name, - Help: help, - }, labels) -} - -func (m objectServiceMetrics) IncGetReqCounter(success bool) { - m.getCounter.Inc(success) -} - -func (m objectServiceMetrics) IncPutReqCounter(success bool) { - m.putCounter.Inc(success) -} - -func (m objectServiceMetrics) IncHeadReqCounter(success bool) { - m.headCounter.Inc(success) -} - -func (m objectServiceMetrics) IncSearchReqCounter(success bool) { - m.searchCounter.Inc(success) -} - -func (m objectServiceMetrics) IncDeleteReqCounter(success bool) { - m.deleteCounter.Inc(success) -} - -func (m objectServiceMetrics) IncRangeReqCounter(success bool) { - m.rangeCounter.Inc(success) -} - -func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) { - m.rangeHashCounter.Inc(success) -} - -func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) { - m.getDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddPutReqDuration(d time.Duration) { - m.putDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddHeadReqDuration(d time.Duration) { - m.headDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddSearchReqDuration(d time.Duration) { - m.searchDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddDeleteReqDuration(d time.Duration) { - m.deleteDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddRangeReqDuration(d time.Duration) { - m.rangeDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddRangeHashReqDuration(d time.Duration) { - m.rangeHashDuration.Add(float64(d)) -} - -func (m objectServiceMetrics) AddPutPayload(ln int) { - m.putPayload.Add(float64(ln)) -} - -func (m objectServiceMetrics) AddGetPayload(ln int) { - m.getPayload.Add(float64(ln)) -} - -func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, delta int) { - m.shardMetrics.With( - prometheus.Labels{ - shardIDLabelKey: shardID, - counterTypeLabelKey: objectType, - }, - ).Add(float64(delta)) -} - -func (m objectServiceMetrics) SetObjectCounter(shardID, objectType string, v uint64) { - m.shardMetrics.With( - prometheus.Labels{ - shardIDLabelKey: shardID, - counterTypeLabelKey: objectType, - }, - ).Set(float64(v)) -} - -func (m objectServiceMetrics) SetReadonly(shardID string, readonly bool) { - var flag float64 - if readonly { - flag = 1 - } - m.shardsReadonly.With( - prometheus.Labels{ - shardIDLabelKey: shardID, - }, - ).Set(flag) +func (m *objectServiceMetrics) AddPayloadSize(method string, size int) { + m.payloadCounter.With(prometheus.Labels{ + methodLabelKey: method, + }).Add(float64(size)) } diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go index 0deafe916..dbcc67338 100644 --- a/pkg/metrics/replicator.go +++ b/pkg/metrics/replicator.go @@ -7,30 +7,39 @@ import ( const replicatorSubsystem = "replicator" +//TODO + +type ReplicatorMetrics interface { + IncInFlightRequest() + DecInFlightRequest() + IncProcessedObjects() + AddPayloadSize(size int64) +} + type replicatorMetrics struct { inFlightRequests prometheus.Gauge processedObjects prometheus.Counter totalReplicatedPayloadSize prometheus.Counter } -func (m replicatorMetrics) IncInFlightRequest() { +func (m *replicatorMetrics) IncInFlightRequest() { m.inFlightRequests.Inc() } -func (m replicatorMetrics) DecInFlightRequest() { +func (m *replicatorMetrics) DecInFlightRequest() { m.inFlightRequests.Dec() } -func (m replicatorMetrics) IncProcessedObjects() { +func (m *replicatorMetrics) IncProcessedObjects() { m.processedObjects.Inc() } -func (m replicatorMetrics) AddPayloadSize(size int64) { +func (m *replicatorMetrics) AddPayloadSize(size int64) { m.totalReplicatedPayloadSize.Add(float64(size)) } -func newReplicatorMetrics() replicatorMetrics { - return replicatorMetrics{ +func newReplicatorMetrics() *replicatorMetrics { + return &replicatorMetrics{ inFlightRequests: newReplicatorGauge("in_flight_requests", "Number of in-flight requests"), processedObjects: newReplicatorCounter("processed_objects", "Number of objects processed since the node startup"), totalReplicatedPayloadSize: newReplicatorCounter("total_replicated_payload_size", "Total size of payloads replicated"), diff --git a/pkg/metrics/state.go b/pkg/metrics/state.go index 893849911..76a8d8ac9 100644 --- a/pkg/metrics/state.go +++ b/pkg/metrics/state.go @@ -7,12 +7,16 @@ import ( const stateSubsystem = "state" +type StateMetrics interface { + SetHealth(s int32) +} + type stateMetrics struct { healthCheck prometheus.Gauge } -func newStateMetrics() stateMetrics { - return stateMetrics{ +func newStateMetrics() *stateMetrics { + return &stateMetrics{ healthCheck: metrics.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: stateSubsystem, @@ -22,6 +26,6 @@ func newStateMetrics() stateMetrics { } } -func (m stateMetrics) SetHealth(s int32) { +func (m *stateMetrics) SetHealth(s int32) { m.healthCheck.Set(float64(s)) } diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index 3ea16dafd..487374940 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -28,24 +28,8 @@ type ( } MetricRegister interface { - IncGetReqCounter(success bool) - IncPutReqCounter(success bool) - IncHeadReqCounter(success bool) - IncSearchReqCounter(success bool) - IncDeleteReqCounter(success bool) - IncRangeReqCounter(success bool) - IncRangeHashReqCounter(success bool) - - AddGetReqDuration(time.Duration) - AddPutReqDuration(time.Duration) - AddHeadReqDuration(time.Duration) - AddSearchReqDuration(time.Duration) - AddDeleteReqDuration(time.Duration) - AddRangeReqDuration(time.Duration) - AddRangeHashReqDuration(time.Duration) - - AddPutPayload(int) - AddGetPayload(int) + AddRequestDuration(string, time.Duration, bool) + AddPayloadSize(string, int) } ) @@ -61,8 +45,7 @@ func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (er if m.enabled { t := time.Now() defer func() { - m.metrics.IncGetReqCounter(err == nil) - m.metrics.AddGetReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Get", time.Since(t), err == nil) }() err = m.next.Get(req, &getStreamMetric{ ServerStream: stream, @@ -99,8 +82,7 @@ func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) res, err := m.next.Head(ctx, request) - m.metrics.IncHeadReqCounter(err == nil) - m.metrics.AddHeadReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Head", time.Since(t), err == nil) return res, err } @@ -113,8 +95,7 @@ func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream) err := m.next.Search(req, stream) - m.metrics.IncSearchReqCounter(err == nil) - m.metrics.AddSearchReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Search", time.Since(t), err == nil) return err } @@ -127,8 +108,7 @@ func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteReque res, err := m.next.Delete(ctx, request) - m.metrics.IncDeleteReqCounter(err == nil) - m.metrics.AddDeleteReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("Delete", time.Since(t), err == nil) return res, err } return m.next.Delete(ctx, request) @@ -140,8 +120,7 @@ func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectR err := m.next.GetRange(req, stream) - m.metrics.IncRangeReqCounter(err == nil) - m.metrics.AddRangeReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("GetRange", time.Since(t), err == nil) return err } @@ -154,8 +133,7 @@ func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRa res, err := m.next.GetRangeHash(ctx, request) - m.metrics.IncRangeHashReqCounter(err == nil) - m.metrics.AddRangeHashReqDuration(time.Since(t)) + m.metrics.AddRequestDuration("GetRangeHash", time.Since(t), err == nil) return res, err } @@ -173,7 +151,7 @@ func (m *MetricCollector) Disable() { func (s getStreamMetric) Send(resp *object.GetResponse) error { chunk, ok := resp.GetBody().GetObjectPart().(*object.GetObjectPartChunk) if ok { - s.metrics.AddGetPayload(len(chunk.GetChunk())) + s.metrics.AddPayloadSize("Get", len(chunk.GetChunk())) } return s.stream.Send(resp) @@ -182,7 +160,7 @@ func (s getStreamMetric) Send(resp *object.GetResponse) error { func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error { chunk, ok := req.GetBody().GetObjectPart().(*object.PutObjectPartChunk) if ok { - s.metrics.AddPutPayload(len(chunk.GetChunk())) + s.metrics.AddPayloadSize("Put", len(chunk.GetChunk())) } return s.stream.Send(ctx, req) @@ -191,8 +169,7 @@ func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { res, err := s.stream.CloseAndRecv(ctx) - s.metrics.IncPutReqCounter(err == nil) - s.metrics.AddPutReqDuration(time.Since(s.start)) + s.metrics.AddRequestDuration("Put", time.Since(s.start), err == nil) return res, err } From 847732605c025d6796dcb7923cccfe987cda6bbd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 10:16:19 +0300 Subject: [PATCH 0334/1943] [#424] metrics: Rename counter to total Suffix total should be used for a unit-less accumulating count. Signed-off-by: Dmitrii Stepanov --- pkg/metrics/engine.go | 4 ++-- pkg/metrics/gc.go | 4 ++-- pkg/metrics/morph.go | 4 ++-- pkg/metrics/replicator.go | 6 +++--- pkg/metrics/writecache.go | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 19e3591a6..d3f3238ac 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -43,14 +43,14 @@ func newEngineMetrics() *engineMetrics { return &engineMetrics{ containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), - errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}), + errorCounter: newEngineGaugeVector("errors_total", "Shard's error counter", []string{shardIDLabelKey}), methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: "request_duration_seconds", Help: "Duration of Engine requests", }, []string{engineMethod}), - objectCounter: newEngineGaugeVector("object_counter", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), + objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), shardsReadonly: newEngineGaugeVector("mode", "Shard mode", []string{shardIDLabelKey}), gc: newGCMetrics(), writeCache: newWriteCacheMetrics(), diff --git a/pkg/metrics/gc.go b/pkg/metrics/gc.go index 9c00d8722..c0319562c 100644 --- a/pkg/metrics/gc.go +++ b/pkg/metrics/gc.go @@ -43,7 +43,7 @@ func newGCMetrics() *gcMetrics { deletedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, - Name: "deleted_objects_count", + Name: "deleted_objects_total", Help: "Total count of objects GC deleted or failed to delete from disk", }, []string{gcShardID, gcStatus}), expCollectDuration: metrics.NewCounterVec(prometheus.CounterOpts{ @@ -55,7 +55,7 @@ func newGCMetrics() *gcMetrics { inhumedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, - Name: "marked_for_removal_objects_count", + Name: "marked_for_removal_objects_total", Help: "Total count of expired objects GC marked to remove", }, []string{gcShardID, gcObjectType}), } diff --git a/pkg/metrics/morph.go b/pkg/metrics/morph.go index cc851d3db..cd5deb5e6 100644 --- a/pkg/metrics/morph.go +++ b/pkg/metrics/morph.go @@ -30,7 +30,7 @@ func NewMorphClientMetrics() morphmetrics.Register { switchCount: metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: morphSubsystem, - Name: "switch_count", + Name: "switches_total", Help: "Number of endpoint switches", }), lastBlock: metrics.NewGauge(prometheus.GaugeOpts{ @@ -42,7 +42,7 @@ func NewMorphClientMetrics() morphmetrics.Register { notificationCount: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: morphSubsystem, - Name: "notification_count", + Name: "notifications_total", Help: "Number of notifications received by notification type", }, []string{morphNotificationTypeLabel}), invokeDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go index dbcc67338..f16c296c6 100644 --- a/pkg/metrics/replicator.go +++ b/pkg/metrics/replicator.go @@ -40,9 +40,9 @@ func (m *replicatorMetrics) AddPayloadSize(size int64) { func newReplicatorMetrics() *replicatorMetrics { return &replicatorMetrics{ - inFlightRequests: newReplicatorGauge("in_flight_requests", "Number of in-flight requests"), - processedObjects: newReplicatorCounter("processed_objects", "Number of objects processed since the node startup"), - totalReplicatedPayloadSize: newReplicatorCounter("total_replicated_payload_size", "Total size of payloads replicated"), + inFlightRequests: newReplicatorGauge("in_flight_requests_total", "Number of in-flight requests"), + processedObjects: newReplicatorCounter("processed_objects_total", "Number of objects processed since the node startup"), + totalReplicatedPayloadSize: newReplicatorCounter("total_replicated_payload_size_bytes", "Total size of payloads replicated"), } } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 31636a295..8498e3930 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -62,10 +62,10 @@ func newWriteCacheMetrics() *writeCacheMetrics { operationCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: wcSubsystem, - Name: "operation_count", + Name: "operations_total", Help: "The number of writecache operations processed", }, []string{wcShardID, wcStorage, wcSuccess, wcOperation}), - actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}), + actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}), estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), modeMtx: sync.RWMutex{}, modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc), From 4449006862cafebaafc67424a2fc777e3f85006b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 11:00:44 +0300 Subject: [PATCH 0335/1943] [#424] metrics: Use mode value as metric value for shard Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/metrics.go | 3 +- pkg/local_object_storage/engine/shards.go | 6 +-- .../shard/metrics_test.go | 10 ++-- pkg/local_object_storage/shard/mode.go | 2 +- pkg/local_object_storage/shard/shard.go | 4 +- pkg/metrics/engine.go | 27 ++++------ pkg/metrics/mode.go | 39 +++++++++++++++ pkg/metrics/writecache.go | 49 ++----------------- 8 files changed, 66 insertions(+), 74 deletions(-) create mode 100644 pkg/metrics/mode.go diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index dd1240ea7..fcac2dc60 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -3,6 +3,7 @@ package engine import ( "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) @@ -12,7 +13,7 @@ type MetricRegister interface { SetObjectCounter(shardID, objectType string, v uint64) AddToObjectCounter(shardID, objectType string, delta int) - SetReadonly(shardID string, readonly bool) + SetMode(shardID string, mode mode.Mode) AddToContainerSize(cnrID string, size int64) AddToPayloadCounter(shardID string, size int64) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 5f4f13635..cfa64929b 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -50,8 +50,8 @@ func (m *metricsWithID) DecObjectCounter(objectType string) { m.mw.AddToObjectCounter(m.id, objectType, -1) } -func (m *metricsWithID) SetReadonly(readonly bool) { - m.mw.SetReadonly(m.id, readonly) +func (m *metricsWithID) SetMode(mode mode.Mode) { + m.mw.SetMode(m.id, mode) } func (m *metricsWithID) AddToContainerSize(cnr string, size int64) { @@ -90,7 +90,7 @@ func (e *StorageEngine) AddShard(opts ...shard.Option) (*shard.ID, error) { } if e.cfg.metrics != nil { - e.cfg.metrics.SetReadonly(sh.ID().String(), sh.GetMode() != mode.ReadWrite) + e.cfg.metrics.SetMode(sh.ID().String(), sh.GetMode()) } return sh.ID(), nil diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index f9e1bc760..d76c240b7 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -22,7 +22,7 @@ type metricsStore struct { objCounters map[string]uint64 cnrSize map[string]int64 pldSize int64 - readOnly bool + mode mode.Mode errCounter int64 } @@ -57,8 +57,8 @@ func (m metricsStore) DecObjectCounter(objectType string) { m.AddToObjectCounter(objectType, -1) } -func (m *metricsStore) SetReadonly(r bool) { - m.readOnly = r +func (m *metricsStore) SetMode(mode mode.Mode) { + m.mode = mode } func (m metricsStore) AddToContainerSize(cnr string, size int64) { @@ -91,9 +91,9 @@ func TestCounters(t *testing.T) { sh, mm := shardWithMetrics(t, dir) sh.SetMode(mode.ReadOnly) - require.True(t, mm.readOnly) + require.Equal(t, mode.ReadOnly, mm.mode) sh.SetMode(mode.ReadWrite) - require.False(t, mm.readOnly) + require.Equal(t, mode.ReadWrite, mm.mode) const objNumber = 10 oo := make([]*object.Object, objNumber) diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index efd41863b..a59f08704 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -64,7 +64,7 @@ func (s *Shard) setMode(m mode.Mode) error { s.info.Mode = m if s.metricsWriter != nil { - s.metricsWriter.SetReadonly(s.info.Mode != mode.ReadWrite) + s.metricsWriter.SetMode(s.info.Mode) } s.log.Info(logs.ShardShardModeSetSuccessfully, diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 8c4db87b5..1d2cab9f2 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -73,8 +73,8 @@ type MetricsWriter interface { // SetShardID must set (update) the shard identifier that will be used in // metrics. SetShardID(id string) - // SetReadonly must set shard readonly state. - SetReadonly(readonly bool) + // SetReadonly must set shard mode. + SetMode(mode mode.Mode) // IncErrorCounter increment error counter. IncErrorCounter() // ClearErrorCounter clear error counter. diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index d3f3238ac..7eac85861 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -3,6 +3,7 @@ package metrics import ( "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -16,7 +17,7 @@ type EngineMetrics interface { AddToObjectCounter(shardID, objectType string, delta int) SetObjectCounter(shardID, objectType string, v uint64) AddToPayloadCounter(shardID string, size int64) - SetReadonly(shardID string, readonly bool) + SetMode(shardID string, mode mode.Mode) WriteCache() WriteCacheMetrics GC() GCMetrics @@ -28,7 +29,7 @@ type engineMetrics struct { containerSize *prometheus.GaugeVec payloadSize *prometheus.GaugeVec errorCounter *prometheus.GaugeVec - shardsReadonly *prometheus.GaugeVec + mode *shardIDModeValue gc *gcMetrics writeCache *writeCacheMetrics @@ -50,10 +51,10 @@ func newEngineMetrics() *engineMetrics { Name: "request_duration_seconds", Help: "Duration of Engine requests", }, []string{engineMethod}), - objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), - shardsReadonly: newEngineGaugeVector("mode", "Shard mode", []string{shardIDLabelKey}), - gc: newGCMetrics(), - writeCache: newWriteCacheMetrics(), + objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), + gc: newGCMetrics(), + writeCache: newWriteCacheMetrics(), + mode: newShardIDMode(engineSubsystem, "mode", "Shard mode"), } } @@ -92,7 +93,7 @@ func (m *engineMetrics) DeleteShardMetrics(shardID string) { m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID}) m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID}) m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabelKey: shardID}) - m.shardsReadonly.Delete(prometheus.Labels{shardIDLabelKey: shardID}) + m.mode.Delete(shardID) } func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) { @@ -113,16 +114,8 @@ func (m *engineMetrics) SetObjectCounter(shardID, objectType string, v uint64) { ).Set(float64(v)) } -func (m *engineMetrics) SetReadonly(shardID string, readonly bool) { - var flag float64 - if readonly { - flag = 1 - } - m.shardsReadonly.With( - prometheus.Labels{ - shardIDLabelKey: shardID, - }, - ).Set(flag) +func (m *engineMetrics) SetMode(shardID string, mode mode.Mode) { + m.mode.SetMode(shardID, mode.String()) } func (m *engineMetrics) WriteCache() WriteCacheMetrics { diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go new file mode 100644 index 000000000..7ae3fd5f9 --- /dev/null +++ b/pkg/metrics/mode.go @@ -0,0 +1,39 @@ +package metrics + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type shardIDModeValue struct { + modeValue *prometheus.GaugeVec +} + +func newShardIDMode(subsystem, name, help string) *shardIDModeValue { + return &shardIDModeValue{ + modeValue: metrics.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: name, + Help: help, + }, []string{wcShardID, wcMode}), + } +} + +func (m *shardIDModeValue) SetMode(shardID string, mode string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + wcShardID: shardID, + }) + + m.modeValue.With(prometheus.Labels{ + wcShardID: shardID, + wcMode: mode, + }).Set(1) +} + +func (m *shardIDModeValue) Delete(shardID string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + wcShardID: shardID, + }) +} diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 8498e3930..373772a62 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -2,7 +2,6 @@ package metrics import ( "fmt" - "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -19,10 +18,6 @@ const ( wcOperation = "operation" ) -type shardIDMode struct { - shardID, mode string -} - type WriteCacheMetrics interface { AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) @@ -46,9 +41,7 @@ type writeCacheMetrics struct { estimatedSize *prometheus.GaugeVec - modeMetrics map[shardIDMode]prometheus.GaugeFunc - modeValues map[string]string - modeMtx sync.RWMutex + mode *shardIDModeValue } func newWriteCacheMetrics() *writeCacheMetrics { @@ -67,9 +60,7 @@ func newWriteCacheMetrics() *writeCacheMetrics { }, []string{wcShardID, wcStorage, wcSuccess, wcOperation}), actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}), estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), - modeMtx: sync.RWMutex{}, - modeMetrics: make(map[shardIDMode]prometheus.GaugeFunc), - modeValues: make(map[string]string), + mode: newShardIDMode(wcSubsystem, "mode", "Writecache mode value"), } } @@ -113,39 +104,7 @@ func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storage } func (m *writeCacheMetrics) SetMode(shardID string, mode string) { - m.modeMtx.Lock() - defer m.modeMtx.Unlock() - - m.modeValues[shardID] = mode - key := shardIDMode{ - shardID: shardID, - mode: mode, - } - if _, found := m.modeMetrics[key]; found { - return - } - - metric := metrics.NewGaugeFunc( - prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: wcSubsystem, - Name: "writecache_mode", - Help: "Writecache mode value", - ConstLabels: prometheus.Labels{ - wcShardID: shardID, - wcMode: mode, - }, - }, func() float64 { - m.modeMtx.RLock() - defer m.modeMtx.RUnlock() - - value := m.modeValues[shardID] - if value == mode { - return 1 - } - return 0 - }) - m.modeMetrics[key] = metric + m.mode.SetMode(shardID, mode) } func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) { @@ -158,7 +117,7 @@ func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string } func (m *writeCacheMetrics) Close(shardID string) { - m.SetMode(shardID, "CLOSED") + m.mode.Delete(shardID) m.methodDuration.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) m.operationCounter.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) m.actualCount.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) From 26b305f82b830eebf6c727f013606b576299e45d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 12:12:47 +0300 Subject: [PATCH 0336/1943] [#424] morph: Fix cache metrics Use separate morph cache metrics for node and IR Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/morph.go | 3 +-- pkg/metrics/innerring.go | 9 +++++---- pkg/metrics/morphcache.go | 8 ++++++-- pkg/morph/client/constructor.go | 1 + pkg/morph/metrics/metrics.go | 4 ++++ 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index ae50b8a8a..63d1605ef 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -30,8 +30,6 @@ const ( ) func initMorphComponents(ctx context.Context, c *cfg) { - var err error - addresses := morphconfig.RPCEndpoint(c.appCfg) // Morph client stable-sorts endpoints by priority. Shuffle here to randomize @@ -50,6 +48,7 @@ func initMorphComponents(ctx context.Context, c *cfg) { c.internalErr <- errors.New("morph connection has been lost") }), client.WithSwitchInterval(morphconfig.SwitchInterval(c.appCfg)), + client.WithMorphCacheMetrics(metrics.NewNodeMorphCacheMetrics()), ) if err != nil { c.log.Info(logs.FrostFSNodeFailedToCreateNeoRPCClient, diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index b3cb3e285..0aee068c5 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -12,6 +12,7 @@ const ( innerRingSubsystem = "ir" innerRingLabelSuccess = "success" innerRingLabelType = "type" + innerRingNamespace = "frostfs_ir" ) // InnerRingServiceMetrics contains metrics collected by inner ring. @@ -26,19 +27,19 @@ type InnerRingServiceMetrics struct { func NewInnerRingMetrics() *InnerRingServiceMetrics { var ( epoch = metrics.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, + Namespace: innerRingNamespace, Subsystem: innerRingSubsystem, Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }) health = metrics.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, + Namespace: innerRingNamespace, Subsystem: innerRingSubsystem, Name: "health", Help: "Current inner-ring node state.", }) eventDuration = metrics.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, + Namespace: innerRingNamespace, Subsystem: innerRingSubsystem, Name: "event_duration_seconds", Help: "Duration of processing of inner-ring events", @@ -49,7 +50,7 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { epoch: epoch, health: health, eventDuration: eventDuration, - morphCacheMetrics: newMorphCacheMetrics(), + morphCacheMetrics: newMorphCacheMetrics(innerRingNamespace), } } diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go index 9cf2c8a0c..3f215b5bf 100644 --- a/pkg/metrics/morphcache.go +++ b/pkg/metrics/morphcache.go @@ -24,10 +24,14 @@ type morphCacheMetrics struct { var _ MorphCacheMetrics = (*morphCacheMetrics)(nil) -func newMorphCacheMetrics() *morphCacheMetrics { +func NewNodeMorphCacheMetrics() MorphCacheMetrics { + return newMorphCacheMetrics(namespace) +} + +func newMorphCacheMetrics(ns string) *morphCacheMetrics { return &morphCacheMetrics{ methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: namespace, + Namespace: ns, Subsystem: mcSubsystem, Name: "request_duration_seconds", Help: "Morph cache request process duration", diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 6cc66c5cc..e7e1bbca9 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -68,6 +68,7 @@ func defaultConfig() *cfg { signer: &transaction.Signer{ Scopes: transaction.Global, }, + morphCacheMetrics: &morphmetrics.NoopMorphCacheMetrics{}, } } diff --git a/pkg/morph/metrics/metrics.go b/pkg/morph/metrics/metrics.go index 9e41a0b86..5d74b054d 100644 --- a/pkg/morph/metrics/metrics.go +++ b/pkg/morph/metrics/metrics.go @@ -15,3 +15,7 @@ func (NoopRegister) IncSwitchCount() { func (NoopRegister) SetLastBlock(uint32) {} func (NoopRegister) IncNotificationCount(string) {} func (NoopRegister) ObserveInvoke(string, string, string, bool, time.Duration) {} + +type NoopMorphCacheMetrics struct{} + +func (m *NoopMorphCacheMetrics) AddMethodDuration(string, bool, time.Duration) {} From 69b788a90b437b04da17996166361f016e8c0e25 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 14 Jun 2023 18:39:49 +0300 Subject: [PATCH 0337/1943] [#424] metrics: Rename mode to mode_info To follow best practice. Signed-off-by: Dmitrii Stepanov --- pkg/metrics/engine.go | 2 +- pkg/metrics/writecache.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 7eac85861..395ab3356 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -54,7 +54,7 @@ func newEngineMetrics() *engineMetrics { objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), gc: newGCMetrics(), writeCache: newWriteCacheMetrics(), - mode: newShardIDMode(engineSubsystem, "mode", "Shard mode"), + mode: newShardIDMode(engineSubsystem, "mode_info", "Shard mode"), } } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 373772a62..a9c1b33a7 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -60,7 +60,7 @@ func newWriteCacheMetrics() *writeCacheMetrics { }, []string{wcShardID, wcStorage, wcSuccess, wcOperation}), actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}), estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), - mode: newShardIDMode(wcSubsystem, "mode", "Writecache mode value"), + mode: newShardIDMode(wcSubsystem, "mode_info", "Writecache mode value"), } } From 20b84f183a2936fe3b4522f13ee06468df63ef98 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 15 Jun 2023 11:42:54 +0300 Subject: [PATCH 0338/1943] [#446] engine: Simplify logs for shard mode change Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/engine.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 7c36811e7..9967b2e2b 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -83,32 +83,28 @@ func (e *StorageEngine) setModeLoop() { } func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { + sid := sh.ID() + log := e.log.With( + zap.Stringer("shard_id", sid), + zap.Uint32("error count", errCount)) + e.mtx.RLock() defer e.mtx.RUnlock() - sid := sh.ID() err := sh.SetMode(mode.DegradedReadOnly) if err != nil { - e.log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount), + log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, zap.Error(err)) err = sh.SetMode(mode.ReadOnly) if err != nil { - e.log.Error(logs.EngineFailedToMoveShardInReadonlyMode, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount), + log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) } else { - e.log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount)) + log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } } else { - e.log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold, - zap.Stringer("shard_id", sid), - zap.Uint32("error count", errCount)) + log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) } } From fe0178181171cff0bf00e7b24943d15dedacb77b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 15 Jun 2023 13:19:36 +0300 Subject: [PATCH 0339/1943] [#446] los: Wrap SSD errors in a separate type Signed-off-by: Evgenii Stratonikov --- .../internal/metaerr/error.go | 33 +++++++++ .../internal/metaerr/error_test.go | 67 +++++++++++++++++++ pkg/local_object_storage/metabase/children.go | 3 +- .../metabase/containers.go | 5 +- pkg/local_object_storage/metabase/control.go | 15 +++-- pkg/local_object_storage/metabase/counter.go | 3 +- pkg/local_object_storage/metabase/delete.go | 3 +- pkg/local_object_storage/metabase/exists.go | 3 +- pkg/local_object_storage/metabase/expired.go | 3 +- pkg/local_object_storage/metabase/get.go | 3 +- .../metabase/graveyard.go | 9 +-- pkg/local_object_storage/metabase/inhume.go | 3 +- .../metabase/iterators.go | 5 +- pkg/local_object_storage/metabase/list.go | 3 +- pkg/local_object_storage/metabase/lock.go | 11 +-- pkg/local_object_storage/metabase/movable.go | 11 +-- pkg/local_object_storage/metabase/put.go | 3 +- pkg/local_object_storage/metabase/select.go | 5 +- pkg/local_object_storage/metabase/shard_id.go | 7 +- .../metabase/storage_id.go | 5 +- pkg/local_object_storage/pilorama/boltdb.go | 45 +++++++------ pkg/local_object_storage/writecache/delete.go | 3 +- pkg/local_object_storage/writecache/flush.go | 9 +-- pkg/local_object_storage/writecache/get.go | 8 ++- .../writecache/iterate.go | 5 +- pkg/local_object_storage/writecache/put.go | 3 +- .../writecache/writecache.go | 5 +- 27 files changed, 202 insertions(+), 76 deletions(-) create mode 100644 pkg/local_object_storage/internal/metaerr/error.go create mode 100644 pkg/local_object_storage/internal/metaerr/error_test.go diff --git a/pkg/local_object_storage/internal/metaerr/error.go b/pkg/local_object_storage/internal/metaerr/error.go new file mode 100644 index 000000000..41b8504bc --- /dev/null +++ b/pkg/local_object_storage/internal/metaerr/error.go @@ -0,0 +1,33 @@ +package metaerr + +import "errors" + +// Error is a wrapper for SSD-related errors. +// In our model it unites metabase, pilorama and write-cache errors. +type Error struct { + err error +} + +// New returns simple error with a provided error message. +func New(msg string) Error { + return Error{err: errors.New(msg)} +} + +// Error implements the error interface. +func (e Error) Error() string { + return e.err.Error() +} + +// Wrap wraps arbitrary error. +// Returns nil if err == nil. +func Wrap(err error) error { + if err != nil { + return Error{err: err} + } + return nil +} + +// Unwrap returns underlying error. +func (e Error) Unwrap() error { + return e.err +} diff --git a/pkg/local_object_storage/internal/metaerr/error_test.go b/pkg/local_object_storage/internal/metaerr/error_test.go new file mode 100644 index 000000000..5a16aa501 --- /dev/null +++ b/pkg/local_object_storage/internal/metaerr/error_test.go @@ -0,0 +1,67 @@ +package metaerr + +import ( + "errors" + "fmt" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestError(t *testing.T) { + t.Run("errors.Is", func(t *testing.T) { + e1 := errors.New("some error") + ee := Wrap(e1) + require.ErrorIs(t, ee, e1) + + e2 := fmt.Errorf("wrap: %w", e1) + ee = Wrap(e2) + require.ErrorIs(t, ee, e1) + require.ErrorIs(t, ee, e2) + + require.Equal(t, errors.Unwrap(ee), e2) + }) + + t.Run("errors.As", func(t *testing.T) { + e1 := testError{42} + ee := Wrap(e1) + + { + var actual testError + require.ErrorAs(t, ee, &actual) + require.Equal(t, e1.data, actual.data) + } + { + var actual Error + require.ErrorAs(t, ee, &actual) + require.Equal(t, e1, actual.err) + } + + e2 := fmt.Errorf("wrap: %w", e1) + ee = Wrap(e2) + + { + var actual testError + require.ErrorAs(t, ee, &actual) + require.Equal(t, e1.data, actual.data) + } + }) +} +func TestNilWrap(t *testing.T) { + require.NoError(t, Wrap(nil)) +} + +func TestErrorMessage(t *testing.T) { + msg := "sth to report" + err := New(msg) + require.Contains(t, err.Error(), msg) +} + +type testError struct { + data uint64 +} + +func (e testError) Error() string { + return strconv.FormatUint(e.data, 10) +} diff --git a/pkg/local_object_storage/metabase/children.go b/pkg/local_object_storage/metabase/children.go index f0591b43c..500e83e7a 100644 --- a/pkg/local_object_storage/metabase/children.go +++ b/pkg/local_object_storage/metabase/children.go @@ -1,6 +1,7 @@ package meta import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -50,7 +51,7 @@ func (db *DB) GetChildren(addresses []oid.Address) (map[oid.Address][]oid.Addres }) if err != nil { - return nil, err + return nil, metaerr.Wrap(err) } return result, nil diff --git a/pkg/local_object_storage/metabase/containers.go b/pkg/local_object_storage/metabase/containers.go index 3d69649a9..fe38e0b6d 100644 --- a/pkg/local_object_storage/metabase/containers.go +++ b/pkg/local_object_storage/metabase/containers.go @@ -3,6 +3,7 @@ package meta import ( "encoding/binary" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.etcd.io/bbolt" ) @@ -21,7 +22,7 @@ func (db *DB) Containers() (list []cid.ID, err error) { return err }) - return list, err + return list, metaerr.Wrap(err) } func (db *DB) containers(tx *bbolt.Tx) ([]cid.ID, error) { @@ -55,7 +56,7 @@ func (db *DB) ContainerSize(id cid.ID) (size uint64, err error) { return err }) - return size, err + return size, metaerr.Wrap(err) } func (db *DB) containerSize(tx *bbolt.Tx, id cid.ID) (uint64, error) { diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 4ae802aaa..370fddda1 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -6,6 +6,7 @@ import ( "path/filepath" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -34,7 +35,7 @@ func (db *DB) Open(readOnly bool) error { } db.boltOptions.ReadOnly = readOnly - return db.openBolt() + return metaerr.Wrap(db.openBolt()) } func (db *DB) openBolt() error { @@ -79,7 +80,7 @@ func (db *DB) openBolt() error { // Does nothing if metabase has already been initialized and filled. To roll back the database to its initial state, // use Reset. func (db *DB) Init() error { - return db.init(false) + return metaerr.Wrap(db.init(false)) } // Reset resets metabase. Works similar to Init but cleans up all static buckets and @@ -92,7 +93,7 @@ func (db *DB) Reset() error { return ErrDegradedMode } - return db.init(true) + return metaerr.Wrap(db.init(true)) } func (db *DB) init(reset bool) error { @@ -167,15 +168,15 @@ func (db *DB) SyncCounters() error { return ErrReadOnlyMode } - return db.boltDB.Update(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { return syncCounter(tx, true) - }) + })) } // Close closes boltDB instance. func (db *DB) Close() error { if db.boltDB != nil { - return db.boltDB.Close() + return metaerr.Wrap(db.boltDB.Close()) } return nil } @@ -203,7 +204,7 @@ func (db *DB) Reload(opts ...Option) (bool, error) { db.mode = mode.Degraded db.info.Path = c.info.Path if err := db.openBolt(); err != nil { - return false, fmt.Errorf("%w: %v", ErrDegradedMode, err) + return false, metaerr.Wrap(fmt.Errorf("%w: %v", ErrDegradedMode, err)) } db.mode = mode.ReadWrite diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index a07328026..c0dc7886e 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -67,7 +68,7 @@ func (db *DB) ObjectCounters() (cc ObjectCounters, err error) { return nil }) - return + return cc, metaerr.Wrap(err) } // updateCounter updates the object counter. Tx MUST be writable. diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 6ddf96593..7e8e0e5dd 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -108,7 +109,7 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { availableRemoved: availableRemoved, sizes: sizes, availableSizes: availableSizes, - }, err + }, metaerr.Wrap(err) } // deleteGroup deletes object from the metabase. Handles removal of the diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index 74faf1634..f9c563bdd 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -5,6 +5,7 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -65,7 +66,7 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err return err }) - return + return res, metaerr.Wrap(err) } func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (exists bool, err error) { diff --git a/pkg/local_object_storage/metabase/expired.go b/pkg/local_object_storage/metabase/expired.go index d20fdbfa9..aac158ba4 100644 --- a/pkg/local_object_storage/metabase/expired.go +++ b/pkg/local_object_storage/metabase/expired.go @@ -7,6 +7,7 @@ import ( "strconv" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -65,7 +66,7 @@ func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.A }) if err != nil { - return nil, err + return nil, metaerr.Wrap(err) } return result, nil } diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index 2f94c72d0..2ce82bcee 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -74,7 +75,7 @@ func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { return err }) - return + return res, metaerr.Wrap(err) } func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw bool, currEpoch uint64) (*objectSDK.Object, error) { diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index 393c9f4d0..e2530bd31 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -65,9 +66,9 @@ func (db *DB) IterateOverGarbage(p GarbageIterationPrm) error { return ErrDegradedMode } - return db.boltDB.View(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateDeletedObj(tx, gcHandler{p.h}, p.offset) - }) + })) } // TombstonedObject represents descriptor of the @@ -132,9 +133,9 @@ func (db *DB) IterateOverGraveyard(p GraveyardIterationPrm) error { return ErrDegradedMode } - return db.boltDB.View(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateDeletedObj(tx, graveyardHandler{p.h}, p.offset) - }) + })) } type kvHandler interface { diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index e4af1d1b9..7865a8771 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -138,7 +139,7 @@ func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err err return db.inhumeTx(tx, currEpoch, prm, &res) }) - return + return res, metaerr.Wrap(err) } func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes) error { diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index ba22566a3..dfa048621 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -6,6 +6,7 @@ import ( "strconv" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -51,9 +52,9 @@ func (db *DB) IterateExpired(epoch uint64, h ExpiredObjectHandler) error { return ErrDegradedMode } - return db.boltDB.View(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateExpired(tx, epoch, h) - }) + })) } func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) error { diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 7fe1040bb..7c77cd703 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -2,6 +2,7 @@ package meta import ( objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -75,7 +76,7 @@ func (db *DB) ListWithCursor(prm ListPrm) (res ListRes, err error) { return err }) - return res, err + return res, metaerr.Wrap(err) } func (db *DB) listWithCursor(tx *bbolt.Tx, result []objectcore.AddressWithType, count int, cursor *Cursor) ([]objectcore.AddressWithType, *Cursor, error) { diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 2b5fcd775..b84e0bf46 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -5,6 +5,7 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -63,7 +64,7 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. } key := make([]byte, cidSize) - return db.boltDB.Update(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { if firstIrregularObjectType(tx, cnr, bucketKeysLocked...) != object.TypeRegular { return logicerr.Wrap(apistatus.LockNonRegularObject{}) } @@ -109,7 +110,7 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. } return nil - }) + })) } // FreeLockedBy unlocks all objects in DB which are locked by lockers. @@ -135,7 +136,7 @@ func (db *DB) FreeLockedBy(lockers []oid.Address) ([]oid.Address, error) { return nil }); err != nil { - return nil, err + return nil, metaerr.Wrap(err) } return unlockedObjects, nil } @@ -292,8 +293,8 @@ func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, e return res, ErrDegradedMode } - return res, db.boltDB.View(func(tx *bbolt.Tx) error { + return res, metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { res.locked = objectLocked(tx, prm.addr.Container(), prm.addr.Object()) return nil - }) + })) } diff --git a/pkg/local_object_storage/metabase/movable.go b/pkg/local_object_storage/metabase/movable.go index 033b9c718..763e49a5d 100644 --- a/pkg/local_object_storage/metabase/movable.go +++ b/pkg/local_object_storage/metabase/movable.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -76,7 +77,7 @@ func (db *DB) ToMoveIt(ctx context.Context, prm ToMoveItPrm) (res ToMoveItRes, e return toMoveIt.Put(key, zeroValue) }) - return + return res, metaerr.Wrap(err) } // DoNotMove removes `MoveIt` mark from the object. @@ -98,7 +99,7 @@ func (db *DB) DoNotMove(prm DoNotMovePrm) (res DoNotMoveRes, err error) { return toMoveIt.Delete(key) }) - return + return res, metaerr.Wrap(err) } // Movable returns list of marked objects to move into other shard. @@ -121,7 +122,7 @@ func (db *DB) Movable(_ MovablePrm) (MovableRes, error) { }) }) if err != nil { - return MovableRes{}, err + return MovableRes{}, metaerr.Wrap(err) } // we can parse strings to structures in-place, but probably it seems @@ -132,8 +133,8 @@ func (db *DB) Movable(_ MovablePrm) (MovableRes, error) { for i := range strAddrs { err = decodeAddressFromKey(&addrs[i], []byte(strAddrs[i])) if err != nil { - return MovableRes{}, fmt.Errorf("can't parse object address %v: %w", - strAddrs[i], err) + return MovableRes{}, metaerr.Wrap(fmt.Errorf("can't parse object address %v: %w", + strAddrs[i], err)) } } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 86830f7da..b7cff68dc 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -9,6 +9,7 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -83,7 +84,7 @@ func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { storagelog.OpField("metabase PUT")) } - return + return res, metaerr.Wrap(err) } func (db *DB) put(tx *bbolt.Tx, diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index b6b8f5b10..466476d4e 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -9,6 +9,7 @@ import ( v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -80,11 +81,11 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err currEpoch := db.epochState.CurrentEpoch() - return res, db.boltDB.View(func(tx *bbolt.Tx) error { + return res, metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { res.addrList, err = db.selectObjects(tx, prm.cnr, prm.filters, currEpoch) return err - }) + })) } func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs object.SearchFilters, currEpoch uint64) ([]oid.Address, error) { diff --git a/pkg/local_object_storage/metabase/shard_id.go b/pkg/local_object_storage/metabase/shard_id.go index fac8a079f..f60a4724d 100644 --- a/pkg/local_object_storage/metabase/shard_id.go +++ b/pkg/local_object_storage/metabase/shard_id.go @@ -1,6 +1,7 @@ package meta import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" ) @@ -28,7 +29,7 @@ func (db *DB) ReadShardID() ([]byte, error) { } return nil }) - return id, err + return id, metaerr.Wrap(err) } // WriteShardID writes shard it to db. @@ -42,11 +43,11 @@ func (db *DB) WriteShardID(id []byte) error { return ErrReadOnlyMode } - return db.boltDB.Update(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { b, err := tx.CreateBucketIfNotExists(shardInfoBucket) if err != nil { return err } return b.Put(shardIDKey, id) - }) + })) } diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index b53010a1e..6ba5a60cb 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -54,7 +55,7 @@ func (db *DB) StorageID(ctx context.Context, prm StorageIDPrm) (res StorageIDRes return err }) - return + return res, metaerr.Wrap(err) } func (db *DB) storageID(tx *bbolt.Tx, addr oid.Address) ([]byte, error) { @@ -113,5 +114,5 @@ func (db *DB) UpdateStorageID(prm UpdateStorageIDPrm) (res UpdateStorageIDRes, e return err }) - return + return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 5b2c97f10..e7bcb110b 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -105,7 +106,7 @@ func (t *boltForest) SetMode(m mode.Mode) error { func (t *boltForest) Open(readOnly bool) error { err := util.MkdirAllX(filepath.Dir(t.path), t.perm) if err != nil { - return fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err) + return metaerr.Wrap(fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)) } opts := *bbolt.DefaultOptions @@ -116,7 +117,7 @@ func (t *boltForest) Open(readOnly bool) error { t.db, err = bbolt.Open(t.path, t.perm, &opts) if err != nil { - return fmt.Errorf("can't open the pilorama DB: %w", err) + return metaerr.Wrap(fmt.Errorf("can't open the pilorama DB: %w", err)) } t.db.MaxBatchSize = t.maxBatchSize @@ -174,7 +175,7 @@ func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID strin lm := *m fullID := bucketName(d.CID, treeID) - return &lm, t.db.Batch(func(tx *bbolt.Tx) error { + return &lm, metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { bLog, bTree, err := t.getTreeBuckets(tx, fullID) if err != nil { return err @@ -185,7 +186,7 @@ func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID strin lm.Child = t.findSpareID(bTree) } return t.do(bLog, bTree, make([]byte, 17), &lm) - }) + })) } func (t *boltForest) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { @@ -219,7 +220,7 @@ func (t *boltForest) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID strin if err == nil { err = retErr } - return height, err + return height, metaerr.Wrap(err) } // TreeExists implements the Forest interface. @@ -247,7 +248,7 @@ func (t *boltForest) TreeExists(ctx context.Context, cid cidSDK.ID, treeID strin return nil }) - return exists, err + return exists, metaerr.Wrap(err) } var syncHeightKey = []byte{'h'} @@ -267,7 +268,7 @@ func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID binary.LittleEndian.PutUint64(rawHeight, height) buck := bucketName(cid, treeID) - return t.db.Batch(func(tx *bbolt.Tx) error { + return metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(buck) if treeRoot == nil { return ErrTreeNotFound @@ -275,7 +276,7 @@ func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID b := treeRoot.Bucket(dataBucket) return b.Put(syncHeightKey, rawHeight) - }) + })) } // TreeLastSyncHeight implements the pilorama.Forest interface. @@ -304,7 +305,7 @@ func (t *boltForest) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, tree } return nil }) - return height, err + return height, metaerr.Wrap(err) } // TreeAddByPath implements the Forest interface. @@ -384,7 +385,7 @@ func (t *boltForest) TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID } return t.do(bLog, bTree, key[:], &lm[len(lm)-1]) }) - return lm, err + return lm, metaerr.Wrap(err) } // getLatestTimestamp returns timestamp for a new operation which is guaranteed to be bigger than @@ -450,13 +451,13 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string return nil }) if err != nil || seen { - return err + return metaerr.Wrap(err) } } if t.db.MaxBatchSize == 1 { fullID := bucketName(cnr, treeID) - return t.db.Update(func(tx *bbolt.Tx) error { + return metaerr.Wrap(t.db.Update(func(tx *bbolt.Tx) error { bLog, bTree, err := t.getTreeBuckets(tx, fullID) if err != nil { return err @@ -464,12 +465,12 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string var lm Move return t.applyOperation(bLog, bTree, []*Move{m}, &lm) - }) + })) } ch := make(chan error, 1) t.addBatch(cnr, treeID, m, ch) - return <-ch + return metaerr.Wrap(<-ch) } func (t *boltForest) addBatch(cnr cidSDK.ID, treeID string, m *Move, ch chan error) { @@ -751,7 +752,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st var nodes []Node - return nodes, t.db.View(func(tx *bbolt.Tx) error { + return nodes, metaerr.Wrap(t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cid, treeID)) if treeRoot == nil { return ErrTreeNotFound @@ -788,7 +789,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st childKey, _ = c.Next() } return nil - }) + })) } // TreeGetMeta implements the forest interface. @@ -828,7 +829,7 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri return m.FromBytes(meta) }) - return m, parentID, err + return m, parentID, metaerr.Wrap(err) } // TreeGetChildren implements the Forest interface. @@ -869,7 +870,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID return nil }) - return children, err + return children, metaerr.Wrap(err) } // TreeList implements the Forest interface. @@ -907,7 +908,7 @@ func (t *boltForest) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, err return nil }) if err != nil { - return nil, fmt.Errorf("could not list trees: %w", err) + return nil, metaerr.Wrap(fmt.Errorf("could not list trees: %w", err)) } return ids, nil @@ -949,7 +950,7 @@ func (t *boltForest) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID str return nil }) - return lm, err + return lm, metaerr.Wrap(err) } // TreeDrop implements the pilorama.Forest interface. @@ -971,7 +972,7 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) return ErrReadOnlyMode } - return t.db.Batch(func(tx *bbolt.Tx) error { + return metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { if treeID == "" { c := tx.Cursor() prefix := make([]byte, 32) @@ -989,7 +990,7 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) return ErrTreeNotFound } return err - }) + })) } func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node, error) { diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index 796fda623..aeab88b0b 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -77,5 +78,5 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { deleted = true } - return err + return metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index da8b3a0fa..21f968b04 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -10,6 +10,7 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -193,7 +194,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { data, err := f() if err != nil { - c.reportFlushError("can't read a file", sAddr, err) + c.reportFlushError("can't read a file", sAddr, metaerr.Wrap(err)) if ignoreErrors { return nil } @@ -203,7 +204,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { var obj object.Object err = obj.Unmarshal(data) if err != nil { - c.reportFlushError("can't unmarshal an object", sAddr, err) + c.reportFlushError("can't unmarshal an object", sAddr, metaerr.Wrap(err)) if ignoreErrors { return nil } @@ -314,7 +315,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { for k, data := cs.Seek(nil); k != nil; k, data = cs.Next() { sa := string(k) if err := addr.DecodeString(sa); err != nil { - c.reportFlushError("can't decode object address from the DB", sa, err) + c.reportFlushError("can't decode object address from the DB", sa, metaerr.Wrap(err)) if ignoreErrors { continue } @@ -323,7 +324,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { var obj object.Object if err := obj.Unmarshal(data); err != nil { - c.reportFlushError("can't unmarshal an object from the DB", sa, err) + c.reportFlushError("can't unmarshal an object from the DB", sa, metaerr.Wrap(err)) if ignoreErrors { continue } diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index aac5759ae..2546bada9 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -5,6 +5,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -28,7 +29,8 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e )) defer span.End() - return c.getInternal(ctx, saddr, addr) + obj, err := c.getInternal(ctx, saddr, addr) + return obj, metaerr.Wrap(err) } func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) (*objectSDK.Object, error) { @@ -71,7 +73,7 @@ func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, obj, err := c.getInternal(ctx, saddr, addr) if err != nil { - return nil, err + return nil, metaerr.Wrap(err) } return obj.CutPayload(), nil @@ -95,5 +97,5 @@ func Get(db *bbolt.DB, key []byte) ([]byte, error) { value = slice.Copy(value) return nil }) - return value, err + return value, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/writecache/iterate.go b/pkg/local_object_storage/writecache/iterate.go index 59ace93bd..5349c069c 100644 --- a/pkg/local_object_storage/writecache/iterate.go +++ b/pkg/local_object_storage/writecache/iterate.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -18,7 +19,7 @@ var ErrNoDefaultBucket = errors.New("no default bucket") // // DB must not be nil and should be opened. func IterateDB(db *bbolt.DB, f func(oid.Address) error) error { - return db.View(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.View(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) if b == nil { return ErrNoDefaultBucket @@ -34,5 +35,5 @@ func IterateDB(db *bbolt.DB, f func(oid.Address) error) error { return f(addr) }) - }) + })) } diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 61e81ec30..619b2bd26 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -72,7 +73,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro if err == nil { added = true } - return common.PutRes{}, err + return common.PutRes{}, metaerr.Wrap(err) } // putSmall persists small objects to the write-cache database and diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 6fa0e36de..962c9c39a 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -129,14 +130,14 @@ func (c *cache) DumpInfo() Info { func (c *cache) Open(readOnly bool) error { err := c.openStore(readOnly) if err != nil { - return err + return metaerr.Wrap(err) } // Opening after Close is done during maintenance mode, // thus we need to create a channel here. c.closeCh = make(chan struct{}) - return c.initCounters() + return metaerr.Wrap(c.initCounters()) } // Init runs necessary services. From 69df0d21c2d5f14a28d45951c32702ceec9da759 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 15 Jun 2023 14:50:51 +0300 Subject: [PATCH 0340/1943] [#446] engine: Move to read-only on blobstor errors Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/engine.go | 36 +++++++++++-------- pkg/local_object_storage/engine/error_test.go | 4 +-- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 9967b2e2b..3061d6383 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -45,6 +46,7 @@ type shardWrapper struct { type setModeRequest struct { sh *shard.Shard + isMeta bool errorCount uint32 } @@ -70,7 +72,7 @@ func (e *StorageEngine) setModeLoop() { if !ok { inProgress[sid] = struct{}{} go func() { - e.moveToDegraded(r.sh, r.errorCount) + e.moveToDegraded(r.sh, r.errorCount, r.isMeta) mtx.Lock() delete(inProgress, sid) @@ -82,7 +84,7 @@ func (e *StorageEngine) setModeLoop() { } } -func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { +func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) { sid := sh.ID() log := e.log.With( zap.Stringer("shard_id", sid), @@ -91,21 +93,23 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { e.mtx.RLock() defer e.mtx.RUnlock() - err := sh.SetMode(mode.DegradedReadOnly) - if err != nil { + if isMeta { + err := sh.SetMode(mode.DegradedReadOnly) + if err == nil { + log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) + return + } log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, zap.Error(err)) - - err = sh.SetMode(mode.ReadOnly) - if err != nil { - log.Error(logs.EngineFailedToMoveShardInReadonlyMode, - zap.Error(err)) - } else { - log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) - } - } else { - log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) } + + err := sh.SetMode(mode.ReadOnly) + if err != nil { + log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) + return + } + + log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } // reportShardErrorBackground increases shard error counter and logs an error. @@ -169,11 +173,13 @@ func (e *StorageEngine) reportShardErrorWithFlags( return } + isMeta := errors.As(err, new(metaerr.Error)) if block { - e.moveToDegraded(sh, errCount) + e.moveToDegraded(sh, errCount, isMeta) } else { req := setModeRequest{ errorCount: errCount, + isMeta: isMeta, sh: sh, } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index dc28d35fa..18bc72d65 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -154,7 +154,7 @@ func TestErrorReporting(t *testing.T) { for i := uint32(0); i < 2; i++ { _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) - checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly) + checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } @@ -219,7 +219,7 @@ func TestBlobstorFailback(t *testing.T) { require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) } - checkShardState(t, te.ng, te.shards[0].id, 1, mode.DegradedReadOnly) + checkShardState(t, te.ng, te.shards[0].id, 2, mode.ReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } From 50caa388b0467b9011650e38930167c7891093cd Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Jun 2023 11:55:06 +0300 Subject: [PATCH 0341/1943] [#303] ir: Use pub key when validate container deletion Signed-off-by: Anton Nikiforov --- pkg/core/container/delete.go | 46 ++++--------------- .../processors/container/process_container.go | 1 + pkg/morph/client/container/delete.go | 15 ++++-- pkg/morph/event/container/delete.go | 3 +- pkg/morph/event/container/delete_notary.go | 5 ++ pkg/services/container/morph/executor.go | 8 ++-- 6 files changed, 31 insertions(+), 47 deletions(-) diff --git a/pkg/core/container/delete.go b/pkg/core/container/delete.go index e3379446f..8e0aaebb9 100644 --- a/pkg/core/container/delete.go +++ b/pkg/core/container/delete.go @@ -1,6 +1,7 @@ package container import ( + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" ) @@ -8,43 +9,14 @@ import ( // RemovalWitness groups the information required // to prove and verify the removal of a container. type RemovalWitness struct { - cnr cid.ID + // ContainerID returns the identifier of the container + // to be removed. + ContainerID cid.ID - sig []byte + // Signature the signature of the container identifier. + Signature *refs.Signature - token *session.Container -} - -// ContainerID returns the identifier of the container -// to be removed. -func (x RemovalWitness) ContainerID() cid.ID { - return x.cnr -} - -// SetContainerID sets the identifier of the container -// to be removed. -func (x *RemovalWitness) SetContainerID(id cid.ID) { - x.cnr = id -} - -// Signature returns the signature of the container identifier. -func (x RemovalWitness) Signature() []byte { - return x.sig -} - -// SetSignature sets a signature of the container identifier. -func (x *RemovalWitness) SetSignature(sig []byte) { - x.sig = sig -} - -// SessionToken returns the token of the session within -// which the container was removed. -func (x RemovalWitness) SessionToken() *session.Container { - return x.token -} - -// SetSessionToken sets the token of the session within -// which the container was removed. -func (x *RemovalWitness) SetSessionToken(tok *session.Container) { - x.token = tok + // SessionToken the token of the session within + // which the container was removed. + SessionToken *session.Container } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 33ef90034..2629b9d29 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -148,6 +148,7 @@ func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { binTokenSession: e.SessionToken(), signature: e.Signature(), signedData: binCnr, + binPublicKey: e.PublicKeyValue, }) if err != nil { return fmt.Errorf("auth container removal: %w", err) diff --git a/pkg/morph/client/container/delete.go b/pkg/morph/client/container/delete.go index c9105a3ca..5bc8fc188 100644 --- a/pkg/morph/client/container/delete.go +++ b/pkg/morph/client/container/delete.go @@ -14,14 +14,15 @@ import ( // Returns error if container ID is nil. func Delete(c *Client, witness core.RemovalWitness) error { binCnr := make([]byte, sha256.Size) - witness.ContainerID().Encode(binCnr) + witness.ContainerID.Encode(binCnr) var prm DeletePrm prm.SetCID(binCnr) - prm.SetSignature(witness.Signature()) + prm.SetSignature(witness.Signature.GetSign()) + prm.SetKey(witness.Signature.GetKey()) - if tok := witness.SessionToken(); tok != nil { + if tok := witness.SessionToken; tok != nil { prm.SetToken(tok.Marshal()) } @@ -33,6 +34,7 @@ type DeletePrm struct { cnr []byte signature []byte token []byte + key []byte client.InvokePrmOptional } @@ -52,6 +54,11 @@ func (d *DeletePrm) SetToken(token []byte) { d.token = token } +// SetKey sets public key. +func (d *DeletePrm) SetKey(key []byte) { + d.key = key +} + // Delete removes the container from FrostFS system // through Container contract call. // @@ -66,7 +73,7 @@ func (c *Client) Delete(p DeletePrm) error { prm := client.InvokePrm{} prm.SetMethod(deleteMethod) - prm.SetArgs(p.cnr, p.signature, p.token) + prm.SetArgs(p.cnr, p.signature, p.key, p.token) prm.InvokePrmOptional = p.InvokePrmOptional err := c.client.Invoke(prm) diff --git a/pkg/morph/event/container/delete.go b/pkg/morph/event/container/delete.go index 4926af27d..a206307f8 100644 --- a/pkg/morph/event/container/delete.go +++ b/pkg/morph/event/container/delete.go @@ -15,6 +15,7 @@ type Delete struct { ContainerIDValue []byte SignatureValue []byte TokenValue []byte + PublicKeyValue []byte // For notary notifications only. // Contains raw transactions of notary request. @@ -42,7 +43,7 @@ func (d Delete) NotaryRequest() *payload.P2PNotaryRequest { return d.NotaryRequestValue } -const expectedItemNumDelete = 3 +const expectedItemNumDelete = 4 // DeleteSuccess structures notification event of successful container removal // thrown by Container contract. diff --git a/pkg/morph/event/container/delete_notary.go b/pkg/morph/event/container/delete_notary.go index 23f13acbb..9711636e7 100644 --- a/pkg/morph/event/container/delete_notary.go +++ b/pkg/morph/event/container/delete_notary.go @@ -17,6 +17,10 @@ func (d *Delete) setSignature(v []byte) { } } +func (d *Delete) setPublicKey(v []byte) { + d.PublicKeyValue = v +} + func (d *Delete) setToken(v []byte) { if v != nil { d.TokenValue = v @@ -26,6 +30,7 @@ func (d *Delete) setToken(v []byte) { var deleteFieldSetters = []func(*Delete, []byte){ // order on stack is reversed (*Delete).setToken, + (*Delete).setPublicKey, (*Delete).setSignature, (*Delete).setContainerID, } diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index 8e6b30856..ae37da520 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -109,8 +109,6 @@ func (s *morphExecutor) Delete(_ context.Context, tokV2 *sessionV2.Token, body * return nil, fmt.Errorf("invalid container ID: %w", err) } - sig := body.GetSignature().GetSign() - var tok *session.Container if tokV2 != nil { @@ -124,9 +122,9 @@ func (s *morphExecutor) Delete(_ context.Context, tokV2 *sessionV2.Token, body * var rmWitness containercore.RemovalWitness - rmWitness.SetContainerID(id) - rmWitness.SetSignature(sig) - rmWitness.SetSessionToken(tok) + rmWitness.ContainerID = id + rmWitness.Signature = body.GetSignature() + rmWitness.SessionToken = tok err = s.wrt.Delete(rmWitness) if err != nil { From 01a0c977608ab6008d491b68097213d1265d309b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Jun 2023 11:59:18 +0300 Subject: [PATCH 0342/1943] [#453] engine: Set Disabled mode to deleted shard Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/shards.go | 9 ++++++++- pkg/local_object_storage/shard/errors.go | 3 +++ pkg/local_object_storage/shard/exists.go | 5 ++++- pkg/local_object_storage/shard/get.go | 5 +++++ pkg/local_object_storage/shard/mode.go | 8 +++++--- pkg/local_object_storage/shard/mode/mode.go | 4 ++++ pkg/local_object_storage/shard/range.go | 5 +++++ pkg/local_object_storage/shard/tree.go | 7 +++++++ 9 files changed, 42 insertions(+), 5 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index dc54f2d2a..2ba9e6e4b 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -492,4 +492,5 @@ const ( ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" ShardDeleteCantDeleteFromWriteCache = "can't delete object from write cache" FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" + EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled" ) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index cfa64929b..e80d7a158 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -203,7 +203,14 @@ func (e *StorageEngine) removeShards(ids ...string) { e.mtx.Unlock() for _, sh := range ss { - err := sh.Close() + err := sh.SetMode(mode.Disabled) + if err != nil { + e.log.Error(logs.EngineCouldNotChangeShardModeToDisabled, + zap.Stringer("id", sh.ID()), + zap.Error(err), + ) + } + err = sh.Close() if err != nil { e.log.Error(logs.EngineCouldNotCloseRemovedShard, zap.Stringer("id", sh.ID()), diff --git a/pkg/local_object_storage/shard/errors.go b/pkg/local_object_storage/shard/errors.go index 3e5224eb9..2958a492c 100644 --- a/pkg/local_object_storage/shard/errors.go +++ b/pkg/local_object_storage/shard/errors.go @@ -4,9 +4,12 @@ import ( "errors" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) +var ErrShardDisabled = logicerr.New("shard disabled") + // IsErrNotFound checks if error returned by Shard Get/Head/GetRange method // corresponds to missing object. func IsErrNotFound(err error) bool { diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 60809da6a..2cdb8dfa8 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -38,6 +38,7 @@ func (p ExistsRes) Exists() bool { // // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been marked as removed. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. +// Returns the ErrShardDisabled if the shard is disabled. func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Exists", trace.WithAttributes( @@ -52,7 +53,9 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { s.m.RLock() defer s.m.RUnlock() - if s.info.Mode.NoMetabase() { + if s.info.Mode.Disabled() { + return ExistsRes{}, ErrShardDisabled + } else if s.info.Mode.NoMetabase() { var p common.ExistsPrm p.Address = prm.addr diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 589ec53c9..3eb70784e 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -66,6 +66,7 @@ func (r GetRes) HasMeta() bool { // Returns an error of type apistatus.ObjectNotFound if the requested object is missing in shard. // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. +// Returns the ErrShardDisabled if the shard is disabled. func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Get", trace.WithAttributes( @@ -78,6 +79,10 @@ func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { s.m.RLock() defer s.m.RUnlock() + if s.info.Mode.Disabled() { + return GetRes{}, ErrShardDisabled + } + cb := func(stor *blobstor.BlobStor, id []byte) (*objectSDK.Object, error) { var getPrm common.GetPrm getPrm.Address = prm.addr diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index a59f08704..1bab57448 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -56,9 +56,11 @@ func (s *Shard) setMode(m mode.Mode) error { } } - for i := range components { - if err := components[i].SetMode(m); err != nil { - return err + if !m.Disabled() { + for i := range components { + if err := components[i].SetMode(m); err != nil { + return err + } } } diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go index 65b2b5c89..49c888d63 100644 --- a/pkg/local_object_storage/shard/mode/mode.go +++ b/pkg/local_object_storage/shard/mode/mode.go @@ -57,3 +57,7 @@ func (m Mode) NoMetabase() bool { func (m Mode) ReadOnly() bool { return m&ReadOnly != 0 } + +func (m Mode) Disabled() bool { + return m == Disabled +} diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 895dc2fe0..362fd86c5 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -72,6 +72,7 @@ func (r RngRes) HasMeta() bool { // Returns an error of type apistatus.ObjectNotFound if the requested object is missing. // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. +// Returns the ErrShardDisabled if the shard is disabled. func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.GetRange", trace.WithAttributes( @@ -86,6 +87,10 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { s.m.RLock() defer s.m.RUnlock() + if s.info.Mode.Disabled() { + return RngRes{}, ErrShardDisabled + } + cb := func(stor *blobstor.BlobStor, id []byte) (*object.Object, error) { var getRngPrm common.GetRangePrm getRngPrm.Address = prm.addr diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 2331d37a8..81477325a 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -265,6 +265,13 @@ func (s *Shard) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (u ) defer span.End() + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return 0, ErrDegradedMode + } + if s.pilorama == nil { return 0, ErrPiloramaDisabled } From 028d4a805859bcd338446ced8674d02fb98018b4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Jun 2023 11:23:17 +0300 Subject: [PATCH 0343/1943] [#373] blobovnicza: Add metrics Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/blobovnicza.go | 10 ++++ .../blobovnicza/control.go | 14 +++++- .../blobovnicza/metrics.go | 16 +++++++ pkg/local_object_storage/blobovnicza/sizes.go | 2 + pkg/local_object_storage/engine/shards.go | 48 +++++++++++-------- 5 files changed, 67 insertions(+), 23 deletions(-) create mode 100644 pkg/local_object_storage/blobovnicza/metrics.go diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index 21c9f9e58..ecd1dc5e5 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -31,6 +31,8 @@ type cfg struct { objSizeLimit uint64 log *logger.Logger + + metrics Metrics } type boltDBCfg struct { @@ -52,6 +54,7 @@ func defaultCfg(c *cfg) { fullSizeLimit: 1 << 30, // 1GB objSizeLimit: 1 << 20, // 1MB log: &logger.Logger{Logger: zap.L()}, + metrics: &noopMetrics{}, } } @@ -112,3 +115,10 @@ func WithReadOnly(ro bool) Option { c.boltOptions.ReadOnly = ro } } + +// WithMetrics returns an option to set metrics storage. +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index 84274528a..c776afe06 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -35,6 +35,9 @@ func (b *Blobovnicza) Open() error { ) b.boltDB, err = bbolt.Open(b.path, b.perm, b.boltOptions) + if err == nil { + b.metrics.IncOpenCount() + } return err } @@ -81,7 +84,9 @@ func (b *Blobovnicza) Init() error { return fmt.Errorf("can't determine DB size: %w", err) } - b.filled.Store(uint64(info.Size())) + sz := uint64(info.Size()) + b.filled.Store(sz) + b.metrics.IncSize(sz) return err } @@ -91,5 +96,10 @@ func (b *Blobovnicza) Close() error { zap.String("path", b.path), ) - return b.boltDB.Close() + err := b.boltDB.Close() + if err == nil { + b.metrics.DecOpenCount() + b.metrics.DecSize(b.filled.Load()) + } + return err } diff --git a/pkg/local_object_storage/blobovnicza/metrics.go b/pkg/local_object_storage/blobovnicza/metrics.go new file mode 100644 index 000000000..1ffb7b1e2 --- /dev/null +++ b/pkg/local_object_storage/blobovnicza/metrics.go @@ -0,0 +1,16 @@ +package blobovnicza + +type Metrics interface { + IncOpenCount() + DecOpenCount() + + IncSize(size uint64) + DecSize(size uint64) +} + +type noopMetrics struct{} + +func (m *noopMetrics) IncOpenCount() {} +func (m *noopMetrics) DecOpenCount() {} +func (m *noopMetrics) IncSize(uint64) {} +func (m *noopMetrics) DecSize(uint64) {} diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index 1cc100d19..bdbc77d18 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -41,10 +41,12 @@ func upperPowerOfTwo(v uint64) uint64 { func (b *Blobovnicza) incSize(sz uint64) { b.filled.Add(sz) + b.metrics.IncSize(sz) } func (b *Blobovnicza) decSize(sz uint64) { b.filled.Add(^(sz - 1)) + b.metrics.DecSize(sz) } func (b *Blobovnicza) full() bool { diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index e80d7a158..f362e2a03 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -102,15 +102,35 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { return nil, fmt.Errorf("could not generate shard ID: %w", err) } + opts = e.appendMetrics(id, opts) + + sh := shard.New(append(opts, + shard.WithID(id), + shard.WithExpiredTombstonesCallback(e.processExpiredTombstones), + shard.WithExpiredLocksCallback(e.processExpiredLocks), + shard.WithDeletedLockCallback(e.processDeletedLocks), + shard.WithReportErrorFunc(e.reportShardErrorBackground), + )...) + + if err := sh.UpdateID(); err != nil { + return nil, fmt.Errorf("could not update shard ID: %w", err) + } + + return sh, err +} + +func (e *StorageEngine) appendMetrics(id *shard.ID, opts []shard.Option) []shard.Option { e.mtx.RLock() + defer e.mtx.RUnlock() if e.metrics != nil { - opts = append(opts, shard.WithMetricsWriter( - &metricsWithID{ - id: id.String(), - mw: e.metrics, - }, - ), + opts = append(opts, + shard.WithMetricsWriter( + &metricsWithID{ + id: id.String(), + mw: e.metrics, + }, + ), shard.WithExtraWriteCacheOptions(writecache.WithMetrics( &writeCacheMetrics{ shardID: id.String(), @@ -126,21 +146,7 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { ) } - e.mtx.RUnlock() - - sh := shard.New(append(opts, - shard.WithID(id), - shard.WithExpiredTombstonesCallback(e.processExpiredTombstones), - shard.WithExpiredLocksCallback(e.processExpiredLocks), - shard.WithDeletedLockCallback(e.processDeletedLocks), - shard.WithReportErrorFunc(e.reportShardErrorBackground), - )...) - - if err := sh.UpdateID(); err != nil { - return nil, fmt.Errorf("could not update shard ID: %w", err) - } - - return sh, err + return opts } func (e *StorageEngine) addShard(sh *shard.Shard) error { From a8526d45e9451275939e894ed4464d50356843ed Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Jun 2023 11:24:14 +0300 Subject: [PATCH 0344/1943] [#373] blobovnizca: Add missed/fix tracing spans Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/blobovnicza_test.go | 2 +- .../blobovnicza/delete.go | 12 ++++++------ .../blobovnicza/exists.go | 19 ++++++++++++++++--- pkg/local_object_storage/blobovnicza/get.go | 1 + .../blobovnicza/get_test.go | 6 +++--- .../blobovnicza/iterate.go | 12 ++++++++++++ .../blobovnicza/iterate_test.go | 2 +- pkg/local_object_storage/blobovnicza/put.go | 14 +++++++++++++- .../blobstor/blobovniczatree/exists.go | 2 +- .../blobstor/blobovniczatree/put.go | 8 ++++---- 10 files changed, 58 insertions(+), 20 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index 5deaf5e4a..05f8906ac 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -21,7 +21,7 @@ func testPutGet(t *testing.T, blz *Blobovnicza, addr oid.Address, sz uint64, ass var pPut PutPrm pPut.SetAddress(addr) pPut.SetMarshaledObject(data) - _, err := blz.Put(pPut) + _, err := blz.Put(context.Background(), pPut) if assertErrPut != nil { require.True(t, assertErrPut(err)) } else { diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index e880815a6..8fbd363db 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -38,13 +38,14 @@ func (p *DeletePrm) SetAddress(addr oid.Address) { func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Delete", trace.WithAttributes( + attribute.String("path", b.path), attribute.String("address", prm.addr.EncodeToString()), )) defer span.End() addrKey := addressKey(prm.addr) - removed := false + found := false err := b.boltDB.Update(func(tx *bbolt.Tx) error { return b.iterateBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { @@ -56,9 +57,6 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err sz := uint64(len(objData)) - // decrease fullness counter - b.decSize(sz) - // remove object from the bucket err := buck.Delete(addrKey) @@ -67,16 +65,18 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err zap.String("binary size", stringifyByteSize(sz)), zap.String("range", stringifyBounds(lower, upper)), ) + // decrease fullness counter + b.decSize(sz) } - removed = true + found = true // stop iteration return true, err }) }) - if err == nil && !removed { + if err == nil && !found { var errNotFound apistatus.ObjectNotFound return DeleteRes{}, errNotFound diff --git a/pkg/local_object_storage/blobovnicza/exists.go b/pkg/local_object_storage/blobovnicza/exists.go index 8ac45c4aa..e6d28f938 100644 --- a/pkg/local_object_storage/blobovnicza/exists.go +++ b/pkg/local_object_storage/blobovnicza/exists.go @@ -1,17 +1,30 @@ package blobovnicza import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Exists check if object with the specified address is stored in b. -func (b *Blobovnicza) Exists(addr oid.Address) (bool, error) { +func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error) { var ( - exists bool - addrKey = addressKey(addr) + exists = false ) + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Exists", + trace.WithAttributes( + attribute.String("path", b.path), + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + + addrKey := addressKey(addr) + err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(_ []byte, buck *bbolt.Bucket) error { exists = buck.Get(addrKey) != nil diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index ff29358b0..d492b7559 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -46,6 +46,7 @@ var errInterruptForEach = errors.New("interrupt for-each") func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Get", trace.WithAttributes( + attribute.String("path", b.path), attribute.String("address", prm.addr.EncodeToString()), )) defer span.End() diff --git a/pkg/local_object_storage/blobovnicza/get_test.go b/pkg/local_object_storage/blobovnicza/get_test.go index ad30e8d94..40c434eb7 100644 --- a/pkg/local_object_storage/blobovnicza/get_test.go +++ b/pkg/local_object_storage/blobovnicza/get_test.go @@ -41,7 +41,7 @@ func TestBlobovnicza_Get(t *testing.T) { addr := oidtest.Address() obj := make([]byte, firstBucketBound+1) - exists, err := blz.Exists(addr) + exists, err := blz.Exists(context.Background(), addr) require.NoError(t, err) require.False(t, exists) @@ -50,7 +50,7 @@ func TestBlobovnicza_Get(t *testing.T) { prmPut.SetMarshaledObject(obj) // place object to [32K:64K] bucket - _, err = blz.Put(prmPut) + _, err = blz.Put(context.Background(), prmPut) require.NoError(t, err) var prmGet GetPrm @@ -61,7 +61,7 @@ func TestBlobovnicza_Get(t *testing.T) { require.NoError(t, err) require.Equal(t, obj, res.Object()) - exists, err := blz.Exists(addr) + exists, err := blz.Exists(context.Background(), addr) require.NoError(t, err) require.True(t, exists) } diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index 8f8357053..c2031ea54 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -4,8 +4,11 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) func (b *Blobovnicza) iterateBuckets(tx *bbolt.Tx, f func(uint64, uint64, *bbolt.Bucket) (bool, error)) error { @@ -119,6 +122,15 @@ type IterateRes struct { // // Handler should not retain object data. Handler must not be nil. func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Iterate", + trace.WithAttributes( + attribute.String("path", b.path), + attribute.Bool("decode_addresses", prm.decodeAddresses), + attribute.Bool("without_data", prm.withoutData), + attribute.Bool("ignore_errors", prm.ignoreErrors), + )) + defer span.End() + var elem IterationElement if err := b.boltDB.View(func(tx *bbolt.Tx) error { diff --git a/pkg/local_object_storage/blobovnicza/iterate_test.go b/pkg/local_object_storage/blobovnicza/iterate_test.go index 505685ced..90308723c 100644 --- a/pkg/local_object_storage/blobovnicza/iterate_test.go +++ b/pkg/local_object_storage/blobovnicza/iterate_test.go @@ -20,7 +20,7 @@ func TestBlobovniczaIterate(t *testing.T) { data := [][]byte{{0, 1, 2, 3}, {5, 6, 7, 8}} addr := oidtest.Address() - _, err := b.Put(PutPrm{addr: addr, objData: data[0]}) + _, err := b.Put(context.Background(), PutPrm{addr: addr, objData: data[0]}) require.NoError(t, err) require.NoError(t, b.boltDB.Update(func(tx *bbolt.Tx) error { diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index 37ed57e1c..cc816b3e1 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -1,11 +1,15 @@ package blobovnicza import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // PutPrm groups the parameters of Put operation. @@ -47,7 +51,15 @@ func (p *PutPrm) SetMarshaledObject(data []byte) { // Returns ErrFull if blobovnicza is filled. // // Should not be called in read-only configuration. -func (b *Blobovnicza) Put(prm PutPrm) (PutRes, error) { +func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Put", + trace.WithAttributes( + attribute.String("path", b.path), + attribute.String("address", prm.addr.EncodeToString()), + attribute.Int("size", len(prm.objData)), + )) + defer span.End() + sz := uint64(len(prm.objData)) bucketName := bucketForSize(sz) key := addressKey(prm.addr) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index 3324507cf..c2a4740d8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -30,7 +30,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common return common.ExistsRes{}, err } - exists, err := blz.Exists(prm.Address) + exists, err := blz.Exists(ctx, prm.Address) return common.ExistsRes{Exists: exists}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 95ed15540..a567baa3a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -45,7 +45,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe PutPrm: putPrm, } - if err := b.iterateDeepest(ctx, prm.Address, it.iterate); err != nil { + if err := b.iterateDeepest(ctx, prm.Address, func(s string) (bool, error) { return it.iterate(ctx, s) }); err != nil { return common.PutRes{}, err } else if it.ID == nil { if it.AllFull { @@ -64,7 +64,7 @@ type putIterator struct { PutPrm blobovnicza.PutPrm } -func (i *putIterator) iterate(path string) (bool, error) { +func (i *putIterator) iterate(ctx context.Context, path string) (bool, error) { active, err := i.B.getActivated(path) if err != nil { if !isLogical(err) { @@ -77,7 +77,7 @@ func (i *putIterator) iterate(path string) (bool, error) { return false, nil } - if _, err := active.blz.Put(i.PutPrm); err != nil { + if _, err := active.blz.Put(ctx, i.PutPrm); err != nil { // Check if blobovnicza is full. We could either receive `blobovnicza.ErrFull` error // or update active blobovnicza in other thread. In the latter case the database will be closed // and `updateActive` takes care of not updating the active blobovnicza twice. @@ -99,7 +99,7 @@ func (i *putIterator) iterate(path string) (bool, error) { return false, nil } - return i.iterate(path) + return i.iterate(ctx, path) } i.AllFull = false From 3ae3c8dfdb806efc33ee809c512be5bce157a1ec Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 2 Jun 2023 16:04:02 +0300 Subject: [PATCH 0345/1943] [#373] fstree: Add metrics Signed-off-by: Dmitrii Stepanov --- .../blobstor/fstree/control.go | 6 +- .../blobstor/fstree/fstree.go | 100 ++++++++++++++++-- .../blobstor/fstree/metrics.go | 26 +++++ .../blobstor/fstree/option.go | 6 ++ 4 files changed, 126 insertions(+), 12 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/fstree/metrics.go diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index 1ff74893d..f41b7aacd 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -7,6 +7,7 @@ import ( // Open implements common.Storage. func (t *FSTree) Open(ro bool) error { t.readOnly = ro + t.metrics.SetMode(ro) return nil } @@ -16,4 +17,7 @@ func (t *FSTree) Init() error { } // Close implements common.Storage. -func (*FSTree) Close() error { return nil } +func (t *FSTree) Close() error { + t.metrics.Close() + return nil +} diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 76487813e..645d4e5f7 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "syscall" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" @@ -35,6 +36,7 @@ type FSTree struct { noSync bool readOnly bool + metrics Metrics } // Info groups the information about file storage. @@ -64,6 +66,7 @@ func New(opts ...Option) *FSTree { Config: nil, Depth: 4, DirNameLen: DirNameLen, + metrics: &noopMetrics{}, } for i := range opts { opts[i](f) @@ -101,7 +104,24 @@ func addressFromString(s string) (oid.Address, error) { // Iterate iterates over all stored objects. func (t *FSTree) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { - return common.IterateRes{}, t.iterate(ctx, 0, []string{t.RootPath}, prm) + var ( + err error + startedAt = time.Now() + ) + + defer func() { + t.metrics.Iterate(time.Since(startedAt), err == nil) + }() + + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Iterate", + trace.WithAttributes( + attribute.String("path", t.RootPath), + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + + err = t.iterate(ctx, 0, []string{t.RootPath}, prm) + return common.IterateRes{}, err } func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, prm common.IteratePrm) error { @@ -202,19 +222,29 @@ func (t *FSTree) treePath(addr oid.Address) string { // Delete removes the object with the specified address from the storage. func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + var ( + err error + startedAt = time.Now() + ) + defer func() { + t.metrics.Delete(time.Since(startedAt), err == nil) + }() + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Delete", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), )) defer span.End() if t.readOnly { - return common.DeleteRes{}, common.ErrReadOnly + err = common.ErrReadOnly + return common.DeleteRes{}, err } p := t.treePath(prm.Address) - err := os.Remove(p) + err = os.Remove(p) if err != nil && os.IsNotExist(err) { err = logicerr.Wrap(apistatus.ObjectNotFound{}) } @@ -224,8 +254,17 @@ func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Delet // Exists returns the path to the file with object contents if it exists in the storage // and an error otherwise. func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + var ( + success = false + startedAt = time.Now() + ) + defer func() { + t.metrics.Exists(time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Exists", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), )) defer span.End() @@ -237,27 +276,40 @@ func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exist if os.IsNotExist(err) { err = nil } + success = err == nil return common.ExistsRes{Exists: found}, err } // Put puts an object in the storage. func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + var ( + size int + startedAt = time.Now() + err error + ) + defer func() { + t.metrics.Put(time.Since(startedAt), size, err == nil) + }() + _, span := tracing.StartSpanFromContext(ctx, "FSTree.Put", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.Bool("dont_compress", prm.DontCompress), )) defer span.End() if t.readOnly { - return common.PutRes{}, common.ErrReadOnly + err = common.ErrReadOnly + return common.PutRes{}, err } p := t.treePath(prm.Address) - if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { + if err = util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { if errors.Is(err, syscall.ENOSPC) { - return common.PutRes{}, common.ErrNoSpace + err = common.ErrNoSpace + return common.PutRes{}, err } return common.PutRes{}, err } @@ -287,17 +339,19 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err // to be so hecking simple. // In a very rare situation we can have multiple partially written copies on disk, // this will be fixed in another issue (we should remove garbage on start). + size = len(prm.RawData) const retryCount = 5 for i := 0; i < retryCount; i++ { tmpPath := p + "#" + strconv.FormatUint(uint64(i), 10) - err := t.writeAndRename(tmpPath, p, prm.RawData) + err = t.writeAndRename(tmpPath, p, prm.RawData) if err != syscall.EEXIST || i == retryCount-1 { return common.PutRes{StorageID: []byte{}}, err } } + err = fmt.Errorf("couldn't read file after %d retries", retryCount) // unreachable, but precaution never hurts, especially 1 day before release. - return common.PutRes{StorageID: []byte{}}, fmt.Errorf("couldn't read file after %d retries", retryCount) + return common.PutRes{StorageID: []byte{}}, err } // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. @@ -365,8 +419,18 @@ func (t *FSTree) PutStream(addr oid.Address, handler func(*os.File) error) error // Get returns an object from the storage by address. func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { + var ( + startedAt = time.Now() + success = false + size = 0 + ) + defer func() { + t.metrics.Get(time.Since(startedAt), size, success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "FSTree.Get", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.Bool("raw", prm.Raw), attribute.String("address", prm.Address.EncodeToString()), )) @@ -394,19 +458,30 @@ func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, err if err != nil { return common.GetRes{}, err } + size = len(data) obj := objectSDK.New() if err := obj.Unmarshal(data); err != nil { return common.GetRes{}, err } - - return common.GetRes{Object: obj, RawData: data}, err + success = true + return common.GetRes{Object: obj, RawData: data}, nil } // GetRange implements common.Storage. func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.GetRangeRes, error) { + var ( + startedAt = time.Now() + success = false + size = 0 + ) + defer func() { + t.metrics.GetRange(time.Since(startedAt), size, success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "FSTree.GetRange", trace.WithAttributes( + attribute.String("path", t.RootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), attribute.String("length", strconv.FormatUint(prm.Range.GetLength(), 10)), @@ -426,8 +501,11 @@ func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.G return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectOutOfRange{}) } + success = true + data := payload[from:to] + size = len(data) return common.GetRangeRes{ - Data: payload[from:to], + Data: data, }, nil } diff --git a/pkg/local_object_storage/blobstor/fstree/metrics.go b/pkg/local_object_storage/blobstor/fstree/metrics.go new file mode 100644 index 000000000..04386d9b8 --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/metrics.go @@ -0,0 +1,26 @@ +package fstree + +import "time" + +type Metrics interface { + SetMode(readOnly bool) + Close() + + Iterate(d time.Duration, success bool) + Delete(d time.Duration, success bool) + Exists(d time.Duration, success bool) + Put(d time.Duration, size int, success bool) + Get(d time.Duration, size int, success bool) + GetRange(d time.Duration, size int, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) Iterate(time.Duration, bool) {} +func (m *noopMetrics) Delete(time.Duration, bool) {} +func (m *noopMetrics) Exists(time.Duration, bool) {} +func (m *noopMetrics) Put(time.Duration, int, bool) {} +func (m *noopMetrics) Get(time.Duration, int, bool) {} +func (m *noopMetrics) GetRange(time.Duration, int, bool) {} diff --git a/pkg/local_object_storage/blobstor/fstree/option.go b/pkg/local_object_storage/blobstor/fstree/option.go index 07e547444..52c8718c2 100644 --- a/pkg/local_object_storage/blobstor/fstree/option.go +++ b/pkg/local_object_storage/blobstor/fstree/option.go @@ -35,3 +35,9 @@ func WithNoSync(noSync bool) Option { f.noSync = noSync } } + +func WithMetrics(m Metrics) Option { + return func(f *FSTree) { + f.metrics = m + } +} From 8318d90ad0c34847d4ea0d2576f44f97fe6148ac Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Jun 2023 10:25:25 +0300 Subject: [PATCH 0346/1943] [#373] blobovniczatree: Add metrics Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/blobovnicza.go | 2 +- .../blobovnicza/metrics.go | 10 +++--- .../blobstor/blobovniczatree/control.go | 7 +++- .../blobstor/blobovniczatree/delete.go | 16 ++++++++- .../blobstor/blobovniczatree/exists.go | 13 ++++++- .../blobstor/blobovniczatree/get.go | 20 ++++++++++- .../blobstor/blobovniczatree/get_range.go | 27 +++++++++++--- .../blobstor/blobovniczatree/iterate.go | 22 +++++++++++- .../blobstor/blobovniczatree/metrics.go | 35 +++++++++++++++++++ .../blobstor/blobovniczatree/option.go | 8 +++++ .../blobstor/blobovniczatree/put.go | 12 +++++++ 11 files changed, 157 insertions(+), 15 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/metrics.go diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index ecd1dc5e5..d5741fba7 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -54,7 +54,7 @@ func defaultCfg(c *cfg) { fullSizeLimit: 1 << 30, // 1GB objSizeLimit: 1 << 20, // 1MB log: &logger.Logger{Logger: zap.L()}, - metrics: &noopMetrics{}, + metrics: &NoopMetrics{}, } } diff --git a/pkg/local_object_storage/blobovnicza/metrics.go b/pkg/local_object_storage/blobovnicza/metrics.go index 1ffb7b1e2..6127370bc 100644 --- a/pkg/local_object_storage/blobovnicza/metrics.go +++ b/pkg/local_object_storage/blobovnicza/metrics.go @@ -8,9 +8,9 @@ type Metrics interface { DecSize(size uint64) } -type noopMetrics struct{} +type NoopMetrics struct{} -func (m *noopMetrics) IncOpenCount() {} -func (m *noopMetrics) DecOpenCount() {} -func (m *noopMetrics) IncSize(uint64) {} -func (m *noopMetrics) DecSize(uint64) {} +func (m *NoopMetrics) IncOpenCount() {} +func (m *NoopMetrics) DecOpenCount() {} +func (m *NoopMetrics) IncSize(uint64) {} +func (m *NoopMetrics) DecSize(uint64) {} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 0b2135631..0045e08db 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -13,6 +13,7 @@ import ( // Open opens blobovnicza tree. func (b *Blobovniczas) Open(readOnly bool) error { b.readOnly = readOnly + b.metrics.SetMode(readOnly) return nil } @@ -70,6 +71,7 @@ func (b *Blobovniczas) Close() error { } b.active = make(map[string]blobovniczaWithIndex) + b.metrics.Close() b.lruMtx.Unlock() @@ -123,9 +125,12 @@ func (b *Blobovniczas) openBlobovniczaNoCache(p string) (*blobovnicza.Blobovnicz b.openMtx.Lock() defer b.openMtx.Unlock() + path := filepath.Join(b.rootPath, p) + blz := blobovnicza.New(append(b.blzOpts, blobovnicza.WithReadOnly(b.readOnly), - blobovnicza.WithPath(filepath.Join(b.rootPath, p)), + blobovnicza.WithPath(path), + blobovnicza.WithMetrics(b.metrics.BlobovnicaMetrics(path)), )...) if err := blz.Open(); err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 1e38c6ea1..4d0801ef1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "path/filepath" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" @@ -21,8 +22,17 @@ import ( // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res common.DeleteRes, err error) { + var ( + success = false + startedAt = time.Now() + ) + defer func() { + b.metrics.Delete(time.Since(startedAt), success, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Delete", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), )) @@ -42,7 +52,10 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co return res, err } - return b.deleteObject(ctx, blz, bPrm) + if res, err = b.deleteObject(ctx, blz, bPrm); err == nil { + success = true + } + return res, err } activeCache := make(map[string]struct{}) @@ -78,6 +91,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co // not found in any blobovnicza return common.DeleteRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } + success = err == nil return } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index c2a4740d8..cd553ec30 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "path/filepath" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" @@ -16,8 +17,18 @@ import ( // Exists implements common.Storage. func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + var ( + startedAt = time.Now() + success = false + found = false + ) + defer func() { + b.metrics.Exists(time.Since(startedAt), success, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Exists", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), )) @@ -39,7 +50,6 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common var gPrm blobovnicza.GetPrm gPrm.SetAddress(prm.Address) - var found bool err := b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) @@ -59,5 +69,6 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common return found, nil }) + success = err == nil return common.ExistsRes{Exists: found}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 3c8e288d4..300e3ce3a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "path/filepath" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" @@ -23,8 +24,18 @@ import ( // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.GetRes, err error) { + var ( + startedAt = time.Now() + found = false + size = 0 + ) + defer func() { + b.metrics.Get(time.Since(startedAt), size, found, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Get", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), attribute.Bool("raw", prm.Raw), @@ -41,7 +52,11 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G return res, err } - return b.getObject(ctx, blz, bPrm) + res, err = b.getObject(ctx, blz, bPrm) + if err == nil { + found = true + size = len(res.RawData) + } } activeCache := make(map[string]struct{}) @@ -72,6 +87,9 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G return res, logicerr.Wrap(apistatus.ObjectNotFound{}) } + found = true + size = len(res.RawData) + return } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 6e9620adf..3530c1a41 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -6,6 +6,7 @@ import ( "fmt" "path/filepath" "strconv" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" @@ -24,8 +25,18 @@ import ( // If blobocvnicza ID is specified, only this blobovnicza is processed. // Otherwise, all Blobovniczas are processed descending weight. func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (res common.GetRangeRes, err error) { + var ( + startedAt = time.Now() + found = false + size = 0 + ) + defer func() { + b.metrics.GetRange(time.Since(startedAt), size, found, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.GetRange", trace.WithAttributes( + attribute.String("path", b.rootPath), attribute.String("address", prm.Address.EncodeToString()), attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), @@ -40,11 +51,15 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re return common.GetRangeRes{}, err } - return b.getObjectRange(ctx, blz, prm) + res, err := b.getObjectRange(ctx, blz, prm) + if err == nil { + size = len(res.Data) + found = true + } + return res, err } activeCache := make(map[string]struct{}) - objectFound := false err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) @@ -67,17 +82,21 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re activeCache[dirPath] = struct{}{} - objectFound = err == nil + found = err == nil // abort iterator if found, otherwise process all Blobovniczas return err == nil, nil }) - if err == nil && !objectFound { + if err == nil && !found { // not found in any blobovnicza return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } + if err == nil { + size = len(res.Data) + } + return } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 140716690..0154fe2ca 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -4,16 +4,35 @@ import ( "context" "fmt" "path/filepath" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Iterate iterates over all objects in b. func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { - return common.IterateRes{}, b.iterateBlobovniczas(ctx, prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { + var ( + startedAt = time.Now() + err error + ) + defer func() { + b.metrics.Iterate(time.Since(startedAt), err == nil) + }() + + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Iterate", + trace.WithAttributes( + attribute.String("path", b.rootPath), + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + + err = b.iterateBlobovniczas(ctx, prm.IgnoreErrors, func(p string, blz *blobovnicza.Blobovnicza) error { var subPrm blobovnicza.IteratePrm subPrm.SetHandler(func(elem blobovnicza.IterationElement) error { data, err := b.compression.Decompress(elem.ObjectData()) @@ -43,6 +62,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm _, err := blz.Iterate(ctx, subPrm) return err }) + return common.IterateRes{}, err } // iterator over all Blobovniczas in unsorted order. Break on f's error return. diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go new file mode 100644 index 000000000..ffa3c8a8b --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -0,0 +1,35 @@ +package blobovniczatree + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" +) + +type Metrics interface { + BlobovnicaMetrics(path string) blobovnicza.Metrics + + SetMode(readOnly bool) + Close() + + Delete(d time.Duration, success, withStorageID bool) + Exists(d time.Duration, success, withStorageID bool) + GetRange(d time.Duration, size int, success, withStorageID bool) + Get(d time.Duration, size int, success, withStorageID bool) + Iterate(d time.Duration, success bool) + Put(d time.Duration, size int, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) BlobovnicaMetrics(string) blobovnicza.Metrics { + return &blobovnicza.NoopMetrics{} +} +func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) Delete(time.Duration, bool, bool) {} +func (m *noopMetrics) Exists(time.Duration, bool, bool) {} +func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Iterate(time.Duration, bool) {} +func (m *noopMetrics) Put(time.Duration, int, bool) {} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 95ef8635a..d0503f23b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -21,6 +21,7 @@ type cfg struct { blzOpts []blobovnicza.Option // reportError is the function called when encountering disk errors. reportError func(string, error) + metrics Metrics } type Option func(*cfg) @@ -40,6 +41,7 @@ func initConfig(c *cfg) { blzShallowDepth: defaultBlzShallowDepth, blzShallowWidth: defaultBlzShallowWidth, reportError: func(string, error) {}, + metrics: &noopMetrics{}, } } @@ -91,3 +93,9 @@ func WithObjectSizeLimit(sz uint64) Option { c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz)) } } + +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index a567baa3a..038d5244d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -4,6 +4,7 @@ import ( "context" "errors" "path/filepath" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" @@ -19,6 +20,15 @@ import ( // // returns error if could not save object in any blobovnicza. func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + var ( + success bool + size int + startedAt = time.Now() + ) + defer func() { + b.metrics.Put(time.Since(startedAt), size, success) + }() + _, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Put", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -33,6 +43,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe if !prm.DontCompress { prm.RawData = b.compression.Compress(prm.RawData) } + size = len(prm.RawData) var putPrm blobovnicza.PutPrm putPrm.SetAddress(prm.Address) @@ -54,6 +65,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe return common.PutRes{}, errPutFailed } + success = true return common.PutRes{StorageID: it.ID.Bytes()}, nil } From f54cc0b607a3f02b167c79d7bc09bd67d8442878 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Jun 2023 09:05:52 +0300 Subject: [PATCH 0347/1943] [#373] blobstor: Add metrics Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/get.go | 8 +++--- .../blobstor/blobovniczatree/get_range.go | 12 +++++---- pkg/local_object_storage/blobstor/blobstor.go | 8 ++++++ pkg/local_object_storage/blobstor/control.go | 4 +++ pkg/local_object_storage/blobstor/delete.go | 11 ++++++++ pkg/local_object_storage/blobstor/exists.go | 18 +++++++++++-- pkg/local_object_storage/blobstor/get.go | 18 ++++++++++--- .../blobstor/get_range.go | 18 ++++++++++--- pkg/local_object_storage/blobstor/iterate.go | 18 +++++++++++++ pkg/local_object_storage/blobstor/metrics.go | 26 +++++++++++++++++++ pkg/local_object_storage/blobstor/put.go | 12 +++++++++ 11 files changed, 134 insertions(+), 19 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/metrics.go diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 300e3ce3a..5dafedd1c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -26,11 +26,11 @@ import ( func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.GetRes, err error) { var ( startedAt = time.Now() - found = false + success = false size = 0 ) defer func() { - b.metrics.Get(time.Since(startedAt), size, found, prm.StorageID != nil) + b.metrics.Get(time.Since(startedAt), size, success, prm.StorageID != nil) }() ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.Get", @@ -54,7 +54,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G res, err = b.getObject(ctx, blz, bPrm) if err == nil { - found = true + success = true size = len(res.RawData) } } @@ -87,7 +87,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G return res, logicerr.Wrap(apistatus.ObjectNotFound{}) } - found = true + success = true size = len(res.RawData) return diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 3530c1a41..8579502c0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -27,11 +27,11 @@ import ( func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (res common.GetRangeRes, err error) { var ( startedAt = time.Now() - found = false + success = false size = 0 ) defer func() { - b.metrics.GetRange(time.Since(startedAt), size, found, prm.StorageID != nil) + b.metrics.GetRange(time.Since(startedAt), size, success, prm.StorageID != nil) }() ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.GetRange", @@ -54,12 +54,13 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re res, err := b.getObjectRange(ctx, blz, prm) if err == nil { size = len(res.Data) - found = true + success = true } return res, err } activeCache := make(map[string]struct{}) + objectFound := false err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { dirPath := filepath.Dir(p) @@ -82,18 +83,19 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re activeCache[dirPath] = struct{}{} - found = err == nil + objectFound = err == nil // abort iterator if found, otherwise process all Blobovniczas return err == nil, nil }) - if err == nil && !found { + if err == nil && !objectFound { // not found in any blobovnicza return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } if err == nil { + success = true size = len(res.Data) } diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index a6fe9935e..a422b2bce 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -43,10 +43,12 @@ type cfg struct { compression compression.Config log *logger.Logger storage []SubStorage + metrics Metrics } func initConfig(c *cfg) { c.log = &logger.Logger{Logger: zap.L()} + c.metrics = &noopMetrics{} } // New creates, initializes and returns new BlobStor instance. @@ -113,3 +115,9 @@ func (b *BlobStor) SetReportErrorFunc(f func(string, error)) { b.storage[i].Storage.SetReportErrorFunc(f) } } + +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index abe39575b..6b439dcf0 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -18,6 +18,7 @@ func (b *BlobStor) Open(readOnly bool) error { return err } } + b.metrics.SetMode(readOnly) return nil } @@ -65,5 +66,8 @@ func (b *BlobStor) Close() error { if firstErr == nil { firstErr = err } + if firstErr == nil { + b.metrics.Close() + } return firstErr } diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index f1b14481c..fe49fc46a 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "errors" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -13,6 +14,14 @@ import ( ) func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + b.metrics.Delete(time.Since(startedAt), success, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Delete", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -28,6 +37,7 @@ func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.Del res, err := b.storage[i].Storage.Delete(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { if err == nil { + success = true logOp(b.log, deleteOp, prm.Address, b.storage[i].Storage.Type(), prm.StorageID) } return res, err @@ -45,6 +55,7 @@ func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.Del res, err := st.Delete(ctx, prm) if err == nil { + success = true logOp(b.log, deleteOp, prm.Address, st.Type(), prm.StorageID) } diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 760e7b2a4..03dad392a 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -3,6 +3,7 @@ package blobstor import ( "context" "encoding/hex" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -17,6 +18,14 @@ import ( // Returns any error encountered that did not allow // to completely check object existence. func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + var ( + exists = false + startedAt = time.Now() + ) + defer func() { + b.metrics.Exists(time.Since(startedAt), exists, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Exists", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -29,9 +38,13 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi if prm.StorageID != nil { if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.Exists(ctx, prm) + res, err := b.storage[len(b.storage)-1].Storage.Exists(ctx, prm) + exists = err == nil && res.Exists + return res, err } - return b.storage[0].Storage.Exists(ctx, prm) + res, err := b.storage[0].Storage.Exists(ctx, prm) + exists = err == nil && res.Exists + return res, err } // If there was an error during existence check below, @@ -47,6 +60,7 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi for i := range b.storage { res, err := b.storage[i].Storage.Exists(ctx, prm) if err == nil && res.Exists { + exists = true return res, nil } else if err != nil { errors = append(errors, err) diff --git a/pkg/local_object_storage/blobstor/get.go b/pkg/local_object_storage/blobstor/get.go index eadb990c8..fab86ea5d 100644 --- a/pkg/local_object_storage/blobstor/get.go +++ b/pkg/local_object_storage/blobstor/get.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "errors" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -16,7 +17,14 @@ import ( // Get reads the object from b. // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. -func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { +func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (res common.GetRes, err error) { + var ( + startedAt = time.Now() + ) + defer func() { + b.metrics.Get(time.Since(startedAt), len(res.RawData), err == nil, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Get", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -30,7 +38,7 @@ func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, e if prm.StorageID == nil { for i := range b.storage { - res, err := b.storage[i].Storage.Get(ctx, prm) + res, err = b.storage[i].Storage.Get(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { return res, err } @@ -39,7 +47,9 @@ func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, e return common.GetRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.Get(ctx, prm) + res, err = b.storage[len(b.storage)-1].Storage.Get(ctx, prm) + } else { + res, err = b.storage[0].Storage.Get(ctx, prm) } - return b.storage[0].Storage.Get(ctx, prm) + return res, err } diff --git a/pkg/local_object_storage/blobstor/get_range.go b/pkg/local_object_storage/blobstor/get_range.go index ca4e41f33..671fbf02e 100644 --- a/pkg/local_object_storage/blobstor/get_range.go +++ b/pkg/local_object_storage/blobstor/get_range.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "errors" "strconv" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -17,7 +18,14 @@ import ( // GetRange reads object payload data from b. // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. -func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (common.GetRangeRes, error) { +func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (res common.GetRangeRes, err error) { + var ( + startedAt = time.Now() + ) + defer func() { + b.metrics.GetRange(time.Since(startedAt), len(res.Data), err == nil, prm.StorageID != nil) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.GetRange", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -32,7 +40,7 @@ func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (common if prm.StorageID == nil { for i := range b.storage { - res, err := b.storage[i].Storage.GetRange(ctx, prm) + res, err = b.storage[i].Storage.GetRange(ctx, prm) if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { return res, err } @@ -41,7 +49,9 @@ func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (common return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) } if len(prm.StorageID) == 0 { - return b.storage[len(b.storage)-1].Storage.GetRange(ctx, prm) + res, err = b.storage[len(b.storage)-1].Storage.GetRange(ctx, prm) + } else { + res, err = b.storage[0].Storage.GetRange(ctx, prm) } - return b.storage[0].Storage.GetRange(ctx, prm) + return res, err } diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index 4e52f0abf..5a41e4c4f 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -3,10 +3,14 @@ package blobstor import ( "context" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -18,6 +22,19 @@ import ( // // If handler returns an error, method wraps and returns it immediately. func (b *BlobStor) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + b.metrics.Iterate(time.Since(startedAt), success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Iterate", + trace.WithAttributes( + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + b.modeMtx.RLock() defer b.modeMtx.RUnlock() @@ -27,6 +44,7 @@ func (b *BlobStor) Iterate(ctx context.Context, prm common.IteratePrm) (common.I return common.IterateRes{}, fmt.Errorf("blobstor iterator failure: %w", err) } } + success = true return common.IterateRes{}, nil } diff --git a/pkg/local_object_storage/blobstor/metrics.go b/pkg/local_object_storage/blobstor/metrics.go new file mode 100644 index 000000000..4fd6f3e54 --- /dev/null +++ b/pkg/local_object_storage/blobstor/metrics.go @@ -0,0 +1,26 @@ +package blobstor + +import "time" + +type Metrics interface { + SetMode(readOnly bool) + Close() + + Delete(d time.Duration, success, withStorageID bool) + Exists(d time.Duration, success, withStorageID bool) + GetRange(d time.Duration, size int, success, withStorageID bool) + Get(d time.Duration, size int, success, withStorageID bool) + Iterate(d time.Duration, success bool) + Put(d time.Duration, size int, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) Delete(time.Duration, bool, bool) {} +func (m *noopMetrics) Exists(time.Duration, bool, bool) {} +func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} +func (m *noopMetrics) Iterate(time.Duration, bool) {} +func (m *noopMetrics) Put(time.Duration, int, bool) {} diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index 125b445b6..a748750dd 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -3,6 +3,7 @@ package blobstor import ( "context" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -26,6 +27,15 @@ var ErrNoPlaceFound = logicerr.New("couldn't find a place to store an object") // Returns any error encountered that // did not allow to completely save the object. func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + var ( + startedAt = time.Now() + success = false + size = 0 + ) + defer func() { + b.metrics.Put(time.Since(startedAt), size, success) + }() + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.Put", trace.WithAttributes( attribute.String("address", prm.Address.EncodeToString()), @@ -47,11 +57,13 @@ func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e } prm.RawData = data } + size = len(prm.RawData) for i := range b.storage { if b.storage[i].Policy == nil || b.storage[i].Policy(prm.Object, prm.RawData) { res, err := b.storage[i].Storage.Put(ctx, prm) if err == nil { + success = true logOp(b.log, putOp, prm.Address, b.storage[i].Storage.Type(), res.StorageID) } return res, err From 059e9e88a25ab86752629d12165c02a3c2c32f46 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Jun 2023 12:27:19 +0300 Subject: [PATCH 0348/1943] [#373] metabase: Add metrics Signed-off-by: Dmitrii Stepanov --- .../internal/meta/list-garbage.go | 2 +- .../internal/meta/list-graveyard.go | 2 +- cmd/frostfs-node/container.go | 2 +- cmd/frostfs-node/notificator.go | 2 +- pkg/local_object_storage/engine/container.go | 14 +++--- pkg/local_object_storage/engine/evacuate.go | 2 +- .../engine/evacuate_test.go | 4 +- pkg/local_object_storage/engine/list.go | 5 +- pkg/local_object_storage/engine/list_test.go | 2 +- .../engine/remove_copies.go | 2 +- pkg/local_object_storage/engine/select.go | 12 ++--- pkg/local_object_storage/metabase/children.go | 24 +++++++++- .../metabase/containers.go | 18 +++++++- .../metabase/containers_test.go | 13 +++--- pkg/local_object_storage/metabase/control.go | 10 +++- pkg/local_object_storage/metabase/db.go | 2 + pkg/local_object_storage/metabase/delete.go | 10 ++++ pkg/local_object_storage/metabase/exists.go | 11 ++++- pkg/local_object_storage/metabase/expired.go | 20 ++++++++ pkg/local_object_storage/metabase/get.go | 11 ++++- .../metabase/graveyard.go | 46 +++++++++++++++++-- .../metabase/graveyard_test.go | 30 ++++++------ pkg/local_object_storage/metabase/inhume.go | 10 +++- .../metabase/iterators.go | 36 +++++++++++++-- .../metabase/iterators_test.go | 6 +-- pkg/local_object_storage/metabase/list.go | 24 +++++++++- .../metabase/list_test.go | 5 +- pkg/local_object_storage/metabase/lock.go | 42 ++++++++++++++--- pkg/local_object_storage/metabase/metrics.go | 20 ++++++++ pkg/local_object_storage/metabase/mode.go | 1 + pkg/local_object_storage/metabase/put.go | 10 ++++ pkg/local_object_storage/metabase/select.go | 12 ++++- pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/shard/gc.go | 14 +++--- pkg/local_object_storage/shard/list.go | 36 ++++++++++++--- pkg/local_object_storage/shard/list_test.go | 2 +- pkg/local_object_storage/shard/shard.go | 4 +- pkg/services/policer/process.go | 2 +- pkg/services/policer/queue.go | 5 +- 39 files changed, 379 insertions(+), 96 deletions(-) create mode 100644 pkg/local_object_storage/metabase/metrics.go diff --git a/cmd/frostfs-lens/internal/meta/list-garbage.go b/cmd/frostfs-lens/internal/meta/list-garbage.go index 3ab9a8f88..61b10ca1f 100644 --- a/cmd/frostfs-lens/internal/meta/list-garbage.go +++ b/cmd/frostfs-lens/internal/meta/list-garbage.go @@ -28,6 +28,6 @@ func listGarbageFunc(cmd *cobra.Command, _ []string) { return nil }) - err := db.IterateOverGarbage(garbPrm) + err := db.IterateOverGarbage(cmd.Context(), garbPrm) common.ExitOnErr(cmd, common.Errf("could not iterate over garbage bucket: %w", err)) } diff --git a/cmd/frostfs-lens/internal/meta/list-graveyard.go b/cmd/frostfs-lens/internal/meta/list-graveyard.go index db90513eb..19a93691c 100644 --- a/cmd/frostfs-lens/internal/meta/list-graveyard.go +++ b/cmd/frostfs-lens/internal/meta/list-graveyard.go @@ -33,6 +33,6 @@ func listGraveyardFunc(cmd *cobra.Command, _ []string) { return nil }) - err := db.IterateOverGraveyard(gravePrm) + err := db.IterateOverGraveyard(cmd.Context(), gravePrm) common.ExitOnErr(cmd, common.Errf("could not iterate over graveyard bucket: %w", err)) } diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 569e4a7ca..d5a5afce1 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -452,7 +452,7 @@ type localStorageLoad struct { } func (d *localStorageLoad) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontroller.UsedSpaceHandler) error { - idList, err := engine.ListContainers(d.engine) + idList, err := engine.ListContainers(context.TODO(), d.engine) if err != nil { return fmt.Errorf("list containers on engine failure: %w", err) } diff --git a/cmd/frostfs-node/notificator.go b/cmd/frostfs-node/notificator.go index 358b39a72..3fa486955 100644 --- a/cmd/frostfs-node/notificator.go +++ b/cmd/frostfs-node/notificator.go @@ -27,7 +27,7 @@ type notificationSource struct { func (n *notificationSource) Iterate(ctx context.Context, epoch uint64, handler func(topic string, addr oid.Address)) { log := n.l.With(zap.Uint64("epoch", epoch)) - listRes, err := n.e.ListContainers(engine.ListContainersPrm{}) + listRes, err := n.e.ListContainers(ctx, engine.ListContainersPrm{}) if err != nil { log.Error(logs.FrostFSNodeNotificatorCouldNotListContainers, zap.Error(err)) return diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 061e2fea0..e45f502ac 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -1,6 +1,8 @@ package engine import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" @@ -92,9 +94,9 @@ func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRe // ListContainers returns a unique container IDs presented in the engine objects. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) ListContainers(_ ListContainersPrm) (res ListContainersRes, err error) { +func (e *StorageEngine) ListContainers(ctx context.Context, _ ListContainersPrm) (res ListContainersRes, err error) { err = e.execIfNotBlocked(func() error { - res, err = e.listContainers() + res, err = e.listContainers(ctx) return err }) @@ -102,10 +104,10 @@ func (e *StorageEngine) ListContainers(_ ListContainersPrm) (res ListContainersR } // ListContainers calls ListContainers method on engine to get a unique container IDs presented in the engine objects. -func ListContainers(e *StorageEngine) ([]cid.ID, error) { +func ListContainers(ctx context.Context, e *StorageEngine) ([]cid.ID, error) { var prm ListContainersPrm - res, err := e.ListContainers(prm) + res, err := e.ListContainers(ctx, prm) if err != nil { return nil, err } @@ -113,7 +115,7 @@ func ListContainers(e *StorageEngine) ([]cid.ID, error) { return res.Containers(), nil } -func (e *StorageEngine) listContainers() (ListContainersRes, error) { +func (e *StorageEngine) listContainers(ctx context.Context) (ListContainersRes, error) { if e.metrics != nil { defer elapsed("ListContainers", e.metrics.AddMethodDuration)() } @@ -121,7 +123,7 @@ func (e *StorageEngine) listContainers() (ListContainersRes, error) { uniqueIDs := make(map[string]cid.ID) e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - res, err := sh.Shard.ListContainers(shard.ListContainersPrm{}) + res, err := sh.Shard.ListContainers(ctx, shard.ListContainersPrm{}) if err != nil { e.reportShardError(sh, "can't get list of containers", err) return false diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 73b7a7830..98a3a202d 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -261,7 +261,7 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E // TODO (@fyrchik): #1731 this approach doesn't work in degraded modes // because ListWithCursor works only with the metabase. - listRes, err := sh.ListWithCursor(listPrm) + listRes, err := sh.ListWithCursor(ctx, listPrm) if err != nil { if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { break diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 43737e7f7..13ac59f24 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -69,7 +69,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng err := e.Put(context.Background(), putPrm) require.NoError(t, err) - res, err := e.shards[ids[len(ids)-1].String()].List() + res, err := e.shards[ids[len(ids)-1].String()].List(context.Background()) require.NoError(t, err) if len(res.AddressList()) == objPerShard { break @@ -209,7 +209,7 @@ func TestEvacuateNetwork(t *testing.T) { var totalCount uint64 for i := range evacuateIDs { - res, err := e.shards[ids[i].String()].List() + res, err := e.shards[ids[i].String()].List(context.Background()) require.NoError(t, err) totalCount += uint64(len(res.AddressList())) diff --git a/pkg/local_object_storage/engine/list.go b/pkg/local_object_storage/engine/list.go index 8781416f4..f9229a2b1 100644 --- a/pkg/local_object_storage/engine/list.go +++ b/pkg/local_object_storage/engine/list.go @@ -1,6 +1,7 @@ package engine import ( + "context" "math/rand" "sort" @@ -96,7 +97,7 @@ func (l ListWithCursorRes) Cursor() *Cursor { // // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. -func (e *StorageEngine) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) { +func (e *StorageEngine) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (ListWithCursorRes, error) { result := make([]objectcore.AddressWithType, 0, prm.count) // Set initial cursors @@ -142,7 +143,7 @@ func (e *StorageEngine) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes shardPrm.WithCount(count) shardPrm.WithCursor(cursor.getCurrentShardCursor()) - res, err := shardInstance.ListWithCursor(shardPrm) + res, err := shardInstance.ListWithCursor(ctx, shardPrm) if err != nil { cursor.setShardRead(curr) continue diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 5b927cf11..6cea2d0f4 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -107,7 +107,7 @@ func TestListWithCursor(t *testing.T) { var prm ListWithCursorPrm prm.count = tt.batchSize for { - res, err := e.ListWithCursor(prm) + res, err := e.ListWithCursor(context.Background(), prm) if err == ErrEndOfListing { require.Empty(t, res.AddressList()) break diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index 7681e0e50..4b48d179c 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -69,7 +69,7 @@ func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicat var listPrm shard.ListWithCursorPrm listPrm.WithCount(uint32(prm.Concurrency)) listPrm.WithCursor(cursor) - res, err := sh.ListWithCursor(listPrm) + res, err := sh.ListWithCursor(ctx, listPrm) if err != nil { if errors.Is(err, meta.ErrEndOfListing) { return nil diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 9f651845f..6a7bf1b7f 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -98,16 +98,16 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, // If limit is zero, then returns all available object addresses. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) List(limit uint64) (res SelectRes, err error) { +func (e *StorageEngine) List(ctx context.Context, limit uint64) (res SelectRes, err error) { err = e.execIfNotBlocked(func() error { - res, err = e.list(limit) + res, err = e.list(ctx, limit) return err }) return } -func (e *StorageEngine) list(limit uint64) (SelectRes, error) { +func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, error) { if e.metrics != nil { defer elapsed("ListObjects", e.metrics.AddMethodDuration)() } @@ -118,7 +118,7 @@ func (e *StorageEngine) list(limit uint64) (SelectRes, error) { // consider iterating over shuffled shards e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - res, err := sh.List() // consider limit result of shard iterator + res, err := sh.List(ctx) // consider limit result of shard iterator if err != nil { e.reportShardError(sh, "could not select objects from shard", err) } else { @@ -159,8 +159,8 @@ func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs object.S // List returns `limit` available physically storage object addresses in // engine. If limit is zero, then returns all available object addresses. -func List(storage *StorageEngine, limit uint64) ([]oid.Address, error) { - res, err := storage.List(limit) +func List(ctx context.Context, storage *StorageEngine, limit uint64) ([]oid.Address, error) { + res, err := storage.List(ctx, limit) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/metabase/children.go b/pkg/local_object_storage/metabase/children.go index 500e83e7a..6816358d2 100644 --- a/pkg/local_object_storage/metabase/children.go +++ b/pkg/local_object_storage/metabase/children.go @@ -1,14 +1,34 @@ package meta import ( + "context" + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // GetChildren returns parent -> children map. // If an object has no children, then map will contain addr -> empty slice value. -func (db *DB) GetChildren(addresses []oid.Address) (map[oid.Address][]oid.Address, error) { +func (db *DB) GetChildren(ctx context.Context, addresses []oid.Address) (map[oid.Address][]oid.Address, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("GetChildren", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.GetChildren", + trace.WithAttributes( + attribute.Int("addr_count", len(addresses)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -53,6 +73,6 @@ func (db *DB) GetChildren(addresses []oid.Address) (map[oid.Address][]oid.Addres if err != nil { return nil, metaerr.Wrap(err) } - + success = true return result, nil } diff --git a/pkg/local_object_storage/metabase/containers.go b/pkg/local_object_storage/metabase/containers.go index fe38e0b6d..472b2affc 100644 --- a/pkg/local_object_storage/metabase/containers.go +++ b/pkg/local_object_storage/metabase/containers.go @@ -1,14 +1,28 @@ package meta import ( + "context" "encoding/binary" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.etcd.io/bbolt" ) -func (db *DB) Containers() (list []cid.ID, err error) { +func (db *DB) Containers(ctx context.Context) (list []cid.ID, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Containers", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.Containers") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -21,7 +35,7 @@ func (db *DB) Containers() (list []cid.ID, err error) { return err }) - + success = err == nil return list, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index c0565b35a..4e2dd550d 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "math/rand" "sort" "testing" @@ -34,7 +35,7 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, err) } - lst, err := db.Containers() + lst, err := db.Containers(context.Background()) require.NoError(t, err) for _, cnr := range lst { @@ -60,7 +61,7 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, putBig(db, obj)) - cnrs, err := db.Containers() + cnrs, err := db.Containers(context.Background()) require.NoError(t, err) cnr, _ := obj.ContainerID() @@ -68,7 +69,7 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, metaInhume(db, object.AddressOf(obj), oidtest.Address())) - cnrs, err = db.Containers() + cnrs, err = db.Containers(context.Background()) require.NoError(t, err) assertContains(cnrs, cnr) }) @@ -78,14 +79,14 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, putBig(db, obj)) - cnrs, err := db.Containers() + cnrs, err := db.Containers(context.Background()) require.NoError(t, err) cnr, _ := obj.ContainerID() assertContains(cnrs, cnr) require.NoError(t, metaToMoveIt(db, object.AddressOf(obj))) - cnrs, err = db.Containers() + cnrs, err = db.Containers(context.Background()) require.NoError(t, err) assertContains(cnrs, cnr) }) @@ -126,7 +127,7 @@ func TestDB_ContainersCount(t *testing.T) { return expected[i].EncodeToString() < expected[j].EncodeToString() }) - got, err := db.Containers() + got, err := db.Containers(context.Background()) require.NoError(t, err) sort.Slice(got, func(i, j int) bool { diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 370fddda1..d0a9c4723 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -175,10 +175,14 @@ func (db *DB) SyncCounters() error { // Close closes boltDB instance. func (db *DB) Close() error { + var err error if db.boltDB != nil { - return metaerr.Wrap(db.boltDB.Close()) + err = metaerr.Wrap(db.boltDB.Close()) } - return nil + if err == nil { + db.metrics.Close() + } + return err } // Reload reloads part of the configuration. @@ -202,12 +206,14 @@ func (db *DB) Reload(opts ...Option) (bool, error) { } db.mode = mode.Degraded + db.metrics.SetMode(mode.Degraded) db.info.Path = c.info.Path if err := db.openBolt(); err != nil { return false, metaerr.Wrap(fmt.Errorf("%w: %v", ErrDegradedMode, err)) } db.mode = mode.ReadWrite + db.metrics.SetMode(mode.ReadWrite) return true, nil } diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index 5a9ca3aa9..c54ed98d0 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -60,6 +60,7 @@ type cfg struct { log *logger.Logger epochState EpochState + metrics Metrics } func defaultCfg() *cfg { @@ -70,6 +71,7 @@ func defaultCfg() *cfg { boltBatchDelay: bbolt.DefaultMaxBatchDelay, boltBatchSize: bbolt.DefaultMaxBatchSize, log: &logger.Logger{Logger: zap.L()}, + metrics: &noopMetrics{}, } } diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 7e8e0e5dd..d387b3d04 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" @@ -71,6 +72,14 @@ type referenceCounter map[string]*referenceNumber // Delete removed object records from metabase indexes. func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { + var ( + startedAt = time.Now() + deleted = false + ) + defer func() { + db.metrics.AddMethodDuration("Delete", time.Since(startedAt), deleted) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Delete", trace.WithAttributes( attribute.Int("addr_count", len(prm.addrs)), @@ -98,6 +107,7 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { return err }) if err == nil { + deleted = true for i := range prm.addrs { storagelog.Write(db.log, storagelog.AddressField(prm.addrs[i]), diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index f9c563bdd..b6e5ea052 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -3,6 +3,7 @@ package meta import ( "context" "fmt" + "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" @@ -45,6 +46,14 @@ func (p ExistsRes) Exists() bool { // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Exists", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Exists", trace.WithAttributes( attribute.String("address", prm.addr.EncodeToString()), @@ -65,7 +74,7 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err return err }) - + success = err == nil return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/expired.go b/pkg/local_object_storage/metabase/expired.go index aac158ba4..43933d12d 100644 --- a/pkg/local_object_storage/metabase/expired.go +++ b/pkg/local_object_storage/metabase/expired.go @@ -5,18 +5,37 @@ import ( "errors" "fmt" "strconv" + "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // FilterExpired return expired items from addresses. // Address considered expired if metabase does contain information about expiration and // expiration epoch is less than epoch. func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.Address) ([]oid.Address, error) { + var ( + startedAt = time.Now() + success = true + ) + defer func() { + db.metrics.AddMethodDuration("FilterExpired", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.FilterExpired", + trace.WithAttributes( + attribute.String("epoch", strconv.FormatUint(epoch, 10)), + attribute.Int("addr_count", len(addresses)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -68,6 +87,7 @@ func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.A if err != nil { return nil, metaerr.Wrap(err) } + success = true return result, nil } diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index 2ce82bcee..ad35b4c18 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -3,6 +3,7 @@ package meta import ( "context" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -52,6 +53,14 @@ func (r GetRes) Header() *objectSDK.Object { // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Get", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Get", trace.WithAttributes( attribute.String("address", prm.addr.EncodeToString()), @@ -74,7 +83,7 @@ func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { return err }) - + success = err == nil return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index e2530bd31..df9a3d302 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -2,10 +2,13 @@ package meta import ( "bytes" + "context" "errors" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -58,7 +61,18 @@ func (g *GarbageIterationPrm) SetOffset(offset oid.Address) { // // If h returns ErrInterruptIterator, nil returns immediately. // Returns other errors of h directly. -func (db *DB) IterateOverGarbage(p GarbageIterationPrm) error { +func (db *DB) IterateOverGarbage(ctx context.Context, p GarbageIterationPrm) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateOverGarbage", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateOverGarbage") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -66,9 +80,11 @@ func (db *DB) IterateOverGarbage(p GarbageIterationPrm) error { return ErrDegradedMode } - return metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateDeletedObj(tx, gcHandler{p.h}, p.offset) })) + success = err == nil + return err } // TombstonedObject represents descriptor of the @@ -125,7 +141,18 @@ func (g *GraveyardIterationPrm) SetOffset(offset oid.Address) { // // If h returns ErrInterruptIterator, nil returns immediately. // Returns other errors of h directly. -func (db *DB) IterateOverGraveyard(p GraveyardIterationPrm) error { +func (db *DB) IterateOverGraveyard(ctx context.Context, p GraveyardIterationPrm) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateOverGraveyard", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateOverGraveyard") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -232,7 +259,18 @@ func graveFromKV(k, v []byte) (res TombstonedObject, err error) { // graveyard bucket. // // Returns any error appeared during deletion process. -func (db *DB) DropGraves(tss []TombstonedObject) error { +func (db *DB) DropGraves(ctx context.Context, tss []TombstonedObject) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("DropGraves", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.DropGraves") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index 8cd09e3f7..7476608f2 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -23,7 +23,7 @@ func TestDB_IterateDeletedObjects_EmptyDB(t *testing.T) { return nil }) - err := db.IterateOverGraveyard(iterGravePRM) + err := db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) require.Zero(t, counter) @@ -33,7 +33,7 @@ func TestDB_IterateDeletedObjects_EmptyDB(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) require.Zero(t, counter) } @@ -83,7 +83,7 @@ func TestDB_Iterate_OffsetNotFound(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) // the second object would be put after the @@ -99,7 +99,7 @@ func TestDB_Iterate_OffsetNotFound(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) // the third object would be put before the @@ -164,7 +164,7 @@ func TestDB_IterateDeletedObjects(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGravePRM) + err = db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) var iterGCPRM meta.GarbageIterationPrm @@ -175,7 +175,7 @@ func TestDB_IterateDeletedObjects(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGCPRM) + err = db.IterateOverGarbage(context.Background(), iterGCPRM) require.NoError(t, err) // objects covered with a tombstone @@ -255,7 +255,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGraveyardPrm) + err = db.IterateOverGraveyard(context.Background(), iterGraveyardPrm) require.NoError(t, err) require.Equal(t, firstIterationSize, counter) require.Equal(t, firstIterationSize, len(gotGraveyard)) @@ -272,7 +272,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGraveyardPrm) + err = db.IterateOverGraveyard(context.Background(), iterGraveyardPrm) require.NoError(t, err) require.Equal(t, len(expectedGraveyard), counter) require.ElementsMatch(t, gotGraveyard, expectedGraveyard) @@ -287,7 +287,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGraveyardPrm) + err = db.IterateOverGraveyard(context.Background(), iterGraveyardPrm) require.NoError(t, err) require.False(t, iWasCalled) } @@ -348,7 +348,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGarbagePrm) + err = db.IterateOverGarbage(context.Background(), iterGarbagePrm) require.NoError(t, err) require.Equal(t, firstIterationSize, counter) require.Equal(t, firstIterationSize, len(gotGarbage)) @@ -363,7 +363,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGarbagePrm) + err = db.IterateOverGarbage(context.Background(), iterGarbagePrm) require.NoError(t, err) require.Equal(t, len(expectedGarbage), counter) require.ElementsMatch(t, gotGarbage, expectedGarbage) @@ -378,7 +378,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { return nil }) - err = db.IterateOverGarbage(iterGarbagePrm) + err = db.IterateOverGarbage(context.Background(), iterGarbagePrm) require.NoError(t, err) require.False(t, iWasCalled) } @@ -418,11 +418,11 @@ func TestDB_DropGraves(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGravePRM) + err = db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) require.Equal(t, 2, counter) - err = db.DropGraves(buriedTS) + err = db.DropGraves(context.Background(), buriedTS) require.NoError(t, err) counter = 0 @@ -431,7 +431,7 @@ func TestDB_DropGraves(t *testing.T) { return nil }) - err = db.IterateOverGraveyard(iterGravePRM) + err = db.IterateOverGraveyard(context.Background(), iterGravePRM) require.NoError(t, err) require.Zero(t, counter) } diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 7865a8771..76744aa33 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -122,6 +123,13 @@ var ErrLockObjectRemoval = logicerr.New("lock object removal") // NOTE: Marks any object with GC mark (despite any prohibitions on operations // with that object) if WithForceGCMark option has been provided. func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Inhume", time.Since(startedAt), success) + }() _, span := tracing.StartSpanFromContext(ctx, "metabase.Inhume") defer span.End() @@ -138,7 +146,7 @@ func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err err err = db.boltDB.Update(func(tx *bbolt.Tx) error { return db.inhumeTx(tx, currEpoch, prm, &res) }) - + success = err == nil return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index dfa048621..78bfd2914 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -1,17 +1,22 @@ package meta import ( + "context" "errors" "fmt" "strconv" + "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ExpiredObject is a descriptor of expired object from DB. @@ -44,7 +49,20 @@ var ErrInterruptIterator = logicerr.New("iterator is interrupted") // // If h returns ErrInterruptIterator, nil returns immediately. // Returns other errors of h directly. -func (db *DB) IterateExpired(epoch uint64, h ExpiredObjectHandler) error { +func (db *DB) IterateExpired(ctx context.Context, epoch uint64, h ExpiredObjectHandler) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateExpired", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateExpired", + trace.WithAttributes( + attribute.String("epoch", strconv.FormatUint(epoch, 10)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -52,9 +70,11 @@ func (db *DB) IterateExpired(epoch uint64, h ExpiredObjectHandler) error { return ErrDegradedMode } - return metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { return db.iterateExpired(tx, epoch, h) })) + success = err == nil + return err } func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) error { @@ -125,7 +145,17 @@ func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) // Returns other errors of h directly. // // Does not modify tss. -func (db *DB) IterateCoveredByTombstones(tss map[string]oid.Address, h func(oid.Address) error) error { +func (db *DB) IterateCoveredByTombstones(ctx context.Context, tss map[string]oid.Address, h func(oid.Address) error) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateCoveredByTombstones", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateCoveredByTombstones") + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 69bf2bee5..e07184eb6 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -36,7 +36,7 @@ func TestDB_IterateExpired(t *testing.T) { require.NoError(t, db.Lock(context.Background(), expiredLocked.Container(), oidtest.ID(), []oid.ID{expiredLocked.Object()})) - err := db.IterateExpired(epoch, func(exp *meta.ExpiredObject) error { + err := db.IterateExpired(context.Background(), epoch, func(exp *meta.ExpiredObject) error { if addr, ok := mAlive[exp.Type()]; ok { require.NotEqual(t, addr, exp.Address()) } @@ -96,7 +96,7 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { ts.EncodeToString(): ts, } - err = db.IterateCoveredByTombstones(tss, func(addr oid.Address) error { + err = db.IterateCoveredByTombstones(context.Background(), tss, func(addr oid.Address) error { handled = append(handled, addr) return nil }) @@ -112,7 +112,7 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { handled = handled[:0] - err = db.IterateCoveredByTombstones(tss, func(addr oid.Address) error { + err = db.IterateCoveredByTombstones(context.Background(), tss, func(addr oid.Address) error { handled = append(handled, addr) return nil }) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 7c77cd703..337265318 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -1,13 +1,19 @@ package meta import ( + "context" + "time" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // ErrEndOfListing is returned from object listing with cursor @@ -61,7 +67,21 @@ func (l ListRes) Cursor() *Cursor { // // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. -func (db *DB) ListWithCursor(prm ListPrm) (res ListRes, err error) { +func (db *DB) ListWithCursor(ctx context.Context, prm ListPrm) (res ListRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("ListWithCursor", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.ListWithCursor", + trace.WithAttributes( + attribute.Int("count", prm.count), + attribute.Bool("has_cursor", prm.cursor != nil), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -75,7 +95,7 @@ func (db *DB) ListWithCursor(prm ListPrm) (res ListRes, err error) { res.addrList, res.cursor, err = db.listWithCursor(tx, result, prm.count, prm.cursor) return err }) - + success = err == nil return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 4bf3ca827..abb55c9d1 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "errors" "sort" "testing" @@ -51,7 +52,7 @@ func benchmarkListWithCursor(b *testing.B, db *meta.DB, batchSize int) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - res, err := db.ListWithCursor(prm) + res, err := db.ListWithCursor(context.Background(), prm) if err != nil { if err != meta.ErrEndOfListing { b.Fatalf("error: %v", err) @@ -225,6 +226,6 @@ func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]objec listPrm.SetCount(count) listPrm.SetCursor(cursor) - r, err := db.ListWithCursor(listPrm) + r, err := db.ListWithCursor(context.Background(), listPrm) return r.AddressList(), r.Cursor(), err } diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index b84e0bf46..f3388daf0 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -36,6 +37,14 @@ func bucketNameLockers(idCnr cid.ID, key []byte) []byte { // // Locked list should be unique. Panics if it is empty. func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid.ID) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Lock", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Lock", trace.WithAttributes( attribute.String("container_id", cnr.EncodeToString()), @@ -57,7 +66,12 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. panic("empty locked list") } - // check if all objects are regular + err := db.lockInternal(locked, cnr, locker) + success = err == nil + return err +} + +func (db *DB) lockInternal(locked []oid.ID, cnr cid.ID, locker oid.ID) error { bucketKeysLocked := make([][]byte, len(locked)) for i := range locked { bucketKeysLocked[i] = objectKey(locked[i], make([]byte, objectKeySize)) @@ -83,7 +97,6 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. loop: for i := range bucketKeysLocked { - // decode list of already existing lockers exLockers, err = decodeList(bucketLockedContainer.Get(bucketKeysLocked[i])) if err != nil { return fmt.Errorf("decode list of object lockers: %w", err) @@ -95,14 +108,11 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. } } - // update the list of lockers updLockers, err = encodeList(append(exLockers, keyLocker)) if err != nil { - // maybe continue for the best effort? return fmt.Errorf("encode list of object lockers: %w", err) } - // write updated list of lockers err = bucketLockedContainer.Put(bucketKeysLocked[i], updLockers) if err != nil { return fmt.Errorf("update list of object lockers: %w", err) @@ -116,6 +126,14 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. // FreeLockedBy unlocks all objects in DB which are locked by lockers. // Returns slice of unlocked object ID's or an error. func (db *DB) FreeLockedBy(lockers []oid.Address) ([]oid.Address, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("FreeLockedBy", time.Since(startedAt), success) + }() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -138,6 +156,7 @@ func (db *DB) FreeLockedBy(lockers []oid.Address) ([]oid.Address, error) { }); err != nil { return nil, metaerr.Wrap(err) } + success = true return unlockedObjects, nil } @@ -280,6 +299,14 @@ func (i IsLockedRes) Locked() bool { // // Returns only non-logical errors related to underlying database. func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IsLocked", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.IsLocked", trace.WithAttributes( attribute.String("address", prm.addr.EncodeToString()), @@ -292,9 +319,10 @@ func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, e if db.mode.NoMetabase() { return res, ErrDegradedMode } - - return res, metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { + err = metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { res.locked = objectLocked(tx, prm.addr.Container(), prm.addr.Object()) return nil })) + success = err == nil + return res, err } diff --git a/pkg/local_object_storage/metabase/metrics.go b/pkg/local_object_storage/metabase/metrics.go new file mode 100644 index 000000000..120579e4c --- /dev/null +++ b/pkg/local_object_storage/metabase/metrics.go @@ -0,0 +1,20 @@ +package meta + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) + +type Metrics interface { + SetMode(m mode.Mode) + Close() + + AddMethodDuration(method string, d time.Duration, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetMode(mode.Mode) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index dd1cdc900..28beca8f3 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -40,5 +40,6 @@ func (db *DB) SetMode(m mode.Mode) error { } db.mode = m + db.metrics.SetMode(m) return nil } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index b7cff68dc..7a24485b3 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" gio "io" + "time" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" @@ -58,6 +59,14 @@ var ( // Returns an error of type apistatus.ObjectAlreadyRemoved if object has been placed in graveyard. // Returns the object.ErrObjectIsExpired if the object is presented but already expired. func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Put", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Put", trace.WithAttributes( attribute.String("address", objectCore.AddressOf(prm.obj).EncodeToString()), @@ -79,6 +88,7 @@ func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { return db.put(tx, prm.obj, prm.id, nil, currEpoch) }) if err == nil { + success = true storagelog.Write(db.log, storagelog.AddressField(objectCore.AddressOf(prm.obj)), storagelog.OpField("metabase PUT")) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 466476d4e..1f32d2cde 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strings" + "time" v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -62,6 +63,14 @@ func (r SelectRes) AddressList() []oid.Address { // Select returns list of addresses of objects that match search filters. func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("Select", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.Select", trace.WithAttributes( attribute.String("container_id", prm.cnr.EncodeToString()), @@ -76,6 +85,7 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err } if blindlyProcess(prm.filters) { + success = true return res, nil } @@ -83,7 +93,7 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err return res, metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { res.addrList, err = db.selectObjects(tx, prm.cnr, prm.filters, currEpoch) - + success = err == nil return err })) } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 84efa1d31..bc514933b 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -138,7 +138,7 @@ func (s *Shard) Init(ctx context.Context) error { } } - s.updateMetrics() + s.updateMetrics(ctx) s.gc = &gc{ gcCfg: &s.gcCfg, diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 2580173c2..b8d516871 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -270,7 +270,7 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { // iterate over metabase's objects with GC mark // (no more than s.rmBatchSize objects) - err := s.metaBase.IterateOverGarbage(iterPrm) + err := s.metaBase.IterateOverGarbage(ctx, iterPrm) if err != nil { s.log.Warn(logs.ShardIteratorOverMetabaseGraveyardFailed, zap.String("error", err.Error()), @@ -382,7 +382,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } - expired, err := s.getExpiredWithLinked(expired) + expired, err := s.getExpiredWithLinked(ctx, expired) if err != nil { s.log.Warn(logs.ShardGCFailedToGetExpiredWithLinked, zap.Error(err)) return @@ -414,9 +414,9 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) } } -func (s *Shard) getExpiredWithLinked(source []oid.Address) ([]oid.Address, error) { +func (s *Shard) getExpiredWithLinked(ctx context.Context, source []oid.Address) ([]oid.Address, error) { result := make([]oid.Address, 0, len(source)) - parentToChildren, err := s.metaBase.GetChildren(source) + parentToChildren, err := s.metaBase.GetChildren(ctx, source) if err != nil { return nil, err } @@ -469,7 +469,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { return } - err = s.metaBase.IterateOverGraveyard(iterPrm) + err = s.metaBase.IterateOverGraveyard(ctx, iterPrm) if err != nil { log.Error(logs.ShardIteratorOverGraveyardFailed, zap.Error(err)) s.m.RUnlock() @@ -560,7 +560,7 @@ func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, onExpiredFo return ErrDegradedMode } - err := s.metaBase.IterateExpired(epoch, func(expiredObject *meta.ExpiredObject) error { + err := s.metaBase.IterateExpired(ctx, epoch, func(expiredObject *meta.ExpiredObject) error { select { case <-ctx.Done(): return meta.ErrInterruptIterator @@ -628,7 +628,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston // drop just processed expired tombstones // from graveyard - err = s.metaBase.DropGraves(tss) + err = s.metaBase.DropGraves(ctx, tss) if err != nil { s.log.Warn(logs.ShardCouldNotDropExpiredGraveRecords, zap.Error(err)) } diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index aaa1112cd..dd21745cc 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -7,8 +7,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -65,7 +68,13 @@ func (r ListWithCursorRes) Cursor() *Cursor { } // List returns all objects physically stored in the Shard. -func (s *Shard) List() (res SelectRes, err error) { +func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.List", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + )) + defer span.End() + s.m.RLock() defer s.m.RUnlock() @@ -73,7 +82,7 @@ func (s *Shard) List() (res SelectRes, err error) { return SelectRes{}, ErrDegradedMode } - lst, err := s.metaBase.Containers() + lst, err := s.metaBase.Containers(ctx) if err != nil { return res, fmt.Errorf("can't list stored containers: %w", err) } @@ -86,7 +95,7 @@ func (s *Shard) List() (res SelectRes, err error) { sPrm.SetContainerID(lst[i]) sPrm.SetFilters(filters) - sRes, err := s.metaBase.Select(context.TODO(), sPrm) // consider making List in metabase + sRes, err := s.metaBase.Select(ctx, sPrm) // consider making List in metabase if err != nil { s.log.Debug(logs.ShardCantSelectAllObjects, zap.Stringer("cid", lst[i]), @@ -101,12 +110,18 @@ func (s *Shard) List() (res SelectRes, err error) { return res, nil } -func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) { +func (s *Shard) ListContainers(ctx context.Context, _ ListContainersPrm) (ListContainersRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.ListContainers", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + )) + defer span.End() + if s.GetMode().NoMetabase() { return ListContainersRes{}, ErrDegradedMode } - containers, err := s.metaBase.Containers() + containers, err := s.metaBase.Containers(ctx) if err != nil { return ListContainersRes{}, fmt.Errorf("could not get list of containers: %w", err) } @@ -122,7 +137,14 @@ func (s *Shard) ListContainers(_ ListContainersPrm) (ListContainersRes, error) { // // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. -func (s *Shard) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) { +func (s *Shard) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (ListWithCursorRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "shard.ListWithCursor", + trace.WithAttributes( + attribute.Int64("count", int64(prm.count)), + attribute.Bool("has_cursor", prm.cursor != nil), + )) + defer span.End() + if s.GetMode().NoMetabase() { return ListWithCursorRes{}, ErrDegradedMode } @@ -130,7 +152,7 @@ func (s *Shard) ListWithCursor(prm ListWithCursorPrm) (ListWithCursorRes, error) var metaPrm meta.ListPrm metaPrm.SetCount(prm.count) metaPrm.SetCursor(prm.cursor) - res, err := s.metaBase.ListWithCursor(metaPrm) + res, err := s.metaBase.ListWithCursor(ctx, metaPrm) if err != nil { return ListWithCursorRes{}, fmt.Errorf("could not get list of objects: %w", err) } diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index bbce28430..63e7651c8 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -71,7 +71,7 @@ func testShardList(t *testing.T, sh *shard.Shard) { } require.NoError(t, errG.Wait()) - res, err := sh.List() + res, err := sh.List(context.Background()) require.NoError(t, err) for _, objID := range res.AddressList() { diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 1d2cab9f2..10c1acd40 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -370,7 +370,7 @@ const ( logical = "logic" ) -func (s *Shard) updateMetrics() { +func (s *Shard) updateMetrics(ctx context.Context) { if s.cfg.metricsWriter != nil && !s.GetMode().NoMetabase() { cc, err := s.metaBase.ObjectCounters() if err != nil { @@ -384,7 +384,7 @@ func (s *Shard) updateMetrics() { s.cfg.metricsWriter.SetObjectCounter(physical, cc.Phy()) s.cfg.metricsWriter.SetObjectCounter(logical, cc.Logic()) - cnrList, err := s.metaBase.Containers() + cnrList, err := s.metaBase.Containers(ctx) if err != nil { s.log.Warn(logs.ShardMetaCantReadContainerList, zap.Error(err)) return diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 4a40f00ba..60e924755 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -34,7 +34,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { default: } - addrs, cursor, err = p.jobQueue.Select(cursor, p.batchSize) + addrs, cursor, err = p.jobQueue.Select(ctx, cursor, p.batchSize) if err != nil { if errors.Is(err, engine.ErrEndOfListing) { time.Sleep(time.Second) // finished whole cycle, sleep a bit diff --git a/pkg/services/policer/queue.go b/pkg/services/policer/queue.go index b8af44049..22012c835 100644 --- a/pkg/services/policer/queue.go +++ b/pkg/services/policer/queue.go @@ -1,6 +1,7 @@ package policer import ( + "context" "fmt" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -11,12 +12,12 @@ type jobQueue struct { localStorage *engine.StorageEngine } -func (q *jobQueue) Select(cursor *engine.Cursor, count uint32) ([]objectcore.AddressWithType, *engine.Cursor, error) { +func (q *jobQueue) Select(ctx context.Context, cursor *engine.Cursor, count uint32) ([]objectcore.AddressWithType, *engine.Cursor, error) { var prm engine.ListWithCursorPrm prm.WithCursor(cursor) prm.WithCount(count) - res, err := q.localStorage.ListWithCursor(prm) + res, err := q.localStorage.ListWithCursor(ctx, prm) if err != nil { return nil, nil, fmt.Errorf("cannot list objects in engine: %w", err) } From d5aaec1107bcc7437abe85418904df68e94414e4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 12:27:53 +0300 Subject: [PATCH 0349/1943] [#373] pilorama: Add metrics Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/pilorama/boltdb.go | 145 +++++++++++++++++-- pkg/local_object_storage/pilorama/metrics.go | 20 +++ pkg/local_object_storage/pilorama/option.go | 7 + 3 files changed, 159 insertions(+), 13 deletions(-) create mode 100644 pkg/local_object_storage/pilorama/metrics.go diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index e7bcb110b..1b8ee7f18 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -72,6 +72,7 @@ func NewBoltForest(opts ...Option) ForestStorage { maxBatchDelay: bbolt.DefaultMaxBatchDelay, maxBatchSize: bbolt.DefaultMaxBatchSize, openFile: os.OpenFile, + metrics: &noopMetrics{}, }, } @@ -101,6 +102,7 @@ func (t *boltForest) SetMode(m mode.Mode) error { } t.mode = m + t.metrics.SetMode(m) return nil } func (t *boltForest) Open(readOnly bool) error { @@ -122,7 +124,11 @@ func (t *boltForest) Open(readOnly bool) error { t.db.MaxBatchSize = t.maxBatchSize t.db.MaxBatchDelay = t.maxBatchDelay - + m := mode.ReadWrite + if readOnly { + m = mode.ReadOnly + } + t.metrics.SetMode(m) return nil } func (t *boltForest) Init() error { @@ -142,10 +148,14 @@ func (t *boltForest) Init() error { }) } func (t *boltForest) Close() error { + var err error if t.db != nil { - return t.db.Close() + err = t.db.Close() } - return nil + if err == nil { + t.metrics.Close() + } + return err } // TreeMove implements the Forest interface. @@ -225,6 +235,14 @@ func (t *boltForest) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID strin // TreeExists implements the Forest interface. func (t *boltForest) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (bool, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeExists", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeExists", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -247,7 +265,7 @@ func (t *boltForest) TreeExists(ctx context.Context, cid cidSDK.ID, treeID strin exists = treeRoot != nil return nil }) - + success = err == nil return exists, metaerr.Wrap(err) } @@ -255,6 +273,14 @@ var syncHeightKey = []byte{'h'} // TreeUpdateLastSyncHeight implements the pilorama.Forest interface. func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeUpdateLastSyncHeight", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeUpdateLastSyncHeight", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -268,7 +294,7 @@ func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID binary.LittleEndian.PutUint64(rawHeight, height) buck := bucketName(cid, treeID) - return metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(buck) if treeRoot == nil { return ErrTreeNotFound @@ -277,10 +303,20 @@ func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID b := treeRoot.Bucket(dataBucket) return b.Put(syncHeightKey, rawHeight) })) + success = err == nil + return err } // TreeLastSyncHeight implements the pilorama.Forest interface. func (t *boltForest) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID string) (uint64, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeLastSyncHeight", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeLastSyncHeight", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -305,11 +341,20 @@ func (t *boltForest) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, tree } return nil }) + success = err == nil return height, metaerr.Wrap(err) } // TreeAddByPath implements the Forest interface. func (t *boltForest) TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID string, attr string, path []string, meta []KeyValue) ([]Move, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeAddByPath", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeAddByPath", trace.WithAttributes( attribute.String("container_id", d.CID.EncodeToString()), @@ -323,6 +368,12 @@ func (t *boltForest) TreeAddByPath(ctx context.Context, d CIDDescriptor, treeID ) defer span.End() + res, err := t.addByPathInternal(d, attr, treeID, path, meta) + success = err == nil + return res, err +} + +func (t *boltForest) addByPathInternal(d CIDDescriptor, attr string, treeID string, path []string, meta []KeyValue) ([]Move, error) { if !d.checkValid() { return nil, ErrInvalidCIDDescriptor } @@ -417,6 +468,14 @@ func (t *boltForest) findSpareID(bTree *bbolt.Bucket) uint64 { // TreeApply implements the Forest interface. func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string, m *Move, backgroundSync bool) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeApply", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeApply", trace.WithAttributes( attribute.String("container_id", cnr.EncodeToString()), @@ -440,6 +499,7 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string err := t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cnr, treeID)) if treeRoot == nil { + success = true return nil } @@ -448,16 +508,18 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string var logKey [8]byte binary.BigEndian.PutUint64(logKey[:], m.Time) seen = b.Get(logKey[:]) != nil + success = true return nil }) if err != nil || seen { + success = err == nil return metaerr.Wrap(err) } } if t.db.MaxBatchSize == 1 { fullID := bucketName(cnr, treeID) - return metaerr.Wrap(t.db.Update(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.Update(func(tx *bbolt.Tx) error { bLog, bTree, err := t.getTreeBuckets(tx, fullID) if err != nil { return err @@ -466,11 +528,15 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string var lm Move return t.applyOperation(bLog, bTree, []*Move{m}, &lm) })) + success = err == nil + return err } ch := make(chan error, 1) t.addBatch(cnr, treeID, m, ch) - return metaerr.Wrap(<-ch) + err := <-ch + success = err == nil + return metaerr.Wrap(err) } func (t *boltForest) addBatch(cnr cidSDK.ID, treeID string, m *Move, ch chan error) { @@ -724,6 +790,14 @@ func (t *boltForest) isAncestor(b *bbolt.Bucket, parent, child Node) bool { // TreeGetByPath implements the Forest interface. func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetByPath", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetByPath", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -740,6 +814,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st } if len(path) == 0 { + success = true return nil, nil } @@ -752,7 +827,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st var nodes []Node - return nodes, metaerr.Wrap(t.db.View(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cid, treeID)) if treeRoot == nil { return ErrTreeNotFound @@ -790,10 +865,20 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st } return nil })) + success = err == nil + return nodes, err } // TreeGetMeta implements the forest interface. func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetMeta", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetMeta", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -828,12 +913,20 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri _, _, meta, _ := t.getState(b, stateKey(key, nodeID)) return m.FromBytes(meta) }) - + success = err == nil return m, parentID, metaerr.Wrap(err) } // TreeGetChildren implements the Forest interface. func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetChildren", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetChildren", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -869,12 +962,20 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID } return nil }) - + success = err == nil return children, metaerr.Wrap(err) } // TreeList implements the Forest interface. func (t *boltForest) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeList", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeList", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -910,12 +1011,20 @@ func (t *boltForest) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, err if err != nil { return nil, metaerr.Wrap(fmt.Errorf("could not list trees: %w", err)) } - + success = true return ids, nil } // TreeGetOpLog implements the pilorama.Forest interface. func (t *boltForest) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeGetOpLog", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeGetOpLog", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -949,12 +1058,20 @@ func (t *boltForest) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID str } return nil }) - + success = err == nil return lm, metaerr.Wrap(err) } // TreeDrop implements the pilorama.Forest interface. func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeDrop", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeDrop", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -972,7 +1089,7 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) return ErrReadOnlyMode } - return metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { + err := metaerr.Wrap(t.db.Batch(func(tx *bbolt.Tx) error { if treeID == "" { c := tx.Cursor() prefix := make([]byte, 32) @@ -991,6 +1108,8 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) } return err })) + success = err == nil + return err } func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node, error) { diff --git a/pkg/local_object_storage/pilorama/metrics.go b/pkg/local_object_storage/pilorama/metrics.go new file mode 100644 index 000000000..8b113eaa7 --- /dev/null +++ b/pkg/local_object_storage/pilorama/metrics.go @@ -0,0 +1,20 @@ +package pilorama + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) + +type Metrics interface { + SetMode(m mode.Mode) + Close() + + AddMethodDuration(method string, d time.Duration, success bool) +} + +type noopMetrics struct{} + +func (m *noopMetrics) SetMode(mode.Mode) {} +func (m *noopMetrics) Close() {} +func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/pilorama/option.go b/pkg/local_object_storage/pilorama/option.go index 0dd5e63d4..d576d427f 100644 --- a/pkg/local_object_storage/pilorama/option.go +++ b/pkg/local_object_storage/pilorama/option.go @@ -15,6 +15,7 @@ type cfg struct { maxBatchDelay time.Duration maxBatchSize int openFile func(string, int, fs.FileMode) (*os.File, error) + metrics Metrics } func WithPath(path string) Option { @@ -52,3 +53,9 @@ func WithOpenFile(openFile func(string, int, fs.FileMode) (*os.File, error)) Opt c.openFile = openFile } } + +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} From d8ecc69d0091db029cb976456b8e923d332e1102 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 14:39:03 +0300 Subject: [PATCH 0350/1943] [#373] local storage: Pass parent ID This is required to add shard ID as metric label. Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/blobovnicza.go | 4 ++++ .../blobstor/blobovniczatree/metrics.go | 3 +++ pkg/local_object_storage/blobstor/blobstor.go | 7 +++++++ pkg/local_object_storage/blobstor/common/storage.go | 1 + pkg/local_object_storage/blobstor/fstree/fstree.go | 4 ++++ pkg/local_object_storage/blobstor/fstree/metrics.go | 3 +++ pkg/local_object_storage/blobstor/memstore/control.go | 1 + pkg/local_object_storage/blobstor/metrics.go | 2 ++ pkg/local_object_storage/blobstor/teststore/teststore.go | 2 ++ pkg/local_object_storage/metabase/db.go | 5 +++++ pkg/local_object_storage/metabase/metrics.go | 3 +++ pkg/local_object_storage/pilorama/boltdb.go | 4 ++++ pkg/local_object_storage/pilorama/forest.go | 1 + pkg/local_object_storage/pilorama/interface.go | 1 + pkg/local_object_storage/pilorama/metrics.go | 3 +++ pkg/local_object_storage/shard/id.go | 3 +++ 16 files changed, 47 insertions(+) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index d99e9aa89..1b0af342d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -256,3 +256,7 @@ func (b *Blobovniczas) SetCompressor(cc *compression.Config) { func (b *Blobovniczas) SetReportErrorFunc(f func(string, error)) { b.reportError = f } + +func (b *Blobovniczas) SetParentID(parentID string) { + b.metrics.SetParentID(parentID) +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index ffa3c8a8b..39fe34e28 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -9,6 +9,8 @@ import ( type Metrics interface { BlobovnicaMetrics(path string) blobovnicza.Metrics + SetParentID(parentID string) + SetMode(readOnly bool) Close() @@ -25,6 +27,7 @@ type noopMetrics struct{} func (m *noopMetrics) BlobovnicaMetrics(string) blobovnicza.Metrics { return &blobovnicza.NoopMetrics{} } +func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) Delete(time.Duration, bool, bool) {} diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index a422b2bce..6c6aed87f 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -72,6 +72,13 @@ func (b *BlobStor) SetLogger(l *logger.Logger) { b.log = l } +func (b *BlobStor) SetParentID(parentID string) { + b.metrics.SetParentID(parentID) + for _, ss := range b.storage { + ss.Storage.SetParentID(parentID) + } +} + // WithStorages provides sub-blobstors. func WithStorages(st []SubStorage) Option { return func(c *cfg) { diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index c5d187f30..b808480e2 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -19,6 +19,7 @@ type Storage interface { // SetReportErrorFunc allows to provide a function to be called on disk errors. // This function MUST be called before Open. SetReportErrorFunc(f func(string, error)) + SetParentID(parentID string) Get(context.Context, GetPrm) (GetRes, error) GetRange(context.Context, GetRangePrm) (GetRangeRes, error) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 645d4e5f7..9ca5d4bd9 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -554,3 +554,7 @@ func (t *FSTree) SetCompressor(cc *compression.Config) { func (t *FSTree) SetReportErrorFunc(_ func(string, error)) { // Do nothing, FSTree can encounter only one error which is returned. } + +func (t *FSTree) SetParentID(parentID string) { + t.metrics.SetParentID(parentID) +} diff --git a/pkg/local_object_storage/blobstor/fstree/metrics.go b/pkg/local_object_storage/blobstor/fstree/metrics.go index 04386d9b8..ca6a54975 100644 --- a/pkg/local_object_storage/blobstor/fstree/metrics.go +++ b/pkg/local_object_storage/blobstor/fstree/metrics.go @@ -3,6 +3,8 @@ package fstree import "time" type Metrics interface { + SetParentID(parentID string) + SetMode(readOnly bool) Close() @@ -16,6 +18,7 @@ type Metrics interface { type noopMetrics struct{} +func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) Iterate(time.Duration, bool) {} diff --git a/pkg/local_object_storage/blobstor/memstore/control.go b/pkg/local_object_storage/blobstor/memstore/control.go index 4deb9f6e2..e6943626b 100644 --- a/pkg/local_object_storage/blobstor/memstore/control.go +++ b/pkg/local_object_storage/blobstor/memstore/control.go @@ -13,3 +13,4 @@ func (s *memstoreImpl) Type() string { return Type } func (s *memstoreImpl) Path() string { return s.rootPath } func (s *memstoreImpl) SetCompressor(cc *compression.Config) { s.compression = cc } func (s *memstoreImpl) SetReportErrorFunc(f func(string, error)) { s.reportError = f } +func (s *memstoreImpl) SetParentID(string) {} diff --git a/pkg/local_object_storage/blobstor/metrics.go b/pkg/local_object_storage/blobstor/metrics.go index 4fd6f3e54..4a7b40092 100644 --- a/pkg/local_object_storage/blobstor/metrics.go +++ b/pkg/local_object_storage/blobstor/metrics.go @@ -3,6 +3,7 @@ package blobstor import "time" type Metrics interface { + SetParentID(parentID string) SetMode(readOnly bool) Close() @@ -16,6 +17,7 @@ type Metrics interface { type noopMetrics struct{} +func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) Delete(time.Duration, bool, bool) {} diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index c7179eb45..2508d74f6 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -214,3 +214,5 @@ func (s *TestStore) Iterate(ctx context.Context, req common.IteratePrm) (common. panic(fmt.Sprintf("unexpected storage call: Iterate(%+v)", req)) } } + +func (s *TestStore) SetParentID(string) {} diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index c54ed98d0..33f272117 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -294,6 +294,11 @@ func (db *DB) SetLogger(l *logger.Logger) { db.log = l } +// SetParentID sets parent ID to nested components. It is used after the shard ID was generated to use it in logs. +func (db *DB) SetParentID(parentID string) { + db.metrics.SetParentID(parentID) +} + // WithLogger returns option to set logger of DB. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { diff --git a/pkg/local_object_storage/metabase/metrics.go b/pkg/local_object_storage/metabase/metrics.go index 120579e4c..fc971bd81 100644 --- a/pkg/local_object_storage/metabase/metrics.go +++ b/pkg/local_object_storage/metabase/metrics.go @@ -7,6 +7,8 @@ import ( ) type Metrics interface { + SetParentID(parentID string) + SetMode(m mode.Mode) Close() @@ -15,6 +17,7 @@ type Metrics interface { type noopMetrics struct{} +func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(mode.Mode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 1b8ee7f18..a19561139 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -158,6 +158,10 @@ func (t *boltForest) Close() error { return err } +func (t *boltForest) SetParentID(id string) { + t.metrics.SetParentID(id) +} + // TreeMove implements the Forest interface. func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID string, m *Move) (*Move, error) { _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeMove", diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 7c5897ed0..76220c1db 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -119,6 +119,7 @@ func (f *memoryForest) SetMode(mode.Mode) error { func (f *memoryForest) Close() error { return nil } +func (f *memoryForest) SetParentID(string) {} // TreeGetByPath implements the Forest interface. func (f *memoryForest) TreeGetByPath(_ context.Context, cid cid.ID, treeID string, attr string, path []string, latest bool) ([]Node, error) { diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index c8287c6d4..89c752627 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -61,6 +61,7 @@ type ForestStorage interface { Open(bool) error Close() error SetMode(m mode.Mode) error + SetParentID(id string) Forest } diff --git a/pkg/local_object_storage/pilorama/metrics.go b/pkg/local_object_storage/pilorama/metrics.go index 8b113eaa7..543ad3e31 100644 --- a/pkg/local_object_storage/pilorama/metrics.go +++ b/pkg/local_object_storage/pilorama/metrics.go @@ -7,6 +7,8 @@ import ( ) type Metrics interface { + SetParentID(id string) + SetMode(m mode.Mode) Close() @@ -15,6 +17,7 @@ type Metrics interface { type noopMetrics struct{} +func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(mode.Mode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index 992a86c01..4f6fd0651 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -55,6 +55,9 @@ func (s *Shard) UpdateID() (err error) { if s.hasWriteCache() { s.writeCache.SetLogger(s.log) } + s.metaBase.SetParentID(s.info.ID.String()) + s.blobStor.SetParentID(s.info.ID.String()) + s.pilorama.SetParentID(s.info.ID.String()) if len(id) != 0 { return nil From 16a142cd0c403c51556e4e49a48fa1be02db9102 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 17:06:02 +0300 Subject: [PATCH 0351/1943] [#373] metrics: Add FSTree metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 21 +++-- pkg/local_object_storage/metrics/consts.go | 3 + pkg/local_object_storage/metrics/fstree.go | 61 ++++++++++++++ pkg/local_object_storage/shard/id.go | 4 +- pkg/metrics/consts.go | 15 ++++ pkg/metrics/fstree.go | 96 ++++++++++++++++++++++ pkg/metrics/mode.go | 36 ++++++++ pkg/metrics/node.go | 6 ++ 8 files changed, 236 insertions(+), 6 deletions(-) create mode 100644 pkg/local_object_storage/metrics/consts.go create mode 100644 pkg/local_object_storage/metrics/fstree.go create mode 100644 pkg/metrics/consts.go create mode 100644 pkg/metrics/fstree.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c88b7224e..44da2e981 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -36,6 +36,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + lsmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" shardmode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -752,12 +753,22 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { }, }) case fstree.Type: + fstreeOpts := []fstree.Option{ + fstree.WithPath(sRead.path), + fstree.WithPerm(sRead.perm), + fstree.WithDepth(sRead.depth), + fstree.WithNoSync(sRead.noSync), + } + if c.metricsCollector != nil { + fstreeOpts = append(fstreeOpts, + fstree.WithMetrics( + lsmetrics.NewFSTreeMetricsWithoutShardID(sRead.path, c.metricsCollector.FSTree()), + ), + ) + } + ss = append(ss, blobstor.SubStorage{ - Storage: fstree.New( - fstree.WithPath(sRead.path), - fstree.WithPerm(sRead.perm), - fstree.WithDepth(sRead.depth), - fstree.WithNoSync(sRead.noSync)), + Storage: fstree.New(fstreeOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { return true }, diff --git a/pkg/local_object_storage/metrics/consts.go b/pkg/local_object_storage/metrics/consts.go new file mode 100644 index 000000000..519930710 --- /dev/null +++ b/pkg/local_object_storage/metrics/consts.go @@ -0,0 +1,3 @@ +package metrics + +const undefined = "undefined" diff --git a/pkg/local_object_storage/metrics/fstree.go b/pkg/local_object_storage/metrics/fstree.go new file mode 100644 index 000000000..d3749d9bc --- /dev/null +++ b/pkg/local_object_storage/metrics/fstree.go @@ -0,0 +1,61 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewFSTreeMetricsWithoutShardID(path string, m metrics_impl.FSTreeMetrics) fstree.Metrics { + return &fstreeMetrics{ + shardID: undefined, + path: path, + m: m, + } +} + +type fstreeMetrics struct { + shardID string + path string + m metrics_impl.FSTreeMetrics +} + +func (m *fstreeMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *fstreeMetrics) SetMode(readOnly bool) { + m.m.SetMode(m.shardID, m.path, readOnly) +} +func (m *fstreeMetrics) Close() { + m.m.Close(m.shardID, m.path) +} + +func (m *fstreeMetrics) Iterate(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Iterate", d, success) +} +func (m *fstreeMetrics) Delete(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Delete", d, success) +} +func (m *fstreeMetrics) Exists(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Exists", d, success) +} +func (m *fstreeMetrics) Put(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Put", d, success) + if success { + m.m.AddPut(m.shardID, m.path, size) + } +} +func (m *fstreeMetrics) Get(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Get", d, success) + if success { + m.m.AddGet(m.shardID, m.path, size) + } +} +func (m *fstreeMetrics) GetRange(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "GetRange", d, success) + if success { + m.m.AddGet(m.shardID, m.path, size) + } +} diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index 4f6fd0651..e2ac423fd 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -57,7 +57,9 @@ func (s *Shard) UpdateID() (err error) { } s.metaBase.SetParentID(s.info.ID.String()) s.blobStor.SetParentID(s.info.ID.String()) - s.pilorama.SetParentID(s.info.ID.String()) + if s.pilorama != nil { + s.pilorama.SetParentID(s.info.ID.String()) + } if len(id) != 0 { return nil diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go new file mode 100644 index 000000000..d65ee51c1 --- /dev/null +++ b/pkg/metrics/consts.go @@ -0,0 +1,15 @@ +package metrics + +const ( + fstreeSubSystem = "fstree" + + successLabel = "success" + shardIDLabel = "shardID" + modeLabel = "mode" + pathLabel = "path" + methodLabel = "method" + + readWriteMode = "READ_WRITE" + readOnlyMode = "READ_ONLY" + closedMode = "CLOSED" +) diff --git a/pkg/metrics/fstree.go b/pkg/metrics/fstree.go new file mode 100644 index 000000000..5bc43757a --- /dev/null +++ b/pkg/metrics/fstree.go @@ -0,0 +1,96 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type FSTreeMetrics interface { + SetMode(shardID, path string, readOnly bool) + Close(shardID, path string) + + MethodDuration(shardID, path string, method string, d time.Duration, success bool) + AddGet(shardID, path string, size int) + AddPut(shardID, path string, size int) +} + +type fstreeMetrics struct { + mode *shardIDPathModeValue + reqDuration *prometheus.HistogramVec + put *prometheus.CounterVec + get *prometheus.CounterVec +} + +func newFSTreeMetrics() *fstreeMetrics { + return &fstreeMetrics{ + mode: newShardIDPathMode(fstreeSubSystem, "mode", "FSTree mode value"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated FSTree request process duration", + }, []string{shardIDLabel, successLabel, pathLabel, methodLabel}), + put: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to FSTree", + }, []string{shardIDLabel, pathLabel}), + get: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: fstreeSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from FSTree", + }, []string{shardIDLabel, pathLabel}), + } +} + +func (m *fstreeMetrics) SetMode(shardID, path string, readOnly bool) { + modeValue := readWriteMode + if readOnly { + modeValue = readOnlyMode + } + m.mode.SetMode(shardID, path, modeValue) +} + +func (m *fstreeMetrics) Close(shardID, path string) { + m.mode.SetMode(shardID, path, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + m.get.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + m.put.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func (m *fstreeMetrics) MethodDuration(shardID, path string, method string, d time.Duration, success bool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + }).Observe(d.Seconds()) +} + +func (m *fstreeMetrics) AddGet(shardID, path string, size int) { + m.get.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (m *fstreeMetrics) AddPut(shardID, path string, size int) { + m.put.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go index 7ae3fd5f9..aa2246676 100644 --- a/pkg/metrics/mode.go +++ b/pkg/metrics/mode.go @@ -37,3 +37,39 @@ func (m *shardIDModeValue) Delete(shardID string) { wcShardID: shardID, }) } + +type shardIDPathModeValue struct { + modeValue *prometheus.GaugeVec +} + +func newShardIDPathMode(subsystem, name, help string) *shardIDPathModeValue { + return &shardIDPathModeValue{ + modeValue: metrics.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: name, + Help: help, + }, []string{shardIDLabel, pathLabel, modeLabel}), + } +} + +func (m *shardIDPathModeValue) SetMode(shardID, path string, mode string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + + m.modeValue.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + modeLabel: mode, + }).Set(1) +} + +func (m *shardIDPathModeValue) Delete(shardID, path string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 232c178d8..0b04b635b 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -14,6 +14,7 @@ type NodeMetrics struct { objectService *objectServiceMetrics treeService *treeServiceMetrics epoch prometheus.Gauge + fstree *fstreeMetrics } func NewNodeMetrics() *NodeMetrics { @@ -41,6 +42,7 @@ func NewNodeMetrics() *NodeMetrics { replicator: replicator, treeService: treeService, epoch: epoch, + fstree: newFSTreeMetrics(), } } @@ -68,3 +70,7 @@ func (m *NodeMetrics) Engine() EngineMetrics { func (m *NodeMetrics) State() StateMetrics { return m.state } + +func (m *NodeMetrics) FSTree() FSTreeMetrics { + return m.fstree +} From 56f320dd85a5c065812becc9eeef15acecf0f371 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 7 Jun 2023 18:04:23 +0300 Subject: [PATCH 0352/1943] [#373] metrics: Add blobstor metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 18 ++-- pkg/local_object_storage/metrics/blobstore.go | 65 ++++++++++++++ pkg/metrics/blobstore.go | 87 +++++++++++++++++++ pkg/metrics/consts.go | 14 +-- pkg/metrics/fstree.go | 6 +- pkg/metrics/mode.go | 8 ++ pkg/metrics/node.go | 42 ++++----- 7 files changed, 198 insertions(+), 42 deletions(-) create mode 100644 pkg/local_object_storage/metrics/blobstore.go create mode 100644 pkg/metrics/blobstore.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 44da2e981..bc73f8bc5 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -786,19 +786,23 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { piloramaOpts := c.getPiloramaOpts(shCfg) ss := c.getSubstorageOpts(shCfg) + blobstoreOpts := []blobstor.Option{ + blobstor.WithCompressObjects(shCfg.compress), + blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), + blobstor.WithStorages(ss), + blobstor.WithLogger(c.log), + } + if c.metricsCollector != nil { + blobstoreOpts = append(blobstoreOpts, blobstor.WithMetrics(lsmetrics.NewBlobstoreMetrics(c.metricsCollector.Blobstore()))) + } + var sh shardOptsWithID sh.configID = shCfg.id() sh.shOpts = []shard.Option{ shard.WithLogger(c.log), shard.WithRefillMetabase(shCfg.refillMetabase), shard.WithMode(shCfg.mode), - shard.WithBlobStorOptions( - blobstor.WithCompressObjects(shCfg.compress), - blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), - blobstor.WithStorages(ss), - - blobstor.WithLogger(c.log), - ), + shard.WithBlobStorOptions(blobstoreOpts...), shard.WithMetaBaseOptions( meta.WithPath(shCfg.metaCfg.path), meta.WithPermissions(shCfg.metaCfg.perm), diff --git a/pkg/local_object_storage/metrics/blobstore.go b/pkg/local_object_storage/metrics/blobstore.go new file mode 100644 index 000000000..48249e89c --- /dev/null +++ b/pkg/local_object_storage/metrics/blobstore.go @@ -0,0 +1,65 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +type blobstoreMetrics struct { + shardID string + m metrics_impl.BlobstoreMetrics +} + +func NewBlobstoreMetrics(m metrics_impl.BlobstoreMetrics) blobstor.Metrics { + return &blobstoreMetrics{ + shardID: undefined, + m: m, + } +} + +func (m *blobstoreMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *blobstoreMetrics) SetMode(readOnly bool) { + m.m.SetMode(m.shardID, readOnly) +} + +func (m *blobstoreMetrics) Close() { + m.m.Close(m.shardID) +} + +func (m *blobstoreMetrics) Delete(d time.Duration, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Delete", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +} + +func (m *blobstoreMetrics) Exists(d time.Duration, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Exists", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) +} + +func (m *blobstoreMetrics) GetRange(d time.Duration, size int, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "GetRange", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) + if success { + m.m.AddGet(m.shardID, size) + } +} + +func (m *blobstoreMetrics) Get(d time.Duration, size int, success, withStorageID bool) { + m.m.MethodDuration(m.shardID, "Get", d, success, metrics_impl.NullBool{Bool: withStorageID, Valid: true}) + if success { + m.m.AddGet(m.shardID, size) + } +} + +func (m *blobstoreMetrics) Iterate(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, "Iterate", d, success, metrics_impl.NullBool{}) +} + +func (m *blobstoreMetrics) Put(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, "Put", d, success, metrics_impl.NullBool{}) + if success { + m.m.AddPut(m.shardID, size) + } +} diff --git a/pkg/metrics/blobstore.go b/pkg/metrics/blobstore.go new file mode 100644 index 000000000..d9bb3f029 --- /dev/null +++ b/pkg/metrics/blobstore.go @@ -0,0 +1,87 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type BlobstoreMetrics interface { + SetMode(shardID string, readOnly bool) + Close(shardID string) + + MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool) + AddPut(shardID string, size int) + AddGet(shardID string, size int) +} + +type blobstoreMetrics struct { + mode *shardIDModeValue + reqDuration *prometheus.HistogramVec + put *prometheus.CounterVec + get *prometheus.CounterVec +} + +func newBlobstoreMetrics() *blobstoreMetrics { + return &blobstoreMetrics{ + mode: newShardIDMode(blobstoreSubSystem, "mode", "Blobstore mode value"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Blobstore request process duration", + }, []string{shardIDLabel, successLabel, methodLabel, withStorageIDLabel}), + put: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to Blobstore", + }, []string{shardIDLabel}), + get: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobstoreSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from Blobstore", + }, []string{shardIDLabel}), + } +} + +func (m *blobstoreMetrics) SetMode(shardID string, readOnly bool) { + m.mode.SetMode(shardID, modeFromBool(readOnly)) +} + +func (m *blobstoreMetrics) Close(shardID string) { + m.mode.SetMode(shardID, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) + m.get.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) + m.put.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) +} + +func (m *blobstoreMetrics) MethodDuration(shardID string, method string, d time.Duration, success bool, withStorageID NullBool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), + methodLabel: method, + withStorageIDLabel: withStorageID.String(), + }).Observe(d.Seconds()) +} + +func (m *blobstoreMetrics) AddPut(shardID string, size int) { + m.put.With(prometheus.Labels{ + shardIDLabel: shardID, + }).Add(float64(size)) +} + +func (m *blobstoreMetrics) AddGet(shardID string, size int) { + m.get.With(prometheus.Labels{ + shardIDLabel: shardID, + }).Add(float64(size)) +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index d65ee51c1..d06d15902 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -1,13 +1,15 @@ package metrics const ( - fstreeSubSystem = "fstree" + fstreeSubSystem = "fstree" + blobstoreSubSystem = "blobstore" - successLabel = "success" - shardIDLabel = "shardID" - modeLabel = "mode" - pathLabel = "path" - methodLabel = "method" + successLabel = "success" + shardIDLabel = "shardID" + modeLabel = "mode" + pathLabel = "path" + methodLabel = "method" + withStorageIDLabel = "withStorageID" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" diff --git a/pkg/metrics/fstree.go b/pkg/metrics/fstree.go index 5bc43757a..4d4f0693b 100644 --- a/pkg/metrics/fstree.go +++ b/pkg/metrics/fstree.go @@ -49,11 +49,7 @@ func newFSTreeMetrics() *fstreeMetrics { } func (m *fstreeMetrics) SetMode(shardID, path string, readOnly bool) { - modeValue := readWriteMode - if readOnly { - modeValue = readOnlyMode - } - m.mode.SetMode(shardID, path, modeValue) + m.mode.SetMode(shardID, path, modeFromBool(readOnly)) } func (m *fstreeMetrics) Close(shardID, path string) { diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go index aa2246676..ad959e18e 100644 --- a/pkg/metrics/mode.go +++ b/pkg/metrics/mode.go @@ -73,3 +73,11 @@ func (m *shardIDPathModeValue) Delete(shardID, path string) { pathLabel: path, }) } + +func modeFromBool(readOnly bool) string { + modeValue := readWriteMode + if readOnly { + modeValue = readOnlyMode + } + return modeValue +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 0b04b635b..bf26aab61 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -15,34 +15,24 @@ type NodeMetrics struct { treeService *treeServiceMetrics epoch prometheus.Gauge fstree *fstreeMetrics + blobstore *blobstoreMetrics } func NewNodeMetrics() *NodeMetrics { - objectService := newObjectServiceMetrics() - - engine := newEngineMetrics() - - state := newStateMetrics() - - replicator := newReplicatorMetrics() - - treeService := newTreeServiceMetrics() - - epoch := metrics.NewGauge(prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: innerRingSubsystem, - Name: "epoch", - Help: "Current epoch as seen by inner-ring node.", - }) - return &NodeMetrics{ - objectService: objectService, - engine: engine, - state: state, - replicator: replicator, - treeService: treeService, - epoch: epoch, - fstree: newFSTreeMetrics(), + objectService: newObjectServiceMetrics(), + engine: newEngineMetrics(), + state: newStateMetrics(), + replicator: newReplicatorMetrics(), + treeService: newTreeServiceMetrics(), + epoch: metrics.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: innerRingSubsystem, + Name: "epoch", + Help: "Current epoch as seen by inner-ring node.", + }), + fstree: newFSTreeMetrics(), + blobstore: newBlobstoreMetrics(), } } @@ -74,3 +64,7 @@ func (m *NodeMetrics) State() StateMetrics { func (m *NodeMetrics) FSTree() FSTreeMetrics { return m.fstree } + +func (m *NodeMetrics) Blobstore() BlobstoreMetrics { + return m.blobstore +} From af608da952dbf9c531c7a31b008a81bfee55ef16 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Jun 2023 12:14:32 +0300 Subject: [PATCH 0353/1943] [#373] metrics: Add blobovnizca metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 27 ++-- .../blobovnicza/control.go | 8 +- .../blobovnicza/metrics.go | 16 +- pkg/local_object_storage/blobovnicza/sizes.go | 4 +- .../blobstor/blobovniczatree/control.go | 2 +- .../blobstor/blobovniczatree/metrics.go | 8 +- .../metrics/blobovnizca.go | 98 ++++++++++++ pkg/metrics/blobovnizca.go | 141 ++++++++++++++++++ pkg/metrics/consts.go | 5 +- pkg/metrics/node.go | 10 +- 10 files changed, 287 insertions(+), 32 deletions(-) create mode 100644 pkg/local_object_storage/metrics/blobovnizca.go create mode 100644 pkg/metrics/blobovnizca.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index bc73f8bc5..0f2db8725 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -738,16 +738,25 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { for _, sRead := range shCfg.subStorages { switch sRead.typ { case blobovniczatree.Type: - ss = append(ss, blobstor.SubStorage{ - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(sRead.path), - blobovniczatree.WithPermissions(sRead.perm), - blobovniczatree.WithBlobovniczaSize(sRead.size), - blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), - blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), - blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobTreeOpts := []blobovniczatree.Option{ + blobovniczatree.WithRootPath(sRead.path), + blobovniczatree.WithPermissions(sRead.perm), + blobovniczatree.WithBlobovniczaSize(sRead.size), + blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), + blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), + blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobovniczatree.WithLogger(c.log), + } - blobovniczatree.WithLogger(c.log)), + if c.metricsCollector != nil { + blobTreeOpts = append(blobTreeOpts, + blobovniczatree.WithMetrics( + lsmetrics.NewBlobovniczaTreeMetrics(sRead.path, c.metricsCollector.BlobobvnizcaTreeMetrics()), + ), + ) + } + ss = append(ss, blobstor.SubStorage{ + Storage: blobovniczatree.NewBlobovniczaTree(blobTreeOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < shCfg.smallSizeObjectLimit }, diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index c776afe06..da4c870bd 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -36,7 +36,7 @@ func (b *Blobovnicza) Open() error { b.boltDB, err = bbolt.Open(b.path, b.perm, b.boltOptions) if err == nil { - b.metrics.IncOpenCount() + b.metrics.IncOpenBlobovnizcaCount() } return err @@ -86,7 +86,7 @@ func (b *Blobovnicza) Init() error { sz := uint64(info.Size()) b.filled.Store(sz) - b.metrics.IncSize(sz) + b.metrics.AddSize(sz) return err } @@ -98,8 +98,8 @@ func (b *Blobovnicza) Close() error { err := b.boltDB.Close() if err == nil { - b.metrics.DecOpenCount() - b.metrics.DecSize(b.filled.Load()) + b.metrics.DecOpenBlobovnizcaCount() + b.metrics.SubSize(b.filled.Load()) } return err } diff --git a/pkg/local_object_storage/blobovnicza/metrics.go b/pkg/local_object_storage/blobovnicza/metrics.go index 6127370bc..d511f90f1 100644 --- a/pkg/local_object_storage/blobovnicza/metrics.go +++ b/pkg/local_object_storage/blobovnicza/metrics.go @@ -1,16 +1,16 @@ package blobovnicza type Metrics interface { - IncOpenCount() - DecOpenCount() + IncOpenBlobovnizcaCount() + DecOpenBlobovnizcaCount() - IncSize(size uint64) - DecSize(size uint64) + AddSize(size uint64) + SubSize(size uint64) } type NoopMetrics struct{} -func (m *NoopMetrics) IncOpenCount() {} -func (m *NoopMetrics) DecOpenCount() {} -func (m *NoopMetrics) IncSize(uint64) {} -func (m *NoopMetrics) DecSize(uint64) {} +func (m *NoopMetrics) IncOpenBlobovnizcaCount() {} +func (m *NoopMetrics) DecOpenBlobovnizcaCount() {} +func (m *NoopMetrics) AddSize(uint64) {} +func (m *NoopMetrics) SubSize(uint64) {} diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index bdbc77d18..7e10b728e 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -41,12 +41,12 @@ func upperPowerOfTwo(v uint64) uint64 { func (b *Blobovnicza) incSize(sz uint64) { b.filled.Add(sz) - b.metrics.IncSize(sz) + b.metrics.AddSize(sz) } func (b *Blobovnicza) decSize(sz uint64) { b.filled.Add(^(sz - 1)) - b.metrics.DecSize(sz) + b.metrics.SubSize(sz) } func (b *Blobovnicza) full() bool { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 0045e08db..bc3d7d60c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -130,7 +130,7 @@ func (b *Blobovniczas) openBlobovniczaNoCache(p string) (*blobovnicza.Blobovnicz blz := blobovnicza.New(append(b.blzOpts, blobovnicza.WithReadOnly(b.readOnly), blobovnicza.WithPath(path), - blobovnicza.WithMetrics(b.metrics.BlobovnicaMetrics(path)), + blobovnicza.WithMetrics(b.metrics.Blobovnizca()), )...) if err := blz.Open(); err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index 39fe34e28..6c9b600c0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -7,7 +7,7 @@ import ( ) type Metrics interface { - BlobovnicaMetrics(path string) blobovnicza.Metrics + Blobovnizca() blobovnicza.Metrics SetParentID(parentID string) @@ -24,9 +24,6 @@ type Metrics interface { type noopMetrics struct{} -func (m *noopMetrics) BlobovnicaMetrics(string) blobovnicza.Metrics { - return &blobovnicza.NoopMetrics{} -} func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} @@ -36,3 +33,6 @@ func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} func (m *noopMetrics) Iterate(time.Duration, bool) {} func (m *noopMetrics) Put(time.Duration, int, bool) {} +func (m *noopMetrics) Blobovnizca() blobovnicza.Metrics { + return &blobovnicza.NoopMetrics{} +} diff --git a/pkg/local_object_storage/metrics/blobovnizca.go b/pkg/local_object_storage/metrics/blobovnizca.go new file mode 100644 index 000000000..a498d822e --- /dev/null +++ b/pkg/local_object_storage/metrics/blobovnizca.go @@ -0,0 +1,98 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewBlobovniczaTreeMetrics(path string, m metrics_impl.BlobobvnizcaMetrics) blobovniczatree.Metrics { + return &blobovniczaTreeMetrics{ + path: path, + shardID: undefined, + m: m, + } +} + +type blobovniczaTreeMetrics struct { + shardID string + path string + m metrics_impl.BlobobvnizcaMetrics +} + +func (m *blobovniczaTreeMetrics) Blobovnizca() blobovnicza.Metrics { + return &blobovniczaMetrics{ + shardID: func() string { return m.shardID }, + path: m.path, + m: m.m, + } +} + +func (m *blobovniczaTreeMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *blobovniczaTreeMetrics) SetMode(readOnly bool) { + m.m.SetBlobobvnizcaTreeMode(m.shardID, m.path, readOnly) +} + +func (m *blobovniczaTreeMetrics) Close() { + m.m.CloseBlobobvnizcaTree(m.shardID, m.path) +} + +func (m *blobovniczaTreeMetrics) Delete(d time.Duration, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Delete", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) +} + +func (m *blobovniczaTreeMetrics) Exists(d time.Duration, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Exists", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) +} + +func (m *blobovniczaTreeMetrics) GetRange(d time.Duration, size int, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "GetRange", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) + if success { + m.m.AddBlobobvnizcaTreeGet(m.shardID, m.path, size) + } +} + +func (m *blobovniczaTreeMetrics) Get(d time.Duration, size int, success, withStorageID bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Get", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) + if success { + m.m.AddBlobobvnizcaTreeGet(m.shardID, m.path, size) + } +} + +func (m *blobovniczaTreeMetrics) Iterate(d time.Duration, success bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Iterate", d, success, metrics_impl.NullBool{}) +} + +func (m *blobovniczaTreeMetrics) Put(d time.Duration, size int, success bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Put", d, success, metrics_impl.NullBool{}) + if success { + m.m.AddBlobobvnizcaTreePut(m.shardID, m.path, size) + } +} + +type blobovniczaMetrics struct { + m metrics_impl.BlobobvnizcaMetrics + shardID func() string + path string +} + +func (m *blobovniczaMetrics) AddSize(size uint64) { + m.m.AddTreeSize(m.shardID(), m.path, size) +} + +func (m *blobovniczaMetrics) SubSize(size uint64) { + m.m.SubTreeSize(m.shardID(), m.path, size) +} + +func (m *blobovniczaMetrics) IncOpenBlobovnizcaCount() { + m.m.IncOpenBlobovnizcaCount(m.shardID(), m.path) +} + +func (m *blobovniczaMetrics) DecOpenBlobovnizcaCount() { + m.m.DecOpenBlobovnizcaCount(m.shardID(), m.path) +} diff --git a/pkg/metrics/blobovnizca.go b/pkg/metrics/blobovnizca.go new file mode 100644 index 000000000..9dc3ed572 --- /dev/null +++ b/pkg/metrics/blobovnizca.go @@ -0,0 +1,141 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type BlobobvnizcaMetrics interface { + SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) + CloseBlobobvnizcaTree(shardID, path string) + BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) + AddBlobobvnizcaTreePut(shardID, path string, size int) + AddBlobobvnizcaTreeGet(shardID, path string, size int) + + AddTreeSize(shardID, path string, size uint64) + SubTreeSize(shardID, path string, size uint64) + + IncOpenBlobovnizcaCount(shardID, path string) + DecOpenBlobovnizcaCount(shardID, path string) +} + +type blobovnizca struct { + treeMode *shardIDPathModeValue + treeReqDuration *prometheus.HistogramVec + treePut *prometheus.CounterVec + treeGet *prometheus.CounterVec + treeSize *prometheus.GaugeVec + treeOpenCounter *prometheus.GaugeVec +} + +func newBlobovnizca() *blobovnizca { + return &blobovnizca{ + treeMode: newShardIDPathMode(blobovnizaTreeSubSystem, "mode", "Blobovnizca tree mode"), + + treeReqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Blobovnizca tree request process duration", + }, []string{shardIDLabel, pathLabel, successLabel, methodLabel, withStorageIDLabel}), + treePut: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + treeGet: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + treeSize: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "size_bytes", + Help: "Blobovnizca tree size", + }, []string{shardIDLabel, pathLabel}), + treeOpenCounter: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: blobovnizaTreeSubSystem, + Name: "open_blobovnizca_count", + Help: "Count of opened blobovnizcas of Blobovnizca tree", + }, []string{shardIDLabel, pathLabel}), + } +} + +func (b *blobovnizca) SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) { + b.treeMode.SetMode(shardID, path, modeFromBool(readOnly)) +} + +func (b *blobovnizca) CloseBlobobvnizcaTree(shardID, path string) { + b.treeMode.SetMode(shardID, path, closedMode) + b.treeReqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + b.treeGet.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + b.treePut.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func (b *blobovnizca) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) { + b.treeReqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + withStorageIDLabel: withStorageID.String(), + }).Observe(d.Seconds()) +} + +func (b *blobovnizca) AddBlobobvnizcaTreePut(shardID, path string, size int) { + b.treePut.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) AddBlobobvnizcaTreeGet(shardID, path string, size int) { + b.treeGet.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) AddTreeSize(shardID, path string, size uint64) { + b.treeSize.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +func (b *blobovnizca) SubTreeSize(shardID, path string, size uint64) { + b.treeSize.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Sub(float64(size)) +} + +func (b *blobovnizca) IncOpenBlobovnizcaCount(shardID, path string) { + b.treeOpenCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Inc() +} + +func (b *blobovnizca) DecOpenBlobovnizcaCount(shardID, path string) { + b.treeOpenCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Dec() +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index d06d15902..d79e11696 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -1,8 +1,9 @@ package metrics const ( - fstreeSubSystem = "fstree" - blobstoreSubSystem = "blobstore" + fstreeSubSystem = "fstree" + blobstoreSubSystem = "blobstore" + blobovnizaTreeSubSystem = "blobovniza_tree" successLabel = "success" shardIDLabel = "shardID" diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index bf26aab61..babd6cdcf 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -16,6 +16,7 @@ type NodeMetrics struct { epoch prometheus.Gauge fstree *fstreeMetrics blobstore *blobstoreMetrics + blobobvnizca *blobovnizca } func NewNodeMetrics() *NodeMetrics { @@ -31,8 +32,9 @@ func NewNodeMetrics() *NodeMetrics { Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }), - fstree: newFSTreeMetrics(), - blobstore: newBlobstoreMetrics(), + fstree: newFSTreeMetrics(), + blobstore: newBlobstoreMetrics(), + blobobvnizca: newBlobovnizca(), } } @@ -68,3 +70,7 @@ func (m *NodeMetrics) FSTree() FSTreeMetrics { func (m *NodeMetrics) Blobstore() BlobstoreMetrics { return m.blobstore } + +func (m *NodeMetrics) BlobobvnizcaTreeMetrics() BlobobvnizcaMetrics { + return m.blobobvnizca +} From e89fa110c75230c341315c7ae76141a894388d29 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Jun 2023 12:45:31 +0300 Subject: [PATCH 0354/1943] [#373] metrics: Add metabase metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 28 +++++----- pkg/local_object_storage/metabase/db.go | 7 +++ pkg/local_object_storage/metrics/metabase.go | 39 ++++++++++++++ pkg/metrics/consts.go | 1 + pkg/metrics/metabase.go | 54 ++++++++++++++++++++ pkg/metrics/node.go | 6 +++ 6 files changed, 123 insertions(+), 12 deletions(-) create mode 100644 pkg/local_object_storage/metrics/metabase.go create mode 100644 pkg/metrics/metabase.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 0f2db8725..1c8c49606 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -805,6 +805,21 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { blobstoreOpts = append(blobstoreOpts, blobstor.WithMetrics(lsmetrics.NewBlobstoreMetrics(c.metricsCollector.Blobstore()))) } + mbOptions := []meta.Option{ + meta.WithPath(shCfg.metaCfg.path), + meta.WithPermissions(shCfg.metaCfg.perm), + meta.WithMaxBatchSize(shCfg.metaCfg.maxBatchSize), + meta.WithMaxBatchDelay(shCfg.metaCfg.maxBatchDelay), + meta.WithBoltDBOptions(&bbolt.Options{ + Timeout: 100 * time.Millisecond, + }), + meta.WithLogger(c.log), + meta.WithEpochState(c.cfgNetmap.state), + } + if c.metricsCollector != nil { + mbOptions = append(mbOptions, meta.WithMetrics(lsmetrics.NewMetabaseMetrics(shCfg.metaCfg.path, c.metricsCollector.MetabaseMetrics()))) + } + var sh shardOptsWithID sh.configID = shCfg.id() sh.shOpts = []shard.Option{ @@ -812,18 +827,7 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { shard.WithRefillMetabase(shCfg.refillMetabase), shard.WithMode(shCfg.mode), shard.WithBlobStorOptions(blobstoreOpts...), - shard.WithMetaBaseOptions( - meta.WithPath(shCfg.metaCfg.path), - meta.WithPermissions(shCfg.metaCfg.perm), - meta.WithMaxBatchSize(shCfg.metaCfg.maxBatchSize), - meta.WithMaxBatchDelay(shCfg.metaCfg.maxBatchDelay), - meta.WithBoltDBOptions(&bbolt.Options{ - Timeout: 100 * time.Millisecond, - }), - - meta.WithLogger(c.log), - meta.WithEpochState(c.cfgNetmap.state), - ), + shard.WithMetaBaseOptions(mbOptions...), shard.WithPiloramaOptions(piloramaOpts...), shard.WithWriteCache(shCfg.writecacheCfg.enabled), shard.WithWriteCacheOptions(writeCacheOpts...), diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index 33f272117..e7de46e15 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -356,3 +356,10 @@ func WithEpochState(s EpochState) Option { c.epochState = s } } + +// WithMetrics returns option to specify metrics collector. +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/metrics/metabase.go b/pkg/local_object_storage/metrics/metabase.go new file mode 100644 index 000000000..d0fb31936 --- /dev/null +++ b/pkg/local_object_storage/metrics/metabase.go @@ -0,0 +1,39 @@ +package metrics + +import ( + "time" + + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewMetabaseMetrics(path string, m metrics_impl.MetabaseMetrics) meta.Metrics { + return &metabaseMetrics{ + shardID: undefined, + path: path, + m: m, + } +} + +type metabaseMetrics struct { + shardID string + path string + m metrics_impl.MetabaseMetrics +} + +func (m *metabaseMetrics) SetParentID(parentID string) { + m.shardID = parentID +} + +func (m *metabaseMetrics) SetMode(mode mode.Mode) { + m.m.SetMode(m.shardID, m.path, mode.String()) +} + +func (m *metabaseMetrics) Close() { + m.m.Close(m.shardID, m.path) +} + +func (m *metabaseMetrics) AddMethodDuration(method string, d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, method, d, success) +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index d79e11696..a9781abe1 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -4,6 +4,7 @@ const ( fstreeSubSystem = "fstree" blobstoreSubSystem = "blobstore" blobovnizaTreeSubSystem = "blobovniza_tree" + metabaseSubSystem = "metabase" successLabel = "success" shardIDLabel = "shardID" diff --git a/pkg/metrics/metabase.go b/pkg/metrics/metabase.go new file mode 100644 index 000000000..640c7f721 --- /dev/null +++ b/pkg/metrics/metabase.go @@ -0,0 +1,54 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type MetabaseMetrics interface { + SetMode(shardID, path string, mode string) + Close(shardID, path string) + + MethodDuration(shardID, path string, method string, d time.Duration, success bool) +} + +func newMetabaseMetrics() *metabaseMetrics { + return &metabaseMetrics{ + mode: newShardIDPathMode(metabaseSubSystem, "mode", "Metabase mode"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: metabaseSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Metabase request process duration", + }, []string{shardIDLabel, successLabel, pathLabel, methodLabel}), + } +} + +type metabaseMetrics struct { + mode *shardIDPathModeValue + reqDuration *prometheus.HistogramVec +} + +func (m *metabaseMetrics) SetMode(shardID, path string, mode string) { + m.mode.SetMode(shardID, path, mode) +} + +func (m *metabaseMetrics) Close(shardID, path string) { + m.mode.SetMode(shardID, path, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +func (m *metabaseMetrics) MethodDuration(shardID, path string, method string, d time.Duration, success bool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + }).Observe(d.Seconds()) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index babd6cdcf..e4e98fe1f 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -17,6 +17,7 @@ type NodeMetrics struct { fstree *fstreeMetrics blobstore *blobstoreMetrics blobobvnizca *blobovnizca + metabase *metabaseMetrics } func NewNodeMetrics() *NodeMetrics { @@ -35,6 +36,7 @@ func NewNodeMetrics() *NodeMetrics { fstree: newFSTreeMetrics(), blobstore: newBlobstoreMetrics(), blobobvnizca: newBlobovnizca(), + metabase: newMetabaseMetrics(), } } @@ -74,3 +76,7 @@ func (m *NodeMetrics) Blobstore() BlobstoreMetrics { func (m *NodeMetrics) BlobobvnizcaTreeMetrics() BlobobvnizcaMetrics { return m.blobobvnizca } + +func (m *NodeMetrics) MetabaseMetrics() MetabaseMetrics { + return m.metabase +} From b5d9f4a2856d19db37251f655082b8f482444bab Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Jun 2023 16:19:47 +0300 Subject: [PATCH 0355/1943] [#373] metrics: Add pilorama metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 ++ pkg/local_object_storage/metrics/pilorama.go | 37 ++++++++++++++ pkg/metrics/consts.go | 1 + pkg/metrics/node.go | 6 +++ pkg/metrics/pilorama.go | 53 ++++++++++++++++++++ 5 files changed, 100 insertions(+) create mode 100644 pkg/local_object_storage/metrics/pilorama.go create mode 100644 pkg/metrics/pilorama.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 1c8c49606..2cb785d30 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -729,6 +729,9 @@ func (c *cfg) getPiloramaOpts(shCfg shardCfg) []pilorama.Option { pilorama.WithMaxBatchSize(prRead.maxBatchSize), pilorama.WithMaxBatchDelay(prRead.maxBatchDelay), ) + if c.metricsCollector != nil { + piloramaOpts = append(piloramaOpts, pilorama.WithMetrics(lsmetrics.NewPiloramaMetrics(c.metricsCollector.PiloramaMetrics()))) + } } return piloramaOpts } diff --git a/pkg/local_object_storage/metrics/pilorama.go b/pkg/local_object_storage/metrics/pilorama.go new file mode 100644 index 000000000..21f027a6e --- /dev/null +++ b/pkg/local_object_storage/metrics/pilorama.go @@ -0,0 +1,37 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewPiloramaMetrics(m metrics_impl.PiloramaMetrics) pilorama.Metrics { + return &piloramaMetrics{ + shardID: undefined, + m: m, + } +} + +type piloramaMetrics struct { + shardID string + m metrics_impl.PiloramaMetrics +} + +func (m *piloramaMetrics) SetParentID(id string) { + m.shardID = id +} + +func (m *piloramaMetrics) SetMode(mode mode.Mode) { + m.m.SetMode(m.shardID, mode) +} + +func (m *piloramaMetrics) Close() { + m.m.Close(m.shardID) +} + +func (m *piloramaMetrics) AddMethodDuration(method string, d time.Duration, success bool) { + m.m.AddMethodDuration(m.shardID, method, d, success) +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index a9781abe1..878e85a35 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -5,6 +5,7 @@ const ( blobstoreSubSystem = "blobstore" blobovnizaTreeSubSystem = "blobovniza_tree" metabaseSubSystem = "metabase" + piloramaSubSystem = "pilorama" successLabel = "success" shardIDLabel = "shardID" diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index e4e98fe1f..3d3712450 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -18,6 +18,7 @@ type NodeMetrics struct { blobstore *blobstoreMetrics blobobvnizca *blobovnizca metabase *metabaseMetrics + pilorama *piloramaMetrics } func NewNodeMetrics() *NodeMetrics { @@ -37,6 +38,7 @@ func NewNodeMetrics() *NodeMetrics { blobstore: newBlobstoreMetrics(), blobobvnizca: newBlobovnizca(), metabase: newMetabaseMetrics(), + pilorama: newPiloramaMetrics(), } } @@ -80,3 +82,7 @@ func (m *NodeMetrics) BlobobvnizcaTreeMetrics() BlobobvnizcaMetrics { func (m *NodeMetrics) MetabaseMetrics() MetabaseMetrics { return m.metabase } + +func (m *NodeMetrics) PiloramaMetrics() PiloramaMetrics { + return m.pilorama +} diff --git a/pkg/metrics/pilorama.go b/pkg/metrics/pilorama.go new file mode 100644 index 000000000..41672a4b5 --- /dev/null +++ b/pkg/metrics/pilorama.go @@ -0,0 +1,53 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type PiloramaMetrics interface { + SetMode(shardID string, m mode.Mode) + Close(shardID string) + + AddMethodDuration(shardID string, method string, d time.Duration, success bool) +} + +func newPiloramaMetrics() *piloramaMetrics { + return &piloramaMetrics{ + mode: newShardIDMode(piloramaSubSystem, "mode", "Pilorama mode"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: piloramaSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated Pilorama request process duration", + }, []string{shardIDLabel, successLabel, methodLabel}), + } +} + +type piloramaMetrics struct { + mode *shardIDModeValue + reqDuration *prometheus.HistogramVec +} + +func (m *piloramaMetrics) SetMode(shardID string, mode mode.Mode) { + m.mode.SetMode(shardID, mode.String()) +} + +func (m *piloramaMetrics) AddMethodDuration(shardID string, method string, d time.Duration, success bool) { + m.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), + methodLabel: method, + }).Observe(d.Seconds()) +} + +func (m *piloramaMetrics) Close(shardID string) { + m.mode.SetMode(shardID, closedMode) + m.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) +} From 03aa210145fe397cd5a53331adbc8df7aeb8ade4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 16 Jun 2023 10:13:22 +0300 Subject: [PATCH 0356/1943] [#373] metrics: Move labels to consts To unify label naming all lable keys and other consts are moved to one file. Signed-off-by: Dmitrii Stepanov --- pkg/metrics/consts.go | 37 +++++++++++++++++---- pkg/metrics/engine.go | 35 +++++++++----------- pkg/metrics/gc.go | 40 +++++++++-------------- pkg/metrics/innerring.go | 13 ++------ pkg/metrics/mode.go | 10 +++--- pkg/metrics/morph.go | 23 ++++--------- pkg/metrics/morphcache.go | 14 +++----- pkg/metrics/node.go | 2 -- pkg/metrics/object.go | 20 +++--------- pkg/metrics/replicator.go | 2 -- pkg/metrics/state.go | 2 -- pkg/metrics/treeservice.go | 15 ++++----- pkg/metrics/writecache.go | 66 ++++++++++++++++---------------------- 13 files changed, 119 insertions(+), 160 deletions(-) diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index 878e85a35..ae3ac7839 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -1,20 +1,45 @@ package metrics const ( + namespace = "frostfs_node" + innerRingNamespace = "frostfs_ir" + fstreeSubSystem = "fstree" blobstoreSubSystem = "blobstore" blobovnizaTreeSubSystem = "blobovniza_tree" metabaseSubSystem = "metabase" piloramaSubSystem = "pilorama" + engineSubsystem = "engine" + gcSubsystem = "garbage_collector" + innerRingSubsystem = "ir" + morphSubsystem = "morph" + morphCacheSubsystem = "morphcache" + objectSubsystem = "object" + replicatorSubsystem = "replicator" + stateSubsystem = "state" + treeServiceSubsystem = "treeservice" + writeCacheSubsystem = "writecache" - successLabel = "success" - shardIDLabel = "shardID" - modeLabel = "mode" - pathLabel = "path" - methodLabel = "method" - withStorageIDLabel = "withStorageID" + successLabel = "success" + shardIDLabel = "shard_id" + modeLabel = "mode" + pathLabel = "path" + methodLabel = "method" + withStorageIDLabel = "with_storage_id" + statusLabel = "status" + objectTypeLabel = "object_type" + typeLabel = "type" + notificationTypeLabel = "notification_type" + invokeTypeLabel = "invoke_type" + contractLabel = "contract" + containerIDLabelKey = "cid" + storageLabel = "storage" + operationLabel = "operation" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" closedMode = "CLOSED" + + failedToDeleteStatus = "failed_to_delete" + deletedStatus = "deleted" ) diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 395ab3356..23d799e28 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -35,23 +35,18 @@ type engineMetrics struct { writeCache *writeCacheMetrics } -const ( - engineSubsystem = "engine" - engineMethod = "method" -) - func newEngineMetrics() *engineMetrics { return &engineMetrics{ containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), - payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), - errorCounter: newEngineGaugeVector("errors_total", "Shard's error counter", []string{shardIDLabelKey}), + payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabel}), + errorCounter: newEngineGaugeVector("errors_total", "Shard's error counter", []string{shardIDLabel}), methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: "request_duration_seconds", Help: "Duration of Engine requests", - }, []string{engineMethod}), - objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}), + }, []string{methodLabel}), + objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabel, typeLabel}), gc: newGCMetrics(), writeCache: newWriteCacheMetrics(), mode: newShardIDMode(engineSubsystem, "mode_info", "Shard mode"), @@ -69,7 +64,7 @@ func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeV func (m *engineMetrics) AddMethodDuration(method string, d time.Duration) { m.methodDuration.With(prometheus.Labels{ - engineMethod: method, + methodLabel: method, }).Observe(d.Seconds()) } @@ -78,29 +73,29 @@ func (m *engineMetrics) AddToContainerSize(cnrID string, size int64) { } func (m *engineMetrics) AddToPayloadCounter(shardID string, size int64) { - m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) + m.payloadSize.With(prometheus.Labels{shardIDLabel: shardID}).Add(float64(size)) } func (m *engineMetrics) IncErrorCounter(shardID string) { - m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Inc() + m.errorCounter.With(prometheus.Labels{shardIDLabel: shardID}).Inc() } func (m *engineMetrics) ClearErrorCounter(shardID string) { - m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0) + m.errorCounter.With(prometheus.Labels{shardIDLabel: shardID}).Set(0) } func (m *engineMetrics) DeleteShardMetrics(shardID string) { - m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID}) - m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID}) - m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabelKey: shardID}) + m.errorCounter.Delete(prometheus.Labels{shardIDLabel: shardID}) + m.payloadSize.Delete(prometheus.Labels{shardIDLabel: shardID}) + m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) m.mode.Delete(shardID) } func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) { m.objectCounter.With( prometheus.Labels{ - shardIDLabelKey: shardID, - counterTypeLabelKey: objectType, + shardIDLabel: shardID, + typeLabel: objectType, }, ).Add(float64(delta)) } @@ -108,8 +103,8 @@ func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int func (m *engineMetrics) SetObjectCounter(shardID, objectType string, v uint64) { m.objectCounter.With( prometheus.Labels{ - shardIDLabelKey: shardID, - counterTypeLabelKey: objectType, + shardIDLabel: shardID, + typeLabel: objectType, }, ).Set(float64(v)) } diff --git a/pkg/metrics/gc.go b/pkg/metrics/gc.go index c0319562c..53bfef0e5 100644 --- a/pkg/metrics/gc.go +++ b/pkg/metrics/gc.go @@ -8,16 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const ( - gcSubsystem = "garbage_collector" - gcShardID = "shard_id" - gcSuccess = "success" - gcStatus = "status" - gcDeleted = "deleted" - gcFailed = "failed_to_delete" - gcObjectType = "object_type" -) - type GCMetrics interface { AddRunDuration(shardID string, d time.Duration, success bool) AddDeletedCount(shardID string, deleted, failed uint64) @@ -39,60 +29,60 @@ func newGCMetrics() *gcMetrics { Subsystem: gcSubsystem, Name: "delete_duration_seconds", Help: "The total time of GC runs to delete objects from disk", - }, []string{gcShardID, gcSuccess}), + }, []string{shardIDLabel, successLabel}), deletedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "deleted_objects_total", Help: "Total count of objects GC deleted or failed to delete from disk", - }, []string{gcShardID, gcStatus}), + }, []string{shardIDLabel, statusLabel}), expCollectDuration: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "marking_duration_seconds", Help: "The total time of GC runs to mark expired objects as removed", - }, []string{gcShardID, gcSuccess, gcObjectType}), + }, []string{shardIDLabel, successLabel, objectTypeLabel}), inhumedCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: gcSubsystem, Name: "marked_for_removal_objects_total", Help: "Total count of expired objects GC marked to remove", - }, []string{gcShardID, gcObjectType}), + }, []string{shardIDLabel, objectTypeLabel}), } } func (m *gcMetrics) AddRunDuration(shardID string, d time.Duration, success bool) { m.runDuration.With(prometheus.Labels{ - gcShardID: shardID, - gcSuccess: strconv.FormatBool(success), + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), }).Add(d.Seconds()) } func (m *gcMetrics) AddDeletedCount(shardID string, deleted, failed uint64) { m.deletedCounter.With( prometheus.Labels{ - gcShardID: shardID, - gcStatus: gcDeleted, + shardIDLabel: shardID, + statusLabel: deletedStatus, }).Add(float64(deleted)) m.deletedCounter.With( prometheus.Labels{ - gcShardID: shardID, - gcStatus: gcFailed, + shardIDLabel: shardID, + statusLabel: failedToDeleteStatus, }).Add(float64(failed)) } func (m *gcMetrics) AddExpiredObjectCollectionDuration(shardID string, d time.Duration, success bool, objectType string) { m.expCollectDuration.With(prometheus.Labels{ - gcShardID: shardID, - gcSuccess: strconv.FormatBool(success), - gcObjectType: objectType, + shardIDLabel: shardID, + successLabel: strconv.FormatBool(success), + objectTypeLabel: objectType, }).Add(d.Seconds()) } func (m *gcMetrics) AddInhumedObjectCount(shardID string, count uint64, objectType string) { m.inhumedCounter.With( prometheus.Labels{ - gcShardID: shardID, - gcObjectType: objectType, + shardIDLabel: shardID, + objectTypeLabel: objectType, }).Add(float64(count)) } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 0aee068c5..d93b3c432 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -8,13 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const ( - innerRingSubsystem = "ir" - innerRingLabelSuccess = "success" - innerRingLabelType = "type" - innerRingNamespace = "frostfs_ir" -) - // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { epoch prometheus.Gauge @@ -43,7 +36,7 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { Subsystem: innerRingSubsystem, Name: "event_duration_seconds", Help: "Duration of processing of inner-ring events", - }, []string{innerRingLabelType, innerRingLabelSuccess}) + }, []string{typeLabel, successLabel}) ) return &InnerRingServiceMetrics{ @@ -66,8 +59,8 @@ func (m *InnerRingServiceMetrics) SetHealth(s int32) { func (m *InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) { m.eventDuration.With(prometheus.Labels{ - innerRingLabelType: typ, - innerRingLabelSuccess: strconv.FormatBool(success), + typeLabel: typ, + successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) } diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go index ad959e18e..312a6b33d 100644 --- a/pkg/metrics/mode.go +++ b/pkg/metrics/mode.go @@ -17,24 +17,24 @@ func newShardIDMode(subsystem, name, help string) *shardIDModeValue { Subsystem: subsystem, Name: name, Help: help, - }, []string{wcShardID, wcMode}), + }, []string{shardIDLabel, modeLabel}), } } func (m *shardIDModeValue) SetMode(shardID string, mode string) { m.modeValue.DeletePartialMatch(prometheus.Labels{ - wcShardID: shardID, + shardIDLabel: shardID, }) m.modeValue.With(prometheus.Labels{ - wcShardID: shardID, - wcMode: mode, + shardIDLabel: shardID, + modeLabel: mode, }).Set(1) } func (m *shardIDModeValue) Delete(shardID string) { m.modeValue.DeletePartialMatch(prometheus.Labels{ - wcShardID: shardID, + shardIDLabel: shardID, }) } diff --git a/pkg/metrics/morph.go b/pkg/metrics/morph.go index cd5deb5e6..5215c674b 100644 --- a/pkg/metrics/morph.go +++ b/pkg/metrics/morph.go @@ -9,15 +9,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const ( - morphSubsystem = "morph" - morphNotificationTypeLabel = "notification_type" - morphInvokeTypeLabel = "invoke_type" - morphContractLabel = "contract" - morphMethodLabel = "method" - morphSuccessLabel = "success" -) - type morphClientMetrics struct { switchCount prometheus.Counter lastBlock prometheus.Gauge @@ -44,13 +35,13 @@ func NewMorphClientMetrics() morphmetrics.Register { Subsystem: morphSubsystem, Name: "notifications_total", Help: "Number of notifications received by notification type", - }, []string{morphNotificationTypeLabel}), + }, []string{notificationTypeLabel}), invokeDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: morphSubsystem, Name: "invoke_duration_seconds", Help: "Cummulative duration of contract invocations", - }, []string{morphInvokeTypeLabel, morphContractLabel, morphMethodLabel, morphSuccessLabel}), + }, []string{invokeTypeLabel, contractLabel, methodLabel, successLabel}), } } @@ -65,7 +56,7 @@ func (m *morphClientMetrics) SetLastBlock(index uint32) { func (m *morphClientMetrics) IncNotificationCount(typ string) { m.notificationCount.With( prometheus.Labels{ - morphNotificationTypeLabel: typ, + notificationTypeLabel: typ, }, ).Inc() } @@ -73,10 +64,10 @@ func (m *morphClientMetrics) IncNotificationCount(typ string) { func (m *morphClientMetrics) ObserveInvoke(typ string, contract string, method string, success bool, d time.Duration) { m.invokeDuration.With( prometheus.Labels{ - morphInvokeTypeLabel: typ, - morphContractLabel: contract, - morphMethodLabel: method, - morphSuccessLabel: strconv.FormatBool(success), + invokeTypeLabel: typ, + contractLabel: contract, + methodLabel: method, + successLabel: strconv.FormatBool(success), }, ).Observe(d.Seconds()) } diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go index 3f215b5bf..a4dbbccfc 100644 --- a/pkg/metrics/morphcache.go +++ b/pkg/metrics/morphcache.go @@ -8,12 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const ( - mcSubsystem = "morphcache" - mcSuccess = "success" - mcMethod = "method" -) - type MorphCacheMetrics interface { AddMethodDuration(method string, success bool, d time.Duration) } @@ -32,18 +26,18 @@ func newMorphCacheMetrics(ns string) *morphCacheMetrics { return &morphCacheMetrics{ methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: ns, - Subsystem: mcSubsystem, + Subsystem: morphCacheSubsystem, Name: "request_duration_seconds", Help: "Morph cache request process duration", - }, []string{mcSuccess, mcMethod}), + }, []string{successLabel, methodLabel}), } } func (m *morphCacheMetrics) AddMethodDuration(method string, success bool, d time.Duration) { m.methodDuration.With( prometheus.Labels{ - mcSuccess: strconv.FormatBool(success), - mcMethod: method, + successLabel: strconv.FormatBool(success), + methodLabel: method, }, ).Observe(d.Seconds()) } diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 3d3712450..45d50b5b0 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -5,8 +5,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const namespace = "frostfs_node" - type NodeMetrics struct { engine *engineMetrics state *stateMetrics diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index 6e1b3a5c4..0ba994ed3 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -8,8 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const objectSubsystem = "object" - type ObjectServiceMetrics interface { AddRequestDuration(method string, d time.Duration, success bool) AddPayloadSize(method string, size int) @@ -20,14 +18,6 @@ type objectServiceMetrics struct { payloadCounter *prometheus.CounterVec } -const ( - shardIDLabelKey = "shard" - counterTypeLabelKey = "type" - containerIDLabelKey = "cid" - methodLabelKey = "method" - successLabelKey = "success" -) - func newObjectServiceMetrics() *objectServiceMetrics { return &objectServiceMetrics{ methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ @@ -35,25 +25,25 @@ func newObjectServiceMetrics() *objectServiceMetrics { Subsystem: objectSubsystem, Name: "request_duration_seconds", Help: "Object Service request process duration", - }, []string{methodLabelKey, successLabelKey}), + }, []string{methodLabel, successLabel}), payloadCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: "request_payload_bytes", Help: "Object Service request payload", - }, []string{methodLabelKey}), + }, []string{methodLabel}), } } func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool) { m.methodDuration.With(prometheus.Labels{ - methodLabelKey: method, - successLabelKey: strconv.FormatBool(success), + methodLabel: method, + successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) } func (m *objectServiceMetrics) AddPayloadSize(method string, size int) { m.payloadCounter.With(prometheus.Labels{ - methodLabelKey: method, + methodLabel: method, }).Add(float64(size)) } diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go index f16c296c6..a1519ac95 100644 --- a/pkg/metrics/replicator.go +++ b/pkg/metrics/replicator.go @@ -5,8 +5,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const replicatorSubsystem = "replicator" - //TODO type ReplicatorMetrics interface { diff --git a/pkg/metrics/state.go b/pkg/metrics/state.go index 76a8d8ac9..243f648e5 100644 --- a/pkg/metrics/state.go +++ b/pkg/metrics/state.go @@ -5,8 +5,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const stateSubsystem = "state" - type StateMetrics interface { SetHealth(s int32) } diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go index ae24c41b2..6702aa83c 100644 --- a/pkg/metrics/treeservice.go +++ b/pkg/metrics/treeservice.go @@ -8,8 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const treeServiceLabelSuccess = "success" - type TreeMetricsRegister interface { AddReplicateTaskDuration(time.Duration, bool) AddReplicateWaitDuration(time.Duration, bool) @@ -25,43 +23,42 @@ type treeServiceMetrics struct { var _ TreeMetricsRegister = (*treeServiceMetrics)(nil) func newTreeServiceMetrics() *treeServiceMetrics { - const treeServiceSubsystem = "treeservice" return &treeServiceMetrics{ replicateTaskDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "replicate_task_duration_seconds", Help: "Duration of individual replication tasks executed as part of replication loops", - }, []string{treeServiceLabelSuccess}), + }, []string{successLabel}), replicateWaitDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "replicate_wait_duration_seconds", Help: "Duration of overall waiting time for replication loops", - }, []string{treeServiceLabelSuccess}), + }, []string{successLabel}), syncOpDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: treeServiceSubsystem, Name: "sync_duration_seconds", Help: "Duration of synchronization operations", - }, []string{treeServiceLabelSuccess}), + }, []string{successLabel}), } } func (m *treeServiceMetrics) AddReplicateTaskDuration(d time.Duration, success bool) { m.replicateTaskDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: strconv.FormatBool(success), + successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddReplicateWaitDuration(d time.Duration, success bool) { m.replicateWaitDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: strconv.FormatBool(success), + successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) } func (m *treeServiceMetrics) AddSyncDuration(d time.Duration, success bool) { m.syncOpDuration.With(prometheus.Labels{ - treeServiceLabelSuccess: strconv.FormatBool(success), + successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index a9c1b33a7..7e6083a49 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -8,16 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const ( - wcSubsystem = "writecache" - wcShardID = "shard_id" - wcSuccess = "success" - wcStorage = "storage" - wcMode = "mode" - wcMethod = "method" - wcOperation = "operation" -) - type WriteCacheMetrics interface { AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) @@ -48,58 +38,58 @@ func newWriteCacheMetrics() *writeCacheMetrics { return &writeCacheMetrics{ methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, - Subsystem: wcSubsystem, + Subsystem: writeCacheSubsystem, Name: "request_duration_seconds", Help: "Writecache request process duration", - }, []string{wcShardID, wcSuccess, wcStorage, wcMethod}), + }, []string{shardIDLabel, successLabel, storageLabel, methodLabel}), operationCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, - Subsystem: wcSubsystem, + Subsystem: writeCacheSubsystem, Name: "operations_total", Help: "The number of writecache operations processed", - }, []string{wcShardID, wcStorage, wcSuccess, wcOperation}), - actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{wcShardID, wcStorage}), - estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}), - mode: newShardIDMode(wcSubsystem, "mode_info", "Writecache mode value"), + }, []string{shardIDLabel, storageLabel, successLabel, operationLabel}), + actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{shardIDLabel, storageLabel}), + estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{shardIDLabel, storageLabel}), + mode: newShardIDMode(writeCacheSubsystem, "mode_info", "Writecache mode value"), } } func (m *writeCacheMetrics) AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) { m.methodDuration.With( prometheus.Labels{ - wcShardID: shardID, - wcSuccess: fmt.Sprintf("%v", success), - wcStorage: storageType, - wcMethod: method, + shardIDLabel: shardID, + successLabel: fmt.Sprintf("%v", success), + storageLabel: storageType, + methodLabel: method, }, ).Observe(d.Seconds()) } func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { m.actualCount.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, + shardIDLabel: shardID, + storageLabel: storageType, }).Inc() } func (m *writeCacheMetrics) DecActualCount(shardID string, storageType string) { m.actualCount.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, + shardIDLabel: shardID, + storageLabel: storageType, }).Dec() } func (m *writeCacheMetrics) SetActualCount(shardID string, count uint64, storageType string) { m.actualCount.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, + shardIDLabel: shardID, + storageLabel: storageType, }).Set(float64(count)) } func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storageType string) { m.estimatedSize.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, + shardIDLabel: shardID, + storageLabel: storageType, }).Set(float64(size)) } @@ -109,25 +99,25 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) { m.operationCounter.With(prometheus.Labels{ - wcShardID: shardID, - wcStorage: storageType, - wcOperation: operation, - wcSuccess: success.String(), + shardIDLabel: shardID, + storageLabel: storageType, + operationLabel: operation, + successLabel: success.String(), }).Inc() } func (m *writeCacheMetrics) Close(shardID string) { m.mode.Delete(shardID) - m.methodDuration.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) - m.operationCounter.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) - m.actualCount.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) - m.estimatedSize.DeletePartialMatch(prometheus.Labels{wcShardID: shardID}) + m.methodDuration.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.operationCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.actualCount.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.estimatedSize.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) } func newWCGaugeVec(name, help string, labels []string) *prometheus.GaugeVec { return metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, - Subsystem: wcSubsystem, + Subsystem: writeCacheSubsystem, Name: name, Help: help, }, labels) From 4d48377cec9f78046e1f2bb37def07be3e98e890 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 21 Jun 2023 16:42:58 +0300 Subject: [PATCH 0357/1943] [#459] blobovniczatree: Fix get error Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobstor/blobovniczatree/get.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 5dafedd1c..e88310f8a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -57,6 +57,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G success = true size = len(res.RawData) } + return res, err } activeCache := make(map[string]struct{}) From 785d81a68ae347aa6d991132d3da60b71bed479d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 19 Jun 2023 13:52:26 +0300 Subject: [PATCH 0358/1943] [#460] services/object: Reduce `distibutedTarget` memory footprint Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/distributed.go | 11 ++--------- pkg/services/object/put/streamer.go | 15 ++++++++++----- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index b24218621..51678fa7b 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -25,8 +25,6 @@ type preparedObjectTarget interface { type distributedTarget struct { traversal traversal - remotePool, localPool util.WorkerPool - obj *objectSDK.Object objMeta object.ContentMeta @@ -34,7 +32,7 @@ type distributedTarget struct { nodeTargetInitializer func(nodeDesc) preparedObjectTarget - isLocalKey func([]byte) bool + getWorkerPool func([]byte) (util.WorkerPool, bool) relay func(context.Context, nodeDesc) error @@ -223,13 +221,8 @@ func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *pla wg.Add(1) addr := addrs[i] - isLocal := t.isLocalKey(addr.PublicKey()) - - workerPool := t.remotePool - if isLocal { - workerPool = t.localPool - } + workerPool, isLocal := t.getWorkerPool(addr.PublicKey()) if err := workerPool.Submit(func() { defer wg.Done() diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 6d0d8062e..8be9c75a0 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" @@ -219,9 +220,8 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { extraBroadcastEnabled: withBroadcast, }, - payload: getPayload(), - remotePool: p.remotePool, - localPool: p.localPool, + payload: getPayload(), + getWorkerPool: p.getWorkerPool, nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { return &localTarget{ @@ -242,8 +242,6 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { relay: relay, fmt: p.fmtValidator, log: p.log, - - isLocalKey: p.netmapKeys.IsLocalKey, } } @@ -280,3 +278,10 @@ func (p *Streamer) Close(ctx context.Context) (*PutResponse, error) { id: ids.SelfID, }, nil } + +func (p *Streamer) getWorkerPool(pub []byte) (pkgutil.WorkerPool, bool) { + if p.netmapKeys.IsLocalKey(pub) { + return p.localPool, true + } + return p.remotePool, false +} From 167a67f0b846cdcfb5405f5bc6f3d13057e3161f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 17:48:50 +0300 Subject: [PATCH 0359/1943] [#460] services/util: Remove `HandleUnaryRequest` There is no need in a wrapper with many from-`interface{}` conversions. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/accounting.go | 6 +- cmd/frostfs-node/container.go | 19 ++-- cmd/frostfs-node/netmap.go | 20 ++-- cmd/frostfs-node/session.go | 5 +- pkg/services/accounting/executor.go | 10 +- pkg/services/accounting/response.go | 37 ------- pkg/services/container/executor.go | 14 ++- pkg/services/container/response.go | 115 -------------------- pkg/services/netmap/executor.go | 11 +- pkg/services/netmap/response.go | 63 ----------- pkg/services/object/response.go | 27 ++--- pkg/services/session/executor.go | 11 +- pkg/services/session/response.go | 37 ------- pkg/services/util/response/client_stream.go | 6 +- pkg/services/util/response/server_stream.go | 6 +- pkg/services/util/response/service.go | 7 +- pkg/services/util/response/unary.go | 21 ---- 17 files changed, 76 insertions(+), 339 deletions(-) delete mode 100644 pkg/services/accounting/response.go delete mode 100644 pkg/services/container/response.go delete mode 100644 pkg/services/netmap/response.go delete mode 100644 pkg/services/session/response.go delete mode 100644 pkg/services/util/response/unary.go diff --git a/cmd/frostfs-node/accounting.go b/cmd/frostfs-node/accounting.go index 6a35f37d0..d04f34ff1 100644 --- a/cmd/frostfs-node/accounting.go +++ b/cmd/frostfs-node/accounting.go @@ -21,10 +21,8 @@ func initAccountingService(ctx context.Context, c *cfg) { server := accountingTransportGRPC.New( accountingService.NewSignService( &c.key.PrivateKey, - accountingService.NewResponseService( - accountingService.NewExecutionService( - accounting.NewExecutor(balanceMorphWrapper), - ), + accountingService.NewExecutionService( + accounting.NewExecutor(balanceMorphWrapper), c.respSvc, ), ), diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index d5a5afce1..9a6cfd02b 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -83,16 +83,13 @@ func initContainerService(ctx context.Context, c *cfg) { server := containerTransportGRPC.New( containerService.NewSignService( &c.key.PrivateKey, - containerService.NewResponseService( - &usedSpaceService{ - Server: containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt)), - loadWriterProvider: loadRouter, - loadPlacementBuilder: loadPlacementBuilder, - routeBuilder: routeBuilder, - cfg: c, - }, - c.respSvc, - ), + &usedSpaceService{ + Server: containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), + loadWriterProvider: loadRouter, + loadPlacementBuilder: loadPlacementBuilder, + routeBuilder: routeBuilder, + cfg: c, + }, ), ) @@ -575,6 +572,8 @@ func (c *usedSpaceService) AnnounceUsedSpace(ctx context.Context, req *container resp := new(containerV2.AnnounceUsedSpaceResponse) resp.SetBody(respBody) + c.cfg.respSvc.SetMeta(resp) + return resp, nil } diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index f1ea8b40e..b7b09a00b 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -148,17 +148,15 @@ func initNetmapService(ctx context.Context, c *cfg) { server := netmapTransportGRPC.New( netmapService.NewSignService( &c.key.PrivateKey, - netmapService.NewResponseService( - netmapService.NewExecutionService( - c, - c.apiVersion, - &netInfo{ - netState: c.cfgNetmap.state, - magic: c.cfgMorph.client, - morphClientNetMap: c.cfgNetmap.wrapper, - msPerBlockRdr: c.cfgMorph.client.MsPerBlock, - }, - ), + netmapService.NewExecutionService( + c, + c.apiVersion, + &netInfo{ + netState: c.cfgNetmap.state, + magic: c.cfgMorph.client, + morphClientNetMap: c.cfgNetmap.wrapper, + msPerBlockRdr: c.cfgMorph.client.MsPerBlock, + }, c.respSvc, ), ), diff --git a/cmd/frostfs-node/session.go b/cmd/frostfs-node/session.go index 95e3b8205..f9c1811a1 100644 --- a/cmd/frostfs-node/session.go +++ b/cmd/frostfs-node/session.go @@ -53,10 +53,7 @@ func initSessionService(c *cfg) { server := sessionTransportGRPC.New( sessionSvc.NewSignService( &c.key.PrivateKey, - sessionSvc.NewResponseService( - sessionSvc.NewExecutionService(c.privateTokenStore, c.log), - c.respSvc, - ), + sessionSvc.NewExecutionService(c.privateTokenStore, c.respSvc, c.log), ), ) diff --git a/pkg/services/accounting/executor.go b/pkg/services/accounting/executor.go index 402845957..b0722cf8a 100644 --- a/pkg/services/accounting/executor.go +++ b/pkg/services/accounting/executor.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" ) type ServiceExecutor interface { @@ -12,13 +13,15 @@ type ServiceExecutor interface { } type executorSvc struct { - exec ServiceExecutor + exec ServiceExecutor + respSvc *response.Service } // NewExecutionService wraps ServiceExecutor and returns Accounting Service interface. -func NewExecutionService(exec ServiceExecutor) Server { +func NewExecutionService(exec ServiceExecutor, respSvc *response.Service) Server { return &executorSvc{ - exec: exec, + exec: exec, + respSvc: respSvc, } } @@ -31,5 +34,6 @@ func (s *executorSvc) Balance(ctx context.Context, req *accounting.BalanceReques resp := new(accounting.BalanceResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/accounting/response.go b/pkg/services/accounting/response.go deleted file mode 100644 index a78ac6fd6..000000000 --- a/pkg/services/accounting/response.go +++ /dev/null @@ -1,37 +0,0 @@ -package accounting - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns accounting service instance that passes internal service -// call to response service. -func NewResponseService(accSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: accSvc, - } -} - -func (s *responseService) Balance(ctx context.Context, req *accounting.BalanceRequest) (*accounting.BalanceResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Balance(ctx, req.(*accounting.BalanceRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*accounting.BalanceResponse), nil -} diff --git a/pkg/services/container/executor.go b/pkg/services/container/executor.go index b4705d258..d4ae11d62 100644 --- a/pkg/services/container/executor.go +++ b/pkg/services/container/executor.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" ) type ServiceExecutor interface { @@ -21,12 +22,15 @@ type executorSvc struct { Server exec ServiceExecutor + + respSvc *response.Service } // NewExecutionService wraps ServiceExecutor and returns Container Service interface. -func NewExecutionService(exec ServiceExecutor) Server { +func NewExecutionService(exec ServiceExecutor, respSvc *response.Service) Server { return &executorSvc{ - exec: exec, + exec: exec, + respSvc: respSvc, } } @@ -44,6 +48,7 @@ func (s *executorSvc) Put(ctx context.Context, req *container.PutRequest) (*cont resp := new(container.PutResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -61,6 +66,7 @@ func (s *executorSvc) Delete(ctx context.Context, req *container.DeleteRequest) resp := new(container.DeleteResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -73,6 +79,7 @@ func (s *executorSvc) Get(ctx context.Context, req *container.GetRequest) (*cont resp := new(container.GetResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -85,6 +92,7 @@ func (s *executorSvc) List(ctx context.Context, req *container.ListRequest) (*co resp := new(container.ListResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -102,6 +110,7 @@ func (s *executorSvc) SetExtendedACL(ctx context.Context, req *container.SetExte resp := new(container.SetExtendedACLResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } @@ -114,5 +123,6 @@ func (s *executorSvc) GetExtendedACL(ctx context.Context, req *container.GetExte resp := new(container.GetExtendedACLResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/container/response.go b/pkg/services/container/response.go deleted file mode 100644 index 138974537..000000000 --- a/pkg/services/container/response.go +++ /dev/null @@ -1,115 +0,0 @@ -package container - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns container service instance that passes internal service -// call to response service. -func NewResponseService(cnrSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: cnrSvc, - } -} - -func (s *responseService) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Put(ctx, req.(*container.PutRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.PutResponse), nil -} - -func (s *responseService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*container.DeleteRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.DeleteResponse), nil -} - -func (s *responseService) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Get(ctx, req.(*container.GetRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.GetResponse), nil -} - -func (s *responseService) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.List(ctx, req.(*container.ListRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.ListResponse), nil -} - -func (s *responseService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.SetExtendedACL(ctx, req.(*container.SetExtendedACLRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.SetExtendedACLResponse), nil -} - -func (s *responseService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetExtendedACL(ctx, req.(*container.GetExtendedACLRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.GetExtendedACLResponse), nil -} - -func (s *responseService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.AnnounceUsedSpace(ctx, req.(*container.AnnounceUsedSpaceRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*container.AnnounceUsedSpaceResponse), nil -} diff --git a/pkg/services/netmap/executor.go b/pkg/services/netmap/executor.go index d77c69d4d..d1e7a949e 100644 --- a/pkg/services/netmap/executor.go +++ b/pkg/services/netmap/executor.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/version" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" versionsdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" ) @@ -18,6 +19,8 @@ type executorSvc struct { state NodeState netInfo NetworkInfo + + respSvc *response.Service } // NodeState encapsulates information @@ -42,8 +45,8 @@ type NetworkInfo interface { Dump(versionsdk.Version) (*netmapSDK.NetworkInfo, error) } -func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo) Server { - if s == nil || netInfo == nil || !version.IsValid(v) { +func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo, respSvc *response.Service) Server { + if s == nil || netInfo == nil || !version.IsValid(v) || respSvc == nil { // this should never happen, otherwise it programmers bug panic("can't create netmap execution service") } @@ -51,6 +54,7 @@ func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo) res := &executorSvc{ state: s, netInfo: netInfo, + respSvc: respSvc, } v.WriteToV2(&res.version) @@ -96,6 +100,7 @@ func (s *executorSvc) LocalNodeInfo( resp := new(netmap.LocalNodeInfoResponse) resp.SetBody(body) + s.respSvc.SetMeta(resp) return resp, nil } @@ -126,6 +131,7 @@ func (s *executorSvc) NetworkInfo( resp := new(netmap.NetworkInfoResponse) resp.SetBody(body) + s.respSvc.SetMeta(resp) return resp, nil } @@ -143,5 +149,6 @@ func (s *executorSvc) Snapshot(_ context.Context, _ *netmap.SnapshotRequest) (*n resp := new(netmap.SnapshotResponse) resp.SetBody(body) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/netmap/response.go b/pkg/services/netmap/response.go deleted file mode 100644 index 8b035e461..000000000 --- a/pkg/services/netmap/response.go +++ /dev/null @@ -1,63 +0,0 @@ -package netmap - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns netmap service instance that passes internal service -// call to response service. -func NewResponseService(nmSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: nmSvc, - } -} - -func (s *responseService) LocalNodeInfo(ctx context.Context, req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.LocalNodeInfo(ctx, req.(*netmap.LocalNodeInfoRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*netmap.LocalNodeInfoResponse), nil -} - -func (s *responseService) NetworkInfo(ctx context.Context, req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.NetworkInfo(ctx, req.(*netmap.NetworkInfoRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*netmap.NetworkInfoResponse), nil -} - -func (s *responseService) Snapshot(ctx context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Snapshot(ctx, req.(*netmap.SnapshotRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*netmap.SnapshotResponse), nil -} diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index def934ea6..4eef1dfb9 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -91,16 +91,13 @@ func (s *ResponseService) Put() (PutObjectStream, error) { } func (s *ResponseService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Head(ctx, req.(*object.HeadRequest)) - }, - ) + resp, err := s.svc.Head(ctx, req) if err != nil { return nil, err } - return resp.(*object.HeadResponse), nil + s.respSvc.SetMeta(resp) + return resp, nil } func (s *searchStreamResponser) Send(resp *object.SearchResponse) error { @@ -117,16 +114,13 @@ func (s *ResponseService) Search(req *object.SearchRequest, stream SearchStream) } func (s *ResponseService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*object.DeleteRequest)) - }, - ) + resp, err := s.svc.Delete(ctx, req) if err != nil { return nil, err } - return resp.(*object.DeleteResponse), nil + s.respSvc.SetMeta(resp) + return resp, nil } func (s *getRangeStreamResponser) Send(resp *object.GetRangeResponse) error { @@ -143,14 +137,11 @@ func (s *ResponseService) GetRange(req *object.GetRangeRequest, stream GetObject } func (s *ResponseService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetRangeHash(ctx, req.(*object.GetRangeHashRequest)) - }, - ) + resp, err := s.svc.GetRangeHash(ctx, req) if err != nil { return nil, err } - return resp.(*object.GetRangeHashResponse), nil + s.respSvc.SetMeta(resp) + return resp, nil } diff --git a/pkg/services/session/executor.go b/pkg/services/session/executor.go index 5ad1d6518..76c220fab 100644 --- a/pkg/services/session/executor.go +++ b/pkg/services/session/executor.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -17,14 +18,17 @@ type ServiceExecutor interface { type executorSvc struct { exec ServiceExecutor + respSvc *response.Service + log *logger.Logger } // NewExecutionService wraps ServiceExecutor and returns Session Service interface. -func NewExecutionService(exec ServiceExecutor, l *logger.Logger) Server { +func NewExecutionService(exec ServiceExecutor, respSvc *response.Service, l *logger.Logger) Server { return &executorSvc{ - exec: exec, - log: l, + exec: exec, + log: l, + respSvc: respSvc, } } @@ -42,5 +46,6 @@ func (s *executorSvc) Create(ctx context.Context, req *session.CreateRequest) (* resp := new(session.CreateResponse) resp.SetBody(respBody) + s.respSvc.SetMeta(resp) return resp, nil } diff --git a/pkg/services/session/response.go b/pkg/services/session/response.go deleted file mode 100644 index cbf93fb1f..000000000 --- a/pkg/services/session/response.go +++ /dev/null @@ -1,37 +0,0 @@ -package session - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" -) - -type responseService struct { - respSvc *response.Service - - svc Server -} - -// NewResponseService returns session service instance that passes internal service -// call to response service. -func NewResponseService(ssSvc Server, respSvc *response.Service) Server { - return &responseService{ - respSvc: respSvc, - svc: ssSvc, - } -} - -func (s *responseService) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - resp, err := s.respSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Create(ctx, req.(*session.CreateRequest)) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*session.CreateResponse), nil -} diff --git a/pkg/services/util/response/client_stream.go b/pkg/services/util/response/client_stream.go index b541c73db..962f2bc26 100644 --- a/pkg/services/util/response/client_stream.go +++ b/pkg/services/util/response/client_stream.go @@ -10,7 +10,7 @@ import ( // ClientMessageStreamer represents client-side message streamer // that sets meta values to the response. type ClientMessageStreamer struct { - cfg *cfg + srv *Service send util.RequestMessageWriter @@ -33,7 +33,7 @@ func (s *ClientMessageStreamer) CloseAndRecv(ctx context.Context) (util.Response return nil, fmt.Errorf("(%T) could not close stream and receive response: %w", s, err) } - setMeta(resp, s.cfg) + s.srv.SetMeta(resp) return resp, nil } @@ -41,7 +41,7 @@ func (s *ClientMessageStreamer) CloseAndRecv(ctx context.Context) (util.Response // CreateRequestStreamer wraps stream methods and returns ClientMessageStreamer instance. func (s *Service) CreateRequestStreamer(sender util.RequestMessageWriter, closer util.ClientStreamCloser) *ClientMessageStreamer { return &ClientMessageStreamer{ - cfg: s.cfg, + srv: s, send: sender, close: closer, } diff --git a/pkg/services/util/response/server_stream.go b/pkg/services/util/response/server_stream.go index 8a19fc4e7..df1c00177 100644 --- a/pkg/services/util/response/server_stream.go +++ b/pkg/services/util/response/server_stream.go @@ -9,7 +9,7 @@ import ( // ServerMessageStreamer represents server-side message streamer // that sets meta values to all response messages. type ServerMessageStreamer struct { - cfg *cfg + srv *Service recv util.ResponseMessageReader } @@ -22,7 +22,7 @@ func (s *ServerMessageStreamer) Recv() (util.ResponseMessage, error) { return nil, fmt.Errorf("could not receive response message for signing: %w", err) } - setMeta(m, s.cfg) + s.srv.SetMeta(m) return m, nil } @@ -30,7 +30,7 @@ func (s *ServerMessageStreamer) Recv() (util.ResponseMessage, error) { // HandleServerStreamRequest builds internal streamer via handlers, wraps it to ServerMessageStreamer and returns the result. func (s *Service) HandleServerStreamRequest(respWriter util.ResponseMessageWriter) util.ResponseMessageWriter { return func(resp util.ResponseMessage) error { - setMeta(resp, s.cfg) + s.SetMeta(resp) return respWriter(resp) } diff --git a/pkg/services/util/response/service.go b/pkg/services/util/response/service.go index 87cc8383d..a63d5343b 100644 --- a/pkg/services/util/response/service.go +++ b/pkg/services/util/response/service.go @@ -44,11 +44,12 @@ func NewService(opts ...Option) *Service { } } -func setMeta(resp util.ResponseMessage, cfg *cfg) { +// SetMeta sets adds meta-header to resp. +func (s *Service) SetMeta(resp util.ResponseMessage) { meta := new(session.ResponseMetaHeader) - meta.SetVersion(&cfg.version) + meta.SetVersion(&s.cfg.version) meta.SetTTL(1) // FIXME: #1160 TTL must be calculated - meta.SetEpoch(cfg.state.CurrentEpoch()) + meta.SetEpoch(s.cfg.state.CurrentEpoch()) if origin := resp.GetMetaHeader(); origin != nil { // FIXME: #1160 what if origin is set by local server? diff --git a/pkg/services/util/response/unary.go b/pkg/services/util/response/unary.go deleted file mode 100644 index 29cb95314..000000000 --- a/pkg/services/util/response/unary.go +++ /dev/null @@ -1,21 +0,0 @@ -package response - -import ( - "context" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" -) - -// HandleUnaryRequest call passes request to handler, sets response meta header values and returns it. -func (s *Service) HandleUnaryRequest(ctx context.Context, req any, handler util.UnaryHandler) (util.ResponseMessage, error) { - // process request - resp, err := handler(ctx, req) - if err != nil { - return nil, fmt.Errorf("could not handle request: %w", err) - } - - setMeta(resp, s.cfg) - - return resp, nil -} From a64dc9ad7083e2021b202aefffe02d48ce221b5d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 17:55:03 +0300 Subject: [PATCH 0360/1943] [#460] services/util: Remove `HandleServerStreamRequest` There is no need in a wrapper with many from-`interface{}` conversions. Signed-off-by: Evgenii Stratonikov --- pkg/services/object/response.go | 39 ++++++++++----------- pkg/services/util/response/server_stream.go | 37 ------------------- 2 files changed, 18 insertions(+), 58 deletions(-) delete mode 100644 pkg/services/util/response/server_stream.go diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index 4eef1dfb9..0bec72163 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -16,21 +16,21 @@ type ResponseService struct { } type searchStreamResponser struct { - util.ServerStream + SearchStream - respWriter util.ResponseMessageWriter + respSvc *response.Service } type getStreamResponser struct { - util.ServerStream + GetObjectStream - respWriter util.ResponseMessageWriter + respSvc *response.Service } type getRangeStreamResponser struct { - util.ServerStream + GetObjectRangeStream - respWriter util.ResponseMessageWriter + respSvc *response.Service } type putStreamResponser struct { @@ -47,15 +47,14 @@ func NewResponseService(objSvc ServiceServer, respSvc *response.Service) *Respon } func (s *getStreamResponser) Send(resp *object.GetResponse) error { - return s.respWriter(resp) + s.respSvc.SetMeta(resp) + return s.GetObjectStream.Send(resp) } func (s *ResponseService) Get(req *object.GetRequest, stream GetObjectStream) error { return s.svc.Get(req, &getStreamResponser{ - ServerStream: stream, - respWriter: s.respSvc.HandleServerStreamRequest(func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetResponse)) - }), + GetObjectStream: stream, + respSvc: s.respSvc, }) } @@ -101,15 +100,14 @@ func (s *ResponseService) Head(ctx context.Context, req *object.HeadRequest) (*o } func (s *searchStreamResponser) Send(resp *object.SearchResponse) error { - return s.respWriter(resp) + s.respSvc.SetMeta(resp) + return s.SearchStream.Send(resp) } func (s *ResponseService) Search(req *object.SearchRequest, stream SearchStream) error { return s.svc.Search(req, &searchStreamResponser{ - ServerStream: stream, - respWriter: s.respSvc.HandleServerStreamRequest(func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.SearchResponse)) - }), + SearchStream: stream, + respSvc: s.respSvc, }) } @@ -124,15 +122,14 @@ func (s *ResponseService) Delete(ctx context.Context, req *object.DeleteRequest) } func (s *getRangeStreamResponser) Send(resp *object.GetRangeResponse) error { - return s.respWriter(resp) + s.respSvc.SetMeta(resp) + return s.GetObjectRangeStream.Send(resp) } func (s *ResponseService) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { return s.svc.GetRange(req, &getRangeStreamResponser{ - ServerStream: stream, - respWriter: s.respSvc.HandleServerStreamRequest(func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetRangeResponse)) - }), + GetObjectRangeStream: stream, + respSvc: s.respSvc, }) } diff --git a/pkg/services/util/response/server_stream.go b/pkg/services/util/response/server_stream.go deleted file mode 100644 index df1c00177..000000000 --- a/pkg/services/util/response/server_stream.go +++ /dev/null @@ -1,37 +0,0 @@ -package response - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" -) - -// ServerMessageStreamer represents server-side message streamer -// that sets meta values to all response messages. -type ServerMessageStreamer struct { - srv *Service - - recv util.ResponseMessageReader -} - -// Recv calls Recv method of internal streamer, sets response meta -// values and returns the response. -func (s *ServerMessageStreamer) Recv() (util.ResponseMessage, error) { - m, err := s.recv() - if err != nil { - return nil, fmt.Errorf("could not receive response message for signing: %w", err) - } - - s.srv.SetMeta(m) - - return m, nil -} - -// HandleServerStreamRequest builds internal streamer via handlers, wraps it to ServerMessageStreamer and returns the result. -func (s *Service) HandleServerStreamRequest(respWriter util.ResponseMessageWriter) util.ResponseMessageWriter { - return func(resp util.ResponseMessage) error { - s.SetMeta(resp) - - return respWriter(resp) - } -} From 40eae221099b5c4bd872ab4331d22cae3a92f364 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 18:01:51 +0300 Subject: [PATCH 0361/1943] [#460] services/util: Remove `CreateRequestStreamer` There is no need in a wrapper with many from-`interface{}` conversions. Signed-off-by: Evgenii Stratonikov --- pkg/services/object/response.go | 24 +++++------ pkg/services/util/response/client_stream.go | 48 --------------------- 2 files changed, 11 insertions(+), 61 deletions(-) delete mode 100644 pkg/services/util/response/client_stream.go diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index 0bec72163..81d6aaaaf 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -5,7 +5,6 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" ) @@ -34,7 +33,8 @@ type getRangeStreamResponser struct { } type putStreamResponser struct { - stream *response.ClientMessageStreamer + stream PutObjectStream + respSvc *response.Service } // NewResponseService returns object service instance that passes internal service @@ -59,16 +59,20 @@ func (s *ResponseService) Get(req *object.GetRequest, stream GetObjectStream) er } func (s *putStreamResponser) Send(ctx context.Context, req *object.PutRequest) error { - return s.stream.Send(ctx, req) + if err := s.stream.Send(ctx, req); err != nil { + return fmt.Errorf("could not send the request: %w", err) + } + return nil } func (s *putStreamResponser) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { r, err := s.stream.CloseAndRecv(ctx) if err != nil { - return nil, fmt.Errorf("(%T) could not receive response: %w", s, err) + return nil, fmt.Errorf("could not close stream and receive response: %w", err) } - return r.(*object.PutResponse), nil + s.respSvc.SetMeta(r) + return r, nil } func (s *ResponseService) Put() (PutObjectStream, error) { @@ -78,14 +82,8 @@ func (s *ResponseService) Put() (PutObjectStream, error) { } return &putStreamResponser{ - stream: s.respSvc.CreateRequestStreamer( - func(ctx context.Context, req any) error { - return stream.Send(ctx, req.(*object.PutRequest)) - }, - func(ctx context.Context) (util.ResponseMessage, error) { - return stream.CloseAndRecv(ctx) - }, - ), + stream: stream, + respSvc: s.respSvc, }, nil } diff --git a/pkg/services/util/response/client_stream.go b/pkg/services/util/response/client_stream.go deleted file mode 100644 index 962f2bc26..000000000 --- a/pkg/services/util/response/client_stream.go +++ /dev/null @@ -1,48 +0,0 @@ -package response - -import ( - "context" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" -) - -// ClientMessageStreamer represents client-side message streamer -// that sets meta values to the response. -type ClientMessageStreamer struct { - srv *Service - - send util.RequestMessageWriter - - close util.ClientStreamCloser -} - -// Send calls send method of internal streamer. -func (s *ClientMessageStreamer) Send(ctx context.Context, req any) error { - if err := s.send(ctx, req); err != nil { - return fmt.Errorf("(%T) could not send the request: %w", s, err) - } - return nil -} - -// CloseAndRecv closes internal stream, receivers the response, -// sets meta values and returns the result. -func (s *ClientMessageStreamer) CloseAndRecv(ctx context.Context) (util.ResponseMessage, error) { - resp, err := s.close(ctx) - if err != nil { - return nil, fmt.Errorf("(%T) could not close stream and receive response: %w", s, err) - } - - s.srv.SetMeta(resp) - - return resp, nil -} - -// CreateRequestStreamer wraps stream methods and returns ClientMessageStreamer instance. -func (s *Service) CreateRequestStreamer(sender util.RequestMessageWriter, closer util.ClientStreamCloser) *ClientMessageStreamer { - return &ClientMessageStreamer{ - srv: s, - send: sender, - close: closer, - } -} From 71889234b772297fbc5b016d1d6ec384b6ae8d21 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 23 Jun 2023 11:25:08 +0300 Subject: [PATCH 0362/1943] [#449] tree: Allow reading requests signed by keys from allow list Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config/tree/config.go | 21 +++++++++++++++++++++ cmd/frostfs-node/tree.go | 1 + pkg/services/tree/options.go | 12 ++++++++++++ pkg/services/tree/signature.go | 15 +++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/cmd/frostfs-node/config/tree/config.go b/cmd/frostfs-node/config/tree/config.go index f6087c53d..8a8919999 100644 --- a/cmd/frostfs-node/config/tree/config.go +++ b/cmd/frostfs-node/config/tree/config.go @@ -1,9 +1,11 @@ package treeconfig import ( + "fmt" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) const ( @@ -71,3 +73,22 @@ func (c TreeConfig) ReplicationWorkerCount() int { func (c TreeConfig) SyncInterval() time.Duration { return config.DurationSafe(c.cfg, "sync_interval") } + +// AuthorizedKeys parses and returns an array of "authorized_keys" config +// parameter from "tree" section. +// +// Returns an empty list if not set. +func (c TreeConfig) AuthorizedKeys() keys.PublicKeys { + authorizedKeysStr := config.StringSliceSafe(c.cfg, "authorized_keys") + authorizedKeys := make(keys.PublicKeys, 0, len(authorizedKeysStr)) + + for i := range authorizedKeysStr { + pub, err := keys.NewPublicKeyFromString(authorizedKeysStr[i]) + if err != nil { + panic(fmt.Errorf("could not parse Tree authorized key %s: %w", authorizedKeysStr[i], err)) + } + + authorizedKeys = append(authorizedKeys, pub) + } + return authorizedKeys +} diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index fffaa01d1..1f73b7956 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -56,6 +56,7 @@ func initTreeService(c *cfg) { tree.WithReplicationTimeout(treeConfig.ReplicationTimeout()), tree.WithReplicationChannelCapacity(treeConfig.ReplicationChannelCapacity()), tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount()), + tree.WithAuthorizedKeys(treeConfig.AuthorizedKeys()), tree.WithMetrics(c.metricsCollector.TreeService())) for _, srv := range c.cfgGRPC.servers { diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index bcaf21f92..a6e23c625 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -33,6 +33,7 @@ type cfg struct { replicatorWorkerCount int replicatorTimeout time.Duration containerCacheSize int + authorizedKeys [][]byte metrics MetricsRegister } @@ -124,3 +125,14 @@ func WithMetrics(v MetricsRegister) Option { c.metrics = v } } + +// WithAuthorizedKeys returns option to add list of public +// keys that have rights to use Tree service. +func WithAuthorizedKeys(keys keys.PublicKeys) Option { + return func(c *cfg) { + c.authorizedKeys = nil + for _, key := range keys { + c.authorizedKeys = append(c.authorizedKeys, key.Bytes()) + } + } +} diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 439912969..976fc8d07 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -51,6 +51,21 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op if err != nil { return err } + if op == acl.OpObjectGet { + // verify if the request for a client operation + // was signed by a key from authorized list. + // Operation must be one of READ. + sign := req.GetSignature() + if sign == nil { + return errors.New("missing signature") + } + var key = sign.GetKey() + for i := range s.authorizedKeys { + if bytes.Equal(s.authorizedKeys[i], key) { + return nil + } + } + } cnr, err := s.cnrSource.Get(cid) if err != nil { From ab489265b3f27435a8a6e013fd2074319d9bf3e0 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 21 Jun 2023 14:18:09 +0300 Subject: [PATCH 0363/1943] [#449] cli: Allow to exec `tree get-by-path` without bearer token Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/tree/get_by_path.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index 6f6fc7ff9..69a3dc7ea 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -71,7 +71,9 @@ func getByPath(cmd *cobra.Command, _ []string) { Path: strings.Split(path, "/"), LatestOnly: latestOnly, AllAttributes: true, - BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), + } + if btok := common.ReadBearerToken(cmd, bearerFlagKey); btok != nil { + req.Body.BearerToken = btok.Marshal() } commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) From 72fedff7ad4d7c523bc6433025eda7e3f98ef093 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Jun 2023 16:33:05 +0300 Subject: [PATCH 0364/1943] [#426] cli: Add object nodes command Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 358 ++++++++++++++++++++++++ cmd/frostfs-cli/modules/object/root.go | 4 +- 2 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-cli/modules/object/nodes.go diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go new file mode 100644 index 000000000..358a9d618 --- /dev/null +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -0,0 +1,358 @@ +package object + +import ( + "context" + "crypto/ecdsa" + "errors" + "fmt" + "strconv" + "sync" + "text/tabwriter" + + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" +) + +const ( + verifyPresenceAllFlag = "verify-presence-all" +) + +type objectNodesInfo struct { + containerID cid.ID + objectID oid.ID + relatedObjectIDs []oid.ID + isLock bool +} + +type boolError struct { + value bool + err error +} + +var objectNodesCmd = &cobra.Command{ + Use: "nodes", + Short: "List of nodes where the object is stored", + Long: `List of nodes where the object should be stored and where it is actually stored. + Lock objects must exist on all nodes of the container. + For complex objects, a node is considered to store an object if the node stores at least one part of the complex object. + By default, the actual storage of the object is checked only on the nodes that should store the object. To check all nodes, use the flag --verify-presence-all.`, + Run: objectNodes, +} + +func initObjectNodesCmd() { + commonflags.Init(objectNodesCmd) + + flags := objectNodesCmd.Flags() + + flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + _ = objectGetCmd.MarkFlagRequired(commonflags.CIDFlag) + + flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage) + _ = objectGetCmd.MarkFlagRequired(commonflags.OIDFlag) + + flags.Bool("verify-presence-all", false, "Verify the actual presence of the object on all netmap nodes") +} + +func objectNodes(cmd *cobra.Command, _ []string) { + var cnrID cid.ID + var objID oid.ID + readObjectAddress(cmd, &cnrID, &objID) + + pk := key.GetOrGenerate(cmd) + cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) + + objectInfo := getObjectInfo(cmd, cnrID, objID, cli, pk) + + placementPolicy, netmap := getPlacementPolicyAndNetmap(cmd, cnrID, cli) + + requiredPlacement := getRequiredPlacement(cmd, objectInfo, placementPolicy, netmap) + + actualPlacement := getActualPlacement(cmd, netmap, requiredPlacement, pk, objectInfo) + + printPlacement(cmd, netmap, requiredPlacement, actualPlacement) +} + +func getObjectInfo(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) *objectNodesInfo { + var addrObj oid.Address + addrObj.SetContainer(cnrID) + addrObj.SetObject(objID) + + var prmHead internalclient.HeadObjectPrm + prmHead.SetClient(cli) + prmHead.SetAddress(addrObj) + prmHead.SetRawFlag(true) + + Prepare(cmd, &prmHead) + readSession(cmd, &prmHead, pk, cnrID, objID) + + res, err := internalclient.HeadObject(cmd.Context(), prmHead) + if err == nil { + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + isLock: res.Header().Type() == objectSDK.TypeLock, + } + } + + var errSplitInfo *objectSDK.SplitInfoError + + if !errors.As(err, &errSplitInfo) { + commonCmd.ExitOnErr(cmd, "failed to get object info: %w", err) + return nil + } + + splitInfo := errSplitInfo.SplitInfo() + + if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnrID); ok { + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + relatedObjectIDs: members, + } + } + + if members, ok := tryGetSplitMembersBySplitID(cmd, splitInfo, cli, cnrID); ok { + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + relatedObjectIDs: members, + } + } + + members := tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnrID, objID) + return &objectNodesInfo{ + containerID: cnrID, + objectID: objID, + relatedObjectIDs: members, + } +} + +func getPlacementPolicyAndNetmap(cmd *cobra.Command, cnrID cid.ID, cli *client.Client) (placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) { + eg, egCtx := errgroup.WithContext(cmd.Context()) + eg.Go(func() (e error) { + placementPolicy, e = getPlacementPolicy(egCtx, cnrID, cli) + return + }) + eg.Go(func() (e error) { + netmap, e = getNetMap(egCtx, cli) + return + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", eg.Wait()) + return +} + +func getPlacementPolicy(ctx context.Context, cnrID cid.ID, cli *client.Client) (netmapSDK.PlacementPolicy, error) { + var prm internalclient.GetContainerPrm + prm.SetClient(cli) + prm.SetContainer(cnrID) + + res, err := internalclient.GetContainer(ctx, prm) + if err != nil { + return netmapSDK.PlacementPolicy{}, err + } + + return res.Container().PlacementPolicy(), nil +} + +func getNetMap(ctx context.Context, cli *client.Client) (*netmapSDK.NetMap, error) { + var prm internalclient.NetMapSnapshotPrm + prm.SetClient(cli) + + res, err := internalclient.NetMapSnapshot(ctx, prm) + if err != nil { + return nil, err + } + nm := res.NetMap() + return &nm, nil +} + +func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) map[uint64]netmapSDK.NodeInfo { + nodes := make(map[uint64]netmapSDK.NodeInfo) + placementBuilder := placement.NewNetworkMapBuilder(netmap) + placement, err := placementBuilder.BuildPlacement(objInfo.containerID, &objInfo.objectID, placementPolicy) + commonCmd.ExitOnErr(cmd, "failed to get required placement: %w", err) + for repIdx, rep := range placement { + numOfReplicas := placementPolicy.ReplicaNumberByIndex(repIdx) + var nodeIdx uint32 + for _, n := range rep { + if !objInfo.isLock && nodeIdx == numOfReplicas { //lock object should be on all container nodes + break + } + nodes[n.Hash()] = n + nodeIdx++ + } + } + + for _, relatedObjID := range objInfo.relatedObjectIDs { + placement, err = placementBuilder.BuildPlacement(objInfo.containerID, &relatedObjID, placementPolicy) + commonCmd.ExitOnErr(cmd, "failed to get required placement for related object: %w", err) + for _, rep := range placement { + for _, n := range rep { + nodes[n.Hash()] = n + } + } + } + + return nodes +} + +func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, + pk *ecdsa.PrivateKey, objInfo *objectNodesInfo) map[uint64]boolError { + result := make(map[uint64]boolError) + resultMtx := &sync.Mutex{} + + var candidates []netmapSDK.NodeInfo + checkAllNodes, _ := cmd.Flags().GetBool(verifyPresenceAllFlag) + if checkAllNodes { + candidates = netmap.Nodes() + } else { + for _, n := range requiredPlacement { + candidates = append(candidates, n) + } + } + + var err error + eg, egCtx := errgroup.WithContext(cmd.Context()) + for _, cand := range candidates { + cand := cand + + eg.Go(func() error { + var cli *client.Client + cli, err = createClient(egCtx, cmd, cand, pk) + if err != nil { + resultMtx.Lock() + defer resultMtx.Unlock() + result[cand.Hash()] = boolError{err: err} + return nil + } + + eg.Go(func() error { + var v boolError + v.value, v.err = isObjectStoredOnNode(egCtx, cmd, objInfo.containerID, objInfo.objectID, cli, pk) + resultMtx.Lock() + defer resultMtx.Unlock() + if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) { + return nil + } + result[cand.Hash()] = v + return nil + }) + + for _, rObjID := range objInfo.relatedObjectIDs { + rObjID := rObjID + eg.Go(func() error { + var v boolError + v.value, v.err = isObjectStoredOnNode(egCtx, cmd, objInfo.containerID, rObjID, cli, pk) + resultMtx.Lock() + defer resultMtx.Unlock() + if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) { + return nil + } + result[cand.Hash()] = v + return nil + }) + } + return nil + }) + } + + egErr := eg.Wait() + if err != nil || egErr != nil { + if err == nil { + err = egErr + } + commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", err) + } + return result +} + +func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.NodeInfo, pk *ecdsa.PrivateKey) (*client.Client, error) { + var cli *client.Client + var addresses []string + candidate.IterateNetworkEndpoints(func(s string) bool { + addresses = append(addresses, s) + return false + }) + var lastErr error + for _, address := range addresses { + var networkAddr network.Address + lastErr = networkAddr.FromString(address) + if lastErr != nil { + continue + } + cli, lastErr = internalclient.GetSDKClient(ctx, cmd, pk, networkAddr) + if lastErr == nil { + break + } + } + if lastErr != nil { + return nil, lastErr + } + if cli == nil { + return nil, fmt.Errorf("failed to create client: no available endpoint") + } + return cli, nil +} + +func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) (bool, error) { + var addrObj oid.Address + addrObj.SetContainer(cnrID) + addrObj.SetObject(objID) + + var prmHead internalclient.HeadObjectPrm + prmHead.SetClient(cli) + prmHead.SetAddress(addrObj) + + Prepare(cmd, &prmHead) + prmHead.SetTTL(1) + readSession(cmd, &prmHead, pk, cnrID, objID) + + res, err := internalclient.HeadObject(ctx, prmHead) + if err == nil && res != nil { + return true, nil + } + var notFound *apistatus.ObjectNotFound + var removed *apistatus.ObjectAlreadyRemoved + if errors.As(err, ¬Found) || errors.As(err, &removed) { + return false, nil + } + return false, err +} + +func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, actualPlacement map[uint64]boolError) { + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug) + defer func() { + commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", w.Flush()) + }() + fmt.Fprintln(w, "Netmap node\tShould contain object\tActually contains object\t") + for _, n := range netmap.Nodes() { + var address string + n.IterateNetworkEndpoints(func(s string) bool { + address = s + return s != "" + }) + _, required := requiredPlacement[n.Hash()] + actual, actualExists := actualPlacement[n.Hash()] + actualStr := "" + if actualExists { + if actual.err != nil { + actualStr = fmt.Sprintf("error: %v", actual.err) + } else { + actualStr = strconv.FormatBool(actual.value) + } + } + fmt.Fprintf(w, "%s\t%s\t%s\t\n", address, strconv.FormatBool(required), actualStr) + } +} diff --git a/cmd/frostfs-cli/modules/object/root.go b/cmd/frostfs-cli/modules/object/root.go index 23badc576..c46fc058a 100644 --- a/cmd/frostfs-cli/modules/object/root.go +++ b/cmd/frostfs-cli/modules/object/root.go @@ -30,7 +30,8 @@ func init() { objectHeadCmd, objectHashCmd, objectRangeCmd, - objectLockCmd} + objectLockCmd, + objectNodesCmd} Cmd.AddCommand(objectChildCommands...) @@ -47,4 +48,5 @@ func init() { initObjectHashCmd() initObjectRangeCmd() initCommandObjectLock() + initObjectNodesCmd() } From dd3874eff1cf8a0111ce5567ee9f46a5829245d7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 14:23:33 +0300 Subject: [PATCH 0365/1943] [#447] pilorama: Add benchmark for create ops Signed-off-by: Evgenii Stratonikov --- .../pilorama/bench_test.go | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 pkg/local_object_storage/pilorama/bench_test.go diff --git a/pkg/local_object_storage/pilorama/bench_test.go b/pkg/local_object_storage/pilorama/bench_test.go new file mode 100644 index 000000000..e729b9ea6 --- /dev/null +++ b/pkg/local_object_storage/pilorama/bench_test.go @@ -0,0 +1,55 @@ +package pilorama + +import ( + "context" + "os" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "github.com/stretchr/testify/require" +) + +func getTimestamp(reorder int, ts Timestamp) Timestamp { + base := ts / Timestamp(reorder) + rem := ts % Timestamp(reorder) + return base*Timestamp(reorder) + Timestamp(reorder) - rem +} + +func BenchmarkCreate(b *testing.B) { + // Use `os.TempDir` because we construct multiple times in the same test. + tmpDir, err := os.MkdirTemp(os.TempDir(), "*") + require.NoError(b, err) + + f := NewBoltForest( + WithPath(filepath.Join(tmpDir, "test.db")), + WithMaxBatchSize(runtime.GOMAXPROCS(0))) + require.NoError(b, f.Open(false)) + require.NoError(b, f.Init()) + b.Cleanup(func() { + require.NoError(b, f.Close()) + require.NoError(b, os.RemoveAll(tmpDir)) + }) + + cid := cidtest.ID() + treeID := "tree" + ctx := context.Background() + var index atomic.Int32 + index.Store(-1) + b.SetParallelism(2) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + i := index.Add(1) + op := &Move{ + Meta: Meta{Time: getTimestamp(runtime.GOMAXPROCS(0)*2, Timestamp(i+1))}, + Child: Node(i + 1), + Parent: RootID, + } + if err := f.TreeApply(ctx, cid, treeID, op, true); err != nil { + b.FailNow() + } + } + }) +} From b4ce0b0412fed1554528cf8758744c1204a2885d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Apr 2023 15:06:16 +0300 Subject: [PATCH 0366/1943] [#447] pilorama: Do not undo log for create ops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` goos: linux goarch: amd64 cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ old │ new │ │ sec/op │ sec/op vs base │ Create-8 36.48µ ± 11% 30.34µ ± 14% -16.84% (p=0.000 n=10) │ old │ new │ │ B/op │ B/op vs base │ Create-8 43.01Ki ± 4% 37.78Ki ± 5% -12.15% (p=0.000 n=10) │ old │ new │ │ allocs/op │ allocs/op vs base │ Create-8 166.0 ± 3% 146.0 ± 3% -12.05% (p=0.000 n=10) ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/batch.go | 64 +++++++++++++++++++++- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/pilorama/batch.go b/pkg/local_object_storage/pilorama/batch.go index 5722c68aa..bccd640f1 100644 --- a/pkg/local_object_storage/pilorama/batch.go +++ b/pkg/local_object_storage/pilorama/batch.go @@ -1,6 +1,7 @@ package pilorama import ( + "encoding/binary" "sort" "sync" "time" @@ -49,10 +50,67 @@ func (b *batch) run() { sort.Slice(b.operations, func(i, j int) bool { return b.operations[i].Time < b.operations[j].Time }) - b.operations = removeDuplicatesInPlace(b.operations) - var lm Move - return b.forest.applyOperation(bLog, bTree, b.operations, &lm) + + // Our main use-case is addition of new items. In this case, + // we do not need to perform undo()/redo(), just do(). + // https://github.com/trvedata/move-op/blob/6c23447c12a7862ff31b7fc2205f6c90fbdb9dc0/proof/Move_Create.thy#L259 + // + // For this optimization to work we need to ensure three things: + // 1. The node itself is not yet in tree. + // 2. The node is not a parent. This case is not mentioned in the article, because + // they consider a "static order" (perform all CREATE operations before MOVE). + // We need this because if node _is_ a parent, we could violate (3) for some late operation. + // See TestForest_ApplySameOperation for details. + // 3. Parent of each operation is already in tree. + var parents map[uint64]struct{} + var cKey [17]byte + var slow bool + for i := range b.operations { + _, _, _, inTree := b.forest.getState(bTree, stateKey(cKey[:], b.operations[i].Child)) + if inTree { + slow = true + break + } + + key := childrenKey(cKey[:], b.operations[i].Child, 0) + k, _ := bTree.Cursor().Seek(key) + if len(k) == 17 && binary.LittleEndian.Uint64(k[1:]) == b.operations[i].Child { + slow = true + break + } + + if b.operations[i].Parent == RootID { + continue + } else if parents == nil { + // Attaching key only to root is done frequently, + // no allocations are performed unless necessary. + parents = make(map[uint64]struct{}) + } else if _, ok := parents[b.operations[i].Parent]; ok { + continue + } + + p := b.operations[i].Parent + _, ts, _, inTree := b.forest.getState(bTree, stateKey(cKey[:], p)) + if !inTree || b.operations[0].Time < ts { + slow = true + break + } + parents[b.operations[i].Parent] = struct{}{} + } + + if slow { + var lm Move + return b.forest.applyOperation(bLog, bTree, b.operations, &lm) + } + + var key [17]byte + for i := range b.operations { + if err := b.forest.do(bLog, bTree, key[:], b.operations[i]); err != nil { + return err + } + } + return nil }) for i := range b.results { b.results[i] <- err From 4bf345225c8dc98004cc79b6d9dd88008c936dd3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 20 Jun 2023 13:39:18 +0300 Subject: [PATCH 0367/1943] [#447] pilorama: Use named constant for the key size Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/batch.go | 7 +++---- pkg/local_object_storage/pilorama/boltdb.go | 15 ++++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/local_object_storage/pilorama/batch.go b/pkg/local_object_storage/pilorama/batch.go index bccd640f1..c65488b74 100644 --- a/pkg/local_object_storage/pilorama/batch.go +++ b/pkg/local_object_storage/pilorama/batch.go @@ -64,7 +64,7 @@ func (b *batch) run() { // See TestForest_ApplySameOperation for details. // 3. Parent of each operation is already in tree. var parents map[uint64]struct{} - var cKey [17]byte + var cKey [maxKeySize]byte var slow bool for i := range b.operations { _, _, _, inTree := b.forest.getState(bTree, stateKey(cKey[:], b.operations[i].Child)) @@ -75,7 +75,7 @@ func (b *batch) run() { key := childrenKey(cKey[:], b.operations[i].Child, 0) k, _ := bTree.Cursor().Seek(key) - if len(k) == 17 && binary.LittleEndian.Uint64(k[1:]) == b.operations[i].Child { + if len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == b.operations[i].Child { slow = true break } @@ -104,9 +104,8 @@ func (b *batch) run() { return b.forest.applyOperation(bLog, bTree, b.operations, &lm) } - var key [17]byte for i := range b.operations { - if err := b.forest.do(bLog, bTree, key[:], b.operations[i]); err != nil { + if err := b.forest.do(bLog, bTree, cKey[:], b.operations[i]); err != nil { return err } } diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index a19561139..53a52433d 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -37,6 +37,11 @@ type boltForest struct { cfg } +const ( + childrenKeySize = 17 + maxKeySize = childrenKeySize +) + var ( dataBucket = []byte{0} logBucket = []byte{1} @@ -199,7 +204,7 @@ func (t *boltForest) TreeMove(ctx context.Context, d CIDDescriptor, treeID strin if lm.Child == RootID { lm.Child = t.findSpareID(bTree) } - return t.do(bLog, bTree, make([]byte, 17), &lm) + return t.do(bLog, bTree, make([]byte, maxKeySize), &lm) })) } @@ -395,7 +400,7 @@ func (t *boltForest) addByPathInternal(d CIDDescriptor, attr string, treeID stri } var lm []Move - var key [17]byte + var key [maxKeySize]byte fullID := bucketName(d.CID, treeID) err := t.db.Batch(func(tx *bbolt.Tx) error { @@ -612,7 +617,7 @@ func (t *boltForest) getTreeBuckets(tx *bbolt.Tx, treeRoot []byte) (*bbolt.Bucke // applyOperations applies log operations. Assumes lm are sorted by timestamp. func (t *boltForest) applyOperation(logBucket, treeBucket *bbolt.Bucket, ms []*Move, lm *Move) error { var tmp Move - var cKey [17]byte + var cKey [maxKeySize]byte c := logBucket.Cursor() @@ -961,7 +966,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID b := treeRoot.Bucket(dataBucket) c := b.Cursor() - for k, _ := c.Seek(key); len(k) == 17 && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { + for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { children = append(children, binary.LittleEndian.Uint64(k[9:])) } return nil @@ -1216,7 +1221,7 @@ func childrenKey(key []byte, child, parent Node) []byte { key[0] = 'c' binary.LittleEndian.PutUint64(key[1:], parent) binary.LittleEndian.PutUint64(key[9:], child) - return key[:17] + return key[:childrenKeySize] } // 'i' + attribute name (string) + attribute value (string) + parent (id) + node (id) -> 0/1. From 71a63b8e9c629fb7efb5343e108995296df97ed3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 26 Jun 2023 10:07:38 +0300 Subject: [PATCH 0368/1943] [#439] node: Update SDK-Go version This fixed version: null header for complex object. Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0fdcb4409..7a1c2f074 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230602115440-ec59ebfd8826 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230623104802-aa8ffebc6328 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 5da5a9a9e..dfa8632aa 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230602115440-ec59ebfd8826 h1:dIk/pUsCME/nbhDdGKe1n7XOdrfvLCCTzJbnuI5zkOk= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230602115440-ec59ebfd8826/go.mod h1:xpCauXjq+v7mgbjr+2Ebs6/Xx8gxKxXHFfHEZUH3N4E= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230623104802-aa8ffebc6328 h1:d2v3W0vLdcqYTY1/P0scdwUukmBfvt+CzkkD0EvskPA= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230623104802-aa8ffebc6328/go.mod h1:w+s3ozlbFfTDFHhjX0A3Iif3BRtnTkwiACxFZD+Q0cQ= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From cac4ed93d6a73e673864896453ab5197056e6458 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 22 Jun 2023 10:46:56 +0300 Subject: [PATCH 0369/1943] [#428] engine: Add low_mem config parameter Concurrent initialization in case of the metabase resync leads to high memory consumption and potential OOM. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 4 ++- cmd/frostfs-node/config/engine/config.go | 5 ++++ docs/storage-node-configuration.md | 1 + pkg/local_object_storage/engine/control.go | 29 +++++++++++++++++----- pkg/local_object_storage/engine/engine.go | 9 +++++++ pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/shard/shard.go | 4 +-- 7 files changed, 44 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 2cb785d30..5d7adab29 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -98,6 +98,7 @@ type applicationConfiguration struct { errorThreshold uint32 shardPoolSize uint32 shards []shardCfg + lowMem bool } } @@ -200,6 +201,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { a.EngineCfg.errorThreshold = engineconfig.ShardErrorThreshold(c) a.EngineCfg.shardPoolSize = engineconfig.ShardPoolSize(c) + a.EngineCfg.lowMem = engineconfig.EngineLowMemoryConsumption(c) return engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error { return a.updateShardConfig(c, sc) }) } @@ -675,8 +677,8 @@ func (c *cfg) engineOpts() []engine.Option { opts = append(opts, engine.WithShardPoolSize(c.EngineCfg.shardPoolSize), engine.WithErrorThreshold(c.EngineCfg.errorThreshold), - engine.WithLogger(c.log), + engine.WithLowMemoryConsumption(c.EngineCfg.lowMem), ) if c.metricsCollector != nil { diff --git a/cmd/frostfs-node/config/engine/config.go b/cmd/frostfs-node/config/engine/config.go index 36684f093..c944d1c58 100644 --- a/cmd/frostfs-node/config/engine/config.go +++ b/cmd/frostfs-node/config/engine/config.go @@ -83,3 +83,8 @@ func ShardPoolSize(c *config.Config) uint32 { func ShardErrorThreshold(c *config.Config) uint32 { return config.Uint32Safe(c.Sub(subsection), "shard_ro_error_threshold") } + +// EngineLowMemoryConsumption returns value of "lowmem" config parmeter from "storage" section. +func EngineLowMemoryConsumption(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "low_mem") +} diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 2c78cf6b1..4469b1e10 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -167,6 +167,7 @@ Local storage engine configuration. |----------------------------|-----------------------------------|---------------|------------------------------------------------------------------------------------------------------------------| | `shard_pool_size` | `int` | `20` | Pool size for shard workers. Limits the amount of concurrent `PUT` operations on each shard. | | `shard_ro_error_threshold` | `int` | `0` | Maximum amount of storage errors to encounter before shard automatically moves to `Degraded` or `ReadOnly` mode. | +| `low_mem` | `bool` | `false` | Reduce memory consumption by reducing performance. | | `shard` | [Shard config](#shard-subsection) | | Configuration for separate shards. | ## `shard` subsection diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 9ad4fcf9c..bd166b3ff 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -12,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) type shardInitError struct { @@ -74,23 +75,30 @@ func (e *StorageEngine) Init(ctx context.Context) error { e.mtx.Lock() defer e.mtx.Unlock() - var wg sync.WaitGroup var errCh = make(chan shardInitError, len(e.shards)) + var eg errgroup.Group + if e.cfg.lowMem && e.anyShardRequiresRefill() { + eg.SetLimit(1) + } for id, sh := range e.shards { - wg.Add(1) - go func(id string, sh *shard.Shard) { - defer wg.Done() + id := id + sh := sh + eg.Go(func() error { if err := sh.Init(ctx); err != nil { errCh <- shardInitError{ err: err, id: id, } } - }(id, sh.Shard) + return nil + }) } - wg.Wait() + err := eg.Wait() close(errCh) + if err != nil { + return fmt.Errorf("failed to initialize shards: %w", err) + } for res := range errCh { if res.err != nil { @@ -125,6 +133,15 @@ func (e *StorageEngine) Init(ctx context.Context) error { return nil } +func (e *StorageEngine) anyShardRequiresRefill() bool { + for _, sh := range e.shards { + if sh.NeedRefillMetabase() { + return true + } + } + return false +} + var errClosed = errors.New("storage engine is closed") // Close releases all StorageEngine's components. Waits for all data-related operations to complete. diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 3061d6383..7bc84e6c9 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -210,6 +210,8 @@ type cfg struct { metrics MetricRegister shardPoolSize uint32 + + lowMem bool } func defaultCfg() *cfg { @@ -265,3 +267,10 @@ func WithErrorThreshold(sz uint32) Option { c.errorsThreshold = sz } } + +// WithLowMemoryConsumption returns an option to set the flag to reduce memory consumption by reducing performance. +func WithLowMemoryConsumption(lowMemCons bool) Option { + return func(c *cfg) { + c.lowMem = lowMemCons + } +} diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index bc514933b..db8248c02 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -98,7 +98,7 @@ func (s *Shard) Init(ctx context.Context) error { if !s.GetMode().NoMetabase() { var initMetabase initializer - if s.needRefillMetabase() { + if s.NeedRefillMetabase() { initMetabase = (*metabaseSynchronizer)(s) } else { initMetabase = s.metaBase diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 10c1acd40..05799d236 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -231,8 +231,8 @@ func (s *Shard) hasWriteCache() bool { return s.cfg.useWriteCache } -// needRefillMetabase returns true if metabase is needed to be refilled. -func (s *Shard) needRefillMetabase() bool { +// NeedRefillMetabase returns true if metabase is needed to be refilled. +func (s *Shard) NeedRefillMetabase() bool { return s.cfg.refillMetabase } From 43d263c3d512914427fe6ac5fdcc845a61778ee1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 22 Jun 2023 10:50:06 +0300 Subject: [PATCH 0370/1943] [#428] linter: Fix unkeyed assignment Thanks to gopls. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/tree_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index c2bae9772..4cd35a088 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -37,7 +37,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { b.Fatal(err) } _, err = te.ng.TreeAddByPath(context.Background(), d, treeID, pilorama.AttributeFilename, nil, - []pilorama.KeyValue{{pilorama.AttributeFilename, []byte(strconv.Itoa(i))}}) + []pilorama.KeyValue{{Key: pilorama.AttributeFilename, Value: []byte(strconv.Itoa(i))}}) if err != nil { b.Fatal(err) } From 0c866f62c5821039fdfd3accbf8f5c8fa0443d42 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 27 Jun 2023 11:12:01 +0300 Subject: [PATCH 0371/1943] [#473] placement: Fix backwards compatibility for `copies_number` Signed-off-by: Evgenii Stratonikov --- pkg/services/object_manager/placement/traverser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 7b9b26a0b..47f79a627 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -86,7 +86,7 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } // backward compatibility for scalar `copies_number` - if len(cfg.copyNumbers) == 1 { + if len(cfg.copyNumbers) == 1 && cfg.copyNumbers[0] != 0 { cfg.flatSuccess = &cfg.copyNumbers[0] } From 13b53258b46bf696f49715d6080bfb5b660f1284 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 13:51:01 +0300 Subject: [PATCH 0372/1943] [#1] go.mod: Remove retract version Signed-off-by: Evgenii Stratonikov --- go.mod | 5 ----- 1 file changed, 5 deletions(-) diff --git a/go.mod b/go.mod index 7a1c2f074..aca1d2d6f 100644 --- a/go.mod +++ b/go.mod @@ -119,8 +119,3 @@ require ( lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) - -retract ( - v1.22.1 // Contains retraction only. - v1.22.0 // Published accidentally. -) From 64e1383df604f635c77f7a646e4b198aaab7962c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 13:52:58 +0300 Subject: [PATCH 0373/1943] [#1] Makefile: Remove `image-storage-testnet` target Signed-off-by: Evgenii Stratonikov --- .docker/Dockerfile.storage-testnet | 19 ------------------- Makefile | 2 +- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 .docker/Dockerfile.storage-testnet diff --git a/.docker/Dockerfile.storage-testnet b/.docker/Dockerfile.storage-testnet deleted file mode 100644 index 28bbf7a1b..000000000 --- a/.docker/Dockerfile.storage-testnet +++ /dev/null @@ -1,19 +0,0 @@ -FROM golang:1.20 as builder -ARG BUILD=now -ARG VERSION=dev -ARG REPO=repository -WORKDIR /src -COPY . /src - -RUN make bin/frostfs-node - -# Executable image -FROM alpine AS frostfs-node -RUN apk add --no-cache bash - -WORKDIR / - -COPY --from=builder /src/bin/frostfs-node /bin/frostfs-node -COPY --from=builder /src/config/testnet/config.yml /config.yml - -CMD ["frostfs-node", "--config", "/config.yml"] diff --git a/Makefile b/Makefile index cf2095ec0..e1b719206 100755 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ image-%: -t $(HUB_IMAGE)-$*:$(HUB_TAG) . # Build all Docker images -images: image-storage image-ir image-cli image-adm image-storage-testnet +images: image-storage image-ir image-cli image-adm # Build dirty local Docker images dirty-images: image-dirty-storage image-dirty-ir image-dirty-cli image-dirty-adm From f4c71cea659d6966f753ff9d0bc157012d616cb4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 13:58:22 +0300 Subject: [PATCH 0374/1943] [#1] *: Replace outdated FIXME/TODO links Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/container/list.go | 3 ++- cmd/frostfs-cli/modules/container/list_objects.go | 3 ++- pkg/core/object/fmt.go | 1 - pkg/innerring/processors/container/common.go | 3 --- pkg/local_object_storage/metabase/select.go | 1 - pkg/services/object/internal/client/client.go | 2 +- 6 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 1c9e4767f..189cc05c7 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -78,7 +78,8 @@ var listContainersCmd = &cobra.Command{ if flagVarListPrintAttr { cnr.IterateAttributes(func(key, val string) { if !strings.HasPrefix(key, container.SysAttributePrefix) && !strings.HasPrefix(key, container.SysAttributePrefixNeoFS) { - // FIXME(@cthulhu-rider): neofs-sdk-go#314 use dedicated method to skip system attributes + // FIXME(@cthulhu-rider): https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/97 + // Use dedicated method to skip system attributes. cmd.Printf(" %s: %s\n", key, val) } }) diff --git a/cmd/frostfs-cli/modules/container/list_objects.go b/cmd/frostfs-cli/modules/container/list_objects.go index e417560e8..d00cc3576 100644 --- a/cmd/frostfs-cli/modules/container/list_objects.go +++ b/cmd/frostfs-cli/modules/container/list_objects.go @@ -71,7 +71,8 @@ var listContainerObjectsCmd = &cobra.Command{ for i := range attrs { attrKey := attrs[i].Key() if !strings.HasPrefix(attrKey, v2object.SysAttributePrefix) && !strings.HasPrefix(attrKey, v2object.SysAttributePrefixNeoFS) { - // FIXME(@cthulhu-rider): neofs-sdk-go#226 use dedicated method to skip system attributes + // FIXME(@cthulhu-rider): https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/97 + // Use dedicated method to skip system attributes. cmd.Printf(" %s: %s\n", attrKey, attrs[i].Value()) } } diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index 946cfc462..bfd22c9ae 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -133,7 +133,6 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unpr } func (v *FormatValidator) validateSignatureKey(obj *object.Object) error { - // FIXME(@cthulhu-rider): temp solution, see neofs-sdk-go#233 sig := obj.Signature() if sig == nil { // TODO(@cthulhu-rider): #1387 use "const" error diff --git a/pkg/innerring/processors/container/common.go b/pkg/innerring/processors/container/common.go index 375e4c179..97eb6f559 100644 --- a/pkg/innerring/processors/container/common.go +++ b/pkg/innerring/processors/container/common.go @@ -63,7 +63,6 @@ func (cp *Processor) verifySignature(v signatureVerificationData) error { } if keyProvided { - // TODO(@cthulhu-rider): #1387 use another approach after neofs-sdk-go#233 var idFromKey user.ID user.IDFromKey(&idFromKey, (ecdsa.PublicKey)(key)) @@ -118,8 +117,6 @@ func (cp *Processor) verifyByTokenSession(v signatureVerificationData, key *fros return errors.New("invalid session token signature") } - // FIXME(@cthulhu-rider): #1387 check token is signed by container owner, see neofs-sdk-go#233 - if keyProvided && !tok.AssertAuthKey(key) { return errors.New("signed with a non-session key") } diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 1f32d2cde..0f53a8f36 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -572,6 +572,5 @@ func blindlyProcess(fs object.SearchFilters) bool { // returns true if string key is a reserved system filter key. func isSystemKey(key string) bool { - // FIXME: #1147 version-dependent approach return strings.HasPrefix(key, v2object.ReservedFilterPrefix) } diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 8214c784d..17bf6a40f 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -81,7 +81,7 @@ type readPrmCommon struct { // // By default current epoch on the server will be used. func (x *readPrmCommon) SetNetmapEpoch(_ uint64) { - // FIXME: (neofs-node#1194) not supported by client + // FIXME(@fyrchik): https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/465 } // GetObjectPrm groups parameters of GetObject operation. From 73a71a71b06adb053933e7d7eab117ceb3406784 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 14:43:25 +0300 Subject: [PATCH 0375/1943] [#1] node: Use a proper validation of a substorage type Signed-off-by: Evgenii Stratonikov --- .../config/engine/shard/blobstor/config.go | 13 ++++--------- cmd/frostfs-node/validate.go | 6 +----- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/config.go index a6e34e80f..f8b2e2e9b 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/config.go @@ -5,8 +5,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/storage" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" ) // Config is a wrapper over the config section @@ -25,14 +23,11 @@ func (x *Config) Storages() []*storage.Config { typ := config.String( (*config.Config)(x), strconv.Itoa(i)+".type") - switch typ { - case "": + if typ == "" { return ss - case fstree.Type, blobovniczatree.Type: - sub := storage.From((*config.Config)(x).Sub(strconv.Itoa(i))) - ss = append(ss, sub) - default: - panic("invalid type") } + + sub := storage.From((*config.Config)(x).Sub(strconv.Itoa(i))) + ss = append(ss, sub) } } diff --git a/cmd/frostfs-node/validate.go b/cmd/frostfs-node/validate.go index 3896fd6be..e07afb2ca 100644 --- a/cmd/frostfs-node/validate.go +++ b/cmd/frostfs-node/validate.go @@ -51,17 +51,13 @@ func validateConfig(c *config.Config) error { blobstor := sc.BlobStor().Storages() if len(blobstor) != 2 { - // TODO (@fyrcik): remove after #1522 return fmt.Errorf("blobstor section must have 2 components, got: %d", len(blobstor)) } for i := range blobstor { switch blobstor[i].Type() { case fstree.Type, blobovniczatree.Type: default: - // FIXME #1764 (@fyrchik): this line is currently unreachable, - // because we panic in `sc.BlobStor().Storages()`. - return fmt.Errorf("unexpected storage type: %s (shard %d)", - blobstor[i].Type(), shardNum) + return fmt.Errorf("unexpected storage type: %s (shard %d)", blobstor[i].Type(), shardNum) } if blobstor[i].Perm()&0600 != 0600 { return fmt.Errorf("invalid permissions for blobstor component: %s, "+ From cab51c8cbe959739b7808a6c797ff0f04b45ff56 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 14:49:08 +0300 Subject: [PATCH 0376/1943] [#1] metabase: Rename blindlyProcess() Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/select.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 0f53a8f36..998ce9b94 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -84,7 +84,7 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err return res, ErrDegradedMode } - if blindlyProcess(prm.filters) { + if checkNonEmpty(prm.filters) { success = true return res, nil } @@ -556,15 +556,12 @@ func markAddressInCache(cache map[string]int, fNum int, addr string) { } } -// returns true if query leads to a deliberately empty result. -func blindlyProcess(fs object.SearchFilters) bool { +// Returns true if at least 1 object can satisfy fs. +func checkNonEmpty(fs object.SearchFilters) bool { for i := range fs { if fs[i].Operation() == object.MatchNotPresent && isSystemKey(fs[i].Header()) { return true } - - // TODO: #1148 check other cases - // e.g. (a == b) && (a != b) } return false From 4f413fe86e33836dc99b5d9629efaca27f924493 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 15:00:44 +0300 Subject: [PATCH 0377/1943] [#1] treesvc: Properly check for secure transport Signed-off-by: Evgenii Stratonikov --- pkg/network/address.go | 2 +- pkg/network/group.go | 2 +- pkg/network/tls.go | 4 ++-- pkg/network/tls_test.go | 2 +- pkg/services/tree/cache.go | 4 +--- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pkg/network/address.go b/pkg/network/address.go index 020882980..8ad285725 100644 --- a/pkg/network/address.go +++ b/pkg/network/address.go @@ -47,7 +47,7 @@ func (a Address) URIAddr() string { panic(fmt.Errorf("could not get host addr: %w", err)) } - if !a.isTLSEnabled() { + if !a.IsTLSEnabled() { return host } diff --git a/pkg/network/group.go b/pkg/network/group.go index c18feac27..a6de0653e 100644 --- a/pkg/network/group.go +++ b/pkg/network/group.go @@ -57,7 +57,7 @@ func (x AddressGroup) Len() int { // Less returns true if i-th address in AddressGroup supports TLS // and j-th one doesn't. func (x AddressGroup) Less(i, j int) bool { - return x[i].isTLSEnabled() && !x[j].isTLSEnabled() + return x[i].IsTLSEnabled() && !x[j].IsTLSEnabled() } // Swap swaps i-th and j-th addresses in AddressGroup. diff --git a/pkg/network/tls.go b/pkg/network/tls.go index de2c93694..9aac89c47 100644 --- a/pkg/network/tls.go +++ b/pkg/network/tls.go @@ -11,8 +11,8 @@ const ( // tls var is used for (un)wrapping other multiaddrs around TLS multiaddr. var tls, _ = multiaddr.NewMultiaddr("/" + tlsProtocolName) -// isTLSEnabled searches for wrapped TLS protocol in multiaddr. -func (a Address) isTLSEnabled() bool { +// IsTLSEnabled searches for wrapped TLS protocol in multiaddr. +func (a Address) IsTLSEnabled() bool { for _, protoc := range a.ma.Protocols() { if protoc.Code == multiaddr.P_TLS { return true diff --git a/pkg/network/tls_test.go b/pkg/network/tls_test.go index 25775eaf1..d93ea6a12 100644 --- a/pkg/network/tls_test.go +++ b/pkg/network/tls_test.go @@ -24,6 +24,6 @@ func TestAddress_TLSEnabled(t *testing.T) { err := addr.FromString(test.input) require.NoError(t, err) - require.Equal(t, test.wantTLS, addr.isTLSEnabled(), test.input) + require.Equal(t, test.wantTLS, addr.IsTLSEnabled(), test.input) } } diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 97218da08..ef0c4b465 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strings" "sync" "time" @@ -100,8 +99,7 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, ), } - // FIXME(@fyrchik): ugly hack #1322 - if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") { + if netAddr.IsTLSEnabled() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } From 8a4e250daefdd469ea0e59e33d587a0823b07dee Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Jun 2023 16:18:39 +0300 Subject: [PATCH 0378/1943] [#468] *: replace outdated TODO crypto-related links Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/util.go | 2 +- cmd/frostfs-cli/modules/object/head.go | 2 +- cmd/frostfs-node/object.go | 2 +- pkg/core/object/fmt.go | 2 +- pkg/morph/client/container/eacl.go | 2 +- pkg/morph/client/container/eacl_set.go | 2 +- pkg/morph/client/container/get.go | 2 +- pkg/morph/client/container/put.go | 2 +- pkg/services/container/morph/executor.go | 4 ++-- pkg/services/control/ir/server/sign.go | 8 ++++---- pkg/services/control/server/sign.go | 8 ++++---- pkg/services/object/acl/v2/util.go | 2 +- pkg/services/tree/signature.go | 2 +- 13 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/util.go b/cmd/frostfs-cli/modules/control/util.go index fdf17244b..5ad675c0e 100644 --- a/cmd/frostfs-cli/modules/control/util.go +++ b/cmd/frostfs-cli/modules/control/util.go @@ -40,7 +40,7 @@ func verifyResponse(cmd *cobra.Command, commonCmd.ExitOnErr(cmd, "", errors.New("missing response signature")) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetScheme(refs.ECDSA_SHA512) sigV2.SetKey(sigControl.GetKey()) diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index 6fce04490..04467744d 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -163,7 +163,7 @@ func printHeader(cmd *cobra.Command, obj *object.Object) error { if signature := obj.Signature(); signature != nil { cmd.Print("ID signature:\n") - // TODO(@carpawell): #1387 implement and use another approach to avoid conversion + // TODO(@carpawell): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature signature.WriteToV2(&sigV2) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 0476dbcac..afc4bcb41 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -435,7 +435,7 @@ func (s *morphEACLFetcher) GetEACL(cnr cid.ID) (*containercore.EACL, error) { } if !eaclInfo.Signature.Verify(binTable) { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return nil, errors.New("invalid signature of the eACL table") } diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index bfd22c9ae..ba9868693 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -135,7 +135,7 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unpr func (v *FormatValidator) validateSignatureKey(obj *object.Object) error { sig := obj.Signature() if sig == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("missing signature") } diff --git a/pkg/morph/client/container/eacl.go b/pkg/morph/client/container/eacl.go index 56c36c179..54c83737d 100644 --- a/pkg/morph/client/container/eacl.go +++ b/pkg/morph/client/container/eacl.go @@ -86,7 +86,7 @@ func (c *Client) GetEACL(cnr cid.ID) (*container.EACL, error) { } } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature sigV2.SetKey(pub) sigV2.SetSign(sig) diff --git a/pkg/morph/client/container/eacl_set.go b/pkg/morph/client/container/eacl_set.go index 86eae4c2b..2d2ffb456 100644 --- a/pkg/morph/client/container/eacl_set.go +++ b/pkg/morph/client/container/eacl_set.go @@ -31,7 +31,7 @@ func PutEACL(c *Client, eaclInfo containercore.EACL) error { prm.SetToken(eaclInfo.Session.Marshal()) } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature eaclInfo.Signature.WriteToV2(&sigV2) diff --git a/pkg/morph/client/container/get.go b/pkg/morph/client/container/get.go index 009b22f3c..0513eea4b 100644 --- a/pkg/morph/client/container/get.go +++ b/pkg/morph/client/container/get.go @@ -105,7 +105,7 @@ func (c *Client) Get(cid []byte) (*containercore.Container, error) { } } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature sigV2.SetKey(pub) sigV2.SetSign(sigBytes) diff --git a/pkg/morph/client/container/put.go b/pkg/morph/client/container/put.go index 2c97446c6..5c23eb36d 100644 --- a/pkg/morph/client/container/put.go +++ b/pkg/morph/client/container/put.go @@ -28,7 +28,7 @@ func Put(c *Client, cnr containercore.Container) (*cid.ID, error) { prm.SetToken(cnr.Session.Marshal()) } - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var sigV2 refs.Signature cnr.Signature.WriteToV2(&sigV2) diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index ae37da520..dec022219 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -52,7 +52,7 @@ func NewExecutor(rdr Reader, wrt Writer) containerSvc.ServiceExecutor { func (s *morphExecutor) Put(_ context.Context, tokV2 *sessionV2.Token, body *container.PutRequestBody) (*container.PutResponseBody, error) { sigV2 := body.GetSignature() if sigV2 == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return nil, errors.New("missing signature") } @@ -206,7 +206,7 @@ func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) func (s *morphExecutor) SetExtendedACL(_ context.Context, tokV2 *sessionV2.Token, body *container.SetExtendedACLRequestBody) (*container.SetExtendedACLResponseBody, error) { sigV2 := body.GetSignature() if sigV2 == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return nil, errors.New("missing signature") } diff --git a/pkg/services/control/ir/server/sign.go b/pkg/services/control/ir/server/sign.go index 4ada98468..f72d51f9e 100644 --- a/pkg/services/control/ir/server/sign.go +++ b/pkg/services/control/ir/server/sign.go @@ -24,7 +24,7 @@ var errDisallowedKey = errors.New("key is not in the allowed list") func (s *Server) isValidRequest(req SignedMessage) error { sign := req.GetSignature() if sign == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("missing signature") } @@ -50,7 +50,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { return fmt.Errorf("marshal request body: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetKey(sign.GetKey()) sigV2.SetSign(sign.GetSign()) @@ -62,7 +62,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { } if !sig.Verify(binBody) { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("invalid signature") } @@ -83,7 +83,7 @@ func SignMessage(key *ecdsa.PrivateKey, msg SignedMessage) error { return fmt.Errorf("calculate signature: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sig.WriteToV2(&sigV2) diff --git a/pkg/services/control/server/sign.go b/pkg/services/control/server/sign.go index 726cdf341..acc405821 100644 --- a/pkg/services/control/server/sign.go +++ b/pkg/services/control/server/sign.go @@ -24,7 +24,7 @@ var errDisallowedKey = errors.New("key is not in the allowed list") func (s *Server) isValidRequest(req SignedMessage) error { sign := req.GetSignature() if sign == nil { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("missing signature") } @@ -50,7 +50,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { return fmt.Errorf("marshal request body: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetKey(sign.GetKey()) sigV2.SetSign(sign.GetSign()) @@ -62,7 +62,7 @@ func (s *Server) isValidRequest(req SignedMessage) error { } if !sig.Verify(binBody) { - // TODO(@cthulhu-rider): #1387 use "const" error + // TODO(@cthulhu-rider): #468 use "const" error return errors.New("invalid signature") } @@ -83,7 +83,7 @@ func SignMessage(key *ecdsa.PrivateKey, msg SignedMessage) error { return fmt.Errorf("calculate signature: %w", err) } - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sig.WriteToV2(&sigV2) diff --git a/pkg/services/object/acl/v2/util.go b/pkg/services/object/acl/v2/util.go index aa5d67584..cd45b63fc 100644 --- a/pkg/services/object/acl/v2/util.go +++ b/pkg/services/object/acl/v2/util.go @@ -118,7 +118,7 @@ func ownerFromToken(token *sessionSDK.Object) (*user.ID, *keys.PublicKey, error) } // 2. Then check if session token owner issued the session token - // TODO(@cthulhu-rider): #1387 implement and use another approach to avoid conversion + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion var tokV2 sessionV2.Token token.WriteToV2(&tokV2) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 976fc8d07..08aa226fa 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -159,7 +159,7 @@ func verifyMessage(m message) error { sig := m.GetSignature() - // TODO(@cthulhu-rider): #1387 use Signature message from NeoFS API to avoid conversion + // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion var sigV2 refs.Signature sigV2.SetKey(sig.GetKey()) sigV2.SetSign(sig.GetSign()) From f91cfb36e6eb33bcdaa1f35dc5f77a58bbb98d9d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 28 Jun 2023 11:09:19 +0300 Subject: [PATCH 0379/1943] [#478] .forgejo: Add build/test workflows Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/build.yml | 38 +++++++++++++++++++++++++ .forgejo/workflows/tests.yml | 54 ++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 .forgejo/workflows/build.yml create mode 100644 .forgejo/workflows/tests.yml diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml new file mode 100644 index 000000000..5e4a97b93 --- /dev/null +++ b/.forgejo/workflows/build.yml @@ -0,0 +1,38 @@ +name: Build + +on: [pull_request] + +jobs: + build: + name: Build Components + runs-on: ubuntu-latest + strategy: + matrix: + go_versions: [ '1.19', '1.20' ] + + steps: + - uses: actions/checkout@v3 + with: + # Allows to fetch all history for all branches and tags. + # Need this for proper versioning. + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '${{ matrix.go_versions }}' + + - name: Build CLI + run: make bin/frostfs-cli + + - name: Build NODE + run: make bin/frostfs-node + + - name: Build IR + run: make bin/frostfs-ir + + - name: Build ADM + run: make bin/frostfs-adm + + - name: Build LENS + run: make bin/frostfs-lens diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml new file mode 100644 index 000000000..2d44d34f8 --- /dev/null +++ b/.forgejo/workflows/tests.yml @@ -0,0 +1,54 @@ +name: Tests and linters +on: [pull_request] + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + cache: true + + - name: golangci-lint + uses: https://github.com/golangci/golangci-lint-action@v3 + with: + version: latest + + tests: + name: Tests + runs-on: ubuntu-latest + strategy: + matrix: + go_versions: [ '1.19', '1.20' ] + fail-fast: false + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '${{ matrix.go_versions }}' + cache: true + + - name: Run tests + run: make test + + tests-race: + name: Tests with -race + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + cache: true + + - name: Run tests + run: go test ./... -count=1 -race From 33d9ebbe7f70c639a7b45373262cda8b3dac00af Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 28 Jun 2023 14:05:09 +0300 Subject: [PATCH 0380/1943] [#478] .golangci.yml: Increase timeout Make it enough for CI. Signed-off-by: Evgenii Stratonikov --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index fae355a3d..4a3fec6e3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -4,7 +4,7 @@ # options for analysis running run: # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 10m + timeout: 20m # include test files or not, default is true tests: false From d0ab552a90bc0359b159f558e7a7a36fbe0afdf7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 28 Jun 2023 14:10:06 +0300 Subject: [PATCH 0381/1943] [#478] *: Fix funlen linter warnings Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/signature.go | 94 +++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 40 deletions(-) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 08aa226fa..63485a707 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -51,20 +51,10 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op if err != nil { return err } - if op == acl.OpObjectGet { - // verify if the request for a client operation - // was signed by a key from authorized list. - // Operation must be one of READ. - sign := req.GetSignature() - if sign == nil { - return errors.New("missing signature") - } - var key = sign.GetKey() - for i := range s.authorizedKeys { - if bytes.Equal(s.authorizedKeys[i], key) { - return nil - } - } + + isAuthorized, err := s.isAuthorized(req, op) + if isAuthorized || err != nil { + return err } cnr, err := s.cnrSource.Get(cid) @@ -74,18 +64,9 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op eaclOp := eACLOp(op) - var bt *bearer.Token - if len(rawBearer) > 0 { - bt = new(bearer.Token) - if err = bt.Unmarshal(rawBearer); err != nil { - return eACLErr(eaclOp, fmt.Errorf("invalid bearer token: %w", err)) - } - if !bt.AssertContainer(cid) { - return eACLErr(eaclOp, errBearerWrongContainer) - } - if !bt.VerifySignature() { - return eACLErr(eaclOp, errBearerSignature) - } + bt, err := parseBearer(rawBearer, cid, eaclOp) + if err != nil { + return err } role, err := roleFromReq(cnr, req, bt) @@ -117,32 +98,65 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op var tb eacl.Table signer := req.GetSignature().GetKey() - if tableFromBearer { - if bt.Impersonate() { - tbCore, err := s.eaclSource.GetEACL(cid) - if err != nil { - return handleGetEACLError(err) - } - tb = *tbCore.Value - signer = bt.SigningKeyBytes() - } else { - if !bearer.ResolveIssuer(*bt).Equals(cnr.Value.Owner()) { - return eACLErr(eaclOp, errBearerWrongOwner) - } - tb = bt.EACLTable() + if tableFromBearer && !bt.Impersonate() { + if !bearer.ResolveIssuer(*bt).Equals(cnr.Value.Owner()) { + return eACLErr(eaclOp, errBearerWrongOwner) } + tb = bt.EACLTable() } else { tbCore, err := s.eaclSource.GetEACL(cid) if err != nil { return handleGetEACLError(err) } - tb = *tbCore.Value + + if bt.Impersonate() { + signer = bt.SigningKeyBytes() + } } return checkEACL(tb, signer, eACLRole(role), eaclOp) } +// Returns true iff the operation is read-only and request was signed +// with one of the authorized keys. +func (s *Service) isAuthorized(req message, op acl.Op) (bool, error) { + if op != acl.OpObjectGet { + return false, nil + } + + sign := req.GetSignature() + if sign == nil { + return false, errors.New("missing signature") + } + + key := sign.GetKey() + for i := range s.authorizedKeys { + if bytes.Equal(s.authorizedKeys[i], key) { + return true, nil + } + } + return false, nil +} + +func parseBearer(rawBearer []byte, cid cidSDK.ID, eaclOp eacl.Operation) (*bearer.Token, error) { + if len(rawBearer) == 0 { + return nil, nil + } + + bt := new(bearer.Token) + if err := bt.Unmarshal(rawBearer); err != nil { + return nil, eACLErr(eaclOp, fmt.Errorf("invalid bearer token: %w", err)) + } + if !bt.AssertContainer(cid) { + return nil, eACLErr(eaclOp, errBearerWrongContainer) + } + if !bt.VerifySignature() { + return nil, eACLErr(eaclOp, errBearerSignature) + } + return bt, nil +} + func handleGetEACLError(err error) error { if client.IsErrEACLNotFound(err) { return nil From d01c064674758bb320c5970bb324e44c11bef011 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 27 Jun 2023 14:26:39 +0300 Subject: [PATCH 0382/1943] [#455] Fix non-informative message when parse node attributes Signed-off-by: Anton Nikiforov --- pkg/util/attributes/parser.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/util/attributes/parser.go b/pkg/util/attributes/parser.go index 8016c0160..f8cc97189 100644 --- a/pkg/util/attributes/parser.go +++ b/pkg/util/attributes/parser.go @@ -1,7 +1,6 @@ package attributes import ( - "errors" "fmt" "strings" @@ -21,7 +20,7 @@ func ReadNodeAttributes(dst *netmap.NodeInfo, attrs []string) error { k, v, found := strings.Cut(line, keyValueSeparator) if !found { - return errors.New("missing attribute key and/or value") + return fmt.Errorf("wrong format for node attribute: '%s'", attrs[i]) } _, ok := cache[k] @@ -36,9 +35,9 @@ func ReadNodeAttributes(dst *netmap.NodeInfo, attrs []string) error { v = replaceEscaping(v, true) if k == "" { - return errors.New("empty key") + return fmt.Errorf("empty key in node attribute: '%s'", attrs[i]) } else if v == "" { - return errors.New("empty value") + return fmt.Errorf("empty value in node attribute: '%s'", attrs[i]) } dst.SetAttribute(k, v) From f437ab8f15800b2b1baaca87afd3d0aa29e49a39 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 7 Jun 2023 13:04:55 +0300 Subject: [PATCH 0383/1943] [#197] object: Make Delete method return correct status Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/services/object/delete/exec.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index 91bc6b3d7..343efbc04 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + apiclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -75,8 +76,9 @@ func (exec *execCtx) newAddress(id oid.ID) oid.Address { } func (exec *execCtx) formSplitInfo(ctx context.Context) bool { - var err error + success := false + var err error exec.splitInfo, err = exec.svc.header.splitInfo(ctx, exec) switch { @@ -87,12 +89,17 @@ func (exec *execCtx) formSplitInfo(ctx context.Context) bool { exec.log.Debug(logs.DeleteCouldNotComposeSplitInfo, zap.String("error", err.Error()), ) - case err == nil: + case err == nil, apiclient.IsErrObjectAlreadyRemoved(err): + // IsErrObjectAlreadyRemoved check is required because splitInfo + // implicitly performs Head request that may return ObjectAlreadyRemoved + // status that is not specified for Delete + exec.status = statusOK exec.err = nil + success = true } - return err == nil + return success } func (exec *execCtx) collectMembers(ctx context.Context) (ok bool) { From 0c5b0257889bf13743fe30f615f9707676ad0bcd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 26 Jun 2023 18:27:42 +0300 Subject: [PATCH 0384/1943] [#470] grpc: Increase message limits For send message limit set to 2GiB, but there are custom GET/GET RANGE limiters. For receive message limit set to 256 MiB, but actual chunk size will be managed by client. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/grpc.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index b62ae9c45..1dd0f0729 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -17,11 +17,13 @@ import ( "google.golang.org/grpc/credentials" ) +const maxRecvMsgSize = 256 << 20 + func initGRPC(c *cfg) { var successCount int grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) { serverOpts := []grpc.ServerOption{ - grpc.MaxSendMsgSize(maxMsgSize), + grpc.MaxRecvMsgSize(maxRecvMsgSize), grpc.ChainUnaryInterceptor( metrics.NewUnaryServerInterceptor(), tracing.NewUnaryServerInterceptor(), From f9730f090d1e9066d423f4a7b291bc9ffbb17a6b Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 29 Jun 2023 12:13:01 +0300 Subject: [PATCH 0385/1943] [#92] Refactor policer and add some unit tests Signed-off-by: Alejandro Lopez --- cmd/frostfs-node/keyspaceiterator.go | 28 +++ cmd/frostfs-node/object.go | 22 +- pkg/services/control/server/evacuate.go | 11 +- pkg/services/policer/check.go | 88 ++----- pkg/services/policer/check_test.go | 8 +- pkg/services/policer/nodecache.go | 57 +++++ pkg/services/policer/option.go | 185 +++++++++++++++ pkg/services/policer/policer.go | 149 ------------ pkg/services/policer/policer_test.go | 299 ++++++++++++++++++++++++ pkg/services/policer/process.go | 18 +- pkg/services/policer/queue.go | 26 --- pkg/services/replicator/process.go | 24 +- pkg/services/replicator/task.go | 35 +-- 13 files changed, 640 insertions(+), 310 deletions(-) create mode 100644 cmd/frostfs-node/keyspaceiterator.go create mode 100644 pkg/services/policer/nodecache.go create mode 100644 pkg/services/policer/option.go create mode 100644 pkg/services/policer/policer_test.go delete mode 100644 pkg/services/policer/queue.go diff --git a/cmd/frostfs-node/keyspaceiterator.go b/cmd/frostfs-node/keyspaceiterator.go new file mode 100644 index 000000000..8991964a0 --- /dev/null +++ b/cmd/frostfs-node/keyspaceiterator.go @@ -0,0 +1,28 @@ +package main + +import ( + "context" + "fmt" + + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" +) + +type keySpaceIterator struct { + ng *engine.StorageEngine + cur *engine.Cursor +} + +func (it *keySpaceIterator) Next(ctx context.Context, batchSize uint32) ([]objectcore.AddressWithType, error) { + var prm engine.ListWithCursorPrm + prm.WithCursor(it.cur) + prm.WithCount(batchSize) + + res, err := it.ng.ListWithCursor(ctx, prm) + if err != nil { + return nil, fmt.Errorf("cannot list objects in engine: %w", err) + } + + it.cur = res.Cursor() + return res.AddressList(), nil +} diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index afc4bcb41..4106f5dc1 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -38,6 +38,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -211,15 +212,30 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl ls := c.cfgObject.cfgLocalStorage.localStorage + buryFn := func(ctx context.Context, addr oid.Address) error { + var prm engine.InhumePrm + prm.MarkAsGarbage(addr) + prm.WithForceRemoval() + + _, err := ls.Inhume(ctx, prm) + return err + } + + remoteHeader := headsvc.NewRemoteHeader(keyStorage, clientConstructor) + pol := policer.New( policer.WithLogger(c.log), - policer.WithLocalStorage(ls), + policer.WithKeySpaceIterator(&keySpaceIterator{ng: ls}), + policer.WithBuryFunc(buryFn), policer.WithContainerSource(c.cfgObject.cnrSource), policer.WithPlacementBuilder( placement.NewNetworkMapSourceBuilder(c.netMapSource), ), - policer.WithRemoteHeader( - headsvc.NewRemoteHeader(keyStorage, clientConstructor), + policer.WithRemoteObjectHeaderFunc( + func(ctx context.Context, ni netmapSDK.NodeInfo, a oid.Address) (*objectSDK.Object, error) { + prm := new(headsvc.RemoteHeadPrm).WithNodeInfo(ni).WithObjectAddress(a) + return remoteHeader.Head(ctx, prm) + }, ), policer.WithNetmapKeys(c), policer.WithHeadTimeout( diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index fc6dd3f60..8f62c3489 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -84,11 +84,12 @@ func (s *Server) replicate(ctx context.Context, addr oid.Address, obj *objectSDK } var res replicatorResult - var task replicator.Task - task.SetObject(obj) - task.SetObjectAddress(addr) - task.SetCopiesNumber(1) - task.SetNodes(nodes) + task := replicator.Task{ + NumCopies: 1, + Addr: addr, + Obj: obj, + Nodes: nodes, + } s.replicator.HandleTask(ctx, task, &res) if res.count == 0 { diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index e91b8871b..db297b68a 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -7,8 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - headsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/head" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -18,55 +16,6 @@ import ( "go.uber.org/zap" ) -// tracks Policer's check progress. -type nodeCache map[uint64]bool - -func newNodeCache() *nodeCache { - m := make(map[uint64]bool) - return (*nodeCache)(&m) -} - -func (n *nodeCache) set(node netmap.NodeInfo, val bool) { - (*n)[node.Hash()] = val -} - -// submits storage node as a candidate to store the object replica in case of -// shortage. -func (n *nodeCache) submitReplicaCandidate(node netmap.NodeInfo) { - n.set(node, false) -} - -// submits storage node as a current object replica holder. -func (n *nodeCache) submitReplicaHolder(node netmap.NodeInfo) { - n.set(node, true) -} - -// processStatus returns current processing status of the storage node -// -// >0 if node does not currently hold the object -// 0 if node already holds the object -// <0 if node has not been processed yet -func (n *nodeCache) processStatus(node netmap.NodeInfo) int8 { - val, ok := (*n)[node.Hash()] - if !ok { - return -1 - } - - if val { - return 0 - } - - return 1 -} - -// SubmitSuccessfulReplication marks given storage node as a current object -// replica holder. -// -// SubmitSuccessfulReplication implements replicator.TaskResult. -func (n *nodeCache) SubmitSuccessfulReplication(node netmap.NodeInfo) { - n.submitReplicaHolder(node) -} - func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.AddressWithType) { addr := addrWithType.Address idCnr := addr.Container() @@ -79,11 +28,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add zap.String("error", err.Error()), ) if container.IsErrNotFound(err) { - var prm engine.InhumePrm - prm.MarkAsGarbage(addrWithType.Address) - prm.WithForceRemoval() - - _, err := p.jobQueue.localStorage.Inhume(ctx, prm) + err := p.buryFn(ctx, addrWithType.Address) if err != nil { p.log.Error(logs.PolicerCouldNotInhumeObjectWithMissingContainer, zap.Stringer("cid", idCnr), @@ -145,10 +90,9 @@ type placementRequirements struct { } func (p *Policer) processNodes(ctx context.Context, requirements *placementRequirements, addrWithType objectcore.AddressWithType, - nodes []netmap.NodeInfo, shortage uint32, checkedNodes *nodeCache) { + nodes []netmap.NodeInfo, shortage uint32, checkedNodes nodeCache) { addr := addrWithType.Address typ := addrWithType.Type - prm := new(headsvc.RemoteHeadPrm).WithObjectAddress(addr) // Number of copies that are stored on maintenance nodes. var uncheckedCopies int @@ -175,8 +119,8 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi } else if nodes[i].IsMaintenance() { shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) } else { - if status := checkedNodes.processStatus(nodes[i]); status >= 0 { - if status == 0 { + if status := checkedNodes.processStatus(nodes[i]); status.Processed() { + if status == nodeHoldsObject { // node already contains replica, no need to replicate nodes = append(nodes[:i], nodes[i+1:]...) i-- @@ -188,7 +132,7 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi callCtx, cancel := context.WithTimeout(ctx, p.headTimeout) - _, err := p.remoteHeader.Head(callCtx, prm.WithNodeInfo(nodes[i])) + _, err := p.remoteHeader(callCtx, nodes[i], addr) cancel() @@ -224,7 +168,7 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi // prevent spam with new replicas. // However, additional copies should not be removed in this case, // because we can remove the only copy this way. -func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes *nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { +func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { checkedNodes.submitReplicaHolder(node) shortage-- uncheckedCopies++ @@ -236,25 +180,29 @@ func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes *nodeCach } func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address, requirements *placementRequirements, - nodes []netmap.NodeInfo, checkedNodes *nodeCache, shortage uint32, uncheckedCopies int) { - if shortage > 0 { + nodes []netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) { + switch { + case shortage > 0: p.log.Debug(logs.PolicerShortageOfObjectCopiesDetected, zap.Stringer("object", addr), zap.Uint32("shortage", shortage), ) - var task replicator.Task - task.SetObjectAddress(addr) - task.SetNodes(nodes) - task.SetCopiesNumber(shortage) + task := replicator.Task{ + NumCopies: shortage, + Addr: addr, + Nodes: nodes, + } p.replicator.HandleTask(ctx, task, checkedNodes) - } else if uncheckedCopies > 0 { + + case uncheckedCopies > 0: // If we have more copies than needed, but some of them are from the maintenance nodes, // save the local copy. p.log.Debug(logs.PolicerSomeOfTheCopiesAreStoredOnNodesUnderMaintenance, zap.Int("count", uncheckedCopies)) - } else if uncheckedCopies == 0 { + + case uncheckedCopies == 0: // Safe to remove: checked all copies, shortage == 0. requirements.removeLocalCopy = true } diff --git a/pkg/services/policer/check_test.go b/pkg/services/policer/check_test.go index b40ee90d2..d4c7ccbf9 100644 --- a/pkg/services/policer/check_test.go +++ b/pkg/services/policer/check_test.go @@ -11,14 +11,14 @@ func TestNodeCache(t *testing.T) { cache := newNodeCache() node := netmaptest.NodeInfo() - require.Negative(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeNotProcessed) cache.SubmitSuccessfulReplication(node) - require.Zero(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeHoldsObject) cache.submitReplicaCandidate(node) - require.Positive(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeDoesNotHoldObject) cache.submitReplicaHolder(node) - require.Zero(t, cache.processStatus(node)) + require.Equal(t, cache.processStatus(node), nodeHoldsObject) } diff --git a/pkg/services/policer/nodecache.go b/pkg/services/policer/nodecache.go new file mode 100644 index 000000000..cd47cb0fc --- /dev/null +++ b/pkg/services/policer/nodecache.go @@ -0,0 +1,57 @@ +package policer + +import "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + +type nodeProcessStatus int8 + +const ( + nodeNotProcessed nodeProcessStatus = iota + nodeDoesNotHoldObject + nodeHoldsObject +) + +func (st nodeProcessStatus) Processed() bool { + return st != nodeNotProcessed +} + +// nodeCache tracks Policer's check progress. +type nodeCache map[uint64]bool + +func newNodeCache() nodeCache { + return make(map[uint64]bool) +} + +func (n nodeCache) set(node netmap.NodeInfo, val bool) { + n[node.Hash()] = val +} + +// submits storage node as a candidate to store the object replica in case of +// shortage. +func (n nodeCache) submitReplicaCandidate(node netmap.NodeInfo) { + n.set(node, false) +} + +// submits storage node as a current object replica holder. +func (n nodeCache) submitReplicaHolder(node netmap.NodeInfo) { + n.set(node, true) +} + +// processStatus returns current processing status of the storage node. +func (n nodeCache) processStatus(node netmap.NodeInfo) nodeProcessStatus { + switch val, ok := n[node.Hash()]; { + case !ok: + return nodeNotProcessed + case val: + return nodeHoldsObject + default: + return nodeDoesNotHoldObject + } +} + +// SubmitSuccessfulReplication marks given storage node as a current object +// replica holder. +// +// SubmitSuccessfulReplication implements replicator.TaskResult. +func (n nodeCache) SubmitSuccessfulReplication(node netmap.NodeInfo) { + n.submitReplicaHolder(node) +} diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go new file mode 100644 index 000000000..6f17b2947 --- /dev/null +++ b/pkg/services/policer/option.go @@ -0,0 +1,185 @@ +package policer + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/panjf2000/ants/v2" + "go.uber.org/zap" +) + +// KeySpaceIterator is the interface that allows iterating over the key space +// of local storage. +// Note that the underlying implementation might be circular: i.e. it can restart +// when the end of the key space is reached. +type KeySpaceIterator interface { + Next(context.Context, uint32) ([]objectcore.AddressWithType, error) +} + +// RedundantCopyCallback is a callback to pass +// the redundant local copy of the object. +type RedundantCopyCallback func(context.Context, oid.Address) + +// BuryFunc is the function to bury (i.e. inhume) an object. +type BuryFunc func(context.Context, oid.Address) error + +// Replicator is the interface to a consumer of replication tasks. +type Replicator interface { + HandleTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) +} + +// RemoteObjectHeaderFunc is the function to obtain HEAD info from a specific remote node. +type RemoteObjectHeaderFunc func(context.Context, netmapSDK.NodeInfo, oid.Address) (*objectSDK.Object, error) + +// NodeLoader provides application load statistics. +type nodeLoader interface { + // ObjectServiceLoad returns object service load value in [0:1] range. + ObjectServiceLoad() float64 +} + +type cfg struct { + headTimeout time.Duration + + log *logger.Logger + + keySpaceIterator KeySpaceIterator + + buryFn BuryFunc + + cnrSrc container.Source + + placementBuilder placement.Builder + + remoteHeader RemoteObjectHeaderFunc + + netmapKeys netmap.AnnouncedKeys + + replicator Replicator + + cbRedundantCopy RedundantCopyCallback + + taskPool *ants.Pool + + loader nodeLoader + + maxCapacity int + + batchSize, cacheSize uint32 + + rebalanceFreq, evictDuration time.Duration +} + +func defaultCfg() *cfg { + return &cfg{ + log: &logger.Logger{Logger: zap.L()}, + batchSize: 10, + cacheSize: 1024, // 1024 * address size = 1024 * 64 = 64 MiB + rebalanceFreq: 1 * time.Second, + evictDuration: 30 * time.Second, + } +} + +// Option is an option for Policer constructor. +type Option func(*cfg) + +// WithHeadTimeout returns option to set Head timeout of Policer. +func WithHeadTimeout(v time.Duration) Option { + return func(c *cfg) { + c.headTimeout = v + } +} + +// WithLogger returns option to set Logger of Policer. +func WithLogger(v *logger.Logger) Option { + return func(c *cfg) { + c.log = v + } +} + +func WithKeySpaceIterator(it KeySpaceIterator) Option { + return func(c *cfg) { + c.keySpaceIterator = it + } +} + +func WithBuryFunc(f BuryFunc) Option { + return func(c *cfg) { + c.buryFn = f + } +} + +// WithContainerSource returns option to set container source of Policer. +func WithContainerSource(v container.Source) Option { + return func(c *cfg) { + c.cnrSrc = v + } +} + +// WithPlacementBuilder returns option to set object placement builder of Policer. +func WithPlacementBuilder(v placement.Builder) Option { + return func(c *cfg) { + c.placementBuilder = v + } +} + +// WithRemoteObjectHeader returns option to set object header receiver of Policer. +func WithRemoteObjectHeaderFunc(v RemoteObjectHeaderFunc) Option { + return func(c *cfg) { + c.remoteHeader = v + } +} + +// WithNetmapKeys returns option to set tool to work with announced public keys. +func WithNetmapKeys(v netmap.AnnouncedKeys) Option { + return func(c *cfg) { + c.netmapKeys = v + } +} + +// WithReplicator returns option to set object replicator of Policer. +func WithReplicator(v Replicator) Option { + return func(c *cfg) { + c.replicator = v + } +} + +// WithRedundantCopyCallback returns option to set +// callback to pass redundant local object copies +// detected by Policer. +func WithRedundantCopyCallback(cb RedundantCopyCallback) Option { + return func(c *cfg) { + c.cbRedundantCopy = cb + } +} + +// WithMaxCapacity returns option to set max capacity +// that can be set to the pool. +func WithMaxCapacity(capacity int) Option { + return func(c *cfg) { + c.maxCapacity = capacity + } +} + +// WithPool returns option to set pool for +// policy and replication operations. +func WithPool(p *ants.Pool) Option { + return func(c *cfg) { + c.taskPool = p + } +} + +// WithNodeLoader returns option to set FrostFS node load source. +func WithNodeLoader(l nodeLoader) Option { + return func(c *cfg) { + c.loader = l + } +} diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index 541ab599c..4be13e0d8 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -1,29 +1,15 @@ package policer import ( - "context" "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - headsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/head" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" - "github.com/panjf2000/ants/v2" "go.uber.org/zap" ) -// NodeLoader provides application load statistics. -type nodeLoader interface { - // ObjectServiceLoad returns object service load value in [0:1] range. - ObjectServiceLoad() float64 -} - type objectsInWork struct { m sync.RWMutex objs map[oid.Address]struct{} @@ -59,53 +45,6 @@ type Policer struct { objsInWork *objectsInWork } -// Option is an option for Policer constructor. -type Option func(*cfg) - -// RedundantCopyCallback is a callback to pass -// the redundant local copy of the object. -type RedundantCopyCallback func(context.Context, oid.Address) - -type cfg struct { - headTimeout time.Duration - - log *logger.Logger - - jobQueue jobQueue - - cnrSrc container.Source - - placementBuilder placement.Builder - - remoteHeader *headsvc.RemoteHeader - - netmapKeys netmap.AnnouncedKeys - - replicator *replicator.Replicator - - cbRedundantCopy RedundantCopyCallback - - taskPool *ants.Pool - - loader nodeLoader - - maxCapacity int - - batchSize, cacheSize uint32 - - rebalanceFreq, evictDuration time.Duration -} - -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - batchSize: 10, - cacheSize: 1024, // 1024 * address size = 1024 * 64 = 64 MiB - rebalanceFreq: 1 * time.Second, - evictDuration: 30 * time.Second, - } -} - // New creates, initializes and returns Policer instance. func New(opts ...Option) *Policer { c := defaultCfg() @@ -129,91 +68,3 @@ func New(opts ...Option) *Policer { }, } } - -// WithHeadTimeout returns option to set Head timeout of Policer. -func WithHeadTimeout(v time.Duration) Option { - return func(c *cfg) { - c.headTimeout = v - } -} - -// WithLogger returns option to set Logger of Policer. -func WithLogger(v *logger.Logger) Option { - return func(c *cfg) { - c.log = v - } -} - -// WithLocalStorage returns option to set local object storage of Policer. -func WithLocalStorage(v *engine.StorageEngine) Option { - return func(c *cfg) { - c.jobQueue.localStorage = v - } -} - -// WithContainerSource returns option to set container source of Policer. -func WithContainerSource(v container.Source) Option { - return func(c *cfg) { - c.cnrSrc = v - } -} - -// WithPlacementBuilder returns option to set object placement builder of Policer. -func WithPlacementBuilder(v placement.Builder) Option { - return func(c *cfg) { - c.placementBuilder = v - } -} - -// WithRemoteHeader returns option to set object header receiver of Policer. -func WithRemoteHeader(v *headsvc.RemoteHeader) Option { - return func(c *cfg) { - c.remoteHeader = v - } -} - -// WithNetmapKeys returns option to set tool to work with announced public keys. -func WithNetmapKeys(v netmap.AnnouncedKeys) Option { - return func(c *cfg) { - c.netmapKeys = v - } -} - -// WithReplicator returns option to set object replicator of Policer. -func WithReplicator(v *replicator.Replicator) Option { - return func(c *cfg) { - c.replicator = v - } -} - -// WithRedundantCopyCallback returns option to set -// callback to pass redundant local object copies -// detected by Policer. -func WithRedundantCopyCallback(cb RedundantCopyCallback) Option { - return func(c *cfg) { - c.cbRedundantCopy = cb - } -} - -// WithMaxCapacity returns option to set max capacity -// that can be set to the pool. -func WithMaxCapacity(capacity int) Option { - return func(c *cfg) { - c.maxCapacity = capacity - } -} - -// WithPool returns option to set pool for -// policy and replication operations. -func WithPool(p *ants.Pool) Option { - return func(c *cfg) { - c.taskPool = p - } -} - -// WithNodeLoader returns option to set FrostFS node load source. -func WithNodeLoader(l nodeLoader) Option { - return func(c *cfg) { - c.loader = l - } -} diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go new file mode 100644 index 000000000..a09957895 --- /dev/null +++ b/pkg/services/policer/policer_test.go @@ -0,0 +1,299 @@ +package policer + +import ( + "bytes" + "context" + "errors" + "sort" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/panjf2000/ants/v2" + "github.com/stretchr/testify/require" +) + +func TestBuryObjectWithoutContainer(t *testing.T) { + // Key space + addr := oidtest.Address() + objs := []objectcore.AddressWithType{ + { + Address: addr, + Type: object.TypeRegular, + }, + } + + // Container source and bury function + buryCh := make(chan oid.Address) + containerSrc := func(id cid.ID) (*container.Container, error) { + return nil, apistatus.ContainerNotFound{} + } + buryFn := func(ctx context.Context, a oid.Address) error { + buryCh <- a + return nil + } + + // Task pool + pool, err := ants.NewPool(4) + require.NoError(t, err) + + // Policer instance + p := New( + WithKeySpaceIterator(&sliceKeySpaceIterator{objs: objs}), + WithContainerSource(containerSrcFunc(containerSrc)), + WithBuryFunc(buryFn), + WithPool(pool), + WithNodeLoader(constNodeLoader(0)), + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Run(ctx) + + require.Equal(t, addr, <-buryCh) +} + +func TestProcessObject(t *testing.T) { + // Notes: + // - nodes are referred to by their index throughout, which is embedded in the public key + // - node with index 0 always refers to the local node, so there's no need to add it to objHolders + // - policy is used only to match the number of replicas for each index in the placement + tests := []struct { + desc string + objType object.Type + nodeCount int + policy string + placement [][]int + objHolders []int + maintenanceNodes []int + wantRemoveRedundant bool + wantReplicateTo []int + }{ + { + desc: "1 copy already held by local node", + nodeCount: 1, + policy: `REP 1`, + placement: [][]int{{0}}, + }, + { + desc: "1 copy already held by the remote node", + nodeCount: 2, + policy: `REP 1`, + placement: [][]int{{1}}, + objHolders: []int{1}, + wantRemoveRedundant: true, + }, + { + desc: "1 copy not yet held by the remote node", + nodeCount: 2, + policy: `REP 1`, + placement: [][]int{{1}}, + wantReplicateTo: []int{1}, + }, + { + desc: "2 copies already held by local and remote node", + nodeCount: 2, + policy: `REP 2`, + placement: [][]int{{0, 1}}, + objHolders: []int{1}, + }, + { + desc: "2 copies but not held by remote node", + nodeCount: 2, + policy: `REP 2`, + placement: [][]int{{0, 1}}, + wantReplicateTo: []int{1}, + }, + { + desc: "multiple vectors already held by remote node", + nodeCount: 2, + policy: `REP 2 REP 2`, + placement: [][]int{{0, 1}, {0, 1}}, + objHolders: []int{1}, + }, + { + desc: "multiple vectors not yet held by remote node", + nodeCount: 2, + policy: `REP 2 REP 2`, + placement: [][]int{{0, 1}, {0, 1}}, + wantReplicateTo: []int{1, 1}, // is this actually good? + }, + { + desc: "lock object must be replicated to all nodes", + objType: object.TypeLock, + nodeCount: 3, + policy: `REP 1`, + placement: [][]int{{0, 1, 2}}, + wantReplicateTo: []int{1, 2}, + }, + { + desc: "preserve local copy when maintenance nodes exist", + nodeCount: 3, + policy: `REP 2`, + placement: [][]int{{1, 2}}, + objHolders: []int{1}, + maintenanceNodes: []int{2}, + }, + } + + for i := range tests { + ti := tests[i] + t.Run(ti.desc, func(t *testing.T) { + addr := oidtest.Address() + + // Netmap, placement policy and placement builder + nodes := make([]netmap.NodeInfo, ti.nodeCount) + for i := range nodes { + nodes[i].SetPublicKey([]byte{byte(i)}) + } + for _, i := range ti.maintenanceNodes { + nodes[i].SetMaintenance() + } + + var policy netmap.PlacementPolicy + require.NoError(t, policy.DecodeString(ti.policy)) + + placementVectors := make([][]netmap.NodeInfo, len(ti.placement)) + for i, pv := range ti.placement { + for _, nj := range pv { + placementVectors[i] = append(placementVectors[i], nodes[nj]) + } + } + placementBuilder := func(cnr cid.ID, obj *oid.ID, p netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { + if cnr.Equals(addr.Container()) && obj != nil && obj.Equals(addr.Object()) { + return placementVectors, nil + } + t.Errorf("unexpected placement build: cid=%v oid=%v", cnr, obj) + return nil, errors.New("unexpected placement build") + } + + // Object remote header + headFn := func(_ context.Context, ni netmap.NodeInfo, a oid.Address) (*object.Object, error) { + index := int(ni.PublicKey()[0]) + if a != addr || index < 1 || index >= ti.nodeCount { + t.Errorf("unexpected remote object head: node=%+v addr=%v", ni, a) + return nil, errors.New("unexpected object head") + } + for _, i := range ti.objHolders { + if index == i { + return nil, nil + } + } + return nil, apistatus.ObjectNotFound{} + } + + // Container source + cnr := &container.Container{} + cnr.Value.Init() + cnr.Value.SetPlacementPolicy(policy) + containerSrc := func(id cid.ID) (*container.Container, error) { + if id.Equals(addr.Container()) { + return cnr, nil + } + t.Errorf("unexpected container requested: got=%v, want=%v", id, addr.Container()) + return nil, apistatus.ContainerNotFound{} + } + buryFn := func(ctx context.Context, a oid.Address) error { + t.Errorf("unexpected object buried: %v", a) + return nil + } + + // Policer instance + var gotRemoveRedundant bool + var gotReplicateTo []int + + p := New( + WithContainerSource(containerSrcFunc(containerSrc)), + WithPlacementBuilder(placementBuilderFunc(placementBuilder)), + WithNetmapKeys(announcedKeysFunc(func(k []byte) bool { + return bytes.Equal(k, nodes[0].PublicKey()) + })), + WithRemoteObjectHeaderFunc(headFn), + WithBuryFunc(buryFn), + WithRedundantCopyCallback(func(_ context.Context, a oid.Address) { + require.True(t, eqAddr(a, addr), "unexpected redundant copy callback: a=%v", a) + gotRemoveRedundant = true + }), + WithReplicator(replicatorFunc(func(_ context.Context, task replicator.Task, res replicator.TaskResult) { + require.True(t, eqAddr(task.Addr, addr), "unexpected replicator task: %+v", task) + for _, node := range task.Nodes { + gotReplicateTo = append(gotReplicateTo, int(node.PublicKey()[0])) + } + })), + ) + + addrWithType := objectcore.AddressWithType{ + Address: addr, + Type: ti.objType, + } + + p.processObject(context.Background(), addrWithType) + sort.Ints(gotReplicateTo) + + require.Equal(t, ti.wantRemoveRedundant, gotRemoveRedundant) + require.Equal(t, ti.wantReplicateTo, gotReplicateTo) + }) + } +} + +// TODO(https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/101) +func eqAddr(a, b oid.Address) bool { + return a.Container().Equals(b.Container()) && a.Object().Equals(b.Object()) +} + +// sliceKeySpaceIterator is a KeySpaceIterator backed by a slice. +type sliceKeySpaceIterator struct { + objs []objectcore.AddressWithType + cur int +} + +func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectcore.AddressWithType, error) { + if it.cur >= len(it.objs) { + it.cur = 0 + return nil, engine.ErrEndOfListing + } + end := it.cur + int(size) + if end > len(it.objs) { + end = len(it.objs) + } + ret := it.objs[it.cur:end] + it.cur = end + return ret, nil +} + +// containerSrcFunc is a container.Source backed by a function. +type containerSrcFunc func(cid.ID) (*container.Container, error) + +func (f containerSrcFunc) Get(id cid.ID) (*container.Container, error) { return f(id) } + +// placementBuilderFunc is a placement.Builder backed by a function +type placementBuilderFunc func(cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) + +func (f placementBuilderFunc) BuildPlacement(c cid.ID, o *oid.ID, p netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { + return f(c, o, p) +} + +// announcedKeysFunc is a netmap.AnnouncedKeys backed by a function. +type announcedKeysFunc func([]byte) bool + +func (f announcedKeysFunc) IsLocalKey(k []byte) bool { return f(k) } + +// constNodeLoader is a nodeLoader that always returns a fixed value. +type constNodeLoader float64 + +func (f constNodeLoader) ObjectServiceLoad() float64 { return float64(f) } + +// replicatorFunc is a Replicator backed by a function. +type replicatorFunc func(context.Context, replicator.Task, replicator.TaskResult) + +func (f replicatorFunc) HandleTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) { + f(ctx, task, res) +} diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 60e924755..6bd38f618 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -6,27 +6,17 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "go.uber.org/zap" ) func (p *Policer) Run(ctx context.Context) { - defer func() { - p.log.Info(logs.PolicerRoutineStopped) - }() - go p.poolCapacityWorker(ctx) p.shardPolicyWorker(ctx) + p.log.Info(logs.PolicerRoutineStopped) } func (p *Policer) shardPolicyWorker(ctx context.Context) { - var ( - addrs []objectcore.AddressWithType - cursor *engine.Cursor - err error - ) - for { select { case <-ctx.Done(): @@ -34,7 +24,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { default: } - addrs, cursor, err = p.jobQueue.Select(ctx, cursor, p.batchSize) + addrs, err := p.keySpaceIterator.Next(ctx, p.batchSize) if err != nil { if errors.Is(err, engine.ErrEndOfListing) { time.Sleep(time.Second) // finished whole cycle, sleep a bit @@ -55,7 +45,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { continue } - err = p.taskPool.Submit(func() { + err := p.taskPool.Submit(func() { v, ok := p.cache.Get(addr.Address) if ok && time.Since(v) < p.evictDuration { return @@ -78,10 +68,10 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { func (p *Policer) poolCapacityWorker(ctx context.Context) { ticker := time.NewTicker(p.rebalanceFreq) + defer ticker.Stop() for { select { case <-ctx.Done(): - ticker.Stop() return case <-ticker.C: frostfsSysLoad := p.loader.ObjectServiceLoad() diff --git a/pkg/services/policer/queue.go b/pkg/services/policer/queue.go deleted file mode 100644 index 22012c835..000000000 --- a/pkg/services/policer/queue.go +++ /dev/null @@ -1,26 +0,0 @@ -package policer - -import ( - "context" - "fmt" - - objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" -) - -type jobQueue struct { - localStorage *engine.StorageEngine -} - -func (q *jobQueue) Select(ctx context.Context, cursor *engine.Cursor, count uint32) ([]objectcore.AddressWithType, *engine.Cursor, error) { - var prm engine.ListWithCursorPrm - prm.WithCursor(cursor) - prm.WithCount(count) - - res, err := q.localStorage.ListWithCursor(ctx, prm) - if err != nil { - return nil, nil, fmt.Errorf("cannot list objects in engine: %w", err) - } - - return res.AddressList(), res.Cursor(), nil -} diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 0f82ff232..a54668e12 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -24,16 +24,16 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) defer p.metrics.DecInFlightRequest() defer func() { p.log.Debug(logs.ReplicatorFinishWork, - zap.Uint32("amount of unfinished replicas", task.quantity), + zap.Uint32("amount of unfinished replicas", task.NumCopies), ) }() - if task.obj == nil { + if task.Obj == nil { var err error - task.obj, err = engine.Get(ctx, p.localStorage, task.addr) + task.Obj, err = engine.Get(ctx, p.localStorage, task.Addr) if err != nil { p.log.Error(logs.ReplicatorCouldNotGetObjectFromLocalStorage, - zap.Stringer("object", task.addr), + zap.Stringer("object", task.Addr), zap.Error(err)) return @@ -41,9 +41,9 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) } prm := new(putsvc.RemotePutPrm). - WithObject(task.obj) + WithObject(task.Obj) - for i := 0; task.quantity > 0 && i < len(task.nodes); i++ { + for i := 0; task.NumCopies > 0 && i < len(task.Nodes); i++ { select { case <-ctx.Done(): return @@ -51,13 +51,13 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) } log := p.log.With( - zap.String("node", netmap.StringifyPublicKey(task.nodes[i])), - zap.Stringer("object", task.addr), + zap.String("node", netmap.StringifyPublicKey(task.Nodes[i])), + zap.Stringer("object", task.Addr), ) callCtx, cancel := context.WithTimeout(ctx, p.putTimeout) - err := p.remoteSender.PutObject(callCtx, prm.WithNodeInfo(task.nodes[i])) + err := p.remoteSender.PutObject(callCtx, prm.WithNodeInfo(task.Nodes[i])) cancel() @@ -68,12 +68,12 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) } else { log.Debug(logs.ReplicatorObjectSuccessfullyReplicated) - task.quantity-- + task.NumCopies-- - res.SubmitSuccessfulReplication(task.nodes[i]) + res.SubmitSuccessfulReplication(task.Nodes[i]) p.metrics.IncProcessedObjects() - p.metrics.AddPayloadSize(int64(task.obj.PayloadSize())) + p.metrics.AddPayloadSize(int64(task.Obj.PayloadSize())) } } } diff --git a/pkg/services/replicator/task.go b/pkg/services/replicator/task.go index ec1b55788..d2b5b2506 100644 --- a/pkg/services/replicator/task.go +++ b/pkg/services/replicator/task.go @@ -8,31 +8,12 @@ import ( // Task represents group of Replicator task parameters. type Task struct { - quantity uint32 - - addr oid.Address - - obj *objectSDK.Object - - nodes []netmap.NodeInfo -} - -// SetCopiesNumber sets number of copies to replicate. -func (t *Task) SetCopiesNumber(v uint32) { - t.quantity = v -} - -// SetObjectAddress sets address of local object. -func (t *Task) SetObjectAddress(v oid.Address) { - t.addr = v -} - -// SetObject sets object to avoid fetching it from the local storage. -func (t *Task) SetObject(obj *objectSDK.Object) { - t.obj = obj -} - -// SetNodes sets a list of potential object holders. -func (t *Task) SetNodes(v []netmap.NodeInfo) { - t.nodes = v + // NumCopies is the number of copies to replicate. + NumCopies uint32 + // Addr is the address of the local object. + Addr oid.Address + // Obj is the object to avoid fetching it from the local storage. + Obj *objectSDK.Object + // Nodes is a list of potential object holders. + Nodes []netmap.NodeInfo } From 3223402c9050db37ff421560e855cd01e967a989 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 30 Jun 2023 10:08:36 +0300 Subject: [PATCH 0386/1943] [#92] Embed policer's objectsInWork mutex Signed-off-by: Alejandro Lopez --- pkg/services/policer/policer.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index 4be13e0d8..d49ad4632 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -11,28 +11,28 @@ import ( ) type objectsInWork struct { - m sync.RWMutex + sync.RWMutex objs map[oid.Address]struct{} } func (oiw *objectsInWork) inWork(addr oid.Address) bool { - oiw.m.RLock() + oiw.RLock() _, ok := oiw.objs[addr] - oiw.m.RUnlock() + oiw.RUnlock() return ok } func (oiw *objectsInWork) remove(addr oid.Address) { - oiw.m.Lock() + oiw.Lock() delete(oiw.objs, addr) - oiw.m.Unlock() + oiw.Unlock() } func (oiw *objectsInWork) add(addr oid.Address) { - oiw.m.Lock() + oiw.Lock() oiw.objs[addr] = struct{}{} - oiw.m.Unlock() + oiw.Unlock() } // Policer represents the utility that verifies From 26acf5689e19eeb15248a12eb5e4a3a8477e6fda Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 30 Jun 2023 10:12:17 +0300 Subject: [PATCH 0387/1943] [#92] Ensure policer objects cannot be worked on concurrently Signed-off-by: Alejandro Lopez --- pkg/services/policer/policer.go | 4 +++- pkg/services/policer/process.go | 11 +++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index d49ad4632..a68b194d4 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -29,10 +29,12 @@ func (oiw *objectsInWork) remove(addr oid.Address) { oiw.Unlock() } -func (oiw *objectsInWork) add(addr oid.Address) { +func (oiw *objectsInWork) add(addr oid.Address) bool { oiw.Lock() + _, exists := oiw.objs[addr] oiw.objs[addr] = struct{}{} oiw.Unlock() + return !exists } // Policer represents the utility that verifies diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 6bd38f618..39b61c8a0 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -51,12 +51,11 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { return } - p.objsInWork.add(addr.Address) - - p.processObject(ctx, addr) - - p.cache.Add(addr.Address, time.Now()) - p.objsInWork.remove(addr.Address) + if p.objsInWork.add(addr.Address) { + p.processObject(ctx, addr) + p.cache.Add(addr.Address, time.Now()) + p.objsInWork.remove(addr.Address) + } }) if err != nil { p.log.Warn(logs.PolicerPoolSubmission, zap.Error(err)) From 8d16d95376a7631ad8078cb988a056b8719a0291 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 29 Jun 2023 10:26:38 +0300 Subject: [PATCH 0388/1943] [#484] morph: Add expired tx logging Signed-off-by: Dmitrii Stepanov --- pkg/morph/event/listener.go | 5 ++++- pkg/morph/event/notary_preparator.go | 18 ++++++++++++++---- pkg/morph/event/notary_preparator_test.go | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 405165702..ca5031415 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -365,11 +365,14 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { // prepare the notary event notaryEvent, err := l.notaryEventsPreparator.Prepare(nr.NotaryRequest) if err != nil { + var expErr *ExpiredTXError switch { case errors.Is(err, ErrTXAlreadyHandled): - case errors.Is(err, ErrMainTXExpired): + case errors.As(err, &expErr): l.log.Warn(logs.EventSkipExpiredMainTXNotaryEvent, zap.String("error", err.Error()), + zap.Uint32("current_block_height", expErr.CurrentBlockHeight), + zap.Uint32("fallback_tx_not_valid_before_height", expErr.FallbackTXNotValidBeforeHeight), ) default: l.log.Warn(logs.EventCouldNotPrepareAndValidateNotaryEvent, diff --git a/pkg/morph/event/notary_preparator.go b/pkg/morph/event/notary_preparator.go index a8b7376fa..298a6d574 100644 --- a/pkg/morph/event/notary_preparator.go +++ b/pkg/morph/event/notary_preparator.go @@ -39,11 +39,18 @@ var ( // ErrTXAlreadyHandled is returned if received TX has already been signed. ErrTXAlreadyHandled = errors.New("received main tx has already been handled") - - // ErrMainTXExpired is returned if received fallback TX is already valid. - ErrMainTXExpired = errors.New("received main tx has expired") ) +// ExpiredTXError is returned if received fallback TX is already valid. +type ExpiredTXError struct { + CurrentBlockHeight uint32 + FallbackTXNotValidBeforeHeight uint32 +} + +func (e *ExpiredTXError) Error() string { + return "received main tx has expired" +} + // BlockCounter must return block count of the network // from which notary requests are received. type BlockCounter interface { @@ -308,7 +315,10 @@ func (p Preparator) validateExpiration(fbTX *transaction.Transaction) error { } if currBlock >= nvb.Height { - return ErrMainTXExpired + return &ExpiredTXError{ + CurrentBlockHeight: currBlock, + FallbackTXNotValidBeforeHeight: nvb.Height, + } } return nil diff --git a/pkg/morph/event/notary_preparator_test.go b/pkg/morph/event/notary_preparator_test.go index b2e46890b..8da9d868a 100644 --- a/pkg/morph/event/notary_preparator_test.go +++ b/pkg/morph/event/notary_preparator_test.go @@ -331,7 +331,7 @@ func TestPrepare_IncorrectNR(t *testing.T) { {}, }, }, - expErr: ErrMainTXExpired, + expErr: &ExpiredTXError{}, }, { name: "incorrect invoker TX Alphabet witness", From 90e9a85acca029521ed3ab23c20f5d0e977b80c8 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 29 Jun 2023 09:44:11 +0300 Subject: [PATCH 0389/1943] [#483] Update dependencies Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/internal/client/client.go | 6 +- go.mod | 69 +++---- go.sum | 180 +++++++++--------- pkg/core/client/client.go | 2 +- pkg/network/cache/multi.go | 2 +- pkg/services/object/internal/client/client.go | 6 +- 6 files changed, 130 insertions(+), 135 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 0e0aadbb3..d0bbc9cff 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -396,7 +396,7 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { return nil, fmt.Errorf("init object writing: %w", err) } - if wrt.WriteHeader(*prm.hdr) { + if wrt.WriteHeader(ctx, *prm.hdr) { if prm.headerCallback != nil { prm.headerCallback(prm.hdr) } @@ -426,7 +426,7 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { for { n, err = prm.rdr.Read(buf) if n > 0 { - if !wrt.WritePayloadChunk(buf[:n]) { + if !wrt.WritePayloadChunk(ctx, buf[:n]) { break } @@ -442,7 +442,7 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { } } - cliRes, err := wrt.Close() + cliRes, err := wrt.Close(ctx) if err != nil { // here err already carries both status and client errors return nil, fmt.Errorf("client failure: %w", err) } diff --git a/go.mod b/go.mod index aca1d2d6f..4be07aef7 100644 --- a/go.mod +++ b/go.mod @@ -4,40 +4,40 @@ go 1.19 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb - git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb + git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230623104802-aa8ffebc6328 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230628121302-5d62cef27e6c git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/uuid v1.3.0 - github.com/hashicorp/golang-lru/v2 v2.0.2 - github.com/klauspost/compress v1.16.5 + github.com/hashicorp/golang-lru/v2 v2.0.4 + github.com/klauspost/compress v1.16.6 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.9.0 - github.com/nats-io/nats.go v1.25.0 + github.com/nats-io/nats.go v1.27.1 github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc github.com/olekukonko/tablewriter v0.0.5 - github.com/panjf2000/ants/v2 v2.7.4 + github.com/panjf2000/ants/v2 v2.7.5 github.com/paulmach/orb v0.9.2 - github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_golang v1.16.0 github.com/spf13/cast v1.5.1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.15.0 - github.com/stretchr/testify v1.8.3 + github.com/spf13/viper v1.16.0 + github.com/stretchr/testify v1.8.4 go.etcd.io/bbolt v1.3.7 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc - golang.org/x/sync v0.2.0 - golang.org/x/term v0.8.0 - google.golang.org/grpc v1.55.0 - google.golang.org/protobuf v1.30.0 + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df + golang.org/x/sync v0.3.0 + golang.org/x/term v0.9.0 + google.golang.org/grpc v1.56.1 + google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -45,12 +45,13 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect - github.com/benbjohnson/clock v1.1.0 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.8.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.9.1 // indirect + github.com/consensys/gnark-crypto v0.11.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -61,36 +62,36 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 // indirect - github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect - github.com/multiformats/go-multihash v0.2.1 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/nats-io/nats-server/v2 v2.7.4 // indirect github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230601131642-a0117042e8fc // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.7 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.43.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect @@ -99,22 +100,24 @@ require ( github.com/subosito/gotenv v1.4.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect github.com/twmb/murmur3 v1.1.8 // indirect - github.com/urfave/cli v1.22.13 // indirect - go.mongodb.org/mongo-driver v1.11.6 // indirect + github.com/urfave/cli v1.22.14 // indirect + go.mongodb.org/mongo-driver v1.12.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 // indirect go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/otel/sdk v1.16.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.opentelemetry.io/proto/otlp v0.20.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect + google.golang.org/genproto v0.0.0-20230628200519-e449d1ea0e82 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230628200519-e449d1ea0e82 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230628200519-e449d1ea0e82 // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index dfa8632aa..03e604641 100644 --- a/go.sum +++ b/go.sum @@ -38,14 +38,14 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb h1:gaq4wgk7b+i9+rkDa+goPkv5FzqYlMxEuCKdZ57mSc0= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb h1:S/TrbOOu9qEXZRZ9/Ddw7crnxbBUQLo68PSzQWYrc9M= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a h1:EeaOunifY30ATPzOo1F32vUDW299UXDHYukZrIVxBZ0= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230623104802-aa8ffebc6328 h1:d2v3W0vLdcqYTY1/P0scdwUukmBfvt+CzkkD0EvskPA= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230623104802-aa8ffebc6328/go.mod h1:w+s3ozlbFfTDFHhjX0A3Iif3BRtnTkwiACxFZD+Q0cQ= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230628121302-5d62cef27e6c h1:nH9mifyAuiZeyc79DDwNZjbJeomz0YW4Ja6Mk/iuaLI= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230628121302-5d62cef27e6c/go.mod h1:w+s3ozlbFfTDFHhjX0A3Iif3BRtnTkwiACxFZD+Q0cQ= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= @@ -53,13 +53,12 @@ git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+ git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1:MJCkWUBhi9pn/CrYO1Q3P687y2KeahrOPS9BD9LDGb0= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/ishell/v2 v2.0.2/go.mod h1:E4oTCXfo6QjoCart0QYa5m9w4S+deXs/P/9jA77A9Bs= @@ -76,12 +75,15 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= +github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -97,7 +99,6 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -118,15 +119,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark-crypto v0.9.1 h1:mru55qKdWl3E035hAoh1jj9d7hVnYY5pfb6tmovSmII= -github.com/consensys/gnark-crypto v0.9.1/go.mod h1:a2DQL4+5ywF6safEeZFEPGRiiGbjzGFRUN2sg06VuU4= +github.com/consensys/gnark-crypto v0.11.0 h1:QqzHQlwEqlQr5jfWblGDkwlKHpT+4QodYqqExkAtyks= +github.com/consensys/gnark-crypto v0.11.0/go.mod h1:Iq/P3HHl0ElSjsg2E1gsMwhAyxnxoKK5nVyZKd+/KhU= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= @@ -144,8 +141,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= @@ -184,7 +179,6 @@ github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaL github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -263,19 +257,18 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 h1:mdLirNAJBxnGgyB6pjZLcs6ue/6eZGBui6gXspfq4ks= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0/go.mod h1:kdXbOySqcQeTxiqglW7aahTmWZy3Pgi6SYL36yvKeyA= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 h1:o95KDiV/b1xdkumY5YbLR0/n2+wBxUpgf3HgfKgTyLI= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3/go.mod h1:hTxjzRcX49ogbTGVJ1sM5mz5s+SSgiGIyL3jjPxl32E= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5 h1:3IZOAnD058zZllQTZNBioTlrzrBG/IjpiZ133IEtusM= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5/go.mod h1:xbKERva94Pw2cPen0s79J3uXmGzbbpDYFBFDlZ4mV/w= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAszqCJMbfTISJ7oMftp8+UGV08= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.4 h1:7GHuZcgid37q8o5i3QI9KMT4nCWQQ3Kx3Ov6bb9MfK0= +github.com/hashicorp/golang-lru/v2 v2.0.4/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= @@ -306,11 +299,10 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= +github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -343,8 +335,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -369,8 +361,8 @@ github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sgg github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -378,8 +370,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 h1:vU9tpM3apjYlLLeY23zRWJ9Zktr5jp+mloR942LEOpY= github.com/nats-io/nats-server/v2 v2.7.4 h1:c+BZJ3rGzUKCBIM4IXO8uNT2u1vajGbD1kPA6wqCEaM= github.com/nats-io/nats-server/v2 v2.7.4/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc= -github.com/nats-io/nats.go v1.25.0 h1:t5/wCPGciR7X3Mu8QOi4jiJaXaWM8qtkLu4lzGZvYHE= -github.com/nats-io/nats.go v1.25.0/go.mod h1:D2WALIhz7V8M0pH8Scx8JZXlg6Oqz5VG+nQkK8nJdvg= +github.com/nats-io/nats.go v1.27.1 h1:OuYnal9aKVSnOzLQIzf7554OXMCG7KbaTkCSBHRcSoo= +github.com/nats-io/nats.go v1.27.1/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -400,8 +392,8 @@ github.com/nspcc-dev/neo-go v0.99.4/go.mod h1:mKTolfRUfKjFso5HPvGSQtUZc70n0VKBMs github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc h1:fySIWvUQsitK5e5qYIHnTDCXuPpwzz89SEUEIyY11sg= github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc/go.mod h1:s9QhjMC784MWqTURovMbyYduIJc86mnCruxcMiAebpc= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230601131642-a0117042e8fc h1:yJM7a9iW4ZIvos2R5jn/Z7I4iZCqWdBVPclRs3UebzY= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230601131642-a0117042e8fc/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce h1:vLGuUNDkmQrWMa4rr4vTd1u8ULqejWxVmNz1L7ocTEI= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= @@ -429,13 +421,13 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/panjf2000/ants/v2 v2.7.4 h1:mJqMDtMckZltyL458pq81IGNfiDhEgzX5s/lhjwPWIM= -github.com/panjf2000/ants/v2 v2.7.4/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8= +github.com/panjf2000/ants/v2 v2.7.5 h1:/vhh0Hza9G1vP1PdCj9hl6MUzCRbmtcTJL0OsnmytuU= +github.com/panjf2000/ants/v2 v2.7.5/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8= github.com/paulmach/orb v0.9.2 h1:p/YWV2uJwamAynnDOJGNbPBVtDHj3vG51k9tR1rFwJE= github.com/paulmach/orb v0.9.2/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= -github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= -github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -451,8 +443,8 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -465,8 +457,8 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.43.0 h1:iq+BVjvYLei5f27wiuNiB1DN6DYQkp1c8Bx0Vykh5us= -github.com/prometheus/common v0.43.0/go.mod h1:NCvr5cQIh3Y/gy73/RdVtC9r8xxrxwJnB+2lB3BxrFc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= @@ -474,8 +466,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= +github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -490,7 +482,6 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= @@ -503,8 +494,8 @@ github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmq github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= -github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= +github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= +github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -518,34 +509,36 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.13 h1:wsLILXG8qCJNse/qAgLNf23737Cx05GflHg/PJGe1Ok= -github.com/urfave/cli v1.22.13/go.mod h1:VufqObjsMTF2BBwKawpx9R8eAneNEWhoO0yx8Vd+FkE= +github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= +github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -553,8 +546,8 @@ go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.mongodb.org/mongo-driver v1.11.6 h1:XM7G6PjiGAO5betLF13BIa5TlLUUE3uJ/2Ox3Lz1K+o= -go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY= +go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE= +go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -578,8 +571,8 @@ go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.20.0 h1:BLOA1cZBAGSbRiNuGCCKiFrCdYB7deeHDeD1SueyOfA= +go.opentelemetry.io/proto/otlp v0.20.0/go.mod h1:3QgjzPALBIv9pcknj2EXGPXjYPFdUh/RQfF8Lz3+Vnw= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -607,10 +600,11 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -621,8 +615,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -647,7 +641,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -685,15 +680,14 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -703,9 +697,7 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -719,9 +711,10 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -775,10 +768,8 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -786,26 +777,28 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -863,6 +856,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -924,7 +918,6 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -933,9 +926,12 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230628200519-e449d1ea0e82 h1:Wdfp5Hc1bqGCWYZNrir4A1Jb+SmVaV2j1DL/pbMMTGI= +google.golang.org/genproto v0.0.0-20230628200519-e449d1ea0e82/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto/googleapis/api v0.0.0-20230628200519-e449d1ea0e82 h1:iI5Fmsfz4zDINYxJLxn2YChI//ypkHM/KuVSvlN7ZXk= +google.golang.org/genproto/googleapis/api v0.0.0-20230628200519-e449d1ea0e82/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230628200519-e449d1ea0e82 h1:6b+zGQBiXFlAMpQr+cCarAdrZD4QgXSG7uUZadYysgg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230628200519-e449d1ea0e82/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -954,13 +950,9 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= -google.golang.org/grpc/examples v0.0.0-20210424002626-9572fd6faeae/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -975,8 +967,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/core/client/client.go b/pkg/core/client/client.go index bd186006e..422ca1a1a 100644 --- a/pkg/core/client/client.go +++ b/pkg/core/client/client.go @@ -12,7 +12,7 @@ import ( // node's client. type Client interface { ContainerAnnounceUsedSpace(context.Context, client.PrmAnnounceSpace) (*client.ResAnnounceSpace, error) - ObjectPutInit(context.Context, client.PrmObjectPutInit) (*client.ObjectWriter, error) + ObjectPutInit(context.Context, client.PrmObjectPutInit) (client.ObjectWriter, error) ObjectDelete(context.Context, client.PrmObjectDelete) (*client.ResObjectDelete, error) ObjectGetInit(context.Context, client.PrmObjectGet) (*client.ObjectReader, error) ObjectHead(context.Context, client.PrmObjectHead) (*client.ResObjectHead, error) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index ba81df16c..b7f7dc20f 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -219,7 +219,7 @@ func (s *singleClient) invalidate() { s.Unlock() } -func (x *multiClient) ObjectPutInit(ctx context.Context, p client.PrmObjectPutInit) (res *client.ObjectWriter, err error) { +func (x *multiClient) ObjectPutInit(ctx context.Context, p client.PrmObjectPutInit) (res client.ObjectWriter, err error) { err = x.iterateClients(ctx, func(c clientcore.Client) error { res, err = c.ObjectPutInit(ctx, p) return err diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 17bf6a40f..a54f6cd01 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -429,11 +429,11 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { return nil, fmt.Errorf("init object writing on client: %w", err) } - if w.WriteHeader(*prm.obj) { - w.WritePayloadChunk(prm.obj.Payload()) + if w.WriteHeader(ctx, *prm.obj) { + w.WritePayloadChunk(ctx, prm.obj.Payload()) } - cliRes, err := w.Close() + cliRes, err := w.Close(ctx) if err == nil { err = apistatus.ErrFromStatus(cliRes.Status()) } else { From 4c248d573eb7951abca58e6ef89556f6d9f0dc1a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 29 Jun 2023 09:48:32 +0300 Subject: [PATCH 0390/1943] [#483] cli: Allow to split object on the client side Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/internal/client/client.go | 64 +++++++++++++++++------ cmd/frostfs-cli/modules/object/put.go | 36 ++++++++----- 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index d0bbc9cff..9545c60cb 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -336,6 +336,8 @@ type PutObjectPrm struct { rdr io.Reader headerCallback func(*object.Object) + + prepareLocally bool } // SetHeader sets object header. @@ -360,6 +362,41 @@ func (x *PutObjectPrm) SetCopiesNumberByVectors(copiesNumbers []uint32) { x.copyNum = copiesNumbers } +// PrepareLocally generate object header on the client side. +// For big object - split locally too. +func (x *PutObjectPrm) PrepareLocally() { + x.prepareLocally = true +} + +func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) { + var putPrm client.PrmObjectPutInit + if !x.prepareLocally && x.sessionToken != nil { + putPrm.WithinSession(*x.sessionToken) + } + + if x.bearerToken != nil { + putPrm.WithBearerToken(*x.bearerToken) + } + + if x.local { + putPrm.MarkLocal() + } + + putPrm.WithXHeaders(x.xHeaders...) + putPrm.SetCopiesNumberByVectors(x.copyNum) + + if x.prepareLocally { + res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{}) + if err != nil { + return client.PrmObjectPutInit{}, err + } + putPrm.WithObjectMaxSize(res.Info().MaxObjectSize()) + putPrm.WithEpochSource(epochSource(res.Info().CurrentEpoch())) + putPrm.WithoutHomomorphicHash(res.Info().HomomorphicHashingDisabled()) + } + return putPrm, nil +} + // PutObjectRes groups the resulting values of PutObject operation. type PutObjectRes struct { id oid.ID @@ -370,28 +407,21 @@ func (x PutObjectRes) ID() oid.ID { return x.id } +type epochSource uint64 + +func (s epochSource) CurrentEpoch() uint64 { + return uint64(s) +} + // PutObject saves the object in FrostFS network. // // Returns any error which prevented the operation from completing correctly in error return. func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { - var putPrm client.PrmObjectPutInit - - if prm.sessionToken != nil { - putPrm.WithinSession(*prm.sessionToken) + sdkPrm, err := prm.convertToSDKPrm(ctx) + if err != nil { + return nil, fmt.Errorf("unable to create parameters of object put operation: %w", err) } - - if prm.bearerToken != nil { - putPrm.WithBearerToken(*prm.bearerToken) - } - - if prm.local { - putPrm.MarkLocal() - } - - putPrm.WithXHeaders(prm.xHeaders...) - putPrm.SetCopiesNumberByVectors(prm.copyNum) - - wrt, err := prm.cli.ObjectPutInit(ctx, putPrm) + wrt, err := prm.cli.ObjectPutInit(ctx, sdkPrm) if err != nil { return nil, fmt.Errorf("init object writing: %w", err) } diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index af9e9fd0b..19edbb975 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -23,9 +23,10 @@ import ( ) const ( - noProgressFlag = "no-progress" - notificationFlag = "notify" - copiesNumberFlag = "copies-number" + noProgressFlag = "no-progress" + notificationFlag = "notify" + copiesNumberFlag = "copies-number" + prepareLocallyFlag = "prepare-locally" ) var putExpiredOn uint64 @@ -54,6 +55,7 @@ func initObjectPutCmd() { flags.Bool("disable-timestamp", false, "Do not set well-known timestamp attribute") flags.Uint64VarP(&putExpiredOn, commonflags.ExpireAt, "e", 0, "The last active epoch in the life of the object") flags.Bool(noProgressFlag, false, "Do not show progress bar") + flags.Bool(prepareLocallyFlag, false, "Generate object header on the client side (for big object - split locally too)") flags.String(notificationFlag, "", "Object notification in the form of *epoch*:*topic*; '-' topic means using default") flags.Bool(binaryFlag, false, "Deserialize object structure from given file.") @@ -102,7 +104,11 @@ func putObject(cmd *cobra.Command, _ []string) { } var prm internalclient.PutObjectPrm - ReadOrOpenSession(cmd, &prm, pk, cnr, nil) + if prepareLocally, _ := cmd.Flags().GetBool(prepareLocallyFlag); prepareLocally { + prm.PrepareLocally() + } else { + ReadOrOpenSession(cmd, &prm, pk, cnr, nil) + } Prepare(cmd, &prm) prm.SetHeader(obj) @@ -121,15 +127,7 @@ func putObject(cmd *cobra.Command, _ []string) { copyNum, err := cmd.Flags().GetString(copiesNumberFlag) commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) - if len(copyNum) > 0 { - var cn []uint32 - for _, num := range strings.Split(copyNum, ",") { - val, err := strconv.ParseUint(num, 10, 32) - commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) - cn = append(cn, uint32(val)) - } - prm.SetCopiesNumberByVectors(cn) - } + prm.SetCopiesNumberByVectors(parseCopyNumber(cmd, copyNum)) res, err := internalclient.PutObject(cmd.Context(), prm) if p != nil { @@ -141,6 +139,18 @@ func putObject(cmd *cobra.Command, _ []string) { cmd.Printf(" OID: %s\n CID: %s\n", res.ID(), cnr) } +func parseCopyNumber(cmd *cobra.Command, copyNum string) []uint32 { + var cn []uint32 + if len(copyNum) > 0 { + for _, num := range strings.Split(copyNum, ",") { + val, err := strconv.ParseUint(num, 10, 32) + commonCmd.ExitOnErr(cmd, "can't parse object copies numbers information: %w", err) + cn = append(cn, uint32(val)) + } + } + return cn +} + func readFilePayload(filename string, cmd *cobra.Command) (io.Reader, cid.ID, user.ID) { buf, err := os.ReadFile(filename) commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err) From b520a3049e6f4ffeb12fa0f7a4cf7c5d95579bf0 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 4 Jul 2023 18:41:25 +0300 Subject: [PATCH 0391/1943] [#483] cli: Fix `object put` cmd with flag `--prepare-locally` Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/object/put.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index 19edbb975..c5b4e6e06 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -105,6 +105,7 @@ func putObject(cmd *cobra.Command, _ []string) { var prm internalclient.PutObjectPrm if prepareLocally, _ := cmd.Flags().GetBool(prepareLocallyFlag); prepareLocally { + prm.SetClient(internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC)) prm.PrepareLocally() } else { ReadOrOpenSession(cmd, &prm, pk, cnr, nil) From c6df6c84ae3ef2e9f67aa79382405a2aa58eb025 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 5 Jul 2023 19:49:58 +0300 Subject: [PATCH 0392/1943] [#473] Do not modify traverse plan when `[0]` copies number vector is provided Signed-off-by: Alex Vanin --- pkg/services/object_manager/placement/traverser.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 47f79a627..c59146e2b 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -97,10 +97,16 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } else { rem = defaultCopiesVector(cfg.policy) + // compatibleZeroVector is a bool flag which is set when cfg.copyNumbers + // is [0]. In this case we should not modify `rem` slice unless track + // copies are ignored, because [0] means that all copies should be + // stored before returning OK to the client. + compatibleZeroVector := len(cfg.copyNumbers) == 1 && cfg.copyNumbers[0] == 0 + for i := range rem { if !cfg.trackCopies { rem[i] = -1 - } else if len(cfg.copyNumbers) > i { + } else if len(cfg.copyNumbers) > i && !compatibleZeroVector { rem[i] = int(cfg.copyNumbers[i]) } } From f354b8a270b3cb5486df4927f4c461715a999b56 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 5 Jul 2023 20:12:55 +0300 Subject: [PATCH 0393/1943] [#473] Add tests for rem values of traverser Signed-off-by: Alex Vanin --- .../placement/traverser_test.go | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index 66fd8afe0..1b307da6f 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -208,3 +208,55 @@ func TestTraverserObjectScenarios(t *testing.T) { require.True(t, tr.Success()) }) } + +func TestTraverserRemValues(t *testing.T) { + selectors := []int{3, 4, 5} + replicas := []int{2, 3, 4} + + nodes, cnr := testPlacement(t, selectors, replicas) + nodesCopy := copyVectors(nodes) + + testCases := [...]struct { + name string + copyNumbers []uint32 + expectedRem []int + }{ + { + name: "zero copy numbers", + copyNumbers: []uint32{}, + expectedRem: replicas, + }, + { + name: "compatible zero copy numbers", + copyNumbers: []uint32{0}, + expectedRem: replicas, + }, + { + name: "copy numbers for all replicas", + copyNumbers: []uint32{1, 1, 1}, + expectedRem: []int{1, 1, 1}, + }, + { + name: "single copy numbers for multiple replicas", + copyNumbers: []uint32{1}, + expectedRem: []int{1}, // may be a bit unexpected + }, + { + name: "multiple copy numbers for multiple replicas", + copyNumbers: []uint32{1, 1}, + expectedRem: []int{1, 1, 4}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + tr, err := NewTraverser( + ForContainer(cnr), + UseBuilder(&testBuilder{vectors: nodesCopy}), + WithCopyNumbers(testCase.copyNumbers), + ) + require.NoError(t, err) + require.Equal(t, testCase.expectedRem, tr.rem) + }) + } +} From 6eefe9747e28c709d1b1ab6248a528581bd76285 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 5 Jul 2023 17:07:52 +0300 Subject: [PATCH 0394/1943] treesvc: Do not provide credentials unless TLS is used 4f413fe86e was perfect, except it did the opposite of what we needed. Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index ef0c4b465..f50aa0b0d 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -99,7 +99,7 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, ), } - if netAddr.IsTLSEnabled() { + if !netAddr.IsTLSEnabled() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } From 4bbe9cc9361b1c8be06b8ba1f6706048549eb56b Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 6 Jul 2023 15:35:23 +0300 Subject: [PATCH 0395/1943] [#496] .golangci.yml: Add importas linter Add importas linter Signed-off-by: Alexander Chuprov --- .golangci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 4a3fec6e3..643390075 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -31,6 +31,12 @@ linters-settings: statements: 60 # default 40 gocognit: min-complexity: 40 # default 30 + importas: + no-unaliased: true + no-extra-aliases: false + alias: + pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object + alias: objectSDK linters: enable: @@ -62,5 +68,6 @@ linters: - funlen - gocognit - contextcheck + - importas disable-all: true fast: false From 033eaf77e102e66bd4f144197bfa22f8bfc847d1 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 6 Jul 2023 15:36:41 +0300 Subject: [PATCH 0396/1943] [#496] node: Fix linter importas Standardize the alias of the import frostfs-sdk-go/object as objectSDK. Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/internal/client/client.go | 38 +++++++-------- .../modules/container/list_objects.go | 4 +- cmd/frostfs-cli/modules/object/get.go | 4 +- cmd/frostfs-cli/modules/object/head.go | 12 ++--- cmd/frostfs-cli/modules/object/put.go | 32 ++++++------- cmd/frostfs-cli/modules/object/range.go | 14 +++--- cmd/frostfs-cli/modules/object/search.go | 22 ++++----- cmd/frostfs-cli/modules/object/util.go | 20 ++++---- cmd/frostfs-cli/modules/tree/add_by_path.go | 4 +- cmd/frostfs-cli/modules/tree/get_by_path.go | 4 +- .../internal/blobovnicza/inspect.go | 4 +- cmd/frostfs-lens/internal/meta/inspect.go | 4 +- cmd/frostfs-lens/internal/printers.go | 4 +- .../internal/writecache/inspect.go | 4 +- pkg/core/object/address.go | 4 +- pkg/core/object/fmt.go | 34 +++++++------- pkg/core/object/fmt_test.go | 40 ++++++++-------- pkg/core/object/object.go | 4 +- .../engine/engine_test.go | 6 +-- pkg/local_object_storage/engine/head_test.go | 6 +-- pkg/local_object_storage/engine/lock_test.go | 16 +++---- pkg/local_object_storage/engine/select.go | 8 ++-- pkg/local_object_storage/engine/tree_test.go | 6 +-- .../internal/testutil/generators.go | 14 +++--- .../internal/testutil/object.go | 16 +++---- .../metabase/counter_test.go | 10 ++-- pkg/local_object_storage/metabase/db.go | 16 +++---- .../metabase/expired_test.go | 4 +- pkg/local_object_storage/metabase/inhume.go | 6 +-- .../metabase/iterators.go | 7 +-- .../metabase/iterators_test.go | 18 ++++---- pkg/local_object_storage/metabase/list.go | 12 ++--- pkg/local_object_storage/metabase/lock.go | 4 +- .../metabase/lock_test.go | 20 ++++---- pkg/local_object_storage/metabase/select.go | 46 +++++++++---------- pkg/local_object_storage/metabase/util.go | 14 +++--- pkg/local_object_storage/shard/gc.go | 6 +-- .../shard/gc_internal_test.go | 4 +- pkg/local_object_storage/shard/list.go | 4 +- pkg/local_object_storage/shard/lock_test.go | 6 +-- .../shard/metrics_test.go | 6 +-- pkg/local_object_storage/shard/put.go | 6 +-- pkg/local_object_storage/shard/range.go | 16 +++---- pkg/local_object_storage/shard/select.go | 6 +-- pkg/local_object_storage/shard/shard_test.go | 4 +- pkg/local_object_storage/util/splitinfo.go | 4 +- .../util/splitinfo_test.go | 16 +++---- pkg/local_object_storage/writecache/flush.go | 12 ++--- .../writecache/flush_test.go | 10 ++-- .../writecache/writecache.go | 12 ++--- pkg/network/cache/multi.go | 6 +-- pkg/services/object/acl/eacl/v2/eacl_test.go | 10 ++-- pkg/services/object/acl/eacl/v2/headers.go | 10 ++-- pkg/services/object/acl/eacl/v2/object.go | 4 +- pkg/services/object/delete/exec.go | 14 +++--- pkg/services/object/delete/local.go | 4 +- pkg/services/object/delete/service.go | 6 +-- pkg/services/object/delete/util.go | 12 ++--- pkg/services/object/get/get.go | 4 +- pkg/services/object/get/v2/get_forwarder.go | 10 ++-- .../object/get/v2/get_range_forwarder.go | 8 ++-- pkg/services/object/get/v2/head_forwarder.go | 10 ++-- pkg/services/object/get/v2/service.go | 8 ++-- pkg/services/object/get/v2/streamer.go | 4 +- pkg/services/object/get/v2/util.go | 26 +++++------ pkg/services/object/get/writer.go | 14 +++--- pkg/services/object/head/remote.go | 4 +- pkg/services/object/internal/client/client.go | 40 ++++++++-------- pkg/services/object/put/local.go | 12 ++--- pkg/services/object/put/prm.go | 6 +-- pkg/services/object/put/remote.go | 10 ++-- pkg/services/object/put/streamer.go | 4 +- pkg/services/object/put/v2/util.go | 4 +- pkg/services/object/search/exec.go | 4 +- pkg/services/object/search/prm.go | 6 +-- pkg/services/object/search/v2/util.go | 4 +- pkg/services/object/util/chain.go | 30 ++++++------ .../object_manager/tombstone/checker.go | 6 +-- pkg/services/policer/check.go | 4 +- pkg/services/policer/policer_test.go | 10 ++-- 80 files changed, 444 insertions(+), 443 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 9545c60cb..babc0ef38 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -13,7 +13,7 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" ) @@ -331,17 +331,17 @@ type PutObjectPrm struct { copyNum []uint32 - hdr *object.Object + hdr *objectSDK.Object rdr io.Reader - headerCallback func(*object.Object) + headerCallback func(*objectSDK.Object) prepareLocally bool } // SetHeader sets object header. -func (x *PutObjectPrm) SetHeader(hdr *object.Object) { +func (x *PutObjectPrm) SetHeader(hdr *objectSDK.Object) { x.hdr = hdr } @@ -352,7 +352,7 @@ func (x *PutObjectPrm) SetPayloadReader(rdr io.Reader) { // SetHeaderCallback sets callback which is called on the object after the header is received // but before the payload is written. -func (x *PutObjectPrm) SetHeaderCallback(f func(*object.Object)) { +func (x *PutObjectPrm) SetHeaderCallback(f func(*objectSDK.Object)) { x.headerCallback = f } @@ -532,22 +532,22 @@ type GetObjectPrm struct { objectAddressPrm rawPrm payloadWriterPrm - headerCallback func(*object.Object) + headerCallback func(*objectSDK.Object) } // SetHeaderCallback sets callback which is called on the object after the header is received // but before the payload is written. -func (p *GetObjectPrm) SetHeaderCallback(f func(*object.Object)) { +func (p *GetObjectPrm) SetHeaderCallback(f func(*objectSDK.Object)) { p.headerCallback = f } // GetObjectRes groups the resulting values of GetObject operation. type GetObjectRes struct { - hdr *object.Object + hdr *objectSDK.Object } // Header returns the header of the request object. -func (x GetObjectRes) Header() *object.Object { +func (x GetObjectRes) Header() *objectSDK.Object { return x.hdr } @@ -585,7 +585,7 @@ func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { return nil, fmt.Errorf("init object reading on client: %w", err) } - var hdr object.Object + var hdr objectSDK.Object if !rdr.ReadHeader(&hdr) { _, err = rdr.Close() @@ -621,11 +621,11 @@ func (x *HeadObjectPrm) SetMainOnlyFlag(v bool) { // HeadObjectRes groups the resulting values of HeadObject operation. type HeadObjectRes struct { - hdr *object.Object + hdr *objectSDK.Object } // Header returns the requested object header. -func (x HeadObjectRes) Header() *object.Object { +func (x HeadObjectRes) Header() *objectSDK.Object { return x.hdr } @@ -661,7 +661,7 @@ func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) return nil, fmt.Errorf("read object header via client: %w", err) } - var hdr object.Object + var hdr objectSDK.Object if !res.ReadHeader(&hdr) { return nil, fmt.Errorf("missing header in response") @@ -677,11 +677,11 @@ type SearchObjectsPrm struct { commonObjectPrm containerIDPrm - filters object.SearchFilters + filters objectSDK.SearchFilters } // SetFilters sets search filters. -func (x *SearchObjectsPrm) SetFilters(filters object.SearchFilters) { +func (x *SearchObjectsPrm) SetFilters(filters objectSDK.SearchFilters) { x.filters = filters } @@ -754,7 +754,7 @@ type HashPayloadRangesPrm struct { tz bool - rngs []*object.Range + rngs []*objectSDK.Range salt []byte } @@ -765,7 +765,7 @@ func (x *HashPayloadRangesPrm) TZ() { } // SetRanges sets a list of payload ranges to hash. -func (x *HashPayloadRangesPrm) SetRanges(rngs []*object.Range) { +func (x *HashPayloadRangesPrm) SetRanges(rngs []*objectSDK.Range) { x.rngs = rngs } @@ -839,11 +839,11 @@ type PayloadRangePrm struct { rawPrm payloadWriterPrm - rng *object.Range + rng *objectSDK.Range } // SetRange sets payload range to read. -func (x *PayloadRangePrm) SetRange(rng *object.Range) { +func (x *PayloadRangePrm) SetRange(rng *objectSDK.Range) { x.rng = rng } diff --git a/cmd/frostfs-cli/modules/container/list_objects.go b/cmd/frostfs-cli/modules/container/list_objects.go index d00cc3576..1e0aeb4db 100644 --- a/cmd/frostfs-cli/modules/container/list_objects.go +++ b/cmd/frostfs-cli/modules/container/list_objects.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" objectCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/object" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -31,7 +31,7 @@ var listContainerObjectsCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { id := parseContainerID(cmd) - filters := new(object.SearchFilters) + filters := new(objectSDK.SearchFilters) filters.AddRootFilter() // search only user created objects cli := internalclient.GetSDKClientByFlag(cmd, key.GetOrGenerate(cmd), commonflags.RPC) diff --git a/cmd/frostfs-cli/modules/object/get.go b/cmd/frostfs-cli/modules/object/get.go index c72a26239..3136f086f 100644 --- a/cmd/frostfs-cli/modules/object/get.go +++ b/cmd/frostfs-cli/modules/object/get.go @@ -11,7 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/cheggaaa/pb" "github.com/spf13/cobra" @@ -84,7 +84,7 @@ func getObject(cmd *cobra.Command, _ []string) { p = pb.New64(0) p.Output = cmd.OutOrStdout() prm.SetPayloadWriter(p.NewProxyWriter(payloadWriter)) - prm.SetHeaderCallback(func(o *object.Object) { + prm.SetHeaderCallback(func(o *objectSDK.Object) { p.SetTotal64(int64(o.PayloadSize())) p.Start() }) diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index 04467744d..db466e588 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -13,7 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -77,7 +77,7 @@ func getObjectHeader(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "", err) } -func saveAndPrintHeader(cmd *cobra.Command, obj *object.Object, filename string) error { +func saveAndPrintHeader(cmd *cobra.Command, obj *objectSDK.Object, filename string) error { bs, err := marshalHeader(cmd, obj) if err != nil { return fmt.Errorf("could not marshal header: %w", err) @@ -97,7 +97,7 @@ func saveAndPrintHeader(cmd *cobra.Command, obj *object.Object, filename string) return printHeader(cmd, obj) } -func marshalHeader(cmd *cobra.Command, hdr *object.Object) ([]byte, error) { +func marshalHeader(cmd *cobra.Command, hdr *objectSDK.Object) ([]byte, error) { toJSON, _ := cmd.Flags().GetBool(commonflags.JSON) toProto, _ := cmd.Flags().GetBool("proto") switch { @@ -138,7 +138,7 @@ func printContainerID(cmd *cobra.Command, recv func() (cid.ID, bool)) { cmd.Printf("CID: %s\n", strID) } -func printHeader(cmd *cobra.Command, obj *object.Object) error { +func printHeader(cmd *cobra.Command, obj *objectSDK.Object) error { printObjectID(cmd, obj.ID) printContainerID(cmd, obj.ContainerID) cmd.Printf("Owner: %s\n", obj.OwnerID()) @@ -150,7 +150,7 @@ func printHeader(cmd *cobra.Command, obj *object.Object) error { cmd.Println("Attributes:") for _, attr := range obj.Attributes() { - if attr.Key() == object.AttributeTimestamp { + if attr.Key() == objectSDK.AttributeTimestamp { cmd.Printf(" %s=%s (%s)\n", attr.Key(), attr.Value(), @@ -174,7 +174,7 @@ func printHeader(cmd *cobra.Command, obj *object.Object) error { return printSplitHeader(cmd, obj) } -func printSplitHeader(cmd *cobra.Command, obj *object.Object) error { +func printSplitHeader(cmd *cobra.Command, obj *objectSDK.Object) error { if splitID := obj.SplitID(); splitID != nil { cmd.Printf("Split ID: %s\n", splitID) } diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index c5b4e6e06..97bb12dbc 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -16,7 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/cheggaaa/pb" "github.com/spf13/cobra" @@ -81,7 +81,7 @@ func putObject(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "", fmt.Errorf("can't open file '%s': %w", filename, err)) } var payloadReader io.Reader = f - obj := object.New() + obj := objectSDK.New() if binary { payloadReader, cnr, ownerID = readFilePayload(filename, cmd) @@ -155,7 +155,7 @@ func parseCopyNumber(cmd *cobra.Command, copyNum string) []uint32 { func readFilePayload(filename string, cmd *cobra.Command) (io.Reader, cid.ID, user.ID) { buf, err := os.ReadFile(filename) commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err) - objTemp := object.New() + objTemp := objectSDK.New() // TODO(@acid-ant): #1932 Use streams to marshal/unmarshal payload commonCmd.ExitOnErr(cmd, "can't unmarshal object from given file: %w", objTemp.Unmarshal(buf)) payloadReader := bytes.NewReader(objTemp.Payload()) @@ -174,19 +174,19 @@ func setFilePayloadReader(cmd *cobra.Command, f *os.File, prm *internalclient.Pu p := pb.New64(fi.Size()) p.Output = cmd.OutOrStdout() prm.SetPayloadReader(p.NewProxyReader(f)) - prm.SetHeaderCallback(func(o *object.Object) { p.Start() }) + prm.SetHeaderCallback(func(o *objectSDK.Object) { p.Start() }) return p } -func setBinaryPayloadReader(cmd *cobra.Command, obj *object.Object, prm *internalclient.PutObjectPrm, payloadReader io.Reader) *pb.ProgressBar { +func setBinaryPayloadReader(cmd *cobra.Command, obj *objectSDK.Object, prm *internalclient.PutObjectPrm, payloadReader io.Reader) *pb.ProgressBar { p := pb.New(len(obj.Payload())) p.Output = cmd.OutOrStdout() prm.SetPayloadReader(p.NewProxyReader(payloadReader)) - prm.SetHeaderCallback(func(o *object.Object) { p.Start() }) + prm.SetHeaderCallback(func(o *objectSDK.Object) { p.Start() }) return p } -func getAllObjectAttributes(cmd *cobra.Command) []object.Attribute { +func getAllObjectAttributes(cmd *cobra.Command) []objectSDK.Attribute { attrs, err := parseObjectAttrs(cmd) commonCmd.ExitOnErr(cmd, "can't parse object attributes: %w", err) @@ -205,7 +205,7 @@ func getAllObjectAttributes(cmd *cobra.Command) []object.Attribute { if !expAttrFound { index := len(attrs) - attrs = append(attrs, object.Attribute{}) + attrs = append(attrs, objectSDK.Attribute{}) attrs[index].SetKey(objectV2.SysAttributeExpEpoch) attrs[index].SetValue(expAttrValue) } @@ -213,7 +213,7 @@ func getAllObjectAttributes(cmd *cobra.Command) []object.Attribute { return attrs } -func parseObjectAttrs(cmd *cobra.Command) ([]object.Attribute, error) { +func parseObjectAttrs(cmd *cobra.Command) ([]objectSDK.Attribute, error) { var rawAttrs []string raw := cmd.Flag("attributes").Value.String() @@ -221,7 +221,7 @@ func parseObjectAttrs(cmd *cobra.Command) ([]object.Attribute, error) { rawAttrs = strings.Split(raw, ",") } - attrs := make([]object.Attribute, len(rawAttrs), len(rawAttrs)+2) // name + timestamp attributes + attrs := make([]objectSDK.Attribute, len(rawAttrs), len(rawAttrs)+2) // name + timestamp attributes for i := range rawAttrs { k, v, found := strings.Cut(rawAttrs[i], "=") if !found { @@ -235,23 +235,23 @@ func parseObjectAttrs(cmd *cobra.Command) ([]object.Attribute, error) { if !disableFilename { filename := filepath.Base(cmd.Flag(fileFlag).Value.String()) index := len(attrs) - attrs = append(attrs, object.Attribute{}) - attrs[index].SetKey(object.AttributeFileName) + attrs = append(attrs, objectSDK.Attribute{}) + attrs[index].SetKey(objectSDK.AttributeFileName) attrs[index].SetValue(filename) } disableTime, _ := cmd.Flags().GetBool("disable-timestamp") if !disableTime { index := len(attrs) - attrs = append(attrs, object.Attribute{}) - attrs[index].SetKey(object.AttributeTimestamp) + attrs = append(attrs, objectSDK.Attribute{}) + attrs[index].SetKey(objectSDK.AttributeTimestamp) attrs[index].SetValue(strconv.FormatInt(time.Now().Unix(), 10)) } return attrs, nil } -func parseObjectNotifications(cmd *cobra.Command) (*object.NotificationInfo, error) { +func parseObjectNotifications(cmd *cobra.Command) (*objectSDK.NotificationInfo, error) { const ( separator = ":" useDefaultTopic = "-" @@ -267,7 +267,7 @@ func parseObjectNotifications(cmd *cobra.Command) (*object.NotificationInfo, err return nil, fmt.Errorf("notification must be in the form of: *epoch*%s*topic*, got %s", separator, raw) } - ni := new(object.NotificationInfo) + ni := new(objectSDK.NotificationInfo) epoch, err := strconv.ParseUint(before, 10, 64) if err != nil { diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index 76425d948..591355b95 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -14,7 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -102,7 +102,7 @@ func getObjectRange(cmd *cobra.Command, _ []string) { } func printSplitInfoErr(cmd *cobra.Command, err error) bool { - var errSplitInfo *object.SplitInfoError + var errSplitInfo *objectSDK.SplitInfoError ok := errors.As(err, &errSplitInfo) @@ -114,14 +114,14 @@ func printSplitInfoErr(cmd *cobra.Command, err error) bool { return ok } -func printSplitInfo(cmd *cobra.Command, info *object.SplitInfo) { +func printSplitInfo(cmd *cobra.Command, info *objectSDK.SplitInfo) { bs, err := marshalSplitInfo(cmd, info) commonCmd.ExitOnErr(cmd, "can't marshal split info: %w", err) cmd.Println(string(bs)) } -func marshalSplitInfo(cmd *cobra.Command, info *object.SplitInfo) ([]byte, error) { +func marshalSplitInfo(cmd *cobra.Command, info *objectSDK.SplitInfo) ([]byte, error) { toJSON, _ := cmd.Flags().GetBool(commonflags.JSON) toProto, _ := cmd.Flags().GetBool("proto") switch { @@ -146,13 +146,13 @@ func marshalSplitInfo(cmd *cobra.Command, info *object.SplitInfo) ([]byte, error } } -func getRangeList(cmd *cobra.Command) ([]*object.Range, error) { +func getRangeList(cmd *cobra.Command) ([]*objectSDK.Range, error) { v := cmd.Flag("range").Value.String() if len(v) == 0 { return nil, nil } vs := strings.Split(v, ",") - rs := make([]*object.Range, len(vs)) + rs := make([]*objectSDK.Range, len(vs)) for i := range vs { before, after, found := strings.Cut(vs[i], rangeSep) if !found { @@ -176,7 +176,7 @@ func getRangeList(cmd *cobra.Command) ([]*object.Range, error) { return nil, fmt.Errorf("invalid '%s' range: uint64 overflow", vs[i]) } - rs[i] = object.NewRange() + rs[i] = objectSDK.NewRange() rs[i].SetOffset(offset) rs[i].SetLength(length) } diff --git a/cmd/frostfs-cli/modules/object/search.go b/cmd/frostfs-cli/modules/object/search.go index b603e5fe8..ca5d78bc9 100644 --- a/cmd/frostfs-cli/modules/object/search.go +++ b/cmd/frostfs-cli/modules/object/search.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -72,18 +72,18 @@ func searchObject(cmd *cobra.Command, _ []string) { } } -var searchUnaryOpVocabulary = map[string]object.SearchMatchType{ - "NOPRESENT": object.MatchNotPresent, +var searchUnaryOpVocabulary = map[string]objectSDK.SearchMatchType{ + "NOPRESENT": objectSDK.MatchNotPresent, } -var searchBinaryOpVocabulary = map[string]object.SearchMatchType{ - "EQ": object.MatchStringEqual, - "NE": object.MatchStringNotEqual, - "COMMON_PREFIX": object.MatchCommonPrefix, +var searchBinaryOpVocabulary = map[string]objectSDK.SearchMatchType{ + "EQ": objectSDK.MatchStringEqual, + "NE": objectSDK.MatchStringNotEqual, + "COMMON_PREFIX": objectSDK.MatchCommonPrefix, } -func parseSearchFilters(cmd *cobra.Command) (object.SearchFilters, error) { - var fs object.SearchFilters +func parseSearchFilters(cmd *cobra.Command) (objectSDK.SearchFilters, error) { + var fs objectSDK.SearchFilters for i := range searchFilters { words := strings.Fields(searchFilters[i]) @@ -97,7 +97,7 @@ func parseSearchFilters(cmd *cobra.Command) (object.SearchFilters, error) { return nil, fmt.Errorf("could not read attributes filter from file: %w", err) } - subFs := object.NewSearchFilters() + subFs := objectSDK.NewSearchFilters() if err := subFs.UnmarshalJSON(data); err != nil { return nil, fmt.Errorf("could not unmarshal attributes filter from file: %w", err) @@ -138,7 +138,7 @@ func parseSearchFilters(cmd *cobra.Command) (object.SearchFilters, error) { return nil, fmt.Errorf("could not parse object ID: %w", err) } - fs.AddObjectIDFilter(object.MatchStringEqual, id) + fs.AddObjectIDFilter(objectSDK.MatchStringEqual, id) } return fs, nil diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index 094b62314..37e9f74e0 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -16,7 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "github.com/spf13/cobra" @@ -87,7 +87,7 @@ func readObjectAddress(cmd *cobra.Command, cnr *cid.ID, obj *oid.ID) oid.Address func readObjectAddressBin(cmd *cobra.Command, cnr *cid.ID, obj *oid.ID, filename string) oid.Address { buf, err := os.ReadFile(filename) commonCmd.ExitOnErr(cmd, "unable to read given file: %w", err) - objTemp := object.New() + objTemp := objectSDK.New() commonCmd.ExitOnErr(cmd, "can't unmarshal object from given file: %w", objTemp.Unmarshal(buf)) var addr oid.Address @@ -356,7 +356,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, _, err := internal.HeadObject(cmd.Context(), prmHead) - var errSplit *object.SplitInfoError + var errSplit *objectSDK.SplitInfoError switch { default: @@ -381,7 +381,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, return tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnr, obj) } -func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *object.SplitInfo, prmHead internal.HeadObjectPrm, cnr cid.ID) ([]oid.ID, bool) { +func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, prmHead internal.HeadObjectPrm, cnr cid.ID) ([]oid.ID, bool) { // collect split chain by the descending ease of operations (ease is evaluated heuristically). // If any approach fails, we don't try the next since we assume that it will fail too. @@ -413,12 +413,12 @@ func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *object.Spl return nil, false } -func tryGetSplitMembersBySplitID(cmd *cobra.Command, splitInfo *object.SplitInfo, cli *client.Client, cnr cid.ID) ([]oid.ID, bool) { +func tryGetSplitMembersBySplitID(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, cli *client.Client, cnr cid.ID) ([]oid.ID, bool) { if idSplit := splitInfo.SplitID(); idSplit != nil { common.PrintVerbose(cmd, "Collecting split members by split ID...") - var query object.SearchFilters - query.AddSplitIDFilter(object.MatchStringEqual, idSplit) + var query objectSDK.SearchFilters + query.AddSplitIDFilter(objectSDK.MatchStringEqual, idSplit) var prm internal.SearchObjectsPrm prm.SetContainerID(cnr) @@ -437,7 +437,7 @@ func tryGetSplitMembersBySplitID(cmd *cobra.Command, splitInfo *object.SplitInfo return nil, false } -func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *object.SplitInfo, prmHead internal.HeadObjectPrm, cli *client.Client, cnr cid.ID, obj oid.ID) []oid.ID { +func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, prmHead internal.HeadObjectPrm, cli *client.Client, cnr cid.ID, obj oid.ID) []oid.ID { var addrObj oid.Address addrObj.SetContainer(cnr) @@ -482,8 +482,8 @@ func tryRestoreChainInReverse(cmd *cobra.Command, splitInfo *object.SplitInfo, p common.PrintVerbose(cmd, "Looking for a linking object...") - var query object.SearchFilters - query.AddParentIDFilter(object.MatchStringEqual, obj) + var query objectSDK.SearchFilters + query.AddParentIDFilter(objectSDK.MatchStringEqual, obj) var prmSearch internal.SearchObjectsPrm prmSearch.SetClient(cli) diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index 6b182e1e4..90f4a6364 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -10,7 +10,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/spf13/cobra" ) @@ -68,7 +68,7 @@ func addByPath(cmd *cobra.Command, _ []string) { req.Body = &tree.AddByPathRequest_Body{ ContainerId: rawCID, TreeId: tid, - PathAttribute: object.AttributeFileName, + PathAttribute: objectSDK.AttributeFileName, // PathAttribute: pAttr, Path: strings.Split(path, "/"), Meta: meta, diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index 69a3dc7ea..38ad2bff5 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -10,7 +10,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/spf13/cobra" ) @@ -66,7 +66,7 @@ func getByPath(cmd *cobra.Command, _ []string) { req.Body = &tree.GetNodeByPathRequest_Body{ ContainerId: rawCID, TreeId: tid, - PathAttribute: object.AttributeFileName, + PathAttribute: objectSDK.AttributeFileName, // PathAttribute: pAttr, Path: strings.Split(path, "/"), LatestOnly: latestOnly, diff --git a/cmd/frostfs-lens/internal/blobovnicza/inspect.go b/cmd/frostfs-lens/internal/blobovnicza/inspect.go index 13442a4b8..b1a6e3fd2 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/inspect.go +++ b/cmd/frostfs-lens/internal/blobovnicza/inspect.go @@ -3,7 +3,7 @@ package blobovnicza import ( common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -38,7 +38,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { data := res.Object() - var o object.Object + var o objectSDK.Object common.ExitOnErr(cmd, common.Errf("could not unmarshal object: %w", o.Unmarshal(data)), ) diff --git a/cmd/frostfs-lens/internal/meta/inspect.go b/cmd/frostfs-lens/internal/meta/inspect.go index bc7f28a3a..de0f24aeb 100644 --- a/cmd/frostfs-lens/internal/meta/inspect.go +++ b/cmd/frostfs-lens/internal/meta/inspect.go @@ -7,7 +7,7 @@ import ( common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -49,7 +49,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { prm.SetAddress(addr) prm.SetRaw(true) - siErr := new(object.SplitInfoError) + siErr := new(objectSDK.SplitInfoError) res, err := db.Get(cmd.Context(), prm) if errors.As(err, &siErr) { diff --git a/cmd/frostfs-lens/internal/printers.go b/cmd/frostfs-lens/internal/printers.go index a232409d6..dd73a5552 100644 --- a/cmd/frostfs-lens/internal/printers.go +++ b/cmd/frostfs-lens/internal/printers.go @@ -4,14 +4,14 @@ import ( "os" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) // PrintObjectHeader prints passed object's header fields via // the passed cobra command. Does nothing with the payload. -func PrintObjectHeader(cmd *cobra.Command, h object.Object) { +func PrintObjectHeader(cmd *cobra.Command, h objectSDK.Object) { cmd.Println("Version:", h.Version()) cmd.Println("Type:", h.Type()) printContainerID(cmd, h.ContainerID) diff --git a/cmd/frostfs-lens/internal/writecache/inspect.go b/cmd/frostfs-lens/internal/writecache/inspect.go index 91f12ed0f..7d3c8ab22 100644 --- a/cmd/frostfs-lens/internal/writecache/inspect.go +++ b/cmd/frostfs-lens/internal/writecache/inspect.go @@ -3,7 +3,7 @@ package writecache import ( common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/spf13/cobra" ) @@ -27,7 +27,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { data, err := writecache.Get(db, []byte(vAddress)) common.ExitOnErr(cmd, common.Errf("could not fetch object: %w", err)) - var o object.Object + var o objectSDK.Object common.ExitOnErr(cmd, common.Errf("could not unmarshal object: %w", o.Unmarshal(data))) common.PrintObjectHeader(cmd, o) diff --git a/pkg/core/object/address.go b/pkg/core/object/address.go index cd5559d9f..12e5c89ce 100644 --- a/pkg/core/object/address.go +++ b/pkg/core/object/address.go @@ -1,7 +1,7 @@ package object import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -9,5 +9,5 @@ import ( // object type. type AddressWithType struct { Address oid.Address - Type object.Type + Type objectSDK.Type } diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index ba9868693..abf8f02cb 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -12,7 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -87,7 +87,7 @@ func NewFormatValidator(opts ...FormatValidatorOption) *FormatValidator { // If unprepared is true, only fields set by user are validated. // // Returns nil error if the object has valid structure. -func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unprepared bool) error { +func (v *FormatValidator) Validate(ctx context.Context, obj *objectSDK.Object, unprepared bool) error { if obj == nil { return errNilObject } @@ -119,7 +119,7 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unpr return fmt.Errorf("object did not pass expiration check: %w", err) } - if err := object.CheckHeaderVerificationFields(obj); err != nil { + if err := objectSDK.CheckHeaderVerificationFields(obj); err != nil { return fmt.Errorf("(%T) could not validate header fields: %w", v, err) } } @@ -132,7 +132,7 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *object.Object, unpr return nil } -func (v *FormatValidator) validateSignatureKey(obj *object.Object) error { +func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { sig := obj.Signature() if sig == nil { // TODO(@cthulhu-rider): #468 use "const" error @@ -178,13 +178,13 @@ func (v *FormatValidator) checkOwnerKey(id user.ID, key frostfsecdsa.PublicKey) // - object.TypeTombstone; // - object.TypeLock. type ContentMeta struct { - typ object.Type + typ objectSDK.Type objs []oid.ID } // Type returns object's type. -func (i ContentMeta) Type() object.Type { +func (i ContentMeta) Type() objectSDK.Type { return i.typ } @@ -197,17 +197,17 @@ func (i ContentMeta) Objects() []oid.ID { } // ValidateContent validates payload content according to the object type. -func (v *FormatValidator) ValidateContent(o *object.Object) (ContentMeta, error) { +func (v *FormatValidator) ValidateContent(o *objectSDK.Object) (ContentMeta, error) { meta := ContentMeta{ typ: o.Type(), } switch o.Type() { - case object.TypeTombstone: + case objectSDK.TypeTombstone: if err := v.fillAndValidateTombstoneMeta(o, &meta); err != nil { return ContentMeta{}, err } - case object.TypeLock: + case objectSDK.TypeLock: if err := v.fillAndValidateLockMeta(o, &meta); err != nil { return ContentMeta{}, err } @@ -218,7 +218,7 @@ func (v *FormatValidator) ValidateContent(o *object.Object) (ContentMeta, error) return meta, nil } -func (v *FormatValidator) fillAndValidateLockMeta(o *object.Object, meta *ContentMeta) error { +func (v *FormatValidator) fillAndValidateLockMeta(o *objectSDK.Object, meta *ContentMeta) error { if len(o.Payload()) == 0 { return errors.New("empty payload in lock") } @@ -240,7 +240,7 @@ func (v *FormatValidator) fillAndValidateLockMeta(o *object.Object, meta *Conten return fmt.Errorf("lock object expiration: %d; current: %d", lockExp, currEpoch) } - var lock object.Lock + var lock objectSDK.Lock if err = lock.Unmarshal(o.Payload()); err != nil { return fmt.Errorf("decode lock payload: %w", err) @@ -256,12 +256,12 @@ func (v *FormatValidator) fillAndValidateLockMeta(o *object.Object, meta *Conten return nil } -func (v *FormatValidator) fillAndValidateTombstoneMeta(o *object.Object, meta *ContentMeta) error { +func (v *FormatValidator) fillAndValidateTombstoneMeta(o *objectSDK.Object, meta *ContentMeta) error { if len(o.Payload()) == 0 { return fmt.Errorf("(%T) empty payload in tombstone", v) } - tombstone := object.NewTombstone() + tombstone := objectSDK.NewTombstone() if err := tombstone.Unmarshal(o.Payload()); err != nil { return fmt.Errorf("(%T) could not unmarshal tombstone content: %w", v, err) @@ -287,7 +287,7 @@ func (v *FormatValidator) fillAndValidateTombstoneMeta(o *object.Object, meta *C var errExpired = errors.New("object has expired") -func (v *FormatValidator) checkExpiration(ctx context.Context, obj *object.Object) error { +func (v *FormatValidator) checkExpiration(ctx context.Context, obj *objectSDK.Object) error { exp, err := expirationEpochAttribute(obj) if err != nil { if errors.Is(err, errNoExpirationEpoch) { @@ -321,7 +321,7 @@ func (v *FormatValidator) checkExpiration(ctx context.Context, obj *object.Objec return nil } -func expirationEpochAttribute(obj *object.Object) (uint64, error) { +func expirationEpochAttribute(obj *objectSDK.Object) (uint64, error) { for _, a := range obj.Attributes() { if a.Key() != objectV2.SysAttributeExpEpoch && a.Key() != objectV2.SysAttributeExpEpochNeoFS { continue @@ -338,7 +338,7 @@ var ( errEmptyAttrVal = errors.New("empty attribute value") ) -func (v *FormatValidator) checkAttributes(obj *object.Object) error { +func (v *FormatValidator) checkAttributes(obj *objectSDK.Object) error { as := obj.Attributes() mUnique := make(map[string]struct{}, len(as)) @@ -362,7 +362,7 @@ func (v *FormatValidator) checkAttributes(obj *object.Object) error { var errIncorrectOwner = errors.New("incorrect object owner") -func (v *FormatValidator) checkOwner(obj *object.Object) error { +func (v *FormatValidator) checkOwner(obj *objectSDK.Object) error { if idOwner := obj.OwnerID(); idOwner == nil || len(idOwner.WalletBytes()) == 0 { return errIncorrectOwner } diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index 2cf5099ba..d04c16709 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -8,7 +8,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" sessiontest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session/test" @@ -17,11 +17,11 @@ import ( "github.com/stretchr/testify/require" ) -func blankValidObject(key *ecdsa.PrivateKey) *object.Object { +func blankValidObject(key *ecdsa.PrivateKey) *objectSDK.Object { var idOwner user.ID user.IDFromKey(&idOwner, key.PublicKey) - obj := object.New() + obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) obj.SetOwnerID(&idOwner) @@ -66,20 +66,20 @@ func TestFormatValidator_Validate(t *testing.T) { }) t.Run("nil identifier", func(t *testing.T) { - obj := object.New() + obj := objectSDK.New() require.ErrorIs(t, v.Validate(context.Background(), obj, false), errNilID) }) t.Run("nil container identifier", func(t *testing.T) { - obj := object.New() + obj := objectSDK.New() obj.SetID(oidtest.ID()) require.ErrorIs(t, v.Validate(context.Background(), obj, true), errNilCID) }) t.Run("unsigned object", func(t *testing.T) { - obj := object.New() + obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) obj.SetID(oidtest.ID()) @@ -94,12 +94,12 @@ func TestFormatValidator_Validate(t *testing.T) { err := tok.Sign(ownerKey.PrivateKey) require.NoError(t, err) - obj := object.New() + obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) obj.SetSessionToken(tok) obj.SetOwnerID(&idOwner) - require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) + require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) require.NoError(t, v.Validate(context.Background(), obj, false)) }) @@ -107,20 +107,20 @@ func TestFormatValidator_Validate(t *testing.T) { t.Run("correct w/o session token", func(t *testing.T) { obj := blankValidObject(&ownerKey.PrivateKey) - require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) + require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) require.NoError(t, v.Validate(context.Background(), obj, false)) }) t.Run("tombstone content", func(t *testing.T) { - obj := object.New() - obj.SetType(object.TypeTombstone) + obj := objectSDK.New() + obj.SetType(objectSDK.TypeTombstone) obj.SetContainerID(cidtest.ID()) _, err := v.ValidateContent(obj) require.Error(t, err) // no tombstone content - content := object.NewTombstone() + content := objectSDK.NewTombstone() content.SetMembers([]oid.ID{oidtest.ID()}) data, err := content.Marshal() @@ -141,7 +141,7 @@ func TestFormatValidator_Validate(t *testing.T) { _, err = v.ValidateContent(obj) require.Error(t, err) // no expiration epoch in tombstone - var expirationAttribute object.Attribute + var expirationAttribute objectSDK.Attribute expirationAttribute.SetKey(objectV2.SysAttributeExpEpoch) expirationAttribute.SetValue(strconv.Itoa(10)) @@ -163,20 +163,20 @@ func TestFormatValidator_Validate(t *testing.T) { require.NoError(t, err) // all good require.EqualValues(t, []oid.ID{id}, contentGot.Objects()) - require.Equal(t, object.TypeTombstone, contentGot.Type()) + require.Equal(t, objectSDK.TypeTombstone, contentGot.Type()) }) t.Run("expiration", func(t *testing.T) { - fn := func(val string) *object.Object { + fn := func(val string) *objectSDK.Object { obj := blankValidObject(&ownerKey.PrivateKey) - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(val) obj.SetAttributes(a) - require.NoError(t, object.SetIDWithSignature(ownerKey.PrivateKey, obj)) + require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) return obj } @@ -221,11 +221,11 @@ func TestFormatValidator_Validate(t *testing.T) { t.Run("duplication", func(t *testing.T) { obj := blankValidObject(&ownerKey.PrivateKey) - var a1 object.Attribute + var a1 objectSDK.Attribute a1.SetKey("key1") a1.SetValue("val1") - var a2 object.Attribute + var a2 objectSDK.Attribute a2.SetKey("key2") a2.SetValue("val2") @@ -244,7 +244,7 @@ func TestFormatValidator_Validate(t *testing.T) { t.Run("empty value", func(t *testing.T) { obj := blankValidObject(&ownerKey.PrivateKey) - var a object.Attribute + var a objectSDK.Attribute a.SetKey("key") obj.SetAttributes(a) diff --git a/pkg/core/object/object.go b/pkg/core/object/object.go index 8fb656acd..9c450966c 100644 --- a/pkg/core/object/object.go +++ b/pkg/core/object/object.go @@ -1,12 +1,12 @@ package object import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) // AddressOf returns the address of the object. -func AddressOf(obj *object.Object) oid.Address { +func AddressOf(obj *objectSDK.Object) oid.Address { var addr oid.Address id, ok := obj.ID() diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 11c0848fe..26de2e5eb 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -18,7 +18,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "git.frostfs.info/TrueCloudLab/hrw" "github.com/panjf2000/ants/v2" @@ -148,7 +148,7 @@ func newStorages(root string, smallSize uint64) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1), blobovniczatree.WithPermissions(0700)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < smallSize }, }, @@ -176,7 +176,7 @@ func newTestStorages(root string, smallSize uint64) ([]blobstor.SubStorage, *tes return []blobstor.SubStorage{ { Storage: smallFileStorage, - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < smallSize }, }, diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go index bf00c4289..d5bf8429c 100644 --- a/pkg/local_object_storage/engine/head_test.go +++ b/pkg/local_object_storage/engine/head_test.go @@ -8,7 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -17,7 +17,7 @@ func TestHeadRaw(t *testing.T) { defer os.RemoveAll(t.Name()) cnr := cidtest.ID() - splitID := object.NewSplitID() + splitID := objectSDK.NewSplitID() parent := testutil.GenerateObjectWithCID(cnr) testutil.AddAttribute(parent, "foo", "bar") @@ -70,7 +70,7 @@ func TestHeadRaw(t *testing.T) { _, err = e.Head(context.Background(), headPrm) require.Error(t, err) - var si *object.SplitInfoError + var si *objectSDK.SplitInfoError require.ErrorAs(t, err, &si) // SplitInfoError should contain info from both shards diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 7796913ea..a7f0a1552 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -15,7 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/panjf2000/ants/v2" @@ -83,7 +83,7 @@ func TestLockUserScenario(t *testing.T) { lockerAddr.SetContainer(cnr) lockerAddr.SetObject(lockerID) - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(strconv.Itoa(lockerExpiresAfter)) @@ -105,9 +105,9 @@ func TestLockUserScenario(t *testing.T) { require.NoError(t, err) // 2. - var locker object.Lock + var locker objectSDK.Lock locker.WriteMembers([]oid.ID{id}) - object.WriteLock(lockerObj, locker) + objectSDK.WriteLock(lockerObj, locker) err = Put(context.Background(), e, lockerObj) require.NoError(t, err) @@ -123,7 +123,7 @@ func TestLockUserScenario(t *testing.T) { require.ErrorAs(t, err, new(apistatus.ObjectLocked)) // 4. - tombObj.SetType(object.TypeTombstone) + tombObj.SetType(objectSDK.TypeTombstone) tombObj.SetID(tombForLockID) tombObj.SetAttributes(a) @@ -188,12 +188,12 @@ func TestLockExpiration(t *testing.T) { require.NoError(t, err) // 2. - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(strconv.Itoa(lockerExpiresAfter)) lock := testutil.GenerateObjectWithCID(cnr) - lock.SetType(object.TypeLock) + lock.SetType(objectSDK.TypeLock) lock.SetAttributes(a) err = Put(context.Background(), e, lock) @@ -266,7 +266,7 @@ func TestLockForceRemoval(t *testing.T) { // 2. lock := testutil.GenerateObjectWithCID(cnr) - lock.SetType(object.TypeLock) + lock.SetType(objectSDK.TypeLock) err = Put(context.Background(), e, lock) require.NoError(t, err) diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 6a7bf1b7f..6a8c9fab9 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -15,7 +15,7 @@ import ( // SelectPrm groups the parameters of Select operation. type SelectPrm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters } // SelectRes groups the resulting values of Select operation. @@ -29,7 +29,7 @@ func (p *SelectPrm) WithContainerID(cnr cid.ID) { } // WithFilters is a Select option to set the object filters. -func (p *SelectPrm) WithFilters(fs object.SearchFilters) { +func (p *SelectPrm) WithFilters(fs objectSDK.SearchFilters) { p.filters = fs } @@ -144,7 +144,7 @@ func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, erro } // Select selects objects from local storage using provided filters. -func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs object.SearchFilters) ([]oid.Address, error) { +func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs objectSDK.SearchFilters) ([]oid.Address, error) { var selectPrm SelectPrm selectPrm.WithContainerID(cnr) selectPrm.WithFilters(fs) diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index 4cd35a088..f1650b5ae 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -8,7 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) func BenchmarkTreeVsSearch(b *testing.B) { @@ -47,8 +47,8 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { var prm SelectPrm prm.WithContainerID(cid) - var fs object.SearchFilters - fs.AddFilter(pilorama.AttributeFilename, strconv.Itoa(objCount/2), object.MatchStringEqual) + var fs objectSDK.SearchFilters + fs.AddFilter(pilorama.AttributeFilename, strconv.Itoa(objCount/2), objectSDK.MatchStringEqual) prm.WithFilters(fs) for i := 0; i < b.N; i++ { diff --git a/pkg/local_object_storage/internal/testutil/generators.go b/pkg/local_object_storage/internal/testutil/generators.go index 059f00b3a..383c596af 100644 --- a/pkg/local_object_storage/internal/testutil/generators.go +++ b/pkg/local_object_storage/internal/testutil/generators.go @@ -6,7 +6,7 @@ import ( "testing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" "golang.org/x/exp/rand" @@ -48,7 +48,7 @@ func (g RandAddrGenerator) Next() oid.Address { // ObjectGenerator is the interface of types that generate object entries. type ObjectGenerator interface { - Next() *object.Object + Next() *objectSDK.Object } // SeqObjGenerator is an ObjectGenerator that generates entries with random payloads of size objSize and sequential IDs. @@ -59,7 +59,7 @@ type SeqObjGenerator struct { var _ ObjectGenerator = &SeqObjGenerator{} -func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *object.Object { +func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *objectSDK.Object { data := make([]byte, sz) _, _ = rand.Read(data) obj := GenerateObjectWithCIDWithPayload(cid, data) @@ -67,7 +67,7 @@ func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *ob return obj } -func (g *SeqObjGenerator) Next() *object.Object { +func (g *SeqObjGenerator) Next() *objectSDK.Object { var id oid.ID binary.LittleEndian.PutUint64(id[:], g.cnt.Add(1)) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) @@ -80,7 +80,7 @@ type RandObjGenerator struct { var _ ObjectGenerator = &RandObjGenerator{} -func (g *RandObjGenerator) Next() *object.Object { +func (g *RandObjGenerator) Next() *objectSDK.Object { var id oid.ID _, _ = rand.Read(id[:]) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) @@ -92,13 +92,13 @@ type OverwriteObjGenerator struct { MaxObjects uint64 } -func (g *OverwriteObjGenerator) Next() *object.Object { +func (g *OverwriteObjGenerator) Next() *objectSDK.Object { var id oid.ID binary.LittleEndian.PutUint64(id[:], uint64(1+rand.Int63n(int64(g.MaxObjects)))) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) } -func AddressFromObject(t testing.TB, obj *object.Object) oid.Address { +func AddressFromObject(t testing.TB, obj *objectSDK.Object) oid.Address { var addr oid.Address id, isSet := obj.ID() diff --git a/pkg/local_object_storage/internal/testutil/object.go b/pkg/local_object_storage/internal/testutil/object.go index 4f6d95816..9cbce27bf 100644 --- a/pkg/local_object_storage/internal/testutil/object.go +++ b/pkg/local_object_storage/internal/testutil/object.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" @@ -16,17 +16,17 @@ import ( const defaultDataSize = 32 -func GenerateObject() *object.Object { +func GenerateObject() *objectSDK.Object { return GenerateObjectWithCID(cidtest.ID()) } -func GenerateObjectWithCID(cnr cid.ID) *object.Object { +func GenerateObjectWithCID(cnr cid.ID) *objectSDK.Object { data := make([]byte, defaultDataSize) _, _ = rand.Read(data) return GenerateObjectWithCIDWithPayload(cnr, data) } -func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *object.Object { +func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *objectSDK.Object { var ver version.Version ver.SetMajor(2) ver.SetMinor(1) @@ -37,7 +37,7 @@ func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *object.Object { var csumTZ checksum.Checksum csumTZ.SetTillichZemor(tz.Sum(csum.Value())) - obj := object.New() + obj := objectSDK.New() obj.SetID(oidtest.ID()) obj.SetOwnerID(usertest.ID()) obj.SetContainerID(cnr) @@ -49,8 +49,8 @@ func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *object.Object { return obj } -func AddAttribute(obj *object.Object, key, val string) { - var attr object.Attribute +func AddAttribute(obj *objectSDK.Object, key, val string) { + var attr objectSDK.Attribute attr.SetKey(key) attr.SetValue(val) @@ -59,7 +59,7 @@ func AddAttribute(obj *object.Object, key, val string) { obj.SetAttributes(attrs...) } -func AddPayload(obj *object.Object, size int) { +func AddPayload(obj *objectSDK.Object, size int) { buf := make([]byte, size) _, _ = rand.Read(buf) diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 507bfcd89..2c5fa0233 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -8,7 +8,7 @@ import ( objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -34,7 +34,7 @@ func TestCounters(t *testing.T) { t.Run("put", func(t *testing.T) { t.Parallel() db := newDB(t) - oo := make([]*object.Object, 0, objCount) + oo := make([]*objectSDK.Object, 0, objCount) for i := 0; i < objCount; i++ { oo = append(oo, testutil.GenerateObject()) } @@ -191,7 +191,7 @@ func TestCounters_Expired(t *testing.T) { oo := make([]oid.Address, objCount) for i := range oo { - oo[i] = putWithExpiration(t, db, object.TypeRegular, epoch+1) + oo[i] = putWithExpiration(t, db, objectSDK.TypeRegular, epoch+1) } // 1. objects are available and counters are correct @@ -275,12 +275,12 @@ func TestCounters_Expired(t *testing.T) { require.Equal(t, uint64(len(oo)), c.Logic()) } -func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*object.Object { +func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*objectSDK.Object { var prm meta.PutPrm var err error parent := testutil.GenerateObject() - oo := make([]*object.Object, 0, count) + oo := make([]*objectSDK.Object, 0, count) for i := 0; i < count; i++ { o := testutil.GenerateObject() if withParent { diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index e7de46e15..f5341ff2e 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -14,7 +14,7 @@ import ( v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/mr-tron/base58" "go.etcd.io/bbolt" "go.uber.org/zap" @@ -39,7 +39,7 @@ type DB struct { modeMtx sync.RWMutex mode mode.Mode - matchers map[object.SearchMatchType]matcher + matchers map[objectSDK.SearchMatchType]matcher boltDB *bbolt.DB @@ -89,20 +89,20 @@ func New(opts ...Option) *DB { return &DB{ cfg: c, - matchers: map[object.SearchMatchType]matcher{ - object.MatchUnknown: { + matchers: map[objectSDK.SearchMatchType]matcher{ + objectSDK.MatchUnknown: { matchSlow: unknownMatcher, matchBucket: unknownMatcherBucket, }, - object.MatchStringEqual: { + objectSDK.MatchStringEqual: { matchSlow: stringEqualMatcher, matchBucket: stringEqualMatcherBucket, }, - object.MatchStringNotEqual: { + objectSDK.MatchStringNotEqual: { matchSlow: stringNotEqualMatcher, matchBucket: stringNotEqualMatcherBucket, }, - object.MatchCommonPrefix: { + objectSDK.MatchCommonPrefix: { matchSlow: stringCommonPrefixMatcher, matchBucket: stringCommonPrefixMatcherBucket, }, @@ -276,7 +276,7 @@ func bucketKeyHelper(hdr string, val string) []byte { return v case v2object.FilterHeaderSplitID: - s := object.NewSplitID() + s := objectSDK.NewSplitID() err := s.Parse(val) if err != nil { diff --git a/pkg/local_object_storage/metabase/expired_test.go b/pkg/local_object_storage/metabase/expired_test.go index 5755b5cdb..9a6bcc5db 100644 --- a/pkg/local_object_storage/metabase/expired_test.go +++ b/pkg/local_object_storage/metabase/expired_test.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -77,7 +77,7 @@ func TestDB_SelectExpired(t *testing.T) { require.Contains(t, expired, getAddressSafe(t, expiredObj22)) } -func getAddressSafe(t *testing.T, o *object.Object) oid.Address { +func getAddressSafe(t *testing.T, o *objectSDK.Object) oid.Address { cid, set := o.ContainerID() if !set { t.Fatalf("container id required") diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 76744aa33..7ba5a68a2 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -12,7 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -276,7 +276,7 @@ func (db *DB) markAsGC(graveyardBKT, garbageBKT *bbolt.Bucket, key []byte) (bool return false, garbageBKT.Put(key, zeroValue) } -func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Bucket, targetKey []byte, cnr cid.ID, obj *object.Object, res *InhumeRes) error { +func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Bucket, targetKey []byte, cnr cid.ID, obj *objectSDK.Object, res *InhumeRes) error { containerID, _ := obj.ContainerID() if inGraveyardWithKey(targetKey, graveyardBKT, garbageBKT) == 0 { res.availableImhumed++ @@ -285,7 +285,7 @@ func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Buc // if object is stored, and it is regular object then update bucket // with container size estimations - if obj.Type() == object.TypeRegular { + if obj.Type() == objectSDK.TypeRegular { err := changeContainerSize(tx, cnr, obj.PayloadSize(), false) if err != nil { return err diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index 78bfd2914..a1e21ef25 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -12,7 +12,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -21,13 +22,13 @@ import ( // ExpiredObject is a descriptor of expired object from DB. type ExpiredObject struct { - typ object.Type + typ objectSDK.Type addr oid.Address } // Type returns type of the expired object. -func (e *ExpiredObject) Type() object.Type { +func (e *ExpiredObject) Type() objectSDK.Type { return e.typ } diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index e07184eb6..034a931d2 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -9,7 +9,7 @@ import ( object2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -20,19 +20,19 @@ func TestDB_IterateExpired(t *testing.T) { const epoch = 13 - mAlive := map[object.Type]oid.Address{} - mExpired := map[object.Type]oid.Address{} + mAlive := map[objectSDK.Type]oid.Address{} + mExpired := map[objectSDK.Type]oid.Address{} - for _, typ := range []object.Type{ - object.TypeRegular, - object.TypeTombstone, - object.TypeLock, + for _, typ := range []objectSDK.Type{ + objectSDK.TypeRegular, + objectSDK.TypeTombstone, + objectSDK.TypeLock, } { mAlive[typ] = putWithExpiration(t, db, typ, epoch) mExpired[typ] = putWithExpiration(t, db, typ, epoch-1) } - expiredLocked := putWithExpiration(t, db, object.TypeRegular, epoch-1) + expiredLocked := putWithExpiration(t, db, objectSDK.TypeRegular, epoch-1) require.NoError(t, db.Lock(context.Background(), expiredLocked.Container(), oidtest.ID(), []oid.ID{expiredLocked.Object()})) @@ -56,7 +56,7 @@ func TestDB_IterateExpired(t *testing.T) { require.Empty(t, mExpired) } -func putWithExpiration(t *testing.T, db *meta.DB, typ object.Type, expiresAt uint64) oid.Address { +func putWithExpiration(t *testing.T, db *meta.DB, typ objectSDK.Type, expiresAt uint64) oid.Address { obj := testutil.GenerateObject() obj.SetType(typ) testutil.AddAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.FormatUint(expiresAt, 10)) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 337265318..37a574a02 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -124,15 +124,15 @@ loop: continue } - var objType object.Type + var objType objectSDK.Type switch prefix { case primaryPrefix: - objType = object.TypeRegular + objType = objectSDK.TypeRegular case lockersPrefix: - objType = object.TypeLock + objType = objectSDK.TypeLock case tombstonePrefix: - objType = object.TypeTombstone + objType = objectSDK.TypeTombstone default: continue } @@ -175,7 +175,7 @@ loop: // selectNFromBucket similar to selectAllFromBucket but uses cursor to find // object to start selecting from. Ignores inhumed objects. func selectNFromBucket(bkt *bbolt.Bucket, // main bucket - objType object.Type, // type of the objects stored in the main bucket + objType objectSDK.Type, // type of the objects stored in the main bucket graveyardBkt, garbageBkt *bbolt.Bucket, // cached graveyard buckets cidRaw []byte, // container ID prefix, optimization cnt cid.ID, // container ID diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index f3388daf0..50aac223b 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -11,7 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -79,7 +79,7 @@ func (db *DB) lockInternal(locked []oid.ID, cnr cid.ID, locker oid.ID) error { key := make([]byte, cidSize) return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { - if firstIrregularObjectType(tx, cnr, bucketKeysLocked...) != object.TypeRegular { + if firstIrregularObjectType(tx, cnr, bucketKeysLocked...) != objectSDK.TypeRegular { return logicerr.Wrap(apistatus.LockNonRegularObject{}) } diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index d5e063431..b442296fd 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -9,7 +9,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" @@ -28,10 +28,10 @@ func TestDB_Lock(t *testing.T) { }) t.Run("(ir)regular", func(t *testing.T) { - for _, typ := range [...]object.Type{ - object.TypeTombstone, - object.TypeLock, - object.TypeRegular, + for _, typ := range [...]objectSDK.Type{ + objectSDK.TypeTombstone, + objectSDK.TypeLock, + objectSDK.TypeRegular, } { obj := objecttest.Object() obj.SetType(typ) @@ -47,7 +47,7 @@ func TestDB_Lock(t *testing.T) { // try to lock it err = db.Lock(context.Background(), cnr, oidtest.ID(), []oid.ID{id}) - if typ == object.TypeRegular { + if typ == objectSDK.TypeRegular { require.NoError(t, err, typ) } else { require.ErrorAs(t, err, &e, typ) @@ -180,7 +180,7 @@ func TestDB_Lock_Expired(t *testing.T) { db := newDB(t, meta.WithEpochState(es)) // put an object - addr := putWithExpiration(t, db, object.TypeRegular, 124) + addr := putWithExpiration(t, db, objectSDK.TypeRegular, 124) // expire the obj es.e = 125 @@ -242,10 +242,10 @@ func TestDB_IsLocked(t *testing.T) { } // putAndLockObj puts object, returns it and its locker. -func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*object.Object, *object.Object) { +func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*objectSDK.Object, *objectSDK.Object) { cnr := cidtest.ID() - lockedObjs := make([]*object.Object, 0, numOfLockedObjs) + lockedObjs := make([]*objectSDK.Object, 0, numOfLockedObjs) lockedObjIDs := make([]oid.ID, 0, numOfLockedObjs) for i := 0; i < numOfLockedObjs; i++ { @@ -261,7 +261,7 @@ func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*object.Ob lockObj := testutil.GenerateObjectWithCID(cnr) lockID, _ := lockObj.ID() - lockObj.SetType(object.TypeLock) + lockObj.SetType(objectSDK.TypeLock) err := putBig(db, lockObj) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 998ce9b94..6a7acd7cc 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -13,7 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -31,14 +31,14 @@ type ( cnr cid.ID - fastFilters, slowFilters object.SearchFilters + fastFilters, slowFilters objectSDK.SearchFilters } ) // SelectPrm groups the parameters of Select operation. type SelectPrm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters } // SelectRes groups the resulting values of Select operation. @@ -52,7 +52,7 @@ func (p *SelectPrm) SetContainerID(cnr cid.ID) { } // SetFilters is a Select option to set the object filters. -func (p *SelectPrm) SetFilters(fs object.SearchFilters) { +func (p *SelectPrm) SetFilters(fs objectSDK.SearchFilters) { p.filters = fs } @@ -98,7 +98,7 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err })) } -func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs object.SearchFilters, currEpoch uint64) ([]oid.Address, error) { +func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters, currEpoch uint64) ([]oid.Address, error) { group, err := groupFilters(fs) if err != nil { return nil, err @@ -186,7 +186,7 @@ func selectAllFromBucket(tx *bbolt.Tx, name []byte, to map[string]int, fNum int) func (db *DB) selectFastFilter( tx *bbolt.Tx, cnr cid.ID, // container we search on - f object.SearchFilter, // fast filter + f objectSDK.SearchFilter, // fast filter to map[string]int, // resulting cache fNum int, // index of filter ) { @@ -220,7 +220,7 @@ func (db *DB) selectFastFilter( default: // user attribute bucketName := attributeBucketName(cnr, f.Header(), bucketName) - if f.Operation() == object.MatchNotPresent { + if f.Operation() == objectSDK.MatchNotPresent { selectOutsideFKBT(tx, allBucketNames(cnr), bucketName, to, fNum) } else { db.selectFromFKBT(tx, bucketName, f, to, fNum) @@ -244,7 +244,7 @@ func allBucketNames(cnr cid.ID) (names [][]byte) { return } -func bucketNamesForType(cnr cid.ID, mType object.SearchMatchType, typeVal string) (names [][]byte) { +func bucketNamesForType(cnr cid.ID, mType objectSDK.SearchMatchType, typeVal string) (names [][]byte) { appendNames := func(key string) { fns, ok := mBucketNaming[key] if ok { @@ -256,15 +256,15 @@ func bucketNamesForType(cnr cid.ID, mType object.SearchMatchType, typeVal string switch mType { default: - case object.MatchStringNotEqual: + case objectSDK.MatchStringNotEqual: for key := range mBucketNaming { if key != typeVal { appendNames(key) } } - case object.MatchStringEqual: + case objectSDK.MatchStringEqual: appendNames(typeVal) - case object.MatchCommonPrefix: + case objectSDK.MatchCommonPrefix: for key := range mBucketNaming { if strings.HasPrefix(key, typeVal) { appendNames(key) @@ -280,7 +280,7 @@ func bucketNamesForType(cnr cid.ID, mType object.SearchMatchType, typeVal string func (db *DB) selectFromFKBT( tx *bbolt.Tx, name []byte, // fkbt root bucket name - f object.SearchFilter, // filter for operation and value + f objectSDK.SearchFilter, // filter for operation and value to map[string]int, // resulting cache fNum int, // index of filter ) { // @@ -357,7 +357,7 @@ func selectOutsideFKBT( func (db *DB) selectFromList( tx *bbolt.Tx, name []byte, // list root bucket name - f object.SearchFilter, // filter for operation and value + f objectSDK.SearchFilter, // filter for operation and value to map[string]int, // resulting cache fNum int, // index of filter ) { // @@ -372,7 +372,7 @@ func (db *DB) selectFromList( ) switch op := f.Operation(); op { - case object.MatchStringEqual: + case objectSDK.MatchStringEqual: lst, err = decodeList(bkt.Get(bucketKeyHelper(f.Header(), f.Value()))) if err != nil { db.log.Debug(logs.MetabaseCantDecodeListBucketLeaf, zap.String("error", err.Error())) @@ -416,7 +416,7 @@ func (db *DB) selectFromList( // selectObjectID processes objectID filter with in-place optimizations. func (db *DB) selectObjectID( tx *bbolt.Tx, - f object.SearchFilter, + f objectSDK.SearchFilter, cnr cid.ID, to map[string]int, // resulting cache fNum int, // index of filter @@ -436,7 +436,7 @@ func (db *DB) selectObjectID( } switch op := f.Operation(); op { - case object.MatchStringEqual: + case objectSDK.MatchStringEqual: var id oid.ID if err := id.DecodeString(f.Value()); err == nil { appendOID(id) @@ -451,7 +451,7 @@ func (db *DB) selectObjectID( return } - for _, bucketName := range bucketNamesForType(cnr, object.MatchStringNotEqual, "") { + for _, bucketName := range bucketNamesForType(cnr, objectSDK.MatchStringNotEqual, "") { // copy-paste from DB.selectAllFrom bkt := tx.Bucket(bucketName) if bkt == nil { @@ -475,7 +475,7 @@ func (db *DB) selectObjectID( } // matchSlowFilters return true if object header is matched by all slow filters. -func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f object.SearchFilters, currEpoch uint64) bool { +func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.SearchFilters, currEpoch uint64) bool { if len(f) == 0 { return true } @@ -521,10 +521,10 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f object.SearchFi // groupFilters divides filters in two groups: fast and slow. Fast filters // processed by indexes and slow filters processed after by unmarshaling // object headers. -func groupFilters(filters object.SearchFilters) (filterGroup, error) { +func groupFilters(filters objectSDK.SearchFilters) (filterGroup, error) { res := filterGroup{ - fastFilters: make(object.SearchFilters, 0, len(filters)), - slowFilters: make(object.SearchFilters, 0, len(filters)), + fastFilters: make(objectSDK.SearchFilters, 0, len(filters)), + slowFilters: make(objectSDK.SearchFilters, 0, len(filters)), } for i := range filters { @@ -557,9 +557,9 @@ func markAddressInCache(cache map[string]int, fNum int, addr string) { } // Returns true if at least 1 object can satisfy fs. -func checkNonEmpty(fs object.SearchFilters) bool { +func checkNonEmpty(fs objectSDK.SearchFilters) bool { for i := range fs { - if fs[i].Operation() == object.MatchNotPresent && isSystemKey(fs[i].Header()) { + if fs[i].Operation() == objectSDK.MatchNotPresent && isSystemKey(fs[i].Header()) { return true } } diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index c50fd051f..4e58ec20b 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -6,7 +6,7 @@ import ( "fmt" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -120,7 +120,7 @@ const ( addressKeySize = cidSize + objectKeySize ) -var splitInfoError *object.SplitInfoError // for errors.As comparisons +var splitInfoError *objectSDK.SplitInfoError // for errors.As comparisons func bucketName(cnr cid.ID, prefix byte, key []byte) []byte { key[0] = prefix @@ -221,7 +221,7 @@ func objectKey(obj oid.ID, key []byte) []byte { // if meets irregular object container in objs - returns its type, otherwise returns object.TypeRegular. // // firstIrregularObjectType(tx, cnr, obj) usage allows getting object type. -func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) object.Type { +func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) objectSDK.Type { if len(objs) == 0 { panic("empty object list in firstIrregularObjectType") } @@ -229,11 +229,11 @@ func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) object var keys [2][1 + cidSize]byte irregularTypeBuckets := [...]struct { - typ object.Type + typ objectSDK.Type name []byte }{ - {object.TypeTombstone, tombstoneBucketName(idCnr, keys[0][:])}, - {object.TypeLock, bucketNameLockers(idCnr, keys[1][:])}, + {objectSDK.TypeTombstone, tombstoneBucketName(idCnr, keys[0][:])}, + {objectSDK.TypeLock, bucketNameLockers(idCnr, keys[1][:])}, } for i := range objs { @@ -244,7 +244,7 @@ func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) object } } - return object.TypeRegular + return objectSDK.TypeRegular } // return true if provided object is of LOCK type. diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index b8d516871..2221d57c1 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -335,7 +335,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { errGroup.Go(func() error { batch := make([]oid.Address, 0, batchSize) expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { - if o.Type() != object.TypeTombstone && o.Type() != object.TypeLock { + if o.Type() != objectSDK.TypeTombstone && o.Type() != objectSDK.TypeLock { batch = append(batch, o.Address()) if len(batch) == batchSize { @@ -519,7 +519,7 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { batch := make([]oid.Address, 0, batchSize) expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { - if o.Type() == object.TypeLock { + if o.Type() == objectSDK.TypeLock { batch = append(batch, o.Address()) if len(batch) == batchSize { diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index d94029976..bc895d67b 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -17,7 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" @@ -41,7 +41,7 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return len(data) <= 1<<20 }, }, diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index dd21745cc..bc13c622b 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -9,7 +9,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -87,7 +87,7 @@ func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { return res, fmt.Errorf("can't list stored containers: %w", err) } - filters := object.NewSearchFilters() + filters := objectSDK.NewSearchFilters() filters.AddPhyFilter() for i := range lst { diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 75010179a..61f2bb9f0 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -15,7 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -38,7 +38,7 @@ func TestShard_Lock(t *testing.T) { blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(2), blobovniczatree.WithBlobovniczaShallowWidth(2)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return len(data) <= 1<<20 }, }, @@ -70,7 +70,7 @@ func TestShard_Lock(t *testing.T) { objID, _ := obj.ID() lock := testutil.GenerateObjectWithCID(cnr) - lock.SetType(object.TypeLock) + lock.SetType(objectSDK.TypeLock) lockID, _ := lock.ID() // put the object diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index d76c240b7..ad0fc15ca 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -13,7 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -96,7 +96,7 @@ func TestCounters(t *testing.T) { require.Equal(t, mode.ReadWrite, mm.mode) const objNumber = 10 - oo := make([]*object.Object, objNumber) + oo := make([]*objectSDK.Object, objNumber) for i := 0; i < objNumber; i++ { oo[i] = testutil.GenerateObject() } @@ -240,7 +240,7 @@ func shardWithMetrics(t *testing.T, path string) (*shard.Shard, *metricsStore) { return sh, mm } -func addrFromObjs(oo []*object.Object) []oid.Address { +func addrFromObjs(oo []*objectSDK.Object) []oid.Address { aa := make([]oid.Address, len(oo)) for i := 0; i < len(oo); i++ { diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index 79dc4846e..688b7aae7 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -17,14 +17,14 @@ import ( // PutPrm groups the parameters of Put operation. type PutPrm struct { - obj *object.Object + obj *objectSDK.Object } // PutRes groups the resulting values of Put operation. type PutRes struct{} // SetObject is a Put option to set object to save. -func (p *PutPrm) SetObject(obj *object.Object) { +func (p *PutPrm) SetObject(obj *objectSDK.Object) { p.obj = obj } diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 362fd86c5..e94482e36 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -29,7 +29,7 @@ type RngPrm struct { // RngRes groups the resulting values of GetRange operation. type RngRes struct { - obj *object.Object + obj *objectSDK.Object hasMeta bool } @@ -54,7 +54,7 @@ func (p *RngPrm) SetIgnoreMeta(ignore bool) { // Object returns the requested object part. // // Instance payload contains the requested range of the original object. -func (r RngRes) Object() *object.Object { +func (r RngRes) Object() *objectSDK.Object { return r.obj } @@ -71,7 +71,7 @@ func (r RngRes) HasMeta() bool { // Returns ErrRangeOutOfBounds if the requested object range is out of bounds. // Returns an error of type apistatus.ObjectNotFound if the requested object is missing. // Returns an error of type apistatus.ObjectAlreadyRemoved if the requested object has been marked as removed in shard. -// Returns the object.ErrObjectIsExpired if the object is presented but already expired. +// Returns the objectSDK.ErrObjectIsExpired if the object is presented but already expired. // Returns the ErrShardDisabled if the shard is disabled. func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.GetRange", @@ -91,7 +91,7 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { return RngRes{}, ErrShardDisabled } - cb := func(stor *blobstor.BlobStor, id []byte) (*object.Object, error) { + cb := func(stor *blobstor.BlobStor, id []byte) (*objectSDK.Object, error) { var getRngPrm common.GetRangePrm getRngPrm.Address = prm.addr getRngPrm.Range.SetOffset(prm.off) @@ -103,13 +103,13 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { return nil, err } - obj := object.New() + obj := objectSDK.New() obj.SetPayload(res.Data) return obj, nil } - wc := func(c writecache.Cache) (*object.Object, error) { + wc := func(c writecache.Cache) (*objectSDK.Object, error) { res, err := c.Get(ctx, prm.addr) if err != nil { return nil, err @@ -122,7 +122,7 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { return nil, logicerr.Wrap(apistatus.ObjectOutOfRange{}) } - obj := object.New() + obj := objectSDK.New() obj.SetPayload(payload[from:to]) return obj, nil } diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index 2d4d473b4..1615f5fbe 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -7,7 +7,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -16,7 +16,7 @@ import ( // SelectPrm groups the parameters of Select operation. type SelectPrm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters } // SelectRes groups the resulting values of Select operation. @@ -30,7 +30,7 @@ func (p *SelectPrm) SetContainerID(cnr cid.ID) { } // SetFilters is a Select option to set the object filters. -func (p *SelectPrm) SetFilters(fs object.SearchFilters) { +func (p *SelectPrm) SetFilters(fs objectSDK.SearchFilters) { p.filters = fs } diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 7b2fdb5d1..a9a8e4ea7 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -15,7 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" @@ -56,7 +56,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), - Policy: func(_ *object.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, data []byte) bool { return len(data) <= 1<<20 }, }, diff --git a/pkg/local_object_storage/util/splitinfo.go b/pkg/local_object_storage/util/splitinfo.go index 4a6d22268..6ae1c3e46 100644 --- a/pkg/local_object_storage/util/splitinfo.go +++ b/pkg/local_object_storage/util/splitinfo.go @@ -1,12 +1,12 @@ package util import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) // MergeSplitInfo ignores conflicts and rewrites `to` with non empty values // from `from`. -func MergeSplitInfo(from, to *object.SplitInfo) *object.SplitInfo { +func MergeSplitInfo(from, to *objectSDK.SplitInfo) *objectSDK.SplitInfo { to.SetSplitID(from.SplitID()) // overwrite SplitID and ignore conflicts if lp, ok := from.LastPart(); ok { diff --git a/pkg/local_object_storage/util/splitinfo_test.go b/pkg/local_object_storage/util/splitinfo_test.go index a0626db28..642fef3b8 100644 --- a/pkg/local_object_storage/util/splitinfo_test.go +++ b/pkg/local_object_storage/util/splitinfo_test.go @@ -5,7 +5,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -15,7 +15,7 @@ func TestMergeSplitInfo(t *testing.T) { uid, err := uuid.NewUUID() require.NoError(t, err) - splitID := object.NewSplitID() + splitID := objectSDK.NewSplitID() splitID.SetUUID(uid) var rawLinkID, rawLastID [32]byte @@ -30,35 +30,35 @@ func TestMergeSplitInfo(t *testing.T) { require.NoError(t, err) lastID.SetSHA256(rawLastID) - target := object.NewSplitInfo() // target is SplitInfo struct with all fields set + target := objectSDK.NewSplitInfo() // target is SplitInfo struct with all fields set target.SetSplitID(splitID) target.SetLastPart(lastID) target.SetLink(linkID) t.Run("merge empty", func(t *testing.T) { - to := object.NewSplitInfo() + to := objectSDK.NewSplitInfo() result := util.MergeSplitInfo(target, to) require.Equal(t, result, target) }) t.Run("merge link", func(t *testing.T) { - from := object.NewSplitInfo() + from := objectSDK.NewSplitInfo() from.SetSplitID(splitID) from.SetLastPart(lastID) - to := object.NewSplitInfo() + to := objectSDK.NewSplitInfo() to.SetLink(linkID) result := util.MergeSplitInfo(from, to) require.Equal(t, result, target) }) t.Run("merge last", func(t *testing.T) { - from := object.NewSplitInfo() + from := objectSDK.NewSplitInfo() from.SetSplitID(splitID) from.SetLink(linkID) - to := object.NewSplitInfo() + to := objectSDK.NewSplitInfo() to.SetLastPart(lastID) result := util.MergeSplitInfo(from, to) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 21f968b04..779bac39a 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -13,7 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/mr-tron/base58" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -129,7 +129,7 @@ func (c *cache) flushSmallObjects() { var count int for i := range m { - obj := object.New() + obj := objectSDK.New() if err := obj.Unmarshal(m[i].data); err != nil { continue } @@ -201,7 +201,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - var obj object.Object + var obj objectSDK.Object err = obj.Unmarshal(data) if err != nil { c.reportFlushError("can't unmarshal an object", sAddr, metaerr.Wrap(err)) @@ -231,7 +231,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { func (c *cache) workerFlushSmall() { defer c.wg.Done() - var obj *object.Object + var obj *objectSDK.Object for { // Give priority to direct put. select { @@ -251,7 +251,7 @@ func (c *cache) workerFlushSmall() { } // flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(ctx context.Context, obj *object.Object, data []byte, st StorageType) error { +func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []byte, st StorageType) error { var err error defer func() { @@ -322,7 +322,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { return err } - var obj object.Object + var obj objectSDK.Object if err := obj.Unmarshal(data); err != nil { c.reportFlushError("can't unmarshal an object from the DB", sa, metaerr.Wrap(err)) if ignoreErrors { diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index e8224ce5d..2223bef02 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -16,7 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" @@ -28,7 +28,7 @@ import ( type objectPair struct { addr oid.Address - obj *object.Object + obj *objectSDK.Object } func TestFlush(t *testing.T) { @@ -211,14 +211,14 @@ func putObject(t *testing.T, c Cache, size int) objectPair { } -func newObject(t *testing.T, size int) (*object.Object, []byte) { - obj := object.New() +func newObject(t *testing.T, size int) (*objectSDK.Object, []byte) { + obj := objectSDK.New() ver := versionSDK.Current() obj.SetID(oidtest.ID()) obj.SetOwnerID(usertest.ID()) obj.SetContainerID(cidtest.ID()) - obj.SetType(object.TypeRegular) + obj.SetType(objectSDK.TypeRegular) obj.SetVersion(&ver) obj.SetPayloadChecksum(checksumtest.Checksum()) obj.SetPayloadHomomorphicHash(checksumtest.Checksum()) diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 962c9c39a..067ff5ae5 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.uber.org/zap" @@ -24,8 +24,8 @@ type Info struct { // Cache represents write-cache for objects. type Cache interface { - Get(ctx context.Context, address oid.Address) (*object.Object, error) - Head(context.Context, oid.Address) (*object.Object, error) + Get(ctx context.Context, address oid.Address) (*objectSDK.Object, error) + Head(context.Context, oid.Address) (*objectSDK.Object, error) // Delete removes object referenced by the given oid.Address from the // Cache. Returns any error encountered that prevented the object to be // removed. @@ -58,7 +58,7 @@ type cache struct { compressFlags map[string]struct{} // flushCh is a channel with objects to flush. - flushCh chan *object.Object + flushCh chan *objectSDK.Object // closeCh is close channel, protected by modeMtx. closeCh chan struct{} // wg is a wait group for flush workers. @@ -75,7 +75,7 @@ const wcStorageType = "write-cache" type objectInfo struct { addr string data []byte - obj *object.Object + obj *objectSDK.Object } const ( @@ -91,7 +91,7 @@ var ( // New creates new writecache instance. func New(opts ...Option) Cache { c := &cache{ - flushCh: make(chan *object.Object), + flushCh: make(chan *objectSDK.Object), mode: mode.ReadWrite, compressFlags: make(map[string]struct{}), diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index b7f7dc20f..18155849b 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -13,7 +13,7 @@ import ( metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -166,7 +166,7 @@ func (x *multiClient) iterateClients(ctx context.Context, f func(clientcore.Clie // non-status logic error that could be returned // from the SDK client; should not be considered // as a connection error - var siErr *object.SplitInfoError + var siErr *objectSDK.SplitInfoError success := err == nil || errors.Is(err, context.Canceled) || errors.As(err, &siErr) if success || firstErr == nil || errors.Is(firstErr, errRecentlyFailed) { @@ -195,7 +195,7 @@ func (x *multiClient) ReportError(err error) { // non-status logic error that could be returned // from the SDK client; should not be considered // as a connection error - var siErr *object.SplitInfoError + var siErr *objectSDK.SplitInfoError if errors.As(err, &siErr) { return } diff --git a/pkg/services/object/acl/eacl/v2/eacl_test.go b/pkg/services/object/acl/eacl/v2/eacl_test.go index ce5d98d5d..6f879123b 100644 --- a/pkg/services/object/acl/eacl/v2/eacl_test.go +++ b/pkg/services/object/acl/eacl/v2/eacl_test.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -22,12 +22,12 @@ type testLocalStorage struct { expAddr oid.Address - obj *object.Object + obj *objectSDK.Object err error } -func (s *testLocalStorage) Head(ctx context.Context, addr oid.Address) (*object.Object, error) { +func (s *testLocalStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { require.True(s.t, addr.Container().Equals(s.expAddr.Container())) require.True(s.t, addr.Object().Equals(s.expAddr.Object())) @@ -69,11 +69,11 @@ func TestHeadRequest(t *testing.T) { meta.SetXHeaders(xHdrs) - obj := object.New() + obj := objectSDK.New() attrKey := "attr_key" attrVal := "attr_val" - var attr object.Attribute + var attr objectSDK.Attribute attr.SetKey(attrKey) attr.SetValue(attrVal) obj.SetAttributes(attr) diff --git a/pkg/services/object/acl/eacl/v2/headers.go b/pkg/services/object/acl/eacl/v2/headers.go index 095810848..c7a1b7729 100644 --- a/pkg/services/object/acl/eacl/v2/headers.go +++ b/pkg/services/object/acl/eacl/v2/headers.go @@ -11,7 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -28,7 +28,7 @@ type cfg struct { } type ObjectStorage interface { - Head(context.Context, oid.Address) (*object.Object, error) + Head(context.Context, oid.Address) (*objectSDK.Object, error) } type Request interface { @@ -141,7 +141,7 @@ func (h *cfg) readObjectHeadersFromRequestXHeaderSource(m requestXHeaderSource, oV2.SetObjectID(v.GetObjectID()) oV2.SetHeader(v.GetHeader()) - dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) + dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) } case *objectV2.SearchRequest: cnrV2 := req.GetBody().GetContainerID() @@ -171,7 +171,7 @@ func (h *cfg) readObjectHeadersResponseXHeaderSource(m responseXHeaderSource, ds oV2.SetObjectID(v.GetObjectID()) oV2.SetHeader(v.GetHeader()) - dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) + dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) } case *objectV2.HeadResponse: oV2 := new(objectV2.Object) @@ -197,7 +197,7 @@ func (h *cfg) readObjectHeadersResponseXHeaderSource(m responseXHeaderSource, ds oV2.SetHeader(hdr) - dst.objectHeaders = headersFromObject(object.NewFromV2(oV2), h.cnr, h.obj) + dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) } return nil } diff --git a/pkg/services/object/acl/eacl/v2/object.go b/pkg/services/object/acl/eacl/v2/object.go index 0a63981cb..690e4aa70 100644 --- a/pkg/services/object/acl/eacl/v2/object.go +++ b/pkg/services/object/acl/eacl/v2/object.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -26,7 +26,7 @@ func u64Value(v uint64) string { return strconv.FormatUint(v, 10) } -func headersFromObject(obj *object.Object, cnr cid.ID, oid *oid.ID) []eaclSDK.Header { +func headersFromObject(obj *objectSDK.Object, cnr cid.ID, oid *oid.ID) []eaclSDK.Header { var count int for obj := obj; obj != nil; obj = obj.Parent() { count += 9 + len(obj.Attributes()) diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index 343efbc04..b10f045ee 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apiclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -29,11 +29,11 @@ type execCtx struct { log *logger.Logger - tombstone *object.Tombstone + tombstone *objectSDK.Tombstone - splitInfo *object.SplitInfo + splitInfo *objectSDK.SplitInfo - tombstoneObj *object.Object + tombstoneObj *objectSDK.Object } const ( @@ -241,9 +241,9 @@ func (exec *execCtx) initTombstoneObject() bool { return false } - exec.tombstoneObj = object.New() + exec.tombstoneObj = objectSDK.New() exec.tombstoneObj.SetContainerID(exec.containerID()) - exec.tombstoneObj.SetType(object.TypeTombstone) + exec.tombstoneObj.SetType(objectSDK.TypeTombstone) exec.tombstoneObj.SetPayload(payload) tokenSession := exec.commonParameters().SessionToken() @@ -256,7 +256,7 @@ func (exec *execCtx) initTombstoneObject() bool { exec.tombstoneObj.SetOwnerID(&localUser) } - var a object.Attribute + var a objectSDK.Attribute a.SetKey(objectV2.SysAttributeExpEpoch) a.SetValue(strconv.FormatUint(exec.tombstone.ExpirationEpoch(), 10)) diff --git a/pkg/services/object/delete/local.go b/pkg/services/object/delete/local.go index 34839b194..ad3e10bc6 100644 --- a/pkg/services/object/delete/local.go +++ b/pkg/services/object/delete/local.go @@ -4,7 +4,7 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -35,7 +35,7 @@ func (exec *execCtx) formTombstone(ctx context.Context) (ok bool) { return false } - exec.tombstone = object.NewTombstone() + exec.tombstone = objectSDK.NewTombstone() exec.tombstone.SetExpirationEpoch( exec.svc.netInfo.CurrentEpoch() + tsLifetime, ) diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index 11ff13b45..5c09ad123 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -9,7 +9,7 @@ import ( searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "go.uber.org/zap" @@ -41,7 +41,7 @@ type cfg struct { header interface { // must return (nil, nil) for PHY objects - splitInfo(context.Context, *execCtx) (*object.SplitInfo, error) + splitInfo(context.Context, *execCtx) (*objectSDK.SplitInfo, error) children(context.Context, *execCtx) ([]oid.ID, error) @@ -85,7 +85,7 @@ func New(opts ...Option) *Service { // WithLogger returns option to specify Delete service's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Delete service"))} + c.log = &logger.Logger{Logger: l.With(zap.String("component", "objectSDK.Delete service"))} } } diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index f6341f02a..b8e8e6324 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -7,7 +7,7 @@ import ( getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -21,7 +21,7 @@ type simpleIDWriter struct { ids []oid.ID } -func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oid.Address) (*object.Object, error) { +func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oid.Address) (*objectSDK.Object, error) { wr := getsvc.NewSimpleObjectWriter() p := getsvc.HeadPrm{} @@ -38,10 +38,10 @@ func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oi return wr.Object(), nil } -func (w *headSvcWrapper) splitInfo(ctx context.Context, exec *execCtx) (*object.SplitInfo, error) { +func (w *headSvcWrapper) splitInfo(ctx context.Context, exec *execCtx) (*objectSDK.SplitInfo, error) { _, err := w.headAddress(ctx, exec, exec.address()) - var errSplitInfo *object.SplitInfoError + var errSplitInfo *objectSDK.SplitInfoError switch { case err == nil: @@ -83,8 +83,8 @@ func (w *headSvcWrapper) previous(ctx context.Context, exec *execCtx, id oid.ID) } func (w *searchSvcWrapper) splitMembers(ctx context.Context, exec *execCtx) ([]oid.ID, error) { - fs := object.SearchFilters{} - fs.AddSplitIDFilter(object.MatchStringEqual, exec.splitInfo.SplitID()) + fs := objectSDK.SearchFilters{} + fs.AddSplitIDFilter(objectSDK.MatchStringEqual, exec.splitInfo.SplitID()) wr := new(simpleIDWriter) diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 457193a59..e3037a70b 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -5,7 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" ) @@ -74,7 +74,7 @@ func (s *Service) get(ctx context.Context, prm RequestParameters) error { localStorage: s.localStorage, prm: prm, - infoSplit: object.NewSplitInfo(), + infoSplit: objectSDK.NewSplitInfo(), } exec.setLogger(s.log) diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index 580c0b58c..40aa3f62e 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -17,7 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -31,7 +31,7 @@ type getRequestForwarder struct { Stream *streamObjectWriter } -func (f *getRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *getRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*objectSDK.Object, error) { ctx, span := tracing.StartSpanFromContext(ctx, "getRequestForwarder.forwardRequestToNode", trace.WithAttributes(attribute.String("address", addr.String())), ) @@ -85,7 +85,7 @@ func (f *getRequestForwarder) writeHeader(ctx context.Context, v *objectV2.GetOb var err error f.OnceHeaderSending.Do(func() { - err = f.Stream.WriteHeader(ctx, object.NewFromV2(obj)) + err = f.Stream.WriteHeader(ctx, objectSDK.NewFromV2(obj)) }) if err != nil { return errCouldNotWriteObjHeader(err) @@ -164,8 +164,8 @@ func (f *getRequestForwarder) readStream(ctx context.Context, c client.MultiAddr localProgress += len(origChunk) f.GlobalProgress += len(chunk) case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - return object.NewSplitInfoError(si) + si := objectSDK.NewSplitInfoFromV2(v) + return objectSDK.NewSplitInfoError(si) } } return nil diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 7858e2b53..8a56c59a6 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -17,7 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -30,7 +30,7 @@ type getRangeRequestForwarder struct { Stream *streamObjectRangeWriter } -func (f *getRangeRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *getRangeRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*objectSDK.Object, error) { ctx, span := tracing.StartSpanFromContext(ctx, "getRangeRequestForwarder.forwardRequestToNode", trace.WithAttributes(attribute.String("address", addr.String())), ) @@ -130,8 +130,8 @@ func (f *getRangeRequestForwarder) readStream(ctx context.Context, rangeStream * localProgress += len(origChunk) f.GlobalProgress += len(chunk) case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - return object.NewSplitInfoError(si) + si := objectSDK.NewSplitInfoFromV2(v) + return objectSDK.NewSplitInfoError(si) } } return nil diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index fa1506435..a1bce1517 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -16,7 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -30,7 +30,7 @@ type headRequestForwarder struct { Key *ecdsa.PrivateKey } -func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*object.Object, error) { +func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*objectSDK.Object, error) { ctx, span := tracing.StartSpanFromContext(ctx, "headRequestForwarder.forwardRequestToNode", trace.WithAttributes(attribute.String("address", addr.String())), ) @@ -82,15 +82,15 @@ func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr ne return nil, err } case *objectV2.SplitInfo: - si := object.NewSplitInfoFromV2(v) - return nil, object.NewSplitInfoError(si) + si := objectSDK.NewSplitInfoFromV2(v) + return nil, objectSDK.NewSplitInfoError(si) } objv2 := new(objectV2.Object) objv2.SetHeader(hdr) objv2.SetSignature(idSig) - obj := object.NewFromV2(objv2) + obj := objectSDK.NewFromV2(objv2) obj.SetID(f.ObjectAddr.Object()) return obj, nil diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index 1bd8befaf..d4bce178a 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -8,7 +8,7 @@ import ( objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" objutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) // Service implements Get operation of Object service v2. @@ -47,7 +47,7 @@ func (s *Service) Get(req *objectV2.GetRequest, stream objectSvc.GetObjectStream err = s.svc.Get(stream.Context(), *p) - var splitErr *object.SplitInfoError + var splitErr *objectSDK.SplitInfoError switch { case errors.As(err, &splitErr): @@ -66,7 +66,7 @@ func (s *Service) GetRange(req *objectV2.GetRangeRequest, stream objectSvc.GetOb err = s.svc.GetRange(stream.Context(), *p) - var splitErr *object.SplitInfoError + var splitErr *objectSDK.SplitInfoError switch { case errors.As(err, &splitErr): @@ -103,7 +103,7 @@ func (s *Service) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV err = s.svc.Head(ctx, *p) - var splitErr *object.SplitInfoError + var splitErr *objectSDK.SplitInfoError if errors.As(err, &splitErr) { setSplitInfoHeadResponse(splitErr.SplitInfo(), resp) diff --git a/pkg/services/object/get/v2/streamer.go b/pkg/services/object/get/v2/streamer.go index 4347ef416..ce9a5c767 100644 --- a/pkg/services/object/get/v2/streamer.go +++ b/pkg/services/object/get/v2/streamer.go @@ -5,7 +5,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) type streamObjectWriter struct { @@ -16,7 +16,7 @@ type streamObjectRangeWriter struct { objectSvc.GetObjectRangeStream } -func (s *streamObjectWriter) WriteHeader(_ context.Context, obj *object.Object) error { +func (s *streamObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Object) error { p := new(objectV2.GetObjectPartInit) objV2 := obj.ToV2() diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 519d9afa0..9ecc9167f 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -16,7 +16,7 @@ import ( getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" versionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "git.frostfs.info/TrueCloudLab/tzhash/tz" @@ -98,7 +98,7 @@ func (s *Service) toRangePrm(req *objectV2.GetRangeRequest, stream objectSvc.Get p.WithAddress(addr) p.WithRawFlag(body.GetRaw()) p.SetChunkWriter(streamWrapper) - p.SetRange(object.NewRangeFromV2(body.GetRange())) + p.SetRange(objectSDK.NewRangeFromV2(body.GetRange())) err = p.Validate() if err != nil { @@ -167,10 +167,10 @@ func (s *Service) toHashRangePrm(req *objectV2.GetRangeHashRequest) (*getsvc.Ran } rngsV2 := body.GetRanges() - rngs := make([]object.Range, len(rngsV2)) + rngs := make([]objectSDK.Range, len(rngsV2)) for i := range rngsV2 { - rngs[i] = *object.NewRangeFromV2(&rngsV2[i]) + rngs[i] = *objectSDK.NewRangeFromV2(&rngsV2[i]) } p.SetRangeList(rngs) @@ -198,7 +198,7 @@ type headResponseWriter struct { body *objectV2.HeadResponseBody } -func (w *headResponseWriter) WriteHeader(_ context.Context, hdr *object.Object) error { +func (w *headResponseWriter) WriteHeader(_ context.Context, hdr *objectSDK.Object) error { if w.mainOnly { w.body.SetHeaderPart(toShortObjectHeader(hdr)) } else { @@ -259,7 +259,7 @@ func (s *Service) toHeadPrm(req *objectV2.HeadRequest, resp *objectV2.HeadRespon return p, nil } -func splitInfoResponse(info *object.SplitInfo) *objectV2.GetResponse { +func splitInfoResponse(info *objectSDK.SplitInfo) *objectV2.GetResponse { resp := new(objectV2.GetResponse) body := new(objectV2.GetResponseBody) @@ -270,7 +270,7 @@ func splitInfoResponse(info *object.SplitInfo) *objectV2.GetResponse { return resp } -func splitInfoRangeResponse(info *object.SplitInfo) *objectV2.GetRangeResponse { +func splitInfoRangeResponse(info *objectSDK.SplitInfo) *objectV2.GetRangeResponse { resp := new(objectV2.GetRangeResponse) body := new(objectV2.GetRangeResponseBody) @@ -281,7 +281,7 @@ func splitInfoRangeResponse(info *object.SplitInfo) *objectV2.GetRangeResponse { return resp } -func setSplitInfoHeadResponse(info *object.SplitInfo, resp *objectV2.HeadResponse) { +func setSplitInfoHeadResponse(info *objectSDK.SplitInfo, resp *objectV2.HeadResponse) { resp.GetBody().SetHeaderPart(info.ToV2()) } @@ -297,7 +297,7 @@ func toHashResponse(typ refs.ChecksumType, res *getsvc.RangeHashRes) *objectV2.G return resp } -func toFullObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { +func toFullObjectHeader(hdr *objectSDK.Object) objectV2.GetHeaderPart { obj := hdr.ToV2() hs := new(objectV2.HeaderWithSignature) @@ -307,7 +307,7 @@ func toFullObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { return hs } -func toShortObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { +func toShortObjectHeader(hdr *objectSDK.Object) objectV2.GetHeaderPart { hdrV2 := hdr.ToV2().GetHeader() sh := new(objectV2.ShortHeader) @@ -322,11 +322,11 @@ func toShortObjectHeader(hdr *object.Object) objectV2.GetHeaderPart { return sh } -func groupAddressRequestForwarder(f func(context.Context, network.Address, client.MultiAddressClient, []byte) (*object.Object, error)) getsvc.RequestForwarder { - return func(ctx context.Context, info client.NodeInfo, c client.MultiAddressClient) (*object.Object, error) { +func groupAddressRequestForwarder(f func(context.Context, network.Address, client.MultiAddressClient, []byte) (*objectSDK.Object, error)) getsvc.RequestForwarder { + return func(ctx context.Context, info client.NodeInfo, c client.MultiAddressClient) (*objectSDK.Object, error) { var ( firstErr error - res *object.Object + res *objectSDK.Object key = info.PublicKey() ) diff --git a/pkg/services/object/get/writer.go b/pkg/services/object/get/writer.go index 78af5db41..1b842adeb 100644 --- a/pkg/services/object/get/writer.go +++ b/pkg/services/object/get/writer.go @@ -4,7 +4,7 @@ import ( "context" "io" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) // ChunkWriter is an interface of target component @@ -16,7 +16,7 @@ type ChunkWriter interface { // HeaderWriter is an interface of target component // to write object header. type HeaderWriter interface { - WriteHeader(context.Context, *object.Object) error + WriteHeader(context.Context, *objectSDK.Object) error } // ObjectWriter is an interface of target component to write object. @@ -26,7 +26,7 @@ type ObjectWriter interface { } type SimpleObjectWriter struct { - obj *object.Object + obj *objectSDK.Object pld []byte } @@ -45,11 +45,11 @@ type hasherWrapper struct { func NewSimpleObjectWriter() *SimpleObjectWriter { return &SimpleObjectWriter{ - obj: object.New(), + obj: objectSDK.New(), } } -func (s *SimpleObjectWriter) WriteHeader(_ context.Context, obj *object.Object) error { +func (s *SimpleObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Object) error { s.obj = obj s.pld = make([]byte, 0, obj.PayloadSize()) @@ -62,7 +62,7 @@ func (s *SimpleObjectWriter) WriteChunk(_ context.Context, p []byte) error { return nil } -func (s *SimpleObjectWriter) Object() *object.Object { +func (s *SimpleObjectWriter) Object() *objectSDK.Object { if len(s.pld) > 0 { s.obj.SetPayload(s.pld) } @@ -74,7 +74,7 @@ func (w *partWriter) WriteChunk(ctx context.Context, p []byte) error { return w.chunkWriter.WriteChunk(ctx, p) } -func (w *partWriter) WriteHeader(ctx context.Context, o *object.Object) error { +func (w *partWriter) WriteHeader(ctx context.Context, o *objectSDK.Object) error { return w.headWriter.WriteHeader(ctx, o) } diff --git a/pkg/services/object/head/remote.go b/pkg/services/object/head/remote.go index bcba181f2..c9c17d4d8 100644 --- a/pkg/services/object/head/remote.go +++ b/pkg/services/object/head/remote.go @@ -10,7 +10,7 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -64,7 +64,7 @@ func (p *RemoteHeadPrm) WithObjectAddress(v oid.Address) *RemoteHeadPrm { } // Head requests object header from the remote node. -func (h *RemoteHeader) Head(ctx context.Context, prm *RemoteHeadPrm) (*object.Object, error) { +func (h *RemoteHeader) Head(ctx context.Context, prm *RemoteHeadPrm) (*objectSDK.Object, error) { key, err := h.keyStorage.GetKey(nil) if err != nil { return nil, fmt.Errorf("(%T) could not receive private key: %w", h, err) diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index a54f6cd01..cfab77efe 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -14,7 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" ) @@ -77,7 +77,7 @@ type readPrmCommon struct { commonPrm } -// SetNetmapEpoch sets the epoch number to be used to locate the object. +// SetNetmapEpoch sets the epoch number to be used to locate the objectSDK. // // By default current epoch on the server will be used. func (x *readPrmCommon) SetNetmapEpoch(_ uint64) { @@ -111,11 +111,11 @@ func (x *GetObjectPrm) SetAddress(addr oid.Address) { // GetObjectRes groups the resulting values of GetObject operation. type GetObjectRes struct { - obj *object.Object + obj *objectSDK.Object } -// Object returns requested object. -func (x GetObjectRes) Object() *object.Object { +// Object returns requested objectSDK. +func (x GetObjectRes) Object() *objectSDK.Object { return x.obj } @@ -125,10 +125,10 @@ func (x GetObjectRes) Object() *object.Object { // // Returns any error which prevented the operation from completing correctly in error return. // Returns: -// - error of type *object.SplitInfoError if object raw flag is set and requested object is virtual; +// - error of type *objectSDK.SplitInfoError if object raw flag is set and requested object is virtual; // - error of type *apistatus.ObjectAlreadyRemoved if the requested object is marked to be removed. // -// GetObject ignores the provided session if it is not related to the requested object. +// GetObject ignores the provided session if it is not related to the requested objectSDK. func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { // here we ignore session if it is opened for other object since such // request will almost definitely fail. The case can occur, for example, @@ -155,7 +155,7 @@ func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { return nil, fmt.Errorf("init object reading: %w", err) } - var obj object.Object + var obj objectSDK.Object if !rdr.ReadHeader(&obj) { res, err := rdr.Close() @@ -210,11 +210,11 @@ func (x *HeadObjectPrm) SetAddress(addr oid.Address) { // HeadObjectRes groups the resulting values of GetObject operation. type HeadObjectRes struct { - hdr *object.Object + hdr *objectSDK.Object } // Header returns requested object header. -func (x HeadObjectRes) Header() *object.Object { +func (x HeadObjectRes) Header() *objectSDK.Object { return x.hdr } @@ -225,10 +225,10 @@ func (x HeadObjectRes) Header() *object.Object { // Returns any error which prevented the operation from completing correctly in error return. // Returns: // -// error of type *object.SplitInfoError if object raw flag is set and requested object is virtual; +// error of type *objectSDK.SplitInfoError if object raw flag is set and requested object is virtual; // error of type *apistatus.ObjectAlreadyRemoved if the requested object is marked to be removed. // -// HeadObject ignores the provided session if it is not related to the requested object. +// HeadObject ignores the provided session if it is not related to the requested objectSDK. func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) { if prm.local { prm.cliPrm.MarkLocal() @@ -255,7 +255,7 @@ func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) return nil, fmt.Errorf("read object header from FrostFS: %w", err) } - var hdr object.Object + var hdr objectSDK.Object if !cliRes.ReadHeader(&hdr) { return nil, errors.New("missing object header in the response") @@ -296,7 +296,7 @@ func (x *PayloadRangePrm) SetAddress(addr oid.Address) { // SetRange range of the object payload to be read. // // Required parameter. -func (x *PayloadRangePrm) SetRange(rng *object.Range) { +func (x *PayloadRangePrm) SetRange(rng *objectSDK.Range) { x.cliPrm.SetOffset(rng.GetOffset()) x.ln = rng.GetLength() } @@ -323,11 +323,11 @@ const maxInitialBufferSize = 1024 * 1024 // 1 MiB // Returns any error which prevented the operation from completing correctly in error return. // Returns: // -// error of type *object.SplitInfoError if object raw flag is set and requested object is virtual; +// error of type *objectSDK.SplitInfoError if object raw flag is set and requested object is virtual; // error of type *apistatus.ObjectAlreadyRemoved if the requested object is marked to be removed; // error of type *apistatus.ObjectOutOfRange if the requested range is too big. // -// PayloadRange ignores the provided session if it is not related to the requested object. +// PayloadRange ignores the provided session if it is not related to the requested objectSDK. func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, error) { if prm.local { prm.cliPrm.MarkLocal() @@ -377,13 +377,13 @@ func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, e type PutObjectPrm struct { commonPrm - obj *object.Object + obj *objectSDK.Object } // SetObject sets object to be stored. // // Required parameter. -func (x *PutObjectPrm) SetObject(obj *object.Object) { +func (x *PutObjectPrm) SetObject(obj *objectSDK.Object) { x.obj = obj } @@ -392,7 +392,7 @@ type PutObjectRes struct { id oid.ID } -// ID returns identifier of the stored object. +// ID returns identifier of the stored objectSDK. func (x PutObjectRes) ID() oid.ID { return x.id } @@ -464,7 +464,7 @@ func (x *SearchObjectsPrm) SetContainerID(id cid.ID) { } // SetFilters sets search filters. -func (x *SearchObjectsPrm) SetFilters(fs object.SearchFilters) { +func (x *SearchObjectsPrm) SetFilters(fs objectSDK.SearchFilters) { x.cliPrm.SetFilters(fs) } diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index f07122729..be202892a 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -5,7 +5,7 @@ import ( "fmt" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" ) @@ -14,7 +14,7 @@ import ( type ObjectStorage interface { // Put must save passed object // and return any appeared error. - Put(context.Context, *object.Object) error + Put(context.Context, *objectSDK.Object) error // Delete must delete passed objects // and return any appeared error. Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error @@ -28,11 +28,11 @@ type ObjectStorage interface { type localTarget struct { storage ObjectStorage - obj *object.Object + obj *objectSDK.Object meta objectCore.ContentMeta } -func (t *localTarget) WriteObject(obj *object.Object, meta objectCore.ContentMeta) error { +func (t *localTarget) WriteObject(obj *objectSDK.Object, meta objectCore.ContentMeta) error { t.obj = obj t.meta = meta @@ -41,12 +41,12 @@ func (t *localTarget) WriteObject(obj *object.Object, meta objectCore.ContentMet func (t *localTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { switch t.meta.Type() { - case object.TypeTombstone: + case objectSDK.TypeTombstone: err := t.storage.Delete(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) if err != nil { return nil, fmt.Errorf("could not delete objects from tombstone locally: %w", err) } - case object.TypeLock: + case objectSDK.TypeLock: err := t.storage.Lock(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) if err != nil { return nil, fmt.Errorf("could not lock object from lock objects locally: %w", err) diff --git a/pkg/services/object/put/prm.go b/pkg/services/object/put/prm.go index c8d1b29a2..52a7c102c 100644 --- a/pkg/services/object/put/prm.go +++ b/pkg/services/object/put/prm.go @@ -7,13 +7,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) type PutInitPrm struct { common *util.CommonPrm - hdr *object.Object + hdr *objectSDK.Object cnr containerSDK.Container @@ -34,7 +34,7 @@ func (p *PutInitPrm) WithCommonPrm(v *util.CommonPrm) *PutInitPrm { return p } -func (p *PutInitPrm) WithObject(v *object.Object) *PutInitPrm { +func (p *PutInitPrm) WithObject(v *objectSDK.Object) *PutInitPrm { if p != nil { p.hdr = v } diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/put/remote.go index bcc566b74..a5b3f643c 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/put/remote.go @@ -11,7 +11,7 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" ) @@ -22,7 +22,7 @@ type remoteTarget struct { nodeInfo clientcore.NodeInfo - obj *object.Object + obj *objectSDK.Object clientConstructor ClientConstructor } @@ -39,10 +39,10 @@ type RemoteSender struct { type RemotePutPrm struct { node netmap.NodeInfo - obj *object.Object + obj *objectSDK.Object } -func (t *remoteTarget) WriteObject(obj *object.Object, _ objectcore.ContentMeta) error { +func (t *remoteTarget) WriteObject(obj *objectSDK.Object, _ objectcore.ContentMeta) error { t.obj = obj return nil @@ -89,7 +89,7 @@ func (p *RemotePutPrm) WithNodeInfo(v netmap.NodeInfo) *RemotePutPrm { } // WithObject sets transferred object. -func (p *RemotePutPrm) WithObject(v *object.Object) *RemotePutPrm { +func (p *RemotePutPrm) WithObject(v *objectSDK.Object) *RemotePutPrm { if p != nil { p.obj = v } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 8be9c75a0..bf6c20588 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -12,7 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -212,7 +212,7 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { // enable additional container broadcast on non-local operation // if object has TOMBSTONE or LOCK type. typ := prm.hdr.Type() - withBroadcast := !prm.common.LocalOnly() && (typ == object.TypeTombstone || typ == object.TypeLock) + withBroadcast := !prm.common.LocalOnly() && (typ == objectSDK.TypeTombstone || typ == objectSDK.TypeLock) return &distributedTarget{ traversal: traversal{ diff --git a/pkg/services/object/put/v2/util.go b/pkg/services/object/put/v2/util.go index 758470f6c..a157a9542 100644 --- a/pkg/services/object/put/v2/util.go +++ b/pkg/services/object/put/v2/util.go @@ -5,7 +5,7 @@ import ( refsV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) func (s *streamer) toInitPrm(part *objectV2.PutObjectPartInit, req *objectV2.PutRequest) (*putsvc.PutInitPrm, error) { @@ -21,7 +21,7 @@ func (s *streamer) toInitPrm(part *objectV2.PutObjectPartInit, req *objectV2.Put return new(putsvc.PutInitPrm). WithObject( - object.NewFromV2(oV2), + objectSDK.NewFromV2(oV2), ). WithRelay(s.relayRequest). WithCommonPrm(commonPrm). diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index 475a31b98..c1a9a0c1c 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -5,7 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -56,7 +56,7 @@ func (exec *execCtx) containerID() cid.ID { return exec.prm.cnr } -func (exec *execCtx) searchFilters() object.SearchFilters { +func (exec *execCtx) searchFilters() objectSDK.SearchFilters { return exec.prm.filters } diff --git a/pkg/services/object/search/prm.go b/pkg/services/object/search/prm.go index da46dfeb6..d2918d6e7 100644 --- a/pkg/services/object/search/prm.go +++ b/pkg/services/object/search/prm.go @@ -6,7 +6,7 @@ import ( coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -18,7 +18,7 @@ type Prm struct { cnr cid.ID - filters object.SearchFilters + filters objectSDK.SearchFilters forwarder RequestForwarder } @@ -55,6 +55,6 @@ func (p *Prm) WithContainerID(id cid.ID) { } // WithSearchFilters sets search filters. -func (p *Prm) WithSearchFilters(fs object.SearchFilters) { +func (p *Prm) WithSearchFilters(fs objectSDK.SearchFilters) { p.filters = fs } diff --git a/pkg/services/object/search/v2/util.go b/pkg/services/object/search/v2/util.go index cfccaede6..e971fa8e5 100644 --- a/pkg/services/object/search/v2/util.go +++ b/pkg/services/object/search/v2/util.go @@ -12,7 +12,7 @@ import ( searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -58,7 +58,7 @@ func (s *Service) toPrm(req *objectV2.SearchRequest, stream objectSvc.SearchStre } p.WithContainerID(id) - p.WithSearchFilters(object.NewSearchFiltersFromV2(body.GetFilters())) + p.WithSearchFilters(objectSDK.NewSearchFiltersFromV2(body.GetFilters())) return p, nil } diff --git a/pkg/services/object/util/chain.go b/pkg/services/object/util/chain.go index 96dafd10e..b574d5eb6 100644 --- a/pkg/services/object/util/chain.go +++ b/pkg/services/object/util/chain.go @@ -5,7 +5,7 @@ import ( "fmt" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -22,11 +22,11 @@ type HeadReceiver interface { // // If reverseDirection arg is true, then the traversal is done in reverse order. // Stop boolean result provides the ability to interrupt the traversal. -type SplitMemberHandler func(member *object.Object, reverseDirection bool) (stop bool) +type SplitMemberHandler func(member *objectSDK.Object, reverseDirection bool) (stop bool) // IterateAllSplitLeaves is an iterator over all object split-tree leaves in direct order. -func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Object)) error { - return IterateSplitLeaves(r, addr, func(leaf *object.Object) bool { +func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*objectSDK.Object)) error { + return IterateSplitLeaves(r, addr, func(leaf *objectSDK.Object) bool { h(leaf) return false }) @@ -35,13 +35,13 @@ func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Obje // IterateSplitLeaves is an iterator over object split-tree leaves in direct order. // // If member handler returns true, then the iterator aborts without error. -func IterateSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Object) bool) error { +func IterateSplitLeaves(r HeadReceiver, addr oid.Address, h func(*objectSDK.Object) bool) error { var ( reverse bool - leaves []*object.Object + leaves []*objectSDK.Object ) - if err := TraverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if err := TraverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { reverse = reverseDirection if reverse { @@ -84,9 +84,9 @@ func traverseSplitChain(r HeadReceiver, addr oid.Address, h SplitMemberHandler) switch res := v.(type) { default: panic(fmt.Sprintf("unexpected result of %T: %T", r, v)) - case *object.Object: + case *objectSDK.Object: return h(res, false), nil - case *object.SplitInfo: + case *objectSDK.SplitInfo: link, withLink := res.Link() last, withLast := res.LastPart() @@ -108,7 +108,7 @@ func traverseByLink(cnr cid.ID, link oid.ID, r HeadReceiver, h SplitMemberHandle chain := make([]oid.ID, 0) - if _, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if _, err := traverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { children := member.Children() if reverseDirection { @@ -122,12 +122,12 @@ func traverseByLink(cnr cid.ID, link oid.ID, r HeadReceiver, h SplitMemberHandle return false, err } - var reverseChain []*object.Object + var reverseChain []*objectSDK.Object for i := range chain { addr.SetObject(chain[i]) - if stop, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if stop, err := traverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { if !reverseDirection { return h(member, false) } @@ -147,16 +147,16 @@ func traverseByLink(cnr cid.ID, link oid.ID, r HeadReceiver, h SplitMemberHandle return false, nil } -func traverseByLast(cnr cid.ID, last oid.ID, withLast bool, res *object.SplitInfo, r HeadReceiver, h SplitMemberHandler) (bool, error) { +func traverseByLast(cnr cid.ID, last oid.ID, withLast bool, res *objectSDK.SplitInfo, r HeadReceiver, h SplitMemberHandler) (bool, error) { var addr oid.Address addr.SetContainer(cnr) for last, withLast = res.LastPart(); withLast; { addr.SetObject(last) - var directChain []*object.Object + var directChain []*objectSDK.Object - if _, err := traverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) { + if _, err := traverseSplitChain(r, addr, func(member *objectSDK.Object, reverseDirection bool) (stop bool) { if reverseDirection { last, withLast = member.PreviousID() return h(member, true) diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go index 46fcc9840..66a0ec7d3 100644 --- a/pkg/services/object_manager/tombstone/checker.go +++ b/pkg/services/object_manager/tombstone/checker.go @@ -7,7 +7,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" "go.uber.org/zap" @@ -21,7 +21,7 @@ type Source interface { // // Tombstone MUST return (nil, nil) if requested tombstone is // missing in the storage for the provided epoch. - Tombstone(ctx context.Context, a oid.Address, epoch uint64) (*object.Object, error) + Tombstone(ctx context.Context, a oid.Address, epoch uint64) (*objectSDK.Object, error) } // ExpirationChecker is a tombstone source wrapper. @@ -72,7 +72,7 @@ func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Addr return false } -func (g *ExpirationChecker) handleTS(addr string, ts *object.Object, reqEpoch uint64) bool { +func (g *ExpirationChecker) handleTS(addr string, ts *objectSDK.Object, reqEpoch uint64) bool { for _, atr := range ts.Attributes() { if atr.Key() == objectV2.SysAttributeExpEpoch || atr.Key() == objectV2.SysAttributeExpEpochNeoFS { epoch, err := strconv.ParseUint(atr.Value(), 10, 64) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index db297b68a..a45c019eb 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -11,7 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -97,7 +97,7 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi // Number of copies that are stored on maintenance nodes. var uncheckedCopies int - if typ == object.TypeLock { + if typ == objectSDK.TypeLock { // all nodes of a container must store the `LOCK` objects // for correct object removal protection: // - `LOCK` objects are broadcast on their PUT requests; diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index a09957895..0ead48ef4 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -14,7 +14,7 @@ import ( apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/panjf2000/ants/v2" @@ -27,7 +27,7 @@ func TestBuryObjectWithoutContainer(t *testing.T) { objs := []objectcore.AddressWithType{ { Address: addr, - Type: object.TypeRegular, + Type: objectSDK.TypeRegular, }, } @@ -68,7 +68,7 @@ func TestProcessObject(t *testing.T) { // - policy is used only to match the number of replicas for each index in the placement tests := []struct { desc string - objType object.Type + objType objectSDK.Type nodeCount int policy string placement [][]int @@ -128,7 +128,7 @@ func TestProcessObject(t *testing.T) { }, { desc: "lock object must be replicated to all nodes", - objType: object.TypeLock, + objType: objectSDK.TypeLock, nodeCount: 3, policy: `REP 1`, placement: [][]int{{0, 1, 2}}, @@ -176,7 +176,7 @@ func TestProcessObject(t *testing.T) { } // Object remote header - headFn := func(_ context.Context, ni netmap.NodeInfo, a oid.Address) (*object.Object, error) { + headFn := func(_ context.Context, ni netmap.NodeInfo, a oid.Address) (*objectSDK.Object, error) { index := int(ni.PublicKey()[0]) if a != addr || index < 1 || index >= ti.nodeCount { t.Errorf("unexpected remote object head: node=%+v addr=%v", ni, a) From a0d51090a4620beccb5a86b0808ef085c1f84f87 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 5 Jul 2023 11:57:16 +0300 Subject: [PATCH 0397/1943] [#482] Enable staticcheck in foregjo actions Signed-off-by: Anton Nikiforov --- .forgejo/workflows/tests.yml | 18 ++++++++++++++++++ Makefile | 6 +++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 2d44d34f8..13395a64b 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -52,3 +52,21 @@ jobs: - name: Run tests run: go test ./... -count=1 -race + + staticcheck: + name: Staticcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + cache: true + + - name: Install staticcheck + run: make staticcheck-install + + - name: Run staticcheck + run: make staticcheck-run diff --git a/Makefile b/Makefile index e1b719206..df53b8772 100755 --- a/Makefile +++ b/Makefile @@ -135,8 +135,12 @@ pre-commit-run: lint: @golangci-lint --timeout=5m run +# Install staticcheck +staticcheck-install: + @go install honnef.co/go/tools/cmd/staticcheck@latest + # Run staticcheck -staticcheck: +staticcheck-run: @staticcheck ./... # Run linters in Docker From e858479a746604813130aa84edb991d0861b5f62 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 6 Jul 2023 18:23:52 +0300 Subject: [PATCH 0398/1943] [#498] policer: Explicitly Rewind() iterator after finish Previously, we can continue to return `EndOfListing` infinitely. Reflect iterator reuse via Rewind() method. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/keyspaceiterator.go | 4 ++ pkg/services/policer/option.go | 1 + pkg/services/policer/policer_test.go | 89 +++++++++++++++++++++++++++- pkg/services/policer/process.go | 1 + 4 files changed, 94 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/keyspaceiterator.go b/cmd/frostfs-node/keyspaceiterator.go index 8991964a0..e7214aacb 100644 --- a/cmd/frostfs-node/keyspaceiterator.go +++ b/cmd/frostfs-node/keyspaceiterator.go @@ -26,3 +26,7 @@ func (it *keySpaceIterator) Next(ctx context.Context, batchSize uint32) ([]objec it.cur = res.Cursor() return res.AddressList(), nil } + +func (it *keySpaceIterator) Rewind() { + it.cur = nil +} diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 6f17b2947..e182c6be7 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -23,6 +23,7 @@ import ( // when the end of the key space is reached. type KeySpaceIterator interface { Next(context.Context, uint32) ([]objectcore.AddressWithType, error) + Rewind() } // RedundantCopyCallback is a callback to pass diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 0ead48ef4..ad82d2477 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -244,11 +244,95 @@ func TestProcessObject(t *testing.T) { } } +func TestIteratorContract(t *testing.T) { + addr := oidtest.Address() + objs := []objectcore.AddressWithType{{ + Address: addr, + Type: object.TypeRegular, + }} + + containerSrc := func(id cid.ID) (*container.Container, error) { + return nil, apistatus.ContainerNotFound{} + } + buryFn := func(ctx context.Context, a oid.Address) error { + return nil + } + + pool, err := ants.NewPool(4) + require.NoError(t, err) + + it := &predefinedIterator{ + scenario: []nextResult{ + {objs, nil}, + {nil, errors.New("opaque")}, + {nil, engine.ErrEndOfListing}, + {nil, engine.ErrEndOfListing}, + {nil, errors.New("opaque")}, + {objs, engine.ErrEndOfListing}, + }, + finishCh: make(chan struct{}), + } + + p := New( + WithKeySpaceIterator(it), + WithContainerSource(containerSrcFunc(containerSrc)), + WithBuryFunc(buryFn), + WithPool(pool), + WithNodeLoader(constNodeLoader(0)), + ) + + ctx, cancel := context.WithCancel(context.Background()) + go p.Run(ctx) + + <-it.finishCh + cancel() + require.Equal(t, []string{ + "Next", + "Next", + "Next", + "Rewind", + "Next", + "Rewind", + "Next", + "Next", + "Rewind", + }, it.calls) +} + // TODO(https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/101) func eqAddr(a, b oid.Address) bool { return a.Container().Equals(b.Container()) && a.Object().Equals(b.Object()) } +type nextResult struct { + objs []objectcore.AddressWithType + err error +} + +type predefinedIterator struct { + scenario []nextResult + finishCh chan struct{} + pos int + calls []string +} + +func (it *predefinedIterator) Next(ctx context.Context, size uint32) ([]objectcore.AddressWithType, error) { + if it.pos == len(it.scenario) { + close(it.finishCh) + <-ctx.Done() + return nil, nil + } + + res := it.scenario[it.pos] + it.pos += 1 + it.calls = append(it.calls, "Next") + return res.objs, res.err +} + +func (it *predefinedIterator) Rewind() { + it.calls = append(it.calls, "Rewind") +} + // sliceKeySpaceIterator is a KeySpaceIterator backed by a slice. type sliceKeySpaceIterator struct { objs []objectcore.AddressWithType @@ -257,7 +341,6 @@ type sliceKeySpaceIterator struct { func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectcore.AddressWithType, error) { if it.cur >= len(it.objs) { - it.cur = 0 return nil, engine.ErrEndOfListing } end := it.cur + int(size) @@ -269,6 +352,10 @@ func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectc return ret, nil } +func (it *sliceKeySpaceIterator) Rewind() { + it.cur = 0 +} + // containerSrcFunc is a container.Source backed by a function. type containerSrcFunc func(cid.ID) (*container.Container, error) diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 39b61c8a0..cdc92ed12 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -27,6 +27,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { addrs, err := p.keySpaceIterator.Next(ctx, p.batchSize) if err != nil { if errors.Is(err, engine.ErrEndOfListing) { + p.keySpaceIterator.Rewind() time.Sleep(time.Second) // finished whole cycle, sleep a bit continue } From 2310a5c7ba5577ac61837418a2de94ea20fc4c7a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 6 Jul 2023 18:29:16 +0300 Subject: [PATCH 0399/1943] [#498] policer: Allow to set sleep duration between iterations Speed up tests on CI. Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/option.go | 3 ++- pkg/services/policer/policer_test.go | 4 ++++ pkg/services/policer/process.go | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index e182c6be7..4194353ca 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -76,7 +76,7 @@ type cfg struct { batchSize, cacheSize uint32 - rebalanceFreq, evictDuration time.Duration + rebalanceFreq, evictDuration, sleepDuration time.Duration } func defaultCfg() *cfg { @@ -85,6 +85,7 @@ func defaultCfg() *cfg { batchSize: 10, cacheSize: 1024, // 1024 * address size = 1024 * 64 = 64 MiB rebalanceFreq: 1 * time.Second, + sleepDuration: 1 * time.Second, evictDuration: 30 * time.Second, } } diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index ad82d2477..664e7d105 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -6,6 +6,7 @@ import ( "errors" "sort" "testing" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -279,6 +280,9 @@ func TestIteratorContract(t *testing.T) { WithBuryFunc(buryFn), WithPool(pool), WithNodeLoader(constNodeLoader(0)), + func(c *cfg) { + c.sleepDuration = time.Millisecond + }, ) ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index cdc92ed12..3b54bf929 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -28,7 +28,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { if err != nil { if errors.Is(err, engine.ErrEndOfListing) { p.keySpaceIterator.Rewind() - time.Sleep(time.Second) // finished whole cycle, sleep a bit + time.Sleep(p.sleepDuration) // finished whole cycle, sleep a bit continue } p.log.Warn(logs.PolicerFailureAtObjectSelectForReplication, zap.Error(err)) From 140d970a95eab5df1e5c44b0d8c2c64bb85092c3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 7 Jul 2023 18:52:40 +0300 Subject: [PATCH 0400/1943] [#498] policer: Fix objectSDK import Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/policer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 664e7d105..42428df23 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -249,7 +249,7 @@ func TestIteratorContract(t *testing.T) { addr := oidtest.Address() objs := []objectcore.AddressWithType{{ Address: addr, - Type: object.TypeRegular, + Type: objectSDK.TypeRegular, }} containerSrc := func(id cid.ID) (*container.Container, error) { From 040a623d391d7879d665f698966baab9db7af219 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 7 Jul 2023 18:52:32 +0300 Subject: [PATCH 0401/1943] [#476] cli: Fix `object nodes` command Do not fail if client creation failed. Use external addresses to create the client too. Use public key as node ID. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 358a9d618..d2d20dc08 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -3,6 +3,7 @@ package object import ( "context" "crypto/ecdsa" + "encoding/hex" "errors" "fmt" "strconv" @@ -223,14 +224,12 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPl } } - var err error eg, egCtx := errgroup.WithContext(cmd.Context()) for _, cand := range candidates { cand := cand eg.Go(func() error { - var cli *client.Client - cli, err = createClient(egCtx, cmd, cand, pk) + cli, err := createClient(egCtx, cmd, cand, pk) if err != nil { resultMtx.Lock() defer resultMtx.Unlock() @@ -268,13 +267,7 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPl }) } - egErr := eg.Wait() - if err != nil || egErr != nil { - if err == nil { - err = egErr - } - commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", err) - } + commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", eg.Wait()) return result } @@ -285,6 +278,7 @@ func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.N addresses = append(addresses, s) return false }) + addresses = append(addresses, candidate.ExternalAddresses()...) var lastErr error for _, address := range addresses { var networkAddr network.Address @@ -336,13 +330,9 @@ func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacem defer func() { commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", w.Flush()) }() - fmt.Fprintln(w, "Netmap node\tShould contain object\tActually contains object\t") + fmt.Fprintln(w, "Node ID\tShould contain object\tActually contains object\t") for _, n := range netmap.Nodes() { - var address string - n.IterateNetworkEndpoints(func(s string) bool { - address = s - return s != "" - }) + nodeID := hex.EncodeToString(n.PublicKey()) _, required := requiredPlacement[n.Hash()] actual, actualExists := actualPlacement[n.Hash()] actualStr := "" @@ -353,6 +343,6 @@ func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacem actualStr = strconv.FormatBool(actual.value) } } - fmt.Fprintf(w, "%s\t%s\t%s\t\n", address, strconv.FormatBool(required), actualStr) + fmt.Fprintf(w, "%s\t%s\t%s\t\n", nodeID, strconv.FormatBool(required), actualStr) } } From 9be5d44a46bf29dcf2a21b66c14ed19030714ef9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Jul 2023 11:35:51 +0300 Subject: [PATCH 0402/1943] [#486] node: Update api-go and sdk-go versions Signed-off-by: Dmitrii Stepanov --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 4be07aef7..9a53bb8fa 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230628121302-5d62cef27e6c + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230706140617-98cab7ed6166 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 03e604641..31a4cb9e1 100644 --- a/go.sum +++ b/go.sum @@ -36,16 +36,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb h1:gaq4wgk7b+i9+rkDa+goPkv5FzqYlMxEuCKdZ57mSc0= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230602142716-68021b910acb/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe h1:SB102RiEg+4h9qcwyG97zHBtwduMRbedbtkwRDVSps8= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a h1:EeaOunifY30ATPzOo1F32vUDW299UXDHYukZrIVxBZ0= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230628121302-5d62cef27e6c h1:nH9mifyAuiZeyc79DDwNZjbJeomz0YW4Ja6Mk/iuaLI= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230628121302-5d62cef27e6c/go.mod h1:w+s3ozlbFfTDFHhjX0A3Iif3BRtnTkwiACxFZD+Q0cQ= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230706140617-98cab7ed6166 h1:HSwD/CDbrUp45gQmfn9KYag8zN0GD+HA0l2+U+c3Ayo= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230706140617-98cab7ed6166/go.mod h1:r5Fir/4jCVXzdfOyCUbikSDB99nVqnHNq7mzVcidnlA= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 7b76527759924bd82c5849c8c8bb451244117d98 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Jul 2023 11:36:20 +0300 Subject: [PATCH 0403/1943] [#486] node: Add PutSingle wrappers Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 4 ++ pkg/network/transport/object/grpc/service.go | 14 +++++ pkg/services/object/acl/v2/service.go | 66 +++++++++++++++++++- pkg/services/object/acl/v2/util.go | 6 ++ pkg/services/object/common.go | 8 +++ pkg/services/object/metrics.go | 16 +++++ pkg/services/object/put/v2/service.go | 6 ++ pkg/services/object/response.go | 10 +++ pkg/services/object/server.go | 1 + pkg/services/object/sign.go | 16 +++++ pkg/services/object/transport_splitter.go | 4 ++ 11 files changed, 148 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 4106f5dc1..1d99d958b 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -70,6 +70,10 @@ func (s *objectSvc) Put() (objectService.PutObjectStream, error) { return s.put.Put() } +func (s *objectSvc) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + return s.put.PutSingle(ctx, req) +} + func (s *objectSvc) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { return s.get.Head(ctx, req) } diff --git a/pkg/network/transport/object/grpc/service.go b/pkg/network/transport/object/grpc/service.go index 7fa60f99c..7c6b395d5 100644 --- a/pkg/network/transport/object/grpc/service.go +++ b/pkg/network/transport/object/grpc/service.go @@ -110,3 +110,17 @@ func (s *Server) GetRangeHash(ctx context.Context, req *objectGRPC.GetRangeHashR return resp.ToGRPCMessage().(*objectGRPC.GetRangeHashResponse), nil } + +func (s *Server) PutSingle(ctx context.Context, req *objectGRPC.PutSingleRequest) (*objectGRPC.PutSingleResponse, error) { + putSingleReq := &object.PutSingleRequest{} + if err := putSingleReq.FromGRPCMessage(req); err != nil { + return nil, err + } + + resp, err := s.srv.PutSingle(ctx, putSingleReq) + if err != nil { + return nil, err + } + + return resp.ToGRPCMessage().(*objectGRPC.PutSingleResponse), nil +} diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 6544d78d7..c75bd326f 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -6,6 +6,7 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" @@ -443,6 +444,65 @@ func (b Service) GetRangeHash( return b.next.GetRangeHash(ctx, request) } +func (b Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { + cnr, err := getContainerIDFromRequest(request) + if err != nil { + return nil, err + } + + idV2 := request.GetBody().GetObject().GetHeader().GetOwnerID() + if idV2 == nil { + return nil, errors.New("missing object owner") + } + + var idOwner user.ID + + err = idOwner.ReadFromV2(*idV2) + if err != nil { + return nil, fmt.Errorf("invalid object owner: %w", err) + } + + obj, err := getObjectIDFromRefObjectID(request.GetBody().GetObject().GetObjectID()) + if err != nil { + return nil, err + } + + var sTok *sessionSDK.Object + sTok, err = readSessionToken(cnr, obj, request.GetMetaHeader().GetSessionToken()) + if err != nil { + return nil, err + } + + bTok, err := originalBearerToken(request.GetMetaHeader()) + if err != nil { + return nil, err + } + + req := MetaWithToken{ + vheader: request.GetVerificationHeader(), + token: sTok, + bearer: bTok, + src: request, + } + + reqInfo, err := b.findRequestInfo(req, cnr, acl.OpObjectPut) + if err != nil { + return nil, err + } + + reqInfo.obj = obj + + if !b.checker.CheckBasicACL(reqInfo) || !b.checker.StickyBitCheck(reqInfo, idOwner) { + return nil, basicACLErr(reqInfo) + } + + if err := b.checker.CheckEACL(request, reqInfo); err != nil { + return nil, eACLErr(reqInfo, err) + } + + return b.next.PutSingle(ctx, request) +} + func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { body := request.GetBody() if body == nil { @@ -481,7 +541,7 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe } var sTok *sessionSDK.Object - sTok, err = p.readSessionToken(cnr, obj, request) + sTok, err = readSessionToken(cnr, obj, request.GetMetaHeader().GetSessionToken()) if err != nil { return err } @@ -515,10 +575,10 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe return p.next.Send(ctx, request) } -func (p putStreamBasicChecker) readSessionToken(cnr cid.ID, obj *oid.ID, request *objectV2.PutRequest) (*sessionSDK.Object, error) { +func readSessionToken(cnr cid.ID, obj *oid.ID, tokV2 *session.Token) (*sessionSDK.Object, error) { var sTok *sessionSDK.Object - if tokV2 := request.GetMetaHeader().GetSessionToken(); tokV2 != nil { + if tokV2 != nil { sTok = new(sessionSDK.Object) err := sTok.ReadFromV2(*tokV2) diff --git a/pkg/services/object/acl/v2/util.go b/pkg/services/object/acl/v2/util.go index cd45b63fc..feda6a3cf 100644 --- a/pkg/services/object/acl/v2/util.go +++ b/pkg/services/object/acl/v2/util.go @@ -44,6 +44,8 @@ func getContainerIDFromRequest(req any) (cid.ID, error) { idV2 = v.GetBody().GetAddress().GetContainerID() case *objectV2.GetRangeHashRequest: idV2 = v.GetBody().GetAddress().GetContainerID() + case *objectV2.PutSingleRequest: + idV2 = v.GetBody().GetObject().GetHeader().GetContainerID() default: return cid.ID{}, errors.New("unknown request type") } @@ -97,6 +99,10 @@ func originalSessionToken(header *sessionV2.RequestMetaHeader) (*sessionSDK.Obje // object reference's holders. Returns an error if object ID is missing in the request. func getObjectIDFromRequestBody(body interface{ GetAddress() *refsV2.Address }) (*oid.ID, error) { idV2 := body.GetAddress().GetObjectID() + return getObjectIDFromRefObjectID(idV2) +} + +func getObjectIDFromRefObjectID(idV2 *refsV2.ObjectID) (*oid.ID, error) { if idV2 == nil { return nil, errors.New("missing object ID") } diff --git a/pkg/services/object/common.go b/pkg/services/object/common.go index 5b139d8eb..0d39dce0b 100644 --- a/pkg/services/object/common.go +++ b/pkg/services/object/common.go @@ -89,3 +89,11 @@ func (x *Common) GetRangeHash(ctx context.Context, req *objectV2.GetRangeHashReq return x.nextHandler.GetRangeHash(ctx, req) } + +func (x *Common) PutSingle(ctx context.Context, req *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { + if x.state.IsMaintenance() { + return nil, errMaintenance + } + + return x.nextHandler.PutSingle(ctx, req) +} diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index 487374940..f972f43ae 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -76,6 +76,22 @@ func (m MetricCollector) Put() (PutObjectStream, error) { return m.next.Put() } +func (m MetricCollector) PutSingle(ctx context.Context, request *object.PutSingleRequest) (*object.PutSingleResponse, error) { + if m.enabled { + t := time.Now() + + res, err := m.next.PutSingle(ctx, request) + + m.metrics.AddRequestDuration("PutSingle", time.Since(t), err == nil) + if err == nil { + m.metrics.AddPayloadSize("PutSingle", len(request.GetBody().GetObject().GetPayload())) + } + + return res, err + } + return m.next.PutSingle(ctx, request) +} + func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) { if m.enabled { t := time.Now() diff --git a/pkg/services/object/put/v2/service.go b/pkg/services/object/put/v2/service.go index 656f8df9c..5af62cd40 100644 --- a/pkg/services/object/put/v2/service.go +++ b/pkg/services/object/put/v2/service.go @@ -1,8 +1,10 @@ package putsvc import ( + "context" "fmt" + objectAPI "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -47,6 +49,10 @@ func (s *Service) Put() (object.PutObjectStream, error) { }, nil } +func (s *Service) PutSingle(context.Context, *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { + return nil, fmt.Errorf("unimplemented") //TODO +} + func WithInternalService(v *putsvc.Service) Option { return func(c *cfg) { c.svc = v diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index 81d6aaaaf..a10f26a34 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -87,6 +87,16 @@ func (s *ResponseService) Put() (PutObjectStream, error) { }, nil } +func (s *ResponseService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + resp, err := s.svc.PutSingle(ctx, req) + if err != nil { + return nil, err + } + + s.respSvc.SetMeta(resp) + return resp, nil +} + func (s *ResponseService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { resp, err := s.svc.Head(ctx, req) if err != nil { diff --git a/pkg/services/object/server.go b/pkg/services/object/server.go index ccce9c4f4..73b88f233 100644 --- a/pkg/services/object/server.go +++ b/pkg/services/object/server.go @@ -41,4 +41,5 @@ type ServiceServer interface { Delete(context.Context, *object.DeleteRequest) (*object.DeleteResponse, error) GetRange(*object.GetRangeRequest, GetObjectRangeStream) error GetRangeHash(context.Context, *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) + PutSingle(context.Context, *object.PutSingleRequest) (*object.PutSingleResponse, error) } diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 9d66c76ba..5b3578e29 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -120,6 +120,22 @@ func (s *SignService) Head(ctx context.Context, req *object.HeadRequest) (*objec return resp.(*object.HeadResponse), nil } +func (s *SignService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, + func(ctx context.Context, req any) (util.ResponseMessage, error) { + return s.svc.PutSingle(ctx, req.(*object.PutSingleRequest)) + }, + func() util.ResponseMessage { + return new(object.PutSingleResponse) + }, + ) + if err != nil { + return nil, err + } + + return resp.(*object.PutSingleResponse), nil +} + func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { s.nonEmptyResp = true return s.respWriter(resp) diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index a7d1c486a..2d9810cd3 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -107,6 +107,10 @@ func (c TransportSplitter) Delete(ctx context.Context, request *object.DeleteReq return c.next.Delete(ctx, request) } +func (c TransportSplitter) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + return c.next.PutSingle(ctx, req) +} + func (s *rangeStreamMsgSizeCtrl) Send(resp *object.GetRangeResponse) error { body := resp.GetBody() From fcbf90d31b7c932fda251fc4d186c07b7ccf1aba Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Jul 2023 15:12:59 +0300 Subject: [PATCH 0404/1943] [#486] node: Add PutSingle implemetation Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/single.go | 369 ++++++++++++++++++++++++++ pkg/services/object/put/v2/service.go | 4 +- 2 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 pkg/services/object/put/single.go diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go new file mode 100644 index 000000000..32c1ca73a --- /dev/null +++ b/pkg/services/object/put/single.go @@ -0,0 +1,369 @@ +package putsvc + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "hash" + "sync" + "sync/atomic" + + objectAPI "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" + svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/tzhash/tz" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +type putSingleRequestSigner struct { + req *objectAPI.PutSingleRequest + keyStorage *svcutil.KeyStorage + signer *sync.Once +} + +func (s *putSingleRequestSigner) GetRequestWithSignedHeader() (*objectAPI.PutSingleRequest, error) { + var resErr error + s.signer.Do(func() { + metaHdr := new(sessionV2.RequestMetaHeader) + meta := s.req.GetMetaHeader() + + metaHdr.SetTTL(meta.GetTTL() - 1) + metaHdr.SetOrigin(meta) + s.req.SetMetaHeader(metaHdr) + + privateKey, err := s.keyStorage.GetKey(nil) + if err != nil { + resErr = err + return + } + resErr = signature.SignServiceMessage(privateKey, s.req) + }) + return s.req, resErr +} + +func (s *Service) PutSingle(ctx context.Context, req *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "putsvc.PutSingle") + defer span.End() + + obj := objectSDK.NewFromV2(req.GetBody().GetObject()) + + if err := s.validatePutSingle(ctx, obj); err != nil { + return nil, err + } + + if err := s.saveToNodes(ctx, obj, req); err != nil { + return nil, err + } + + resp := &objectAPI.PutSingleResponse{} + resp.SetBody(&objectAPI.PutSingleResponseBody{}) + return resp, nil +} + +func (s *Service) validatePutSingle(ctx context.Context, obj *objectSDK.Object) error { + if err := s.validarePutSingleSize(obj); err != nil { + return err + } + + if err := s.validatePutSingleChecksum(obj); err != nil { + return err + } + + return s.validatePutSingleObject(ctx, obj) +} + +func (s *Service) validarePutSingleSize(obj *objectSDK.Object) error { + if uint64(len(obj.Payload())) != obj.PayloadSize() { + return ErrWrongPayloadSize + } + + maxAllowedSize := s.maxSizeSrc.MaxObjectSize() + if obj.PayloadSize() > maxAllowedSize { + return ErrExceedingMaxSize + } + + return nil +} + +func (s *Service) validatePutSingleChecksum(obj *objectSDK.Object) error { + cs, csSet := obj.PayloadChecksum() + if !csSet { + return errors.New("missing payload checksum") + } + + var hash hash.Hash + + switch typ := cs.Type(); typ { + default: + return fmt.Errorf("unsupported payload checksum type %v", typ) + case checksum.SHA256: + hash = sha256.New() + case checksum.TZ: + hash = tz.New() + } + + if _, err := hash.Write(obj.Payload()); err != nil { + return fmt.Errorf("could not compute payload hash: %w", err) + } + + if !bytes.Equal(hash.Sum(nil), cs.Value()) { + return fmt.Errorf("incorrect payload checksum") + } + + return nil +} + +func (s *Service) validatePutSingleObject(ctx context.Context, obj *objectSDK.Object) error { + if err := s.fmtValidator.Validate(ctx, obj, false); err != nil { + return fmt.Errorf("coult not validate object format: %w", err) + } + + _, err := s.fmtValidator.ValidateContent(obj) + if err != nil { + return fmt.Errorf("could not validate payload content: %w", err) + } + + return nil +} + +func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *objectAPI.PutSingleRequest) error { + localOnly := req.GetMetaHeader().GetTTL() <= 1 + placementOptions, err := s.getPutSinglePlacementOptions(obj, req.GetBody().GetCopiesNumber(), localOnly) + if err != nil { + return err + } + traversal := &traversal{ + opts: placementOptions, + extraBroadcastEnabled: len(obj.Children()) > 0 || + (!localOnly && (obj.Type() == objectSDK.TypeTombstone || obj.Type() == objectSDK.TypeLock)), + mtx: sync.RWMutex{}, + mExclude: make(map[string]struct{}), + } + signer := &putSingleRequestSigner{ + req: req, + keyStorage: s.keyStorage, + signer: &sync.Once{}, + } + return s.saveAccordingToPlacement(ctx, obj, signer, traversal) +} + +func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) ([]placement.Option, error) { + var result []placement.Option + if len(copiesNumber) > 0 { + result = append(result, placement.WithCopyNumbers(copiesNumber)) + } + + cnrID, ok := obj.ContainerID() + if !ok { + return nil, errors.New("missing container ID") + } + cnrInfo, err := s.cnrSrc.Get(cnrID) + if err != nil { + return nil, fmt.Errorf("could not get container by ID: %w", err) + } + result = append(result, placement.ForContainer(cnrInfo.Value)) + + objID, ok := obj.ID() + if !ok { + return nil, errors.New("missing object ID") + } + result = append(result, placement.ForObject(objID)) + + latestNetmap, err := netmap.GetLatestNetworkMap(s.netMapSrc) + if err != nil { + return nil, fmt.Errorf("could not get latest network map: %w", err) + } + builder := placement.NewNetworkMapBuilder(latestNetmap) + if localOnly { + result = append(result, placement.SuccessAfter(1)) + builder = svcutil.NewLocalPlacement(builder, s.netmapKeys) + } + result = append(result, placement.UseBuilder(builder)) + return result, nil +} + +func (s *Service) saveAccordingToPlacement(ctx context.Context, obj *objectSDK.Object, signer *putSingleRequestSigner, traversal *traversal) error { + traverser, err := placement.NewTraverser(traversal.opts...) + if err != nil { + return fmt.Errorf("could not create object placement traverser: %w", err) + } + + var resultError atomic.Value + for { + addrs := traverser.Next() + if len(addrs) == 0 { + break + } + + if stop := s.saveToPlacementNodes(ctx, obj, signer, traversal, traverser, addrs, &resultError); stop { + break + } + } + + if !traverser.Success() { + var err errIncompletePut + err.singleErr, _ = resultError.Load().(error) + return err + } + + if traversal.submitPrimaryPlacementFinish() { + err = s.saveAccordingToPlacement(ctx, obj, signer, traversal) + if err != nil { + s.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) + } + } + + return nil +} + +func (s *Service) saveToPlacementNodes(ctx context.Context, + obj *objectSDK.Object, + signer *putSingleRequestSigner, + traversal *traversal, + traverser *placement.Traverser, + nodeAddresses []placement.Node, + resultError *atomic.Value, +) bool { + wg := sync.WaitGroup{} + + for _, nodeAddress := range nodeAddresses { + nodeAddress := nodeAddress + if traversal.processed(nodeAddress) { + continue + } + + local := false + workerPool := s.remotePool + if s.netmapKeys.IsLocalKey(nodeAddress.PublicKey()) { + local = true + workerPool = s.localPool + } + + wg.Add(1) + if err := workerPool.Submit(func() { + defer wg.Done() + + err := s.saveToPlacementNode(ctx, &nodeDesc{local: local, info: nodeAddress}, obj, signer) + + traversal.submitProcessed(nodeAddress) + + if err != nil { + resultError.Store(err) + svcutil.LogServiceError(s.log, "PUT", nodeAddress.Addresses(), err) + return + } + + traverser.SubmitSuccess() + }); err != nil { + wg.Done() + svcutil.LogWorkerPoolError(s.log, "PUT", err) + return true + } + } + + wg.Wait() + + return false +} + +func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *nodeDesc, obj *objectSDK.Object, signer *putSingleRequestSigner) error { + if nodeDesc.local { + return s.localStore.Put(ctx, obj) + } + + var info client.NodeInfo + + client.NodeInfoFromNetmapElement(&info, nodeDesc.info) + + c, err := s.clientConstructor.Get(info) + if err != nil { + return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) + } + + return s.redirectPutSingleRequest(ctx, signer, obj, info, c) +} + +func (s *Service) redirectPutSingleRequest(ctx context.Context, + signer *putSingleRequestSigner, + obj *objectSDK.Object, + info client.NodeInfo, + c client.MultiAddressClient) error { + ctx, span := tracing.StartSpanFromContext(ctx, "putService.redirectPutSingleRequest") + defer span.End() + + var req *objectAPI.PutSingleRequest + var firstErr error + req, firstErr = signer.GetRequestWithSignedHeader() + if firstErr != nil { + return firstErr + } + + info.AddressGroup().IterateAddresses(func(addr network.Address) (stop bool) { + ctx, span := tracing.StartSpanFromContext(ctx, "putService.redirectPutSingleRequest.IterateAddresses", + trace.WithAttributes( + attribute.String("address", addr.String()), + )) + defer span.End() + + var err error + + defer func() { + if err != nil { + objID, _ := obj.ID() + cnrID, _ := obj.ContainerID() + s.log.Warn("failed to redirect PutSingle request", + zap.Error(err), + zap.Stringer("address", addr), + zap.Stringer("object_id", objID), + zap.Stringer("container_id", cnrID), + ) + } + + stop = err == nil + if stop || firstErr == nil { + firstErr = err + } + }() + + var resp *objectAPI.PutSingleResponse + + err = c.RawForAddress(ctx, addr, func(cli *rawclient.Client) error { + var e error + resp, e = rpc.PutSingleObject(cli, req, rawclient.WithContext(ctx)) + return e + }) + if err != nil { + err = fmt.Errorf("failed to execute request: %w", err) + return + } + + if err = internal.VerifyResponseKeyV2(info.PublicKey(), resp); err != nil { + return + } + + err = signature.VerifyServiceMessage(resp) + if err != nil { + err = fmt.Errorf("response verification failed: %w", err) + } + + return + }) + + return firstErr +} diff --git a/pkg/services/object/put/v2/service.go b/pkg/services/object/put/v2/service.go index 5af62cd40..78655edc7 100644 --- a/pkg/services/object/put/v2/service.go +++ b/pkg/services/object/put/v2/service.go @@ -49,8 +49,8 @@ func (s *Service) Put() (object.PutObjectStream, error) { }, nil } -func (s *Service) PutSingle(context.Context, *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { - return nil, fmt.Errorf("unimplemented") //TODO +func (s *Service) PutSingle(ctx context.Context, req *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { + return s.svc.PutSingle(ctx, req) } func WithInternalService(v *putsvc.Service) Option { From a65e26878b287a304713845d23a9c4ae1516c8aa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 10 Jul 2023 15:42:31 +0300 Subject: [PATCH 0405/1943] [#486] put service: Fix error typo Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/single.go | 2 +- pkg/services/object/put/validation.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 32c1ca73a..200830e15 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -131,7 +131,7 @@ func (s *Service) validatePutSingleChecksum(obj *objectSDK.Object) error { func (s *Service) validatePutSingleObject(ctx context.Context, obj *objectSDK.Object) error { if err := s.fmtValidator.Validate(ctx, obj, false); err != nil { - return fmt.Errorf("coult not validate object format: %w", err) + return fmt.Errorf("coud not validate object format: %w", err) } _, err := s.fmtValidator.ValidateContent(obj) diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/put/validation.go index 406304422..d4b08a038 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/put/validation.go @@ -48,7 +48,7 @@ var ( func (t *validatingTarget) WriteHeader(ctx context.Context, obj *objectSDK.Object) error { if err := t.fmt.Validate(ctx, obj, true); err != nil { - return fmt.Errorf("(%T) coult not validate object format: %w", t, err) + return fmt.Errorf("(%T) could not validate object format: %w", t, err) } return t.nextTarget.WriteHeader(ctx, obj) @@ -93,7 +93,7 @@ func (t *validatingPreparedTarget) WriteHeader(ctx context.Context, obj *objectS t.checksum = cs.Value() if err := t.fmt.Validate(ctx, obj, false); err != nil { - return fmt.Errorf("(%T) coult not validate object format: %w", t, err) + return fmt.Errorf("(%T) could not validate object format: %w", t, err) } err := t.nextTarget.WriteHeader(ctx, obj) From c42db4e761790d7221814e95df7c74cd7ae46036 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 11 Jul 2023 12:43:48 +0300 Subject: [PATCH 0406/1943] [#390] cli: Support more tree service API calls in CLI * Support healthcheck API call * Support move API call * Support remove API call * Support getSubtree API call * Support getOpLog API call Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-cli/modules/tree/add_by_path.go | 1 - cmd/frostfs-cli/modules/tree/get_by_path.go | 10 +- cmd/frostfs-cli/modules/tree/get_op_log.go | 83 +++++++++++++++ cmd/frostfs-cli/modules/tree/healthcheck.go | 42 ++++++++ cmd/frostfs-cli/modules/tree/move.go | 107 ++++++++++++++++++++ cmd/frostfs-cli/modules/tree/remove.go | 74 ++++++++++++++ cmd/frostfs-cli/modules/tree/root.go | 18 +++- cmd/frostfs-cli/modules/tree/subtree.go | 101 ++++++++++++++++++ 8 files changed, 430 insertions(+), 6 deletions(-) create mode 100644 cmd/frostfs-cli/modules/tree/get_op_log.go create mode 100644 cmd/frostfs-cli/modules/tree/healthcheck.go create mode 100644 cmd/frostfs-cli/modules/tree/move.go create mode 100644 cmd/frostfs-cli/modules/tree/remove.go create mode 100644 cmd/frostfs-cli/modules/tree/subtree.go diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index 90f4a6364..26e3dddfc 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -62,7 +62,6 @@ func addByPath(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "meta data parsing: %w", err) path, _ := cmd.Flags().GetString(pathFlagKey) - // pAttr, _ := cmd.Flags().GetString(pathAttributeFlagKey) req := new(tree.AddByPathRequest) req.Body = &tree.AddByPathRequest_Body{ diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index 38ad2bff5..c76027938 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -60,7 +60,11 @@ func getByPath(cmd *cobra.Command, _ []string) { latestOnly, _ := cmd.Flags().GetBool(latestOnlyFlagKey) path, _ := cmd.Flags().GetString(pathFlagKey) - // pAttr, _ := cmd.Flags().GetString(pathAttributeFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } req := new(tree.GetNodeByPathRequest) req.Body = &tree.GetNodeByPathRequest_Body{ @@ -71,9 +75,7 @@ func getByPath(cmd *cobra.Command, _ []string) { Path: strings.Split(path, "/"), LatestOnly: latestOnly, AllAttributes: true, - } - if btok := common.ReadBearerToken(cmd, bearerFlagKey); btok != nil { - req.Body.BearerToken = btok.Marshal() + BearerToken: bt, } commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) diff --git a/cmd/frostfs-cli/modules/tree/get_op_log.go b/cmd/frostfs-cli/modules/tree/get_op_log.go new file mode 100644 index 000000000..6efa76133 --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/get_op_log.go @@ -0,0 +1,83 @@ +package tree + +import ( + "crypto/sha256" + "errors" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var getOpLogCmd = &cobra.Command{ + Use: "get-op-log", + Short: "Get logged operations starting with some height", + Run: getOpLog, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initGetOpLogCmd() { + commonflags.Init(getOpLogCmd) + initCTID(getOpLogCmd) + + ff := getOpLogCmd.Flags() + ff.Uint64(heightFlagKey, 0, "Height to start with") + ff.Uint64(countFlagKey, 10, "Logged operations count") + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func getOpLog(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + + cidRaw, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidRaw) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + height, _ := cmd.Flags().GetUint64(heightFlagKey) + count, _ := cmd.Flags().GetUint64(countFlagKey) + + req := &tree.GetOpLogRequest{ + Body: &tree.GetOpLogRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + Height: height, + Count: count, + }, + } + + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) + + resp, err := cli.GetOpLog(ctx, req) + commonCmd.ExitOnErr(cmd, "get op log: %w", err) + + opLogResp, err := resp.Recv() + for ; err == nil; opLogResp, err = resp.Recv() { + o := opLogResp.GetBody().GetOperation() + + cmd.Println("Parent ID: ", o.GetParentId()) + + cmd.Println("\tChild ID: ", o.GetChildId()) + cmd.Printf("\tMeta: %s\n", o.GetMeta()) + } + if !errors.Is(err, io.EOF) { + commonCmd.ExitOnErr(cmd, "get op log response stream: %w", err) + } +} diff --git a/cmd/frostfs-cli/modules/tree/healthcheck.go b/cmd/frostfs-cli/modules/tree/healthcheck.go new file mode 100644 index 000000000..46d4c3409 --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/healthcheck.go @@ -0,0 +1,42 @@ +package tree + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + "github.com/spf13/cobra" +) + +var healthcheckCmd = &cobra.Command{ + Use: "healthcheck", + Short: "Check tree service availability", + Run: healthcheck, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initHealthcheckCmd() { + commonflags.Init(healthcheckCmd) + ff := healthcheckCmd.Flags() + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func healthcheck(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "client: %w", err) + + req := &tree.HealthcheckRequest{ + Body: &tree.HealthcheckRequest_Body{}, + } + commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + + _, err = cli.Healthcheck(ctx, req) + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + + cmd.Println("Success.") +} diff --git a/cmd/frostfs-cli/modules/tree/move.go b/cmd/frostfs-cli/modules/tree/move.go new file mode 100644 index 000000000..b91769185 --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/move.go @@ -0,0 +1,107 @@ +package tree + +import ( + "crypto/sha256" + "errors" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var moveCmd = &cobra.Command{ + Use: "move", + Short: "Move node", + Run: move, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initMoveCmd() { + commonflags.Init(moveCmd) + initCTID(moveCmd) + + ff := moveCmd.Flags() + ff.Uint64(nodeIDFlagKey, 0, "Node ID.") + ff.Uint64(parentIDFlagKey, 0, "Parent ID.") + + _ = getSubtreeCmd.MarkFlagRequired(nodeIDFlagKey) + _ = getSubtreeCmd.MarkFlagRequired(parentIDFlagKey) + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func move(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + cidString, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidString) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + pid, _ := cmd.Flags().GetUint64(parentIDFlagKey) + nid, _ := cmd.Flags().GetUint64(nodeIDFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + + subTreeReq := &tree.GetSubTreeRequest{ + Body: &tree.GetSubTreeRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + RootId: nid, + Depth: 1, + BearerToken: bt, + }, + } + commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(subTreeReq, pk)) + resp, err := cli.GetSubTree(ctx, subTreeReq) + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + + var meta []*tree.KeyValue + subtreeResp, err := resp.Recv() + for ; err == nil; subtreeResp, err = resp.Recv() { + meta = subtreeResp.GetBody().GetMeta() + } + if !errors.Is(err, io.EOF) { + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + } + var metaErr error + if len(meta) == 0 { + metaErr = errors.New("no meta for given node ID") + } + commonCmd.ExitOnErr(cmd, "unexpected rpc call result: %w", metaErr) + + req := &tree.MoveRequest{ + Body: &tree.MoveRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + ParentId: pid, + NodeId: nid, + Meta: meta, + }, + } + + commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + + _, err = cli.Move(ctx, req) + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + cmd.Println("Success.") +} diff --git a/cmd/frostfs-cli/modules/tree/remove.go b/cmd/frostfs-cli/modules/tree/remove.go new file mode 100644 index 000000000..7002ac9a4 --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/remove.go @@ -0,0 +1,74 @@ +package tree + +import ( + "crypto/sha256" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var removeCmd = &cobra.Command{ + Use: "remove", + Short: "Remove node", + Run: remove, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initRemoveCmd() { + commonflags.Init(removeCmd) + initCTID(removeCmd) + + ff := removeCmd.Flags() + ff.Uint64(nodeIDFlagKey, 0, "Node ID.") + + _ = getSubtreeCmd.MarkFlagRequired(nodeIDFlagKey) + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func remove(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + cidString, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidString) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + + nid, _ := cmd.Flags().GetUint64(nodeIDFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + req := &tree.RemoveRequest{ + Body: &tree.RemoveRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + NodeId: nid, + BearerToken: bt, + }, + } + + commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + + _, err = cli.Remove(ctx, req) + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + cmd.Println("Success.") +} diff --git a/cmd/frostfs-cli/modules/tree/root.go b/cmd/frostfs-cli/modules/tree/root.go index b5de0dd6f..701a78f2a 100644 --- a/cmd/frostfs-cli/modules/tree/root.go +++ b/cmd/frostfs-cli/modules/tree/root.go @@ -15,16 +15,28 @@ func init() { Cmd.AddCommand(getByPathCmd) Cmd.AddCommand(addByPathCmd) Cmd.AddCommand(listCmd) + Cmd.AddCommand(healthcheckCmd) + Cmd.AddCommand(moveCmd) + Cmd.AddCommand(removeCmd) + Cmd.AddCommand(getSubtreeCmd) + Cmd.AddCommand(getOpLogCmd) initAddCmd() initGetByPathCmd() initAddByPathCmd() initListCmd() + initHealthcheckCmd() + initMoveCmd() + initRemoveCmd() + initGetSubtreeCmd() + initGetOpLogCmd() } const ( treeIDFlagKey = "tid" parentIDFlagKey = "pid" + nodeIDFlagKey = "nid" + rootIDFlagKey = "root" metaFlagKey = "meta" @@ -34,6 +46,10 @@ const ( latestOnlyFlagKey = "latest" bearerFlagKey = "bearer" + + heightFlagKey = "height" + countFlagKey = "count" + depthFlagKey = "depth" ) func initCTID(cmd *cobra.Command) { @@ -45,5 +61,5 @@ func initCTID(cmd *cobra.Command) { ff.String(treeIDFlagKey, "", "Tree ID") _ = cmd.MarkFlagRequired(treeIDFlagKey) - ff.StringP(bearerFlagKey, "", "", "Path to bearer token") + ff.String(bearerFlagKey, "", "Path to bearer token") } diff --git a/cmd/frostfs-cli/modules/tree/subtree.go b/cmd/frostfs-cli/modules/tree/subtree.go new file mode 100644 index 000000000..f6dcf85ff --- /dev/null +++ b/cmd/frostfs-cli/modules/tree/subtree.go @@ -0,0 +1,101 @@ +package tree + +import ( + "crypto/sha256" + "errors" + "io" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +var getSubtreeCmd = &cobra.Command{ + Use: "get-subtree", + Short: "Get subtree", + Run: getSubTree, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func initGetSubtreeCmd() { + commonflags.Init(getSubtreeCmd) + initCTID(getSubtreeCmd) + + ff := getSubtreeCmd.Flags() + ff.Uint64(rootIDFlagKey, 0, "Root ID to traverse from.") + ff.Uint32(depthFlagKey, 10, "Traversal depth.") + + _ = getSubtreeCmd.MarkFlagRequired(commonflags.CIDFlag) + _ = getSubtreeCmd.MarkFlagRequired(treeIDFlagKey) + + _ = cobra.MarkFlagRequired(ff, commonflags.RPC) +} + +func getSubTree(cmd *cobra.Command, _ []string) { + pk := key.GetOrGenerate(cmd) + cidString, _ := cmd.Flags().GetString(commonflags.CIDFlag) + + var cnr cid.ID + err := cnr.DecodeString(cidString) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) + + ctx := cmd.Context() + + cli, err := _client(ctx) + commonCmd.ExitOnErr(cmd, "client: %w", err) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + tid, _ := cmd.Flags().GetString(treeIDFlagKey) + + rid, _ := cmd.Flags().GetUint64(rootIDFlagKey) + + depth, _ := cmd.Flags().GetUint32(depthFlagKey) + + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + + req := &tree.GetSubTreeRequest{ + Body: &tree.GetSubTreeRequest_Body{ + ContainerId: rawCID, + TreeId: tid, + RootId: rid, + Depth: depth, + BearerToken: bt, + }, + } + + commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + + resp, err := cli.GetSubTree(ctx, req) + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + + subtreeResp, err := resp.Recv() + for ; err == nil; subtreeResp, err = resp.Recv() { + b := subtreeResp.GetBody() + + cmd.Printf("Node ID: %d\n", b.GetNodeId()) + + cmd.Println("\tParent ID: ", b.GetParentId()) + cmd.Println("\tTimestamp: ", b.GetTimestamp()) + + if meta := b.GetMeta(); len(meta) > 0 { + cmd.Println("\tMeta pairs: ") + for _, kv := range meta { + cmd.Printf("\t\t%s: %s\n", kv.GetKey(), string(kv.GetValue())) + } + } + } + if !errors.Is(err, io.EOF) { + commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + } +} From 676a3efa79e136fc95ced5c8263c16032dd64f7b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 11 Jul 2023 12:36:33 +0300 Subject: [PATCH 0407/1943] [#390] cli: Make tree commands errors and messages more verbose Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-cli/modules/tree/add.go | 6 +++--- cmd/frostfs-cli/modules/tree/add_by_path.go | 6 +++--- cmd/frostfs-cli/modules/tree/get_by_path.go | 6 +++--- cmd/frostfs-cli/modules/tree/healthcheck.go | 9 +++++---- cmd/frostfs-cli/modules/tree/list.go | 6 +++--- cmd/frostfs-cli/modules/tree/move.go | 12 ++++++------ cmd/frostfs-cli/modules/tree/remove.go | 8 ++++---- cmd/frostfs-cli/modules/tree/subtree.go | 6 +++--- 8 files changed, 30 insertions(+), 29 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index ea9245271..28718687d 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -50,7 +50,7 @@ func add(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -64,10 +64,10 @@ func add(cmd *cobra.Command, _ []string) { BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.Add(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to cal add: %w", err) cmd.Println("Node ID: ", resp.Body.NodeId) } diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index 26e3dddfc..4661853a5 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -53,7 +53,7 @@ func addByPath(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -74,10 +74,10 @@ func addByPath(cmd *cobra.Command, _ []string) { BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.AddByPath(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to addByPath %w", err) cmd.Printf("Parent ID: %d\n", resp.GetBody().GetParentId()) diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index c76027938..f239066cd 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -53,7 +53,7 @@ func getByPath(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -78,10 +78,10 @@ func getByPath(cmd *cobra.Command, _ []string) { BearerToken: bt, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.GetNodeByPath(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to call getNodeByPath: %w", err) nn := resp.GetBody().GetNodes() if len(nn) == 0 { diff --git a/cmd/frostfs-cli/modules/tree/healthcheck.go b/cmd/frostfs-cli/modules/tree/healthcheck.go index 46d4c3409..f0506467e 100644 --- a/cmd/frostfs-cli/modules/tree/healthcheck.go +++ b/cmd/frostfs-cli/modules/tree/healthcheck.go @@ -1,6 +1,7 @@ package tree import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -28,15 +29,15 @@ func healthcheck(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) req := &tree.HealthcheckRequest{ Body: &tree.HealthcheckRequest_Body{}, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) _, err = cli.Healthcheck(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to call healthcheck: %w", err) - cmd.Println("Success.") + common.PrintVerbose(cmd, "Successful healthcheck invocation.") } diff --git a/cmd/frostfs-cli/modules/tree/list.go b/cmd/frostfs-cli/modules/tree/list.go index 8e4d2bd4c..a25d066d5 100644 --- a/cmd/frostfs-cli/modules/tree/list.go +++ b/cmd/frostfs-cli/modules/tree/list.go @@ -41,7 +41,7 @@ func list(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -52,10 +52,10 @@ func list(cmd *cobra.Command, _ []string) { }, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.TreeList(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to call treeList %w", err) for _, treeID := range resp.GetBody().GetIds() { cmd.Println(treeID) diff --git a/cmd/frostfs-cli/modules/tree/move.go b/cmd/frostfs-cli/modules/tree/move.go index b91769185..84b2fb80e 100644 --- a/cmd/frostfs-cli/modules/tree/move.go +++ b/cmd/frostfs-cli/modules/tree/move.go @@ -48,7 +48,7 @@ func move(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -71,7 +71,7 @@ func move(cmd *cobra.Command, _ []string) { BearerToken: bt, }, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(subTreeReq, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(subTreeReq, pk)) resp, err := cli.GetSubTree(ctx, subTreeReq) commonCmd.ExitOnErr(cmd, "rpc call: %w", err) @@ -81,7 +81,7 @@ func move(cmd *cobra.Command, _ []string) { meta = subtreeResp.GetBody().GetMeta() } if !errors.Is(err, io.EOF) { - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to read getSubTree response stream: %w", err) } var metaErr error if len(meta) == 0 { @@ -99,9 +99,9 @@ func move(cmd *cobra.Command, _ []string) { }, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) _, err = cli.Move(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) - cmd.Println("Success.") + commonCmd.ExitOnErr(cmd, "failed to call move: %w", err) + common.PrintVerbose(cmd, "Successful move invocation.") } diff --git a/cmd/frostfs-cli/modules/tree/remove.go b/cmd/frostfs-cli/modules/tree/remove.go index 7002ac9a4..74e9d9749 100644 --- a/cmd/frostfs-cli/modules/tree/remove.go +++ b/cmd/frostfs-cli/modules/tree/remove.go @@ -44,7 +44,7 @@ func remove(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -66,9 +66,9 @@ func remove(cmd *cobra.Command, _ []string) { }, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) _, err = cli.Remove(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) - cmd.Println("Success.") + commonCmd.ExitOnErr(cmd, "failed to call remove: %w", err) + common.PrintVerbose(cmd, "Successful remove invocation.") } diff --git a/cmd/frostfs-cli/modules/tree/subtree.go b/cmd/frostfs-cli/modules/tree/subtree.go index f6dcf85ff..64cb351ec 100644 --- a/cmd/frostfs-cli/modules/tree/subtree.go +++ b/cmd/frostfs-cli/modules/tree/subtree.go @@ -48,7 +48,7 @@ func getSubTree(cmd *cobra.Command, _ []string) { ctx := cmd.Context() cli, err := _client(ctx) - commonCmd.ExitOnErr(cmd, "client: %w", err) + commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) @@ -74,10 +74,10 @@ func getSubTree(cmd *cobra.Command, _ []string) { }, } - commonCmd.ExitOnErr(cmd, "message signing: %w", tree.SignMessage(req, pk)) + commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.GetSubTree(ctx, req) - commonCmd.ExitOnErr(cmd, "rpc call: %w", err) + commonCmd.ExitOnErr(cmd, "failed to call getSubTree: %w", err) subtreeResp, err := resp.Recv() for ; err == nil; subtreeResp, err = resp.Recv() { From 0754e6e65403b28cc73b1115f309939b2dd24c44 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 11 Jul 2023 13:31:49 +0300 Subject: [PATCH 0408/1943] [#390] cli: Fix bearer token reading for tree subcommands Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-cli/modules/tree/add.go | 7 ++++++- cmd/frostfs-cli/modules/tree/add_by_path.go | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index 28718687d..4ac1ed198 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -55,13 +55,18 @@ func add(cmd *cobra.Command, _ []string) { rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + req := new(tree.AddRequest) req.Body = &tree.AddRequest_Body{ ContainerId: rawCID, TreeId: tid, ParentId: pid, Meta: meta, - BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), + BearerToken: bt, } commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index 4661853a5..ea815dbfe 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -63,6 +63,11 @@ func addByPath(cmd *cobra.Command, _ []string) { path, _ := cmd.Flags().GetString(pathFlagKey) + var bt []byte + if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { + bt = t.Marshal() + } + req := new(tree.AddByPathRequest) req.Body = &tree.AddByPathRequest_Body{ ContainerId: rawCID, @@ -71,7 +76,7 @@ func addByPath(cmd *cobra.Command, _ []string) { // PathAttribute: pAttr, Path: strings.Split(path, "/"), Meta: meta, - BearerToken: common.ReadBearerToken(cmd, bearerFlagKey).Marshal(), + BearerToken: bt, } commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) From 80481c015c236fac0dfb7a600952291f92738858 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Jul 2023 17:33:13 +0300 Subject: [PATCH 0409/1943] [#509] putsvc: Omit AccessIdentifiers from iteratePlacement() Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/distributed.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 51678fa7b..c8294c0b8 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -146,7 +146,14 @@ func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdent t.traversal.extraBroadcastEnabled = true } - return t.iteratePlacement(ctx) + if err := t.iteratePlacement(ctx); err != nil { + return nil, err + } + + id, _ := t.obj.ID() + return &transformer.AccessIdentifiers{ + SelfID: id, + }, nil } func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error { @@ -164,14 +171,14 @@ func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error return nil } -func (t *distributedTarget) iteratePlacement(ctx context.Context) (*transformer.AccessIdentifiers, error) { +func (t *distributedTarget) iteratePlacement(ctx context.Context) error { id, _ := t.obj.ID() traverser, err := placement.NewTraverser( append(t.traversal.opts, placement.ForObject(id))..., ) if err != nil { - return nil, fmt.Errorf("(%T) could not create object placement traverser: %w", t, err) + return fmt.Errorf("(%T) could not create object placement traverser: %w", t, err) } resErr := &atomic.Value{} @@ -190,23 +197,19 @@ func (t *distributedTarget) iteratePlacement(ctx context.Context) (*transformer. if !traverser.Success() { var err errIncompletePut err.singleErr, _ = resErr.Load().(error) - return nil, err + return err } // perform additional container broadcast if needed if t.traversal.submitPrimaryPlacementFinish() { - _, err = t.iteratePlacement(ctx) + err = t.iteratePlacement(ctx) if err != nil { t.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) // we don't fail primary operation because of broadcast failure } } - id, _ = t.obj.ID() - - return &transformer.AccessIdentifiers{ - SelfID: id, - }, nil + return nil } func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *placement.Traverser, addrs []placement.Node, resErr *atomic.Value) bool { From 7da284f3e84d596fca56e93947c98bc74e579706 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Jul 2023 17:32:00 +0300 Subject: [PATCH 0410/1943] [#509] go.mod: Update sdk-go Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- pkg/services/object/put/distributed.go | 31 ++++++++++++++++---------- pkg/services/object/put/streamer.go | 6 ++--- pkg/services/object/put/validation.go | 4 ++-- 5 files changed, 27 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 9a53bb8fa..e2735b223 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230706140617-98cab7ed6166 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230711142135-998fe1a7ab31 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 31a4cb9e1..ddbc77633 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230706140617-98cab7ed6166 h1:HSwD/CDbrUp45gQmfn9KYag8zN0GD+HA0l2+U+c3Ayo= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230706140617-98cab7ed6166/go.mod h1:r5Fir/4jCVXzdfOyCUbikSDB99nVqnHNq7mzVcidnlA= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230711142135-998fe1a7ab31 h1:PSHVtyD3vw2NTbdWBUz0Wql1WH42nSCP+4j6PmIgZ8Y= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230711142135-998fe1a7ab31/go.mod h1:MlLlhZb4qbLhUScDngfujUZKhs0/2YKW9D8LTY3BApY= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index c8294c0b8..408ee099c 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -135,18 +135,7 @@ func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdent t.obj.SetPayload(t.payload.Data) - var err error - - if t.objMeta, err = t.fmt.ValidateContent(t.obj); err != nil { - return nil, fmt.Errorf("(%T) could not validate payload content: %w", t, err) - } - - if len(t.obj.Children()) > 0 { - // enabling extra broadcast for linking objects - t.traversal.extraBroadcastEnabled = true - } - - if err := t.iteratePlacement(ctx); err != nil { + if err := t.WriteObject(ctx, t.obj); err != nil { return nil, err } @@ -156,6 +145,24 @@ func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdent }, nil } +// WriteObject implements the transformer.ObjectWriter interface. +func (t *distributedTarget) WriteObject(ctx context.Context, obj *objectSDK.Object) error { + t.obj = obj + + var err error + + if t.objMeta, err = t.fmt.ValidateContent(t.obj); err != nil { + return fmt.Errorf("(%T) could not validate payload content: %w", t, err) + } + + if len(t.obj.Children()) > 0 { + // enabling extra broadcast for linking objects + t.traversal.extraBroadcastEnabled = true + } + + return t.iteratePlacement(ctx) +} + func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error { if !node.local && t.relay != nil { return t.relay(ctx, node) diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index bf6c20588..2dccafa47 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -22,7 +22,7 @@ type Streamer struct { sessionKey *ecdsa.PrivateKey - target transformer.ObjectTarget + target transformer.ChunkedObjectWriter relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error @@ -129,7 +129,7 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { fmt: p.fmtValidator, nextTarget: transformer.NewPayloadSizeLimiter(transformer.Params{ Key: sessionKey, - NextTargetInit: func() transformer.ObjectTarget { return p.newCommonTarget(prm) }, + NextTargetInit: func() transformer.ObjectWriter { return p.newCommonTarget(prm) }, NetworkState: p.networkState, MaxSize: p.maxPayloadSz, WithoutHomomorphicHash: containerSDK.IsHomomorphicHashingDisabled(prm.cnr), @@ -192,7 +192,7 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error { return nil } -func (p *Streamer) newCommonTarget(prm *PutInitPrm) transformer.ObjectTarget { +func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { var relay func(context.Context, nodeDesc) error if p.relay != nil { relay = func(ctx context.Context, node nodeDesc) error { diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/put/validation.go index d4b08a038..c2b078ef5 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/put/validation.go @@ -17,14 +17,14 @@ import ( // validatingTarget validates unprepared object format and content (streaming PUT case). type validatingTarget struct { - nextTarget transformer.ObjectTarget + nextTarget transformer.ChunkedObjectWriter fmt *object.FormatValidator } // validatingPreparedTarget validates prepared object format and content. type validatingPreparedTarget struct { - nextTarget transformer.ObjectTarget + nextTarget transformer.ChunkedObjectWriter fmt *object.FormatValidator From 61541eaec2c0feab381b57dc512a6380395c7b2f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 10:05:09 +0300 Subject: [PATCH 0411/1943] [#294] aclsvc: Refactor checker constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 11 +++--- pkg/services/object/acl/acl.go | 55 ++++++----------------------- pkg/services/object/acl/acl_test.go | 11 +++--- 3 files changed, 20 insertions(+), 57 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 1d99d958b..45b1089a7 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -429,12 +429,11 @@ func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter) v2.Se ), v2.WithNextService(splitSvc), v2.WithEACLChecker( - acl.NewChecker(new(acl.CheckerPrm). - SetNetmapState(c.cfgNetmap.state). - SetEACLSource(c.cfgObject.eaclSource). - SetValidator(eaclSDK.NewValidator()). - SetLocalStorage(ls), - ), + acl.NewChecker( + c.cfgNetmap.state, + c.cfgObject.eaclSource, + eaclSDK.NewValidator(), + ls), ), ) } diff --git a/pkg/services/object/acl/acl.go b/pkg/services/object/acl/acl.go index 351b4ad3b..cfe49396e 100644 --- a/pkg/services/object/acl/acl.go +++ b/pkg/services/object/acl/acl.go @@ -21,35 +21,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) -// CheckerPrm groups parameters for Checker -// constructor. -type CheckerPrm struct { - eaclSrc container.EACLSource - validator *eaclSDK.Validator - localStorage *engine.StorageEngine - state netmap.State -} - -func (c *CheckerPrm) SetEACLSource(v container.EACLSource) *CheckerPrm { - c.eaclSrc = v - return c -} - -func (c *CheckerPrm) SetValidator(v *eaclSDK.Validator) *CheckerPrm { - c.validator = v - return c -} - -func (c *CheckerPrm) SetLocalStorage(v *engine.StorageEngine) *CheckerPrm { - c.localStorage = v - return c -} - -func (c *CheckerPrm) SetNetmapState(v netmap.State) *CheckerPrm { - c.state = v - return c -} - // Checker implements v2.ACLChecker interfaces and provides // ACL/eACL validation functionality. type Checker struct { @@ -71,23 +42,17 @@ var ( // NewChecker creates Checker. // Panics if at least one of the parameter is nil. -func NewChecker(prm *CheckerPrm) *Checker { - panicOnNil := func(fieldName string, field any) { - if field == nil { - panic(fmt.Sprintf("incorrect field %s (%T): %v", fieldName, field, field)) - } - } - - panicOnNil("EACLSource", prm.eaclSrc) - panicOnNil("EACLValidator", prm.validator) - panicOnNil("LocalStorageEngine", prm.localStorage) - panicOnNil("NetmapState", prm.state) - +func NewChecker( + state netmap.State, + eaclSrc container.EACLSource, + validator *eaclSDK.Validator, + localStorage *engine.StorageEngine, +) *Checker { return &Checker{ - eaclSrc: prm.eaclSrc, - validator: prm.validator, - localStorage: prm.localStorage, - state: prm.state, + eaclSrc: eaclSrc, + validator: validator, + localStorage: localStorage, + state: state, } } diff --git a/pkg/services/object/acl/acl_test.go b/pkg/services/object/acl/acl_test.go index d3ad1e6fd..b9b82dac8 100644 --- a/pkg/services/object/acl/acl_test.go +++ b/pkg/services/object/acl/acl_test.go @@ -27,12 +27,11 @@ func (e emptyNetmapState) CurrentEpoch() uint64 { } func TestStickyCheck(t *testing.T) { - checker := NewChecker(new(CheckerPrm). - SetLocalStorage(&engine.StorageEngine{}). - SetValidator(eaclSDK.NewValidator()). - SetEACLSource(emptyEACLSource{}). - SetNetmapState(emptyNetmapState{}), - ) + checker := NewChecker( + emptyNetmapState{}, + emptyEACLSource{}, + eaclSDK.NewValidator(), + &engine.StorageEngine{}) t.Run("system role", func(t *testing.T) { var info v2.RequestInfo From 18d8898b00d94b33fd2c3bddee83dbadbd6ce15b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 11:28:36 +0300 Subject: [PATCH 0412/1943] [#294] aclsvc: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/acl/acl.go | 36 +++++++++-------- pkg/services/object/acl/eacl/v2/eacl_test.go | 6 +-- pkg/services/object/acl/eacl/v2/headers.go | 20 ++++------ pkg/services/object/acl/eacl/v2/localstore.go | 22 ----------- pkg/services/object/acl/eacl/v2/opts.go | 39 ------------------- pkg/services/object/acl/eacl/v2/xheader.go | 10 ++++- 6 files changed, 39 insertions(+), 94 deletions(-) delete mode 100644 pkg/services/object/acl/eacl/v2/localstore.go diff --git a/pkg/services/object/acl/acl.go b/pkg/services/object/acl/acl.go index cfe49396e..921545c8b 100644 --- a/pkg/services/object/acl/acl.go +++ b/pkg/services/object/acl/acl.go @@ -1,10 +1,12 @@ package acl import ( + "context" "crypto/ecdsa" "crypto/elliptic" "errors" "fmt" + "io" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -17,6 +19,8 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) @@ -30,6 +34,18 @@ type Checker struct { state netmap.State } +type localStorage struct { + ls *engine.StorageEngine +} + +func (s *localStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { + if s.ls == nil { + return nil, io.ErrUnexpectedEOF + } + + return engine.Head(ctx, s.ls, addr) +} + // Various EACL check errors. var ( errEACLDeniedByRule = errors.New("denied by rule") @@ -158,26 +174,14 @@ func getRole(reqInfo v2.RequestInfo) eaclSDK.Role { } func (c *Checker) getHeaderSource(cnr cid.ID, msg any, reqInfo v2.RequestInfo) (eaclSDK.TypedHeaderSource, error) { - hdrSrcOpts := make([]eaclV2.Option, 0, 3) - - hdrSrcOpts = append(hdrSrcOpts, - eaclV2.WithLocalObjectStorage(c.localStorage), - eaclV2.WithCID(cnr), - eaclV2.WithOID(reqInfo.ObjectID()), - ) - + var xHeaderSource eaclV2.XHeaderSource if req, ok := msg.(eaclV2.Request); ok { - hdrSrcOpts = append(hdrSrcOpts, eaclV2.WithServiceRequest(req)) + xHeaderSource = eaclV2.NewRequestXHeaderSource(req) } else { - hdrSrcOpts = append(hdrSrcOpts, - eaclV2.WithServiceResponse( - msg.(eaclV2.Response), - reqInfo.Request().(eaclV2.Request), - ), - ) + xHeaderSource = eaclV2.NewResponseXHeaderSource(msg.(eaclV2.Response), reqInfo.Request().(eaclV2.Request)) } - hdrSrc, err := eaclV2.NewMessageHeaderSource(hdrSrcOpts...) + hdrSrc, err := eaclV2.NewMessageHeaderSource(&localStorage{ls: c.localStorage}, xHeaderSource, cnr, eaclV2.WithOID(reqInfo.ObjectID())) if err != nil { return nil, fmt.Errorf("can't parse headers: %w", err) } diff --git a/pkg/services/object/acl/eacl/v2/eacl_test.go b/pkg/services/object/acl/eacl/v2/eacl_test.go index 6f879123b..023b99239 100644 --- a/pkg/services/object/acl/eacl/v2/eacl_test.go +++ b/pkg/services/object/acl/eacl/v2/eacl_test.go @@ -103,9 +103,9 @@ func TestHeadRequest(t *testing.T) { newSource := func(t *testing.T) eaclSDK.TypedHeaderSource { hdrSrc, err := NewMessageHeaderSource( - WithObjectStorage(lStorage), - WithServiceRequest(req), - WithCID(addr.Container()), + lStorage, + NewRequestXHeaderSource(req), + addr.Container(), WithOID(&id)) require.NoError(t, err) return hdrSrc diff --git a/pkg/services/object/acl/eacl/v2/headers.go b/pkg/services/object/acl/eacl/v2/headers.go index c7a1b7729..7408c96e9 100644 --- a/pkg/services/object/acl/eacl/v2/headers.go +++ b/pkg/services/object/acl/eacl/v2/headers.go @@ -21,7 +21,7 @@ type Option func(*cfg) type cfg struct { storage ObjectStorage - msg xHeaderSource + msg XHeaderSource cnr cid.ID obj *oid.ID @@ -46,14 +46,12 @@ type headerSource struct { incompleteObjectHeaders bool } -func defaultCfg() *cfg { - return &cfg{ - storage: new(localStorage), +func NewMessageHeaderSource(os ObjectStorage, xhs XHeaderSource, cnrID cid.ID, opts ...Option) (eaclSDK.TypedHeaderSource, error) { + cfg := &cfg{ + storage: os, + cnr: cnrID, + msg: xhs, } -} - -func NewMessageHeaderSource(opts ...Option) (eaclSDK.TypedHeaderSource, error) { - cfg := defaultCfg() for i := range opts { opts[i](cfg) @@ -70,7 +68,7 @@ func NewMessageHeaderSource(opts ...Option) (eaclSDK.TypedHeaderSource, error) { return nil, err } - res.requestHeaders = requestHeaders(cfg.msg) + res.requestHeaders = cfg.msg.GetXHeaders() return res, nil } @@ -96,10 +94,6 @@ func (x xHeader) Value() string { return (*session.XHeader)(&x).GetValue() } -func requestHeaders(msg xHeaderSource) []eaclSDK.Header { - return msg.GetXHeaders() -} - var errMissingOID = errors.New("object ID is missing") func (h *cfg) readObjectHeaders(dst *headerSource) error { diff --git a/pkg/services/object/acl/eacl/v2/localstore.go b/pkg/services/object/acl/eacl/v2/localstore.go deleted file mode 100644 index 0f23e9881..000000000 --- a/pkg/services/object/acl/eacl/v2/localstore.go +++ /dev/null @@ -1,22 +0,0 @@ -package v2 - -import ( - "context" - "io" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -type localStorage struct { - ls *engine.StorageEngine -} - -func (s *localStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { - if s.ls == nil { - return nil, io.ErrUnexpectedEOF - } - - return engine.Head(ctx, s.ls, addr) -} diff --git a/pkg/services/object/acl/eacl/v2/opts.go b/pkg/services/object/acl/eacl/v2/opts.go index 7657e8780..d91a21c75 100644 --- a/pkg/services/object/acl/eacl/v2/opts.go +++ b/pkg/services/object/acl/eacl/v2/opts.go @@ -1,48 +1,9 @@ package v2 import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -func WithObjectStorage(v ObjectStorage) Option { - return func(c *cfg) { - c.storage = v - } -} - -func WithLocalObjectStorage(v *engine.StorageEngine) Option { - return func(c *cfg) { - c.storage = &localStorage{ - ls: v, - } - } -} - -func WithServiceRequest(v Request) Option { - return func(c *cfg) { - c.msg = requestXHeaderSource{ - req: v, - } - } -} - -func WithServiceResponse(resp Response, req Request) Option { - return func(c *cfg) { - c.msg = responseXHeaderSource{ - resp: resp, - req: req, - } - } -} - -func WithCID(v cid.ID) Option { - return func(c *cfg) { - c.cnr = v - } -} - func WithOID(v *oid.ID) Option { return func(c *cfg) { c.obj = v diff --git a/pkg/services/object/acl/eacl/v2/xheader.go b/pkg/services/object/acl/eacl/v2/xheader.go index 246714af7..c1fdea9d8 100644 --- a/pkg/services/object/acl/eacl/v2/xheader.go +++ b/pkg/services/object/acl/eacl/v2/xheader.go @@ -5,7 +5,7 @@ import ( eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" ) -type xHeaderSource interface { +type XHeaderSource interface { GetXHeaders() []eaclSDK.Header } @@ -13,12 +13,20 @@ type requestXHeaderSource struct { req Request } +func NewRequestXHeaderSource(req Request) XHeaderSource { + return requestXHeaderSource{req: req} +} + type responseXHeaderSource struct { resp Response req Request } +func NewResponseXHeaderSource(resp Response, req Request) XHeaderSource { + return responseXHeaderSource{resp: resp, req: req} +} + func (s requestXHeaderSource) GetXHeaders() []eaclSDK.Header { ln := 0 From 70a108198884211b35f10b1a196eefe3234e56ef Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 11:44:37 +0300 Subject: [PATCH 0413/1943] [#294] aclsvcv2: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 22 +++++++-------- pkg/services/object/acl/v2/opts.go | 39 --------------------------- pkg/services/object/acl/v2/service.go | 34 ++++++++++------------- 3 files changed, 23 insertions(+), 72 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 45b1089a7..da7458170 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -421,20 +421,16 @@ func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter) v2.Se irFetcher := createInnerRingFetcher(c) return v2.New( + splitSvc, + c.netMapSource, + newCachedIRFetcher(irFetcher), + acl.NewChecker( + c.cfgNetmap.state, + c.cfgObject.eaclSource, + eaclSDK.NewValidator(), + ls), + c.cfgObject.cnrSource, v2.WithLogger(c.log), - v2.WithIRFetcher(newCachedIRFetcher(irFetcher)), - v2.WithNetmapSource(c.netMapSource), - v2.WithContainerSource( - c.cfgObject.cnrSource, - ), - v2.WithNextService(splitSvc), - v2.WithEACLChecker( - acl.NewChecker( - c.cfgNetmap.state, - c.cfgObject.eaclSource, - eaclSDK.NewValidator(), - ls), - ), ) } diff --git a/pkg/services/object/acl/v2/opts.go b/pkg/services/object/acl/v2/opts.go index 7e937da06..15fcce884 100644 --- a/pkg/services/object/acl/v2/opts.go +++ b/pkg/services/object/acl/v2/opts.go @@ -1,9 +1,6 @@ package v2 import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" ) @@ -13,39 +10,3 @@ func WithLogger(v *logger.Logger) Option { c.log = v } } - -// WithNetmapSource return option to set -// netmap source. -func WithNetmapSource(v netmap.Source) Option { - return func(c *cfg) { - c.nm = v - } -} - -// WithContainerSource returns option to set container source. -func WithContainerSource(v container.Source) Option { - return func(c *cfg) { - c.containers = v - } -} - -// WithNextService returns option to set next object service. -func WithNextService(v objectSvc.ServiceServer) Option { - return func(c *cfg) { - c.next = v - } -} - -// WithEACLChecker returns option to set eACL checker. -func WithEACLChecker(v ACLChecker) Option { - return func(c *cfg) { - c.checker = v - } -} - -// WithIRFetcher returns option to set inner ring fetcher. -func WithIRFetcher(v InnerRingFetcher) Option { - return func(c *cfg) { - c.irFetcher = v - } -} diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index c75bd326f..412708b1c 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -73,32 +73,26 @@ type cfg struct { next object.ServiceServer } -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - } -} - // New is a constructor for object ACL checking service. -func New(opts ...Option) Service { - cfg := defaultCfg() +func New(next object.ServiceServer, + nm netmap.Source, + irf InnerRingFetcher, + acl ACLChecker, + cs container.Source, + opts ...Option) Service { + cfg := &cfg{ + log: &logger.Logger{Logger: zap.L()}, + next: next, + nm: nm, + irFetcher: irf, + checker: acl, + containers: cs, + } for i := range opts { opts[i](cfg) } - panicOnNil := func(v any, name string) { - if v == nil { - panic(fmt.Sprintf("ACL service: %s is nil", name)) - } - } - - panicOnNil(cfg.next, "next Service") - panicOnNil(cfg.nm, "netmap client") - panicOnNil(cfg.irFetcher, "inner Ring fetcher") - panicOnNil(cfg.checker, "acl checker") - panicOnNil(cfg.containers, "container source") - return Service{ cfg: cfg, c: senderClassifier{ From a476d8285abd39cc7b74284ae8ae75fed05a5ed7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 11:53:18 +0300 Subject: [PATCH 0414/1943] [#294] deletesvcv2: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 4 +--- pkg/services/object/delete/v2/service.go | 23 ++--------------------- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index da7458170..3f0db1461 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -397,9 +397,7 @@ func createDeleteService(c *cfg, keyStorage *util.KeyStorage, sGet *getsvc.Servi } func createDeleteServiceV2(sDelete *deletesvc.Service) *deletesvcV2.Service { - return deletesvcV2.NewService( - deletesvcV2.WithInternalService(sDelete), - ) + return deletesvcV2.NewService(sDelete) } func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Service, diff --git a/pkg/services/object/delete/v2/service.go b/pkg/services/object/delete/v2/service.go index 51759c5df..10dcd0e87 100644 --- a/pkg/services/object/delete/v2/service.go +++ b/pkg/services/object/delete/v2/service.go @@ -9,26 +9,13 @@ import ( // Service implements Delete operation of Object service v2. type Service struct { - *cfg -} - -// Option represents Service constructor option. -type Option func(*cfg) - -type cfg struct { svc *deletesvc.Service } // NewService constructs Service instance from provided options. -func NewService(opts ...Option) *Service { - c := new(cfg) - - for i := range opts { - opts[i](c) - } - +func NewService(svc *deletesvc.Service) *Service { return &Service{ - cfg: c, + svc: svc, } } @@ -51,9 +38,3 @@ func (s *Service) Delete(ctx context.Context, req *objectV2.DeleteRequest) (*obj return resp, nil } - -func WithInternalService(v *deletesvc.Service) Option { - return func(c *cfg) { - c.svc = v - } -} From 1420b8b9eae4f315c3bb1701589b1591718b6bba Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 12:04:00 +0300 Subject: [PATCH 0415/1943] [#294] deletesvc: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 14 +++---- pkg/services/object/delete/service.go | 58 +++++++-------------------- 2 files changed, 21 insertions(+), 51 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 3f0db1461..1f763dd9a 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -382,17 +382,17 @@ func createGetServiceV2(sGet *getsvc.Service, keyStorage *util.KeyStorage) *gets func createDeleteService(c *cfg, keyStorage *util.KeyStorage, sGet *getsvc.Service, sSearch *searchsvc.Service, sPut *putsvc.Service) *deletesvc.Service { return deletesvc.New( - deletesvc.WithLogger(c.log), - deletesvc.WithHeadService(sGet), - deletesvc.WithSearchService(sSearch), - deletesvc.WithPutService(sPut), - deletesvc.WithNetworkInfo(&delNetInfo{ + sGet, + sSearch, + sPut, + &delNetInfo{ State: c.cfgNetmap.state, tsLifetime: c.cfgObject.tombstoneLifetime, cfg: c, - }), - deletesvc.WithKeyStorage(keyStorage), + }, + keyStorage, + deletesvc.WithLogger(c.log), ) } diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index 5c09ad123..e208a298b 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -62,16 +62,22 @@ type cfg struct { keyStorage *util.KeyStorage } -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - } -} - // New creates, initializes and returns utility serving // Object.Get service requests. -func New(opts ...Option) *Service { - c := defaultCfg() +func New(gs *getsvc.Service, + ss *searchsvc.Service, + ps *putsvc.Service, + ni NetworkInfo, + ks *util.KeyStorage, + opts ...Option) *Service { + c := &cfg{ + log: &logger.Logger{Logger: zap.L()}, + header: (*headSvcWrapper)(gs), + searcher: (*searchSvcWrapper)(ss), + placer: (*putSvcWrapper)(ps), + netInfo: ni, + keyStorage: ks, + } for i := range opts { opts[i](c) @@ -88,39 +94,3 @@ func WithLogger(l *logger.Logger) Option { c.log = &logger.Logger{Logger: l.With(zap.String("component", "objectSDK.Delete service"))} } } - -// WithHeadService returns option to set Head service -// to work with object headers. -func WithHeadService(h *getsvc.Service) Option { - return func(c *cfg) { - c.header = (*headSvcWrapper)(h) - } -} - -// WithSearchService returns option to set search service. -func WithSearchService(s *searchsvc.Service) Option { - return func(c *cfg) { - c.searcher = (*searchSvcWrapper)(s) - } -} - -// WithPutService returns option to specify put service. -func WithPutService(p *putsvc.Service) Option { - return func(c *cfg) { - c.placer = (*putSvcWrapper)(p) - } -} - -// WithNetworkInfo returns option to set network information source. -func WithNetworkInfo(netInfo NetworkInfo) Option { - return func(c *cfg) { - c.netInfo = netInfo - } -} - -// WithKeyStorage returns option to set local private key storage. -func WithKeyStorage(ks *util.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = ks - } -} From 800a685e8477289f8c5ed6f7a58cbc97b823c895 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 12:21:03 +0300 Subject: [PATCH 0416/1943] [#294] putsvc: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 16 +++--- pkg/services/object/put/service.go | 84 ++++++++---------------------- 2 files changed, 30 insertions(+), 70 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 1f763dd9a..43d68bdcd 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -313,14 +313,14 @@ func createPutSvc(c *cfg, keyStorage *util.KeyStorage) *putsvc.Service { } return putsvc.NewService( - putsvc.WithKeyStorage(keyStorage), - putsvc.WithClientConstructor(c.putClientCache), - putsvc.WithMaxSizeSource(newCachedMaxObjectSizeSource(c)), - putsvc.WithObjectStorage(os), - putsvc.WithContainerSource(c.cfgObject.cnrSource), - putsvc.WithNetworkMapSource(c.netMapSource), - putsvc.WithNetmapKeys(c), - putsvc.WithNetworkState(c.cfgNetmap.state), + keyStorage, + c.putClientCache, + newCachedMaxObjectSizeSource(c), + os, + c.cfgObject.cnrSource, + c.netMapSource, + c, + c.cfgNetmap.state, putsvc.WithWorkerPools(c.cfgObject.pool.putRemote, c.cfgObject.pool.putLocal), putsvc.WithLogger(c.log), ) diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 567a3fea1..7f2600f9c 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -46,8 +46,6 @@ type cfg struct { fmtValidator *object.FormatValidator - fmtValidatorOpts []object.FormatValidatorOption - networkState netmap.State clientConstructor ClientConstructor @@ -55,22 +53,34 @@ type cfg struct { log *logger.Logger } -func defaultCfg() *cfg { - return &cfg{ - remotePool: util.NewPseudoWorkerPool(), - localPool: util.NewPseudoWorkerPool(), - log: &logger.Logger{Logger: zap.L()}, +func NewService(ks *objutil.KeyStorage, + cc ClientConstructor, + ms MaxSizeSource, + os ObjectStorage, + cs container.Source, + ns netmap.Source, + nk netmap.AnnouncedKeys, + nst netmap.State, + opts ...Option) *Service { + c := &cfg{ + remotePool: util.NewPseudoWorkerPool(), + localPool: util.NewPseudoWorkerPool(), + log: &logger.Logger{Logger: zap.L()}, + keyStorage: ks, + clientConstructor: cc, + maxSizeSrc: ms, + localStore: os, + cnrSrc: cs, + netMapSrc: ns, + netmapKeys: nk, + networkState: nst, } -} - -func NewService(opts ...Option) *Service { - c := defaultCfg() for i := range opts { opts[i](c) } - c.fmtValidator = object.NewFormatValidator(c.fmtValidatorOpts...) + c.fmtValidator = object.NewFormatValidator(object.WithLockSource(os), object.WithNetState(nst)) return &Service{ cfg: c, @@ -83,62 +93,12 @@ func (p *Service) Put() (*Streamer, error) { }, nil } -func WithKeyStorage(v *objutil.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = v - } -} - -func WithMaxSizeSource(v MaxSizeSource) Option { - return func(c *cfg) { - c.maxSizeSrc = v - } -} - -func WithObjectStorage(v ObjectStorage) Option { - return func(c *cfg) { - c.localStore = v - c.fmtValidatorOpts = append(c.fmtValidatorOpts, object.WithLockSource(v)) - } -} - -func WithContainerSource(v container.Source) Option { - return func(c *cfg) { - c.cnrSrc = v - } -} - -func WithNetworkMapSource(v netmap.Source) Option { - return func(c *cfg) { - c.netMapSrc = v - } -} - func WithWorkerPools(remote, local util.WorkerPool) Option { return func(c *cfg) { c.remotePool, c.localPool = remote, local } } -func WithNetmapKeys(v netmap.AnnouncedKeys) Option { - return func(c *cfg) { - c.netmapKeys = v - } -} - -func WithNetworkState(v netmap.State) Option { - return func(c *cfg) { - c.networkState = v - c.fmtValidatorOpts = append(c.fmtValidatorOpts, object.WithNetState(v)) - } -} - -func WithClientConstructor(v ClientConstructor) Option { - return func(c *cfg) { - c.clientConstructor = v - } -} - func WithLogger(l *logger.Logger) Option { return func(c *cfg) { c.log = l From ec9b73846593666629a3c306369b6ddb0d99a912 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 12:31:03 +0300 Subject: [PATCH 0417/1943] [#294] putsvcv2: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 5 +---- pkg/services/object/put/v2/service.go | 30 +++------------------------ 2 files changed, 4 insertions(+), 31 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 43d68bdcd..37dcc7b3e 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -327,10 +327,7 @@ func createPutSvc(c *cfg, keyStorage *util.KeyStorage) *putsvc.Service { } func createPutSvcV2(sPut *putsvc.Service, keyStorage *util.KeyStorage) *putsvcV2.Service { - return putsvcV2.NewService( - putsvcV2.WithInternalService(sPut), - putsvcV2.WithKeyStorage(keyStorage), - ) + return putsvcV2.NewService(sPut, keyStorage) } func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache) *searchsvc.Service { diff --git a/pkg/services/object/put/v2/service.go b/pkg/services/object/put/v2/service.go index 78655edc7..db902ae59 100644 --- a/pkg/services/object/put/v2/service.go +++ b/pkg/services/object/put/v2/service.go @@ -12,27 +12,15 @@ import ( // Service implements Put operation of Object service v2. type Service struct { - *cfg -} - -// Option represents Service constructor option. -type Option func(*cfg) - -type cfg struct { svc *putsvc.Service keyStorage *util.KeyStorage } // NewService constructs Service instance from provided options. -func NewService(opts ...Option) *Service { - c := new(cfg) - - for i := range opts { - opts[i](c) - } - +func NewService(svc *putsvc.Service, ks *util.KeyStorage) *Service { return &Service{ - cfg: c, + svc: svc, + keyStorage: ks, } } @@ -52,15 +40,3 @@ func (s *Service) Put() (object.PutObjectStream, error) { func (s *Service) PutSingle(ctx context.Context, req *objectAPI.PutSingleRequest) (*objectAPI.PutSingleResponse, error) { return s.svc.PutSingle(ctx, req) } - -func WithInternalService(v *putsvc.Service) Option { - return func(c *cfg) { - c.svc = v - } -} - -func WithKeyStorage(ks *util.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = ks - } -} From e8091101c7c2b83d6c1e3a708fc252b3e1f3afe2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 12:53:37 +0300 Subject: [PATCH 0418/1943] [#294] searchsvc: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 16 +++--- pkg/services/object/search/service.go | 72 ++++++++------------------- 2 files changed, 27 insertions(+), 61 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 37dcc7b3e..02336bcbf 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -334,16 +334,14 @@ func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Trav ls := c.cfgObject.cfgLocalStorage.localStorage return searchsvc.New( - searchsvc.WithLogger(c.log), - searchsvc.WithLocalStorageEngine(ls), - searchsvc.WithClientConstructor(coreConstructor), - searchsvc.WithTraverserGenerator( - traverseGen.WithTraverseOptions( - placement.WithoutSuccessTracking(), - ), + ls, + coreConstructor, + traverseGen.WithTraverseOptions( + placement.WithoutSuccessTracking(), ), - searchsvc.WithNetMapSource(c.netMapSource), - searchsvc.WithKeyStorage(keyStorage), + c.netMapSource, + keyStorage, + searchsvc.WithLogger(c.log), ) } diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index 708979d79..16b82a620 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -55,17 +55,28 @@ type cfg struct { keyStore *util.KeyStorage } -func defaultCfg() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - clientConstructor: new(clientConstructorWrapper), - } -} - // New creates, initializes and returns utility serving // Object.Get service requests. -func New(opts ...Option) *Service { - c := defaultCfg() +func New(e *engine.StorageEngine, + cc ClientConstructor, + tg *util.TraverserGenerator, + ns netmap.Source, + ks *util.KeyStorage, + opts ...Option) *Service { + c := &cfg{ + log: &logger.Logger{Logger: zap.L()}, + clientConstructor: &clientConstructorWrapper{ + constructor: cc, + }, + localStorage: &storageEngineWrapper{ + storage: e, + }, + traverserGenerator: (*traverseGeneratorWrapper)(tg), + currentEpochReceiver: &nmSrcWrapper{ + nmSrc: ns, + }, + keyStore: ks, + } for i := range opts { opts[i](c) @@ -82,46 +93,3 @@ func WithLogger(l *logger.Logger) Option { c.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Search service"))} } } - -// WithLocalStorageEngine returns option to set local storage -// instance. -func WithLocalStorageEngine(e *engine.StorageEngine) Option { - return func(c *cfg) { - c.localStorage = &storageEngineWrapper{ - storage: e, - } - } -} - -// WithClientConstructor returns option to set constructor of remote node clients. -func WithClientConstructor(v ClientConstructor) Option { - return func(c *cfg) { - c.clientConstructor.(*clientConstructorWrapper).constructor = v - } -} - -// WithTraverserGenerator returns option to set generator of -// placement traverser to get the objects from containers. -func WithTraverserGenerator(t *util.TraverserGenerator) Option { - return func(c *cfg) { - c.traverserGenerator = (*traverseGeneratorWrapper)(t) - } -} - -// WithNetMapSource returns option to set network -// map storage to receive current network state. -func WithNetMapSource(nmSrc netmap.Source) Option { - return func(c *cfg) { - c.currentEpochReceiver = &nmSrcWrapper{ - nmSrc: nmSrc, - } - } -} - -// WithKeyStorage returns option to set private -// key storage for session tokens and node key. -func WithKeyStorage(store *util.KeyStorage) Option { - return func(c *cfg) { - c.keyStore = store - } -} From c83e7c875f7be3439208758669a81a7298ba28e9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jul 2023 12:56:45 +0300 Subject: [PATCH 0419/1943] [#294] searchsvcv2: Refactor service constructor Pass required deps as args. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 5 +--- pkg/services/object/search/v2/service.go | 36 +++--------------------- 2 files changed, 5 insertions(+), 36 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 02336bcbf..1b9f0c817 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -346,10 +346,7 @@ func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Trav } func createSearchSvcV2(sSearch *searchsvc.Service, keyStorage *util.KeyStorage) *searchsvcV2.Service { - return searchsvcV2.NewService( - searchsvcV2.WithInternalService(sSearch), - searchsvcV2.WithKeyStorage(keyStorage), - ) + return searchsvcV2.NewService(sSearch, keyStorage) } func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, diff --git a/pkg/services/object/search/v2/service.go b/pkg/services/object/search/v2/service.go index 17e1bc7e0..78b72ac79 100644 --- a/pkg/services/object/search/v2/service.go +++ b/pkg/services/object/search/v2/service.go @@ -9,28 +9,15 @@ import ( // Service implements Search operation of Object service v2. type Service struct { - *cfg -} - -// Option represents Service constructor option. -type Option func(*cfg) - -type cfg struct { - svc *searchsvc.Service - + svc *searchsvc.Service keyStorage *objutil.KeyStorage } // NewService constructs Service instance from provided options. -func NewService(opts ...Option) *Service { - c := new(cfg) - - for i := range opts { - opts[i](c) - } - +func NewService(s *searchsvc.Service, ks *objutil.KeyStorage) *Service { return &Service{ - cfg: c, + svc: s, + keyStorage: ks, } } @@ -43,18 +30,3 @@ func (s *Service) Search(req *objectV2.SearchRequest, stream objectSvc.SearchStr return s.svc.Search(stream.Context(), *p) } - -// WithInternalService returns option to set entity -// that handles request payload. -func WithInternalService(v *searchsvc.Service) Option { - return func(c *cfg) { - c.svc = v - } -} - -// WithKeyStorage returns option to set local private key storage. -func WithKeyStorage(ks *objutil.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = ks - } -} From 24eb98889724f58338aeb0874b40de6db1f0cbc5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 10 Jul 2023 12:18:56 +0300 Subject: [PATCH 0420/1943] [#294] deletesvc: Drop cast Signed-off-by: Dmitrii Stepanov --- pkg/services/object/delete/service.go | 6 +++--- pkg/services/object/delete/util.go | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index e208a298b..b74a4c7ba 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -72,9 +72,9 @@ func New(gs *getsvc.Service, opts ...Option) *Service { c := &cfg{ log: &logger.Logger{Logger: zap.L()}, - header: (*headSvcWrapper)(gs), - searcher: (*searchSvcWrapper)(ss), - placer: (*putSvcWrapper)(ps), + header: &headSvcWrapper{s: gs}, + searcher: &searchSvcWrapper{s: ss}, + placer: &putSvcWrapper{s: ps}, netInfo: ni, keyStorage: ks, } diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index b8e8e6324..439abca2b 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -11,11 +11,17 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -type headSvcWrapper getsvc.Service +type headSvcWrapper struct { + s *getsvc.Service +} -type searchSvcWrapper searchsvc.Service +type searchSvcWrapper struct { + s *searchsvc.Service +} -type putSvcWrapper putsvc.Service +type putSvcWrapper struct { + s *putsvc.Service +} type simpleIDWriter struct { ids []oid.ID @@ -30,7 +36,7 @@ func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oi p.WithRawFlag(true) p.WithAddress(addr) - err := (*getsvc.Service)(w).Head(ctx, p) + err := w.s.Head(ctx, p) if err != nil { return nil, err } @@ -94,7 +100,7 @@ func (w *searchSvcWrapper) splitMembers(ctx context.Context, exec *execCtx) ([]o p.WithContainerID(exec.containerID()) p.WithSearchFilters(fs) - err := (*searchsvc.Service)(w).Search(ctx, p) + err := w.s.Search(ctx, p) if err != nil { return nil, err } @@ -109,7 +115,7 @@ func (s *simpleIDWriter) WriteIDs(ids []oid.ID) error { } func (w *putSvcWrapper) put(ctx context.Context, exec *execCtx) (*oid.ID, error) { - streamer, err := (*putsvc.Service)(w).Put() + streamer, err := w.s.Put() if err != nil { return nil, err } From b8bcfac53106332ce10ec04dfe8e889abd2814a8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 12 Jul 2023 10:21:35 +0300 Subject: [PATCH 0421/1943] [#510] treesvc: Fix panic in bearer token processing Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/signature.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 63485a707..b932f6de6 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -110,7 +110,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op } tb = *tbCore.Value - if bt.Impersonate() { + if tableFromBearer && bt.Impersonate() { signer = bt.SigningKeyBytes() } } From 8a9fc2c372576e5d8334088fd8620b2126ea1a41 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 12 Jul 2023 10:26:18 +0300 Subject: [PATCH 0422/1943] [#510] treesvc: Rename `tableFromBearer` to `useBearer` With impersonation, the old name is no longer descriptive. Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/signature.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index b932f6de6..7a466955c 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -84,7 +84,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op return nil } - var tableFromBearer bool + var useBearer bool if len(rawBearer) != 0 { if !basicACL.AllowedBearerRules(op) { s.log.Debug(logs.TreeBearerPresentedButNotAllowedByACL, @@ -92,13 +92,13 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op zap.String("op", op.String()), ) } else { - tableFromBearer = true + useBearer = true } } var tb eacl.Table signer := req.GetSignature().GetKey() - if tableFromBearer && !bt.Impersonate() { + if useBearer && !bt.Impersonate() { if !bearer.ResolveIssuer(*bt).Equals(cnr.Value.Owner()) { return eACLErr(eaclOp, errBearerWrongOwner) } @@ -110,7 +110,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op } tb = *tbCore.Value - if tableFromBearer && bt.Impersonate() { + if useBearer && bt.Impersonate() { signer = bt.SigningKeyBytes() } } From 11027945d8de36cfc44c1b5b3980f17be7cd2dc4 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 13 Jul 2023 12:13:53 +0300 Subject: [PATCH 0423/1943] [#479] writecache: Fix writecache fstree flush premature ctx cancel Signed-off-by: Alejandro Lopez --- pkg/local_object_storage/writecache/flush.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 779bac39a..243be4627 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -37,7 +37,6 @@ const ( // runFlushLoop starts background workers which periodically flush objects to the blobstor. func (c *cache) runFlushLoop() { ctx, cancel := context.WithCancel(context.Background()) - defer cancel() ch := c.closeCh c.wg.Add(1) From 397131b0ea3fc742c03bcae97df89dbad67d183f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Jul 2023 16:43:41 +0300 Subject: [PATCH 0424/1943] [#520] objectcore: Refactor format validator Remove redundant FIXME. Move error to consts. Signed-off-by: Dmitrii Stepanov --- pkg/core/object/fmt.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index abf8f02cb..e6d8174fa 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -64,6 +64,8 @@ var errNoExpirationEpoch = errors.New("missing expiration epoch attribute") var errTombstoneExpiration = errors.New("tombstone body and header contain different expiration values") +var errMissingSignature = errors.New("missing signature") + func defaultCfg() *cfg { return new(cfg) } @@ -135,8 +137,7 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *objectSDK.Object, u func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { sig := obj.Signature() if sig == nil { - // TODO(@cthulhu-rider): #468 use "const" error - return errors.New("missing signature") + return errMissingSignature } var sigV2 refs.Signature @@ -157,8 +158,6 @@ func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { return v.checkOwnerKey(*obj.OwnerID(), key) } - // FIXME: #1159 perform token verification - return nil } From 486287c2f7802bfa5d6436df566f4fac4942f57e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 14 Jul 2023 16:08:01 +0300 Subject: [PATCH 0425/1943] [#524] cli: Add impersonate flag for bearer token creation Signed-off-by: Denis Kirillov --- cmd/frostfs-cli/modules/bearer/create.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/bearer/create.go b/cmd/frostfs-cli/modules/bearer/create.go index 2f1623d9b..a05de9a76 100644 --- a/cmd/frostfs-cli/modules/bearer/create.go +++ b/cmd/frostfs-cli/modules/bearer/create.go @@ -24,6 +24,7 @@ const ( ownerFlag = "owner" outFlag = "out" jsonFlag = commonflags.JSON + impersonateFlag = "impersonate" ) var createCmd = &cobra.Command{ @@ -39,15 +40,18 @@ is set to current epoch + n. } func init() { - createCmd.Flags().StringP(eaclFlag, "e", "", "Path to the extended ACL table") + createCmd.Flags().StringP(eaclFlag, "e", "", "Path to the extended ACL table (mutually exclusive with --impersonate flag)") createCmd.Flags().StringP(issuedAtFlag, "i", "", "Epoch to issue token at") createCmd.Flags().StringP(notValidBeforeFlag, "n", "", "Not valid before epoch") createCmd.Flags().StringP(commonflags.ExpireAt, "x", "", "The last active epoch for the token") createCmd.Flags().StringP(ownerFlag, "o", "", "Token owner") createCmd.Flags().String(outFlag, "", "File to write token to") createCmd.Flags().Bool(jsonFlag, false, "Output token in JSON") + createCmd.Flags().Bool(impersonateFlag, false, "Mark token as impersonate to consider the token signer as the request owner (mutually exclusive with --eacl flag)") createCmd.Flags().StringP(commonflags.RPC, commonflags.RPCShorthand, commonflags.RPCDefault, commonflags.RPCUsage) + createCmd.MarkFlagsMutuallyExclusive(eaclFlag, impersonateFlag) + _ = cobra.MarkFlagFilename(createCmd.Flags(), eaclFlag) _ = cobra.MarkFlagRequired(createCmd.Flags(), issuedAtFlag) @@ -101,6 +105,9 @@ func createToken(cmd *cobra.Command, _ []string) { b.SetIat(iat) b.ForUser(ownerID) + impersonate, _ := cmd.Flags().GetBool(impersonateFlag) + b.SetImpersonate(impersonate) + eaclPath, _ := cmd.Flags().GetString(eaclFlag) if eaclPath != "" { table := eaclSDK.NewTable() From d8e37a827f5361596d6e4262c3864d745de590aa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 12 Jul 2023 15:37:35 +0300 Subject: [PATCH 0426/1943] [#497] config: Add examples and unit tests Add examples and unit tests for tree.authorized_keys section. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config/tree/config_test.go | 11 +++++++++++ config/example/node.env | 1 + config/example/node.json | 6 +++++- config/example/node.yaml | 3 +++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config/tree/config_test.go b/cmd/frostfs-node/config/tree/config_test.go index a39aa4553..898f7e715 100644 --- a/cmd/frostfs-node/config/tree/config_test.go +++ b/cmd/frostfs-node/config/tree/config_test.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" ) @@ -21,10 +22,19 @@ func TestTreeSection(t *testing.T) { require.Equal(t, 0, treeSec.ReplicationChannelCapacity()) require.Equal(t, 0, treeSec.ReplicationWorkerCount()) require.Equal(t, time.Duration(0), treeSec.ReplicationTimeout()) + require.Equal(t, 0, len(treeSec.AuthorizedKeys())) }) const path = "../../../../config/example/node" + var expectedKeys keys.PublicKeys + key, err := keys.NewPublicKeyFromString("0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0") + require.NoError(t, err) + expectedKeys = append(expectedKeys, key) + key, err = keys.NewPublicKeyFromString("02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56") + require.NoError(t, err) + expectedKeys = append(expectedKeys, key) + var fileConfigTest = func(c *config.Config) { treeSec := treeconfig.Tree(c) @@ -34,6 +44,7 @@ func TestTreeSection(t *testing.T) { require.Equal(t, 32, treeSec.ReplicationWorkerCount()) require.Equal(t, 5*time.Second, treeSec.ReplicationTimeout()) require.Equal(t, time.Hour, treeSec.SyncInterval()) + require.Equal(t, expectedKeys, treeSec.AuthorizedKeys()) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/config/example/node.env b/config/example/node.env index 143bf0388..089021767 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -36,6 +36,7 @@ FROSTFS_TREE_REPLICATION_CHANNEL_CAPACITY=32 FROSTFS_TREE_REPLICATION_WORKER_COUNT=32 FROSTFS_TREE_REPLICATION_TIMEOUT=5s FROSTFS_TREE_SYNC_INTERVAL=1h +FROSTFS_TREE_AUTHORIZED_KEYS="0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0 02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56" # gRPC section ## 0 server diff --git a/config/example/node.json b/config/example/node.json index 04aabdd42..e4b85bc81 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -75,7 +75,11 @@ "replication_channel_capacity": 32, "replication_worker_count": 32, "replication_timeout": "5s", - "sync_interval": "1h" + "sync_interval": "1h", + "authorized_keys": [ + "0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0", + "02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56" + ] }, "control": { "authorized_keys": [ diff --git a/config/example/node.yaml b/config/example/node.yaml index bc665a688..897f4e15b 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -62,6 +62,9 @@ tree: replication_channel_capacity: 32 replication_timeout: 5s sync_interval: 1h + authorized_keys: # list of hex-encoded public keys that have rights to use the Tree Service with frostfs-cli + - 0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0 + - 02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56 control: authorized_keys: # list of hex-encoded public keys that have rights to use the Control Service From a0c7045f29a64fec1ef60178cbd3bcbf1100c714 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Jul 2023 11:58:13 +0300 Subject: [PATCH 0427/1943] [#512] cli: Refactor UX for bearer create command Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/bearer/create.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-cli/modules/bearer/create.go b/cmd/frostfs-cli/modules/bearer/create.go index a05de9a76..b85115047 100644 --- a/cmd/frostfs-cli/modules/bearer/create.go +++ b/cmd/frostfs-cli/modules/bearer/create.go @@ -41,8 +41,8 @@ is set to current epoch + n. func init() { createCmd.Flags().StringP(eaclFlag, "e", "", "Path to the extended ACL table (mutually exclusive with --impersonate flag)") - createCmd.Flags().StringP(issuedAtFlag, "i", "", "Epoch to issue token at") - createCmd.Flags().StringP(notValidBeforeFlag, "n", "", "Not valid before epoch") + createCmd.Flags().StringP(issuedAtFlag, "i", "+0", "Epoch to issue token at") + createCmd.Flags().StringP(notValidBeforeFlag, "n", "+0", "Not valid before epoch") createCmd.Flags().StringP(commonflags.ExpireAt, "x", "", "The last active epoch for the token") createCmd.Flags().StringP(ownerFlag, "o", "", "Token owner") createCmd.Flags().String(outFlag, "", "File to write token to") @@ -54,8 +54,6 @@ func init() { _ = cobra.MarkFlagFilename(createCmd.Flags(), eaclFlag) - _ = cobra.MarkFlagRequired(createCmd.Flags(), issuedAtFlag) - _ = cobra.MarkFlagRequired(createCmd.Flags(), notValidBeforeFlag) _ = cobra.MarkFlagRequired(createCmd.Flags(), commonflags.ExpireAt) _ = cobra.MarkFlagRequired(createCmd.Flags(), ownerFlag) _ = cobra.MarkFlagRequired(createCmd.Flags(), outFlag) @@ -72,10 +70,14 @@ func createToken(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "can't parse --"+notValidBeforeFlag+" flag: %w", err) if iatRelative || expRelative || nvbRelative { + endpoint, _ := cmd.Flags().GetString(commonflags.RPC) + if len(endpoint) == 0 { + commonCmd.ExitOnErr(cmd, "can't fetch current epoch: %w", fmt.Errorf("'%s' flag value must be specified", commonflags.RPC)) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - endpoint, _ := cmd.Flags().GetString(commonflags.RPC) currEpoch, err := internalclient.GetCurrentEpoch(ctx, cmd, endpoint) commonCmd.ExitOnErr(cmd, "can't fetch current epoch: %w", err) From 3e8de14e7d921146f0fdb40307d9cc7fccd8c8e2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 11 Jul 2023 16:13:03 +0300 Subject: [PATCH 0428/1943] [#382] evacuate: Fix unit tests Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate_test.go | 2 +- pkg/local_object_storage/engine/put.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 13ac59f24..b642ea065 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -28,7 +28,7 @@ import ( func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEngine, []*shard.ID, []*objectSDK.Object) { dir := t.TempDir() - te := testNewEngine(t, WithShardPoolSize(1)). + te := testNewEngine(t). setShardsNumOpts(t, shardNum, func(id int) []shard.Option { return []shard.Option{ shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 98c4504e6..2f96b8296 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -154,6 +154,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, putSuccess = true }); err != nil { + e.log.Warn(logs.EngineCouldNotPutObjectToShard, zap.Error(err)) close(exitCh) } From 3b66f98f27755e33c00ed845c41eb8996ef55a0e Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 13 Jul 2023 15:04:29 +0300 Subject: [PATCH 0429/1943] [#519] Use Address.Equals in policer tests Signed-off-by: Alejandro Lopez --- pkg/services/policer/policer_test.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 42428df23..56dab413e 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -220,11 +220,11 @@ func TestProcessObject(t *testing.T) { WithRemoteObjectHeaderFunc(headFn), WithBuryFunc(buryFn), WithRedundantCopyCallback(func(_ context.Context, a oid.Address) { - require.True(t, eqAddr(a, addr), "unexpected redundant copy callback: a=%v", a) + require.True(t, a.Equals(addr), "unexpected redundant copy callback: a=%v", a) gotRemoveRedundant = true }), WithReplicator(replicatorFunc(func(_ context.Context, task replicator.Task, res replicator.TaskResult) { - require.True(t, eqAddr(task.Addr, addr), "unexpected replicator task: %+v", task) + require.True(t, task.Addr.Equals(addr), "unexpected replicator task: %+v", task) for _, node := range task.Nodes { gotReplicateTo = append(gotReplicateTo, int(node.PublicKey()[0])) } @@ -303,11 +303,6 @@ func TestIteratorContract(t *testing.T) { }, it.calls) } -// TODO(https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/101) -func eqAddr(a, b oid.Address) bool { - return a.Container().Equals(b.Container()) && a.Object().Equals(b.Object()) -} - type nextResult struct { objs []objectcore.AddressWithType err error From b2487e8cc516c789e4c53e09859450f310689abb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Jul 2023 16:22:38 +0300 Subject: [PATCH 0430/1943] [#516] node: Do not bootstrap if node is online candidate Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 ++- cmd/frostfs-node/netmap.go | 19 +++++++++++++------ internal/logs/logs.go | 1 + 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 5d7adab29..d22d68ba0 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -337,7 +337,8 @@ type internals struct { apiVersion version.Version healthStatus *atomic.Int32 // is node under maintenance - isMaintenance atomic.Bool + isMaintenance atomic.Bool + isOnlineCandidate bool } // starts node's maintenance. diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index b7b09a00b..96b866b11 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -227,6 +227,10 @@ func bootstrapNode(c *cfg) { c.log.Info(logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) return } + if c.isOnlineCandidate { + c.log.Info(logs.NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap) + return + } err := c.bootstrap() fatalOnErrDetails("bootstrap error", err) } @@ -258,7 +262,8 @@ func initNetmapState(c *cfg) { epoch, err := c.cfgNetmap.wrapper.Epoch() fatalOnErrDetails("could not initialize current epoch number", err) - ni, err := c.netmapInitLocalNodeState(epoch) + var ni *netmapSDK.NodeInfo + ni, c.isOnlineCandidate, err = c.netmapInitLocalNodeState(epoch) fatalOnErrDetails("could not init network state", err) stateWord := nodeState(ni) @@ -291,27 +296,29 @@ func nodeState(ni *netmapSDK.NodeInfo) string { return "undefined" } -func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { +func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, bool, error) { nmNodes, err := c.cfgNetmap.wrapper.GetCandidates() if err != nil { - return nil, err + return nil, false, err } var candidate *netmapSDK.NodeInfo + isOnlineCandidate := false for i := range nmNodes { if bytes.Equal(nmNodes[i].PublicKey(), c.binPublicKey) { candidate = &nmNodes[i] + isOnlineCandidate = candidate.IsOnline() break } } node, err := c.netmapLocalNodeState(epoch) if err != nil { - return nil, err + return nil, false, err } if candidate == nil { - return node, nil + return node, false, nil } nmState := nodeState(node) @@ -323,7 +330,7 @@ func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error zap.String("netmap", nmState), zap.String("candidate", candidateState)) } - return candidate, nil + return candidate, isOnlineCandidate, nil } func (c *cfg) netmapLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 2ba9e6e4b..82e04a16d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -493,4 +493,5 @@ const ( ShardDeleteCantDeleteFromWriteCache = "can't delete object from write cache" FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled" + NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap = "the node is already in candidate list with online state, skip initial bootstrap" ) From 8966dd8e35c738306ca7e99accf324e97cb27601 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 12 Jul 2023 17:47:42 +0300 Subject: [PATCH 0431/1943] [#463] putsvc: Use PutSingle RPC for remote target Signed-off-by: Dmitrii Stepanov --- pkg/core/client/client.go | 1 + pkg/network/cache/multi.go | 9 ++++ pkg/services/object/internal/client/client.go | 48 +++++++++++++++++++ pkg/services/object/put/remote.go | 20 ++++++++ 4 files changed, 78 insertions(+) diff --git a/pkg/core/client/client.go b/pkg/core/client/client.go index 422ca1a1a..8c92901f2 100644 --- a/pkg/core/client/client.go +++ b/pkg/core/client/client.go @@ -13,6 +13,7 @@ import ( type Client interface { ContainerAnnounceUsedSpace(context.Context, client.PrmAnnounceSpace) (*client.ResAnnounceSpace, error) ObjectPutInit(context.Context, client.PrmObjectPutInit) (client.ObjectWriter, error) + ObjectPutSingle(context.Context, client.PrmObjectPutSingle) (*client.ResObjectPutSingle, error) ObjectDelete(context.Context, client.PrmObjectDelete) (*client.ResObjectDelete, error) ObjectGetInit(context.Context, client.PrmObjectGet) (*client.ObjectReader, error) ObjectHead(context.Context, client.PrmObjectHead) (*client.ResObjectHead, error) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 18155849b..98d2f33e7 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -228,6 +228,15 @@ func (x *multiClient) ObjectPutInit(ctx context.Context, p client.PrmObjectPutIn return } +func (x *multiClient) ObjectPutSingle(ctx context.Context, p client.PrmObjectPutSingle) (res *client.ResObjectPutSingle, err error) { + err = x.iterateClients(ctx, func(c clientcore.Client) error { + res, err = c.ObjectPutSingle(ctx, p) + return err + }) + + return +} + func (x *multiClient) ContainerAnnounceUsedSpace(ctx context.Context, prm client.PrmAnnounceSpace) (res *client.ResAnnounceSpace, err error) { err = x.iterateClients(ctx, func(c clientcore.Client) error { res, err = c.ContainerAnnounceUsedSpace(ctx, prm) diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index cfab77efe..73f4ff7c4 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -449,6 +449,54 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { }, nil } +// PutObjectSingle saves the object in local storage of the remote node with PutSingle RPC. +// +// Client and key must be set. +// +// Returns any error which prevented the operation from completing correctly in error return. +func PutObjectSingle(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "client.PutObjectSingle") + defer span.End() + + objID, isSet := prm.obj.ID() + if !isSet { + return nil, errors.New("missing object id") + } + + var prmCli client.PrmObjectPutSingle + + prmCli.ExecuteLocal() + + if prm.key != nil { + prmCli.UseKey(prm.key) + } + + if prm.tokenSession != nil { + prmCli.WithinSession(*prm.tokenSession) + } + + if prm.tokenBearer != nil { + prmCli.WithBearerToken(*prm.tokenBearer) + } + + prmCli.WithXHeaders(prm.xHeaders...) + prmCli.SetObject(prm.obj.ToV2()) + + res, err := prm.cli.ObjectPutSingle(ctx, prmCli) + if err != nil { + ReportError(prm.cli, err) + return nil, fmt.Errorf("put single object on client: %w", err) + } + + if err = apistatus.ErrFromStatus(res.Status()); err != nil { + return nil, fmt.Errorf("put single object via client: %w", err) + } + + return &PutObjectRes{ + id: objID, + }, nil +} + // SearchObjectsPrm groups parameters of SearchObjects operation. type SearchObjectsPrm struct { readPrmCommon diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/put/remote.go index a5b3f643c..8116243ec 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/put/remote.go @@ -13,6 +13,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type remoteTarget struct { @@ -63,6 +65,15 @@ func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifier prm.SetXHeaders(t.commonPrm.XHeaders()) prm.SetObject(t.obj) + res, err := t.putSingle(ctx, prm) + if status.Code(err) != codes.Unimplemented { + return res, err + } + + return t.putStream(ctx, prm) +} + +func (t *remoteTarget) putStream(ctx context.Context, prm internalclient.PutObjectPrm) (*transformer.AccessIdentifiers, error) { res, err := internalclient.PutObject(ctx, prm) if err != nil { return nil, fmt.Errorf("(%T) could not put object to %s: %w", t, t.nodeInfo.AddressGroup(), err) @@ -71,6 +82,15 @@ func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifier return &transformer.AccessIdentifiers{SelfID: res.ID()}, nil } +func (t *remoteTarget) putSingle(ctx context.Context, prm internalclient.PutObjectPrm) (*transformer.AccessIdentifiers, error) { + res, err := internalclient.PutObjectSingle(ctx, prm) + if err != nil { + return nil, fmt.Errorf("(%T) could not put single object to %s: %w", t, t.nodeInfo.AddressGroup(), err) + } + + return &transformer.AccessIdentifiers{SelfID: res.ID()}, nil +} + // NewRemoteSender creates, initializes and returns new RemoteSender instance. func NewRemoteSender(keyStorage *util.KeyStorage, cons ClientConstructor) *RemoteSender { return &RemoteSender{ From 8b78db74bcfbf0117dedd6e898da4f95f0e191e6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Jul 2023 14:58:15 +0300 Subject: [PATCH 0432/1943] [#463] replicator: Add tracing span Signed-off-by: Dmitrii Stepanov --- pkg/services/replicator/process.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index a54668e12..16bcec9c5 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -6,7 +6,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -28,6 +31,13 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) ) }() + ctx, span := tracing.StartSpanFromContext(ctx, "Replicator.HandleTask", + trace.WithAttributes( + attribute.Stringer("address", task.Addr), + attribute.Int64("number_of_copies", int64(task.NumCopies)), + )) + defer span.End() + if task.Obj == nil { var err error task.Obj, err = engine.Get(ctx, p.localStorage, task.Addr) From f0355a453e3e43250e454b5ec544a73c3cf4f965 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Jul 2023 15:08:46 +0300 Subject: [PATCH 0433/1943] [#463] policer: Remove capacity rebalance logic Current implementation has some quirks. For example, using only half of object.put.pool_size_remote threads tells replicator that is node is 50% loaded, but in reality we could be putting lot's of big objects. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 7 ------- cmd/frostfs-node/object.go | 1 - internal/logs/logs.go | 1 - pkg/services/policer/option.go | 15 --------------- pkg/services/policer/policer_test.go | 7 ------- pkg/services/policer/process.go | 25 ------------------------- 6 files changed, 56 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d22d68ba0..267a05aa8 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -997,13 +997,6 @@ func (c *cfg) needBootstrap() bool { return c.cfgNetmap.needBootstrap } -// ObjectServiceLoad implements system loader interface for policer component. -// It is calculated as size/capacity ratio of "remote object put" worker. -// Returns float value between 0.0 and 1.0. -func (c *cfg) ObjectServiceLoad() float64 { - return float64(c.cfgObject.pool.putRemote.Running()) / float64(c.cfgObject.pool.putRemoteCapacity) -} - type dCmp struct { name string reloadFunc func() error diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 1b9f0c817..84411d31b 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -259,7 +259,6 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl }), policer.WithMaxCapacity(c.cfgObject.pool.replicatorPoolSize), policer.WithPool(c.cfgObject.pool.replication), - policer.WithNodeLoader(c), ) c.workers = append(c.workers, worker{ diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 82e04a16d..a2ff8dcb9 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -52,7 +52,6 @@ const ( PolicerRoutineStopped = "routine stopped" // Info in ../node/pkg/services/policer/process.go PolicerFailureAtObjectSelectForReplication = "failure at object select for replication" // Warn in ../node/pkg/services/policer/process.go PolicerPoolSubmission = "pool submission" // Warn in ../node/pkg/services/policer/process.go - PolicerTuneReplicationCapacity = "tune replication capacity" // Debug in ../node/pkg/services/policer/process.go ReplicatorFinishWork = "finish work" // Debug in ../node/pkg/services/replicator/process.go ReplicatorCouldNotGetObjectFromLocalStorage = "could not get object from local storage" // Error in ../node/pkg/services/replicator/process.go ReplicatorCouldNotReplicateObject = "could not replicate object" // Error in ../node/pkg/services/replicator/process.go diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 4194353ca..5058b026b 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -41,12 +41,6 @@ type Replicator interface { // RemoteObjectHeaderFunc is the function to obtain HEAD info from a specific remote node. type RemoteObjectHeaderFunc func(context.Context, netmapSDK.NodeInfo, oid.Address) (*objectSDK.Object, error) -// NodeLoader provides application load statistics. -type nodeLoader interface { - // ObjectServiceLoad returns object service load value in [0:1] range. - ObjectServiceLoad() float64 -} - type cfg struct { headTimeout time.Duration @@ -70,8 +64,6 @@ type cfg struct { taskPool *ants.Pool - loader nodeLoader - maxCapacity int batchSize, cacheSize uint32 @@ -178,10 +170,3 @@ func WithPool(p *ants.Pool) Option { c.taskPool = p } } - -// WithNodeLoader returns option to set FrostFS node load source. -func WithNodeLoader(l nodeLoader) Option { - return func(c *cfg) { - c.loader = l - } -} diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 56dab413e..c0aeac515 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -52,7 +52,6 @@ func TestBuryObjectWithoutContainer(t *testing.T) { WithContainerSource(containerSrcFunc(containerSrc)), WithBuryFunc(buryFn), WithPool(pool), - WithNodeLoader(constNodeLoader(0)), ) ctx, cancel := context.WithCancel(context.Background()) @@ -279,7 +278,6 @@ func TestIteratorContract(t *testing.T) { WithContainerSource(containerSrcFunc(containerSrc)), WithBuryFunc(buryFn), WithPool(pool), - WithNodeLoader(constNodeLoader(0)), func(c *cfg) { c.sleepDuration = time.Millisecond }, @@ -372,11 +370,6 @@ type announcedKeysFunc func([]byte) bool func (f announcedKeysFunc) IsLocalKey(k []byte) bool { return f(k) } -// constNodeLoader is a nodeLoader that always returns a fixed value. -type constNodeLoader float64 - -func (f constNodeLoader) ObjectServiceLoad() float64 { return float64(f) } - // replicatorFunc is a Replicator backed by a function. type replicatorFunc func(context.Context, replicator.Task, replicator.TaskResult) diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 3b54bf929..1f61c69f4 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -11,7 +11,6 @@ import ( ) func (p *Policer) Run(ctx context.Context) { - go p.poolCapacityWorker(ctx) p.shardPolicyWorker(ctx) p.log.Info(logs.PolicerRoutineStopped) } @@ -65,27 +64,3 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { } } } - -func (p *Policer) poolCapacityWorker(ctx context.Context) { - ticker := time.NewTicker(p.rebalanceFreq) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - frostfsSysLoad := p.loader.ObjectServiceLoad() - newCapacity := int((1.0 - frostfsSysLoad) * float64(p.maxCapacity)) - if newCapacity == 0 { - newCapacity++ - } - - if p.taskPool.Cap() != newCapacity { - p.taskPool.Tune(newCapacity) - p.log.Debug(logs.PolicerTuneReplicationCapacity, - zap.Float64("system_load", frostfsSysLoad), - zap.Int("new_capacity", newCapacity)) - } - } - } -} From 4680087711f4fd8a42669a8e8e7eef53b5b8323d Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Mon, 17 Jul 2023 13:59:37 +0300 Subject: [PATCH 0434/1943] [#527] Add support for select-filter expressions in policy playground Signed-off-by: Alejandro Lopez --- .../modules/container/policy_playground.go | 33 ++++++++++++++----- go.mod | 2 +- go.sum | 4 +-- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 65255f9c2..97e30a3cd 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -82,15 +82,32 @@ func (repl *policyPlaygroundREPL) handleRemove(args []string) error { } func (repl *policyPlaygroundREPL) handleEval(args []string) error { - policyStr := strings.Join(args, " ") - placementPolicy, err := parseContainerPolicy(repl.cmd, policyStr) - if err != nil { - return fmt.Errorf("parsing placement policy: %v", err) - } + policyStr := strings.TrimSpace(strings.Join(args, " ")) + var nodes [][]netmap.NodeInfo nm := repl.netMap() - nodes, err := nm.ContainerNodes(*placementPolicy, nil) - if err != nil { - return fmt.Errorf("building container nodes: %v", err) + + if strings.HasPrefix(policyStr, "CBF") || strings.HasPrefix(policyStr, "SELECT") || strings.HasPrefix(policyStr, "FILTER") { + // Assume that the input is a partial SELECT-FILTER expression. + // Full inline policies always start with UNIQUE or REP keywords, + // or different prefixes when it's the case of an external file. + sfExpr, err := netmap.DecodeSelectFilterString(policyStr) + if err != nil { + return fmt.Errorf("parsing select-filter expression: %v", err) + } + nodes, err = nm.SelectFilterNodes(sfExpr) + if err != nil { + return fmt.Errorf("building select-filter nodes: %v", err) + } + } else { + // Assume that the input is a full policy or input file otherwise. + placementPolicy, err := parseContainerPolicy(repl.cmd, policyStr) + if err != nil { + return fmt.Errorf("parsing placement policy: %v", err) + } + nodes, err = nm.ContainerNodes(*placementPolicy, nil) + if err != nil { + return fmt.Errorf("building container nodes: %v", err) + } } for i, ns := range nodes { var ids []string diff --git a/go.mod b/go.mod index e2735b223..7378621c4 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230711142135-998fe1a7ab31 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230717102948-b91f9d8c7910 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index ddbc77633..2ea573e1e 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230711142135-998fe1a7ab31 h1:PSHVtyD3vw2NTbdWBUz0Wql1WH42nSCP+4j6PmIgZ8Y= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230711142135-998fe1a7ab31/go.mod h1:MlLlhZb4qbLhUScDngfujUZKhs0/2YKW9D8LTY3BApY= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230717102948-b91f9d8c7910 h1:UmTLvK7/G8KbOOBr+u577fO7C1JNDEOnG0CB5kRBT3Y= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230717102948-b91f9d8c7910/go.mod h1:MlLlhZb4qbLhUScDngfujUZKhs0/2YKW9D8LTY3BApY= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 64299755843b41b7ea19ca53b40de98e9c164541 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 17 Jul 2023 18:10:16 +0300 Subject: [PATCH 0435/1943] [#511] docs: Remove GitHub mentions from CONTRIBUTING.md Signed-off-by: Dmitrii Stepanov --- CONTRIBUTING.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce617f7f6..53ff7c8df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,8 +3,8 @@ First, thank you for contributing! We love and encourage pull requests from everyone. Please follow the guidelines: -- Check the open [issues](https://github.com/TrueCloudLab/frostfs-node/issues) and - [pull requests](https://github.com/TrueCloudLab/frostfs-node/pulls) for existing +- Check the open [issues](https://git.frostfs.info/TrueCloudLab/frostfs-node/issues) and + [pull requests](https://git.frostfs.info/TrueCloudLab/frostfs-node/pulls) for existing discussions. - Open an issue first, to discuss a new feature or enhancement. @@ -27,19 +27,19 @@ Start by forking the `frostfs-node` repository, make changes in a branch and the send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: -### Set up your GitHub Repository -Fork [FrostFS node upstream](https://github.com/TrueCloudLab/frostfs-node/fork) source +### Set up your Forgejo repository +Fork [FrostFS node upstream](https://git.frostfs.info/TrueCloudLab/frostfs-node) source repository to your own personal repository. Copy the URL of your fork (you will need it for the `git clone` command below). ```sh -$ git clone https://github.com/TrueCloudLab/frostfs-node +$ git clone https://git.frostfs.info/TrueCloudLab/frostfs-node ``` ### Set up git remote as ``upstream`` ```sh $ cd frostfs-node -$ git remote add upstream https://github.com/TrueCloudLab/frostfs-node +$ git remote add upstream https://git.frostfs.info/TrueCloudLab/frostfs-node $ git fetch upstream $ git merge upstream/master ... @@ -58,7 +58,7 @@ $ git checkout -b feature/123-something_awesome After your code changes, make sure - To add test cases for the new code. -- To run `make lint` +- To run `make lint` and `make staticcheck-run` - To squash your commits into a single commit or a series of logically separated commits run `git rebase -i`. It's okay to force update your pull request. - To run `make test` and `make all` completes. @@ -89,8 +89,8 @@ $ git push origin feature/123-something_awesome ``` ### Create a Pull Request -Pull requests can be created via GitHub. Refer to [this -document](https://help.github.com/articles/creating-a-pull-request/) for +Pull requests can be created via Forgejo. Refer to [this +document](https://docs.codeberg.org/collaborating/pull-requests-and-git-flow/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged. From a9d04ba86f2edc375ca0c688ebed0658c7728ff3 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Mon, 10 Jul 2023 11:03:02 +0300 Subject: [PATCH 0436/1943] [#244] Remove --local-dump from frostfs-adm config Signed-off-by: Alejandro Lopez --- cmd/frostfs-adm/internal/modules/morph/initialize.go | 6 +++--- cmd/frostfs-adm/internal/modules/morph/local_client.go | 3 +-- cmd/frostfs-adm/internal/modules/morph/root.go | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 9eb867faa..dec1fba20 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -210,11 +210,11 @@ func validateInit(cmd *cobra.Command) error { func createClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) (Client, error) { var c Client var err error - if v.GetString(localDumpFlag) != "" { - if v.GetString(endpointFlag) != "" { + if ldf := cmd.Flags().Lookup(localDumpFlag); ldf != nil && ldf.Changed { + if cmd.Flags().Changed(endpointFlag) { return nil, fmt.Errorf("`%s` and `%s` flags are mutually exclusive", endpointFlag, localDumpFlag) } - c, err = newLocalClient(cmd, v, wallets) + c, err = newLocalClient(cmd, v, wallets, ldf.Value.String()) } else { c, err = getN3Client(v) } diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index 45d09c387..0367f7479 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -51,7 +51,7 @@ type localClient struct { maxGasInvoke int64 } -func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) (*localClient, error) { +func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet, dumpPath string) (*localClient, error) { cfg, err := config.LoadFile(v.GetString(protoConfigPath)) if err != nil { return nil, err @@ -87,7 +87,6 @@ func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet go bc.Run() - dumpPath := v.GetString(localDumpFlag) if cmd.Name() != "init" { f, err := os.OpenFile(dumpPath, os.O_RDONLY, 0600) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 431be125c..6a9e5b9c1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -77,7 +77,6 @@ var ( _ = viper.BindPFlag(containerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) _ = viper.BindPFlag(withdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) _ = viper.BindPFlag(protoConfigPath, cmd.Flags().Lookup(protoConfigPath)) - _ = viper.BindPFlag(localDumpFlag, cmd.Flags().Lookup(localDumpFlag)) }, RunE: initializeSideChainCmd, } From 24dffdac6fa4941b48e38dbf9c6e3d9ba50790d4 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 14 Jul 2023 11:11:45 +0300 Subject: [PATCH 0437/1943] [#521] cli: Add netmap load command to policy playground Signed-off-by: Alejandro Lopez --- .../modules/container/policy_playground.go | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 97e30a3cd..7e9d72a08 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -3,6 +3,7 @@ package container import ( "bufio" "encoding/hex" + "encoding/json" "fmt" "io" "os" @@ -39,7 +40,7 @@ func (repl *policyPlaygroundREPL) handleLs(args []string) error { for id, node := range repl.nodes { var attrs []string node.IterateAttributes(func(k, v string) { - attrs = append(attrs, fmt.Sprintf("%s:%s", k, v)) + attrs = append(attrs, fmt.Sprintf("%s:%q", k, v)) }) fmt.Printf("\t%2d: id=%s attrs={%v}\n", i, id, strings.Join(attrs, " ")) i++ @@ -69,6 +70,40 @@ func (repl *policyPlaygroundREPL) handleAdd(args []string) error { return nil } +func (repl *policyPlaygroundREPL) handleLoad(args []string) error { + if len(args) != 1 { + return fmt.Errorf("too few arguments for command 'add': got %d, want 1", len(args)) + } + + jsonNetmap := map[string]map[string]string{} + + b, err := os.ReadFile(args[0]) + if err != nil { + return fmt.Errorf("reading netmap file %q: %v", args[0], err) + } + + if err := json.Unmarshal(b, &jsonNetmap); err != nil { + return fmt.Errorf("decoding json netmap: %v", err) + } + + repl.nodes = make(map[string]netmap.NodeInfo) + for id, attrs := range jsonNetmap { + key, err := hex.DecodeString(id) + if err != nil { + return fmt.Errorf("node id must be a hex string: got %q: %v", id, err) + } + + node := repl.nodes[id] + node.SetPublicKey(key) + for k, v := range attrs { + node.SetAttribute(k, v) + } + repl.nodes[id] = node + } + + return nil +} + func (repl *policyPlaygroundREPL) handleRemove(args []string) error { if len(args) == 0 { return fmt.Errorf("too few arguments for command 'remove': got %d, want >0", len(args)) @@ -150,6 +185,7 @@ func (repl *policyPlaygroundREPL) run() error { cmdHandlers := map[string]func([]string) error{ "ls": repl.handleLs, "add": repl.handleAdd, + "load": repl.handleLoad, "remove": repl.handleRemove, "eval": repl.handleEval, } From 57e7fb5ccfec50de5675ff2a0d9f421428004a7b Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Mon, 17 Jul 2023 10:23:51 +0300 Subject: [PATCH 0438/1943] [#521] cli: Add common aliases to policy playground commands Signed-off-by: Alejandro Lopez --- cmd/frostfs-cli/modules/container/policy_playground.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 7e9d72a08..1ac41f08c 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -183,10 +183,12 @@ func (repl *policyPlaygroundREPL) run() error { } cmdHandlers := map[string]func([]string) error{ + "list": repl.handleLs, "ls": repl.handleLs, "add": repl.handleAdd, "load": repl.handleLoad, "remove": repl.handleRemove, + "rm": repl.handleRemove, "eval": repl.handleEval, } for reader := bufio.NewReader(os.Stdin); ; { From 94df541426e0433653175b0f9c1f1e97da68ec70 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 19 Jul 2023 16:11:56 +0300 Subject: [PATCH 0439/1943] [#530] go.mod: Update frostfs-sdk-go and frostfs-api-go versions Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 7378621c4..2e7d8fd56 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230719100335-582d94c81c74 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230717102948-b91f9d8c7910 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230719130356-5defed4ab435 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 2ea573e1e..c8737ebdc 100644 --- a/go.sum +++ b/go.sum @@ -36,16 +36,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe h1:SB102RiEg+4h9qcwyG97zHBtwduMRbedbtkwRDVSps8= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230704092742-285516a94ebe/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230719100335-582d94c81c74 h1:xgeSzZP40SbMMVGfG1V7xrC8m7DS6llZ9kkjO9L8+jY= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230719100335-582d94c81c74/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a h1:EeaOunifY30ATPzOo1F32vUDW299UXDHYukZrIVxBZ0= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230717102948-b91f9d8c7910 h1:UmTLvK7/G8KbOOBr+u577fO7C1JNDEOnG0CB5kRBT3Y= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230717102948-b91f9d8c7910/go.mod h1:MlLlhZb4qbLhUScDngfujUZKhs0/2YKW9D8LTY3BApY= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230719130356-5defed4ab435 h1:N6T2OF5nyxizGFb3bV6GaSn5HsTurJG7RvF25SKHg0A= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230719130356-5defed4ab435/go.mod h1:3GPr75siKbjGwUUz09nXfd98RBMozlDUdtKmtgxcCG8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From b4e72a2dfdede508443e301d12bf619987f428ba Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 11 Jul 2023 11:39:17 +0300 Subject: [PATCH 0440/1943] [#335] treesvc: Sort nodes by Filename in GetSubTree Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/tree.go | 4 +- pkg/local_object_storage/pilorama/boltdb.go | 19 +- pkg/local_object_storage/pilorama/forest.go | 12 +- .../pilorama/interface.go | 2 +- pkg/local_object_storage/pilorama/types.go | 6 + pkg/local_object_storage/shard/tree.go | 2 +- pkg/services/tree/service.go | 49 +- pkg/services/tree/service.pb.go | 703 +++++++++++------- pkg/services/tree/service.proto | 9 + pkg/services/tree/service_frostfs.pb.go | 30 + pkg/services/tree/service_grpc.pb.go | 56 +- pkg/services/tree/types.pb.go | 2 +- 12 files changed, 565 insertions(+), 329 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 08c6d26b0..df3e919ec 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -172,7 +172,7 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s } // TreeGetChildren implements the pilorama.Forest interface. -func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { +func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetChildren", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -183,7 +183,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree defer span.End() var err error - var nodes []uint64 + var nodes []pilorama.NodeInfo for _, sh := range e.sortShardsByWeight(cid) { nodes, err = sh.TreeGetChildren(ctx, cid, treeID, nodeID) if err != nil { diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 53a52433d..a729e2a22 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -927,7 +927,7 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri } // TreeGetChildren implements the Forest interface. -func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) { +func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) { var ( startedAt = time.Now() success = false @@ -956,7 +956,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID key[0] = 'c' binary.LittleEndian.PutUint64(key[1:], nodeID) - var children []uint64 + var result []NodeInfo err := t.db.View(func(tx *bbolt.Tx) error { treeRoot := tx.Bucket(bucketName(cid, treeID)) @@ -967,12 +967,23 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID b := treeRoot.Bucket(dataBucket) c := b.Cursor() for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { - children = append(children, binary.LittleEndian.Uint64(k[9:])) + childID := binary.LittleEndian.Uint64(k[9:]) + childInfo := NodeInfo{ + ID: childID, + } + parentID, _, metaBytes, found := t.getState(b, stateKey(key, childID)) + if found { + childInfo.ParentID = parentID + if err := childInfo.Meta.FromBytes(metaBytes); err != nil { + return err + } + } + result = append(result, childInfo) } return nil }) success = err == nil - return children, metaerr.Wrap(err) + return result, metaerr.Wrap(err) } // TreeList implements the Forest interface. diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 76220c1db..8fb519128 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -148,7 +148,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string, } // TreeGetChildren implements the Forest interface. -func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]uint64, error) { +func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]NodeInfo, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -156,8 +156,14 @@ func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID str } children := s.tree.getChildren(nodeID) - res := make([]Node, len(children)) - copy(res, children) + res := make([]NodeInfo, 0, len(children)) + for _, childID := range children { + res = append(res, NodeInfo{ + ID: childID, + Meta: s.infoMap[childID].Meta, + ParentID: s.infoMap[childID].Parent, + }) + } return res, nil } diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 89c752627..ea171a479 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -32,7 +32,7 @@ type Forest interface { TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) (Meta, Node, error) // TreeGetChildren returns children of the node with the specified ID. The order is arbitrary. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. - TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]uint64, error) + TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) // TreeGetOpLog returns first log operation stored at or above the height. // In case no such operation is found, empty Move and nil error should be returned. TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) diff --git a/pkg/local_object_storage/pilorama/types.go b/pkg/local_object_storage/pilorama/types.go index 99918683d..8d8616364 100644 --- a/pkg/local_object_storage/pilorama/types.go +++ b/pkg/local_object_storage/pilorama/types.go @@ -55,3 +55,9 @@ var ( func isAttributeInternal(key string) bool { return key == AttributeFilename } + +type NodeInfo struct { + ID Node + Meta Meta + ParentID Node +} diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 81477325a..7e2c80152 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -159,7 +159,7 @@ func (s *Shard) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, n } // TreeGetChildren implements the pilorama.Forest interface. -func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]uint64, error) { +func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node) ([]pilorama.NodeInfo, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetChildren", trace.WithAttributes( attribute.String("shard_id", s.ID().String()), diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 12d970c42..57767f87e 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "sort" "sync" "sync/atomic" @@ -440,7 +441,15 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { // Traverse the tree in a DFS manner. Because we need to support arbitrary depth, // recursive implementation is not suitable here, so we maintain explicit stack. - stack := [][]uint64{{b.GetRootId()}} + m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId()) + if err != nil { + return err + } + stack := [][]pilorama.NodeInfo{{{ + ID: b.GetRootId(), + Meta: m, + ParentID: p, + }}} for { if len(stack) == 0 { @@ -450,19 +459,15 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD continue } - nodeID := stack[len(stack)-1][0] + node := stack[len(stack)-1][0] stack[len(stack)-1] = stack[len(stack)-1][1:] - m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), nodeID) - if err != nil { - return err - } err = srv.Send(&GetSubTreeResponse{ Body: &GetSubTreeResponse_Body{ - NodeId: nodeID, - ParentId: p, - Timestamp: m.Time, - Meta: metaToProto(m.Items), + NodeId: node.ID, + ParentId: node.ParentID, + Timestamp: node.Meta.Time, + Meta: metaToProto(node.Meta.Items), }, }) if err != nil { @@ -470,7 +475,11 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD } if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() { - children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), nodeID) + children, err := forest.TreeGetChildren(ctx, cid, b.GetTreeId(), node.ID) + if err != nil { + return err + } + children, err = sortByFilename(children, b.GetOrderBy().GetDirection()) if err != nil { return err } @@ -482,6 +491,24 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD return nil } +func sortByFilename(nodes []pilorama.NodeInfo, d GetSubTreeRequest_Body_Order_Direction) ([]pilorama.NodeInfo, error) { + switch d { + case GetSubTreeRequest_Body_Order_None: + return nodes, nil + case GetSubTreeRequest_Body_Order_Asc: + if len(nodes) == 0 { + return nodes, nil + } + less := func(i, j int) bool { + return bytes.Compare(nodes[i].Meta.GetAttr(pilorama.AttributeFilename), nodes[j].Meta.GetAttr(pilorama.AttributeFilename)) < 0 + } + sort.Slice(nodes, less) + return nodes, nil + default: + return nil, fmt.Errorf("unsupported order direction: %s", d.String()) + } +} + // Apply locally applies operation from the remote node to the tree. func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) { err := verifyMessage(req) diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go index 08664a6d0..63f3e714a 100644 --- a/pkg/services/tree/service.pb.go +++ b/pkg/services/tree/service.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.12.4 +// protoc v3.21.9 // source: pkg/services/tree/service.proto package tree @@ -23,6 +23,52 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type GetSubTreeRequest_Body_Order_Direction int32 + +const ( + GetSubTreeRequest_Body_Order_None GetSubTreeRequest_Body_Order_Direction = 0 + GetSubTreeRequest_Body_Order_Asc GetSubTreeRequest_Body_Order_Direction = 1 +) + +// Enum value maps for GetSubTreeRequest_Body_Order_Direction. +var ( + GetSubTreeRequest_Body_Order_Direction_name = map[int32]string{ + 0: "None", + 1: "Asc", + } + GetSubTreeRequest_Body_Order_Direction_value = map[string]int32{ + "None": 0, + "Asc": 1, + } +) + +func (x GetSubTreeRequest_Body_Order_Direction) Enum() *GetSubTreeRequest_Body_Order_Direction { + p := new(GetSubTreeRequest_Body_Order_Direction) + *p = x + return p +} + +func (x GetSubTreeRequest_Body_Order_Direction) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GetSubTreeRequest_Body_Order_Direction) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_services_tree_service_proto_enumTypes[0].Descriptor() +} + +func (GetSubTreeRequest_Body_Order_Direction) Type() protoreflect.EnumType { + return &file_pkg_services_tree_service_proto_enumTypes[0] +} + +func (x GetSubTreeRequest_Body_Order_Direction) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GetSubTreeRequest_Body_Order_Direction.Descriptor instead. +func (GetSubTreeRequest_Body_Order_Direction) EnumDescriptor() ([]byte, []int) { + return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{10, 0, 0, 0} +} + type AddRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1940,6 +1986,8 @@ type GetSubTreeRequest_Body struct { Depth uint32 `protobuf:"varint,4,opt,name=depth,proto3" json:"depth,omitempty"` // Bearer token in V2 format. BearerToken []byte `protobuf:"bytes,5,opt,name=bearer_token,json=bearerToken,proto3" json:"bearer_token,omitempty"` + // Result ordering. + OrderBy *GetSubTreeRequest_Body_Order `protobuf:"bytes,6,opt,name=order_by,json=orderBy,proto3" json:"order_by,omitempty"` } func (x *GetSubTreeRequest_Body) Reset() { @@ -2009,6 +2057,60 @@ func (x *GetSubTreeRequest_Body) GetBearerToken() []byte { return nil } +func (x *GetSubTreeRequest_Body) GetOrderBy() *GetSubTreeRequest_Body_Order { + if x != nil { + return x.OrderBy + } + return nil +} + +type GetSubTreeRequest_Body_Order struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Direction GetSubTreeRequest_Body_Order_Direction `protobuf:"varint,1,opt,name=direction,proto3,enum=tree.GetSubTreeRequest_Body_Order_Direction" json:"direction,omitempty"` +} + +func (x *GetSubTreeRequest_Body_Order) Reset() { + *x = GetSubTreeRequest_Body_Order{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_tree_service_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetSubTreeRequest_Body_Order) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSubTreeRequest_Body_Order) ProtoMessage() {} + +func (x *GetSubTreeRequest_Body_Order) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_tree_service_proto_msgTypes[32] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSubTreeRequest_Body_Order.ProtoReflect.Descriptor instead. +func (*GetSubTreeRequest_Body_Order) Descriptor() ([]byte, []int) { + return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{10, 0, 0} +} + +func (x *GetSubTreeRequest_Body_Order) GetDirection() GetSubTreeRequest_Body_Order_Direction { + if x != nil { + return x.Direction + } + return GetSubTreeRequest_Body_Order_None +} + type GetSubTreeResponse_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2027,7 +2129,7 @@ type GetSubTreeResponse_Body struct { func (x *GetSubTreeResponse_Body) Reset() { *x = GetSubTreeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[32] + mi := &file_pkg_services_tree_service_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2040,7 +2142,7 @@ func (x *GetSubTreeResponse_Body) String() string { func (*GetSubTreeResponse_Body) ProtoMessage() {} func (x *GetSubTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[32] + mi := &file_pkg_services_tree_service_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2096,7 +2198,7 @@ type TreeListRequest_Body struct { func (x *TreeListRequest_Body) Reset() { *x = TreeListRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[33] + mi := &file_pkg_services_tree_service_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2109,7 +2211,7 @@ func (x *TreeListRequest_Body) String() string { func (*TreeListRequest_Body) ProtoMessage() {} func (x *TreeListRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[33] + mi := &file_pkg_services_tree_service_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2144,7 +2246,7 @@ type TreeListResponse_Body struct { func (x *TreeListResponse_Body) Reset() { *x = TreeListResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[34] + mi := &file_pkg_services_tree_service_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2157,7 +2259,7 @@ func (x *TreeListResponse_Body) String() string { func (*TreeListResponse_Body) ProtoMessage() {} func (x *TreeListResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[34] + mi := &file_pkg_services_tree_service_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2196,7 +2298,7 @@ type ApplyRequest_Body struct { func (x *ApplyRequest_Body) Reset() { *x = ApplyRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[35] + mi := &file_pkg_services_tree_service_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2209,7 +2311,7 @@ func (x *ApplyRequest_Body) String() string { func (*ApplyRequest_Body) ProtoMessage() {} func (x *ApplyRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[35] + mi := &file_pkg_services_tree_service_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2255,7 +2357,7 @@ type ApplyResponse_Body struct { func (x *ApplyResponse_Body) Reset() { *x = ApplyResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[36] + mi := &file_pkg_services_tree_service_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2268,7 +2370,7 @@ func (x *ApplyResponse_Body) String() string { func (*ApplyResponse_Body) ProtoMessage() {} func (x *ApplyResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[36] + mi := &file_pkg_services_tree_service_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2302,7 +2404,7 @@ type GetOpLogRequest_Body struct { func (x *GetOpLogRequest_Body) Reset() { *x = GetOpLogRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[37] + mi := &file_pkg_services_tree_service_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2315,7 +2417,7 @@ func (x *GetOpLogRequest_Body) String() string { func (*GetOpLogRequest_Body) ProtoMessage() {} func (x *GetOpLogRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[37] + mi := &file_pkg_services_tree_service_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2371,7 +2473,7 @@ type GetOpLogResponse_Body struct { func (x *GetOpLogResponse_Body) Reset() { *x = GetOpLogResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[38] + mi := &file_pkg_services_tree_service_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2384,7 +2486,7 @@ func (x *GetOpLogResponse_Body) String() string { func (*GetOpLogResponse_Body) ProtoMessage() {} func (x *GetOpLogResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[38] + mi := &file_pkg_services_tree_service_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2416,7 +2518,7 @@ type HealthcheckResponse_Body struct { func (x *HealthcheckResponse_Body) Reset() { *x = HealthcheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[39] + mi := &file_pkg_services_tree_service_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2429,7 +2531,7 @@ func (x *HealthcheckResponse_Body) String() string { func (*HealthcheckResponse_Body) ProtoMessage() {} func (x *HealthcheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[39] + mi := &file_pkg_services_tree_service_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2454,7 +2556,7 @@ type HealthcheckRequest_Body struct { func (x *HealthcheckRequest_Body) Reset() { *x = HealthcheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[40] + mi := &file_pkg_services_tree_service_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2467,7 +2569,7 @@ func (x *HealthcheckRequest_Body) String() string { func (*HealthcheckRequest_Body) ProtoMessage() {} func (x *HealthcheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[40] + mi := &file_pkg_services_tree_service_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2640,14 +2742,14 @@ var file_pkg_services_tree_service_proto_rawDesc = []byte{ 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, - 0x73, 0x22, 0x8b, 0x02, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, + 0x73, 0x22, 0xbf, 0x03, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x30, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x94, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xc8, 0x02, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, @@ -2656,146 +2758,157 @@ var file_pkg_services_tree_service_proto_rawDesc = []byte{ 0x72, 0x6f, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, - 0xf6, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, - 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, - 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x7e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x72, - 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x70, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x12, 0x22, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x04, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x22, 0x9b, 0x01, 0x0a, 0x0f, 0x54, 0x72, 0x65, - 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2e, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x74, 0x72, 0x65, - 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x29, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0x8c, 0x01, 0x0a, 0x10, 0x54, 0x72, 0x65, 0x65, 0x4c, - 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x72, 0x65, 0x65, - 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x18, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x03, 0x69, 0x64, 0x73, 0x22, 0xdb, 0x01, 0x0a, 0x0c, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, - 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x6f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, - 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x74, 0x72, 0x65, 0x65, - 0x2e, 0x4c, 0x6f, 0x67, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x22, 0x74, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xe2, 0x01, 0x0a, 0x0f, 0x47, 0x65, - 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2e, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x74, 0x72, - 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, + 0x28, 0x0c, 0x52, 0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, + 0x3d, 0x0a, 0x08, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x5f, 0x62, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, + 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, + 0x4f, 0x72, 0x64, 0x65, 0x72, 0x52, 0x07, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x42, 0x79, 0x1a, 0x73, + 0x0a, 0x05, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x4a, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x74, 0x72, 0x65, + 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x2e, 0x44, + 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x22, 0x1e, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x73, + 0x63, 0x10, 0x01, 0x22, 0xf6, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, + 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x70, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, - 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xa7, - 0x01, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, - 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x33, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2b, 0x0a, 0x09, 0x6f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4c, 0x6f, 0x67, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x09, 0x6f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x80, 0x01, 0x0a, 0x13, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x32, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, - 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x7e, 0x0a, 0x12, 0x48, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x31, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, - 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xd6, 0x04, 0x0a, 0x0b, - 0x54, 0x72, 0x65, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x2a, 0x0a, 0x03, 0x41, - 0x64, 0x64, 0x12, 0x10, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, 0x64, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x09, 0x41, 0x64, 0x64, 0x42, 0x79, - 0x50, 0x61, 0x74, 0x68, 0x12, 0x16, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, 0x64, 0x42, - 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74, - 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x12, - 0x13, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x04, 0x4d, 0x6f, - 0x76, 0x65, 0x12, 0x11, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4d, 0x6f, 0x76, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0d, 0x47, 0x65, 0x74, - 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1a, 0x2e, 0x74, 0x72, 0x65, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x7e, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, + 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x22, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4b, 0x65, + 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x22, 0x9b, 0x01, 0x0a, + 0x0f, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x2e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, + 0x29, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0x8c, 0x01, 0x0a, 0x10, 0x54, + 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, + 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, + 0x18, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x64, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x69, 0x64, 0x73, 0x22, 0xdb, 0x01, 0x0a, 0x0c, 0x41, 0x70, + 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, + 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, + 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, + 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x6f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, + 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, + 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x09, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, + 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4c, 0x6f, 0x67, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x09, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x74, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x6c, 0x79, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, + 0x70, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xe2, 0x01, + 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x2e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x1a, 0x70, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, + 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, + 0x65, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x14, 0x0a, 0x05, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, + 0x6e, 0x74, 0x22, 0xa7, 0x01, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, + 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, + 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x33, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x2b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4c, 0x6f, 0x67, 0x4d, 0x6f, 0x76, + 0x65, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x80, 0x01, 0x0a, + 0x13, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, + 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, + 0x7e, 0x0a, 0x12, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, + 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, + 0xd6, 0x04, 0x0a, 0x0b, 0x54, 0x72, 0x65, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, + 0x2a, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x12, 0x10, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, + 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, + 0x41, 0x64, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x09, 0x41, + 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x12, 0x16, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, + 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, + 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x06, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x12, 0x13, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, + 0x0a, 0x04, 0x4d, 0x6f, 0x76, 0x65, 0x12, 0x11, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4d, 0x6f, + 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x74, 0x72, 0x65, 0x65, + 0x2e, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, + 0x0d, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1a, + 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, + 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, - 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, - 0x65, 0x12, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, - 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x74, 0x72, 0x65, - 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x39, 0x0a, 0x08, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, - 0x73, 0x74, 0x12, 0x15, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, - 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x74, 0x72, 0x65, 0x65, - 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x30, 0x0a, 0x05, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x12, 0x12, 0x2e, 0x74, 0x72, 0x65, - 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x12, - 0x15, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, - 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, - 0x12, 0x42, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x12, - 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, - 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x74, 0x72, 0x65, 0x65, - 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, - 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, - 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, - 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, - 0x74, 0x72, 0x65, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x53, 0x75, + 0x62, 0x54, 0x72, 0x65, 0x65, 0x12, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, + 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, + 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x39, 0x0a, 0x08, 0x54, 0x72, + 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x15, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, + 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, + 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x12, 0x12, + 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4f, 0x70, + 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, + 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x74, 0x72, 0x65, + 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, + 0x65, 0x63, 0x6b, 0x12, 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, + 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x2e, + 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, + 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, + 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x65, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2810,127 +2923,132 @@ func file_pkg_services_tree_service_proto_rawDescGZIP() []byte { return file_pkg_services_tree_service_proto_rawDescData } -var file_pkg_services_tree_service_proto_msgTypes = make([]protoimpl.MessageInfo, 41) +var file_pkg_services_tree_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_pkg_services_tree_service_proto_msgTypes = make([]protoimpl.MessageInfo, 42) var file_pkg_services_tree_service_proto_goTypes = []interface{}{ - (*AddRequest)(nil), // 0: tree.AddRequest - (*AddResponse)(nil), // 1: tree.AddResponse - (*AddByPathRequest)(nil), // 2: tree.AddByPathRequest - (*AddByPathResponse)(nil), // 3: tree.AddByPathResponse - (*RemoveRequest)(nil), // 4: tree.RemoveRequest - (*RemoveResponse)(nil), // 5: tree.RemoveResponse - (*MoveRequest)(nil), // 6: tree.MoveRequest - (*MoveResponse)(nil), // 7: tree.MoveResponse - (*GetNodeByPathRequest)(nil), // 8: tree.GetNodeByPathRequest - (*GetNodeByPathResponse)(nil), // 9: tree.GetNodeByPathResponse - (*GetSubTreeRequest)(nil), // 10: tree.GetSubTreeRequest - (*GetSubTreeResponse)(nil), // 11: tree.GetSubTreeResponse - (*TreeListRequest)(nil), // 12: tree.TreeListRequest - (*TreeListResponse)(nil), // 13: tree.TreeListResponse - (*ApplyRequest)(nil), // 14: tree.ApplyRequest - (*ApplyResponse)(nil), // 15: tree.ApplyResponse - (*GetOpLogRequest)(nil), // 16: tree.GetOpLogRequest - (*GetOpLogResponse)(nil), // 17: tree.GetOpLogResponse - (*HealthcheckResponse)(nil), // 18: tree.HealthcheckResponse - (*HealthcheckRequest)(nil), // 19: tree.HealthcheckRequest - (*AddRequest_Body)(nil), // 20: tree.AddRequest.Body - (*AddResponse_Body)(nil), // 21: tree.AddResponse.Body - (*AddByPathRequest_Body)(nil), // 22: tree.AddByPathRequest.Body - (*AddByPathResponse_Body)(nil), // 23: tree.AddByPathResponse.Body - (*RemoveRequest_Body)(nil), // 24: tree.RemoveRequest.Body - (*RemoveResponse_Body)(nil), // 25: tree.RemoveResponse.Body - (*MoveRequest_Body)(nil), // 26: tree.MoveRequest.Body - (*MoveResponse_Body)(nil), // 27: tree.MoveResponse.Body - (*GetNodeByPathRequest_Body)(nil), // 28: tree.GetNodeByPathRequest.Body - (*GetNodeByPathResponse_Info)(nil), // 29: tree.GetNodeByPathResponse.Info - (*GetNodeByPathResponse_Body)(nil), // 30: tree.GetNodeByPathResponse.Body - (*GetSubTreeRequest_Body)(nil), // 31: tree.GetSubTreeRequest.Body - (*GetSubTreeResponse_Body)(nil), // 32: tree.GetSubTreeResponse.Body - (*TreeListRequest_Body)(nil), // 33: tree.TreeListRequest.Body - (*TreeListResponse_Body)(nil), // 34: tree.TreeListResponse.Body - (*ApplyRequest_Body)(nil), // 35: tree.ApplyRequest.Body - (*ApplyResponse_Body)(nil), // 36: tree.ApplyResponse.Body - (*GetOpLogRequest_Body)(nil), // 37: tree.GetOpLogRequest.Body - (*GetOpLogResponse_Body)(nil), // 38: tree.GetOpLogResponse.Body - (*HealthcheckResponse_Body)(nil), // 39: tree.HealthcheckResponse.Body - (*HealthcheckRequest_Body)(nil), // 40: tree.HealthcheckRequest.Body - (*Signature)(nil), // 41: tree.Signature - (*KeyValue)(nil), // 42: tree.KeyValue - (*LogMove)(nil), // 43: tree.LogMove + (GetSubTreeRequest_Body_Order_Direction)(0), // 0: tree.GetSubTreeRequest.Body.Order.Direction + (*AddRequest)(nil), // 1: tree.AddRequest + (*AddResponse)(nil), // 2: tree.AddResponse + (*AddByPathRequest)(nil), // 3: tree.AddByPathRequest + (*AddByPathResponse)(nil), // 4: tree.AddByPathResponse + (*RemoveRequest)(nil), // 5: tree.RemoveRequest + (*RemoveResponse)(nil), // 6: tree.RemoveResponse + (*MoveRequest)(nil), // 7: tree.MoveRequest + (*MoveResponse)(nil), // 8: tree.MoveResponse + (*GetNodeByPathRequest)(nil), // 9: tree.GetNodeByPathRequest + (*GetNodeByPathResponse)(nil), // 10: tree.GetNodeByPathResponse + (*GetSubTreeRequest)(nil), // 11: tree.GetSubTreeRequest + (*GetSubTreeResponse)(nil), // 12: tree.GetSubTreeResponse + (*TreeListRequest)(nil), // 13: tree.TreeListRequest + (*TreeListResponse)(nil), // 14: tree.TreeListResponse + (*ApplyRequest)(nil), // 15: tree.ApplyRequest + (*ApplyResponse)(nil), // 16: tree.ApplyResponse + (*GetOpLogRequest)(nil), // 17: tree.GetOpLogRequest + (*GetOpLogResponse)(nil), // 18: tree.GetOpLogResponse + (*HealthcheckResponse)(nil), // 19: tree.HealthcheckResponse + (*HealthcheckRequest)(nil), // 20: tree.HealthcheckRequest + (*AddRequest_Body)(nil), // 21: tree.AddRequest.Body + (*AddResponse_Body)(nil), // 22: tree.AddResponse.Body + (*AddByPathRequest_Body)(nil), // 23: tree.AddByPathRequest.Body + (*AddByPathResponse_Body)(nil), // 24: tree.AddByPathResponse.Body + (*RemoveRequest_Body)(nil), // 25: tree.RemoveRequest.Body + (*RemoveResponse_Body)(nil), // 26: tree.RemoveResponse.Body + (*MoveRequest_Body)(nil), // 27: tree.MoveRequest.Body + (*MoveResponse_Body)(nil), // 28: tree.MoveResponse.Body + (*GetNodeByPathRequest_Body)(nil), // 29: tree.GetNodeByPathRequest.Body + (*GetNodeByPathResponse_Info)(nil), // 30: tree.GetNodeByPathResponse.Info + (*GetNodeByPathResponse_Body)(nil), // 31: tree.GetNodeByPathResponse.Body + (*GetSubTreeRequest_Body)(nil), // 32: tree.GetSubTreeRequest.Body + (*GetSubTreeRequest_Body_Order)(nil), // 33: tree.GetSubTreeRequest.Body.Order + (*GetSubTreeResponse_Body)(nil), // 34: tree.GetSubTreeResponse.Body + (*TreeListRequest_Body)(nil), // 35: tree.TreeListRequest.Body + (*TreeListResponse_Body)(nil), // 36: tree.TreeListResponse.Body + (*ApplyRequest_Body)(nil), // 37: tree.ApplyRequest.Body + (*ApplyResponse_Body)(nil), // 38: tree.ApplyResponse.Body + (*GetOpLogRequest_Body)(nil), // 39: tree.GetOpLogRequest.Body + (*GetOpLogResponse_Body)(nil), // 40: tree.GetOpLogResponse.Body + (*HealthcheckResponse_Body)(nil), // 41: tree.HealthcheckResponse.Body + (*HealthcheckRequest_Body)(nil), // 42: tree.HealthcheckRequest.Body + (*Signature)(nil), // 43: tree.Signature + (*KeyValue)(nil), // 44: tree.KeyValue + (*LogMove)(nil), // 45: tree.LogMove } var file_pkg_services_tree_service_proto_depIdxs = []int32{ - 20, // 0: tree.AddRequest.body:type_name -> tree.AddRequest.Body - 41, // 1: tree.AddRequest.signature:type_name -> tree.Signature - 21, // 2: tree.AddResponse.body:type_name -> tree.AddResponse.Body - 41, // 3: tree.AddResponse.signature:type_name -> tree.Signature - 22, // 4: tree.AddByPathRequest.body:type_name -> tree.AddByPathRequest.Body - 41, // 5: tree.AddByPathRequest.signature:type_name -> tree.Signature - 23, // 6: tree.AddByPathResponse.body:type_name -> tree.AddByPathResponse.Body - 41, // 7: tree.AddByPathResponse.signature:type_name -> tree.Signature - 24, // 8: tree.RemoveRequest.body:type_name -> tree.RemoveRequest.Body - 41, // 9: tree.RemoveRequest.signature:type_name -> tree.Signature - 25, // 10: tree.RemoveResponse.body:type_name -> tree.RemoveResponse.Body - 41, // 11: tree.RemoveResponse.signature:type_name -> tree.Signature - 26, // 12: tree.MoveRequest.body:type_name -> tree.MoveRequest.Body - 41, // 13: tree.MoveRequest.signature:type_name -> tree.Signature - 27, // 14: tree.MoveResponse.body:type_name -> tree.MoveResponse.Body - 41, // 15: tree.MoveResponse.signature:type_name -> tree.Signature - 28, // 16: tree.GetNodeByPathRequest.body:type_name -> tree.GetNodeByPathRequest.Body - 41, // 17: tree.GetNodeByPathRequest.signature:type_name -> tree.Signature - 30, // 18: tree.GetNodeByPathResponse.body:type_name -> tree.GetNodeByPathResponse.Body - 41, // 19: tree.GetNodeByPathResponse.signature:type_name -> tree.Signature - 31, // 20: tree.GetSubTreeRequest.body:type_name -> tree.GetSubTreeRequest.Body - 41, // 21: tree.GetSubTreeRequest.signature:type_name -> tree.Signature - 32, // 22: tree.GetSubTreeResponse.body:type_name -> tree.GetSubTreeResponse.Body - 41, // 23: tree.GetSubTreeResponse.signature:type_name -> tree.Signature - 33, // 24: tree.TreeListRequest.body:type_name -> tree.TreeListRequest.Body - 41, // 25: tree.TreeListRequest.signature:type_name -> tree.Signature - 34, // 26: tree.TreeListResponse.body:type_name -> tree.TreeListResponse.Body - 41, // 27: tree.TreeListResponse.signature:type_name -> tree.Signature - 35, // 28: tree.ApplyRequest.body:type_name -> tree.ApplyRequest.Body - 41, // 29: tree.ApplyRequest.signature:type_name -> tree.Signature - 36, // 30: tree.ApplyResponse.body:type_name -> tree.ApplyResponse.Body - 41, // 31: tree.ApplyResponse.signature:type_name -> tree.Signature - 37, // 32: tree.GetOpLogRequest.body:type_name -> tree.GetOpLogRequest.Body - 41, // 33: tree.GetOpLogRequest.signature:type_name -> tree.Signature - 38, // 34: tree.GetOpLogResponse.body:type_name -> tree.GetOpLogResponse.Body - 41, // 35: tree.GetOpLogResponse.signature:type_name -> tree.Signature - 39, // 36: tree.HealthcheckResponse.body:type_name -> tree.HealthcheckResponse.Body - 41, // 37: tree.HealthcheckResponse.signature:type_name -> tree.Signature - 40, // 38: tree.HealthcheckRequest.body:type_name -> tree.HealthcheckRequest.Body - 41, // 39: tree.HealthcheckRequest.signature:type_name -> tree.Signature - 42, // 40: tree.AddRequest.Body.meta:type_name -> tree.KeyValue - 42, // 41: tree.AddByPathRequest.Body.meta:type_name -> tree.KeyValue - 42, // 42: tree.MoveRequest.Body.meta:type_name -> tree.KeyValue - 42, // 43: tree.GetNodeByPathResponse.Info.meta:type_name -> tree.KeyValue - 29, // 44: tree.GetNodeByPathResponse.Body.nodes:type_name -> tree.GetNodeByPathResponse.Info - 42, // 45: tree.GetSubTreeResponse.Body.meta:type_name -> tree.KeyValue - 43, // 46: tree.ApplyRequest.Body.operation:type_name -> tree.LogMove - 43, // 47: tree.GetOpLogResponse.Body.operation:type_name -> tree.LogMove - 0, // 48: tree.TreeService.Add:input_type -> tree.AddRequest - 2, // 49: tree.TreeService.AddByPath:input_type -> tree.AddByPathRequest - 4, // 50: tree.TreeService.Remove:input_type -> tree.RemoveRequest - 6, // 51: tree.TreeService.Move:input_type -> tree.MoveRequest - 8, // 52: tree.TreeService.GetNodeByPath:input_type -> tree.GetNodeByPathRequest - 10, // 53: tree.TreeService.GetSubTree:input_type -> tree.GetSubTreeRequest - 12, // 54: tree.TreeService.TreeList:input_type -> tree.TreeListRequest - 14, // 55: tree.TreeService.Apply:input_type -> tree.ApplyRequest - 16, // 56: tree.TreeService.GetOpLog:input_type -> tree.GetOpLogRequest - 19, // 57: tree.TreeService.Healthcheck:input_type -> tree.HealthcheckRequest - 1, // 58: tree.TreeService.Add:output_type -> tree.AddResponse - 3, // 59: tree.TreeService.AddByPath:output_type -> tree.AddByPathResponse - 5, // 60: tree.TreeService.Remove:output_type -> tree.RemoveResponse - 7, // 61: tree.TreeService.Move:output_type -> tree.MoveResponse - 9, // 62: tree.TreeService.GetNodeByPath:output_type -> tree.GetNodeByPathResponse - 11, // 63: tree.TreeService.GetSubTree:output_type -> tree.GetSubTreeResponse - 13, // 64: tree.TreeService.TreeList:output_type -> tree.TreeListResponse - 15, // 65: tree.TreeService.Apply:output_type -> tree.ApplyResponse - 17, // 66: tree.TreeService.GetOpLog:output_type -> tree.GetOpLogResponse - 18, // 67: tree.TreeService.Healthcheck:output_type -> tree.HealthcheckResponse - 58, // [58:68] is the sub-list for method output_type - 48, // [48:58] is the sub-list for method input_type - 48, // [48:48] is the sub-list for extension type_name - 48, // [48:48] is the sub-list for extension extendee - 0, // [0:48] is the sub-list for field type_name + 21, // 0: tree.AddRequest.body:type_name -> tree.AddRequest.Body + 43, // 1: tree.AddRequest.signature:type_name -> tree.Signature + 22, // 2: tree.AddResponse.body:type_name -> tree.AddResponse.Body + 43, // 3: tree.AddResponse.signature:type_name -> tree.Signature + 23, // 4: tree.AddByPathRequest.body:type_name -> tree.AddByPathRequest.Body + 43, // 5: tree.AddByPathRequest.signature:type_name -> tree.Signature + 24, // 6: tree.AddByPathResponse.body:type_name -> tree.AddByPathResponse.Body + 43, // 7: tree.AddByPathResponse.signature:type_name -> tree.Signature + 25, // 8: tree.RemoveRequest.body:type_name -> tree.RemoveRequest.Body + 43, // 9: tree.RemoveRequest.signature:type_name -> tree.Signature + 26, // 10: tree.RemoveResponse.body:type_name -> tree.RemoveResponse.Body + 43, // 11: tree.RemoveResponse.signature:type_name -> tree.Signature + 27, // 12: tree.MoveRequest.body:type_name -> tree.MoveRequest.Body + 43, // 13: tree.MoveRequest.signature:type_name -> tree.Signature + 28, // 14: tree.MoveResponse.body:type_name -> tree.MoveResponse.Body + 43, // 15: tree.MoveResponse.signature:type_name -> tree.Signature + 29, // 16: tree.GetNodeByPathRequest.body:type_name -> tree.GetNodeByPathRequest.Body + 43, // 17: tree.GetNodeByPathRequest.signature:type_name -> tree.Signature + 31, // 18: tree.GetNodeByPathResponse.body:type_name -> tree.GetNodeByPathResponse.Body + 43, // 19: tree.GetNodeByPathResponse.signature:type_name -> tree.Signature + 32, // 20: tree.GetSubTreeRequest.body:type_name -> tree.GetSubTreeRequest.Body + 43, // 21: tree.GetSubTreeRequest.signature:type_name -> tree.Signature + 34, // 22: tree.GetSubTreeResponse.body:type_name -> tree.GetSubTreeResponse.Body + 43, // 23: tree.GetSubTreeResponse.signature:type_name -> tree.Signature + 35, // 24: tree.TreeListRequest.body:type_name -> tree.TreeListRequest.Body + 43, // 25: tree.TreeListRequest.signature:type_name -> tree.Signature + 36, // 26: tree.TreeListResponse.body:type_name -> tree.TreeListResponse.Body + 43, // 27: tree.TreeListResponse.signature:type_name -> tree.Signature + 37, // 28: tree.ApplyRequest.body:type_name -> tree.ApplyRequest.Body + 43, // 29: tree.ApplyRequest.signature:type_name -> tree.Signature + 38, // 30: tree.ApplyResponse.body:type_name -> tree.ApplyResponse.Body + 43, // 31: tree.ApplyResponse.signature:type_name -> tree.Signature + 39, // 32: tree.GetOpLogRequest.body:type_name -> tree.GetOpLogRequest.Body + 43, // 33: tree.GetOpLogRequest.signature:type_name -> tree.Signature + 40, // 34: tree.GetOpLogResponse.body:type_name -> tree.GetOpLogResponse.Body + 43, // 35: tree.GetOpLogResponse.signature:type_name -> tree.Signature + 41, // 36: tree.HealthcheckResponse.body:type_name -> tree.HealthcheckResponse.Body + 43, // 37: tree.HealthcheckResponse.signature:type_name -> tree.Signature + 42, // 38: tree.HealthcheckRequest.body:type_name -> tree.HealthcheckRequest.Body + 43, // 39: tree.HealthcheckRequest.signature:type_name -> tree.Signature + 44, // 40: tree.AddRequest.Body.meta:type_name -> tree.KeyValue + 44, // 41: tree.AddByPathRequest.Body.meta:type_name -> tree.KeyValue + 44, // 42: tree.MoveRequest.Body.meta:type_name -> tree.KeyValue + 44, // 43: tree.GetNodeByPathResponse.Info.meta:type_name -> tree.KeyValue + 30, // 44: tree.GetNodeByPathResponse.Body.nodes:type_name -> tree.GetNodeByPathResponse.Info + 33, // 45: tree.GetSubTreeRequest.Body.order_by:type_name -> tree.GetSubTreeRequest.Body.Order + 0, // 46: tree.GetSubTreeRequest.Body.Order.direction:type_name -> tree.GetSubTreeRequest.Body.Order.Direction + 44, // 47: tree.GetSubTreeResponse.Body.meta:type_name -> tree.KeyValue + 45, // 48: tree.ApplyRequest.Body.operation:type_name -> tree.LogMove + 45, // 49: tree.GetOpLogResponse.Body.operation:type_name -> tree.LogMove + 1, // 50: tree.TreeService.Add:input_type -> tree.AddRequest + 3, // 51: tree.TreeService.AddByPath:input_type -> tree.AddByPathRequest + 5, // 52: tree.TreeService.Remove:input_type -> tree.RemoveRequest + 7, // 53: tree.TreeService.Move:input_type -> tree.MoveRequest + 9, // 54: tree.TreeService.GetNodeByPath:input_type -> tree.GetNodeByPathRequest + 11, // 55: tree.TreeService.GetSubTree:input_type -> tree.GetSubTreeRequest + 13, // 56: tree.TreeService.TreeList:input_type -> tree.TreeListRequest + 15, // 57: tree.TreeService.Apply:input_type -> tree.ApplyRequest + 17, // 58: tree.TreeService.GetOpLog:input_type -> tree.GetOpLogRequest + 20, // 59: tree.TreeService.Healthcheck:input_type -> tree.HealthcheckRequest + 2, // 60: tree.TreeService.Add:output_type -> tree.AddResponse + 4, // 61: tree.TreeService.AddByPath:output_type -> tree.AddByPathResponse + 6, // 62: tree.TreeService.Remove:output_type -> tree.RemoveResponse + 8, // 63: tree.TreeService.Move:output_type -> tree.MoveResponse + 10, // 64: tree.TreeService.GetNodeByPath:output_type -> tree.GetNodeByPathResponse + 12, // 65: tree.TreeService.GetSubTree:output_type -> tree.GetSubTreeResponse + 14, // 66: tree.TreeService.TreeList:output_type -> tree.TreeListResponse + 16, // 67: tree.TreeService.Apply:output_type -> tree.ApplyResponse + 18, // 68: tree.TreeService.GetOpLog:output_type -> tree.GetOpLogResponse + 19, // 69: tree.TreeService.Healthcheck:output_type -> tree.HealthcheckResponse + 60, // [60:70] is the sub-list for method output_type + 50, // [50:60] is the sub-list for method input_type + 50, // [50:50] is the sub-list for extension type_name + 50, // [50:50] is the sub-list for extension extendee + 0, // [0:50] is the sub-list for field type_name } func init() { file_pkg_services_tree_service_proto_init() } @@ -3325,7 +3443,7 @@ func file_pkg_services_tree_service_proto_init() { } } file_pkg_services_tree_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetSubTreeResponse_Body); i { + switch v := v.(*GetSubTreeRequest_Body_Order); i { case 0: return &v.state case 1: @@ -3337,7 +3455,7 @@ func file_pkg_services_tree_service_proto_init() { } } file_pkg_services_tree_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TreeListRequest_Body); i { + switch v := v.(*GetSubTreeResponse_Body); i { case 0: return &v.state case 1: @@ -3349,7 +3467,7 @@ func file_pkg_services_tree_service_proto_init() { } } file_pkg_services_tree_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TreeListResponse_Body); i { + switch v := v.(*TreeListRequest_Body); i { case 0: return &v.state case 1: @@ -3361,7 +3479,7 @@ func file_pkg_services_tree_service_proto_init() { } } file_pkg_services_tree_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ApplyRequest_Body); i { + switch v := v.(*TreeListResponse_Body); i { case 0: return &v.state case 1: @@ -3373,7 +3491,7 @@ func file_pkg_services_tree_service_proto_init() { } } file_pkg_services_tree_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ApplyResponse_Body); i { + switch v := v.(*ApplyRequest_Body); i { case 0: return &v.state case 1: @@ -3385,7 +3503,7 @@ func file_pkg_services_tree_service_proto_init() { } } file_pkg_services_tree_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOpLogRequest_Body); i { + switch v := v.(*ApplyResponse_Body); i { case 0: return &v.state case 1: @@ -3397,7 +3515,7 @@ func file_pkg_services_tree_service_proto_init() { } } file_pkg_services_tree_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOpLogResponse_Body); i { + switch v := v.(*GetOpLogRequest_Body); i { case 0: return &v.state case 1: @@ -3409,7 +3527,7 @@ func file_pkg_services_tree_service_proto_init() { } } file_pkg_services_tree_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthcheckResponse_Body); i { + switch v := v.(*GetOpLogResponse_Body); i { case 0: return &v.state case 1: @@ -3421,6 +3539,18 @@ func file_pkg_services_tree_service_proto_init() { } } file_pkg_services_tree_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HealthcheckResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_tree_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*HealthcheckRequest_Body); i { case 0: return &v.state @@ -3438,13 +3568,14 @@ func file_pkg_services_tree_service_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_tree_service_proto_rawDesc, - NumEnums: 0, - NumMessages: 41, + NumEnums: 1, + NumMessages: 42, NumExtensions: 0, NumServices: 1, }, GoTypes: file_pkg_services_tree_service_proto_goTypes, DependencyIndexes: file_pkg_services_tree_service_proto_depIdxs, + EnumInfos: file_pkg_services_tree_service_proto_enumTypes, MessageInfos: file_pkg_services_tree_service_proto_msgTypes, }.Build() File_pkg_services_tree_service_proto = out.File diff --git a/pkg/services/tree/service.proto b/pkg/services/tree/service.proto index 182d8adb2..ec63d88ec 100644 --- a/pkg/services/tree/service.proto +++ b/pkg/services/tree/service.proto @@ -238,6 +238,13 @@ message GetNodeByPathResponse { message GetSubTreeRequest { message Body { + message Order { + enum Direction { + None = 0; + Asc = 1; + } + Direction direction = 1; + } // Container ID in V2 format. bytes container_id = 1; // The name of the tree. @@ -249,6 +256,8 @@ message GetSubTreeRequest { uint32 depth = 4; // Bearer token in V2 format. bytes bearer_token = 5; + // Result ordering. + Order order_by = 6; } // Request body. diff --git a/pkg/services/tree/service_frostfs.pb.go b/pkg/services/tree/service_frostfs.pb.go index 42b7ba3fc..b272e4389 100644 --- a/pkg/services/tree/service_frostfs.pb.go +++ b/pkg/services/tree/service_frostfs.pb.go @@ -892,6 +892,34 @@ func (x *GetNodeByPathResponse) SetSignature(sig *Signature) { x.Signature = sig } +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetSubTreeRequest_Body_Order) StableSize() (size int) { + size += proto.EnumSize(1, int32(x.Direction)) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetSubTreeRequest_Body_Order) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.EnumMarshal(1, buf[offset:], int32(x.Direction)) + return buf +} + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -901,6 +929,7 @@ func (x *GetSubTreeRequest_Body) StableSize() (size int) { size += proto.UInt64Size(3, x.RootId) size += proto.UInt32Size(4, x.Depth) size += proto.BytesSize(5, x.BearerToken) + size += proto.NestedStructureSize(6, x.OrderBy) return size } @@ -925,6 +954,7 @@ func (x *GetSubTreeRequest_Body) StableMarshal(buf []byte) []byte { offset += proto.UInt64Marshal(3, buf[offset:], x.RootId) offset += proto.UInt32Marshal(4, buf[offset:], x.Depth) offset += proto.BytesMarshal(5, buf[offset:], x.BearerToken) + offset += proto.NestedStructureMarshal(6, buf[offset:], x.OrderBy) return buf } diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index fa259e804..2c0828951 100644 --- a/pkg/services/tree/service_grpc.pb.go +++ b/pkg/services/tree/service_grpc.pb.go @@ -1,7 +1,10 @@ +//* +// Service for working with CRDT tree. + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.3.0 +// - protoc v3.21.9 // source: pkg/services/tree/service.proto package tree @@ -18,6 +21,19 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + TreeService_Add_FullMethodName = "/tree.TreeService/Add" + TreeService_AddByPath_FullMethodName = "/tree.TreeService/AddByPath" + TreeService_Remove_FullMethodName = "/tree.TreeService/Remove" + TreeService_Move_FullMethodName = "/tree.TreeService/Move" + TreeService_GetNodeByPath_FullMethodName = "/tree.TreeService/GetNodeByPath" + TreeService_GetSubTree_FullMethodName = "/tree.TreeService/GetSubTree" + TreeService_TreeList_FullMethodName = "/tree.TreeService/TreeList" + TreeService_Apply_FullMethodName = "/tree.TreeService/Apply" + TreeService_GetOpLog_FullMethodName = "/tree.TreeService/GetOpLog" + TreeService_Healthcheck_FullMethodName = "/tree.TreeService/Healthcheck" +) + // TreeServiceClient is the client API for TreeService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -55,7 +71,7 @@ func NewTreeServiceClient(cc grpc.ClientConnInterface) TreeServiceClient { func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*AddResponse, error) { out := new(AddResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Add", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Add_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -64,7 +80,7 @@ func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grp func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, opts ...grpc.CallOption) (*AddByPathResponse, error) { out := new(AddByPathResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/AddByPath", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_AddByPath_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -73,7 +89,7 @@ func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts ...grpc.CallOption) (*RemoveResponse, error) { out := new(RemoveResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Remove", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Remove_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -82,7 +98,7 @@ func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...grpc.CallOption) (*MoveResponse, error) { out := new(MoveResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Move", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Move_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -91,7 +107,7 @@ func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...g func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPathRequest, opts ...grpc.CallOption) (*GetNodeByPathResponse, error) { out := new(GetNodeByPathResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/GetNodeByPath", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_GetNodeByPath_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -99,7 +115,7 @@ func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPath } func (c *treeServiceClient) GetSubTree(ctx context.Context, in *GetSubTreeRequest, opts ...grpc.CallOption) (TreeService_GetSubTreeClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], "/tree.TreeService/GetSubTree", opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], TreeService_GetSubTree_FullMethodName, opts...) if err != nil { return nil, err } @@ -132,7 +148,7 @@ func (x *treeServiceGetSubTreeClient) Recv() (*GetSubTreeResponse, error) { func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, opts ...grpc.CallOption) (*TreeListResponse, error) { out := new(TreeListResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/TreeList", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_TreeList_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -141,7 +157,7 @@ func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, o func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts ...grpc.CallOption) (*ApplyResponse, error) { out := new(ApplyResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Apply", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Apply_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -149,7 +165,7 @@ func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts .. } func (c *treeServiceClient) GetOpLog(ctx context.Context, in *GetOpLogRequest, opts ...grpc.CallOption) (TreeService_GetOpLogClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], "/tree.TreeService/GetOpLog", opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], TreeService_GetOpLog_FullMethodName, opts...) if err != nil { return nil, err } @@ -182,7 +198,7 @@ func (x *treeServiceGetOpLogClient) Recv() (*GetOpLogResponse, error) { func (c *treeServiceClient) Healthcheck(ctx context.Context, in *HealthcheckRequest, opts ...grpc.CallOption) (*HealthcheckResponse, error) { out := new(HealthcheckResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Healthcheck", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Healthcheck_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -272,7 +288,7 @@ func _TreeService_Add_Handler(srv interface{}, ctx context.Context, dec func(int } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Add", + FullMethod: TreeService_Add_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Add(ctx, req.(*AddRequest)) @@ -290,7 +306,7 @@ func _TreeService_AddByPath_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/AddByPath", + FullMethod: TreeService_AddByPath_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).AddByPath(ctx, req.(*AddByPathRequest)) @@ -308,7 +324,7 @@ func _TreeService_Remove_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Remove", + FullMethod: TreeService_Remove_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Remove(ctx, req.(*RemoveRequest)) @@ -326,7 +342,7 @@ func _TreeService_Move_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Move", + FullMethod: TreeService_Move_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Move(ctx, req.(*MoveRequest)) @@ -344,7 +360,7 @@ func _TreeService_GetNodeByPath_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/GetNodeByPath", + FullMethod: TreeService_GetNodeByPath_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).GetNodeByPath(ctx, req.(*GetNodeByPathRequest)) @@ -383,7 +399,7 @@ func _TreeService_TreeList_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/TreeList", + FullMethod: TreeService_TreeList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).TreeList(ctx, req.(*TreeListRequest)) @@ -401,7 +417,7 @@ func _TreeService_Apply_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Apply", + FullMethod: TreeService_Apply_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Apply(ctx, req.(*ApplyRequest)) @@ -440,7 +456,7 @@ func _TreeService_Healthcheck_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Healthcheck", + FullMethod: TreeService_Healthcheck_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Healthcheck(ctx, req.(*HealthcheckRequest)) diff --git a/pkg/services/tree/types.pb.go b/pkg/services/tree/types.pb.go index 45d889177..b4d6981ef 100644 --- a/pkg/services/tree/types.pb.go +++ b/pkg/services/tree/types.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.12.4 +// protoc v3.21.9 // source: pkg/services/tree/types.proto package tree From af82c2865ec1ce28f62fa928eeab98fd041e40cd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Jul 2023 11:50:13 +0300 Subject: [PATCH 0441/1943] [#335] treesvc: Fix inmemory unit tests and nil meta items Bolt forest saves empty slice of items. Now inmemory forest does it the same way. Signed-off-by: Dmitrii Stepanov --- .../pilorama/forest_test.go | 19 +++++++++++++------ pkg/local_object_storage/pilorama/inmemory.go | 8 ++++++-- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 9e8e98863..5c143d3eb 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -147,16 +147,23 @@ func testForestTreeGetChildren(t *testing.T, s Forest) { treeAdd(t, 2, 0) treeAdd(t, 7, 0) - testGetChildren := func(t *testing.T, nodeID Node, expected []Node) { + testGetChildren := func(t *testing.T, nodeID Node, expected []NodeInfo) { actual, err := s.TreeGetChildren(context.Background(), cid, treeID, nodeID) require.NoError(t, err) require.ElementsMatch(t, expected, actual) } - testGetChildren(t, 0, []uint64{10, 2, 7}) - testGetChildren(t, 10, []uint64{3, 6}) + testGetChildren(t, 0, []NodeInfo{ + {ID: 10, Meta: Meta{Time: 1, Items: []KeyValue{}}}, + {ID: 2, Meta: Meta{Time: 5, Items: []KeyValue{}}}, + {ID: 7, Meta: Meta{Time: 6, Items: []KeyValue{}}}, + }) + testGetChildren(t, 10, []NodeInfo{ + {ID: 3, ParentID: 10, Meta: Meta{Time: 2, Items: []KeyValue{}}}, + {ID: 6, ParentID: 10, Meta: Meta{Time: 3, Items: []KeyValue{}}}, + }) testGetChildren(t, 3, nil) - testGetChildren(t, 6, []uint64{11}) + testGetChildren(t, 6, []NodeInfo{{ID: 11, ParentID: 6, Meta: Meta{Time: 4, Items: []KeyValue{}}}}) testGetChildren(t, 11, nil) testGetChildren(t, 2, nil) testGetChildren(t, 7, nil) @@ -495,11 +502,11 @@ func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ nodes, err := s.TreeGetChildren(ctx, cid, treeID, RootID) require.NoError(t, err) - require.Equal(t, []Node{1}, nodes) + require.Equal(t, []NodeInfo{{ID: 1, ParentID: RootID, Meta: meta[0]}}, nodes) nodes, err = s.TreeGetChildren(ctx, cid, treeID, 1) require.NoError(t, err) - require.Equal(t, []Node{2}, nodes) + require.Equal(t, []NodeInfo{{ID: 2, ParentID: 1, Meta: meta[1]}}, nodes) } t.Run("expected", func(t *testing.T) { diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index 1bde312ac..c9f5df3b7 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -68,10 +68,14 @@ func (s *memoryTree) Apply(op *Move) error { // do performs a single move operation on a tree. func (s *memoryTree) do(op *Move) move { + m := op.Meta + if m.Items == nil { + m.Items = []KeyValue{} + } lm := move{ Move: Move{ Parent: op.Parent, - Meta: op.Meta, + Meta: m, Child: op.Child, }, } @@ -91,7 +95,7 @@ func (s *memoryTree) do(op *Move) move { p.Meta.Time = op.Time } - p.Meta = op.Meta + p.Meta = m p.Parent = op.Parent s.tree.infoMap[op.Child] = p From eed594431f9260e9dbf65a9e86c1be33b2f645ca Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Jul 2023 11:51:19 +0300 Subject: [PATCH 0442/1943] [#335] treesvc: Add GetSubTree ordering unit test Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/getsubtree_test.go | 61 ++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index dc4ce29aa..88a5b5e06 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -3,6 +3,8 @@ package tree import ( "context" "errors" + "path" + "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -119,6 +121,65 @@ func TestGetSubTree(t *testing.T) { }) } +func TestGetSubTreeOrderAsc(t *testing.T) { + d := pilorama.CIDDescriptor{CID: cidtest.ID(), Size: 1} + treeID := "sometree" + p := pilorama.NewMemoryForest() + + tree := []struct { + path []string + id uint64 + }{ + {path: []string{"dir1"}}, + {path: []string{"dir2"}}, + {path: []string{"dir1", "sub1"}}, + {path: []string{"dir2", "sub1"}}, + {path: []string{"dir2", "sub2"}}, + {path: []string{"dir2", "sub1", "subsub1"}}, + } + + for i := range tree { + path := tree[i].path + meta := []pilorama.KeyValue{ + {Key: pilorama.AttributeFilename, Value: []byte(path[len(path)-1])}} + + lm, err := p.TreeAddByPath(context.Background(), d, treeID, pilorama.AttributeFilename, path[:len(path)-1], meta) + require.NoError(t, err) + require.Equal(t, 1, len(lm)) + tree[i].id = lm[0].Child + } + + acc := subTreeAcc{errIndex: -1} + err := getSubTree(context.Background(), &acc, d.CID, &GetSubTreeRequest_Body{ + TreeId: treeID, + OrderBy: &GetSubTreeRequest_Body_Order{ + Direction: GetSubTreeRequest_Body_Order_Asc, + }, + }, p) + require.NoError(t, err) + // GetSubTree must return child only after is has returned the parent. + require.Equal(t, uint64(0), acc.seen[0].Body.NodeId) + + paths := make([]string, 0, len(acc.seen)) + for i := range acc.seen { + if i == 0 { + continue + } + found := false + for j := range tree { + if acc.seen[i].Body.NodeId == tree[j].id { + found = true + paths = append(paths, path.Join(tree[j].path...)) + } + } + require.True(t, found, "unknown node") + } + + require.True(t, sort.SliceIsSorted(paths, func(i, j int) bool { + return paths[i] < paths[j] + })) +} + var ( errSubTreeSend = errors.New("send finished with error") errSubTreeSendAfterError = errors.New("send was invoked after an error occurred") From 5a4054eeb67436147939af4617aa0215b93fb955 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 20 Jul 2023 11:23:26 +0300 Subject: [PATCH 0443/1943] [#535] .forgejo: Add vulncheck workflow Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/vulncheck.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .forgejo/workflows/vulncheck.yml diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml new file mode 100644 index 000000000..9390ad2d6 --- /dev/null +++ b/.forgejo/workflows/vulncheck.yml @@ -0,0 +1,22 @@ +name: Vulncheck +on: [pull_request] + +jobs: + vulncheck: + name: Vulncheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.20' + + - name: Install govulncheck + run: go install golang.org/x/vuln/cmd/govulncheck@latest + + - name: Run govulncheck + run: govulncheck ./... From fef172c5b03f424953601eaa281b70874a4bd087 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 18:04:24 +0300 Subject: [PATCH 0444/1943] [#6] services/util: Simplify `response.Service` It has only 1 parameter, which is obligatory. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config.go | 2 +- pkg/services/util/response/service.go | 40 ++++----------------------- 2 files changed, 7 insertions(+), 35 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 267a05aa8..c088d2f07 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -622,7 +622,7 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt key: key, binPublicKey: key.PublicKey().Bytes(), localAddr: netAddr, - respSvc: response.NewService(response.WithNetworkState(netState)), + respSvc: response.NewService(netState), clientCache: cache.NewSDKClientCache(cacheOpts), bgClientCache: cache.NewSDKClientCache(cacheOpts), putClientCache: cache.NewSDKClientCache(cacheOpts), diff --git a/pkg/services/util/response/service.go b/pkg/services/util/response/service.go index a63d5343b..005a643e5 100644 --- a/pkg/services/util/response/service.go +++ b/pkg/services/util/response/service.go @@ -11,45 +11,24 @@ import ( // Service represents universal v2 service // that sets response meta header values. type Service struct { - cfg *cfg -} - -// Option is an option of Service constructor. -type Option func(*cfg) - -type cfg struct { version refs.Version state netmap.State } -func defaultCfg() *cfg { - var c cfg - - version.Current().WriteToV2(&c.version) - - return &c -} - // NewService creates, initializes and returns Service instance. -func NewService(opts ...Option) *Service { - c := defaultCfg() - - for i := range opts { - opts[i](c) - } - - return &Service{ - cfg: c, - } +func NewService(nmState netmap.State) *Service { + s := &Service{state: nmState} + version.Current().WriteToV2(&s.version) + return s } // SetMeta sets adds meta-header to resp. func (s *Service) SetMeta(resp util.ResponseMessage) { meta := new(session.ResponseMetaHeader) - meta.SetVersion(&s.cfg.version) + meta.SetVersion(&s.version) meta.SetTTL(1) // FIXME: #1160 TTL must be calculated - meta.SetEpoch(s.cfg.state.CurrentEpoch()) + meta.SetEpoch(s.state.CurrentEpoch()) if origin := resp.GetMetaHeader(); origin != nil { // FIXME: #1160 what if origin is set by local server? @@ -58,10 +37,3 @@ func (s *Service) SetMeta(resp util.ResponseMessage) { resp.SetMetaHeader(meta) } - -// WithNetworkState returns option to set network state of Service. -func WithNetworkState(v netmap.State) Option { - return func(c *cfg) { - c.state = v - } -} From 372160d04807b12ffddacaa8ed949b93b29f98be Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 18:26:43 +0300 Subject: [PATCH 0445/1943] [#6] services/util: Remove `SignService.HandleUnaryRequest` There is no need in a wrapper with many from-`interface{}` conversions. Signed-off-by: Evgenii Stratonikov --- pkg/services/accounting/sign.go | 15 +--- pkg/services/container/sign.go | 119 ++++++++++---------------------- pkg/services/netmap/sign.go | 51 ++++---------- pkg/services/object/sign.go | 68 ++++++------------ pkg/services/session/sign.go | 17 ++--- pkg/services/util/sign.go | 45 ++++++------ 6 files changed, 97 insertions(+), 218 deletions(-) diff --git a/pkg/services/accounting/sign.go b/pkg/services/accounting/sign.go index e98d9b3af..9efb063f5 100644 --- a/pkg/services/accounting/sign.go +++ b/pkg/services/accounting/sign.go @@ -22,17 +22,6 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { } func (s *signService) Balance(ctx context.Context, req *accounting.BalanceRequest) (*accounting.BalanceResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Balance(ctx, req.(*accounting.BalanceRequest)) - }, - func() util.ResponseMessage { - return new(accounting.BalanceResponse) - }, - ) - if err != nil { - return nil, err - } - - return resp.(*accounting.BalanceResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Balance(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } diff --git a/pkg/services/container/sign.go b/pkg/services/container/sign.go index 9e77e2e21..55125335f 100644 --- a/pkg/services/container/sign.go +++ b/pkg/services/container/sign.go @@ -22,113 +22,64 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { } func (s *signService) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Put(ctx, req.(*container.PutRequest)) - }, - func() util.ResponseMessage { - return new(container.PutResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.PutResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.PutResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Put(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*container.DeleteRequest)) - }, - func() util.ResponseMessage { - return new(container.DeleteResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.DeleteResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.DeleteResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Get(ctx, req.(*container.GetRequest)) - }, - func() util.ResponseMessage { - return new(container.GetResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.GetResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.GetResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Get(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.List(ctx, req.(*container.ListRequest)) - }, - func() util.ResponseMessage { - return new(container.ListResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.ListResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.ListResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.List(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.SetExtendedACL(ctx, req.(*container.SetExtendedACLRequest)) - }, - func() util.ResponseMessage { - return new(container.SetExtendedACLResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.SetExtendedACLResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.SetExtendedACLResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.SetExtendedACL(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetExtendedACL(ctx, req.(*container.GetExtendedACLRequest)) - }, - func() util.ResponseMessage { - return new(container.GetExtendedACLResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.GetExtendedACLResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.GetExtendedACLResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.GetExtendedACL(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.AnnounceUsedSpace(ctx, req.(*container.AnnounceUsedSpaceRequest)) - }, - func() util.ResponseMessage { - return new(container.AnnounceUsedSpaceResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.AnnounceUsedSpaceResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*container.AnnounceUsedSpaceResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.AnnounceUsedSpace(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } diff --git a/pkg/services/netmap/sign.go b/pkg/services/netmap/sign.go index 85b19d862..e665519e1 100644 --- a/pkg/services/netmap/sign.go +++ b/pkg/services/netmap/sign.go @@ -24,49 +24,28 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) LocalNodeInfo( ctx context.Context, req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.LocalNodeInfo(ctx, req.(*netmap.LocalNodeInfoRequest)) - }, - func() util.ResponseMessage { - return new(netmap.LocalNodeInfoResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(netmap.LocalNodeInfoResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*netmap.LocalNodeInfoResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.LocalNodeInfo(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) NetworkInfo(ctx context.Context, req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.NetworkInfo(ctx, req.(*netmap.NetworkInfoRequest)) - }, - func() util.ResponseMessage { - return new(netmap.NetworkInfoResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(netmap.NetworkInfoResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*netmap.NetworkInfoResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.NetworkInfo(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *signService) Snapshot(ctx context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Snapshot(ctx, req.(*netmap.SnapshotRequest)) - }, - func() util.ResponseMessage { - return new(netmap.SnapshotResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(netmap.SnapshotResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*netmap.SnapshotResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Snapshot(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 5b3578e29..f160b4c56 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -105,35 +105,21 @@ func (s *SignService) Put() (PutObjectStream, error) { } func (s *SignService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Head(ctx, req.(*object.HeadRequest)) - }, - func() util.ResponseMessage { - return new(object.HeadResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.HeadResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*object.HeadResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Head(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *SignService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.PutSingle(ctx, req.(*object.PutSingleRequest)) - }, - func() util.ResponseMessage { - return new(object.PutSingleResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.PutSingleResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*object.PutSingleResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.PutSingle(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { @@ -172,19 +158,12 @@ func (s *SignService) Search(req *object.SearchRequest, stream SearchStream) err } func (s *SignService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Delete(ctx, req.(*object.DeleteRequest)) - }, - func() util.ResponseMessage { - return new(object.DeleteResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.DeleteResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*object.DeleteResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } func (s *getRangeStreamSigner) Send(resp *object.GetRangeResponse) error { @@ -209,17 +188,10 @@ func (s *SignService) GetRange(req *object.GetRangeRequest, stream GetObjectRang } func (s *SignService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.GetRangeHash(ctx, req.(*object.GetRangeHashRequest)) - }, - func() util.ResponseMessage { - return new(object.GetRangeHashResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.GetRangeHashResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*object.GetRangeHashResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.GetRangeHash(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } diff --git a/pkg/services/session/sign.go b/pkg/services/session/sign.go index 1156dc538..33c0a531c 100644 --- a/pkg/services/session/sign.go +++ b/pkg/services/session/sign.go @@ -22,17 +22,10 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { } func (s *signService) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - resp, err := s.sigSvc.HandleUnaryRequest(ctx, req, - func(ctx context.Context, req any) (util.ResponseMessage, error) { - return s.svc.Create(ctx, req.(*session.CreateRequest)) - }, - func() util.ResponseMessage { - return new(session.CreateResponse) - }, - ) - if err != nil { - return nil, err + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(session.CreateResponse) + return resp, s.sigSvc.SignResponse(req, resp, err) } - - return resp.(*session.CreateResponse), nil + resp, err := util.EnsureNonNilResponse(s.svc.Create(ctx, req)) + return resp, s.sigSvc.SignResponse(req, resp, err) } diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index cb4be3084..e41575663 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -172,44 +172,39 @@ func (s *SignService) HandleServerStreamRequest( return nil } -func (s *SignService) HandleUnaryRequest(ctx context.Context, req any, handler UnaryHandler, blankResp ResponseConstructor) (ResponseMessage, error) { +func (s *SignService) SignResponse(req RequestMessage, resp ResponseMessage, err error) error { // handle protocol versions <=2.10 (API statuses was introduced in 2.11 only) // req argument should be strengthen with type RequestMessage - statusSupported := isStatusSupported(req.(RequestMessage)) // panic is OK here for now - - var ( - resp ResponseMessage - err error - ) - - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { - var sigErr apistatus.SignatureVerification - sigErr.SetMessage(err.Error()) - - err = sigErr - } else { - // process request - resp, err = handler(ctx, req) - } + statusSupported := isStatusSupported(req) if err != nil { if !statusSupported { - return nil, err + return err } - resp = blankResp() - setStatusV2(resp, err) } // sign the response - if err = signResponse(s.key, resp, statusSupported); err != nil { - return nil, err - } + return signResponse(s.key, resp, statusSupported) +} - return resp, nil +func (s *SignService) VerifyRequest(req RequestMessage) error { + if err := signature.VerifyServiceMessage(req); err != nil { + var sigErr apistatus.SignatureVerification + sigErr.SetMessage(err.Error()) + return sigErr + } + return nil +} + +// EnsureNonNilResponse creates an appropriate response struct if it is nil. +func EnsureNonNilResponse[T any](resp *T, err error) (*T, error) { + if resp != nil { + return resp, err + } + return new(T), err } func isStatusSupported(req RequestMessage) bool { From c2617baf6323bde3fa387aa1741c277715d7eb97 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 30 Dec 2022 20:01:13 +0300 Subject: [PATCH 0446/1943] [#6] services/util: Remove remaining stream wrappers Signed-off-by: Evgenii Stratonikov --- pkg/services/accounting/sign.go | 2 +- pkg/services/container/sign.go | 28 ++--- pkg/services/netmap/sign.go | 12 +- pkg/services/object/sign.go | 188 ++++++++++++++++---------------- pkg/services/session/sign.go | 4 +- pkg/services/util/sign.go | 188 ++++---------------------------- 6 files changed, 139 insertions(+), 283 deletions(-) diff --git a/pkg/services/accounting/sign.go b/pkg/services/accounting/sign.go index 9efb063f5..be7b08a39 100644 --- a/pkg/services/accounting/sign.go +++ b/pkg/services/accounting/sign.go @@ -23,5 +23,5 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) Balance(ctx context.Context, req *accounting.BalanceRequest) (*accounting.BalanceResponse, error) { resp, err := util.EnsureNonNilResponse(s.svc.Balance(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/container/sign.go b/pkg/services/container/sign.go index 55125335f..b336f19c3 100644 --- a/pkg/services/container/sign.go +++ b/pkg/services/container/sign.go @@ -24,62 +24,62 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.PutResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Put(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.DeleteResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.GetResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Get(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.ListResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.List(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.SetExtendedACLResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.SetExtendedACL(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.GetExtendedACLResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.GetExtendedACL(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.AnnounceUsedSpaceResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.AnnounceUsedSpace(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/netmap/sign.go b/pkg/services/netmap/sign.go index e665519e1..2d01164a3 100644 --- a/pkg/services/netmap/sign.go +++ b/pkg/services/netmap/sign.go @@ -26,26 +26,26 @@ func (s *signService) LocalNodeInfo( req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(netmap.LocalNodeInfoResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.LocalNodeInfo(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) NetworkInfo(ctx context.Context, req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(netmap.NetworkInfoResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.NetworkInfo(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *signService) Snapshot(ctx context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(netmap.SnapshotResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Snapshot(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index f160b4c56..c516872e3 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -18,27 +18,30 @@ type SignService struct { } type searchStreamSigner struct { - util.ServerStream - - respWriter util.ResponseMessageWriter + SearchStream + statusSupported bool + sigSvc *util.SignService nonEmptyResp bool // set on first Send call } type getStreamSigner struct { - util.ServerStream - - respWriter util.ResponseMessageWriter + GetObjectStream + statusSupported bool + sigSvc *util.SignService } type putStreamSigner struct { - stream *util.RequestMessageStreamer + sigSvc *util.SignService + stream PutObjectStream + statusSupported bool + err error } type getRangeStreamSigner struct { - util.ServerStream - - respWriter util.ResponseMessageWriter + GetObjectRangeStream + statusSupported bool + sigSvc *util.SignService } func NewSignService(key *ecdsa.PrivateKey, svc ServiceServer) *SignService { @@ -50,37 +53,50 @@ func NewSignService(key *ecdsa.PrivateKey, svc ServiceServer) *SignService { } func (s *getStreamSigner) Send(resp *object.GetResponse) error { - return s.respWriter(resp) + if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + return err + } + return s.GetObjectStream.Send(resp) } func (s *SignService) Get(req *object.GetRequest, stream GetObjectStream) error { - return s.sigSvc.HandleServerStreamRequest(req, - func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetResponse)) - }, - func() util.ResponseMessage { - return new(object.GetResponse) - }, - func(respWriter util.ResponseMessageWriter) error { - return s.svc.Get(req, &getStreamSigner{ - ServerStream: stream, - respWriter: respWriter, - }) - }, - ) + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.GetResponse) + _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return stream.Send(resp) + } + + return s.svc.Get(req, &getStreamSigner{ + GetObjectStream: stream, + sigSvc: s.sigSvc, + statusSupported: util.IsStatusSupported(req), + }) } func (s *putStreamSigner) Send(ctx context.Context, req *object.PutRequest) error { - return s.stream.Send(ctx, req) + s.statusSupported = util.IsStatusSupported(req) + + if s.err = s.sigSvc.VerifyRequest(req); s.err != nil { + return util.ErrAbortStream + } + if s.err = s.stream.Send(ctx, req); s.err != nil { + return util.ErrAbortStream + } + return nil } -func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { - r, err := s.stream.CloseAndRecv(ctx) - if err != nil { - return nil, fmt.Errorf("could not receive response: %w", err) +func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.PutResponse, err error) { + if s.err != nil { + err = s.err + resp = new(object.PutResponse) + } else { + resp, err = s.stream.CloseAndRecv(ctx) + if err != nil { + return nil, fmt.Errorf("could not close stream and receive response: %w", err) + } } - return r.(*object.PutResponse), nil + return resp, s.sigSvc.SignResponse(s.statusSupported, resp, err) } func (s *SignService) Put() (PutObjectStream, error) { @@ -90,108 +106,96 @@ func (s *SignService) Put() (PutObjectStream, error) { } return &putStreamSigner{ - stream: s.sigSvc.CreateRequestStreamer( - func(ctx context.Context, req any) error { - return stream.Send(ctx, req.(*object.PutRequest)) - }, - func(ctx context.Context) (util.ResponseMessage, error) { - return stream.CloseAndRecv(ctx) - }, - func() util.ResponseMessage { - return new(object.PutResponse) - }, - ), + stream: stream, + sigSvc: s.sigSvc, }, nil } func (s *SignService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.HeadResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Head(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *SignService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.PutSingleResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.PutSingle(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { s.nonEmptyResp = true - return s.respWriter(resp) + if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + return err + } + return s.SearchStream.Send(resp) } func (s *SignService) Search(req *object.SearchRequest, stream SearchStream) error { - return s.sigSvc.HandleServerStreamRequest(req, - func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.SearchResponse)) - }, - func() util.ResponseMessage { - return new(object.SearchResponse) - }, - func(respWriter util.ResponseMessageWriter) error { - stream := &searchStreamSigner{ - ServerStream: stream, - respWriter: respWriter, - } + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.SearchResponse) + _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return stream.Send(resp) + } - err := s.svc.Search(req, stream) - - if err == nil && !stream.nonEmptyResp { - // The higher component does not write any response in the case of an empty result (which is correct). - // With the introduction of status returns at least one answer must be signed and sent to the client. - // This approach is supported by clients who do not know how to work with statuses (one could make - // a switch according to the protocol version from the request, but the costs of sending an empty - // answer can be neglected due to the gradual refusal to use the "old" clients). - return stream.Send(new(object.SearchResponse)) - } - - return err - }, - ) + ss := &searchStreamSigner{ + SearchStream: stream, + sigSvc: s.sigSvc, + statusSupported: util.IsStatusSupported(req), + } + err := s.svc.Search(req, ss) + if err == nil && !ss.nonEmptyResp { + // The higher component does not write any response in the case of an empty result (which is correct). + // With the introduction of status returns at least one answer must be signed and sent to the client. + // This approach is supported by clients who do not know how to work with statuses (one could make + // a switch according to the protocol version from the request, but the costs of sending an empty + // answer can be neglected due to the gradual refusal to use the "old" clients). + return stream.Send(new(object.SearchResponse)) + } + return err } func (s *SignService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.DeleteResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } func (s *getRangeStreamSigner) Send(resp *object.GetRangeResponse) error { - return s.respWriter(resp) + if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + return err + } + return s.GetObjectRangeStream.Send(resp) } func (s *SignService) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { - return s.sigSvc.HandleServerStreamRequest(req, - func(resp util.ResponseMessage) error { - return stream.Send(resp.(*object.GetRangeResponse)) - }, - func() util.ResponseMessage { - return new(object.GetRangeResponse) - }, - func(respWriter util.ResponseMessageWriter) error { - return s.svc.GetRange(req, &getRangeStreamSigner{ - ServerStream: stream, - respWriter: respWriter, - }) - }, - ) + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(object.GetRangeResponse) + _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return stream.Send(resp) + } + + return s.svc.GetRange(req, &getRangeStreamSigner{ + GetObjectRangeStream: stream, + sigSvc: s.sigSvc, + statusSupported: util.IsStatusSupported(req), + }) } func (s *SignService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.GetRangeHashResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.GetRangeHash(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/session/sign.go b/pkg/services/session/sign.go index 33c0a531c..ffce0621e 100644 --- a/pkg/services/session/sign.go +++ b/pkg/services/session/sign.go @@ -24,8 +24,8 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(session.CreateResponse) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Create(ctx, req)) - return resp, s.sigSvc.SignResponse(req, resp, err) + return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) } diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index e41575663..a3e0c946e 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -1,7 +1,6 @@ package util import ( - "context" "crypto/ecdsa" "errors" "fmt" @@ -21,163 +20,23 @@ type ResponseMessage interface { SetMetaHeader(*session.ResponseMetaHeader) } -type UnaryHandler func(context.Context, any) (ResponseMessage, error) - type SignService struct { key *ecdsa.PrivateKey } -type ResponseMessageWriter func(ResponseMessage) error - -type ServerStreamHandler func(context.Context, any) (ResponseMessageReader, error) - -type ResponseMessageReader func() (ResponseMessage, error) - var ErrAbortStream = errors.New("abort message stream") -type ResponseConstructor func() ResponseMessage - -type RequestMessageWriter func(context.Context, any) error - -type ClientStreamCloser func(context.Context) (ResponseMessage, error) - -type RequestMessageStreamer struct { - key *ecdsa.PrivateKey - - send RequestMessageWriter - - close ClientStreamCloser - - respCons ResponseConstructor - - statusSupported bool - - sendErr error -} - func NewUnarySignService(key *ecdsa.PrivateKey) *SignService { return &SignService{ key: key, } } -func (s *RequestMessageStreamer) Send(ctx context.Context, req any) error { - // req argument should be strengthen with type RequestMessage - s.statusSupported = isStatusSupported(req.(RequestMessage)) // panic is OK here for now - - var err error - - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { - err = fmt.Errorf("could not verify request: %w", err) - } else { - err = s.send(ctx, req) - } - - if err != nil { - if !s.statusSupported { - return err - } - - s.sendErr = err - - return ErrAbortStream - } - - return nil -} - -func (s *RequestMessageStreamer) CloseAndRecv(ctx context.Context) (ResponseMessage, error) { - var ( - resp ResponseMessage - err error - ) - - if s.sendErr != nil { - err = s.sendErr - } else { - resp, err = s.close(ctx) - if err != nil { - err = fmt.Errorf("could not close stream and receive response: %w", err) - } - } - - if err != nil { - if !s.statusSupported { - return nil, err - } - - resp = s.respCons() - - setStatusV2(resp, err) - } - - if err = signResponse(s.key, resp, s.statusSupported); err != nil { - return nil, err - } - - return resp, nil -} - -func (s *SignService) CreateRequestStreamer(sender RequestMessageWriter, closer ClientStreamCloser, blankResp ResponseConstructor) *RequestMessageStreamer { - return &RequestMessageStreamer{ - key: s.key, - send: sender, - close: closer, - - respCons: blankResp, - } -} - -func (s *SignService) HandleServerStreamRequest( - req any, - respWriter ResponseMessageWriter, - blankResp ResponseConstructor, - respWriterCaller func(ResponseMessageWriter) error, -) error { - // handle protocol versions <=2.10 (API statuses was introduced in 2.11 only) - - // req argument should be strengthen with type RequestMessage - statusSupported := isStatusSupported(req.(RequestMessage)) // panic is OK here for now - - var err error - - // verify request signatures - if err = signature.VerifyServiceMessage(req); err != nil { - err = fmt.Errorf("could not verify request: %w", err) - } else { - err = respWriterCaller(func(resp ResponseMessage) error { - if err := signResponse(s.key, resp, statusSupported); err != nil { - return err - } - - return respWriter(resp) - }) - } - - if err != nil { - if !statusSupported { - return err - } - - resp := blankResp() - - setStatusV2(resp, err) - - _ = signResponse(s.key, resp, false) // panics or returns nil with false arg - - return respWriter(resp) - } - - return nil -} - -func (s *SignService) SignResponse(req RequestMessage, resp ResponseMessage, err error) error { - // handle protocol versions <=2.10 (API statuses was introduced in 2.11 only) - - // req argument should be strengthen with type RequestMessage - statusSupported := isStatusSupported(req) - +// SignResponse response with private key via signature.SignServiceMessage. +// The signature error affects the result depending on the protocol version: +// - if status return is supported, panics since we cannot return the failed status, because it will not be signed. +// - otherwise, returns error in order to transport it directly. +func (s *SignService) SignResponse(statusSupported bool, resp ResponseMessage, err error) error { if err != nil { if !statusSupported { return err @@ -186,8 +45,18 @@ func (s *SignService) SignResponse(req RequestMessage, resp ResponseMessage, err setStatusV2(resp, err) } - // sign the response - return signResponse(s.key, resp, statusSupported) + err = signature.SignServiceMessage(s.key, resp) + if err != nil { + err = fmt.Errorf("could not sign response: %w", err) + + if statusSupported { + // We can't pass this error as status code since response will be unsigned. + // Isn't expected in practice, so panic is ok here. + panic(err) + } + } + + return err } func (s *SignService) VerifyRequest(req RequestMessage) error { @@ -207,7 +76,9 @@ func EnsureNonNilResponse[T any](resp *T, err error) (*T, error) { return new(T), err } -func isStatusSupported(req RequestMessage) bool { +// IsStatusSupported returns true iff request version implies expecting status return. +// This allows us to handle protocol versions <=2.10 (API statuses was introduced in 2.11 only). +func IsStatusSupported(req RequestMessage) bool { version := req.GetMetaHeader().GetVersion() mjr := version.GetMajor() @@ -223,22 +94,3 @@ func setStatusV2(resp ResponseMessage, err error) { session.SetStatus(resp, apistatus.ToStatusV2(apistatus.ErrToStatus(err))) } - -// signs response with private key via signature.SignServiceMessage. -// The signature error affects the result depending on the protocol version: -// - if status return is supported, panics since we cannot return the failed status, because it will not be signed; -// - otherwise, returns error in order to transport it directly. -func signResponse(key *ecdsa.PrivateKey, resp any, statusSupported bool) error { - err := signature.SignServiceMessage(key, resp) - if err != nil { - err = fmt.Errorf("could not sign response: %w", err) - - if statusSupported { - // We can't pass this error as status code since response will be unsigned. - // Isn't expected in practice, so panic is ok here. - panic(err) - } - } - - return err -} From 448b48287c7b4df99cbe24426747f2b28bfa68df Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 31 Dec 2022 09:43:13 +0300 Subject: [PATCH 0447/1943] [#6] services/util: Do not panic in sign function Signed-off-by: Evgenii Stratonikov --- pkg/services/util/sign.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index a3e0c946e..a26bd311c 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -47,16 +47,10 @@ func (s *SignService) SignResponse(statusSupported bool, resp ResponseMessage, e err = signature.SignServiceMessage(s.key, resp) if err != nil { - err = fmt.Errorf("could not sign response: %w", err) - - if statusSupported { - // We can't pass this error as status code since response will be unsigned. - // Isn't expected in practice, so panic is ok here. - panic(err) - } + return fmt.Errorf("could not sign response: %w", err) } - return err + return nil } func (s *SignService) VerifyRequest(req RequestMessage) error { From 5ff82ff04f673f728ee7b9a081d9a79809681c2c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 10 Jan 2023 15:10:54 +0300 Subject: [PATCH 0448/1943] [#6] services/object: Simplify local/remote targets We do not use the return result from Close() and we always execute both methods in succession. It makes sense to unite them. Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/distributed.go | 8 ++--- pkg/services/object/put/local.go | 34 ++++++--------------- pkg/services/object/put/remote.go | 41 +++++++++----------------- pkg/services/object/put/streamer.go | 2 +- 4 files changed, 27 insertions(+), 58 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 408ee099c..cf5cc558e 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -18,8 +18,7 @@ import ( ) type preparedObjectTarget interface { - WriteObject(*objectSDK.Object, object.ContentMeta) error - Close(ctx context.Context) (*transformer.AccessIdentifiers, error) + WriteObject(context.Context, *objectSDK.Object, object.ContentMeta) error } type distributedTarget struct { @@ -170,10 +169,9 @@ func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error target := t.nodeTargetInitializer(node) - if err := target.WriteObject(t.obj, t.objMeta); err != nil { + err := target.WriteObject(ctx, t.obj, t.objMeta) + if err != nil { return fmt.Errorf("could not write header: %w", err) - } else if _, err := target.Close(ctx); err != nil { - return fmt.Errorf("could not close object stream: %w", err) } return nil } diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index be202892a..54649adc7 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -7,7 +7,6 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" ) // ObjectStorage is an object storage interface. @@ -27,41 +26,26 @@ type ObjectStorage interface { type localTarget struct { storage ObjectStorage - - obj *objectSDK.Object - meta objectCore.ContentMeta } -func (t *localTarget) WriteObject(obj *objectSDK.Object, meta objectCore.ContentMeta) error { - t.obj = obj - t.meta = meta - - return nil -} - -func (t *localTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { - switch t.meta.Type() { +func (t localTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, meta objectCore.ContentMeta) error { + switch meta.Type() { case objectSDK.TypeTombstone: - err := t.storage.Delete(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) + err := t.storage.Delete(ctx, objectCore.AddressOf(obj), meta.Objects()) if err != nil { - return nil, fmt.Errorf("could not delete objects from tombstone locally: %w", err) + return fmt.Errorf("could not delete objects from tombstone locally: %w", err) } case objectSDK.TypeLock: - err := t.storage.Lock(ctx, objectCore.AddressOf(t.obj), t.meta.Objects()) + err := t.storage.Lock(ctx, objectCore.AddressOf(obj), meta.Objects()) if err != nil { - return nil, fmt.Errorf("could not lock object from lock objects locally: %w", err) + return fmt.Errorf("could not lock object from lock objects locally: %w", err) } default: // objects that do not change meta storage } - if err := t.storage.Put(ctx, t.obj); err != nil { //TODO - return nil, fmt.Errorf("(%T) could not put object to local storage: %w", t, err) + if err := t.storage.Put(ctx, obj); err != nil { + return fmt.Errorf("(%T) could not put object to local storage: %w", t, err) } - - id, _ := t.obj.ID() - - return &transformer.AccessIdentifiers{ - SelfID: id, - }, nil + return nil } diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/put/remote.go index 8116243ec..ee8d64e7a 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/put/remote.go @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -24,8 +23,6 @@ type remoteTarget struct { nodeInfo clientcore.NodeInfo - obj *objectSDK.Object - clientConstructor ClientConstructor } @@ -44,16 +41,10 @@ type RemotePutPrm struct { obj *objectSDK.Object } -func (t *remoteTarget) WriteObject(obj *objectSDK.Object, _ objectcore.ContentMeta) error { - t.obj = obj - - return nil -} - -func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { +func (t *remoteTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, _ objectcore.ContentMeta) error { c, err := t.clientConstructor.Get(t.nodeInfo) if err != nil { - return nil, fmt.Errorf("(%T) could not create SDK client %s: %w", t, t.nodeInfo, err) + return fmt.Errorf("(%T) could not create SDK client %s: %w", t, t.nodeInfo, err) } var prm internalclient.PutObjectPrm @@ -63,32 +54,30 @@ func (t *remoteTarget) Close(ctx context.Context) (*transformer.AccessIdentifier prm.SetSessionToken(t.commonPrm.SessionToken()) prm.SetBearerToken(t.commonPrm.BearerToken()) prm.SetXHeaders(t.commonPrm.XHeaders()) - prm.SetObject(t.obj) + prm.SetObject(obj) - res, err := t.putSingle(ctx, prm) + err = t.putSingle(ctx, prm) if status.Code(err) != codes.Unimplemented { - return res, err + return err } return t.putStream(ctx, prm) } -func (t *remoteTarget) putStream(ctx context.Context, prm internalclient.PutObjectPrm) (*transformer.AccessIdentifiers, error) { - res, err := internalclient.PutObject(ctx, prm) +func (t *remoteTarget) putStream(ctx context.Context, prm internalclient.PutObjectPrm) error { + _, err := internalclient.PutObject(ctx, prm) if err != nil { - return nil, fmt.Errorf("(%T) could not put object to %s: %w", t, t.nodeInfo.AddressGroup(), err) + return fmt.Errorf("(%T) could not put object to %s: %w", t, t.nodeInfo.AddressGroup(), err) } - - return &transformer.AccessIdentifiers{SelfID: res.ID()}, nil + return nil } -func (t *remoteTarget) putSingle(ctx context.Context, prm internalclient.PutObjectPrm) (*transformer.AccessIdentifiers, error) { - res, err := internalclient.PutObjectSingle(ctx, prm) +func (t *remoteTarget) putSingle(ctx context.Context, prm internalclient.PutObjectPrm) error { + _, err := internalclient.PutObjectSingle(ctx, prm) if err != nil { - return nil, fmt.Errorf("(%T) could not put single object to %s: %w", t, t.nodeInfo.AddressGroup(), err) + return fmt.Errorf("(%T) could not put single object to %s: %w", t, t.nodeInfo.AddressGroup(), err) } - - return &transformer.AccessIdentifiers{SelfID: res.ID()}, nil + return nil } // NewRemoteSender creates, initializes and returns new RemoteSender instance. @@ -134,9 +123,7 @@ func (s *RemoteSender) PutObject(ctx context.Context, p *RemotePutPrm) error { return fmt.Errorf("parse client node info: %w", err) } - if err := t.WriteObject(p.obj, objectcore.ContentMeta{}); err != nil { - return fmt.Errorf("(%T) could not send object header: %w", s, err) - } else if _, err := t.Close(ctx); err != nil { + if err := t.WriteObject(ctx, p.obj, objectcore.ContentMeta{}); err != nil { return fmt.Errorf("(%T) could not send object: %w", s, err) } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 2dccafa47..80b1c2541 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -224,7 +224,7 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { getWorkerPool: p.getWorkerPool, nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { - return &localTarget{ + return localTarget{ storage: p.localStore, } } From 32c77f3a2376acad9affbd901f8e451be4348c16 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 25 Jul 2023 10:07:38 +0300 Subject: [PATCH 0449/1943] [#537] node: Add runtime.memory_limit config parameter This parameter allows to set soft memory limit for Go GC. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 4 +++ cmd/frostfs-node/config/runtime/config.go | 23 ++++++++++++++ .../config/runtime/config_test.go | 30 +++++++++++++++++++ cmd/frostfs-node/main.go | 1 + cmd/frostfs-node/runtime.go | 26 ++++++++++++++++ config/example/node.env | 2 ++ config/example/node.json | 3 ++ config/example/node.yaml | 3 ++ docs/storage-node-configuration.md | 13 ++++++++ internal/logs/logs.go | 2 ++ 10 files changed, 107 insertions(+) create mode 100644 cmd/frostfs-node/config/runtime/config.go create mode 100644 cmd/frostfs-node/config/runtime/config_test.go create mode 100644 cmd/frostfs-node/runtime.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c088d2f07..283cf501a 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1055,6 +1055,10 @@ func (c *cfg) reloadConfig(ctx context.Context) { } components = append(components, dCmp{"logger", logPrm.Reload}) + components = append(components, dCmp{"runtime", func() error { + setRuntimeParameters(c) + return nil + }}) components = append(components, dCmp{"tracing", func() error { updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg)) if updated { diff --git a/cmd/frostfs-node/config/runtime/config.go b/cmd/frostfs-node/config/runtime/config.go new file mode 100644 index 000000000..ad6cce43b --- /dev/null +++ b/cmd/frostfs-node/config/runtime/config.go @@ -0,0 +1,23 @@ +package runtime + +import ( + "math" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" +) + +const ( + subsection = "runtime" + memoryLimitDefault = math.MaxInt64 +) + +// GCMemoryLimitBytes returns the value of "soft_memory_limit" config parameter from "runtime" section. +func GCMemoryLimitBytes(c *config.Config) int64 { + l := config.SizeInBytesSafe(c.Sub(subsection), "soft_memory_limit") + + if l > 0 { + return int64(l) + } + + return memoryLimitDefault +} diff --git a/cmd/frostfs-node/config/runtime/config_test.go b/cmd/frostfs-node/config/runtime/config_test.go new file mode 100644 index 000000000..1bfa42ad8 --- /dev/null +++ b/cmd/frostfs-node/config/runtime/config_test.go @@ -0,0 +1,30 @@ +package runtime + +import ( + "math" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "github.com/stretchr/testify/require" +) + +func TestGCMemoryLimit(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + empty := configtest.EmptyConfig() + + require.Equal(t, int64(math.MaxInt64), GCMemoryLimitBytes(empty)) + }) + + const path = "../../../../config/example/node" + + fileConfigTest := func(c *config.Config) { + require.Equal(t, int64(1073741824), GCMemoryLimitBytes(c)) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) +} diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 425cf25a0..bf872da03 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -84,6 +84,7 @@ func initApp(ctx context.Context, c *cfg) { c.wg.Done() }() + setRuntimeParameters(c) metrics, _ := metricsComponent(c) initAndLog(c, "profiler", initProfilerService) initAndLog(c, metrics.name, metrics.init) diff --git a/cmd/frostfs-node/runtime.go b/cmd/frostfs-node/runtime.go new file mode 100644 index 000000000..d858ba490 --- /dev/null +++ b/cmd/frostfs-node/runtime.go @@ -0,0 +1,26 @@ +package main + +import ( + "os" + "runtime/debug" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/runtime" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "go.uber.org/zap" +) + +func setRuntimeParameters(c *cfg) { + if len(os.Getenv("GOMEMLIMIT")) != 0 { + // default limit < yaml limit < app env limit < GOMEMLIMIT + c.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) + return + } + + memLimitBytes := runtime.GCMemoryLimitBytes(c.appCfg) + previous := debug.SetMemoryLimit(memLimitBytes) + if memLimitBytes != previous { + c.log.Info(logs.RuntimeSoftMemoryLimitUpdated, + zap.Int64("new_value", memLimitBytes), + zap.Int64("old_value", previous)) + } +} diff --git a/config/example/node.env b/config/example/node.env index 089021767..3abb744be 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -188,3 +188,5 @@ FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m FROSTFS_TRACING_ENABLED=true FROSTFS_TRACING_ENDPOINT="localhost" FROSTFS_TRACING_EXPORTER="otlp_grpc" + +FROSTFS_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 diff --git a/config/example/node.json b/config/example/node.json index e4b85bc81..6c98903f1 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -245,5 +245,8 @@ "enabled": true, "endpoint": "localhost:9090", "exporter": "otlp_grpc" + }, + "runtime": { + "soft_memory_limit": 1073741824 } } diff --git a/config/example/node.yaml b/config/example/node.yaml index 897f4e15b..0ef5fea7f 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -217,3 +217,6 @@ tracing: enabled: true exporter: "otlp_grpc" endpoint: "localhost" + +runtime: + soft_memory_limit: 1gb diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 4469b1e10..439edf598 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -24,6 +24,7 @@ There are some custom types used for brevity: | `policer` | [Policer service configuration](#policer-section) | | `replicator` | [Replicator service configuration](#replicator-section) | | `storage` | [Storage engine configuration](#storage-section) | +| `runtime` | [Runtime configuration](#runtime-section) | # `control` section @@ -426,3 +427,15 @@ object: | `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | | `put.pool_size_remote` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | | `put.pool_size_local` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | + +# `runtime` section +Contains runtime parameters. + +```yaml +runtime: + soft_memory_limit: 1GB +``` + +| Parameter | Type | Default value | Description | +|---------------------|--------|---------------|--------------------------------------------------------------------------| +| `soft_memory_limit` | `size` | 0 | Soft memory limit for the runtime. Zero or no value stands for no limit. If `GOMEMLIMIT` environment variable is set, the value from the configuration file will be ignored. | diff --git a/internal/logs/logs.go b/internal/logs/logs.go index a2ff8dcb9..a400187cc 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -493,4 +493,6 @@ const ( FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled" NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap = "the node is already in candidate list with online state, skip initial bootstrap" + RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" + RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" ) From 286242cad0cea9eb1977be23191be9e469765b3c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Jul 2023 13:40:47 +0300 Subject: [PATCH 0450/1943] [#539] getsvc: Use buffer to assemble object To reduce memory consumption. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assemble.go | 3 ++- pkg/services/object/get/assembler.go | 13 ++++++++----- pkg/services/object/get/get_test.go | 1 + pkg/services/object/get/writer.go | 10 +++++++++- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index 777822ac3..43f2c470b 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -113,8 +113,9 @@ func (r *request) HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Object, return w.Object(), nil } -func (r *request) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range) (*objectSDK.Object, error) { +func (r *request) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, payloadBuffer []byte) (*objectSDK.Object, error) { w := NewSimpleObjectWriter() + w.SetPayloadBuffer(payloadBuffer) p := r.prm p.common = p.common.WithLocalOnly(false) diff --git a/pkg/services/object/get/assembler.go b/pkg/services/object/get/assembler.go index 321d4b16d..2299751e1 100644 --- a/pkg/services/object/get/assembler.go +++ b/pkg/services/object/get/assembler.go @@ -10,7 +10,7 @@ import ( ) type objectGetter interface { - GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range) (*objectSDK.Object, error) + GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, buffer []byte) (*objectSDK.Object, error) HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Object, error) } @@ -77,7 +77,7 @@ func (a *assembler) getLastPartOrLinkObjectID() (oid.ID, bool) { } func (a *assembler) initializeFromSourceObjectID(ctx context.Context, id oid.ID) (*oid.ID, []oid.ID, error) { - sourceObject, err := a.getChildObject(ctx, id, nil, true) + sourceObject, err := a.getChildObject(ctx, id, nil, true, nil) if err != nil { return nil, nil, err } @@ -131,8 +131,8 @@ func (a *assembler) initializeFromSourceObjectID(ctx context.Context, id oid.ID) return nil, sourceObject.Children(), nil } -func (a *assembler) getChildObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, verifyIsChild bool) (*objectSDK.Object, error) { - obj, err := a.objGetter.GetObject(ctx, id, rng) +func (a *assembler) getChildObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, verifyIsChild bool, payloadBuffer []byte) (*objectSDK.Object, error) { + obj, err := a.objGetter.GetObject(ctx, id, rng, payloadBuffer) if err != nil { return nil, err } @@ -176,13 +176,14 @@ func (a *assembler) assemleObjectByPreviousIDInReverse(ctx context.Context, prev func (a *assembler) assemblePayloadByObjectIDs(ctx context.Context, writer ObjectWriter, partIDs []oid.ID, partRanges []objectSDK.Range, verifyIsChild bool) error { withRng := len(partRanges) > 0 && a.rng != nil + var buffer []byte for i := range partIDs { var r *objectSDK.Range if withRng { r = &partRanges[i] } - child, err := a.getChildObject(ctx, partIDs[i], r, verifyIsChild) + child, err := a.getChildObject(ctx, partIDs[i], r, verifyIsChild, buffer) if err != nil { return err } @@ -190,6 +191,8 @@ func (a *assembler) assemblePayloadByObjectIDs(ctx context.Context, writer Objec if err := writer.WriteChunk(ctx, child.Payload()); err != nil { return err } + + buffer = child.Payload() } return nil } diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 9c5506064..b6b77506e 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -629,6 +629,7 @@ func TestGetRemoteSmall(t *testing.T) { *c1, *c2 = *c2, *c1 + w.SetPayloadBuffer(make([]byte, 0)) err = svc.Get(ctx, p) require.NoError(t, err) require.Equal(t, obj, w.Object()) diff --git a/pkg/services/object/get/writer.go b/pkg/services/object/get/writer.go index 1b842adeb..0f8c71f7b 100644 --- a/pkg/services/object/get/writer.go +++ b/pkg/services/object/get/writer.go @@ -49,10 +49,18 @@ func NewSimpleObjectWriter() *SimpleObjectWriter { } } +func (s *SimpleObjectWriter) SetPayloadBuffer(buffer []byte) { + if buffer != nil { + s.pld = buffer[:0] + } +} + func (s *SimpleObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Object) error { s.obj = obj - s.pld = make([]byte, 0, obj.PayloadSize()) + if s.pld == nil { + s.pld = make([]byte, 0, obj.PayloadSize()) + } return nil } From 99bb488ebdc031db91a7a5b65a115699223f61a9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Jul 2023 14:34:56 +0300 Subject: [PATCH 0451/1943] [#539] getsvc: Write payload direct to out stream To reduce memory allocations. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assemble.go | 9 +++++---- pkg/services/object/get/assembler.go | 19 +++++++------------ pkg/services/object/get/get_test.go | 1 - pkg/services/object/get/writer.go | 26 +++++++++++++++----------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index 43f2c470b..72ad506c7 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -113,9 +113,10 @@ func (r *request) HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Object, return w.Object(), nil } -func (r *request) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, payloadBuffer []byte) (*objectSDK.Object, error) { - w := NewSimpleObjectWriter() - w.SetPayloadBuffer(payloadBuffer) +func (r *request) GetObjectAndWritePayload(ctx context.Context, id oid.ID, rng *objectSDK.Range, writer ChunkWriter) (*objectSDK.Object, error) { + w := &payloadWriter{ + origin: writer, + } p := r.prm p.common = p.common.WithLocalOnly(false) @@ -128,7 +129,7 @@ func (r *request) GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range if err := r.getObjectWithIndependentRequest(ctx, p); err != nil { return nil, err } - return w.Object(), nil + return w.obj, nil } func (r *request) getObjectWithIndependentRequest(ctx context.Context, prm RequestParameters) error { diff --git a/pkg/services/object/get/assembler.go b/pkg/services/object/get/assembler.go index 2299751e1..61de0ee99 100644 --- a/pkg/services/object/get/assembler.go +++ b/pkg/services/object/get/assembler.go @@ -10,7 +10,7 @@ import ( ) type objectGetter interface { - GetObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, buffer []byte) (*objectSDK.Object, error) + GetObjectAndWritePayload(ctx context.Context, id oid.ID, rng *objectSDK.Range, writer ChunkWriter) (*objectSDK.Object, error) HeadObject(ctx context.Context, id oid.ID) (*objectSDK.Object, error) } @@ -77,10 +77,12 @@ func (a *assembler) getLastPartOrLinkObjectID() (oid.ID, bool) { } func (a *assembler) initializeFromSourceObjectID(ctx context.Context, id oid.ID) (*oid.ID, []oid.ID, error) { - sourceObject, err := a.getChildObject(ctx, id, nil, true, nil) + w := NewSimpleObjectWriter() + sourceObject, err := a.getChildObject(ctx, id, nil, true, w) if err != nil { return nil, nil, err } + sourceObject.SetPayload(w.pld) parentObject := sourceObject.Parent() if parentObject == nil { @@ -131,8 +133,8 @@ func (a *assembler) initializeFromSourceObjectID(ctx context.Context, id oid.ID) return nil, sourceObject.Children(), nil } -func (a *assembler) getChildObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, verifyIsChild bool, payloadBuffer []byte) (*objectSDK.Object, error) { - obj, err := a.objGetter.GetObject(ctx, id, rng, payloadBuffer) +func (a *assembler) getChildObject(ctx context.Context, id oid.ID, rng *objectSDK.Range, verifyIsChild bool, writer ChunkWriter) (*objectSDK.Object, error) { + obj, err := a.objGetter.GetObjectAndWritePayload(ctx, id, rng, writer) if err != nil { return nil, err } @@ -176,23 +178,16 @@ func (a *assembler) assemleObjectByPreviousIDInReverse(ctx context.Context, prev func (a *assembler) assemblePayloadByObjectIDs(ctx context.Context, writer ObjectWriter, partIDs []oid.ID, partRanges []objectSDK.Range, verifyIsChild bool) error { withRng := len(partRanges) > 0 && a.rng != nil - var buffer []byte for i := range partIDs { var r *objectSDK.Range if withRng { r = &partRanges[i] } - child, err := a.getChildObject(ctx, partIDs[i], r, verifyIsChild, buffer) + _, err := a.getChildObject(ctx, partIDs[i], r, verifyIsChild, writer) if err != nil { return err } - - if err := writer.WriteChunk(ctx, child.Payload()); err != nil { - return err - } - - buffer = child.Payload() } return nil } diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index b6b77506e..9c5506064 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -629,7 +629,6 @@ func TestGetRemoteSmall(t *testing.T) { *c1, *c2 = *c2, *c1 - w.SetPayloadBuffer(make([]byte, 0)) err = svc.Get(ctx, p) require.NoError(t, err) require.Equal(t, obj, w.Object()) diff --git a/pkg/services/object/get/writer.go b/pkg/services/object/get/writer.go index 0f8c71f7b..3aa4d66ac 100644 --- a/pkg/services/object/get/writer.go +++ b/pkg/services/object/get/writer.go @@ -49,19 +49,9 @@ func NewSimpleObjectWriter() *SimpleObjectWriter { } } -func (s *SimpleObjectWriter) SetPayloadBuffer(buffer []byte) { - if buffer != nil { - s.pld = buffer[:0] - } -} - func (s *SimpleObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Object) error { s.obj = obj - - if s.pld == nil { - s.pld = make([]byte, 0, obj.PayloadSize()) - } - + s.pld = make([]byte, 0, obj.PayloadSize()) return nil } @@ -90,3 +80,17 @@ func (h *hasherWrapper) WriteChunk(_ context.Context, p []byte) error { _, err := h.hash.Write(p) return err } + +type payloadWriter struct { + origin ChunkWriter + obj *objectSDK.Object +} + +func (w *payloadWriter) WriteChunk(ctx context.Context, p []byte) error { + return w.origin.WriteChunk(ctx, p) +} + +func (w *payloadWriter) WriteHeader(_ context.Context, o *objectSDK.Object) error { + w.obj = o + return nil +} From ea329134308c5aad5d26b9885f41c65f9835f0ad Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 27 Jul 2023 16:28:02 +0300 Subject: [PATCH 0452/1943] [#543] putsvc: Fix PutSingle implementation Add Lock and Delete handlers to local PutSingle. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/single.go | 48 +++++++++++++++++++------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 200830e15..b5a88c502 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -18,6 +18,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -63,11 +64,12 @@ func (s *Service) PutSingle(ctx context.Context, req *objectAPI.PutSingleRequest obj := objectSDK.NewFromV2(req.GetBody().GetObject()) - if err := s.validatePutSingle(ctx, obj); err != nil { + meta, err := s.validatePutSingle(ctx, obj) + if err != nil { return nil, err } - if err := s.saveToNodes(ctx, obj, req); err != nil { + if err := s.saveToNodes(ctx, obj, req, meta); err != nil { return nil, err } @@ -76,13 +78,13 @@ func (s *Service) PutSingle(ctx context.Context, req *objectAPI.PutSingleRequest return resp, nil } -func (s *Service) validatePutSingle(ctx context.Context, obj *objectSDK.Object) error { +func (s *Service) validatePutSingle(ctx context.Context, obj *objectSDK.Object) (object.ContentMeta, error) { if err := s.validarePutSingleSize(obj); err != nil { - return err + return object.ContentMeta{}, err } if err := s.validatePutSingleChecksum(obj); err != nil { - return err + return object.ContentMeta{}, err } return s.validatePutSingleObject(ctx, obj) @@ -129,20 +131,20 @@ func (s *Service) validatePutSingleChecksum(obj *objectSDK.Object) error { return nil } -func (s *Service) validatePutSingleObject(ctx context.Context, obj *objectSDK.Object) error { +func (s *Service) validatePutSingleObject(ctx context.Context, obj *objectSDK.Object) (object.ContentMeta, error) { if err := s.fmtValidator.Validate(ctx, obj, false); err != nil { - return fmt.Errorf("coud not validate object format: %w", err) + return object.ContentMeta{}, fmt.Errorf("coud not validate object format: %w", err) } - _, err := s.fmtValidator.ValidateContent(obj) + meta, err := s.fmtValidator.ValidateContent(obj) if err != nil { - return fmt.Errorf("could not validate payload content: %w", err) + return object.ContentMeta{}, fmt.Errorf("could not validate payload content: %w", err) } - return nil + return meta, nil } -func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *objectAPI.PutSingleRequest) error { +func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { localOnly := req.GetMetaHeader().GetTTL() <= 1 placementOptions, err := s.getPutSinglePlacementOptions(obj, req.GetBody().GetCopiesNumber(), localOnly) if err != nil { @@ -160,7 +162,7 @@ func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *o keyStorage: s.keyStorage, signer: &sync.Once{}, } - return s.saveAccordingToPlacement(ctx, obj, signer, traversal) + return s.saveAccordingToPlacement(ctx, obj, signer, traversal, meta) } func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) ([]placement.Option, error) { @@ -198,7 +200,8 @@ func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumb return result, nil } -func (s *Service) saveAccordingToPlacement(ctx context.Context, obj *objectSDK.Object, signer *putSingleRequestSigner, traversal *traversal) error { +func (s *Service) saveAccordingToPlacement(ctx context.Context, obj *objectSDK.Object, signer *putSingleRequestSigner, + traversal *traversal, meta object.ContentMeta) error { traverser, err := placement.NewTraverser(traversal.opts...) if err != nil { return fmt.Errorf("could not create object placement traverser: %w", err) @@ -211,7 +214,7 @@ func (s *Service) saveAccordingToPlacement(ctx context.Context, obj *objectSDK.O break } - if stop := s.saveToPlacementNodes(ctx, obj, signer, traversal, traverser, addrs, &resultError); stop { + if stop := s.saveToPlacementNodes(ctx, obj, signer, traversal, traverser, addrs, meta, &resultError); stop { break } } @@ -223,7 +226,7 @@ func (s *Service) saveAccordingToPlacement(ctx context.Context, obj *objectSDK.O } if traversal.submitPrimaryPlacementFinish() { - err = s.saveAccordingToPlacement(ctx, obj, signer, traversal) + err = s.saveAccordingToPlacement(ctx, obj, signer, traversal, meta) if err != nil { s.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) } @@ -238,6 +241,7 @@ func (s *Service) saveToPlacementNodes(ctx context.Context, traversal *traversal, traverser *placement.Traverser, nodeAddresses []placement.Node, + meta object.ContentMeta, resultError *atomic.Value, ) bool { wg := sync.WaitGroup{} @@ -259,7 +263,7 @@ func (s *Service) saveToPlacementNodes(ctx context.Context, if err := workerPool.Submit(func() { defer wg.Done() - err := s.saveToPlacementNode(ctx, &nodeDesc{local: local, info: nodeAddress}, obj, signer) + err := s.saveToPlacementNode(ctx, &nodeDesc{local: local, info: nodeAddress}, obj, signer, meta) traversal.submitProcessed(nodeAddress) @@ -282,9 +286,10 @@ func (s *Service) saveToPlacementNodes(ctx context.Context, return false } -func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *nodeDesc, obj *objectSDK.Object, signer *putSingleRequestSigner) error { +func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *nodeDesc, obj *objectSDK.Object, + signer *putSingleRequestSigner, meta object.ContentMeta) error { if nodeDesc.local { - return s.localStore.Put(ctx, obj) + return s.saveLocal(ctx, obj, meta) } var info client.NodeInfo @@ -299,6 +304,13 @@ func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *nodeDesc, o return s.redirectPutSingleRequest(ctx, signer, obj, info, c) } +func (s *Service) saveLocal(ctx context.Context, obj *objectSDK.Object, meta object.ContentMeta) error { + localTarget := &localTarget{ + storage: s.localStore, + } + return localTarget.WriteObject(ctx, obj, meta) +} + func (s *Service) redirectPutSingleRequest(ctx context.Context, signer *putSingleRequestSigner, obj *objectSDK.Object, From ad5f527bd3b2aa14a664c82e2090ac566c4b56dc Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 27 Jul 2023 19:56:37 +0300 Subject: [PATCH 0453/1943] [#541] writecache/test: Remove initWC() Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/writecache/flush_test.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 2223bef02..1d471e5eb 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -65,7 +65,7 @@ func TestFlush(t *testing.T) { WithBlobstor(bs), }, opts...)...) require.NoError(t, wc.Open(false)) - initWC(t, wc) + require.NoError(t, wc.Init()) // First set mode for metabase and blobstor to prevent background flushes. require.NoError(t, mb.SetMode(mode.ReadOnly)) @@ -229,10 +229,6 @@ func newObject(t *testing.T, size int) (*objectSDK.Object, []byte) { return obj, data } -func initWC(t *testing.T, wc Cache) { - require.NoError(t, wc.Init()) -} - type dummyEpoch struct{} func (dummyEpoch) CurrentEpoch() uint64 { From ec8b4fdc484e8e41938d7913d91f47170eb913a9 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 27 Jul 2023 19:59:00 +0300 Subject: [PATCH 0454/1943] [#541] writecache/test: Close writecache on exit Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/writecache/flush_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 1d471e5eb..997d23a3d 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -64,6 +64,7 @@ func TestFlush(t *testing.T) { WithMetabase(mb), WithBlobstor(bs), }, opts...)...) + t.Cleanup(func() { require.NoError(t, wc.Close()) }) require.NoError(t, wc.Open(false)) require.NoError(t, wc.Init()) From 7b0fdf0202d26026fa81535adfd92e774d081e52 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Jul 2023 15:47:32 +0300 Subject: [PATCH 0455/1943] [#533] services: Assume API supports status codes Signed-off-by: Evgenii Stratonikov --- pkg/services/accounting/sign.go | 2 +- pkg/services/container/sign.go | 28 ++++++++--------- pkg/services/netmap/sign.go | 12 +++---- pkg/services/object/sign.go | 55 ++++++++++++++------------------- pkg/services/session/sign.go | 4 +-- pkg/services/util/sign.go | 16 +--------- 6 files changed, 47 insertions(+), 70 deletions(-) diff --git a/pkg/services/accounting/sign.go b/pkg/services/accounting/sign.go index be7b08a39..cd6ff0307 100644 --- a/pkg/services/accounting/sign.go +++ b/pkg/services/accounting/sign.go @@ -23,5 +23,5 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) Balance(ctx context.Context, req *accounting.BalanceRequest) (*accounting.BalanceResponse, error) { resp, err := util.EnsureNonNilResponse(s.svc.Balance(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } diff --git a/pkg/services/container/sign.go b/pkg/services/container/sign.go index b336f19c3..bba717f60 100644 --- a/pkg/services/container/sign.go +++ b/pkg/services/container/sign.go @@ -24,62 +24,62 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.PutResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Put(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *signService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.DeleteResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *signService) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.GetResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Get(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *signService) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.ListResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.List(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *signService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.SetExtendedACLResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.SetExtendedACL(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *signService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.GetExtendedACLResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.GetExtendedACL(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *signService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.AnnounceUsedSpaceResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.AnnounceUsedSpace(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } diff --git a/pkg/services/netmap/sign.go b/pkg/services/netmap/sign.go index 2d01164a3..305d3443e 100644 --- a/pkg/services/netmap/sign.go +++ b/pkg/services/netmap/sign.go @@ -26,26 +26,26 @@ func (s *signService) LocalNodeInfo( req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(netmap.LocalNodeInfoResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.LocalNodeInfo(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *signService) NetworkInfo(ctx context.Context, req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(netmap.NetworkInfoResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.NetworkInfo(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *signService) Snapshot(ctx context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(netmap.SnapshotResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Snapshot(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index c516872e3..f4f40dd7c 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -19,29 +19,25 @@ type SignService struct { type searchStreamSigner struct { SearchStream - statusSupported bool - sigSvc *util.SignService + sigSvc *util.SignService nonEmptyResp bool // set on first Send call } type getStreamSigner struct { GetObjectStream - statusSupported bool - sigSvc *util.SignService + sigSvc *util.SignService } type putStreamSigner struct { - sigSvc *util.SignService - stream PutObjectStream - statusSupported bool - err error + sigSvc *util.SignService + stream PutObjectStream + err error } type getRangeStreamSigner struct { GetObjectRangeStream - statusSupported bool - sigSvc *util.SignService + sigSvc *util.SignService } func NewSignService(key *ecdsa.PrivateKey, svc ServiceServer) *SignService { @@ -53,7 +49,7 @@ func NewSignService(key *ecdsa.PrivateKey, svc ServiceServer) *SignService { } func (s *getStreamSigner) Send(resp *object.GetResponse) error { - if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + if err := s.sigSvc.SignResponse(resp, nil); err != nil { return err } return s.GetObjectStream.Send(resp) @@ -62,20 +58,17 @@ func (s *getStreamSigner) Send(resp *object.GetResponse) error { func (s *SignService) Get(req *object.GetRequest, stream GetObjectStream) error { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.GetResponse) - _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + _ = s.sigSvc.SignResponse(resp, err) return stream.Send(resp) } return s.svc.Get(req, &getStreamSigner{ GetObjectStream: stream, sigSvc: s.sigSvc, - statusSupported: util.IsStatusSupported(req), }) } func (s *putStreamSigner) Send(ctx context.Context, req *object.PutRequest) error { - s.statusSupported = util.IsStatusSupported(req) - if s.err = s.sigSvc.VerifyRequest(req); s.err != nil { return util.ErrAbortStream } @@ -96,7 +89,7 @@ func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.PutRes } } - return resp, s.sigSvc.SignResponse(s.statusSupported, resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *SignService) Put() (PutObjectStream, error) { @@ -114,24 +107,24 @@ func (s *SignService) Put() (PutObjectStream, error) { func (s *SignService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.HeadResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Head(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *SignService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.PutSingleResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.PutSingle(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { s.nonEmptyResp = true - if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + if err := s.sigSvc.SignResponse(resp, nil); err != nil { return err } return s.SearchStream.Send(resp) @@ -140,14 +133,13 @@ func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { func (s *SignService) Search(req *object.SearchRequest, stream SearchStream) error { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.SearchResponse) - _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + _ = s.sigSvc.SignResponse(resp, err) return stream.Send(resp) } ss := &searchStreamSigner{ - SearchStream: stream, - sigSvc: s.sigSvc, - statusSupported: util.IsStatusSupported(req), + SearchStream: stream, + sigSvc: s.sigSvc, } err := s.svc.Search(req, ss) if err == nil && !ss.nonEmptyResp { @@ -164,14 +156,14 @@ func (s *SignService) Search(req *object.SearchRequest, stream SearchStream) err func (s *SignService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.DeleteResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Delete(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } func (s *getRangeStreamSigner) Send(resp *object.GetRangeResponse) error { - if err := s.sigSvc.SignResponse(s.statusSupported, resp, nil); err != nil { + if err := s.sigSvc.SignResponse(resp, nil); err != nil { return err } return s.GetObjectRangeStream.Send(resp) @@ -180,22 +172,21 @@ func (s *getRangeStreamSigner) Send(resp *object.GetRangeResponse) error { func (s *SignService) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.GetRangeResponse) - _ = s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + _ = s.sigSvc.SignResponse(resp, err) return stream.Send(resp) } return s.svc.GetRange(req, &getRangeStreamSigner{ GetObjectRangeStream: stream, sigSvc: s.sigSvc, - statusSupported: util.IsStatusSupported(req), }) } func (s *SignService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.GetRangeHashResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.GetRangeHash(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } diff --git a/pkg/services/session/sign.go b/pkg/services/session/sign.go index ffce0621e..690fff896 100644 --- a/pkg/services/session/sign.go +++ b/pkg/services/session/sign.go @@ -24,8 +24,8 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(session.CreateResponse) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } resp, err := util.EnsureNonNilResponse(s.svc.Create(ctx, req)) - return resp, s.sigSvc.SignResponse(util.IsStatusSupported(req), resp, err) + return resp, s.sigSvc.SignResponse(resp, err) } diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index a26bd311c..26586ff49 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -36,12 +36,8 @@ func NewUnarySignService(key *ecdsa.PrivateKey) *SignService { // The signature error affects the result depending on the protocol version: // - if status return is supported, panics since we cannot return the failed status, because it will not be signed. // - otherwise, returns error in order to transport it directly. -func (s *SignService) SignResponse(statusSupported bool, resp ResponseMessage, err error) error { +func (s *SignService) SignResponse(resp ResponseMessage, err error) error { if err != nil { - if !statusSupported { - return err - } - setStatusV2(resp, err) } @@ -70,16 +66,6 @@ func EnsureNonNilResponse[T any](resp *T, err error) (*T, error) { return new(T), err } -// IsStatusSupported returns true iff request version implies expecting status return. -// This allows us to handle protocol versions <=2.10 (API statuses was introduced in 2.11 only). -func IsStatusSupported(req RequestMessage) bool { - version := req.GetMetaHeader().GetVersion() - - mjr := version.GetMajor() - - return mjr > 2 || mjr == 2 && version.GetMinor() >= 11 -} - func setStatusV2(resp ResponseMessage, err error) { // unwrap error for e := errors.Unwrap(err); e != nil; e = errors.Unwrap(err) { From 05ac9e36375ffeae0818a4eae2ff2c931aa759bc Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Jul 2023 16:01:02 +0300 Subject: [PATCH 0456/1943] [#547] objectsvc: Work with `traversal` struct from a single thread Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/distributed.go | 20 ++++++-------------- pkg/services/object/put/single.go | 5 ++--- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index cf5cc558e..f4afd44aa 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -47,9 +47,6 @@ type traversal struct { // need of additional broadcast after the object is saved extraBroadcastEnabled bool - // mtx protects mExclude map. - mtx sync.RWMutex - // container nodes which was processed during the primary object placement mExclude map[string]struct{} } @@ -75,21 +72,17 @@ func (x *traversal) submitProcessed(n placement.Node) { if x.extraBroadcastEnabled { key := string(n.PublicKey()) - x.mtx.Lock() if x.mExclude == nil { x.mExclude = make(map[string]struct{}, 1) } x.mExclude[key] = struct{}{} - x.mtx.Unlock() } } // checks if specified node was processed during the primary object placement. func (x *traversal) processed(n placement.Node) bool { - x.mtx.RLock() _, ok := x.mExclude[string(n.PublicKey())] - x.mtx.RUnlock() return ok } @@ -235,13 +228,6 @@ func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *pla defer wg.Done() err := t.sendObject(ctx, nodeDesc{local: isLocal, info: addr}) - - // mark the container node as processed in order to exclude it - // in subsequent container broadcast. Note that we don't - // process this node during broadcast if primary placement - // on it failed. - t.traversal.submitProcessed(addr) - if err != nil { resErr.Store(err) svcutil.LogServiceError(t.log, "PUT", addr.Addresses(), err) @@ -254,6 +240,12 @@ func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *pla svcutil.LogWorkerPoolError(t.log, "PUT", err) return true } + + // mark the container node as processed in order to exclude it + // in subsequent container broadcast. Note that we don't + // process this node during broadcast if primary placement + // on it failed. + t.traversal.submitProcessed(addr) } wg.Wait() diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index b5a88c502..9d17a8128 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -154,7 +154,6 @@ func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *o opts: placementOptions, extraBroadcastEnabled: len(obj.Children()) > 0 || (!localOnly && (obj.Type() == objectSDK.TypeTombstone || obj.Type() == objectSDK.TypeLock)), - mtx: sync.RWMutex{}, mExclude: make(map[string]struct{}), } signer := &putSingleRequestSigner{ @@ -265,8 +264,6 @@ func (s *Service) saveToPlacementNodes(ctx context.Context, err := s.saveToPlacementNode(ctx, &nodeDesc{local: local, info: nodeAddress}, obj, signer, meta) - traversal.submitProcessed(nodeAddress) - if err != nil { resultError.Store(err) svcutil.LogServiceError(s.log, "PUT", nodeAddress.Addresses(), err) @@ -279,6 +276,8 @@ func (s *Service) saveToPlacementNodes(ctx context.Context, svcutil.LogWorkerPoolError(s.log, "PUT", err) return true } + + traversal.submitProcessed(nodeAddress) } wg.Wait() From 35dc64bd7b29dbdb1ae51428096f080d22ecee1b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 26 Jul 2023 16:02:08 +0300 Subject: [PATCH 0457/1943] [#547] metabase: Fix datarace in tests Quite an old one bf9e938a3b2829199. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/put.go | 1 + pkg/local_object_storage/metabase/select.go | 1 + pkg/local_object_storage/metabase/util.go | 2 -- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 7a24485b3..28f12851f 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -111,6 +111,7 @@ func (db *DB) put(tx *bbolt.Tx, exists, err := db.exists(tx, objectCore.AddressOf(obj), currEpoch) + var splitInfoError *objectSDK.SplitInfoError if errors.As(err, &splitInfoError) { exists = true // object exists, however it is virtual } else if err != nil { diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 6a7acd7cc..8b086a89f 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -427,6 +427,7 @@ func (db *DB) selectObjectID( addr.SetContainer(cnr) addr.SetObject(id) + var splitInfoError *objectSDK.SplitInfoError ok, err := db.exists(tx, addr, currEpoch) if (err == nil && ok) || errors.As(err, &splitInfoError) { raw := make([]byte, objectKeySize) diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index 4e58ec20b..c9d9bb947 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -120,8 +120,6 @@ const ( addressKeySize = cidSize + objectKeySize ) -var splitInfoError *objectSDK.SplitInfoError // for errors.As comparisons - func bucketName(cnr cid.ID, prefix byte, key []byte) []byte { key[0] = prefix cnr.Encode(key[1:]) From b8052c794ef9ea6b51c9581dadc9c44ad6a9d8bb Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 31 Jul 2023 14:35:07 +0300 Subject: [PATCH 0458/1943] [#547] go.mod: Update api-go, sdk-go Signed-off-by: Evgenii Stratonikov --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 2e7d8fd56..05e720a75 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230719100335-582d94c81c74 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230726155259-7a5ee927c8a2 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230719130356-5defed4ab435 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230731113339-0fe0d716784b git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index c8737ebdc..e3310bbe5 100644 --- a/go.sum +++ b/go.sum @@ -36,16 +36,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230719100335-582d94c81c74 h1:xgeSzZP40SbMMVGfG1V7xrC8m7DS6llZ9kkjO9L8+jY= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230719100335-582d94c81c74/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230726155259-7a5ee927c8a2 h1:5QUnWafFc0RprHCpQ3d1T4MEo2fvGjL/3GPeB0w54mA= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230726155259-7a5ee927c8a2/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a h1:EeaOunifY30ATPzOo1F32vUDW299UXDHYukZrIVxBZ0= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230719130356-5defed4ab435 h1:N6T2OF5nyxizGFb3bV6GaSn5HsTurJG7RvF25SKHg0A= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230719130356-5defed4ab435/go.mod h1:3GPr75siKbjGwUUz09nXfd98RBMozlDUdtKmtgxcCG8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230731113339-0fe0d716784b h1:RJT771u77ezYJxUVzvCH1uLxZ+P4BAuWU7X87zj9z8E= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230731113339-0fe0d716784b/go.mod h1:NWY5C3PfVNBOv/phNh9J86ZQdsXoItRNPx8GyPeleFU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 9ca63ac8c35f15075597eb7264b7331a72d6dc0e Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 31 Jul 2023 16:56:11 +0300 Subject: [PATCH 0459/1943] [#542] Fix bug in `SizeInBytesSafe` function Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config/cast.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config/cast.go b/cmd/frostfs-node/config/cast.go index cf0017ca5..9036c3ab0 100644 --- a/cmd/frostfs-node/config/cast.go +++ b/cmd/frostfs-node/config/cast.go @@ -199,7 +199,7 @@ func parseSizeInBytes(sizeStr string) uint64 { if sizeStr[lastChar] == 'b' || sizeStr[lastChar] == 'B' { lastChar-- } - if lastChar > 0 { + if lastChar >= 0 { switch unicode.ToLower(rune(sizeStr[lastChar])) { case 'k': multiplier = 1 << 10 From ec8a631d311e052ae37f5ef740acbd60babf169a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 31 Jul 2023 16:57:08 +0300 Subject: [PATCH 0460/1943] [#542] Update test for `SizeInBytesSafe` function Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config/cast_test.go | 5 +++++ cmd/frostfs-node/config/test/config.json | 9 +++++++-- cmd/frostfs-node/config/test/config.yaml | 9 +++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-node/config/cast_test.go b/cmd/frostfs-node/config/cast_test.go index ef48bdc5b..f8c1ee28e 100644 --- a/cmd/frostfs-node/config/cast_test.go +++ b/cmd/frostfs-node/config/cast_test.go @@ -125,10 +125,15 @@ func TestSizeInBytes(t *testing.T) { ) configtest.ForEachFileType("test/config", func(c *config.Config) { c = c.Sub("sizes") + require.EqualValues(t, 1, config.SizeInBytesSafe(c, "size_b")) + require.EqualValues(t, kb, config.SizeInBytesSafe(c, "size_k")) require.EqualValues(t, kb, config.SizeInBytesSafe(c, "size_kb")) require.EqualValues(t, 2*kb, config.SizeInBytesSafe(c, "size_kb_no_space")) + require.EqualValues(t, 12*mb, config.SizeInBytesSafe(c, "size_m")) require.EqualValues(t, 12*mb, config.SizeInBytesSafe(c, "size_mb")) + require.EqualValues(t, 4*gb, config.SizeInBytesSafe(c, "size_g")) require.EqualValues(t, 4*gb, config.SizeInBytesSafe(c, "size_gb")) + require.EqualValues(t, 5*tb, config.SizeInBytesSafe(c, "size_t")) require.EqualValues(t, 5*tb, config.SizeInBytesSafe(c, "size_tb")) require.EqualValues(t, 12, config.SizeInBytesSafe(c, "size_i_am_not_very_clever")) require.EqualValues(t, tb/2, config.SizeInBytesSafe(c, "size_float")) diff --git a/cmd/frostfs-node/config/test/config.json b/cmd/frostfs-node/config/test/config.json index 45635c7ab..7eec1b73e 100644 --- a/cmd/frostfs-node/config/test/config.json +++ b/cmd/frostfs-node/config/test/config.json @@ -49,10 +49,15 @@ }, "sizes": { + "size_b": "1b", + "size_k": "1 k", "size_kb": "1 kb", "size_kb_no_space": "2kb", - "size_mb": "12m", - "size_gb": "4g", + "size_m": "12m", + "size_mb": "12mb", + "size_g": "4g", + "size_gb": "4gb", + "size_t": "5 T", "size_tb": "5 TB", "size_float": ".5t", "size_float_big": "14.123 gb", diff --git a/cmd/frostfs-node/config/test/config.yaml b/cmd/frostfs-node/config/test/config.yaml index 102c045a1..3ae459ea2 100644 --- a/cmd/frostfs-node/config/test/config.yaml +++ b/cmd/frostfs-node/config/test/config.yaml @@ -42,10 +42,15 @@ bool: incorrect: not true sizes: + size_b: 1b + size_k: 1 k size_kb: 1 kb size_kb_no_space: 2kb - size_mb: 12m - size_gb: 4g + size_m: 12m + size_mb: 12mb + size_g: 4g + size_gb: 4gb + size_t: 5 T size_tb: 5 TB size_float: .5t size_float_big: 14.123 gb From b3695411d907c3c65485bab04f9ff8479a72906b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 1 Aug 2023 16:21:34 +0300 Subject: [PATCH 0461/1943] [#553] eacl: Fix bug with casting to ObjectAccessDenied error Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/services/object/acl/v2/errors.go | 4 +-- pkg/services/object/acl/v2/errors_test.go | 30 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 pkg/services/object/acl/v2/errors_test.go diff --git a/pkg/services/object/acl/v2/errors.go b/pkg/services/object/acl/v2/errors.go index 70c1ae991..7fc20d618 100644 --- a/pkg/services/object/acl/v2/errors.go +++ b/pkg/services/object/acl/v2/errors.go @@ -25,14 +25,14 @@ const accessDeniedACLReasonFmt = "access to operation %s is denied by basic ACL const accessDeniedEACLReasonFmt = "access to operation %s is denied by extended ACL check: %v" func basicACLErr(info RequestInfo) error { - var errAccessDenied apistatus.ObjectAccessDenied + errAccessDenied := &apistatus.ObjectAccessDenied{} errAccessDenied.WriteReason(fmt.Sprintf(accessDeniedACLReasonFmt, info.operation)) return errAccessDenied } func eACLErr(info RequestInfo, err error) error { - var errAccessDenied apistatus.ObjectAccessDenied + errAccessDenied := &apistatus.ObjectAccessDenied{} errAccessDenied.WriteReason(fmt.Sprintf(accessDeniedEACLReasonFmt, info.operation, err)) return errAccessDenied diff --git a/pkg/services/object/acl/v2/errors_test.go b/pkg/services/object/acl/v2/errors_test.go new file mode 100644 index 000000000..2d2b7bc8d --- /dev/null +++ b/pkg/services/object/acl/v2/errors_test.go @@ -0,0 +1,30 @@ +package v2 + +import ( + "errors" + "testing" + + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "github.com/stretchr/testify/require" +) + +func TestBasicACLErr(t *testing.T) { + var reqInfo RequestInfo + err := basicACLErr(reqInfo) + + var errAccessDenied *apistatus.ObjectAccessDenied + + require.ErrorAs(t, err, &errAccessDenied, + "basicACLErr must be able to be casted to apistatus.ObjectAccessDenied") +} + +func TestEACLErr(t *testing.T) { + var reqInfo RequestInfo + testErr := errors.New("test-eacl") + err := eACLErr(reqInfo, testErr) + + var errAccessDenied *apistatus.ObjectAccessDenied + + require.ErrorAs(t, err, &errAccessDenied, + "eACLErr must be able to be casted to apistatus.ObjectAccessDenied") +} From 1b7b54ba8931d4783e34f902dbbabf55842a2111 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 24 Jul 2023 15:23:25 +0300 Subject: [PATCH 0462/1943] [#540] cli: refactor client parameters usage Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-cli/modules/container/create.go | 13 +++++++------ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index e5bbeacd4..29c2422ed 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -13,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -99,13 +100,13 @@ It will be stored in sidechain when inner ring will accepts it.`, _, err = internalclient.SyncContainerSettings(cmd.Context(), syncContainerPrm) commonCmd.ExitOnErr(cmd, "syncing container's settings rpc error: %w", err) - var putPrm internalclient.PutContainerPrm - putPrm.SetClient(cli) - putPrm.SetContainer(cnr) - - if tok != nil { - putPrm.WithinSession(*tok) + putPrm := internalclient.PutContainerPrm{ + PrmContainerPut: client.PrmContainerPut{ + Container: &cnr, + Session: tok, + }, } + putPrm.SetClient(cli) res, err := internalclient.PutContainer(cmd.Context(), putPrm) commonCmd.ExitOnErr(cmd, "put container rpc error: %w", err) diff --git a/go.mod b/go.mod index 05e720a75..980086dcd 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230726155259-7a5ee927c8a2 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230731113339-0fe0d716784b + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230801095957-13d0b170d2ad git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index e3310bbe5..e7c11c7a9 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230731113339-0fe0d716784b h1:RJT771u77ezYJxUVzvCH1uLxZ+P4BAuWU7X87zj9z8E= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230731113339-0fe0d716784b/go.mod h1:NWY5C3PfVNBOv/phNh9J86ZQdsXoItRNPx8GyPeleFU= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230801095957-13d0b170d2ad h1:jD6Dv6Eg/HApmQOFoJPSVXP9xQF/KSVoSf9AZkmKyRA= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230801095957-13d0b170d2ad/go.mod h1:NWY5C3PfVNBOv/phNh9J86ZQdsXoItRNPx8GyPeleFU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 93c46cfdf00727dd3e5dbecdcd03ce96847d71e1 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 1 Aug 2023 13:03:34 +0300 Subject: [PATCH 0463/1943] [#550] cli: make get-op-log meta pretty formatted Close #550 Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-cli/modules/tree/get_op_log.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/tree/get_op_log.go b/cmd/frostfs-cli/modules/tree/get_op_log.go index 6efa76133..b1b307f62 100644 --- a/cmd/frostfs-cli/modules/tree/get_op_log.go +++ b/cmd/frostfs-cli/modules/tree/get_op_log.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/spf13/cobra" @@ -75,7 +76,15 @@ func getOpLog(cmd *cobra.Command, _ []string) { cmd.Println("Parent ID: ", o.GetParentId()) cmd.Println("\tChild ID: ", o.GetChildId()) - cmd.Printf("\tMeta: %s\n", o.GetMeta()) + + m := &pilorama.Meta{} + err = m.FromBytes(o.GetMeta()) + commonCmd.ExitOnErr(cmd, "could not unmarshal meta: %w", err) + cmd.Printf("\tMeta:\n") + cmd.Printf("\t\tTime: %d\n", m.Time) + for _, item := range m.Items { + cmd.Printf("\t\t%s: %s\n", item.Key, item.Value) + } } if !errors.Is(err, io.EOF) { commonCmd.ExitOnErr(cmd, "get op log response stream: %w", err) From 435a581b5ee93b3bc93a5c7505b5ffba6c33546f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 2 Aug 2023 12:47:34 +0300 Subject: [PATCH 0464/1943] [#557] go.mod: Update sdk-go and api-go Signed-off-by: Evgenii Stratonikov --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 980086dcd..f6be26a9e 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.19 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230726155259-7a5ee927c8a2 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230801095957-13d0b170d2ad + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802090410-95b987b818d9 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index e7c11c7a9..e13be1d2d 100644 --- a/go.sum +++ b/go.sum @@ -36,16 +36,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230726155259-7a5ee927c8a2 h1:5QUnWafFc0RprHCpQ3d1T4MEo2fvGjL/3GPeB0w54mA= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230726155259-7a5ee927c8a2/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 h1:v6JqBD/VzZx3QSxbaXnUwnnJ1KEYheU4LzLGr3IhsAE= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a h1:EeaOunifY30ATPzOo1F32vUDW299UXDHYukZrIVxBZ0= git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230801095957-13d0b170d2ad h1:jD6Dv6Eg/HApmQOFoJPSVXP9xQF/KSVoSf9AZkmKyRA= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230801095957-13d0b170d2ad/go.mod h1:NWY5C3PfVNBOv/phNh9J86ZQdsXoItRNPx8GyPeleFU= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802090410-95b987b818d9 h1:sCCTrMISLhfwqzbmUFvNW8MbwO1jzIHYgcEkxmokLd4= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802090410-95b987b818d9/go.mod h1:LI2GOj0pEx0jYTjB3QHja2PNhQFYL2pCm71RAFwDv0M= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 1e8b4b8a17c4f5cb6c8fa129801a4f327b97b589 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 2 Aug 2023 12:48:17 +0300 Subject: [PATCH 0465/1943] [#557] services: Regenerate stable marshalers Signed-off-by: Evgenii Stratonikov --- pkg/services/control/ir/service_frostfs.pb.go | 36 +++++ pkg/services/control/ir/types_frostfs.pb.go | 3 + pkg/services/control/service_frostfs.pb.go | 150 ++++++++++++++++++ pkg/services/control/types_frostfs.pb.go | 18 +++ pkg/services/tree/service_frostfs.pb.go | 126 +++++++++++++++ pkg/services/tree/types_frostfs.pb.go | 9 ++ 6 files changed, 342 insertions(+) diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index c93253105..f39d3b043 100644 --- a/pkg/services/control/ir/service_frostfs.pb.go +++ b/pkg/services/control/ir/service_frostfs.pb.go @@ -8,6 +8,9 @@ import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" // // Structures with the same field values have the same binary size. func (x *HealthCheckRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -27,6 +30,9 @@ func (x *HealthCheckRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *HealthCheckRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -79,6 +85,9 @@ func (x *HealthCheckRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *HealthCheckResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.EnumSize(1, int32(x.HealthStatus)) return size } @@ -107,6 +116,9 @@ func (x *HealthCheckResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *HealthCheckResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -159,6 +171,9 @@ func (x *HealthCheckResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *TickEpochRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -178,6 +193,9 @@ func (x *TickEpochRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *TickEpochRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -230,6 +248,9 @@ func (x *TickEpochRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *TickEpochResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -249,6 +270,9 @@ func (x *TickEpochResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *TickEpochResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -301,6 +325,9 @@ func (x *TickEpochResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *RemoveNodeRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.Key) return size } @@ -329,6 +356,9 @@ func (x *RemoveNodeRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *RemoveNodeRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -381,6 +411,9 @@ func (x *RemoveNodeRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *RemoveNodeResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -400,6 +433,9 @@ func (x *RemoveNodeResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *RemoveNodeResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size diff --git a/pkg/services/control/ir/types_frostfs.pb.go b/pkg/services/control/ir/types_frostfs.pb.go index 50679e785..ef2fc458e 100644 --- a/pkg/services/control/ir/types_frostfs.pb.go +++ b/pkg/services/control/ir/types_frostfs.pb.go @@ -8,6 +8,9 @@ import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" // // Structures with the same field values have the same binary size. func (x *Signature) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.Key) size += proto.BytesSize(2, x.Sign) return size diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 979c22685..cc96a98df 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -8,6 +8,9 @@ import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" // // Structures with the same field values have the same binary size. func (x *HealthCheckRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -27,6 +30,9 @@ func (x *HealthCheckRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *HealthCheckRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -79,6 +85,9 @@ func (x *HealthCheckRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *HealthCheckResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.EnumSize(1, int32(x.NetmapStatus)) size += proto.EnumSize(2, int32(x.HealthStatus)) return size @@ -109,6 +118,9 @@ func (x *HealthCheckResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *HealthCheckResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -161,6 +173,9 @@ func (x *HealthCheckResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *SetNetmapStatusRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.EnumSize(1, int32(x.Status)) size += proto.BoolSize(2, x.ForceMaintenance) return size @@ -191,6 +206,9 @@ func (x *SetNetmapStatusRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *SetNetmapStatusRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -243,6 +261,9 @@ func (x *SetNetmapStatusRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *SetNetmapStatusResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -262,6 +283,9 @@ func (x *SetNetmapStatusResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *SetNetmapStatusResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -314,6 +338,9 @@ func (x *SetNetmapStatusResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *DropObjectsRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.RepeatedBytesSize(1, x.AddressList) return size } @@ -342,6 +369,9 @@ func (x *DropObjectsRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *DropObjectsRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -394,6 +424,9 @@ func (x *DropObjectsRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *DropObjectsResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -413,6 +446,9 @@ func (x *DropObjectsResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *DropObjectsResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -465,6 +501,9 @@ func (x *DropObjectsResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *ListShardsRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -484,6 +523,9 @@ func (x *ListShardsRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *ListShardsRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -536,6 +578,9 @@ func (x *ListShardsRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *ListShardsResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } for i := range x.Shards { size += proto.NestedStructureSize(1, x.Shards[i]) } @@ -568,6 +613,9 @@ func (x *ListShardsResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *ListShardsResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -620,6 +668,9 @@ func (x *ListShardsResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *SetShardModeRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.RepeatedBytesSize(1, x.Shard_ID) size += proto.EnumSize(2, int32(x.Mode)) size += proto.BoolSize(3, x.ResetErrorCounter) @@ -652,6 +703,9 @@ func (x *SetShardModeRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *SetShardModeRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -704,6 +758,9 @@ func (x *SetShardModeRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *SetShardModeResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -723,6 +780,9 @@ func (x *SetShardModeResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *SetShardModeResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -775,6 +835,9 @@ func (x *SetShardModeResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *SynchronizeTreeRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.ContainerId) size += proto.StringSize(2, x.TreeId) size += proto.UInt64Size(3, x.Height) @@ -807,6 +870,9 @@ func (x *SynchronizeTreeRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *SynchronizeTreeRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -859,6 +925,9 @@ func (x *SynchronizeTreeRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *SynchronizeTreeResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -878,6 +947,9 @@ func (x *SynchronizeTreeResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *SynchronizeTreeResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -930,6 +1002,9 @@ func (x *SynchronizeTreeResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *EvacuateShardRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.RepeatedBytesSize(1, x.Shard_ID) size += proto.BoolSize(2, x.IgnoreErrors) return size @@ -960,6 +1035,9 @@ func (x *EvacuateShardRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *EvacuateShardRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1012,6 +1090,9 @@ func (x *EvacuateShardRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *EvacuateShardResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.UInt32Size(1, x.Count) return size } @@ -1040,6 +1121,9 @@ func (x *EvacuateShardResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *EvacuateShardResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1092,6 +1176,9 @@ func (x *EvacuateShardResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *FlushCacheRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.RepeatedBytesSize(1, x.Shard_ID) return size } @@ -1120,6 +1207,9 @@ func (x *FlushCacheRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *FlushCacheRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1172,6 +1262,9 @@ func (x *FlushCacheRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *FlushCacheResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -1191,6 +1284,9 @@ func (x *FlushCacheResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *FlushCacheResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1243,6 +1339,9 @@ func (x *FlushCacheResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *DoctorRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.UInt32Size(1, x.Concurrency) size += proto.BoolSize(2, x.RemoveDuplicates) return size @@ -1273,6 +1372,9 @@ func (x *DoctorRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *DoctorRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1325,6 +1427,9 @@ func (x *DoctorRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *DoctorResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -1344,6 +1449,9 @@ func (x *DoctorResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *DoctorResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1396,6 +1504,9 @@ func (x *DoctorResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *StartShardEvacuationRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.RepeatedBytesSize(1, x.Shard_ID) size += proto.BoolSize(2, x.IgnoreErrors) return size @@ -1426,6 +1537,9 @@ func (x *StartShardEvacuationRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *StartShardEvacuationRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1478,6 +1592,9 @@ func (x *StartShardEvacuationRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *StartShardEvacuationResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -1497,6 +1614,9 @@ func (x *StartShardEvacuationResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *StartShardEvacuationResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1549,6 +1669,9 @@ func (x *StartShardEvacuationResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *GetShardEvacuationStatusRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -1568,6 +1691,9 @@ func (x *GetShardEvacuationStatusRequest_Body) StableMarshal(buf []byte) []byte // // Structures with the same field values have the same binary size. func (x *GetShardEvacuationStatusRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1620,6 +1746,9 @@ func (x *GetShardEvacuationStatusRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.Int64Size(1, x.Value) return size } @@ -1648,6 +1777,9 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) StableMarshal(buf // // Structures with the same field values have the same binary size. func (x *GetShardEvacuationStatusResponse_Body_Duration) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.Int64Size(1, x.Seconds) return size } @@ -1676,6 +1808,9 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) StableMarshal(buf []byt // // Structures with the same field values have the same binary size. func (x *GetShardEvacuationStatusResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.UInt64Size(1, x.Total) size += proto.UInt64Size(2, x.Evacuated) size += proto.UInt64Size(3, x.Failed) @@ -1718,6 +1853,9 @@ func (x *GetShardEvacuationStatusResponse_Body) StableMarshal(buf []byte) []byte // // Structures with the same field values have the same binary size. func (x *GetShardEvacuationStatusResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1770,6 +1908,9 @@ func (x *GetShardEvacuationStatusResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *StopShardEvacuationRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -1789,6 +1930,9 @@ func (x *StopShardEvacuationRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *StopShardEvacuationRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1841,6 +1985,9 @@ func (x *StopShardEvacuationRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *StopShardEvacuationResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -1860,6 +2007,9 @@ func (x *StopShardEvacuationResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *StopShardEvacuationResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size diff --git a/pkg/services/control/types_frostfs.pb.go b/pkg/services/control/types_frostfs.pb.go index 1d05102b3..ab246dbdc 100644 --- a/pkg/services/control/types_frostfs.pb.go +++ b/pkg/services/control/types_frostfs.pb.go @@ -8,6 +8,9 @@ import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" // // Structures with the same field values have the same binary size. func (x *Signature) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.Key) size += proto.BytesSize(2, x.Sign) return size @@ -38,6 +41,9 @@ func (x *Signature) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *NodeInfo_Attribute) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.StringSize(1, x.Key) size += proto.StringSize(2, x.Value) size += proto.RepeatedStringSize(3, x.Parents) @@ -70,6 +76,9 @@ func (x *NodeInfo_Attribute) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *NodeInfo) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.PublicKey) size += proto.RepeatedStringSize(2, x.Addresses) for i := range x.Attributes { @@ -108,6 +117,9 @@ func (x *NodeInfo) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *Netmap) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.UInt64Size(1, x.Epoch) for i := range x.Nodes { size += proto.NestedStructureSize(2, x.Nodes[i]) @@ -142,6 +154,9 @@ func (x *Netmap) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *ShardInfo) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.Shard_ID) size += proto.StringSize(2, x.MetabasePath) for i := range x.Blobstor { @@ -186,6 +201,9 @@ func (x *ShardInfo) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *BlobstorInfo) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.StringSize(1, x.Path) size += proto.StringSize(2, x.Type) return size diff --git a/pkg/services/tree/service_frostfs.pb.go b/pkg/services/tree/service_frostfs.pb.go index b272e4389..4b50af2aa 100644 --- a/pkg/services/tree/service_frostfs.pb.go +++ b/pkg/services/tree/service_frostfs.pb.go @@ -8,6 +8,9 @@ import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" // // Structures with the same field values have the same binary size. func (x *AddRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.ContainerId) size += proto.StringSize(2, x.TreeId) size += proto.UInt64Size(3, x.ParentId) @@ -48,6 +51,9 @@ func (x *AddRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *AddRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -100,6 +106,9 @@ func (x *AddRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *AddResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.UInt64Size(1, x.NodeId) return size } @@ -128,6 +137,9 @@ func (x *AddResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *AddResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -180,6 +192,9 @@ func (x *AddResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *AddByPathRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.ContainerId) size += proto.StringSize(2, x.TreeId) size += proto.StringSize(3, x.PathAttribute) @@ -222,6 +237,9 @@ func (x *AddByPathRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *AddByPathRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -274,6 +292,9 @@ func (x *AddByPathRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *AddByPathResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } var n int n, _ = proto.RepeatedUInt64Size(1, x.Nodes) size += n @@ -306,6 +327,9 @@ func (x *AddByPathResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *AddByPathResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -358,6 +382,9 @@ func (x *AddByPathResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *RemoveRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.ContainerId) size += proto.StringSize(2, x.TreeId) size += proto.UInt64Size(3, x.NodeId) @@ -392,6 +419,9 @@ func (x *RemoveRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *RemoveRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -444,6 +474,9 @@ func (x *RemoveRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *RemoveResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -463,6 +496,9 @@ func (x *RemoveResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *RemoveResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -515,6 +551,9 @@ func (x *RemoveResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *MoveRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.ContainerId) size += proto.StringSize(2, x.TreeId) size += proto.UInt64Size(3, x.ParentId) @@ -557,6 +596,9 @@ func (x *MoveRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *MoveRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -609,6 +651,9 @@ func (x *MoveRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *MoveResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -628,6 +673,9 @@ func (x *MoveResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *MoveResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -680,6 +728,9 @@ func (x *MoveResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *GetNodeByPathRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.ContainerId) size += proto.StringSize(2, x.TreeId) size += proto.StringSize(3, x.PathAttribute) @@ -722,6 +773,9 @@ func (x *GetNodeByPathRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *GetNodeByPathRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -774,6 +828,9 @@ func (x *GetNodeByPathRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *GetNodeByPathResponse_Info) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.UInt64Size(1, x.NodeId) size += proto.UInt64Size(2, x.Timestamp) for i := range x.Meta { @@ -812,6 +869,9 @@ func (x *GetNodeByPathResponse_Info) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *GetNodeByPathResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } for i := range x.Nodes { size += proto.NestedStructureSize(1, x.Nodes[i]) } @@ -844,6 +904,9 @@ func (x *GetNodeByPathResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *GetNodeByPathResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -896,6 +959,9 @@ func (x *GetNodeByPathResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *GetSubTreeRequest_Body_Order) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.EnumSize(1, int32(x.Direction)) return size } @@ -924,6 +990,9 @@ func (x *GetSubTreeRequest_Body_Order) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *GetSubTreeRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.ContainerId) size += proto.StringSize(2, x.TreeId) size += proto.UInt64Size(3, x.RootId) @@ -962,6 +1031,9 @@ func (x *GetSubTreeRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *GetSubTreeRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1014,6 +1086,9 @@ func (x *GetSubTreeRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *GetSubTreeResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.UInt64Size(1, x.NodeId) size += proto.UInt64Size(2, x.ParentId) size += proto.UInt64Size(3, x.Timestamp) @@ -1052,6 +1127,9 @@ func (x *GetSubTreeResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *GetSubTreeResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1104,6 +1182,9 @@ func (x *GetSubTreeResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *TreeListRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.ContainerId) return size } @@ -1132,6 +1213,9 @@ func (x *TreeListRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *TreeListRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1184,6 +1268,9 @@ func (x *TreeListRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *TreeListResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.RepeatedStringSize(1, x.Ids) return size } @@ -1212,6 +1299,9 @@ func (x *TreeListResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *TreeListResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1264,6 +1354,9 @@ func (x *TreeListResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *ApplyRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.ContainerId) size += proto.StringSize(2, x.TreeId) size += proto.NestedStructureSize(3, x.Operation) @@ -1296,6 +1389,9 @@ func (x *ApplyRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *ApplyRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1348,6 +1444,9 @@ func (x *ApplyRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *ApplyResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -1367,6 +1466,9 @@ func (x *ApplyResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *ApplyResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1419,6 +1521,9 @@ func (x *ApplyResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *GetOpLogRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.ContainerId) size += proto.StringSize(2, x.TreeId) size += proto.UInt64Size(3, x.Height) @@ -1453,6 +1558,9 @@ func (x *GetOpLogRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *GetOpLogRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1505,6 +1613,9 @@ func (x *GetOpLogRequest) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *GetOpLogResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Operation) return size } @@ -1533,6 +1644,9 @@ func (x *GetOpLogResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *GetOpLogResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1585,6 +1699,9 @@ func (x *GetOpLogResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *HealthcheckResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -1604,6 +1721,9 @@ func (x *HealthcheckResponse_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *HealthcheckResponse) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size @@ -1656,6 +1776,9 @@ func (x *HealthcheckResponse) SetSignature(sig *Signature) { // // Structures with the same field values have the same binary size. func (x *HealthcheckRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } return size } @@ -1675,6 +1798,9 @@ func (x *HealthcheckRequest_Body) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *HealthcheckRequest) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.NestedStructureSize(1, x.Body) size += proto.NestedStructureSize(2, x.Signature) return size diff --git a/pkg/services/tree/types_frostfs.pb.go b/pkg/services/tree/types_frostfs.pb.go index 8e3c6a026..707fcc3cc 100644 --- a/pkg/services/tree/types_frostfs.pb.go +++ b/pkg/services/tree/types_frostfs.pb.go @@ -8,6 +8,9 @@ import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" // // Structures with the same field values have the same binary size. func (x *KeyValue) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.StringSize(1, x.Key) size += proto.BytesSize(2, x.Value) return size @@ -38,6 +41,9 @@ func (x *KeyValue) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *LogMove) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.UInt64Size(1, x.ParentId) size += proto.BytesSize(2, x.Meta) size += proto.UInt64Size(3, x.ChildId) @@ -70,6 +76,9 @@ func (x *LogMove) StableMarshal(buf []byte) []byte { // // Structures with the same field values have the same binary size. func (x *Signature) StableSize() (size int) { + if x == nil { + return 0 + } size += proto.BytesSize(1, x.Key) size += proto.BytesSize(2, x.Sign) return size From 65c72f3e0b0a2095625f6f7b278645bec7173199 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 3 Aug 2023 10:00:49 +0300 Subject: [PATCH 0466/1943] [#559] Remove manual path handling in fstree tests Signed-off-by: Alejandro Lopez --- .../blobstor/fstree/generic_test.go | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/generic_test.go b/pkg/local_object_storage/blobstor/fstree/generic_test.go index 49f487d35..757482c78 100644 --- a/pkg/local_object_storage/blobstor/fstree/generic_test.go +++ b/pkg/local_object_storage/blobstor/fstree/generic_test.go @@ -1,9 +1,6 @@ package fstree import ( - "os" - "path/filepath" - "strconv" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -11,39 +8,31 @@ import ( ) func TestGeneric(t *testing.T) { - defer func() { _ = os.RemoveAll(t.Name()) }() - - helper := func(t *testing.T, dir string) common.Storage { + newTreeFromPath := func(path string) common.Storage { return New( - WithPath(dir), + WithPath(path), WithDepth(2), WithDirNameLen(2)) } - var n int newTree := func(t *testing.T) common.Storage { - dir := filepath.Join(t.Name(), strconv.Itoa(n)) - return helper(t, dir) + return newTreeFromPath(t.TempDir()) } blobstortest.TestAll(t, newTree, 2048, 16*1024) t.Run("info", func(t *testing.T) { - dir := filepath.Join(t.Name(), "info") - blobstortest.TestInfo(t, func(t *testing.T) common.Storage { - return helper(t, dir) - }, Type, dir) + path := t.TempDir() + blobstortest.TestInfo(t, func(*testing.T) common.Storage { + return newTreeFromPath(path) + }, Type, path) }) } func TestControl(t *testing.T) { - defer func() { _ = os.RemoveAll(t.Name()) }() - - var n int newTree := func(t *testing.T) common.Storage { - dir := filepath.Join(t.Name(), strconv.Itoa(n)) return New( - WithPath(dir), + WithPath(t.TempDir()), WithDepth(2), WithDirNameLen(2)) } From 1a0cb0f34a3239df8a27be865528b5e41eafb8b0 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 22 Jun 2023 14:55:30 +0300 Subject: [PATCH 0467/1943] [#421] Try using badger for the write-cache Signed-off-by: Alejandro Lopez --- cmd/frostfs-lens/internal/flags.go | 7 + .../internal/writecache/inspect.go | 35 +- cmd/frostfs-lens/internal/writecache/list.go | 30 +- cmd/frostfs-lens/internal/writecache/root.go | 13 +- cmd/frostfs-node/config.go | 53 ++- .../config/engine/shard/writecache/config.go | 36 ++ config/example/node.json | 1 + config/example/node.yaml | 1 + docs/storage-node-configuration.md | 2 + go.mod | 12 + go.sum | 19 + internal/logs/logs.go | 372 +++++++++--------- .../engine/control_test.go | 53 +-- pkg/local_object_storage/engine/shards.go | 5 +- .../internal/testutil/object.go | 6 + .../shard/control_test.go | 13 +- pkg/local_object_storage/shard/gc_test.go | 11 +- pkg/local_object_storage/shard/range_test.go | 13 +- pkg/local_object_storage/shard/shard.go | 40 +- pkg/local_object_storage/shard/shard_test.go | 24 +- .../shard/shutdown_test.go | 12 +- .../writecache/benchmark/writecache_test.go | 52 +++ .../writecache/config/config.go | 22 ++ pkg/local_object_storage/writecache/doc.go | 11 - .../writecache/flush_test.go | 237 ----------- .../writecache/generic_test.go | 30 -- .../writecache/metrics.go | 20 +- .../writecache/writecache.go | 144 +------ .../writecachebadger/cachebadger.go | 129 ++++++ .../writecache/writecachebadger/delete.go | 70 ++++ .../writecache/writecachebadger/flush.go | 257 ++++++++++++ .../writecache/writecachebadger/flush_test.go | 65 +++ .../writecache/writecachebadger/gc.go | 31 ++ .../writecachebadger/generic_test.go | 20 + .../writecache/writecachebadger/get.go | 95 +++++ .../writecache/writecachebadger/iterate.go | 32 ++ .../writecache/{ => writecachebadger}/mode.go | 9 +- .../writecache/writecachebadger/options.go | 141 +++++++ .../writecache/writecachebadger/put.go | 82 ++++ .../writecache/writecachebadger/state.go | 57 +++ .../writecache/writecachebadger/storage.go | 91 +++++ .../writecache/writecachebadger/util.go | 36 ++ .../writecache/writecachebbolt/cachebbolt.go | 146 +++++++ .../{ => writecachebbolt}/delete.go | 11 +- .../writecache/{ => writecachebbolt}/flush.go | 11 +- .../writecache/writecachebbolt/flush_test.go | 106 +++++ .../writecachebbolt/generic_test.go | 18 + .../writecache/{ => writecachebbolt}/get.go | 9 +- .../{ => writecachebbolt}/iterate.go | 2 +- .../writecache/writecachebbolt/mode.go | 75 ++++ .../{ => writecachebbolt}/options.go | 7 +- .../writecache/{ => writecachebbolt}/put.go | 11 +- .../writecache/{ => writecachebbolt}/state.go | 2 +- .../{ => writecachebbolt}/storage.go | 7 +- .../writecache/{ => writecachebbolt}/util.go | 2 +- .../writecache/writecachetest/flush.go | 185 +++++++++ 56 files changed, 2234 insertions(+), 747 deletions(-) create mode 100644 pkg/local_object_storage/writecache/benchmark/writecache_test.go create mode 100644 pkg/local_object_storage/writecache/config/config.go delete mode 100644 pkg/local_object_storage/writecache/doc.go delete mode 100644 pkg/local_object_storage/writecache/flush_test.go delete mode 100644 pkg/local_object_storage/writecache/generic_test.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/cachebadger.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/delete.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/flush.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/flush_test.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/gc.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/generic_test.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/get.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/iterate.go rename pkg/local_object_storage/writecache/{ => writecachebadger}/mode.go (82%) create mode 100644 pkg/local_object_storage/writecache/writecachebadger/options.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/put.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/state.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/storage.go create mode 100644 pkg/local_object_storage/writecache/writecachebadger/util.go create mode 100644 pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go rename pkg/local_object_storage/writecache/{ => writecachebbolt}/delete.go (87%) rename pkg/local_object_storage/writecache/{ => writecachebbolt}/flush.go (94%) create mode 100644 pkg/local_object_storage/writecache/writecachebbolt/flush_test.go create mode 100644 pkg/local_object_storage/writecache/writecachebbolt/generic_test.go rename pkg/local_object_storage/writecache/{ => writecachebbolt}/get.go (92%) rename pkg/local_object_storage/writecache/{ => writecachebbolt}/iterate.go (97%) create mode 100644 pkg/local_object_storage/writecache/writecachebbolt/mode.go rename pkg/local_object_storage/writecache/{ => writecachebbolt}/options.go (96%) rename pkg/local_object_storage/writecache/{ => writecachebbolt}/put.go (91%) rename pkg/local_object_storage/writecache/{ => writecachebbolt}/state.go (98%) rename pkg/local_object_storage/writecache/{ => writecachebbolt}/storage.go (94%) rename pkg/local_object_storage/writecache/{ => writecachebbolt}/util.go (95%) create mode 100644 pkg/local_object_storage/writecache/writecachetest/flush.go diff --git a/cmd/frostfs-lens/internal/flags.go b/cmd/frostfs-lens/internal/flags.go index 8a987a2d4..95710f7c6 100644 --- a/cmd/frostfs-lens/internal/flags.go +++ b/cmd/frostfs-lens/internal/flags.go @@ -8,6 +8,7 @@ const ( flagAddress = "address" flagEnginePath = "path" flagOutFile = "out" + flagDBType = "dbtype" ) // AddAddressFlag adds the address flag to the passed cobra command. @@ -33,3 +34,9 @@ func AddOutputFileFlag(cmd *cobra.Command, v *string) { "File to save object payload") _ = cmd.MarkFlagFilename(flagOutFile) } + +// AddDBTypeFlag adds the DB type flag to the passed cobra command. +func AddDBTypeFlag(cmd *cobra.Command, v *string) { + cmd.Flags().StringVar(v, flagOutFile, "bbolt", + "Type of DB used by write cache (default: bbolt)") +} diff --git a/cmd/frostfs-lens/internal/writecache/inspect.go b/cmd/frostfs-lens/internal/writecache/inspect.go index 7d3c8ab22..1a733513b 100644 --- a/cmd/frostfs-lens/internal/writecache/inspect.go +++ b/cmd/frostfs-lens/internal/writecache/inspect.go @@ -1,8 +1,13 @@ package writecache import ( + "fmt" + "os" + common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/spf13/cobra" ) @@ -18,14 +23,34 @@ func init() { common.AddAddressFlag(inspectCMD, &vAddress) common.AddComponentPathFlag(inspectCMD, &vPath) common.AddOutputFileFlag(inspectCMD, &vOut) + common.AddDBTypeFlag(inspectCMD, &vDBType) } func inspectFunc(cmd *cobra.Command, _ []string) { - db := openWC(cmd) - defer db.Close() + var data []byte - data, err := writecache.Get(db, []byte(vAddress)) - common.ExitOnErr(cmd, common.Errf("could not fetch object: %w", err)) + switch vDBType { + case "bbolt": + db, err := writecachebbolt.OpenDB(vPath, true, os.OpenFile) + common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) + defer db.Close() + + data, err = writecachebbolt.Get(db, []byte(vAddress)) + common.ExitOnErr(cmd, common.Errf("could not fetch object: %w", err)) + + case "badger": + log, err := logger.NewLogger(&logger.Prm{}) + common.ExitOnErr(cmd, common.Errf("could not create logger: %w", err)) + + db, err := writecachebadger.OpenDB(vPath, true, log) + common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) + + data, err = writecachebadger.Get(db, []byte(vAddress)) + common.ExitOnErr(cmd, common.Errf("could not fetch object: %w", err)) + + default: + common.ExitOnErr(cmd, fmt.Errorf("invalid dbtype: %q (possible values: bbolt, badger)", vDBType)) + } var o objectSDK.Object common.ExitOnErr(cmd, common.Errf("could not unmarshal object: %w", o.Unmarshal(data))) diff --git a/cmd/frostfs-lens/internal/writecache/list.go b/cmd/frostfs-lens/internal/writecache/list.go index f6d0cfff0..df02a82f7 100644 --- a/cmd/frostfs-lens/internal/writecache/list.go +++ b/cmd/frostfs-lens/internal/writecache/list.go @@ -3,9 +3,12 @@ package writecache import ( "fmt" "io" + "os" common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -30,9 +33,26 @@ func listFunc(cmd *cobra.Command, _ []string) { return err } - db := openWC(cmd) - defer db.Close() + switch vDBType { + case "bbolt": + db, err := writecachebbolt.OpenDB(vPath, true, os.OpenFile) + common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) + defer db.Close() - err := writecache.IterateDB(db, wAddr) - common.ExitOnErr(cmd, common.Errf("write-cache iterator failure: %w", err)) + err = writecachebbolt.IterateDB(db, wAddr) + common.ExitOnErr(cmd, common.Errf("write-cache iterator failure: %w", err)) + + case "badger": + log, err := logger.NewLogger(&logger.Prm{}) + common.ExitOnErr(cmd, common.Errf("could not create logger: %w", err)) + + db, err := writecachebadger.OpenDB(vPath, true, log) + common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) + + err = writecachebadger.IterateDB(db, wAddr) + common.ExitOnErr(cmd, common.Errf("write-cache iterator failure: %w", err)) + + default: + common.ExitOnErr(cmd, fmt.Errorf("invalid dbtype: %q (possible values: bbolt, badger)", vDBType)) + } } diff --git a/cmd/frostfs-lens/internal/writecache/root.go b/cmd/frostfs-lens/internal/writecache/root.go index 4a1305848..11a8bb96b 100644 --- a/cmd/frostfs-lens/internal/writecache/root.go +++ b/cmd/frostfs-lens/internal/writecache/root.go @@ -1,18 +1,14 @@ package writecache import ( - "os" - - common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "github.com/spf13/cobra" - "go.etcd.io/bbolt" ) var ( vAddress string vPath string vOut string + vDBType string ) // Root contains `write-cache` command definition. @@ -24,10 +20,3 @@ var Root = &cobra.Command{ func init() { Root.AddCommand(listCMD, inspectCMD) } - -func openWC(cmd *cobra.Command) *bbolt.DB { - db, err := writecache.OpenDB(vPath, true, os.OpenFile) - common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) - - return db -} diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 283cf501a..2a84805dc 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -40,7 +40,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" shardmode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" containerClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" @@ -127,6 +129,7 @@ type shardCfg struct { writecacheCfg struct { enabled bool + typ writecacheconfig.Type path string maxBatchSize int maxBatchDelay time.Duration @@ -135,6 +138,7 @@ type shardCfg struct { flushWorkerCount int sizeLimit uint64 noSync bool + gcInterval time.Duration } piloramaCfg struct { @@ -238,6 +242,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc := &newConfig.writecacheCfg wc.enabled = true + wc.typ = writeCacheCfg.Type() wc.path = writeCacheCfg.Path() wc.maxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize() wc.maxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay() @@ -246,6 +251,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc.flushWorkerCount = writeCacheCfg.WorkersNumber() wc.sizeLimit = writeCacheCfg.SizeLimit() wc.noSync = writeCacheCfg.NoSync() + wc.gcInterval = writeCacheCfg.GCInterval() } } @@ -704,20 +710,37 @@ func (c *cfg) shardOpts() []shardOptsWithID { return shards } -func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { - var writeCacheOpts []writecache.Option +func (c *cfg) getWriteCacheOpts(shCfg shardCfg) writecacheconfig.Options { + var writeCacheOpts writecacheconfig.Options if wcRead := shCfg.writecacheCfg; wcRead.enabled { - writeCacheOpts = append(writeCacheOpts, - writecache.WithPath(wcRead.path), - writecache.WithMaxBatchSize(wcRead.maxBatchSize), - writecache.WithMaxBatchDelay(wcRead.maxBatchDelay), - writecache.WithMaxObjectSize(wcRead.maxObjSize), - writecache.WithSmallObjectSize(wcRead.smallObjectSize), - writecache.WithFlushWorkersCount(wcRead.flushWorkerCount), - writecache.WithMaxCacheSize(wcRead.sizeLimit), - writecache.WithNoSync(wcRead.noSync), - writecache.WithLogger(c.log), - ) + switch wcRead.typ { + case writecacheconfig.TypeBBolt: + writeCacheOpts.Type = writecacheconfig.TypeBBolt + writeCacheOpts.BBoltOptions = append(writeCacheOpts.BBoltOptions, + writecachebbolt.WithPath(wcRead.path), + writecachebbolt.WithMaxBatchSize(wcRead.maxBatchSize), + writecachebbolt.WithMaxBatchDelay(wcRead.maxBatchDelay), + writecachebbolt.WithMaxObjectSize(wcRead.maxObjSize), + writecachebbolt.WithSmallObjectSize(wcRead.smallObjectSize), + writecachebbolt.WithFlushWorkersCount(wcRead.flushWorkerCount), + writecachebbolt.WithMaxCacheSize(wcRead.sizeLimit), + writecachebbolt.WithNoSync(wcRead.noSync), + writecachebbolt.WithLogger(c.log), + ) + case writecacheconfig.TypeBadger: + writeCacheOpts.Type = writecacheconfig.TypeBBolt + writeCacheOpts.BadgerOptions = append(writeCacheOpts.BadgerOptions, + writecachebadger.WithPath(wcRead.path), + writecachebadger.WithMaxObjectSize(wcRead.maxObjSize), + writecachebadger.WithFlushWorkersCount(wcRead.flushWorkerCount), + writecachebadger.WithMaxCacheSize(wcRead.sizeLimit), + writecachebadger.WithNoSync(wcRead.noSync), + writecachebadger.WithLogger(c.log), + writecachebadger.WithGCInterval(wcRead.gcInterval), + ) + default: + panic(fmt.Sprintf("unknown writecache type: %q", wcRead.typ)) + } } return writeCacheOpts } @@ -836,7 +859,7 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { shard.WithMetaBaseOptions(mbOptions...), shard.WithPiloramaOptions(piloramaOpts...), shard.WithWriteCache(shCfg.writecacheCfg.enabled), - shard.WithWriteCacheOptions(writeCacheOpts...), + shard.WithWriteCacheOptions(writeCacheOpts), shard.WithRemoverBatchSize(shCfg.gcCfg.removerBatchSize), shard.WithGCRemoverSleepInterval(shCfg.gcCfg.removerSleepInterval), shard.WithExpiredCollectorBatchSize(shCfg.gcCfg.expiredCollectorBatchSize), diff --git a/cmd/frostfs-node/config/engine/shard/writecache/config.go b/cmd/frostfs-node/config/engine/shard/writecache/config.go index c003cefa6..504fe3ca2 100644 --- a/cmd/frostfs-node/config/engine/shard/writecache/config.go +++ b/cmd/frostfs-node/config/engine/shard/writecache/config.go @@ -1,8 +1,12 @@ package writecacheconfig import ( + "fmt" + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb" + writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" ) // Config is a wrapper over the config section @@ -21,6 +25,9 @@ const ( // SizeLimitDefault is a default write-cache size limit. SizeLimitDefault = 1 << 30 + + // DefaultGCInterval is the default duration of the GC cycle interval. + DefaultGCInterval = 1 * time.Minute ) // From wraps config section into Config. @@ -35,6 +42,22 @@ func (x *Config) Enabled() bool { return config.Bool((*config.Config)(x), "enabled") } +// Type returns the writecache implementation type to use. +// +// Panics if the type is not recognized. +func (x *Config) Type() writecacheconfig.Type { + t := config.String((*config.Config)(x), "type") + + switch t { + case "bbolt", "": + return writecacheconfig.TypeBBolt + case "badger": + return writecacheconfig.TypeBadger + } + + panic(fmt.Sprintf("invalid writecache type: %q", t)) +} + // Path returns the value of "path" config parameter. // // Panics if the value is not a non-empty string. @@ -126,3 +149,16 @@ func (x *Config) NoSync() bool { func (x *Config) BoltDB() *boltdbconfig.Config { return (*boltdbconfig.Config)(x) } + +// GCInterval returns the value of "gc_interval" config parameter. +// +// Returns DefaultGCInterval if the value is not a positive duration. +func (x *Config) GCInterval() time.Duration { + d := config.DurationSafe((*config.Config)(x), "gc_interval") + + if d > 0 { + return d + } + + return DefaultGCInterval +} diff --git a/config/example/node.json b/config/example/node.json index 6c98903f1..6e995112e 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -194,6 +194,7 @@ "resync_metabase": true, "writecache": { "enabled": true, + "type": "bbolt", "path": "tmp/1/cache", "memcache_capacity": 2147483648, "small_object_size": 16384, diff --git a/config/example/node.yaml b/config/example/node.yaml index 0ef5fea7f..acce3741a 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -122,6 +122,7 @@ storage: writecache: enabled: true + type: bbolt small_object_size: 16k # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes max_object_size: 134217728 # size threshold for "big" objects which bypass write-cache and go to the storage directly, bytes workers_number: 30 # number of write-cache flusher threads diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 439edf598..2e2d04088 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -273,6 +273,7 @@ metabase: ```yaml writecache: enabled: true + type: bbolt path: /path/to/writecache capacity: 4294967296 small_object_size: 16384 @@ -282,6 +283,7 @@ writecache: | Parameter | Type | Default value | Description | |----------------------|------------|---------------|----------------------------------------------------------------------------------------------------------------------| +| `type` | `string` | | Type of write cache backing implementation to use (`bbolt`, `badger`). | | `path` | `string` | | Path to the metabase file. | | `capacity` | `size` | unrestricted | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | | `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | diff --git a/go.mod b/go.mod index f6be26a9e..7ad0fad88 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,17 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require ( + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.1.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + go.opencensus.io v0.24.0 // indirect +) + require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect @@ -55,6 +66,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/dgraph-io/badger/v4 v4.1.0 github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index e13be1d2d..07411ad4a 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,14 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/dgraph-io/badger/v4 v4.1.0 h1:E38jc0f+RATYrycSUf9LMv/t47XAy+3CApyYSq4APOQ= +github.com/dgraph-io/badger/v4 v4.1.0/go.mod h1:P50u28d39ibBRmIJuQC/NSdBOg46HnHw7al2SW5QRHg= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -177,12 +185,16 @@ github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8w github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -216,6 +228,8 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -224,6 +238,7 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -554,6 +569,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= @@ -677,6 +694,7 @@ golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -779,6 +797,7 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/logs/logs.go b/internal/logs/logs.go index a400187cc..dbe4ab7d1 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -291,191 +291,193 @@ const ( ShardFailureToMarkLockersAsGarbage = "failure to mark lockers as garbage" // Warn in ../node/pkg/local_object_storage/shard/gc.go ShardFailureToGetExpiredUnlockedObjects = "failure to get expired unlocked objects" // Warn in ../node/pkg/local_object_storage/shard/gc.go ShardCouldNotMarkObjectToDeleteInMetabase = "could not mark object to delete in metabase" // Debug in ../node/pkg/local_object_storage/shard/inhume.go - WritecacheTriedToFlushItemsFromWritecache = "tried to flush items from write-cache" // Debug in ../node/pkg/local_object_storage/writecache/flush.go - WritecacheWaitingForChannelsToFlush = "waiting for channels to flush" // Info in ../node/pkg/local_object_storage/writecache/mode.go - WritecacheFillingFlushMarksForObjectsInFSTree = "filling flush marks for objects in FSTree" // Info in ../node/pkg/local_object_storage/writecache/init.go - WritecacheFinishedUpdatingFSTreeFlushMarks = "finished updating FSTree flush marks" // Info in ../node/pkg/local_object_storage/writecache/init.go - WritecacheFillingFlushMarksForObjectsInDatabase = "filling flush marks for objects in database" // Info in ../node/pkg/local_object_storage/writecache/init.go - WritecacheFinishedUpdatingFlushMarks = "finished updating flush marks" // Info in ../node/pkg/local_object_storage/writecache/init.go - WritecacheCantRemoveObjectsFromTheDatabase = "can't remove objects from the database" // Error in ../node/pkg/local_object_storage/writecache/storage.go - WritecacheCantParseAddress = "can't parse address" // Error in ../node/pkg/local_object_storage/writecache/storage.go - WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" // Error in ../node/pkg/local_object_storage/writecache/storage.go - BlobovniczatreeCouldNotGetObjectFromLevel = "could not get object from level" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go - BlobovniczatreeCouldNotReadPayloadRangeFromOpenedBlobovnicza = "could not read payload range from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go - BlobovniczatreeCouldNotReadPayloadRangeFromActiveBlobovnicza = "could not read payload range from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go - BlobovniczatreeCouldNotCloseBlobovnicza = "could not close Blobovnicza" // Error in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go - BlobovniczatreeBlobovniczaSuccessfullyClosedOnEvict = "blobovnicza successfully closed on evict" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go - BlobovniczatreeUpdatingActiveBlobovnicza = "updating active blobovnicza..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go - BlobovniczatreeActiveBlobovniczaSuccessfullyUpdated = "active blobovnicza successfully updated" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go - BlobovniczatreeBlobovniczaSuccessfullyActivated = "blobovnicza successfully activated" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go - BlobovniczatreeCouldNotRemoveObjectFromLevel = "could not remove object from level" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go - BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza = "could not remove object from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go - BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza = "could not remove object from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go - BlobovniczatreeCouldNotGetActiveBlobovnicza = "could not get active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go - BlobovniczatreeBlobovniczaOverflowed = "blobovnicza overflowed" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go - BlobovniczatreeCouldNotUpdateActiveBlobovnicza = "could not update active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go - BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza = "could not put object to active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go - BlobovniczatreeCouldNotReadObjectFromOpenedBlobovnicza = "could not read object from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get.go - BlobovniczatreeCouldNotGetObjectFromActiveBlobovnicza = "could not get object from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get.go - BlobovniczatreeInitializingBlobovniczas = "initializing Blobovnicza's" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go - BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization = "read-only mode, skip blobovniczas initialization..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go - BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing = "blobovnicza successfully initialized, closing..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go - BlobovniczatreeCouldNotCloseActiveBlobovnicza = "could not close active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go - AlphabetTick = "tick" // Info in ../node/pkg/innerring/processors/alphabet/handlers.go - AlphabetAlphabetProcessorWorkerPoolDrained = "alphabet processor worker pool drained" // Warn in ../node/pkg/innerring/processors/alphabet/handlers.go - AlphabetNonAlphabetModeIgnoreGasEmissionEvent = "non alphabet mode, ignore gas emission event" // Info in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent = "node is out of alphabet range, ignore gas emission event" // Debug in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetCantInvokeAlphabetEmitMethod = "can't invoke alphabet emit method" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetStorageNodeEmissionIsOff = "storage node emission is off" // Info in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes = "can't get netmap snapshot to emit gas to storage nodes" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetGasEmission = "gas emission" // Debug in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetCantParseNodePublicKey = "can't parse node public key" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetCantTransferGas = "can't transfer gas" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetCantTransferGasToWallet = "can't transfer gas to wallet" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetAlphabetWorkerPool = "alphabet worker pool" // Debug in ../node/pkg/innerring/processors/alphabet/processor.go - BalanceBalanceWorkerPoolDrained = "balance worker pool drained" // Warn in ../node/pkg/innerring/processors/balance/handlers.go - BalanceNonAlphabetModeIgnoreBalanceLock = "non alphabet mode, ignore balance lock" // Info in ../node/pkg/innerring/processors/balance/process_assets.go - BalanceCantSendLockAssetTx = "can't send lock asset tx" // Error in ../node/pkg/innerring/processors/balance/process_assets.go - BalanceBalanceWorkerPool = "balance worker pool" // Debug in ../node/pkg/innerring/processors/balance/processor.go - ContainerContainerWorkerPool = "container worker pool" // Debug in ../node/pkg/innerring/processors/container/processor.go - ContainerContainerProcessorWorkerPoolDrained = "container processor worker pool drained" // Warn in ../node/pkg/innerring/processors/container/handlers.go - ContainerNonAlphabetModeIgnoreContainerPut = "non alphabet mode, ignore container put" // Info in ../node/pkg/innerring/processors/container/process_container.go - ContainerPutContainerCheckFailed = "put container check failed" // Error in ../node/pkg/innerring/processors/container/process_container.go - ContainerCouldNotApprovePutContainer = "could not approve put container" // Error in ../node/pkg/innerring/processors/container/process_container.go - ContainerNonAlphabetModeIgnoreContainerDelete = "non alphabet mode, ignore container delete" // Info in ../node/pkg/innerring/processors/container/process_container.go - ContainerDeleteContainerCheckFailed = "delete container check failed" // Error in ../node/pkg/innerring/processors/container/process_container.go - ContainerCouldNotApproveDeleteContainer = "could not approve delete container" // Error in ../node/pkg/innerring/processors/container/process_container.go - ContainerNonAlphabetModeIgnoreSetEACL = "non alphabet mode, ignore set EACL" // Info in ../node/pkg/innerring/processors/container/process_eacl.go - ContainerSetEACLCheckFailed = "set EACL check failed" // Error in ../node/pkg/innerring/processors/container/process_eacl.go - ContainerCouldNotApproveSetEACL = "could not approve set EACL" // Error in ../node/pkg/innerring/processors/container/process_eacl.go - FrostFSNonAlphabetModeIgnoreBind = "non alphabet mode, ignore bind" // Info in ../node/pkg/innerring/processors/frostfs/process_bind.go - FrostFSInvalidManageKeyEvent = "invalid manage key event" // Error in ../node/pkg/innerring/processors/frostfs/process_bind.go - FrostFSCouldNotDecodeScriptHashFromBytes = "could not decode script hash from bytes" // Error in ../node/pkg/innerring/processors/frostfs/process_bind.go - FrostFSNonAlphabetModeIgnoreConfig = "non alphabet mode, ignore config" // Info in ../node/pkg/innerring/processors/frostfs/process_config.go - FrostFSCantRelaySetConfigEvent = "can't relay set config event" // Error in ../node/pkg/innerring/processors/frostfs/process_config.go - FrostFSFrostfsWorkerPool = "frostfs worker pool" // Debug in ../node/pkg/innerring/processors/frostfs/processor.go - FrostFSFrostfsProcessorWorkerPoolDrained = "frostfs processor worker pool drained" // Warn in ../node/pkg/innerring/processors/frostfs/handlers.go - FrostFSNonAlphabetModeIgnoreDeposit = "non alphabet mode, ignore deposit" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantTransferAssetsToBalanceContract = "can't transfer assets to balance contract" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSDoubleMintEmissionDeclined = "double mint emission declined" // Warn in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantGetGasBalanceOfTheNode = "can't get gas balance of the node" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSGasBalanceThresholdHasBeenReached = "gas balance threshold has been reached" // Warn in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantTransferNativeGasToReceiver = "can't transfer native gas to receiver" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSNonAlphabetModeIgnoreWithdraw = "non alphabet mode, ignore withdraw" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantCreateLockAccount = "can't create lock account" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantLockAssetsForWithdraw = "can't lock assets for withdraw" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSNonAlphabetModeIgnoreCheque = "non alphabet mode, ignore cheque" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantTransferAssetsToFedContract = "can't transfer assets to fed contract" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - GovernanceNewEvent = "new event" // Info in ../node/pkg/innerring/processors/governance/handlers.go - GovernanceGovernanceWorkerPoolDrained = "governance worker pool drained" // Warn in ../node/pkg/innerring/processors/governance/handlers.go - GovernanceNonAlphabetModeIgnoreAlphabetSync = "non alphabet mode, ignore alphabet sync" // Info in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantFetchAlphabetListFromMainNet = "can't fetch alphabet list from main net" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantFetchAlphabetListFromSideChain = "can't fetch alphabet list from side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantMergeAlphabetListsFromMainNetAndSideChain = "can't merge alphabet lists from main net and side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged = "no governance update, alphabet list has not been changed" // Info in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceAlphabetListHasBeenChangedStartingUpdate = "alphabet list has been changed, starting update" // Info in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantVoteForSideChainCommittee = "can't vote for side chain committee" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceFinishedAlphabetListUpdate = "finished alphabet list update" // Info in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantFetchInnerRingListFromSideChain = "can't fetch inner ring list from side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys = "can't create new inner ring list with new alphabet keys" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceUpdateOfTheInnerRingList = "update of the inner ring list" // Info in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantUpdateInnerRingListWithNewAlphabetKeys = "can't update inner ring list with new alphabet keys" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantUpdateListOfNotaryNodesInSideChain = "can't update list of notary nodes in side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract = "can't update list of alphabet nodes in frostfs contract" // Error in ../node/pkg/innerring/processors/governance/process_update.go - NetmapNetmapWorkerPool = "netmap worker pool" // Debug in ../node/pkg/innerring/processors/netmap/processor.go - NetmapTick = "tick" // Info in ../node/pkg/innerring/processors/netmap/handlers.go - NetmapNetmapWorkerPoolDrained = "netmap worker pool drained" // Warn in ../node/pkg/innerring/processors/netmap/handlers.go - NetmapNetmapCleanUpRoutineIsDisabled518 = "netmap clean up routine is disabled" // Debug in ../node/pkg/innerring/processors/netmap/handlers.go - NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick = "non alphabet mode, ignore new netmap cleanup tick" // Info in ../node/pkg/innerring/processors/netmap/process_cleanup.go - NetmapCantDecodePublicKeyOfNetmapNode = "can't decode public key of netmap node" // Warn in ../node/pkg/innerring/processors/netmap/process_cleanup.go - NetmapVoteToRemoveNodeFromNetmap = "vote to remove node from netmap" // Info in ../node/pkg/innerring/processors/netmap/process_cleanup.go - NetmapCantInvokeNetmapUpdateState = "can't invoke netmap.UpdateState" // Error in ../node/pkg/innerring/processors/netmap/process_cleanup.go - NetmapCantIterateOnNetmapCleanerCache = "can't iterate on netmap cleaner cache" // Warn in ../node/pkg/innerring/processors/netmap/process_cleanup.go - NetmapCantGetEpochDuration = "can't get epoch duration" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapCantGetTransactionHeight = "can't get transaction height" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapCantResetEpochTimer = "can't reset epoch timer" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapCantGetNetmapSnapshotToPerformCleanup = "can't get netmap snapshot to perform cleanup" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapCantStartContainerSizeEstimation = "can't start container size estimation" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapNonAlphabetModeIgnoreNewEpochTick = "non alphabet mode, ignore new epoch tick" // Info in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapNextEpoch = "next epoch" // Debug in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapCantInvokeNetmapNewEpoch = "can't invoke netmap.NewEpoch" // Error in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapNonAlphabetModeIgnoreNewPeerNotification = "non alphabet mode, ignore new peer notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapNonhaltNotaryTransaction = "non-halt notary transaction" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCantParseNetworkMapCandidate = "can't parse network map candidate" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate = "could not verify and update information about network map candidate" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapApprovingNetworkMapCandidate = "approving network map candidate" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCantInvokeNetmapAddPeer = "can't invoke netmap.AddPeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapNonAlphabetModeIgnoreUpdatePeerNotification = "non alphabet mode, ignore update peer notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapPreventSwitchingNodeToMaintenanceState = "prevent switching node to maintenance state" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go // Debug in ../node/pkg/innerring/processors/reputation/processor.go - FrostFSIRInternalError = "internal error" // Info in ../node/cmd/frostfs-ir/main.go - FrostFSIRCouldNotShutdownHTTPServer = "could not shutdown HTTP server" // Debug in ../node/cmd/frostfs-ir/main.go - FrostFSIRApplicationStopped = "application stopped" // Info in ../node/cmd/frostfs-ir/main.go - FrostFSIRCouldntCreateRPCClientForEndpoint = "could not create RPC client for endpoint" // Debug in ../node/pkg/morph/client/constructor.go - FrostFSIRCreatedRPCClientForEndpoint = "created RPC client for endpoint" // Info in ../node/pkg/morph/client/constructor.go - FrostFSIRReloadExtraWallets = "reload extra wallets" // Info in ../node/cmd/frostfs-ir/config.go - FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" // Error in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" // Error in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeStartListeningGRPCEndpoint = "start listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeStoppingGRPCServer = "stopping gRPC server..." // Info in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeGRPCCannotShutdownGracefullyForcingStop = "gRPC cannot shutdown gracefully, forcing stop" // Info in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeGRPCServerStoppedSuccessfully = "gRPC server stopped successfully" // Info in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeWaitingForAllProcessesToStop = "waiting for all processes to stop" // Debug in ../node/cmd/frostfs-node/main.go - FrostFSNodeStartedLocalNodesMaintenance = "started local node's maintenance" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeStoppedLocalNodesMaintenance = "stopped local node's maintenance" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeFailedToAttachShardToEngine = "failed to attach shard to engine" // Error in ../node/cmd/frostfs-node/config.go - FrostFSNodeShardAttachedToEngine = "shard attached to engine" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeClosingComponentsOfTheStorageEngine = "closing components of the storage engine..." // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeStorageEngineClosingFailure = "storage engine closing failure" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully = "all components of the storage engine closed successfully" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeBootstrappingWithTheMaintenanceState = "bootstrapping with the maintenance state" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeBootstrappingWithOnlineState = "bootstrapping with online state" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeTerminationSignalHasBeenReceivedStopping = "termination signal has been received, stopping..." // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeTerminationSignalProcessingIsComplete = "termination signal processing is complete" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeInternalApplicationError = "internal application error" // Warn in ../node/cmd/frostfs-node/config.go - FrostFSNodeInternalErrorProcessingIsComplete = "internal error processing is complete" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration = "SIGHUP has been received, rereading configuration..." // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeConfigurationReading = "configuration reading" // Error in ../node/cmd/frostfs-node/config.go - FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" // Error in ../node/cmd/frostfs-node/config.go - FrostFSNodeTracingConfigationUpdated = "tracing configation updated" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeStorageEngineConfigurationUpdate = "storage engine configuration update" // Error in ../node/cmd/frostfs-node/config.go - FrostFSNodeUpdatedConfigurationApplying = "updated configuration applying" // Error in ../node/cmd/frostfs-node/config.go - FrostFSNodeConfigurationHasBeenReloadedSuccessfully = "configuration has been reloaded successfully" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeReadNewlyCreatedContainerAfterTheNotification = "read newly created container after the notification" // Error in ../node/cmd/frostfs-node/container.go - FrostFSNodeContainerCreationEventsReceipt = "container creation event's receipt" // Debug in ../node/cmd/frostfs-node/container.go - FrostFSNodeContainerRemovalEventsReceipt = "container removal event's receipt" // Debug in ../node/cmd/frostfs-node/container.go - FrostFSNodeSaveUsedSpaceAnnouncementInContract = "save used space announcement in contract" // Debug in ../node/cmd/frostfs-node/container.go - FrostFSNodeFailedToCalculateContainerSizeInStorageEngine = "failed to calculate container size in storage engine" // Debug in ../node/cmd/frostfs-node/container.go - FrostFSNodeContainerSizeInStorageEngineCalculatedSuccessfully = "container size in storage engine calculated successfully" // Debug in ../node/cmd/frostfs-node/container.go - FrostFSNodeNotificatorCouldNotListContainers = "notificator: could not list containers" // Error in ../node/cmd/frostfs-node/notificator.go - FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer = "notificator: could not select objects from container" // Error in ../node/cmd/frostfs-node/notificator.go - FrostFSNodeNotificatorCouldNotProcessObject = "notificator: could not process object" // Error in ../node/cmd/frostfs-node/notificator.go - FrostFSNodeNotificatorFinishedProcessingObjectNotifications = "notificator: finished processing object notifications" // Debug in ../node/cmd/frostfs-node/notificator.go - FrostFSNodeCouldNotWriteObjectNotification = "could not write object notification" // Warn in ../node/cmd/frostfs-node/notificator.go - FrostFSNodeCouldNotGetMaxObjectSizeValue = "could not get max object size value" // Error in ../node/cmd/frostfs-node/object.go - FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage = "could not inhume mark redundant copy as garbage" // Warn in ../node/cmd/frostfs-node/object.go - FrostFSNodeFailedInitTracing = "failed init tracing" // Error in ../node/cmd/frostfs-node/tracing.go - FrostFSNodeFailedShutdownTracing = "failed shutdown tracing" // Error in ../node/cmd/frostfs-node/tracing.go - FrostFSNodeFailedToCreateNeoRPCClient = "failed to create neo RPC client" // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeClosingMorphComponents = "closing morph components..." // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal = "failed to set group signer scope, continue with Global" // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeNotarySupport = "notary support" // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeMorphcacheTTLFetchedFromNetwork = "morph.cache_ttl fetched from network" // Debug in ../node/cmd/frostfs-node/morph.go - FrostFSNodeNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" // Warn in ../node/cmd/frostfs-node/morph.go - FrostFSNodeNewEpochEventFromSidechain = "new epoch event from sidechain" // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeNewBlock = "new block" // Debug in ../node/cmd/frostfs-node/morph.go - FrostFSNodeCantUpdatePersistentState = "can't update persistent state" // Warn in ../node/cmd/frostfs-node/morph.go - FrostFSNodeCantSendRebootstrapTx = "can't send re-bootstrap tx" // Warn in ../node/cmd/frostfs-node/netmap.go - FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch = "could not update node state on new epoch" // Error in ../node/cmd/frostfs-node/netmap.go - FrostFSNodeCouldNotMakeNotaryDeposit = "could not make notary deposit" // Error in ../node/cmd/frostfs-node/netmap.go - FrostFSNodeInitialNetworkState = "initial network state" // Info in ../node/cmd/frostfs-node/netmap.go - FrostFSNodeTreeServiceIsNotEnabledSkipInitialization = "tree service is not enabled, skip initialization" // Info in ../node/cmd/frostfs-node/tree.go - FrostFSNodeCouldNotSynchronizeTreeService = "could not synchronize Tree Service" // Error in ../node/cmd/frostfs-node/tree.go - FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" // Debug in ../node/cmd/frostfs-node/tree.go - FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" // Error in ../node/cmd/frostfs-node/tree.go - FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" // Error in ../node/cmd/frostfs-node/control.go + WritecacheBadgerInitExperimental = "initializing badger-backed experimental writecache" + WritecacheTriedToFlushItemsFromWritecache = "tried to flush items from write-cache" // Debug in ../node/pkg/local_object_storage/writecache/flush.go + WritecacheWaitingForChannelsToFlush = "waiting for channels to flush" // Info in ../node/pkg/local_object_storage/writecache/mode.go + WritecacheFillingFlushMarksForObjectsInFSTree = "filling flush marks for objects in FSTree" // Info in ../node/pkg/local_object_storage/writecache/init.go + WritecacheFinishedUpdatingFSTreeFlushMarks = "finished updating FSTree flush marks" // Info in ../node/pkg/local_object_storage/writecache/init.go + WritecacheFillingFlushMarksForObjectsInDatabase = "filling flush marks for objects in database" // Info in ../node/pkg/local_object_storage/writecache/init.go + WritecacheFinishedUpdatingFlushMarks = "finished updating flush marks" // Info in ../node/pkg/local_object_storage/writecache/init.go + WritecacheCantRemoveObjectsFromTheDatabase = "can't remove objects from the database" // Error in ../node/pkg/local_object_storage/writecache/storage.go + WritecacheCantParseAddress = "can't parse address" // Error in ../node/pkg/local_object_storage/writecache/storage.go + WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" // Error in ../node/pkg/local_object_storage/writecache/storage.go + WritecacheDBValueLogGCRunCompleted = "value log GC run completed" + BlobovniczatreeCouldNotGetObjectFromLevel = "could not get object from level" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go + BlobovniczatreeCouldNotReadPayloadRangeFromOpenedBlobovnicza = "could not read payload range from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go + BlobovniczatreeCouldNotReadPayloadRangeFromActiveBlobovnicza = "could not read payload range from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go + BlobovniczatreeCouldNotCloseBlobovnicza = "could not close Blobovnicza" // Error in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go + BlobovniczatreeBlobovniczaSuccessfullyClosedOnEvict = "blobovnicza successfully closed on evict" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go + BlobovniczatreeUpdatingActiveBlobovnicza = "updating active blobovnicza..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go + BlobovniczatreeActiveBlobovniczaSuccessfullyUpdated = "active blobovnicza successfully updated" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go + BlobovniczatreeBlobovniczaSuccessfullyActivated = "blobovnicza successfully activated" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go + BlobovniczatreeCouldNotRemoveObjectFromLevel = "could not remove object from level" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go + BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza = "could not remove object from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go + BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza = "could not remove object from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go + BlobovniczatreeCouldNotGetActiveBlobovnicza = "could not get active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go + BlobovniczatreeBlobovniczaOverflowed = "blobovnicza overflowed" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go + BlobovniczatreeCouldNotUpdateActiveBlobovnicza = "could not update active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go + BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza = "could not put object to active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go + BlobovniczatreeCouldNotReadObjectFromOpenedBlobovnicza = "could not read object from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get.go + BlobovniczatreeCouldNotGetObjectFromActiveBlobovnicza = "could not get object from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get.go + BlobovniczatreeInitializingBlobovniczas = "initializing Blobovnicza's" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go + BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization = "read-only mode, skip blobovniczas initialization..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go + BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing = "blobovnicza successfully initialized, closing..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go + BlobovniczatreeCouldNotCloseActiveBlobovnicza = "could not close active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go + AlphabetTick = "tick" // Info in ../node/pkg/innerring/processors/alphabet/handlers.go + AlphabetAlphabetProcessorWorkerPoolDrained = "alphabet processor worker pool drained" // Warn in ../node/pkg/innerring/processors/alphabet/handlers.go + AlphabetNonAlphabetModeIgnoreGasEmissionEvent = "non alphabet mode, ignore gas emission event" // Info in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent = "node is out of alphabet range, ignore gas emission event" // Debug in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetCantInvokeAlphabetEmitMethod = "can't invoke alphabet emit method" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetStorageNodeEmissionIsOff = "storage node emission is off" // Info in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes = "can't get netmap snapshot to emit gas to storage nodes" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetGasEmission = "gas emission" // Debug in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetCantParseNodePublicKey = "can't parse node public key" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetCantTransferGas = "can't transfer gas" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetCantTransferGasToWallet = "can't transfer gas to wallet" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go + AlphabetAlphabetWorkerPool = "alphabet worker pool" // Debug in ../node/pkg/innerring/processors/alphabet/processor.go + BalanceBalanceWorkerPoolDrained = "balance worker pool drained" // Warn in ../node/pkg/innerring/processors/balance/handlers.go + BalanceNonAlphabetModeIgnoreBalanceLock = "non alphabet mode, ignore balance lock" // Info in ../node/pkg/innerring/processors/balance/process_assets.go + BalanceCantSendLockAssetTx = "can't send lock asset tx" // Error in ../node/pkg/innerring/processors/balance/process_assets.go + BalanceBalanceWorkerPool = "balance worker pool" // Debug in ../node/pkg/innerring/processors/balance/processor.go + ContainerContainerWorkerPool = "container worker pool" // Debug in ../node/pkg/innerring/processors/container/processor.go + ContainerContainerProcessorWorkerPoolDrained = "container processor worker pool drained" // Warn in ../node/pkg/innerring/processors/container/handlers.go + ContainerNonAlphabetModeIgnoreContainerPut = "non alphabet mode, ignore container put" // Info in ../node/pkg/innerring/processors/container/process_container.go + ContainerPutContainerCheckFailed = "put container check failed" // Error in ../node/pkg/innerring/processors/container/process_container.go + ContainerCouldNotApprovePutContainer = "could not approve put container" // Error in ../node/pkg/innerring/processors/container/process_container.go + ContainerNonAlphabetModeIgnoreContainerDelete = "non alphabet mode, ignore container delete" // Info in ../node/pkg/innerring/processors/container/process_container.go + ContainerDeleteContainerCheckFailed = "delete container check failed" // Error in ../node/pkg/innerring/processors/container/process_container.go + ContainerCouldNotApproveDeleteContainer = "could not approve delete container" // Error in ../node/pkg/innerring/processors/container/process_container.go + ContainerNonAlphabetModeIgnoreSetEACL = "non alphabet mode, ignore set EACL" // Info in ../node/pkg/innerring/processors/container/process_eacl.go + ContainerSetEACLCheckFailed = "set EACL check failed" // Error in ../node/pkg/innerring/processors/container/process_eacl.go + ContainerCouldNotApproveSetEACL = "could not approve set EACL" // Error in ../node/pkg/innerring/processors/container/process_eacl.go + FrostFSNonAlphabetModeIgnoreBind = "non alphabet mode, ignore bind" // Info in ../node/pkg/innerring/processors/frostfs/process_bind.go + FrostFSInvalidManageKeyEvent = "invalid manage key event" // Error in ../node/pkg/innerring/processors/frostfs/process_bind.go + FrostFSCouldNotDecodeScriptHashFromBytes = "could not decode script hash from bytes" // Error in ../node/pkg/innerring/processors/frostfs/process_bind.go + FrostFSNonAlphabetModeIgnoreConfig = "non alphabet mode, ignore config" // Info in ../node/pkg/innerring/processors/frostfs/process_config.go + FrostFSCantRelaySetConfigEvent = "can't relay set config event" // Error in ../node/pkg/innerring/processors/frostfs/process_config.go + FrostFSFrostfsWorkerPool = "frostfs worker pool" // Debug in ../node/pkg/innerring/processors/frostfs/processor.go + FrostFSFrostfsProcessorWorkerPoolDrained = "frostfs processor worker pool drained" // Warn in ../node/pkg/innerring/processors/frostfs/handlers.go + FrostFSNonAlphabetModeIgnoreDeposit = "non alphabet mode, ignore deposit" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantTransferAssetsToBalanceContract = "can't transfer assets to balance contract" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSDoubleMintEmissionDeclined = "double mint emission declined" // Warn in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantGetGasBalanceOfTheNode = "can't get gas balance of the node" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSGasBalanceThresholdHasBeenReached = "gas balance threshold has been reached" // Warn in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantTransferNativeGasToReceiver = "can't transfer native gas to receiver" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSNonAlphabetModeIgnoreWithdraw = "non alphabet mode, ignore withdraw" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantCreateLockAccount = "can't create lock account" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantLockAssetsForWithdraw = "can't lock assets for withdraw" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSNonAlphabetModeIgnoreCheque = "non alphabet mode, ignore cheque" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go + FrostFSCantTransferAssetsToFedContract = "can't transfer assets to fed contract" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go + GovernanceNewEvent = "new event" // Info in ../node/pkg/innerring/processors/governance/handlers.go + GovernanceGovernanceWorkerPoolDrained = "governance worker pool drained" // Warn in ../node/pkg/innerring/processors/governance/handlers.go + GovernanceNonAlphabetModeIgnoreAlphabetSync = "non alphabet mode, ignore alphabet sync" // Info in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantFetchAlphabetListFromMainNet = "can't fetch alphabet list from main net" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantFetchAlphabetListFromSideChain = "can't fetch alphabet list from side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantMergeAlphabetListsFromMainNetAndSideChain = "can't merge alphabet lists from main net and side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged = "no governance update, alphabet list has not been changed" // Info in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceAlphabetListHasBeenChangedStartingUpdate = "alphabet list has been changed, starting update" // Info in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantVoteForSideChainCommittee = "can't vote for side chain committee" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceFinishedAlphabetListUpdate = "finished alphabet list update" // Info in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantFetchInnerRingListFromSideChain = "can't fetch inner ring list from side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys = "can't create new inner ring list with new alphabet keys" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceUpdateOfTheInnerRingList = "update of the inner ring list" // Info in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantUpdateInnerRingListWithNewAlphabetKeys = "can't update inner ring list with new alphabet keys" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantUpdateListOfNotaryNodesInSideChain = "can't update list of notary nodes in side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go + GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract = "can't update list of alphabet nodes in frostfs contract" // Error in ../node/pkg/innerring/processors/governance/process_update.go + NetmapNetmapWorkerPool = "netmap worker pool" // Debug in ../node/pkg/innerring/processors/netmap/processor.go + NetmapTick = "tick" // Info in ../node/pkg/innerring/processors/netmap/handlers.go + NetmapNetmapWorkerPoolDrained = "netmap worker pool drained" // Warn in ../node/pkg/innerring/processors/netmap/handlers.go + NetmapNetmapCleanUpRoutineIsDisabled518 = "netmap clean up routine is disabled" // Debug in ../node/pkg/innerring/processors/netmap/handlers.go + NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick = "non alphabet mode, ignore new netmap cleanup tick" // Info in ../node/pkg/innerring/processors/netmap/process_cleanup.go + NetmapCantDecodePublicKeyOfNetmapNode = "can't decode public key of netmap node" // Warn in ../node/pkg/innerring/processors/netmap/process_cleanup.go + NetmapVoteToRemoveNodeFromNetmap = "vote to remove node from netmap" // Info in ../node/pkg/innerring/processors/netmap/process_cleanup.go + NetmapCantInvokeNetmapUpdateState = "can't invoke netmap.UpdateState" // Error in ../node/pkg/innerring/processors/netmap/process_cleanup.go + NetmapCantIterateOnNetmapCleanerCache = "can't iterate on netmap cleaner cache" // Warn in ../node/pkg/innerring/processors/netmap/process_cleanup.go + NetmapCantGetEpochDuration = "can't get epoch duration" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapCantGetTransactionHeight = "can't get transaction height" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapCantResetEpochTimer = "can't reset epoch timer" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapCantGetNetmapSnapshotToPerformCleanup = "can't get netmap snapshot to perform cleanup" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapCantStartContainerSizeEstimation = "can't start container size estimation" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapNonAlphabetModeIgnoreNewEpochTick = "non alphabet mode, ignore new epoch tick" // Info in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapNextEpoch = "next epoch" // Debug in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapCantInvokeNetmapNewEpoch = "can't invoke netmap.NewEpoch" // Error in ../node/pkg/innerring/processors/netmap/process_epoch.go + NetmapNonAlphabetModeIgnoreNewPeerNotification = "non alphabet mode, ignore new peer notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapNonhaltNotaryTransaction = "non-halt notary transaction" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCantParseNetworkMapCandidate = "can't parse network map candidate" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate = "could not verify and update information about network map candidate" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapApprovingNetworkMapCandidate = "approving network map candidate" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCantInvokeNetmapAddPeer = "can't invoke netmap.AddPeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapNonAlphabetModeIgnoreUpdatePeerNotification = "non alphabet mode, ignore update peer notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapPreventSwitchingNodeToMaintenanceState = "prevent switching node to maintenance state" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go + NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go // Debug in ../node/pkg/innerring/processors/reputation/processor.go + FrostFSIRInternalError = "internal error" // Info in ../node/cmd/frostfs-ir/main.go + FrostFSIRCouldNotShutdownHTTPServer = "could not shutdown HTTP server" // Debug in ../node/cmd/frostfs-ir/main.go + FrostFSIRApplicationStopped = "application stopped" // Info in ../node/cmd/frostfs-ir/main.go + FrostFSIRCouldntCreateRPCClientForEndpoint = "could not create RPC client for endpoint" // Debug in ../node/pkg/morph/client/constructor.go + FrostFSIRCreatedRPCClientForEndpoint = "created RPC client for endpoint" // Info in ../node/pkg/morph/client/constructor.go + FrostFSIRReloadExtraWallets = "reload extra wallets" // Info in ../node/cmd/frostfs-ir/config.go + FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" // Error in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" // Error in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeStartListeningGRPCEndpoint = "start listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeStoppingGRPCServer = "stopping gRPC server..." // Info in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeGRPCCannotShutdownGracefullyForcingStop = "gRPC cannot shutdown gracefully, forcing stop" // Info in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeGRPCServerStoppedSuccessfully = "gRPC server stopped successfully" // Info in ../node/cmd/frostfs-node/grpc.go + FrostFSNodeWaitingForAllProcessesToStop = "waiting for all processes to stop" // Debug in ../node/cmd/frostfs-node/main.go + FrostFSNodeStartedLocalNodesMaintenance = "started local node's maintenance" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeStoppedLocalNodesMaintenance = "stopped local node's maintenance" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeFailedToAttachShardToEngine = "failed to attach shard to engine" // Error in ../node/cmd/frostfs-node/config.go + FrostFSNodeShardAttachedToEngine = "shard attached to engine" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeClosingComponentsOfTheStorageEngine = "closing components of the storage engine..." // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeStorageEngineClosingFailure = "storage engine closing failure" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully = "all components of the storage engine closed successfully" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeBootstrappingWithTheMaintenanceState = "bootstrapping with the maintenance state" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeBootstrappingWithOnlineState = "bootstrapping with online state" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeTerminationSignalHasBeenReceivedStopping = "termination signal has been received, stopping..." // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeTerminationSignalProcessingIsComplete = "termination signal processing is complete" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeInternalApplicationError = "internal application error" // Warn in ../node/cmd/frostfs-node/config.go + FrostFSNodeInternalErrorProcessingIsComplete = "internal error processing is complete" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration = "SIGHUP has been received, rereading configuration..." // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeConfigurationReading = "configuration reading" // Error in ../node/cmd/frostfs-node/config.go + FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" // Error in ../node/cmd/frostfs-node/config.go + FrostFSNodeTracingConfigationUpdated = "tracing configation updated" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeStorageEngineConfigurationUpdate = "storage engine configuration update" // Error in ../node/cmd/frostfs-node/config.go + FrostFSNodeUpdatedConfigurationApplying = "updated configuration applying" // Error in ../node/cmd/frostfs-node/config.go + FrostFSNodeConfigurationHasBeenReloadedSuccessfully = "configuration has been reloaded successfully" // Info in ../node/cmd/frostfs-node/config.go + FrostFSNodeReadNewlyCreatedContainerAfterTheNotification = "read newly created container after the notification" // Error in ../node/cmd/frostfs-node/container.go + FrostFSNodeContainerCreationEventsReceipt = "container creation event's receipt" // Debug in ../node/cmd/frostfs-node/container.go + FrostFSNodeContainerRemovalEventsReceipt = "container removal event's receipt" // Debug in ../node/cmd/frostfs-node/container.go + FrostFSNodeSaveUsedSpaceAnnouncementInContract = "save used space announcement in contract" // Debug in ../node/cmd/frostfs-node/container.go + FrostFSNodeFailedToCalculateContainerSizeInStorageEngine = "failed to calculate container size in storage engine" // Debug in ../node/cmd/frostfs-node/container.go + FrostFSNodeContainerSizeInStorageEngineCalculatedSuccessfully = "container size in storage engine calculated successfully" // Debug in ../node/cmd/frostfs-node/container.go + FrostFSNodeNotificatorCouldNotListContainers = "notificator: could not list containers" // Error in ../node/cmd/frostfs-node/notificator.go + FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer = "notificator: could not select objects from container" // Error in ../node/cmd/frostfs-node/notificator.go + FrostFSNodeNotificatorCouldNotProcessObject = "notificator: could not process object" // Error in ../node/cmd/frostfs-node/notificator.go + FrostFSNodeNotificatorFinishedProcessingObjectNotifications = "notificator: finished processing object notifications" // Debug in ../node/cmd/frostfs-node/notificator.go + FrostFSNodeCouldNotWriteObjectNotification = "could not write object notification" // Warn in ../node/cmd/frostfs-node/notificator.go + FrostFSNodeCouldNotGetMaxObjectSizeValue = "could not get max object size value" // Error in ../node/cmd/frostfs-node/object.go + FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage = "could not inhume mark redundant copy as garbage" // Warn in ../node/cmd/frostfs-node/object.go + FrostFSNodeFailedInitTracing = "failed init tracing" // Error in ../node/cmd/frostfs-node/tracing.go + FrostFSNodeFailedShutdownTracing = "failed shutdown tracing" // Error in ../node/cmd/frostfs-node/tracing.go + FrostFSNodeFailedToCreateNeoRPCClient = "failed to create neo RPC client" // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeClosingMorphComponents = "closing morph components..." // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal = "failed to set group signer scope, continue with Global" // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeNotarySupport = "notary support" // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeMorphcacheTTLFetchedFromNetwork = "morph.cache_ttl fetched from network" // Debug in ../node/cmd/frostfs-node/morph.go + FrostFSNodeNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" // Warn in ../node/cmd/frostfs-node/morph.go + FrostFSNodeNewEpochEventFromSidechain = "new epoch event from sidechain" // Info in ../node/cmd/frostfs-node/morph.go + FrostFSNodeNewBlock = "new block" // Debug in ../node/cmd/frostfs-node/morph.go + FrostFSNodeCantUpdatePersistentState = "can't update persistent state" // Warn in ../node/cmd/frostfs-node/morph.go + FrostFSNodeCantSendRebootstrapTx = "can't send re-bootstrap tx" // Warn in ../node/cmd/frostfs-node/netmap.go + FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch = "could not update node state on new epoch" // Error in ../node/cmd/frostfs-node/netmap.go + FrostFSNodeCouldNotMakeNotaryDeposit = "could not make notary deposit" // Error in ../node/cmd/frostfs-node/netmap.go + FrostFSNodeInitialNetworkState = "initial network state" // Info in ../node/cmd/frostfs-node/netmap.go + FrostFSNodeTreeServiceIsNotEnabledSkipInitialization = "tree service is not enabled, skip initialization" // Info in ../node/cmd/frostfs-node/tree.go + FrostFSNodeCouldNotSynchronizeTreeService = "could not synchronize Tree Service" // Error in ../node/cmd/frostfs-node/tree.go + FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" // Debug in ../node/cmd/frostfs-node/tree.go + FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" // Error in ../node/cmd/frostfs-node/tree.go + FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" // Error in ../node/cmd/frostfs-node/control.go FrostFSNodePolicerIsDisabled = "policer is disabled" CommonApplicationStarted = "application started" ShardGCCollectingExpiredObjectsStarted = "collecting expired objects started" diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 0c433f226..593eb9971 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -20,7 +20,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" @@ -35,9 +36,8 @@ func TestInitializationFailure(t *testing.T) { type openFileFunc func(string, int, fs.FileMode) (*os.File, error) type testShardOpts struct { - openFileMetabase openFileFunc - openFileWriteCache openFileFunc - openFilePilorama openFileFunc + openFileMetabase openFileFunc + openFilePilorama openFileFunc } testShard := func(opts testShardOpts) ([]shard.Option, *teststore.TestStore, *teststore.TestStore) { @@ -52,6 +52,13 @@ func TestInitializationFailure(t *testing.T) { storages, smallFileStorage, largeFileStorage := newTestStorages(blobstorPath, 1<<20) + wcOpts := writecacheconfig.Options{ + Type: writecacheconfig.TypeBBolt, + BBoltOptions: []writecachebbolt.Option{ + writecachebbolt.WithPath(writecachePath), + }, + } + return []shard.Option{ shard.WithID(sid), shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), @@ -66,10 +73,7 @@ func TestInitializationFailure(t *testing.T) { meta.WithPermissions(0700), meta.WithEpochState(epochState{})), shard.WithWriteCache(true), - shard.WithWriteCacheOptions( - writecache.WithPath(writecachePath), - writecache.WithOpenFile(opts.openFileWriteCache), - ), + shard.WithWriteCacheOptions(wcOpts), shard.WithPiloramaOptions( pilorama.WithPath(piloramaPath), pilorama.WithOpenFile(opts.openFilePilorama), @@ -79,9 +83,8 @@ func TestInitializationFailure(t *testing.T) { t.Run("blobstor", func(t *testing.T) { shardOpts, _, largeFileStorage := testShard(testShardOpts{ - openFileMetabase: os.OpenFile, - openFileWriteCache: os.OpenFile, - openFilePilorama: os.OpenFile, + openFileMetabase: os.OpenFile, + openFilePilorama: os.OpenFile, }) largeFileStorage.SetOption(teststore.WithOpen(func(ro bool) error { return teststore.ErrDiskExploded @@ -103,30 +106,11 @@ func TestInitializationFailure(t *testing.T) { openFileMetabaseSucceed.Store(true) } shardOpts, _, _ := testShard(testShardOpts{ - openFileMetabase: openFileMetabase, - openFileWriteCache: os.OpenFile, - openFilePilorama: os.OpenFile, + openFileMetabase: openFileMetabase, + openFilePilorama: os.OpenFile, }) testEngineFailInitAndReload(t, true, shardOpts, beforeReload) }) - t.Run("write-cache", func(t *testing.T) { - var openFileWriteCacheSucceed atomic.Bool - openFileWriteCache := func(p string, f int, mode fs.FileMode) (*os.File, error) { - if openFileWriteCacheSucceed.Load() { - return os.OpenFile(p, f, mode) - } - return nil, teststore.ErrDiskExploded - } - beforeReload := func() { - openFileWriteCacheSucceed.Store(true) - } - shardOpts, _, _ := testShard(testShardOpts{ - openFileMetabase: os.OpenFile, - openFileWriteCache: openFileWriteCache, - openFilePilorama: os.OpenFile, - }) - testEngineFailInitAndReload(t, false, shardOpts, beforeReload) - }) t.Run("pilorama", func(t *testing.T) { var openFilePiloramaSucceed atomic.Bool openFilePilorama := func(p string, f int, mode fs.FileMode) (*os.File, error) { @@ -139,9 +123,8 @@ func TestInitializationFailure(t *testing.T) { openFilePiloramaSucceed.Store(true) } shardOpts, _, _ := testShard(testShardOpts{ - openFileMetabase: os.OpenFile, - openFileWriteCache: os.OpenFile, - openFilePilorama: openFilePilorama, + openFileMetabase: os.OpenFile, + openFilePilorama: openFilePilorama, }) testEngineFailInitAndReload(t, false, shardOpts, beforeReload) }) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index f362e2a03..f12a63e91 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" "github.com/google/uuid" @@ -131,11 +130,11 @@ func (e *StorageEngine) appendMetrics(id *shard.ID, opts []shard.Option) []shard mw: e.metrics, }, ), - shard.WithExtraWriteCacheOptions(writecache.WithMetrics( + shard.WithWriteCacheMetrics( &writeCacheMetrics{ shardID: id.String(), metrics: e.metrics.WriteCache(), - }), + }, ), shard.WithGCMetrics( &gcMetrics{ diff --git a/pkg/local_object_storage/internal/testutil/object.go b/pkg/local_object_storage/internal/testutil/object.go index 9cbce27bf..573a099ff 100644 --- a/pkg/local_object_storage/internal/testutil/object.go +++ b/pkg/local_object_storage/internal/testutil/object.go @@ -20,6 +20,12 @@ func GenerateObject() *objectSDK.Object { return GenerateObjectWithCID(cidtest.ID()) } +func GenerateObjectWithSize(sz int) *objectSDK.Object { + data := make([]byte, sz) + _, _ = rand.Read(data) + return GenerateObjectWithCIDWithPayload(cidtest.ID(), data) +} + func GenerateObjectWithCID(cnr cid.ID) *objectSDK.Object { data := make([]byte, defaultDataSize) _, _ = rand.Read(data) diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 82b107196..23677c58a 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -17,7 +17,8 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -63,6 +64,13 @@ func TestShardOpen(t *testing.T) { return nil, fs.ErrPermission } + wcOpts := writecacheconfig.Options{ + Type: writecacheconfig.TypeBBolt, + BBoltOptions: []writecachebbolt.Option{ + writecachebbolt.WithPath(filepath.Join(dir, "wc")), + }, + } + newShard := func() *Shard { return New( WithID(NewIDFromBytes([]byte{})), @@ -79,8 +87,7 @@ func TestShardOpen(t *testing.T) { WithPiloramaOptions( pilorama.WithPath(filepath.Join(dir, "pilorama"))), WithWriteCache(true), - WithWriteCacheOptions( - writecache.WithPath(filepath.Join(dir, "wc")))) + WithWriteCacheOptions(wcOpts)) } allowedMode.Store(int64(os.O_RDWR)) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 263a0ea4d..4b154462c 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -25,7 +26,10 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { Value: 100, } - sh := newCustomShard(t, t.TempDir(), false, nil, nil, []meta.Option{meta.WithEpochState(epoch)}) + wcOpts := writecacheconfig.Options{ + Type: writecacheconfig.TypeBBolt, + } + sh := newCustomShard(t, t.TempDir(), false, wcOpts, nil, []meta.Option{meta.WithEpochState(epoch)}) t.Cleanup(func() { releaseShard(sh, t) @@ -122,7 +126,10 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { linkID, _ := link.ID() - sh := newCustomShard(t, t.TempDir(), false, nil, nil, []meta.Option{meta.WithEpochState(epoch)}) + wcOpts := writecacheconfig.Options{ + Type: writecacheconfig.TypeBBolt, + } + sh := newCustomShard(t, t.TempDir(), false, wcOpts, nil, []meta.Option{meta.WithEpochState(epoch)}) t.Cleanup(func() { releaseShard(sh, t) diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 9ef2106b0..11a8aab5e 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -12,7 +12,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -70,8 +71,14 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { testCase{true, "object in write-cache, out of range, big offset", 100, newRange(101, math.MaxUint64-10)}) } - sh := newCustomShard(t, t.TempDir(), hasWriteCache, - []writecache.Option{writecache.WithMaxObjectSize(writeCacheMaxSize)}, + wcOpts := writecacheconfig.Options{ + Type: writecacheconfig.TypeBBolt, + BBoltOptions: []writecachebbolt.Option{ + writecachebbolt.WithMaxObjectSize(writeCacheMaxSize), + }, + } + + sh := newCustomShard(t, t.TempDir(), hasWriteCache, wcOpts, []blobstor.Option{blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 05799d236..00f4fbb9e 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -2,6 +2,7 @@ package shard import ( "context" + "fmt" "sync" "sync/atomic" "time" @@ -12,6 +13,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -98,7 +102,7 @@ type cfg struct { metaOpts []meta.Option - writeCacheOpts []writecache.Option + writeCacheOpts writecacheconfig.Options piloramaOpts []pilorama.Option @@ -153,11 +157,22 @@ func New(opts ...Option) *Shard { s.blobStor.SetReportErrorFunc(reportFunc) if c.useWriteCache { - s.writeCache = writecache.New( - append(c.writeCacheOpts, - writecache.WithReportErrorFunc(reportFunc), - writecache.WithBlobstor(bs), - writecache.WithMetabase(mb))...) + switch c.writeCacheOpts.Type { + case writecacheconfig.TypeBBolt: + s.writeCache = writecachebbolt.New( + append(c.writeCacheOpts.BBoltOptions, + writecachebbolt.WithReportErrorFunc(reportFunc), + writecachebbolt.WithBlobstor(bs), + writecachebbolt.WithMetabase(mb))...) + case writecacheconfig.TypeBadger: + s.writeCache = writecachebadger.New( + append(c.writeCacheOpts.BadgerOptions, + writecachebadger.WithReportErrorFunc(reportFunc), + writecachebadger.WithBlobstor(bs), + writecachebadger.WithMetabase(mb))...) + default: + panic(fmt.Sprintf("invalid writecache type: %v", c.writeCacheOpts.Type)) + } } if s.piloramaOpts != nil { @@ -191,16 +206,21 @@ func WithMetaBaseOptions(opts ...meta.Option) Option { } // WithWriteCacheOptions returns option to set internal write cache options. -func WithWriteCacheOptions(opts ...writecache.Option) Option { +func WithWriteCacheOptions(opts writecacheconfig.Options) Option { return func(c *cfg) { c.writeCacheOpts = opts } } -// WithExtraWriteCacheOptions returns option to add extra write cache options. -func WithExtraWriteCacheOptions(opts ...writecache.Option) Option { +// WithWriteCacheMetrics returns an option to set the metrics register used by the write cache. +func WithWriteCacheMetrics(wcMetrics writecache.Metrics) Option { return func(c *cfg) { - c.writeCacheOpts = append(c.writeCacheOpts, opts...) + switch c.writeCacheOpts.Type { + case writecacheconfig.TypeBBolt: + c.writeCacheOpts.BBoltOptions = append(c.writeCacheOpts.BBoltOptions, writecachebbolt.WithMetrics(wcMetrics)) + case writecacheconfig.TypeBadger: + c.writeCacheOpts.BadgerOptions = append(c.writeCacheOpts.BadgerOptions, writecachebadger.WithMetrics(wcMetrics)) + } } } diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index a9a8e4ea7..3b7c7c4a3 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -12,7 +12,9 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -33,15 +35,25 @@ func (s epochState) CurrentEpoch() uint64 { func newShard(t testing.TB, enableWriteCache bool) *shard.Shard { return newCustomShard(t, t.TempDir(), enableWriteCache, - nil, + writecacheconfig.Options{Type: writecacheconfig.TypeBBolt}, nil, nil) } -func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts []writecache.Option, bsOpts []blobstor.Option, metaOptions []meta.Option) *shard.Shard { +func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts writecacheconfig.Options, bsOpts []blobstor.Option, metaOptions []meta.Option) *shard.Shard { var sh *shard.Shard if enableWriteCache { rootPath = filepath.Join(rootPath, "wc") + switch wcOpts.Type { + case writecacheconfig.TypeBBolt: + wcOpts.BBoltOptions = append( + []writecachebbolt.Option{writecachebbolt.WithPath(filepath.Join(rootPath, "wcache"))}, + wcOpts.BBoltOptions...) + case writecacheconfig.TypeBadger: + wcOpts.BadgerOptions = append( + []writecachebadger.Option{writecachebadger.WithPath(filepath.Join(rootPath, "wcache"))}, + wcOpts.BadgerOptions...) + } } else { rootPath = filepath.Join(rootPath, "nowc") } @@ -79,11 +91,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts ), shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))), shard.WithWriteCache(enableWriteCache), - shard.WithWriteCacheOptions( - append( - []writecache.Option{writecache.WithPath(filepath.Join(rootPath, "wcache"))}, - wcOpts...)..., - ), + shard.WithWriteCacheOptions(wcOpts), shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { sh.HandleDeletedLocks(addresses) }), diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 5fe9fd7e9..15bff7f1e 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -8,7 +8,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" @@ -36,9 +37,12 @@ func TestWriteCacheObjectLoss(t *testing.T) { } dir := t.TempDir() - wcOpts := []writecache.Option{ - writecache.WithSmallObjectSize(smallSize), - writecache.WithMaxObjectSize(smallSize * 2)} + wcOpts := writecacheconfig.Options{ + Type: writecacheconfig.TypeBBolt, + BBoltOptions: []writecachebbolt.Option{ + writecachebbolt.WithMaxObjectSize(smallSize * 2), + }, + } sh := newCustomShard(t, dir, true, wcOpts, nil, nil) diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go new file mode 100644 index 000000000..4c44b1d44 --- /dev/null +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -0,0 +1,52 @@ +package benchmark + +import ( + "context" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" +) + +func BenchmarkWritecache(b *testing.B) { + b.Run("bbolt", func(b *testing.B) { + cache := writecachebbolt.New( + writecachebbolt.WithPath(b.TempDir()), + ) + benchmarkPut(b, cache) + }) + b.Run("badger", func(b *testing.B) { + cache := writecachebadger.New( + writecachebadger.WithPath(b.TempDir()), + ) + benchmarkPut(b, cache) + }) +} + +func benchmarkPut(b *testing.B, cache writecache.Cache) { + if err := cache.Open(false); err != nil { + b.Fatalf("initializing: %v", err) + } + if err := cache.Init(); err != nil { + b.Fatalf("opening: %v", err) + } + defer cache.Close() + + ctx := context.Background() + objGen := testutil.RandObjGenerator{ObjSize: 8 << 10} + + b.ResetTimer() + for n := 0; n < b.N; n++ { + prm := common.PutPrm{ + Address: oidtest.Address(), + Object: objGen.Next(), + } + if _, err := cache.Put(ctx, prm); err != nil { + b.Fatalf("putting: %v", err) + } + } +} diff --git a/pkg/local_object_storage/writecache/config/config.go b/pkg/local_object_storage/writecache/config/config.go new file mode 100644 index 000000000..91f097e17 --- /dev/null +++ b/pkg/local_object_storage/writecache/config/config.go @@ -0,0 +1,22 @@ +// Package config provides the common configuration options for write cache implementations. +package config + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" +) + +// Type is the write cache implementation type. +type Type int + +const ( + TypeBBolt Type = iota + TypeBadger +) + +// Options are the configuration options for the write cache. +type Options struct { + Type Type + BBoltOptions []writecachebbolt.Option + BadgerOptions []writecachebadger.Option +} diff --git a/pkg/local_object_storage/writecache/doc.go b/pkg/local_object_storage/writecache/doc.go deleted file mode 100644 index f2e904030..000000000 --- a/pkg/local_object_storage/writecache/doc.go +++ /dev/null @@ -1,11 +0,0 @@ -// Package writecache implements write-cache for objects. -// -// Write-cache has 2 components: -// 1. Key-value (bbolt) database for storing small objects. -// 2. Filesystem tree for storing big objects. -// -// Flushing from the writecache to the main storage is done in the background. -// To make it possible to serve Read requests after the object was flushed, -// we maintain an LRU cache containing addresses of all the objects that -// could be safely deleted. The actual deletion is done during eviction from this cache. -package writecache diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go deleted file mode 100644 index 997d23a3d..000000000 --- a/pkg/local_object_storage/writecache/flush_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package writecache - -import ( - "context" - "os" - "path/filepath" - "sync/atomic" - "testing" - - objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" - versionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" - "github.com/stretchr/testify/require" - "go.etcd.io/bbolt" - "go.uber.org/zap/zaptest" -) - -type objectPair struct { - addr oid.Address - obj *objectSDK.Object -} - -func TestFlush(t *testing.T) { - const ( - objCount = 4 - smallSize = 256 - ) - - newCache := func(t *testing.T, opts ...Option) (Cache, *blobstor.BlobStor, *meta.DB) { - dir := t.TempDir() - mb := meta.New( - meta.WithPath(filepath.Join(dir, "meta")), - meta.WithEpochState(dummyEpoch{})) - require.NoError(t, mb.Open(false)) - require.NoError(t, mb.Init()) - - bs := blobstor.New(blobstor.WithStorages([]blobstor.SubStorage{ - { - Storage: fstree.New( - fstree.WithPath(filepath.Join(dir, "blob")), - fstree.WithDepth(0), - fstree.WithDirNameLen(1)), - }, - })) - require.NoError(t, bs.Open(false)) - require.NoError(t, bs.Init()) - - wc := New( - append([]Option{ - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), - WithPath(filepath.Join(dir, "writecache")), - WithSmallObjectSize(smallSize), - WithMetabase(mb), - WithBlobstor(bs), - }, opts...)...) - t.Cleanup(func() { require.NoError(t, wc.Close()) }) - require.NoError(t, wc.Open(false)) - require.NoError(t, wc.Init()) - - // First set mode for metabase and blobstor to prevent background flushes. - require.NoError(t, mb.SetMode(mode.ReadOnly)) - require.NoError(t, bs.SetMode(mode.ReadOnly)) - - return wc, bs, mb - } - - putObjects := func(t *testing.T, c Cache) []objectPair { - objects := make([]objectPair, objCount) - for i := range objects { - objects[i] = putObject(t, c, 1+(i%2)*smallSize) - } - return objects - } - - check := func(t *testing.T, mb *meta.DB, bs *blobstor.BlobStor, objects []objectPair) { - for i := range objects { - var mPrm meta.StorageIDPrm - mPrm.SetAddress(objects[i].addr) - - mRes, err := mb.StorageID(context.Background(), mPrm) - require.NoError(t, err) - - var prm common.GetPrm - prm.Address = objects[i].addr - prm.StorageID = mRes.StorageID() - - res, err := bs.Get(context.Background(), prm) - require.NoError(t, err) - require.Equal(t, objects[i].obj, res.Object) - } - } - - t.Run("no errors", func(t *testing.T) { - wc, bs, mb := newCache(t) - objects := putObjects(t, wc) - - require.NoError(t, bs.SetMode(mode.ReadWrite)) - require.NoError(t, mb.SetMode(mode.ReadWrite)) - - require.NoError(t, wc.Flush(context.Background(), false)) - - check(t, mb, bs, objects) - }) - - t.Run("flush on moving to degraded mode", func(t *testing.T) { - wc, bs, mb := newCache(t) - objects := putObjects(t, wc) - - // Blobstor is read-only, so we expect en error from `flush` here. - require.Error(t, wc.SetMode(mode.Degraded)) - - // First move to read-only mode to close background workers. - require.NoError(t, wc.SetMode(mode.ReadOnly)) - require.NoError(t, bs.SetMode(mode.ReadWrite)) - require.NoError(t, mb.SetMode(mode.ReadWrite)) - require.NoError(t, wc.SetMode(mode.Degraded)) - - check(t, mb, bs, objects) - }) - - t.Run("ignore errors", func(t *testing.T) { - testIgnoreErrors := func(t *testing.T, f func(*cache)) { - var errCount atomic.Uint32 - wc, bs, mb := newCache(t, WithReportErrorFunc(func(message string, err error) { - errCount.Add(1) - })) - objects := putObjects(t, wc) - f(wc.(*cache)) - - require.NoError(t, wc.SetMode(mode.ReadOnly)) - require.NoError(t, bs.SetMode(mode.ReadWrite)) - require.NoError(t, mb.SetMode(mode.ReadWrite)) - - require.Equal(t, uint32(0), errCount.Load()) - require.Error(t, wc.Flush(context.Background(), false)) - require.True(t, errCount.Load() > 0) - require.NoError(t, wc.Flush(context.Background(), true)) - - check(t, mb, bs, objects) - } - t.Run("db, invalid address", func(t *testing.T) { - testIgnoreErrors(t, func(c *cache) { - _, data := newObject(t, 1) - require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - return b.Put([]byte{1, 2, 3}, data) - })) - }) - }) - t.Run("db, invalid object", func(t *testing.T) { - testIgnoreErrors(t, func(c *cache) { - require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - return b.Put([]byte(oidtest.Address().EncodeToString()), []byte{1, 2, 3}) - })) - }) - }) - t.Run("fs, read error", func(t *testing.T) { - testIgnoreErrors(t, func(c *cache) { - obj, data := newObject(t, 1) - - var prm common.PutPrm - prm.Address = objectCore.AddressOf(obj) - prm.RawData = data - - _, err := c.fsTree.Put(context.Background(), prm) - require.NoError(t, err) - - p := prm.Address.Object().EncodeToString() + "." + prm.Address.Container().EncodeToString() - p = filepath.Join(c.fsTree.RootPath, p[:1], p[1:]) - - _, err = os.Stat(p) // sanity check - require.NoError(t, err) - require.NoError(t, os.Truncate(p, 0)) // corrupt the file contents, so that it can't be unmarshalled - }) - }) - t.Run("fs, invalid object", func(t *testing.T) { - testIgnoreErrors(t, func(c *cache) { - var prm common.PutPrm - prm.Address = oidtest.Address() - prm.RawData = []byte{1, 2, 3} - _, err := c.fsTree.Put(context.Background(), prm) - require.NoError(t, err) - }) - }) - }) -} - -func putObject(t *testing.T, c Cache, size int) objectPair { - obj, data := newObject(t, size) - - var prm common.PutPrm - prm.Address = objectCore.AddressOf(obj) - prm.Object = obj - prm.RawData = data - - _, err := c.Put(context.Background(), prm) - require.NoError(t, err) - - return objectPair{prm.Address, prm.Object} - -} - -func newObject(t *testing.T, size int) (*objectSDK.Object, []byte) { - obj := objectSDK.New() - ver := versionSDK.Current() - - obj.SetID(oidtest.ID()) - obj.SetOwnerID(usertest.ID()) - obj.SetContainerID(cidtest.ID()) - obj.SetType(objectSDK.TypeRegular) - obj.SetVersion(&ver) - obj.SetPayloadChecksum(checksumtest.Checksum()) - obj.SetPayloadHomomorphicHash(checksumtest.Checksum()) - obj.SetPayload(make([]byte, size)) - - data, err := obj.Marshal() - require.NoError(t, err) - return obj, data -} - -type dummyEpoch struct{} - -func (dummyEpoch) CurrentEpoch() uint64 { - return 0 -} diff --git a/pkg/local_object_storage/writecache/generic_test.go b/pkg/local_object_storage/writecache/generic_test.go deleted file mode 100644 index 53d6624b7..000000000 --- a/pkg/local_object_storage/writecache/generic_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package writecache - -import ( - "os" - "path/filepath" - "strconv" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" -) - -func TestGeneric(t *testing.T) { - defer func() { _ = os.RemoveAll(t.Name()) }() - - var n int - newCache := func(t *testing.T) storagetest.Component { - n++ - dir := filepath.Join(t.Name(), strconv.Itoa(n)) - require.NoError(t, os.MkdirAll(dir, os.ModePerm)) - return New( - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), - WithFlushWorkersCount(2), - WithPath(dir)) - } - - storagetest.TestAll(t, newCache) -} diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go index 957bf2770..5eac06698 100644 --- a/pkg/local_object_storage/writecache/metrics.go +++ b/pkg/local_object_storage/writecache/metrics.go @@ -31,22 +31,24 @@ type Metrics interface { Close() } +func DefaultMetrics() Metrics { return metricsStub{} } + type metricsStub struct{} -func (s *metricsStub) Get(time.Duration, bool, StorageType) {} +func (metricsStub) Get(time.Duration, bool, StorageType) {} -func (s *metricsStub) Delete(time.Duration, bool, StorageType) {} +func (metricsStub) Delete(time.Duration, bool, StorageType) {} -func (s *metricsStub) Put(time.Duration, bool, StorageType) {} +func (metricsStub) Put(time.Duration, bool, StorageType) {} -func (s *metricsStub) SetEstimateSize(uint64, uint64) {} +func (metricsStub) SetEstimateSize(uint64, uint64) {} -func (s *metricsStub) SetMode(mode.Mode) {} +func (metricsStub) SetMode(mode.Mode) {} -func (s *metricsStub) SetActualCounters(uint64, uint64) {} +func (metricsStub) SetActualCounters(uint64, uint64) {} -func (s *metricsStub) Flush(bool, StorageType) {} +func (metricsStub) Flush(bool, StorageType) {} -func (s *metricsStub) Evict(StorageType) {} +func (metricsStub) Evict(StorageType) {} -func (s *metricsStub) Close() {} +func (metricsStub) Close() {} diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 067ff5ae5..084c9a3ac 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -2,18 +2,14 @@ package writecache import ( "context" - "os" - "sync" + "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.etcd.io/bbolt" - "go.uber.org/zap" ) // Info groups the information about write-cache. @@ -44,133 +40,13 @@ type Cache interface { Close() error } -type cache struct { - options - - // mtx protects statistics, counters and compressFlags. - mtx sync.RWMutex - - mode mode.Mode - modeMtx sync.RWMutex - - // compressFlags maps address of a big object to boolean value indicating - // whether object should be compressed. - compressFlags map[string]struct{} - - // flushCh is a channel with objects to flush. - flushCh chan *objectSDK.Object - // closeCh is close channel, protected by modeMtx. - closeCh chan struct{} - // wg is a wait group for flush workers. - wg sync.WaitGroup - // store contains underlying database. - store - // fsTree contains big files stored directly on file-system. - fsTree *fstree.FSTree -} - -// wcStorageType is used for write-cache operations logging. -const wcStorageType = "write-cache" - -type objectInfo struct { - addr string - data []byte - obj *objectSDK.Object -} - -const ( - defaultMaxObjectSize = 64 * 1024 * 1024 // 64 MiB - defaultSmallObjectSize = 32 * 1024 // 32 KiB - defaultMaxCacheSize = 1 << 30 // 1 GiB -) - var ( - defaultBucket = []byte{0} + // ErrReadOnly is returned when Put/Write is performed in a read-only mode. + ErrReadOnly = logicerr.New("write-cache is in read-only mode") + // ErrNotInitialized is returned when write-cache is initializing. + ErrNotInitialized = logicerr.New("write-cache is not initialized yet") + // ErrBigObject is returned when object is too big to be placed in cache. + ErrBigObject = errors.New("too big object") + // ErrOutOfSpace is returned when there is no space left to put a new object. + ErrOutOfSpace = errors.New("no space left in the write cache") ) - -// New creates new writecache instance. -func New(opts ...Option) Cache { - c := &cache{ - flushCh: make(chan *objectSDK.Object), - mode: mode.ReadWrite, - - compressFlags: make(map[string]struct{}), - options: options{ - log: &logger.Logger{Logger: zap.NewNop()}, - maxObjectSize: defaultMaxObjectSize, - smallObjectSize: defaultSmallObjectSize, - workersCount: defaultFlushWorkersCount, - maxCacheSize: defaultMaxCacheSize, - maxBatchSize: bbolt.DefaultMaxBatchSize, - maxBatchDelay: bbolt.DefaultMaxBatchDelay, - openFile: os.OpenFile, - metrics: &metricsStub{}, - }, - } - - for i := range opts { - opts[i](&c.options) - } - - return c -} - -// SetLogger sets logger. It is used after the shard ID was generated to use it in logs. -func (c *cache) SetLogger(l *logger.Logger) { - c.log = l -} - -func (c *cache) DumpInfo() Info { - return Info{ - Path: c.path, - } -} - -// Open opens and initializes database. Reads object counters from the ObjectCounters instance. -func (c *cache) Open(readOnly bool) error { - err := c.openStore(readOnly) - if err != nil { - return metaerr.Wrap(err) - } - - // Opening after Close is done during maintenance mode, - // thus we need to create a channel here. - c.closeCh = make(chan struct{}) - - return metaerr.Wrap(c.initCounters()) -} - -// Init runs necessary services. -func (c *cache) Init() error { - c.metrics.SetMode(c.mode) - c.runFlushLoop() - return nil -} - -// Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. -func (c *cache) Close() error { - // We cannot lock mutex for the whole operation duration - // because it is taken by some background workers, so `wg.Wait()` is done without modeMtx. - c.modeMtx.Lock() - if c.closeCh != nil { - close(c.closeCh) - } - c.mode = mode.DegradedReadOnly // prevent new operations from being processed - c.modeMtx.Unlock() - - c.wg.Wait() - - c.modeMtx.Lock() - defer c.modeMtx.Unlock() - - c.closeCh = nil - var err error - if c.db != nil { - err = c.db.Close() - if err != nil { - c.db = nil - } - } - c.metrics.Close() - return nil -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go new file mode 100644 index 000000000..837e76a0b --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go @@ -0,0 +1,129 @@ +package writecachebadger + +import ( + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" +) + +type cache struct { + options + + mode mode.Mode + modeMtx sync.RWMutex + + // flushCh is a channel with objects to flush. + flushCh chan *objectSDK.Object + // closeCh is close channel, protected by modeMtx. + closeCh chan struct{} + // wg is a wait group for flush workers. + wg sync.WaitGroup + // store contains underlying database. + store +} + +// wcStorageType is used for write-cache operations logging. +const wcStorageType = "write-cache" + +type objectInfo struct { + addr oid.Address + data []byte + obj *objectSDK.Object +} + +const ( + defaultMaxObjectSize = 64 << 20 // 64 MiB + defaultSmallObjectSize = 32 << 10 // 32 KiB + defaultMaxCacheSize = 1 << 30 // 1 GiB +) + +// New creates new writecache instance. +func New(opts ...Option) writecache.Cache { + c := &cache{ + flushCh: make(chan *objectSDK.Object), + mode: mode.ReadWrite, + + options: options{ + log: &logger.Logger{Logger: zap.NewNop()}, + maxObjectSize: defaultMaxObjectSize, + workersCount: defaultFlushWorkersCount, + maxCacheSize: defaultMaxCacheSize, + metrics: writecache.DefaultMetrics(), + }, + } + + for i := range opts { + opts[i](&c.options) + } + + return c +} + +// SetLogger sets logger. It is used after the shard ID was generated to use it in logs. +func (c *cache) SetLogger(l *logger.Logger) { + c.log = l +} + +func (c *cache) DumpInfo() writecache.Info { + return writecache.Info{ + Path: c.path, + } +} + +// Open opens and initializes database. Reads object counters from the ObjectCounters instance. +func (c *cache) Open(readOnly bool) error { + err := c.openStore(readOnly) + if err != nil { + return metaerr.Wrap(err) + } + + // Opening after Close is done during maintenance mode, + // thus we need to create a channel here. + c.closeCh = make(chan struct{}) + + return metaerr.Wrap(c.initCounters()) +} + +// Init runs necessary services. +func (c *cache) Init() error { + c.log.Info(logs.WritecacheBadgerInitExperimental) + c.metrics.SetMode(c.mode) + c.runFlushLoop() + c.runGCLoop() + return nil +} + +// Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. +func (c *cache) Close() error { + // We cannot lock mutex for the whole operation duration + // because it is taken by some background workers, so `wg.Wait()` is done without modeMtx. + c.modeMtx.Lock() + if c.closeCh != nil { + close(c.closeCh) + } + c.mode = mode.DegradedReadOnly // prevent new operations from being processed + c.modeMtx.Unlock() + + c.wg.Wait() + + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + + c.closeCh = nil + var err error + if c.db != nil { + err = c.db.Close() + if err != nil { + c.db = nil + } + } + c.metrics.Close() + return nil +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/delete.go b/pkg/local_object_storage/writecache/writecachebadger/delete.go new file mode 100644 index 000000000..1b46b2be9 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/delete.go @@ -0,0 +1,70 @@ +package writecachebadger + +import ( + "context" + "time" + + storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/dgraph-io/badger/v4" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// Delete removes object from write-cache. +// +// Returns an error of type apistatus.ObjectNotFound if object is missing in write-cache. +func (c *cache) Delete(ctx context.Context, addr oid.Address) error { + _, span := tracing.StartSpanFromContext(ctx, "writecache.Delete", + trace.WithAttributes( + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + + deleted := false + storageType := writecache.StorageTypeUndefined + startedAt := time.Now() + defer func() { + c.metrics.Delete(time.Since(startedAt), deleted, storageType) + }() + + c.modeMtx.RLock() + defer c.modeMtx.RUnlock() + if c.readOnly() { + return writecache.ErrReadOnly + } + + saddr := addr.EncodeToString() + + err := c.db.Update(func(tx *badger.Txn) error { + it, err := tx.Get([]byte(saddr)) + if err != nil { + if err == badger.ErrKeyNotFound { + return logicerr.Wrap(apistatus.ObjectNotFound{}) + } + return err + } + if it.ValueSize() > 0 { + storageType = writecache.StorageTypeDB + err := tx.Delete([]byte(saddr)) + if err == nil { + storagelog.Write(c.log, + storagelog.AddressField(saddr), + storagelog.StorageTypeField(wcStorageType), + storagelog.OpField("db DELETE"), + ) + deleted = true + c.objCounters.DecDB() + } + return err + } + return nil + }) + + return metaerr.Wrap(err) +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush.go b/pkg/local_object_storage/writecache/writecachebadger/flush.go new file mode 100644 index 000000000..d8bdddb56 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/flush.go @@ -0,0 +1,257 @@ +package writecachebadger + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "github.com/dgraph-io/badger/v4" + "github.com/mr-tron/base58" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +const ( + // flushBatchSize is amount of keys which will be read from cache to be flushed + // to the main storage. It is used to reduce contention between cache put + // and cache persist. + flushBatchSize = 512 + // defaultFlushWorkersCount is number of workers for putting objects in main storage. + defaultFlushWorkersCount = 20 + // defaultFlushInterval is default time interval between successive flushes. + defaultFlushInterval = time.Second +) + +// runFlushLoop starts background workers which periodically flush objects to the blobstor. +func (c *cache) runFlushLoop() { + for i := 0; i < c.workersCount; i++ { + c.wg.Add(1) + go c.workerFlushSmall() + } + + c.wg.Add(1) + go func() { + defer c.wg.Done() + + tt := time.NewTimer(defaultFlushInterval) + defer tt.Stop() + + for { + select { + case <-tt.C: + c.flushSmallObjects() + tt.Reset(defaultFlushInterval) + case <-c.closeCh: + return + } + } + }() +} + +func (c *cache) flushSmallObjects() { + var lastKey internalKey + var m []objectInfo + for { + select { + case <-c.closeCh: + return + default: + } + + m = m[:0] + + c.modeMtx.RLock() + if c.readOnly() { + c.modeMtx.RUnlock() + time.Sleep(time.Second) + continue + } + + _ = c.db.View(func(tx *badger.Txn) error { + it := tx.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + if len(lastKey) == 0 { + it.Rewind() + } else { + it.Seek(lastKey[:]) + if it.Valid() && bytes.Equal(it.Item().Key(), lastKey[:]) { + it.Next() + } + } + for ; it.Valid() && len(m) < flushBatchSize; it.Next() { + if got, want := int(it.Item().KeySize()), len(lastKey); got != want { + return fmt.Errorf("invalid db key len: got %d, want %d", got, want) + } + it.Item().KeyCopy(lastKey[:]) + value, err := it.Item().ValueCopy(nil) + if err != nil { + return err + } + m = append(m, objectInfo{ + addr: lastKey.address(), + data: value, + }) + } + return nil + }) + + var count int + for i := range m { + obj := objectSDK.New() + if err := obj.Unmarshal(m[i].data); err != nil { + continue + } + + count++ + select { + case c.flushCh <- obj: + case <-c.closeCh: + c.modeMtx.RUnlock() + return + } + } + + if count == 0 { + c.modeMtx.RUnlock() + break + } + + c.modeMtx.RUnlock() + + c.log.Debug(logs.WritecacheTriedToFlushItemsFromWritecache, + zap.Int("count", count), + zap.String("start", base58.Encode(lastKey[:]))) + } +} + +func (c *cache) reportFlushError(msg string, addr string, err error) { + if c.reportError != nil { + c.reportError(msg, err) + } else { + c.log.Error(msg, + zap.String("address", addr), + zap.Error(err)) + } +} + +// workerFlushSmall writes small objects to the main storage. +func (c *cache) workerFlushSmall() { + defer c.wg.Done() + + var obj *objectSDK.Object + for { + // Give priority to direct put. + select { + case obj = <-c.flushCh: + case <-c.closeCh: + return + } + + err := c.flushObject(context.TODO(), obj, nil, writecache.StorageTypeDB) + if err != nil { + // Error is handled in flushObject. + continue + } + + c.deleteFromDB([]string{objectCore.AddressOf(obj).EncodeToString()}) + } +} + +// flushObject is used to write object directly to the main storage. +func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []byte, st writecache.StorageType) error { + var err error + + defer func() { + c.metrics.Flush(err == nil, st) + }() + + addr := objectCore.AddressOf(obj) + + var prm common.PutPrm + prm.Object = obj + prm.RawData = data + + res, err := c.blobstor.Put(ctx, prm) + if err != nil { + if !errors.Is(err, common.ErrNoSpace) && !errors.Is(err, common.ErrReadOnly) && + !errors.Is(err, blobstor.ErrNoPlaceFound) { + c.reportFlushError("can't flush an object to blobstor", + addr.EncodeToString(), err) + } + return err + } + + var updPrm meta.UpdateStorageIDPrm + updPrm.SetAddress(addr) + updPrm.SetStorageID(res.StorageID) + + _, err = c.metabase.UpdateStorageID(updPrm) + if err != nil { + c.reportFlushError("can't update object storage ID", + addr.EncodeToString(), err) + } + return err +} + +// Flush flushes all objects from the write-cache to the main storage. +// Write-cache must be in readonly mode to ensure correctness of an operation and +// to prevent interference with background flush workers. +func (c *cache) Flush(ctx context.Context, ignoreErrors bool) error { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Flush", + trace.WithAttributes( + attribute.Bool("ignore_errors", ignoreErrors), + )) + defer span.End() + + c.modeMtx.RLock() + defer c.modeMtx.RUnlock() + + return c.flush(ctx, ignoreErrors) +} + +func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { + return c.db.View(func(tx *badger.Txn) error { + it := tx.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + var key internalKey + for it.Rewind(); it.Valid(); it.Next() { + if got, want := int(it.Item().KeySize()), len(key); got != want { + err := fmt.Errorf("invalid db key len: got %d, want %d", got, want) + c.reportFlushError("can't decode object address from the DB", hex.EncodeToString(it.Item().Key()), metaerr.Wrap(err)) + if ignoreErrors { + continue + } + return err + } + if err := it.Item().Value(func(data []byte) error { + var obj objectSDK.Object + if err := obj.Unmarshal(data); err != nil { + copy(key[:], it.Item().Key()) + c.reportFlushError("can't unmarshal an object from the DB", key.address().EncodeToString(), metaerr.Wrap(err)) + if ignoreErrors { + return nil + } + return err + } + + return c.flushObject(ctx, &obj, data, writecache.StorageTypeDB) + }); err != nil { + return err + } + } + return nil + }) +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go new file mode 100644 index 000000000..4d65d5855 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go @@ -0,0 +1,65 @@ +package writecachebadger + +import ( + "path/filepath" + "sync/atomic" + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachetest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/dgraph-io/badger/v4" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" +) + +func TestFlush(t *testing.T) { + createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs *blobstor.BlobStor, opts ...Option) writecache.Cache { + return New( + append([]Option{ + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithPath(filepath.Join(t.TempDir(), "writecache")), + WithMetabase(mb), + WithBlobstor(bs), + WithGCInterval(1 * time.Second), + }, opts...)...) + } + + errCountOpt := func() (Option, *atomic.Uint32) { + cnt := &atomic.Uint32{} + return WithReportErrorFunc(func(string, error) { + cnt.Add(1) + }), cnt + } + + failures := []writecachetest.TestFailureInjector[Option]{ + { + Desc: "db, invalid address", + InjectFn: func(t *testing.T, wc writecache.Cache) { + c := wc.(*cache) + obj := testutil.GenerateObject() + data, err := obj.Marshal() + require.NoError(t, err) + require.NoError(t, c.db.Update(func(tx *badger.Txn) error { + return tx.Set([]byte{1, 2, 3}, data) + })) + }, + }, + { + Desc: "db, invalid object", + InjectFn: func(t *testing.T, wc writecache.Cache) { + c := wc.(*cache) + require.NoError(t, c.db.Update(func(tx *badger.Txn) error { + return tx.Set([]byte(oidtest.Address().EncodeToString()), []byte{1, 2, 3}) + })) + }, + }, + } + + writecachetest.TestFlush(t, createCacheFn, errCountOpt, failures...) +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/gc.go b/pkg/local_object_storage/writecache/writecachebadger/gc.go new file mode 100644 index 000000000..51d3e9763 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/gc.go @@ -0,0 +1,31 @@ +package writecachebadger + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" +) + +func (c *cache) runGCLoop() { + c.wg.Add(1) + + go func() { + defer c.wg.Done() + + t := time.NewTicker(c.gcInterval) + defer t.Stop() + + for { + select { + case <-c.closeCh: + return + case <-t.C: + // 0.5 is the recommended value so that write amplification of the value log is 2. + // See https://pkg.go.dev/github.com/dgraph-io/badger/v4#DB.RunValueLogGC for more info. + for c.db.RunValueLogGC(0.5) == nil { + c.log.Debug(logs.WritecacheDBValueLogGCRunCompleted) + } + } + } + }() +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/generic_test.go b/pkg/local_object_storage/writecache/writecachebadger/generic_test.go new file mode 100644 index 000000000..be0a40e00 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/generic_test.go @@ -0,0 +1,20 @@ +package writecachebadger + +import ( + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "go.uber.org/zap/zaptest" +) + +func TestGeneric(t *testing.T) { + storagetest.TestAll(t, func(t *testing.T) storagetest.Component { + return New( + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithFlushWorkersCount(2), + WithPath(t.TempDir()), + WithGCInterval(1*time.Second)) + }) +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/get.go b/pkg/local_object_storage/writecache/writecachebadger/get.go new file mode 100644 index 000000000..36896c569 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/get.go @@ -0,0 +1,95 @@ +package writecachebadger + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/dgraph-io/badger/v4" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// Get returns object from write-cache. +// +// Returns an error of type apistatus.ObjectNotFound if the requested object is missing in write-cache. +func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { + _, span := tracing.StartSpanFromContext(ctx, "writecache.Get", + trace.WithAttributes( + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + + obj, err := c.getInternal(addr) + return obj, metaerr.Wrap(err) +} + +func (c *cache) getInternal(addr oid.Address) (*objectSDK.Object, error) { + found := false + storageType := writecache.StorageTypeUndefined + startedAt := time.Now() + defer func() { + c.metrics.Get(time.Since(startedAt), found, storageType) + }() + + k := addr2key(addr) + value, err := Get(c.db, k[:]) + if err == nil { + obj := objectSDK.New() + found = true + storageType = writecache.StorageTypeDB + return obj, obj.Unmarshal(value) + } + + return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) +} + +// Head returns object header from write-cache. +// +// Returns an error of type apistatus.ObjectNotFound if the requested object is missing in write-cache. +func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { + _, span := tracing.StartSpanFromContext(ctx, "writecache.Head", + trace.WithAttributes( + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + + obj, err := c.getInternal(addr) + if err != nil { + return nil, metaerr.Wrap(err) + } + + return obj.CutPayload(), nil +} + +// Get fetches object from the underlying database. +// Key should be a stringified address. +// +// Returns an error of type apistatus.ObjectNotFound if the requested object is missing in db. +func Get(db *badger.DB, key []byte) ([]byte, error) { + var value []byte + + err := db.View(func(tx *badger.Txn) error { + it, err := tx.Get(key) + if err != nil { + if err == badger.ErrKeyNotFound { + return logicerr.Wrap(apistatus.ObjectNotFound{}) + } + return err + } + v, err := it.ValueCopy(nil) + if err != nil { + return err + } + value = v + return nil + }) + + return value, metaerr.Wrap(err) +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/iterate.go b/pkg/local_object_storage/writecache/writecachebadger/iterate.go new file mode 100644 index 000000000..111242048 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/iterate.go @@ -0,0 +1,32 @@ +package writecachebadger + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/dgraph-io/badger/v4" +) + +// IterateDB iterates over all objects stored in badger.DB instance and passes them to f until error return. +// It is assumed that db is an underlying database of some WriteCache instance. +// +// DB must not be nil and should be opened. +func IterateDB(db *badger.DB, f func(oid.Address) error) error { + return metaerr.Wrap(db.View(func(tx *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false + it := tx.NewIterator(opts) + for it.Rewind(); it.Valid(); it.Next() { + var key internalKey + if got, want := len(it.Item().Key()), len(key); got != want { + return fmt.Errorf("invalid db key len: got %d, want %d", got, want) + } + copy(key[:], it.Item().Key()) + if err := f(key.address()); err != nil { + return err + } + } + return nil + })) +} diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/writecachebadger/mode.go similarity index 82% rename from pkg/local_object_storage/writecache/mode.go rename to pkg/local_object_storage/writecache/writecachebadger/mode.go index bdbbec7c9..9a39fa41a 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/writecachebadger/mode.go @@ -1,4 +1,4 @@ -package writecache +package writecachebadger import ( "context" @@ -7,18 +7,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) -// ErrReadOnly is returned when Put/Write is performed in a read-only mode. -var ErrReadOnly = logicerr.New("write-cache is in read-only mode") - -// ErrNotInitialized is returned when write-cache is initializing. -var ErrNotInitialized = logicerr.New("write-cache is not initialized yet") - // SetMode sets write-cache mode of operation. // When shard is put in read-only mode all objects in memory are flushed to disk // and all background jobs are suspended. diff --git a/pkg/local_object_storage/writecache/writecachebadger/options.go b/pkg/local_object_storage/writecache/writecachebadger/options.go new file mode 100644 index 000000000..635c1418f --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/options.go @@ -0,0 +1,141 @@ +package writecachebadger + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.uber.org/zap" +) + +// Option represents write-cache configuration option. +type Option func(*options) + +// meta is an interface for a metabase. +type metabase interface { + Exists(context.Context, meta.ExistsPrm) (meta.ExistsRes, error) + StorageID(context.Context, meta.StorageIDPrm) (meta.StorageIDRes, error) + UpdateStorageID(meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error) +} + +// blob is an interface for the blobstor. +type blob interface { + Put(context.Context, common.PutPrm) (common.PutRes, error) + NeedsCompression(obj *objectSDK.Object) bool + Exists(ctx context.Context, res common.ExistsPrm) (common.ExistsRes, error) +} + +type options struct { + log *logger.Logger + // path is a path to a directory for write-cache. + path string + // blobstor is the main persistent storage. + blobstor blob + // metabase is the metabase instance. + metabase metabase + // maxObjectSize is the maximum size of the object stored in the write-cache. + maxObjectSize uint64 + // workersCount is the number of workers flushing objects in parallel. + workersCount int + // maxCacheSize is the maximum total size of all objects saved in cache (DB + FS). + // 1 GiB by default. + maxCacheSize uint64 + // objCounters contains atomic counters for the number of objects stored in cache. + objCounters counters + // noSync is true iff FSTree allows unsynchronized writes. + noSync bool + // reportError is the function called when encountering disk errors in background workers. + reportError func(string, error) + // metrics is metrics implementation + metrics writecache.Metrics + // gcInterval is the interval duration to run the GC cycle. + gcInterval time.Duration +} + +// WithLogger sets logger. +func WithLogger(log *logger.Logger) Option { + return func(o *options) { + o.log = &logger.Logger{Logger: log.With(zap.String("component", "WriteCache"))} + } +} + +// WithPath sets path to writecache db. +func WithPath(path string) Option { + return func(o *options) { + o.path = path + } +} + +// WithBlobstor sets main object storage. +func WithBlobstor(bs *blobstor.BlobStor) Option { + return func(o *options) { + o.blobstor = bs + } +} + +// WithMetabase sets metabase. +func WithMetabase(db *meta.DB) Option { + return func(o *options) { + o.metabase = db + } +} + +// WithMaxObjectSize sets maximum object size to be stored in write-cache. +func WithMaxObjectSize(sz uint64) Option { + return func(o *options) { + if sz > 0 { + o.maxObjectSize = sz + } + } +} + +func WithFlushWorkersCount(c int) Option { + return func(o *options) { + if c > 0 { + o.workersCount = c + } + } +} + +// WithMaxCacheSize sets maximum write-cache size in bytes. +func WithMaxCacheSize(sz uint64) Option { + return func(o *options) { + o.maxCacheSize = sz + } +} + +// WithNoSync sets an option to allow returning to caller on PUT before write is persisted. +// Note, that we use this flag for FSTree only and DO NOT use it for a bolt DB because +// we cannot yet properly handle the corrupted database during the startup. This SHOULD NOT +// be relied upon and may be changed in future. +func WithNoSync(noSync bool) Option { + return func(o *options) { + o.noSync = noSync + } +} + +// WithReportErrorFunc sets error reporting function. +func WithReportErrorFunc(f func(string, error)) Option { + return func(o *options) { + o.reportError = f + } +} + +// WithMetrics sets metrics implementation. +func WithMetrics(metrics writecache.Metrics) Option { + return func(o *options) { + o.metrics = metrics + } +} + +// WithGCInterval sets the duration of the interval to run GC cycles. +func WithGCInterval(d time.Duration) Option { + return func(o *options) { + o.gcInterval = d + } +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/put.go b/pkg/local_object_storage/writecache/writecachebadger/put.go new file mode 100644 index 000000000..c03a0d336 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/put.go @@ -0,0 +1,82 @@ +package writecachebadger + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// Put puts object to write-cache. +// +// Returns ErrReadOnly if write-cache is in R/O mode. +// Returns ErrNotInitialized if write-cache has not been initialized yet. +// Returns ErrOutOfSpace if saving an object leads to WC's size overflow. +// Returns ErrBigObject if an objects exceeds maximum object size. +func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "writecache.Put", + trace.WithAttributes( + attribute.String("address", prm.Address.EncodeToString()), + attribute.Bool("dont_compress", prm.DontCompress), + )) + defer span.End() + + startedAt := time.Now() + added := false + storageType := writecache.StorageTypeUndefined + defer func() { + c.metrics.Put(time.Since(startedAt), added, storageType) + }() + + c.modeMtx.RLock() + defer c.modeMtx.RUnlock() + if c.readOnly() { + return common.PutRes{}, writecache.ErrReadOnly + } + + sz := uint64(len(prm.RawData)) + if sz > c.maxObjectSize { + return common.PutRes{}, writecache.ErrBigObject + } + + oi := objectInfo{ + addr: prm.Address, + obj: prm.Object, + data: prm.RawData, + } + + storageType = writecache.StorageTypeDB + err := c.put(oi) + if err == nil { + added = true + } + return common.PutRes{}, err +} + +// put persists objects to the write-cache database and +// pushes the to the flush workers queue. +func (c *cache) put(obj objectInfo) error { + cacheSize := c.estimateCacheSize() + if c.maxCacheSize < c.incSizeDB(cacheSize) { + return writecache.ErrOutOfSpace + } + + wb := c.db.NewWriteBatch() + k := addr2key(obj.addr) + _ = wb.Set(k[:], obj.data) + err := wb.Flush() + if err == nil { + storagelog.Write(c.log, + storagelog.AddressField(obj.addr), + storagelog.StorageTypeField(wcStorageType), + storagelog.OpField("db PUT"), + ) + c.objCounters.IncDB() + } + return err +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/state.go b/pkg/local_object_storage/writecache/writecachebadger/state.go new file mode 100644 index 000000000..994dfa3d5 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/state.go @@ -0,0 +1,57 @@ +package writecachebadger + +import ( + "fmt" + "math" + "sync/atomic" + + "github.com/dgraph-io/badger/v4" +) + +func (c *cache) estimateCacheSize() uint64 { + onDiskSize, _ := c.db.EstimateSize(nil) + c.metrics.SetEstimateSize(onDiskSize, 0) + return onDiskSize +} + +func (c *cache) incSizeDB(sz uint64) uint64 { + return sz + c.maxObjectSize +} + +type counters struct { + cDB atomic.Uint64 +} + +func (x *counters) IncDB() { + x.cDB.Add(1) +} + +func (x *counters) DecDB() { + x.cDB.Add(math.MaxUint64) +} + +func (x *counters) DB() uint64 { + return x.cDB.Load() +} + +func (c *cache) initCounters() error { + var inDB uint64 + err := c.db.View(func(tx *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false + it := tx.NewIterator(opts) + defer it.Close() + for it.Rewind(); it.Valid(); it.Next() { + inDB++ + } + return nil + }) + if err != nil { + return fmt.Errorf("could not read write-cache DB counter: %w", err) + } + + c.objCounters.cDB.Store(inDB) + c.metrics.SetActualCounters(inDB, 0) + + return nil +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/storage.go b/pkg/local_object_storage/writecache/writecachebadger/storage.go new file mode 100644 index 000000000..25d1900d1 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/storage.go @@ -0,0 +1,91 @@ +package writecachebadger + +import ( + "fmt" + "os" + "path/filepath" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/dgraph-io/badger/v4" + "go.uber.org/zap" +) + +// store represents persistent storage with in-memory LRU cache +// for flushed items on top of it. +type store struct { + db *badger.DB +} + +type internalKey [len(cid.ID{}) + len(oid.ID{})]byte + +func (k internalKey) address() oid.Address { + var addr oid.Address + var cnr cid.ID + var obj oid.ID + copy(cnr[:], k[:len(cnr)]) + copy(obj[:], k[len(cnr):]) + addr.SetContainer(cnr) + addr.SetObject(obj) + return addr +} + +func addr2key(addr oid.Address) internalKey { + var key internalKey + cnr, obj := addr.Container(), addr.Object() + copy(key[:len(cnr)], cnr[:]) + copy(key[len(cnr):], obj[:]) + return key +} + +const dbName = "small.badger" + +func (c *cache) openStore(readOnly bool) error { + err := util.MkdirAllX(c.path, os.ModePerm) + if err != nil { + return err + } + + c.db, err = OpenDB(filepath.Join(c.path, dbName), readOnly, c.log) + if err != nil { + return fmt.Errorf("could not open database: %w", err) + } + + return nil +} + +func (c *cache) deleteFromDB(keys []string) []string { + if len(keys) == 0 { + return keys + } + + wb := c.db.NewWriteBatch() + + var errorIndex int + for errorIndex = range keys { + if err := wb.Delete([]byte(keys[errorIndex])); err != nil { + break + } + } + + for i := 0; i < errorIndex; i++ { + c.objCounters.DecDB() + c.metrics.Evict(writecache.StorageTypeDB) + storagelog.Write(c.log, + storagelog.AddressField(keys[i]), + storagelog.StorageTypeField(wcStorageType), + storagelog.OpField("db DELETE"), + ) + } + + if err := wb.Flush(); err != nil { + c.log.Error(logs.WritecacheCantRemoveObjectsFromTheDatabase, zap.Error(err)) + } + + copy(keys, keys[errorIndex:]) + return keys[:len(keys)-errorIndex] +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/util.go b/pkg/local_object_storage/writecache/writecachebadger/util.go new file mode 100644 index 000000000..1bb278f0a --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebadger/util.go @@ -0,0 +1,36 @@ +package writecachebadger + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "github.com/dgraph-io/badger/v4" +) + +// OpenDB opens a badger instance for write-cache. Opens in read-only mode if ro is true. +func OpenDB(p string, ro bool, l *logger.Logger) (*badger.DB, error) { + return badger.Open(badger.DefaultOptions(p). + WithReadOnly(ro). + WithLoggingLevel(badger.ERROR). + WithLogger(badgerLoggerWrapper{l})) +} + +type badgerLoggerWrapper struct { + l *logger.Logger +} + +func (w badgerLoggerWrapper) Errorf(msg string, args ...any) { + w.l.Error(fmt.Sprintf(msg, args...)) +} + +func (w badgerLoggerWrapper) Warningf(msg string, args ...any) { + w.l.Error(fmt.Sprintf(msg, args...)) +} + +func (w badgerLoggerWrapper) Infof(msg string, args ...any) { + w.l.Error(fmt.Sprintf(msg, args...)) +} + +func (w badgerLoggerWrapper) Debugf(msg string, args ...any) { + w.l.Error(fmt.Sprintf(msg, args...)) +} diff --git a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go new file mode 100644 index 000000000..407d1a9ce --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go @@ -0,0 +1,146 @@ +package writecachebbolt + +import ( + "os" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.etcd.io/bbolt" + "go.uber.org/zap" +) + +type cache struct { + options + + // mtx protects statistics, counters and compressFlags. + mtx sync.RWMutex + + mode mode.Mode + modeMtx sync.RWMutex + + // compressFlags maps address of a big object to boolean value indicating + // whether object should be compressed. + compressFlags map[string]struct{} + + // flushCh is a channel with objects to flush. + flushCh chan *objectSDK.Object + // closeCh is close channel, protected by modeMtx. + closeCh chan struct{} + // wg is a wait group for flush workers. + wg sync.WaitGroup + // store contains underlying database. + store + // fsTree contains big files stored directly on file-system. + fsTree *fstree.FSTree +} + +// wcStorageType is used for write-cache operations logging. +const wcStorageType = "write-cache" + +type objectInfo struct { + addr string + data []byte + obj *objectSDK.Object +} + +const ( + defaultMaxObjectSize = 64 * 1024 * 1024 // 64 MiB + defaultSmallObjectSize = 32 * 1024 // 32 KiB + defaultMaxCacheSize = 1 << 30 // 1 GiB +) + +var ( + defaultBucket = []byte{0} +) + +// New creates new writecache instance. +func New(opts ...Option) writecache.Cache { + c := &cache{ + flushCh: make(chan *objectSDK.Object), + mode: mode.ReadWrite, + + compressFlags: make(map[string]struct{}), + options: options{ + log: &logger.Logger{Logger: zap.NewNop()}, + maxObjectSize: defaultMaxObjectSize, + smallObjectSize: defaultSmallObjectSize, + workersCount: defaultFlushWorkersCount, + maxCacheSize: defaultMaxCacheSize, + maxBatchSize: bbolt.DefaultMaxBatchSize, + maxBatchDelay: bbolt.DefaultMaxBatchDelay, + openFile: os.OpenFile, + metrics: writecache.DefaultMetrics(), + }, + } + + for i := range opts { + opts[i](&c.options) + } + + return c +} + +// SetLogger sets logger. It is used after the shard ID was generated to use it in logs. +func (c *cache) SetLogger(l *logger.Logger) { + c.log = l +} + +func (c *cache) DumpInfo() writecache.Info { + return writecache.Info{ + Path: c.path, + } +} + +// Open opens and initializes database. Reads object counters from the ObjectCounters instance. +func (c *cache) Open(readOnly bool) error { + err := c.openStore(readOnly) + if err != nil { + return metaerr.Wrap(err) + } + + // Opening after Close is done during maintenance mode, + // thus we need to create a channel here. + c.closeCh = make(chan struct{}) + + return metaerr.Wrap(c.initCounters()) +} + +// Init runs necessary services. +func (c *cache) Init() error { + c.metrics.SetMode(c.mode) + c.runFlushLoop() + return nil +} + +// Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. +func (c *cache) Close() error { + // We cannot lock mutex for the whole operation duration + // because it is taken by some background workers, so `wg.Wait()` is done without modeMtx. + c.modeMtx.Lock() + if c.closeCh != nil { + close(c.closeCh) + } + c.mode = mode.DegradedReadOnly // prevent new operations from being processed + c.modeMtx.Unlock() + + c.wg.Wait() + + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + + c.closeCh = nil + var err error + if c.db != nil { + err = c.db.Close() + if err != nil { + c.db = nil + } + } + c.metrics.Close() + return nil +} diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/writecachebbolt/delete.go similarity index 87% rename from pkg/local_object_storage/writecache/delete.go rename to pkg/local_object_storage/writecache/writecachebbolt/delete.go index aeab88b0b..b0cc091ab 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/delete.go @@ -1,4 +1,4 @@ -package writecache +package writecachebbolt import ( "context" @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -25,7 +26,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { defer span.End() deleted := false - storageType := StorageTypeUndefined + storageType := writecache.StorageTypeUndefined startedAt := time.Now() defer func() { c.metrics.Delete(time.Since(startedAt), deleted, storageType) @@ -34,7 +35,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { - return ErrReadOnly + return writecache.ErrReadOnly } saddr := addr.EncodeToString() @@ -47,7 +48,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { }) if dataSize > 0 { - storageType = StorageTypeDB + storageType = writecache.StorageTypeDB err := c.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) err := b.Delete([]byte(saddr)) @@ -66,7 +67,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { return nil } - storageType = StorageTypeFSTree + storageType = writecache.StorageTypeFSTree _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err == nil { storagelog.Write(c.log, diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/writecachebbolt/flush.go similarity index 94% rename from pkg/local_object_storage/writecache/flush.go rename to pkg/local_object_storage/writecache/writecachebbolt/flush.go index 243be4627..78018eeae 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush.go @@ -1,4 +1,4 @@ -package writecache +package writecachebbolt import ( "bytes" @@ -12,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -210,7 +211,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - err = c.flushObject(ctx, &obj, data, StorageTypeFSTree) + err = c.flushObject(ctx, &obj, data, writecache.StorageTypeFSTree) if err != nil { if ignoreErrors { return nil @@ -239,7 +240,7 @@ func (c *cache) workerFlushSmall() { return } - err := c.flushObject(context.TODO(), obj, nil, StorageTypeDB) + err := c.flushObject(context.TODO(), obj, nil, writecache.StorageTypeDB) if err != nil { // Error is handled in flushObject. continue @@ -250,7 +251,7 @@ func (c *cache) workerFlushSmall() { } // flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []byte, st StorageType) error { +func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []byte, st writecache.StorageType) error { var err error defer func() { @@ -330,7 +331,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { return err } - if err := c.flushObject(ctx, &obj, data, StorageTypeDB); err != nil { + if err := c.flushObject(ctx, &obj, data, writecache.StorageTypeDB); err != nil { return err } } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go new file mode 100644 index 000000000..465410bac --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go @@ -0,0 +1,106 @@ +package writecachebbolt + +import ( + "context" + "os" + "path/filepath" + "sync/atomic" + "testing" + + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachetest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" + "go.uber.org/zap/zaptest" +) + +func TestFlush(t *testing.T) { + createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs *blobstor.BlobStor, opts ...Option) writecache.Cache { + return New( + append([]Option{ + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithPath(filepath.Join(t.TempDir(), "writecache")), + WithSmallObjectSize(smallSize), + WithMetabase(mb), + WithBlobstor(bs), + }, opts...)...) + } + + errCountOpt := func() (Option, *atomic.Uint32) { + cnt := &atomic.Uint32{} + return WithReportErrorFunc(func(string, error) { + cnt.Add(1) + }), cnt + } + + failures := []writecachetest.TestFailureInjector[Option]{ + { + Desc: "db, invalid address", + InjectFn: func(t *testing.T, wc writecache.Cache) { + c := wc.(*cache) + obj := testutil.GenerateObject() + data, err := obj.Marshal() + require.NoError(t, err) + require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error { + b := tx.Bucket(defaultBucket) + return b.Put([]byte{1, 2, 3}, data) + })) + }, + }, + { + Desc: "db, invalid object", + InjectFn: func(t *testing.T, wc writecache.Cache) { + c := wc.(*cache) + require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error { + b := tx.Bucket(defaultBucket) + k := []byte(oidtest.Address().EncodeToString()) + v := []byte{1, 2, 3} + return b.Put(k, v) + })) + }, + }, + { + Desc: "fs, read error", + InjectFn: func(t *testing.T, wc writecache.Cache) { + c := wc.(*cache) + obj := testutil.GenerateObject() + data, err := obj.Marshal() + require.NoError(t, err) + + var prm common.PutPrm + prm.Address = objectCore.AddressOf(obj) + prm.RawData = data + + _, err = c.fsTree.Put(context.Background(), prm) + require.NoError(t, err) + + p := prm.Address.Object().EncodeToString() + "." + prm.Address.Container().EncodeToString() + p = filepath.Join(c.fsTree.RootPath, p[:1], p[1:]) + + _, err = os.Stat(p) // sanity check + require.NoError(t, err) + require.NoError(t, os.Truncate(p, 0)) // corrupt the file contents, so that it can't be unmarshalled + }, + }, + { + Desc: "fs, invalid object", + InjectFn: func(t *testing.T, wc writecache.Cache) { + c := wc.(*cache) + var prm common.PutPrm + prm.Address = oidtest.Address() + prm.RawData = []byte{1, 2, 3} + _, err := c.fsTree.Put(context.Background(), prm) + require.NoError(t, err) + }, + }, + } + + writecachetest.TestFlush(t, createCacheFn, errCountOpt, failures...) +} diff --git a/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go b/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go new file mode 100644 index 000000000..509efdd60 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go @@ -0,0 +1,18 @@ +package writecachebbolt + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "go.uber.org/zap/zaptest" +) + +func TestGeneric(t *testing.T) { + storagetest.TestAll(t, func(t *testing.T) storagetest.Component { + return New( + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithFlushWorkersCount(2), + WithPath(t.TempDir())) + }) +} diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/writecachebbolt/get.go similarity index 92% rename from pkg/local_object_storage/writecache/get.go rename to pkg/local_object_storage/writecache/writecachebbolt/get.go index 2546bada9..9d2bc39dc 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/get.go @@ -1,4 +1,4 @@ -package writecache +package writecachebbolt import ( "context" @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -35,7 +36,7 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) (*objectSDK.Object, error) { found := false - storageType := StorageTypeUndefined + storageType := writecache.StorageTypeUndefined startedAt := time.Now() defer func() { c.metrics.Get(time.Since(startedAt), found, storageType) @@ -45,7 +46,7 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) if err == nil { obj := objectSDK.New() found = true - storageType = StorageTypeDB + storageType = writecache.StorageTypeDB return obj, obj.Unmarshal(value) } @@ -55,7 +56,7 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) } found = true - storageType = StorageTypeFSTree + storageType = writecache.StorageTypeFSTree return res.Object, nil } diff --git a/pkg/local_object_storage/writecache/iterate.go b/pkg/local_object_storage/writecache/writecachebbolt/iterate.go similarity index 97% rename from pkg/local_object_storage/writecache/iterate.go rename to pkg/local_object_storage/writecache/writecachebbolt/iterate.go index 5349c069c..530db42a6 100644 --- a/pkg/local_object_storage/writecache/iterate.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/iterate.go @@ -1,4 +1,4 @@ -package writecache +package writecachebbolt import ( "errors" diff --git a/pkg/local_object_storage/writecache/writecachebbolt/mode.go b/pkg/local_object_storage/writecache/writecachebbolt/mode.go new file mode 100644 index 000000000..f7a9fffa3 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachebbolt/mode.go @@ -0,0 +1,75 @@ +package writecachebbolt + +import ( + "context" + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// SetMode sets write-cache mode of operation. +// When shard is put in read-only mode all objects in memory are flushed to disk +// and all background jobs are suspended. +func (c *cache) SetMode(m mode.Mode) error { + ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.SetMode", + trace.WithAttributes( + attribute.String("mode", m.String()), + )) + defer span.End() + + err := c.setMode(ctx, m) + if err == nil { + c.metrics.SetMode(m) + } + return err +} + +// setMode applies new mode. Must be called with cache.modeMtx lock taken. +func (c *cache) setMode(ctx context.Context, m mode.Mode) error { + var err error + turnOffMeta := m.NoMetabase() + + if turnOffMeta && !c.mode.NoMetabase() { + err = c.flush(ctx, true) + if err != nil { + return err + } + } + + if c.db != nil { + if err = c.db.Close(); err != nil { + return fmt.Errorf("can't close write-cache database: %w", err) + } + } + + // Suspend producers to ensure there are channel send operations in fly. + // flushCh is populated by `flush` with `modeMtx` taken, thus waiting until it is empty + // guarantees that there are no in-fly operations. + for len(c.flushCh) != 0 { + c.log.Info(logs.WritecacheWaitingForChannelsToFlush) + time.Sleep(time.Second) + } + + if turnOffMeta { + c.mode = m + return nil + } + + if err = c.openStore(m.ReadOnly()); err != nil { + return err + } + + c.mode = m + return nil +} + +// readOnly returns true if current mode is read-only. +// `c.modeMtx` must be taken. +func (c *cache) readOnly() bool { + return c.mode.ReadOnly() +} diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/writecachebbolt/options.go similarity index 96% rename from pkg/local_object_storage/writecache/options.go rename to pkg/local_object_storage/writecache/writecachebbolt/options.go index bea40aa36..0a21421ca 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/options.go @@ -1,4 +1,4 @@ -package writecache +package writecachebbolt import ( "context" @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" @@ -61,7 +62,7 @@ type options struct { // openFile is the function called internally by bbolt to open database files. Useful for hermetic testing. openFile func(string, int, fs.FileMode) (*os.File, error) // metrics is metrics implementation - metrics Metrics + metrics writecache.Metrics } // WithLogger sets logger. @@ -168,7 +169,7 @@ func WithOpenFile(f func(string, int, fs.FileMode) (*os.File, error)) Option { } // WithMetrics sets metrics implementation. -func WithMetrics(metrics Metrics) Option { +func WithMetrics(metrics writecache.Metrics) Option { return func(o *options) { o.metrics = metrics } diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/writecachebbolt/put.go similarity index 91% rename from pkg/local_object_storage/writecache/put.go rename to pkg/local_object_storage/writecache/writecachebbolt/put.go index 619b2bd26..505d091a5 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/put.go @@ -1,4 +1,4 @@ -package writecache +package writecachebbolt import ( "context" @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" @@ -37,7 +38,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro startedAt := time.Now() added := false - storageType := StorageTypeUndefined + storageType := writecache.StorageTypeUndefined defer func() { c.metrics.Put(time.Since(startedAt), added, storageType) }() @@ -45,7 +46,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { - return common.PutRes{}, ErrReadOnly + return common.PutRes{}, writecache.ErrReadOnly } sz := uint64(len(prm.RawData)) @@ -60,7 +61,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro } if sz <= c.smallObjectSize { - storageType = StorageTypeDB + storageType = writecache.StorageTypeDB err := c.putSmall(oi) if err == nil { added = true @@ -68,7 +69,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro return common.PutRes{}, err } - storageType = StorageTypeFSTree + storageType = writecache.StorageTypeFSTree err := c.putBig(ctx, oi.addr, prm) if err == nil { added = true diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/writecachebbolt/state.go similarity index 98% rename from pkg/local_object_storage/writecache/state.go rename to pkg/local_object_storage/writecache/writecachebbolt/state.go index 14103e626..950379751 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/state.go @@ -1,4 +1,4 @@ -package writecache +package writecachebbolt import ( "fmt" diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/writecachebbolt/storage.go similarity index 94% rename from pkg/local_object_storage/writecache/storage.go rename to pkg/local_object_storage/writecache/writecachebbolt/storage.go index 3bd3813d1..ab0b59906 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/storage.go @@ -1,4 +1,4 @@ -package writecache +package writecachebbolt import ( "context" @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -79,7 +80,7 @@ func (c *cache) deleteFromDB(keys []string) []string { }) for i := 0; i < errorIndex; i++ { c.objCounters.DecDB() - c.metrics.Evict(StorageTypeDB) + c.metrics.Evict(writecache.StorageTypeDB) storagelog.Write(c.log, storagelog.AddressField(keys[i]), storagelog.StorageTypeField(wcStorageType), @@ -122,7 +123,7 @@ func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree DELETE"), ) - c.metrics.Evict(StorageTypeFSTree) + c.metrics.Evict(writecache.StorageTypeFSTree) c.objCounters.DecFS() } } diff --git a/pkg/local_object_storage/writecache/util.go b/pkg/local_object_storage/writecache/writecachebbolt/util.go similarity index 95% rename from pkg/local_object_storage/writecache/util.go rename to pkg/local_object_storage/writecache/writecachebbolt/util.go index 0ed4a954e..fe225583c 100644 --- a/pkg/local_object_storage/writecache/util.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/util.go @@ -1,4 +1,4 @@ -package writecache +package writecachebbolt import ( "io/fs" diff --git a/pkg/local_object_storage/writecache/writecachetest/flush.go b/pkg/local_object_storage/writecache/writecachetest/flush.go new file mode 100644 index 000000000..e36778e03 --- /dev/null +++ b/pkg/local_object_storage/writecache/writecachetest/flush.go @@ -0,0 +1,185 @@ +package writecachetest + +import ( + "context" + "path/filepath" + "sync/atomic" + "testing" + + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/stretchr/testify/require" +) + +const ( + objCount = 4 + smallSize = 256 +) + +type CreateCacheFunc[Option any] func( + t *testing.T, + smallSize uint64, + meta *meta.DB, + bs *blobstor.BlobStor, + opts ...Option, +) writecache.Cache + +type TestFailureInjector[Option any] struct { + Desc string + InjectFn func(*testing.T, writecache.Cache) +} + +type objectPair struct { + addr oid.Address + obj *objectSDK.Object +} + +func TestFlush[Option any]( + t *testing.T, + createCacheFn CreateCacheFunc[Option], + errCountOption func() (Option, *atomic.Uint32), + failures ...TestFailureInjector[Option], +) { + t.Run("no errors", func(t *testing.T) { + wc, bs, mb := newCache(t, createCacheFn, smallSize) + objects := putObjects(t, wc) + + require.NoError(t, bs.SetMode(mode.ReadWrite)) + require.NoError(t, mb.SetMode(mode.ReadWrite)) + + require.NoError(t, wc.Flush(context.Background(), false)) + + check(t, mb, bs, objects) + }) + + t.Run("flush on moving to degraded mode", func(t *testing.T) { + wc, bs, mb := newCache(t, createCacheFn, smallSize) + objects := putObjects(t, wc) + + // Blobstor is read-only, so we expect en error from `flush` here. + require.Error(t, wc.SetMode(mode.Degraded)) + + // First move to read-only mode to close background workers. + require.NoError(t, wc.SetMode(mode.ReadOnly)) + require.NoError(t, bs.SetMode(mode.ReadWrite)) + require.NoError(t, mb.SetMode(mode.ReadWrite)) + require.NoError(t, wc.SetMode(mode.Degraded)) + + check(t, mb, bs, objects) + }) + + t.Run("ignore errors", func(t *testing.T) { + for _, f := range failures { + f := f + t.Run(f.Desc, func(t *testing.T) { + errCountOpt, errCount := errCountOption() + wc, bs, mb := newCache(t, createCacheFn, smallSize, errCountOpt) + objects := putObjects(t, wc) + f.InjectFn(t, wc) + + require.NoError(t, wc.SetMode(mode.ReadOnly)) + require.NoError(t, bs.SetMode(mode.ReadWrite)) + require.NoError(t, mb.SetMode(mode.ReadWrite)) + + require.Equal(t, uint32(0), errCount.Load()) + require.Error(t, wc.Flush(context.Background(), false)) + require.True(t, errCount.Load() > 0) + require.NoError(t, wc.Flush(context.Background(), true)) + + check(t, mb, bs, objects) + }) + } + }) +} + +func newCache[Option any]( + t *testing.T, + createCacheFn CreateCacheFunc[Option], + smallSize uint64, + opts ...Option, +) (writecache.Cache, *blobstor.BlobStor, *meta.DB) { + dir := t.TempDir() + mb := meta.New( + meta.WithPath(filepath.Join(dir, "meta")), + meta.WithEpochState(dummyEpoch{})) + require.NoError(t, mb.Open(false)) + require.NoError(t, mb.Init()) + + bs := blobstor.New(blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(dir, "blob")), + fstree.WithDepth(0), + fstree.WithDirNameLen(1)), + }, + })) + require.NoError(t, bs.Open(false)) + require.NoError(t, bs.Init()) + + wc := createCacheFn(t, smallSize, mb, bs, opts...) + t.Cleanup(func() { require.NoError(t, wc.Close()) }) + require.NoError(t, wc.Open(false)) + require.NoError(t, wc.Init()) + + // First set mode for metabase and blobstor to prevent background flushes. + require.NoError(t, mb.SetMode(mode.ReadOnly)) + require.NoError(t, bs.SetMode(mode.ReadOnly)) + + return wc, bs, mb +} + +func putObject(t *testing.T, c writecache.Cache, size int) objectPair { + obj := testutil.GenerateObjectWithSize(size) + data, err := obj.Marshal() + require.NoError(t, err) + + var prm common.PutPrm + prm.Address = objectCore.AddressOf(obj) + prm.Object = obj + prm.RawData = data + + _, err = c.Put(context.Background(), prm) + require.NoError(t, err) + + return objectPair{prm.Address, prm.Object} +} + +func putObjects(t *testing.T, c writecache.Cache) []objectPair { + objects := make([]objectPair, objCount) + for i := range objects { + objects[i] = putObject(t, c, 1+(i%2)*smallSize) + } + return objects +} + +func check(t *testing.T, mb *meta.DB, bs *blobstor.BlobStor, objects []objectPair) { + for i := range objects { + var mPrm meta.StorageIDPrm + mPrm.SetAddress(objects[i].addr) + + mRes, err := mb.StorageID(context.Background(), mPrm) + require.NoError(t, err) + + var prm common.GetPrm + prm.Address = objects[i].addr + prm.StorageID = mRes.StorageID() + + res, err := bs.Get(context.Background(), prm) + require.NoError(t, err) + require.Equal(t, objects[i].obj, res.Object) + } +} + +type dummyEpoch struct{} + +func (dummyEpoch) CurrentEpoch() uint64 { + return 0 +} From 5bbfebba2d62782141fe4684f69b636c51c80de1 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Mon, 7 Aug 2023 12:05:00 +0300 Subject: [PATCH 0468/1943] [#570] Fix writecache type constant copy-pasta bug Signed-off-by: Alejandro Lopez --- cmd/frostfs-node/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 2a84805dc..31b1a793c 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -728,7 +728,7 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) writecacheconfig.Options { writecachebbolt.WithLogger(c.log), ) case writecacheconfig.TypeBadger: - writeCacheOpts.Type = writecacheconfig.TypeBBolt + writeCacheOpts.Type = writecacheconfig.TypeBadger writeCacheOpts.BadgerOptions = append(writeCacheOpts.BadgerOptions, writecachebadger.WithPath(wcRead.path), writecachebadger.WithMaxObjectSize(wcRead.maxObjSize), From 0e697266c37dd17470a06f50bf90be56d9923d91 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 7 Aug 2023 13:31:56 +0300 Subject: [PATCH 0469/1943] [#563] writecache: Fix metrics and bolt delete Estimate cache size after delete objects to update metric. Update counters on small object deletion. Do not count bbolt DB file as FSTree object. Signed-off-by: Dmitrii Stepanov --- .../writecache/writecachebbolt/flush.go | 2 +- .../writecache/writecachebbolt/state.go | 3 +++ .../writecache/writecachebbolt/storage.go | 27 ++++++------------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush.go b/pkg/local_object_storage/writecache/writecachebbolt/flush.go index 78018eeae..4c1fb60e3 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush.go @@ -246,7 +246,7 @@ func (c *cache) workerFlushSmall() { continue } - c.deleteFromDB([]string{objectCore.AddressOf(obj).EncodeToString()}) + c.deleteFromDB(objectCore.AddressOf(obj).EncodeToString()) } } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/state.go b/pkg/local_object_storage/writecache/writecachebbolt/state.go index 950379751..03283dfef 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/state.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/state.go @@ -68,6 +68,9 @@ func (c *cache) initCounters() error { if err != nil { return fmt.Errorf("could not read write-cache FS counter: %w", err) } + if inFS > 0 { + inFS-- //small.bolt DB file + } c.objCounters.cDB.Store(inDB) c.objCounters.cFS.Store(inFS) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/storage.go b/pkg/local_object_storage/writecache/writecachebbolt/storage.go index ab0b59906..23f873d11 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/storage.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/storage.go @@ -63,36 +63,24 @@ func (c *cache) openStore(readOnly bool) error { return nil } -func (c *cache) deleteFromDB(keys []string) []string { - if len(keys) == 0 { - return keys - } - - var errorIndex int +func (c *cache) deleteFromDB(key string) { err := c.db.Batch(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) - for errorIndex = range keys { - if err := b.Delete([]byte(keys[errorIndex])); err != nil { - return err - } - } - return nil + return b.Delete([]byte(key)) }) - for i := 0; i < errorIndex; i++ { + + if err == nil { c.objCounters.DecDB() c.metrics.Evict(writecache.StorageTypeDB) storagelog.Write(c.log, - storagelog.AddressField(keys[i]), + storagelog.AddressField(key), storagelog.StorageTypeField(wcStorageType), storagelog.OpField("db DELETE"), ) - } - if err != nil { + c.estimateCacheSize() + } else { c.log.Error(logs.WritecacheCantRemoveObjectsFromTheDatabase, zap.Error(err)) } - - copy(keys, keys[errorIndex:]) - return keys[:len(keys)-errorIndex] } func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { @@ -125,6 +113,7 @@ func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { ) c.metrics.Evict(writecache.StorageTypeFSTree) c.objCounters.DecFS() + c.estimateCacheSize() } } From d3a52ec73a3c2052a121ec95549ddae9b1cd6f51 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Aug 2023 11:58:25 +0300 Subject: [PATCH 0470/1943] [#516] node: Send bootstrap request if attributes were updated Consider following situation: 1. Epoch tick. 2. Update node attributes in the config. 3. Restart node. Because we already sent bootstrap query after (1) and we are still online, new bootstrap query won't be sent. This leads to the node attributes being updated after another epoch. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config.go | 4 ++-- cmd/frostfs-node/netmap.go | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 31b1a793c..47f848279 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -343,8 +343,8 @@ type internals struct { apiVersion version.Version healthStatus *atomic.Int32 // is node under maintenance - isMaintenance atomic.Bool - isOnlineCandidate bool + isMaintenance atomic.Bool + alreadyBootstraped bool } // starts node's maintenance. diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 96b866b11..60a59940f 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -227,7 +227,7 @@ func bootstrapNode(c *cfg) { c.log.Info(logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) return } - if c.isOnlineCandidate { + if c.alreadyBootstraped { c.log.Info(logs.NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap) return } @@ -263,7 +263,7 @@ func initNetmapState(c *cfg) { fatalOnErrDetails("could not initialize current epoch number", err) var ni *netmapSDK.NodeInfo - ni, c.isOnlineCandidate, err = c.netmapInitLocalNodeState(epoch) + ni, c.alreadyBootstraped, err = c.netmapInitLocalNodeState(epoch) fatalOnErrDetails("could not init network state", err) stateWord := nodeState(ni) @@ -282,6 +282,13 @@ func initNetmapState(c *cfg) { c.handleLocalNodeInfo(ni) } +func sameNodeInfo(a, b *netmapSDK.NodeInfo) bool { + // Suboptimal, but we do this once on the node startup. + rawA := a.Marshal() + rawB := b.Marshal() + return bytes.Equal(rawA, rawB) +} + func nodeState(ni *netmapSDK.NodeInfo) string { if ni != nil { switch { @@ -303,11 +310,11 @@ func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, bool, } var candidate *netmapSDK.NodeInfo - isOnlineCandidate := false + alreadyBootstraped := false for i := range nmNodes { if bytes.Equal(nmNodes[i].PublicKey(), c.binPublicKey) { candidate = &nmNodes[i] - isOnlineCandidate = candidate.IsOnline() + alreadyBootstraped = candidate.IsOnline() && sameNodeInfo(&c.cfgNodeInfo.localInfo, candidate) break } } @@ -330,7 +337,7 @@ func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, bool, zap.String("netmap", nmState), zap.String("candidate", candidateState)) } - return candidate, isOnlineCandidate, nil + return candidate, alreadyBootstraped, nil } func (c *cfg) netmapLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { From 7da4306e38f2bbf6abf58de76f5c0029cc0368fc Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 8 Aug 2023 09:45:36 +0300 Subject: [PATCH 0471/1943] [#575] writecache: Fix log level for badger writecache Signed-off-by: Alejandro Lopez --- .../writecache/writecachebadger/util.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/writecache/writecachebadger/util.go b/pkg/local_object_storage/writecache/writecachebadger/util.go index 1bb278f0a..6d72c2039 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/util.go +++ b/pkg/local_object_storage/writecache/writecachebadger/util.go @@ -24,13 +24,13 @@ func (w badgerLoggerWrapper) Errorf(msg string, args ...any) { } func (w badgerLoggerWrapper) Warningf(msg string, args ...any) { - w.l.Error(fmt.Sprintf(msg, args...)) + w.l.Warn(fmt.Sprintf(msg, args...)) } func (w badgerLoggerWrapper) Infof(msg string, args ...any) { - w.l.Error(fmt.Sprintf(msg, args...)) + w.l.Info(fmt.Sprintf(msg, args...)) } func (w badgerLoggerWrapper) Debugf(msg string, args ...any) { - w.l.Error(fmt.Sprintf(msg, args...)) + w.l.Debug(fmt.Sprintf(msg, args...)) } From 8d589314b5b76d213f02a9f2cd152bec8bcd7d06 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 3 Aug 2023 10:39:48 +0300 Subject: [PATCH 0472/1943] [#560] node: Fix `Put` in multi REP with intersecting sets of nodes Once the node was processed it skipped, at the step of forming result in case when all nodes skipped, because processed for previous REP, service mark the whole request as incomplete. Example of policies which are unblocked: - REP 1 REP 1 CBF 1 - REP 4 IN X REP 4 IN Y CBF 4 SELECT 2 FROM FX AS X SELECT 2 FROM FY AS Y FILTER Country EQ Russia OR Country EQ Sweden OR Country EQ Finland AS FY FILTER Price GE 0 AS FX Signed-off-by: Anton Nikiforov --- pkg/services/object/put/distributed.go | 29 +++++++++++++------------- pkg/services/object/put/single.go | 11 +++++++--- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index f4afd44aa..4740ad1fe 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -48,7 +48,7 @@ type traversal struct { extraBroadcastEnabled bool // container nodes which was processed during the primary object placement - mExclude map[string]struct{} + mExclude map[string]*bool } // updates traversal parameters after the primary placement finish and @@ -68,24 +68,18 @@ func (x *traversal) submitPrimaryPlacementFinish() bool { } // marks the container node as processed during the primary object placement. -func (x *traversal) submitProcessed(n placement.Node) { +func (x *traversal) submitProcessed(n placement.Node, item *bool) { if x.extraBroadcastEnabled { key := string(n.PublicKey()) if x.mExclude == nil { - x.mExclude = make(map[string]struct{}, 1) + x.mExclude = make(map[string]*bool, 1) } - x.mExclude[key] = struct{}{} + x.mExclude[key] = item } } -// checks if specified node was processed during the primary object placement. -func (x *traversal) processed(n placement.Node) bool { - _, ok := x.mExclude[string(n.PublicKey())] - return ok -} - type nodeDesc struct { local bool @@ -181,6 +175,8 @@ func (t *distributedTarget) iteratePlacement(ctx context.Context) error { resErr := &atomic.Value{} + // Must iterate over all replicas, regardless of whether there are identical nodes there. + // At the same time need to exclude identical nodes from processing. for { addrs := traverser.Next() if len(addrs) == 0 { @@ -214,14 +210,18 @@ func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *pla wg := &sync.WaitGroup{} for i := range addrs { - if t.traversal.processed(addrs[i]) { + addr := addrs[i] + if val := t.traversal.mExclude[string(addr.PublicKey())]; val != nil { + // Check is node processed successful on the previous iteration. + if *val { + traverser.SubmitSuccess() + } // it can happen only during additional container broadcast continue } wg.Add(1) - - addr := addrs[i] + item := new(bool) workerPool, isLocal := t.getWorkerPool(addr.PublicKey()) if err := workerPool.Submit(func() { @@ -235,6 +235,7 @@ func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *pla } traverser.SubmitSuccess() + *item = true }); err != nil { wg.Done() svcutil.LogWorkerPoolError(t.log, "PUT", err) @@ -245,7 +246,7 @@ func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *pla // in subsequent container broadcast. Note that we don't // process this node during broadcast if primary placement // on it failed. - t.traversal.submitProcessed(addr) + t.traversal.submitProcessed(addr, item) } wg.Wait() diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 9d17a8128..7c7955c87 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -154,7 +154,7 @@ func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *o opts: placementOptions, extraBroadcastEnabled: len(obj.Children()) > 0 || (!localOnly && (obj.Type() == objectSDK.TypeTombstone || obj.Type() == objectSDK.TypeLock)), - mExclude: make(map[string]struct{}), + mExclude: make(map[string]*bool), } signer := &putSingleRequestSigner{ req: req, @@ -247,7 +247,10 @@ func (s *Service) saveToPlacementNodes(ctx context.Context, for _, nodeAddress := range nodeAddresses { nodeAddress := nodeAddress - if traversal.processed(nodeAddress) { + if ok := traversal.mExclude[string(nodeAddress.PublicKey())]; ok != nil { + if *ok { + traverser.SubmitSuccess() + } continue } @@ -258,6 +261,7 @@ func (s *Service) saveToPlacementNodes(ctx context.Context, workerPool = s.localPool } + item := new(bool) wg.Add(1) if err := workerPool.Submit(func() { defer wg.Done() @@ -271,13 +275,14 @@ func (s *Service) saveToPlacementNodes(ctx context.Context, } traverser.SubmitSuccess() + *item = true }); err != nil { wg.Done() svcutil.LogWorkerPoolError(s.log, "PUT", err) return true } - traversal.submitProcessed(nodeAddress) + traversal.submitProcessed(nodeAddress, item) } wg.Wait() From ae322e9f7338ed627a459bf4096a15e4d5ec3d2c Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Wed, 9 Aug 2023 11:28:26 +0300 Subject: [PATCH 0473/1943] [#576] Set SyncWrites for badger writecache by default Signed-off-by: Alejandro Lopez --- cmd/frostfs-node/config.go | 1 - .../writecache/writecachebadger/options.go | 12 ------------ .../writecache/writecachebadger/util.go | 1 + 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 47f848279..f5c106f77 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -734,7 +734,6 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) writecacheconfig.Options { writecachebadger.WithMaxObjectSize(wcRead.maxObjSize), writecachebadger.WithFlushWorkersCount(wcRead.flushWorkerCount), writecachebadger.WithMaxCacheSize(wcRead.sizeLimit), - writecachebadger.WithNoSync(wcRead.noSync), writecachebadger.WithLogger(c.log), writecachebadger.WithGCInterval(wcRead.gcInterval), ) diff --git a/pkg/local_object_storage/writecache/writecachebadger/options.go b/pkg/local_object_storage/writecache/writecachebadger/options.go index 635c1418f..1737885ba 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/options.go +++ b/pkg/local_object_storage/writecache/writecachebadger/options.go @@ -47,8 +47,6 @@ type options struct { maxCacheSize uint64 // objCounters contains atomic counters for the number of objects stored in cache. objCounters counters - // noSync is true iff FSTree allows unsynchronized writes. - noSync bool // reportError is the function called when encountering disk errors in background workers. reportError func(string, error) // metrics is metrics implementation @@ -109,16 +107,6 @@ func WithMaxCacheSize(sz uint64) Option { } } -// WithNoSync sets an option to allow returning to caller on PUT before write is persisted. -// Note, that we use this flag for FSTree only and DO NOT use it for a bolt DB because -// we cannot yet properly handle the corrupted database during the startup. This SHOULD NOT -// be relied upon and may be changed in future. -func WithNoSync(noSync bool) Option { - return func(o *options) { - o.noSync = noSync - } -} - // WithReportErrorFunc sets error reporting function. func WithReportErrorFunc(f func(string, error)) Option { return func(o *options) { diff --git a/pkg/local_object_storage/writecache/writecachebadger/util.go b/pkg/local_object_storage/writecache/writecachebadger/util.go index 6d72c2039..e55ed119d 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/util.go +++ b/pkg/local_object_storage/writecache/writecachebadger/util.go @@ -11,6 +11,7 @@ import ( func OpenDB(p string, ro bool, l *logger.Logger) (*badger.DB, error) { return badger.Open(badger.DefaultOptions(p). WithReadOnly(ro). + WithSyncWrites(true). WithLoggingLevel(badger.ERROR). WithLogger(badgerLoggerWrapper{l})) } From de3d1eb99c11cc2256bf22c35824eaed076c56cb Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 9 Aug 2023 09:40:50 +0300 Subject: [PATCH 0474/1943] [#581] Bump required go version to go1.20 Signed-off-by: Evgenii Stratonikov --- .docker/Dockerfile.adm | 2 +- .docker/Dockerfile.ci | 2 +- .docker/Dockerfile.cli | 2 +- .docker/Dockerfile.ir | 2 +- .docker/Dockerfile.storage | 2 +- .forgejo/workflows/build.yml | 2 +- .forgejo/workflows/tests.yml | 8 ++++---- .forgejo/workflows/vulncheck.yml | 2 +- CHANGELOG.md | 2 +- Makefile | 2 +- README.md | 2 +- go.mod | 2 +- 12 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.docker/Dockerfile.adm b/.docker/Dockerfile.adm index f0075c7ab..eeccaab79 100644 --- a/.docker/Dockerfile.adm +++ b/.docker/Dockerfile.adm @@ -1,4 +1,4 @@ -FROM golang:1.20 as builder +FROM golang:1.21 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ci b/.docker/Dockerfile.ci index e22762b7b..ef6586a64 100644 --- a/.docker/Dockerfile.ci +++ b/.docker/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM golang:1.20 +FROM golang:1.21 WORKDIR /tmp diff --git a/.docker/Dockerfile.cli b/.docker/Dockerfile.cli index 8fad863be..0dd4cebcf 100644 --- a/.docker/Dockerfile.cli +++ b/.docker/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM golang:1.20 as builder +FROM golang:1.21 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ir b/.docker/Dockerfile.ir index 0b01ea7e4..4015df673 100644 --- a/.docker/Dockerfile.ir +++ b/.docker/Dockerfile.ir @@ -1,4 +1,4 @@ -FROM golang:1.20 as builder +FROM golang:1.21 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage b/.docker/Dockerfile.storage index dd9ff2684..ced6ea538 100644 --- a/.docker/Dockerfile.storage +++ b/.docker/Dockerfile.storage @@ -1,4 +1,4 @@ -FROM golang:1.20 as builder +FROM golang:1.21 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 5e4a97b93..51f63e5a5 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.19', '1.20' ] + go_versions: [ '1.20', '1.21' ] steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 13395a64b..6e1afac9e 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -11,7 +11,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.21' cache: true - name: golangci-lint @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.19', '1.20' ] + go_versions: [ '1.20', '1.21' ] fail-fast: false steps: - uses: actions/checkout@v3 @@ -47,7 +47,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.21' cache: true - name: Run tests @@ -62,7 +62,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.21' cache: true - name: Install staticcheck diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 9390ad2d6..8ea01749b 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.20' + go-version: '1.21' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 6afb8f700..00837c79b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,7 @@ Changelog for FrostFS Node - `github.com/multiformats/go-multiaddr` to `v0.9.0` - `github.com/hashicorp/golang-lru/v2` to `v2.0.2` - `go.uber.org/atomic` to `v1.11.0` -- Minimum go version to v1.19 +- Minimum go version to v1.20 - `github.com/prometheus/client_golang` to `v1.15.1` - `github.com/prometheus/client_model` to `v0.4.0` - `go.opentelemetry.io/otel` to `v1.15.1` diff --git a/Makefile b/Makefile index df53b8772..e1464dd54 100755 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 HUB_IMAGE ?= truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" -GO_VERSION ?= 1.19 +GO_VERSION ?= 1.21 LINT_VERSION ?= 1.52.2 ARCH = amd64 diff --git a/README.md b/README.md index f228cd426..c3a9bf09c 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The latest version of frostfs-node works with frostfs-contract # Building -To make all binaries you need Go 1.19+ and `make`: +To make all binaries you need Go 1.20+ and `make`: ``` make all ``` diff --git a/go.mod b/go.mod index 7ad0fad88..14ee91aa4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-node -go 1.19 +go 1.20 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 From 5b7e4a51b76313a7502c65f048f4f695788a3039 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 4 Aug 2023 14:14:07 +0300 Subject: [PATCH 0475/1943] [#481] Update frostfs-sdk-go and error pointer receivers Signed-off-by: Alejandro Lopez --- cmd/frostfs-cli/internal/client/client.go | 2 +- cmd/frostfs-node/cache.go | 2 +- go.mod | 2 +- go.sum | 4 +- pkg/core/container/storage.go | 9 --- .../processors/container/handlers_test.go | 2 +- .../blobovnicza/blobovnicza_test.go | 5 +- .../blobovnicza/delete.go | 4 +- .../blobovnicza/errors.go | 13 ---- pkg/local_object_storage/blobovnicza/get.go | 4 +- .../blobstor/blobovniczatree/delete.go | 11 +-- .../blobstor/blobovniczatree/errors.go | 3 +- .../blobstor/blobovniczatree/exists.go | 3 +- .../blobstor/blobovniczatree/get.go | 11 +-- .../blobstor/blobovniczatree/get_range.go | 13 ++-- pkg/local_object_storage/blobstor/delete.go | 5 +- .../blobstor/fstree/fstree.go | 6 +- pkg/local_object_storage/blobstor/get.go | 6 +- .../blobstor/get_range.go | 6 +- .../blobstor/internal/blobstortest/delete.go | 10 +-- .../blobstor/internal/blobstortest/get.go | 4 +- .../internal/blobstortest/get_range.go | 11 +-- .../blobstor/memstore/memstore.go | 6 +- pkg/local_object_storage/engine/delete.go | 13 ++-- .../engine/delete_test.go | 21 +++--- pkg/local_object_storage/engine/error_test.go | 3 +- pkg/local_object_storage/engine/exists.go | 9 ++- pkg/local_object_storage/engine/get.go | 9 +-- pkg/local_object_storage/engine/head.go | 15 ++-- pkg/local_object_storage/engine/inhume.go | 10 +-- pkg/local_object_storage/engine/lock.go | 6 +- pkg/local_object_storage/engine/lock_test.go | 11 +-- pkg/local_object_storage/engine/range.go | 15 ++-- .../metabase/control_test.go | 3 +- pkg/local_object_storage/metabase/delete.go | 5 +- .../metabase/delete_test.go | 6 +- pkg/local_object_storage/metabase/errors.go | 9 --- pkg/local_object_storage/metabase/exists.go | 4 +- .../metabase/exists_test.go | 4 +- pkg/local_object_storage/metabase/get.go | 12 ++-- pkg/local_object_storage/metabase/get_test.go | 6 +- pkg/local_object_storage/metabase/inhume.go | 2 +- .../metabase/inhume_test.go | 15 ++-- pkg/local_object_storage/metabase/lock.go | 2 +- .../metabase/lock_test.go | 16 +++-- pkg/local_object_storage/shard/control.go | 3 +- .../shard/control_test.go | 13 ++-- pkg/local_object_storage/shard/delete.go | 3 +- pkg/local_object_storage/shard/delete_test.go | 6 +- pkg/local_object_storage/shard/errors.go | 15 +--- .../shard/gc_internal_test.go | 5 +- pkg/local_object_storage/shard/gc_test.go | 5 +- pkg/local_object_storage/shard/get.go | 5 +- pkg/local_object_storage/shard/get_test.go | 5 +- pkg/local_object_storage/shard/head_test.go | 5 +- pkg/local_object_storage/shard/inhume_test.go | 4 +- pkg/local_object_storage/shard/lock_test.go | 9 ++- pkg/local_object_storage/shard/range.go | 2 +- pkg/local_object_storage/shard/range_test.go | 3 +- .../writecache/writecachebadger/delete.go | 2 +- .../writecache/writecachebadger/get.go | 4 +- .../writecache/writecachebbolt/get.go | 4 +- .../writecache/writecachebbolt/storage.go | 5 +- pkg/morph/client/container/eacl.go | 4 +- pkg/morph/client/container/get.go | 4 +- pkg/services/object/acl/v2/service.go | 2 +- pkg/services/object/common.go | 18 +++-- pkg/services/object/get/assemble.go | 4 +- pkg/services/object/get/get_test.go | 71 +++++++++---------- pkg/services/object/get/local.go | 4 +- pkg/services/object/get/remote.go | 4 +- pkg/services/object/get/v2/util.go | 4 +- pkg/services/object/util/key.go | 8 +-- .../object_manager/tombstone/source/source.go | 5 +- pkg/services/policer/check.go | 4 +- pkg/services/policer/policer_test.go | 8 +-- pkg/services/util/sign.go | 2 +- 77 files changed, 265 insertions(+), 313 deletions(-) delete mode 100644 pkg/local_object_storage/blobovnicza/errors.go diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index babc0ef38..e250f8692 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -109,7 +109,7 @@ type GetContainerPrm struct { // SetContainer sets identifier of the container to be read. func (x *GetContainerPrm) SetContainer(id cid.ID) { - x.cliPrm.SetContainer(id) + *x.cliPrm.ContainerID = id } // GetContainerRes groups the resulting values of GetContainer operation. diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index bae38b299..fa5513640 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -208,7 +208,7 @@ func newCachedContainerStorage(v container.Source, ttl time.Duration) ttlContain } func (s ttlContainerStorage) handleRemoval(cnr cid.ID) { - s.set(cnr, nil, apistatus.ContainerNotFound{}) + s.set(cnr, nil, new(apistatus.ContainerNotFound)) } // Get returns container value from the cache. If value is missing in the cache diff --git a/go.mod b/go.mod index 14ee91aa4..e9f6ac9d9 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802090410-95b987b818d9 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230804083501-3dc8129ed794 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 07411ad4a..0a442b1ed 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802090410-95b987b818d9 h1:sCCTrMISLhfwqzbmUFvNW8MbwO1jzIHYgcEkxmokLd4= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230802090410-95b987b818d9/go.mod h1:LI2GOj0pEx0jYTjB3QHja2PNhQFYL2pCm71RAFwDv0M= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230804083501-3dc8129ed794 h1:gCIhE79b9fAM0Pixgv/Ot7iVmNJbAS8ZwBHLBnWS7Zc= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230804083501-3dc8129ed794/go.mod h1:LI2GOj0pEx0jYTjB3QHja2PNhQFYL2pCm71RAFwDv0M= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/pkg/core/container/storage.go b/pkg/core/container/storage.go index 3ed54529c..c79b7c041 100644 --- a/pkg/core/container/storage.go +++ b/pkg/core/container/storage.go @@ -1,9 +1,6 @@ package container import ( - "errors" - - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" @@ -37,12 +34,6 @@ type Source interface { Get(cid.ID) (*Container, error) } -// IsErrNotFound checks if the error returned by Source.Get corresponds -// to the missing container. -func IsErrNotFound(err error) bool { - return errors.As(err, new(apistatus.ContainerNotFound)) -} - // EACL groups information about the FrostFS container's extended ACL stored in // the FrostFS network. type EACL struct { diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index 1e518f474..77dbe876a 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -282,7 +282,7 @@ func (c *testContainerClient) Get(cid []byte) (*containercore.Container, error) if cont, found := c.get[key]; found { return cont, nil } - return nil, apistatus.ContainerNotFound{} + return nil, new(apistatus.ContainerNotFound) } type testIDClient struct { diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index 05f8906ac..c7b7bb8a9 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -8,6 +8,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -77,7 +78,7 @@ func TestBlobovnicza(t *testing.T) { require.NoError(t, blz.Init()) // try to read non-existent address - testGet(t, blz, oidtest.Address(), nil, IsErrNotFound) + testGet(t, blz, oidtest.Address(), nil, client.IsErrObjectNotFound) filled := uint64(15 * 1 << 10) @@ -92,7 +93,7 @@ func TestBlobovnicza(t *testing.T) { require.NoError(t, err) // should return 404 - testGet(t, blz, addr, nil, IsErrNotFound) + testGet(t, blz, addr, nil, client.IsErrObjectNotFound) // fill Blobovnicza fully for ; filled < sizeLim; filled += objSizeLim { diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 8fbd363db..419e073c6 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -77,9 +77,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err }) if err == nil && !found { - var errNotFound apistatus.ObjectNotFound - - return DeleteRes{}, errNotFound + return DeleteRes{}, new(apistatus.ObjectNotFound) } return DeleteRes{}, err diff --git a/pkg/local_object_storage/blobovnicza/errors.go b/pkg/local_object_storage/blobovnicza/errors.go deleted file mode 100644 index 2b0575c92..000000000 --- a/pkg/local_object_storage/blobovnicza/errors.go +++ /dev/null @@ -1,13 +0,0 @@ -package blobovnicza - -import ( - "errors" - - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" -) - -// IsErrNotFound checks if the error returned by Blobovnicza Get/Delete method -// corresponds to the missing object. -func IsErrNotFound(err error) bool { - return errors.As(err, new(apistatus.ObjectNotFound)) -} diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index d492b7559..3a613cfeb 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -72,9 +72,7 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { } if data == nil { - var errNotFound apistatus.ObjectNotFound - - return GetRes{}, errNotFound + return GetRes{}, new(apistatus.ObjectNotFound) } return GetRes{ diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 4d0801ef1..0698c232d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -69,7 +70,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co res, err = b.deleteObjectFromLevel(ctx, bPrm, p, !ok) if err != nil { - if !blobovnicza.IsErrNotFound(err) { + if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), @@ -89,7 +90,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co if err == nil && !objectFound { // not found in any blobovnicza - return common.DeleteRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) + return common.DeleteRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } success = err == nil @@ -109,7 +110,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicz if ok { if res, err := b.deleteObject(ctx, v, prm); err == nil { return res, err - } else if !blobovnicza.IsErrNotFound(err) { + } else if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), @@ -128,7 +129,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicz if ok && tryActive { if res, err := b.deleteObject(ctx, active.blz, prm); err == nil { return res, err - } else if !blobovnicza.IsErrNotFound(err) { + } else if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), @@ -142,7 +143,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicz // (Blobovniczas "after" the active one are empty anyway, // and it's pointless to open them). if u64FromHexString(filepath.Base(blzPath)) > active.ind { - return common.DeleteRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) + return common.DeleteRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } // open blobovnicza (cached inside) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/errors.go b/pkg/local_object_storage/blobstor/blobovniczatree/errors.go index 4ef053d54..04247280a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/errors.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/errors.go @@ -8,7 +8,8 @@ import ( ) func isErrOutOfRange(err error) bool { - return errors.As(err, new(apistatus.ObjectOutOfRange)) + var target *apistatus.ObjectOutOfRange + return errors.As(err, &target) } func isLogical(err error) bool { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index cd553ec30..c1964c227 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -57,7 +58,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common _, err := b.getObjectFromLevel(ctx, gPrm, p, !ok) if err != nil { - if !blobovnicza.IsErrNotFound(err) { + if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error())) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index e88310f8a..671d045b9 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -12,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" @@ -69,7 +70,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G res, err = b.getObjectFromLevel(ctx, bPrm, p, !ok) if err != nil { - if !blobovnicza.IsErrNotFound(err) { + if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), @@ -85,7 +86,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G if err == nil && res.Object == nil { // not found in any blobovnicza - return res, logicerr.Wrap(apistatus.ObjectNotFound{}) + return res, logicerr.Wrap(new(apistatus.ObjectNotFound)) } success = true @@ -107,7 +108,7 @@ func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.G if ok { if res, err := b.getObject(ctx, v, prm); err == nil { return res, err - } else if !blobovnicza.IsErrNotFound(err) { + } else if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotReadObjectFromOpenedBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), @@ -127,7 +128,7 @@ func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.G if ok && tryActive { if res, err := b.getObject(ctx, active.blz, prm); err == nil { return res, err - } else if !blobovnicza.IsErrNotFound(err) { + } else if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromActiveBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), @@ -141,7 +142,7 @@ func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.G // (Blobovniczas "after" the active one are empty anyway, // and it's pointless to open them). if u64FromHexString(filepath.Base(blzPath)) > active.ind { - return common.GetRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) + return common.GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } // open blobovnicza (cached inside) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 8579502c0..088ebe249 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -13,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" @@ -70,7 +71,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re res, err = b.getRangeFromLevel(ctx, prm, p, !ok) if err != nil { outOfBounds := isErrOutOfRange(err) - if !outOfBounds && !blobovnicza.IsErrNotFound(err) { + if !outOfBounds && !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), @@ -91,7 +92,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re if err == nil && !objectFound { // not found in any blobovnicza - return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) + return common.GetRangeRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } if err == nil { @@ -119,7 +120,7 @@ func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRang isErrOutOfRange(err): return res, err default: - if !blobovnicza.IsErrNotFound(err) { + if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotReadPayloadRangeFromOpenedBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), @@ -144,7 +145,7 @@ func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRang isErrOutOfRange(err): return res, err default: - if !blobovnicza.IsErrNotFound(err) { + if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotReadPayloadRangeFromActiveBlobovnicza, zap.String("path", blzPath), zap.String("error", err.Error()), @@ -159,7 +160,7 @@ func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRang // (Blobovniczas "after" the active one are empty anyway, // and it's pointless to open them). if u64FromHexString(filepath.Base(blzPath)) > active.ind { - return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) + return common.GetRangeRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } // open blobovnicza (cached inside) @@ -202,7 +203,7 @@ func (b *Blobovniczas) getObjectRange(ctx context.Context, blz *blobovnicza.Blob payload := obj.Payload() if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to { - return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectOutOfRange{}) + return common.GetRangeRes{}, logicerr.Wrap(new(apistatus.ObjectOutOfRange)) } return common.GetRangeRes{ diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index fe49fc46a..8930980a6 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -3,12 +3,11 @@ package blobstor import ( "context" "encoding/hex" - "errors" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -35,7 +34,7 @@ func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.Del if prm.StorageID == nil { for i := range b.storage { res, err := b.storage[i].Storage.Delete(ctx, prm) - if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { + if err == nil || !client.IsErrObjectNotFound(err) { if err == nil { success = true logOp(b.log, deleteOp, prm.Address, b.storage[i].Storage.Type(), prm.StorageID) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 9ca5d4bd9..5fbbdfea9 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -246,7 +246,7 @@ func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Delet err = os.Remove(p) if err != nil && os.IsNotExist(err) { - err = logicerr.Wrap(apistatus.ObjectNotFound{}) + err = logicerr.Wrap(new(apistatus.ObjectNotFound)) } return common.DeleteRes{}, err } @@ -439,7 +439,7 @@ func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, err p := t.treePath(prm.Address) if _, err := os.Stat(p); os.IsNotExist(err) { - return common.GetRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) + return common.GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } var data []byte @@ -498,7 +498,7 @@ func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.G to := from + prm.Range.GetLength() if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to { - return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectOutOfRange{}) + return common.GetRangeRes{}, logicerr.Wrap(new(apistatus.ObjectOutOfRange)) } success = true diff --git a/pkg/local_object_storage/blobstor/get.go b/pkg/local_object_storage/blobstor/get.go index fab86ea5d..00ed18d7b 100644 --- a/pkg/local_object_storage/blobstor/get.go +++ b/pkg/local_object_storage/blobstor/get.go @@ -3,12 +3,12 @@ package blobstor import ( "context" "encoding/hex" - "errors" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -39,12 +39,12 @@ func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (res common.GetRe if prm.StorageID == nil { for i := range b.storage { res, err = b.storage[i].Storage.Get(ctx, prm) - if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { + if err == nil || !client.IsErrObjectNotFound(err) { return res, err } } - return common.GetRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) + return common.GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } if len(prm.StorageID) == 0 { res, err = b.storage[len(b.storage)-1].Storage.Get(ctx, prm) diff --git a/pkg/local_object_storage/blobstor/get_range.go b/pkg/local_object_storage/blobstor/get_range.go index 671fbf02e..50abd19fa 100644 --- a/pkg/local_object_storage/blobstor/get_range.go +++ b/pkg/local_object_storage/blobstor/get_range.go @@ -3,13 +3,13 @@ package blobstor import ( "context" "encoding/hex" - "errors" "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -41,12 +41,12 @@ func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (res co if prm.StorageID == nil { for i := range b.storage { res, err = b.storage[i].Storage.GetRange(ctx, prm) - if err == nil || !errors.As(err, new(apistatus.ObjectNotFound)) { + if err == nil || !client.IsErrObjectNotFound(err) { return res, err } } - return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) + return common.GetRangeRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } if len(prm.StorageID) == 0 { res, err = b.storage[len(b.storage)-1].Storage.GetRange(ctx, prm) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go index 7532a5b5f..0bfac2772 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go @@ -5,7 +5,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) @@ -23,7 +23,7 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { prm.Address = oidtest.Address() _, err := s.Delete(context.Background(), prm) - require.Error(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) }) t.Run("with storage ID", func(t *testing.T) { @@ -43,12 +43,12 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { t.Run("get fail", func(t *testing.T) { prm := common.GetPrm{Address: oidtest.Address()} _, err := s.Get(context.Background(), prm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) }) t.Run("getrange fail", func(t *testing.T) { prm := common.GetRangePrm{Address: oidtest.Address()} _, err := s.GetRange(context.Background(), prm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) }) }) t.Run("without storage ID", func(t *testing.T) { @@ -68,7 +68,7 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { require.NoError(t, err) _, err = s.Delete(context.Background(), prm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) }) t.Run("non-deleted object is still available", func(t *testing.T) { diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go index c5755dfba..9a7ebed09 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go @@ -5,7 +5,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) @@ -21,7 +21,7 @@ func TestGet(t *testing.T, cons Constructor, min, max uint64) { t.Run("missing object", func(t *testing.T) { gPrm := common.GetPrm{Address: oidtest.Address()} _, err := s.Get(context.Background(), gPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) }) for i := range objects { diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go index b0c8aa95a..1a9da0726 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go @@ -6,6 +6,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -22,7 +23,7 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { t.Run("missing object", func(t *testing.T) { gPrm := common.GetRangePrm{Address: oidtest.Address()} _, err := s.GetRange(context.Background(), gPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) }) payload := objects[0].obj.Payload() @@ -56,7 +57,7 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { gPrm.Range.SetLength(10) _, err := s.GetRange(context.Background(), gPrm) - require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange)) + require.ErrorAs(t, err, new(*apistatus.ObjectOutOfRange)) }) t.Run("offset + length > len(payload)", func(t *testing.T) { @@ -64,7 +65,7 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { gPrm.Range.SetLength(uint64(len(payload))) _, err := s.GetRange(context.Background(), gPrm) - require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange)) + require.ErrorAs(t, err, new(*apistatus.ObjectOutOfRange)) }) t.Run("length is negative when converted to int64", func(t *testing.T) { @@ -72,7 +73,7 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { gPrm.Range.SetLength(1 << 63) _, err := s.GetRange(context.Background(), gPrm) - require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange)) + require.ErrorAs(t, err, new(*apistatus.ObjectOutOfRange)) }) t.Run("offset + length overflow uint64", func(t *testing.T) { @@ -80,6 +81,6 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { gPrm.Range.SetLength(math.MaxUint64 - 2) _, err := s.GetRange(context.Background(), gPrm) - require.ErrorAs(t, err, new(apistatus.ObjectOutOfRange)) + require.ErrorAs(t, err, new(*apistatus.ObjectOutOfRange)) }) } diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index b6cca2551..9428f457f 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -41,7 +41,7 @@ func (s *memstoreImpl) Get(_ context.Context, req common.GetPrm) (common.GetRes, s.mu.RUnlock() if !exists { - return common.GetRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) + return common.GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } // Decompress the data. @@ -73,7 +73,7 @@ func (s *memstoreImpl) GetRange(ctx context.Context, req common.GetRangePrm) (co to := from + req.Range.GetLength() if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to { - return common.GetRangeRes{}, logicerr.Wrap(apistatus.ObjectOutOfRange{}) + return common.GetRangeRes{}, logicerr.Wrap(new(apistatus.ObjectOutOfRange)) } return common.GetRangeRes{ @@ -123,7 +123,7 @@ func (s *memstoreImpl) Delete(_ context.Context, req common.DeletePrm) (common.D return common.DeleteRes{}, nil } - return common.DeleteRes{}, logicerr.Wrap(apistatus.ObjectNotFound{}) + return common.DeleteRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } func (s *memstoreImpl) Iterate(_ context.Context, req common.IteratePrm) (common.IterateRes, error) { diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 4d6d838bc..8e3180560 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -71,8 +72,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e } var locked struct { - is bool - err apistatus.ObjectLocked + is bool } var splitInfo *objectSDK.SplitInfo @@ -85,13 +85,13 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e resExists, err := sh.Exists(ctx, existsPrm) if err != nil { - if shard.IsErrRemoved(err) || shard.IsErrObjectExpired(err) { + if client.IsErrObjectAlreadyRemoved(err) || shard.IsErrObjectExpired(err) { return true } var splitErr *objectSDK.SplitInfoError if !errors.As(err, &splitErr) { - if !shard.IsErrNotFound(err) { + if !client.IsErrObjectNotFound(err) { e.reportShardError(sh, "could not check object existence", err) } return false @@ -111,7 +111,8 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e if err != nil { e.reportShardError(sh, "could not inhume object in shard", err) - locked.is = errors.As(err, &locked.err) + var target *apistatus.ObjectLocked + locked.is = errors.As(err, &target) return locked.is } @@ -121,7 +122,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e }) if locked.is { - return DeleteRes{}, locked.err + return DeleteRes{}, new(apistatus.ObjectLocked) } if splitInfo != nil { diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index bbc27615a..3ea0291da 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -65,16 +65,14 @@ func TestDeleteBigObject(t *testing.T) { } require.NoError(t, Put(context.Background(), e, link)) - var splitErr *objectSDK.SplitInfoError - addrParent := object.AddressOf(parent) - checkGetError(t, e, addrParent, &splitErr) + checkGetError[*objectSDK.SplitInfoError](t, e, addrParent, true) addrLink := object.AddressOf(link) - checkGetError(t, e, addrLink, nil) + checkGetError[error](t, e, addrLink, false) for i := range children { - checkGetError(t, e, object.AddressOf(children[i]), nil) + checkGetError[error](t, e, object.AddressOf(children[i]), false) } var deletePrm DeletePrm @@ -84,20 +82,21 @@ func TestDeleteBigObject(t *testing.T) { _, err := e.Delete(context.Background(), deletePrm) require.NoError(t, err) - checkGetError(t, e, addrParent, &apistatus.ObjectNotFound{}) - checkGetError(t, e, addrLink, &apistatus.ObjectNotFound{}) + checkGetError[*apistatus.ObjectNotFound](t, e, addrParent, true) + checkGetError[*apistatus.ObjectNotFound](t, e, addrLink, true) for i := range children { - checkGetError(t, e, object.AddressOf(children[i]), &apistatus.ObjectNotFound{}) + checkGetError[*apistatus.ObjectNotFound](t, e, object.AddressOf(children[i]), true) } } -func checkGetError(t *testing.T, e *StorageEngine, addr oid.Address, expected any) { +func checkGetError[E error](t *testing.T, e *StorageEngine, addr oid.Address, shouldFail bool) { var getPrm GetPrm getPrm.WithAddress(addr) _, err := e.Get(context.Background(), getPrm) - if expected != nil { - require.ErrorAs(t, err, expected) + if shouldFail { + var target E + require.ErrorAs(t, err, &target) } else { require.NoError(t, err) } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 18bc72d65..a77149e7c 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -18,7 +18,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" @@ -216,7 +215,7 @@ func TestBlobstorFailback(t *testing.T) { require.Equal(t, objs[i].Payload()[1:11], rngRes.Object().Payload()) _, err = te.ng.GetRange(context.Background(), RngPrm{addr: addr, off: errSmallSize + 10, ln: 1}) - require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) + require.True(t, shard.IsErrOutOfRange(err)) } checkShardState(t, te.ng, te.shards[0].id, 2, mode.ReadOnly) diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index 6208461e9..ef6292768 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -5,6 +5,7 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -19,7 +20,7 @@ func (e *StorageEngine) exists(ctx context.Context, addr oid.Address) (bool, err e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { res, err := sh.Exists(ctx, shPrm) if err != nil { - if shard.IsErrRemoved(err) { + if client.IsErrObjectAlreadyRemoved(err) { alreadyRemoved = true return true @@ -34,7 +35,7 @@ func (e *StorageEngine) exists(ctx context.Context, addr oid.Address) (bool, err return true } - if !shard.IsErrNotFound(err) { + if !client.IsErrObjectNotFound(err) { e.reportShardError(sh, "could not check existence of object in shard", err) } return false @@ -48,9 +49,7 @@ func (e *StorageEngine) exists(ctx context.Context, addr oid.Address) (bool, err }) if alreadyRemoved { - var errRemoved apistatus.ObjectAlreadyRemoved - - return false, errRemoved + return false, new(apistatus.ObjectAlreadyRemoved) } return exists, nil diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index bd094770a..2f736146c 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -67,7 +68,7 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { defer elapsed("Get", e.metrics.AddMethodDuration)() } - var errNotFound apistatus.ObjectNotFound + errNotFound := new(apistatus.ObjectNotFound) var shPrm shard.GetPrm shPrm.SetAddress(prm.addr) @@ -90,7 +91,7 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { } if it.Object == nil { - if !it.HasDegraded && it.ShardWithMeta.Shard == nil || !shard.IsErrNotFound(it.OutError) { + if !it.HasDegraded && it.ShardWithMeta.Shard == nil || !client.IsErrObjectNotFound(it.OutError) { return GetRes{}, it.OutError } @@ -144,7 +145,7 @@ func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { i.MetaError = err } switch { - case shard.IsErrNotFound(err): + case client.IsErrObjectNotFound(err): return false // ignore, go to next shard case errors.As(err, &i.splitInfoErr): if i.SplitInfo == nil { @@ -158,7 +159,7 @@ func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { // stop iterating over shards if SplitInfo structure is complete return withLink && withLast - case shard.IsErrRemoved(err): + case client.IsErrObjectAlreadyRemoved(err): i.OutError = err return true // stop, return it back case shard.IsErrObjectExpired(err): diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index ca51015db..ba5e7cc1d 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -74,11 +75,9 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) var ( head *objectSDK.Object siErr *objectSDK.SplitInfoError + outSI *objectSDK.SplitInfo - errNotFound apistatus.ObjectNotFound - - outSI *objectSDK.SplitInfo - outError error = errNotFound + outError error = new(apistatus.ObjectNotFound) ) var shPrm shard.HeadPrm @@ -89,7 +88,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) res, err := sh.Head(ctx, shPrm) if err != nil { switch { - case shard.IsErrNotFound(err): + case client.IsErrObjectNotFound(err): return false // ignore, go to next shard case errors.As(err, &siErr): if outSI == nil { @@ -107,16 +106,14 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) } return false - case shard.IsErrRemoved(err): + case client.IsErrObjectAlreadyRemoved(err): outError = err return true // stop, return it back case shard.IsErrObjectExpired(err): - var notFoundErr apistatus.ObjectNotFound - // object is found but should not // be returned - outError = notFoundErr + outError = new(apistatus.ObjectNotFound) return true default: diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 0b9ae602b..e0f9d7ab3 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -8,6 +8,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -94,8 +95,7 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e zap.Error(err), zap.Stringer("addr", prm.addrs[i])) } else if locked { - var lockedErr apistatus.ObjectLocked - return InhumeRes{}, lockedErr + return InhumeRes{}, new(apistatus.ObjectLocked) } } @@ -125,7 +125,6 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e // Returns ok if object was inhumed during this invocation or before. func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm shard.InhumePrm, checkExists bool) (bool, error) { root := false - var errLocked apistatus.ObjectLocked var existPrm shard.ExistsPrm var retErr error var ok bool @@ -143,7 +142,7 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh existPrm.SetAddress(addr) exRes, err := sh.Exists(ctx, existPrm) if err != nil { - if shard.IsErrRemoved(err) || shard.IsErrObjectExpired(err) { + if client.IsErrObjectAlreadyRemoved(err) || shard.IsErrObjectExpired(err) { // inhumed once - no need to be inhumed again ok = true return true @@ -163,9 +162,10 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh _, err := sh.Inhume(ctx, prm) if err != nil { + var errLocked *apistatus.ObjectLocked switch { case errors.As(err, &errLocked): - retErr = apistatus.ObjectLocked{} + retErr = new(apistatus.ObjectLocked) return true case errors.Is(err, shard.ErrLockObjectRemoval): retErr = meta.ErrLockObjectRemoval diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 61a5a0dc9..5ad603421 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -41,11 +41,11 @@ func (e *StorageEngine) lock(ctx context.Context, idCnr cid.ID, locker oid.ID, l for i := range locked { switch e.lockSingle(ctx, idCnr, locker, locked[i], true) { case 1: - return logicerr.Wrap(apistatus.LockNonRegularObject{}) + return logicerr.Wrap(new(apistatus.LockNonRegularObject)) case 0: switch e.lockSingle(ctx, idCnr, locker, locked[i], false) { case 1: - return logicerr.Wrap(apistatus.LockNonRegularObject{}) + return logicerr.Wrap(new(apistatus.LockNonRegularObject)) case 0: return logicerr.Wrap(errLockFailed) } @@ -62,7 +62,6 @@ func (e *StorageEngine) lock(ctx context.Context, idCnr cid.ID, locker oid.ID, l func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, locked oid.ID, checkExists bool) (status uint8) { // code is pretty similar to inhumeAddr, maybe unify? root := false - var errIrregular apistatus.LockNonRegularObject var addrLocked oid.Address addrLocked.SetContainer(idCnr) @@ -105,6 +104,7 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo if err != nil { e.reportShardError(sh, "could not lock object in shard", err) + var errIrregular *apistatus.LockNonRegularObject if errors.As(err, &errIrregular) { status = 1 return true diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index a7f0a1552..3e253bfc7 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -119,8 +119,9 @@ func TestLockUserScenario(t *testing.T) { var inhumePrm InhumePrm inhumePrm.WithTarget(tombAddr, objAddr) + var objLockedErr *apistatus.ObjectLocked _, err = e.Inhume(context.Background(), inhumePrm) - require.ErrorAs(t, err, new(apistatus.ObjectLocked)) + require.ErrorAs(t, err, &objLockedErr) // 4. tombObj.SetType(objectSDK.TypeTombstone) @@ -208,8 +209,9 @@ func TestLockExpiration(t *testing.T) { var inhumePrm InhumePrm inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj)) + var objLockedErr *apistatus.ObjectLocked _, err = e.Inhume(context.Background(), inhumePrm) - require.ErrorAs(t, err, new(apistatus.ObjectLocked)) + require.ErrorAs(t, err, &objLockedErr) // 3. e.HandleNewEpoch(lockerExpiresAfter + 1) @@ -281,13 +283,14 @@ func TestLockForceRemoval(t *testing.T) { var inhumePrm InhumePrm inhumePrm.MarkAsGarbage(objectcore.AddressOf(obj)) + var objLockedErr *apistatus.ObjectLocked _, err = e.Inhume(context.Background(), inhumePrm) - require.ErrorAs(t, err, new(apistatus.ObjectLocked)) + require.ErrorAs(t, err, &objLockedErr) inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj)) _, err = e.Inhume(context.Background(), inhumePrm) - require.ErrorAs(t, err, new(apistatus.ObjectLocked)) + require.ErrorAs(t, err, &objLockedErr) // 4. var deletePrm DeletePrm diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 328df4587..b3aaea6f5 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -83,14 +84,12 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error defer elapsed("GetRange", e.metrics.AddMethodDuration)() } - var errNotFound apistatus.ObjectNotFound - var shPrm shard.RngPrm shPrm.SetAddress(prm.addr) shPrm.SetRange(prm.off, prm.ln) it := &getRangeShardIterator{ - OutError: errNotFound, + OutError: new(apistatus.ObjectNotFound), ShardPrm: shPrm, Address: prm.addr, Engine: e, @@ -105,7 +104,7 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error if it.Object == nil { // If any shard is in a degraded mode, we should assume that metabase could store // info about some object. - if it.ShardWithMeta.Shard == nil && !it.HasDegraded || !shard.IsErrNotFound(it.OutError) { + if it.ShardWithMeta.Shard == nil && !it.HasDegraded || !client.IsErrObjectNotFound(it.OutError) { return RngRes{}, it.OutError } @@ -171,7 +170,7 @@ func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) { i.MetaError = err } switch { - case shard.IsErrNotFound(err): + case client.IsErrObjectNotFound(err): return false // ignore, go to next shard case errors.As(err, &i.SplitInfoError): if i.SplitInfo == nil { @@ -186,7 +185,7 @@ func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) { // stop iterating over shards if SplitInfo structure is complete return withLink && withLast case - shard.IsErrRemoved(err), + client.IsErrObjectAlreadyRemoved(err), shard.IsErrOutOfRange(err): i.OutError = err @@ -212,9 +211,7 @@ func (i *getRangeShardIterator) tryGetFromBlobstor(ctx context.Context) { res, err := sh.GetRange(ctx, i.ShardPrm) if shard.IsErrOutOfRange(err) { - var errOutOfRange apistatus.ObjectOutOfRange - - i.OutError = errOutOfRange + i.OutError = new(apistatus.ObjectOutOfRange) return true } i.Object = res.Object() diff --git a/pkg/local_object_storage/metabase/control_test.go b/pkg/local_object_storage/metabase/control_test.go index b67e748b3..8f36423fd 100644 --- a/pkg/local_object_storage/metabase/control_test.go +++ b/pkg/local_object_storage/metabase/control_test.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -43,7 +44,7 @@ func TestReset(t *testing.T) { require.NoError(t, err) assertExists(addr, true, nil) - assertExists(addrToInhume, false, meta.IsErrRemoved) + assertExists(addrToInhume, false, client.IsErrObjectAlreadyRemoved) err = db.Reset() require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index d387b3d04..f7ff7a129 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -11,7 +11,7 @@ import ( storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -204,9 +204,8 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter obj, err := db.get(tx, addr, key, false, true, currEpoch) if err != nil { var siErr *objectSDK.SplitInfoError - var notFoundErr apistatus.ObjectNotFound - if errors.As(err, ¬FoundErr) || errors.As(err, &siErr) { + if client.IsErrObjectNotFound(err) || errors.As(err, &siErr) { return false, false, 0, nil } diff --git a/pkg/local_object_storage/metabase/delete_test.go b/pkg/local_object_storage/metabase/delete_test.go index d2a4bfa7b..9b68f0bf8 100644 --- a/pkg/local_object_storage/metabase/delete_test.go +++ b/pkg/local_object_storage/metabase/delete_test.go @@ -8,7 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -68,11 +68,11 @@ func TestDB_Delete(t *testing.T) { // check if they marked as already removed ok, err := metaExists(db, object.AddressOf(child)) - require.ErrorAs(t, err, &apistatus.ObjectAlreadyRemoved{}) + require.True(t, client.IsErrObjectAlreadyRemoved(err)) require.False(t, ok) ok, err = metaExists(db, object.AddressOf(parent)) - require.ErrorAs(t, err, &apistatus.ObjectAlreadyRemoved{}) + require.True(t, client.IsErrObjectAlreadyRemoved(err)) require.False(t, ok) } diff --git a/pkg/local_object_storage/metabase/errors.go b/pkg/local_object_storage/metabase/errors.go index bc16dc01e..e9ffab439 100644 --- a/pkg/local_object_storage/metabase/errors.go +++ b/pkg/local_object_storage/metabase/errors.go @@ -1,19 +1,10 @@ package meta import ( - "errors" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) // ErrObjectIsExpired is returned when the requested object's // epoch is less than the current one. Such objects are considered // as removed and should not be returned from the Storage Engine. var ErrObjectIsExpired = logicerr.New("object is expired") - -// IsErrRemoved checks if error returned by Shard Exists/Get/Put method -// corresponds to removed object. -func IsErrRemoved(err error) bool { - return errors.As(err, new(apistatus.ObjectAlreadyRemoved)) -} diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index b6e5ea052..aa9aba106 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -82,9 +82,9 @@ func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (exists b // check graveyard and object expiration first switch objectStatus(tx, addr, currEpoch) { case 1: - return false, logicerr.Wrap(apistatus.ObjectNotFound{}) + return false, logicerr.Wrap(new(apistatus.ObjectNotFound)) case 2: - return false, logicerr.Wrap(apistatus.ObjectAlreadyRemoved{}) + return false, logicerr.Wrap(new(apistatus.ObjectAlreadyRemoved)) case 3: return false, ErrObjectIsExpired } diff --git a/pkg/local_object_storage/metabase/exists_test.go b/pkg/local_object_storage/metabase/exists_test.go index 034bc0005..06394339a 100644 --- a/pkg/local_object_storage/metabase/exists_test.go +++ b/pkg/local_object_storage/metabase/exists_test.go @@ -7,7 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -40,7 +40,7 @@ func TestDB_Exists(t *testing.T) { require.NoError(t, err) exists, err := metaExists(db, object.AddressOf(regular)) - require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) + require.True(t, client.IsErrObjectAlreadyRemoved(err)) require.False(t, exists) }) }) diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index ad35b4c18..d18331a3d 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -91,9 +91,9 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b if checkStatus { switch objectStatus(tx, addr, currEpoch) { case 1: - return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) + return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) case 2: - return nil, logicerr.Wrap(apistatus.ObjectAlreadyRemoved{}) + return nil, logicerr.Wrap(new(apistatus.ObjectAlreadyRemoved)) case 3: return nil, ErrObjectIsExpired } @@ -143,7 +143,7 @@ func getVirtualObject(tx *bbolt.Tx, cnr cid.ID, key []byte, raw bool) (*objectSD bucketName := make([]byte, bucketKeySize) parentBucket := tx.Bucket(parentBucketName(cnr, bucketName)) if parentBucket == nil { - return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) + return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) } relativeLst, err := decodeList(parentBucket.Get(key)) @@ -152,7 +152,7 @@ func getVirtualObject(tx *bbolt.Tx, cnr cid.ID, key []byte, raw bool) (*objectSD } if len(relativeLst) == 0 { // this should never happen though - return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) + return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) } // pick last item, for now there is not difference which address to pick @@ -171,7 +171,7 @@ func getVirtualObject(tx *bbolt.Tx, cnr cid.ID, key []byte, raw bool) (*objectSD par := child.Parent() if par == nil { // this should never happen though - return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) + return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) } return par, nil @@ -183,5 +183,5 @@ func getSplitInfoError(tx *bbolt.Tx, cnr cid.ID, key []byte) error { return logicerr.Wrap(objectSDK.NewSplitInfoError(splitInfo)) } - return logicerr.Wrap(apistatus.ObjectNotFound{}) + return logicerr.Wrap(new(apistatus.ObjectNotFound)) } diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index c19fea8d8..98a4bd960 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -12,7 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -114,7 +114,7 @@ func TestDB_Get(t *testing.T) { require.NoError(t, metaInhume(db, obj, ts)) _, err := metaGet(db, obj, false) - require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) + require.True(t, client.IsErrObjectAlreadyRemoved(err)) obj = oidtest.Address() @@ -124,7 +124,7 @@ func TestDB_Get(t *testing.T) { _, err = db.Inhume(context.Background(), prm) require.NoError(t, err) _, err = metaGet(db, obj, false) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) }) t.Run("expired object", func(t *testing.T) { diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 7ba5a68a2..fe8b8873e 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -166,7 +166,7 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes // prevent locked objects to be inhumed if !prm.forceRemoval && objectLocked(tx, cnr, id) { - return apistatus.ObjectLocked{} + return new(apistatus.ObjectLocked) } var lockWasChecked bool diff --git a/pkg/local_object_storage/metabase/inhume_test.go b/pkg/local_object_storage/metabase/inhume_test.go index 0f0774227..378e38e79 100644 --- a/pkg/local_object_storage/metabase/inhume_test.go +++ b/pkg/local_object_storage/metabase/inhume_test.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -28,10 +29,10 @@ func TestDB_Inhume(t *testing.T) { require.NoError(t, err) _, err = metaExists(db, object.AddressOf(raw)) - require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) + require.True(t, client.IsErrObjectAlreadyRemoved(err)) _, err = metaGet(db, object.AddressOf(raw), false) - require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) + require.True(t, client.IsErrObjectAlreadyRemoved(err)) } func TestInhumeTombOnTomb(t *testing.T) { @@ -58,7 +59,7 @@ func TestInhumeTombOnTomb(t *testing.T) { // addr1 should become inhumed {addr1:addr2} _, err = db.Exists(context.Background(), existsPrm) - require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) + require.True(t, client.IsErrObjectAlreadyRemoved(err)) inhumePrm.SetAddresses(addr3) inhumePrm.SetTombstoneAddress(addr1) @@ -72,13 +73,13 @@ func TestInhumeTombOnTomb(t *testing.T) { // NOT ObjectAlreadyRemoved since that record has been removed // from graveyard but addr1 is still marked with GC _, err = db.Exists(context.Background(), existsPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) existsPrm.SetAddress(addr3) // addr3 should be inhumed {addr3: addr1} _, err = db.Exists(context.Background(), existsPrm) - require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) + require.True(t, client.IsErrObjectAlreadyRemoved(err)) inhumePrm.SetAddresses(addr1) inhumePrm.SetTombstoneAddress(oidtest.Address()) @@ -93,7 +94,7 @@ func TestInhumeTombOnTomb(t *testing.T) { // (tomb can not be inhumed) but should be kept as object // with GC mark _, err = db.Exists(context.Background(), existsPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) } func TestInhumeLocked(t *testing.T) { @@ -109,7 +110,7 @@ func TestInhumeLocked(t *testing.T) { _, err = db.Inhume(context.Background(), prm) - var e apistatus.ObjectLocked + var e *apistatus.ObjectLocked require.ErrorAs(t, err, &e) } diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 50aac223b..30a31ab87 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -80,7 +80,7 @@ func (db *DB) lockInternal(locked []oid.ID, cnr cid.ID, locker oid.ID) error { return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { if firstIrregularObjectType(tx, cnr, bucketKeysLocked...) != objectSDK.TypeRegular { - return logicerr.Wrap(apistatus.LockNonRegularObject{}) + return logicerr.Wrap(new(apistatus.LockNonRegularObject)) } bucketLocked := tx.Bucket(bucketNameLocked) diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index b442296fd..834ab07a7 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -41,7 +41,7 @@ func TestDB_Lock(t *testing.T) { err := metaPut(db, obj, nil) require.NoError(t, err, typ) - var e apistatus.LockNonRegularObject + var e *apistatus.LockNonRegularObject id, _ := obj.ID() @@ -66,13 +66,15 @@ func TestDB_Lock(t *testing.T) { // check locking relation + var objLockedErr *apistatus.ObjectLocked + inhumePrm.SetAddresses(objAddr) _, err := db.Inhume(context.Background(), inhumePrm) - require.ErrorAs(t, err, new(apistatus.ObjectLocked)) + require.ErrorAs(t, err, &objLockedErr) inhumePrm.SetTombstoneAddress(oidtest.Address()) _, err = db.Inhume(context.Background(), inhumePrm) - require.ErrorAs(t, err, new(apistatus.ObjectLocked)) + require.ErrorAs(t, err, &objLockedErr) // try to remove lock object inhumePrm.SetAddresses(lockAddr) @@ -84,11 +86,11 @@ func TestDB_Lock(t *testing.T) { inhumePrm.SetAddresses(objAddr) _, err = db.Inhume(context.Background(), inhumePrm) - require.ErrorAs(t, err, new(apistatus.ObjectLocked)) + require.ErrorAs(t, err, &objLockedErr) inhumePrm.SetTombstoneAddress(oidtest.Address()) _, err = db.Inhume(context.Background(), inhumePrm) - require.ErrorAs(t, err, new(apistatus.ObjectLocked)) + require.ErrorAs(t, err, &objLockedErr) }) t.Run("lock-unlock scenario", func(t *testing.T) { @@ -97,9 +99,11 @@ func TestDB_Lock(t *testing.T) { objAddr := objectcore.AddressOf(objs[0]) lockAddr := objectcore.AddressOf(lockObj) + var objLockedErr *apistatus.ObjectLocked + // try to inhume locked object using tombstone err := metaInhume(db, objAddr, lockAddr) - require.ErrorAs(t, err, new(apistatus.ObjectLocked)) + require.ErrorAs(t, err, &objLockedErr) // free locked object var inhumePrm meta.InhumePrm diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index db8248c02..a49a79043 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -11,6 +11,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" @@ -195,7 +196,7 @@ func (s *Shard) refillMetabase(ctx context.Context) error { mPrm.SetStorageID(descriptor) _, err = s.metaBase.Put(ctx, mPrm) - if err != nil && !meta.IsErrRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) { + if err != nil && !client.IsErrObjectAlreadyRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) { return err } diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 23677c58a..deeb39300 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -20,6 +20,7 @@ import ( writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" @@ -170,7 +171,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) { var getPrm GetPrm getPrm.SetAddress(addr) _, err = sh.Get(context.Background(), getPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) require.NoError(t, sh.Close()) } @@ -297,7 +298,7 @@ func TestRefillMetabase(t *testing.T) { res, err := sh.Head(context.Background(), headPrm) if expObj == nil { - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) return } @@ -322,9 +323,9 @@ func TestRefillMetabase(t *testing.T) { _, err := sh.Head(context.Background(), headPrm) if exists { - require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) + require.True(t, client.IsErrObjectAlreadyRemoved(err)) } else { - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) } } } @@ -339,9 +340,9 @@ func TestRefillMetabase(t *testing.T) { var prm InhumePrm prm.MarkAsGarbage(addr) + var target *apistatus.ObjectLocked _, err := sh.Inhume(context.Background(), prm) - require.ErrorAs(t, err, new(apistatus.ObjectLocked), - "object %s should be locked", locked[i]) + require.ErrorAs(t, err, &target, "object %s should be locked", locked[i]) } } diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 2c7e0af27..c683d65a8 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -9,6 +9,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -79,7 +80,7 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { func (s *Shard) deleteObjectFromWriteCacheSafe(ctx context.Context, addr oid.Address) { if s.hasWriteCache() { err := s.writeCache.Delete(ctx, addr) - if err != nil && !IsErrNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { + if err != nil && !client.IsErrObjectNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.Error(err)) } } diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index 441e1c455..aba204f36 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -7,7 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" ) @@ -57,7 +57,7 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { require.NoError(t, err) _, err = sh.Get(context.Background(), getPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) }) t.Run("small object", func(t *testing.T) { @@ -81,6 +81,6 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { require.NoError(t, err) _, err = sh.Get(context.Background(), getPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) }) } diff --git a/pkg/local_object_storage/shard/errors.go b/pkg/local_object_storage/shard/errors.go index 2958a492c..045ad1bba 100644 --- a/pkg/local_object_storage/shard/errors.go +++ b/pkg/local_object_storage/shard/errors.go @@ -10,22 +10,11 @@ import ( var ErrShardDisabled = logicerr.New("shard disabled") -// IsErrNotFound checks if error returned by Shard Get/Head/GetRange method -// corresponds to missing object. -func IsErrNotFound(err error) bool { - return errors.As(err, new(apistatus.ObjectNotFound)) -} - -// IsErrRemoved checks if error returned by Shard Exists/Get/Head/GetRange method -// corresponds to removed object. -func IsErrRemoved(err error) bool { - return errors.As(err, new(apistatus.ObjectAlreadyRemoved)) -} - // IsErrOutOfRange checks if an error returned by Shard GetRange method // corresponds to exceeding the object bounds. func IsErrOutOfRange(err error) bool { - return errors.As(err, new(apistatus.ObjectOutOfRange)) + var target *apistatus.ObjectOutOfRange + return errors.As(err, &target) } // IsErrObjectExpired checks if an error returned by Shard corresponds to diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index bc895d67b..cf6633962 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -16,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -109,7 +110,7 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { require.NoError(t, err, "failed to inhume") _, err = sh.Get(context.Background(), getPrm) require.Error(t, err, "get returned error") - require.True(t, IsErrNotFound(err), "invalid error type") + require.True(t, client.IsErrObjectNotFound(err), "invalid error type") //storageID var metaStIDPrm meta.StorageIDPrm @@ -140,5 +141,5 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { //get should return object not found _, err = sh.Get(context.Background(), getPrm) require.Error(t, err, "get returned no error") - require.True(t, IsErrNotFound(err), "invalid error type") + require.True(t, client.IsErrObjectNotFound(err), "invalid error type") } diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 4b154462c..9d2771ae4 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -12,6 +12,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -74,7 +75,7 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { getPrm.SetAddress(objectCore.AddressOf(obj)) require.Eventually(t, func() bool { _, err = sh.Get(context.Background(), getPrm) - return shard.IsErrNotFound(err) + return client.IsErrObjectNotFound(err) }, 3*time.Second, 1*time.Second, "expired object must be deleted") } @@ -171,6 +172,6 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { require.Eventually(t, func() bool { _, err = sh.Get(context.Background(), getPrm) - return shard.IsErrNotFound(err) + return client.IsErrObjectNotFound(err) }, 3*time.Second, 1*time.Second, "expired complex object must be deleted on epoch after lock expires") } diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 3eb70784e..91565a346 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -129,7 +130,7 @@ func (s *Shard) fetchObjectData(ctx context.Context, addr oid.Address, skipMeta } if !mRes.Exists() { - return nil, false, logicerr.Wrap(apistatus.ObjectNotFound{}) + return nil, false, logicerr.Wrap(new(apistatus.ObjectNotFound)) } } else { s.log.Warn(logs.ShardFetchingObjectWithoutMeta, zap.Stringer("addr", addr)) @@ -140,7 +141,7 @@ func (s *Shard) fetchObjectData(ctx context.Context, addr oid.Address, skipMeta if err == nil || IsErrOutOfRange(err) { return res, false, err } - if IsErrNotFound(err) { + if client.IsErrObjectNotFound(err) { s.log.Debug(logs.ShardObjectIsMissingInWritecache, zap.Stringer("addr", addr), zap.Bool("skip_meta", skipMeta)) diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index 2db86c48a..25639902d 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -119,10 +120,10 @@ func testGet(t *testing.T, sh *shard.Shard, getPrm shard.GetPrm, hasWriteCache b res, err := sh.Get(context.Background(), getPrm) if hasWriteCache { require.Eventually(t, func() bool { - if shard.IsErrNotFound(err) { + if client.IsErrObjectNotFound(err) { res, err = sh.Get(context.Background(), getPrm) } - return !shard.IsErrNotFound(err) + return !client.IsErrObjectNotFound(err) }, time.Second, time.Millisecond*100) } return res, err diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index 7e336ea06..3a7457ace 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" @@ -90,10 +91,10 @@ func testHead(t *testing.T, sh *shard.Shard, headPrm shard.HeadPrm, hasWriteCach res, err := sh.Head(context.Background(), headPrm) if hasWriteCache { require.Eventually(t, func() bool { - if shard.IsErrNotFound(err) { + if client.IsErrObjectNotFound(err) { res, err = sh.Head(context.Background(), headPrm) } - return !shard.IsErrNotFound(err) + return !client.IsErrObjectNotFound(err) }, time.Second, time.Millisecond*100) } return res, err diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 4151d6218..3fa6bc0a3 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -7,7 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" ) @@ -56,5 +56,5 @@ func testShardInhume(t *testing.T, hasWriteCache bool) { require.NoError(t, err) _, err = sh.Get(context.Background(), getPrm) - require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) + require.True(t, client.IsErrObjectAlreadyRemoved(err)) } diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 61f2bb9f0..da71c4808 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -13,6 +13,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -96,12 +97,14 @@ func TestShard_Lock(t *testing.T) { var inhumePrm shard.InhumePrm inhumePrm.SetTarget(objectcore.AddressOf(ts), objectcore.AddressOf(obj)) + var objLockedErr *apistatus.ObjectLocked + _, err = sh.Inhume(context.Background(), inhumePrm) - require.ErrorAs(t, err, new(apistatus.ObjectLocked)) + require.ErrorAs(t, err, &objLockedErr) inhumePrm.MarkAsGarbage(objectcore.AddressOf(obj)) _, err = sh.Inhume(context.Background(), inhumePrm) - require.ErrorAs(t, err, new(apistatus.ObjectLocked)) + require.ErrorAs(t, err, &objLockedErr) }) t.Run("inhuming lock objects", func(t *testing.T) { @@ -141,7 +144,7 @@ func TestShard_Lock(t *testing.T) { getPrm.SetAddress(objectcore.AddressOf(obj)) _, err = sh.Get(context.Background(), getPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, client.IsErrObjectNotFound(err)) }) } diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index e94482e36..9491543c4 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -119,7 +119,7 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { from := prm.off to := from + prm.ln if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to { - return nil, logicerr.Wrap(apistatus.ObjectOutOfRange{}) + return nil, logicerr.Wrap(new(apistatus.ObjectOutOfRange)) } obj := objectSDK.New() diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 11a8aab5e..1806990ab 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -119,7 +119,8 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { res, err := sh.GetRange(context.Background(), rngPrm) if tc.hasErr { - require.ErrorAs(t, err, &apistatus.ObjectOutOfRange{}) + var target *apistatus.ObjectOutOfRange + require.ErrorAs(t, err, &target) } else { require.Equal(t, payload[tc.rng.GetOffset():tc.rng.GetOffset()+tc.rng.GetLength()], diff --git a/pkg/local_object_storage/writecache/writecachebadger/delete.go b/pkg/local_object_storage/writecache/writecachebadger/delete.go index 1b46b2be9..f37371098 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/delete.go +++ b/pkg/local_object_storage/writecache/writecachebadger/delete.go @@ -45,7 +45,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { it, err := tx.Get([]byte(saddr)) if err != nil { if err == badger.ErrKeyNotFound { - return logicerr.Wrap(apistatus.ObjectNotFound{}) + return logicerr.Wrap(new(apistatus.ObjectNotFound)) } return err } diff --git a/pkg/local_object_storage/writecache/writecachebadger/get.go b/pkg/local_object_storage/writecache/writecachebadger/get.go index 36896c569..42403e551 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/get.go +++ b/pkg/local_object_storage/writecache/writecachebadger/get.go @@ -47,7 +47,7 @@ func (c *cache) getInternal(addr oid.Address) (*objectSDK.Object, error) { return obj, obj.Unmarshal(value) } - return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) + return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) } // Head returns object header from write-cache. @@ -79,7 +79,7 @@ func Get(db *badger.DB, key []byte) ([]byte, error) { it, err := tx.Get(key) if err != nil { if err == badger.ErrKeyNotFound { - return logicerr.Wrap(apistatus.ObjectNotFound{}) + return logicerr.Wrap(new(apistatus.ObjectNotFound)) } return err } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/get.go b/pkg/local_object_storage/writecache/writecachebbolt/get.go index 9d2bc39dc..dede1fece 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/get.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/get.go @@ -52,7 +52,7 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) res, err := c.fsTree.Get(ctx, common.GetPrm{Address: addr}) if err != nil { - return nil, logicerr.Wrap(apistatus.ObjectNotFound{}) + return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) } found = true @@ -93,7 +93,7 @@ func Get(db *bbolt.DB, key []byte) ([]byte, error) { } value = b.Get(key) if value == nil { - return logicerr.Wrap(apistatus.ObjectNotFound{}) + return logicerr.Wrap(new(apistatus.ObjectNotFound)) } value = slice.Copy(value) return nil diff --git a/pkg/local_object_storage/writecache/writecachebbolt/storage.go b/pkg/local_object_storage/writecache/writecachebbolt/storage.go index 23f873d11..67397f985 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/storage.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/storage.go @@ -2,7 +2,6 @@ package writecachebbolt import ( "context" - "errors" "fmt" "os" @@ -12,7 +11,7 @@ import ( storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.uber.org/zap" @@ -98,7 +97,7 @@ func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { } _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) - if err != nil && !errors.As(err, new(apistatus.ObjectNotFound)) { + if err != nil && !client.IsErrObjectNotFound(err) { c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) // Save the key for the next iteration. diff --git a/pkg/morph/client/container/eacl.go b/pkg/morph/client/container/eacl.go index 54c83737d..8e9455050 100644 --- a/pkg/morph/client/container/eacl.go +++ b/pkg/morph/client/container/eacl.go @@ -55,9 +55,7 @@ func (c *Client) GetEACL(cnr cid.ID) (*container.EACL, error) { // The absence of a signature in the response can be taken as an eACL absence criterion, // since unsigned table cannot be approved in the storage by design. if len(sig) == 0 { - var errEACLNotFound apistatus.EACLNotFound - - return nil, errEACLNotFound + return nil, new(apistatus.EACLNotFound) } pub, err := client.BytesFromStackItem(arr[2]) diff --git a/pkg/morph/client/container/get.go b/pkg/morph/client/container/get.go index 0513eea4b..6715f870f 100644 --- a/pkg/morph/client/container/get.go +++ b/pkg/morph/client/container/get.go @@ -51,9 +51,7 @@ func (c *Client) Get(cid []byte) (*containercore.Container, error) { res, err := c.client.TestInvoke(prm) if err != nil { if strings.Contains(err.Error(), containerContract.NotFoundError) { - var errNotFound apistatus.ContainerNotFound - - return nil, errNotFound + return nil, new(apistatus.ContainerNotFound) } return nil, fmt.Errorf("could not perform test invocation (%s): %w", getMethod, err) } else if ln := len(res); ln != 1 { diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 412708b1c..271c4d20b 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -639,7 +639,7 @@ func (b Service) findRequestInfo(req MetaWithToken, idCnr cid.ID, op acl.Op) (in return info, errors.New("can't fetch current epoch") } if req.token.ExpiredAt(currentEpoch) { - return info, apistatus.SessionTokenExpired{} + return info, new(apistatus.SessionTokenExpired) } if req.token.InvalidAt(currentEpoch) { return info, fmt.Errorf("%s: token is invalid at %d epoch)", diff --git a/pkg/services/object/common.go b/pkg/services/object/common.go index 0d39dce0b..73ee9f81b 100644 --- a/pkg/services/object/common.go +++ b/pkg/services/object/common.go @@ -32,11 +32,9 @@ func (x *Common) Init(state NodeState, nextHandler ServiceServer) { x.nextHandler = nextHandler } -var errMaintenance apistatus.NodeUnderMaintenance - func (x *Common) Get(req *objectV2.GetRequest, stream GetObjectStream) error { if x.state.IsMaintenance() { - return errMaintenance + return new(apistatus.NodeUnderMaintenance) } return x.nextHandler.Get(req, stream) @@ -44,7 +42,7 @@ func (x *Common) Get(req *objectV2.GetRequest, stream GetObjectStream) error { func (x *Common) Put() (PutObjectStream, error) { if x.state.IsMaintenance() { - return nil, errMaintenance + return nil, new(apistatus.NodeUnderMaintenance) } return x.nextHandler.Put() @@ -52,7 +50,7 @@ func (x *Common) Put() (PutObjectStream, error) { func (x *Common) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { if x.state.IsMaintenance() { - return nil, errMaintenance + return nil, new(apistatus.NodeUnderMaintenance) } return x.nextHandler.Head(ctx, req) @@ -60,7 +58,7 @@ func (x *Common) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV2 func (x *Common) Search(req *objectV2.SearchRequest, stream SearchStream) error { if x.state.IsMaintenance() { - return errMaintenance + return new(apistatus.NodeUnderMaintenance) } return x.nextHandler.Search(req, stream) @@ -68,7 +66,7 @@ func (x *Common) Search(req *objectV2.SearchRequest, stream SearchStream) error func (x *Common) Delete(ctx context.Context, req *objectV2.DeleteRequest) (*objectV2.DeleteResponse, error) { if x.state.IsMaintenance() { - return nil, errMaintenance + return nil, new(apistatus.NodeUnderMaintenance) } return x.nextHandler.Delete(ctx, req) @@ -76,7 +74,7 @@ func (x *Common) Delete(ctx context.Context, req *objectV2.DeleteRequest) (*obje func (x *Common) GetRange(req *objectV2.GetRangeRequest, stream GetObjectRangeStream) error { if x.state.IsMaintenance() { - return errMaintenance + return new(apistatus.NodeUnderMaintenance) } return x.nextHandler.GetRange(req, stream) @@ -84,7 +82,7 @@ func (x *Common) GetRange(req *objectV2.GetRangeRequest, stream GetObjectRangeSt func (x *Common) GetRangeHash(ctx context.Context, req *objectV2.GetRangeHashRequest) (*objectV2.GetRangeHashResponse, error) { if x.state.IsMaintenance() { - return nil, errMaintenance + return nil, new(apistatus.NodeUnderMaintenance) } return x.nextHandler.GetRangeHash(ctx, req) @@ -92,7 +90,7 @@ func (x *Common) GetRangeHash(ctx context.Context, req *objectV2.GetRangeHashReq func (x *Common) PutSingle(ctx context.Context, req *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { if x.state.IsMaintenance() { - return nil, errMaintenance + return nil, new(apistatus.NodeUnderMaintenance) } return x.nextHandler.PutSingle(ctx, req) diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index 72ad506c7..6a8c5c818 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -63,8 +63,8 @@ func (r *request) assemble(ctx context.Context) { var errSplitInfo *objectSDK.SplitInfoError var errRemovedRemote *apistatus.ObjectAlreadyRemoved var errOutOfRangeRemote *apistatus.ObjectOutOfRange - var errRemovedLocal apistatus.ObjectAlreadyRemoved - var errOutOfRangeLocal apistatus.ObjectOutOfRange + var errRemovedLocal *apistatus.ObjectAlreadyRemoved + var errOutOfRangeLocal *apistatus.ObjectOutOfRange switch { default: diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 9c5506064..5b391fe2b 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -16,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -128,9 +129,7 @@ func (c *testClient) addResult(addr oid.Address, obj *objectSDK.Object, err erro func (c *testClient) Get(ctx context.Context, address oid.Address, requestParams RemoteRequestParams) (*objectSDK.Object, error) { v, ok := c.results[address.EncodeToString()] if !ok { - var errNotFound apistatus.ObjectNotFound - - return nil, errNotFound + return nil, new(apistatus.ObjectNotFound) } if v.err != nil { @@ -172,9 +171,7 @@ func (s *testStorage) Range(_ context.Context, address oid.Address, rng *objectS ) if _, ok = s.inhumed[sAddr]; ok { - var errRemoved apistatus.ObjectAlreadyRemoved - - return nil, errRemoved + return nil, new(apistatus.ObjectAlreadyRemoved) } if info, ok := s.virtual[sAddr]; ok { @@ -185,9 +182,7 @@ func (s *testStorage) Range(_ context.Context, address oid.Address, rng *objectS return cutToRange(obj, rng), nil } - var errNotFound apistatus.ObjectNotFound - - return nil, errNotFound + return nil, new(apistatus.ObjectNotFound) } func cutToRange(o *objectSDK.Object, rng *objectSDK.Range) *objectSDK.Object { @@ -374,19 +369,19 @@ func TestGetLocalOnly(t *testing.T) { err := svc.Get(ctx, p) - require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) + require.True(t, clientSDK.IsErrObjectAlreadyRemoved(err)) rngPrm := newRngPrm(false, nil, 0, 0) rngPrm.WithAddress(addr) err = svc.GetRange(ctx, rngPrm) - require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) + require.True(t, clientSDK.IsErrObjectAlreadyRemoved(err)) headPrm := newHeadPrm(false, nil) headPrm.WithAddress(addr) err = svc.Head(ctx, headPrm) - require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) + require.True(t, clientSDK.IsErrObjectAlreadyRemoved(err)) }) t.Run("404", func(t *testing.T) { @@ -401,20 +396,20 @@ func TestGetLocalOnly(t *testing.T) { err := svc.Get(ctx, p) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) rngPrm := newRngPrm(false, nil, 0, 0) rngPrm.WithAddress(addr) err = svc.GetRange(ctx, rngPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) headPrm := newHeadPrm(false, nil) headPrm.WithAddress(addr) err = svc.Head(ctx, headPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) }) t.Run("VIRTUAL", func(t *testing.T) { @@ -679,19 +674,19 @@ func TestGetRemoteSmall(t *testing.T) { p.WithAddress(addr) err := svc.Get(ctx, p) - require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) + require.True(t, clientSDK.IsErrObjectAlreadyRemoved(err)) rngPrm := newRngPrm(false, nil, 0, 0) rngPrm.WithAddress(addr) err = svc.GetRange(ctx, rngPrm) - require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) + require.True(t, clientSDK.IsErrObjectAlreadyRemoved(err)) headPrm := newHeadPrm(false, nil) headPrm.WithAddress(addr) err = svc.Head(ctx, headPrm) - require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) + require.True(t, clientSDK.IsErrObjectAlreadyRemoved(err)) }) t.Run("404", func(t *testing.T) { @@ -723,19 +718,19 @@ func TestGetRemoteSmall(t *testing.T) { p.WithAddress(addr) err := svc.Get(ctx, p) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) rngPrm := newRngPrm(false, nil, 0, 0) rngPrm.WithAddress(addr) err = svc.GetRange(ctx, rngPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) headPrm := newHeadPrm(false, nil) headPrm.WithAddress(addr) err = svc.Head(ctx, headPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) }) t.Run("VIRTUAL", func(t *testing.T) { @@ -768,11 +763,11 @@ func TestGetRemoteSmall(t *testing.T) { c1 := newTestClient() c1.addResult(addr, nil, errors.New("any error")) - c1.addResult(splitAddr, nil, apistatus.ObjectNotFound{}) + c1.addResult(splitAddr, nil, new(apistatus.ObjectNotFound)) c2 := newTestClient() c2.addResult(addr, nil, objectSDK.NewSplitInfoError(splitInfo)) - c2.addResult(splitAddr, nil, apistatus.ObjectNotFound{}) + c2.addResult(splitAddr, nil, new(apistatus.ObjectNotFound)) builder := &testPlacementBuilder{ vectors: map[string][][]netmap.NodeInfo{ @@ -794,13 +789,13 @@ func TestGetRemoteSmall(t *testing.T) { p.WithAddress(addr) err := svc.Get(ctx, p) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) rngPrm := newRngPrm(false, nil, 0, 0) rngPrm.WithAddress(addr) err = svc.GetRange(ctx, rngPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) }) t.Run("get chain element failure", func(t *testing.T) { @@ -845,7 +840,7 @@ func TestGetRemoteSmall(t *testing.T) { c2.addResult(addr, nil, objectSDK.NewSplitInfoError(splitInfo)) c2.addResult(linkAddr, linkingObj, nil) c2.addResult(child1Addr, children[0], nil) - c2.addResult(child2Addr, nil, apistatus.ObjectNotFound{}) + c2.addResult(child2Addr, nil, new(apistatus.ObjectNotFound)) builder := &testPlacementBuilder{ vectors: map[string][][]netmap.NodeInfo{ @@ -869,13 +864,13 @@ func TestGetRemoteSmall(t *testing.T) { p.WithAddress(addr) err := svc.Get(ctx, p) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) rngPrm := newRngPrm(false, NewSimpleObjectWriter(), 0, 1) rngPrm.WithAddress(addr) err = svc.GetRange(ctx, rngPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) }) t.Run("OK", func(t *testing.T) { @@ -1315,11 +1310,11 @@ func TestGetRemoteSmall(t *testing.T) { c1 := newTestClient() c1.addResult(addr, nil, errors.New("any error")) - c1.addResult(splitAddr, nil, apistatus.ObjectNotFound{}) + c1.addResult(splitAddr, nil, new(apistatus.ObjectNotFound)) c2 := newTestClient() c2.addResult(addr, nil, objectSDK.NewSplitInfoError(splitInfo)) - c2.addResult(splitAddr, nil, apistatus.ObjectNotFound{}) + c2.addResult(splitAddr, nil, new(apistatus.ObjectNotFound)) builder := &testPlacementBuilder{ vectors: map[string][][]netmap.NodeInfo{ @@ -1341,13 +1336,13 @@ func TestGetRemoteSmall(t *testing.T) { p.WithAddress(addr) err := svc.Get(ctx, p) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) rngPrm := newRngPrm(false, nil, 0, 0) rngPrm.WithAddress(addr) err = svc.GetRange(ctx, rngPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) }) t.Run("get chain element failure", func(t *testing.T) { @@ -1403,19 +1398,19 @@ func TestGetRemoteSmall(t *testing.T) { testHeadVirtual(svc, addr, splitInfo) headSvc := newTestClient() - headSvc.addResult(preRightAddr, nil, apistatus.ObjectNotFound{}) + headSvc.addResult(preRightAddr, nil, new(apistatus.ObjectNotFound)) p := newPrm(false, NewSimpleObjectWriter()) p.WithAddress(addr) err := svc.Get(ctx, p) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) rngPrm := newRngPrm(false, nil, 0, 1) rngPrm.WithAddress(addr) err = svc.GetRange(ctx, rngPrm) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) }) t.Run("child has different parent", func(t *testing.T) { @@ -1711,7 +1706,7 @@ func TestGetFromPastEpoch(t *testing.T) { p.WithAddress(addr) err := svc.Get(ctx, p) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) commonPrm.SetNetmapLookupDepth(1) @@ -1734,7 +1729,7 @@ func TestGetFromPastEpoch(t *testing.T) { rp.SetRange(r) err = svc.GetRange(ctx, rp) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) w = NewSimpleObjectWriter() rp.SetChunkWriter(w) @@ -1751,7 +1746,7 @@ func TestGetFromPastEpoch(t *testing.T) { hp.WithAddress(addr) err = svc.Head(ctx, hp) - require.ErrorAs(t, err, new(apistatus.ObjectNotFound)) + require.True(t, clientSDK.IsErrObjectNotFound(err)) w = NewSimpleObjectWriter() hp.SetHeaderWriter(w) diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index 03ede58cc..257465019 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -22,8 +22,8 @@ func (r *request) executeLocal(ctx context.Context) { r.collectedObject, err = r.get(ctx) var errSplitInfo *objectSDK.SplitInfoError - var errRemoved apistatus.ObjectAlreadyRemoved - var errOutOfRange apistatus.ObjectOutOfRange + var errRemoved *apistatus.ObjectAlreadyRemoved + var errOutOfRange *apistatus.ObjectOutOfRange switch { default: diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index e3464f941..4434f036a 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -31,10 +31,8 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { switch { default: - var errNotFound apistatus.ObjectNotFound - r.status = statusUndefined - r.err = errNotFound + r.err = new(apistatus.ObjectNotFound) r.log.Debug(logs.GetRemoteCallFailed, zap.Error(err)) case err == nil: diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 9ecc9167f..7f7dd7480 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -3,7 +3,6 @@ package getsvc import ( "context" "crypto/sha256" - "errors" "hash" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" @@ -15,6 +14,7 @@ import ( objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -154,7 +154,7 @@ func (s *Service) toHashRangePrm(req *objectV2.GetRangeHashRequest) (*getsvc.Ran ID: tok.ID(), Owner: tok.Issuer(), }) - if err != nil && errors.As(err, new(apistatus.SessionTokenNotFound)) { + if err != nil && clientSDK.IsErrSessionNotFound(err) { commonPrm.ForgetTokens() signerKey, err = s.keyStorage.GetKey(nil) } diff --git a/pkg/services/object/util/key.go b/pkg/services/object/util/key.go index e2ece5849..8304bf13a 100644 --- a/pkg/services/object/util/key.go +++ b/pkg/services/object/util/key.go @@ -68,16 +68,12 @@ func (s *KeyStorage) GetKey(info *SessionInfo) (*ecdsa.PrivateKey, error) { pToken := s.tokenStore.Get(info.Owner, binID) if pToken != nil { if pToken.ExpiredAt() <= s.networkState.CurrentEpoch() { - var errExpired apistatus.SessionTokenExpired - - return nil, errExpired + return nil, new(apistatus.SessionTokenExpired) } return pToken.SessionKey(), nil } - var errNotFound apistatus.SessionTokenNotFound - - return nil, errNotFound + return nil, new(apistatus.SessionTokenNotFound) } return s.key, nil diff --git a/pkg/services/object_manager/tombstone/source/source.go b/pkg/services/object_manager/tombstone/source/source.go index 4d0a81cde..1ff07b05a 100644 --- a/pkg/services/object_manager/tombstone/source/source.go +++ b/pkg/services/object_manager/tombstone/source/source.go @@ -2,12 +2,11 @@ package tsourse import ( "context" - "errors" "fmt" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -68,7 +67,7 @@ func (s Source) Tombstone(ctx context.Context, a oid.Address, _ uint64) (*object err := s.s.Head(ctx, headPrm) switch { - case errors.As(err, new(apistatus.ObjectNotFound)) || errors.As(err, new(apistatus.ObjectAlreadyRemoved)): + case client.IsErrObjectNotFound(err) || client.IsErrObjectAlreadyRemoved(err): return nil, nil case err != nil: return nil, fmt.Errorf("could not get tombstone from the source: %w", err) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index a45c019eb..800ddb006 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -5,7 +5,6 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -27,7 +26,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add zap.Stringer("cid", idCnr), zap.String("error", err.Error()), ) - if container.IsErrNotFound(err) { + if client.IsErrContainerNotFound(err) { err := p.buryFn(ctx, addrWithType.Address) if err != nil { p.log.Error(logs.PolicerCouldNotInhumeObjectWithMissingContainer, @@ -218,7 +217,6 @@ func isClientErrMaintenance(err error) bool { default: return false case - apistatus.NodeUnderMaintenance, *apistatus.NodeUnderMaintenance: return true } diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index c0aeac515..37d6c0506 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -35,7 +35,7 @@ func TestBuryObjectWithoutContainer(t *testing.T) { // Container source and bury function buryCh := make(chan oid.Address) containerSrc := func(id cid.ID) (*container.Container, error) { - return nil, apistatus.ContainerNotFound{} + return nil, new(apistatus.ContainerNotFound) } buryFn := func(ctx context.Context, a oid.Address) error { buryCh <- a @@ -187,7 +187,7 @@ func TestProcessObject(t *testing.T) { return nil, nil } } - return nil, apistatus.ObjectNotFound{} + return nil, new(apistatus.ObjectNotFound) } // Container source @@ -199,7 +199,7 @@ func TestProcessObject(t *testing.T) { return cnr, nil } t.Errorf("unexpected container requested: got=%v, want=%v", id, addr.Container()) - return nil, apistatus.ContainerNotFound{} + return nil, new(apistatus.ContainerNotFound) } buryFn := func(ctx context.Context, a oid.Address) error { t.Errorf("unexpected object buried: %v", a) @@ -252,7 +252,7 @@ func TestIteratorContract(t *testing.T) { }} containerSrc := func(id cid.ID) (*container.Container, error) { - return nil, apistatus.ContainerNotFound{} + return nil, new(apistatus.ContainerNotFound) } buryFn := func(ctx context.Context, a oid.Address) error { return nil diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index 26586ff49..bce43d6e8 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -51,7 +51,7 @@ func (s *SignService) SignResponse(resp ResponseMessage, err error) error { func (s *SignService) VerifyRequest(req RequestMessage) error { if err := signature.VerifyServiceMessage(req); err != nil { - var sigErr apistatus.SignatureVerification + sigErr := new(apistatus.SignatureVerification) sigErr.SetMessage(err.Error()) return sigErr } From 33c11be0cf6bd5dd796aaa686302e6546ed2dd85 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 9 Aug 2023 16:01:23 +0300 Subject: [PATCH 0476/1943] [#587] .golangci.yml: Fix pre-commit warnings Signed-off-by: Evgenii Stratonikov --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 643390075..eba0d605c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -31,7 +31,7 @@ linters-settings: statements: 60 # default 40 gocognit: min-complexity: 40 # default 30 - importas: + importas: no-unaliased: true no-extra-aliases: false alias: From d641cba2fc9d33d895da95387a8cf8ad3affe9bb Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 9 Aug 2023 16:01:44 +0300 Subject: [PATCH 0477/1943] [#587] Do not use math/rand.Read Fix staticcheck warnings after go1.20 update. Signed-off-by: Evgenii Stratonikov --- .../blobovnicza/blobovnicza_test.go | 4 +-- .../blobstor/internal/blobstortest/common.go | 6 ++-- .../metabase/index_test.go | 7 +++-- .../pilorama/forest_test.go | 31 ++++++++++--------- .../pilorama/meta_test.go | 2 +- .../shard/shutdown_test.go | 2 +- .../util/splitinfo_test.go | 2 +- pkg/morph/client/multy_test.go | 4 +-- pkg/morph/client/netmap/netmap_test.go | 2 +- 9 files changed, 33 insertions(+), 27 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index c7b7bb8a9..f8f638564 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -2,8 +2,8 @@ package blobovnicza import ( "context" + "crypto/rand" "errors" - "math/rand" "os" "testing" @@ -54,8 +54,6 @@ func testGet(t *testing.T, blz *Blobovnicza, addr oid.Address, expObj []byte, as } func TestBlobovnicza(t *testing.T) { - rand.Seed(1024) - p := "./test_blz" sizeLim := uint64(256 * 1 << 10) // 256KB diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/common.go b/pkg/local_object_storage/blobstor/internal/blobstortest/common.go index e31f3280a..1f234198e 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/common.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/common.go @@ -2,7 +2,8 @@ package blobstortest import ( "context" - "math/rand" + "crypto/rand" + mrand "math/rand" "testing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -53,8 +54,9 @@ func TestInfo(t *testing.T, cons Constructor, expectedType string, expectedPath func prepare(t *testing.T, count int, s common.Storage, min, max uint64) []objectDesc { objects := make([]objectDesc, count) + r := mrand.New(mrand.NewSource(0)) for i := range objects { - objects[i].obj = NewObject(min + uint64(rand.Intn(int(max-min+1)))) // not too large + objects[i].obj = NewObject(min + uint64(r.Intn(int(max-min+1)))) // not too large objects[i].addr = objectCore.AddressOf(objects[i].obj) raw, err := objects[i].obj.Marshal() diff --git a/pkg/local_object_storage/metabase/index_test.go b/pkg/local_object_storage/metabase/index_test.go index eb7238a59..45b9bc756 100644 --- a/pkg/local_object_storage/metabase/index_test.go +++ b/pkg/local_object_storage/metabase/index_test.go @@ -1,9 +1,11 @@ package meta import ( + "crypto/rand" "math" - "math/rand" + mrand "math/rand" "testing" + "time" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/stretchr/testify/require" @@ -39,9 +41,10 @@ func Test_decodeList(t *testing.T) { require.Error(t, err) }) t.Run("random", func(t *testing.T) { + r := mrand.New(mrand.NewSource(time.Now().Unix())) expected := make([][]byte, 20) for i := range expected { - expected[i] = make([]byte, rand.Uint32()%10) + expected[i] = make([]byte, r.Uint32()%10) rand.Read(expected[i]) } diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 5c143d3eb..0e2dd5ccb 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -2,12 +2,14 @@ package pilorama import ( "context" + "crypto/rand" "fmt" - "math/rand" + mrand "math/rand" "path/filepath" "strconv" "sync" "testing" + "time" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" @@ -767,9 +769,10 @@ func prepareRandomTree(nodeCount, opCount int) []Move { rand.Read(ops[i].Meta.Items[1].Value) } + r := mrand.New(mrand.NewSource(time.Now().Unix())) for i := nodeCount; i < len(ops); i++ { ops[i] = Move{ - Parent: rand.Uint64() % uint64(nodeCount+12), + Parent: r.Uint64() % uint64(nodeCount+12), Meta: Meta{ Time: Timestamp(i + nodeCount), Items: []KeyValue{ @@ -777,9 +780,9 @@ func prepareRandomTree(nodeCount, opCount int) []Move { {Value: make([]byte, 10)}, }, }, - Child: rand.Uint64() % uint64(nodeCount+10), + Child: r.Uint64() % uint64(nodeCount+10), } - if rand.Uint32()%5 == 0 { + if r.Uint32()%5 == 0 { ops[i].Parent = TrashID } rand.Read(ops[i].Meta.Items[1].Value) @@ -813,7 +816,7 @@ func compareForests(t *testing.T, expected, actual Forest, cid cidSDK.ID, treeID } func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ ...Option) Forest, batchSize, opCount, iterCount int) { - rand.Seed(42) + r := mrand.New(mrand.NewSource(42)) const nodeCount = 5 @@ -829,7 +832,7 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ for i := 0; i < iterCount; i++ { // Shuffle random operations, leave initialization in place. - rand.Shuffle(len(ops), func(i, j int) { ops[i], ops[j] = ops[j], ops[i] }) + r.Shuffle(len(ops), func(i, j int) { ops[i], ops[j] = ops[j], ops[i] }) actual := constructor(t, WithMaxBatchSize(batchSize)) wg := new(sync.WaitGroup) @@ -855,7 +858,7 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ } func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ ...Option) Forest) { - rand.Seed(42) + r := mrand.New(mrand.NewSource(42)) const ( nodeCount = 5 @@ -875,7 +878,7 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ .. const iterCount = 200 for i := 0; i < iterCount; i++ { // Shuffle random operations, leave initialization in place. - rand.Shuffle(len(ops), func(i, j int) { ops[i], ops[j] = ops[j], ops[i] }) + r.Shuffle(len(ops), func(i, j int) { ops[i], ops[j] = ops[j], ops[i] }) actual := constructor(t) for i := range ops { @@ -897,17 +900,18 @@ func BenchmarkApplySequential(b *testing.B) { b.Run(providers[i].name, func(b *testing.B) { for _, bs := range batchSizes { b.Run("batchsize="+strconv.Itoa(bs), func(b *testing.B) { + r := mrand.New(mrand.NewSource(time.Now().Unix())) s := providers[i].construct(b, WithMaxBatchSize(bs)) benchmarkApply(b, s, func(opCount int) []Move { ops := make([]Move, opCount) for i := range ops { ops[i] = Move{ - Parent: uint64(rand.Intn(benchNodeCount)), + Parent: uint64(r.Intn(benchNodeCount)), Meta: Meta{ Time: Timestamp(i), Items: []KeyValue{{Value: []byte{0, 1, 2, 3, 4}}}, }, - Child: uint64(rand.Intn(benchNodeCount)), + Child: uint64(r.Intn(benchNodeCount)), } } return ops @@ -930,17 +934,18 @@ func BenchmarkApplyReorderLast(b *testing.B) { b.Run(providers[i].name, func(b *testing.B) { for _, bs := range batchSizes { b.Run("batchsize="+strconv.Itoa(bs), func(b *testing.B) { + r := mrand.New(mrand.NewSource(time.Now().Unix())) s := providers[i].construct(b, WithMaxBatchSize(bs)) benchmarkApply(b, s, func(opCount int) []Move { ops := make([]Move, opCount) for i := range ops { ops[i] = Move{ - Parent: uint64(rand.Intn(benchNodeCount)), + Parent: uint64(r.Intn(benchNodeCount)), Meta: Meta{ Time: Timestamp(i), Items: []KeyValue{{Value: []byte{0, 1, 2, 3, 4}}}, }, - Child: uint64(rand.Intn(benchNodeCount)), + Child: uint64(r.Intn(benchNodeCount)), } if i != 0 && i%blockSize == 0 { for j := 0; j < blockSize/2; j++ { @@ -957,8 +962,6 @@ func BenchmarkApplyReorderLast(b *testing.B) { } func benchmarkApply(b *testing.B, s Forest, genFunc func(int) []Move) { - rand.Seed(42) - ops := genFunc(b.N) cid := cidtest.ID() treeID := "version" diff --git a/pkg/local_object_storage/pilorama/meta_test.go b/pkg/local_object_storage/pilorama/meta_test.go index 7adb97abd..9df4c7e94 100644 --- a/pkg/local_object_storage/pilorama/meta_test.go +++ b/pkg/local_object_storage/pilorama/meta_test.go @@ -1,7 +1,7 @@ package pilorama import ( - "math/rand" + "crypto/rand" "testing" "github.com/stretchr/testify/require" diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 15bff7f1e..68ef90963 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -2,7 +2,7 @@ package shard_test import ( "context" - "math/rand" + "crypto/rand" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" diff --git a/pkg/local_object_storage/util/splitinfo_test.go b/pkg/local_object_storage/util/splitinfo_test.go index 642fef3b8..0b7be5af3 100644 --- a/pkg/local_object_storage/util/splitinfo_test.go +++ b/pkg/local_object_storage/util/splitinfo_test.go @@ -1,7 +1,7 @@ package util_test import ( - "math/rand" + "crypto/rand" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" diff --git a/pkg/morph/client/multy_test.go b/pkg/morph/client/multy_test.go index 4bc38c70c..84a07b0a4 100644 --- a/pkg/morph/client/multy_test.go +++ b/pkg/morph/client/multy_test.go @@ -9,11 +9,11 @@ import ( ) func TestInitEndpoints(t *testing.T) { - rand.Seed(time.Now().UnixNano()) + r := rand.New(rand.NewSource(time.Now().UnixNano())) ee := make([]Endpoint, 100) for i := range ee { - ee[i].Priority = rand.Int() + ee[i].Priority = r.Int() } var eeInternal endpoints diff --git a/pkg/morph/client/netmap/netmap_test.go b/pkg/morph/client/netmap/netmap_test.go index d6172a682..a8a306197 100644 --- a/pkg/morph/client/netmap/netmap_test.go +++ b/pkg/morph/client/netmap/netmap_test.go @@ -1,8 +1,8 @@ package netmap import ( + "crypto/rand" "math/big" - "math/rand" "strconv" "testing" From 023b90342cf9448cd9867a28dae04d7e90d4450c Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 10 Aug 2023 10:14:06 +0300 Subject: [PATCH 0478/1943] [#584] Disable compression in badger writecache Signed-off-by: Alejandro Lopez --- pkg/local_object_storage/writecache/writecachebadger/util.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/local_object_storage/writecache/writecachebadger/util.go b/pkg/local_object_storage/writecache/writecachebadger/util.go index e55ed119d..e6079e370 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/util.go +++ b/pkg/local_object_storage/writecache/writecachebadger/util.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/dgraph-io/badger/v4" + badgeroptions "github.com/dgraph-io/badger/v4/options" ) // OpenDB opens a badger instance for write-cache. Opens in read-only mode if ro is true. @@ -12,6 +13,7 @@ func OpenDB(p string, ro bool, l *logger.Logger) (*badger.DB, error) { return badger.Open(badger.DefaultOptions(p). WithReadOnly(ro). WithSyncWrites(true). + WithCompression(badgeroptions.None). WithLoggingLevel(badger.ERROR). WithLogger(badgerLoggerWrapper{l})) } From 8f994163eea2088df4ce7ade816ffefe876d911f Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Wed, 9 Aug 2023 15:54:08 +0300 Subject: [PATCH 0479/1943] [#586] Fix writecache benchmarks and refactor hacky NeedsCompression Signed-off-by: Alejandro Lopez --- .../blobstor/blobovniczatree/blobovnicza.go | 4 + pkg/local_object_storage/blobstor/blobstor.go | 4 + .../blobstor/blobstor_test.go | 16 +-- .../blobstor/common/storage.go | 3 + .../blobstor/fstree/fstree.go | 4 + .../blobstor/memstore/control.go | 1 + pkg/local_object_storage/blobstor/put.go | 9 -- .../blobstor/teststore/option.go | 5 + .../blobstor/teststore/teststore.go | 13 ++ .../writecache/benchmark/writecache_test.go | 116 ++++++++++++++---- .../writecache/writecache.go | 14 +++ .../writecache/writecachebadger/flush_test.go | 3 +- .../writecache/writecachebadger/options.go | 27 +--- .../writecache/writecachebbolt/flush_test.go | 3 +- .../writecache/writecachebbolt/options.go | 27 +--- .../writecache/writecachebbolt/put.go | 2 +- .../writecache/writecachetest/flush.go | 2 +- 17 files changed, 161 insertions(+), 92 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 1b0af342d..a23bba425 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -252,6 +252,10 @@ func (b *Blobovniczas) SetCompressor(cc *compression.Config) { b.compression = cc } +func (b *Blobovniczas) Compressor() *compression.Config { + return b.compression +} + // SetReportErrorFunc implements common.Storage. func (b *Blobovniczas) SetReportErrorFunc(f func(string, error)) { b.reportError = f diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index 6c6aed87f..d2a2338a3 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -128,3 +128,7 @@ func WithMetrics(m Metrics) Option { c.metrics = m } } + +func (b *BlobStor) Compressor() *compression.Config { + return &b.cfg.compression +} diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index ef85f0f1d..878452069 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -146,33 +146,33 @@ func TestBlobstor_needsCompression(t *testing.T) { b := newBlobStor(t, true, "audio/*", "*/x-mpeg", "*/mpeg", "application/x-midi") obj := newObjectWithCt("video/mpeg") - require.False(t, b.NeedsCompression(obj)) + require.False(t, b.compression.NeedsCompression(obj)) obj = newObjectWithCt("audio/aiff") - require.False(t, b.NeedsCompression(obj)) + require.False(t, b.compression.NeedsCompression(obj)) obj = newObjectWithCt("application/x-midi") - require.False(t, b.NeedsCompression(obj)) + require.False(t, b.compression.NeedsCompression(obj)) obj = newObjectWithCt("text/plain") - require.True(t, b.NeedsCompression(obj)) + require.True(t, b.compression.NeedsCompression(obj)) obj = newObjectWithCt("") - require.True(t, b.NeedsCompression(obj)) + require.True(t, b.compression.NeedsCompression(obj)) }) t.Run("content-types omitted", func(t *testing.T) { b := newBlobStor(t, true) obj := newObjectWithCt("video/mpeg") - require.True(t, b.NeedsCompression(obj)) + require.True(t, b.compression.NeedsCompression(obj)) }) t.Run("compress disabled", func(t *testing.T) { b := newBlobStor(t, false, "video/mpeg") obj := newObjectWithCt("video/mpeg") - require.False(t, b.NeedsCompression(obj)) + require.False(t, b.compression.NeedsCompression(obj)) obj = newObjectWithCt("text/plain") - require.False(t, b.NeedsCompression(obj)) + require.False(t, b.compression.NeedsCompression(obj)) }) } diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index b808480e2..852f020cc 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -15,7 +15,10 @@ type Storage interface { Type() string Path() string + SetCompressor(cc *compression.Config) + Compressor() *compression.Config + // SetReportErrorFunc allows to provide a function to be called on disk errors. // This function MUST be called before Open. SetReportErrorFunc(f func(string, error)) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 5fbbdfea9..637bf8438 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -550,6 +550,10 @@ func (t *FSTree) SetCompressor(cc *compression.Config) { t.Config = cc } +func (t *FSTree) Compressor() *compression.Config { + return t.Config +} + // SetReportErrorFunc implements common.Storage. func (t *FSTree) SetReportErrorFunc(_ func(string, error)) { // Do nothing, FSTree can encounter only one error which is returned. diff --git a/pkg/local_object_storage/blobstor/memstore/control.go b/pkg/local_object_storage/blobstor/memstore/control.go index e6943626b..7f6e85a16 100644 --- a/pkg/local_object_storage/blobstor/memstore/control.go +++ b/pkg/local_object_storage/blobstor/memstore/control.go @@ -12,5 +12,6 @@ func (s *memstoreImpl) Close() error { return nil } func (s *memstoreImpl) Type() string { return Type } func (s *memstoreImpl) Path() string { return s.rootPath } func (s *memstoreImpl) SetCompressor(cc *compression.Config) { s.compression = cc } +func (s *memstoreImpl) Compressor() *compression.Config { return s.compression } func (s *memstoreImpl) SetReportErrorFunc(f func(string, error)) { s.reportError = f } func (s *memstoreImpl) SetParentID(string) {} diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index a748750dd..1adae303d 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -72,11 +71,3 @@ func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e return common.PutRes{}, ErrNoPlaceFound } - -// NeedsCompression returns true if the object should be compressed. -// For an object to be compressed 2 conditions must hold: -// 1. Compression is enabled in settings. -// 2. Object MIME Content-Type is allowed for compression. -func (b *BlobStor) NeedsCompression(obj *objectSDK.Object) bool { - return b.cfg.compression.NeedsCompression(obj) -} diff --git a/pkg/local_object_storage/blobstor/teststore/option.go b/pkg/local_object_storage/blobstor/teststore/option.go index 2797a1ee5..b897832cf 100644 --- a/pkg/local_object_storage/blobstor/teststore/option.go +++ b/pkg/local_object_storage/blobstor/teststore/option.go @@ -15,6 +15,7 @@ type cfg struct { Type func() string Path func() string SetCompressor func(cc *compression.Config) + Compressor func() *compression.Config SetReportErrorFunc func(f func(string, error)) Get func(common.GetPrm) (common.GetRes, error) @@ -45,6 +46,10 @@ func WithSetCompressor(f func(*compression.Config)) Option { return func(c *cfg) { c.overrides.SetCompressor = f } } +func WithCompressor(f func() *compression.Config) Option { + return func(c *cfg) { c.overrides.Compressor = f } +} + func WithReportErrorFunc(f func(func(string, error))) Option { return func(c *cfg) { c.overrides.SetReportErrorFunc = f } } diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 2508d74f6..3e5b21251 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -128,6 +128,19 @@ func (s *TestStore) SetCompressor(cc *compression.Config) { } } +func (s *TestStore) Compressor() *compression.Config { + s.mu.RLock() + defer s.mu.RUnlock() + switch { + case s.overrides.Compressor != nil: + return s.overrides.Compressor() + case s.st != nil: + return s.st.Compressor() + default: + panic("unexpected storage call: Compressor()") + } +} + func (s *TestStore) SetReportErrorFunc(f func(string, error)) { s.mu.RLock() defer s.mu.RUnlock() diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index 4c44b1d44..6ae04a92a 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -3,50 +3,120 @@ package benchmark import ( "context" "testing" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" - oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" ) -func BenchmarkWritecache(b *testing.B) { - b.Run("bbolt", func(b *testing.B) { - cache := writecachebbolt.New( - writecachebbolt.WithPath(b.TempDir()), - ) - benchmarkPut(b, cache) +func BenchmarkWritecacheSeq(b *testing.B) { + const payloadSize = 8 << 10 + b.Run("bbolt_seq", func(b *testing.B) { + benchmarkPutSeq(b, newBBoltCache(b), payloadSize) }) - b.Run("badger", func(b *testing.B) { - cache := writecachebadger.New( - writecachebadger.WithPath(b.TempDir()), - ) - benchmarkPut(b, cache) + b.Run("badger_seq", func(b *testing.B) { + benchmarkPutSeq(b, newBadgerCache(b), payloadSize) }) } -func benchmarkPut(b *testing.B, cache writecache.Cache) { - if err := cache.Open(false); err != nil { - b.Fatalf("initializing: %v", err) - } - if err := cache.Init(); err != nil { - b.Fatalf("opening: %v", err) - } - defer cache.Close() +func BenchmarkWritecachePar(b *testing.B) { + const payloadSize = 8 << 10 + b.Run("bbolt_par", func(b *testing.B) { + benchmarkPutPar(b, newBBoltCache(b), payloadSize) + }) + b.Run("badger_par", func(b *testing.B) { + benchmarkPutPar(b, newBadgerCache(b), payloadSize) + }) +} + +func benchmarkPutSeq(b *testing.B, cache writecache.Cache, size uint64) { + benchmarkPutPrepare(b, cache) ctx := context.Background() - objGen := testutil.RandObjGenerator{ObjSize: 8 << 10} + objGen := testutil.RandObjGenerator{ObjSize: size} b.ResetTimer() for n := 0; n < b.N; n++ { + obj := objGen.Next() + rawData, err := obj.Marshal() + require.NoError(b, err, "marshaling object") prm := common.PutPrm{ - Address: oidtest.Address(), - Object: objGen.Next(), + Address: testutil.AddressFromObject(b, obj), + Object: obj, + RawData: rawData, } if _, err := cache.Put(ctx, prm); err != nil { b.Fatalf("putting: %v", err) } } } + +func benchmarkPutPar(b *testing.B, cache writecache.Cache, size uint64) { + benchmarkPutPrepare(b, cache) + + ctx := context.Background() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + objGen := testutil.RandObjGenerator{ObjSize: size} + for pb.Next() { + obj := objGen.Next() + rawData, err := obj.Marshal() + require.NoError(b, err, "marshaling object") + prm := common.PutPrm{ + Address: testutil.AddressFromObject(b, obj), + Object: obj, + RawData: rawData, + } + if _, err := cache.Put(ctx, prm); err != nil { + b.Fatalf("putting: %v", err) + } + } + }) +} + +func benchmarkPutPrepare(b *testing.B, cache writecache.Cache) { + require.NoError(b, cache.Open(false), "opening") + require.NoError(b, cache.Init(), "initializing") + b.Cleanup(func() { + require.NoError(b, cache.Close(), "closing") + }) +} + +type testMetabase struct{} + +func (testMetabase) UpdateStorageID(meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error) { + return meta.UpdateStorageIDRes{}, nil +} + +func newBBoltCache(b *testing.B) writecache.Cache { + bs := teststore.New( + teststore.WithPut(func(pp common.PutPrm) (common.PutRes, error) { return common.PutRes{}, nil }), + ) + return writecachebbolt.New( + writecachebbolt.WithPath(b.TempDir()), + writecachebbolt.WithBlobstor(bs), + writecachebbolt.WithMetabase(testMetabase{}), + writecachebbolt.WithMaxCacheSize(256<<30), + writecachebbolt.WithSmallObjectSize(128<<10), + ) +} + +func newBadgerCache(b *testing.B) writecache.Cache { + bs := teststore.New( + teststore.WithPut(func(pp common.PutPrm) (common.PutRes, error) { return common.PutRes{}, nil }), + ) + return writecachebadger.New( + writecachebadger.WithPath(b.TempDir()), + writecachebadger.WithBlobstor(bs), + writecachebadger.WithMetabase(testMetabase{}), + writecachebadger.WithMaxCacheSize(256<<30), + writecachebadger.WithGCInterval(10*time.Second), + ) +} diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 084c9a3ac..8c8637358 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -5,6 +5,8 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -40,6 +42,18 @@ type Cache interface { Close() error } +// MainStorage is the interface of the underlying storage of Cache implementations. +type MainStorage interface { + Compressor() *compression.Config + Exists(context.Context, common.ExistsPrm) (common.ExistsRes, error) + Put(context.Context, common.PutPrm) (common.PutRes, error) +} + +// Metabase is the interface of the metabase used by Cache implementations. +type Metabase interface { + UpdateStorageID(meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error) +} + var ( // ErrReadOnly is returned when Put/Write is performed in a read-only mode. ErrReadOnly = logicerr.New("write-cache is in read-only mode") diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go index 4d65d5855..90e584e02 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" @@ -19,7 +18,7 @@ import ( ) func TestFlush(t *testing.T) { - createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs *blobstor.BlobStor, opts ...Option) writecache.Cache { + createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { return New( append([]Option{ WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), diff --git a/pkg/local_object_storage/writecache/writecachebadger/options.go b/pkg/local_object_storage/writecache/writecachebadger/options.go index 1737885ba..63bfb196c 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/options.go +++ b/pkg/local_object_storage/writecache/writecachebadger/options.go @@ -1,43 +1,24 @@ package writecachebadger import ( - "context" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" ) // Option represents write-cache configuration option. type Option func(*options) -// meta is an interface for a metabase. -type metabase interface { - Exists(context.Context, meta.ExistsPrm) (meta.ExistsRes, error) - StorageID(context.Context, meta.StorageIDPrm) (meta.StorageIDRes, error) - UpdateStorageID(meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error) -} - -// blob is an interface for the blobstor. -type blob interface { - Put(context.Context, common.PutPrm) (common.PutRes, error) - NeedsCompression(obj *objectSDK.Object) bool - Exists(ctx context.Context, res common.ExistsPrm) (common.ExistsRes, error) -} - type options struct { log *logger.Logger // path is a path to a directory for write-cache. path string // blobstor is the main persistent storage. - blobstor blob + blobstor writecache.MainStorage // metabase is the metabase instance. - metabase metabase + metabase writecache.Metabase // maxObjectSize is the maximum size of the object stored in the write-cache. maxObjectSize uint64 // workersCount is the number of workers flushing objects in parallel. @@ -70,14 +51,14 @@ func WithPath(path string) Option { } // WithBlobstor sets main object storage. -func WithBlobstor(bs *blobstor.BlobStor) Option { +func WithBlobstor(bs writecache.MainStorage) Option { return func(o *options) { o.blobstor = bs } } // WithMetabase sets metabase. -func WithMetabase(db *meta.DB) Option { +func WithMetabase(db writecache.Metabase) Option { return func(o *options) { o.metabase = db } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go index 465410bac..acd180130 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go @@ -8,7 +8,6 @@ import ( "testing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -22,7 +21,7 @@ import ( ) func TestFlush(t *testing.T) { - createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs *blobstor.BlobStor, opts ...Option) writecache.Cache { + createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { return New( append([]Option{ WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), diff --git a/pkg/local_object_storage/writecache/writecachebbolt/options.go b/pkg/local_object_storage/writecache/writecachebbolt/options.go index 0a21421ca..d8eedfc79 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/options.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/options.go @@ -1,45 +1,26 @@ package writecachebbolt import ( - "context" "io/fs" "os" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" ) // Option represents write-cache configuration option. type Option func(*options) -// meta is an interface for a metabase. -type metabase interface { - Exists(context.Context, meta.ExistsPrm) (meta.ExistsRes, error) - StorageID(context.Context, meta.StorageIDPrm) (meta.StorageIDRes, error) - UpdateStorageID(meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error) -} - -// blob is an interface for the blobstor. -type blob interface { - Put(context.Context, common.PutPrm) (common.PutRes, error) - NeedsCompression(obj *objectSDK.Object) bool - Exists(ctx context.Context, res common.ExistsPrm) (common.ExistsRes, error) -} - type options struct { log *logger.Logger // path is a path to a directory for write-cache. path string // blobstor is the main persistent storage. - blobstor blob + blobstor writecache.MainStorage // metabase is the metabase instance. - metabase metabase + metabase writecache.Metabase // maxObjectSize is the maximum size of the object stored in the write-cache. maxObjectSize uint64 // smallObjectSize is the maximum size of the object stored in the database. @@ -80,14 +61,14 @@ func WithPath(path string) Option { } // WithBlobstor sets main object storage. -func WithBlobstor(bs *blobstor.BlobStor) Option { +func WithBlobstor(bs writecache.MainStorage) Option { return func(o *options) { o.blobstor = bs } } // WithMetabase sets metabase. -func WithMetabase(db *meta.DB) Option { +func WithMetabase(db writecache.Metabase) Option { return func(o *options) { o.metabase = db } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/put.go b/pkg/local_object_storage/writecache/writecachebbolt/put.go index 505d091a5..48b130d26 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/put.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/put.go @@ -112,7 +112,7 @@ func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) erro return err } - if c.blobstor.NeedsCompression(prm.Object) { + if compressor := c.blobstor.Compressor(); compressor != nil && compressor.NeedsCompression(prm.Object) { c.mtx.Lock() c.compressFlags[addr] = struct{}{} c.mtx.Unlock() diff --git a/pkg/local_object_storage/writecache/writecachetest/flush.go b/pkg/local_object_storage/writecache/writecachetest/flush.go index e36778e03..05b4b23ac 100644 --- a/pkg/local_object_storage/writecache/writecachetest/flush.go +++ b/pkg/local_object_storage/writecache/writecachetest/flush.go @@ -28,7 +28,7 @@ type CreateCacheFunc[Option any] func( t *testing.T, smallSize uint64, meta *meta.DB, - bs *blobstor.BlobStor, + bs writecache.MainStorage, opts ...Option, ) writecache.Cache From c3c0574e3c5578e646fffc794bbc8104a85cc968 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 10 Aug 2023 10:56:26 +0300 Subject: [PATCH 0480/1943] [#589] cli: Add `control shards evacuate` command It was accidentally removed in f7c0b50d70f. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/shards.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index 742109673..6208c560b 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -13,6 +13,7 @@ var shardsCmd = &cobra.Command{ func initControlShardsCmd() { shardsCmd.AddCommand(listShardsCmd) shardsCmd.AddCommand(setShardModeCmd) + shardsCmd.AddCommand(evacuateShardCmd) shardsCmd.AddCommand(evacuationShardCmd) shardsCmd.AddCommand(flushCacheCmd) shardsCmd.AddCommand(doctorCmd) From 6186329aec351fd04aebec625710f48f6d4fb303 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 10 Aug 2023 11:09:36 +0300 Subject: [PATCH 0481/1943] [#590] cli: fix SDK PrmContainerGet usage for GetContainerPrm Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/client.go | 19 ++++++++++++------- cmd/frostfs-cli/modules/container/create.go | 9 ++++++--- cmd/frostfs-cli/modules/container/delete.go | 19 +++++++++++++------ cmd/frostfs-cli/modules/container/get.go | 10 +++++++--- cmd/frostfs-cli/modules/container/list.go | 7 ++++--- cmd/frostfs-cli/modules/object/nodes.go | 9 ++++++--- 6 files changed, 48 insertions(+), 25 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index e250f8692..0d4d47cb3 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -103,13 +103,15 @@ func PutContainer(ctx context.Context, prm PutContainerPrm) (res PutContainerRes // GetContainerPrm groups parameters of GetContainer operation. type GetContainerPrm struct { - commonPrm - cliPrm client.PrmContainerGet + Client *client.Client + ClientParams client.PrmContainerGet } // SetContainer sets identifier of the container to be read. +// +// Deprecated: Use GetContainerPrm.ClientParams.ContainerID instead. func (x *GetContainerPrm) SetContainer(id cid.ID) { - *x.cliPrm.ContainerID = id + x.ClientParams.ContainerID = &id } // GetContainerRes groups the resulting values of GetContainer operation. @@ -126,7 +128,7 @@ func (x GetContainerRes) Container() containerSDK.Container { // // Returns any error which prevented the operation from completing correctly in error return. func GetContainer(ctx context.Context, prm GetContainerPrm) (res GetContainerRes, err error) { - res.cliRes, err = prm.cli.ContainerGet(ctx, prm.cliPrm) + res.cliRes, err = prm.Client.ContainerGet(ctx, prm.ClientParams) return } @@ -134,9 +136,12 @@ func GetContainer(ctx context.Context, prm GetContainerPrm) (res GetContainerRes // IsACLExtendable checks if ACL of the container referenced by the given identifier // can be extended. Client connection MUST BE correctly established in advance. func IsACLExtendable(ctx context.Context, c *client.Client, cnr cid.ID) (bool, error) { - var prm GetContainerPrm - prm.SetClient(c) - prm.SetContainer(cnr) + prm := GetContainerPrm{ + Client: c, + ClientParams: client.PrmContainerGet{ + ContainerID: &cnr, + }, + } res, err := GetContainer(ctx, prm) if err != nil { diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 29c2422ed..cf6c77956 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -118,9 +118,12 @@ It will be stored in sidechain when inner ring will accepts it.`, if containerAwait { cmd.Println("awaiting...") - var getPrm internalclient.GetContainerPrm - getPrm.SetClient(cli) - getPrm.SetContainer(id) + getPrm := internalclient.GetContainerPrm{ + Client: cli, + ClientParams: client.PrmContainerGet{ + ContainerID: &id, + }, + } for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) diff --git a/cmd/frostfs-cli/modules/container/delete.go b/cmd/frostfs-cli/modules/container/delete.go index f5b69edfd..893bf842a 100644 --- a/cmd/frostfs-cli/modules/container/delete.go +++ b/cmd/frostfs-cli/modules/container/delete.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/spf13/cobra" @@ -30,9 +31,12 @@ Only owner of the container has a permission to remove container.`, if force, _ := cmd.Flags().GetBool(commonflags.ForceFlag); !force { common.PrintVerbose(cmd, "Reading the container to check ownership...") - var getPrm internalclient.GetContainerPrm - getPrm.SetClient(cli) - getPrm.SetContainer(id) + getPrm := internalclient.GetContainerPrm{ + Client: cli, + ClientParams: client.PrmContainerGet{ + ContainerID: &id, + }, + } resGet, err := internalclient.GetContainer(cmd.Context(), getPrm) commonCmd.ExitOnErr(cmd, "can't get the container: %w", err) @@ -99,9 +103,12 @@ Only owner of the container has a permission to remove container.`, if containerAwait { cmd.Println("awaiting...") - var getPrm internalclient.GetContainerPrm - getPrm.SetClient(cli) - getPrm.SetContainer(id) + getPrm := internalclient.GetContainerPrm{ + Client: cli, + ClientParams: client.PrmContainerGet{ + ContainerID: &id, + }, + } for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) diff --git a/cmd/frostfs-cli/modules/container/get.go b/cmd/frostfs-cli/modules/container/get.go index 90bcc190a..f0590c7a1 100644 --- a/cmd/frostfs-cli/modules/container/get.go +++ b/cmd/frostfs-cli/modules/container/get.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -147,9 +148,12 @@ func getContainer(cmd *cobra.Command) (container.Container, *ecdsa.PrivateKey) { pk = key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - var prm internalclient.GetContainerPrm - prm.SetClient(cli) - prm.SetContainer(id) + prm := internalclient.GetContainerPrm{ + Client: cli, + ClientParams: client.PrmContainerGet{ + ContainerID: &id, + }, + } res, err := internalclient.GetContainer(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 189cc05c7..0e8ea7ad4 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -52,8 +52,9 @@ var listContainersCmd = &cobra.Command{ res, err := internalclient.ListContainers(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - var prmGet internalclient.GetContainerPrm - prmGet.SetClient(cli) + prmGet := internalclient.GetContainerPrm{ + Client: cli, + } containerIDs := res.IDList() for _, cnrID := range containerIDs { @@ -62,7 +63,7 @@ var listContainersCmd = &cobra.Command{ continue } - prmGet.SetContainer(cnrID) + prmGet.ClientParams.ContainerID = &cnrID res, err := internalclient.GetContainer(cmd.Context(), prmGet) if err != nil { cmd.Printf(" failed to read attributes: %v\n", err) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index d2d20dc08..5499e5eec 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -155,9 +155,12 @@ func getPlacementPolicyAndNetmap(cmd *cobra.Command, cnrID cid.ID, cli *client.C } func getPlacementPolicy(ctx context.Context, cnrID cid.ID, cli *client.Client) (netmapSDK.PlacementPolicy, error) { - var prm internalclient.GetContainerPrm - prm.SetClient(cli) - prm.SetContainer(cnrID) + prm := internalclient.GetContainerPrm{ + Client: cli, + ClientParams: client.PrmContainerGet{ + ContainerID: &cnrID, + }, + } res, err := internalclient.GetContainer(ctx, prm) if err != nil { From 34b5d904413f893ea6307ece569126ff0e696afa Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 10 Aug 2023 11:43:17 +0300 Subject: [PATCH 0482/1943] [#590] cli: fix SDK PrmContainerPut usage for PutContainerPrm Signed-off-by: Airat Arifullin Date: Tue, 8 Aug 2023 18:03:32 +0300 Subject: [PATCH 0483/1943] [#578] gendoc: Replace NSPCC with FrostFS Signed-off-by: Evgenii Stratonikov --- pkg/util/gendoc/gendoc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/gendoc/gendoc.go b/pkg/util/gendoc/gendoc.go index df0d32b21..e7228d0e0 100644 --- a/pkg/util/gendoc/gendoc.go +++ b/pkg/util/gendoc/gendoc.go @@ -67,7 +67,7 @@ In this case there is a number of helper functions which can be used: case gendocMan: hdr := &doc.GenManHeader{ Section: "1", - Source: "NSPCC & Morphbits", + Source: "FrostFS", } return doc.GenManTree(rootCmd, hdr, args[0]) default: From b02a1a34c130570c71b659a31018d48e12512280 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 8 Aug 2023 18:08:00 +0300 Subject: [PATCH 0484/1943] [#578] gendoc: Allow to customize man pages Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/root.go | 2 +- cmd/frostfs-cli/modules/root.go | 2 +- cmd/frostfs-lens/root.go | 2 +- pkg/util/gendoc/gendoc.go | 25 ++++++++++++++++++------ 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/root.go b/cmd/frostfs-adm/internal/modules/root.go index fd1517f90..0fa0f7f69 100644 --- a/cmd/frostfs-adm/internal/modules/root.go +++ b/cmd/frostfs-adm/internal/modules/root.go @@ -45,7 +45,7 @@ func init() { rootCmd.AddCommand(storagecfg.RootCmd) rootCmd.AddCommand(autocomplete.Command("frostfs-adm")) - rootCmd.AddCommand(gendoc.Command(rootCmd)) + rootCmd.AddCommand(gendoc.Command(rootCmd, gendoc.Options{})) } func Execute() error { diff --git a/cmd/frostfs-cli/modules/root.go b/cmd/frostfs-cli/modules/root.go index 93513479a..808bd6d07 100644 --- a/cmd/frostfs-cli/modules/root.go +++ b/cmd/frostfs-cli/modules/root.go @@ -85,7 +85,7 @@ func init() { rootCmd.AddCommand(objectCli.Cmd) rootCmd.AddCommand(containerCli.Cmd) rootCmd.AddCommand(tree.Cmd) - rootCmd.AddCommand(gendoc.Command(rootCmd)) + rootCmd.AddCommand(gendoc.Command(rootCmd, gendoc.Options{})) } func entryPoint(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-lens/root.go b/cmd/frostfs-lens/root.go index ab937addc..96ade802c 100644 --- a/cmd/frostfs-lens/root.go +++ b/cmd/frostfs-lens/root.go @@ -38,7 +38,7 @@ func init() { blobovnicza.Root, meta.Root, writecache.Root, - gendoc.Command(command), + gendoc.Command(command, gendoc.Options{}), ) } diff --git a/pkg/util/gendoc/gendoc.go b/pkg/util/gendoc/gendoc.go index e7228d0e0..14361d8f6 100644 --- a/pkg/util/gendoc/gendoc.go +++ b/pkg/util/gendoc/gendoc.go @@ -23,8 +23,25 @@ const ( extensionFlag = "extension" ) +// Options for doc generation. +type Options struct { + // Parameters for man generation. By default use (1) section and `FrostFS` source. + ManHeader *doc.GenManHeader +} + +func (o *Options) fillDefaults() { + if o.ManHeader == nil { + o.ManHeader = &doc.GenManHeader{ + Section: "1", + Source: "FrostFS", + } + } +} + // Command returns command which generates user documentation for the argument. -func Command(rootCmd *cobra.Command) *cobra.Command { +func Command(rootCmd *cobra.Command, opts Options) *cobra.Command { + opts.fillDefaults() + gendocCmd := &cobra.Command{ Use: "gendoc ", Short: "Generate documentation for this command", @@ -65,11 +82,7 @@ In this case there is a number of helper functions which can be used: case gendocMarkdown: return doc.GenMarkdownTree(rootCmd, args[0]) case gendocMan: - hdr := &doc.GenManHeader{ - Section: "1", - Source: "FrostFS", - } - return doc.GenManTree(rootCmd, hdr, args[0]) + return doc.GenManTree(rootCmd, opts.ManHeader, args[0]) default: return errors.New("type must be 'md' or 'man'") } From dca31e88884905881b8254e15f2298a2476ecd7f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 8 Aug 2023 18:12:42 +0300 Subject: [PATCH 0485/1943] [#578] gendoc: Add date to man pages ``` HISTORY 8-Aug-2023 Auto generated by spf13/cobra ``` Signed-off-by: Evgenii Stratonikov --- pkg/util/gendoc/gendoc.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/util/gendoc/gendoc.go b/pkg/util/gendoc/gendoc.go index 14361d8f6..16ebe9f98 100644 --- a/pkg/util/gendoc/gendoc.go +++ b/pkg/util/gendoc/gendoc.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" "text/template" + "time" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" @@ -31,9 +32,11 @@ type Options struct { func (o *Options) fillDefaults() { if o.ManHeader == nil { + now := time.Now() o.ManHeader = &doc.GenManHeader{ Section: "1", Source: "FrostFS", + Date: &now, } } } From 6bcba277578996d36d82b6ae5597d9e452f63e7e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 8 Aug 2023 18:14:21 +0300 Subject: [PATCH 0486/1943] [#578] gendoc: Remove flag shorthands Make it harder to encounter conflicts in already existing commands. Because the command is executed once, I don't think the usability is worse. Signed-off-by: Evgenii Stratonikov --- debian/rules | 4 ++-- pkg/util/gendoc/gendoc.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/debian/rules b/debian/rules index b85356b99..0dd8ee399 100755 --- a/debian/rules +++ b/debian/rules @@ -12,8 +12,8 @@ override_dh_auto_install: echo $(DEB_BUILD_OPTIONS) dh_auto_install - bin/frostfs-adm gendoc -t man man/ - bin/frostfs-cli gendoc -t man man/ + bin/frostfs-adm gendoc --type man man/ + bin/frostfs-cli gendoc --type man man/ bin/frostfs-adm completion bash > debian/frostfs-adm.bash-completion bin/frostfs-cli completion bash > debian/frostfs-cli.bash-completion diff --git a/pkg/util/gendoc/gendoc.go b/pkg/util/gendoc/gendoc.go index 16ebe9f98..5da211e2b 100644 --- a/pkg/util/gendoc/gendoc.go +++ b/pkg/util/gendoc/gendoc.go @@ -93,9 +93,9 @@ In this case there is a number of helper functions which can be used: } ff := gendocCmd.Flags() - ff.StringP(gendocTypeFlag, "t", gendocMarkdown, "Type for the documentation ('md' or 'man')") + ff.String(gendocTypeFlag, gendocMarkdown, "Type for the documentation ('md' or 'man')") ff.Int(depthFlag, 1, "If template is specified, unify all commands starting from depth in a single file. Default: 1.") - ff.StringP(extensionFlag, "e", "", "If the template is specified, string to append to the output file names") + ff.String(extensionFlag, "", "If the template is specified, string to append to the output file names") return gendocCmd } From c3e23a14489b97aab71392e436a435e99f6d7361 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 8 Aug 2023 18:19:11 +0300 Subject: [PATCH 0487/1943] [#578] gendoc: Allow to override flags The command is used in multiple places across the whole FrostFS ecosystem. While we want to have uniform interfaces everywhere, sometimes we can't: already defined global flags can be harder to change because of our obligations to the users. Cobra framework doesn't allow conflicting flags (we can have global ones), so allow to override them. Signed-off-by: Evgenii Stratonikov --- pkg/util/gendoc/gendoc.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/pkg/util/gendoc/gendoc.go b/pkg/util/gendoc/gendoc.go index 5da211e2b..8897bf1e5 100644 --- a/pkg/util/gendoc/gendoc.go +++ b/pkg/util/gendoc/gendoc.go @@ -28,6 +28,18 @@ const ( type Options struct { // Parameters for man generation. By default use (1) section and `FrostFS` source. ManHeader *doc.GenManHeader + // TypeFlag is the flag to use for type, without leading `--`. + // Do not use unless really necessary. + // Default: `type`. + TypeFlag string + // DepthFlag is the flag to use for depth, without leading `--`. + // Do not use unless really necessary. + // Default: `depth`. + DepthFlag string + // ExtensionFlag is the flag to use for extension, without leading `--`. + // Do not use unless really necessary. + // Default: `extension`. + ExtensionFlag string } func (o *Options) fillDefaults() { @@ -39,6 +51,15 @@ func (o *Options) fillDefaults() { Date: &now, } } + if o.TypeFlag == "" { + o.TypeFlag = gendocTypeFlag + } + if o.DepthFlag == "" { + o.DepthFlag = depthFlag + } + if o.ExtensionFlag == "" { + o.ExtensionFlag = extensionFlag + } } // Command returns command which generates user documentation for the argument. @@ -93,9 +114,9 @@ In this case there is a number of helper functions which can be used: } ff := gendocCmd.Flags() - ff.String(gendocTypeFlag, gendocMarkdown, "Type for the documentation ('md' or 'man')") - ff.Int(depthFlag, 1, "If template is specified, unify all commands starting from depth in a single file. Default: 1.") - ff.String(extensionFlag, "", "If the template is specified, string to append to the output file names") + ff.String(opts.TypeFlag, gendocMarkdown, "Type for the documentation ('md' or 'man')") + ff.Int(opts.DepthFlag, 1, "If template is specified, unify all commands starting from depth in a single file. Default: 1.") + ff.String(opts.ExtensionFlag, "", "If the template is specified, string to append to the output file names") return gendocCmd } From 4ad0ebb32fa6cd99303944f637edd5994400e68a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 9 Aug 2023 09:23:03 +0300 Subject: [PATCH 0488/1943] [#565] Add metrics for current GRPC endpoint status Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config.go | 2 ++ cmd/frostfs-node/grpc.go | 5 +++++ pkg/metrics/consts.go | 2 ++ pkg/metrics/grpc.go | 35 +++++++++++++++++++++++++++++++++++ pkg/metrics/node.go | 6 ++++++ 5 files changed, 50 insertions(+) create mode 100644 pkg/metrics/grpc.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index f5c106f77..935aefe7f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -446,6 +446,8 @@ type cfgGRPC struct { servers []*grpc.Server + endpoints []string + maxChunkSize uint64 maxAddrAmount uint64 diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index 1dd0f0729..10d387b13 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -69,11 +69,14 @@ func initGRPC(c *cfg) { lis, err := net.Listen("tcp", sc.Endpoint()) if err != nil { + c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(sc.Endpoint()) c.log.Error(logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) return } + c.metricsCollector.GrpcServerMetrics().MarkHealthy(sc.Endpoint()) c.cfgGRPC.listeners = append(c.cfgGRPC.listeners, lis) + c.cfgGRPC.endpoints = append(c.cfgGRPC.endpoints, sc.Endpoint()) srv := grpc.NewServer(serverOpts...) @@ -96,6 +99,7 @@ func serveGRPC(c *cfg) { srv := c.cfgGRPC.servers[i] lis := c.cfgGRPC.listeners[i] + endpoint := c.cfgGRPC.endpoints[i] go func() { defer func() { @@ -111,6 +115,7 @@ func serveGRPC(c *cfg) { ) if err := srv.Serve(lis); err != nil { + c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(endpoint) fmt.Println("gRPC server error", err) } }() diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index ae3ac7839..691aaa375 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -19,6 +19,7 @@ const ( stateSubsystem = "state" treeServiceSubsystem = "treeservice" writeCacheSubsystem = "writecache" + grpcServerSubsystem = "grpc_server" successLabel = "success" shardIDLabel = "shard_id" @@ -35,6 +36,7 @@ const ( containerIDLabelKey = "cid" storageLabel = "storage" operationLabel = "operation" + endpointLabel = "endpoint" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" diff --git a/pkg/metrics/grpc.go b/pkg/metrics/grpc.go new file mode 100644 index 000000000..a83f53998 --- /dev/null +++ b/pkg/metrics/grpc.go @@ -0,0 +1,35 @@ +package metrics + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type GrpcServerMetrics interface { + MarkHealthy(endpoint string) + MarkUnhealthy(endpoint string) +} + +type grpcServerMetrics struct { + endpointHealth *prometheus.GaugeVec +} + +func newGrpcServerMetrics() *grpcServerMetrics { + return &grpcServerMetrics{ + endpointHealth: metrics.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: grpcServerSubsystem, + Name: "health", + Help: "GRPC Server Endpoint health", + }, []string{endpointLabel}), + } +} + +func (m *grpcServerMetrics) MarkHealthy(endpoint string) { + m.endpointHealth.With(prometheus.Labels{endpointLabel: endpoint}).Set(float64(1)) +} + +func (m *grpcServerMetrics) MarkUnhealthy(endpoint string) { + m.endpointHealth.With(prometheus.Labels{endpointLabel: endpoint}).Set(float64(0)) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 45d50b5b0..ce650c220 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -17,6 +17,7 @@ type NodeMetrics struct { blobobvnizca *blobovnizca metabase *metabaseMetrics pilorama *piloramaMetrics + grpc *grpcServerMetrics } func NewNodeMetrics() *NodeMetrics { @@ -37,6 +38,7 @@ func NewNodeMetrics() *NodeMetrics { blobobvnizca: newBlobovnizca(), metabase: newMetabaseMetrics(), pilorama: newPiloramaMetrics(), + grpc: newGrpcServerMetrics(), } } @@ -84,3 +86,7 @@ func (m *NodeMetrics) MetabaseMetrics() MetabaseMetrics { func (m *NodeMetrics) PiloramaMetrics() PiloramaMetrics { return m.pilorama } + +func (m *NodeMetrics) GrpcServerMetrics() GrpcServerMetrics { + return m.grpc +} From 26a78aa366ef2874d97997cd1a13a60dbd748d13 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 3 Aug 2023 14:18:56 +0300 Subject: [PATCH 0489/1943] [#248] adm: Remove obsolete network parameters Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/README.md | 2 -- cmd/frostfs-adm/docs/deploy.md | 2 -- cmd/frostfs-adm/internal/modules/config/config.go | 6 ------ cmd/frostfs-adm/internal/modules/config/config_test.go | 2 -- cmd/frostfs-adm/internal/modules/morph/config.go | 6 ++---- cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go | 2 -- cmd/frostfs-adm/internal/modules/morph/netmap_util.go | 2 -- cmd/frostfs-adm/internal/modules/morph/root.go | 6 ------ 8 files changed, 2 insertions(+), 26 deletions(-) diff --git a/cmd/frostfs-adm/README.md b/cmd/frostfs-adm/README.md index dac264239..f3dbe6950 100644 --- a/cmd/frostfs-adm/README.md +++ b/cmd/frostfs-adm/README.md @@ -36,9 +36,7 @@ alphabet-wallets: /path # path to consensus node / alphabet wallets s network: max_object_size: 67108864 # max size of a single FrostFS object, bytes epoch_duration: 240 # duration of a FrostFS epoch in blocks, consider block generation frequency in the sidechain - basic_income_rate: 0 # basic income rate, for private consider 0 fee: - audit: 0 # network audit fee, for private installation consider 0 candidate: 0 # inner ring candidate registration fee, for private installation consider 0 container: 0 # container creation fee, for private installation consider 0 container_alias: 0 # container nice-name registration fee, for private installation consider 0 diff --git a/cmd/frostfs-adm/docs/deploy.md b/cmd/frostfs-adm/docs/deploy.md index 974c2a93c..b40dc75b0 100644 --- a/cmd/frostfs-adm/docs/deploy.md +++ b/cmd/frostfs-adm/docs/deploy.md @@ -34,9 +34,7 @@ alphabet-wallets: /home/user/deploy/alphabet-wallets network: max_object_size: 67108864 epoch_duration: 240 - basic_income_rate: 0 fee: - audit: 0 candidate: 0 container: 0 withdraw: 0 diff --git a/cmd/frostfs-adm/internal/modules/config/config.go b/cmd/frostfs-adm/internal/modules/config/config.go index ebefe1d2c..9dfab9f85 100644 --- a/cmd/frostfs-adm/internal/modules/config/config.go +++ b/cmd/frostfs-adm/internal/modules/config/config.go @@ -18,8 +18,6 @@ type configTemplate struct { AlphabetDir string MaxObjectSize int EpochDuration int - BasicIncomeRate int - AuditFee int CandidateFee int ContainerFee int ContainerAliasFee int @@ -33,10 +31,8 @@ alphabet-wallets: {{ .AlphabetDir}} network: max_object_size: {{ .MaxObjectSize}} epoch_duration: {{ .EpochDuration}} - basic_income_rate: {{ .BasicIncomeRate}} homomorphic_hash_disabled: {{ .HomomorphicHashDisabled}} fee: - audit: {{ .AuditFee}} candidate: {{ .CandidateFee}} container: {{ .ContainerFee}} container_alias: {{ .ContainerAliasFee }} @@ -111,9 +107,7 @@ func generateConfigExample(appDir string, credSize int) (string, error) { Endpoint: "https://neo.rpc.node:30333", MaxObjectSize: 67108864, // 64 MiB EpochDuration: 240, // 1 hour with 15s per block - BasicIncomeRate: 1_0000_0000, // 0.0001 GAS per GiB (Fixed12) HomomorphicHashDisabled: false, // object homomorphic hash is enabled - AuditFee: 1_0000, // 0.00000001 GAS per audit (Fixed12) CandidateFee: 100_0000_0000, // 100.0 GAS (Fixed8) ContainerFee: 1000, // 0.000000001 * 7 GAS per container (Fixed12) ContainerAliasFee: 500, // ContainerFee / 2 diff --git a/cmd/frostfs-adm/internal/modules/config/config_test.go b/cmd/frostfs-adm/internal/modules/config/config_test.go index 292fe76da..3fa800a06 100644 --- a/cmd/frostfs-adm/internal/modules/config/config_test.go +++ b/cmd/frostfs-adm/internal/modules/config/config_test.go @@ -28,8 +28,6 @@ func TestGenerateConfigExample(t *testing.T) { require.Equal(t, filepath.Join(appDir, "alphabet-wallets"), v.GetString("alphabet-wallets")) require.Equal(t, 67108864, v.GetInt("network.max_object_size")) require.Equal(t, 240, v.GetInt("network.epoch_duration")) - require.Equal(t, 100000000, v.GetInt("network.basic_income_rate")) - require.Equal(t, 10000, v.GetInt("network.fee.audit")) require.Equal(t, 10000000000, v.GetInt("network.fee.candidate")) require.Equal(t, 1000, v.GetInt("network.fee.container")) require.Equal(t, 100000000, v.GetInt("network.fee.withdraw")) diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index 09d071b53..a86829aaa 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -54,8 +54,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { } for k, v := range m { switch k { - case netmap.AuditFeeConfig, netmap.BasicIncomeRateConfig, - netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, + case netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, netmap.EpochDurationConfig, netmap.IrCandidateFeeConfig, netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig: nbuf := make([]byte, 8) @@ -134,8 +133,7 @@ func parseConfigPair(kvStr string, force bool) (key string, val any, err error) valRaw := v switch key { - case netmap.AuditFeeConfig, netmap.BasicIncomeRateConfig, - netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, + case netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, netmap.EpochDurationConfig, netmap.IrCandidateFeeConfig, netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig: val, err = strconv.ParseInt(valRaw, 10, 64) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 8a0a88899..dc6bc646c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -69,10 +69,8 @@ var ( netmapConfigKeys = []string{ netmap.EpochDurationConfig, netmap.MaxObjectSizeConfig, - netmap.AuditFeeConfig, netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, - netmap.BasicIncomeRateConfig, netmap.IrCandidateFeeConfig, netmap.WithdrawFeeConfig, netmap.HomomorphicHashingDisabledKey, diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_util.go b/cmd/frostfs-adm/internal/modules/morph/netmap_util.go index 23cfd120c..fa7aa0af3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap_util.go @@ -12,10 +12,8 @@ func getDefaultNetmapContractConfigMap() map[string]any { m := make(map[string]any) m[netmap.EpochDurationConfig] = viper.GetInt64(epochDurationInitFlag) m[netmap.MaxObjectSizeConfig] = viper.GetInt64(maxObjectSizeInitFlag) - m[netmap.AuditFeeConfig] = viper.GetInt64(auditFeeInitFlag) m[netmap.ContainerFeeConfig] = viper.GetInt64(containerFeeInitFlag) m[netmap.ContainerAliasFeeConfig] = viper.GetInt64(containerAliasFeeInitFlag) - m[netmap.BasicIncomeRateConfig] = viper.GetInt64(incomeRateInitFlag) m[netmap.IrCandidateFeeConfig] = viper.GetInt64(candidateFeeInitFlag) m[netmap.WithdrawFeeConfig] = viper.GetInt64(withdrawFeeInitFlag) m[netmap.HomomorphicHashingDisabledKey] = viper.GetBool(homomorphicHashDisabledInitFlag) diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 6a9e5b9c1..bee1837a3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -18,10 +18,6 @@ const ( maxObjectSizeCLIFlag = "max-object-size" epochDurationInitFlag = "network.epoch_duration" epochDurationCLIFlag = "epoch-duration" - incomeRateInitFlag = "network.basic_income_rate" - incomeRateCLIFlag = "basic-income-rate" - auditFeeInitFlag = "network.fee.audit" - auditFeeCLIFlag = "audit-fee" containerFeeInitFlag = "network.fee.container" containerAliasFeeInitFlag = "network.fee.container_alias" containerFeeCLIFlag = "container-fee" @@ -69,9 +65,7 @@ var ( _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) _ = viper.BindPFlag(epochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) _ = viper.BindPFlag(maxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) - _ = viper.BindPFlag(incomeRateInitFlag, cmd.Flags().Lookup(incomeRateCLIFlag)) _ = viper.BindPFlag(homomorphicHashDisabledInitFlag, cmd.Flags().Lookup(homomorphicHashDisabledCLIFlag)) - _ = viper.BindPFlag(auditFeeInitFlag, cmd.Flags().Lookup(auditFeeCLIFlag)) _ = viper.BindPFlag(candidateFeeInitFlag, cmd.Flags().Lookup(candidateFeeCLIFlag)) _ = viper.BindPFlag(containerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) _ = viper.BindPFlag(containerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) From c54fcb297d994bc4a04a9162dc12a314771e327b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 3 Aug 2023 14:19:46 +0300 Subject: [PATCH 0490/1943] [#248] cli,node: Remove obsolete network parameters Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/netmap/netinfo.go | 2 -- cmd/frostfs-node/netmap.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/cmd/frostfs-cli/modules/netmap/netinfo.go b/cmd/frostfs-cli/modules/netmap/netinfo.go index c6ab0b6f6..3c1839114 100644 --- a/cmd/frostfs-cli/modules/netmap/netinfo.go +++ b/cmd/frostfs-cli/modules/netmap/netinfo.go @@ -38,8 +38,6 @@ var netInfoCmd = &cobra.Command{ const format = " %s: %v\n" cmd.Println("FrostFS network configuration (system)") - cmd.Printf(format, "Audit fee", netInfo.AuditFee()) - cmd.Printf(format, "Storage price", netInfo.StoragePrice()) cmd.Printf(format, "Container fee", netInfo.ContainerFee()) cmd.Printf(format, "Epoch duration", netInfo.EpochDuration()) cmd.Printf(format, "Inner Ring candidate fee", netInfo.IRCandidateFee()) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 60a59940f..b0acac032 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -477,8 +477,6 @@ func (n *netInfo) Dump(ver version.Version) (*netmapSDK.NetworkInfo, error) { ni.SetMsPerBlock(msPerBlock) ni.SetMaxObjectSize(netInfoMorph.MaxObjectSize) - ni.SetStoragePrice(netInfoMorph.StoragePrice) - ni.SetAuditFee(netInfoMorph.AuditFee) ni.SetEpochDuration(netInfoMorph.EpochDuration) ni.SetContainerFee(netInfoMorph.ContainerFee) ni.SetNamedContainerFee(netInfoMorph.ContainerAliasFee) From 55ce4dc075ef3a78eba86ff53e7cd861f18bafea Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 3 Aug 2023 14:20:09 +0300 Subject: [PATCH 0491/1943] [#248] morph: Remove obsolete network parameters Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/netmap/config.go | 32 ------------------------------- 1 file changed, 32 deletions(-) diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index c837cbf0c..09ae09b51 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -11,8 +11,6 @@ import ( const ( MaxObjectSizeConfig = "MaxObjectSize" - BasicIncomeRateConfig = "BasicIncomeRate" - AuditFeeConfig = "AuditFee" EpochDurationConfig = "EpochDuration" ContainerFeeConfig = "ContainerFee" ContainerAliasFeeConfig = "ContainerAliasFee" @@ -33,28 +31,6 @@ func (c *Client) MaxObjectSize() (uint64, error) { return objectSize, nil } -// BasicIncomeRate returns basic income rate configuration value from network -// config in netmap contract. -func (c *Client) BasicIncomeRate() (uint64, error) { - rate, err := c.readUInt64Config(BasicIncomeRateConfig) - if err != nil { - return 0, fmt.Errorf("(%T) could not get basic income rate: %w", c, err) - } - - return rate, nil -} - -// AuditFee returns audit fee configuration value from network -// config in netmap contract. -func (c *Client) AuditFee() (uint64, error) { - fee, err := c.readUInt64Config(AuditFeeConfig) - if err != nil { - return 0, fmt.Errorf("(%T) could not get audit fee: %w", c, err) - } - - return fee, nil -} - // EpochDuration returns number of sidechain blocks per one FrostFS epoch. func (c *Client) EpochDuration() (uint64, error) { epochDuration, err := c.readUInt64Config(EpochDurationConfig) @@ -201,10 +177,6 @@ type RawNetworkParameter struct { type NetworkConfiguration struct { MaxObjectSize uint64 - StoragePrice uint64 - - AuditFee uint64 - EpochDuration uint64 ContainerFee uint64 @@ -262,10 +234,6 @@ func (c *Client) ReadNetworkConfiguration() (NetworkConfiguration, error) { }) case MaxObjectSizeConfig: res.MaxObjectSize = bytesToUint64(value) - case BasicIncomeRateConfig: - res.StoragePrice = bytesToUint64(value) - case AuditFeeConfig: - res.AuditFee = bytesToUint64(value) case EpochDurationConfig: res.EpochDuration = bytesToUint64(value) case ContainerFeeConfig: From 2685b1d548a613adafb89e1fcfd165dc5ba8c4e8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 3 Aug 2023 14:23:05 +0300 Subject: [PATCH 0492/1943] [#248] adm: Do not deploy audit contract Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/docs/deploy.md | 1 - cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go | 4 ---- 2 files changed, 5 deletions(-) diff --git a/cmd/frostfs-adm/docs/deploy.md b/cmd/frostfs-adm/docs/deploy.md index b40dc75b0..3b9f4c98c 100644 --- a/cmd/frostfs-adm/docs/deploy.md +++ b/cmd/frostfs-adm/docs/deploy.md @@ -140,7 +140,6 @@ Waiting for transactions to persist... Stage 7: set addresses in NNS. Waiting for transactions to persist... NNS: Set alphabet0.frostfs -> f692dfb4d43a15b464eb51a7041160fb29c44b6a -NNS: Set audit.frostfs -> 7df847b993affb3852074345a7c2bd622171ee0d NNS: Set balance.frostfs -> 103519b3067a66307080a66570c0491ee8f68879 NNS: Set container.frostfs -> cae60bdd689d185901e495352d0247752ce50846 NNS: Set frostfsid.frostfs -> c421fb60a3895865a8f24d197d6a80ef686041d2 diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index dc6bc646c..a7cd537b7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -41,7 +41,6 @@ const ( frostfsContract = "frostfs" // not deployed in side-chain. processingContract = "processing" // not deployed in side-chain. alphabetContract = "alphabet" - auditContract = "audit" balanceContract = "balance" containerContract = "container" frostfsIDContract = "frostfsid" @@ -51,7 +50,6 @@ const ( var ( contractList = []string{ - auditContract, balanceContract, containerContract, frostfsIDContract, @@ -531,8 +529,6 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an case processingContract: items = append(items, c.Contracts[frostfsContract].Hash) return items[1:] // no notary info - case auditContract: - items = append(items, c.Contracts[netmapContract].Hash) case balanceContract: items = append(items, c.Contracts[netmapContract].Hash, From 779da6ec359aeb08d8aa4ec52d461587b3247a4c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 3 Aug 2023 14:24:44 +0300 Subject: [PATCH 0493/1943] [#248] morph: Remove audit contract name Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/nns.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index 53dbe180e..758b220a2 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -21,8 +21,6 @@ import ( const ( nnsContractID = 1 // NNS contract must be deployed first in the sidechain - // NNSAuditContractName is a name of the audit contract in NNS. - NNSAuditContractName = "audit.frostfs" // NNSBalanceContractName is a name of the balance contract in NNS. NNSBalanceContractName = "balance.frostfs" // NNSContainerContractName is a name of the container contract in NNS. From e084c47bd665a853d15aff30bf10ca00b4102578 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 3 Aug 2023 14:26:58 +0300 Subject: [PATCH 0494/1943] [#248] innerring: Remove audit from tests Signed-off-by: Evgenii Stratonikov --- pkg/innerring/contracts_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/innerring/contracts_test.go b/pkg/innerring/contracts_test.go index 5f877133c..0fb0442b2 100644 --- a/pkg/innerring/contracts_test.go +++ b/pkg/innerring/contracts_test.go @@ -16,7 +16,6 @@ func TestParseContractsSuccess(t *testing.T) { contracts: frostfs: ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62 processing: 597f5894867113a41e192801709c02497f611de8 - audit: 219e37aed2180b87e7fe945dbf97d67125e8d73f balance: d2aa48d14b17b11bc4c68205027884a96706dd16 container: ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6 frostfsid: 9f5866decbc751a099e74c7c7bc89f609201755a @@ -138,13 +137,12 @@ contracts: func TestParseContractsInvalid(t *testing.T) { t.Parallel() - t.Run("invalid audit contract", func(t *testing.T) { + t.Run("invalid frostfs contract", func(t *testing.T) { t.Parallel() file := strings.NewReader(` contracts: frostfs: invalid_data processing: 597f5894867113a41e192801709c02497f611de8 - audit: 219e37aed2180b87e7fe945dbf97d67125e8d73f balance: d2aa48d14b17b11bc4c68205027884a96706dd16 container: ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6 frostfsid: 9f5866decbc751a099e74c7c7bc89f609201755a @@ -171,7 +169,6 @@ contracts: contracts: frostfs: ee3dee6d05dc79c24a5b8f6985e10d68b7cacc62 processing: 597f5894867113a41e192801709c02497f611de8 - audit: 219e37aed2180b87e7fe945dbf97d67125e8d73f balance: d2aa48d14b17b11bc4c68205027884a96706dd16 container: ed4a7a66fe3f9bfe50f214b49be8f215a3c886b6 frostfsid: 9f5866decbc751a099e74c7c7bc89f609201755a From 32a9f5158641c34e1b590142fe93ae36bd935b3e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 7 Aug 2023 11:41:52 +0300 Subject: [PATCH 0495/1943] [#562] logs: Remove superfluous comment Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index dbe4ab7d1..ad4003bed 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -409,7 +409,7 @@ const ( NetmapCantInvokeNetmapAddPeer = "can't invoke netmap.AddPeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapNonAlphabetModeIgnoreUpdatePeerNotification = "non alphabet mode, ignore update peer notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go NetmapPreventSwitchingNodeToMaintenanceState = "prevent switching node to maintenance state" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go // Debug in ../node/pkg/innerring/processors/reputation/processor.go + NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go FrostFSIRInternalError = "internal error" // Info in ../node/cmd/frostfs-ir/main.go FrostFSIRCouldNotShutdownHTTPServer = "could not shutdown HTTP server" // Debug in ../node/cmd/frostfs-ir/main.go FrostFSIRApplicationStopped = "application stopped" // Info in ../node/cmd/frostfs-ir/main.go From dd988a5912e8a6fff3fb3aeb23588e836f1b3fae Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 10 Aug 2023 15:03:16 +0300 Subject: [PATCH 0496/1943] [#592] adm: Allow to tick epoch concurrently Epoch can be ticked by IR in the background. Because the only usecase for this is to apply some changes, the right behaviour is ignoring an error, not retrying. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/epoch.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/epoch.go index 1f622fa89..a96efa43f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/epoch.go @@ -3,6 +3,7 @@ package morph import ( "errors" "fmt" + "strings" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" @@ -38,7 +39,14 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { return err } - return wCtx.awaitTx() + if err := wCtx.awaitTx(); err != nil { + if strings.Contains(err.Error(), "invalid epoch") { + cmd.Println("Epoch has already ticked.") + return nil + } + return err + } + return nil } func emitNewEpochCall(bw *io.BufBinWriter, wCtx *initializeContext, nmHash util.Uint160) error { From 20af34ecdb1f2f147c5a5102212846fa4f05ca29 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 10 Aug 2023 13:05:12 +0300 Subject: [PATCH 0497/1943] [#591] cli: fix SDK PrmContainerDelete usage for DeleteContainerPrm Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/client.go | 6 +++--- cmd/frostfs-cli/modules/container/delete.go | 12 ++++++------ go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 44e2d5fcd..f09468973 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -153,8 +153,8 @@ func IsACLExtendable(ctx context.Context, c *client.Client, cnr cid.ID) (bool, e // DeleteContainerPrm groups parameters of DeleteContainerPrm operation. type DeleteContainerPrm struct { - commonPrm - client.PrmContainerDelete + Client *client.Client + ClientParams client.PrmContainerDelete } // DeleteContainerRes groups the resulting values of DeleteContainer operation. @@ -169,7 +169,7 @@ type DeleteContainerRes struct{} // // Returns any error which prevented the operation from completing correctly in error return. func DeleteContainer(ctx context.Context, prm DeleteContainerPrm) (res DeleteContainerRes, err error) { - _, err = prm.cli.ContainerDelete(ctx, prm.PrmContainerDelete) + _, err = prm.Client.ContainerDelete(ctx, prm.ClientParams) return } diff --git a/cmd/frostfs-cli/modules/container/delete.go b/cmd/frostfs-cli/modules/container/delete.go index 893bf842a..20de84f74 100644 --- a/cmd/frostfs-cli/modules/container/delete.go +++ b/cmd/frostfs-cli/modules/container/delete.go @@ -87,12 +87,12 @@ Only owner of the container has a permission to remove container.`, } } - var delPrm internalclient.DeleteContainerPrm - delPrm.SetClient(cli) - delPrm.SetContainer(id) - - if tok != nil { - delPrm.WithinSession(*tok) + delPrm := internalclient.DeleteContainerPrm{ + Client: cli, + ClientParams: client.PrmContainerDelete{ + ContainerID: &id, + Session: tok, + }, } _, err := internalclient.DeleteContainer(cmd.Context(), delPrm) diff --git a/go.mod b/go.mod index e9f6ac9d9..4173ad0df 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230804083501-3dc8129ed794 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230809065235-d48788c7a946 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 0a442b1ed..7dda28828 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230804083501-3dc8129ed794 h1:gCIhE79b9fAM0Pixgv/Ot7iVmNJbAS8ZwBHLBnWS7Zc= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230804083501-3dc8129ed794/go.mod h1:LI2GOj0pEx0jYTjB3QHja2PNhQFYL2pCm71RAFwDv0M= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230809065235-d48788c7a946 h1:/WVghZ/Za1eAwERCL4JgW5MNvbe7d2DU8qhFZBFgNGE= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230809065235-d48788c7a946/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From b44a8dd46c0c1d2ab7c352bc49b5cb7711cf23f6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 11 Aug 2023 09:54:48 +0300 Subject: [PATCH 0498/1943] [#597] *: Fix linter warnings Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/container/list.go | 1 + .../blobstor/internal/blobstortest/common.go | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 0e8ea7ad4..c8b891d32 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -63,6 +63,7 @@ var listContainersCmd = &cobra.Command{ continue } + cnrID := cnrID prmGet.ClientParams.ContainerID = &cnrID res, err := internalclient.GetContainer(cmd.Context(), prmGet) if err != nil { diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/common.go b/pkg/local_object_storage/blobstor/internal/blobstortest/common.go index 1f234198e..c08e39bf1 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/common.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/common.go @@ -2,9 +2,9 @@ package blobstortest import ( "context" - "crypto/rand" mrand "math/rand" "testing" + "time" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -87,7 +87,8 @@ func NewObject(sz uint64) *objectSDK.Object { raw.SetContainerID(cidtest.ID()) payload := make([]byte, sz) - rand.Read(payload) + r := mrand.New(mrand.NewSource(time.Now().UnixNano())) + r.Read(payload) raw.SetPayload(payload) // fit the binary size to the required From 4d2af137e9c570469d417516b8a38783612ffa85 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Jul 2023 13:48:27 +0300 Subject: [PATCH 0499/1943] [#500] .forgejo: Add DCO action Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/dco.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .forgejo/workflows/dco.yml diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml new file mode 100644 index 000000000..6746408aa --- /dev/null +++ b/.forgejo/workflows/dco.yml @@ -0,0 +1,21 @@ +name: DCO action +on: [pull_request] + +jobs: + dco: + name: DCO + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.21' + + - name: Run commit format checker + uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2 + with: + from: 'origin/${{ github.event.pull_request.base.ref }}' From abdb0910ccd1bb0078cdad423331ca28a0a7d390 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 11 Aug 2023 12:13:09 +0300 Subject: [PATCH 0500/1943] [#600] adm/tests: Add missing WaitGroup.Add() ``` panic: sync: negative WaitGroup counter ``` Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/generate_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate_test.go index 457813df0..7f0a2708c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate_test.go @@ -75,6 +75,7 @@ func TestGenerateAlphabet(t *testing.T) { var wg sync.WaitGroup for i := uint64(0); i < size; i++ { i := i + wg.Add(1) go func() { defer wg.Done() p := filepath.Join(walletDir, innerring.GlagoliticLetter(i).String()+".json") From a5f51add25872bedc655037d011c0a3e1366f06b Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 9 Aug 2023 15:53:01 +0300 Subject: [PATCH 0501/1943] [#162] ci: Add noliteral linter Signed-off-by: Alexander Chuprov --- .forgejo/workflows/tests.yml | 9 +++++---- .golangci.yml | 9 +++++++++ Makefile | 22 +++++++++++++++++++--- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 6e1afac9e..f66a2c401 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -14,10 +14,11 @@ jobs: go-version: '1.21' cache: true - - name: golangci-lint - uses: https://github.com/golangci/golangci-lint-action@v3 - with: - version: latest + - name: Install linters + run: make lint-install + + - name: Run linters + run: make lint tests: name: Tests diff --git a/.golangci.yml b/.golangci.yml index eba0d605c..12ebe04cb 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -37,6 +37,14 @@ linters-settings: alias: pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object alias: objectSDK + custom: + noliteral: + path: bin/external_linters.so + original-url: git.frostfs.info/TrueCloudLab/linters.git +# settings: +# target-methods : ["reportFlushError", "reportError"] +# disable-packages: ["codes", "err", "res","exec"] +# constants-repository: "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" linters: enable: @@ -69,5 +77,6 @@ linters: - gocognit - contextcheck - importas + - noliteral disable-all: true fast: false diff --git a/Makefile b/Makefile index e1464dd54..ed222efcf 100755 --- a/Makefile +++ b/Makefile @@ -8,8 +8,7 @@ HUB_IMAGE ?= truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.21 -LINT_VERSION ?= 1.52.2 -ARCH = amd64 +LINT_VERSION ?= 1.53.3 BIN = bin RELEASE = release @@ -25,6 +24,9 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \ sed "s/-/~/")-${OS_RELEASE} +OUTPUT_LINT_DIR ?= $(shell pwd)/bin +TMP_DIR := .cache + .PHONY: help all images dep clean fmts fmt imports test lint docker/lint prepare-release debpackage pre-commit unpre-commit @@ -131,9 +133,23 @@ test: pre-commit-run: @pre-commit run -a --hook-stage manual +# Install linters +lint-install: + @mkdir -p $(TMP_DIR) + @rm -rf $(TMP_DIR)/linters + @git clone --depth 1 https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters + @make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR) + @rm -rf $(TMP_DIR)/linters + @rmdir $(TMP_DIR) 2>/dev/null || true + @GOBIN=$(OUTPUT_LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) + + # Run linters lint: - @golangci-lint --timeout=5m run + NOLITERAL_TARGET_METHODS="reportFlushError,reportError" \ + NOLITERAL_DISABLE_PACKAGES="codes,err,res,exec" \ + NOLITERAL_CONSTANTS_PACKAGE="git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" \ + $(OUTPUT_LINT_DIR)/golangci-lint --timeout=10m run # Install staticcheck staticcheck-install: From 21800e9fccfdaee73aaf23de77e1610e1ddd5f2d Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 9 Aug 2023 15:53:13 +0300 Subject: [PATCH 0502/1943] [#162] core: Move literals to constants Signed-off-by: Alexander Chuprov --- cmd/frostfs-node/netmap.go | 2 +- internal/logs/logs.go | 16 ++++++++++++++++ pkg/local_object_storage/shard/delete.go | 4 ++-- .../writecache/writecachebadger/flush.go | 8 ++++---- .../writecache/writecachebbolt/flush.go | 12 ++++++------ pkg/morph/subscriber/subscriber.go | 4 ++-- pkg/services/object/put/single.go | 2 +- pkg/services/object_manager/tombstone/checker.go | 2 +- 8 files changed, 33 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index b0acac032..e7d3271b1 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -333,7 +333,7 @@ func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, bool, if nmState != candidateState { // This happens when the node was switched to maintenance without epoch tick. // We expect it to continue staying in maintenance. - c.log.Info("candidate status is different from the netmap status, the former takes priority", + c.log.Info(logs.CandidateStatusPriority, zap.String("netmap", nmState), zap.String("candidate", candidateState)) } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index ad4003bed..68914d6a3 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -495,6 +495,22 @@ const ( FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled" NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap = "the node is already in candidate list with online state, skip initial bootstrap" + RPConnectionLost = "RPC connection lost, attempting reconnect" + RPCNodeSwitchFailure = "can't switch RPC node" + FSTreeCantReadFile = "can't read a file" + FSTreeCantUnmarshalObject = "can't unmarshal an object" + FSTreeCantFushObjectBlobstor = "can't flush an object to blobstor" + FSTreeCantUpdateID = "can't update object storage ID" + FSTreeCantDecodeDBObjectAddress = "can't decode object address from the DB" + PutSingleRedirectFailure = "failed to redirect PutSingle request" + StorageIDRetrievalFailure = "can't get storage ID from metabase" + ObjectRemovalFailureBlobStor = "can't remove object from blobStor" + CandidateStatusPriority = "candidate status is different from the netmap status, the former takes priority" + TombstoneExpirationParseFailure = "tombstone getter: could not parse tombstone expiration epoch" + FrostFSNodeCantUpdateObjectStorageID = "can't update object storage ID" + FrostFSNodeCantFlushObjectToBlobstor = "can't flush an object to blobstor" + FrostFSNodeCantDecodeObjectAddressFromDB = "can't decode object address from the DB" // Error in ../node/cmd/frostfs-node/morph.go + FrostFSNodeCantUnmarshalObjectFromDB = "can't unmarshal an object from the DB" // Error in ../node/cmd/frostfs-node/morph.go RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" ) diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index c683d65a8..662c0835a 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -92,7 +92,7 @@ func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { res, err := s.metaBase.StorageID(ctx, sPrm) if err != nil { - s.log.Debug("can't get storage ID from metabase", + s.log.Debug(logs.StorageIDRetrievalFailure, zap.Stringer("object", addr), zap.String("error", err.Error())) } @@ -104,7 +104,7 @@ func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { _, err = s.blobStor.Delete(ctx, delPrm) if err != nil { - s.log.Debug("can't remove object from blobStor", + s.log.Debug(logs.ObjectRemovalFailureBlobStor, zap.Stringer("object_address", addr), zap.String("error", err.Error())) } diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush.go b/pkg/local_object_storage/writecache/writecachebadger/flush.go index d8bdddb56..fec6ecf88 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush.go @@ -188,7 +188,7 @@ func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []b if err != nil { if !errors.Is(err, common.ErrNoSpace) && !errors.Is(err, common.ErrReadOnly) && !errors.Is(err, blobstor.ErrNoPlaceFound) { - c.reportFlushError("can't flush an object to blobstor", + c.reportFlushError(logs.FrostFSNodeCantFlushObjectToBlobstor, addr.EncodeToString(), err) } return err @@ -200,7 +200,7 @@ func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []b _, err = c.metabase.UpdateStorageID(updPrm) if err != nil { - c.reportFlushError("can't update object storage ID", + c.reportFlushError(logs.FrostFSNodeCantUpdateObjectStorageID, addr.EncodeToString(), err) } return err @@ -230,7 +230,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { for it.Rewind(); it.Valid(); it.Next() { if got, want := int(it.Item().KeySize()), len(key); got != want { err := fmt.Errorf("invalid db key len: got %d, want %d", got, want) - c.reportFlushError("can't decode object address from the DB", hex.EncodeToString(it.Item().Key()), metaerr.Wrap(err)) + c.reportFlushError(logs.FrostFSNodeCantDecodeObjectAddressFromDB, hex.EncodeToString(it.Item().Key()), metaerr.Wrap(err)) if ignoreErrors { continue } @@ -240,7 +240,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { var obj objectSDK.Object if err := obj.Unmarshal(data); err != nil { copy(key[:], it.Item().Key()) - c.reportFlushError("can't unmarshal an object from the DB", key.address().EncodeToString(), metaerr.Wrap(err)) + c.reportFlushError(logs.FrostFSNodeCantUnmarshalObjectFromDB, key.address().EncodeToString(), metaerr.Wrap(err)) if ignoreErrors { return nil } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush.go b/pkg/local_object_storage/writecache/writecachebbolt/flush.go index 4c1fb60e3..c3b0f89bb 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush.go @@ -194,7 +194,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { data, err := f() if err != nil { - c.reportFlushError("can't read a file", sAddr, metaerr.Wrap(err)) + c.reportFlushError(logs.FSTreeCantReadFile, sAddr, metaerr.Wrap(err)) if ignoreErrors { return nil } @@ -204,7 +204,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { var obj objectSDK.Object err = obj.Unmarshal(data) if err != nil { - c.reportFlushError("can't unmarshal an object", sAddr, metaerr.Wrap(err)) + c.reportFlushError(logs.FSTreeCantUnmarshalObject, sAddr, metaerr.Wrap(err)) if ignoreErrors { return nil } @@ -268,7 +268,7 @@ func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []b if err != nil { if !errors.Is(err, common.ErrNoSpace) && !errors.Is(err, common.ErrReadOnly) && !errors.Is(err, blobstor.ErrNoPlaceFound) { - c.reportFlushError("can't flush an object to blobstor", + c.reportFlushError(logs.FSTreeCantFushObjectBlobstor, addr.EncodeToString(), err) } return err @@ -280,7 +280,7 @@ func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []b _, err = c.metabase.UpdateStorageID(updPrm) if err != nil { - c.reportFlushError("can't update object storage ID", + c.reportFlushError(logs.FSTreeCantUpdateID, addr.EncodeToString(), err) } return err @@ -315,7 +315,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { for k, data := cs.Seek(nil); k != nil; k, data = cs.Next() { sa := string(k) if err := addr.DecodeString(sa); err != nil { - c.reportFlushError("can't decode object address from the DB", sa, metaerr.Wrap(err)) + c.reportFlushError(logs.FSTreeCantDecodeDBObjectAddress, sa, metaerr.Wrap(err)) if ignoreErrors { continue } @@ -324,7 +324,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { var obj objectSDK.Object if err := obj.Unmarshal(data); err != nil { - c.reportFlushError("can't unmarshal an object from the DB", sa, metaerr.Wrap(err)) + c.reportFlushError(logs.FSTreeCantDecodeDBObjectAddress, sa, metaerr.Wrap(err)) if ignoreErrors { continue } diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 4076111f0..c2d8494fa 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -250,9 +250,9 @@ routeloop: } func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) (bool, <-chan rpcclient.Notification) { - s.log.Info("RPC connection lost, attempting reconnect") + s.log.Info(logs.RPConnectionLost) if !s.client.SwitchRPC(ctx) { - s.log.Error("can't switch RPC node") + s.log.Error(logs.RPCNodeSwitchFailure) return false, nil } diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 7c7955c87..eb18f0f46 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -343,7 +343,7 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, if err != nil { objID, _ := obj.ID() cnrID, _ := obj.ContainerID() - s.log.Warn("failed to redirect PutSingle request", + s.log.Warn(logs.PutSingleRedirectFailure, zap.Error(err), zap.Stringer("address", addr), zap.Stringer("object_id", objID), diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go index 66a0ec7d3..c3c810001 100644 --- a/pkg/services/object_manager/tombstone/checker.go +++ b/pkg/services/object_manager/tombstone/checker.go @@ -78,7 +78,7 @@ func (g *ExpirationChecker) handleTS(addr string, ts *objectSDK.Object, reqEpoch epoch, err := strconv.ParseUint(atr.Value(), 10, 64) if err != nil { g.log.Warn( - "tombstone getter: could not parse tombstone expiration epoch", + logs.TombstoneExpirationParseFailure, zap.Error(err), ) From ad87493c41c7b6225ba439fbc044821f47b2ac12 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 11 Aug 2023 13:05:16 +0300 Subject: [PATCH 0503/1943] [#8] Bump required go version to go1.20 Signed-off-by: Alexander Chuprov --- .golangci.yml | 8 ++++---- Makefile | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 12ebe04cb..845ab2202 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -41,10 +41,10 @@ linters-settings: noliteral: path: bin/external_linters.so original-url: git.frostfs.info/TrueCloudLab/linters.git -# settings: -# target-methods : ["reportFlushError", "reportError"] -# disable-packages: ["codes", "err", "res","exec"] -# constants-repository: "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + settings: + target-methods : ["reportFlushError", "reportError"] + disable-packages: ["codes", "err", "res","exec"] + constants-package: "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" linters: enable: diff --git a/Makefile b/Makefile index ed222efcf..3914986b5 100755 --- a/Makefile +++ b/Makefile @@ -8,7 +8,8 @@ HUB_IMAGE ?= truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.21 -LINT_VERSION ?= 1.53.3 +LINT_VERSION ?= 1.54.0 +ARCH = amd64 BIN = bin RELEASE = release @@ -25,6 +26,7 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed "s/-/~/")-${OS_RELEASE} OUTPUT_LINT_DIR ?= $(shell pwd)/bin +LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION) TMP_DIR := .cache .PHONY: help all images dep clean fmts fmt imports test lint docker/lint @@ -138,18 +140,18 @@ lint-install: @mkdir -p $(TMP_DIR) @rm -rf $(TMP_DIR)/linters @git clone --depth 1 https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters - @make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR) + @@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR) @rm -rf $(TMP_DIR)/linters @rmdir $(TMP_DIR) 2>/dev/null || true - @GOBIN=$(OUTPUT_LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) - + @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) # Run linters lint: - NOLITERAL_TARGET_METHODS="reportFlushError,reportError" \ - NOLITERAL_DISABLE_PACKAGES="codes,err,res,exec" \ - NOLITERAL_CONSTANTS_PACKAGE="git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" \ - $(OUTPUT_LINT_DIR)/golangci-lint --timeout=10m run + @if [ ! -d "$(LINT_DIR)" ]; then \ + echo "Run make lint-install"; \ + exit 1; \ + fi + $(LINT_DIR)/golangci-lint run # Install staticcheck staticcheck-install: From 376f03a445ca9c708b84110a4dbd3ec1ca560ab9 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Fri, 11 Aug 2023 16:43:23 +0300 Subject: [PATCH 0504/1943] [#598] Hold mode mutex when setting mode Signed-off-by: Alejandro Lopez --- pkg/local_object_storage/writecache/writecachebadger/mode.go | 3 +++ pkg/local_object_storage/writecache/writecachebbolt/mode.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/pkg/local_object_storage/writecache/writecachebadger/mode.go b/pkg/local_object_storage/writecache/writecachebadger/mode.go index 9a39fa41a..03d861831 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/mode.go +++ b/pkg/local_object_storage/writecache/writecachebadger/mode.go @@ -22,6 +22,9 @@ func (c *cache) SetMode(m mode.Mode) error { )) defer span.End() + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + err := c.setMode(ctx, m) if err == nil { c.metrics.SetMode(m) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/mode.go b/pkg/local_object_storage/writecache/writecachebbolt/mode.go index f7a9fffa3..b187996a1 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/mode.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/mode.go @@ -22,6 +22,9 @@ func (c *cache) SetMode(m mode.Mode) error { )) defer span.End() + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + err := c.setMode(ctx, m) if err == nil { c.metrics.SetMode(m) From ae8be495c8c9bf729e59526e8fbd526a19f98910 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Mon, 14 Aug 2023 14:01:39 +0300 Subject: [PATCH 0505/1943] [#xx] Avoid manual management of files in tests Signed-off-by: Alejandro Lopez --- .../blobstor/blobovniczatree/generic_test.go | 17 +++-------------- .../engine/control_test.go | 18 ++++-------------- pkg/local_object_storage/engine/delete_test.go | 3 --- pkg/local_object_storage/engine/engine_test.go | 8 +++----- pkg/local_object_storage/engine/head_test.go | 3 --- pkg/local_object_storage/engine/inhume_test.go | 3 --- pkg/local_object_storage/engine/list_test.go | 10 +++------- pkg/local_object_storage/engine/lock_test.go | 4 ---- pkg/local_object_storage/engine/shards_test.go | 2 -- .../metabase/counter_test.go | 4 ---- pkg/local_object_storage/metabase/db_test.go | 5 ++--- .../metabase/generic_test.go | 9 +-------- .../pilorama/generic_test.go | 10 +--------- 13 files changed, 17 insertions(+), 79 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go index f1bc0b225..7f95e750b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go @@ -1,9 +1,6 @@ package blobovniczatree import ( - "os" - "path/filepath" - "strconv" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -15,8 +12,6 @@ import ( func TestGeneric(t *testing.T) { const maxObjectSize = 1 << 16 - defer func() { _ = os.RemoveAll(t.Name()) }() - helper := func(t *testing.T, dir string) common.Storage { return NewBlobovniczaTree( WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), @@ -27,16 +22,14 @@ func TestGeneric(t *testing.T) { WithBlobovniczaSize(1<<20)) } - var n int newTree := func(t *testing.T) common.Storage { - dir := filepath.Join(t.Name(), strconv.Itoa(n)) - return helper(t, dir) + return helper(t, t.TempDir()) } blobstortest.TestAll(t, newTree, 1024, maxObjectSize) t.Run("info", func(t *testing.T) { - dir := filepath.Join(t.Name(), "info") + dir := t.TempDir() blobstortest.TestInfo(t, func(t *testing.T) common.Storage { return helper(t, dir) }, Type, dir) @@ -46,17 +39,13 @@ func TestGeneric(t *testing.T) { func TestControl(t *testing.T) { const maxObjectSize = 2048 - defer func() { _ = os.RemoveAll(t.Name()) }() - - var n int newTree := func(t *testing.T) common.Storage { - dir := filepath.Join(t.Name(), strconv.Itoa(n)) return NewBlobovniczaTree( WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), - WithRootPath(dir), + WithRootPath(t.TempDir()), WithBlobovniczaSize(1<<20)) } diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 593eb9971..804bef6ec 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -44,18 +44,12 @@ func TestInitializationFailure(t *testing.T) { sid, err := generateShardID() require.NoError(t, err) - tempDir := t.TempDir() - blobstorPath := filepath.Join(tempDir, "bs") - metabasePath := filepath.Join(tempDir, "mb") - writecachePath := filepath.Join(tempDir, "wc") - piloramaPath := filepath.Join(tempDir, "pl") - - storages, smallFileStorage, largeFileStorage := newTestStorages(blobstorPath, 1<<20) + storages, smallFileStorage, largeFileStorage := newTestStorages(t.TempDir(), 1<<20) wcOpts := writecacheconfig.Options{ Type: writecacheconfig.TypeBBolt, BBoltOptions: []writecachebbolt.Option{ - writecachebbolt.WithPath(writecachePath), + writecachebbolt.WithPath(t.TempDir()), }, } @@ -69,13 +63,13 @@ func TestInitializationFailure(t *testing.T) { Timeout: 100 * time.Millisecond, OpenFile: opts.openFileMetabase, }), - meta.WithPath(metabasePath), + meta.WithPath(filepath.Join(t.TempDir(), "metabase")), meta.WithPermissions(0700), meta.WithEpochState(epochState{})), shard.WithWriteCache(true), shard.WithWriteCacheOptions(wcOpts), shard.WithPiloramaOptions( - pilorama.WithPath(piloramaPath), + pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama")), pilorama.WithOpenFile(opts.openFilePilorama), ), }, smallFileStorage, largeFileStorage @@ -178,10 +172,6 @@ func testEngineFailInitAndReload(t *testing.T, errOnAdd bool, opts []shard.Optio func TestExecBlocks(t *testing.T) { e := testNewEngine(t).setShardsNum(t, 2).engine // number doesn't matter in this test, 2 is several but not many - t.Cleanup(func() { - os.RemoveAll(t.Name()) - }) - // put some object obj := testutil.GenerateObjectWithCID(cidtest.ID()) diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 3ea0291da..e4a516512 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -2,7 +2,6 @@ package engine import ( "context" - "os" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -20,8 +19,6 @@ import ( func TestDeleteBigObject(t *testing.T) { t.Parallel() - defer os.RemoveAll(t.Name()) - cnr := cidtest.ID() parentID := oidtest.ID() splitID := objectSDK.NewSplitID() diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 26de2e5eb..e3dd49728 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -2,7 +2,6 @@ package engine import ( "context" - "fmt" "os" "path/filepath" "sync/atomic" @@ -204,11 +203,10 @@ func testDefaultShardOptions(t testing.TB, id int) []shard.Option { shard.WithLogger(&logger.Logger{Logger: zap.L()}), shard.WithBlobStorOptions( blobstor.WithStorages( - newStorages(filepath.Join(t.Name(), fmt.Sprintf("%d.blobstor", id)), - 1<<20))), - shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.pilorama", id)))), + newStorages(t.TempDir(), 1<<20))), + shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))), shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(t.Name(), fmt.Sprintf("%d.metabase", id))), + meta.WithPath(filepath.Join(t.TempDir(), "metabase")), meta.WithPermissions(0700), meta.WithEpochState(epochState{}), )} diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go index d5bf8429c..e212ffa32 100644 --- a/pkg/local_object_storage/engine/head_test.go +++ b/pkg/local_object_storage/engine/head_test.go @@ -2,7 +2,6 @@ package engine import ( "context" - "os" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" @@ -14,8 +13,6 @@ import ( ) func TestHeadRaw(t *testing.T) { - defer os.RemoveAll(t.Name()) - cnr := cidtest.ID() splitID := objectSDK.NewSplitID() diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 924cf518b..a5f242071 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -2,7 +2,6 @@ package engine import ( "context" - "os" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -14,8 +13,6 @@ import ( ) func TestStorageEngine_Inhume(t *testing.T) { - defer os.RemoveAll(t.Name()) - cnr := cidtest.ID() splitID := objectSDK.NewSplitID() diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 6cea2d0f4..9d5d12683 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -2,8 +2,6 @@ package engine import ( "context" - "fmt" - "os" "path/filepath" "sort" "testing" @@ -71,11 +69,10 @@ func TestListWithCursor(t *testing.T) { shard.WithLogger(&logger.Logger{Logger: zap.L()}), shard.WithBlobStorOptions( blobstor.WithStorages( - newStorages(filepath.Join(t.Name(), tt.name, fmt.Sprintf("%d.blobstor", id)), - 1<<20))), - shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.Name(), tt.name, fmt.Sprintf("%d.pilorama", id)))), + newStorages(t.TempDir(), 1<<20))), + shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))), shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(t.Name(), tt.name, fmt.Sprintf("%d.metabase", id))), + meta.WithPath(filepath.Join(t.TempDir(), "metabase")), meta.WithPermissions(0700), meta.WithEpochState(epochState{}), )} @@ -85,7 +82,6 @@ func TestListWithCursor(t *testing.T) { t.Cleanup(func() { e.Close() - os.RemoveAll(t.Name()) }) expected := make([]object.AddressWithType, 0, tt.objectNum) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 3e253bfc7..906a867e2 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -2,7 +2,6 @@ package engine import ( "context" - "os" "strconv" "testing" "time" @@ -65,7 +64,6 @@ func TestLockUserScenario(t *testing.T) { t.Cleanup(func() { _ = e.Close() - _ = os.RemoveAll(t.Name()) }) lockerID := oidtest.ID() @@ -174,7 +172,6 @@ func TestLockExpiration(t *testing.T) { t.Cleanup(func() { _ = e.Close() - _ = os.RemoveAll(t.Name()) }) const lockerExpiresAfter = 13 @@ -254,7 +251,6 @@ func TestLockForceRemoval(t *testing.T) { require.NoError(t, e.Init(context.Background())) t.Cleanup(func() { _ = e.Close() - _ = os.RemoveAll(t.Name()) }) cnr := cidtest.ID() diff --git a/pkg/local_object_storage/engine/shards_test.go b/pkg/local_object_storage/engine/shards_test.go index 1bc0b880c..3631e33ed 100644 --- a/pkg/local_object_storage/engine/shards_test.go +++ b/pkg/local_object_storage/engine/shards_test.go @@ -1,7 +1,6 @@ package engine import ( - "os" "testing" "github.com/stretchr/testify/require" @@ -14,7 +13,6 @@ func TestRemoveShard(t *testing.T) { e, ids := te.engine, te.shardIDs t.Cleanup(func() { e.Close() - os.RemoveAll(t.Name()) }) require.Equal(t, numOfShards, len(e.shardPools)) diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 2c5fa0233..89b52c887 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -2,7 +2,6 @@ package meta_test import ( "context" - "os" "testing" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -18,9 +17,6 @@ const objCount = 10 func TestCounters(t *testing.T) { t.Parallel() - t.Cleanup(func() { - require.NoError(t, os.RemoveAll(t.Name())) - }) t.Run("defaults", func(t *testing.T) { t.Parallel() diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index 9ef7bf8bc..6169dd81e 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -2,6 +2,7 @@ package meta_test import ( "os" + "path/filepath" "strconv" "testing" @@ -40,11 +41,9 @@ func testSelect(t *testing.T, db *meta.DB, cnr cid.ID, fs objectSDK.SearchFilter } func newDB(t testing.TB, opts ...meta.Option) *meta.DB { - path := t.Name() - bdb := meta.New( append([]meta.Option{ - meta.WithPath(path), + meta.WithPath(filepath.Join(t.TempDir(), "metabase")), meta.WithPermissions(0600), meta.WithEpochState(epochState{}), }, opts...)..., diff --git a/pkg/local_object_storage/metabase/generic_test.go b/pkg/local_object_storage/metabase/generic_test.go index 9d15b6f7a..52581b2a0 100644 --- a/pkg/local_object_storage/metabase/generic_test.go +++ b/pkg/local_object_storage/metabase/generic_test.go @@ -1,9 +1,7 @@ package meta import ( - "os" "path/filepath" - "strconv" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest" @@ -12,15 +10,10 @@ import ( func TestGeneric(t *testing.T) { t.Parallel() - defer func() { _ = os.RemoveAll(t.Name()) }() - - var n int newMetabase := func(t *testing.T) storagetest.Component { - n++ - dir := filepath.Join(t.Name(), strconv.Itoa(n)) return New( WithEpochState(epochStateImpl{}), - WithPath(dir)) + WithPath(filepath.Join(t.TempDir(), "metabase"))) } storagetest.TestAll(t, newMetabase) diff --git a/pkg/local_object_storage/pilorama/generic_test.go b/pkg/local_object_storage/pilorama/generic_test.go index cac03c4e1..d6a9f333b 100644 --- a/pkg/local_object_storage/pilorama/generic_test.go +++ b/pkg/local_object_storage/pilorama/generic_test.go @@ -1,23 +1,15 @@ package pilorama import ( - "os" "path/filepath" - "strconv" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest" ) func TestGeneric(t *testing.T) { - defer func() { _ = os.RemoveAll(t.Name()) }() - - var n int newPilorama := func(t *testing.T) storagetest.Component { - n++ - dir := filepath.Join(t.Name(), strconv.Itoa(n)) - return NewBoltForest( - WithPath(dir)) + return NewBoltForest(WithPath(filepath.Join(t.TempDir(), "pilorama"))) } storagetest.TestAll(t, newPilorama) From 88b6755c5e95a0176e828272d380efab62a959f1 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Mon, 14 Aug 2023 10:54:09 +0300 Subject: [PATCH 0506/1943] [#598] Fix use-after-close bug in badger writecache Signed-off-by: Alejandro Lopez --- .../writecache/writecachebadger/flush.go | 7 +++++++ .../writecache/writecachebadger/gc.go | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush.go b/pkg/local_object_storage/writecache/writecachebadger/flush.go index fec6ecf88..8630026cd 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush.go @@ -80,6 +80,13 @@ func (c *cache) flushSmallObjects() { continue } + // Using the db after Close will panic and badger won't wait for outstanding txs, + // so we need to check manually. + if c.db.IsClosed() { + c.modeMtx.RUnlock() + return + } + _ = c.db.View(func(tx *badger.Txn) error { it := tx.NewIterator(badger.DefaultIteratorOptions) defer it.Close() diff --git a/pkg/local_object_storage/writecache/writecachebadger/gc.go b/pkg/local_object_storage/writecache/writecachebadger/gc.go index 51d3e9763..8432a9c04 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/gc.go +++ b/pkg/local_object_storage/writecache/writecachebadger/gc.go @@ -20,6 +20,14 @@ func (c *cache) runGCLoop() { case <-c.closeCh: return case <-t.C: + // This serves to synchronize the c.db field when changing mode as well. + c.modeMtx.RLock() + ro := c.readOnly() + c.modeMtx.RUnlock() + if ro { + continue + } + // 0.5 is the recommended value so that write amplification of the value log is 2. // See https://pkg.go.dev/github.com/dgraph-io/badger/v4#DB.RunValueLogGC for more info. for c.db.RunValueLogGC(0.5) == nil { From bc425b5bad7871aba7efd43b93ed3f329caa4456 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 14 Aug 2023 15:53:34 +0300 Subject: [PATCH 0507/1943] [#608] pre-commit: Fix linter Signed-off-by: Alexander Chuprov --- .golangci.yml | 2 +- .pre-commit-config.yaml | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 845ab2202..6741f1278 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -40,7 +40,7 @@ linters-settings: custom: noliteral: path: bin/external_linters.so - original-url: git.frostfs.info/TrueCloudLab/linters.git + original-url: git.frostfs.info/TrueCloudLab/linters.git settings: target-methods : ["reportFlushError", "reportError"] disable-packages: ["codes", "err", "res","exec"] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d9b2b8f2..f891f8860 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,10 +30,13 @@ repos: hooks: - id: shellcheck - - repo: https://github.com/golangci/golangci-lint - rev: v1.52.2 + - repo: local hooks: - - id: golangci-lint + - id: make-lint + name: Run Make Lint + entry: make lint + language: system + pass_filenames: false - repo: local hooks: From d15199c5d81748cf31a01b56ecdfdad2cf75520c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 16 Aug 2023 10:39:41 +0300 Subject: [PATCH 0508/1943] [#596] engine: Consider context errors as logical Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/engine.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 7bc84e6c9..bb0b682d6 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -1,6 +1,7 @@ package engine import ( + "context" "errors" "sync" "sync/atomic" @@ -196,7 +197,7 @@ func (e *StorageEngine) reportShardErrorWithFlags( } func isLogical(err error) bool { - return errors.As(err, &logicerr.Logical{}) + return errors.As(err, &logicerr.Logical{}) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) } // Option represents StorageEngine's constructor option. From eec97d177e27c02bfe54e28b822bebbadb018e5c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Aug 2023 17:14:41 +0300 Subject: [PATCH 0509/1943] [#585] writecache: Count items periodically Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + .../writecache/writecachebbolt/cachebbolt.go | 3 +- .../writecache/writecachebbolt/delete.go | 2 - .../writecache/writecachebbolt/put.go | 2 - .../writecache/writecachebbolt/state.go | 87 +++++++++++-------- .../writecache/writecachebbolt/storage.go | 2 - 6 files changed, 55 insertions(+), 42 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 68914d6a3..8d3712590 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -513,4 +513,5 @@ const ( FrostFSNodeCantUnmarshalObjectFromDB = "can't unmarshal an object from the DB" // Error in ../node/cmd/frostfs-node/morph.go RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" + FailedToCountWritecacheItems = "failed to count writecache items" ) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go index 407d1a9ce..69e418bb3 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go @@ -107,13 +107,14 @@ func (c *cache) Open(readOnly bool) error { // thus we need to create a channel here. c.closeCh = make(chan struct{}) - return metaerr.Wrap(c.initCounters()) + return metaerr.Wrap(c.setCounters()) } // Init runs necessary services. func (c *cache) Init() error { c.metrics.SetMode(c.mode) c.runFlushLoop() + c.runDBCounterLoop() return nil } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/delete.go b/pkg/local_object_storage/writecache/writecachebbolt/delete.go index b0cc091ab..105526c27 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/delete.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/delete.go @@ -63,7 +63,6 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.OpField("db DELETE"), ) deleted = true - c.objCounters.DecDB() return nil } @@ -75,7 +74,6 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree DELETE"), ) - c.objCounters.DecFS() deleted = true } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/put.go b/pkg/local_object_storage/writecache/writecachebbolt/put.go index 48b130d26..645e0bf8c 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/put.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/put.go @@ -95,7 +95,6 @@ func (c *cache) putSmall(obj objectInfo) error { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("db PUT"), ) - c.objCounters.IncDB() } return err } @@ -117,7 +116,6 @@ func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) erro c.compressFlags[addr] = struct{}{} c.mtx.Unlock() } - c.objCounters.IncFS() storagelog.Write(c.log, storagelog.AddressField(addr), storagelog.StorageTypeField(wcStorageType), diff --git a/pkg/local_object_storage/writecache/writecachebbolt/state.go b/pkg/local_object_storage/writecache/writecachebbolt/state.go index 03283dfef..49b802aef 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/state.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/state.go @@ -2,10 +2,13 @@ package writecachebbolt import ( "fmt" - "math" "sync/atomic" + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "go.etcd.io/bbolt" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) func (c *cache) estimateCacheSize() uint64 { @@ -27,54 +30,68 @@ type counters struct { cDB, cFS atomic.Uint64 } -func (x *counters) IncDB() { - x.cDB.Add(1) -} - -func (x *counters) DecDB() { - x.cDB.Add(math.MaxUint64) -} - func (x *counters) DB() uint64 { return x.cDB.Load() } -func (x *counters) IncFS() { - x.cFS.Add(1) -} - -func (x *counters) DecFS() { - x.cFS.Add(math.MaxUint64) -} - func (x *counters) FS() uint64 { return x.cFS.Load() } -func (c *cache) initCounters() error { +func (c *cache) setCounters() error { var inDB uint64 - err := c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - if b != nil { - inDB = uint64(b.Stats().KeyN) + var inFS uint64 + + var eg errgroup.Group + + eg.Go(func() error { + err := c.db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(defaultBucket) + if b != nil { + inDB = uint64(b.Stats().KeyN) + } + return nil + }) + if err != nil { + return fmt.Errorf("could not read write-cache DB counter: %w", err) } + c.objCounters.cDB.Store(inDB) return nil }) - if err != nil { - return fmt.Errorf("could not read write-cache DB counter: %w", err) - } - inFS, err := c.fsTree.NumberOfObjects() - if err != nil { - return fmt.Errorf("could not read write-cache FS counter: %w", err) + eg.Go(func() error { + var err error + inFS, err = c.fsTree.NumberOfObjects() + if err != nil { + return fmt.Errorf("could not read write-cache FS counter: %w", err) + } + if inFS > 0 { + inFS-- //small.bolt DB file + } + c.objCounters.cFS.Store(inFS) + return nil + }) + if err := eg.Wait(); err != nil { + return err } - if inFS > 0 { - inFS-- //small.bolt DB file - } - - c.objCounters.cDB.Store(inDB) - c.objCounters.cFS.Store(inFS) c.metrics.SetActualCounters(inDB, inFS) - return nil } + +func (c *cache) runDBCounterLoop() { + go func() { + t := time.NewTicker(time.Second * 30) + defer t.Stop() + for { + select { + case <-t.C: + err := c.setCounters() + if err != nil { + c.log.Warn(logs.FailedToCountWritecacheItems, zap.Error(err)) + } + case <-c.closeCh: + return + } + } + }() +} diff --git a/pkg/local_object_storage/writecache/writecachebbolt/storage.go b/pkg/local_object_storage/writecache/writecachebbolt/storage.go index 67397f985..c651b7445 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/storage.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/storage.go @@ -69,7 +69,6 @@ func (c *cache) deleteFromDB(key string) { }) if err == nil { - c.objCounters.DecDB() c.metrics.Evict(writecache.StorageTypeDB) storagelog.Write(c.log, storagelog.AddressField(key), @@ -111,7 +110,6 @@ func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { storagelog.OpField("fstree DELETE"), ) c.metrics.Evict(writecache.StorageTypeFSTree) - c.objCounters.DecFS() c.estimateCacheSize() } } From 0c52186572d26021cca1492293d4395e75cefd4e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 10 Aug 2023 18:42:35 +0300 Subject: [PATCH 0510/1943] [#585] fstree: Remove unused method Signed-off-by: Dmitrii Stepanov --- .../blobstor/fstree/fstree.go | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 637bf8438..2692fbf7e 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -396,27 +396,6 @@ func (t *FSTree) writeFile(p string, data []byte) error { return err } -// PutStream puts executes handler on a file opened for write. -func (t *FSTree) PutStream(addr oid.Address, handler func(*os.File) error) error { - if t.readOnly { - return common.ErrReadOnly - } - - p := t.treePath(addr) - - if err := util.MkdirAllX(filepath.Dir(p), t.Permissions); err != nil { - return err - } - - f, err := os.OpenFile(p, t.writeFlags(), t.Permissions) - if err != nil { - return err - } - defer f.Close() - - return handler(f) -} - // Get returns an object from the storage by address. func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { var ( From baad49990cffafee1e5ccd8952b83a00c869a52f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 10 Aug 2023 18:46:24 +0300 Subject: [PATCH 0511/1943] [#585] fstree: Return logical error if object deleted Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobstor/fstree/fstree.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 2692fbf7e..5a9adde0f 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -429,6 +429,9 @@ func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, err data, err = os.ReadFile(p) if err != nil { + if os.IsNotExist(err) { + return common.GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) + } return common.GetRes{}, err } } From 58c8722c815320206cc14bfa878772f1b8c6efc8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 11 Aug 2023 11:32:43 +0300 Subject: [PATCH 0512/1943] [#585] fstree: Add optional file counter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/cache.go | 70 ++--------- .../blobstor/fstree/control.go | 5 +- .../blobstor/fstree/counter.go | 32 +++++ .../blobstor/fstree/fstree.go | 114 +++++++++++------- .../blobstor/fstree/fstree_test.go | 71 +++++++++++ .../blobstor/fstree/option.go | 10 ++ .../blobstor/perf_test.go | 13 +- .../writecache/writecachebbolt/cachebbolt.go | 3 +- .../writecache/writecachebbolt/state.go | 95 ++++++--------- .../writecache/writecachebbolt/storage.go | 7 +- pkg/util/sync/key_locker.go | 56 +++++++++ .../util/sync/key_locker_test.go | 8 +- 12 files changed, 312 insertions(+), 172 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/fstree/counter.go create mode 100644 pkg/util/sync/key_locker.go rename cmd/frostfs-node/cache_test.go => pkg/util/sync/key_locker_test.go (82%) diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index fa5513640..6d138d894 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" + utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -24,59 +25,6 @@ type valueWithTime[V any] struct { e error } -type locker struct { - mtx sync.Mutex - waiters int // not protected by mtx, must used outer mutex to update concurrently -} - -type keyLocker[K comparable] struct { - lockers map[K]*locker - lockersMtx sync.Mutex -} - -func newKeyLocker[K comparable]() *keyLocker[K] { - return &keyLocker[K]{ - lockers: make(map[K]*locker), - } -} - -func (l *keyLocker[K]) LockKey(key K) { - l.lockersMtx.Lock() - - if locker, found := l.lockers[key]; found { - locker.waiters++ - l.lockersMtx.Unlock() - - locker.mtx.Lock() - return - } - - locker := &locker{ - waiters: 1, - } - locker.mtx.Lock() - - l.lockers[key] = locker - l.lockersMtx.Unlock() -} - -func (l *keyLocker[K]) UnlockKey(key K) { - l.lockersMtx.Lock() - defer l.lockersMtx.Unlock() - - locker, found := l.lockers[key] - if !found { - return - } - - if locker.waiters == 1 { - delete(l.lockers, key) - } - locker.waiters-- - - locker.mtx.Unlock() -} - // entity that provides TTL cache interface. type ttlNetCache[K comparable, V any] struct { ttl time.Duration @@ -87,7 +35,7 @@ type ttlNetCache[K comparable, V any] struct { netRdr netValueReader[K, V] - keyLocker *keyLocker[K] + keyLocker *utilSync.KeyLocker[K] } // complicates netValueReader with TTL caching mechanism. @@ -100,7 +48,7 @@ func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr n sz: sz, cache: cache, netRdr: netRdr, - keyLocker: newKeyLocker[K](), + keyLocker: utilSync.NewKeyLocker[K](), } } @@ -115,8 +63,8 @@ func (c *ttlNetCache[K, V]) get(key K) (V, error) { return val.v, val.e } - c.keyLocker.LockKey(key) - defer c.keyLocker.UnlockKey(key) + c.keyLocker.Lock(key) + defer c.keyLocker.Unlock(key) val, ok = c.cache.Peek(key) if ok && time.Since(val.t) < c.ttl { @@ -135,8 +83,8 @@ func (c *ttlNetCache[K, V]) get(key K) (V, error) { } func (c *ttlNetCache[K, V]) set(k K, v V, e error) { - c.keyLocker.LockKey(k) - defer c.keyLocker.UnlockKey(k) + c.keyLocker.Lock(k) + defer c.keyLocker.Unlock(k) c.cache.Add(k, &valueWithTime[V]{ v: v, @@ -146,8 +94,8 @@ func (c *ttlNetCache[K, V]) set(k K, v V, e error) { } func (c *ttlNetCache[K, V]) remove(key K) { - c.keyLocker.LockKey(key) - defer c.keyLocker.UnlockKey(key) + c.keyLocker.Lock(key) + defer c.keyLocker.Unlock(key) c.cache.Remove(key) } diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index f41b7aacd..c56312d38 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -13,7 +13,10 @@ func (t *FSTree) Open(ro bool) error { // Init implements common.Storage. func (t *FSTree) Init() error { - return util.MkdirAllX(t.RootPath, t.Permissions) + if err := util.MkdirAllX(t.RootPath, t.Permissions); err != nil { + return err + } + return t.initFileCounter() } // Close implements common.Storage. diff --git a/pkg/local_object_storage/blobstor/fstree/counter.go b/pkg/local_object_storage/blobstor/fstree/counter.go new file mode 100644 index 000000000..70b346093 --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/counter.go @@ -0,0 +1,32 @@ +package fstree + +import ( + "math" + "sync/atomic" +) + +// FileCounter used to count files in FSTree. The implementation must be thread-safe. +type FileCounter interface { + Set(v uint64) + Inc() + Dec() +} + +type noopCounter struct{} + +func (c *noopCounter) Set(uint64) {} +func (c *noopCounter) Inc() {} +func (c *noopCounter) Dec() {} + +type SimpleCounter struct { + v atomic.Uint64 +} + +func NewSimpleCounter() *SimpleCounter { + return &SimpleCounter{} +} + +func (c *SimpleCounter) Set(v uint64) { c.v.Store(v) } +func (c *SimpleCounter) Inc() { c.v.Add(1) } +func (c *SimpleCounter) Dec() { c.v.Add(math.MaxUint64) } +func (c *SimpleCounter) Value() uint64 { return c.v.Load() } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 5a9adde0f..243a7239e 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strconv" "strings" + "sync/atomic" "syscall" "time" @@ -26,6 +27,16 @@ import ( "go.opentelemetry.io/otel/trace" ) +type keyLock interface { + Lock(string) + Unlock(string) +} + +type noopKeyLock struct{} + +func (l *noopKeyLock) Lock(string) {} +func (l *noopKeyLock) Unlock(string) {} + // FSTree represents an object storage as a filesystem tree. type FSTree struct { Info @@ -37,6 +48,12 @@ type FSTree struct { noSync bool readOnly bool metrics Metrics + + fileGuard keyLock + fileCounter FileCounter + fileCounterEnabled bool + + suffix atomic.Uint64 } // Info groups the information about file storage. @@ -63,10 +80,12 @@ func New(opts ...Option) *FSTree { Permissions: 0700, RootPath: "./", }, - Config: nil, - Depth: 4, - DirNameLen: DirNameLen, - metrics: &noopMetrics{}, + Config: nil, + Depth: 4, + DirNameLen: DirNameLen, + metrics: &noopMetrics{}, + fileGuard: &noopKeyLock{}, + fileCounter: &noopCounter{}, } for i := range opts { opts[i](f) @@ -244,7 +263,17 @@ func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Delet p := t.treePath(prm.Address) - err = os.Remove(p) + if t.fileCounterEnabled { + t.fileGuard.Lock(p) + err = os.Remove(p) + t.fileGuard.Unlock(p) + if err == nil { + t.fileCounter.Dec() + } + } else { + err = os.Remove(p) + } + if err != nil && os.IsNotExist(err) { err = logicerr.Wrap(new(apistatus.ObjectNotFound)) } @@ -317,45 +346,19 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err prm.RawData = t.Compress(prm.RawData) } - // Here is a situation: - // Feb 09 13:10:37 buky neofs-node[32445]: 2023-02-09T13:10:37.161Z info log/log.go:13 local object storage operation {"shard_id": "SkT8BfjouW6t93oLuzQ79s", "address": "7NxFz4SruSi8TqXacr2Ae22nekMhgYk1sfkddJo9PpWk/5enyUJGCyU1sfrURDnHEjZFdbGqANVhayYGfdSqtA6wA", "op": "PUT", "type": "fstree", "storage_id": ""} - // Feb 09 13:10:37 buky neofs-node[32445]: 2023-02-09T13:10:37.183Z info log/log.go:13 local object storage operation {"shard_id": "SkT8BfjouW6t93oLuzQ79s", "address": "7NxFz4SruSi8TqXacr2Ae22nekMhgYk1sfkddJo9PpWk/5enyUJGCyU1sfrURDnHEjZFdbGqANVhayYGfdSqtA6wA", "op": "metabase PUT"} - // Feb 09 13:10:37 buky neofs-node[32445]: 2023-02-09T13:10:37.862Z debug policer/check.go:231 shortage of object copies detected {"component": "Object Policer", "object": "7NxFz4SruSi8TqXacr2Ae22nekMhgYk1sfkddJo9PpWk/5enyUJGCyU1sfrURDnHEjZFdbGqANVhayYGfdSqtA6wA", "shortage": 1} - // Feb 09 13:10:37 buky neofs-node[32445]: 2023-02-09T13:10:37.862Z debug shard/get.go:124 object is missing in write-cache {"shard_id": "SkT8BfjouW6t93oLuzQ79s", "addr": "7NxFz4SruSi8TqXacr2Ae22nekMhgYk1sfkddJo9PpWk/5enyUJGCyU1sfrURDnHEjZFdbGqANVhayYGfdSqtA6wA", "skip_meta": false} - // - // 1. We put an object on node 1. - // 2. Relentless policer sees that it has only 1 copy and tries to PUT it to node 2. - // 3. PUT operation started by client at (1) also puts an object here. - // 4. Now we have concurrent writes and one of `Rename` calls will return `no such file` error. - // Even more than that, concurrent writes can corrupt data. - // - // So here is a solution: - // 1. Write a file to 'name + 1'. - // 2. If it exists, retry with temporary name being 'name + 2'. - // 3. Set some reasonable number of attempts. - // - // It is a bit kludgey, but I am unusually proud about having found this out after - // hours of research on linux kernel, dirsync mount option and ext4 FS, turned out - // to be so hecking simple. - // In a very rare situation we can have multiple partially written copies on disk, - // this will be fixed in another issue (we should remove garbage on start). size = len(prm.RawData) - const retryCount = 5 - for i := 0; i < retryCount; i++ { - tmpPath := p + "#" + strconv.FormatUint(uint64(i), 10) - err = t.writeAndRename(tmpPath, p, prm.RawData) - if err != syscall.EEXIST || i == retryCount-1 { - return common.PutRes{StorageID: []byte{}}, err - } - } - - err = fmt.Errorf("couldn't read file after %d retries", retryCount) - // unreachable, but precaution never hurts, especially 1 day before release. + tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10) + err = t.writeAndRename(tmpPath, p, prm.RawData) return common.PutRes{StorageID: []byte{}}, err } // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { + if t.fileCounterEnabled { + t.fileGuard.Lock(p) + defer t.fileGuard.Unlock(p) + } + err := t.writeFile(tmpPath, data) if err != nil { var pe *fs.PathError @@ -364,10 +367,21 @@ func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { case syscall.ENOSPC: err = common.ErrNoSpace _ = os.RemoveAll(tmpPath) - case syscall.EEXIST: - return syscall.EEXIST } } + return err + } + + if t.fileCounterEnabled { + t.fileCounter.Inc() + var targetFileExists bool + if _, e := os.Stat(p); e == nil { + targetFileExists = true + } + err = os.Rename(tmpPath, p) + if err == nil && targetFileExists { + t.fileCounter.Dec() + } } else { err = os.Rename(tmpPath, p) } @@ -491,11 +505,23 @@ func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.G }, nil } -// NumberOfObjects walks the file tree rooted at FSTree's root -// and returns number of stored objects. -func (t *FSTree) NumberOfObjects() (uint64, error) { - var counter uint64 +// initFileCounter walks the file tree rooted at FSTree's root, +// counts total items count, inits counter and returns number of stored objects. +func (t *FSTree) initFileCounter() error { + if !t.fileCounterEnabled { + return nil + } + counter, err := t.countFiles() + if err != nil { + return err + } + t.fileCounter.Set(counter) + return nil +} + +func (t *FSTree) countFiles() (uint64, error) { + var counter uint64 // it is simpler to just consider every file // that is not directory as an object err := filepath.WalkDir(t.RootPath, diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_test.go b/pkg/local_object_storage/blobstor/fstree/fstree_test.go index 0e5525e77..b81ce43f1 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_test.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_test.go @@ -1,10 +1,16 @@ package fstree import ( + "context" + "errors" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) func TestAddressToString(t *testing.T) { @@ -28,3 +34,68 @@ func Benchmark_addressFromString(b *testing.B) { } } } + +func TestObjectCounter(t *testing.T) { + t.Parallel() + counter := NewSimpleCounter() + fst := New( + WithPath(t.TempDir()), + WithDepth(2), + WithDirNameLen(2), + WithFileCounter(counter)) + require.NoError(t, fst.Open(false)) + require.NoError(t, fst.Init()) + + counterValue := counter.Value() + require.Equal(t, uint64(0), counterValue) + + defer func() { + require.NoError(t, fst.Close()) + }() + + addr := oidtest.Address() + obj := objectSDK.New() + obj.SetID(addr.Object()) + obj.SetContainerID(addr.Container()) + obj.SetPayload([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}) + + var putPrm common.PutPrm + putPrm.Address = addr + putPrm.RawData, _ = obj.Marshal() + + var getPrm common.GetPrm + getPrm.Address = putPrm.Address + + var delPrm common.DeletePrm + delPrm.Address = addr + + eg, egCtx := errgroup.WithContext(context.Background()) + + eg.Go(func() error { + for j := 0; j < 1_000; j++ { + _, err := fst.Put(egCtx, putPrm) + if err != nil { + return err + } + } + return nil + }) + + eg.Go(func() error { + var le logicerr.Logical + for j := 0; j < 1_000; j++ { + _, err := fst.Delete(egCtx, delPrm) + if err != nil && !errors.As(err, &le) { + return err + } + } + return nil + }) + + require.NoError(t, eg.Wait()) + + counterValue = counter.Value() + realCount, err := fst.countFiles() + require.NoError(t, err) + require.Equal(t, realCount, counterValue) +} diff --git a/pkg/local_object_storage/blobstor/fstree/option.go b/pkg/local_object_storage/blobstor/fstree/option.go index 52c8718c2..21d46ac4d 100644 --- a/pkg/local_object_storage/blobstor/fstree/option.go +++ b/pkg/local_object_storage/blobstor/fstree/option.go @@ -2,6 +2,8 @@ package fstree import ( "io/fs" + + utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" ) type Option func(*FSTree) @@ -41,3 +43,11 @@ func WithMetrics(m Metrics) Option { f.metrics = m } } + +func WithFileCounter(c FileCounter) Option { + return func(f *FSTree) { + f.fileCounterEnabled = true + f.fileCounter = c + f.fileGuard = utilSync.NewKeyLocker[string]() + } +} diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index 5245146cb..c773ea0ee 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -53,7 +53,7 @@ var storages = []storage{ }, }, { - desc: "fstree", + desc: "fstree_without_object_counter", create: func(dir string) common.Storage { return fstree.New( fstree.WithPath(dir), @@ -62,6 +62,17 @@ var storages = []storage{ ) }, }, + { + desc: "fstree_with_object_counter", + create: func(dir string) common.Storage { + return fstree.New( + fstree.WithPath(dir), + fstree.WithDepth(2), + fstree.WithDirNameLen(2), + fstree.WithFileCounter(fstree.NewSimpleCounter()), + ) + }, + }, { desc: "blobovniczatree", create: func(dir string) common.Storage { diff --git a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go index 69e418bb3..407d1a9ce 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go @@ -107,14 +107,13 @@ func (c *cache) Open(readOnly bool) error { // thus we need to create a channel here. c.closeCh = make(chan struct{}) - return metaerr.Wrap(c.setCounters()) + return metaerr.Wrap(c.initCounters()) } // Init runs necessary services. func (c *cache) Init() error { c.metrics.SetMode(c.mode) c.runFlushLoop() - c.runDBCounterLoop() return nil } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/state.go b/pkg/local_object_storage/writecache/writecachebbolt/state.go index 49b802aef..91d54b0ea 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/state.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/state.go @@ -2,20 +2,24 @@ package writecachebbolt import ( "fmt" + "math" "sync/atomic" - "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "go.etcd.io/bbolt" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" ) func (c *cache) estimateCacheSize() uint64 { - db := c.objCounters.DB() * c.smallObjectSize - fstree := c.objCounters.FS() * c.maxObjectSize - c.metrics.SetEstimateSize(db, fstree) - return db + fstree + dbCount := c.objCounters.DB() + fsCount := c.objCounters.FS() + if fsCount > 0 { + fsCount-- //db file + } + dbSize := dbCount * c.smallObjectSize + fsSize := fsCount * c.maxObjectSize + c.metrics.SetEstimateSize(dbSize, fsSize) + c.metrics.SetActualCounters(dbCount, fsCount) + return dbSize + fsSize } func (c *cache) incSizeDB(sz uint64) uint64 { @@ -26,6 +30,8 @@ func (c *cache) incSizeFS(sz uint64) uint64 { return sz + c.maxObjectSize } +var _ fstree.FileCounter = &counters{} + type counters struct { cDB, cFS atomic.Uint64 } @@ -38,60 +44,33 @@ func (x *counters) FS() uint64 { return x.cFS.Load() } -func (c *cache) setCounters() error { +// Set implements fstree.ObjectCounter. +func (x *counters) Set(v uint64) { + x.cFS.Store(v) +} + +// Inc implements fstree.ObjectCounter. +func (x *counters) Inc() { + x.cFS.Add(1) +} + +// Dec implements fstree.ObjectCounter. +func (x *counters) Dec() { + x.cFS.Add(math.MaxUint64) +} + +func (c *cache) initCounters() error { var inDB uint64 - var inFS uint64 - - var eg errgroup.Group - - eg.Go(func() error { - err := c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - if b != nil { - inDB = uint64(b.Stats().KeyN) - } - return nil - }) - if err != nil { - return fmt.Errorf("could not read write-cache DB counter: %w", err) + err := c.db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(defaultBucket) + if b != nil { + inDB = uint64(b.Stats().KeyN) } - c.objCounters.cDB.Store(inDB) return nil }) - - eg.Go(func() error { - var err error - inFS, err = c.fsTree.NumberOfObjects() - if err != nil { - return fmt.Errorf("could not read write-cache FS counter: %w", err) - } - if inFS > 0 { - inFS-- //small.bolt DB file - } - c.objCounters.cFS.Store(inFS) - return nil - }) - if err := eg.Wait(); err != nil { - return err + if err != nil { + return fmt.Errorf("could not read write-cache DB counter: %w", err) } - c.metrics.SetActualCounters(inDB, inFS) + c.objCounters.cDB.Store(inDB) return nil } - -func (c *cache) runDBCounterLoop() { - go func() { - t := time.NewTicker(time.Second * 30) - defer t.Stop() - for { - select { - case <-t.C: - err := c.setCounters() - if err != nil { - c.log.Warn(logs.FailedToCountWritecacheItems, zap.Error(err)) - } - case <-c.closeCh: - return - } - } - }() -} diff --git a/pkg/local_object_storage/writecache/writecachebbolt/storage.go b/pkg/local_object_storage/writecache/writecachebbolt/storage.go index c651b7445..d79eb2963 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/storage.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/storage.go @@ -54,10 +54,15 @@ func (c *cache) openStore(readOnly bool) error { fstree.WithPerm(os.ModePerm), fstree.WithDepth(1), fstree.WithDirNameLen(1), - fstree.WithNoSync(c.noSync)) + fstree.WithNoSync(c.noSync), + fstree.WithFileCounter(&c.objCounters), + ) if err := c.fsTree.Open(readOnly); err != nil { return fmt.Errorf("could not open FSTree: %w", err) } + if err := c.fsTree.Init(); err != nil { + return fmt.Errorf("could not init FSTree: %w", err) + } return nil } diff --git a/pkg/util/sync/key_locker.go b/pkg/util/sync/key_locker.go new file mode 100644 index 000000000..97de0386d --- /dev/null +++ b/pkg/util/sync/key_locker.go @@ -0,0 +1,56 @@ +package sync + +import "sync" + +type locker struct { + mtx sync.Mutex + waiters int // not protected by mtx, must used outer mutex to update concurrently +} + +type KeyLocker[K comparable] struct { + lockers map[K]*locker + lockersMtx sync.Mutex +} + +func NewKeyLocker[K comparable]() *KeyLocker[K] { + return &KeyLocker[K]{ + lockers: make(map[K]*locker), + } +} + +func (l *KeyLocker[K]) Lock(key K) { + l.lockersMtx.Lock() + + if locker, found := l.lockers[key]; found { + locker.waiters++ + l.lockersMtx.Unlock() + + locker.mtx.Lock() + return + } + + locker := &locker{ + waiters: 1, + } + locker.mtx.Lock() + + l.lockers[key] = locker + l.lockersMtx.Unlock() +} + +func (l *KeyLocker[K]) Unlock(key K) { + l.lockersMtx.Lock() + defer l.lockersMtx.Unlock() + + locker, found := l.lockers[key] + if !found { + return + } + + if locker.waiters == 1 { + delete(l.lockers, key) + } + locker.waiters-- + + locker.mtx.Unlock() +} diff --git a/cmd/frostfs-node/cache_test.go b/pkg/util/sync/key_locker_test.go similarity index 82% rename from cmd/frostfs-node/cache_test.go rename to pkg/util/sync/key_locker_test.go index a3e1c4ea6..3b3e6a694 100644 --- a/cmd/frostfs-node/cache_test.go +++ b/pkg/util/sync/key_locker_test.go @@ -1,4 +1,4 @@ -package main +package sync import ( "context" @@ -12,11 +12,11 @@ import ( func TestKeyLocker(t *testing.T) { taken := false eg, _ := errgroup.WithContext(context.Background()) - keyLocker := newKeyLocker[int]() + keyLocker := NewKeyLocker[int]() for i := 0; i < 100; i++ { eg.Go(func() error { - keyLocker.LockKey(0) - defer keyLocker.UnlockKey(0) + keyLocker.Lock(0) + defer keyLocker.Unlock(0) require.False(t, taken) taken = true From 2efe9cc1beec5df36727a1cd5562a86b77f79eb8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 11 Aug 2023 12:51:41 +0300 Subject: [PATCH 0513/1943] [#585] writecache: Fix DB counter Signed-off-by: Dmitrii Stepanov --- .../writecache/writecachebbolt/delete.go | 12 ++++++++++-- .../writecache/writecachebbolt/put.go | 13 ++++++++++++- .../writecache/writecachebbolt/storage.go | 11 +++++++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/delete.go b/pkg/local_object_storage/writecache/writecachebbolt/delete.go index 105526c27..e4d9c219f 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/delete.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/delete.go @@ -2,6 +2,7 @@ package writecachebbolt import ( "context" + "math" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -49,9 +50,12 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { if dataSize > 0 { storageType = writecache.StorageTypeDB + var recordDeleted bool err := c.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) - err := b.Delete([]byte(saddr)) + key := []byte(saddr) + recordDeleted = b.Get(key) != nil + err := b.Delete(key) return err }) if err != nil { @@ -62,6 +66,10 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("db DELETE"), ) + if recordDeleted { + c.objCounters.cDB.Add(math.MaxUint64) + c.estimateCacheSize() + } deleted = true return nil } @@ -75,7 +83,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.OpField("fstree DELETE"), ) deleted = true + c.estimateCacheSize() } - return metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/put.go b/pkg/local_object_storage/writecache/writecachebbolt/put.go index 645e0bf8c..1d9957219 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/put.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/put.go @@ -85,9 +85,15 @@ func (c *cache) putSmall(obj objectInfo) error { return ErrOutOfSpace } + var newRecord bool err := c.db.Batch(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) - return b.Put([]byte(obj.addr), obj.data) + key := []byte(obj.addr) + newRecord = b.Get(key) == nil + if newRecord { + return b.Put(key, obj.data) + } + return nil }) if err == nil { storagelog.Write(c.log, @@ -95,6 +101,10 @@ func (c *cache) putSmall(obj objectInfo) error { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("db PUT"), ) + if newRecord { + c.objCounters.cDB.Add(1) + c.estimateCacheSize() + } } return err } @@ -121,6 +131,7 @@ func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) erro storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree PUT"), ) + c.estimateCacheSize() return nil } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/storage.go b/pkg/local_object_storage/writecache/writecachebbolt/storage.go index d79eb2963..bbd66d57c 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/storage.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/storage.go @@ -3,6 +3,7 @@ package writecachebbolt import ( "context" "fmt" + "math" "os" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -68,9 +69,12 @@ func (c *cache) openStore(readOnly bool) error { } func (c *cache) deleteFromDB(key string) { + var recordDeleted bool err := c.db.Batch(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) - return b.Delete([]byte(key)) + key := []byte(key) + recordDeleted = !recordDeleted && b.Get(key) != nil + return b.Delete(key) }) if err == nil { @@ -80,7 +84,10 @@ func (c *cache) deleteFromDB(key string) { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("db DELETE"), ) - c.estimateCacheSize() + if recordDeleted { + c.objCounters.cDB.Add(math.MaxUint64) + c.estimateCacheSize() + } } else { c.log.Error(logs.WritecacheCantRemoveObjectsFromTheDatabase, zap.Error(err)) } From 5335e7089ea5d9b04e0992cab3355bd75a22646b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 16 Aug 2023 12:28:19 +0300 Subject: [PATCH 0514/1943] [#615] pilorama: Speedup TestForest_ApplyRandom() Some of our pilorama tests fail on CI. The reasons are not obvious, but one possible improvement is using `WithNoSync` option for these. It should have much effect, because we are writing on the tmpfs, but doesn't hurt anyway. If I replace `t.TempDir()` with a local directory, test execution time goes down from 5s (sync) to 0.4s (nosync), which is the same time as with `t.TempDir()`. Maybe we have some strange CI configuration. ``` panic: test timed out after 10m0s running tests: TestForest_ApplyRandom (8m22s) TestForest_ApplyRandom/bbolt (8m21s) ... goroutine 170 [syscall]: syscall.Syscall(0xc000100000?, 0xc00047b758?, 0x6aff9a?, 0xc00041c1b0?) /opt/hostedtoolcache/go/1.20.7/x64/src/syscall/syscall_linux.go:69 +0x27 syscall.Fdatasync(0x9e35c0?) /opt/hostedtoolcache/go/1.20.7/x64/src/syscall/zsyscall_linux_amd64.go:418 +0x2a go.etcd.io/bbolt.fdatasync(0xc000189000?) ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/forest_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 0e2dd5ccb..6f657873a 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -825,7 +825,7 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ cid := cidtest.ID() treeID := "version" - expected := constructor(t) + expected := constructor(t, WithNoSync(true)) for i := range ops { require.NoError(t, expected.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } @@ -834,7 +834,7 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ // Shuffle random operations, leave initialization in place. r.Shuffle(len(ops), func(i, j int) { ops[i], ops[j] = ops[j], ops[i] }) - actual := constructor(t, WithMaxBatchSize(batchSize)) + actual := constructor(t, WithMaxBatchSize(batchSize), WithNoSync(true)) wg := new(sync.WaitGroup) ch := make(chan *Move) for i := 0; i < batchSize; i++ { @@ -870,7 +870,7 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ .. cid := cidtest.ID() treeID := "version" - expected := constructor(t) + expected := constructor(t, WithNoSync(true)) for i := range ops { require.NoError(t, expected.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } @@ -880,7 +880,7 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ .. // Shuffle random operations, leave initialization in place. r.Shuffle(len(ops), func(i, j int) { ops[i], ops[j] = ops[j], ops[i] }) - actual := constructor(t) + actual := constructor(t, WithNoSync(true)) for i := range ops { require.NoError(t, actual.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } From 6407bb5bd1ef138501ece8dcac80f2ba8d884181 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 17 Aug 2023 11:27:20 +0300 Subject: [PATCH 0515/1943] [#619] node: Fix object put when copies numbers contains only zeros In this case object should placement according to replicas. Signed-off-by: Anton Nikiforov --- .../object_manager/placement/traverser.go | 16 +++++++++++----- .../object_manager/placement/traverser_test.go | 12 +++++++++++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index c59146e2b..ad1077047 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -97,16 +97,22 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } else { rem = defaultCopiesVector(cfg.policy) - // compatibleZeroVector is a bool flag which is set when cfg.copyNumbers - // is [0]. In this case we should not modify `rem` slice unless track - // copies are ignored, because [0] means that all copies should be + // Bool flag which is set when cfg.copyNumbers contains not only zeros. + // In this case we should not modify `rem` slice unless track + // copies are ignored, because [0, ...] means that all copies should be // stored before returning OK to the client. - compatibleZeroVector := len(cfg.copyNumbers) == 1 && cfg.copyNumbers[0] == 0 + var considerCopiesNumber bool + for _, val := range cfg.copyNumbers { + if val != 0 { + considerCopiesNumber = true + break + } + } for i := range rem { if !cfg.trackCopies { rem[i] = -1 - } else if len(cfg.copyNumbers) > i && !compatibleZeroVector { + } else if considerCopiesNumber && len(cfg.copyNumbers) > i { rem[i] = int(cfg.copyNumbers[i]) } } diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index 1b307da6f..e16f33759 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -227,10 +227,20 @@ func TestTraverserRemValues(t *testing.T) { expectedRem: replicas, }, { - name: "compatible zero copy numbers", + name: "compatible zero copy numbers, len 1", copyNumbers: []uint32{0}, expectedRem: replicas, }, + { + name: "compatible zero copy numbers, len 2", + copyNumbers: []uint32{0, 0}, + expectedRem: replicas, + }, + { + name: "compatible zero copy numbers, len 3", + copyNumbers: []uint32{0, 0, 0}, + expectedRem: replicas, + }, { name: "copy numbers for all replicas", copyNumbers: []uint32{1, 1, 1}, From 5a51b78946e277519a035a83a7985f77eb9b0246 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 17 Aug 2023 14:24:38 +0300 Subject: [PATCH 0516/1943] [#620] object: Send status response for server-side streams Previously status responses were wrapped in the gRPC error and thus couldn't be correctly handled on client. Introduced in c2617baf63, thanks @ale64bit for having found. Signed-off-by: Evgenii Stratonikov --- pkg/services/object/sign.go | 40 +++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index f4f40dd7c..faad26489 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -49,7 +49,11 @@ func NewSignService(key *ecdsa.PrivateKey, svc ServiceServer) *SignService { } func (s *getStreamSigner) Send(resp *object.GetResponse) error { - if err := s.sigSvc.SignResponse(resp, nil); err != nil { + return s.send(resp, nil) +} + +func (s *getStreamSigner) send(resp *object.GetResponse, err error) error { + if err := s.sigSvc.SignResponse(resp, err); err != nil { return err } return s.GetObjectStream.Send(resp) @@ -62,10 +66,14 @@ func (s *SignService) Get(req *object.GetRequest, stream GetObjectStream) error return stream.Send(resp) } - return s.svc.Get(req, &getStreamSigner{ + w := &getStreamSigner{ GetObjectStream: stream, sigSvc: s.sigSvc, - }) + } + if err := s.svc.Get(req, w); err != nil { + return w.send(new(object.GetResponse), err) + } + return nil } func (s *putStreamSigner) Send(ctx context.Context, req *object.PutRequest) error { @@ -124,7 +132,11 @@ func (s *SignService) PutSingle(ctx context.Context, req *object.PutSingleReques func (s *searchStreamSigner) Send(resp *object.SearchResponse) error { s.nonEmptyResp = true - if err := s.sigSvc.SignResponse(resp, nil); err != nil { + return s.send(resp, nil) +} + +func (s *searchStreamSigner) send(resp *object.SearchResponse, err error) error { + if err := s.sigSvc.SignResponse(resp, err); err != nil { return err } return s.SearchStream.Send(resp) @@ -142,15 +154,15 @@ func (s *SignService) Search(req *object.SearchRequest, stream SearchStream) err sigSvc: s.sigSvc, } err := s.svc.Search(req, ss) - if err == nil && !ss.nonEmptyResp { + if err != nil || !ss.nonEmptyResp { // The higher component does not write any response in the case of an empty result (which is correct). // With the introduction of status returns at least one answer must be signed and sent to the client. // This approach is supported by clients who do not know how to work with statuses (one could make // a switch according to the protocol version from the request, but the costs of sending an empty // answer can be neglected due to the gradual refusal to use the "old" clients). - return stream.Send(new(object.SearchResponse)) + return ss.send(new(object.SearchResponse), err) } - return err + return nil } func (s *SignService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { @@ -163,7 +175,11 @@ func (s *SignService) Delete(ctx context.Context, req *object.DeleteRequest) (*o } func (s *getRangeStreamSigner) Send(resp *object.GetRangeResponse) error { - if err := s.sigSvc.SignResponse(resp, nil); err != nil { + return s.send(resp, nil) +} + +func (s *getRangeStreamSigner) send(resp *object.GetRangeResponse, err error) error { + if err := s.sigSvc.SignResponse(resp, err); err != nil { return err } return s.GetObjectRangeStream.Send(resp) @@ -176,10 +192,14 @@ func (s *SignService) GetRange(req *object.GetRangeRequest, stream GetObjectRang return stream.Send(resp) } - return s.svc.GetRange(req, &getRangeStreamSigner{ + w := &getRangeStreamSigner{ GetObjectRangeStream: stream, sigSvc: s.sigSvc, - }) + } + if err := s.svc.GetRange(req, w); err != nil { + return w.send(new(object.GetRangeResponse), err) + } + return nil } func (s *SignService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { From a8de37c8a279a1a086ff02e49187f4d31475c76b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 14 Aug 2023 14:06:17 +0300 Subject: [PATCH 0517/1943] [#607] *: Remove redundant if on error returns Semantic patch: ``` @@ @@ -if err != nil { return err } -return nil +return err ``` Signed-off-by: Evgenii Stratonikov --- pkg/innerring/initialization.go | 12 ++---------- .../blobstor/compression/compress.go | 6 +----- pkg/local_object_storage/pilorama/boltdb.go | 5 +---- pkg/morph/client/notifications.go | 6 +----- pkg/morph/event/notary_preparator.go | 6 +----- pkg/services/notificator/nats/service.go | 6 +----- pkg/services/session/storage/persistent/storage.go | 6 +----- 7 files changed, 8 insertions(+), 39 deletions(-) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 52ffb10a1..84112d121 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -246,11 +246,7 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { } err = bindMorphProcessor(s.alphabetProcessor, s) - if err != nil { - return err - } - - return nil + return err } func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.Client, @@ -454,11 +450,7 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien } err = s.initAlphabetProcessor(cfg) - if err != nil { - return err - } - - return nil + return err } func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- error) (*chainParams, error) { diff --git a/pkg/local_object_storage/blobstor/compression/compress.go b/pkg/local_object_storage/blobstor/compression/compress.go index 4ebf057d3..270c2b18d 100644 --- a/pkg/local_object_storage/blobstor/compression/compress.go +++ b/pkg/local_object_storage/blobstor/compression/compress.go @@ -33,11 +33,7 @@ func (c *Config) Init() error { } c.decoder, err = zstd.NewReader(nil) - if err != nil { - return err - } - - return nil + return err } // NeedsCompression returns true if the object should be compressed. diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index a729e2a22..33d2b6878 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -146,10 +146,7 @@ func (t *boltForest) Init() error { return err } _, err = tx.CreateBucketIfNotExists(logBucket) - if err != nil { - return err - } - return nil + return err }) } func (t *boltForest) Close() error { diff --git a/pkg/morph/client/notifications.go b/pkg/morph/client/notifications.go index dbca00d7c..121dccfb7 100644 --- a/pkg/morph/client/notifications.go +++ b/pkg/morph/client/notifications.go @@ -101,9 +101,5 @@ func (c *Client) UnsubscribeAll() error { } err := c.client.UnsubscribeAll() - if err != nil { - return err - } - - return nil + return err } diff --git a/pkg/morph/event/notary_preparator.go b/pkg/morph/event/notary_preparator.go index 298a6d574..37091f768 100644 --- a/pkg/morph/event/notary_preparator.go +++ b/pkg/morph/event/notary_preparator.go @@ -243,11 +243,7 @@ func (p Preparator) validateParameterOpcodes(ops []Op) error { } err = validateNestedArgs(argsLen, ops[:l-2]) - if err != nil { - return err - } - - return nil + return err } func validateNestedArgs(expArgLen int64, ops []Op) error { diff --git a/pkg/services/notificator/nats/service.go b/pkg/services/notificator/nats/service.go index 19f0cbde1..7f525df95 100644 --- a/pkg/services/notificator/nats/service.go +++ b/pkg/services/notificator/nats/service.go @@ -74,11 +74,7 @@ func (n *Writer) Notify(topic string, address oid.Address) error { } _, err := n.js.Publish(topic, []byte(address.EncodeToString()), nats.MsgId(messageID)) - if err != nil { - return err - } - - return nil + return err } // New creates new Writer. diff --git a/pkg/services/session/storage/persistent/storage.go b/pkg/services/session/storage/persistent/storage.go index 25f067d62..e4c2091c0 100644 --- a/pkg/services/session/storage/persistent/storage.go +++ b/pkg/services/session/storage/persistent/storage.go @@ -102,11 +102,7 @@ func (s *TokenStore) Get(ownerID user.ID, tokenID []byte) (t *storage.PrivateTok var err error t, err = s.unpackToken(rawToken) - if err != nil { - return err - } - - return nil + return err }) if err != nil { s.l.Error(logs.PersistentCouldNotGetSessionFromPersistentStorage, From e604a3d749be413a35e2cb44d5ee29b33eb9db66 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 14 Aug 2023 14:08:18 +0300 Subject: [PATCH 0518/1943] [#607] *: Use zap.Stringer() where possible Semantic patch: ``` @@ var f expression var t expression var a expression @@ f( ..., - zap.String(t, a.String()), + zap.Stringer(t, a), ..., ) ``` Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/grpc.go | 4 ++-- pkg/innerring/processors/frostfs/process_assets.go | 2 +- pkg/local_object_storage/engine/remove_copies.go | 2 +- pkg/local_object_storage/shard/id.go | 2 +- pkg/services/tree/signature.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index 10d387b13..42059ec09 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -104,14 +104,14 @@ func serveGRPC(c *cfg) { go func() { defer func() { c.log.Info(logs.FrostFSNodeStopListeningGRPCEndpoint, - zap.String("endpoint", lis.Addr().String()), + zap.Stringer("endpoint", lis.Addr()), ) c.wg.Done() }() c.log.Info(logs.FrostFSNodeStartListeningGRPCEndpoint, - zap.String("endpoint", lis.Addr().String()), + zap.Stringer("endpoint", lis.Addr()), ) if err := srv.Serve(lis); err != nil { diff --git a/pkg/innerring/processors/frostfs/process_assets.go b/pkg/innerring/processors/frostfs/process_assets.go index 327a4a3aa..c72aeceee 100644 --- a/pkg/innerring/processors/frostfs/process_assets.go +++ b/pkg/innerring/processors/frostfs/process_assets.go @@ -45,7 +45,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { val, ok := np.mintEmitCache.Get(receiver.String()) if ok && val+np.mintEmitThreshold >= curEpoch { np.log.Warn(logs.FrostFSDoubleMintEmissionDeclined, - zap.String("receiver", receiver.String()), + zap.Stringer("receiver", receiver), zap.Uint64("last_emission", val), zap.Uint64("current_epoch", curEpoch)) diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index 4b48d179c..573153268 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -55,7 +55,7 @@ func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicat // This is not currently the case, because `FreeSpace` metric used by weight sorting is always 0. // However we could change weights in future and easily forget this function. for _, sh := range e.shards { - e.log.Debug(logs.EngineStartedDuplicatesRemovalRoutine, zap.String("shard_id", sh.ID().String())) + e.log.Debug(logs.EngineStartedDuplicatesRemovalRoutine, zap.Stringer("shard_id", sh.ID())) ch := make(chan oid.Address) errG, ctx := errgroup.WithContext(ctx) diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index e2ac423fd..49f9a3043 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -49,7 +49,7 @@ func (s *Shard) UpdateID() (err error) { } } - s.log = &logger.Logger{Logger: s.log.With(zap.String("shard_id", s.info.ID.String()))} + s.log = &logger.Logger{Logger: s.log.With(zap.Stringer("shard_id", s.info.ID))} s.metaBase.SetLogger(s.log) s.blobStor.SetLogger(s.log) if s.hasWriteCache() { diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 7a466955c..6dad22774 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -89,7 +89,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op if !basicACL.AllowedBearerRules(op) { s.log.Debug(logs.TreeBearerPresentedButNotAllowedByACL, zap.String("cid", cid.EncodeToString()), - zap.String("op", op.String()), + zap.Stringer("op", op), ) } else { useBearer = true From 127c6767866eb1bfe273aed41e52b7843e9be2fe Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 14 Aug 2023 14:17:45 +0300 Subject: [PATCH 0519/1943] [#607] *: Use keys.PublicKeys.Copy() where possible Semantic patch: ``` @@ var dst identifier var src identifier var keys identifier @@ import keys "github.com/nspcc-dev/neo-go/pkg/crypto/keys" -dst := make(keys.PublicKeys, len(src)) -copy(dst, src) +dst := src.Copy() ``` Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/generate.go | 3 +-- pkg/innerring/processors/governance/list_test.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index ccdc4519f..55321582b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -104,8 +104,7 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er bftCount := smartcontract.GetDefaultHonestNodeCount(size) for i := range wallets { i := i - ps := make(keys.PublicKeys, len(pubs)) - copy(ps, pubs) + ps := pubs.Copy() errG.Go(func() error { if err := addMultisigAccount(wallets[i], majCount, committeeAccountName, passwords[i], ps); err != nil { return fmt.Errorf("can't create committee account: %w", err) diff --git a/pkg/innerring/processors/governance/list_test.go b/pkg/innerring/processors/governance/list_test.go index e0121ab62..d099ec837 100644 --- a/pkg/innerring/processors/governance/list_test.go +++ b/pkg/innerring/processors/governance/list_test.go @@ -60,8 +60,7 @@ func TestNewAlphabetList(t *testing.T) { orig := keys.PublicKeys{k[1], k[2], k[3], k[4]} main := keys.PublicKeys{k[1], k[2], k[5], k[4]} - exp := make(keys.PublicKeys, len(main)) - copy(exp, main) + exp := main.Copy() sort.Sort(exp) got, err := newAlphabetList(orig, main) From f7042c5a6f680073c44e85aea33f99609767fa72 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 17 Aug 2023 15:49:24 +0300 Subject: [PATCH 0520/1943] [#609] Replace zaptest.NewLogger() with zap.L() Semantic patch: ``` @@ @@ -import "go.uber.org/zap/zaptest" +import "go.uber.org/zap" -zaptest.NewLogger(t) +zap.L() ``` Signed-off-by: Evgenii Stratonikov --- .../blobstor/blobovniczatree/exists_test.go | 4 ++-- .../blobstor/blobovniczatree/generic_test.go | 6 +++--- .../blobstor/memstore/memstore_test.go | 4 ++-- pkg/local_object_storage/engine/control_test.go | 3 +-- pkg/local_object_storage/engine/delete_test.go | 4 ++-- pkg/local_object_storage/engine/engine_test.go | 3 +-- pkg/local_object_storage/engine/error_test.go | 4 ++-- pkg/local_object_storage/engine/evacuate_test.go | 4 ++-- pkg/local_object_storage/shard/control_test.go | 4 ++-- pkg/local_object_storage/shard/gc_internal_test.go | 8 ++++---- pkg/local_object_storage/shard/range_test.go | 4 ++-- pkg/local_object_storage/shard/reload_test.go | 4 ++-- pkg/local_object_storage/shard/shard_test.go | 5 ++--- .../writecache/writecachebadger/flush_test.go | 4 ++-- .../writecache/writecachebadger/generic_test.go | 4 ++-- .../writecache/writecachebbolt/flush_test.go | 4 ++-- .../writecache/writecachebbolt/generic_test.go | 4 ++-- pkg/services/tree/signature_test.go | 4 ++-- 18 files changed, 37 insertions(+), 40 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 8d9fe526e..f9448ae92 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -12,13 +12,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/util/slice" "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func TestExistsInvalidStorageID(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(&logger.Logger{Logger: zap.L()}), WithObjectSizeLimit(1024), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go index 7f95e750b..5a713b9b9 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func TestGeneric(t *testing.T) { @@ -14,7 +14,7 @@ func TestGeneric(t *testing.T) { helper := func(t *testing.T, dir string) common.Storage { return NewBlobovniczaTree( - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(&logger.Logger{Logger: zap.L()}), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), @@ -41,7 +41,7 @@ func TestControl(t *testing.T) { newTree := func(t *testing.T) common.Storage { return NewBlobovniczaTree( - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(&logger.Logger{Logger: zap.L()}), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index 125276290..7979b1504 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -10,13 +10,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func TestSimpleLifecycle(t *testing.T) { s := New( WithRootPath("memstore"), - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(&logger.Logger{Logger: zap.L()}), ) t.Cleanup(func() { _ = s.Close() }) require.NoError(t, s.Open(false)) diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 804bef6ec..04591687d 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -27,7 +27,6 @@ import ( "github.com/stretchr/testify/require" "go.etcd.io/bbolt" "go.uber.org/zap" - "go.uber.org/zap/zaptest" ) // TestInitializationFailure checks that shard is initialized and closed even if media @@ -55,7 +54,7 @@ func TestInitializationFailure(t *testing.T) { return []shard.Option{ shard.WithID(sid), - shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + shard.WithLogger(&logger.Logger{Logger: zap.L()}), shard.WithBlobStorOptions( blobstor.WithStorages(storages)), shard.WithMetaBaseOptions( diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index e4a516512..802b6b0b5 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -13,7 +13,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func TestDeleteBigObject(t *testing.T) { @@ -54,7 +54,7 @@ func TestDeleteBigObject(t *testing.T) { s3 := testNewShard(t, 3) e := testNewEngine(t).setInitializedShards(t, s1, s2, s3).engine - e.log = &logger.Logger{Logger: zaptest.NewLogger(t)} + e.log = &logger.Logger{Logger: zap.L()} defer e.Close() for i := range children { diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index e3dd49728..39c5e2e1e 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -23,7 +23,6 @@ import ( "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" "go.uber.org/zap" - "go.uber.org/zap/zaptest" ) type epochState struct{} @@ -81,7 +80,7 @@ type testEngineWrapper struct { } func testNewEngine(t testing.TB, opts ...Option) *testEngineWrapper { - engine := New(WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)})) + engine := New(WithLogger(&logger.Logger{Logger: zap.L()})) for _, opt := range opts { opt(engine.cfg) } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index a77149e7c..bbed8cfb9 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -21,7 +21,7 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) const errSmallSize = 256 @@ -56,7 +56,7 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) largeFileStorage: largeFileStorage, } return []shard.Option{ - shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + shard.WithLogger(&logger.Logger{Logger: zap.L()}), shard.WithBlobStorOptions(blobstor.WithStorages(storages)), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", id))), diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index b642ea065..63296ccd7 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -21,7 +21,7 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" "golang.org/x/sync/errgroup" ) @@ -31,7 +31,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng te := testNewEngine(t). setShardsNumOpts(t, shardNum, func(id int) []shard.Option { return []shard.Option{ - shard.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + shard.WithLogger(&logger.Logger{Logger: zap.L()}), shard.WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{{ Storage: fstree.New( diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index deeb39300..053d89a8a 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -30,7 +30,7 @@ import ( objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) type epochState struct{} @@ -75,7 +75,7 @@ func TestShardOpen(t *testing.T) { newShard := func() *Shard { return New( WithID(NewIDFromBytes([]byte{})), - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(&logger.Logger{Logger: zap.L()}), WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{ {Storage: st}, diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index cf6633962..21f300e2c 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -22,7 +22,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { @@ -32,13 +32,13 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { var sh *Shard - l := &logger.Logger{Logger: zaptest.NewLogger(t)} + l := &logger.Logger{Logger: zap.L()} blobOpts := []blobstor.Option{ - blobstor.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + blobstor.WithLogger(&logger.Logger{Logger: zap.L()}), blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + blobovniczatree.WithLogger(&logger.Logger{Logger: zap.L()}), blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 1806990ab..e764a0763 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -19,7 +19,7 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/nspcc-dev/neo-go/pkg/util/slice" "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func TestShard_GetRange(t *testing.T) { @@ -82,7 +82,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { []blobstor.Option{blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + blobovniczatree.WithLogger(&logger.Logger{Logger: zap.L()}), blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 0b964ba2e..75ce386bc 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -20,7 +20,7 @@ import ( usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func TestShardReload(t *testing.T) { @@ -29,7 +29,7 @@ func TestShardReload(t *testing.T) { p := t.Name() defer os.RemoveAll(p) - l := &logger.Logger{Logger: zaptest.NewLogger(t)} + l := &logger.Logger{Logger: zap.L()} blobOpts := []blobstor.Option{ blobstor.WithLogger(l), blobstor.WithStorages([]blobstor.SubStorage{ diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 3b7c7c4a3..4ec73191c 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -22,7 +22,6 @@ import ( "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" "go.uber.org/zap" - "go.uber.org/zap/zaptest" ) type epochState struct { @@ -60,11 +59,11 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts if bsOpts == nil { bsOpts = []blobstor.Option{ - blobstor.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + blobstor.WithLogger(&logger.Logger{Logger: zap.L()}), blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + blobovniczatree.WithLogger(&logger.Logger{Logger: zap.L()}), blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go index 90e584e02..1bf50e000 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go @@ -14,14 +14,14 @@ import ( oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func TestFlush(t *testing.T) { createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { return New( append([]Option{ - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(&logger.Logger{Logger: zap.L()}), WithPath(filepath.Join(t.TempDir(), "writecache")), WithMetabase(mb), WithBlobstor(bs), diff --git a/pkg/local_object_storage/writecache/writecachebadger/generic_test.go b/pkg/local_object_storage/writecache/writecachebadger/generic_test.go index be0a40e00..641da2293 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/generic_test.go +++ b/pkg/local_object_storage/writecache/writecachebadger/generic_test.go @@ -6,13 +6,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func TestGeneric(t *testing.T) { storagetest.TestAll(t, func(t *testing.T) storagetest.Component { return New( - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(&logger.Logger{Logger: zap.L()}), WithFlushWorkersCount(2), WithPath(t.TempDir()), WithGCInterval(1*time.Second)) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go index acd180130..e7ba66fae 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go @@ -17,14 +17,14 @@ import ( oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func TestFlush(t *testing.T) { createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { return New( append([]Option{ - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(&logger.Logger{Logger: zap.L()}), WithPath(filepath.Join(t.TempDir(), "writecache")), WithSmallObjectSize(smallSize), WithMetabase(mb), diff --git a/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go b/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go index 509efdd60..b34d53f3e 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go @@ -5,13 +5,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) func TestGeneric(t *testing.T) { storagetest.TestAll(t, func(t *testing.T) storagetest.Component { return New( - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(&logger.Logger{Logger: zap.L()}), WithFlushWorkersCount(2), WithPath(t.TempDir())) }) diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index eaf9b8b79..482c16157 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -20,7 +20,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" ) type dummyNetmapSource struct { @@ -97,7 +97,7 @@ func TestMessageSign(t *testing.T) { s := &Service{ cfg: cfg{ - log: &logger.Logger{Logger: zaptest.NewLogger(t)}, + log: &logger.Logger{Logger: zap.L()}, key: &privs[0].PrivateKey, nmSource: dummyNetmapSource{}, cnrSource: dummyContainerSource{ From 2e49d7ea7ee96b76629c0ae5bf6fd548e5d9aa8a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 15 Aug 2023 16:53:38 +0300 Subject: [PATCH 0521/1943] [#602] blobovnicza: Init before using Fix blobovnicza size: after restart size metric resets. Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/control.go | 34 +++++++++---------- .../blobstor/blobovniczatree/control.go | 3 ++ 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index da4c870bd..79bad91e2 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -45,8 +45,6 @@ func (b *Blobovnicza) Open() error { // Init initializes internal database structure. // // If Blobovnicza is already initialized, no action is taken. -// -// Should not be called in read-only configuration. func (b *Blobovnicza) Init() error { b.log.Debug(logs.BlobovniczaInitializing, zap.Uint64("object size limit", b.objSizeLimit), @@ -58,25 +56,27 @@ func (b *Blobovnicza) Init() error { return nil } - err := b.boltDB.Update(func(tx *bbolt.Tx) error { - return b.iterateBucketKeys(func(lower, upper uint64, key []byte) (bool, error) { - // create size range bucket + if !b.boltOptions.ReadOnly { + err := b.boltDB.Update(func(tx *bbolt.Tx) error { + return b.iterateBucketKeys(func(lower, upper uint64, key []byte) (bool, error) { + // create size range bucket - rangeStr := stringifyBounds(lower, upper) - b.log.Debug(logs.BlobovniczaCreatingBucketForSizeRange, - zap.String("range", rangeStr)) + rangeStr := stringifyBounds(lower, upper) + b.log.Debug(logs.BlobovniczaCreatingBucketForSizeRange, + zap.String("range", rangeStr)) - _, err := tx.CreateBucketIfNotExists(key) - if err != nil { - return false, fmt.Errorf("(%T) could not create bucket for bounds %s: %w", - b, rangeStr, err) - } + _, err := tx.CreateBucketIfNotExists(key) + if err != nil { + return false, fmt.Errorf("(%T) could not create bucket for bounds %s: %w", + b, rangeStr, err) + } - return false, nil + return false, nil + }) }) - }) - if err != nil { - return err + if err != nil { + return err + } } info, err := os.Stat(b.path) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index bc3d7d60c..3a06310f6 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -136,5 +136,8 @@ func (b *Blobovniczas) openBlobovniczaNoCache(p string) (*blobovnicza.Blobovnicz if err := blz.Open(); err != nil { return nil, fmt.Errorf("could not open blobovnicza %s: %w", p, err) } + if err := blz.Init(); err != nil { + return nil, fmt.Errorf("could not init blobovnicza %s: %w", p, err) + } return blz, nil } From 809e97626b367d4ae054cf6e9846fa617812b183 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Aug 2023 11:12:19 +0300 Subject: [PATCH 0522/1943] [#602] blobovnicza: Fix size counter Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/blobovnicza.go | 6 +- .../blobovnicza/control.go | 67 +++++++++++++++---- .../blobovnicza/delete.go | 35 +++++----- .../blobovnicza/iterate.go | 25 ++++--- pkg/local_object_storage/blobovnicza/put.go | 4 +- pkg/local_object_storage/blobovnicza/sizes.go | 11 +-- .../blobovnicza/sizes_test.go | 3 +- 7 files changed, 99 insertions(+), 52 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index d5741fba7..22168de22 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -3,6 +3,7 @@ package blobovnicza import ( "io/fs" "os" + "sync" "sync/atomic" "time" @@ -15,9 +16,12 @@ import ( type Blobovnicza struct { cfg - filled atomic.Uint64 + dataSize atomic.Uint64 boltDB *bbolt.DB + + opened bool + controlMtx sync.Mutex } // Option is an option of Blobovnicza's constructor. diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index 79bad91e2..7b5552907 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -1,8 +1,8 @@ package blobovnicza import ( + "errors" "fmt" - "os" "path/filepath" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -14,7 +14,15 @@ import ( // Open opens an internal database at the configured path with the configured permissions. // // If the database file does not exist, it will be created automatically. +// If blobovnizca is already open, does nothing. func (b *Blobovnicza) Open() error { + b.controlMtx.Lock() + defer b.controlMtx.Unlock() + + if b.opened { + return nil + } + b.log.Debug(logs.BlobovniczaCreatingDirectoryForBoltDB, zap.String("path", b.path), zap.Bool("ro", b.boltOptions.ReadOnly), @@ -36,6 +44,7 @@ func (b *Blobovnicza) Open() error { b.boltDB, err = bbolt.Open(b.path, b.perm, b.boltOptions) if err == nil { + b.opened = true b.metrics.IncOpenBlobovnizcaCount() } @@ -45,20 +54,28 @@ func (b *Blobovnicza) Open() error { // Init initializes internal database structure. // // If Blobovnicza is already initialized, no action is taken. +// Blobovnizca must be open, otherwise an error will return. func (b *Blobovnicza) Init() error { + b.controlMtx.Lock() + defer b.controlMtx.Unlock() + + if !b.opened { + return errors.New("blobovnizca is not open") + } + b.log.Debug(logs.BlobovniczaInitializing, zap.Uint64("object size limit", b.objSizeLimit), zap.Uint64("storage size limit", b.fullSizeLimit), ) - if size := b.filled.Load(); size != 0 { + if size := b.dataSize.Load(); size != 0 { b.log.Debug(logs.BlobovniczaAlreadyInitialized, zap.Uint64("size", size)) return nil } if !b.boltOptions.ReadOnly { err := b.boltDB.Update(func(tx *bbolt.Tx) error { - return b.iterateBucketKeys(func(lower, upper uint64, key []byte) (bool, error) { + return b.iterateBucketKeys(true, func(lower, upper uint64, key []byte) (bool, error) { // create size range bucket rangeStr := stringifyBounds(lower, upper) @@ -79,27 +96,49 @@ func (b *Blobovnicza) Init() error { } } - info, err := os.Stat(b.path) + return b.initializeSize() +} + +func (b *Blobovnicza) initializeSize() error { + var size uint64 + err := b.boltDB.View(func(tx *bbolt.Tx) error { + return b.iterateAllBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) { + size += uint64(b.Stats().KeyN) * upper + return false, nil + }) + }) if err != nil { return fmt.Errorf("can't determine DB size: %w", err) } - - sz := uint64(info.Size()) - b.filled.Store(sz) - b.metrics.AddSize(sz) - return err + b.dataSize.Store(size) + b.metrics.AddSize(size) + return nil } // Close releases all internal database resources. +// +// If blobovnizca is already closed, does nothing. func (b *Blobovnicza) Close() error { + b.controlMtx.Lock() + defer b.controlMtx.Unlock() + + if !b.opened { + return nil + } + b.log.Debug(logs.BlobovniczaClosingBoltDB, zap.String("path", b.path), ) - err := b.boltDB.Close() - if err == nil { - b.metrics.DecOpenBlobovnizcaCount() - b.metrics.SubSize(b.filled.Load()) + if err := b.boltDB.Close(); err != nil { + return err } - return err + + b.metrics.DecOpenBlobovnizcaCount() + b.metrics.SubSize(b.dataSize.Load()) + b.dataSize.Store(0) + + b.opened = false + + return nil } diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 419e073c6..f332173c1 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -46,33 +46,22 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err addrKey := addressKey(prm.addr) found := false + var sizeUpperBound uint64 + var sizeLowerBound uint64 + var dataSize uint64 err := b.boltDB.Update(func(tx *bbolt.Tx) error { - return b.iterateBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { + return b.iterateAllBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { objData := buck.Get(addrKey) if objData == nil { // object is not in bucket => continue iterating return false, nil } - - sz := uint64(len(objData)) - - // remove object from the bucket - err := buck.Delete(addrKey) - - if err == nil { - b.log.Debug(logs.BlobovniczaObjectWasRemovedFromBucket, - zap.String("binary size", stringifyByteSize(sz)), - zap.String("range", stringifyBounds(lower, upper)), - ) - // decrease fullness counter - b.decSize(sz) - } - + dataSize = uint64(len(objData)) + sizeLowerBound = lower + sizeUpperBound = upper found = true - - // stop iteration - return true, err + return true, buck.Delete(addrKey) }) }) @@ -80,5 +69,13 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err return DeleteRes{}, new(apistatus.ObjectNotFound) } + if err == nil && found { + b.log.Debug(logs.BlobovniczaObjectWasRemovedFromBucket, + zap.String("binary size", stringifyByteSize(dataSize)), + zap.String("range", stringifyBounds(sizeLowerBound, sizeUpperBound)), + ) + b.decSize(sizeUpperBound) + } + return DeleteRes{}, err } diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index c2031ea54..b29ccb43c 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -3,6 +3,7 @@ package blobovnicza import ( "context" "fmt" + "math" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -11,28 +12,32 @@ import ( "go.opentelemetry.io/otel/trace" ) -func (b *Blobovnicza) iterateBuckets(tx *bbolt.Tx, f func(uint64, uint64, *bbolt.Bucket) (bool, error)) error { - return b.iterateBucketKeys(func(lower uint64, upper uint64, key []byte) (bool, error) { +// iterateAllBuckets iterates all buckets in db +// +// If the maximum size of the object (b.objSizeLimit) has been changed to lower value, +// then there may be more buckets than the current limit of the object size. +func (b *Blobovnicza) iterateAllBuckets(tx *bbolt.Tx, f func(uint64, uint64, *bbolt.Bucket) (bool, error)) error { + return b.iterateBucketKeys(false, func(lower uint64, upper uint64, key []byte) (bool, error) { buck := tx.Bucket(key) if buck == nil { - // expected to happen: - // - before initialization step (incorrect usage by design) - // - if DB is corrupted (in future this case should be handled) - return false, fmt.Errorf("(%T) could not get bucket %s", b, stringifyBounds(lower, upper)) + return true, nil } return f(lower, upper, buck) }) } -func (b *Blobovnicza) iterateBucketKeys(f func(uint64, uint64, []byte) (bool, error)) error { - return b.iterateBounds(func(lower, upper uint64) (bool, error) { +func (b *Blobovnicza) iterateBucketKeys(useObjLimitBound bool, f func(uint64, uint64, []byte) (bool, error)) error { + return b.iterateBounds(useObjLimitBound, func(lower, upper uint64) (bool, error) { return f(lower, upper, bucketKeyFromBounds(upper)) }) } -func (b *Blobovnicza) iterateBounds(f func(uint64, uint64) (bool, error)) error { - objLimitBound := upperPowerOfTwo(b.objSizeLimit) +func (b *Blobovnicza) iterateBounds(useObjLimitBound bool, f func(uint64, uint64) (bool, error)) error { + var objLimitBound uint64 = math.MaxUint64 + if useObjLimitBound { + objLimitBound = upperPowerOfTwo(b.objSizeLimit) + } for upper := firstBucketBound; upper <= max(objLimitBound, firstBucketBound); upper *= 2 { var lower uint64 diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index cc816b3e1..b8cc9954a 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -61,7 +61,7 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { defer span.End() sz := uint64(len(prm.objData)) - bucketName := bucketForSize(sz) + bucketName, upperBound := bucketForSize(sz) key := addressKey(prm.addr) err := b.boltDB.Batch(func(tx *bbolt.Tx) error { @@ -85,7 +85,7 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { return nil }) if err == nil { - b.incSize(sz) + b.incSize(upperBound) } return PutRes{}, err diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index 7e10b728e..482242c3f 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -28,8 +28,9 @@ func bucketKeyFromBounds(upperBound uint64) []byte { return buf[:ln] } -func bucketForSize(sz uint64) []byte { - return bucketKeyFromBounds(upperPowerOfTwo(sz)) +func bucketForSize(sz uint64) ([]byte, uint64) { + upperBound := upperPowerOfTwo(sz) + return bucketKeyFromBounds(upperBound), upperBound } func upperPowerOfTwo(v uint64) uint64 { @@ -40,15 +41,15 @@ func upperPowerOfTwo(v uint64) uint64 { } func (b *Blobovnicza) incSize(sz uint64) { - b.filled.Add(sz) + b.dataSize.Add(sz) b.metrics.AddSize(sz) } func (b *Blobovnicza) decSize(sz uint64) { - b.filled.Add(^(sz - 1)) + b.dataSize.Add(^(sz - 1)) b.metrics.SubSize(sz) } func (b *Blobovnicza) full() bool { - return b.filled.Load() >= b.fullSizeLimit + return b.dataSize.Load() >= b.fullSizeLimit } diff --git a/pkg/local_object_storage/blobovnicza/sizes_test.go b/pkg/local_object_storage/blobovnicza/sizes_test.go index 2bf451f12..d2f576fd3 100644 --- a/pkg/local_object_storage/blobovnicza/sizes_test.go +++ b/pkg/local_object_storage/blobovnicza/sizes_test.go @@ -34,7 +34,8 @@ func TestSizes(t *testing.T) { upperBound: 4 * firstBucketBound, }, } { - require.Equal(t, bucketKeyFromBounds(item.upperBound), bucketForSize(item.sz)) + key, _ := bucketForSize(item.sz) + require.Equal(t, bucketKeyFromBounds(item.upperBound), key) } } From 10e63537b27eeaf7dd9720e4b0d4492dbc84026d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Aug 2023 11:58:47 +0300 Subject: [PATCH 0523/1943] [#602] metrics: Rename blobovnicza size metric `Size` is not size, but open db size. Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/control.go | 4 +-- .../blobovnicza/metrics.go | 12 +++---- pkg/local_object_storage/blobovnicza/sizes.go | 4 +-- .../metrics/blobovnizca.go | 8 ++--- pkg/metrics/blobovnizca.go | 32 +++++++++---------- pkg/metrics/consts.go | 32 +++++++++---------- 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index 7b5552907..0e3204968 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -111,7 +111,7 @@ func (b *Blobovnicza) initializeSize() error { return fmt.Errorf("can't determine DB size: %w", err) } b.dataSize.Store(size) - b.metrics.AddSize(size) + b.metrics.AddOpenBlobovnizcaSize(size) return nil } @@ -135,7 +135,7 @@ func (b *Blobovnicza) Close() error { } b.metrics.DecOpenBlobovnizcaCount() - b.metrics.SubSize(b.dataSize.Load()) + b.metrics.SubOpenBlobovnizcaSize(b.dataSize.Load()) b.dataSize.Store(0) b.opened = false diff --git a/pkg/local_object_storage/blobovnicza/metrics.go b/pkg/local_object_storage/blobovnicza/metrics.go index d511f90f1..5f4f94cbe 100644 --- a/pkg/local_object_storage/blobovnicza/metrics.go +++ b/pkg/local_object_storage/blobovnicza/metrics.go @@ -4,13 +4,13 @@ type Metrics interface { IncOpenBlobovnizcaCount() DecOpenBlobovnizcaCount() - AddSize(size uint64) - SubSize(size uint64) + AddOpenBlobovnizcaSize(size uint64) + SubOpenBlobovnizcaSize(size uint64) } type NoopMetrics struct{} -func (m *NoopMetrics) IncOpenBlobovnizcaCount() {} -func (m *NoopMetrics) DecOpenBlobovnizcaCount() {} -func (m *NoopMetrics) AddSize(uint64) {} -func (m *NoopMetrics) SubSize(uint64) {} +func (m *NoopMetrics) IncOpenBlobovnizcaCount() {} +func (m *NoopMetrics) DecOpenBlobovnizcaCount() {} +func (m *NoopMetrics) AddOpenBlobovnizcaSize(uint64) {} +func (m *NoopMetrics) SubOpenBlobovnizcaSize(uint64) {} diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index 482242c3f..d4ac19201 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -42,12 +42,12 @@ func upperPowerOfTwo(v uint64) uint64 { func (b *Blobovnicza) incSize(sz uint64) { b.dataSize.Add(sz) - b.metrics.AddSize(sz) + b.metrics.AddOpenBlobovnizcaSize(sz) } func (b *Blobovnicza) decSize(sz uint64) { b.dataSize.Add(^(sz - 1)) - b.metrics.SubSize(sz) + b.metrics.SubOpenBlobovnizcaSize(sz) } func (b *Blobovnicza) full() bool { diff --git a/pkg/local_object_storage/metrics/blobovnizca.go b/pkg/local_object_storage/metrics/blobovnizca.go index a498d822e..72c45c31d 100644 --- a/pkg/local_object_storage/metrics/blobovnizca.go +++ b/pkg/local_object_storage/metrics/blobovnizca.go @@ -81,12 +81,12 @@ type blobovniczaMetrics struct { path string } -func (m *blobovniczaMetrics) AddSize(size uint64) { - m.m.AddTreeSize(m.shardID(), m.path, size) +func (m *blobovniczaMetrics) AddOpenBlobovnizcaSize(size uint64) { + m.m.AddOpenBlobovnizcaSize(m.shardID(), m.path, size) } -func (m *blobovniczaMetrics) SubSize(size uint64) { - m.m.SubTreeSize(m.shardID(), m.path, size) +func (m *blobovniczaMetrics) SubOpenBlobovnizcaSize(size uint64) { + m.m.SubOpenBlobovnizcaSize(m.shardID(), m.path, size) } func (m *blobovniczaMetrics) IncOpenBlobovnizcaCount() { diff --git a/pkg/metrics/blobovnizca.go b/pkg/metrics/blobovnizca.go index 9dc3ed572..b80ec5fc1 100644 --- a/pkg/metrics/blobovnizca.go +++ b/pkg/metrics/blobovnizca.go @@ -15,8 +15,8 @@ type BlobobvnizcaMetrics interface { AddBlobobvnizcaTreePut(shardID, path string, size int) AddBlobobvnizcaTreeGet(shardID, path string, size int) - AddTreeSize(shardID, path string, size uint64) - SubTreeSize(shardID, path string, size uint64) + AddOpenBlobovnizcaSize(shardID, path string, size uint64) + SubOpenBlobovnizcaSize(shardID, path string, size uint64) IncOpenBlobovnizcaCount(shardID, path string) DecOpenBlobovnizcaCount(shardID, path string) @@ -27,41 +27,41 @@ type blobovnizca struct { treeReqDuration *prometheus.HistogramVec treePut *prometheus.CounterVec treeGet *prometheus.CounterVec - treeSize *prometheus.GaugeVec + treeOpenSize *prometheus.GaugeVec treeOpenCounter *prometheus.GaugeVec } func newBlobovnizca() *blobovnizca { return &blobovnizca{ - treeMode: newShardIDPathMode(blobovnizaTreeSubSystem, "mode", "Blobovnizca tree mode"), + treeMode: newShardIDPathMode(blobovniczaTreeSubSystem, "mode", "Blobovnizca tree mode"), treeReqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, - Subsystem: blobovnizaTreeSubSystem, + Subsystem: blobovniczaTreeSubSystem, Name: "request_duration_seconds", Help: "Accumulated Blobovnizca tree request process duration", }, []string{shardIDLabel, pathLabel, successLabel, methodLabel, withStorageIDLabel}), treePut: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, - Subsystem: blobovnizaTreeSubSystem, + Subsystem: blobovniczaTreeSubSystem, Name: "put_bytes", Help: "Accumulated payload size written to Blobovnizca tree", }, []string{shardIDLabel, pathLabel}), treeGet: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, - Subsystem: blobovnizaTreeSubSystem, + Subsystem: blobovniczaTreeSubSystem, Name: "get_bytes", Help: "Accumulated payload size read from Blobovnizca tree", }, []string{shardIDLabel, pathLabel}), - treeSize: metrics.NewGaugeVec(prometheus.GaugeOpts{ + treeOpenSize: metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, - Subsystem: blobovnizaTreeSubSystem, - Name: "size_bytes", - Help: "Blobovnizca tree size", + Subsystem: blobovniczaTreeSubSystem, + Name: "open_blobovnizca_size_bytes", + Help: "Size of opened blobovnizcas of Blobovnizca tree", }, []string{shardIDLabel, pathLabel}), treeOpenCounter: metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, - Subsystem: blobovnizaTreeSubSystem, + Subsystem: blobovniczaTreeSubSystem, Name: "open_blobovnizca_count", Help: "Count of opened blobovnizcas of Blobovnizca tree", }, []string{shardIDLabel, pathLabel}), @@ -112,15 +112,15 @@ func (b *blobovnizca) AddBlobobvnizcaTreeGet(shardID, path string, size int) { }).Add(float64(size)) } -func (b *blobovnizca) AddTreeSize(shardID, path string, size uint64) { - b.treeSize.With(prometheus.Labels{ +func (b *blobovnizca) AddOpenBlobovnizcaSize(shardID, path string, size uint64) { + b.treeOpenSize.With(prometheus.Labels{ shardIDLabel: shardID, pathLabel: path, }).Add(float64(size)) } -func (b *blobovnizca) SubTreeSize(shardID, path string, size uint64) { - b.treeSize.With(prometheus.Labels{ +func (b *blobovnizca) SubOpenBlobovnizcaSize(shardID, path string, size uint64) { + b.treeOpenSize.With(prometheus.Labels{ shardIDLabel: shardID, pathLabel: path, }).Sub(float64(size)) diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index 691aaa375..550eec424 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -4,22 +4,22 @@ const ( namespace = "frostfs_node" innerRingNamespace = "frostfs_ir" - fstreeSubSystem = "fstree" - blobstoreSubSystem = "blobstore" - blobovnizaTreeSubSystem = "blobovniza_tree" - metabaseSubSystem = "metabase" - piloramaSubSystem = "pilorama" - engineSubsystem = "engine" - gcSubsystem = "garbage_collector" - innerRingSubsystem = "ir" - morphSubsystem = "morph" - morphCacheSubsystem = "morphcache" - objectSubsystem = "object" - replicatorSubsystem = "replicator" - stateSubsystem = "state" - treeServiceSubsystem = "treeservice" - writeCacheSubsystem = "writecache" - grpcServerSubsystem = "grpc_server" + fstreeSubSystem = "fstree" + blobstoreSubSystem = "blobstore" + blobovniczaTreeSubSystem = "blobovnicza_tree" + metabaseSubSystem = "metabase" + piloramaSubSystem = "pilorama" + engineSubsystem = "engine" + gcSubsystem = "garbage_collector" + innerRingSubsystem = "ir" + morphSubsystem = "morph" + morphCacheSubsystem = "morphcache" + objectSubsystem = "object" + replicatorSubsystem = "replicator" + stateSubsystem = "state" + treeServiceSubsystem = "treeservice" + writeCacheSubsystem = "writecache" + grpcServerSubsystem = "grpc_server" successLabel = "success" shardIDLabel = "shard_id" From c4e1d8eb07dc91b588cb46db0021dbf95e4db121 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 18 Aug 2023 11:14:10 +0300 Subject: [PATCH 0524/1943] [#602] node: Fix blobovnicza typos Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/control.go | 16 +++---- .../blobovnicza/metrics.go | 16 +++---- pkg/local_object_storage/blobovnicza/sizes.go | 4 +- .../blobstor/blobovniczatree/blobovnicza.go | 2 +- .../blobstor/blobovniczatree/control.go | 2 +- .../blobstor/blobovniczatree/metrics.go | 4 +- pkg/local_object_storage/metabase/put_test.go | 2 +- .../{blobovnizca.go => blobovnicza.go} | 18 +++---- .../{blobovnizca.go => blobovnicza.go} | 48 +++++++++---------- pkg/metrics/node.go | 4 +- 10 files changed, 58 insertions(+), 58 deletions(-) rename pkg/local_object_storage/metrics/{blobovnizca.go => blobovnicza.go} (83%) rename pkg/metrics/{blobovnizca.go => blobovnicza.go} (72%) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index 0e3204968..aa6be6025 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -14,7 +14,7 @@ import ( // Open opens an internal database at the configured path with the configured permissions. // // If the database file does not exist, it will be created automatically. -// If blobovnizca is already open, does nothing. +// If blobovnicza is already open, does nothing. func (b *Blobovnicza) Open() error { b.controlMtx.Lock() defer b.controlMtx.Unlock() @@ -45,7 +45,7 @@ func (b *Blobovnicza) Open() error { b.boltDB, err = bbolt.Open(b.path, b.perm, b.boltOptions) if err == nil { b.opened = true - b.metrics.IncOpenBlobovnizcaCount() + b.metrics.IncOpenBlobovniczaCount() } return err @@ -54,13 +54,13 @@ func (b *Blobovnicza) Open() error { // Init initializes internal database structure. // // If Blobovnicza is already initialized, no action is taken. -// Blobovnizca must be open, otherwise an error will return. +// Blobovnicza must be open, otherwise an error will return. func (b *Blobovnicza) Init() error { b.controlMtx.Lock() defer b.controlMtx.Unlock() if !b.opened { - return errors.New("blobovnizca is not open") + return errors.New("blobovnicza is not open") } b.log.Debug(logs.BlobovniczaInitializing, @@ -111,13 +111,13 @@ func (b *Blobovnicza) initializeSize() error { return fmt.Errorf("can't determine DB size: %w", err) } b.dataSize.Store(size) - b.metrics.AddOpenBlobovnizcaSize(size) + b.metrics.AddOpenBlobovniczaSize(size) return nil } // Close releases all internal database resources. // -// If blobovnizca is already closed, does nothing. +// If blobovnicza is already closed, does nothing. func (b *Blobovnicza) Close() error { b.controlMtx.Lock() defer b.controlMtx.Unlock() @@ -134,8 +134,8 @@ func (b *Blobovnicza) Close() error { return err } - b.metrics.DecOpenBlobovnizcaCount() - b.metrics.SubOpenBlobovnizcaSize(b.dataSize.Load()) + b.metrics.DecOpenBlobovniczaCount() + b.metrics.SubOpenBlobovniczaSize(b.dataSize.Load()) b.dataSize.Store(0) b.opened = false diff --git a/pkg/local_object_storage/blobovnicza/metrics.go b/pkg/local_object_storage/blobovnicza/metrics.go index 5f4f94cbe..9ce703201 100644 --- a/pkg/local_object_storage/blobovnicza/metrics.go +++ b/pkg/local_object_storage/blobovnicza/metrics.go @@ -1,16 +1,16 @@ package blobovnicza type Metrics interface { - IncOpenBlobovnizcaCount() - DecOpenBlobovnizcaCount() + IncOpenBlobovniczaCount() + DecOpenBlobovniczaCount() - AddOpenBlobovnizcaSize(size uint64) - SubOpenBlobovnizcaSize(size uint64) + AddOpenBlobovniczaSize(size uint64) + SubOpenBlobovniczaSize(size uint64) } type NoopMetrics struct{} -func (m *NoopMetrics) IncOpenBlobovnizcaCount() {} -func (m *NoopMetrics) DecOpenBlobovnizcaCount() {} -func (m *NoopMetrics) AddOpenBlobovnizcaSize(uint64) {} -func (m *NoopMetrics) SubOpenBlobovnizcaSize(uint64) {} +func (m *NoopMetrics) IncOpenBlobovniczaCount() {} +func (m *NoopMetrics) DecOpenBlobovniczaCount() {} +func (m *NoopMetrics) AddOpenBlobovniczaSize(uint64) {} +func (m *NoopMetrics) SubOpenBlobovniczaSize(uint64) {} diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index d4ac19201..a2f558f55 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -42,12 +42,12 @@ func upperPowerOfTwo(v uint64) uint64 { func (b *Blobovnicza) incSize(sz uint64) { b.dataSize.Add(sz) - b.metrics.AddOpenBlobovnizcaSize(sz) + b.metrics.AddOpenBlobovniczaSize(sz) } func (b *Blobovnicza) decSize(sz uint64) { b.dataSize.Add(^(sz - 1)) - b.metrics.SubOpenBlobovnizcaSize(sz) + b.metrics.SubOpenBlobovniczaSize(sz) } func (b *Blobovnicza) full() bool { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index a23bba425..d4fedb14c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -89,7 +89,7 @@ var _ common.Storage = (*Blobovniczas)(nil) var errPutFailed = errors.New("could not save the object in any blobovnicza") -// NewBlobovniczaTree returns new instance of blobovnizas tree. +// NewBlobovniczaTree returns new instance of blobovniczas tree. func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz = new(Blobovniczas) initConfig(&blz.cfg) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 3a06310f6..d7b0799c1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -130,7 +130,7 @@ func (b *Blobovniczas) openBlobovniczaNoCache(p string) (*blobovnicza.Blobovnicz blz := blobovnicza.New(append(b.blzOpts, blobovnicza.WithReadOnly(b.readOnly), blobovnicza.WithPath(path), - blobovnicza.WithMetrics(b.metrics.Blobovnizca()), + blobovnicza.WithMetrics(b.metrics.Blobovnicza()), )...) if err := blz.Open(); err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index 6c9b600c0..032d8cf71 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -7,7 +7,7 @@ import ( ) type Metrics interface { - Blobovnizca() blobovnicza.Metrics + Blobovnicza() blobovnicza.Metrics SetParentID(parentID string) @@ -33,6 +33,6 @@ func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} func (m *noopMetrics) Iterate(time.Duration, bool) {} func (m *noopMetrics) Put(time.Duration, int, bool) {} -func (m *noopMetrics) Blobovnizca() blobovnicza.Metrics { +func (m *noopMetrics) Blobovnicza() blobovnicza.Metrics { return &blobovnicza.NoopMetrics{} } diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index dbf89c9c8..a4cb2edc1 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -80,7 +80,7 @@ func BenchmarkPut(b *testing.B) { }) } -func TestDB_PutBlobovnicaUpdate(t *testing.T) { +func TestDB_PutBlobovniczaUpdate(t *testing.T) { db := newDB(t) raw1 := testutil.GenerateObject() diff --git a/pkg/local_object_storage/metrics/blobovnizca.go b/pkg/local_object_storage/metrics/blobovnicza.go similarity index 83% rename from pkg/local_object_storage/metrics/blobovnizca.go rename to pkg/local_object_storage/metrics/blobovnicza.go index 72c45c31d..5261f4080 100644 --- a/pkg/local_object_storage/metrics/blobovnizca.go +++ b/pkg/local_object_storage/metrics/blobovnicza.go @@ -22,7 +22,7 @@ type blobovniczaTreeMetrics struct { m metrics_impl.BlobobvnizcaMetrics } -func (m *blobovniczaTreeMetrics) Blobovnizca() blobovnicza.Metrics { +func (m *blobovniczaTreeMetrics) Blobovnicza() blobovnicza.Metrics { return &blobovniczaMetrics{ shardID: func() string { return m.shardID }, path: m.path, @@ -81,18 +81,18 @@ type blobovniczaMetrics struct { path string } -func (m *blobovniczaMetrics) AddOpenBlobovnizcaSize(size uint64) { - m.m.AddOpenBlobovnizcaSize(m.shardID(), m.path, size) +func (m *blobovniczaMetrics) AddOpenBlobovniczaSize(size uint64) { + m.m.AddOpenBlobovniczaSize(m.shardID(), m.path, size) } -func (m *blobovniczaMetrics) SubOpenBlobovnizcaSize(size uint64) { - m.m.SubOpenBlobovnizcaSize(m.shardID(), m.path, size) +func (m *blobovniczaMetrics) SubOpenBlobovniczaSize(size uint64) { + m.m.SubOpenBlobovniczaSize(m.shardID(), m.path, size) } -func (m *blobovniczaMetrics) IncOpenBlobovnizcaCount() { - m.m.IncOpenBlobovnizcaCount(m.shardID(), m.path) +func (m *blobovniczaMetrics) IncOpenBlobovniczaCount() { + m.m.IncOpenBlobovniczaCount(m.shardID(), m.path) } -func (m *blobovniczaMetrics) DecOpenBlobovnizcaCount() { - m.m.DecOpenBlobovnizcaCount(m.shardID(), m.path) +func (m *blobovniczaMetrics) DecOpenBlobovniczaCount() { + m.m.DecOpenBlobovniczaCount(m.shardID(), m.path) } diff --git a/pkg/metrics/blobovnizca.go b/pkg/metrics/blobovnicza.go similarity index 72% rename from pkg/metrics/blobovnizca.go rename to pkg/metrics/blobovnicza.go index b80ec5fc1..9c5e17520 100644 --- a/pkg/metrics/blobovnizca.go +++ b/pkg/metrics/blobovnicza.go @@ -15,14 +15,14 @@ type BlobobvnizcaMetrics interface { AddBlobobvnizcaTreePut(shardID, path string, size int) AddBlobobvnizcaTreeGet(shardID, path string, size int) - AddOpenBlobovnizcaSize(shardID, path string, size uint64) - SubOpenBlobovnizcaSize(shardID, path string, size uint64) + AddOpenBlobovniczaSize(shardID, path string, size uint64) + SubOpenBlobovniczaSize(shardID, path string, size uint64) - IncOpenBlobovnizcaCount(shardID, path string) - DecOpenBlobovnizcaCount(shardID, path string) + IncOpenBlobovniczaCount(shardID, path string) + DecOpenBlobovniczaCount(shardID, path string) } -type blobovnizca struct { +type blobovnicza struct { treeMode *shardIDPathModeValue treeReqDuration *prometheus.HistogramVec treePut *prometheus.CounterVec @@ -31,48 +31,48 @@ type blobovnizca struct { treeOpenCounter *prometheus.GaugeVec } -func newBlobovnizca() *blobovnizca { - return &blobovnizca{ - treeMode: newShardIDPathMode(blobovniczaTreeSubSystem, "mode", "Blobovnizca tree mode"), +func newBlobovnicza() *blobovnicza { + return &blobovnicza{ + treeMode: newShardIDPathMode(blobovniczaTreeSubSystem, "mode", "Blobovnicza tree mode"), treeReqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: blobovniczaTreeSubSystem, Name: "request_duration_seconds", - Help: "Accumulated Blobovnizca tree request process duration", + Help: "Accumulated Blobovnicza tree request process duration", }, []string{shardIDLabel, pathLabel, successLabel, methodLabel, withStorageIDLabel}), treePut: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: blobovniczaTreeSubSystem, Name: "put_bytes", - Help: "Accumulated payload size written to Blobovnizca tree", + Help: "Accumulated payload size written to Blobovnicza tree", }, []string{shardIDLabel, pathLabel}), treeGet: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: blobovniczaTreeSubSystem, Name: "get_bytes", - Help: "Accumulated payload size read from Blobovnizca tree", + Help: "Accumulated payload size read from Blobovnicza tree", }, []string{shardIDLabel, pathLabel}), treeOpenSize: metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: blobovniczaTreeSubSystem, - Name: "open_blobovnizca_size_bytes", - Help: "Size of opened blobovnizcas of Blobovnizca tree", + Name: "open_blobovnicza_size_bytes", + Help: "Size of opened blobovniczas of Blobovnicza tree", }, []string{shardIDLabel, pathLabel}), treeOpenCounter: metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: blobovniczaTreeSubSystem, - Name: "open_blobovnizca_count", - Help: "Count of opened blobovnizcas of Blobovnizca tree", + Name: "open_blobovnicza_count", + Help: "Count of opened blobovniczas of Blobovnicza tree", }, []string{shardIDLabel, pathLabel}), } } -func (b *blobovnizca) SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) { +func (b *blobovnicza) SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) { b.treeMode.SetMode(shardID, path, modeFromBool(readOnly)) } -func (b *blobovnizca) CloseBlobobvnizcaTree(shardID, path string) { +func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) { b.treeMode.SetMode(shardID, path, closedMode) b.treeReqDuration.DeletePartialMatch(prometheus.Labels{ shardIDLabel: shardID, @@ -88,7 +88,7 @@ func (b *blobovnizca) CloseBlobobvnizcaTree(shardID, path string) { }) } -func (b *blobovnizca) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) { +func (b *blobovnicza) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) { b.treeReqDuration.With(prometheus.Labels{ shardIDLabel: shardID, pathLabel: path, @@ -98,42 +98,42 @@ func (b *blobovnizca) BlobobvnizcaTreeMethodDuration(shardID, path string, metho }).Observe(d.Seconds()) } -func (b *blobovnizca) AddBlobobvnizcaTreePut(shardID, path string, size int) { +func (b *blobovnicza) AddBlobobvnizcaTreePut(shardID, path string, size int) { b.treePut.With(prometheus.Labels{ shardIDLabel: shardID, pathLabel: path, }).Add(float64(size)) } -func (b *blobovnizca) AddBlobobvnizcaTreeGet(shardID, path string, size int) { +func (b *blobovnicza) AddBlobobvnizcaTreeGet(shardID, path string, size int) { b.treeGet.With(prometheus.Labels{ shardIDLabel: shardID, pathLabel: path, }).Add(float64(size)) } -func (b *blobovnizca) AddOpenBlobovnizcaSize(shardID, path string, size uint64) { +func (b *blobovnicza) AddOpenBlobovniczaSize(shardID, path string, size uint64) { b.treeOpenSize.With(prometheus.Labels{ shardIDLabel: shardID, pathLabel: path, }).Add(float64(size)) } -func (b *blobovnizca) SubOpenBlobovnizcaSize(shardID, path string, size uint64) { +func (b *blobovnicza) SubOpenBlobovniczaSize(shardID, path string, size uint64) { b.treeOpenSize.With(prometheus.Labels{ shardIDLabel: shardID, pathLabel: path, }).Sub(float64(size)) } -func (b *blobovnizca) IncOpenBlobovnizcaCount(shardID, path string) { +func (b *blobovnicza) IncOpenBlobovniczaCount(shardID, path string) { b.treeOpenCounter.With(prometheus.Labels{ shardIDLabel: shardID, pathLabel: path, }).Inc() } -func (b *blobovnizca) DecOpenBlobovnizcaCount(shardID, path string) { +func (b *blobovnicza) DecOpenBlobovniczaCount(shardID, path string) { b.treeOpenCounter.With(prometheus.Labels{ shardIDLabel: shardID, pathLabel: path, diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index ce650c220..ad6864b8f 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -14,7 +14,7 @@ type NodeMetrics struct { epoch prometheus.Gauge fstree *fstreeMetrics blobstore *blobstoreMetrics - blobobvnizca *blobovnizca + blobobvnizca *blobovnicza metabase *metabaseMetrics pilorama *piloramaMetrics grpc *grpcServerMetrics @@ -35,7 +35,7 @@ func NewNodeMetrics() *NodeMetrics { }), fstree: newFSTreeMetrics(), blobstore: newBlobstoreMetrics(), - blobobvnizca: newBlobovnizca(), + blobobvnizca: newBlobovnicza(), metabase: newMetabaseMetrics(), pilorama: newPiloramaMetrics(), grpc: newGrpcServerMetrics(), From f2811f858575336f10a2f38aa04d39606a7a829a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 18 Aug 2023 13:01:27 +0300 Subject: [PATCH 0525/1943] [#602] metrics: Add blobovnicza items counter Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/blobovnicza.go | 3 ++- .../blobovnicza/control.go | 19 +++++++++++---- .../blobovnicza/delete.go | 2 +- .../blobovnicza/metrics.go | 13 ++++++---- pkg/local_object_storage/blobovnicza/put.go | 2 +- pkg/local_object_storage/blobovnicza/sizes.go | 17 ++++++++----- .../metrics/blobovnicza.go | 8 +++++++ pkg/metrics/blobovnicza.go | 24 +++++++++++++++++++ 8 files changed, 70 insertions(+), 18 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index 22168de22..c787f9d5e 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -16,7 +16,8 @@ import ( type Blobovnicza struct { cfg - dataSize atomic.Uint64 + dataSize atomic.Uint64 + itemsCount atomic.Uint64 boltDB *bbolt.DB diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index aa6be6025..ad554a0ad 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -68,8 +68,10 @@ func (b *Blobovnicza) Init() error { zap.Uint64("storage size limit", b.fullSizeLimit), ) - if size := b.dataSize.Load(); size != 0 { - b.log.Debug(logs.BlobovniczaAlreadyInitialized, zap.Uint64("size", size)) + size := b.dataSize.Load() + items := b.itemsCount.Load() + if size != 0 || items != 0 { + b.log.Debug(logs.BlobovniczaAlreadyInitialized, zap.Uint64("size", size), zap.Uint64("items", items)) return nil } @@ -96,14 +98,17 @@ func (b *Blobovnicza) Init() error { } } - return b.initializeSize() + return b.initializeCounters() } -func (b *Blobovnicza) initializeSize() error { +func (b *Blobovnicza) initializeCounters() error { var size uint64 + var items uint64 err := b.boltDB.View(func(tx *bbolt.Tx) error { return b.iterateAllBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) { - size += uint64(b.Stats().KeyN) * upper + keysN := uint64(b.Stats().KeyN) + size += keysN * upper + items += keysN return false, nil }) }) @@ -111,7 +116,9 @@ func (b *Blobovnicza) initializeSize() error { return fmt.Errorf("can't determine DB size: %w", err) } b.dataSize.Store(size) + b.itemsCount.Store(items) b.metrics.AddOpenBlobovniczaSize(size) + b.metrics.AddOpenBlobovniczaItems(items) return nil } @@ -136,7 +143,9 @@ func (b *Blobovnicza) Close() error { b.metrics.DecOpenBlobovniczaCount() b.metrics.SubOpenBlobovniczaSize(b.dataSize.Load()) + b.metrics.SubOpenBlobovniczaItems(b.itemsCount.Load()) b.dataSize.Store(0) + b.itemsCount.Store(0) b.opened = false diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index f332173c1..5a5da235f 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -74,7 +74,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err zap.String("binary size", stringifyByteSize(dataSize)), zap.String("range", stringifyBounds(sizeLowerBound, sizeUpperBound)), ) - b.decSize(sizeUpperBound) + b.itemDeleted(sizeUpperBound) } return DeleteRes{}, err diff --git a/pkg/local_object_storage/blobovnicza/metrics.go b/pkg/local_object_storage/blobovnicza/metrics.go index 9ce703201..37352b083 100644 --- a/pkg/local_object_storage/blobovnicza/metrics.go +++ b/pkg/local_object_storage/blobovnicza/metrics.go @@ -6,11 +6,16 @@ type Metrics interface { AddOpenBlobovniczaSize(size uint64) SubOpenBlobovniczaSize(size uint64) + + AddOpenBlobovniczaItems(items uint64) + SubOpenBlobovniczaItems(items uint64) } type NoopMetrics struct{} -func (m *NoopMetrics) IncOpenBlobovniczaCount() {} -func (m *NoopMetrics) DecOpenBlobovniczaCount() {} -func (m *NoopMetrics) AddOpenBlobovniczaSize(uint64) {} -func (m *NoopMetrics) SubOpenBlobovniczaSize(uint64) {} +func (m *NoopMetrics) IncOpenBlobovniczaCount() {} +func (m *NoopMetrics) DecOpenBlobovniczaCount() {} +func (m *NoopMetrics) AddOpenBlobovniczaSize(uint64) {} +func (m *NoopMetrics) SubOpenBlobovniczaSize(uint64) {} +func (m *NoopMetrics) AddOpenBlobovniczaItems(uint64) {} +func (m *NoopMetrics) SubOpenBlobovniczaItems(uint64) {} diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index b8cc9954a..e43b89ec6 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -85,7 +85,7 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { return nil }) if err == nil { - b.incSize(upperBound) + b.itemAdded(upperBound) } return PutRes{}, err diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index a2f558f55..290df9c93 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -3,6 +3,7 @@ package blobovnicza import ( "encoding/binary" "fmt" + "math" "math/bits" "strconv" ) @@ -40,14 +41,18 @@ func upperPowerOfTwo(v uint64) uint64 { return 1 << bits.Len64(v-1) } -func (b *Blobovnicza) incSize(sz uint64) { - b.dataSize.Add(sz) - b.metrics.AddOpenBlobovniczaSize(sz) +func (b *Blobovnicza) itemAdded(itemSize uint64) { + b.dataSize.Add(itemSize) + b.itemsCount.Add(1) + b.metrics.AddOpenBlobovniczaSize(itemSize) + b.metrics.AddOpenBlobovniczaItems(1) } -func (b *Blobovnicza) decSize(sz uint64) { - b.dataSize.Add(^(sz - 1)) - b.metrics.SubOpenBlobovniczaSize(sz) +func (b *Blobovnicza) itemDeleted(itemSize uint64) { + b.dataSize.Add(^(itemSize - 1)) + b.itemsCount.Add(math.MaxUint64) + b.metrics.SubOpenBlobovniczaSize(itemSize) + b.metrics.SubOpenBlobovniczaItems(1) } func (b *Blobovnicza) full() bool { diff --git a/pkg/local_object_storage/metrics/blobovnicza.go b/pkg/local_object_storage/metrics/blobovnicza.go index 5261f4080..0d0318b3b 100644 --- a/pkg/local_object_storage/metrics/blobovnicza.go +++ b/pkg/local_object_storage/metrics/blobovnicza.go @@ -96,3 +96,11 @@ func (m *blobovniczaMetrics) IncOpenBlobovniczaCount() { func (m *blobovniczaMetrics) DecOpenBlobovniczaCount() { m.m.DecOpenBlobovniczaCount(m.shardID(), m.path) } + +func (m *blobovniczaMetrics) AddOpenBlobovniczaItems(items uint64) { + m.m.AddOpenBlobovniczaItems(m.shardID(), m.path, items) +} + +func (m *blobovniczaMetrics) SubOpenBlobovniczaItems(items uint64) { + m.m.SubOpenBlobovniczaItems(m.shardID(), m.path, items) +} diff --git a/pkg/metrics/blobovnicza.go b/pkg/metrics/blobovnicza.go index 9c5e17520..a1ecbc700 100644 --- a/pkg/metrics/blobovnicza.go +++ b/pkg/metrics/blobovnicza.go @@ -18,6 +18,9 @@ type BlobobvnizcaMetrics interface { AddOpenBlobovniczaSize(shardID, path string, size uint64) SubOpenBlobovniczaSize(shardID, path string, size uint64) + AddOpenBlobovniczaItems(shardID, path string, items uint64) + SubOpenBlobovniczaItems(shardID, path string, items uint64) + IncOpenBlobovniczaCount(shardID, path string) DecOpenBlobovniczaCount(shardID, path string) } @@ -28,6 +31,7 @@ type blobovnicza struct { treePut *prometheus.CounterVec treeGet *prometheus.CounterVec treeOpenSize *prometheus.GaugeVec + treeOpenItems *prometheus.GaugeVec treeOpenCounter *prometheus.GaugeVec } @@ -59,6 +63,12 @@ func newBlobovnicza() *blobovnicza { Name: "open_blobovnicza_size_bytes", Help: "Size of opened blobovniczas of Blobovnicza tree", }, []string{shardIDLabel, pathLabel}), + treeOpenItems: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: blobovniczaTreeSubSystem, + Name: "open_blobovnicza_items_total", + Help: "Count of items in opened blobovniczas of Blobovnicza tree", + }, []string{shardIDLabel, pathLabel}), treeOpenCounter: metrics.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: blobovniczaTreeSubSystem, @@ -139,3 +149,17 @@ func (b *blobovnicza) DecOpenBlobovniczaCount(shardID, path string) { pathLabel: path, }).Dec() } + +func (b *blobovnicza) AddOpenBlobovniczaItems(shardID, path string, items uint64) { + b.treeOpenItems.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(items)) +} + +func (b *blobovnicza) SubOpenBlobovniczaItems(shardID, path string, items uint64) { + b.treeOpenItems.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Sub(float64(items)) +} From a1f1d233cc296f347510d86e9a73414433e4c4b1 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 16 Aug 2023 18:26:36 +0300 Subject: [PATCH 0526/1943] [#618] linters: bump truecloudlab-linters to 0.0.2 Signed-off-by: Alexander Chuprov --- .golangci.yml | 11 ++++++----- Makefile | 5 +++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 6741f1278..ef99fc152 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -38,13 +38,14 @@ linters-settings: pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object alias: objectSDK custom: - noliteral: + truecloudlab-linters: path: bin/external_linters.so original-url: git.frostfs.info/TrueCloudLab/linters.git settings: - target-methods : ["reportFlushError", "reportError"] - disable-packages: ["codes", "err", "res","exec"] - constants-package: "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + noliteral: + target-methods : ["reportFlushError", "reportError"] + disable-packages: ["codes", "err", "res","exec"] + constants-package: "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" linters: enable: @@ -77,6 +78,6 @@ linters: - gocognit - contextcheck - importas - - noliteral + - truecloudlab-linters disable-all: true fast: false diff --git a/Makefile b/Makefile index 3914986b5..604115469 100755 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.21 LINT_VERSION ?= 1.54.0 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.2 ARCH = amd64 BIN = bin @@ -26,7 +27,7 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed "s/-/~/")-${OS_RELEASE} OUTPUT_LINT_DIR ?= $(shell pwd)/bin -LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION) +LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION) TMP_DIR := .cache .PHONY: help all images dep clean fmts fmt imports test lint docker/lint @@ -139,7 +140,7 @@ pre-commit-run: lint-install: @mkdir -p $(TMP_DIR) @rm -rf $(TMP_DIR)/linters - @git clone --depth 1 https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters + @git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters @@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR) @rm -rf $(TMP_DIR)/linters @rmdir $(TMP_DIR) 2>/dev/null || true From 4d9a6c07fbecb39f29439eb8558136ccc0fe45f8 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 16 Aug 2023 18:30:33 +0300 Subject: [PATCH 0527/1943] [#618] core: Replace fmt.Sprintf with strconv.FormatUint Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/engine/tree.go | 10 +++++----- pkg/local_object_storage/pilorama/boltdb.go | 9 +++++---- pkg/local_object_storage/shard/tree.go | 10 +++++----- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index df3e919ec..55b9f9c50 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -3,7 +3,7 @@ package engine import ( "context" "errors" - "fmt" + "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -145,7 +145,7 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() @@ -177,7 +177,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() @@ -208,7 +208,7 @@ func (e *StorageEngine) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("height", fmt.Sprintf("%d", height)), + attribute.String("height", strconv.FormatUint(height, 10)), ), ) defer span.End() @@ -333,7 +333,7 @@ func (e *StorageEngine) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("height", fmt.Sprintf("%d", height)), + attribute.String("height", strconv.FormatUint(height, 10)), ), ) defer span.End() diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 33d2b6878..c9a9f039e 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -9,6 +9,7 @@ import ( "math/rand" "os" "path/filepath" + "strconv" "sync" "time" @@ -291,7 +292,7 @@ func (t *boltForest) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("height", fmt.Sprintf("%d", height)), + attribute.String("height", strconv.FormatUint(height, 10)), ), ) defer span.End() @@ -889,7 +890,7 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() @@ -937,7 +938,7 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() @@ -1046,7 +1047,7 @@ func (t *boltForest) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID str trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("height", fmt.Sprintf("%d", height)), + attribute.String("height", strconv.FormatUint(height, 10)), ), ) defer span.End() diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 7e2c80152..7795b820d 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -2,7 +2,7 @@ package shard import ( "context" - "fmt" + "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -140,7 +140,7 @@ func (s *Shard) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, n attribute.String("shard_id", s.ID().String()), attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() @@ -165,7 +165,7 @@ func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID strin attribute.String("shard_id", s.ID().String()), attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", fmt.Sprintf("%d", nodeID)), + attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() @@ -190,7 +190,7 @@ func (s *Shard) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, attribute.String("shard_id", s.ID().String()), attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("height", fmt.Sprintf("%d", height)), + attribute.String("height", strconv.FormatUint(height, 10)), ), ) defer span.End() @@ -309,7 +309,7 @@ func (s *Shard) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, tre attribute.String("shard_id", s.ID().String()), attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("height", fmt.Sprintf("%d", height)), + attribute.String("height", strconv.FormatUint(height, 10)), ), ) defer span.End() From d3904ec5994299e02fa22b6e8be19f0be4e65ff6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Aug 2023 18:06:33 +0300 Subject: [PATCH 0528/1943] [#602] config: Add blobovnicza leaf width parameter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 2 ++ cmd/frostfs-node/config/engine/config_test.go | 2 ++ .../config/engine/shard/blobstor/blobovnicza/config.go | 10 ++++++++++ config/example/node.env | 2 ++ config/example/node.json | 6 ++++-- config/example/node.yaml | 1 + 6 files changed, 21 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 935aefe7f..45ddeec83 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -174,6 +174,7 @@ type subStorageCfg struct { // blobovnicza-specific size uint64 width uint64 + leafWidth uint64 openedCacheSize int } @@ -288,6 +289,7 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.size = sub.Size() sCfg.depth = sub.ShallowDepth() sCfg.width = sub.ShallowWidth() + sCfg.leafWidth = sub.LeafWidth() sCfg.openedCacheSize = sub.OpenedCacheSize() case fstree.Type: sub := fstreeconfig.From((*config.Config)(storagesCfg[i])) diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index acf8c2a60..b8e95db6d 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -94,6 +94,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 1, blz.ShallowDepth()) require.EqualValues(t, 4, blz.ShallowWidth()) require.EqualValues(t, 50, blz.OpenedCacheSize()) + require.EqualValues(t, 10, blz.LeafWidth()) require.Equal(t, "tmp/0/blob", ss[1].Path()) require.EqualValues(t, 0644, ss[1].Perm()) @@ -142,6 +143,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 1, blz.ShallowDepth()) require.EqualValues(t, 4, blz.ShallowWidth()) require.EqualValues(t, 50, blz.OpenedCacheSize()) + require.EqualValues(t, 10, blz.LeafWidth()) require.Equal(t, "tmp/1/blob", ss[1].Path()) require.EqualValues(t, 0644, ss[1].Perm()) diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index 5595f10bb..a780ea927 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -102,3 +102,13 @@ func (x *Config) OpenedCacheSize() int { func (x *Config) BoltDB() *boltdbconfig.Config { return (*boltdbconfig.Config)(x) } + +// LeafWidth returns the value of "leaf_width" config parameter. +// +// Returns 0 if the value is not a positive number. +func (x *Config) LeafWidth() uint64 { + return config.UintSafe( + (*config.Config)(x), + "leaf_width", + ) +} diff --git a/config/example/node.env b/config/example/node.env index 3abb744be..7ed818ef9 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -121,6 +121,7 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 +FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 ### FSTree config FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH=tmp/0/blob @@ -167,6 +168,7 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 +FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10 ### FSTree config FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH=tmp/1/blob diff --git a/config/example/node.json b/config/example/node.json index 6e995112e..a480c4a0e 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -168,7 +168,8 @@ "size": 4194304, "depth": 1, "width": 4, - "opened_cache_capacity": 50 + "opened_cache_capacity": 50, + "leaf_width": 10 }, { "type": "fstree", @@ -218,7 +219,8 @@ "size": 4194304, "depth": 1, "width": 4, - "opened_cache_capacity": 50 + "opened_cache_capacity": 50, + "leaf_width": 10 }, { "type": "fstree", diff --git a/config/example/node.yaml b/config/example/node.yaml index acce3741a..39fb7714d 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -145,6 +145,7 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files + leaf_width: 10 # max count of key-value DB on leafs of object tree storage - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) depth: 5 # max depth of object tree storage in FS From 354a92ea2c5aeaeaae0705219ee39c2b1efc1f06 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 18 Aug 2023 09:08:26 +0300 Subject: [PATCH 0529/1943] [#602] blobovnicza: Add leaf width implementation Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 + .../blobstor/blobovniczatree/blobovnicza.go | 16 ++++++++++++---- .../blobstor/blobovniczatree/iterate.go | 7 ++++++- .../blobstor/blobovniczatree/option.go | 7 +++++++ 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 45ddeec83..808a78d28 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -776,6 +776,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaSize(sRead.size), blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), + blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), blobovniczatree.WithLogger(c.log), } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index d4fedb14c..80d064c80 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -98,6 +98,10 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { opts[i](&blz.cfg) } + if blz.blzLeafWidth == 0 { + blz.blzLeafWidth = blz.blzShallowWidth + } + cache, err := simplelru.NewLRU[string, *blobovnicza.Blobovnicza](blz.openedCacheSize, func(p string, value *blobovnicza.Blobovnicza) { lvlPath := filepath.Dir(p) if b, ok := blz.active[lvlPath]; ok && b.ind == u64FromHexString(filepath.Base(p)) { @@ -120,13 +124,17 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { panic(fmt.Errorf("could not create LRU cache of size %d: %w", blz.openedCacheSize, err)) } - cp := uint64(1) + activeMapCapacity := uint64(1) for i := uint64(0); i < blz.blzShallowDepth; i++ { - cp *= blz.blzShallowWidth + if i+1 == blz.blzShallowDepth { + activeMapCapacity *= blz.blzLeafWidth + } else { + activeMapCapacity *= blz.blzShallowWidth + } } blz.opened = cache - blz.active = make(map[string]blobovniczaWithIndex, cp) + blz.active = make(map[string]blobovniczaWithIndex, activeMapCapacity) return blz } @@ -161,7 +169,7 @@ func (b *Blobovniczas) updateAndGet(lvlPath string, old *uint64) (blobovniczaWit if ok { if old != nil { - if active.ind == b.blzShallowWidth-1 { + if active.ind == b.blzLeafWidth-1 { return active, logicerr.New("no more Blobovniczas") } else if active.ind != *old { // sort of CAS in order to control concurrent diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 0154fe2ca..aeac854f7 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -115,7 +115,12 @@ func (b *Blobovniczas) iterateDeepest(ctx context.Context, addr oid.Address, f f // iterator over particular level of directories. func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) { - indices := indexSlice(b.blzShallowWidth) + isLeafLevel := uint64(len(curPath)) == b.blzShallowDepth + levelWidth := b.blzShallowWidth + if isLeafLevel { + levelWidth = b.blzLeafWidth + } + indices := indexSlice(levelWidth) hrw.SortSliceByValue(indices, addressHash(addr, filepath.Join(curPath...))) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index d0503f23b..d11185652 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -17,6 +17,7 @@ type cfg struct { openedCacheSize int blzShallowDepth uint64 blzShallowWidth uint64 + blzLeafWidth uint64 compression *compression.Config blzOpts []blobovnicza.Option // reportError is the function called when encountering disk errors. @@ -64,6 +65,12 @@ func WithBlobovniczaShallowWidth(width uint64) Option { } } +func WithBlobovniczaLeafWidth(w uint64) Option { + return func(c *cfg) { + c.blzLeafWidth = w + } +} + func WithBlobovniczaShallowDepth(depth uint64) Option { return func(c *cfg) { c.blzShallowDepth = depth From 84ea07558767bfddf9edf1f3aa36baf224545a54 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 18 Aug 2023 17:44:17 +0300 Subject: [PATCH 0530/1943] [#625] cli: Fix SDK EACLPrm usage for PrmContainerEACL Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/client.go | 6 +++--- cmd/frostfs-cli/modules/container/get_eacl.go | 10 +++++++--- cmd/frostfs-cli/modules/container/set_eacl.go | 10 +++++++--- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index f09468973..ae76780bd 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -176,8 +176,8 @@ func DeleteContainer(ctx context.Context, prm DeleteContainerPrm) (res DeleteCon // EACLPrm groups parameters of EACL operation. type EACLPrm struct { - commonPrm - client.PrmContainerEACL + Client *client.Client + ClientParams client.PrmContainerEACL } // EACLRes groups the resulting values of EACL operation. @@ -194,7 +194,7 @@ func (x EACLRes) EACL() eacl.Table { // // Returns any error which prevented the operation from completing correctly in error return. func EACL(ctx context.Context, prm EACLPrm) (res EACLRes, err error) { - res.cliRes, err = prm.cli.ContainerEACL(ctx, prm.PrmContainerEACL) + res.cliRes, err = prm.Client.ContainerEACL(ctx, prm.ClientParams) return } diff --git a/cmd/frostfs-cli/modules/container/get_eacl.go b/cmd/frostfs-cli/modules/container/get_eacl.go index 21ea5b5bc..6746132eb 100644 --- a/cmd/frostfs-cli/modules/container/get_eacl.go +++ b/cmd/frostfs-cli/modules/container/get_eacl.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" ) @@ -20,9 +21,12 @@ var getExtendedACLCmd = &cobra.Command{ pk := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - var eaclPrm internalclient.EACLPrm - eaclPrm.SetClient(cli) - eaclPrm.SetContainer(id) + eaclPrm := internalclient.EACLPrm{ + Client: cli, + ClientParams: client.PrmContainerEACL{ + ContainerID: &id, + }, + } res, err := internalclient.EACL(cmd.Context(), eaclPrm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) diff --git a/cmd/frostfs-cli/modules/container/set_eacl.go b/cmd/frostfs-cli/modules/container/set_eacl.go index c88d5767b..51805d69d 100644 --- a/cmd/frostfs-cli/modules/container/set_eacl.go +++ b/cmd/frostfs-cli/modules/container/set_eacl.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" ) @@ -65,9 +66,12 @@ Container ID in EACL table will be substituted with ID from the CLI.`, cmd.Println("awaiting...") - var getEACLPrm internalclient.EACLPrm - getEACLPrm.SetClient(cli) - getEACLPrm.SetContainer(id) + getEACLPrm := internalclient.EACLPrm{ + Client: cli, + ClientParams: client.PrmContainerEACL{ + ContainerID: &id, + }, + } for i := 0; i < awaitTimeout; i++ { time.Sleep(1 * time.Second) diff --git a/go.mod b/go.mod index 4173ad0df..a1d99fcc6 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230809065235-d48788c7a946 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 7dda28828..fd7be9463 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230809065235-d48788c7a946 h1:/WVghZ/Za1eAwERCL4JgW5MNvbe7d2DU8qhFZBFgNGE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230809065235-d48788c7a946/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e h1:+78sZdTFeK2t+2qJ4Tmm5CItUjfcF4/nFMum3JnZ6PQ= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 62c2ad4b22182ef09f11f6d864f53fc124cd0afc Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 21 Aug 2023 11:11:58 +0300 Subject: [PATCH 0531/1943] [#626] logs: Remove autogenerated comments Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 922 +++++++++++++++++++++--------------------- 1 file changed, 461 insertions(+), 461 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 8d3712590..6c261785f 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -17,467 +17,467 @@ const ( ) const ( - InnerringNonalphabetModeDoNotStopContainerEstimations = "non-alphabet mode, do not stop container estimations" // Debug in ../node/pkg/innerring/blocktimer.go - InnerringCantStopEpochEstimation = "can't stop epoch estimation" // Warn in ../node/pkg/innerring/blocktimer.go - InnerringCantMakeNotaryDepositInMainChain = "can't make notary deposit in main chain" // Error in ../node/pkg/innerring/notary.go - InnerringCantMakeNotaryDepositInSideChain = "can't make notary deposit in side chain" // Error in ../node/pkg/innerring/notary.go - InnerringNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" // Info in ../node/pkg/innerring/notary.go - InnerringCantGetInnerRingIndex = "can't get inner ring index" // Error in ../node/pkg/innerring/state.go - InnerringCantGetInnerRingSize = "can't get inner ring size" // Error in ../node/pkg/innerring/state.go - InnerringCantGetAlphabetIndex = "can't get alphabet index" // Error in ../node/pkg/innerring/state.go - InnerringIgnoreValidatorVoteNodeNotInAlphabetRange = "ignore validator vote: node not in alphabet range" // Info in ../node/pkg/innerring/state.go - InnerringIgnoreValidatorVoteEmptyValidatorsList = "ignore validator vote: empty validators list" // Info in ../node/pkg/innerring/state.go - InnerringCantInvokeVoteMethodInAlphabetContract = "can't invoke vote method in alphabet contract" // Warn in ../node/pkg/innerring/state.go - InnerringCantGetLastProcessedMainChainBlockNumber = "can't get last processed main chain block number" // Warn in ../node/pkg/innerring/initialization.go - InnerringNotarySupport = "notary support" // Info in ../node/pkg/innerring/initialization.go - InnerringAlphabetKeysSyncIsDisabled = "alphabet keys sync is disabled" // Debug in ../node/pkg/innerring/initialization.go - InnerringNoControlServerEndpointSpecified = "no Control server endpoint specified, service is disabled" // Info in ../node/pkg/innerring/initialization.go - InnerringCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" // Warn in ../node/pkg/innerring/initialization.go - InnerringFailedToSetGroupSignerScope = "failed to set group signer scope, continue with Global" // Info in ../node/pkg/innerring/initialization.go - InnerringCantVoteForPreparedValidators = "can't vote for prepared validators" // Warn in ../node/pkg/innerring/innerring.go - InnerringNewBlock = "new block" // Debug in ../node/pkg/innerring/innerring.go - InnerringCantUpdatePersistentState = "can't update persistent state" // Warn in ../node/pkg/innerring/innerring.go - InnerringCloserError = "closer error" // Warn in ../node/pkg/innerring/innerring.go - InnerringReadConfigFromBlockchain = "read config from blockchain" // Debug in ../node/pkg/innerring/innerring.go - NotificatorNotificatorStartProcessingObjectNotifications = "notificator: start processing object notifications" // Debug in ../node/pkg/services/notificator/service.go - NotificatorNotificatorProcessingObjectNotification = "notificator: processing object notification" // Debug in ../node/pkg/services/notificator/service.go - PolicerCouldNotGetContainer = "could not get container" // Error in ../node/pkg/services/policer/check.go - PolicerCouldNotInhumeObjectWithMissingContainer = "could not inhume object with missing container" // Error in ../node/pkg/services/policer/check.go - PolicerCouldNotBuildPlacementVectorForObject = "could not build placement vector for object" // Error in ../node/pkg/services/policer/check.go - PolicerRedundantLocalObjectCopyDetected = "redundant local object copy detected" // Info in ../node/pkg/services/policer/check.go - PolicerReceiveObjectHeaderToCheckPolicyCompliance = "receive object header to check policy compliance" // Error in ../node/pkg/services/policer/check.go - PolicerConsiderNodeUnderMaintenanceAsOK = "consider node under maintenance as OK" // Debug in ../node/pkg/services/policer/check.go - PolicerShortageOfObjectCopiesDetected = "shortage of object copies detected" // Debug in ../node/pkg/services/policer/check.go - PolicerSomeOfTheCopiesAreStoredOnNodesUnderMaintenance = "some of the copies are stored on nodes under maintenance, save local copy" // Debug in ../node/pkg/services/policer/check.go - PolicerRoutineStopped = "routine stopped" // Info in ../node/pkg/services/policer/process.go - PolicerFailureAtObjectSelectForReplication = "failure at object select for replication" // Warn in ../node/pkg/services/policer/process.go - PolicerPoolSubmission = "pool submission" // Warn in ../node/pkg/services/policer/process.go - ReplicatorFinishWork = "finish work" // Debug in ../node/pkg/services/replicator/process.go - ReplicatorCouldNotGetObjectFromLocalStorage = "could not get object from local storage" // Error in ../node/pkg/services/replicator/process.go - ReplicatorCouldNotReplicateObject = "could not replicate object" // Error in ../node/pkg/services/replicator/process.go - ReplicatorObjectSuccessfullyReplicated = "object successfully replicated" // Debug in ../node/pkg/services/replicator/process.go - TreeRedirectingTreeServiceQuery = "redirecting tree service query" // Debug in ../node/pkg/services/tree/redirect.go - TreeBearerPresentedButNotAllowedByACL = "bearer presented but not allowed by ACL" // Debug in ../node/pkg/services/tree/signature.go - TreeCouldNotGetLastSynchronizedHeightForATree = "could not get last synchronized height for a tree" // Warn in ../node/pkg/services/tree/sync.go - TreeCouldNotUpdateLastSynchronizedHeightForATree = "could not update last synchronized height for a tree" // Warn in ../node/pkg/services/tree/sync.go - TreeSynchronizeTree = "synchronize tree" // Debug in ../node/pkg/services/tree/sync.go - TreeFailedToRunTreeSynchronizationOverAllNodes = "failed to run tree synchronization over all nodes" // Warn in ../node/pkg/services/tree/sync.go - TreeSyncingTrees = "syncing trees..." // Debug in ../node/pkg/services/tree/sync.go - TreeCouldNotFetchContainers = "could not fetch containers" // Error in ../node/pkg/services/tree/sync.go - TreeTreesHaveBeenSynchronized = "trees have been synchronized" // Debug in ../node/pkg/services/tree/sync.go - TreeSyncingContainerTrees = "syncing container trees..." // Debug in ../node/pkg/services/tree/sync.go - TreeCouldNotSyncTrees = "could not sync trees" // Error in ../node/pkg/services/tree/sync.go - TreeContainerTreesHaveBeenSynced = "container trees have been synced" // Debug in ../node/pkg/services/tree/sync.go - TreeCouldNotQueryTreesForSynchronization = "could not query trees for synchronization" // Error in ../node/pkg/services/tree/sync.go - TreeRemovingRedundantTrees = "removing redundant trees..." // Debug in ../node/pkg/services/tree/sync.go - TreeCouldNotRemoveRedundantTree = "could not remove redundant tree" // Error in ../node/pkg/services/tree/sync.go - TreeCouldNotCalculateContainerNodes = "could not calculate container nodes" // Error in ../node/pkg/services/tree/sync.go - TreeFailedToApplyReplicatedOperation = "failed to apply replicated operation" // Error in ../node/pkg/services/tree/replicator.go - TreeDoNotSendUpdateToTheNode = "do not send update to the node" // Debug in ../node/pkg/services/tree/replicator.go - TreeFailedToSentUpdateToTheNode = "failed to sent update to the node" // Warn in ../node/pkg/services/tree/replicator.go - TreeErrorDuringReplication = "error during replication" // Error in ../node/pkg/services/tree/replicator.go - PersistentCouldNotGetSessionFromPersistentStorage = "could not get session from persistent storage" // Error in ../node/pkg/services/session/storage/persistent/storage.go - PersistentCouldNotDeleteSToken = "could not delete token" // Error in ../node/pkg/services/session/storage/persistent/storage.go - PersistentCouldNotCleanUpExpiredTokens = "could not clean up expired tokens" // Error in ../node/pkg/services/session/storage/persistent/storage.go - ControllerReportIsAlreadyStarted = "report is already started" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - TombstoneCouldNotGetTheTombstoneTheSource = "tombstone getter: could not get the tombstone the source" // Warn in ../node/pkg/services/object_manager/tombstone/checker.go - DeleteRequestIsNotRolledOverToTheContainer = "request is not rolled over to the container" // Debug in ../node/pkg/services/object/delete/container.go - DeleteCouldNotComposeSplitInfo = "could not compose split info" // Debug in ../node/pkg/services/object/delete/exec.go - DeleteNoSplitInfoObjectIsPHY = "no split info, object is PHY" // Debug in ../node/pkg/services/object/delete/exec.go - DeleteAssemblingChain = "assembling chain..." // Debug in ../node/pkg/services/object/delete/exec.go - DeleteCouldNotGetPreviousSplitElement = "could not get previous split element" // Debug in ../node/pkg/services/object/delete/exec.go - DeleteCollectingChildren = "collecting children..." // Debug in ../node/pkg/services/object/delete/exec.go - DeleteCouldNotCollectObjectChildren = "could not collect object children" // Debug in ../node/pkg/services/object/delete/exec.go - DeleteSupplementBySplitID = "supplement by split ID" // Debug in ../node/pkg/services/object/delete/exec.go - DeleteCouldNotSearchForSplitChainMembers = "could not search for split chain members" // Debug in ../node/pkg/services/object/delete/exec.go - DeleteCouldNotMarshalTombstoneStructure = "could not marshal tombstone structure" // Debug in ../node/pkg/services/object/delete/exec.go - DeleteCouldNotSaveTheTombstone = "could not save the tombstone" // Debug in ../node/pkg/services/object/delete/exec.go - DeleteFormingTombstoneStructure = "forming tombstone structure..." // Debug in ../node/pkg/services/object/delete/local.go - DeleteTombstoneStructureSuccessfullyFormedSaving = "tombstone structure successfully formed, saving..." // Debug in ../node/pkg/services/object/delete/local.go - DeleteCouldNotReadTombstoneLifetimeConfig = "could not read tombstone lifetime config" // Debug in ../node/pkg/services/object/delete/local.go - DeleteFormingSplitInfo = "forming split info..." // Debug in ../node/pkg/services/object/delete/local.go - DeleteSplitInfoSuccessfullyFormedCollectingMembers = "split info successfully formed, collecting members..." // Debug in ../node/pkg/services/object/delete/local.go - DeleteMembersSuccessfullyCollected = "members successfully collected" // Debug in ../node/pkg/services/object/delete/local.go - GetRemoteCallFailed = "remote call failed" // Debug in ../node/pkg/services/object/get/remote.go - GetCanNotAssembleTheObject = "can not assemble the object" // Debug in ../node/pkg/services/object/get/assemble.go - GetTryingToAssembleTheObject = "trying to assemble the object..." // Debug in ../node/pkg/services/object/get/assemble.go - GetAssemblingSplittedObject = "assembling splitted object..." // Debug in ../node/pkg/services/object/get/assemble.go - GetAssemblingSplittedObjectCompleted = "assembling splitted object completed" // Debug in ../node/pkg/services/object/get/assemble.go - GetFailedToAssembleSplittedObject = "failed to assemble splitted object" // Warn in ../node/pkg/services/object/get/assemble.go - GetCouldNotGenerateContainerTraverser = "could not generate container traverser" // Debug in ../node/pkg/services/object/get/exec.go - GetCouldNotConstructRemoteNodeClient = "could not construct remote node client" // Debug in ../node/pkg/services/object/get/exec.go - GetCouldNotWriteHeader = "could not write header" // Debug in ../node/pkg/services/object/get/exec.go - GetCouldNotWritePayloadChunk = "could not write payload chunk" // Debug in ../node/pkg/services/object/get/exec.go - GetLocalGetFailed = "local get failed" // Debug in ../node/pkg/services/object/get/local.go - GetReturnResultDirectly = "return result directly" // Debug in ../node/pkg/services/object/get/container.go - GetCompletingTheOperation = "completing the operation" // Debug in ../node/pkg/services/object/get/container.go - GetRequestedObjectWasMarkedAsRemoved = "requested object was marked as removed" // Debug in ../node/pkg/services/object/get/get.go - GetRequestedObjectIsVirtual = "requested object is virtual" // Debug in ../node/pkg/services/object/get/get.go - GetRequestedRangeIsOutOfObjectBounds = "requested range is out of object bounds" // Debug in ../node/pkg/services/object/get/get.go - PutAdditionalContainerBroadcastFailure = "additional container broadcast failure" // Error in ../node/pkg/services/object/put/distributed.go - SearchReturnResultDirectly = "return result directly" // Debug in ../node/pkg/services/object/search/container.go - SearchCouldNotConstructRemoteNodeClient = "could not construct remote node client" // Debug in ../node/pkg/services/object/search/container.go - SearchRemoteOperationFailed = "remote operation failed" // Debug in ../node/pkg/services/object/search/container.go - SearchCouldNotGenerateContainerTraverser = "could not generate container traverser" // Debug in ../node/pkg/services/object/search/exec.go - SearchCouldNotWriteObjectIdentifiers = "could not write object identifiers" // Debug in ../node/pkg/services/object/search/exec.go - SearchLocalOperationFailed = "local operation failed" // Debug in ../node/pkg/services/object/search/local.go - UtilObjectServiceError = "object service error" // Error in ../node/pkg/services/object/util/log.go - UtilCouldNotPushTaskToWorkerPool = "could not push task to worker pool" // Error in ../node/pkg/services/object/util/log.go - V2CantCheckIfRequestFromInnerRing = "can't check if request from inner ring" // Debug in ../node/pkg/services/object/acl/v2/classifier.go - V2CantCheckIfRequestFromContainerNode = "can't check if request from container node" // Debug in ../node/pkg/services/object/acl/v2/classifier.go - NatsNatsConnectionWasLost = "nats: connection was lost" // Error in ../node/pkg/services/notificator/nats/service.go - NatsNatsReconnectedToTheServer = "nats: reconnected to the server" // Warn in ../node/pkg/services/notificator/nats/service.go - NatsNatsClosingConnectionAsTheContextIsDone = "nats: closing connection as the context is done" // Info in ../node/pkg/services/notificator/nats/service.go - ControllerStartingToAnnounceTheValuesOfTheMetrics = "starting to announce the values of the metrics" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerCouldNotInitializeIteratorOverLocallyCollectedMetrics = "could not initialize iterator over locally collected metrics" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerCouldNotInitializeAnnouncementAccumulator = "could not initialize announcement accumulator" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerIteratorOverLocallyCollectedMetricsAborted = "iterator over locally collected metrics aborted" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerCouldNotFinishWritingLocalAnnouncements = "could not finish writing local announcements" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerTrustAnnouncementSuccessfullyFinished = "trust announcement successfully finished" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerAnnouncementIsAlreadyStarted = "announcement is already started" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerAnnouncementSuccessfullyInterrupted = "announcement successfully interrupted" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerAnnouncementIsNotStartedOrAlreadyInterrupted = "announcement is not started or already interrupted" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerCouldNotInitializeIteratorOverLocallyAccumulatedAnnouncements = "could not initialize iterator over locally accumulated announcements" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerCouldNotInitializeResultTarget = "could not initialize result target" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerIteratorOverLocalAnnouncementsAborted = "iterator over local announcements aborted" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - ControllerCouldNotFinishWritingLoadEstimations = "could not finish writing load estimations" // Debug in ../node/pkg/services/container/announcement/load/controller/calls.go - RouteCouldNotInitializeWriterProvider = "could not initialize writer provider" // Debug in ../node/pkg/services/container/announcement/load/route/calls.go - RouteCouldNotInitializeWriter = "could not initialize writer" // Debug in ../node/pkg/services/container/announcement/load/route/calls.go - RouteCouldNotPutTheValue = "could not put the value" // Debug in ../node/pkg/services/container/announcement/load/route/calls.go - RouteCouldNotCloseRemoteServerWriter = "could not close remote server writer" // Debug in ../node/pkg/services/container/announcement/load/route/calls.go - ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch = "could not restore block subscription after RPC switch" // Error in ../node/pkg/morph/client/notifications.go - ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch = "could not restore notification subscription after RPC switch" // Error in ../node/pkg/morph/client/notifications.go - ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch = "could not restore notary notification subscription after RPC switch" // Error in ../node/pkg/morph/client/notifications.go - ClientCouldNotEstablishConnectionToTheSwitchedRPCNode = "could not establish connection to the switched RPC node" // Warn in ../node/pkg/morph/client/multi.go - ClientConnectionToTheNewRPCNodeHasBeenEstablished = "connection to the new RPC node has been established" // Info in ../node/pkg/morph/client/multi.go - ClientSwitchingToTheNextRPCNode = "switching to the next RPC node" // Warn in ../node/pkg/morph/client/multi.go - ClientCouldNotEstablishConnectionToAnyRPCNode = "could not establish connection to any RPC node" // Error in ../node/pkg/morph/client/multi.go - ClientCouldNotCreateClientToTheHigherPriorityNode = "could not create client to the higher priority node" // Warn in ../node/pkg/morph/client/multi.go - ClientSwitchedToTheHigherPriorityRPC = "switched to the higher priority RPC" // Info in ../node/pkg/morph/client/multi.go - ClientCouldNotRestoreSideChainSubscriptionsUsingNode = "could not restore side chain subscriptions using node" // Warn in ../node/pkg/morph/client/multi.go - ClientNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" // Info in ../node/pkg/morph/client/notary.go - ClientNotaryDepositInvoke = "notary deposit invoke" // Info in ../node/pkg/morph/client/notary.go - ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked" // Debug in ../node/pkg/morph/client/notary.go - ClientNotaryRequestInvoked = "notary request invoked" // Debug in ../node/pkg/morph/client/notary.go - ClientNeoClientInvoke = "neo client invoke" // Debug in ../node/pkg/morph/client/client.go - ClientNativeGasTransferInvoke = "native gas transfer invoke" // Debug in ../node/pkg/morph/client/client.go - ClientBatchGasTransferInvoke = "batch gas transfer invoke" // Debug in ../node/pkg/morph/client/client.go - ClientCantGetBlockchainHeight = "can't get blockchain height" // Error in ../node/pkg/morph/client/client.go - ClientCantGetBlockchainHeight243 = "can't get blockchain height" // Error in ../node/pkg/morph/client/client.go - EventCouldNotSubmitHandlerToWorkerPool = "could not Submit handler to worker pool" // Warn in ../node/pkg/morph/event/utils.go - EventCouldNotStartListenToEvents = "could not start listen to events" // Error in ../node/pkg/morph/event/listener.go - EventStopEventListenerByError = "stop event listener by error" // Error in ../node/pkg/morph/event/listener.go - EventStopEventListenerByContext = "stop event listener by context" // Info in ../node/pkg/morph/event/listener.go - EventStopEventListenerByNotificationChannel = "stop event listener by notification channel" // Warn in ../node/pkg/morph/event/listener.go - EventNilNotificationEventWasCaught = "nil notification event was caught" // Warn in ../node/pkg/morph/event/listener.go - EventStopEventListenerByNotaryChannel = "stop event listener by notary channel" // Warn in ../node/pkg/morph/event/listener.go - EventNilNotaryEventWasCaught = "nil notary event was caught" // Warn in ../node/pkg/morph/event/listener.go - EventStopEventListenerByBlockChannel = "stop event listener by block channel" // Warn in ../node/pkg/morph/event/listener.go - EventNilBlockWasCaught = "nil block was caught" // Warn in ../node/pkg/morph/event/listener.go - EventListenerWorkerPoolDrained = "listener worker pool drained" // Warn in ../node/pkg/morph/event/listener.go - EventEventParserNotSet = "event parser not set" // Debug in ../node/pkg/morph/event/listener.go - EventCouldNotParseNotificationEvent = "could not parse notification event" // Warn in ../node/pkg/morph/event/listener.go - EventNotificationHandlersForParsedNotificationEventWereNotRegistered = "notification handlers for parsed notification event were not registered" // Info in ../node/pkg/morph/event/listener.go - EventSkipExpiredMainTXNotaryEvent = "skip expired main TX notary event" // Warn in ../node/pkg/morph/event/listener.go - EventCouldNotPrepareAndValidateNotaryEvent = "could not prepare and validate notary event" // Warn in ../node/pkg/morph/event/listener.go - EventNotaryParserNotSet = "notary parser not set" // Debug in ../node/pkg/morph/event/listener.go - EventCouldNotParseNotaryEvent = "could not parse notary event" // Warn in ../node/pkg/morph/event/listener.go - EventNotaryHandlersForParsedNotificationEventWereNotRegistered = "notary handlers for parsed notification event were not registered" // Info in ../node/pkg/morph/event/listener.go - EventIgnoreNilEventParser = "ignore nil event parser" // Info in ../node/pkg/morph/event/listener.go - EventListenerHasBeenAlreadyStartedIgnoreParser = "listener has been already started, ignore parser" // Warn in ../node/pkg/morph/event/listener.go - EventRegisteredNewEventParser = "registered new event parser" // Debug in ../node/pkg/morph/event/listener.go - EventIgnoreNilEventHandler = "ignore nil event handler" // Warn in ../node/pkg/morph/event/listener.go - EventIgnoreHandlerOfEventWoParser = "ignore handler of event w/o parser" // Warn in ../node/pkg/morph/event/listener.go - EventRegisteredNewEventHandler = "registered new event handler" // Debug in ../node/pkg/morph/event/listener.go - EventIgnoreNilNotaryEventParser = "ignore nil notary event parser" // Info in ../node/pkg/morph/event/listener.go - EventListenerHasBeenAlreadyStartedIgnoreNotaryParser = "listener has been already started, ignore notary parser" // Warn in ../node/pkg/morph/event/listener.go - EventIgnoreNilNotaryEventHandler = "ignore nil notary event handler" // Warn in ../node/pkg/morph/event/listener.go - EventIgnoreHandlerOfNotaryEventWoParser = "ignore handler of notary event w/o parser" // Warn in ../node/pkg/morph/event/listener.go - EventIgnoreNilBlockHandler = "ignore nil block handler" // Warn in ../node/pkg/morph/event/listener.go - SubscriberRemoteNotificationChannelHasBeenClosed = "remote notification channel has been closed" // Warn in ../node/pkg/morph/subscriber/subscriber.go - SubscriberCantCastNotifyEventValueToTheNotifyStruct = "can't cast notify event value to the notify struct" // Error in ../node/pkg/morph/subscriber/subscriber.go - SubscriberNewNotificationEventFromSidechain = "new notification event from sidechain" // Debug in ../node/pkg/morph/subscriber/subscriber.go - SubscriberCantCastBlockEventValueToBlock = "can't cast block event value to block" // Error in ../node/pkg/morph/subscriber/subscriber.go - SubscriberCantCastNotifyEventValueToTheNotaryRequestStruct = "can't cast notify event value to the notary request struct" // Error in ../node/pkg/morph/subscriber/subscriber.go - SubscriberUnsupportedNotificationFromTheChain = "unsupported notification from the chain" // Debug in ../node/pkg/morph/subscriber/subscriber.go - BlobovniczaCreatingDirectoryForBoltDB = "creating directory for BoltDB" // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go - BlobovniczaOpeningBoltDB = "opening BoltDB" // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go - BlobovniczaInitializing = "initializing..." // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go - BlobovniczaAlreadyInitialized = "already initialized" // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go - BlobovniczaCreatingBucketForSizeRange = "creating bucket for size range" // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go - BlobovniczaClosingBoltDB = "closing BoltDB" // Debug in ../node/pkg/local_object_storage/blobovnicza/control.go - BlobovniczaObjectWasRemovedFromBucket = "object was removed from bucket" // Debug in ../node/pkg/local_object_storage/blobovnicza/delete.go - BlobstorOpening = "opening..." // Debug in ../node/pkg/local_object_storage/blobstor/control.go - BlobstorInitializing = "initializing..." // Debug in ../node/pkg/local_object_storage/blobstor/control.go - BlobstorClosing = "closing..." // Debug in ../node/pkg/local_object_storage/blobstor/control.go - BlobstorCouldntCloseStorage = "couldn't close storage" // Info in ../node/pkg/local_object_storage/blobstor/control.go - BlobstorErrorOccurredDuringObjectExistenceChecking = "error occurred during object existence checking" // Warn in ../node/pkg/local_object_storage/blobstor/exists.go - BlobstorErrorOccurredDuringTheIteration = "error occurred during the iteration" // Warn in ../node/pkg/local_object_storage/blobstor/iterate.go - EngineShardHasBeenRemoved = "shard has been removed" // Info in ../node/pkg/local_object_storage/engine/shards.go - EngineCouldNotCloseRemovedShard = "could not close removed shard" // Error in ../node/pkg/local_object_storage/engine/shards.go - EngineCouldNotOpenShardClosingAndSkipping = "could not open shard, closing and skipping" // Error in ../node/pkg/local_object_storage/engine/control.go - EngineCouldNotClosePartiallyInitializedShard = "could not close partially initialized shard" // Error in ../node/pkg/local_object_storage/engine/control.go - EngineCouldNotInitializeShardClosingAndSkipping = "could not initialize shard, closing and skipping" // Error in ../node/pkg/local_object_storage/engine/control.go - EngineCouldNotCloseShard = "could not close shard" // Debug in ../node/pkg/local_object_storage/engine/control.go - EngineCouldNotReloadAShard = "could not reload a shard" // Error in ../node/pkg/local_object_storage/engine/control.go - EngineAddedNewShard = "added new shard" // Info in ../node/pkg/local_object_storage/engine/control.go - EngineCouldNotMarkObjectForShardRelocation = "could not mark object for shard relocation" // Warn in ../node/pkg/local_object_storage/engine/put.go - EngineCouldNotPutObjectToShard = "could not put object to shard" // Warn in ../node/pkg/local_object_storage/engine/put.go - EngineErrorDuringSearchingForObjectChildren = "error during searching for object children" // Warn in ../node/pkg/local_object_storage/engine/delete.go - EngineCouldNotInhumeObjectInShard = "could not inhume object in shard" // Debug in ../node/pkg/local_object_storage/engine/delete.go - EngineStartingRemovalOfLocallyredundantCopies = "starting removal of locally-redundant copies" // Info in ../node/pkg/local_object_storage/engine/remove_copies.go - EngineStartedDuplicatesRemovalRoutine = "started duplicates removal routine" // Debug in ../node/pkg/local_object_storage/engine/remove_copies.go - EngineFinishedRemovalOfLocallyredundantCopies = "finished removal of locally-redundant copies" // Error in ../node/pkg/local_object_storage/engine/remove_copies.go - EngineRemovingAnObjectWithoutFullLockingCheck = "removing an object without full locking check" // Warn in ../node/pkg/local_object_storage/engine/inhume.go - EngineInterruptProcessingTheExpiredLocks = "interrupt processing the expired locks" // Info in ../node/pkg/local_object_storage/engine/inhume.go - EngineInterruptProcessingTheDeletedLocks = "interrupt processing the deleted locks" // Info in ../node/pkg/local_object_storage/engine/inhume.go - EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly = "failed to move shard in degraded-read-only mode, moving to read-only" // Error in ../node/pkg/local_object_storage/engine/engine.go - EngineFailedToMoveShardInReadonlyMode = "failed to move shard in read-only mode" // Error in ../node/pkg/local_object_storage/engine/engine.go - EngineShardIsMovedInReadonlyModeDueToErrorThreshold = "shard is moved in read-only mode due to error threshold" // Info in ../node/pkg/local_object_storage/engine/engine.go - EngineShardIsMovedInDegradedModeDueToErrorThreshold = "shard is moved in degraded mode due to error threshold" // Info in ../node/pkg/local_object_storage/engine/engine.go - EngineModeChangeIsInProgressIgnoringSetmodeRequest = "mode change is in progress, ignoring set-mode request" // Debug in ../node/pkg/local_object_storage/engine/engine.go - EngineStartedShardsEvacuation = "started shards evacuation" // Info in ../node/pkg/local_object_storage/engine/evacuate.go - EngineFinishedSuccessfullyShardsEvacuation = "shards evacuation finished successfully" // Info in ../node/pkg/local_object_storage/engine/evacuate.go - EngineFinishedWithErrorShardsEvacuation = "shards evacuation finished with error" // Error in ../node/pkg/local_object_storage/engine/evacuate.go - EngineObjectIsMovedToAnotherShard = "object is moved to another shard" // Debug in ../node/pkg/local_object_storage/engine/evacuate.go - MetabaseMissingMatcher = "missing matcher" // Debug in ../node/pkg/local_object_storage/metabase/select.go - MetabaseErrorInFKBTSelection = "error in FKBT selection" // Debug in ../node/pkg/local_object_storage/metabase/select.go - MetabaseCantDecodeListBucketLeaf = "can't decode list bucket leaf" // Debug in ../node/pkg/local_object_storage/metabase/select.go - MetabaseUnknownOperation = "unknown operation" // Debug in ../node/pkg/local_object_storage/metabase/select.go - MetabaseCantIterateOverTheBucket = "can't iterate over the bucket" // Debug in ../node/pkg/local_object_storage/metabase/select.go - MetabaseCouldNotIterateOverTheBuckets = "could not iterate over the buckets" // Debug in ../node/pkg/local_object_storage/metabase/select.go - MetabaseCreatedDirectoryForMetabase = "created directory for Metabase" // Debug in ../node/pkg/local_object_storage/metabase/control.go - MetabaseOpenedBoltDBInstanceForMetabase = "opened boltDB instance for Metabase" // Debug in ../node/pkg/local_object_storage/metabase/control.go - MetabaseCheckingMetabaseVersion = "checking metabase version" // Debug in ../node/pkg/local_object_storage/metabase/control.go - ShardCantSelectAllObjects = "can't select all objects" // Debug in ../node/pkg/local_object_storage/shard/list.go - ShardSettingShardMode = "setting shard mode" // Info in ../node/pkg/local_object_storage/shard/mode.go - ShardShardModeSetSuccessfully = "shard mode set successfully" // Info in ../node/pkg/local_object_storage/shard/mode.go - ShardCouldNotMarkObjectForShardRelocationInMetabase = "could not mark object for shard relocation in metabase" // Debug in ../node/pkg/local_object_storage/shard/move.go - ShardCantDeleteObjectFromWriteCache = "can't delete object from write cache" // Warn in ../node/pkg/local_object_storage/shard/delete.go - ShardCantGetStorageIDFromMetabase = "can't get storage ID from metabase" // Debug in ../node/pkg/local_object_storage/shard/delete.go - ShardCantRemoveObjectFromBlobStor = "can't remove object from blobStor" // Debug in ../node/pkg/local_object_storage/shard/delete.go - ShardFetchingObjectWithoutMeta = "fetching object without meta" // Warn in ../node/pkg/local_object_storage/shard/get.go - ShardObjectIsMissingInWritecache = "object is missing in write-cache" // Debug in ../node/pkg/local_object_storage/shard/get.go - ShardFailedToFetchObjectFromWritecache = "failed to fetch object from write-cache" // Error in ../node/pkg/local_object_storage/shard/get.go - ShardCantPutObjectToTheWritecacheTryingBlobstor = "can't put object to the write-cache, trying blobstor" // Debug in ../node/pkg/local_object_storage/shard/put.go - ShardMetaObjectCounterRead = "meta: object counter read" // Warn in ../node/pkg/local_object_storage/shard/shard.go - ShardMetaCantReadContainerList = "meta: can't read container list" // Warn in ../node/pkg/local_object_storage/shard/shard.go - ShardMetaCantReadContainerSize = "meta: can't read container size" // Warn in ../node/pkg/local_object_storage/shard/shard.go - ShardMetabaseFailureSwitchingMode = "metabase failure, switching mode" // Error in ../node/pkg/local_object_storage/shard/control.go - ShardCantMoveShardToReadonlySwitchMode = "can't move shard to readonly, switch mode" // Error in ../node/pkg/local_object_storage/shard/control.go - ShardCouldNotUnmarshalObject = "could not unmarshal object" // Warn in ../node/pkg/local_object_storage/shard/control.go - ShardCouldNotCloseShardComponent = "could not close shard component" // Error in ../node/pkg/local_object_storage/shard/control.go - ShardCantOpenMetabaseMoveToADegradedMode = "can't open metabase, move to a degraded mode" // Error in ../node/pkg/local_object_storage/shard/control.go - ShardCantInitializeMetabaseMoveToADegradedreadonlyMode = "can't initialize metabase, move to a degraded-read-only mode" // Error in ../node/pkg/local_object_storage/shard/control.go - ShardTryingToRestoreReadwriteMode = "trying to restore read-write mode" // Info in ../node/pkg/local_object_storage/shard/control.go - ShardStopEventListenerByClosedChannel = "stop event listener by closed channel" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardCouldNotSubmitGCJobToWorkerPool = "could not submit GC job to worker pool" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardGCIsStopped = "GC is stopped" // Debug in ../node/pkg/local_object_storage/shard/gc.go - ShardWaitingForGCWorkersToStop = "waiting for GC workers to stop..." // Info in ../node/pkg/local_object_storage/shard/gc.go - ShardIteratorOverMetabaseGraveyardFailed = "iterator over metabase graveyard failed" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardCouldNotDeleteTheObjects = "could not delete the objects" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardIteratorOverExpiredObjectsFailed = "iterator over expired objects failed" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardCouldNotInhumeTheObjects = "could not inhume the objects" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardStartedExpiredTombstonesHandling = "started expired tombstones handling" // Debug in ../node/pkg/local_object_storage/shard/gc.go - ShardIteratingTombstones = "iterating tombstones" // Debug in ../node/pkg/local_object_storage/shard/gc.go - ShardShardIsInADegradedModeSkipCollectingExpiredTombstones = "shard is in a degraded mode, skip collecting expired tombstones" // Debug in ../node/pkg/local_object_storage/shard/gc.go - ShardIteratorOverGraveyardFailed = "iterator over graveyard failed" // Error in ../node/pkg/local_object_storage/shard/gc.go - ShardHandlingExpiredTombstonesBatch = "handling expired tombstones batch" // Debug in ../node/pkg/local_object_storage/shard/gc.go - ShardFinishedExpiredTombstonesHandling = "finished expired tombstones handling" // Debug in ../node/pkg/local_object_storage/shard/gc.go - ShardIteratorOverExpiredLocksFailed = "iterator over expired locks failed" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardCouldNotMarkTombstonesAsGarbage = "could not mark tombstones as garbage" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardCouldNotDropExpiredGraveRecords = "could not drop expired grave records" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardFailureToUnlockObjects = "failure to unlock objects" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardFailureToMarkLockersAsGarbage = "failure to mark lockers as garbage" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardFailureToGetExpiredUnlockedObjects = "failure to get expired unlocked objects" // Warn in ../node/pkg/local_object_storage/shard/gc.go - ShardCouldNotMarkObjectToDeleteInMetabase = "could not mark object to delete in metabase" // Debug in ../node/pkg/local_object_storage/shard/inhume.go + InnerringNonalphabetModeDoNotStopContainerEstimations = "non-alphabet mode, do not stop container estimations" + InnerringCantStopEpochEstimation = "can't stop epoch estimation" + InnerringCantMakeNotaryDepositInMainChain = "can't make notary deposit in main chain" + InnerringCantMakeNotaryDepositInSideChain = "can't make notary deposit in side chain" + InnerringNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" + InnerringCantGetInnerRingIndex = "can't get inner ring index" + InnerringCantGetInnerRingSize = "can't get inner ring size" + InnerringCantGetAlphabetIndex = "can't get alphabet index" + InnerringIgnoreValidatorVoteNodeNotInAlphabetRange = "ignore validator vote: node not in alphabet range" + InnerringIgnoreValidatorVoteEmptyValidatorsList = "ignore validator vote: empty validators list" + InnerringCantInvokeVoteMethodInAlphabetContract = "can't invoke vote method in alphabet contract" + InnerringCantGetLastProcessedMainChainBlockNumber = "can't get last processed main chain block number" + InnerringNotarySupport = "notary support" + InnerringAlphabetKeysSyncIsDisabled = "alphabet keys sync is disabled" + InnerringNoControlServerEndpointSpecified = "no Control server endpoint specified, service is disabled" + InnerringCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" + InnerringFailedToSetGroupSignerScope = "failed to set group signer scope, continue with Global" + InnerringCantVoteForPreparedValidators = "can't vote for prepared validators" + InnerringNewBlock = "new block" + InnerringCantUpdatePersistentState = "can't update persistent state" + InnerringCloserError = "closer error" + InnerringReadConfigFromBlockchain = "read config from blockchain" + NotificatorNotificatorStartProcessingObjectNotifications = "notificator: start processing object notifications" + NotificatorNotificatorProcessingObjectNotification = "notificator: processing object notification" + PolicerCouldNotGetContainer = "could not get container" + PolicerCouldNotInhumeObjectWithMissingContainer = "could not inhume object with missing container" + PolicerCouldNotBuildPlacementVectorForObject = "could not build placement vector for object" + PolicerRedundantLocalObjectCopyDetected = "redundant local object copy detected" + PolicerReceiveObjectHeaderToCheckPolicyCompliance = "receive object header to check policy compliance" + PolicerConsiderNodeUnderMaintenanceAsOK = "consider node under maintenance as OK" + PolicerShortageOfObjectCopiesDetected = "shortage of object copies detected" + PolicerSomeOfTheCopiesAreStoredOnNodesUnderMaintenance = "some of the copies are stored on nodes under maintenance, save local copy" + PolicerRoutineStopped = "routine stopped" + PolicerFailureAtObjectSelectForReplication = "failure at object select for replication" + PolicerPoolSubmission = "pool submission" + ReplicatorFinishWork = "finish work" + ReplicatorCouldNotGetObjectFromLocalStorage = "could not get object from local storage" + ReplicatorCouldNotReplicateObject = "could not replicate object" + ReplicatorObjectSuccessfullyReplicated = "object successfully replicated" + TreeRedirectingTreeServiceQuery = "redirecting tree service query" + TreeBearerPresentedButNotAllowedByACL = "bearer presented but not allowed by ACL" + TreeCouldNotGetLastSynchronizedHeightForATree = "could not get last synchronized height for a tree" + TreeCouldNotUpdateLastSynchronizedHeightForATree = "could not update last synchronized height for a tree" + TreeSynchronizeTree = "synchronize tree" + TreeFailedToRunTreeSynchronizationOverAllNodes = "failed to run tree synchronization over all nodes" + TreeSyncingTrees = "syncing trees..." + TreeCouldNotFetchContainers = "could not fetch containers" + TreeTreesHaveBeenSynchronized = "trees have been synchronized" + TreeSyncingContainerTrees = "syncing container trees..." + TreeCouldNotSyncTrees = "could not sync trees" + TreeContainerTreesHaveBeenSynced = "container trees have been synced" + TreeCouldNotQueryTreesForSynchronization = "could not query trees for synchronization" + TreeRemovingRedundantTrees = "removing redundant trees..." + TreeCouldNotRemoveRedundantTree = "could not remove redundant tree" + TreeCouldNotCalculateContainerNodes = "could not calculate container nodes" + TreeFailedToApplyReplicatedOperation = "failed to apply replicated operation" + TreeDoNotSendUpdateToTheNode = "do not send update to the node" + TreeFailedToSentUpdateToTheNode = "failed to sent update to the node" + TreeErrorDuringReplication = "error during replication" + PersistentCouldNotGetSessionFromPersistentStorage = "could not get session from persistent storage" + PersistentCouldNotDeleteSToken = "could not delete token" + PersistentCouldNotCleanUpExpiredTokens = "could not clean up expired tokens" + ControllerReportIsAlreadyStarted = "report is already started" + TombstoneCouldNotGetTheTombstoneTheSource = "tombstone getter: could not get the tombstone the source" + DeleteRequestIsNotRolledOverToTheContainer = "request is not rolled over to the container" + DeleteCouldNotComposeSplitInfo = "could not compose split info" + DeleteNoSplitInfoObjectIsPHY = "no split info, object is PHY" + DeleteAssemblingChain = "assembling chain..." + DeleteCouldNotGetPreviousSplitElement = "could not get previous split element" + DeleteCollectingChildren = "collecting children..." + DeleteCouldNotCollectObjectChildren = "could not collect object children" + DeleteSupplementBySplitID = "supplement by split ID" + DeleteCouldNotSearchForSplitChainMembers = "could not search for split chain members" + DeleteCouldNotMarshalTombstoneStructure = "could not marshal tombstone structure" + DeleteCouldNotSaveTheTombstone = "could not save the tombstone" + DeleteFormingTombstoneStructure = "forming tombstone structure..." + DeleteTombstoneStructureSuccessfullyFormedSaving = "tombstone structure successfully formed, saving..." + DeleteCouldNotReadTombstoneLifetimeConfig = "could not read tombstone lifetime config" + DeleteFormingSplitInfo = "forming split info..." + DeleteSplitInfoSuccessfullyFormedCollectingMembers = "split info successfully formed, collecting members..." + DeleteMembersSuccessfullyCollected = "members successfully collected" + GetRemoteCallFailed = "remote call failed" + GetCanNotAssembleTheObject = "can not assemble the object" + GetTryingToAssembleTheObject = "trying to assemble the object..." + GetAssemblingSplittedObject = "assembling splitted object..." + GetAssemblingSplittedObjectCompleted = "assembling splitted object completed" + GetFailedToAssembleSplittedObject = "failed to assemble splitted object" + GetCouldNotGenerateContainerTraverser = "could not generate container traverser" + GetCouldNotConstructRemoteNodeClient = "could not construct remote node client" + GetCouldNotWriteHeader = "could not write header" + GetCouldNotWritePayloadChunk = "could not write payload chunk" + GetLocalGetFailed = "local get failed" + GetReturnResultDirectly = "return result directly" + GetCompletingTheOperation = "completing the operation" + GetRequestedObjectWasMarkedAsRemoved = "requested object was marked as removed" + GetRequestedObjectIsVirtual = "requested object is virtual" + GetRequestedRangeIsOutOfObjectBounds = "requested range is out of object bounds" + PutAdditionalContainerBroadcastFailure = "additional container broadcast failure" + SearchReturnResultDirectly = "return result directly" + SearchCouldNotConstructRemoteNodeClient = "could not construct remote node client" + SearchRemoteOperationFailed = "remote operation failed" + SearchCouldNotGenerateContainerTraverser = "could not generate container traverser" + SearchCouldNotWriteObjectIdentifiers = "could not write object identifiers" + SearchLocalOperationFailed = "local operation failed" + UtilObjectServiceError = "object service error" + UtilCouldNotPushTaskToWorkerPool = "could not push task to worker pool" + V2CantCheckIfRequestFromInnerRing = "can't check if request from inner ring" + V2CantCheckIfRequestFromContainerNode = "can't check if request from container node" + NatsNatsConnectionWasLost = "nats: connection was lost" + NatsNatsReconnectedToTheServer = "nats: reconnected to the server" + NatsNatsClosingConnectionAsTheContextIsDone = "nats: closing connection as the context is done" + ControllerStartingToAnnounceTheValuesOfTheMetrics = "starting to announce the values of the metrics" + ControllerCouldNotInitializeIteratorOverLocallyCollectedMetrics = "could not initialize iterator over locally collected metrics" + ControllerCouldNotInitializeAnnouncementAccumulator = "could not initialize announcement accumulator" + ControllerIteratorOverLocallyCollectedMetricsAborted = "iterator over locally collected metrics aborted" + ControllerCouldNotFinishWritingLocalAnnouncements = "could not finish writing local announcements" + ControllerTrustAnnouncementSuccessfullyFinished = "trust announcement successfully finished" + ControllerAnnouncementIsAlreadyStarted = "announcement is already started" + ControllerAnnouncementSuccessfullyInterrupted = "announcement successfully interrupted" + ControllerAnnouncementIsNotStartedOrAlreadyInterrupted = "announcement is not started or already interrupted" + ControllerCouldNotInitializeIteratorOverLocallyAccumulatedAnnouncements = "could not initialize iterator over locally accumulated announcements" + ControllerCouldNotInitializeResultTarget = "could not initialize result target" + ControllerIteratorOverLocalAnnouncementsAborted = "iterator over local announcements aborted" + ControllerCouldNotFinishWritingLoadEstimations = "could not finish writing load estimations" + RouteCouldNotInitializeWriterProvider = "could not initialize writer provider" + RouteCouldNotInitializeWriter = "could not initialize writer" + RouteCouldNotPutTheValue = "could not put the value" + RouteCouldNotCloseRemoteServerWriter = "could not close remote server writer" + ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch = "could not restore block subscription after RPC switch" + ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch = "could not restore notification subscription after RPC switch" + ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch = "could not restore notary notification subscription after RPC switch" + ClientCouldNotEstablishConnectionToTheSwitchedRPCNode = "could not establish connection to the switched RPC node" + ClientConnectionToTheNewRPCNodeHasBeenEstablished = "connection to the new RPC node has been established" + ClientSwitchingToTheNextRPCNode = "switching to the next RPC node" + ClientCouldNotEstablishConnectionToAnyRPCNode = "could not establish connection to any RPC node" + ClientCouldNotCreateClientToTheHigherPriorityNode = "could not create client to the higher priority node" + ClientSwitchedToTheHigherPriorityRPC = "switched to the higher priority RPC" + ClientCouldNotRestoreSideChainSubscriptionsUsingNode = "could not restore side chain subscriptions using node" + ClientNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" + ClientNotaryDepositInvoke = "notary deposit invoke" + ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked" + ClientNotaryRequestInvoked = "notary request invoked" + ClientNeoClientInvoke = "neo client invoke" + ClientNativeGasTransferInvoke = "native gas transfer invoke" + ClientBatchGasTransferInvoke = "batch gas transfer invoke" + ClientCantGetBlockchainHeight = "can't get blockchain height" + ClientCantGetBlockchainHeight243 = "can't get blockchain height" + EventCouldNotSubmitHandlerToWorkerPool = "could not Submit handler to worker pool" + EventCouldNotStartListenToEvents = "could not start listen to events" + EventStopEventListenerByError = "stop event listener by error" + EventStopEventListenerByContext = "stop event listener by context" + EventStopEventListenerByNotificationChannel = "stop event listener by notification channel" + EventNilNotificationEventWasCaught = "nil notification event was caught" + EventStopEventListenerByNotaryChannel = "stop event listener by notary channel" + EventNilNotaryEventWasCaught = "nil notary event was caught" + EventStopEventListenerByBlockChannel = "stop event listener by block channel" + EventNilBlockWasCaught = "nil block was caught" + EventListenerWorkerPoolDrained = "listener worker pool drained" + EventEventParserNotSet = "event parser not set" + EventCouldNotParseNotificationEvent = "could not parse notification event" + EventNotificationHandlersForParsedNotificationEventWereNotRegistered = "notification handlers for parsed notification event were not registered" + EventSkipExpiredMainTXNotaryEvent = "skip expired main TX notary event" + EventCouldNotPrepareAndValidateNotaryEvent = "could not prepare and validate notary event" + EventNotaryParserNotSet = "notary parser not set" + EventCouldNotParseNotaryEvent = "could not parse notary event" + EventNotaryHandlersForParsedNotificationEventWereNotRegistered = "notary handlers for parsed notification event were not registered" + EventIgnoreNilEventParser = "ignore nil event parser" + EventListenerHasBeenAlreadyStartedIgnoreParser = "listener has been already started, ignore parser" + EventRegisteredNewEventParser = "registered new event parser" + EventIgnoreNilEventHandler = "ignore nil event handler" + EventIgnoreHandlerOfEventWoParser = "ignore handler of event w/o parser" + EventRegisteredNewEventHandler = "registered new event handler" + EventIgnoreNilNotaryEventParser = "ignore nil notary event parser" + EventListenerHasBeenAlreadyStartedIgnoreNotaryParser = "listener has been already started, ignore notary parser" + EventIgnoreNilNotaryEventHandler = "ignore nil notary event handler" + EventIgnoreHandlerOfNotaryEventWoParser = "ignore handler of notary event w/o parser" + EventIgnoreNilBlockHandler = "ignore nil block handler" + SubscriberRemoteNotificationChannelHasBeenClosed = "remote notification channel has been closed" + SubscriberCantCastNotifyEventValueToTheNotifyStruct = "can't cast notify event value to the notify struct" + SubscriberNewNotificationEventFromSidechain = "new notification event from sidechain" + SubscriberCantCastBlockEventValueToBlock = "can't cast block event value to block" + SubscriberCantCastNotifyEventValueToTheNotaryRequestStruct = "can't cast notify event value to the notary request struct" + SubscriberUnsupportedNotificationFromTheChain = "unsupported notification from the chain" + BlobovniczaCreatingDirectoryForBoltDB = "creating directory for BoltDB" + BlobovniczaOpeningBoltDB = "opening BoltDB" + BlobovniczaInitializing = "initializing..." + BlobovniczaAlreadyInitialized = "already initialized" + BlobovniczaCreatingBucketForSizeRange = "creating bucket for size range" + BlobovniczaClosingBoltDB = "closing BoltDB" + BlobovniczaObjectWasRemovedFromBucket = "object was removed from bucket" + BlobstorOpening = "opening..." + BlobstorInitializing = "initializing..." + BlobstorClosing = "closing..." + BlobstorCouldntCloseStorage = "couldn't close storage" + BlobstorErrorOccurredDuringObjectExistenceChecking = "error occurred during object existence checking" + BlobstorErrorOccurredDuringTheIteration = "error occurred during the iteration" + EngineShardHasBeenRemoved = "shard has been removed" + EngineCouldNotCloseRemovedShard = "could not close removed shard" + EngineCouldNotOpenShardClosingAndSkipping = "could not open shard, closing and skipping" + EngineCouldNotClosePartiallyInitializedShard = "could not close partially initialized shard" + EngineCouldNotInitializeShardClosingAndSkipping = "could not initialize shard, closing and skipping" + EngineCouldNotCloseShard = "could not close shard" + EngineCouldNotReloadAShard = "could not reload a shard" + EngineAddedNewShard = "added new shard" + EngineCouldNotMarkObjectForShardRelocation = "could not mark object for shard relocation" + EngineCouldNotPutObjectToShard = "could not put object to shard" + EngineErrorDuringSearchingForObjectChildren = "error during searching for object children" + EngineCouldNotInhumeObjectInShard = "could not inhume object in shard" + EngineStartingRemovalOfLocallyredundantCopies = "starting removal of locally-redundant copies" + EngineStartedDuplicatesRemovalRoutine = "started duplicates removal routine" + EngineFinishedRemovalOfLocallyredundantCopies = "finished removal of locally-redundant copies" + EngineRemovingAnObjectWithoutFullLockingCheck = "removing an object without full locking check" + EngineInterruptProcessingTheExpiredLocks = "interrupt processing the expired locks" + EngineInterruptProcessingTheDeletedLocks = "interrupt processing the deleted locks" + EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly = "failed to move shard in degraded-read-only mode, moving to read-only" + EngineFailedToMoveShardInReadonlyMode = "failed to move shard in read-only mode" + EngineShardIsMovedInReadonlyModeDueToErrorThreshold = "shard is moved in read-only mode due to error threshold" + EngineShardIsMovedInDegradedModeDueToErrorThreshold = "shard is moved in degraded mode due to error threshold" + EngineModeChangeIsInProgressIgnoringSetmodeRequest = "mode change is in progress, ignoring set-mode request" + EngineStartedShardsEvacuation = "started shards evacuation" + EngineFinishedSuccessfullyShardsEvacuation = "shards evacuation finished successfully" + EngineFinishedWithErrorShardsEvacuation = "shards evacuation finished with error" + EngineObjectIsMovedToAnotherShard = "object is moved to another shard" + MetabaseMissingMatcher = "missing matcher" + MetabaseErrorInFKBTSelection = "error in FKBT selection" + MetabaseCantDecodeListBucketLeaf = "can't decode list bucket leaf" + MetabaseUnknownOperation = "unknown operation" + MetabaseCantIterateOverTheBucket = "can't iterate over the bucket" + MetabaseCouldNotIterateOverTheBuckets = "could not iterate over the buckets" + MetabaseCreatedDirectoryForMetabase = "created directory for Metabase" + MetabaseOpenedBoltDBInstanceForMetabase = "opened boltDB instance for Metabase" + MetabaseCheckingMetabaseVersion = "checking metabase version" + ShardCantSelectAllObjects = "can't select all objects" + ShardSettingShardMode = "setting shard mode" + ShardShardModeSetSuccessfully = "shard mode set successfully" + ShardCouldNotMarkObjectForShardRelocationInMetabase = "could not mark object for shard relocation in metabase" + ShardCantDeleteObjectFromWriteCache = "can't delete object from write cache" + ShardCantGetStorageIDFromMetabase = "can't get storage ID from metabase" + ShardCantRemoveObjectFromBlobStor = "can't remove object from blobStor" + ShardFetchingObjectWithoutMeta = "fetching object without meta" + ShardObjectIsMissingInWritecache = "object is missing in write-cache" + ShardFailedToFetchObjectFromWritecache = "failed to fetch object from write-cache" + ShardCantPutObjectToTheWritecacheTryingBlobstor = "can't put object to the write-cache, trying blobstor" + ShardMetaObjectCounterRead = "meta: object counter read" + ShardMetaCantReadContainerList = "meta: can't read container list" + ShardMetaCantReadContainerSize = "meta: can't read container size" + ShardMetabaseFailureSwitchingMode = "metabase failure, switching mode" + ShardCantMoveShardToReadonlySwitchMode = "can't move shard to readonly, switch mode" + ShardCouldNotUnmarshalObject = "could not unmarshal object" + ShardCouldNotCloseShardComponent = "could not close shard component" + ShardCantOpenMetabaseMoveToADegradedMode = "can't open metabase, move to a degraded mode" + ShardCantInitializeMetabaseMoveToADegradedreadonlyMode = "can't initialize metabase, move to a degraded-read-only mode" + ShardTryingToRestoreReadwriteMode = "trying to restore read-write mode" + ShardStopEventListenerByClosedChannel = "stop event listener by closed channel" + ShardCouldNotSubmitGCJobToWorkerPool = "could not submit GC job to worker pool" + ShardGCIsStopped = "GC is stopped" + ShardWaitingForGCWorkersToStop = "waiting for GC workers to stop..." + ShardIteratorOverMetabaseGraveyardFailed = "iterator over metabase graveyard failed" + ShardCouldNotDeleteTheObjects = "could not delete the objects" + ShardIteratorOverExpiredObjectsFailed = "iterator over expired objects failed" + ShardCouldNotInhumeTheObjects = "could not inhume the objects" + ShardStartedExpiredTombstonesHandling = "started expired tombstones handling" + ShardIteratingTombstones = "iterating tombstones" + ShardShardIsInADegradedModeSkipCollectingExpiredTombstones = "shard is in a degraded mode, skip collecting expired tombstones" + ShardIteratorOverGraveyardFailed = "iterator over graveyard failed" + ShardHandlingExpiredTombstonesBatch = "handling expired tombstones batch" + ShardFinishedExpiredTombstonesHandling = "finished expired tombstones handling" + ShardIteratorOverExpiredLocksFailed = "iterator over expired locks failed" + ShardCouldNotMarkTombstonesAsGarbage = "could not mark tombstones as garbage" + ShardCouldNotDropExpiredGraveRecords = "could not drop expired grave records" + ShardFailureToUnlockObjects = "failure to unlock objects" + ShardFailureToMarkLockersAsGarbage = "failure to mark lockers as garbage" + ShardFailureToGetExpiredUnlockedObjects = "failure to get expired unlocked objects" + ShardCouldNotMarkObjectToDeleteInMetabase = "could not mark object to delete in metabase" WritecacheBadgerInitExperimental = "initializing badger-backed experimental writecache" - WritecacheTriedToFlushItemsFromWritecache = "tried to flush items from write-cache" // Debug in ../node/pkg/local_object_storage/writecache/flush.go - WritecacheWaitingForChannelsToFlush = "waiting for channels to flush" // Info in ../node/pkg/local_object_storage/writecache/mode.go - WritecacheFillingFlushMarksForObjectsInFSTree = "filling flush marks for objects in FSTree" // Info in ../node/pkg/local_object_storage/writecache/init.go - WritecacheFinishedUpdatingFSTreeFlushMarks = "finished updating FSTree flush marks" // Info in ../node/pkg/local_object_storage/writecache/init.go - WritecacheFillingFlushMarksForObjectsInDatabase = "filling flush marks for objects in database" // Info in ../node/pkg/local_object_storage/writecache/init.go - WritecacheFinishedUpdatingFlushMarks = "finished updating flush marks" // Info in ../node/pkg/local_object_storage/writecache/init.go - WritecacheCantRemoveObjectsFromTheDatabase = "can't remove objects from the database" // Error in ../node/pkg/local_object_storage/writecache/storage.go - WritecacheCantParseAddress = "can't parse address" // Error in ../node/pkg/local_object_storage/writecache/storage.go - WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" // Error in ../node/pkg/local_object_storage/writecache/storage.go + WritecacheTriedToFlushItemsFromWritecache = "tried to flush items from write-cache" + WritecacheWaitingForChannelsToFlush = "waiting for channels to flush" + WritecacheFillingFlushMarksForObjectsInFSTree = "filling flush marks for objects in FSTree" + WritecacheFinishedUpdatingFSTreeFlushMarks = "finished updating FSTree flush marks" + WritecacheFillingFlushMarksForObjectsInDatabase = "filling flush marks for objects in database" + WritecacheFinishedUpdatingFlushMarks = "finished updating flush marks" + WritecacheCantRemoveObjectsFromTheDatabase = "can't remove objects from the database" + WritecacheCantParseAddress = "can't parse address" + WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" WritecacheDBValueLogGCRunCompleted = "value log GC run completed" - BlobovniczatreeCouldNotGetObjectFromLevel = "could not get object from level" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go - BlobovniczatreeCouldNotReadPayloadRangeFromOpenedBlobovnicza = "could not read payload range from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go - BlobovniczatreeCouldNotReadPayloadRangeFromActiveBlobovnicza = "could not read payload range from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go - BlobovniczatreeCouldNotCloseBlobovnicza = "could not close Blobovnicza" // Error in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go - BlobovniczatreeBlobovniczaSuccessfullyClosedOnEvict = "blobovnicza successfully closed on evict" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go - BlobovniczatreeUpdatingActiveBlobovnicza = "updating active blobovnicza..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go - BlobovniczatreeActiveBlobovniczaSuccessfullyUpdated = "active blobovnicza successfully updated" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go - BlobovniczatreeBlobovniczaSuccessfullyActivated = "blobovnicza successfully activated" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go - BlobovniczatreeCouldNotRemoveObjectFromLevel = "could not remove object from level" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go - BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza = "could not remove object from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go - BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza = "could not remove object from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/delete.go - BlobovniczatreeCouldNotGetActiveBlobovnicza = "could not get active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go - BlobovniczatreeBlobovniczaOverflowed = "blobovnicza overflowed" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go - BlobovniczatreeCouldNotUpdateActiveBlobovnicza = "could not update active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go - BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza = "could not put object to active blobovnicza" // reportError in ../node/pkg/local_object_storage/blobstor/blobovniczatree/put.go - BlobovniczatreeCouldNotReadObjectFromOpenedBlobovnicza = "could not read object from opened blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get.go - BlobovniczatreeCouldNotGetObjectFromActiveBlobovnicza = "could not get object from active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/get.go - BlobovniczatreeInitializingBlobovniczas = "initializing Blobovnicza's" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go - BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization = "read-only mode, skip blobovniczas initialization..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go - BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing = "blobovnicza successfully initialized, closing..." // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go - BlobovniczatreeCouldNotCloseActiveBlobovnicza = "could not close active blobovnicza" // Debug in ../node/pkg/local_object_storage/blobstor/blobovniczatree/control.go - AlphabetTick = "tick" // Info in ../node/pkg/innerring/processors/alphabet/handlers.go - AlphabetAlphabetProcessorWorkerPoolDrained = "alphabet processor worker pool drained" // Warn in ../node/pkg/innerring/processors/alphabet/handlers.go - AlphabetNonAlphabetModeIgnoreGasEmissionEvent = "non alphabet mode, ignore gas emission event" // Info in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent = "node is out of alphabet range, ignore gas emission event" // Debug in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetCantInvokeAlphabetEmitMethod = "can't invoke alphabet emit method" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetStorageNodeEmissionIsOff = "storage node emission is off" // Info in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes = "can't get netmap snapshot to emit gas to storage nodes" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetGasEmission = "gas emission" // Debug in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetCantParseNodePublicKey = "can't parse node public key" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetCantTransferGas = "can't transfer gas" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetCantTransferGasToWallet = "can't transfer gas to wallet" // Warn in ../node/pkg/innerring/processors/alphabet/process_emit.go - AlphabetAlphabetWorkerPool = "alphabet worker pool" // Debug in ../node/pkg/innerring/processors/alphabet/processor.go - BalanceBalanceWorkerPoolDrained = "balance worker pool drained" // Warn in ../node/pkg/innerring/processors/balance/handlers.go - BalanceNonAlphabetModeIgnoreBalanceLock = "non alphabet mode, ignore balance lock" // Info in ../node/pkg/innerring/processors/balance/process_assets.go - BalanceCantSendLockAssetTx = "can't send lock asset tx" // Error in ../node/pkg/innerring/processors/balance/process_assets.go - BalanceBalanceWorkerPool = "balance worker pool" // Debug in ../node/pkg/innerring/processors/balance/processor.go - ContainerContainerWorkerPool = "container worker pool" // Debug in ../node/pkg/innerring/processors/container/processor.go - ContainerContainerProcessorWorkerPoolDrained = "container processor worker pool drained" // Warn in ../node/pkg/innerring/processors/container/handlers.go - ContainerNonAlphabetModeIgnoreContainerPut = "non alphabet mode, ignore container put" // Info in ../node/pkg/innerring/processors/container/process_container.go - ContainerPutContainerCheckFailed = "put container check failed" // Error in ../node/pkg/innerring/processors/container/process_container.go - ContainerCouldNotApprovePutContainer = "could not approve put container" // Error in ../node/pkg/innerring/processors/container/process_container.go - ContainerNonAlphabetModeIgnoreContainerDelete = "non alphabet mode, ignore container delete" // Info in ../node/pkg/innerring/processors/container/process_container.go - ContainerDeleteContainerCheckFailed = "delete container check failed" // Error in ../node/pkg/innerring/processors/container/process_container.go - ContainerCouldNotApproveDeleteContainer = "could not approve delete container" // Error in ../node/pkg/innerring/processors/container/process_container.go - ContainerNonAlphabetModeIgnoreSetEACL = "non alphabet mode, ignore set EACL" // Info in ../node/pkg/innerring/processors/container/process_eacl.go - ContainerSetEACLCheckFailed = "set EACL check failed" // Error in ../node/pkg/innerring/processors/container/process_eacl.go - ContainerCouldNotApproveSetEACL = "could not approve set EACL" // Error in ../node/pkg/innerring/processors/container/process_eacl.go - FrostFSNonAlphabetModeIgnoreBind = "non alphabet mode, ignore bind" // Info in ../node/pkg/innerring/processors/frostfs/process_bind.go - FrostFSInvalidManageKeyEvent = "invalid manage key event" // Error in ../node/pkg/innerring/processors/frostfs/process_bind.go - FrostFSCouldNotDecodeScriptHashFromBytes = "could not decode script hash from bytes" // Error in ../node/pkg/innerring/processors/frostfs/process_bind.go - FrostFSNonAlphabetModeIgnoreConfig = "non alphabet mode, ignore config" // Info in ../node/pkg/innerring/processors/frostfs/process_config.go - FrostFSCantRelaySetConfigEvent = "can't relay set config event" // Error in ../node/pkg/innerring/processors/frostfs/process_config.go - FrostFSFrostfsWorkerPool = "frostfs worker pool" // Debug in ../node/pkg/innerring/processors/frostfs/processor.go - FrostFSFrostfsProcessorWorkerPoolDrained = "frostfs processor worker pool drained" // Warn in ../node/pkg/innerring/processors/frostfs/handlers.go - FrostFSNonAlphabetModeIgnoreDeposit = "non alphabet mode, ignore deposit" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantTransferAssetsToBalanceContract = "can't transfer assets to balance contract" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSDoubleMintEmissionDeclined = "double mint emission declined" // Warn in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantGetGasBalanceOfTheNode = "can't get gas balance of the node" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSGasBalanceThresholdHasBeenReached = "gas balance threshold has been reached" // Warn in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantTransferNativeGasToReceiver = "can't transfer native gas to receiver" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSNonAlphabetModeIgnoreWithdraw = "non alphabet mode, ignore withdraw" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantCreateLockAccount = "can't create lock account" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantLockAssetsForWithdraw = "can't lock assets for withdraw" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSNonAlphabetModeIgnoreCheque = "non alphabet mode, ignore cheque" // Info in ../node/pkg/innerring/processors/frostfs/process_assets.go - FrostFSCantTransferAssetsToFedContract = "can't transfer assets to fed contract" // Error in ../node/pkg/innerring/processors/frostfs/process_assets.go - GovernanceNewEvent = "new event" // Info in ../node/pkg/innerring/processors/governance/handlers.go - GovernanceGovernanceWorkerPoolDrained = "governance worker pool drained" // Warn in ../node/pkg/innerring/processors/governance/handlers.go - GovernanceNonAlphabetModeIgnoreAlphabetSync = "non alphabet mode, ignore alphabet sync" // Info in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantFetchAlphabetListFromMainNet = "can't fetch alphabet list from main net" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantFetchAlphabetListFromSideChain = "can't fetch alphabet list from side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantMergeAlphabetListsFromMainNetAndSideChain = "can't merge alphabet lists from main net and side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged = "no governance update, alphabet list has not been changed" // Info in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceAlphabetListHasBeenChangedStartingUpdate = "alphabet list has been changed, starting update" // Info in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantVoteForSideChainCommittee = "can't vote for side chain committee" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceFinishedAlphabetListUpdate = "finished alphabet list update" // Info in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantFetchInnerRingListFromSideChain = "can't fetch inner ring list from side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys = "can't create new inner ring list with new alphabet keys" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceUpdateOfTheInnerRingList = "update of the inner ring list" // Info in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantUpdateInnerRingListWithNewAlphabetKeys = "can't update inner ring list with new alphabet keys" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantUpdateListOfNotaryNodesInSideChain = "can't update list of notary nodes in side chain" // Error in ../node/pkg/innerring/processors/governance/process_update.go - GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract = "can't update list of alphabet nodes in frostfs contract" // Error in ../node/pkg/innerring/processors/governance/process_update.go - NetmapNetmapWorkerPool = "netmap worker pool" // Debug in ../node/pkg/innerring/processors/netmap/processor.go - NetmapTick = "tick" // Info in ../node/pkg/innerring/processors/netmap/handlers.go - NetmapNetmapWorkerPoolDrained = "netmap worker pool drained" // Warn in ../node/pkg/innerring/processors/netmap/handlers.go - NetmapNetmapCleanUpRoutineIsDisabled518 = "netmap clean up routine is disabled" // Debug in ../node/pkg/innerring/processors/netmap/handlers.go - NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick = "non alphabet mode, ignore new netmap cleanup tick" // Info in ../node/pkg/innerring/processors/netmap/process_cleanup.go - NetmapCantDecodePublicKeyOfNetmapNode = "can't decode public key of netmap node" // Warn in ../node/pkg/innerring/processors/netmap/process_cleanup.go - NetmapVoteToRemoveNodeFromNetmap = "vote to remove node from netmap" // Info in ../node/pkg/innerring/processors/netmap/process_cleanup.go - NetmapCantInvokeNetmapUpdateState = "can't invoke netmap.UpdateState" // Error in ../node/pkg/innerring/processors/netmap/process_cleanup.go - NetmapCantIterateOnNetmapCleanerCache = "can't iterate on netmap cleaner cache" // Warn in ../node/pkg/innerring/processors/netmap/process_cleanup.go - NetmapCantGetEpochDuration = "can't get epoch duration" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapCantGetTransactionHeight = "can't get transaction height" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapCantResetEpochTimer = "can't reset epoch timer" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapCantGetNetmapSnapshotToPerformCleanup = "can't get netmap snapshot to perform cleanup" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapCantStartContainerSizeEstimation = "can't start container size estimation" // Warn in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapNonAlphabetModeIgnoreNewEpochTick = "non alphabet mode, ignore new epoch tick" // Info in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapNextEpoch = "next epoch" // Debug in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapCantInvokeNetmapNewEpoch = "can't invoke netmap.NewEpoch" // Error in ../node/pkg/innerring/processors/netmap/process_epoch.go - NetmapNonAlphabetModeIgnoreNewPeerNotification = "non alphabet mode, ignore new peer notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapNonhaltNotaryTransaction = "non-halt notary transaction" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCantParseNetworkMapCandidate = "can't parse network map candidate" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate = "could not verify and update information about network map candidate" // Warn in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapApprovingNetworkMapCandidate = "approving network map candidate" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCantInvokeNetmapAddPeer = "can't invoke netmap.AddPeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapNonAlphabetModeIgnoreUpdatePeerNotification = "non alphabet mode, ignore update peer notification" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapPreventSwitchingNodeToMaintenanceState = "prevent switching node to maintenance state" // Info in ../node/pkg/innerring/processors/netmap/process_peers.go - NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" // Error in ../node/pkg/innerring/processors/netmap/process_peers.go - FrostFSIRInternalError = "internal error" // Info in ../node/cmd/frostfs-ir/main.go - FrostFSIRCouldNotShutdownHTTPServer = "could not shutdown HTTP server" // Debug in ../node/cmd/frostfs-ir/main.go - FrostFSIRApplicationStopped = "application stopped" // Info in ../node/cmd/frostfs-ir/main.go - FrostFSIRCouldntCreateRPCClientForEndpoint = "could not create RPC client for endpoint" // Debug in ../node/pkg/morph/client/constructor.go - FrostFSIRCreatedRPCClientForEndpoint = "created RPC client for endpoint" // Info in ../node/pkg/morph/client/constructor.go - FrostFSIRReloadExtraWallets = "reload extra wallets" // Info in ../node/cmd/frostfs-ir/config.go - FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" // Error in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" // Error in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeStartListeningGRPCEndpoint = "start listening gRPC endpoint" // Info in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeStoppingGRPCServer = "stopping gRPC server..." // Info in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeGRPCCannotShutdownGracefullyForcingStop = "gRPC cannot shutdown gracefully, forcing stop" // Info in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeGRPCServerStoppedSuccessfully = "gRPC server stopped successfully" // Info in ../node/cmd/frostfs-node/grpc.go - FrostFSNodeWaitingForAllProcessesToStop = "waiting for all processes to stop" // Debug in ../node/cmd/frostfs-node/main.go - FrostFSNodeStartedLocalNodesMaintenance = "started local node's maintenance" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeStoppedLocalNodesMaintenance = "stopped local node's maintenance" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeFailedToAttachShardToEngine = "failed to attach shard to engine" // Error in ../node/cmd/frostfs-node/config.go - FrostFSNodeShardAttachedToEngine = "shard attached to engine" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeClosingComponentsOfTheStorageEngine = "closing components of the storage engine..." // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeStorageEngineClosingFailure = "storage engine closing failure" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully = "all components of the storage engine closed successfully" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeBootstrappingWithTheMaintenanceState = "bootstrapping with the maintenance state" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeBootstrappingWithOnlineState = "bootstrapping with online state" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeTerminationSignalHasBeenReceivedStopping = "termination signal has been received, stopping..." // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeTerminationSignalProcessingIsComplete = "termination signal processing is complete" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeInternalApplicationError = "internal application error" // Warn in ../node/cmd/frostfs-node/config.go - FrostFSNodeInternalErrorProcessingIsComplete = "internal error processing is complete" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration = "SIGHUP has been received, rereading configuration..." // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeConfigurationReading = "configuration reading" // Error in ../node/cmd/frostfs-node/config.go - FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" // Error in ../node/cmd/frostfs-node/config.go - FrostFSNodeTracingConfigationUpdated = "tracing configation updated" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeStorageEngineConfigurationUpdate = "storage engine configuration update" // Error in ../node/cmd/frostfs-node/config.go - FrostFSNodeUpdatedConfigurationApplying = "updated configuration applying" // Error in ../node/cmd/frostfs-node/config.go - FrostFSNodeConfigurationHasBeenReloadedSuccessfully = "configuration has been reloaded successfully" // Info in ../node/cmd/frostfs-node/config.go - FrostFSNodeReadNewlyCreatedContainerAfterTheNotification = "read newly created container after the notification" // Error in ../node/cmd/frostfs-node/container.go - FrostFSNodeContainerCreationEventsReceipt = "container creation event's receipt" // Debug in ../node/cmd/frostfs-node/container.go - FrostFSNodeContainerRemovalEventsReceipt = "container removal event's receipt" // Debug in ../node/cmd/frostfs-node/container.go - FrostFSNodeSaveUsedSpaceAnnouncementInContract = "save used space announcement in contract" // Debug in ../node/cmd/frostfs-node/container.go - FrostFSNodeFailedToCalculateContainerSizeInStorageEngine = "failed to calculate container size in storage engine" // Debug in ../node/cmd/frostfs-node/container.go - FrostFSNodeContainerSizeInStorageEngineCalculatedSuccessfully = "container size in storage engine calculated successfully" // Debug in ../node/cmd/frostfs-node/container.go - FrostFSNodeNotificatorCouldNotListContainers = "notificator: could not list containers" // Error in ../node/cmd/frostfs-node/notificator.go - FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer = "notificator: could not select objects from container" // Error in ../node/cmd/frostfs-node/notificator.go - FrostFSNodeNotificatorCouldNotProcessObject = "notificator: could not process object" // Error in ../node/cmd/frostfs-node/notificator.go - FrostFSNodeNotificatorFinishedProcessingObjectNotifications = "notificator: finished processing object notifications" // Debug in ../node/cmd/frostfs-node/notificator.go - FrostFSNodeCouldNotWriteObjectNotification = "could not write object notification" // Warn in ../node/cmd/frostfs-node/notificator.go - FrostFSNodeCouldNotGetMaxObjectSizeValue = "could not get max object size value" // Error in ../node/cmd/frostfs-node/object.go - FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage = "could not inhume mark redundant copy as garbage" // Warn in ../node/cmd/frostfs-node/object.go - FrostFSNodeFailedInitTracing = "failed init tracing" // Error in ../node/cmd/frostfs-node/tracing.go - FrostFSNodeFailedShutdownTracing = "failed shutdown tracing" // Error in ../node/cmd/frostfs-node/tracing.go - FrostFSNodeFailedToCreateNeoRPCClient = "failed to create neo RPC client" // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeClosingMorphComponents = "closing morph components..." // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal = "failed to set group signer scope, continue with Global" // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeNotarySupport = "notary support" // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeMorphcacheTTLFetchedFromNetwork = "morph.cache_ttl fetched from network" // Debug in ../node/cmd/frostfs-node/morph.go - FrostFSNodeNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" // Warn in ../node/cmd/frostfs-node/morph.go - FrostFSNodeNewEpochEventFromSidechain = "new epoch event from sidechain" // Info in ../node/cmd/frostfs-node/morph.go - FrostFSNodeNewBlock = "new block" // Debug in ../node/cmd/frostfs-node/morph.go - FrostFSNodeCantUpdatePersistentState = "can't update persistent state" // Warn in ../node/cmd/frostfs-node/morph.go - FrostFSNodeCantSendRebootstrapTx = "can't send re-bootstrap tx" // Warn in ../node/cmd/frostfs-node/netmap.go - FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch = "could not update node state on new epoch" // Error in ../node/cmd/frostfs-node/netmap.go - FrostFSNodeCouldNotMakeNotaryDeposit = "could not make notary deposit" // Error in ../node/cmd/frostfs-node/netmap.go - FrostFSNodeInitialNetworkState = "initial network state" // Info in ../node/cmd/frostfs-node/netmap.go - FrostFSNodeTreeServiceIsNotEnabledSkipInitialization = "tree service is not enabled, skip initialization" // Info in ../node/cmd/frostfs-node/tree.go - FrostFSNodeCouldNotSynchronizeTreeService = "could not synchronize Tree Service" // Error in ../node/cmd/frostfs-node/tree.go - FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" // Debug in ../node/cmd/frostfs-node/tree.go - FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" // Error in ../node/cmd/frostfs-node/tree.go - FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" // Error in ../node/cmd/frostfs-node/control.go + BlobovniczatreeCouldNotGetObjectFromLevel = "could not get object from level" + BlobovniczatreeCouldNotReadPayloadRangeFromOpenedBlobovnicza = "could not read payload range from opened blobovnicza" + BlobovniczatreeCouldNotReadPayloadRangeFromActiveBlobovnicza = "could not read payload range from active blobovnicza" + BlobovniczatreeCouldNotCloseBlobovnicza = "could not close Blobovnicza" + BlobovniczatreeBlobovniczaSuccessfullyClosedOnEvict = "blobovnicza successfully closed on evict" + BlobovniczatreeUpdatingActiveBlobovnicza = "updating active blobovnicza..." + BlobovniczatreeActiveBlobovniczaSuccessfullyUpdated = "active blobovnicza successfully updated" + BlobovniczatreeBlobovniczaSuccessfullyActivated = "blobovnicza successfully activated" + BlobovniczatreeCouldNotRemoveObjectFromLevel = "could not remove object from level" + BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza = "could not remove object from opened blobovnicza" + BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza = "could not remove object from active blobovnicza" + BlobovniczatreeCouldNotGetActiveBlobovnicza = "could not get active blobovnicza" + BlobovniczatreeBlobovniczaOverflowed = "blobovnicza overflowed" + BlobovniczatreeCouldNotUpdateActiveBlobovnicza = "could not update active blobovnicza" + BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza = "could not put object to active blobovnicza" + BlobovniczatreeCouldNotReadObjectFromOpenedBlobovnicza = "could not read object from opened blobovnicza" + BlobovniczatreeCouldNotGetObjectFromActiveBlobovnicza = "could not get object from active blobovnicza" + BlobovniczatreeInitializingBlobovniczas = "initializing Blobovnicza's" + BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization = "read-only mode, skip blobovniczas initialization..." + BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing = "blobovnicza successfully initialized, closing..." + BlobovniczatreeCouldNotCloseActiveBlobovnicza = "could not close active blobovnicza" + AlphabetTick = "tick" + AlphabetAlphabetProcessorWorkerPoolDrained = "alphabet processor worker pool drained" + AlphabetNonAlphabetModeIgnoreGasEmissionEvent = "non alphabet mode, ignore gas emission event" + AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent = "node is out of alphabet range, ignore gas emission event" + AlphabetCantInvokeAlphabetEmitMethod = "can't invoke alphabet emit method" + AlphabetStorageNodeEmissionIsOff = "storage node emission is off" + AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes = "can't get netmap snapshot to emit gas to storage nodes" + AlphabetGasEmission = "gas emission" + AlphabetCantParseNodePublicKey = "can't parse node public key" + AlphabetCantTransferGas = "can't transfer gas" + AlphabetCantTransferGasToWallet = "can't transfer gas to wallet" + AlphabetAlphabetWorkerPool = "alphabet worker pool" + BalanceBalanceWorkerPoolDrained = "balance worker pool drained" + BalanceNonAlphabetModeIgnoreBalanceLock = "non alphabet mode, ignore balance lock" + BalanceCantSendLockAssetTx = "can't send lock asset tx" + BalanceBalanceWorkerPool = "balance worker pool" + ContainerContainerWorkerPool = "container worker pool" + ContainerContainerProcessorWorkerPoolDrained = "container processor worker pool drained" + ContainerNonAlphabetModeIgnoreContainerPut = "non alphabet mode, ignore container put" + ContainerPutContainerCheckFailed = "put container check failed" + ContainerCouldNotApprovePutContainer = "could not approve put container" + ContainerNonAlphabetModeIgnoreContainerDelete = "non alphabet mode, ignore container delete" + ContainerDeleteContainerCheckFailed = "delete container check failed" + ContainerCouldNotApproveDeleteContainer = "could not approve delete container" + ContainerNonAlphabetModeIgnoreSetEACL = "non alphabet mode, ignore set EACL" + ContainerSetEACLCheckFailed = "set EACL check failed" + ContainerCouldNotApproveSetEACL = "could not approve set EACL" + FrostFSNonAlphabetModeIgnoreBind = "non alphabet mode, ignore bind" + FrostFSInvalidManageKeyEvent = "invalid manage key event" + FrostFSCouldNotDecodeScriptHashFromBytes = "could not decode script hash from bytes" + FrostFSNonAlphabetModeIgnoreConfig = "non alphabet mode, ignore config" + FrostFSCantRelaySetConfigEvent = "can't relay set config event" + FrostFSFrostfsWorkerPool = "frostfs worker pool" + FrostFSFrostfsProcessorWorkerPoolDrained = "frostfs processor worker pool drained" + FrostFSNonAlphabetModeIgnoreDeposit = "non alphabet mode, ignore deposit" + FrostFSCantTransferAssetsToBalanceContract = "can't transfer assets to balance contract" + FrostFSDoubleMintEmissionDeclined = "double mint emission declined" + FrostFSCantGetGasBalanceOfTheNode = "can't get gas balance of the node" + FrostFSGasBalanceThresholdHasBeenReached = "gas balance threshold has been reached" + FrostFSCantTransferNativeGasToReceiver = "can't transfer native gas to receiver" + FrostFSNonAlphabetModeIgnoreWithdraw = "non alphabet mode, ignore withdraw" + FrostFSCantCreateLockAccount = "can't create lock account" + FrostFSCantLockAssetsForWithdraw = "can't lock assets for withdraw" + FrostFSNonAlphabetModeIgnoreCheque = "non alphabet mode, ignore cheque" + FrostFSCantTransferAssetsToFedContract = "can't transfer assets to fed contract" + GovernanceNewEvent = "new event" + GovernanceGovernanceWorkerPoolDrained = "governance worker pool drained" + GovernanceNonAlphabetModeIgnoreAlphabetSync = "non alphabet mode, ignore alphabet sync" + GovernanceCantFetchAlphabetListFromMainNet = "can't fetch alphabet list from main net" + GovernanceCantFetchAlphabetListFromSideChain = "can't fetch alphabet list from side chain" + GovernanceCantMergeAlphabetListsFromMainNetAndSideChain = "can't merge alphabet lists from main net and side chain" + GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged = "no governance update, alphabet list has not been changed" + GovernanceAlphabetListHasBeenChangedStartingUpdate = "alphabet list has been changed, starting update" + GovernanceCantVoteForSideChainCommittee = "can't vote for side chain committee" + GovernanceFinishedAlphabetListUpdate = "finished alphabet list update" + GovernanceCantFetchInnerRingListFromSideChain = "can't fetch inner ring list from side chain" + GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys = "can't create new inner ring list with new alphabet keys" + GovernanceUpdateOfTheInnerRingList = "update of the inner ring list" + GovernanceCantUpdateInnerRingListWithNewAlphabetKeys = "can't update inner ring list with new alphabet keys" + GovernanceCantUpdateListOfNotaryNodesInSideChain = "can't update list of notary nodes in side chain" + GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract = "can't update list of alphabet nodes in frostfs contract" + NetmapNetmapWorkerPool = "netmap worker pool" + NetmapTick = "tick" + NetmapNetmapWorkerPoolDrained = "netmap worker pool drained" + NetmapNetmapCleanUpRoutineIsDisabled518 = "netmap clean up routine is disabled" + NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick = "non alphabet mode, ignore new netmap cleanup tick" + NetmapCantDecodePublicKeyOfNetmapNode = "can't decode public key of netmap node" + NetmapVoteToRemoveNodeFromNetmap = "vote to remove node from netmap" + NetmapCantInvokeNetmapUpdateState = "can't invoke netmap.UpdateState" + NetmapCantIterateOnNetmapCleanerCache = "can't iterate on netmap cleaner cache" + NetmapCantGetEpochDuration = "can't get epoch duration" + NetmapCantGetTransactionHeight = "can't get transaction height" + NetmapCantResetEpochTimer = "can't reset epoch timer" + NetmapCantGetNetmapSnapshotToPerformCleanup = "can't get netmap snapshot to perform cleanup" + NetmapCantStartContainerSizeEstimation = "can't start container size estimation" + NetmapNonAlphabetModeIgnoreNewEpochTick = "non alphabet mode, ignore new epoch tick" + NetmapNextEpoch = "next epoch" + NetmapCantInvokeNetmapNewEpoch = "can't invoke netmap.NewEpoch" + NetmapNonAlphabetModeIgnoreNewPeerNotification = "non alphabet mode, ignore new peer notification" + NetmapNonhaltNotaryTransaction = "non-halt notary transaction" + NetmapCantParseNetworkMapCandidate = "can't parse network map candidate" + NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate = "could not verify and update information about network map candidate" + NetmapApprovingNetworkMapCandidate = "approving network map candidate" + NetmapCantInvokeNetmapAddPeer = "can't invoke netmap.AddPeer" + NetmapNonAlphabetModeIgnoreUpdatePeerNotification = "non alphabet mode, ignore update peer notification" + NetmapPreventSwitchingNodeToMaintenanceState = "prevent switching node to maintenance state" + NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" + FrostFSIRInternalError = "internal error" + FrostFSIRCouldNotShutdownHTTPServer = "could not shutdown HTTP server" + FrostFSIRApplicationStopped = "application stopped" + FrostFSIRCouldntCreateRPCClientForEndpoint = "could not create RPC client for endpoint" + FrostFSIRCreatedRPCClientForEndpoint = "created RPC client for endpoint" + FrostFSIRReloadExtraWallets = "reload extra wallets" + FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" + FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" + FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" + FrostFSNodeStartListeningGRPCEndpoint = "start listening gRPC endpoint" + FrostFSNodeStoppingGRPCServer = "stopping gRPC server..." + FrostFSNodeGRPCCannotShutdownGracefullyForcingStop = "gRPC cannot shutdown gracefully, forcing stop" + FrostFSNodeGRPCServerStoppedSuccessfully = "gRPC server stopped successfully" + FrostFSNodeWaitingForAllProcessesToStop = "waiting for all processes to stop" + FrostFSNodeStartedLocalNodesMaintenance = "started local node's maintenance" + FrostFSNodeStoppedLocalNodesMaintenance = "stopped local node's maintenance" + FrostFSNodeFailedToAttachShardToEngine = "failed to attach shard to engine" + FrostFSNodeShardAttachedToEngine = "shard attached to engine" + FrostFSNodeClosingComponentsOfTheStorageEngine = "closing components of the storage engine..." + FrostFSNodeStorageEngineClosingFailure = "storage engine closing failure" + FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully = "all components of the storage engine closed successfully" + FrostFSNodeBootstrappingWithTheMaintenanceState = "bootstrapping with the maintenance state" + FrostFSNodeBootstrappingWithOnlineState = "bootstrapping with online state" + FrostFSNodeTerminationSignalHasBeenReceivedStopping = "termination signal has been received, stopping..." + FrostFSNodeTerminationSignalProcessingIsComplete = "termination signal processing is complete" + FrostFSNodeInternalApplicationError = "internal application error" + FrostFSNodeInternalErrorProcessingIsComplete = "internal error processing is complete" + FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration = "SIGHUP has been received, rereading configuration..." + FrostFSNodeConfigurationReading = "configuration reading" + FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" + FrostFSNodeTracingConfigationUpdated = "tracing configation updated" + FrostFSNodeStorageEngineConfigurationUpdate = "storage engine configuration update" + FrostFSNodeUpdatedConfigurationApplying = "updated configuration applying" + FrostFSNodeConfigurationHasBeenReloadedSuccessfully = "configuration has been reloaded successfully" + FrostFSNodeReadNewlyCreatedContainerAfterTheNotification = "read newly created container after the notification" + FrostFSNodeContainerCreationEventsReceipt = "container creation event's receipt" + FrostFSNodeContainerRemovalEventsReceipt = "container removal event's receipt" + FrostFSNodeSaveUsedSpaceAnnouncementInContract = "save used space announcement in contract" + FrostFSNodeFailedToCalculateContainerSizeInStorageEngine = "failed to calculate container size in storage engine" + FrostFSNodeContainerSizeInStorageEngineCalculatedSuccessfully = "container size in storage engine calculated successfully" + FrostFSNodeNotificatorCouldNotListContainers = "notificator: could not list containers" + FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer = "notificator: could not select objects from container" + FrostFSNodeNotificatorCouldNotProcessObject = "notificator: could not process object" + FrostFSNodeNotificatorFinishedProcessingObjectNotifications = "notificator: finished processing object notifications" + FrostFSNodeCouldNotWriteObjectNotification = "could not write object notification" + FrostFSNodeCouldNotGetMaxObjectSizeValue = "could not get max object size value" + FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage = "could not inhume mark redundant copy as garbage" + FrostFSNodeFailedInitTracing = "failed init tracing" + FrostFSNodeFailedShutdownTracing = "failed shutdown tracing" + FrostFSNodeFailedToCreateNeoRPCClient = "failed to create neo RPC client" + FrostFSNodeClosingMorphComponents = "closing morph components..." + FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal = "failed to set group signer scope, continue with Global" + FrostFSNodeNotarySupport = "notary support" + FrostFSNodeMorphcacheTTLFetchedFromNetwork = "morph.cache_ttl fetched from network" + FrostFSNodeNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" + FrostFSNodeCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" + FrostFSNodeNewEpochEventFromSidechain = "new epoch event from sidechain" + FrostFSNodeNewBlock = "new block" + FrostFSNodeCantUpdatePersistentState = "can't update persistent state" + FrostFSNodeCantSendRebootstrapTx = "can't send re-bootstrap tx" + FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch = "could not update node state on new epoch" + FrostFSNodeCouldNotMakeNotaryDeposit = "could not make notary deposit" + FrostFSNodeInitialNetworkState = "initial network state" + FrostFSNodeTreeServiceIsNotEnabledSkipInitialization = "tree service is not enabled, skip initialization" + FrostFSNodeCouldNotSynchronizeTreeService = "could not synchronize Tree Service" + FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" + FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" + FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" FrostFSNodePolicerIsDisabled = "policer is disabled" CommonApplicationStarted = "application started" ShardGCCollectingExpiredObjectsStarted = "collecting expired objects started" @@ -509,8 +509,8 @@ const ( TombstoneExpirationParseFailure = "tombstone getter: could not parse tombstone expiration epoch" FrostFSNodeCantUpdateObjectStorageID = "can't update object storage ID" FrostFSNodeCantFlushObjectToBlobstor = "can't flush an object to blobstor" - FrostFSNodeCantDecodeObjectAddressFromDB = "can't decode object address from the DB" // Error in ../node/cmd/frostfs-node/morph.go - FrostFSNodeCantUnmarshalObjectFromDB = "can't unmarshal an object from the DB" // Error in ../node/cmd/frostfs-node/morph.go + FrostFSNodeCantDecodeObjectAddressFromDB = "can't decode object address from the DB" + FrostFSNodeCantUnmarshalObjectFromDB = "can't unmarshal an object from the DB" RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" FailedToCountWritecacheItems = "failed to count writecache items" From 42fb6fb372b85e5192ef93aa8bafe28945c9e578 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 18 Aug 2023 17:12:24 +0300 Subject: [PATCH 0532/1943] [#574] morph: Add DeletionInfo method for morph client Signed-off-by: Airat Arifullin --- pkg/core/container/storage.go | 9 ++++ pkg/morph/client/container/client.go | 1 + pkg/morph/client/container/deletion_info.go | 52 +++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 pkg/morph/client/container/deletion_info.go diff --git a/pkg/core/container/storage.go b/pkg/core/container/storage.go index c79b7c041..bdff45cc6 100644 --- a/pkg/core/container/storage.go +++ b/pkg/core/container/storage.go @@ -20,6 +20,15 @@ type Container struct { Session *session.Container } +// DelInfo contains info about removed container. +type DelInfo struct { + // Container owner. + Owner []byte + + // Epoch indicates when the container was removed. + Epoch int +} + // Source is an interface that wraps // basic container receiving method. type Source interface { diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index 2b5996cd7..c8702b1c7 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -29,6 +29,7 @@ const ( containersOfMethod = "containersOf" eaclMethod = "eACL" setEACLMethod = "setEACL" + deletionInfoMethod = "deletionInfo" startEstimationMethod = "startContainerEstimation" stopEstimationMethod = "stopContainerEstimation" diff --git a/pkg/morph/client/container/deletion_info.go b/pkg/morph/client/container/deletion_info.go new file mode 100644 index 000000000..094d4e118 --- /dev/null +++ b/pkg/morph/client/container/deletion_info.go @@ -0,0 +1,52 @@ +package container + +import ( + "fmt" + "strings" + + containerContract "git.frostfs.info/TrueCloudLab/frostfs-contract/container" + containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" +) + +func (c *Client) DeletionInfo(cid cid.ID) (*containercore.DelInfo, error) { + prm := client.TestInvokePrm{} + prm.SetMethod(deletionInfoMethod) + prm.SetArgs(cid) + + res, err := c.client.TestInvoke(prm) + if err != nil { + if strings.Contains(err.Error(), containerContract.NotFoundError) { + return nil, new(apistatus.ContainerNotFound) + } + return nil, fmt.Errorf("could not perform test invocation (%s): %w", deletionInfoMethod, err) + } else if ln := len(res); ln != 1 { + return nil, fmt.Errorf("unexpected stack item count (%s): %d", deletionInfoMethod, ln) + } + + arr, err := client.ArrayFromStackItem(res[0]) + if err != nil { + return nil, fmt.Errorf("could not get item array of container (%s): %w", deletionInfoMethod, err) + } + + if len(arr) != 2 { + return nil, fmt.Errorf("unexpected container stack item count (%s): %d", deletionInfoMethod, len(arr)) + } + + owner, err := client.BytesFromStackItem(arr[0]) + if err != nil { + return nil, fmt.Errorf("could not get byte array of container (%s): %w", deletionInfoMethod, err) + } + + epoch, err := client.IntFromStackItem(arr[1]) + if err != nil { + return nil, fmt.Errorf("could not get byte array of container signature (%s): %w", deletionInfoMethod, err) + } + + return &containercore.DelInfo{ + Owner: owner, + Epoch: int(epoch), + }, nil +} From 23be3eb627ad967ce7e3d4941f5cdc43cd179b0e Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 18 Aug 2023 17:20:30 +0300 Subject: [PATCH 0533/1943] [#574] tree: Check if container is really removed * Use DeletionInfo method from morph client to check if the container has been really removed from neo-go Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- cmd/frostfs-node/tree.go | 4 ++++ internal/logs/logs.go | 1 + pkg/services/tree/options.go | 3 +++ pkg/services/tree/signature_test.go | 4 ++++ pkg/services/tree/sync.go | 23 ++++++++++++++++++++++- 5 files changed, 34 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index 1f73b7956..175748ac5 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -31,6 +31,10 @@ func (c cnrSource) Get(id cid.ID) (*container.Container, error) { return c.src.Get(id) } +func (c cnrSource) DeletionInfo(cid cid.ID) (*container.DelInfo, error) { + return c.cli.DeletionInfo(cid) +} + func (c cnrSource) List() ([]cid.ID, error) { return c.cli.ContainersOf(nil) } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 6c261785f..3f0a3f56e 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -70,6 +70,7 @@ const ( TreeContainerTreesHaveBeenSynced = "container trees have been synced" TreeCouldNotQueryTreesForSynchronization = "could not query trees for synchronization" TreeRemovingRedundantTrees = "removing redundant trees..." + TreeCouldNotCheckIfContainerExisted = "could not check if the container ever existed" TreeCouldNotRemoveRedundantTree = "could not remove redundant tree" TreeCouldNotCalculateContainerNodes = "could not calculate container nodes" TreeFailedToApplyReplicatedOperation = "failed to apply replicated operation" diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index a6e23c625..043e12cb2 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -14,6 +14,9 @@ import ( type ContainerSource interface { container.Source + + DeletionInfo(cid.ID) (*container.DelInfo, error) + // List must return list of all the containers in the FrostFS network // at the moment of a call and any error that does not allow fetching // container information. diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 482c16157..9d4776ba1 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -53,6 +53,10 @@ func (s dummyContainerSource) Get(id cid.ID) (*containercore.Container, error) { return cnt, nil } +func (s dummyContainerSource) DeletionInfo(id cid.ID) (*containercore.DelInfo, error) { + return &containercore.DelInfo{}, nil +} + type dummyEACLSource map[string]*containercore.EACL func (s dummyEACLSource) GetEACL(id cid.ID) (*containercore.EACL, error) { diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index e44e8dbbf..62a127c6b 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -18,6 +18,7 @@ import ( metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" tracing_grpc "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" @@ -440,6 +441,18 @@ func (s *Service) syncContainers(ctx context.Context, cnrs []cid.ID) { wg.Wait() } +func (s *Service) containerEverExisted(cid cid.ID) (bool, error) { + _, err := s.cnrSource.DeletionInfo(cid) + if err == nil { + return true, nil + } + var errContainerNotFound *apistatus.ContainerNotFound + if errors.As(err, &errContainerNotFound) { + return false, nil + } + return false, err +} + func (s *Service) removeContainers(ctx context.Context, newContainers map[cid.ID]struct{}) { ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.removeContainers") defer span.End() @@ -452,7 +465,15 @@ func (s *Service) removeContainers(ctx context.Context, newContainers map[cid.ID if _, ok := newContainers[cnr]; ok { continue } - removed = append(removed, cnr) + + existed, err := s.containerEverExisted(cnr) + if err != nil { + s.log.Error(logs.TreeCouldNotCheckIfContainerExisted, + zap.Stringer("cid", cnr), + zap.Error(err)) + } else if existed { + removed = append(removed, cnr) + } } for i := range removed { delete(s.cnrMap, removed[i]) From dc3bc08c073c41158ea0466788901df040c2dee4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 21 Aug 2023 20:06:15 +0300 Subject: [PATCH 0534/1943] [#631] lens: Fix db type flag name Typo from 1a0cb0f34a32. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-lens/internal/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-lens/internal/flags.go b/cmd/frostfs-lens/internal/flags.go index 95710f7c6..004519167 100644 --- a/cmd/frostfs-lens/internal/flags.go +++ b/cmd/frostfs-lens/internal/flags.go @@ -37,6 +37,6 @@ func AddOutputFileFlag(cmd *cobra.Command, v *string) { // AddDBTypeFlag adds the DB type flag to the passed cobra command. func AddDBTypeFlag(cmd *cobra.Command, v *string) { - cmd.Flags().StringVar(v, flagOutFile, "bbolt", + cmd.Flags().StringVar(v, flagDBType, "bbolt", "Type of DB used by write cache (default: bbolt)") } From 345a1a69a228cce3810199eb1b51396b89d0db93 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 22 Aug 2023 10:51:42 +0300 Subject: [PATCH 0535/1943] [#635] Use internal key type when deleting from badger wc Signed-off-by: Alejandro Lopez --- pkg/local_object_storage/writecache/writecachebadger/flush.go | 2 +- .../writecache/writecachebadger/storage.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush.go b/pkg/local_object_storage/writecache/writecachebadger/flush.go index 8630026cd..7f41920ba 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush.go @@ -173,7 +173,7 @@ func (c *cache) workerFlushSmall() { continue } - c.deleteFromDB([]string{objectCore.AddressOf(obj).EncodeToString()}) + c.deleteFromDB([]internalKey{addr2key(objectCore.AddressOf(obj))}) } } diff --git a/pkg/local_object_storage/writecache/writecachebadger/storage.go b/pkg/local_object_storage/writecache/writecachebadger/storage.go index 25d1900d1..9ff54bee0 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/storage.go +++ b/pkg/local_object_storage/writecache/writecachebadger/storage.go @@ -58,7 +58,7 @@ func (c *cache) openStore(readOnly bool) error { return nil } -func (c *cache) deleteFromDB(keys []string) []string { +func (c *cache) deleteFromDB(keys []internalKey) []internalKey { if len(keys) == 0 { return keys } @@ -67,7 +67,7 @@ func (c *cache) deleteFromDB(keys []string) []string { var errorIndex int for errorIndex = range keys { - if err := wb.Delete([]byte(keys[errorIndex])); err != nil { + if err := wb.Delete(keys[errorIndex][:]); err != nil { break } } From 238b8f10a0799fa0d0d1d36de3cf1e8af57e25a0 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 21 Aug 2023 19:11:43 +0300 Subject: [PATCH 0536/1943] [#630] cli: Fix SDK SetEACLPrm usage for PrmContainerSetEACL Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/client.go | 6 +++--- cmd/frostfs-cli/modules/container/set_eacl.go | 12 ++++++------ go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index ae76780bd..0c6d2a38d 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -201,8 +201,8 @@ func EACL(ctx context.Context, prm EACLPrm) (res EACLRes, err error) { // SetEACLPrm groups parameters of SetEACL operation. type SetEACLPrm struct { - commonPrm - client.PrmContainerSetEACL + Client *client.Client + ClientParams client.PrmContainerSetEACL } // SetEACLRes groups the resulting values of SetEACL operation. @@ -217,7 +217,7 @@ type SetEACLRes struct{} // // Returns any error which prevented the operation from completing correctly in error return. func SetEACL(ctx context.Context, prm SetEACLPrm) (res SetEACLRes, err error) { - _, err = prm.cli.ContainerSetEACL(ctx, prm.PrmContainerSetEACL) + _, err = prm.Client.ContainerSetEACL(ctx, prm.ClientParams) return } diff --git a/cmd/frostfs-cli/modules/container/set_eacl.go b/cmd/frostfs-cli/modules/container/set_eacl.go index 51805d69d..f2f60f5c3 100644 --- a/cmd/frostfs-cli/modules/container/set_eacl.go +++ b/cmd/frostfs-cli/modules/container/set_eacl.go @@ -49,12 +49,12 @@ Container ID in EACL table will be substituted with ID from the CLI.`, cmd.Println("ACL extension is enabled in the container, continue processing.") } - var setEACLPrm internalclient.SetEACLPrm - setEACLPrm.SetClient(cli) - setEACLPrm.SetTable(*eaclTable) - - if tok != nil { - setEACLPrm.WithinSession(*tok) + setEACLPrm := internalclient.SetEACLPrm{ + Client: cli, + ClientParams: client.PrmContainerSetEACL{ + Table: eaclTable, + Session: tok, + }, } _, err := internalclient.SetEACL(cmd.Context(), setEACLPrm) diff --git a/go.mod b/go.mod index a1d99fcc6..46639d49c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index fd7be9463..58f1fb217 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e h1:+78sZdTFeK2t+2qJ4Tmm5CItUjfcF4/nFMum3JnZ6PQ= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230816095347-6fdbe755179e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 h1:OuViMF54N87FXmaBEpYw3jhzaLrJ/EWOlPL1wUkimE0= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 82cc453be9cdca1da43424b0b45676bd7fa3a3f5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 23 Aug 2023 10:43:16 +0300 Subject: [PATCH 0537/1943] [#xx] shard: Fix data race in metrics tests Protect test metric store fields with a mutex. Probably, not every field should be protected, but better safe than sorry. Signed-off-by: Evgenii Stratonikov --- .../shard/metrics_test.go | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index ad0fc15ca..60adc0724 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -3,6 +3,7 @@ package shard_test import ( "context" "path/filepath" + "sync" "testing" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -19,6 +20,7 @@ import ( ) type metricsStore struct { + mtx sync.Mutex objCounters map[string]uint64 cnrSize map[string]int64 pldSize int64 @@ -49,35 +51,51 @@ func (m metricsStore) AddToObjectCounter(objectType string, delta int) { } } -func (m metricsStore) IncObjectCounter(objectType string) { +func (m *metricsStore) IncObjectCounter(objectType string) { + m.mtx.Lock() + defer m.mtx.Unlock() m.objCounters[objectType] += 1 } -func (m metricsStore) DecObjectCounter(objectType string) { +func (m *metricsStore) DecObjectCounter(objectType string) { + m.mtx.Lock() + defer m.mtx.Unlock() m.AddToObjectCounter(objectType, -1) } func (m *metricsStore) SetMode(mode mode.Mode) { + m.mtx.Lock() + defer m.mtx.Unlock() m.mode = mode } -func (m metricsStore) AddToContainerSize(cnr string, size int64) { +func (m *metricsStore) AddToContainerSize(cnr string, size int64) { + m.mtx.Lock() + defer m.mtx.Unlock() m.cnrSize[cnr] += size } func (m *metricsStore) AddToPayloadSize(size int64) { + m.mtx.Lock() + defer m.mtx.Unlock() m.pldSize += size } func (m *metricsStore) IncErrorCounter() { + m.mtx.Lock() + defer m.mtx.Unlock() m.errCounter += 1 } func (m *metricsStore) ClearErrorCounter() { + m.mtx.Lock() + defer m.mtx.Unlock() m.errCounter = 0 } func (m *metricsStore) DeleteShardMetrics() { + m.mtx.Lock() + defer m.mtx.Unlock() m.errCounter = 0 } From 02b03d9c4fdb05212f585d1974ec1c089102adb8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 23 Aug 2023 10:48:23 +0300 Subject: [PATCH 0538/1943] [#638] logger: Remove sampling from test loggers Losing logs is always a bad idea, especially when we debug tests. Signed-off-by: Evgenii Stratonikov --- pkg/util/logger/test/logger.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pkg/util/logger/test/logger.go b/pkg/util/logger/test/logger.go index e78a117fa..4a2870870 100644 --- a/pkg/util/logger/test/logger.go +++ b/pkg/util/logger/test/logger.go @@ -9,8 +9,6 @@ import ( "go.uber.org/zap/zapcore" ) -const sampling = 1000 - // NewLogger creates a new logger. // // If debug, development logger is created. @@ -20,11 +18,6 @@ func NewLogger(t testing.TB, debug bool) *logger.Logger { if debug { cfg := zap.NewDevelopmentConfig() - cfg.Sampling = &zap.SamplingConfig{ - Initial: sampling, - Thereafter: sampling, - } - cfg.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder log, err := cfg.Build() From 322c1dc273a08319cc3e62bdabbb25f482e2849e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 23 Aug 2023 10:53:42 +0300 Subject: [PATCH 0539/1943] [#638] Use test.NewLogger() in tests Semantic patch (restricted to **/*_test.go): ``` @@ @@ +import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" -import "go.uber.org/zap" -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" -&logger.Logger{Logger: zap.L()} +test.NewLogger(t, false) ``` Signed-off-by: Evgenii Stratonikov --- .../blobstor/blobovniczatree/exists_test.go | 5 ++--- .../blobstor/blobovniczatree/generic_test.go | 7 +++---- .../blobstor/memstore/memstore_test.go | 5 ++--- pkg/local_object_storage/engine/control_test.go | 7 +++---- pkg/local_object_storage/engine/delete_test.go | 5 ++--- pkg/local_object_storage/engine/engine_test.go | 7 +++---- pkg/local_object_storage/engine/error_test.go | 5 ++--- pkg/local_object_storage/engine/evacuate_test.go | 5 ++--- pkg/local_object_storage/engine/list_test.go | 5 ++--- pkg/local_object_storage/shard/control_test.go | 5 ++--- pkg/local_object_storage/shard/gc_internal_test.go | 10 ++++------ pkg/local_object_storage/shard/range_test.go | 5 ++--- pkg/local_object_storage/shard/reload_test.go | 6 ++---- pkg/local_object_storage/shard/shard_test.go | 9 ++++----- .../writecache/writecachebadger/flush_test.go | 5 ++--- .../writecache/writecachebadger/generic_test.go | 5 ++--- .../writecache/writecachebbolt/flush_test.go | 5 ++--- .../writecache/writecachebbolt/generic_test.go | 5 ++--- pkg/services/tree/signature_test.go | 5 ++--- 19 files changed, 45 insertions(+), 66 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index f9448ae92..a1fdbc127 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -9,16 +9,15 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "github.com/nspcc-dev/neo-go/pkg/util/slice" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func TestExistsInvalidStorageID(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( - WithLogger(&logger.Logger{Logger: zap.L()}), + WithLogger(test.NewLogger(t, false)), WithObjectSizeLimit(1024), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go index 5a713b9b9..764cbc26b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go @@ -5,8 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" ) func TestGeneric(t *testing.T) { @@ -14,7 +13,7 @@ func TestGeneric(t *testing.T) { helper := func(t *testing.T, dir string) common.Storage { return NewBlobovniczaTree( - WithLogger(&logger.Logger{Logger: zap.L()}), + WithLogger(test.NewLogger(t, false)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), @@ -41,7 +40,7 @@ func TestControl(t *testing.T) { newTree := func(t *testing.T) common.Storage { return NewBlobovniczaTree( - WithLogger(&logger.Logger{Logger: zap.L()}), + WithLogger(test.NewLogger(t, false)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index 7979b1504..a080f2bb9 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -7,16 +7,15 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func TestSimpleLifecycle(t *testing.T) { s := New( WithRootPath("memstore"), - WithLogger(&logger.Logger{Logger: zap.L()}), + WithLogger(test.NewLogger(t, false)), ) t.Cleanup(func() { _ = s.Close() }) require.NoError(t, s.Open(false)) diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 04591687d..bbb52a271 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -22,11 +22,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/zap" ) // TestInitializationFailure checks that shard is initialized and closed even if media @@ -54,7 +53,7 @@ func TestInitializationFailure(t *testing.T) { return []shard.Option{ shard.WithID(sid), - shard.WithLogger(&logger.Logger{Logger: zap.L()}), + shard.WithLogger(test.NewLogger(t, false)), shard.WithBlobStorOptions( blobstor.WithStorages(storages)), shard.WithMetaBaseOptions( @@ -295,7 +294,7 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str te := testNewEngine(t). setShardsNumOpts(t, num, func(id int) []shard.Option { return []shard.Option{ - shard.WithLogger(&logger.Logger{Logger: zap.L()}), + shard.WithLogger(test.NewLogger(t, false)), shard.WithBlobStorOptions( blobstor.WithStorages(newStorages(filepath.Join(addPath, strconv.Itoa(id)), errSmallSize))), shard.WithMetaBaseOptions( diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 802b6b0b5..08b1adab8 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -6,14 +6,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func TestDeleteBigObject(t *testing.T) { @@ -54,7 +53,7 @@ func TestDeleteBigObject(t *testing.T) { s3 := testNewShard(t, 3) e := testNewEngine(t).setInitializedShards(t, s1, s2, s3).engine - e.log = &logger.Logger{Logger: zap.L()} + e.log = test.NewLogger(t, false) defer e.Close() for i := range children { diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 39c5e2e1e..ba7cdb5b3 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -15,14 +15,13 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "git.frostfs.info/TrueCloudLab/hrw" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) type epochState struct{} @@ -80,7 +79,7 @@ type testEngineWrapper struct { } func testNewEngine(t testing.TB, opts ...Option) *testEngineWrapper { - engine := New(WithLogger(&logger.Logger{Logger: zap.L()})) + engine := New(WithLogger(test.NewLogger(t, false))) for _, opt := range opts { opt(engine.cfg) } @@ -199,7 +198,7 @@ func testNewShard(t testing.TB, id int) *shard.Shard { func testDefaultShardOptions(t testing.TB, id int) []shard.Option { return []shard.Option{ - shard.WithLogger(&logger.Logger{Logger: zap.L()}), + shard.WithLogger(test.NewLogger(t, false)), shard.WithBlobStorOptions( blobstor.WithStorages( newStorages(t.TempDir(), 1<<20))), diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index bbed8cfb9..2bc02cf96 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -17,11 +17,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) const errSmallSize = 256 @@ -56,7 +55,7 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) largeFileStorage: largeFileStorage, } return []shard.Option{ - shard.WithLogger(&logger.Logger{Logger: zap.L()}), + shard.WithLogger(test.NewLogger(t, false)), shard.WithBlobStorOptions(blobstor.WithStorages(storages)), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", id))), diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 63296ccd7..40f3bcf8c 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -16,12 +16,11 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" - "go.uber.org/zap" "golang.org/x/sync/errgroup" ) @@ -31,7 +30,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng te := testNewEngine(t). setShardsNumOpts(t, shardNum, func(id int) []shard.Option { return []shard.Option{ - shard.WithLogger(&logger.Logger{Logger: zap.L()}), + shard.WithLogger(test.NewLogger(t, false)), shard.WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{{ Storage: fstree.New( diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 9d5d12683..0a9748f43 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -12,11 +12,10 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithType { @@ -66,7 +65,7 @@ func TestListWithCursor(t *testing.T) { t.Parallel() e := testNewEngine(t).setShardsNumOpts(t, tt.shardNum, func(id int) []shard.Option { return []shard.Option{ - shard.WithLogger(&logger.Logger{Logger: zap.L()}), + shard.WithLogger(test.NewLogger(t, false)), shard.WithBlobStorOptions( blobstor.WithStorages( newStorages(t.TempDir(), 1<<20))), diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 053d89a8a..7ee928875 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -19,7 +19,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -30,7 +30,6 @@ import ( objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/zap" ) type epochState struct{} @@ -75,7 +74,7 @@ func TestShardOpen(t *testing.T) { newShard := func() *Shard { return New( WithID(NewIDFromBytes([]byte{})), - WithLogger(&logger.Logger{Logger: zap.L()}), + WithLogger(test.NewLogger(t, false)), WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{ {Storage: st}, diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index 21f300e2c..af2fffd63 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -15,14 +15,13 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { @@ -31,14 +30,13 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { rootPath := t.TempDir() var sh *Shard - - l := &logger.Logger{Logger: zap.L()} + l := test.NewLogger(t, false) blobOpts := []blobstor.Option{ - blobstor.WithLogger(&logger.Logger{Logger: zap.L()}), + blobstor.WithLogger(test.NewLogger(t, false)), blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(&logger.Logger{Logger: zap.L()}), + blobovniczatree.WithLogger(test.NewLogger(t, false)), blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index e764a0763..243bda201 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -14,12 +14,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/nspcc-dev/neo-go/pkg/util/slice" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func TestShard_GetRange(t *testing.T) { @@ -82,7 +81,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { []blobstor.Option{blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(&logger.Logger{Logger: zap.L()}), + blobovniczatree.WithLogger(test.NewLogger(t, false)), blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 75ce386bc..b98c36064 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -11,7 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -20,7 +20,6 @@ import ( usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func TestShardReload(t *testing.T) { @@ -28,8 +27,7 @@ func TestShardReload(t *testing.T) { p := t.Name() defer os.RemoveAll(p) - - l := &logger.Logger{Logger: zap.L()} + l := test.NewLogger(t, false) blobOpts := []blobstor.Option{ blobstor.WithLogger(l), blobstor.WithStorages([]blobstor.SubStorage{ diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 4ec73191c..ff8bfe313 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -16,12 +16,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) type epochState struct { @@ -59,11 +58,11 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts if bsOpts == nil { bsOpts = []blobstor.Option{ - blobstor.WithLogger(&logger.Logger{Logger: zap.L()}), + blobstor.WithLogger(test.NewLogger(t, false)), blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(&logger.Logger{Logger: zap.L()}), + blobovniczatree.WithLogger(test.NewLogger(t, false)), blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), @@ -81,7 +80,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts opts := []shard.Option{ shard.WithID(shard.NewIDFromBytes([]byte{})), - shard.WithLogger(&logger.Logger{Logger: zap.L()}), + shard.WithLogger(test.NewLogger(t, false)), shard.WithBlobStorOptions(bsOpts...), shard.WithMetaBaseOptions( append([]meta.Option{ diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go index 1bf50e000..119c04107 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go @@ -10,18 +10,17 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachetest" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func TestFlush(t *testing.T) { createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { return New( append([]Option{ - WithLogger(&logger.Logger{Logger: zap.L()}), + WithLogger(test.NewLogger(t, false)), WithPath(filepath.Join(t.TempDir(), "writecache")), WithMetabase(mb), WithBlobstor(bs), diff --git a/pkg/local_object_storage/writecache/writecachebadger/generic_test.go b/pkg/local_object_storage/writecache/writecachebadger/generic_test.go index 641da2293..d21adf41f 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/generic_test.go +++ b/pkg/local_object_storage/writecache/writecachebadger/generic_test.go @@ -5,14 +5,13 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" ) func TestGeneric(t *testing.T) { storagetest.TestAll(t, func(t *testing.T) storagetest.Component { return New( - WithLogger(&logger.Logger{Logger: zap.L()}), + WithLogger(test.NewLogger(t, false)), WithFlushWorkersCount(2), WithPath(t.TempDir()), WithGCInterval(1*time.Second)) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go index e7ba66fae..5c73891a7 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go @@ -13,18 +13,17 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachetest" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" - "go.uber.org/zap" ) func TestFlush(t *testing.T) { createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { return New( append([]Option{ - WithLogger(&logger.Logger{Logger: zap.L()}), + WithLogger(test.NewLogger(t, false)), WithPath(filepath.Join(t.TempDir(), "writecache")), WithSmallObjectSize(smallSize), WithMetabase(mb), diff --git a/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go b/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go index b34d53f3e..64d853a78 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go @@ -4,14 +4,13 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" ) func TestGeneric(t *testing.T) { storagetest.TestAll(t, func(t *testing.T) storagetest.Component { return New( - WithLogger(&logger.Logger{Logger: zap.L()}), + WithLogger(test.NewLogger(t, false)), WithFlushWorkersCount(2), WithPath(t.TempDir())) }) diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 9d4776ba1..2f044b971 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -9,7 +9,7 @@ import ( aclV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" @@ -20,7 +20,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) type dummyNetmapSource struct { @@ -101,7 +100,7 @@ func TestMessageSign(t *testing.T) { s := &Service{ cfg: cfg{ - log: &logger.Logger{Logger: zap.L()}, + log: test.NewLogger(t, false), key: &privs[0].PrivateKey, nmSource: dummyNetmapSource{}, cnrSource: dummyContainerSource{ From 96e690883fa0b8604a15dde69e7352f969c91083 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 23 Aug 2023 10:55:36 +0300 Subject: [PATCH 0540/1943] [#638] Unify test loggers In some places we have debug=false, in others debug=true. Let's be consistent. Semantic patch: ``` @@ @@ -test.NewLogger(..., false) +test.NewLogger(..., true) ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobovnicza/blobovnicza_test.go | 2 +- .../blobstor/blobovniczatree/exists_test.go | 2 +- .../blobstor/blobovniczatree/generic_test.go | 4 ++-- .../blobstor/memstore/memstore_test.go | 2 +- pkg/local_object_storage/engine/control_test.go | 4 ++-- pkg/local_object_storage/engine/delete_test.go | 2 +- pkg/local_object_storage/engine/engine_test.go | 4 ++-- pkg/local_object_storage/engine/error_test.go | 2 +- pkg/local_object_storage/engine/evacuate_test.go | 2 +- pkg/local_object_storage/engine/list_test.go | 2 +- pkg/local_object_storage/shard/control_test.go | 2 +- pkg/local_object_storage/shard/gc_internal_test.go | 6 +++--- pkg/local_object_storage/shard/range_test.go | 2 +- pkg/local_object_storage/shard/reload_test.go | 2 +- pkg/local_object_storage/shard/shard_test.go | 6 +++--- .../writecache/writecachebadger/flush_test.go | 2 +- .../writecache/writecachebadger/generic_test.go | 2 +- .../writecache/writecachebbolt/flush_test.go | 2 +- .../writecache/writecachebbolt/generic_test.go | 2 +- pkg/services/object/get/get_test.go | 6 +++--- pkg/services/object/search/search_test.go | 8 ++++---- pkg/services/tree/signature_test.go | 2 +- 22 files changed, 34 insertions(+), 34 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index f8f638564..48a189c9d 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -64,7 +64,7 @@ func TestBlobovnicza(t *testing.T) { WithPath(p), WithObjectSizeLimit(objSizeLim), WithFullSizeLimit(sizeLim), - WithLogger(test.NewLogger(t, false)), + WithLogger(test.NewLogger(t, true)), ) defer os.Remove(p) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index a1fdbc127..8cbdf613c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -17,7 +17,7 @@ import ( func TestExistsInvalidStorageID(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( - WithLogger(test.NewLogger(t, false)), + WithLogger(test.NewLogger(t, true)), WithObjectSizeLimit(1024), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go index 764cbc26b..6eb8b81ae 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go @@ -13,7 +13,7 @@ func TestGeneric(t *testing.T) { helper := func(t *testing.T, dir string) common.Storage { return NewBlobovniczaTree( - WithLogger(test.NewLogger(t, false)), + WithLogger(test.NewLogger(t, true)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), @@ -40,7 +40,7 @@ func TestControl(t *testing.T) { newTree := func(t *testing.T) common.Storage { return NewBlobovniczaTree( - WithLogger(test.NewLogger(t, false)), + WithLogger(test.NewLogger(t, true)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index a080f2bb9..eaa2a4b61 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -15,7 +15,7 @@ import ( func TestSimpleLifecycle(t *testing.T) { s := New( WithRootPath("memstore"), - WithLogger(test.NewLogger(t, false)), + WithLogger(test.NewLogger(t, true)), ) t.Cleanup(func() { _ = s.Close() }) require.NoError(t, s.Open(false)) diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index bbb52a271..eb601122b 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -53,7 +53,7 @@ func TestInitializationFailure(t *testing.T) { return []shard.Option{ shard.WithID(sid), - shard.WithLogger(test.NewLogger(t, false)), + shard.WithLogger(test.NewLogger(t, true)), shard.WithBlobStorOptions( blobstor.WithStorages(storages)), shard.WithMetaBaseOptions( @@ -294,7 +294,7 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str te := testNewEngine(t). setShardsNumOpts(t, num, func(id int) []shard.Option { return []shard.Option{ - shard.WithLogger(test.NewLogger(t, false)), + shard.WithLogger(test.NewLogger(t, true)), shard.WithBlobStorOptions( blobstor.WithStorages(newStorages(filepath.Join(addPath, strconv.Itoa(id)), errSmallSize))), shard.WithMetaBaseOptions( diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 08b1adab8..57b9ec2e3 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -53,7 +53,7 @@ func TestDeleteBigObject(t *testing.T) { s3 := testNewShard(t, 3) e := testNewEngine(t).setInitializedShards(t, s1, s2, s3).engine - e.log = test.NewLogger(t, false) + e.log = test.NewLogger(t, true) defer e.Close() for i := range children { diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index ba7cdb5b3..a8fd9eeee 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -79,7 +79,7 @@ type testEngineWrapper struct { } func testNewEngine(t testing.TB, opts ...Option) *testEngineWrapper { - engine := New(WithLogger(test.NewLogger(t, false))) + engine := New(WithLogger(test.NewLogger(t, true))) for _, opt := range opts { opt(engine.cfg) } @@ -198,7 +198,7 @@ func testNewShard(t testing.TB, id int) *shard.Shard { func testDefaultShardOptions(t testing.TB, id int) []shard.Option { return []shard.Option{ - shard.WithLogger(test.NewLogger(t, false)), + shard.WithLogger(test.NewLogger(t, true)), shard.WithBlobStorOptions( blobstor.WithStorages( newStorages(t.TempDir(), 1<<20))), diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 2bc02cf96..750918880 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -55,7 +55,7 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) largeFileStorage: largeFileStorage, } return []shard.Option{ - shard.WithLogger(test.NewLogger(t, false)), + shard.WithLogger(test.NewLogger(t, true)), shard.WithBlobStorOptions(blobstor.WithStorages(storages)), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", id))), diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 40f3bcf8c..6030ba494 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -30,7 +30,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng te := testNewEngine(t). setShardsNumOpts(t, shardNum, func(id int) []shard.Option { return []shard.Option{ - shard.WithLogger(test.NewLogger(t, false)), + shard.WithLogger(test.NewLogger(t, true)), shard.WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{{ Storage: fstree.New( diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 0a9748f43..8234c3c28 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -65,7 +65,7 @@ func TestListWithCursor(t *testing.T) { t.Parallel() e := testNewEngine(t).setShardsNumOpts(t, tt.shardNum, func(id int) []shard.Option { return []shard.Option{ - shard.WithLogger(test.NewLogger(t, false)), + shard.WithLogger(test.NewLogger(t, true)), shard.WithBlobStorOptions( blobstor.WithStorages( newStorages(t.TempDir(), 1<<20))), diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 7ee928875..73612d840 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -74,7 +74,7 @@ func TestShardOpen(t *testing.T) { newShard := func() *Shard { return New( WithID(NewIDFromBytes([]byte{})), - WithLogger(test.NewLogger(t, false)), + WithLogger(test.NewLogger(t, true)), WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{ {Storage: st}, diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index af2fffd63..c258b6c99 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -30,13 +30,13 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { rootPath := t.TempDir() var sh *Shard - l := test.NewLogger(t, false) + l := test.NewLogger(t, true) blobOpts := []blobstor.Option{ - blobstor.WithLogger(test.NewLogger(t, false)), + blobstor.WithLogger(test.NewLogger(t, true)), blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(test.NewLogger(t, false)), + blobovniczatree.WithLogger(test.NewLogger(t, true)), blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 243bda201..68693f769 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -81,7 +81,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { []blobstor.Option{blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(test.NewLogger(t, false)), + blobovniczatree.WithLogger(test.NewLogger(t, true)), blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index b98c36064..c89c0cb7e 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -27,7 +27,7 @@ func TestShardReload(t *testing.T) { p := t.Name() defer os.RemoveAll(p) - l := test.NewLogger(t, false) + l := test.NewLogger(t, true) blobOpts := []blobstor.Option{ blobstor.WithLogger(l), blobstor.WithStorages([]blobstor.SubStorage{ diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index ff8bfe313..1083bf014 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -58,11 +58,11 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts if bsOpts == nil { bsOpts = []blobstor.Option{ - blobstor.WithLogger(test.NewLogger(t, false)), + blobstor.WithLogger(test.NewLogger(t, true)), blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(test.NewLogger(t, false)), + blobovniczatree.WithLogger(test.NewLogger(t, true)), blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), @@ -80,7 +80,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts opts := []shard.Option{ shard.WithID(shard.NewIDFromBytes([]byte{})), - shard.WithLogger(test.NewLogger(t, false)), + shard.WithLogger(test.NewLogger(t, true)), shard.WithBlobStorOptions(bsOpts...), shard.WithMetaBaseOptions( append([]meta.Option{ diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go index 119c04107..19bc15614 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go @@ -20,7 +20,7 @@ func TestFlush(t *testing.T) { createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { return New( append([]Option{ - WithLogger(test.NewLogger(t, false)), + WithLogger(test.NewLogger(t, true)), WithPath(filepath.Join(t.TempDir(), "writecache")), WithMetabase(mb), WithBlobstor(bs), diff --git a/pkg/local_object_storage/writecache/writecachebadger/generic_test.go b/pkg/local_object_storage/writecache/writecachebadger/generic_test.go index d21adf41f..08845665f 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/generic_test.go +++ b/pkg/local_object_storage/writecache/writecachebadger/generic_test.go @@ -11,7 +11,7 @@ import ( func TestGeneric(t *testing.T) { storagetest.TestAll(t, func(t *testing.T) storagetest.Component { return New( - WithLogger(test.NewLogger(t, false)), + WithLogger(test.NewLogger(t, true)), WithFlushWorkersCount(2), WithPath(t.TempDir()), WithGCInterval(1*time.Second)) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go index 5c73891a7..5db4df318 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go @@ -23,7 +23,7 @@ func TestFlush(t *testing.T) { createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { return New( append([]Option{ - WithLogger(test.NewLogger(t, false)), + WithLogger(test.NewLogger(t, true)), WithPath(filepath.Join(t.TempDir(), "writecache")), WithSmallObjectSize(smallSize), WithMetabase(mb), diff --git a/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go b/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go index 64d853a78..7eadd1afc 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go @@ -10,7 +10,7 @@ import ( func TestGeneric(t *testing.T) { storagetest.TestAll(t, func(t *testing.T) storagetest.Component { return New( - WithLogger(test.NewLogger(t, false)), + WithLogger(test.NewLogger(t, true)), WithFlushWorkersCount(2), WithPath(t.TempDir())) }) diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 5b391fe2b..3ac487265 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -273,7 +273,7 @@ func TestGetLocalOnly(t *testing.T) { newSvc := func(storage *testStorage) *Service { return &Service{ - log: test.NewLogger(t, false), + log: test.NewLogger(t, true), localStorage: storage, } } @@ -535,7 +535,7 @@ func TestGetRemoteSmall(t *testing.T) { const curEpoch = 13 return &Service{ - log: test.NewLogger(t, false), + log: test.NewLogger(t, true), localStorage: newTestStorage(), traverserGenerator: &testTraverserGenerator{ c: cnr, @@ -1667,7 +1667,7 @@ func TestGetFromPastEpoch(t *testing.T) { const curEpoch = 13 svc := &Service{ - log: test.NewLogger(t, false), + log: test.NewLogger(t, true), localStorage: newTestStorage(), epochSource: testEpochReceiver(curEpoch), traverserGenerator: &testTraverserGenerator{ diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index 75059103f..1eb32a29a 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -152,7 +152,7 @@ func TestGetLocalOnly(t *testing.T) { newSvc := func(storage *testStorage) *Service { svc := &Service{cfg: new(cfg)} - svc.log = test.NewLogger(t, false) + svc.log = test.NewLogger(t, true) svc.localStorage = storage return svc @@ -254,7 +254,7 @@ func TestGetRemoteSmall(t *testing.T) { newSvc := func(b *testPlacementBuilder, c *testClientCache) *Service { svc := &Service{cfg: new(cfg)} - svc.log = test.NewLogger(t, false) + svc.log = test.NewLogger(t, true) svc.localStorage = newTestStorage() const curEpoch = 13 @@ -363,7 +363,7 @@ func TestGetFromPastEpoch(t *testing.T) { c22.addResult(idCnr, ids22, nil) svc := &Service{cfg: new(cfg)} - svc.log = test.NewLogger(t, false) + svc.log = test.NewLogger(t, true) svc.localStorage = newTestStorage() const curEpoch = 13 @@ -476,7 +476,7 @@ func TestGetWithSessionToken(t *testing.T) { w := new(simpleIDWriter) svc := &Service{cfg: new(cfg)} - svc.log = test.NewLogger(t, false) + svc.log = test.NewLogger(t, true) svc.localStorage = localStorage const curEpoch = 13 diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 2f044b971..1449d5756 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -100,7 +100,7 @@ func TestMessageSign(t *testing.T) { s := &Service{ cfg: cfg{ - log: test.NewLogger(t, false), + log: test.NewLogger(t, true), key: &privs[0].PrivateKey, nmSource: dummyNetmapSource{}, cnrSource: dummyContainerSource{ From 3fcf56f2fbbd5b9a4257cc7b29fa135f03b8d938 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 23 Aug 2023 20:14:37 +0300 Subject: [PATCH 0541/1943] [#643] objsvc/put: Copy config to `distributedTarget` Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/distributed.go | 12 +++--------- pkg/services/object/put/streamer.go | 14 ++++++-------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 4740ad1fe..df01be9bf 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -10,8 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "go.uber.org/zap" @@ -27,17 +25,13 @@ type distributedTarget struct { obj *objectSDK.Object objMeta object.ContentMeta + *cfg + payload *payload nodeTargetInitializer func(nodeDesc) preparedObjectTarget - getWorkerPool func([]byte) (util.WorkerPool, bool) - relay func(context.Context, nodeDesc) error - - fmt *object.FormatValidator - - log *logger.Logger } // parameters and state of container traversal. @@ -137,7 +131,7 @@ func (t *distributedTarget) WriteObject(ctx context.Context, obj *objectSDK.Obje var err error - if t.objMeta, err = t.fmt.ValidateContent(t.obj); err != nil { + if t.objMeta, err = t.fmtValidator.ValidateContent(t.obj); err != nil { return fmt.Errorf("(%T) could not validate payload content: %w", t, err) } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 80b1c2541..1b5a926a1 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -215,13 +215,13 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { withBroadcast := !prm.common.LocalOnly() && (typ == objectSDK.TypeTombstone || typ == objectSDK.TypeLock) return &distributedTarget{ + cfg: p.cfg, traversal: traversal{ opts: prm.traverseOpts, extraBroadcastEnabled: withBroadcast, }, - payload: getPayload(), - getWorkerPool: p.getWorkerPool, + payload: getPayload(), nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { return localTarget{ @@ -240,8 +240,6 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { return rt }, relay: relay, - fmt: p.fmtValidator, - log: p.log, } } @@ -279,9 +277,9 @@ func (p *Streamer) Close(ctx context.Context) (*PutResponse, error) { }, nil } -func (p *Streamer) getWorkerPool(pub []byte) (pkgutil.WorkerPool, bool) { - if p.netmapKeys.IsLocalKey(pub) { - return p.localPool, true +func (c *cfg) getWorkerPool(pub []byte) (pkgutil.WorkerPool, bool) { + if c.netmapKeys.IsLocalKey(pub) { + return c.localPool, true } - return p.remotePool, false + return c.remotePool, false } From facd3b2c4b943ce6feca1db41afb1acc7b374982 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 23 Aug 2023 20:31:03 +0300 Subject: [PATCH 0542/1943] [#643] objsvc/put: Unify placement iterators Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/common.go | 114 +++++++++++++++++++++++++ pkg/services/object/put/distributed.go | 99 ++------------------- pkg/services/object/put/single.go | 107 ++--------------------- pkg/services/object/put/streamer.go | 11 +-- 4 files changed, 132 insertions(+), 199 deletions(-) create mode 100644 pkg/services/object/put/common.go diff --git a/pkg/services/object/put/common.go b/pkg/services/object/put/common.go new file mode 100644 index 000000000..97e127e19 --- /dev/null +++ b/pkg/services/object/put/common.go @@ -0,0 +1,114 @@ +package putsvc + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "go.uber.org/zap" +) + +type nodeIterator struct { + traversal + cfg *cfg +} + +func (c *cfg) newNodeIterator(opts []placement.Option) *nodeIterator { + return &nodeIterator{ + traversal: traversal{ + opts: opts, + mExclude: make(map[string]*bool), + }, + cfg: c, + } +} + +func (n *nodeIterator) forEachNode(ctx context.Context, f func(context.Context, nodeDesc) error) error { + traverser, err := placement.NewTraverser(n.traversal.opts...) + if err != nil { + return fmt.Errorf("could not create object placement traverser: %w", err) + } + + resErr := &atomic.Value{} + + // Must iterate over all replicas, regardless of whether there are identical nodes there. + // At the same time need to exclude identical nodes from processing. + for { + addrs := traverser.Next() + if len(addrs) == 0 { + break + } + + if n.forEachAddress(ctx, traverser, addrs, f, resErr) { + break + } + } + + if !traverser.Success() { + var err errIncompletePut + err.singleErr, _ = resErr.Load().(error) + return err + } + + // perform additional container broadcast if needed + if n.traversal.submitPrimaryPlacementFinish() { + err := n.forEachNode(ctx, f) + if err != nil { + n.cfg.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) + // we don't fail primary operation because of broadcast failure + } + } + + return nil +} + +func (n *nodeIterator) forEachAddress(ctx context.Context, traverser *placement.Traverser, addrs []placement.Node, f func(context.Context, nodeDesc) error, resErr *atomic.Value) bool { + var wg sync.WaitGroup + + for _, addr := range addrs { + addr := addr + if ok := n.mExclude[string(addr.PublicKey())]; ok != nil { + if *ok { + traverser.SubmitSuccess() + } + // This can happen only during additional container broadcast. + continue + } + + workerPool, isLocal := n.cfg.getWorkerPool(addr.PublicKey()) + + item := new(bool) + wg.Add(1) + if err := workerPool.Submit(func() { + defer wg.Done() + + err := f(ctx, nodeDesc{local: isLocal, info: addr}) + if err != nil { + resErr.Store(err) + svcutil.LogServiceError(n.cfg.log, "PUT", addr.Addresses(), err) + return + } + + traverser.SubmitSuccess() + *item = true + }); err != nil { + wg.Done() + svcutil.LogWorkerPoolError(n.cfg.log, "PUT", err) + return true + } + + // Mark the container node as processed in order to exclude it + // in subsequent container broadcast. Note that we don't + // process this node during broadcast if primary placement + // on it failed. + n.traversal.submitProcessed(addr, item) + } + + wg.Wait() + + return false +} diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index df01be9bf..577c5a15c 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -3,16 +3,11 @@ package putsvc import ( "context" "fmt" - "sync" - "sync/atomic" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" - "go.uber.org/zap" ) type preparedObjectTarget interface { @@ -20,7 +15,8 @@ type preparedObjectTarget interface { } type distributedTarget struct { - traversal traversal + placementOpts []placement.Option + extraBroadcastEnabled bool obj *objectSDK.Object objMeta object.ContentMeta @@ -137,7 +133,7 @@ func (t *distributedTarget) WriteObject(ctx context.Context, obj *objectSDK.Obje if len(t.obj.Children()) > 0 { // enabling extra broadcast for linking objects - t.traversal.extraBroadcastEnabled = true + t.extraBroadcastEnabled = true } return t.iteratePlacement(ctx) @@ -160,90 +156,7 @@ func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error func (t *distributedTarget) iteratePlacement(ctx context.Context) error { id, _ := t.obj.ID() - traverser, err := placement.NewTraverser( - append(t.traversal.opts, placement.ForObject(id))..., - ) - if err != nil { - return fmt.Errorf("(%T) could not create object placement traverser: %w", t, err) - } - - resErr := &atomic.Value{} - - // Must iterate over all replicas, regardless of whether there are identical nodes there. - // At the same time need to exclude identical nodes from processing. - for { - addrs := traverser.Next() - if len(addrs) == 0 { - break - } - - if t.iterateAddresses(ctx, traverser, addrs, resErr) { - break - } - } - - if !traverser.Success() { - var err errIncompletePut - err.singleErr, _ = resErr.Load().(error) - return err - } - - // perform additional container broadcast if needed - if t.traversal.submitPrimaryPlacementFinish() { - err = t.iteratePlacement(ctx) - if err != nil { - t.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) - // we don't fail primary operation because of broadcast failure - } - } - - return nil -} - -func (t *distributedTarget) iterateAddresses(ctx context.Context, traverser *placement.Traverser, addrs []placement.Node, resErr *atomic.Value) bool { - wg := &sync.WaitGroup{} - - for i := range addrs { - addr := addrs[i] - if val := t.traversal.mExclude[string(addr.PublicKey())]; val != nil { - // Check is node processed successful on the previous iteration. - if *val { - traverser.SubmitSuccess() - } - // it can happen only during additional container broadcast - continue - } - - wg.Add(1) - item := new(bool) - - workerPool, isLocal := t.getWorkerPool(addr.PublicKey()) - if err := workerPool.Submit(func() { - defer wg.Done() - - err := t.sendObject(ctx, nodeDesc{local: isLocal, info: addr}) - if err != nil { - resErr.Store(err) - svcutil.LogServiceError(t.log, "PUT", addr.Addresses(), err) - return - } - - traverser.SubmitSuccess() - *item = true - }); err != nil { - wg.Done() - svcutil.LogWorkerPoolError(t.log, "PUT", err) - return true - } - - // mark the container node as processed in order to exclude it - // in subsequent container broadcast. Note that we don't - // process this node during broadcast if primary placement - // on it failed. - t.traversal.submitProcessed(addr, item) - } - - wg.Wait() - - return false + iter := t.cfg.newNodeIterator(append(t.placementOpts, placement.ForObject(id))) + iter.extraBroadcastEnabled = t.extraBroadcastEnabled + return iter.forEachNode(ctx, t.sendObject) } diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index eb18f0f46..8a7f192bb 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -8,7 +8,6 @@ import ( "fmt" "hash" "sync" - "sync/atomic" objectAPI "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" @@ -150,18 +149,19 @@ func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *o if err != nil { return err } - traversal := &traversal{ - opts: placementOptions, - extraBroadcastEnabled: len(obj.Children()) > 0 || - (!localOnly && (obj.Type() == objectSDK.TypeTombstone || obj.Type() == objectSDK.TypeLock)), - mExclude: make(map[string]*bool), - } + + iter := s.cfg.newNodeIterator(placementOptions) + iter.extraBroadcastEnabled = len(obj.Children()) > 0 || (!localOnly && (obj.Type() == objectSDK.TypeTombstone || obj.Type() == objectSDK.TypeLock)) + signer := &putSingleRequestSigner{ req: req, keyStorage: s.keyStorage, signer: &sync.Once{}, } - return s.saveAccordingToPlacement(ctx, obj, signer, traversal, meta) + + return iter.forEachNode(ctx, func(ctx context.Context, nd nodeDesc) error { + return s.saveToPlacementNode(ctx, &nd, obj, signer, meta) + }) } func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) ([]placement.Option, error) { @@ -199,97 +199,6 @@ func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumb return result, nil } -func (s *Service) saveAccordingToPlacement(ctx context.Context, obj *objectSDK.Object, signer *putSingleRequestSigner, - traversal *traversal, meta object.ContentMeta) error { - traverser, err := placement.NewTraverser(traversal.opts...) - if err != nil { - return fmt.Errorf("could not create object placement traverser: %w", err) - } - - var resultError atomic.Value - for { - addrs := traverser.Next() - if len(addrs) == 0 { - break - } - - if stop := s.saveToPlacementNodes(ctx, obj, signer, traversal, traverser, addrs, meta, &resultError); stop { - break - } - } - - if !traverser.Success() { - var err errIncompletePut - err.singleErr, _ = resultError.Load().(error) - return err - } - - if traversal.submitPrimaryPlacementFinish() { - err = s.saveAccordingToPlacement(ctx, obj, signer, traversal, meta) - if err != nil { - s.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) - } - } - - return nil -} - -func (s *Service) saveToPlacementNodes(ctx context.Context, - obj *objectSDK.Object, - signer *putSingleRequestSigner, - traversal *traversal, - traverser *placement.Traverser, - nodeAddresses []placement.Node, - meta object.ContentMeta, - resultError *atomic.Value, -) bool { - wg := sync.WaitGroup{} - - for _, nodeAddress := range nodeAddresses { - nodeAddress := nodeAddress - if ok := traversal.mExclude[string(nodeAddress.PublicKey())]; ok != nil { - if *ok { - traverser.SubmitSuccess() - } - continue - } - - local := false - workerPool := s.remotePool - if s.netmapKeys.IsLocalKey(nodeAddress.PublicKey()) { - local = true - workerPool = s.localPool - } - - item := new(bool) - wg.Add(1) - if err := workerPool.Submit(func() { - defer wg.Done() - - err := s.saveToPlacementNode(ctx, &nodeDesc{local: local, info: nodeAddress}, obj, signer, meta) - - if err != nil { - resultError.Store(err) - svcutil.LogServiceError(s.log, "PUT", nodeAddress.Addresses(), err) - return - } - - traverser.SubmitSuccess() - *item = true - }); err != nil { - wg.Done() - svcutil.LogWorkerPoolError(s.log, "PUT", err) - return true - } - - traversal.submitProcessed(nodeAddress, item) - } - - wg.Wait() - - return false -} - func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *nodeDesc, obj *objectSDK.Object, signer *putSingleRequestSigner, meta object.ContentMeta) error { if nodeDesc.local { diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 1b5a926a1..10f932849 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -215,13 +215,10 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { withBroadcast := !prm.common.LocalOnly() && (typ == objectSDK.TypeTombstone || typ == objectSDK.TypeLock) return &distributedTarget{ - cfg: p.cfg, - traversal: traversal{ - opts: prm.traverseOpts, - - extraBroadcastEnabled: withBroadcast, - }, - payload: getPayload(), + cfg: p.cfg, + placementOpts: prm.traverseOpts, + extraBroadcastEnabled: withBroadcast, + payload: getPayload(), nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { return localTarget{ From c9e3c9956ed9e5470438e76a8b41bde5f13467fa Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 23 Aug 2023 20:38:25 +0300 Subject: [PATCH 0543/1943] [#643] objsvc/put: Unify extraBroadcastEnabled usage Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/common.go | 5 +++++ pkg/services/object/put/distributed.go | 8 +------- pkg/services/object/put/single.go | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/services/object/put/common.go b/pkg/services/object/put/common.go index 97e127e19..6696a192b 100644 --- a/pkg/services/object/put/common.go +++ b/pkg/services/object/put/common.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" ) @@ -112,3 +113,7 @@ func (n *nodeIterator) forEachAddress(ctx context.Context, traverser *placement. return false } + +func needAdditionalBroadcast(obj *objectSDK.Object, localOnly bool) bool { + return len(obj.Children()) > 0 || (!localOnly && (obj.Type() == objectSDK.TypeTombstone || obj.Type() == objectSDK.TypeLock)) +} diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 577c5a15c..509f4aee0 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -130,12 +130,6 @@ func (t *distributedTarget) WriteObject(ctx context.Context, obj *objectSDK.Obje if t.objMeta, err = t.fmtValidator.ValidateContent(t.obj); err != nil { return fmt.Errorf("(%T) could not validate payload content: %w", t, err) } - - if len(t.obj.Children()) > 0 { - // enabling extra broadcast for linking objects - t.extraBroadcastEnabled = true - } - return t.iteratePlacement(ctx) } @@ -157,6 +151,6 @@ func (t *distributedTarget) iteratePlacement(ctx context.Context) error { id, _ := t.obj.ID() iter := t.cfg.newNodeIterator(append(t.placementOpts, placement.ForObject(id))) - iter.extraBroadcastEnabled = t.extraBroadcastEnabled + iter.extraBroadcastEnabled = needAdditionalBroadcast(t.obj, false /* Distributed target is for cluster-wide PUT */) return iter.forEachNode(ctx, t.sendObject) } diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 8a7f192bb..adac194a0 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -151,7 +151,7 @@ func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *o } iter := s.cfg.newNodeIterator(placementOptions) - iter.extraBroadcastEnabled = len(obj.Children()) > 0 || (!localOnly && (obj.Type() == objectSDK.TypeTombstone || obj.Type() == objectSDK.TypeLock)) + iter.extraBroadcastEnabled = needAdditionalBroadcast(obj, localOnly) signer := &putSingleRequestSigner{ req: req, From ba58144de14355680ec8347500c70bff33882639 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 24 Aug 2023 20:54:29 +0300 Subject: [PATCH 0544/1943] [#647] objsvc/search: Remove netmap.Source wrapper Signed-off-by: Evgenii Stratonikov --- pkg/services/object/search/exec.go | 2 +- pkg/services/object/search/search_test.go | 2 +- pkg/services/object/search/service.go | 10 ++++------ pkg/services/object/search/util.go | 9 --------- 4 files changed, 6 insertions(+), 17 deletions(-) diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index c1a9a0c1c..f471da20e 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -74,7 +74,7 @@ func (exec *execCtx) initEpoch() bool { return true } - e, err := exec.svc.currentEpochReceiver.currentEpoch() + e, err := exec.svc.currentEpochReceiver.Epoch() switch { default: diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index 1eb32a29a..d8c5ad090 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -56,7 +56,7 @@ type simpleIDWriter struct { type testEpochReceiver uint64 -func (e testEpochReceiver) currentEpoch() (uint64, error) { +func (e testEpochReceiver) Epoch() (uint64, error) { return uint64(e), nil } diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index 16b82a620..fd3dbb07b 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -49,7 +49,7 @@ type cfg struct { } currentEpochReceiver interface { - currentEpoch() (uint64, error) + Epoch() (uint64, error) } keyStore *util.KeyStorage @@ -71,11 +71,9 @@ func New(e *engine.StorageEngine, localStorage: &storageEngineWrapper{ storage: e, }, - traverserGenerator: (*traverseGeneratorWrapper)(tg), - currentEpochReceiver: &nmSrcWrapper{ - nmSrc: ns, - }, - keyStore: ks, + traverserGenerator: (*traverseGeneratorWrapper)(tg), + currentEpochReceiver: ns, + keyStore: ks, } for i := range opts { diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index b5b351a3b..6ea8e3f14 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -5,7 +5,6 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -36,10 +35,6 @@ type storageEngineWrapper struct { type traverseGeneratorWrapper util.TraverserGenerator -type nmSrcWrapper struct { - nmSrc netmap.Source -} - func newUniqueAddressWriter(w IDListWriter) IDListWriter { return &uniqueIDWriter{ written: make(map[oid.ID]struct{}), @@ -143,7 +138,3 @@ func idsFromAddresses(addrs []oid.Address) []oid.ID { func (e *traverseGeneratorWrapper) generateTraverser(cnr cid.ID, epoch uint64) (*placement.Traverser, error) { return (*util.TraverserGenerator)(e).GenerateTraverser(cnr, nil, epoch) } - -func (n *nmSrcWrapper) currentEpoch() (uint64, error) { - return n.nmSrc.Epoch() -} From 56f841b022eec6a9bd37349154718cfdc9b3c426 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 24 Aug 2023 20:57:53 +0300 Subject: [PATCH 0545/1943] [#647] objsvc/search: Remove TraverserGenerator wrapper Signed-off-by: Evgenii Stratonikov --- pkg/services/object/search/exec.go | 2 +- pkg/services/object/search/search_test.go | 2 +- pkg/services/object/search/service.go | 4 ++-- pkg/services/object/search/util.go | 8 -------- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index f471da20e..d69cb9b50 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -93,7 +93,7 @@ func (exec *execCtx) initEpoch() bool { } func (exec *execCtx) generateTraverser(cnr cid.ID) (*placement.Traverser, bool) { - t, err := exec.svc.traverserGenerator.generateTraverser(cnr, exec.curProcEpoch) + t, err := exec.svc.traverserGenerator.GenerateTraverser(cnr, nil, exec.curProcEpoch) switch { default: diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index d8c5ad090..9bfc54739 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -71,7 +71,7 @@ func newTestStorage() *testStorage { } } -func (g *testTraverserGenerator) generateTraverser(_ cid.ID, epoch uint64) (*placement.Traverser, error) { +func (g *testTraverserGenerator) GenerateTraverser(_ cid.ID, _ *oid.ID, epoch uint64) (*placement.Traverser, error) { return placement.NewTraverser( placement.ForContainer(g.c), placement.UseBuilder(g.b[epoch]), diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index fd3dbb07b..863312200 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -45,7 +45,7 @@ type cfg struct { } traverserGenerator interface { - generateTraverser(cid.ID, uint64) (*placement.Traverser, error) + GenerateTraverser(cid.ID, *oid.ID, uint64) (*placement.Traverser, error) } currentEpochReceiver interface { @@ -71,7 +71,7 @@ func New(e *engine.StorageEngine, localStorage: &storageEngineWrapper{ storage: e, }, - traverserGenerator: (*traverseGeneratorWrapper)(tg), + traverserGenerator: tg, currentEpochReceiver: ns, keyStore: ks, } diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index 6ea8e3f14..37d215256 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -8,8 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -33,8 +31,6 @@ type storageEngineWrapper struct { storage *engine.StorageEngine } -type traverseGeneratorWrapper util.TraverserGenerator - func newUniqueAddressWriter(w IDListWriter) IDListWriter { return &uniqueIDWriter{ written: make(map[oid.ID]struct{}), @@ -134,7 +130,3 @@ func idsFromAddresses(addrs []oid.Address) []oid.ID { return ids } - -func (e *traverseGeneratorWrapper) generateTraverser(cnr cid.ID, epoch uint64) (*placement.Traverser, error) { - return (*util.TraverserGenerator)(e).GenerateTraverser(cnr, nil, epoch) -} From 966ad22abfc9e47397e112f7947a138b5d619c0d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 24 Aug 2023 21:44:55 +0300 Subject: [PATCH 0546/1943] [#647] objsvc/search: Simplify error handling Signed-off-by: Evgenii Stratonikov --- pkg/services/object/search/container.go | 41 ++++++-------- pkg/services/object/search/exec.go | 72 ++++--------------------- pkg/services/object/search/local.go | 20 ++++--- pkg/services/object/search/search.go | 35 ++++++------ 4 files changed, 51 insertions(+), 117 deletions(-) diff --git a/pkg/services/object/search/container.go b/pkg/services/object/search/container.go index 2b6101a98..a8865f5f0 100644 --- a/pkg/services/object/search/container.go +++ b/pkg/services/object/search/container.go @@ -3,6 +3,7 @@ package searchsvc import ( "context" "encoding/hex" + "fmt" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -10,12 +11,7 @@ import ( "go.uber.org/zap" ) -func (exec *execCtx) executeOnContainer(ctx context.Context) { - if exec.isLocal() { - exec.log.Debug(logs.SearchReturnResultDirectly) - return - } - +func (exec *execCtx) executeOnContainer(ctx context.Context) error { lookupDepth := exec.netmapLookupDepth() exec.log.Debug(logs.TryingToExecuteInContainer, @@ -23,13 +19,12 @@ func (exec *execCtx) executeOnContainer(ctx context.Context) { ) // initialize epoch number - ok := exec.initEpoch() - if !ok { - return + if err := exec.initEpoch(); err != nil { + return fmt.Errorf("%s: %w", logs.CouldNotGetCurrentEpochNumber, err) } for { - if exec.processCurrentEpoch(ctx) { + if err := exec.processCurrentEpoch(ctx); err != nil { break } @@ -44,18 +39,17 @@ func (exec *execCtx) executeOnContainer(ctx context.Context) { exec.curProcEpoch-- } - exec.status = statusOK - exec.err = nil + return nil } -func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { +func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { exec.log.Debug(logs.ProcessEpoch, zap.Uint64("number", exec.curProcEpoch), ) - traverser, ok := exec.generateTraverser(exec.containerID()) - if !ok { - return true + traverser, err := exec.svc.traverserGenerator.GenerateTraverser(exec.containerID(), nil, exec.curProcEpoch) + if err != nil { + return fmt.Errorf("%s: %w", logs.SearchCouldNotGenerateContainerTraverser, err) } ctx, cancel := context.WithCancel(ctx) @@ -91,12 +85,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { c, err := exec.svc.clientConstructor.get(info) if err != nil { - mtx.Lock() - exec.status = statusUndefined - exec.err = err - mtx.Unlock() - - exec.log.Debug(logs.SearchCouldNotConstructRemoteNodeClient) + exec.log.Debug(logs.SearchCouldNotConstructRemoteNodeClient, zap.String("error", err.Error())) return } @@ -109,13 +98,17 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) bool { } mtx.Lock() - exec.writeIDList(ids) + err = exec.writeIDList(ids) mtx.Unlock() + if err != nil { + exec.log.Debug(logs.SearchCouldNotWriteObjectIdentifiers, zap.String("error", err.Error())) + return + } }(i) } wg.Wait() } - return false + return nil } diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index d69cb9b50..62ef5827f 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -1,8 +1,6 @@ package searchsvc import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -10,28 +8,16 @@ import ( "go.uber.org/zap" ) -type statusError struct { - status int - err error -} - type execCtx struct { svc *Service prm Prm - statusError - log *logger.Logger curProcEpoch uint64 } -const ( - statusUndefined int = iota - statusOK -) - func (exec *execCtx) prepare() { if _, ok := exec.prm.writer.(*uniqueIDWriter); !ok { exec.prm.writer = newUniqueAddressWriter(exec.prm.writer) @@ -68,64 +54,24 @@ func (exec *execCtx) netmapLookupDepth() uint64 { return exec.prm.common.NetmapLookupDepth() } -func (exec *execCtx) initEpoch() bool { +func (exec *execCtx) initEpoch() error { exec.curProcEpoch = exec.netmapEpoch() if exec.curProcEpoch > 0 { - return true + return nil } e, err := exec.svc.currentEpochReceiver.Epoch() - - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.CouldNotGetCurrentEpochNumber, - zap.String("error", err.Error()), - ) - - return false - case err == nil: - exec.curProcEpoch = e - return true + if err != nil { + return err } + + exec.curProcEpoch = e + return nil } -func (exec *execCtx) generateTraverser(cnr cid.ID) (*placement.Traverser, bool) { - t, err := exec.svc.traverserGenerator.GenerateTraverser(cnr, nil, exec.curProcEpoch) - - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.SearchCouldNotGenerateContainerTraverser, - zap.String("error", err.Error()), - ) - - return nil, false - case err == nil: - return t, true - } -} - -func (exec *execCtx) writeIDList(ids []oid.ID) { +func (exec *execCtx) writeIDList(ids []oid.ID) error { ids = exec.filterAllowedObjectIDs(ids) - err := exec.prm.writer.WriteIDs(ids) - - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.SearchCouldNotWriteObjectIdentifiers, - zap.String("error", err.Error()), - ) - case err == nil: - exec.status = statusOK - exec.err = nil - } + return exec.prm.writer.WriteIDs(ids) } func (exec *execCtx) filterAllowedObjectIDs(objIDs []oid.ID) []oid.ID { diff --git a/pkg/services/object/search/local.go b/pkg/services/object/search/local.go index 1af69caf1..cfaed13b8 100644 --- a/pkg/services/object/search/local.go +++ b/pkg/services/object/search/local.go @@ -2,24 +2,22 @@ package searchsvc import ( "context" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "go.uber.org/zap" ) -func (exec *execCtx) executeLocal(ctx context.Context) { +func (exec *execCtx) executeLocal(ctx context.Context) error { ids, err := exec.svc.localStorage.search(ctx, exec) - if err != nil { - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.SearchLocalOperationFailed, - zap.String("error", err.Error()), - ) - - return + exec.log.Debug(logs.SearchLocalOperationFailed, zap.String("error", err.Error())) + return err } - exec.writeIDList(ids) + if err := exec.writeIDList(ids); err != nil { + return fmt.Errorf("%s: %w", logs.SearchCouldNotWriteObjectIdentifiers, err) + } + + return nil } diff --git a/pkg/services/object/search/search.go b/pkg/services/object/search/search.go index 7a7cbfc5b..2be6bb0e3 100644 --- a/pkg/services/object/search/search.go +++ b/pkg/services/object/search/search.go @@ -18,33 +18,30 @@ func (s *Service) Search(ctx context.Context, prm Prm) error { exec.setLogger(s.log) - exec.execute(ctx) - - return exec.statusError.err + return exec.execute(ctx) } -func (exec *execCtx) execute(ctx context.Context) { +func (exec *execCtx) execute(ctx context.Context) error { exec.log.Debug(logs.ServingRequest) - // perform local operation - exec.executeLocal(ctx) + err := exec.executeLocal(ctx) + exec.logResult(err) - exec.analyzeStatus(ctx, true) + if exec.isLocal() { + exec.log.Debug(logs.SearchReturnResultDirectly) + return err + } + + err = exec.executeOnContainer(ctx) + exec.logResult(err) + return err } -func (exec *execCtx) analyzeStatus(ctx context.Context, execCnr bool) { - // analyze local result - switch exec.status { +func (exec *execCtx) logResult(err error) { + switch { default: - exec.log.Debug(logs.OperationFinishedWithError, - zap.String("error", exec.err.Error()), - ) - case statusOK: + exec.log.Debug(logs.OperationFinishedWithError, zap.String("error", err.Error())) + case err == nil: exec.log.Debug(logs.OperationFinishedSuccessfully) } - - if execCnr { - exec.executeOnContainer(ctx) - exec.analyzeStatus(ctx, false) - } } From 4db2cbc9274b8314157b6856fa093191aed73fa9 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 24 Aug 2023 21:55:36 +0300 Subject: [PATCH 0547/1943] [#647] objsvc/search: Wrap in uniqueIDWriter during parameter setting Signed-off-by: Evgenii Stratonikov --- pkg/services/object/search/exec.go | 6 ------ pkg/services/object/search/prm.go | 4 ++-- pkg/services/object/search/search.go | 2 -- pkg/services/object/search/util.go | 5 ++++- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index 62ef5827f..2e0d48773 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -18,12 +18,6 @@ type execCtx struct { curProcEpoch uint64 } -func (exec *execCtx) prepare() { - if _, ok := exec.prm.writer.(*uniqueIDWriter); !ok { - exec.prm.writer = newUniqueAddressWriter(exec.prm.writer) - } -} - func (exec *execCtx) setLogger(l *logger.Logger) { exec.log = &logger.Logger{Logger: l.With( zap.String("request", "SEARCH"), diff --git a/pkg/services/object/search/prm.go b/pkg/services/object/search/prm.go index d2918d6e7..95fe82e2f 100644 --- a/pkg/services/object/search/prm.go +++ b/pkg/services/object/search/prm.go @@ -12,7 +12,7 @@ import ( // Prm groups parameters of Get service call. type Prm struct { - writer IDListWriter + writer *uniqueIDWriter common *util.CommonPrm @@ -40,7 +40,7 @@ func (p *Prm) SetCommonParameters(common *util.CommonPrm) { // SetWriter sets target component to write list of object identifiers. func (p *Prm) SetWriter(w IDListWriter) { - p.writer = w + p.writer = newUniqueAddressWriter(w) } // SetRequestForwarder sets callback for forwarding diff --git a/pkg/services/object/search/search.go b/pkg/services/object/search/search.go index 2be6bb0e3..4a5c414d5 100644 --- a/pkg/services/object/search/search.go +++ b/pkg/services/object/search/search.go @@ -14,8 +14,6 @@ func (s *Service) Search(ctx context.Context, prm Prm) error { prm: prm, } - exec.prepare() - exec.setLogger(s.log) return exec.execute(ctx) diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index 37d215256..67b6c0d01 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -31,7 +31,10 @@ type storageEngineWrapper struct { storage *engine.StorageEngine } -func newUniqueAddressWriter(w IDListWriter) IDListWriter { +func newUniqueAddressWriter(w IDListWriter) *uniqueIDWriter { + if w, ok := w.(*uniqueIDWriter); ok { + return w + } return &uniqueIDWriter{ written: make(map[oid.ID]struct{}), writer: w, From 40b556fc197666a454b4b84a518c6c59e78ce4bb Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 24 Aug 2023 22:17:27 +0300 Subject: [PATCH 0548/1943] [#647] objsvc/search: Improve testing coverage Signed-off-by: Evgenii Stratonikov --- pkg/services/object/search/search_test.go | 113 +++++++++++++++++----- 1 file changed, 90 insertions(+), 23 deletions(-) diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index 9bfc54739..09d98eff2 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -22,6 +22,7 @@ import ( frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" sessionsdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -60,6 +61,14 @@ func (e testEpochReceiver) Epoch() (uint64, error) { return uint64(e), nil } +type errIDWriter struct { + err error +} + +func (e errIDWriter) WriteIDs(ids []oid.ID) error { + return e.err +} + func (s *simpleIDWriter) WriteIDs(ids []oid.ID) error { s.ids = append(s.ids, ids...) return nil @@ -194,6 +203,20 @@ func TestGetLocalOnly(t *testing.T) { w := new(simpleIDWriter) p := newPrm(cnr, w) + err := svc.Search(ctx, p) + require.ErrorIs(t, err, testErr) + }) + t.Run("FAIL while writing ID", func(t *testing.T) { + storage := newTestStorage() + svc := newSvc(storage) + + cnr := cidtest.ID() + storage.addResult(cnr, []oid.ID{oidtest.ID()}, nil) + + testErr := errors.New("any error") + w := errIDWriter{testErr} + p := newPrm(cnr, w) + err := svc.Search(ctx, p) require.ErrorIs(t, err, testErr) }) @@ -280,33 +303,34 @@ func TestGetRemoteSmall(t *testing.T) { return p } + var addr oid.Address + addr.SetContainer(id) + + ns, as := testNodeMatrix(t, placementDim) + + builder := &testPlacementBuilder{ + vectors: map[string][][]netmap.NodeInfo{ + addr.EncodeToString(): ns, + }, + } + + c1 := newTestStorage() + ids1 := generateIDs(10) + + c2 := newTestStorage() + ids2 := generateIDs(10) + + svc := newSvc(builder, &testClientCache{ + clients: map[string]*testStorage{ + as[0][0]: c1, + as[0][1]: c2, + }, + }) + t.Run("OK", func(t *testing.T) { - var addr oid.Address - addr.SetContainer(id) - - ns, as := testNodeMatrix(t, placementDim) - - builder := &testPlacementBuilder{ - vectors: map[string][][]netmap.NodeInfo{ - addr.EncodeToString(): ns, - }, - } - - c1 := newTestStorage() - ids1 := generateIDs(10) c1.addResult(id, ids1, nil) - - c2 := newTestStorage() - ids2 := generateIDs(10) c2.addResult(id, ids2, nil) - svc := newSvc(builder, &testClientCache{ - clients: map[string]*testStorage{ - as[0][0]: c1, - as[0][1]: c2, - }, - }) - w := new(simpleIDWriter) p := newPrm(id, w) @@ -319,6 +343,49 @@ func TestGetRemoteSmall(t *testing.T) { require.Contains(t, w.ids, id) } }) + t.Run("non-local fail is not a FAIL", func(t *testing.T) { + testErr := errors.New("opaque") + + c1.addResult(id, ids1, nil) + c2.addResult(id, nil, testErr) + + w := new(simpleIDWriter) + p := newPrm(id, w) + + err := svc.Search(ctx, p) + require.NoError(t, err) + require.Equal(t, ids1, w.ids) + }) + t.Run("client init fail is not a FAIL", func(t *testing.T) { + svc := newSvc(builder, &testClientCache{ + clients: map[string]*testStorage{ + as[0][0]: c1, + }, + }) + c1.addResult(id, ids1, nil) + c2.addResult(id, ids2, nil) + + w := new(simpleIDWriter) + p := newPrm(id, w) + + err := svc.Search(ctx, p) + require.NoError(t, err) + require.Equal(t, ids1, w.ids) + }) + t.Run("context is respected", func(t *testing.T) { + c1.addResult(id, ids1, nil) + c2.addResult(id, ids2, nil) + + w := new(simpleIDWriter) + p := newPrm(id, w) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := svc.Search(ctx, p) + require.NoError(t, err) + require.Empty(t, w.ids) + }) } func TestGetFromPastEpoch(t *testing.T) { From d2084ece413ceb80a98bdf1bf96b0af91c82b8ac Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 25 Aug 2023 10:43:49 +0300 Subject: [PATCH 0549/1943] [#648] objsvc/delete: Remove redundant logs We never propagate delete requests to the container node, because tombstone broadcast is done via PUT. No need to pollute logs. Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 1 - pkg/services/object/delete/container.go | 7 ------- pkg/services/object/delete/delete.go | 10 ++-------- 3 files changed, 2 insertions(+), 16 deletions(-) delete mode 100644 pkg/services/object/delete/container.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 3f0a3f56e..a94e868f2 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -82,7 +82,6 @@ const ( PersistentCouldNotCleanUpExpiredTokens = "could not clean up expired tokens" ControllerReportIsAlreadyStarted = "report is already started" TombstoneCouldNotGetTheTombstoneTheSource = "tombstone getter: could not get the tombstone the source" - DeleteRequestIsNotRolledOverToTheContainer = "request is not rolled over to the container" DeleteCouldNotComposeSplitInfo = "could not compose split info" DeleteNoSplitInfoObjectIsPHY = "no split info, object is PHY" DeleteAssemblingChain = "assembling chain..." diff --git a/pkg/services/object/delete/container.go b/pkg/services/object/delete/container.go deleted file mode 100644 index 3106d8efd..000000000 --- a/pkg/services/object/delete/container.go +++ /dev/null @@ -1,7 +0,0 @@ -package deletesvc - -import "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - -func (exec *execCtx) executeOnContainer() { - exec.log.Debug(logs.DeleteRequestIsNotRolledOverToTheContainer) -} diff --git a/pkg/services/object/delete/delete.go b/pkg/services/object/delete/delete.go index ebc191538..cf07875a8 100644 --- a/pkg/services/object/delete/delete.go +++ b/pkg/services/object/delete/delete.go @@ -39,11 +39,10 @@ func (exec *execCtx) execute(ctx context.Context) { // perform local operation exec.executeLocal(ctx) - - exec.analyzeStatus(true) + exec.analyzeStatus() } -func (exec *execCtx) analyzeStatus(execCnr bool) { +func (exec *execCtx) analyzeStatus() { // analyze local result switch exec.status { case statusOK: @@ -52,10 +51,5 @@ func (exec *execCtx) analyzeStatus(execCnr bool) { exec.log.Debug(logs.OperationFinishedWithError, zap.String("error", exec.err.Error()), ) - - if execCnr { - exec.executeOnContainer() - exec.analyzeStatus(false) - } } } From f8ba60aa0c933354960530b25313e4b22b75da0d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 25 Aug 2023 10:57:31 +0300 Subject: [PATCH 0550/1943] [#648] objsvc/delete: Handle errors in Go style Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 7 -- pkg/services/object/delete/delete.go | 26 ++-- pkg/services/object/delete/exec.go | 171 +++++++-------------------- pkg/services/object/delete/local.go | 39 ++---- 4 files changed, 62 insertions(+), 181 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index a94e868f2..b826ae08b 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -82,19 +82,12 @@ const ( PersistentCouldNotCleanUpExpiredTokens = "could not clean up expired tokens" ControllerReportIsAlreadyStarted = "report is already started" TombstoneCouldNotGetTheTombstoneTheSource = "tombstone getter: could not get the tombstone the source" - DeleteCouldNotComposeSplitInfo = "could not compose split info" DeleteNoSplitInfoObjectIsPHY = "no split info, object is PHY" DeleteAssemblingChain = "assembling chain..." - DeleteCouldNotGetPreviousSplitElement = "could not get previous split element" DeleteCollectingChildren = "collecting children..." - DeleteCouldNotCollectObjectChildren = "could not collect object children" DeleteSupplementBySplitID = "supplement by split ID" - DeleteCouldNotSearchForSplitChainMembers = "could not search for split chain members" - DeleteCouldNotMarshalTombstoneStructure = "could not marshal tombstone structure" - DeleteCouldNotSaveTheTombstone = "could not save the tombstone" DeleteFormingTombstoneStructure = "forming tombstone structure..." DeleteTombstoneStructureSuccessfullyFormedSaving = "tombstone structure successfully formed, saving..." - DeleteCouldNotReadTombstoneLifetimeConfig = "could not read tombstone lifetime config" DeleteFormingSplitInfo = "forming split info..." DeleteSplitInfoSuccessfullyFormedCollectingMembers = "split info successfully formed, collecting members..." DeleteMembersSuccessfullyCollected = "members successfully collected" diff --git a/pkg/services/object/delete/delete.go b/pkg/services/object/delete/delete.go index cf07875a8..88454625d 100644 --- a/pkg/services/object/delete/delete.go +++ b/pkg/services/object/delete/delete.go @@ -29,27 +29,17 @@ func (s *Service) Delete(ctx context.Context, prm Prm) error { exec.setLogger(s.log) - exec.execute(ctx) - - return exec.statusError.err + return exec.execute(ctx) } -func (exec *execCtx) execute(ctx context.Context) { +func (exec *execCtx) execute(ctx context.Context) error { exec.log.Debug(logs.ServingRequest) - // perform local operation - exec.executeLocal(ctx) - exec.analyzeStatus() -} - -func (exec *execCtx) analyzeStatus() { - // analyze local result - switch exec.status { - case statusOK: - exec.log.Debug(logs.OperationFinishedSuccessfully) - default: - exec.log.Debug(logs.OperationFinishedWithError, - zap.String("error", exec.err.Error()), - ) + if err := exec.executeLocal(ctx); err != nil { + exec.log.Debug(logs.OperationFinishedWithError, zap.String("error", err.Error())) + return err } + + exec.log.Debug(logs.OperationFinishedSuccessfully) + return nil } diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index b10f045ee..aac8c8860 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -2,6 +2,7 @@ package deletesvc import ( "context" + "fmt" "strconv" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" @@ -15,18 +16,11 @@ import ( "go.uber.org/zap" ) -type statusError struct { - status int - err error -} - type execCtx struct { svc *Service prm Prm - statusError - log *logger.Logger tombstone *objectSDK.Tombstone @@ -36,11 +30,6 @@ type execCtx struct { tombstoneObj *objectSDK.Object } -const ( - statusUndefined int = iota - statusOK -) - func (exec *execCtx) setLogger(l *logger.Logger) { exec.log = &logger.Logger{Logger: l.With( zap.String("request", "DELETE"), @@ -75,48 +64,34 @@ func (exec *execCtx) newAddress(id oid.ID) oid.Address { return a } -func (exec *execCtx) formSplitInfo(ctx context.Context) bool { - success := false - +func (exec *execCtx) formSplitInfo(ctx context.Context) error { var err error exec.splitInfo, err = exec.svc.header.splitInfo(ctx, exec) - - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.DeleteCouldNotComposeSplitInfo, - zap.String("error", err.Error()), - ) - case err == nil, apiclient.IsErrObjectAlreadyRemoved(err): + if err != nil && !apiclient.IsErrObjectAlreadyRemoved(err) { // IsErrObjectAlreadyRemoved check is required because splitInfo // implicitly performs Head request that may return ObjectAlreadyRemoved - // status that is not specified for Delete - - exec.status = statusOK - exec.err = nil - success = true + // status that is not specified for Delete. + return err } - return success + return nil } -func (exec *execCtx) collectMembers(ctx context.Context) (ok bool) { +func (exec *execCtx) collectMembers(ctx context.Context) error { if exec.splitInfo == nil { exec.log.Debug(logs.DeleteNoSplitInfoObjectIsPHY) - return true + return nil } + var err error if _, withLink := exec.splitInfo.Link(); withLink { - ok = exec.collectChildren(ctx) + err = exec.collectChildren(ctx) } - if !ok { + if err != nil { if _, withLast := exec.splitInfo.LastPart(); withLast { - ok = exec.collectChain(ctx) - if !ok { - return + if err := exec.collectChain(ctx); err != nil { + return err } } } // may be fail if neither right nor linking ID is set? @@ -124,7 +99,7 @@ func (exec *execCtx) collectMembers(ctx context.Context) (ok bool) { return exec.supplementBySplitID(ctx) } -func (exec *execCtx) collectChain(ctx context.Context) bool { +func (exec *execCtx) collectChain(ctx context.Context) error { var chain []oid.ID exec.log.Debug(logs.DeleteAssemblingChain) @@ -133,84 +108,43 @@ func (exec *execCtx) collectChain(ctx context.Context) bool { chain = append(chain, prev) p, err := exec.svc.header.previous(ctx, exec, prev) + if err != nil { + return fmt.Errorf("get previous split element for %s: %w", prev, err) + } - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.DeleteCouldNotGetPreviousSplitElement, - zap.Stringer("id", prev), - zap.String("error", err.Error()), - ) - - return false - case err == nil: - exec.status = statusOK - exec.err = nil - - withPrev = p != nil - if withPrev { - prev = *p - } + withPrev = p != nil + if withPrev { + prev = *p } } exec.addMembers(chain) - - return true + return nil } -func (exec *execCtx) collectChildren(ctx context.Context) bool { +func (exec *execCtx) collectChildren(ctx context.Context) error { exec.log.Debug(logs.DeleteCollectingChildren) children, err := exec.svc.header.children(ctx, exec) - - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.DeleteCouldNotCollectObjectChildren, - zap.String("error", err.Error()), - ) - - return false - case err == nil: - exec.status = statusOK - exec.err = nil - - link, _ := exec.splitInfo.Link() - - exec.addMembers(append(children, link)) - - return true + if err != nil { + return fmt.Errorf("collect children: %w", err) } + + link, _ := exec.splitInfo.Link() + exec.addMembers(append(children, link)) + return nil } -func (exec *execCtx) supplementBySplitID(ctx context.Context) bool { +func (exec *execCtx) supplementBySplitID(ctx context.Context) error { exec.log.Debug(logs.DeleteSupplementBySplitID) chain, err := exec.svc.searcher.splitMembers(ctx, exec) - - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.DeleteCouldNotSearchForSplitChainMembers, - zap.String("error", err.Error()), - ) - - return false - case err == nil: - exec.status = statusOK - exec.err = nil - - exec.addMembers(chain) - - return true + if err != nil { + return fmt.Errorf("search split chain members: %w", err) } + + exec.addMembers(chain) + return nil } func (exec *execCtx) addMembers(incoming []oid.ID) { @@ -228,17 +162,10 @@ func (exec *execCtx) addMembers(incoming []oid.ID) { exec.tombstone.SetMembers(append(members, incoming...)) } -func (exec *execCtx) initTombstoneObject() bool { +func (exec *execCtx) initTombstoneObject() error { payload, err := exec.tombstone.Marshal() if err != nil { - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.DeleteCouldNotMarshalTombstoneStructure, - zap.String("error", err.Error()), - ) - - return false + return fmt.Errorf("marshal tombstone: %w", err) } exec.tombstoneObj = objectSDK.New() @@ -262,29 +189,15 @@ func (exec *execCtx) initTombstoneObject() bool { exec.tombstoneObj.SetAttributes(a) - return true + return nil } -func (exec *execCtx) saveTombstone(ctx context.Context) bool { +func (exec *execCtx) saveTombstone(ctx context.Context) error { id, err := exec.svc.placer.put(ctx, exec) - - switch { - default: - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.DeleteCouldNotSaveTheTombstone, - zap.String("error", err.Error()), - ) - - return false - case err == nil: - exec.status = statusOK - exec.err = nil - - exec.prm.tombAddrWriter. - SetAddress(exec.newAddress(*id)) + if err != nil { + return fmt.Errorf("save tombstone: %w", err) } - return true + exec.prm.tombAddrWriter.SetAddress(exec.newAddress(*id)) + return nil } diff --git a/pkg/services/object/delete/local.go b/pkg/services/object/delete/local.go index ad3e10bc6..55ce4408d 100644 --- a/pkg/services/object/delete/local.go +++ b/pkg/services/object/delete/local.go @@ -2,37 +2,29 @@ package deletesvc import ( "context" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.uber.org/zap" ) -func (exec *execCtx) executeLocal(ctx context.Context) { +func (exec *execCtx) executeLocal(ctx context.Context) error { exec.log.Debug(logs.DeleteFormingTombstoneStructure) - ok := exec.formTombstone(ctx) - if !ok { - return + if err := exec.formTombstone(ctx); err != nil { + return err } exec.log.Debug(logs.DeleteTombstoneStructureSuccessfullyFormedSaving) - exec.saveTombstone(ctx) + return exec.saveTombstone(ctx) } -func (exec *execCtx) formTombstone(ctx context.Context) (ok bool) { +func (exec *execCtx) formTombstone(ctx context.Context) error { tsLifetime, err := exec.svc.netInfo.TombstoneLifetime() if err != nil { - exec.status = statusUndefined - exec.err = err - - exec.log.Debug(logs.DeleteCouldNotReadTombstoneLifetimeConfig, - zap.String("error", err.Error()), - ) - - return false + return fmt.Errorf("fetch tombstone lifetime: %w", err) } exec.tombstone = objectSDK.NewTombstone() @@ -43,26 +35,19 @@ func (exec *execCtx) formTombstone(ctx context.Context) (ok bool) { exec.log.Debug(logs.DeleteFormingSplitInfo) - ok = exec.formSplitInfo(ctx) - if !ok { - return + if err := exec.formSplitInfo(ctx); err != nil { + return fmt.Errorf("form split info: %w", err) } exec.log.Debug(logs.DeleteSplitInfoSuccessfullyFormedCollectingMembers) exec.tombstone.SetSplitID(exec.splitInfo.SplitID()) - ok = exec.collectMembers(ctx) - if !ok { - return + if err := exec.collectMembers(ctx); err != nil { + return err } exec.log.Debug(logs.DeleteMembersSuccessfullyCollected) - ok = exec.initTombstoneObject() - if !ok { - return - } - - return true + return exec.initTombstoneObject() } From c4db8e769054ac9ba26a6a20d80bab402ed193eb Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 25 Aug 2023 11:43:35 +0300 Subject: [PATCH 0551/1943] [#637] shard/test: Fix data race Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/metrics_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 60adc0724..91e158b04 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -28,13 +28,17 @@ type metricsStore struct { errCounter int64 } -func (m metricsStore) SetShardID(_ string) {} +func (m *metricsStore) SetShardID(_ string) {} -func (m metricsStore) SetObjectCounter(objectType string, v uint64) { +func (m *metricsStore) SetObjectCounter(objectType string, v uint64) { + m.mtx.Lock() + defer m.mtx.Unlock() m.objCounters[objectType] = v } -func (m metricsStore) AddToObjectCounter(objectType string, delta int) { +func (m *metricsStore) AddToObjectCounter(objectType string, delta int) { + m.mtx.Lock() + defer m.mtx.Unlock() switch { case delta > 0: m.objCounters[objectType] += uint64(delta) From 9072772a09c93f2bcdb9ad1441344512980a4a39 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 25 Aug 2023 12:04:34 +0300 Subject: [PATCH 0552/1943] [#649] shard/test: Increase GC remover interval This was set in #348 to speed up tests. It seems 100ms doesn't increase overall test time, but it reduces the amount of logs by 100x factor. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/delete_test.go | 13 +++++++++---- pkg/local_object_storage/shard/shard_test.go | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index aba204f36..bfafcdc74 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -3,6 +3,7 @@ package shard_test import ( "context" "testing" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" @@ -56,8 +57,10 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { _, err = sh.Delete(context.TODO(), delPrm) require.NoError(t, err) - _, err = sh.Get(context.Background(), getPrm) - require.True(t, client.IsErrObjectNotFound(err)) + require.Eventually(t, func() bool { + _, err = sh.Get(context.Background(), getPrm) + return client.IsErrObjectNotFound(err) + }, time.Second, 50*time.Millisecond) }) t.Run("small object", func(t *testing.T) { @@ -80,7 +83,9 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { _, err = sh.Delete(context.Background(), delPrm) require.NoError(t, err) - _, err = sh.Get(context.Background(), getPrm) - require.True(t, client.IsErrObjectNotFound(err)) + require.Eventually(t, func() bool { + _, err = sh.Get(context.Background(), getPrm) + return client.IsErrObjectNotFound(err) + }, time.Second, 50*time.Millisecond) }) } diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 1083bf014..6337b0b6e 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -101,7 +101,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts require.NoError(t, err) return pool }), - shard.WithGCRemoverSleepInterval(1 * time.Millisecond), + shard.WithGCRemoverSleepInterval(100 * time.Millisecond), } sh = shard.New(opts...) From 554ff2c06b397ce447a55f651f5185e97f781c8c Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 24 Aug 2023 15:27:24 +0300 Subject: [PATCH 0553/1943] [#574] core: Extend Source interface with DeletionInfo method * Introduce common method EverExisted * Define DeletionInfo for struct that must implement Source * Refactor tree srv Signed-off-by: Airat Arifullin --- cmd/frostfs-node/cache.go | 23 +++++++-- cmd/frostfs-node/container.go | 17 ++++--- cmd/frostfs-node/tree.go | 2 +- pkg/core/container/storage.go | 2 + pkg/core/container/util.go | 22 +++++++++ pkg/morph/client/container/deletion_info.go | 22 ++++++++- pkg/services/policer/policer_test.go | 54 ++++++++++++++------- pkg/services/tree/sync.go | 16 +----- 8 files changed, 115 insertions(+), 43 deletions(-) create mode 100644 pkg/core/container/util.go diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index 6d138d894..f8266e562 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -142,7 +142,8 @@ func (c *lruNetCache) get(key uint64) (*netmapSDK.NetMap, error) { // wrapper over TTL cache of values read from the network // that implements container storage. type ttlContainerStorage struct { - *ttlNetCache[cid.ID, *container.Container] + containerCache *ttlNetCache[cid.ID, *container.Container] + delInfoCache *ttlNetCache[cid.ID, *container.DelInfo] } func newCachedContainerStorage(v container.Source, ttl time.Duration) ttlContainerStorage { @@ -151,18 +152,32 @@ func newCachedContainerStorage(v container.Source, ttl time.Duration) ttlContain lruCnrCache := newNetworkTTLCache(containerCacheSize, ttl, func(id cid.ID) (*container.Container, error) { return v.Get(id) }) + lruDelInfoCache := newNetworkTTLCache(containerCacheSize, ttl, func(id cid.ID) (*container.DelInfo, error) { + return v.DeletionInfo(id) + }) - return ttlContainerStorage{lruCnrCache} + return ttlContainerStorage{ + containerCache: lruCnrCache, + delInfoCache: lruDelInfoCache, + } } func (s ttlContainerStorage) handleRemoval(cnr cid.ID) { - s.set(cnr, nil, new(apistatus.ContainerNotFound)) + s.containerCache.set(cnr, nil, new(apistatus.ContainerNotFound)) + + // The removal causes the cache miss and thus deletion info (that contains + // ownerID and epoch) for the container will be updated from sidechain. + s.delInfoCache.remove(cnr) } // Get returns container value from the cache. If value is missing in the cache // or expired, then it returns value from side chain and updates the cache. func (s ttlContainerStorage) Get(cnr cid.ID) (*container.Container, error) { - return s.get(cnr) + return s.containerCache.get(cnr) +} + +func (s ttlContainerStorage) DeletionInfo(cnr cid.ID) (*container.DelInfo, error) { + return s.delInfoCache.get(cnr) } type ttlEACLStorage struct { diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 9a6cfd02b..8c2ae54b3 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -28,6 +28,7 @@ import ( containerMorph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apiClient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -113,7 +114,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c c.cfgObject.eaclSource = eACLFetcher cnrRdr.eacl = eACLFetcher c.cfgObject.cnrSource = cnrSrc - cnrRdr.get = cnrSrc + cnrRdr.src = cnrSrc cnrRdr.lister = client } else { // use RPC node as source of Container contract items (with caching) @@ -131,7 +132,8 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c cnr, err := cnrSrc.Get(ev.ID) if err == nil { cachedContainerLister.update(cnr.Value.Owner(), ev.ID, true) - cachedContainerStorage.set(ev.ID, cnr, nil) + cachedContainerStorage.containerCache.set(ev.ID, cnr, nil) + cachedContainerStorage.delInfoCache.set(ev.ID, nil, new(apistatus.ContainerNotFound)) } else { // unlike removal, we expect successful receive of the container // after successful creation, so logging can be useful @@ -159,7 +161,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c } cachedContainerStorage.handleRemoval(ev.ID) - c.log.Debug(logs.FrostFSNodeContainerRemovalEventsReceipt, zap.Stringer("id", ev.ID), ) @@ -170,7 +171,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c cnrRdr.lister = cachedContainerLister cnrRdr.eacl = c.cfgObject.eaclSource - cnrRdr.get = c.cfgObject.cnrSource + cnrRdr.src = c.cfgObject.cnrSource cnrWrt.cacheEnabled = true cnrWrt.eacls = cachedEACLStorage @@ -641,7 +642,7 @@ func (c *usedSpaceService) processLoadValue(_ context.Context, a containerSDK.Si type morphContainerReader struct { eacl containerCore.EACLSource - get containerCore.Source + src containerCore.Source lister interface { List(*user.ID) ([]cid.ID, error) @@ -649,7 +650,11 @@ type morphContainerReader struct { } func (x *morphContainerReader) Get(id cid.ID) (*containerCore.Container, error) { - return x.get.Get(id) + return x.src.Get(id) +} + +func (x *morphContainerReader) DeletionInfo(id cid.ID) (*containerCore.DelInfo, error) { + return x.src.DeletionInfo(id) } func (x *morphContainerReader) GetEACL(id cid.ID) (*containerCore.EACL, error) { diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index 175748ac5..f7c0f2a36 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -32,7 +32,7 @@ func (c cnrSource) Get(id cid.ID) (*container.Container, error) { } func (c cnrSource) DeletionInfo(cid cid.ID) (*container.DelInfo, error) { - return c.cli.DeletionInfo(cid) + return c.src.DeletionInfo(cid) } func (c cnrSource) List() ([]cid.ID, error) { diff --git a/pkg/core/container/storage.go b/pkg/core/container/storage.go index bdff45cc6..0766ced31 100644 --- a/pkg/core/container/storage.go +++ b/pkg/core/container/storage.go @@ -41,6 +41,8 @@ type Source interface { // Implementations must not retain the container pointer and modify // the container through it. Get(cid.ID) (*Container, error) + + DeletionInfo(cid.ID) (*DelInfo, error) } // EACL groups information about the FrostFS container's extended ACL stored in diff --git a/pkg/core/container/util.go b/pkg/core/container/util.go new file mode 100644 index 000000000..58ceb03ba --- /dev/null +++ b/pkg/core/container/util.go @@ -0,0 +1,22 @@ +package container + +import ( + "errors" + + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" +) + +// EverExisted checks whether the container ever existed or +// it just has not been created yet at the current epoch. +func EverExisted(s Source, cid cid.ID) (bool, error) { + _, err := s.DeletionInfo(cid) + if err == nil { + return true, nil + } + var errContainerNotFound *apistatus.ContainerNotFound + if errors.As(err, &errContainerNotFound) { + return false, nil + } + return false, err +} diff --git a/pkg/morph/client/container/deletion_info.go b/pkg/morph/client/container/deletion_info.go index 094d4e118..b34ec32db 100644 --- a/pkg/morph/client/container/deletion_info.go +++ b/pkg/morph/client/container/deletion_info.go @@ -1,6 +1,7 @@ package container import ( + "crypto/sha256" "fmt" "strings" @@ -11,7 +12,26 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) -func (c *Client) DeletionInfo(cid cid.ID) (*containercore.DelInfo, error) { +func (x *containerSource) DeletionInfo(cnr cid.ID) (*containercore.DelInfo, error) { + return DeletionInfo((*Client)(x), cnr) +} + +type deletionInfo interface { + DeletionInfo(cid []byte) (*containercore.DelInfo, error) +} + +func AsContainerSpecInfoProvider(w *Client) containercore.Source { + return (*containerSource)(w) +} + +func DeletionInfo(c deletionInfo, cnr cid.ID) (*containercore.DelInfo, error) { + binCnr := make([]byte, sha256.Size) + cnr.Encode(binCnr) + + return c.DeletionInfo(binCnr) +} + +func (c *Client) DeletionInfo(cid []byte) (*containercore.DelInfo, error) { prm := client.TestInvokePrm{} prm.SetMethod(deletionInfoMethod) prm.SetArgs(cid) diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 37d6c0506..8b3eecd3b 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -34,8 +34,13 @@ func TestBuryObjectWithoutContainer(t *testing.T) { // Container source and bury function buryCh := make(chan oid.Address) - containerSrc := func(id cid.ID) (*container.Container, error) { - return nil, new(apistatus.ContainerNotFound) + containerSrc := containerSrc{ + get: func(id cid.ID) (*container.Container, error) { + return nil, new(apistatus.ContainerNotFound) + }, + deletionInfo: func(id cid.ID) (*container.DelInfo, error) { + return &container.DelInfo{}, nil + }, } buryFn := func(ctx context.Context, a oid.Address) error { buryCh <- a @@ -49,7 +54,7 @@ func TestBuryObjectWithoutContainer(t *testing.T) { // Policer instance p := New( WithKeySpaceIterator(&sliceKeySpaceIterator{objs: objs}), - WithContainerSource(containerSrcFunc(containerSrc)), + WithContainerSource(containerSrc), WithBuryFunc(buryFn), WithPool(pool), ) @@ -194,12 +199,17 @@ func TestProcessObject(t *testing.T) { cnr := &container.Container{} cnr.Value.Init() cnr.Value.SetPlacementPolicy(policy) - containerSrc := func(id cid.ID) (*container.Container, error) { - if id.Equals(addr.Container()) { - return cnr, nil - } - t.Errorf("unexpected container requested: got=%v, want=%v", id, addr.Container()) - return nil, new(apistatus.ContainerNotFound) + containerSrc := containerSrc{ + get: func(id cid.ID) (*container.Container, error) { + if id.Equals(addr.Container()) { + return cnr, nil + } + t.Errorf("unexpected container requested: got=%v, want=%v", id, addr.Container()) + return nil, new(apistatus.ContainerNotFound) + }, + deletionInfo: func(id cid.ID) (*container.DelInfo, error) { + return &container.DelInfo{}, nil + }, } buryFn := func(ctx context.Context, a oid.Address) error { t.Errorf("unexpected object buried: %v", a) @@ -211,7 +221,7 @@ func TestProcessObject(t *testing.T) { var gotReplicateTo []int p := New( - WithContainerSource(containerSrcFunc(containerSrc)), + WithContainerSource(containerSrc), WithPlacementBuilder(placementBuilderFunc(placementBuilder)), WithNetmapKeys(announcedKeysFunc(func(k []byte) bool { return bytes.Equal(k, nodes[0].PublicKey()) @@ -251,9 +261,6 @@ func TestIteratorContract(t *testing.T) { Type: objectSDK.TypeRegular, }} - containerSrc := func(id cid.ID) (*container.Container, error) { - return nil, new(apistatus.ContainerNotFound) - } buryFn := func(ctx context.Context, a oid.Address) error { return nil } @@ -273,9 +280,18 @@ func TestIteratorContract(t *testing.T) { finishCh: make(chan struct{}), } + containerSrc := containerSrc{ + get: func(id cid.ID) (*container.Container, error) { + return nil, new(apistatus.ContainerNotFound) + }, + deletionInfo: func(id cid.ID) (*container.DelInfo, error) { + return &container.DelInfo{}, nil + }, + } + p := New( WithKeySpaceIterator(it), - WithContainerSource(containerSrcFunc(containerSrc)), + WithContainerSource(containerSrc), WithBuryFunc(buryFn), WithPool(pool), func(c *cfg) { @@ -353,10 +369,14 @@ func (it *sliceKeySpaceIterator) Rewind() { it.cur = 0 } -// containerSrcFunc is a container.Source backed by a function. -type containerSrcFunc func(cid.ID) (*container.Container, error) +type containerSrc struct { + get func(id cid.ID) (*container.Container, error) + deletionInfo func(id cid.ID) (*container.DelInfo, error) +} -func (f containerSrcFunc) Get(id cid.ID) (*container.Container, error) { return f(id) } +func (f containerSrc) Get(id cid.ID) (*container.Container, error) { return f.get(id) } + +func (f containerSrc) DeletionInfo(id cid.ID) (*container.DelInfo, error) { return f.deletionInfo(id) } // placementBuilderFunc is a placement.Builder backed by a function type placementBuilderFunc func(cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 62a127c6b..c681bb078 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -12,13 +12,13 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" tracing_grpc "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" @@ -441,18 +441,6 @@ func (s *Service) syncContainers(ctx context.Context, cnrs []cid.ID) { wg.Wait() } -func (s *Service) containerEverExisted(cid cid.ID) (bool, error) { - _, err := s.cnrSource.DeletionInfo(cid) - if err == nil { - return true, nil - } - var errContainerNotFound *apistatus.ContainerNotFound - if errors.As(err, &errContainerNotFound) { - return false, nil - } - return false, err -} - func (s *Service) removeContainers(ctx context.Context, newContainers map[cid.ID]struct{}) { ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.removeContainers") defer span.End() @@ -466,7 +454,7 @@ func (s *Service) removeContainers(ctx context.Context, newContainers map[cid.ID continue } - existed, err := s.containerEverExisted(cnr) + existed, err := containerCore.EverExisted(s.cnrSource, cnr) if err != nil { s.log.Error(logs.TreeCouldNotCheckIfContainerExisted, zap.Stringer("cid", cnr), From 4ea0df77d0b5336fac5420df5bfba89e650f33cc Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 24 Aug 2023 15:35:19 +0300 Subject: [PATCH 0554/1943] [#574] policer: Check if the container was really removed Signed-off-by: Airat Arifullin --- cmd/frostfs-node/cache.go | 3 +-- cmd/frostfs-node/container.go | 2 -- internal/logs/logs.go | 1 + pkg/core/container/util.go | 4 ++-- pkg/morph/client/container/deletion_info.go | 4 ---- pkg/services/policer/check.go | 13 +++++++++++-- pkg/services/tree/sync.go | 2 +- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index f8266e562..6a5d5d182 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -165,8 +165,7 @@ func newCachedContainerStorage(v container.Source, ttl time.Duration) ttlContain func (s ttlContainerStorage) handleRemoval(cnr cid.ID) { s.containerCache.set(cnr, nil, new(apistatus.ContainerNotFound)) - // The removal causes the cache miss and thus deletion info (that contains - // ownerID and epoch) for the container will be updated from sidechain. + // The removal invalidates possibly stored error response. s.delInfoCache.remove(cnr) } diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 8c2ae54b3..5d88ff9ea 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -28,7 +28,6 @@ import ( containerMorph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apiClient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -133,7 +132,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c if err == nil { cachedContainerLister.update(cnr.Value.Owner(), ev.ID, true) cachedContainerStorage.containerCache.set(ev.ID, cnr, nil) - cachedContainerStorage.delInfoCache.set(ev.ID, nil, new(apistatus.ContainerNotFound)) } else { // unlike removal, we expect successful receive of the container // after successful creation, so logging can be useful diff --git a/internal/logs/logs.go b/internal/logs/logs.go index b826ae08b..95961cd47 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -42,6 +42,7 @@ const ( NotificatorNotificatorStartProcessingObjectNotifications = "notificator: start processing object notifications" NotificatorNotificatorProcessingObjectNotification = "notificator: processing object notification" PolicerCouldNotGetContainer = "could not get container" + PolicerCouldNotConfirmContainerRemoval = "could not confirm container removal" PolicerCouldNotInhumeObjectWithMissingContainer = "could not inhume object with missing container" PolicerCouldNotBuildPlacementVectorForObject = "could not build placement vector for object" PolicerRedundantLocalObjectCopyDetected = "redundant local object copy detected" diff --git a/pkg/core/container/util.go b/pkg/core/container/util.go index 58ceb03ba..98919284e 100644 --- a/pkg/core/container/util.go +++ b/pkg/core/container/util.go @@ -7,9 +7,9 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) -// EverExisted checks whether the container ever existed or +// WasRemoved checks whether the container ever existed or // it just has not been created yet at the current epoch. -func EverExisted(s Source, cid cid.ID) (bool, error) { +func WasRemoved(s Source, cid cid.ID) (bool, error) { _, err := s.DeletionInfo(cid) if err == nil { return true, nil diff --git a/pkg/morph/client/container/deletion_info.go b/pkg/morph/client/container/deletion_info.go index b34ec32db..40eb267d6 100644 --- a/pkg/morph/client/container/deletion_info.go +++ b/pkg/morph/client/container/deletion_info.go @@ -20,10 +20,6 @@ type deletionInfo interface { DeletionInfo(cid []byte) (*containercore.DelInfo, error) } -func AsContainerSpecInfoProvider(w *Client) containercore.Source { - return (*containerSource)(w) -} - func DeletionInfo(c deletionInfo, cnr cid.ID) (*containercore.DelInfo, error) { binCnr := make([]byte, sha256.Size) cnr.Encode(binCnr) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 800ddb006..bc82b144d 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -5,6 +5,7 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -27,12 +28,20 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add zap.String("error", err.Error()), ) if client.IsErrContainerNotFound(err) { - err := p.buryFn(ctx, addrWithType.Address) + existed, err := containercore.WasRemoved(p.cnrSrc, idCnr) if err != nil { - p.log.Error(logs.PolicerCouldNotInhumeObjectWithMissingContainer, + p.log.Error(logs.PolicerCouldNotConfirmContainerRemoval, zap.Stringer("cid", idCnr), zap.Stringer("oid", idObj), zap.String("error", err.Error())) + } else if existed { + err := p.buryFn(ctx, addrWithType.Address) + if err != nil { + p.log.Error(logs.PolicerCouldNotInhumeObjectWithMissingContainer, + zap.Stringer("cid", idCnr), + zap.Stringer("oid", idObj), + zap.String("error", err.Error())) + } } } diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index c681bb078..9cff8b351 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -454,7 +454,7 @@ func (s *Service) removeContainers(ctx context.Context, newContainers map[cid.ID continue } - existed, err := containerCore.EverExisted(s.cnrSource, cnr) + existed, err := containerCore.WasRemoved(s.cnrSource, cnr) if err != nil { s.log.Error(logs.TreeCouldNotCheckIfContainerExisted, zap.Stringer("cid", cnr), From ab2614ec2d444e06c1407854f6a451cc36a03f1d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 17 Jul 2023 16:46:46 +0300 Subject: [PATCH 0555/1943] [#528] objectcore: Validate token issuer Add token issuer against object owner validation. Signed-off-by: Dmitrii Stepanov --- pkg/core/object/fmt.go | 7 ++++++- pkg/core/object/fmt_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index e6d8174fa..fe6654517 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -153,9 +153,14 @@ func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { } token := obj.SessionToken() + ownerID := *obj.OwnerID() if token == nil || !token.AssertAuthKey(&key) { - return v.checkOwnerKey(*obj.OwnerID(), key) + return v.checkOwnerKey(ownerID, key) + } + + if !token.Issuer().Equals(ownerID) { + return fmt.Errorf("(%T) different token issuer and object owner identifiers %s/%s", v, token.Issuer(), ownerID) } return nil diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index d04c16709..392ecf605 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -8,11 +8,13 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" sessiontest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" ) @@ -104,6 +106,34 @@ func TestFormatValidator_Validate(t *testing.T) { require.NoError(t, v.Validate(context.Background(), obj, false)) }) + t.Run("invalid w/ session token", func(t *testing.T) { + var idOwner user.ID + user.IDFromKey(&idOwner, ownerKey.PrivateKey.PublicKey) + + var randomUserID user.ID + randPrivKey, err := keys.NewPrivateKey() + require.NoError(t, err) + user.IDFromKey(&randomUserID, randPrivKey.PrivateKey.PublicKey) + + tok := sessiontest.Object() + fsPubKey := frostfsecdsa.PublicKey(*ownerKey.PublicKey()) + tok.SetID(uuid.New()) + tok.SetAuthKey(&fsPubKey) + tok.SetExp(100500) + tok.SetIat(1) + tok.SetNbf(1) + err = tok.Sign(ownerKey.PrivateKey) + require.NoError(t, err) + + obj := objectSDK.New() + obj.SetContainerID(cidtest.ID()) + obj.SetSessionToken(tok) + obj.SetOwnerID(&randomUserID) + require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) + + require.Error(t, v.Validate(context.Background(), obj, false)) //invalid owner + }) + t.Run("correct w/o session token", func(t *testing.T) { obj := blankValidObject(&ownerKey.PrivateKey) From ae81d6660adee15617c10354ad5a17a75382ba5d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Jul 2023 15:44:35 +0300 Subject: [PATCH 0556/1943] [#529] objectcore: Fix object content validation There are old objects where the owner of the object may not match the one who issued the token. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 7 +- cmd/frostfs-node/config/object/config.go | 5 + cmd/frostfs-node/config/object/config_test.go | 2 + cmd/frostfs-node/object.go | 14 +- config/example/node.env | 1 + config/example/node.json | 3 +- config/example/node.yaml | 1 + pkg/core/object/fmt.go | 154 +++++++- pkg/core/object/fmt_test.go | 364 ++++++++++++++++-- pkg/services/object/put/service.go | 26 +- 10 files changed, 535 insertions(+), 42 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 808a78d28..8e103b527 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -511,6 +511,8 @@ type cfgObject struct { cfgLocalStorage cfgLocalStorage tombstoneLifetime uint64 + + skipSessionTokenIssuerVerification bool } type cfgNotifications struct { @@ -677,8 +679,9 @@ func initCfgGRPC() cfgGRPC { func initCfgObject(appCfg *config.Config) cfgObject { return cfgObject{ - pool: initObjectPool(appCfg), - tombstoneLifetime: objectconfig.TombstoneLifetime(appCfg), + pool: initObjectPool(appCfg), + tombstoneLifetime: objectconfig.TombstoneLifetime(appCfg), + skipSessionTokenIssuerVerification: objectconfig.Put(appCfg).SkipSessionTokenIssuerVerification(), } } diff --git a/cmd/frostfs-node/config/object/config.go b/cmd/frostfs-node/config/object/config.go index cd969852b..f7a33b5e0 100644 --- a/cmd/frostfs-node/config/object/config.go +++ b/cmd/frostfs-node/config/object/config.go @@ -51,3 +51,8 @@ func (g PutConfig) PoolSizeLocal() int { return PutPoolSizeDefault } + +// SkipSessionTokenIssuerVerification returns the value of "skip_session_token_issuer_verification" config parameter or `false“ if is not defined. +func (g PutConfig) SkipSessionTokenIssuerVerification() bool { + return config.BoolSafe(g.cfg, "skip_session_token_issuer_verification") +} diff --git a/cmd/frostfs-node/config/object/config_test.go b/cmd/frostfs-node/config/object/config_test.go index 81c1ccd57..513b6e9c5 100644 --- a/cmd/frostfs-node/config/object/config_test.go +++ b/cmd/frostfs-node/config/object/config_test.go @@ -16,6 +16,7 @@ func TestObjectSection(t *testing.T) { require.Equal(t, objectconfig.PutPoolSizeDefault, objectconfig.Put(empty).PoolSizeRemote()) require.Equal(t, objectconfig.PutPoolSizeDefault, objectconfig.Put(empty).PoolSizeLocal()) require.EqualValues(t, objectconfig.DefaultTombstoneLifetime, objectconfig.TombstoneLifetime(empty)) + require.False(t, objectconfig.Put(empty).SkipSessionTokenIssuerVerification()) }) const path = "../../../../config/example/node" @@ -24,6 +25,7 @@ func TestObjectSection(t *testing.T) { require.Equal(t, 100, objectconfig.Put(c).PoolSizeRemote()) require.Equal(t, 200, objectconfig.Put(c).PoolSizeLocal()) require.EqualValues(t, 10, objectconfig.TombstoneLifetime(c)) + require.True(t, objectconfig.Put(c).SkipSessionTokenIssuerVerification()) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 84411d31b..34847e36f 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -160,8 +160,9 @@ func initObjectService(c *cfg) { addPolicer(c, keyStorage, c.bgClientCache) traverseGen := util.NewTraverserGenerator(c.netMapSource, c.cfgObject.cnrSource, c) + irFetcher := newCachedIRFetcher(createInnerRingFetcher(c)) - sPut := createPutSvc(c, keyStorage) + sPut := createPutSvc(c, keyStorage, &irFetcher) sPutV2 := createPutSvcV2(sPut, keyStorage) @@ -184,7 +185,7 @@ func initObjectService(c *cfg) { splitSvc := createSplitService(c, sPutV2, sGetV2, sSearchV2, sDeleteV2) - aclSvc := createACLServiceV2(c, splitSvc) + aclSvc := createACLServiceV2(c, splitSvc, &irFetcher) var commonSvc objectService.Common commonSvc.Init(&c.internals, aclSvc) @@ -295,7 +296,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa ) } -func createPutSvc(c *cfg, keyStorage *util.KeyStorage) *putsvc.Service { +func createPutSvc(c *cfg, keyStorage *util.KeyStorage, irFetcher *cachedIRFetcher) *putsvc.Service { ls := c.cfgObject.cfgLocalStorage.localStorage var os putsvc.ObjectStorage = engineWithoutNotifications{ @@ -320,8 +321,10 @@ func createPutSvc(c *cfg, keyStorage *util.KeyStorage) *putsvc.Service { c.netMapSource, c, c.cfgNetmap.state, + irFetcher, putsvc.WithWorkerPools(c.cfgObject.pool.putRemote, c.cfgObject.pool.putLocal), putsvc.WithLogger(c.log), + putsvc.WithVerifySessionTokenIssuer(!c.cfgObject.skipSessionTokenIssuerVerification), ) } @@ -405,14 +408,13 @@ func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Servi ) } -func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter) v2.Service { +func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter, irFetcher *cachedIRFetcher) v2.Service { ls := c.cfgObject.cfgLocalStorage.localStorage - irFetcher := createInnerRingFetcher(c) return v2.New( splitSvc, c.netMapSource, - newCachedIRFetcher(irFetcher), + irFetcher, acl.NewChecker( c.cfgNetmap.state, c.cfgObject.eaclSource, diff --git a/config/example/node.env b/config/example/node.env index 7ed818ef9..fde65173b 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -86,6 +86,7 @@ FROSTFS_REPLICATOR_POOL_SIZE=10 # Object service section FROSTFS_OBJECT_PUT_POOL_SIZE_REMOTE=100 FROSTFS_OBJECT_PUT_POOL_SIZE_LOCAL=200 +FROSTFS_OBJECT_PUT_SKIP_SESSION_TOKEN_ISSUER_VERIFICATION=true FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10 # Storage engine section diff --git a/config/example/node.json b/config/example/node.json index a480c4a0e..e8455ee55 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -130,7 +130,8 @@ }, "put": { "pool_size_remote": 100, - "pool_size_local": 200 + "pool_size_local": 200, + "skip_session_token_issuer_verification": true } }, "storage": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 39fb7714d..2ca1b426c 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -110,6 +110,7 @@ object: put: pool_size_remote: 100 # number of async workers for remote PUT operations pool_size_local: 200 # number of async workers for local PUT operations + skip_session_token_issuer_verification: true # session token issuer verification will be skipped if true storage: # note: shard configuration can be omitted for relay node (see `node.relay`) diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index fe6654517..90365cead 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -1,20 +1,26 @@ package object import ( + "bytes" "context" "crypto/ecdsa" + "crypto/elliptic" + "crypto/sha256" "errors" "fmt" "strconv" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) // FormatValidator represents an object format validator. @@ -26,8 +32,16 @@ type FormatValidator struct { type FormatValidatorOption func(*cfg) type cfg struct { - netState netmap.State - e LockSource + netState netmap.State + e LockSource + ir InnerRing + netmap netmap.Source + containers container.Source + verifyTokenIssuer bool +} + +type InnerRing interface { + InnerRingKeys() ([][]byte, error) } // DeleteHandler is an interface of delete queue processor. @@ -159,13 +173,117 @@ func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { return v.checkOwnerKey(ownerID, key) } - if !token.Issuer().Equals(ownerID) { - return fmt.Errorf("(%T) different token issuer and object owner identifiers %s/%s", v, token.Issuer(), ownerID) + if v.verifyTokenIssuer { + signerIsIROrContainerNode, err := v.isIROrContainerNode(obj, binKey) + if err != nil { + return err + } + + if signerIsIROrContainerNode { + return nil + } + + if !token.Issuer().Equals(ownerID) { + return fmt.Errorf("(%T) different token issuer and object owner identifiers %s/%s", v, token.Issuer(), ownerID) + } + return nil } return nil } +func (v *FormatValidator) isIROrContainerNode(obj *objectSDK.Object, signerKey []byte) (bool, error) { + pKey, err := keys.NewPublicKeyFromBytes(signerKey, elliptic.P256()) + if err != nil { + return false, fmt.Errorf("(%T) failed to unmarshal signer public key: %w", v, err) + } + + isIR, err := v.isInnerRingKey(pKey.Bytes()) + if err != nil { + return false, fmt.Errorf("(%T) failed to check if signer is inner ring node: %w", v, err) + } + if isIR { + return true, nil + } + + isContainerNode, err := v.isContainerNode(pKey.Bytes(), obj) + if err != nil { + return false, fmt.Errorf("(%T) failed to check if signer is container node: %w", v, err) + } + return isContainerNode, nil +} + +func (v *FormatValidator) isInnerRingKey(key []byte) (bool, error) { + innerRingKeys, err := v.ir.InnerRingKeys() + if err != nil { + return false, err + } + + for i := range innerRingKeys { + if bytes.Equal(innerRingKeys[i], key) { + return true, nil + } + } + + return false, nil +} + +func (v *FormatValidator) isContainerNode(key []byte, obj *objectSDK.Object) (bool, error) { + cnrID, containerIDSet := obj.ContainerID() + if !containerIDSet { + return false, errNilCID + } + + cnrIDBin := make([]byte, sha256.Size) + cnrID.Encode(cnrIDBin) + + cnr, err := v.containers.Get(cnrID) + if err != nil { + return false, fmt.Errorf("failed to get container (id=%s): %w", cnrID.EncodeToString(), err) + } + + lastNetmap, err := netmap.GetLatestNetworkMap(v.netmap) + if err != nil { + return false, fmt.Errorf("failed to get latest netmap: %w", err) + } + + isContainerNode, err := v.isContainerNodeKey(lastNetmap, cnr, cnrIDBin, key) + if err != nil { + return false, fmt.Errorf("failed to check latest netmap for container nodes: %w", err) + } + if isContainerNode { + return true, nil + } + + previousNetmap, err := netmap.GetPreviousNetworkMap(v.netmap) + if err != nil { + return false, fmt.Errorf("failed to get previous netmap: %w", err) + } + + isContainerNode, err = v.isContainerNodeKey(previousNetmap, cnr, cnrIDBin, key) + if err != nil { + return false, fmt.Errorf("failed to check previous netmap for container nodes: %w", err) + } + return isContainerNode, nil +} + +func (v *FormatValidator) isContainerNodeKey(nm *netmapSDK.NetMap, cnr *container.Container, cnrIDBin, key []byte) (bool, error) { + cnrVectors, err := nm.ContainerNodes(cnr.Value.PlacementPolicy(), cnrIDBin) + if err != nil { + return false, err + } + + for i := range cnrVectors { + for j := range cnrVectors[i] { + if bytes.Equal(cnrVectors[i][j].PublicKey(), key) { + return true, nil + } + } + } + + return false, nil +} + func (v *FormatValidator) checkOwnerKey(id user.ID, key frostfsecdsa.PublicKey) error { var id2 user.ID user.IDFromKey(&id2, (ecdsa.PublicKey)(key)) @@ -387,3 +505,31 @@ func WithLockSource(e LockSource) FormatValidatorOption { c.e = e } } + +// WithInnerRing return option to set Inner Ring source. +func WithInnerRing(ir InnerRing) FormatValidatorOption { + return func(c *cfg) { + c.ir = ir + } +} + +// WithNetmapSource return option to set Netmap source. +func WithNetmapSource(ns netmap.Source) FormatValidatorOption { + return func(c *cfg) { + c.netmap = ns + } +} + +// WithContainersSource return option to set Containers source. +func WithContainersSource(cs container.Source) FormatValidatorOption { + return func(c *cfg) { + c.containers = cs + } +} + +// WithVerifySessionTokenIssuer return option to set verify session token issuer value. +func WithVerifySessionTokenIssuer(verifySessionTokenIssuer bool) FormatValidatorOption { + return func(c *cfg) { + c.verifyTokenIssuer = verifySessionTokenIssuer + } +} diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index 392ecf605..fbd82906d 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -3,12 +3,17 @@ package object import ( "context" "crypto/ecdsa" + "fmt" "strconv" "testing" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -106,34 +111,6 @@ func TestFormatValidator_Validate(t *testing.T) { require.NoError(t, v.Validate(context.Background(), obj, false)) }) - t.Run("invalid w/ session token", func(t *testing.T) { - var idOwner user.ID - user.IDFromKey(&idOwner, ownerKey.PrivateKey.PublicKey) - - var randomUserID user.ID - randPrivKey, err := keys.NewPrivateKey() - require.NoError(t, err) - user.IDFromKey(&randomUserID, randPrivKey.PrivateKey.PublicKey) - - tok := sessiontest.Object() - fsPubKey := frostfsecdsa.PublicKey(*ownerKey.PublicKey()) - tok.SetID(uuid.New()) - tok.SetAuthKey(&fsPubKey) - tok.SetExp(100500) - tok.SetIat(1) - tok.SetNbf(1) - err = tok.Sign(ownerKey.PrivateKey) - require.NoError(t, err) - - obj := objectSDK.New() - obj.SetContainerID(cidtest.ID()) - obj.SetSessionToken(tok) - obj.SetOwnerID(&randomUserID) - require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) - - require.Error(t, v.Validate(context.Background(), obj, false)) //invalid owner - }) - t.Run("correct w/o session token", func(t *testing.T) { obj := blankValidObject(&ownerKey.PrivateKey) @@ -284,3 +261,334 @@ func TestFormatValidator_Validate(t *testing.T) { }) }) } + +func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { + const curEpoch = 13 + + ls := testLockSource{ + m: make(map[oid.Address]bool), + } + + signer, err := keys.NewPrivateKey() + require.NoError(t, err) + + var owner user.ID + ownerPrivKey, err := keys.NewPrivateKey() + require.NoError(t, err) + user.IDFromKey(&owner, ownerPrivKey.PrivateKey.PublicKey) + + t.Run("different issuer and owner, verify issuer disabled", func(t *testing.T) { + t.Parallel() + v := NewFormatValidator( + WithNetState(testNetState{ + epoch: curEpoch, + }), + WithLockSource(ls), + WithVerifySessionTokenIssuer(false), + ) + + tok := sessiontest.Object() + fsPubKey := frostfsecdsa.PublicKey(*signer.PublicKey()) + tok.SetID(uuid.New()) + tok.SetAuthKey(&fsPubKey) + tok.SetExp(100500) + tok.SetIat(1) + tok.SetNbf(1) + require.NoError(t, tok.Sign(signer.PrivateKey)) + + obj := objectSDK.New() + obj.SetContainerID(cidtest.ID()) + obj.SetSessionToken(tok) + obj.SetOwnerID(&owner) + require.NoError(t, objectSDK.SetIDWithSignature(signer.PrivateKey, obj)) + + require.NoError(t, v.Validate(context.Background(), obj, false)) + }) + + t.Run("different issuer and owner, issuer is IR node, verify issuer enabled", func(t *testing.T) { + t.Parallel() + v := NewFormatValidator( + WithNetState(testNetState{ + epoch: curEpoch, + }), + WithLockSource(ls), + WithVerifySessionTokenIssuer(true), + WithInnerRing(&testIRSource{ + irNodes: [][]byte{signer.PublicKey().Bytes()}, + }), + ) + + tok := sessiontest.Object() + fsPubKey := frostfsecdsa.PublicKey(*signer.PublicKey()) + tok.SetID(uuid.New()) + tok.SetAuthKey(&fsPubKey) + tok.SetExp(100500) + tok.SetIat(1) + tok.SetNbf(1) + require.NoError(t, tok.Sign(signer.PrivateKey)) + + obj := objectSDK.New() + obj.SetContainerID(cidtest.ID()) + obj.SetSessionToken(tok) + obj.SetOwnerID(&owner) + require.NoError(t, objectSDK.SetIDWithSignature(signer.PrivateKey, obj)) + + require.NoError(t, v.Validate(context.Background(), obj, false)) + }) + + t.Run("different issuer and owner, issuer is container node in current epoch, verify issuer enabled", func(t *testing.T) { + t.Parallel() + + tok := sessiontest.Object() + fsPubKey := frostfsecdsa.PublicKey(*signer.PublicKey()) + tok.SetID(uuid.New()) + tok.SetAuthKey(&fsPubKey) + tok.SetExp(100500) + tok.SetIat(1) + tok.SetNbf(1) + require.NoError(t, tok.Sign(signer.PrivateKey)) + + cnrID := cidtest.ID() + cont := containerSDK.Container{} + cont.Init() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + cont.SetPlacementPolicy(pp) + + var node netmap.NodeInfo + node.SetPublicKey(signer.PublicKey().Bytes()) + currentEpochNM := &netmap.NetMap{} + currentEpochNM.SetEpoch(curEpoch) + currentEpochNM.SetNodes([]netmap.NodeInfo{node}) + + obj := objectSDK.New() + obj.SetContainerID(cnrID) + obj.SetSessionToken(tok) + obj.SetOwnerID(&owner) + require.NoError(t, objectSDK.SetIDWithSignature(signer.PrivateKey, obj)) + + v := NewFormatValidator( + WithNetState(testNetState{ + epoch: curEpoch, + }), + WithLockSource(ls), + WithVerifySessionTokenIssuer(true), + WithInnerRing(&testIRSource{ + irNodes: [][]byte{}, + }), + WithContainersSource( + &testContainerSource{ + containers: map[cid.ID]*container.Container{ + cnrID: { + Value: cont, + }, + }, + }, + ), + WithNetmapSource( + &testNetmapSource{ + netmaps: map[uint64]*netmap.NetMap{ + curEpoch: currentEpochNM, + }, + currentEpoch: curEpoch, + }, + ), + ) + + require.NoError(t, v.Validate(context.Background(), obj, false)) + }) + + t.Run("different issuer and owner, issuer is container node in previous epoch, verify issuer enabled", func(t *testing.T) { + t.Parallel() + + tok := sessiontest.Object() + fsPubKey := frostfsecdsa.PublicKey(*signer.PublicKey()) + tok.SetID(uuid.New()) + tok.SetAuthKey(&fsPubKey) + tok.SetExp(100500) + tok.SetIat(1) + tok.SetNbf(1) + require.NoError(t, tok.Sign(signer.PrivateKey)) + + cnrID := cidtest.ID() + cont := containerSDK.Container{} + cont.Init() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + cont.SetPlacementPolicy(pp) + + var issuerNode netmap.NodeInfo + issuerNode.SetPublicKey(signer.PublicKey().Bytes()) + + var nonIssuerNode netmap.NodeInfo + nonIssuerKey, err := keys.NewPrivateKey() + require.NoError(t, err) + nonIssuerNode.SetPublicKey(nonIssuerKey.PublicKey().Bytes()) + + currentEpochNM := &netmap.NetMap{} + currentEpochNM.SetEpoch(curEpoch) + currentEpochNM.SetNodes([]netmap.NodeInfo{nonIssuerNode}) + + previousEpochNM := &netmap.NetMap{} + previousEpochNM.SetEpoch(curEpoch - 1) + previousEpochNM.SetNodes([]netmap.NodeInfo{issuerNode}) + + obj := objectSDK.New() + obj.SetContainerID(cnrID) + obj.SetSessionToken(tok) + obj.SetOwnerID(&owner) + require.NoError(t, objectSDK.SetIDWithSignature(signer.PrivateKey, obj)) + + v := NewFormatValidator( + WithNetState(testNetState{ + epoch: curEpoch, + }), + WithLockSource(ls), + WithVerifySessionTokenIssuer(true), + WithInnerRing(&testIRSource{ + irNodes: [][]byte{}, + }), + WithContainersSource( + &testContainerSource{ + containers: map[cid.ID]*container.Container{ + cnrID: { + Value: cont, + }, + }, + }, + ), + WithNetmapSource( + &testNetmapSource{ + netmaps: map[uint64]*netmap.NetMap{ + curEpoch: currentEpochNM, + curEpoch - 1: previousEpochNM, + }, + currentEpoch: curEpoch, + }, + ), + ) + + require.NoError(t, v.Validate(context.Background(), obj, false)) + }) + + t.Run("different issuer and owner, issuer is unknown, verify issuer enabled", func(t *testing.T) { + t.Parallel() + + tok := sessiontest.Object() + fsPubKey := frostfsecdsa.PublicKey(*signer.PublicKey()) + tok.SetID(uuid.New()) + tok.SetAuthKey(&fsPubKey) + tok.SetExp(100500) + tok.SetIat(1) + tok.SetNbf(1) + require.NoError(t, tok.Sign(signer.PrivateKey)) + + cnrID := cidtest.ID() + cont := containerSDK.Container{} + cont.Init() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + cont.SetPlacementPolicy(pp) + + var nonIssuerNode1 netmap.NodeInfo + nonIssuerKey1, err := keys.NewPrivateKey() + require.NoError(t, err) + nonIssuerNode1.SetPublicKey(nonIssuerKey1.PublicKey().Bytes()) + + var nonIssuerNode2 netmap.NodeInfo + nonIssuerKey2, err := keys.NewPrivateKey() + require.NoError(t, err) + nonIssuerNode2.SetPublicKey(nonIssuerKey2.PublicKey().Bytes()) + + currentEpochNM := &netmap.NetMap{} + currentEpochNM.SetEpoch(curEpoch) + currentEpochNM.SetNodes([]netmap.NodeInfo{nonIssuerNode1}) + + previousEpochNM := &netmap.NetMap{} + previousEpochNM.SetEpoch(curEpoch - 1) + previousEpochNM.SetNodes([]netmap.NodeInfo{nonIssuerNode2}) + + obj := objectSDK.New() + obj.SetContainerID(cnrID) + obj.SetSessionToken(tok) + obj.SetOwnerID(&owner) + require.NoError(t, objectSDK.SetIDWithSignature(signer.PrivateKey, obj)) + + v := NewFormatValidator( + WithNetState(testNetState{ + epoch: curEpoch, + }), + WithLockSource(ls), + WithVerifySessionTokenIssuer(true), + WithInnerRing(&testIRSource{ + irNodes: [][]byte{}, + }), + WithContainersSource( + &testContainerSource{ + containers: map[cid.ID]*container.Container{ + cnrID: { + Value: cont, + }, + }, + }, + ), + WithNetmapSource( + &testNetmapSource{ + netmaps: map[uint64]*netmap.NetMap{ + curEpoch: currentEpochNM, + curEpoch - 1: previousEpochNM, + }, + currentEpoch: curEpoch, + }, + ), + ) + + require.Error(t, v.Validate(context.Background(), obj, false)) + }) +} + +type testIRSource struct { + irNodes [][]byte +} + +func (s *testIRSource) InnerRingKeys() ([][]byte, error) { + return s.irNodes, nil +} + +type testContainerSource struct { + containers map[cid.ID]*container.Container +} + +func (s *testContainerSource) Get(cnrID cid.ID) (*container.Container, error) { + if cnr, found := s.containers[cnrID]; found { + return cnr, nil + } + return nil, fmt.Errorf("container not found") +} + +func (s *testContainerSource) DeletionInfo(cid.ID) (*container.DelInfo, error) { + return nil, nil +} + +type testNetmapSource struct { + netmaps map[uint64]*netmap.NetMap + currentEpoch uint64 +} + +func (s *testNetmapSource) GetNetMap(diff uint64) (*netmap.NetMap, error) { + if diff >= s.currentEpoch { + return nil, fmt.Errorf("invalid diff") + } + return s.GetNetMapByEpoch(s.currentEpoch - diff) +} + +func (s *testNetmapSource) GetNetMapByEpoch(epoch uint64) (*netmap.NetMap, error) { + if nm, found := s.netmaps[epoch]; found { + return nm, nil + } + return nil, fmt.Errorf("netmap not found") +} + +func (s *testNetmapSource) Epoch() (uint64, error) { + return s.currentEpoch, nil +} diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 7f2600f9c..4095cf124 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -29,6 +29,14 @@ type ClientConstructor interface { Get(client.NodeInfo) (client.MultiAddressClient, error) } +type InnerRing interface { + InnerRingKeys() ([][]byte, error) +} + +type FormatValidatorConfig interface { + VerifySessionTokenIssuer() bool +} + type cfg struct { keyStorage *objutil.KeyStorage @@ -51,6 +59,8 @@ type cfg struct { clientConstructor ClientConstructor log *logger.Logger + + verifySessionTokenIssuer bool } func NewService(ks *objutil.KeyStorage, @@ -61,6 +71,7 @@ func NewService(ks *objutil.KeyStorage, ns netmap.Source, nk netmap.AnnouncedKeys, nst netmap.State, + ir InnerRing, opts ...Option) *Service { c := &cfg{ remotePool: util.NewPseudoWorkerPool(), @@ -80,7 +91,14 @@ func NewService(ks *objutil.KeyStorage, opts[i](c) } - c.fmtValidator = object.NewFormatValidator(object.WithLockSource(os), object.WithNetState(nst)) + c.fmtValidator = object.NewFormatValidator( + object.WithLockSource(os), + object.WithNetState(nst), + object.WithInnerRing(ir), + object.WithNetmapSource(ns), + object.WithContainersSource(cs), + object.WithVerifySessionTokenIssuer(c.verifySessionTokenIssuer), + ) return &Service{ cfg: c, @@ -104,3 +122,9 @@ func WithLogger(l *logger.Logger) Option { c.log = l } } + +func WithVerifySessionTokenIssuer(v bool) Option { + return func(c *cfg) { + c.verifySessionTokenIssuer = v + } +} From 55b82e744b822a60f4b84c8b389e08ad13d1cca1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 7 Aug 2023 09:54:47 +0300 Subject: [PATCH 0557/1943] [#529] objectcore: Use common sender classifier Use common sender classifier for ACL service and format validator. Signed-off-by: Dmitrii Stepanov --- pkg/core/object/fmt.go | 98 +++---------------- pkg/core/object/fmt_test.go | 27 ++++- .../object/sender_classifier.go} | 70 +++++++------ pkg/services/object/acl/v2/service.go | 19 ++-- pkg/services/object/put/service.go | 1 + 5 files changed, 95 insertions(+), 120 deletions(-) rename pkg/{services/object/acl/v2/classifier.go => core/object/sender_classifier.go} (68%) diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index 90365cead..e65767723 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -1,10 +1,8 @@ package object import ( - "bytes" "context" "crypto/ecdsa" - "crypto/elliptic" "crypto/sha256" "errors" "fmt" @@ -14,18 +12,20 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" - netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) // FormatValidator represents an object format validator. type FormatValidator struct { *cfg + + senderClassifier SenderClassifier } // FormatValidatorOption represents a FormatValidator constructor option. @@ -37,13 +37,10 @@ type cfg struct { ir InnerRing netmap netmap.Source containers container.Source + log *logger.Logger verifyTokenIssuer bool } -type InnerRing interface { - InnerRingKeys() ([][]byte, error) -} - // DeleteHandler is an interface of delete queue processor. type DeleteHandler interface { // DeleteObjects places objects to a removal queue. @@ -93,7 +90,8 @@ func NewFormatValidator(opts ...FormatValidatorOption) *FormatValidator { } return &FormatValidator{ - cfg: cfg, + cfg: cfg, + senderClassifier: NewSenderClassifier(cfg.ir, cfg.netmap, cfg.log), } } @@ -193,42 +191,6 @@ func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { } func (v *FormatValidator) isIROrContainerNode(obj *objectSDK.Object, signerKey []byte) (bool, error) { - pKey, err := keys.NewPublicKeyFromBytes(signerKey, elliptic.P256()) - if err != nil { - return false, fmt.Errorf("(%T) failed to unmarshal signer public key: %w", v, err) - } - - isIR, err := v.isInnerRingKey(pKey.Bytes()) - if err != nil { - return false, fmt.Errorf("(%T) failed to check if signer is inner ring node: %w", v, err) - } - if isIR { - return true, nil - } - - isContainerNode, err := v.isContainerNode(pKey.Bytes(), obj) - if err != nil { - return false, fmt.Errorf("(%T) failed to check if signer is container node: %w", v, err) - } - return isContainerNode, nil -} - -func (v *FormatValidator) isInnerRingKey(key []byte) (bool, error) { - innerRingKeys, err := v.ir.InnerRingKeys() - if err != nil { - return false, err - } - - for i := range innerRingKeys { - if bytes.Equal(innerRingKeys[i], key) { - return true, nil - } - } - - return false, nil -} - -func (v *FormatValidator) isContainerNode(key []byte, obj *objectSDK.Object) (bool, error) { cnrID, containerIDSet := obj.ContainerID() if !containerIDSet { return false, errNilCID @@ -242,46 +204,11 @@ func (v *FormatValidator) isContainerNode(key []byte, obj *objectSDK.Object) (bo return false, fmt.Errorf("failed to get container (id=%s): %w", cnrID.EncodeToString(), err) } - lastNetmap, err := netmap.GetLatestNetworkMap(v.netmap) - if err != nil { - return false, fmt.Errorf("failed to get latest netmap: %w", err) - } - - isContainerNode, err := v.isContainerNodeKey(lastNetmap, cnr, cnrIDBin, key) - if err != nil { - return false, fmt.Errorf("failed to check latest netmap for container nodes: %w", err) - } - if isContainerNode { - return true, nil - } - - previousNetmap, err := netmap.GetPreviousNetworkMap(v.netmap) - if err != nil { - return false, fmt.Errorf("failed to get previous netmap: %w", err) - } - - isContainerNode, err = v.isContainerNodeKey(previousNetmap, cnr, cnrIDBin, key) - if err != nil { - return false, fmt.Errorf("failed to check previous netmap for container nodes: %w", err) - } - return isContainerNode, nil -} - -func (v *FormatValidator) isContainerNodeKey(nm *netmapSDK.NetMap, cnr *container.Container, cnrIDBin, key []byte) (bool, error) { - cnrVectors, err := nm.ContainerNodes(cnr.Value.PlacementPolicy(), cnrIDBin) + res, err := v.senderClassifier.IsInnerRingOrContainerNode(signerKey, cnrID, cnr.Value) if err != nil { return false, err } - - for i := range cnrVectors { - for j := range cnrVectors[i] { - if bytes.Equal(cnrVectors[i][j].PublicKey(), key) { - return true, nil - } - } - } - - return false, nil + return res.Role == acl.RoleContainer || res.Role == acl.RoleInnerRing, nil } func (v *FormatValidator) checkOwnerKey(id user.ID, key frostfsecdsa.PublicKey) error { @@ -533,3 +460,10 @@ func WithVerifySessionTokenIssuer(verifySessionTokenIssuer bool) FormatValidator c.verifyTokenIssuer = verifySessionTokenIssuer } } + +// WithLogger return option to set logger. +func WithLogger(l *logger.Logger) FormatValidatorOption { + return func(c *cfg) { + c.log = l + } +} diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index fbd82906d..2a5b5690c 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -9,6 +9,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" @@ -22,6 +23,7 @@ import ( "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" ) func blankValidObject(key *ecdsa.PrivateKey) *objectSDK.Object { @@ -63,6 +65,7 @@ func TestFormatValidator_Validate(t *testing.T) { epoch: curEpoch, }), WithLockSource(ls), + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), ) ownerKey, err := keys.NewPrivateKey() @@ -285,6 +288,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { }), WithLockSource(ls), WithVerifySessionTokenIssuer(false), + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), ) tok := sessiontest.Object() @@ -307,6 +311,14 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { t.Run("different issuer and owner, issuer is IR node, verify issuer enabled", func(t *testing.T) { t.Parallel() + + cnrID := cidtest.ID() + cont := containerSDK.Container{} + cont.Init() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + cont.SetPlacementPolicy(pp) + v := NewFormatValidator( WithNetState(testNetState{ epoch: curEpoch, @@ -316,6 +328,16 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { WithInnerRing(&testIRSource{ irNodes: [][]byte{signer.PublicKey().Bytes()}, }), + WithContainersSource( + &testContainerSource{ + containers: map[cid.ID]*container.Container{ + cnrID: { + Value: cont, + }, + }, + }, + ), + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), ) tok := sessiontest.Object() @@ -328,7 +350,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { require.NoError(t, tok.Sign(signer.PrivateKey)) obj := objectSDK.New() - obj.SetContainerID(cidtest.ID()) + obj.SetContainerID(cnrID) obj.SetSessionToken(tok) obj.SetOwnerID(&owner) require.NoError(t, objectSDK.SetIDWithSignature(signer.PrivateKey, obj)) @@ -393,6 +415,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { currentEpoch: curEpoch, }, ), + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), ) require.NoError(t, v.Validate(context.Background(), obj, false)) @@ -466,6 +489,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { currentEpoch: curEpoch, }, ), + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), ) require.NoError(t, v.Validate(context.Background(), obj, false)) @@ -541,6 +565,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { currentEpoch: curEpoch, }, ), + WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), ) require.Error(t, v.Validate(context.Background(), obj, false)) diff --git a/pkg/services/object/acl/v2/classifier.go b/pkg/core/object/sender_classifier.go similarity index 68% rename from pkg/services/object/acl/v2/classifier.go rename to pkg/core/object/sender_classifier.go index cdc5fb623..79bf12ce3 100644 --- a/pkg/services/object/acl/v2/classifier.go +++ b/pkg/core/object/sender_classifier.go @@ -1,4 +1,4 @@ -package v2 +package object import ( "bytes" @@ -11,50 +11,64 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.uber.org/zap" ) -type senderClassifier struct { +type InnerRing interface { + InnerRingKeys() ([][]byte, error) +} + +type SenderClassifier struct { log *logger.Logger - innerRing InnerRingFetcher + innerRing InnerRing netmap core.Source } -type classifyResult struct { - role acl.Role - key []byte +func NewSenderClassifier(innerRing InnerRing, netmap core.Source, log *logger.Logger) SenderClassifier { + return SenderClassifier{ + log: log, + innerRing: innerRing, + netmap: netmap, + } } -func (c senderClassifier) classify( - req MetaWithToken, - idCnr cid.ID, - cnr container.Container) (res *classifyResult, err error) { - ownerID, ownerKey, err := req.RequestOwner() - if err != nil { - return nil, err - } +type ClassifyResult struct { + Role acl.Role + Key []byte +} +func (c SenderClassifier) Classify( + ownerID *user.ID, + ownerKey *keys.PublicKey, + idCnr cid.ID, + cnr container.Container) (res *ClassifyResult, err error) { ownerKeyInBytes := ownerKey.Bytes() // TODO: #767 get owner from frostfs.id if present // if request owner is the same as container owner, return RoleUser if ownerID.Equals(cnr.Owner()) { - return &classifyResult{ - role: acl.RoleOwner, - key: ownerKeyInBytes, + return &ClassifyResult{ + Role: acl.RoleOwner, + Key: ownerKeyInBytes, }, nil } + return c.IsInnerRingOrContainerNode(ownerKeyInBytes, idCnr, cnr) +} + +func (c SenderClassifier) IsInnerRingOrContainerNode(ownerKeyInBytes []byte, idCnr cid.ID, cnr container.Container) (*ClassifyResult, error) { isInnerRingNode, err := c.isInnerRingKey(ownerKeyInBytes) if err != nil { // do not throw error, try best case matching c.log.Debug(logs.V2CantCheckIfRequestFromInnerRing, zap.String("error", err.Error())) } else if isInnerRingNode { - return &classifyResult{ - role: acl.RoleInnerRing, - key: ownerKeyInBytes, + return &ClassifyResult{ + Role: acl.RoleInnerRing, + Key: ownerKeyInBytes, }, nil } @@ -69,20 +83,20 @@ func (c senderClassifier) classify( c.log.Debug(logs.V2CantCheckIfRequestFromContainerNode, zap.String("error", err.Error())) } else if isContainerNode { - return &classifyResult{ - role: acl.RoleContainer, - key: ownerKeyInBytes, + return &ClassifyResult{ + Role: acl.RoleContainer, + Key: ownerKeyInBytes, }, nil } // if none of above, return RoleOthers - return &classifyResult{ - role: acl.RoleOthers, - key: ownerKeyInBytes, + return &ClassifyResult{ + Role: acl.RoleOthers, + Key: ownerKeyInBytes, }, nil } -func (c senderClassifier) isInnerRingKey(owner []byte) (bool, error) { +func (c SenderClassifier) isInnerRingKey(owner []byte) (bool, error) { innerRingKeys, err := c.innerRing.InnerRingKeys() if err != nil { return false, err @@ -98,7 +112,7 @@ func (c senderClassifier) isInnerRingKey(owner []byte) (bool, error) { return false, nil } -func (c senderClassifier) isContainerKey( +func (c SenderClassifier) isContainerKey( owner, idCnr []byte, cnr container.Container) (bool, error) { nm, err := core.GetLatestNetworkMap(c.netmap) // first check current netmap diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 271c4d20b..8239403a7 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -24,7 +25,7 @@ import ( type Service struct { *cfg - c senderClassifier + c objectCore.SenderClassifier } type putStreamBasicChecker struct { @@ -95,11 +96,7 @@ func New(next object.ServiceServer, return Service{ cfg: cfg, - c: senderClassifier{ - log: cfg.log, - innerRing: cfg.irFetcher, - netmap: cfg.nm, - }, + c: objectCore.NewSenderClassifier(cfg.irFetcher, cfg.nm, cfg.log), } } @@ -652,20 +649,24 @@ func (b Service) findRequestInfo(req MetaWithToken, idCnr cid.ID, op acl.Op) (in } // find request role and key - res, err := b.c.classify(req, idCnr, cnr.Value) + ownerID, ownerKey, err := req.RequestOwner() + if err != nil { + return info, err + } + res, err := b.c.Classify(ownerID, ownerKey, idCnr, cnr.Value) if err != nil { return info, err } info.basicACL = cnr.Value.BasicACL() - info.requestRole = res.role + info.requestRole = res.Role info.operation = op info.cnrOwner = cnr.Value.Owner() info.idCnr = idCnr // it is assumed that at the moment the key will be valid, // otherwise the request would not pass validation - info.senderKey = res.key + info.senderKey = res.Key // add bearer token if it is present in request info.bearer = req.bearer diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 4095cf124..3a7dcefd6 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -98,6 +98,7 @@ func NewService(ks *objutil.KeyStorage, object.WithNetmapSource(ns), object.WithContainersSource(cs), object.WithVerifySessionTokenIssuer(c.verifySessionTokenIssuer), + object.WithLogger(c.log), ) return &Service{ From a724debb199c9a4b28473a5847979495cd99dd35 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 29 Aug 2023 12:41:45 +0300 Subject: [PATCH 0558/1943] [#632] .forgejo: Print --version Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 51f63e5a5..3c74d9434 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -24,6 +24,7 @@ jobs: - name: Build CLI run: make bin/frostfs-cli + - run: bin/frostfs-cli --version - name: Build NODE run: make bin/frostfs-node @@ -33,6 +34,8 @@ jobs: - name: Build ADM run: make bin/frostfs-adm + - run: bin/frostfs-adm --version - name: Build LENS run: make bin/frostfs-lens + - run: bin/frostfs-lens --version From 7654847f79265ca6663eea1224d2e4da546a9215 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 29 Aug 2023 13:41:57 +0300 Subject: [PATCH 0559/1943] [#659] adm: Remove nspcc.ru from the default email Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/deploy.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/initialize_nns.go | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/deploy.go b/cmd/frostfs-adm/internal/modules/morph/deploy.go index a9098891c..a4b945438 100644 --- a/cmd/frostfs-adm/internal/modules/morph/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/deploy.go @@ -145,12 +145,12 @@ func registerNNS(nnsCs *state.Contract, c *initializeContext, zone string, domai emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, zone, c.CommitteeAcc.Contract.ScriptHash(), - "ops@nspcc.ru", int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) + frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, domain, c.CommitteeAcc.Contract.ScriptHash(), - "ops@nspcc.ru", int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) + frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) } else { s, ok, err := c.nnsRegisterDomainScript(nnsCs.Hash, cs.Hash, domain) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go index 15657a6d9..6758b4dd8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go @@ -27,6 +27,8 @@ import ( const defaultExpirationTime = 10 * 365 * 24 * time.Hour / time.Second +const frostfsOpsEmail = "ops@frostfs.info" + func (c *initializeContext) setNNS() error { nnsCs, err := c.Client.GetContractStateByID(1) if err != nil { @@ -40,7 +42,7 @@ func (c *initializeContext) setNNS() error { bw := io.NewBufBinWriter() emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, "frostfs", c.CommitteeAcc.Contract.ScriptHash(), - "ops@nspcc.ru", int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) + frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) if err := c.sendCommitteeTx(bw.Bytes(), true); err != nil { return fmt.Errorf("can't add domain root to NNS: %w", err) @@ -122,7 +124,7 @@ func (c *initializeContext) emitUpdateNNSGroupScript(bw *io.BufBinWriter, nnsHas if isAvail { emit.AppCall(bw.BinWriter, nnsHash, "register", callflag.All, morphClient.NNSGroupKeyName, c.CommitteeAcc.Contract.ScriptHash(), - "ops@nspcc.ru", int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) + frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) } @@ -170,7 +172,7 @@ func (c *initializeContext) nnsRegisterDomainScript(nnsHash, expectedHash util.U bw := io.NewBufBinWriter() emit.AppCall(bw.BinWriter, nnsHash, "register", callflag.All, domain, c.CommitteeAcc.Contract.ScriptHash(), - "ops@nspcc.ru", int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) + frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) if bw.Err != nil { From 91f3745b58cfc551d17e4048a275df7509eb46b7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 29 Aug 2023 13:43:26 +0300 Subject: [PATCH 0560/1943] [#659] debian: Remove nspcc email Signed-off-by: Evgenii Stratonikov --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 575a27e33..f3f214bca 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: frostfs-node Section: misc Priority: optional -Maintainer: NeoSPCC +Maintainer: TrueCloudLab Build-Depends: debhelper-compat (= 13), dh-sequence-bash-completion, devscripts Standards-Version: 4.5.1 Homepage: https://fs.neo.org/ From fe5aa06a75a3afa36a2fd304e6f791d493284b35 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 30 Aug 2023 13:43:43 +0300 Subject: [PATCH 0561/1943] [#665] node: Bind length of copies number to number of replicas Allow to use one digit in copies number array for backward compatibility. Signed-off-by: Anton Nikiforov --- .../object_manager/placement/traverser.go | 8 +++++++ .../placement/traverser_test.go | 21 ++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index ad1077047..dc9ab5e7a 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -58,6 +58,9 @@ var errNilBuilder = errors.New("placement builder is nil") var errNilPolicy = errors.New("placement policy is nil") +var errCopiesNumberLen = errors.New("copies number accepts only one number or array with length " + + "equal to length of replicas") + func defaultCfg() *cfg { return &cfg{ trackCopies: true, @@ -74,6 +77,11 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } } + cnLen := len(cfg.copyNumbers) + if cnLen > 0 && cnLen != 1 && cnLen != cfg.policy.NumberOfReplicas() { + return nil, errCopiesNumberLen + } + if cfg.builder == nil { return nil, fmt.Errorf("%s: %w", invalidOptsMsg, errNilBuilder) } else if !cfg.policySet { diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index e16f33759..9b70efc73 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -220,6 +220,7 @@ func TestTraverserRemValues(t *testing.T) { name string copyNumbers []uint32 expectedRem []int + expectedErr error }{ { name: "zero copy numbers", @@ -231,11 +232,6 @@ func TestTraverserRemValues(t *testing.T) { copyNumbers: []uint32{0}, expectedRem: replicas, }, - { - name: "compatible zero copy numbers, len 2", - copyNumbers: []uint32{0, 0}, - expectedRem: replicas, - }, { name: "compatible zero copy numbers, len 3", copyNumbers: []uint32{0, 0, 0}, @@ -253,9 +249,14 @@ func TestTraverserRemValues(t *testing.T) { }, { name: "multiple copy numbers for multiple replicas", - copyNumbers: []uint32{1, 1}, + copyNumbers: []uint32{1, 1, 4}, expectedRem: []int{1, 1, 4}, }, + { + name: "incompatible copies number vector", + copyNumbers: []uint32{1, 1}, + expectedErr: errCopiesNumberLen, + }, } for _, testCase := range testCases { @@ -265,8 +266,12 @@ func TestTraverserRemValues(t *testing.T) { UseBuilder(&testBuilder{vectors: nodesCopy}), WithCopyNumbers(testCase.copyNumbers), ) - require.NoError(t, err) - require.Equal(t, testCase.expectedRem, tr.rem) + if testCase.expectedErr == nil { + require.NoError(t, err, testCase.name) + require.Equal(t, testCase.expectedRem, tr.rem, testCase.name) + } else { + require.Error(t, err, testCase.expectedErr, testCase.name) + } }) } } From 1daef2ceeb44efe26d4f1a4f4c6d59fe3891c6a6 Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Tue, 29 Aug 2023 15:26:22 +0300 Subject: [PATCH 0562/1943] [#660] writecache: Fix remaining addr2key uses Signed-off-by: Alejandro Lopez --- .../writecache/writecachebadger/delete.go | 8 ++++---- .../writecache/writecachebadger/flush_test.go | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/writecache/writecachebadger/delete.go b/pkg/local_object_storage/writecache/writecachebadger/delete.go index f37371098..afab78547 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/delete.go +++ b/pkg/local_object_storage/writecache/writecachebadger/delete.go @@ -39,10 +39,10 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { return writecache.ErrReadOnly } - saddr := addr.EncodeToString() + key := addr2key(addr) err := c.db.Update(func(tx *badger.Txn) error { - it, err := tx.Get([]byte(saddr)) + it, err := tx.Get(key[:]) if err != nil { if err == badger.ErrKeyNotFound { return logicerr.Wrap(new(apistatus.ObjectNotFound)) @@ -51,10 +51,10 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { } if it.ValueSize() > 0 { storageType = writecache.StorageTypeDB - err := tx.Delete([]byte(saddr)) + err := tx.Delete(key[:]) if err == nil { storagelog.Write(c.log, - storagelog.AddressField(saddr), + storagelog.AddressField(addr.EncodeToString()), storagelog.StorageTypeField(wcStorageType), storagelog.OpField("db DELETE"), ) diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go index 19bc15614..855d2214f 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go @@ -52,8 +52,9 @@ func TestFlush(t *testing.T) { Desc: "db, invalid object", InjectFn: func(t *testing.T, wc writecache.Cache) { c := wc.(*cache) + key := addr2key(oidtest.Address()) require.NoError(t, c.db.Update(func(tx *badger.Txn) error { - return tx.Set([]byte(oidtest.Address().EncodeToString()), []byte{1, 2, 3}) + return tx.Set(key[:], []byte{1, 2, 3}) })) }, }, From 806cc13d9f49a48a22c09f170127bff326019a8a Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 29 Aug 2023 13:25:54 +0300 Subject: [PATCH 0563/1943] [#658] client: Refactor PrmObjectGet/Head/Range usage Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/client.go | 96 +++++++------------ go.mod | 2 +- go.sum | 4 +- pkg/services/object/internal/client/client.go | 86 +++++++---------- 4 files changed, 73 insertions(+), 115 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 0c6d2a38d..22ca0aa7d 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -563,28 +563,19 @@ func (x GetObjectRes) Header() *objectSDK.Object { // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { - var getPrm client.PrmObjectGet - getPrm.FromContainer(prm.objAddr.Container()) - getPrm.ByID(prm.objAddr.Object()) + cnr := prm.objAddr.Container() + obj := prm.objAddr.Object() - if prm.sessionToken != nil { - getPrm.WithinSession(*prm.sessionToken) + getPrm := client.PrmObjectGet{ + XHeaders: prm.xHeaders, + BearerToken: prm.bearerToken, + Session: prm.sessionToken, + Raw: prm.raw, + Local: prm.local, + ContainerID: &cnr, + ObjectID: &obj, } - if prm.bearerToken != nil { - getPrm.WithBearerToken(*prm.bearerToken) - } - - if prm.raw { - getPrm.MarkRaw() - } - - if prm.local { - getPrm.MarkLocal() - } - - getPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectGetInit(ctx, getPrm) if err != nil { return nil, fmt.Errorf("init object reading on client: %w", err) @@ -639,29 +630,20 @@ func (x HeadObjectRes) Header() *objectSDK.Object { // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) { - var cliPrm client.PrmObjectHead - cliPrm.FromContainer(prm.objAddr.Container()) - cliPrm.ByID(prm.objAddr.Object()) + cnr := prm.objAddr.Container() + obj := prm.objAddr.Object() - if prm.sessionToken != nil { - cliPrm.WithinSession(*prm.sessionToken) + headPrm := client.PrmObjectHead{ + XHeaders: prm.xHeaders, + BearerToken: prm.bearerToken, + Session: prm.sessionToken, + Raw: prm.raw, + Local: prm.local, + ContainerID: &cnr, + ObjectID: &obj, } - if prm.bearerToken != nil { - cliPrm.WithBearerToken(*prm.bearerToken) - } - - if prm.raw { - cliPrm.MarkRaw() - } - - if prm.local { - cliPrm.MarkLocal() - } - - cliPrm.WithXHeaders(prm.xHeaders...) - - res, err := prm.cli.ObjectHead(ctx, cliPrm) + res, err := prm.cli.ObjectHead(ctx, headPrm) if err != nil { return nil, fmt.Errorf("read object header via client: %w", err) } @@ -862,32 +844,22 @@ type PayloadRangeRes struct{} // Returns any error which prevented the operation from completing correctly in error return. // For raw reading, returns *object.SplitInfoError error if object is virtual. func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, error) { - var cliPrm client.PrmObjectRange - cliPrm.FromContainer(prm.objAddr.Container()) - cliPrm.ByID(prm.objAddr.Object()) + cnr := prm.objAddr.Container() + obj := prm.objAddr.Object() - if prm.sessionToken != nil { - cliPrm.WithinSession(*prm.sessionToken) + rangePrm := client.PrmObjectRange{ + XHeaders: prm.xHeaders, + BearerToken: prm.bearerToken, + Session: prm.sessionToken, + Raw: prm.raw, + Local: prm.local, + ContainerID: &cnr, + ObjectID: &obj, + Offset: prm.rng.GetOffset(), + Length: prm.rng.GetLength(), } - if prm.bearerToken != nil { - cliPrm.WithBearerToken(*prm.bearerToken) - } - - if prm.raw { - cliPrm.MarkRaw() - } - - if prm.local { - cliPrm.MarkLocal() - } - - cliPrm.SetOffset(prm.rng.GetOffset()) - cliPrm.SetLength(prm.rng.GetLength()) - - cliPrm.WithXHeaders(prm.xHeaders...) - - rdr, err := prm.cli.ObjectRangeInit(ctx, cliPrm) + rdr, err := prm.cli.ObjectRangeInit(ctx, rangePrm) if err != nil { return nil, fmt.Errorf("init payload reading: %w", err) } diff --git a/go.mod b/go.mod index 46639d49c..cbe20e39f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230828082657-84e7e69f98ac git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 58f1fb217..4b5e40174 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05 h1:OuViMF54N87FXmaBEpYw3jhzaLrJ/EWOlPL1wUkimE0= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230821090303-202412230a05/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230828082657-84e7e69f98ac h1:BRmJd04eUgzmdLe/2I0Ok/LNOkTyEAPymYUrCCF0a1U= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230828082657-84e7e69f98ac/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 73f4ff7c4..0301a593a 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -88,7 +88,7 @@ func (x *readPrmCommon) SetNetmapEpoch(_ uint64) { type GetObjectPrm struct { readPrmCommon - cliPrm client.PrmObjectGet + ClientParams client.PrmObjectGet obj oid.ID } @@ -97,7 +97,7 @@ type GetObjectPrm struct { // // By default request will not be raw. func (x *GetObjectPrm) SetRawFlag() { - x.cliPrm.MarkRaw() + x.ClientParams.Raw = true } // SetAddress sets object address. @@ -105,8 +105,10 @@ func (x *GetObjectPrm) SetRawFlag() { // Required parameter. func (x *GetObjectPrm) SetAddress(addr oid.Address) { x.obj = addr.Object() - x.cliPrm.FromContainer(addr.Container()) - x.cliPrm.ByID(x.obj) + cnr := addr.Container() + + x.ClientParams.ContainerID = &cnr + x.ClientParams.ObjectID = &x.obj } // GetObjectRes groups the resulting values of GetObject operation. @@ -134,23 +136,15 @@ func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { // request will almost definitely fail. The case can occur, for example, // when session is bound to the parent object and child object is requested. if prm.tokenSession != nil && prm.tokenSession.AssertObject(prm.obj) { - prm.cliPrm.WithinSession(*prm.tokenSession) + prm.ClientParams.Session = prm.tokenSession } - if prm.tokenBearer != nil { - prm.cliPrm.WithBearerToken(*prm.tokenBearer) - } + prm.ClientParams.XHeaders = prm.xHeaders + prm.ClientParams.BearerToken = prm.tokenBearer + prm.ClientParams.Local = prm.local + prm.ClientParams.Key = prm.key - if prm.local { - prm.cliPrm.MarkLocal() - } - - prm.cliPrm.WithXHeaders(prm.xHeaders...) - if prm.key != nil { - prm.cliPrm.UseKey(*prm.key) - } - - rdr, err := prm.cli.ObjectGetInit(ctx, prm.cliPrm) + rdr, err := prm.cli.ObjectGetInit(ctx, prm.ClientParams) if err != nil { return nil, fmt.Errorf("init object reading: %w", err) } @@ -187,7 +181,7 @@ func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { type HeadObjectPrm struct { readPrmCommon - cliPrm client.PrmObjectHead + ClientParams client.PrmObjectHead obj oid.ID } @@ -196,7 +190,7 @@ type HeadObjectPrm struct { // // By default request will not be raw. func (x *HeadObjectPrm) SetRawFlag() { - x.cliPrm.MarkRaw() + x.ClientParams.Raw = true } // SetAddress sets object address. @@ -204,8 +198,10 @@ func (x *HeadObjectPrm) SetRawFlag() { // Required parameter. func (x *HeadObjectPrm) SetAddress(addr oid.Address) { x.obj = addr.Object() - x.cliPrm.FromContainer(addr.Container()) - x.cliPrm.ByID(x.obj) + cnr := addr.Container() + + x.ClientParams.ContainerID = &cnr + x.ClientParams.ObjectID = &x.obj } // HeadObjectRes groups the resulting values of GetObject operation. @@ -230,22 +226,16 @@ func (x HeadObjectRes) Header() *objectSDK.Object { // // HeadObject ignores the provided session if it is not related to the requested objectSDK. func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) { - if prm.local { - prm.cliPrm.MarkLocal() - } - // see details in same statement of GetObject if prm.tokenSession != nil && prm.tokenSession.AssertObject(prm.obj) { - prm.cliPrm.WithinSession(*prm.tokenSession) + prm.ClientParams.Session = prm.tokenSession } - if prm.tokenBearer != nil { - prm.cliPrm.WithBearerToken(*prm.tokenBearer) - } + prm.ClientParams.BearerToken = prm.tokenBearer + prm.ClientParams.Local = prm.local + prm.ClientParams.XHeaders = prm.xHeaders - prm.cliPrm.WithXHeaders(prm.xHeaders...) - - cliRes, err := prm.cli.ObjectHead(ctx, prm.cliPrm) + cliRes, err := prm.cli.ObjectHead(ctx, prm.ClientParams) if err == nil { // pull out an error from status err = apistatus.ErrFromStatus(cliRes.Status()) @@ -272,7 +262,7 @@ type PayloadRangePrm struct { ln uint64 - cliPrm client.PrmObjectRange + ClientParams client.PrmObjectRange obj oid.ID } @@ -281,7 +271,7 @@ type PayloadRangePrm struct { // // By default request will not be raw. func (x *PayloadRangePrm) SetRawFlag() { - x.cliPrm.MarkRaw() + x.ClientParams.Raw = true } // SetAddress sets object address. @@ -289,15 +279,17 @@ func (x *PayloadRangePrm) SetRawFlag() { // Required parameter. func (x *PayloadRangePrm) SetAddress(addr oid.Address) { x.obj = addr.Object() - x.cliPrm.FromContainer(addr.Container()) - x.cliPrm.ByID(x.obj) + cnr := addr.Container() + + x.ClientParams.ContainerID = &cnr + x.ClientParams.ObjectID = &x.obj } // SetRange range of the object payload to be read. // // Required parameter. func (x *PayloadRangePrm) SetRange(rng *objectSDK.Range) { - x.cliPrm.SetOffset(rng.GetOffset()) + x.ClientParams.Offset = rng.GetOffset() x.ln = rng.GetLength() } @@ -329,23 +321,17 @@ const maxInitialBufferSize = 1024 * 1024 // 1 MiB // // PayloadRange ignores the provided session if it is not related to the requested objectSDK. func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, error) { - if prm.local { - prm.cliPrm.MarkLocal() - } - // see details in same statement of GetObject if prm.tokenSession != nil && prm.tokenSession.AssertObject(prm.obj) { - prm.cliPrm.WithinSession(*prm.tokenSession) + prm.ClientParams.Session = prm.tokenSession } - if prm.tokenBearer != nil { - prm.cliPrm.WithBearerToken(*prm.tokenBearer) - } + prm.ClientParams.XHeaders = prm.xHeaders + prm.ClientParams.BearerToken = prm.tokenBearer + prm.ClientParams.Local = prm.local + prm.ClientParams.Length = prm.ln - prm.cliPrm.SetLength(prm.ln) - prm.cliPrm.WithXHeaders(prm.xHeaders...) - - rdr, err := prm.cli.ObjectRangeInit(ctx, prm.cliPrm) + rdr, err := prm.cli.ObjectRangeInit(ctx, prm.ClientParams) if err != nil { return nil, fmt.Errorf("init payload reading: %w", err) } From 4dff9555f15145768608b130b752b4788955a4c6 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 23 Aug 2023 10:05:40 +0300 Subject: [PATCH 0564/1943] [#568] writecache: Improve flushing scheme for badger Signed-off-by: Anton Nikiforov --- go.mod | 2 +- internal/logs/logs.go | 1 + .../writecachebadger/cachebadger.go | 9 +- .../writecache/writecachebadger/flush.go | 142 ++++++++++-------- 4 files changed, 89 insertions(+), 65 deletions(-) diff --git a/go.mod b/go.mod index cbe20e39f..114c95adb 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 + github.com/dgraph-io/ristretto v0.1.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/uuid v1.3.0 github.com/hashicorp/golang-lru/v2 v2.0.4 @@ -42,7 +43,6 @@ require ( ) require ( - github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.1.0 // indirect diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 95961cd47..6ceee4f17 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -296,6 +296,7 @@ const ( WritecacheCantParseAddress = "can't parse address" WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" WritecacheDBValueLogGCRunCompleted = "value log GC run completed" + WritecacheBadgerObjAlreadyScheduled = "object already scheduled for flush" BlobovniczatreeCouldNotGetObjectFromLevel = "could not get object from level" BlobovniczatreeCouldNotReadPayloadRangeFromOpenedBlobovnicza = "could not read payload range from opened blobovnicza" BlobovniczatreeCouldNotReadPayloadRangeFromActiveBlobovnicza = "could not read payload range from active blobovnicza" diff --git a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go index 837e76a0b..28aa7e766 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go +++ b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go @@ -21,6 +21,10 @@ type cache struct { // flushCh is a channel with objects to flush. flushCh chan *objectSDK.Object + // scheduled4Flush contains objects scheduled for flush via flushCh + // helps to avoid multiple flushing of one object + scheduled4Flush map[oid.Address]struct{} + scheduled4FlushMtx sync.RWMutex // closeCh is close channel, protected by modeMtx. closeCh chan struct{} // wg is a wait group for flush workers. @@ -47,8 +51,9 @@ const ( // New creates new writecache instance. func New(opts ...Option) writecache.Cache { c := &cache{ - flushCh: make(chan *objectSDK.Object), - mode: mode.ReadWrite, + flushCh: make(chan *objectSDK.Object), + mode: mode.ReadWrite, + scheduled4Flush: map[oid.Address]struct{}{}, options: options{ log: &logger.Logger{Logger: zap.NewNop()}, diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush.go b/pkg/local_object_storage/writecache/writecachebadger/flush.go index 7f41920ba..ab6910a80 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush.go @@ -18,7 +18,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/dgraph-io/badger/v4" - "github.com/mr-tron/base58" + "github.com/dgraph-io/ristretto/z" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -35,6 +35,65 @@ const ( defaultFlushInterval = time.Second ) +type collector struct { + cache *cache + scheduled int + processed int + cancel func() +} + +func (c *collector) Send(buf *z.Buffer) error { + list, err := badger.BufferToKVList(buf) + if err != nil { + return err + } + for _, kv := range list.Kv { + select { + case <-c.cache.closeCh: + c.cancel() + return nil + default: + } + if kv.StreamDone { + return nil + } + if c.scheduled >= flushBatchSize { + c.cancel() + return nil + } + if got, want := len(kv.Key), len(internalKey{}); got != want { + c.cache.log.Debug( + fmt.Sprintf("not expected db key len: got %d, want %d", got, want)) + continue + } + c.processed++ + obj := objectSDK.New() + val := bytes.Clone(kv.Value) + if err = obj.Unmarshal(val); err != nil { + continue + } + addr := objectCore.AddressOf(obj) + c.cache.scheduled4FlushMtx.RLock() + _, ok := c.cache.scheduled4Flush[addr] + c.cache.scheduled4FlushMtx.RUnlock() + if ok { + c.cache.log.Debug(logs.WritecacheBadgerObjAlreadyScheduled, zap.Stringer("obj", addr)) + continue + } + c.cache.scheduled4FlushMtx.Lock() + c.cache.scheduled4Flush[addr] = struct{}{} + c.cache.scheduled4FlushMtx.Unlock() + c.scheduled++ + select { + case c.cache.flushCh <- obj: + case <-c.cache.closeCh: + c.cancel() + return nil + } + } + return nil +} + // runFlushLoop starts background workers which periodically flush objects to the blobstor. func (c *cache) runFlushLoop() { for i := 0; i < c.workersCount; i++ { @@ -62,17 +121,12 @@ func (c *cache) runFlushLoop() { } func (c *cache) flushSmallObjects() { - var lastKey internalKey - var m []objectInfo for { select { case <-c.closeCh: return default: } - - m = m[:0] - c.modeMtx.RLock() if c.readOnly() { c.modeMtx.RUnlock() @@ -86,61 +140,24 @@ func (c *cache) flushSmallObjects() { c.modeMtx.RUnlock() return } - - _ = c.db.View(func(tx *badger.Txn) error { - it := tx.NewIterator(badger.DefaultIteratorOptions) - defer it.Close() - if len(lastKey) == 0 { - it.Rewind() - } else { - it.Seek(lastKey[:]) - if it.Valid() && bytes.Equal(it.Item().Key(), lastKey[:]) { - it.Next() - } - } - for ; it.Valid() && len(m) < flushBatchSize; it.Next() { - if got, want := int(it.Item().KeySize()), len(lastKey); got != want { - return fmt.Errorf("invalid db key len: got %d, want %d", got, want) - } - it.Item().KeyCopy(lastKey[:]) - value, err := it.Item().ValueCopy(nil) - if err != nil { - return err - } - m = append(m, objectInfo{ - addr: lastKey.address(), - data: value, - }) - } - return nil - }) - - var count int - for i := range m { - obj := objectSDK.New() - if err := obj.Unmarshal(m[i].data); err != nil { - continue - } - - count++ - select { - case c.flushCh <- obj: - case <-c.closeCh: - c.modeMtx.RUnlock() - return - } + ctx, cancel := context.WithCancel(context.TODO()) + coll := collector{ + cache: c, + cancel: cancel, } - - if count == 0 { - c.modeMtx.RUnlock() + stream := c.db.NewStream() + // All calls to Send are done by a single goroutine + stream.Send = coll.Send + if err := stream.Orchestrate(ctx); err != nil { + c.log.Debug(fmt.Sprintf( + "error during flushing object from wc: %s", err)) + } + c.modeMtx.RUnlock() + if coll.scheduled == 0 { break } - - c.modeMtx.RUnlock() - c.log.Debug(logs.WritecacheTriedToFlushItemsFromWritecache, - zap.Int("count", count), - zap.String("start", base58.Encode(lastKey[:]))) + zap.Int("scheduled", coll.scheduled), zap.Int("processed", coll.processed)) } } @@ -167,13 +184,14 @@ func (c *cache) workerFlushSmall() { return } + addr := objectCore.AddressOf(obj) err := c.flushObject(context.TODO(), obj, nil, writecache.StorageTypeDB) - if err != nil { - // Error is handled in flushObject. - continue + if err == nil { + c.deleteFromDB([]internalKey{addr2key(addr)}) } - - c.deleteFromDB([]internalKey{addr2key(objectCore.AddressOf(obj))}) + c.scheduled4FlushMtx.Lock() + delete(c.scheduled4Flush, addr) + c.scheduled4FlushMtx.Unlock() } } From b9b86d2ec87318c3e41cd0d3b799919d3e821151 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 30 Aug 2023 15:24:31 +0300 Subject: [PATCH 0565/1943] [#666] shard/test: Fix data race in metrics tests Signed-off-by: Evgenii Stratonikov --- .../shard/metrics_test.go | 71 ++++++++++++------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 91e158b04..3d904331e 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -36,6 +36,29 @@ func (m *metricsStore) SetObjectCounter(objectType string, v uint64) { m.objCounters[objectType] = v } +func (m *metricsStore) getObjectCounter(objectType string) uint64 { + m.mtx.Lock() + defer m.mtx.Unlock() + return m.objCounters[objectType] +} + +func (m *metricsStore) containerSizes() map[string]int64 { + m.mtx.Lock() + defer m.mtx.Unlock() + + r := make(map[string]int64, len(m.cnrSize)) + for c, s := range m.cnrSize { + r[c] = s + } + return r +} + +func (m *metricsStore) payloadSize() int64 { + m.mtx.Lock() + defer m.mtx.Unlock() + return m.pldSize +} + func (m *metricsStore) AddToObjectCounter(objectType string, delta int) { m.mtx.Lock() defer m.mtx.Unlock() @@ -124,10 +147,10 @@ func TestCounters(t *testing.T) { } t.Run("defaults", func(t *testing.T) { - require.Zero(t, mm.objCounters[physical]) - require.Zero(t, mm.objCounters[logical]) - require.Empty(t, mm.cnrSize) - require.Zero(t, mm.pldSize) + require.Zero(t, mm.getObjectCounter(physical)) + require.Zero(t, mm.getObjectCounter(logical)) + require.Empty(t, mm.containerSizes()) + require.Zero(t, mm.payloadSize()) }) var totalPayload int64 @@ -150,10 +173,10 @@ func TestCounters(t *testing.T) { require.NoError(t, err) } - require.Equal(t, uint64(objNumber), mm.objCounters[physical]) - require.Equal(t, uint64(objNumber), mm.objCounters[logical]) - require.Equal(t, expectedSizes, mm.cnrSize) - require.Equal(t, totalPayload, mm.pldSize) + require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical)) + require.Equal(t, uint64(objNumber), mm.getObjectCounter(logical)) + require.Equal(t, expectedSizes, mm.containerSizes()) + require.Equal(t, totalPayload, mm.payloadSize()) }) t.Run("inhume_GC", func(t *testing.T) { @@ -167,10 +190,10 @@ func TestCounters(t *testing.T) { require.NoError(t, err) } - require.Equal(t, uint64(objNumber), mm.objCounters[physical]) - require.Equal(t, uint64(objNumber-inhumedNumber), mm.objCounters[logical]) - require.Equal(t, expectedSizes, mm.cnrSize) - require.Equal(t, totalPayload, mm.pldSize) + require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical)) + require.Equal(t, uint64(objNumber-inhumedNumber), mm.getObjectCounter(logical)) + require.Equal(t, expectedSizes, mm.containerSizes()) + require.Equal(t, totalPayload, mm.payloadSize()) oo = oo[inhumedNumber:] }) @@ -179,8 +202,8 @@ func TestCounters(t *testing.T) { var prm shard.InhumePrm ts := objectcore.AddressOf(testutil.GenerateObject()) - phy := mm.objCounters[physical] - logic := mm.objCounters[logical] + phy := mm.getObjectCounter(physical) + logic := mm.getObjectCounter(logical) inhumedNumber := int(phy / 4) prm.SetTarget(ts, addrFromObjs(oo[:inhumedNumber])...) @@ -188,10 +211,10 @@ func TestCounters(t *testing.T) { _, err := sh.Inhume(context.Background(), prm) require.NoError(t, err) - require.Equal(t, phy, mm.objCounters[physical]) - require.Equal(t, logic-uint64(inhumedNumber), mm.objCounters[logical]) - require.Equal(t, expectedSizes, mm.cnrSize) - require.Equal(t, totalPayload, mm.pldSize) + require.Equal(t, phy, mm.getObjectCounter(physical)) + require.Equal(t, logic-uint64(inhumedNumber), mm.getObjectCounter(logical)) + require.Equal(t, expectedSizes, mm.containerSizes()) + require.Equal(t, totalPayload, mm.payloadSize()) oo = oo[inhumedNumber:] }) @@ -199,8 +222,8 @@ func TestCounters(t *testing.T) { t.Run("Delete", func(t *testing.T) { var prm shard.DeletePrm - phy := mm.objCounters[physical] - logic := mm.objCounters[logical] + phy := mm.getObjectCounter(physical) + logic := mm.getObjectCounter(logical) deletedNumber := int(phy / 4) prm.SetAddresses(addrFromObjs(oo[:deletedNumber])...) @@ -208,8 +231,8 @@ func TestCounters(t *testing.T) { _, err := sh.Delete(context.Background(), prm) require.NoError(t, err) - require.Equal(t, phy-uint64(deletedNumber), mm.objCounters[physical]) - require.Equal(t, logic-uint64(deletedNumber), mm.objCounters[logical]) + require.Equal(t, phy-uint64(deletedNumber), mm.getObjectCounter(physical)) + require.Equal(t, logic-uint64(deletedNumber), mm.getObjectCounter(logical)) var totalRemovedpayload uint64 for i := range oo[:deletedNumber] { removedPayload := oo[i].PayloadSize() @@ -218,8 +241,8 @@ func TestCounters(t *testing.T) { cnr, _ := oo[i].ContainerID() expectedSizes[cnr.EncodeToString()] -= int64(removedPayload) } - require.Equal(t, expectedSizes, mm.cnrSize) - require.Equal(t, totalPayload-int64(totalRemovedpayload), mm.pldSize) + require.Equal(t, expectedSizes, mm.containerSizes()) + require.Equal(t, totalPayload-int64(totalRemovedpayload), mm.payloadSize()) }) } From c672f59ab8a2b80f0fd84dd9c22897f0a7324075 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 30 Aug 2023 23:36:48 +0300 Subject: [PATCH 0566/1943] [#536] blobovnicza: Drop cache Each blobovnicza instance is opened while is in use. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + .../blobovnicza/blobovnicza_test.go | 5 +- pkg/local_object_storage/blobovnicza/put.go | 8 - pkg/local_object_storage/blobovnicza/sizes.go | 2 +- .../blobstor/blobovniczatree/active.go | 213 +++++++++++++++ .../blobstor/blobovniczatree/blobovnicza.go | 143 +---------- .../blobovniczatree/concurrency_test.go | 59 +++++ .../blobstor/blobovniczatree/control.go | 111 ++------ .../blobstor/blobovniczatree/delete.go | 67 +---- .../blobstor/blobovniczatree/errors.go | 2 + .../blobstor/blobovniczatree/exists.go | 14 +- .../blobstor/blobovniczatree/get.go | 67 +---- .../blobstor/blobovniczatree/get_range.go | 76 +----- .../blobstor/blobovniczatree/iterate.go | 4 +- .../blobstor/blobovniczatree/manager.go | 243 ++++++++++++++++++ .../blobstor/blobovniczatree/put.go | 45 +--- 16 files changed, 586 insertions(+), 474 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/active.go create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/manager.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 6ceee4f17..ed88e615b 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -509,4 +509,5 @@ const ( RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" FailedToCountWritecacheItems = "failed to count writecache items" + AttemtToCloseAlreadyClosedBlobovnicza = "attempt to close an already closed blobovnicza" ) diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index 48a189c9d..8d701ae5c 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -3,7 +3,6 @@ package blobovnicza import ( "context" "crypto/rand" - "errors" "os" "testing" @@ -98,9 +97,9 @@ func TestBlobovnicza(t *testing.T) { testPutGet(t, blz, oidtest.Address(), objSizeLim, nil, nil) } - // from now objects should not be saved + // blobovnizca accepts object event if full testPutGet(t, blz, oidtest.Address(), 1024, func(err error) bool { - return errors.Is(err, ErrFull) + return err == nil }, nil) require.NoError(t, blz.Close()) diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index e43b89ec6..787372211 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -23,10 +23,6 @@ type PutPrm struct { type PutRes struct { } -// ErrFull is returned when trying to save an -// object to a filled blobovnicza. -var ErrFull = logicerr.New("blobovnicza is full") - // SetAddress sets the address of the saving object. func (p *PutPrm) SetAddress(addr oid.Address) { p.addr = addr @@ -65,10 +61,6 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { key := addressKey(prm.addr) err := b.boltDB.Batch(func(tx *bbolt.Tx) error { - if b.full() { - return ErrFull - } - buck := tx.Bucket(bucketName) if buck == nil { // expected to happen: diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index 290df9c93..cd1f69725 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -55,6 +55,6 @@ func (b *Blobovnicza) itemDeleted(itemSize uint64) { b.metrics.SubOpenBlobovniczaItems(1) } -func (b *Blobovnicza) full() bool { +func (b *Blobovnicza) IsFull() bool { return b.dataSize.Load() >= b.fullSizeLimit } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/active.go b/pkg/local_object_storage/blobstor/blobovniczatree/active.go new file mode 100644 index 000000000..526699b45 --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/active.go @@ -0,0 +1,213 @@ +package blobovniczatree + +import ( + "path/filepath" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" +) + +type activeDB struct { + blz *blobovnicza.Blobovnicza + shDB *sharedDB +} + +func (db *activeDB) Blobovnicza() *blobovnicza.Blobovnicza { + return db.blz +} + +func (db *activeDB) Close() { + db.shDB.Close() +} + +func (db *activeDB) Path() string { + return db.shDB.Path() +} + +// activeDBManager manages active blobovnicza instances (that is, those that are being used for Put). +// +// Uses dbManager for opening/closing sharedDB instances. +// Stores a reference to an open active sharedDB, so dbManager does not close it. +// When changing the active sharedDB, releases the reference to the previous active sharedDB. +type activeDBManager struct { + levelToActiveDBGuard *sync.RWMutex + levelToActiveDB map[string]*sharedDB + levelLock *utilSync.KeyLocker[string] + closed bool + + dbManager *dbManager + leafWidth uint64 +} + +func newActiveDBManager(dbManager *dbManager, leafWidth uint64) *activeDBManager { + return &activeDBManager{ + levelToActiveDBGuard: &sync.RWMutex{}, + levelToActiveDB: make(map[string]*sharedDB), + levelLock: utilSync.NewKeyLocker[string](), + + dbManager: dbManager, + leafWidth: leafWidth, + } +} + +// GetOpenedActiveDBForLevel returns active DB for level. +// DB must be closed after use. +func (m *activeDBManager) GetOpenedActiveDBForLevel(lvlPath string) (*activeDB, error) { + activeDB, err := m.getCurrentActiveIfOk(lvlPath) + if err != nil { + return nil, err + } + if activeDB != nil { + return activeDB, nil + } + + return m.updateAndGetActive(lvlPath) +} + +func (m *activeDBManager) Open() { + m.levelToActiveDBGuard.Lock() + defer m.levelToActiveDBGuard.Unlock() + + m.closed = false +} + +func (m *activeDBManager) Close() { + m.levelToActiveDBGuard.Lock() + defer m.levelToActiveDBGuard.Unlock() + + for _, db := range m.levelToActiveDB { + db.Close() + } + m.levelToActiveDB = make(map[string]*sharedDB) + m.closed = true +} + +func (m *activeDBManager) getCurrentActiveIfOk(lvlPath string) (*activeDB, error) { + m.levelToActiveDBGuard.RLock() + defer m.levelToActiveDBGuard.RUnlock() + + if m.closed { + return nil, errClosed + } + + db, ok := m.levelToActiveDB[lvlPath] + if !ok { + return nil, nil + } + + blz, err := db.Open() //open db for usage, will be closed on activeDB.Close() + if err != nil { + return nil, err + } + + if blz.IsFull() { + db.Close() + return nil, nil + } + + return &activeDB{ + blz: blz, + shDB: db, + }, nil +} + +func (m *activeDBManager) updateAndGetActive(lvlPath string) (*activeDB, error) { + m.levelLock.Lock(lvlPath) + defer m.levelLock.Unlock(lvlPath) + + current, err := m.getCurrentActiveIfOk(lvlPath) + if err != nil { + return nil, err + } + if current != nil { + return current, nil + } + + nextShDB, err := m.getNextSharedDB(lvlPath) + if err != nil { + return nil, err + } + + if nextShDB == nil { + return nil, nil + } + + blz, err := nextShDB.Open() // open db for client, client must call Close() after usage + if err != nil { + return nil, err + } + return &activeDB{ + blz: blz, + shDB: nextShDB, + }, nil +} + +func (m *activeDBManager) getNextSharedDB(lvlPath string) (*sharedDB, error) { + var idx uint64 + var iterCount uint64 + hasActive, currentIdx := m.hasActiveDB(lvlPath) + if hasActive { + idx = (currentIdx + 1) % m.leafWidth + } + + var next *sharedDB + + for iterCount < m.leafWidth { + path := filepath.Join(lvlPath, u64ToHexString(idx)) + shDB := m.dbManager.GetByPath(path) + db, err := shDB.Open() //open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close() + if err != nil { + return nil, err + } + if db.IsFull() { + shDB.Close() + } else { + next = shDB + break + } + idx = (idx + 1) % m.leafWidth + iterCount++ + } + + previous, updated := m.replace(lvlPath, next) + if !updated && next != nil { + next.Close() // manager is closed, so don't hold active DB open + } + if updated && previous != nil { + previous.Close() + } + return next, nil +} + +func (m *activeDBManager) hasActiveDB(lvlPath string) (bool, uint64) { + m.levelToActiveDBGuard.RLock() + defer m.levelToActiveDBGuard.RUnlock() + + if m.closed { + return false, 0 + } + + db, ok := m.levelToActiveDB[lvlPath] + if !ok { + return false, 0 + } + return true, u64FromHexString(filepath.Base(db.Path())) +} + +func (m *activeDBManager) replace(lvlPath string, shDB *sharedDB) (*sharedDB, bool) { + m.levelToActiveDBGuard.Lock() + defer m.levelToActiveDBGuard.Unlock() + + if m.closed { + return nil, false + } + + previous := m.levelToActiveDB[lvlPath] + if shDB == nil { + delete(m.levelToActiveDB, lvlPath) + } else { + m.levelToActiveDB[lvlPath] = shDB + } + return previous, true +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 80d064c80..46a8df685 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -3,19 +3,12 @@ package blobovniczatree import ( "errors" "fmt" - "path/filepath" "strconv" - "sync" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" - "github.com/hashicorp/golang-lru/v2/simplelru" - "go.uber.org/zap" ) // Blobovniczas represents the storage of the "small" objects. @@ -61,28 +54,8 @@ import ( type Blobovniczas struct { cfg - // cache of opened filled Blobovniczas - opened *simplelru.LRU[string, *blobovnicza.Blobovnicza] - // lruMtx protects opened cache. - // It isn't RWMutex because `Get` calls must - // lock this mutex on write, as LRU info is updated. - // It must be taken after activeMtx in case when eviction is possible - // i.e. `Add`, `Purge` and `Remove` calls. - lruMtx sync.Mutex - - // mutex to exclude parallel bbolt.Open() calls - // bbolt.Open() deadlocks if it tries to open already opened file - openMtx sync.Mutex - - // list of active (opened, non-filled) Blobovniczas - activeMtx sync.RWMutex - active map[string]blobovniczaWithIndex -} - -type blobovniczaWithIndex struct { - ind uint64 - - blz *blobovnicza.Blobovnicza + commondbManager *dbManager + activeDBManager *activeDBManager } var _ common.Storage = (*Blobovniczas)(nil) @@ -102,120 +75,12 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz.blzLeafWidth = blz.blzShallowWidth } - cache, err := simplelru.NewLRU[string, *blobovnicza.Blobovnicza](blz.openedCacheSize, func(p string, value *blobovnicza.Blobovnicza) { - lvlPath := filepath.Dir(p) - if b, ok := blz.active[lvlPath]; ok && b.ind == u64FromHexString(filepath.Base(p)) { - // This branch is taken if we have recently updated active blobovnicza and remove - // it from opened cache. - return - } else if err := value.Close(); err != nil { - blz.log.Error(logs.BlobovniczatreeCouldNotCloseBlobovnicza, - zap.String("id", p), - zap.String("error", err.Error()), - ) - } else { - blz.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyClosedOnEvict, - zap.String("id", p), - ) - } - }) - if err != nil { - // occurs only if the size is not positive - panic(fmt.Errorf("could not create LRU cache of size %d: %w", blz.openedCacheSize, err)) - } - - activeMapCapacity := uint64(1) - for i := uint64(0); i < blz.blzShallowDepth; i++ { - if i+1 == blz.blzShallowDepth { - activeMapCapacity *= blz.blzLeafWidth - } else { - activeMapCapacity *= blz.blzShallowWidth - } - } - - blz.opened = cache - blz.active = make(map[string]blobovniczaWithIndex, activeMapCapacity) + blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.blzLeafWidth, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) + blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) return blz } -// activates and returns activated blobovnicza of p-level (dir). -// -// returns error if blobvnicza could not be activated. -func (b *Blobovniczas) getActivated(lvlPath string) (blobovniczaWithIndex, error) { - return b.updateAndGet(lvlPath, nil) -} - -// updates active blobovnicza of p-level (dir). -// -// if current active blobovnicza's index is not old, it remains unchanged. -func (b *Blobovniczas) updateActive(lvlPath string, old *uint64) error { - b.log.Debug(logs.BlobovniczatreeUpdatingActiveBlobovnicza, zap.String("path", lvlPath)) - - _, err := b.updateAndGet(lvlPath, old) - - b.log.Debug(logs.BlobovniczatreeActiveBlobovniczaSuccessfullyUpdated, zap.String("path", lvlPath)) - - return err -} - -// updates and returns active blobovnicza of p-level (dir). -// -// if current active blobovnicza's index is not old, it is returned unchanged. -func (b *Blobovniczas) updateAndGet(lvlPath string, old *uint64) (blobovniczaWithIndex, error) { - b.activeMtx.RLock() - active, ok := b.active[lvlPath] - b.activeMtx.RUnlock() - - if ok { - if old != nil { - if active.ind == b.blzLeafWidth-1 { - return active, logicerr.New("no more Blobovniczas") - } else if active.ind != *old { - // sort of CAS in order to control concurrent - // updateActive calls - return active, nil - } - } else { - return active, nil - } - - active.ind++ - } - - var err error - if active.blz, err = b.openBlobovnicza(filepath.Join(lvlPath, u64ToHexString(active.ind))); err != nil { - return active, err - } - - b.activeMtx.Lock() - defer b.activeMtx.Unlock() - - // check 2nd time to find out if it blobovnicza was activated while thread was locked - tryActive, ok := b.active[lvlPath] - if ok && tryActive.blz == active.blz { - return tryActive, nil - } - - // Remove from opened cache (active blobovnicza should always be opened). - // Because `onEvict` callback is called in `Remove`, we need to update - // active map beforehand. - b.active[lvlPath] = active - - activePath := filepath.Join(lvlPath, u64ToHexString(active.ind)) - b.lruMtx.Lock() - b.opened.Remove(activePath) - if ok { - b.opened.Add(filepath.Join(lvlPath, u64ToHexString(tryActive.ind)), tryActive.blz) - } - b.lruMtx.Unlock() - - b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyActivated, - zap.String("path", activePath)) - - return active, nil -} - // returns hash of the object address. func addressHash(addr *oid.Address, path string) uint64 { var a string diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go new file mode 100644 index 000000000..018ecce3c --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go @@ -0,0 +1,59 @@ +package blobovniczatree + +import ( + "context" + "sync" + "sync/atomic" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "github.com/stretchr/testify/require" +) + +func TestBlobovniczaTree_Concurrency(t *testing.T) { + t.Parallel() + const n = 1000 + + st := NewBlobovniczaTree( + WithLogger(test.NewLogger(t, true)), + WithObjectSizeLimit(1024), + WithBlobovniczaShallowWidth(10), + WithBlobovniczaShallowDepth(1), + WithRootPath(t.TempDir())) + require.NoError(t, st.Open(false)) + require.NoError(t, st.Init()) + t.Cleanup(func() { + require.NoError(t, st.Close()) + }) + + objGen := &testutil.SeqObjGenerator{ObjSize: 1} + + var cnt atomic.Int64 + var wg sync.WaitGroup + for i := 0; i < 1000; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for cnt.Add(1) <= n { + obj := objGen.Next() + addr := testutil.AddressFromObject(t, obj) + + raw, err := obj.Marshal() + require.NoError(t, err) + + _, err = st.Put(context.Background(), common.PutPrm{ + Address: addr, + RawData: raw, + }) + require.NoError(t, err) + + _, err = st.Get(context.Background(), common.GetPrm{Address: addr}) + require.NoError(t, err) // fails very often, correlated to how many goroutines are started + } + }() + } + + wg.Wait() +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index d7b0799c1..c3d12088a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -2,11 +2,8 @@ package blobovniczatree import ( "context" - "fmt" - "path/filepath" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "go.uber.org/zap" ) @@ -14,6 +11,7 @@ import ( func (b *Blobovniczas) Open(readOnly bool) error { b.readOnly = readOnly b.metrics.SetMode(readOnly) + b.openManagers() return nil } @@ -23,59 +21,35 @@ func (b *Blobovniczas) Open(readOnly bool) error { func (b *Blobovniczas) Init() error { b.log.Debug(logs.BlobovniczatreeInitializingBlobovniczas) + b.openManagers() + if b.readOnly { b.log.Debug(logs.BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization) return nil } return b.iterateLeaves(context.TODO(), func(p string) (bool, error) { - blz, err := b.openBlobovniczaNoCache(p) + shBlz := b.openBlobovniczaNoCache(p) + _, err := shBlz.Open() if err != nil { return true, err } - defer blz.Close() - - if err := blz.Init(); err != nil { - return true, fmt.Errorf("could not initialize blobovnicza structure %s: %w", p, err) - } + defer shBlz.Close() b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) return false, nil }) } +func (b *Blobovniczas) openManagers() { + b.commondbManager.Open() //order important + b.activeDBManager.Open() +} + // Close implements common.Storage. func (b *Blobovniczas) Close() error { - b.activeMtx.Lock() - - b.lruMtx.Lock() - - for p, v := range b.active { - if err := v.blz.Close(); err != nil { - b.log.Debug(logs.BlobovniczatreeCouldNotCloseActiveBlobovnicza, - zap.String("path", p), - zap.String("error", err.Error()), - ) - } - b.opened.Remove(p) - } - for _, k := range b.opened.Keys() { - blz, _ := b.opened.Get(k) - if err := blz.Close(); err != nil { - b.log.Debug(logs.BlobovniczatreeCouldNotCloseActiveBlobovnicza, - zap.String("path", k), - zap.String("error", err.Error()), - ) - } - b.opened.Remove(k) - } - - b.active = make(map[string]blobovniczaWithIndex) - b.metrics.Close() - - b.lruMtx.Unlock() - - b.activeMtx.Unlock() + b.activeDBManager.Close() //order important + b.commondbManager.Close() return nil } @@ -83,61 +57,10 @@ func (b *Blobovniczas) Close() error { // opens and returns blobovnicza with path p. // // If blobovnicza is already opened and cached, instance from cache is returned w/o changes. -func (b *Blobovniczas) openBlobovnicza(p string) (*blobovnicza.Blobovnicza, error) { - b.lruMtx.Lock() - v, ok := b.opened.Get(p) - b.lruMtx.Unlock() - if ok { - // blobovnicza should be opened in cache - return v, nil - } - - lvlPath := filepath.Dir(p) - curIndex := u64FromHexString(filepath.Base(p)) - - b.activeMtx.RLock() - defer b.activeMtx.RUnlock() - - active, ok := b.active[lvlPath] - if ok && active.ind == curIndex { - return active.blz, nil - } - - b.lruMtx.Lock() - defer b.lruMtx.Unlock() - - v, ok = b.opened.Get(p) - if ok { - return v, nil - } - - blz, err := b.openBlobovniczaNoCache(p) - if err != nil { - return nil, err - } - - b.opened.Add(p, blz) - - return blz, nil +func (b *Blobovniczas) openBlobovnicza(p string) *sharedDB { + return b.openBlobovniczaNoCache(p) } -func (b *Blobovniczas) openBlobovniczaNoCache(p string) (*blobovnicza.Blobovnicza, error) { - b.openMtx.Lock() - defer b.openMtx.Unlock() - - path := filepath.Join(b.rootPath, p) - - blz := blobovnicza.New(append(b.blzOpts, - blobovnicza.WithReadOnly(b.readOnly), - blobovnicza.WithPath(path), - blobovnicza.WithMetrics(b.metrics.Blobovnicza()), - )...) - - if err := blz.Open(); err != nil { - return nil, fmt.Errorf("could not open blobovnicza %s: %w", p, err) - } - if err := blz.Init(); err != nil { - return nil, fmt.Errorf("could not init blobovnicza %s: %w", p, err) - } - return blz, nil +func (b *Blobovniczas) openBlobovniczaNoCache(p string) *sharedDB { + return b.commondbManager.GetByPath(p) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 0698c232d..bc7735e74 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -3,7 +3,6 @@ package blobovniczatree import ( "context" "encoding/hex" - "path/filepath" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -48,10 +47,12 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co if prm.StorageID != nil { id := blobovnicza.NewIDFromBytes(prm.StorageID) - blz, err := b.openBlobovnicza(id.String()) + shBlz := b.openBlobovnicza(id.String()) + blz, err := shBlz.Open() if err != nil { return res, err } + defer shBlz.Close() if res, err = b.deleteObject(ctx, blz, bPrm); err == nil { success = true @@ -59,16 +60,10 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co return res, err } - activeCache := make(map[string]struct{}) objectFound := false err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { - dirPath := filepath.Dir(p) - - // don't process active blobovnicza of the level twice - _, ok := activeCache[dirPath] - - res, err = b.deleteObjectFromLevel(ctx, bPrm, p, !ok) + res, err = b.deleteObjectFromLevel(ctx, bPrm, p) if err != nil { if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, @@ -78,8 +73,6 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co } } - activeCache[dirPath] = struct{}{} - if err == nil { objectFound = true } @@ -100,57 +93,13 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co // tries to delete object from particular blobovnicza. // // returns no error if object was removed from some blobovnicza of the same level. -func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicza.DeletePrm, blzPath string, tryActive bool) (common.DeleteRes, error) { - lvlPath := filepath.Dir(blzPath) - - // try to remove from blobovnicza if it is opened - b.lruMtx.Lock() - v, ok := b.opened.Get(blzPath) - b.lruMtx.Unlock() - if ok { - if res, err := b.deleteObject(ctx, v, prm); err == nil { - return res, err - } else if !client.IsErrObjectNotFound(err) { - b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza, - zap.String("path", blzPath), - zap.String("error", err.Error()), - ) - } - } - - // therefore the object is possibly placed in a lighter blobovnicza - - // next we check in the active level blobobnicza: - // * the active blobovnicza is always opened. - b.activeMtx.RLock() - active, ok := b.active[lvlPath] - b.activeMtx.RUnlock() - - if ok && tryActive { - if res, err := b.deleteObject(ctx, active.blz, prm); err == nil { - return res, err - } else if !client.IsErrObjectNotFound(err) { - b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza, - zap.String("path", blzPath), - zap.String("error", err.Error()), - ) - } - } - - // then object is possibly placed in closed blobovnicza - - // check if it makes sense to try to open the blob - // (Blobovniczas "after" the active one are empty anyway, - // and it's pointless to open them). - if u64FromHexString(filepath.Base(blzPath)) > active.ind { - return common.DeleteRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) - } - - // open blobovnicza (cached inside) - blz, err := b.openBlobovnicza(blzPath) +func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicza.DeletePrm, blzPath string) (common.DeleteRes, error) { + shBlz := b.openBlobovnicza(blzPath) + blz, err := shBlz.Open() if err != nil { return common.DeleteRes{}, err } + defer shBlz.Close() return b.deleteObject(ctx, blz, prm) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/errors.go b/pkg/local_object_storage/blobstor/blobovniczatree/errors.go index 04247280a..be0fd81c3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/errors.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/errors.go @@ -7,6 +7,8 @@ import ( apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) +var errClosed = logicerr.New("blobvnicza is closed") + func isErrOutOfRange(err error) bool { var target *apistatus.ObjectOutOfRange return errors.As(err, &target) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index c1964c227..f7a192526 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -3,7 +3,6 @@ package blobovniczatree import ( "context" "encoding/hex" - "path/filepath" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -37,26 +36,22 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common if prm.StorageID != nil { id := blobovnicza.NewIDFromBytes(prm.StorageID) - blz, err := b.openBlobovnicza(id.String()) + shBlz := b.openBlobovnicza(id.String()) + blz, err := shBlz.Open() if err != nil { return common.ExistsRes{}, err } + defer shBlz.Close() exists, err := blz.Exists(ctx, prm.Address) return common.ExistsRes{Exists: exists}, err } - activeCache := make(map[string]struct{}) - var gPrm blobovnicza.GetPrm gPrm.SetAddress(prm.Address) err := b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { - dirPath := filepath.Dir(p) - - _, ok := activeCache[dirPath] - - _, err := b.getObjectFromLevel(ctx, gPrm, p, !ok) + _, err := b.getObjectFromLevel(ctx, gPrm, p) if err != nil { if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, @@ -65,7 +60,6 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common } } - activeCache[dirPath] = struct{}{} found = err == nil return found, nil }) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 671d045b9..aca39bf3b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "fmt" - "path/filepath" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -48,10 +47,12 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G if prm.StorageID != nil { id := blobovnicza.NewIDFromBytes(prm.StorageID) - blz, err := b.openBlobovnicza(id.String()) + shBlz := b.openBlobovnicza(id.String()) + blz, err := shBlz.Open() if err != nil { return res, err } + defer shBlz.Close() res, err = b.getObject(ctx, blz, bPrm) if err == nil { @@ -61,14 +62,8 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G return res, err } - activeCache := make(map[string]struct{}) - err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { - dirPath := filepath.Dir(p) - - _, ok := activeCache[dirPath] - - res, err = b.getObjectFromLevel(ctx, bPrm, p, !ok) + res, err = b.getObjectFromLevel(ctx, bPrm, p) if err != nil { if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, @@ -78,8 +73,6 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G } } - activeCache[dirPath] = struct{}{} - // abort iterator if found, otherwise process all Blobovniczas return err == nil, nil }) @@ -98,58 +91,14 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G // tries to read object from particular blobovnicza. // // returns error if object could not be read from any blobovnicza of the same level. -func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.GetPrm, blzPath string, tryActive bool) (common.GetRes, error) { - lvlPath := filepath.Dir(blzPath) - - // try to read from blobovnicza if it is opened - b.lruMtx.Lock() - v, ok := b.opened.Get(blzPath) - b.lruMtx.Unlock() - if ok { - if res, err := b.getObject(ctx, v, prm); err == nil { - return res, err - } else if !client.IsErrObjectNotFound(err) { - b.log.Debug(logs.BlobovniczatreeCouldNotReadObjectFromOpenedBlobovnicza, - zap.String("path", blzPath), - zap.String("error", err.Error()), - ) - } - } - - // therefore the object is possibly placed in a lighter blobovnicza - - // next we check in the active level blobobnicza: - // * the freshest objects are probably the most demanded; - // * the active blobovnicza is always opened. - b.activeMtx.RLock() - active, ok := b.active[lvlPath] - b.activeMtx.RUnlock() - - if ok && tryActive { - if res, err := b.getObject(ctx, active.blz, prm); err == nil { - return res, err - } else if !client.IsErrObjectNotFound(err) { - b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromActiveBlobovnicza, - zap.String("path", blzPath), - zap.String("error", err.Error()), - ) - } - } - - // then object is possibly placed in closed blobovnicza - - // check if it makes sense to try to open the blob - // (Blobovniczas "after" the active one are empty anyway, - // and it's pointless to open them). - if u64FromHexString(filepath.Base(blzPath)) > active.ind { - return common.GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) - } - +func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.GetPrm, blzPath string) (common.GetRes, error) { // open blobovnicza (cached inside) - blz, err := b.openBlobovnicza(blzPath) + shBlz := b.openBlobovnicza(blzPath) + blz, err := shBlz.Open() if err != nil { return common.GetRes{}, err } + defer shBlz.Close() return b.getObject(ctx, blz, prm) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 088ebe249..3d1a6b988 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "fmt" - "path/filepath" "strconv" "time" @@ -47,10 +46,12 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re if prm.StorageID != nil { id := blobovnicza.NewIDFromBytes(prm.StorageID) - blz, err := b.openBlobovnicza(id.String()) + shBlz := b.openBlobovnicza(id.String()) + blz, err := shBlz.Open() if err != nil { return common.GetRangeRes{}, err } + defer shBlz.Close() res, err := b.getObjectRange(ctx, blz, prm) if err == nil { @@ -60,15 +61,10 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re return res, err } - activeCache := make(map[string]struct{}) objectFound := false err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { - dirPath := filepath.Dir(p) - - _, ok := activeCache[dirPath] - - res, err = b.getRangeFromLevel(ctx, prm, p, !ok) + res, err = b.getRangeFromLevel(ctx, prm, p) if err != nil { outOfBounds := isErrOutOfRange(err) if !outOfBounds && !client.IsErrObjectNotFound(err) { @@ -82,8 +78,6 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re } } - activeCache[dirPath] = struct{}{} - objectFound = err == nil // abort iterator if found, otherwise process all Blobovniczas @@ -106,68 +100,14 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re // tries to read range of object payload data from particular blobovnicza. // // returns error if object could not be read from any blobovnicza of the same level. -func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRangePrm, blzPath string, tryActive bool) (common.GetRangeRes, error) { - lvlPath := filepath.Dir(blzPath) - - // try to read from blobovnicza if it is opened - b.lruMtx.Lock() - v, ok := b.opened.Get(blzPath) - b.lruMtx.Unlock() - if ok { - res, err := b.getObjectRange(ctx, v, prm) - switch { - case err == nil, - isErrOutOfRange(err): - return res, err - default: - if !client.IsErrObjectNotFound(err) { - b.log.Debug(logs.BlobovniczatreeCouldNotReadPayloadRangeFromOpenedBlobovnicza, - zap.String("path", blzPath), - zap.String("error", err.Error()), - ) - } - } - } - - // therefore the object is possibly placed in a lighter blobovnicza - - // next we check in the active level blobobnicza: - // * the freshest objects are probably the most demanded; - // * the active blobovnicza is always opened. - b.activeMtx.RLock() - active, ok := b.active[lvlPath] - b.activeMtx.RUnlock() - - if ok && tryActive { - res, err := b.getObjectRange(ctx, active.blz, prm) - switch { - case err == nil, - isErrOutOfRange(err): - return res, err - default: - if !client.IsErrObjectNotFound(err) { - b.log.Debug(logs.BlobovniczatreeCouldNotReadPayloadRangeFromActiveBlobovnicza, - zap.String("path", blzPath), - zap.String("error", err.Error()), - ) - } - } - } - - // then object is possibly placed in closed blobovnicza - - // check if it makes sense to try to open the blob - // (Blobovniczas "after" the active one are empty anyway, - // and it's pointless to open them). - if u64FromHexString(filepath.Base(blzPath)) > active.ind { - return common.GetRangeRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) - } - +func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRangePrm, blzPath string) (common.GetRangeRes, error) { // open blobovnicza (cached inside) - blz, err := b.openBlobovnicza(blzPath) + shBlz := b.openBlobovnicza(blzPath) + blz, err := shBlz.Open() if err != nil { return common.GetRangeRes{}, err } + defer shBlz.Close() return b.getObjectRange(ctx, blz, prm) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index aeac854f7..cc273839a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -68,13 +68,15 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm // iterator over all Blobovniczas in unsorted order. Break on f's error return. func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error { return b.iterateLeaves(ctx, func(p string) (bool, error) { - blz, err := b.openBlobovnicza(p) + shBlz := b.openBlobovnicza(p) + blz, err := shBlz.Open() if err != nil { if ignoreErrors { return false, nil } return false, fmt.Errorf("could not open blobovnicza %s: %w", p, err) } + defer shBlz.Close() err = f(p, blz) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go new file mode 100644 index 000000000..941d9961d --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -0,0 +1,243 @@ +package blobovniczatree + +import ( + "fmt" + "path/filepath" + "sync" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "go.uber.org/zap" +) + +// sharedDB is responsible for opening and closing a file of single blobovnicza. +type sharedDB struct { + guard *sync.RWMutex + blcza *blobovnicza.Blobovnicza + refCount uint32 + + openDBCounter *openDBCounter + closedFlag *atomic.Bool + options []blobovnicza.Option + path string + readOnly bool + metrics blobovnicza.Metrics + log *logger.Logger +} + +func newSharedDB(options []blobovnicza.Option, path string, readOnly bool, + metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlag *atomic.Bool, log *logger.Logger) *sharedDB { + return &sharedDB{ + guard: &sync.RWMutex{}, + + options: options, + path: path, + readOnly: readOnly, + metrics: metrics, + closedFlag: closedFlag, + log: log, + openDBCounter: openDBCounter, + } +} + +func (b *sharedDB) Open() (*blobovnicza.Blobovnicza, error) { + if b.closedFlag.Load() { + return nil, errClosed + } + + b.guard.Lock() + defer b.guard.Unlock() + + if b.refCount > 0 { + b.refCount++ + return b.blcza, nil + } + + blz := blobovnicza.New(append(b.options, + blobovnicza.WithReadOnly(b.readOnly), + blobovnicza.WithPath(b.path), + blobovnicza.WithMetrics(b.metrics), + )...) + + if err := blz.Open(); err != nil { + return nil, fmt.Errorf("could not open blobovnicza %s: %w", b.path, err) + } + if err := blz.Init(); err != nil { + return nil, fmt.Errorf("could not init blobovnicza %s: %w", b.path, err) + } + + b.refCount++ + b.blcza = blz + b.openDBCounter.Inc() + + return blz, nil +} + +func (b *sharedDB) Close() { + b.guard.Lock() + defer b.guard.Unlock() + + if b.refCount == 0 { + b.log.Error(logs.AttemtToCloseAlreadyClosedBlobovnicza, zap.String("id", b.path)) + return + } + + if b.refCount == 1 { + b.refCount = 0 + if err := b.blcza.Close(); err != nil { + b.log.Error(logs.BlobovniczatreeCouldNotCloseBlobovnicza, + zap.String("id", b.path), + zap.String("error", err.Error()), + ) + } + b.blcza = nil + b.openDBCounter.Dec() + return + } + + b.refCount-- +} + +func (b *sharedDB) Path() string { + return b.path +} + +// levelDbManager stores pointers of the sharedDB's for the leaf directory of the blobovnicza tree. +type levelDbManager struct { + databases []*sharedDB +} + +func newLevelDBManager(width uint64, options []blobovnicza.Option, rootPath string, lvlPath string, + readOnly bool, metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlog *atomic.Bool, log *logger.Logger) *levelDbManager { + result := &levelDbManager{ + databases: make([]*sharedDB, width), + } + var idx uint64 + for idx = 0; idx < width; idx++ { + result.databases[idx] = newSharedDB(options, filepath.Join(rootPath, lvlPath, u64ToHexString(idx)), readOnly, metrics, openDBCounter, closedFlog, log) + } + return result +} + +func (m *levelDbManager) GetByIndex(idx uint64) *sharedDB { + return m.databases[idx] +} + +// dbManager manages the opening and closing of blobovnicza instances. +// +// The blobovnicza opens at the first request, closes after the last request. +type dbManager struct { + levelToManager map[string]*levelDbManager + levelToManagerGuard *sync.RWMutex + closedFlag *atomic.Bool + dbCounter *openDBCounter + + rootPath string + options []blobovnicza.Option + readOnly bool + metrics blobovnicza.Metrics + leafWidth uint64 + log *logger.Logger +} + +func newDBManager(rootPath string, options []blobovnicza.Option, leafWidth uint64, readOnly bool, metrics blobovnicza.Metrics, log *logger.Logger) *dbManager { + return &dbManager{ + rootPath: rootPath, + options: options, + readOnly: readOnly, + metrics: metrics, + leafWidth: leafWidth, + levelToManager: make(map[string]*levelDbManager), + levelToManagerGuard: &sync.RWMutex{}, + log: log, + closedFlag: &atomic.Bool{}, + dbCounter: newOpenDBCounter(), + } +} + +func (m *dbManager) GetByPath(path string) *sharedDB { + lvlPath := filepath.Dir(path) + curIndex := u64FromHexString(filepath.Base(path)) + levelManager := m.getLevelManager(lvlPath) + return levelManager.GetByIndex(curIndex) +} + +func (m *dbManager) Open() { + m.closedFlag.Store(false) +} + +func (m *dbManager) Close() { + m.closedFlag.Store(true) + m.dbCounter.WaitUntilAllClosed() +} + +func (m *dbManager) getLevelManager(lvlPath string) *levelDbManager { + result := m.getLevelManagerIfExists(lvlPath) + if result != nil { + return result + } + return m.getOrCreateLevelManager(lvlPath) +} + +func (m *dbManager) getLevelManagerIfExists(lvlPath string) *levelDbManager { + m.levelToManagerGuard.RLock() + defer m.levelToManagerGuard.RUnlock() + + return m.levelToManager[lvlPath] +} + +func (m *dbManager) getOrCreateLevelManager(lvlPath string) *levelDbManager { + m.levelToManagerGuard.Lock() + defer m.levelToManagerGuard.Unlock() + + if result, ok := m.levelToManager[lvlPath]; ok { + return result + } + + result := newLevelDBManager(m.leafWidth, m.options, m.rootPath, lvlPath, m.readOnly, m.metrics, m.dbCounter, m.closedFlag, m.log) + m.levelToManager[lvlPath] = result + return result +} + +type openDBCounter struct { + cond *sync.Cond + count uint64 +} + +func newOpenDBCounter() *openDBCounter { + return &openDBCounter{ + cond: &sync.Cond{ + L: &sync.Mutex{}, + }, + } +} + +func (c *openDBCounter) Inc() { + c.cond.L.Lock() + defer c.cond.L.Unlock() + + c.count++ +} + +func (c *openDBCounter) Dec() { + c.cond.L.Lock() + defer c.cond.L.Unlock() + + if c.count > 0 { + c.count-- + } + + if c.count == 0 { + c.cond.Broadcast() + } +} + +func (c *openDBCounter) WaitUntilAllClosed() { + c.cond.L.Lock() + for c.count > 0 { + c.cond.Wait() + } + c.cond.L.Unlock() +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 038d5244d..aae280e0b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -2,7 +2,6 @@ package blobovniczatree import ( "context" - "errors" "path/filepath" "time" @@ -10,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -76,8 +74,8 @@ type putIterator struct { PutPrm blobovnicza.PutPrm } -func (i *putIterator) iterate(ctx context.Context, path string) (bool, error) { - active, err := i.B.getActivated(path) +func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) { + active, err := i.B.activeDBManager.GetOpenedActiveDBForLevel(lvlPath) if err != nil { if !isLogical(err) { i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) @@ -89,46 +87,29 @@ func (i *putIterator) iterate(ctx context.Context, path string) (bool, error) { return false, nil } - if _, err := active.blz.Put(ctx, i.PutPrm); err != nil { - // Check if blobovnicza is full. We could either receive `blobovnicza.ErrFull` error - // or update active blobovnicza in other thread. In the latter case the database will be closed - // and `updateActive` takes care of not updating the active blobovnicza twice. - if isFull := errors.Is(err, blobovnicza.ErrFull); isFull || errors.Is(err, bbolt.ErrDatabaseNotOpen) { - if isFull { - i.B.log.Debug(logs.BlobovniczatreeBlobovniczaOverflowed, - zap.String("path", filepath.Join(path, u64ToHexString(active.ind)))) - } + if active == nil { + i.B.log.Debug(logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath)) + return false, nil + } + defer active.Close() - if err := i.B.updateActive(path, &active.ind); err != nil { - if !isLogical(err) { - i.B.reportError(logs.BlobovniczatreeCouldNotUpdateActiveBlobovnicza, err) - } else { - i.B.log.Debug(logs.BlobovniczatreeCouldNotUpdateActiveBlobovnicza, - zap.String("level", path), - zap.String("error", err.Error())) - } + i.AllFull = false - return false, nil - } - - return i.iterate(ctx, path) - } - - i.AllFull = false + _, err = active.Blobovnicza().Put(ctx, i.PutPrm) + if err != nil { if !isLogical(err) { i.B.reportError(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err) } else { i.B.log.Debug(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, - zap.String("path", filepath.Join(path, u64ToHexString(active.ind))), + zap.String("path", active.Path()), zap.String("error", err.Error())) } return false, nil } - path = filepath.Join(path, u64ToHexString(active.ind)) - - i.ID = blobovnicza.NewIDFromBytes([]byte(path)) + idx := u64FromHexString(filepath.Base(active.Path())) + i.ID = blobovnicza.NewIDFromBytes([]byte(filepath.Join(lvlPath, u64ToHexString(idx)))) return true, nil } From 7456c8556a8f924f065c7e18f1b80b1624847dd0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 31 Aug 2023 11:32:09 +0300 Subject: [PATCH 0567/1943] [#536] blobovnicza: Add blobovniczatree DB cache Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/blobovnicza.go | 2 + .../blobstor/blobovniczatree/cache.go | 103 ++++++++++++++++++ .../blobovniczatree/concurrency_test.go | 2 +- .../blobstor/blobovniczatree/control.go | 18 +-- .../blobstor/blobovniczatree/delete.go | 4 +- .../blobstor/blobovniczatree/exists.go | 2 +- .../blobstor/blobovniczatree/get.go | 4 +- .../blobstor/blobovniczatree/get_range.go | 4 +- .../blobstor/blobovniczatree/iterate.go | 2 +- .../blobstor/blobovniczatree/manager.go | 3 +- 10 files changed, 124 insertions(+), 20 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/cache.go diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 46a8df685..fd5155ee3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -56,6 +56,7 @@ type Blobovniczas struct { commondbManager *dbManager activeDBManager *activeDBManager + dbCache *dbCache } var _ common.Storage = (*Blobovniczas)(nil) @@ -77,6 +78,7 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.blzLeafWidth, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) + blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager) return blz } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go new file mode 100644 index 000000000..9bdee6df3 --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -0,0 +1,103 @@ +package blobovniczatree + +import ( + "fmt" + "sync" + + utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" + "github.com/hashicorp/golang-lru/v2/simplelru" +) + +// dbCache caches sharedDB instances that are NOT open for Put. +// +// Uses dbManager for opening/closing sharedDB instances. +// Stores a reference to an cached sharedDB, so dbManager does not close it. +type dbCache struct { + cacheGuard *sync.RWMutex + cache simplelru.LRUCache[string, *sharedDB] + pathLock *utilSync.KeyLocker[string] + closed bool + + dbManager *dbManager +} + +func newDBCache(size int, dbManager *dbManager) *dbCache { + cache, err := simplelru.NewLRU[string, *sharedDB](size, func(_ string, evictedDB *sharedDB) { + evictedDB.Close() + }) + if err != nil { + // occurs only if the size is not positive + panic(fmt.Errorf("could not create LRU cache of size %d: %w", size, err)) + } + return &dbCache{ + cacheGuard: &sync.RWMutex{}, + cache: cache, + dbManager: dbManager, + pathLock: utilSync.NewKeyLocker[string](), + } +} + +func (c *dbCache) Open() { + c.cacheGuard.Lock() + defer c.cacheGuard.Unlock() + + c.closed = false +} + +func (c *dbCache) Close() { + c.cacheGuard.Lock() + defer c.cacheGuard.Unlock() + c.cache.Purge() + c.closed = true +} + +func (c *dbCache) GetOrCreate(path string) *sharedDB { + value := c.getExisted(path) + if value != nil { + return value + } + return c.create(path) +} + +func (c *dbCache) getExisted(path string) *sharedDB { + c.cacheGuard.Lock() + defer c.cacheGuard.Unlock() + + if value, ok := c.cache.Get(path); ok { + return value + } + return nil +} + +func (c *dbCache) create(path string) *sharedDB { + c.pathLock.Lock(path) + defer c.pathLock.Unlock(path) + + value := c.getExisted(path) + if value != nil { + return value + } + + value = c.dbManager.GetByPath(path) + + _, err := value.Open() //open db to hold reference, closed by evictedDB.Close() or if cache closed + if err != nil { + return value + } + if added := c.put(path, value); !added { + value.Close() + } + return value +} + +func (c *dbCache) put(path string, db *sharedDB) bool { + c.cacheGuard.Lock() + defer c.cacheGuard.Unlock() + + if !c.closed { + c.cache.Add(path, db) + return true + } + + return false +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go index 018ecce3c..24cb93865 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go @@ -50,7 +50,7 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) { require.NoError(t, err) _, err = st.Get(context.Background(), common.GetPrm{Address: addr}) - require.NoError(t, err) // fails very often, correlated to how many goroutines are started + require.NoError(t, err) } }() } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index c3d12088a..5ea426e1c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -21,15 +21,13 @@ func (b *Blobovniczas) Open(readOnly bool) error { func (b *Blobovniczas) Init() error { b.log.Debug(logs.BlobovniczatreeInitializingBlobovniczas) - b.openManagers() - if b.readOnly { b.log.Debug(logs.BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization) return nil } return b.iterateLeaves(context.TODO(), func(p string) (bool, error) { - shBlz := b.openBlobovniczaNoCache(p) + shBlz := b.getBlobovniczaWithoutCaching(p) _, err := shBlz.Open() if err != nil { return true, err @@ -44,23 +42,25 @@ func (b *Blobovniczas) Init() error { func (b *Blobovniczas) openManagers() { b.commondbManager.Open() //order important b.activeDBManager.Open() + b.dbCache.Open() } // Close implements common.Storage. func (b *Blobovniczas) Close() error { - b.activeDBManager.Close() //order important + b.dbCache.Close() //order important + b.activeDBManager.Close() b.commondbManager.Close() return nil } -// opens and returns blobovnicza with path p. +// returns blobovnicza with path p // -// If blobovnicza is already opened and cached, instance from cache is returned w/o changes. -func (b *Blobovniczas) openBlobovnicza(p string) *sharedDB { - return b.openBlobovniczaNoCache(p) +// If blobovnicza is already cached, instance from cache is returned w/o changes. +func (b *Blobovniczas) getBlobovnicza(p string) *sharedDB { + return b.dbCache.GetOrCreate(p) } -func (b *Blobovniczas) openBlobovniczaNoCache(p string) *sharedDB { +func (b *Blobovniczas) getBlobovniczaWithoutCaching(p string) *sharedDB { return b.commondbManager.GetByPath(p) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index bc7735e74..ca1e46a15 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -47,7 +47,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co if prm.StorageID != nil { id := blobovnicza.NewIDFromBytes(prm.StorageID) - shBlz := b.openBlobovnicza(id.String()) + shBlz := b.getBlobovnicza(id.String()) blz, err := shBlz.Open() if err != nil { return res, err @@ -94,7 +94,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co // // returns no error if object was removed from some blobovnicza of the same level. func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicza.DeletePrm, blzPath string) (common.DeleteRes, error) { - shBlz := b.openBlobovnicza(blzPath) + shBlz := b.getBlobovnicza(blzPath) blz, err := shBlz.Open() if err != nil { return common.DeleteRes{}, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index f7a192526..c94881168 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -36,7 +36,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common if prm.StorageID != nil { id := blobovnicza.NewIDFromBytes(prm.StorageID) - shBlz := b.openBlobovnicza(id.String()) + shBlz := b.getBlobovnicza(id.String()) blz, err := shBlz.Open() if err != nil { return common.ExistsRes{}, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index aca39bf3b..f2399e414 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -47,7 +47,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G if prm.StorageID != nil { id := blobovnicza.NewIDFromBytes(prm.StorageID) - shBlz := b.openBlobovnicza(id.String()) + shBlz := b.getBlobovnicza(id.String()) blz, err := shBlz.Open() if err != nil { return res, err @@ -93,7 +93,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G // returns error if object could not be read from any blobovnicza of the same level. func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.GetPrm, blzPath string) (common.GetRes, error) { // open blobovnicza (cached inside) - shBlz := b.openBlobovnicza(blzPath) + shBlz := b.getBlobovnicza(blzPath) blz, err := shBlz.Open() if err != nil { return common.GetRes{}, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 3d1a6b988..56b41351b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -46,7 +46,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re if prm.StorageID != nil { id := blobovnicza.NewIDFromBytes(prm.StorageID) - shBlz := b.openBlobovnicza(id.String()) + shBlz := b.getBlobovnicza(id.String()) blz, err := shBlz.Open() if err != nil { return common.GetRangeRes{}, err @@ -102,7 +102,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re // returns error if object could not be read from any blobovnicza of the same level. func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRangePrm, blzPath string) (common.GetRangeRes, error) { // open blobovnicza (cached inside) - shBlz := b.openBlobovnicza(blzPath) + shBlz := b.getBlobovnicza(blzPath) blz, err := shBlz.Open() if err != nil { return common.GetRangeRes{}, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index cc273839a..a2afbb8aa 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -68,7 +68,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm // iterator over all Blobovniczas in unsorted order. Break on f's error return. func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error { return b.iterateLeaves(ctx, func(p string) (bool, error) { - shBlz := b.openBlobovnicza(p) + shBlz := b.getBlobovnicza(p) blz, err := shBlz.Open() if err != nil { if ignoreErrors { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index 941d9961d..7bc8e2827 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -114,8 +114,7 @@ func newLevelDBManager(width uint64, options []blobovnicza.Option, rootPath stri result := &levelDbManager{ databases: make([]*sharedDB, width), } - var idx uint64 - for idx = 0; idx < width; idx++ { + for idx := uint64(0); idx < width; idx++ { result.databases[idx] = newSharedDB(options, filepath.Join(rootPath, lvlPath, u64ToHexString(idx)), readOnly, metrics, openDBCounter, closedFlog, log) } return result From a54b4472de7b2970e2280285ffabc42be3b55160 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 4 Sep 2023 15:13:49 +0300 Subject: [PATCH 0568/1943] [#674] network: Close connections on address updates Signed-off-by: Evgenii Stratonikov --- pkg/network/cache/multi.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 98d2f33e7..d03edd962 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -130,6 +130,7 @@ loop: continue loop } } + x.clients[a].invalidate() delete(x.clients, a) } From 054e3ef3d30790650729875f9835b025570c800a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 4 Sep 2023 15:33:21 +0300 Subject: [PATCH 0569/1943] [#674] pre-commit: Update shellcheck-py Signed-off-by: Evgenii Stratonikov --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f891f8860..5f20bdc84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: exclude: ".key$" - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.2 + rev: v0.9.0.5 hooks: - id: shellcheck From 88d50e4c7734219b151ba93f494fb5bd29e8a14f Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 4 Sep 2023 12:00:49 +0300 Subject: [PATCH 0570/1943] [#656] policer: Add "bad" testcase Signed-off-by: Anton Nikiforov --- internal/logs/logs.go | 1 + pkg/services/policer/check.go | 33 +++++++++------------------- pkg/services/policer/policer_test.go | 33 +++++++++++++++++++++++++++- pkg/services/policer/process.go | 7 +++++- 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index ed88e615b..1c5815c6c 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -53,6 +53,7 @@ const ( PolicerRoutineStopped = "routine stopped" PolicerFailureAtObjectSelectForReplication = "failure at object select for replication" PolicerPoolSubmission = "pool submission" + PolicerUnableToProcessObj = "unable to process object" ReplicatorFinishWork = "finish work" ReplicatorCouldNotGetObjectFromLocalStorage = "could not get object from local storage" ReplicatorCouldNotReplicateObject = "could not replicate object" diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index bc82b144d..c04273468 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -3,6 +3,7 @@ package policer import ( "context" "errors" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" @@ -16,48 +17,33 @@ import ( "go.uber.org/zap" ) -func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.AddressWithType) { +func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.AddressWithType) error { addr := addrWithType.Address idCnr := addr.Container() idObj := addr.Object() cnr, err := p.cnrSrc.Get(idCnr) if err != nil { - p.log.Error(logs.PolicerCouldNotGetContainer, - zap.Stringer("cid", idCnr), - zap.String("error", err.Error()), - ) if client.IsErrContainerNotFound(err) { - existed, err := containercore.WasRemoved(p.cnrSrc, idCnr) - if err != nil { - p.log.Error(logs.PolicerCouldNotConfirmContainerRemoval, - zap.Stringer("cid", idCnr), - zap.Stringer("oid", idObj), - zap.String("error", err.Error())) + existed, errWasRemoved := containercore.WasRemoved(p.cnrSrc, idCnr) + if errWasRemoved != nil { + return fmt.Errorf("%s: %w", logs.PolicerCouldNotConfirmContainerRemoval, errWasRemoved) } else if existed { err := p.buryFn(ctx, addrWithType.Address) if err != nil { - p.log.Error(logs.PolicerCouldNotInhumeObjectWithMissingContainer, - zap.Stringer("cid", idCnr), - zap.Stringer("oid", idObj), - zap.String("error", err.Error())) + return fmt.Errorf("%s: %w", logs.PolicerCouldNotInhumeObjectWithMissingContainer, err) } } } - return + return fmt.Errorf("%s: %w", logs.PolicerCouldNotGetContainer, err) } policy := cnr.Value.PlacementPolicy() nn, err := p.placementBuilder.BuildPlacement(idCnr, &idObj, policy) if err != nil { - p.log.Error(logs.PolicerCouldNotBuildPlacementVectorForObject, - zap.Stringer("cid", idCnr), - zap.String("error", err.Error()), - ) - - return + return fmt.Errorf("%s: %w", logs.PolicerCouldNotBuildPlacementVectorForObject, err) } c := &placementRequirements{} @@ -73,7 +59,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add for i := range nn { select { case <-ctx.Done(): - return + return ctx.Err() default: } @@ -87,6 +73,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add p.cbRedundantCopy(ctx, addr) } + return nil } type placementRequirements struct { diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 8b3eecd3b..c73d33629 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -12,6 +12,7 @@ import ( objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -245,7 +246,8 @@ func TestProcessObject(t *testing.T) { Type: ti.objType, } - p.processObject(context.Background(), addrWithType) + err := p.processObject(context.Background(), addrWithType) + require.NoError(t, err) sort.Ints(gotReplicateTo) require.Equal(t, ti.wantRemoveRedundant, gotRemoveRedundant) @@ -254,6 +256,35 @@ func TestProcessObject(t *testing.T) { } } +func TestProcessObjectError(t *testing.T) { + addr := oidtest.Address() + // Container source + cnr := &container.Container{} + cnr.Value.Init() + source := containerSrc{ + get: func(id cid.ID) (*container.Container, error) { + return nil, new(apistatus.ContainerNotFound) + }, + deletionInfo: func(id cid.ID) (*container.DelInfo, error) { + return nil, new(apistatus.ContainerNotFound) + }, + } + buryFn := func(ctx context.Context, a oid.Address) error { + t.Errorf("unexpected object buried: %v", a) + return nil + } + p := New( + WithContainerSource(source), + WithBuryFunc(buryFn), + ) + + addrWithType := objectcore.AddressWithType{ + Address: addr, + } + + require.True(t, client.IsErrContainerNotFound(p.processObject(context.Background(), addrWithType))) +} + func TestIteratorContract(t *testing.T) { addr := oidtest.Address() objs := []objectcore.AddressWithType{{ diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 1f61c69f4..2e8fe9295 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -52,7 +52,12 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { } if p.objsInWork.add(addr.Address) { - p.processObject(ctx, addr) + err := p.processObject(ctx, addr) + if err != nil { + p.log.Error(logs.PolicerUnableToProcessObj, + zap.Stringer("object", addr.Address), + zap.String("error", err.Error())) + } p.cache.Add(addr.Address, time.Now()) p.objsInWork.remove(addr.Address) } From d9de9e2bbb8d56990f66bd89337b71645746c3e1 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 4 Sep 2023 18:40:22 +0300 Subject: [PATCH 0571/1943] [#675] client: Refactor PrmObjectDelete usage Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/client.go | 19 ++++++++----------- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 22ca0aa7d..7fbfd5105 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -507,20 +507,17 @@ func (x DeleteObjectRes) Tombstone() oid.ID { // // Returns any error which prevented the operation from completing correctly in error return. func DeleteObject(ctx context.Context, prm DeleteObjectPrm) (*DeleteObjectRes, error) { - var delPrm client.PrmObjectDelete - delPrm.FromContainer(prm.objAddr.Container()) - delPrm.ByID(prm.objAddr.Object()) + cnr := prm.objAddr.Container() + obj := prm.objAddr.Object() - if prm.sessionToken != nil { - delPrm.WithinSession(*prm.sessionToken) + delPrm := client.PrmObjectDelete{ + XHeaders: prm.xHeaders, + ContainerID: &cnr, + ObjectID: &obj, + Session: prm.sessionToken, + BearerToken: prm.bearerToken, } - if prm.bearerToken != nil { - delPrm.WithBearerToken(*prm.bearerToken) - } - - delPrm.WithXHeaders(prm.xHeaders...) - cliRes, err := prm.cli.ObjectDelete(ctx, delPrm) if err != nil { return nil, fmt.Errorf("remove object via client: %w", err) diff --git a/go.mod b/go.mod index 114c95adb..6d5382b75 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230828082657-84e7e69f98ac + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904141422-5a471e5002a7 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 4b5e40174..6c8b41704 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230828082657-84e7e69f98ac h1:BRmJd04eUgzmdLe/2I0Ok/LNOkTyEAPymYUrCCF0a1U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230828082657-84e7e69f98ac/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904141422-5a471e5002a7 h1:fwEDv/1INboZGbfQDxlsdbZGvHNUL/73rQe4fG8G3DA= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904141422-5a471e5002a7/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From aeeb8193d22e984a7959e25a572119893bd0ce1d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 6 Sep 2023 17:06:54 +0300 Subject: [PATCH 0572/1943] [#676] node: Fix header source creation when checking eacl Signed-off-by: Anton Nikiforov --- pkg/services/object/acl/eacl/v2/headers.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/services/object/acl/eacl/v2/headers.go b/pkg/services/object/acl/eacl/v2/headers.go index 7408c96e9..34975e1e6 100644 --- a/pkg/services/object/acl/eacl/v2/headers.go +++ b/pkg/services/object/acl/eacl/v2/headers.go @@ -137,6 +137,8 @@ func (h *cfg) readObjectHeadersFromRequestXHeaderSource(m requestXHeaderSource, dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) } + case *objectV2.PutSingleRequest: + dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(req.GetBody().GetObject()), h.cnr, h.obj) case *objectV2.SearchRequest: cnrV2 := req.GetBody().GetContainerID() var cnr cid.ID From da8f384324fb91e7376ce4df5d82936512cde24c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 30 Aug 2023 17:10:21 +0300 Subject: [PATCH 0573/1943] [#668] shard/test: Fix typo in `existence` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/gc_internal_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index c258b6c99..9a4f52029 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -116,13 +116,13 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { storageID, err := sh.metaBase.StorageID(context.Background(), metaStIDPrm) require.NoError(t, err, "failed to get storage ID") - //check existance in blobstore + //check existence in blobstore var bsExisted common.ExistsPrm bsExisted.Address = addr bsExisted.StorageID = storageID.StorageID() exRes, err := sh.blobStor.Exists(context.Background(), bsExisted) - require.NoError(t, err, "failed to check blobstore existance") - require.True(t, exRes.Exists, "invalid blobstore existance result") + require.NoError(t, err, "failed to check blobstore existence") + require.True(t, exRes.Exists, "invalid blobstore existence result") //drop from blobstor var bsDeletePrm common.DeletePrm @@ -131,10 +131,10 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { _, err = sh.blobStor.Delete(context.Background(), bsDeletePrm) require.NoError(t, err, "failed to delete from blobstore") - //check existance in blobstore + //check existence in blobstore exRes, err = sh.blobStor.Exists(context.Background(), bsExisted) - require.NoError(t, err, "failed to check blobstore existance") - require.False(t, exRes.Exists, "invalid blobstore existance result") + require.NoError(t, err, "failed to check blobstore existence") + require.False(t, exRes.Exists, "invalid blobstore existence result") //get should return object not found _, err = sh.Get(context.Background(), getPrm) From aa23c6a83aaebee17f102e2f7301b773132212f9 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 30 Aug 2023 18:25:56 +0300 Subject: [PATCH 0574/1943] [#668] shard/test: Remove subtest from TestCounters Otherwise, individual tests cannot be run. Signed-off-by: Evgenii Stratonikov --- .../shard/metrics_test.go | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 3d904331e..280741f7b 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -163,21 +163,19 @@ func TestCounters(t *testing.T) { totalPayload += oSize } - t.Run("put", func(t *testing.T) { - var prm shard.PutPrm + var prm shard.PutPrm - for i := 0; i < objNumber; i++ { - prm.SetObject(oo[i]) + for i := 0; i < objNumber; i++ { + prm.SetObject(oo[i]) - _, err := sh.Put(context.Background(), prm) - require.NoError(t, err) - } + _, err := sh.Put(context.Background(), prm) + require.NoError(t, err) + } - require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical)) - require.Equal(t, uint64(objNumber), mm.getObjectCounter(logical)) - require.Equal(t, expectedSizes, mm.containerSizes()) - require.Equal(t, totalPayload, mm.payloadSize()) - }) + require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical)) + require.Equal(t, uint64(objNumber), mm.getObjectCounter(logical)) + require.Equal(t, expectedSizes, mm.containerSizes()) + require.Equal(t, totalPayload, mm.payloadSize()) t.Run("inhume_GC", func(t *testing.T) { var prm shard.InhumePrm From bdecfbc1be41935fa8faeee04fc60bb5909809fe Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 31 Aug 2023 15:47:06 +0300 Subject: [PATCH 0575/1943] [#668] shard/test: Move tests to the main package Semantic patch (also, duplicate definitions are removed): ``` @@ var e identifier @@ -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" -shard.e +e ``` Signed-off-by: Evgenii Stratonikov --- .../shard/control_test.go | 6 --- pkg/local_object_storage/shard/delete_test.go | 11 +++--- pkg/local_object_storage/shard/gc_test.go | 15 ++++---- pkg/local_object_storage/shard/get_test.go | 9 ++--- pkg/local_object_storage/shard/head_test.go | 9 ++--- pkg/local_object_storage/shard/inhume_test.go | 9 ++--- pkg/local_object_storage/shard/list_test.go | 7 ++-- pkg/local_object_storage/shard/lock_test.go | 33 ++++++++--------- .../shard/metrics_test.go | 28 ++++++-------- pkg/local_object_storage/shard/range_test.go | 7 ++-- pkg/local_object_storage/shard/shard_test.go | 37 +++++++++---------- .../shard/shutdown_test.go | 7 ++-- 12 files changed, 79 insertions(+), 99 deletions(-) diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 73612d840..128e5cc0c 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -32,12 +32,6 @@ import ( "go.etcd.io/bbolt" ) -type epochState struct{} - -func (s epochState) CurrentEpoch() uint64 { - return 0 -} - type objAddr struct { obj *objectSDK.Object addr oid.Address diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index bfafcdc74..8ee613186 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -1,4 +1,4 @@ -package shard_test +package shard import ( "context" @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" @@ -36,8 +35,8 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { obj := testutil.GenerateObjectWithCID(cnr) testutil.AddAttribute(obj, "foo", "bar") - var putPrm shard.PutPrm - var getPrm shard.GetPrm + var putPrm PutPrm + var getPrm GetPrm t.Run("big object", func(t *testing.T) { testutil.AddPayload(obj, 1<<20) @@ -45,7 +44,7 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { putPrm.SetObject(obj) getPrm.SetAddress(object.AddressOf(obj)) - var delPrm shard.DeletePrm + var delPrm DeletePrm delPrm.SetAddresses(object.AddressOf(obj)) _, err := sh.Put(context.Background(), putPrm) @@ -71,7 +70,7 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { putPrm.SetObject(obj) getPrm.SetAddress(object.AddressOf(obj)) - var delPrm shard.DeletePrm + var delPrm DeletePrm delPrm.SetAddresses(object.AddressOf(obj)) _, err := sh.Put(context.Background(), putPrm) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 9d2771ae4..24038f10b 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -1,4 +1,4 @@ -package shard_test +package shard import ( "context" @@ -10,7 +10,6 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" @@ -55,7 +54,7 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { lock.SetAttributes(lockExpirationAttr) lockID, _ := lock.ID() - var putPrm shard.PutPrm + var putPrm PutPrm putPrm.SetObject(obj) _, err := sh.Put(context.Background(), putPrm) @@ -69,9 +68,9 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { require.NoError(t, err) epoch.Value = 105 - sh.NotificationChannel() <- shard.EventNewEpoch(epoch.Value) + sh.NotificationChannel() <- EventNewEpoch(epoch.Value) - var getPrm shard.GetPrm + var getPrm GetPrm getPrm.SetAddress(objectCore.AddressOf(obj)) require.Eventually(t, func() bool { _, err = sh.Get(context.Background(), getPrm) @@ -141,7 +140,7 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { lock.SetAttributes(lockExpirationAttr) lockID, _ := lock.ID() - var putPrm shard.PutPrm + var putPrm PutPrm for _, child := range children { putPrm.SetObject(child) @@ -160,7 +159,7 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { _, err = sh.Put(context.Background(), putPrm) require.NoError(t, err) - var getPrm shard.GetPrm + var getPrm GetPrm getPrm.SetAddress(objectCore.AddressOf(parent)) _, err = sh.Get(context.Background(), getPrm) @@ -168,7 +167,7 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { require.True(t, errors.As(err, &splitInfoError), "split info must be provided") epoch.Value = 105 - sh.NotificationChannel() <- shard.EventNewEpoch(epoch.Value) + sh.NotificationChannel() <- EventNewEpoch(epoch.Value) require.Eventually(t, func() bool { _, err = sh.Get(context.Background(), getPrm) diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index 25639902d..d3aeb5339 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -1,4 +1,4 @@ -package shard_test +package shard import ( "bytes" @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -35,8 +34,8 @@ func testShardGet(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) defer releaseShard(sh, t) - var putPrm shard.PutPrm - var getPrm shard.GetPrm + var putPrm PutPrm + var getPrm GetPrm t.Run("small object", func(t *testing.T) { obj := testutil.GenerateObject() @@ -116,7 +115,7 @@ func testShardGet(t *testing.T, hasWriteCache bool) { }) } -func testGet(t *testing.T, sh *shard.Shard, getPrm shard.GetPrm, hasWriteCache bool) (shard.GetRes, error) { +func testGet(t *testing.T, sh *Shard, getPrm GetPrm, hasWriteCache bool) (GetRes, error) { res, err := sh.Get(context.Background(), getPrm) if hasWriteCache { require.Eventually(t, func() bool { diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index 3a7457ace..bf6ba990b 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -1,4 +1,4 @@ -package shard_test +package shard import ( "context" @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -33,8 +32,8 @@ func testShardHead(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) defer releaseShard(sh, t) - var putPrm shard.PutPrm - var headPrm shard.HeadPrm + var putPrm PutPrm + var headPrm HeadPrm t.Run("regular object", func(t *testing.T) { obj := testutil.GenerateObject() @@ -87,7 +86,7 @@ func testShardHead(t *testing.T, hasWriteCache bool) { }) } -func testHead(t *testing.T, sh *shard.Shard, headPrm shard.HeadPrm, hasWriteCache bool) (shard.HeadRes, error) { +func testHead(t *testing.T, sh *Shard, headPrm HeadPrm, hasWriteCache bool) (HeadRes, error) { res, err := sh.Head(context.Background(), headPrm) if hasWriteCache { require.Eventually(t, func() bool { diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 3fa6bc0a3..33d483b5e 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -1,4 +1,4 @@ -package shard_test +package shard import ( "context" @@ -6,7 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" @@ -37,13 +36,13 @@ func testShardInhume(t *testing.T, hasWriteCache bool) { ts := testutil.GenerateObjectWithCID(cnr) - var putPrm shard.PutPrm + var putPrm PutPrm putPrm.SetObject(obj) - var inhPrm shard.InhumePrm + var inhPrm InhumePrm inhPrm.SetTarget(object.AddressOf(ts), object.AddressOf(obj)) - var getPrm shard.GetPrm + var getPrm GetPrm getPrm.SetAddress(object.AddressOf(obj)) _, err := sh.Put(context.Background(), putPrm) diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index 63e7651c8..33b94ce06 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -1,4 +1,4 @@ -package shard_test +package shard import ( "context" @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -31,7 +30,7 @@ func TestShard_List(t *testing.T) { }) } -func testShardList(t *testing.T, sh *shard.Shard) { +func testShardList(t *testing.T, sh *Shard) { const C = 10 const N = 5 @@ -59,7 +58,7 @@ func testShardList(t *testing.T, sh *shard.Shard) { objs[object.AddressOf(obj).EncodeToString()] = 0 mtx.Unlock() - var putPrm shard.PutPrm + var putPrm PutPrm putPrm.SetObject(obj) _, err := sh.Put(context.Background(), putPrm) diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index da71c4808..be9a0ba9a 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -1,4 +1,4 @@ -package shard_test +package shard import ( "context" @@ -11,7 +11,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -26,13 +25,13 @@ import ( func TestShard_Lock(t *testing.T) { t.Parallel() - var sh *shard.Shard + var sh *Shard rootPath := t.TempDir() - opts := []shard.Option{ - shard.WithID(shard.NewIDFromBytes([]byte{})), - shard.WithLogger(&logger.Logger{Logger: zap.NewNop()}), - shard.WithBlobStorOptions( + opts := []Option{ + WithID(NewIDFromBytes([]byte{})), + WithLogger(&logger.Logger{Logger: zap.NewNop()}), + WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( @@ -49,16 +48,16 @@ func TestShard_Lock(t *testing.T) { }, }), ), - shard.WithMetaBaseOptions( + WithMetaBaseOptions( meta.WithPath(filepath.Join(rootPath, "meta")), meta.WithEpochState(epochState{}), ), - shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { + WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { sh.HandleDeletedLocks(addresses) }), } - sh = shard.New(opts...) + sh = New(opts...) require.NoError(t, sh.Open()) require.NoError(t, sh.Init(context.Background())) @@ -76,7 +75,7 @@ func TestShard_Lock(t *testing.T) { // put the object - var putPrm shard.PutPrm + var putPrm PutPrm putPrm.SetObject(obj) _, err := sh.Put(context.Background(), putPrm) @@ -94,7 +93,7 @@ func TestShard_Lock(t *testing.T) { t.Run("inhuming locked objects", func(t *testing.T) { ts := testutil.GenerateObjectWithCID(cnr) - var inhumePrm shard.InhumePrm + var inhumePrm InhumePrm inhumePrm.SetTarget(objectcore.AddressOf(ts), objectcore.AddressOf(obj)) var objLockedErr *apistatus.ObjectLocked @@ -110,7 +109,7 @@ func TestShard_Lock(t *testing.T) { t.Run("inhuming lock objects", func(t *testing.T) { ts := testutil.GenerateObjectWithCID(cnr) - var inhumePrm shard.InhumePrm + var inhumePrm InhumePrm inhumePrm.SetTarget(objectcore.AddressOf(ts), objectcore.AddressOf(lock)) _, err = sh.Inhume(context.Background(), inhumePrm) @@ -122,7 +121,7 @@ func TestShard_Lock(t *testing.T) { }) t.Run("force objects inhuming", func(t *testing.T) { - var inhumePrm shard.InhumePrm + var inhumePrm InhumePrm inhumePrm.MarkAsGarbage(objectcore.AddressOf(lock)) inhumePrm.ForceRemoval() @@ -132,7 +131,7 @@ func TestShard_Lock(t *testing.T) { // it should be possible to remove // lock object now - inhumePrm = shard.InhumePrm{} + inhumePrm = InhumePrm{} inhumePrm.MarkAsGarbage(objectcore.AddressOf(obj)) _, err = sh.Inhume(context.Background(), inhumePrm) @@ -140,7 +139,7 @@ func TestShard_Lock(t *testing.T) { // check that object has been removed - var getPrm shard.GetPrm + var getPrm GetPrm getPrm.SetAddress(objectcore.AddressOf(obj)) _, err = sh.Get(context.Background(), getPrm) @@ -160,7 +159,7 @@ func TestShard_IsLocked(t *testing.T) { // put the object - var putPrm shard.PutPrm + var putPrm PutPrm putPrm.SetObject(obj) _, err := sh.Put(context.Background(), putPrm) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 280741f7b..ae993c79c 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -1,4 +1,4 @@ -package shard_test +package shard import ( "context" @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -126,9 +125,6 @@ func (m *metricsStore) DeleteShardMetrics() { m.errCounter = 0 } -const physical = "phy" -const logical = "logic" - func TestCounters(t *testing.T) { t.Parallel() @@ -163,7 +159,7 @@ func TestCounters(t *testing.T) { totalPayload += oSize } - var prm shard.PutPrm + var prm PutPrm for i := 0; i < objNumber; i++ { prm.SetObject(oo[i]) @@ -178,7 +174,7 @@ func TestCounters(t *testing.T) { require.Equal(t, totalPayload, mm.payloadSize()) t.Run("inhume_GC", func(t *testing.T) { - var prm shard.InhumePrm + var prm InhumePrm inhumedNumber := objNumber / 4 for i := 0; i < inhumedNumber; i++ { @@ -197,7 +193,7 @@ func TestCounters(t *testing.T) { }) t.Run("inhume_TS", func(t *testing.T) { - var prm shard.InhumePrm + var prm InhumePrm ts := objectcore.AddressOf(testutil.GenerateObject()) phy := mm.getObjectCounter(physical) @@ -218,7 +214,7 @@ func TestCounters(t *testing.T) { }) t.Run("Delete", func(t *testing.T) { - var prm shard.DeletePrm + var prm DeletePrm phy := mm.getObjectCounter(physical) logic := mm.getObjectCounter(logical) @@ -244,7 +240,7 @@ func TestCounters(t *testing.T) { }) } -func shardWithMetrics(t *testing.T, path string) (*shard.Shard, *metricsStore) { +func shardWithMetrics(t *testing.T, path string) (*Shard, *metricsStore) { blobOpts := []blobstor.Option{ blobstor.WithStorages([]blobstor.SubStorage{ { @@ -264,14 +260,14 @@ func shardWithMetrics(t *testing.T, path string) (*shard.Shard, *metricsStore) { cnrSize: make(map[string]int64), } - sh := shard.New( - shard.WithID(shard.NewIDFromBytes([]byte{})), - shard.WithBlobStorOptions(blobOpts...), - shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(path, "pilorama"))), - shard.WithMetaBaseOptions( + sh := New( + WithID(NewIDFromBytes([]byte{})), + WithBlobStorOptions(blobOpts...), + WithPiloramaOptions(pilorama.WithPath(filepath.Join(path, "pilorama"))), + WithMetaBaseOptions( meta.WithPath(filepath.Join(path, "meta")), meta.WithEpochState(epochState{})), - shard.WithMetricsWriter(mm), + WithMetricsWriter(mm), ) require.NoError(t, sh.Open()) require.NoError(t, sh.Init(context.Background())) diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 68693f769..4741e5163 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -1,4 +1,4 @@ -package shard_test +package shard import ( "context" @@ -11,7 +11,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" @@ -106,13 +105,13 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { addr := object.AddressOf(obj) payload := slice.Copy(obj.Payload()) - var putPrm shard.PutPrm + var putPrm PutPrm putPrm.SetObject(obj) _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) - var rngPrm shard.RngPrm + var rngPrm RngPrm rngPrm.SetAddress(addr) rngPrm.SetRange(tc.rng.GetOffset(), tc.rng.GetLength()) diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 6337b0b6e..d1bfc0880 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -1,4 +1,4 @@ -package shard_test +package shard import ( "context" @@ -11,7 +11,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" @@ -31,15 +30,15 @@ func (s epochState) CurrentEpoch() uint64 { return s.Value } -func newShard(t testing.TB, enableWriteCache bool) *shard.Shard { +func newShard(t testing.TB, enableWriteCache bool) *Shard { return newCustomShard(t, t.TempDir(), enableWriteCache, writecacheconfig.Options{Type: writecacheconfig.TypeBBolt}, nil, nil) } -func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts writecacheconfig.Options, bsOpts []blobstor.Option, metaOptions []meta.Option) *shard.Shard { - var sh *shard.Shard +func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts writecacheconfig.Options, bsOpts []blobstor.Option, metaOptions []meta.Option) *Shard { + var sh *Shard if enableWriteCache { rootPath = filepath.Join(rootPath, "wc") switch wcOpts.Type { @@ -78,33 +77,33 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts } } - opts := []shard.Option{ - shard.WithID(shard.NewIDFromBytes([]byte{})), - shard.WithLogger(test.NewLogger(t, true)), - shard.WithBlobStorOptions(bsOpts...), - shard.WithMetaBaseOptions( + opts := []Option{ + WithID(NewIDFromBytes([]byte{})), + WithLogger(test.NewLogger(t, true)), + WithBlobStorOptions(bsOpts...), + WithMetaBaseOptions( append([]meta.Option{ meta.WithPath(filepath.Join(rootPath, "meta")), meta.WithEpochState(epochState{})}, metaOptions...)..., ), - shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))), - shard.WithWriteCache(enableWriteCache), - shard.WithWriteCacheOptions(wcOpts), - shard.WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { + WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))), + WithWriteCache(enableWriteCache), + WithWriteCacheOptions(wcOpts), + WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { sh.HandleDeletedLocks(addresses) }), - shard.WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { + WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { sh.HandleExpiredLocks(ctx, epoch, a) }), - shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { pool, err := ants.NewPool(sz) require.NoError(t, err) return pool }), - shard.WithGCRemoverSleepInterval(100 * time.Millisecond), + WithGCRemoverSleepInterval(100 * time.Millisecond), } - sh = shard.New(opts...) + sh = New(opts...) require.NoError(t, sh.Open()) require.NoError(t, sh.Init(context.Background())) @@ -112,6 +111,6 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts return sh } -func releaseShard(s *shard.Shard, t testing.TB) { +func releaseShard(s *Shard, t testing.TB) { require.NoError(t, s.Close()) } diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 68ef90963..304448b1b 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -1,4 +1,4 @@ -package shard_test +package shard import ( "context" @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" @@ -50,7 +49,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { for i := range objects { obj := objects[i] errG.Go(func() error { - var putPrm shard.PutPrm + var putPrm PutPrm putPrm.SetObject(obj) _, err := sh.Put(context.Background(), putPrm) return err @@ -62,7 +61,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { sh = newCustomShard(t, dir, true, wcOpts, nil, nil) defer releaseShard(sh, t) - var getPrm shard.GetPrm + var getPrm GetPrm for i := range objects { getPrm.SetAddress(object.AddressOf(objects[i])) From 42696016de769a50a3009aa7aeaed84ddff7b2b6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 31 Aug 2023 16:33:37 +0300 Subject: [PATCH 0576/1943] [#668] shard: Close stopChannel in GC It is done once, but now we could read it from multiple places. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/gc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 2221d57c1..8be91fcaf 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -220,7 +220,7 @@ func (gc *gc) tickRemover(ctx context.Context) { func (gc *gc) stop() { gc.onceStop.Do(func() { - gc.stopChannel <- struct{}{} + close(gc.stopChannel) }) gc.log.Info(logs.ShardWaitingForGCWorkersToStop) From 382eb8a485dda328ba1f2886d98efd2756dd7add Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 31 Aug 2023 18:17:55 +0300 Subject: [PATCH 0577/1943] [#668] shard/test: Disable GC where it is not needed Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/gc.go | 9 ++++++++- pkg/local_object_storage/shard/gc_internal_test.go | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 8be91fcaf..38baccd4b 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -119,6 +119,8 @@ type gcCfg struct { expiredCollectorBatchSize int metrics GCMectrics + + testHookRemover func(ctx context.Context) gcRunResult } func defaultGCCfg() gcCfg { @@ -209,7 +211,12 @@ func (gc *gc) tickRemover(ctx context.Context) { case <-timer.C: startedAt := time.Now() - result := gc.remover(ctx) + var result gcRunResult + if gc.testHookRemover != nil { + result = gc.testHookRemover(ctx) + } else { + result = gc.remover(ctx) + } timer.Reset(gc.removerInterval) gc.metrics.AddRunDuration(time.Since(startedAt), result.success) diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index 9a4f52029..c8925e01f 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -75,7 +75,7 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { } sh = New(opts...) - + sh.gcCfg.testHookRemover = func(context.Context) gcRunResult { return gcRunResult{} } require.NoError(t, sh.Open()) require.NoError(t, sh.Init(context.Background())) From 429f941cdaed1dbf8d77e4e25c27e3dc90ae269a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Sep 2023 17:18:04 +0300 Subject: [PATCH 0578/1943] [#668] shard/test: Release shard in t.Cleanup() Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/delete_test.go | 1 - pkg/local_object_storage/shard/get_test.go | 1 - pkg/local_object_storage/shard/head_test.go | 1 - pkg/local_object_storage/shard/inhume_test.go | 1 - pkg/local_object_storage/shard/list_test.go | 2 -- pkg/local_object_storage/shard/shard_test.go | 4 +++- 6 files changed, 3 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index 8ee613186..3421ac9e0 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -28,7 +28,6 @@ func TestShard_Delete(t *testing.T) { func testShardDelete(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) - defer releaseShard(sh, t) cnr := cidtest.ID() diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index d3aeb5339..19a5e8d70 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -32,7 +32,6 @@ func TestShard_Get(t *testing.T) { func testShardGet(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) - defer releaseShard(sh, t) var putPrm PutPrm var getPrm GetPrm diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index bf6ba990b..dfae48e84 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -30,7 +30,6 @@ func TestShard_Head(t *testing.T) { func testShardHead(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) - defer releaseShard(sh, t) var putPrm PutPrm var headPrm HeadPrm diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 33d483b5e..6c8e46faf 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -27,7 +27,6 @@ func TestShard_Inhume(t *testing.T) { func testShardInhume(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) - defer releaseShard(sh, t) cnr := cidtest.ID() diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index 33b94ce06..9ca1753c4 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -18,14 +18,12 @@ func TestShard_List(t *testing.T) { t.Run("without write cache", func(t *testing.T) { t.Parallel() sh := newShard(t, false) - defer releaseShard(sh, t) testShardList(t, sh) }) t.Run("with write cache", func(t *testing.T) { t.Parallel() shWC := newShard(t, true) - defer releaseShard(shWC, t) testShardList(t, shWC) }) } diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index d1bfc0880..1550af5c0 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -31,10 +31,12 @@ func (s epochState) CurrentEpoch() uint64 { } func newShard(t testing.TB, enableWriteCache bool) *Shard { - return newCustomShard(t, t.TempDir(), enableWriteCache, + sh := newCustomShard(t, t.TempDir(), enableWriteCache, writecacheconfig.Options{Type: writecacheconfig.TypeBBolt}, nil, nil) + t.Cleanup(func() { releaseShard(sh, t) }) + return sh } func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts writecacheconfig.Options, bsOpts []blobstor.Option, metaOptions []meta.Option) *Shard { From 268adb79cbab63efe7c9c248cc848c4580887e61 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Sep 2023 17:32:14 +0300 Subject: [PATCH 0579/1943] [#668] shard/test: Simplify shard construction newCustomShard() has many parameters but only the first is obligatory. `enableWriteCache` is left as-is, because it directly affects the functionality. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/gc_test.go | 4 +- pkg/local_object_storage/shard/range_test.go | 38 +++++++------ pkg/local_object_storage/shard/shard_test.go | 53 +++++++++++-------- .../shard/shutdown_test.go | 4 +- 4 files changed, 55 insertions(+), 44 deletions(-) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 24038f10b..a7daadac8 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -29,7 +29,7 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { wcOpts := writecacheconfig.Options{ Type: writecacheconfig.TypeBBolt, } - sh := newCustomShard(t, t.TempDir(), false, wcOpts, nil, []meta.Option{meta.WithEpochState(epoch)}) + sh := newCustomShard(t, false, shardOptions{rootPath: t.TempDir(), wcOpts: wcOpts, metaOptions: []meta.Option{meta.WithEpochState(epoch)}}) t.Cleanup(func() { releaseShard(sh, t) @@ -129,7 +129,7 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { wcOpts := writecacheconfig.Options{ Type: writecacheconfig.TypeBBolt, } - sh := newCustomShard(t, t.TempDir(), false, wcOpts, nil, []meta.Option{meta.WithEpochState(epoch)}) + sh := newCustomShard(t, false, shardOptions{rootPath: t.TempDir(), wcOpts: wcOpts, metaOptions: []meta.Option{meta.WithEpochState(epoch)}}) t.Cleanup(func() { releaseShard(sh, t) diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 4741e5163..8c2f400d4 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -76,24 +76,28 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { }, } - sh := newCustomShard(t, t.TempDir(), hasWriteCache, wcOpts, - []blobstor.Option{blobstor.WithStorages([]blobstor.SubStorage{ - { - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(test.NewLogger(t, true)), - blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")), - blobovniczatree.WithBlobovniczaShallowDepth(1), - blobovniczatree.WithBlobovniczaShallowWidth(1)), - Policy: func(_ *objectSDK.Object, data []byte) bool { - return len(data) <= smallObjectSize + sh := newCustomShard(t, hasWriteCache, shardOptions{ + rootPath: t.TempDir(), + wcOpts: wcOpts, + bsOpts: []blobstor.Option{ + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithLogger(test.NewLogger(t, true)), + blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")), + blobovniczatree.WithBlobovniczaShallowDepth(1), + blobovniczatree.WithBlobovniczaShallowWidth(1)), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return len(data) <= smallObjectSize + }, }, - }, - { - Storage: fstree.New( - fstree.WithPath(filepath.Join(t.TempDir(), "blob"))), - }, - })}, - nil) + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(t.TempDir(), "blob"))), + }, + }), + }, + }) defer releaseShard(sh, t) for _, tc := range testCases { diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 1550af5c0..1318bd28f 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -30,41 +30,48 @@ func (s epochState) CurrentEpoch() uint64 { return s.Value } +type shardOptions struct { + rootPath string + wcOpts writecacheconfig.Options + bsOpts []blobstor.Option + metaOptions []meta.Option +} + func newShard(t testing.TB, enableWriteCache bool) *Shard { - sh := newCustomShard(t, t.TempDir(), enableWriteCache, - writecacheconfig.Options{Type: writecacheconfig.TypeBBolt}, - nil, - nil) + sh := newCustomShard(t, enableWriteCache, shardOptions{ + rootPath: t.TempDir(), + wcOpts: writecacheconfig.Options{Type: writecacheconfig.TypeBBolt}, + }) t.Cleanup(func() { releaseShard(sh, t) }) return sh } -func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts writecacheconfig.Options, bsOpts []blobstor.Option, metaOptions []meta.Option) *Shard { +func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard { var sh *Shard if enableWriteCache { - rootPath = filepath.Join(rootPath, "wc") - switch wcOpts.Type { + o.rootPath = filepath.Join(o.rootPath, "wc") + switch o.wcOpts.Type { case writecacheconfig.TypeBBolt: - wcOpts.BBoltOptions = append( - []writecachebbolt.Option{writecachebbolt.WithPath(filepath.Join(rootPath, "wcache"))}, - wcOpts.BBoltOptions...) + o.wcOpts.BBoltOptions = append( + []writecachebbolt.Option{writecachebbolt.WithPath(filepath.Join(o.rootPath, "wcache"))}, + o.wcOpts.BBoltOptions...) case writecacheconfig.TypeBadger: - wcOpts.BadgerOptions = append( - []writecachebadger.Option{writecachebadger.WithPath(filepath.Join(rootPath, "wcache"))}, - wcOpts.BadgerOptions...) + o.wcOpts.BadgerOptions = append( + []writecachebadger.Option{writecachebadger.WithPath(filepath.Join(o.rootPath, "wcache"))}, + o.wcOpts.BadgerOptions...) } } else { - rootPath = filepath.Join(rootPath, "nowc") + o.rootPath = filepath.Join(o.rootPath, "nowc") } - if bsOpts == nil { - bsOpts = []blobstor.Option{ + if o.bsOpts == nil { + o.bsOpts = []blobstor.Option{ blobstor.WithLogger(test.NewLogger(t, true)), blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( blobovniczatree.WithLogger(test.NewLogger(t, true)), - blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), + blobovniczatree.WithRootPath(filepath.Join(o.rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), Policy: func(_ *objectSDK.Object, data []byte) bool { @@ -73,7 +80,7 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts }, { Storage: fstree.New( - fstree.WithPath(filepath.Join(rootPath, "blob"))), + fstree.WithPath(filepath.Join(o.rootPath, "blob"))), }, }), } @@ -82,15 +89,15 @@ func newCustomShard(t testing.TB, rootPath string, enableWriteCache bool, wcOpts opts := []Option{ WithID(NewIDFromBytes([]byte{})), WithLogger(test.NewLogger(t, true)), - WithBlobStorOptions(bsOpts...), + WithBlobStorOptions(o.bsOpts...), WithMetaBaseOptions( append([]meta.Option{ - meta.WithPath(filepath.Join(rootPath, "meta")), meta.WithEpochState(epochState{})}, - metaOptions...)..., + meta.WithPath(filepath.Join(o.rootPath, "meta")), meta.WithEpochState(epochState{})}, + o.metaOptions...)..., ), - WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))), + WithPiloramaOptions(pilorama.WithPath(filepath.Join(o.rootPath, "pilorama"))), WithWriteCache(enableWriteCache), - WithWriteCacheOptions(wcOpts), + WithWriteCacheOptions(o.wcOpts), WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { sh.HandleDeletedLocks(addresses) }), diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 304448b1b..6cfa5fb6b 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -43,7 +43,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { }, } - sh := newCustomShard(t, dir, true, wcOpts, nil, nil) + sh := newCustomShard(t, true, shardOptions{rootPath: dir, wcOpts: wcOpts}) var errG errgroup.Group for i := range objects { @@ -58,7 +58,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { require.NoError(t, errG.Wait()) require.NoError(t, sh.Close()) - sh = newCustomShard(t, dir, true, wcOpts, nil, nil) + sh = newCustomShard(t, true, shardOptions{rootPath: dir, wcOpts: wcOpts}) defer releaseShard(sh, t) var getPrm GetPrm From c661ba13126aa58dfdc56db62f0513571e0d09ef Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Sep 2023 17:48:00 +0300 Subject: [PATCH 0580/1943] [#668] shard/test: Use sane defaults in the test constructor Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/gc_test.go | 11 ++--------- pkg/local_object_storage/shard/range_test.go | 3 +-- pkg/local_object_storage/shard/shard_test.go | 12 ++++++++---- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index a7daadac8..d6a7e39e2 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -10,7 +10,6 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -26,10 +25,7 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { Value: 100, } - wcOpts := writecacheconfig.Options{ - Type: writecacheconfig.TypeBBolt, - } - sh := newCustomShard(t, false, shardOptions{rootPath: t.TempDir(), wcOpts: wcOpts, metaOptions: []meta.Option{meta.WithEpochState(epoch)}}) + sh := newCustomShard(t, false, shardOptions{metaOptions: []meta.Option{meta.WithEpochState(epoch)}}) t.Cleanup(func() { releaseShard(sh, t) @@ -126,10 +122,7 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { linkID, _ := link.ID() - wcOpts := writecacheconfig.Options{ - Type: writecacheconfig.TypeBBolt, - } - sh := newCustomShard(t, false, shardOptions{rootPath: t.TempDir(), wcOpts: wcOpts, metaOptions: []meta.Option{meta.WithEpochState(epoch)}}) + sh := newCustomShard(t, false, shardOptions{metaOptions: []meta.Option{meta.WithEpochState(epoch)}}) t.Cleanup(func() { releaseShard(sh, t) diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 8c2f400d4..77a20966a 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -77,8 +77,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { } sh := newCustomShard(t, hasWriteCache, shardOptions{ - rootPath: t.TempDir(), - wcOpts: wcOpts, + wcOpts: wcOpts, bsOpts: []blobstor.Option{ blobstor.WithStorages([]blobstor.SubStorage{ { diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 1318bd28f..33560407e 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -38,15 +38,19 @@ type shardOptions struct { } func newShard(t testing.TB, enableWriteCache bool) *Shard { - sh := newCustomShard(t, enableWriteCache, shardOptions{ - rootPath: t.TempDir(), - wcOpts: writecacheconfig.Options{Type: writecacheconfig.TypeBBolt}, - }) + sh := newCustomShard(t, enableWriteCache, shardOptions{}) t.Cleanup(func() { releaseShard(sh, t) }) return sh } func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard { + if o.rootPath == "" { + o.rootPath = t.TempDir() + } + if enableWriteCache && o.wcOpts.Type == 0 { + o.wcOpts.Type = writecacheconfig.TypeBBolt + } + var sh *Shard if enableWriteCache { o.rootPath = filepath.Join(o.rootPath, "wc") From 39879fa868bdaac111711ec40159028ffbdd7469 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Sep 2023 17:51:15 +0300 Subject: [PATCH 0581/1943] [#668] shard/test: Add dontRelease options Most of the time we would like to close shard with minor exceptions. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/gc_test.go | 8 -------- pkg/local_object_storage/shard/range_test.go | 1 - pkg/local_object_storage/shard/shard_test.go | 9 ++++++--- pkg/local_object_storage/shard/shutdown_test.go | 3 +-- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index d6a7e39e2..4f19db878 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -27,10 +27,6 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { sh := newCustomShard(t, false, shardOptions{metaOptions: []meta.Option{meta.WithEpochState(epoch)}}) - t.Cleanup(func() { - releaseShard(sh, t) - }) - cnr := cidtest.ID() var objExpirationAttr objectSDK.Attribute @@ -124,10 +120,6 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { sh := newCustomShard(t, false, shardOptions{metaOptions: []meta.Option{meta.WithEpochState(epoch)}}) - t.Cleanup(func() { - releaseShard(sh, t) - }) - lock := testutil.GenerateObjectWithCID(cnr) lock.SetType(objectSDK.TypeLock) lock.SetAttributes(lockExpirationAttr) diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 77a20966a..a8bc83307 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -97,7 +97,6 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { }), }, }) - defer releaseShard(sh, t) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 33560407e..eb2aba580 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -32,15 +32,14 @@ func (s epochState) CurrentEpoch() uint64 { type shardOptions struct { rootPath string + dontRelease bool wcOpts writecacheconfig.Options bsOpts []blobstor.Option metaOptions []meta.Option } func newShard(t testing.TB, enableWriteCache bool) *Shard { - sh := newCustomShard(t, enableWriteCache, shardOptions{}) - t.Cleanup(func() { releaseShard(sh, t) }) - return sh + return newCustomShard(t, enableWriteCache, shardOptions{}) } func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard { @@ -121,6 +120,10 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard require.NoError(t, sh.Open()) require.NoError(t, sh.Init(context.Background())) + if !o.dontRelease { + t.Cleanup(func() { releaseShard(sh, t) }) + } + return sh } diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 6cfa5fb6b..163c3a4ae 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -43,7 +43,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { }, } - sh := newCustomShard(t, true, shardOptions{rootPath: dir, wcOpts: wcOpts}) + sh := newCustomShard(t, true, shardOptions{dontRelease: true, rootPath: dir, wcOpts: wcOpts}) var errG errgroup.Group for i := range objects { @@ -59,7 +59,6 @@ func TestWriteCacheObjectLoss(t *testing.T) { require.NoError(t, sh.Close()) sh = newCustomShard(t, true, shardOptions{rootPath: dir, wcOpts: wcOpts}) - defer releaseShard(sh, t) var getPrm GetPrm From 36759f843418c6728897701de99b76feda78ef76 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Sep 2023 18:03:04 +0300 Subject: [PATCH 0582/1943] [#668] shard/test: Properly check event processing See https://git.frostfs.info/TrueCloudLab/frostfs-node/actions/runs/1594/jobs/2 Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/gc.go | 46 +++++++++++--------- pkg/local_object_storage/shard/gc_test.go | 32 ++++++++------ pkg/local_object_storage/shard/shard_test.go | 3 ++ 3 files changed, 47 insertions(+), 34 deletions(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 38baccd4b..13ab39ae0 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -160,33 +160,37 @@ func (gc *gc) listenEvents(ctx context.Context) { return } - v, ok := gc.mEventHandler[event.typ()] - if !ok { - continue - } + gc.handleEvent(ctx, event) + } +} - v.cancelFunc() - v.prevGroup.Wait() +func (gc *gc) handleEvent(ctx context.Context, event Event) { + v, ok := gc.mEventHandler[event.typ()] + if !ok { + return + } - var runCtx context.Context - runCtx, v.cancelFunc = context.WithCancel(ctx) + v.cancelFunc() + v.prevGroup.Wait() - v.prevGroup.Add(len(v.handlers)) + var runCtx context.Context + runCtx, v.cancelFunc = context.WithCancel(ctx) - for i := range v.handlers { - h := v.handlers[i] + v.prevGroup.Add(len(v.handlers)) - err := gc.workerPool.Submit(func() { - defer v.prevGroup.Done() - h(runCtx, event) - }) - if err != nil { - gc.log.Warn(logs.ShardCouldNotSubmitGCJobToWorkerPool, - zap.String("error", err.Error()), - ) + for i := range v.handlers { + h := v.handlers[i] - v.prevGroup.Done() - } + err := gc.workerPool.Submit(func() { + defer v.prevGroup.Done() + h(runCtx, event) + }) + if err != nil { + gc.log.Warn(logs.ShardCouldNotSubmitGCJobToWorkerPool, + zap.String("error", err.Error()), + ) + + v.prevGroup.Done() } } } diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 4f19db878..8b535200d 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -4,12 +4,12 @@ import ( "context" "errors" "testing" - "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -25,7 +25,12 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { Value: 100, } - sh := newCustomShard(t, false, shardOptions{metaOptions: []meta.Option{meta.WithEpochState(epoch)}}) + sh := newCustomShard(t, false, shardOptions{ + metaOptions: []meta.Option{meta.WithEpochState(epoch)}, + additionalShardOptions: []Option{WithGCWorkerPoolInitializer(func(int) util.WorkerPool { + return util.NewPseudoWorkerPool() // synchronous event processing + })}, + }) cnr := cidtest.ID() @@ -60,14 +65,12 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { require.NoError(t, err) epoch.Value = 105 - sh.NotificationChannel() <- EventNewEpoch(epoch.Value) + sh.gc.handleEvent(context.Background(), EventNewEpoch(epoch.Value)) var getPrm GetPrm getPrm.SetAddress(objectCore.AddressOf(obj)) - require.Eventually(t, func() bool { - _, err = sh.Get(context.Background(), getPrm) - return client.IsErrObjectNotFound(err) - }, 3*time.Second, 1*time.Second, "expired object must be deleted") + _, err = sh.Get(context.Background(), getPrm) + require.True(t, client.IsErrObjectNotFound(err), "expired object must be deleted") } func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { @@ -118,7 +121,12 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { linkID, _ := link.ID() - sh := newCustomShard(t, false, shardOptions{metaOptions: []meta.Option{meta.WithEpochState(epoch)}}) + sh := newCustomShard(t, false, shardOptions{ + metaOptions: []meta.Option{meta.WithEpochState(epoch)}, + additionalShardOptions: []Option{WithGCWorkerPoolInitializer(func(int) util.WorkerPool { + return util.NewPseudoWorkerPool() // synchronous event processing + })}, + }) lock := testutil.GenerateObjectWithCID(cnr) lock.SetType(objectSDK.TypeLock) @@ -152,10 +160,8 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { require.True(t, errors.As(err, &splitInfoError), "split info must be provided") epoch.Value = 105 - sh.NotificationChannel() <- EventNewEpoch(epoch.Value) + sh.gc.handleEvent(context.Background(), EventNewEpoch(epoch.Value)) - require.Eventually(t, func() bool { - _, err = sh.Get(context.Background(), getPrm) - return client.IsErrObjectNotFound(err) - }, 3*time.Second, 1*time.Second, "expired complex object must be deleted on epoch after lock expires") + _, err = sh.Get(context.Background(), getPrm) + require.True(t, client.IsErrObjectNotFound(err), "expired complex object must be deleted on epoch after lock expires") } diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index eb2aba580..669ac287c 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -36,6 +36,8 @@ type shardOptions struct { wcOpts writecacheconfig.Options bsOpts []blobstor.Option metaOptions []meta.Option + + additionalShardOptions []Option } func newShard(t testing.TB, enableWriteCache bool) *Shard { @@ -114,6 +116,7 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard }), WithGCRemoverSleepInterval(100 * time.Millisecond), } + opts = append(opts, o.additionalShardOptions...) sh = New(opts...) From a716db99dbcb64fdc947ddfc1366d80a87dd1f8d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Sep 2023 18:08:23 +0300 Subject: [PATCH 0583/1943] [#668] shard/test: Do not alter rootPath option Supposedly, this was added to allow creating 2 different shards without subtest. Now we use t.TempDir() everywhere, so this should not be a problem. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/shard_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 669ac287c..5dcccd9c9 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -54,7 +54,6 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard var sh *Shard if enableWriteCache { - o.rootPath = filepath.Join(o.rootPath, "wc") switch o.wcOpts.Type { case writecacheconfig.TypeBBolt: o.wcOpts.BBoltOptions = append( @@ -65,8 +64,6 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard []writecachebadger.Option{writecachebadger.WithPath(filepath.Join(o.rootPath, "wcache"))}, o.wcOpts.BadgerOptions...) } - } else { - o.rootPath = filepath.Join(o.rootPath, "nowc") } if o.bsOpts == nil { From 8a81af5a3b1b23381ac09983b38b02f476788072 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Thu, 31 Aug 2023 19:26:47 +0300 Subject: [PATCH 0584/1943] [#653] Add context parameter to Open functions Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-lens/internal/meta/root.go | 2 +- cmd/frostfs-node/config.go | 6 ++--- cmd/frostfs-node/main.go | 4 ++-- .../blobstor/blobstor_test.go | 8 +++---- pkg/local_object_storage/blobstor/control.go | 8 ++++++- .../blobstor/exists_test.go | 2 +- .../blobstor/iterate_test.go | 2 +- pkg/local_object_storage/blobstor/mode.go | 3 ++- pkg/local_object_storage/engine/control.go | 24 +++++++++---------- .../engine/control_test.go | 14 +++++------ .../engine/delete_test.go | 2 +- .../engine/engine_test.go | 8 +++---- pkg/local_object_storage/engine/error_test.go | 4 ++-- .../engine/evacuate_test.go | 2 +- pkg/local_object_storage/engine/head_test.go | 2 +- .../engine/inhume_test.go | 4 ++-- pkg/local_object_storage/engine/list_test.go | 4 ++-- pkg/local_object_storage/engine/lock_test.go | 12 +++++----- pkg/local_object_storage/engine/shards.go | 9 +++---- .../engine/shards_test.go | 3 ++- .../internal/storagetest/storage.go | 19 ++++++++------- pkg/local_object_storage/metabase/control.go | 3 ++- pkg/local_object_storage/metabase/db_test.go | 3 ++- pkg/local_object_storage/metabase/mode.go | 5 ++-- .../metabase/version_test.go | 15 ++++++------ .../pilorama/bench_test.go | 2 +- pkg/local_object_storage/pilorama/boltdb.go | 4 ++-- pkg/local_object_storage/pilorama/forest.go | 2 +- .../pilorama/forest_test.go | 4 ++-- .../pilorama/interface.go | 2 +- pkg/local_object_storage/shard/control.go | 10 ++++---- .../shard/control_test.go | 14 +++++------ .../shard/gc_internal_test.go | 2 +- pkg/local_object_storage/shard/id.go | 6 +++-- pkg/local_object_storage/shard/lock_test.go | 2 +- .../shard/metrics_test.go | 2 +- pkg/local_object_storage/shard/reload_test.go | 2 +- pkg/local_object_storage/shard/shard_test.go | 2 +- .../writecache/benchmark/writecache_test.go | 2 +- .../writecache/writecache.go | 2 +- .../writecachebadger/cachebadger.go | 3 ++- .../writecache/writecachebbolt/cachebbolt.go | 3 ++- .../writecache/writecachetest/flush.go | 6 ++--- 43 files changed, 129 insertions(+), 109 deletions(-) diff --git a/cmd/frostfs-lens/internal/meta/root.go b/cmd/frostfs-lens/internal/meta/root.go index 9f35c26c2..a59574b6c 100644 --- a/cmd/frostfs-lens/internal/meta/root.go +++ b/cmd/frostfs-lens/internal/meta/root.go @@ -43,7 +43,7 @@ func openMeta(cmd *cobra.Command) *meta.DB { }), meta.WithEpochState(epochState{}), ) - common.ExitOnErr(cmd, common.Errf("could not open metabase: %w", db.Open(true))) + common.ExitOnErr(cmd, common.Errf("could not open metabase: %w", db.Open(cmd.Context(), true))) return db } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 8e103b527..8286bc7d7 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -901,7 +901,7 @@ func (c *cfg) LocalAddress() network.AddressGroup { return c.localAddr } -func initLocalStorage(c *cfg) { +func initLocalStorage(ctx context.Context, c *cfg) { ls := engine.New(c.engineOpts()...) addNewEpochAsyncNotificationHandler(c, func(ev event.Event) { @@ -914,7 +914,7 @@ func initLocalStorage(c *cfg) { var shardsAttached int for _, optsWithMeta := range c.shardOpts() { - id, err := ls.AddShard(append(optsWithMeta.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))...) + id, err := ls.AddShard(ctx, append(optsWithMeta.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))...) if err != nil { c.log.Error(logs.FrostFSNodeFailedToAttachShardToEngine, zap.Error(err)) } else { @@ -931,7 +931,7 @@ func initLocalStorage(c *cfg) { c.onShutdown(func() { c.log.Info(logs.FrostFSNodeClosingComponentsOfTheStorageEngine) - err := ls.Close() + err := ls.Close(context.Background()) if err != nil { c.log.Info(logs.FrostFSNodeStorageEngineClosingFailure, zap.String("error", err.Error()), diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index bf872da03..88032ebdb 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -91,10 +91,10 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) }) - initLocalStorage(c) + initLocalStorage(ctx, c) initAndLog(c, "storage engine", func(c *cfg) { - fatalOnErr(c.cfgObject.cfgLocalStorage.localStorage.Open()) + fatalOnErr(c.cfgObject.cfgLocalStorage.localStorage.Open(ctx)) fatalOnErr(c.cfgObject.cfgLocalStorage.localStorage.Init(ctx)) }) diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index 878452069..f1d567da7 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -51,7 +51,7 @@ func TestCompression(t *testing.T) { bs := New( WithCompressObjects(compress), WithStorages(defaultStorages(dir, smallSizeLimit))) - require.NoError(t, bs.Open(false)) + require.NoError(t, bs.Open(context.Background(), false)) require.NoError(t, bs.Init()) return bs } @@ -126,7 +126,7 @@ func TestBlobstor_needsCompression(t *testing.T) { Storage: fstree.New(fstree.WithPath(dir)), }, })) - require.NoError(t, bs.Open(false)) + require.NoError(t, bs.Open(context.Background(), false)) require.NoError(t, bs.Init()) return bs } @@ -188,7 +188,7 @@ func TestConcurrentPut(t *testing.T) { blobStor := New( WithStorages(defaultStorages(dir, smallSizeLimit))) - require.NoError(t, blobStor.Open(false)) + require.NoError(t, blobStor.Open(context.Background(), false)) require.NoError(t, blobStor.Init()) testGet := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { @@ -268,7 +268,7 @@ func TestConcurrentDelete(t *testing.T) { blobStor := New( WithStorages(defaultStorages(dir, smallSizeLimit))) - require.NoError(t, blobStor.Open(false)) + require.NoError(t, blobStor.Open(context.Background(), false)) require.NoError(t, blobStor.Init()) testPut := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 6b439dcf0..4b8a36de8 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -1,6 +1,7 @@ package blobstor import ( + "context" "errors" "fmt" @@ -9,10 +10,15 @@ import ( ) // Open opens BlobStor. -func (b *BlobStor) Open(readOnly bool) error { +func (b *BlobStor) Open(ctx context.Context, readOnly bool) error { b.log.Debug(logs.BlobstorOpening) for i := range b.storage { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } err := b.storage[i].Storage.Open(readOnly) if err != nil { return err diff --git a/pkg/local_object_storage/blobstor/exists_test.go b/pkg/local_object_storage/blobstor/exists_test.go index 7a3dcd617..367b63af1 100644 --- a/pkg/local_object_storage/blobstor/exists_test.go +++ b/pkg/local_object_storage/blobstor/exists_test.go @@ -20,7 +20,7 @@ func TestExists(t *testing.T) { b := New(WithStorages(storages)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(context.Background(), false)) require.NoError(t, b.Init()) objects := []*objectSDK.Object{ diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index c35869655..ef3fda991 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -26,7 +26,7 @@ func TestIterateObjects(t *testing.T) { defer os.RemoveAll(p) // open Blobstor - require.NoError(t, blobStor.Open(false)) + require.NoError(t, blobStor.Open(context.Background(), false)) // initialize Blobstor require.NoError(t, blobStor.Init()) diff --git a/pkg/local_object_storage/blobstor/mode.go b/pkg/local_object_storage/blobstor/mode.go index e6d0edc03..2f4473bd8 100644 --- a/pkg/local_object_storage/blobstor/mode.go +++ b/pkg/local_object_storage/blobstor/mode.go @@ -1,6 +1,7 @@ package blobstor import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -21,7 +22,7 @@ func (b *BlobStor) SetMode(m mode.Mode) error { err := b.Close() if err == nil { - if err = b.Open(m.ReadOnly()); err == nil { + if err = b.Open(context.TODO(), m.ReadOnly()); err == nil { err = b.Init() } } diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index bd166b3ff..d97148f73 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -21,11 +21,11 @@ type shardInitError struct { } // Open opens all StorageEngine's components. -func (e *StorageEngine) Open() error { - return e.open() +func (e *StorageEngine) Open(ctx context.Context) error { + return e.open(ctx) } -func (e *StorageEngine) open() error { +func (e *StorageEngine) open(ctx context.Context) error { e.mtx.Lock() defer e.mtx.Unlock() @@ -36,7 +36,7 @@ func (e *StorageEngine) open() error { wg.Add(1) go func(id string, sh *shard.Shard) { defer wg.Done() - if err := sh.Open(); err != nil { + if err := sh.Open(ctx); err != nil { errCh <- shardInitError{ err: err, id: id, @@ -148,10 +148,10 @@ var errClosed = errors.New("storage engine is closed") // After the call, all the next ones will fail. // // The method MUST only be called when the application exits. -func (e *StorageEngine) Close() error { +func (e *StorageEngine) Close(ctx context.Context) error { close(e.closeCh) defer e.wg.Wait() - return e.setBlockExecErr(errClosed) + return e.setBlockExecErr(ctx, errClosed) } // closes all shards. Never returns an error, shard errors are logged. @@ -197,7 +197,7 @@ func (e *StorageEngine) execIfNotBlocked(op func() error) error { // - otherwise, resumes execution. If exec was blocked, calls open method. // // Can be called concurrently with exec. In this case it waits for all executions to complete. -func (e *StorageEngine) setBlockExecErr(err error) error { +func (e *StorageEngine) setBlockExecErr(ctx context.Context, err error) error { e.blockExec.mtx.Lock() defer e.blockExec.mtx.Unlock() @@ -212,7 +212,7 @@ func (e *StorageEngine) setBlockExecErr(err error) error { if err == nil { if prevErr != nil { // block -> ok - return e.open() + return e.open(ctx) } } else if prevErr == nil { // ok -> block return e.close(errors.Is(err, errClosed)) @@ -235,7 +235,7 @@ func (e *StorageEngine) setBlockExecErr(err error) error { // Note: technically passing nil error will resume the execution, otherwise, it is recommended to call ResumeExecution // for this. func (e *StorageEngine) BlockExecution(err error) error { - return e.setBlockExecErr(err) + return e.setBlockExecErr(context.Background(), err) } // ResumeExecution resumes the execution of any data-related operation. @@ -247,7 +247,7 @@ func (e *StorageEngine) BlockExecution(err error) error { // // Must not be called concurrently with either Open or Init. func (e *StorageEngine) ResumeExecution() error { - return e.setBlockExecErr(nil) + return e.setBlockExecErr(context.Background(), nil) } type ReConfiguration struct { @@ -334,14 +334,14 @@ loop: } for _, newID := range shardsToAdd { - sh, err := e.createShard(rcfg.shards[newID]) + sh, err := e.createShard(ctx, rcfg.shards[newID]) if err != nil { return fmt.Errorf("could not add new shard with '%s' metabase path: %w", newID, err) } idStr := sh.ID().String() - err = sh.Open() + err = sh.Open(ctx) if err == nil { err = sh.Init(ctx) } diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index eb601122b..0c5ff94da 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -126,7 +126,7 @@ func testEngineFailInitAndReload(t *testing.T, errOnAdd bool, opts []shard.Optio var configID string e := New() - _, err := e.AddShard(opts...) + _, err := e.AddShard(context.Background(), opts...) if errOnAdd { require.Error(t, err) // This branch is only taken when we cannot update shard ID in the metabase. @@ -144,7 +144,7 @@ func testEngineFailInitAndReload(t *testing.T, errOnAdd bool, opts []shard.Optio configID = calculateShardID(e.shards[id].Shard.DumpInfo()) e.mtx.RUnlock() - err = e.Open() + err = e.Open(context.Background()) if err == nil { require.Error(t, e.Init(context.Background())) } @@ -193,7 +193,7 @@ func TestExecBlocks(t *testing.T) { require.NoError(t, err) // close - require.NoError(t, e.Close()) + require.NoError(t, e.Close(context.Background())) // try exec after close _, err = Head(context.Background(), e, addr) @@ -209,13 +209,13 @@ func TestPersistentShardID(t *testing.T) { te := newEngineWithErrorThreshold(t, dir, 1) checkShardState(t, te.ng, te.shards[0].id, 0, mode.ReadWrite) - require.NoError(t, te.ng.Close()) + require.NoError(t, te.ng.Close(context.Background())) newTe := newEngineWithErrorThreshold(t, dir, 1) for i := 0; i < len(newTe.shards); i++ { require.Equal(t, te.shards[i].id, newTe.shards[i].id) } - require.NoError(t, newTe.ng.Close()) + require.NoError(t, newTe.ng.Close(context.Background())) p1 := newTe.ng.shards[te.shards[0].id.String()].Shard.DumpInfo().MetaBaseInfo.Path p2 := newTe.ng.shards[te.shards[1].id.String()].Shard.DumpInfo().MetaBaseInfo.Path @@ -227,7 +227,7 @@ func TestPersistentShardID(t *testing.T) { newTe = newEngineWithErrorThreshold(t, dir, 1) require.Equal(t, te.shards[1].id, newTe.shards[0].id) require.Equal(t, te.shards[0].id, newTe.shards[1].id) - require.NoError(t, newTe.ng.Close()) + require.NoError(t, newTe.ng.Close(context.Background())) } @@ -313,7 +313,7 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str require.Equal(t, num, len(e.shards)) require.Equal(t, num, len(e.shardPools)) - require.NoError(t, e.Open()) + require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) return e, currShards diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 57b9ec2e3..8e94732ae 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -54,7 +54,7 @@ func TestDeleteBigObject(t *testing.T) { e := testNewEngine(t).setInitializedShards(t, s1, s2, s3).engine e.log = test.NewLogger(t, true) - defer e.Close() + defer e.Close(context.Background()) for i := range children { require.NoError(t, Put(context.Background(), e, children[i])) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index a8fd9eeee..934bea8bb 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -50,7 +50,7 @@ func benchmarkExists(b *testing.B, shardNum int) { e := testNewEngine(b).setInitializedShards(b, shards...).engine b.Cleanup(func() { - _ = e.Close() + _ = e.Close(context.Background()) _ = os.RemoveAll(b.Name()) }) @@ -119,7 +119,7 @@ func (te *testEngineWrapper) setShardsNum(t testing.TB, num int) *testEngineWrap func (te *testEngineWrapper) setShardsNumOpts(t testing.TB, num int, shardOpts func(id int) []shard.Option) *testEngineWrapper { for i := 0; i < num; i++ { opts := shardOpts(i) - id, err := te.engine.AddShard(opts...) + id, err := te.engine.AddShard(context.Background(), opts...) require.NoError(t, err) te.shardIDs = append(te.shardIDs, id) } @@ -130,7 +130,7 @@ func (te *testEngineWrapper) setShardsNumAdditionalOpts(t testing.TB, num int, s for i := 0; i < num; i++ { defaultOpts := testDefaultShardOptions(t, i) opts := append(defaultOpts, shardOpts(i)...) - id, err := te.engine.AddShard(opts...) + id, err := te.engine.AddShard(context.Background(), opts...) require.NoError(t, err) te.shardIDs = append(te.shardIDs, id) } @@ -190,7 +190,7 @@ func testNewShard(t testing.TB, id int) *shard.Shard { shardOpts := append([]shard.Option{shard.WithID(sid)}, testDefaultShardOptions(t, id)...) s := shard.New(shardOpts...) - require.NoError(t, s.Open()) + require.NoError(t, s.Open(context.Background())) require.NoError(t, s.Init(context.Background())) return s diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 750918880..0a48f8188 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -68,7 +68,7 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) } }) e := te.engine - require.NoError(t, e.Open()) + require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) for i, id := range te.shardIDs { @@ -192,7 +192,7 @@ func TestBlobstorFailback(t *testing.T) { } checkShardState(t, te.ng, te.shards[0].id, 0, mode.ReadWrite) - require.NoError(t, te.ng.Close()) + require.NoError(t, te.ng.Close(context.Background())) p1 := te.ng.shards[te.shards[0].id.String()].Shard.DumpInfo().BlobStorInfo.SubStorages[1].Path p2 := te.ng.shards[te.shards[1].id.String()].Shard.DumpInfo().BlobStorInfo.SubStorages[1].Path diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 6030ba494..c0c05d661 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -44,7 +44,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng } }) e, ids := te.engine, te.shardIDs - require.NoError(t, e.Open()) + require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) objects := make([]*objectSDK.Object, 0, objPerShard*len(ids)) diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go index e212ffa32..5c123d617 100644 --- a/pkg/local_object_storage/engine/head_test.go +++ b/pkg/local_object_storage/engine/head_test.go @@ -43,7 +43,7 @@ func TestHeadRaw(t *testing.T) { s2 := testNewShard(t, 2) e := testNewEngine(t).setInitializedShards(t, s1, s2).engine - defer e.Close() + defer e.Close(context.Background()) var putPrmLeft shard.PutPrm putPrmLeft.SetObject(child) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index a5f242071..8fff6280c 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -37,7 +37,7 @@ func TestStorageEngine_Inhume(t *testing.T) { t.Run("delete small object", func(t *testing.T) { e := testNewEngine(t).setShardsNum(t, 1).engine - defer e.Close() + defer e.Close(context.Background()) err := Put(context.Background(), e, parent) require.NoError(t, err) @@ -58,7 +58,7 @@ func TestStorageEngine_Inhume(t *testing.T) { s2 := testNewShard(t, 2) e := testNewEngine(t).setInitializedShards(t, s1, s2).engine - defer e.Close() + defer e.Close(context.Background()) var putChild shard.PutPrm putChild.SetObject(child) diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 8234c3c28..eef25d209 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -76,11 +76,11 @@ func TestListWithCursor(t *testing.T) { meta.WithEpochState(epochState{}), )} }).engine - require.NoError(t, e.Open()) + require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) t.Cleanup(func() { - e.Close() + e.Close(context.Background()) }) expected := make([]object.AddressWithType, 0, tt.objectNum) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 906a867e2..d5c080222 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -59,11 +59,11 @@ func TestLockUserScenario(t *testing.T) { } }) e := testEngine.engine - require.NoError(t, e.Open()) + require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) t.Cleanup(func() { - _ = e.Close() + _ = e.Close(context.Background()) }) lockerID := oidtest.ID() @@ -167,11 +167,11 @@ func TestLockExpiration(t *testing.T) { } }) e := testEngine.engine - require.NoError(t, e.Open()) + require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) t.Cleanup(func() { - _ = e.Close() + _ = e.Close(context.Background()) }) const lockerExpiresAfter = 13 @@ -247,10 +247,10 @@ func TestLockForceRemoval(t *testing.T) { shard.WithDeletedLockCallback(e.processDeletedLocks), } }).engine - require.NoError(t, e.Open()) + require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) t.Cleanup(func() { - _ = e.Close() + _ = e.Close(context.Background()) }) cnr := cidtest.ID() diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index f12a63e91..4b9d8752a 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -1,6 +1,7 @@ package engine import ( + "context" "fmt" "sync/atomic" @@ -77,8 +78,8 @@ func (m *metricsWithID) DeleteShardMetrics() { // // Returns any error encountered that did not allow adding a shard. // Otherwise returns the ID of the added shard. -func (e *StorageEngine) AddShard(opts ...shard.Option) (*shard.ID, error) { - sh, err := e.createShard(opts) +func (e *StorageEngine) AddShard(ctx context.Context, opts ...shard.Option) (*shard.ID, error) { + sh, err := e.createShard(ctx, opts) if err != nil { return nil, fmt.Errorf("could not create a shard: %w", err) } @@ -95,7 +96,7 @@ func (e *StorageEngine) AddShard(opts ...shard.Option) (*shard.ID, error) { return sh.ID(), nil } -func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { +func (e *StorageEngine) createShard(ctx context.Context, opts []shard.Option) (*shard.Shard, error) { id, err := generateShardID() if err != nil { return nil, fmt.Errorf("could not generate shard ID: %w", err) @@ -111,7 +112,7 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { shard.WithReportErrorFunc(e.reportShardErrorBackground), )...) - if err := sh.UpdateID(); err != nil { + if err := sh.UpdateID(ctx); err != nil { return nil, fmt.Errorf("could not update shard ID: %w", err) } diff --git a/pkg/local_object_storage/engine/shards_test.go b/pkg/local_object_storage/engine/shards_test.go index 3631e33ed..e13017e24 100644 --- a/pkg/local_object_storage/engine/shards_test.go +++ b/pkg/local_object_storage/engine/shards_test.go @@ -1,6 +1,7 @@ package engine import ( + "context" "testing" "github.com/stretchr/testify/require" @@ -12,7 +13,7 @@ func TestRemoveShard(t *testing.T) { te := testNewEngine(t).setShardsNum(t, numOfShards) e, ids := te.engine, te.shardIDs t.Cleanup(func() { - e.Close() + e.Close(context.Background()) }) require.Equal(t, numOfShards, len(e.shardPools)) diff --git a/pkg/local_object_storage/internal/storagetest/storage.go b/pkg/local_object_storage/internal/storagetest/storage.go index 74400a981..ec60a2d0e 100644 --- a/pkg/local_object_storage/internal/storagetest/storage.go +++ b/pkg/local_object_storage/internal/storagetest/storage.go @@ -1,6 +1,7 @@ package storagetest import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -9,7 +10,7 @@ import ( // Component represents single storage component. type Component interface { - Open(bool) error + Open(context.Context, bool) error SetMode(mode.Mode) error Init() error Close() error @@ -57,18 +58,18 @@ func TestCloseAfterOpen(t *testing.T, cons Constructor) { t.Run("RW", func(t *testing.T) { // Use-case: irrecoverable error on some components, close everything. s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(context.Background(), false)) require.NoError(t, s.Close()) }) t.Run("RO", func(t *testing.T) { // Use-case: irrecoverable error on some components, close everything. // Open in read-only must be done after the db is here. s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(context.Background(), false)) require.NoError(t, s.Init()) require.NoError(t, s.Close()) - require.NoError(t, s.Open(true)) + require.NoError(t, s.Open(context.Background(), true)) require.NoError(t, s.Close()) }) } @@ -77,7 +78,7 @@ func TestCloseAfterOpen(t *testing.T, cons Constructor) { func TestCloseTwice(t *testing.T, cons Constructor) { // Use-case: move to maintenance mode twice, first time failed. s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(context.Background(), false)) require.NoError(t, s.Init()) require.NoError(t, s.Close()) require.NoError(t, s.Close()) // already closed, no-op @@ -89,12 +90,12 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { // Use-case: metabase `Init` failed, // call `SetMode` on all not-yet-initialized components. s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(context.Background(), false)) require.NoError(t, s.SetMode(m)) t.Run("after open in RO", func(t *testing.T) { require.NoError(t, s.Close()) - require.NoError(t, s.Open(true)) + require.NoError(t, s.Open(context.Background(), true)) require.NoError(t, s.SetMode(m)) }) @@ -103,7 +104,7 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { t.Run("after init", func(t *testing.T) { s := cons(t) // Use-case: notmal node operation. - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(context.Background(), false)) require.NoError(t, s.Init()) require.NoError(t, s.SetMode(m)) require.NoError(t, s.Close()) @@ -113,7 +114,7 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { func TestModeTransition(t *testing.T, cons Constructor, from, to mode.Mode) { // Use-case: normal node operation. s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(context.Background(), false)) require.NoError(t, s.Init()) require.NoError(t, s.SetMode(from)) require.NoError(t, s.SetMode(to)) diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index d0a9c4723..3f155eeb5 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -1,6 +1,7 @@ package meta import ( + "context" "errors" "fmt" "path/filepath" @@ -21,7 +22,7 @@ var ErrDegradedMode = logicerr.New("metabase is in a degraded mode") var ErrReadOnlyMode = logicerr.New("metabase is in a read-only mode") // Open boltDB instance for metabase. -func (db *DB) Open(readOnly bool) error { +func (db *DB) Open(_ context.Context, readOnly bool) error { err := util.MkdirAllX(filepath.Dir(db.info.Path), db.info.Permission) if err != nil { return fmt.Errorf("can't create dir %s for metabase: %w", db.info.Path, err) diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index 6169dd81e..bc5015b60 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "os" "path/filepath" "strconv" @@ -49,7 +50,7 @@ func newDB(t testing.TB, opts ...meta.Option) *meta.DB { }, opts...)..., ) - require.NoError(t, bdb.Open(false)) + require.NoError(t, bdb.Open(context.Background(), false)) require.NoError(t, bdb.Init()) t.Cleanup(func() { diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index 28beca8f3..a18095f3e 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -1,6 +1,7 @@ package meta import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -27,9 +28,9 @@ func (db *DB) SetMode(m mode.Mode) error { case m.NoMetabase(): db.boltDB = nil case m.ReadOnly(): - err = db.Open(true) + err = db.Open(context.TODO(), true) default: - err = db.Open(false) + err = db.Open(context.TODO(), false) } if err == nil && !m.NoMetabase() && !m.ReadOnly() { err = db.Init() diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index 70ded67a4..6f011c246 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -1,6 +1,7 @@ package meta import ( + "context" "encoding/binary" "errors" "fmt" @@ -42,13 +43,13 @@ func TestVersion(t *testing.T) { } t.Run("simple", func(t *testing.T) { db := newDB(t) - require.NoError(t, db.Open(false)) + require.NoError(t, db.Open(context.Background(), false)) require.NoError(t, db.Init()) check(t, db) require.NoError(t, db.Close()) t.Run("reopen", func(t *testing.T) { - require.NoError(t, db.Open(false)) + require.NoError(t, db.Open(context.Background(), false)) require.NoError(t, db.Init()) check(t, db) require.NoError(t, db.Close()) @@ -56,29 +57,29 @@ func TestVersion(t *testing.T) { }) t.Run("old data", func(t *testing.T) { db := newDB(t) - require.NoError(t, db.Open(false)) + require.NoError(t, db.Open(context.Background(), false)) require.NoError(t, db.WriteShardID([]byte{1, 2, 3, 4})) require.NoError(t, db.Close()) - require.NoError(t, db.Open(false)) + require.NoError(t, db.Open(context.Background(), false)) require.NoError(t, db.Init()) check(t, db) require.NoError(t, db.Close()) }) t.Run("invalid version", func(t *testing.T) { db := newDB(t) - require.NoError(t, db.Open(false)) + require.NoError(t, db.Open(context.Background(), false)) require.NoError(t, db.boltDB.Update(func(tx *bbolt.Tx) error { return updateVersion(tx, version+1) })) require.NoError(t, db.Close()) - require.NoError(t, db.Open(false)) + require.NoError(t, db.Open(context.Background(), false)) require.Error(t, db.Init()) require.NoError(t, db.Close()) t.Run("reset", func(t *testing.T) { - require.NoError(t, db.Open(false)) + require.NoError(t, db.Open(context.Background(), false)) require.NoError(t, db.Reset()) check(t, db) require.NoError(t, db.Close()) diff --git a/pkg/local_object_storage/pilorama/bench_test.go b/pkg/local_object_storage/pilorama/bench_test.go index e729b9ea6..3d5ff1a7c 100644 --- a/pkg/local_object_storage/pilorama/bench_test.go +++ b/pkg/local_object_storage/pilorama/bench_test.go @@ -26,7 +26,7 @@ func BenchmarkCreate(b *testing.B) { f := NewBoltForest( WithPath(filepath.Join(tmpDir, "test.db")), WithMaxBatchSize(runtime.GOMAXPROCS(0))) - require.NoError(b, f.Open(false)) + require.NoError(b, f.Open(context.Background(), false)) require.NoError(b, f.Init()) b.Cleanup(func() { require.NoError(b, f.Close()) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index c9a9f039e..2689e345a 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -99,7 +99,7 @@ func (t *boltForest) SetMode(m mode.Mode) error { err := t.Close() if err == nil && !m.NoMetabase() { - if err = t.Open(m.ReadOnly()); err == nil { + if err = t.Open(context.TODO(), m.ReadOnly()); err == nil { err = t.Init() } } @@ -111,7 +111,7 @@ func (t *boltForest) SetMode(m mode.Mode) error { t.metrics.SetMode(m) return nil } -func (t *boltForest) Open(readOnly bool) error { +func (t *boltForest) Open(_ context.Context, readOnly bool) error { err := util.MkdirAllX(filepath.Dir(t.path), t.perm) if err != nil { return metaerr.Wrap(fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 8fb519128..e5612d2b9 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -110,7 +110,7 @@ func (f *memoryForest) Init() error { return nil } -func (f *memoryForest) Open(bool) error { +func (f *memoryForest) Open(context.Context, bool) error { return nil } func (f *memoryForest) SetMode(mode.Mode) error { diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 6f657873a..8e7fec200 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -24,7 +24,7 @@ var providers = []struct { }{ {"inmemory", func(t testing.TB, _ ...Option) Forest { f := NewMemoryForest() - require.NoError(t, f.Open(false)) + require.NoError(t, f.Open(context.Background(), false)) require.NoError(t, f.Init()) t.Cleanup(func() { require.NoError(t, f.Close()) @@ -37,7 +37,7 @@ var providers = []struct { append([]Option{ WithPath(filepath.Join(t.TempDir(), "test.db")), WithMaxBatchSize(1)}, opts...)...) - require.NoError(t, f.Open(false)) + require.NoError(t, f.Open(context.Background(), false)) require.NoError(t, f.Init()) t.Cleanup(func() { require.NoError(t, f.Close()) diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index ea171a479..e7f7eb512 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -58,7 +58,7 @@ type ForestStorage interface { // DumpInfo returns information about the pilorama. DumpInfo() Info Init() error - Open(bool) error + Open(context.Context, bool) error Close() error SetMode(m mode.Mode) error SetParentID(id string) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index a49a79043..257498b32 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -41,8 +41,10 @@ func (s *Shard) handleMetabaseFailure(stage string, err error) error { } // Open opens all Shard's components. -func (s *Shard) Open() error { - components := []interface{ Open(bool) error }{ +func (s *Shard) Open(ctx context.Context) error { + components := []interface { + Open(context.Context, bool) error + }{ s.blobStor, s.metaBase, } @@ -55,12 +57,12 @@ func (s *Shard) Open() error { } for i, component := range components { - if err := component.Open(false); err != nil { + if err := component.Open(ctx, false); err != nil { if component == s.metaBase { // We must first open all other components to avoid // opening non-existent DB in read-only mode. for j := i + 1; j < len(components); j++ { - if err := components[j].Open(false); err != nil { + if err := components[j].Open(ctx, false); err != nil { // Other components must be opened, fail. return fmt.Errorf("could not open %T: %w", components[j], err) } diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 128e5cc0c..749229cc3 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -87,7 +87,7 @@ func TestShardOpen(t *testing.T) { allowedMode.Store(int64(os.O_RDWR)) sh := newShard() - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) require.Equal(t, mode.ReadWrite, sh.GetMode()) require.NoError(t, sh.Close()) @@ -96,7 +96,7 @@ func TestShardOpen(t *testing.T) { allowedMode.Store(int64(os.O_RDONLY)) sh = newShard() - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) require.Equal(t, mode.ReadOnly, sh.GetMode()) require.Error(t, sh.SetMode(mode.ReadWrite)) @@ -107,7 +107,7 @@ func TestShardOpen(t *testing.T) { allowedMode.Store(math.MaxInt64) sh = newShard() - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) require.Equal(t, mode.DegradedReadOnly, sh.GetMode()) require.NoError(t, sh.Close()) @@ -135,7 +135,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) { WithBlobStorOptions(blobOpts...), WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))), WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta")), meta.WithEpochState(epochState{}))) - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) obj := objecttest.Object() @@ -158,7 +158,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) { WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))), WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta_new")), meta.WithEpochState(epochState{})), WithRefillMetabase(true)) - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) var getPrm GetPrm @@ -197,7 +197,7 @@ func TestRefillMetabase(t *testing.T) { ) // open Blobstor - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) // initialize Blobstor require.NoError(t, sh.Init(context.Background())) @@ -365,7 +365,7 @@ func TestRefillMetabase(t *testing.T) { ) // open Blobstor - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) // initialize Blobstor require.NoError(t, sh.Init(context.Background())) diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index c8925e01f..332cdf5be 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -76,7 +76,7 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { sh = New(opts...) sh.gcCfg.testHookRemover = func(context.Context) gcRunResult { return gcRunResult{} } - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) t.Cleanup(func() { diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index 49f9a3043..b0d95e54c 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -1,6 +1,8 @@ package shard import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/mr-tron/base58" "go.uber.org/zap" @@ -27,8 +29,8 @@ func (s *Shard) ID() *ID { } // UpdateID reads shard ID saved in the metabase and updates it if it is missing. -func (s *Shard) UpdateID() (err error) { - if err = s.metaBase.Open(false); err != nil { +func (s *Shard) UpdateID(ctx context.Context) (err error) { + if err = s.metaBase.Open(ctx, false); err != nil { return err } defer func() { diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index be9a0ba9a..ca6b0ca38 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -58,7 +58,7 @@ func TestShard_Lock(t *testing.T) { } sh = New(opts...) - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) t.Cleanup(func() { diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index ae993c79c..23721af6e 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -269,7 +269,7 @@ func shardWithMetrics(t *testing.T, path string) (*Shard, *metricsStore) { meta.WithEpochState(epochState{})), WithMetricsWriter(mm), ) - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) t.Cleanup(func() { diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index c89c0cb7e..9cfa267e8 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -52,7 +52,7 @@ func TestShardReload(t *testing.T) { pilorama.WithPath(filepath.Join(p, "pilorama")))} sh := New(opts...) - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) objects := make([]objAddr, 5) diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 5dcccd9c9..9da9eb6b8 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -117,7 +117,7 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard sh = New(opts...) - require.NoError(t, sh.Open()) + require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) if !o.dontRelease { diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index 6ae04a92a..16c6d73b2 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -82,7 +82,7 @@ func benchmarkPutPar(b *testing.B, cache writecache.Cache, size uint64) { } func benchmarkPutPrepare(b *testing.B, cache writecache.Cache) { - require.NoError(b, cache.Open(false), "opening") + require.NoError(b, cache.Open(context.Background(), false), "opening") require.NoError(b, cache.Init(), "initializing") b.Cleanup(func() { require.NoError(b, cache.Close(), "closing") diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 8c8637358..c2fdc100f 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -38,7 +38,7 @@ type Cache interface { Flush(context.Context, bool) error Init() error - Open(readOnly bool) error + Open(ctx context.Context, readOnly bool) error Close() error } diff --git a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go index 28aa7e766..6dd4755b3 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go +++ b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go @@ -1,6 +1,7 @@ package writecachebadger import ( + "context" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -83,7 +84,7 @@ func (c *cache) DumpInfo() writecache.Info { } // Open opens and initializes database. Reads object counters from the ObjectCounters instance. -func (c *cache) Open(readOnly bool) error { +func (c *cache) Open(_ context.Context, readOnly bool) error { err := c.openStore(readOnly) if err != nil { return metaerr.Wrap(err) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go index 407d1a9ce..11bdbe07b 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go @@ -1,6 +1,7 @@ package writecachebbolt import ( + "context" "os" "sync" @@ -97,7 +98,7 @@ func (c *cache) DumpInfo() writecache.Info { } // Open opens and initializes database. Reads object counters from the ObjectCounters instance. -func (c *cache) Open(readOnly bool) error { +func (c *cache) Open(_ context.Context, readOnly bool) error { err := c.openStore(readOnly) if err != nil { return metaerr.Wrap(err) diff --git a/pkg/local_object_storage/writecache/writecachetest/flush.go b/pkg/local_object_storage/writecache/writecachetest/flush.go index 05b4b23ac..6911344a2 100644 --- a/pkg/local_object_storage/writecache/writecachetest/flush.go +++ b/pkg/local_object_storage/writecache/writecachetest/flush.go @@ -110,7 +110,7 @@ func newCache[Option any]( mb := meta.New( meta.WithPath(filepath.Join(dir, "meta")), meta.WithEpochState(dummyEpoch{})) - require.NoError(t, mb.Open(false)) + require.NoError(t, mb.Open(context.Background(), false)) require.NoError(t, mb.Init()) bs := blobstor.New(blobstor.WithStorages([]blobstor.SubStorage{ @@ -121,12 +121,12 @@ func newCache[Option any]( fstree.WithDirNameLen(1)), }, })) - require.NoError(t, bs.Open(false)) + require.NoError(t, bs.Open(context.Background(), false)) require.NoError(t, bs.Init()) wc := createCacheFn(t, smallSize, mb, bs, opts...) t.Cleanup(func() { require.NoError(t, wc.Close()) }) - require.NoError(t, wc.Open(false)) + require.NoError(t, wc.Open(context.Background(), false)) require.NoError(t, wc.Init()) // First set mode for metabase and blobstor to prevent background flushes. From aa9f8dce3deae69c2f5df8aa11a76db2d578a4b0 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 6 Sep 2023 15:53:53 +0300 Subject: [PATCH 0585/1943] [#677] client: Refactor PrmAnnounceSpace/EndpointInfo/NetworkInfo usage Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/client.go | 12 ++++++------ cmd/frostfs-cli/modules/netmap/get_epoch.go | 5 +++-- cmd/frostfs-cli/modules/netmap/netinfo.go | 5 +++-- cmd/frostfs-cli/modules/netmap/nodeinfo.go | 5 +++-- cmd/frostfs-cli/modules/session/create.go | 5 +++-- cmd/frostfs-node/container.go | 6 +++--- go.mod | 2 +- go.sum | 4 ++-- 8 files changed, 24 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 7fbfd5105..ffed877c7 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -224,8 +224,8 @@ func SetEACL(ctx context.Context, prm SetEACLPrm) (res SetEACLRes, err error) { // NetworkInfoPrm groups parameters of NetworkInfo operation. type NetworkInfoPrm struct { - commonPrm - client.PrmNetworkInfo + Client *client.Client + ClientParams client.PrmNetworkInfo } // NetworkInfoRes groups the resulting values of NetworkInfo operation. @@ -242,15 +242,15 @@ func (x NetworkInfoRes) NetworkInfo() netmap.NetworkInfo { // // Returns any error which prevented the operation from completing correctly in error return. func NetworkInfo(ctx context.Context, prm NetworkInfoPrm) (res NetworkInfoRes, err error) { - res.cliRes, err = prm.cli.NetworkInfo(ctx, prm.PrmNetworkInfo) + res.cliRes, err = prm.Client.NetworkInfo(ctx, prm.ClientParams) return } // NodeInfoPrm groups parameters of NodeInfo operation. type NodeInfoPrm struct { - commonPrm - client.PrmEndpointInfo + Client *client.Client + ClientParams client.PrmEndpointInfo } // NodeInfoRes groups the resulting values of NodeInfo operation. @@ -272,7 +272,7 @@ func (x NodeInfoRes) LatestVersion() version.Version { // // Returns any error which prevented the operation from completing correctly in error return. func NodeInfo(ctx context.Context, prm NodeInfoPrm) (res NodeInfoRes, err error) { - res.cliRes, err = prm.cli.EndpointInfo(ctx, prm.PrmEndpointInfo) + res.cliRes, err = prm.Client.EndpointInfo(ctx, prm.ClientParams) return } diff --git a/cmd/frostfs-cli/modules/netmap/get_epoch.go b/cmd/frostfs-cli/modules/netmap/get_epoch.go index a9c2e1f19..1670f16c0 100644 --- a/cmd/frostfs-cli/modules/netmap/get_epoch.go +++ b/cmd/frostfs-cli/modules/netmap/get_epoch.go @@ -16,8 +16,9 @@ var getEpochCmd = &cobra.Command{ p := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, p, commonflags.RPC) - var prm internalclient.NetworkInfoPrm - prm.SetClient(cli) + prm := internalclient.NetworkInfoPrm{ + Client: cli, + } res, err := internalclient.NetworkInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) diff --git a/cmd/frostfs-cli/modules/netmap/netinfo.go b/cmd/frostfs-cli/modules/netmap/netinfo.go index 3c1839114..ceea6a890 100644 --- a/cmd/frostfs-cli/modules/netmap/netinfo.go +++ b/cmd/frostfs-cli/modules/netmap/netinfo.go @@ -20,8 +20,9 @@ var netInfoCmd = &cobra.Command{ p := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, p, commonflags.RPC) - var prm internalclient.NetworkInfoPrm - prm.SetClient(cli) + prm := internalclient.NetworkInfoPrm{ + Client: cli, + } res, err := internalclient.NetworkInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) diff --git a/cmd/frostfs-cli/modules/netmap/nodeinfo.go b/cmd/frostfs-cli/modules/netmap/nodeinfo.go index 3b2113efb..bf73286ed 100644 --- a/cmd/frostfs-cli/modules/netmap/nodeinfo.go +++ b/cmd/frostfs-cli/modules/netmap/nodeinfo.go @@ -22,8 +22,9 @@ var nodeInfoCmd = &cobra.Command{ p := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, p, commonflags.RPC) - var prm internalclient.NodeInfoPrm - prm.SetClient(cli) + prm := internalclient.NodeInfoPrm{ + Client: cli, + } res, err := internalclient.NodeInfo(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) diff --git a/cmd/frostfs-cli/modules/session/create.go b/cmd/frostfs-cli/modules/session/create.go index 53f6e8bc4..c578fc6b7 100644 --- a/cmd/frostfs-cli/modules/session/create.go +++ b/cmd/frostfs-cli/modules/session/create.go @@ -91,8 +91,9 @@ func createSession(cmd *cobra.Command, _ []string) { // // Fills ID, lifetime and session key. func CreateSession(ctx context.Context, dst *session.Object, c *client.Client, lifetime uint64) error { - var netInfoPrm internalclient.NetworkInfoPrm - netInfoPrm.SetClient(c) + netInfoPrm := internalclient.NetworkInfoPrm{ + Client: c, + } ni, err := internalclient.NetworkInfo(ctx, netInfoPrm) if err != nil { diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 5d88ff9ea..d54bf13dd 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -383,9 +383,9 @@ func (r *remoteLoadAnnounceWriter) Put(a containerSDK.SizeEstimation) error { } func (r *remoteLoadAnnounceWriter) Close(ctx context.Context) error { - var cliPrm apiClient.PrmAnnounceSpace - - cliPrm.SetValues(r.buf) + cliPrm := apiClient.PrmAnnounceSpace{ + Announcements: r.buf, + } _, err := r.client.ContainerAnnounceUsedSpace(ctx, cliPrm) return err diff --git a/go.mod b/go.mod index 6d5382b75..e9b00cd67 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904141422-5a471e5002a7 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904165523-55a1f23e7170 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 6c8b41704..1d00dcf37 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904141422-5a471e5002a7 h1:fwEDv/1INboZGbfQDxlsdbZGvHNUL/73rQe4fG8G3DA= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904141422-5a471e5002a7/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904165523-55a1f23e7170 h1:nOS6QWW8O4xDpL61TXe+NBpVXTB/fP4zy0lRh0eAWfI= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904165523-55a1f23e7170/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From f435ab1b26c93769ec6b08daf328f8d5541bf280 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 12 Sep 2023 10:05:44 +0300 Subject: [PATCH 0586/1943] [#682] go.mod: Update sdk-go Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/client/client.go | 45 ++++++++++------------- cmd/frostfs-cli/modules/session/create.go | 2 +- go.mod | 2 +- go.sum | 4 +- 4 files changed, 24 insertions(+), 29 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index ffed877c7..4cc206925 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -8,6 +8,7 @@ import ( "io" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/accounting" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -773,39 +774,33 @@ func (x HashPayloadRangesRes) HashList() [][]byte { // Returns any error which prevented the operation from completing correctly in error return. // Returns an error if number of received hashes differs with the number of requested ranges. func HashPayloadRanges(ctx context.Context, prm HashPayloadRangesPrm) (*HashPayloadRangesRes, error) { - var cliPrm client.PrmObjectHash - cliPrm.FromContainer(prm.objAddr.Container()) - cliPrm.ByID(prm.objAddr.Object()) + rngs := make([]objectSDK.Range, len(prm.rngs)) - if prm.local { - cliPrm.MarkLocal() + for i, r := range prm.rngs { + if r != nil { + rngs[i] = *r + } } - cliPrm.UseSalt(prm.salt) - - rngs := make([]uint64, 2*len(prm.rngs)) - - for i := range prm.rngs { - rngs[2*i] = prm.rngs[i].GetOffset() - rngs[2*i+1] = prm.rngs[i].GetLength() - } - - cliPrm.SetRangeList(rngs...) - + cs := checksum.SHA256 if prm.tz { - cliPrm.TillichZemorAlgo() + cs = checksum.TZ } - if prm.sessionToken != nil { - cliPrm.WithinSession(*prm.sessionToken) + cnr := prm.objAddr.Container() + obj := prm.objAddr.Object() + cliPrm := client.PrmObjectHash{ + ContainerID: &cnr, + ObjectID: &obj, + Local: prm.local, + Salt: prm.salt, + Ranges: rngs, + ChecksumType: cs, + Session: prm.sessionToken, + BearerToken: prm.bearerToken, + XHeaders: prm.xHeaders, } - if prm.bearerToken != nil { - cliPrm.WithBearerToken(*prm.bearerToken) - } - - cliPrm.WithXHeaders(prm.xHeaders...) - res, err := prm.cli.ObjectHash(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("read payload hashes via client: %w", err) diff --git a/cmd/frostfs-cli/modules/session/create.go b/cmd/frostfs-cli/modules/session/create.go index c578fc6b7..f9705c023 100644 --- a/cmd/frostfs-cli/modules/session/create.go +++ b/cmd/frostfs-cli/modules/session/create.go @@ -105,7 +105,7 @@ func CreateSession(ctx context.Context, dst *session.Object, c *client.Client, l var sessionPrm internalclient.CreateSessionPrm sessionPrm.SetClient(c) - sessionPrm.SetExp(exp) + sessionPrm.Expiration = exp sessionRes, err := internalclient.CreateSession(ctx, sessionPrm) if err != nil { diff --git a/go.mod b/go.mod index e9b00cd67..a6582d429 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904165523-55a1f23e7170 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230911122224-ac8fc6d4400c git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 1d00dcf37..a054f882a 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904165523-55a1f23e7170 h1:nOS6QWW8O4xDpL61TXe+NBpVXTB/fP4zy0lRh0eAWfI= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230904165523-55a1f23e7170/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230911122224-ac8fc6d4400c h1:gD+dj5IZx9jlniDu8TlLQdRGCd8KIOzYjjdDd1KcVdI= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230911122224-ac8fc6d4400c/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 3c76884182450c48f2cb4723cd5c61fbc401d409 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 12 Sep 2023 10:41:29 +0300 Subject: [PATCH 0587/1943] [#682] cli: Unify array of ranges type Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/client/client.go | 14 +++----------- cmd/frostfs-cli/modules/object/range.go | 7 +++---- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 4cc206925..e8869bc8c 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -739,7 +739,7 @@ type HashPayloadRangesPrm struct { tz bool - rngs []*objectSDK.Range + rngs []objectSDK.Range salt []byte } @@ -750,7 +750,7 @@ func (x *HashPayloadRangesPrm) TZ() { } // SetRanges sets a list of payload ranges to hash. -func (x *HashPayloadRangesPrm) SetRanges(rngs []*objectSDK.Range) { +func (x *HashPayloadRangesPrm) SetRanges(rngs []objectSDK.Range) { x.rngs = rngs } @@ -774,14 +774,6 @@ func (x HashPayloadRangesRes) HashList() [][]byte { // Returns any error which prevented the operation from completing correctly in error return. // Returns an error if number of received hashes differs with the number of requested ranges. func HashPayloadRanges(ctx context.Context, prm HashPayloadRangesPrm) (*HashPayloadRangesRes, error) { - rngs := make([]objectSDK.Range, len(prm.rngs)) - - for i, r := range prm.rngs { - if r != nil { - rngs[i] = *r - } - } - cs := checksum.SHA256 if prm.tz { cs = checksum.TZ @@ -794,7 +786,7 @@ func HashPayloadRanges(ctx context.Context, prm HashPayloadRangesPrm) (*HashPayl ObjectID: &obj, Local: prm.local, Salt: prm.salt, - Ranges: rngs, + Ranges: prm.rngs, ChecksumType: cs, Session: prm.sessionToken, BearerToken: prm.bearerToken, diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index 591355b95..0eee7bdba 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -84,7 +84,7 @@ func getObjectRange(cmd *cobra.Command, _ []string) { raw, _ := cmd.Flags().GetBool(rawFlag) prm.SetRawFlag(raw) prm.SetAddress(objAddr) - prm.SetRange(ranges[0]) + prm.SetRange(&ranges[0]) prm.SetPayloadWriter(out) _, err = internalclient.PayloadRange(cmd.Context(), prm) @@ -146,13 +146,13 @@ func marshalSplitInfo(cmd *cobra.Command, info *objectSDK.SplitInfo) ([]byte, er } } -func getRangeList(cmd *cobra.Command) ([]*objectSDK.Range, error) { +func getRangeList(cmd *cobra.Command) ([]objectSDK.Range, error) { v := cmd.Flag("range").Value.String() if len(v) == 0 { return nil, nil } vs := strings.Split(v, ",") - rs := make([]*objectSDK.Range, len(vs)) + rs := make([]objectSDK.Range, len(vs)) for i := range vs { before, after, found := strings.Cut(vs[i], rangeSep) if !found { @@ -176,7 +176,6 @@ func getRangeList(cmd *cobra.Command) ([]*objectSDK.Range, error) { return nil, fmt.Errorf("invalid '%s' range: uint64 overflow", vs[i]) } - rs[i] = objectSDK.NewRange() rs[i].SetOffset(offset) rs[i].SetLength(length) } From 58239d1b2c5326879f362e10a14a84b5f669485f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 12 Sep 2023 10:44:43 +0300 Subject: [PATCH 0588/1943] [#683] cli: Add context to policy parsing errors Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/container/create.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 10d53a7d8..37827b617 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -185,12 +185,12 @@ func parseContainerPolicy(cmd *cobra.Command, policyString string) (*netmap.Plac return &result, nil } - if err = result.UnmarshalJSON([]byte(policyString)); err == nil { + if err := result.UnmarshalJSON([]byte(policyString)); err == nil { common.PrintVerbose(cmd, "Parsed JSON encoded policy") return &result, nil } - return nil, errors.New("can't parse placement policy") + return nil, fmt.Errorf("can't parse placement policy: %w", err) } func parseAttributes(dst *container.Container, attributes []string) error { From c6af4a3ec80e6d2170c9f19c0235454a00405f4d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 13 Sep 2023 11:25:59 +0300 Subject: [PATCH 0589/1943] [#679] engine: Do not increase error counter on meta mismatch It was introduced in 69e1e6ca to help node determine faulty shards. However, the situation is possible in a real-life scenario: 1. Object O is evacuated from shard A to B. 2. Shard A is unmounted because of lower-level errors. 3. We now have object in meta on A and in blobstor on B. Technically we have it in meta on shard B too, but we still got the error if B goes to a degraded mode. Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/error_test.go | 2 +- pkg/local_object_storage/engine/get.go | 9 ++++++--- pkg/local_object_storage/engine/range.go | 8 +++++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 1c5815c6c..c18d191f8 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -258,6 +258,7 @@ const ( ShardMetaObjectCounterRead = "meta: object counter read" ShardMetaCantReadContainerList = "meta: can't read container list" ShardMetaCantReadContainerSize = "meta: can't read container size" + ShardMetaInfoPresentButObjectNotFound = "meta info was present, but the object is missing" ShardMetabaseFailureSwitchingMode = "metabase failure, switching mode" ShardCantMoveShardToReadonlySwitchMode = "can't move shard to readonly, switch mode" ShardCouldNotUnmarshalObject = "could not unmarshal object" diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 0a48f8188..90356104e 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -217,7 +217,7 @@ func TestBlobstorFailback(t *testing.T) { require.True(t, shard.IsErrOutOfRange(err)) } - checkShardState(t, te.ng, te.shards[0].id, 2, mode.ReadOnly) + checkShardState(t, te.ng, te.shards[0].id, 0, mode.ReadWrite) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 2f736146c..5d888238e 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -100,9 +101,11 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { if it.Object == nil { return GetRes{}, it.OutError } - if it.ShardWithMeta.Shard != nil { - e.reportShardError(it.ShardWithMeta, "meta info was present, but object is missing", - it.MetaError, zap.Stringer("address", prm.addr)) + if it.ShardWithMeta.Shard != nil && it.MetaError != nil { + e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound, + zap.Stringer("shard_id", it.ShardWithMeta.ID()), + zap.String("error", it.MetaError.Error()), + zap.Stringer("address", prm.addr)) } } diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index b3aaea6f5..fb0873ee2 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -5,6 +5,7 @@ import ( "errors" "strconv" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -113,9 +114,10 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error if it.Object == nil { return RngRes{}, it.OutError } - if it.ShardWithMeta.Shard != nil { - e.reportShardError(it.ShardWithMeta, "meta info was present, but object is missing", - it.MetaError, + if it.ShardWithMeta.Shard != nil && it.MetaError != nil { + e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound, + zap.Stringer("shard_id", it.ShardWithMeta.ID()), + zap.String("error", it.MetaError.Error()), zap.Stringer("address", prm.addr)) } } From 10570fc0350e51dd7f66fee19419d81d9227cec7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 14 Sep 2023 14:47:55 +0300 Subject: [PATCH 0590/1943] [#690] go.mod: Update contract and api-go Signed-off-by: Evgenii Stratonikov --- go.mod | 8 +- go.sum | 681 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 674 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index a6582d429..89f437216 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.20 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 - git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0 + git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230911122224-ac8fc6d4400c git.frostfs.info/TrueCloudLab/hrw v1.2.1 @@ -20,7 +20,7 @@ require ( github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.9.0 github.com/nats-io/nats.go v1.27.1 - github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc + github.com/nspcc-dev/neo-go v0.101.5-0.20230808195420-5fc61be5f6c5 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.7.5 github.com/paulmach/orb v0.9.2 @@ -97,7 +97,7 @@ require ( github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230808195420-5fc61be5f6c5 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index a054f882a..5bcd5e428 100644 --- a/go.sum +++ b/go.sum @@ -17,29 +17,378 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= +cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= +cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= +cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= +cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= +cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= +cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= +cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= +cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= +cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= +cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= +cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= +cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= +cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= +cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= +cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= +cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= +cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= +cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= +cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= +cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= +cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= +cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= +cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= +cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= +cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= +cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= +cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= +cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= +cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= +cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= +cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= +cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= +cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= +cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= +cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= +cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= +cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= +cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= +cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= +cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44 h1:v6JqBD/VzZx3QSxbaXnUwnnJ1KEYheU4LzLGr3IhsAE= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.15.1-0.20230802075510-964c3edb3f44/go.mod h1:pKJJRLOChW4zDQsAt1e8k/snWKljJtpkiPfxV53ngjI= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a h1:EeaOunifY30ATPzOo1F32vUDW299UXDHYukZrIVxBZ0= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230627134746-36f3d39c406a/go.mod h1:nkR5gaGeez3Zv2SE7aceP0YwxG2FzIB5cGKpQO2vV2o= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0 h1:vy6leTEGcKVrLmKfeK5pGGIi3D1vn6rX32Hy4gIOWto= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.0 h1:9ahw69njrwf2legz5xNVTA+4A6UwcBzy9oBdbJmoBxU= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.0/go.mod h1:3V8FyzpbIIxzpgfUaSlOJBAT11IzhZzkQnGpYvRQR5E= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= @@ -59,6 +408,7 @@ github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1 github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/ishell/v2 v2.0.2/go.mod h1:E4oTCXfo6QjoCart0QYa5m9w4S+deXs/P/9jA77A9Bs= @@ -73,6 +423,7 @@ github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGn github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -99,6 +450,9 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -119,9 +473,18 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.9.1/go.mod h1:a2DQL4+5ywF6safEeZFEPGRiiGbjzGFRUN2sg06VuU4= github.com/consensys/gnark-crypto v0.11.0 h1:QqzHQlwEqlQr5jfWblGDkwlKHpT+4QodYqqExkAtyks= github.com/consensys/gnark-crypto v0.11.0/go.mod h1:Iq/P3HHl0ElSjsg2E1gsMwhAyxnxoKK5nVyZKd+/KhU= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -133,6 +496,8 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= @@ -149,8 +514,14 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -160,6 +531,7 @@ github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWT github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -188,6 +560,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -203,6 +576,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -218,11 +592,13 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= @@ -242,12 +618,15 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -258,14 +637,31 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -275,6 +671,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0/go github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5 h1:3IZOAnD058zZllQTZNBioTlrzrBG/IjpiZ133IEtusM= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5/go.mod h1:xbKERva94Pw2cPen0s79J3uXmGzbbpDYFBFDlZ4mV/w= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -290,8 +688,10 @@ github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25 github.com/holiman/uint256 v1.2.2 h1:TXKcSGc2WaxPD2+bmzAsVthL4+pEN0YwXcL5qED83vk= github.com/holiman/uint256 v1.2.2/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= @@ -325,11 +725,15 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -396,30 +800,39 @@ github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= -github.com/nspcc-dev/dbft v0.0.0-20220902113116-58a5e763e647/go.mod h1:g9xisXmX9NP9MjioaTe862n9SlZTrP+6PVUWLBYOr98= +github.com/nspcc-dev/dbft v0.0.0-20220629112714-fd49ca59d354/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= +github.com/nspcc-dev/dbft v0.0.0-20221020093431-31c1bbdc74f2/go.mod h1:g9xisXmX9NP9MjioaTe862n9SlZTrP+6PVUWLBYOr98= +github.com/nspcc-dev/dbft v0.0.0-20230515113611-25db6ba61d5c/go.mod h1:kjBC9F8L25GR+kIHy/1KgG/KfcoGnVwIiyovgq1uszk= github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 h1:n4ZaFCKt1pQJd7PXoMJabZWK9ejjbLOVrkl/lOUmshg= github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= -github.com/nspcc-dev/neo-go v0.99.4/go.mod h1:mKTolfRUfKjFso5HPvGSQtUZc70n0VKBMs16eGuC5gA= -github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc h1:fySIWvUQsitK5e5qYIHnTDCXuPpwzz89SEUEIyY11sg= -github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc/go.mod h1:s9QhjMC784MWqTURovMbyYduIJc86mnCruxcMiAebpc= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220927123257-24c107e3a262/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce h1:vLGuUNDkmQrWMa4rr4vTd1u8ULqejWxVmNz1L7ocTEI= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= +github.com/nspcc-dev/neo-go v0.99.2/go.mod h1:9P0yWqhZX7i/ChJ+zjtiStO1uPTolPFUM+L5oNznU8E= +github.com/nspcc-dev/neo-go v0.100.1/go.mod h1:Nnp7F4e9IBccsgtCeLtUWV+0T6gk1PtP5HRtA13hUfc= +github.com/nspcc-dev/neo-go v0.101.5-0.20230808195420-5fc61be5f6c5 h1:AXI2upTPeTqX+n4xrBEzPATgEviOM/Prg6UQ6KDm+DU= +github.com/nspcc-dev/neo-go v0.101.5-0.20230808195420-5fc61be5f6c5/go.mod h1:Z0kpjwnTJj/ik/X6z18xjCL0X2+RNbqlnhKrl+MYgP8= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220809123759-3094d3e0c14b/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230420112658-c50ab951645a/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230808195420-5fc61be5f6c5 h1:/d7mY5hYlNhmEXexKcyqSR0b1Hdl5hf/c5o8Vi/1vt4= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230808195420-5fc61be5f6c5/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= +github.com/nspcc-dev/neofs-api-go/v2 v2.14.0/go.mod h1:DRIr0Ic1s+6QgdqmNFNLIqMqd7lNMJfYwkczlm1hDtM= +github.com/nspcc-dev/neofs-contract v0.16.0/go.mod h1:gN5bo2TlMvLbySImmg76DVj3jVmYgti2VVlQ+h/tcr0= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.9/go.mod h1:fTsdTU/M9rvv/f9jlp7vHOm3DRp+NSfjfTv9NohrKTE= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= +github.com/nspcc-dev/tzhash v1.7.0/go.mod h1:Dnx9LUlOLr5paL2Rtc96x0PPs8D9eIkUtowt1n+KQus= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -444,10 +857,12 @@ github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKf github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -489,6 +904,7 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -497,12 +913,17 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= @@ -552,6 +973,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= @@ -560,6 +982,7 @@ go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE= go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0= @@ -569,6 +992,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= @@ -588,17 +1012,21 @@ go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.20.0 h1:BLOA1cZBAGSbRiNuGCCKiFrCdYB7deeHDeD1SueyOfA= go.opentelemetry.io/proto/otlp v0.20.0/go.mod h1:3QgjzPALBIv9pcknj2EXGPXjYPFdUh/RQfF8Lz3+Vnw= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -611,6 +1039,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -618,8 +1047,11 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -632,6 +1064,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/exp v0.0.0-20221227203929-1b447090c38c/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -647,6 +1081,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -657,8 +1092,13 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -697,13 +1137,32 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -715,8 +1174,23 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -730,6 +1204,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= @@ -785,25 +1260,62 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -812,16 +1324,22 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -874,14 +1392,26 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -901,6 +1431,38 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -944,7 +1506,82 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230628200519-e449d1ea0e82 h1:Wdfp5Hc1bqGCWYZNrir4A1Jb+SmVaV2j1DL/pbMMTGI= google.golang.org/genproto v0.0.0-20230628200519-e449d1ea0e82/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/genproto/googleapis/api v0.0.0-20230628200519-e449d1ea0e82 h1:iI5Fmsfz4zDINYxJLxn2YChI//ypkHM/KuVSvlN7ZXk= @@ -969,9 +1606,30 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -985,6 +1643,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= From 3889e829e6d1a858c71b3b649eed615b41be2480 Mon Sep 17 00:00:00 2001 From: "a.arifullin" Date: Thu, 14 Sep 2023 14:08:43 +0300 Subject: [PATCH 0591/1943] [#667] writecache: Add logs for report error func in tests Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- .../writecache/writecachebadger/flush_test.go | 6 +++++- .../writecache/writecachebbolt/flush_test.go | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go index 855d2214f..392654e48 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go @@ -14,9 +14,12 @@ import ( oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/require" + "go.uber.org/zap" ) func TestFlush(t *testing.T) { + testlogger := test.NewLogger(t, true) + createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { return New( append([]Option{ @@ -30,8 +33,9 @@ func TestFlush(t *testing.T) { errCountOpt := func() (Option, *atomic.Uint32) { cnt := &atomic.Uint32{} - return WithReportErrorFunc(func(string, error) { + return WithReportErrorFunc(func(msg string, err error) { cnt.Add(1) + testlogger.Warn(msg, zap.Uint32("error_count", cnt.Load()), zap.Error(err)) }), cnt } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go index 5db4df318..533cec1d2 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go @@ -17,13 +17,16 @@ import ( oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" + "go.uber.org/zap" ) func TestFlush(t *testing.T) { + testlogger := test.NewLogger(t, true) + createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { return New( append([]Option{ - WithLogger(test.NewLogger(t, true)), + WithLogger(testlogger), WithPath(filepath.Join(t.TempDir(), "writecache")), WithSmallObjectSize(smallSize), WithMetabase(mb), @@ -33,8 +36,9 @@ func TestFlush(t *testing.T) { errCountOpt := func() (Option, *atomic.Uint32) { cnt := &atomic.Uint32{} - return WithReportErrorFunc(func(string, error) { + return WithReportErrorFunc(func(msg string, err error) { cnt.Add(1) + testlogger.Warn(msg, zap.Uint32("error_count", cnt.Load()), zap.Error(err)) }), cnt } From d9b93b12c1536953cb3286753f115b4162f88c65 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 18 Sep 2023 10:05:17 +0300 Subject: [PATCH 0592/1943] [#333] Sort shards by shard_ID in cli output Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/modules/control/shards_list.go | 14 ++++++++++++-- cmd/frostfs-cli/modules/control/shards_set_mode.go | 6 ++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/shards_list.go b/cmd/frostfs-cli/modules/control/shards_list.go index ec515e6b4..4d09667b8 100644 --- a/cmd/frostfs-cli/modules/control/shards_list.go +++ b/cmd/frostfs-cli/modules/control/shards_list.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "sort" "strings" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -49,11 +50,14 @@ func listShards(cmd *cobra.Command, _ []string) { verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + shards := resp.GetBody().GetShards() + sortShardsByID(shards) + isJSON, _ := cmd.Flags().GetBool(commonflags.JSON) if isJSON { - prettyPrintShardsJSON(cmd, resp.GetBody().GetShards()) + prettyPrintShardsJSON(cmd, shards) } else { - prettyPrintShards(cmd, resp.GetBody().GetShards()) + prettyPrintShards(cmd, shards) } } @@ -115,3 +119,9 @@ func shardModeToString(m control.ShardMode) string { return "unknown" } + +func sortShardsByID(ii []*control.ShardInfo) { + sort.Slice(ii, func(i, j int) bool { + return bytes.Compare(ii[i].Shard_ID, ii[j].Shard_ID) < 0 + }) +} diff --git a/cmd/frostfs-cli/modules/control/shards_set_mode.go b/cmd/frostfs-cli/modules/control/shards_set_mode.go index 135c0efa6..78f768965 100644 --- a/cmd/frostfs-cli/modules/control/shards_set_mode.go +++ b/cmd/frostfs-cli/modules/control/shards_set_mode.go @@ -1,7 +1,9 @@ package control import ( + "bytes" "fmt" + "sort" "strings" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -167,5 +169,9 @@ func getShardIDList(cmd *cobra.Command) [][]byte { res = append(res, raw) } + sort.Slice(res, func(i, j int) bool { + return bytes.Compare(res[i], res[j]) < 0 + }) + return res } From c14c9a023cbd96e12a60909f87f0d1e56ffba97b Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Wed, 20 Sep 2023 18:26:28 +0300 Subject: [PATCH 0593/1943] [#333] Sort objects by ID in SearchObjects Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/internal/client/client.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index e8869bc8c..26d8aa83b 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "io" + "sort" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/accounting" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" @@ -727,6 +729,11 @@ func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes return nil, fmt.Errorf("read object list: %w", err) } + sort.Slice(list, func(i, j int) bool { + lhs, rhs := list[i].EncodeToString(), list[j].EncodeToString() + return strings.Compare(lhs, rhs) < 0 + }) + return &SearchObjectsRes{ ids: list, }, nil From b8c3c2486d13a2a2bcf4ea0b8b85c03571f80e57 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 25 Sep 2023 09:58:26 +0300 Subject: [PATCH 0594/1943] [#333] Sort containers by ID Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/internal/client/client.go | 10 ++++++++++ cmd/frostfs-cli/modules/container/list.go | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 26d8aa83b..9cb7c8161 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -71,6 +71,16 @@ func ListContainers(ctx context.Context, prm ListContainersPrm) (res ListContain return } +// SortedIDList returns sorted list of identifiers of user's containers. +func (x ListContainersRes) SortedIDList() []cid.ID { + list := x.cliRes.Containers() + sort.Slice(list, func(i, j int) bool { + lhs, rhs := list[i].EncodeToString(), list[j].EncodeToString() + return strings.Compare(lhs, rhs) < 0 + }) + return list +} + // PutContainerPrm groups parameters of PutContainer operation. type PutContainerPrm struct { Client *client.Client diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index c8b891d32..1c7787760 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -56,7 +56,7 @@ var listContainersCmd = &cobra.Command{ Client: cli, } - containerIDs := res.IDList() + containerIDs := res.SortedIDList() for _, cnrID := range containerIDs { if flagVarListName == "" && !flagVarListPrintAttr { cmd.Println(cnrID.String()) From c1e4130020dfde27387ddf26bdf1cafd5444a00b Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 27 Sep 2023 11:02:06 +0300 Subject: [PATCH 0595/1943] [#146] node: Add trace_id to logs Signed-off-by: Alexander Chuprov --- .../blobovnicza/delete.go | 2 ++ .../blobstor/blobovniczatree/delete.go | 2 ++ .../blobstor/blobovniczatree/exists.go | 4 ++- .../blobstor/blobovniczatree/get.go | 2 ++ .../blobstor/blobovniczatree/get_range.go | 3 +- .../blobstor/blobovniczatree/put.go | 10 ++++-- pkg/local_object_storage/blobstor/exists.go | 4 ++- pkg/local_object_storage/engine/delete.go | 7 ++-- pkg/local_object_storage/engine/evacuate.go | 19 +++++++---- pkg/local_object_storage/engine/get.go | 4 ++- pkg/local_object_storage/engine/inhume.go | 7 ++-- pkg/local_object_storage/engine/put.go | 5 ++- pkg/local_object_storage/engine/range.go | 4 ++- pkg/local_object_storage/engine/tree.go | 34 +++++++++++++------ pkg/local_object_storage/shard/delete.go | 7 ++-- pkg/local_object_storage/shard/get.go | 7 ++-- pkg/local_object_storage/shard/inhume.go | 2 ++ pkg/local_object_storage/shard/list.go | 4 ++- pkg/local_object_storage/shard/move.go | 2 ++ pkg/services/object/put/single.go | 5 +-- pkg/services/replicator/process.go | 5 ++- pkg/services/tree/redirect.go | 5 ++- pkg/services/tree/replicator.go | 7 ++-- pkg/tracing/trace.go | 19 +++++++++++ 24 files changed, 129 insertions(+), 41 deletions(-) create mode 100644 pkg/tracing/trace.go diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 5a5da235f..4bc13cc95 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -4,6 +4,7 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -73,6 +74,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err b.log.Debug(logs.BlobovniczaObjectWasRemovedFromBucket, zap.String("binary size", stringifyByteSize(dataSize)), zap.String("range", stringifyBounds(sizeLowerBound, sizeUpperBound)), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) b.itemDeleted(sizeUpperBound) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index ca1e46a15..28e3a8f36 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -69,6 +70,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index c94881168..e1a6f5ed5 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "go.opentelemetry.io/otel/attribute" @@ -56,7 +57,8 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common if !client.IsErrObjectNotFound(err) { b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index f2399e414..49849e759 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -69,6 +70,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 56b41351b..91d1e3ce0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -71,7 +72,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), - ) + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } if outOfBounds { return true, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index aae280e0b..6f9c8c0de 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -81,14 +82,16 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) } else { i.B.log.Debug(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return false, nil } if active == nil { - i.B.log.Debug(logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath)) + i.B.log.Debug(logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return false, nil } defer active.Close() @@ -102,7 +105,8 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) } else { i.B.log.Debug(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", active.Path()), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return false, nil diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 03dad392a..43feec7c9 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -74,7 +75,8 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi for _, err := range errors[:len(errors)-1] { b.log.Warn(logs.BlobstorErrorOccurredDuringObjectExistenceChecking, zap.Stringer("address", prm.Address), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return common.ExistsRes{}, errors[len(errors)-1] diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 8e3180560..68a7325c6 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -150,7 +151,8 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo if err != nil { e.log.Warn(logs.EngineErrorDuringSearchingForObjectChildren, zap.Stringer("addr", addr), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return false } @@ -161,7 +163,8 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo if err != nil { e.log.Debug(logs.EngineCouldNotInhumeObjectInShard, zap.Stringer("addr", addr), - zap.String("err", err.Error())) + zap.String("err", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) continue } } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 98a3a202d..192070c1c 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -11,6 +11,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -200,11 +201,13 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p e.evacuateLimiter.Complete(err) }() - e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField) + e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) err = e.getTotalObjectsCount(ctx, shardsToEvacuate, res) if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField) + e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } @@ -266,7 +269,8 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { break } - e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField) + e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } @@ -342,7 +346,8 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to res.failed.Add(1) continue } - e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) + e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } @@ -363,7 +368,8 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to err = prm.handler(ctx, addr, getRes.Object()) if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) + e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } res.evacuated.Add(1) @@ -392,7 +398,8 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), zap.Stringer("addr", addr), - evacuationOperationLogField) + evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return true, nil } diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 5d888238e..f77c44226 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -105,7 +106,8 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound, zap.Stringer("shard_id", it.ShardWithMeta.ID()), zap.String("error", it.MetaError.Error()), - zap.Stringer("address", prm.addr)) + zap.Stringer("address", prm.addr), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } } diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index e0f9d7ab3..293746f70 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -93,7 +94,8 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e if err != nil { e.log.Warn(logs.EngineRemovingAnObjectWithoutFullLockingCheck, zap.Error(err), - zap.Stringer("addr", prm.addrs[i])) + zap.Stringer("addr", prm.addrs[i]), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } else if locked { return InhumeRes{}, new(apistatus.ObjectLocked) } @@ -201,7 +203,8 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { locked, err = h.Shard.IsLocked(ctx, addr) if err != nil { - e.reportShardError(h, "can't check object's lockers", err, zap.Stringer("addr", addr)) + e.reportShardError(h, "can't check object's lockers", err, zap.Stringer("addr", addr), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) outErr = err return false } diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 2f96b8296..79ee3a997 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -128,6 +129,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, e.log.Warn(logs.EngineCouldNotMarkObjectForShardRelocation, zap.Stringer("shard", sh.ID()), zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } } @@ -144,7 +146,8 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, errors.Is(err, common.ErrReadOnly) || errors.Is(err, common.ErrNoSpace) { e.log.Warn(logs.EngineCouldNotPutObjectToShard, zap.Stringer("shard_id", sh.ID()), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return } diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index fb0873ee2..e45845d6c 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -118,7 +119,8 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound, zap.Stringer("shard_id", it.ShardWithMeta.ID()), zap.String("error", it.MetaError.Error()), - zap.Stringer("address", prm.addr)) + zap.Stringer("address", prm.addr), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } } diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 55b9f9c50..6c3594bab 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.opentelemetry.io/otel/attribute" @@ -38,7 +39,8 @@ func (e *StorageEngine) TreeMove(ctx context.Context, d pilorama.CIDDescriptor, if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't perform `TreeMove`", err, zap.Stringer("cid", d.CID), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return nil, err @@ -71,7 +73,8 @@ func (e *StorageEngine) TreeAddByPath(ctx context.Context, d pilorama.CIDDescrip if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't perform `TreeAddByPath`", err, zap.Stringer("cid", d.CID), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return nil, err } @@ -99,7 +102,8 @@ func (e *StorageEngine) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID str if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't perform `TreeApply`", err, zap.Stringer("cid", cnr), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return err } @@ -130,7 +134,8 @@ func (e *StorageEngine) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(sh, "can't perform `TreeGetByPath`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } @@ -162,7 +167,8 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(sh, "can't perform `TreeGetMeta`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } @@ -193,7 +199,8 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(sh, "can't perform `TreeGetChildren`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } @@ -224,7 +231,8 @@ func (e *StorageEngine) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(sh, "can't perform `TreeGetOpLog`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } @@ -253,7 +261,8 @@ func (e *StorageEngine) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID stri if !errors.Is(err, pilorama.ErrTreeNotFound) && !errors.Is(err, shard.ErrReadOnlyMode) { e.reportShardError(sh, "can't perform `TreeDrop`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } @@ -281,7 +290,8 @@ func (e *StorageEngine) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, } e.reportShardError(sh, "can't perform `TreeList`", err, - zap.Stringer("cid", cid)) + zap.Stringer("cid", cid), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) // returns as much info about // trees as possible @@ -347,7 +357,8 @@ func (e *StorageEngine) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK if err != nil && !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(lst[index], "can't update tree synchronization height", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return err } @@ -373,7 +384,8 @@ func (e *StorageEngine) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, t if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(sh, "can't read tree synchronization height", err, zap.Stringer("cid", cid), - zap.String("tree", treeID)) + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } continue } diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 662c0835a..ea481300b 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -94,7 +95,8 @@ func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { if err != nil { s.log.Debug(logs.StorageIDRetrievalFailure, zap.Stringer("object", addr), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } storageID := res.StorageID() @@ -106,7 +108,8 @@ func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { if err != nil { s.log.Debug(logs.ObjectRemovalFailureBlobStor, zap.Stringer("object_address", addr), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } } diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 91565a346..2e7c84bcd 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -10,6 +10,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -144,12 +145,14 @@ func (s *Shard) fetchObjectData(ctx context.Context, addr oid.Address, skipMeta if client.IsErrObjectNotFound(err) { s.log.Debug(logs.ShardObjectIsMissingInWritecache, zap.Stringer("addr", addr), - zap.Bool("skip_meta", skipMeta)) + zap.Bool("skip_meta", skipMeta), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } else { s.log.Error(logs.ShardFailedToFetchObjectFromWritecache, zap.Error(err), zap.Stringer("addr", addr), - zap.Bool("skip_meta", skipMeta)) + zap.Bool("skip_meta", skipMeta), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } } if skipMeta || mErr != nil { diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 6a2f9311d..a5f8960c3 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" @@ -110,6 +111,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { s.log.Debug(logs.ShardCouldNotMarkObjectToDeleteInMetabase, zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) s.m.RUnlock() diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index bc13c622b..d6e4d7e50 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -99,7 +100,8 @@ func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { if err != nil { s.log.Debug(logs.ShardCantSelectAllObjects, zap.Stringer("cid", lst[i]), - zap.String("error", err.Error())) + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) continue } diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go index 9832c9c84..539dbc97f 100644 --- a/pkg/local_object_storage/shard/move.go +++ b/pkg/local_object_storage/shard/move.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" @@ -53,6 +54,7 @@ func (s *Shard) ToMoveIt(ctx context.Context, prm ToMoveItPrm) (ToMoveItRes, err if err != nil { s.log.Debug(logs.ShardCouldNotMarkObjectForShardRelocationInMetabase, zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index adac194a0..3865aabb9 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -22,6 +22,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -242,8 +243,7 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, info.AddressGroup().IterateAddresses(func(addr network.Address) (stop bool) { ctx, span := tracing.StartSpanFromContext(ctx, "putService.redirectPutSingleRequest.IterateAddresses", trace.WithAttributes( - attribute.String("address", addr.String()), - )) + attribute.String("address", addr.String()))) defer span.End() var err error @@ -257,6 +257,7 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, zap.Stringer("address", addr), zap.Stringer("object_id", objID), zap.Stringer("container_id", cnrID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 16bcec9c5..3a46e5f04 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" @@ -44,7 +45,8 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) if err != nil { p.log.Error(logs.ReplicatorCouldNotGetObjectFromLocalStorage, zap.Stringer("object", task.Addr), - zap.Error(err)) + zap.Error(err), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return } @@ -63,6 +65,7 @@ func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) log := p.log.With( zap.String("node", netmap.StringifyPublicKey(task.Nodes[i])), zap.Stringer("object", task.Addr), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) callCtx, cancel := context.WithTimeout(ctx, p.putTimeout) diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index 0afd3439a..ec41a60d4 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -6,6 +6,7 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" @@ -39,7 +40,9 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo return false } - s.log.Debug(logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint)) + s.log.Debug(logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + called = true stop = f(c) return true diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 0ca30273e..49a37b8be 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -107,12 +108,14 @@ func (s *Service) replicationWorker(ctx context.Context) { if lastErr != nil { if errors.Is(lastErr, errRecentlyFailed) { s.log.Debug(logs.TreeDoNotSendUpdateToTheNode, - zap.String("last_error", lastErr.Error())) + zap.String("last_error", lastErr.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } else { s.log.Warn(logs.TreeFailedToSentUpdateToTheNode, zap.String("last_error", lastErr.Error()), zap.String("address", lastAddr), - zap.String("key", hex.EncodeToString(task.n.PublicKey()))) + zap.String("key", hex.EncodeToString(task.n.PublicKey())), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } s.metrics.AddReplicateTaskDuration(time.Since(start), false) } else { diff --git a/pkg/tracing/trace.go b/pkg/tracing/trace.go new file mode 100644 index 000000000..dc2f90259 --- /dev/null +++ b/pkg/tracing/trace.go @@ -0,0 +1,19 @@ +package tracing + +import ( + "context" + + "go.opentelemetry.io/otel/trace" +) + +var emptyTraceID = [16]byte{} + +// GetTraceID retrieves the trace ID from the provided context. +// It returns an empty string if no trace ID is found. +func GetTraceID(ctx context.Context) string { + span := trace.SpanFromContext(ctx) + if span == nil || span.SpanContext().TraceID() == emptyTraceID { + return "" + } + return span.SpanContext().TraceID().String() +} From a0a35ffbecc110ee17bfeefc3bbc4dbe9f328ec7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 29 Sep 2023 18:41:48 +0300 Subject: [PATCH 0596/1943] [#702] node: Update SDK version Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 89f437216..28008128c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0 git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230911122224-ac8fc6d4400c + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230928142024-84b9d29fc98c git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 5bcd5e428..8490c167c 100644 --- a/go.sum +++ b/go.sum @@ -393,8 +393,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230911122224-ac8fc6d4400c h1:gD+dj5IZx9jlniDu8TlLQdRGCd8KIOzYjjdDd1KcVdI= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230911122224-ac8fc6d4400c/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230928142024-84b9d29fc98c h1:c8mduKlc8Zioppz5o06QRYS5KYX3BFRO+NgKj2q6kD8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230928142024-84b9d29fc98c/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 627b3027456fc87605d9a82490088cd2c767dbe4 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 2 Oct 2023 10:38:11 +0300 Subject: [PATCH 0597/1943] [#709] node: Put in log info about listening endpoints Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/control.go | 7 ++++++- cmd/frostfs-node/grpc.go | 3 ++- cmd/frostfs-node/httpcomponent.go | 5 +++++ cmd/frostfs-node/notificator.go | 2 ++ internal/logs/logs.go | 3 ++- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index 3ed6bc54a..787fe3351 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -14,6 +14,8 @@ import ( "google.golang.org/grpc" ) +const serviceNameControl = "control" + type treeSynchronizer struct { treeSvc *tree.Service } @@ -66,7 +68,10 @@ func initControlService(c *cfg) { control.RegisterControlServiceServer(c.cfgControlService.server, ctlSvc) c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { - runAndLog(ctx, c, "control", false, func(context.Context, *cfg) { + runAndLog(ctx, c, serviceNameControl, false, func(context.Context, *cfg) { + c.log.Info(logs.FrostFSNodeStartListeningEndpoint, + zap.String("service", serviceNameControl), + zap.String("endpoint", endpoint)) fatalOnErr(c.cfgControlService.server.Serve(lis)) }) })) diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index 42059ec09..74df7a18f 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -110,7 +110,8 @@ func serveGRPC(c *cfg) { c.wg.Done() }() - c.log.Info(logs.FrostFSNodeStartListeningGRPCEndpoint, + c.log.Info(logs.FrostFSNodeStartListeningEndpoint, + zap.String("service", "gRPC"), zap.Stringer("endpoint", lis.Addr()), ) diff --git a/cmd/frostfs-node/httpcomponent.go b/cmd/frostfs-node/httpcomponent.go index db9239384..6e8fa8f2c 100644 --- a/cmd/frostfs-node/httpcomponent.go +++ b/cmd/frostfs-node/httpcomponent.go @@ -6,7 +6,9 @@ import ( "net/http" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" + "go.uber.org/zap" ) type httpComponent struct { @@ -42,6 +44,9 @@ func (cmp *httpComponent) init(c *cfg) { cmp.name, func(ctx context.Context) { runAndLog(ctx, c, cmp.name, false, func(context.Context, *cfg) { + c.log.Info(logs.FrostFSNodeStartListeningEndpoint, + zap.String("service", cmp.name), + zap.String("endpoint", cmp.address)) fatalOnErr(srv.Serve()) }) }, diff --git a/cmd/frostfs-node/notificator.go b/cmd/frostfs-node/notificator.go index 3fa486955..8e581505f 100644 --- a/cmd/frostfs-node/notificator.go +++ b/cmd/frostfs-node/notificator.go @@ -167,5 +167,7 @@ func connectNats(ctx context.Context, c *cfg) { err := c.cfgNotifications.nw.w.Connect(ctx, endpoint) if err != nil { panic(fmt.Sprintf("could not connect to a nats endpoint %s: %v", endpoint, err)) + } else { + c.log.Info(logs.NatsConnectedToEndpoint, zap.String("endpoint", endpoint)) } } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index c18d191f8..56312f83f 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -123,6 +123,7 @@ const ( NatsNatsConnectionWasLost = "nats: connection was lost" NatsNatsReconnectedToTheServer = "nats: reconnected to the server" NatsNatsClosingConnectionAsTheContextIsDone = "nats: closing connection as the context is done" + NatsConnectedToEndpoint = "nats: successfully connected to endpoint" ControllerStartingToAnnounceTheValuesOfTheMetrics = "starting to announce the values of the metrics" ControllerCouldNotInitializeIteratorOverLocallyCollectedMetrics = "could not initialize iterator over locally collected metrics" ControllerCouldNotInitializeAnnouncementAccumulator = "could not initialize announcement accumulator" @@ -413,10 +414,10 @@ const ( FrostFSIRCouldntCreateRPCClientForEndpoint = "could not create RPC client for endpoint" FrostFSIRCreatedRPCClientForEndpoint = "created RPC client for endpoint" FrostFSIRReloadExtraWallets = "reload extra wallets" + FrostFSNodeStartListeningEndpoint = "start listening endpoint" FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" - FrostFSNodeStartListeningGRPCEndpoint = "start listening gRPC endpoint" FrostFSNodeStoppingGRPCServer = "stopping gRPC server..." FrostFSNodeGRPCCannotShutdownGracefullyForcingStop = "gRPC cannot shutdown gracefully, forcing stop" FrostFSNodeGRPCServerStoppedSuccessfully = "gRPC server stopped successfully" From f3278d76a93a63d2004ac62c0a6f0f70ea81560b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 5 Oct 2023 10:57:58 +0300 Subject: [PATCH 0598/1943] [#721] netmap: Send bootstrap at each epoch tick Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/netmap.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index e7d3271b1..cc3c0c48d 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -179,15 +179,8 @@ func addNewEpochNotificationHandlers(c *cfg) { return } - n := ev.(netmapEvent.NewEpoch).EpochNumber() - - const reBootstrapInterval = 2 - - if (n-c.cfgNetmap.startEpoch)%reBootstrapInterval == 0 { - err := c.bootstrap() - if err != nil { - c.log.Warn(logs.FrostFSNodeCantSendRebootstrapTx, zap.Error(err)) - } + if err := c.bootstrap(); err != nil { + c.log.Warn(logs.FrostFSNodeCantSendRebootstrapTx, zap.Error(err)) } }) From bd5bf8b1a955bef17e2ba722b62579428433860d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 5 Oct 2023 11:30:41 +0300 Subject: [PATCH 0599/1943] [#721] netmap: Drop already bootstraped check Because of this check, under certain conditions, the node could be removed from the network map, although the node was functioning normally. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 +-- cmd/frostfs-node/netmap.go | 25 ++++++------------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 8286bc7d7..60e567c5a 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -345,8 +345,7 @@ type internals struct { apiVersion version.Version healthStatus *atomic.Int32 // is node under maintenance - isMaintenance atomic.Bool - alreadyBootstraped bool + isMaintenance atomic.Bool } // starts node's maintenance. diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index cc3c0c48d..ebe152e47 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -220,10 +220,6 @@ func bootstrapNode(c *cfg) { c.log.Info(logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) return } - if c.alreadyBootstraped { - c.log.Info(logs.NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap) - return - } err := c.bootstrap() fatalOnErrDetails("bootstrap error", err) } @@ -256,7 +252,7 @@ func initNetmapState(c *cfg) { fatalOnErrDetails("could not initialize current epoch number", err) var ni *netmapSDK.NodeInfo - ni, c.alreadyBootstraped, err = c.netmapInitLocalNodeState(epoch) + ni, err = c.netmapInitLocalNodeState(epoch) fatalOnErrDetails("could not init network state", err) stateWord := nodeState(ni) @@ -275,13 +271,6 @@ func initNetmapState(c *cfg) { c.handleLocalNodeInfo(ni) } -func sameNodeInfo(a, b *netmapSDK.NodeInfo) bool { - // Suboptimal, but we do this once on the node startup. - rawA := a.Marshal() - rawB := b.Marshal() - return bytes.Equal(rawA, rawB) -} - func nodeState(ni *netmapSDK.NodeInfo) string { if ni != nil { switch { @@ -296,29 +285,27 @@ func nodeState(ni *netmapSDK.NodeInfo) string { return "undefined" } -func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, bool, error) { +func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { nmNodes, err := c.cfgNetmap.wrapper.GetCandidates() if err != nil { - return nil, false, err + return nil, err } var candidate *netmapSDK.NodeInfo - alreadyBootstraped := false for i := range nmNodes { if bytes.Equal(nmNodes[i].PublicKey(), c.binPublicKey) { candidate = &nmNodes[i] - alreadyBootstraped = candidate.IsOnline() && sameNodeInfo(&c.cfgNodeInfo.localInfo, candidate) break } } node, err := c.netmapLocalNodeState(epoch) if err != nil { - return nil, false, err + return nil, err } if candidate == nil { - return node, false, nil + return node, nil } nmState := nodeState(node) @@ -330,7 +317,7 @@ func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, bool, zap.String("netmap", nmState), zap.String("candidate", candidateState)) } - return candidate, alreadyBootstraped, nil + return candidate, nil } func (c *cfg) netmapLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { From 997ac7cd8d386700674516c026b90127e416d518 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 6 Oct 2023 10:22:35 +0300 Subject: [PATCH 0600/1943] [#726] writecache: Fix small object flush for BBolt Do not marshal object twice. Signed-off-by: Dmitrii Stepanov --- .../writecache/writecachebbolt/cachebbolt.go | 4 ++-- .../writecache/writecachebbolt/flush.go | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go index 11bdbe07b..363ee8448 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go @@ -29,7 +29,7 @@ type cache struct { compressFlags map[string]struct{} // flushCh is a channel with objects to flush. - flushCh chan *objectSDK.Object + flushCh chan objectInfo // closeCh is close channel, protected by modeMtx. closeCh chan struct{} // wg is a wait group for flush workers. @@ -62,7 +62,7 @@ var ( // New creates new writecache instance. func New(opts ...Option) writecache.Cache { c := &cache{ - flushCh: make(chan *objectSDK.Object), + flushCh: make(chan objectInfo), mode: mode.ReadWrite, compressFlags: make(map[string]struct{}), diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush.go b/pkg/local_object_storage/writecache/writecachebbolt/flush.go index c3b0f89bb..89b26cd64 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush.go @@ -79,7 +79,6 @@ func (c *cache) runFlushLoop() { func (c *cache) flushSmallObjects() { var lastKey []byte - var m []objectInfo for { select { case <-c.closeCh: @@ -87,7 +86,7 @@ func (c *cache) flushSmallObjects() { default: } - m = m[:0] + var m []objectInfo c.modeMtx.RLock() if c.readOnly() { @@ -133,10 +132,11 @@ func (c *cache) flushSmallObjects() { if err := obj.Unmarshal(m[i].data); err != nil { continue } + m[i].obj = obj count++ select { - case c.flushCh <- obj: + case c.flushCh <- m[i]: case <-c.closeCh: c.modeMtx.RUnlock() return @@ -231,22 +231,22 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { func (c *cache) workerFlushSmall() { defer c.wg.Done() - var obj *objectSDK.Object + var objInfo objectInfo for { // Give priority to direct put. select { - case obj = <-c.flushCh: + case objInfo = <-c.flushCh: case <-c.closeCh: return } - err := c.flushObject(context.TODO(), obj, nil, writecache.StorageTypeDB) + err := c.flushObject(context.TODO(), objInfo.obj, objInfo.data, writecache.StorageTypeDB) if err != nil { // Error is handled in flushObject. continue } - c.deleteFromDB(objectCore.AddressOf(obj).EncodeToString()) + c.deleteFromDB(objInfo.addr) } } From d07afd803ccb1dc111e545d8ac27c37987b3831c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 6 Oct 2023 11:31:40 +0300 Subject: [PATCH 0601/1943] [#726] writecache: Fix small object flush for Badger Do not marshal object twice. Signed-off-by: Dmitrii Stepanov --- .../writecache/writecachebadger/cachebadger.go | 4 ++-- .../writecache/writecachebadger/flush.go | 17 ++++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go index 6dd4755b3..d5da77635 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go +++ b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go @@ -21,7 +21,7 @@ type cache struct { modeMtx sync.RWMutex // flushCh is a channel with objects to flush. - flushCh chan *objectSDK.Object + flushCh chan objectInfo // scheduled4Flush contains objects scheduled for flush via flushCh // helps to avoid multiple flushing of one object scheduled4Flush map[oid.Address]struct{} @@ -52,7 +52,7 @@ const ( // New creates new writecache instance. func New(opts ...Option) writecache.Cache { c := &cache{ - flushCh: make(chan *objectSDK.Object), + flushCh: make(chan objectInfo), mode: mode.ReadWrite, scheduled4Flush: map[oid.Address]struct{}{}, diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush.go b/pkg/local_object_storage/writecache/writecachebadger/flush.go index ab6910a80..3f2bdb043 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush.go @@ -85,7 +85,11 @@ func (c *collector) Send(buf *z.Buffer) error { c.cache.scheduled4FlushMtx.Unlock() c.scheduled++ select { - case c.cache.flushCh <- obj: + case c.cache.flushCh <- objectInfo{ + addr: addr, + data: val, + obj: obj, + }: case <-c.cache.closeCh: c.cancel() return nil @@ -175,22 +179,21 @@ func (c *cache) reportFlushError(msg string, addr string, err error) { func (c *cache) workerFlushSmall() { defer c.wg.Done() - var obj *objectSDK.Object + var objInfo objectInfo for { // Give priority to direct put. select { - case obj = <-c.flushCh: + case objInfo = <-c.flushCh: case <-c.closeCh: return } - addr := objectCore.AddressOf(obj) - err := c.flushObject(context.TODO(), obj, nil, writecache.StorageTypeDB) + err := c.flushObject(context.TODO(), objInfo.obj, objInfo.data, writecache.StorageTypeDB) if err == nil { - c.deleteFromDB([]internalKey{addr2key(addr)}) + c.deleteFromDB([]internalKey{addr2key(objInfo.addr)}) } c.scheduled4FlushMtx.Lock() - delete(c.scheduled4Flush, addr) + delete(c.scheduled4Flush, objInfo.addr) c.scheduled4FlushMtx.Unlock() } } From 4caa934eea89e061bfc7b8bf31643e4e6c587572 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 6 Oct 2023 13:54:48 +0300 Subject: [PATCH 0602/1943] [#729] containersvc: Remove load announcement IR code was removed in 8879c6ea. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/container.go | 436 +----------------- .../announcement/load/controller/calls.go | 307 ------------ .../load/controller/calls_test.go | 192 -------- .../load/controller/controller.go | 94 ---- .../announcement/load/controller/deps.go | 103 ----- .../announcement/load/controller/opts.go | 28 -- .../announcement/load/controller/util.go | 36 -- .../announcement/load/route/calls.go | 145 ------ .../container/announcement/load/route/deps.go | 31 -- .../container/announcement/load/route/opts.go | 28 -- .../load/route/placement/builder.go | 49 -- .../load/route/placement/calls.go | 47 -- .../announcement/load/route/placement/deps.go | 14 - .../announcement/load/route/router.go | 87 ---- .../container/announcement/load/route/util.go | 49 -- .../announcement/load/storage/storage.go | 151 ------ .../announcement/load/storage/storage_test.go | 50 -- 17 files changed, 2 insertions(+), 1845 deletions(-) delete mode 100644 pkg/services/container/announcement/load/controller/calls.go delete mode 100644 pkg/services/container/announcement/load/controller/calls_test.go delete mode 100644 pkg/services/container/announcement/load/controller/controller.go delete mode 100644 pkg/services/container/announcement/load/controller/deps.go delete mode 100644 pkg/services/container/announcement/load/controller/opts.go delete mode 100644 pkg/services/container/announcement/load/controller/util.go delete mode 100644 pkg/services/container/announcement/load/route/calls.go delete mode 100644 pkg/services/container/announcement/load/route/deps.go delete mode 100644 pkg/services/container/announcement/load/route/opts.go delete mode 100644 pkg/services/container/announcement/load/route/placement/builder.go delete mode 100644 pkg/services/container/announcement/load/route/placement/calls.go delete mode 100644 pkg/services/container/announcement/load/route/placement/deps.go delete mode 100644 pkg/services/container/announcement/load/route/router.go delete mode 100644 pkg/services/container/announcement/load/route/util.go delete mode 100644 pkg/services/container/announcement/load/storage/storage.go delete mode 100644 pkg/services/container/announcement/load/storage/storage_test.go diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index d54bf13dd..9239df640 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -3,44 +3,22 @@ package main import ( "bytes" "context" - "crypto/ecdsa" - "crypto/sha256" - "errors" - "fmt" - "strconv" - containerV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" containerTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/container/grpc" containerService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container" - loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" - loadroute "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/route" - placementrouter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/route/placement" - loadstorage "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/storage" containerMorph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/morph" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - apiClient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "go.uber.org/zap" ) -const ( - startEstimationNotifyEvent = "StartEstimation" - stopEstimationNotifyEvent = "StopEstimation" -) - -func initContainerService(ctx context.Context, c *cfg) { +func initContainerService(_ context.Context, c *cfg) { // container wrapper that tries to invoke notary // requests if chain is configured so wrap, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0, cntClient.TryNotary()) @@ -52,44 +30,10 @@ func initContainerService(ctx context.Context, c *cfg) { cnrRdr, cnrWrt := configureEACLAndContainerSources(c, wrap, cnrSrc) - loadAccumulator := loadstorage.New(loadstorage.Prm{}) - - loadPlacementBuilder := &loadPlacementBuilder{ - log: c.log, - nmSrc: c.netMapSource, - cnrSrc: cnrSrc, - } - - routeBuilder := placementrouter.New(placementrouter.Prm{ - PlacementBuilder: loadPlacementBuilder, - }) - - loadRouter := loadroute.New( - loadroute.Prm{ - LocalServerInfo: c, - RemoteWriterProvider: &remoteLoadAnnounceProvider{ - key: &c.key.PrivateKey, - netmapKeys: c, - clientCache: c.bgClientCache, - deadEndProvider: loadcontroller.SimpleWriterProvider(loadAccumulator), - }, - Builder: routeBuilder, - }, - loadroute.WithLogger(c.log), - ) - - setLoadController(ctx, c, loadRouter, loadAccumulator) - server := containerTransportGRPC.New( containerService.NewSignService( &c.key.PrivateKey, - &usedSpaceService{ - Server: containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), - loadWriterProvider: loadRouter, - loadPlacementBuilder: loadPlacementBuilder, - routeBuilder: routeBuilder, - cfg: c, - }, + containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), ), ) @@ -178,50 +122,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c return cnrRdr, cnrWrt } -func setLoadController(ctx context.Context, c *cfg, loadRouter *loadroute.Router, loadAccumulator *loadstorage.Storage) { - pubKey := c.key.PublicKey().Bytes() - - // container wrapper that always sends non-notary - // requests - wrapperNoNotary, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0) - fatalOnErr(err) - - resultWriter := &morphLoadWriter{ - log: c.log, - cnrMorphClient: wrapperNoNotary, - key: pubKey, - } - - localMetrics := &localStorageLoad{ - log: c.log, - engine: c.cfgObject.cfgLocalStorage.localStorage, - } - - ctrl := loadcontroller.New( - loadcontroller.Prm{ - LocalMetrics: loadcontroller.SimpleIteratorProvider(localMetrics), - AnnouncementAccumulator: loadcontroller.SimpleIteratorProvider(loadAccumulator), - LocalAnnouncementTarget: loadRouter, - ResultReceiver: loadcontroller.SimpleWriterProvider(resultWriter), - }, - loadcontroller.WithLogger(c.log), - ) - - setContainerNotificationParser(c, startEstimationNotifyEvent, containerEvent.ParseStartEstimation) - addContainerAsyncNotificationHandler(c, startEstimationNotifyEvent, func(ev event.Event) { - ctrl.Start(ctx, loadcontroller.StartPrm{ - Epoch: ev.(containerEvent.StartEstimation).Epoch(), - }) - }) - - setContainerNotificationParser(c, stopEstimationNotifyEvent, containerEvent.ParseStopEstimation) - addContainerAsyncNotificationHandler(c, stopEstimationNotifyEvent, func(ev event.Event) { - ctrl.Stop(ctx, loadcontroller.StopPrm{ - Epoch: ev.(containerEvent.StopEstimation).Epoch(), - }) - }) -} - // addContainerNotificationHandler adds handler that will be executed synchronously. func addContainerNotificationHandler(c *cfg, sTyp string, h event.Handler) { typ := event.TypeFromString(sTyp) @@ -284,219 +184,6 @@ func setContainerNotificationParser(c *cfg, sTyp string, p event.NotificationPar c.cfgContainer.parsers[typ] = p } -type morphLoadWriter struct { - log *logger.Logger - - cnrMorphClient *cntClient.Client - - key []byte -} - -func (w *morphLoadWriter) Put(a containerSDK.SizeEstimation) error { - w.log.Debug(logs.FrostFSNodeSaveUsedSpaceAnnouncementInContract, - zap.Uint64("epoch", a.Epoch()), - zap.Stringer("cid", a.Container()), - zap.Uint64("size", a.Value()), - ) - - prm := cntClient.AnnounceLoadPrm{} - - prm.SetAnnouncement(a) - prm.SetReporter(w.key) - - return w.cnrMorphClient.AnnounceLoad(prm) -} - -func (*morphLoadWriter) Close(context.Context) error { - return nil -} - -type nopLoadWriter struct{} - -func (nopLoadWriter) Put(containerSDK.SizeEstimation) error { - return nil -} - -func (nopLoadWriter) Close(context.Context) error { - return nil -} - -type remoteLoadAnnounceProvider struct { - key *ecdsa.PrivateKey - - netmapKeys netmapCore.AnnouncedKeys - - clientCache interface { - Get(client.NodeInfo) (client.MultiAddressClient, error) - } - - deadEndProvider loadcontroller.WriterProvider -} - -func (r *remoteLoadAnnounceProvider) InitRemote(srv loadcontroller.ServerInfo) (loadcontroller.WriterProvider, error) { - if srv == nil { - return r.deadEndProvider, nil - } - - if r.netmapKeys.IsLocalKey(srv.PublicKey()) { - // if local => return no-op writer - return loadcontroller.SimpleWriterProvider(new(nopLoadWriter)), nil - } - - var info client.NodeInfo - - err := client.NodeInfoFromRawNetmapElement(&info, srv) - if err != nil { - return nil, fmt.Errorf("parse client node info: %w", err) - } - - c, err := r.clientCache.Get(info) - if err != nil { - return nil, fmt.Errorf("could not initialize API client: %w", err) - } - - return &remoteLoadAnnounceWriterProvider{ - client: c, - }, nil -} - -type remoteLoadAnnounceWriterProvider struct { - client client.Client -} - -func (p *remoteLoadAnnounceWriterProvider) InitWriter([]loadcontroller.ServerInfo) (loadcontroller.Writer, error) { - return &remoteLoadAnnounceWriter{ - client: p.client, - }, nil -} - -type remoteLoadAnnounceWriter struct { - client client.Client - - buf []containerSDK.SizeEstimation -} - -func (r *remoteLoadAnnounceWriter) Put(a containerSDK.SizeEstimation) error { - r.buf = append(r.buf, a) - - return nil -} - -func (r *remoteLoadAnnounceWriter) Close(ctx context.Context) error { - cliPrm := apiClient.PrmAnnounceSpace{ - Announcements: r.buf, - } - - _, err := r.client.ContainerAnnounceUsedSpace(ctx, cliPrm) - return err -} - -type loadPlacementBuilder struct { - log *logger.Logger - - nmSrc netmapCore.Source - - cnrSrc containerCore.Source -} - -func (l *loadPlacementBuilder) BuildPlacement(epoch uint64, cnr cid.ID) ([][]netmap.NodeInfo, error) { - cnrNodes, nm, err := l.buildPlacement(epoch, cnr) - if err != nil { - return nil, err - } - - const pivotPrefix = "load_announcement_" - - pivot := []byte( - pivotPrefix + strconv.FormatUint(epoch, 10), - ) - - placement, err := nm.PlacementVectors(cnrNodes, pivot) - if err != nil { - return nil, fmt.Errorf("could not build placement vectors: %w", err) - } - - return placement, nil -} - -func (l *loadPlacementBuilder) buildPlacement(epoch uint64, idCnr cid.ID) ([][]netmap.NodeInfo, *netmap.NetMap, error) { - cnr, err := l.cnrSrc.Get(idCnr) - if err != nil { - return nil, nil, err - } - - nm, err := l.nmSrc.GetNetMapByEpoch(epoch) - if err != nil { - return nil, nil, fmt.Errorf("could not get network map: %w", err) - } - - binCnr := make([]byte, sha256.Size) - idCnr.Encode(binCnr) - - cnrNodes, err := nm.ContainerNodes(cnr.Value.PlacementPolicy(), binCnr) - if err != nil { - return nil, nil, fmt.Errorf("could not build container nodes: %w", err) - } - - return cnrNodes, nm, nil -} - -type localStorageLoad struct { - log *logger.Logger - - engine *engine.StorageEngine -} - -func (d *localStorageLoad) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontroller.UsedSpaceHandler) error { - idList, err := engine.ListContainers(context.TODO(), d.engine) - if err != nil { - return fmt.Errorf("list containers on engine failure: %w", err) - } - - for i := range idList { - sz, err := engine.ContainerSize(d.engine, idList[i]) - if err != nil { - d.log.Debug(logs.FrostFSNodeFailedToCalculateContainerSizeInStorageEngine, - zap.Stringer("cid", idList[i]), - zap.String("error", err.Error()), - ) - - continue - } - - d.log.Debug(logs.FrostFSNodeContainerSizeInStorageEngineCalculatedSuccessfully, - zap.Uint64("size", sz), - zap.Stringer("cid", idList[i]), - ) - - var a containerSDK.SizeEstimation - a.SetContainer(idList[i]) - a.SetValue(sz) - - if f != nil && !f(a) { - continue - } - - if err := h(a); err != nil { - return err - } - } - - return nil -} - -type usedSpaceService struct { - containerService.Server - - loadWriterProvider loadcontroller.WriterProvider - - loadPlacementBuilder *loadPlacementBuilder - - routeBuilder loadroute.Builder - - cfg *cfg -} - func (c *cfg) PublicKey() []byte { return nodeKeyFromNetmap(c) } @@ -517,125 +204,6 @@ func (c *cfg) ExternalAddresses() []string { return c.cfgNodeInfo.localInfo.ExternalAddresses() } -func (c *usedSpaceService) PublicKey() []byte { - return nodeKeyFromNetmap(c.cfg) -} - -func (c *usedSpaceService) IterateAddresses(f func(string) bool) { - c.cfg.iterateNetworkAddresses(f) -} - -func (c *usedSpaceService) NumberOfAddresses() int { - return c.cfg.addressNum() -} - -func (c *usedSpaceService) ExternalAddresses() []string { - return c.cfg.ExternalAddresses() -} - -func (c *usedSpaceService) AnnounceUsedSpace(ctx context.Context, req *containerV2.AnnounceUsedSpaceRequest) (*containerV2.AnnounceUsedSpaceResponse, error) { - var passedRoute []loadcontroller.ServerInfo - - for hdr := req.GetVerificationHeader(); hdr != nil; hdr = hdr.GetOrigin() { - passedRoute = append(passedRoute, &containerOnlyKeyRemoteServerInfo{ - key: hdr.GetBodySignature().GetKey(), - }) - } - - for left, right := 0, len(passedRoute)-1; left < right; left, right = left+1, right-1 { - passedRoute[left], passedRoute[right] = passedRoute[right], passedRoute[left] - } - - passedRoute = append(passedRoute, c) - - w, err := c.loadWriterProvider.InitWriter(passedRoute) - if err != nil { - return nil, fmt.Errorf("could not initialize container's used space writer: %w", err) - } - - var est containerSDK.SizeEstimation - - for _, aV2 := range req.GetBody().GetAnnouncements() { - err = est.ReadFromV2(aV2) - if err != nil { - return nil, fmt.Errorf("invalid size announcement: %w", err) - } - - if err := c.processLoadValue(ctx, est, passedRoute, w); err != nil { - return nil, err - } - } - - respBody := new(containerV2.AnnounceUsedSpaceResponseBody) - - resp := new(containerV2.AnnounceUsedSpaceResponse) - resp.SetBody(respBody) - - c.cfg.respSvc.SetMeta(resp) - - return resp, nil -} - -var errNodeOutsideContainer = errors.New("node outside the container") - -type containerOnlyKeyRemoteServerInfo struct { - key []byte -} - -func (i *containerOnlyKeyRemoteServerInfo) PublicKey() []byte { - return i.key -} - -func (*containerOnlyKeyRemoteServerInfo) IterateAddresses(func(string) bool) { -} - -func (*containerOnlyKeyRemoteServerInfo) NumberOfAddresses() int { - return 0 -} - -func (*containerOnlyKeyRemoteServerInfo) ExternalAddresses() []string { - return nil -} - -func (l *loadPlacementBuilder) isNodeFromContainerKey(epoch uint64, cnr cid.ID, key []byte) (bool, error) { - cnrNodes, _, err := l.buildPlacement(epoch, cnr) - if err != nil { - return false, err - } - - for i := range cnrNodes { - for j := range cnrNodes[i] { - if bytes.Equal(cnrNodes[i][j].PublicKey(), key) { - return true, nil - } - } - } - - return false, nil -} - -func (c *usedSpaceService) processLoadValue(_ context.Context, a containerSDK.SizeEstimation, - route []loadcontroller.ServerInfo, w loadcontroller.Writer) error { - fromCnr, err := c.loadPlacementBuilder.isNodeFromContainerKey(a.Epoch(), a.Container(), route[0].PublicKey()) - if err != nil { - return fmt.Errorf("could not verify that the sender belongs to the container: %w", err) - } else if !fromCnr { - return errNodeOutsideContainer - } - - err = loadroute.CheckRoute(c.routeBuilder, a, route) - if err != nil { - return fmt.Errorf("wrong route of container's used space value: %w", err) - } - - err = w.Put(a) - if err != nil { - return fmt.Errorf("could not write container's used space value: %w", err) - } - - return nil -} - // implements interface required by container service provided by morph executor. type morphContainerReader struct { eacl containerCore.EACLSource diff --git a/pkg/services/container/announcement/load/controller/calls.go b/pkg/services/container/announcement/load/controller/calls.go deleted file mode 100644 index e1ed6e496..000000000 --- a/pkg/services/container/announcement/load/controller/calls.go +++ /dev/null @@ -1,307 +0,0 @@ -package loadcontroller - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - "go.uber.org/zap" -) - -// StartPrm groups the required parameters of the Controller.Start method. -type StartPrm struct { - // Epoch number by which you want to select - // the values of the used space of containers. - Epoch uint64 -} - -type commonContext struct { - epoch uint64 - - ctrl *Controller - - log *logger.Logger -} - -type announcer struct { - commonContext -} - -// Start starts the processing of container.SizeEstimation values. -// -// Single Start operation overtakes all data from LocalMetrics to -// LocalAnnouncementTarget (Controller's parameters). -// No filter by epoch is used for the iterator, since it is expected -// that the source of metrics does not track the change of epochs. -// -// Each call acquires an announcement context for an Epoch parameter. -// At the very end of the operation, the context is released. -func (c *Controller) Start(ctx context.Context, prm StartPrm) { - var announcer *announcer - // acquire announcement - ctx, announcer = c.acquireAnnouncement(ctx, prm) - if announcer == nil { - return - } - - // finally stop and free the announcement - defer announcer.freeAnnouncement() - - // announce local values - announcer.announce(ctx) -} - -func (c *announcer) announce(ctx context.Context) { - c.log.Debug(logs.ControllerStartingToAnnounceTheValuesOfTheMetrics) - - var ( - metricsIterator Iterator - err error - ) - - // initialize iterator over locally collected metrics - metricsIterator, err = c.ctrl.prm.LocalMetrics.InitIterator() - if err != nil { - c.log.Debug(logs.ControllerCouldNotInitializeIteratorOverLocallyCollectedMetrics, - zap.String("error", err.Error()), - ) - - return - } - - // initialize target of local announcements - targetWriter, err := c.ctrl.prm.LocalAnnouncementTarget.InitWriter(nil) - if err != nil { - c.log.Debug(logs.ControllerCouldNotInitializeAnnouncementAccumulator, - zap.String("error", err.Error()), - ) - - return - } - - // iterate over all collected metrics and write them to the target - err = metricsIterator.Iterate( - func(container.SizeEstimation) bool { - return true // local metrics don't know about epochs - }, - func(a container.SizeEstimation) error { - a.SetEpoch(c.epoch) // set epoch explicitly - return targetWriter.Put(a) - }, - ) - if err != nil { - c.log.Debug(logs.ControllerIteratorOverLocallyCollectedMetricsAborted, - zap.String("error", err.Error()), - ) - - return - } - - // finish writing - err = targetWriter.Close(ctx) - if err != nil { - c.log.Debug(logs.ControllerCouldNotFinishWritingLocalAnnouncements, - zap.String("error", err.Error()), - ) - - return - } - - c.log.Debug(logs.ControllerTrustAnnouncementSuccessfullyFinished) -} - -func (c *Controller) acquireAnnouncement(ctx context.Context, prm StartPrm) (context.Context, *announcer) { - started := true - c.announceMtx.Lock() - { - if cancel := c.mAnnounceCtx[prm.Epoch]; cancel == nil { - ctx, cancel = context.WithCancel(ctx) - c.mAnnounceCtx[prm.Epoch] = cancel - started = false - } - } - c.announceMtx.Unlock() - - log := &logger.Logger{Logger: c.opts.log.With( - zap.Uint64("epoch", prm.Epoch), - )} - - if started { - log.Debug(logs.ControllerAnnouncementIsAlreadyStarted) - return ctx, nil - } - - return ctx, &announcer{ - commonContext: commonContext{ - epoch: prm.Epoch, - ctrl: c, - log: log, - }, - } -} - -func (c *commonContext) freeAnnouncement() { - var stopped bool - - c.ctrl.announceMtx.Lock() - - { - var cancel context.CancelFunc - - cancel, stopped = c.ctrl.mAnnounceCtx[c.epoch] - - if stopped { - cancel() - delete(c.ctrl.mAnnounceCtx, c.epoch) - } - } - - c.ctrl.announceMtx.Unlock() - - if stopped { - c.log.Debug(logs.ControllerAnnouncementSuccessfullyInterrupted) - } else { - c.log.Debug(logs.ControllerAnnouncementIsNotStartedOrAlreadyInterrupted) - } -} - -// StopPrm groups the required parameters of the Controller.Stop method. -type StopPrm struct { - // Epoch number the analysis of the values of which must be interrupted. - Epoch uint64 -} - -type reporter struct { - commonContext -} - -// Stop interrupts the processing of container.SizeEstimation values. -// -// Single Stop operation releases an announcement context and overtakes -// all data from AnnouncementAccumulator to ResultReceiver (Controller's -// parameters). Only values for the specified Epoch parameter are processed. -// -// Each call acquires a report context for an Epoch parameter. -// At the very end of the operation, the context is released. -func (c *Controller) Stop(ctx context.Context, prm StopPrm) { - var reporter *reporter - ctx, reporter = c.acquireReport(ctx, prm) - if reporter == nil { - return - } - - // finally stop and free reporting - defer reporter.freeReport() - - // interrupt announcement - reporter.freeAnnouncement() - - // report the estimations - reporter.report(ctx) -} - -func (c *Controller) acquireReport(ctx context.Context, prm StopPrm) (context.Context, *reporter) { - started := true - - c.reportMtx.Lock() - { - if cancel := c.mReportCtx[prm.Epoch]; cancel == nil { - ctx, cancel = context.WithCancel(ctx) - c.mReportCtx[prm.Epoch] = cancel - started = false - } - } - - c.reportMtx.Unlock() - - log := &logger.Logger{Logger: c.opts.log.With( - zap.Uint64("epoch", prm.Epoch), - )} - - if started { - log.Debug(logs.ControllerReportIsAlreadyStarted) - return ctx, nil - } - - return ctx, &reporter{ - commonContext: commonContext{ - epoch: prm.Epoch, - ctrl: c, - log: log, - }, - } -} - -func (c *commonContext) freeReport() { - var stopped bool - - c.ctrl.reportMtx.Lock() - - { - var cancel context.CancelFunc - - cancel, stopped = c.ctrl.mReportCtx[c.epoch] - - if stopped { - cancel() - delete(c.ctrl.mReportCtx, c.epoch) - } - } - - c.ctrl.reportMtx.Unlock() - - if stopped { - c.log.Debug(logs.ControllerAnnouncementSuccessfullyInterrupted) - } else { - c.log.Debug(logs.ControllerAnnouncementIsNotStartedOrAlreadyInterrupted) - } -} - -func (c *reporter) report(ctx context.Context) { - var ( - localIterator Iterator - err error - ) - - // initialize iterator over locally accumulated announcements - localIterator, err = c.ctrl.prm.AnnouncementAccumulator.InitIterator() - if err != nil { - c.log.Debug(logs.ControllerCouldNotInitializeIteratorOverLocallyAccumulatedAnnouncements, - zap.String("error", err.Error()), - ) - - return - } - - // initialize final destination of load estimations - resultWriter, err := c.ctrl.prm.ResultReceiver.InitWriter(nil) - if err != nil { - c.log.Debug(logs.ControllerCouldNotInitializeResultTarget, - zap.String("error", err.Error()), - ) - - return - } - - // iterate over all accumulated announcements and write them to the target - err = localIterator.Iterate( - usedSpaceFilterEpochEQ(c.epoch), - resultWriter.Put, - ) - if err != nil { - c.log.Debug(logs.ControllerIteratorOverLocalAnnouncementsAborted, - zap.String("error", err.Error()), - ) - - return - } - - // finish writing - err = resultWriter.Close(ctx) - if err != nil { - c.log.Debug(logs.ControllerCouldNotFinishWritingLoadEstimations, - zap.String("error", err.Error()), - ) - } -} diff --git a/pkg/services/container/announcement/load/controller/calls_test.go b/pkg/services/container/announcement/load/controller/calls_test.go deleted file mode 100644 index 6ca24e869..000000000 --- a/pkg/services/container/announcement/load/controller/calls_test.go +++ /dev/null @@ -1,192 +0,0 @@ -package loadcontroller_test - -import ( - "context" - "math/rand" - "sync" - "testing" - - loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "github.com/stretchr/testify/require" -) - -type testAnnouncementStorage struct { - w loadcontroller.Writer - - i loadcontroller.Iterator - - mtx sync.RWMutex - - m map[uint64][]container.SizeEstimation -} - -func newTestStorage() *testAnnouncementStorage { - return &testAnnouncementStorage{ - m: make(map[uint64][]container.SizeEstimation), - } -} - -func (s *testAnnouncementStorage) InitIterator() (loadcontroller.Iterator, error) { - if s.i != nil { - return s.i, nil - } - - return s, nil -} - -func (s *testAnnouncementStorage) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontroller.UsedSpaceHandler) error { - s.mtx.RLock() - defer s.mtx.RUnlock() - - for _, v := range s.m { - for _, a := range v { - if f(a) { - if err := h(a); err != nil { - return err - } - } - } - } - - return nil -} - -func (s *testAnnouncementStorage) InitWriter([]loadcontroller.ServerInfo) (loadcontroller.Writer, error) { - if s.w != nil { - return s.w, nil - } - - return s, nil -} - -func (s *testAnnouncementStorage) Put(v container.SizeEstimation) error { - s.mtx.Lock() - s.m[v.Epoch()] = append(s.m[v.Epoch()], v) - s.mtx.Unlock() - - return nil -} - -func (s *testAnnouncementStorage) Close(context.Context) error { - return nil -} - -func randAnnouncement() (a container.SizeEstimation) { - a.SetContainer(cidtest.ID()) - a.SetValue(rand.Uint64()) - - return -} - -func TestSimpleScenario(t *testing.T) { - // create storage to write final estimations - resultStorage := newTestStorage() - - // create storages to accumulate announcements - accumulatingStorageN2 := newTestStorage() - - // create storage of local metrics - localStorageN1 := newTestStorage() - localStorageN2 := newTestStorage() - - // create 2 controllers: 1st writes announcements to 2nd, 2nd directly to final destination - ctrlN1 := loadcontroller.New(loadcontroller.Prm{ - LocalMetrics: localStorageN1, - AnnouncementAccumulator: newTestStorage(), - LocalAnnouncementTarget: &testAnnouncementStorage{ - w: accumulatingStorageN2, - }, - ResultReceiver: resultStorage, - }) - - ctrlN2 := loadcontroller.New(loadcontroller.Prm{ - LocalMetrics: localStorageN2, - AnnouncementAccumulator: accumulatingStorageN2, - LocalAnnouncementTarget: &testAnnouncementStorage{ - w: resultStorage, - }, - ResultReceiver: resultStorage, - }) - - const processEpoch uint64 = 10 - - const goodNum = 4 - - // create 2 random values for processing epoch and 1 for some different - announces := make([]container.SizeEstimation, 0, goodNum) - - for i := 0; i < goodNum; i++ { - a := randAnnouncement() - a.SetEpoch(processEpoch) - - announces = append(announces, a) - } - - // store one half of "good" announcements to 1st metrics storage, another - to 2nd - // and "bad" to both - for i := 0; i < goodNum/2; i++ { - require.NoError(t, localStorageN1.Put(announces[i])) - } - - for i := goodNum / 2; i < goodNum; i++ { - require.NoError(t, localStorageN2.Put(announces[i])) - } - - wg := new(sync.WaitGroup) - wg.Add(2) - - startPrm := loadcontroller.StartPrm{ - Epoch: processEpoch, - } - - // start both controllers - go func() { - ctrlN1.Start(context.Background(), startPrm) - wg.Done() - }() - - go func() { - ctrlN2.Start(context.Background(), startPrm) - wg.Done() - }() - - wg.Wait() - wg.Add(2) - - stopPrm := loadcontroller.StopPrm{ - Epoch: processEpoch, - } - - // stop both controllers - go func() { - ctrlN1.Stop(context.Background(), stopPrm) - wg.Done() - }() - - go func() { - ctrlN2.Stop(context.Background(), stopPrm) - wg.Done() - }() - - wg.Wait() - - // result target should contain all "good" announcements and shoult not container the "bad" one - var res []container.SizeEstimation - - err := resultStorage.Iterate( - func(a container.SizeEstimation) bool { - return true - }, - func(a container.SizeEstimation) error { - res = append(res, a) - return nil - }, - ) - require.NoError(t, err) - - for i := range announces { - require.Contains(t, res, announces[i]) - } -} diff --git a/pkg/services/container/announcement/load/controller/controller.go b/pkg/services/container/announcement/load/controller/controller.go deleted file mode 100644 index ef6dbade7..000000000 --- a/pkg/services/container/announcement/load/controller/controller.go +++ /dev/null @@ -1,94 +0,0 @@ -package loadcontroller - -import ( - "context" - "fmt" - "sync" -) - -// Prm groups the required parameters of the Controller's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Iterator over the used space values of the containers - // collected by the node locally. - LocalMetrics IteratorProvider - - // Place of recording the local values of - // the used space of containers. - LocalAnnouncementTarget WriterProvider - - // Iterator over the summarized used space scores - // from the various network participants. - AnnouncementAccumulator IteratorProvider - - // Place of recording the final estimates of - // the used space of containers. - ResultReceiver WriterProvider -} - -// Controller represents main handler for starting -// and interrupting container volume estimation. -// -// It binds the interfaces of the local value stores -// to the target storage points. Controller is abstracted -// from the internal storage device and the network location -// of the connecting components. At its core, it is a -// high-level start-stop trigger for calculations. -// -// For correct operation, the controller must be created -// using the constructor (New) based on the required parameters -// and optional components. After successful creation, -// the constructor is immediately ready to work through -// API of external control of calculations and data transfer. -type Controller struct { - prm Prm - - opts *options - - announceMtx sync.Mutex - mAnnounceCtx map[uint64]context.CancelFunc - - reportMtx sync.Mutex - mReportCtx map[uint64]context.CancelFunc -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the Controller. -// -// Panics if at least one value of the parameters is invalid. -// -// The created Controller does not require additional -// initialization and is completely ready for work. -func New(prm Prm, opts ...Option) *Controller { - switch { - case prm.LocalMetrics == nil: - panicOnPrmValue("LocalMetrics", prm.LocalMetrics) - case prm.AnnouncementAccumulator == nil: - panicOnPrmValue("AnnouncementAccumulator", prm.AnnouncementAccumulator) - case prm.LocalAnnouncementTarget == nil: - panicOnPrmValue("LocalAnnouncementTarget", prm.LocalAnnouncementTarget) - case prm.ResultReceiver == nil: - panicOnPrmValue("ResultReceiver", prm.ResultReceiver) - } - - o := defaultOpts() - - for _, opt := range opts { - opt(o) - } - - return &Controller{ - prm: prm, - opts: o, - mAnnounceCtx: make(map[uint64]context.CancelFunc), - mReportCtx: make(map[uint64]context.CancelFunc), - } -} diff --git a/pkg/services/container/announcement/load/controller/deps.go b/pkg/services/container/announcement/load/controller/deps.go deleted file mode 100644 index 99da8594f..000000000 --- a/pkg/services/container/announcement/load/controller/deps.go +++ /dev/null @@ -1,103 +0,0 @@ -package loadcontroller - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" -) - -// UsedSpaceHandler describes the signature of the container.SizeEstimation -// value handling function. -// -// Termination of processing without failures is usually signaled -// with a zero error, while a specific value may describe the reason -// for failure. -type UsedSpaceHandler func(container.SizeEstimation) error - -// UsedSpaceFilter describes the signature of the function for -// checking whether a value meets a certain criterion. -// -// Return of true means conformity, false - vice versa. -type UsedSpaceFilter func(container.SizeEstimation) bool - -// Iterator is a group of methods provided by entity -// which can iterate over a group of container.SizeEstimation values. -type Iterator interface { - // Iterate must start an iterator over values that - // meet the filter criterion (returns true). - // For each such value should call a handler, the error - // of which should be directly returned from the method. - // - // Internal failures of the iterator are also signaled via - // an error. After a successful call to the last value - // handler, nil should be returned. - Iterate(UsedSpaceFilter, UsedSpaceHandler) error -} - -// IteratorProvider is a group of methods provided -// by entity which generates iterators over -// container.SizeEstimation values. -type IteratorProvider interface { - // InitIterator should return an initialized Iterator. - // - // Initialization problems are reported via error. - // If no error was returned, then the Iterator must not be nil. - // - // Implementations can have different logic for different - // contexts, so specific ones may document their own behavior. - InitIterator() (Iterator, error) -} - -// Writer describes the interface for storing container.SizeEstimation values. -// -// This interface is provided by both local storage -// of values and remote (wrappers over the RPC). -type Writer interface { - // Put performs a write operation of container.SizeEstimation value - // and returns any error encountered. - // - // All values after the Close call must be flushed to the - // physical target. Implementations can cache values before - // Close operation. - // - // Put must not be called after Close. - Put(container.SizeEstimation) error - - // Close exits with method-providing Writer. - // - // All cached values must be flushed before - // the Close's return. - // - // Methods must not be called after Close. - Close(ctx context.Context) error -} - -// WriterProvider is a group of methods provided -// by entity which generates keepers of -// container.SizeEstimation values. -type WriterProvider interface { - // InitWriter should return an initialized Writer. - // - // Initialization problems are reported via error. - // If no error was returned, then the Writer must not be nil. - InitWriter(route []ServerInfo) (Writer, error) -} - -// ServerInfo describes a set of -// characteristics of a point in a route. -type ServerInfo interface { - // PublicKey returns public key of the node - // from the route in a binary representation. - PublicKey() []byte - - // Iterates over network addresses of the node - // in the route. Breaks iterating on true return - // of the handler. - IterateAddresses(func(string) bool) - - // Returns number of server's network addresses. - NumberOfAddresses() int - - // ExternalAddresses returns external node's addresses. - ExternalAddresses() []string -} diff --git a/pkg/services/container/announcement/load/controller/opts.go b/pkg/services/container/announcement/load/controller/opts.go deleted file mode 100644 index 29148def0..000000000 --- a/pkg/services/container/announcement/load/controller/opts.go +++ /dev/null @@ -1,28 +0,0 @@ -package loadcontroller - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// Option sets an optional parameter of Controller. -type Option func(*options) - -type options struct { - log *logger.Logger -} - -func defaultOpts() *options { - return &options{ - log: &logger.Logger{Logger: zap.L()}, - } -} - -// WithLogger returns option to specify logging component. -func WithLogger(l *logger.Logger) Option { - return func(o *options) { - if l != nil { - o.log = l - } - } -} diff --git a/pkg/services/container/announcement/load/controller/util.go b/pkg/services/container/announcement/load/controller/util.go deleted file mode 100644 index 223de13ba..000000000 --- a/pkg/services/container/announcement/load/controller/util.go +++ /dev/null @@ -1,36 +0,0 @@ -package loadcontroller - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" -) - -func usedSpaceFilterEpochEQ(epoch uint64) UsedSpaceFilter { - return func(a container.SizeEstimation) bool { - return a.Epoch() == epoch - } -} - -type storageWrapper struct { - w Writer - i Iterator -} - -func (s storageWrapper) InitIterator() (Iterator, error) { - return s.i, nil -} - -func (s storageWrapper) InitWriter([]ServerInfo) (Writer, error) { - return s.w, nil -} - -func SimpleIteratorProvider(i Iterator) IteratorProvider { - return &storageWrapper{ - i: i, - } -} - -func SimpleWriterProvider(w Writer) WriterProvider { - return &storageWrapper{ - w: w, - } -} diff --git a/pkg/services/container/announcement/load/route/calls.go b/pkg/services/container/announcement/load/route/calls.go deleted file mode 100644 index 9a483aed0..000000000 --- a/pkg/services/container/announcement/load/route/calls.go +++ /dev/null @@ -1,145 +0,0 @@ -package loadroute - -import ( - "context" - "encoding/hex" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - "go.uber.org/zap" -) - -// InitWriter initializes and returns Writer that sends each value to its next route point. -// -// If route is present, then it is taken into account, -// and the value will be sent to its continuation. Otherwise, the route will be laid -// from scratch and the value will be sent to its primary point. -// -// After building a list of remote points of the next leg of the route, the value is sent -// sequentially to all of them. If any transmissions (even all) fail, an error will not -// be returned. -// -// Close of the composed Writer calls Close method on each internal Writer generated in -// runtime and never returns an error. -// -// Always returns nil error. -func (r *Router) InitWriter(route []loadcontroller.ServerInfo) (loadcontroller.Writer, error) { - if len(route) == 0 { - route = []loadcontroller.ServerInfo{r.localSrvInfo} - } - - return &loadWriter{ - router: r, - route: route, - mRoute: make(map[routeKey]*valuesRoute), - mServers: make(map[string]loadcontroller.Writer), - }, nil -} - -type routeKey struct { - epoch uint64 - - cid string -} - -type valuesRoute struct { - route []loadcontroller.ServerInfo - - values []container.SizeEstimation -} - -type loadWriter struct { - router *Router - - route []loadcontroller.ServerInfo - - routeMtx sync.RWMutex - mRoute map[routeKey]*valuesRoute - - mServers map[string]loadcontroller.Writer -} - -func (w *loadWriter) Put(a container.SizeEstimation) error { - w.routeMtx.Lock() - defer w.routeMtx.Unlock() - - key := routeKey{ - epoch: a.Epoch(), - cid: a.Container().EncodeToString(), - } - - routeValues, ok := w.mRoute[key] - if !ok { - route, err := w.router.routeBuilder.NextStage(a, w.route) - if err != nil { - return err - } else if len(route) == 0 { - route = []loadcontroller.ServerInfo{nil} - } - - routeValues = &valuesRoute{ - route: route, - values: []container.SizeEstimation{a}, - } - - w.mRoute[key] = routeValues - } - - for _, remoteInfo := range routeValues.route { - var key string - - if remoteInfo != nil { - key = hex.EncodeToString(remoteInfo.PublicKey()) - } - - remoteWriter, ok := w.mServers[key] - if !ok { - provider, err := w.router.remoteProvider.InitRemote(remoteInfo) - if err != nil { - w.router.log.Debug(logs.RouteCouldNotInitializeWriterProvider, - zap.String("error", err.Error()), - ) - - continue // best effort - } - - remoteWriter, err = provider.InitWriter(w.route) - if err != nil { - w.router.log.Debug(logs.RouteCouldNotInitializeWriter, - zap.String("error", err.Error()), - ) - - continue // best effort - } - - w.mServers[key] = remoteWriter - } - - err := remoteWriter.Put(a) - if err != nil { - w.router.log.Debug(logs.RouteCouldNotPutTheValue, - zap.String("error", err.Error()), - ) - } - - // continue best effort - } - - return nil -} - -func (w *loadWriter) Close(ctx context.Context) error { - for key, wRemote := range w.mServers { - err := wRemote.Close(ctx) - if err != nil { - w.router.log.Debug(logs.RouteCouldNotCloseRemoteServerWriter, - zap.String("key", key), - zap.String("error", err.Error()), - ) - } - } - - return nil -} diff --git a/pkg/services/container/announcement/load/route/deps.go b/pkg/services/container/announcement/load/route/deps.go deleted file mode 100644 index b255900f7..000000000 --- a/pkg/services/container/announcement/load/route/deps.go +++ /dev/null @@ -1,31 +0,0 @@ -package loadroute - -import ( - loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" -) - -// Builder groups methods to route values in the network. -type Builder interface { - // NextStage must return next group of route points for the value a - // based on the passed route. - // - // Empty passed list means being at the starting point of the route. - // - // Must return empty list and no error if the endpoint of the route is reached. - // If there are more than one point to go and the last passed point is included - // in that list (means that point is the last point in one of the route groups), - // returned route must contain nil point that should be interpreted as signal to, - // among sending to other route points, save the announcement in that point. - NextStage(a container.SizeEstimation, passed []loadcontroller.ServerInfo) ([]loadcontroller.ServerInfo, error) -} - -// RemoteWriterProvider describes the component -// for sending values to a fixed route point. -type RemoteWriterProvider interface { - // InitRemote must return WriterProvider to the route point - // corresponding to info. - // - // Nil info matches the end of the route. - InitRemote(info loadcontroller.ServerInfo) (loadcontroller.WriterProvider, error) -} diff --git a/pkg/services/container/announcement/load/route/opts.go b/pkg/services/container/announcement/load/route/opts.go deleted file mode 100644 index ab140ab4c..000000000 --- a/pkg/services/container/announcement/load/route/opts.go +++ /dev/null @@ -1,28 +0,0 @@ -package loadroute - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// Option sets an optional parameter of Router. -type Option func(*options) - -type options struct { - log *logger.Logger -} - -func defaultOpts() *options { - return &options{ - log: &logger.Logger{Logger: zap.L()}, - } -} - -// WithLogger returns Option to specify logging component. -func WithLogger(l *logger.Logger) Option { - return func(o *options) { - if l != nil { - o.log = l - } - } -} diff --git a/pkg/services/container/announcement/load/route/placement/builder.go b/pkg/services/container/announcement/load/route/placement/builder.go deleted file mode 100644 index 493b89723..000000000 --- a/pkg/services/container/announcement/load/route/placement/builder.go +++ /dev/null @@ -1,49 +0,0 @@ -package placementrouter - -import "fmt" - -// Prm groups the required parameters of the Builder's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Calculator of the container members. - // - // Must not be nil. - PlacementBuilder PlacementBuilder -} - -// Builder represents component that routes used container space -// values between nodes from the container. -// -// For correct operation, Builder must be created using -// the constructor (New) based on the required parameters -// and optional components. After successful creation, -// the Builder is immediately ready to work through API. -type Builder struct { - placementBuilder PlacementBuilder -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the Builder. -// -// Panics if at least one value of the parameters is invalid. -// -// The created Builder does not require additional -// initialization and is completely ready for work. -func New(prm Prm) *Builder { - switch { - case prm.PlacementBuilder == nil: - panicOnPrmValue("PlacementBuilder", prm.PlacementBuilder) - } - - return &Builder{ - placementBuilder: prm.PlacementBuilder, - } -} diff --git a/pkg/services/container/announcement/load/route/placement/calls.go b/pkg/services/container/announcement/load/route/placement/calls.go deleted file mode 100644 index 68bdb43a7..000000000 --- a/pkg/services/container/announcement/load/route/placement/calls.go +++ /dev/null @@ -1,47 +0,0 @@ -package placementrouter - -import ( - "bytes" - "fmt" - - netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" -) - -// NextStage composes container nodes for the container and epoch from a, -// and returns the list of nodes with maximum weight (one from each vector). -// -// If passed route has more than one point, then endpoint of the route is reached. -// -// The traversed route is not checked, it is assumed to be correct. -func (b *Builder) NextStage(a container.SizeEstimation, passed []loadcontroller.ServerInfo) ([]loadcontroller.ServerInfo, error) { - if len(passed) > 1 { - return nil, nil - } - - cnr := a.Container() - - placement, err := b.placementBuilder.BuildPlacement(a.Epoch(), cnr) - if err != nil { - return nil, fmt.Errorf("could not build placement %s: %w", cnr, err) - } - - res := make([]loadcontroller.ServerInfo, 0, len(placement)) - - for i := range placement { - if len(placement[i]) == 0 { - continue - } - - if len(passed) == 1 && bytes.Equal(passed[0].PublicKey(), placement[i][0].PublicKey()) { - // add nil element so the announcement will be saved in local memory - res = append(res, nil) - } else { - // add element with remote node to send announcement to - res = append(res, netmapcore.Node(placement[i][0])) - } - } - - return res, nil -} diff --git a/pkg/services/container/announcement/load/route/placement/deps.go b/pkg/services/container/announcement/load/route/placement/deps.go deleted file mode 100644 index 43339eb47..000000000 --- a/pkg/services/container/announcement/load/route/placement/deps.go +++ /dev/null @@ -1,14 +0,0 @@ -package placementrouter - -import ( - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" -) - -// PlacementBuilder describes interface of FrostFS placement calculator. -type PlacementBuilder interface { - // BuildPlacement must compose and sort (according to a specific algorithm) - // storage nodes from the container by its identifier using network map - // of particular epoch. - BuildPlacement(epoch uint64, cnr cid.ID) ([][]netmap.NodeInfo, error) -} diff --git a/pkg/services/container/announcement/load/route/router.go b/pkg/services/container/announcement/load/route/router.go deleted file mode 100644 index c8f784b16..000000000 --- a/pkg/services/container/announcement/load/route/router.go +++ /dev/null @@ -1,87 +0,0 @@ -package loadroute - -import ( - "fmt" - - loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" -) - -// Prm groups the required parameters of the Router's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Characteristics of the local node's server. - // - // Must not be nil. - LocalServerInfo loadcontroller.ServerInfo - - // Component for sending values to a fixed route point. - // - // Must not be nil. - RemoteWriterProvider RemoteWriterProvider - - // Route planner. - // - // Must not be nil. - Builder Builder -} - -// Router represents component responsible for routing -// used container space values over the network. -// -// For each fixed pair (container ID, epoch) there is a -// single value route on the network. Router provides the -// interface for writing values to the next point of the route. -// -// For correct operation, Router must be created using -// the constructor (New) based on the required parameters -// and optional components. After successful creation, -// the Router is immediately ready to work through API. -type Router struct { - log *logger.Logger - - remoteProvider RemoteWriterProvider - - routeBuilder Builder - - localSrvInfo loadcontroller.ServerInfo -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the Router. -// -// Panics if at least one value of the parameters is invalid. -// -// The created Router does not require additional -// initialization and is completely ready for work. -func New(prm Prm, opts ...Option) *Router { - switch { - case prm.RemoteWriterProvider == nil: - panicOnPrmValue("RemoteWriterProvider", prm.RemoteWriterProvider) - case prm.Builder == nil: - panicOnPrmValue("Builder", prm.Builder) - case prm.LocalServerInfo == nil: - panicOnPrmValue("LocalServerInfo", prm.LocalServerInfo) - } - - o := defaultOpts() - - for i := range opts { - opts[i](o) - } - - return &Router{ - log: o.log, - remoteProvider: prm.RemoteWriterProvider, - routeBuilder: prm.Builder, - localSrvInfo: prm.LocalServerInfo, - } -} diff --git a/pkg/services/container/announcement/load/route/util.go b/pkg/services/container/announcement/load/route/util.go deleted file mode 100644 index ea0f51aad..000000000 --- a/pkg/services/container/announcement/load/route/util.go +++ /dev/null @@ -1,49 +0,0 @@ -package loadroute - -import ( - "bytes" - "errors" - - loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" -) - -var errWrongRoute = errors.New("wrong route") - -// CheckRoute checks if the route is a route correctly constructed by the builder for value a. -// -// Returns nil if route is correct, otherwise an error clarifying the inconsistency. -func CheckRoute(builder Builder, a container.SizeEstimation, route []loadcontroller.ServerInfo) error { - for i := 1; i < len(route); i++ { - servers, err := builder.NextStage(a, route[:i]) - if err != nil { - return err - } else if len(servers) == 0 { - break - } - - found := false - - for j := range servers { - if servers[j] == nil { - // nil route point means that - // (i-1)-th node in the route - // must, among other things, - // save the announcement to its - // local memory - continue - } - - if bytes.Equal(servers[j].PublicKey(), route[i].PublicKey()) { - found = true - break - } - } - - if !found { - return errWrongRoute - } - } - - return nil -} diff --git a/pkg/services/container/announcement/load/storage/storage.go b/pkg/services/container/announcement/load/storage/storage.go deleted file mode 100644 index 4d3104c76..000000000 --- a/pkg/services/container/announcement/load/storage/storage.go +++ /dev/null @@ -1,151 +0,0 @@ -package loadstorage - -import ( - "context" - "sort" - "sync" - - loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" -) - -type usedSpaceEstimations struct { - announcement container.SizeEstimation - - sizes []uint64 -} - -type storageKey struct { - epoch uint64 - - cid string -} - -// Storage represents in-memory storage of -// container.SizeEstimation values. -// -// The write operation has the usual behavior - to save -// the next number of used container space for a specific epoch. -// All values related to one key (epoch, container ID) are stored -// as a list. -// -// Storage also provides an iterator interface, into the handler -// of which the final score is passed, built on all values saved -// at the time of the call. Currently the only possible estimation -// formula is used - the average between 10th and 90th percentile. -// -// For correct operation, Storage must be created -// using the constructor (New) based on the required parameters -// and optional components. After successful creation, -// Storage is immediately ready to work through API. -type Storage struct { - mtx sync.RWMutex - - mItems map[storageKey]*usedSpaceEstimations -} - -// Prm groups the required parameters of the Storage's constructor. -// -// The component is not parameterizable at the moment. -type Prm struct{} - -// New creates a new instance of the Storage. -// -// The created Storage does not require additional -// initialization and is completely ready for work. -func New(_ Prm) *Storage { - return &Storage{ - mItems: make(map[storageKey]*usedSpaceEstimations), - } -} - -// Put appends the next value of the occupied container space for the epoch -// to the list of already saved values. -// -// Always returns nil error. -func (s *Storage) Put(a container.SizeEstimation) error { - s.mtx.Lock() - - { - key := storageKey{ - epoch: a.Epoch(), - cid: a.Container().EncodeToString(), - } - - estimations, ok := s.mItems[key] - if !ok { - estimations = &usedSpaceEstimations{ - announcement: a, - sizes: make([]uint64, 0, 1), - } - - s.mItems[key] = estimations - } - - estimations.sizes = append(estimations.sizes, a.Value()) - } - - s.mtx.Unlock() - - return nil -} - -func (s *Storage) Close(context.Context) error { - return nil -} - -// Iterate goes through all the lists with the key (container ID, epoch), -// calculates the final grade for all values, and passes it to the handler. -// -// Final grade is the average between 10th and 90th percentiles. -func (s *Storage) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontroller.UsedSpaceHandler) (err error) { - s.mtx.RLock() - - { - for _, v := range s.mItems { - if f(v.announcement) { - // calculate estimation based on 90th percentile - v.announcement.SetValue(finalEstimation(v.sizes)) - - if err = h(v.announcement); err != nil { - break - } - } - } - } - - s.mtx.RUnlock() - - return -} - -func finalEstimation(vals []uint64) uint64 { - sort.Slice(vals, func(i, j int) bool { - return vals[i] < vals[j] - }) - - const ( - lowerRank = 10 - upperRank = 90 - ) - - if len(vals) >= lowerRank { - lowerInd := percentile(lowerRank, vals) - upperInd := percentile(upperRank, vals) - - vals = vals[lowerInd:upperInd] - } - - sum := uint64(0) - - for i := range vals { - sum += vals[i] - } - - return sum / uint64(len(vals)) -} - -func percentile(rank int, vals []uint64) int { - p := len(vals) * rank / 100 - return p -} diff --git a/pkg/services/container/announcement/load/storage/storage_test.go b/pkg/services/container/announcement/load/storage/storage_test.go deleted file mode 100644 index 20e73627d..000000000 --- a/pkg/services/container/announcement/load/storage/storage_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package loadstorage - -import ( - "math/rand" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - "github.com/stretchr/testify/require" -) - -func TestStorage(t *testing.T) { - const epoch uint64 = 13 - - var a container.SizeEstimation - a.SetContainer(cidtest.ID()) - a.SetEpoch(epoch) - - const opinionsNum = 100 - - s := New(Prm{}) - - opinions := make([]uint64, opinionsNum) - for i := range opinions { - opinions[i] = rand.Uint64() - - a.SetValue(opinions[i]) - - require.NoError(t, s.Put(a)) - } - - iterCounter := 0 - - err := s.Iterate( - func(ai container.SizeEstimation) bool { - return ai.Epoch() == epoch - }, - func(ai container.SizeEstimation) error { - iterCounter++ - - require.Equal(t, epoch, ai.Epoch()) - require.Equal(t, a.Container(), ai.Container()) - require.Equal(t, finalEstimation(opinions), ai.Value()) - - return nil - }, - ) - require.NoError(t, err) - require.Equal(t, 1, iterCounter) -} From 5e229dc2481c3a745ec702f6c74353e648ce649d Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Wed, 4 Oct 2023 19:20:59 +0300 Subject: [PATCH 0603/1943] [#701] metrics: add metric to evaluate policer performance Add processed objects counter in policerMetrics, add policer field to NodeMetrics Signed-off-by: Ekaterina Lebedeva --- pkg/metrics/consts.go | 1 + pkg/metrics/node.go | 6 ++++++ pkg/metrics/policer.go | 29 +++++++++++++++++++++++++++++ pkg/services/policer/metrics.go | 5 +++++ pkg/services/policer/option.go | 2 ++ pkg/services/policer/process.go | 1 + 6 files changed, 44 insertions(+) create mode 100644 pkg/metrics/policer.go create mode 100644 pkg/services/policer/metrics.go diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index 550eec424..f7a8fd771 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -20,6 +20,7 @@ const ( treeServiceSubsystem = "treeservice" writeCacheSubsystem = "writecache" grpcServerSubsystem = "grpc_server" + policerSubsystem = "policer" successLabel = "success" shardIDLabel = "shard_id" diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index ad6864b8f..052defa4d 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -18,6 +18,7 @@ type NodeMetrics struct { metabase *metabaseMetrics pilorama *piloramaMetrics grpc *grpcServerMetrics + policer *policerMetrics } func NewNodeMetrics() *NodeMetrics { @@ -39,6 +40,7 @@ func NewNodeMetrics() *NodeMetrics { metabase: newMetabaseMetrics(), pilorama: newPiloramaMetrics(), grpc: newGrpcServerMetrics(), + policer: newPolicerMetrics(), } } @@ -90,3 +92,7 @@ func (m *NodeMetrics) PiloramaMetrics() PiloramaMetrics { func (m *NodeMetrics) GrpcServerMetrics() GrpcServerMetrics { return m.grpc } + +func (m *NodeMetrics) PolicerMetrics() PolicerMetrics { + return m.policer +} diff --git a/pkg/metrics/policer.go b/pkg/metrics/policer.go new file mode 100644 index 000000000..e4bdc944e --- /dev/null +++ b/pkg/metrics/policer.go @@ -0,0 +1,29 @@ +package metrics + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type PolicerMetrics interface { + IncProcessedObjects() +} + +type policerMetrics struct { + processedObjectsCounter prometheus.Counter +} + +func newPolicerMetrics() *policerMetrics { + return &policerMetrics{ + processedObjectsCounter: metrics.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: policerSubsystem, + Name: "processed_objects_total", + Help: "Total number of objects processed by policer", + }), + } +} + +func (m *policerMetrics) IncProcessedObjects() { + m.processedObjectsCounter.Inc() +} diff --git a/pkg/services/policer/metrics.go b/pkg/services/policer/metrics.go new file mode 100644 index 000000000..0b83f298c --- /dev/null +++ b/pkg/services/policer/metrics.go @@ -0,0 +1,5 @@ +package policer + +type MetricsRegister interface { + IncProcessedObjects() +} diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 5058b026b..3a2fdc507 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -69,6 +69,8 @@ type cfg struct { batchSize, cacheSize uint32 rebalanceFreq, evictDuration, sleepDuration time.Duration + + metrics MetricsRegister } func defaultCfg() *cfg { diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 2e8fe9295..873a1efb0 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -60,6 +60,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { } p.cache.Add(addr.Address, time.Now()) p.objsInWork.remove(addr.Address) + p.metrics.IncProcessedObjects() } }) if err != nil { From aca11d7474d0341b37a17e08a1a719bb3bd158fc Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 11 Oct 2023 15:13:28 +0300 Subject: [PATCH 0604/1943] [#735] policer: Allow to provide metrics from the outside Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/metrics.go | 4 ++++ pkg/services/policer/option.go | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/pkg/services/policer/metrics.go b/pkg/services/policer/metrics.go index 0b83f298c..c2ad2b0b5 100644 --- a/pkg/services/policer/metrics.go +++ b/pkg/services/policer/metrics.go @@ -3,3 +3,7 @@ package policer type MetricsRegister interface { IncProcessedObjects() } + +type noopMetrics struct{} + +func (noopMetrics) IncProcessedObjects() {} diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 3a2fdc507..fe47555c4 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -81,6 +81,7 @@ func defaultCfg() *cfg { rebalanceFreq: 1 * time.Second, sleepDuration: 1 * time.Second, evictDuration: 30 * time.Second, + metrics: noopMetrics{}, } } @@ -172,3 +173,10 @@ func WithPool(p *ants.Pool) Option { c.taskPool = p } } + +// WithMetrics returns option to set metrics. +func WithMetrics(m MetricsRegister) Option { + return func(c *cfg) { + c.metrics = m + } +} From 994f48f8bba365c22ca15dbba83275f87957c6a7 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 4 Oct 2023 10:03:37 +0300 Subject: [PATCH 0605/1943] [#680] metrics: Export log and morph with script Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/morph.go | 5 ++--- pkg/metrics/morph.go | 3 +-- pkg/metrics/morphcache.go | 4 ---- pkg/metrics/node.go | 13 +++++++++++++ pkg/util/logger/logger.go | 2 +- pkg/util/logger/metrics.go | 8 ++++---- scripts/export-metrics/main.go | 3 +++ 7 files changed, 24 insertions(+), 14 deletions(-) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 63d1605ef..5c3db0d27 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -9,7 +9,6 @@ import ( morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -42,13 +41,13 @@ func initMorphComponents(ctx context.Context, c *cfg) { c.key, client.WithDialTimeout(morphconfig.DialTimeout(c.appCfg)), client.WithLogger(c.log), - client.WithMetrics(metrics.NewMorphClientMetrics()), + client.WithMetrics(c.metricsCollector.MorphClientMetrics()), client.WithEndpoints(addresses...), client.WithConnLostCallback(func() { c.internalErr <- errors.New("morph connection has been lost") }), client.WithSwitchInterval(morphconfig.SwitchInterval(c.appCfg)), - client.WithMorphCacheMetrics(metrics.NewNodeMorphCacheMetrics()), + client.WithMorphCacheMetrics(c.metricsCollector.MorphCacheMetrics()), ) if err != nil { c.log.Info(logs.FrostFSNodeFailedToCreateNeoRPCClient, diff --git a/pkg/metrics/morph.go b/pkg/metrics/morph.go index 5215c674b..02d7517bc 100644 --- a/pkg/metrics/morph.go +++ b/pkg/metrics/morph.go @@ -4,7 +4,6 @@ import ( "strconv" "time" - morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -16,7 +15,7 @@ type morphClientMetrics struct { invokeDuration *prometheus.HistogramVec } -func NewMorphClientMetrics() morphmetrics.Register { +func newMorphClientMetrics() *morphClientMetrics { return &morphClientMetrics{ switchCount: metrics.NewCounter(prometheus.CounterOpts{ Namespace: namespace, diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go index a4dbbccfc..388cb11e8 100644 --- a/pkg/metrics/morphcache.go +++ b/pkg/metrics/morphcache.go @@ -18,10 +18,6 @@ type morphCacheMetrics struct { var _ MorphCacheMetrics = (*morphCacheMetrics)(nil) -func NewNodeMorphCacheMetrics() MorphCacheMetrics { - return newMorphCacheMetrics(namespace) -} - func newMorphCacheMetrics(ns string) *morphCacheMetrics { return &morphCacheMetrics{ methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 052defa4d..614300aad 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -1,6 +1,7 @@ package metrics import ( + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -19,6 +20,8 @@ type NodeMetrics struct { pilorama *piloramaMetrics grpc *grpcServerMetrics policer *policerMetrics + morphClient *morphClientMetrics + morphCache *morphCacheMetrics } func NewNodeMetrics() *NodeMetrics { @@ -41,6 +44,8 @@ func NewNodeMetrics() *NodeMetrics { pilorama: newPiloramaMetrics(), grpc: newGrpcServerMetrics(), policer: newPolicerMetrics(), + morphClient: newMorphClientMetrics(), + morphCache: newMorphCacheMetrics(namespace), } } @@ -96,3 +101,11 @@ func (m *NodeMetrics) GrpcServerMetrics() GrpcServerMetrics { func (m *NodeMetrics) PolicerMetrics() PolicerMetrics { return m.policer } + +func (m *NodeMetrics) MorphClientMetrics() morphmetrics.Register { + return m.morphClient +} + +func (m *NodeMetrics) MorphCacheMetrics() MorphCacheMetrics { + return m.morphCache +} diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index fcac09321..cd4297767 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -82,7 +82,7 @@ func NewLogger(prm *Prm) (*Logger, error) { lvl := zap.NewAtomicLevelAt(prm.level) - m := newLogMetrics(prm.MetricsNamespace) + m := NewLogMetrics(prm.MetricsNamespace) c := zap.NewProductionConfig() c.Level = lvl diff --git a/pkg/util/logger/metrics.go b/pkg/util/logger/metrics.go index 708583473..1fa0669a2 100644 --- a/pkg/util/logger/metrics.go +++ b/pkg/util/logger/metrics.go @@ -14,12 +14,12 @@ const ( logDroppedLabel = "dropped" ) -type logMetrics struct { +type LogMetrics struct { logCount *prometheus.CounterVec } -func newLogMetrics(namespace string) *logMetrics { - return &logMetrics{ +func NewLogMetrics(namespace string) *LogMetrics { + return &LogMetrics{ logCount: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: logSubsystem, @@ -29,7 +29,7 @@ func newLogMetrics(namespace string) *logMetrics { } } -func (m *logMetrics) Inc(level zapcore.Level, dropped bool) { +func (m *LogMetrics) Inc(level zapcore.Level, dropped bool) { m.logCount.With(prometheus.Labels{ logLevelLabel: level.String(), logDroppedLabel: strconv.FormatBool(dropped), diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go index f29eca37c..45b155260 100644 --- a/scripts/export-metrics/main.go +++ b/scripts/export-metrics/main.go @@ -7,6 +7,7 @@ import ( "os" local_metrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) @@ -27,9 +28,11 @@ func main() { switch { case *node != "": _ = local_metrics.NewNodeMetrics() + _ = logger.NewLogMetrics("frostfs_node") filename = *node case *ir != "": _ = local_metrics.NewInnerRingMetrics() + _ = logger.NewLogMetrics("frostfs_ir") filename = *ir default: From bf082348d43819df0a0a7239ae2825e000483fe6 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 5 Oct 2023 18:00:16 +0300 Subject: [PATCH 0606/1943] [#680] metrics: Add step `export-metrics` in Makefile Signed-off-by: Anton Nikiforov --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 604115469..7fa4e4317 100755 --- a/Makefile +++ b/Makefile @@ -70,6 +70,12 @@ dep: CGO_ENABLED=0 \ go mod tidy -v && echo OK +# Build export-metrics +export-metrics: dep + @printf "⇒ Build export-metrics\n" + CGO_ENABLED=0 \ + go build -v -trimpath -o bin/export-metrics ./scripts/export-metrics + # Regenerate proto files: protoc: @GOPRIVATE=github.com/TrueCloudLab go mod vendor From 3a997d120767eef9636e11b878b805d0d19ffc68 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Sun, 8 Oct 2023 12:32:00 +0300 Subject: [PATCH 0607/1943] [#680] metrics: Initialize log metrics together with services Signed-off-by: Anton Nikiforov --- cmd/frostfs-ir/main.go | 8 +++++--- cmd/frostfs-node/config.go | 37 ++++++++++++++++------------------ pkg/innerring/innerring.go | 5 +++-- pkg/metrics/innerring.go | 7 +++++++ pkg/metrics/node.go | 7 +++++++ pkg/util/logger/logger.go | 10 ++++----- pkg/util/logger/metrics.go | 19 +++++++++++++---- scripts/export-metrics/main.go | 3 --- 8 files changed, 58 insertions(+), 38 deletions(-) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 70199b094..2b39f78e3 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" + irMetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/spf13/viper" "go.uber.org/zap" @@ -61,12 +62,13 @@ func main() { cfg, err = newConfig() exitErr(err) - logPrm.MetricsNamespace = "frostfs_ir" + metrics := irMetrics.NewInnerRingMetrics() + err = logPrm.SetLevelString( cfg.GetString("logger.level"), ) exitErr(err) - + logPrm.SamplingHook = metrics.LogMetrics().GetSamplingHook() log, err = logger.NewLogger(logPrm) exitErr(err) @@ -78,7 +80,7 @@ func main() { metricsCmp = newMetricsComponent() metricsCmp.init() - innerRing, err = innerring.New(ctx, log, cfg, intErr) + innerRing, err = innerring.New(ctx, log, cfg, intErr, metrics) exitErr(err) pprofCmp.start() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 60e567c5a..b35200d27 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -554,22 +554,21 @@ func initCfg(appCfg *config.Config) *cfg { key := nodeconfig.Key(appCfg) + relayOnly := nodeconfig.Relay(appCfg) + + netState := newNetworkState() + netState.metrics = c.metricsCollector + + c.shared = initShared(appCfg, key, netState, relayOnly) + logPrm, err := c.loggerPrm() fatalOnErr(err) - - logPrm.MetricsNamespace = "frostfs_node" - + logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook() log, err := logger.NewLogger(logPrm) fatalOnErr(err) c.internals = initInternals(appCfg, log) - relayOnly := nodeconfig.Relay(appCfg) - - netState := newNetworkState() - - c.shared = initShared(appCfg, key, netState, relayOnly) - c.cfgAccounting = cfgAccounting{ scriptHash: contractsconfig.Balance(appCfg), } @@ -586,9 +585,6 @@ func initCfg(appCfg *config.Config) *cfg { user.IDFromKey(&c.ownerIDFromKey, key.PrivateKey.PublicKey) - c.metricsCollector = metrics.NewNodeMetrics() - netState.metrics = c.metricsCollector - c.onShutdown(c.clientCache.CloseAll) // clean up connections c.onShutdown(c.bgClientCache.CloseAll) // clean up connections c.onShutdown(c.putClientCache.CloseAll) // clean up connections @@ -630,14 +626,15 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt } return shared{ - key: key, - binPublicKey: key.PublicKey().Bytes(), - localAddr: netAddr, - respSvc: response.NewService(netState), - clientCache: cache.NewSDKClientCache(cacheOpts), - bgClientCache: cache.NewSDKClientCache(cacheOpts), - putClientCache: cache.NewSDKClientCache(cacheOpts), - persistate: persistate, + key: key, + binPublicKey: key.PublicKey().Bytes(), + localAddr: netAddr, + respSvc: response.NewService(netState), + clientCache: cache.NewSDKClientCache(cacheOpts), + bgClientCache: cache.NewSDKClientCache(cacheOpts), + putClientCache: cache.NewSDKClientCache(cacheOpts), + persistate: persistate, + metricsCollector: metrics.NewNodeMetrics(), } } diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 1567e40d3..6f2463c74 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -325,11 +325,12 @@ func (s *Server) registerStarter(f func() error) { } // New creates instance of inner ring sever structure. -func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error) (*Server, error) { +func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error, + metrics *metrics.InnerRingServiceMetrics) (*Server, error) { var err error server := &Server{ log: log, - irMetrics: metrics.NewInnerRingMetrics(), + irMetrics: metrics, } server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED) diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index d93b3c432..3e438a01b 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -4,6 +4,7 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -14,6 +15,7 @@ type InnerRingServiceMetrics struct { health prometheus.Gauge eventDuration *prometheus.HistogramVec morphCacheMetrics *morphCacheMetrics + logMetrics logger.LogMetrics } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. @@ -44,6 +46,7 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { health: health, eventDuration: eventDuration, morphCacheMetrics: newMorphCacheMetrics(innerRingNamespace), + logMetrics: logger.NewLogMetrics(innerRingNamespace), } } @@ -67,3 +70,7 @@ func (m *InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success func (m *InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { return m.morphCacheMetrics } + +func (m *InnerRingServiceMetrics) LogMetrics() logger.LogMetrics { + return m.logMetrics +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 614300aad..0dd86d90e 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -2,6 +2,7 @@ package metrics import ( morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -22,6 +23,7 @@ type NodeMetrics struct { policer *policerMetrics morphClient *morphClientMetrics morphCache *morphCacheMetrics + log logger.LogMetrics } func NewNodeMetrics() *NodeMetrics { @@ -46,6 +48,7 @@ func NewNodeMetrics() *NodeMetrics { policer: newPolicerMetrics(), morphClient: newMorphClientMetrics(), morphCache: newMorphCacheMetrics(namespace), + log: logger.NewLogMetrics(namespace), } } @@ -109,3 +112,7 @@ func (m *NodeMetrics) MorphClientMetrics() morphmetrics.Register { func (m *NodeMetrics) MorphCacheMetrics() MorphCacheMetrics { return m.morphCache } + +func (m *NodeMetrics) LogMetrics() logger.LogMetrics { + return m.log +} diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index cd4297767..f9a5fe38f 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -31,8 +31,8 @@ type Prm struct { // support runtime rereading level zapcore.Level - // MetricsNamespace is the namespace string used for log counter metrics - MetricsNamespace string + // SamplingHook hook for the zap.Logger + SamplingHook func(e zapcore.Entry, sd zapcore.SamplingDecision) // do not support runtime rereading } @@ -82,14 +82,12 @@ func NewLogger(prm *Prm) (*Logger, error) { lvl := zap.NewAtomicLevelAt(prm.level) - m := NewLogMetrics(prm.MetricsNamespace) - c := zap.NewProductionConfig() c.Level = lvl c.Encoding = "console" c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - c.Sampling.Hook = func(e zapcore.Entry, sd zapcore.SamplingDecision) { - m.Inc(e.Level, sd == zapcore.LogDropped) + if prm.SamplingHook != nil { + c.Sampling.Hook = prm.SamplingHook } lZap, err := c.Build( diff --git a/pkg/util/logger/metrics.go b/pkg/util/logger/metrics.go index 1fa0669a2..7e62e6383 100644 --- a/pkg/util/logger/metrics.go +++ b/pkg/util/logger/metrics.go @@ -14,12 +14,17 @@ const ( logDroppedLabel = "dropped" ) -type LogMetrics struct { +type LogMetrics interface { + Inc(level zapcore.Level, dropped bool) + GetSamplingHook() func(e zapcore.Entry, sd zapcore.SamplingDecision) +} + +type logMetrics struct { logCount *prometheus.CounterVec } -func NewLogMetrics(namespace string) *LogMetrics { - return &LogMetrics{ +func NewLogMetrics(namespace string) LogMetrics { + return &logMetrics{ logCount: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: logSubsystem, @@ -29,9 +34,15 @@ func NewLogMetrics(namespace string) *LogMetrics { } } -func (m *LogMetrics) Inc(level zapcore.Level, dropped bool) { +func (m *logMetrics) Inc(level zapcore.Level, dropped bool) { m.logCount.With(prometheus.Labels{ logLevelLabel: level.String(), logDroppedLabel: strconv.FormatBool(dropped), }).Inc() } + +func (m *logMetrics) GetSamplingHook() func(zapcore.Entry, zapcore.SamplingDecision) { + return func(e zapcore.Entry, sd zapcore.SamplingDecision) { + m.Inc(e.Level, sd == zapcore.LogDropped) + } +} diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go index 45b155260..f29eca37c 100644 --- a/scripts/export-metrics/main.go +++ b/scripts/export-metrics/main.go @@ -7,7 +7,6 @@ import ( "os" local_metrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) @@ -28,11 +27,9 @@ func main() { switch { case *node != "": _ = local_metrics.NewNodeMetrics() - _ = logger.NewLogMetrics("frostfs_node") filename = *node case *ir != "": _ = local_metrics.NewInnerRingMetrics() - _ = logger.NewLogMetrics("frostfs_ir") filename = *ir default: From d5c10612f4e679fe8d5d8edcc61ba7c6666103e4 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 12 Oct 2023 09:31:36 +0300 Subject: [PATCH 0608/1943] [#735] policer: Register metrics Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/object.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 34847e36f..82ddf897f 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -260,6 +260,7 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl }), policer.WithMaxCapacity(c.cfgObject.pool.replicatorPoolSize), policer.WithPool(c.cfgObject.pool.replication), + policer.WithMetrics(c.metricsCollector.PolicerMetrics()), ) c.workers = append(c.workers, worker{ From 7e0c5a55de8e14ee9bdf4484d186db87923383b8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 13 Oct 2023 13:50:25 +0300 Subject: [PATCH 0609/1943] [#734] writecache: Fix flush Now UpdateStorageID doesn't return error in case of logical error. If object is in graveyard or GC market, it is still required to update storage ID. Signed-off-by: Dmitrii Stepanov --- .../metabase/storage_id.go | 5 ++++- .../metabase/storage_id_test.go | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index 6ba5a60cb..7e43e30eb 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -5,6 +5,7 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/util/slice" @@ -107,7 +108,9 @@ func (db *DB) UpdateStorageID(prm UpdateStorageIDPrm) (res UpdateStorageIDRes, e err = db.boltDB.Batch(func(tx *bbolt.Tx) error { exists, err := db.exists(tx, prm.addr, currEpoch) - if err == nil && exists || errors.Is(err, ErrObjectIsExpired) { + if err == nil && exists { + err = updateStorageID(tx, prm.addr, prm.id) + } else if errors.As(err, new(logicerr.Logical)) { err = updateStorageID(tx, prm.addr, prm.id) } diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go index b3652a680..7278eec57 100644 --- a/pkg/local_object_storage/metabase/storage_id_test.go +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -18,6 +18,7 @@ func TestDB_StorageID(t *testing.T) { raw1 := testutil.GenerateObject() raw2 := testutil.GenerateObject() + deleted := testutil.GenerateObject() storageID := []byte{1, 2, 3, 4} @@ -34,6 +35,15 @@ func TestDB_StorageID(t *testing.T) { err = putBig(db, raw2) require.NoError(t, err) + // put object with storageID and delete it + err = metaPut(db, deleted, storageID) + require.NoError(t, err) + + cnrID, ok := deleted.ContainerID() + require.True(t, ok) + ts := testutil.GenerateObjectWithCID(cnrID) + require.NoError(t, metaInhume(db, object.AddressOf(deleted), object.AddressOf(ts))) + // check StorageID for object without storageID fetchedStorageID, err = metaStorageID(db, object.AddressOf(raw2)) require.NoError(t, err) @@ -44,12 +54,23 @@ func TestDB_StorageID(t *testing.T) { require.NoError(t, err) require.Equal(t, storageID, fetchedStorageID) + // check StorageID for deleted object with storageID + fetchedStorageID, err = metaStorageID(db, object.AddressOf(deleted)) + require.NoError(t, err) + require.Equal(t, storageID, fetchedStorageID) + t.Run("update", func(t *testing.T) { + storageID := []byte{1, 2, 3, 4, 5} require.NoError(t, metaUpdateStorageID(db, object.AddressOf(raw2), storageID)) + require.NoError(t, metaUpdateStorageID(db, object.AddressOf(deleted), storageID)) fetchedStorageID, err = metaStorageID(db, object.AddressOf(raw2)) require.NoError(t, err) require.Equal(t, storageID, fetchedStorageID) + + fetchedStorageID, err = metaStorageID(db, object.AddressOf(deleted)) + require.NoError(t, err) + require.Equal(t, storageID, fetchedStorageID) }) } From f26233b47a93750d52903e5141d4c145e9cf3787 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 13 Oct 2023 14:01:14 +0300 Subject: [PATCH 0610/1943] [#734] metabase: Include UpdateStorageID in metrics and traces Signed-off-by: Dmitrii Stepanov --- .../metabase/storage_id.go | 20 +++++++++++++++++-- .../metabase/storage_id_test.go | 2 +- .../writecache/benchmark/writecache_test.go | 2 +- .../writecache/writecache.go | 2 +- .../writecache/writecachebadger/flush.go | 2 +- .../writecache/writecachebbolt/flush.go | 2 +- 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index 7e43e30eb..0a64da856 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -3,6 +3,7 @@ package meta import ( "context" "errors" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -94,7 +95,22 @@ func (p *UpdateStorageIDPrm) SetStorageID(id []byte) { } // UpdateStorageID updates storage descriptor for objects from the blobstor. -func (db *DB) UpdateStorageID(prm UpdateStorageIDPrm) (res UpdateStorageIDRes, err error) { +func (db *DB) UpdateStorageID(ctx context.Context, prm UpdateStorageIDPrm) (res UpdateStorageIDRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("UpdateStorageID", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.UpdateStorageID", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + attribute.String("storage_id", string(prm.id)), + )) + defer span.End() + db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -116,6 +132,6 @@ func (db *DB) UpdateStorageID(prm UpdateStorageIDPrm) (res UpdateStorageIDRes, e return err }) - + success = err == nil return res, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go index 7278eec57..63dbc2f3f 100644 --- a/pkg/local_object_storage/metabase/storage_id_test.go +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -79,7 +79,7 @@ func metaUpdateStorageID(db *meta.DB, addr oid.Address, id []byte) error { sidPrm.SetAddress(addr) sidPrm.SetStorageID(id) - _, err := db.UpdateStorageID(sidPrm) + _, err := db.UpdateStorageID(context.Background(), sidPrm) return err } diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index 16c6d73b2..ea4bf0d33 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -91,7 +91,7 @@ func benchmarkPutPrepare(b *testing.B, cache writecache.Cache) { type testMetabase struct{} -func (testMetabase) UpdateStorageID(meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error) { +func (testMetabase) UpdateStorageID(context.Context, meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error) { return meta.UpdateStorageIDRes{}, nil } diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index c2fdc100f..3d3501969 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -51,7 +51,7 @@ type MainStorage interface { // Metabase is the interface of the metabase used by Cache implementations. type Metabase interface { - UpdateStorageID(meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error) + UpdateStorageID(context.Context, meta.UpdateStorageIDPrm) (meta.UpdateStorageIDRes, error) } var ( diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush.go b/pkg/local_object_storage/writecache/writecachebadger/flush.go index 3f2bdb043..e0333eff7 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush.go @@ -226,7 +226,7 @@ func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []b updPrm.SetAddress(addr) updPrm.SetStorageID(res.StorageID) - _, err = c.metabase.UpdateStorageID(updPrm) + _, err = c.metabase.UpdateStorageID(ctx, updPrm) if err != nil { c.reportFlushError(logs.FrostFSNodeCantUpdateObjectStorageID, addr.EncodeToString(), err) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush.go b/pkg/local_object_storage/writecache/writecachebbolt/flush.go index 89b26cd64..fd322e9fa 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush.go @@ -278,7 +278,7 @@ func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []b updPrm.SetAddress(addr) updPrm.SetStorageID(res.StorageID) - _, err = c.metabase.UpdateStorageID(updPrm) + _, err = c.metabase.UpdateStorageID(ctx, updPrm) if err != nil { c.reportFlushError(logs.FSTreeCantUpdateID, addr.EncodeToString(), err) From f2437f7ae962bae537666bc0914bc9db9a5ddb75 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 13 Oct 2023 16:28:04 +0300 Subject: [PATCH 0611/1943] [#734] shard: Fix Delete method Due to the flushing data from the writecache to the storage and simultaneous deletion, a partial deletion situation is possible. So as a solution, deletion is allowed only when the object is in storage, because object will be deleted from writecache by flush goroutine. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/delete.go | 37 ++++++++----------- pkg/local_object_storage/shard/delete_test.go | 34 +++++++++++------ pkg/local_object_storage/shard/gc.go | 2 +- 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index ea481300b..e24bd8541 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -2,15 +2,12 @@ package shard import ( "context" - "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -34,8 +31,7 @@ func (p *DeletePrm) SetAddresses(addr ...oid.Address) { p.addr = append(p.addr, addr...) } -// Delete removes data from the shard's writeCache, metaBase and -// blobStor. +// Delete removes data from the shard's metaBase and// blobStor. func (s *Shard) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Delete", trace.WithAttributes( @@ -47,10 +43,10 @@ func (s *Shard) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { s.m.RLock() defer s.m.RUnlock() - return s.delete(ctx, prm) + return s.delete(ctx, prm, false) } -func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { +func (s *Shard) delete(ctx context.Context, prm DeletePrm, skipFailed bool) (DeleteRes, error) { if s.info.Mode.ReadOnly() { return DeleteRes{}, ErrReadOnlyMode } else if s.info.Mode.NoMetabase() { @@ -65,12 +61,18 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { default: } - s.deleteObjectFromWriteCacheSafe(ctx, addr) - - s.deleteFromBlobstorSafe(ctx, addr) + if err := s.deleteFromBlobstor(ctx, addr); err != nil { + if skipFailed { + continue + } + return result, err + } if err := s.deleteFromMetabase(ctx, addr); err != nil { - return result, err // stop on metabase error ? + if skipFailed { + continue + } + return result, err } result.deleted++ } @@ -78,16 +80,7 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { return result, nil } -func (s *Shard) deleteObjectFromWriteCacheSafe(ctx context.Context, addr oid.Address) { - if s.hasWriteCache() { - err := s.writeCache.Delete(ctx, addr) - if err != nil && !client.IsErrObjectNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) { - s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.Error(err)) - } - } -} - -func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { +func (s *Shard) deleteFromBlobstor(ctx context.Context, addr oid.Address) error { var sPrm meta.StorageIDPrm sPrm.SetAddress(addr) @@ -97,6 +90,7 @@ func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { zap.Stringer("object", addr), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + return err } storageID := res.StorageID() @@ -111,6 +105,7 @@ func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) { zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } + return err } func (s *Shard) deleteFromMetabase(ctx context.Context, addr oid.Address) error { diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index 3421ac9e0..910528f84 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -52,13 +52,18 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { _, err = testGet(t, sh, getPrm, hasWriteCache) require.NoError(t, err) - _, err = sh.Delete(context.TODO(), delPrm) - require.NoError(t, err) + if hasWriteCache { + require.Eventually(t, func() bool { + _, err = sh.Delete(context.Background(), delPrm) + return err == nil + }, 30*time.Second, 100*time.Millisecond) + } else { + _, err = sh.Delete(context.Background(), delPrm) + require.NoError(t, err) + } - require.Eventually(t, func() bool { - _, err = sh.Get(context.Background(), getPrm) - return client.IsErrObjectNotFound(err) - }, time.Second, 50*time.Millisecond) + _, err = sh.Get(context.Background(), getPrm) + require.True(t, client.IsErrObjectNotFound(err)) }) t.Run("small object", func(t *testing.T) { @@ -78,12 +83,17 @@ func testShardDelete(t *testing.T, hasWriteCache bool) { _, err = sh.Get(context.Background(), getPrm) require.NoError(t, err) - _, err = sh.Delete(context.Background(), delPrm) - require.NoError(t, err) + if hasWriteCache { + require.Eventually(t, func() bool { + _, err = sh.Delete(context.Background(), delPrm) + return err == nil + }, 10*time.Second, 100*time.Millisecond) + } else { + _, err = sh.Delete(context.Background(), delPrm) + require.NoError(t, err) + } - require.Eventually(t, func() bool { - _, err = sh.Get(context.Background(), getPrm) - return client.IsErrObjectNotFound(err) - }, time.Second, 50*time.Millisecond) + _, err = sh.Get(context.Background(), getPrm) + require.True(t, client.IsErrObjectNotFound(err)) }) } diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 13ab39ae0..0ce99b475 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -297,7 +297,7 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { deletePrm.SetAddresses(buf...) // delete accumulated objects - res, err := s.delete(ctx, deletePrm) + res, err := s.delete(ctx, deletePrm, true) result.deleted = res.deleted result.failedToDelete = uint64(len(buf)) - res.deleted From 189dbb01be21608f2596a2e21d896232eb8c32bb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 16 Oct 2023 18:15:04 +0300 Subject: [PATCH 0612/1943] [#733] frostfs-cli: Add `control ir remove-container` Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/ir.go | 2 + .../modules/control/ir_remove_container.go | 94 ++++ pkg/innerring/initialization.go | 3 +- pkg/innerring/innerring.go | 22 +- pkg/morph/client/container/delete.go | 2 +- pkg/morph/client/static.go | 5 + pkg/services/control/ir/rpc.go | 15 +- pkg/services/control/ir/server/calls.go | 64 +++ pkg/services/control/ir/server/server.go | 16 +- pkg/services/control/ir/service.pb.go | 446 ++++++++++++++---- pkg/services/control/ir/service.proto | 19 + pkg/services/control/ir/service_frostfs.pb.go | 165 +++++++ pkg/services/control/ir/service_grpc.pb.go | 61 ++- pkg/services/control/ir/types.pb.go | 2 +- 14 files changed, 806 insertions(+), 110 deletions(-) create mode 100644 cmd/frostfs-cli/modules/control/ir_remove_container.go diff --git a/cmd/frostfs-cli/modules/control/ir.go b/cmd/frostfs-cli/modules/control/ir.go index 396d5d0a5..ac1371db7 100644 --- a/cmd/frostfs-cli/modules/control/ir.go +++ b/cmd/frostfs-cli/modules/control/ir.go @@ -12,8 +12,10 @@ func initControlIRCmd() { irCmd.AddCommand(tickEpochCmd) irCmd.AddCommand(removeNodeCmd) irCmd.AddCommand(irHealthCheckCmd) + irCmd.AddCommand(removeContainerCmd) initControlIRTickEpochCmd() initControlIRRemoveNodeCmd() initControlIRHealthCheckCmd() + initControlIRRemoveContainerCmd() } diff --git a/cmd/frostfs-cli/modules/control/ir_remove_container.go b/cmd/frostfs-cli/modules/control/ir_remove_container.go new file mode 100644 index 000000000..43173bcaa --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir_remove_container.go @@ -0,0 +1,94 @@ +package control + +import ( + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/spf13/cobra" +) + +const ( + ownerFlag = "owner" +) + +var removeContainerCmd = &cobra.Command{ + Use: "remove-container", + Short: "Schedules a container removal", + Long: `Schedules a container removal via a notary request. +Container data will be deleted asynchronously by policer. +To check removal status "frostfs-cli container list" command can be used.`, + Run: removeContainer, +} + +func initControlIRRemoveContainerCmd() { + initControlFlags(removeContainerCmd) + + flags := removeContainerCmd.Flags() + flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + flags.String(ownerFlag, "", "Container owner's wallet address.") + removeContainerCmd.MarkFlagsMutuallyExclusive(commonflags.CIDFlag, ownerFlag) +} + +func removeContainer(cmd *cobra.Command, _ []string) { + req := prepareRemoveContainerRequest(cmd) + + pk := key.Get(cmd) + c := getClient(cmd, pk) + + commonCmd.ExitOnErr(cmd, "could not sign request: %w", ircontrolsrv.SignMessage(pk, req)) + + var resp *ircontrol.RemoveContainerResponse + err := c.ExecRaw(func(client *rawclient.Client) error { + var err error + resp, err = ircontrol.RemoveContainer(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "failed to execute request: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + if len(req.GetBody().GetContainerId()) > 0 { + cmd.Println("Container scheduled to removal") + } else { + cmd.Println("User containers sheduled to removal") + } +} + +func prepareRemoveContainerRequest(cmd *cobra.Command) *ircontrol.RemoveContainerRequest { + req := &ircontrol.RemoveContainerRequest{ + Body: &ircontrol.RemoveContainerRequest_Body{}, + } + + cidStr, err := cmd.Flags().GetString(commonflags.CIDFlag) + commonCmd.ExitOnErr(cmd, "failed to get cid: ", err) + + ownerStr, err := cmd.Flags().GetString(ownerFlag) + commonCmd.ExitOnErr(cmd, "failed to get owner: ", err) + + if len(ownerStr) == 0 && len(cidStr) == 0 { + commonCmd.ExitOnErr(cmd, "invalid usage: %w", errors.New("neither owner's wallet address nor container ID are specified")) + } + + if len(ownerStr) > 0 { + var owner user.ID + commonCmd.ExitOnErr(cmd, "invalid owner ID: %w", owner.DecodeString(ownerStr)) + var ownerID refs.OwnerID + owner.WriteToV2(&ownerID) + req.Body.Owner = ownerID.StableMarshal(nil) + } + + if len(cidStr) > 0 { + var containerID cid.ID + commonCmd.ExitOnErr(cmd, "invalid container ID: %w", containerID.DecodeString(cidStr)) + req.Body.ContainerId = containerID[:] + } + return req +} diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 84112d121..eb1c4b2d4 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -343,7 +343,7 @@ func (s *Server) initGRPCServer(cfg *viper.Viper) error { p.SetPrivateKey(*s.key) p.SetHealthChecker(s) - controlSvc := controlsrv.New(p, s.netmapClient, + controlSvc := controlsrv.New(p, s.netmapClient, s.containerClient, controlsrv.WithAllowedKeys(authKeys), ) @@ -389,6 +389,7 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { if err != nil { return nil, err } + s.containerClient = result.CnrClient s.netmapClient, err = nmClient.NewFromMorph(s.morphClient, s.contracts.netmap, fee, nmClient.TryNotary(), nmClient.AsAlphabet()) if err != nil { diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 6f2463c74..ae2ad4019 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -16,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/subscriber" @@ -46,16 +47,17 @@ type ( epochTimer *timer.BlockTimer // global state - morphClient *client.Client - mainnetClient *client.Client - epochCounter atomic.Uint64 - epochDuration atomic.Uint64 - statusIndex *innerRingIndexer - precision precision.Fixed8Converter - healthStatus atomic.Int32 - balanceClient *balanceClient.Client - netmapClient *nmClient.Client - persistate *state.PersistentStorage + morphClient *client.Client + mainnetClient *client.Client + epochCounter atomic.Uint64 + epochDuration atomic.Uint64 + statusIndex *innerRingIndexer + precision precision.Fixed8Converter + healthStatus atomic.Int32 + balanceClient *balanceClient.Client + netmapClient *nmClient.Client + persistate *state.PersistentStorage + containerClient *container.Client // metrics irMetrics *metrics.InnerRingServiceMetrics diff --git a/pkg/morph/client/container/delete.go b/pkg/morph/client/container/delete.go index 5bc8fc188..c1d346e70 100644 --- a/pkg/morph/client/container/delete.go +++ b/pkg/morph/client/container/delete.go @@ -67,7 +67,7 @@ func (d *DeletePrm) SetKey(key []byte) { // // If TryNotary is provided, calls notary contract. func (c *Client) Delete(p DeletePrm) error { - if len(p.signature) == 0 { + if len(p.signature) == 0 && p.IsControl() { return errNilArgument } diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index 7aa17a70f..0531eacdf 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -115,6 +115,11 @@ func (i *InvokePrmOptional) SetControlTX(b bool) { i.controlTX = b } +// IsControl gets whether a control transaction will be used. +func (i *InvokePrmOptional) IsControl() bool { + return i.controlTX +} + // Invoke calls Invoke method of Client with static internal script hash and fee. // Supported args types are the same as in Client. // diff --git a/pkg/services/control/ir/rpc.go b/pkg/services/control/ir/rpc.go index 1b635c149..0c9400f6c 100644 --- a/pkg/services/control/ir/rpc.go +++ b/pkg/services/control/ir/rpc.go @@ -9,9 +9,10 @@ import ( const serviceName = "ircontrol.ControlService" const ( - rpcHealthCheck = "HealthCheck" - rpcTickEpoch = "TickEpoch" - rpcRemoveNode = "RemoveNode" + rpcHealthCheck = "HealthCheck" + rpcTickEpoch = "TickEpoch" + rpcRemoveNode = "RemoveNode" + rpcRemoveContainer = "RemoveContainer" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -40,6 +41,14 @@ func RemoveNode( return sendUnary[RemoveNodeRequest, RemoveNodeResponse](cli, rpcRemoveNode, req, opts...) } +func RemoveContainer( + cli *client.Client, + req *RemoveContainerRequest, + opts ...client.CallOption, +) (*RemoveContainerResponse, error) { + return sendUnary[RemoveContainerRequest, RemoveContainerResponse](cli, rpcRemoveContainer, req, opts...) +} + func sendUnary[I, O grpc.Message](cli *client.Client, rpcName string, req *I, opts ...client.CallOption) (*O, error) { var resp O wResp := &responseWrapper[*O]{ diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 680d1e606..537905840 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -5,8 +5,12 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -99,3 +103,63 @@ func (s *Server) RemoveNode(_ context.Context, req *control.RemoveNodeRequest) ( return resp, nil } + +// RemoveContainer forces a container removal. +func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainerRequest) (*control.RemoveContainerResponse, error) { + if err := s.isValidRequest(req); err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + if len(req.Body.GetContainerId()) > 0 && len(req.Body.GetOwner()) > 0 { + return nil, status.Error(codes.InvalidArgument, "specify the owner and container at the same time is not allowed") + } + + if len(req.Body.GetContainerId()) > 0 { + var containerID cid.ID + if err := containerID.Decode(req.Body.GetContainerId()); err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed to parse container ID: %s", err.Error())) + } + if err := s.removeContainer(containerID); err != nil { + return nil, err + } + } else { + var ownerID refs.OwnerID + if err := ownerID.Unmarshal(req.GetBody().GetOwner()); err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed to parse ownerID: %s", err.Error())) + } + var owner user.ID + if err := owner.ReadFromV2(ownerID); err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed to read owner: %s", err.Error())) + } + + cids, err := s.containerClient.ContainersOf(&owner) + if err != nil { + return nil, fmt.Errorf("failed to get owner's containers: %w", err) + } + + for _, containerID := range cids { + if err := s.removeContainer(containerID); err != nil { + return nil, err + } + } + } + + resp := &control.RemoveContainerResponse{ + Body: &control.RemoveContainerResponse_Body{}, + } + if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil +} + +func (s *Server) removeContainer(containerID cid.ID) error { + var prm container.DeletePrm + prm.SetCID(containerID[:]) + prm.SetControlTX(true) + + if err := s.containerClient.Delete(prm); err != nil { + return fmt.Errorf("forcing container removal: %w", err) + } + return nil +} diff --git a/pkg/services/control/ir/server/server.go b/pkg/services/control/ir/server/server.go index dc00809a6..c2a4f88a6 100644 --- a/pkg/services/control/ir/server/server.go +++ b/pkg/services/control/ir/server/server.go @@ -3,6 +3,7 @@ package control import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" ) @@ -12,10 +13,10 @@ import ( // To gain access to the service, any request must be // signed with a key from the white list. type Server struct { - prm Prm - netmapClient *netmap.Client - - allowedKeys [][]byte + prm Prm + netmapClient *netmap.Client + containerClient *container.Client + allowedKeys [][]byte } func panicOnPrmValue(n string, v any) { @@ -32,7 +33,7 @@ func panicOnPrmValue(n string, v any) { // Forms white list from all keys specified via // WithAllowedKeys option and a public key of // the parameterized private key. -func New(prm Prm, netmapClient *netmap.Client, opts ...Option) *Server { +func New(prm Prm, netmapClient *netmap.Client, containerClient *container.Client, opts ...Option) *Server { // verify required parameters switch { case prm.healthChecker == nil: @@ -47,8 +48,9 @@ func New(prm Prm, netmapClient *netmap.Client, opts ...Option) *Server { } return &Server{ - prm: prm, - netmapClient: netmapClient, + prm: prm, + netmapClient: netmapClient, + containerClient: containerClient, allowedKeys: append(o.allowedKeys, prm.key.PublicKey().Bytes()), } diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index bec74a3be..56d52be4c 100644 --- a/pkg/services/control/ir/service.pb.go +++ b/pkg/services/control/ir/service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.12.4 +// protoc v4.24.4 // source: pkg/services/control/ir/service.proto package control @@ -358,6 +358,116 @@ func (x *RemoveNodeResponse) GetSignature() *Signature { return nil } +type RemoveContainerRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *RemoveContainerRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *RemoveContainerRequest) Reset() { + *x = RemoveContainerRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveContainerRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveContainerRequest) ProtoMessage() {} + +func (x *RemoveContainerRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveContainerRequest.ProtoReflect.Descriptor instead. +func (*RemoveContainerRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{6} +} + +func (x *RemoveContainerRequest) GetBody() *RemoveContainerRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *RemoveContainerRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +type RemoveContainerResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *RemoveContainerResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *RemoveContainerResponse) Reset() { + *x = RemoveContainerResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveContainerResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveContainerResponse) ProtoMessage() {} + +func (x *RemoveContainerResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveContainerResponse.ProtoReflect.Descriptor instead. +func (*RemoveContainerResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{7} +} + +func (x *RemoveContainerResponse) GetBody() *RemoveContainerResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *RemoveContainerResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + // Health check request body. type HealthCheckRequest_Body struct { state protoimpl.MessageState @@ -368,7 +478,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -381,7 +491,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -410,7 +520,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -423,7 +533,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -455,7 +565,7 @@ type TickEpochRequest_Body struct { func (x *TickEpochRequest_Body) Reset() { *x = TickEpochRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[8] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -468,7 +578,7 @@ func (x *TickEpochRequest_Body) String() string { func (*TickEpochRequest_Body) ProtoMessage() {} func (x *TickEpochRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[8] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -493,7 +603,7 @@ type TickEpochResponse_Body struct { func (x *TickEpochResponse_Body) Reset() { *x = TickEpochResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[9] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -506,7 +616,7 @@ func (x *TickEpochResponse_Body) String() string { func (*TickEpochResponse_Body) ProtoMessage() {} func (x *TickEpochResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[9] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -533,7 +643,7 @@ type RemoveNodeRequest_Body struct { func (x *RemoveNodeRequest_Body) Reset() { *x = RemoveNodeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[10] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -546,7 +656,7 @@ func (x *RemoveNodeRequest_Body) String() string { func (*RemoveNodeRequest_Body) ProtoMessage() {} func (x *RemoveNodeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[10] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -578,7 +688,7 @@ type RemoveNodeResponse_Body struct { func (x *RemoveNodeResponse_Body) Reset() { *x = RemoveNodeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[11] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -591,7 +701,7 @@ func (x *RemoveNodeResponse_Body) String() string { func (*RemoveNodeResponse_Body) ProtoMessage() {} func (x *RemoveNodeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[11] + mi := &file_pkg_services_control_ir_service_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -607,6 +717,99 @@ func (*RemoveNodeResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{5, 0} } +type RemoveContainerRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` + Owner []byte `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` +} + +func (x *RemoveContainerRequest_Body) Reset() { + *x = RemoveContainerRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveContainerRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveContainerRequest_Body) ProtoMessage() {} + +func (x *RemoveContainerRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveContainerRequest_Body.ProtoReflect.Descriptor instead. +func (*RemoveContainerRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{6, 0} +} + +func (x *RemoveContainerRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} + +func (x *RemoveContainerRequest_Body) GetOwner() []byte { + if x != nil { + return x.Owner + } + return nil +} + +type RemoveContainerResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *RemoveContainerResponse_Body) Reset() { + *x = RemoveContainerResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveContainerResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveContainerResponse_Body) ProtoMessage() {} + +func (x *RemoveContainerResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_ir_service_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveContainerResponse_Body.ProtoReflect.Descriptor instead. +func (*RemoveContainerResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{7, 0} +} + var File_pkg_services_control_ir_service_proto protoreflect.FileDescriptor var file_pkg_services_control_ir_service_proto_rawDesc = []byte{ @@ -672,27 +875,54 @@ var file_pkg_services_control_ir_service_proto_rawDesc = []byte{ 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x32, 0xf1, 0x01, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x09, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, - 0x68, 0x12, 0x1b, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, - 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, - 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, - 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x0a, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x69, 0x72, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, - 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, - 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, - 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x2f, 0x69, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x64, 0x79, 0x22, 0xc9, 0x01, 0x0a, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3a, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x69, + 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x3f, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x77, 0x6e, + 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x22, + 0x92, 0x01, 0x0a, 0x17, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x69, 0x72, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x32, 0xcb, 0x02, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x09, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, + 0x63, 0x68, 0x12, 0x1b, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, + 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1c, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, + 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, + 0x0a, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x69, 0x72, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, + 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x0f, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x21, 0x2e, 0x69, 0x72, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, + 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, + 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, + 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, + 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x69, 0x72, + 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -707,48 +937,58 @@ func file_pkg_services_control_ir_service_proto_rawDescGZIP() []byte { return file_pkg_services_control_ir_service_proto_rawDescData } -var file_pkg_services_control_ir_service_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_pkg_services_control_ir_service_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_pkg_services_control_ir_service_proto_goTypes = []interface{}{ - (*HealthCheckRequest)(nil), // 0: ircontrol.HealthCheckRequest - (*HealthCheckResponse)(nil), // 1: ircontrol.HealthCheckResponse - (*TickEpochRequest)(nil), // 2: ircontrol.TickEpochRequest - (*TickEpochResponse)(nil), // 3: ircontrol.TickEpochResponse - (*RemoveNodeRequest)(nil), // 4: ircontrol.RemoveNodeRequest - (*RemoveNodeResponse)(nil), // 5: ircontrol.RemoveNodeResponse - (*HealthCheckRequest_Body)(nil), // 6: ircontrol.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 7: ircontrol.HealthCheckResponse.Body - (*TickEpochRequest_Body)(nil), // 8: ircontrol.TickEpochRequest.Body - (*TickEpochResponse_Body)(nil), // 9: ircontrol.TickEpochResponse.Body - (*RemoveNodeRequest_Body)(nil), // 10: ircontrol.RemoveNodeRequest.Body - (*RemoveNodeResponse_Body)(nil), // 11: ircontrol.RemoveNodeResponse.Body - (*Signature)(nil), // 12: ircontrol.Signature - (HealthStatus)(0), // 13: ircontrol.HealthStatus + (*HealthCheckRequest)(nil), // 0: ircontrol.HealthCheckRequest + (*HealthCheckResponse)(nil), // 1: ircontrol.HealthCheckResponse + (*TickEpochRequest)(nil), // 2: ircontrol.TickEpochRequest + (*TickEpochResponse)(nil), // 3: ircontrol.TickEpochResponse + (*RemoveNodeRequest)(nil), // 4: ircontrol.RemoveNodeRequest + (*RemoveNodeResponse)(nil), // 5: ircontrol.RemoveNodeResponse + (*RemoveContainerRequest)(nil), // 6: ircontrol.RemoveContainerRequest + (*RemoveContainerResponse)(nil), // 7: ircontrol.RemoveContainerResponse + (*HealthCheckRequest_Body)(nil), // 8: ircontrol.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 9: ircontrol.HealthCheckResponse.Body + (*TickEpochRequest_Body)(nil), // 10: ircontrol.TickEpochRequest.Body + (*TickEpochResponse_Body)(nil), // 11: ircontrol.TickEpochResponse.Body + (*RemoveNodeRequest_Body)(nil), // 12: ircontrol.RemoveNodeRequest.Body + (*RemoveNodeResponse_Body)(nil), // 13: ircontrol.RemoveNodeResponse.Body + (*RemoveContainerRequest_Body)(nil), // 14: ircontrol.RemoveContainerRequest.Body + (*RemoveContainerResponse_Body)(nil), // 15: ircontrol.RemoveContainerResponse.Body + (*Signature)(nil), // 16: ircontrol.Signature + (HealthStatus)(0), // 17: ircontrol.HealthStatus } var file_pkg_services_control_ir_service_proto_depIdxs = []int32{ - 6, // 0: ircontrol.HealthCheckRequest.body:type_name -> ircontrol.HealthCheckRequest.Body - 12, // 1: ircontrol.HealthCheckRequest.signature:type_name -> ircontrol.Signature - 7, // 2: ircontrol.HealthCheckResponse.body:type_name -> ircontrol.HealthCheckResponse.Body - 12, // 3: ircontrol.HealthCheckResponse.signature:type_name -> ircontrol.Signature - 8, // 4: ircontrol.TickEpochRequest.body:type_name -> ircontrol.TickEpochRequest.Body - 12, // 5: ircontrol.TickEpochRequest.signature:type_name -> ircontrol.Signature - 9, // 6: ircontrol.TickEpochResponse.body:type_name -> ircontrol.TickEpochResponse.Body - 12, // 7: ircontrol.TickEpochResponse.signature:type_name -> ircontrol.Signature - 10, // 8: ircontrol.RemoveNodeRequest.body:type_name -> ircontrol.RemoveNodeRequest.Body - 12, // 9: ircontrol.RemoveNodeRequest.signature:type_name -> ircontrol.Signature - 11, // 10: ircontrol.RemoveNodeResponse.body:type_name -> ircontrol.RemoveNodeResponse.Body - 12, // 11: ircontrol.RemoveNodeResponse.signature:type_name -> ircontrol.Signature - 13, // 12: ircontrol.HealthCheckResponse.Body.health_status:type_name -> ircontrol.HealthStatus - 0, // 13: ircontrol.ControlService.HealthCheck:input_type -> ircontrol.HealthCheckRequest - 2, // 14: ircontrol.ControlService.TickEpoch:input_type -> ircontrol.TickEpochRequest - 4, // 15: ircontrol.ControlService.RemoveNode:input_type -> ircontrol.RemoveNodeRequest - 1, // 16: ircontrol.ControlService.HealthCheck:output_type -> ircontrol.HealthCheckResponse - 3, // 17: ircontrol.ControlService.TickEpoch:output_type -> ircontrol.TickEpochResponse - 5, // 18: ircontrol.ControlService.RemoveNode:output_type -> ircontrol.RemoveNodeResponse - 16, // [16:19] is the sub-list for method output_type - 13, // [13:16] is the sub-list for method input_type - 13, // [13:13] is the sub-list for extension type_name - 13, // [13:13] is the sub-list for extension extendee - 0, // [0:13] is the sub-list for field type_name + 8, // 0: ircontrol.HealthCheckRequest.body:type_name -> ircontrol.HealthCheckRequest.Body + 16, // 1: ircontrol.HealthCheckRequest.signature:type_name -> ircontrol.Signature + 9, // 2: ircontrol.HealthCheckResponse.body:type_name -> ircontrol.HealthCheckResponse.Body + 16, // 3: ircontrol.HealthCheckResponse.signature:type_name -> ircontrol.Signature + 10, // 4: ircontrol.TickEpochRequest.body:type_name -> ircontrol.TickEpochRequest.Body + 16, // 5: ircontrol.TickEpochRequest.signature:type_name -> ircontrol.Signature + 11, // 6: ircontrol.TickEpochResponse.body:type_name -> ircontrol.TickEpochResponse.Body + 16, // 7: ircontrol.TickEpochResponse.signature:type_name -> ircontrol.Signature + 12, // 8: ircontrol.RemoveNodeRequest.body:type_name -> ircontrol.RemoveNodeRequest.Body + 16, // 9: ircontrol.RemoveNodeRequest.signature:type_name -> ircontrol.Signature + 13, // 10: ircontrol.RemoveNodeResponse.body:type_name -> ircontrol.RemoveNodeResponse.Body + 16, // 11: ircontrol.RemoveNodeResponse.signature:type_name -> ircontrol.Signature + 14, // 12: ircontrol.RemoveContainerRequest.body:type_name -> ircontrol.RemoveContainerRequest.Body + 16, // 13: ircontrol.RemoveContainerRequest.signature:type_name -> ircontrol.Signature + 15, // 14: ircontrol.RemoveContainerResponse.body:type_name -> ircontrol.RemoveContainerResponse.Body + 16, // 15: ircontrol.RemoveContainerResponse.signature:type_name -> ircontrol.Signature + 17, // 16: ircontrol.HealthCheckResponse.Body.health_status:type_name -> ircontrol.HealthStatus + 0, // 17: ircontrol.ControlService.HealthCheck:input_type -> ircontrol.HealthCheckRequest + 2, // 18: ircontrol.ControlService.TickEpoch:input_type -> ircontrol.TickEpochRequest + 4, // 19: ircontrol.ControlService.RemoveNode:input_type -> ircontrol.RemoveNodeRequest + 6, // 20: ircontrol.ControlService.RemoveContainer:input_type -> ircontrol.RemoveContainerRequest + 1, // 21: ircontrol.ControlService.HealthCheck:output_type -> ircontrol.HealthCheckResponse + 3, // 22: ircontrol.ControlService.TickEpoch:output_type -> ircontrol.TickEpochResponse + 5, // 23: ircontrol.ControlService.RemoveNode:output_type -> ircontrol.RemoveNodeResponse + 7, // 24: ircontrol.ControlService.RemoveContainer:output_type -> ircontrol.RemoveContainerResponse + 21, // [21:25] is the sub-list for method output_type + 17, // [17:21] is the sub-list for method input_type + 17, // [17:17] is the sub-list for extension type_name + 17, // [17:17] is the sub-list for extension extendee + 0, // [0:17] is the sub-list for field type_name } func init() { file_pkg_services_control_ir_service_proto_init() } @@ -831,7 +1071,7 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*RemoveContainerRequest); i { case 0: return &v.state case 1: @@ -843,7 +1083,7 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { + switch v := v.(*RemoveContainerResponse); i { case 0: return &v.state case 1: @@ -855,7 +1095,7 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TickEpochRequest_Body); i { + switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state case 1: @@ -867,7 +1107,7 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TickEpochResponse_Body); i { + switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state case 1: @@ -879,7 +1119,7 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveNodeRequest_Body); i { + switch v := v.(*TickEpochRequest_Body); i { case 0: return &v.state case 1: @@ -891,6 +1131,30 @@ func file_pkg_services_control_ir_service_proto_init() { } } file_pkg_services_control_ir_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TickEpochResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_ir_service_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoveNodeRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_ir_service_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RemoveNodeResponse_Body); i { case 0: return &v.state @@ -902,6 +1166,30 @@ func file_pkg_services_control_ir_service_proto_init() { return nil } } + file_pkg_services_control_ir_service_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoveContainerRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_ir_service_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoveContainerResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -909,7 +1197,7 @@ func file_pkg_services_control_ir_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_ir_service_proto_rawDesc, NumEnums: 0, - NumMessages: 12, + NumMessages: 16, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/ir/service.proto b/pkg/services/control/ir/service.proto index d647db0df..6c85ba4b9 100644 --- a/pkg/services/control/ir/service.proto +++ b/pkg/services/control/ir/service.proto @@ -14,6 +14,8 @@ service ControlService { rpc TickEpoch (TickEpochRequest) returns (TickEpochResponse); // Forces a node removal to be signaled by the IR node with high probability. rpc RemoveNode (RemoveNodeRequest) returns (RemoveNodeResponse); + // Forces a container removal to be signaled by the IR node with high probability. + rpc RemoveContainer(RemoveContainerRequest) returns (RemoveContainerResponse); } // Health check request. @@ -75,3 +77,20 @@ message RemoveNodeResponse { Body body = 1; Signature signature = 2; } + +message RemoveContainerRequest { + message Body{ + bytes container_id = 1; + bytes owner = 2; + } + + Body body = 1; + Signature signature = 2; +} + +message RemoveContainerResponse { + message Body{} + + Body body = 1; + Signature signature = 2; +} \ No newline at end of file diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index f39d3b043..131ec78da 100644 --- a/pkg/services/control/ir/service_frostfs.pb.go +++ b/pkg/services/control/ir/service_frostfs.pb.go @@ -483,3 +483,168 @@ func (x *RemoveNodeResponse) ReadSignedData(buf []byte) ([]byte, error) { func (x *RemoveNodeResponse) SetSignature(sig *Signature) { x.Signature = sig } + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveContainerRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.BytesSize(1, x.ContainerId) + size += proto.BytesSize(2, x.Owner) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveContainerRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) + offset += proto.BytesMarshal(2, buf[offset:], x.Owner) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveContainerRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveContainerRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *RemoveContainerRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *RemoveContainerRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *RemoveContainerRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveContainerResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveContainerResponse_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveContainerResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveContainerResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *RemoveContainerResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *RemoveContainerResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *RemoveContainerResponse) SetSignature(sig *Signature) { + x.Signature = sig +} diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index 6ba214da0..004c82446 100644 --- a/pkg/services/control/ir/service_grpc.pb.go +++ b/pkg/services/control/ir/service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.24.4 // source: pkg/services/control/ir/service.proto package control @@ -18,6 +18,13 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + ControlService_HealthCheck_FullMethodName = "/ircontrol.ControlService/HealthCheck" + ControlService_TickEpoch_FullMethodName = "/ircontrol.ControlService/TickEpoch" + ControlService_RemoveNode_FullMethodName = "/ircontrol.ControlService/RemoveNode" + ControlService_RemoveContainer_FullMethodName = "/ircontrol.ControlService/RemoveContainer" +) + // ControlServiceClient is the client API for ControlService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -28,6 +35,8 @@ type ControlServiceClient interface { TickEpoch(ctx context.Context, in *TickEpochRequest, opts ...grpc.CallOption) (*TickEpochResponse, error) // Forces a node removal to be signaled by the IR node with high probability. RemoveNode(ctx context.Context, in *RemoveNodeRequest, opts ...grpc.CallOption) (*RemoveNodeResponse, error) + // Forces a container removal to be signaled by the IR node with high probability. + RemoveContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*RemoveContainerResponse, error) } type controlServiceClient struct { @@ -40,7 +49,7 @@ func NewControlServiceClient(cc grpc.ClientConnInterface) ControlServiceClient { func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { out := new(HealthCheckResponse) - err := c.cc.Invoke(ctx, "/ircontrol.ControlService/HealthCheck", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_HealthCheck_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -49,7 +58,7 @@ func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckR func (c *controlServiceClient) TickEpoch(ctx context.Context, in *TickEpochRequest, opts ...grpc.CallOption) (*TickEpochResponse, error) { out := new(TickEpochResponse) - err := c.cc.Invoke(ctx, "/ircontrol.ControlService/TickEpoch", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_TickEpoch_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -58,7 +67,16 @@ func (c *controlServiceClient) TickEpoch(ctx context.Context, in *TickEpochReque func (c *controlServiceClient) RemoveNode(ctx context.Context, in *RemoveNodeRequest, opts ...grpc.CallOption) (*RemoveNodeResponse, error) { out := new(RemoveNodeResponse) - err := c.cc.Invoke(ctx, "/ircontrol.ControlService/RemoveNode", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_RemoveNode_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlServiceClient) RemoveContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*RemoveContainerResponse, error) { + out := new(RemoveContainerResponse) + err := c.cc.Invoke(ctx, ControlService_RemoveContainer_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -75,6 +93,8 @@ type ControlServiceServer interface { TickEpoch(context.Context, *TickEpochRequest) (*TickEpochResponse, error) // Forces a node removal to be signaled by the IR node with high probability. RemoveNode(context.Context, *RemoveNodeRequest) (*RemoveNodeResponse, error) + // Forces a container removal to be signaled by the IR node with high probability. + RemoveContainer(context.Context, *RemoveContainerRequest) (*RemoveContainerResponse, error) } // UnimplementedControlServiceServer should be embedded to have forward compatible implementations. @@ -90,6 +110,9 @@ func (UnimplementedControlServiceServer) TickEpoch(context.Context, *TickEpochRe func (UnimplementedControlServiceServer) RemoveNode(context.Context, *RemoveNodeRequest) (*RemoveNodeResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method RemoveNode not implemented") } +func (UnimplementedControlServiceServer) RemoveContainer(context.Context, *RemoveContainerRequest) (*RemoveContainerResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RemoveContainer not implemented") +} // UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ControlServiceServer will @@ -112,7 +135,7 @@ func _ControlService_HealthCheck_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/ircontrol.ControlService/HealthCheck", + FullMethod: ControlService_HealthCheck_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).HealthCheck(ctx, req.(*HealthCheckRequest)) @@ -130,7 +153,7 @@ func _ControlService_TickEpoch_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/ircontrol.ControlService/TickEpoch", + FullMethod: ControlService_TickEpoch_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).TickEpoch(ctx, req.(*TickEpochRequest)) @@ -148,7 +171,7 @@ func _ControlService_RemoveNode_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/ircontrol.ControlService/RemoveNode", + FullMethod: ControlService_RemoveNode_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).RemoveNode(ctx, req.(*RemoveNodeRequest)) @@ -156,6 +179,24 @@ func _ControlService_RemoveNode_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } +func _ControlService_RemoveContainer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RemoveContainerRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).RemoveContainer(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_RemoveContainer_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).RemoveContainer(ctx, req.(*RemoveContainerRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -175,6 +216,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "RemoveNode", Handler: _ControlService_RemoveNode_Handler, }, + { + MethodName: "RemoveContainer", + Handler: _ControlService_RemoveContainer_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "pkg/services/control/ir/service.proto", diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go index 8107b917e..0ff8f45e9 100644 --- a/pkg/services/control/ir/types.pb.go +++ b/pkg/services/control/ir/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.12.4 +// protoc v4.24.4 // source: pkg/services/control/ir/types.proto package control From dc4d27201bc36cd9983e559028ebd4400f7703b3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 19 Oct 2023 18:01:19 +0300 Subject: [PATCH 0613/1943] [#733] morph: Fix delete container signature check Committed invalid condition, it was just for debug. Signed-off-by: Dmitrii Stepanov --- pkg/morph/client/container/delete.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/morph/client/container/delete.go b/pkg/morph/client/container/delete.go index c1d346e70..b520120d2 100644 --- a/pkg/morph/client/container/delete.go +++ b/pkg/morph/client/container/delete.go @@ -67,7 +67,7 @@ func (d *DeletePrm) SetKey(key []byte) { // // If TryNotary is provided, calls notary contract. func (c *Client) Delete(p DeletePrm) error { - if len(p.signature) == 0 && p.IsControl() { + if len(p.signature) == 0 && !p.IsControl() { return errNilArgument } From 12b7cf2533b71616105e09dcab8da8352358cc6b Mon Sep 17 00:00:00 2001 From: aarifullin Date: Thu, 19 Oct 2023 18:19:55 +0300 Subject: [PATCH 0614/1943] [#747] client: Refactor PrmObjectPutSingle usage Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 ++-- pkg/services/object/internal/client/client.go | 24 ++++++------------- 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 28008128c..a97e04f33 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0 git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230928142024-84b9d29fc98c + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231019083813-5804128ff3d5 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 8490c167c..29134ea57 100644 --- a/go.sum +++ b/go.sum @@ -393,8 +393,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230928142024-84b9d29fc98c h1:c8mduKlc8Zioppz5o06QRYS5KYX3BFRO+NgKj2q6kD8= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230928142024-84b9d29fc98c/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231019083813-5804128ff3d5 h1:P2HCnBTx+LwhMYSrs4sF8+jHvD1+HdS0fMtUPe2kTns= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231019083813-5804128ff3d5/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 0301a593a..27548862e 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -449,25 +449,15 @@ func PutObjectSingle(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, erro return nil, errors.New("missing object id") } - var prmCli client.PrmObjectPutSingle - - prmCli.ExecuteLocal() - - if prm.key != nil { - prmCli.UseKey(prm.key) + prmCli := client.PrmObjectPutSingle{ + XHeaders: prm.xHeaders, + BearerToken: prm.tokenBearer, + Session: prm.tokenSession, + Local: true, + Key: prm.key, + Object: prm.obj, } - if prm.tokenSession != nil { - prmCli.WithinSession(*prm.tokenSession) - } - - if prm.tokenBearer != nil { - prmCli.WithBearerToken(*prm.tokenBearer) - } - - prmCli.WithXHeaders(prm.xHeaders...) - prmCli.SetObject(prm.obj.ToV2()) - res, err := prm.cli.ObjectPutSingle(ctx, prmCli) if err != nil { ReportError(prm.cli, err) From 58b6224dd890ceb554e205bd480085f9e71d2ff3 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Fri, 20 Oct 2023 10:33:24 +0300 Subject: [PATCH 0615/1943] [#747] client: Refactor PrmObjectPutInit usage Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/client.go | 27 +++++++------------ pkg/services/object/internal/client/client.go | 22 +++++---------- 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 9cb7c8161..b370ff3aa 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -387,30 +387,23 @@ func (x *PutObjectPrm) PrepareLocally() { } func (x *PutObjectPrm) convertToSDKPrm(ctx context.Context) (client.PrmObjectPutInit, error) { - var putPrm client.PrmObjectPutInit - if !x.prepareLocally && x.sessionToken != nil { - putPrm.WithinSession(*x.sessionToken) + putPrm := client.PrmObjectPutInit{ + XHeaders: x.xHeaders, + BearerToken: x.bearerToken, + Local: x.local, + CopiesNumber: x.copyNum, } - if x.bearerToken != nil { - putPrm.WithBearerToken(*x.bearerToken) - } - - if x.local { - putPrm.MarkLocal() - } - - putPrm.WithXHeaders(x.xHeaders...) - putPrm.SetCopiesNumberByVectors(x.copyNum) - if x.prepareLocally { res, err := x.cli.NetworkInfo(ctx, client.PrmNetworkInfo{}) if err != nil { return client.PrmObjectPutInit{}, err } - putPrm.WithObjectMaxSize(res.Info().MaxObjectSize()) - putPrm.WithEpochSource(epochSource(res.Info().CurrentEpoch())) - putPrm.WithoutHomomorphicHash(res.Info().HomomorphicHashingDisabled()) + putPrm.MaxSize = res.Info().MaxObjectSize() + putPrm.EpochSource = epochSource(res.Info().CurrentEpoch()) + putPrm.WithoutHomomorphHash = res.Info().HomomorphicHashingDisabled() + } else { + putPrm.Session = x.sessionToken } return putPrm, nil } diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 27548862e..e0a7e562a 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -392,24 +392,14 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "client.PutObject") defer span.End() - var prmCli client.PrmObjectPutInit - - prmCli.MarkLocal() - - if prm.key != nil { - prmCli.UseKey(*prm.key) + prmCli := client.PrmObjectPutInit{ + XHeaders: prm.xHeaders, + BearerToken: prm.tokenBearer, + Session: prm.tokenSession, + Local: true, + Key: prm.key, } - if prm.tokenSession != nil { - prmCli.WithinSession(*prm.tokenSession) - } - - if prm.tokenBearer != nil { - prmCli.WithBearerToken(*prm.tokenBearer) - } - - prmCli.WithXHeaders(prm.xHeaders...) - w, err := prm.cli.ObjectPutInit(ctx, prmCli) if err != nil { return nil, fmt.Errorf("init object writing on client: %w", err) From b0cf100427dc9d813ac0fc4facd8412f6399b370 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 20 Oct 2023 09:52:25 +0300 Subject: [PATCH 0616/1943] [#49] node: React on SIGHUP only when node in READY state Add more info in logs when node is going to shut down, but initialization process still in progress. Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config.go | 16 ++++++++++++++-- cmd/frostfs-node/control.go | 14 ++++++++++++-- cmd/frostfs-node/main.go | 6 +++--- internal/logs/logs.go | 3 +++ pkg/services/control/types.pb.go | 32 +++++++++++++++++++------------- pkg/services/control/types.proto | 3 +++ 6 files changed, 54 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index b35200d27..8a7317231 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1040,7 +1040,6 @@ func (c *cfg) signalWatcher(ctx context.Context) { c.reloadConfig(ctx) case syscall.SIGTERM, syscall.SIGINT: c.log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) - // TODO (@acid-ant): #49 need to cover case when stuck at the middle(node health UNDEFINED or STARTING) c.shutdown() @@ -1062,6 +1061,12 @@ func (c *cfg) signalWatcher(ctx context.Context) { func (c *cfg) reloadConfig(ctx context.Context) { c.log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) + if !c.compareAndSwapHealthStatus(control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) { + c.log.Info(logs.FrostFSNodeSIGHUPSkip) + return + } + defer c.compareAndSwapHealthStatus(control.HealthStatus_RECONFIGURING, control.HealthStatus_READY) + err := c.readConfig(c.appCfg) if err != nil { c.log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) @@ -1142,7 +1147,14 @@ func (c *cfg) createTombstoneSource() *tombstone.ExpirationChecker { } func (c *cfg) shutdown() { - c.setHealthStatus(control.HealthStatus_SHUTTING_DOWN) + old := c.swapHealthStatus(control.HealthStatus_SHUTTING_DOWN) + if old == control.HealthStatus_SHUTTING_DOWN { + c.log.Info(logs.FrostFSNodeShutdownSkip) + return + } + if old == control.HealthStatus_STARTING { + c.log.Warn(logs.FrostFSNodeShutdownWhenNotReady) + } c.ctxCancel() c.done <- struct{}{} diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index 787fe3351..98d893c38 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -83,10 +83,20 @@ func (c *cfg) NetmapStatus() control.NetmapStatus { func (c *cfg) setHealthStatus(st control.HealthStatus) { c.healthStatus.Store(int32(st)) + c.metricsCollector.State().SetHealth(int32(st)) +} - if c.metricsCollector != nil { - c.metricsCollector.State().SetHealth(int32(st)) +func (c *cfg) compareAndSwapHealthStatus(oldSt, newSt control.HealthStatus) (swapped bool) { + if swapped = c.healthStatus.CompareAndSwap(int32(oldSt), int32(newSt)); swapped { + c.metricsCollector.State().SetHealth(int32(newSt)) } + return +} + +func (c *cfg) swapHealthStatus(st control.HealthStatus) (old control.HealthStatus) { + old = control.HealthStatus(c.healthStatus.Swap(int32(st))) + c.metricsCollector.State().SetHealth(int32(st)) + return } func (c *cfg) HealthStatus() control.HealthStatus { diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 88032ebdb..2791efcef 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -60,13 +60,13 @@ func main() { var ctx context.Context ctx, c.ctxCancel = context.WithCancel(context.Background()) - initApp(ctx, c) - c.setHealthStatus(control.HealthStatus_STARTING) + initApp(ctx, c) + bootUp(ctx, c) - c.setHealthStatus(control.HealthStatus_READY) + c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY) wait(c) } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 56312f83f..f8d6cc9b4 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -436,6 +436,9 @@ const ( FrostFSNodeInternalApplicationError = "internal application error" FrostFSNodeInternalErrorProcessingIsComplete = "internal error processing is complete" FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration = "SIGHUP has been received, rereading configuration..." + FrostFSNodeSIGHUPSkip = "node not ready for reconfiguration, skipped SIGHUP" + FrostFSNodeShutdownSkip = "node already is going to shutting down, skipped shutdown" + FrostFSNodeShutdownWhenNotReady = "node is going to shutting down when subsystems still initializing" FrostFSNodeConfigurationReading = "configuration reading" FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" FrostFSNodeTracingConfigationUpdated = "tracing configation updated" diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index d2ee50770..b385bc674 100644 --- a/pkg/services/control/types.pb.go +++ b/pkg/services/control/types.pb.go @@ -89,6 +89,8 @@ const ( HealthStatus_READY HealthStatus = 2 // Storage node application is shutting down. HealthStatus_SHUTTING_DOWN HealthStatus = 3 + // Storage node application is reconfiguring. + HealthStatus_RECONFIGURING HealthStatus = 4 ) // Enum value maps for HealthStatus. @@ -98,12 +100,14 @@ var ( 1: "STARTING", 2: "READY", 3: "SHUTTING_DOWN", + 4: "RECONFIGURING", } HealthStatus_value = map[string]int32{ "HEALTH_STATUS_UNDEFINED": 0, "STARTING": 1, "READY": 2, "SHUTTING_DOWN": 3, + "RECONFIGURING": 4, } ) @@ -709,24 +713,26 @@ var file_pkg_services_control_types_proto_rawDesc = []byte{ 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x46, 0x46, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x0f, 0x0a, - 0x0b, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x2a, 0x57, + 0x0b, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x2a, 0x6a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x17, 0x48, 0x45, 0x41, 0x4c, 0x54, 0x48, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x48, 0x55, 0x54, 0x54, 0x49, 0x4e, 0x47, - 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x2a, 0x6a, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x4d, 0x4f, - 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, - 0x0a, 0x0a, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0d, - 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x12, 0x0c, 0x0a, - 0x08, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x44, - 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, - 0x59, 0x10, 0x04, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, - 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, - 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, - 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x43, 0x4f, 0x4e, + 0x46, 0x49, 0x47, 0x55, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x04, 0x2a, 0x6a, 0x0a, 0x09, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x48, 0x41, 0x52, 0x44, + 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, + 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, + 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x16, + 0x0a, 0x12, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x5f, + 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x04, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, + 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, + 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, + 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto index 5b4844580..b34988c2e 100644 --- a/pkg/services/control/types.proto +++ b/pkg/services/control/types.proto @@ -115,6 +115,9 @@ enum HealthStatus { // Storage node application is shutting down. SHUTTING_DOWN = 3; + + // Storage node application is reconfiguring. + RECONFIGURING = 4; } // Shard description. From c0b86f2d933fc34e82663872cc154315560af718 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 19 Sep 2023 08:43:13 +0300 Subject: [PATCH 0617/1943] [#642] writecache: Remove usage of close channel in badger Signed-off-by: Anton Nikiforov --- .../writecachebadger/cachebadger.go | 21 +++++----- .../writecache/writecachebadger/flush.go | 38 +++++++++---------- .../writecache/writecachebadger/gc.go | 5 ++- 3 files changed, 30 insertions(+), 34 deletions(-) diff --git a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go index d5da77635..be7046d05 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go +++ b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go @@ -26,12 +26,12 @@ type cache struct { // helps to avoid multiple flushing of one object scheduled4Flush map[oid.Address]struct{} scheduled4FlushMtx sync.RWMutex - // closeCh is close channel, protected by modeMtx. - closeCh chan struct{} // wg is a wait group for flush workers. wg sync.WaitGroup // store contains underlying database. store + // cancel is cancel function, protected by modeMtx in Close. + cancel func() } // wcStorageType is used for write-cache operations logging. @@ -89,11 +89,6 @@ func (c *cache) Open(_ context.Context, readOnly bool) error { if err != nil { return metaerr.Wrap(err) } - - // Opening after Close is done during maintenance mode, - // thus we need to create a channel here. - c.closeCh = make(chan struct{}) - return metaerr.Wrap(c.initCounters()) } @@ -101,8 +96,10 @@ func (c *cache) Open(_ context.Context, readOnly bool) error { func (c *cache) Init() error { c.log.Info(logs.WritecacheBadgerInitExperimental) c.metrics.SetMode(c.mode) - c.runFlushLoop() - c.runGCLoop() + ctx, cancel := context.WithCancel(context.Background()) + c.cancel = cancel + c.runFlushLoop(ctx) + c.runGCLoop(ctx) return nil } @@ -111,8 +108,9 @@ func (c *cache) Close() error { // We cannot lock mutex for the whole operation duration // because it is taken by some background workers, so `wg.Wait()` is done without modeMtx. c.modeMtx.Lock() - if c.closeCh != nil { - close(c.closeCh) + if c.cancel != nil { + c.cancel() + c.cancel = nil } c.mode = mode.DegradedReadOnly // prevent new operations from being processed c.modeMtx.Unlock() @@ -122,7 +120,6 @@ func (c *cache) Close() error { c.modeMtx.Lock() defer c.modeMtx.Unlock() - c.closeCh = nil var err error if c.db != nil { err = c.db.Close() diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush.go b/pkg/local_object_storage/writecache/writecachebadger/flush.go index e0333eff7..1ee6bbb66 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush.go @@ -39,18 +39,16 @@ type collector struct { cache *cache scheduled int processed int - cancel func() } -func (c *collector) Send(buf *z.Buffer) error { +func (c *collector) send(ctx context.Context, cancel func(), buf *z.Buffer) error { list, err := badger.BufferToKVList(buf) if err != nil { return err } for _, kv := range list.Kv { select { - case <-c.cache.closeCh: - c.cancel() + case <-ctx.Done(): return nil default: } @@ -58,7 +56,7 @@ func (c *collector) Send(buf *z.Buffer) error { return nil } if c.scheduled >= flushBatchSize { - c.cancel() + cancel() return nil } if got, want := len(kv.Key), len(internalKey{}); got != want { @@ -90,8 +88,7 @@ func (c *collector) Send(buf *z.Buffer) error { data: val, obj: obj, }: - case <-c.cache.closeCh: - c.cancel() + case <-ctx.Done(): return nil } } @@ -99,10 +96,10 @@ func (c *collector) Send(buf *z.Buffer) error { } // runFlushLoop starts background workers which periodically flush objects to the blobstor. -func (c *cache) runFlushLoop() { +func (c *cache) runFlushLoop(ctx context.Context) { for i := 0; i < c.workersCount; i++ { c.wg.Add(1) - go c.workerFlushSmall() + go c.workerFlushSmall(ctx) } c.wg.Add(1) @@ -115,19 +112,19 @@ func (c *cache) runFlushLoop() { for { select { case <-tt.C: - c.flushSmallObjects() + c.flushSmallObjects(ctx) tt.Reset(defaultFlushInterval) - case <-c.closeCh: + case <-ctx.Done(): return } } }() } -func (c *cache) flushSmallObjects() { +func (c *cache) flushSmallObjects(ctx context.Context) { for { select { - case <-c.closeCh: + case <-ctx.Done(): return default: } @@ -144,14 +141,15 @@ func (c *cache) flushSmallObjects() { c.modeMtx.RUnlock() return } - ctx, cancel := context.WithCancel(context.TODO()) + ctx, cancel := context.WithCancel(ctx) coll := collector{ - cache: c, - cancel: cancel, + cache: c, } stream := c.db.NewStream() // All calls to Send are done by a single goroutine - stream.Send = coll.Send + stream.Send = func(buf *z.Buffer) error { + return coll.send(ctx, cancel, buf) + } if err := stream.Orchestrate(ctx); err != nil { c.log.Debug(fmt.Sprintf( "error during flushing object from wc: %s", err)) @@ -176,7 +174,7 @@ func (c *cache) reportFlushError(msg string, addr string, err error) { } // workerFlushSmall writes small objects to the main storage. -func (c *cache) workerFlushSmall() { +func (c *cache) workerFlushSmall(ctx context.Context) { defer c.wg.Done() var objInfo objectInfo @@ -184,11 +182,11 @@ func (c *cache) workerFlushSmall() { // Give priority to direct put. select { case objInfo = <-c.flushCh: - case <-c.closeCh: + case <-ctx.Done(): return } - err := c.flushObject(context.TODO(), objInfo.obj, objInfo.data, writecache.StorageTypeDB) + err := c.flushObject(ctx, objInfo.obj, objInfo.data, writecache.StorageTypeDB) if err == nil { c.deleteFromDB([]internalKey{addr2key(objInfo.addr)}) } diff --git a/pkg/local_object_storage/writecache/writecachebadger/gc.go b/pkg/local_object_storage/writecache/writecachebadger/gc.go index 8432a9c04..b856efc3d 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/gc.go +++ b/pkg/local_object_storage/writecache/writecachebadger/gc.go @@ -1,12 +1,13 @@ package writecachebadger import ( + "context" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" ) -func (c *cache) runGCLoop() { +func (c *cache) runGCLoop(ctx context.Context) { c.wg.Add(1) go func() { @@ -17,7 +18,7 @@ func (c *cache) runGCLoop() { for { select { - case <-c.closeCh: + case <-ctx.Done(): return case <-t.C: // This serves to synchronize the c.db field when changing mode as well. From 559ad58ab191652c7a144cd4203bb458f9a225f9 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 19 Sep 2023 08:46:19 +0300 Subject: [PATCH 0618/1943] [#642] writecache: Remove usage of close channel in bbolt Signed-off-by: Anton Nikiforov --- .../writecache/writecachebbolt/cachebbolt.go | 18 +++++------ .../writecache/writecachebbolt/flush.go | 32 +++++++------------ 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go index 363ee8448..9d54cd5c2 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go @@ -30,8 +30,8 @@ type cache struct { // flushCh is a channel with objects to flush. flushCh chan objectInfo - // closeCh is close channel, protected by modeMtx. - closeCh chan struct{} + // cancel is cancel function, protected by modeMtx in Close. + cancel func() // wg is a wait group for flush workers. wg sync.WaitGroup // store contains underlying database. @@ -104,17 +104,15 @@ func (c *cache) Open(_ context.Context, readOnly bool) error { return metaerr.Wrap(err) } - // Opening after Close is done during maintenance mode, - // thus we need to create a channel here. - c.closeCh = make(chan struct{}) - return metaerr.Wrap(c.initCounters()) } // Init runs necessary services. func (c *cache) Init() error { c.metrics.SetMode(c.mode) - c.runFlushLoop() + ctx, cancel := context.WithCancel(context.Background()) + c.cancel = cancel + c.runFlushLoop(ctx) return nil } @@ -123,8 +121,9 @@ func (c *cache) Close() error { // We cannot lock mutex for the whole operation duration // because it is taken by some background workers, so `wg.Wait()` is done without modeMtx. c.modeMtx.Lock() - if c.closeCh != nil { - close(c.closeCh) + if c.cancel != nil { + c.cancel() + c.cancel = nil } c.mode = mode.DegradedReadOnly // prevent new operations from being processed c.modeMtx.Unlock() @@ -134,7 +133,6 @@ func (c *cache) Close() error { c.modeMtx.Lock() defer c.modeMtx.Unlock() - c.closeCh = nil var err error if c.db != nil { err = c.db.Close() diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush.go b/pkg/local_object_storage/writecache/writecachebbolt/flush.go index fd322e9fa..393776dd1 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush.go @@ -36,20 +36,10 @@ const ( ) // runFlushLoop starts background workers which periodically flush objects to the blobstor. -func (c *cache) runFlushLoop() { - ctx, cancel := context.WithCancel(context.Background()) - - ch := c.closeCh - c.wg.Add(1) - go func() { - <-ch - cancel() - c.wg.Done() - }() - +func (c *cache) runFlushLoop(ctx context.Context) { for i := 0; i < c.workersCount; i++ { c.wg.Add(1) - go c.workerFlushSmall() + go c.workerFlushSmall(ctx) } c.wg.Add(1) @@ -68,20 +58,20 @@ func (c *cache) runFlushLoop() { for { select { case <-tt.C: - c.flushSmallObjects() + c.flushSmallObjects(ctx) tt.Reset(defaultFlushInterval) - case <-c.closeCh: + case <-ctx.Done(): return } } }() } -func (c *cache) flushSmallObjects() { +func (c *cache) flushSmallObjects(ctx context.Context) { var lastKey []byte for { select { - case <-c.closeCh: + case <-ctx.Done(): return default: } @@ -137,7 +127,7 @@ func (c *cache) flushSmallObjects() { count++ select { case c.flushCh <- m[i]: - case <-c.closeCh: + case <-ctx.Done(): c.modeMtx.RUnlock() return } @@ -170,7 +160,7 @@ func (c *cache) workerFlushBig(ctx context.Context) { _ = c.flushFSTree(ctx, true) c.modeMtx.RUnlock() - case <-c.closeCh: + case <-ctx.Done(): return } } @@ -228,7 +218,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { } // workerFlushSmall writes small objects to the main storage. -func (c *cache) workerFlushSmall() { +func (c *cache) workerFlushSmall(ctx context.Context) { defer c.wg.Done() var objInfo objectInfo @@ -236,11 +226,11 @@ func (c *cache) workerFlushSmall() { // Give priority to direct put. select { case objInfo = <-c.flushCh: - case <-c.closeCh: + case <-ctx.Done(): return } - err := c.flushObject(context.TODO(), objInfo.obj, objInfo.data, writecache.StorageTypeDB) + err := c.flushObject(ctx, objInfo.obj, objInfo.data, writecache.StorageTypeDB) if err != nil { // Error is handled in flushObject. continue From fe1acf9e9a42367ae78e411f8124aaeed5fb707d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 24 Oct 2023 12:34:30 +0300 Subject: [PATCH 0619/1943] [#750] morph: Remove deprecated channel use Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 9 --------- pkg/morph/subscriber/subscriber.go | 15 ++++----------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 606f3bd66..290e651f7 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -527,15 +527,6 @@ func (c *Client) IsValidScript(script []byte, signers []transaction.Signer) (val return res.State == vmstate.Halt.String(), nil } -// NotificationChannel returns channel than receives subscribed -// notification from the connected RPC node. -// Channel is closed when connection to the RPC node is lost. -func (c *Client) NotificationChannel() <-chan rpcclient.Notification { - c.switchLock.RLock() - defer c.switchLock.RUnlock() - return c.client.Notifications //lint:ignore SA1019 waits for neo-go v0.102.0 https://github.com/nspcc-dev/neo-go/pull/2980 -} - func (c *Client) Metrics() morphmetrics.Register { return c.metrics } diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index c2d8494fa..7f3c91385 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -12,7 +12,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" ) @@ -183,8 +182,6 @@ func New(ctx context.Context, p *Params) (Subscriber, error) { func (s *subscriber) routeNotifications(ctx context.Context) { var ( - // TODO: not needed after nspcc-dev/neo-go#2980. - cliCh = s.client.NotificationChannel() restoreCh = make(chan bool) restoreInProgress bool ) @@ -220,8 +217,6 @@ routeloop: } else { connLost = true } - case _, ok := <-cliCh: - connLost = !ok case ok := <-restoreCh: restoreInProgress = false if !ok { @@ -230,7 +225,7 @@ routeloop: } if connLost { if !restoreInProgress { - restoreInProgress, cliCh = s.switchEndpoint(ctx, restoreCh) + restoreInProgress = s.switchEndpoint(ctx, restoreCh) if !restoreInProgress { break routeloop } @@ -249,15 +244,13 @@ routeloop: close(s.notaryChan) } -func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) (bool, <-chan rpcclient.Notification) { +func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) bool { s.log.Info(logs.RPConnectionLost) if !s.client.SwitchRPC(ctx) { s.log.Error(logs.RPCNodeSwitchFailure) - return false, nil + return false } - cliCh := s.client.NotificationChannel() - s.Lock() chs := newSubChannels() go func() { @@ -267,7 +260,7 @@ func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) ( s.Unlock() s.client.Metrics().IncSwitchCount() - return true, cliCh + return true } func newSubChannels() subChannels { From efb37b0e658b041b4b6bb1365da5fa6d25120a01 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 24 Oct 2023 13:36:03 +0300 Subject: [PATCH 0620/1943] [#750] adm: Fix invalid tests Introduced in a9d04ba86f2. Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/initialize_test.go | 10 ++++++++-- cmd/frostfs-adm/internal/modules/morph/root.go | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index 30a7168dd..39da56662 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -20,7 +20,7 @@ import ( ) const ( - contractsPath = "../../../../../../frostfs-contract/frostfs-contract-v0.16.0.tar.gz" + contractsPath = "../../../../../../contract/frostfs-contract-v0.18.0.tar.gz" protoFileName = "proto.yml" ) @@ -58,7 +58,9 @@ func testInitialize(t *testing.T, committeeSize int) { // Set to the path or remove the next statement to download from the network. require.NoError(t, initCmd.Flags().Set(contractsInitFlag, contractsPath)) - v.Set(localDumpFlag, filepath.Join(testdataDir, "out")) + + dumpPath := filepath.Join(testdataDir, "out") + require.NoError(t, initCmd.Flags().Set(localDumpFlag, dumpPath)) v.Set(alphabetWalletsFlag, testdataDir) v.Set(epochDurationInitFlag, 1) v.Set(maxObjectSizeInitFlag, 1024) @@ -67,12 +69,15 @@ func testInitialize(t *testing.T, committeeSize int) { require.NoError(t, initializeSideChainCmd(initCmd, nil)) t.Run("force-new-epoch", func(t *testing.T) { + require.NoError(t, forceNewEpoch.Flags().Set(localDumpFlag, dumpPath)) require.NoError(t, forceNewEpochCmd(forceNewEpoch, nil)) }) t.Run("set-config", func(t *testing.T) { + require.NoError(t, setConfig.Flags().Set(localDumpFlag, dumpPath)) require.NoError(t, setConfigCmd(setConfig, []string{"MaintenanceModeAllowed=true"})) }) t.Run("set-policy", func(t *testing.T) { + require.NoError(t, setPolicy.Flags().Set(localDumpFlag, dumpPath)) require.NoError(t, setPolicyCmd(setPolicy, []string{"ExecFeeFactor=1"})) }) t.Run("remove-node", func(t *testing.T) { @@ -80,6 +85,7 @@ func testInitialize(t *testing.T, committeeSize int) { require.NoError(t, err) pub := hex.EncodeToString(pk.PublicKey().Bytes()) + require.NoError(t, removeNodes.Flags().Set(localDumpFlag, dumpPath)) require.NoError(t, removeNodesCmd(removeNodes, []string{pub})) }) } diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index bee1837a3..133d5162f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -320,6 +320,7 @@ func initSetConfigCmd() { setConfig.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") setConfig.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") setConfig.Flags().Bool(forceConfigSet, false, "Force setting not well-known configuration key") + setConfig.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initDumpNetworkConfigCmd() { @@ -337,18 +338,21 @@ func initSetPolicyCmd() { RootCmd.AddCommand(setPolicy) setPolicy.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") setPolicy.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + setPolicy.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initRemoveNodesCmd() { RootCmd.AddCommand(removeNodes) removeNodes.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") removeNodes.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + removeNodes.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initForceNewEpochCmd() { RootCmd.AddCommand(forceNewEpoch) forceNewEpoch.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") forceNewEpoch.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + forceNewEpoch.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initGenerateStorageCmd() { From 7f8ccc105bb85c4889f66655c2552caeeef89a17 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 24 Oct 2023 12:47:46 +0300 Subject: [PATCH 0621/1943] [#750] adm: Drop deprecated GetNetwork() Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/initialize_transfer.go | 3 ++- cmd/frostfs-adm/internal/modules/morph/local_client.go | 5 ----- cmd/frostfs-adm/internal/modules/morph/n3client.go | 2 -- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go index 1f8e53416..b032780be 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go @@ -93,12 +93,13 @@ func (c *initializeContext) multiSignAndSend(tx *transaction.Transaction, accTyp } func (c *initializeContext) multiSign(tx *transaction.Transaction, accType string) error { - network, err := c.Client.GetNetwork() + version, err := c.Client.GetVersion() if err != nil { // error appears only if client // has not been initialized panic(err) } + network := version.Protocol.Network // Use parameter context to avoid dealing with signature order. pc := scContext.NewParameterContext("", network, tx) diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index 0367f7479..84cf40e97 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -10,7 +10,6 @@ import ( "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/config" - "github.com/nspcc-dev/neo-go/pkg/config/netmode" "github.com/nspcc-dev/neo-go/pkg/core" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/chaindump" @@ -138,10 +137,6 @@ func (l *localClient) GetNativeContracts() ([]state.NativeContract, error) { return l.bc.GetNatives(), nil } -func (l *localClient) GetNetwork() (netmode.Magic, error) { - return l.bc.GetConfig().Magic, nil -} - func (l *localClient) GetApplicationLog(h util.Uint256, t *trigger.Type) (*result.ApplicationLog, error) { aer, err := l.bc.GetAppExecResults(h, *t) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/n3client.go b/cmd/frostfs-adm/internal/modules/morph/n3client.go index 138943b6e..77f207c9f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/n3client.go @@ -6,7 +6,6 @@ import ( "fmt" "time" - "github.com/nspcc-dev/neo-go/pkg/config/netmode" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -32,7 +31,6 @@ type Client interface { GetContractStateByID(int32) (*state.Contract, error) GetContractStateByHash(util.Uint160) (*state.Contract, error) GetNativeContracts() ([]state.NativeContract, error) - GetNetwork() (netmode.Magic, error) GetApplicationLog(util.Uint256, *trigger.Type) (*result.ApplicationLog, error) GetVersion() (*result.Version, error) CreateTxFromScript([]byte, *wallet.Account, int64, int64, []rpcclient.SignerAccount) (*transaction.Transaction, error) From 00a0045d9a50faca916461bdbb3d0de261aceb86 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 24 Oct 2023 13:06:30 +0300 Subject: [PATCH 0622/1943] [#750] adm: Drop deprecated GetContract*() Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/balance.go | 4 +++- cmd/frostfs-adm/internal/modules/morph/config.go | 7 +++++-- .../internal/modules/morph/container.go | 13 ++++++++----- cmd/frostfs-adm/internal/modules/morph/deploy.go | 3 ++- .../internal/modules/morph/dump_hashes.go | 4 +++- cmd/frostfs-adm/internal/modules/morph/epoch.go | 4 +++- .../internal/modules/morph/initialize.go | 4 +++- .../internal/modules/morph/initialize_deploy.go | 16 +++++++++++----- .../internal/modules/morph/initialize_nns.go | 4 +++- .../internal/modules/morph/local_client.go | 15 --------------- .../internal/modules/morph/n3client.go | 2 -- .../internal/modules/morph/netmap_candidates.go | 4 +++- .../internal/modules/morph/remove_node.go | 4 +++- 13 files changed, 47 insertions(+), 37 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance.go index 6debc50b9..9b706c7bf 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance.go @@ -16,6 +16,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/gas" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/rpcclient/rolemgmt" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" @@ -56,7 +57,8 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { inv := invoker.New(c, nil) if dumpStorage || dumpAlphabet || dumpProxy { - nnsCs, err = c.GetContractStateByID(1) + r := management.NewReader(inv) + nnsCs, err = r.GetContractByID(1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index a86829aaa..3c4029bca 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -13,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/vm/emit" @@ -29,8 +30,9 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { } inv := invoker.New(c, nil) + r := management.NewReader(inv) - cs, err := c.GetContractStateByID(1) + cs, err := r.GetContractByID(1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) } @@ -87,7 +89,8 @@ func setConfigCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't initialize context: %w", err) } - cs, err := wCtx.Client.GetContractStateByID(1) + r := management.NewReader(wCtx.ReadOnlyInvoker) + cs, err := r.GetContractByID(1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/container.go b/cmd/frostfs-adm/internal/modules/morph/container.go index 687d7e84e..f3faa3044 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container.go @@ -11,6 +11,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/hash" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/util" @@ -22,14 +23,15 @@ import ( var errInvalidContainerResponse = errors.New("invalid response from container contract") -func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker, c Client) (util.Uint160, error) { +func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker) (util.Uint160, error) { s, err := cmd.Flags().GetString(containerContractFlag) var ch util.Uint160 if err == nil { ch, err = util.Uint160DecodeStringLE(s) } if err != nil { - nnsCs, err := c.GetContractStateByID(1) + r := management.NewReader(inv) + nnsCs, err := r.GetContractByID(1) if err != nil { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } @@ -78,7 +80,7 @@ func dumpContainers(cmd *cobra.Command, _ []string) error { inv := invoker.New(c, nil) - ch, err := getContainerContractHash(cmd, inv, c) + ch, err := getContainerContractHash(cmd, inv) if err != nil { return fmt.Errorf("unable to get contaract hash: %w", err) } @@ -168,7 +170,7 @@ func listContainers(cmd *cobra.Command, _ []string) error { inv := invoker.New(c, nil) - ch, err := getContainerContractHash(cmd, inv, c) + ch, err := getContainerContractHash(cmd, inv) if err != nil { return fmt.Errorf("unable to get contaract hash: %w", err) } @@ -298,7 +300,8 @@ func parseContainers(filename string) ([]Container, error) { } func fetchContainerContractHash(wCtx *initializeContext) (util.Uint160, error) { - nnsCs, err := wCtx.Client.GetContractStateByID(1) + r := management.NewReader(wCtx.ReadOnlyInvoker) + nnsCs, err := r.GetContractByID(1) if err != nil { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/deploy.go b/cmd/frostfs-adm/internal/modules/morph/deploy.go index a4b945438..54b506715 100644 --- a/cmd/frostfs-adm/internal/modules/morph/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/deploy.go @@ -76,7 +76,8 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { return err } - nnsCs, err := c.Client.GetContractStateByID(1) + r := management.NewReader(c.ReadOnlyInvoker) + nnsCs, err := r.GetContractByID(1) if err != nil { return fmt.Errorf("can't fetch NNS contract state: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go index 69db5c7bd..585de6d6d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go @@ -11,6 +11,7 @@ import ( morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/util" @@ -36,7 +37,8 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't create N3 client: %w", err) } - cs, err := c.GetContractStateByID(1) + r := management.NewReader(invoker.New(c, nil)) + cs, err := r.GetContractByID(1) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/epoch.go index a96efa43f..453e984b0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/epoch.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/util" @@ -20,7 +21,8 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't to initialize context: %w", err) } - cs, err := wCtx.Client.GetContractStateByID(1) + r := management.NewReader(wCtx.ReadOnlyInvoker) + cs, err := r.GetContractByID(1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index dec1fba20..4184e1a80 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -15,6 +15,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" @@ -312,7 +313,8 @@ func (c *initializeContext) nnsContractState() (*state.Contract, error) { return c.nnsCs, nil } - cs, err := c.Client.GetContractStateByID(1) + r := management.NewReader(c.ReadOnlyInvoker) + cs, err := r.GetContractByID(1) if err != nil { return nil, err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index a7cd537b7..6896b4ff0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -94,7 +94,10 @@ func (c *initializeContext) deployNNS(method string) error { h := cs.Hash nnsCs, err := c.nnsContractState() - if err == nil { + if err != nil { + return err + } + if nnsCs != nil { if nnsCs.NEF.Checksum == cs.NEF.Checksum { if method == deployMethodName { c.Command.Println("NNS contract is already deployed.") @@ -366,8 +369,9 @@ func (c *initializeContext) deployContracts() error { } func (c *initializeContext) isUpdated(ctrHash util.Uint160, cs *contractState) bool { - realCs, err := c.Client.GetContractStateByHash(ctrHash) - return err == nil && realCs.NEF.Checksum == cs.NEF.Checksum + r := management.NewReader(c.ReadOnlyInvoker) + realCs, err := r.GetContract(ctrHash) + return err == nil && realCs != nil && realCs.NEF.Checksum == cs.NEF.Checksum } func (c *initializeContext) getContract(ctrName string) *contractState { @@ -536,7 +540,8 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an case containerContract: // In case if NNS is updated multiple times, we can't calculate // it's actual hash based on local data, thus query chain. - nnsCs, err := c.Client.GetContractStateByID(1) + r := management.NewReader(c.ReadOnlyInvoker) + nnsCs, err := r.GetContractByID(1) if err != nil { panic("NNS is not yet deployed") } @@ -590,7 +595,8 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an } func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, error) { - cs, err := c.Client.GetContractStateByID(1) + r := management.NewReader(c.ReadOnlyInvoker) + cs, err := r.GetContractByID(1) if err != nil { return nil, fmt.Errorf("NNS is not yet deployed: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go index 6758b4dd8..1bf4c4bee 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go @@ -15,6 +15,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" nnsClient "github.com/nspcc-dev/neo-go/pkg/rpcclient/nns" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" @@ -30,7 +31,8 @@ const defaultExpirationTime = 10 * 365 * 24 * time.Hour / time.Second const frostfsOpsEmail = "ops@frostfs.info" func (c *initializeContext) setNNS() error { - nnsCs, err := c.Client.GetContractStateByID(1) + r := management.NewReader(c.ReadOnlyInvoker) + nnsCs, err := r.GetContractByID(1) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index 84cf40e97..cd094ca29 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -118,21 +118,6 @@ func (l *localClient) GetBlockCount() (uint32, error) { return l.bc.BlockHeight(), nil } -func (l *localClient) GetContractStateByID(id int32) (*state.Contract, error) { - h, err := l.bc.GetContractScriptHash(id) - if err != nil { - return nil, err - } - return l.GetContractStateByHash(h) -} - -func (l *localClient) GetContractStateByHash(h util.Uint160) (*state.Contract, error) { - if cs := l.bc.GetContractState(h); cs != nil { - return cs, nil - } - return nil, storage.ErrKeyNotFound -} - func (l *localClient) GetNativeContracts() ([]state.NativeContract, error) { return l.bc.GetNatives(), nil } diff --git a/cmd/frostfs-adm/internal/modules/morph/n3client.go b/cmd/frostfs-adm/internal/modules/morph/n3client.go index 77f207c9f..7ba5a7ce3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/n3client.go @@ -28,8 +28,6 @@ type Client interface { invoker.RPCInvoke GetBlockCount() (uint32, error) - GetContractStateByID(int32) (*state.Contract, error) - GetContractStateByHash(util.Uint160) (*state.Contract, error) GetNativeContracts() ([]state.NativeContract, error) GetApplicationLog(util.Uint256, *trigger.Type) (*result.ApplicationLog, error) GetVersion() (*result.Version, error) diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go index 222b9902a..30f7c69c4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go @@ -5,6 +5,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -14,8 +15,9 @@ func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "can't create N3 client: %w", err) inv := invoker.New(c, nil) + r := management.NewReader(inv) - cs, err := c.GetContractStateByID(1) + cs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract info: %w", err) nmHash, err := nnsResolveHash(inv, cs.Hash, netmapContract+".frostfs") diff --git a/cmd/frostfs-adm/internal/modules/morph/remove_node.go b/cmd/frostfs-adm/internal/modules/morph/remove_node.go index df67433c4..ba7bb90ff 100644 --- a/cmd/frostfs-adm/internal/modules/morph/remove_node.go +++ b/cmd/frostfs-adm/internal/modules/morph/remove_node.go @@ -7,6 +7,7 @@ import ( netmapcontract "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/spf13/cobra" @@ -33,7 +34,8 @@ func removeNodesCmd(cmd *cobra.Command, args []string) error { } defer wCtx.close() - cs, err := wCtx.Client.GetContractStateByID(1) + r := management.NewReader(wCtx.ReadOnlyInvoker) + cs, err := r.GetContractByID(1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) } From 4368243bed365983e992a71c14fcdc08fe4cff4d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 24 Oct 2023 13:13:49 +0300 Subject: [PATCH 0623/1943] [#750] adm: Drop deprecated NEP17BalanceOf() Signed-off-by: Evgenii Stratonikov --- .../modules/morph/initialize_register.go | 7 +++++-- .../modules/morph/initialize_transfer.go | 12 ++++++++---- .../internal/modules/morph/local_client.go | 17 ----------------- .../internal/modules/morph/n3client.go | 1 - 4 files changed, 13 insertions(+), 24 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index 469b269de..60bb7f8ed 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -3,6 +3,7 @@ package morph import ( "errors" "fmt" + "math/big" "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -11,6 +12,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/neo" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/util" @@ -134,8 +136,9 @@ func (c *initializeContext) transferNEOToAlphabetContracts() error { } func (c *initializeContext) transferNEOFinished(neoHash util.Uint160) (bool, error) { - bal, err := c.Client.NEP17BalanceOf(neoHash, c.CommitteeAcc.Contract.ScriptHash()) - return bal < native.NEOTotalSupply, err + r := nep17.NewReader(c.ReadOnlyInvoker, neoHash) + bal, err := r.BalanceOf(c.CommitteeAcc.Contract.ScriptHash()) + return bal.Cmp(big.NewInt(native.NEOTotalSupply)) == -1, err } var errGetPriceInvalid = errors.New("`getRegisterPrice`: invalid response") diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go index b032780be..45db6489f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go @@ -2,6 +2,7 @@ package morph import ( "fmt" + "math/big" "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -9,6 +10,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/gas" "github.com/nspcc-dev/neo-go/pkg/rpcclient/neo" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" scContext "github.com/nspcc-dev/neo-go/pkg/smartcontract/context" "github.com/nspcc-dev/neo-go/pkg/vm/emit" @@ -80,8 +82,9 @@ func (c *initializeContext) transferFunds() error { func (c *initializeContext) transferFundsFinished() (bool, error) { acc := c.Accounts[0] - res, err := c.Client.NEP17BalanceOf(gas.Hash, acc.Contract.ScriptHash()) - return res > initialAlphabetGASAmount/2, err + r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) + res, err := r.BalanceOf(acc.Contract.ScriptHash()) + return res.Cmp(big.NewInt(initialAlphabetGASAmount/2)) == 1, err } func (c *initializeContext) multiSignAndSend(tx *transaction.Transaction, accType string) error { @@ -147,8 +150,9 @@ func (c *initializeContext) multiSign(tx *transaction.Transaction, accType strin func (c *initializeContext) transferGASToProxy() error { proxyCs := c.getContract(proxyContract) - bal, err := c.Client.NEP17BalanceOf(gas.Hash, proxyCs.Hash) - if err != nil || bal > 0 { + r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) + bal, err := r.BalanceOf(proxyCs.Hash) + if err != nil || bal.Sign() > 0 { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index cd094ca29..2ef8774bb 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -35,7 +35,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -324,22 +323,6 @@ func getSigners(sender *wallet.Account, cosigners []rpcclient.SignerAccount) ([] return signers, accounts, nil } -func (l *localClient) NEP17BalanceOf(h util.Uint160, acc util.Uint160) (int64, error) { - res, err := invokeFunction(l, h, "balanceOf", []any{acc}, nil) - if err != nil { - return 0, err - } - if res.State != vmstate.Halt.String() || len(res.Stack) == 0 { - return 0, fmt.Errorf("`balance`: invalid response (empty: %t): %s", - len(res.Stack) == 0, res.FaultException) - } - bi, err := res.Stack[0].TryInteger() - if err != nil || !bi.IsInt64() { - return 0, fmt.Errorf("`balance`: invalid response") - } - return bi.Int64(), nil -} - func (l *localClient) InvokeScript(script []byte, signers []transaction.Signer) (*result.Invoke, error) { lastBlock, err := l.bc.GetBlock(l.bc.CurrentBlockHash()) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/n3client.go b/cmd/frostfs-adm/internal/modules/morph/n3client.go index 7ba5a7ce3..ac9abee8c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/n3client.go @@ -32,7 +32,6 @@ type Client interface { GetApplicationLog(util.Uint256, *trigger.Type) (*result.ApplicationLog, error) GetVersion() (*result.Version, error) CreateTxFromScript([]byte, *wallet.Account, int64, int64, []rpcclient.SignerAccount) (*transaction.Transaction, error) - NEP17BalanceOf(util.Uint160, util.Uint160) (int64, error) SendRawTransaction(*transaction.Transaction) (util.Uint256, error) GetCommittee() (keys.PublicKeys, error) CalculateNotaryFee(uint8) (int64, error) From a788c24e6d6a4ab7681102da27f44bc77676d099 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 24 Oct 2023 13:23:12 +0300 Subject: [PATCH 0624/1943] [#750] adm: Drop deprecated AddNetworkFee() Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/local_client.go | 114 ++++++++++-------- .../internal/modules/morph/n3client.go | 1 - 2 files changed, 65 insertions(+), 50 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index 2ef8774bb..eca9f45b7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -13,12 +13,10 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/chaindump" - "github.com/nspcc-dev/neo-go/pkg/core/fee" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/storage" "github.com/nspcc-dev/neo-go/pkg/core/transaction" - "github.com/nspcc-dev/neo-go/pkg/crypto/hash" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" @@ -30,8 +28,10 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" @@ -150,12 +150,17 @@ func (l *localClient) CreateTxFromScript(script []byte, acc *wallet.Account, sys tx := transaction.New(script, sysFee) tx.Signers = signers tx.ValidUntilBlock = l.bc.BlockHeight() + 2 + tx.Scripts = make([]transaction.Witness, len(accounts)) + for i := range tx.Scripts { + tx.Scripts[i].VerificationScript = accounts[i].Contract.Script + } - err = l.AddNetworkFee(tx, netFee, accounts...) + fee, err := l.CalculateNetworkFee(tx) if err != nil { return nil, fmt.Errorf("failed to add network fee: %w", err) } + tx.NetworkFee = fee + netFee return tx, nil } @@ -233,67 +238,78 @@ func (l *localClient) InvokeContractVerify(util.Uint160, []smartcontract.Paramet // CalculateNetworkFee calculates network fee for the given transaction. // Copied from neo-go with minor corrections (no need to support non-notary mode): -// https://github.com/nspcc-dev/neo-go/blob/v0.99.2/pkg/services/rpcsrv/server.go#L744 +// https://github.com/nspcc-dev/neo-go/blob/v0.103.0/pkg/services/rpcsrv/server.go#L911 func (l *localClient) CalculateNetworkFee(tx *transaction.Transaction) (int64, error) { + // Avoid setting hash for this tx: server code doesn't touch client transaction. + data := tx.Bytes() + tx, err := transaction.NewTransactionFromBytes(data) + if err != nil { + return 0, err + } + hashablePart, err := tx.EncodeHashableFields() if err != nil { - return 0, fmt.Errorf("failed to compute tx size: %w", err) + return 0, err } - size := len(hashablePart) + io.GetVarSize(len(tx.Signers)) - ef := l.bc.GetBaseExecFee() - - var netFee int64 + var ( + netFee int64 + // Verification GAS cost can't exceed this policy. + gasLimit = l.bc.GetMaxVerificationGAS() + ) for i, signer := range tx.Signers { - var verificationScript []byte - for _, w := range tx.Scripts { - if w.VerificationScript != nil && hash.Hash160(w.VerificationScript).Equals(signer.Account) { - verificationScript = w.VerificationScript - break + w := tx.Scripts[i] + if len(w.InvocationScript) == 0 { // No invocation provided, try to infer one. + var paramz []manifest.Parameter + if len(w.VerificationScript) == 0 { // Contract-based verification + cs := l.bc.GetContractState(signer.Account) + if cs == nil { + return 0, fmt.Errorf("signer %d has no verification script and no deployed contract", i) + } + md := cs.Manifest.ABI.GetMethod(manifest.MethodVerify, -1) + if md == nil || md.ReturnType != smartcontract.BoolType { + return 0, fmt.Errorf("signer %d has no verify method in deployed contract", i) + } + paramz = md.Parameters // Might as well have none params and it's OK. + } else { // Regular signature verification. + if vm.IsSignatureContract(w.VerificationScript) { + paramz = []manifest.Parameter{{Type: smartcontract.SignatureType}} + } else if nSigs, _, ok := vm.ParseMultiSigContract(w.VerificationScript); ok { + paramz = make([]manifest.Parameter, nSigs) + for j := 0; j < nSigs; j++ { + paramz[j] = manifest.Parameter{Type: smartcontract.SignatureType} + } + } } - } - if verificationScript == nil { - gasConsumed, err := l.bc.VerifyWitness(signer.Account, tx, &tx.Scripts[i], l.maxGasInvoke) - if err != nil { - return 0, fmt.Errorf("invalid signature: %w", err) + inv := io.NewBufBinWriter() + for _, p := range paramz { + p.Type.EncodeDefaultValue(inv.BinWriter) } - netFee += gasConsumed - size += io.GetVarSize([]byte{}) + io.GetVarSize(tx.Scripts[i].InvocationScript) - continue + if inv.Err != nil { + return 0, fmt.Errorf("failed to create dummy invocation script (signer %d): %s", i, inv.Err.Error()) + } + w.InvocationScript = inv.Bytes() + } + gasConsumed, err := l.bc.VerifyWitness(signer.Account, tx, &w, gasLimit) + if err != nil && !errors.Is(err, core.ErrInvalidSignature) { + return 0, err + } + gasLimit -= gasConsumed + netFee += gasConsumed + size += io.GetVarSize(w.VerificationScript) + io.GetVarSize(w.InvocationScript) + } + if l.bc.P2PSigExtensionsEnabled() { + attrs := tx.GetAttributes(transaction.NotaryAssistedT) + if len(attrs) != 0 { + na := attrs[0].Value.(*transaction.NotaryAssisted) + netFee += (int64(na.NKeys) + 1) * l.bc.GetNotaryServiceFeePerKey() } - - fee, sizeDelta := fee.Calculate(ef, verificationScript) - netFee += fee - size += sizeDelta } - fee := l.bc.FeePerByte() netFee += int64(size) * fee - return netFee, nil } -// AddNetworkFee adds network fee for each witness script and optional extra -// network fee to transaction. `accs` is an array signer's accounts. -// Copied from neo-go with minor corrections (no need to support contract signers): -// https://github.com/nspcc-dev/neo-go/blob/6ff11baa1b9e4c71ef0d1de43b92a8c541ca732c/pkg/rpc/client/rpc.go#L960 -func (l *localClient) AddNetworkFee(tx *transaction.Transaction, extraFee int64, accs ...*wallet.Account) error { - if len(tx.Signers) != len(accs) { - return errors.New("number of signers must match number of scripts") - } - - size := io.GetVarSize(tx) - ef := l.bc.GetBaseExecFee() - for i := range tx.Signers { - netFee, sizeDelta := fee.Calculate(ef, accs[i].Contract.Script) - tx.NetworkFee += netFee - size += sizeDelta - } - - tx.NetworkFee += int64(size)*l.bc.FeePerByte() + extraFee - return nil -} - // getSigners returns an array of transaction signers and corresponding accounts from // given sender and cosigners. If cosigners list already contains sender, the sender // will be placed at the start of the list. diff --git a/cmd/frostfs-adm/internal/modules/morph/n3client.go b/cmd/frostfs-adm/internal/modules/morph/n3client.go index ac9abee8c..e3042740b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/n3client.go @@ -36,7 +36,6 @@ type Client interface { GetCommittee() (keys.PublicKeys, error) CalculateNotaryFee(uint8) (int64, error) CalculateNetworkFee(tx *transaction.Transaction) (int64, error) - AddNetworkFee(*transaction.Transaction, int64, ...*wallet.Account) error SignAndPushInvocationTx([]byte, *wallet.Account, int64, fixedn.Fixed8, []rpcclient.SignerAccount) (util.Uint256, error) SignAndPushP2PNotaryRequest(*transaction.Transaction, []byte, int64, int64, uint32, *wallet.Account) (*payload.P2PNotaryRequest, error) } From b0d303f3ed40a500cb8a838aac794be4a55f1a63 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 24 Oct 2023 14:52:46 +0300 Subject: [PATCH 0625/1943] [#750] adm: Drop unused methods from Client Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/local_client.go | 17 ----------------- .../internal/modules/morph/n3client.go | 5 ----- 2 files changed, 22 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index eca9f45b7..b00eca6b4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -19,10 +19,8 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" - "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/network/payload" "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" @@ -184,21 +182,6 @@ func (l *localClient) InvokeFunction(h util.Uint160, method string, sPrm []smart return invokeFunction(l, h, method, pp, ss) } -func (l *localClient) CalculateNotaryFee(_ uint8) (int64, error) { - // not used by `morph init` command - panic("unexpected call") -} - -func (l *localClient) SignAndPushP2PNotaryRequest(_ *transaction.Transaction, _ []byte, _ int64, _ int64, _ uint32, _ *wallet.Account) (*payload.P2PNotaryRequest, error) { - // not used by `morph init` command - panic("unexpected call") -} - -func (l *localClient) SignAndPushInvocationTx(_ []byte, _ *wallet.Account, _ int64, _ fixedn.Fixed8, _ []rpcclient.SignerAccount) (util.Uint256, error) { - // not used by `morph init` command - panic("unexpected call") -} - func (l *localClient) TerminateSession(_ uuid.UUID) (bool, error) { // not used by `morph init` command panic("unexpected call") diff --git a/cmd/frostfs-adm/internal/modules/morph/n3client.go b/cmd/frostfs-adm/internal/modules/morph/n3client.go index e3042740b..d99715592 100644 --- a/cmd/frostfs-adm/internal/modules/morph/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/n3client.go @@ -9,9 +9,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/network/payload" "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -34,10 +32,7 @@ type Client interface { CreateTxFromScript([]byte, *wallet.Account, int64, int64, []rpcclient.SignerAccount) (*transaction.Transaction, error) SendRawTransaction(*transaction.Transaction) (util.Uint256, error) GetCommittee() (keys.PublicKeys, error) - CalculateNotaryFee(uint8) (int64, error) CalculateNetworkFee(tx *transaction.Transaction) (int64, error) - SignAndPushInvocationTx([]byte, *wallet.Account, int64, fixedn.Fixed8, []rpcclient.SignerAccount) (util.Uint256, error) - SignAndPushP2PNotaryRequest(*transaction.Transaction, []byte, int64, int64, uint32, *wallet.Account) (*payload.P2PNotaryRequest, error) } type hashVUBPair struct { From 7f35f2fb1dbc2b192e1cba32aa847f358f19eee7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 24 Oct 2023 15:12:59 +0300 Subject: [PATCH 0626/1943] [#750] adm: Drop deprecated CreateTxFromScript() Signed-off-by: Evgenii Stratonikov --- .../modules/morph/initialize_deploy.go | 20 +----- .../modules/morph/initialize_register.go | 11 +++- .../modules/morph/initialize_transfer.go | 30 ++++----- .../internal/modules/morph/local_client.go | 63 ------------------- .../internal/modules/morph/n3client.go | 1 - 5 files changed, 25 insertions(+), 100 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 6896b4ff0..350cd611c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -18,10 +18,8 @@ import ( morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/encoding/address" io2 "github.com/nspcc-dev/neo-go/pkg/io" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" @@ -33,7 +31,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" ) const ( @@ -115,28 +112,13 @@ func (c *initializeContext) deployNNS(method string) error { } params := getContractDeployParameters(cs, nil) - signer := transaction.Signer{ - Account: c.CommitteeAcc.Contract.ScriptHash(), - Scopes: transaction.CalledByEntry, - } invokeHash := management.Hash if method == updateMethodName { invokeHash = nnsCs.Hash } - res, err := invokeFunction(c.Client, invokeHash, method, params, []transaction.Signer{signer}) - if err != nil { - return fmt.Errorf("can't deploy NNS contract: %w", err) - } - if res.State != vmstate.Halt.String() { - return fmt.Errorf("can't deploy NNS contract: %s", res.FaultException) - } - - tx, err := c.Client.CreateTxFromScript(res.Script, c.CommitteeAcc, res.GasConsumed, 0, []rpcclient.SignerAccount{{ - Signer: signer, - Account: c.CommitteeAcc, - }}) + tx, err := c.CommitteeAct.MakeCall(invokeHash, method, params...) if err != nil { return fmt.Errorf("failed to create deploy tx for %s: %w", nnsContract, err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index 60bb7f8ed..92145a796 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -10,6 +10,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/neo" "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" @@ -43,12 +44,12 @@ func (c *initializeContext) registerCandidateRange(start, end int) error { panic(fmt.Sprintf("BUG: %v", w.Err)) } - signers := []rpcclient.SignerAccount{{ + signers := []actor.SignerAccount{{ Signer: c.getSigner(false, c.CommitteeAcc), Account: c.CommitteeAcc, }} for _, acc := range c.Accounts[start:end] { - signers = append(signers, rpcclient.SignerAccount{ + signers = append(signers, actor.SignerAccount{ Signer: transaction.Signer{ Account: acc.Contract.ScriptHash(), Scopes: transaction.CustomContracts, @@ -58,7 +59,11 @@ func (c *initializeContext) registerCandidateRange(start, end int) error { }) } - tx, err := c.Client.CreateTxFromScript(w.Bytes(), c.CommitteeAcc, -1, 0, signers) + act, err := actor.New(c.Client, signers) + if err != nil { + return fmt.Errorf("can't create actor: %w", err) + } + tx, err := act.MakeRun(w.Bytes()) if err != nil { return fmt.Errorf("can't create tx: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go index 45db6489f..3d214d0cf 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go @@ -8,6 +8,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/gas" "github.com/nspcc-dev/neo-go/pkg/rpcclient/neo" "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" @@ -61,13 +62,7 @@ func (c *initializeContext) transferFunds() error { }, ) - tx, err := createNEP17MultiTransferTx(c.Client, c.ConsensusAcc, 0, transfers, []rpcclient.SignerAccount{{ - Signer: transaction.Signer{ - Account: c.ConsensusAcc.Contract.ScriptHash(), - Scopes: transaction.CalledByEntry, - }, - Account: c.ConsensusAcc, - }}) + tx, err := createNEP17MultiTransferTx(c.Client, c.ConsensusAcc, transfers) if err != nil { return fmt.Errorf("can't create transfer transaction: %w", err) } @@ -156,11 +151,11 @@ func (c *initializeContext) transferGASToProxy() error { return err } - tx, err := createNEP17MultiTransferTx(c.Client, c.CommitteeAcc, 0, []rpcclient.TransferTarget{{ + tx, err := createNEP17MultiTransferTx(c.Client, c.CommitteeAcc, []rpcclient.TransferTarget{{ Token: gas.Hash, Address: proxyCs.Hash, Amount: initialProxyGASAmount, - }}, nil) + }}) if err != nil { return err } @@ -172,8 +167,8 @@ func (c *initializeContext) transferGASToProxy() error { return c.awaitTx() } -func createNEP17MultiTransferTx(c Client, acc *wallet.Account, netFee int64, - recipients []rpcclient.TransferTarget, cosigners []rpcclient.SignerAccount) (*transaction.Transaction, error) { +func createNEP17MultiTransferTx(c Client, acc *wallet.Account, + recipients []rpcclient.TransferTarget) (*transaction.Transaction, error) { from := acc.Contract.ScriptHash() w := io.NewBufBinWriter() @@ -185,11 +180,18 @@ func createNEP17MultiTransferTx(c Client, acc *wallet.Account, netFee int64, if w.Err != nil { return nil, fmt.Errorf("failed to create transfer script: %w", w.Err) } - return c.CreateTxFromScript(w.Bytes(), acc, -1, netFee, append([]rpcclient.SignerAccount{{ + + signers := []actor.SignerAccount{{ Signer: transaction.Signer{ - Account: from, + Account: acc.Contract.ScriptHash(), Scopes: transaction.CalledByEntry, }, Account: acc, - }}, cosigners...)) + }} + + act, err := actor.New(c, signers) + if err != nil { + return nil, fmt.Errorf("can't create actor: %w", err) + } + return act.MakeRun(w.Bytes()) } diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index b00eca6b4..24dcb7dec 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -21,7 +21,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract" @@ -129,39 +128,6 @@ func (l *localClient) GetApplicationLog(h util.Uint256, t *trigger.Type) (*resul return &a, nil } -func (l *localClient) CreateTxFromScript(script []byte, acc *wallet.Account, sysFee int64, netFee int64, cosigners []rpcclient.SignerAccount) (*transaction.Transaction, error) { - signers, accounts, err := getSigners(acc, cosigners) - if err != nil { - return nil, fmt.Errorf("failed to construct tx signers: %w", err) - } - if sysFee < 0 { - res, err := l.InvokeScript(script, signers) - if err != nil { - return nil, fmt.Errorf("can't add system fee to transaction: %w", err) - } - if res.State != "HALT" { - return nil, fmt.Errorf("can't add system fee to transaction: bad vm state: %s due to an error: %s", res.State, res.FaultException) - } - sysFee = res.GasConsumed - } - - tx := transaction.New(script, sysFee) - tx.Signers = signers - tx.ValidUntilBlock = l.bc.BlockHeight() + 2 - tx.Scripts = make([]transaction.Witness, len(accounts)) - for i := range tx.Scripts { - tx.Scripts[i].VerificationScript = accounts[i].Contract.Script - } - - fee, err := l.CalculateNetworkFee(tx) - if err != nil { - return nil, fmt.Errorf("failed to add network fee: %w", err) - } - - tx.NetworkFee = fee + netFee - return tx, nil -} - func (l *localClient) GetCommittee() (keys.PublicKeys, error) { // not used by `morph init` command panic("unexpected call") @@ -293,35 +259,6 @@ func (l *localClient) CalculateNetworkFee(tx *transaction.Transaction) (int64, e return netFee, nil } -// getSigners returns an array of transaction signers and corresponding accounts from -// given sender and cosigners. If cosigners list already contains sender, the sender -// will be placed at the start of the list. -// Copied from neo-go with minor corrections: -// https://github.com/nspcc-dev/neo-go/blob/6ff11baa1b9e4c71ef0d1de43b92a8c541ca732c/pkg/rpc/client/rpc.go#L735 -func getSigners(sender *wallet.Account, cosigners []rpcclient.SignerAccount) ([]transaction.Signer, []*wallet.Account, error) { - var ( - signers []transaction.Signer - accounts []*wallet.Account - ) - - from := sender.Contract.ScriptHash() - s := transaction.Signer{ - Account: from, - Scopes: transaction.None, - } - for _, c := range cosigners { - if c.Signer.Account == from { - s = c.Signer - continue - } - signers = append(signers, c.Signer) - accounts = append(accounts, c.Account) - } - signers = append([]transaction.Signer{s}, signers...) - accounts = append([]*wallet.Account{sender}, accounts...) - return signers, accounts, nil -} - func (l *localClient) InvokeScript(script []byte, signers []transaction.Signer) (*result.Invoke, error) { lastBlock, err := l.bc.GetBlock(l.bc.CurrentBlockHash()) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/n3client.go b/cmd/frostfs-adm/internal/modules/morph/n3client.go index d99715592..3550895c3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/n3client.go @@ -29,7 +29,6 @@ type Client interface { GetNativeContracts() ([]state.NativeContract, error) GetApplicationLog(util.Uint256, *trigger.Type) (*result.ApplicationLog, error) GetVersion() (*result.Version, error) - CreateTxFromScript([]byte, *wallet.Account, int64, int64, []rpcclient.SignerAccount) (*transaction.Transaction, error) SendRawTransaction(*transaction.Transaction) (util.Uint256, error) GetCommittee() (keys.PublicKeys, error) CalculateNetworkFee(tx *transaction.Transaction) (int64, error) From 4239f1e81771affc467ff93376f5d94cf85512be Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 24 Oct 2023 15:20:38 +0300 Subject: [PATCH 0627/1943] [#750] adm: Drop deprecated rpcclient.TransferTarget We do not use `nep17` wrapper, because transfers of different tokens are possible in a single transaction. Signed-off-by: Evgenii Stratonikov --- .../modules/morph/initialize_transfer.go | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go index 3d214d0cf..12b94a2ed 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go @@ -7,13 +7,13 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/io" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/gas" "github.com/nspcc-dev/neo-go/pkg/rpcclient/neo" "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" scContext "github.com/nspcc-dev/neo-go/pkg/smartcontract/context" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" "github.com/nspcc-dev/neo-go/pkg/wallet" @@ -36,11 +36,11 @@ func (c *initializeContext) transferFunds() error { return err } - var transfers []rpcclient.TransferTarget + var transfers []transferTarget for _, acc := range c.Accounts { to := acc.Contract.ScriptHash() transfers = append(transfers, - rpcclient.TransferTarget{ + transferTarget{ Token: gas.Hash, Address: to, Amount: initialAlphabetGASAmount, @@ -50,12 +50,12 @@ func (c *initializeContext) transferFunds() error { // It is convenient to have all funds at the committee account. transfers = append(transfers, - rpcclient.TransferTarget{ + transferTarget{ Token: gas.Hash, Address: c.CommitteeAcc.Contract.ScriptHash(), Amount: (gasInitialTotalSupply - initialAlphabetGASAmount*int64(len(c.Wallets))) / 2, }, - rpcclient.TransferTarget{ + transferTarget{ Token: neo.Hash, Address: c.CommitteeAcc.Contract.ScriptHash(), Amount: native.NEOTotalSupply, @@ -151,7 +151,7 @@ func (c *initializeContext) transferGASToProxy() error { return err } - tx, err := createNEP17MultiTransferTx(c.Client, c.CommitteeAcc, []rpcclient.TransferTarget{{ + tx, err := createNEP17MultiTransferTx(c.Client, c.CommitteeAcc, []transferTarget{{ Token: gas.Hash, Address: proxyCs.Hash, Amount: initialProxyGASAmount, @@ -167,8 +167,14 @@ func (c *initializeContext) transferGASToProxy() error { return c.awaitTx() } -func createNEP17MultiTransferTx(c Client, acc *wallet.Account, - recipients []rpcclient.TransferTarget) (*transaction.Transaction, error) { +type transferTarget struct { + Token util.Uint160 + Address util.Uint160 + Amount int64 + Data any +} + +func createNEP17MultiTransferTx(c Client, acc *wallet.Account, recipients []transferTarget) (*transaction.Transaction, error) { from := acc.Contract.ScriptHash() w := io.NewBufBinWriter() From 2dbf5c612ad69f48dabe87b6dca79ab931e1d148 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Oct 2023 10:30:13 +0300 Subject: [PATCH 0628/1943] [#752] go.mod: Update neo-go to v0.103.0 Signed-off-by: Evgenii Stratonikov --- go.mod | 25 ++++++++++++------------- go.sum | 57 +++++++++++++++++++++++++++++---------------------------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index a97e04f33..0d2d67ca0 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.9.0 github.com/nats-io/nats.go v1.27.1 - github.com/nspcc-dev/neo-go v0.101.5-0.20230808195420-5fc61be5f6c5 + github.com/nspcc-dev/neo-go v0.103.0 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.7.5 github.com/paulmach/orb v0.9.2 @@ -34,10 +34,10 @@ require ( go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df + golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/sync v0.3.0 - golang.org/x/term v0.9.0 - google.golang.org/grpc v1.56.1 + golang.org/x/term v0.13.0 + google.golang.org/grpc v1.57.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -62,7 +62,7 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.11.0 // indirect + github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -97,7 +97,7 @@ require ( github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230808195420-5fc61be5f6c5 // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231020160724-c3955f87d1b5 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -123,13 +123,12 @@ require ( go.opentelemetry.io/proto/otlp v0.20.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.10.0 // indirect - golang.org/x/net v0.11.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect - google.golang.org/genproto v0.0.0-20230628200519-e449d1ea0e82 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230628200519-e449d1ea0e82 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230628200519-e449d1ea0e82 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 29134ea57..31c705226 100644 --- a/go.sum +++ b/go.sum @@ -485,8 +485,8 @@ github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWH github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.9.1/go.mod h1:a2DQL4+5ywF6safEeZFEPGRiiGbjzGFRUN2sg06VuU4= -github.com/consensys/gnark-crypto v0.11.0 h1:QqzHQlwEqlQr5jfWblGDkwlKHpT+4QodYqqExkAtyks= -github.com/consensys/gnark-crypto v0.11.0/go.mod h1:Iq/P3HHl0ElSjsg2E1gsMwhAyxnxoKK5nVyZKd+/KhU= +github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb h1:f0BMgIjhZy4lSRHCXFbQst85f5agZAjtDMixQqBWNpc= +github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= @@ -739,13 +739,13 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= @@ -811,13 +811,15 @@ github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1: github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= github.com/nspcc-dev/neo-go v0.99.2/go.mod h1:9P0yWqhZX7i/ChJ+zjtiStO1uPTolPFUM+L5oNznU8E= github.com/nspcc-dev/neo-go v0.100.1/go.mod h1:Nnp7F4e9IBccsgtCeLtUWV+0T6gk1PtP5HRtA13hUfc= -github.com/nspcc-dev/neo-go v0.101.5-0.20230808195420-5fc61be5f6c5 h1:AXI2upTPeTqX+n4xrBEzPATgEviOM/Prg6UQ6KDm+DU= github.com/nspcc-dev/neo-go v0.101.5-0.20230808195420-5fc61be5f6c5/go.mod h1:Z0kpjwnTJj/ik/X6z18xjCL0X2+RNbqlnhKrl+MYgP8= +github.com/nspcc-dev/neo-go v0.103.0 h1:UVyWPhzZdfYFG35ORP3FRDLh8J/raRQ6m8SptDdlgfM= +github.com/nspcc-dev/neo-go v0.103.0/go.mod h1:x+wmcYqpZYJwLp1l/pHZrqNp3RSWlkMymWGDij3/OPo= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220809123759-3094d3e0c14b/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230420112658-c50ab951645a/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230808195420-5fc61be5f6c5 h1:/d7mY5hYlNhmEXexKcyqSR0b1Hdl5hf/c5o8Vi/1vt4= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230808195420-5fc61be5f6c5/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231020160724-c3955f87d1b5 h1:09CpI5uwsxb1EeFPIKQRwwWlfCmDD/Dwwh01lPiQScM= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231020160724-c3955f87d1b5/go.mod h1:J/Mk6+nKeKSW4wygkZQFLQ6SkLOSGX5Ga0RuuuktEag= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.14.0/go.mod h1:DRIr0Ic1s+6QgdqmNFNLIqMqd7lNMJfYwkczlm1hDtM= @@ -1052,8 +1054,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1066,8 +1068,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/exp v0.0.0-20221227203929-1b447090c38c/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1099,7 +1101,7 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1163,8 +1165,8 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1306,8 +1308,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1316,8 +1318,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1332,8 +1334,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1402,7 +1404,7 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1582,12 +1584,11 @@ google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZV google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230628200519-e449d1ea0e82 h1:Wdfp5Hc1bqGCWYZNrir4A1Jb+SmVaV2j1DL/pbMMTGI= -google.golang.org/genproto v0.0.0-20230628200519-e449d1ea0e82/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230628200519-e449d1ea0e82 h1:iI5Fmsfz4zDINYxJLxn2YChI//ypkHM/KuVSvlN7ZXk= -google.golang.org/genproto/googleapis/api v0.0.0-20230628200519-e449d1ea0e82/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230628200519-e449d1ea0e82 h1:6b+zGQBiXFlAMpQr+cCarAdrZD4QgXSG7uUZadYysgg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230628200519-e449d1ea0e82/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e h1:xIXmWJ303kJCuogpj0bHq+dcjcZHU+XFyc1I0Yl9cRg= +google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 h1:XVeBY8d/FaK4848myy41HBqnDwvxeV3zMZhwN1TvAMU= +google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1627,8 +1628,8 @@ google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= -google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= -google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 4f62fded0106e2b22a828bcc96b60e334d937735 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Oct 2023 10:33:06 +0300 Subject: [PATCH 0629/1943] [#752] go.mod: Update dependencies Signed-off-by: Evgenii Stratonikov --- go.mod | 22 ++++++++++------------ go.sum | 48 +++++++++++++++++++++++------------------------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index 0d2d67ca0..bbba2e3c3 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,9 @@ require ( github.com/chzyer/readline v1.5.1 github.com/dgraph-io/ristretto v0.1.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 - github.com/google/uuid v1.3.0 - github.com/hashicorp/golang-lru/v2 v2.0.4 - github.com/klauspost/compress v1.16.6 + github.com/google/uuid v1.3.1 + github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/klauspost/compress v1.17.2 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.9.0 @@ -33,11 +33,11 @@ require ( go.etcd.io/bbolt v1.3.7 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 - go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 - golang.org/x/sync v0.3.0 + go.uber.org/zap v1.26.0 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d + golang.org/x/sync v0.4.0 golang.org/x/term v0.13.0 - google.golang.org/grpc v1.57.0 + google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -45,7 +45,7 @@ require ( require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.1.0 // indirect + github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -56,7 +56,6 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect - github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.8.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect @@ -121,14 +120,13 @@ require ( go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/otel/sdk v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.20.0 // indirect - go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 31c705226..bb5757b45 100644 --- a/go.sum +++ b/go.sum @@ -427,8 +427,6 @@ github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12/g github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= -github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -561,8 +559,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -646,8 +644,9 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= @@ -680,8 +679,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.4 h1:7GHuZcgid37q8o5i3QI9KMT4nCWQQ3Kx3Ov6bb9MfK0= -github.com/hashicorp/golang-lru/v2 v2.0.4/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= @@ -714,8 +713,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= -github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1021,8 +1020,6 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -1033,8 +1030,9 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1068,8 +1066,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/exp v0.0.0-20221227203929-1b447090c38c/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1101,7 +1099,7 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1208,8 +1206,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1404,7 +1402,7 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= -golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1584,11 +1582,11 @@ google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZV google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e h1:xIXmWJ303kJCuogpj0bHq+dcjcZHU+XFyc1I0Yl9cRg= -google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 h1:XVeBY8d/FaK4848myy41HBqnDwvxeV3zMZhwN1TvAMU= -google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= +google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a h1:fwgW9j3vHirt4ObdHoYNwuO24BEZjSzbh+zPaNWoiY8= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1628,8 +1626,8 @@ google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 695031296706487052b61918243356a6538ea220 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Oct 2023 11:32:53 +0300 Subject: [PATCH 0630/1943] [#752] morph: Drop loop copy kludges Signed-off-by: Evgenii Stratonikov --- pkg/morph/subscriber/subscriber.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 7f3c91385..c291988fd 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -310,7 +310,6 @@ func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotific // notification events restoration for contract := range s.subscribedEvents { - contract := contract // See https://github.com/nspcc-dev/neo-go/issues/2890 _, err = s.client.ReceiveExecutionNotifications(contract, notifCh) if err != nil { s.log.Error(logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) @@ -320,7 +319,6 @@ func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotific // notary notification events restoration for signer := range s.subscribedNotaryEvents { - signer := signer // See https://github.com/nspcc-dev/neo-go/issues/2890 _, err = s.client.ReceiveNotaryRequests(signer, notaryCh) if err != nil { s.log.Error(logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) From 0a9830564ff2c91d8777156389c3aaf3bdfb2b8a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Oct 2023 15:58:22 +0300 Subject: [PATCH 0631/1943] [#752] morph: Adopt neo-go RPC statuses Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/notary.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 17644361a..0bbe526a6 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -706,13 +706,17 @@ func WithProxyContract(h util.Uint160) NotaryOption { } } -// Neo RPC node can return `core.ErrInvalidAttribute` error with +// Neo RPC node can return `neorpc.ErrInvalidAttribute` error with // `conflicting transaction <> is already on chain` message. This // error is expected and ignored. As soon as main tx persisted on // chain everything is fine. This happens because notary contract // requires 5 out of 7 signatures to send main tx, thus last two // notary requests may be processed after main tx appeared on chain. func alreadyOnChainError(err error) bool { + if !errors.Is(err, neorpc.ErrInvalidAttribute) { + return false + } + const alreadyOnChainErrorMessage = "already on chain" return strings.Contains(err.Error(), alreadyOnChainErrorMessage) From cddc58ace282f74385d065119b779f9e9a5fdf10 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Oct 2023 16:04:21 +0300 Subject: [PATCH 0632/1943] [#752] innerring: Optimize keyPosition() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ old │ new │ │ sec/op │ sec/op vs base │ KeyPosition-8 2771.50n ± 10% 40.32n ± 4% -98.55% (p=0.000 n=10) │ old │ new │ │ B/op │ B/op vs base │ KeyPosition-8 1.531Ki ± 0% 0.000Ki ± 0% -100.00% (p=0.000 n=10) │ old │ new │ │ allocs/op │ allocs/op vs base │ KeyPosition-8 21.00 ± 0% 0.00 ± 0% -100.00% (p=0.000 n=10) ``` Signed-off-by: Evgenii Stratonikov --- pkg/innerring/indexer.go | 4 +--- pkg/innerring/indexer_test.go | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/pkg/innerring/indexer.go b/pkg/innerring/indexer.go index ac5fb93ef..5d0a87d64 100644 --- a/pkg/innerring/indexer.go +++ b/pkg/innerring/indexer.go @@ -1,7 +1,6 @@ package innerring import ( - "bytes" "fmt" "sync" "time" @@ -113,10 +112,9 @@ func (s *innerRingIndexer) AlphabetIndex() (int32, error) { // index of the key. func keyPosition(key *keys.PublicKey, list keys.PublicKeys) (result int32) { result = -1 - rawBytes := key.Bytes() for i := range list { - if bytes.Equal(list[i].Bytes(), rawBytes) { + if key.Equal(list[i]) { result = int32(i) break } diff --git a/pkg/innerring/indexer_test.go b/pkg/innerring/indexer_test.go index 1937f7a49..5bc2cc988 100644 --- a/pkg/innerring/indexer_test.go +++ b/pkg/innerring/indexer_test.go @@ -223,3 +223,23 @@ func (f *testIRFetcher) InnerRingKeys() (keys.PublicKeys, error) { f.calls.Add(1) return f.keys, f.err } + +func BenchmarkKeyPosition(b *testing.B) { + list := make(keys.PublicKeys, 7) + for i := range list { + p, err := keys.NewPrivateKey() + require.NoError(b, err) + list[i] = p.PublicKey() + } + + key := new(keys.PublicKey) + require.NoError(b, key.DecodeBytes(list[5].Bytes())) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if keyPosition(key, list) != 5 { + b.FailNow() + } + } +} From 9f7c2d88106151f15fd134c3288e557b93c91976 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Oct 2023 16:06:12 +0300 Subject: [PATCH 0633/1943] [#752] innerring: Simplify keyPosition() Signed-off-by: Evgenii Stratonikov --- pkg/innerring/indexer.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pkg/innerring/indexer.go b/pkg/innerring/indexer.go index 5d0a87d64..45135a57b 100644 --- a/pkg/innerring/indexer.go +++ b/pkg/innerring/indexer.go @@ -110,15 +110,11 @@ func (s *innerRingIndexer) AlphabetIndex() (int32, error) { // keyPosition returns "-1" if key is not found in the list, otherwise returns // index of the key. -func keyPosition(key *keys.PublicKey, list keys.PublicKeys) (result int32) { - result = -1 - +func keyPosition(key *keys.PublicKey, list keys.PublicKeys) int32 { for i := range list { if key.Equal(list[i]) { - result = int32(i) - break + return int32(i) } } - - return result + return -1 } From 121f5c4dd8ff5d6d0f6f1d91bcafce2d3712fd4d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 26 Oct 2023 10:36:22 +0300 Subject: [PATCH 0634/1943] [#757] ir: Do not exclude node in maintenance mode from netmap Signed-off-by: Anton Nikiforov --- pkg/innerring/processors/netmap/cleanup_table.go | 5 ++++- .../processors/netmap/cleanup_table_test.go | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/innerring/processors/netmap/cleanup_table.go b/pkg/innerring/processors/netmap/cleanup_table.go index 80117247d..705e21d99 100644 --- a/pkg/innerring/processors/netmap/cleanup_table.go +++ b/pkg/innerring/processors/netmap/cleanup_table.go @@ -24,6 +24,8 @@ type ( epochStamp binNodeInfo []byte + + maintenance bool } ) @@ -58,6 +60,7 @@ func (c *cleanupTable) update(snapshot netmap.NetMap, now uint64) { } access.binNodeInfo = binNodeInfo + access.maintenance = nmNodes[i].IsMaintenance() newMap[keyString] = access } @@ -105,7 +108,7 @@ func (c *cleanupTable) forEachRemoveCandidate(epoch uint64, f func(string) error defer c.Unlock() for keyString, access := range c.lastAccess { - if epoch-access.epoch > c.threshold { + if !access.maintenance && epoch-access.epoch > c.threshold { access.removeFlag = true // set remove flag c.lastAccess[keyString] = access diff --git a/pkg/innerring/processors/netmap/cleanup_table_test.go b/pkg/innerring/processors/netmap/cleanup_table_test.go index 959710393..ae5620733 100644 --- a/pkg/innerring/processors/netmap/cleanup_table_test.go +++ b/pkg/innerring/processors/netmap/cleanup_table_test.go @@ -124,6 +124,21 @@ func TestCleanupTable(t *testing.T) { })) require.EqualValues(t, len(infos)-1, cnt) }) + + t.Run("skip maintenance nodes", func(t *testing.T) { + cnt := 0 + infos[1].SetMaintenance() + key := netmap.StringifyPublicKey(infos[1]) + c.update(networkMap, 5) + + require.NoError(t, + c.forEachRemoveCandidate(5, func(s string) error { + cnt++ + require.NotEqual(t, s, key) + return nil + })) + require.EqualValues(t, len(infos)-1, cnt) + }) }) } From d4b6ebe7e797d3f573c6d2b858588be121dabb05 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 27 Oct 2023 09:53:12 +0300 Subject: [PATCH 0635/1943] [#725] writecache: Fix metric values Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/writecache.go | 7 ------- .../writecache/writecachebadger/delete.go | 2 +- .../writecache/writecachebadger/put.go | 2 +- .../writecache/writecachebadger/state.go | 10 ++++++++++ .../writecache/writecachebadger/storage.go | 2 +- .../writecache/writecachebbolt/delete.go | 1 + .../writecache/writecachebbolt/flush.go | 1 + .../writecache/writecachebbolt/put.go | 1 + .../writecache/writecachebbolt/state.go | 1 + .../writecache/writecachebbolt/storage.go | 3 ++- pkg/metrics/writecache.go | 20 ------------------- 11 files changed, 19 insertions(+), 31 deletions(-) diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 2e518c6ff..00a40105e 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -68,16 +68,10 @@ func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.Sto func (m *writeCacheMetrics) Delete(d time.Duration, success bool, st writecache.StorageType) { m.metrics.AddMethodDuration(m.shardID, "Delete", success, d, st.String()) - if success { - m.metrics.DecActualCount(m.shardID, st.String()) - } } func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.StorageType) { m.metrics.AddMethodDuration(m.shardID, "Put", success, d, st.String()) - if success { - m.metrics.IncActualCount(m.shardID, st.String()) - } } func (m *writeCacheMetrics) SetEstimateSize(db, fstree uint64) { @@ -99,7 +93,6 @@ func (m *writeCacheMetrics) Flush(success bool, st writecache.StorageType) { } func (m *writeCacheMetrics) Evict(st writecache.StorageType) { - m.metrics.DecActualCount(m.shardID, st.String()) m.metrics.IncOperationCounter(m.shardID, "Evict", metrics.NullBool{}, st.String()) } diff --git a/pkg/local_object_storage/writecache/writecachebadger/delete.go b/pkg/local_object_storage/writecache/writecachebadger/delete.go index afab78547..f96bf2707 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/delete.go +++ b/pkg/local_object_storage/writecache/writecachebadger/delete.go @@ -59,7 +59,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.OpField("db DELETE"), ) deleted = true - c.objCounters.DecDB() + c.decDB() } return err } diff --git a/pkg/local_object_storage/writecache/writecachebadger/put.go b/pkg/local_object_storage/writecache/writecachebadger/put.go index c03a0d336..2071ba1d0 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/put.go +++ b/pkg/local_object_storage/writecache/writecachebadger/put.go @@ -76,7 +76,7 @@ func (c *cache) put(obj objectInfo) error { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("db PUT"), ) - c.objCounters.IncDB() + c.incDB() } return err } diff --git a/pkg/local_object_storage/writecache/writecachebadger/state.go b/pkg/local_object_storage/writecache/writecachebadger/state.go index 994dfa3d5..e098eb060 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/state.go +++ b/pkg/local_object_storage/writecache/writecachebadger/state.go @@ -55,3 +55,13 @@ func (c *cache) initCounters() error { return nil } + +func (c *cache) incDB() { + c.objCounters.IncDB() + c.metrics.SetActualCounters(c.objCounters.DB(), 0) +} + +func (c *cache) decDB() { + c.objCounters.DecDB() + c.metrics.SetActualCounters(c.objCounters.DB(), 0) +} diff --git a/pkg/local_object_storage/writecache/writecachebadger/storage.go b/pkg/local_object_storage/writecache/writecachebadger/storage.go index 9ff54bee0..04337b7a6 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/storage.go +++ b/pkg/local_object_storage/writecache/writecachebadger/storage.go @@ -73,7 +73,7 @@ func (c *cache) deleteFromDB(keys []internalKey) []internalKey { } for i := 0; i < errorIndex; i++ { - c.objCounters.DecDB() + c.decDB() c.metrics.Evict(writecache.StorageTypeDB) storagelog.Write(c.log, storagelog.AddressField(keys[i]), diff --git a/pkg/local_object_storage/writecache/writecachebbolt/delete.go b/pkg/local_object_storage/writecache/writecachebbolt/delete.go index e4d9c219f..15c83eed1 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/delete.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/delete.go @@ -83,6 +83,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storagelog.OpField("fstree DELETE"), ) deleted = true + // counter changed by fstree c.estimateCacheSize() } return metaerr.Wrap(err) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush.go b/pkg/local_object_storage/writecache/writecachebbolt/flush.go index 393776dd1..9e46226d1 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush.go @@ -60,6 +60,7 @@ func (c *cache) runFlushLoop(ctx context.Context) { case <-tt.C: c.flushSmallObjects(ctx) tt.Reset(defaultFlushInterval) + c.estimateCacheSize() case <-ctx.Done(): return } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/put.go b/pkg/local_object_storage/writecache/writecachebbolt/put.go index 1d9957219..63fa544ed 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/put.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/put.go @@ -131,6 +131,7 @@ func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) erro storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree PUT"), ) + // counter changed by fstree c.estimateCacheSize() return nil diff --git a/pkg/local_object_storage/writecache/writecachebbolt/state.go b/pkg/local_object_storage/writecache/writecachebbolt/state.go index 91d54b0ea..8749ec601 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/state.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/state.go @@ -72,5 +72,6 @@ func (c *cache) initCounters() error { return fmt.Errorf("could not read write-cache DB counter: %w", err) } c.objCounters.cDB.Store(inDB) + c.estimateCacheSize() return nil } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/storage.go b/pkg/local_object_storage/writecache/writecachebbolt/storage.go index bbd66d57c..bff9385dc 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/storage.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/storage.go @@ -73,7 +73,7 @@ func (c *cache) deleteFromDB(key string) { err := c.db.Batch(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) key := []byte(key) - recordDeleted = !recordDeleted && b.Get(key) != nil + recordDeleted = b.Get(key) != nil return b.Delete(key) }) @@ -122,6 +122,7 @@ func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { storagelog.OpField("fstree DELETE"), ) c.metrics.Evict(writecache.StorageTypeFSTree) + // counter changed by fstree c.estimateCacheSize() } } diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 7e6083a49..c8f815ca5 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -10,16 +10,10 @@ import ( type WriteCacheMetrics interface { AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) - - IncActualCount(shardID string, storageType string) - DecActualCount(shardID string, storageType string) SetActualCount(shardID string, count uint64, storageType string) - SetEstimateSize(shardID string, size uint64, storageType string) SetMode(shardID string, mode string) - IncOperationCounter(shardID string, operation string, success NullBool, storageType string) - Close(shardID string) } @@ -65,20 +59,6 @@ func (m *writeCacheMetrics) AddMethodDuration(shardID string, method string, suc ).Observe(d.Seconds()) } -func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) { - m.actualCount.With(prometheus.Labels{ - shardIDLabel: shardID, - storageLabel: storageType, - }).Inc() -} - -func (m *writeCacheMetrics) DecActualCount(shardID string, storageType string) { - m.actualCount.With(prometheus.Labels{ - shardIDLabel: shardID, - storageLabel: storageType, - }).Dec() -} - func (m *writeCacheMetrics) SetActualCount(shardID string, count uint64, storageType string) { m.actualCount.With(prometheus.Labels{ shardIDLabel: shardID, From 869518be0afaf8c717513a634dc8cd2228c466e6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 30 Oct 2023 17:11:04 +0300 Subject: [PATCH 0636/1943] [#728] writecache: Fix Badger writecache race. Signed-off-by: Dmitrii Stepanov --- .../writecachebadger/cachebadger.go | 6 ++++ .../writecache/writecachebadger/gc.go | 31 +++++++++++-------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go index be7046d05..484f01815 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go +++ b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go @@ -85,6 +85,9 @@ func (c *cache) DumpInfo() writecache.Info { // Open opens and initializes database. Reads object counters from the ObjectCounters instance. func (c *cache) Open(_ context.Context, readOnly bool) error { + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + err := c.openStore(readOnly) if err != nil { return metaerr.Wrap(err) @@ -94,6 +97,9 @@ func (c *cache) Open(_ context.Context, readOnly bool) error { // Init runs necessary services. func (c *cache) Init() error { + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + c.log.Info(logs.WritecacheBadgerInitExperimental) c.metrics.SetMode(c.mode) ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/local_object_storage/writecache/writecachebadger/gc.go b/pkg/local_object_storage/writecache/writecachebadger/gc.go index b856efc3d..8937ff295 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/gc.go +++ b/pkg/local_object_storage/writecache/writecachebadger/gc.go @@ -21,20 +21,25 @@ func (c *cache) runGCLoop(ctx context.Context) { case <-ctx.Done(): return case <-t.C: - // This serves to synchronize the c.db field when changing mode as well. - c.modeMtx.RLock() - ro := c.readOnly() - c.modeMtx.RUnlock() - if ro { - continue - } - - // 0.5 is the recommended value so that write amplification of the value log is 2. - // See https://pkg.go.dev/github.com/dgraph-io/badger/v4#DB.RunValueLogGC for more info. - for c.db.RunValueLogGC(0.5) == nil { - c.log.Debug(logs.WritecacheDBValueLogGCRunCompleted) - } + c.runGC() } } }() } + +func (c *cache) runGC() { + // This serves to synchronize the c.db field when changing mode as well. + c.modeMtx.RLock() + defer c.modeMtx.RUnlock() + + ro := c.readOnly() + if ro { + return + } + + // 0.5 is the recommended value so that write amplification of the value log is 2. + // See https://pkg.go.dev/github.com/dgraph-io/badger/v4#DB.RunValueLogGC for more info. + for c.db.RunValueLogGC(0.5) == nil { + c.log.Debug(logs.WritecacheDBValueLogGCRunCompleted) + } +} From 261d281154e443816baed8ff16b287b4c20bb706 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 30 Oct 2023 10:28:41 +0300 Subject: [PATCH 0637/1943] [#762] go.mod: Update sdk-go Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/client/sdk.go | 1 - cmd/frostfs-cli/modules/accounting/balance.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- pkg/network/cache/multi.go | 1 + 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/sdk.go b/cmd/frostfs-cli/internal/client/sdk.go index 79d3dcb0d..6923e4f17 100644 --- a/cmd/frostfs-cli/internal/client/sdk.go +++ b/cmd/frostfs-cli/internal/client/sdk.go @@ -50,7 +50,6 @@ func GetSDKClient(ctx context.Context, cmd *cobra.Command, key *ecdsa.PrivateKey ) prmInit.SetDefaultPrivateKey(*key) - prmInit.ResolveFrostFSFailures() prmDial.SetServerURI(addr.URIAddr()) if timeout := viper.GetDuration(commonflags.Timeout); timeout > 0 { // In CLI we can only set a timeout for the whole operation. diff --git a/cmd/frostfs-cli/modules/accounting/balance.go b/cmd/frostfs-cli/modules/accounting/balance.go index 5ed8f9403..8e975fb03 100644 --- a/cmd/frostfs-cli/modules/accounting/balance.go +++ b/cmd/frostfs-cli/modules/accounting/balance.go @@ -39,7 +39,7 @@ var accountingBalanceCmd = &cobra.Command{ var prm internalclient.BalanceOfPrm prm.SetClient(cli) - prm.SetAccount(idUser) + prm.Account = &idUser res, err := internalclient.BalanceOf(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) diff --git a/go.mod b/go.mod index bbba2e3c3..0c6731363 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0 git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231019083813-5804128ff3d5 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231027075445-a02c0bfac885 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index bb5757b45..6fb57f4f4 100644 --- a/go.sum +++ b/go.sum @@ -393,8 +393,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231019083813-5804128ff3d5 h1:P2HCnBTx+LwhMYSrs4sF8+jHvD1+HdS0fMtUPe2kTns= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231019083813-5804128ff3d5/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231027075445-a02c0bfac885 h1:tbOUgoaN3usFTUA9k62z96uQDRNJUiz9TxzJr1AtSFo= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231027075445-a02c0bfac885/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index d03edd962..c9b13826a 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -57,6 +57,7 @@ func (x *multiClient) createForAddress(ctx context.Context, addr network.Address prmDial client.PrmDial ) + prmInit.DisableFrostFSFailuresResolution() prmDial.SetServerURI(addr.URIAddr()) if x.opts.Key != nil { From b8f79f422771397491cfd9c9f87341bde8ddc12d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 31 Oct 2023 14:21:42 +0300 Subject: [PATCH 0638/1943] [#633] shard/test: Fix race conditions in TestCounters() Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/metrics_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 23721af6e..107938139 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -268,6 +268,7 @@ func shardWithMetrics(t *testing.T, path string) (*Shard, *metricsStore) { meta.WithPath(filepath.Join(path, "meta")), meta.WithEpochState(epochState{})), WithMetricsWriter(mm), + WithGCRemoverSleepInterval(time.Hour), ) require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) From 00aa6d974990b5a84a2fcdff1f9e91d2f6b30011 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 31 Oct 2023 14:16:34 +0300 Subject: [PATCH 0639/1943] [#633] shard/test: Fix TestCounters() Introduced in 362f24953a7, forgotten to be changed because test generator didn't provide payload size. Signed-off-by: Evgenii Stratonikov --- .../internal/testutil/object.go | 1 + .../shard/metrics_test.go | 25 +++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/internal/testutil/object.go b/pkg/local_object_storage/internal/testutil/object.go index 573a099ff..60e9211d5 100644 --- a/pkg/local_object_storage/internal/testutil/object.go +++ b/pkg/local_object_storage/internal/testutil/object.go @@ -49,6 +49,7 @@ func GenerateObjectWithCIDWithPayload(cnr cid.ID, data []byte) *objectSDK.Object obj.SetContainerID(cnr) obj.SetVersion(&ver) obj.SetPayload(data) + obj.SetPayloadSize(uint64(len(data))) obj.SetPayloadChecksum(csum) obj.SetPayloadHomomorphicHash(csumTZ) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 107938139..7724aa222 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "sync" "testing" + "time" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -151,11 +152,11 @@ func TestCounters(t *testing.T) { var totalPayload int64 - expectedSizes := make(map[string]int64) + expectedLogicalSizes := make(map[string]int64) for i := range oo { cnr, _ := oo[i].ContainerID() oSize := int64(oo[i].PayloadSize()) - expectedSizes[cnr.EncodeToString()] += oSize + expectedLogicalSizes[cnr.EncodeToString()] += oSize totalPayload += oSize } @@ -170,7 +171,7 @@ func TestCounters(t *testing.T) { require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical)) require.Equal(t, uint64(objNumber), mm.getObjectCounter(logical)) - require.Equal(t, expectedSizes, mm.containerSizes()) + require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) t.Run("inhume_GC", func(t *testing.T) { @@ -182,11 +183,15 @@ func TestCounters(t *testing.T) { _, err := sh.Inhume(context.Background(), prm) require.NoError(t, err) + + cid, ok := oo[i].ContainerID() + require.True(t, ok) + expectedLogicalSizes[cid.EncodeToString()] -= int64(oo[i].PayloadSize()) } require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical)) require.Equal(t, uint64(objNumber-inhumedNumber), mm.getObjectCounter(logical)) - require.Equal(t, expectedSizes, mm.containerSizes()) + require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) oo = oo[inhumedNumber:] @@ -205,9 +210,15 @@ func TestCounters(t *testing.T) { _, err := sh.Inhume(context.Background(), prm) require.NoError(t, err) + for i := 0; i < inhumedNumber; i++ { + cid, ok := oo[i].ContainerID() + require.True(t, ok) + expectedLogicalSizes[cid.EncodeToString()] -= int64(oo[i].PayloadSize()) + } + require.Equal(t, phy, mm.getObjectCounter(physical)) require.Equal(t, logic-uint64(inhumedNumber), mm.getObjectCounter(logical)) - require.Equal(t, expectedSizes, mm.containerSizes()) + require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) oo = oo[inhumedNumber:] @@ -233,9 +244,9 @@ func TestCounters(t *testing.T) { totalRemovedpayload += removedPayload cnr, _ := oo[i].ContainerID() - expectedSizes[cnr.EncodeToString()] -= int64(removedPayload) + expectedLogicalSizes[cnr.EncodeToString()] -= int64(removedPayload) } - require.Equal(t, expectedSizes, mm.containerSizes()) + require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload-int64(totalRemovedpayload), mm.payloadSize()) }) } From 79088baa06bb0ee0da9489f72a8cdad5ea77e477 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 31 Oct 2023 14:56:55 +0300 Subject: [PATCH 0640/1943] [#772] node: Apply gofumpt Signed-off-by: Dmitrii Stepanov --- .../internal/modules/config/config.go | 4 ++-- .../internal/modules/morph/generate.go | 2 +- .../internal/modules/morph/local_client.go | 2 +- cmd/frostfs-adm/internal/modules/root.go | 16 +++++++--------- .../internal/modules/storagecfg/root.go | 2 +- cmd/frostfs-cli/modules/acl/extended/create.go | 2 +- cmd/frostfs-cli/modules/bearer/create.go | 2 +- cmd/frostfs-cli/modules/container/get.go | 2 +- cmd/frostfs-cli/modules/container/get_eacl.go | 2 +- cmd/frostfs-cli/modules/container/nodes.go | 2 +- cmd/frostfs-cli/modules/object/get.go | 2 +- cmd/frostfs-cli/modules/object/nodes.go | 5 +++-- cmd/frostfs-cli/modules/object/root.go | 3 ++- cmd/frostfs-cli/modules/session/create.go | 2 +- cmd/frostfs-cli/modules/util/convert_eacl.go | 2 +- cmd/frostfs-cli/modules/util/keyer.go | 2 +- cmd/frostfs-cli/modules/util/sign_bearer.go | 2 +- cmd/frostfs-cli/modules/util/sign_session.go | 2 +- cmd/frostfs-ir/config.go | 2 +- cmd/frostfs-lens/internal/printers.go | 2 +- .../config/apiclient/config_test.go | 2 +- cmd/frostfs-node/config/config.go | 1 - cmd/frostfs-node/config/configdir_test.go | 4 ++-- .../config/contracts/config_test.go | 2 +- cmd/frostfs-node/config/control/config_test.go | 2 +- cmd/frostfs-node/config/engine/config_test.go | 14 +++++++------- .../engine/shard/blobstor/storage/config.go | 2 +- .../config/engine/shard/boltdb/boltdb.go | 2 +- .../config/engine/shard/pilorama/config.go | 2 +- cmd/frostfs-node/config/grpc/config_test.go | 2 +- cmd/frostfs-node/config/logger/config_test.go | 2 +- cmd/frostfs-node/config/metrics/config_test.go | 2 +- cmd/frostfs-node/config/morph/config_test.go | 12 +++++------- cmd/frostfs-node/config/node/config_test.go | 2 +- cmd/frostfs-node/config/object/config_test.go | 2 +- cmd/frostfs-node/config/policer/config_test.go | 2 +- .../config/profiler/config_test.go | 2 +- .../config/replicator/config_test.go | 2 +- cmd/frostfs-node/config/tree/config_test.go | 2 +- cmd/frostfs-node/morph.go | 3 ++- cmd/frostfs-node/object.go | 9 ++++++--- cmd/frostfs-node/tracing.go | 2 +- cmd/frostfs-node/validate.go | 2 +- cmd/internal/common/netmap.go | 3 ++- pkg/core/client/util.go | 6 ++++-- pkg/core/object/sender_classifier.go | 9 ++++++--- pkg/innerring/blocktimer_test.go | 1 + pkg/innerring/config/fee_test.go | 1 - pkg/innerring/initialization.go | 6 ++++-- pkg/innerring/innerring.go | 3 ++- .../processors/alphabet/handlers_test.go | 1 + .../processors/balance/handlers_test.go | 3 +-- .../processors/container/handlers_test.go | 1 + .../processors/frostfs/handlers_test.go | 6 ++++-- .../processors/governance/handlers_test.go | 2 +- .../processors/netmap/handlers_test.go | 3 +++ pkg/local_object_storage/blobovnicza/delete.go | 3 +-- pkg/local_object_storage/blobovnicza/exists.go | 4 +--- .../blobovnicza/iterate.go | 3 +-- pkg/local_object_storage/blobovnicza/put.go | 3 +-- .../blobstor/blobovniczatree/active.go | 4 ++-- .../blobstor/blobovniczatree/cache.go | 2 +- .../blobstor/blobovniczatree/control.go | 4 ++-- .../blobstor/blobovniczatree/exists_test.go | 2 +- .../blobstor/blobovniczatree/manager.go | 6 ++++-- .../blobstor/blobovniczatree/option.go | 2 +- .../blobstor/fstree/fstree.go | 2 +- .../blobstor/generic_test.go | 1 - pkg/local_object_storage/blobstor/get.go | 4 +--- pkg/local_object_storage/blobstor/get_range.go | 4 +--- .../blobstor/internal/blobstortest/iterate.go | 2 +- pkg/local_object_storage/blobstor/logger.go | 6 ++++-- pkg/local_object_storage/engine/control.go | 4 ++-- .../engine/control_test.go | 5 ++--- pkg/local_object_storage/engine/engine.go | 6 ++++-- pkg/local_object_storage/engine/engine_test.go | 9 +++++---- pkg/local_object_storage/engine/error_test.go | 4 ++-- pkg/local_object_storage/engine/evacuate.go | 13 ++++++++----- .../engine/evacuate_test.go | 4 ++-- pkg/local_object_storage/engine/list_test.go | 5 +++-- .../internal/metaerr/error_test.go | 1 + pkg/local_object_storage/metabase/children.go | 1 - pkg/local_object_storage/metabase/counter.go | 6 ++++-- pkg/local_object_storage/metabase/db_test.go | 2 +- pkg/local_object_storage/metabase/delete.go | 4 ++-- pkg/local_object_storage/metabase/expired.go | 1 - pkg/local_object_storage/metabase/list.go | 2 +- pkg/local_object_storage/metabase/list_test.go | 1 - pkg/local_object_storage/metabase/lock.go | 1 - pkg/local_object_storage/metabase/put.go | 3 ++- .../metabase/version_test.go | 2 +- pkg/local_object_storage/metrics/fstree.go | 6 ++++++ pkg/local_object_storage/pilorama/boltdb.go | 3 +++ pkg/local_object_storage/pilorama/forest.go | 5 ++++- .../pilorama/forest_test.go | 18 ++++++++++++------ pkg/local_object_storage/pilorama/meta_test.go | 6 ++++-- .../shard/gc_internal_test.go | 12 ++++++------ pkg/local_object_storage/shard/reload_test.go | 6 ++++-- pkg/local_object_storage/shard/shard_test.go | 3 ++- .../writecache/writecachebbolt/cachebbolt.go | 4 +--- .../writecache/writecachebbolt/state.go | 2 +- pkg/metrics/replicator.go | 2 -- pkg/morph/client/constructor.go | 4 +--- pkg/morph/client/container/client.go | 4 +--- pkg/morph/client/netmap/peer.go | 2 +- pkg/morph/event/frostfs/ir_update_test.go | 12 +++++------- pkg/morph/event/listener_test.go | 2 +- pkg/morph/event/utils.go | 1 - pkg/morph/subscriber/subscriber.go | 3 ++- pkg/network/transport/netmap/grpc/service.go | 3 ++- pkg/services/container/morph/executor_test.go | 3 ++- pkg/services/control/service_test.go | 1 + pkg/services/control/types_test.go | 3 ++- pkg/services/netmap/executor.go | 6 ++++-- pkg/services/netmap/sign.go | 3 ++- pkg/services/object/acl/v2/errors.go | 6 ++++-- pkg/services/object/acl/v2/service.go | 12 ++++++++---- pkg/services/object/delete/service.go | 3 ++- pkg/services/object/get/assembler.go | 3 ++- pkg/services/object/get/get.go | 1 + pkg/services/object/get/get_test.go | 10 +++------- pkg/services/object/internal/key.go | 3 ++- pkg/services/object/put/service.go | 3 ++- pkg/services/object/put/single.go | 6 ++++-- pkg/services/object/search/service.go | 3 ++- pkg/services/object/util/prm.go | 3 ++- pkg/services/policer/check.go | 9 +++++---- .../storage/persistent/executor_test.go | 4 ++-- .../session/storage/persistent/storage.go | 2 +- pkg/services/tree/getsubtree_test.go | 6 ++++-- pkg/services/tree/signature.go | 14 +++++++++----- pkg/services/tree/sync.go | 14 +++++++------- pkg/util/locode/table/csv/opts.go | 2 +- pkg/util/os.go | 2 +- pkg/util/state/storage.go | 2 +- scripts/export-metrics/main.go | 2 +- 136 files changed, 293 insertions(+), 239 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/config/config.go b/cmd/frostfs-adm/internal/modules/config/config.go index 9dfab9f85..dd98a98b2 100644 --- a/cmd/frostfs-adm/internal/modules/config/config.go +++ b/cmd/frostfs-adm/internal/modules/config/config.go @@ -50,12 +50,12 @@ func initConfig(cmd *cobra.Command, _ []string) error { } pathDir := filepath.Dir(configPath) - err = os.MkdirAll(pathDir, 0700) + err = os.MkdirAll(pathDir, 0o700) if err != nil { return fmt.Errorf("create dir %s: %w", pathDir, err) } - f, err := os.OpenFile(configPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_SYNC, 0600) + f, err := os.OpenFile(configPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_SYNC, 0o600) if err != nil { return fmt.Errorf("open %s: %w", configPath, err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index 55321582b..b8e006962 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -76,7 +76,7 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er } p := filepath.Join(walletDir, innerring.GlagoliticLetter(i).String()+".json") - f, err := os.OpenFile(p, os.O_CREATE, 0644) + f, err := os.OpenFile(p, os.O_CREATE, 0o644) if err != nil { return nil, fmt.Errorf("can't create wallet file: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/local_client.go index 24dcb7dec..1a723042c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/local_client.go @@ -83,7 +83,7 @@ func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet go bc.Run() if cmd.Name() != "init" { - f, err := os.OpenFile(dumpPath, os.O_RDONLY, 0600) + f, err := os.OpenFile(dumpPath, os.O_RDONLY, 0o600) if err != nil { return nil, fmt.Errorf("can't open local dump: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/root.go b/cmd/frostfs-adm/internal/modules/root.go index 0fa0f7f69..8595483ab 100644 --- a/cmd/frostfs-adm/internal/modules/root.go +++ b/cmd/frostfs-adm/internal/modules/root.go @@ -15,16 +15,14 @@ import ( "github.com/spf13/viper" ) -var ( - rootCmd = &cobra.Command{ - Use: "frostfs-adm", - Short: "FrostFS Administrative Tool", - Long: `FrostFS Administrative Tool provides functions to setup and +var rootCmd = &cobra.Command{ + Use: "frostfs-adm", + Short: "FrostFS Administrative Tool", + Long: `FrostFS Administrative Tool provides functions to setup and manage FrostFS network deployment.`, - RunE: entryPoint, - SilenceUsage: true, - } -) + RunE: entryPoint, + SilenceUsage: true, +} func init() { cobra.OnInitialize(func() { initConfig(rootCmd) }) diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/root.go b/cmd/frostfs-adm/internal/modules/storagecfg/root.go index 90edf7d75..127272da5 100644 --- a/cmd/frostfs-adm/internal/modules/storagecfg/root.go +++ b/cmd/frostfs-adm/internal/modules/storagecfg/root.go @@ -145,7 +145,7 @@ func storageConfig(cmd *cobra.Command, args []string) { } out := applyTemplate(c) - fatalOnErr(os.WriteFile(outPath, out, 0644)) + fatalOnErr(os.WriteFile(outPath, out, 0o644)) cmd.Println("Node is ready for work! Run `frostfs-node -config " + outPath + "`") } diff --git a/cmd/frostfs-cli/modules/acl/extended/create.go b/cmd/frostfs-cli/modules/acl/extended/create.go index 7da26f8eb..59dfabba2 100644 --- a/cmd/frostfs-cli/modules/acl/extended/create.go +++ b/cmd/frostfs-cli/modules/acl/extended/create.go @@ -106,7 +106,7 @@ func createEACL(cmd *cobra.Command, _ []string) { return } - err = os.WriteFile(outArg, buf.Bytes(), 0644) + err = os.WriteFile(outArg, buf.Bytes(), 0o644) if err != nil { cmd.PrintErrln(err) os.Exit(1) diff --git a/cmd/frostfs-cli/modules/bearer/create.go b/cmd/frostfs-cli/modules/bearer/create.go index b85115047..8a7500db3 100644 --- a/cmd/frostfs-cli/modules/bearer/create.go +++ b/cmd/frostfs-cli/modules/bearer/create.go @@ -130,6 +130,6 @@ func createToken(cmd *cobra.Command, _ []string) { } out, _ := cmd.Flags().GetString(outFlag) - err = os.WriteFile(out, data, 0644) + err = os.WriteFile(out, data, 0o644) commonCmd.ExitOnErr(cmd, "can't write token to file: %w", err) } diff --git a/cmd/frostfs-cli/modules/container/get.go b/cmd/frostfs-cli/modules/container/get.go index f0590c7a1..cd920f0c4 100644 --- a/cmd/frostfs-cli/modules/container/get.go +++ b/cmd/frostfs-cli/modules/container/get.go @@ -51,7 +51,7 @@ var getContainerInfoCmd = &cobra.Command{ data = cnr.Marshal() } - err = os.WriteFile(containerPathTo, data, 0644) + err = os.WriteFile(containerPathTo, data, 0o644) commonCmd.ExitOnErr(cmd, "can't write container to file: %w", err) } }, diff --git a/cmd/frostfs-cli/modules/container/get_eacl.go b/cmd/frostfs-cli/modules/container/get_eacl.go index 6746132eb..8896db769 100644 --- a/cmd/frostfs-cli/modules/container/get_eacl.go +++ b/cmd/frostfs-cli/modules/container/get_eacl.go @@ -52,7 +52,7 @@ var getExtendedACLCmd = &cobra.Command{ cmd.Println("dumping data to file:", containerPathTo) - err = os.WriteFile(containerPathTo, data, 0644) + err = os.WriteFile(containerPathTo, data, 0o644) commonCmd.ExitOnErr(cmd, "could not write eACL to file: %w", err) }, } diff --git a/cmd/frostfs-cli/modules/container/nodes.go b/cmd/frostfs-cli/modules/container/nodes.go index 8b0f266a7..9273dd651 100644 --- a/cmd/frostfs-cli/modules/container/nodes.go +++ b/cmd/frostfs-cli/modules/container/nodes.go @@ -20,7 +20,7 @@ var containerNodesCmd = &cobra.Command{ Short: "Show nodes for container", Long: "Show nodes taking part in a container at the current epoch.", Run: func(cmd *cobra.Command, args []string) { - var cnr, pkey = getContainer(cmd) + cnr, pkey := getContainer(cmd) if pkey == nil { pkey = key.GetOrGenerate(cmd) diff --git a/cmd/frostfs-cli/modules/object/get.go b/cmd/frostfs-cli/modules/object/get.go index 3136f086f..9a888ccd3 100644 --- a/cmd/frostfs-cli/modules/object/get.go +++ b/cmd/frostfs-cli/modules/object/get.go @@ -132,7 +132,7 @@ func createOutWriter(cmd *cobra.Command, filename string) (out io.Writer, closer out = os.Stdout closer = func() {} } else { - f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) if err != nil { commonCmd.ExitOnErr(cmd, "", fmt.Errorf("can't open file '%s': %w", filename, err)) } diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 5499e5eec..ef6c29179 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -191,7 +191,7 @@ func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placemen numOfReplicas := placementPolicy.ReplicaNumberByIndex(repIdx) var nodeIdx uint32 for _, n := range rep { - if !objInfo.isLock && nodeIdx == numOfReplicas { //lock object should be on all container nodes + if !objInfo.isLock && nodeIdx == numOfReplicas { // lock object should be on all container nodes break } nodes[n.Hash()] = n @@ -213,7 +213,8 @@ func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placemen } func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, - pk *ecdsa.PrivateKey, objInfo *objectNodesInfo) map[uint64]boolError { + pk *ecdsa.PrivateKey, objInfo *objectNodesInfo, +) map[uint64]boolError { result := make(map[uint64]boolError) resultMtx := &sync.Mutex{} diff --git a/cmd/frostfs-cli/modules/object/root.go b/cmd/frostfs-cli/modules/object/root.go index c46fc058a..d9d4bb84d 100644 --- a/cmd/frostfs-cli/modules/object/root.go +++ b/cmd/frostfs-cli/modules/object/root.go @@ -31,7 +31,8 @@ func init() { objectHashCmd, objectRangeCmd, objectLockCmd, - objectNodesCmd} + objectNodesCmd, + } Cmd.AddCommand(objectChildCommands...) diff --git a/cmd/frostfs-cli/modules/session/create.go b/cmd/frostfs-cli/modules/session/create.go index f9705c023..94ae595a8 100644 --- a/cmd/frostfs-cli/modules/session/create.go +++ b/cmd/frostfs-cli/modules/session/create.go @@ -81,7 +81,7 @@ func createSession(cmd *cobra.Command, _ []string) { } filename, _ := cmd.Flags().GetString(outFlag) - err = os.WriteFile(filename, data, 0644) + err = os.WriteFile(filename, data, 0o644) commonCmd.ExitOnErr(cmd, "can't write token to file: %w", err) } diff --git a/cmd/frostfs-cli/modules/util/convert_eacl.go b/cmd/frostfs-cli/modules/util/convert_eacl.go index 1fb76c50e..774d500bc 100644 --- a/cmd/frostfs-cli/modules/util/convert_eacl.go +++ b/cmd/frostfs-cli/modules/util/convert_eacl.go @@ -48,7 +48,7 @@ func convertEACLTable(cmd *cobra.Command, _ []string) { return } - err = os.WriteFile(to, data, 0644) + err = os.WriteFile(to, data, 0o644) commonCmd.ExitOnErr(cmd, "can't write exteded ACL table to file: %w", err) cmd.Printf("extended ACL table was successfully dumped to %s\n", to) diff --git a/cmd/frostfs-cli/modules/util/keyer.go b/cmd/frostfs-cli/modules/util/keyer.go index 53082a15e..ee2497348 100644 --- a/cmd/frostfs-cli/modules/util/keyer.go +++ b/cmd/frostfs-cli/modules/util/keyer.go @@ -78,7 +78,7 @@ func keyerGenerate(filename string, d *keyer.Dashboard) error { } if filename != "" { - return os.WriteFile(filename, key, 0600) + return os.WriteFile(filename, key, 0o600) } return nil diff --git a/cmd/frostfs-cli/modules/util/sign_bearer.go b/cmd/frostfs-cli/modules/util/sign_bearer.go index c641cf0ac..991216958 100644 --- a/cmd/frostfs-cli/modules/util/sign_bearer.go +++ b/cmd/frostfs-cli/modules/util/sign_bearer.go @@ -56,7 +56,7 @@ func signBearerToken(cmd *cobra.Command, _ []string) { return } - err = os.WriteFile(to, data, 0644) + err = os.WriteFile(to, data, 0o644) commonCmd.ExitOnErr(cmd, "can't write signed bearer token to file: %w", err) cmd.Printf("signed bearer token was successfully dumped to %s\n", to) diff --git a/cmd/frostfs-cli/modules/util/sign_session.go b/cmd/frostfs-cli/modules/util/sign_session.go index 2c77ee4df..ba76678dc 100644 --- a/cmd/frostfs-cli/modules/util/sign_session.go +++ b/cmd/frostfs-cli/modules/util/sign_session.go @@ -76,7 +76,7 @@ func signSessionToken(cmd *cobra.Command, _ []string) { return } - err = os.WriteFile(to, data, 0644) + err = os.WriteFile(to, data, 0o644) if err != nil { commonCmd.ExitOnErr(cmd, "", fmt.Errorf("can't write signed session token to %s: %w", to, err)) } diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 54c7d18e3..be870052c 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -13,7 +13,7 @@ import ( func newConfig() (*viper.Viper, error) { var err error - var dv = viper.New() + dv := viper.New() defaultConfiguration(dv) diff --git a/cmd/frostfs-lens/internal/printers.go b/cmd/frostfs-lens/internal/printers.go index dd73a5552..ea0cbc8e0 100644 --- a/cmd/frostfs-lens/internal/printers.go +++ b/cmd/frostfs-lens/internal/printers.go @@ -59,7 +59,7 @@ func WriteObjectToFile(cmd *cobra.Command, path string, data []byte) { } ExitOnErr(cmd, Errf("could not write file: %w", - os.WriteFile(path, data, 0644))) + os.WriteFile(path, data, 0o644))) cmd.Printf("\nSaved payload to '%s' file\n", path) } diff --git a/cmd/frostfs-node/config/apiclient/config_test.go b/cmd/frostfs-node/config/apiclient/config_test.go index f4f09d4ff..cdfa5c401 100644 --- a/cmd/frostfs-node/config/apiclient/config_test.go +++ b/cmd/frostfs-node/config/apiclient/config_test.go @@ -22,7 +22,7 @@ func TestApiclientSection(t *testing.T) { const path = "../../../../config/example/node" - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { require.Equal(t, 15*time.Second, apiclientconfig.DialTimeout(c)) require.Equal(t, 20*time.Second, apiclientconfig.StreamTimeout(c)) require.Equal(t, 30*time.Second, apiclientconfig.ReconnectTimeout(c)) diff --git a/cmd/frostfs-node/config/config.go b/cmd/frostfs-node/config/config.go index 77e34d613..d74e820ac 100644 --- a/cmd/frostfs-node/config/config.go +++ b/cmd/frostfs-node/config/config.go @@ -38,7 +38,6 @@ func New(configFile, configDir, envPrefix string) *Config { configViper.WithConfigFile(configFile), configViper.WithConfigDir(configDir), configViper.WithEnvPrefix(envPrefix)) - if err != nil { panic(err) } diff --git a/cmd/frostfs-node/config/configdir_test.go b/cmd/frostfs-node/config/configdir_test.go index ede15a522..35dae97d9 100644 --- a/cmd/frostfs-node/config/configdir_test.go +++ b/cmd/frostfs-node/config/configdir_test.go @@ -15,8 +15,8 @@ func TestConfigDir(t *testing.T) { cfgFileName0 := path.Join(dir, "cfg_00.json") cfgFileName1 := path.Join(dir, "cfg_01.yml") - require.NoError(t, os.WriteFile(cfgFileName0, []byte(`{"storage":{"shard_pool_size":15}}`), 0777)) - require.NoError(t, os.WriteFile(cfgFileName1, []byte("logger:\n level: debug"), 0777)) + require.NoError(t, os.WriteFile(cfgFileName0, []byte(`{"storage":{"shard_pool_size":15}}`), 0o777)) + require.NoError(t, os.WriteFile(cfgFileName1, []byte("logger:\n level: debug"), 0o777)) c := New("", dir, "") require.Equal(t, "debug", cast.ToString(c.Sub("logger").Value("level"))) diff --git a/cmd/frostfs-node/config/contracts/config_test.go b/cmd/frostfs-node/config/contracts/config_test.go index d816ea1e4..c85a625c5 100644 --- a/cmd/frostfs-node/config/contracts/config_test.go +++ b/cmd/frostfs-node/config/contracts/config_test.go @@ -35,7 +35,7 @@ func TestContractsSection(t *testing.T) { expProxy, err := util.Uint160DecodeStringLE("ad7c6b55b737b696e5c82c85445040964a03e97f") require.NoError(t, err) - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { balance := contractsconfig.Balance(c) container := contractsconfig.Container(c) netmap := contractsconfig.Netmap(c) diff --git a/cmd/frostfs-node/config/control/config_test.go b/cmd/frostfs-node/config/control/config_test.go index 1aed229e2..f702d83ae 100644 --- a/cmd/frostfs-node/config/control/config_test.go +++ b/cmd/frostfs-node/config/control/config_test.go @@ -24,7 +24,7 @@ func TestControlSection(t *testing.T) { pubs[0], _ = keys.NewPublicKeyFromString("035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11") pubs[1], _ = keys.NewPublicKeyFromString("028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6") - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { require.Equal(t, pubs, controlconfig.AuthorizedKeys(c)) require.Equal(t, "localhost:8090", controlconfig.GRPC(c).Endpoint()) } diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index b8e95db6d..4077b1744 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -42,7 +42,7 @@ func TestEngineSection(t *testing.T) { const path = "../../../../config/example/node" - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { num := 0 require.EqualValues(t, 100, engineconfig.ShardErrorThreshold(c)) @@ -78,7 +78,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 3221225472, wc.SizeLimit()) require.Equal(t, "tmp/0/meta", meta.Path()) - require.Equal(t, fs.FileMode(0644), meta.BoltDB().Perm()) + require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) require.Equal(t, 100, meta.BoltDB().MaxBatchSize()) require.Equal(t, 10*time.Millisecond, meta.BoltDB().MaxBatchDelay()) @@ -89,7 +89,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, 2, len(ss)) blz := blobovniczaconfig.From((*config.Config)(ss[0])) require.Equal(t, "tmp/0/blob/blobovnicza", ss[0].Path()) - require.EqualValues(t, 0644, blz.BoltDB().Perm()) + require.EqualValues(t, 0o644, blz.BoltDB().Perm()) require.EqualValues(t, 4194304, blz.Size()) require.EqualValues(t, 1, blz.ShallowDepth()) require.EqualValues(t, 4, blz.ShallowWidth()) @@ -97,7 +97,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 10, blz.LeafWidth()) require.Equal(t, "tmp/0/blob", ss[1].Path()) - require.EqualValues(t, 0644, ss[1].Perm()) + require.EqualValues(t, 0o644, ss[1].Perm()) fst := fstreeconfig.From((*config.Config)(ss[1])) require.EqualValues(t, 5, fst.Depth()) @@ -112,7 +112,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, mode.ReadOnly, sc.Mode()) case 1: require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path()) - require.Equal(t, fs.FileMode(0644), pl.Perm()) + require.Equal(t, fs.FileMode(0o644), pl.Perm()) require.True(t, pl.NoSync()) require.Equal(t, 5*time.Millisecond, pl.MaxBatchDelay()) require.Equal(t, 100, pl.MaxBatchSize()) @@ -127,7 +127,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 4294967296, wc.SizeLimit()) require.Equal(t, "tmp/1/meta", meta.Path()) - require.Equal(t, fs.FileMode(0644), meta.BoltDB().Perm()) + require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) require.Equal(t, 200, meta.BoltDB().MaxBatchSize()) require.Equal(t, 20*time.Millisecond, meta.BoltDB().MaxBatchDelay()) @@ -146,7 +146,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 10, blz.LeafWidth()) require.Equal(t, "tmp/1/blob", ss[1].Path()) - require.EqualValues(t, 0644, ss[1].Perm()) + require.EqualValues(t, 0o644, ss[1].Perm()) fst := fstreeconfig.From((*config.Config)(ss[1])) require.EqualValues(t, 5, fst.Depth()) diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go index 649e4980d..e83c69de2 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go @@ -9,7 +9,7 @@ import ( type Config config.Config // PermDefault are default permission bits for BlobStor data. -const PermDefault = 0660 +const PermDefault = 0o660 func From(x *config.Config) *Config { return (*Config)(x) diff --git a/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go b/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go index 83d4e45c5..9e334cd8f 100644 --- a/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go +++ b/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go @@ -13,7 +13,7 @@ type Config config.Config const ( // PermDefault is a default permission bits for metabase file. - PermDefault = 0660 + PermDefault = 0o660 ) // Perm returns the value of "perm" config parameter as a fs.FileMode. diff --git a/cmd/frostfs-node/config/engine/shard/pilorama/config.go b/cmd/frostfs-node/config/engine/shard/pilorama/config.go index 7ac0eca91..28671ca55 100644 --- a/cmd/frostfs-node/config/engine/shard/pilorama/config.go +++ b/cmd/frostfs-node/config/engine/shard/pilorama/config.go @@ -13,7 +13,7 @@ type Config config.Config const ( // PermDefault is a default permission bits for metabase file. - PermDefault = 0660 + PermDefault = 0o660 ) // From wraps config section into Config. diff --git a/cmd/frostfs-node/config/grpc/config_test.go b/cmd/frostfs-node/config/grpc/config_test.go index 1ada792ec..13ce4294e 100644 --- a/cmd/frostfs-node/config/grpc/config_test.go +++ b/cmd/frostfs-node/config/grpc/config_test.go @@ -17,7 +17,7 @@ func TestGRPCSection(t *testing.T) { const path = "../../../../config/example/node" - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { num := 0 IterateEndpoints(c, func(sc *Config) { diff --git a/cmd/frostfs-node/config/logger/config_test.go b/cmd/frostfs-node/config/logger/config_test.go index 8d37bdb1b..8a0acbd85 100644 --- a/cmd/frostfs-node/config/logger/config_test.go +++ b/cmd/frostfs-node/config/logger/config_test.go @@ -17,7 +17,7 @@ func TestLoggerSection_Level(t *testing.T) { const path = "../../../../config/example/node" - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { v := loggerconfig.Level(c) require.Equal(t, "debug", v) } diff --git a/cmd/frostfs-node/config/metrics/config_test.go b/cmd/frostfs-node/config/metrics/config_test.go index 4c03729ee..c2a1b1fc4 100644 --- a/cmd/frostfs-node/config/metrics/config_test.go +++ b/cmd/frostfs-node/config/metrics/config_test.go @@ -22,7 +22,7 @@ func TestMetricsSection(t *testing.T) { const path = "../../../../config/example/node" - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { to := metricsconfig.ShutdownTimeout(c) addr := metricsconfig.Address(c) diff --git a/cmd/frostfs-node/config/morph/config_test.go b/cmd/frostfs-node/config/morph/config_test.go index be61d8608..192140446 100644 --- a/cmd/frostfs-node/config/morph/config_test.go +++ b/cmd/frostfs-node/config/morph/config_test.go @@ -23,14 +23,12 @@ func TestMorphSection(t *testing.T) { const path = "../../../../config/example/node" - var ( - rpcs = []client.Endpoint{ - {"wss://rpc1.morph.frostfs.info:40341/ws", 1}, - {"wss://rpc2.morph.frostfs.info:40341/ws", 2}, - } - ) + rpcs := []client.Endpoint{ + {"wss://rpc1.morph.frostfs.info:40341/ws", 1}, + {"wss://rpc2.morph.frostfs.info:40341/ws", 2}, + } - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { require.Equal(t, rpcs, morphconfig.RPCEndpoint(c)) require.Equal(t, 30*time.Second, morphconfig.DialTimeout(c)) require.Equal(t, 15*time.Second, morphconfig.CacheTTL(c)) diff --git a/cmd/frostfs-node/config/node/config_test.go b/cmd/frostfs-node/config/node/config_test.go index 3a1120491..b0041c870 100644 --- a/cmd/frostfs-node/config/node/config_test.go +++ b/cmd/frostfs-node/config/node/config_test.go @@ -56,7 +56,7 @@ func TestNodeSection(t *testing.T) { const path = "../../../../config/example/node" - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { key := Key(c) addrs := BootstrapAddresses(c) attributes := Attributes(c) diff --git a/cmd/frostfs-node/config/object/config_test.go b/cmd/frostfs-node/config/object/config_test.go index 513b6e9c5..e2bb105d9 100644 --- a/cmd/frostfs-node/config/object/config_test.go +++ b/cmd/frostfs-node/config/object/config_test.go @@ -21,7 +21,7 @@ func TestObjectSection(t *testing.T) { const path = "../../../../config/example/node" - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { require.Equal(t, 100, objectconfig.Put(c).PoolSizeRemote()) require.Equal(t, 200, objectconfig.Put(c).PoolSizeLocal()) require.EqualValues(t, 10, objectconfig.TombstoneLifetime(c)) diff --git a/cmd/frostfs-node/config/policer/config_test.go b/cmd/frostfs-node/config/policer/config_test.go index a4c23eae7..95f0c3af2 100644 --- a/cmd/frostfs-node/config/policer/config_test.go +++ b/cmd/frostfs-node/config/policer/config_test.go @@ -19,7 +19,7 @@ func TestPolicerSection(t *testing.T) { const path = "../../../../config/example/node" - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { require.Equal(t, 15*time.Second, policerconfig.HeadTimeout(c)) } diff --git a/cmd/frostfs-node/config/profiler/config_test.go b/cmd/frostfs-node/config/profiler/config_test.go index 355874387..2f1cb1788 100644 --- a/cmd/frostfs-node/config/profiler/config_test.go +++ b/cmd/frostfs-node/config/profiler/config_test.go @@ -25,7 +25,7 @@ func TestProfilerSection(t *testing.T) { const path = "../../../../config/example/node" - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { to := profilerconfig.ShutdownTimeout(c) addr := profilerconfig.Address(c) diff --git a/cmd/frostfs-node/config/replicator/config_test.go b/cmd/frostfs-node/config/replicator/config_test.go index 0118aa0b2..2129c01b4 100644 --- a/cmd/frostfs-node/config/replicator/config_test.go +++ b/cmd/frostfs-node/config/replicator/config_test.go @@ -20,7 +20,7 @@ func TestReplicatorSection(t *testing.T) { const path = "../../../../config/example/node" - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { require.Equal(t, 15*time.Second, replicatorconfig.PutTimeout(c)) require.Equal(t, 10, replicatorconfig.PoolSize(c)) } diff --git a/cmd/frostfs-node/config/tree/config_test.go b/cmd/frostfs-node/config/tree/config_test.go index 898f7e715..285ea0725 100644 --- a/cmd/frostfs-node/config/tree/config_test.go +++ b/cmd/frostfs-node/config/tree/config_test.go @@ -35,7 +35,7 @@ func TestTreeSection(t *testing.T) { require.NoError(t, err) expectedKeys = append(expectedKeys, key) - var fileConfigTest = func(c *config.Config) { + fileConfigTest := func(c *config.Config) { treeSec := treeconfig.Tree(c) require.True(t, treeSec.Enabled()) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 5c3db0d27..edb1412aa 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -239,7 +239,8 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { } func registerNotificationHandlers(scHash util.Uint160, lis event.Listener, parsers map[event.Type]event.NotificationParser, - subs map[event.Type][]event.Handler) { + subs map[event.Type][]event.Handler, +) { for typ, handlers := range subs { pi := event.NotificationParserInfo{} pi.SetType(typ) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 82ddf897f..2f714b821 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -353,7 +353,8 @@ func createSearchSvcV2(sSearch *searchsvc.Service, keyStorage *util.KeyStorage) } func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, - coreConstructor *cache.ClientCache) *getsvc.Service { + coreConstructor *cache.ClientCache, +) *getsvc.Service { ls := c.cfgObject.cfgLocalStorage.localStorage return getsvc.New( @@ -375,7 +376,8 @@ func createGetServiceV2(sGet *getsvc.Service, keyStorage *util.KeyStorage) *gets } func createDeleteService(c *cfg, keyStorage *util.KeyStorage, sGet *getsvc.Service, - sSearch *searchsvc.Service, sPut *putsvc.Service) *deletesvc.Service { + sSearch *searchsvc.Service, sPut *putsvc.Service, +) *deletesvc.Service { return deletesvc.New( sGet, sSearch, @@ -396,7 +398,8 @@ func createDeleteServiceV2(sDelete *deletesvc.Service) *deletesvcV2.Service { } func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Service, - sSearchV2 *searchsvcV2.Service, sDeleteV2 *deletesvcV2.Service) *objectService.TransportSplitter { + sSearchV2 *searchsvcV2.Service, sDeleteV2 *deletesvcV2.Service, +) *objectService.TransportSplitter { return objectService.NewTransportSplitter( c.cfgGRPC.maxChunkSize, c.cfgGRPC.maxAddrAmount, diff --git a/cmd/frostfs-node/tracing.go b/cmd/frostfs-node/tracing.go index 08dc049da..312adfb8d 100644 --- a/cmd/frostfs-node/tracing.go +++ b/cmd/frostfs-node/tracing.go @@ -23,7 +23,7 @@ func initTracing(ctx context.Context, c *cfg) { fn: func() { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - err := tracing.Shutdown(ctx) //cfg context cancels before close + err := tracing.Shutdown(ctx) // cfg context cancels before close if err != nil { c.log.Error(logs.FrostFSNodeFailedShutdownTracing, zap.Error(err)) } diff --git a/cmd/frostfs-node/validate.go b/cmd/frostfs-node/validate.go index e07afb2ca..80c90ec44 100644 --- a/cmd/frostfs-node/validate.go +++ b/cmd/frostfs-node/validate.go @@ -59,7 +59,7 @@ func validateConfig(c *config.Config) error { default: return fmt.Errorf("unexpected storage type: %s (shard %d)", blobstor[i].Type(), shardNum) } - if blobstor[i].Perm()&0600 != 0600 { + if blobstor[i].Perm()&0o600 != 0o600 { return fmt.Errorf("invalid permissions for blobstor component: %s, "+ "expected at least rw- for the owner (shard %d)", blobstor[i].Perm(), shardNum) diff --git a/cmd/internal/common/netmap.go b/cmd/internal/common/netmap.go index 94964e0d2..79b03a726 100644 --- a/cmd/internal/common/netmap.go +++ b/cmd/internal/common/netmap.go @@ -10,7 +10,8 @@ import ( // PrettyPrintNodeInfo print information about network node with given indent and index. // To avoid printing attribute list use short parameter. func PrettyPrintNodeInfo(cmd *cobra.Command, node netmap.NodeInfo, - index int, indent string, short bool) { + index int, indent string, short bool, +) { var strState string switch { diff --git a/pkg/core/client/util.go b/pkg/core/client/util.go index 80c8f49b7..d4bc0cf68 100644 --- a/pkg/core/client/util.go +++ b/pkg/core/client/util.go @@ -22,7 +22,8 @@ func NodeInfoFromRawNetmapElement(dst *NodeInfo, info interface { IterateAddresses(func(string) bool) NumberOfAddresses() int ExternalAddresses() []string -}) error { +}, +) error { var a network.AddressGroup err := a.FromIterator(info) @@ -49,7 +50,8 @@ func NodeInfoFromNetmapElement(dst *NodeInfo, info interface { PublicKey() []byte Addresses() network.AddressGroup ExternalAddresses() network.AddressGroup -}) { +}, +) { nodeInfoFromKeyAddr(dst, info.PublicKey(), info.Addresses(), info.ExternalAddresses()) } diff --git a/pkg/core/object/sender_classifier.go b/pkg/core/object/sender_classifier.go index 79bf12ce3..ac881431e 100644 --- a/pkg/core/object/sender_classifier.go +++ b/pkg/core/object/sender_classifier.go @@ -43,7 +43,8 @@ func (c SenderClassifier) Classify( ownerID *user.ID, ownerKey *keys.PublicKey, idCnr cid.ID, - cnr container.Container) (res *ClassifyResult, err error) { + cnr container.Container, +) (res *ClassifyResult, err error) { ownerKeyInBytes := ownerKey.Bytes() // TODO: #767 get owner from frostfs.id if present @@ -114,7 +115,8 @@ func (c SenderClassifier) isInnerRingKey(owner []byte) (bool, error) { func (c SenderClassifier) isContainerKey( owner, idCnr []byte, - cnr container.Container) (bool, error) { + cnr container.Container, +) (bool, error) { nm, err := core.GetLatestNetworkMap(c.netmap) // first check current netmap if err != nil { return false, err @@ -140,7 +142,8 @@ func (c SenderClassifier) isContainerKey( func lookupKeyInContainer( nm *netmap.NetMap, owner, idCnr []byte, - cnr container.Container) (bool, error) { + cnr container.Container, +) (bool, error) { cnrVectors, err := nm.ContainerNodes(cnr.PlacementPolicy(), idCnr) if err != nil { return false, err diff --git a/pkg/innerring/blocktimer_test.go b/pkg/innerring/blocktimer_test.go index 224aa5c58..9d02721c2 100644 --- a/pkg/innerring/blocktimer_test.go +++ b/pkg/innerring/blocktimer_test.go @@ -117,6 +117,7 @@ type testEpochState struct { func (s *testEpochState) EpochCounter() uint64 { return s.counter } + func (s *testEpochState) EpochDuration() uint64 { return s.duration } diff --git a/pkg/innerring/config/fee_test.go b/pkg/innerring/config/fee_test.go index f7330c6ca..ced21b238 100644 --- a/pkg/innerring/config/fee_test.go +++ b/pkg/innerring/config/fee_test.go @@ -60,5 +60,4 @@ fee: require.Equal(t, fixedn.Fixed8(10), config.MainChainFee(), "main chain fee invalid") require.Equal(t, fixedn.Fixed8(0), config.SideChainFee(), "side chain fee invalid") }) - } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index eb1c4b2d4..f8ebe89fe 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -35,7 +35,8 @@ import ( func (s *Server) initNetmapProcessor(cfg *viper.Viper, cnrClient *container.Client, - alphaSync event.Handler) error { + alphaSync event.Handler, +) error { locodeValidator, err := s.newLocodeValidator(cfg) if err != nil { return err @@ -250,7 +251,8 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { } func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.Client, - frostfsIDClient *frostfsid.Client) error { + frostfsIDClient *frostfsid.Client, +) error { // container processor containerProcessor, err := cont.New(&cont.Params{ Log: s.log, diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index ae2ad4019..49b770811 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -328,7 +328,8 @@ func (s *Server) registerStarter(f func() error) { // New creates instance of inner ring sever structure. func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error, - metrics *metrics.InnerRingServiceMetrics) (*Server, error) { + metrics *metrics.InnerRingServiceMetrics, +) (*Server, error) { var err error server := &Server{ log: log, diff --git a/pkg/innerring/processors/alphabet/handlers_test.go b/pkg/innerring/processors/alphabet/handlers_test.go index c098ca27d..e39244a36 100644 --- a/pkg/innerring/processors/alphabet/handlers_test.go +++ b/pkg/innerring/processors/alphabet/handlers_test.go @@ -256,6 +256,7 @@ func (c *testMorphClient) Invoke(contract util.Uint160, fee fixedn.Fixed8, metho }) return nil } + func (c *testMorphClient) TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error { c.transferedGas = append(c.transferedGas, transferGas{ receiver: receiver, diff --git a/pkg/innerring/processors/balance/handlers_test.go b/pkg/innerring/processors/balance/handlers_test.go index 3470fba2d..10ae8c60e 100644 --- a/pkg/innerring/processors/balance/handlers_test.go +++ b/pkg/innerring/processors/balance/handlers_test.go @@ -73,8 +73,7 @@ func (s *testAlphabetState) IsAlphabet() bool { return s.isAlphabet } -type testPresicionConverter struct { -} +type testPresicionConverter struct{} func (c *testPresicionConverter) ToFixed8(v int64) int64 { return v diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index 77dbe876a..f129127a6 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -319,6 +319,7 @@ func (e *testPutEvent) Signature() []byte { func (e *testPutEvent) SessionToken() []byte { return e.st } + func (e *testPutEvent) NotaryRequest() *payload.P2PNotaryRequest { return e.nr } diff --git a/pkg/innerring/processors/frostfs/handlers_test.go b/pkg/innerring/processors/frostfs/handlers_test.go index db7835811..475ddaaac 100644 --- a/pkg/innerring/processors/frostfs/handlers_test.go +++ b/pkg/innerring/processors/frostfs/handlers_test.go @@ -301,8 +301,7 @@ func (s *testAlphabetState) IsAlphabet() bool { return s.isAlphabet } -type testPrecisionConverter struct { -} +type testPrecisionConverter struct{} func (c *testPrecisionConverter) ToBalancePrecision(v int64) int64 { return v @@ -318,10 +317,12 @@ func (c *testBalaceClient) Mint(p balance.MintPrm) error { c.mint = append(c.mint, p) return nil } + func (c *testBalaceClient) Lock(p balance.LockPrm) error { c.lock = append(c.lock, p) return nil } + func (c *testBalaceClient) Burn(p balance.BurnPrm) error { c.burn = append(c.burn, p) return nil @@ -349,6 +350,7 @@ type testMorphClient struct { func (c *testMorphClient) GasBalance() (res int64, err error) { return c.balance, nil } + func (c *testMorphClient) TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error { c.transferGas = append(c.transferGas, transferGas{ receiver: receiver, diff --git a/pkg/innerring/processors/governance/handlers_test.go b/pkg/innerring/processors/governance/handlers_test.go index 63d156dac..adf674173 100644 --- a/pkg/innerring/processors/governance/handlers_test.go +++ b/pkg/innerring/processors/governance/handlers_test.go @@ -205,7 +205,7 @@ func generateTestKeys(t *testing.T) testKeys { require.NoError(t, err, "failed to create expected new alphabet") if len(result.newAlphabetExp) == 0 { - continue //can be happen because of random and sort + continue // can be happen because of random and sort } var irKeys keys.PublicKeys diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index 6c9e265cc..fa87cff92 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -309,12 +309,15 @@ type testEpochState struct { func (s *testEpochState) SetEpochCounter(c uint64) { s.counter = c } + func (s *testEpochState) EpochCounter() uint64 { return s.counter } + func (s *testEpochState) SetEpochDuration(d uint64) { s.duration = d } + func (s *testEpochState) EpochDuration() uint64 { return s.duration } diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 4bc13cc95..d256c1c69 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -20,8 +20,7 @@ type DeletePrm struct { } // DeleteRes groups the resulting values of Delete operation. -type DeleteRes struct { -} +type DeleteRes struct{} // SetAddress sets the address of the requested object. func (p *DeletePrm) SetAddress(addr oid.Address) { diff --git a/pkg/local_object_storage/blobovnicza/exists.go b/pkg/local_object_storage/blobovnicza/exists.go index e6d28f938..9b190416f 100644 --- a/pkg/local_object_storage/blobovnicza/exists.go +++ b/pkg/local_object_storage/blobovnicza/exists.go @@ -12,9 +12,7 @@ import ( // Exists check if object with the specified address is stored in b. func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error) { - var ( - exists = false - ) + exists := false _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.Exists", trace.WithAttributes( diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index b29ccb43c..f4b85f5d0 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -116,8 +116,7 @@ func (x *IteratePrm) IgnoreErrors() { } // IterateRes groups the resulting values of Iterate operation. -type IterateRes struct { -} +type IterateRes struct{} // Iterate goes through all stored objects, and passes IterationElement to parameterized handler until error return. // diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index 787372211..859f7dbbf 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -20,8 +20,7 @@ type PutPrm struct { } // PutRes groups the resulting values of Put operation. -type PutRes struct { -} +type PutRes struct{} // SetAddress sets the address of the saving object. func (p *PutPrm) SetAddress(addr oid.Address) { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/active.go b/pkg/local_object_storage/blobstor/blobovniczatree/active.go index 526699b45..def197318 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/active.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/active.go @@ -96,7 +96,7 @@ func (m *activeDBManager) getCurrentActiveIfOk(lvlPath string) (*activeDB, error return nil, nil } - blz, err := db.Open() //open db for usage, will be closed on activeDB.Close() + blz, err := db.Open() // open db for usage, will be closed on activeDB.Close() if err != nil { return nil, err } @@ -156,7 +156,7 @@ func (m *activeDBManager) getNextSharedDB(lvlPath string) (*sharedDB, error) { for iterCount < m.leafWidth { path := filepath.Join(lvlPath, u64ToHexString(idx)) shDB := m.dbManager.GetByPath(path) - db, err := shDB.Open() //open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close() + db, err := shDB.Open() // open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close() if err != nil { return nil, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go index 9bdee6df3..3f62c49fa 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -80,7 +80,7 @@ func (c *dbCache) create(path string) *sharedDB { value = c.dbManager.GetByPath(path) - _, err := value.Open() //open db to hold reference, closed by evictedDB.Close() or if cache closed + _, err := value.Open() // open db to hold reference, closed by evictedDB.Close() or if cache closed if err != nil { return value } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 5ea426e1c..d993767b7 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -40,14 +40,14 @@ func (b *Blobovniczas) Init() error { } func (b *Blobovniczas) openManagers() { - b.commondbManager.Open() //order important + b.commondbManager.Open() // order important b.activeDBManager.Open() b.dbCache.Open() } // Close implements common.Storage. func (b *Blobovniczas) Close() error { - b.dbCache.Close() //order important + b.dbCache.Close() // order important b.activeDBManager.Close() b.commondbManager.Close() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 8cbdf613c..198e4741b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -55,7 +55,7 @@ func TestExistsInvalidStorageID(t *testing.T) { // An invalid boltdb file is created so that it returns an error when opened require.NoError(t, os.MkdirAll(filepath.Join(dir, relBadFileDir), os.ModePerm)) - require.NoError(t, os.WriteFile(filepath.Join(dir, relBadFileDir, badFileName), []byte("not a boltdb file content"), 0777)) + require.NoError(t, os.WriteFile(filepath.Join(dir, relBadFileDir, badFileName), []byte("not a boltdb file content"), 0o777)) res, err := b.Exists(context.Background(), common.ExistsPrm{Address: addr, StorageID: []byte(filepath.Join(relBadFileDir, badFileName))}) require.Error(t, err) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index 7bc8e2827..d695cb199 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -28,7 +28,8 @@ type sharedDB struct { } func newSharedDB(options []blobovnicza.Option, path string, readOnly bool, - metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlag *atomic.Bool, log *logger.Logger) *sharedDB { + metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlag *atomic.Bool, log *logger.Logger, +) *sharedDB { return &sharedDB{ guard: &sync.RWMutex{}, @@ -110,7 +111,8 @@ type levelDbManager struct { } func newLevelDBManager(width uint64, options []blobovnicza.Option, rootPath string, lvlPath string, - readOnly bool, metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlog *atomic.Bool, log *logger.Logger) *levelDbManager { + readOnly bool, metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlog *atomic.Bool, log *logger.Logger, +) *levelDbManager { result := &levelDbManager{ databases: make([]*sharedDB, width), } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index d11185652..561b83761 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -28,7 +28,7 @@ type cfg struct { type Option func(*cfg) const ( - defaultPerm = 0700 + defaultPerm = 0o700 defaultOpenedCacheSize = 50 defaultBlzShallowDepth = 2 defaultBlzShallowWidth = 16 diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 243a7239e..bf3400bcd 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -77,7 +77,7 @@ var _ common.Storage = (*FSTree)(nil) func New(opts ...Option) *FSTree { f := &FSTree{ Info: Info{ - Permissions: 0700, + Permissions: 0o700, RootPath: "./", }, Config: nil, diff --git a/pkg/local_object_storage/blobstor/generic_test.go b/pkg/local_object_storage/blobstor/generic_test.go index 204bdf61d..b58ab8a68 100644 --- a/pkg/local_object_storage/blobstor/generic_test.go +++ b/pkg/local_object_storage/blobstor/generic_test.go @@ -7,7 +7,6 @@ import ( ) func TestGeneric(t *testing.T) { - newMetabase := func(t *testing.T) storagetest.Component { return New( WithStorages(defaultStorages(t.TempDir(), 128))) diff --git a/pkg/local_object_storage/blobstor/get.go b/pkg/local_object_storage/blobstor/get.go index 00ed18d7b..d00ef2f21 100644 --- a/pkg/local_object_storage/blobstor/get.go +++ b/pkg/local_object_storage/blobstor/get.go @@ -18,9 +18,7 @@ import ( // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. func (b *BlobStor) Get(ctx context.Context, prm common.GetPrm) (res common.GetRes, err error) { - var ( - startedAt = time.Now() - ) + startedAt := time.Now() defer func() { b.metrics.Get(time.Since(startedAt), len(res.RawData), err == nil, prm.StorageID != nil) }() diff --git a/pkg/local_object_storage/blobstor/get_range.go b/pkg/local_object_storage/blobstor/get_range.go index 50abd19fa..9bded4720 100644 --- a/pkg/local_object_storage/blobstor/get_range.go +++ b/pkg/local_object_storage/blobstor/get_range.go @@ -19,9 +19,7 @@ import ( // If the descriptor is present, only one sub-storage is tried, // Otherwise, each sub-storage is tried in order. func (b *BlobStor) GetRange(ctx context.Context, prm common.GetRangePrm) (res common.GetRangeRes, err error) { - var ( - startedAt = time.Now() - ) + startedAt := time.Now() defer func() { b.metrics.GetRange(time.Since(startedAt), len(res.Data), err == nil, prm.StorageID != nil) }() diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index 34622c857..72f107664 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -91,7 +91,7 @@ func runTestIgnoreLogicalErrors(t *testing.T, s common.Storage, objects []object seen := make(map[string]objectDesc) var n int - var logicErr = errors.New("logic error") + logicErr := errors.New("logic error") var iterPrm common.IteratePrm iterPrm.IgnoreErrors = true iterPrm.Handler = func(elem common.IterationElement) error { diff --git a/pkg/local_object_storage/blobstor/logger.go b/pkg/local_object_storage/blobstor/logger.go index f201f18d7..7e057a0e3 100644 --- a/pkg/local_object_storage/blobstor/logger.go +++ b/pkg/local_object_storage/blobstor/logger.go @@ -6,8 +6,10 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -const deleteOp = "DELETE" -const putOp = "PUT" +const ( + deleteOp = "DELETE" + putOp = "PUT" +) func logOp(l *logger.Logger, op string, addr oid.Address, typ string, sID []byte) { storagelog.Write(l, diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index d97148f73..03196400a 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -30,7 +30,7 @@ func (e *StorageEngine) open(ctx context.Context) error { defer e.mtx.Unlock() var wg sync.WaitGroup - var errCh = make(chan shardInitError, len(e.shards)) + errCh := make(chan shardInitError, len(e.shards)) for id, sh := range e.shards { wg.Add(1) @@ -75,7 +75,7 @@ func (e *StorageEngine) Init(ctx context.Context) error { e.mtx.Lock() defer e.mtx.Unlock() - var errCh = make(chan shardInitError, len(e.shards)) + errCh := make(chan shardInitError, len(e.shards)) var eg errgroup.Group if e.cfg.lowMem && e.anyShardRequiresRefill() { eg.SetLimit(1) diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 0c5ff94da..9cfb311ec 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -62,7 +62,7 @@ func TestInitializationFailure(t *testing.T) { OpenFile: opts.openFileMetabase, }), meta.WithPath(filepath.Join(t.TempDir(), "metabase")), - meta.WithPermissions(0700), + meta.WithPermissions(0o700), meta.WithEpochState(epochState{})), shard.WithWriteCache(true), shard.WithWriteCacheOptions(wcOpts), @@ -228,7 +228,6 @@ func TestPersistentShardID(t *testing.T) { require.Equal(t, te.shards[1].id, newTe.shards[0].id) require.Equal(t, te.shards[0].id, newTe.shards[1].id) require.NoError(t, newTe.ng.Close(context.Background())) - } func TestReload(t *testing.T) { @@ -299,7 +298,7 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str blobstor.WithStorages(newStorages(filepath.Join(addPath, strconv.Itoa(id)), errSmallSize))), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(addPath, fmt.Sprintf("%d.metabase", id))), - meta.WithPermissions(0700), + meta.WithPermissions(0o700), meta.WithEpochState(epochState{}), ), } diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index bb0b682d6..e312e9f0d 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -143,7 +143,8 @@ func (e *StorageEngine) reportShardError( sh hashedShard, msg string, err error, - fields ...zap.Field) { + fields ...zap.Field, +) { if isLogical(err) { e.log.Warn(msg, zap.Stringer("shard_id", sh.ID()), @@ -162,7 +163,8 @@ func (e *StorageEngine) reportShardErrorWithFlags( block bool, msg string, err error, - fields ...zap.Field) { + fields ...zap.Field, +) { sid := sh.ID() e.log.Warn(msg, append([]zap.Field{ zap.Stringer("shard_id", sid), diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 934bea8bb..21bf37dc7 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -144,7 +144,7 @@ func newStorages(root string, smallSize uint64) []blobstor.SubStorage { blobovniczatree.WithRootPath(filepath.Join(root, "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1), - blobovniczatree.WithPermissions(0700)), + blobovniczatree.WithPermissions(0o700)), Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < smallSize }, @@ -163,7 +163,7 @@ func newTestStorages(root string, smallSize uint64) ([]blobstor.SubStorage, *tes blobovniczatree.WithRootPath(filepath.Join(root, "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1), - blobovniczatree.WithPermissions(0700)), + blobovniczatree.WithPermissions(0o700)), )) largeFileStorage := teststore.New( teststore.WithSubstorage(fstree.New( @@ -205,7 +205,8 @@ func testDefaultShardOptions(t testing.TB, id int) []shard.Option { shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(t.TempDir(), "metabase")), - meta.WithPermissions(0700), + meta.WithPermissions(0o700), meta.WithEpochState(epochState{}), - )} + ), + } } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 90356104e..cf62302f2 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -59,12 +59,12 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) shard.WithBlobStorOptions(blobstor.WithStorages(storages)), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", id))), - meta.WithPermissions(0700), + meta.WithPermissions(0o700), meta.WithEpochState(epochState{}), ), shard.WithPiloramaOptions( pilorama.WithPath(filepath.Join(dir, fmt.Sprintf("%d.pilorama", id))), - pilorama.WithPerm(0700)), + pilorama.WithPerm(0o700)), } }) e := te.engine diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 192070c1c..f5f1b658d 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -163,7 +163,6 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev res := NewEvacuateShardRes() ctx = ctxOrBackground(ctx, prm.async) eg, egCtx, err := e.evacuateLimiter.TryStart(ctx, shardIDs, res) - if err != nil { return nil, err } @@ -187,7 +186,8 @@ func ctxOrBackground(ctx context.Context, background bool) context.Context { } func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, +) error { var err error ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShards", trace.WithAttributes( @@ -246,7 +246,8 @@ func (e *StorageEngine) getTotalObjectsCount(ctx context.Context, shardsToEvacua } func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, +) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShard", trace.WithAttributes( attribute.String("shardID", shardID), @@ -322,7 +323,8 @@ func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) } func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, toEvacuate []object.AddressWithType, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard) error { + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, +) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateObjects", trace.WithAttributes( attribute.Int("objects_count", len(toEvacuate)), @@ -378,7 +380,8 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to } func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) (bool, error) { + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes, +) (bool, error) { hrw.SortHasherSliceByWeightValue(shards, weights, hrw.StringHash(addr.EncodeToString())) for j := range shards { select { diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index c0c05d661..13ba94f5b 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -39,7 +39,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng }})), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", id))), - meta.WithPermissions(0700), + meta.WithPermissions(0o700), meta.WithEpochState(epochState{})), } }) @@ -137,7 +137,7 @@ func TestEvacuateShard(t *testing.T) { func TestEvacuateNetwork(t *testing.T) { t.Parallel() - var errReplication = errors.New("handler error") + errReplication := errors.New("handler error") acceptOneOf := func(objects []*objectSDK.Object, max uint64) func(context.Context, oid.Address, *objectSDK.Object) error { var n uint64 diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index eef25d209..246118687 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -72,9 +72,10 @@ func TestListWithCursor(t *testing.T) { shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(t.TempDir(), "metabase")), - meta.WithPermissions(0700), + meta.WithPermissions(0o700), meta.WithEpochState(epochState{}), - )} + ), + } }).engine require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) diff --git a/pkg/local_object_storage/internal/metaerr/error_test.go b/pkg/local_object_storage/internal/metaerr/error_test.go index 5a16aa501..acde48793 100644 --- a/pkg/local_object_storage/internal/metaerr/error_test.go +++ b/pkg/local_object_storage/internal/metaerr/error_test.go @@ -48,6 +48,7 @@ func TestError(t *testing.T) { } }) } + func TestNilWrap(t *testing.T) { require.NoError(t, Wrap(nil)) } diff --git a/pkg/local_object_storage/metabase/children.go b/pkg/local_object_storage/metabase/children.go index 6816358d2..acd367951 100644 --- a/pkg/local_object_storage/metabase/children.go +++ b/pkg/local_object_storage/metabase/children.go @@ -69,7 +69,6 @@ func (db *DB) GetChildren(ctx context.Context, addresses []oid.Address) (map[oid } return nil }) - if err != nil { return nil, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index c0dc7886e..d4afe1878 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -10,8 +10,10 @@ import ( "go.etcd.io/bbolt" ) -var objectPhyCounterKey = []byte("phy_counter") -var objectLogicCounterKey = []byte("logic_counter") +var ( + objectPhyCounterKey = []byte("phy_counter") + objectLogicCounterKey = []byte("logic_counter") +) type objectType uint8 diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index bc5015b60..b3c38f3d7 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -45,7 +45,7 @@ func newDB(t testing.TB, opts ...meta.Option) *meta.DB { bdb := meta.New( append([]meta.Option{ meta.WithPath(filepath.Join(t.TempDir(), "metabase")), - meta.WithPermissions(0600), + meta.WithPermissions(0o600), meta.WithEpochState(epochState{}), }, opts...)..., ) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index f7ff7a129..e84759b88 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -98,8 +98,8 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { var rawRemoved uint64 var availableRemoved uint64 var err error - var sizes = make([]uint64, len(prm.addrs)) - var availableSizes = make([]uint64, len(prm.addrs)) + sizes := make([]uint64, len(prm.addrs)) + availableSizes := make([]uint64, len(prm.addrs)) err = db.boltDB.Update(func(tx *bbolt.Tx) error { // We need to clear slice because tx can try to execute multiple times. diff --git a/pkg/local_object_storage/metabase/expired.go b/pkg/local_object_storage/metabase/expired.go index 43933d12d..aa2cb6f20 100644 --- a/pkg/local_object_storage/metabase/expired.go +++ b/pkg/local_object_storage/metabase/expired.go @@ -83,7 +83,6 @@ func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.A } return nil }) - if err != nil { return nil, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 37a574a02..5099a161b 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -115,7 +115,7 @@ func (db *DB) listWithCursor(tx *bbolt.Tx, result []objectcore.AddressWithType, graveyardBkt := tx.Bucket(graveyardBucketName) garbageBkt := tx.Bucket(garbageBucketName) - var rawAddr = make([]byte, cidSize, addressKeySize) + rawAddr := make([]byte, cidSize, addressKeySize) loop: for ; name != nil; name, _ = c.Next() { diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index abb55c9d1..3e598ed16 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -211,7 +211,6 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) { for _, v := range expected { require.Equal(t, 1, v) } - } func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithType { diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 30a31ab87..0bc2b06f0 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -244,7 +244,6 @@ func freePotentialLocks(tx *bbolt.Tx, idCnr cid.ID, locker oid.ID) ([]oid.Addres return nil }) - if err != nil { return nil, err } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 28f12851f..875388e58 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -101,7 +101,8 @@ func (db *DB) put(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *objectSDK.SplitInfo, - currEpoch uint64) error { + currEpoch uint64, +) error { cnr, ok := obj.ContainerID() if !ok { return errors.New("missing container in object") diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index 6f011c246..ddeddd189 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -23,7 +23,7 @@ func TestVersion(t *testing.T) { newDB := func(t *testing.T) *DB { return New(WithPath(filepath.Join(dir, t.Name())), - WithPermissions(0600), WithEpochState(epochStateImpl{})) + WithPermissions(0o600), WithEpochState(epochStateImpl{})) } check := func(t *testing.T, db *DB) { require.NoError(t, db.boltDB.View(func(tx *bbolt.Tx) error { diff --git a/pkg/local_object_storage/metrics/fstree.go b/pkg/local_object_storage/metrics/fstree.go index d3749d9bc..0def3210c 100644 --- a/pkg/local_object_storage/metrics/fstree.go +++ b/pkg/local_object_storage/metrics/fstree.go @@ -28,6 +28,7 @@ func (m *fstreeMetrics) SetParentID(parentID string) { func (m *fstreeMetrics) SetMode(readOnly bool) { m.m.SetMode(m.shardID, m.path, readOnly) } + func (m *fstreeMetrics) Close() { m.m.Close(m.shardID, m.path) } @@ -35,24 +36,29 @@ func (m *fstreeMetrics) Close() { func (m *fstreeMetrics) Iterate(d time.Duration, success bool) { m.m.MethodDuration(m.shardID, m.path, "Iterate", d, success) } + func (m *fstreeMetrics) Delete(d time.Duration, success bool) { m.m.MethodDuration(m.shardID, m.path, "Delete", d, success) } + func (m *fstreeMetrics) Exists(d time.Duration, success bool) { m.m.MethodDuration(m.shardID, m.path, "Exists", d, success) } + func (m *fstreeMetrics) Put(d time.Duration, size int, success bool) { m.m.MethodDuration(m.shardID, m.path, "Put", d, success) if success { m.m.AddPut(m.shardID, m.path, size) } } + func (m *fstreeMetrics) Get(d time.Duration, size int, success bool) { m.m.MethodDuration(m.shardID, m.path, "Get", d, success) if success { m.m.AddGet(m.shardID, m.path, size) } } + func (m *fstreeMetrics) GetRange(d time.Duration, size int, success bool) { m.m.MethodDuration(m.shardID, m.path, "GetRange", d, success) if success { diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 2689e345a..1bc161341 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -111,6 +111,7 @@ func (t *boltForest) SetMode(m mode.Mode) error { t.metrics.SetMode(m) return nil } + func (t *boltForest) Open(_ context.Context, readOnly bool) error { err := util.MkdirAllX(filepath.Dir(t.path), t.perm) if err != nil { @@ -137,6 +138,7 @@ func (t *boltForest) Open(_ context.Context, readOnly bool) error { t.metrics.SetMode(m) return nil } + func (t *boltForest) Init() error { if t.mode.NoMetabase() || t.db.IsReadOnly() { return nil @@ -150,6 +152,7 @@ func (t *boltForest) Init() error { return err }) } + func (t *boltForest) Close() error { var err error if t.db != nil { diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index e5612d2b9..b82e5b3bd 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -71,7 +71,8 @@ func (f *memoryForest) TreeAddByPath(_ context.Context, d CIDDescriptor, treeID Parent: node, Meta: Meta{ Time: s.timestamp(d.Position, d.Size), - Items: []KeyValue{{Key: attr, Value: []byte(path[j])}}}, + Items: []KeyValue{{Key: attr, Value: []byte(path[j])}}, + }, Child: s.findSpareID(), }) lm[j-i] = op.Move @@ -113,9 +114,11 @@ func (f *memoryForest) Init() error { func (f *memoryForest) Open(context.Context, bool) error { return nil } + func (f *memoryForest) SetMode(mode.Mode) error { return nil } + func (f *memoryForest) Close() error { return nil } diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 8e7fec200..67454b686 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -36,7 +36,8 @@ var providers = []struct { f := NewBoltForest( append([]Option{ WithPath(filepath.Join(t.TempDir(), "test.db")), - WithMaxBatchSize(1)}, opts...)...) + WithMaxBatchSize(1), + }, opts...)...) require.NoError(t, f.Open(context.Background(), false)) require.NoError(t, f.Init()) t.Cleanup(func() { @@ -68,7 +69,8 @@ func testForestTreeMove(t *testing.T, s Forest) { meta := []KeyValue{ {Key: AttributeVersion, Value: []byte("XXX")}, - {Key: AttributeFilename, Value: []byte("file.txt")}} + {Key: AttributeFilename, Value: []byte("file.txt")}, + } lm, err := s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, []string{"path", "to"}, meta) require.NoError(t, err) require.Equal(t, 3, len(lm)) @@ -261,7 +263,8 @@ func testForestTreeAdd(t *testing.T, s Forest) { meta := []KeyValue{ {Key: AttributeVersion, Value: []byte("XXX")}, - {Key: AttributeFilename, Value: []byte("file.txt")}} + {Key: AttributeFilename, Value: []byte("file.txt")}, + } m := &Move{ Parent: RootID, Child: RootID, @@ -306,7 +309,8 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { meta := []KeyValue{ {Key: AttributeVersion, Value: []byte("XXX")}, - {Key: AttributeFilename, Value: []byte("file.txt")}} + {Key: AttributeFilename, Value: []byte("file.txt")}, + } t.Run("invalid descriptor", func(t *testing.T) { _, err := s.TreeAddByPath(context.Background(), CIDDescriptor{cid, 0, 0}, treeID, AttributeFilename, []string{"yyy"}, meta) @@ -381,7 +385,8 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { testMeta(t, s, cid, treeID, oldMove.Child, oldMove.Parent, Meta{Time: oldMove.Time, Items: []KeyValue{ {AttributeVersion, []byte("SomeValue")}, - {AttributeFilename, []byte("another")}}}) + {AttributeFilename, []byte("another")}, + }}) t.Run("get by path", func(t *testing.T) { nodes, err := s.TreeGetByPath(context.Background(), cid, treeID, AttributeFilename, []string{"path", "another"}, false) @@ -399,7 +404,8 @@ func testForestTreeAddByPath(t *testing.T, s Forest) { t.Run("empty component", func(t *testing.T) { meta := []KeyValue{ {Key: AttributeVersion, Value: []byte("XXX")}, - {Key: AttributeFilename, Value: []byte{}}} + {Key: AttributeFilename, Value: []byte{}}, + } lm, err := s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, []string{"path", "to"}, meta) require.NoError(t, err) require.Equal(t, 1, len(lm)) diff --git a/pkg/local_object_storage/pilorama/meta_test.go b/pkg/local_object_storage/pilorama/meta_test.go index 9df4c7e94..f329f6092 100644 --- a/pkg/local_object_storage/pilorama/meta_test.go +++ b/pkg/local_object_storage/pilorama/meta_test.go @@ -21,7 +21,8 @@ func TestMeta_Bytes(t *testing.T) { Items: []KeyValue{ {"abc", []byte{1, 2, 3}}, {AttributeFilename, []byte{}}, - }} + }, + } data := expected.Bytes() @@ -35,7 +36,8 @@ func TestMeta_Bytes(t *testing.T) { Items: []KeyValue{ {"abc", []byte{1, 2, 3}}, {"xyz", []byte{5, 6, 7, 8}}, - }} + }, + } data := expected.Bytes() diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index 332cdf5be..b64cb9c40 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -101,7 +101,7 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { _, err = sh.Get(context.Background(), getPrm) require.NoError(t, err, "failed to get") - //inhume + // inhume var inhumePrm InhumePrm inhumePrm.MarkAsGarbage(addr) _, err = sh.Inhume(context.Background(), inhumePrm) @@ -110,13 +110,13 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { require.Error(t, err, "get returned error") require.True(t, client.IsErrObjectNotFound(err), "invalid error type") - //storageID + // storageID var metaStIDPrm meta.StorageIDPrm metaStIDPrm.SetAddress(addr) storageID, err := sh.metaBase.StorageID(context.Background(), metaStIDPrm) require.NoError(t, err, "failed to get storage ID") - //check existence in blobstore + // check existence in blobstore var bsExisted common.ExistsPrm bsExisted.Address = addr bsExisted.StorageID = storageID.StorageID() @@ -124,19 +124,19 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { require.NoError(t, err, "failed to check blobstore existence") require.True(t, exRes.Exists, "invalid blobstore existence result") - //drop from blobstor + // drop from blobstor var bsDeletePrm common.DeletePrm bsDeletePrm.Address = addr bsDeletePrm.StorageID = storageID.StorageID() _, err = sh.blobStor.Delete(context.Background(), bsDeletePrm) require.NoError(t, err, "failed to delete from blobstore") - //check existence in blobstore + // check existence in blobstore exRes, err = sh.blobStor.Exists(context.Background(), bsExisted) require.NoError(t, err, "failed to check blobstore existence") require.False(t, exRes.Exists, "invalid blobstore existence result") - //get should return object not found + // get should return object not found _, err = sh.Get(context.Background(), getPrm) require.Error(t, err, "get returned no error") require.True(t, client.IsErrObjectNotFound(err), "invalid error type") diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 9cfa267e8..fc1fc2d20 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -41,7 +41,8 @@ func TestShardReload(t *testing.T) { metaOpts := []meta.Option{ meta.WithPath(filepath.Join(p, "meta")), - meta.WithEpochState(epochState{})} + meta.WithEpochState(epochState{}), + } opts := []Option{ WithID(NewIDFromBytes([]byte{})), @@ -49,7 +50,8 @@ func TestShardReload(t *testing.T) { WithBlobStorOptions(blobOpts...), WithMetaBaseOptions(metaOpts...), WithPiloramaOptions( - pilorama.WithPath(filepath.Join(p, "pilorama")))} + pilorama.WithPath(filepath.Join(p, "pilorama"))), + } sh := New(opts...) require.NoError(t, sh.Open(context.Background())) diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 9da9eb6b8..811c7f3ab 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -94,7 +94,8 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard WithBlobStorOptions(o.bsOpts...), WithMetaBaseOptions( append([]meta.Option{ - meta.WithPath(filepath.Join(o.rootPath, "meta")), meta.WithEpochState(epochState{})}, + meta.WithPath(filepath.Join(o.rootPath, "meta")), meta.WithEpochState(epochState{}), + }, o.metaOptions...)..., ), WithPiloramaOptions(pilorama.WithPath(filepath.Join(o.rootPath, "pilorama"))), diff --git a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go index 9d54cd5c2..9bbf5b7df 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go @@ -55,9 +55,7 @@ const ( defaultMaxCacheSize = 1 << 30 // 1 GiB ) -var ( - defaultBucket = []byte{0} -) +var defaultBucket = []byte{0} // New creates new writecache instance. func New(opts ...Option) writecache.Cache { diff --git a/pkg/local_object_storage/writecache/writecachebbolt/state.go b/pkg/local_object_storage/writecache/writecachebbolt/state.go index 8749ec601..9261b2606 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/state.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/state.go @@ -13,7 +13,7 @@ func (c *cache) estimateCacheSize() uint64 { dbCount := c.objCounters.DB() fsCount := c.objCounters.FS() if fsCount > 0 { - fsCount-- //db file + fsCount-- // db file } dbSize := dbCount * c.smallObjectSize fsSize := fsCount * c.maxObjectSize diff --git a/pkg/metrics/replicator.go b/pkg/metrics/replicator.go index a1519ac95..ca72a3031 100644 --- a/pkg/metrics/replicator.go +++ b/pkg/metrics/replicator.go @@ -5,8 +5,6 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -//TODO - type ReplicatorMetrics interface { IncInFlightRequest() DecInFlightRequest() diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index e7e1bbca9..50a9572d4 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -55,9 +55,7 @@ const ( defaultWaitInterval = 500 * time.Millisecond ) -var ( - ErrNoHealthyEndpoint = errors.New("no healthy endpoint") -) +var ErrNoHealthyEndpoint = errors.New("no healthy endpoint") func defaultConfig() *cfg { return &cfg{ diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index c8702b1c7..9dd3a337b 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -42,9 +42,7 @@ const ( putNamedMethod = "putNamed" ) -var ( - errNilArgument = errors.New("empty argument") -) +var errNilArgument = errors.New("empty argument") // NewFromMorph returns the wrapper instance from the raw morph client. // diff --git a/pkg/morph/client/netmap/peer.go b/pkg/morph/client/netmap/peer.go index 7ceaa0250..4769c038c 100644 --- a/pkg/morph/client/netmap/peer.go +++ b/pkg/morph/client/netmap/peer.go @@ -22,7 +22,7 @@ func (a *AddPeerPrm) SetNodeInfo(nodeInfo netmap.NodeInfo) { // AddPeer registers peer in FrostFS network through // Netmap contract call. func (c *Client) AddPeer(p AddPeerPrm) error { - var method = addPeerMethod + method := addPeerMethod if c.client.WithNotary() && c.client.IsAlpha() { // In notary environments Alphabet must calls AddPeerIR method instead of AddPeer. diff --git a/pkg/morph/event/frostfs/ir_update_test.go b/pkg/morph/event/frostfs/ir_update_test.go index 8ce6fdc36..fae87e5f9 100644 --- a/pkg/morph/event/frostfs/ir_update_test.go +++ b/pkg/morph/event/frostfs/ir_update_test.go @@ -16,13 +16,11 @@ func genKey(t *testing.T) *keys.PrivateKey { } func TestParseUpdateInnerRing(t *testing.T) { - var ( - publicKeys = []*keys.PublicKey{ - genKey(t).PublicKey(), - genKey(t).PublicKey(), - genKey(t).PublicKey(), - } - ) + publicKeys := []*keys.PublicKey{ + genKey(t).PublicKey(), + genKey(t).PublicKey(), + genKey(t).PublicKey(), + } t.Run("wrong number of parameters", func(t *testing.T) { prms := []stackitem.Item{ diff --git a/pkg/morph/event/listener_test.go b/pkg/morph/event/listener_test.go index dc7ac3b81..ae92bd580 100644 --- a/pkg/morph/event/listener_test.go +++ b/pkg/morph/event/listener_test.go @@ -147,7 +147,6 @@ func TestErrorPassing(t *testing.T) { require.ErrorIs(t, err, bErr, "invalid block error") }) - } type testSubscriber struct { @@ -166,6 +165,7 @@ func (s *testSubscriber) UnsubscribeForNotification() {} func (s *testSubscriber) BlockNotifications() error { return s.blockErr } + func (s *testSubscriber) SubscribeForNotaryRequests(mainTXSigner util.Uint160) error { return nil } diff --git a/pkg/morph/event/utils.go b/pkg/morph/event/utils.go index 2a7c6250d..f3b6443fb 100644 --- a/pkg/morph/event/utils.go +++ b/pkg/morph/event/utils.go @@ -88,7 +88,6 @@ func WorkerPoolHandler(w util2.WorkerPool, h Handler, log *logger.Logger) Handle err := w.Submit(func() { h(e) }) - if err != nil { log.Warn(logs.EventCouldNotSubmitHandlerToWorkerPool, zap.String("error", err.Error()), diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index c291988fd..ee5466a7d 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -296,7 +296,8 @@ drainloop: // restoreSubscriptions restores subscriptions according to // cached information about them. func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotificationEvent, - blCh chan<- *block.Block, notaryCh chan<- *result.NotaryRequestEvent) bool { + blCh chan<- *block.Block, notaryCh chan<- *result.NotaryRequestEvent, +) bool { var err error // new block events restoration diff --git a/pkg/network/transport/netmap/grpc/service.go b/pkg/network/transport/netmap/grpc/service.go index 7a3aec86c..406c77e58 100644 --- a/pkg/network/transport/netmap/grpc/service.go +++ b/pkg/network/transport/netmap/grpc/service.go @@ -24,7 +24,8 @@ func New(c netmapsvc.Server) *Server { // LocalNodeInfo converts gRPC request message and passes it to internal netmap service. func (s *Server) LocalNodeInfo( ctx context.Context, - req *netmapGRPC.LocalNodeInfoRequest) (*netmapGRPC.LocalNodeInfoResponse, error) { + req *netmapGRPC.LocalNodeInfoRequest, +) (*netmapGRPC.LocalNodeInfoResponse, error) { nodeInfoReq := new(netmap.LocalNodeInfoRequest) if err := nodeInfoReq.FromGRPCMessage(req); err != nil { return nil, err diff --git a/pkg/services/container/morph/executor_test.go b/pkg/services/container/morph/executor_test.go index a270ee856..42dd0b77e 100644 --- a/pkg/services/container/morph/executor_test.go +++ b/pkg/services/container/morph/executor_test.go @@ -51,7 +51,8 @@ func TestInvalidToken(t *testing.T) { sign := func(reqBody interface { StableMarshal([]byte) []byte SetSignature(signature *refs.Signature) - }) { + }, + ) { signer := frostfsecdsa.Signer(priv.PrivateKey) var sig frostfscrypto.Signature require.NoError(t, sig.Calculate(signer, reqBody.StableMarshal(nil))) diff --git a/pkg/services/control/service_test.go b/pkg/services/control/service_test.go index 8c96e2b73..1d98cc6f1 100644 --- a/pkg/services/control/service_test.go +++ b/pkg/services/control/service_test.go @@ -103,6 +103,7 @@ func equalListShardResponseBodies(b1, b2 *control.ListShardsResponse_Body) bool return true } + func compareBlobstorInfo(a, b []*control.BlobstorInfo) bool { if len(a) != len(b) { return false diff --git a/pkg/services/control/types_test.go b/pkg/services/control/types_test.go index 1505a985c..df0cdf141 100644 --- a/pkg/services/control/types_test.go +++ b/pkg/services/control/types_test.go @@ -142,7 +142,8 @@ func generateShardInfo(id int) *control.ShardInfo { si.SetMetabasePath(filepath.Join(path, "meta")) si.Blobstor = []*control.BlobstorInfo{ {Type: fstree.Type, Path: filepath.Join(path, "fstree")}, - {Type: blobovniczatree.Type, Path: filepath.Join(path, "blobtree")}} + {Type: blobovniczatree.Type, Path: filepath.Join(path, "blobtree")}, + } si.SetWriteCachePath(filepath.Join(path, "writecache")) si.SetPiloramaPath(filepath.Join(path, "pilorama")) diff --git a/pkg/services/netmap/executor.go b/pkg/services/netmap/executor.go index d1e7a949e..496b07a98 100644 --- a/pkg/services/netmap/executor.go +++ b/pkg/services/netmap/executor.go @@ -64,7 +64,8 @@ func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo, func (s *executorSvc) LocalNodeInfo( _ context.Context, - req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { + req *netmap.LocalNodeInfoRequest, +) (*netmap.LocalNodeInfoResponse, error) { verV2 := req.GetMetaHeader().GetVersion() if verV2 == nil { return nil, errors.New("missing version") @@ -106,7 +107,8 @@ func (s *executorSvc) LocalNodeInfo( func (s *executorSvc) NetworkInfo( _ context.Context, - req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { + req *netmap.NetworkInfoRequest, +) (*netmap.NetworkInfoResponse, error) { verV2 := req.GetMetaHeader().GetVersion() if verV2 == nil { return nil, errors.New("missing protocol version in meta header") diff --git a/pkg/services/netmap/sign.go b/pkg/services/netmap/sign.go index 305d3443e..9a16ad8f1 100644 --- a/pkg/services/netmap/sign.go +++ b/pkg/services/netmap/sign.go @@ -23,7 +23,8 @@ func NewSignService(key *ecdsa.PrivateKey, svc Server) Server { func (s *signService) LocalNodeInfo( ctx context.Context, - req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { + req *netmap.LocalNodeInfoRequest, +) (*netmap.LocalNodeInfoResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(netmap.LocalNodeInfoResponse) return resp, s.sigSvc.SignResponse(resp, err) diff --git a/pkg/services/object/acl/v2/errors.go b/pkg/services/object/acl/v2/errors.go index 7fc20d618..11b9e6e5f 100644 --- a/pkg/services/object/acl/v2/errors.go +++ b/pkg/services/object/acl/v2/errors.go @@ -21,8 +21,10 @@ var ( errInvalidVerb = malformedRequestError("session token verb is invalid") ) -const accessDeniedACLReasonFmt = "access to operation %s is denied by basic ACL check" -const accessDeniedEACLReasonFmt = "access to operation %s is denied by extended ACL check: %v" +const ( + accessDeniedACLReasonFmt = "access to operation %s is denied by basic ACL check" + accessDeniedEACLReasonFmt = "access to operation %s is denied by extended ACL check: %v" +) func basicACLErr(info RequestInfo) error { errAccessDenied := &apistatus.ObjectAccessDenied{} diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 8239403a7..b5bd3d4f4 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -80,7 +80,8 @@ func New(next object.ServiceServer, irf InnerRingFetcher, acl ACLChecker, cs container.Source, - opts ...Option) Service { + opts ...Option, +) Service { cfg := &cfg{ log: &logger.Logger{Logger: zap.L()}, next: next, @@ -168,7 +169,8 @@ func (b Service) Put() (object.PutObjectStream, error) { func (b Service) Head( ctx context.Context, - request *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { + request *objectV2.HeadRequest, +) (*objectV2.HeadResponse, error) { cnr, err := getContainerIDFromRequest(request) if err != nil { return nil, err @@ -276,7 +278,8 @@ func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStr func (b Service) Delete( ctx context.Context, - request *objectV2.DeleteRequest) (*objectV2.DeleteResponse, error) { + request *objectV2.DeleteRequest, +) (*objectV2.DeleteResponse, error) { cnr, err := getContainerIDFromRequest(request) if err != nil { return nil, err @@ -384,7 +387,8 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb func (b Service) GetRangeHash( ctx context.Context, - request *objectV2.GetRangeHashRequest) (*objectV2.GetRangeHashResponse, error) { + request *objectV2.GetRangeHashRequest, +) (*objectV2.GetRangeHashResponse, error) { cnr, err := getContainerIDFromRequest(request) if err != nil { return nil, err diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index b74a4c7ba..0ba4da437 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -69,7 +69,8 @@ func New(gs *getsvc.Service, ps *putsvc.Service, ni NetworkInfo, ks *util.KeyStorage, - opts ...Option) *Service { + opts ...Option, +) *Service { c := &cfg{ log: &logger.Logger{Logger: zap.L()}, header: &headSvcWrapper{s: gs}, diff --git a/pkg/services/object/get/assembler.go b/pkg/services/object/get/assembler.go index 61de0ee99..f10c67e52 100644 --- a/pkg/services/object/get/assembler.go +++ b/pkg/services/object/get/assembler.go @@ -29,7 +29,8 @@ func newAssembler( addr oid.Address, splitInfo *objectSDK.SplitInfo, rng *objectSDK.Range, - objGetter objectGetter) *assembler { + objGetter objectGetter, +) *assembler { return &assembler{ addr: addr, rng: rng, diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index e3037a70b..3d5547047 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -23,6 +23,7 @@ func (s *Service) GetRange(ctx context.Context, prm RangePrm) error { rng: prm.rng, }) } + func (s *Service) GetRangeHash(ctx context.Context, prm RangeHashPrm) (*RangeHashRes, error) { hashes := make([][]byte, 0, len(prm.rngs)) diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 3ac487265..f4d4f7372 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -233,8 +233,7 @@ func (whe *writeHeaderError) Error() string { return "write header error" } -type writeHeaderErrorObjectWriter struct { -} +type writeHeaderErrorObjectWriter struct{} func (w *writeHeaderErrorObjectWriter) WriteHeader(_ context.Context, _ *objectSDK.Object) error { return &writeHeaderError{} @@ -250,8 +249,7 @@ func (whe *writePayloadError) Error() string { return "write payload error" } -type writePayloadErrorObjectWriter struct { -} +type writePayloadErrorObjectWriter struct{} func (w *writePayloadErrorObjectWriter) WriteHeader(_ context.Context, _ *objectSDK.Object) error { return nil @@ -261,8 +259,7 @@ func (w *writePayloadErrorObjectWriter) WriteChunk(_ context.Context, _ []byte) return &writePayloadError{} } -type testKeyStorage struct { -} +type testKeyStorage struct{} func (ks *testKeyStorage) GetKey(_ *util.SessionInfo) (*ecdsa.PrivateKey, error) { return &ecdsa.PrivateKey{}, nil @@ -1289,7 +1286,6 @@ func TestGetRemoteSmall(t *testing.T) { err := svc.GetRange(ctx, p) require.ErrorAs(t, err, new(*apistatus.ObjectOutOfRange)) }) - }) t.Run("right child", func(t *testing.T) { diff --git a/pkg/services/object/internal/key.go b/pkg/services/object/internal/key.go index 7ab5f082c..eba716976 100644 --- a/pkg/services/object/internal/key.go +++ b/pkg/services/object/internal/key.go @@ -10,7 +10,8 @@ import ( // VerifyResponseKeyV2 checks if response is signed with expected key. Returns client.ErrWrongPublicKey if not. func VerifyResponseKeyV2(expectedKey []byte, resp interface { GetVerificationHeader() *session.ResponseVerificationHeader -}) error { +}, +) error { if !bytes.Equal(resp.GetVerificationHeader().GetBodySignature().GetKey(), expectedKey) { return client.ErrWrongPublicKey } diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 3a7dcefd6..a93873738 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -72,7 +72,8 @@ func NewService(ks *objutil.KeyStorage, nk netmap.AnnouncedKeys, nst netmap.State, ir InnerRing, - opts ...Option) *Service { + opts ...Option, +) *Service { c := &cfg{ remotePool: util.NewPseudoWorkerPool(), localPool: util.NewPseudoWorkerPool(), diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 3865aabb9..183b2310d 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -201,7 +201,8 @@ func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumb } func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *nodeDesc, obj *objectSDK.Object, - signer *putSingleRequestSigner, meta object.ContentMeta) error { + signer *putSingleRequestSigner, meta object.ContentMeta, +) error { if nodeDesc.local { return s.saveLocal(ctx, obj, meta) } @@ -229,7 +230,8 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, signer *putSingleRequestSigner, obj *objectSDK.Object, info client.NodeInfo, - c client.MultiAddressClient) error { + c client.MultiAddressClient, +) error { ctx, span := tracing.StartSpanFromContext(ctx, "putService.redirectPutSingleRequest") defer span.End() diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index 863312200..0a54f54c5 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -62,7 +62,8 @@ func New(e *engine.StorageEngine, tg *util.TraverserGenerator, ns netmap.Source, ks *util.KeyStorage, - opts ...Option) *Service { + opts ...Option, +) *Service { c := &cfg{ log: &logger.Logger{Logger: zap.L()}, clientConstructor: &clientConstructorWrapper{ diff --git a/pkg/services/object/util/prm.go b/pkg/services/object/util/prm.go index c09c07cc1..022b9fe5b 100644 --- a/pkg/services/object/util/prm.go +++ b/pkg/services/object/util/prm.go @@ -109,7 +109,8 @@ func (p *CommonPrm) ForgetTokens() { func CommonPrmFromV2(req interface { GetMetaHeader() *session.RequestMetaHeader -}) (*CommonPrm, error) { +}, +) (*CommonPrm, error) { meta := req.GetMetaHeader() ttl := meta.GetTTL() diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index c04273468..9b44aceda 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -85,7 +85,8 @@ type placementRequirements struct { } func (p *Policer) processNodes(ctx context.Context, requirements *placementRequirements, addrWithType objectcore.AddressWithType, - nodes []netmap.NodeInfo, shortage uint32, checkedNodes nodeCache) { + nodes []netmap.NodeInfo, shortage uint32, checkedNodes nodeCache, +) { addr := addrWithType.Address typ := addrWithType.Type @@ -175,7 +176,8 @@ func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes nodeCache } func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address, requirements *placementRequirements, - nodes []netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) { + nodes []netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int, +) { switch { case shortage > 0: p.log.Debug(logs.PolicerShortageOfObjectCopiesDetected, @@ -212,8 +214,7 @@ func isClientErrMaintenance(err error) bool { switch unwrapErr(err).(type) { default: return false - case - *apistatus.NodeUnderMaintenance: + case *apistatus.NodeUnderMaintenance: return true } } diff --git a/pkg/services/session/storage/persistent/executor_test.go b/pkg/services/session/storage/persistent/executor_test.go index 860b95897..ee0c8f915 100644 --- a/pkg/services/session/storage/persistent/executor_test.go +++ b/pkg/services/session/storage/persistent/executor_test.go @@ -166,14 +166,14 @@ func TestTokenStore_RemoveOld(t *testing.T) { // // If this test is passing, TokenStore works correctly. func TestBolt_Cursor(t *testing.T) { - db, err := bbolt.Open(filepath.Join(t.TempDir(), ".storage"), 0666, nil) + db, err := bbolt.Open(filepath.Join(t.TempDir(), ".storage"), 0o666, nil) require.NoError(t, err) defer db.Close() cursorKeys := make(map[string]struct{}) - var bucketName = []byte("bucket") + bucketName := []byte("bucket") err = db.Update(func(tx *bbolt.Tx) (err error) { b, err := tx.CreateBucket(bucketName) diff --git a/pkg/services/session/storage/persistent/storage.go b/pkg/services/session/storage/persistent/storage.go index e4c2091c0..71711e371 100644 --- a/pkg/services/session/storage/persistent/storage.go +++ b/pkg/services/session/storage/persistent/storage.go @@ -40,7 +40,7 @@ func NewTokenStore(path string, opts ...Option) (*TokenStore, error) { o(cfg) } - db, err := bbolt.Open(path, 0600, + db, err := bbolt.Open(path, 0o600, &bbolt.Options{ Timeout: cfg.timeout, }) diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index 88a5b5e06..63cdc73a2 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -33,7 +33,8 @@ func TestGetSubTree(t *testing.T) { for i := range tree { path := tree[i].path meta := []pilorama.KeyValue{ - {Key: pilorama.AttributeFilename, Value: []byte(path[len(path)-1])}} + {Key: pilorama.AttributeFilename, Value: []byte(path[len(path)-1])}, + } lm, err := p.TreeAddByPath(context.Background(), d, treeID, pilorama.AttributeFilename, path[:len(path)-1], meta) require.NoError(t, err) @@ -141,7 +142,8 @@ func TestGetSubTreeOrderAsc(t *testing.T) { for i := range tree { path := tree[i].path meta := []pilorama.KeyValue{ - {Key: pilorama.AttributeFilename, Value: []byte(path[len(path)-1])}} + {Key: pilorama.AttributeFilename, Value: []byte(path[len(path)-1])}, + } lm, err := p.TreeAddByPath(context.Background(), d, treeID, pilorama.AttributeFilename, path[:len(path)-1], meta) require.NoError(t, err) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 6dad22774..15067d3cd 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -37,9 +37,11 @@ func eACLErr(op eacl.Operation, err error) error { return fmt.Errorf("access to operation %s is denied by extended ACL check: %w", op, err) } -var errBearerWrongOwner = errors.New("bearer token must be signed by the container owner") -var errBearerWrongContainer = errors.New("bearer token is created for another container") -var errBearerSignature = errors.New("invalid bearer token signature") +var ( + errBearerWrongOwner = errors.New("bearer token must be signed by the container owner") + errBearerWrongContainer = errors.New("bearer token is created for another container") + errBearerSignature = errors.New("invalid bearer token signature") +) // verifyClient verifies if the request for a client operation // was signed by a key allowed by (e)ACL rules. @@ -261,8 +263,10 @@ func eACLRole(role acl.Role) eacl.Role { } } -var errDENY = errors.New("DENY eACL rule") -var errNoAllowRules = errors.New("not found allowing rules for the request") +var ( + errDENY = errors.New("DENY eACL rule") + errNoAllowRules = errors.New("not found allowing rules for the request") +) // checkEACL searches for the eACL rules that could be applied to the request // (a tuple of a signer key, his FrostFS role and a request operation). diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 9cff8b351..f5a7fbce6 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -176,7 +176,8 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram } func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID string, - operationStream <-chan *pilorama.Move) uint64 { + operationStream <-chan *pilorama.Move, +) uint64 { errGroup, _ := errgroup.WithContext(ctx) const workersCount = 1024 errGroup.SetLimit(workersCount) @@ -215,7 +216,8 @@ func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID s } func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, - height uint64, treeClient TreeServiceClient, opsCh chan<- *pilorama.Move) (uint64, error) { + height uint64, treeClient TreeServiceClient, opsCh chan<- *pilorama.Move, +) (uint64, error) { rawCID := make([]byte, sha256.Size) cid.Encode(rawCID) @@ -261,11 +263,9 @@ func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, // already applied operation and keep good batching. // The method returns a height that service should start sync from in the next time. func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, - treeID string, nodes []netmapSDK.NodeInfo) uint64 { - s.log.Debug(logs.TreeSynchronizeTree, - zap.Stringer("cid", cid), - zap.String("tree", treeID), - zap.Uint64("from", from)) + treeID string, nodes []netmapSDK.NodeInfo, +) uint64 { + s.log.Debug(logs.TreeSynchronizeTree, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.Uint64("from", from)) errGroup, egCtx := errgroup.WithContext(ctx) const workersCount = 1024 diff --git a/pkg/util/locode/table/csv/opts.go b/pkg/util/locode/table/csv/opts.go index 5aaffd7c1..68e442899 100644 --- a/pkg/util/locode/table/csv/opts.go +++ b/pkg/util/locode/table/csv/opts.go @@ -15,7 +15,7 @@ type options struct { func defaultOpts() *options { return &options{ - mode: 0700, + mode: 0o700, } } diff --git a/pkg/util/os.go b/pkg/util/os.go index 1c4c97806..30e08a8c3 100644 --- a/pkg/util/os.go +++ b/pkg/util/os.go @@ -6,5 +6,5 @@ import "os" // but with +x for a user and a group. This makes the created // dir openable regardless of the passed permissions. func MkdirAllX(path string, perm os.FileMode) error { - return os.MkdirAll(path, perm|0110) + return os.MkdirAll(path, perm|0o110) } diff --git a/pkg/util/state/storage.go b/pkg/util/state/storage.go index 0485b1481..ee957f270 100644 --- a/pkg/util/state/storage.go +++ b/pkg/util/state/storage.go @@ -19,7 +19,7 @@ var stateBucket = []byte("state") // NewPersistentStorage creates a new instance of a storage with 0600 rights. func NewPersistentStorage(path string) (*PersistentStorage, error) { - db, err := bbolt.Open(path, 0600, nil) + db, err := bbolt.Open(path, 0o600, nil) if err != nil { return nil, fmt.Errorf("can't open bbolt at %s: %w", path, err) } diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go index f29eca37c..63571e96d 100644 --- a/scripts/export-metrics/main.go +++ b/scripts/export-metrics/main.go @@ -45,7 +45,7 @@ func main() { os.Exit(1) } - if err := os.WriteFile(filename, data, 0644); err != nil { + if err := os.WriteFile(filename, data, 0o644); err != nil { fmt.Fprintf(os.Stderr, "Could write to file: %v\n", err) os.Exit(1) } From 990f9f2d2b5228503840d07e78b6c14815359cdc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 31 Oct 2023 16:59:38 +0300 Subject: [PATCH 0641/1943] [#772] makefile: Replace gofmt with gofumpt Signed-off-by: Dmitrii Stepanov --- Makefile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 7fa4e4317..14e32e40f 100755 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ OUTPUT_LINT_DIR ?= $(shell pwd)/bin LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION) TMP_DIR := .cache -.PHONY: help all images dep clean fmts fmt imports test lint docker/lint +.PHONY: help all images dep clean fmts fumpt imports test lint docker/lint prepare-release debpackage pre-commit unpre-commit # To build a specific binary, use it's name prefix with bin/ as a target @@ -122,18 +122,17 @@ docker/%: # Run all code formatters -fmts: fmt imports - -# Reformat code -fmt: - @echo "⇒ Processing gofmt check" - @gofmt -s -w cmd/ pkg/ misc/ +fmts: fumpt imports # Reformat imports imports: @echo "⇒ Processing goimports check" @goimports -w cmd/ pkg/ misc/ +fumpt: + @echo "⇒ Processing gofumpt check" + @gofumpt -l -w cmd/ pkg/ misc/ + # Run Unit Test with go test test: @echo "⇒ Running go test" From 8a82335b5c4844bf5758d5948a7b51304d0b362f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 31 Oct 2023 17:00:57 +0300 Subject: [PATCH 0642/1943] [#772] pre-commit: Add gofumpt check Signed-off-by: Dmitrii Stepanov --- .pre-commit-config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f20bdc84..ecd70390c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,6 +47,15 @@ repos: types: [go] language: system + - repo: local + hooks: + - id: gofumpt + name: gofumpt + entry: make fumpt + pass_filenames: false + types: [go] + language: system + - repo: https://github.com/TekWizely/pre-commit-golang rev: v1.0.0-rc.1 hooks: From 05b508f79a04dd8281b4137ee54313044b1b1a12 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 31 Oct 2023 17:01:27 +0300 Subject: [PATCH 0643/1943] [#772] proto: Fix file ending Signed-off-by: Dmitrii Stepanov --- pkg/services/control/ir/service.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/control/ir/service.proto b/pkg/services/control/ir/service.proto index 6c85ba4b9..cd890d438 100644 --- a/pkg/services/control/ir/service.proto +++ b/pkg/services/control/ir/service.proto @@ -93,4 +93,4 @@ message RemoveContainerResponse { Body body = 1; Signature signature = 2; -} \ No newline at end of file +} From c80b46fad351e4860c743614d32fa5e81ca8ecce Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 31 Oct 2023 14:45:22 +0300 Subject: [PATCH 0644/1943] [#754] blobstor: Estimate compressability Now it is possible to enable compressability estimation. If data is likely uncompressable, it should reduce CPU time and memory. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 9 +++- cmd/frostfs-node/config/cast.go | 12 +++++ cmd/frostfs-node/config/engine/config_test.go | 2 + .../config/engine/shard/config.go | 31 +++++++++++- config/example/node.env | 2 + config/example/node.json | 2 + config/example/node.yaml | 2 + docs/storage-node-configuration.md | 20 ++++---- pkg/local_object_storage/blobstor/blobstor.go | 21 ++++++++ .../blobstor/compression/bench_test.go | 49 +++++++++++++++++++ .../blobstor/compression/compress.go | 15 ++++++ 11 files changed, 153 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 8a7317231..6580de157 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -105,7 +105,10 @@ type applicationConfiguration struct { } type shardCfg struct { - compress bool + compress bool + estimateCompressibility bool + estimateCompressibilityThreshold float64 + smallSizeObjectLimit uint64 uncompressableContentType []string refillMetabase bool @@ -217,6 +220,8 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, oldConfig newConfig.refillMetabase = oldConfig.RefillMetabase() newConfig.mode = oldConfig.Mode() newConfig.compress = oldConfig.Compress() + newConfig.estimateCompressibility = oldConfig.EstimateCompressibility() + newConfig.estimateCompressibilityThreshold = oldConfig.EstimateCompressibilityThreshold() newConfig.uncompressableContentType = oldConfig.UncompressableContentTypes() newConfig.smallSizeObjectLimit = oldConfig.SmallSizeLimit() @@ -830,6 +835,8 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { blobstoreOpts := []blobstor.Option{ blobstor.WithCompressObjects(shCfg.compress), blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), + blobstor.WithCompressibilityEstimate(shCfg.estimateCompressibility), + blobstor.WithCompressibilityEstimateThreshold(shCfg.estimateCompressibilityThreshold), blobstor.WithStorages(ss), blobstor.WithLogger(c.log), } diff --git a/cmd/frostfs-node/config/cast.go b/cmd/frostfs-node/config/cast.go index 9036c3ab0..c99d33569 100644 --- a/cmd/frostfs-node/config/cast.go +++ b/cmd/frostfs-node/config/cast.go @@ -223,3 +223,15 @@ func parseSizeInBytes(sizeStr string) uint64 { size := cast.ToFloat64(sizeStr) return safeMul(size, multiplier) } + +// FloatOrDefault reads a configuration value +// from c by name and casts it to float64. +// +// Returns defaultValue if the value can not be casted. +func FloatOrDefault(c *Config, name string, defaultValue float64) float64 { + v, err := cast.ToFloat64E(c.Value(name)) + if err != nil { + return defaultValue + } + return v +} diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 4077b1744..6b7c268ce 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -84,6 +84,8 @@ func TestEngineSection(t *testing.T) { require.Equal(t, true, sc.Compress()) require.Equal(t, []string{"audio/*", "video/*"}, sc.UncompressableContentTypes()) + require.Equal(t, true, sc.EstimateCompressibility()) + require.Equal(t, float64(0.7), sc.EstimateCompressibilityThreshold()) require.EqualValues(t, 102400, sc.SmallSizeLimit()) require.Equal(t, 2, len(ss)) diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go index 1dc32fb86..16100c3a7 100644 --- a/cmd/frostfs-node/config/engine/shard/config.go +++ b/cmd/frostfs-node/config/engine/shard/config.go @@ -16,8 +16,11 @@ import ( // which provides access to Shard configurations. type Config config.Config -// SmallSizeLimitDefault is a default limit of small objects payload in bytes. -const SmallSizeLimitDefault = 1 << 20 +const ( + // SmallSizeLimitDefault is a default limit of small objects payload in bytes. + SmallSizeLimitDefault = 1 << 20 + EstimateCompressibilityThresholdDefault = 0.1 +) // From wraps config section into Config. func From(c *config.Config) *Config { @@ -43,6 +46,30 @@ func (x *Config) UncompressableContentTypes() []string { "compression_exclude_content_types") } +// EstimateCompressibility returns the value of "estimate_compressibility" config parameter. +// +// Returns false if the value is not a valid bool. +func (x *Config) EstimateCompressibility() bool { + return config.BoolSafe( + (*config.Config)(x), + "compression_estimate_compressibility", + ) +} + +// EstimateCompressibilityThreshold returns the value of "estimate_compressibility_threshold" config parameter. +// +// Returns EstimateCompressibilityThresholdDefault if the value is not defined, not valid float or not in range [0.0; 1.0]. +func (x *Config) EstimateCompressibilityThreshold() float64 { + v := config.FloatOrDefault( + (*config.Config)(x), + "compression_estimate_compressibility_threshold", + EstimateCompressibilityThresholdDefault) + if v < 0.0 || v > 1.0 { + return EstimateCompressibilityThresholdDefault + } + return v +} + // SmallSizeLimit returns the value of "small_object_size" config parameter. // // Returns SmallSizeLimitDefault if the value is not a positive number. diff --git a/config/example/node.env b/config/example/node.env index fde65173b..dda740cf1 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -113,6 +113,8 @@ FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_DELAY=10ms ### Blobstor config FROSTFS_STORAGE_SHARD_0_COMPRESS=true FROSTFS_STORAGE_SHARD_0_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*" +FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY=true +FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY_THRESHOLD=0.7 FROSTFS_STORAGE_SHARD_0_SMALL_OBJECT_SIZE=102400 ### Blobovnicza config FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH=tmp/0/blob/blobovnicza diff --git a/config/example/node.json b/config/example/node.json index e8455ee55..1038d5e5c 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -160,6 +160,8 @@ "compression_exclude_content_types": [ "audio/*", "video/*" ], + "compression_estimate_compressibility": true, + "compression_estimate_compressibility_threshold": 0.7, "small_object_size": 102400, "blobstor": [ { diff --git a/config/example/node.yaml b/config/example/node.yaml index 2ca1b426c..8b2046e95 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -178,6 +178,8 @@ storage: compression_exclude_content_types: - audio/* - video/* + compression_estimate_compressibility: true + compression_estimate_compressibility_threshold: 0.7 blobstor: - type: blobovnicza diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 2e2d04088..5e9f3caf7 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -179,15 +179,17 @@ The following table describes configuration for each shard. | Parameter | Type | Default value | Description | |-------------------------------------|---------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `compress` | `bool` | `false` | Flag to enable compression. | -| `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). | -| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` | -| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. | -| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. | -| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. | -| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | -| `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. | -| `gc` | [GC config](#gc-subsection) | | GC configuration. | +| `compress` | `bool` | `false` | Flag to enable compression. | +| `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). | +| `compression_estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. | +| `compression_estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. | +| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` | +| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. | +| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. | +| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. | +| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | +| `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. | +| `gc` | [GC config](#gc-subsection) | | GC configuration. | ### `blobstor` subsection diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index d2a2338a3..bc9ab2b99 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -107,6 +107,27 @@ func WithCompressObjects(comp bool) Option { } } +// WithCompressibilityEstimate returns an option to use +// normilized compressibility estimate to decide compress +// data or not. +// +// See https://github.com/klauspost/compress/blob/v1.17.2/compressible.go#L5 +func WithCompressibilityEstimate(v bool) Option { + return func(c *cfg) { + c.compression.UseCompressEstimation = v + } +} + +// WithCompressibilityEstimateThreshold returns an option to set +// normilized compressibility estimate threshold. +// +// See https://github.com/klauspost/compress/blob/v1.17.2/compressible.go#L5 +func WithCompressibilityEstimateThreshold(threshold float64) Option { + return func(c *cfg) { + c.compression.CompressEstimationThreshold = threshold + } +} + // WithUncompressableContentTypes returns option to disable decompression // for specific content types as seen by object.AttributeContentType attribute. func WithUncompressableContentTypes(values []string) Option { diff --git a/pkg/local_object_storage/blobstor/compression/bench_test.go b/pkg/local_object_storage/blobstor/compression/bench_test.go index 6e05366cf..986912985 100644 --- a/pkg/local_object_storage/blobstor/compression/bench_test.go +++ b/pkg/local_object_storage/blobstor/compression/bench_test.go @@ -3,8 +3,10 @@ package compression import ( "crypto/rand" "fmt" + "log" "testing" + "github.com/klauspost/compress" "github.com/stretchr/testify/require" ) @@ -47,3 +49,50 @@ func notSoRandomSlice(size, blockSize int) []byte { } return data } + +func BenchmarkCompressionRealVSEstimate(b *testing.B) { + var total float64 // to prevent from compiler optimizations + maxSize := 60 * 1024 * 1024 + b.Run("estimate", func(b *testing.B) { + b.ResetTimer() + + c := &Config{ + Enabled: true, + } + require.NoError(b, c.Init()) + + for size := 1024; size <= maxSize; size *= 2 { + data := make([]byte, size) + _, err := rand.Reader.Read(data) + require.NoError(b, err) + + b.StartTimer() + estimation := compress.Estimate(data) + total += estimation + b.StopTimer() + } + }) + + b.Run("compress", func(b *testing.B) { + b.ResetTimer() + + c := &Config{ + Enabled: true, + } + require.NoError(b, c.Init()) + + for size := 1024; size <= maxSize; size *= 2 { + data := make([]byte, size) + _, err := rand.Reader.Read(data) + require.NoError(b, err) + + b.StartTimer() + maxSize := c.encoder.MaxEncodedSize(len(data)) + compressed := c.encoder.EncodeAll(data, make([]byte, 0, maxSize)) + total += float64(len(compressed)) / float64(len(data)) + b.StopTimer() + } + }) + + log.Println(total) +} diff --git a/pkg/local_object_storage/blobstor/compression/compress.go b/pkg/local_object_storage/blobstor/compression/compress.go index 270c2b18d..85ab47692 100644 --- a/pkg/local_object_storage/blobstor/compression/compress.go +++ b/pkg/local_object_storage/blobstor/compression/compress.go @@ -5,6 +5,7 @@ import ( "strings" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "github.com/klauspost/compress" "github.com/klauspost/compress/zstd" ) @@ -13,6 +14,9 @@ type Config struct { Enabled bool UncompressableContentTypes []string + UseCompressEstimation bool + CompressEstimationThreshold float64 + encoder *zstd.Encoder decoder *zstd.Decoder } @@ -82,6 +86,17 @@ func (c *Config) Compress(data []byte) []byte { if c == nil || !c.Enabled { return data } + if c.UseCompressEstimation { + estimated := compress.Estimate(data) + if estimated >= c.CompressEstimationThreshold { + return c.compress(data) + } + return data + } + return c.compress(data) +} + +func (c *Config) compress(data []byte) []byte { maxSize := c.encoder.MaxEncodedSize(len(data)) compressed := c.encoder.EncodeAll(data, make([]byte, 0, maxSize)) if len(data) < len(compressed) { From a26483fc30b29a57aa07b8dad0afe43252280cb5 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 27 Oct 2023 15:02:14 +0300 Subject: [PATCH 0645/1943] [#749] morph: Fix panic when closing morph client Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/morph.go | 4 +++ pkg/morph/client/client.go | 2 ++ pkg/morph/client/multi.go | 2 ++ pkg/morph/client/notifications.go | 5 ++- pkg/morph/event/listener.go | 54 +++++++++++++++++++++---------- 5 files changed, 49 insertions(+), 18 deletions(-) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index edb1412aa..f7100d0bf 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -207,6 +207,10 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { }) fatalOnErr(err) + c.onShutdown(func() { + lis.Stop() + }) + c.workers = append(c.workers, newWorkerFromFunc(func(wCtx context.Context) { runAndLog(wCtx, c, "morph notification", false, func(lCtx context.Context, c *cfg) { lis.ListenWithError(lCtx, c.internalErr) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 290e651f7..050838648 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -73,6 +73,8 @@ type Client struct { // channel for internal stop closeChan chan struct{} + closed atomic.Bool + wg sync.WaitGroup // indicates that Client is not able to // establish connection to any of the diff --git a/pkg/morph/client/multi.go b/pkg/morph/client/multi.go index e006ca69a..25ec626d1 100644 --- a/pkg/morph/client/multi.go +++ b/pkg/morph/client/multi.go @@ -75,6 +75,8 @@ func (c *Client) SwitchRPC(ctx context.Context) bool { } func (c *Client) closeWaiter(ctx context.Context) { + c.wg.Add(1) + defer c.wg.Done() select { case <-ctx.Done(): case <-c.closeChan: diff --git a/pkg/morph/client/notifications.go b/pkg/morph/client/notifications.go index 121dccfb7..a013631df 100644 --- a/pkg/morph/client/notifications.go +++ b/pkg/morph/client/notifications.go @@ -16,7 +16,10 @@ func (c *Client) Close() { // closing should be done via the channel // to prevent switching to another RPC node // in the notification loop - close(c.closeChan) + if c.closed.CompareAndSwap(false, true) { + close(c.closeChan) + } + c.wg.Wait() } // ReceiveExecutionNotifications performs subscription for notifications diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index ca5031415..4fb92da50 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -96,6 +96,8 @@ type ListenerParams struct { type listener struct { mtx sync.RWMutex + wg sync.WaitGroup + startOnce, stopOnce sync.Once started bool @@ -124,6 +126,12 @@ var ( errNilLogger = errors.New("nil logger") errNilSubscriber = errors.New("nil event subscriber") + + errNotificationSubscrConnectionTerminated = errors.New("event subscriber connection has been terminated") + + errNotarySubscrConnectionTerminated = errors.New("notary event subscriber connection has been terminated") + + errBlockNotificationChannelClosed = errors.New("new block notification channel is closed") ) // Listen starts the listening for events with registered handlers. @@ -133,6 +141,8 @@ var ( // Returns an error if listener was already started. func (l *listener) Listen(ctx context.Context) { l.startOnce.Do(func() { + l.wg.Add(1) + defer l.wg.Done() if err := l.listen(ctx, nil); err != nil { l.log.Error(logs.EventCouldNotStartListenToEvents, zap.String("error", err.Error()), @@ -149,11 +159,13 @@ func (l *listener) Listen(ctx context.Context) { // Returns an error if listener was already started. func (l *listener) ListenWithError(ctx context.Context, intError chan<- error) { l.startOnce.Do(func() { + l.wg.Add(1) + defer l.wg.Done() if err := l.listen(ctx, intError); err != nil { l.log.Error(logs.EventCouldNotStartListenToEvents, zap.String("error", err.Error()), ) - intError <- err + l.sendError(ctx, intError, err) } }) } @@ -172,6 +184,8 @@ func (l *listener) listen(ctx context.Context, intError chan<- error) error { } func (l *listener) subscribe(errCh chan error) { + l.wg.Add(1) + defer l.wg.Done() // create the list of listening contract hashes hashes := make([]util.Uint160, 0) @@ -212,6 +226,23 @@ func (l *listener) subscribe(errCh chan error) { } } +func (l *listener) sendError(ctx context.Context, intErr chan<- error, err error) bool { + if intErr == nil { + return false + } + // This select required because were are reading from error channel and closing listener + // in the same routine when shutting down node. + select { + case <-ctx.Done(): + l.log.Info(logs.EventStopEventListenerByContext, + zap.String("reason", ctx.Err().Error()), + ) + return false + case intErr <- err: + return true + } +} + func (l *listener) listenLoop(ctx context.Context, intErr chan<- error, subErrCh chan error) { chs := l.subscriber.NotificationChannels() @@ -219,12 +250,9 @@ loop: for { select { case err := <-subErrCh: - if intErr != nil { - intErr <- err - } else { + if !l.sendError(ctx, intErr, err) { l.log.Error(logs.EventStopEventListenerByError, zap.Error(err)) } - break loop case <-ctx.Done(): l.log.Info(logs.EventStopEventListenerByContext, @@ -234,10 +262,7 @@ loop: case notifyEvent, ok := <-chs.NotificationsCh: if !ok { l.log.Warn(logs.EventStopEventListenerByNotificationChannel) - if intErr != nil { - intErr <- errors.New("event subscriber connection has been terminated") - } - + l.sendError(ctx, intErr, errNotificationSubscrConnectionTerminated) break loop } else if notifyEvent == nil { l.log.Warn(logs.EventNilNotificationEventWasCaught) @@ -248,10 +273,7 @@ loop: case notaryEvent, ok := <-chs.NotaryRequestsCh: if !ok { l.log.Warn(logs.EventStopEventListenerByNotaryChannel) - if intErr != nil { - intErr <- errors.New("notary event subscriber connection has been terminated") - } - + l.sendError(ctx, intErr, errNotarySubscrConnectionTerminated) break loop } else if notaryEvent == nil { l.log.Warn(logs.EventNilNotaryEventWasCaught) @@ -262,10 +284,7 @@ loop: case b, ok := <-chs.BlockCh: if !ok { l.log.Warn(logs.EventStopEventListenerByBlockChannel) - if intErr != nil { - intErr <- errors.New("new block notification channel is closed") - } - + l.sendError(ctx, intErr, errBlockNotificationChannelClosed) break loop } else if b == nil { l.log.Warn(logs.EventNilBlockWasCaught) @@ -603,6 +622,7 @@ func (l *listener) Stop() { l.stopOnce.Do(func() { l.subscriber.Close() }) + l.wg.Wait() } func (l *listener) RegisterBlockHandler(handler BlockHandler) { From c7a72294840e5a932b7f2637ddbb5dfde16a803a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 31 Oct 2023 15:57:06 +0300 Subject: [PATCH 0646/1943] [#764] metrics: Fix epoch metric Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 6580de157..e3e56e5de 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -562,10 +562,11 @@ func initCfg(appCfg *config.Config) *cfg { relayOnly := nodeconfig.Relay(appCfg) netState := newNetworkState() - netState.metrics = c.metricsCollector c.shared = initShared(appCfg, key, netState, relayOnly) + netState.metrics = c.metricsCollector + logPrm, err := c.loggerPrm() fatalOnErr(err) logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook() From 0b0e5dab2435c046ff5da467cbec586df8b00a17 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 1 Nov 2023 11:07:56 +0300 Subject: [PATCH 0647/1943] [#756] adm: Add polling interval increase Signed-off-by: Dmitrii Stepanov --- .../internal/modules/morph/initialize.go | 60 ++++++++++++------- .../internal/modules/morph/initialize_test.go | 35 +++++++++++ 2 files changed, 72 insertions(+), 23 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 4184e1a80..e288361dd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -7,6 +7,7 @@ import ( "path/filepath" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" @@ -376,36 +377,18 @@ func (c *clientContext) awaitTx(cmd *cobra.Command) error { func awaitTx(cmd *cobra.Command, c Client, txs []hashVUBPair) error { cmd.Println("Waiting for transactions to persist...") - const pollInterval = time.Second - - tick := time.NewTicker(pollInterval) - defer tick.Stop() - at := trigger.Application var retErr error - currBlock, err := c.GetBlockCount() - if err != nil { - return fmt.Errorf("can't fetch current block height: %w", err) - } - loop: for i := range txs { - res, err := c.GetApplicationLog(txs[i].hash, &at) - if err == nil { - if retErr == nil && len(res.Executions) > 0 && res.Executions[0].VMState != vmstate.Halt { - retErr = fmt.Errorf("tx %d persisted in %s state: %s", - i, res.Executions[0].VMState, res.Executions[0].FaultException) - } - continue loop - } - if txs[i].vub < currBlock { - return fmt.Errorf("tx was not persisted: vub=%d, height=%d", txs[i].vub, currBlock) - } - for range tick.C { + var it int + var pollInterval time.Duration + var pollIntervalChanged bool + for { // We must fetch current height before application log, to avoid race condition. - currBlock, err = c.GetBlockCount() + currBlock, err := c.GetBlockCount() if err != nil { return fmt.Errorf("can't fetch current block height: %w", err) } @@ -420,12 +403,43 @@ loop: if txs[i].vub < currBlock { return fmt.Errorf("tx was not persisted: vub=%d, height=%d", txs[i].vub, currBlock) } + + pollInterval, pollIntervalChanged = nextPollInterval(it, pollInterval) + if pollIntervalChanged && viper.GetBool(commonflags.Verbose) { + cmd.Printf("Pool interval to check transaction persistence changed: %s\n", pollInterval.String()) + } + + timer := time.NewTimer(pollInterval) + select { + case <-cmd.Context().Done(): + return cmd.Context().Err() + case <-timer.C: + } + + it++ } } return retErr } +func nextPollInterval(it int, previous time.Duration) (time.Duration, bool) { + const minPollInterval = 1 * time.Second + const maxPollInterval = 16 * time.Second + const changeAfter = 5 + if it == 0 { + return minPollInterval, true + } + if it%changeAfter != 0 { + return previous, false + } + nextInterval := previous * 2 + if nextInterval > maxPollInterval { + return maxPollInterval, previous != maxPollInterval + } + return nextInterval, true +} + // sendCommitteeTx creates transaction from script, signs it by committee nodes and sends it to RPC. // If tryGroup is false, global scope is used for the signer (useful when // working with native contracts). diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index 39da56662..613c63894 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -145,3 +145,38 @@ func setTestCredentials(v *viper.Viper, size int) { } v.Set("credentials.contract", testContractPassword) } + +func TestNextPollInterval(t *testing.T) { + var pollInterval time.Duration + var iteration int + + pollInterval, hasChanged := nextPollInterval(iteration, pollInterval) + require.True(t, hasChanged) + require.Equal(t, time.Second, pollInterval) + + iteration = 4 + pollInterval, hasChanged = nextPollInterval(iteration, pollInterval) + require.False(t, hasChanged) + require.Equal(t, time.Second, pollInterval) + + iteration = 5 + pollInterval, hasChanged = nextPollInterval(iteration, pollInterval) + require.True(t, hasChanged) + require.Equal(t, 2*time.Second, pollInterval) + + iteration = 10 + pollInterval, hasChanged = nextPollInterval(iteration, pollInterval) + require.True(t, hasChanged) + require.Equal(t, 4*time.Second, pollInterval) + + iteration = 20 + pollInterval = 32 * time.Second + pollInterval, hasChanged = nextPollInterval(iteration, pollInterval) + require.True(t, hasChanged) // from 32s to 16s + require.Equal(t, 16*time.Second, pollInterval) + + pollInterval = 16 * time.Second + pollInterval, hasChanged = nextPollInterval(iteration, pollInterval) + require.False(t, hasChanged) + require.Equal(t, 16*time.Second, pollInterval) +} From 1ab567870a50ad24f3feff7e25f5ba057786c7b7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 2 Nov 2023 10:04:02 +0300 Subject: [PATCH 0648/1943] [#779] adm: Support deploying policy contract Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 350cd611c..8d46793e5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -42,6 +42,7 @@ const ( containerContract = "container" frostfsIDContract = "frostfsid" netmapContract = "netmap" + policyContract = "policy" proxyContract = "proxy" ) @@ -51,6 +52,7 @@ var ( containerContract, frostfsIDContract, netmapContract, + policyContract, proxyContract, } @@ -568,7 +570,7 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an c.Contracts[containerContract].Hash, keysParam, configParam) - case proxyContract: + case proxyContract, policyContract: items = nil default: panic(fmt.Sprintf("invalid contract name: %s", ctrName)) From 7c8591f83baf07997d914ca43cb092614aa6d02f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 2 Nov 2023 10:09:04 +0300 Subject: [PATCH 0649/1943] [#779] go.mod: Update frostfs-contract Signed-off-by: Evgenii Stratonikov --- go.mod | 6 +- go.sum | 1698 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 1679 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 0c6731363..bebb9346a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0 - git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.0 + git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231027075445-a02c0bfac885 git.frostfs.info/TrueCloudLab/hrw v1.2.1 @@ -43,11 +43,11 @@ require ( ) require ( - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/flatbuffers v1.12.1 // indirect + github.com/google/flatbuffers v2.0.8+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect go.opencensus.io v0.24.0 // indirect ) diff --git a/go.sum b/go.sum index 6fb57f4f4..22f42ef85 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -35,47 +37,102 @@ cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34h cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= +cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= +cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= +cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= +cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= +cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps= +cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= +cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= +cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= +cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= +cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= +cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= +cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= +cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= +cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= +cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= +cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= +cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= +cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= +cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= +cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= +cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= +cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= +cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= +cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= +cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= +cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= +cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= +cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= +cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/baremetalsolution v1.1.1/go.mod h1:D1AV6xwOksJMV4OSlWHtWuFNZZYujJknMAP4Qa27QIA= cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/batch v1.3.1/go.mod h1:VguXeQKXIYaeeIYbuozUmBR13AfL4SJP7IltNPS+A4A= cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= +cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= +cloud.google.com/go/beyondcorp v1.0.0/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -85,26 +142,50 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= +cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= +cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= +cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= +cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= +cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= +cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= +cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= +cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= +cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= +cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= +cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= +cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= +cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= +cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= +cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -117,224 +198,437 @@ cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= +cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= +cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= +cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= +cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= +cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= +cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= +cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= +cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= +cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= +cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= +cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= +cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= +cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= +cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= +cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= +cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= +cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= +cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= +cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= +cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= +cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= +cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= +cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= +cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= +cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= +cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= +cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= +cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= +cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= +cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= +cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= +cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= +cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= +cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= +cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= +cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= +cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= +cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= +cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= +cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= +cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= +cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= +cloud.google.com/go/gkebackup v1.3.0/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= +cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= +cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= +cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= +cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= +cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= +cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= +cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= +cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= +cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= +cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= +cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= +cloud.google.com/go/kms v1.11.0/go.mod h1:hwdiYC0xjnWsKQQCQQmIQnS9asjYVSK6jtXm+zFqXLM= +cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= +cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= +cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= +cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/longrunning v0.4.2/go.mod h1:OHrnaYyLUV6oqwh0xiS7e5sLQhP1m0QU9R+WhGDMgIQ= +cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= +cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= +cloud.google.com/go/maps v1.3.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= +cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= +cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= +cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= +cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= +cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= +cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= +cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= +cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= +cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= +cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= +cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= +cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= +cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= +cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= +cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= +cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= +cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= +cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= +cloud.google.com/go/orgpolicy v1.11.0/go.mod h1:2RK748+FtVvnfuynxBzdnyu7sygtoZa1za/0ZfpOs1M= +cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= +cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= +cloud.google.com/go/osconfig v1.12.0/go.mod h1:8f/PaYzoS3JMVfdfTubkowZYGmAhUCjjwnjqWI7NVBc= +cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= +cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= +cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= +cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= +cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= +cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= +cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= +cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= +cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= +cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= +cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= +cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= +cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= +cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= +cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= +cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= +cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= +cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= +cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= +cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= +cloud.google.com/go/run v1.2.0/go.mod h1:36V1IlDzQ0XxbQjUx6IYbw8H3TJnWvhii963WW3B/bo= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= +cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= +cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= +cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= +cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= +cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= +cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= +cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= +cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= +cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= +cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= +cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= +cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= +cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= +cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= +cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= +cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= +cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -344,51 +638,96 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= +cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= +cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= +cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= +cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= +cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= +cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= +cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= +cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= +cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= +cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= +cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= +cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= +cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= +cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= +cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= +cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= +cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= +cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= +cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= +cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= +cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= +cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= +cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= +cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0 h1:vy6leTEGcKVrLmKfeK5pGGIi3D1vn6rX32Hy4gIOWto= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.0 h1:9ahw69njrwf2legz5xNVTA+4A6UwcBzy9oBdbJmoBxU= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.0/go.mod h1:3V8FyzpbIIxzpgfUaSlOJBAT11IzhZzkQnGpYvRQR5E= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989 h1:hcWlSHcgkg2wnqAInu0AGQcdEY/zr7+BDbGK0rDfXLQ= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989/go.mod h1:PxsAxe6J4hyYJoU6hyYxD9q5K/DFxdquKHL1BrRaGkg= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= @@ -401,38 +740,143 @@ git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9m git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20221206110420-d395f97c4830/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1:MJCkWUBhi9pn/CrYO1Q3P687y2KeahrOPS9BD9LDGb0= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= +github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= +github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= +github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= +github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= +github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= +github.com/Microsoft/hcsshim v0.9.6/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= +github.com/Microsoft/hcsshim v0.10.0-rc.8/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= +github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= github.com/abiosoft/ishell/v2 v2.0.2/go.mod h1:E4oTCXfo6QjoCart0QYa5m9w4S+deXs/P/9jA77A9Bs= github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= +github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -445,19 +889,37 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytecodealliance/wasmtime-go v0.36.0/go.mod h1:q320gUxqyI8yB+ZqRuaJOEnGkAnHh6WtJjMaT2CW4wI= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= +github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= @@ -467,6 +929,13 @@ github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObk github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= +github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= +github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= +github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -480,33 +949,242 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark-crypto v0.9.1/go.mod h1:a2DQL4+5ywF6safEeZFEPGRiiGbjzGFRUN2sg06VuU4= +github.com/consensys/gnark v0.9.1/go.mod h1:udWvWGXnfBE7mn7BsNoGAvZDnUhcONBEtNijvVjfY80= github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb h1:f0BMgIjhZy4lSRHCXFbQst85f5agZAjtDMixQqBWNpc= github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/container-orchestrated-devices/container-device-interface v0.5.4/go.mod h1:DjE95rfPiiSmG7uVXtg0z6MnPm/Lx4wxKCIts0ZE0vg= +github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/btrfs/v2 v2.0.0/go.mod h1:swkD/7j9HApWpzl8OHfrHNxppPd9l44DFZdF94BUj9k= +github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= +github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= +github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= +github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= +github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= +github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= +github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE= +github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0= +github.com/containerd/containerd v1.6.8/go.mod h1:By6p5KqPK0/7/CgO/A6t/Gz+CUYUu2zf1hUaaymVXB0= +github.com/containerd/containerd v1.6.9/go.mod h1:XVicUvkxOrftE2Q1YWUXgZwkkAxwQYNOFzYWvfVfEfQ= +github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= +github.com/containerd/containerd v1.7.3/go.mod h1:32FOM4/O0RkNg7AjQj3hDzN9cUGtu+HMvaKUNiqCZB8= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= +github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= +github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= +github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/containerd/continuity v0.4.1/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= +github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= +github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= +github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= +github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= +github.com/containerd/go-cni v1.1.6/go.mod h1:BWtoWl5ghVymxu6MBjg79W9NZrCRyHIdUtk4cauMe34= +github.com/containerd/go-cni v1.1.9/go.mod h1:XYrZJ1d5W6E2VOvjffL3IZq0Dz6bsVlERHbekNK90PM= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= +github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= +github.com/containerd/imgcrypt v1.1.4/go.mod h1:LorQnPtzL/T0IyCeftcsMEO7AqxUDbdO8j/tSUpgxvo= +github.com/containerd/imgcrypt v1.1.7/go.mod h1:FD8gqIcX5aTotCtOmjeCsi3A1dHmTZpnMISGKSczt4k= +github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/nri v0.3.0/go.mod h1:Zw9q2lP16sdg0zYybemZ9yTDy8g7fPCIB3KXOGlggXI= +github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= +github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= +github.com/containerd/ttrpc v1.1.1-0.20220420014843-944ef4a40df3/go.mod h1:YYyNVhZrTMiaf51Vj6WhAJqJw+vl/nzABhj8pWrzle4= +github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= +github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= +github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= +github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v1.1.0/go.mod h1:oZF9wBnrnQjpWLaPKEinrx3TQ9a+W/RJO7Zb41d8YLE= +github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= +github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= +github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= +github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= +github.com/containernetworking/plugins v1.2.0/go.mod h1:/VjX4uHecW5vVimFa1wkG4s+r/s9qIfPdqlLF4TW8c4= +github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= +github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= +github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/containers/ocicrypt v1.1.3/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g= +github.com/containers/ocicrypt v1.1.6/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= +github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= +github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= +github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= +github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M= github.com/dgraph-io/badger/v4 v4.1.0 h1:E38jc0f+RATYrycSUf9LMv/t47XAy+3CApyYSq4APOQ= github.com/dgraph-io/badger/v4 v4.1.0/go.mod h1:P50u28d39ibBRmIJuQC/NSdBOg46HnHw7al2SW5QRHg= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/distribution/distribution/v3 v3.0.0-20220526142353-ffbd94cbe269/go.mod h1:28YO/VJk9/64+sTGNuYaBjWxrXTPrj0C0XmgTIOjxX4= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v23.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= +github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v23.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v23.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= +github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -517,50 +1195,137 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= +github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/foxcpp/go-mockdns v0.0.0-20210729171921-fb145fc6f897/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= +github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ini/ini v1.66.6/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -599,11 +1364,16 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.12.6/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -620,11 +1390,16 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= +github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -637,11 +1412,18 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= +github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -650,6 +1432,8 @@ github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -660,20 +1444,59 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 h1:mdLirNAJBxnGgyB6pjZLcs6ue/6eZGBui6gXspfq4ks= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0/go.mod h1:kdXbOySqcQeTxiqglW7aahTmWZy3Pgi6SYL36yvKeyA= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5 h1:3IZOAnD058zZllQTZNBioTlrzrBG/IjpiZ133IEtusM= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5/go.mod h1:xbKERva94Pw2cPen0s79J3uXmGzbbpDYFBFDlZ4mV/w= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -683,6 +1506,10 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/holiman/uint256 v1.2.2 h1:TXKcSGc2WaxPD2+bmzAsVthL4+pEN0YwXcL5qED83vk= github.com/holiman/uint256 v1.2.2/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= @@ -690,13 +1517,34 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= +github.com/intel/goresctrl v0.3.0/go.mod h1:fdz3mD85cmP9sHD8JUlrNWAxvwM86CrbmVXltEKd7zk= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -706,71 +1554,162 @@ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= +github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= +github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= +github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= +github.com/lestrrat-go/jwx v1.2.25/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xwxBBK1NxY= +github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mistifyio/go-zfs/v3 v3.0.1/go.mod h1:CzVgeB0RvF2EGzQnytKVvVSDwmKJXxkOTUGbNrTja/k= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= +github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= +github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= @@ -783,8 +1722,11 @@ github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7B github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 h1:vU9tpM3apjYlLLeY23zRWJ9Zktr5jp+mloR942LEOpY= github.com/nats-io/nats-server/v2 v2.7.4 h1:c+BZJ3rGzUKCBIM4IXO8uNT2u1vajGbD1kPA6wqCEaM= github.com/nats-io/nats-server/v2 v2.7.4/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc= @@ -794,12 +1736,14 @@ github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/networkplumbing/go-nft v0.2.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= -github.com/nspcc-dev/dbft v0.0.0-20220629112714-fd49ca59d354/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= github.com/nspcc-dev/dbft v0.0.0-20221020093431-31c1bbdc74f2/go.mod h1:g9xisXmX9NP9MjioaTe862n9SlZTrP+6PVUWLBYOr98= github.com/nspcc-dev/dbft v0.0.0-20230515113611-25db6ba61d5c/go.mod h1:kjBC9F8L25GR+kIHy/1KgG/KfcoGnVwIiyovgq1uszk= github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= @@ -808,58 +1752,144 @@ github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= -github.com/nspcc-dev/neo-go v0.99.2/go.mod h1:9P0yWqhZX7i/ChJ+zjtiStO1uPTolPFUM+L5oNznU8E= github.com/nspcc-dev/neo-go v0.100.1/go.mod h1:Nnp7F4e9IBccsgtCeLtUWV+0T6gk1PtP5HRtA13hUfc= -github.com/nspcc-dev/neo-go v0.101.5-0.20230808195420-5fc61be5f6c5/go.mod h1:Z0kpjwnTJj/ik/X6z18xjCL0X2+RNbqlnhKrl+MYgP8= github.com/nspcc-dev/neo-go v0.103.0 h1:UVyWPhzZdfYFG35ORP3FRDLh8J/raRQ6m8SptDdlgfM= github.com/nspcc-dev/neo-go v0.103.0/go.mod h1:x+wmcYqpZYJwLp1l/pHZrqNp3RSWlkMymWGDij3/OPo= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20220809123759-3094d3e0c14b/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230420112658-c50ab951645a/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230808195420-5fc61be5f6c5/go.mod h1:ZUuXOkdtHZgaC13za/zMgXfQFncZ0jLzfQTe+OsDOtg= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231020160724-c3955f87d1b5 h1:09CpI5uwsxb1EeFPIKQRwwWlfCmDD/Dwwh01lPiQScM= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231020160724-c3955f87d1b5/go.mod h1:J/Mk6+nKeKSW4wygkZQFLQ6SkLOSGX5Ga0RuuuktEag= github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= github.com/nspcc-dev/neofs-api-go/v2 v2.14.0/go.mod h1:DRIr0Ic1s+6QgdqmNFNLIqMqd7lNMJfYwkczlm1hDtM= -github.com/nspcc-dev/neofs-contract v0.16.0/go.mod h1:gN5bo2TlMvLbySImmg76DVj3jVmYgti2VVlQ+h/tcr0= github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.9/go.mod h1:fTsdTU/M9rvv/f9jlp7vHOm3DRp+NSfjfTv9NohrKTE= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11/go.mod h1:W+ImTNRnSNMH8w43H1knCcIqwu7dLHePXtlJNZ7EFIs= github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/tzhash v1.7.0/go.mod h1:Dnx9LUlOLr5paL2Rtc96x0PPs8D9eIkUtowt1n+KQus= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.24.2 h1:J/tulyYK6JwBldPViHJReihxxZ+22FHs0piGjQAvoUE= +github.com/onsi/gomega v1.24.2/go.mod h1:gs3J10IS7Z7r7eXRoNJIrNqU4ToQukCJhFtKrWgHWnk= +github.com/open-policy-agent/opa v0.42.2/go.mod h1:MrmoTi/BsKWT58kXlVayBb+rYVeaMwuBm3nYAN3923s= +github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= +github.com/opencontainers/runc v1.1.8/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.1.0-rc.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI= +github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= +github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/panjf2000/ants/v2 v2.7.5 h1:/vhh0Hza9G1vP1PdCj9hl6MUzCRbmtcTJL0OsnmytuU= github.com/panjf2000/ants/v2 v2.7.5/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulmach/orb v0.9.2 h1:p/YWV2uJwamAynnDOJGNbPBVtDHj3vG51k9tR1rFwJE= github.com/paulmach/orb v0.9.2/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= +github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -867,76 +1897,160 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/safchain/ethtool v0.2.0/go.mod h1:WkKB1DnNtvsMlDmQ50sgwowDJV/hGbJSOvJoEXs1AJQ= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -944,31 +2058,75 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tchap/go-patricia/v2 v2.3.1/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= +github.com/testcontainers/testcontainers-go v0.22.0/go.mod h1:k0YiPa26xJCRUbUkYqy5rY6NGvSbVCeUBXCvucscBR4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= +github.com/vektah/gqlparser/v2 v2.4.5/go.mod h1:flJWIR04IMQPGz+BXLrORkrARBxv/rtyIAFvd/MceW0= +github.com/veraison/go-cose v1.0.0-rc.1/go.mod h1:7ziE85vSq4ScFTg6wyoMXjucIGOf4JkFEZi/an96Ct4= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= +github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yashtewari/glob-intersection v0.1.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -979,14 +2137,37 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/pkg/v3 v3.5.5/go.mod h1:6ksYFxttiUGzC2uxyqiyOEvhAiD0tuIqSZkX3TyPdaE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/raft/v3 v3.5.5/go.mod h1:76TA48q03g1y1VpTue92jZLr9lIHKUNcYdZOOGyx8rI= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +go.etcd.io/etcd/server/v3 v3.5.5/go.mod h1:rZ95vDw/jrvsbj9XpTqPrTAB9/kzchVdhRirySPkUBc= go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE= go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -996,68 +2177,156 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.40.0/go.mod h1:UMklln0+MRhZC4e3PwmN3pCtq4DyIadWw4yikh6bNrw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.32.0/go.mod h1:5eCOqeGphOyz6TsY3ZDNjE33SM/TFAK3RGuCL2naTgY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= +go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= +go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= +go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= +go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= +go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0/go.mod h1:M1hVZHNxcbkAlcvrOMlpQ4YOO3Awf+4N2dxkZL3xm04= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0/go.mod h1:ceUgdyfNv4h4gLxHR0WNfDiiVmZFodZhZSbOLhpxqXE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0/go.mod h1:E+/KKhwOSw8yoPxSSuUHG6vKppkvhN+S1Jc7Nib3k3o= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0/go.mod h1:+N7zNjIJv4K+DeX67XXET0P+eIciESgaFDBqh+ZJFS4= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 h1:+XWJd3jf75RXJq29mxbuXhCXFDG3S3R4vBUeSI2P7tE= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0/go.mod h1:hqgzBPTf4yONMFgdZvL/bK42R/iinTyVQtiWihs3SZc= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/metric v0.30.0/go.mod h1:/ShZ7+TS4dHzDFmfi1kSXMhMVubNoP0oIaBp70J6UXU= +go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= +go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s= go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= +go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= +go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU= +go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= +go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= +go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= +go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= +go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= +go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= +go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= +go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.16.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.20.0 h1:BLOA1cZBAGSbRiNuGCCKiFrCdYB7deeHDeD1SueyOfA= go.opentelemetry.io/proto/otlp v0.20.0/go.mod h1:3QgjzPALBIv9pcknj2EXGPXjYPFdUh/RQfF8Lz3+Vnw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -1065,11 +2334,25 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= -golang.org/x/exp v0.0.0-20221227203929-1b447090c38c/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1099,23 +2382,38 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1132,20 +2430,27 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -1161,8 +2466,15 @@ golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1191,6 +2503,10 @@ golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1204,14 +2520,21 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1220,25 +2543,43 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1249,25 +2590,41 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1276,10 +2633,19 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1288,16 +2654,23 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1305,17 +2678,31 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1332,31 +2719,55 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1376,18 +2787,23 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1398,10 +2814,17 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.11.1/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1410,8 +2833,16 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= +google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1463,6 +2894,17 @@ google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91 google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjYK+5E= +google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= +google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= +google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1470,11 +2912,13 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1483,6 +2927,7 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -1491,17 +2936,21 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1533,6 +2982,7 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= @@ -1576,21 +3026,64 @@ google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= +google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108= google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a h1:fwgW9j3vHirt4ObdHoYNwuO24BEZjSzbh+zPaNWoiY8= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= +google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1615,6 +3108,7 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -1625,7 +3119,13 @@ google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= @@ -1644,20 +3144,37 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1668,9 +3185,16 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= +gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1678,10 +3202,140 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= +k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= +k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= +k8s.io/api v0.26.2/go.mod h1:1kjMQsFE+QHPfskEcVNgL3+Hp88B80uj0QtSOlj8itU= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= +k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= +k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0= +k8s.io/apimachinery v0.26.2/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= +k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= +k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ= +k8s.io/apiserver v0.26.2/go.mod h1:GHcozwXgXsPuOJ28EnQ/jXEM9QeG6HT22YxSNmpYNh8= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= +k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= +k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= +k8s.io/client-go v0.26.2/go.mod h1:u5EjOuSyBa09yqqyY7m3abZeovO/7D/WehVVlZ2qcqU= +k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= +k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= +k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= +k8s.io/component-base v0.26.2/go.mod h1:DxbuIe9M3IZPRxPIzhch2m1eT7uFrSBJUBuVCQEBivs= +k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= +k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= +k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= +k8s.io/cri-api v0.25.0/go.mod h1:J1rAyQkSJ2Q6I+aBMOVgg2/cbbebso6FNa0UagiR0kc= +k8s.io/cri-api v0.25.3/go.mod h1:riC/P0yOGUf2K1735wW+CXs1aY2ctBgePtnnoFLd0dU= +k8s.io/cri-api v0.27.1/go.mod h1:+Ts/AVYbIo04S86XbTD73UPp/DkTiYxtsFeOFEu32L0= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kms v0.26.2/go.mod h1:69qGnf1NsFOQP07fBYqNLZklqEHSJF024JqYCaeVxHg= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= +modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= +modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= +modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= +modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA= +modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0= +modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= +modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= +modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= +oras.land/oras-go v1.2.0/go.mod h1:pFNs7oHp2dYsYMSS82HaX5l4mpnGO7hbpPN6EWH2ltc= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 7b1eda5107afdcae552bf9ccffeffee2e9f585f4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 2 Nov 2023 17:32:41 +0300 Subject: [PATCH 0650/1943] [#531] go.mod: Update api-go version Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bebb9346a..9e97238c5 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.20 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231027075445-a02c0bfac885 diff --git a/go.sum b/go.sum index 22f42ef85..80b600167 100644 --- a/go.sum +++ b/go.sum @@ -724,8 +724,8 @@ cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvo dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0 h1:vy6leTEGcKVrLmKfeK5pGGIi3D1vn6rX32Hy4gIOWto= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd h1:MxiIXu4vUATX9bNYFnM1dXAYkx5dnPOGCMzwNNXJEvM= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989 h1:hcWlSHcgkg2wnqAInu0AGQcdEY/zr7+BDbGK0rDfXLQ= git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989/go.mod h1:PxsAxe6J4hyYJoU6hyYxD9q5K/DFxdquKHL1BrRaGkg= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= From 20d6132f313f1fed011d18325cf6f2265f823ceb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 2 Nov 2023 17:34:33 +0300 Subject: [PATCH 0651/1943] [#531] signSvc: Add SetMarshaledData method call To reduce memory allocations add `SetMarshaledData` method call to return already marshalled data in next `StableMarshal` calls. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/sign.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index faad26489..2f4f9cb9a 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -122,6 +122,7 @@ func (s *SignService) Head(ctx context.Context, req *object.HeadRequest) (*objec } func (s *SignService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + req.GetBody().SetMarshalData(nil) if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.PutSingleResponse) return resp, s.sigSvc.SignResponse(resp, err) From cae50ecb21e91e32143ee5da65069430ed8ddfd4 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 2 Nov 2023 16:20:37 +0300 Subject: [PATCH 0652/1943] [#716] adm: Add dump policy Signed-off-by: Alexander Chuprov --- .../internal/modules/morph/policy.go | 33 +++++++++++++++++++ .../internal/modules/morph/root.go | 15 +++++++++ 2 files changed, 48 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/policy.go b/cmd/frostfs-adm/internal/modules/morph/policy.go index 0703ebc2d..005a02711 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy.go @@ -1,11 +1,15 @@ package morph import ( + "bytes" "fmt" "strconv" "strings" + "text/tabwriter" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/policy" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/vm/emit" @@ -52,3 +56,32 @@ func setPolicyCmd(cmd *cobra.Command, args []string) error { return wCtx.awaitTx() } + +func dumpPolicyCmd(cmd *cobra.Command, _ []string) error { + c, err := getN3Client(viper.GetViper()) + commonCmd.ExitOnErr(cmd, "can't create N3 client:", err) + + inv := invoker.New(c, nil) + policyContract := policy.NewReader(inv) + + execFee, err := policyContract.GetExecFeeFactor() + commonCmd.ExitOnErr(cmd, "can't get execution fee factor:", err) + + feePerByte, err := policyContract.GetFeePerByte() + commonCmd.ExitOnErr(cmd, "can't get fee per byte:", err) + + storagePrice, err := policyContract.GetStoragePrice() + commonCmd.ExitOnErr(cmd, "can't get storage price:", err) + + buf := bytes.NewBuffer(nil) + tw := tabwriter.NewWriter(buf, 0, 2, 2, ' ', 0) + + _, _ = tw.Write([]byte(fmt.Sprintf("Execution Fee Factor:\t%d (int)\n", execFee))) + _, _ = tw.Write([]byte(fmt.Sprintf("Fee Per Byte:\t%d (int)\n", feePerByte))) + _, _ = tw.Write([]byte(fmt.Sprintf("Storage Price:\t%d (int)\n", storagePrice))) + + _ = tw.Flush() + cmd.Print(buf.String()) + + return nil +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 133d5162f..707ae733d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -146,6 +146,15 @@ var ( }, } + dumpPolicy = &cobra.Command{ + Use: "dump-policy", + Short: "Dump FrostFS policy", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + RunE: dumpPolicyCmd, + } + dumpContractHashesCmd = &cobra.Command{ Use: "dump-hashes", Short: "Dump deployed contract hashes", @@ -239,6 +248,7 @@ func init() { initForceNewEpochCmd() initRemoveNodesCmd() initSetPolicyCmd() + initDumpPolicyCmd() initDumpContractHashesCmd() initDumpNetworkConfigCmd() initSetConfigCmd() @@ -341,6 +351,11 @@ func initSetPolicyCmd() { setPolicy.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } +func initDumpPolicyCmd() { + RootCmd.AddCommand(dumpPolicy) + dumpPolicy.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") +} + func initRemoveNodesCmd() { RootCmd.AddCommand(removeNodes) removeNodes.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") From 74c91eeef5fe520ae3f73a882315b1c9189ede53 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Wed, 1 Nov 2023 18:18:28 +0300 Subject: [PATCH 0653/1943] [#777] client: Refactor PrmContainerList, PrmObjectSearch usage Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/client.go | 23 +++++----------- cmd/frostfs-cli/modules/container/list.go | 2 +- go.mod | 2 +- go.sum | 4 +-- pkg/services/object/internal/client/client.go | 26 +++++-------------- 5 files changed, 18 insertions(+), 39 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index b370ff3aa..9807e111c 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -689,24 +689,15 @@ func (x SearchObjectsRes) IDList() []oid.ID { // // Returns any error which prevented the operation from completing correctly in error return. func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes, error) { - var cliPrm client.PrmObjectSearch - cliPrm.InContainer(prm.cnrID) - cliPrm.SetFilters(prm.filters) - - if prm.sessionToken != nil { - cliPrm.WithinSession(*prm.sessionToken) + cliPrm := client.PrmObjectSearch{ + XHeaders: prm.xHeaders, + Local: prm.local, + BearerToken: prm.bearerToken, + Session: prm.sessionToken, + ContainerID: &prm.cnrID, + Filters: prm.filters, } - if prm.bearerToken != nil { - cliPrm.WithBearerToken(*prm.bearerToken) - } - - if prm.local { - cliPrm.MarkLocal() - } - - cliPrm.WithXHeaders(prm.xHeaders...) - rdr, err := prm.cli.ObjectSearchInit(ctx, cliPrm) if err != nil { return nil, fmt.Errorf("init object search: %w", err) diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 1c7787760..1e7ba96ea 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -47,7 +47,7 @@ var listContainersCmd = &cobra.Command{ var prm internalclient.ListContainersPrm prm.SetClient(cli) - prm.SetAccount(idUser) + prm.Account = &idUser res, err := internalclient.ListContainers(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) diff --git a/go.mod b/go.mod index 9e97238c5..37cb477bf 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231027075445-a02c0bfac885 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 80b600167..62fd85c86 100644 --- a/go.sum +++ b/go.sum @@ -732,8 +732,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231027075445-a02c0bfac885 h1:tbOUgoaN3usFTUA9k62z96uQDRNJUiz9TxzJr1AtSFo= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231027075445-a02c0bfac885/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d h1:KVHsNsEoedQ1+f+eDq88AFKkLePGfnygt4bcUQrCgPw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index e0a7e562a..dd2de0fd1 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -474,12 +474,12 @@ type SearchObjectsPrm struct { // // Required parameter. func (x *SearchObjectsPrm) SetContainerID(id cid.ID) { - x.cliPrm.InContainer(id) + x.cliPrm.ContainerID = &id } // SetFilters sets search filters. func (x *SearchObjectsPrm) SetFilters(fs objectSDK.SearchFilters) { - x.cliPrm.SetFilters(fs) + x.cliPrm.Filters = fs } // SearchObjectsRes groups the resulting values of SearchObjects operation. @@ -496,23 +496,11 @@ func (x SearchObjectsRes) IDList() []oid.ID { // // Returns any error which prevented the operation from completing correctly in error return. func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes, error) { - if prm.local { - prm.cliPrm.MarkLocal() - } - - if prm.tokenSession != nil { - prm.cliPrm.WithinSession(*prm.tokenSession) - } - - if prm.tokenBearer != nil { - prm.cliPrm.WithBearerToken(*prm.tokenBearer) - } - - prm.cliPrm.WithXHeaders(prm.xHeaders...) - - if prm.key != nil { - prm.cliPrm.UseKey(*prm.key) - } + prm.cliPrm.Local = prm.local + prm.cliPrm.Session = prm.tokenSession + prm.cliPrm.BearerToken = prm.tokenBearer + prm.cliPrm.XHeaders = prm.xHeaders + prm.cliPrm.Key = prm.key rdr, err := prm.cli.ObjectSearchInit(ctx, prm.cliPrm) if err != nil { From 523fb3ca5163d37dfe189a5a760932002efe4fc3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 2 Nov 2023 17:57:12 +0300 Subject: [PATCH 0654/1943] [#684] proto: Add skipped count to evacuation status response Signed-off-by: Dmitrii Stepanov --- pkg/services/control/service.pb.go | 207 +++++++++++---------- pkg/services/control/service.proto | 5 +- pkg/services/control/service_frostfs.pb.go | 2 + pkg/services/control/service_grpc.pb.go | 2 +- pkg/services/control/types.pb.go | 2 +- 5 files changed, 117 insertions(+), 101 deletions(-) diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index b1bebb1e2..443afb1aa 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.21.9 +// protoc v4.24.4 // source: pkg/services/control/service.proto package control @@ -2458,7 +2458,7 @@ type GetShardEvacuationStatusResponse_Body struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Total objects to evacuate count. The value is approximate, so evacuated + failed == total is not guaranteed after completion. + // Total objects to evacuate count. The value is approximate, so evacuated + failed + skipped == total is not guaranteed after completion. Total uint64 `protobuf:"varint,1,opt,name=total,proto3" json:"total,omitempty"` // Evacuated objects count. Evacuated uint64 `protobuf:"varint,2,opt,name=evacuated,proto3" json:"evacuated,omitempty"` @@ -2474,6 +2474,8 @@ type GetShardEvacuationStatusResponse_Body struct { StartedAt *GetShardEvacuationStatusResponse_Body_UnixTimestamp `protobuf:"bytes,7,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` // Error message if evacuation failed. ErrorMessage string `protobuf:"bytes,8,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` + // Skipped objects count. + Skipped uint64 `protobuf:"varint,9,opt,name=skipped,proto3" json:"skipped,omitempty"` } func (x *GetShardEvacuationStatusResponse_Body) Reset() { @@ -2564,6 +2566,13 @@ func (x *GetShardEvacuationStatusResponse_Body) GetErrorMessage() string { return "" } +func (x *GetShardEvacuationStatusResponse_Body) GetSkipped() uint64 { + if x != nil { + return x.Skipped + } + return 0 +} + // Unix timestamp value. type GetShardEvacuationStatusResponse_Body_UnixTimestamp struct { state protoimpl.MessageState @@ -2975,7 +2984,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xc6, 0x05, 0x0a, 0x20, + 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xe0, 0x05, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, @@ -2985,7 +2994,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xab, 0x04, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xc5, 0x04, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, @@ -3010,103 +3019,105 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x25, 0x0a, 0x0d, - 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x1a, 0x24, 0x0a, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x18, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x23, 0x0a, 0x1f, 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, - 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, - 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, - 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, - 0x45, 0x44, 0x10, 0x02, 0x22, 0x94, 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x73, + 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x1a, 0x25, 0x0a, 0x0d, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x24, 0x0a, + 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x63, + 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73, 0x65, 0x63, 0x6f, + 0x6e, 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x23, 0x0a, + 0x1f, 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, + 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0x94, + 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, + 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xee, + 0x07, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, + 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, + 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, + 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, + 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, + 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, + 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, + 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, + 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, + 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, - 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x32, 0xee, 0x07, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, - 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, - 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, - 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, - 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, - 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, - 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, + 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, - 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, - 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, - 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, - 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, - 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, - 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, - 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, + 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, + 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, + 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, + 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, + 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, + 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, + 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index a80deb2da..cbcf74e53 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -362,7 +362,7 @@ message GetShardEvacuationStatusResponse { int64 seconds = 1; } - // Total objects to evacuate count. The value is approximate, so evacuated + failed == total is not guaranteed after completion. + // Total objects to evacuate count. The value is approximate, so evacuated + failed + skipped == total is not guaranteed after completion. uint64 total = 1; // Evacuated objects count. uint64 evacuated = 2; @@ -379,6 +379,9 @@ message GetShardEvacuationStatusResponse { UnixTimestamp started_at = 7; // Error message if evacuation failed. string error_message = 8; + + // Skipped objects count. + uint64 skipped = 9; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index cc96a98df..56ff81ace 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -1819,6 +1819,7 @@ func (x *GetShardEvacuationStatusResponse_Body) StableSize() (size int) { size += proto.NestedStructureSize(6, x.Duration) size += proto.NestedStructureSize(7, x.StartedAt) size += proto.StringSize(8, x.ErrorMessage) + size += proto.UInt64Size(9, x.Skipped) return size } @@ -1846,6 +1847,7 @@ func (x *GetShardEvacuationStatusResponse_Body) StableMarshal(buf []byte) []byte offset += proto.NestedStructureMarshal(6, buf[offset:], x.Duration) offset += proto.NestedStructureMarshal(7, buf[offset:], x.StartedAt) offset += proto.StringMarshal(8, buf[offset:], x.ErrorMessage) + offset += proto.UInt64Marshal(9, buf[offset:], x.Skipped) return buf } diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 8afc6086a..967c739f5 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.9 +// - protoc v4.24.4 // source: pkg/services/control/service.proto package control diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index b385bc674..33700c5b9 100644 --- a/pkg/services/control/types.pb.go +++ b/pkg/services/control/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.21.9 +// protoc v4.24.4 // source: pkg/services/control/types.proto package control From 1e21733ea56ea4c115770be5b900617c6e5c59a6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 2 Nov 2023 18:09:43 +0300 Subject: [PATCH 0655/1943] [#684] cli: Add skipped count to evacuation status output Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/evacuation.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 4eb6505cf..735897b81 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -220,12 +220,12 @@ func appendEstimation(sb *strings.Builder, resp *control.GetShardEvacuationStatu if resp.GetBody().GetStatus() != control.GetShardEvacuationStatusResponse_Body_RUNNING || resp.GetBody().GetDuration() == nil || resp.GetBody().GetTotal() == 0 || - resp.GetBody().GetEvacuated()+resp.GetBody().GetFailed() == 0 { + resp.GetBody().GetEvacuated()+resp.GetBody().GetFailed()+resp.Body.GetSkipped() == 0 { return } durationSeconds := float64(resp.GetBody().GetDuration().GetSeconds()) - evacuated := float64(resp.GetBody().GetEvacuated() + resp.GetBody().GetFailed()) + evacuated := float64(resp.GetBody().GetEvacuated() + resp.GetBody().GetFailed() + resp.GetBody().GetSkipped()) avgObjEvacuationTimeSeconds := durationSeconds / evacuated objectsLeft := float64(resp.GetBody().GetTotal()) - evacuated leftSeconds := avgObjEvacuationTimeSeconds * objectsLeft @@ -285,10 +285,11 @@ func appendShardIDs(sb *strings.Builder, resp *control.GetShardEvacuationStatusR } func appendCounts(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { - sb.WriteString(fmt.Sprintf(" Evacuated %d object out of %d, failed to evacuate %d objects.", + sb.WriteString(fmt.Sprintf(" Evacuated %d objects out of %d, failed to evacuate: %d, skipped: %d.", resp.GetBody().GetEvacuated(), - resp.Body.GetTotal(), - resp.Body.GetFailed())) + resp.GetBody().GetTotal(), + resp.GetBody().GetFailed(), + resp.GetBody().GetSkipped())) } func initControlEvacuationShardCmd() { From 226e84d782ed635e5e642547fba733bd3a4c87be Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 2 Nov 2023 18:22:58 +0300 Subject: [PATCH 0656/1943] [#684] node: Add skipped objects count to evacuation result Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 16 ++++++++++++++++ .../engine/evacuate_limiter.go | 7 +++++++ pkg/services/control/server/convert.go | 1 + 3 files changed, 24 insertions(+) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index f5f1b658d..7bb37ef61 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -41,6 +41,7 @@ type EvacuateShardRes struct { evacuated *atomic.Uint64 total *atomic.Uint64 failed *atomic.Uint64 + skipped *atomic.Uint64 } // NewEvacuateShardRes creates new EvacuateShardRes instance. @@ -49,6 +50,7 @@ func NewEvacuateShardRes() *EvacuateShardRes { evacuated: new(atomic.Uint64), total: new(atomic.Uint64), failed: new(atomic.Uint64), + skipped: new(atomic.Uint64), } } @@ -97,6 +99,14 @@ func (p *EvacuateShardRes) Failed() uint64 { return p.failed.Load() } +// Skipped returns count of skipped objects. +func (p *EvacuateShardRes) Skipped() uint64 { + if p == nil { + return 0 + } + return p.skipped.Load() +} + // DeepCopy returns deep copy of result instance. func (p *EvacuateShardRes) DeepCopy() *EvacuateShardRes { if p == nil { @@ -107,11 +117,13 @@ func (p *EvacuateShardRes) DeepCopy() *EvacuateShardRes { evacuated: new(atomic.Uint64), total: new(atomic.Uint64), failed: new(atomic.Uint64), + skipped: new(atomic.Uint64), } res.evacuated.Store(p.evacuated.Load()) res.total.Store(p.total.Load()) res.failed.Store(p.failed.Load()) + res.skipped.Store(p.skipped.Load()) return res } @@ -224,6 +236,7 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p zap.Uint64("total", res.Total()), zap.Uint64("evacuated", res.Evacuated()), zap.Uint64("failed", res.Failed()), + zap.Uint64("skipped", res.Skipped()), ) return nil } @@ -404,6 +417,9 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } + if exists { + res.skipped.Add(1) + } return true, nil } } diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go index 62795fa1a..83acd30d9 100644 --- a/pkg/local_object_storage/engine/evacuate_limiter.go +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -55,6 +55,13 @@ func (s *EvacuationState) Failed() uint64 { return s.result.Failed() } +func (s *EvacuationState) Skipped() uint64 { + if s == nil { + return 0 + } + return s.result.Skipped() +} + func (s *EvacuationState) ProcessingStatus() EvacuateProcessState { if s == nil { return EvacuateProcessStateUndefined diff --git a/pkg/services/control/server/convert.go b/pkg/services/control/server/convert.go index 1d29ed406..f922d7278 100644 --- a/pkg/services/control/server/convert.go +++ b/pkg/services/control/server/convert.go @@ -55,6 +55,7 @@ func stateToResponse(state *engine.EvacuationState) (*control.GetShardEvacuation StartedAt: startedAt, Duration: duration, ErrorMessage: state.ErrorMessage(), + Skipped: state.Skipped(), }, }, nil } From 9c98fa6152640aa3bfd6f50f1dc343857731f8fc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 1 Nov 2023 20:53:24 +0300 Subject: [PATCH 0657/1943] [#763] metabase: Add container objects counter Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/control.go | 15 +- pkg/local_object_storage/metabase/counter.go | 276 +++++++++++++++++- .../metabase/counter_test.go | 201 ++++++++++++- pkg/local_object_storage/metabase/delete.go | 44 ++- pkg/local_object_storage/metabase/inhume.go | 24 +- pkg/local_object_storage/metabase/put.go | 12 +- pkg/local_object_storage/metabase/util.go | 12 +- 7 files changed, 535 insertions(+), 49 deletions(-) diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 3f155eeb5..00fe1e02b 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -103,12 +103,13 @@ func (db *DB) init(reset bool) error { } mStaticBuckets := map[string]struct{}{ - string(containerVolumeBucketName): {}, - string(graveyardBucketName): {}, - string(toMoveItBucketName): {}, - string(garbageBucketName): {}, - string(shardInfoBucket): {}, - string(bucketNameLocked): {}, + string(containerVolumeBucketName): {}, + string(containerCounterBucketName): {}, + string(graveyardBucketName): {}, + string(toMoveItBucketName): {}, + string(garbageBucketName): {}, + string(shardInfoBucket): {}, + string(bucketNameLocked): {}, } return db.boltDB.Update(func(tx *bbolt.Tx) error { @@ -135,7 +136,7 @@ func (db *DB) init(reset bool) error { } } - if !reset { + if !reset { // counters will be recalculated by refill metabase err = syncCounter(tx, false) if err != nil { return fmt.Errorf("could not sync object counter: %w", err) diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index d4afe1878..d1aafa3db 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -1,10 +1,15 @@ package meta import ( + "bytes" + "context" "encoding/binary" + "errors" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -73,9 +78,128 @@ func (db *DB) ObjectCounters() (cc ObjectCounters, err error) { return cc, metaerr.Wrap(err) } -// updateCounter updates the object counter. Tx MUST be writable. -// If inc == `true`, increases the counter, decreases otherwise. -func (db *DB) updateCounter(tx *bbolt.Tx, typ objectType, delta uint64, inc bool) error { +type ContainerCounters struct { + Logical map[cid.ID]uint64 + Physical map[cid.ID]uint64 +} + +// ContainerCounters returns object counters for each container +// that metabase has tracked since it was opened and initialized. +// +// Returns only the errors that do not allow reading counter +// in Bolt database. +// +// It is guaranteed that the ContainerCounters fields are not nil. +func (db *DB) ContainerCounters(ctx context.Context) (ContainerCounters, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("ContainerCounters", time.Since(startedAt), success) + }() + + ctx, span := tracing.StartSpanFromContext(ctx, "metabase.ContainerCounters") + defer span.End() + + cc := ContainerCounters{ + Logical: make(map[cid.ID]uint64), + Physical: make(map[cid.ID]uint64), + } + + lastKey := make([]byte, cidSize) + + // there is no limit for containers count, so use batching with cancellation + for { + select { + case <-ctx.Done(): + return cc, ctx.Err() + default: + } + + completed, err := db.containerCountersNextBatch(lastKey, &cc) + if err != nil { + return cc, err + } + if completed { + break + } + } + + success = true + return cc, nil +} + +func (db *DB) containerCountersNextBatch(lastKey []byte, cc *ContainerCounters) (bool, error) { + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return false, ErrDegradedMode + } + + counter := 0 + const batchSize = 1000 + + err := db.boltDB.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(containerCounterBucketName) + if b == nil { + return ErrInterruptIterator + } + c := b.Cursor() + var key, value []byte + for key, value = c.Seek(lastKey); key != nil; key, value = c.Next() { + if bytes.Equal(lastKey, key) { + continue + } + copy(lastKey, key) + + cnrID, err := parseContainerCounterKey(key) + if err != nil { + return err + } + phy, logic, err := parseContainerCounterValue(value) + if err != nil { + return err + } + if phy > 0 { + cc.Physical[cnrID] = phy + } + if logic > 0 { + cc.Logical[cnrID] = logic + } + + counter++ + if counter == batchSize { + break + } + } + + if counter < batchSize { // last batch + return ErrInterruptIterator + } + return nil + }) + if err != nil { + if errors.Is(err, ErrInterruptIterator) { + return true, nil + } + return false, metaerr.Wrap(err) + } + return false, nil +} + +func (db *DB) incCounters(tx *bbolt.Tx, cnrID cid.ID) error { + if err := db.updateShardObjectCounter(tx, phy, 1, true); err != nil { + return fmt.Errorf("could not increase phy object counter: %w", err) + } + if err := db.updateShardObjectCounter(tx, logical, 1, true); err != nil { + return fmt.Errorf("could not increase logical object counter: %w", err) + } + return db.incContainerObjectCounter(tx, cnrID) +} + +func (db *DB) updateShardObjectCounter(tx *bbolt.Tx, typ objectType, delta uint64, inc bool) error { b := tx.Bucket(shardInfoBucket) if b == nil { return nil @@ -112,6 +236,63 @@ func (db *DB) updateCounter(tx *bbolt.Tx, typ objectType, delta uint64, inc bool return b.Put(counterKey, newCounter) } +func (db *DB) updateContainerCounter(tx *bbolt.Tx, delta map[cid.ID]ObjectCounters, inc bool) error { + b := tx.Bucket(containerCounterBucketName) + if b == nil { + return nil + } + + key := make([]byte, cidSize) + for cnrID, cnrDelta := range delta { + cnrID.Encode(key) + if err := db.editContainerCounterValue(b, key, cnrDelta, inc); err != nil { + return err + } + } + return nil +} + +func (*DB) editContainerCounterValue(b *bbolt.Bucket, key []byte, delta ObjectCounters, inc bool) error { + var phyValue, logicValue uint64 + var err error + data := b.Get(key) + if len(data) > 0 { + phyValue, logicValue, err = parseContainerCounterValue(data) + if err != nil { + return err + } + } + phyValue = nextValue(phyValue, delta.phy, inc) + logicValue = nextValue(logicValue, delta.logic, inc) + if phyValue > 0 || logicValue > 0 { + value := containerCounterValue(phyValue, logicValue) + return b.Put(key, value) + } + return b.Delete(key) +} + +func nextValue(existed, delta uint64, inc bool) uint64 { + if inc { + existed += delta + } else if existed <= delta { + existed = 0 + } else { + existed -= delta + } + return existed +} + +func (db *DB) incContainerObjectCounter(tx *bbolt.Tx, cnrID cid.ID) error { + b := tx.Bucket(containerCounterBucketName) + if b == nil { + return nil + } + + key := make([]byte, cidSize) + cnrID.Encode(key) + return db.editContainerCounterValue(b, key, ObjectCounters{logic: 1, phy: 1}, true) +} + // syncCounter updates object counters according to metabase state: // it counts all the physically/logically stored objects using internal // indexes. Tx MUST be writable. @@ -119,26 +300,38 @@ func (db *DB) updateCounter(tx *bbolt.Tx, typ objectType, delta uint64, inc bool // Does nothing if counters are not empty and force is false. If force is // true, updates the counters anyway. func syncCounter(tx *bbolt.Tx, force bool) error { - b, err := tx.CreateBucketIfNotExists(shardInfoBucket) + shardInfoB, err := tx.CreateBucketIfNotExists(shardInfoBucket) if err != nil { return fmt.Errorf("could not get shard info bucket: %w", err) } - - if !force && len(b.Get(objectPhyCounterKey)) == 8 && len(b.Get(objectLogicCounterKey)) == 8 { + shardObjectCounterInitialized := len(shardInfoB.Get(objectPhyCounterKey)) == 8 && len(shardInfoB.Get(objectLogicCounterKey)) == 8 + containerCounterInitialized := tx.Bucket(containerCounterBucketName) != nil + if !force && shardObjectCounterInitialized && containerCounterInitialized { // the counters are already inited return nil } + containerCounterB, err := tx.CreateBucketIfNotExists(containerCounterBucketName) + if err != nil { + return fmt.Errorf("could not get container counter bucket: %w", err) + } + var addr oid.Address - var phyCounter uint64 - var logicCounter uint64 + counters := make(map[cid.ID]ObjectCounters) graveyardBKT := tx.Bucket(graveyardBucketName) garbageBKT := tx.Bucket(garbageBucketName) key := make([]byte, addressKeySize) err = iteratePhyObjects(tx, func(cnr cid.ID, obj oid.ID) error { - phyCounter++ + if v, ok := counters[cnr]; ok { + v.phy++ + counters[cnr] = v + } else { + counters[cnr] = ObjectCounters{ + phy: 1, + } + } addr.SetContainer(cnr) addr.SetObject(obj) @@ -146,7 +339,14 @@ func syncCounter(tx *bbolt.Tx, force bool) error { // check if an object is available: not with GCMark // and not covered with a tombstone if inGraveyardWithKey(addressKey(addr, key), graveyardBKT, garbageBKT) == 0 { - logicCounter++ + if v, ok := counters[cnr]; ok { + v.logic++ + counters[cnr] = v + } else { + counters[cnr] = ObjectCounters{ + logic: 1, + } + } } return nil @@ -155,21 +355,65 @@ func syncCounter(tx *bbolt.Tx, force bool) error { return fmt.Errorf("could not iterate objects: %w", err) } - data := make([]byte, 8) - binary.LittleEndian.PutUint64(data, phyCounter) + return setObjectCounters(counters, shardInfoB, containerCounterB) +} - err = b.Put(objectPhyCounterKey, data) +func setObjectCounters(counters map[cid.ID]ObjectCounters, shardInfoB, containerCounterB *bbolt.Bucket) error { + var phyTotal uint64 + var logicTotal uint64 + key := make([]byte, cidSize) + for cnrID, count := range counters { + phyTotal += count.phy + logicTotal += count.logic + + cnrID.Encode(key) + value := containerCounterValue(count.phy, count.logic) + err := containerCounterB.Put(key, value) + if err != nil { + return fmt.Errorf("could not update phy container object counter: %w", err) + } + } + phyData := make([]byte, 8) + binary.LittleEndian.PutUint64(phyData, phyTotal) + + err := shardInfoB.Put(objectPhyCounterKey, phyData) if err != nil { return fmt.Errorf("could not update phy object counter: %w", err) } - data = make([]byte, 8) - binary.LittleEndian.PutUint64(data, logicCounter) + logData := make([]byte, 8) + binary.LittleEndian.PutUint64(logData, logicTotal) - err = b.Put(objectLogicCounterKey, data) + err = shardInfoB.Put(objectLogicCounterKey, logData) if err != nil { return fmt.Errorf("could not update logic object counter: %w", err) } return nil } + +func containerCounterValue(phy, logic uint64) []byte { + res := make([]byte, 16) + binary.LittleEndian.PutUint64(res, phy) + binary.LittleEndian.PutUint64(res[8:], logic) + return res +} + +func parseContainerCounterKey(buf []byte) (cid.ID, error) { + if len(buf) != cidSize { + return cid.ID{}, fmt.Errorf("invalid key length") + } + var cnrID cid.ID + if err := cnrID.Decode(buf); err != nil { + return cid.ID{}, fmt.Errorf("failed to decode container ID: %w", err) + } + return cnrID, nil +} + +// parseContainerCounterValue return phy, logic values. +func parseContainerCounterValue(buf []byte) (uint64, uint64, error) { + if len(buf) != 16 { + return 0, 0, fmt.Errorf("invalid value length") + } + return binary.LittleEndian.Uint64(buf), binary.LittleEndian.Uint64(buf[8:]), nil +} diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 89b52c887..51062f1fe 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -7,6 +7,7 @@ import ( objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -25,6 +26,11 @@ func TestCounters(t *testing.T) { require.NoError(t, err) require.Zero(t, c.Phy()) require.Zero(t, c.Logic()) + + cc, err := db.ContainerCounters(context.Background()) + require.NoError(t, err) + require.Zero(t, len(cc.Physical)) + require.Zero(t, len(cc.Logical)) }) t.Run("put", func(t *testing.T) { @@ -36,9 +42,14 @@ func TestCounters(t *testing.T) { } var prm meta.PutPrm + expPhy := make(map[cid.ID]uint64) + expLog := make(map[cid.ID]uint64) for i := 0; i < objCount; i++ { prm.SetObject(oo[i]) + cnrID, _ := oo[i].ContainerID() + expPhy[cnrID]++ + expLog[cnrID]++ _, err := db.Put(context.Background(), prm) require.NoError(t, err) @@ -48,6 +59,12 @@ func TestCounters(t *testing.T) { require.Equal(t, uint64(i+1), c.Phy()) require.Equal(t, uint64(i+1), c.Logic()) + + cc, err := db.ContainerCounters(context.Background()) + require.NoError(t, err) + + require.Equal(t, expPhy, cc.Physical) + require.Equal(t, expLog, cc.Logical) } }) @@ -56,6 +73,14 @@ func TestCounters(t *testing.T) { db := newDB(t) oo := putObjs(t, db, objCount, false) + expPhy := make(map[cid.ID]uint64) + expLog := make(map[cid.ID]uint64) + for _, obj := range oo { + cnrID, _ := obj.ContainerID() + expPhy[cnrID]++ + expLog[cnrID]++ + } + var prm meta.DeletePrm for i := objCount - 1; i >= 0; i-- { prm.SetAddresses(objectcore.AddressOf(oo[i])) @@ -69,6 +94,27 @@ func TestCounters(t *testing.T) { require.Equal(t, uint64(i), c.Phy()) require.Equal(t, uint64(i), c.Logic()) + + cnrID, _ := oo[i].ContainerID() + if v, ok := expPhy[cnrID]; ok { + if v == 1 { + delete(expPhy, cnrID) + } else { + expPhy[cnrID]-- + } + } + if v, ok := expLog[cnrID]; ok { + if v == 1 { + delete(expLog, cnrID) + } else { + expLog[cnrID]-- + } + } + + cc, err := db.ContainerCounters(context.Background()) + require.NoError(t, err) + require.Equal(t, expPhy, cc.Physical) + require.Equal(t, expLog, cc.Logical) } }) @@ -77,6 +123,14 @@ func TestCounters(t *testing.T) { db := newDB(t) oo := putObjs(t, db, objCount, false) + expPhy := make(map[cid.ID]uint64) + expLog := make(map[cid.ID]uint64) + for _, obj := range oo { + cnrID, _ := obj.ContainerID() + expPhy[cnrID]++ + expLog[cnrID]++ + } + inhumedObjs := make([]oid.Address, objCount/2) for i, o := range oo { @@ -87,6 +141,16 @@ func TestCounters(t *testing.T) { inhumedObjs[i] = objectcore.AddressOf(o) } + for _, addr := range inhumedObjs { + if v, ok := expLog[addr.Container()]; ok { + if v == 1 { + delete(expLog, addr.Container()) + } else { + expLog[addr.Container()]-- + } + } + } + var prm meta.InhumePrm prm.SetTombstoneAddress(oidtest.Address()) prm.SetAddresses(inhumedObjs...) @@ -100,6 +164,12 @@ func TestCounters(t *testing.T) { require.Equal(t, uint64(objCount), c.Phy()) require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic()) + + cc, err := db.ContainerCounters(context.Background()) + require.NoError(t, err) + + require.Equal(t, expPhy, cc.Physical) + require.Equal(t, expLog, cc.Logical) }) t.Run("put_split", func(t *testing.T) { @@ -107,6 +177,9 @@ func TestCounters(t *testing.T) { db := newDB(t) parObj := testutil.GenerateObject() + expPhy := make(map[cid.ID]uint64) + expLog := make(map[cid.ID]uint64) + // put objects and check that parent info // does not affect the counter for i := 0; i < objCount; i++ { @@ -115,12 +188,21 @@ func TestCounters(t *testing.T) { o.SetParent(parObj) } + cnrID, _ := o.ContainerID() + expLog[cnrID]++ + expPhy[cnrID]++ + require.NoError(t, putBig(db, o)) c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(i+1), c.Phy()) require.Equal(t, uint64(i+1), c.Logic()) + + cc, err := db.ContainerCounters(context.Background()) + require.NoError(t, err) + require.Equal(t, expPhy, cc.Physical) + require.Equal(t, expLog, cc.Logical) } }) @@ -129,16 +211,40 @@ func TestCounters(t *testing.T) { db := newDB(t) oo := putObjs(t, db, objCount, true) + expPhy := make(map[cid.ID]uint64) + expLog := make(map[cid.ID]uint64) + for _, obj := range oo { + cnrID, _ := obj.ContainerID() + expPhy[cnrID]++ + expLog[cnrID]++ + } + // delete objects that have parent info // and check that it does not affect // the counter for i, o := range oo { - require.NoError(t, metaDelete(db, objectcore.AddressOf(o))) + addr := objectcore.AddressOf(o) + require.NoError(t, metaDelete(db, addr)) c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(objCount-i-1), c.Phy()) require.Equal(t, uint64(objCount-i-1), c.Logic()) + + if v, ok := expPhy[addr.Container()]; ok { + if v == 1 { + delete(expPhy, addr.Container()) + } else { + expPhy[addr.Container()]-- + } + } + if v, ok := expLog[addr.Container()]; ok { + if v == 1 { + delete(expLog, addr.Container()) + } else { + expLog[addr.Container()]-- + } + } } }) @@ -147,6 +253,14 @@ func TestCounters(t *testing.T) { db := newDB(t) oo := putObjs(t, db, objCount, true) + expPhy := make(map[cid.ID]uint64) + expLog := make(map[cid.ID]uint64) + for _, obj := range oo { + cnrID, _ := obj.ContainerID() + expPhy[cnrID]++ + expLog[cnrID]++ + } + inhumedObjs := make([]oid.Address, objCount/2) for i, o := range oo { @@ -157,6 +271,16 @@ func TestCounters(t *testing.T) { inhumedObjs[i] = objectcore.AddressOf(o) } + for _, addr := range inhumedObjs { + if v, ok := expLog[addr.Container()]; ok { + if v == 1 { + delete(expLog, addr.Container()) + } else { + expLog[addr.Container()]-- + } + } + } + var prm meta.InhumePrm prm.SetTombstoneAddress(oidtest.Address()) prm.SetAddresses(inhumedObjs...) @@ -169,6 +293,12 @@ func TestCounters(t *testing.T) { require.Equal(t, uint64(objCount), c.Phy()) require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic()) + + cc, err := db.ContainerCounters(context.Background()) + require.NoError(t, err) + + require.Equal(t, expPhy, cc.Physical) + require.Equal(t, expLog, cc.Logical) }) } @@ -190,6 +320,13 @@ func TestCounters_Expired(t *testing.T) { oo[i] = putWithExpiration(t, db, objectSDK.TypeRegular, epoch+1) } + expPhy := make(map[cid.ID]uint64) + expLog := make(map[cid.ID]uint64) + for _, addr := range oo { + expPhy[addr.Container()]++ + expLog[addr.Container()]++ + } + // 1. objects are available and counters are correct c, err := db.ObjectCounters() @@ -197,6 +334,12 @@ func TestCounters_Expired(t *testing.T) { require.Equal(t, uint64(objCount), c.Phy()) require.Equal(t, uint64(objCount), c.Logic()) + cc, err := db.ContainerCounters(context.Background()) + require.NoError(t, err) + + require.Equal(t, expPhy, cc.Physical) + require.Equal(t, expLog, cc.Logical) + for _, o := range oo { _, err := metaGet(db, o, true) require.NoError(t, err) @@ -212,6 +355,12 @@ func TestCounters_Expired(t *testing.T) { require.Equal(t, uint64(objCount), c.Phy()) require.Equal(t, uint64(objCount), c.Logic()) + cc, err = db.ContainerCounters(context.Background()) + require.NoError(t, err) + + require.Equal(t, expPhy, cc.Physical) + require.Equal(t, expLog, cc.Logical) + for _, o := range oo { _, err := metaGet(db, o, true) require.ErrorIs(t, err, meta.ErrObjectIsExpired) @@ -235,6 +384,20 @@ func TestCounters_Expired(t *testing.T) { require.Equal(t, uint64(len(oo)), c.Phy()) require.Equal(t, uint64(len(oo)-1), c.Logic()) + if v, ok := expLog[oo[0].Container()]; ok { + if v == 1 { + delete(expLog, oo[0].Container()) + } else { + expLog[oo[0].Container()]-- + } + } + + cc, err = db.ContainerCounters(context.Background()) + require.NoError(t, err) + + require.Equal(t, expPhy, cc.Physical) + require.Equal(t, expLog, cc.Logical) + // 4. `Delete` an object with GCMark should decrease the // phy counter but does not affect the logic counter (after // that step they should be equal) @@ -246,6 +409,14 @@ func TestCounters_Expired(t *testing.T) { require.NoError(t, err) require.Zero(t, deleteRes.AvailableObjectsRemoved()) + if v, ok := expPhy[oo[0].Container()]; ok { + if v == 1 { + delete(expPhy, oo[0].Container()) + } else { + expPhy[oo[0].Container()]-- + } + } + oo = oo[1:] c, err = db.ObjectCounters() @@ -253,6 +424,12 @@ func TestCounters_Expired(t *testing.T) { require.Equal(t, uint64(len(oo)), c.Phy()) require.Equal(t, uint64(len(oo)), c.Logic()) + cc, err = db.ContainerCounters(context.Background()) + require.NoError(t, err) + + require.Equal(t, expPhy, cc.Physical) + require.Equal(t, expLog, cc.Logical) + // 5 `Delete` an expired object (like it would the control // service do) should decrease both counters despite the // expiration fact @@ -263,12 +440,34 @@ func TestCounters_Expired(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(1), deleteRes.AvailableObjectsRemoved()) + if v, ok := expLog[oo[0].Container()]; ok { + if v == 1 { + delete(expLog, oo[0].Container()) + } else { + expLog[oo[0].Container()]-- + } + } + + if v, ok := expPhy[oo[0].Container()]; ok { + if v == 1 { + delete(expPhy, oo[0].Container()) + } else { + expPhy[oo[0].Container()]-- + } + } + oo = oo[1:] c, err = db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(len(oo)), c.Phy()) require.Equal(t, uint64(len(oo)), c.Logic()) + + cc, err = db.ContainerCounters(context.Background()) + require.NoError(t, err) + + require.Equal(t, expPhy, cc.Physical) + require.Equal(t, expLog, cc.Logical) } func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*objectSDK.Object { diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index e84759b88..653a88ae7 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -12,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -132,8 +133,9 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address, sizes []uint64, ava refCounter := make(referenceCounter, len(addrs)) currEpoch := db.epochState.CurrentEpoch() - var rawDeleted uint64 - var availableDeleted uint64 + cnrIDDelta := make(map[cid.ID]ObjectCounters) + var rawDeletedTotal uint64 + var availableDeletedTotal uint64 for i := range addrs { removed, available, size, err := db.delete(tx, addrs[i], refCounter, currEpoch) @@ -142,40 +144,62 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address, sizes []uint64, ava } if removed { - rawDeleted++ + if v, ok := cnrIDDelta[addrs[i].Container()]; ok { + v.phy++ + cnrIDDelta[addrs[i].Container()] = v + } else { + cnrIDDelta[addrs[i].Container()] = ObjectCounters{ + phy: 1, + } + } + + rawDeletedTotal++ sizes[i] = size } if available { - availableDeleted++ + if v, ok := cnrIDDelta[addrs[i].Container()]; ok { + v.logic++ + cnrIDDelta[addrs[i].Container()] = v + } else { + cnrIDDelta[addrs[i].Container()] = ObjectCounters{ + logic: 1, + } + } + + availableDeletedTotal++ availableSizes[i] = size } } - if rawDeleted > 0 { - err := db.updateCounter(tx, phy, rawDeleted, false) + if rawDeletedTotal > 0 { + err := db.updateShardObjectCounter(tx, phy, rawDeletedTotal, false) if err != nil { return 0, 0, fmt.Errorf("could not decrease phy object counter: %w", err) } } - if availableDeleted > 0 { - err := db.updateCounter(tx, logical, availableDeleted, false) + if availableDeletedTotal > 0 { + err := db.updateShardObjectCounter(tx, logical, availableDeletedTotal, false) if err != nil { return 0, 0, fmt.Errorf("could not decrease logical object counter: %w", err) } } + if err := db.updateContainerCounter(tx, cnrIDDelta, false); err != nil { + return 0, 0, fmt.Errorf("could not decrease container object counter: %w", err) + } + for _, refNum := range refCounter { if refNum.cur == refNum.all { err := db.deleteObject(tx, refNum.obj, true) if err != nil { - return rawDeleted, availableDeleted, err // maybe log and continue? + return rawDeletedTotal, availableDeletedTotal, err // maybe log and continue? } } } - return rawDeleted, availableDeleted, nil + return rawDeletedTotal, availableDeletedTotal, nil } // delete removes object indexes from the metabase. Counts the references diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index fe8b8873e..e06e57109 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -73,6 +73,7 @@ func (i *InhumeRes) storeDeletionInfo(containerID cid.ID, deletedSize uint64) { Size: deletedSize, CID: containerID, }) + i.availableImhumed++ } // SetAddresses sets a list of object addresses that should be inhumed. @@ -224,7 +225,27 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes } } - return db.updateCounter(tx, logical, res.availableImhumed, false) + return db.applyInhumeResToCounters(tx, res) +} + +func (db *DB) applyInhumeResToCounters(tx *bbolt.Tx, res *InhumeRes) error { + var inhumedCount uint64 + inhumedbyCnr := make(map[cid.ID]ObjectCounters) + for _, dd := range res.deletionDetails { + if v, ok := inhumedbyCnr[dd.CID]; ok { + v.logic++ + inhumedbyCnr[dd.CID] = v + } else { + inhumedbyCnr[dd.CID] = ObjectCounters{logic: 1} + } + inhumedCount++ + } + + if err := db.updateShardObjectCounter(tx, logical, inhumedCount, false); err != nil { + return err + } + + return db.updateContainerCounter(tx, inhumedbyCnr, false) } // getInhumeTargetBucketAndValue return target bucket to store inhume result and value that will be put in the bucket. @@ -279,7 +300,6 @@ func (db *DB) markAsGC(graveyardBKT, garbageBKT *bbolt.Bucket, key []byte) (bool func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Bucket, targetKey []byte, cnr cid.ID, obj *objectSDK.Object, res *InhumeRes) error { containerID, _ := obj.ContainerID() if inGraveyardWithKey(targetKey, graveyardBKT, garbageBKT) == 0 { - res.availableImhumed++ res.storeDeletionInfo(containerID, obj.PayloadSize()) } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 875388e58..2b5c5421d 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -183,16 +183,8 @@ func (db *DB) insertObject(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *o } if !isParent { - err = db.updateCounter(tx, phy, 1, true) - if err != nil { - return fmt.Errorf("could not increase phy object counter: %w", err) - } - - // it is expected that putting an unavailable object is - // impossible and should be handled on the higher levels - err = db.updateCounter(tx, logical, 1, true) - if err != nil { - return fmt.Errorf("could not increase logical object counter: %w", err) + if err = db.incCounters(tx, cnr); err != nil { + return err } } diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index c9d9bb947..c04aff61c 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -19,9 +19,10 @@ var ( graveyardBucketName = []byte{graveyardPrefix} // garbageBucketName stores rows with the objects that should be physically // deleted by the node (Garbage Collector routine). - garbageBucketName = []byte{garbagePrefix} - toMoveItBucketName = []byte{toMoveItPrefix} - containerVolumeBucketName = []byte{containerVolumePrefix} + garbageBucketName = []byte{garbagePrefix} + toMoveItBucketName = []byte{toMoveItPrefix} + containerVolumeBucketName = []byte{containerVolumePrefix} + containerCounterBucketName = []byte{containerCountersPrefix} zeroValue = []byte{0xFF} ) @@ -111,6 +112,11 @@ const ( // Key: split ID // Value: list of object IDs splitPrefix + + // containerCountersPrefix is used for storing container object counters. + // Key: container ID + type + // Value: container size in bytes as little-endian uint64 + containerCountersPrefix ) const ( From 70ab1ebd541f89926b112944bb3eaeee08d33619 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 2 Nov 2023 13:50:52 +0300 Subject: [PATCH 0658/1943] [#763] metrics: Add container_objects_total metric Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/metrics.go | 4 + pkg/local_object_storage/engine/shards.go | 12 +++ pkg/local_object_storage/metabase/delete.go | 78 +++++++-------- pkg/local_object_storage/metabase/inhume.go | 28 +++++- pkg/local_object_storage/shard/delete.go | 1 + pkg/local_object_storage/shard/gc.go | 3 + pkg/local_object_storage/shard/inhume.go | 1 + .../shard/metrics_test.go | 94 ++++++++++++++++++- pkg/local_object_storage/shard/put.go | 2 +- pkg/local_object_storage/shard/shard.go | 34 ++++++- pkg/metrics/engine.go | 46 ++++++++- 12 files changed, 250 insertions(+), 54 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index f8d6cc9b4..5f361cefb 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -516,4 +516,5 @@ const ( RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" FailedToCountWritecacheItems = "failed to count writecache items" AttemtToCloseAlreadyClosedBlobovnicza = "attempt to close an already closed blobovnicza" + FailedToGetContainerCounters = "failed to get container counters values" ) diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index fcac2dc60..363d098f6 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -21,6 +21,10 @@ type MetricRegister interface { ClearErrorCounter(shardID string) DeleteShardMetrics(shardID string) + SetContainerObjectCounter(shardID, contID, objectType string, v uint64) + IncContainerObjectCounter(shardID, contID, objectType string) + SubContainerObjectCounter(shardID, contID, objectType string, v uint64) + WriteCache() metrics.WriteCacheMetrics GC() metrics.GCMetrics } diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 4b9d8752a..2e1cc7515 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -74,6 +74,18 @@ func (m *metricsWithID) DeleteShardMetrics() { m.mw.DeleteShardMetrics(m.id) } +func (m *metricsWithID) SetContainerObjectsCount(cnrID string, objectType string, value uint64) { + m.mw.SetContainerObjectCounter(m.id, cnrID, objectType, value) +} + +func (m *metricsWithID) IncContainerObjectsCount(cnrID string, objectType string) { + m.mw.IncContainerObjectCounter(m.id, cnrID, objectType) +} + +func (m *metricsWithID) SubContainerObjectsCount(cnrID string, objectType string, value uint64) { + m.mw.SubContainerObjectCounter(m.id, cnrID, objectType, value) +} + // AddShard adds a new shard to the storage engine. // // Returns any error encountered that did not allow adding a shard. diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 653a88ae7..5acd71164 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -31,6 +31,7 @@ type DeleteRes struct { availableRemoved uint64 sizes []uint64 availableSizes []uint64 + removedByCnrID map[cid.ID]ObjectCounters } // AvailableObjectsRemoved returns the number of removed available @@ -39,6 +40,11 @@ func (d DeleteRes) AvailableObjectsRemoved() uint64 { return d.availableRemoved } +// RemovedByCnrID returns the number of removed objects by container ID. +func (d DeleteRes) RemovedByCnrID() map[cid.ID]ObjectCounters { + return d.removedByCnrID +} + // RawObjectsRemoved returns the number of removed raw objects. func (d DeleteRes) RawObjectsRemoved() uint64 { return d.rawRemoved @@ -96,15 +102,11 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { return DeleteRes{}, ErrReadOnlyMode } - var rawRemoved uint64 - var availableRemoved uint64 var err error - sizes := make([]uint64, len(prm.addrs)) - availableSizes := make([]uint64, len(prm.addrs)) + var res DeleteRes err = db.boltDB.Update(func(tx *bbolt.Tx) error { - // We need to clear slice because tx can try to execute multiple times. - rawRemoved, availableRemoved, err = db.deleteGroup(tx, prm.addrs, sizes, availableSizes) + res, err = db.deleteGroup(tx, prm.addrs) return err }) if err == nil { @@ -115,91 +117,83 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { storagelog.OpField("metabase DELETE")) } } - return DeleteRes{ - rawRemoved: rawRemoved, - availableRemoved: availableRemoved, - sizes: sizes, - availableSizes: availableSizes, - }, metaerr.Wrap(err) + return res, metaerr.Wrap(err) } // deleteGroup deletes object from the metabase. Handles removal of the // references of the split objects. -// The first return value is a physical objects removed number: physical -// objects that were stored. The second return value is a logical objects -// removed number: objects that were available (without Tombstones, GCMarks -// non-expired, etc.) -func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address, sizes []uint64, availableSizes []uint64) (uint64, uint64, error) { +func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) { + res := DeleteRes{ + sizes: make([]uint64, len(addrs)), + availableSizes: make([]uint64, len(addrs)), + removedByCnrID: make(map[cid.ID]ObjectCounters), + } refCounter := make(referenceCounter, len(addrs)) currEpoch := db.epochState.CurrentEpoch() - cnrIDDelta := make(map[cid.ID]ObjectCounters) - var rawDeletedTotal uint64 - var availableDeletedTotal uint64 - for i := range addrs { removed, available, size, err := db.delete(tx, addrs[i], refCounter, currEpoch) if err != nil { - return 0, 0, err // maybe log and continue? + return DeleteRes{}, err // maybe log and continue? } if removed { - if v, ok := cnrIDDelta[addrs[i].Container()]; ok { + if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { v.phy++ - cnrIDDelta[addrs[i].Container()] = v + res.removedByCnrID[addrs[i].Container()] = v } else { - cnrIDDelta[addrs[i].Container()] = ObjectCounters{ + res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ phy: 1, } } - rawDeletedTotal++ - sizes[i] = size + res.rawRemoved++ + res.sizes[i] = size } if available { - if v, ok := cnrIDDelta[addrs[i].Container()]; ok { + if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { v.logic++ - cnrIDDelta[addrs[i].Container()] = v + res.removedByCnrID[addrs[i].Container()] = v } else { - cnrIDDelta[addrs[i].Container()] = ObjectCounters{ + res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ logic: 1, } } - availableDeletedTotal++ - availableSizes[i] = size + res.availableRemoved++ + res.availableSizes[i] = size } } - if rawDeletedTotal > 0 { - err := db.updateShardObjectCounter(tx, phy, rawDeletedTotal, false) + if res.rawRemoved > 0 { + err := db.updateShardObjectCounter(tx, phy, res.rawRemoved, false) if err != nil { - return 0, 0, fmt.Errorf("could not decrease phy object counter: %w", err) + return DeleteRes{}, fmt.Errorf("could not decrease phy object counter: %w", err) } } - if availableDeletedTotal > 0 { - err := db.updateShardObjectCounter(tx, logical, availableDeletedTotal, false) + if res.availableRemoved > 0 { + err := db.updateShardObjectCounter(tx, logical, res.availableRemoved, false) if err != nil { - return 0, 0, fmt.Errorf("could not decrease logical object counter: %w", err) + return DeleteRes{}, fmt.Errorf("could not decrease logical object counter: %w", err) } } - if err := db.updateContainerCounter(tx, cnrIDDelta, false); err != nil { - return 0, 0, fmt.Errorf("could not decrease container object counter: %w", err) + if err := db.updateContainerCounter(tx, res.removedByCnrID, false); err != nil { + return DeleteRes{}, fmt.Errorf("could not decrease container object counter: %w", err) } for _, refNum := range refCounter { if refNum.cur == refNum.all { err := db.deleteObject(tx, refNum.obj, true) if err != nil { - return rawDeletedTotal, availableDeletedTotal, err // maybe log and continue? + return DeleteRes{}, err // maybe log and continue? } } } - return rawDeletedTotal, availableDeletedTotal, nil + return res, nil } // delete removes object indexes from the metabase. Counts the references diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index e06e57109..cf7921992 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -37,14 +37,21 @@ type DeletionInfo struct { // InhumeRes encapsulates results of Inhume operation. type InhumeRes struct { deletedLockObj []oid.Address - availableImhumed uint64 + availableInhumed uint64 + inhumedByCnrID map[cid.ID]ObjectCounters deletionDetails []DeletionInfo } // AvailableInhumed return number of available object // that have been inhumed. func (i InhumeRes) AvailableInhumed() uint64 { - return i.availableImhumed + return i.availableInhumed +} + +// InhumedByCnrID return number of object +// that have been inhumed by container ID. +func (i InhumeRes) InhumedByCnrID() map[cid.ID]ObjectCounters { + return i.inhumedByCnrID } // DeletedLockObjects returns deleted object of LOCK @@ -73,7 +80,15 @@ func (i *InhumeRes) storeDeletionInfo(containerID cid.ID, deletedSize uint64) { Size: deletedSize, CID: containerID, }) - i.availableImhumed++ + i.availableInhumed++ + if v, ok := i.inhumedByCnrID[containerID]; ok { + v.logic++ + i.inhumedByCnrID[containerID] = v + } else { + i.inhumedByCnrID[containerID] = ObjectCounters{ + logic: 1, + } + } } // SetAddresses sets a list of object addresses that should be inhumed. @@ -123,7 +138,7 @@ var ErrLockObjectRemoval = logicerr.New("lock object removal") // // NOTE: Marks any object with GC mark (despite any prohibitions on operations // with that object) if WithForceGCMark option has been provided. -func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err error) { +func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { var ( startedAt = time.Now() success = false @@ -143,8 +158,11 @@ func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err err return InhumeRes{}, ErrReadOnlyMode } + res := InhumeRes{ + inhumedByCnrID: make(map[cid.ID]ObjectCounters), + } currEpoch := db.epochState.CurrentEpoch() - err = db.boltDB.Update(func(tx *bbolt.Tx) error { + err := db.boltDB.Update(func(tx *bbolt.Tx) error { return db.inhumeTx(tx, currEpoch, prm, &res) }) success = err == nil diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index e24bd8541..663d781ad 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -118,6 +118,7 @@ func (s *Shard) deleteFromMetabase(ctx context.Context, addr oid.Address) error } s.decObjectCounterBy(physical, res.RawObjectsRemoved()) s.decObjectCounterBy(logical, res.AvailableObjectsRemoved()) + s.decContainerObjectCounter(res.RemovedByCnrID()) removedPayload := res.RemovedPhysicalObjectSizes()[0] logicalRemovedPayload := res.RemovedLogicalObjectSizes()[0] if logicalRemovedPayload > 0 { diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 0ce99b475..346903c5c 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -416,6 +416,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeRegular) s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.decContainerObjectCounter(res.InhumedByCnrID()) i := 0 for i < res.GetDeletionInfoLength() { @@ -629,6 +630,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeTombstone) s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.decContainerObjectCounter(res.InhumedByCnrID()) i := 0 for i < res.GetDeletionInfoLength() { @@ -675,6 +677,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeLock) s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.decContainerObjectCounter(res.InhumedByCnrID()) i := 0 for i < res.GetDeletionInfoLength() { diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index a5f8960c3..335df82f0 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -122,6 +122,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { s.m.RUnlock() s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.decContainerObjectCounter(res.InhumedByCnrID()) i := 0 for i < res.GetDeletionInfoLength() { diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 7724aa222..1e9bbf9fc 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -14,6 +14,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" @@ -23,6 +24,7 @@ type metricsStore struct { mtx sync.Mutex objCounters map[string]uint64 cnrSize map[string]int64 + cnrCount map[string]uint64 pldSize int64 mode mode.Mode errCounter int64 @@ -126,6 +128,39 @@ func (m *metricsStore) DeleteShardMetrics() { m.errCounter = 0 } +func (m *metricsStore) SetContainerObjectsCount(cnrID string, objectType string, value uint64) { + m.mtx.Lock() + defer m.mtx.Unlock() + m.cnrCount[cnrID+objectType] = value +} + +func (m *metricsStore) IncContainerObjectsCount(cnrID string, objectType string) { + m.mtx.Lock() + defer m.mtx.Unlock() + m.cnrCount[cnrID+objectType]++ +} + +func (m *metricsStore) SubContainerObjectsCount(cnrID string, objectType string, value uint64) { + m.mtx.Lock() + defer m.mtx.Unlock() + existed := m.cnrCount[cnrID+objectType] + if existed < value { + panic("existed value smaller than value to sustract") + } + if existed == value { + delete(m.cnrCount, cnrID+objectType) + } else { + m.cnrCount[cnrID+objectType] -= value + } +} + +func (m *metricsStore) getContainerCount(cnrID, objectType string) (uint64, bool) { + m.mtx.Lock() + defer m.mtx.Unlock() + v, ok := m.cnrCount[cnrID+objectType] + return v, ok +} + func TestCounters(t *testing.T) { t.Parallel() @@ -143,21 +178,37 @@ func TestCounters(t *testing.T) { oo[i] = testutil.GenerateObject() } + cc := meta.ContainerCounters{Logical: make(map[cid.ID]uint64), Physical: make(map[cid.ID]uint64)} + t.Run("defaults", func(t *testing.T) { require.Zero(t, mm.getObjectCounter(physical)) require.Zero(t, mm.getObjectCounter(logical)) require.Empty(t, mm.containerSizes()) require.Zero(t, mm.payloadSize()) + + for _, obj := range oo { + contID, _ := obj.ContainerID() + v, ok := mm.getContainerCount(contID.EncodeToString(), physical) + require.Zero(t, v) + require.False(t, ok) + v, ok = mm.getContainerCount(contID.EncodeToString(), logical) + require.Zero(t, v) + require.False(t, ok) + } }) var totalPayload int64 expectedLogicalSizes := make(map[string]int64) + expectedLogCC := make(map[cid.ID]uint64) + expectedPhyCC := make(map[cid.ID]uint64) for i := range oo { cnr, _ := oo[i].ContainerID() oSize := int64(oo[i].PayloadSize()) expectedLogicalSizes[cnr.EncodeToString()] += oSize totalPayload += oSize + expectedLogCC[cnr]++ + expectedPhyCC[cnr]++ } var prm PutPrm @@ -174,6 +225,11 @@ func TestCounters(t *testing.T) { require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) + cc, err := sh.metaBase.ContainerCounters(context.Background()) + require.NoError(t, err) + require.Equal(t, expectedLogCC, cc.Logical) + require.Equal(t, expectedPhyCC, cc.Physical) + t.Run("inhume_GC", func(t *testing.T) { var prm InhumePrm inhumedNumber := objNumber / 4 @@ -187,6 +243,11 @@ func TestCounters(t *testing.T) { cid, ok := oo[i].ContainerID() require.True(t, ok) expectedLogicalSizes[cid.EncodeToString()] -= int64(oo[i].PayloadSize()) + + expectedLogCC[cid]-- + if expectedLogCC[cid] == 0 { + delete(expectedLogCC, cid) + } } require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical)) @@ -194,6 +255,11 @@ func TestCounters(t *testing.T) { require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) + cc, err := sh.metaBase.ContainerCounters(context.Background()) + require.NoError(t, err) + require.Equal(t, expectedLogCC, cc.Logical) + require.Equal(t, expectedPhyCC, cc.Physical) + oo = oo[inhumedNumber:] }) @@ -214,6 +280,11 @@ func TestCounters(t *testing.T) { cid, ok := oo[i].ContainerID() require.True(t, ok) expectedLogicalSizes[cid.EncodeToString()] -= int64(oo[i].PayloadSize()) + + expectedLogCC[cid]-- + if expectedLogCC[cid] == 0 { + delete(expectedLogCC, cid) + } } require.Equal(t, phy, mm.getObjectCounter(physical)) @@ -221,6 +292,11 @@ func TestCounters(t *testing.T) { require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) + cc, err = sh.metaBase.ContainerCounters(context.Background()) + require.NoError(t, err) + require.Equal(t, expectedLogCC, cc.Logical) + require.Equal(t, expectedPhyCC, cc.Physical) + oo = oo[inhumedNumber:] }) @@ -245,9 +321,24 @@ func TestCounters(t *testing.T) { cnr, _ := oo[i].ContainerID() expectedLogicalSizes[cnr.EncodeToString()] -= int64(removedPayload) + + expectedLogCC[cnr]-- + if expectedLogCC[cnr] == 0 { + delete(expectedLogCC, cnr) + } + + expectedPhyCC[cnr]-- + if expectedPhyCC[cnr] == 0 { + delete(expectedPhyCC, cnr) + } } require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload-int64(totalRemovedpayload), mm.payloadSize()) + + cc, err = sh.metaBase.ContainerCounters(context.Background()) + require.NoError(t, err) + require.Equal(t, expectedLogCC, cc.Logical) + require.Equal(t, expectedPhyCC, cc.Physical) }) } @@ -268,7 +359,8 @@ func shardWithMetrics(t *testing.T, path string) (*Shard, *metricsStore) { "phy": 0, "logic": 0, }, - cnrSize: make(map[string]int64), + cnrSize: make(map[string]int64), + cnrCount: make(map[string]uint64), } sh := New( diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index 688b7aae7..10983d041 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -90,7 +90,7 @@ func (s *Shard) Put(ctx context.Context, prm PutPrm) (PutRes, error) { return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err) } - s.incObjectCounter() + s.incObjectCounter(putPrm.Address.Container()) s.addToPayloadSize(int64(prm.obj.PayloadSize())) s.addToContainerSize(putPrm.Address.Container().EncodeToString(), int64(prm.obj.PayloadSize())) } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 00f4fbb9e..07b774022 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -18,6 +18,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -85,6 +86,12 @@ type MetricsWriter interface { ClearErrorCounter() // DeleteShardMetrics deletes shard metrics from registry. DeleteShardMetrics() + // SetContainerObjectsCount sets container object count. + SetContainerObjectsCount(cnrID string, objectType string, value uint64) + // IncContainerObjectsCount increments container object count. + IncContainerObjectsCount(cnrID string, objectType string) + // SubContainerObjectsCount subtracts container object count. + SubContainerObjectsCount(cnrID string, objectType string, value uint64) } type cfg struct { @@ -425,15 +432,29 @@ func (s *Shard) updateMetrics(ctx context.Context) { } s.metricsWriter.AddToPayloadSize(int64(totalPayload)) + + contCount, err := s.metaBase.ContainerCounters(ctx) + if err != nil { + s.log.Warn(logs.FailedToGetContainerCounters, zap.Error(err)) + return + } + for contID, count := range contCount.Physical { + s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), physical, count) + } + for contID, count := range contCount.Logical { + s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), logical, count) + } } } // incObjectCounter increment both physical and logical object // counters. -func (s *Shard) incObjectCounter() { +func (s *Shard) incObjectCounter(cnrID cid.ID) { if s.cfg.metricsWriter != nil { s.cfg.metricsWriter.IncObjectCounter(physical) s.cfg.metricsWriter.IncObjectCounter(logical) + s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), physical) + s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), logical) } } @@ -443,6 +464,17 @@ func (s *Shard) decObjectCounterBy(typ string, v uint64) { } } +func (s *Shard) decContainerObjectCounter(byCnr map[cid.ID]meta.ObjectCounters) { + if s.cfg.metricsWriter == nil { + return + } + + for cnrID, count := range byCnr { + s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), physical, count.Phy()) + s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), logical, count.Logic()) + } +} + func (s *Shard) addToContainerSize(cnr string, size int64) { if s.cfg.metricsWriter != nil { s.cfg.metricsWriter.AddToContainerSize(cnr, size) diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 23d799e28..609c30f97 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -18,6 +18,9 @@ type EngineMetrics interface { SetObjectCounter(shardID, objectType string, v uint64) AddToPayloadCounter(shardID string, size int64) SetMode(shardID string, mode mode.Mode) + SetContainerObjectCounter(shardID, contID, objectType string, v uint64) + IncContainerObjectCounter(shardID, contID, objectType string) + SubContainerObjectCounter(shardID, contID, objectType string, v uint64) WriteCache() WriteCacheMetrics GC() GCMetrics @@ -30,6 +33,7 @@ type engineMetrics struct { payloadSize *prometheus.GaugeVec errorCounter *prometheus.GaugeVec mode *shardIDModeValue + contObjCounter *prometheus.GaugeVec gc *gcMetrics writeCache *writeCacheMetrics @@ -46,10 +50,13 @@ func newEngineMetrics() *engineMetrics { Name: "request_duration_seconds", Help: "Duration of Engine requests", }, []string{methodLabel}), - objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards", []string{shardIDLabel, typeLabel}), - gc: newGCMetrics(), - writeCache: newWriteCacheMetrics(), - mode: newShardIDMode(engineSubsystem, "mode_info", "Shard mode"), + objectCounter: newEngineGaugeVector("objects_total", + "Objects counters per shards. DEPRECATED: Will be deleted in next releasese, use frostfs_node_engine_container_objects_total metric.", + []string{shardIDLabel, typeLabel}), + gc: newGCMetrics(), + writeCache: newWriteCacheMetrics(), + mode: newShardIDMode(engineSubsystem, "mode_info", "Shard mode"), + contObjCounter: newEngineGaugeVector("container_objects_total", "Count of objects for each container", []string{shardIDLabel, containerIDLabelKey, typeLabel}), } } @@ -88,6 +95,7 @@ func (m *engineMetrics) DeleteShardMetrics(shardID string) { m.errorCounter.Delete(prometheus.Labels{shardIDLabel: shardID}) m.payloadSize.Delete(prometheus.Labels{shardIDLabel: shardID}) m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.contObjCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) m.mode.Delete(shardID) } @@ -109,6 +117,36 @@ func (m *engineMetrics) SetObjectCounter(shardID, objectType string, v uint64) { ).Set(float64(v)) } +func (m *engineMetrics) SetContainerObjectCounter(shardID, contID, objectType string, v uint64) { + m.contObjCounter.With( + prometheus.Labels{ + shardIDLabel: shardID, + containerIDLabelKey: contID, + typeLabel: objectType, + }, + ).Set(float64(v)) +} + +func (m *engineMetrics) IncContainerObjectCounter(shardID, contID, objectType string) { + m.contObjCounter.With( + prometheus.Labels{ + shardIDLabel: shardID, + containerIDLabelKey: contID, + typeLabel: objectType, + }, + ).Inc() +} + +func (m *engineMetrics) SubContainerObjectCounter(shardID, contID, objectType string, v uint64) { + m.contObjCounter.With( + prometheus.Labels{ + shardIDLabel: shardID, + containerIDLabelKey: contID, + typeLabel: objectType, + }, + ).Sub(float64(v)) +} + func (m *engineMetrics) SetMode(shardID string, mode mode.Mode) { m.mode.SetMode(shardID, mode.String()) } From 3a2c319b87adcfbb371cf6acee6df3807d90cadc Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 31 Oct 2023 18:36:42 +0300 Subject: [PATCH 0659/1943] [#770] control: Generate gRPC methods to manipulate APE chains * Define new types and gRPC methods to manipulate APE chains in control service. * Stub gRPC handlers for the generated methods. Signed-off-by: Airat Arifullin --- pkg/services/control/server/policy_engine.go | 24 + pkg/services/control/service.pb.go | 1740 +++++++++++++++--- pkg/services/control/service.proto | 114 ++ pkg/services/control/service_frostfs.pb.go | 694 +++++++ pkg/services/control/service_grpc.pb.go | 156 ++ 5 files changed, 2465 insertions(+), 263 deletions(-) create mode 100644 pkg/services/control/server/policy_engine.go diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go new file mode 100644 index 000000000..058839bfb --- /dev/null +++ b/pkg/services/control/server/policy_engine.go @@ -0,0 +1,24 @@ +package control + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" +) + +func (s *Server) AddChainLocalOverride(ctx context.Context, req *control.AddChainLocalOverrideRequest) (*control.AddChainLocalOverrideResponse, error) { + return nil, fmt.Errorf("not implemented yet") +} + +func (s *Server) GetChainLocalOverride(ctx context.Context, req *control.GetChainLocalOverrideRequest) (*control.GetChainLocalOverrideResponse, error) { + return nil, fmt.Errorf("not implemented yet") +} + +func (s *Server) ListChainLocalOverrides(ctx context.Context, req *control.ListChainLocalOverridesRequest) (*control.ListChainLocalOverridesResponse, error) { + return nil, fmt.Errorf("not implemented yet") +} + +func (s *Server) RemoveChainLocalOverride(ctx context.Context, req *control.RemoveChainLocalOverrideRequest) (*control.RemoveChainLocalOverrideResponse, error) { + return nil, fmt.Errorf("not implemented yet") +} diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 443afb1aa..547ea7e7e 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -1438,6 +1438,452 @@ func (x *StopShardEvacuationResponse) GetSignature() *Signature { return nil } +// AddChainLocalOverride request. +type AddChainLocalOverrideRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *AddChainLocalOverrideRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *AddChainLocalOverrideRequest) Reset() { + *x = AddChainLocalOverrideRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AddChainLocalOverrideRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AddChainLocalOverrideRequest) ProtoMessage() {} + +func (x *AddChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[24] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AddChainLocalOverrideRequest.ProtoReflect.Descriptor instead. +func (*AddChainLocalOverrideRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24} +} + +func (x *AddChainLocalOverrideRequest) GetBody() *AddChainLocalOverrideRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *AddChainLocalOverrideRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// AddChainLocalOverride response. +type AddChainLocalOverrideResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *AddChainLocalOverrideResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *AddChainLocalOverrideResponse) Reset() { + *x = AddChainLocalOverrideResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AddChainLocalOverrideResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AddChainLocalOverrideResponse) ProtoMessage() {} + +func (x *AddChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[25] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AddChainLocalOverrideResponse.ProtoReflect.Descriptor instead. +func (*AddChainLocalOverrideResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25} +} + +func (x *AddChainLocalOverrideResponse) GetBody() *AddChainLocalOverrideResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *AddChainLocalOverrideResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// GetChainLocalOverride request. +type GetChainLocalOverrideRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *GetChainLocalOverrideRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *GetChainLocalOverrideRequest) Reset() { + *x = GetChainLocalOverrideRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetChainLocalOverrideRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetChainLocalOverrideRequest) ProtoMessage() {} + +func (x *GetChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[26] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetChainLocalOverrideRequest.ProtoReflect.Descriptor instead. +func (*GetChainLocalOverrideRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26} +} + +func (x *GetChainLocalOverrideRequest) GetBody() *GetChainLocalOverrideRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *GetChainLocalOverrideRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// GetChainLocalOverride response. +type GetChainLocalOverrideResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *GetChainLocalOverrideResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *GetChainLocalOverrideResponse) Reset() { + *x = GetChainLocalOverrideResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetChainLocalOverrideResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetChainLocalOverrideResponse) ProtoMessage() {} + +func (x *GetChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[27] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetChainLocalOverrideResponse.ProtoReflect.Descriptor instead. +func (*GetChainLocalOverrideResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27} +} + +func (x *GetChainLocalOverrideResponse) GetBody() *GetChainLocalOverrideResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *GetChainLocalOverrideResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// ListChainLocalOverrides request. +type ListChainLocalOverridesRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *ListChainLocalOverridesRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *ListChainLocalOverridesRequest) Reset() { + *x = ListChainLocalOverridesRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListChainLocalOverridesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListChainLocalOverridesRequest) ProtoMessage() {} + +func (x *ListChainLocalOverridesRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[28] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListChainLocalOverridesRequest.ProtoReflect.Descriptor instead. +func (*ListChainLocalOverridesRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28} +} + +func (x *ListChainLocalOverridesRequest) GetBody() *ListChainLocalOverridesRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *ListChainLocalOverridesRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// ListChainLocalOverrides response. +type ListChainLocalOverridesResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *ListChainLocalOverridesResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *ListChainLocalOverridesResponse) Reset() { + *x = ListChainLocalOverridesResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListChainLocalOverridesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListChainLocalOverridesResponse) ProtoMessage() {} + +func (x *ListChainLocalOverridesResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[29] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListChainLocalOverridesResponse.ProtoReflect.Descriptor instead. +func (*ListChainLocalOverridesResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29} +} + +func (x *ListChainLocalOverridesResponse) GetBody() *ListChainLocalOverridesResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *ListChainLocalOverridesResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +type RemoveChainLocalOverrideRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *RemoveChainLocalOverrideRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *RemoveChainLocalOverrideRequest) Reset() { + *x = RemoveChainLocalOverrideRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveChainLocalOverrideRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveChainLocalOverrideRequest) ProtoMessage() {} + +func (x *RemoveChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[30] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveChainLocalOverrideRequest.ProtoReflect.Descriptor instead. +func (*RemoveChainLocalOverrideRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30} +} + +func (x *RemoveChainLocalOverrideRequest) GetBody() *RemoveChainLocalOverrideRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *RemoveChainLocalOverrideRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +type RemoveChainLocalOverrideResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *RemoveChainLocalOverrideResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *RemoveChainLocalOverrideResponse) Reset() { + *x = RemoveChainLocalOverrideResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveChainLocalOverrideResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveChainLocalOverrideResponse) ProtoMessage() {} + +func (x *RemoveChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[31] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveChainLocalOverrideResponse.ProtoReflect.Descriptor instead. +func (*RemoveChainLocalOverrideResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31} +} + +func (x *RemoveChainLocalOverrideResponse) GetBody() *RemoveChainLocalOverrideResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *RemoveChainLocalOverrideResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + // Health check request body. type HealthCheckRequest_Body struct { state protoimpl.MessageState @@ -1448,7 +1894,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1461,7 +1907,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1492,7 +1938,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1505,7 +1951,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1556,7 +2002,7 @@ type SetNetmapStatusRequest_Body struct { func (x *SetNetmapStatusRequest_Body) Reset() { *x = SetNetmapStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1569,7 +2015,7 @@ func (x *SetNetmapStatusRequest_Body) String() string { func (*SetNetmapStatusRequest_Body) ProtoMessage() {} func (x *SetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1609,7 +2055,7 @@ type SetNetmapStatusResponse_Body struct { func (x *SetNetmapStatusResponse_Body) Reset() { *x = SetNetmapStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1622,7 +2068,7 @@ func (x *SetNetmapStatusResponse_Body) String() string { func (*SetNetmapStatusResponse_Body) ProtoMessage() {} func (x *SetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1652,7 +2098,7 @@ type DropObjectsRequest_Body struct { func (x *DropObjectsRequest_Body) Reset() { *x = DropObjectsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1665,7 +2111,7 @@ func (x *DropObjectsRequest_Body) String() string { func (*DropObjectsRequest_Body) ProtoMessage() {} func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1698,7 +2144,7 @@ type DropObjectsResponse_Body struct { func (x *DropObjectsResponse_Body) Reset() { *x = DropObjectsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1711,7 +2157,7 @@ func (x *DropObjectsResponse_Body) String() string { func (*DropObjectsResponse_Body) ProtoMessage() {} func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1737,7 +2183,7 @@ type ListShardsRequest_Body struct { func (x *ListShardsRequest_Body) Reset() { *x = ListShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1750,7 +2196,7 @@ func (x *ListShardsRequest_Body) String() string { func (*ListShardsRequest_Body) ProtoMessage() {} func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1779,7 +2225,7 @@ type ListShardsResponse_Body struct { func (x *ListShardsResponse_Body) Reset() { *x = ListShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1792,7 +2238,7 @@ func (x *ListShardsResponse_Body) String() string { func (*ListShardsResponse_Body) ProtoMessage() {} func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1832,7 +2278,7 @@ type SetShardModeRequest_Body struct { func (x *SetShardModeRequest_Body) Reset() { *x = SetShardModeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1845,7 +2291,7 @@ func (x *SetShardModeRequest_Body) String() string { func (*SetShardModeRequest_Body) ProtoMessage() {} func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1892,7 +2338,7 @@ type SetShardModeResponse_Body struct { func (x *SetShardModeResponse_Body) Reset() { *x = SetShardModeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1905,7 +2351,7 @@ func (x *SetShardModeResponse_Body) String() string { func (*SetShardModeResponse_Body) ProtoMessage() {} func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1936,7 +2382,7 @@ type SynchronizeTreeRequest_Body struct { func (x *SynchronizeTreeRequest_Body) Reset() { *x = SynchronizeTreeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1949,7 +2395,7 @@ func (x *SynchronizeTreeRequest_Body) String() string { func (*SynchronizeTreeRequest_Body) ProtoMessage() {} func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1996,7 +2442,7 @@ type SynchronizeTreeResponse_Body struct { func (x *SynchronizeTreeResponse_Body) Reset() { *x = SynchronizeTreeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2009,7 +2455,7 @@ func (x *SynchronizeTreeResponse_Body) String() string { func (*SynchronizeTreeResponse_Body) ProtoMessage() {} func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2040,7 +2486,7 @@ type EvacuateShardRequest_Body struct { func (x *EvacuateShardRequest_Body) Reset() { *x = EvacuateShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2053,7 +2499,7 @@ func (x *EvacuateShardRequest_Body) String() string { func (*EvacuateShardRequest_Body) ProtoMessage() {} func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2095,7 +2541,7 @@ type EvacuateShardResponse_Body struct { func (x *EvacuateShardResponse_Body) Reset() { *x = EvacuateShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2108,7 +2554,7 @@ func (x *EvacuateShardResponse_Body) String() string { func (*EvacuateShardResponse_Body) ProtoMessage() {} func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2144,7 +2590,7 @@ type FlushCacheRequest_Body struct { func (x *FlushCacheRequest_Body) Reset() { *x = FlushCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2157,7 +2603,7 @@ func (x *FlushCacheRequest_Body) String() string { func (*FlushCacheRequest_Body) ProtoMessage() {} func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2190,7 +2636,7 @@ type FlushCacheResponse_Body struct { func (x *FlushCacheResponse_Body) Reset() { *x = FlushCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2203,7 +2649,7 @@ func (x *FlushCacheResponse_Body) String() string { func (*FlushCacheResponse_Body) ProtoMessage() {} func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2234,7 +2680,7 @@ type DoctorRequest_Body struct { func (x *DoctorRequest_Body) Reset() { *x = DoctorRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2247,7 +2693,7 @@ func (x *DoctorRequest_Body) String() string { func (*DoctorRequest_Body) ProtoMessage() {} func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2287,7 +2733,7 @@ type DoctorResponse_Body struct { func (x *DoctorResponse_Body) Reset() { *x = DoctorResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2300,7 +2746,7 @@ func (x *DoctorResponse_Body) String() string { func (*DoctorResponse_Body) ProtoMessage() {} func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2331,7 +2777,7 @@ type StartShardEvacuationRequest_Body struct { func (x *StartShardEvacuationRequest_Body) Reset() { *x = StartShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2344,7 +2790,7 @@ func (x *StartShardEvacuationRequest_Body) String() string { func (*StartShardEvacuationRequest_Body) ProtoMessage() {} func (x *StartShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2384,7 +2830,7 @@ type StartShardEvacuationResponse_Body struct { func (x *StartShardEvacuationResponse_Body) Reset() { *x = StartShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2397,7 +2843,7 @@ func (x *StartShardEvacuationResponse_Body) String() string { func (*StartShardEvacuationResponse_Body) ProtoMessage() {} func (x *StartShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2423,7 +2869,7 @@ type GetShardEvacuationStatusRequest_Body struct { func (x *GetShardEvacuationStatusRequest_Body) Reset() { *x = GetShardEvacuationStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2436,7 +2882,7 @@ func (x *GetShardEvacuationStatusRequest_Body) String() string { func (*GetShardEvacuationStatusRequest_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2481,7 +2927,7 @@ type GetShardEvacuationStatusResponse_Body struct { func (x *GetShardEvacuationStatusResponse_Body) Reset() { *x = GetShardEvacuationStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2494,7 +2940,7 @@ func (x *GetShardEvacuationStatusResponse_Body) String() string { func (*GetShardEvacuationStatusResponse_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2585,7 +3031,7 @@ type GetShardEvacuationStatusResponse_Body_UnixTimestamp struct { func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) Reset() { *x = GetShardEvacuationStatusResponse_Body_UnixTimestamp{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2598,7 +3044,7 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) String() string { func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2633,7 +3079,7 @@ type GetShardEvacuationStatusResponse_Body_Duration struct { func (x *GetShardEvacuationStatusResponse_Body_Duration) Reset() { *x = GetShardEvacuationStatusResponse_Body_Duration{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2646,7 +3092,7 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) String() string { func (*GetShardEvacuationStatusResponse_Body_Duration) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_Duration) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2679,7 +3125,7 @@ type StopShardEvacuationRequest_Body struct { func (x *StopShardEvacuationRequest_Body) Reset() { *x = StopShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2692,7 +3138,7 @@ func (x *StopShardEvacuationRequest_Body) String() string { func (*StopShardEvacuationRequest_Body) ProtoMessage() {} func (x *StopShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2718,7 +3164,7 @@ type StopShardEvacuationResponse_Body struct { func (x *StopShardEvacuationResponse_Body) Reset() { *x = StopShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2731,7 +3177,7 @@ func (x *StopShardEvacuationResponse_Body) String() string { func (*StopShardEvacuationResponse_Body) ProtoMessage() {} func (x *StopShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2747,6 +3193,416 @@ func (*StopShardEvacuationResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0} } +type AddChainLocalOverrideRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Container id for which the overrides are applied. + ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` + // Serialized rule chain. + Chain []byte `protobuf:"bytes,2,opt,name=chain,proto3" json:"chain,omitempty"` +} + +func (x *AddChainLocalOverrideRequest_Body) Reset() { + *x = AddChainLocalOverrideRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[58] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AddChainLocalOverrideRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AddChainLocalOverrideRequest_Body) ProtoMessage() {} + +func (x *AddChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[58] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AddChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. +func (*AddChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24, 0} +} + +func (x *AddChainLocalOverrideRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} + +func (x *AddChainLocalOverrideRequest_Body) GetChain() []byte { + if x != nil { + return x.Chain + } + return nil +} + +type AddChainLocalOverrideResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Chain ID assigned for the added rule chain. + ChainId string `protobuf:"bytes,1,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` +} + +func (x *AddChainLocalOverrideResponse_Body) Reset() { + *x = AddChainLocalOverrideResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[59] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AddChainLocalOverrideResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AddChainLocalOverrideResponse_Body) ProtoMessage() {} + +func (x *AddChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[59] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AddChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. +func (*AddChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25, 0} +} + +func (x *AddChainLocalOverrideResponse_Body) GetChainId() string { + if x != nil { + return x.ChainId + } + return "" +} + +type GetChainLocalOverrideRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Container id for which the overrides are defined. + ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` + // Chain ID assigned for the added rule chain. + ChainId string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` +} + +func (x *GetChainLocalOverrideRequest_Body) Reset() { + *x = GetChainLocalOverrideRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[60] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetChainLocalOverrideRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetChainLocalOverrideRequest_Body) ProtoMessage() {} + +func (x *GetChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[60] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. +func (*GetChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26, 0} +} + +func (x *GetChainLocalOverrideRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} + +func (x *GetChainLocalOverrideRequest_Body) GetChainId() string { + if x != nil { + return x.ChainId + } + return "" +} + +type GetChainLocalOverrideResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Serialized rule chain. + Chain []byte `protobuf:"bytes,1,opt,name=chain,proto3" json:"chain,omitempty"` +} + +func (x *GetChainLocalOverrideResponse_Body) Reset() { + *x = GetChainLocalOverrideResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[61] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetChainLocalOverrideResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetChainLocalOverrideResponse_Body) ProtoMessage() {} + +func (x *GetChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[61] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. +func (*GetChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27, 0} +} + +func (x *GetChainLocalOverrideResponse_Body) GetChain() []byte { + if x != nil { + return x.Chain + } + return nil +} + +type ListChainLocalOverridesRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Container id for which the overrides are defined. + ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` +} + +func (x *ListChainLocalOverridesRequest_Body) Reset() { + *x = ListChainLocalOverridesRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[62] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListChainLocalOverridesRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListChainLocalOverridesRequest_Body) ProtoMessage() {} + +func (x *ListChainLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[62] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListChainLocalOverridesRequest_Body.ProtoReflect.Descriptor instead. +func (*ListChainLocalOverridesRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28, 0} +} + +func (x *ListChainLocalOverridesRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} + +type ListChainLocalOverridesResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The list of serialized rule chain. + Chains [][]byte `protobuf:"bytes,1,rep,name=chains,proto3" json:"chains,omitempty"` +} + +func (x *ListChainLocalOverridesResponse_Body) Reset() { + *x = ListChainLocalOverridesResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[63] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListChainLocalOverridesResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListChainLocalOverridesResponse_Body) ProtoMessage() {} + +func (x *ListChainLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[63] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListChainLocalOverridesResponse_Body.ProtoReflect.Descriptor instead. +func (*ListChainLocalOverridesResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29, 0} +} + +func (x *ListChainLocalOverridesResponse_Body) GetChains() [][]byte { + if x != nil { + return x.Chains + } + return nil +} + +type RemoveChainLocalOverrideRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Container id for which the overrides are defined. + ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` + // Chain ID assigned for the added rule chain. + ChainId string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` +} + +func (x *RemoveChainLocalOverrideRequest_Body) Reset() { + *x = RemoveChainLocalOverrideRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[64] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveChainLocalOverrideRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveChainLocalOverrideRequest_Body) ProtoMessage() {} + +func (x *RemoveChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[64] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. +func (*RemoveChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30, 0} +} + +func (x *RemoveChainLocalOverrideRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} + +func (x *RemoveChainLocalOverrideRequest_Body) GetChainId() string { + if x != nil { + return x.ChainId + } + return "" +} + +type RemoveChainLocalOverrideResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Removed bool `protobuf:"varint,1,opt,name=removed,proto3" json:"removed,omitempty"` +} + +func (x *RemoveChainLocalOverrideResponse_Body) Reset() { + *x = RemoveChainLocalOverrideResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[65] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveChainLocalOverrideResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveChainLocalOverrideResponse_Body) ProtoMessage() {} + +func (x *RemoveChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[65] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. +func (*RemoveChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31, 0} +} + +func (x *RemoveChainLocalOverrideResponse_Body) GetRemoved() bool { + if x != nil { + return x.Removed + } + return false +} + var File_pkg_services_control_service_proto protoreflect.FileDescriptor var file_pkg_services_control_service_proto_rawDesc = []byte{ @@ -3049,75 +3905,201 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xee, - 0x07, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, - 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xd1, + 0x01, 0x0a, 0x1c, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x3f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x22, 0xb5, 0x01, 0x0a, 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xd6, 0x01, 0x0a, 0x1c, 0x47, + 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x44, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, + 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, + 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xbf, 0x01, 0x0a, 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x29, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, 0x73, + 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x73, 0x22, 0xdc, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x44, 0x0a, 0x04, 0x42, 0x6f, + 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, + 0x22, 0xba, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x20, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x32, 0x9d, 0x0b, + 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, + 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, + 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, + 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, - 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, - 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, - 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, - 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, - 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, - 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, - 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, - 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, - 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, - 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, - 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, - 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, - 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, - 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, - 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, + 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, + 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, + 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, + 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, + 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, + 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, + 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, + 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, + 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, + 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, + 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, + 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, + 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -3133,7 +4115,7 @@ func file_pkg_services_control_service_proto_rawDescGZIP() []byte { } var file_pkg_services_control_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 50) +var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 66) var file_pkg_services_control_service_proto_goTypes = []interface{}{ (GetShardEvacuationStatusResponse_Body_Status)(0), // 0: control.GetShardEvacuationStatusResponse.Body.Status (*HealthCheckRequest)(nil), // 1: control.HealthCheckRequest @@ -3160,124 +4142,164 @@ var file_pkg_services_control_service_proto_goTypes = []interface{}{ (*GetShardEvacuationStatusResponse)(nil), // 22: control.GetShardEvacuationStatusResponse (*StopShardEvacuationRequest)(nil), // 23: control.StopShardEvacuationRequest (*StopShardEvacuationResponse)(nil), // 24: control.StopShardEvacuationResponse - (*HealthCheckRequest_Body)(nil), // 25: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 26: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 27: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 28: control.SetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 29: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 30: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 31: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 32: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 33: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 34: control.SetShardModeResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 35: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 36: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 37: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 38: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 39: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 40: control.FlushCacheResponse.Body - (*DoctorRequest_Body)(nil), // 41: control.DoctorRequest.Body - (*DoctorResponse_Body)(nil), // 42: control.DoctorResponse.Body - (*StartShardEvacuationRequest_Body)(nil), // 43: control.StartShardEvacuationRequest.Body - (*StartShardEvacuationResponse_Body)(nil), // 44: control.StartShardEvacuationResponse.Body - (*GetShardEvacuationStatusRequest_Body)(nil), // 45: control.GetShardEvacuationStatusRequest.Body - (*GetShardEvacuationStatusResponse_Body)(nil), // 46: control.GetShardEvacuationStatusResponse.Body - (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 47: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 48: control.GetShardEvacuationStatusResponse.Body.Duration - (*StopShardEvacuationRequest_Body)(nil), // 49: control.StopShardEvacuationRequest.Body - (*StopShardEvacuationResponse_Body)(nil), // 50: control.StopShardEvacuationResponse.Body - (*Signature)(nil), // 51: control.Signature - (NetmapStatus)(0), // 52: control.NetmapStatus - (HealthStatus)(0), // 53: control.HealthStatus - (*ShardInfo)(nil), // 54: control.ShardInfo - (ShardMode)(0), // 55: control.ShardMode + (*AddChainLocalOverrideRequest)(nil), // 25: control.AddChainLocalOverrideRequest + (*AddChainLocalOverrideResponse)(nil), // 26: control.AddChainLocalOverrideResponse + (*GetChainLocalOverrideRequest)(nil), // 27: control.GetChainLocalOverrideRequest + (*GetChainLocalOverrideResponse)(nil), // 28: control.GetChainLocalOverrideResponse + (*ListChainLocalOverridesRequest)(nil), // 29: control.ListChainLocalOverridesRequest + (*ListChainLocalOverridesResponse)(nil), // 30: control.ListChainLocalOverridesResponse + (*RemoveChainLocalOverrideRequest)(nil), // 31: control.RemoveChainLocalOverrideRequest + (*RemoveChainLocalOverrideResponse)(nil), // 32: control.RemoveChainLocalOverrideResponse + (*HealthCheckRequest_Body)(nil), // 33: control.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 34: control.HealthCheckResponse.Body + (*SetNetmapStatusRequest_Body)(nil), // 35: control.SetNetmapStatusRequest.Body + (*SetNetmapStatusResponse_Body)(nil), // 36: control.SetNetmapStatusResponse.Body + (*DropObjectsRequest_Body)(nil), // 37: control.DropObjectsRequest.Body + (*DropObjectsResponse_Body)(nil), // 38: control.DropObjectsResponse.Body + (*ListShardsRequest_Body)(nil), // 39: control.ListShardsRequest.Body + (*ListShardsResponse_Body)(nil), // 40: control.ListShardsResponse.Body + (*SetShardModeRequest_Body)(nil), // 41: control.SetShardModeRequest.Body + (*SetShardModeResponse_Body)(nil), // 42: control.SetShardModeResponse.Body + (*SynchronizeTreeRequest_Body)(nil), // 43: control.SynchronizeTreeRequest.Body + (*SynchronizeTreeResponse_Body)(nil), // 44: control.SynchronizeTreeResponse.Body + (*EvacuateShardRequest_Body)(nil), // 45: control.EvacuateShardRequest.Body + (*EvacuateShardResponse_Body)(nil), // 46: control.EvacuateShardResponse.Body + (*FlushCacheRequest_Body)(nil), // 47: control.FlushCacheRequest.Body + (*FlushCacheResponse_Body)(nil), // 48: control.FlushCacheResponse.Body + (*DoctorRequest_Body)(nil), // 49: control.DoctorRequest.Body + (*DoctorResponse_Body)(nil), // 50: control.DoctorResponse.Body + (*StartShardEvacuationRequest_Body)(nil), // 51: control.StartShardEvacuationRequest.Body + (*StartShardEvacuationResponse_Body)(nil), // 52: control.StartShardEvacuationResponse.Body + (*GetShardEvacuationStatusRequest_Body)(nil), // 53: control.GetShardEvacuationStatusRequest.Body + (*GetShardEvacuationStatusResponse_Body)(nil), // 54: control.GetShardEvacuationStatusResponse.Body + (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 55: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 56: control.GetShardEvacuationStatusResponse.Body.Duration + (*StopShardEvacuationRequest_Body)(nil), // 57: control.StopShardEvacuationRequest.Body + (*StopShardEvacuationResponse_Body)(nil), // 58: control.StopShardEvacuationResponse.Body + (*AddChainLocalOverrideRequest_Body)(nil), // 59: control.AddChainLocalOverrideRequest.Body + (*AddChainLocalOverrideResponse_Body)(nil), // 60: control.AddChainLocalOverrideResponse.Body + (*GetChainLocalOverrideRequest_Body)(nil), // 61: control.GetChainLocalOverrideRequest.Body + (*GetChainLocalOverrideResponse_Body)(nil), // 62: control.GetChainLocalOverrideResponse.Body + (*ListChainLocalOverridesRequest_Body)(nil), // 63: control.ListChainLocalOverridesRequest.Body + (*ListChainLocalOverridesResponse_Body)(nil), // 64: control.ListChainLocalOverridesResponse.Body + (*RemoveChainLocalOverrideRequest_Body)(nil), // 65: control.RemoveChainLocalOverrideRequest.Body + (*RemoveChainLocalOverrideResponse_Body)(nil), // 66: control.RemoveChainLocalOverrideResponse.Body + (*Signature)(nil), // 67: control.Signature + (NetmapStatus)(0), // 68: control.NetmapStatus + (HealthStatus)(0), // 69: control.HealthStatus + (*ShardInfo)(nil), // 70: control.ShardInfo + (ShardMode)(0), // 71: control.ShardMode } var file_pkg_services_control_service_proto_depIdxs = []int32{ - 25, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 51, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 26, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 51, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 27, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 51, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 28, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 51, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 29, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 51, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature - 30, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 51, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature - 31, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 51, // 13: control.ListShardsRequest.signature:type_name -> control.Signature - 32, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 51, // 15: control.ListShardsResponse.signature:type_name -> control.Signature - 33, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 51, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature - 34, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 51, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature - 35, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 51, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 36, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 51, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 37, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 51, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature - 38, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 51, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature - 39, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 51, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature - 40, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 51, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature - 41, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body - 51, // 33: control.DoctorRequest.signature:type_name -> control.Signature - 42, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body - 51, // 35: control.DoctorResponse.signature:type_name -> control.Signature - 43, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body - 51, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature - 44, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body - 51, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature - 45, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body - 51, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 46, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body - 51, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 49, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body - 51, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature - 50, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body - 51, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature - 52, // 48: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 53, // 49: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 52, // 50: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 54, // 51: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 55, // 52: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 0, // 53: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status - 48, // 54: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration - 47, // 55: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - 1, // 56: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 3, // 57: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 5, // 58: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 7, // 59: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 9, // 60: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 11, // 61: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 13, // 62: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 19, // 63: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest - 21, // 64: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest - 23, // 65: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest - 15, // 66: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 17, // 67: control.ControlService.Doctor:input_type -> control.DoctorRequest - 2, // 68: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 4, // 69: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 6, // 70: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 8, // 71: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 10, // 72: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 12, // 73: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 14, // 74: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 20, // 75: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse - 22, // 76: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse - 24, // 77: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse - 16, // 78: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 18, // 79: control.ControlService.Doctor:output_type -> control.DoctorResponse - 68, // [68:80] is the sub-list for method output_type - 56, // [56:68] is the sub-list for method input_type - 56, // [56:56] is the sub-list for extension type_name - 56, // [56:56] is the sub-list for extension extendee - 0, // [0:56] is the sub-list for field type_name + 33, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body + 67, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature + 34, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body + 67, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature + 35, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body + 67, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature + 36, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body + 67, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature + 37, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body + 67, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature + 38, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body + 67, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature + 39, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body + 67, // 13: control.ListShardsRequest.signature:type_name -> control.Signature + 40, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body + 67, // 15: control.ListShardsResponse.signature:type_name -> control.Signature + 41, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body + 67, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature + 42, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body + 67, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature + 43, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body + 67, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature + 44, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body + 67, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature + 45, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body + 67, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature + 46, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body + 67, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature + 47, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body + 67, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature + 48, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body + 67, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature + 49, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body + 67, // 33: control.DoctorRequest.signature:type_name -> control.Signature + 50, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body + 67, // 35: control.DoctorResponse.signature:type_name -> control.Signature + 51, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body + 67, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature + 52, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body + 67, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature + 53, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body + 67, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 54, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body + 67, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 57, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body + 67, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature + 58, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body + 67, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature + 59, // 48: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body + 67, // 49: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature + 60, // 50: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body + 67, // 51: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature + 61, // 52: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body + 67, // 53: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature + 62, // 54: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body + 67, // 55: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature + 63, // 56: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body + 67, // 57: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature + 64, // 58: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body + 67, // 59: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature + 65, // 60: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body + 67, // 61: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature + 66, // 62: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body + 67, // 63: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature + 68, // 64: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus + 69, // 65: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus + 68, // 66: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus + 70, // 67: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo + 71, // 68: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode + 0, // 69: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status + 56, // 70: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration + 55, // 71: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + 1, // 72: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 3, // 73: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 5, // 74: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 7, // 75: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 9, // 76: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 11, // 77: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 13, // 78: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 19, // 79: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest + 21, // 80: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest + 23, // 81: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest + 15, // 82: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 17, // 83: control.ControlService.Doctor:input_type -> control.DoctorRequest + 25, // 84: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest + 27, // 85: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest + 29, // 86: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest + 31, // 87: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest + 2, // 88: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 4, // 89: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 6, // 90: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 8, // 91: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 10, // 92: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 12, // 93: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 14, // 94: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 20, // 95: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse + 22, // 96: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse + 24, // 97: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse + 16, // 98: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 18, // 99: control.ControlService.Doctor:output_type -> control.DoctorResponse + 26, // 100: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse + 28, // 101: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse + 30, // 102: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse + 32, // 103: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse + 88, // [88:104] is the sub-list for method output_type + 72, // [72:88] is the sub-list for method input_type + 72, // [72:72] is the sub-list for extension type_name + 72, // [72:72] is the sub-list for extension extendee + 0, // [0:72] is the sub-list for field type_name } func init() { file_pkg_services_control_service_proto_init() } @@ -3576,7 +4598,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*AddChainLocalOverrideRequest); i { case 0: return &v.state case 1: @@ -3588,7 +4610,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { + switch v := v.(*AddChainLocalOverrideResponse); i { case 0: return &v.state case 1: @@ -3600,7 +4622,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusRequest_Body); i { + switch v := v.(*GetChainLocalOverrideRequest); i { case 0: return &v.state case 1: @@ -3612,7 +4634,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusResponse_Body); i { + switch v := v.(*GetChainLocalOverrideResponse); i { case 0: return &v.state case 1: @@ -3624,7 +4646,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest_Body); i { + switch v := v.(*ListChainLocalOverridesRequest); i { case 0: return &v.state case 1: @@ -3636,7 +4658,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse_Body); i { + switch v := v.(*ListChainLocalOverridesResponse); i { case 0: return &v.state case 1: @@ -3648,7 +4670,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest_Body); i { + switch v := v.(*RemoveChainLocalOverrideRequest); i { case 0: return &v.state case 1: @@ -3660,7 +4682,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse_Body); i { + switch v := v.(*RemoveChainLocalOverrideResponse); i { case 0: return &v.state case 1: @@ -3672,7 +4694,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest_Body); i { + switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state case 1: @@ -3684,7 +4706,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse_Body); i { + switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state case 1: @@ -3696,7 +4718,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest_Body); i { + switch v := v.(*SetNetmapStatusRequest_Body); i { case 0: return &v.state case 1: @@ -3708,7 +4730,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse_Body); i { + switch v := v.(*SetNetmapStatusResponse_Body); i { case 0: return &v.state case 1: @@ -3720,7 +4742,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest_Body); i { + switch v := v.(*DropObjectsRequest_Body); i { case 0: return &v.state case 1: @@ -3732,7 +4754,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse_Body); i { + switch v := v.(*DropObjectsResponse_Body); i { case 0: return &v.state case 1: @@ -3744,7 +4766,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest_Body); i { + switch v := v.(*ListShardsRequest_Body); i { case 0: return &v.state case 1: @@ -3756,7 +4778,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheResponse_Body); i { + switch v := v.(*ListShardsResponse_Body); i { case 0: return &v.state case 1: @@ -3768,7 +4790,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorRequest_Body); i { + switch v := v.(*SetShardModeRequest_Body); i { case 0: return &v.state case 1: @@ -3780,7 +4802,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorResponse_Body); i { + switch v := v.(*SetShardModeResponse_Body); i { case 0: return &v.state case 1: @@ -3792,7 +4814,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationRequest_Body); i { + switch v := v.(*SynchronizeTreeRequest_Body); i { case 0: return &v.state case 1: @@ -3804,7 +4826,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationResponse_Body); i { + switch v := v.(*SynchronizeTreeResponse_Body); i { case 0: return &v.state case 1: @@ -3816,7 +4838,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusRequest_Body); i { + switch v := v.(*EvacuateShardRequest_Body); i { case 0: return &v.state case 1: @@ -3828,7 +4850,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body); i { + switch v := v.(*EvacuateShardResponse_Body); i { case 0: return &v.state case 1: @@ -3840,7 +4862,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { + switch v := v.(*FlushCacheRequest_Body); i { case 0: return &v.state case 1: @@ -3852,7 +4874,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { + switch v := v.(*FlushCacheResponse_Body); i { case 0: return &v.state case 1: @@ -3864,7 +4886,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationRequest_Body); i { + switch v := v.(*DoctorRequest_Body); i { case 0: return &v.state case 1: @@ -3876,6 +4898,102 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DoctorResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StartShardEvacuationRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StartShardEvacuationResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetShardEvacuationStatusRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetShardEvacuationStatusResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StopShardEvacuationRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*StopShardEvacuationResponse_Body); i { case 0: return &v.state @@ -3887,6 +5005,102 @@ func file_pkg_services_control_service_proto_init() { return nil } } + file_pkg_services_control_service_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AddChainLocalOverrideRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AddChainLocalOverrideResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetChainLocalOverrideRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetChainLocalOverrideResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListChainLocalOverridesRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListChainLocalOverridesResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -3894,7 +5108,7 @@ func file_pkg_services_control_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_service_proto_rawDesc, NumEnums: 1, - NumMessages: 50, + NumMessages: 66, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index cbcf74e53..e7d218f20 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -44,6 +44,18 @@ service ControlService { // Doctor performs storage restructuring operations on engine. rpc Doctor (DoctorRequest) returns (DoctorResponse); + + // Add local access policy engine overrides to a node. + rpc AddChainLocalOverride (AddChainLocalOverrideRequest) returns (AddChainLocalOverrideResponse); + + // Get local access policy engine overrides stored in the node by chain id. + rpc GetChainLocalOverride (GetChainLocalOverrideRequest) returns (GetChainLocalOverrideResponse); + + // List local access policy engine overrides stored in the node by container id. + rpc ListChainLocalOverrides (ListChainLocalOverridesRequest) returns (ListChainLocalOverridesResponse); + + // Remove local access policy engine overrides stored in the node by chaind id. + rpc RemoveChainLocalOverride (RemoveChainLocalOverrideRequest) returns (RemoveChainLocalOverrideResponse); } // Health check request. @@ -405,3 +417,105 @@ message StopShardEvacuationResponse { Body body = 1; Signature signature = 2; } + +// AddChainLocalOverride request. +message AddChainLocalOverrideRequest { + message Body { + // Container id for which the overrides are applied. + bytes container_id = 1; + + // Serialized rule chain. + bytes chain = 2; + } + + Body body = 1; + + Signature signature = 2; +} + +// AddChainLocalOverride response. +message AddChainLocalOverrideResponse { + message Body { + // Chain ID assigned for the added rule chain. + string chain_id = 1; + } + + Body body = 1; + + Signature signature = 2; +} + +// GetChainLocalOverride request. +message GetChainLocalOverrideRequest { + message Body { + // Container id for which the overrides are defined. + bytes container_id = 1; + + // Chain ID assigned for the added rule chain. + string chain_id = 2; + } + + Body body = 1; + + Signature signature = 2; +} + +// GetChainLocalOverride response. +message GetChainLocalOverrideResponse { + message Body { + // Serialized rule chain. + bytes chain = 1; + } + + Body body = 1; + + Signature signature = 2; +} + +// ListChainLocalOverrides request. +message ListChainLocalOverridesRequest { + message Body { + // Container id for which the overrides are defined. + bytes container_id = 1; + } + + Body body = 1; + + Signature signature = 2; +} + +// ListChainLocalOverrides response. +message ListChainLocalOverridesResponse { + message Body { + // The list of serialized rule chain. + repeated bytes chains = 1; + } + + Body body = 1; + + Signature signature = 2; +} + +message RemoveChainLocalOverrideRequest { + message Body { + // Container id for which the overrides are defined. + bytes container_id = 1; + + // Chain ID assigned for the added rule chain. + string chain_id = 2; + } + + Body body = 1; + + Signature signature = 2; +} + +message RemoveChainLocalOverrideResponse { + message Body { + bool removed = 1; + } + + Body body = 1; + + Signature signature = 2; +} diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 56ff81ace..852788a30 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -2059,3 +2059,697 @@ func (x *StopShardEvacuationResponse) ReadSignedData(buf []byte) ([]byte, error) func (x *StopShardEvacuationResponse) SetSignature(sig *Signature) { x.Signature = sig } + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *AddChainLocalOverrideRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.BytesSize(1, x.ContainerId) + size += proto.BytesSize(2, x.Chain) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *AddChainLocalOverrideRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) + offset += proto.BytesMarshal(2, buf[offset:], x.Chain) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *AddChainLocalOverrideRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *AddChainLocalOverrideRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *AddChainLocalOverrideRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *AddChainLocalOverrideRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *AddChainLocalOverrideRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *AddChainLocalOverrideResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.StringSize(1, x.ChainId) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *AddChainLocalOverrideResponse_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.StringMarshal(1, buf[offset:], x.ChainId) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *AddChainLocalOverrideResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *AddChainLocalOverrideResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *AddChainLocalOverrideResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *AddChainLocalOverrideResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *AddChainLocalOverrideResponse) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetChainLocalOverrideRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.BytesSize(1, x.ContainerId) + size += proto.StringSize(2, x.ChainId) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetChainLocalOverrideRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) + offset += proto.StringMarshal(2, buf[offset:], x.ChainId) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetChainLocalOverrideRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetChainLocalOverrideRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *GetChainLocalOverrideRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *GetChainLocalOverrideRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *GetChainLocalOverrideRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetChainLocalOverrideResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.BytesSize(1, x.Chain) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetChainLocalOverrideResponse_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.BytesMarshal(1, buf[offset:], x.Chain) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetChainLocalOverrideResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetChainLocalOverrideResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *GetChainLocalOverrideResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *GetChainLocalOverrideResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *GetChainLocalOverrideResponse) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListChainLocalOverridesRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.BytesSize(1, x.ContainerId) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ListChainLocalOverridesRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListChainLocalOverridesRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ListChainLocalOverridesRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *ListChainLocalOverridesRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *ListChainLocalOverridesRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *ListChainLocalOverridesRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListChainLocalOverridesResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.RepeatedBytesSize(1, x.Chains) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ListChainLocalOverridesResponse_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Chains) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListChainLocalOverridesResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ListChainLocalOverridesResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *ListChainLocalOverridesResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *ListChainLocalOverridesResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *ListChainLocalOverridesResponse) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveChainLocalOverrideRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.BytesSize(1, x.ContainerId) + size += proto.StringSize(2, x.ChainId) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveChainLocalOverrideRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) + offset += proto.StringMarshal(2, buf[offset:], x.ChainId) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveChainLocalOverrideRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveChainLocalOverrideRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *RemoveChainLocalOverrideRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *RemoveChainLocalOverrideRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *RemoveChainLocalOverrideRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveChainLocalOverrideResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.BoolSize(1, x.Removed) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveChainLocalOverrideResponse_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.BoolMarshal(1, buf[offset:], x.Removed) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveChainLocalOverrideResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveChainLocalOverrideResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *RemoveChainLocalOverrideResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *RemoveChainLocalOverrideResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *RemoveChainLocalOverrideResponse) SetSignature(sig *Signature) { + x.Signature = sig +} diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 967c739f5..b5881ccb2 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -31,6 +31,10 @@ const ( ControlService_StopShardEvacuation_FullMethodName = "/control.ControlService/StopShardEvacuation" ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache" ControlService_Doctor_FullMethodName = "/control.ControlService/Doctor" + ControlService_AddChainLocalOverride_FullMethodName = "/control.ControlService/AddChainLocalOverride" + ControlService_GetChainLocalOverride_FullMethodName = "/control.ControlService/GetChainLocalOverride" + ControlService_ListChainLocalOverrides_FullMethodName = "/control.ControlService/ListChainLocalOverrides" + ControlService_RemoveChainLocalOverride_FullMethodName = "/control.ControlService/RemoveChainLocalOverride" ) // ControlServiceClient is the client API for ControlService service. @@ -62,6 +66,14 @@ type ControlServiceClient interface { FlushCache(ctx context.Context, in *FlushCacheRequest, opts ...grpc.CallOption) (*FlushCacheResponse, error) // Doctor performs storage restructuring operations on engine. Doctor(ctx context.Context, in *DoctorRequest, opts ...grpc.CallOption) (*DoctorResponse, error) + // Add local access policy engine overrides to a node. + AddChainLocalOverride(ctx context.Context, in *AddChainLocalOverrideRequest, opts ...grpc.CallOption) (*AddChainLocalOverrideResponse, error) + // Get local access policy engine overrides stored in the node by chain id. + GetChainLocalOverride(ctx context.Context, in *GetChainLocalOverrideRequest, opts ...grpc.CallOption) (*GetChainLocalOverrideResponse, error) + // List local access policy engine overrides stored in the node by container id. + ListChainLocalOverrides(ctx context.Context, in *ListChainLocalOverridesRequest, opts ...grpc.CallOption) (*ListChainLocalOverridesResponse, error) + // Remove local access policy engine overrides stored in the node by chaind id. + RemoveChainLocalOverride(ctx context.Context, in *RemoveChainLocalOverrideRequest, opts ...grpc.CallOption) (*RemoveChainLocalOverrideResponse, error) } type controlServiceClient struct { @@ -180,6 +192,42 @@ func (c *controlServiceClient) Doctor(ctx context.Context, in *DoctorRequest, op return out, nil } +func (c *controlServiceClient) AddChainLocalOverride(ctx context.Context, in *AddChainLocalOverrideRequest, opts ...grpc.CallOption) (*AddChainLocalOverrideResponse, error) { + out := new(AddChainLocalOverrideResponse) + err := c.cc.Invoke(ctx, ControlService_AddChainLocalOverride_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlServiceClient) GetChainLocalOverride(ctx context.Context, in *GetChainLocalOverrideRequest, opts ...grpc.CallOption) (*GetChainLocalOverrideResponse, error) { + out := new(GetChainLocalOverrideResponse) + err := c.cc.Invoke(ctx, ControlService_GetChainLocalOverride_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlServiceClient) ListChainLocalOverrides(ctx context.Context, in *ListChainLocalOverridesRequest, opts ...grpc.CallOption) (*ListChainLocalOverridesResponse, error) { + out := new(ListChainLocalOverridesResponse) + err := c.cc.Invoke(ctx, ControlService_ListChainLocalOverrides_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlServiceClient) RemoveChainLocalOverride(ctx context.Context, in *RemoveChainLocalOverrideRequest, opts ...grpc.CallOption) (*RemoveChainLocalOverrideResponse, error) { + out := new(RemoveChainLocalOverrideResponse) + err := c.cc.Invoke(ctx, ControlService_RemoveChainLocalOverride_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // ControlServiceServer is the server API for ControlService service. // All implementations should embed UnimplementedControlServiceServer // for forward compatibility @@ -209,6 +257,14 @@ type ControlServiceServer interface { FlushCache(context.Context, *FlushCacheRequest) (*FlushCacheResponse, error) // Doctor performs storage restructuring operations on engine. Doctor(context.Context, *DoctorRequest) (*DoctorResponse, error) + // Add local access policy engine overrides to a node. + AddChainLocalOverride(context.Context, *AddChainLocalOverrideRequest) (*AddChainLocalOverrideResponse, error) + // Get local access policy engine overrides stored in the node by chain id. + GetChainLocalOverride(context.Context, *GetChainLocalOverrideRequest) (*GetChainLocalOverrideResponse, error) + // List local access policy engine overrides stored in the node by container id. + ListChainLocalOverrides(context.Context, *ListChainLocalOverridesRequest) (*ListChainLocalOverridesResponse, error) + // Remove local access policy engine overrides stored in the node by chaind id. + RemoveChainLocalOverride(context.Context, *RemoveChainLocalOverrideRequest) (*RemoveChainLocalOverrideResponse, error) } // UnimplementedControlServiceServer should be embedded to have forward compatible implementations. @@ -251,6 +307,18 @@ func (UnimplementedControlServiceServer) FlushCache(context.Context, *FlushCache func (UnimplementedControlServiceServer) Doctor(context.Context, *DoctorRequest) (*DoctorResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Doctor not implemented") } +func (UnimplementedControlServiceServer) AddChainLocalOverride(context.Context, *AddChainLocalOverrideRequest) (*AddChainLocalOverrideResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AddChainLocalOverride not implemented") +} +func (UnimplementedControlServiceServer) GetChainLocalOverride(context.Context, *GetChainLocalOverrideRequest) (*GetChainLocalOverrideResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetChainLocalOverride not implemented") +} +func (UnimplementedControlServiceServer) ListChainLocalOverrides(context.Context, *ListChainLocalOverridesRequest) (*ListChainLocalOverridesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListChainLocalOverrides not implemented") +} +func (UnimplementedControlServiceServer) RemoveChainLocalOverride(context.Context, *RemoveChainLocalOverrideRequest) (*RemoveChainLocalOverrideResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RemoveChainLocalOverride not implemented") +} // UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ControlServiceServer will @@ -479,6 +547,78 @@ func _ControlService_Doctor_Handler(srv interface{}, ctx context.Context, dec fu return interceptor(ctx, in, info, handler) } +func _ControlService_AddChainLocalOverride_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AddChainLocalOverrideRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).AddChainLocalOverride(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_AddChainLocalOverride_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).AddChainLocalOverride(ctx, req.(*AddChainLocalOverrideRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ControlService_GetChainLocalOverride_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetChainLocalOverrideRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).GetChainLocalOverride(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_GetChainLocalOverride_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).GetChainLocalOverride(ctx, req.(*GetChainLocalOverrideRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ControlService_ListChainLocalOverrides_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListChainLocalOverridesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).ListChainLocalOverrides(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_ListChainLocalOverrides_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).ListChainLocalOverrides(ctx, req.(*ListChainLocalOverridesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ControlService_RemoveChainLocalOverride_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RemoveChainLocalOverrideRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).RemoveChainLocalOverride(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_RemoveChainLocalOverride_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).RemoveChainLocalOverride(ctx, req.(*RemoveChainLocalOverrideRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -534,6 +674,22 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "Doctor", Handler: _ControlService_Doctor_Handler, }, + { + MethodName: "AddChainLocalOverride", + Handler: _ControlService_AddChainLocalOverride_Handler, + }, + { + MethodName: "GetChainLocalOverride", + Handler: _ControlService_GetChainLocalOverride_Handler, + }, + { + MethodName: "ListChainLocalOverrides", + Handler: _ControlService_ListChainLocalOverrides_Handler, + }, + { + MethodName: "RemoveChainLocalOverride", + Handler: _ControlService_RemoveChainLocalOverride_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "pkg/services/control/service.proto", From 5ec73fe8a0b6c3c2ab31b093a43c6fdfa7423773 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 30 Oct 2023 16:48:02 +0300 Subject: [PATCH 0660/1943] [#770] node: Introduce ape chain source * Provide methods to access rule chains with access policy engine (APE) chain source * Initialize apeChainSource within object service initialization * Share apeChainSource with control service * Implement dummy apeChainSource instance based on in-memory implementation Signed-off-by: Airat Arifullin --- cmd/frostfs-node/config.go | 5 +++++ cmd/frostfs-node/control.go | 1 + cmd/frostfs-node/object.go | 2 ++ cmd/frostfs-node/policy_engine.go | 28 +++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 ++ pkg/core/container/storage.go | 8 ++++++++ pkg/services/control/server/server.go | 10 ++++++++++ 8 files changed, 57 insertions(+) create mode 100644 cmd/frostfs-node/policy_engine.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e3e56e5de..a41b73d92 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -510,6 +510,11 @@ type cfgObject struct { eaclSource container.EACLSource + // Access policy chain source is used by object service to + // check for operation permissions but this source is also shared with + // control service that dispatches local overrides. + apeChainSource container.AccessPolicyEngineChainSource + pool cfgObjectRoutines cfgLocalStorage cfgLocalStorage diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index 98d893c38..30d644803 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -51,6 +51,7 @@ func initControlService(c *cfg) { controlSvc.WithTreeService(treeSynchronizer{ c.treeService, }), + controlSvc.WithAPEChainSource(c.cfgObject.apeChainSource), ) lis, err := net.Listen("tcp", endpoint) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 2f714b821..bbaec01ed 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -157,6 +157,8 @@ func initObjectService(c *cfg) { c.replicator = createReplicator(c, keyStorage, c.bgClientCache) + c.cfgObject.apeChainSource = NewAPESource() + addPolicer(c, keyStorage, c.bgClientCache) traverseGen := util.NewTraverserGenerator(c.netMapSource, c.cfgObject.cnrSource, c) diff --git a/cmd/frostfs-node/policy_engine.go b/cmd/frostfs-node/policy_engine.go new file mode 100644 index 000000000..039124a6b --- /dev/null +++ b/cmd/frostfs-node/policy_engine.go @@ -0,0 +1,28 @@ +package main + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine" +) + +type apeChainSourceImpl struct { + localChainStorage map[cid.ID]policyengine.CachedChainStorage +} + +func NewAPESource() container.AccessPolicyEngineChainSource { + return &apeChainSourceImpl{ + localChainStorage: make(map[cid.ID]policyengine.CachedChainStorage), + } +} + +var _ container.AccessPolicyEngineChainSource = (*apeChainSourceImpl)(nil) + +func (c *apeChainSourceImpl) GetChainSource(cid cid.ID) (policyengine.CachedChainStorage, error) { + s, ok := c.localChainStorage[cid] + if ok { + return s, nil + } + c.localChainStorage[cid] = policyengine.NewInMemory() + return c.localChainStorage[cid], nil +} diff --git a/go.mod b/go.mod index 37cb477bf..4f6505e87 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d git.frostfs.info/TrueCloudLab/hrw v1.2.1 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231101082425-5eee1a733432 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 diff --git a/go.sum b/go.sum index 62fd85c86..05a870b80 100644 --- a/go.sum +++ b/go.sum @@ -736,6 +736,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231101082425-5eee1a733432 h1:z0PqdiEIHXK2qC83e6pmxUZ5peP9CIL0Bh5mP/d+4Xc= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231101082425-5eee1a733432/go.mod h1:qf3B9hSz6gCMfcfvqkhTu5ak+Gx2R+wo4Hc87LnKxPg= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/pkg/core/container/storage.go b/pkg/core/container/storage.go index 0766ced31..f48fc73ac 100644 --- a/pkg/core/container/storage.go +++ b/pkg/core/container/storage.go @@ -6,6 +6,7 @@ import ( frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine" ) // Container groups information about the FrostFS container stored in the FrostFS network. @@ -70,3 +71,10 @@ type EACLSource interface { // eACL table is not in source. GetEACL(cid.ID) (*EACL, error) } + +// AccessPolicyEngineChainSource interface provides methods to access and manipulate +// policy engine chain storage. +type AccessPolicyEngineChainSource interface { + // TODO (aarifullin): Better to use simpler interface instead CachedChainStorage. + GetChainSource(cid cid.ID) (policyengine.CachedChainStorage, error) +} diff --git a/pkg/services/control/server/server.go b/pkg/services/control/server/server.go index a0ad44e23..ae024a82d 100644 --- a/pkg/services/control/server/server.go +++ b/pkg/services/control/server/server.go @@ -59,6 +59,8 @@ type cfg struct { cnrSrc container.Source + apeChainSrc container.AccessPolicyEngineChainSource + replicator *replicator.Replicator nodeState NodeState @@ -151,3 +153,11 @@ func WithTreeService(s TreeService) Option { c.treeService = s } } + +// WithAPEChainSource returns the option to set access policy engine +// chain source. +func WithAPEChainSource(apeChainSrc container.AccessPolicyEngineChainSource) Option { + return func(c *cfg) { + c.apeChainSrc = apeChainSrc + } +} From 8e11ef46b89e1b305b4a555d1ec445f97f37151a Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 30 Oct 2023 18:30:23 +0300 Subject: [PATCH 0661/1943] [#770] object: Introduce ape chain checker for object svc * Introduce Request type converted from RequestInfo type to implement policy-engine's Request interface * Implement basic ape checker to check if a request is permitted to be performed * Make put handlers use APE checker instead EACL Signed-off-by: Airat Arifullin --- cmd/frostfs-node/object.go | 1 + cmd/frostfs-node/policy_engine.go | 6 ++ pkg/services/object/acl/ape.go | 56 +++++++++++++ pkg/services/object/acl/ape_request.go | 105 +++++++++++++++++++++++++ pkg/services/object/acl/v2/service.go | 18 +++-- pkg/services/object/acl/v2/types.go | 6 ++ 6 files changed, 184 insertions(+), 8 deletions(-) create mode 100644 pkg/services/object/acl/ape.go create mode 100644 pkg/services/object/acl/ape_request.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index bbaec01ed..79f492190 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -426,6 +426,7 @@ func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter, irFet c.cfgObject.eaclSource, eaclSDK.NewValidator(), ls), + acl.NewAPEChecker(c.log, c.cfgObject.apeChainSource), c.cfgObject.cnrSource, v2.WithLogger(c.log), ) diff --git a/cmd/frostfs-node/policy_engine.go b/cmd/frostfs-node/policy_engine.go index 039124a6b..f0bd78629 100644 --- a/cmd/frostfs-node/policy_engine.go +++ b/cmd/frostfs-node/policy_engine.go @@ -1,12 +1,15 @@ package main import ( + "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" policyengine "git.frostfs.info/TrueCloudLab/policy-engine" ) type apeChainSourceImpl struct { + mtx sync.Mutex localChainStorage map[cid.ID]policyengine.CachedChainStorage } @@ -19,6 +22,9 @@ func NewAPESource() container.AccessPolicyEngineChainSource { var _ container.AccessPolicyEngineChainSource = (*apeChainSourceImpl)(nil) func (c *apeChainSourceImpl) GetChainSource(cid cid.ID) (policyengine.CachedChainStorage, error) { + c.mtx.Lock() + defer c.mtx.Unlock() + s, ok := c.localChainStorage[cid] if ok { return s, nil diff --git a/pkg/services/object/acl/ape.go b/pkg/services/object/acl/ape.go new file mode 100644 index 000000000..b064b1eec --- /dev/null +++ b/pkg/services/object/acl/ape.go @@ -0,0 +1,56 @@ +package acl + +import ( + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine" +) + +var ( + errAPEChainNoSource = errors.New("could not get ape chain source for the container") +) + +type apeCheckerImpl struct { + log *logger.Logger + apeSrc container.AccessPolicyEngineChainSource +} + +func NewAPEChecker(log *logger.Logger, apeSrc container.AccessPolicyEngineChainSource) v2.APEChainChecker { + return &apeCheckerImpl{ + log: log, + apeSrc: apeSrc, + } +} + +func (c *apeCheckerImpl) CheckIfRequestPermitted(reqInfo v2.RequestInfo) error { + cnr := reqInfo.ContainerID() + + chainCache, err := c.apeSrc.GetChainSource(cnr) + if err != nil { + return errAPEChainNoSource + } + + request := new(Request) + request.FromRequestInfo(reqInfo) + + status, ruleFound := chainCache.IsAllowed(policyengine.Ingress, "", request) + + if !ruleFound || status == policyengine.Allow { + return nil + } + + return apeErr(reqInfo, status) +} + +const accessDeniedAPEReasonFmt = "access to operation %s is denied by access policy engine: %s" + +func apeErr(req v2.RequestInfo, status policyengine.Status) error { + errAccessDenied := &apistatus.ObjectAccessDenied{} + errAccessDenied.WriteReason(fmt.Sprintf(accessDeniedAPEReasonFmt, req.Operation(), status.String())) + return errAccessDenied +} diff --git a/pkg/services/object/acl/ape_request.go b/pkg/services/object/acl/ape_request.go new file mode 100644 index 000000000..3e8bb173d --- /dev/null +++ b/pkg/services/object/acl/ape_request.go @@ -0,0 +1,105 @@ +package acl + +import ( + "fmt" + + v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" + aclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine" +) + +type Request struct { + operation string + resource *resource + properties map[string]string +} + +var _ policyengine.Request = (*Request)(nil) + +type resource struct { + name string + properties map[string]string +} + +var _ policyengine.Resource = (*resource)(nil) + +func (r *resource) Name() string { + return r.name +} + +func (r *resource) Property(key string) string { + return r.properties[key] +} + +// TODO (aarifullin): these stringified verbs, properties and namespaces +// should be non-implementation-specific. +func getResource(reqInfo v2.RequestInfo) *resource { + cid := reqInfo.ContainerID() + oid := "*" + if reqOID := reqInfo.ObjectID(); reqOID != nil { + oid = reqOID.EncodeToString() + } + name := fmt.Sprintf("native:::object/%s/%s", + cid, + oid) + + return &resource{ + name: name, + properties: make(map[string]string), + } +} + +func getProperties(_ v2.RequestInfo) map[string]string { + return map[string]string{ + "Actor": "", + } +} + +// TODO (aarifullin): these stringified verbs, properties and namespaces +// should be non-implementation-specific. +func getOperation(reqInfo v2.RequestInfo) string { + var verb string + switch op := reqInfo.Operation(); op { + case aclSDK.OpObjectGet: + verb = "GetObject" + case aclSDK.OpObjectHead: + verb = "HeadObject" + case aclSDK.OpObjectPut: + verb = "PutObject" + case aclSDK.OpObjectDelete: + verb = "DeleteObject" + case aclSDK.OpObjectSearch: + verb = "SearchObject" + case aclSDK.OpObjectRange: + verb = "RangeObject" + case aclSDK.OpObjectHash: + verb = "HashObject" + } + + return "native:" + verb +} + +func NewRequest() *Request { + return &Request{ + resource: new(resource), + properties: map[string]string{}, + } +} + +func (r *Request) FromRequestInfo(ri v2.RequestInfo) { + r.operation = getOperation(ri) + r.resource = getResource(ri) + r.properties = getProperties(ri) +} + +func (r *Request) Operation() string { + return r.operation +} + +func (r *Request) Property(key string) string { + return r.properties[key] +} + +func (r *Request) Resource() policyengine.Resource { + return r.resource +} diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index b5bd3d4f4..ba9d1dfa5 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -67,6 +67,10 @@ type cfg struct { checker ACLChecker + // TODO(aarifullin): apeCheck is temporarily the part of + // acl service and must be standalone. + apeChecker APEChainChecker + irFetcher InnerRingFetcher nm netmap.Source @@ -79,6 +83,7 @@ func New(next object.ServiceServer, nm netmap.Source, irf InnerRingFetcher, acl ACLChecker, + apeChecker APEChainChecker, cs container.Source, opts ...Option, ) Service { @@ -88,6 +93,7 @@ func New(next object.ServiceServer, nm: nm, irFetcher: irf, checker: acl, + apeChecker: apeChecker, containers: cs, } @@ -212,10 +218,8 @@ func (b Service) Head( reqInfo.obj = obj - if !b.checker.CheckBasicACL(reqInfo) { - return nil, basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return nil, eACLErr(reqInfo, err) + if err := b.apeChecker.CheckIfRequestPermitted(reqInfo); err != nil { + return nil, err } resp, err := b.next.Head(ctx, request) @@ -560,10 +564,8 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe reqInfo.obj = obj - if !p.source.checker.CheckBasicACL(reqInfo) || !p.source.checker.StickyBitCheck(reqInfo, idOwner) { - return basicACLErr(reqInfo) - } else if err := p.source.checker.CheckEACL(request, reqInfo); err != nil { - return eACLErr(reqInfo, err) + if err := p.source.apeChecker.CheckIfRequestPermitted(reqInfo); err != nil { + return err } } diff --git a/pkg/services/object/acl/v2/types.go b/pkg/services/object/acl/v2/types.go index 061cd26b6..a113c4693 100644 --- a/pkg/services/object/acl/v2/types.go +++ b/pkg/services/object/acl/v2/types.go @@ -26,3 +26,9 @@ type InnerRingFetcher interface { // the actual inner ring. InnerRingKeys() ([][]byte, error) } + +// APEChainChecker is the interface that provides methods to +// check if the access policy engine permits to perform the request. +type APEChainChecker interface { + CheckIfRequestPermitted(RequestInfo) error +} From 66848d32888fda0eb6a37ef2b7c02dd9f54db124 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 31 Oct 2023 11:55:42 +0300 Subject: [PATCH 0662/1943] [#770] cli: Add methods to work with APE rules via control svc * Add methods to frostfs-cli * Implement rpc in control service Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/control/add_rule.go | 90 +++++++++ cmd/frostfs-cli/modules/control/get_rule.go | 69 +++++++ cmd/frostfs-cli/modules/control/list_rules.go | 72 +++++++ .../modules/control/remove_rule.go | 72 +++++++ cmd/frostfs-cli/modules/control/root.go | 8 + cmd/frostfs-cli/modules/util/ape.go | 179 ++++++++++++++++++ cmd/frostfs-cli/modules/util/ape_test.go | 129 +++++++++++++ pkg/services/control/rpc.go | 56 ++++++ pkg/services/control/server/policy_engine.go | 140 +++++++++++++- pkg/services/control/server/server.go | 6 + pkg/services/object/acl/v2/service.go | 4 - 11 files changed, 813 insertions(+), 12 deletions(-) create mode 100644 cmd/frostfs-cli/modules/control/add_rule.go create mode 100644 cmd/frostfs-cli/modules/control/get_rule.go create mode 100644 cmd/frostfs-cli/modules/control/list_rules.go create mode 100644 cmd/frostfs-cli/modules/control/remove_rule.go create mode 100644 cmd/frostfs-cli/modules/util/ape.go create mode 100644 cmd/frostfs-cli/modules/util/ape_test.go diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go new file mode 100644 index 000000000..0c6732108 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -0,0 +1,90 @@ +package control + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + ape "git.frostfs.info/TrueCloudLab/policy-engine" + "github.com/spf13/cobra" +) + +const ( + ruleFlag = "rule" +) + +var addRuleCmd = &cobra.Command{ + Use: "add-rule", + Short: "Add local override", + Long: "Add local APE rule to a node with following format:\n[:action_detail] [ ...] ", + Example: `allow Object.Get * +deny Object.Get EbxzAdz5LB4uqxuz6crWKAumBNtZyK2rKsqQP7TdZvwr/* +deny:QuotaLimitReached Object.Put Object.Resource:Department=HR * +`, + Run: addRule, +} + +func prettyJSONFormat(cmd *cobra.Command, serializedChain []byte) string { + wr := bytes.NewBufferString("") + err := json.Indent(wr, serializedChain, "", " ") + commonCmd.ExitOnErr(cmd, "%w", err) + return wr.String() +} + +func addRule(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + var cnr cid.ID + cidStr, _ := cmd.Flags().GetString(commonflags.CIDFlag) + commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(cidStr)) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + rule, _ := cmd.Flags().GetString(ruleFlag) + + chain := new(ape.Chain) + commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, []string{rule})) + serializedChain := chain.Bytes() + + cmd.Println("Container ID: " + cidStr) + cmd.Println("Parsed chain:\n" + prettyJSONFormat(cmd, serializedChain)) + + req := &control.AddChainLocalOverrideRequest{ + Body: &control.AddChainLocalOverrideRequest_Body{ + ContainerId: rawCID, + Chain: serializedChain, + }, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.AddChainLocalOverrideResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.AddChainLocalOverride(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Rule has been added. Chain id: ", resp.GetBody().GetChainId()) +} + +func initControlAddRuleCmd() { + initControlFlags(addRuleCmd) + + ff := addRuleCmd.Flags() + ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + ff.String(ruleFlag, "", "Rule statement") +} diff --git a/cmd/frostfs-cli/modules/control/get_rule.go b/cmd/frostfs-cli/modules/control/get_rule.go new file mode 100644 index 000000000..df2bf94fe --- /dev/null +++ b/cmd/frostfs-cli/modules/control/get_rule.go @@ -0,0 +1,69 @@ +package control + +import ( + "crypto/sha256" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + "github.com/spf13/cobra" +) + +var getRuleCmd = &cobra.Command{ + Use: "get-rule", + Short: "Get local override", + Long: "Get local APE override of the node", + Run: getRule, +} + +func getRule(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + var cnr cid.ID + cidStr, _ := cmd.Flags().GetString(commonflags.CIDFlag) + commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(cidStr)) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + chainID, _ := cmd.Flags().GetString(chainIDFlag) + + req := &control.GetChainLocalOverrideRequest{ + Body: &control.GetChainLocalOverrideRequest_Body{ + ContainerId: rawCID, + ChainId: chainID, + }, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.GetChainLocalOverrideResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.GetChainLocalOverride(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + var chain policyengine.Chain + commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(resp.GetBody().GetChain())) + + // TODO (aarifullin): make pretty-formatted output for chains. + cmd.Println("Parsed chain:\n" + prettyJSONFormat(cmd, chain.Bytes())) +} + +func initControGetRuleCmd() { + initControlFlags(getRuleCmd) + + ff := getRuleCmd.Flags() + ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + ff.String(chainIDFlag, "", "Chain id") +} diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go new file mode 100644 index 000000000..b3d7a5b9c --- /dev/null +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -0,0 +1,72 @@ +package control + +import ( + "crypto/sha256" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + "github.com/spf13/cobra" +) + +var listRulesCmd = &cobra.Command{ + Use: "list-rules", + Short: "List local overrides", + Long: "List local APE overrides of the node", + Run: listRules, +} + +func listRules(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + var cnr cid.ID + cidStr, _ := cmd.Flags().GetString(commonflags.CIDFlag) + commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(cidStr)) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + req := &control.ListChainLocalOverridesRequest{ + Body: &control.ListChainLocalOverridesRequest_Body{ + ContainerId: rawCID, + }, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.ListChainLocalOverridesResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.ListChainLocalOverrides(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + chains := resp.GetBody().GetChains() + if len(chains) == 0 { + cmd.Println("Local overrides are not defined for the container.") + return + } + + for _, c := range chains { + // TODO (aarifullin): make pretty-formatted output for chains. + var chain policyengine.Chain + commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(c)) + cmd.Println("Parsed chain:\n" + prettyJSONFormat(cmd, chain.Bytes())) + } +} + +func initControlListRulesCmd() { + initControlFlags(listRulesCmd) + + ff := listRulesCmd.Flags() + ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) +} diff --git a/cmd/frostfs-cli/modules/control/remove_rule.go b/cmd/frostfs-cli/modules/control/remove_rule.go new file mode 100644 index 000000000..de064536b --- /dev/null +++ b/cmd/frostfs-cli/modules/control/remove_rule.go @@ -0,0 +1,72 @@ +package control + +import ( + "crypto/sha256" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +const ( + chainIDFlag = "chain-id" +) + +var removeRuleCmd = &cobra.Command{ + Use: "remove-rule", + Short: "Remove local override", + Long: "Remove local APE override of the node", + Run: removeRule, +} + +func removeRule(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + var cnr cid.ID + cidStr, _ := cmd.Flags().GetString(commonflags.CIDFlag) + commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(cidStr)) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + chainID, _ := cmd.Flags().GetString(chainIDFlag) + + req := &control.RemoveChainLocalOverrideRequest{ + Body: &control.RemoveChainLocalOverrideRequest_Body{ + ContainerId: rawCID, + ChainId: chainID, + }, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.RemoveChainLocalOverrideResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.RemoveChainLocalOverride(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + if resp.GetBody().Removed { + cmd.Println("Rule has been removed.") + } else { + cmd.Println("Rule has not been removed.") + } +} + +func initControlRemoveRuleCmd() { + initControlFlags(removeRuleCmd) + + ff := removeRuleCmd.Flags() + ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + ff.String(chainIDFlag, "", "Chain id") +} diff --git a/cmd/frostfs-cli/modules/control/root.go b/cmd/frostfs-cli/modules/control/root.go index 015676185..f7c846eba 100644 --- a/cmd/frostfs-cli/modules/control/root.go +++ b/cmd/frostfs-cli/modules/control/root.go @@ -34,6 +34,10 @@ func init() { shardsCmd, synchronizeTreeCmd, irCmd, + addRuleCmd, + removeRuleCmd, + listRulesCmd, + getRuleCmd, ) initControlHealthCheckCmd() @@ -42,4 +46,8 @@ func init() { initControlShardsCmd() initControlSynchronizeTreeCmd() initControlIRCmd() + initControlAddRuleCmd() + initControlRemoveRuleCmd() + initControlListRulesCmd() + initControGetRuleCmd() } diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go new file mode 100644 index 000000000..881546450 --- /dev/null +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -0,0 +1,179 @@ +package util + +import ( + "errors" + "fmt" + "strings" + + policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + "github.com/flynn-archive/go-shlex" +) + +var ( + errInvalidStatementFormat = errors.New("invalid statement format") + errInvalidConditionFormat = errors.New("invalid condition format") + errUnknownAction = errors.New("action is not recognized") + errUnknownOperation = errors.New("operation is not recognized") + errUnknownActionDetail = errors.New("action detail is not recognized") + errUnknownBinaryOperator = errors.New("binary operator is not recognized") + errUnknownCondObjectType = errors.New("condition object type is not recognized") +) + +// ParseAPEChain parses APE chain rules. +func ParseAPEChain(chain *policyengine.Chain, rules []string) error { + if len(rules) == 0 { + return errors.New("no APE rules provided") + } + + for _, rule := range rules { + r := new(policyengine.Rule) + if err := ParseAPERule(r, rule); err != nil { + return err + } + chain.Rules = append(chain.Rules, *r) + } + + return nil +} + +// ParseAPERule parses access-policy-engine statement from the following form: +// [:action_detail] [ ...] +// +// Examples: +// deny Object.Put * +// deny:QuotaLimitReached Object.Put * +// allow Object.Put * +// allow Object.Get Object.Resource:Department=HR Object.Request:Actor=ownerA * +// +//nolint:godot +func ParseAPERule(r *policyengine.Rule, rule string) error { + lexemes, err := shlex.Split(rule) + if err != nil { + return fmt.Errorf("can't parse rule '%s': %v", rule, err) + } + return parseRuleLexemes(r, lexemes) +} + +func parseRuleLexemes(r *policyengine.Rule, lexemes []string) error { + if len(lexemes) < 2 { + return errInvalidStatementFormat + } + + var err error + r.Status, err = parseStatus(lexemes[0]) + if err != nil { + return err + } + + r.Action, err = parseAction(lexemes[1]) + if err != nil { + return err + } + + r.Condition, err = parseConditions(lexemes[2 : len(lexemes)-1]) + if err != nil { + return err + } + + r.Resource, err = parseResource(lexemes[len(lexemes)-1]) + return err +} + +func parseStatus(lexeme string) (policyengine.Status, error) { + action, expression, found := strings.Cut(lexeme, ":") + switch action = strings.ToLower(action); action { + case "deny": + if !found { + return policyengine.AccessDenied, nil + } else if strings.EqualFold(expression, "QuotaLimitReached") { + return policyengine.QuotaLimitReached, nil + } else { + return 0, fmt.Errorf("%w: %s", errUnknownActionDetail, expression) + } + case "allow": + if found { + return 0, errUnknownActionDetail + } + return policyengine.Allow, nil + default: + return 0, errUnknownAction + } +} + +func parseAction(lexeme string) ([]string, error) { + switch strings.ToLower(lexeme) { + case "object.put": + return []string{"native:PutObject"}, nil + case "object.get": + return []string{"native:GetObject"}, nil + case "object.head": + return []string{"native:HeadObject"}, nil + case "object.delete": + return []string{"native:DeleteObject"}, nil + case "object.search": + return []string{"native:SearchObject"}, nil + case "object.range": + return []string{"native:RangeObject"}, nil + case "object.hash": + return []string{"native:HashObject"}, nil + default: + } + return nil, fmt.Errorf("%w: %s", errUnknownOperation, lexeme) +} + +func parseResource(lexeme string) ([]string, error) { + return []string{fmt.Sprintf("native:::object/%s", lexeme)}, nil +} + +const ( + ObjectResource = "object.resource" + ObjectRequest = "object.request" + ObjectActor = "object.actor" +) + +var typeToCondObject = map[string]policyengine.ObjectType{ + ObjectResource: policyengine.ObjectResource, + ObjectRequest: policyengine.ObjectRequest, + ObjectActor: policyengine.ObjectActor, +} + +func parseConditions(lexemes []string) ([]policyengine.Condition, error) { + conds := make([]policyengine.Condition, 0) + + for _, lexeme := range lexemes { + typ, expression, found := strings.Cut(lexeme, ":") + typ = strings.ToLower(typ) + + objType, ok := typeToCondObject[typ] + if ok { + if !found { + return nil, fmt.Errorf("%w: %s", errInvalidConditionFormat, lexeme) + } + + var lhs, rhs string + var binExpFound bool + + var cond policyengine.Condition + cond.Object = objType + + lhs, rhs, binExpFound = strings.Cut(expression, "!=") + if !binExpFound { + lhs, rhs, binExpFound = strings.Cut(expression, "=") + if !binExpFound { + return nil, fmt.Errorf("%w: %s", errUnknownBinaryOperator, expression) + } + cond.Op = policyengine.CondStringEquals + } else { + cond.Op = policyengine.CondStringNotEquals + } + + cond.Key, cond.Value = lhs, rhs + + conds = append(conds, cond) + } else { + return nil, fmt.Errorf("%w: %s", errUnknownCondObjectType, typ) + } + } + + return conds, nil +} diff --git a/cmd/frostfs-cli/modules/util/ape_test.go b/cmd/frostfs-cli/modules/util/ape_test.go new file mode 100644 index 000000000..e71544a9b --- /dev/null +++ b/cmd/frostfs-cli/modules/util/ape_test.go @@ -0,0 +1,129 @@ +package util + +import ( + "testing" + + policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + "github.com/stretchr/testify/require" +) + +func TestParseAPERule(t *testing.T) { + tests := [...]struct { + name string + rule string + expectErr error + expectRule policyengine.Rule + }{ + { + name: "Valid allow rule", + rule: "allow Object.Put *", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Action: []string{"native:PutObject"}, + Resource: []string{"native:::object/*"}, + Condition: []policyengine.Condition{}, + }, + }, + { + name: "Valid deny rule", + rule: "deny Object.Put *", + expectRule: policyengine.Rule{ + Status: policyengine.AccessDenied, + Action: []string{"native:PutObject"}, + Resource: []string{"native:::object/*"}, + Condition: []policyengine.Condition{}, + }, + }, + { + name: "Valid deny rule with action detail", + rule: "deny:QuotaLimitReached Object.Put *", + expectRule: policyengine.Rule{ + Status: policyengine.QuotaLimitReached, + Action: []string{"native:PutObject"}, + Resource: []string{"native:::object/*"}, + Condition: []policyengine.Condition{}, + }, + }, + { + name: "Valid allow rule with conditions", + rule: "allow Object.Get Object.Resource:Department=HR Object.Request:Actor!=ownerA *", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Action: []string{"native:GetObject"}, + Resource: []string{"native:::object/*"}, + Condition: []policyengine.Condition{ + { + Op: policyengine.CondStringEquals, + Object: policyengine.ObjectResource, + Key: "Department", + Value: "HR", + }, + { + Op: policyengine.CondStringNotEquals, + Object: policyengine.ObjectRequest, + Key: "Actor", + Value: "ownerA", + }, + }, + }, + }, + { + name: "Valid rule with conditions with action detail", + rule: "deny:QuotaLimitReached Object.Get Object.Resource:Department=HR Object.Request:Actor!=ownerA *", + expectRule: policyengine.Rule{ + Status: policyengine.QuotaLimitReached, + Action: []string{"native:GetObject"}, + Resource: []string{"native:::object/*"}, + Condition: []policyengine.Condition{ + { + Op: policyengine.CondStringEquals, + Object: policyengine.ObjectResource, + Key: "Department", + Value: "HR", + }, + { + Op: policyengine.CondStringNotEquals, + Object: policyengine.ObjectRequest, + Key: "Actor", + Value: "ownerA", + }, + }, + }, + }, + { + name: "Invalid rule with unknown action", + rule: "permit Object.Put *", + expectErr: errUnknownAction, + }, + { + name: "Invalid rule with unknown operation", + rule: "allow Object.PutOut *", + expectErr: errUnknownOperation, + }, + { + name: "Invalid rule with unknown action detail", + rule: "deny:UnknownActionDetail Object.Put *", + expectErr: errUnknownActionDetail, + }, + { + name: "Invalid rule with unknown condition binary operator", + rule: "deny Object.Put Object.Resource:Department
Date: Tue, 7 Nov 2023 15:19:55 +0300 Subject: [PATCH 0663/1943] [#xx] adm: Drop notaryDisabled deploy parameter Refs https://git.frostfs.info/TrueCloudLab/frostfs-contract/pulls/50 Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/initialize_deploy.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 8d46793e5..f6100f4f9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -505,8 +505,7 @@ func getContractDeployParameters(cs *contractState, deployData []any) []any { } func (c *initializeContext) getContractDeployData(ctrName string, keysParam []any, method string) []any { - items := make([]any, 1, 6) - items[0] = false // notaryDisabled is false + items := make([]any, 0, 6) switch ctrName { case frostfsContract: @@ -596,12 +595,11 @@ func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, } func (c *initializeContext) getAlphabetDeployItems(i, n int) []any { - items := make([]any, 6) - items[0] = false - items[1] = c.Contracts[netmapContract].Hash - items[2] = c.Contracts[proxyContract].Hash - items[3] = innerring.GlagoliticLetter(i).String() - items[4] = int64(i) - items[5] = int64(n) + items := make([]any, 5) + items[0] = c.Contracts[netmapContract].Hash + items[1] = c.Contracts[proxyContract].Hash + items[2] = innerring.GlagoliticLetter(i).String() + items[3] = int64(i) + items[4] = int64(n) return items } From 3534d6d05bdddcf40783d51aa281434e2e128cc0 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Wed, 8 Nov 2023 17:12:45 +0300 Subject: [PATCH 0664/1943] [#794] objectsvc: Return accidentally removed acl checks for Head Signed-off-by: Airat Arifullin --- pkg/services/object/acl/v2/service.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 6d6f9c448..ee167d0e4 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -218,6 +218,12 @@ func (b Service) Head( reqInfo.obj = obj + if !b.checker.CheckBasicACL(reqInfo) { + return nil, basicACLErr(reqInfo) + } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { + return nil, eACLErr(reqInfo, err) + } + resp, err := b.next.Head(ctx, request) if err == nil { if err = b.checker.CheckEACL(resp, reqInfo); err != nil { From 1b22801eed60949f853e63ccd069dea557b9c3d6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 9 Nov 2023 10:25:46 +0300 Subject: [PATCH 0665/1943] [#788] engine: Fix flaky tests Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/lock_test.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index d5c080222..ec18fc5e3 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -137,13 +137,12 @@ func TestLockUserScenario(t *testing.T) { // 5. e.HandleNewEpoch(lockerExpiresAfter + 1) - // delay for GC - time.Sleep(time.Second) - inhumePrm.WithTarget(tombAddr, objAddr) - _, err = e.Inhume(context.Background(), inhumePrm) - require.NoError(t, err) + require.Eventually(t, func() bool { + _, err = e.Inhume(context.Background(), inhumePrm) + return err == nil + }, 30*time.Second, time.Second) } func TestLockExpiration(t *testing.T) { @@ -213,15 +212,13 @@ func TestLockExpiration(t *testing.T) { // 3. e.HandleNewEpoch(lockerExpiresAfter + 1) - // delay for GC processing. It can't be estimated, but making it bigger - // will slow down test - time.Sleep(time.Second) - // 4. inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj)) - _, err = e.Inhume(context.Background(), inhumePrm) - require.NoError(t, err) + require.Eventually(t, func() bool { + _, err = e.Inhume(context.Background(), inhumePrm) + return err == nil + }, 30*time.Second, time.Second) } func TestLockForceRemoval(t *testing.T) { From 9133b4389e764611d7211576b97b01aa531c8e8c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 9 Nov 2023 10:27:32 +0300 Subject: [PATCH 0666/1943] [#788] objectsvc: Fix formatting (gofumpt) Signed-off-by: Dmitrii Stepanov --- pkg/services/object/acl/ape.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/services/object/acl/ape.go b/pkg/services/object/acl/ape.go index b064b1eec..86af440fe 100644 --- a/pkg/services/object/acl/ape.go +++ b/pkg/services/object/acl/ape.go @@ -11,9 +11,7 @@ import ( policyengine "git.frostfs.info/TrueCloudLab/policy-engine" ) -var ( - errAPEChainNoSource = errors.New("could not get ape chain source for the container") -) +var errAPEChainNoSource = errors.New("could not get ape chain source for the container") type apeCheckerImpl struct { log *logger.Logger From 1bca8f118f6e29f9dbf015be12b03e5eb4a670dc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 8 Nov 2023 15:43:54 +0300 Subject: [PATCH 0667/1943] [#792] makefile: Fix protoc and staticcheck versions Signed-off-by: Dmitrii Stepanov --- .golangci.yml | 2 +- Makefile | 61 +++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index ef99fc152..5c7e9d918 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -39,7 +39,7 @@ linters-settings: alias: objectSDK custom: truecloudlab-linters: - path: bin/external_linters.so + path: bin/linters/external_linters.so original-url: git.frostfs.info/TrueCloudLab/linters.git settings: noliteral: diff --git a/Makefile b/Makefile index 14e32e40f..3fcb5027c 100755 --- a/Makefile +++ b/Makefile @@ -10,6 +10,14 @@ HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.21 LINT_VERSION ?= 1.54.0 TRUECLOUDLAB_LINT_VERSION ?= 0.0.2 +PROTOC_VERSION ?= 25.0 +PROTOC_GEN_GO_VERSION ?= $(shell go list -f '{{.Version}}' -m google.golang.org/protobuf) +PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2) +PROTOC_OS_VERSION=osx-x86_64 +ifeq ($(shell uname), Linux) + PROTOC_OS_VERSION=linux-x86_64 +endif +STATICCHECK_VERSION ?= 2023.1.6 ARCH = amd64 BIN = bin @@ -26,9 +34,15 @@ PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \ sed "s/-/~/")-${OS_RELEASE} -OUTPUT_LINT_DIR ?= $(shell pwd)/bin +OUTPUT_LINT_DIR ?= $(abspath $(BIN))/linters LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION) TMP_DIR := .cache +PROTOBUF_DIR ?= $(abspath $(BIN))/protobuf +PROTOC_DIR ?= $(PROTOBUF_DIR)/protoc-v$(PROTOC_VERSION) +PROTOC_GEN_GO_DIR ?= $(PROTOBUF_DIR)/protoc-gen-go-$(PROTOC_GEN_GO_VERSION) +PROTOGEN_FROSTFS_DIR ?= $(PROTOBUF_DIR)/protogen-$(PROTOGEN_FROSTFS_VERSION) +STATICCHECK_DIR ?= $(abspath $(BIN))/staticcheck +STATICCHECK_VERSION_DIR ?= $(STATICCHECK_DIR)/$(STATICCHECK_VERSION) .PHONY: help all images dep clean fmts fumpt imports test lint docker/lint prepare-release debpackage pre-commit unpre-commit @@ -78,22 +92,32 @@ export-metrics: dep # Regenerate proto files: protoc: - @GOPRIVATE=github.com/TrueCloudLab go mod vendor - # Install specific version for protobuf lib - @go list -f '{{.Path}}/...@{{.Version}}' -m github.com/golang/protobuf | xargs go install -v - @GOBIN=$(abspath $(BIN)) go install -mod=mod -v git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/protogen - # Protoc generate - @for f in `find . -type f -name '*.proto' -not -path './vendor/*'`; do \ + @if [ ! -d "$(PROTOC_DIR)" ] || [ ! -d "$(PROTOC_GEN_GO_DIR)" ] || [ ! -d "$(PROTOGEN_FROSTFS_DIR)" ]; then \ + make protoc-install; \ + fi + @for f in `find . -type f -name '*.proto' -not -path './bin/*'`; do \ echo "⇒ Processing $$f "; \ - protoc \ - --proto_path=.:./vendor:/usr/local/include \ - --plugin=protoc-gen-go-frostfs=$(BIN)/protogen \ + $(PROTOC_DIR)/bin/protoc \ + --proto_path=.:$(PROTOC_DIR)/include:/usr/local/include \ + --plugin=protoc-gen-go=$(PROTOC_GEN_GO_DIR)/protoc-gen-go \ + --plugin=protoc-gen-go-frostfs=$(PROTOGEN_FROSTFS_DIR)/protogen \ --go-frostfs_out=. --go-frostfs_opt=paths=source_relative \ --go_out=. --go_opt=paths=source_relative \ --go-grpc_opt=require_unimplemented_servers=false \ --go-grpc_out=. --go-grpc_opt=paths=source_relative $$f; \ done - rm -rf vendor + +protoc-install: + @rm -rf $(PROTOBUF_DIR) + @mkdir $(PROTOBUF_DIR) + @echo "⇒ Installing protoc... " + @wget -q -O $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip 'https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_OS_VERSION).zip' + @unzip -q -o $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip -d $(PROTOC_DIR) + @rm $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip + @echo "⇒ Installing protoc-gen-go..." + @GOBIN=$(PROTOC_GEN_GO_DIR) go install -v google.golang.org/protobuf/...@$(PROTOC_GEN_GO_VERSION) + @echo "⇒ Instaling protogen FrostFS plugin..." + @GOBIN=$(PROTOGEN_FROSTFS_DIR) go install -mod=mod -v git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/protogen@$(PROTOGEN_FROSTFS_VERSION) # Build FrostFS component's docker image image-%: @@ -143,6 +167,8 @@ pre-commit-run: # Install linters lint-install: + @rm -rf $(OUTPUT_LINT_DIR) + @mkdir $(OUTPUT_LINT_DIR) @mkdir -p $(TMP_DIR) @rm -rf $(TMP_DIR)/linters @git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters @@ -154,18 +180,22 @@ lint-install: # Run linters lint: @if [ ! -d "$(LINT_DIR)" ]; then \ - echo "Run make lint-install"; \ - exit 1; \ + make lint-install; \ fi $(LINT_DIR)/golangci-lint run # Install staticcheck staticcheck-install: - @go install honnef.co/go/tools/cmd/staticcheck@latest + @rm -rf $(STATICCHECK_DIR) + @mkdir $(STATICCHECK_DIR) + @GOBIN=$(STATICCHECK_VERSION_DIR) go install honnef.co/go/tools/cmd/staticcheck@$(STATICCHECK_VERSION) # Run staticcheck staticcheck-run: - @staticcheck ./... + @if [ ! -d "$(STATICCHECK_VERSION_DIR)" ]; then \ + make staticcheck-install; \ + fi + @$(STATICCHECK_VERSION_DIR)/staticcheck ./... # Run linters in Docker docker/lint: @@ -189,7 +219,6 @@ version: # Delete built artifacts clean: - rm -rf vendor rm -rf .cache rm -rf $(BIN) rm -rf $(RELEASE) From 7cdae4f66075977cdb216de25f5809aa70839fe5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 8 Nov 2023 15:45:03 +0300 Subject: [PATCH 0668/1943] [#792] proto: Regenerate with fixed version Signed-off-by: Dmitrii Stepanov --- pkg/services/control/ir/service.pb.go | 4 ++-- pkg/services/control/ir/service_grpc.pb.go | 2 +- pkg/services/control/ir/types.pb.go | 4 ++-- pkg/services/control/service.pb.go | 4 ++-- pkg/services/control/service_grpc.pb.go | 2 +- pkg/services/control/types.pb.go | 4 ++-- pkg/services/tree/service.pb.go | 4 ++-- pkg/services/tree/service_grpc.pb.go | 2 +- pkg/services/tree/types.pb.go | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index 56d52be4c..1b81a5619 100644 --- a/pkg/services/control/ir/service.pb.go +++ b/pkg/services/control/ir/service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v4.24.4 +// protoc-gen-go v1.31.0 +// protoc v4.25.0 // source: pkg/services/control/ir/service.proto package control diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index 004c82446..724149c44 100644 --- a/pkg/services/control/ir/service_grpc.pb.go +++ b/pkg/services/control/ir/service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v4.24.4 +// - protoc v4.25.0 // source: pkg/services/control/ir/service.proto package control diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go index 0ff8f45e9..cc39ad572 100644 --- a/pkg/services/control/ir/types.pb.go +++ b/pkg/services/control/ir/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v4.24.4 +// protoc-gen-go v1.31.0 +// protoc v4.25.0 // source: pkg/services/control/ir/types.proto package control diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 547ea7e7e..346a43664 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v4.24.4 +// protoc-gen-go v1.31.0 +// protoc v4.25.0 // source: pkg/services/control/service.proto package control diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index b5881ccb2..c3976c54a 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v4.24.4 +// - protoc v4.25.0 // source: pkg/services/control/service.proto package control diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index 33700c5b9..f031f8043 100644 --- a/pkg/services/control/types.pb.go +++ b/pkg/services/control/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v4.24.4 +// protoc-gen-go v1.31.0 +// protoc v4.25.0 // source: pkg/services/control/types.proto package control diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go index 63f3e714a..534661389 100644 --- a/pkg/services/tree/service.pb.go +++ b/pkg/services/tree/service.pb.go @@ -3,8 +3,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v3.21.9 +// protoc-gen-go v1.31.0 +// protoc v4.25.0 // source: pkg/services/tree/service.proto package tree diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index 2c0828951..63f96e11a 100644 --- a/pkg/services/tree/service_grpc.pb.go +++ b/pkg/services/tree/service_grpc.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.9 +// - protoc v4.25.0 // source: pkg/services/tree/service.proto package tree diff --git a/pkg/services/tree/types.pb.go b/pkg/services/tree/types.pb.go index b4d6981ef..f6b19c20f 100644 --- a/pkg/services/tree/types.pb.go +++ b/pkg/services/tree/types.pb.go @@ -3,8 +3,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v3.21.9 +// protoc-gen-go v1.31.0 +// protoc v4.25.0 // source: pkg/services/tree/types.proto package tree From 0f75e48138220675a50fd473318acd1a7a7e7500 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 9 Nov 2023 13:33:03 +0300 Subject: [PATCH 0669/1943] [#796] policer: Fix tombstone objects replication Tombstone objects must be replicated to all container nodes. Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/check.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 9b44aceda..9e40219d1 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -93,10 +93,10 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi // Number of copies that are stored on maintenance nodes. var uncheckedCopies int - if typ == objectSDK.TypeLock { - // all nodes of a container must store the `LOCK` objects + if typ == objectSDK.TypeLock || typ == objectSDK.TypeTombstone { + // all nodes of a container must store the `LOCK` and `TOMBSTONE` objects // for correct object removal protection: - // - `LOCK` objects are broadcast on their PUT requests; + // - `LOCK` and `TOMBSTONE` objects are broadcast on their PUT requests; // - `LOCK` object removal is a prohibited action in the GC. shortage = uint32(len(nodes)) } From 78cfb6aea86c01df34a534020dc63cefbad61da0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 9 Nov 2023 13:33:59 +0300 Subject: [PATCH 0670/1943] [#796] cli: Fix object nodes command Tombstone objects must be present on all container nodes. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index ef6c29179..70a3eb75f 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -31,10 +31,10 @@ const ( ) type objectNodesInfo struct { - containerID cid.ID - objectID oid.ID - relatedObjectIDs []oid.ID - isLock bool + containerID cid.ID + objectID oid.ID + relatedObjectIDs []oid.ID + isLockOrTombstone bool } type boolError struct { @@ -101,9 +101,9 @@ func getObjectInfo(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.C res, err := internalclient.HeadObject(cmd.Context(), prmHead) if err == nil { return &objectNodesInfo{ - containerID: cnrID, - objectID: objID, - isLock: res.Header().Type() == objectSDK.TypeLock, + containerID: cnrID, + objectID: objID, + isLockOrTombstone: res.Header().Type() == objectSDK.TypeLock || res.Header().Type() == objectSDK.TypeTombstone, } } @@ -191,7 +191,7 @@ func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placemen numOfReplicas := placementPolicy.ReplicaNumberByIndex(repIdx) var nodeIdx uint32 for _, n := range rep { - if !objInfo.isLock && nodeIdx == numOfReplicas { // lock object should be on all container nodes + if !objInfo.isLockOrTombstone && nodeIdx == numOfReplicas { // lock and tombstone objects should be on all container nodes break } nodes[n.Hash()] = n From bdfa5234875141c6152284ea19eba440aeaed702 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 7 Nov 2023 16:44:43 +0300 Subject: [PATCH 0671/1943] [#787] proto: Add VUB field for IR service Signed-off-by: Dmitrii Stepanov --- pkg/services/control/ir/service.pb.go | 219 ++++++++++++------ pkg/services/control/ir/service.proto | 27 ++- pkg/services/control/ir/service_frostfs.pb.go | 40 ++++ 3 files changed, 203 insertions(+), 83 deletions(-) diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index 1b81a5619..a9be56cd9 100644 --- a/pkg/services/control/ir/service.pb.go +++ b/pkg/services/control/ir/service.pb.go @@ -560,6 +560,9 @@ type TickEpochRequest_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + // Valid until block value override. + Vub uint32 `protobuf:"varint,1,opt,name=vub,proto3" json:"vub,omitempty"` } func (x *TickEpochRequest_Body) Reset() { @@ -594,10 +597,20 @@ func (*TickEpochRequest_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{2, 0} } +func (x *TickEpochRequest_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} + type TickEpochResponse_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + // Valid until block value for transaction. + Vub uint32 `protobuf:"varint,1,opt,name=vub,proto3" json:"vub,omitempty"` } func (x *TickEpochResponse_Body) Reset() { @@ -632,12 +645,21 @@ func (*TickEpochResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{3, 0} } +func (x *TickEpochResponse_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} + type RemoveNodeRequest_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // Valid until block value override. + Vub uint32 `protobuf:"varint,2,opt,name=vub,proto3" json:"vub,omitempty"` } func (x *RemoveNodeRequest_Body) Reset() { @@ -679,10 +701,20 @@ func (x *RemoveNodeRequest_Body) GetKey() []byte { return nil } +func (x *RemoveNodeRequest_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} + type RemoveNodeResponse_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + // Valid until block value for transaction. + Vub uint32 `protobuf:"varint,1,opt,name=vub,proto3" json:"vub,omitempty"` } func (x *RemoveNodeResponse_Body) Reset() { @@ -717,6 +749,13 @@ func (*RemoveNodeResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{5, 0} } +func (x *RemoveNodeResponse_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} + type RemoveContainerRequest_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -724,6 +763,8 @@ type RemoveContainerRequest_Body struct { ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` Owner []byte `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + // Valid until block value override. + Vub uint32 `protobuf:"varint,3,opt,name=vub,proto3" json:"vub,omitempty"` } func (x *RemoveContainerRequest_Body) Reset() { @@ -772,10 +813,20 @@ func (x *RemoveContainerRequest_Body) GetOwner() []byte { return nil } +func (x *RemoveContainerRequest_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} + type RemoveContainerResponse_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + // Valid until block value for transaction. + Vub uint32 `protobuf:"varint,1,opt,name=vub,proto3" json:"vub,omitempty"` } func (x *RemoveContainerResponse_Body) Reset() { @@ -810,6 +861,13 @@ func (*RemoveContainerResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{7, 0} } +func (x *RemoveContainerResponse_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} + var File_pkg_services_control_ir_service_proto protoreflect.FileDescriptor var file_pkg_services_control_ir_service_proto_rawDesc = []byte{ @@ -839,7 +897,7 @@ var file_pkg_services_control_ir_service_proto_rawDesc = []byte{ 0x3c, 0x0a, 0x0d, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x0c, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x84, 0x01, + 0x0c, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x96, 0x01, 0x0a, 0x10, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, @@ -847,82 +905,89 @@ var file_pkg_services_control_ir_service_proto_rawDesc = []byte{ 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x22, 0x86, 0x01, 0x0a, 0x11, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, - 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x98, 0x01, - 0x0a, 0x11, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x21, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, - 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x18, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x88, 0x01, 0x0a, 0x12, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, - 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, - 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, - 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x22, 0xc9, 0x01, 0x0a, 0x16, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3a, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x69, - 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, - 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x3f, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x77, 0x6e, - 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x22, - 0x92, 0x01, 0x0a, 0x17, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x69, 0x72, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x32, 0xcb, 0x02, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x09, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, - 0x63, 0x68, 0x12, 0x1b, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, - 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1c, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, - 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, - 0x0a, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x69, 0x72, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, - 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x0f, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x21, 0x2e, 0x69, 0x72, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, - 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, - 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, - 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, - 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x69, 0x72, - 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x18, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x75, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x03, 0x76, 0x75, 0x62, 0x22, 0x98, 0x01, 0x0a, 0x11, 0x54, 0x69, 0x63, 0x6b, 0x45, + 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x69, 0x72, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x18, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x76, 0x75, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x76, 0x75, + 0x62, 0x22, 0xaa, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, + 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x1a, 0x2a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x76, 0x75, 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x76, 0x75, 0x62, 0x22, 0x9a, + 0x01, 0x0a, 0x12, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x18, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x75, 0x62, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x76, 0x75, 0x62, 0x22, 0xdb, 0x01, 0x0a, 0x16, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x51, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, + 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, + 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x75, 0x62, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x76, 0x75, 0x62, 0x22, 0xa4, 0x01, 0x0a, 0x17, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x18, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x76, 0x75, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x76, 0x75, 0x62, + 0x32, 0xcb, 0x02, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x12, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1e, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, + 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x46, 0x0a, 0x09, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x1b, + 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, + 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x69, 0x72, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, + 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x0a, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x0f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x21, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x69, 0x72, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x44, + 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, + 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, + 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, + 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x69, 0x72, 0x2f, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/services/control/ir/service.proto b/pkg/services/control/ir/service.proto index cd890d438..967ff979e 100644 --- a/pkg/services/control/ir/service.proto +++ b/pkg/services/control/ir/service.proto @@ -21,8 +21,7 @@ service ControlService { // Health check request. message HealthCheckRequest { // Health check request body. - message Body { - } + message Body {} // Body of health check request message. Body body = 1; @@ -49,14 +48,20 @@ message HealthCheckResponse { } message TickEpochRequest { - message Body{} + message Body{ + // Valid until block value override. + uint32 vub = 1; + } Body body = 1; Signature signature = 2; } message TickEpochResponse { - message Body{} + message Body{ + // Valid until block value for transaction. + uint32 vub = 1; + } Body body = 1; Signature signature = 2; @@ -65,6 +70,8 @@ message TickEpochResponse { message RemoveNodeRequest { message Body{ bytes key = 1; + // Valid until block value override. + uint32 vub = 2; } Body body = 1; @@ -72,7 +79,10 @@ message RemoveNodeRequest { } message RemoveNodeResponse { - message Body{} + message Body{ + // Valid until block value for transaction. + uint32 vub = 1; + } Body body = 1; Signature signature = 2; @@ -82,6 +92,8 @@ message RemoveContainerRequest { message Body{ bytes container_id = 1; bytes owner = 2; + // Valid until block value override. + uint32 vub = 3; } Body body = 1; @@ -89,7 +101,10 @@ message RemoveContainerRequest { } message RemoveContainerResponse { - message Body{} + message Body{ + // Valid until block value for transaction. + uint32 vub = 1; + } Body body = 1; Signature signature = 2; diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index 131ec78da..e22d0013f 100644 --- a/pkg/services/control/ir/service_frostfs.pb.go +++ b/pkg/services/control/ir/service_frostfs.pb.go @@ -174,6 +174,7 @@ func (x *TickEpochRequest_Body) StableSize() (size int) { if x == nil { return 0 } + size += proto.UInt32Size(1, x.Vub) return size } @@ -186,6 +187,14 @@ func (x *TickEpochRequest_Body) StableSize() (size int) { // // Structures with the same field values have the same binary format. func (x *TickEpochRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.UInt32Marshal(1, buf[offset:], x.Vub) return buf } @@ -251,6 +260,7 @@ func (x *TickEpochResponse_Body) StableSize() (size int) { if x == nil { return 0 } + size += proto.UInt32Size(1, x.Vub) return size } @@ -263,6 +273,14 @@ func (x *TickEpochResponse_Body) StableSize() (size int) { // // Structures with the same field values have the same binary format. func (x *TickEpochResponse_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.UInt32Marshal(1, buf[offset:], x.Vub) return buf } @@ -329,6 +347,7 @@ func (x *RemoveNodeRequest_Body) StableSize() (size int) { return 0 } size += proto.BytesSize(1, x.Key) + size += proto.UInt32Size(2, x.Vub) return size } @@ -349,6 +368,7 @@ func (x *RemoveNodeRequest_Body) StableMarshal(buf []byte) []byte { } var offset int offset += proto.BytesMarshal(1, buf[offset:], x.Key) + offset += proto.UInt32Marshal(2, buf[offset:], x.Vub) return buf } @@ -414,6 +434,7 @@ func (x *RemoveNodeResponse_Body) StableSize() (size int) { if x == nil { return 0 } + size += proto.UInt32Size(1, x.Vub) return size } @@ -426,6 +447,14 @@ func (x *RemoveNodeResponse_Body) StableSize() (size int) { // // Structures with the same field values have the same binary format. func (x *RemoveNodeResponse_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.UInt32Marshal(1, buf[offset:], x.Vub) return buf } @@ -493,6 +522,7 @@ func (x *RemoveContainerRequest_Body) StableSize() (size int) { } size += proto.BytesSize(1, x.ContainerId) size += proto.BytesSize(2, x.Owner) + size += proto.UInt32Size(3, x.Vub) return size } @@ -514,6 +544,7 @@ func (x *RemoveContainerRequest_Body) StableMarshal(buf []byte) []byte { var offset int offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) offset += proto.BytesMarshal(2, buf[offset:], x.Owner) + offset += proto.UInt32Marshal(3, buf[offset:], x.Vub) return buf } @@ -579,6 +610,7 @@ func (x *RemoveContainerResponse_Body) StableSize() (size int) { if x == nil { return 0 } + size += proto.UInt32Size(1, x.Vub) return size } @@ -591,6 +623,14 @@ func (x *RemoveContainerResponse_Body) StableSize() (size int) { // // Structures with the same field values have the same binary format. func (x *RemoveContainerResponse_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.UInt32Marshal(1, buf[offset:], x.Vub) return buf } From 5466e884447401fa8d7441683111ac0b5a13608c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 7 Nov 2023 17:35:09 +0300 Subject: [PATCH 0672/1943] [#787] cli: Add `vub` for `control ir` commands Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/ir.go | 15 ++++++++++++++- .../modules/control/ir_remove_container.go | 6 +++++- cmd/frostfs-cli/modules/control/ir_remove_node.go | 4 +++- cmd/frostfs-cli/modules/control/ir_tick_epoch.go | 7 +++++-- cmd/frostfs-cli/modules/control/util.go | 11 +++++++++++ 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/ir.go b/cmd/frostfs-cli/modules/control/ir.go index ac1371db7..2a38f1e97 100644 --- a/cmd/frostfs-cli/modules/control/ir.go +++ b/cmd/frostfs-cli/modules/control/ir.go @@ -1,6 +1,9 @@ package control -import "github.com/spf13/cobra" +import ( + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/spf13/cobra" +) var irCmd = &cobra.Command{ Use: "ir", @@ -19,3 +22,13 @@ func initControlIRCmd() { initControlIRHealthCheckCmd() initControlIRRemoveContainerCmd() } + +func printVUB(cmd *cobra.Command, vub uint32) { + cmd.Printf("Transaction's valid until block is %d\n", vub) +} + +func parseVUB(cmd *cobra.Command) uint32 { + vub, err := cmd.Flags().GetUint32(irFlagNameVUB) + commonCmd.ExitOnErr(cmd, "invalid valid until block value: %w", err) + return vub +} diff --git a/cmd/frostfs-cli/modules/control/ir_remove_container.go b/cmd/frostfs-cli/modules/control/ir_remove_container.go index 43173bcaa..52eaa574d 100644 --- a/cmd/frostfs-cli/modules/control/ir_remove_container.go +++ b/cmd/frostfs-cli/modules/control/ir_remove_container.go @@ -29,7 +29,7 @@ To check removal status "frostfs-cli container list" command can be used.`, } func initControlIRRemoveContainerCmd() { - initControlFlags(removeContainerCmd) + initControlIRFlags(removeContainerCmd) flags := removeContainerCmd.Flags() flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) @@ -60,6 +60,7 @@ func removeContainer(cmd *cobra.Command, _ []string) { } else { cmd.Println("User containers sheduled to removal") } + printVUB(cmd, resp.GetBody().GetVub()) } func prepareRemoveContainerRequest(cmd *cobra.Command) *ircontrol.RemoveContainerRequest { @@ -90,5 +91,8 @@ func prepareRemoveContainerRequest(cmd *cobra.Command) *ircontrol.RemoveContaine commonCmd.ExitOnErr(cmd, "invalid container ID: %w", containerID.DecodeString(cidStr)) req.Body.ContainerId = containerID[:] } + + req.Body.Vub = parseVUB(cmd) + return req } diff --git a/cmd/frostfs-cli/modules/control/ir_remove_node.go b/cmd/frostfs-cli/modules/control/ir_remove_node.go index f5b968b7f..412dc7934 100644 --- a/cmd/frostfs-cli/modules/control/ir_remove_node.go +++ b/cmd/frostfs-cli/modules/control/ir_remove_node.go @@ -20,7 +20,7 @@ var removeNodeCmd = &cobra.Command{ } func initControlIRRemoveNodeCmd() { - initControlFlags(removeNodeCmd) + initControlIRFlags(removeNodeCmd) flags := removeNodeCmd.Flags() flags.String("node", "", "Node public key as a hex string") @@ -41,6 +41,7 @@ func removeNode(cmd *cobra.Command, _ []string) { req := new(ircontrol.RemoveNodeRequest) req.SetBody(&ircontrol.RemoveNodeRequest_Body{ Key: nodeKey, + Vub: parseVUB(cmd), }) commonCmd.ExitOnErr(cmd, "could not sign request: %w", ircontrolsrv.SignMessage(pk, req)) @@ -55,4 +56,5 @@ func removeNode(cmd *cobra.Command, _ []string) { verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) cmd.Println("Node removed") + printVUB(cmd, resp.GetBody().GetVub()) } diff --git a/cmd/frostfs-cli/modules/control/ir_tick_epoch.go b/cmd/frostfs-cli/modules/control/ir_tick_epoch.go index 3e6af0081..6965b5dca 100644 --- a/cmd/frostfs-cli/modules/control/ir_tick_epoch.go +++ b/cmd/frostfs-cli/modules/control/ir_tick_epoch.go @@ -17,7 +17,7 @@ var tickEpochCmd = &cobra.Command{ } func initControlIRTickEpochCmd() { - initControlFlags(tickEpochCmd) + initControlIRFlags(tickEpochCmd) } func tickEpoch(cmd *cobra.Command, _ []string) { @@ -25,7 +25,9 @@ func tickEpoch(cmd *cobra.Command, _ []string) { c := getClient(cmd, pk) req := new(ircontrol.TickEpochRequest) - req.SetBody(new(ircontrol.TickEpochRequest_Body)) + req.SetBody(&ircontrol.TickEpochRequest_Body{ + Vub: parseVUB(cmd), + }) err := ircontrolsrv.SignMessage(pk, req) commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) @@ -40,4 +42,5 @@ func tickEpoch(cmd *cobra.Command, _ []string) { verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) cmd.Println("Epoch tick requested") + printVUB(cmd, resp.GetBody().GetVub()) } diff --git a/cmd/frostfs-cli/modules/control/util.go b/cmd/frostfs-cli/modules/control/util.go index 5ad675c0e..ffaceff13 100644 --- a/cmd/frostfs-cli/modules/control/util.go +++ b/cmd/frostfs-cli/modules/control/util.go @@ -14,6 +14,10 @@ import ( "github.com/spf13/cobra" ) +const ( + irFlagNameVUB = "vub" +) + func initControlFlags(cmd *cobra.Command) { ff := cmd.Flags() ff.StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, commonflags.WalletPathDefault, commonflags.WalletPathUsage) @@ -22,6 +26,13 @@ func initControlFlags(cmd *cobra.Command) { ff.DurationP(commonflags.Timeout, commonflags.TimeoutShorthand, commonflags.TimeoutDefault, commonflags.TimeoutUsage) } +func initControlIRFlags(cmd *cobra.Command) { + initControlFlags(cmd) + + ff := cmd.Flags() + ff.Uint32(irFlagNameVUB, 0, "Valid until block value for notary transaction") +} + func signRequest(cmd *cobra.Command, pk *ecdsa.PrivateKey, req controlSvc.SignedMessage) { err := controlSvc.SignMessage(pk, req) commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) From 518f3baf41db089f3a790bdf1fd665af36fb21d6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 7 Nov 2023 18:13:26 +0300 Subject: [PATCH 0673/1943] [#787] morph: Return VUB from `Invoke` method Signed-off-by: Dmitrii Stepanov --- .../processors/alphabet/handlers_test.go | 4 +-- .../processors/alphabet/process_emit.go | 2 +- .../processors/alphabet/processor.go | 2 +- pkg/innerring/processors/netmap/wrappers.go | 3 ++- pkg/innerring/state.go | 2 +- pkg/morph/client/balance/burn.go | 3 ++- pkg/morph/client/balance/lock.go | 3 ++- pkg/morph/client/balance/mint.go | 3 ++- pkg/morph/client/balance/transfer.go | 2 +- pkg/morph/client/client.go | 9 ++++--- pkg/morph/client/container/delete.go | 2 +- pkg/morph/client/container/eacl_set.go | 2 +- pkg/morph/client/container/estimations.go | 4 +-- pkg/morph/client/container/load.go | 2 +- pkg/morph/client/container/put.go | 2 +- pkg/morph/client/frostfs/bind.go | 4 +-- pkg/morph/client/frostfs/cheque.go | 6 +++-- pkg/morph/client/frostfsid/addrm_keys.go | 4 +-- pkg/morph/client/netmap/config.go | 3 ++- pkg/morph/client/netmap/innerring.go | 3 ++- pkg/morph/client/netmap/new_epoch.go | 2 +- pkg/morph/client/netmap/peer.go | 2 +- pkg/morph/client/netmap/update_state.go | 2 +- pkg/morph/client/notary.go | 27 ++++++++++--------- pkg/morph/client/static.go | 19 +++++++++---- 25 files changed, 69 insertions(+), 48 deletions(-) diff --git a/pkg/innerring/processors/alphabet/handlers_test.go b/pkg/innerring/processors/alphabet/handlers_test.go index e39244a36..f94c8e78d 100644 --- a/pkg/innerring/processors/alphabet/handlers_test.go +++ b/pkg/innerring/processors/alphabet/handlers_test.go @@ -246,7 +246,7 @@ type testMorphClient struct { batchTransferedGas []batchTransferGas } -func (c *testMorphClient) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) error { +func (c *testMorphClient) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) { c.invokedMethods = append(c.invokedMethods, invokedMethod{ contract: contract, @@ -254,7 +254,7 @@ func (c *testMorphClient) Invoke(contract util.Uint160, fee fixedn.Fixed8, metho method: method, args: args, }) - return nil + return 0, nil } func (c *testMorphClient) TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error { diff --git a/pkg/innerring/processors/alphabet/process_emit.go b/pkg/innerring/processors/alphabet/process_emit.go index 8a2336011..2317f3e98 100644 --- a/pkg/innerring/processors/alphabet/process_emit.go +++ b/pkg/innerring/processors/alphabet/process_emit.go @@ -30,7 +30,7 @@ func (ap *Processor) processEmit() bool { } // there is no signature collecting, so we don't need extra fee - err := ap.morphClient.Invoke(contract, 0, emitMethod) + _, err := ap.morphClient.Invoke(contract, 0, emitMethod) if err != nil { ap.log.Warn(logs.AlphabetCantInvokeAlphabetEmitMethod, zap.String("error", err.Error())) diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index 04dde80f7..ce6679969 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -40,7 +40,7 @@ type ( } morphClient interface { - Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) error + Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error BatchTransferGas(receivers []util.Uint160, amount fixedn.Fixed8) error } diff --git a/pkg/innerring/processors/netmap/wrappers.go b/pkg/innerring/processors/netmap/wrappers.go index 255d498d3..f9af703de 100644 --- a/pkg/innerring/processors/netmap/wrappers.go +++ b/pkg/innerring/processors/netmap/wrappers.go @@ -23,7 +23,8 @@ func (w *netmapClientWrapper) UpdatePeerState(p netmapclient.UpdatePeerPrm) erro } func (w *netmapClientWrapper) MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { - return w.netmapClient.Morph().NotaryInvoke(contract, fee, nonce, vub, method, args...) + _, err := w.netmapClient.Morph().NotaryInvoke(contract, fee, nonce, vub, method, args...) + return err } func (w *netmapClientWrapper) ContractAddress() util.Uint160 { diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index c5adb71eb..f2521bd08 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -125,7 +125,7 @@ func (s *Server) voteForSidechainValidator(prm governance.VoteValidatorPrm) erro } s.contracts.alphabet.iterate(func(letter GlagoliticLetter, contract util.Uint160) { - err := s.morphClient.NotaryInvoke(contract, s.feeConfig.SideChainFee(), nonce, vubP, voteMethod, epoch, validators) + _, err := s.morphClient.NotaryInvoke(contract, s.feeConfig.SideChainFee(), nonce, vubP, voteMethod, epoch, validators) if err != nil { s.log.Warn(logs.InnerringCantInvokeVoteMethodInAlphabetContract, zap.Int8("alphabet_index", int8(letter)), diff --git a/pkg/morph/client/balance/burn.go b/pkg/morph/client/balance/burn.go index 88a1aa8f1..4befbef45 100644 --- a/pkg/morph/client/balance/burn.go +++ b/pkg/morph/client/balance/burn.go @@ -36,5 +36,6 @@ func (c *Client) Burn(p BurnPrm) error { prm.SetArgs(p.to, p.amount, p.id) prm.InvokePrmOptional = p.InvokePrmOptional - return c.client.Invoke(prm) + _, err := c.client.Invoke(prm) + return err } diff --git a/pkg/morph/client/balance/lock.go b/pkg/morph/client/balance/lock.go index 7b270808e..a5b206799 100644 --- a/pkg/morph/client/balance/lock.go +++ b/pkg/morph/client/balance/lock.go @@ -48,5 +48,6 @@ func (c *Client) Lock(p LockPrm) error { prm.SetArgs(p.id, p.user, p.lock, p.amount, p.dueEpoch) prm.InvokePrmOptional = p.InvokePrmOptional - return c.client.Invoke(prm) + _, err := c.client.Invoke(prm) + return err } diff --git a/pkg/morph/client/balance/mint.go b/pkg/morph/client/balance/mint.go index cda78c077..73448da31 100644 --- a/pkg/morph/client/balance/mint.go +++ b/pkg/morph/client/balance/mint.go @@ -36,5 +36,6 @@ func (c *Client) Mint(p MintPrm) error { prm.SetArgs(p.to, p.amount, p.id) prm.InvokePrmOptional = p.InvokePrmOptional - return c.client.Invoke(prm) + _, err := c.client.Invoke(prm) + return err } diff --git a/pkg/morph/client/balance/transfer.go b/pkg/morph/client/balance/transfer.go index 5206f69fa..08fb05289 100644 --- a/pkg/morph/client/balance/transfer.go +++ b/pkg/morph/client/balance/transfer.go @@ -39,7 +39,7 @@ func (c *Client) TransferX(p TransferPrm) error { prm.SetArgs(from, to, p.Amount, p.Details) prm.InvokePrmOptional = p.InvokePrmOptional - err = c.client.Invoke(prm) + _, err = c.client.Invoke(prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", transferXMethod, err) } diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 050838648..93a4176f9 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -174,8 +174,9 @@ func wrapFrostFSError(err error) error { } // Invoke invokes contract method by sending transaction into blockchain. +// Returns valid until block value. // Supported args types: int64, string, util.Uint160, []byte and bool. -func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) error { +func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) { start := time.Now() success := false defer func() { @@ -186,12 +187,12 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, defer c.switchLock.RUnlock() if c.inactive { - return ErrConnectionLost + return 0, ErrConnectionLost } txHash, vub, err := c.rpcActor.SendTunedCall(contract, method, nil, addFeeCheckerModifier(int64(fee)), args...) if err != nil { - return fmt.Errorf("could not invoke %s: %w", method, err) + return 0, fmt.Errorf("could not invoke %s: %w", method, err) } c.logger.Debug(logs.ClientNeoClientInvoke, @@ -200,7 +201,7 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, zap.Stringer("tx_hash", txHash.Reverse())) success = true - return nil + return vub, nil } // TestInvokeIterator invokes contract method returning an iterator and executes cb on each element. diff --git a/pkg/morph/client/container/delete.go b/pkg/morph/client/container/delete.go index b520120d2..c6cf7aa8d 100644 --- a/pkg/morph/client/container/delete.go +++ b/pkg/morph/client/container/delete.go @@ -76,7 +76,7 @@ func (c *Client) Delete(p DeletePrm) error { prm.SetArgs(p.cnr, p.signature, p.key, p.token) prm.InvokePrmOptional = p.InvokePrmOptional - err := c.client.Invoke(prm) + _, err := c.client.Invoke(prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", deleteMethod, err) } diff --git a/pkg/morph/client/container/eacl_set.go b/pkg/morph/client/container/eacl_set.go index 2d2ffb456..c68104593 100644 --- a/pkg/morph/client/container/eacl_set.go +++ b/pkg/morph/client/container/eacl_set.go @@ -85,7 +85,7 @@ func (c *Client) PutEACL(p PutEACLPrm) error { prm.SetArgs(p.table, p.sig, p.key, p.token) prm.InvokePrmOptional = p.InvokePrmOptional - err := c.client.Invoke(prm) + _, err := c.client.Invoke(prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", setEACLMethod, err) } diff --git a/pkg/morph/client/container/estimations.go b/pkg/morph/client/container/estimations.go index 6adf67476..f288c63cf 100644 --- a/pkg/morph/client/container/estimations.go +++ b/pkg/morph/client/container/estimations.go @@ -34,7 +34,7 @@ func (c *Client) StartEstimation(p StartEstimationPrm) error { prm.SetArgs(p.epoch) prm.InvokePrmOptional = p.InvokePrmOptional - if err := c.client.Invoke(prm); err != nil { + if _, err := c.client.Invoke(prm); err != nil { return fmt.Errorf("could not invoke method (%s): %w", startEstimationMethod, err) } return nil @@ -47,7 +47,7 @@ func (c *Client) StopEstimation(p StopEstimationPrm) error { prm.SetArgs(p.epoch) prm.InvokePrmOptional = p.InvokePrmOptional - if err := c.client.Invoke(prm); err != nil { + if _, err := c.client.Invoke(prm); err != nil { return fmt.Errorf("could not invoke method (%s): %w", stopEstimationMethod, err) } return nil diff --git a/pkg/morph/client/container/load.go b/pkg/morph/client/container/load.go index dcf89f73e..b5263d7a6 100644 --- a/pkg/morph/client/container/load.go +++ b/pkg/morph/client/container/load.go @@ -41,7 +41,7 @@ func (c *Client) AnnounceLoad(p AnnounceLoadPrm) error { prm.SetArgs(p.a.Epoch(), binCnr, p.a.Value(), p.key) prm.InvokePrmOptional = p.InvokePrmOptional - err := c.client.Invoke(prm) + _, err := c.client.Invoke(prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", putSizeMethod, err) } diff --git a/pkg/morph/client/container/put.go b/pkg/morph/client/container/put.go index 5c23eb36d..ee323af00 100644 --- a/pkg/morph/client/container/put.go +++ b/pkg/morph/client/container/put.go @@ -116,7 +116,7 @@ func (c *Client) Put(p PutPrm) error { prm.SetMethod(method) prm.InvokePrmOptional = p.InvokePrmOptional - err := c.client.Invoke(prm) + _, err := c.client.Invoke(prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", method, err) } diff --git a/pkg/morph/client/frostfs/bind.go b/pkg/morph/client/frostfs/bind.go index 5b15d5c7b..f7b91ddb8 100644 --- a/pkg/morph/client/frostfs/bind.go +++ b/pkg/morph/client/frostfs/bind.go @@ -41,7 +41,7 @@ func (x *Client) BindKeys(p BindKeysPrm) error { prm.SetArgs(p.scriptHash, p.keys) prm.InvokePrmOptional = p.InvokePrmOptional - err := x.client.Invoke(prm) + _, err := x.client.Invoke(prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", bindKeysMethod, err) } @@ -62,7 +62,7 @@ func (x *Client) UnbindKeys(args UnbindKeysPrm) error { prm.SetArgs(args.scriptHash, args.keys) prm.InvokePrmOptional = args.InvokePrmOptional - err := x.client.Invoke(prm) + _, err := x.client.Invoke(prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", unbindKeysMethod, err) } diff --git a/pkg/morph/client/frostfs/cheque.go b/pkg/morph/client/frostfs/cheque.go index e8f4f7f18..016b56f8f 100644 --- a/pkg/morph/client/frostfs/cheque.go +++ b/pkg/morph/client/frostfs/cheque.go @@ -43,7 +43,8 @@ func (x *Client) Cheque(p ChequePrm) error { prm.SetArgs(p.id, p.user, p.amount, p.lock) prm.InvokePrmOptional = p.InvokePrmOptional - return x.client.Invoke(prm) + _, err := x.client.Invoke(prm) + return err } // AlphabetUpdatePrm groups parameters of AlphabetUpdate operation. @@ -71,5 +72,6 @@ func (x *Client) AlphabetUpdate(p AlphabetUpdatePrm) error { prm.SetArgs(p.id, p.pubs) prm.InvokePrmOptional = p.InvokePrmOptional - return x.client.Invoke(prm) + _, err := x.client.Invoke(prm) + return err } diff --git a/pkg/morph/client/frostfsid/addrm_keys.go b/pkg/morph/client/frostfsid/addrm_keys.go index cbbd05142..e9004f366 100644 --- a/pkg/morph/client/frostfsid/addrm_keys.go +++ b/pkg/morph/client/frostfsid/addrm_keys.go @@ -36,7 +36,7 @@ func (x *Client) AddKeys(p CommonBindPrm) error { prm.SetArgs(p.ownerID, p.keys) prm.InvokePrmOptional = p.InvokePrmOptional - err := x.client.Invoke(prm) + _, err := x.client.Invoke(prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", addKeysMethod, err) } @@ -52,7 +52,7 @@ func (x *Client) RemoveKeys(args CommonBindPrm) error { prm.SetArgs(args.ownerID, args.keys) prm.InvokePrmOptional = args.InvokePrmOptional - err := x.client.Invoke(prm) + _, err := x.client.Invoke(prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", removeKeysMethod, err) } diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 09ae09b51..21ba145d4 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -159,7 +159,8 @@ func (c *Client) SetConfig(p SetConfigPrm) error { prm.SetArgs(p.id, p.key, p.value) prm.InvokePrmOptional = p.InvokePrmOptional - return c.client.Invoke(prm) + _, err := c.client.Invoke(prm) + return err } // RawNetworkParameter is a FrostFS network parameter which is transmitted but diff --git a/pkg/morph/client/netmap/innerring.go b/pkg/morph/client/netmap/innerring.go index 742165b9a..d6f8c56b2 100644 --- a/pkg/morph/client/netmap/innerring.go +++ b/pkg/morph/client/netmap/innerring.go @@ -34,7 +34,8 @@ func (c *Client) UpdateInnerRing(p UpdateIRPrm) error { prm.SetArgs(args) prm.InvokePrmOptional = p.InvokePrmOptional - return c.client.Invoke(prm) + _, err := c.client.Invoke(prm) + return err } // GetInnerRingList return current IR list. diff --git a/pkg/morph/client/netmap/new_epoch.go b/pkg/morph/client/netmap/new_epoch.go index 7a63f14d7..5f9f1ce5c 100644 --- a/pkg/morph/client/netmap/new_epoch.go +++ b/pkg/morph/client/netmap/new_epoch.go @@ -17,7 +17,7 @@ func (c *Client) NewEpoch(epoch uint64, force bool) error { prm.SetArgs(epoch) prm.SetControlTX(force) - if err := c.client.Invoke(prm); err != nil { + if _, err := c.client.Invoke(prm); err != nil { return fmt.Errorf("could not invoke method (%s): %w", newEpochMethod, err) } return nil diff --git a/pkg/morph/client/netmap/peer.go b/pkg/morph/client/netmap/peer.go index 4769c038c..0ce2d3b42 100644 --- a/pkg/morph/client/netmap/peer.go +++ b/pkg/morph/client/netmap/peer.go @@ -36,7 +36,7 @@ func (c *Client) AddPeer(p AddPeerPrm) error { prm.SetArgs(p.nodeInfo.Marshal()) prm.InvokePrmOptional = p.InvokePrmOptional - if err := c.client.Invoke(prm); err != nil { + if _, err := c.client.Invoke(prm); err != nil { return fmt.Errorf("could not invoke method (%s): %w", method, err) } return nil diff --git a/pkg/morph/client/netmap/update_state.go b/pkg/morph/client/netmap/update_state.go index 02967453b..a15459021 100644 --- a/pkg/morph/client/netmap/update_state.go +++ b/pkg/morph/client/netmap/update_state.go @@ -55,7 +55,7 @@ func (c *Client) UpdatePeerState(p UpdatePeerPrm) error { prm.SetArgs(int64(p.state), p.key) prm.InvokePrmOptional = p.InvokePrmOptional - if err := c.client.Invoke(prm); err != nil { + if _, err := c.client.Invoke(prm); err != nil { return fmt.Errorf("could not invoke smart contract: %w", err) } return nil diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 0bbe526a6..ca3d37c2b 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -355,13 +355,15 @@ func (c *Client) UpdateNeoFSAlphabetList(prm UpdateAlphabetListPrm) error { // blockchain. Fallback tx is a `RET`. If Notary support is not enabled // it fallbacks to a simple `Invoke()`. // +// Returns valid until block value. +// // `nonce` and `vub` are used only if notary is enabled. -func (c *Client) NotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { +func (c *Client) NotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) (uint32, error) { c.switchLock.RLock() defer c.switchLock.RUnlock() if c.inactive { - return ErrConnectionLost + return 0, ErrConnectionLost } if c.notary == nil { @@ -376,12 +378,12 @@ func (c *Client) NotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce ui // not expected to be signed by the current node. // // Considered to be used by non-IR nodes. -func (c *Client) NotaryInvokeNotAlpha(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) error { +func (c *Client) NotaryInvokeNotAlpha(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) { c.switchLock.RLock() defer c.switchLock.RUnlock() if c.inactive { - return ErrConnectionLost + return 0, ErrConnectionLost } if c.notary == nil { @@ -440,10 +442,11 @@ func (c *Client) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { func (c *Client) notaryInvokeAsCommittee(method string, nonce, vub uint32, args ...any) error { designate := c.GetDesignateHash() - return c.notaryInvoke(true, true, designate, nonce, &vub, method, args...) + _, err := c.notaryInvoke(true, true, designate, nonce, &vub, method, args...) + return err } -func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint160, nonce uint32, vub *uint32, method string, args ...any) error { +func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint160, nonce uint32, vub *uint32, method string, args ...any) (uint32, error) { start := time.Now() success := false defer func() { @@ -452,22 +455,22 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint alphabetList, err := c.notary.alphabetSource() if err != nil { - return err + return 0, err } until, err := c.getUntilValue(vub) if err != nil { - return err + return 0, err } cosigners, err := c.notaryCosigners(invokedByAlpha, alphabetList, committee) if err != nil { - return err + return 0, err } nAct, err := notary.NewActor(c.client, cosigners, c.acc) if err != nil { - return err + return 0, err } mainH, fbH, untilActual, err := nAct.Notarize(nAct.MakeTunedCall(contract, method, nil, func(r *result.Invoke, t *transaction.Transaction) error { @@ -482,7 +485,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint }, args...)) if err != nil && !alreadyOnChainError(err) { - return err + return 0, err } c.logger.Debug(logs.ClientNotaryRequestInvoked, @@ -492,7 +495,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint zap.String("fallback_hash", fbH.StringLE())) success = true - return nil + return until, nil } func (c *Client) notaryCosignersFromTx(mainTx *transaction.Transaction, alphabetList keys.PublicKeys) ([]actor.SignerAccount, error) { diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index 0531eacdf..1dea4c4b7 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -120,6 +120,10 @@ func (i *InvokePrmOptional) IsControl() bool { return i.controlTX } +type InvokeRes struct { + VUB uint32 +} + // Invoke calls Invoke method of Client with static internal script hash and fee. // Supported args types are the same as in Client. // @@ -129,7 +133,9 @@ func (i *InvokePrmOptional) IsControl() bool { // // If fee for the operation executed using specified method is customized, then StaticClient uses it. // Otherwise, default fee is used. -func (s StaticClient) Invoke(prm InvokePrm) error { +func (s StaticClient) Invoke(prm InvokePrm) (InvokeRes, error) { + var res InvokeRes + var err error if s.tryNotary { if s.alpha { var ( @@ -146,24 +152,27 @@ func (s StaticClient) Invoke(prm InvokePrm) error { nonce, vub, err = s.client.CalculateNonceAndVUB(*prm.hash) } if err != nil { - return fmt.Errorf("could not calculate nonce and VUB for notary alphabet invoke: %w", err) + return InvokeRes{}, fmt.Errorf("could not calculate nonce and VUB for notary alphabet invoke: %w", err) } vubP = &vub } - return s.client.NotaryInvoke(s.scScriptHash, s.fee, nonce, vubP, prm.method, prm.args...) + res.VUB, err = s.client.NotaryInvoke(s.scScriptHash, s.fee, nonce, vubP, prm.method, prm.args...) + return res, err } - return s.client.NotaryInvokeNotAlpha(s.scScriptHash, s.fee, prm.method, prm.args...) + res.VUB, err = s.client.NotaryInvokeNotAlpha(s.scScriptHash, s.fee, prm.method, prm.args...) + return res, err } - return s.client.Invoke( + res.VUB, err = s.client.Invoke( s.scScriptHash, s.fee, prm.method, prm.args..., ) + return res, err } // TestInvokePrm groups parameters of the TestInvoke operation. From 2393d13e4d9c325aa64d45b134b6ddfff7ffdd01 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 8 Nov 2023 12:05:03 +0300 Subject: [PATCH 0674/1943] [#787] morph: Return VUB for IR service calls Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/netmap.go | 3 +- .../processors/netmap/handlers_test.go | 2 +- .../processors/netmap/process_epoch.go | 2 +- pkg/innerring/processors/netmap/processor.go | 2 +- pkg/innerring/processors/netmap/wrappers.go | 8 +++-- pkg/morph/client/container/delete.go | 15 ++++----- pkg/morph/client/netmap/new_epoch.go | 11 ++++--- pkg/morph/client/netmap/peer.go | 13 +++++--- pkg/morph/client/netmap/update_state.go | 9 +++--- pkg/morph/client/notary.go | 4 +-- pkg/morph/client/static.go | 19 ++++++++++-- pkg/services/control/ir/server/calls.go | 31 +++++++++++++------ 12 files changed, 78 insertions(+), 41 deletions(-) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index ebe152e47..347121e56 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -418,7 +418,8 @@ func (c *cfg) updateNetMapState(stateSetter func(*nmClient.UpdatePeerPrm)) error prm.SetKey(c.key.PublicKey().Bytes()) stateSetter(&prm) - return c.cfgNetmap.wrapper.UpdatePeerState(prm) + _, err := c.cfgNetmap.wrapper.UpdatePeerState(prm) + return err } type netInfo struct { diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index fa87cff92..164ee41da 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -399,7 +399,7 @@ func (c *testNetmapClient) NetMap() (*netmap.NetMap, error) { return c.netmap, nil } -func (c *testNetmapClient) NewEpoch(epoch uint64, force bool) error { +func (c *testNetmapClient) NewEpoch(epoch uint64) error { c.newEpochs = append(c.newEpochs, epoch) return nil } diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index 01bfbae67..f3cb9837f 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -77,7 +77,7 @@ func (np *Processor) processNewEpochTick() bool { nextEpoch := np.epochState.EpochCounter() + 1 np.log.Debug(logs.NetmapNextEpoch, zap.Uint64("value", nextEpoch)) - err := np.netmapClient.NewEpoch(nextEpoch, false) + err := np.netmapClient.NewEpoch(nextEpoch) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) return false diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index 6b8a24a62..e8fb8721b 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -60,7 +60,7 @@ type ( EpochDuration() (uint64, error) MorphTxHeight(h util.Uint256) (res uint32, err error) NetMap() (*netmap.NetMap, error) - NewEpoch(epoch uint64, force bool) error + NewEpoch(epoch uint64) error MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error } diff --git a/pkg/innerring/processors/netmap/wrappers.go b/pkg/innerring/processors/netmap/wrappers.go index f9af703de..618c1fb8f 100644 --- a/pkg/innerring/processors/netmap/wrappers.go +++ b/pkg/innerring/processors/netmap/wrappers.go @@ -19,7 +19,8 @@ type netmapClientWrapper struct { } func (w *netmapClientWrapper) UpdatePeerState(p netmapclient.UpdatePeerPrm) error { - return w.netmapClient.UpdatePeerState(p) + _, err := w.netmapClient.UpdatePeerState(p) + return err } func (w *netmapClientWrapper) MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { @@ -43,8 +44,9 @@ func (w *netmapClientWrapper) NetMap() (*netmap.NetMap, error) { return w.netmapClient.NetMap() } -func (w *netmapClientWrapper) NewEpoch(epoch uint64, force bool) error { - return w.netmapClient.NewEpoch(epoch, force) +func (w *netmapClientWrapper) NewEpoch(epoch uint64) error { + _, err := w.netmapClient.NewEpoch(epoch, 0, false) + return err } func (w *netmapClientWrapper) MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) { diff --git a/pkg/morph/client/container/delete.go b/pkg/morph/client/container/delete.go index c6cf7aa8d..20351b570 100644 --- a/pkg/morph/client/container/delete.go +++ b/pkg/morph/client/container/delete.go @@ -26,7 +26,8 @@ func Delete(c *Client, witness core.RemovalWitness) error { prm.SetToken(tok.Marshal()) } - return c.Delete(prm) + _, err := c.Delete(prm) + return err } // DeletePrm groups parameters of Delete client operation. @@ -62,13 +63,13 @@ func (d *DeletePrm) SetKey(key []byte) { // Delete removes the container from FrostFS system // through Container contract call. // -// Returns any error encountered that caused +// Returns valid until block and any error encountered that caused // the removal to interrupt. // // If TryNotary is provided, calls notary contract. -func (c *Client) Delete(p DeletePrm) error { +func (c *Client) Delete(p DeletePrm) (uint32, error) { if len(p.signature) == 0 && !p.IsControl() { - return errNilArgument + return 0, errNilArgument } prm := client.InvokePrm{} @@ -76,9 +77,9 @@ func (c *Client) Delete(p DeletePrm) error { prm.SetArgs(p.cnr, p.signature, p.key, p.token) prm.InvokePrmOptional = p.InvokePrmOptional - _, err := c.client.Invoke(prm) + res, err := c.client.Invoke(prm) if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", deleteMethod, err) + return 0, fmt.Errorf("could not invoke method (%s): %w", deleteMethod, err) } - return nil + return res.VUB, nil } diff --git a/pkg/morph/client/netmap/new_epoch.go b/pkg/morph/client/netmap/new_epoch.go index 5f9f1ce5c..a596b9991 100644 --- a/pkg/morph/client/netmap/new_epoch.go +++ b/pkg/morph/client/netmap/new_epoch.go @@ -11,14 +11,17 @@ import ( // If `force` is true, this call is normally initiated by a control // service command and uses a control notary transaction internally // to ensure all nodes produce the same transaction with high probability. -func (c *Client) NewEpoch(epoch uint64, force bool) error { +// If vub > 0, vub will be used as valid until block value. +func (c *Client) NewEpoch(epoch uint64, vub uint32, force bool) (uint32, error) { prm := client.InvokePrm{} prm.SetMethod(newEpochMethod) prm.SetArgs(epoch) prm.SetControlTX(force) + prm.SetVUB(vub) - if _, err := c.client.Invoke(prm); err != nil { - return fmt.Errorf("could not invoke method (%s): %w", newEpochMethod, err) + res, err := c.client.Invoke(prm) + if err != nil { + return 0, fmt.Errorf("could not invoke method (%s): %w", newEpochMethod, err) } - return nil + return res.VUB, nil } diff --git a/pkg/morph/client/netmap/peer.go b/pkg/morph/client/netmap/peer.go index 0ce2d3b42..30f51f699 100644 --- a/pkg/morph/client/netmap/peer.go +++ b/pkg/morph/client/netmap/peer.go @@ -43,17 +43,20 @@ func (c *Client) AddPeer(p AddPeerPrm) error { } // ForceRemovePeer marks the given peer as offline via a notary control transaction. -func (c *Client) ForceRemovePeer(nodeInfo netmap.NodeInfo) error { +// If vub > 0, vub will be used as valid until block value. +func (c *Client) ForceRemovePeer(nodeInfo netmap.NodeInfo, vub uint32) (uint32, error) { if !c.client.WithNotary() { - return fmt.Errorf("peer can be forcefully removed only in notary environment") + return 0, fmt.Errorf("peer can be forcefully removed only in notary environment") } prm := UpdatePeerPrm{} prm.SetKey(nodeInfo.PublicKey()) prm.SetControlTX(true) + prm.SetVUB(vub) - if err := c.UpdatePeerState(prm); err != nil { - return fmt.Errorf("updating peer state: %v", err) + vub, err := c.UpdatePeerState(prm) + if err != nil { + return 0, fmt.Errorf("updating peer state: %v", err) } - return nil + return vub, nil } diff --git a/pkg/morph/client/netmap/update_state.go b/pkg/morph/client/netmap/update_state.go index a15459021..7c3a4e8cd 100644 --- a/pkg/morph/client/netmap/update_state.go +++ b/pkg/morph/client/netmap/update_state.go @@ -36,7 +36,7 @@ func (u *UpdatePeerPrm) SetMaintenance() { } // UpdatePeerState changes peer status through Netmap contract call. -func (c *Client) UpdatePeerState(p UpdatePeerPrm) error { +func (c *Client) UpdatePeerState(p UpdatePeerPrm) (uint32, error) { method := updateStateMethod if c.client.WithNotary() && c.client.IsAlpha() { @@ -55,8 +55,9 @@ func (c *Client) UpdatePeerState(p UpdatePeerPrm) error { prm.SetArgs(int64(p.state), p.key) prm.InvokePrmOptional = p.InvokePrmOptional - if _, err := c.client.Invoke(prm); err != nil { - return fmt.Errorf("could not invoke smart contract: %w", err) + res, err := c.client.Invoke(prm) + if err != nil { + return 0, fmt.Errorf("could not invoke smart contract: %w", err) } - return nil + return res.VUB, nil } diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index ca3d37c2b..dfdd7c03b 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -378,7 +378,7 @@ func (c *Client) NotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce ui // not expected to be signed by the current node. // // Considered to be used by non-IR nodes. -func (c *Client) NotaryInvokeNotAlpha(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) { +func (c *Client) NotaryInvokeNotAlpha(contract util.Uint160, fee fixedn.Fixed8, vubP *uint32, method string, args ...any) (uint32, error) { c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -390,7 +390,7 @@ func (c *Client) NotaryInvokeNotAlpha(contract util.Uint160, fee fixedn.Fixed8, return c.Invoke(contract, fee, method, args...) } - return c.notaryInvoke(false, false, contract, rand.Uint32(), nil, method, args...) + return c.notaryInvoke(false, false, contract, rand.Uint32(), vubP, method, args...) } // NotarySignAndInvokeTX signs and sends notary request that was received from diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index 1dea4c4b7..595e11421 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -100,6 +100,8 @@ type InvokePrmOptional struct { // It's only used by notary transactions and it affects only the // computation of `validUntilBlock` values. controlTX bool + // vub is used to set custom valid until block value. + vub uint32 } // SetHash sets optional hash of the transaction. @@ -120,6 +122,11 @@ func (i *InvokePrmOptional) IsControl() bool { return i.controlTX } +// SetVUB sets valid until block value. +func (i *InvokePrmOptional) SetVUB(v uint32) { + i.vub = v +} + type InvokeRes struct { VUB uint32 } @@ -136,11 +143,11 @@ type InvokeRes struct { func (s StaticClient) Invoke(prm InvokePrm) (InvokeRes, error) { var res InvokeRes var err error + var vubP *uint32 if s.tryNotary { if s.alpha { var ( nonce uint32 = 1 - vubP *uint32 vub uint32 err error ) @@ -158,11 +165,19 @@ func (s StaticClient) Invoke(prm InvokePrm) (InvokeRes, error) { vubP = &vub } + if prm.vub > 0 { + vubP = &prm.vub + } + res.VUB, err = s.client.NotaryInvoke(s.scScriptHash, s.fee, nonce, vubP, prm.method, prm.args...) return res, err } - res.VUB, err = s.client.NotaryInvokeNotAlpha(s.scScriptHash, s.fee, prm.method, prm.args...) + if prm.vub > 0 { + vubP = &prm.vub + } + + res.VUB, err = s.client.NotaryInvokeNotAlpha(s.scScriptHash, s.fee, vubP, prm.method, prm.args...) return res, err } diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 537905840..4224ea81c 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -53,9 +53,11 @@ func (s *Server) TickEpoch(_ context.Context, req *control.TickEpochRequest) (*c return nil, fmt.Errorf("getting current epoch: %w", err) } - if err := s.netmapClient.NewEpoch(epoch+1, true); err != nil { + vub, err := s.netmapClient.NewEpoch(epoch+1, req.GetBody().GetVub(), true) + if err != nil { return nil, fmt.Errorf("forcing new epoch: %w", err) } + resp.Body.Vub = vub if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -93,9 +95,11 @@ func (s *Server) RemoveNode(_ context.Context, req *control.RemoveNodeRequest) ( return nil, status.Error(codes.FailedPrecondition, "node is already offline") } - if err := s.netmapClient.ForceRemovePeer(nodeInfo); err != nil { + vub, err := s.netmapClient.ForceRemovePeer(nodeInfo, req.GetBody().GetVub()) + if err != nil { return nil, fmt.Errorf("forcing node removal: %w", err) } + resp.Body.Vub = vub if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -113,13 +117,15 @@ func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainer if len(req.Body.GetContainerId()) > 0 && len(req.Body.GetOwner()) > 0 { return nil, status.Error(codes.InvalidArgument, "specify the owner and container at the same time is not allowed") } - + var vub uint32 if len(req.Body.GetContainerId()) > 0 { var containerID cid.ID if err := containerID.Decode(req.Body.GetContainerId()); err != nil { return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed to parse container ID: %s", err.Error())) } - if err := s.removeContainer(containerID); err != nil { + var err error + vub, err = s.removeContainer(containerID, req.GetBody().GetVub()) + if err != nil { return nil, err } } else { @@ -138,14 +144,17 @@ func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainer } for _, containerID := range cids { - if err := s.removeContainer(containerID); err != nil { + vub, err = s.removeContainer(containerID, req.GetBody().GetVub()) + if err != nil { return nil, err } } } resp := &control.RemoveContainerResponse{ - Body: &control.RemoveContainerResponse_Body{}, + Body: &control.RemoveContainerResponse_Body{ + Vub: vub, + }, } if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -153,13 +162,15 @@ func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainer return resp, nil } -func (s *Server) removeContainer(containerID cid.ID) error { +func (s *Server) removeContainer(containerID cid.ID, vub uint32) (uint32, error) { var prm container.DeletePrm prm.SetCID(containerID[:]) prm.SetControlTX(true) + prm.SetVUB(vub) - if err := s.containerClient.Delete(prm); err != nil { - return fmt.Errorf("forcing container removal: %w", err) + vub, err := s.containerClient.Delete(prm) + if err != nil { + return 0, fmt.Errorf("forcing container removal: %w", err) } - return nil + return vub, nil } From c8a62ffeddf47ed2cf51305250ab9f0bb777b024 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 8 Nov 2023 12:28:34 +0300 Subject: [PATCH 0675/1943] [#787] morph: Calculate VUB and nonce when hash is nil Signed-off-by: Dmitrii Stepanov --- pkg/innerring/state.go | 2 +- pkg/morph/client/notary.go | 32 ++++++++++++++++++++++---------- pkg/morph/client/static.go | 4 ++-- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index f2521bd08..f5a03fa29 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -117,7 +117,7 @@ func (s *Server) voteForSidechainValidator(prm governance.VoteValidatorPrm) erro ) if prm.Hash != nil { - nonce, vub, err = s.morphClient.CalculateNonceAndVUB(*prm.Hash) + nonce, vub, err = s.morphClient.CalculateNonceAndVUB(prm.Hash) if err != nil { return fmt.Errorf("could not calculate nonce and `validUntilBlock` values: %w", err) } diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index dfdd7c03b..5b817a805 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -289,7 +289,7 @@ func (c *Client) UpdateNotaryList(prm UpdateNotaryListPrm) error { panic(notaryNotEnabledPanicMsg) } - nonce, vub, err := c.CalculateNonceAndVUB(prm.hash) + nonce, vub, err := c.CalculateNonceAndVUB(&prm.hash) if err != nil { return fmt.Errorf("could not calculate nonce and `valicUntilBlock` values: %w", err) } @@ -337,7 +337,7 @@ func (c *Client) UpdateNeoFSAlphabetList(prm UpdateAlphabetListPrm) error { panic(notaryNotEnabledPanicMsg) } - nonce, vub, err := c.CalculateNonceAndVUB(prm.hash) + nonce, vub, err := c.CalculateNonceAndVUB(&prm.hash) if err != nil { return fmt.Errorf("could not calculate nonce and `valicUntilBlock` values: %w", err) } @@ -763,17 +763,19 @@ func CalculateNotaryDepositAmount(c *Client, gasMul, gasDiv int64) (fixedn.Fixed // CalculateNonceAndVUB calculates nonce and ValidUntilBlock values // based on transaction hash. -func (c *Client) CalculateNonceAndVUB(hash util.Uint256) (nonce uint32, vub uint32, err error) { +func (c *Client) CalculateNonceAndVUB(hash *util.Uint256) (nonce uint32, vub uint32, err error) { return c.calculateNonceAndVUB(hash, false) } // CalculateNonceAndVUBControl calculates nonce and rounded ValidUntilBlock values // based on transaction hash for use in control transactions. -func (c *Client) CalculateNonceAndVUBControl(hash util.Uint256) (nonce uint32, vub uint32, err error) { +func (c *Client) CalculateNonceAndVUBControl(hash *util.Uint256) (nonce uint32, vub uint32, err error) { return c.calculateNonceAndVUB(hash, true) } -func (c *Client) calculateNonceAndVUB(hash util.Uint256, roundBlockHeight bool) (nonce uint32, vub uint32, err error) { +// If hash specified, transaction's height and hash are used to compute VUB and nonce. +// If not, then current block height used to compute VUB and nonce. +func (c *Client) calculateNonceAndVUB(hash *util.Uint256, roundBlockHeight bool) (nonce uint32, vub uint32, err error) { c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -785,11 +787,18 @@ func (c *Client) calculateNonceAndVUB(hash util.Uint256, roundBlockHeight bool) return 0, 0, nil } - nonce = binary.LittleEndian.Uint32(hash.BytesLE()) + var height uint32 - height, err := c.getTransactionHeight(hash) - if err != nil { - return 0, 0, fmt.Errorf("could not get transaction height: %w", err) + if hash != nil { + height, err = c.getTransactionHeight(*hash) + if err != nil { + return 0, 0, fmt.Errorf("could not get transaction height: %w", err) + } + } else { + height, err = c.rpcActor.GetBlockCount() + if err != nil { + return 0, 0, fmt.Errorf("could not get chain height: %w", err) + } } // For control transactions, we round down the block height to control the @@ -800,7 +809,10 @@ func (c *Client) calculateNonceAndVUB(hash util.Uint256, roundBlockHeight bool) height = height / inc * inc } - return nonce, height + c.notary.txValidTime, nil + if hash != nil { + return binary.LittleEndian.Uint32(hash.BytesLE()), height + c.notary.txValidTime, nil + } + return height + c.notary.txValidTime, height + c.notary.txValidTime, nil } func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index 595e11421..dfcf62b83 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -154,9 +154,9 @@ func (s StaticClient) Invoke(prm InvokePrm) (InvokeRes, error) { if prm.hash != nil { if prm.controlTX { - nonce, vub, err = s.client.CalculateNonceAndVUBControl(*prm.hash) + nonce, vub, err = s.client.CalculateNonceAndVUBControl(prm.hash) } else { - nonce, vub, err = s.client.CalculateNonceAndVUB(*prm.hash) + nonce, vub, err = s.client.CalculateNonceAndVUB(prm.hash) } if err != nil { return InvokeRes{}, fmt.Errorf("could not calculate nonce and VUB for notary alphabet invoke: %w", err) From 8088063195f971d10ad7bb7bf5497b214000c5ec Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 13 Nov 2023 17:22:31 +0300 Subject: [PATCH 0676/1943] [#787] netmap: Refactor `NewEpoch` method Split for user and control methods. Signed-off-by: Dmitrii Stepanov --- pkg/innerring/processors/netmap/wrappers.go | 3 +-- pkg/morph/client/netmap/new_epoch.go | 24 +++++++++++++++------ pkg/services/control/ir/server/calls.go | 2 +- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/pkg/innerring/processors/netmap/wrappers.go b/pkg/innerring/processors/netmap/wrappers.go index 618c1fb8f..e75fdaf40 100644 --- a/pkg/innerring/processors/netmap/wrappers.go +++ b/pkg/innerring/processors/netmap/wrappers.go @@ -45,8 +45,7 @@ func (w *netmapClientWrapper) NetMap() (*netmap.NetMap, error) { } func (w *netmapClientWrapper) NewEpoch(epoch uint64) error { - _, err := w.netmapClient.NewEpoch(epoch, 0, false) - return err + return w.netmapClient.NewEpoch(epoch) } func (w *netmapClientWrapper) MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) { diff --git a/pkg/morph/client/netmap/new_epoch.go b/pkg/morph/client/netmap/new_epoch.go index a596b9991..ded386c86 100644 --- a/pkg/morph/client/netmap/new_epoch.go +++ b/pkg/morph/client/netmap/new_epoch.go @@ -8,15 +8,27 @@ import ( // NewEpoch updates FrostFS epoch number through // Netmap contract call. -// If `force` is true, this call is normally initiated by a control -// service command and uses a control notary transaction internally -// to ensure all nodes produce the same transaction with high probability. -// If vub > 0, vub will be used as valid until block value. -func (c *Client) NewEpoch(epoch uint64, vub uint32, force bool) (uint32, error) { +func (c *Client) NewEpoch(epoch uint64) error { prm := client.InvokePrm{} prm.SetMethod(newEpochMethod) prm.SetArgs(epoch) - prm.SetControlTX(force) + + _, err := c.client.Invoke(prm) + if err != nil { + return fmt.Errorf("could not invoke method (%s): %w", newEpochMethod, err) + } + return nil +} + +// NewEpochControl updates FrostFS epoch number through +// control notary transaction internally to ensure all +// nodes produce the same transaction with high probability. +// If vub > 0, vub will be used as valid until block value. +func (c *Client) NewEpochControl(epoch uint64, vub uint32) (uint32, error) { + prm := client.InvokePrm{} + prm.SetMethod(newEpochMethod) + prm.SetArgs(epoch) + prm.SetControlTX(true) prm.SetVUB(vub) res, err := c.client.Invoke(prm) diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 4224ea81c..7baae0329 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -53,7 +53,7 @@ func (s *Server) TickEpoch(_ context.Context, req *control.TickEpochRequest) (*c return nil, fmt.Errorf("getting current epoch: %w", err) } - vub, err := s.netmapClient.NewEpoch(epoch+1, req.GetBody().GetVub(), true) + vub, err := s.netmapClient.NewEpochControl(epoch+1, req.GetBody().GetVub()) if err != nil { return nil, fmt.Errorf("forcing new epoch: %w", err) } From f04806ccd342be552cf172b4ebf856899fbc7a70 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 14 Nov 2023 14:21:58 +0300 Subject: [PATCH 0677/1943] [#506] container: Use user.ID in DeletionInfo response Signed-off-by: Evgenii Stratonikov --- pkg/core/container/storage.go | 3 ++- pkg/morph/client/container/deletion_info.go | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/core/container/storage.go b/pkg/core/container/storage.go index f48fc73ac..05e8fce49 100644 --- a/pkg/core/container/storage.go +++ b/pkg/core/container/storage.go @@ -6,6 +6,7 @@ import ( frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" policyengine "git.frostfs.info/TrueCloudLab/policy-engine" ) @@ -24,7 +25,7 @@ type Container struct { // DelInfo contains info about removed container. type DelInfo struct { // Container owner. - Owner []byte + Owner user.ID // Epoch indicates when the container was removed. Epoch int diff --git a/pkg/morph/client/container/deletion_info.go b/pkg/morph/client/container/deletion_info.go index 40eb267d6..61e26506d 100644 --- a/pkg/morph/client/container/deletion_info.go +++ b/pkg/morph/client/container/deletion_info.go @@ -10,6 +10,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/mr-tron/base58" ) func (x *containerSource) DeletionInfo(cnr cid.ID) (*containercore.DelInfo, error) { @@ -51,11 +53,16 @@ func (c *Client) DeletionInfo(cid []byte) (*containercore.DelInfo, error) { return nil, fmt.Errorf("unexpected container stack item count (%s): %d", deletionInfoMethod, len(arr)) } - owner, err := client.BytesFromStackItem(arr[0]) + rawOwner, err := client.BytesFromStackItem(arr[0]) if err != nil { return nil, fmt.Errorf("could not get byte array of container (%s): %w", deletionInfoMethod, err) } + var owner user.ID + if err := owner.DecodeString(base58.Encode(rawOwner)); err != nil { + return nil, fmt.Errorf("could not decode container owner id (%s): %w", deletionInfoMethod, err) + } + epoch, err := client.IntFromStackItem(arr[1]) if err != nil { return nil, fmt.Errorf("could not get byte array of container signature (%s): %w", deletionInfoMethod, err) From a952a406a2077c603b9feeef9395cc7aabe0dc11 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 14 Nov 2023 14:47:31 +0300 Subject: [PATCH 0678/1943] [#506] container: Use uint64 for epoch type It is `uint64` in netmap source interfaces and other code. Signed-off-by: Evgenii Stratonikov --- pkg/core/container/storage.go | 2 +- pkg/morph/client/container/deletion_info.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/core/container/storage.go b/pkg/core/container/storage.go index 05e8fce49..ebafd22db 100644 --- a/pkg/core/container/storage.go +++ b/pkg/core/container/storage.go @@ -28,7 +28,7 @@ type DelInfo struct { Owner user.ID // Epoch indicates when the container was removed. - Epoch int + Epoch uint64 } // Source is an interface that wraps diff --git a/pkg/morph/client/container/deletion_info.go b/pkg/morph/client/container/deletion_info.go index 61e26506d..dda6bf98c 100644 --- a/pkg/morph/client/container/deletion_info.go +++ b/pkg/morph/client/container/deletion_info.go @@ -63,13 +63,13 @@ func (c *Client) DeletionInfo(cid []byte) (*containercore.DelInfo, error) { return nil, fmt.Errorf("could not decode container owner id (%s): %w", deletionInfoMethod, err) } - epoch, err := client.IntFromStackItem(arr[1]) + epoch, err := client.BigIntFromStackItem(arr[1]) if err != nil { return nil, fmt.Errorf("could not get byte array of container signature (%s): %w", deletionInfoMethod, err) } return &containercore.DelInfo{ Owner: owner, - Epoch: int(epoch), + Epoch: epoch.Uint64(), }, nil } From f13f5d3b0f22dec9103f51c3d1330ff25131ba44 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 14 Nov 2023 14:27:34 +0300 Subject: [PATCH 0679/1943] [#506] node: Use DeletionInfo() method to get deleted container owner Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/container.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 9239df640..e9a7079f1 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -93,16 +93,12 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c subscribeToContainerRemoval(c, func(e event.Event) { ev := e.(containerEvent.DeleteSuccess) - // read owner of the removed container in order to update the listing cache. - // It's strange to read already removed container, but we can successfully hit - // the cache. - // TODO: use owner directly from the event after neofs-contract#256 will become resolved - cnr, err := cachedContainerStorage.Get(ev.ID) + cachedContainerStorage.handleRemoval(ev.ID) + info, err := cachedContainerStorage.DeletionInfo(ev.ID) if err == nil { - cachedContainerLister.update(cnr.Value.Owner(), ev.ID, false) + cachedContainerLister.update(info.Owner, ev.ID, false) } - cachedContainerStorage.handleRemoval(ev.ID) c.log.Debug(logs.FrostFSNodeContainerRemovalEventsReceipt, zap.Stringer("id", ev.ID), ) From b62008dacaa581c4f84bbdc11b0c739907b1ad15 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 14 Nov 2023 14:35:23 +0300 Subject: [PATCH 0680/1943] [#506] node: Invalidate `list` cache after container add/removal `update` already has problems mentioned in its doc-comment and the code itself is not straightforward. Invalidating cache altogether seems like a better option because we don't construct cache output ourselves (thus, no "impossible" results). Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/cache.go | 46 ++--------------------------------- cmd/frostfs-node/container.go | 4 +-- 2 files changed, 4 insertions(+), 46 deletions(-) diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index 6a5d5d182..ea5bdfda0 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -308,51 +308,9 @@ func (s ttlContainerLister) List(id *user.ID) ([]cid.ID, error) { return res, nil } -// updates cached list of owner's containers: cnr is added if flag is true, otherwise it's removed. -// Concurrent calls can lead to some races: -// - two parallel additions to missing owner's cache can lead to only one container to be cached -// - async cache value eviction can lead to idle addition -// -// All described race cases aren't critical since cache values expire anyway, we just try -// to increase cache actuality w/o huge overhead on synchronization. -func (s *ttlContainerLister) update(owner user.ID, cnr cid.ID, add bool) { +func (s *ttlContainerLister) invalidate(owner user.ID) { strOwner := owner.EncodeToString() - - val, ok := s.inner.cache.Peek(strOwner) - if !ok { - // we could cache the single cnr but in this case we will disperse - // with the Sidechain a lot - return - } - - if s.inner.ttl <= time.Since(val.t) { - return - } - - item := val.v - - item.mtx.Lock() - { - found := false - - for i := range item.list { - if found = item.list[i].Equals(cnr); found { - if !add { - item.list = append(item.list[:i], item.list[i+1:]...) - // if list became empty we don't remove the value from the cache - // since empty list is a correct value, and we don't want to insta - // re-request it from the Sidechain - } - - break - } - } - - if add && !found { - item.list = append(item.list, cnr) - } - } - item.mtx.Unlock() + s.inner.remove(strOwner) } type cachedIRFetcher struct { diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index e9a7079f1..c7360a1bf 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -74,7 +74,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c // creation success are most commonly tracked by polling GET op. cnr, err := cnrSrc.Get(ev.ID) if err == nil { - cachedContainerLister.update(cnr.Value.Owner(), ev.ID, true) + cachedContainerLister.invalidate(cnr.Value.Owner()) cachedContainerStorage.containerCache.set(ev.ID, cnr, nil) } else { // unlike removal, we expect successful receive of the container @@ -96,7 +96,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c cachedContainerStorage.handleRemoval(ev.ID) info, err := cachedContainerStorage.DeletionInfo(ev.ID) if err == nil { - cachedContainerLister.update(info.Owner, ev.ID, false) + cachedContainerLister.invalidate(info.Owner) } c.log.Debug(logs.FrostFSNodeContainerRemovalEventsReceipt, From f871f5cc6c2535646e529f9a8d1f5f3817d7459b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 8 Nov 2023 16:36:55 +0300 Subject: [PATCH 0681/1943] [#793] adm: Support new FrostFS ID contract Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/frostfsid_util.go | 41 +++++++++++++++++ .../modules/morph/frostfsid_util_test.go | 45 +++++++++++++++++++ .../modules/morph/initialize_deploy.go | 12 +++-- 3 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go new file mode 100644 index 000000000..54a64a67a --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go @@ -0,0 +1,41 @@ +package morph + +import ( + "fmt" + + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/nspcc-dev/neo-go/pkg/smartcontract" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/spf13/viper" +) + +// neo-go doesn't support []util.Uint160 type: +// https://github.com/nspcc-dev/neo-go/blob/v0.103.0/pkg/smartcontract/parameter.go#L262 +// Thus, return []smartcontract.Parameter. +func getFrostfsIDAuthorizedKeys(v *viper.Viper) ([]smartcontract.Parameter, error) { + var res []smartcontract.Parameter + + ks := v.GetStringSlice(frostfsIDAuthorizedKeysConfigKey) + for i := range ks { + h, err := address.StringToUint160(ks[i]) + if err == nil { + res = append(res, smartcontract.Parameter{Type: smartcontract.Hash160Type, Value: h}) + continue + } + + h, err = util.Uint160DecodeStringLE(ks[i]) + if err == nil { + res = append(res, smartcontract.Parameter{Type: smartcontract.Hash160Type, Value: h}) + continue + } + + pk, err := keys.NewPublicKeyFromString(ks[i]) + if err == nil { + res = append(res, smartcontract.Parameter{Type: smartcontract.Hash160Type, Value: pk.GetScriptHash()}) + continue + } + return nil, fmt.Errorf("frostfsid: #%d item in authorized_keys is invalid: '%s'", i, ks[i]) + } + return res, nil +} diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go new file mode 100644 index 000000000..9982c0815 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go @@ -0,0 +1,45 @@ +package morph + +import ( + "encoding/hex" + "testing" + + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/nspcc-dev/neo-go/pkg/smartcontract" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestFrostfsIDConfig(t *testing.T) { + pks := make([]*keys.PrivateKey, 4) + for i := range pks { + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + pks[i] = pk + } + + v := viper.New() + v.Set("frostfsid.authorized_keys", []string{ + pks[0].GetScriptHash().StringLE(), + address.Uint160ToString(pks[1].GetScriptHash()), + hex.EncodeToString(pks[2].PublicKey().UncompressedBytes()), + hex.EncodeToString(pks[3].PublicKey().Bytes()), + }) + + actual, err := getFrostfsIDAuthorizedKeys(v) + require.NoError(t, err) + require.Equal(t, len(pks), len(actual)) + for i := range pks { + require.Equal(t, smartcontract.Hash160Type, actual[i].Type) + require.Equal(t, pks[i].GetScriptHash(), actual[i].Value) + } + + t.Run("bad key", func(t *testing.T) { + v := viper.New() + v.Set("frostfsid.authorized_keys", []string{"abc"}) + + _, err := getFrostfsIDAuthorizedKeys(v) + require.Error(t, err) + }) +} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index f6100f4f9..34475fe8b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -31,6 +31,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "github.com/spf13/viper" ) const ( @@ -46,6 +47,8 @@ const ( proxyContract = "proxy" ) +const frostfsIDAuthorizedKeysConfigKey = "frostfsid.authorized_keys" + var ( contractList = []string{ balanceContract, @@ -535,9 +538,12 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an nnsCs.Hash, "container") case frostfsIDContract: - items = append(items, - c.Contracts[netmapContract].Hash, - c.Contracts[containerContract].Hash) + hs, err := getFrostfsIDAuthorizedKeys(viper.GetViper()) + if err != nil { + panic(err) + } + + items = append(items, hs) case netmapContract: md := getDefaultNetmapContractConfigMap() if method == updateMethodName { From c1ec6e33b45ee8f45636fb16020d0937aa718723 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 13 Nov 2023 12:41:40 +0300 Subject: [PATCH 0682/1943] [#793] adm: Always use committee as FrostFS ID owner Committee should be able to authorize everything, there are no other usecases for the frostfs-adm currently. Also, it somewhat eases configuration, because committee hash depends on the protocol configuration. Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/frostfsid_util.go | 3 ++- .../internal/modules/morph/frostfsid_util_test.go | 14 +++++++++----- .../internal/modules/morph/initialize_deploy.go | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go index 54a64a67a..248ad76b0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go @@ -13,8 +13,9 @@ import ( // neo-go doesn't support []util.Uint160 type: // https://github.com/nspcc-dev/neo-go/blob/v0.103.0/pkg/smartcontract/parameter.go#L262 // Thus, return []smartcontract.Parameter. -func getFrostfsIDAuthorizedKeys(v *viper.Viper) ([]smartcontract.Parameter, error) { +func getFrostfsIDAuthorizedKeys(v *viper.Viper, defaultOwner util.Uint160) ([]smartcontract.Parameter, error) { var res []smartcontract.Parameter + res = append(res, smartcontract.Parameter{Type: smartcontract.Hash160Type, Value: defaultOwner}) ks := v.GetStringSlice(frostfsIDAuthorizedKeysConfigKey) for i := range ks { diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go index 9982c0815..f4b6a66f6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go @@ -7,6 +7,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/smartcontract" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -27,19 +28,22 @@ func TestFrostfsIDConfig(t *testing.T) { hex.EncodeToString(pks[3].PublicKey().Bytes()), }) - actual, err := getFrostfsIDAuthorizedKeys(v) + comm := util.Uint160{1, 2, 3} + actual, err := getFrostfsIDAuthorizedKeys(v, comm) require.NoError(t, err) - require.Equal(t, len(pks), len(actual)) + require.Equal(t, len(pks)+1, len(actual)) + require.Equal(t, smartcontract.Hash160Type, actual[0].Type) + require.Equal(t, comm, actual[0].Value) for i := range pks { - require.Equal(t, smartcontract.Hash160Type, actual[i].Type) - require.Equal(t, pks[i].GetScriptHash(), actual[i].Value) + require.Equal(t, smartcontract.Hash160Type, actual[i+1].Type) + require.Equal(t, pks[i].GetScriptHash(), actual[i+1].Value) } t.Run("bad key", func(t *testing.T) { v := viper.New() v.Set("frostfsid.authorized_keys", []string{"abc"}) - _, err := getFrostfsIDAuthorizedKeys(v) + _, err := getFrostfsIDAuthorizedKeys(v, comm) require.Error(t, err) }) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 34475fe8b..5183114f9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -538,7 +538,7 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an nnsCs.Hash, "container") case frostfsIDContract: - hs, err := getFrostfsIDAuthorizedKeys(viper.GetViper()) + hs, err := getFrostfsIDAuthorizedKeys(viper.GetViper(), c.CommitteeAcc.PublicKey().GetScriptHash()) if err != nil { panic(err) } From 364f835b7e203fb96805499ec6a50a30efcdbcf8 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 13 Nov 2023 11:41:11 +0300 Subject: [PATCH 0683/1943] [#740] logs: Add Loki Signed-off-by: Alexander Chuprov --- cmd/frostfs-node/config.go | 8 ++++ cmd/frostfs-node/config/logger/config.go | 54 +++++++++++++++++++++++- go.mod | 2 +- go.sum | 4 +- 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index a41b73d92..a43b59ab2 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -62,6 +62,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state" + "git.frostfs.info/TrueCloudLab/frostfs-observability/logging/lokicore" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -72,6 +73,7 @@ import ( "github.com/panjf2000/ants/v2" "go.etcd.io/bbolt" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "google.golang.org/grpc" ) @@ -577,6 +579,12 @@ func initCfg(appCfg *config.Config) *cfg { logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook() log, err := logger.NewLogger(logPrm) fatalOnErr(err) + if loggerconfig.ToLokiConfig(appCfg).Enabled { + log.Logger = log.Logger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + lokiCore := lokicore.New(core, loggerconfig.ToLokiConfig(appCfg)) + return lokiCore + })) + } c.internals = initInternals(appCfg, log) diff --git a/cmd/frostfs-node/config/logger/config.go b/cmd/frostfs-node/config/logger/config.go index 78e4377a9..02645c543 100644 --- a/cmd/frostfs-node/config/logger/config.go +++ b/cmd/frostfs-node/config/logger/config.go @@ -1,12 +1,21 @@ package loggerconfig import ( + "os" + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "git.frostfs.info/TrueCloudLab/frostfs-observability/logging/lokicore/loki" ) const ( // LevelDefault is a default logger level. - LevelDefault = "info" + LevelDefault = "info" + subsection = "logger" + lokiSubsection = "loki" + AddressDefault = "localhost:3100" + BatchEntriesNumberDefault = 100 + BatchWaitDefault = time.Second ) // Level returns the value of "level" config parameter @@ -15,7 +24,7 @@ const ( // Returns LevelDefault if the value is not a non-empty string. func Level(c *config.Config) string { v := config.StringSafe( - c.Sub("logger"), + c.Sub(subsection), "level", ) if v != "" { @@ -24,3 +33,44 @@ func Level(c *config.Config) string { return LevelDefault } + +// ToLokiConfig extracts loki config. +func ToLokiConfig(c *config.Config) loki.Config { + hostname, _ := os.Hostname() + return loki.Config{ + Enabled: config.BoolSafe(c.Sub(subsection).Sub(lokiSubsection), "enabled"), + BatchWait: getBatchWait(c), + BatchEntriesNumber: getBatchEntriesNumber(c), + Endpoint: getEndpoint(c), + Labels: map[string]string{ + "hostname": hostname, + }, + } +} + +func getBatchWait(c *config.Config) time.Duration { + v := config.DurationSafe(c.Sub(subsection).Sub(lokiSubsection), "max_batch_delay") + if v > 0 { + return v + } + + return BatchWaitDefault +} + +func getBatchEntriesNumber(c *config.Config) int { + v := config.IntSafe(c.Sub(subsection).Sub(lokiSubsection), "max_batch_size") + if v > 0 { + return int(v) + } + + return BatchEntriesNumberDefault +} + +func getEndpoint(c *config.Config) string { + v := config.StringSafe(c.Sub(subsection).Sub(lokiSubsection), "endpoint") + if v != "" { + return v + } + + return AddressDefault +} diff --git a/go.mod b/go.mod index 4f6505e87..5c22fdfc2 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989 - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231101082425-5eee1a733432 diff --git a/go.sum b/go.sum index 05a870b80..806648251 100644 --- a/go.sum +++ b/go.sum @@ -730,8 +730,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845a git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989/go.mod h1:PxsAxe6J4hyYJoU6hyYxD9q5K/DFxdquKHL1BrRaGkg= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d h1:KVHsNsEoedQ1+f+eDq88AFKkLePGfnygt4bcUQrCgPw= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= From fd9128d0513d86503e88b5cab4f4710072f0d3a9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 10 Nov 2023 11:40:06 +0300 Subject: [PATCH 0684/1943] [#800] node: eACL -> APE converter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/util/ape.go | 32 +- cmd/frostfs-cli/modules/util/ape_test.go | 25 +- go.mod | 2 +- go.sum | 4 +- internal/ape/converter.go | 246 ++++++++++++ internal/ape/converter_test.go | 470 +++++++++++++++++++++++ 6 files changed, 749 insertions(+), 30 deletions(-) create mode 100644 internal/ape/converter.go create mode 100644 internal/ape/converter_test.go diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index 881546450..47ce37bb7 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -6,6 +6,7 @@ import ( "strings" policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/flynn-archive/go-shlex" ) @@ -65,7 +66,7 @@ func parseRuleLexemes(r *policyengine.Rule, lexemes []string) error { return err } - r.Action, err = parseAction(lexemes[1]) + r.Actions, err = parseAction(lexemes[1]) if err != nil { return err } @@ -75,7 +76,7 @@ func parseRuleLexemes(r *policyengine.Rule, lexemes []string) error { return err } - r.Resource, err = parseResource(lexemes[len(lexemes)-1]) + r.Resources, err = parseResource(lexemes[len(lexemes)-1]) return err } @@ -100,41 +101,42 @@ func parseStatus(lexeme string) (policyengine.Status, error) { } } -func parseAction(lexeme string) ([]string, error) { +func parseAction(lexeme string) (policyengine.Actions, error) { switch strings.ToLower(lexeme) { case "object.put": - return []string{"native:PutObject"}, nil + return policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, nil case "object.get": - return []string{"native:GetObject"}, nil + return policyengine.Actions{Names: []string{nativeschema.MethodGetObject}}, nil case "object.head": - return []string{"native:HeadObject"}, nil + return policyengine.Actions{Names: []string{nativeschema.MethodHeadObject}}, nil case "object.delete": - return []string{"native:DeleteObject"}, nil + return policyengine.Actions{Names: []string{nativeschema.MethodDeleteObject}}, nil case "object.search": - return []string{"native:SearchObject"}, nil + return policyengine.Actions{Names: []string{nativeschema.MethodSearchObject}}, nil case "object.range": - return []string{"native:RangeObject"}, nil + return policyengine.Actions{Names: []string{nativeschema.MethodRangeObject}}, nil case "object.hash": - return []string{"native:HashObject"}, nil + return policyengine.Actions{Names: []string{nativeschema.MethodHashObject}}, nil default: } - return nil, fmt.Errorf("%w: %s", errUnknownOperation, lexeme) + return policyengine.Actions{}, fmt.Errorf("%w: %s", errUnknownOperation, lexeme) } -func parseResource(lexeme string) ([]string, error) { - return []string{fmt.Sprintf("native:::object/%s", lexeme)}, nil +func parseResource(lexeme string) (policyengine.Resources, error) { + if lexeme == "*" { + return policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, nil + } + return policyengine.Resources{Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, lexeme)}}, nil } const ( ObjectResource = "object.resource" ObjectRequest = "object.request" - ObjectActor = "object.actor" ) var typeToCondObject = map[string]policyengine.ObjectType{ ObjectResource: policyengine.ObjectResource, ObjectRequest: policyengine.ObjectRequest, - ObjectActor: policyengine.ObjectActor, } func parseConditions(lexemes []string) ([]policyengine.Condition, error) { diff --git a/cmd/frostfs-cli/modules/util/ape_test.go b/cmd/frostfs-cli/modules/util/ape_test.go index e71544a9b..b2e9d1d87 100644 --- a/cmd/frostfs-cli/modules/util/ape_test.go +++ b/cmd/frostfs-cli/modules/util/ape_test.go @@ -4,6 +4,7 @@ import ( "testing" policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/stretchr/testify/require" ) @@ -19,8 +20,8 @@ func TestParseAPERule(t *testing.T) { rule: "allow Object.Put *", expectRule: policyengine.Rule{ Status: policyengine.Allow, - Action: []string{"native:PutObject"}, - Resource: []string{"native:::object/*"}, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, Condition: []policyengine.Condition{}, }, }, @@ -29,8 +30,8 @@ func TestParseAPERule(t *testing.T) { rule: "deny Object.Put *", expectRule: policyengine.Rule{ Status: policyengine.AccessDenied, - Action: []string{"native:PutObject"}, - Resource: []string{"native:::object/*"}, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, Condition: []policyengine.Condition{}, }, }, @@ -39,8 +40,8 @@ func TestParseAPERule(t *testing.T) { rule: "deny:QuotaLimitReached Object.Put *", expectRule: policyengine.Rule{ Status: policyengine.QuotaLimitReached, - Action: []string{"native:PutObject"}, - Resource: []string{"native:::object/*"}, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, Condition: []policyengine.Condition{}, }, }, @@ -48,9 +49,9 @@ func TestParseAPERule(t *testing.T) { name: "Valid allow rule with conditions", rule: "allow Object.Get Object.Resource:Department=HR Object.Request:Actor!=ownerA *", expectRule: policyengine.Rule{ - Status: policyengine.Allow, - Action: []string{"native:GetObject"}, - Resource: []string{"native:::object/*"}, + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodGetObject}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, Condition: []policyengine.Condition{ { Op: policyengine.CondStringEquals, @@ -71,9 +72,9 @@ func TestParseAPERule(t *testing.T) { name: "Valid rule with conditions with action detail", rule: "deny:QuotaLimitReached Object.Get Object.Resource:Department=HR Object.Request:Actor!=ownerA *", expectRule: policyengine.Rule{ - Status: policyengine.QuotaLimitReached, - Action: []string{"native:GetObject"}, - Resource: []string{"native:::object/*"}, + Status: policyengine.QuotaLimitReached, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodGetObject}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, Condition: []policyengine.Condition{ { Op: policyengine.CondStringEquals, diff --git a/go.mod b/go.mod index 5c22fdfc2..5def2fade 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231101082425-5eee1a733432 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231114100951-38985e4ec86b git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 diff --git a/go.sum b/go.sum index 806648251..5c30a75e4 100644 --- a/go.sum +++ b/go.sum @@ -736,8 +736,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231101082425-5eee1a733432 h1:z0PqdiEIHXK2qC83e6pmxUZ5peP9CIL0Bh5mP/d+4Xc= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231101082425-5eee1a733432/go.mod h1:qf3B9hSz6gCMfcfvqkhTu5ak+Gx2R+wo4Hc87LnKxPg= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231114100951-38985e4ec86b h1:io+LMCjNfP1IA7Jku7QVzAnlBjJXDNBNrHw1fpbsoeI= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231114100951-38985e4ec86b/go.mod h1:qf3B9hSz6gCMfcfvqkhTu5ak+Gx2R+wo4Hc87LnKxPg= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/internal/ape/converter.go b/internal/ape/converter.go new file mode 100644 index 000000000..e268626e6 --- /dev/null +++ b/internal/ape/converter.go @@ -0,0 +1,246 @@ +package ape + +import ( + "encoding/hex" + "fmt" + + v2acl "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" +) + +type ConvertEACLError struct { + nested error +} + +func (e *ConvertEACLError) Error() string { + if e == nil { + return "" + } + return fmt.Sprintf("failed to convert eACL table to policy engine chain: %s", e.nested.Error()) +} + +func (e *ConvertEACLError) Unwrap() error { + if e == nil { + return nil + } + return e.nested +} + +// ConvertEACLToAPE converts eacl.Table to policyengine.Chain. +func ConvertEACLToAPE(eaclTable *eacl.Table) (*policyengine.Chain, error) { + if eaclTable == nil { + return nil, nil + } + res := &policyengine.Chain{} + + resource := getResource(eaclTable) + + for _, eaclRecord := range eaclTable.Records() { + if len(eaclRecord.Targets()) == 0 { + // see https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/src/commit/ab75edd70939564421936d207ef80d6c1398b51b/eacl/validator.go#L101 + // and https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/src/commit/ab75edd70939564421936d207ef80d6c1398b51b/eacl/validator.go#L36 + // such record doesn't have any effect + continue + } + + st, err := actionToStatus(eaclRecord.Action()) + if err != nil { + return nil, err + } + act, err := operationToAction(eaclRecord.Operation()) + if err != nil { + return nil, err + } + + if len(eaclRecord.Filters()) == 0 { + res.Rules = appendTargetsOnly(res.Rules, st, act, resource, eaclRecord.Targets()) + } else { + res.Rules, err = appendTargetsAndFilters(res.Rules, st, act, resource, eaclRecord.Targets(), eaclRecord.Filters()) + if err != nil { + return nil, err + } + } + } + + return res, nil +} + +func appendTargetsOnly(source []policyengine.Rule, st policyengine.Status, act policyengine.Actions, res policyengine.Resources, targets []eacl.Target) []policyengine.Rule { + // see https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/src/commit/ab75edd70939564421936d207ef80d6c1398b51b/eacl/validator.go#L101 + // role OR public key must be equal + rule := policyengine.Rule{ + Status: st, + Actions: act, + Resources: res, + Any: true, + } + for _, target := range targets { + var roleCondition policyengine.Condition + roleCondition.Object = policyengine.ObjectRequest + roleCondition.Key = nativeschema.PropertyKeyActorRole + roleCondition.Value = target.Role().String() + roleCondition.Op = policyengine.CondStringEquals + rule.Condition = append(rule.Condition, roleCondition) + + for _, binKey := range target.BinaryKeys() { + var pubKeyCondition policyengine.Condition + pubKeyCondition.Object = policyengine.ObjectRequest + pubKeyCondition.Key = nativeschema.PropertyKeyActorPublicKey + pubKeyCondition.Value = hex.EncodeToString(binKey) + pubKeyCondition.Op = policyengine.CondStringEquals + rule.Condition = append(rule.Condition, pubKeyCondition) + } + } + return append(source, rule) +} + +func appendTargetsAndFilters(source []policyengine.Rule, st policyengine.Status, act policyengine.Actions, res policyengine.Resources, + targets []eacl.Target, filters []eacl.Filter, +) ([]policyengine.Rule, error) { + // see https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/src/commit/ab75edd70939564421936d207ef80d6c1398b51b/eacl/validator.go#L101 + // role OR public key must be equal + // so filters are repeated for each role and public key + var err error + for _, target := range targets { + rule := policyengine.Rule{ + Status: st, + Actions: act, + Resources: res, + } + var roleCondition policyengine.Condition + roleCondition.Object = policyengine.ObjectRequest + roleCondition.Key = nativeschema.PropertyKeyActorRole + roleCondition.Value = target.Role().String() + roleCondition.Op = policyengine.CondStringEquals + + rule.Condition = append(rule.Condition, roleCondition) + rule.Condition, err = appendFilters(rule.Condition, filters) + if err != nil { + return nil, err + } + + source = append(source, rule) + + for _, binKey := range target.BinaryKeys() { + rule := policyengine.Rule{ + Status: st, + Actions: act, + Resources: res, + } + var pubKeyCondition policyengine.Condition + pubKeyCondition.Object = policyengine.ObjectRequest + pubKeyCondition.Key = nativeschema.PropertyKeyActorPublicKey + pubKeyCondition.Value = hex.EncodeToString(binKey) + pubKeyCondition.Op = policyengine.CondStringEquals + + rule.Condition = append(rule.Condition, pubKeyCondition) + rule.Condition, err = appendFilters(rule.Condition, filters) + if err != nil { + return nil, err + } + + source = append(source, rule) + } + } + + return source, nil +} + +func appendFilters(source []policyengine.Condition, filters []eacl.Filter) ([]policyengine.Condition, error) { + for _, filter := range filters { + var cond policyengine.Condition + var isObject bool + if filter.From() == eacl.HeaderFromObject { + cond.Object = policyengine.ObjectResource + isObject = true + } else if filter.From() == eacl.HeaderFromRequest { + cond.Object = policyengine.ObjectRequest + } else { + return nil, &ConvertEACLError{nested: fmt.Errorf("unknown filter from: %d", filter.From())} + } + + if filter.Matcher() == eacl.MatchStringEqual { + cond.Op = policyengine.CondStringEquals + } else if filter.Matcher() == eacl.MatchStringNotEqual { + cond.Op = policyengine.CondStringNotEquals + } else { + return nil, &ConvertEACLError{nested: fmt.Errorf("unknown filter matcher: %d", filter.Matcher())} + } + + cond.Key = eaclKeyToAPEKey(filter.Key(), isObject) + cond.Value = filter.Value() + + source = append(source, cond) + } + return source, nil +} + +func eaclKeyToAPEKey(key string, isObject bool) string { + if !isObject { + return key + } + switch key { + default: + return key + case v2acl.FilterObjectVersion: + return nativeschema.PropertyKeyObjectVersion + case v2acl.FilterObjectID: + return nativeschema.PropertyKeyObjectID + case v2acl.FilterObjectContainerID: + return nativeschema.PropertyKeyObjectContainerID + case v2acl.FilterObjectOwnerID: + return nativeschema.PropertyKeyObjectOwnerID + case v2acl.FilterObjectCreationEpoch: + return nativeschema.PropertyKeyObjectCreationEpoch + case v2acl.FilterObjectPayloadLength: + return nativeschema.PropertyKeyObjectPayloadLength + case v2acl.FilterObjectPayloadHash: + return nativeschema.PropertyKeyObjectPayloadHash + case v2acl.FilterObjectType: + return nativeschema.PropertyKeyObjectType + case v2acl.FilterObjectHomomorphicHash: + return nativeschema.PropertyKeyObjectHomomorphicHash + } +} + +func getResource(eaclTable *eacl.Table) policyengine.Resources { + cnrID, isSet := eaclTable.CID() + if isSet { + return policyengine.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, + } + } + return policyengine.Resources{ + Names: []string{nativeschema.ResourceFormatRootObjects}, + } +} + +func actionToStatus(a eacl.Action) (policyengine.Status, error) { + switch a { + case eacl.ActionAllow: + return policyengine.Allow, nil + case eacl.ActionDeny: + return policyengine.AccessDenied, nil + default: + return policyengine.NoRuleFound, &ConvertEACLError{nested: fmt.Errorf("unknown action: %d", a)} + } +} + +var eaclOperationToEngineAction = map[eacl.Operation]policyengine.Actions{ + eacl.OperationGet: {Names: []string{nativeschema.MethodGetObject}}, + eacl.OperationHead: {Names: []string{nativeschema.MethodHeadObject}}, + eacl.OperationPut: {Names: []string{nativeschema.MethodPutObject}}, + eacl.OperationDelete: {Names: []string{nativeschema.MethodDeleteObject}}, + eacl.OperationSearch: {Names: []string{nativeschema.MethodSearchObject}}, + eacl.OperationRange: {Names: []string{nativeschema.MethodRangeObject}}, + eacl.OperationRangeHash: {Names: []string{nativeschema.MethodHashObject}}, +} + +func operationToAction(op eacl.Operation) (policyengine.Actions, error) { + if v, ok := eaclOperationToEngineAction[op]; ok { + return v, nil + } + return policyengine.Actions{}, &ConvertEACLError{nested: fmt.Errorf("unknown operation: %d", op)} +} diff --git a/internal/ape/converter_test.go b/internal/ape/converter_test.go new file mode 100644 index 000000000..bf6205ec9 --- /dev/null +++ b/internal/ape/converter_test.go @@ -0,0 +1,470 @@ +package ape + +import ( + "encoding/hex" + "fmt" + "testing" + + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestEACLTableWithoutRecords(t *testing.T) { + t.Parallel() + + tb := eacl.NewTable() + ch, err := ConvertEACLToAPE(tb) + require.NoError(t, err) + + vu := &eacl.ValidationUnit{} + vu.WithEACLTable(tb) + req := &testRequest{ + res: &testResource{name: nativeschema.ResourceFormatRootObjects}, + } + + compare(t, tb, vu, ch, req) + + cnrID := cidtest.ID() + tb.SetCID(cnrID) + vu.WithContainerID(&cnrID) + req.res.name = fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString()) + + ch, err = ConvertEACLToAPE(tb) + require.NoError(t, err) + + compare(t, tb, vu, ch, req) +} + +func TestNoTargets(t *testing.T) { + t.Parallel() + for _, act := range []eacl.Action{eacl.ActionAllow, eacl.ActionDeny} { + cnrID := cidtest.ID() + tb := eacl.NewTable() + tb.SetCID(cnrID) + + vu := &eacl.ValidationUnit{} + vu.WithEACLTable(tb) + vu.WithContainerID(&cnrID) + vu.WithRole(eacl.RoleOthers) + + // deny delete without role or key specified + record := eacl.NewRecord() + record.SetAction(act) + record.SetOperation(eacl.OperationDelete) + record.AddObjectContainerIDFilter(eacl.MatchStringEqual, cnrID) + + tb.AddRecord(record) + + ch, err := ConvertEACLToAPE(tb) + require.NoError(t, err) + + req := &testRequest{ + props: map[string]string{ + nativeschema.PropertyKeyActorRole: eacl.RoleOthers.String(), + }, + res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, + } + compare(t, tb, vu, ch, req) + } +} + +func TestNoFilters(t *testing.T) { + t.Parallel() + + t.Run("target match by role only", func(t *testing.T) { + t.Parallel() + + for _, act := range []eacl.Action{eacl.ActionAllow, eacl.ActionDeny} { + cnrID := cidtest.ID() + tb := eacl.NewTable() + tb.SetCID(cnrID) + + vu := &eacl.ValidationUnit{} + vu.WithEACLTable(tb) + vu.WithContainerID(&cnrID) + vu.WithRole(eacl.RoleOthers) + + // allow/deny for OTHERS + record := eacl.NewRecord() + record.SetAction(act) + record.SetOperation(eacl.OperationDelete) + + target := eacl.NewTarget() + target.SetRole(eacl.RoleOthers) + record.SetTargets(*target) + + tb.AddRecord(record) + + ch, err := ConvertEACLToAPE(tb) + require.NoError(t, err) + + req := &testRequest{ + props: map[string]string{ + nativeschema.PropertyKeyActorRole: eacl.RoleOthers.String(), + }, + res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, + } + compare(t, tb, vu, ch, req) + } + }) + + t.Run("target match by role and public key", func(t *testing.T) { + t.Parallel() + + for _, act := range []eacl.Action{eacl.ActionAllow, eacl.ActionDeny} { + cnrID := cidtest.ID() + tb := eacl.NewTable() + tb.SetCID(cnrID) + + vu := &eacl.ValidationUnit{} + vu.WithEACLTable(tb) + vu.WithContainerID(&cnrID) + vu.WithRole(eacl.RoleOthers) + + // allow/deny for OTHERS + record := eacl.NewRecord() + record.SetAction(act) + record.SetOperation(eacl.OperationDelete) + + p1, err := keys.NewPrivateKey() + require.NoError(t, err) + p2, err := keys.NewPrivateKey() + require.NoError(t, err) + + vu.WithSenderKey(p2.PublicKey().Bytes()) + + target := eacl.NewTarget() + target.SetRole(eacl.RoleOthers) + target.SetBinaryKeys([][]byte{p1.PublicKey().Bytes(), p2.PublicKey().Bytes()}) + record.SetTargets(*target) + + tb.AddRecord(record) + + ch, err := ConvertEACLToAPE(tb) + require.NoError(t, err) + + req := &testRequest{ + props: map[string]string{ + nativeschema.PropertyKeyActorRole: eacl.RoleOthers.String(), + nativeschema.PropertyKeyActorPublicKey: string(p2.PublicKey().Bytes()), + }, + res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, + } + compare(t, tb, vu, ch, req) + } + }) + + t.Run("target match by public key only", func(t *testing.T) { + t.Parallel() + + for _, act := range []eacl.Action{eacl.ActionAllow, eacl.ActionDeny} { + cnrID := cidtest.ID() + tb := eacl.NewTable() + tb.SetCID(cnrID) + + vu := &eacl.ValidationUnit{} + vu.WithEACLTable(tb) + vu.WithContainerID(&cnrID) + + // allow/deny for OTHERS + record := eacl.NewRecord() + record.SetAction(act) + record.SetOperation(eacl.OperationDelete) + + p1, err := keys.NewPrivateKey() + require.NoError(t, err) + p2, err := keys.NewPrivateKey() + require.NoError(t, err) + + vu.WithSenderKey(p2.PublicKey().Bytes()) + + target := eacl.NewTarget() + target.SetRole(eacl.RoleOthers) + target.SetBinaryKeys([][]byte{p1.PublicKey().Bytes(), p2.PublicKey().Bytes()}) + record.SetTargets(*target) + + tb.AddRecord(record) + + ch, err := ConvertEACLToAPE(tb) + require.NoError(t, err) + + req := &testRequest{ + props: map[string]string{ + nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(p2.PublicKey().Bytes()), + }, + res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, + } + compare(t, tb, vu, ch, req) + } + }) + + t.Run("target doesn't match", func(t *testing.T) { + t.Parallel() + + for _, act := range []eacl.Action{eacl.ActionAllow, eacl.ActionDeny} { + cnrID := cidtest.ID() + tb := eacl.NewTable() + tb.SetCID(cnrID) + + vu := &eacl.ValidationUnit{} + vu.WithEACLTable(tb) + vu.WithContainerID(&cnrID) + vu.WithRole(eacl.RoleSystem) + + // allow/deny for OTHERS + record := eacl.NewRecord() + record.SetAction(act) + record.SetOperation(eacl.OperationDelete) + + target := eacl.NewTarget() + target.SetRole(eacl.RoleOthers) + record.SetTargets(*target) + + tb.AddRecord(record) + + ch, err := ConvertEACLToAPE(tb) + require.NoError(t, err) + + req := &testRequest{ + props: map[string]string{ + nativeschema.PropertyKeyActorRole: eacl.RoleSystem.String(), + }, + res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, + } + compare(t, tb, vu, ch, req) + } + }) +} + +func TestWithFilters(t *testing.T) { + t.Parallel() + + t.Run("object attributes", func(t *testing.T) { + t.Parallel() + + const attrKey = "attribute_1" + const attrValue = "attribute_1_value" + + for _, act := range []eacl.Action{eacl.ActionAllow, eacl.ActionDeny} { + cnrID := cidtest.ID() + tb := eacl.NewTable() + tb.SetCID(cnrID) + + vu := &eacl.ValidationUnit{} + vu.WithEACLTable(tb) + vu.WithContainerID(&cnrID) + vu.WithRole(eacl.RoleOthers) + vu.WithHeaderSource(&testHeaderSource{ + headers: map[eacl.FilterHeaderType][]eacl.Header{ + eacl.HeaderFromObject: {&testHeader{key: attrKey, value: attrValue}}, + }, + }) + + // allow/deny for OTHERS + record := eacl.NewRecord() + record.SetAction(act) + record.SetOperation(eacl.OperationDelete) + + target := eacl.NewTarget() + target.SetRole(eacl.RoleOthers) + record.SetTargets(*target) + + record.AddObjectAttributeFilter(eacl.MatchStringEqual, attrKey, attrValue) + + tb.AddRecord(record) + + ch, err := ConvertEACLToAPE(tb) + require.NoError(t, err) + + req := &testRequest{ + props: map[string]string{ + nativeschema.PropertyKeyActorRole: eacl.RoleOthers.String(), + }, + res: &testResource{ + name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString()), + props: map[string]string{ + attrKey: attrValue, + }, + }, + } + compare(t, tb, vu, ch, req) + } + }) + + t.Run("request attributes", func(t *testing.T) { + t.Parallel() + + const attrKey = "attribute_1" + const attrValue = "attribute_1_value" + + for _, act := range []eacl.Action{eacl.ActionAllow, eacl.ActionDeny} { + cnrID := cidtest.ID() + tb := eacl.NewTable() + tb.SetCID(cnrID) + + vu := &eacl.ValidationUnit{} + vu.WithEACLTable(tb) + vu.WithContainerID(&cnrID) + vu.WithRole(eacl.RoleOthers) + vu.WithHeaderSource(&testHeaderSource{ + headers: map[eacl.FilterHeaderType][]eacl.Header{ + eacl.HeaderFromRequest: {&testHeader{key: attrKey, value: attrValue}}, + }, + }) + + // allow/deny for OTHERS + record := eacl.NewRecord() + record.SetAction(act) + record.SetOperation(eacl.OperationDelete) + + target := eacl.NewTarget() + target.SetRole(eacl.RoleOthers) + record.SetTargets(*target) + + record.AddFilter(eacl.HeaderFromRequest, eacl.MatchStringEqual, attrKey, attrValue) + + tb.AddRecord(record) + + ch, err := ConvertEACLToAPE(tb) + require.NoError(t, err) + + req := &testRequest{ + props: map[string]string{ + nativeschema.PropertyKeyActorRole: eacl.RoleOthers.String(), + attrKey: attrValue, + }, + res: &testResource{ + name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString()), + }, + } + compare(t, tb, vu, ch, req) + } + }) +} + +func TestNoHeader(t *testing.T) { + t.Skip("Should pass after https://git.frostfs.info/TrueCloudLab/policy-engine/issues/8#issuecomment-26126") + + t.Parallel() + + const attrKey = "attribute_1" + cnrID := cidtest.ID() + tb := eacl.NewTable() + tb.SetCID(cnrID) + + vu := &eacl.ValidationUnit{} + vu.WithEACLTable(tb) + vu.WithContainerID(&cnrID) + vu.WithRole(eacl.RoleOthers) + vu.WithHeaderSource(&testHeaderSource{ + headers: map[eacl.FilterHeaderType][]eacl.Header{ + eacl.HeaderFromRequest: {}, + }, + }) + + // allow/deny for OTHERS + record := eacl.NewRecord() + record.SetAction(eacl.ActionDeny) + record.SetOperation(eacl.OperationDelete) + + target := eacl.NewTarget() + target.SetRole(eacl.RoleOthers) + record.SetTargets(*target) + + record.AddFilter(eacl.HeaderFromRequest, eacl.MatchStringEqual, attrKey, "") + + tb.AddRecord(record) + + ch, err := ConvertEACLToAPE(tb) + require.NoError(t, err) + + req := &testRequest{ + props: map[string]string{ + nativeschema.PropertyKeyActorRole: eacl.RoleOthers.String(), + }, + res: &testResource{ + name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString()), + }, + } + compare(t, tb, vu, ch, req) +} + +func compare(t *testing.T, eaclTable *eacl.Table, vu *eacl.ValidationUnit, ch *policyengine.Chain, req *testRequest) { + validator := eacl.NewValidator() + for eaclOp, apeOp := range eaclOperationToEngineAction { + vu.WithOperation(eaclOp) + req.op = apeOp.Names[0] + + eaclAct, recordFound := validator.CalculateAction(vu) + apeSt, ruleFound := ch.Match(req) + + require.Equal(t, recordFound, ruleFound) + require.NotEqual(t, eacl.ActionUnknown, eaclAct) + if eaclAct == eacl.ActionAllow { + if recordFound { + require.Equal(t, policyengine.Allow, apeSt) + } else { + require.Equal(t, policyengine.NoRuleFound, apeSt) + } + } else { + require.Equal(t, policyengine.AccessDenied, apeSt) + } + } +} + +type testRequest struct { + op string + props map[string]string + res *testResource +} + +func (r *testRequest) Operation() string { + return r.op +} + +func (r *testRequest) Property(key string) string { + if v, ok := r.props[key]; ok { + return v + } + return "" +} + +func (r *testRequest) Resource() policyengine.Resource { + return r.res +} + +type testResource struct { + name string + props map[string]string +} + +func (r *testResource) Name() string { + return r.name +} + +func (r *testResource) Property(key string) string { + if v, ok := r.props[key]; ok { + return v + } + return "" +} + +type testHeaderSource struct { + headers map[eacl.FilterHeaderType][]eacl.Header +} + +func (s *testHeaderSource) HeadersOfType(t eacl.FilterHeaderType) ([]eacl.Header, bool) { + v, ok := s.headers[t] + return v, ok +} + +type testHeader struct { + key, value string +} + +func (h *testHeader) Key() string { return h.key } +func (h *testHeader) Value() string { return h.value } From 4d5be5ccb51179454728cc98bc76a53b22b49d69 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Thu, 16 Nov 2023 10:58:55 +0300 Subject: [PATCH 0685/1943] [#811] ape: Update policy-engine module version and rebase Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/control/add_rule.go | 4 +- cmd/frostfs-cli/modules/control/get_rule.go | 4 +- cmd/frostfs-cli/modules/control/list_rules.go | 4 +- cmd/frostfs-cli/modules/util/ape.go | 58 +++++++------- cmd/frostfs-cli/modules/util/ape_test.go | 2 +- cmd/frostfs-node/policy_engine.go | 11 +-- go.mod | 2 +- go.sum | 4 +- internal/ape/converter.go | 76 +++++++++---------- internal/ape/converter_test.go | 13 ++-- pkg/core/container/storage.go | 4 +- pkg/services/control/server/policy_engine.go | 42 +++++++--- pkg/services/object/acl/ape.go | 11 ++- pkg/services/object/acl/ape_request.go | 47 +++++------- 14 files changed, 151 insertions(+), 131 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index 0c6732108..d7177dd96 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -12,7 +12,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - ape "git.frostfs.info/TrueCloudLab/policy-engine" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" ) @@ -50,7 +50,7 @@ func addRule(cmd *cobra.Command, _ []string) { rule, _ := cmd.Flags().GetString(ruleFlag) - chain := new(ape.Chain) + chain := new(apechain.Chain) commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, []string{rule})) serializedChain := chain.Bytes() diff --git a/cmd/frostfs-cli/modules/control/get_rule.go b/cmd/frostfs-cli/modules/control/get_rule.go index df2bf94fe..0c34a696e 100644 --- a/cmd/frostfs-cli/modules/control/get_rule.go +++ b/cmd/frostfs-cli/modules/control/get_rule.go @@ -9,7 +9,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" ) @@ -53,7 +53,7 @@ func getRule(cmd *cobra.Command, _ []string) { verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - var chain policyengine.Chain + var chain apechain.Chain commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(resp.GetBody().GetChain())) // TODO (aarifullin): make pretty-formatted output for chains. diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index b3d7a5b9c..6a0879d0e 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -9,7 +9,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" ) @@ -58,7 +58,7 @@ func listRules(cmd *cobra.Command, _ []string) { for _, c := range chains { // TODO (aarifullin): make pretty-formatted output for chains. - var chain policyengine.Chain + var chain apechain.Chain commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(c)) cmd.Println("Parsed chain:\n" + prettyJSONFormat(cmd, chain.Bytes())) } diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index 47ce37bb7..c5f8526e2 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/flynn-archive/go-shlex" ) @@ -21,13 +21,13 @@ var ( ) // ParseAPEChain parses APE chain rules. -func ParseAPEChain(chain *policyengine.Chain, rules []string) error { +func ParseAPEChain(chain *apechain.Chain, rules []string) error { if len(rules) == 0 { return errors.New("no APE rules provided") } for _, rule := range rules { - r := new(policyengine.Rule) + r := new(apechain.Rule) if err := ParseAPERule(r, rule); err != nil { return err } @@ -47,7 +47,7 @@ func ParseAPEChain(chain *policyengine.Chain, rules []string) error { // allow Object.Get Object.Resource:Department=HR Object.Request:Actor=ownerA * // //nolint:godot -func ParseAPERule(r *policyengine.Rule, rule string) error { +func ParseAPERule(r *apechain.Rule, rule string) error { lexemes, err := shlex.Split(rule) if err != nil { return fmt.Errorf("can't parse rule '%s': %v", rule, err) @@ -55,7 +55,7 @@ func ParseAPERule(r *policyengine.Rule, rule string) error { return parseRuleLexemes(r, lexemes) } -func parseRuleLexemes(r *policyengine.Rule, lexemes []string) error { +func parseRuleLexemes(r *apechain.Rule, lexemes []string) error { if len(lexemes) < 2 { return errInvalidStatementFormat } @@ -80,14 +80,14 @@ func parseRuleLexemes(r *policyengine.Rule, lexemes []string) error { return err } -func parseStatus(lexeme string) (policyengine.Status, error) { +func parseStatus(lexeme string) (apechain.Status, error) { action, expression, found := strings.Cut(lexeme, ":") switch action = strings.ToLower(action); action { case "deny": if !found { - return policyengine.AccessDenied, nil + return apechain.AccessDenied, nil } else if strings.EqualFold(expression, "QuotaLimitReached") { - return policyengine.QuotaLimitReached, nil + return apechain.QuotaLimitReached, nil } else { return 0, fmt.Errorf("%w: %s", errUnknownActionDetail, expression) } @@ -95,38 +95,38 @@ func parseStatus(lexeme string) (policyengine.Status, error) { if found { return 0, errUnknownActionDetail } - return policyengine.Allow, nil + return apechain.Allow, nil default: return 0, errUnknownAction } } -func parseAction(lexeme string) (policyengine.Actions, error) { +func parseAction(lexeme string) (apechain.Actions, error) { switch strings.ToLower(lexeme) { case "object.put": - return policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, nil + return apechain.Actions{Names: []string{nativeschema.MethodPutObject}}, nil case "object.get": - return policyengine.Actions{Names: []string{nativeschema.MethodGetObject}}, nil + return apechain.Actions{Names: []string{nativeschema.MethodGetObject}}, nil case "object.head": - return policyengine.Actions{Names: []string{nativeschema.MethodHeadObject}}, nil + return apechain.Actions{Names: []string{nativeschema.MethodHeadObject}}, nil case "object.delete": - return policyengine.Actions{Names: []string{nativeschema.MethodDeleteObject}}, nil + return apechain.Actions{Names: []string{nativeschema.MethodDeleteObject}}, nil case "object.search": - return policyengine.Actions{Names: []string{nativeschema.MethodSearchObject}}, nil + return apechain.Actions{Names: []string{nativeschema.MethodSearchObject}}, nil case "object.range": - return policyengine.Actions{Names: []string{nativeschema.MethodRangeObject}}, nil + return apechain.Actions{Names: []string{nativeschema.MethodRangeObject}}, nil case "object.hash": - return policyengine.Actions{Names: []string{nativeschema.MethodHashObject}}, nil + return apechain.Actions{Names: []string{nativeschema.MethodHashObject}}, nil default: } - return policyengine.Actions{}, fmt.Errorf("%w: %s", errUnknownOperation, lexeme) + return apechain.Actions{}, fmt.Errorf("%w: %s", errUnknownOperation, lexeme) } -func parseResource(lexeme string) (policyengine.Resources, error) { +func parseResource(lexeme string) (apechain.Resources, error) { if lexeme == "*" { - return policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, nil + return apechain.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, nil } - return policyengine.Resources{Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, lexeme)}}, nil + return apechain.Resources{Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, lexeme)}}, nil } const ( @@ -134,13 +134,13 @@ const ( ObjectRequest = "object.request" ) -var typeToCondObject = map[string]policyengine.ObjectType{ - ObjectResource: policyengine.ObjectResource, - ObjectRequest: policyengine.ObjectRequest, +var typeToCondObject = map[string]apechain.ObjectType{ + ObjectResource: apechain.ObjectResource, + ObjectRequest: apechain.ObjectRequest, } -func parseConditions(lexemes []string) ([]policyengine.Condition, error) { - conds := make([]policyengine.Condition, 0) +func parseConditions(lexemes []string) ([]apechain.Condition, error) { + conds := make([]apechain.Condition, 0) for _, lexeme := range lexemes { typ, expression, found := strings.Cut(lexeme, ":") @@ -155,7 +155,7 @@ func parseConditions(lexemes []string) ([]policyengine.Condition, error) { var lhs, rhs string var binExpFound bool - var cond policyengine.Condition + var cond apechain.Condition cond.Object = objType lhs, rhs, binExpFound = strings.Cut(expression, "!=") @@ -164,9 +164,9 @@ func parseConditions(lexemes []string) ([]policyengine.Condition, error) { if !binExpFound { return nil, fmt.Errorf("%w: %s", errUnknownBinaryOperator, expression) } - cond.Op = policyengine.CondStringEquals + cond.Op = apechain.CondStringEquals } else { - cond.Op = policyengine.CondStringNotEquals + cond.Op = apechain.CondStringNotEquals } cond.Key, cond.Value = lhs, rhs diff --git a/cmd/frostfs-cli/modules/util/ape_test.go b/cmd/frostfs-cli/modules/util/ape_test.go index b2e9d1d87..1cab8e6ae 100644 --- a/cmd/frostfs-cli/modules/util/ape_test.go +++ b/cmd/frostfs-cli/modules/util/ape_test.go @@ -3,7 +3,7 @@ package util import ( "testing" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/stretchr/testify/require" ) diff --git a/cmd/frostfs-node/policy_engine.go b/cmd/frostfs-node/policy_engine.go index f0bd78629..248cddb11 100644 --- a/cmd/frostfs-node/policy_engine.go +++ b/cmd/frostfs-node/policy_engine.go @@ -5,23 +5,24 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" ) type apeChainSourceImpl struct { mtx sync.Mutex - localChainStorage map[cid.ID]policyengine.CachedChainStorage + localChainStorage map[cid.ID]engine.LocalOverrideEngine } func NewAPESource() container.AccessPolicyEngineChainSource { return &apeChainSourceImpl{ - localChainStorage: make(map[cid.ID]policyengine.CachedChainStorage), + localChainStorage: make(map[cid.ID]engine.LocalOverrideEngine), } } var _ container.AccessPolicyEngineChainSource = (*apeChainSourceImpl)(nil) -func (c *apeChainSourceImpl) GetChainSource(cid cid.ID) (policyengine.CachedChainStorage, error) { +func (c *apeChainSourceImpl) GetChainSource(cid cid.ID) (engine.LocalOverrideEngine, error) { c.mtx.Lock() defer c.mtx.Unlock() @@ -29,6 +30,6 @@ func (c *apeChainSourceImpl) GetChainSource(cid cid.ID) (policyengine.CachedChai if ok { return s, nil } - c.localChainStorage[cid] = policyengine.NewInMemory() + c.localChainStorage[cid] = inmemory.NewInMemoryLocalOverrides() return c.localChainStorage[cid], nil } diff --git a/go.mod b/go.mod index 5def2fade..120c05ff4 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231114100951-38985e4ec86b + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231115094736-5db67021e10f git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 diff --git a/go.sum b/go.sum index 5c30a75e4..642280d95 100644 --- a/go.sum +++ b/go.sum @@ -736,8 +736,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231114100951-38985e4ec86b h1:io+LMCjNfP1IA7Jku7QVzAnlBjJXDNBNrHw1fpbsoeI= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231114100951-38985e4ec86b/go.mod h1:qf3B9hSz6gCMfcfvqkhTu5ak+Gx2R+wo4Hc87LnKxPg= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231115094736-5db67021e10f h1:Rq95TuEkqc3T1EN5ZU1Vgf6H33TR95hz97ca8jrUciQ= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231115094736-5db67021e10f/go.mod h1:qf3B9hSz6gCMfcfvqkhTu5ak+Gx2R+wo4Hc87LnKxPg= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/internal/ape/converter.go b/internal/ape/converter.go index e268626e6..dec6f80d8 100644 --- a/internal/ape/converter.go +++ b/internal/ape/converter.go @@ -6,7 +6,7 @@ import ( v2acl "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" ) @@ -28,12 +28,12 @@ func (e *ConvertEACLError) Unwrap() error { return e.nested } -// ConvertEACLToAPE converts eacl.Table to policyengine.Chain. -func ConvertEACLToAPE(eaclTable *eacl.Table) (*policyengine.Chain, error) { +// ConvertEACLToAPE converts eacl.Table to apechain.Chain. +func ConvertEACLToAPE(eaclTable *eacl.Table) (*apechain.Chain, error) { if eaclTable == nil { return nil, nil } - res := &policyengine.Chain{} + res := &apechain.Chain{} resource := getResource(eaclTable) @@ -67,53 +67,53 @@ func ConvertEACLToAPE(eaclTable *eacl.Table) (*policyengine.Chain, error) { return res, nil } -func appendTargetsOnly(source []policyengine.Rule, st policyengine.Status, act policyengine.Actions, res policyengine.Resources, targets []eacl.Target) []policyengine.Rule { +func appendTargetsOnly(source []apechain.Rule, st apechain.Status, act apechain.Actions, res apechain.Resources, targets []eacl.Target) []apechain.Rule { // see https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/src/commit/ab75edd70939564421936d207ef80d6c1398b51b/eacl/validator.go#L101 // role OR public key must be equal - rule := policyengine.Rule{ + rule := apechain.Rule{ Status: st, Actions: act, Resources: res, Any: true, } for _, target := range targets { - var roleCondition policyengine.Condition - roleCondition.Object = policyengine.ObjectRequest + var roleCondition apechain.Condition + roleCondition.Object = apechain.ObjectRequest roleCondition.Key = nativeschema.PropertyKeyActorRole roleCondition.Value = target.Role().String() - roleCondition.Op = policyengine.CondStringEquals + roleCondition.Op = apechain.CondStringEquals rule.Condition = append(rule.Condition, roleCondition) for _, binKey := range target.BinaryKeys() { - var pubKeyCondition policyengine.Condition - pubKeyCondition.Object = policyengine.ObjectRequest + var pubKeyCondition apechain.Condition + pubKeyCondition.Object = apechain.ObjectRequest pubKeyCondition.Key = nativeschema.PropertyKeyActorPublicKey pubKeyCondition.Value = hex.EncodeToString(binKey) - pubKeyCondition.Op = policyengine.CondStringEquals + pubKeyCondition.Op = apechain.CondStringEquals rule.Condition = append(rule.Condition, pubKeyCondition) } } return append(source, rule) } -func appendTargetsAndFilters(source []policyengine.Rule, st policyengine.Status, act policyengine.Actions, res policyengine.Resources, +func appendTargetsAndFilters(source []apechain.Rule, st apechain.Status, act apechain.Actions, res apechain.Resources, targets []eacl.Target, filters []eacl.Filter, -) ([]policyengine.Rule, error) { +) ([]apechain.Rule, error) { // see https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/src/commit/ab75edd70939564421936d207ef80d6c1398b51b/eacl/validator.go#L101 // role OR public key must be equal // so filters are repeated for each role and public key var err error for _, target := range targets { - rule := policyengine.Rule{ + rule := apechain.Rule{ Status: st, Actions: act, Resources: res, } - var roleCondition policyengine.Condition - roleCondition.Object = policyengine.ObjectRequest + var roleCondition apechain.Condition + roleCondition.Object = apechain.ObjectRequest roleCondition.Key = nativeschema.PropertyKeyActorRole roleCondition.Value = target.Role().String() - roleCondition.Op = policyengine.CondStringEquals + roleCondition.Op = apechain.CondStringEquals rule.Condition = append(rule.Condition, roleCondition) rule.Condition, err = appendFilters(rule.Condition, filters) @@ -124,16 +124,16 @@ func appendTargetsAndFilters(source []policyengine.Rule, st policyengine.Status, source = append(source, rule) for _, binKey := range target.BinaryKeys() { - rule := policyengine.Rule{ + rule := apechain.Rule{ Status: st, Actions: act, Resources: res, } - var pubKeyCondition policyengine.Condition - pubKeyCondition.Object = policyengine.ObjectRequest + var pubKeyCondition apechain.Condition + pubKeyCondition.Object = apechain.ObjectRequest pubKeyCondition.Key = nativeschema.PropertyKeyActorPublicKey pubKeyCondition.Value = hex.EncodeToString(binKey) - pubKeyCondition.Op = policyengine.CondStringEquals + pubKeyCondition.Op = apechain.CondStringEquals rule.Condition = append(rule.Condition, pubKeyCondition) rule.Condition, err = appendFilters(rule.Condition, filters) @@ -148,23 +148,23 @@ func appendTargetsAndFilters(source []policyengine.Rule, st policyengine.Status, return source, nil } -func appendFilters(source []policyengine.Condition, filters []eacl.Filter) ([]policyengine.Condition, error) { +func appendFilters(source []apechain.Condition, filters []eacl.Filter) ([]apechain.Condition, error) { for _, filter := range filters { - var cond policyengine.Condition + var cond apechain.Condition var isObject bool if filter.From() == eacl.HeaderFromObject { - cond.Object = policyengine.ObjectResource + cond.Object = apechain.ObjectResource isObject = true } else if filter.From() == eacl.HeaderFromRequest { - cond.Object = policyengine.ObjectRequest + cond.Object = apechain.ObjectRequest } else { return nil, &ConvertEACLError{nested: fmt.Errorf("unknown filter from: %d", filter.From())} } if filter.Matcher() == eacl.MatchStringEqual { - cond.Op = policyengine.CondStringEquals + cond.Op = apechain.CondStringEquals } else if filter.Matcher() == eacl.MatchStringNotEqual { - cond.Op = policyengine.CondStringNotEquals + cond.Op = apechain.CondStringNotEquals } else { return nil, &ConvertEACLError{nested: fmt.Errorf("unknown filter matcher: %d", filter.Matcher())} } @@ -205,30 +205,30 @@ func eaclKeyToAPEKey(key string, isObject bool) string { } } -func getResource(eaclTable *eacl.Table) policyengine.Resources { +func getResource(eaclTable *eacl.Table) apechain.Resources { cnrID, isSet := eaclTable.CID() if isSet { - return policyengine.Resources{ + return apechain.Resources{ Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, } } - return policyengine.Resources{ + return apechain.Resources{ Names: []string{nativeschema.ResourceFormatRootObjects}, } } -func actionToStatus(a eacl.Action) (policyengine.Status, error) { +func actionToStatus(a eacl.Action) (apechain.Status, error) { switch a { case eacl.ActionAllow: - return policyengine.Allow, nil + return apechain.Allow, nil case eacl.ActionDeny: - return policyengine.AccessDenied, nil + return apechain.AccessDenied, nil default: - return policyengine.NoRuleFound, &ConvertEACLError{nested: fmt.Errorf("unknown action: %d", a)} + return apechain.NoRuleFound, &ConvertEACLError{nested: fmt.Errorf("unknown action: %d", a)} } } -var eaclOperationToEngineAction = map[eacl.Operation]policyengine.Actions{ +var eaclOperationToEngineAction = map[eacl.Operation]apechain.Actions{ eacl.OperationGet: {Names: []string{nativeschema.MethodGetObject}}, eacl.OperationHead: {Names: []string{nativeschema.MethodHeadObject}}, eacl.OperationPut: {Names: []string{nativeschema.MethodPutObject}}, @@ -238,9 +238,9 @@ var eaclOperationToEngineAction = map[eacl.Operation]policyengine.Actions{ eacl.OperationRangeHash: {Names: []string{nativeschema.MethodHashObject}}, } -func operationToAction(op eacl.Operation) (policyengine.Actions, error) { +func operationToAction(op eacl.Operation) (apechain.Actions, error) { if v, ok := eaclOperationToEngineAction[op]; ok { return v, nil } - return policyengine.Actions{}, &ConvertEACLError{nested: fmt.Errorf("unknown operation: %d", op)} + return apechain.Actions{}, &ConvertEACLError{nested: fmt.Errorf("unknown operation: %d", op)} } diff --git a/internal/ape/converter_test.go b/internal/ape/converter_test.go index bf6205ec9..de72408b1 100644 --- a/internal/ape/converter_test.go +++ b/internal/ape/converter_test.go @@ -7,7 +7,8 @@ import ( cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" @@ -393,7 +394,7 @@ func TestNoHeader(t *testing.T) { compare(t, tb, vu, ch, req) } -func compare(t *testing.T, eaclTable *eacl.Table, vu *eacl.ValidationUnit, ch *policyengine.Chain, req *testRequest) { +func compare(t *testing.T, eaclTable *eacl.Table, vu *eacl.ValidationUnit, ch *apechain.Chain, req *testRequest) { validator := eacl.NewValidator() for eaclOp, apeOp := range eaclOperationToEngineAction { vu.WithOperation(eaclOp) @@ -406,12 +407,12 @@ func compare(t *testing.T, eaclTable *eacl.Table, vu *eacl.ValidationUnit, ch *p require.NotEqual(t, eacl.ActionUnknown, eaclAct) if eaclAct == eacl.ActionAllow { if recordFound { - require.Equal(t, policyengine.Allow, apeSt) + require.Equal(t, apechain.Allow, apeSt) } else { - require.Equal(t, policyengine.NoRuleFound, apeSt) + require.Equal(t, apechain.NoRuleFound, apeSt) } } else { - require.Equal(t, policyengine.AccessDenied, apeSt) + require.Equal(t, apechain.AccessDenied, apeSt) } } } @@ -433,7 +434,7 @@ func (r *testRequest) Property(key string) string { return "" } -func (r *testRequest) Resource() policyengine.Resource { +func (r *testRequest) Resource() resource.Resource { return r.res } diff --git a/pkg/core/container/storage.go b/pkg/core/container/storage.go index ebafd22db..cc358b436 100644 --- a/pkg/core/container/storage.go +++ b/pkg/core/container/storage.go @@ -7,7 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" ) // Container groups information about the FrostFS container stored in the FrostFS network. @@ -77,5 +77,5 @@ type EACLSource interface { // policy engine chain storage. type AccessPolicyEngineChainSource interface { // TODO (aarifullin): Better to use simpler interface instead CachedChainStorage. - GetChainSource(cid cid.ID) (policyengine.CachedChainStorage, error) + GetChainSource(cid cid.ID) (engine.LocalOverrideEngine, error) } diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 19d1b7148..8565cd9c1 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -2,11 +2,14 @@ package control import ( "context" + "errors" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + engine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -22,7 +25,7 @@ func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainL return nil, status.Error(codes.InvalidArgument, err.Error()) } - var chain policyengine.Chain + var chain apechain.Chain if err = chain.DecodeBytes(req.GetBody().GetChain()); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -34,9 +37,12 @@ func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainL s.apeChainCounter.Add(1) // TODO (aarifullin): the such chain id is not well-designed yet. - chain.ID = policyengine.ChainID(fmt.Sprintf("%s:%d", policyengine.Ingress, s.apeChainCounter.Load())) + chain.ID = apechain.ID(fmt.Sprintf("%s:%d", apechain.Ingress, s.apeChainCounter.Load())) - src.AddOverride(policyengine.Ingress, &chain) + resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) + if _, err = src.LocalStorage().AddOverride(apechain.Ingress, resource, &chain); err != nil { + return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) + } resp := &control.AddChainLocalOverrideResponse{ Body: &control.AddChainLocalOverrideResponse_Body{ @@ -66,10 +72,10 @@ func (s *Server) GetChainLocalOverride(_ context.Context, req *control.GetChainL return nil, status.Error(codes.Internal, err.Error()) } - chain, found := src.GetOverride(policyengine.Ingress, policyengine.ChainID(req.GetBody().GetChainId())) - if !found { - err = fmt.Errorf("local override has not been found") - return nil, status.Error(codes.NotFound, err.Error()) + resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) + chain, err := src.LocalStorage().GetOverride(apechain.Ingress, resource, apechain.ID(req.GetBody().GetChainId())) + if err != nil { + return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } resp := &control.GetChainLocalOverrideResponse{ @@ -100,7 +106,11 @@ func (s *Server) ListChainLocalOverrides(_ context.Context, req *control.ListCha return nil, status.Error(codes.Internal, err.Error()) } - chains := src.ListOverrides(policyengine.Ingress) + resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) + chains, err := src.LocalStorage().ListOverrides(apechain.Ingress, resource) + if err != nil { + return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) + } serializedChains := make([][]byte, 0, len(chains)) for _, chain := range chains { serializedChains = append(serializedChains, chain.Bytes()) @@ -134,10 +144,13 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove return nil, status.Error(codes.Internal, err.Error()) } - removed := src.RemoveOverride(policyengine.Ingress, policyengine.ChainID(req.GetBody().GetChainId())) + resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) + if err = src.LocalStorage().RemoveOverride(apechain.Ingress, resource, apechain.ID(req.GetBody().GetChainId())); err != nil { + return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) + } resp := &control.RemoveChainLocalOverrideResponse{ Body: &control.RemoveChainLocalOverrideResponse_Body{ - Removed: removed, + Removed: true, }, } err = SignMessage(s.key, resp) @@ -146,3 +159,10 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove } return resp, nil } + +func getCodeByLocalStorageErr(err error) codes.Code { + if errors.Is(err, engine.ErrChainNotFound) { + return codes.NotFound + } + return codes.Internal +} diff --git a/pkg/services/object/acl/ape.go b/pkg/services/object/acl/ape.go index 86af440fe..525ed5930 100644 --- a/pkg/services/object/acl/ape.go +++ b/pkg/services/object/acl/ape.go @@ -8,7 +8,7 @@ import ( v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" ) var errAPEChainNoSource = errors.New("could not get ape chain source for the container") @@ -36,9 +36,12 @@ func (c *apeCheckerImpl) CheckIfRequestPermitted(reqInfo v2.RequestInfo) error { request := new(Request) request.FromRequestInfo(reqInfo) - status, ruleFound := chainCache.IsAllowed(policyengine.Ingress, "", request) + status, ruleFound, err := chainCache.IsAllowed(apechain.Ingress, "", request) + if err != nil { + return err + } - if !ruleFound || status == policyengine.Allow { + if !ruleFound || status == apechain.Allow { return nil } @@ -47,7 +50,7 @@ func (c *apeCheckerImpl) CheckIfRequestPermitted(reqInfo v2.RequestInfo) error { const accessDeniedAPEReasonFmt = "access to operation %s is denied by access policy engine: %s" -func apeErr(req v2.RequestInfo, status policyengine.Status) error { +func apeErr(req v2.RequestInfo, status apechain.Status) error { errAccessDenied := &apistatus.ObjectAccessDenied{} errAccessDenied.WriteReason(fmt.Sprintf(accessDeniedAPEReasonFmt, req.Operation(), status.String())) return errAccessDenied diff --git a/pkg/services/object/acl/ape_request.go b/pkg/services/object/acl/ape_request.go index 3e8bb173d..accbdce5f 100644 --- a/pkg/services/object/acl/ape_request.go +++ b/pkg/services/object/acl/ape_request.go @@ -5,7 +5,8 @@ import ( v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" aclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine" + aperesource "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" ) type Request struct { @@ -14,14 +15,14 @@ type Request struct { properties map[string]string } -var _ policyengine.Request = (*Request)(nil) +var _ aperesource.Request = (*Request)(nil) type resource struct { name string properties map[string]string } -var _ policyengine.Resource = (*resource)(nil) +var _ aperesource.Resource = (*resource)(nil) func (r *resource) Name() string { return r.name @@ -31,18 +32,14 @@ func (r *resource) Property(key string) string { return r.properties[key] } -// TODO (aarifullin): these stringified verbs, properties and namespaces -// should be non-implementation-specific. func getResource(reqInfo v2.RequestInfo) *resource { + var name string cid := reqInfo.ContainerID() - oid := "*" - if reqOID := reqInfo.ObjectID(); reqOID != nil { - oid = reqOID.EncodeToString() + if oid := reqInfo.ObjectID(); oid != nil { + name = fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, cid.EncodeToString(), oid.EncodeToString()) + } else { + name = fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) } - name := fmt.Sprintf("native:::object/%s/%s", - cid, - oid) - return &resource{ name: name, properties: make(map[string]string), @@ -51,32 +48,30 @@ func getResource(reqInfo v2.RequestInfo) *resource { func getProperties(_ v2.RequestInfo) map[string]string { return map[string]string{ - "Actor": "", + nativeschema.PropertyKeyActorPublicKey: "", + nativeschema.PropertyKeyActorRole: "", } } -// TODO (aarifullin): these stringified verbs, properties and namespaces -// should be non-implementation-specific. func getOperation(reqInfo v2.RequestInfo) string { - var verb string switch op := reqInfo.Operation(); op { case aclSDK.OpObjectGet: - verb = "GetObject" + return nativeschema.MethodGetObject case aclSDK.OpObjectHead: - verb = "HeadObject" + return nativeschema.MethodHeadObject case aclSDK.OpObjectPut: - verb = "PutObject" + return nativeschema.MethodPutObject case aclSDK.OpObjectDelete: - verb = "DeleteObject" + return nativeschema.MethodDeleteObject case aclSDK.OpObjectSearch: - verb = "SearchObject" + return nativeschema.MethodSearchObject case aclSDK.OpObjectRange: - verb = "RangeObject" + return nativeschema.MethodRangeObject case aclSDK.OpObjectHash: - verb = "HashObject" + return nativeschema.MethodHashObject + default: + return "" } - - return "native:" + verb } func NewRequest() *Request { @@ -100,6 +95,6 @@ func (r *Request) Property(key string) string { return r.properties[key] } -func (r *Request) Resource() policyengine.Resource { +func (r *Request) Resource() aperesource.Resource { return r.resource } From 137e987a4e65ee253da0e16f4bcd602619b73c0d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 15 Nov 2023 13:12:23 +0300 Subject: [PATCH 0686/1943] [#655] storage: Drop `LazyHandler` LazyHandler is implemented and used incorrectly. Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/iterate.go | 4 +-- .../blobstor/common/iterate.go | 1 - .../blobstor/fstree/fstree.go | 35 ++++++++----------- .../blobstor/internal/blobstortest/iterate.go | 27 -------------- .../blobstor/memstore/memstore.go | 4 --- .../writecache/writecachebbolt/flush.go | 17 +++------ 6 files changed, 19 insertions(+), 69 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index a2afbb8aa..7129e088a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -53,9 +53,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm StorageID: []byte(p), }) } - return prm.LazyHandler(elem.Address(), func() ([]byte, error) { - return data, err - }) + return nil }) subPrm.DecodeAddresses() diff --git a/pkg/local_object_storage/blobstor/common/iterate.go b/pkg/local_object_storage/blobstor/common/iterate.go index a6f0da26b..1fbb08d32 100644 --- a/pkg/local_object_storage/blobstor/common/iterate.go +++ b/pkg/local_object_storage/blobstor/common/iterate.go @@ -15,7 +15,6 @@ type IterationHandler func(IterationElement) error // IteratePrm groups the parameters of Iterate operation. type IteratePrm struct { Handler IterationHandler - LazyHandler func(oid.Address, func() ([]byte, error)) error IgnoreErrors bool ErrorHandler func(oid.Address, error) error } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index bf3400bcd..06309f53c 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -188,31 +188,24 @@ func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, pr continue } - if prm.LazyHandler != nil { - err = prm.LazyHandler(addr, func() ([]byte, error) { - return data, err - }) - } else { - if err == nil { - data, err = t.Decompress(data) - } - if err != nil { - if prm.IgnoreErrors { - if prm.ErrorHandler != nil { - return prm.ErrorHandler(addr, err) - } - continue + if err == nil { + data, err = t.Decompress(data) + } + if err != nil { + if prm.IgnoreErrors { + if prm.ErrorHandler != nil { + return prm.ErrorHandler(addr, err) } - return err + continue } - - err = prm.Handler(common.IterationElement{ - Address: addr, - ObjectData: data, - StorageID: []byte{}, - }) + return err } + err = prm.Handler(common.IterationElement{ + Address: addr, + ObjectData: data, + StorageID: []byte{}, + }) if err != nil { return err } diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index 72f107664..c73a663b6 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -6,7 +6,6 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -30,8 +29,6 @@ func TestIterate(t *testing.T, cons Constructor, min, max uint64) { runTestNormalHandler(t, s, objects) - runTestLazyHandler(t, s, objects) - runTestIgnoreLogicalErrors(t, s, objects) } @@ -62,30 +59,6 @@ func runTestNormalHandler(t *testing.T, s common.Storage, objects []objectDesc) }) } -func runTestLazyHandler(t *testing.T, s common.Storage, objects []objectDesc) { - t.Run("lazy handler", func(t *testing.T) { - seen := make(map[string]func() ([]byte, error)) - - var iterPrm common.IteratePrm - iterPrm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { - seen[addr.String()] = f - return nil - } - - _, err := s.Iterate(context.Background(), iterPrm) - require.NoError(t, err) - require.Equal(t, len(objects), len(seen)) - for i := range objects { - f, ok := seen[objects[i].addr.String()] - require.True(t, ok) - - data, err := f() - require.NoError(t, err) - require.Equal(t, objects[i].raw, data) - } - }) -} - func runTestIgnoreLogicalErrors(t *testing.T, s common.Storage, objects []objectDesc) { t.Run("ignore errors doesn't work for logical errors", func(t *testing.T) { seen := make(map[string]objectDesc) diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index 9428f457f..63e1a3e99 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -154,10 +154,6 @@ func (s *memstoreImpl) Iterate(_ context.Context, req common.IteratePrm) (common if err := req.Handler(elem); err != nil { return common.IterateRes{}, err } - case req.LazyHandler != nil: - if err := req.LazyHandler(elem.Address, func() ([]byte, error) { return elem.ObjectData, nil }); err != nil { - return common.IterateRes{}, err - } default: if !req.IgnoreErrors { return common.IterateRes{}, logicerr.Wrap(fmt.Errorf("(%T) no Handler or LazyHandler set for IteratePrm", s)) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush.go b/pkg/local_object_storage/writecache/writecachebbolt/flush.go index 9e46226d1..9efee1dda 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush.go @@ -180,20 +180,11 @@ func (c *cache) reportFlushError(msg string, addr string, err error) { func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { var prm common.IteratePrm prm.IgnoreErrors = ignoreErrors - prm.LazyHandler = func(addr oid.Address, f func() ([]byte, error)) error { - sAddr := addr.EncodeToString() - - data, err := f() - if err != nil { - c.reportFlushError(logs.FSTreeCantReadFile, sAddr, metaerr.Wrap(err)) - if ignoreErrors { - return nil - } - return err - } + prm.Handler = func(e common.IterationElement) error { + sAddr := e.Address.EncodeToString() var obj objectSDK.Object - err = obj.Unmarshal(data) + err := obj.Unmarshal(e.ObjectData) if err != nil { c.reportFlushError(logs.FSTreeCantUnmarshalObject, sAddr, metaerr.Wrap(err)) if ignoreErrors { @@ -202,7 +193,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - err = c.flushObject(ctx, &obj, data, writecache.StorageTypeFSTree) + err = c.flushObject(ctx, &obj, e.ObjectData, writecache.StorageTypeFSTree) if err != nil { if ignoreErrors { return nil From 29fe8c41f33bff2668ab0a020a5401a07859fe38 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 15 Nov 2023 14:18:03 +0300 Subject: [PATCH 0687/1943] [#655] storage: Drop `ErrorHandler` The only one usage was for logging. Now logging performed by storage anyway. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 + .../blobstor/blobovniczatree/iterate.go | 14 ++++++++--- .../blobstor/common/iterate.go | 1 - .../blobstor/fstree/fstree.go | 23 ++++++++++++++----- .../blobstor/fstree/option.go | 8 +++++++ pkg/local_object_storage/blobstor/iterate.go | 15 ++++++------ .../blobstor/memstore/memstore.go | 3 --- 7 files changed, 45 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index a43b59ab2..3be33a2a1 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -818,6 +818,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { fstree.WithPerm(sRead.perm), fstree.WithDepth(sRead.depth), fstree.WithNoSync(sRead.noSync), + fstree.WithLogger(c.log), } if c.metricsCollector != nil { fstreeOpts = append(fstreeOpts, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 7129e088a..d2b285c65 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -6,6 +6,7 @@ import ( "path/filepath" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -13,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/hrw" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" ) // Iterate iterates over all objects in b. @@ -38,9 +40,11 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm data, err := b.compression.Decompress(elem.ObjectData()) if err != nil { if prm.IgnoreErrors { - if prm.ErrorHandler != nil { - return prm.ErrorHandler(elem.Address(), err) - } + b.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, + zap.Stringer("address", elem.Address()), + zap.String("err", err.Error()), + zap.String("storage_id", p), + zap.String("root_path", b.rootPath)) return nil } return fmt.Errorf("could not decompress object data: %w", err) @@ -70,6 +74,10 @@ func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors boo blz, err := shBlz.Open() if err != nil { if ignoreErrors { + b.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, + zap.String("err", err.Error()), + zap.String("storage_id", p), + zap.String("root_path", b.rootPath)) return false, nil } return false, fmt.Errorf("could not open blobovnicza %s: %w", p, err) diff --git a/pkg/local_object_storage/blobstor/common/iterate.go b/pkg/local_object_storage/blobstor/common/iterate.go index 1fbb08d32..a1b8ff047 100644 --- a/pkg/local_object_storage/blobstor/common/iterate.go +++ b/pkg/local_object_storage/blobstor/common/iterate.go @@ -16,7 +16,6 @@ type IterationHandler func(IterationElement) error type IteratePrm struct { Handler IterationHandler IgnoreErrors bool - ErrorHandler func(oid.Address, error) error } // IterateRes groups the resulting values of Iterate operation. diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 06309f53c..d1fe85901 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -14,10 +14,12 @@ import ( "syscall" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -25,6 +27,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" ) type keyLock interface { @@ -41,6 +44,8 @@ func (l *noopKeyLock) Unlock(string) {} type FSTree struct { Info + log *logger.Logger + *compression.Config Depth uint64 DirNameLen int @@ -86,6 +91,7 @@ func New(opts ...Option) *FSTree { metrics: &noopMetrics{}, fileGuard: &noopKeyLock{}, fileCounter: &noopCounter{}, + log: &logger.Logger{Logger: zap.L()}, } for i := range opts { opts[i](f) @@ -145,9 +151,13 @@ func (t *FSTree) Iterate(ctx context.Context, prm common.IteratePrm) (common.Ite func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, prm common.IteratePrm) error { curName := strings.Join(curPath[1:], "") - des, err := os.ReadDir(filepath.Join(curPath...)) + dirPath := filepath.Join(curPath...) + des, err := os.ReadDir(dirPath) if err != nil { if prm.IgnoreErrors { + t.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, + zap.String("err", err.Error()), + zap.String("directory_path", dirPath)) return nil } return err @@ -182,8 +192,8 @@ func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, pr if err != nil { continue } - - data, err := os.ReadFile(filepath.Join(curPath...)) + path := filepath.Join(curPath...) + data, err := os.ReadFile(path) if err != nil && os.IsNotExist(err) { continue } @@ -193,9 +203,10 @@ func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, pr } if err != nil { if prm.IgnoreErrors { - if prm.ErrorHandler != nil { - return prm.ErrorHandler(addr, err) - } + t.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, + zap.Stringer("address", addr), + zap.String("err", err.Error()), + zap.String("path", path)) continue } return err diff --git a/pkg/local_object_storage/blobstor/fstree/option.go b/pkg/local_object_storage/blobstor/fstree/option.go index 21d46ac4d..e6c7a2608 100644 --- a/pkg/local_object_storage/blobstor/fstree/option.go +++ b/pkg/local_object_storage/blobstor/fstree/option.go @@ -3,7 +3,9 @@ package fstree import ( "io/fs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" + "go.uber.org/zap" ) type Option func(*FSTree) @@ -51,3 +53,9 @@ func WithFileCounter(c FileCounter) Option { f.fileGuard = utilSync.NewKeyLocker[string]() } } + +func WithLogger(l *logger.Logger) Option { + return func(f *FSTree) { + f.log = &logger.Logger{Logger: l.With(zap.String("component", "FSTree"))} + } +} diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index 5a41e4c4f..f213d7547 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -40,7 +40,14 @@ func (b *BlobStor) Iterate(ctx context.Context, prm common.IteratePrm) (common.I for i := range b.storage { _, err := b.storage[i].Storage.Iterate(ctx, prm) - if err != nil && !prm.IgnoreErrors { + if err != nil { + if prm.IgnoreErrors { + b.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, + zap.String("storage_path", b.storage[i].Storage.Path()), + zap.String("storage_type", b.storage[i].Storage.Type()), + zap.String("err", err.Error())) + continue + } return common.IterateRes{}, fmt.Errorf("blobstor iterator failure: %w", err) } } @@ -57,12 +64,6 @@ func IterateBinaryObjects(ctx context.Context, blz *BlobStor, f func(addr oid.Ad return f(elem.Address, elem.ObjectData, elem.StorageID) } prm.IgnoreErrors = true - prm.ErrorHandler = func(addr oid.Address, err error) error { - blz.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, - zap.Stringer("address", addr), - zap.String("err", err.Error())) - return nil - } _, err := blz.Iterate(ctx, prm) diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index 63e1a3e99..cc4f6921a 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -142,9 +142,6 @@ func (s *memstoreImpl) Iterate(_ context.Context, req common.IteratePrm) (common var err error if elem.ObjectData, err = s.compression.Decompress(elem.ObjectData); err != nil { if req.IgnoreErrors { - if req.ErrorHandler != nil { - return common.IterateRes{}, req.ErrorHandler(elem.Address, err) - } continue } return common.IterateRes{}, logicerr.Wrap(fmt.Errorf("(%T) decompressing data for address %q: %v", s, elem.Address.String(), err)) From 5cfb758e4eecb28ba22f301293032a3e918997b7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 17 Nov 2023 09:05:46 +0300 Subject: [PATCH 0688/1943] [#806] morph: Remove container list cache Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/cache.go | 71 ----------------------------------- cmd/frostfs-node/container.go | 10 +---- 2 files changed, 1 insertion(+), 80 deletions(-) diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index ea5bdfda0..cd383a5c2 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -6,13 +6,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" lru "github.com/hashicorp/golang-lru/v2" ) @@ -244,75 +242,6 @@ func (s *lruNetmapSource) Epoch() (uint64, error) { return s.netState.CurrentEpoch(), nil } -// wrapper over TTL cache of values read from the network -// that implements container lister. -type ttlContainerLister struct { - inner *ttlNetCache[string, *cacheItemContainerList] - client *cntClient.Client -} - -// value type for ttlNetCache used by ttlContainerLister. -type cacheItemContainerList struct { - // protects list from concurrent add/remove ops - mtx sync.RWMutex - // actual list of containers owner by the particular user - list []cid.ID -} - -func newCachedContainerLister(c *cntClient.Client, ttl time.Duration) ttlContainerLister { - const containerListerCacheSize = 100 - - lruCnrListerCache := newNetworkTTLCache(containerListerCacheSize, ttl, func(strID string) (*cacheItemContainerList, error) { - var id *user.ID - - if strID != "" { - id = new(user.ID) - - err := id.DecodeString(strID) - if err != nil { - return nil, err - } - } - - list, err := c.ContainersOf(id) - if err != nil { - return nil, err - } - - return &cacheItemContainerList{ - list: list, - }, nil - }) - - return ttlContainerLister{inner: lruCnrListerCache, client: c} -} - -// List returns list of container IDs from the cache. If list is missing in the -// cache or expired, then it returns container IDs from side chain and updates -// the cache. -func (s ttlContainerLister) List(id *user.ID) ([]cid.ID, error) { - if id == nil { - return s.client.ContainersOf(nil) - } - - item, err := s.inner.get(id.EncodeToString()) - if err != nil { - return nil, err - } - - item.mtx.RLock() - res := make([]cid.ID, len(item.list)) - copy(res, item.list) - item.mtx.RUnlock() - - return res, nil -} - -func (s *ttlContainerLister) invalidate(owner user.ID) { - strOwner := owner.EncodeToString() - s.inner.remove(strOwner) -} - type cachedIRFetcher struct { *ttlNetCache[struct{}, [][]byte] } diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index c7360a1bf..5477947d7 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -63,7 +63,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c // use RPC node as source of Container contract items (with caching) cachedContainerStorage := newCachedContainerStorage(cnrSrc, c.cfgMorph.cacheTTL) cachedEACLStorage := newCachedEACLStorage(eACLFetcher, c.cfgMorph.cacheTTL) - cachedContainerLister := newCachedContainerLister(client, c.cfgMorph.cacheTTL) subscribeToContainerCreation(c, func(e event.Event) { ev := e.(containerEvent.PutSuccess) @@ -74,7 +73,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c // creation success are most commonly tracked by polling GET op. cnr, err := cnrSrc.Get(ev.ID) if err == nil { - cachedContainerLister.invalidate(cnr.Value.Owner()) cachedContainerStorage.containerCache.set(ev.ID, cnr, nil) } else { // unlike removal, we expect successful receive of the container @@ -92,13 +90,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c subscribeToContainerRemoval(c, func(e event.Event) { ev := e.(containerEvent.DeleteSuccess) - cachedContainerStorage.handleRemoval(ev.ID) - info, err := cachedContainerStorage.DeletionInfo(ev.ID) - if err == nil { - cachedContainerLister.invalidate(info.Owner) - } - c.log.Debug(logs.FrostFSNodeContainerRemovalEventsReceipt, zap.Stringer("id", ev.ID), ) @@ -107,7 +99,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c c.cfgObject.eaclSource = cachedEACLStorage c.cfgObject.cnrSource = cachedContainerStorage - cnrRdr.lister = cachedContainerLister + cnrRdr.lister = client cnrRdr.eacl = c.cfgObject.eaclSource cnrRdr.src = c.cfgObject.cnrSource From 47286ebf3242b55961b6ca424802336705d039be Mon Sep 17 00:00:00 2001 From: aarifullin Date: Wed, 15 Nov 2023 12:03:31 +0300 Subject: [PATCH 0689/1943] [#805] pilorama: Fix TreeDrop * If treeID is empty then deleting buckets for cursor may get invalidated. So, buckets should be gathered before deleting. Signed-off-by: Airat Arifullin --- pkg/local_object_storage/pilorama/boltdb.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 1bc161341..52cdec586 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1115,11 +1115,12 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) c := tx.Cursor() prefix := make([]byte, 32) cid.Encode(prefix) - for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = c.Next() { + for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = c.Seek(prefix) { err := tx.DeleteBucket(k) if err != nil { return err } + _, _ = c.First() // rewind the cursor to the root page } return nil } From 76ff26039c4e9f9dceaabda5127c8909c95b290e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 17 Nov 2023 10:58:04 +0300 Subject: [PATCH 0690/1943] [#96] node: Drop neo-go's `slices` package Signed-off-by: Dmitrii Stepanov --- pkg/innerring/processors/frostfs/handlers.go | 11 ++++++++--- pkg/local_object_storage/blobovnicza/get.go | 4 ++-- pkg/local_object_storage/blobovnicza/iterate_test.go | 4 ++-- .../blobstor/blobovniczatree/exists_test.go | 4 ++-- pkg/local_object_storage/metabase/shard_id.go | 5 +++-- pkg/local_object_storage/metabase/storage_id.go | 4 ++-- pkg/local_object_storage/shard/range_test.go | 4 ++-- .../writecache/writecachebbolt/flush.go | 5 ++--- .../writecache/writecachebbolt/get.go | 4 ++-- 9 files changed, 25 insertions(+), 20 deletions(-) diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index ab53d5c48..adb923b87 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -1,21 +1,24 @@ package frostfs import ( + "bytes" "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" - "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.uber.org/zap" + "golang.org/x/exp/slices" ) func (np *Processor) handleDeposit(ev event.Event) { deposit := ev.(frostfsEvent.Deposit) + depositIDBin := bytes.Clone(deposit.ID()) + slices.Reverse(depositIDBin) np.log.Info(logs.Notification, zap.String("type", "deposit"), - zap.String("id", hex.EncodeToString(slice.CopyReverse(deposit.ID())))) + zap.String("id", hex.EncodeToString(depositIDBin))) // send event to the worker pool @@ -31,9 +34,11 @@ func (np *Processor) handleDeposit(ev event.Event) { func (np *Processor) handleWithdraw(ev event.Event) { withdraw := ev.(frostfsEvent.Withdraw) + withdrawBin := bytes.Clone(withdraw.ID()) + slices.Reverse(withdrawBin) np.log.Info(logs.Notification, zap.String("type", "withdraw"), - zap.String("id", hex.EncodeToString(slice.CopyReverse(withdraw.ID())))) + zap.String("id", hex.EncodeToString(withdrawBin))) // send event to the worker pool diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index 3a613cfeb..9975e5e8c 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -1,13 +1,13 @@ package blobovnicza import ( + "bytes" "context" "errors" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -63,7 +63,7 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { return nil } - data = slice.Copy(data) + data = bytes.Clone(data) return errInterruptForEach }) diff --git a/pkg/local_object_storage/blobovnicza/iterate_test.go b/pkg/local_object_storage/blobovnicza/iterate_test.go index 90308723c..5db1e4165 100644 --- a/pkg/local_object_storage/blobovnicza/iterate_test.go +++ b/pkg/local_object_storage/blobovnicza/iterate_test.go @@ -1,13 +1,13 @@ package blobovnicza import ( + "bytes" "context" "errors" "path/filepath" "testing" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - "github.com/nspcc-dev/neo-go/pkg/util/slice" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" ) @@ -30,7 +30,7 @@ func TestBlobovniczaIterate(t *testing.T) { seen := make([][]byte, 0, 2) inc := func(e IterationElement) error { - seen = append(seen, slice.Copy(e.data)) + seen = append(seen, bytes.Clone(e.data)) return nil } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 198e4741b..c89262343 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "bytes" "context" "os" "path/filepath" @@ -10,7 +11,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" - "github.com/nspcc-dev/neo-go/pkg/util/slice" "github.com/stretchr/testify/require" ) @@ -37,7 +37,7 @@ func TestExistsInvalidStorageID(t *testing.T) { t.Run("valid but wrong storage id", func(t *testing.T) { // "0/X/Y" <-> "1/X/Y" - storageID := slice.Copy(putRes.StorageID) + storageID := bytes.Clone(putRes.StorageID) if storageID[0] == '0' { storageID[0]++ } else { diff --git a/pkg/local_object_storage/metabase/shard_id.go b/pkg/local_object_storage/metabase/shard_id.go index f60a4724d..c1ec8c67e 100644 --- a/pkg/local_object_storage/metabase/shard_id.go +++ b/pkg/local_object_storage/metabase/shard_id.go @@ -1,8 +1,9 @@ package meta import ( + "bytes" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" - "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" ) @@ -25,7 +26,7 @@ func (db *DB) ReadShardID() ([]byte, error) { err := db.boltDB.View(func(tx *bbolt.Tx) error { b := tx.Bucket(shardInfoBucket) if b != nil { - id = slice.Copy(b.Get(shardIDKey)) + id = bytes.Clone(b.Get(shardIDKey)) } return nil }) diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index 0a64da856..ef6b44dfd 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -1,6 +1,7 @@ package meta import ( + "bytes" "context" "errors" "time" @@ -9,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -72,7 +72,7 @@ func (db *DB) storageID(tx *bbolt.Tx, addr oid.Address) ([]byte, error) { return nil, nil } - return slice.Copy(storageID), nil + return bytes.Clone(storageID), nil } // UpdateStorageIDPrm groups the parameters of UpdateStorageID operation. diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index a8bc83307..098a584ad 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -1,6 +1,7 @@ package shard import ( + "bytes" "context" "math" "path/filepath" @@ -16,7 +17,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - "github.com/nspcc-dev/neo-go/pkg/util/slice" "github.com/stretchr/testify/require" ) @@ -105,7 +105,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { testutil.AddPayload(obj, tc.payloadSize) addr := object.AddressOf(obj) - payload := slice.Copy(obj.Payload()) + payload := bytes.Clone(obj.Payload()) var putPrm PutPrm putPrm.SetObject(obj) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush.go b/pkg/local_object_storage/writecache/writecachebbolt/flush.go index 9efee1dda..d0e89a385 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush.go @@ -17,7 +17,6 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/mr-tron/base58" - "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -106,12 +105,12 @@ func (c *cache) flushSmallObjects(ctx context.Context) { if len(lastKey) == len(k) { copy(lastKey, k) } else { - lastKey = slice.Copy(k) + lastKey = bytes.Clone(k) } m = append(m, objectInfo{ addr: string(k), - data: slice.Copy(v), + data: bytes.Clone(v), }) } return nil diff --git a/pkg/local_object_storage/writecache/writecachebbolt/get.go b/pkg/local_object_storage/writecache/writecachebbolt/get.go index dede1fece..838b207be 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/get.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/get.go @@ -1,6 +1,7 @@ package writecachebbolt import ( + "bytes" "context" "time" @@ -12,7 +13,6 @@ import ( apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -95,7 +95,7 @@ func Get(db *bbolt.DB, key []byte) ([]byte, error) { if value == nil { return logicerr.Wrap(new(apistatus.ObjectNotFound)) } - value = slice.Copy(value) + value = bytes.Clone(value) return nil }) return value, metaerr.Wrap(err) From 02454df14a54d6e51e5f454ad1dcbaf65ddf2147 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Wed, 15 Nov 2023 15:28:45 +0300 Subject: [PATCH 0691/1943] [#809] client: Refactor PrmInit, PrmDial usage Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/sdk.go | 26 ++++++++------- go.mod | 2 +- go.sum | 4 +-- pkg/network/cache/multi.go | 44 ++++++++++++-------------- 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/sdk.go b/cmd/frostfs-cli/internal/client/sdk.go index 6923e4f17..b3e04e5f5 100644 --- a/cmd/frostfs-cli/internal/client/sdk.go +++ b/cmd/frostfs-cli/internal/client/sdk.go @@ -43,26 +43,28 @@ func getSDKClientByFlag(cmd *cobra.Command, key *ecdsa.PrivateKey, endpointFlag // GetSDKClient returns default frostfs-sdk-go client. func GetSDKClient(ctx context.Context, cmd *cobra.Command, key *ecdsa.PrivateKey, addr network.Address) (*client.Client, error) { - var ( - c client.Client - prmInit client.PrmInit - prmDial client.PrmDial - ) + var c client.Client - prmInit.SetDefaultPrivateKey(*key) - prmDial.SetServerURI(addr.URIAddr()) + prmInit := client.PrmInit{ + Key: *key, + } + + prmDial := client.PrmDial{ + Endpoint: addr.URIAddr(), + GRPCDialOptions: []grpc.DialOption{ + grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInteceptor()), + grpc.WithChainStreamInterceptor(tracing.NewStreamClientInterceptor()), + }, + } if timeout := viper.GetDuration(commonflags.Timeout); timeout > 0 { // In CLI we can only set a timeout for the whole operation. // By also setting stream timeout we ensure that no operation hands // for too long. - prmDial.SetTimeout(timeout) - prmDial.SetStreamTimeout(timeout) + prmDial.DialTimeout = timeout + prmDial.StreamTimeout = timeout common.PrintVerbose(cmd, "Set request timeout to %s.", timeout) } - prmDial.SetGRPCDialOptions( - grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInteceptor()), - grpc.WithChainStreamInterceptor(tracing.NewStreamClientInterceptor())) c.Init(prmInit) diff --git a/go.mod b/go.mod index 120c05ff4..7d2b93486 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231114081800-3787477133f3 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231115094736-5db67021e10f git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index 642280d95..070c246ba 100644 --- a/go.sum +++ b/go.sum @@ -732,8 +732,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d h1:KVHsNsEoedQ1+f+eDq88AFKkLePGfnygt4bcUQrCgPw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231101144515-6fbe1595cb3d/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231114081800-3787477133f3 h1:Qa35bB58plMb14LIsYzu2ibeYfsY2taFnbZytv9Ao3M= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231114081800-3787477133f3/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231115094736-5db67021e10f h1:Rq95TuEkqc3T1EN5ZU1Vgf6H33TR95hz97ca8jrUciQ= diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index c9b13826a..5dd206283 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -51,42 +51,40 @@ func newMultiClient(addr network.AddressGroup, opts ClientCacheOpts) *multiClien } func (x *multiClient) createForAddress(ctx context.Context, addr network.Address) (clientcore.Client, error) { - var ( - c client.Client - prmInit client.PrmInit - prmDial client.PrmDial - ) - - prmInit.DisableFrostFSFailuresResolution() - prmDial.SetServerURI(addr.URIAddr()) + var c client.Client + prmInit := client.PrmInit{ + DisableFrostFSErrorResolution: true, + } if x.opts.Key != nil { - prmInit.SetDefaultPrivateKey(*x.opts.Key) + prmInit.Key = *x.opts.Key } + prmDial := client.PrmDial{ + Endpoint: addr.URIAddr(), + GRPCDialOptions: []grpc.DialOption{ + grpc.WithChainUnaryInterceptor( + metrics.NewUnaryClientInterceptor(), + tracing.NewUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + metrics.NewStreamClientInterceptor(), + tracing.NewStreamClientInterceptor(), + ), + }, + } if x.opts.DialTimeout > 0 { - prmDial.SetTimeout(x.opts.DialTimeout) + prmDial.DialTimeout = x.opts.DialTimeout } if x.opts.StreamTimeout > 0 { - prmDial.SetStreamTimeout(x.opts.StreamTimeout) + prmDial.StreamTimeout = x.opts.StreamTimeout } if x.opts.ResponseCallback != nil { - prmInit.SetResponseInfoCallback(x.opts.ResponseCallback) + prmInit.ResponseInfoCallback = x.opts.ResponseCallback } - prmDial.SetGRPCDialOptions( - grpc.WithChainUnaryInterceptor( - metrics.NewUnaryClientInterceptor(), - tracing.NewUnaryClientInteceptor(), - ), - grpc.WithChainStreamInterceptor( - metrics.NewStreamClientInterceptor(), - tracing.NewStreamClientInterceptor(), - ), - ) - c.Init(prmInit) err := c.Dial(ctx, prmDial) if err != nil { From 8d18fa159e085dde62336ec80f76fdfee358c412 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 17 Nov 2023 17:41:13 +0300 Subject: [PATCH 0692/1943] [#667] writecache: Fix flush test Allow to disable background flusher for testing purposes. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 - .../writecache/writecachebadger/flush.go | 3 ++ .../writecache/writecachebadger/flush_test.go | 1 + .../writecache/writecachebadger/options.go | 9 ++++ .../writecache/writecachebbolt/flush.go | 5 +- .../writecache/writecachebbolt/flush_test.go | 1 + .../writecache/writecachebbolt/options.go | 9 ++++ .../writecache/writecachebbolt/storage.go | 47 +++++-------------- 8 files changed, 40 insertions(+), 36 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 5f361cefb..e8472357c 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -296,7 +296,6 @@ const ( WritecacheFillingFlushMarksForObjectsInDatabase = "filling flush marks for objects in database" WritecacheFinishedUpdatingFlushMarks = "finished updating flush marks" WritecacheCantRemoveObjectsFromTheDatabase = "can't remove objects from the database" - WritecacheCantParseAddress = "can't parse address" WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" WritecacheDBValueLogGCRunCompleted = "value log GC run completed" WritecacheBadgerObjAlreadyScheduled = "object already scheduled for flush" diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush.go b/pkg/local_object_storage/writecache/writecachebadger/flush.go index 1ee6bbb66..48e31dee6 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush.go @@ -97,6 +97,9 @@ func (c *collector) send(ctx context.Context, cancel func(), buf *z.Buffer) erro // runFlushLoop starts background workers which periodically flush objects to the blobstor. func (c *cache) runFlushLoop(ctx context.Context) { + if c.disableBackgroundFlush { + return + } for i := 0; i < c.workersCount; i++ { c.wg.Add(1) go c.workerFlushSmall(ctx) diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go index 392654e48..5e3f60eab 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go @@ -28,6 +28,7 @@ func TestFlush(t *testing.T) { WithMetabase(mb), WithBlobstor(bs), WithGCInterval(1 * time.Second), + WithDisableBackgroundFlush(), }, opts...)...) } diff --git a/pkg/local_object_storage/writecache/writecachebadger/options.go b/pkg/local_object_storage/writecache/writecachebadger/options.go index 63bfb196c..d041a9b88 100644 --- a/pkg/local_object_storage/writecache/writecachebadger/options.go +++ b/pkg/local_object_storage/writecache/writecachebadger/options.go @@ -34,6 +34,8 @@ type options struct { metrics writecache.Metrics // gcInterval is the interval duration to run the GC cycle. gcInterval time.Duration + // disableBackgroundFlush is for testing purposes only. + disableBackgroundFlush bool } // WithLogger sets logger. @@ -108,3 +110,10 @@ func WithGCInterval(d time.Duration) Option { o.gcInterval = d } } + +// WithDisableBackgroundFlush disables background flush, for testing purposes only. +func WithDisableBackgroundFlush() Option { + return func(o *options) { + o.disableBackgroundFlush = true + } +} diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush.go b/pkg/local_object_storage/writecache/writecachebbolt/flush.go index d0e89a385..d73e374f5 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush.go @@ -36,6 +36,9 @@ const ( // runFlushLoop starts background workers which periodically flush objects to the blobstor. func (c *cache) runFlushLoop(ctx context.Context) { + if c.disableBackgroundFlush { + return + } for i := 0; i < c.workersCount; i++ { c.wg.Add(1) go c.workerFlushSmall(ctx) @@ -200,7 +203,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - c.deleteFromDisk(ctx, []string{sAddr}) + c.deleteFromDisk(ctx, e.Address) return nil } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go index 533cec1d2..89add8115 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go @@ -31,6 +31,7 @@ func TestFlush(t *testing.T) { WithSmallObjectSize(smallSize), WithMetabase(mb), WithBlobstor(bs), + WithDisableBackgroundFlush(), }, opts...)...) } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/options.go b/pkg/local_object_storage/writecache/writecachebbolt/options.go index d8eedfc79..3ea329192 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/options.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/options.go @@ -44,6 +44,8 @@ type options struct { openFile func(string, int, fs.FileMode) (*os.File, error) // metrics is metrics implementation metrics writecache.Metrics + // disableBackgroundFlush is for testing purposes only. + disableBackgroundFlush bool } // WithLogger sets logger. @@ -155,3 +157,10 @@ func WithMetrics(metrics writecache.Metrics) Option { o.metrics = metrics } } + +// WithDisableBackgroundFlush disables background flush, for testing purposes only. +func WithDisableBackgroundFlush() Option { + return func(o *options) { + o.disableBackgroundFlush = true + } +} diff --git a/pkg/local_object_storage/writecache/writecachebbolt/storage.go b/pkg/local_object_storage/writecache/writecachebbolt/storage.go index bff9385dc..7a5030625 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/storage.go +++ b/pkg/local_object_storage/writecache/writecachebbolt/storage.go @@ -93,39 +93,18 @@ func (c *cache) deleteFromDB(key string) { } } -func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string { - if len(keys) == 0 { - return keys +func (c *cache) deleteFromDisk(ctx context.Context, addr oid.Address) { + _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) + if err != nil && !client.IsErrObjectNotFound(err) { + c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) + } else if err == nil { + storagelog.Write(c.log, + storagelog.AddressField(addr.EncodeToString()), + storagelog.StorageTypeField(wcStorageType), + storagelog.OpField("fstree DELETE"), + ) + c.metrics.Evict(writecache.StorageTypeFSTree) + // counter changed by fstree + c.estimateCacheSize() } - - var copyIndex int - var addr oid.Address - - for i := range keys { - if err := addr.DecodeString(keys[i]); err != nil { - c.log.Error(logs.WritecacheCantParseAddress, zap.String("address", keys[i])) - continue - } - - _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) - if err != nil && !client.IsErrObjectNotFound(err) { - c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) - - // Save the key for the next iteration. - keys[copyIndex] = keys[i] - copyIndex++ - continue - } else if err == nil { - storagelog.Write(c.log, - storagelog.AddressField(keys[i]), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("fstree DELETE"), - ) - c.metrics.Evict(writecache.StorageTypeFSTree) - // counter changed by fstree - c.estimateCacheSize() - } - } - - return keys[:copyIndex] } From 07390ad4e359b7bc24a09af7e22a8f8bc7f781fc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 21 Nov 2023 13:20:36 +0300 Subject: [PATCH 0693/1943] [#715] node: Unify config parameter names Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 14 +++++++------- cmd/frostfs-node/config/engine/config_test.go | 8 ++++---- .../config/engine/shard/gc/config.go | 6 +++--- .../config/engine/shard/writecache/config.go | 6 +++--- cmd/frostfs-node/config/object/config.go | 8 ++++---- config/example/node.env | 10 +++++----- config/example/node.json | 10 +++++----- config/example/node.yaml | 8 ++++---- config/mainnet/config.yml | 4 ++-- docs/storage-node-configuration.md | 16 ++++++++-------- pkg/local_object_storage/shard/gc.go | 12 ++++++------ pkg/local_object_storage/shard/shard.go | 6 +++--- 12 files changed, 54 insertions(+), 54 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 3be33a2a1..cc106cf95 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -126,10 +126,10 @@ type shardCfg struct { subStorages []subStorageCfg gcCfg struct { - removerBatchSize int - removerSleepInterval time.Duration - expiredCollectorBatchSize int - expiredCollectorWorkersCount int + removerBatchSize int + removerSleepInterval time.Duration + expiredCollectorBatchSize int + expiredCollectorWorkerCount int } writecacheCfg struct { @@ -256,7 +256,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc.maxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay() wc.maxObjSize = writeCacheCfg.MaxObjectSize() wc.smallObjectSize = writeCacheCfg.SmallObjectSize() - wc.flushWorkerCount = writeCacheCfg.WorkersNumber() + wc.flushWorkerCount = writeCacheCfg.WorkerCount() wc.sizeLimit = writeCacheCfg.SizeLimit() wc.noSync = writeCacheCfg.NoSync() wc.gcInterval = writeCacheCfg.GCInterval() @@ -328,7 +328,7 @@ func (a *applicationConfiguration) setGCConfig(newConfig *shardCfg, oldConfig *s newConfig.gcCfg.removerBatchSize = gcCfg.RemoverBatchSize() newConfig.gcCfg.removerSleepInterval = gcCfg.RemoverSleepInterval() newConfig.gcCfg.expiredCollectorBatchSize = gcCfg.ExpiredCollectorBatchSize() - newConfig.gcCfg.expiredCollectorWorkersCount = gcCfg.ExpiredCollectorWorkersCount() + newConfig.gcCfg.expiredCollectorWorkerCount = gcCfg.ExpiredCollectorWorkerCount() } // internals contains application-specific internals that are created @@ -888,7 +888,7 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { shard.WithRemoverBatchSize(shCfg.gcCfg.removerBatchSize), shard.WithGCRemoverSleepInterval(shCfg.gcCfg.removerSleepInterval), shard.WithExpiredCollectorBatchSize(shCfg.gcCfg.expiredCollectorBatchSize), - shard.WithExpiredCollectorWorkersCount(shCfg.gcCfg.expiredCollectorWorkersCount), + shard.WithExpiredCollectorWorkerCount(shCfg.gcCfg.expiredCollectorWorkerCount), shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { pool, err := ants.NewPool(sz) fatalOnErr(err) diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 6b7c268ce..665f70bcb 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -74,7 +74,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, "tmp/0/cache", wc.Path()) require.EqualValues(t, 16384, wc.SmallObjectSize()) require.EqualValues(t, 134217728, wc.MaxObjectSize()) - require.EqualValues(t, 30, wc.WorkersNumber()) + require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 3221225472, wc.SizeLimit()) require.Equal(t, "tmp/0/meta", meta.Path()) @@ -108,7 +108,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 150, gc.RemoverBatchSize()) require.Equal(t, 2*time.Minute, gc.RemoverSleepInterval()) require.Equal(t, 1500, gc.ExpiredCollectorBatchSize()) - require.Equal(t, 15, gc.ExpiredCollectorWorkersCount()) + require.Equal(t, 15, gc.ExpiredCollectorWorkerCount()) require.Equal(t, false, sc.RefillMetabase()) require.Equal(t, mode.ReadOnly, sc.Mode()) @@ -125,7 +125,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, "tmp/1/cache", wc.Path()) require.EqualValues(t, 16384, wc.SmallObjectSize()) require.EqualValues(t, 134217728, wc.MaxObjectSize()) - require.EqualValues(t, 30, wc.WorkersNumber()) + require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 4294967296, wc.SizeLimit()) require.Equal(t, "tmp/1/meta", meta.Path()) @@ -157,7 +157,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 200, gc.RemoverBatchSize()) require.Equal(t, 5*time.Minute, gc.RemoverSleepInterval()) require.Equal(t, gcconfig.ExpiredCollectorBatchSizeDefault, gc.ExpiredCollectorBatchSize()) - require.Equal(t, gcconfig.ExpiredCollectorWorkersCountDefault, gc.ExpiredCollectorWorkersCount()) + require.Equal(t, gcconfig.ExpiredCollectorWorkersCountDefault, gc.ExpiredCollectorWorkerCount()) require.Equal(t, true, sc.RefillMetabase()) require.Equal(t, mode.ReadWrite, sc.Mode()) diff --git a/cmd/frostfs-node/config/engine/shard/gc/config.go b/cmd/frostfs-node/config/engine/shard/gc/config.go index 0500697c8..8cb90d3ff 100644 --- a/cmd/frostfs-node/config/engine/shard/gc/config.go +++ b/cmd/frostfs-node/config/engine/shard/gc/config.go @@ -63,14 +63,14 @@ func (x *Config) RemoverSleepInterval() time.Duration { return RemoverSleepIntervalDefault } -// ExpiredCollectorWorkersCount returns the value of "expired_collector_workers_count" +// ExpiredCollectorWorkerCount returns the value of "expired_collector_worker_count" // config parameter. // // Returns ExpiredCollectorWorkersCountDefault if the value is not a positive number. -func (x *Config) ExpiredCollectorWorkersCount() int { +func (x *Config) ExpiredCollectorWorkerCount() int { s := config.IntSafe( (*config.Config)(x), - "expired_collector_workers_count", + "expired_collector_worker_count", ) if s > 0 { diff --git a/cmd/frostfs-node/config/engine/shard/writecache/config.go b/cmd/frostfs-node/config/engine/shard/writecache/config.go index 504fe3ca2..d0fd4bf75 100644 --- a/cmd/frostfs-node/config/engine/shard/writecache/config.go +++ b/cmd/frostfs-node/config/engine/shard/writecache/config.go @@ -106,13 +106,13 @@ func (x *Config) MaxObjectSize() uint64 { return MaxSizeDefault } -// WorkersNumber returns the value of "workers_number" config parameter. +// WorkerCount returns the value of "flush_worker_count" config parameter. // // Returns WorkersNumberDefault if the value is not a positive number. -func (x *Config) WorkersNumber() int { +func (x *Config) WorkerCount() int { c := config.IntSafe( (*config.Config)(x), - "workers_number", + "flush_worker_count", ) if c > 0 { diff --git a/cmd/frostfs-node/config/object/config.go b/cmd/frostfs-node/config/object/config.go index f7a33b5e0..876dc3ef1 100644 --- a/cmd/frostfs-node/config/object/config.go +++ b/cmd/frostfs-node/config/object/config.go @@ -28,11 +28,11 @@ func Put(c *config.Config) PutConfig { } } -// PoolSizeRemote returns the value of "pool_size_remote" config parameter. +// PoolSizeRemote returns the value of "remote_pool_size" config parameter. // // Returns PutPoolSizeDefault if the value is not a positive number. func (g PutConfig) PoolSizeRemote() int { - v := config.Int(g.cfg, "pool_size_remote") + v := config.Int(g.cfg, "remote_pool_size") if v > 0 { return int(v) } @@ -40,11 +40,11 @@ func (g PutConfig) PoolSizeRemote() int { return PutPoolSizeDefault } -// PoolSizeLocal returns the value of "pool_size_local" config parameter. +// PoolSizeLocal returns the value of "local_pool_size" config parameter. // // Returns PutPoolSizeDefault if the value is not a positive number. func (g PutConfig) PoolSizeLocal() int { - v := config.Int(g.cfg, "pool_size_local") + v := config.Int(g.cfg, "local_pool_size") if v > 0 { return int(v) } diff --git a/config/example/node.env b/config/example/node.env index dda740cf1..f4dc218a3 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -84,8 +84,8 @@ FROSTFS_REPLICATOR_PUT_TIMEOUT=15s FROSTFS_REPLICATOR_POOL_SIZE=10 # Object service section -FROSTFS_OBJECT_PUT_POOL_SIZE_REMOTE=100 -FROSTFS_OBJECT_PUT_POOL_SIZE_LOCAL=200 +FROSTFS_OBJECT_PUT_REMOTE_POOL_SIZE=100 +FROSTFS_OBJECT_PUT_LOCAL_POOL_SIZE=200 FROSTFS_OBJECT_PUT_SKIP_SESSION_TOKEN_ISSUER_VERIFICATION=true FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10 @@ -103,7 +103,7 @@ FROSTFS_STORAGE_SHARD_0_WRITECACHE_NO_SYNC=true FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH=tmp/0/cache FROSTFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384 FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728 -FROSTFS_STORAGE_SHARD_0_WRITECACHE_WORKERS_NUMBER=30 +FROSTFS_STORAGE_SHARD_0_WRITECACHE_FLUSH_WORKER_COUNT=30 FROSTFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472 ### Metabase config FROSTFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta @@ -142,7 +142,7 @@ FROSTFS_STORAGE_SHARD_0_GC_REMOVER_SLEEP_INTERVAL=2m #### Limit of objects to be marked expired by the garbage collector FROSTFS_STORAGE_SHARD_0_GC_EXPIRED_COLLECTOR_BATCH_SIZE=1500 #### Limit of concurrent workers collecting expired objects by the garbage collector -FROSTFS_STORAGE_SHARD_0_GC_EXPIRED_COLLECTOR_WORKERS_COUNT=15 +FROSTFS_STORAGE_SHARD_0_GC_EXPIRED_COLLECTOR_WORKER_COUNT=15 ## 1 shard ### Flag to refill Metabase from BlobStor @@ -154,7 +154,7 @@ FROSTFS_STORAGE_SHARD_1_WRITECACHE_ENABLED=true FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH=tmp/1/cache FROSTFS_STORAGE_SHARD_1_WRITECACHE_SMALL_OBJECT_SIZE=16384 FROSTFS_STORAGE_SHARD_1_WRITECACHE_MAX_OBJECT_SIZE=134217728 -FROSTFS_STORAGE_SHARD_1_WRITECACHE_WORKERS_NUMBER=30 +FROSTFS_STORAGE_SHARD_1_WRITECACHE_FLUSH_WORKER_COUNT=30 FROSTFS_STORAGE_SHARD_1_WRITECACHE_CAPACITY=4294967296 ### Metabase config FROSTFS_STORAGE_SHARD_1_METABASE_PATH=tmp/1/meta diff --git a/config/example/node.json b/config/example/node.json index 1038d5e5c..c2e3d0a5d 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -129,8 +129,8 @@ "tombstone_lifetime": 10 }, "put": { - "pool_size_remote": 100, - "pool_size_local": 200, + "remote_pool_size": 100, + "local_pool_size": 200, "skip_session_token_issuer_verification": true } }, @@ -147,7 +147,7 @@ "path": "tmp/0/cache", "small_object_size": 16384, "max_object_size": 134217728, - "workers_number": 30, + "flush_worker_count": 30, "capacity": 3221225472 }, "metabase": { @@ -190,7 +190,7 @@ "remover_batch_size": 150, "remover_sleep_interval": "2m", "expired_collector_batch_size": 1500, - "expired_collector_workers_count": 15 + "expired_collector_worker_count": 15 } }, "1": { @@ -203,7 +203,7 @@ "memcache_capacity": 2147483648, "small_object_size": 16384, "max_object_size": 134217728, - "workers_number": 30, + "flush_worker_count": 30, "capacity": 4294967296 }, "metabase": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 8b2046e95..d04e122b6 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -108,8 +108,8 @@ object: delete: tombstone_lifetime: 10 # tombstone "local" lifetime in epochs put: - pool_size_remote: 100 # number of async workers for remote PUT operations - pool_size_local: 200 # number of async workers for local PUT operations + remote_pool_size: 100 # number of async workers for remote PUT operations + local_pool_size: 200 # number of async workers for local PUT operations skip_session_token_issuer_verification: true # session token issuer verification will be skipped if true storage: @@ -126,7 +126,7 @@ storage: type: bbolt small_object_size: 16k # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes max_object_size: 134217728 # size threshold for "big" objects which bypass write-cache and go to the storage directly, bytes - workers_number: 30 # number of write-cache flusher threads + flush_worker_count: 30 # number of write-cache flusher threads metabase: perm: 0644 # permissions for metabase files(directories: +x for current user and group) @@ -196,7 +196,7 @@ storage: remover_batch_size: 150 # number of objects to be removed by the garbage collector remover_sleep_interval: 2m # frequency of the garbage collector invocation expired_collector_batch_size: 1500 # number of objects to be marked expired by the garbage collector - expired_collector_workers_count: 15 # number of concurrent workers collecting expired objects by the garbage collector + expired_collector_worker_count: 15 # number of concurrent workers collecting expired objects by the garbage collector 1: writecache: diff --git a/config/mainnet/config.yml b/config/mainnet/config.yml index 7db476e55..d86ea451f 100644 --- a/config/mainnet/config.yml +++ b/config/mainnet/config.yml @@ -50,8 +50,8 @@ prometheus: object: put: - pool_size_remote: 100 - pool_size_local: 100 + remote_pool_size: 100 + local_pool_size: 100 morph: rpc_endpoint: diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 5e9f3caf7..71eabc042 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -243,7 +243,7 @@ gc: remover_batch_size: 200 remover_sleep_interval: 5m expired_collector_batch_size: 500 - expired_collector_workers_count: 5 + expired_collector_worker_count: 5 ``` | Parameter | Type | Default value | Description | @@ -251,7 +251,7 @@ gc: | `remover_batch_size` | `int` | `100` | Amount of objects to grab in a single batch. | | `remover_sleep_interval` | `duration` | `1m` | Time to sleep between iterations. | | `expired_collector_batch_size` | `int` | `500` | Max amount of expired objects to grab in a single batch. | -| `expired_collector_workers_count` | `int` | `5` | Max amount of concurrent expired objects workers. | +| `expired_collector_worker_count` | `int` | `5` | Max amount of concurrent expired objects workers. | ### `metabase` subsection @@ -280,7 +280,7 @@ writecache: capacity: 4294967296 small_object_size: 16384 max_object_size: 134217728 - workers_number: 30 + flush_worker_count: 30 ``` | Parameter | Type | Default value | Description | @@ -290,7 +290,7 @@ writecache: | `capacity` | `size` | unrestricted | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | | `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | | `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. | -| `workers_number` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | +| `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | | `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | | `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | @@ -415,7 +415,7 @@ replicator: | Parameter | Type | Default value | Description | |---------------|------------|----------------------------------------|---------------------------------------------| | `put_timeout` | `duration` | `5s` | Timeout for performing the `PUT` operation. | -| `pool_size` | `int` | Equal to `object.put.pool_size_remote` | Maximum amount of concurrent replications. | +| `pool_size` | `int` | Equal to `object.put.remote_pool_size` | Maximum amount of concurrent replications. | # `object` section Contains object-service related parameters. @@ -423,14 +423,14 @@ Contains object-service related parameters. ```yaml object: put: - pool_size_remote: 100 + remote_pool_size: 100 ``` | Parameter | Type | Default value | Description | |-----------------------------|-------|---------------|------------------------------------------------------------------------------------------------| | `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | -| `put.pool_size_remote` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | -| `put.pool_size_local` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | +| `put.remote_pool_size` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | +| `put.local_pool_size` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | # `runtime` section Contains runtime parameters. diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 346903c5c..e16f89457 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -115,8 +115,8 @@ type gcCfg struct { workerPoolInit func(int) util.WorkerPool - expiredCollectorWorkersCount int - expiredCollectorBatchSize int + expiredCollectorWorkerCount int + expiredCollectorBatchSize int metrics GCMectrics @@ -313,16 +313,16 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { return } -func (s *Shard) getExpiredObjectsParameters() (workersCount, batchSize int) { - workersCount = minExpiredWorkers +func (s *Shard) getExpiredObjectsParameters() (workerCount, batchSize int) { + workerCount = minExpiredWorkers batchSize = minExpiredBatchSize if s.gc.gcCfg.expiredCollectorBatchSize > batchSize { batchSize = s.gc.gcCfg.expiredCollectorBatchSize } - if s.gc.gcCfg.expiredCollectorWorkersCount > workersCount { - workersCount = s.gc.gcCfg.expiredCollectorWorkersCount + if s.gc.gcCfg.expiredCollectorWorkerCount > workerCount { + workerCount = s.gc.gcCfg.expiredCollectorWorkerCount } return } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 07b774022..aa87e0cfc 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -365,11 +365,11 @@ func WithExpiredCollectorBatchSize(size int) Option { } } -// WithExpiredCollectorWorkersCount returns option to set concurrent +// WithExpiredCollectorWorkerCount returns option to set concurrent // workers count of expired object collection operation. -func WithExpiredCollectorWorkersCount(count int) Option { +func WithExpiredCollectorWorkerCount(count int) Option { return func(c *cfg) { - c.gcCfg.expiredCollectorWorkersCount = count + c.gcCfg.expiredCollectorWorkerCount = count } } From c99157f0b20a503a3baab25e96eafbef0cff82cb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 21 Nov 2023 11:42:08 +0300 Subject: [PATCH 0694/1943] [#821] go.mod: Update SDK-Go version Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7d2b93486..2714e1567 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231114081800-3787477133f3 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231115094736-5db67021e10f git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index 070c246ba..7660d9dcd 100644 --- a/go.sum +++ b/go.sum @@ -732,8 +732,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231114081800-3787477133f3 h1:Qa35bB58plMb14LIsYzu2ibeYfsY2taFnbZytv9Ao3M= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231114081800-3787477133f3/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e h1:RoGxgSNyVxDaYgjb40P+Xy6val+rs7b5QQtGkIUpUoY= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231115094736-5db67021e10f h1:Rq95TuEkqc3T1EN5ZU1Vgf6H33TR95hz97ca8jrUciQ= From c516c7c5f46d6a3ae8e183f4582baf112ca0a50e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 21 Nov 2023 11:42:30 +0300 Subject: [PATCH 0695/1943] [#821] node: Pass user.ID by value Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/accounting/balance.go | 2 +- cmd/frostfs-cli/modules/container/list.go | 2 +- cmd/frostfs-cli/modules/object/lock.go | 2 +- cmd/frostfs-cli/modules/object/put.go | 4 ++-- pkg/core/object/fmt.go | 4 ++-- pkg/core/object/fmt_test.go | 14 +++++++------- pkg/services/object/acl/acl_test.go | 6 +++--- pkg/services/object/acl/eacl/v2/object.go | 4 ++-- pkg/services/object/delete/exec.go | 4 ++-- pkg/services/object/put/streamer.go | 2 +- pkg/services/object/util/key_test.go | 4 ++-- .../session/storage/persistent/executor_test.go | 6 +++--- 12 files changed, 27 insertions(+), 27 deletions(-) diff --git a/cmd/frostfs-cli/modules/accounting/balance.go b/cmd/frostfs-cli/modules/accounting/balance.go index 8e975fb03..d10b9eaa1 100644 --- a/cmd/frostfs-cli/modules/accounting/balance.go +++ b/cmd/frostfs-cli/modules/accounting/balance.go @@ -39,7 +39,7 @@ var accountingBalanceCmd = &cobra.Command{ var prm internalclient.BalanceOfPrm prm.SetClient(cli) - prm.Account = &idUser + prm.Account = idUser res, err := internalclient.BalanceOf(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 1e7ba96ea..86c1e5fba 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -47,7 +47,7 @@ var listContainersCmd = &cobra.Command{ var prm internalclient.ListContainersPrm prm.SetClient(cli) - prm.Account = &idUser + prm.Account = idUser res, err := internalclient.ListContainers(cmd.Context(), prm) commonCmd.ExitOnErr(cmd, "rpc error: %w", err) diff --git a/cmd/frostfs-cli/modules/object/lock.go b/cmd/frostfs-cli/modules/object/lock.go index fa1898586..d2e9af24c 100644 --- a/cmd/frostfs-cli/modules/object/lock.go +++ b/cmd/frostfs-cli/modules/object/lock.go @@ -94,7 +94,7 @@ var objectLockCmd = &cobra.Command{ obj := objectSDK.New() obj.SetContainerID(cnr) - obj.SetOwnerID(&idOwner) + obj.SetOwnerID(idOwner) obj.SetType(objectSDK.TypeLock) obj.SetAttributes(expirationAttr) obj.SetPayload(lock.Marshal()) diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index 97bb12dbc..a41d3c0aa 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -93,7 +93,7 @@ func putObject(cmd *cobra.Command, _ []string) { attrs := getAllObjectAttributes(cmd) obj.SetContainerID(cnr) - obj.SetOwnerID(&ownerID) + obj.SetOwnerID(ownerID) obj.SetAttributes(attrs...) notificationInfo, err := parseObjectNotifications(cmd) @@ -160,7 +160,7 @@ func readFilePayload(filename string, cmd *cobra.Command) (io.Reader, cid.ID, us commonCmd.ExitOnErr(cmd, "can't unmarshal object from given file: %w", objTemp.Unmarshal(buf)) payloadReader := bytes.NewReader(objTemp.Payload()) cnr, _ := objTemp.ContainerID() - ownerID := *objTemp.OwnerID() + ownerID := objTemp.OwnerID() return payloadReader, cnr, ownerID } diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index e65767723..e266800b2 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -165,7 +165,7 @@ func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { } token := obj.SessionToken() - ownerID := *obj.OwnerID() + ownerID := obj.OwnerID() if token == nil || !token.AssertAuthKey(&key) { return v.checkOwnerKey(ownerID, key) @@ -412,7 +412,7 @@ func (v *FormatValidator) checkAttributes(obj *objectSDK.Object) error { var errIncorrectOwner = errors.New("incorrect object owner") func (v *FormatValidator) checkOwner(obj *objectSDK.Object) error { - if idOwner := obj.OwnerID(); idOwner == nil || len(idOwner.WalletBytes()) == 0 { + if idOwner := obj.OwnerID(); idOwner.IsEmpty() { return errIncorrectOwner } diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index 2a5b5690c..a8901ad6d 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -32,7 +32,7 @@ func blankValidObject(key *ecdsa.PrivateKey) *objectSDK.Object { obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) - obj.SetOwnerID(&idOwner) + obj.SetOwnerID(idOwner) return obj } @@ -107,7 +107,7 @@ func TestFormatValidator_Validate(t *testing.T) { obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) obj.SetSessionToken(tok) - obj.SetOwnerID(&idOwner) + obj.SetOwnerID(idOwner) require.NoError(t, objectSDK.SetIDWithSignature(ownerKey.PrivateKey, obj)) @@ -303,7 +303,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { obj := objectSDK.New() obj.SetContainerID(cidtest.ID()) obj.SetSessionToken(tok) - obj.SetOwnerID(&owner) + obj.SetOwnerID(owner) require.NoError(t, objectSDK.SetIDWithSignature(signer.PrivateKey, obj)) require.NoError(t, v.Validate(context.Background(), obj, false)) @@ -352,7 +352,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { obj := objectSDK.New() obj.SetContainerID(cnrID) obj.SetSessionToken(tok) - obj.SetOwnerID(&owner) + obj.SetOwnerID(owner) require.NoError(t, objectSDK.SetIDWithSignature(signer.PrivateKey, obj)) require.NoError(t, v.Validate(context.Background(), obj, false)) @@ -386,7 +386,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { obj := objectSDK.New() obj.SetContainerID(cnrID) obj.SetSessionToken(tok) - obj.SetOwnerID(&owner) + obj.SetOwnerID(owner) require.NoError(t, objectSDK.SetIDWithSignature(signer.PrivateKey, obj)) v := NewFormatValidator( @@ -459,7 +459,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { obj := objectSDK.New() obj.SetContainerID(cnrID) obj.SetSessionToken(tok) - obj.SetOwnerID(&owner) + obj.SetOwnerID(owner) require.NoError(t, objectSDK.SetIDWithSignature(signer.PrivateKey, obj)) v := NewFormatValidator( @@ -535,7 +535,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { obj := objectSDK.New() obj.SetContainerID(cnrID) obj.SetSessionToken(tok) - obj.SetOwnerID(&owner) + obj.SetOwnerID(owner) require.NoError(t, objectSDK.SetIDWithSignature(signer.PrivateKey, obj)) v := NewFormatValidator( diff --git a/pkg/services/object/acl/acl_test.go b/pkg/services/object/acl/acl_test.go index b9b82dac8..d63cb1285 100644 --- a/pkg/services/object/acl/acl_test.go +++ b/pkg/services/object/acl/acl_test.go @@ -39,14 +39,14 @@ func TestStickyCheck(t *testing.T) { info.SetSenderKey(make([]byte, 33)) // any non-empty key info.SetRequestRole(acl.RoleContainer) - require.True(t, checker.StickyBitCheck(info, *usertest.ID())) + require.True(t, checker.StickyBitCheck(info, usertest.ID())) var basicACL acl.Basic basicACL.MakeSticky() info.SetBasicACL(basicACL) - require.True(t, checker.StickyBitCheck(info, *usertest.ID())) + require.True(t, checker.StickyBitCheck(info, usertest.ID())) }) t.Run("owner ID and/or public key emptiness", func(t *testing.T) { @@ -72,7 +72,7 @@ func TestStickyCheck(t *testing.T) { var ownerID user.ID if withOwner { - ownerID = *usertest.ID() + ownerID = usertest.ID() } require.Equal(t, expected, checker.StickyBitCheck(info, ownerID)) diff --git a/pkg/services/object/acl/eacl/v2/object.go b/pkg/services/object/acl/eacl/v2/object.go index 690e4aa70..72bd4c2d2 100644 --- a/pkg/services/object/acl/eacl/v2/object.go +++ b/pkg/services/object/acl/eacl/v2/object.go @@ -62,8 +62,8 @@ func headersFromObject(obj *objectSDK.Object, cnr cid.ID, oid *oid.ID) []eaclSDK res = append(res, oidHeader(*oid)) } - if idOwner := obj.OwnerID(); idOwner != nil { - res = append(res, ownerIDHeader(*idOwner)) + if idOwner := obj.OwnerID(); !idOwner.IsEmpty() { + res = append(res, ownerIDHeader(idOwner)) } cs, ok := obj.PayloadChecksum() diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index aac8c8860..d48f8ab7b 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -176,11 +176,11 @@ func (exec *execCtx) initTombstoneObject() error { tokenSession := exec.commonParameters().SessionToken() if tokenSession != nil { issuer := tokenSession.Issuer() - exec.tombstoneObj.SetOwnerID(&issuer) + exec.tombstoneObj.SetOwnerID(issuer) } else { // make local node a tombstone object owner localUser := exec.svc.netInfo.LocalNodeID() - exec.tombstoneObj.SetOwnerID(&localUser) + exec.tombstoneObj.SetOwnerID(localUser) } var a objectSDK.Attribute diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 10f932849..a9abe407d 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -112,7 +112,7 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { // If it isn't owner key, replication attempts will fail, thus this check. if sToken == nil { ownerObj := prm.hdr.OwnerID() - if ownerObj == nil { + if ownerObj.IsEmpty() { return errors.New("missing object owner") } diff --git a/pkg/services/object/util/key_test.go b/pkg/services/object/util/key_test.go index 76e320e0c..cb7ddfde5 100644 --- a/pkg/services/object/util/key_test.go +++ b/pkg/services/object/util/key_test.go @@ -25,7 +25,7 @@ func TestNewKeyStorage(t *testing.T) { tokenStor := tokenStorage.NewTokenStore() stor := util.NewKeyStorage(&nodeKey.PrivateKey, tokenStor, mockedNetworkState{42}) - owner := *usertest.ID() + owner := usertest.ID() t.Run("node key", func(t *testing.T) { key, err := stor.GetKey(nil) @@ -36,7 +36,7 @@ func TestNewKeyStorage(t *testing.T) { t.Run("unknown token", func(t *testing.T) { _, err = stor.GetKey(&util.SessionInfo{ ID: uuid.New(), - Owner: *usertest.ID(), + Owner: usertest.ID(), }) require.Error(t, err) }) diff --git a/pkg/services/session/storage/persistent/executor_test.go b/pkg/services/session/storage/persistent/executor_test.go index ee0c8f915..39cb14359 100644 --- a/pkg/services/session/storage/persistent/executor_test.go +++ b/pkg/services/session/storage/persistent/executor_test.go @@ -22,7 +22,7 @@ func TestTokenStore(t *testing.T) { defer ts.Close() - owner := *usertest.ID() + owner := usertest.ID() var ownerV2 refs.OwnerID owner.WriteToV2(&ownerV2) @@ -66,7 +66,7 @@ func TestTokenStore_Persistent(t *testing.T) { ts, err := NewTokenStore(path) require.NoError(t, err) - idOwner := *usertest.ID() + idOwner := usertest.ID() var idOwnerV2 refs.OwnerID idOwner.WriteToV2(&idOwnerV2) @@ -127,7 +127,7 @@ func TestTokenStore_RemoveOld(t *testing.T) { defer ts.Close() - owner := *usertest.ID() + owner := usertest.ID() var ownerV2 refs.OwnerID owner.WriteToV2(&ownerV2) From 5048236441b2faf8b9b411fc9ed246421659324f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 15 Nov 2023 13:33:30 +0300 Subject: [PATCH 0696/1943] [#808] go.mod: Update dependencies Signed-off-by: Evgenii Stratonikov --- go.mod | 10 +++++----- go.sum | 16 ++++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 2714e1567..cfa6dd7a6 100644 --- a/go.mod +++ b/go.mod @@ -27,11 +27,11 @@ require ( github.com/paulmach/orb v0.9.2 github.com/prometheus/client_golang v1.16.0 github.com/spf13/cast v1.5.1 - github.com/spf13/cobra v1.7.0 + github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 - go.etcd.io/bbolt v1.3.7 + go.etcd.io/bbolt v1.3.8 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.26.0 @@ -63,7 +63,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/dgraph-io/badger/v4 v4.1.0 @@ -124,8 +124,8 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 7660d9dcd..8c3689023 100644 --- a/go.sum +++ b/go.sum @@ -1111,8 +1111,9 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -2028,8 +2029,8 @@ github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -2148,8 +2149,9 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= +go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= @@ -2686,8 +2688,9 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2726,8 +2729,9 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From e81a58b8da3c5b606e892b2927feeeed939dfa55 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 15 Nov 2023 14:26:23 +0300 Subject: [PATCH 0697/1943] [#808] cli: Use MarkFlagsOneRequired after cobra update Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/ir_remove_container.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/ir_remove_container.go b/cmd/frostfs-cli/modules/control/ir_remove_container.go index 52eaa574d..a66d7e06d 100644 --- a/cmd/frostfs-cli/modules/control/ir_remove_container.go +++ b/cmd/frostfs-cli/modules/control/ir_remove_container.go @@ -1,8 +1,6 @@ package control import ( - "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -35,6 +33,7 @@ func initControlIRRemoveContainerCmd() { flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) flags.String(ownerFlag, "", "Container owner's wallet address.") removeContainerCmd.MarkFlagsMutuallyExclusive(commonflags.CIDFlag, ownerFlag) + removeContainerCmd.MarkFlagsOneRequired(commonflags.CIDFlag, ownerFlag) } func removeContainer(cmd *cobra.Command, _ []string) { @@ -74,10 +73,6 @@ func prepareRemoveContainerRequest(cmd *cobra.Command) *ircontrol.RemoveContaine ownerStr, err := cmd.Flags().GetString(ownerFlag) commonCmd.ExitOnErr(cmd, "failed to get owner: ", err) - if len(ownerStr) == 0 && len(cidStr) == 0 { - commonCmd.ExitOnErr(cmd, "invalid usage: %w", errors.New("neither owner's wallet address nor container ID are specified")) - } - if len(ownerStr) > 0 { var owner user.ID commonCmd.ExitOnErr(cmd, "invalid owner ID: %w", owner.DecodeString(ownerStr)) From 5521737f0b4cf5bb1ae22bfcfb3e7539617e8fd0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 15 Nov 2023 14:31:37 +0300 Subject: [PATCH 0698/1943] [#808] cli: Use EnableTraverseRunHooks in cobra Adopt EnableTraverseRunHooks to get rid of tracing boilerplate in multiple commands. Now adding `--trace` flag is sufficient for a command to support tracing. Finally, it looks how it _should_. Refs https://git.frostfs.info/TrueCloudLab/frostfs-node/pulls/406 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/accounting/root.go | 3 --- cmd/frostfs-cli/modules/container/root.go | 3 --- cmd/frostfs-cli/modules/netmap/root.go | 3 --- cmd/frostfs-cli/modules/object/root.go | 3 --- cmd/frostfs-cli/modules/root.go | 5 +++++ cmd/frostfs-cli/modules/session/create.go | 3 --- 6 files changed, 5 insertions(+), 15 deletions(-) diff --git a/cmd/frostfs-cli/modules/accounting/root.go b/cmd/frostfs-cli/modules/accounting/root.go index f94488b6f..8ab8aa125 100644 --- a/cmd/frostfs-cli/modules/accounting/root.go +++ b/cmd/frostfs-cli/modules/accounting/root.go @@ -1,7 +1,6 @@ package accounting import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -18,9 +17,7 @@ var Cmd = &cobra.Command{ _ = viper.BindPFlag(commonflags.WalletPath, flags.Lookup(commonflags.WalletPath)) _ = viper.BindPFlag(commonflags.Account, flags.Lookup(commonflags.Account)) _ = viper.BindPFlag(commonflags.RPC, flags.Lookup(commonflags.RPC)) - common.StartClientCommandSpan(cmd) }, - PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/container/root.go b/cmd/frostfs-cli/modules/container/root.go index f3c3e0e3a..ab7f5fb90 100644 --- a/cmd/frostfs-cli/modules/container/root.go +++ b/cmd/frostfs-cli/modules/container/root.go @@ -1,7 +1,6 @@ package container import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -16,9 +15,7 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) - common.StartClientCommandSpan(cmd) }, - PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/netmap/root.go b/cmd/frostfs-cli/modules/netmap/root.go index 006ac6d9f..aaa83f12f 100644 --- a/cmd/frostfs-cli/modules/netmap/root.go +++ b/cmd/frostfs-cli/modules/netmap/root.go @@ -1,7 +1,6 @@ package netmap import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -15,9 +14,7 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) - common.StartClientCommandSpan(cmd) }, - PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/object/root.go b/cmd/frostfs-cli/modules/object/root.go index d9d4bb84d..e04366f4d 100644 --- a/cmd/frostfs-cli/modules/object/root.go +++ b/cmd/frostfs-cli/modules/object/root.go @@ -1,7 +1,6 @@ package object import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "github.com/spf13/cobra" ) @@ -16,9 +15,7 @@ var Cmd = &cobra.Command{ // the viper before execution commonflags.Bind(cmd) commonflags.BindAPI(cmd) - common.StartClientCommandSpan(cmd) }, - PersistentPostRun: common.StopClientCommandSpan, } func init() { diff --git a/cmd/frostfs-cli/modules/root.go b/cmd/frostfs-cli/modules/root.go index 808bd6d07..87e6a905f 100644 --- a/cmd/frostfs-cli/modules/root.go +++ b/cmd/frostfs-cli/modules/root.go @@ -46,6 +46,10 @@ of frostfs-api and some useful utilities for compiling ACL rules from JSON notation, managing container access through protocol gates, querying network map and much more!`, Run: entryPoint, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + common.StartClientCommandSpan(cmd) + }, + PersistentPostRun: common.StopClientCommandSpan, } // Execute adds all child commands to the root command and sets flags appropriately. @@ -57,6 +61,7 @@ func Execute() { func init() { cobra.OnInitialize(initConfig) + cobra.EnableTraverseRunHooks = true // use stdout as default output for cmd.Print() rootCmd.SetOut(os.Stdout) diff --git a/cmd/frostfs-cli/modules/session/create.go b/cmd/frostfs-cli/modules/session/create.go index 94ae595a8..e92e4baa6 100644 --- a/cmd/frostfs-cli/modules/session/create.go +++ b/cmd/frostfs-cli/modules/session/create.go @@ -6,7 +6,6 @@ import ( "os" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -33,9 +32,7 @@ var createCmd = &cobra.Command{ PersistentPreRun: func(cmd *cobra.Command, args []string) { _ = viper.BindPFlag(commonflags.WalletPath, cmd.Flags().Lookup(commonflags.WalletPath)) _ = viper.BindPFlag(commonflags.Account, cmd.Flags().Lookup(commonflags.Account)) - common.StartClientCommandSpan(cmd) }, - PersistentPostRun: common.StopClientCommandSpan, } func init() { From 306f12e6c5eab0cc9684efe04b267572b6ac60e8 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 28 Nov 2023 10:32:56 +0300 Subject: [PATCH 0699/1943] [#828] adm: Fix policy contract deploy Signed-off-by: Airat Arifullin --- cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 5183114f9..4ba72ef41 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -575,8 +575,10 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an c.Contracts[containerContract].Hash, keysParam, configParam) - case proxyContract, policyContract: + case proxyContract: items = nil + case policyContract: + items = []any{nil} default: panic(fmt.Sprintf("invalid contract name: %s", ctrName)) } From b215817e14698d5dc48f356160ba7aa83b502f62 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 28 Nov 2023 10:12:39 +0300 Subject: [PATCH 0700/1943] [#741] treesvc: Remove unused height variables Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + pkg/services/tree/sync.go | 80 ++++++++++++++++++--------------------- 2 files changed, 37 insertions(+), 44 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index e8472357c..fad5f464d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -64,6 +64,7 @@ const ( TreeCouldNotUpdateLastSynchronizedHeightForATree = "could not update last synchronized height for a tree" TreeSynchronizeTree = "synchronize tree" TreeFailedToRunTreeSynchronizationOverAllNodes = "failed to run tree synchronization over all nodes" + TreeFailedToRunTreeSynchronizationForSpecificNode = "failed to run tree synchronization for specific node" TreeSyncingTrees = "syncing trees..." TreeCouldNotFetchContainers = "could not fetch containers" TreeTreesHaveBeenSynchronized = "trees have been synchronized" diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index f5a7fbce6..6dc3ffb29 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -217,44 +217,41 @@ func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID s func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, height uint64, treeClient TreeServiceClient, opsCh chan<- *pilorama.Move, -) (uint64, error) { +) error { rawCID := make([]byte, sha256.Size) cid.Encode(rawCID) - for { - newHeight := height - req := &GetOpLogRequest{ - Body: &GetOpLogRequest_Body{ - ContainerId: rawCID, - TreeId: treeID, - Height: newHeight, - }, - } - if err := SignMessage(req, s.key); err != nil { - return 0, err - } - - c, err := treeClient.GetOpLog(ctx, req) - if err != nil { - return 0, fmt.Errorf("can't initialize client: %w", err) - } - res, err := c.Recv() - for ; err == nil; res, err = c.Recv() { - lm := res.GetBody().GetOperation() - m := &pilorama.Move{ - Parent: lm.ParentId, - Child: lm.ChildId, - } - if err := m.Meta.FromBytes(lm.Meta); err != nil { - return 0, err - } - opsCh <- m - } - if height == newHeight || err != nil && !errors.Is(err, io.EOF) { - return newHeight, err - } - height = newHeight + req := &GetOpLogRequest{ + Body: &GetOpLogRequest_Body{ + ContainerId: rawCID, + TreeId: treeID, + Height: height, + }, } + if err := SignMessage(req, s.key); err != nil { + return err + } + + c, err := treeClient.GetOpLog(ctx, req) + if err != nil { + return fmt.Errorf("can't initialize client: %w", err) + } + res, err := c.Recv() + for ; err == nil; res, err = c.Recv() { + lm := res.GetBody().GetOperation() + m := &pilorama.Move{ + Parent: lm.ParentId, + Child: lm.ChildId, + } + if err := m.Meta.FromBytes(lm.Meta); err != nil { + return err + } + opsCh <- m + } + if err != nil && !errors.Is(err, io.EOF) { + return err + } + return nil } // synchronizeTree synchronizes operations getting them from different nodes. @@ -291,7 +288,6 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, i := i n := n errGroup.Go(func() error { - height := from n.IterateNetworkEndpoints(func(addr string) bool { var a network.Address if err := a.FromString(addr); err != nil { @@ -315,16 +311,12 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, defer cc.Close() treeClient := NewTreeServiceClient(cc) - for { - h, err := s.startStream(egCtx, cid, treeID, from, treeClient, nodeOperationStreams[i]) - if height < h { - height = h - } - if err != nil || h <= height { - // Error with the response, try the next node. - return true - } + err = s.startStream(egCtx, cid, treeID, from, treeClient, nodeOperationStreams[i]) + if err != nil { + // Error with the response, try the next node. + s.log.Warn(logs.TreeFailedToRunTreeSynchronizationForSpecificNode, zap.Error(err), zap.String("address", addr)) } + return true }) close(nodeOperationStreams[i]) return nil From b21be1abdd3125e715c222ce4ffc1f26a6d6bba9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 28 Nov 2023 10:51:15 +0300 Subject: [PATCH 0701/1943] [#741] treesvc: Do not update sync height if some node is unavailable Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/sync.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 6dc3ffb29..65920fdc0 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -9,6 +9,7 @@ import ( "math" "math/rand" "sync" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -284,10 +285,14 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return nil }) + var allNodesSynced atomic.Bool + allNodesSynced.Store(true) + for i, n := range nodes { i := i n := n errGroup.Go(func() error { + var nodeSynced bool n.IterateNetworkEndpoints(func(addr string) bool { var a network.Address if err := a.FromString(addr); err != nil { @@ -316,13 +321,18 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, // Error with the response, try the next node. s.log.Warn(logs.TreeFailedToRunTreeSynchronizationForSpecificNode, zap.Error(err), zap.String("address", addr)) } + nodeSynced = err == nil return true }) close(nodeOperationStreams[i]) + if !nodeSynced { + allNodesSynced.Store(false) + } return nil }) } if err := errGroup.Wait(); err != nil { + allNodesSynced.Store(false) s.log.Warn(logs.TreeFailedToRunTreeSynchronizationOverAllNodes, zap.Error(err)) } @@ -332,7 +342,10 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, } else { newHeight++ } - return newHeight + if allNodesSynced.Load() { + return newHeight + } + return from } // ErrAlreadySyncing is returned when a service synchronization has already From 26e4f7005c35f5b89642483bd94f8fe45277b377 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 28 Nov 2023 17:04:02 +0300 Subject: [PATCH 0702/1943] [#741] treesvc: Refactor tree sync Fix linter issues. Add error logging. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 2 ++ pkg/services/tree/sync.go | 35 ++++++++++++++++++++--------------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index fad5f464d..a72609b41 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -65,6 +65,8 @@ const ( TreeSynchronizeTree = "synchronize tree" TreeFailedToRunTreeSynchronizationOverAllNodes = "failed to run tree synchronization over all nodes" TreeFailedToRunTreeSynchronizationForSpecificNode = "failed to run tree synchronization for specific node" + TreeFailedToParseAddressForTreeSynchronization = "failed to parse address for tree synchronization" + TreeFailedToConnectForTreeSynchronization = "failed to connect for tree synchronization" TreeSyncingTrees = "syncing trees..." TreeCouldNotFetchContainers = "could not fetch containers" TreeTreesHaveBeenSynchronized = "trees have been synchronized" diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 65920fdc0..485278adf 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -217,8 +217,10 @@ func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID s } func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, - height uint64, treeClient TreeServiceClient, opsCh chan<- *pilorama.Move, + height uint64, cc *grpc.ClientConn, opsCh chan<- *pilorama.Move, ) error { + treeClient := NewTreeServiceClient(cc) + rawCID := make([]byte, sha256.Size) cid.Encode(rawCID) @@ -296,29 +298,19 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, n.IterateNetworkEndpoints(func(addr string) bool { var a network.Address if err := a.FromString(addr); err != nil { + s.log.Warn(logs.TreeFailedToParseAddressForTreeSynchronization, zap.Error(err), zap.String("address", addr)) return false } - cc, err := grpc.DialContext(egCtx, a.URIAddr(), - grpc.WithChainUnaryInterceptor( - metrics.NewUnaryClientInterceptor(), - tracing_grpc.NewUnaryClientInteceptor(), - ), - grpc.WithChainStreamInterceptor( - metrics.NewStreamClientInterceptor(), - tracing_grpc.NewStreamClientInterceptor(), - ), - grpc.WithTransportCredentials(insecure.NewCredentials())) + cc, err := s.dialCtx(egCtx, a) if err != nil { - // Failed to connect, try the next address. + s.log.Warn(logs.TreeFailedToConnectForTreeSynchronization, zap.Error(err), zap.String("address", addr)) return false } defer cc.Close() - treeClient := NewTreeServiceClient(cc) - err = s.startStream(egCtx, cid, treeID, from, treeClient, nodeOperationStreams[i]) + err = s.startStream(egCtx, cid, treeID, from, cc, nodeOperationStreams[i]) if err != nil { - // Error with the response, try the next node. s.log.Warn(logs.TreeFailedToRunTreeSynchronizationForSpecificNode, zap.Error(err), zap.String("address", addr)) } nodeSynced = err == nil @@ -348,6 +340,19 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return from } +func (*Service) dialCtx(egCtx context.Context, a network.Address) (*grpc.ClientConn, error) { + return grpc.DialContext(egCtx, a.URIAddr(), + grpc.WithChainUnaryInterceptor( + metrics.NewUnaryClientInterceptor(), + tracing_grpc.NewUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + metrics.NewStreamClientInterceptor(), + tracing_grpc.NewStreamClientInterceptor(), + ), + grpc.WithTransportCredentials(insecure.NewCredentials())) +} + // ErrAlreadySyncing is returned when a service synchronization has already // been started. var ErrAlreadySyncing = errors.New("service is being synchronized") From 7f6852bbd2f440b5c49b616966e96bfd5ff30c65 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 30 Nov 2023 11:45:31 +0300 Subject: [PATCH 0703/1943] [#639] node: Refactor TTL cache Migrate from internal to external TTL implementation Signed-off-by: Alexander Chuprov --- cmd/frostfs-node/cache.go | 29 ++++++------------ cmd/frostfs-node/cache_test.go | 56 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 cmd/frostfs-node/cache_test.go diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index cd383a5c2..bf68d245f 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -12,38 +12,29 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" lru "github.com/hashicorp/golang-lru/v2" + "github.com/hashicorp/golang-lru/v2/expirable" ) type netValueReader[K any, V any] func(K) (V, error) -type valueWithTime[V any] struct { +type valueWithError[V any] struct { v V - t time.Time // cached error in order to not repeat failed request for some time e error } // entity that provides TTL cache interface. type ttlNetCache[K comparable, V any] struct { - ttl time.Duration - - sz int - - cache *lru.Cache[K, *valueWithTime[V]] - - netRdr netValueReader[K, V] - + cache *expirable.LRU[K, *valueWithError[V]] + netRdr netValueReader[K, V] keyLocker *utilSync.KeyLocker[K] } // complicates netValueReader with TTL caching mechanism. func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr netValueReader[K, V]) *ttlNetCache[K, V] { - cache, err := lru.New[K, *valueWithTime[V]](sz) - fatalOnErr(err) + cache := expirable.NewLRU[K, *valueWithError[V]](sz, nil, ttl) return &ttlNetCache[K, V]{ - ttl: ttl, - sz: sz, cache: cache, netRdr: netRdr, keyLocker: utilSync.NewKeyLocker[K](), @@ -57,7 +48,7 @@ func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr n // returned value should not be modified. func (c *ttlNetCache[K, V]) get(key K) (V, error) { val, ok := c.cache.Peek(key) - if ok && time.Since(val.t) < c.ttl { + if ok { return val.v, val.e } @@ -65,15 +56,14 @@ func (c *ttlNetCache[K, V]) get(key K) (V, error) { defer c.keyLocker.Unlock(key) val, ok = c.cache.Peek(key) - if ok && time.Since(val.t) < c.ttl { + if ok { return val.v, val.e } v, err := c.netRdr(key) - c.cache.Add(key, &valueWithTime[V]{ + c.cache.Add(key, &valueWithError[V]{ v: v, - t: time.Now(), e: err, }) @@ -84,9 +74,8 @@ func (c *ttlNetCache[K, V]) set(k K, v V, e error) { c.keyLocker.Lock(k) defer c.keyLocker.Unlock(k) - c.cache.Add(k, &valueWithTime[V]{ + c.cache.Add(k, &valueWithError[V]{ v: v, - t: time.Now(), e: e, }) } diff --git a/cmd/frostfs-node/cache_test.go b/cmd/frostfs-node/cache_test.go new file mode 100644 index 000000000..6e076abfc --- /dev/null +++ b/cmd/frostfs-node/cache_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestTTLNetCache(t *testing.T) { + ttlDuration := time.Millisecond * 50 + cache := newNetworkTTLCache[string, time.Time](10, ttlDuration, testNetValueReader) + + key := "key" + + t.Run("Test Add and Get", func(t *testing.T) { + ti := time.Now() + cache.set(key, ti, nil) + val, err := cache.get(key) + require.NoError(t, err) + require.Equal(t, ti, val) + }) + + t.Run("Test TTL", func(t *testing.T) { + ti := time.Now() + cache.set(key, ti, nil) + time.Sleep(2 * ttlDuration) + val, err := cache.get(key) + require.NoError(t, err) + require.NotEqual(t, val, ti) + }) + + t.Run("Test Remove", func(t *testing.T) { + ti := time.Now() + cache.set(key, ti, nil) + cache.remove(key) + val, err := cache.get(key) + require.NoError(t, err) + require.NotEqual(t, val, ti) + }) + + t.Run("Test Cache Error", func(t *testing.T) { + cache.set("error", time.Now(), errors.New("mock error")) + _, err := cache.get("error") + require.Error(t, err) + require.Equal(t, "mock error", err.Error()) + }) +} + +func testNetValueReader(key string) (time.Time, error) { + if key == "error" { + return time.Now(), errors.New("mock error") + } + return time.Now(), nil +} From a982c3df18194f6cad2d5289194afaf364ece262 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 21 Nov 2023 17:01:14 +0300 Subject: [PATCH 0704/1943] [#824] cli: Support passing chain ID in add-rule command Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/control/add_rule.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index d7177dd96..f57823ed3 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -41,6 +41,8 @@ func prettyJSONFormat(cmd *cobra.Command, serializedChain []byte) string { func addRule(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) + chainID, _ := cmd.Flags().GetString(chainIDFlag) + var cnr cid.ID cidStr, _ := cmd.Flags().GetString(commonflags.CIDFlag) commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(cidStr)) @@ -52,6 +54,7 @@ func addRule(cmd *cobra.Command, _ []string) { chain := new(apechain.Chain) commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, []string{rule})) + chain.ID = apechain.ID(chainID) serializedChain := chain.Bytes() cmd.Println("Container ID: " + cidStr) @@ -87,4 +90,5 @@ func initControlAddRuleCmd() { ff := addRuleCmd.Flags() ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) ff.String(ruleFlag, "", "Rule statement") + ff.String(chainIDFlag, "", "Assign ID to the parsed chain") } From 2302e5d34222ee0900909dc6616db32b22a132aa Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 29 Nov 2023 10:46:46 +0300 Subject: [PATCH 0705/1943] [#651] shard/test: Refactor Shard_Delete Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/shard/delete_test.go | 98 +++++++------------ 1 file changed, 37 insertions(+), 61 deletions(-) diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index 910528f84..846cb0e3b 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -12,88 +12,64 @@ import ( "github.com/stretchr/testify/require" ) -func TestShard_Delete(t *testing.T) { - t.Parallel() - - t.Run("without write cache", func(t *testing.T) { +func TestShard_Delete_SmallObject(t *testing.T) { + t.Run("small object without write cache", func(t *testing.T) { t.Parallel() - testShardDelete(t, false) + testShard(t, false, 1<<5) }) - t.Run("with write cache", func(t *testing.T) { + t.Run("small object with write cache", func(t *testing.T) { t.Parallel() - testShardDelete(t, true) + testShard(t, true, 1<<5) }) } -func testShardDelete(t *testing.T, hasWriteCache bool) { +func TestShard_Delete_BigObject(t *testing.T) { + t.Run("big object without write cache", func(t *testing.T) { + t.Parallel() + testShard(t, false, 1<<20) + }) + + t.Run("big object with write cache", func(t *testing.T) { + t.Parallel() + testShard(t, true, 1<<20) + }) +} + +func testShard(t *testing.T, hasWriteCache bool, payloadSize int) { sh := newShard(t, hasWriteCache) cnr := cidtest.ID() obj := testutil.GenerateObjectWithCID(cnr) testutil.AddAttribute(obj, "foo", "bar") + testutil.AddPayload(obj, payloadSize) var putPrm PutPrm + putPrm.SetObject(obj) + var getPrm GetPrm + getPrm.SetAddress(object.AddressOf(obj)) - t.Run("big object", func(t *testing.T) { - testutil.AddPayload(obj, 1<<20) + var delPrm DeletePrm + delPrm.SetAddresses(object.AddressOf(obj)) - putPrm.SetObject(obj) - getPrm.SetAddress(object.AddressOf(obj)) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) - var delPrm DeletePrm - delPrm.SetAddresses(object.AddressOf(obj)) + _, err = testGet(t, sh, getPrm, hasWriteCache) + require.NoError(t, err) - _, err := sh.Put(context.Background(), putPrm) - require.NoError(t, err) - - _, err = testGet(t, sh, getPrm, hasWriteCache) - require.NoError(t, err) - - if hasWriteCache { - require.Eventually(t, func() bool { - _, err = sh.Delete(context.Background(), delPrm) - return err == nil - }, 30*time.Second, 100*time.Millisecond) - } else { + if hasWriteCache { + require.Eventually(t, func() bool { _, err = sh.Delete(context.Background(), delPrm) - require.NoError(t, err) - } - - _, err = sh.Get(context.Background(), getPrm) - require.True(t, client.IsErrObjectNotFound(err)) - }) - - t.Run("small object", func(t *testing.T) { - obj := testutil.GenerateObjectWithCID(cnr) - testutil.AddAttribute(obj, "foo", "bar") - testutil.AddPayload(obj, 1<<5) - - putPrm.SetObject(obj) - getPrm.SetAddress(object.AddressOf(obj)) - - var delPrm DeletePrm - delPrm.SetAddresses(object.AddressOf(obj)) - - _, err := sh.Put(context.Background(), putPrm) + return err == nil + }, 30*time.Second, 10*time.Millisecond) + } else { + _, err = sh.Delete(context.Background(), delPrm) require.NoError(t, err) + } - _, err = sh.Get(context.Background(), getPrm) - require.NoError(t, err) - - if hasWriteCache { - require.Eventually(t, func() bool { - _, err = sh.Delete(context.Background(), delPrm) - return err == nil - }, 10*time.Second, 100*time.Millisecond) - } else { - _, err = sh.Delete(context.Background(), delPrm) - require.NoError(t, err) - } - - _, err = sh.Get(context.Background(), getPrm) - require.True(t, client.IsErrObjectNotFound(err)) - }) + _, err = sh.Get(context.Background(), getPrm) + require.True(t, client.IsErrObjectNotFound(err)) } From 445ebcc0e7170c9c9c78b3b9ccc1da1eaced6a91 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 29 Nov 2023 10:49:47 +0300 Subject: [PATCH 0706/1943] [#651] shard/test: Speedup Shard_Delete Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/shard/delete_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index 846cb0e3b..f7687e961 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -61,6 +61,7 @@ func testShard(t *testing.T, hasWriteCache bool, payloadSize int) { require.NoError(t, err) if hasWriteCache { + sh.FlushWriteCache(context.Background(), FlushWriteCachePrm{ignoreErrors: false}) require.Eventually(t, func() bool { _, err = sh.Delete(context.Background(), delPrm) return err == nil From b2c63e57ba101b97c8484d5d7406667b2a545d73 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 29 Nov 2023 10:51:56 +0300 Subject: [PATCH 0707/1943] [#651] engine/test: Speedup StorageEngine_Inhume Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/engine/inhume_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 8fff6280c..4bb128bd7 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -36,6 +36,7 @@ func TestStorageEngine_Inhume(t *testing.T) { link.SetSplitID(splitID) t.Run("delete small object", func(t *testing.T) { + t.Parallel() e := testNewEngine(t).setShardsNum(t, 1).engine defer e.Close(context.Background()) @@ -54,6 +55,7 @@ func TestStorageEngine_Inhume(t *testing.T) { }) t.Run("delete big object", func(t *testing.T) { + t.Parallel() s1 := testNewShard(t, 1) s2 := testNewShard(t, 2) From f1db468d4844a084be154c1fbeb81c74ac0a410e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 4 Dec 2023 16:12:59 +0300 Subject: [PATCH 0708/1943] [#840] adm: Update FrostFS ID deploy arguments Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/frostfsid_util.go | 49 ++++++++----------- .../modules/morph/frostfsid_util_test.go | 36 ++++++++------ .../modules/morph/initialize_deploy.go | 10 ++-- 3 files changed, 47 insertions(+), 48 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go index 248ad76b0..cfe3c1ee9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go @@ -5,38 +5,29 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" - "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/viper" ) -// neo-go doesn't support []util.Uint160 type: -// https://github.com/nspcc-dev/neo-go/blob/v0.103.0/pkg/smartcontract/parameter.go#L262 -// Thus, return []smartcontract.Parameter. -func getFrostfsIDAuthorizedKeys(v *viper.Viper, defaultOwner util.Uint160) ([]smartcontract.Parameter, error) { - var res []smartcontract.Parameter - res = append(res, smartcontract.Parameter{Type: smartcontract.Hash160Type, Value: defaultOwner}) - - ks := v.GetStringSlice(frostfsIDAuthorizedKeysConfigKey) - for i := range ks { - h, err := address.StringToUint160(ks[i]) - if err == nil { - res = append(res, smartcontract.Parameter{Type: smartcontract.Hash160Type, Value: h}) - continue - } - - h, err = util.Uint160DecodeStringLE(ks[i]) - if err == nil { - res = append(res, smartcontract.Parameter{Type: smartcontract.Hash160Type, Value: h}) - continue - } - - pk, err := keys.NewPublicKeyFromString(ks[i]) - if err == nil { - res = append(res, smartcontract.Parameter{Type: smartcontract.Hash160Type, Value: pk.GetScriptHash()}) - continue - } - return nil, fmt.Errorf("frostfsid: #%d item in authorized_keys is invalid: '%s'", i, ks[i]) +func getFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) { + admin := v.GetString(frostfsIDAdminConfigKey) + if admin == "" { + return util.Uint160{}, false, nil } - return res, nil + + h, err := address.StringToUint160(admin) + if err == nil { + return h, true, nil + } + + h, err = util.Uint160DecodeStringLE(admin) + if err == nil { + return h, true, nil + } + + pk, err := keys.NewPublicKeyFromString(admin) + if err == nil { + return pk.GetScriptHash(), true, nil + } + return util.Uint160{}, true, fmt.Errorf("frostfsid: admin is invalid: '%s'", admin) } diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go index f4b6a66f6..192dc9f12 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go @@ -6,8 +6,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" - "github.com/nspcc-dev/neo-go/pkg/smartcontract" - "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -20,30 +18,36 @@ func TestFrostfsIDConfig(t *testing.T) { pks[i] = pk } - v := viper.New() - v.Set("frostfsid.authorized_keys", []string{ + fmts := []string{ pks[0].GetScriptHash().StringLE(), address.Uint160ToString(pks[1].GetScriptHash()), hex.EncodeToString(pks[2].PublicKey().UncompressedBytes()), hex.EncodeToString(pks[3].PublicKey().Bytes()), - }) + } - comm := util.Uint160{1, 2, 3} - actual, err := getFrostfsIDAuthorizedKeys(v, comm) - require.NoError(t, err) - require.Equal(t, len(pks)+1, len(actual)) - require.Equal(t, smartcontract.Hash160Type, actual[0].Type) - require.Equal(t, comm, actual[0].Value) - for i := range pks { - require.Equal(t, smartcontract.Hash160Type, actual[i+1].Type) - require.Equal(t, pks[i].GetScriptHash(), actual[i+1].Value) + for i := range fmts { + v := viper.New() + v.Set("frostfsid.admin", fmts[i]) + + actual, found, err := getFrostfsIDAdmin(v) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, pks[i].GetScriptHash(), actual) } t.Run("bad key", func(t *testing.T) { v := viper.New() - v.Set("frostfsid.authorized_keys", []string{"abc"}) + v.Set("frostfsid.admin", "abc") - _, err := getFrostfsIDAuthorizedKeys(v, comm) + _, found, err := getFrostfsIDAdmin(v) require.Error(t, err) + require.True(t, found) + }) + t.Run("missing key", func(t *testing.T) { + v := viper.New() + + _, found, err := getFrostfsIDAdmin(v) + require.NoError(t, err) + require.False(t, found) }) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 4ba72ef41..533fcad4d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -47,7 +47,7 @@ const ( proxyContract = "proxy" ) -const frostfsIDAuthorizedKeysConfigKey = "frostfsid.authorized_keys" +const frostfsIDAdminConfigKey = "frostfsid.admin" var ( contractList = []string{ @@ -538,12 +538,16 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an nnsCs.Hash, "container") case frostfsIDContract: - hs, err := getFrostfsIDAuthorizedKeys(viper.GetViper(), c.CommitteeAcc.PublicKey().GetScriptHash()) + h, found, err := getFrostfsIDAdmin(viper.GetViper()) if err != nil { panic(err) } - items = append(items, hs) + if found { + items = append(items, h) + } else { + items = append(items, nil) + } case netmapContract: md := getDefaultNetmapContractConfigMap() if method == updateMethodName { From 44806aa9f1968f71dd6210201857d0c39f486a3c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 1 Dec 2023 12:53:09 +0300 Subject: [PATCH 0709/1943] [#837] go.mod: Update dependencies Signed-off-by: Evgenii Stratonikov --- go.mod | 118 +-- go.sum | 3119 +++----------------------------------------------------- 2 files changed, 186 insertions(+), 3051 deletions(-) diff --git a/go.mod b/go.mod index cfa6dd7a6..aa0f689c1 100644 --- a/go.mod +++ b/go.mod @@ -4,40 +4,40 @@ go 1.20 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd - git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989 + git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231129062201-a1b61d394958 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231115094736-5db67021e10f + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231128145636-a0a35bf4bf31 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 github.com/dgraph-io/ristretto v0.1.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 - github.com/google/uuid v1.3.1 + github.com/google/uuid v1.4.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/klauspost/compress v1.17.2 + github.com/klauspost/compress v1.17.4 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 - github.com/multiformats/go-multiaddr v0.9.0 - github.com/nats-io/nats.go v1.27.1 - github.com/nspcc-dev/neo-go v0.103.0 + github.com/multiformats/go-multiaddr v0.12.0 + github.com/nats-io/nats.go v1.31.0 + github.com/nspcc-dev/neo-go v0.104.0 github.com/olekukonko/tablewriter v0.0.5 - github.com/panjf2000/ants/v2 v2.7.5 + github.com/panjf2000/ants/v2 v2.9.0 github.com/paulmach/orb v0.9.2 - github.com/prometheus/client_golang v1.16.0 - github.com/spf13/cast v1.5.1 + github.com/prometheus/client_golang v1.17.0 + github.com/spf13/cast v1.6.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.16.0 + github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 go.etcd.io/bbolt v1.3.8 - go.opentelemetry.io/otel v1.16.0 - go.opentelemetry.io/otel/trace v1.16.0 + go.opentelemetry.io/otel v1.21.0 + go.opentelemetry.io/otel/trace v1.21.0 go.uber.org/zap v1.26.0 - golang.org/x/exp v0.0.0-20231006140011-7918f672742d - golang.org/x/sync v0.4.0 - golang.org/x/term v0.13.0 + golang.org/x/exp v0.0.0-20231127185646-65229373498e + golang.org/x/sync v0.5.0 + golang.org/x/term v0.15.0 google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 @@ -46,10 +46,15 @@ require ( require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.1.2 // indirect + github.com/golang/glog v1.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/flatbuffers v2.0.8+incompatible // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + go.mongodb.org/mongo-driver v1.11.4 // indirect go.opencensus.io v0.24.0 // indirect ) @@ -58,33 +63,32 @@ require ( git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.8.0 // indirect + github.com/bits-and-blooms/bitset v1.11.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb // indirect + github.com/consensys/gnark-crypto v0.12.2-0.20231117165148-e77308824822 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect - github.com/dgraph-io/badger/v4 v4.1.0 - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/dgraph-io/badger/v4 v4.2.0 + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/gorilla/websocket v1.5.0 // indirect - github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 // indirect - github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect - github.com/hashicorp/golang-lru v0.6.0 // indirect + github.com/gorilla/websocket v1.5.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/holiman/uint256 v1.2.2 // indirect + github.com/holiman/uint256 v1.2.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect @@ -93,41 +97,37 @@ require ( github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/nats-io/nats-server/v2 v2.7.4 // indirect - github.com/nats-io/nkeys v0.4.4 // indirect + github.com/nats-io/nkeys v0.4.6 // indirect github.com/nats-io/nuid v1.0.1 // indirect - github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231020160724-c3955f87d1b5 // indirect + github.com/nspcc-dev/go-ordered-json v0.0.0-20231123160306-3374ff1e7a3c // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231201072346-6c0c2a6a98b0 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.9.5 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/subosito/gotenv v1.4.2 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/twmb/murmur3 v1.1.8 // indirect github.com/urfave/cli v1.22.14 // indirect - go.mongodb.org/mongo-driver v1.12.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 // indirect - go.opentelemetry.io/otel/metric v1.16.0 // indirect - go.opentelemetry.io/otel/sdk v1.16.0 // indirect - go.opentelemetry.io/proto/otlp v0.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/sdk v1.21.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 8c3689023..0167d4a51 100644 --- a/go.sum +++ b/go.sum @@ -1,733 +1,8 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= -cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= -cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= -cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= -cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= -cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= -cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= -cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= -cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= -cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= -cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= -cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= -cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= -cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= -cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps= -cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= -cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= -cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= -cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= -cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= -cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= -cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= -cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= -cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= -cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= -cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= -cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= -cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= -cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= -cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= -cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= -cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= -cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= -cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= -cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= -cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= -cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= -cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= -cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= -cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= -cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= -cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= -cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= -cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= -cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= -cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= -cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= -cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= -cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= -cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= -cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= -cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= -cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= -cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= -cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= -cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= -cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= -cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= -cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= -cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= -cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= -cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= -cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= -cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= -cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= -cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= -cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= -cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= -cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= -cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= -cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= -cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= -cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= -cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= -cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= -cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= -cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= -cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= -cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= -cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= -cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= -cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= -cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= -cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= -cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= -cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= -cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= -cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= -cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= -cloud.google.com/go/baremetalsolution v1.1.1/go.mod h1:D1AV6xwOksJMV4OSlWHtWuFNZZYujJknMAP4Qa27QIA= -cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= -cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= -cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= -cloud.google.com/go/batch v1.3.1/go.mod h1:VguXeQKXIYaeeIYbuozUmBR13AfL4SJP7IltNPS+A4A= -cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= -cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= -cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= -cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= -cloud.google.com/go/beyondcorp v1.0.0/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= -cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= -cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= -cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= -cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= -cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= -cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= -cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= -cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= -cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= -cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= -cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= -cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= -cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= -cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= -cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= -cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= -cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= -cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= -cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= -cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= -cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= -cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= -cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= -cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= -cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= -cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= -cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= -cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= -cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= -cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= -cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= -cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= -cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= -cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= -cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= -cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= -cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= -cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= -cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= -cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= -cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= -cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= -cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= -cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= -cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= -cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= -cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= -cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= -cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= -cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= -cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= -cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= -cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= -cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= -cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= -cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= -cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= -cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= -cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= -cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= -cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= -cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= -cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= -cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= -cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= -cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= -cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= -cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= -cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= -cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= -cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= -cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= -cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= -cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= -cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= -cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= -cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= -cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= -cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= -cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= -cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= -cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= -cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= -cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= -cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= -cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= -cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= -cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= -cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= -cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= -cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= -cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= -cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= -cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= -cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= -cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= -cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= -cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= -cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= -cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= -cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= -cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= -cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= -cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= -cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= -cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= -cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= -cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= -cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= -cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= -cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= -cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= -cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= -cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= -cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= -cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= -cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= -cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= -cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= -cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= -cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= -cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= -cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= -cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= -cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= -cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= -cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= -cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= -cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= -cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= -cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= -cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= -cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= -cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= -cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= -cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= -cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= -cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= -cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= -cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= -cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= -cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= -cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= -cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= -cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= -cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= -cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= -cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= -cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= -cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= -cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= -cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= -cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= -cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= -cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= -cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= -cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= -cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= -cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= -cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= -cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= -cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= -cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= -cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= -cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= -cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= -cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= -cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= -cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= -cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= -cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= -cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= -cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= -cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= -cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= -cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= -cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= -cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= -cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= -cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= -cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= -cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= -cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= -cloud.google.com/go/gkebackup v1.3.0/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= -cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= -cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= -cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= -cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= -cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= -cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= -cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= -cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= -cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= -cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= -cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= -cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= -cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= -cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= -cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= -cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= -cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= -cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= -cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= -cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= -cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= -cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= -cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= -cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= -cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= -cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= -cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= -cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= -cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= -cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= -cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= -cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= -cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= -cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= -cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= -cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= -cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= -cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= -cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= -cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= -cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= -cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= -cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= -cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= -cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= -cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= -cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= -cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= -cloud.google.com/go/kms v1.11.0/go.mod h1:hwdiYC0xjnWsKQQCQQmIQnS9asjYVSK6jtXm+zFqXLM= -cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= -cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= -cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= -cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= -cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= -cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= -cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= -cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= -cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= -cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= -cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= -cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= -cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= -cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= -cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= -cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= -cloud.google.com/go/longrunning v0.4.2/go.mod h1:OHrnaYyLUV6oqwh0xiS7e5sLQhP1m0QU9R+WhGDMgIQ= -cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= -cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= -cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= -cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= -cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= -cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= -cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= -cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= -cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= -cloud.google.com/go/maps v1.3.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= -cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= -cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= -cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= -cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= -cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= -cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= -cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= -cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= -cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= -cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= -cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= -cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= -cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= -cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= -cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= -cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= -cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= -cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= -cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= -cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= -cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= -cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= -cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= -cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= -cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= -cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= -cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= -cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= -cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= -cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= -cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= -cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= -cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= -cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= -cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= -cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= -cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= -cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= -cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= -cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= -cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= -cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= -cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= -cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= -cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= -cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= -cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= -cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= -cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= -cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= -cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= -cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= -cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= -cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= -cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= -cloud.google.com/go/orgpolicy v1.11.0/go.mod h1:2RK748+FtVvnfuynxBzdnyu7sygtoZa1za/0ZfpOs1M= -cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= -cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= -cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= -cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= -cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= -cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= -cloud.google.com/go/osconfig v1.12.0/go.mod h1:8f/PaYzoS3JMVfdfTubkowZYGmAhUCjjwnjqWI7NVBc= -cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= -cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= -cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= -cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= -cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= -cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= -cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= -cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= -cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= -cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= -cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= -cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= -cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= -cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= -cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= -cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0= -cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= -cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= -cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= -cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= -cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= -cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= -cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= -cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= -cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= -cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= -cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= -cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= -cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= -cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= -cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= -cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= -cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= -cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= -cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= -cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= -cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= -cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= -cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= -cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= -cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= -cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= -cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= -cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= -cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= -cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= -cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= -cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= -cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= -cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= -cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= -cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= -cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= -cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= -cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= -cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= -cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= -cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= -cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= -cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= -cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= -cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= -cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= -cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= -cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= -cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= -cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= -cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= -cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= -cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= -cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= -cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= -cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= -cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= -cloud.google.com/go/run v1.2.0/go.mod h1:36V1IlDzQ0XxbQjUx6IYbw8H3TJnWvhii963WW3B/bo= -cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= -cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= -cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= -cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= -cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= -cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= -cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= -cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= -cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= -cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= -cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= -cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= -cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= -cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= -cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= -cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= -cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= -cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= -cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= -cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= -cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= -cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= -cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= -cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= -cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= -cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= -cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= -cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= -cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= -cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= -cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= -cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= -cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= -cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= -cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= -cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= -cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= -cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= -cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= -cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= -cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= -cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= -cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= -cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= -cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= -cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= -cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= -cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= -cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= -cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= -cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= -cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= -cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= -cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= -cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= -cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= -cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= -cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= -cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= -cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= -cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= -cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= -cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= -cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= -cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= -cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= -cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= -cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= -cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= -cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= -cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= -cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= -cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= -cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= -cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= -cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= -cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= -cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= -cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= -cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= -cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= -cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= -cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= -cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= -cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= -cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= -cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= -cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= -cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= -cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= -cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= -cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= -cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= -cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= -cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= -cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= -cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= -cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= -cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= -cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= -cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= -cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= -cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= -cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= -cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= -cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= -cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= -cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= -cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= -cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= -cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= -cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= -cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= -cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= -cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= -cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= -cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= -cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= -cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= -cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= -cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= -cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= -cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= -cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= -cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= -cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= -cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= -cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= -cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= -cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= -cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= -cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= -cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= -cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= -cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= -cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= -cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= -cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= -cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= -cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= -cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= -cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= -cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= -cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= -cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= -cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd h1:MxiIXu4vUATX9bNYFnM1dXAYkx5dnPOGCMzwNNXJEvM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989 h1:hcWlSHcgkg2wnqAInu0AGQcdEY/zr7+BDbGK0rDfXLQ= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231102065436-9ed3845aa989/go.mod h1:PxsAxe6J4hyYJoU6hyYxD9q5K/DFxdquKHL1BrRaGkg= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231129062201-a1b61d394958 h1:X9yPizADIhD3K/gdKVCthlAnf9aQ3UJJGnZgIwwixRQ= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231129062201-a1b61d394958/go.mod h1:rQWdsG18NaiFvkJpMguJev913KD/yleHaniRBkUyt0o= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= @@ -736,192 +11,28 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231115094736-5db67021e10f h1:Rq95TuEkqc3T1EN5ZU1Vgf6H33TR95hz97ca8jrUciQ= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231115094736-5db67021e10f/go.mod h1:qf3B9hSz6gCMfcfvqkhTu5ak+Gx2R+wo4Hc87LnKxPg= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231128145636-a0a35bf4bf31 h1:31tE+hNkHar7IMduhzj8LFY8+LT2EswiVI873MEfZj0= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231128145636-a0a35bf4bf31/go.mod h1:ekrDiIySdYhji5rBNAkxYMztFWMXyC9Q8LVz6gGVDu0= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= -git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20221206110420-d395f97c4830/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= -github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= -github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CityOfZion/neo-go v0.62.1-pre.0.20191114145240-e740fbe708f8/go.mod h1:MJCkWUBhi9pn/CrYO1Q3P687y2KeahrOPS9BD9LDGb0= -github.com/CityOfZion/neo-go v0.70.1-pre.0.20191209120015-fccb0085941e/go.mod h1:0enZl0az8xA6PVkwzEOwPWVJGqlt/GO4hA4kmQ5Xzig= -github.com/CityOfZion/neo-go v0.70.1-pre.0.20191212173117-32ac01130d4c/go.mod h1:JtlHfeqLywZLswKIKFnAp+yzezY4Dji9qlfQKB2OD/I= -github.com/CityOfZion/neo-go v0.71.1-pre.0.20200129171427-f773ec69fb84/go.mod h1:FLI526IrRWHmcsO+mHsCbj64pJZhwQFTLJZu+A4PGOA= -github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= -github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= -github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= -github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= -github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= -github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= -github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim v0.9.6/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim v0.10.0-rc.8/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= -github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= -github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= -github.com/abiosoft/ishell v2.0.0+incompatible/go.mod h1:HQR9AqF2R3P4XXpMpI0NAzgHf/aS6+zVXRj14cVk9qg= -github.com/abiosoft/ishell/v2 v2.0.2/go.mod h1:E4oTCXfo6QjoCart0QYa5m9w4S+deXs/P/9jA77A9Bs= -github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db/go.mod h1:rB3B4rKii8V21ydCbIzH5hZiCQE7f5E9SzUb/ZZx530= -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= -github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= -github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= -github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= -github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= -github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= -github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= -github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210521073959-f0d4d129b7f1/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= -github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= -github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= -github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= -github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.43.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= -github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= -github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/bytecodealliance/wasmtime-go v0.36.0/go.mod h1:q320gUxqyI8yB+ZqRuaJOEnGkAnHh6WtJjMaT2CW4wI= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/bits-and-blooms/bitset v1.11.0 h1:RMyy2mBBShArUAhfVRZJ2xyBO58KCBCtZFShw3umo6k= +github.com/bits-and-blooms/bitset v1.11.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= -github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= -github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= -github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= -github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= @@ -931,424 +42,61 @@ github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObk github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= -github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= -github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark v0.9.1/go.mod h1:udWvWGXnfBE7mn7BsNoGAvZDnUhcONBEtNijvVjfY80= -github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb h1:f0BMgIjhZy4lSRHCXFbQst85f5agZAjtDMixQqBWNpc= -github.com/consensys/gnark-crypto v0.12.2-0.20231013160410-1f65e75b6dfb/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= -github.com/container-orchestrated-devices/container-device-interface v0.5.4/go.mod h1:DjE95rfPiiSmG7uVXtg0z6MnPm/Lx4wxKCIts0ZE0vg= -github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= -github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= -github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= -github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/btrfs/v2 v2.0.0/go.mod h1:swkD/7j9HApWpzl8OHfrHNxppPd9l44DFZdF94BUj9k= -github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= -github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= -github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= -github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= -github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= -github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= -github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= -github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= -github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= -github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= -github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= -github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= -github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= -github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE= -github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0= -github.com/containerd/containerd v1.6.8/go.mod h1:By6p5KqPK0/7/CgO/A6t/Gz+CUYUu2zf1hUaaymVXB0= -github.com/containerd/containerd v1.6.9/go.mod h1:XVicUvkxOrftE2Q1YWUXgZwkkAxwQYNOFzYWvfVfEfQ= -github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= -github.com/containerd/containerd v1.7.3/go.mod h1:32FOM4/O0RkNg7AjQj3hDzN9cUGtu+HMvaKUNiqCZB8= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= -github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= -github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= -github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= -github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= -github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= -github.com/containerd/continuity v0.4.1/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= -github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= -github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= -github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= -github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= -github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= -github.com/containerd/go-cni v1.1.6/go.mod h1:BWtoWl5ghVymxu6MBjg79W9NZrCRyHIdUtk4cauMe34= -github.com/containerd/go-cni v1.1.9/go.mod h1:XYrZJ1d5W6E2VOvjffL3IZq0Dz6bsVlERHbekNK90PM= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= -github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= -github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= -github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= -github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= -github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= -github.com/containerd/imgcrypt v1.1.4/go.mod h1:LorQnPtzL/T0IyCeftcsMEO7AqxUDbdO8j/tSUpgxvo= -github.com/containerd/imgcrypt v1.1.7/go.mod h1:FD8gqIcX5aTotCtOmjeCsi3A1dHmTZpnMISGKSczt4k= -github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= -github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/nri v0.3.0/go.mod h1:Zw9q2lP16sdg0zYybemZ9yTDy8g7fPCIB3KXOGlggXI= -github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= -github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= -github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= -github.com/containerd/ttrpc v1.1.1-0.20220420014843-944ef4a40df3/go.mod h1:YYyNVhZrTMiaf51Vj6WhAJqJw+vl/nzABhj8pWrzle4= -github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= -github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= -github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= -github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= -github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= -github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= -github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v1.1.0/go.mod h1:oZF9wBnrnQjpWLaPKEinrx3TQ9a+W/RJO7Zb41d8YLE= -github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= -github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= -github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= -github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= -github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= -github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= -github.com/containernetworking/plugins v1.2.0/go.mod h1:/VjX4uHecW5vVimFa1wkG4s+r/s9qIfPdqlLF4TW8c4= -github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= -github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= -github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/containers/ocicrypt v1.1.3/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g= -github.com/containers/ocicrypt v1.1.6/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/consensys/gnark-crypto v0.12.2-0.20231117165148-e77308824822 h1:PvEjRgB/U4bv0jl9w65Wy9g0nIdkkW7vkNoR8Vq/als= +github.com/consensys/gnark-crypto v0.12.2-0.20231117165148-e77308824822/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= -github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= -github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= -github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= -github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= -github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M= -github.com/dgraph-io/badger/v4 v4.1.0 h1:E38jc0f+RATYrycSUf9LMv/t47XAy+3CApyYSq4APOQ= -github.com/dgraph-io/badger/v4 v4.1.0/go.mod h1:P50u28d39ibBRmIJuQC/NSdBOg46HnHw7al2SW5QRHg= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/distribution/distribution/v3 v3.0.0-20220526142353-ffbd94cbe269/go.mod h1:28YO/VJk9/64+sTGNuYaBjWxrXTPrj0C0XmgTIOjxX4= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v23.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= -github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v23.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v23.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= -github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= -github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= -github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= -github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= -github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= -github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= -github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= -github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/foxcpp/go-mockdns v0.0.0-20210729171921-fb145fc6f897/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= -github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= -github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= -github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= -github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= -github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= -github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= -github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-ini/ini v1.66.6/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= -github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= -github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= -github.com/go-redis/redis v6.10.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= -github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= -github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= -github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= -github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -1358,704 +106,178 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.12.6/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= -github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= -github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= -github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= -github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= -github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= -github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= -github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= -github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= -github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0 h1:mdLirNAJBxnGgyB6pjZLcs6ue/6eZGBui6gXspfq4ks= -github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0-rc.0/go.mod h1:kdXbOySqcQeTxiqglW7aahTmWZy3Pgi6SYL36yvKeyA= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5 h1:3IZOAnD058zZllQTZNBioTlrzrBG/IjpiZ133IEtusM= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5/go.mod h1:xbKERva94Pw2cPen0s79J3uXmGzbbpDYFBFDlZ4mV/w= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= -github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 h1:f4tggROQKKcnh4eItay6z/HbHLqghBxS8g7pyMhmDio= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0/go.mod h1:hKAkSgNkL0FII46ZkJcpVEAai4KV+swlIWCKfekd1pA= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 h1:HcUWd006luQPljE73d5sk+/VgYPGUReEVz2y1/qylwY= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= -github.com/holiman/uint256 v1.2.2 h1:TXKcSGc2WaxPD2+bmzAsVthL4+pEN0YwXcL5qED83vk= -github.com/holiman/uint256 v1.2.2/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= +github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= +github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= -github.com/intel/goresctrl v0.3.0/go.mod h1:fdz3mD85cmP9sHD8JUlrNWAxvwM86CrbmVXltEKd7zk= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= -github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= -github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= -github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= -github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= -github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= -github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= -github.com/lestrrat-go/jwx v1.2.25/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xwxBBK1NxY= -github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= -github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= -github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= -github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= -github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= -github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= -github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= -github.com/mistifyio/go-zfs/v3 v3.0.1/go.mod h1:CzVgeB0RvF2EGzQnytKVvVSDwmKJXxkOTUGbNrTja/k= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= -github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= -github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= -github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= -github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= -github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= +github.com/multiformats/go-multiaddr v0.12.0 h1:1QlibTFkoXJuDjjYsMHhE73TnzJQl8FSWatk/0gxGzE= +github.com/multiformats/go-multiaddr v0.12.0/go.mod h1:WmZXgObOQOYp9r3cslLlppkrz1FYSHmE834dfz/lWu8= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 h1:vU9tpM3apjYlLLeY23zRWJ9Zktr5jp+mloR942LEOpY= -github.com/nats-io/nats-server/v2 v2.7.4 h1:c+BZJ3rGzUKCBIM4IXO8uNT2u1vajGbD1kPA6wqCEaM= -github.com/nats-io/nats-server/v2 v2.7.4/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc= -github.com/nats-io/nats.go v1.27.1 h1:OuYnal9aKVSnOzLQIzf7554OXMCG7KbaTkCSBHRcSoo= -github.com/nats-io/nats.go v1.27.1/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= -github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= -github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= +github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E= +github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8= +github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= +github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/networkplumbing/go-nft v0.2.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nspcc-dev/dbft v0.0.0-20191205084618-dacb1a30c254/go.mod h1:w1Ln2aT+dBlPhLnuZhBV+DfPEdS2CHWWLp5JTScY3bw= -github.com/nspcc-dev/dbft v0.0.0-20191209120240-0d6b7568d9ae/go.mod h1:3FjXOoHmA51EGfb5GS/HOv7VdmngNRTssSeQ729dvGY= -github.com/nspcc-dev/dbft v0.0.0-20200117124306-478e5cfbf03a/go.mod h1:/YFK+XOxxg0Bfm6P92lY5eDSLYfp06XOdL8KAVgXjVk= -github.com/nspcc-dev/dbft v0.0.0-20200219114139-199d286ed6c1/go.mod h1:O0qtn62prQSqizzoagHmuuKoz8QMkU3SzBoKdEvm3aQ= -github.com/nspcc-dev/dbft v0.0.0-20210721160347-1b03241391ac/go.mod h1:U8MSnEShH+o5hexfWJdze6uMFJteP0ko7J2frO7Yu1Y= -github.com/nspcc-dev/dbft v0.0.0-20221020093431-31c1bbdc74f2/go.mod h1:g9xisXmX9NP9MjioaTe862n9SlZTrP+6PVUWLBYOr98= -github.com/nspcc-dev/dbft v0.0.0-20230515113611-25db6ba61d5c/go.mod h1:kjBC9F8L25GR+kIHy/1KgG/KfcoGnVwIiyovgq1uszk= -github.com/nspcc-dev/go-ordered-json v0.0.0-20210915112629-e1b6cce73d02/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= -github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 h1:n4ZaFCKt1pQJd7PXoMJabZWK9ejjbLOVrkl/lOUmshg= -github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= -github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU= -github.com/nspcc-dev/neo-go v0.73.1-pre.0.20200303142215-f5a1b928ce09/go.mod h1:pPYwPZ2ks+uMnlRLUyXOpLieaDQSEaf4NM3zHVbRjmg= -github.com/nspcc-dev/neo-go v0.98.0/go.mod h1:E3cc1x6RXSXrJb2nDWXTXjnXk3rIqVN8YdFyWv+FrqM= -github.com/nspcc-dev/neo-go v0.100.1/go.mod h1:Nnp7F4e9IBccsgtCeLtUWV+0T6gk1PtP5HRtA13hUfc= -github.com/nspcc-dev/neo-go v0.103.0 h1:UVyWPhzZdfYFG35ORP3FRDLh8J/raRQ6m8SptDdlgfM= -github.com/nspcc-dev/neo-go v0.103.0/go.mod h1:x+wmcYqpZYJwLp1l/pHZrqNp3RSWlkMymWGDij3/OPo= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20221202075445-cb5c18dc73eb/go.mod h1:23bBw0v6pBYcrWs8CBEEDIEDJNbcFoIh8pGGcf2Vv8s= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231020160724-c3955f87d1b5 h1:09CpI5uwsxb1EeFPIKQRwwWlfCmDD/Dwwh01lPiQScM= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231020160724-c3955f87d1b5/go.mod h1:J/Mk6+nKeKSW4wygkZQFLQ6SkLOSGX5Ga0RuuuktEag= -github.com/nspcc-dev/neofs-api-go/v2 v2.11.0-pre.0.20211201134523-3604d96f3fe1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.11.1/go.mod h1:oS8dycEh8PPf2Jjp6+8dlwWyEv2Dy77h/XhhcdxYEFs= -github.com/nspcc-dev/neofs-api-go/v2 v2.14.0/go.mod h1:DRIr0Ic1s+6QgdqmNFNLIqMqd7lNMJfYwkczlm1hDtM= -github.com/nspcc-dev/neofs-crypto v0.2.0/go.mod h1:F/96fUzPM3wR+UGsPi3faVNmFlA9KAEAUQR7dMxZmNA= -github.com/nspcc-dev/neofs-crypto v0.2.3/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-crypto v0.3.0/go.mod h1:8w16GEJbH6791ktVqHN9YRNH3s9BEEKYxGhlFnp0cDw= -github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20211201182451-a5b61c4f6477/go.mod h1:dfMtQWmBHYpl9Dez23TGtIUKiFvCIxUZq/CkSIhEpz4= -github.com/nspcc-dev/neofs-sdk-go v0.0.0-20220113123743-7f3162110659/go.mod h1:/jay1lr3w7NQd/VDBkEhkJmDmyPNsu4W+QV2obsUV40= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11/go.mod h1:W+ImTNRnSNMH8w43H1knCcIqwu7dLHePXtlJNZ7EFIs= -github.com/nspcc-dev/rfc6979 v0.1.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= +github.com/nspcc-dev/go-ordered-json v0.0.0-20231123160306-3374ff1e7a3c h1:OOQeE613BH93ICPq3eke5N78gWNeMjcBWkmD2NKyXVg= +github.com/nspcc-dev/go-ordered-json v0.0.0-20231123160306-3374ff1e7a3c/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= +github.com/nspcc-dev/neo-go v0.104.0 h1:FGj3Z46yABcFIAI1SCLd1jQSoh+B00h/2VAgEgY1JKQ= +github.com/nspcc-dev/neo-go v0.104.0/go.mod h1:omsUK5PAtG2/nQ3/evs95QEg3wtkj3LH53e0NKtXVwQ= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231201072346-6c0c2a6a98b0 h1:GYeU/mq2qfq6fNSuTzgxvlQQUydeCBicjzdnggwtuz0= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231201072346-6c0c2a6a98b0/go.mod h1:J/Mk6+nKeKSW4wygkZQFLQ6SkLOSGX5Ga0RuuuktEag= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= -github.com/nspcc-dev/tzhash v1.7.0/go.mod h1:Dnx9LUlOLr5paL2Rtc96x0PPs8D9eIkUtowt1n+KQus= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= -github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= -github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= -github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= -github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= -github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= -github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= -github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= -github.com/onsi/gomega v1.24.2 h1:J/tulyYK6JwBldPViHJReihxxZ+22FHs0piGjQAvoUE= -github.com/onsi/gomega v1.24.2/go.mod h1:gs3J10IS7Z7r7eXRoNJIrNqU4ToQukCJhFtKrWgHWnk= -github.com/open-policy-agent/opa v0.42.2/go.mod h1:MrmoTi/BsKWT58kXlVayBb+rYVeaMwuBm3nYAN3923s= -github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= -github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= -github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= -github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/opencontainers/runc v1.1.8/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= -github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.1.0-rc.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= -github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= -github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI= -github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= -github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= -github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= -github.com/panjf2000/ants/v2 v2.7.5 h1:/vhh0Hza9G1vP1PdCj9hl6MUzCRbmtcTJL0OsnmytuU= -github.com/panjf2000/ants/v2 v2.7.5/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/panjf2000/ants/v2 v2.9.0 h1:SztCLkVxBRigbg+vt0S5QvF5vxAbxbKt09/YfAJ0tEo= +github.com/panjf2000/ants/v2 v2.9.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= github.com/paulmach/orb v0.9.2 h1:p/YWV2uJwamAynnDOJGNbPBVtDHj3vG51k9tR1rFwJE= github.com/paulmach/orb v0.9.2/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= -github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= -github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= -github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= -github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= -github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= -github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/safchain/ethtool v0.2.0/go.mod h1:WkKB1DnNtvsMlDmQ50sgwowDJV/hGbJSOvJoEXs1AJQ= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= -github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= -github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= -github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= -github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= -github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= -github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI= +github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -2065,1076 +287,162 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= -github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/tchap/go-patricia/v2 v2.3.1/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= -github.com/testcontainers/testcontainers-go v0.22.0/go.mod h1:k0YiPa26xJCRUbUkYqy5rY6NGvSbVCeUBXCvucscBR4= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= -github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= -github.com/vektah/gqlparser/v2 v2.4.5/go.mod h1:flJWIR04IMQPGz+BXLrORkrARBxv/rtyIAFvd/MceW0= -github.com/veraison/go-cose v1.0.0-rc.1/go.mod h1:7ziE85vSq4ScFTg6wyoMXjucIGOf4JkFEZi/an96Ct4= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= -github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= -github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yashtewari/glob-intersection v0.1.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= -github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= -github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= -github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4= -go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= -go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/pkg/v3 v3.5.5/go.mod h1:6ksYFxttiUGzC2uxyqiyOEvhAiD0tuIqSZkX3TyPdaE= -go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/raft/v3 v3.5.5/go.mod h1:76TA48q03g1y1VpTue92jZLr9lIHKUNcYdZOOGyx8rI= -go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= -go.etcd.io/etcd/server/v3 v3.5.5/go.mod h1:rZ95vDw/jrvsbj9XpTqPrTAB9/kzchVdhRirySPkUBc= -go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M= +go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE= -go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0= -go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.40.0/go.mod h1:UMklln0+MRhZC4e3PwmN3pCtq4DyIadWw4yikh6bNrw= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.32.0/go.mod h1:5eCOqeGphOyz6TsY3ZDNjE33SM/TFAK3RGuCL2naTgY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= -go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= -go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= -go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= -go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= -go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0/go.mod h1:M1hVZHNxcbkAlcvrOMlpQ4YOO3Awf+4N2dxkZL3xm04= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0/go.mod h1:ceUgdyfNv4h4gLxHR0WNfDiiVmZFodZhZSbOLhpxqXE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0/go.mod h1:E+/KKhwOSw8yoPxSSuUHG6vKppkvhN+S1Jc7Nib3k3o= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0/go.mod h1:+N7zNjIJv4K+DeX67XXET0P+eIciESgaFDBqh+ZJFS4= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 h1:+XWJd3jf75RXJq29mxbuXhCXFDG3S3R4vBUeSI2P7tE= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0/go.mod h1:hqgzBPTf4yONMFgdZvL/bK42R/iinTyVQtiWihs3SZc= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v0.30.0/go.mod h1:/ShZ7+TS4dHzDFmfi1kSXMhMVubNoP0oIaBp70J6UXU= -go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= -go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s= -go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= -go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= -go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= -go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU= -go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= -go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= -go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= -go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= -go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= -go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= -go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= -go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= -go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.opentelemetry.io/proto/otlp v0.16.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= -go.opentelemetry.io/proto/otlp v0.20.0 h1:BLOA1cZBAGSbRiNuGCCKiFrCdYB7deeHDeD1SueyOfA= -go.opentelemetry.io/proto/otlp v0.20.0/go.mod h1:3QgjzPALBIv9pcknj2EXGPXjYPFdUh/RQfF8Lz3+Vnw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 h1:VhlEQAPp9R1ktYfrPk5SOryw1e9LDDTZCbIPFrho0ec= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0/go.mod h1:kB3ufRbfU+CQ4MlUcqtW8Z7YEOBeK2DJ6CmR5rYYF3E= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= -golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= -golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= +golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180318012157-96caea41033d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= -golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/tools v0.11.1/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= -golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= -gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= -gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= -google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= -google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= -google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= -google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= -google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= -google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= -google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= -google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= -google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= -google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= -google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= -google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= -google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= -google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= -google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjYK+5E= -google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= -google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= -google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= -google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= -google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= -google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= -google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= -google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= -google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= -google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= -google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= -google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= -google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= -google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= -google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= -google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= -google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= -google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= -google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= -google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= -google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= -google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108= -google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a h1:fwgW9j3vHirt4ObdHoYNwuO24BEZjSzbh+zPaNWoiY8= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= -google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f h1:Vn+VyHU5guc9KjB5KrjI2q0wCOWEOIh0OEsleqakHJg= +google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 h1:ZcOkrmX74HbKFYnpPY8Qsw93fC29TbJXspYKaBkSXDQ= +google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4/go.mod h1:k2dtGpRrbsSyKcNPKKI5sstZkrNCZwpU/ns96JoHbGg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= -google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= -google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= -google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -3143,205 +451,32 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= -gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= -k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= -k8s.io/api v0.26.2/go.mod h1:1kjMQsFE+QHPfskEcVNgL3+Hp88B80uj0QtSOlj8itU= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= -k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0= -k8s.io/apimachinery v0.26.2/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= -k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= -k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ= -k8s.io/apiserver v0.26.2/go.mod h1:GHcozwXgXsPuOJ28EnQ/jXEM9QeG6HT22YxSNmpYNh8= -k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= -k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= -k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= -k8s.io/client-go v0.26.2/go.mod h1:u5EjOuSyBa09yqqyY7m3abZeovO/7D/WehVVlZ2qcqU= -k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= -k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= -k8s.io/component-base v0.26.2/go.mod h1:DxbuIe9M3IZPRxPIzhch2m1eT7uFrSBJUBuVCQEBivs= -k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= -k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= -k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= -k8s.io/cri-api v0.25.0/go.mod h1:J1rAyQkSJ2Q6I+aBMOVgg2/cbbebso6FNa0UagiR0kc= -k8s.io/cri-api v0.25.3/go.mod h1:riC/P0yOGUf2K1735wW+CXs1aY2ctBgePtnnoFLd0dU= -k8s.io/cri-api v0.27.1/go.mod h1:+Ts/AVYbIo04S86XbTD73UPp/DkTiYxtsFeOFEu32L0= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kms v0.26.2/go.mod h1:69qGnf1NsFOQP07fBYqNLZklqEHSJF024JqYCaeVxHg= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= -lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= -modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= -modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= -modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= -modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= -modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= -modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= -modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI= -modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= -modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= -modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= -modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= -modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= -modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= -modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= -modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= -modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= -modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= -modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= -modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= -modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= -modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= -modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= -modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA= -modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0= -modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= -modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= -modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= -modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= -modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= -modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= -modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= -modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= -modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= -modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= -modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= -modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= -oras.land/oras-go v1.2.0/go.mod h1:pFNs7oHp2dYsYMSS82HaX5l4mpnGO7hbpPN6EWH2ltc= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 6bb27f98ddf009f0fff9808d264238cea9894c00 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 1 Dec 2023 13:12:05 +0300 Subject: [PATCH 0710/1943] [#837] .pre-commit: Update hook versions Signed-off-by: Evgenii Stratonikov --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ecd70390c..f66adda0b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: - id: gitlint-ci - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -26,7 +26,7 @@ repos: exclude: ".key$" - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.5 + rev: v0.9.0.6 hooks: - id: shellcheck From b892feeaf65b89af3b5aa712dca5aefd11d5d660 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Dec 2023 11:57:04 +0300 Subject: [PATCH 0711/1943] [#845] adm: Relax notary-enabled check Starting from v0.104.0 `NativeActivations` config field is no longer present and Notary activation height is always 0. https://github.com/nspcc-dev/neo-go/pull/3212/ https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/issues/59 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/initialize.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index e288361dd..96100ffb5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -519,7 +519,7 @@ func checkNotaryEnabled(c Client) error { nativeHashes := make(map[string]util.Uint160, len(ns)) for i := range ns { if ns[i].Manifest.Name == nativenames.Notary { - notaryEnabled = len(ns[i].UpdateHistory) > 0 + notaryEnabled = true } nativeHashes[ns[i].Manifest.Name] = ns[i].Hash } From 29550fe6003ed5a10daf1ed4ad0c08982d0af022 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 4 Dec 2023 12:54:23 +0300 Subject: [PATCH 0712/1943] [#838] shard: Refactor `updateMetrics` method Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/shard.go | 86 +++++++++++++------------ 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index aa87e0cfc..ff437d7ee 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -398,52 +398,54 @@ const ( ) func (s *Shard) updateMetrics(ctx context.Context) { - if s.cfg.metricsWriter != nil && !s.GetMode().NoMetabase() { - cc, err := s.metaBase.ObjectCounters() + if s.cfg.metricsWriter == nil || s.GetMode().NoMetabase() { + return + } + + cc, err := s.metaBase.ObjectCounters() + if err != nil { + s.log.Warn(logs.ShardMetaObjectCounterRead, + zap.Error(err), + ) + + return + } + + s.cfg.metricsWriter.SetObjectCounter(physical, cc.Phy()) + s.cfg.metricsWriter.SetObjectCounter(logical, cc.Logic()) + + cnrList, err := s.metaBase.Containers(ctx) + if err != nil { + s.log.Warn(logs.ShardMetaCantReadContainerList, zap.Error(err)) + return + } + + var totalPayload uint64 + + for i := range cnrList { + size, err := s.metaBase.ContainerSize(cnrList[i]) if err != nil { - s.log.Warn(logs.ShardMetaObjectCounterRead, - zap.Error(err), - ) - - return + s.log.Warn(logs.ShardMetaCantReadContainerSize, + zap.String("cid", cnrList[i].EncodeToString()), + zap.Error(err)) + continue } + s.metricsWriter.AddToContainerSize(cnrList[i].EncodeToString(), int64(size)) + totalPayload += size + } - s.cfg.metricsWriter.SetObjectCounter(physical, cc.Phy()) - s.cfg.metricsWriter.SetObjectCounter(logical, cc.Logic()) + s.metricsWriter.AddToPayloadSize(int64(totalPayload)) - cnrList, err := s.metaBase.Containers(ctx) - if err != nil { - s.log.Warn(logs.ShardMetaCantReadContainerList, zap.Error(err)) - return - } - - var totalPayload uint64 - - for i := range cnrList { - size, err := s.metaBase.ContainerSize(cnrList[i]) - if err != nil { - s.log.Warn(logs.ShardMetaCantReadContainerSize, - zap.String("cid", cnrList[i].EncodeToString()), - zap.Error(err)) - continue - } - s.metricsWriter.AddToContainerSize(cnrList[i].EncodeToString(), int64(size)) - totalPayload += size - } - - s.metricsWriter.AddToPayloadSize(int64(totalPayload)) - - contCount, err := s.metaBase.ContainerCounters(ctx) - if err != nil { - s.log.Warn(logs.FailedToGetContainerCounters, zap.Error(err)) - return - } - for contID, count := range contCount.Physical { - s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), physical, count) - } - for contID, count := range contCount.Logical { - s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), logical, count) - } + contCount, err := s.metaBase.ContainerCounters(ctx) + if err != nil { + s.log.Warn(logs.FailedToGetContainerCounters, zap.Error(err)) + return + } + for contID, count := range contCount.Physical { + s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), physical, count) + } + for contID, count := range contCount.Logical { + s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), logical, count) } } From f314da4af320ff7df80f69d6928ff85b96318b8e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 4 Dec 2023 17:35:11 +0300 Subject: [PATCH 0713/1943] [#838] metabase: Add user object type counter Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/counter.go | 129 ++++++---- .../metabase/counter_test.go | 239 ++++++++---------- pkg/local_object_storage/metabase/delete.go | 8 +- pkg/local_object_storage/metabase/inhume.go | 8 +- .../shard/control_test.go | 8 +- pkg/local_object_storage/shard/count.go | 2 +- .../shard/metrics_test.go | 59 +++-- pkg/local_object_storage/shard/shard.go | 21 +- 8 files changed, 247 insertions(+), 227 deletions(-) diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index d1aafa3db..be860830a 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -18,6 +18,7 @@ import ( var ( objectPhyCounterKey = []byte("phy_counter") objectLogicCounterKey = []byte("logic_counter") + objectUserCounterKey = []byte("user_counter") ) type objectType uint8 @@ -31,18 +32,13 @@ const ( // ObjectCounters groups object counter // according to metabase state. type ObjectCounters struct { - logic uint64 - phy uint64 + Logic uint64 + Phy uint64 + User uint64 } -// Logic returns logical object counter. -func (o ObjectCounters) Logic() uint64 { - return o.logic -} - -// Phy returns physical object counter. -func (o ObjectCounters) Phy() uint64 { - return o.phy +func (o ObjectCounters) IsZero() bool { + return o.Phy == 0 && o.Logic == 0 && o.User == 0 } // ObjectCounters returns object counters that metabase has @@ -63,12 +59,17 @@ func (db *DB) ObjectCounters() (cc ObjectCounters, err error) { if b != nil { data := b.Get(objectPhyCounterKey) if len(data) == 8 { - cc.phy = binary.LittleEndian.Uint64(data) + cc.Phy = binary.LittleEndian.Uint64(data) } data = b.Get(objectLogicCounterKey) if len(data) == 8 { - cc.logic = binary.LittleEndian.Uint64(data) + cc.Logic = binary.LittleEndian.Uint64(data) + } + + data = b.Get(objectUserCounterKey) + if len(data) == 8 { + cc.User = binary.LittleEndian.Uint64(data) } } @@ -79,8 +80,7 @@ func (db *DB) ObjectCounters() (cc ObjectCounters, err error) { } type ContainerCounters struct { - Logical map[cid.ID]uint64 - Physical map[cid.ID]uint64 + Counts map[cid.ID]ObjectCounters } // ContainerCounters returns object counters for each container @@ -103,8 +103,7 @@ func (db *DB) ContainerCounters(ctx context.Context) (ContainerCounters, error) defer span.End() cc := ContainerCounters{ - Logical: make(map[cid.ID]uint64), - Physical: make(map[cid.ID]uint64), + Counts: make(map[cid.ID]ObjectCounters), } lastKey := make([]byte, cidSize) @@ -158,16 +157,11 @@ func (db *DB) containerCountersNextBatch(lastKey []byte, cc *ContainerCounters) if err != nil { return err } - phy, logic, err := parseContainerCounterValue(value) + ent, err := parseContainerCounterValue(value) if err != nil { return err } - if phy > 0 { - cc.Physical[cnrID] = phy - } - if logic > 0 { - cc.Logical[cnrID] = logic - } + cc.Counts[cnrID] = ent counter++ if counter == batchSize { @@ -253,22 +247,23 @@ func (db *DB) updateContainerCounter(tx *bbolt.Tx, delta map[cid.ID]ObjectCounte } func (*DB) editContainerCounterValue(b *bbolt.Bucket, key []byte, delta ObjectCounters, inc bool) error { - var phyValue, logicValue uint64 + var entity ObjectCounters var err error data := b.Get(key) if len(data) > 0 { - phyValue, logicValue, err = parseContainerCounterValue(data) + entity, err = parseContainerCounterValue(data) if err != nil { return err } } - phyValue = nextValue(phyValue, delta.phy, inc) - logicValue = nextValue(logicValue, delta.logic, inc) - if phyValue > 0 || logicValue > 0 { - value := containerCounterValue(phyValue, logicValue) - return b.Put(key, value) + entity.Phy = nextValue(entity.Phy, delta.Phy, inc) + entity.Logic = nextValue(entity.Logic, delta.Logic, inc) + entity.User = nextValue(entity.User, delta.User, inc) + if entity.IsZero() { + return b.Delete(key) } - return b.Delete(key) + value := containerCounterValue(entity) + return b.Put(key, value) } func nextValue(existed, delta uint64, inc bool) uint64 { @@ -290,7 +285,7 @@ func (db *DB) incContainerObjectCounter(tx *bbolt.Tx, cnrID cid.ID) error { key := make([]byte, cidSize) cnrID.Encode(key) - return db.editContainerCounterValue(b, key, ObjectCounters{logic: 1, phy: 1}, true) + return db.editContainerCounterValue(b, key, ObjectCounters{Logic: 1, Phy: 1}, true) } // syncCounter updates object counters according to metabase state: @@ -304,9 +299,11 @@ func syncCounter(tx *bbolt.Tx, force bool) error { if err != nil { return fmt.Errorf("could not get shard info bucket: %w", err) } - shardObjectCounterInitialized := len(shardInfoB.Get(objectPhyCounterKey)) == 8 && len(shardInfoB.Get(objectLogicCounterKey)) == 8 - containerCounterInitialized := tx.Bucket(containerCounterBucketName) != nil - if !force && shardObjectCounterInitialized && containerCounterInitialized { + shardObjectCounterInitialized := len(shardInfoB.Get(objectPhyCounterKey)) == 8 && + len(shardInfoB.Get(objectLogicCounterKey)) == 8 && + len(shardInfoB.Get(objectUserCounterKey)) == 8 + containerObjectCounterInitialized := containerObjectCounterInitialized(tx) + if !force && shardObjectCounterInitialized && containerObjectCounterInitialized { // the counters are already inited return nil } @@ -325,11 +322,11 @@ func syncCounter(tx *bbolt.Tx, force bool) error { err = iteratePhyObjects(tx, func(cnr cid.ID, obj oid.ID) error { if v, ok := counters[cnr]; ok { - v.phy++ + v.Phy++ counters[cnr] = v } else { counters[cnr] = ObjectCounters{ - phy: 1, + Phy: 1, } } @@ -340,11 +337,11 @@ func syncCounter(tx *bbolt.Tx, force bool) error { // and not covered with a tombstone if inGraveyardWithKey(addressKey(addr, key), graveyardBKT, garbageBKT) == 0 { if v, ok := counters[cnr]; ok { - v.logic++ + v.Logic++ counters[cnr] = v } else { counters[cnr] = ObjectCounters{ - logic: 1, + Logic: 1, } } } @@ -361,13 +358,15 @@ func syncCounter(tx *bbolt.Tx, force bool) error { func setObjectCounters(counters map[cid.ID]ObjectCounters, shardInfoB, containerCounterB *bbolt.Bucket) error { var phyTotal uint64 var logicTotal uint64 + var userTotal uint64 key := make([]byte, cidSize) for cnrID, count := range counters { - phyTotal += count.phy - logicTotal += count.logic + phyTotal += count.Phy + logicTotal += count.Logic + userTotal += count.User cnrID.Encode(key) - value := containerCounterValue(count.phy, count.logic) + value := containerCounterValue(count) err := containerCounterB.Put(key, value) if err != nil { return fmt.Errorf("could not update phy container object counter: %w", err) @@ -389,13 +388,22 @@ func setObjectCounters(counters map[cid.ID]ObjectCounters, shardInfoB, container return fmt.Errorf("could not update logic object counter: %w", err) } + userData := make([]byte, 8) + binary.LittleEndian.PutUint64(userData, userTotal) + + err = shardInfoB.Put(objectUserCounterKey, userData) + if err != nil { + return fmt.Errorf("could not update user object counter: %w", err) + } + return nil } -func containerCounterValue(phy, logic uint64) []byte { - res := make([]byte, 16) - binary.LittleEndian.PutUint64(res, phy) - binary.LittleEndian.PutUint64(res[8:], logic) +func containerCounterValue(entity ObjectCounters) []byte { + res := make([]byte, 24) + binary.LittleEndian.PutUint64(res, entity.Phy) + binary.LittleEndian.PutUint64(res[8:], entity.Logic) + binary.LittleEndian.PutUint64(res[16:], entity.User) return res } @@ -411,9 +419,30 @@ func parseContainerCounterKey(buf []byte) (cid.ID, error) { } // parseContainerCounterValue return phy, logic values. -func parseContainerCounterValue(buf []byte) (uint64, uint64, error) { - if len(buf) != 16 { - return 0, 0, fmt.Errorf("invalid value length") +func parseContainerCounterValue(buf []byte) (ObjectCounters, error) { + if len(buf) != 24 { + return ObjectCounters{}, fmt.Errorf("invalid value length") } - return binary.LittleEndian.Uint64(buf), binary.LittleEndian.Uint64(buf[8:]), nil + return ObjectCounters{ + Phy: binary.LittleEndian.Uint64(buf), + Logic: binary.LittleEndian.Uint64(buf[8:16]), + User: binary.LittleEndian.Uint64(buf[16:]), + }, nil +} + +func containerObjectCounterInitialized(tx *bbolt.Tx) bool { + b := tx.Bucket(containerCounterBucketName) + if b == nil { + return false + } + k, v := b.Cursor().First() + if k == nil && v == nil { + return true + } + _, err := parseContainerCounterKey(k) + if err != nil { + return false + } + _, err = parseContainerCounterValue(v) + return err == nil } diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 51062f1fe..27d0b37ae 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -24,13 +24,13 @@ func TestCounters(t *testing.T) { db := newDB(t) c, err := db.ObjectCounters() require.NoError(t, err) - require.Zero(t, c.Phy()) - require.Zero(t, c.Logic()) + require.Zero(t, c.Phy) + require.Zero(t, c.Logic) + require.Zero(t, c.User) cc, err := db.ContainerCounters(context.Background()) require.NoError(t, err) - require.Zero(t, len(cc.Physical)) - require.Zero(t, len(cc.Logical)) + require.Zero(t, len(cc.Counts)) }) t.Run("put", func(t *testing.T) { @@ -42,29 +42,30 @@ func TestCounters(t *testing.T) { } var prm meta.PutPrm - expPhy := make(map[cid.ID]uint64) - expLog := make(map[cid.ID]uint64) + exp := make(map[cid.ID]meta.ObjectCounters) for i := 0; i < objCount; i++ { prm.SetObject(oo[i]) cnrID, _ := oo[i].ContainerID() - expPhy[cnrID]++ - expLog[cnrID]++ + c := meta.ObjectCounters{} + exp[cnrID] = meta.ObjectCounters{ + Logic: 1, + Phy: 1, + } _, err := db.Put(context.Background(), prm) require.NoError(t, err) - c, err := db.ObjectCounters() + c, err = db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(i+1), c.Phy()) - require.Equal(t, uint64(i+1), c.Logic()) + require.Equal(t, uint64(i+1), c.Phy) + require.Equal(t, uint64(i+1), c.Logic) cc, err := db.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expPhy, cc.Physical) - require.Equal(t, expLog, cc.Logical) + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) } }) @@ -73,12 +74,13 @@ func TestCounters(t *testing.T) { db := newDB(t) oo := putObjs(t, db, objCount, false) - expPhy := make(map[cid.ID]uint64) - expLog := make(map[cid.ID]uint64) + exp := make(map[cid.ID]meta.ObjectCounters) for _, obj := range oo { cnrID, _ := obj.ContainerID() - expPhy[cnrID]++ - expLog[cnrID]++ + exp[cnrID] = meta.ObjectCounters{ + Logic: 1, + Phy: 1, + } } var prm meta.DeletePrm @@ -92,29 +94,23 @@ func TestCounters(t *testing.T) { c, err := db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(i), c.Phy()) - require.Equal(t, uint64(i), c.Logic()) + require.Equal(t, uint64(i), c.Phy) + require.Equal(t, uint64(i), c.Logic) cnrID, _ := oo[i].ContainerID() - if v, ok := expPhy[cnrID]; ok { - if v == 1 { - delete(expPhy, cnrID) + if v, ok := exp[cnrID]; ok { + v.Phy-- + v.Logic-- + if v.IsZero() { + delete(exp, cnrID) } else { - expPhy[cnrID]-- - } - } - if v, ok := expLog[cnrID]; ok { - if v == 1 { - delete(expLog, cnrID) - } else { - expLog[cnrID]-- + exp[cnrID] = v } } cc, err := db.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expPhy, cc.Physical) - require.Equal(t, expLog, cc.Logical) + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) } }) @@ -123,12 +119,13 @@ func TestCounters(t *testing.T) { db := newDB(t) oo := putObjs(t, db, objCount, false) - expPhy := make(map[cid.ID]uint64) - expLog := make(map[cid.ID]uint64) + exp := make(map[cid.ID]meta.ObjectCounters) for _, obj := range oo { cnrID, _ := obj.ContainerID() - expPhy[cnrID]++ - expLog[cnrID]++ + exp[cnrID] = meta.ObjectCounters{ + Logic: 1, + Phy: 1, + } } inhumedObjs := make([]oid.Address, objCount/2) @@ -142,11 +139,12 @@ func TestCounters(t *testing.T) { } for _, addr := range inhumedObjs { - if v, ok := expLog[addr.Container()]; ok { - if v == 1 { - delete(expLog, addr.Container()) + if v, ok := exp[addr.Container()]; ok { + v.Logic-- + if v.IsZero() { + delete(exp, addr.Container()) } else { - expLog[addr.Container()]-- + exp[addr.Container()] = v } } } @@ -162,14 +160,13 @@ func TestCounters(t *testing.T) { c, err := db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(objCount), c.Phy()) - require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic()) + require.Equal(t, uint64(objCount), c.Phy) + require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic) cc, err := db.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expPhy, cc.Physical) - require.Equal(t, expLog, cc.Logical) + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) }) t.Run("put_split", func(t *testing.T) { @@ -177,8 +174,7 @@ func TestCounters(t *testing.T) { db := newDB(t) parObj := testutil.GenerateObject() - expPhy := make(map[cid.ID]uint64) - expLog := make(map[cid.ID]uint64) + exp := make(map[cid.ID]meta.ObjectCounters) // put objects and check that parent info // does not affect the counter @@ -189,20 +185,21 @@ func TestCounters(t *testing.T) { } cnrID, _ := o.ContainerID() - expLog[cnrID]++ - expPhy[cnrID]++ + exp[cnrID] = meta.ObjectCounters{ + Logic: 1, + Phy: 1, + } require.NoError(t, putBig(db, o)) c, err := db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(i+1), c.Phy()) - require.Equal(t, uint64(i+1), c.Logic()) + require.Equal(t, uint64(i+1), c.Phy) + require.Equal(t, uint64(i+1), c.Logic) cc, err := db.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expPhy, cc.Physical) - require.Equal(t, expLog, cc.Logical) + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) } }) @@ -211,12 +208,13 @@ func TestCounters(t *testing.T) { db := newDB(t) oo := putObjs(t, db, objCount, true) - expPhy := make(map[cid.ID]uint64) - expLog := make(map[cid.ID]uint64) + exp := make(map[cid.ID]meta.ObjectCounters) for _, obj := range oo { cnrID, _ := obj.ContainerID() - expPhy[cnrID]++ - expLog[cnrID]++ + exp[cnrID] = meta.ObjectCounters{ + Logic: 1, + Phy: 1, + } } // delete objects that have parent info @@ -228,21 +226,15 @@ func TestCounters(t *testing.T) { c, err := db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(objCount-i-1), c.Phy()) - require.Equal(t, uint64(objCount-i-1), c.Logic()) + require.Equal(t, uint64(objCount-i-1), c.Phy) - if v, ok := expPhy[addr.Container()]; ok { - if v == 1 { - delete(expPhy, addr.Container()) + if v, ok := exp[addr.Container()]; ok { + v.Logic-- + v.Phy-- + if v.IsZero() { + delete(exp, addr.Container()) } else { - expPhy[addr.Container()]-- - } - } - if v, ok := expLog[addr.Container()]; ok { - if v == 1 { - delete(expLog, addr.Container()) - } else { - expLog[addr.Container()]-- + exp[addr.Container()] = v } } } @@ -253,12 +245,13 @@ func TestCounters(t *testing.T) { db := newDB(t) oo := putObjs(t, db, objCount, true) - expPhy := make(map[cid.ID]uint64) - expLog := make(map[cid.ID]uint64) + exp := make(map[cid.ID]meta.ObjectCounters) for _, obj := range oo { cnrID, _ := obj.ContainerID() - expPhy[cnrID]++ - expLog[cnrID]++ + exp[cnrID] = meta.ObjectCounters{ + Logic: 1, + Phy: 1, + } } inhumedObjs := make([]oid.Address, objCount/2) @@ -272,11 +265,12 @@ func TestCounters(t *testing.T) { } for _, addr := range inhumedObjs { - if v, ok := expLog[addr.Container()]; ok { - if v == 1 { - delete(expLog, addr.Container()) + if v, ok := exp[addr.Container()]; ok { + v.Logic-- + if v.IsZero() { + delete(exp, addr.Container()) } else { - expLog[addr.Container()]-- + exp[addr.Container()] = v } } } @@ -291,14 +285,13 @@ func TestCounters(t *testing.T) { c, err := db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(objCount), c.Phy()) - require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic()) + require.Equal(t, uint64(objCount), c.Phy) + require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic) cc, err := db.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expPhy, cc.Physical) - require.Equal(t, expLog, cc.Logical) + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) }) } @@ -320,25 +313,25 @@ func TestCounters_Expired(t *testing.T) { oo[i] = putWithExpiration(t, db, objectSDK.TypeRegular, epoch+1) } - expPhy := make(map[cid.ID]uint64) - expLog := make(map[cid.ID]uint64) + exp := make(map[cid.ID]meta.ObjectCounters) for _, addr := range oo { - expPhy[addr.Container()]++ - expLog[addr.Container()]++ + exp[addr.Container()] = meta.ObjectCounters{ + Logic: 1, + Phy: 1, + } } // 1. objects are available and counters are correct c, err := db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(objCount), c.Phy()) - require.Equal(t, uint64(objCount), c.Logic()) + require.Equal(t, uint64(objCount), c.Phy) + require.Equal(t, uint64(objCount), c.Logic) cc, err := db.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expPhy, cc.Physical) - require.Equal(t, expLog, cc.Logical) + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) for _, o := range oo { _, err := metaGet(db, o, true) @@ -352,14 +345,13 @@ func TestCounters_Expired(t *testing.T) { c, err = db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(objCount), c.Phy()) - require.Equal(t, uint64(objCount), c.Logic()) + require.Equal(t, uint64(objCount), c.Phy) + require.Equal(t, uint64(objCount), c.Logic) cc, err = db.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expPhy, cc.Physical) - require.Equal(t, expLog, cc.Logical) + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) for _, o := range oo { _, err := metaGet(db, o, true) @@ -381,22 +373,22 @@ func TestCounters_Expired(t *testing.T) { c, err = db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(len(oo)), c.Phy()) - require.Equal(t, uint64(len(oo)-1), c.Logic()) + require.Equal(t, uint64(len(oo)), c.Phy) + require.Equal(t, uint64(len(oo)-1), c.Logic) - if v, ok := expLog[oo[0].Container()]; ok { - if v == 1 { - delete(expLog, oo[0].Container()) + if v, ok := exp[oo[0].Container()]; ok { + v.Logic-- + if v.IsZero() { + delete(exp, oo[0].Container()) } else { - expLog[oo[0].Container()]-- + exp[oo[0].Container()] = v } } cc, err = db.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expPhy, cc.Physical) - require.Equal(t, expLog, cc.Logical) + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) // 4. `Delete` an object with GCMark should decrease the // phy counter but does not affect the logic counter (after @@ -409,11 +401,12 @@ func TestCounters_Expired(t *testing.T) { require.NoError(t, err) require.Zero(t, deleteRes.AvailableObjectsRemoved()) - if v, ok := expPhy[oo[0].Container()]; ok { - if v == 1 { - delete(expPhy, oo[0].Container()) + if v, ok := exp[oo[0].Container()]; ok { + v.Phy-- + if v.IsZero() { + delete(exp, oo[0].Container()) } else { - expPhy[oo[0].Container()]-- + exp[oo[0].Container()] = v } } @@ -421,14 +414,13 @@ func TestCounters_Expired(t *testing.T) { c, err = db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(len(oo)), c.Phy()) - require.Equal(t, uint64(len(oo)), c.Logic()) + require.Equal(t, uint64(len(oo)), c.Phy) + require.Equal(t, uint64(len(oo)), c.Logic) cc, err = db.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expPhy, cc.Physical) - require.Equal(t, expLog, cc.Logical) + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) // 5 `Delete` an expired object (like it would the control // service do) should decrease both counters despite the @@ -440,19 +432,13 @@ func TestCounters_Expired(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(1), deleteRes.AvailableObjectsRemoved()) - if v, ok := expLog[oo[0].Container()]; ok { - if v == 1 { - delete(expLog, oo[0].Container()) + if v, ok := exp[oo[0].Container()]; ok { + v.Phy-- + v.Logic-- + if v.IsZero() { + delete(exp, oo[0].Container()) } else { - expLog[oo[0].Container()]-- - } - } - - if v, ok := expPhy[oo[0].Container()]; ok { - if v == 1 { - delete(expPhy, oo[0].Container()) - } else { - expPhy[oo[0].Container()]-- + exp[oo[0].Container()] = v } } @@ -460,14 +446,13 @@ func TestCounters_Expired(t *testing.T) { c, err = db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(len(oo)), c.Phy()) - require.Equal(t, uint64(len(oo)), c.Logic()) + require.Equal(t, uint64(len(oo)), c.Phy) + require.Equal(t, uint64(len(oo)), c.Logic) cc, err = db.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expPhy, cc.Physical) - require.Equal(t, expLog, cc.Logical) + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) } func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*objectSDK.Object { @@ -491,8 +476,8 @@ func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*objectSDK c, err := db.ObjectCounters() require.NoError(t, err) - require.Equal(t, uint64(i+1), c.Phy()) - require.Equal(t, uint64(i+1), c.Logic()) + require.Equal(t, uint64(i+1), c.Phy) + require.Equal(t, uint64(i+1), c.Logic) } return oo diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 5acd71164..a2100f363 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -139,11 +139,11 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) if removed { if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { - v.phy++ + v.Phy++ res.removedByCnrID[addrs[i].Container()] = v } else { res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ - phy: 1, + Phy: 1, } } @@ -153,11 +153,11 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) if available { if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { - v.logic++ + v.Logic++ res.removedByCnrID[addrs[i].Container()] = v } else { res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ - logic: 1, + Logic: 1, } } diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index cf7921992..2a9adff3a 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -82,11 +82,11 @@ func (i *InhumeRes) storeDeletionInfo(containerID cid.ID, deletedSize uint64) { }) i.availableInhumed++ if v, ok := i.inhumedByCnrID[containerID]; ok { - v.logic++ + v.Logic++ i.inhumedByCnrID[containerID] = v } else { i.inhumedByCnrID[containerID] = ObjectCounters{ - logic: 1, + Logic: 1, } } } @@ -251,10 +251,10 @@ func (db *DB) applyInhumeResToCounters(tx *bbolt.Tx, res *InhumeRes) error { inhumedbyCnr := make(map[cid.ID]ObjectCounters) for _, dd := range res.deletionDetails { if v, ok := inhumedbyCnr[dd.CID]; ok { - v.logic++ + v.Logic++ inhumedbyCnr[dd.CID] = v } else { - inhumedbyCnr[dd.CID] = ObjectCounters{logic: 1} + inhumedbyCnr[dd.CID] = ObjectCounters{Logic: 1} } inhumedCount++ } diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 749229cc3..f08af6fc7 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -347,8 +347,8 @@ func TestRefillMetabase(t *testing.T) { c, err := sh.metaBase.ObjectCounters() require.NoError(t, err) - phyBefore := c.Phy() - logicalBefore := c.Logic() + phyBefore := c.Phy + logicalBefore := c.Logic err = sh.Close() require.NoError(t, err) @@ -382,8 +382,8 @@ func TestRefillMetabase(t *testing.T) { c, err = sh.metaBase.ObjectCounters() require.NoError(t, err) - require.Equal(t, phyBefore, c.Phy()) - require.Equal(t, logicalBefore, c.Logic()) + require.Equal(t, phyBefore, c.Phy) + require.Equal(t, logicalBefore, c.Logic) checkAllObjs(true) checkObj(object.AddressOf(tombObj), tombObj) diff --git a/pkg/local_object_storage/shard/count.go b/pkg/local_object_storage/shard/count.go index abed5278e..b3bc6a30b 100644 --- a/pkg/local_object_storage/shard/count.go +++ b/pkg/local_object_storage/shard/count.go @@ -27,5 +27,5 @@ func (s *Shard) LogicalObjectsCount(ctx context.Context) (uint64, error) { if err != nil { return 0, err } - return cc.Logic(), nil + return cc.Logic, nil } diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 1e9bbf9fc..138009a7c 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -178,8 +178,6 @@ func TestCounters(t *testing.T) { oo[i] = testutil.GenerateObject() } - cc := meta.ContainerCounters{Logical: make(map[cid.ID]uint64), Physical: make(map[cid.ID]uint64)} - t.Run("defaults", func(t *testing.T) { require.Zero(t, mm.getObjectCounter(physical)) require.Zero(t, mm.getObjectCounter(logical)) @@ -200,15 +198,16 @@ func TestCounters(t *testing.T) { var totalPayload int64 expectedLogicalSizes := make(map[string]int64) - expectedLogCC := make(map[cid.ID]uint64) - expectedPhyCC := make(map[cid.ID]uint64) + expected := make(map[cid.ID]meta.ObjectCounters) for i := range oo { cnr, _ := oo[i].ContainerID() oSize := int64(oo[i].PayloadSize()) expectedLogicalSizes[cnr.EncodeToString()] += oSize totalPayload += oSize - expectedLogCC[cnr]++ - expectedPhyCC[cnr]++ + expected[cnr] = meta.ObjectCounters{ + Logic: 1, + Phy: 1, + } } var prm PutPrm @@ -227,8 +226,7 @@ func TestCounters(t *testing.T) { cc, err := sh.metaBase.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expectedLogCC, cc.Logical) - require.Equal(t, expectedPhyCC, cc.Physical) + require.Equal(t, meta.ContainerCounters{Counts: expected}, cc) t.Run("inhume_GC", func(t *testing.T) { var prm InhumePrm @@ -244,9 +242,13 @@ func TestCounters(t *testing.T) { require.True(t, ok) expectedLogicalSizes[cid.EncodeToString()] -= int64(oo[i].PayloadSize()) - expectedLogCC[cid]-- - if expectedLogCC[cid] == 0 { - delete(expectedLogCC, cid) + if v, ok := expected[cid]; ok { + v.Logic-- + if v.IsZero() { + delete(expected, cid) + } else { + expected[cid] = v + } } } @@ -257,8 +259,7 @@ func TestCounters(t *testing.T) { cc, err := sh.metaBase.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expectedLogCC, cc.Logical) - require.Equal(t, expectedPhyCC, cc.Physical) + require.Equal(t, meta.ContainerCounters{Counts: expected}, cc) oo = oo[inhumedNumber:] }) @@ -281,9 +282,13 @@ func TestCounters(t *testing.T) { require.True(t, ok) expectedLogicalSizes[cid.EncodeToString()] -= int64(oo[i].PayloadSize()) - expectedLogCC[cid]-- - if expectedLogCC[cid] == 0 { - delete(expectedLogCC, cid) + if v, ok := expected[cid]; ok { + v.Logic-- + if v.IsZero() { + delete(expected, cid) + } else { + expected[cid] = v + } } } @@ -294,8 +299,7 @@ func TestCounters(t *testing.T) { cc, err = sh.metaBase.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expectedLogCC, cc.Logical) - require.Equal(t, expectedPhyCC, cc.Physical) + require.Equal(t, meta.ContainerCounters{Counts: expected}, cc) oo = oo[inhumedNumber:] }) @@ -322,14 +326,14 @@ func TestCounters(t *testing.T) { cnr, _ := oo[i].ContainerID() expectedLogicalSizes[cnr.EncodeToString()] -= int64(removedPayload) - expectedLogCC[cnr]-- - if expectedLogCC[cnr] == 0 { - delete(expectedLogCC, cnr) - } - - expectedPhyCC[cnr]-- - if expectedPhyCC[cnr] == 0 { - delete(expectedPhyCC, cnr) + if v, ok := expected[cnr]; ok { + v.Logic-- + v.Phy-- + if v.IsZero() { + delete(expected, cnr) + } else { + expected[cnr] = v + } } } require.Equal(t, expectedLogicalSizes, mm.containerSizes()) @@ -337,8 +341,7 @@ func TestCounters(t *testing.T) { cc, err = sh.metaBase.ContainerCounters(context.Background()) require.NoError(t, err) - require.Equal(t, expectedLogCC, cc.Logical) - require.Equal(t, expectedPhyCC, cc.Physical) + require.Equal(t, meta.ContainerCounters{Counts: expected}, cc) }) } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index ff437d7ee..853a87b9f 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -395,6 +395,8 @@ const ( // counter type (excludes objects that are // stored but unavailable). logical = "logic" + // user is an available small or big regular object. + user = "user" ) func (s *Shard) updateMetrics(ctx context.Context) { @@ -411,8 +413,9 @@ func (s *Shard) updateMetrics(ctx context.Context) { return } - s.cfg.metricsWriter.SetObjectCounter(physical, cc.Phy()) - s.cfg.metricsWriter.SetObjectCounter(logical, cc.Logic()) + s.cfg.metricsWriter.SetObjectCounter(physical, cc.Phy) + s.cfg.metricsWriter.SetObjectCounter(logical, cc.Logic) + s.cfg.metricsWriter.SetObjectCounter(user, cc.User) cnrList, err := s.metaBase.Containers(ctx) if err != nil { @@ -441,11 +444,10 @@ func (s *Shard) updateMetrics(ctx context.Context) { s.log.Warn(logs.FailedToGetContainerCounters, zap.Error(err)) return } - for contID, count := range contCount.Physical { - s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), physical, count) - } - for contID, count := range contCount.Logical { - s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), logical, count) + for contID, count := range contCount.Counts { + s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), physical, count.Phy) + s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), logical, count.Logic) + s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), user, count.User) } } @@ -472,8 +474,9 @@ func (s *Shard) decContainerObjectCounter(byCnr map[cid.ID]meta.ObjectCounters) } for cnrID, count := range byCnr { - s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), physical, count.Phy()) - s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), logical, count.Logic()) + s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), physical, count.Phy) + s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), logical, count.Logic) + s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), user, count.User) } } From d30ab5f29ea2bef513cff00af10edc915a582b9a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 5 Dec 2023 17:00:16 +0300 Subject: [PATCH 0714/1943] [#838] metabase: Count user objects Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/shards.go | 4 -- pkg/local_object_storage/metabase/counter.go | 48 +++++++++++--- .../metabase/counter_test.go | 30 +++++++++ pkg/local_object_storage/metabase/delete.go | 63 +++++++++++++++---- pkg/local_object_storage/metabase/inhume.go | 49 +++++++++------ .../metabase/iterators.go | 7 ++- pkg/local_object_storage/metabase/put.go | 2 +- pkg/local_object_storage/shard/delete.go | 1 + pkg/local_object_storage/shard/gc.go | 3 + pkg/local_object_storage/shard/inhume.go | 1 + .../shard/metrics_test.go | 19 ++++-- pkg/local_object_storage/shard/put.go | 2 +- pkg/local_object_storage/shard/shard.go | 9 +-- 13 files changed, 180 insertions(+), 58 deletions(-) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 2e1cc7515..5d7fb598e 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -46,10 +46,6 @@ func (m *metricsWithID) IncObjectCounter(objectType string) { m.mw.AddToObjectCounter(m.id, objectType, +1) } -func (m *metricsWithID) DecObjectCounter(objectType string) { - m.mw.AddToObjectCounter(m.id, objectType, -1) -} - func (m *metricsWithID) SetMode(mode mode.Mode) { m.mw.SetMode(m.id, mode) } diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index be860830a..96d80077d 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -27,6 +28,7 @@ const ( _ objectType = iota phy logical + user ) // ObjectCounters groups object counter @@ -183,14 +185,19 @@ func (db *DB) containerCountersNextBatch(lastKey []byte, cc *ContainerCounters) return false, nil } -func (db *DB) incCounters(tx *bbolt.Tx, cnrID cid.ID) error { +func (db *DB) incCounters(tx *bbolt.Tx, cnrID cid.ID, isUserObject bool) error { if err := db.updateShardObjectCounter(tx, phy, 1, true); err != nil { return fmt.Errorf("could not increase phy object counter: %w", err) } if err := db.updateShardObjectCounter(tx, logical, 1, true); err != nil { return fmt.Errorf("could not increase logical object counter: %w", err) } - return db.incContainerObjectCounter(tx, cnrID) + if isUserObject { + if err := db.updateShardObjectCounter(tx, user, 1, true); err != nil { + return fmt.Errorf("could not increase user object counter: %w", err) + } + } + return db.incContainerObjectCounter(tx, cnrID, isUserObject) } func (db *DB) updateShardObjectCounter(tx *bbolt.Tx, typ objectType, delta uint64, inc bool) error { @@ -207,6 +214,8 @@ func (db *DB) updateShardObjectCounter(tx *bbolt.Tx, typ objectType, delta uint6 counterKey = objectPhyCounterKey case logical: counterKey = objectLogicCounterKey + case user: + counterKey = objectUserCounterKey default: panic("unknown object type counter") } @@ -230,7 +239,7 @@ func (db *DB) updateShardObjectCounter(tx *bbolt.Tx, typ objectType, delta uint6 return b.Put(counterKey, newCounter) } -func (db *DB) updateContainerCounter(tx *bbolt.Tx, delta map[cid.ID]ObjectCounters, inc bool) error { +func (db *DB) updateContainerCounter(tx *bbolt.Tx, delta map[cid.ID]ObjectCounters, inc bool) error { // TODO #838 b := tx.Bucket(containerCounterBucketName) if b == nil { return nil @@ -277,7 +286,7 @@ func nextValue(existed, delta uint64, inc bool) uint64 { return existed } -func (db *DB) incContainerObjectCounter(tx *bbolt.Tx, cnrID cid.ID) error { +func (db *DB) incContainerObjectCounter(tx *bbolt.Tx, cnrID cid.ID, isUserObject bool) error { b := tx.Bucket(containerCounterBucketName) if b == nil { return nil @@ -285,7 +294,11 @@ func (db *DB) incContainerObjectCounter(tx *bbolt.Tx, cnrID cid.ID) error { key := make([]byte, cidSize) cnrID.Encode(key) - return db.editContainerCounterValue(b, key, ObjectCounters{Logic: 1, Phy: 1}, true) + c := ObjectCounters{Logic: 1, Phy: 1} + if isUserObject { + c.User = 1 + } + return db.editContainerCounterValue(b, key, c, true) } // syncCounter updates object counters according to metabase state: @@ -319,8 +332,9 @@ func syncCounter(tx *bbolt.Tx, force bool) error { graveyardBKT := tx.Bucket(graveyardBucketName) garbageBKT := tx.Bucket(garbageBucketName) key := make([]byte, addressKeySize) + var isAvailable bool - err = iteratePhyObjects(tx, func(cnr cid.ID, obj oid.ID) error { + err = iteratePhyObjects(tx, func(cnr cid.ID, objID oid.ID, obj *objectSDK.Object) error { if v, ok := counters[cnr]; ok { v.Phy++ counters[cnr] = v @@ -331,7 +345,8 @@ func syncCounter(tx *bbolt.Tx, force bool) error { } addr.SetContainer(cnr) - addr.SetObject(obj) + addr.SetObject(objID) + isAvailable = false // check if an object is available: not with GCMark // and not covered with a tombstone @@ -344,6 +359,18 @@ func syncCounter(tx *bbolt.Tx, force bool) error { Logic: 1, } } + isAvailable = true + } + + if isAvailable && IsUserObject(obj) { + if v, ok := counters[cnr]; ok { + v.User++ + counters[cnr] = v + } else { + counters[cnr] = ObjectCounters{ + User: 1, + } + } } return nil @@ -446,3 +473,10 @@ func containerObjectCounterInitialized(tx *bbolt.Tx) bool { _, err = parseContainerCounterValue(v) return err == nil } + +func IsUserObject(obj *objectSDK.Object) bool { + _, hasParentID := obj.ParentID() + return obj.Type() == objectSDK.TypeRegular && + (obj.SplitID() == nil || + (hasParentID && len(obj.Children()) == 0)) +} diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 27d0b37ae..9241c97a0 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -51,6 +51,7 @@ func TestCounters(t *testing.T) { exp[cnrID] = meta.ObjectCounters{ Logic: 1, Phy: 1, + User: 1, } _, err := db.Put(context.Background(), prm) @@ -80,6 +81,7 @@ func TestCounters(t *testing.T) { exp[cnrID] = meta.ObjectCounters{ Logic: 1, Phy: 1, + User: 1, } } @@ -96,11 +98,13 @@ func TestCounters(t *testing.T) { require.Equal(t, uint64(i), c.Phy) require.Equal(t, uint64(i), c.Logic) + require.Equal(t, uint64(i), c.User) cnrID, _ := oo[i].ContainerID() if v, ok := exp[cnrID]; ok { v.Phy-- v.Logic-- + v.User-- if v.IsZero() { delete(exp, cnrID) } else { @@ -125,6 +129,7 @@ func TestCounters(t *testing.T) { exp[cnrID] = meta.ObjectCounters{ Logic: 1, Phy: 1, + User: 1, } } @@ -141,6 +146,7 @@ func TestCounters(t *testing.T) { for _, addr := range inhumedObjs { if v, ok := exp[addr.Container()]; ok { v.Logic-- + v.User-- if v.IsZero() { delete(exp, addr.Container()) } else { @@ -156,12 +162,14 @@ func TestCounters(t *testing.T) { res, err := db.Inhume(context.Background(), prm) require.NoError(t, err) require.Equal(t, uint64(len(inhumedObjs)), res.AvailableInhumed()) + require.Equal(t, uint64(len(inhumedObjs)), res.UserInhumed()) c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(objCount), c.Phy) require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic) + require.Equal(t, uint64(objCount-len(inhumedObjs)), c.User) cc, err := db.ContainerCounters(context.Background()) require.NoError(t, err) @@ -182,12 +190,14 @@ func TestCounters(t *testing.T) { o := testutil.GenerateObject() if i < objCount/2 { // half of the objs will have the parent o.SetParent(parObj) + o.SetSplitID(objectSDK.NewSplitID()) } cnrID, _ := o.ContainerID() exp[cnrID] = meta.ObjectCounters{ Logic: 1, Phy: 1, + User: 1, } require.NoError(t, putBig(db, o)) @@ -196,6 +206,7 @@ func TestCounters(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(i+1), c.Phy) require.Equal(t, uint64(i+1), c.Logic) + require.Equal(t, uint64(i+1), c.User) cc, err := db.ContainerCounters(context.Background()) require.NoError(t, err) @@ -214,6 +225,7 @@ func TestCounters(t *testing.T) { exp[cnrID] = meta.ObjectCounters{ Logic: 1, Phy: 1, + User: 1, } } @@ -227,10 +239,13 @@ func TestCounters(t *testing.T) { c, err := db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(objCount-i-1), c.Phy) + require.Equal(t, uint64(objCount-i-1), c.Logic) + require.Equal(t, uint64(objCount-i-1), c.User) if v, ok := exp[addr.Container()]; ok { v.Logic-- v.Phy-- + v.User-- if v.IsZero() { delete(exp, addr.Container()) } else { @@ -251,6 +266,7 @@ func TestCounters(t *testing.T) { exp[cnrID] = meta.ObjectCounters{ Logic: 1, Phy: 1, + User: 1, } } @@ -267,6 +283,7 @@ func TestCounters(t *testing.T) { for _, addr := range inhumedObjs { if v, ok := exp[addr.Container()]; ok { v.Logic-- + v.User-- if v.IsZero() { delete(exp, addr.Container()) } else { @@ -287,6 +304,7 @@ func TestCounters(t *testing.T) { require.Equal(t, uint64(objCount), c.Phy) require.Equal(t, uint64(objCount-len(inhumedObjs)), c.Logic) + require.Equal(t, uint64(objCount-len(inhumedObjs)), c.User) cc, err := db.ContainerCounters(context.Background()) require.NoError(t, err) @@ -318,6 +336,7 @@ func TestCounters_Expired(t *testing.T) { exp[addr.Container()] = meta.ObjectCounters{ Logic: 1, Phy: 1, + User: 1, } } @@ -327,6 +346,7 @@ func TestCounters_Expired(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(objCount), c.Phy) require.Equal(t, uint64(objCount), c.Logic) + require.Equal(t, uint64(objCount), c.User) cc, err := db.ContainerCounters(context.Background()) require.NoError(t, err) @@ -347,6 +367,7 @@ func TestCounters_Expired(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(objCount), c.Phy) require.Equal(t, uint64(objCount), c.Logic) + require.Equal(t, uint64(objCount), c.User) cc, err = db.ContainerCounters(context.Background()) require.NoError(t, err) @@ -369,15 +390,18 @@ func TestCounters_Expired(t *testing.T) { inhumeRes, err := db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) require.Equal(t, uint64(1), inhumeRes.AvailableInhumed()) + require.Equal(t, uint64(1), inhumeRes.UserInhumed()) c, err = db.ObjectCounters() require.NoError(t, err) require.Equal(t, uint64(len(oo)), c.Phy) require.Equal(t, uint64(len(oo)-1), c.Logic) + require.Equal(t, uint64(len(oo)-1), c.User) if v, ok := exp[oo[0].Container()]; ok { v.Logic-- + v.User-- if v.IsZero() { delete(exp, oo[0].Container()) } else { @@ -400,6 +424,7 @@ func TestCounters_Expired(t *testing.T) { deleteRes, err := db.Delete(context.Background(), deletePrm) require.NoError(t, err) require.Zero(t, deleteRes.AvailableObjectsRemoved()) + require.Zero(t, deleteRes.UserObjectsRemoved()) if v, ok := exp[oo[0].Container()]; ok { v.Phy-- @@ -416,6 +441,7 @@ func TestCounters_Expired(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(len(oo)), c.Phy) require.Equal(t, uint64(len(oo)), c.Logic) + require.Equal(t, uint64(len(oo)), c.User) cc, err = db.ContainerCounters(context.Background()) require.NoError(t, err) @@ -431,10 +457,12 @@ func TestCounters_Expired(t *testing.T) { deleteRes, err = db.Delete(context.Background(), deletePrm) require.NoError(t, err) require.Equal(t, uint64(1), deleteRes.AvailableObjectsRemoved()) + require.Equal(t, uint64(1), deleteRes.UserObjectsRemoved()) if v, ok := exp[oo[0].Container()]; ok { v.Phy-- v.Logic-- + v.User-- if v.IsZero() { delete(exp, oo[0].Container()) } else { @@ -448,6 +476,7 @@ func TestCounters_Expired(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(len(oo)), c.Phy) require.Equal(t, uint64(len(oo)), c.Logic) + require.Equal(t, uint64(len(oo)), c.User) cc, err = db.ContainerCounters(context.Background()) require.NoError(t, err) @@ -465,6 +494,7 @@ func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*objectSDK o := testutil.GenerateObject() if withParent { o.SetParent(parent) + o.SetSplitID(objectSDK.NewSplitID()) } oo = append(oo, o) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index a2100f363..4fea4b4f2 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -29,6 +29,7 @@ type DeletePrm struct { type DeleteRes struct { rawRemoved uint64 availableRemoved uint64 + userRemoved uint64 sizes []uint64 availableSizes []uint64 removedByCnrID map[cid.ID]ObjectCounters @@ -40,6 +41,10 @@ func (d DeleteRes) AvailableObjectsRemoved() uint64 { return d.availableRemoved } +func (d DeleteRes) UserObjectsRemoved() uint64 { + return d.userRemoved +} + // RemovedByCnrID returns the number of removed objects by container ID. func (d DeleteRes) RemovedByCnrID() map[cid.ID]ObjectCounters { return d.removedByCnrID @@ -132,12 +137,12 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) currEpoch := db.epochState.CurrentEpoch() for i := range addrs { - removed, available, size, err := db.delete(tx, addrs[i], refCounter, currEpoch) + r, err := db.delete(tx, addrs[i], refCounter, currEpoch) if err != nil { return DeleteRes{}, err // maybe log and continue? } - if removed { + if r.Removed { if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { v.Phy++ res.removedByCnrID[addrs[i].Container()] = v @@ -148,10 +153,10 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) } res.rawRemoved++ - res.sizes[i] = size + res.sizes[i] = r.Size } - if available { + if r.Available { if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { v.Logic++ res.removedByCnrID[addrs[i].Container()] = v @@ -162,7 +167,20 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) } res.availableRemoved++ - res.availableSizes[i] = size + res.availableSizes[i] = r.Size + } + + if r.User { + if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { + v.User++ + res.removedByCnrID[addrs[i].Container()] = v + } else { + res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ + User: 1, + } + } + + res.userRemoved++ } } @@ -180,6 +198,13 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) } } + if res.userRemoved > 0 { + err := db.updateShardObjectCounter(tx, user, res.userRemoved, false) + if err != nil { + return DeleteRes{}, fmt.Errorf("could not decrease user object counter: %w", err) + } + } + if err := db.updateContainerCounter(tx, res.removedByCnrID, false); err != nil { return DeleteRes{}, fmt.Errorf("could not decrease container object counter: %w", err) } @@ -196,13 +221,20 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) return res, nil } +type deleteSingleResult struct { + Removed bool + Available bool + User bool + Size uint64 +} + // delete removes object indexes from the metabase. Counts the references // of the object that is being removed. // The first return value indicates if an object has been removed. (removing a // non-exist object is error-free). The second return value indicates if an // object was available before the removal (for calculating the logical object -// counter). The third return value is removed object payload size. -func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter, currEpoch uint64) (bool, bool, uint64, error) { +// counter). The third return value The fourth return value is removed object payload size. +func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter, currEpoch uint64) (deleteSingleResult, error) { key := make([]byte, addressKeySize) addrKey := addressKey(addr, key) garbageBKT := tx.Bucket(garbageBucketName) @@ -214,7 +246,7 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter if garbageBKT != nil { err := garbageBKT.Delete(addrKey) if err != nil { - return false, false, 0, fmt.Errorf("could not remove from garbage bucket: %w", err) + return deleteSingleResult{}, fmt.Errorf("could not remove from garbage bucket: %w", err) } } @@ -224,10 +256,10 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter var siErr *objectSDK.SplitInfoError if client.IsErrObjectNotFound(err) || errors.As(err, &siErr) { - return false, false, 0, nil + return deleteSingleResult{}, nil } - return false, false, 0, err + return deleteSingleResult{}, err } // if object is an only link to a parent, then remove parent @@ -250,13 +282,20 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter nRef.cur++ } + isUserObject := IsUserObject(obj) + // remove object err = db.deleteObject(tx, obj, false) if err != nil { - return false, false, 0, fmt.Errorf("could not remove object: %w", err) + return deleteSingleResult{}, fmt.Errorf("could not remove object: %w", err) } - return true, removeAvailableObject, obj.PayloadSize(), nil + return deleteSingleResult{ + Removed: true, + Available: removeAvailableObject, + User: isUserObject && removeAvailableObject, + Size: obj.PayloadSize(), + }, nil } func (db *DB) deleteObject( diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 2a9adff3a..21db9a629 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -30,14 +30,16 @@ type InhumePrm struct { // DeletionInfo contains details on deleted object. type DeletionInfo struct { - Size uint64 - CID cid.ID + Size uint64 + CID cid.ID + IsUser bool } // InhumeRes encapsulates results of Inhume operation. type InhumeRes struct { deletedLockObj []oid.Address availableInhumed uint64 + userInhumed uint64 inhumedByCnrID map[cid.ID]ObjectCounters deletionDetails []DeletionInfo } @@ -48,6 +50,10 @@ func (i InhumeRes) AvailableInhumed() uint64 { return i.availableInhumed } +func (i InhumeRes) UserInhumed() uint64 { + return i.userInhumed +} + // InhumedByCnrID return number of object // that have been inhumed by container ID. func (i InhumeRes) InhumedByCnrID() map[cid.ID]ObjectCounters { @@ -75,19 +81,31 @@ func (i InhumeRes) GetDeletionInfoByIndex(target int) DeletionInfo { // StoreDeletionInfo stores size of deleted object and associated container ID // in corresponding arrays. -func (i *InhumeRes) storeDeletionInfo(containerID cid.ID, deletedSize uint64) { +func (i *InhumeRes) storeDeletionInfo(containerID cid.ID, deletedSize uint64, isUser bool) { i.deletionDetails = append(i.deletionDetails, DeletionInfo{ - Size: deletedSize, - CID: containerID, + Size: deletedSize, + CID: containerID, + IsUser: isUser, }) i.availableInhumed++ + if isUser { + i.userInhumed++ + } + if v, ok := i.inhumedByCnrID[containerID]; ok { v.Logic++ + if isUser { + v.User++ + } i.inhumedByCnrID[containerID] = v } else { - i.inhumedByCnrID[containerID] = ObjectCounters{ + v = ObjectCounters{ Logic: 1, } + if isUser { + v.User = 1 + } + i.inhumedByCnrID[containerID] = v } } @@ -247,23 +265,14 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes } func (db *DB) applyInhumeResToCounters(tx *bbolt.Tx, res *InhumeRes) error { - var inhumedCount uint64 - inhumedbyCnr := make(map[cid.ID]ObjectCounters) - for _, dd := range res.deletionDetails { - if v, ok := inhumedbyCnr[dd.CID]; ok { - v.Logic++ - inhumedbyCnr[dd.CID] = v - } else { - inhumedbyCnr[dd.CID] = ObjectCounters{Logic: 1} - } - inhumedCount++ + if err := db.updateShardObjectCounter(tx, logical, res.AvailableInhumed(), false); err != nil { + return err } - - if err := db.updateShardObjectCounter(tx, logical, inhumedCount, false); err != nil { + if err := db.updateShardObjectCounter(tx, user, res.UserInhumed(), false); err != nil { return err } - return db.updateContainerCounter(tx, inhumedbyCnr, false) + return db.updateContainerCounter(tx, res.inhumedByCnrID, false) } // getInhumeTargetBucketAndValue return target bucket to store inhume result and value that will be put in the bucket. @@ -318,7 +327,7 @@ func (db *DB) markAsGC(graveyardBKT, garbageBKT *bbolt.Bucket, key []byte) (bool func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Bucket, targetKey []byte, cnr cid.ID, obj *objectSDK.Object, res *InhumeRes) error { containerID, _ := obj.ContainerID() if inGraveyardWithKey(targetKey, graveyardBKT, garbageBKT) == 0 { - res.storeDeletionInfo(containerID, obj.PayloadSize()) + res.storeDeletionInfo(containerID, obj.PayloadSize(), IsUserObject(obj)) } // if object is stored, and it is regular object then update bucket diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index a1e21ef25..7b60b7d50 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -202,9 +202,10 @@ func (db *DB) iterateCoveredByTombstones(tx *bbolt.Tx, tss map[string]oid.Addres return err } -func iteratePhyObjects(tx *bbolt.Tx, f func(cid.ID, oid.ID) error) error { +func iteratePhyObjects(tx *bbolt.Tx, f func(cid.ID, oid.ID, *objectSDK.Object) error) error { var cid cid.ID var oid oid.ID + obj := objectSDK.New() return tx.ForEach(func(name []byte, b *bbolt.Bucket) error { b58CID, postfix := parseContainerIDWithPrefix(&cid, name) @@ -221,8 +222,8 @@ func iteratePhyObjects(tx *bbolt.Tx, f func(cid.ID, oid.ID) error) error { } return b.ForEach(func(k, v []byte) error { - if oid.Decode(k) == nil { - return f(cid, oid) + if oid.Decode(k) == nil && obj.Unmarshal(v) == nil { + return f(cid, oid, obj) } return nil diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 2b5c5421d..2f5108953 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -183,7 +183,7 @@ func (db *DB) insertObject(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *o } if !isParent { - if err = db.incCounters(tx, cnr); err != nil { + if err = db.incCounters(tx, cnr, IsUserObject(obj)); err != nil { return err } } diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 663d781ad..0856bda10 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -118,6 +118,7 @@ func (s *Shard) deleteFromMetabase(ctx context.Context, addr oid.Address) error } s.decObjectCounterBy(physical, res.RawObjectsRemoved()) s.decObjectCounterBy(logical, res.AvailableObjectsRemoved()) + s.decObjectCounterBy(user, res.UserObjectsRemoved()) s.decContainerObjectCounter(res.RemovedByCnrID()) removedPayload := res.RemovedPhysicalObjectSizes()[0] logicalRemovedPayload := res.RemovedLogicalObjectSizes()[0] diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index e16f89457..0f978f26e 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -416,6 +416,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeRegular) s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.decObjectCounterBy(user, res.UserInhumed()) s.decContainerObjectCounter(res.InhumedByCnrID()) i := 0 @@ -630,6 +631,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeTombstone) s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.decObjectCounterBy(user, res.UserInhumed()) s.decContainerObjectCounter(res.InhumedByCnrID()) i := 0 @@ -677,6 +679,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeLock) s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.decObjectCounterBy(user, res.UserInhumed()) s.decContainerObjectCounter(res.InhumedByCnrID()) i := 0 diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 335df82f0..48cfaa98e 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -122,6 +122,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { s.m.RUnlock() s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.decObjectCounterBy(user, res.UserInhumed()) s.decContainerObjectCounter(res.InhumedByCnrID()) i := 0 diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 138009a7c..f59565cb3 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -86,12 +86,6 @@ func (m *metricsStore) IncObjectCounter(objectType string) { m.objCounters[objectType] += 1 } -func (m *metricsStore) DecObjectCounter(objectType string) { - m.mtx.Lock() - defer m.mtx.Unlock() - m.AddToObjectCounter(objectType, -1) -} - func (m *metricsStore) SetMode(mode mode.Mode) { m.mtx.Lock() defer m.mtx.Unlock() @@ -192,6 +186,9 @@ func TestCounters(t *testing.T) { v, ok = mm.getContainerCount(contID.EncodeToString(), logical) require.Zero(t, v) require.False(t, ok) + v, ok = mm.getContainerCount(contID.EncodeToString(), user) + require.Zero(t, v) + require.False(t, ok) } }) @@ -207,6 +204,7 @@ func TestCounters(t *testing.T) { expected[cnr] = meta.ObjectCounters{ Logic: 1, Phy: 1, + User: 1, } } @@ -221,6 +219,7 @@ func TestCounters(t *testing.T) { require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical)) require.Equal(t, uint64(objNumber), mm.getObjectCounter(logical)) + require.Equal(t, uint64(objNumber), mm.getObjectCounter(user)) require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) @@ -244,6 +243,7 @@ func TestCounters(t *testing.T) { if v, ok := expected[cid]; ok { v.Logic-- + v.User-- if v.IsZero() { delete(expected, cid) } else { @@ -254,6 +254,7 @@ func TestCounters(t *testing.T) { require.Equal(t, uint64(objNumber), mm.getObjectCounter(physical)) require.Equal(t, uint64(objNumber-inhumedNumber), mm.getObjectCounter(logical)) + require.Equal(t, uint64(objNumber-inhumedNumber), mm.getObjectCounter(user)) require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) @@ -270,6 +271,7 @@ func TestCounters(t *testing.T) { phy := mm.getObjectCounter(physical) logic := mm.getObjectCounter(logical) + custom := mm.getObjectCounter(user) inhumedNumber := int(phy / 4) prm.SetTarget(ts, addrFromObjs(oo[:inhumedNumber])...) @@ -284,6 +286,7 @@ func TestCounters(t *testing.T) { if v, ok := expected[cid]; ok { v.Logic-- + v.User-- if v.IsZero() { delete(expected, cid) } else { @@ -294,6 +297,7 @@ func TestCounters(t *testing.T) { require.Equal(t, phy, mm.getObjectCounter(physical)) require.Equal(t, logic-uint64(inhumedNumber), mm.getObjectCounter(logical)) + require.Equal(t, custom-uint64(inhumedNumber), mm.getObjectCounter(user)) require.Equal(t, expectedLogicalSizes, mm.containerSizes()) require.Equal(t, totalPayload, mm.payloadSize()) @@ -309,6 +313,7 @@ func TestCounters(t *testing.T) { phy := mm.getObjectCounter(physical) logic := mm.getObjectCounter(logical) + custom := mm.getObjectCounter(user) deletedNumber := int(phy / 4) prm.SetAddresses(addrFromObjs(oo[:deletedNumber])...) @@ -318,6 +323,7 @@ func TestCounters(t *testing.T) { require.Equal(t, phy-uint64(deletedNumber), mm.getObjectCounter(physical)) require.Equal(t, logic-uint64(deletedNumber), mm.getObjectCounter(logical)) + require.Equal(t, custom-uint64(deletedNumber), mm.getObjectCounter(user)) var totalRemovedpayload uint64 for i := range oo[:deletedNumber] { removedPayload := oo[i].PayloadSize() @@ -329,6 +335,7 @@ func TestCounters(t *testing.T) { if v, ok := expected[cnr]; ok { v.Logic-- v.Phy-- + v.User-- if v.IsZero() { delete(expected, cnr) } else { diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index 10983d041..09a98260d 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -90,7 +90,7 @@ func (s *Shard) Put(ctx context.Context, prm PutPrm) (PutRes, error) { return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err) } - s.incObjectCounter(putPrm.Address.Container()) + s.incObjectCounter(putPrm.Address.Container(), meta.IsUserObject(prm.obj)) s.addToPayloadSize(int64(prm.obj.PayloadSize())) s.addToContainerSize(putPrm.Address.Container().EncodeToString(), int64(prm.obj.PayloadSize())) } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 853a87b9f..2c4ce91dc 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -72,9 +72,6 @@ type MetricsWriter interface { // IncObjectCounter must increment shard's object counter taking into account // object type. IncObjectCounter(objectType string) - // DecObjectCounter must decrement shard's object counter taking into account - // object type. - DecObjectCounter(objectType string) // SetShardID must set (update) the shard identifier that will be used in // metrics. SetShardID(id string) @@ -453,12 +450,16 @@ func (s *Shard) updateMetrics(ctx context.Context) { // incObjectCounter increment both physical and logical object // counters. -func (s *Shard) incObjectCounter(cnrID cid.ID) { +func (s *Shard) incObjectCounter(cnrID cid.ID, isUser bool) { if s.cfg.metricsWriter != nil { s.cfg.metricsWriter.IncObjectCounter(physical) s.cfg.metricsWriter.IncObjectCounter(logical) s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), physical) s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), logical) + if isUser { + s.cfg.metricsWriter.IncObjectCounter(user) + s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), user) + } } } From a478050639aac5bb144f5d41b366e26c8956e3c0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 6 Dec 2023 13:02:14 +0300 Subject: [PATCH 0715/1943] [#838] metabase: Resolve funlen linter Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/delete.go | 141 +++++++++++--------- 1 file changed, 76 insertions(+), 65 deletions(-) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 4fea4b4f2..6a5661ed1 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -139,81 +139,21 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) for i := range addrs { r, err := db.delete(tx, addrs[i], refCounter, currEpoch) if err != nil { - return DeleteRes{}, err // maybe log and continue? + return DeleteRes{}, err } - if r.Removed { - if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { - v.Phy++ - res.removedByCnrID[addrs[i].Container()] = v - } else { - res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ - Phy: 1, - } - } - - res.rawRemoved++ - res.sizes[i] = r.Size - } - - if r.Available { - if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { - v.Logic++ - res.removedByCnrID[addrs[i].Container()] = v - } else { - res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ - Logic: 1, - } - } - - res.availableRemoved++ - res.availableSizes[i] = r.Size - } - - if r.User { - if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { - v.User++ - res.removedByCnrID[addrs[i].Container()] = v - } else { - res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ - User: 1, - } - } - - res.userRemoved++ - } + applyDeleteSingleResult(r, &res, addrs, i) } - if res.rawRemoved > 0 { - err := db.updateShardObjectCounter(tx, phy, res.rawRemoved, false) - if err != nil { - return DeleteRes{}, fmt.Errorf("could not decrease phy object counter: %w", err) - } - } - - if res.availableRemoved > 0 { - err := db.updateShardObjectCounter(tx, logical, res.availableRemoved, false) - if err != nil { - return DeleteRes{}, fmt.Errorf("could not decrease logical object counter: %w", err) - } - } - - if res.userRemoved > 0 { - err := db.updateShardObjectCounter(tx, user, res.userRemoved, false) - if err != nil { - return DeleteRes{}, fmt.Errorf("could not decrease user object counter: %w", err) - } - } - - if err := db.updateContainerCounter(tx, res.removedByCnrID, false); err != nil { - return DeleteRes{}, fmt.Errorf("could not decrease container object counter: %w", err) + if err := db.updateCountersDelete(tx, res); err != nil { + return DeleteRes{}, err } for _, refNum := range refCounter { if refNum.cur == refNum.all { err := db.deleteObject(tx, refNum.obj, true) if err != nil { - return DeleteRes{}, err // maybe log and continue? + return DeleteRes{}, err } } } @@ -221,6 +161,77 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) return res, nil } +func (db *DB) updateCountersDelete(tx *bbolt.Tx, res DeleteRes) error { + if res.rawRemoved > 0 { + err := db.updateShardObjectCounter(tx, phy, res.rawRemoved, false) + if err != nil { + return fmt.Errorf("could not decrease phy object counter: %w", err) + } + } + + if res.availableRemoved > 0 { + err := db.updateShardObjectCounter(tx, logical, res.availableRemoved, false) + if err != nil { + return fmt.Errorf("could not decrease logical object counter: %w", err) + } + } + + if res.userRemoved > 0 { + err := db.updateShardObjectCounter(tx, user, res.userRemoved, false) + if err != nil { + return fmt.Errorf("could not decrease user object counter: %w", err) + } + } + + if err := db.updateContainerCounter(tx, res.removedByCnrID, false); err != nil { + return fmt.Errorf("could not decrease container object counter: %w", err) + } + return nil +} + +func applyDeleteSingleResult(r deleteSingleResult, res *DeleteRes, addrs []oid.Address, i int) { + if r.Removed { + if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { + v.Phy++ + res.removedByCnrID[addrs[i].Container()] = v + } else { + res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ + Phy: 1, + } + } + + res.rawRemoved++ + res.sizes[i] = r.Size + } + + if r.Available { + if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { + v.Logic++ + res.removedByCnrID[addrs[i].Container()] = v + } else { + res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ + Logic: 1, + } + } + + res.availableRemoved++ + res.availableSizes[i] = r.Size + } + + if r.User { + if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { + v.User++ + res.removedByCnrID[addrs[i].Container()] = v + } else { + res.removedByCnrID[addrs[i].Container()] = ObjectCounters{ + User: 1, + } + } + + res.userRemoved++ + } +} + type deleteSingleResult struct { Removed bool Available bool From 44552a849b5e975dba7a0576c6ae4d39c8e962fa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 19 Sep 2023 13:50:14 +0300 Subject: [PATCH 0716/1943] [#661] shard: Add blobstor rebuilder Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 4 + pkg/local_object_storage/blobstor/rebuild.go | 15 ++++ pkg/local_object_storage/shard/control.go | 11 +++ pkg/local_object_storage/shard/rebuilder.go | 89 ++++++++++++++++++++ pkg/local_object_storage/shard/shard.go | 2 + 5 files changed, 121 insertions(+) create mode 100644 pkg/local_object_storage/blobstor/rebuild.go create mode 100644 pkg/local_object_storage/shard/rebuilder.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index a72609b41..4b8305e41 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -519,4 +519,8 @@ const ( FailedToCountWritecacheItems = "failed to count writecache items" AttemtToCloseAlreadyClosedBlobovnicza = "attempt to close an already closed blobovnicza" FailedToGetContainerCounters = "failed to get container counters values" + FailedToRebuildBlobstore = "failed to rebuild blobstore" + BlobstoreRebuildStarted = "blobstore rebuild started" + BlobstoreRebuildCompletedSuccessfully = "blobstore rebuild completed successfully" + BlobstoreRebuildStopped = "blobstore rebuild stopped" ) diff --git a/pkg/local_object_storage/blobstor/rebuild.go b/pkg/local_object_storage/blobstor/rebuild.go new file mode 100644 index 000000000..110735fad --- /dev/null +++ b/pkg/local_object_storage/blobstor/rebuild.go @@ -0,0 +1,15 @@ +package blobstor + +import ( + "context" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" +) + +type StorageIDUpdate interface { + UpdateStorageID(ctx context.Context, obj *objectSDK.Object, storageID []byte) error +} + +func (b *BlobStor) Rebuild(_ context.Context, _ StorageIDUpdate) error { + return nil +} diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 257498b32..a988683ad 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -162,6 +162,9 @@ func (s *Shard) Init(ctx context.Context) error { s.gc.init(ctx) + s.rb = newRebuilder() + s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) + return nil } @@ -266,6 +269,9 @@ func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object // Close releases all Shard's components. func (s *Shard) Close() error { + if s.rb != nil { + s.rb.Stop(s.log) + } components := []interface{ Close() error }{} if s.pilorama != nil { @@ -310,6 +316,11 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { unlock := s.lockExclusive() defer unlock() + s.rb.Stop(s.log) + defer func() { + s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) + }() + ok, err := s.metaBase.Reload(c.metaOpts...) if err != nil { if errors.Is(err, meta.ErrDegradedMode) { diff --git a/pkg/local_object_storage/shard/rebuilder.go b/pkg/local_object_storage/shard/rebuilder.go new file mode 100644 index 000000000..3e0c66abf --- /dev/null +++ b/pkg/local_object_storage/shard/rebuilder.go @@ -0,0 +1,89 @@ +package shard + +import ( + "context" + "errors" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.uber.org/zap" +) + +type rebuilder struct { + mtx *sync.Mutex + wg *sync.WaitGroup + cancel func() +} + +func newRebuilder() *rebuilder { + return &rebuilder{ + mtx: &sync.Mutex{}, + wg: &sync.WaitGroup{}, + cancel: nil, + } +} + +func (r *rebuilder) Start(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *logger.Logger) { + r.mtx.Lock() + defer r.mtx.Unlock() + + r.start(ctx, bs, mb, log) +} + +func (r *rebuilder) start(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *logger.Logger) { + if r.cancel != nil { + r.stop(log) + } + ctx, cancel := context.WithCancel(ctx) + r.cancel = cancel + r.wg.Add(1) + go func() { + defer r.wg.Done() + + log.Info(logs.BlobstoreRebuildStarted) + if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}); err != nil { + log.Warn(logs.FailedToRebuildBlobstore, zap.Error(err)) + } else { + log.Info(logs.BlobstoreRebuildCompletedSuccessfully) + } + }() +} + +func (r *rebuilder) Stop(log *logger.Logger) { + r.mtx.Lock() + defer r.mtx.Unlock() + + r.stop(log) +} + +func (r *rebuilder) stop(log *logger.Logger) { + if r.cancel == nil { + return + } + + r.cancel() + r.wg.Wait() + r.cancel = nil + log.Info(logs.BlobstoreRebuildStopped) +} + +var errMBIsNotAvailable = errors.New("metabase is not available") + +type mbStorageIDUpdate struct { + mb *meta.DB +} + +func (u *mbStorageIDUpdate) UpdateStorageID(ctx context.Context, obj *objectSDK.Object, storageID []byte) error { + if u.mb == nil { + return errMBIsNotAvailable + } + var prm meta.PutPrm + prm.SetObject(obj) + prm.SetStorageID(storageID) + _, err := u.mb.Put(ctx, prm) + return err +} diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 2c4ce91dc..915feccb4 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -39,6 +39,8 @@ type Shard struct { tsSource TombstoneSource + rb *rebuilder + gcCancel atomic.Value setModeRequested atomic.Bool } From 484eb59893a09ccae0100aeb16d80e6820ae8837 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 19 Sep 2023 18:08:38 +0300 Subject: [PATCH 0717/1943] [#661] blobovniczatree: Use .db extension for db files Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-lens/internal/meta/inspect.go | 4 +- internal/logs/logs.go | 6 ++ .../blobstor/blobovniczatree/active.go | 2 +- .../blobstor/blobovniczatree/blobovnicza.go | 13 +++- .../blobstor/blobovniczatree/control.go | 47 +++++++++++++ .../blobstor/blobovniczatree/control_test.go | 69 +++++++++++++++++++ .../blobstor/blobovniczatree/delete.go | 4 +- .../blobstor/blobovniczatree/exists.go | 4 +- .../blobstor/blobovniczatree/exists_test.go | 2 +- .../blobstor/blobovniczatree/get.go | 4 +- .../blobstor/blobovniczatree/get_range.go | 4 +- .../blobovniczatree}/id.go | 6 +- .../blobstor/blobovniczatree/iterate.go | 13 +++- .../blobstor/blobovniczatree/manager.go | 2 +- .../blobstor/blobovniczatree/put.go | 4 +- 15 files changed, 160 insertions(+), 24 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/control_test.go rename pkg/local_object_storage/{blobovnicza => blobstor/blobovniczatree}/id.go (71%) diff --git a/cmd/frostfs-lens/internal/meta/inspect.go b/cmd/frostfs-lens/internal/meta/inspect.go index de0f24aeb..9eb60f966 100644 --- a/cmd/frostfs-lens/internal/meta/inspect.go +++ b/cmd/frostfs-lens/internal/meta/inspect.go @@ -5,7 +5,7 @@ import ( "fmt" common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -40,7 +40,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { common.ExitOnErr(cmd, common.Errf("could not check if the obj is small: %w", err)) if id := resStorageID.StorageID(); id != nil { - cmd.Printf("Object storageID: %s\n\n", blobovnicza.NewIDFromBytes(id).String()) + cmd.Printf("Object storageID: %s\n\n", blobovniczatree.NewIDFromBytes(id).Path()) } else { cmd.Printf("Object does not contain storageID\n\n") } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 4b8305e41..0c5e162b3 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -523,4 +523,10 @@ const ( BlobstoreRebuildStarted = "blobstore rebuild started" BlobstoreRebuildCompletedSuccessfully = "blobstore rebuild completed successfully" BlobstoreRebuildStopped = "blobstore rebuild stopped" + BlobovniczaTreeFixingFileExtensions = "fixing blobovnicza tree file extensions..." + BlobovniczaTreeFixingFileExtensionsCompletedSuccessfully = "fixing blobovnicza tree file extensions completed successfully" + BlobovniczaTreeFixingFileExtensionsFailed = "failed to fix blobovnicza tree file extensions" + BlobovniczaTreeFixingFileExtensionForFile = "fixing blobovnicza file extension..." + BlobovniczaTreeFixingFileExtensionCompletedSuccessfully = "fixing blobovnicza file extension completed successfully" + BlobovniczaTreeFixingFileExtensionFailed = "failed to fix blobovnicza file extension" ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/active.go b/pkg/local_object_storage/blobstor/blobovniczatree/active.go index def197318..da8880646 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/active.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/active.go @@ -154,7 +154,7 @@ func (m *activeDBManager) getNextSharedDB(lvlPath string) (*sharedDB, error) { var next *sharedDB for iterCount < m.leafWidth { - path := filepath.Join(lvlPath, u64ToHexString(idx)) + path := filepath.Join(lvlPath, u64ToHexStringExt(idx)) shDB := m.dbManager.GetByPath(path) db, err := shDB.Open() // open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close() if err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index fd5155ee3..d44049fa8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "strconv" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" @@ -63,6 +64,10 @@ var _ common.Storage = (*Blobovniczas)(nil) var errPutFailed = errors.New("could not save the object in any blobovnicza") +const ( + dbExtension = ".db" +) + // NewBlobovniczaTree returns new instance of blobovniczas tree. func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz = new(Blobovniczas) @@ -94,14 +99,16 @@ func addressHash(addr *oid.Address, path string) uint64 { return hrw.StringHash(a + path) } -// converts uint64 to hex string. func u64ToHexString(ind uint64) string { return strconv.FormatUint(ind, 16) } -// converts uint64 hex string to uint64. +func u64ToHexStringExt(ind uint64) string { + return strconv.FormatUint(ind, 16) + dbExtension +} + func u64FromHexString(str string) uint64 { - v, err := strconv.ParseUint(str, 16, 64) + v, err := strconv.ParseUint(strings.TrimSuffix(str, dbExtension), 16, 64) if err != nil { panic(fmt.Sprintf("blobovnicza name is not an index %s", str)) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index d993767b7..75a30ad3d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -2,11 +2,17 @@ package blobovniczatree import ( "context" + "errors" + "os" + "path/filepath" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "go.uber.org/zap" ) +var errFailedToChangeExtensionReadOnly = errors.New("failed to change blobovnicza extension: read only mode") + // Open opens blobovnicza tree. func (b *Blobovniczas) Open(readOnly bool) error { b.readOnly = readOnly @@ -21,6 +27,13 @@ func (b *Blobovniczas) Open(readOnly bool) error { func (b *Blobovniczas) Init() error { b.log.Debug(logs.BlobovniczatreeInitializingBlobovniczas) + b.log.Debug(logs.BlobovniczaTreeFixingFileExtensions) + if err := b.addDBExtensionToDBs(b.rootPath, 0); err != nil { + b.log.Error(logs.BlobovniczaTreeFixingFileExtensionsFailed, zap.Error(err)) + return err + } + b.log.Debug(logs.BlobovniczaTreeFixingFileExtensionsCompletedSuccessfully) + if b.readOnly { b.log.Debug(logs.BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization) return nil @@ -64,3 +77,37 @@ func (b *Blobovniczas) getBlobovnicza(p string) *sharedDB { func (b *Blobovniczas) getBlobovniczaWithoutCaching(p string) *sharedDB { return b.commondbManager.GetByPath(p) } + +func (b *Blobovniczas) addDBExtensionToDBs(path string, depth uint64) error { + entries, err := os.ReadDir(path) + if os.IsNotExist(err) && depth == 0 { + return nil + } + + for _, entry := range entries { + if entry.IsDir() { + if err := b.addDBExtensionToDBs(filepath.Join(path, entry.Name()), depth+1); err != nil { + return err + } + continue + } + + if strings.HasSuffix(entry.Name(), dbExtension) { + continue + } + if b.readOnly { + return errFailedToChangeExtensionReadOnly + } + + sourcePath := filepath.Join(path, entry.Name()) + targetPath := filepath.Join(path, entry.Name()+dbExtension) + b.log.Debug(logs.BlobovniczaTreeFixingFileExtensionForFile, zap.String("source", sourcePath), zap.String("target", targetPath)) + if err := os.Rename(sourcePath, targetPath); err != nil { + b.log.Error(logs.BlobovniczaTreeFixingFileExtensionFailed, zap.String("source", sourcePath), zap.String("target", targetPath), zap.Error(err)) + return err + } + b.log.Debug(logs.BlobovniczaTreeFixingFileExtensionCompletedSuccessfully, zap.String("source", sourcePath), zap.String("target", targetPath)) + } + + return nil +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go new file mode 100644 index 000000000..0bf304fb1 --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go @@ -0,0 +1,69 @@ +package blobovniczatree + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + "github.com/stretchr/testify/require" +) + +func TestDBExtensionFix(t *testing.T) { + root := t.TempDir() + createTestTree(t, 0, 2, 3, root) + t.Run("adds suffix if not exists", func(t *testing.T) { + openAndCloseTestTree(t, 2, 3, root) + validateTestTree(t, root) + }) + + t.Run("not adds second suffix if exists", func(t *testing.T) { + openAndCloseTestTree(t, 2, 3, root) + validateTestTree(t, root) + }) +} + +func createTestTree(t *testing.T, currentDepth, depth, width uint64, path string) { + if currentDepth == depth { + var w uint64 + for ; w < width; w++ { + dbPath := filepath.Join(path, u64ToHexString(w)) + b := blobovnicza.New(blobovnicza.WithPath(dbPath)) + require.NoError(t, b.Open()) + require.NoError(t, b.Init()) + require.NoError(t, b.Close()) + } + return + } + + var w uint64 + for ; w < width; w++ { + createTestTree(t, currentDepth+1, depth, width, filepath.Join(path, u64ToHexString(w))) + } +} + +func openAndCloseTestTree(t *testing.T, depth, width uint64, path string) { + blz := NewBlobovniczaTree( + WithBlobovniczaShallowDepth(depth), + WithBlobovniczaShallowWidth(width), + WithRootPath(path), + ) + require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Init()) + require.NoError(t, blz.Close()) +} + +func validateTestTree(t *testing.T, path string) { + entries, err := os.ReadDir(path) + require.NoError(t, err) + + for _, entry := range entries { + if entry.IsDir() { + validateTestTree(t, filepath.Join(path, entry.Name())) + } else { + require.True(t, strings.HasSuffix(entry.Name(), dbExtension)) + require.False(t, strings.HasSuffix(strings.TrimSuffix(entry.Name(), dbExtension), dbExtension)) + } + } +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 28e3a8f36..9008672da 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -47,8 +47,8 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co bPrm.SetAddress(prm.Address) if prm.StorageID != nil { - id := blobovnicza.NewIDFromBytes(prm.StorageID) - shBlz := b.getBlobovnicza(id.String()) + id := NewIDFromBytes(prm.StorageID) + shBlz := b.getBlobovnicza(id.Path()) blz, err := shBlz.Open() if err != nil { return res, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index e1a6f5ed5..514ee5f95 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -36,8 +36,8 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common defer span.End() if prm.StorageID != nil { - id := blobovnicza.NewIDFromBytes(prm.StorageID) - shBlz := b.getBlobovnicza(id.String()) + id := NewIDFromBytes(prm.StorageID) + shBlz := b.getBlobovnicza(id.Path()) blz, err := shBlz.Open() if err != nil { return common.ExistsRes{}, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index c89262343..63df97595 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -55,7 +55,7 @@ func TestExistsInvalidStorageID(t *testing.T) { // An invalid boltdb file is created so that it returns an error when opened require.NoError(t, os.MkdirAll(filepath.Join(dir, relBadFileDir), os.ModePerm)) - require.NoError(t, os.WriteFile(filepath.Join(dir, relBadFileDir, badFileName), []byte("not a boltdb file content"), 0o777)) + require.NoError(t, os.WriteFile(filepath.Join(dir, relBadFileDir, badFileName+".db"), []byte("not a boltdb file content"), 0o777)) res, err := b.Exists(context.Background(), common.ExistsPrm{Address: addr, StorageID: []byte(filepath.Join(relBadFileDir, badFileName))}) require.Error(t, err) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 49849e759..0a1f29265 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -47,8 +47,8 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G bPrm.SetAddress(prm.Address) if prm.StorageID != nil { - id := blobovnicza.NewIDFromBytes(prm.StorageID) - shBlz := b.getBlobovnicza(id.String()) + id := NewIDFromBytes(prm.StorageID) + shBlz := b.getBlobovnicza(id.Path()) blz, err := shBlz.Open() if err != nil { return res, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 91d1e3ce0..3a8fae758 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -46,8 +46,8 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re defer span.End() if prm.StorageID != nil { - id := blobovnicza.NewIDFromBytes(prm.StorageID) - shBlz := b.getBlobovnicza(id.String()) + id := NewIDFromBytes(prm.StorageID) + shBlz := b.getBlobovnicza(id.Path()) blz, err := shBlz.Open() if err != nil { return common.GetRangeRes{}, err diff --git a/pkg/local_object_storage/blobovnicza/id.go b/pkg/local_object_storage/blobstor/blobovniczatree/id.go similarity index 71% rename from pkg/local_object_storage/blobovnicza/id.go rename to pkg/local_object_storage/blobstor/blobovniczatree/id.go index 3d3ccf8b9..a080819bc 100644 --- a/pkg/local_object_storage/blobovnicza/id.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/id.go @@ -1,4 +1,4 @@ -package blobovnicza +package blobovniczatree // ID represents Blobovnicza identifier. type ID []byte @@ -8,8 +8,8 @@ func NewIDFromBytes(v []byte) *ID { return (*ID)(&v) } -func (id ID) String() string { - return string(id) +func (id ID) Path() string { + return string(id) + dbExtension } func (id ID) Bytes() []byte { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index d2b285c65..a0bfc374f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -54,7 +55,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm return prm.Handler(common.IterationElement{ Address: elem.Address(), ObjectData: data, - StorageID: []byte(p), + StorageID: []byte(strings.TrimSuffix(p, dbExtension)), }) } return nil @@ -140,10 +141,16 @@ func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, cur return false, ctx.Err() default: } + + lastPart := u64ToHexString(indices[i]) + if isLeafLevel { + lastPart = u64ToHexStringExt(indices[i]) + } + if i == 0 { - curPath = append(curPath, u64ToHexString(indices[i])) + curPath = append(curPath, lastPart) } else { - curPath[len(curPath)-1] = u64ToHexString(indices[i]) + curPath[len(curPath)-1] = lastPart } if exec { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index d695cb199..b89ff1de7 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -117,7 +117,7 @@ func newLevelDBManager(width uint64, options []blobovnicza.Option, rootPath stri databases: make([]*sharedDB, width), } for idx := uint64(0); idx < width; idx++ { - result.databases[idx] = newSharedDB(options, filepath.Join(rootPath, lvlPath, u64ToHexString(idx)), readOnly, metrics, openDBCounter, closedFlog, log) + result.databases[idx] = newSharedDB(options, filepath.Join(rootPath, lvlPath, u64ToHexStringExt(idx)), readOnly, metrics, openDBCounter, closedFlog, log) } return result } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 6f9c8c0de..8c8697c20 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -70,7 +70,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe type putIterator struct { B *Blobovniczas - ID *blobovnicza.ID + ID *ID AllFull bool PutPrm blobovnicza.PutPrm } @@ -113,7 +113,7 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) } idx := u64FromHexString(filepath.Base(active.Path())) - i.ID = blobovnicza.NewIDFromBytes([]byte(filepath.Join(lvlPath, u64ToHexString(idx)))) + i.ID = NewIDFromBytes([]byte(filepath.Join(lvlPath, u64ToHexString(idx)))) return true, nil } From c1667a11d29a844d8f3c2f34671ae855b4a67665 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 20 Sep 2023 17:46:10 +0300 Subject: [PATCH 0718/1943] [#661] blobovniczatree: Allow to change depth or width Now it is possible to change depth or with of blobovniczatree. Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/blobovnicza.go | 2 +- .../blobstor/blobovniczatree/control.go | 32 ++++- .../blobstor/blobovniczatree/control_test.go | 120 ++++++++++++++++++ .../blobstor/blobovniczatree/delete.go | 2 +- .../blobstor/blobovniczatree/exists.go | 2 +- .../blobstor/blobovniczatree/get.go | 2 +- .../blobstor/blobovniczatree/get_range.go | 2 +- .../blobstor/blobovniczatree/iterate.go | 109 +++++++++++++++- .../blobstor/blobovniczatree/iterate_test.go | 41 ++++++ .../blobstor/blobovniczatree/manager.go | 70 +++++++--- 10 files changed, 355 insertions(+), 27 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index d44049fa8..14acec60f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -81,7 +81,7 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz.blzLeafWidth = blz.blzShallowWidth } - blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.blzLeafWidth, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) + blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 75a30ad3d..69f3ca97f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -8,6 +8,7 @@ import ( "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "go.uber.org/zap" ) @@ -39,7 +40,36 @@ func (b *Blobovniczas) Init() error { return nil } - return b.iterateLeaves(context.TODO(), func(p string) (bool, error) { + return b.initializeDBs(context.TODO()) +} + +func (b *Blobovniczas) initializeDBs(ctx context.Context) error { + err := util.MkdirAllX(b.rootPath, b.perm) + if err != nil { + return err + } + + visited := make(map[string]struct{}) + err = b.iterateExistingDBPaths(ctx, func(p string) (bool, error) { + visited[p] = struct{}{} + shBlz := b.getBlobovniczaWithoutCaching(p) + _, err := shBlz.Open() + if err != nil { + return true, err + } + defer shBlz.Close() + + b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) + return false, nil + }) + if err != nil { + return err + } + + return b.iterateSortedLeaves(ctx, nil, func(p string) (bool, error) { + if _, found := visited[p]; found { + return false, nil + } shBlz := b.getBlobovniczaWithoutCaching(p) _, err := shBlz.Open() if err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go index 0bf304fb1..11d4e78fe 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go @@ -1,12 +1,16 @@ package blobovniczatree import ( + "context" "os" "path/filepath" "strings" "testing" + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" "github.com/stretchr/testify/require" ) @@ -67,3 +71,119 @@ func validateTestTree(t *testing.T, path string) { } } } + +func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { + t.Parallel() + + rootDir := t.TempDir() + + blz := NewBlobovniczaTree( + WithBlobovniczaShallowDepth(3), + WithBlobovniczaShallowWidth(5), + WithRootPath(rootDir), + ) + + require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Init()) + + obj35 := blobstortest.NewObject(10 * 1024) + addr35 := objectCore.AddressOf(obj35) + raw, err := obj35.Marshal() + require.NoError(t, err) + + pRes35, err := blz.Put(context.Background(), common.PutPrm{ + Address: addr35, + Object: obj35, + RawData: raw, + }) + require.NoError(t, err) + + gRes, err := blz.Get(context.Background(), common.GetPrm{ + Address: addr35, + StorageID: pRes35.StorageID, + }) + require.NoError(t, err) + require.EqualValues(t, obj35, gRes.Object) + + gRes, err = blz.Get(context.Background(), common.GetPrm{ + Address: addr35, + }) + require.NoError(t, err) + require.EqualValues(t, obj35, gRes.Object) + + require.NoError(t, blz.Close()) + + // change depth and width + blz = NewBlobovniczaTree( + WithBlobovniczaShallowDepth(5), + WithBlobovniczaShallowWidth(2), + WithRootPath(rootDir), + ) + + require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Init()) + + gRes, err = blz.Get(context.Background(), common.GetPrm{ + Address: addr35, + StorageID: pRes35.StorageID, + }) + require.NoError(t, err) + require.EqualValues(t, obj35, gRes.Object) + + gRes, err = blz.Get(context.Background(), common.GetPrm{ + Address: addr35, + }) + require.NoError(t, err) + require.EqualValues(t, obj35, gRes.Object) + + obj52 := blobstortest.NewObject(10 * 1024) + addr52 := objectCore.AddressOf(obj52) + raw, err = obj52.Marshal() + require.NoError(t, err) + + pRes52, err := blz.Put(context.Background(), common.PutPrm{ + Address: addr52, + Object: obj52, + RawData: raw, + }) + require.NoError(t, err) + + require.NoError(t, blz.Close()) + + // change depth and width back + blz = NewBlobovniczaTree( + WithBlobovniczaShallowDepth(3), + WithBlobovniczaShallowWidth(5), + WithRootPath(rootDir), + ) + require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Init()) + + gRes, err = blz.Get(context.Background(), common.GetPrm{ + Address: addr35, + StorageID: pRes35.StorageID, + }) + require.NoError(t, err) + require.EqualValues(t, obj35, gRes.Object) + + gRes, err = blz.Get(context.Background(), common.GetPrm{ + Address: addr35, + }) + require.NoError(t, err) + require.EqualValues(t, obj35, gRes.Object) + + gRes, err = blz.Get(context.Background(), common.GetPrm{ + Address: addr52, + StorageID: pRes52.StorageID, + }) + require.NoError(t, err) + require.EqualValues(t, obj52, gRes.Object) + + gRes, err = blz.Get(context.Background(), common.GetPrm{ + Address: addr52, + }) + require.NoError(t, err) + require.EqualValues(t, obj52, gRes.Object) + + require.NoError(t, blz.Close()) +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 9008672da..21c21beb4 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -63,7 +63,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co objectFound := false - err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { + err = b.iterateSortedDBPaths(ctx, prm.Address, func(p string) (bool, error) { res, err = b.deleteObjectFromLevel(ctx, bPrm, p) if err != nil { if !client.IsErrObjectNotFound(err) { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index 514ee5f95..a64b2bbb1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -51,7 +51,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common var gPrm blobovnicza.GetPrm gPrm.SetAddress(prm.Address) - err := b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { + err := b.iterateSortedDBPaths(ctx, prm.Address, func(p string) (bool, error) { _, err := b.getObjectFromLevel(ctx, gPrm, p) if err != nil { if !client.IsErrObjectNotFound(err) { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 0a1f29265..08cacda8a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -63,7 +63,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G return res, err } - err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { + err = b.iterateSortedDBPaths(ctx, prm.Address, func(p string) (bool, error) { res, err = b.getObjectFromLevel(ctx, bPrm, p) if err != nil { if !client.IsErrObjectNotFound(err) { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 3a8fae758..d237ae439 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -64,7 +64,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re objectFound := false - err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) { + err = b.iterateSortedDBPaths(ctx, prm.Address, func(p string) (bool, error) { res, err = b.getRangeFromLevel(ctx, prm, p) if err != nil { outOfBounds := isErrOutOfRange(err) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index a0bfc374f..0522f8910 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -3,6 +3,7 @@ package blobovniczatree import ( "context" "fmt" + "os" "path/filepath" "strings" "time" @@ -70,7 +71,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm // iterator over all Blobovniczas in unsorted order. Break on f's error return. func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error { - return b.iterateLeaves(ctx, func(p string) (bool, error) { + return b.iterateExistingDBPaths(ctx, func(p string) (bool, error) { shBlz := b.getBlobovnicza(p) blz, err := shBlz.Open() if err != nil { @@ -91,7 +92,9 @@ func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors boo }) } -// iterator over the paths of Blobovniczas sorted by weight. +// iterateSortedLeaves iterates over the paths of Blobovniczas sorted by weight. +// +// Uses depth, width and leaf width for iteration. func (b *Blobovniczas) iterateSortedLeaves(ctx context.Context, addr *oid.Address, f func(string) (bool, error)) error { _, err := b.iterateSorted( ctx, @@ -169,9 +172,105 @@ func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, cur return false, nil } -// iterator over the paths of Blobovniczas in random order. -func (b *Blobovniczas) iterateLeaves(ctx context.Context, f func(string) (bool, error)) error { - return b.iterateSortedLeaves(ctx, nil, f) +// iterateExistingDBPaths iterates over the paths of Blobovniczas without any order. +// +// Uses existed blobovnicza files for iteration. +func (b *Blobovniczas) iterateExistingDBPaths(ctx context.Context, f func(string) (bool, error)) error { + _, err := b.iterateExistingDBPathsDFS(ctx, "", f) + return err +} + +func (b *Blobovniczas) iterateExistingDBPathsDFS(ctx context.Context, path string, f func(string) (bool, error)) (bool, error) { + sysPath := filepath.Join(b.rootPath, path) + entries, err := os.ReadDir(sysPath) + if os.IsNotExist(err) && b.readOnly && path == "" { // non initialized tree in read only mode + return false, nil + } + if err != nil { + return false, err + } + for _, entry := range entries { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + if entry.IsDir() { + stop, err := b.iterateExistingDBPathsDFS(ctx, filepath.Join(path, entry.Name()), f) + if err != nil { + return false, err + } + if stop { + return true, nil + } + } else { + stop, err := f(filepath.Join(path, entry.Name())) + if err != nil { + return false, err + } + if stop { + return true, nil + } + } + } + return false, nil +} + +func (b *Blobovniczas) iterateSortedDBPaths(ctx context.Context, addr oid.Address, f func(string) (bool, error)) error { + _, err := b.iterateSordedDBPathsInternal(ctx, "", addr, f) + return err +} + +func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path string, addr oid.Address, f func(string) (bool, error)) (bool, error) { + sysPath := filepath.Join(b.rootPath, path) + entries, err := os.ReadDir(sysPath) + if os.IsNotExist(err) && b.readOnly && path == "" { // non initialized tree in read only mode + return false, nil + } + if err != nil { + return false, err + } + var dbIdxs []uint64 + var dirIdxs []uint64 + + for _, entry := range entries { + idx := u64FromHexString(entry.Name()) + if entry.IsDir() { + dirIdxs = append(dirIdxs, idx) + } else { + dbIdxs = append(dbIdxs, idx) + } + } + + if len(dbIdxs) > 0 { + hrw.SortSliceByValue(dbIdxs, addressHash(&addr, path)) + for _, dbIdx := range dbIdxs { + dbPath := filepath.Join(path, u64ToHexStringExt(dbIdx)) + stop, err := f(dbPath) + if err != nil { + return false, err + } + if stop { + return true, nil + } + } + } + + if len(dirIdxs) > 0 { + hrw.SortSliceByValue(dirIdxs, addressHash(&addr, path)) + for _, dirIdx := range dirIdxs { + dirPath := filepath.Join(path, u64ToHexString(dirIdx)) + stop, err := b.iterateSordedDBPathsInternal(ctx, dirPath, addr, f) + if err != nil { + return false, err + } + if stop { + return true, nil + } + } + } + + return false, nil } // makes slice of uint64 values from 0 to number-1. diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go new file mode 100644 index 000000000..91fa27dd0 --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go @@ -0,0 +1,41 @@ +package blobovniczatree + +import ( + "context" + "testing" + + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +func TestIterateSortedLeavesAndDBPathsAreSame(t *testing.T) { + t.Parallel() + + blz := NewBlobovniczaTree( + WithBlobovniczaShallowDepth(3), + WithBlobovniczaShallowWidth(5), + WithRootPath(t.TempDir()), + ) + require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Init()) + defer func() { + require.NoError(t, blz.Close()) + }() + + addr := oidtest.Address() + + var leaves []string + var dbPaths []string + + blz.iterateSortedLeaves(context.Background(), &addr, func(s string) (bool, error) { + leaves = append(leaves, s) + return false, nil + }) + + blz.iterateSortedDBPaths(context.Background(), addr, func(s string) (bool, error) { + dbPaths = append(dbPaths, s) + return false, nil + }) + + require.Equal(t, leaves, dbPaths) +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index b89ff1de7..4dca13ada 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -107,25 +107,65 @@ func (b *sharedDB) Path() string { // levelDbManager stores pointers of the sharedDB's for the leaf directory of the blobovnicza tree. type levelDbManager struct { - databases []*sharedDB + dbMtx *sync.RWMutex + databases map[uint64]*sharedDB + + options []blobovnicza.Option + path string + readOnly bool + metrics blobovnicza.Metrics + openDBCounter *openDBCounter + closedFlag *atomic.Bool + log *logger.Logger } -func newLevelDBManager(width uint64, options []blobovnicza.Option, rootPath string, lvlPath string, - readOnly bool, metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlog *atomic.Bool, log *logger.Logger, +func newLevelDBManager(options []blobovnicza.Option, rootPath string, lvlPath string, + readOnly bool, metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlag *atomic.Bool, log *logger.Logger, ) *levelDbManager { result := &levelDbManager{ - databases: make([]*sharedDB, width), - } - for idx := uint64(0); idx < width; idx++ { - result.databases[idx] = newSharedDB(options, filepath.Join(rootPath, lvlPath, u64ToHexStringExt(idx)), readOnly, metrics, openDBCounter, closedFlog, log) + databases: make(map[uint64]*sharedDB), + dbMtx: &sync.RWMutex{}, + + options: options, + path: filepath.Join(rootPath, lvlPath), + readOnly: readOnly, + metrics: metrics, + openDBCounter: openDBCounter, + closedFlag: closedFlag, + log: log, } return result } func (m *levelDbManager) GetByIndex(idx uint64) *sharedDB { + res := m.getDBIfExists(idx) + if res != nil { + return res + } + return m.getOrCreateDB(idx) +} + +func (m *levelDbManager) getDBIfExists(idx uint64) *sharedDB { + m.dbMtx.RLock() + defer m.dbMtx.RUnlock() + return m.databases[idx] } +func (m *levelDbManager) getOrCreateDB(idx uint64) *sharedDB { + m.dbMtx.Lock() + defer m.dbMtx.Unlock() + + db := m.databases[idx] + if db != nil { + return db + } + + db = newSharedDB(m.options, filepath.Join(m.path, u64ToHexStringExt(idx)), m.readOnly, m.metrics, m.openDBCounter, m.closedFlag, m.log) + m.databases[idx] = db + return db +} + // dbManager manages the opening and closing of blobovnicza instances. // // The blobovnicza opens at the first request, closes after the last request. @@ -135,21 +175,19 @@ type dbManager struct { closedFlag *atomic.Bool dbCounter *openDBCounter - rootPath string - options []blobovnicza.Option - readOnly bool - metrics blobovnicza.Metrics - leafWidth uint64 - log *logger.Logger + rootPath string + options []blobovnicza.Option + readOnly bool + metrics blobovnicza.Metrics + log *logger.Logger } -func newDBManager(rootPath string, options []blobovnicza.Option, leafWidth uint64, readOnly bool, metrics blobovnicza.Metrics, log *logger.Logger) *dbManager { +func newDBManager(rootPath string, options []blobovnicza.Option, readOnly bool, metrics blobovnicza.Metrics, log *logger.Logger) *dbManager { return &dbManager{ rootPath: rootPath, options: options, readOnly: readOnly, metrics: metrics, - leafWidth: leafWidth, levelToManager: make(map[string]*levelDbManager), levelToManagerGuard: &sync.RWMutex{}, log: log, @@ -197,7 +235,7 @@ func (m *dbManager) getOrCreateLevelManager(lvlPath string) *levelDbManager { return result } - result := newLevelDBManager(m.leafWidth, m.options, m.rootPath, lvlPath, m.readOnly, m.metrics, m.dbCounter, m.closedFlag, m.log) + result := newLevelDBManager(m.options, m.rootPath, lvlPath, m.readOnly, m.metrics, m.dbCounter, m.closedFlag, m.log) m.levelToManager[lvlPath] = result return result } From a531eaf8bc246ee77780ac7accee241d265fc14b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 21 Sep 2023 18:18:20 +0300 Subject: [PATCH 0719/1943] [#661] blobstor: Add Rebuild implementation Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 2 ++ .../blobstor/blobovniczatree/rebuild.go | 11 ++++++++ .../blobstor/common/rebuild.go | 20 +++++++++++++ .../blobstor/common/storage.go | 1 + .../blobstor/fstree/fstree.go | 4 +++ .../blobstor/memstore/memstore.go | 4 +++ pkg/local_object_storage/blobstor/rebuild.go | 28 +++++++++++++++++-- .../blobstor/teststore/teststore.go | 4 +++ 8 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go create mode 100644 pkg/local_object_storage/blobstor/common/rebuild.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 0c5e162b3..c1afdadf7 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -529,4 +529,6 @@ const ( BlobovniczaTreeFixingFileExtensionForFile = "fixing blobovnicza file extension..." BlobovniczaTreeFixingFileExtensionCompletedSuccessfully = "fixing blobovnicza file extension completed successfully" BlobovniczaTreeFixingFileExtensionFailed = "failed to fix blobovnicza file extension" + BlobstorRebuildFailedToRebuildStorages = "failed to rebuild storages" + BlobstorRebuildRebuildStoragesCompleted = "storages rebuild completed" ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go new file mode 100644 index 000000000..27c645abb --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -0,0 +1,11 @@ +package blobovniczatree + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" +) + +func (b *Blobovniczas) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { + return common.RebuildRes{}, nil +} diff --git a/pkg/local_object_storage/blobstor/common/rebuild.go b/pkg/local_object_storage/blobstor/common/rebuild.go new file mode 100644 index 000000000..1cbc2cbd7 --- /dev/null +++ b/pkg/local_object_storage/blobstor/common/rebuild.go @@ -0,0 +1,20 @@ +package common + +import ( + "context" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" +) + +type RebuildRes struct { + ObjectsMoved uint64 + FilesRemoved uint64 +} + +type RebuildPrm struct { + MetaStorage MetaStorage +} + +type MetaStorage interface { + UpdateStorageID(ctx context.Context, obj *objectSDK.Object, storageID []byte) error +} diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 852f020cc..e552fafea 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -30,4 +30,5 @@ type Storage interface { Put(context.Context, PutPrm) (PutRes, error) Delete(context.Context, DeletePrm) (DeleteRes, error) Iterate(context.Context, IteratePrm) (IterateRes, error) + Rebuild(context.Context, RebuildPrm) (RebuildRes, error) } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index d1fe85901..948872fd2 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -574,3 +574,7 @@ func (t *FSTree) SetReportErrorFunc(_ func(string, error)) { func (t *FSTree) SetParentID(parentID string) { t.metrics.SetParentID(parentID) } + +func (t *FSTree) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { + return common.RebuildRes{}, nil +} diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index cc4f6921a..39bed00bf 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -159,3 +159,7 @@ func (s *memstoreImpl) Iterate(_ context.Context, req common.IteratePrm) (common } return common.IterateRes{}, nil } + +func (s *memstoreImpl) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { + return common.RebuildRes{}, nil +} diff --git a/pkg/local_object_storage/blobstor/rebuild.go b/pkg/local_object_storage/blobstor/rebuild.go index 110735fad..29396da7b 100644 --- a/pkg/local_object_storage/blobstor/rebuild.go +++ b/pkg/local_object_storage/blobstor/rebuild.go @@ -3,13 +3,37 @@ package blobstor import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.uber.org/zap" ) type StorageIDUpdate interface { UpdateStorageID(ctx context.Context, obj *objectSDK.Object, storageID []byte) error } -func (b *BlobStor) Rebuild(_ context.Context, _ StorageIDUpdate) error { - return nil +func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate) error { + var summary common.RebuildRes + var rErr error + for _, storage := range b.storage { + res, err := storage.Storage.Rebuild(ctx, common.RebuildPrm{ + MetaStorage: upd, + }) + summary.FilesRemoved += res.FilesRemoved + summary.ObjectsMoved += res.ObjectsMoved + if err != nil { + b.log.Error(logs.BlobstorRebuildFailedToRebuildStorages, + zap.String("failed_storage_path", storage.Storage.Path()), + zap.String("failed_storage_type", storage.Storage.Type()), + zap.Error(err)) + rErr = err + break + } + } + b.log.Info(logs.BlobstorRebuildRebuildStoragesCompleted, + zap.Bool("success", rErr == nil), + zap.Uint64("total_files_removed", summary.FilesRemoved), + zap.Uint64("total_objects_moved", summary.ObjectsMoved)) + return rErr } diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 3e5b21251..c0cdfacf8 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -229,3 +229,7 @@ func (s *TestStore) Iterate(ctx context.Context, req common.IteratePrm) (common. } func (s *TestStore) SetParentID(string) {} + +func (s *TestStore) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { + return common.RebuildRes{}, nil +} From 422226da183d762ae86b285a6ad9ab3825a92d83 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 22 Sep 2023 13:07:32 +0300 Subject: [PATCH 0720/1943] [#661] blobovniczatree: Add Rebuild implementation Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 6 + .../blobstor/blobovniczatree/active.go | 6 +- .../blobstor/blobovniczatree/blobovnicza.go | 5 + .../blobstor/blobovniczatree/cache.go | 29 +++- .../blobstor/blobovniczatree/delete.go | 6 + .../blobstor/blobovniczatree/iterate.go | 10 ++ .../blobstor/blobovniczatree/manager.go | 69 +++++++- .../blobstor/blobovniczatree/option.go | 36 ++-- .../blobstor/blobovniczatree/put.go | 4 +- .../blobstor/blobovniczatree/rebuild.go | 154 +++++++++++++++++- .../blobstor/blobovniczatree/rebuild_test.go | 132 +++++++++++++++ .../blobstor/common/rebuild.go | 4 +- pkg/local_object_storage/blobstor/rebuild.go | 4 +- pkg/local_object_storage/shard/rebuilder.go | 17 +- 14 files changed, 443 insertions(+), 39 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index c1afdadf7..6a4667209 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -531,4 +531,10 @@ const ( BlobovniczaTreeFixingFileExtensionFailed = "failed to fix blobovnicza file extension" BlobstorRebuildFailedToRebuildStorages = "failed to rebuild storages" BlobstorRebuildRebuildStoragesCompleted = "storages rebuild completed" + BlobovniczaTreeCollectingDBToRebuild = "collecting blobovniczas to rebuild..." + BlobovniczaTreeCollectingDBToRebuildFailed = "collecting blobovniczas to rebuild failed" + BlobovniczaTreeCollectingDBToRebuildSuccess = "collecting blobovniczas to rebuild completed successfully" + BlobovniczaTreeRebuildingBlobovnicza = "rebuilding blobovnicza..." + BlobovniczaTreeRebuildingBlobovniczaFailed = "rebuilding blobovnicza failed" + BlobovniczaTreeRebuildingBlobovniczaSuccess = "rebuilding blobovnicza completed successfully" ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/active.go b/pkg/local_object_storage/blobstor/blobovniczatree/active.go index da8880646..0e3497383 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/active.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/active.go @@ -21,8 +21,8 @@ func (db *activeDB) Close() { db.shDB.Close() } -func (db *activeDB) Path() string { - return db.shDB.Path() +func (db *activeDB) SystemPath() string { + return db.shDB.SystemPath() } // activeDBManager manages active blobovnicza instances (that is, those that are being used for Put). @@ -192,7 +192,7 @@ func (m *activeDBManager) hasActiveDB(lvlPath string) (bool, uint64) { if !ok { return false, 0 } - return true, u64FromHexString(filepath.Base(db.Path())) + return true, u64FromHexString(filepath.Base(db.SystemPath())) } func (m *activeDBManager) replace(lvlPath string, shDB *sharedDB) (*sharedDB, bool) { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 14acec60f..0354fa2ac 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -5,6 +5,7 @@ import ( "fmt" "strconv" "strings" + "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" @@ -58,6 +59,8 @@ type Blobovniczas struct { commondbManager *dbManager activeDBManager *activeDBManager dbCache *dbCache + dbFilesGuard *sync.RWMutex + rebuildGuard *sync.RWMutex } var _ common.Storage = (*Blobovniczas)(nil) @@ -84,6 +87,8 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager) + blz.dbFilesGuard = &sync.RWMutex{} + blz.rebuildGuard = &sync.RWMutex{} return blz } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go index 3f62c49fa..317bac5ff 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -15,8 +15,9 @@ import ( type dbCache struct { cacheGuard *sync.RWMutex cache simplelru.LRUCache[string, *sharedDB] - pathLock *utilSync.KeyLocker[string] + pathLock *utilSync.KeyLocker[string] // the order of locks is important: pathLock first, cacheGuard second closed bool + nonCached map[string]struct{} dbManager *dbManager } @@ -34,6 +35,7 @@ func newDBCache(size int, dbManager *dbManager) *dbCache { cache: cache, dbManager: dbManager, pathLock: utilSync.NewKeyLocker[string](), + nonCached: make(map[string]struct{}), } } @@ -59,6 +61,27 @@ func (c *dbCache) GetOrCreate(path string) *sharedDB { return c.create(path) } +func (c *dbCache) EvictAndMarkNonCached(path string) { + c.pathLock.Lock(path) + defer c.pathLock.Unlock(path) + + c.cacheGuard.Lock() + defer c.cacheGuard.Unlock() + + c.cache.Remove(path) + c.nonCached[path] = struct{}{} +} + +func (c *dbCache) RemoveFromNonCached(path string) { + c.pathLock.Lock(path) + defer c.pathLock.Unlock(path) + + c.cacheGuard.Lock() + defer c.cacheGuard.Unlock() + + delete(c.nonCached, path) +} + func (c *dbCache) getExisted(path string) *sharedDB { c.cacheGuard.Lock() defer c.cacheGuard.Unlock() @@ -94,7 +117,9 @@ func (c *dbCache) put(path string, db *sharedDB) bool { c.cacheGuard.Lock() defer c.cacheGuard.Unlock() - if !c.closed { + _, isNonCached := c.nonCached[path] + + if !isNonCached && !c.closed { c.cache.Add(path, db) return true } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 21c21beb4..56803c872 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -43,6 +43,12 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co return common.DeleteRes{}, common.ErrReadOnly } + if b.rebuildGuard.TryRLock() { + defer b.rebuildGuard.RUnlock() + } else { + return common.DeleteRes{}, errRebuildInProgress + } + var bPrm blobovnicza.DeletePrm bPrm.SetAddress(prm.Address) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 0522f8910..736787952 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -181,6 +181,11 @@ func (b *Blobovniczas) iterateExistingDBPaths(ctx context.Context, f func(string } func (b *Blobovniczas) iterateExistingDBPathsDFS(ctx context.Context, path string, f func(string) (bool, error)) (bool, error) { + if path == "" { + b.dbFilesGuard.RLock() + defer b.dbFilesGuard.RUnlock() + } + sysPath := filepath.Join(b.rootPath, path) entries, err := os.ReadDir(sysPath) if os.IsNotExist(err) && b.readOnly && path == "" { // non initialized tree in read only mode @@ -222,6 +227,11 @@ func (b *Blobovniczas) iterateSortedDBPaths(ctx context.Context, addr oid.Addres } func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path string, addr oid.Address, f func(string) (bool, error)) (bool, error) { + if path == "" { + b.dbFilesGuard.RLock() + defer b.dbFilesGuard.RUnlock() + } + sysPath := filepath.Join(b.rootPath, path) entries, err := os.ReadDir(sysPath) if os.IsNotExist(err) && b.readOnly && path == "" { // non initialized tree in read only mode diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index 4dca13ada..4fdde15a9 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -1,7 +1,9 @@ package blobovniczatree import ( + "errors" "fmt" + "os" "path/filepath" "sync" "sync/atomic" @@ -12,9 +14,11 @@ import ( "go.uber.org/zap" ) +var errClosingClosedBlobovnicza = errors.New("closing closed blobovnicza is not allowed") + // sharedDB is responsible for opening and closing a file of single blobovnicza. type sharedDB struct { - guard *sync.RWMutex + cond *sync.Cond blcza *blobovnicza.Blobovnicza refCount uint32 @@ -31,8 +35,9 @@ func newSharedDB(options []blobovnicza.Option, path string, readOnly bool, metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlag *atomic.Bool, log *logger.Logger, ) *sharedDB { return &sharedDB{ - guard: &sync.RWMutex{}, - + cond: &sync.Cond{ + L: &sync.RWMutex{}, + }, options: options, path: path, readOnly: readOnly, @@ -48,8 +53,8 @@ func (b *sharedDB) Open() (*blobovnicza.Blobovnicza, error) { return nil, errClosed } - b.guard.Lock() - defer b.guard.Unlock() + b.cond.L.Lock() + defer b.cond.L.Unlock() if b.refCount > 0 { b.refCount++ @@ -77,11 +82,12 @@ func (b *sharedDB) Open() (*blobovnicza.Blobovnicza, error) { } func (b *sharedDB) Close() { - b.guard.Lock() - defer b.guard.Unlock() + b.cond.L.Lock() + defer b.cond.L.Unlock() if b.refCount == 0 { b.log.Error(logs.AttemtToCloseAlreadyClosedBlobovnicza, zap.String("id", b.path)) + b.cond.Broadcast() return } @@ -99,9 +105,38 @@ func (b *sharedDB) Close() { } b.refCount-- + if b.refCount == 1 { + b.cond.Broadcast() + } } -func (b *sharedDB) Path() string { +func (b *sharedDB) CloseAndRemoveFile() error { + b.cond.L.Lock() + if b.refCount > 1 { + b.cond.Wait() + } + defer b.cond.L.Unlock() + + if b.refCount == 0 { + return errClosingClosedBlobovnicza + } + + if err := b.blcza.Close(); err != nil { + b.log.Error(logs.BlobovniczatreeCouldNotCloseBlobovnicza, + zap.String("id", b.path), + zap.String("error", err.Error()), + ) + return fmt.Errorf("failed to close blobovnicza (path = %s): %w", b.path, err) + } + + b.refCount = 0 + b.blcza = nil + b.openDBCounter.Dec() + + return os.Remove(b.path) +} + +func (b *sharedDB) SystemPath() string { return b.path } @@ -166,6 +201,13 @@ func (m *levelDbManager) getOrCreateDB(idx uint64) *sharedDB { return db } +func (m *levelDbManager) hasAnyDB() bool { + m.dbMtx.RLock() + defer m.dbMtx.RUnlock() + + return len(m.databases) > 0 +} + // dbManager manages the opening and closing of blobovnicza instances. // // The blobovnicza opens at the first request, closes after the last request. @@ -203,6 +245,17 @@ func (m *dbManager) GetByPath(path string) *sharedDB { return levelManager.GetByIndex(curIndex) } +func (m *dbManager) CleanResources(path string) { + lvlPath := filepath.Dir(path) + + m.levelToManagerGuard.Lock() + defer m.levelToManagerGuard.Unlock() + + if result, ok := m.levelToManager[lvlPath]; ok && !result.hasAnyDB() { + delete(m.levelToManager, lvlPath) + } +} + func (m *dbManager) Open() { m.closedFlag.Store(false) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 561b83761..54c521186 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -2,6 +2,7 @@ package blobovniczatree import ( "io/fs" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" @@ -21,28 +22,31 @@ type cfg struct { compression *compression.Config blzOpts []blobovnicza.Option // reportError is the function called when encountering disk errors. - reportError func(string, error) - metrics Metrics + reportError func(string, error) + metrics Metrics + waitBeforeDropDB time.Duration } type Option func(*cfg) const ( - defaultPerm = 0o700 - defaultOpenedCacheSize = 50 - defaultBlzShallowDepth = 2 - defaultBlzShallowWidth = 16 + defaultPerm = 0o700 + defaultOpenedCacheSize = 50 + defaultBlzShallowDepth = 2 + defaultBlzShallowWidth = 16 + defaultWaitBeforeDropDB = 10 * time.Second ) func initConfig(c *cfg) { *c = cfg{ - log: &logger.Logger{Logger: zap.L()}, - perm: defaultPerm, - openedCacheSize: defaultOpenedCacheSize, - blzShallowDepth: defaultBlzShallowDepth, - blzShallowWidth: defaultBlzShallowWidth, - reportError: func(string, error) {}, - metrics: &noopMetrics{}, + log: &logger.Logger{Logger: zap.L()}, + perm: defaultPerm, + openedCacheSize: defaultOpenedCacheSize, + blzShallowDepth: defaultBlzShallowDepth, + blzShallowWidth: defaultBlzShallowWidth, + reportError: func(string, error) {}, + metrics: &noopMetrics{}, + waitBeforeDropDB: defaultWaitBeforeDropDB, } } @@ -106,3 +110,9 @@ func WithMetrics(m Metrics) Option { c.metrics = m } } + +func WithWaitBeforeDropDB(t time.Duration) Option { + return func(c *cfg) { + c.waitBeforeDropDB = t + } +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 8c8697c20..96c4db67d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -104,7 +104,7 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) i.B.reportError(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err) } else { i.B.log.Debug(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, - zap.String("path", active.Path()), + zap.String("path", active.SystemPath()), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } @@ -112,7 +112,7 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) return false, nil } - idx := u64FromHexString(filepath.Base(active.Path())) + idx := u64FromHexString(filepath.Base(active.SystemPath())) i.ID = NewIDFromBytes([]byte(filepath.Join(lvlPath, u64ToHexString(idx)))) return true, nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index 27c645abb..35e001b63 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -2,10 +2,160 @@ package blobovniczatree import ( "context" + "errors" + "os" + "path/filepath" + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" ) -func (b *Blobovniczas) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { - return common.RebuildRes{}, nil +var errRebuildInProgress = errors.New("rebuild is in progress, the operation cannot be performed") + +func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (common.RebuildRes, error) { + if b.readOnly { + return common.RebuildRes{}, common.ErrReadOnly + } + + b.rebuildGuard.Lock() + defer b.rebuildGuard.Unlock() + + b.log.Debug(logs.BlobovniczaTreeCollectingDBToRebuild) + var res common.RebuildRes + dbsToMigrate, err := b.getDBsToRebuild(ctx) + if err != nil { + b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err)) + return res, err + } + b.log.Info(logs.BlobovniczaTreeCollectingDBToRebuildSuccess, zap.Int("blobovniczas_to_rebuild", len(dbsToMigrate))) + for _, db := range dbsToMigrate { + b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db)) + movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage) + res.ObjectsMoved += movedObjects + if err != nil { + b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err)) + return res, err + } + b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) + res.FilesRemoved++ + } + + return res, nil +} + +func (b *Blobovniczas) getDBsToRebuild(ctx context.Context) ([]string, error) { + dbsToMigrate := make(map[string]struct{}) + if err := b.iterateExistingDBPaths(ctx, func(s string) (bool, error) { + dbsToMigrate[s] = struct{}{} + return false, nil + }); err != nil { + return nil, err + } + if err := b.iterateSortedLeaves(ctx, nil, func(s string) (bool, error) { + delete(dbsToMigrate, s) + return false, nil + }); err != nil { + return nil, err + } + result := make([]string, 0, len(dbsToMigrate)) + for db := range dbsToMigrate { + result = append(result, db) + } + return result, nil +} + +func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.MetaStorage) (uint64, error) { + shDB := b.getBlobovnicza(path) + blz, err := shDB.Open() + if err != nil { + return 0, err + } + shDBClosed := false + defer func() { + if shDBClosed { + return + } + shDB.Close() + }() + + migratedObjects, err := b.moveObjects(ctx, blz, meta) + if err != nil { + return migratedObjects, err + } + shDBClosed, err = b.dropDB(ctx, path, shDB) + return migratedObjects, err +} + +func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovnicza, meta common.MetaStorage) (uint64, error) { + var result uint64 + + var prm blobovnicza.IteratePrm + prm.DecodeAddresses() + prm.SetHandler(func(ie blobovnicza.IterationElement) error { + e := b.moveObject(ctx, ie.Address(), ie.ObjectData(), meta) + if e == nil { + result++ + } + return e + }) + + _, err := blz.Iterate(ctx, prm) + return result, err +} + +func (b *Blobovniczas) moveObject(ctx context.Context, addr oid.Address, data []byte, metaStore common.MetaStorage) error { + var pPrm common.PutPrm + pPrm.Address = addr + pPrm.RawData = data + pRes, err := b.Put(ctx, pPrm) + if err != nil { + return err + } + return metaStore.UpdateStorageID(ctx, addr, pRes.StorageID) +} + +func (b *Blobovniczas) dropDB(ctx context.Context, path string, shDb *sharedDB) (bool, error) { + select { + case <-ctx.Done(): + return false, ctx.Err() + case <-time.After(b.waitBeforeDropDB): // to complete requests with old storage ID + } + + b.dbCache.EvictAndMarkNonCached(path) + defer b.dbCache.RemoveFromNonCached(path) + + b.dbFilesGuard.Lock() + defer b.dbFilesGuard.Unlock() + + if err := shDb.CloseAndRemoveFile(); err != nil { + return false, err + } + b.commondbManager.CleanResources(path) + if err := b.dropDirectoryIfEmpty(filepath.Dir(path)); err != nil { + return true, err + } + return true, nil +} + +func (b *Blobovniczas) dropDirectoryIfEmpty(path string) error { + if path == "." { + return nil + } + + sysPath := filepath.Join(b.rootPath, path) + entries, err := os.ReadDir(sysPath) + if err != nil { + return err + } + if len(entries) > 0 { + return nil + } + if err := os.Remove(sysPath); err != nil { + return err + } + return b.dropDirectoryIfEmpty(filepath.Dir(path)) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go new file mode 100644 index 000000000..00b38f69a --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -0,0 +1,132 @@ +package blobovniczatree + +import ( + "context" + "sync" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +func TestBlobovniczaTreeRebuild(t *testing.T) { + t.Parallel() + + t.Run("width increased", func(t *testing.T) { + t.Parallel() + testBlobovniczaTreeRebuildHelper(t, 2, 2, 2, 3, false) + }) + + t.Run("width reduced", func(t *testing.T) { + t.Parallel() + testBlobovniczaTreeRebuildHelper(t, 2, 2, 2, 1, true) + }) + + t.Run("depth increased", func(t *testing.T) { + t.Parallel() + testBlobovniczaTreeRebuildHelper(t, 1, 2, 2, 2, true) + }) + + t.Run("depth reduced", func(t *testing.T) { + t.Parallel() + testBlobovniczaTreeRebuildHelper(t, 2, 2, 1, 2, true) + }) +} + +func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, targetDepth, targetWidth uint64, shouldMigrate bool) { + dir := t.TempDir() + b := NewBlobovniczaTree( + WithLogger(test.NewLogger(t, true)), + WithObjectSizeLimit(2048), + WithBlobovniczaShallowWidth(sourceWidth), + WithBlobovniczaShallowDepth(sourceDepth), + WithRootPath(dir), + WithBlobovniczaSize(100*1024*1024), + WithWaitBeforeDropDB(0), + WithOpenedCacheSize(1000)) + require.NoError(t, b.Open(false)) + require.NoError(t, b.Init()) + + eg, egCtx := errgroup.WithContext(context.Background()) + storageIDs := make(map[oid.Address][]byte) + storageIDsGuard := &sync.Mutex{} + for i := 0; i < 1000; i++ { + eg.Go(func() error { + obj := blobstortest.NewObject(1024) + data, err := obj.Marshal() + if err != nil { + return err + } + var prm common.PutPrm + prm.Address = object.AddressOf(obj) + prm.RawData = data + res, err := b.Put(egCtx, prm) + if err != nil { + return err + } + storageIDsGuard.Lock() + storageIDs[prm.Address] = res.StorageID + storageIDsGuard.Unlock() + return nil + }) + } + + require.NoError(t, eg.Wait()) + require.NoError(t, b.Close()) + + b = NewBlobovniczaTree( + WithLogger(test.NewLogger(t, true)), + WithObjectSizeLimit(2048), + WithBlobovniczaShallowWidth(targetWidth), + WithBlobovniczaShallowDepth(targetDepth), + WithRootPath(dir), + WithBlobovniczaSize(100*1024*1024), + WithWaitBeforeDropDB(0), + WithOpenedCacheSize(1000)) + require.NoError(t, b.Open(false)) + require.NoError(t, b.Init()) + + for addr, storageID := range storageIDs { + var gPrm common.GetPrm + gPrm.Address = addr + gPrm.StorageID = storageID + _, err := b.Get(context.Background(), gPrm) + require.NoError(t, err) + } + + metaStub := &storageIDUpdateStub{ + storageIDs: storageIDs, + } + var rPrm common.RebuildPrm + rPrm.MetaStorage = metaStub + rRes, err := b.Rebuild(context.Background(), rPrm) + require.NoError(t, err) + dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 + require.Equal(t, shouldMigrate, dataMigrated) + + for addr, storageID := range storageIDs { + var gPrm common.GetPrm + gPrm.Address = addr + gPrm.StorageID = storageID + _, err := b.Get(context.Background(), gPrm) + require.NoError(t, err) + } + + require.NoError(t, b.Close()) +} + +type storageIDUpdateStub struct { + storageIDs map[oid.Address][]byte + updatedCount uint64 +} + +func (s *storageIDUpdateStub) UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error { + s.storageIDs[addr] = storageID + s.updatedCount++ + return nil +} diff --git a/pkg/local_object_storage/blobstor/common/rebuild.go b/pkg/local_object_storage/blobstor/common/rebuild.go index 1cbc2cbd7..896ecbb33 100644 --- a/pkg/local_object_storage/blobstor/common/rebuild.go +++ b/pkg/local_object_storage/blobstor/common/rebuild.go @@ -3,7 +3,7 @@ package common import ( "context" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) type RebuildRes struct { @@ -16,5 +16,5 @@ type RebuildPrm struct { } type MetaStorage interface { - UpdateStorageID(ctx context.Context, obj *objectSDK.Object, storageID []byte) error + UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error } diff --git a/pkg/local_object_storage/blobstor/rebuild.go b/pkg/local_object_storage/blobstor/rebuild.go index 29396da7b..882381dcf 100644 --- a/pkg/local_object_storage/blobstor/rebuild.go +++ b/pkg/local_object_storage/blobstor/rebuild.go @@ -5,12 +5,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) type StorageIDUpdate interface { - UpdateStorageID(ctx context.Context, obj *objectSDK.Object, storageID []byte) error + UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error } func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate) error { diff --git a/pkg/local_object_storage/shard/rebuilder.go b/pkg/local_object_storage/shard/rebuilder.go index 3e0c66abf..a636fceb0 100644 --- a/pkg/local_object_storage/shard/rebuilder.go +++ b/pkg/local_object_storage/shard/rebuilder.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -77,13 +77,20 @@ type mbStorageIDUpdate struct { mb *meta.DB } -func (u *mbStorageIDUpdate) UpdateStorageID(ctx context.Context, obj *objectSDK.Object, storageID []byte) error { +func (u *mbStorageIDUpdate) UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if u.mb == nil { return errMBIsNotAvailable } - var prm meta.PutPrm - prm.SetObject(obj) + + var prm meta.UpdateStorageIDPrm + prm.SetAddress(addr) prm.SetStorageID(storageID) - _, err := u.mb.Put(ctx, prm) + _, err := u.mb.UpdateStorageID(ctx, prm) return err } From da4fee2d0b693500bf917b3d850570ca889f0bbd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 22 Sep 2023 17:08:48 +0300 Subject: [PATCH 0721/1943] [#698] blobovniczatree: Init blobovniczas concurrently Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 + cmd/frostfs-node/config/engine/config_test.go | 2 + .../shard/blobstor/blobovnicza/config.go | 19 ++++++ config/example/node.env | 1 + config/example/node.json | 3 +- config/example/node.yaml | 1 + .../blobstor/blobovniczatree/control.go | 49 ++++++++----- .../blobstor/blobovniczatree/option.go | 68 +++++++++++-------- 8 files changed, 102 insertions(+), 44 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index cc106cf95..840dbf332 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -181,6 +181,7 @@ type subStorageCfg struct { width uint64 leafWidth uint64 openedCacheSize int + initWorkerCount int } // readConfig fills applicationConfiguration with raw configuration values @@ -298,6 +299,7 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.width = sub.ShallowWidth() sCfg.leafWidth = sub.LeafWidth() sCfg.openedCacheSize = sub.OpenedCacheSize() + sCfg.initWorkerCount = sub.InitWorkerCount() case fstree.Type: sub := fstreeconfig.From((*config.Config)(storagesCfg[i])) sCfg.depth = sub.Depth() @@ -796,6 +798,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), blobovniczatree.WithLogger(c.log), } diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 665f70bcb..6b2b37226 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -97,6 +97,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 4, blz.ShallowWidth()) require.EqualValues(t, 50, blz.OpenedCacheSize()) require.EqualValues(t, 10, blz.LeafWidth()) + require.EqualValues(t, 10, blz.InitWorkerCount()) require.Equal(t, "tmp/0/blob", ss[1].Path()) require.EqualValues(t, 0o644, ss[1].Perm()) @@ -146,6 +147,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 4, blz.ShallowWidth()) require.EqualValues(t, 50, blz.OpenedCacheSize()) require.EqualValues(t, 10, blz.LeafWidth()) + require.EqualValues(t, blobovniczaconfig.InitWorkerCountDefault, blz.InitWorkerCount()) require.Equal(t, "tmp/1/blob", ss[1].Path()) require.EqualValues(t, 0o644, ss[1].Perm()) diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index a780ea927..1f7fc77f5 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -22,6 +22,9 @@ const ( // OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's. OpenedCacheSizeDefault = 16 + + // InitWorkerCountDefault is a default workers count to initialize Blobovnicza's. + InitWorkerCountDefault = 5 ) // From wraps config section into Config. @@ -112,3 +115,19 @@ func (x *Config) LeafWidth() uint64 { "leaf_width", ) } + +// InitWorkerCount returns the value of "init_worker_count" config parameter. +// +// Returns InitWorkerCountDefault if the value is not a positive number. +func (x *Config) InitWorkerCount() int { + d := config.IntSafe( + (*config.Config)(x), + "init_worker_count", + ) + + if d > 0 { + return int(d) + } + + return InitWorkerCountDefault +} diff --git a/config/example/node.env b/config/example/node.env index f4dc218a3..9d36b5108 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -125,6 +125,7 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 +FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 ### FSTree config FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH=tmp/0/blob diff --git a/config/example/node.json b/config/example/node.json index c2e3d0a5d..b008009d6 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -172,7 +172,8 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, - "leaf_width": 10 + "leaf_width": 10, + "init_worker_count": 10 }, { "type": "fstree", diff --git a/config/example/node.yaml b/config/example/node.yaml index d04e122b6..bf92388a2 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -184,6 +184,7 @@ storage: blobstor: - type: blobovnicza path: tmp/0/blob/blobovnicza + init_worker_count: 10 #count of workers to initialize blobovniczas - type: fstree path: tmp/0/blob # blobstor path diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 69f3ca97f..ad463c113 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) var errFailedToChangeExtensionReadOnly = errors.New("failed to change blobovnicza extension: read only mode") @@ -49,37 +50,53 @@ func (b *Blobovniczas) initializeDBs(ctx context.Context) error { return err } + eg, egCtx := errgroup.WithContext(ctx) + eg.SetLimit(b.blzInitWorkerCount) visited := make(map[string]struct{}) - err = b.iterateExistingDBPaths(ctx, func(p string) (bool, error) { + err = b.iterateExistingDBPaths(egCtx, func(p string) (bool, error) { visited[p] = struct{}{} - shBlz := b.getBlobovniczaWithoutCaching(p) - _, err := shBlz.Open() - if err != nil { - return true, err - } - defer shBlz.Close() + eg.Go(func() error { + shBlz := b.getBlobovniczaWithoutCaching(p) + _, err := shBlz.Open() + if err != nil { + return err + } + defer shBlz.Close() - b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) + b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) + return nil + }) return false, nil }) if err != nil { + _ = eg.Wait() return err } - return b.iterateSortedLeaves(ctx, nil, func(p string) (bool, error) { + err = b.iterateSortedLeaves(egCtx, nil, func(p string) (bool, error) { if _, found := visited[p]; found { return false, nil } - shBlz := b.getBlobovniczaWithoutCaching(p) - _, err := shBlz.Open() - if err != nil { - return true, err - } - defer shBlz.Close() + eg.Go(func() error { + shBlz := b.getBlobovniczaWithoutCaching(p) + _, err := shBlz.Open() + if err != nil { + return err + } + defer shBlz.Close() - b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) + b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) + return nil + }) return false, nil }) + + if err != nil { + _ = eg.Wait() + return err + } + + return eg.Wait() } func (b *Blobovniczas) openManagers() { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 54c521186..f1eb1eb8e 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -11,42 +11,44 @@ import ( ) type cfg struct { - log *logger.Logger - perm fs.FileMode - readOnly bool - rootPath string - openedCacheSize int - blzShallowDepth uint64 - blzShallowWidth uint64 - blzLeafWidth uint64 - compression *compression.Config - blzOpts []blobovnicza.Option - // reportError is the function called when encountering disk errors. - reportError func(string, error) - metrics Metrics - waitBeforeDropDB time.Duration + log *logger.Logger + perm fs.FileMode + readOnly bool + rootPath string + openedCacheSize int + blzShallowDepth uint64 + blzShallowWidth uint64 + blzLeafWidth uint64 + compression *compression.Config + blzOpts []blobovnicza.Option + reportError func(string, error) // reportError is the function called when encountering disk errors. + metrics Metrics + waitBeforeDropDB time.Duration + blzInitWorkerCount int } type Option func(*cfg) const ( - defaultPerm = 0o700 - defaultOpenedCacheSize = 50 - defaultBlzShallowDepth = 2 - defaultBlzShallowWidth = 16 - defaultWaitBeforeDropDB = 10 * time.Second + defaultPerm = 0o700 + defaultOpenedCacheSize = 50 + defaultBlzShallowDepth = 2 + defaultBlzShallowWidth = 16 + defaultWaitBeforeDropDB = 10 * time.Second + defaultBlzInitWorkerCount = 5 ) func initConfig(c *cfg) { *c = cfg{ - log: &logger.Logger{Logger: zap.L()}, - perm: defaultPerm, - openedCacheSize: defaultOpenedCacheSize, - blzShallowDepth: defaultBlzShallowDepth, - blzShallowWidth: defaultBlzShallowWidth, - reportError: func(string, error) {}, - metrics: &noopMetrics{}, - waitBeforeDropDB: defaultWaitBeforeDropDB, + log: &logger.Logger{Logger: zap.L()}, + perm: defaultPerm, + openedCacheSize: defaultOpenedCacheSize, + blzShallowDepth: defaultBlzShallowDepth, + blzShallowWidth: defaultBlzShallowWidth, + reportError: func(string, error) {}, + metrics: &noopMetrics{}, + waitBeforeDropDB: defaultWaitBeforeDropDB, + blzInitWorkerCount: defaultBlzInitWorkerCount, } } @@ -116,3 +118,15 @@ func WithWaitBeforeDropDB(t time.Duration) Option { c.waitBeforeDropDB = t } } + +// WithInitWorkerCount sets maximum workers count to init blobovnicza tree. +// +// Negative or zero value means no limit. +func WithInitWorkerCount(v int) Option { + if v <= 0 { + v = -1 + } + return func(c *cfg) { + c.blzInitWorkerCount = v + } +} From b2769ca3dec21420a800ed79d56ea2abe608084c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Sep 2023 16:25:15 +0300 Subject: [PATCH 0722/1943] [#661] blobovniczatree: Make Rebuild failover safe Now move info stores in blobovnicza, so in case of failover rebuild completes previous operation first. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 11 + .../blobovnicza/control.go | 2 +- .../blobovnicza/delete.go | 2 +- .../blobovnicza/exists.go | 6 +- pkg/local_object_storage/blobovnicza/get.go | 6 +- .../blobovnicza/iterate.go | 10 +- pkg/local_object_storage/blobovnicza/move.go | 108 +++++++ .../blobstor/blobovniczatree/blobovnicza.go | 12 +- .../blobstor/blobovniczatree/control.go | 10 +- .../blobstor/blobovniczatree/delete.go | 7 + .../blobstor/blobovniczatree/iterate.go | 16 +- .../blobstor/blobovniczatree/rebuild.go | 268 +++++++++++++++++- .../blobovniczatree/rebuild_failover_test.go | 192 +++++++++++++ 13 files changed, 615 insertions(+), 35 deletions(-) create mode 100644 pkg/local_object_storage/blobovnicza/move.go create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 6a4667209..17ba33994 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -537,4 +537,15 @@ const ( BlobovniczaTreeRebuildingBlobovnicza = "rebuilding blobovnicza..." BlobovniczaTreeRebuildingBlobovniczaFailed = "rebuilding blobovnicza failed" BlobovniczaTreeRebuildingBlobovniczaSuccess = "rebuilding blobovnicza completed successfully" + BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza = "could not put move info to source blobovnicza" + BlobovniczatreeCouldNotUpdateStorageID = "could not update storage ID" + BlobovniczatreeCouldNotDropMoveInfo = "could not drop move info from source blobovnicza" + BlobovniczatreeCouldNotDeleteFromSource = "could not delete object from source blobovnicza" + BlobovniczaTreeCompletingPreviousRebuild = "completing previous rebuild if failed..." + BlobovniczaTreeCompletedPreviousRebuildSuccess = "previous rebuild completed successfully" + BlobovniczaTreeCompletedPreviousRebuildFailed = "failed to complete previous rebuild" + BlobovniczatreeCouldNotCheckExistenceInSourceDB = "could not check object existence in source blobovnicza" + BlobovniczatreeCouldNotCheckExistenceInTargetDB = "could not check object existence in target blobovnicza" + BlobovniczatreeCouldNotGetObjectFromSourceDB = "could not get object from source blobovnicza" + BlobovniczatreeCouldNotPutObjectToTargetDB = "could not put object to target blobovnicza" ) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index ad554a0ad..c1d7d49a3 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -105,7 +105,7 @@ func (b *Blobovnicza) initializeCounters() error { var size uint64 var items uint64 err := b.boltDB.View(func(tx *bbolt.Tx) error { - return b.iterateAllBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) { + return b.iterateAllDataBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) { keysN := uint64(b.Stats().KeyN) size += keysN * upper items += keysN diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index d256c1c69..0f92ea3f5 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -51,7 +51,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err var dataSize uint64 err := b.boltDB.Update(func(tx *bbolt.Tx) error { - return b.iterateAllBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { + return b.iterateAllDataBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { objData := buck.Get(addrKey) if objData == nil { // object is not in bucket => continue iterating diff --git a/pkg/local_object_storage/blobovnicza/exists.go b/pkg/local_object_storage/blobovnicza/exists.go index 9b190416f..8cafc72ad 100644 --- a/pkg/local_object_storage/blobovnicza/exists.go +++ b/pkg/local_object_storage/blobovnicza/exists.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "bytes" "context" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -24,7 +25,10 @@ func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error addrKey := addressKey(addr) err := b.boltDB.View(func(tx *bbolt.Tx) error { - return tx.ForEach(func(_ []byte, buck *bbolt.Bucket) error { + return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { + if bytes.Equal(bucketName, incompletedMoveBucketName) { + return nil + } exists = buck.Get(addrKey) != nil if exists { return errInterruptForEach diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index 9975e5e8c..88d8e7e4a 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -57,7 +57,11 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { ) if err := b.boltDB.View(func(tx *bbolt.Tx) error { - return tx.ForEach(func(_ []byte, buck *bbolt.Bucket) error { + return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { + if bytes.Equal(bucketName, incompletedMoveBucketName) { + return nil + } + data = buck.Get(addrKey) if data == nil { return nil diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index f4b85f5d0..e19f8f14b 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "bytes" "context" "fmt" "math" @@ -12,11 +13,11 @@ import ( "go.opentelemetry.io/otel/trace" ) -// iterateAllBuckets iterates all buckets in db +// iterateAllDataBuckets iterates all buckets in db // // If the maximum size of the object (b.objSizeLimit) has been changed to lower value, // then there may be more buckets than the current limit of the object size. -func (b *Blobovnicza) iterateAllBuckets(tx *bbolt.Tx, f func(uint64, uint64, *bbolt.Bucket) (bool, error)) error { +func (b *Blobovnicza) iterateAllDataBuckets(tx *bbolt.Tx, f func(uint64, uint64, *bbolt.Bucket) (bool, error)) error { return b.iterateBucketKeys(false, func(lower uint64, upper uint64, key []byte) (bool, error) { buck := tx.Bucket(key) if buck == nil { @@ -138,7 +139,10 @@ func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes, var elem IterationElement if err := b.boltDB.View(func(tx *bbolt.Tx) error { - return tx.ForEach(func(name []byte, buck *bbolt.Bucket) error { + return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { + if bytes.Equal(bucketName, incompletedMoveBucketName) { + return nil + } return buck.ForEach(func(k, v []byte) error { select { case <-ctx.Done(): diff --git a/pkg/local_object_storage/blobovnicza/move.go b/pkg/local_object_storage/blobovnicza/move.go new file mode 100644 index 000000000..255198f66 --- /dev/null +++ b/pkg/local_object_storage/blobovnicza/move.go @@ -0,0 +1,108 @@ +package blobovnicza + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +var incompletedMoveBucketName = []byte("INCOMPLETED_MOVE") + +type MoveInfo struct { + Address oid.Address + TargetStorageID []byte +} + +func (b *Blobovnicza) PutMoveInfo(ctx context.Context, prm MoveInfo) error { + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.PutMoveInfo", + trace.WithAttributes( + attribute.String("path", b.path), + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("target_storage_id", string(prm.TargetStorageID)), + )) + defer span.End() + + key := addressKey(prm.Address) + + return b.boltDB.Update(func(tx *bbolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists(incompletedMoveBucketName) + if err != nil { + return err + } + + if err := bucket.Put(key, prm.TargetStorageID); err != nil { + return fmt.Errorf("(%T) failed to save move info: %w", b, err) + } + + return nil + }) +} + +func (b *Blobovnicza) DropMoveInfo(ctx context.Context, address oid.Address) error { + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.DropMoveInfo", + trace.WithAttributes( + attribute.String("path", b.path), + attribute.String("address", address.EncodeToString()), + )) + defer span.End() + + key := addressKey(address) + + return b.boltDB.Update(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(incompletedMoveBucketName) + if bucket == nil { + return nil + } + + if err := bucket.Delete(key); err != nil { + return fmt.Errorf("(%T) failed to drop move info: %w", b, err) + } + + c := bucket.Cursor() + k, v := c.First() + bucketEmpty := k == nil && v == nil + if bucketEmpty { + return tx.DeleteBucket(incompletedMoveBucketName) + } + + return nil + }) +} + +func (b *Blobovnicza) ListMoveInfo(ctx context.Context) ([]MoveInfo, error) { + _, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.ListMoveInfo", + trace.WithAttributes( + attribute.String("path", b.path), + )) + defer span.End() + + var result []MoveInfo + if err := b.boltDB.View(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(incompletedMoveBucketName) + if bucket == nil { + return nil + } + return bucket.ForEach(func(k, v []byte) error { + var addr oid.Address + storageID := make([]byte, len(v)) + if err := addressFromKey(&addr, k); err != nil { + return err + } + copy(storageID, v) + result = append(result, MoveInfo{ + Address: addr, + TargetStorageID: storageID, + }) + return nil + }) + }); err != nil { + return nil, err + } + + return result, nil +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 0354fa2ac..9f2950094 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -56,11 +56,12 @@ import ( type Blobovniczas struct { cfg - commondbManager *dbManager - activeDBManager *activeDBManager - dbCache *dbCache - dbFilesGuard *sync.RWMutex - rebuildGuard *sync.RWMutex + commondbManager *dbManager + activeDBManager *activeDBManager + dbCache *dbCache + deleteProtectedObjects *addressMap + dbFilesGuard *sync.RWMutex + rebuildGuard *sync.RWMutex } var _ common.Storage = (*Blobovniczas)(nil) @@ -87,6 +88,7 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager) + blz.deleteProtectedObjects = newAddressMap() blz.dbFilesGuard = &sync.RWMutex{} blz.rebuildGuard = &sync.RWMutex{} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index ad463c113..561b08599 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -57,12 +57,20 @@ func (b *Blobovniczas) initializeDBs(ctx context.Context) error { visited[p] = struct{}{} eg.Go(func() error { shBlz := b.getBlobovniczaWithoutCaching(p) - _, err := shBlz.Open() + blz, err := shBlz.Open() if err != nil { return err } defer shBlz.Close() + moveInfo, err := blz.ListMoveInfo(egCtx) + if err != nil { + return err + } + for _, move := range moveInfo { + b.deleteProtectedObjects.Add(move.Address) + } + b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) return nil }) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 56803c872..298de3ad6 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -3,6 +3,7 @@ package blobovniczatree import ( "context" "encoding/hex" + "errors" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -18,6 +19,8 @@ import ( "go.uber.org/zap" ) +var errObjectIsDeleteProtected = errors.New("object is delete protected") + // Delete deletes object from blobovnicza tree. // // If blobocvnicza ID is specified, only this blobovnicza is processed. @@ -49,6 +52,10 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co return common.DeleteRes{}, errRebuildInProgress } + if b.deleteProtectedObjects.Contains(prm.Address) { + return common.DeleteRes{}, errObjectIsDeleteProtected + } + var bPrm blobovnicza.DeletePrm bPrm.SetAddress(prm.Address) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 736787952..5115439b1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -176,16 +176,14 @@ func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, cur // // Uses existed blobovnicza files for iteration. func (b *Blobovniczas) iterateExistingDBPaths(ctx context.Context, f func(string) (bool, error)) error { + b.dbFilesGuard.RLock() + defer b.dbFilesGuard.RUnlock() + _, err := b.iterateExistingDBPathsDFS(ctx, "", f) return err } func (b *Blobovniczas) iterateExistingDBPathsDFS(ctx context.Context, path string, f func(string) (bool, error)) (bool, error) { - if path == "" { - b.dbFilesGuard.RLock() - defer b.dbFilesGuard.RUnlock() - } - sysPath := filepath.Join(b.rootPath, path) entries, err := os.ReadDir(sysPath) if os.IsNotExist(err) && b.readOnly && path == "" { // non initialized tree in read only mode @@ -222,16 +220,14 @@ func (b *Blobovniczas) iterateExistingDBPathsDFS(ctx context.Context, path strin } func (b *Blobovniczas) iterateSortedDBPaths(ctx context.Context, addr oid.Address, f func(string) (bool, error)) error { + b.dbFilesGuard.RLock() + defer b.dbFilesGuard.RUnlock() + _, err := b.iterateSordedDBPathsInternal(ctx, "", addr, f) return err } func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path string, addr oid.Address, f func(string) (bool, error)) (bool, error) { - if path == "" { - b.dbFilesGuard.RLock() - defer b.dbFilesGuard.RUnlock() - } - sysPath := filepath.Join(b.rootPath, path) entries, err := os.ReadDir(sysPath) if os.IsNotExist(err) && b.readOnly && path == "" { // non initialized tree in read only mode diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index 35e001b63..5f87933bf 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -5,11 +5,13 @@ import ( "errors" "os" "path/filepath" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -24,8 +26,18 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm b.rebuildGuard.Lock() defer b.rebuildGuard.Unlock() - b.log.Debug(logs.BlobovniczaTreeCollectingDBToRebuild) var res common.RebuildRes + + b.log.Debug(logs.BlobovniczaTreeCompletingPreviousRebuild) + completedPreviosMoves, err := b.completeIncompletedMove(ctx, prm.MetaStorage) + res.ObjectsMoved += completedPreviosMoves + if err != nil { + b.log.Warn(logs.BlobovniczaTreeCompletedPreviousRebuildFailed, zap.Error(err)) + return res, err + } + b.log.Debug(logs.BlobovniczaTreeCompletedPreviousRebuildSuccess) + + b.log.Debug(logs.BlobovniczaTreeCollectingDBToRebuild) dbsToMigrate, err := b.getDBsToRebuild(ctx) if err != nil { b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err)) @@ -82,7 +94,7 @@ func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.M shDB.Close() }() - migratedObjects, err := b.moveObjects(ctx, blz, meta) + migratedObjects, err := b.moveObjects(ctx, blz, shDB.SystemPath(), meta) if err != nil { return migratedObjects, err } @@ -90,13 +102,13 @@ func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.M return migratedObjects, err } -func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovnicza, meta common.MetaStorage) (uint64, error) { +func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovnicza, blzPath string, meta common.MetaStorage) (uint64, error) { var result uint64 var prm blobovnicza.IteratePrm prm.DecodeAddresses() prm.SetHandler(func(ie blobovnicza.IterationElement) error { - e := b.moveObject(ctx, ie.Address(), ie.ObjectData(), meta) + e := b.moveObject(ctx, blz, blzPath, ie.Address(), ie.ObjectData(), meta) if e == nil { result++ } @@ -107,15 +119,28 @@ func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovn return result, err } -func (b *Blobovniczas) moveObject(ctx context.Context, addr oid.Address, data []byte, metaStore common.MetaStorage) error { - var pPrm common.PutPrm - pPrm.Address = addr - pPrm.RawData = data - pRes, err := b.Put(ctx, pPrm) - if err != nil { - return err +func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string, + addr oid.Address, data []byte, metaStore common.MetaStorage) error { + it := &moveIterator{ + B: b, + ID: nil, + AllFull: true, + Address: addr, + ObjectData: data, + MetaStore: metaStore, + Source: source, + SourceSysPath: sourcePath, } - return metaStore.UpdateStorageID(ctx, addr, pRes.StorageID) + + if err := b.iterateDeepest(ctx, addr, func(lvlPath string) (bool, error) { return it.tryMoveToLvl(ctx, lvlPath) }); err != nil { + return err + } else if it.ID == nil { + if it.AllFull { + return common.ErrNoSpace + } + return errPutFailed + } + return nil } func (b *Blobovniczas) dropDB(ctx context.Context, path string, shDb *sharedDB) (bool, error) { @@ -159,3 +184,222 @@ func (b *Blobovniczas) dropDirectoryIfEmpty(path string) error { } return b.dropDirectoryIfEmpty(filepath.Dir(path)) } + +func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore common.MetaStorage) (uint64, error) { + var count uint64 + return count, b.iterateExistingDBPaths(ctx, func(s string) (bool, error) { + shDB := b.getBlobovnicza(s) + blz, err := shDB.Open() + if err != nil { + return true, err + } + defer shDB.Close() + + incompletedMoves, err := blz.ListMoveInfo(ctx) + if err != nil { + return true, err + } + + for _, move := range incompletedMoves { + if err := b.performMove(ctx, blz, shDB.SystemPath(), move, metaStore); err != nil { + return true, err + } + count++ + } + + return false, nil + }) +} + +func (b *Blobovniczas) performMove(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string, + move blobovnicza.MoveInfo, metaStore common.MetaStorage) error { + targetDB := b.getBlobovnicza(NewIDFromBytes(move.TargetStorageID).Path()) + target, err := targetDB.Open() + if err != nil { + return err + } + defer targetDB.Close() + + existsInSource := true + var gPrm blobovnicza.GetPrm + gPrm.SetAddress(move.Address) + gRes, err := source.Get(ctx, gPrm) + if err != nil { + if client.IsErrObjectNotFound(err) { + existsInSource = false + } else { + b.log.Warn(logs.BlobovniczatreeCouldNotCheckExistenceInTargetDB, zap.Error(err)) + return err + } + } + + if !existsInSource { //object was deleted by Rebuild, need to delete move info + if err = source.DropMoveInfo(ctx, move.Address); err != nil { + b.log.Warn(logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", sourcePath), zap.Error(err)) + return err + } + b.deleteProtectedObjects.Delete(move.Address) + return nil + } + + existsInTarget, err := target.Exists(ctx, move.Address) + if err != nil { + b.log.Warn(logs.BlobovniczatreeCouldNotCheckExistenceInTargetDB, zap.Error(err)) + return err + } + + if !existsInTarget { + var putPrm blobovnicza.PutPrm + putPrm.SetAddress(move.Address) + putPrm.SetMarshaledObject(gRes.Object()) + _, err = target.Put(ctx, putPrm) + if err != nil { + b.log.Warn(logs.BlobovniczatreeCouldNotPutObjectToTargetDB, zap.String("path", targetDB.SystemPath()), zap.Error(err)) + return err + } + } + + if err = metaStore.UpdateStorageID(ctx, move.Address, move.TargetStorageID); err != nil { + b.log.Warn(logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err)) + return err + } + + var deletePrm blobovnicza.DeletePrm + deletePrm.SetAddress(move.Address) + if _, err = source.Delete(ctx, deletePrm); err != nil { + b.log.Warn(logs.BlobovniczatreeCouldNotDeleteFromSource, zap.String("path", sourcePath), zap.Error(err)) + return err + } + + if err = source.DropMoveInfo(ctx, move.Address); err != nil { + b.log.Warn(logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", sourcePath), zap.Error(err)) + return err + } + + b.deleteProtectedObjects.Delete(move.Address) + return nil +} + +type moveIterator struct { + B *Blobovniczas + ID *ID + AllFull bool + Address oid.Address + ObjectData []byte + MetaStore common.MetaStorage + Source *blobovnicza.Blobovnicza + SourceSysPath string +} + +func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, error) { + target, err := i.B.activeDBManager.GetOpenedActiveDBForLevel(lvlPath) + if err != nil { + if !isLogical(err) { + i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) + } else { + i.B.log.Warn(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, zap.Error(err)) + } + return false, nil + } + + if target == nil { + i.B.log.Warn(logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath)) + return false, nil + } + defer target.Close() + + i.AllFull = false + + targetIDx := u64FromHexString(filepath.Base(target.SystemPath())) + targetStorageID := NewIDFromBytes([]byte(filepath.Join(lvlPath, u64ToHexString(targetIDx)))) + + if err = i.Source.PutMoveInfo(ctx, blobovnicza.MoveInfo{ + Address: i.Address, + TargetStorageID: targetStorageID.Bytes(), + }); err != nil { + if !isLogical(err) { + i.B.reportError(logs.BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza, err) + } else { + i.B.log.Warn(logs.BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza, zap.String("path", i.SourceSysPath), zap.Error(err)) + } + return true, nil + } + i.B.deleteProtectedObjects.Add(i.Address) + + var putPrm blobovnicza.PutPrm + putPrm.SetAddress(i.Address) + putPrm.SetMarshaledObject(i.ObjectData) + + _, err = target.Blobovnicza().Put(ctx, putPrm) + if err != nil { + if !isLogical(err) { + i.B.reportError(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err) + } else { + i.B.log.Warn(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", target.SystemPath()), zap.Error(err)) + } + return true, nil + } + + if err = i.MetaStore.UpdateStorageID(ctx, i.Address, targetStorageID.Bytes()); err != nil { + i.B.log.Warn(logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err)) + return true, nil + } + + var deletePrm blobovnicza.DeletePrm + deletePrm.SetAddress(i.Address) + if _, err = i.Source.Delete(ctx, deletePrm); err != nil { + if !isLogical(err) { + i.B.reportError(logs.BlobovniczatreeCouldNotDeleteFromSource, err) + } else { + i.B.log.Warn(logs.BlobovniczatreeCouldNotDeleteFromSource, zap.String("path", i.SourceSysPath), zap.Error(err)) + } + return true, nil + } + + if err = i.Source.DropMoveInfo(ctx, i.Address); err != nil { + if !isLogical(err) { + i.B.reportError(logs.BlobovniczatreeCouldNotDropMoveInfo, err) + } else { + i.B.log.Warn(logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", i.SourceSysPath), zap.Error(err)) + } + return true, nil + } + i.B.deleteProtectedObjects.Delete(i.Address) + + i.ID = targetStorageID + return true, nil +} + +type addressMap struct { + data map[oid.Address]struct{} + guard *sync.RWMutex +} + +func newAddressMap() *addressMap { + return &addressMap{ + data: make(map[oid.Address]struct{}), + guard: &sync.RWMutex{}, + } +} + +func (m *addressMap) Add(address oid.Address) { + m.guard.Lock() + defer m.guard.Unlock() + + m.data[address] = struct{}{} +} + +func (m *addressMap) Delete(address oid.Address) { + m.guard.Lock() + defer m.guard.Unlock() + + delete(m.data, address) +} + +func (m *addressMap) Contains(address oid.Address) bool { + m.guard.RLock() + defer m.guard.RUnlock() + + _, contains := m.data[address] + return contains +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go new file mode 100644 index 000000000..c9ce564c9 --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -0,0 +1,192 @@ +package blobovniczatree + +import ( + "bytes" + "context" + "path/filepath" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/stretchr/testify/require" +) + +func TestRebuildFailover(t *testing.T) { + t.Parallel() + + t.Run("only move info saved", testRebuildFailoverOnlyMoveInfoSaved) + + t.Run("object saved to target", testRebuildFailoverObjectSavedToTarget) + + t.Run("object deleted from source", testRebuildFailoverObjectDeletedFromSource) +} + +func testRebuildFailoverOnlyMoveInfoSaved(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) + require.NoError(t, blz.Open()) + require.NoError(t, blz.Init()) + + obj := blobstortest.NewObject(1024) + data, err := obj.Marshal() + require.NoError(t, err) + + var pPrm blobovnicza.PutPrm + pPrm.SetAddress(object.AddressOf(obj)) + pPrm.SetMarshaledObject(data) + _, err = blz.Put(context.Background(), pPrm) + require.NoError(t, err) + + require.NoError(t, blz.PutMoveInfo(context.Background(), blobovnicza.MoveInfo{ + Address: object.AddressOf(obj), + TargetStorageID: []byte("0/0/0"), + })) + + require.NoError(t, blz.Close()) + + testRebuildFailoverValidate(t, dir, obj, true) +} + +func testRebuildFailoverObjectSavedToTarget(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) + require.NoError(t, blz.Open()) + require.NoError(t, blz.Init()) + + obj := blobstortest.NewObject(1024) + data, err := obj.Marshal() + require.NoError(t, err) + + var pPrm blobovnicza.PutPrm + pPrm.SetAddress(object.AddressOf(obj)) + pPrm.SetMarshaledObject(data) + _, err = blz.Put(context.Background(), pPrm) + require.NoError(t, err) + + require.NoError(t, blz.PutMoveInfo(context.Background(), blobovnicza.MoveInfo{ + Address: object.AddressOf(obj), + TargetStorageID: []byte("0/0/0"), + })) + + require.NoError(t, blz.Close()) + + blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db"))) + require.NoError(t, blz.Open()) + require.NoError(t, blz.Init()) + + _, err = blz.Put(context.Background(), pPrm) + require.NoError(t, err) + + require.NoError(t, blz.Close()) + + testRebuildFailoverValidate(t, dir, obj, true) +} + +func testRebuildFailoverObjectDeletedFromSource(t *testing.T) { + t.Parallel() + dir := t.TempDir() + + blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) + require.NoError(t, blz.Open()) + require.NoError(t, blz.Init()) + + obj := blobstortest.NewObject(1024) + data, err := obj.Marshal() + require.NoError(t, err) + + require.NoError(t, blz.PutMoveInfo(context.Background(), blobovnicza.MoveInfo{ + Address: object.AddressOf(obj), + TargetStorageID: []byte("0/0/0"), + })) + + require.NoError(t, blz.Close()) + + blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db"))) + require.NoError(t, blz.Open()) + require.NoError(t, blz.Init()) + + var pPrm blobovnicza.PutPrm + pPrm.SetAddress(object.AddressOf(obj)) + pPrm.SetMarshaledObject(data) + _, err = blz.Put(context.Background(), pPrm) + require.NoError(t, err) + + require.NoError(t, blz.Close()) + + testRebuildFailoverValidate(t, dir, obj, false) +} + +func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object, mustUpdateStorageID bool) { + b := NewBlobovniczaTree( + WithLogger(test.NewLogger(t, true)), + WithObjectSizeLimit(2048), + WithBlobovniczaShallowWidth(2), + WithBlobovniczaShallowDepth(2), + WithRootPath(dir), + WithBlobovniczaSize(100*1024*1024), + WithWaitBeforeDropDB(0), + WithOpenedCacheSize(1000)) + require.NoError(t, b.Open(false)) + require.NoError(t, b.Init()) + + var dPrm common.DeletePrm + dPrm.Address = object.AddressOf(obj) + dPrm.StorageID = []byte("0/0/1") + _, err := b.Delete(context.Background(), dPrm) + require.ErrorIs(t, err, errObjectIsDeleteProtected) + + metaStub := &storageIDUpdateStub{ + storageIDs: make(map[oid.Address][]byte), + } + rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ + MetaStorage: metaStub, + }) + require.NoError(t, err) + require.Equal(t, uint64(1), rRes.ObjectsMoved) + require.Equal(t, uint64(0), rRes.FilesRemoved) + + require.NoError(t, b.Close()) + + blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) + require.NoError(t, blz.Open()) + require.NoError(t, blz.Init()) + + moveInfo, err := blz.ListMoveInfo(context.Background()) + require.NoError(t, err) + require.Equal(t, 0, len(moveInfo)) + + var gPrm blobovnicza.GetPrm + gPrm.SetAddress(object.AddressOf(obj)) + _, err = blz.Get(context.Background(), gPrm) + require.True(t, client.IsErrObjectNotFound(err)) + + require.NoError(t, blz.Close()) + + blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db"))) + require.NoError(t, blz.Open()) + require.NoError(t, blz.Init()) + + moveInfo, err = blz.ListMoveInfo(context.Background()) + require.NoError(t, err) + require.Equal(t, 0, len(moveInfo)) + + gRes, err := blz.Get(context.Background(), gPrm) + require.NoError(t, err) + require.True(t, len(gRes.Object()) > 0) + + if mustUpdateStorageID { + require.True(t, bytes.Equal([]byte("0/0/0"), metaStub.storageIDs[object.AddressOf(obj)])) + } + + require.NoError(t, blz.Close()) +} From d4d905ecc6cd00a04be5f2b0d8dcf0b91312b551 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Sep 2023 18:21:37 +0300 Subject: [PATCH 0723/1943] [#661] metrics: Add blobovniczatree rebuild metrics Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/control.go | 1 + .../blobstor/blobovniczatree/metrics.go | 12 ++++++ .../blobstor/blobovniczatree/rebuild.go | 17 ++++++++ .../metrics/blobovnicza.go | 8 ++++ pkg/metrics/blobovnicza.go | 42 +++++++++++++++---- pkg/metrics/consts.go | 1 + 6 files changed, 74 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 561b08599..f1d78dc54 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -19,6 +19,7 @@ var errFailedToChangeExtensionReadOnly = errors.New("failed to change blobovnicz func (b *Blobovniczas) Open(readOnly bool) error { b.readOnly = readOnly b.metrics.SetMode(readOnly) + b.metrics.SetRebuildStatus(rebuildStatusNotStarted) b.openManagers() return nil } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index 032d8cf71..ee0326bcf 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -6,6 +6,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" ) +const ( + rebuildStatusNotStarted = "not_started" + rebuildStatusRunning = "running" + rebuildStatusCompleted = "completed" + rebuildStatusFailed = "failed" +) + type Metrics interface { Blobovnicza() blobovnicza.Metrics @@ -14,6 +21,9 @@ type Metrics interface { SetMode(readOnly bool) Close() + SetRebuildStatus(status string) + ObjectMoved(d time.Duration) + Delete(d time.Duration, success, withStorageID bool) Exists(d time.Duration, success, withStorageID bool) GetRange(d time.Duration, size int, success, withStorageID bool) @@ -27,6 +37,8 @@ type noopMetrics struct{} func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} +func (m *noopMetrics) SetRebuildStatus(string) {} +func (m *noopMetrics) ObjectMoved(time.Duration) {} func (m *noopMetrics) Delete(time.Duration, bool, bool) {} func (m *noopMetrics) Exists(time.Duration, bool, bool) {} func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index 5f87933bf..c1f21703c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -23,6 +23,16 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm return common.RebuildRes{}, common.ErrReadOnly } + b.metrics.SetRebuildStatus(rebuildStatusRunning) + success := true + defer func() { + if success { + b.metrics.SetRebuildStatus(rebuildStatusCompleted) + } else { + b.metrics.SetRebuildStatus(rebuildStatusFailed) + } + }() + b.rebuildGuard.Lock() defer b.rebuildGuard.Unlock() @@ -33,6 +43,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm res.ObjectsMoved += completedPreviosMoves if err != nil { b.log.Warn(logs.BlobovniczaTreeCompletedPreviousRebuildFailed, zap.Error(err)) + success = false return res, err } b.log.Debug(logs.BlobovniczaTreeCompletedPreviousRebuildSuccess) @@ -41,6 +52,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm dbsToMigrate, err := b.getDBsToRebuild(ctx) if err != nil { b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err)) + success = false return res, err } b.log.Info(logs.BlobovniczaTreeCollectingDBToRebuildSuccess, zap.Int("blobovniczas_to_rebuild", len(dbsToMigrate))) @@ -50,6 +62,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm res.ObjectsMoved += movedObjects if err != nil { b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err)) + success = false return res, err } b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) @@ -121,6 +134,10 @@ func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovn func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string, addr oid.Address, data []byte, metaStore common.MetaStorage) error { + startedAt := time.Now() + defer func() { + b.metrics.ObjectMoved(time.Since(startedAt)) + }() it := &moveIterator{ B: b, ID: nil, diff --git a/pkg/local_object_storage/metrics/blobovnicza.go b/pkg/local_object_storage/metrics/blobovnicza.go index 0d0318b3b..3d751ba3f 100644 --- a/pkg/local_object_storage/metrics/blobovnicza.go +++ b/pkg/local_object_storage/metrics/blobovnicza.go @@ -42,6 +42,14 @@ func (m *blobovniczaTreeMetrics) Close() { m.m.CloseBlobobvnizcaTree(m.shardID, m.path) } +func (m *blobovniczaTreeMetrics) SetRebuildStatus(status string) { + m.m.BlobovniczaTreeRebuildStatus(m.shardID, m.path, status) +} + +func (m *blobovniczaTreeMetrics) ObjectMoved(d time.Duration) { + m.m.BlobovniczaTreeObjectMoved(m.shardID, m.path, d) +} + func (m *blobovniczaTreeMetrics) Delete(d time.Duration, success, withStorageID bool) { m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Delete", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID}) } diff --git a/pkg/metrics/blobovnicza.go b/pkg/metrics/blobovnicza.go index a1ecbc700..ddeeb406c 100644 --- a/pkg/metrics/blobovnicza.go +++ b/pkg/metrics/blobovnicza.go @@ -23,16 +23,21 @@ type BlobobvnizcaMetrics interface { IncOpenBlobovniczaCount(shardID, path string) DecOpenBlobovniczaCount(shardID, path string) + + BlobovniczaTreeRebuildStatus(shardID, path, status string) + BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration) } type blobovnicza struct { - treeMode *shardIDPathModeValue - treeReqDuration *prometheus.HistogramVec - treePut *prometheus.CounterVec - treeGet *prometheus.CounterVec - treeOpenSize *prometheus.GaugeVec - treeOpenItems *prometheus.GaugeVec - treeOpenCounter *prometheus.GaugeVec + treeMode *shardIDPathModeValue + treeReqDuration *prometheus.HistogramVec + treePut *prometheus.CounterVec + treeGet *prometheus.CounterVec + treeOpenSize *prometheus.GaugeVec + treeOpenItems *prometheus.GaugeVec + treeOpenCounter *prometheus.GaugeVec + treeObjectMoveDuration *prometheus.HistogramVec + treeRebuildStatus *shardIDPathModeValue } func newBlobovnicza() *blobovnicza { @@ -75,6 +80,13 @@ func newBlobovnicza() *blobovnicza { Name: "open_blobovnicza_count", Help: "Count of opened blobovniczas of Blobovnicza tree", }, []string{shardIDLabel, pathLabel}), + treeObjectMoveDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: blobovniczaTreeSubSystem, + Name: "object_move_duration_seconds", + Help: "Accumulated Blobovnicza tree object move duration", + }, []string{shardIDLabel, pathLabel}), + treeRebuildStatus: newShardIDPathMode(blobovniczaTreeSubSystem, "rebuild_status", "Blobovnicza tree rebuild status"), } } @@ -96,6 +108,11 @@ func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) { shardIDLabel: shardID, pathLabel: path, }) + b.treeObjectMoveDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + b.treeRebuildStatus.SetMode(shardID, path, undefinedStatus) } func (b *blobovnicza) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) { @@ -163,3 +180,14 @@ func (b *blobovnicza) SubOpenBlobovniczaItems(shardID, path string, items uint64 pathLabel: path, }).Sub(float64(items)) } + +func (b *blobovnicza) BlobovniczaTreeRebuildStatus(shardID, path, status string) { + b.treeRebuildStatus.SetMode(shardID, path, status) +} + +func (b *blobovnicza) BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration) { + b.treeObjectMoveDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Observe(d.Seconds()) +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index f7a8fd771..f3b83a2e5 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -45,4 +45,5 @@ const ( failedToDeleteStatus = "failed_to_delete" deletedStatus = "deleted" + undefinedStatus = "undefined" ) From f1c7905263c8efb2f3530fd4cf16f80185aae98c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 3 Oct 2023 11:58:35 +0300 Subject: [PATCH 0724/1943] [#661] blobovniczatree: Make Rebuild concurrent Different DBs can be rebuild concurrently. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 5 +- cmd/frostfs-node/config/engine/config.go | 11 ++++ cmd/frostfs-node/config/engine/config_test.go | 2 + config/example/node.env | 1 + config/example/node.json | 1 + config/example/node.yaml | 1 + .../blobstor/blobovniczatree/rebuild.go | 57 ++++++++++++++----- .../blobovniczatree/rebuild_failover_test.go | 5 +- .../blobstor/blobovniczatree/rebuild_test.go | 11 ++++ .../blobstor/common/rebuild.go | 8 ++- pkg/local_object_storage/blobstor/rebuild.go | 10 +++- pkg/local_object_storage/engine/engine.go | 17 +++++- .../engine/rebuild_limiter.go | 26 +++++++++ pkg/local_object_storage/engine/shards.go | 1 + pkg/local_object_storage/shard/control.go | 2 +- .../shard/rebuild_limiter.go | 13 +++++ pkg/local_object_storage/shard/rebuilder.go | 18 +++--- pkg/local_object_storage/shard/shard.go | 11 ++++ 18 files changed, 170 insertions(+), 30 deletions(-) create mode 100644 pkg/local_object_storage/engine/rebuild_limiter.go create mode 100644 pkg/local_object_storage/shard/rebuild_limiter.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 840dbf332..941aa0d0a 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -103,6 +103,7 @@ type applicationConfiguration struct { shardPoolSize uint32 shards []shardCfg lowMem bool + rebuildWorkers uint32 } } @@ -213,6 +214,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { a.EngineCfg.errorThreshold = engineconfig.ShardErrorThreshold(c) a.EngineCfg.shardPoolSize = engineconfig.ShardPoolSize(c) a.EngineCfg.lowMem = engineconfig.EngineLowMemoryConsumption(c) + a.EngineCfg.rebuildWorkers = engineconfig.EngineRebuildWorkersCount(c) return engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error { return a.updateShardConfig(c, sc) }) } @@ -703,13 +705,14 @@ func initCfgObject(appCfg *config.Config) cfgObject { } func (c *cfg) engineOpts() []engine.Option { - opts := make([]engine.Option, 0, 4) + var opts []engine.Option opts = append(opts, engine.WithShardPoolSize(c.EngineCfg.shardPoolSize), engine.WithErrorThreshold(c.EngineCfg.errorThreshold), engine.WithLogger(c.log), engine.WithLowMemoryConsumption(c.EngineCfg.lowMem), + engine.WithRebuildWorkersCount(c.EngineCfg.rebuildWorkers), ) if c.metricsCollector != nil { diff --git a/cmd/frostfs-node/config/engine/config.go b/cmd/frostfs-node/config/engine/config.go index c944d1c58..baa4e3c9d 100644 --- a/cmd/frostfs-node/config/engine/config.go +++ b/cmd/frostfs-node/config/engine/config.go @@ -15,6 +15,9 @@ const ( // ShardPoolSizeDefault is a default value of routine pool size per-shard to // process object PUT operations in a storage engine. ShardPoolSizeDefault = 20 + // RebuildWorkersCountDefault is a default value of the workers count to + // process storage rebuild operations in a storage engine. + RebuildWorkersCountDefault = 100 ) // ErrNoShardConfigured is returned when at least 1 shard is required but none are found. @@ -88,3 +91,11 @@ func ShardErrorThreshold(c *config.Config) uint32 { func EngineLowMemoryConsumption(c *config.Config) bool { return config.BoolSafe(c.Sub(subsection), "low_mem") } + +// EngineRebuildWorkersCount returns value of "rebuild_workers_count" config parmeter from "storage" section. +func EngineRebuildWorkersCount(c *config.Config) uint32 { + if v := config.Uint32Safe(c.Sub(subsection), "rebuild_workers_count"); v > 0 { + return v + } + return RebuildWorkersCountDefault +} diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 6b2b37226..185c6394f 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -38,6 +38,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 0, engineconfig.ShardErrorThreshold(empty)) require.EqualValues(t, engineconfig.ShardPoolSizeDefault, engineconfig.ShardPoolSize(empty)) require.EqualValues(t, mode.ReadWrite, shardconfig.From(empty).Mode()) + require.EqualValues(t, engineconfig.RebuildWorkersCountDefault, engineconfig.EngineRebuildWorkersCount(empty)) }) const path = "../../../../config/example/node" @@ -47,6 +48,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 100, engineconfig.ShardErrorThreshold(c)) require.EqualValues(t, 15, engineconfig.ShardPoolSize(c)) + require.EqualValues(t, uint32(1000), engineconfig.EngineRebuildWorkersCount(c)) err := engineconfig.IterateShards(c, true, func(sc *shardconfig.Config) error { defer func() { diff --git a/config/example/node.env b/config/example/node.env index 9d36b5108..9c8b90d34 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -92,6 +92,7 @@ FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10 # Storage engine section FROSTFS_STORAGE_SHARD_POOL_SIZE=15 FROSTFS_STORAGE_SHARD_RO_ERROR_THRESHOLD=100 +FROSTFS_STORAGE_REBUILD_WORKERS_COUNT=1000 ## 0 shard ### Flag to refill Metabase from BlobStor FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE=false diff --git a/config/example/node.json b/config/example/node.json index b008009d6..c7cac2501 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -137,6 +137,7 @@ "storage": { "shard_pool_size": 15, "shard_ro_error_threshold": 100, + "rebuild_workers_count": 1000, "shard": { "0": { "mode": "read-only", diff --git a/config/example/node.yaml b/config/example/node.yaml index bf92388a2..4cd023f52 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -116,6 +116,7 @@ storage: # note: shard configuration can be omitted for relay node (see `node.relay`) shard_pool_size: 15 # size of per-shard worker pools used for PUT operations shard_ro_error_threshold: 100 # amount of errors to occur before shard is made read-only (default: 0, ignore errors) + rebuild_workers_count: 1000 # count of rebuild storage concurrent workers shard: default: # section with the default shard parameters diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index c1f21703c..a0ddbfdef 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "sync" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -14,6 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) var errRebuildInProgress = errors.New("rebuild is in progress, the operation cannot be performed") @@ -55,21 +57,48 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm success = false return res, err } + b.log.Info(logs.BlobovniczaTreeCollectingDBToRebuildSuccess, zap.Int("blobovniczas_to_rebuild", len(dbsToMigrate))) - for _, db := range dbsToMigrate { - b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db)) - movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage) - res.ObjectsMoved += movedObjects - if err != nil { - b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err)) - success = false + res, err = b.migrateDBs(ctx, dbsToMigrate, prm, res) + if err != nil { + success = false + } + return res, err +} + +func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common.RebuildPrm, res common.RebuildRes) (common.RebuildRes, error) { + eg, ctx := errgroup.WithContext(ctx) + + var movedObjectsAcc atomic.Uint64 + var filesMovedAcc atomic.Uint64 + for _, db := range dbs { + db := db + if err := prm.WorkerLimiter.AcquireWorkSlot(ctx); err != nil { + _ = eg.Wait() + res.FilesRemoved += filesMovedAcc.Load() + res.ObjectsMoved += movedObjectsAcc.Load() return res, err } - b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) - res.FilesRemoved++ + eg.Go(func() error { + defer prm.WorkerLimiter.ReleaseWorkSlot() + + b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db)) + movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage) + movedObjectsAcc.Add(movedObjects) + if err != nil { + b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err)) + return err + } + b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) + filesMovedAcc.Add(1) + return nil + }) } - return res, nil + err := eg.Wait() + res.FilesRemoved += filesMovedAcc.Load() + res.ObjectsMoved += movedObjectsAcc.Load() + return res, err } func (b *Blobovniczas) getDBsToRebuild(ctx context.Context) ([]string, error) { @@ -133,7 +162,8 @@ func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovn } func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string, - addr oid.Address, data []byte, metaStore common.MetaStorage) error { + addr oid.Address, data []byte, metaStore common.MetaStorage, +) error { startedAt := time.Now() defer func() { b.metrics.ObjectMoved(time.Since(startedAt)) @@ -229,7 +259,8 @@ func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore co } func (b *Blobovniczas) performMove(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string, - move blobovnicza.MoveInfo, metaStore common.MetaStorage) error { + move blobovnicza.MoveInfo, metaStore common.MetaStorage, +) error { targetDB := b.getBlobovnicza(NewIDFromBytes(move.TargetStorageID).Path()) target, err := targetDB.Open() if err != nil { @@ -250,7 +281,7 @@ func (b *Blobovniczas) performMove(ctx context.Context, source *blobovnicza.Blob } } - if !existsInSource { //object was deleted by Rebuild, need to delete move info + if !existsInSource { // object was deleted by Rebuild, need to delete move info if err = source.DropMoveInfo(ctx, move.Address); err != nil { b.log.Warn(logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", sourcePath), zap.Error(err)) return err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index c9ce564c9..ff86c036a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "path/filepath" + "sync" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -147,9 +148,11 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object metaStub := &storageIDUpdateStub{ storageIDs: make(map[oid.Address][]byte), + guard: &sync.Mutex{}, } rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ - MetaStorage: metaStub, + MetaStorage: metaStub, + WorkerLimiter: &rebuildLimiterStub{}, }) require.NoError(t, err) require.Equal(t, uint64(1), rRes.ObjectsMoved) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 00b38f69a..34006b270 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -101,9 +101,11 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta metaStub := &storageIDUpdateStub{ storageIDs: storageIDs, + guard: &sync.Mutex{}, } var rPrm common.RebuildPrm rPrm.MetaStorage = metaStub + rPrm.WorkerLimiter = &rebuildLimiterStub{} rRes, err := b.Rebuild(context.Background(), rPrm) require.NoError(t, err) dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 @@ -121,12 +123,21 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta } type storageIDUpdateStub struct { + guard *sync.Mutex storageIDs map[oid.Address][]byte updatedCount uint64 } func (s *storageIDUpdateStub) UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error { + s.guard.Lock() + defer s.guard.Unlock() + s.storageIDs[addr] = storageID s.updatedCount++ return nil } + +type rebuildLimiterStub struct{} + +func (s *rebuildLimiterStub) AcquireWorkSlot(context.Context) error { return nil } +func (s *rebuildLimiterStub) ReleaseWorkSlot() {} diff --git a/pkg/local_object_storage/blobstor/common/rebuild.go b/pkg/local_object_storage/blobstor/common/rebuild.go index 896ecbb33..9f629ef8c 100644 --- a/pkg/local_object_storage/blobstor/common/rebuild.go +++ b/pkg/local_object_storage/blobstor/common/rebuild.go @@ -12,9 +12,15 @@ type RebuildRes struct { } type RebuildPrm struct { - MetaStorage MetaStorage + MetaStorage MetaStorage + WorkerLimiter ConcurrentWorkersLimiter } type MetaStorage interface { UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error } + +type ConcurrentWorkersLimiter interface { + AcquireWorkSlot(ctx context.Context) error + ReleaseWorkSlot() +} diff --git a/pkg/local_object_storage/blobstor/rebuild.go b/pkg/local_object_storage/blobstor/rebuild.go index 882381dcf..101c60752 100644 --- a/pkg/local_object_storage/blobstor/rebuild.go +++ b/pkg/local_object_storage/blobstor/rebuild.go @@ -13,12 +13,18 @@ type StorageIDUpdate interface { UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error } -func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate) error { +type ConcurrentWorkersLimiter interface { + AcquireWorkSlot(ctx context.Context) error + ReleaseWorkSlot() +} + +func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate, limiter ConcurrentWorkersLimiter) error { var summary common.RebuildRes var rErr error for _, storage := range b.storage { res, err := storage.Storage.Rebuild(ctx, common.RebuildPrm{ - MetaStorage: upd, + MetaStorage: upd, + WorkerLimiter: limiter, }) summary.FilesRemoved += res.FilesRemoved summary.ObjectsMoved += res.ObjectsMoved diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index e312e9f0d..026a7c336 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -38,6 +38,7 @@ type StorageEngine struct { err error } evacuateLimiter *evacuationLimiter + rebuildLimiter *rebuildLimiter } type shardWrapper struct { @@ -215,13 +216,15 @@ type cfg struct { shardPoolSize uint32 lowMem bool + + rebuildWorkersCount uint32 } func defaultCfg() *cfg { return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - - shardPoolSize: 20, + log: &logger.Logger{Logger: zap.L()}, + shardPoolSize: 20, + rebuildWorkersCount: 100, } } @@ -240,6 +243,7 @@ func New(opts ...Option) *StorageEngine { closeCh: make(chan struct{}), setModeCh: make(chan setModeRequest), evacuateLimiter: &evacuationLimiter{}, + rebuildLimiter: newRebuildLimiter(c.rebuildWorkersCount), } } @@ -277,3 +281,10 @@ func WithLowMemoryConsumption(lowMemCons bool) Option { c.lowMem = lowMemCons } } + +// WithRebuildWorkersCount returns an option to set the count of concurrent rebuild workers. +func WithRebuildWorkersCount(count uint32) Option { + return func(c *cfg) { + c.rebuildWorkersCount = count + } +} diff --git a/pkg/local_object_storage/engine/rebuild_limiter.go b/pkg/local_object_storage/engine/rebuild_limiter.go new file mode 100644 index 000000000..28b02b0a3 --- /dev/null +++ b/pkg/local_object_storage/engine/rebuild_limiter.go @@ -0,0 +1,26 @@ +package engine + +import "context" + +type rebuildLimiter struct { + semaphore chan struct{} +} + +func newRebuildLimiter(workersCount uint32) *rebuildLimiter { + return &rebuildLimiter{ + semaphore: make(chan struct{}, workersCount), + } +} + +func (l *rebuildLimiter) AcquireWorkSlot(ctx context.Context) error { + select { + case l.semaphore <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (l *rebuildLimiter) ReleaseWorkSlot() { + <-l.semaphore +} diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 5d7fb598e..7e3e85e81 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -118,6 +118,7 @@ func (e *StorageEngine) createShard(ctx context.Context, opts []shard.Option) (* shard.WithExpiredLocksCallback(e.processExpiredLocks), shard.WithDeletedLockCallback(e.processDeletedLocks), shard.WithReportErrorFunc(e.reportShardErrorBackground), + shard.WithRebuildWorkerLimiter(e.rebuildLimiter), )...) if err := sh.UpdateID(ctx); err != nil { diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index a988683ad..f103ebc2b 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -162,7 +162,7 @@ func (s *Shard) Init(ctx context.Context) error { s.gc.init(ctx) - s.rb = newRebuilder() + s.rb = newRebuilder(s.rebuildLimiter) s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) return nil diff --git a/pkg/local_object_storage/shard/rebuild_limiter.go b/pkg/local_object_storage/shard/rebuild_limiter.go new file mode 100644 index 000000000..efc21837c --- /dev/null +++ b/pkg/local_object_storage/shard/rebuild_limiter.go @@ -0,0 +1,13 @@ +package shard + +import "context" + +type RebuildWorkerLimiter interface { + AcquireWorkSlot(ctx context.Context) error + ReleaseWorkSlot() +} + +type noopRebuildLimiter struct{} + +func (l *noopRebuildLimiter) AcquireWorkSlot(context.Context) error { return nil } +func (l *noopRebuildLimiter) ReleaseWorkSlot() {} diff --git a/pkg/local_object_storage/shard/rebuilder.go b/pkg/local_object_storage/shard/rebuilder.go index a636fceb0..f18573c57 100644 --- a/pkg/local_object_storage/shard/rebuilder.go +++ b/pkg/local_object_storage/shard/rebuilder.go @@ -14,16 +14,18 @@ import ( ) type rebuilder struct { - mtx *sync.Mutex - wg *sync.WaitGroup - cancel func() + mtx *sync.Mutex + wg *sync.WaitGroup + cancel func() + limiter RebuildWorkerLimiter } -func newRebuilder() *rebuilder { +func newRebuilder(l RebuildWorkerLimiter) *rebuilder { return &rebuilder{ - mtx: &sync.Mutex{}, - wg: &sync.WaitGroup{}, - cancel: nil, + mtx: &sync.Mutex{}, + wg: &sync.WaitGroup{}, + cancel: nil, + limiter: l, } } @@ -45,7 +47,7 @@ func (r *rebuilder) start(ctx context.Context, bs *blobstor.BlobStor, mb *meta.D defer r.wg.Done() log.Info(logs.BlobstoreRebuildStarted) - if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}); err != nil { + if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, r.limiter); err != nil { log.Warn(logs.FailedToRebuildBlobstore, zap.Error(err)) } else { log.Info(logs.BlobstoreRebuildCompletedSuccessfully) diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 915feccb4..56c6aee1f 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -127,6 +127,8 @@ type cfg struct { metricsWriter MetricsWriter reportErrorFunc func(selfID string, message string, err error) + + rebuildLimiter RebuildWorkerLimiter } func defaultCfg() *cfg { @@ -135,6 +137,7 @@ func defaultCfg() *cfg { log: &logger.Logger{Logger: zap.L()}, gcCfg: defaultGCCfg(), reportErrorFunc: func(string, string, error) {}, + rebuildLimiter: &noopRebuildLimiter{}, } } @@ -372,6 +375,14 @@ func WithExpiredCollectorWorkerCount(count int) Option { } } +// WithRebuildWorkerLimiter return option to set concurrent +// workers count of storage rebuild operation. +func WithRebuildWorkerLimiter(l RebuildWorkerLimiter) Option { + return func(c *cfg) { + c.rebuildLimiter = l + } +} + func (s *Shard) fillInfo() { s.cfg.info.MetaBaseInfo = s.metaBase.DumpInfo() s.cfg.info.BlobStorInfo = s.blobStor.DumpInfo() From c6a739e7467bc0fbbcdb1f813b38d1387f95d2c8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Oct 2023 16:01:35 +0300 Subject: [PATCH 0725/1943] [#661] blobovniczatree: Make Rebuild concurrent for objects Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/option.go | 9 ++ .../blobstor/blobovniczatree/rebuild.go | 98 +++++++++++-------- .../blobstor/blobovniczatree/rebuild_test.go | 6 +- 3 files changed, 72 insertions(+), 41 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index f1eb1eb8e..93dc487ef 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -25,6 +25,7 @@ type cfg struct { metrics Metrics waitBeforeDropDB time.Duration blzInitWorkerCount int + blzMoveBatchSize int } type Option func(*cfg) @@ -36,6 +37,7 @@ const ( defaultBlzShallowWidth = 16 defaultWaitBeforeDropDB = 10 * time.Second defaultBlzInitWorkerCount = 5 + defaulBlzMoveBatchSize = 10000 ) func initConfig(c *cfg) { @@ -49,6 +51,7 @@ func initConfig(c *cfg) { metrics: &noopMetrics{}, waitBeforeDropDB: defaultWaitBeforeDropDB, blzInitWorkerCount: defaultBlzInitWorkerCount, + blzMoveBatchSize: defaulBlzMoveBatchSize, } } @@ -119,6 +122,12 @@ func WithWaitBeforeDropDB(t time.Duration) Option { } } +func WithMoveBatchSize(v int) Option { + return func(c *cfg) { + c.blzMoveBatchSize = v + } +} + // WithInitWorkerCount sets maximum workers count to init blobovnicza tree. // // Negative or zero value means no limit. diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index a0ddbfdef..c7e44fdb3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "bytes" "context" "errors" "os" @@ -18,7 +19,10 @@ import ( "golang.org/x/sync/errgroup" ) -var errRebuildInProgress = errors.New("rebuild is in progress, the operation cannot be performed") +var ( + errRebuildInProgress = errors.New("rebuild is in progress, the operation cannot be performed") + errBatchFull = errors.New("batch full") +) func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (common.RebuildRes, error) { if b.readOnly { @@ -67,38 +71,18 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm } func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common.RebuildPrm, res common.RebuildRes) (common.RebuildRes, error) { - eg, ctx := errgroup.WithContext(ctx) - - var movedObjectsAcc atomic.Uint64 - var filesMovedAcc atomic.Uint64 for _, db := range dbs { - db := db - if err := prm.WorkerLimiter.AcquireWorkSlot(ctx); err != nil { - _ = eg.Wait() - res.FilesRemoved += filesMovedAcc.Load() - res.ObjectsMoved += movedObjectsAcc.Load() + b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db)) + movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage, prm.WorkerLimiter) + res.ObjectsMoved += movedObjects + if err != nil { + b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err)) return res, err } - eg.Go(func() error { - defer prm.WorkerLimiter.ReleaseWorkSlot() - - b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db)) - movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage) - movedObjectsAcc.Add(movedObjects) - if err != nil { - b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err)) - return err - } - b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) - filesMovedAcc.Add(1) - return nil - }) + b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) + res.FilesRemoved++ } - - err := eg.Wait() - res.FilesRemoved += filesMovedAcc.Load() - res.ObjectsMoved += movedObjectsAcc.Load() - return res, err + return res, nil } func (b *Blobovniczas) getDBsToRebuild(ctx context.Context) ([]string, error) { @@ -122,7 +106,7 @@ func (b *Blobovniczas) getDBsToRebuild(ctx context.Context) ([]string, error) { return result, nil } -func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.MetaStorage) (uint64, error) { +func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.MetaStorage, limiter common.ConcurrentWorkersLimiter) (uint64, error) { shDB := b.getBlobovnicza(path) blz, err := shDB.Open() if err != nil { @@ -136,7 +120,7 @@ func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.M shDB.Close() }() - migratedObjects, err := b.moveObjects(ctx, blz, shDB.SystemPath(), meta) + migratedObjects, err := b.moveObjects(ctx, blz, shDB.SystemPath(), meta, limiter) if err != nil { return migratedObjects, err } @@ -144,21 +128,57 @@ func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.M return migratedObjects, err } -func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovnicza, blzPath string, meta common.MetaStorage) (uint64, error) { - var result uint64 +func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovnicza, blzPath string, meta common.MetaStorage, limiter common.ConcurrentWorkersLimiter) (uint64, error) { + var result atomic.Uint64 + batch := make(map[oid.Address][]byte) var prm blobovnicza.IteratePrm prm.DecodeAddresses() prm.SetHandler(func(ie blobovnicza.IterationElement) error { - e := b.moveObject(ctx, blz, blzPath, ie.Address(), ie.ObjectData(), meta) - if e == nil { - result++ + batch[ie.Address()] = bytes.Clone(ie.ObjectData()) + if len(batch) == b.blzMoveBatchSize { + return errBatchFull } - return e + return nil }) - _, err := blz.Iterate(ctx, prm) - return result, err + for { + _, err := blz.Iterate(ctx, prm) + if err != nil && !errors.Is(err, errBatchFull) { + return result.Load(), err + } + + if len(batch) == 0 { + break + } + + eg, egCtx := errgroup.WithContext(ctx) + + for addr, data := range batch { + addr := addr + data := data + + if err := limiter.AcquireWorkSlot(egCtx); err != nil { + _ = eg.Wait() + return result.Load(), err + } + eg.Go(func() error { + defer limiter.ReleaseWorkSlot() + err := b.moveObject(egCtx, blz, blzPath, addr, data, meta) + if err == nil { + result.Add(1) + } + return err + }) + } + if err := eg.Wait(); err != nil { + return result.Load(), err + } + + batch = make(map[oid.Address][]byte) + } + + return result.Load(), nil } func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 34006b270..921515a6a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -48,7 +48,8 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta WithRootPath(dir), WithBlobovniczaSize(100*1024*1024), WithWaitBeforeDropDB(0), - WithOpenedCacheSize(1000)) + WithOpenedCacheSize(1000), + WithMoveBatchSize(3)) require.NoError(t, b.Open(false)) require.NoError(t, b.Init()) @@ -87,7 +88,8 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta WithRootPath(dir), WithBlobovniczaSize(100*1024*1024), WithWaitBeforeDropDB(0), - WithOpenedCacheSize(1000)) + WithOpenedCacheSize(1000), + WithMoveBatchSize(3)) require.NoError(t, b.Open(false)) require.NoError(t, b.Init()) From 2407e5f5ff2399078a984d2e031bf3cefef64053 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Oct 2023 18:22:53 +0300 Subject: [PATCH 0726/1943] [#661] blobovniczatree: Do not sort DB's and indicies Put stores object to next active DB, so there is no need to sort DBs. In addition, it adds unnecessary DB openings. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobstor/blobovniczatree/iterate.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 5115439b1..92014fd55 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -134,7 +134,9 @@ func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, cur } indices := indexSlice(levelWidth) - hrw.SortSliceByValue(indices, addressHash(addr, filepath.Join(curPath...))) + if !isLeafLevel { + hrw.SortSliceByValue(indices, addressHash(addr, filepath.Join(curPath...))) + } exec := uint64(len(curPath)) == execDepth @@ -249,7 +251,6 @@ func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path st } if len(dbIdxs) > 0 { - hrw.SortSliceByValue(dbIdxs, addressHash(&addr, path)) for _, dbIdx := range dbIdxs { dbPath := filepath.Join(path, u64ToHexStringExt(dbIdx)) stop, err := f(dbPath) From 8bbfb2df4340faea12cca59c0b9f219babc58d6c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Oct 2023 18:33:03 +0300 Subject: [PATCH 0727/1943] [#661] blobovniczatree: Pass object size limit from config If actual small object size value lower than default object size limit, then unnecessary buckets created. If actual small object size value greated than default object size limit, then error happens. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 941aa0d0a..a0d5fba92 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -803,6 +803,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), blobovniczatree.WithLogger(c.log), + blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit), } if c.metricsCollector != nil { From 8911656b1a03fdd30fee352e70161d451f399d9f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 11 Oct 2023 09:23:55 +0300 Subject: [PATCH 0728/1943] [#661] metrcis: Add rebuild percent metric Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/metrics.go | 2 ++ .../blobstor/blobovniczatree/rebuild.go | 5 +++++ .../metrics/blobovnicza.go | 4 ++++ pkg/metrics/blobovnicza.go | 19 +++++++++++++++++++ 4 files changed, 30 insertions(+) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index ee0326bcf..ed05f8772 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -23,6 +23,7 @@ type Metrics interface { SetRebuildStatus(status string) ObjectMoved(d time.Duration) + SetRebuildPercent(value uint32) Delete(d time.Duration, success, withStorageID bool) Exists(d time.Duration, success, withStorageID bool) @@ -38,6 +39,7 @@ func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(bool) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) SetRebuildStatus(string) {} +func (m *noopMetrics) SetRebuildPercent(uint32) {} func (m *noopMetrics) ObjectMoved(time.Duration) {} func (m *noopMetrics) Delete(time.Duration, bool, bool) {} func (m *noopMetrics) Exists(time.Duration, bool, bool) {} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index c7e44fdb3..d65ca43f8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -30,6 +30,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm } b.metrics.SetRebuildStatus(rebuildStatusRunning) + b.metrics.SetRebuildPercent(0) success := true defer func() { if success { @@ -71,6 +72,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm } func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common.RebuildPrm, res common.RebuildRes) (common.RebuildRes, error) { + var completedDBCount uint32 for _, db := range dbs { b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db)) movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage, prm.WorkerLimiter) @@ -81,7 +83,10 @@ func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common. } b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) res.FilesRemoved++ + completedDBCount++ + b.metrics.SetRebuildPercent((100 * completedDBCount) / uint32(len(dbs))) } + b.metrics.SetRebuildPercent(100) return res, nil } diff --git a/pkg/local_object_storage/metrics/blobovnicza.go b/pkg/local_object_storage/metrics/blobovnicza.go index 3d751ba3f..3a92a7bc7 100644 --- a/pkg/local_object_storage/metrics/blobovnicza.go +++ b/pkg/local_object_storage/metrics/blobovnicza.go @@ -46,6 +46,10 @@ func (m *blobovniczaTreeMetrics) SetRebuildStatus(status string) { m.m.BlobovniczaTreeRebuildStatus(m.shardID, m.path, status) } +func (m *blobovniczaTreeMetrics) SetRebuildPercent(value uint32) { + m.m.BlobovniczaTreeRebuildPercent(m.shardID, m.path, value) +} + func (m *blobovniczaTreeMetrics) ObjectMoved(d time.Duration) { m.m.BlobovniczaTreeObjectMoved(m.shardID, m.path, d) } diff --git a/pkg/metrics/blobovnicza.go b/pkg/metrics/blobovnicza.go index ddeeb406c..f3a14a96e 100644 --- a/pkg/metrics/blobovnicza.go +++ b/pkg/metrics/blobovnicza.go @@ -25,6 +25,7 @@ type BlobobvnizcaMetrics interface { DecOpenBlobovniczaCount(shardID, path string) BlobovniczaTreeRebuildStatus(shardID, path, status string) + BlobovniczaTreeRebuildPercent(shardID, path string, value uint32) BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration) } @@ -38,6 +39,7 @@ type blobovnicza struct { treeOpenCounter *prometheus.GaugeVec treeObjectMoveDuration *prometheus.HistogramVec treeRebuildStatus *shardIDPathModeValue + treeRebuildPercent *prometheus.GaugeVec } func newBlobovnicza() *blobovnicza { @@ -87,6 +89,12 @@ func newBlobovnicza() *blobovnicza { Help: "Accumulated Blobovnicza tree object move duration", }, []string{shardIDLabel, pathLabel}), treeRebuildStatus: newShardIDPathMode(blobovniczaTreeSubSystem, "rebuild_status", "Blobovnicza tree rebuild status"), + treeRebuildPercent: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: blobovniczaTreeSubSystem, + Name: "rebuild_complete_percent", + Help: "Percent of rebuild completeness", + }, []string{shardIDLabel, pathLabel}), } } @@ -112,6 +120,10 @@ func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) { shardIDLabel: shardID, pathLabel: path, }) + b.treeRebuildPercent.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) b.treeRebuildStatus.SetMode(shardID, path, undefinedStatus) } @@ -191,3 +203,10 @@ func (b *blobovnicza) BlobovniczaTreeObjectMoved(shardID, path string, d time.Du pathLabel: path, }).Observe(d.Seconds()) } + +func (b *blobovnicza) BlobovniczaTreeRebuildPercent(shardID, path string, value uint32) { + b.treeRebuildPercent.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Set(float64(value)) +} From 5e8c08da3e64d2d8fb64a1bd7c4464ce65582843 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 11 Oct 2023 10:33:42 +0300 Subject: [PATCH 0729/1943] [#661] blobstore: Add address to error logs Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index d65ca43f8..2302661d5 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -333,7 +333,7 @@ func (b *Blobovniczas) performMove(ctx context.Context, source *blobovnicza.Blob } if err = metaStore.UpdateStorageID(ctx, move.Address, move.TargetStorageID); err != nil { - b.log.Warn(logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err)) + b.log.Warn(logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err), zap.Stringer("address", move.Address)) return err } @@ -414,7 +414,7 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, } if err = i.MetaStore.UpdateStorageID(ctx, i.Address, targetStorageID.Bytes()); err != nil { - i.B.log.Warn(logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err)) + i.B.log.Warn(logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err), zap.Stringer("address", i.Address)) return true, nil } From e54dc3dc7c9a581328d4766d4d4a4073a5e47f55 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 12 Oct 2023 15:25:16 +0300 Subject: [PATCH 0730/1943] [#698] blobovnicza: Store counter values Blobovnicza initialization take a long time because of bucket Stat() call. So now blobovnicza stores counters in META bucket. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 3 + .../blobovnicza/control.go | 24 ++++ .../blobovnicza/delete.go | 17 ++- .../blobovnicza/exists.go | 3 +- pkg/local_object_storage/blobovnicza/get.go | 2 +- .../blobovnicza/iterate.go | 3 +- pkg/local_object_storage/blobovnicza/meta.go | 103 ++++++++++++++++++ pkg/local_object_storage/blobovnicza/put.go | 4 +- 8 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 pkg/local_object_storage/blobovnicza/meta.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 17ba33994..e1c2cfdee 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -548,4 +548,7 @@ const ( BlobovniczatreeCouldNotCheckExistenceInTargetDB = "could not check object existence in target blobovnicza" BlobovniczatreeCouldNotGetObjectFromSourceDB = "could not get object from source blobovnicza" BlobovniczatreeCouldNotPutObjectToTargetDB = "could not put object to target blobovnicza" + BlobovniczaSavingCountersToMeta = "saving counters to blobovnicza's meta..." + BlobovniczaSavingCountersToMetaSuccess = "saving counters to blobovnicza's meta completed successfully" + BlobovniczaSavingCountersToMetaFailed = "saving counters to blobovnicza's meta failed" ) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index c1d7d49a3..2a85313f1 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -104,7 +104,17 @@ func (b *Blobovnicza) Init() error { func (b *Blobovnicza) initializeCounters() error { var size uint64 var items uint64 + var sizeExists bool + var itemsCountExists bool + err := b.boltDB.View(func(tx *bbolt.Tx) error { + size, sizeExists = hasDataSize(tx) + items, itemsCountExists = hasItemsCount(tx) + + if sizeExists && itemsCountExists { + return nil + } + return b.iterateAllDataBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) { keysN := uint64(b.Stats().KeyN) size += keysN * upper @@ -115,6 +125,20 @@ func (b *Blobovnicza) initializeCounters() error { if err != nil { return fmt.Errorf("can't determine DB size: %w", err) } + if (!sizeExists || !itemsCountExists) && !b.boltOptions.ReadOnly { + b.log.Debug(logs.BlobovniczaSavingCountersToMeta, zap.Uint64("size", size), zap.Uint64("items", items)) + if err := b.boltDB.Update(func(tx *bbolt.Tx) error { + if err := saveDataSize(tx, size); err != nil { + return err + } + return saveItemsCount(tx, items) + }); err != nil { + b.log.Debug(logs.BlobovniczaSavingCountersToMetaFailed, zap.Uint64("size", size), zap.Uint64("items", items)) + return fmt.Errorf("can't save blobovnicza's size and items count: %w", err) + } + b.log.Debug(logs.BlobovniczaSavingCountersToMetaSuccess, zap.Uint64("size", size), zap.Uint64("items", items)) + } + b.dataSize.Store(size) b.itemsCount.Store(items) b.metrics.AddOpenBlobovniczaSize(size) diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 0f92ea3f5..ff3a7a290 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -51,7 +51,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err var dataSize uint64 err := b.boltDB.Update(func(tx *bbolt.Tx) error { - return b.iterateAllDataBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { + err := b.iterateAllDataBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { objData := buck.Get(addrKey) if objData == nil { // object is not in bucket => continue iterating @@ -63,6 +63,21 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err found = true return true, buck.Delete(addrKey) }) + if err != nil { + return err + } + if found { + return updateMeta(tx, func(count, size uint64) (uint64, uint64) { + if count > 0 { + count-- + } + if size >= sizeUpperBound { + size -= sizeUpperBound + } + return count, size + }) + } + return nil }) if err == nil && !found { diff --git a/pkg/local_object_storage/blobovnicza/exists.go b/pkg/local_object_storage/blobovnicza/exists.go index 8cafc72ad..f7bc84d4a 100644 --- a/pkg/local_object_storage/blobovnicza/exists.go +++ b/pkg/local_object_storage/blobovnicza/exists.go @@ -1,7 +1,6 @@ package blobovnicza import ( - "bytes" "context" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -26,7 +25,7 @@ func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { - if bytes.Equal(bucketName, incompletedMoveBucketName) { + if isNonDataBucket(bucketName) { return nil } exists = buck.Get(addrKey) != nil diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index 88d8e7e4a..600323f55 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -58,7 +58,7 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { if err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { - if bytes.Equal(bucketName, incompletedMoveBucketName) { + if isNonDataBucket(bucketName) { return nil } diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index e19f8f14b..32b0ccea7 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -1,7 +1,6 @@ package blobovnicza import ( - "bytes" "context" "fmt" "math" @@ -140,7 +139,7 @@ func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes, if err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { - if bytes.Equal(bucketName, incompletedMoveBucketName) { + if isNonDataBucket(bucketName) { return nil } return buck.ForEach(func(k, v []byte) error { diff --git a/pkg/local_object_storage/blobovnicza/meta.go b/pkg/local_object_storage/blobovnicza/meta.go new file mode 100644 index 000000000..8b913f8a7 --- /dev/null +++ b/pkg/local_object_storage/blobovnicza/meta.go @@ -0,0 +1,103 @@ +package blobovnicza + +import ( + "bytes" + "encoding/binary" + + "go.etcd.io/bbolt" +) + +const ( + dataSizeAndItemsCountBufLength = 8 +) + +var ( + metaBucketName = []byte("META") + dataSizeKey = []byte("data_size") + itemsCountKey = []byte("items_count") +) + +func isNonDataBucket(bucketName []byte) bool { + return bytes.Equal(bucketName, incompletedMoveBucketName) || bytes.Equal(bucketName, metaBucketName) +} + +func hasDataSize(tx *bbolt.Tx) (uint64, bool) { + b := tx.Bucket(metaBucketName) + if b == nil { + return 0, false + } + v := b.Get(dataSizeKey) + if v == nil { + return 0, false + } + if len(v) != dataSizeAndItemsCountBufLength { + return 0, false + } + return binary.LittleEndian.Uint64(v), true +} + +func hasItemsCount(tx *bbolt.Tx) (uint64, bool) { + b := tx.Bucket(metaBucketName) + if b == nil { + return 0, false + } + v := b.Get(itemsCountKey) + if v == nil { + return 0, false + } + if len(v) != dataSizeAndItemsCountBufLength { + return 0, false + } + return binary.LittleEndian.Uint64(v), true +} + +func saveDataSize(tx *bbolt.Tx, size uint64) error { + b, err := tx.CreateBucketIfNotExists(metaBucketName) + if err != nil { + return err + } + buf := make([]byte, dataSizeAndItemsCountBufLength) + binary.LittleEndian.PutUint64(buf, size) + return b.Put(dataSizeKey, buf) +} + +func saveItemsCount(tx *bbolt.Tx, count uint64) error { + b, err := tx.CreateBucketIfNotExists(metaBucketName) + if err != nil { + return err + } + buf := make([]byte, dataSizeAndItemsCountBufLength) + binary.LittleEndian.PutUint64(buf, count) + return b.Put(itemsCountKey, buf) +} + +func updateMeta(tx *bbolt.Tx, updateValues func(count, size uint64) (uint64, uint64)) error { + b, err := tx.CreateBucketIfNotExists(metaBucketName) + if err != nil { + return err + } + + var count uint64 + var size uint64 + + v := b.Get(itemsCountKey) + if v != nil { + count = binary.LittleEndian.Uint64(v) + } + + v = b.Get(dataSizeKey) + if v != nil { + size = binary.LittleEndian.Uint64(v) + } + + count, size = updateValues(count, size) + + buf := make([]byte, dataSizeAndItemsCountBufLength) + binary.LittleEndian.PutUint64(buf, size) + if err := b.Put(dataSizeKey, buf); err != nil { + return err + } + + binary.LittleEndian.PutUint64(buf, count) + return b.Put(itemsCountKey, buf) +} diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index 859f7dbbf..608739bee 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -73,7 +73,9 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { return fmt.Errorf("(%T) could not save object in bucket: %w", b, err) } - return nil + return updateMeta(tx, func(count, size uint64) (uint64, uint64) { + return count + 1, size + upperBound + }) }) if err == nil { b.itemAdded(upperBound) From ad0697adc405bc01eb99127729a5b8e2be3f0e43 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 12 Oct 2023 21:35:08 +0300 Subject: [PATCH 0731/1943] [#661] blobovnicza: Compute size with record size To get more accurate size of blobovnicza use record size (lenght of key + lenght of data). Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobovnicza/control.go | 9 +++++---- pkg/local_object_storage/blobovnicza/delete.go | 10 +++++++--- pkg/local_object_storage/blobovnicza/put.go | 7 ++++--- pkg/local_object_storage/blobovnicza/sizes.go | 5 ++--- pkg/local_object_storage/blobovnicza/sizes_test.go | 2 +- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index 2a85313f1..dfc4f884f 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -116,10 +116,11 @@ func (b *Blobovnicza) initializeCounters() error { } return b.iterateAllDataBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) { - keysN := uint64(b.Stats().KeyN) - size += keysN * upper - items += keysN - return false, nil + return false, b.ForEach(func(k, v []byte) error { + size += uint64(len(k) + len(v)) + items++ + return nil + }) }) }) if err != nil { diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index ff3a7a290..ab8575ad7 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -49,6 +49,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err var sizeUpperBound uint64 var sizeLowerBound uint64 var dataSize uint64 + var recordSize uint64 err := b.boltDB.Update(func(tx *bbolt.Tx) error { err := b.iterateAllDataBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) { @@ -60,6 +61,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err dataSize = uint64(len(objData)) sizeLowerBound = lower sizeUpperBound = upper + recordSize = dataSize + uint64(len(addrKey)) found = true return true, buck.Delete(addrKey) }) @@ -71,8 +73,10 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err if count > 0 { count-- } - if size >= sizeUpperBound { - size -= sizeUpperBound + if size >= recordSize { + size -= recordSize + } else { + size = 0 } return count, size }) @@ -90,7 +94,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err zap.String("range", stringifyBounds(sizeLowerBound, sizeUpperBound)), zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) - b.itemDeleted(sizeUpperBound) + b.itemDeleted(recordSize) } return DeleteRes{}, err diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index 608739bee..d15b6e2b2 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -56,8 +56,9 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { defer span.End() sz := uint64(len(prm.objData)) - bucketName, upperBound := bucketForSize(sz) + bucketName := bucketForSize(sz) key := addressKey(prm.addr) + recordSize := sz + uint64(len(key)) err := b.boltDB.Batch(func(tx *bbolt.Tx) error { buck := tx.Bucket(bucketName) @@ -74,11 +75,11 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { } return updateMeta(tx, func(count, size uint64) (uint64, uint64) { - return count + 1, size + upperBound + return count + 1, size + recordSize }) }) if err == nil { - b.itemAdded(upperBound) + b.itemAdded(recordSize) } return PutRes{}, err diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index cd1f69725..1dff75aed 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -29,9 +29,8 @@ func bucketKeyFromBounds(upperBound uint64) []byte { return buf[:ln] } -func bucketForSize(sz uint64) ([]byte, uint64) { - upperBound := upperPowerOfTwo(sz) - return bucketKeyFromBounds(upperBound), upperBound +func bucketForSize(sz uint64) []byte { + return bucketKeyFromBounds(upperPowerOfTwo(sz)) } func upperPowerOfTwo(v uint64) uint64 { diff --git a/pkg/local_object_storage/blobovnicza/sizes_test.go b/pkg/local_object_storage/blobovnicza/sizes_test.go index d2f576fd3..01093b8d7 100644 --- a/pkg/local_object_storage/blobovnicza/sizes_test.go +++ b/pkg/local_object_storage/blobovnicza/sizes_test.go @@ -34,7 +34,7 @@ func TestSizes(t *testing.T) { upperBound: 4 * firstBucketBound, }, } { - key, _ := bucketForSize(item.sz) + key := bucketForSize(item.sz) require.Equal(t, bucketKeyFromBounds(item.upperBound), key) } } From db49ad16cc2037bd365f2cd6e7b1c20689fadd06 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 27 Nov 2023 16:01:48 +0300 Subject: [PATCH 0732/1943] [#826] blobovniczatree: Do not create DB's on init Blobovniczas will be created on write requests. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 ++ cmd/frostfs-node/config/engine/config_test.go | 1 + .../shard/blobstor/blobovnicza/config.go | 10 ++++++ config/example/node.env | 1 + config/example/node.json | 3 +- config/example/node.yaml | 1 + .../blobstor/blobovniczatree/control.go | 36 ++++++++++--------- .../blobstor/blobovniczatree/iterate_test.go | 1 + .../blobstor/blobovniczatree/option.go | 8 +++++ 9 files changed, 46 insertions(+), 18 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index a0d5fba92..2d80a4115 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -183,6 +183,7 @@ type subStorageCfg struct { leafWidth uint64 openedCacheSize int initWorkerCount int + initInAdvance bool } // readConfig fills applicationConfiguration with raw configuration values @@ -302,6 +303,7 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.leafWidth = sub.LeafWidth() sCfg.openedCacheSize = sub.OpenedCacheSize() sCfg.initWorkerCount = sub.InitWorkerCount() + sCfg.initInAdvance = sub.InitInAdvance() case fstree.Type: sub := fstreeconfig.From((*config.Config)(storagesCfg[i])) sCfg.depth = sub.Depth() @@ -802,6 +804,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), + blobovniczatree.WithInitInAdvance(sRead.initInAdvance), blobovniczatree.WithLogger(c.log), blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit), } diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 185c6394f..6c7b23cc4 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -100,6 +100,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 50, blz.OpenedCacheSize()) require.EqualValues(t, 10, blz.LeafWidth()) require.EqualValues(t, 10, blz.InitWorkerCount()) + require.EqualValues(t, true, blz.InitInAdvance()) require.Equal(t, "tmp/0/blob", ss[1].Path()) require.EqualValues(t, 0o644, ss[1].Perm()) diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index 1f7fc77f5..37191cc72 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -131,3 +131,13 @@ func (x *Config) InitWorkerCount() int { return InitWorkerCountDefault } + +// InitInAdvance returns the value of "init_in_advance" config parameter. +// +// Returns False if the value is not defined or invalid. +func (x *Config) InitInAdvance() bool { + return config.BoolSafe( + (*config.Config)(x), + "init_in_advance", + ) +} diff --git a/config/example/node.env b/config/example/node.env index 9c8b90d34..be16f62bf 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -127,6 +127,7 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 +FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE ### FSTree config FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH=tmp/0/blob diff --git a/config/example/node.json b/config/example/node.json index c7cac2501..ff38e3894 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -174,7 +174,8 @@ "width": 4, "opened_cache_capacity": 50, "leaf_width": 10, - "init_worker_count": 10 + "init_worker_count": 10, + "init_in_advance": true }, { "type": "fstree", diff --git a/config/example/node.yaml b/config/example/node.yaml index 4cd023f52..1355113a3 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -186,6 +186,7 @@ storage: - type: blobovnicza path: tmp/0/blob/blobovnicza init_worker_count: 10 #count of workers to initialize blobovniczas + init_in_advance: true - type: fstree path: tmp/0/blob # blobstor path diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index f1d78dc54..bd98967cb 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -82,27 +82,29 @@ func (b *Blobovniczas) initializeDBs(ctx context.Context) error { return err } - err = b.iterateSortedLeaves(egCtx, nil, func(p string) (bool, error) { - if _, found := visited[p]; found { - return false, nil - } - eg.Go(func() error { - shBlz := b.getBlobovniczaWithoutCaching(p) - _, err := shBlz.Open() - if err != nil { - return err + if b.createDBInAdvance { + err = b.iterateSortedLeaves(egCtx, nil, func(p string) (bool, error) { + if _, found := visited[p]; found { + return false, nil } - defer shBlz.Close() + eg.Go(func() error { + shBlz := b.getBlobovniczaWithoutCaching(p) + _, err := shBlz.Open() + if err != nil { + return err + } + defer shBlz.Close() - b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) - return nil + b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) + return nil + }) + return false, nil }) - return false, nil - }) - if err != nil { - _ = eg.Wait() - return err + if err != nil { + _ = eg.Wait() + return err + } } return eg.Wait() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go index 91fa27dd0..c322fa7a3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go @@ -16,6 +16,7 @@ func TestIterateSortedLeavesAndDBPathsAreSame(t *testing.T) { WithBlobovniczaShallowWidth(5), WithRootPath(t.TempDir()), ) + blz.createDBInAdvance = true require.NoError(t, blz.Open(false)) require.NoError(t, blz.Init()) defer func() { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 93dc487ef..d36074dc8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -26,6 +26,7 @@ type cfg struct { waitBeforeDropDB time.Duration blzInitWorkerCount int blzMoveBatchSize int + createDBInAdvance bool } type Option func(*cfg) @@ -139,3 +140,10 @@ func WithInitWorkerCount(v int) Option { c.blzInitWorkerCount = v } } + +// WithInitInAdvance returns an option to create blobovnicza tree DB's in advance. +func WithInitInAdvance(v bool) Option { + return func(c *cfg) { + c.createDBInAdvance = v + } +} From 39060382a1a0eb6233460775b8b03d641a6f6638 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Tue, 5 Dec 2023 12:11:28 +0300 Subject: [PATCH 0733/1943] [#842] control: Recieve target in gRPC methods for APE managing * Introduce Target type and pass it to all gRPC methods for APE chain managing instead CID. Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/control/add_rule.go | 11 +- cmd/frostfs-cli/modules/control/get_rule.go | 11 +- cmd/frostfs-cli/modules/control/list_rules.go | 9 +- .../modules/control/remove_rule.go | 11 +- pkg/services/control/server/policy_engine.go | 56 +- pkg/services/control/service.pb.go | 478 +++++++++--------- pkg/services/control/service.proto | 22 +- pkg/services/control/service_frostfs.pb.go | 16 +- pkg/services/control/types.pb.go | 230 +++++++-- pkg/services/control/types.proto | 17 + pkg/services/control/types_frostfs.pb.go | 33 ++ 11 files changed, 578 insertions(+), 316 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index f57823ed3..ca9220009 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" "encoding/json" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -13,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/spf13/cobra" ) @@ -60,10 +62,15 @@ func addRule(cmd *cobra.Command, _ []string) { cmd.Println("Container ID: " + cidStr) cmd.Println("Parsed chain:\n" + prettyJSONFormat(cmd, serializedChain)) + name := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cidStr) + req := &control.AddChainLocalOverrideRequest{ Body: &control.AddChainLocalOverrideRequest_Body{ - ContainerId: rawCID, - Chain: serializedChain, + Target: &control.ChainTarget{ + Type: control.ChainTarget_CONTAINER, + Name: name, + }, + Chain: serializedChain, }, } diff --git a/cmd/frostfs-cli/modules/control/get_rule.go b/cmd/frostfs-cli/modules/control/get_rule.go index 0c34a696e..962ef179b 100644 --- a/cmd/frostfs-cli/modules/control/get_rule.go +++ b/cmd/frostfs-cli/modules/control/get_rule.go @@ -2,6 +2,7 @@ package control import ( "crypto/sha256" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -10,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/spf13/cobra" ) @@ -32,10 +34,15 @@ func getRule(cmd *cobra.Command, _ []string) { chainID, _ := cmd.Flags().GetString(chainIDFlag) + name := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cidStr) + req := &control.GetChainLocalOverrideRequest{ Body: &control.GetChainLocalOverrideRequest_Body{ - ContainerId: rawCID, - ChainId: chainID, + Target: &control.ChainTarget{ + Name: name, + Type: control.ChainTarget_CONTAINER, + }, + ChainId: chainID, }, } diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index 6a0879d0e..c1e47f483 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -2,6 +2,7 @@ package control import ( "crypto/sha256" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -10,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/spf13/cobra" ) @@ -30,9 +32,14 @@ func listRules(cmd *cobra.Command, _ []string) { rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) + name := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cidStr) + req := &control.ListChainLocalOverridesRequest{ Body: &control.ListChainLocalOverridesRequest_Body{ - ContainerId: rawCID, + Target: &control.ChainTarget{ + Name: name, + Type: control.ChainTarget_CONTAINER, + }, }, } diff --git a/cmd/frostfs-cli/modules/control/remove_rule.go b/cmd/frostfs-cli/modules/control/remove_rule.go index de064536b..c3f4aeb0c 100644 --- a/cmd/frostfs-cli/modules/control/remove_rule.go +++ b/cmd/frostfs-cli/modules/control/remove_rule.go @@ -2,6 +2,7 @@ package control import ( "crypto/sha256" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -9,6 +10,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/spf13/cobra" ) @@ -35,10 +37,15 @@ func removeRule(cmd *cobra.Command, _ []string) { chainID, _ := cmd.Flags().GetString(chainIDFlag) + name := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cidStr) + req := &control.RemoveChainLocalOverrideRequest{ Body: &control.RemoveChainLocalOverrideRequest_Body{ - ContainerId: rawCID, - ChainId: chainID, + Target: &control.ChainTarget{ + Name: name, + Type: control.ChainTarget_CONTAINER, + }, + ChainId: chainID, }, } diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 8565cd9c1..1c09de918 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -14,13 +15,36 @@ import ( "google.golang.org/grpc/status" ) +// extractCID extracts CID from the schema's pattern. +// TODO (aarifullin): This is temporary solution should be replaced by +// resource name validation. +func extractCID(resource string) (cid.ID, error) { + var cidStr string + + // Sscanf requires to make tokens delimited by spaces. + pattern := strings.Replace(nativeschema.ResourceFormatRootContainerObjects, "/", " ", -1) + resource = strings.Replace(resource, "/", " ", -1) + + if _, err := fmt.Sscanf(resource, pattern, &cidStr); err != nil { + err = fmt.Errorf("could not parse the target name '%s' to CID: %w", resource, err) + return cid.ID{}, err + } + var cid cid.ID + err := cid.DecodeString(cidStr) + return cid, err +} + func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainLocalOverrideRequest) (*control.AddChainLocalOverrideResponse, error) { if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) } - var cid cid.ID - err := cid.Decode(req.GetBody().GetContainerId()) + target := req.GetBody().GetTarget() + if target.Type != control.ChainTarget_CONTAINER { + return nil, status.Error(codes.Internal, fmt.Errorf("target type is not supported: %s", target.Type.String()).Error()) + } + + cid, err := extractCID(target.GetName()) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -37,7 +61,9 @@ func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainL s.apeChainCounter.Add(1) // TODO (aarifullin): the such chain id is not well-designed yet. - chain.ID = apechain.ID(fmt.Sprintf("%s:%d", apechain.Ingress, s.apeChainCounter.Load())) + if chain.ID == "" { + chain.ID = apechain.ID(fmt.Sprintf("%s:%d", apechain.Ingress, s.apeChainCounter.Load())) + } resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) if _, err = src.LocalStorage().AddOverride(apechain.Ingress, resource, &chain); err != nil { @@ -61,8 +87,12 @@ func (s *Server) GetChainLocalOverride(_ context.Context, req *control.GetChainL return nil, status.Error(codes.PermissionDenied, err.Error()) } - var cid cid.ID - err := cid.Decode(req.GetBody().GetContainerId()) + target := req.GetBody().GetTarget() + if target.Type != control.ChainTarget_CONTAINER { + return nil, status.Error(codes.Internal, fmt.Errorf("target type is not supported: %s", target.Type.String()).Error()) + } + + cid, err := extractCID(target.GetName()) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -95,8 +125,12 @@ func (s *Server) ListChainLocalOverrides(_ context.Context, req *control.ListCha return nil, status.Error(codes.PermissionDenied, err.Error()) } - var cid cid.ID - err := cid.Decode(req.GetBody().GetContainerId()) + target := req.GetBody().GetTarget() + if target.Type != control.ChainTarget_CONTAINER { + return nil, status.Error(codes.Internal, fmt.Errorf("target type is not supported: %s", target.Type.String()).Error()) + } + + cid, err := extractCID(target.GetName()) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -133,8 +167,12 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove return nil, status.Error(codes.PermissionDenied, err.Error()) } - var cid cid.ID - err := cid.Decode(req.GetBody().GetContainerId()) + target := req.GetBody().GetTarget() + if target.Type != control.ChainTarget_CONTAINER { + return nil, status.Error(codes.Internal, fmt.Errorf("target type is not supported: %s", target.Type.String()).Error()) + } + + cid, err := extractCID(target.GetName()) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 346a43664..8b8739aba 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -3198,9 +3198,11 @@ type AddChainLocalOverrideRequest_Body struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Container id for which the overrides are applied. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` - // Serialized rule chain. + // Target for which the overrides are applied. + Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` + // Serialized rule chain. If chain ID is left empty + // in the chain, then it will be generated and returned + // in the response. Chain []byte `protobuf:"bytes,2,opt,name=chain,proto3" json:"chain,omitempty"` } @@ -3236,9 +3238,9 @@ func (*AddChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24, 0} } -func (x *AddChainLocalOverrideRequest_Body) GetContainerId() []byte { +func (x *AddChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { if x != nil { - return x.ContainerId + return x.Target } return nil } @@ -3256,6 +3258,8 @@ type AddChainLocalOverrideResponse_Body struct { unknownFields protoimpl.UnknownFields // Chain ID assigned for the added rule chain. + // If chain ID is left empty in the request, then + // it will be generated. ChainId string `protobuf:"bytes,1,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` } @@ -3303,8 +3307,8 @@ type GetChainLocalOverrideRequest_Body struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Container id for which the overrides are defined. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` + // Target for which the overrides are applied. + Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` // Chain ID assigned for the added rule chain. ChainId string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` } @@ -3341,9 +3345,9 @@ func (*GetChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26, 0} } -func (x *GetChainLocalOverrideRequest_Body) GetContainerId() []byte { +func (x *GetChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { if x != nil { - return x.ContainerId + return x.Target } return nil } @@ -3408,8 +3412,8 @@ type ListChainLocalOverridesRequest_Body struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Container id for which the overrides are defined. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` + // Target for which the overrides are applied. + Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` } func (x *ListChainLocalOverridesRequest_Body) Reset() { @@ -3444,9 +3448,9 @@ func (*ListChainLocalOverridesRequest_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28, 0} } -func (x *ListChainLocalOverridesRequest_Body) GetContainerId() []byte { +func (x *ListChainLocalOverridesRequest_Body) GetTarget() *ChainTarget { if x != nil { - return x.ContainerId + return x.Target } return nil } @@ -3504,8 +3508,8 @@ type RemoveChainLocalOverrideRequest_Body struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Container id for which the overrides are defined. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` + // Target for which the overrides are applied. + Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` // Chain ID assigned for the added rule chain. ChainId string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` } @@ -3542,9 +3546,9 @@ func (*RemoveChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30, 0} } -func (x *RemoveChainLocalOverrideRequest_Body) GetContainerId() []byte { +func (x *RemoveChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { if x != nil { - return x.ContainerId + return x.Target } return nil } @@ -3905,7 +3909,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xd1, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xdc, 0x01, 0x0a, 0x1c, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, @@ -3915,191 +3919,194 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x3f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, - 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, - 0x69, 0x6e, 0x22, 0xb5, 0x01, 0x0a, 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, + 0x65, 0x1a, 0x4a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, + 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xb5, 0x01, + 0x0a, 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, + 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, + 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xe1, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, + 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, + 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xca, 0x01, 0x0a, + 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x40, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, + 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, + 0x61, 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, + 0x6e, 0x73, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xba, 0x01, 0x0a, + 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xd6, 0x01, 0x0a, 0x1c, 0x47, - 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x44, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, - 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, - 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, - 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xbf, 0x01, 0x0a, 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x29, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, 0x73, - 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, - 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x61, - 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x73, 0x22, 0xdc, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x44, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, - 0x22, 0xba, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x20, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x32, 0x9d, 0x0b, - 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, - 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, - 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, - 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, - 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, - 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, - 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, - 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, - 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, - 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, - 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, - 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x20, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x32, 0x9d, 0x0b, 0x0a, 0x0e, 0x43, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, + 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, + 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, + 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, + 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, + 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, + 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, + 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, + 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, - 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, - 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, - 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, - 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, - 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, - 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, + 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, + 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, + 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, + 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, + 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, + 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, - 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, - 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, - 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, + 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, + 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, + 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, + 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4189,6 +4196,7 @@ var file_pkg_services_control_service_proto_goTypes = []interface{}{ (HealthStatus)(0), // 69: control.HealthStatus (*ShardInfo)(nil), // 70: control.ShardInfo (ShardMode)(0), // 71: control.ShardMode + (*ChainTarget)(nil), // 72: control.ChainTarget } var file_pkg_services_control_service_proto_depIdxs = []int32{ 33, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body @@ -4263,43 +4271,47 @@ var file_pkg_services_control_service_proto_depIdxs = []int32{ 0, // 69: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status 56, // 70: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration 55, // 71: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - 1, // 72: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 3, // 73: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 5, // 74: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 7, // 75: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 9, // 76: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 11, // 77: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 13, // 78: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 19, // 79: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest - 21, // 80: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest - 23, // 81: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest - 15, // 82: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 17, // 83: control.ControlService.Doctor:input_type -> control.DoctorRequest - 25, // 84: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest - 27, // 85: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest - 29, // 86: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest - 31, // 87: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest - 2, // 88: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 4, // 89: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 6, // 90: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 8, // 91: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 10, // 92: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 12, // 93: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 14, // 94: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 20, // 95: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse - 22, // 96: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse - 24, // 97: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse - 16, // 98: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 18, // 99: control.ControlService.Doctor:output_type -> control.DoctorResponse - 26, // 100: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse - 28, // 101: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse - 30, // 102: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse - 32, // 103: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse - 88, // [88:104] is the sub-list for method output_type - 72, // [72:88] is the sub-list for method input_type - 72, // [72:72] is the sub-list for extension type_name - 72, // [72:72] is the sub-list for extension extendee - 0, // [0:72] is the sub-list for field type_name + 72, // 72: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 72, // 73: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 72, // 74: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget + 72, // 75: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 1, // 76: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 3, // 77: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 5, // 78: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 7, // 79: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 9, // 80: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 11, // 81: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 13, // 82: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 19, // 83: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest + 21, // 84: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest + 23, // 85: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest + 15, // 86: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 17, // 87: control.ControlService.Doctor:input_type -> control.DoctorRequest + 25, // 88: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest + 27, // 89: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest + 29, // 90: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest + 31, // 91: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest + 2, // 92: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 4, // 93: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 6, // 94: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 8, // 95: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 10, // 96: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 12, // 97: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 14, // 98: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 20, // 99: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse + 22, // 100: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse + 24, // 101: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse + 16, // 102: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 18, // 103: control.ControlService.Doctor:output_type -> control.DoctorResponse + 26, // 104: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse + 28, // 105: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse + 30, // 106: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse + 32, // 107: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse + 92, // [92:108] is the sub-list for method output_type + 76, // [76:92] is the sub-list for method input_type + 76, // [76:76] is the sub-list for extension type_name + 76, // [76:76] is the sub-list for extension extendee + 0, // [0:76] is the sub-list for field type_name } func init() { file_pkg_services_control_service_proto_init() } diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index e7d218f20..cbc3aaf54 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -421,10 +421,12 @@ message StopShardEvacuationResponse { // AddChainLocalOverride request. message AddChainLocalOverrideRequest { message Body { - // Container id for which the overrides are applied. - bytes container_id = 1; + // Target for which the overrides are applied. + ChainTarget target = 1; - // Serialized rule chain. + // Serialized rule chain. If chain ID is left empty + // in the chain, then it will be generated and returned + // in the response. bytes chain = 2; } @@ -437,6 +439,8 @@ message AddChainLocalOverrideRequest { message AddChainLocalOverrideResponse { message Body { // Chain ID assigned for the added rule chain. + // If chain ID is left empty in the request, then + // it will be generated. string chain_id = 1; } @@ -448,8 +452,8 @@ message AddChainLocalOverrideResponse { // GetChainLocalOverride request. message GetChainLocalOverrideRequest { message Body { - // Container id for which the overrides are defined. - bytes container_id = 1; + // Target for which the overrides are applied. + ChainTarget target = 1; // Chain ID assigned for the added rule chain. string chain_id = 2; @@ -475,8 +479,8 @@ message GetChainLocalOverrideResponse { // ListChainLocalOverrides request. message ListChainLocalOverridesRequest { message Body { - // Container id for which the overrides are defined. - bytes container_id = 1; + // Target for which the overrides are applied. + ChainTarget target = 1; } Body body = 1; @@ -498,8 +502,8 @@ message ListChainLocalOverridesResponse { message RemoveChainLocalOverrideRequest { message Body { - // Container id for which the overrides are defined. - bytes container_id = 1; + // Target for which the overrides are applied. + ChainTarget target = 1; // Chain ID assigned for the added rule chain. string chain_id = 2; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 852788a30..e163e54b7 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -2067,7 +2067,7 @@ func (x *AddChainLocalOverrideRequest_Body) StableSize() (size int) { if x == nil { return 0 } - size += proto.BytesSize(1, x.ContainerId) + size += proto.NestedStructureSize(1, x.Target) size += proto.BytesSize(2, x.Chain) return size } @@ -2088,7 +2088,7 @@ func (x *AddChainLocalOverrideRequest_Body) StableMarshal(buf []byte) []byte { buf = make([]byte, x.StableSize()) } var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) offset += proto.BytesMarshal(2, buf[offset:], x.Chain) return buf } @@ -2241,7 +2241,7 @@ func (x *GetChainLocalOverrideRequest_Body) StableSize() (size int) { if x == nil { return 0 } - size += proto.BytesSize(1, x.ContainerId) + size += proto.NestedStructureSize(1, x.Target) size += proto.StringSize(2, x.ChainId) return size } @@ -2262,7 +2262,7 @@ func (x *GetChainLocalOverrideRequest_Body) StableMarshal(buf []byte) []byte { buf = make([]byte, x.StableSize()) } var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) offset += proto.StringMarshal(2, buf[offset:], x.ChainId) return buf } @@ -2415,7 +2415,7 @@ func (x *ListChainLocalOverridesRequest_Body) StableSize() (size int) { if x == nil { return 0 } - size += proto.BytesSize(1, x.ContainerId) + size += proto.NestedStructureSize(1, x.Target) return size } @@ -2435,7 +2435,7 @@ func (x *ListChainLocalOverridesRequest_Body) StableMarshal(buf []byte) []byte { buf = make([]byte, x.StableSize()) } var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) return buf } @@ -2587,7 +2587,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) StableSize() (size int) { if x == nil { return 0 } - size += proto.BytesSize(1, x.ContainerId) + size += proto.NestedStructureSize(1, x.Target) size += proto.StringSize(2, x.ChainId) return size } @@ -2608,7 +2608,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) StableMarshal(buf []byte) []byte buf = make([]byte, x.StableSize()) } var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) offset += proto.StringMarshal(2, buf[offset:], x.ChainId) return buf } diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index f031f8043..8f083d695 100644 --- a/pkg/services/control/types.pb.go +++ b/pkg/services/control/types.pb.go @@ -199,6 +199,55 @@ func (ShardMode) EnumDescriptor() ([]byte, []int) { return file_pkg_services_control_types_proto_rawDescGZIP(), []int{2} } +type ChainTarget_TargetType int32 + +const ( + ChainTarget_UNDEFINED ChainTarget_TargetType = 0 + ChainTarget_NAMESPACE ChainTarget_TargetType = 1 + ChainTarget_CONTAINER ChainTarget_TargetType = 2 +) + +// Enum value maps for ChainTarget_TargetType. +var ( + ChainTarget_TargetType_name = map[int32]string{ + 0: "UNDEFINED", + 1: "NAMESPACE", + 2: "CONTAINER", + } + ChainTarget_TargetType_value = map[string]int32{ + "UNDEFINED": 0, + "NAMESPACE": 1, + "CONTAINER": 2, + } +) + +func (x ChainTarget_TargetType) Enum() *ChainTarget_TargetType { + p := new(ChainTarget_TargetType) + *p = x + return p +} + +func (x ChainTarget_TargetType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ChainTarget_TargetType) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_services_control_types_proto_enumTypes[3].Descriptor() +} + +func (ChainTarget_TargetType) Type() protoreflect.EnumType { + return &file_pkg_services_control_types_proto_enumTypes[3] +} + +func (x ChainTarget_TargetType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ChainTarget_TargetType.Descriptor instead. +func (ChainTarget_TargetType) EnumDescriptor() ([]byte, []int) { + return file_pkg_services_control_types_proto_rawDescGZIP(), []int{5, 0} +} + // Signature of some message. type Signature struct { state protoimpl.MessageState @@ -555,6 +604,63 @@ func (x *BlobstorInfo) GetType() string { return "" } +// ChainTarget is an object to which local overrides +// are applied. +type ChainTarget struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Type ChainTarget_TargetType `protobuf:"varint,1,opt,name=type,proto3,enum=control.ChainTarget_TargetType" json:"type,omitempty"` + Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` +} + +func (x *ChainTarget) Reset() { + *x = ChainTarget{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_types_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ChainTarget) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChainTarget) ProtoMessage() {} + +func (x *ChainTarget) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_types_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChainTarget.ProtoReflect.Descriptor instead. +func (*ChainTarget) Descriptor() ([]byte, []int) { + return file_pkg_services_control_types_proto_rawDescGZIP(), []int{5} +} + +func (x *ChainTarget) GetType() ChainTarget_TargetType { + if x != nil { + return x.Type + } + return ChainTarget_UNDEFINED +} + +func (x *ChainTarget) GetName() string { + if x != nil { + return x.Name + } + return "" +} + // Administrator-defined Attributes of the FrostFS Storage Node. // // `Attribute` is a Key-Value metadata pair. Key name must be a valid UTF-8 @@ -608,7 +714,7 @@ type NodeInfo_Attribute struct { func (x *NodeInfo_Attribute) Reset() { *x = NodeInfo_Attribute{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_types_proto_msgTypes[5] + mi := &file_pkg_services_control_types_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -621,7 +727,7 @@ func (x *NodeInfo_Attribute) String() string { func (*NodeInfo_Attribute) ProtoMessage() {} func (x *NodeInfo_Attribute) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_types_proto_msgTypes[5] + mi := &file_pkg_services_control_types_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -708,31 +814,40 @@ var file_pkg_services_control_types_proto_rawDesc = []byte{ 0x6d, 0x61, 0x50, 0x61, 0x74, 0x68, 0x22, 0x36, 0x0a, 0x0c, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x2a, 0x4e, - 0x0a, 0x0c, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, - 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, - 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x46, 0x46, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x0f, 0x0a, - 0x0b, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x2a, 0x6a, - 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, - 0x0a, 0x17, 0x48, 0x45, 0x41, 0x4c, 0x54, 0x48, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, - 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x41, - 0x44, 0x59, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x48, 0x55, 0x54, 0x54, 0x49, 0x4e, 0x47, - 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x43, 0x4f, 0x4e, - 0x46, 0x49, 0x47, 0x55, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x04, 0x2a, 0x6a, 0x0a, 0x09, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x48, 0x41, 0x52, 0x44, - 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, - 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, - 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, - 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x16, - 0x0a, 0x12, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x5f, - 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x04, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, - 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, - 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, - 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x91, + 0x01, 0x0a, 0x0b, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x33, + 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, + 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x39, 0x0a, 0x0a, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, + 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, + 0x10, 0x02, 0x2a, 0x4e, 0x0a, 0x0c, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, + 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x4e, 0x4c, 0x49, + 0x4e, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x46, 0x46, 0x4c, 0x49, 0x4e, 0x45, 0x10, + 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, + 0x10, 0x03, 0x2a, 0x6a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x17, 0x48, 0x45, 0x41, 0x4c, 0x54, 0x48, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, + 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, + 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x48, 0x55, 0x54, + 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x52, + 0x45, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x55, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x04, 0x2a, 0x6a, + 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x53, + 0x48, 0x41, 0x52, 0x44, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, + 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x57, 0x52, + 0x49, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, + 0x4c, 0x59, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, + 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x5f, 0x52, + 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x04, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, + 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, + 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, + 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -747,30 +862,33 @@ func file_pkg_services_control_types_proto_rawDescGZIP() []byte { return file_pkg_services_control_types_proto_rawDescData } -var file_pkg_services_control_types_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_pkg_services_control_types_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_pkg_services_control_types_proto_enumTypes = make([]protoimpl.EnumInfo, 4) +var file_pkg_services_control_types_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_pkg_services_control_types_proto_goTypes = []interface{}{ - (NetmapStatus)(0), // 0: control.NetmapStatus - (HealthStatus)(0), // 1: control.HealthStatus - (ShardMode)(0), // 2: control.ShardMode - (*Signature)(nil), // 3: control.Signature - (*NodeInfo)(nil), // 4: control.NodeInfo - (*Netmap)(nil), // 5: control.Netmap - (*ShardInfo)(nil), // 6: control.ShardInfo - (*BlobstorInfo)(nil), // 7: control.BlobstorInfo - (*NodeInfo_Attribute)(nil), // 8: control.NodeInfo.Attribute + (NetmapStatus)(0), // 0: control.NetmapStatus + (HealthStatus)(0), // 1: control.HealthStatus + (ShardMode)(0), // 2: control.ShardMode + (ChainTarget_TargetType)(0), // 3: control.ChainTarget.TargetType + (*Signature)(nil), // 4: control.Signature + (*NodeInfo)(nil), // 5: control.NodeInfo + (*Netmap)(nil), // 6: control.Netmap + (*ShardInfo)(nil), // 7: control.ShardInfo + (*BlobstorInfo)(nil), // 8: control.BlobstorInfo + (*ChainTarget)(nil), // 9: control.ChainTarget + (*NodeInfo_Attribute)(nil), // 10: control.NodeInfo.Attribute } var file_pkg_services_control_types_proto_depIdxs = []int32{ - 8, // 0: control.NodeInfo.attributes:type_name -> control.NodeInfo.Attribute - 0, // 1: control.NodeInfo.state:type_name -> control.NetmapStatus - 4, // 2: control.Netmap.nodes:type_name -> control.NodeInfo - 7, // 3: control.ShardInfo.blobstor:type_name -> control.BlobstorInfo - 2, // 4: control.ShardInfo.mode:type_name -> control.ShardMode - 5, // [5:5] is the sub-list for method output_type - 5, // [5:5] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 10, // 0: control.NodeInfo.attributes:type_name -> control.NodeInfo.Attribute + 0, // 1: control.NodeInfo.state:type_name -> control.NetmapStatus + 5, // 2: control.Netmap.nodes:type_name -> control.NodeInfo + 8, // 3: control.ShardInfo.blobstor:type_name -> control.BlobstorInfo + 2, // 4: control.ShardInfo.mode:type_name -> control.ShardMode + 3, // 5: control.ChainTarget.type:type_name -> control.ChainTarget.TargetType + 6, // [6:6] is the sub-list for method output_type + 6, // [6:6] is the sub-list for method input_type + 6, // [6:6] is the sub-list for extension type_name + 6, // [6:6] is the sub-list for extension extendee + 0, // [0:6] is the sub-list for field type_name } func init() { file_pkg_services_control_types_proto_init() } @@ -840,6 +958,18 @@ func file_pkg_services_control_types_proto_init() { } } file_pkg_services_control_types_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChainTarget); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_types_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*NodeInfo_Attribute); i { case 0: return &v.state @@ -857,8 +987,8 @@ func file_pkg_services_control_types_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_types_proto_rawDesc, - NumEnums: 3, - NumMessages: 6, + NumEnums: 4, + NumMessages: 7, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto index b34988c2e..d1bc05e29 100644 --- a/pkg/services/control/types.proto +++ b/pkg/services/control/types.proto @@ -169,3 +169,20 @@ enum ShardMode { // DegradedReadOnly. DEGRADED_READ_ONLY = 4; } + + +// ChainTarget is an object to which local overrides +// are applied. +message ChainTarget { + enum TargetType { + UNDEFINED = 0; + + NAMESPACE = 1; + + CONTAINER = 2; + } + + TargetType type = 1; + + string Name = 2; +} \ No newline at end of file diff --git a/pkg/services/control/types_frostfs.pb.go b/pkg/services/control/types_frostfs.pb.go index ab246dbdc..858d85a1c 100644 --- a/pkg/services/control/types_frostfs.pb.go +++ b/pkg/services/control/types_frostfs.pb.go @@ -229,3 +229,36 @@ func (x *BlobstorInfo) StableMarshal(buf []byte) []byte { offset += proto.StringMarshal(2, buf[offset:], x.Type) return buf } + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ChainTarget) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.EnumSize(1, int32(x.Type)) + size += proto.StringSize(2, x.Name) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ChainTarget) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.EnumMarshal(1, buf[offset:], int32(x.Type)) + offset += proto.StringMarshal(2, buf[offset:], x.Name) + return buf +} From e361e017f37ed878a9be0c225abdb9181e8c7a43 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Tue, 5 Dec 2023 13:26:09 +0300 Subject: [PATCH 0734/1943] [#842] control: Pass target instead resource name * Update policy-engine package version in go.mod, go.sum. * Refactor CheckIfRequestPermitted: pass container target instead container ID. Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 ++-- pkg/services/control/server/policy_engine.go | 8 ++++---- pkg/services/object/acl/ape.go | 5 ++++- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index aa0f689c1..43ff1648a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231128145636-a0a35bf4bf31 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231205092054-2d4a9fc6dcb3 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 diff --git a/go.sum b/go.sum index 0167d4a51..bede02d05 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231128145636-a0a35bf4bf31 h1:31tE+hNkHar7IMduhzj8LFY8+LT2EswiVI873MEfZj0= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231128145636-a0a35bf4bf31/go.mod h1:ekrDiIySdYhji5rBNAkxYMztFWMXyC9Q8LVz6gGVDu0= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231205092054-2d4a9fc6dcb3 h1:d4cCtg6vgQ101Qni9FqYaGPkmSJP1ZnEyHYMI+JaTIo= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231205092054-2d4a9fc6dcb3/go.mod h1:ekrDiIySdYhji5rBNAkxYMztFWMXyC9Q8LVz6gGVDu0= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 1c09de918..cbb6ba803 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -66,7 +66,7 @@ func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainL } resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) - if _, err = src.LocalStorage().AddOverride(apechain.Ingress, resource, &chain); err != nil { + if _, err = src.LocalStorage().AddOverride(apechain.Ingress, engine.ContainerTarget(resource), &chain); err != nil { return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } @@ -103,7 +103,7 @@ func (s *Server) GetChainLocalOverride(_ context.Context, req *control.GetChainL } resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) - chain, err := src.LocalStorage().GetOverride(apechain.Ingress, resource, apechain.ID(req.GetBody().GetChainId())) + chain, err := src.LocalStorage().GetOverride(apechain.Ingress, engine.ContainerTarget(resource), apechain.ID(req.GetBody().GetChainId())) if err != nil { return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } @@ -141,7 +141,7 @@ func (s *Server) ListChainLocalOverrides(_ context.Context, req *control.ListCha } resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) - chains, err := src.LocalStorage().ListOverrides(apechain.Ingress, resource) + chains, err := src.LocalStorage().ListOverrides(apechain.Ingress, engine.ContainerTarget(resource)) if err != nil { return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } @@ -183,7 +183,7 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove } resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) - if err = src.LocalStorage().RemoveOverride(apechain.Ingress, resource, apechain.ID(req.GetBody().GetChainId())); err != nil { + if err = src.LocalStorage().RemoveOverride(apechain.Ingress, engine.ContainerTarget(resource), apechain.ID(req.GetBody().GetChainId())); err != nil { return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } resp := &control.RemoveChainLocalOverrideResponse{ diff --git a/pkg/services/object/acl/ape.go b/pkg/services/object/acl/ape.go index 525ed5930..a48bb5c9f 100644 --- a/pkg/services/object/acl/ape.go +++ b/pkg/services/object/acl/ape.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" ) var errAPEChainNoSource = errors.New("could not get ape chain source for the container") @@ -36,7 +37,9 @@ func (c *apeCheckerImpl) CheckIfRequestPermitted(reqInfo v2.RequestInfo) error { request := new(Request) request.FromRequestInfo(reqInfo) - status, ruleFound, err := chainCache.IsAllowed(apechain.Ingress, "", request) + cnrTarget := getResource(reqInfo).Name() + + status, ruleFound, err := chainCache.IsAllowed(apechain.Ingress, policyengine.NewRequestTargetWithContainer(cnrTarget), request) if err != nil { return err } From 0f45e3d344da440797800c4adaa3f90f4e7c787b Mon Sep 17 00:00:00 2001 From: aarifullin Date: Mon, 20 Nov 2023 19:35:16 +0300 Subject: [PATCH 0735/1943] [#804] ape: Implement boltdb storage for local overrides Signed-off-by: Airat Arifullin --- cmd/frostfs-node/config.go | 39 ++- cmd/frostfs-node/config/node/config.go | 55 +++- cmd/frostfs-node/control.go | 2 +- cmd/frostfs-node/main.go | 6 + cmd/frostfs-node/object.go | 4 +- cmd/frostfs-node/policy_engine.go | 70 ++++-- internal/logs/logs.go | 2 + pkg/ape/chainbase/boltdb.go | 250 +++++++++++++++++++ pkg/ape/chainbase/inmemory.go | 30 +++ pkg/ape/chainbase/interface.go | 22 ++ pkg/ape/chainbase/option.go | 67 +++++ pkg/core/container/storage.go | 8 - pkg/services/control/server/policy_engine.go | 105 ++------ pkg/services/control/server/server.go | 19 +- pkg/services/object/acl/ape.go | 23 +- 15 files changed, 560 insertions(+), 142 deletions(-) create mode 100644 pkg/ape/chainbase/boltdb.go create mode 100644 pkg/ape/chainbase/inmemory.go create mode 100644 pkg/ape/chainbase/interface.go create mode 100644 pkg/ape/chainbase/option.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 2d80a4115..82b01b563 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -29,6 +29,7 @@ import ( replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -68,6 +69,7 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" neogoutil "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" @@ -518,10 +520,7 @@ type cfgObject struct { eaclSource container.EACLSource - // Access policy chain source is used by object service to - // check for operation permissions but this source is also shared with - // control service that dispatches local overrides. - apeChainSource container.AccessPolicyEngineChainSource + cfgAccessPolicyEngine cfgAccessPolicyEngine pool cfgObjectRoutines @@ -542,6 +541,10 @@ type cfgLocalStorage struct { localStorage *engine.StorageEngine } +type cfgAccessPolicyEngine struct { + accessPolicyEngine *accessPolicyEngine +} + type cfgObjectRoutines struct { putRemote *ants.Pool @@ -970,6 +973,34 @@ func initLocalStorage(ctx context.Context, c *cfg) { }) } +func initAccessPolicyEngine(_ context.Context, c *cfg) { + var localOverrideDB chainbase.LocalOverrideDatabase + if nodeconfig.PersistentPolicyRules(c.appCfg).Path() == "" { + c.log.Warn(logs.FrostFSNodePersistentRuleStorageDBPathIsNotSetInmemoryWillBeUsed) + localOverrideDB = chainbase.NewInmemoryLocalOverrideDatabase() + } else { + localOverrideDB = chainbase.NewBoltLocalOverrideDatabase( + chainbase.WithLogger(c.log), + chainbase.WithPath(nodeconfig.PersistentPolicyRules(c.appCfg).Path()), + chainbase.WithPerm(nodeconfig.PersistentPolicyRules(c.appCfg).Perm()), + chainbase.WithNoSync(nodeconfig.PersistentPolicyRules(c.appCfg).NoSync()), + ) + } + + morphRuleStorage := inmemory.NewInmemoryMorphRuleChainStorage() + + ape := newAccessPolicyEngine(morphRuleStorage, localOverrideDB) + c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine = ape + + c.onShutdown(func() { + if err := ape.LocalOverrideDatabaseCore().Close(); err != nil { + c.log.Warn(logs.FrostFSNodeAccessPolicyEngineClosingFailure, + zap.Error(err), + ) + } + }) +} + func initObjectPool(cfg *config.Config) (pool cfgObjectRoutines) { var err error diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index 9dfe8ddf4..ac76ad47e 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -2,6 +2,7 @@ package nodeconfig import ( "fmt" + "io/fs" "os" "strconv" "time" @@ -30,11 +31,18 @@ type NotificationConfig struct { cfg *config.Config } +// PersistentPolicyRulesConfig is a wrapper over "persistent_policy_rules" config section +// which provides access to persistent policy rules storage configuration of node. +type PersistentPolicyRulesConfig struct { + cfg *config.Config +} + const ( - subsection = "node" - persistentSessionsSubsection = "persistent_sessions" - persistentStateSubsection = "persistent_state" - notificationSubsection = "notification" + subsection = "node" + persistentSessionsSubsection = "persistent_sessions" + persistentStateSubsection = "persistent_state" + notificationSubsection = "notification" + persistentPolicyRulesSubsection = "persistent_policy_rules" attributePrefix = "attribute" @@ -245,3 +253,42 @@ func (n NotificationConfig) KeyPath() string { func (n NotificationConfig) CAPath() string { return config.StringSafe(n.cfg, "ca") } + +const ( + // PermDefault is a default permission bits for local override storage file. + PermDefault = 0o644 +) + +// PersistentPolicyRules returns structure that provides access to "persistent_policy_rules" +// subsection of "node" section. +func PersistentPolicyRules(c *config.Config) PersistentPolicyRulesConfig { + return PersistentPolicyRulesConfig{ + c.Sub(subsection).Sub(persistentPolicyRulesSubsection), + } +} + +// Path returns the value of "path" config parameter. +// +// Returns empty string if missing, for compatibility with older configurations. +func (l PersistentPolicyRulesConfig) Path() string { + return config.StringSafe(l.cfg, "path") +} + +// Perm returns the value of "perm" config parameter as a fs.FileMode. +// +// Returns PermDefault if the value is not a positive number. +func (l PersistentPolicyRulesConfig) Perm() fs.FileMode { + p := config.UintSafe((*config.Config)(l.cfg), "perm") + if p == 0 { + p = PermDefault + } + + return fs.FileMode(p) +} + +// NoSync returns the value of "no_sync" config parameter as a bool value. +// +// Returns false if the value is not a boolean. +func (l PersistentPolicyRulesConfig) NoSync() bool { + return config.BoolSafe((*config.Config)(l.cfg), "no_sync") +} diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index 30d644803..4f5c5d780 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -51,7 +51,7 @@ func initControlService(c *cfg) { controlSvc.WithTreeService(treeSynchronizer{ c.treeService, }), - controlSvc.WithAPEChainSource(c.cfgObject.apeChainSource), + controlSvc.WithLocalOverrideStorage(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine), ) lis, err := net.Listen("tcp", endpoint) diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 2791efcef..d4dfb7e60 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -98,6 +98,12 @@ func initApp(ctx context.Context, c *cfg) { fatalOnErr(c.cfgObject.cfgLocalStorage.localStorage.Init(ctx)) }) + initAccessPolicyEngine(ctx, c) + initAndLog(c, "access policy engine", func(c *cfg) { + fatalOnErr(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalOverrideDatabaseCore().Open(ctx)) + fatalOnErr(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalOverrideDatabaseCore().Init()) + }) + initAndLog(c, "gRPC", initGRPC) initAndLog(c, "netmap", func(c *cfg) { initNetmapService(ctx, c) }) initAndLog(c, "accounting", func(c *cfg) { initAccountingService(ctx, c) }) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 79f492190..59827179b 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -157,8 +157,6 @@ func initObjectService(c *cfg) { c.replicator = createReplicator(c, keyStorage, c.bgClientCache) - c.cfgObject.apeChainSource = NewAPESource() - addPolicer(c, keyStorage, c.bgClientCache) traverseGen := util.NewTraverserGenerator(c.netMapSource, c.cfgObject.cnrSource, c) @@ -426,7 +424,7 @@ func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter, irFet c.cfgObject.eaclSource, eaclSDK.NewValidator(), ls), - acl.NewAPEChecker(c.log, c.cfgObject.apeChainSource), + acl.NewAPEChecker(c.log, c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.chainRouter), c.cfgObject.cnrSource, v2.WithLogger(c.log), ) diff --git a/cmd/frostfs-node/policy_engine.go b/cmd/frostfs-node/policy_engine.go index 248cddb11..d04322c2e 100644 --- a/cmd/frostfs-node/policy_engine.go +++ b/cmd/frostfs-node/policy_engine.go @@ -3,33 +3,63 @@ package main import ( "sync" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" - "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" ) -type apeChainSourceImpl struct { - mtx sync.Mutex - localChainStorage map[cid.ID]engine.LocalOverrideEngine +type accessPolicyEngine struct { + mtx sync.RWMutex + + chainRouter engine.ChainRouter + + morphChainStorage engine.MorphRuleChainStorage + + localOverrideDatabase chainbase.LocalOverrideDatabase } -func NewAPESource() container.AccessPolicyEngineChainSource { - return &apeChainSourceImpl{ - localChainStorage: make(map[cid.ID]engine.LocalOverrideEngine), +var _ engine.LocalOverrideEngine = (*accessPolicyEngine)(nil) + +func newAccessPolicyEngine( + morphChainStorage engine.MorphRuleChainStorage, + localOverrideDatabase chainbase.LocalOverrideDatabase) *accessPolicyEngine { + return &accessPolicyEngine{ + chainRouter: engine.NewDefaultChainRouterWithLocalOverrides( + morphChainStorage, + localOverrideDatabase, + ), + + morphChainStorage: morphChainStorage, + + localOverrideDatabase: localOverrideDatabase, } } -var _ container.AccessPolicyEngineChainSource = (*apeChainSourceImpl)(nil) +func (a *accessPolicyEngine) IsAllowed(name chain.Name, target engine.RequestTarget, r resource.Request) (status chain.Status, found bool, err error) { + a.mtx.RLock() + defer a.mtx.RUnlock() -func (c *apeChainSourceImpl) GetChainSource(cid cid.ID) (engine.LocalOverrideEngine, error) { - c.mtx.Lock() - defer c.mtx.Unlock() - - s, ok := c.localChainStorage[cid] - if ok { - return s, nil - } - c.localChainStorage[cid] = inmemory.NewInMemoryLocalOverrides() - return c.localChainStorage[cid], nil + return a.chainRouter.IsAllowed(name, target, r) +} + +func (a *accessPolicyEngine) MorphRuleChainStorage() engine.MorphRuleChainStorage { + a.mtx.Lock() + defer a.mtx.Unlock() + + return a.morphChainStorage +} + +func (a *accessPolicyEngine) LocalStorage() engine.LocalOverrideStorage { + a.mtx.Lock() + defer a.mtx.Unlock() + + return a.localOverrideDatabase +} + +func (a *accessPolicyEngine) LocalOverrideDatabaseCore() chainbase.DatabaseCore { + a.mtx.Lock() + defer a.mtx.Unlock() + + return a.localOverrideDatabase } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index e1c2cfdee..befa7342a 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -429,7 +429,9 @@ const ( FrostFSNodeFailedToAttachShardToEngine = "failed to attach shard to engine" FrostFSNodeShardAttachedToEngine = "shard attached to engine" FrostFSNodeClosingComponentsOfTheStorageEngine = "closing components of the storage engine..." + FrostFSNodeAccessPolicyEngineClosingFailure = "ape closing failure" FrostFSNodeStorageEngineClosingFailure = "storage engine closing failure" + FrostFSNodePersistentRuleStorageDBPathIsNotSetInmemoryWillBeUsed = "persistent rule storage db path is not set: in-memory will be used" FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully = "all components of the storage engine closed successfully" FrostFSNodeBootstrappingWithTheMaintenanceState = "bootstrapping with the maintenance state" FrostFSNodeBootstrappingWithOnlineState = "bootstrapping with online state" diff --git a/pkg/ape/chainbase/boltdb.go b/pkg/ape/chainbase/boltdb.go new file mode 100644 index 000000000..c6cd2c014 --- /dev/null +++ b/pkg/ape/chainbase/boltdb.go @@ -0,0 +1,250 @@ +package chainbase + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "github.com/nspcc-dev/neo-go/pkg/util/slice" + "go.etcd.io/bbolt" +) + +type boltLocalOverrideStorage struct { + *cfg + + db *bbolt.DB +} + +var ( + chainBucket = []byte{0} +) + +var ( + ErrChainBucketNotFound = logicerr.New("chain root bucket has not been found") + + ErrChainNotFound = logicerr.New("chain has not been found") + + ErrGlobalNamespaceBucketNotFound = logicerr.New("global namespace bucket has not been found") + + ErrTargetTypeBucketNotFound = logicerr.New("target type bucket has not been found") + + ErrTargetNameBucketNotFound = logicerr.New("target name bucket has not been found") +) + +// NewBoltLocalOverrideDatabase returns storage wrapper for storing access policy engine +// local overrides. +// +// chain storage (chainBucket): +// -> global namespace bucket (nBucket): +// --> target bucket (tBucket) +// ---> target name (resource) bucket (rBucket): +// +// | Key | Value | +// x---------------------x-------------------x +// | chain id (string) | serialized chain | +// x---------------------x-------------------x +// +//nolint:godot +func NewBoltLocalOverrideDatabase(opts ...Option) LocalOverrideDatabase { + c := defaultCfg() + + for i := range opts { + opts[i](c) + } + + return &boltLocalOverrideStorage{ + cfg: c, + } +} + +func (cs *boltLocalOverrideStorage) Init() error { + return cs.db.Update(func(tx *bbolt.Tx) error { + _, err := tx.CreateBucketIfNotExists(chainBucket) + return err + }) +} + +func (cs *boltLocalOverrideStorage) Open(context.Context) error { + err := util.MkdirAllX(filepath.Dir(cs.path), cs.perm) + if err != nil { + return fmt.Errorf("can't create dir %s for the chain DB: %w", cs.path, err) + } + + opts := *bbolt.DefaultOptions + opts.NoSync = cs.noSync + opts.Timeout = 100 * time.Millisecond + + cs.db, err = bbolt.Open(cs.path, cs.perm, &opts) + if err != nil { + return fmt.Errorf("can't open the chain DB: %w", err) + } + + cs.db.MaxBatchSize = cs.maxBatchSize + cs.db.MaxBatchDelay = cs.maxBatchDelay + + return nil +} + +func (cs *boltLocalOverrideStorage) Close() error { + var err error + if cs.db != nil { + err = cs.db.Close() + } + return err +} + +func getTargetBucket(tx *bbolt.Tx, name chain.Name, target policyengine.Target) (*bbolt.Bucket, error) { + cbucket := tx.Bucket(chainBucket) + if cbucket == nil { + return nil, ErrChainBucketNotFound + } + + nbucket := cbucket.Bucket([]byte(name)) + if nbucket == nil { + return nil, fmt.Errorf("global namespace %s: %w", name, ErrGlobalNamespaceBucketNotFound) + } + + typeBucket := nbucket.Bucket([]byte{byte(target.Type)}) + if typeBucket == nil { + return nil, fmt.Errorf("type bucket '%c': %w", target.Type, ErrTargetTypeBucketNotFound) + } + + rbucket := typeBucket.Bucket([]byte(target.Name)) + if rbucket == nil { + return nil, fmt.Errorf("target name bucket %s: %w", target.Name, ErrTargetNameBucketNotFound) + } + + return rbucket, nil +} + +func getTargetBucketCreateIfEmpty(tx *bbolt.Tx, name chain.Name, target policyengine.Target) (*bbolt.Bucket, error) { + cbucket := tx.Bucket(chainBucket) + if cbucket == nil { + return nil, ErrChainBucketNotFound + } + + nbucket := cbucket.Bucket([]byte(name)) + if nbucket == nil { + var err error + nbucket, err = cbucket.CreateBucket([]byte(name)) + if err != nil { + return nil, fmt.Errorf("could not create a bucket for the global chain name %s: %w", name, err) + } + } + + typeBucket := nbucket.Bucket([]byte{byte(target.Type)}) + if typeBucket == nil { + var err error + typeBucket, err = cbucket.CreateBucket([]byte{byte(target.Type)}) + if err != nil { + return nil, fmt.Errorf("could not create a bucket for the target type '%c': %w", target.Type, err) + } + } + + rbucket := typeBucket.Bucket([]byte(target.Name)) + if rbucket == nil { + var err error + rbucket, err = typeBucket.CreateBucket([]byte(target.Name)) + if err != nil { + return nil, fmt.Errorf("could not create a bucket for the target name %s: %w", target.Name, err) + } + } + + return rbucket, nil +} + +func (cs *boltLocalOverrideStorage) AddOverride(name chain.Name, target policyengine.Target, c *chain.Chain) (chain.ID, error) { + if c.ID == "" { + return "", fmt.Errorf("chain ID is not set") + } + + serializedChain := c.Bytes() + + err := cs.db.Update(func(tx *bbolt.Tx) error { + rbuck, err := getTargetBucketCreateIfEmpty(tx, name, target) + if err != nil { + return err + } + return rbuck.Put([]byte(c.ID), serializedChain) + }) + + return c.ID, err +} + +func (cs *boltLocalOverrideStorage) GetOverride(name chain.Name, target policyengine.Target, chainID chain.ID) (*chain.Chain, error) { + var serializedChain []byte + + if err := cs.db.View(func(tx *bbolt.Tx) error { + rbuck, err := getTargetBucket(tx, name, target) + if err != nil { + return err + } + serializedChain = rbuck.Get([]byte(chainID)) + if serializedChain == nil { + return ErrChainNotFound + } + serializedChain = slice.Copy(serializedChain) + return nil + }); err != nil { + return nil, err + } + + c := &chain.Chain{} + if err := json.Unmarshal(serializedChain, c); err != nil { + return nil, err + } + return c, nil +} + +func (cs *boltLocalOverrideStorage) RemoveOverride(name chain.Name, target policyengine.Target, chainID chain.ID) error { + return cs.db.Update(func(tx *bbolt.Tx) error { + rbuck, err := getTargetBucket(tx, name, target) + if err != nil { + return err + } + return rbuck.Delete([]byte(chainID)) + }) +} + +func (cs *boltLocalOverrideStorage) ListOverrides(name chain.Name, target policyengine.Target) ([]*chain.Chain, error) { + var serializedChains [][]byte + var serializedChain []byte + if err := cs.db.View(func(tx *bbolt.Tx) error { + rbuck, err := getTargetBucket(tx, name, target) + if err != nil { + return err + } + return rbuck.ForEach(func(_, v []byte) error { + serializedChain = slice.Copy(v) + serializedChains = append(serializedChains, serializedChain) + return nil + }) + }); err != nil { + if errors.Is(err, ErrGlobalNamespaceBucketNotFound) || errors.Is(err, ErrTargetNameBucketNotFound) { + return []*chain.Chain{}, nil + } + return nil, err + } + chains := make([]*chain.Chain, 0, len(serializedChains)) + for _, serializedChain = range serializedChains { + c := &chain.Chain{} + if err := json.Unmarshal(serializedChain, c); err != nil { + return nil, err + } + chains = append(chains, c) + } + return chains, nil +} + +func (cs *boltLocalOverrideStorage) DropAllOverrides(name chain.Name) error { + return cs.db.Update(func(tx *bbolt.Tx) error { + return tx.DeleteBucket([]byte(name)) + }) +} diff --git a/pkg/ape/chainbase/inmemory.go b/pkg/ape/chainbase/inmemory.go new file mode 100644 index 000000000..27712d959 --- /dev/null +++ b/pkg/ape/chainbase/inmemory.go @@ -0,0 +1,30 @@ +package chainbase + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" +) + +type inmemoryLocalOverrideStorage struct { + engine.LocalOverrideStorage +} + +func NewInmemoryLocalOverrideDatabase() LocalOverrideDatabase { + return &inmemoryLocalOverrideStorage{ + LocalOverrideStorage: inmemory.NewInmemoryLocalStorage(), + } +} + +func (cs *inmemoryLocalOverrideStorage) Init() error { + return nil +} + +func (cs *inmemoryLocalOverrideStorage) Open(_ context.Context) error { + return nil +} + +func (cs *inmemoryLocalOverrideStorage) Close() error { + return nil +} diff --git a/pkg/ape/chainbase/interface.go b/pkg/ape/chainbase/interface.go new file mode 100644 index 000000000..ee445f22c --- /dev/null +++ b/pkg/ape/chainbase/interface.go @@ -0,0 +1,22 @@ +package chainbase + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" +) + +// DatabaseCore interface provides methods to initialize and manage local override storage +// as database. +type DatabaseCore interface { + Init() error + Open(context.Context) error + Close() error +} + +// LocalOverrideDatabase interface provides methods to manage local override storage +// as database and as the APE's local override storage. +type LocalOverrideDatabase interface { + DatabaseCore + engine.LocalOverrideStorage +} diff --git a/pkg/ape/chainbase/option.go b/pkg/ape/chainbase/option.go new file mode 100644 index 000000000..e547701fb --- /dev/null +++ b/pkg/ape/chainbase/option.go @@ -0,0 +1,67 @@ +package chainbase + +import ( + "io/fs" + "os" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "go.etcd.io/bbolt" + "go.uber.org/zap" +) + +type Option func(*cfg) + +type cfg struct { + path string + perm fs.FileMode + noSync bool + maxBatchDelay time.Duration + maxBatchSize int + log *logger.Logger +} + +func defaultCfg() *cfg { + return &cfg{ + perm: os.ModePerm, + maxBatchDelay: bbolt.DefaultMaxBatchDelay, + maxBatchSize: bbolt.DefaultMaxBatchSize, + log: &logger.Logger{Logger: zap.L()}, + } +} + +func WithPath(path string) Option { + return func(c *cfg) { + c.path = path + } +} + +func WithPerm(perm fs.FileMode) Option { + return func(c *cfg) { + c.perm = perm + } +} + +func WithNoSync(noSync bool) Option { + return func(c *cfg) { + c.noSync = noSync + } +} + +func WithMaxBatchDelay(maxBatchDelay time.Duration) Option { + return func(c *cfg) { + c.maxBatchDelay = maxBatchDelay + } +} + +func WithMaxBatchSize(maxBatchSize int) Option { + return func(c *cfg) { + c.maxBatchSize = maxBatchSize + } +} + +func WithLogger(l *logger.Logger) Option { + return func(c *cfg) { + c.log = l + } +} diff --git a/pkg/core/container/storage.go b/pkg/core/container/storage.go index cc358b436..69854f495 100644 --- a/pkg/core/container/storage.go +++ b/pkg/core/container/storage.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" ) // Container groups information about the FrostFS container stored in the FrostFS network. @@ -72,10 +71,3 @@ type EACLSource interface { // eACL table is not in source. GetEACL(cid.ID) (*EACL, error) } - -// AccessPolicyEngineChainSource interface provides methods to access and manipulate -// policy engine chain storage. -type AccessPolicyEngineChainSource interface { - // TODO (aarifullin): Better to use simpler interface instead CachedChainStorage. - GetChainSource(cid cid.ID) (engine.LocalOverrideEngine, error) -} diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index cbb6ba803..7ffa392a8 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -4,34 +4,24 @@ import ( "context" "errors" "fmt" - "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" engine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" - nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) -// extractCID extracts CID from the schema's pattern. -// TODO (aarifullin): This is temporary solution should be replaced by -// resource name validation. -func extractCID(resource string) (cid.ID, error) { - var cidStr string - - // Sscanf requires to make tokens delimited by spaces. - pattern := strings.Replace(nativeschema.ResourceFormatRootContainerObjects, "/", " ", -1) - resource = strings.Replace(resource, "/", " ", -1) - - if _, err := fmt.Sscanf(resource, pattern, &cidStr); err != nil { - err = fmt.Errorf("could not parse the target name '%s' to CID: %w", resource, err) - return cid.ID{}, err +func apeTarget(chainTarget *control.ChainTarget) (engine.Target, error) { + switch chainTarget.GetType() { + case control.ChainTarget_CONTAINER: + return engine.ContainerTarget(chainTarget.GetName()), nil + case control.ChainTarget_NAMESPACE: + return engine.NamespaceTarget(chainTarget.GetName()), nil + default: } - var cid cid.ID - err := cid.DecodeString(cidStr) - return cid, err + return engine.Target{}, status.Error(codes.InvalidArgument, + fmt.Errorf("target type is not supported: %s", chainTarget.GetType().String()).Error()) } func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainLocalOverrideRequest) (*control.AddChainLocalOverrideResponse, error) { @@ -39,34 +29,23 @@ func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainL return nil, status.Error(codes.PermissionDenied, err.Error()) } - target := req.GetBody().GetTarget() - if target.Type != control.ChainTarget_CONTAINER { - return nil, status.Error(codes.Internal, fmt.Errorf("target type is not supported: %s", target.Type.String()).Error()) - } - - cid, err := extractCID(target.GetName()) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - var chain apechain.Chain - if err = chain.DecodeBytes(req.GetBody().GetChain()); err != nil { + if err := chain.DecodeBytes(req.GetBody().GetChain()); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } - src, err := s.apeChainSrc.GetChainSource(cid) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - s.apeChainCounter.Add(1) // TODO (aarifullin): the such chain id is not well-designed yet. if chain.ID == "" { chain.ID = apechain.ID(fmt.Sprintf("%s:%d", apechain.Ingress, s.apeChainCounter.Load())) } - resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) - if _, err = src.LocalStorage().AddOverride(apechain.Ingress, engine.ContainerTarget(resource), &chain); err != nil { + target, err := apeTarget(req.GetBody().GetTarget()) + if err != nil { + return nil, err + } + + if _, err = s.localOverrideStorage.LocalStorage().AddOverride(apechain.Ingress, target, &chain); err != nil { return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } @@ -87,23 +66,11 @@ func (s *Server) GetChainLocalOverride(_ context.Context, req *control.GetChainL return nil, status.Error(codes.PermissionDenied, err.Error()) } - target := req.GetBody().GetTarget() - if target.Type != control.ChainTarget_CONTAINER { - return nil, status.Error(codes.Internal, fmt.Errorf("target type is not supported: %s", target.Type.String()).Error()) - } - - cid, err := extractCID(target.GetName()) + target, err := apeTarget(req.GetBody().GetTarget()) if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) + return nil, err } - - src, err := s.apeChainSrc.GetChainSource(cid) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) - chain, err := src.LocalStorage().GetOverride(apechain.Ingress, engine.ContainerTarget(resource), apechain.ID(req.GetBody().GetChainId())) + chain, err := s.localOverrideStorage.LocalStorage().GetOverride(apechain.Ingress, target, apechain.ID(req.GetBody().GetChainId())) if err != nil { return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } @@ -125,23 +92,12 @@ func (s *Server) ListChainLocalOverrides(_ context.Context, req *control.ListCha return nil, status.Error(codes.PermissionDenied, err.Error()) } - target := req.GetBody().GetTarget() - if target.Type != control.ChainTarget_CONTAINER { - return nil, status.Error(codes.Internal, fmt.Errorf("target type is not supported: %s", target.Type.String()).Error()) - } - - cid, err := extractCID(target.GetName()) + target, err := apeTarget(req.GetBody().GetTarget()) if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) + return nil, err } - src, err := s.apeChainSrc.GetChainSource(cid) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) - chains, err := src.LocalStorage().ListOverrides(apechain.Ingress, engine.ContainerTarget(resource)) + chains, err := s.localOverrideStorage.LocalStorage().ListOverrides(apechain.Ingress, target) if err != nil { return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } @@ -167,23 +123,12 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove return nil, status.Error(codes.PermissionDenied, err.Error()) } - target := req.GetBody().GetTarget() - if target.Type != control.ChainTarget_CONTAINER { - return nil, status.Error(codes.Internal, fmt.Errorf("target type is not supported: %s", target.Type.String()).Error()) - } - - cid, err := extractCID(target.GetName()) + target, err := apeTarget(req.GetBody().GetTarget()) if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) + return nil, err } - src, err := s.apeChainSrc.GetChainSource(cid) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - resource := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) - if err = src.LocalStorage().RemoveOverride(apechain.Ingress, engine.ContainerTarget(resource), apechain.ID(req.GetBody().GetChainId())); err != nil { + if err = s.localOverrideStorage.LocalStorage().RemoveOverride(apechain.Ingress, target, apechain.ID(req.GetBody().GetChainId())); err != nil { return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } resp := &control.RemoveChainLocalOverrideResponse{ diff --git a/pkg/services/control/server/server.go b/pkg/services/control/server/server.go index 48bc85200..7cfa93f05 100644 --- a/pkg/services/control/server/server.go +++ b/pkg/services/control/server/server.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" ) // Server is an entity that serves @@ -51,6 +52,14 @@ type NodeState interface { ForceMaintenance() error } +// LocalOverrideStorageDecorator interface provides methods to decorate LocalOverrideEngine +// interface methods. +type LocalOverrideStorageDecorator interface { + // LocalStorage method can be decorated by using sync primitives in the case if the local + // override storage state should be consistent for chain router. + LocalStorage() policyengine.LocalOverrideStorage +} + // Option of the Server's constructor. type Option func(*cfg) @@ -65,7 +74,7 @@ type cfg struct { cnrSrc container.Source - apeChainSrc container.AccessPolicyEngineChainSource + localOverrideStorage LocalOverrideStorageDecorator replicator *replicator.Replicator @@ -160,10 +169,10 @@ func WithTreeService(s TreeService) Option { } } -// WithAPEChainSource returns the option to set access policy engine -// chain source. -func WithAPEChainSource(apeChainSrc container.AccessPolicyEngineChainSource) Option { +// WithLocalOverrideStorage returns the option to set access policy engine +// chain override storage. +func WithLocalOverrideStorage(localOverrideStorage LocalOverrideStorageDecorator) Option { return func(c *cfg) { - c.apeChainSrc = apeChainSrc + c.localOverrideStorage = localOverrideStorage } } diff --git a/pkg/services/object/acl/ape.go b/pkg/services/object/acl/ape.go index a48bb5c9f..e2a884c96 100644 --- a/pkg/services/object/acl/ape.go +++ b/pkg/services/object/acl/ape.go @@ -1,10 +1,8 @@ package acl import ( - "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -12,34 +10,25 @@ import ( policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" ) -var errAPEChainNoSource = errors.New("could not get ape chain source for the container") - type apeCheckerImpl struct { - log *logger.Logger - apeSrc container.AccessPolicyEngineChainSource + log *logger.Logger + chainRouter policyengine.ChainRouter } -func NewAPEChecker(log *logger.Logger, apeSrc container.AccessPolicyEngineChainSource) v2.APEChainChecker { +func NewAPEChecker(log *logger.Logger, chainRouter policyengine.ChainRouter) v2.APEChainChecker { return &apeCheckerImpl{ - log: log, - apeSrc: apeSrc, + log: log, + chainRouter: chainRouter, } } func (c *apeCheckerImpl) CheckIfRequestPermitted(reqInfo v2.RequestInfo) error { - cnr := reqInfo.ContainerID() - - chainCache, err := c.apeSrc.GetChainSource(cnr) - if err != nil { - return errAPEChainNoSource - } - request := new(Request) request.FromRequestInfo(reqInfo) cnrTarget := getResource(reqInfo).Name() - status, ruleFound, err := chainCache.IsAllowed(apechain.Ingress, policyengine.NewRequestTargetWithContainer(cnrTarget), request) + status, ruleFound, err := c.chainRouter.IsAllowed(apechain.Ingress, policyengine.NewRequestTargetWithContainer(cnrTarget), request) if err != nil { return err } From 70e0c1e082a0c1b833d8a3106790f015c7c52382 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 5 Dec 2023 11:16:44 +0300 Subject: [PATCH 0736/1943] [#841] ir: Execute `netmap.addPeerIR` only for state `online` Signed-off-by: Anton Nikiforov --- pkg/innerring/processors/netmap/handlers_test.go | 15 +++++++++++++++ pkg/innerring/processors/netmap/process_peers.go | 7 ++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index 164ee41da..6a66fe323 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -144,6 +144,21 @@ func TestAddPeer(t *testing.T) { time.Sleep(10 * time.Millisecond) } + require.Nil(t, nc.notaryInvokes, "invalid notary invokes") + + node.SetOnline() + ev = netmapEvent.AddPeer{ + NodeBytes: node.Marshal(), + Request: &payload.P2PNotaryRequest{ + MainTransaction: &transaction.Transaction{}, + }, + } + proc.handleAddPeer(ev) + + for proc.pool.Running() > 0 { + time.Sleep(10 * time.Millisecond) + } + require.EqualValues(t, []notaryInvoke{ { contract: nc.contractAddress, diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index 96b8c8e97..41e4bfb7e 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -57,7 +57,12 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { updated := np.netmapSnapshot.touch(keyString, np.epochState.EpochCounter(), nodeInfoBinary) - if updated { + // `processAddPeer` reacts on `AddPeer` notification, `processNewEpoch` - on `NewEpoch`. + // This two notification produces in order - `NewEpoch` -> `AddPeer`. + // But there is no guarantee that code will be executed in the same order. + // That is why we need to perform `addPeerIR` only in case when node is online, + // because in scope of this method, contract set state `ONLINE` for the node. + if updated && nodeInfo.IsOnline() { np.log.Info(logs.NetmapApprovingNetworkMapCandidate, zap.String("key", keyString)) From 46532fb9ce8b20c40646c91cd6e4f19ff4b6f9fc Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 5 Dec 2023 11:18:25 +0300 Subject: [PATCH 0737/1943] [#841] doc: Describe `epoch` Signed-off-by: Anton Nikiforov --- docs/epoch.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 docs/epoch.md diff --git a/docs/epoch.md b/docs/epoch.md new file mode 100644 index 000000000..2f33dcfd5 --- /dev/null +++ b/docs/epoch.md @@ -0,0 +1,43 @@ +# Epoch + +The main purpose of the `epoch` in `frostfs` environment is to manipulate `netmap`. +Each new epoch, `ir` service trigger revision content of the `netmap` by adding or removing nodes to or from it. +`node` service trigger few internal processes each new epoch - for example, running GC. +Epoch also used in an object lifecycle. + +At the startup, `ir` service initializes an epoch timer which handles new epoch tick. +Epoch timer is a block timer - which means that this timer ticks each block or set of blocks. +The epoch duration stores in the configurable parameter `EpochDuration` in the blockchain. +It is possible to get it via `frostfs-adm`: +```shell +> frostfs-adm morph dump-config -c config.yml -r http://morph-chain.frostfs.devenv:30333 +... +EpochDuration: 240 (int) +... +> +``` +Once epoch timer ticks, `ir` service call method [NewEpoch](https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/commit/a1b61d3949581f4d65b0d32a33d98ba9c193dc2a/netmap/netmap_contract.go#L238) +of the `netmap` contract. Each `ir` instance can do this at the same time, but it is not an issue, +because multiple call of this method with the same set of parameters will give us the same result. + +Utility `frostfs-adm` have a command to trigger new epoch: +```shell +> frostfs-adm morph force-new-epoch -c config.yml -r http://morph-chain.frostfs.devenv:30333 +``` +Command goes directly to the `netmap` contract and call method `NewEpoch`. +Method checks alphabet witness and stores candidates nodes which are not in the `OFFLINE` state as a current netmap. +Then executes method `NewEpoch` in `balance` and `container` contracts. +At the end it produces notification `NewEpoch` which is handled by `node` and `ir` services. + +`ir` handler for `NewEpoch` updates internal state of the netmap, if it is necessary, updates state of the nodes or +marks for exclusion from netmap in the blockchain. + +`node` handler for `NewEpoch` executes method `addPeer` of the `netmap` contract. +This method do nothing, but produces notification which handled by `ir` service. +`ir` in handler for `AddPeer` may update node state in the netmap if it is necessary. + +At the startup, node bootstraps with state `ONLINE`. From the online state, it is possible to move to `MAINTENANCE` or `OFFLINE`. +Node moved to `OFFLINE` state automatically, when there is no bootstrap request from it for a number of epochs. +This number stored in the `ir` config `netmap_cleaner.threshold`. +From `OFFLINE` state node, once it bootstrapped, moves to `ONLINE`. +`MAINTENANCE` state persists even if node rebooted or unavailable for a few epochs. From d5c9dd3c839b46050519f7efdcc3447906c42f5e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Dec 2023 16:55:12 +0300 Subject: [PATCH 0738/1943] [#852] ape: Use first match for eACL->APE converter Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- internal/ape/converter.go | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 43ff1648a..3e79f7ff7 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231205092054-2d4a9fc6dcb3 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231211080303-8c673ee4f4af git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 diff --git a/go.sum b/go.sum index bede02d05..15ae22422 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231205092054-2d4a9fc6dcb3 h1:d4cCtg6vgQ101Qni9FqYaGPkmSJP1ZnEyHYMI+JaTIo= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231205092054-2d4a9fc6dcb3/go.mod h1:ekrDiIySdYhji5rBNAkxYMztFWMXyC9Q8LVz6gGVDu0= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231211080303-8c673ee4f4af h1:QSgejckGChrry5waJqf2votsOY0J7Sfh8tPqos0rCXA= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231211080303-8c673ee4f4af/go.mod h1:iJMX6qk9aIHIu3WVSd4puF5CHsNk5eOi++MaJJfNbXM= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/internal/ape/converter.go b/internal/ape/converter.go index dec6f80d8..f629e262d 100644 --- a/internal/ape/converter.go +++ b/internal/ape/converter.go @@ -33,7 +33,9 @@ func ConvertEACLToAPE(eaclTable *eacl.Table) (*apechain.Chain, error) { if eaclTable == nil { return nil, nil } - res := &apechain.Chain{} + res := &apechain.Chain{ + MatchType: apechain.MatchTypeFirstMatch, + } resource := getResource(eaclTable) From 1cd2bfe51a15a2e3a2707c93653ce76015284766 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 20 Nov 2023 14:32:26 +0300 Subject: [PATCH 0739/1943] [#755] morph: Drop FrostFSID contract usage Unused. Signed-off-by: Dmitrii Stepanov --- pkg/innerring/initialization.go | 23 +--- pkg/innerring/processors/container/common.go | 15 --- .../processors/container/handlers_test.go | 18 +-- .../processors/container/processor.go | 11 -- pkg/innerring/processors/frostfs/handlers.go | 36 ------ .../processors/frostfs/handlers_test.go | 87 -------------- .../processors/frostfs/process_bind.go | 109 ------------------ pkg/innerring/processors/frostfs/processor.go | 32 ----- pkg/morph/client/frostfs/bind.go | 71 ------------ pkg/morph/client/frostfs/client.go | 2 - pkg/morph/client/frostfsid/addrm_keys.go | 61 ---------- pkg/morph/client/frostfsid/client.go | 72 ------------ pkg/morph/client/frostfsid/keys.go | 54 --------- pkg/morph/event/frostfs/bind.go | 92 --------------- pkg/morph/event/frostfs/bind_test.go | 72 ------------ pkg/morph/event/frostfs/cheque_test.go | 9 ++ pkg/morph/event/frostfs/unbind.go | 33 ------ pkg/morph/event/frostfs/unbind_test.go | 63 ---------- 18 files changed, 16 insertions(+), 844 deletions(-) delete mode 100644 pkg/innerring/processors/frostfs/process_bind.go delete mode 100644 pkg/morph/client/frostfs/bind.go delete mode 100644 pkg/morph/client/frostfsid/addrm_keys.go delete mode 100644 pkg/morph/client/frostfsid/client.go delete mode 100644 pkg/morph/client/frostfsid/keys.go delete mode 100644 pkg/morph/event/frostfs/bind.go delete mode 100644 pkg/morph/event/frostfs/bind_test.go delete mode 100644 pkg/morph/event/frostfs/unbind.go delete mode 100644 pkg/morph/event/frostfs/unbind_test.go diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index f8ebe89fe..71571c072 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -20,7 +20,6 @@ import ( balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" frostfsClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" @@ -250,9 +249,7 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { return err } -func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.Client, - frostfsIDClient *frostfsid.Client, -) error { +func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.Client) error { // container processor containerProcessor, err := cont.New(&cont.Params{ Log: s.log, @@ -261,7 +258,6 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C AlphabetState: s, ContainerClient: cnrClient, MorphClient: cnrClient.Morph(), - FrostFSIDClient: frostfsIDClient, NetworkState: s.netmapClient, }) if err != nil { @@ -289,7 +285,7 @@ func (s *Server) initBalanceProcessor(cfg *viper.Viper, frostfsCli *frostfsClien return bindMorphProcessor(balanceProcessor, s) } -func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper, frostfsIDClient *frostfsid.Client) error { +func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper) error { if s.withoutMainNet { return nil } @@ -299,7 +295,6 @@ func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper, frostfsIDClient * Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.frostfs"), FrostFSContract: s.contracts.frostfs, - FrostFSIDClient: frostfsIDClient, BalanceClient: s.balanceClient, NetmapClient: s.netmapClient, MorphClient: s.morphClient, @@ -369,9 +364,8 @@ func (s *Server) initGRPCServer(cfg *viper.Viper) error { } type serverMorphClients struct { - CnrClient *container.Client - FrostFSIDClient *frostfsid.Client - FrostFSClient *frostfsClient.Client + CnrClient *container.Client + FrostFSClient *frostfsClient.Client } func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { @@ -403,11 +397,6 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { return nil, err } - result.FrostFSIDClient, err = frostfsid.NewFromMorph(s.morphClient, s.contracts.frostfsID, fee, frostfsid.TryNotary(), frostfsid.AsAlphabet()) - if err != nil { - return nil, err - } - result.FrostFSClient, err = frostfsClient.NewFromMorph(s.mainnetClient, s.contracts.frostfs, s.feeConfig.MainChainFee(), frostfsClient.TryNotary(), frostfsClient.AsAlphabet()) if err != nil { @@ -437,7 +426,7 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien return err } - err = s.initContainerProcessor(cfg, morphClients.CnrClient, morphClients.FrostFSIDClient) + err = s.initContainerProcessor(cfg, morphClients.CnrClient) if err != nil { return err } @@ -447,7 +436,7 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien return err } - err = s.initFrostFSMainnetProcessor(cfg, morphClients.FrostFSIDClient) + err = s.initFrostFSMainnetProcessor(cfg) if err != nil { return err } diff --git a/pkg/innerring/processors/container/common.go b/pkg/innerring/processors/container/common.go index 97eb6f559..ba12ebb37 100644 --- a/pkg/innerring/processors/container/common.go +++ b/pkg/innerring/processors/container/common.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" @@ -73,20 +72,6 @@ func (cp *Processor) verifySignature(v signatureVerificationData) error { return errors.New("invalid signature calculated by container owner's key") } - } else { - var prm frostfsid.AccountKeysPrm - prm.SetID(v.ownerContainer) - - ownerKeys, err := cp.idClient.AccountKeys(prm) - if err != nil { - return fmt.Errorf("receive owner keys %s: %w", v.ownerContainer, err) - } - - for i := range ownerKeys { - if (*frostfsecdsa.PublicKeyRFC6979)(ownerKeys[i]).Verify(v.signedData, v.signature) { - return nil - } - } } return errors.New("signature is invalid or calculated with the key not bound to the container owner") diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index f129127a6..3acfa2251 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -8,7 +8,6 @@ import ( containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -40,7 +39,6 @@ func TestPutEvent(t *testing.T) { Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, - FrostFSIDClient: &testIDClient{}, NetworkState: nst, ContainerClient: &testContainerClient{}, MorphClient: mc, @@ -95,18 +93,12 @@ func TestDeleteEvent(t *testing.T) { p, err := keys.NewPrivateKey() require.NoError(t, err) - idc := &testIDClient{ - publicKeys: []*keys.PublicKey{ - p.PublicKey(), - }, - } mc := &testMorphClient{} proc, err := New(&Params{ Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, - FrostFSIDClient: idc, NetworkState: nst, ContainerClient: cc, MorphClient: mc, @@ -139,6 +131,7 @@ func TestDeleteEvent(t *testing.T) { ContainerIDValue: cidBin, SignatureValue: p.Sign(cidBin), NotaryRequestValue: nr, + PublicKeyValue: p.PublicKey().Bytes(), } var signature frostfscrypto.Signature @@ -177,7 +170,6 @@ func TestSetEACLEvent(t *testing.T) { Log: test.NewLogger(t, true), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, - FrostFSIDClient: &testIDClient{}, NetworkState: nst, ContainerClient: cc, MorphClient: mc, @@ -285,14 +277,6 @@ func (c *testContainerClient) Get(cid []byte) (*containercore.Container, error) return nil, new(apistatus.ContainerNotFound) } -type testIDClient struct { - publicKeys keys.PublicKeys -} - -func (c *testIDClient) AccountKeys(p frostfsid.AccountKeysPrm) (keys.PublicKeys, error) { - return c.publicKeys, nil -} - var _ putEvent = &testPutEvent{} type testPutEvent struct { diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index fd5348c6f..9f0ae77bf 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -7,13 +7,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/core/mempoolevent" "github.com/nspcc-dev/neo-go/pkg/core/transaction" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" "go.uber.org/zap" @@ -34,10 +32,6 @@ type ( NotarySignAndInvokeTX(mainTx *transaction.Transaction) error } - IDClient interface { - AccountKeys(p frostfsid.AccountKeysPrm) (keys.PublicKeys, error) - } - // Processor of events produced by container contract in the sidechain. Processor struct { log *logger.Logger @@ -46,7 +40,6 @@ type ( alphabetState AlphabetState cnrClient ContClient // notary must be enabled morphClient MorphClient - idClient IDClient netState NetworkState } @@ -58,7 +51,6 @@ type ( AlphabetState AlphabetState ContainerClient ContClient MorphClient MorphClient - FrostFSIDClient IDClient NetworkState NetworkState } ) @@ -92,8 +84,6 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/container: Container client is not set") case p.MorphClient == nil: return nil, errors.New("ir/container: Morph client is not set") - case p.FrostFSIDClient == nil: - return nil, errors.New("ir/container: FrostFS ID client is not set") case p.NetworkState == nil: return nil, errors.New("ir/container: network state is not set") } @@ -116,7 +106,6 @@ func New(p *Params) (*Processor, error) { pool: pool, alphabetState: p.AlphabetState, cnrClient: p.ContainerClient, - idClient: p.FrostFSIDClient, netState: p.NetworkState, morphClient: p.MorphClient, }, nil diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index adb923b87..5782dc744 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -88,39 +88,3 @@ func (np *Processor) handleConfig(ev event.Event) { zap.Int("capacity", np.pool.Cap())) } } - -func (np *Processor) handleBind(ev event.Event) { - e := ev.(frostfsEvent.Bind) - np.log.Info(logs.Notification, - zap.String("type", "bind"), - ) - - // send event to the worker pool - - err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_bind", func() bool { - return np.processBind(e, true) - }) - if err != nil { - // there system can be moved into controlled degradation stage - np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, - zap.Int("capacity", np.pool.Cap())) - } -} - -func (np *Processor) handleUnbind(ev event.Event) { - e := ev.(frostfsEvent.Unbind) - np.log.Info(logs.Notification, - zap.String("type", "unbind"), - ) - - // send event to the worker pool - - err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_unbind", func() bool { - return np.processBind(e, false) - }) - if err != nil { - // there system can be moved into controlled degradation stage - np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, - zap.Int("capacity", np.pool.Cap())) - } -} diff --git a/pkg/innerring/processors/frostfs/handlers_test.go b/pkg/innerring/processors/frostfs/handlers_test.go index 475ddaaac..2685808ed 100644 --- a/pkg/innerring/processors/frostfs/handlers_test.go +++ b/pkg/innerring/processors/frostfs/handlers_test.go @@ -5,12 +5,9 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/stretchr/testify/require" @@ -194,80 +191,11 @@ func TestHandleConfig(t *testing.T) { require.EqualValues(t, []nmClient.SetConfigPrm{expConfig}, nm.config, "invalid config value") } -func TestHandleUnbind(t *testing.T) { - t.Parallel() - es := &testEpochState{ - epochCounter: 100, - } - m := &testMorphClient{ - balance: 150, - } - id := &testIDClient{} - proc, err := newTestProc(t, func(p *Params) { - p.EpochState = es - p.MorphClient = m - p.FrostFSIDClient = id - }) - require.NoError(t, err, "failed to create processor") - - p, err := keys.NewPrivateKey() - require.NoError(t, err) - - evUnbind := frostfsEvent.Unbind{ - BindCommon: frostfsEvent.BindCommon{ - UserValue: util.Uint160{49}.BytesBE(), - KeysValue: [][]byte{ - p.PublicKey().Bytes(), - }, - TxHashValue: util.Uint256{100}, - }, - } - - proc.handleUnbind(evUnbind) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - var userID user.ID - userID.SetScriptHash(util.Uint160{49}) - - var expBind frostfsid.CommonBindPrm - expBind.SetOwnerID(userID.WalletBytes()) - expBind.SetKeys(evUnbind.BindCommon.KeysValue) - expBind.SetHash(evUnbind.BindCommon.TxHashValue) - - var expNilSlice []frostfsid.CommonBindPrm - - require.EqualValues(t, []frostfsid.CommonBindPrm{expBind}, id.remove, "invalid remove keys value") - require.EqualValues(t, expNilSlice, id.add, "invalid add keys value") - - evBind := frostfsEvent.Bind{ - BindCommon: frostfsEvent.BindCommon{ - UserValue: util.Uint160{49}.BytesBE(), - KeysValue: [][]byte{ - p.PublicKey().Bytes(), - }, - TxHashValue: util.Uint256{100}, - }, - } - - proc.handleBind(evBind) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - require.EqualValues(t, []frostfsid.CommonBindPrm{expBind}, id.remove, "invalid remove keys value") - require.EqualValues(t, []frostfsid.CommonBindPrm{expBind}, id.add, "invalid add keys value") -} - func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { p := &Params{ Log: test.NewLogger(t, true), PoolSize: 1, FrostFSContract: util.Uint160{0}, - FrostFSIDClient: &testIDClient{}, BalanceClient: &testBalaceClient{}, NetmapClient: &testNetmapClient{}, MorphClient: &testMorphClient{}, @@ -358,18 +286,3 @@ func (c *testMorphClient) TransferGas(receiver util.Uint160, amount fixedn.Fixed }) return nil } - -type testIDClient struct { - add []frostfsid.CommonBindPrm - remove []frostfsid.CommonBindPrm -} - -func (c *testIDClient) AddKeys(p frostfsid.CommonBindPrm) error { - c.add = append(c.add, p) - return nil -} - -func (c *testIDClient) RemoveKeys(args frostfsid.CommonBindPrm) error { - c.remove = append(c.remove, args) - return nil -} diff --git a/pkg/innerring/processors/frostfs/process_bind.go b/pkg/innerring/processors/frostfs/process_bind.go deleted file mode 100644 index 50c6bf5f5..000000000 --- a/pkg/innerring/processors/frostfs/process_bind.go +++ /dev/null @@ -1,109 +0,0 @@ -package frostfs - -import ( - "crypto/elliptic" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/util" - "go.uber.org/zap" -) - -type bindCommon interface { - User() []byte - Keys() [][]byte - TxHash() util.Uint256 -} - -func (np *Processor) processBind(e bindCommon, bind bool) bool { - if !np.alphabetState.IsAlphabet() { - np.log.Info(logs.FrostFSNonAlphabetModeIgnoreBind) - return true - } - - c := &bindCommonContext{ - bindCommon: e, - bind: bind, - } - - err := np.checkBindCommon(c) - if err != nil { - np.log.Error(logs.FrostFSInvalidManageKeyEvent, - zap.Bool("bind", c.bind), - zap.String("error", err.Error()), - ) - - return false - } - - return np.approveBindCommon(c) == nil -} - -type bindCommonContext struct { - bindCommon - - bind bool - - scriptHash util.Uint160 -} - -func (np *Processor) checkBindCommon(e *bindCommonContext) error { - var err error - - e.scriptHash, err = util.Uint160DecodeBytesBE(e.User()) - if err != nil { - return err - } - - curve := elliptic.P256() - - for _, key := range e.Keys() { - _, err = keys.NewPublicKeyFromBytes(key, curve) - if err != nil { - return err - } - } - - return nil -} - -func (np *Processor) approveBindCommon(e *bindCommonContext) error { - // calculate wallet address - scriptHash := e.User() - - u160, err := util.Uint160DecodeBytesBE(scriptHash) - if err != nil { - np.log.Error(logs.FrostFSCouldNotDecodeScriptHashFromBytes, - zap.String("error", err.Error()), - ) - - return err - } - - var id user.ID - id.SetScriptHash(u160) - - prm := frostfsid.CommonBindPrm{} - prm.SetOwnerID(id.WalletBytes()) - prm.SetKeys(e.Keys()) - prm.SetHash(e.bindCommon.TxHash()) - - var typ string - if e.bind { - typ = "bind" - err = np.frostfsIDClient.AddKeys(prm) - } else { - typ = "unbind" - err = np.frostfsIDClient.RemoveKeys(prm) - } - - if err != nil { - np.log.Error(fmt.Sprintf("could not approve %s", typ), - zap.String("error", err.Error())) - } - - return err -} diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index 20f44adcd..2019857ac 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" @@ -51,11 +50,6 @@ type ( TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error } - IDClient interface { - AddKeys(p frostfsid.CommonBindPrm) error - RemoveKeys(args frostfsid.CommonBindPrm) error - } - // Processor of events produced by frostfs contract in main net. Processor struct { log *logger.Logger @@ -73,7 +67,6 @@ type ( mintEmitThreshold uint64 mintEmitValue fixedn.Fixed8 gasBalanceThreshold int64 - frostfsIDClient IDClient } // Params of the processor constructor. @@ -82,7 +75,6 @@ type ( Metrics metrics.Register PoolSize int FrostFSContract util.Uint160 - FrostFSIDClient IDClient BalanceClient BalanceClient NetmapClient NetmapClient MorphClient MorphClient @@ -101,8 +93,6 @@ const ( withdrawNotification = "Withdraw" chequeNotification = "Cheque" configNotification = "SetConfig" - bindNotification = "Bind" - unbindNotification = "Unbind" ) // New creates frostfs mainnet contract processor instance. @@ -152,8 +142,6 @@ func New(p *Params) (*Processor, error) { mintEmitThreshold: p.MintEmitThreshold, mintEmitValue: p.MintEmitValue, gasBalanceThreshold: p.GasBalanceThreshold, - - frostfsIDClient: p.FrostFSIDClient, }, nil } @@ -187,16 +175,6 @@ func (np *Processor) ListenerNotificationParsers() []event.NotificationParserInf p.SetParser(frostfsEvent.ParseConfig) parsers = append(parsers, p) - // bind event - p.SetType(event.TypeFromString(bindNotification)) - p.SetParser(frostfsEvent.ParseBind) - parsers = append(parsers, p) - - // unbind event - p.SetType(event.TypeFromString(unbindNotification)) - p.SetParser(frostfsEvent.ParseUnbind) - parsers = append(parsers, p) - return parsers } @@ -230,16 +208,6 @@ func (np *Processor) ListenerNotificationHandlers() []event.NotificationHandlerI h.SetHandler(np.handleConfig) handlers = append(handlers, h) - // bind handler - h.SetType(event.TypeFromString(bindNotification)) - h.SetHandler(np.handleBind) - handlers = append(handlers, h) - - // unbind handler - h.SetType(event.TypeFromString(unbindNotification)) - h.SetHandler(np.handleUnbind) - handlers = append(handlers, h) - return handlers } diff --git a/pkg/morph/client/frostfs/bind.go b/pkg/morph/client/frostfs/bind.go deleted file mode 100644 index f7b91ddb8..000000000 --- a/pkg/morph/client/frostfs/bind.go +++ /dev/null @@ -1,71 +0,0 @@ -package frostfscontract - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" -) - -type commonBindArgs struct { - scriptHash []byte // script hash of account identifier - - keys [][]byte // list of serialized public keys - - client.InvokePrmOptional -} - -// SetOptionalPrm sets optional client parameters. -func (x *commonBindArgs) SetOptionalPrm(op client.InvokePrmOptional) { - x.InvokePrmOptional = op -} - -// SetScriptHash sets script hash of the FrostFS account identifier. -func (x *commonBindArgs) SetScriptHash(v []byte) { - x.scriptHash = v -} - -// SetKeys sets a list of public keys in a binary format. -func (x *commonBindArgs) SetKeys(v [][]byte) { - x.keys = v -} - -// BindKeysPrm groups parameters of BindKeys operation. -type BindKeysPrm struct { - commonBindArgs -} - -// BindKeys binds list of public keys from FrostFS account by script hash. -func (x *Client) BindKeys(p BindKeysPrm) error { - prm := client.InvokePrm{} - prm.SetMethod(bindKeysMethod) - prm.SetArgs(p.scriptHash, p.keys) - prm.InvokePrmOptional = p.InvokePrmOptional - - _, err := x.client.Invoke(prm) - if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", bindKeysMethod, err) - } - - return nil -} - -// UnbindKeysPrm groups parameters of UnbindKeys operation. -type UnbindKeysPrm struct { - commonBindArgs -} - -// UnbindKeys invokes the call of key unbinding method -// of FrostFS contract. -func (x *Client) UnbindKeys(args UnbindKeysPrm) error { - prm := client.InvokePrm{} - prm.SetMethod(unbindKeysMethod) - prm.SetArgs(args.scriptHash, args.keys) - prm.InvokePrmOptional = args.InvokePrmOptional - - _, err := x.client.Invoke(prm) - if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", unbindKeysMethod, err) - } - - return nil -} diff --git a/pkg/morph/client/frostfs/client.go b/pkg/morph/client/frostfs/client.go index 3e3e70ec0..571915c27 100644 --- a/pkg/morph/client/frostfs/client.go +++ b/pkg/morph/client/frostfs/client.go @@ -21,8 +21,6 @@ type Client struct { } const ( - bindKeysMethod = "bind" - unbindKeysMethod = "unbind" alphabetUpdateMethod = "alphabetUpdate" chequeMethod = "cheque" ) diff --git a/pkg/morph/client/frostfsid/addrm_keys.go b/pkg/morph/client/frostfsid/addrm_keys.go deleted file mode 100644 index e9004f366..000000000 --- a/pkg/morph/client/frostfsid/addrm_keys.go +++ /dev/null @@ -1,61 +0,0 @@ -package frostfsid - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" -) - -type CommonBindPrm struct { - ownerID []byte // FrostFS account identifier - - keys [][]byte // list of serialized public keys - - client.InvokePrmOptional -} - -func (x *CommonBindPrm) SetOptionalPrm(prm client.InvokePrmOptional) { - x.InvokePrmOptional = prm -} - -// SetOwnerID sets FrostFS account identifier. -func (x *CommonBindPrm) SetOwnerID(v []byte) { - x.ownerID = v -} - -// SetKeys sets a list of public keys in a binary format. -func (x *CommonBindPrm) SetKeys(v [][]byte) { - x.keys = v -} - -// AddKeys adds a list of public keys to/from FrostFS account. -func (x *Client) AddKeys(p CommonBindPrm) error { - prm := client.InvokePrm{} - - prm.SetMethod(addKeysMethod) - prm.SetArgs(p.ownerID, p.keys) - prm.InvokePrmOptional = p.InvokePrmOptional - - _, err := x.client.Invoke(prm) - if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", addKeysMethod, err) - } - - return nil -} - -// RemoveKeys removes a list of public keys to/from FrostFS account. -func (x *Client) RemoveKeys(args CommonBindPrm) error { - prm := client.InvokePrm{} - - prm.SetMethod(removeKeysMethod) - prm.SetArgs(args.ownerID, args.keys) - prm.InvokePrmOptional = args.InvokePrmOptional - - _, err := x.client.Invoke(prm) - if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", removeKeysMethod, err) - } - - return nil -} diff --git a/pkg/morph/client/frostfsid/client.go b/pkg/morph/client/frostfsid/client.go deleted file mode 100644 index dded61aff..000000000 --- a/pkg/morph/client/frostfsid/client.go +++ /dev/null @@ -1,72 +0,0 @@ -package frostfsid - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - "github.com/nspcc-dev/neo-go/pkg/util" -) - -// Client is a wrapper over StaticClient -// which makes calls with the names and arguments -// of the FrostFS ID contract. -// -// Working client must be created via constructor New. -// Using the Client that has been created with new(Client) -// expression (or just declaring a Client variable) is unsafe -// and can lead to panic. -type Client struct { - client *client.StaticClient // static FrostFS ID contract client -} - -const ( - keyListingMethod = "key" - addKeysMethod = "addKey" - removeKeysMethod = "removeKey" -) - -// NewFromMorph wraps client to work with FrostFS ID contract. -func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8, opts ...Option) (*Client, error) { - o := defaultOpts() - - for i := range opts { - opts[i](o) - } - - sc, err := client.NewStatic(cli, contract, fee, ([]client.StaticClientOption)(*o)...) - if err != nil { - return nil, fmt.Errorf("could not create client of FrostFS ID contract: %w", err) - } - - return &Client{client: sc}, nil -} - -// Option allows to set an optional -// parameter of ClientWrapper. -type Option func(*opts) - -type opts []client.StaticClientOption - -func defaultOpts() *opts { - return new(opts) -} - -// TryNotary returns option to enable -// notary invocation tries. -func TryNotary() Option { - return func(o *opts) { - *o = append(*o, client.TryNotary()) - } -} - -// AsAlphabet returns option to sign main TX -// of notary requests with client's private -// key. -// -// Considered to be used by IR nodes only. -func AsAlphabet() Option { - return func(o *opts) { - *o = append(*o, client.AsAlphabet()) - } -} diff --git a/pkg/morph/client/frostfsid/keys.go b/pkg/morph/client/frostfsid/keys.go deleted file mode 100644 index 3bae7adfb..000000000 --- a/pkg/morph/client/frostfsid/keys.go +++ /dev/null @@ -1,54 +0,0 @@ -package frostfsid - -import ( - "crypto/elliptic" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" -) - -// AccountKeysPrm groups parameters of AccountKeys operation. -type AccountKeysPrm struct { - id user.ID -} - -// SetID sets owner ID. -func (a *AccountKeysPrm) SetID(id user.ID) { - a.id = id -} - -// AccountKeys requests public keys of FrostFS account from FrostFS ID contract. -func (x *Client) AccountKeys(p AccountKeysPrm) (keys.PublicKeys, error) { - prm := client.TestInvokePrm{} - prm.SetMethod(keyListingMethod) - prm.SetArgs(p.id.WalletBytes()) - - items, err := x.client.TestInvoke(prm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", keyListingMethod, err) - } else if ln := len(items); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", keyListingMethod, ln) - } - - items, err = client.ArrayFromStackItem(items[0]) - if err != nil { - return nil, fmt.Errorf("1st stack item must be an array (%s)", keyListingMethod) - } - - pubs := make(keys.PublicKeys, len(items)) - for i := range items { - rawPub, err := client.BytesFromStackItem(items[i]) - if err != nil { - return nil, fmt.Errorf("invalid stack item, expected byte array (%s)", keyListingMethod) - } - - pubs[i], err = keys.NewPublicKeyFromBytes(rawPub, elliptic.P256()) - if err != nil { - return nil, fmt.Errorf("received invalid key (%s): %w", keyListingMethod, err) - } - } - - return pubs, nil -} diff --git a/pkg/morph/event/frostfs/bind.go b/pkg/morph/event/frostfs/bind.go deleted file mode 100644 index 8655b1222..000000000 --- a/pkg/morph/event/frostfs/bind.go +++ /dev/null @@ -1,92 +0,0 @@ -package frostfs - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" -) - -type Bind struct { - BindCommon -} - -type BindCommon struct { - UserValue []byte - KeysValue [][]byte - - // TxHashValue is used in notary environmental - // for calculating unique but same for - // all notification receivers values. - TxHashValue util.Uint256 -} - -// TxHash returns hash of the TX with new epoch -// notification. -func (b BindCommon) TxHash() util.Uint256 { - return b.TxHashValue -} - -// MorphEvent implements Neo:Morph Event interface. -func (BindCommon) MorphEvent() {} - -func (b BindCommon) Keys() [][]byte { return b.KeysValue } - -func (b BindCommon) User() []byte { return b.UserValue } - -func ParseBind(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Bind - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - err = parseBind(&ev.BindCommon, params) - if err != nil { - return nil, err - } - - ev.TxHashValue = e.Container - - return ev, nil -} - -func parseBind(dst *BindCommon, params []stackitem.Item) error { - if ln := len(params); ln != 2 { - return event.WrongNumberOfParameters(2, ln) - } - - var err error - - // parse user - dst.UserValue, err = client.BytesFromStackItem(params[0]) - if err != nil { - return fmt.Errorf("could not get bind user: %w", err) - } - - // parse keys - bindKeys, err := client.ArrayFromStackItem(params[1]) - if err != nil { - return fmt.Errorf("could not get bind keys: %w", err) - } - - dst.KeysValue = make([][]byte, 0, len(bindKeys)) - - for i := range bindKeys { - rawKey, err := client.BytesFromStackItem(bindKeys[i]) - if err != nil { - return fmt.Errorf("could not get bind public key: %w", err) - } - - dst.KeysValue = append(dst.KeysValue, rawKey) - } - - return nil -} diff --git a/pkg/morph/event/frostfs/bind_test.go b/pkg/morph/event/frostfs/bind_test.go deleted file mode 100644 index 38a82b8d4..000000000 --- a/pkg/morph/event/frostfs/bind_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package frostfs - -import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" -) - -func TestParseBind(t *testing.T) { - var ( - user = []byte{0x1, 0x2, 0x3} - publicKeys = [][]byte{ - []byte("key1"), - []byte("key2"), - []byte("key3"), - } - ) - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - } - - _, err := ParseBind(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(2, len(prms)).Error()) - }) - - t.Run("wrong first parameter", func(t *testing.T) { - _, err := ParseBind(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong second parameter", func(t *testing.T) { - _, err := ParseBind(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(user), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct", func(t *testing.T) { - ev, err := ParseBind(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(user), - stackitem.NewArray([]stackitem.Item{ - stackitem.NewByteArray(publicKeys[0]), - stackitem.NewByteArray(publicKeys[1]), - stackitem.NewByteArray(publicKeys[2]), - }), - })) - require.NoError(t, err) - - e := ev.(Bind) - - require.Equal(t, user, e.User()) - require.Equal(t, publicKeys, e.Keys()) - }) -} - -func createNotifyEventFromItems(items []stackitem.Item) *state.ContainedNotificationEvent { - return &state.ContainedNotificationEvent{ - NotificationEvent: state.NotificationEvent{ - Item: stackitem.NewArray(items), - }, - } -} diff --git a/pkg/morph/event/frostfs/cheque_test.go b/pkg/morph/event/frostfs/cheque_test.go index be53592ca..ab177757f 100644 --- a/pkg/morph/event/frostfs/cheque_test.go +++ b/pkg/morph/event/frostfs/cheque_test.go @@ -5,6 +5,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" + "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/stretchr/testify/require" @@ -84,3 +85,11 @@ func TestParseCheque(t *testing.T) { }, ev) }) } + +func createNotifyEventFromItems(items []stackitem.Item) *state.ContainedNotificationEvent { + return &state.ContainedNotificationEvent{ + NotificationEvent: state.NotificationEvent{ + Item: stackitem.NewArray(items), + }, + } +} diff --git a/pkg/morph/event/frostfs/unbind.go b/pkg/morph/event/frostfs/unbind.go deleted file mode 100644 index 5a6a8dad9..000000000 --- a/pkg/morph/event/frostfs/unbind.go +++ /dev/null @@ -1,33 +0,0 @@ -package frostfs - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" -) - -type Unbind struct { - BindCommon -} - -func ParseUnbind(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Unbind - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - err = parseBind(&ev.BindCommon, params) - if err != nil { - return nil, err - } - - ev.TxHashValue = e.Container - - return ev, nil -} diff --git a/pkg/morph/event/frostfs/unbind_test.go b/pkg/morph/event/frostfs/unbind_test.go deleted file mode 100644 index 4b79d7c48..000000000 --- a/pkg/morph/event/frostfs/unbind_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package frostfs - -import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" -) - -func TestParseUnbind(t *testing.T) { - var ( - user = []byte{0x1, 0x2, 0x3} - publicKeys = [][]byte{ - []byte("key1"), - []byte("key2"), - []byte("key3"), - } - ) - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - } - - _, err := ParseUnbind(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(2, len(prms)).Error()) - }) - - t.Run("wrong first parameter", func(t *testing.T) { - _, err := ParseUnbind(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("wrong second parameter", func(t *testing.T) { - _, err := ParseUnbind(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(user), - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct", func(t *testing.T) { - ev, err := ParseUnbind(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(user), - stackitem.NewArray([]stackitem.Item{ - stackitem.NewByteArray(publicKeys[0]), - stackitem.NewByteArray(publicKeys[1]), - stackitem.NewByteArray(publicKeys[2]), - }), - })) - require.NoError(t, err) - - e := ev.(Unbind) - - require.Equal(t, user, e.User()) - require.Equal(t, publicKeys, e.Keys()) - }) -} From a3ef7b58b46e7cf8bb1e3e94ed8c127c888fc387 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 20 Nov 2023 17:03:19 +0300 Subject: [PATCH 0740/1943] [#755] innerring: Check container owner namespace Signed-off-by: Dmitrii Stepanov --- pkg/innerring/initialization.go | 16 ++- .../processors/container/handlers_test.go | 11 ++ .../processors/container/process_container.go | 25 +++- .../processors/container/processor.go | 38 +++--- pkg/morph/client/frostfsid/client.go | 31 +++++ pkg/morph/client/frostfsid/subject.go | 117 ++++++++++++++++++ 6 files changed, 218 insertions(+), 20 deletions(-) create mode 100644 pkg/morph/client/frostfsid/client.go create mode 100644 pkg/morph/client/frostfsid/subject.go diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 71571c072..f4d9b4169 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -20,6 +20,7 @@ import ( balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" frostfsClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" @@ -249,7 +250,7 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { return err } -func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.Client) error { +func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.Client, frostfsIDClient *frostfsid.Client) error { // container processor containerProcessor, err := cont.New(&cont.Params{ Log: s.log, @@ -258,6 +259,7 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C AlphabetState: s, ContainerClient: cnrClient, MorphClient: cnrClient.Morph(), + FrostFSIDClient: frostfsIDClient, NetworkState: s.netmapClient, }) if err != nil { @@ -364,8 +366,9 @@ func (s *Server) initGRPCServer(cfg *viper.Viper) error { } type serverMorphClients struct { - CnrClient *container.Client - FrostFSClient *frostfsClient.Client + CnrClient *container.Client + FrostFSIDClient *frostfsid.Client + FrostFSClient *frostfsClient.Client } func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { @@ -397,6 +400,11 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { return nil, err } + result.FrostFSIDClient, err = frostfsid.NewFromMorph(s.morphClient, s.contracts.frostfsID, fee) + if err != nil { + return nil, err + } + result.FrostFSClient, err = frostfsClient.NewFromMorph(s.mainnetClient, s.contracts.frostfs, s.feeConfig.MainChainFee(), frostfsClient.TryNotary(), frostfsClient.AsAlphabet()) if err != nil { @@ -426,7 +434,7 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien return err } - err = s.initContainerProcessor(cfg, morphClients.CnrClient) + err = s.initContainerProcessor(cfg, morphClients.CnrClient, morphClients.FrostFSIDClient) if err != nil { return err } diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index 3acfa2251..9fbfdf811 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -3,9 +3,11 @@ package container import ( "crypto/ecdsa" "encoding/hex" + "fmt" "testing" "time" + frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" @@ -42,6 +44,7 @@ func TestPutEvent(t *testing.T) { NetworkState: nst, ContainerClient: &testContainerClient{}, MorphClient: mc, + FrostFSIDClient: &testFrostFSIDClient{}, }) require.NoError(t, err, "failed to create processor") @@ -102,6 +105,7 @@ func TestDeleteEvent(t *testing.T) { NetworkState: nst, ContainerClient: cc, MorphClient: mc, + FrostFSIDClient: &testFrostFSIDClient{}, }) require.NoError(t, err, "failed to create processor") @@ -173,6 +177,7 @@ func TestSetEACLEvent(t *testing.T) { NetworkState: nst, ContainerClient: cc, MorphClient: mc, + FrostFSIDClient: &testFrostFSIDClient{}, }) require.NoError(t, err, "failed to create processor") @@ -316,3 +321,9 @@ func (c *testMorphClient) NotarySignAndInvokeTX(mainTx *transaction.Transaction) c.transactions = append(c.transactions, mainTx) return nil } + +type testFrostFSIDClient struct{} + +func (c *testFrostFSIDClient) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) { + return nil, fmt.Errorf("subject not found") +} diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 2629b9d29..2ad863b27 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -2,6 +2,7 @@ package container import ( "fmt" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -10,6 +11,7 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "github.com/nspcc-dev/neo-go/pkg/network/payload" + "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" ) @@ -88,7 +90,7 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { } // check native name and zone - err = checkNNS(ctx, cnr) + err = cp.checkNNS(ctx, cnr) if err != nil { return fmt.Errorf("NNS: %w", err) } @@ -157,7 +159,7 @@ func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { return nil } -func checkNNS(ctx *putContainerContext, cnr containerSDK.Container) error { +func (cp *Processor) checkNNS(ctx *putContainerContext, cnr containerSDK.Container) error { // fetch domain info ctx.d = containerSDK.ReadDomain(cnr) @@ -175,6 +177,25 @@ func checkNNS(ctx *putContainerContext, cnr containerSDK.Container) error { } } + namespace, hasNamespace := strings.CutSuffix(ctx.d.Zone(), ".ns") + if !hasNamespace { + return nil + } + + addr, err := util.Uint160DecodeBytesBE(cnr.Owner().WalletBytes()[1 : 1+util.Uint160Size]) + if err != nil { + return fmt.Errorf("could not get container owner address: %w", err) + } + + subject, err := cp.frostFSIDClient.GetSubject(addr) + if err != nil { + return fmt.Errorf("could not get subject from FrostfsID contract: %w", err) + } + + if subject.Namespace != namespace { + return fmt.Errorf("container and owner namespaces do not match") + } + return nil } diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index 9f0ae77bf..8fd9edfb8 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" @@ -32,15 +33,20 @@ type ( NotarySignAndInvokeTX(mainTx *transaction.Transaction) error } + FrostFSIDClient interface { + GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) + } + // Processor of events produced by container contract in the sidechain. Processor struct { - log *logger.Logger - metrics metrics.Register - pool *ants.Pool - alphabetState AlphabetState - cnrClient ContClient // notary must be enabled - morphClient MorphClient - netState NetworkState + log *logger.Logger + metrics metrics.Register + pool *ants.Pool + alphabetState AlphabetState + cnrClient ContClient // notary must be enabled + morphClient MorphClient + netState NetworkState + frostFSIDClient FrostFSIDClient } // Params of the processor constructor. @@ -52,6 +58,7 @@ type ( ContainerClient ContClient MorphClient MorphClient NetworkState NetworkState + FrostFSIDClient FrostFSIDClient } ) @@ -86,6 +93,8 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/container: Morph client is not set") case p.NetworkState == nil: return nil, errors.New("ir/container: network state is not set") + case p.FrostFSIDClient == nil: + return nil, errors.New("ir/container: FrostFSID client is not set") } p.Log.Debug(logs.ContainerContainerWorkerPool, zap.Int("size", p.PoolSize)) @@ -101,13 +110,14 @@ func New(p *Params) (*Processor, error) { } return &Processor{ - log: p.Log, - metrics: metricsRegister, - pool: pool, - alphabetState: p.AlphabetState, - cnrClient: p.ContainerClient, - netState: p.NetworkState, - morphClient: p.MorphClient, + log: p.Log, + metrics: metricsRegister, + pool: pool, + alphabetState: p.AlphabetState, + cnrClient: p.ContainerClient, + netState: p.NetworkState, + morphClient: p.MorphClient, + frostFSIDClient: p.FrostFSIDClient, }, nil } diff --git a/pkg/morph/client/frostfsid/client.go b/pkg/morph/client/frostfsid/client.go new file mode 100644 index 000000000..3efa522bc --- /dev/null +++ b/pkg/morph/client/frostfsid/client.go @@ -0,0 +1,31 @@ +package frostfsid + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" + "github.com/nspcc-dev/neo-go/pkg/util" +) + +// Client is a wrapper over StaticClient +// which makes calls with the names and arguments +// of the FrostFS ID contract. +// +// Working client must be created via constructor New. +// Using the Client that has been created with new(Client) +// expression (or just declaring a Client variable) is unsafe +// and can lead to panic. +type Client struct { + client *client.StaticClient // static FrostFS ID contract client +} + +// NewFromMorph wraps client to work with FrostFS ID contract. +func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8) (*Client, error) { + sc, err := client.NewStatic(cli, contract, fee, client.TryNotary(), client.AsAlphabet()) + if err != nil { + return nil, fmt.Errorf("could not create client of FrostFS ID contract: %w", err) + } + + return &Client{client: sc}, nil +} diff --git a/pkg/morph/client/frostfsid/subject.go b/pkg/morph/client/frostfsid/subject.go new file mode 100644 index 000000000..b14675d58 --- /dev/null +++ b/pkg/morph/client/frostfsid/subject.go @@ -0,0 +1,117 @@ +package frostfsid + +import ( + "errors" + "fmt" + + frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" +) + +const methodGetSubject = "getSubject" + +func (c *Client) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) { + prm := client.TestInvokePrm{} + prm.SetMethod(methodGetSubject) + prm.SetArgs(addr) + + res, err := c.client.TestInvoke(prm) + if err != nil { + return nil, fmt.Errorf("could not perform test invocation (%s): %w", methodGetSubject, err) + } + + subj, err := parseSubject(res) + if err != nil { + return nil, fmt.Errorf("could not parse test invocation result (%s): %w", methodGetSubject, err) + } + + return subj, nil +} + +// parseSubject from https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/commit/dd5919348da9731f24504e7bc485516c2ba5f11c/frostfsid/client/client.go#L592 +func parseSubject(structArr []stackitem.Item) (*frostfsidclient.Subject, error) { + if len(structArr) < 5 { + return nil, errors.New("invalid response subject struct") + } + + var ( + err error + subj frostfsidclient.Subject + ) + + subj.PrimaryKey, err = unwrap.PublicKey(makeValidRes(structArr[0])) + if err != nil { + return nil, err + } + + if !structArr[1].Equals(stackitem.Null{}) { + subj.AdditionalKeys, err = unwrap.ArrayOfPublicKeys(makeValidRes(structArr[1])) + if err != nil { + return nil, err + } + } + + if !structArr[2].Equals(stackitem.Null{}) { + subj.Namespace, err = unwrap.UTF8String(makeValidRes(structArr[2])) + if err != nil { + return nil, err + } + } + + if !structArr[3].Equals(stackitem.Null{}) { + subj.Name, err = unwrap.UTF8String(makeValidRes(structArr[3])) + if err != nil { + return nil, err + } + } + + subj.KV, err = parseMap(structArr[4]) + if err != nil { + return nil, err + } + + return &subj, nil +} + +func makeValidRes(item stackitem.Item) (*result.Invoke, error) { + return &result.Invoke{ + Stack: []stackitem.Item{item}, + State: vmstate.Halt.String(), + }, nil +} + +func parseMap(item stackitem.Item) (map[string]string, error) { + if item.Equals(stackitem.Null{}) { + return nil, nil + } + + metaMap, err := unwrap.Map(makeValidRes(item)) + if err != nil { + return nil, err + } + + meta, ok := metaMap.Value().([]stackitem.MapElement) + if !ok { + return nil, errors.New("invalid map type") + } + + res := make(map[string]string, len(meta)) + for _, element := range meta { + key, err := element.Key.TryBytes() + if err != nil { + return nil, err + } + val, err := element.Value.TryBytes() + if err != nil { + return nil, err + } + res[string(key)] = string(val) + } + + return res, nil +} From 681b2c5fd45f3c6763db2be3509575d3c3f967db Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Dec 2023 16:50:44 +0300 Subject: [PATCH 0741/1943] [#825] policer: Do not drop required linking objects Signed-off-by: Dmitrii Stepanov --- pkg/core/object/address.go | 11 +++++++-- pkg/local_object_storage/metabase/list.go | 27 +++++++++++++++++------ pkg/services/policer/check.go | 6 ++--- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/pkg/core/object/address.go b/pkg/core/object/address.go index 12e5c89ce..a25d853eb 100644 --- a/pkg/core/object/address.go +++ b/pkg/core/object/address.go @@ -1,6 +1,8 @@ package object import ( + "fmt" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -8,6 +10,11 @@ import ( // AddressWithType groups object address with its FrostFS // object type. type AddressWithType struct { - Address oid.Address - Type objectSDK.Type + Address oid.Address + Type objectSDK.Type + IsLinkingObject bool +} + +func (v AddressWithType) String() string { + return fmt.Sprintf("address: %s, type: %s, is linking: %t", v.Address, v.Type, v.IsLinkingObject) } diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 5099a161b..f7a314452 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -102,6 +102,7 @@ func (db *DB) ListWithCursor(ctx context.Context, prm ListPrm) (res ListRes, err func (db *DB) listWithCursor(tx *bbolt.Tx, result []objectcore.AddressWithType, count int, cursor *Cursor) ([]objectcore.AddressWithType, *Cursor, error) { threshold := cursor == nil // threshold is a flag to ignore cursor var bucketName []byte + var err error c := tx.Cursor() name, _ := c.First() @@ -140,8 +141,11 @@ loop: bkt := tx.Bucket(name) if bkt != nil { copy(rawAddr, cidRaw) - result, offset, cursor = selectNFromBucket(bkt, objType, graveyardBkt, garbageBkt, rawAddr, containerID, + result, offset, cursor, err = selectNFromBucket(bkt, objType, graveyardBkt, garbageBkt, rawAddr, containerID, result, count, cursor, threshold) + if err != nil { + return nil, nil, err + } } bucketName = name if len(result) >= count { @@ -183,23 +187,23 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket limit int, // stop listing at `limit` items in result cursor *Cursor, // start from cursor object threshold bool, // ignore cursor and start immediately -) ([]objectcore.AddressWithType, []byte, *Cursor) { +) ([]objectcore.AddressWithType, []byte, *Cursor, error) { if cursor == nil { cursor = new(Cursor) } count := len(to) c := bkt.Cursor() - k, _ := c.First() + k, v := c.First() offset := cursor.inBucketOffset if !threshold { c.Seek(offset) - k, _ = c.Next() // we are looking for objects _after_ the cursor + k, v = c.Next() // we are looking for objects _after_ the cursor } - for ; k != nil; k, _ = c.Next() { + for ; k != nil; k, v = c.Next() { if count >= limit { break } @@ -214,14 +218,23 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket continue } + var isLinkingObj bool + if objType == objectSDK.TypeRegular { + var o objectSDK.Object + if err := o.Unmarshal(v); err != nil { + return nil, nil, nil, err + } + isLinkingObj = isLinkObject(&o) + } + var a oid.Address a.SetContainer(cnt) a.SetObject(obj) - to = append(to, objectcore.AddressWithType{Address: a, Type: objType}) + to = append(to, objectcore.AddressWithType{Address: a, Type: objType, IsLinkingObject: isLinkingObj}) count++ } - return to, offset, cursor + return to, offset, cursor, nil } func parseContainerIDWithPrefix(containerID *cid.ID, name []byte) ([]byte, byte) { diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 9e40219d1..73f83b2a7 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -93,10 +93,10 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi // Number of copies that are stored on maintenance nodes. var uncheckedCopies int - if typ == objectSDK.TypeLock || typ == objectSDK.TypeTombstone { - // all nodes of a container must store the `LOCK` and `TOMBSTONE` objects + if typ == objectSDK.TypeLock || typ == objectSDK.TypeTombstone || addrWithType.IsLinkingObject { + // all nodes of a container must store the `LOCK`, `TOMBSTONE` and linking objects // for correct object removal protection: - // - `LOCK` and `TOMBSTONE` objects are broadcast on their PUT requests; + // - `LOCK`, `TOMBSTONE` and linking objects are broadcast on their PUT requests; // - `LOCK` object removal is a prohibited action in the GC. shortage = uint32(len(nodes)) } From 3b7c0362a8592d439b5b242eee5fbee583a9b6c4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 12 Dec 2023 14:18:42 +0300 Subject: [PATCH 0742/1943] [#861] shard: Fix Delete object It is possible that object doesn't exist in metabase. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + pkg/local_object_storage/shard/delete.go | 29 ++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index befa7342a..c996e830a 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -553,4 +553,5 @@ const ( BlobovniczaSavingCountersToMeta = "saving counters to blobovnicza's meta..." BlobovniczaSavingCountersToMetaSuccess = "saving counters to blobovnicza's meta completed successfully" BlobovniczaSavingCountersToMetaFailed = "saving counters to blobovnicza's meta failed" + ObjectRemovalFailureExistsInWritecache = "can't remove object: object must be flushed from writecache" ) diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 0856bda10..1c510bdb7 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -2,12 +2,14 @@ package shard import ( "context" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -61,6 +63,13 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm, skipFailed bool) (Del default: } + if err := s.validateWritecacheDoesntContainObject(ctx, addr); err != nil { + if skipFailed { + continue + } + return result, err + } + if err := s.deleteFromBlobstor(ctx, addr); err != nil { if skipFailed { continue @@ -80,6 +89,21 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm, skipFailed bool) (Del return result, nil } +func (s *Shard) validateWritecacheDoesntContainObject(ctx context.Context, addr oid.Address) error { + if !s.hasWriteCache() { + return nil + } + _, err := s.writeCache.Head(ctx, addr) + if err == nil { + s.log.Warn(logs.ObjectRemovalFailureExistsInWritecache, zap.Stringer("object_address", addr)) + return fmt.Errorf("object %s must be flushed from writecache", addr) + } + if client.IsErrObjectNotFound(err) { + return nil + } + return err +} + func (s *Shard) deleteFromBlobstor(ctx context.Context, addr oid.Address) error { var sPrm meta.StorageIDPrm sPrm.SetAddress(addr) @@ -99,13 +123,14 @@ func (s *Shard) deleteFromBlobstor(ctx context.Context, addr oid.Address) error delPrm.StorageID = storageID _, err = s.blobStor.Delete(ctx, delPrm) - if err != nil { + if err != nil && !client.IsErrObjectNotFound(err) { s.log.Debug(logs.ObjectRemovalFailureBlobStor, zap.Stringer("object_address", addr), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + return err } - return err + return nil } func (s *Shard) deleteFromMetabase(ctx context.Context, addr oid.Address) error { From d2746a7d679f5a30a6a3a4a18cbfca0d2f4bc3d8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Dec 2023 13:01:32 +0300 Subject: [PATCH 0743/1943] [#857] Makefile: Update linter version Signed-off-by: Dmitrii Stepanov --- Makefile | 4 ++-- cmd/frostfs-node/notificator.go | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3fcb5027c..153640da8 100755 --- a/Makefile +++ b/Makefile @@ -8,8 +8,8 @@ HUB_IMAGE ?= truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.21 -LINT_VERSION ?= 1.54.0 -TRUECLOUDLAB_LINT_VERSION ?= 0.0.2 +LINT_VERSION ?= 1.55.2 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.3 PROTOC_VERSION ?= 25.0 PROTOC_GEN_GO_VERSION ?= $(shell go list -f '{{.Version}}' -m google.golang.org/protobuf) PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2) diff --git a/cmd/frostfs-node/notificator.go b/cmd/frostfs-node/notificator.go index 8e581505f..f9bb31fed 100644 --- a/cmd/frostfs-node/notificator.go +++ b/cmd/frostfs-node/notificator.go @@ -167,7 +167,6 @@ func connectNats(ctx context.Context, c *cfg) { err := c.cfgNotifications.nw.w.Connect(ctx, endpoint) if err != nil { panic(fmt.Sprintf("could not connect to a nats endpoint %s: %v", endpoint, err)) - } else { - c.log.Info(logs.NatsConnectedToEndpoint, zap.String("endpoint", endpoint)) } + c.log.Info(logs.NatsConnectedToEndpoint, zap.String("endpoint", endpoint)) } From 5d7833c89b9ae73fa6b53c8d200d2e549a147bdb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Dec 2023 13:06:03 +0300 Subject: [PATCH 0744/1943] [#857] golangci: Add perfsprint linter Signed-off-by: Dmitrii Stepanov --- .golangci.yml | 1 + pkg/metrics/writecache.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 5c7e9d918..1404dd104 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -79,5 +79,6 @@ linters: - contextcheck - importas - truecloudlab-linters + - perfsprint disable-all: true fast: false diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index c8f815ca5..ea3d8742a 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -1,7 +1,7 @@ package metrics import ( - "fmt" + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -52,7 +52,7 @@ func (m *writeCacheMetrics) AddMethodDuration(shardID string, method string, suc m.methodDuration.With( prometheus.Labels{ shardIDLabel: shardID, - successLabel: fmt.Sprintf("%v", success), + successLabel: strconv.FormatBool(success), storageLabel: storageType, methodLabel: method, }, From 94ffe8bb452a6b8e83f8bb27103891d5307fd68e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Dec 2023 13:09:57 +0300 Subject: [PATCH 0745/1943] [#857] golangci: Add testifylint linter Signed-off-by: Dmitrii Stepanov --- .golangci.yml | 1 + .../blobstor/internal/blobstortest/iterate.go | 2 +- pkg/local_object_storage/writecache/writecachetest/flush.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 1404dd104..4dd82d92f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -80,5 +80,6 @@ linters: - importas - truecloudlab-linters - perfsprint + - testifylint disable-all: true fast: false diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index c73a663b6..985ca0173 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -82,7 +82,7 @@ func runTestIgnoreLogicalErrors(t *testing.T, s common.Storage, objects []object _, err := s.Iterate(context.Background(), iterPrm) require.Equal(t, err, logicErr) - require.Equal(t, len(objects)/2, len(seen)) + require.Len(t, seen, len(objects)/2) for i := range objects { d, ok := seen[objects[i].addr.String()] if ok { diff --git a/pkg/local_object_storage/writecache/writecachetest/flush.go b/pkg/local_object_storage/writecache/writecachetest/flush.go index 6911344a2..2c495df52 100644 --- a/pkg/local_object_storage/writecache/writecachetest/flush.go +++ b/pkg/local_object_storage/writecache/writecachetest/flush.go @@ -91,7 +91,7 @@ func TestFlush[Option any]( require.Equal(t, uint32(0), errCount.Load()) require.Error(t, wc.Flush(context.Background(), false)) - require.True(t, errCount.Load() > 0) + require.Greater(t, errCount.Load(), uint32(0)) require.NoError(t, wc.Flush(context.Background(), true)) check(t, mb, bs, objects) From 11add38e87291cd34cc14e6748fa7900ca2deff6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Dec 2023 13:18:34 +0300 Subject: [PATCH 0746/1943] [#857] golangci: Add protogetter linter Signed-off-by: Dmitrii Stepanov --- .golangci.yml | 1 + cmd/frostfs-cli/modules/control/evacuation.go | 6 +++--- cmd/frostfs-cli/modules/control/remove_rule.go | 2 +- cmd/frostfs-cli/modules/control/shards_list.go | 6 +++--- cmd/frostfs-cli/modules/tree/add.go | 2 +- pkg/services/control/ir/server/calls.go | 6 +++--- pkg/services/control/server/doctor.go | 4 ++-- pkg/services/control/server/set_shard_mode.go | 4 ++-- pkg/services/tree/service.go | 10 ++++++---- pkg/services/tree/sync.go | 6 +++--- 10 files changed, 25 insertions(+), 22 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 4dd82d92f..d209693aa 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -81,5 +81,6 @@ linters: - truecloudlab-linters - perfsprint - testifylint + - protogetter disable-all: true fast: false diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 735897b81..821dd1a6c 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -220,7 +220,7 @@ func appendEstimation(sb *strings.Builder, resp *control.GetShardEvacuationStatu if resp.GetBody().GetStatus() != control.GetShardEvacuationStatusResponse_Body_RUNNING || resp.GetBody().GetDuration() == nil || resp.GetBody().GetTotal() == 0 || - resp.GetBody().GetEvacuated()+resp.GetBody().GetFailed()+resp.Body.GetSkipped() == 0 { + resp.GetBody().GetEvacuated()+resp.GetBody().GetFailed()+resp.GetBody().GetSkipped() == 0 { return } @@ -252,8 +252,8 @@ func appendStartedAt(sb *strings.Builder, resp *control.GetShardEvacuationStatus } func appendError(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { - if len(resp.Body.GetErrorMessage()) > 0 { - sb.WriteString(fmt.Sprintf(" Error: %s.", resp.Body.GetErrorMessage())) + if len(resp.GetBody().GetErrorMessage()) > 0 { + sb.WriteString(fmt.Sprintf(" Error: %s.", resp.GetBody().GetErrorMessage())) } } diff --git a/cmd/frostfs-cli/modules/control/remove_rule.go b/cmd/frostfs-cli/modules/control/remove_rule.go index c3f4aeb0c..60f96afd5 100644 --- a/cmd/frostfs-cli/modules/control/remove_rule.go +++ b/cmd/frostfs-cli/modules/control/remove_rule.go @@ -63,7 +63,7 @@ func removeRule(cmd *cobra.Command, _ []string) { verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - if resp.GetBody().Removed { + if resp.GetBody().GetRemoved() { cmd.Println("Rule has been removed.") } else { cmd.Println("Rule has not been removed.") diff --git a/cmd/frostfs-cli/modules/control/shards_list.go b/cmd/frostfs-cli/modules/control/shards_list.go index 4d09667b8..0b5d6b0a3 100644 --- a/cmd/frostfs-cli/modules/control/shards_list.go +++ b/cmd/frostfs-cli/modules/control/shards_list.go @@ -65,7 +65,7 @@ func prettyPrintShardsJSON(cmd *cobra.Command, ii []*control.ShardInfo) { out := make([]map[string]any, 0, len(ii)) for _, i := range ii { out = append(out, map[string]any{ - "shard_id": base58.Encode(i.Shard_ID), + "shard_id": base58.Encode(i.GetShard_ID()), "mode": shardModeToString(i.GetMode()), "metabase": i.GetMetabasePath(), "blobstor": i.GetBlobstor(), @@ -105,7 +105,7 @@ func prettyPrintShards(cmd *cobra.Command, ii []*control.ShardInfo) { pathPrinter("Write-cache", i.GetWritecachePath())+ pathPrinter("Pilorama", i.GetPiloramaPath())+ fmt.Sprintf("Error count: %d\n", i.GetErrorCount()), - base58.Encode(i.Shard_ID), + base58.Encode(i.GetShard_ID()), shardModeToString(i.GetMode()), ) } @@ -122,6 +122,6 @@ func shardModeToString(m control.ShardMode) string { func sortShardsByID(ii []*control.ShardInfo) { sort.Slice(ii, func(i, j int) bool { - return bytes.Compare(ii[i].Shard_ID, ii[j].Shard_ID) < 0 + return bytes.Compare(ii[i].GetShard_ID(), ii[j].GetShard_ID()) < 0 }) } diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index 4ac1ed198..8ceea4173 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -74,7 +74,7 @@ func add(cmd *cobra.Command, _ []string) { resp, err := cli.Add(ctx, req) commonCmd.ExitOnErr(cmd, "failed to cal add: %w", err) - cmd.Println("Node ID: ", resp.Body.NodeId) + cmd.Println("Node ID: ", resp.GetBody().GetNodeId()) } func parseMeta(cmd *cobra.Command) ([]*tree.KeyValue, error) { diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 7baae0329..58610b0d1 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -114,13 +114,13 @@ func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainer return nil, status.Error(codes.PermissionDenied, err.Error()) } - if len(req.Body.GetContainerId()) > 0 && len(req.Body.GetOwner()) > 0 { + if len(req.GetBody().GetContainerId()) > 0 && len(req.GetBody().GetOwner()) > 0 { return nil, status.Error(codes.InvalidArgument, "specify the owner and container at the same time is not allowed") } var vub uint32 - if len(req.Body.GetContainerId()) > 0 { + if len(req.GetBody().GetContainerId()) > 0 { var containerID cid.ID - if err := containerID.Decode(req.Body.GetContainerId()); err != nil { + if err := containerID.Decode(req.GetBody().GetContainerId()); err != nil { return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed to parse container ID: %s", err.Error())) } var err error diff --git a/pkg/services/control/server/doctor.go b/pkg/services/control/server/doctor.go index 2c91d4c2b..707a6c983 100644 --- a/pkg/services/control/server/doctor.go +++ b/pkg/services/control/server/doctor.go @@ -15,12 +15,12 @@ func (s *Server) Doctor(ctx context.Context, req *control.DoctorRequest) (*contr return nil, status.Error(codes.PermissionDenied, err.Error()) } - if !req.Body.RemoveDuplicates { + if !req.GetBody().GetRemoveDuplicates() { return nil, status.Error(codes.InvalidArgument, "operation not specified") } var prm engine.RemoveDuplicatesPrm - prm.Concurrency = int(req.Body.Concurrency) + prm.Concurrency = int(req.GetBody().GetConcurrency()) err = s.s.RemoveDuplicates(ctx, prm) if err != nil { diff --git a/pkg/services/control/server/set_shard_mode.go b/pkg/services/control/server/set_shard_mode.go index 274e2f65d..18a7eb327 100644 --- a/pkg/services/control/server/set_shard_mode.go +++ b/pkg/services/control/server/set_shard_mode.go @@ -36,8 +36,8 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques return nil, status.Error(codes.Internal, fmt.Sprintf("unknown shard mode: %s", requestedMode)) } - for _, shardID := range s.getShardIDList(req.Body.GetShard_ID()) { - err = s.s.SetShardMode(shardID, m, req.Body.GetResetErrorCounter()) + for _, shardID := range s.getShardIDList(req.GetBody().GetShard_ID()) { + err = s.s.SetShardMode(shardID, m, req.GetBody().GetResetErrorCounter()) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 57767f87e..59b01796e 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -369,13 +369,14 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) x.ParentId = parent x.NodeId = node x.Timestamp = m.Time - if b.AllAttributes { + if b.GetAllAttributes() { x.Meta = metaToProto(m.Items) } else { + var metaValue []*KeyValue for _, kv := range m.Items { for _, attr := range b.GetAttributes() { if kv.Key == attr { - x.Meta = append(x.Meta, &KeyValue{ + metaValue = append(metaValue, &KeyValue{ Key: kv.Key, Value: kv.Value, }) @@ -383,6 +384,7 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) } } } + x.Meta = metaValue } info = append(info, &x) } @@ -670,8 +672,8 @@ func protoToMeta(arr []*KeyValue) []pilorama.KeyValue { meta := make([]pilorama.KeyValue, len(arr)) for i, kv := range arr { if kv != nil { - meta[i].Key = kv.Key - meta[i].Value = kv.Value + meta[i].Key = kv.GetKey() + meta[i].Value = kv.GetValue() } } return meta diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 485278adf..2a19ae18a 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -243,10 +243,10 @@ func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, for ; err == nil; res, err = c.Recv() { lm := res.GetBody().GetOperation() m := &pilorama.Move{ - Parent: lm.ParentId, - Child: lm.ChildId, + Parent: lm.GetParentId(), + Child: lm.GetChildId(), } - if err := m.Meta.FromBytes(lm.Meta); err != nil { + if err := m.Meta.FromBytes(lm.GetMeta()); err != nil { return err } opsCh <- m From 7eb46404a150cff97b45afe5181163bc9605fc29 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 13 Dec 2023 13:34:29 +0300 Subject: [PATCH 0747/1943] [#863] blobovnicza: Fix counters Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobovnicza/meta.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/meta.go b/pkg/local_object_storage/blobovnicza/meta.go index 8b913f8a7..3316d4666 100644 --- a/pkg/local_object_storage/blobovnicza/meta.go +++ b/pkg/local_object_storage/blobovnicza/meta.go @@ -92,12 +92,13 @@ func updateMeta(tx *bbolt.Tx, updateValues func(count, size uint64) (uint64, uin count, size = updateValues(count, size) - buf := make([]byte, dataSizeAndItemsCountBufLength) - binary.LittleEndian.PutUint64(buf, size) - if err := b.Put(dataSizeKey, buf); err != nil { + sizeBuf := make([]byte, dataSizeAndItemsCountBufLength) + binary.LittleEndian.PutUint64(sizeBuf, size) + if err := b.Put(dataSizeKey, sizeBuf); err != nil { return err } - binary.LittleEndian.PutUint64(buf, count) - return b.Put(itemsCountKey, buf) + countBuf := make([]byte, dataSizeAndItemsCountBufLength) + binary.LittleEndian.PutUint64(countBuf, count) + return b.Put(itemsCountKey, countBuf) } From 05f8f4928991dd90875942168682785ac223d281 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 12 Dec 2023 15:36:27 +0300 Subject: [PATCH 0748/1943] [#552] gofumpt changes Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-node/policy_engine.go | 3 ++- pkg/ape/chainbase/boltdb.go | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-node/policy_engine.go b/cmd/frostfs-node/policy_engine.go index d04322c2e..74cbf841c 100644 --- a/cmd/frostfs-node/policy_engine.go +++ b/cmd/frostfs-node/policy_engine.go @@ -23,7 +23,8 @@ var _ engine.LocalOverrideEngine = (*accessPolicyEngine)(nil) func newAccessPolicyEngine( morphChainStorage engine.MorphRuleChainStorage, - localOverrideDatabase chainbase.LocalOverrideDatabase) *accessPolicyEngine { + localOverrideDatabase chainbase.LocalOverrideDatabase, +) *accessPolicyEngine { return &accessPolicyEngine{ chainRouter: engine.NewDefaultChainRouterWithLocalOverrides( morphChainStorage, diff --git a/pkg/ape/chainbase/boltdb.go b/pkg/ape/chainbase/boltdb.go index c6cd2c014..e54d01ea0 100644 --- a/pkg/ape/chainbase/boltdb.go +++ b/pkg/ape/chainbase/boltdb.go @@ -22,9 +22,7 @@ type boltLocalOverrideStorage struct { db *bbolt.DB } -var ( - chainBucket = []byte{0} -) +var chainBucket = []byte{0} var ( ErrChainBucketNotFound = logicerr.New("chain root bucket has not been found") From 9b2dce5763de16e2374428884ec6cf8072256337 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Thu, 16 Nov 2023 02:48:05 +0300 Subject: [PATCH 0749/1943] [#552] Add status notification to systemd Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-node/control.go | 17 +++++++++++++++++ debian/frostfs-ir.service | 3 ++- debian/frostfs-storage.service | 3 ++- go.mod | 1 + go.sum | 12 ++++++++++++ internal/logs/logs.go | 1 + 6 files changed, 35 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index 4f5c5d780..fcd84db3c 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "net" controlconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/control" @@ -10,6 +11,7 @@ import ( controlSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -83,12 +85,14 @@ func (c *cfg) NetmapStatus() control.NetmapStatus { } func (c *cfg) setHealthStatus(st control.HealthStatus) { + c.notifySystemd(st) c.healthStatus.Store(int32(st)) c.metricsCollector.State().SetHealth(int32(st)) } func (c *cfg) compareAndSwapHealthStatus(oldSt, newSt control.HealthStatus) (swapped bool) { if swapped = c.healthStatus.CompareAndSwap(int32(oldSt), int32(newSt)); swapped { + c.notifySystemd(newSt) c.metricsCollector.State().SetHealth(int32(newSt)) } return @@ -96,6 +100,7 @@ func (c *cfg) compareAndSwapHealthStatus(oldSt, newSt control.HealthStatus) (swa func (c *cfg) swapHealthStatus(st control.HealthStatus) (old control.HealthStatus) { old = control.HealthStatus(c.healthStatus.Swap(int32(st))) + c.notifySystemd(st) c.metricsCollector.State().SetHealth(int32(st)) return } @@ -103,3 +108,15 @@ func (c *cfg) swapHealthStatus(st control.HealthStatus) (old control.HealthStatu func (c *cfg) HealthStatus() control.HealthStatus { return control.HealthStatus(c.healthStatus.Load()) } + +func (c *cfg) notifySystemd(st control.HealthStatus) string { + status := fmt.Sprintf("%v, %v", st.Number(), st) + err := sysdnotify.Status(status) + if err == nil { + c.log.Info(fmt.Sprintf("reported STATUS=\"%v\" to systemd", status)) + } + if err != nil { + c.log.Error(logs.FailedToReportStatusToSystemd, zap.String("error", err.Error())) + } + return fmt.Sprintf("%v, %v", st, control.HealthStatus_name[int32(st)]) +} diff --git a/debian/frostfs-ir.service b/debian/frostfs-ir.service index b59295ac5..304017f68 100644 --- a/debian/frostfs-ir.service +++ b/debian/frostfs-ir.service @@ -3,7 +3,8 @@ Description=FrostFS InnerRing node Requires=network.target [Service] -Type=simple +Type=notify +NotifyAccess=all ExecStart=/usr/bin/frostfs-ir --config /etc/frostfs/ir/config.yml User=frostfs-ir Group=frostfs-ir diff --git a/debian/frostfs-storage.service b/debian/frostfs-storage.service index 6e7d1dd5d..573961756 100644 --- a/debian/frostfs-storage.service +++ b/debian/frostfs-storage.service @@ -3,7 +3,8 @@ Description=FrostFS Storage node Requires=network.target [Service] -Type=simple +Type=notify +NotifyAccess=all ExecStart=/usr/bin/frostfs-node --config /etc/frostfs/storage/config.yml User=frostfs-storage Group=frostfs-storage diff --git a/go.mod b/go.mod index 3e79f7ff7..3037a6167 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/uuid v1.4.0 github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/iguanesolutions/go-systemd/v5 v5.1.1 github.com/klauspost/compress v1.17.4 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 diff --git a/go.sum b/go.sum index 15ae22422..780bdbcdb 100644 --- a/go.sum +++ b/go.sum @@ -146,6 +146,18 @@ github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/iguanesolutions/go-systemd/v5 v5.1.1 h1:Hs0Z16knPGCBFnKECrICPh+RQ89Sgy0xyzcalrHMKdw= +github.com/iguanesolutions/go-systemd/v5 v5.1.1/go.mod h1:Quv57scs6S7T0rC6qyLfW20KU/P4p9hrbLPF+ILYrXY= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= diff --git a/internal/logs/logs.go b/internal/logs/logs.go index c996e830a..1775b18cd 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -554,4 +554,5 @@ const ( BlobovniczaSavingCountersToMetaSuccess = "saving counters to blobovnicza's meta completed successfully" BlobovniczaSavingCountersToMetaFailed = "saving counters to blobovnicza's meta failed" ObjectRemovalFailureExistsInWritecache = "can't remove object: object must be flushed from writecache" + FailedToReportStatusToSystemd = "failed to report status to systemd" ) From eca7ac9f0d4096e86131ff0f068862320af5eb82 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 21 Nov 2023 12:08:49 +0300 Subject: [PATCH 0750/1943] [#552] Add sdnotify package To avoid using third-party dependencies. Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-node/control.go | 7 ++--- go.mod | 1 - go.sum | 12 ------- pkg/util/sdnotify/sdnotify.go | 59 +++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 pkg/util/sdnotify/sdnotify.go diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index fcd84db3c..6b603d433 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -10,8 +10,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" controlSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sdnotify" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -109,14 +109,13 @@ func (c *cfg) HealthStatus() control.HealthStatus { return control.HealthStatus(c.healthStatus.Load()) } -func (c *cfg) notifySystemd(st control.HealthStatus) string { +func (c *cfg) notifySystemd(st control.HealthStatus) { status := fmt.Sprintf("%v, %v", st.Number(), st) - err := sysdnotify.Status(status) + err := sdnotify.Status(status) if err == nil { c.log.Info(fmt.Sprintf("reported STATUS=\"%v\" to systemd", status)) } if err != nil { c.log.Error(logs.FailedToReportStatusToSystemd, zap.String("error", err.Error())) } - return fmt.Sprintf("%v, %v", st, control.HealthStatus_name[int32(st)]) } diff --git a/go.mod b/go.mod index 3037a6167..3e79f7ff7 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/uuid v1.4.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/iguanesolutions/go-systemd/v5 v5.1.1 github.com/klauspost/compress v1.17.4 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 diff --git a/go.sum b/go.sum index 780bdbcdb..15ae22422 100644 --- a/go.sum +++ b/go.sum @@ -146,18 +146,6 @@ github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= -github.com/iguanesolutions/go-systemd/v5 v5.1.1 h1:Hs0Z16knPGCBFnKECrICPh+RQ89Sgy0xyzcalrHMKdw= -github.com/iguanesolutions/go-systemd/v5 v5.1.1/go.mod h1:Quv57scs6S7T0rC6qyLfW20KU/P4p9hrbLPF+ILYrXY= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= diff --git a/pkg/util/sdnotify/sdnotify.go b/pkg/util/sdnotify/sdnotify.go new file mode 100644 index 000000000..7c3d2f1d2 --- /dev/null +++ b/pkg/util/sdnotify/sdnotify.go @@ -0,0 +1,59 @@ +package sdnotify + +import ( + "fmt" + "net" + "os" + "strings" +) + +const ( + ReadyEnabled = "READY=1" + StoppingEnabled = "STOPPING=1" + ReloadingEnabled = "RELOADING=1" +) + +var socket *net.UnixAddr + +// Initializes socket with provided name of +// environment variable. +func InitSocket() error { + notifySocket := os.Getenv("NOTIFY_SOCKET") + if notifySocket == "" { + return fmt.Errorf("\"NOTIFY_SOCKET\" environment variable is not present") + } + socket = &net.UnixAddr{ + Name: notifySocket, + Net: "unixgram", + } + return nil +} + +// FlagAndStatus sends systemd a combination of a +// well-known status and STATUS=%s{status}, separated by newline. +func FlagAndStatus(status string) error { + status += "\nSTATUS=" + strings.TrimSuffix(status, "=1") + return Send(status) +} + +// Status sends systemd notify STATUS=%s{status}. +func Status(status string) error { + return Send(fmt.Sprintf("STATUS=%s", status)) +} + +// Send state through the notify socket if any. +// If the notify socket was not detected, it returns an error. +func Send(state string) error { + if socket == nil { + return fmt.Errorf("socket is not initialized") + } + conn, err := net.DialUnix(socket.Net, nil, socket) + if err != nil { + return fmt.Errorf("can't open unix socket: %v", err) + } + defer conn.Close() + if _, err = conn.Write([]byte(state)); err != nil { + return fmt.Errorf("can't write into the unix socket: %v", err) + } + return nil +} From ef07c1a3c96e0a1c39c00eee20b7bbeb7855dcdb Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 21 Nov 2023 15:07:37 +0300 Subject: [PATCH 0751/1943] [#552] Add sysd notifications to storage service Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-node/config.go | 12 ++++++++++++ cmd/frostfs-node/control.go | 19 ++++++++++++++----- config/example/node.yaml | 3 +++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 82b01b563..27c028bae 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -62,6 +62,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sdnotify" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state" "git.frostfs.info/TrueCloudLab/frostfs-observability/logging/lokicore" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -361,6 +362,8 @@ type internals struct { healthStatus *atomic.Int32 // is node under maintenance isMaintenance atomic.Bool + + sdNotify bool } // starts node's maintenance. @@ -632,9 +635,18 @@ func initInternals(appCfg *config.Config, log *logger.Logger) internals { log: log, apiVersion: version.Current(), healthStatus: &healthStatus, + sdNotify: initSdNotify(appCfg), } } +func initSdNotify(appCfg *config.Config) bool { + if config.BoolSafe(appCfg.Sub("systemdnotify"), "enabled") { + fatalOnErr(sdnotify.InitSocket()) + return true + } + return false +} + func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkState, relayOnly bool) shared { var netAddr network.AddressGroup diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index 6b603d433..ee2a2952a 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -110,12 +110,21 @@ func (c *cfg) HealthStatus() control.HealthStatus { } func (c *cfg) notifySystemd(st control.HealthStatus) { - status := fmt.Sprintf("%v, %v", st.Number(), st) - err := sdnotify.Status(status) - if err == nil { - c.log.Info(fmt.Sprintf("reported STATUS=\"%v\" to systemd", status)) + if !c.sdNotify { + return + } + var err error + switch st { + case control.HealthStatus_READY: + err = sdnotify.FlagAndStatus(sdnotify.ReadyEnabled) + case control.HealthStatus_SHUTTING_DOWN: + err = sdnotify.FlagAndStatus(sdnotify.StoppingEnabled) + case control.HealthStatus_RECONFIGURING: + err = sdnotify.FlagAndStatus(sdnotify.ReloadingEnabled) + default: + err = sdnotify.Status(fmt.Sprintf("%v", st)) } if err != nil { - c.log.Error(logs.FailedToReportStatusToSystemd, zap.String("error", err.Error())) + c.log.Error(logs.FailedToReportStatusToSystemd, zap.Error(err)) } } diff --git a/config/example/node.yaml b/config/example/node.yaml index 1355113a3..c6a2be9ac 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -1,6 +1,9 @@ logger: level: debug # logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" +systemdnotify: + enabled: true + pprof: enabled: true address: localhost:6060 # endpoint for Node profiling From 2d4c0a0f4a727694e9b106521e1965e6e11e9ec7 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 5 Dec 2023 17:05:34 +0300 Subject: [PATCH 0752/1943] [#552] Add systemd notifications to ir service Signed-off-by: Ekaterina Lebedeva --- config/example/ir.yaml | 3 +++ pkg/innerring/innerring.go | 14 ++++++++++++++ pkg/innerring/state.go | 20 ++++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/config/example/ir.yaml b/config/example/ir.yaml index a01f3d0bb..401328e72 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -119,3 +119,6 @@ prometheus: enabled: true address: localhost:9090 # Endpoint for application prometheus metrics; disabled by default shutdown_timeout: 30s # Timeout for metrics HTTP server graceful shutdown + +systemdnotify: + enabled: true diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 49b770811..5d7dc5a52 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -24,6 +24,7 @@ import ( control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/precision" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sdnotify" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -73,6 +74,7 @@ type ( predefinedValidators keys.PublicKeys initialEpochTickDelta uint32 withoutMainNet bool + sdNotify bool // runtime processors netmapProcessor *netmap.Processor @@ -336,6 +338,11 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan irMetrics: metrics, } + server.sdNotify, err = server.initSdNotify(cfg) + if err != nil { + return nil, err + } + server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED) // parse notary support @@ -404,6 +411,13 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return server, nil } +func (s *Server) initSdNotify(cfg *viper.Viper) (bool, error) { + if cfg.GetBool("systemdnotify.enabled") { + return true, sdnotify.InitSocket() + } + return false, nil +} + func createListener(ctx context.Context, cli *client.Client, p *chainParams) (event.Listener, error) { var ( sub subscriber.Subscriber diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index f5a03fa29..1616dbb9f 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sdnotify" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/viper" @@ -154,6 +155,7 @@ func (s *Server) ResetEpochTimer(h uint32) error { func (s *Server) setHealthStatus(hs control.HealthStatus) { s.healthStatus.Store(int32(hs)) + s.notifySystemd(hs) if s.irMetrics != nil { s.irMetrics.SetHealth(int32(hs)) } @@ -173,3 +175,21 @@ func initPersistentStateStorage(cfg *viper.Viper) (*state.PersistentStorage, err return persistStorage, nil } + +func (s *Server) notifySystemd(st control.HealthStatus) { + if !s.sdNotify { + return + } + var err error + switch st { + case control.HealthStatus_READY: + err = sdnotify.FlagAndStatus(sdnotify.ReadyEnabled) + case control.HealthStatus_SHUTTING_DOWN: + err = sdnotify.FlagAndStatus(sdnotify.StoppingEnabled) + default: + err = sdnotify.Status(fmt.Sprintf("%v", st)) + } + if err != nil { + s.log.Error(logs.FailedToReportStatusToSystemd, zap.Error(err)) + } +} From 4a64b077030fa8eea4484a9d48aee5000ac80ab1 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 13 Dec 2023 15:25:47 +0300 Subject: [PATCH 0753/1943] [#869] cli: Pass only CID in requests for control API * Fix add-rule, list-rules, remove-rule, get-rule commands: do not convert container ID to native-schema resource format and pass it to control API. Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/control/add_rule.go | 6 +----- cmd/frostfs-cli/modules/control/get_rule.go | 6 +----- cmd/frostfs-cli/modules/control/list_rules.go | 6 +----- cmd/frostfs-cli/modules/control/remove_rule.go | 6 +----- 4 files changed, 4 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index ca9220009..48101c570 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -4,7 +4,6 @@ import ( "bytes" "crypto/sha256" "encoding/json" - "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -14,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" - nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/spf13/cobra" ) @@ -62,13 +60,11 @@ func addRule(cmd *cobra.Command, _ []string) { cmd.Println("Container ID: " + cidStr) cmd.Println("Parsed chain:\n" + prettyJSONFormat(cmd, serializedChain)) - name := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cidStr) - req := &control.AddChainLocalOverrideRequest{ Body: &control.AddChainLocalOverrideRequest_Body{ Target: &control.ChainTarget{ Type: control.ChainTarget_CONTAINER, - Name: name, + Name: cidStr, }, Chain: serializedChain, }, diff --git a/cmd/frostfs-cli/modules/control/get_rule.go b/cmd/frostfs-cli/modules/control/get_rule.go index 962ef179b..6d86d5d74 100644 --- a/cmd/frostfs-cli/modules/control/get_rule.go +++ b/cmd/frostfs-cli/modules/control/get_rule.go @@ -2,7 +2,6 @@ package control import ( "crypto/sha256" - "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -11,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" - nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/spf13/cobra" ) @@ -34,12 +32,10 @@ func getRule(cmd *cobra.Command, _ []string) { chainID, _ := cmd.Flags().GetString(chainIDFlag) - name := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cidStr) - req := &control.GetChainLocalOverrideRequest{ Body: &control.GetChainLocalOverrideRequest_Body{ Target: &control.ChainTarget{ - Name: name, + Name: cidStr, Type: control.ChainTarget_CONTAINER, }, ChainId: chainID, diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index c1e47f483..f087f5367 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -2,7 +2,6 @@ package control import ( "crypto/sha256" - "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -11,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" - nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/spf13/cobra" ) @@ -32,12 +30,10 @@ func listRules(cmd *cobra.Command, _ []string) { rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) - name := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cidStr) - req := &control.ListChainLocalOverridesRequest{ Body: &control.ListChainLocalOverridesRequest_Body{ Target: &control.ChainTarget{ - Name: name, + Name: cidStr, Type: control.ChainTarget_CONTAINER, }, }, diff --git a/cmd/frostfs-cli/modules/control/remove_rule.go b/cmd/frostfs-cli/modules/control/remove_rule.go index 60f96afd5..929d0b1a1 100644 --- a/cmd/frostfs-cli/modules/control/remove_rule.go +++ b/cmd/frostfs-cli/modules/control/remove_rule.go @@ -2,7 +2,6 @@ package control import ( "crypto/sha256" - "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -10,7 +9,6 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/spf13/cobra" ) @@ -37,12 +35,10 @@ func removeRule(cmd *cobra.Command, _ []string) { chainID, _ := cmd.Flags().GetString(chainIDFlag) - name := fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cidStr) - req := &control.RemoveChainLocalOverrideRequest{ Body: &control.RemoveChainLocalOverrideRequest_Body{ Target: &control.ChainTarget{ - Name: name, + Name: cidStr, Type: control.ChainTarget_CONTAINER, }, ChainId: chainID, From bdd43f621111ef85cf07e09f830f67e9dbb86b24 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 13 Dec 2023 15:29:17 +0300 Subject: [PATCH 0754/1943] [#869] object: Pass just CID to chain router * Do not convert CID from request to native-schema resource format - this step is unneccessary for APE. Signed-off-by: Airat Arifullin --- pkg/services/object/acl/ape.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/acl/ape.go b/pkg/services/object/acl/ape.go index e2a884c96..a431ad252 100644 --- a/pkg/services/object/acl/ape.go +++ b/pkg/services/object/acl/ape.go @@ -26,7 +26,7 @@ func (c *apeCheckerImpl) CheckIfRequestPermitted(reqInfo v2.RequestInfo) error { request := new(Request) request.FromRequestInfo(reqInfo) - cnrTarget := getResource(reqInfo).Name() + cnrTarget := reqInfo.ContainerID().EncodeToString() status, ruleFound, err := c.chainRouter.IsAllowed(apechain.Ingress, policyengine.NewRequestTargetWithContainer(cnrTarget), request) if err != nil { From f4877e7b42d84364de1d03f07d9ef199c615af15 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 30 Nov 2023 20:51:23 +0300 Subject: [PATCH 0755/1943] [#835] grpc: Try to reconnect if endpoint listen failed Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/accounting.go | 8 +- cmd/frostfs-node/config.go | 78 +++++++++-- cmd/frostfs-node/config/grpc/config.go | 15 ++ cmd/frostfs-node/container.go | 8 +- cmd/frostfs-node/grpc.go | 182 +++++++++++++++++-------- cmd/frostfs-node/netmap.go | 8 +- cmd/frostfs-node/object.go | 8 +- cmd/frostfs-node/session.go | 8 +- cmd/frostfs-node/tree.go | 8 +- internal/logs/logs.go | 5 + 10 files changed, 243 insertions(+), 85 deletions(-) diff --git a/cmd/frostfs-node/accounting.go b/cmd/frostfs-node/accounting.go index d04f34ff1..ec737f8a0 100644 --- a/cmd/frostfs-node/accounting.go +++ b/cmd/frostfs-node/accounting.go @@ -2,12 +2,14 @@ package main import ( "context" + "net" accountingGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" accountingTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/accounting/grpc" accountingService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/accounting" accounting "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/accounting/morph" + "google.golang.org/grpc" ) func initAccountingService(ctx context.Context, c *cfg) { @@ -28,7 +30,7 @@ func initAccountingService(ctx context.Context, c *cfg) { ), ) - for _, srv := range c.cfgGRPC.servers { - accountingGRPC.RegisterAccountingServiceServer(srv, server) - } + c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { + accountingGRPC.RegisterAccountingServiceServer(s, server) + }) } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 27c028bae..e887c2e6c 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -460,18 +460,80 @@ func (c *cfg) ReadCurrentNetMap(msg *netmapV2.NetMap) error { return nil } +type grpcServer struct { + Listener net.Listener + Server *grpc.Server + Endpoint string +} + type cfgGRPC struct { - listeners []net.Listener - - servers []*grpc.Server - - endpoints []string - - maxChunkSize uint64 + // guard protects connections and handlers + guard sync.RWMutex + // servers must be protected with guard + servers []grpcServer + // handlers must be protected with guard + handlers []func(e string, l net.Listener, s *grpc.Server) + maxChunkSize uint64 maxAddrAmount uint64 } +func (c *cfgGRPC) append(e string, l net.Listener, s *grpc.Server) { + c.guard.Lock() + defer c.guard.Unlock() + + c.servers = append(c.servers, grpcServer{ + Listener: l, + Server: s, + Endpoint: e, + }) +} + +func (c *cfgGRPC) appendAndHandle(e string, l net.Listener, s *grpc.Server) { + c.guard.Lock() + defer c.guard.Unlock() + + c.servers = append(c.servers, grpcServer{ + Listener: l, + Server: s, + Endpoint: e, + }) + + for _, h := range c.handlers { + h(e, l, s) + } +} + +func (c *cfgGRPC) performAndSave(handler func(e string, l net.Listener, s *grpc.Server)) { + c.guard.Lock() + defer c.guard.Unlock() + + for _, conn := range c.servers { + handler(conn.Endpoint, conn.Listener, conn.Server) + } + + c.handlers = append(c.handlers, handler) +} + +func (c *cfgGRPC) dropConnection(endpoint string) { + c.guard.Lock() + defer c.guard.Unlock() + + pos := -1 + for idx, srv := range c.servers { + if srv.Endpoint == endpoint { + pos = idx + break + } + } + if pos < 0 { + return + } + + c.servers[pos].Server.Stop() // closes listener + c.servers = append(c.servers[0:pos], c.servers[pos+1:]...) +} + type cfgMorph struct { client *client.Client @@ -1232,7 +1294,7 @@ func (c *cfg) shutdown() { } c.ctxCancel() - c.done <- struct{}{} + close(c.done) for i := range c.closers { c.closers[len(c.closers)-1-i].fn() } diff --git a/cmd/frostfs-node/config/grpc/config.go b/cmd/frostfs-node/config/grpc/config.go index c25d2e717..37dd76426 100644 --- a/cmd/frostfs-node/config/grpc/config.go +++ b/cmd/frostfs-node/config/grpc/config.go @@ -3,6 +3,7 @@ package grpcconfig import ( "errors" "strconv" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" ) @@ -109,3 +110,17 @@ func IterateEndpoints(c *config.Config, f func(*Config)) { panic("no gRPC server configured") } } + +const DefaultReconnectInterval = time.Minute + +// ReconnectTimeout returns the value of "reconnect_interval" gRPC config parameter. +// +// Returns DefaultReconnectInterval if value is not defined or invalid. +func ReconnectTimeout(c *config.Config) time.Duration { + grpcConf := c.Sub("grpc") + ri := config.DurationSafe(grpcConf, "reconnect_interval") + if ri > 0 { + return ri + } + return DefaultReconnectInterval +} diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 5477947d7..4fc7f5649 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "net" containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -16,6 +17,7 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "go.uber.org/zap" + "google.golang.org/grpc" ) func initContainerService(_ context.Context, c *cfg) { @@ -37,9 +39,9 @@ func initContainerService(_ context.Context, c *cfg) { ), ) - for _, srv := range c.cfgGRPC.servers { - containerGRPC.RegisterContainerServiceServer(srv, server) - } + c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { + containerGRPC.RegisterContainerServiceServer(s, server) + }) } func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc containerCore.Source) (*morphContainerReader, *morphContainerWriter) { diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index 74df7a18f..cc78440bf 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -3,7 +3,6 @@ package main import ( "crypto/tls" "errors" - "fmt" "net" "time" @@ -20,91 +19,154 @@ import ( const maxRecvMsgSize = 256 << 20 func initGRPC(c *cfg) { + var endpointsToReconnect []string var successCount int grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) { - serverOpts := []grpc.ServerOption{ - grpc.MaxRecvMsgSize(maxRecvMsgSize), - grpc.ChainUnaryInterceptor( - metrics.NewUnaryServerInterceptor(), - tracing.NewUnaryServerInterceptor(), - ), - grpc.ChainStreamInterceptor( - metrics.NewStreamServerInterceptor(), - tracing.NewStreamServerInterceptor(), - ), - } - - tlsCfg := sc.TLS() - - if tlsCfg != nil { - cert, err := tls.LoadX509KeyPair(tlsCfg.CertificateFile(), tlsCfg.KeyFile()) - if err != nil { - c.log.Error(logs.FrostFSNodeCouldNotReadCertificateFromFile, zap.Error(err)) - return - } - - var cipherSuites []uint16 - if !tlsCfg.UseInsecureCrypto() { - // This more or less follows the list in https://wiki.mozilla.org/Security/Server_Side_TLS - // excluding: - // 1. TLS 1.3 suites need not be specified here. - // 2. Suites that use DH key exchange are not implemented by stdlib. - cipherSuites = []uint16{ - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, - } - } - creds := credentials.NewTLS(&tls.Config{ - MinVersion: tls.VersionTLS12, - CipherSuites: cipherSuites, - Certificates: []tls.Certificate{cert}, - }) - - serverOpts = append(serverOpts, grpc.Creds(creds)) + serverOpts, ok := getGrpcServerOpts(c, sc) + if !ok { + return } lis, err := net.Listen("tcp", sc.Endpoint()) if err != nil { c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(sc.Endpoint()) c.log.Error(logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) + endpointsToReconnect = append(endpointsToReconnect, sc.Endpoint()) return } c.metricsCollector.GrpcServerMetrics().MarkHealthy(sc.Endpoint()) - c.cfgGRPC.listeners = append(c.cfgGRPC.listeners, lis) - c.cfgGRPC.endpoints = append(c.cfgGRPC.endpoints, sc.Endpoint()) - srv := grpc.NewServer(serverOpts...) c.onShutdown(func() { stopGRPC("FrostFS Public API", srv, c.log) }) - c.cfgGRPC.servers = append(c.cfgGRPC.servers, srv) + c.cfgGRPC.append(sc.Endpoint(), lis, srv) successCount++ }) if successCount == 0 { fatalOnErr(errors.New("could not listen to any gRPC endpoints")) } + + for _, endpoint := range endpointsToReconnect { + scheduleReconnect(endpoint, c) + } +} + +func scheduleReconnect(endpoint string, c *cfg) { + c.wg.Add(1) + go func() { + defer c.wg.Done() + + timeout := grpcconfig.ReconnectTimeout(c.appCfg) + t := time.NewTicker(timeout) + for { + select { + case <-t.C: + c.log.Info(logs.FrostFSNodeGRPCReconnecting, zap.String("endpoint", endpoint)) + var success, found bool + grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) { + if sc.Endpoint() != endpoint { + return + } + found = true + serverOpts, ok := getGrpcServerOpts(c, sc) + if !ok { + return + } + lis, err := net.Listen("tcp", sc.Endpoint()) + if err != nil { + c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(sc.Endpoint()) + c.log.Error(logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) + return + } + c.metricsCollector.GrpcServerMetrics().MarkHealthy(sc.Endpoint()) + + srv := grpc.NewServer(serverOpts...) + + c.onShutdown(func() { + stopGRPC("FrostFS Public API", srv, c.log) + }) + + c.cfgGRPC.appendAndHandle(sc.Endpoint(), lis, srv) + success = true + }) + if !found { + c.log.Warn(logs.FrostFSNodeGRPCServerConfigNotFound, zap.String("endpoint", endpoint)) + return + } + + if success { + c.log.Info(logs.FrostFSNodeGRPCReconnectedSuccessfully, zap.String("endpoint", endpoint)) + return + } + c.log.Warn(logs.FrostFSNodeGRPCReconnectFailed, zap.Duration("next_try_in", timeout)) + case <-c.done: + return + } + } + }() +} + +func getGrpcServerOpts(c *cfg, sc *grpcconfig.Config) ([]grpc.ServerOption, bool) { + serverOpts := []grpc.ServerOption{ + grpc.MaxRecvMsgSize(maxRecvMsgSize), + grpc.ChainUnaryInterceptor( + metrics.NewUnaryServerInterceptor(), + tracing.NewUnaryServerInterceptor(), + ), + grpc.ChainStreamInterceptor( + metrics.NewStreamServerInterceptor(), + tracing.NewStreamServerInterceptor(), + ), + } + + tlsCfg := sc.TLS() + + if tlsCfg != nil { + cert, err := tls.LoadX509KeyPair(tlsCfg.CertificateFile(), tlsCfg.KeyFile()) + if err != nil { + c.log.Error(logs.FrostFSNodeCouldNotReadCertificateFromFile, zap.Error(err)) + return nil, false + } + + var cipherSuites []uint16 + if !tlsCfg.UseInsecureCrypto() { + // This more or less follows the list in https://wiki.mozilla.org/Security/Server_Side_TLS + // excluding: + // 1. TLS 1.3 suites need not be specified here. + // 2. Suites that use DH key exchange are not implemented by stdlib. + cipherSuites = []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + } + } + creds := credentials.NewTLS(&tls.Config{ + MinVersion: tls.VersionTLS12, + CipherSuites: cipherSuites, + Certificates: []tls.Certificate{cert}, + }) + + serverOpts = append(serverOpts, grpc.Creds(creds)) + } + + return serverOpts, true } func serveGRPC(c *cfg) { - for i := range c.cfgGRPC.servers { + c.cfgGRPC.performAndSave(func(e string, l net.Listener, s *grpc.Server) { c.wg.Add(1) - srv := c.cfgGRPC.servers[i] - lis := c.cfgGRPC.listeners[i] - endpoint := c.cfgGRPC.endpoints[i] - go func() { defer func() { c.log.Info(logs.FrostFSNodeStopListeningGRPCEndpoint, - zap.Stringer("endpoint", lis.Addr()), + zap.Stringer("endpoint", l.Addr()), ) c.wg.Done() @@ -112,15 +174,17 @@ func serveGRPC(c *cfg) { c.log.Info(logs.FrostFSNodeStartListeningEndpoint, zap.String("service", "gRPC"), - zap.Stringer("endpoint", lis.Addr()), + zap.Stringer("endpoint", l.Addr()), ) - if err := srv.Serve(lis); err != nil { - c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(endpoint) - fmt.Println("gRPC server error", err) + if err := s.Serve(l); err != nil { + c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(e) + c.log.Error(logs.FrostFSNodeGRPCServerError, zap.Error(err)) + c.cfgGRPC.dropConnection(e) + scheduleReconnect(e, c) } }() - } + }) } func stopGRPC(name string, s *grpc.Server, l *logger.Logger) { diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 347121e56..b21e842c5 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "net" "sync/atomic" netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" @@ -21,6 +22,7 @@ import ( netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "go.uber.org/zap" + "google.golang.org/grpc" ) // primary solution of local network state dump. @@ -162,9 +164,9 @@ func initNetmapService(ctx context.Context, c *cfg) { ), ) - for _, srv := range c.cfgGRPC.servers { - netmapGRPC.RegisterNetmapServiceServer(srv, server) - } + c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { + netmapGRPC.RegisterNetmapServiceServer(s, server) + }) addNewEpochNotificationHandlers(c) } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 59827179b..e84696e6b 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object/grpc" @@ -43,6 +44,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "go.uber.org/zap" + "google.golang.org/grpc" ) type objectSvc struct { @@ -204,9 +206,9 @@ func initObjectService(c *cfg) { signSvc, c.metricsCollector.ObjectService(), metricsconfig.Enabled(c.appCfg)) server := objectTransportGRPC.New(c.shared.metricsSvc) - for _, srv := range c.cfgGRPC.servers { - objectGRPC.RegisterObjectServiceServer(srv, server) - } + c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { + objectGRPC.RegisterObjectServiceServer(s, server) + }) } func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.ClientCache) { diff --git a/cmd/frostfs-node/session.go b/cmd/frostfs-node/session.go index f9c1811a1..ee21ec230 100644 --- a/cmd/frostfs-node/session.go +++ b/cmd/frostfs-node/session.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "net" "time" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -16,6 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage/persistent" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage/temporary" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "google.golang.org/grpc" ) type sessionStorage interface { @@ -57,7 +59,7 @@ func initSessionService(c *cfg) { ), ) - for _, srv := range c.cfgGRPC.servers { - sessionGRPC.RegisterSessionServiceServer(srv, server) - } + c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { + sessionGRPC.RegisterSessionServiceServer(s, server) + }) } diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index f7c0f2a36..dced05bc2 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "net" "time" treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree" @@ -15,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" + "google.golang.org/grpc" ) type cnrSource struct { @@ -63,9 +65,9 @@ func initTreeService(c *cfg) { tree.WithAuthorizedKeys(treeConfig.AuthorizedKeys()), tree.WithMetrics(c.metricsCollector.TreeService())) - for _, srv := range c.cfgGRPC.servers { - tree.RegisterTreeServiceServer(srv, c.treeService) - } + c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { + tree.RegisterTreeServiceServer(s, c.treeService) + }) c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { c.treeService.Start(ctx) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 1775b18cd..c3f4fdc7f 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -423,6 +423,11 @@ const ( FrostFSNodeStoppingGRPCServer = "stopping gRPC server..." FrostFSNodeGRPCCannotShutdownGracefullyForcingStop = "gRPC cannot shutdown gracefully, forcing stop" FrostFSNodeGRPCServerStoppedSuccessfully = "gRPC server stopped successfully" + FrostFSNodeGRPCServerError = "gRPC server error" + FrostFSNodeGRPCReconnecting = "reconnecting gRPC server..." + FrostFSNodeGRPCReconnectedSuccessfully = "gRPC server reconnected successfully" + FrostFSNodeGRPCServerConfigNotFound = "gRPC server config not found" + FrostFSNodeGRPCReconnectFailed = "failed to reconnect gRPC server" FrostFSNodeWaitingForAllProcessesToStop = "waiting for all processes to stop" FrostFSNodeStartedLocalNodesMaintenance = "started local node's maintenance" FrostFSNodeStoppedLocalNodesMaintenance = "stopped local node's maintenance" From 61da7dca24162448689c71a14a116c3d7b401b7b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Dec 2023 18:32:02 +0300 Subject: [PATCH 0756/1943] [#835] node: Fix appCfg concurrent access Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 29 ++++++++++-- cmd/frostfs-node/grpc.go | 91 ++++++++++++++++++++++---------------- 2 files changed, 79 insertions(+), 41 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e887c2e6c..e6c30376f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -426,11 +426,26 @@ type dynamicConfiguration struct { metrics *httpComponent } +type appConfigGuard struct { + mtx sync.RWMutex +} + +func (g *appConfigGuard) LockAppConfigShared() func() { + g.mtx.RLock() + return func() { g.mtx.RUnlock() } +} + +func (g *appConfigGuard) LockAppConfigExclusive() func() { + g.mtx.Lock() + return func() { g.mtx.Unlock() } +} + type cfg struct { applicationConfiguration internals shared dynamicConfiguration + appConfigGuard // configuration of the internal // services @@ -474,8 +489,9 @@ type cfgGRPC struct { // handlers must be protected with guard handlers []func(e string, l net.Listener, s *grpc.Server) - maxChunkSize uint64 - maxAddrAmount uint64 + maxChunkSize uint64 + maxAddrAmount uint64 + reconnectTimeout time.Duration } func (c *cfgGRPC) append(e string, l net.Listener, s *grpc.Server) { @@ -1204,7 +1220,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { } defer c.compareAndSwapHealthStatus(control.HealthStatus_RECONFIGURING, control.HealthStatus_READY) - err := c.readConfig(c.appCfg) + err := c.reloadAppConfig() if err != nil { c.log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) return @@ -1271,6 +1287,13 @@ func (c *cfg) reloadConfig(ctx context.Context) { c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } +func (c *cfg) reloadAppConfig() error { + unlock := c.LockAppConfigExclusive() + defer unlock() + + return c.readConfig(c.appCfg) +} + func (c *cfg) createTombstoneSource() *tombstone.ExpirationChecker { var tssPrm tsourse.TombstoneSourcePrm tssPrm.SetGetService(c.cfgObject.getSvc) diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index cc78440bf..3a38b2cca 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -49,6 +49,7 @@ func initGRPC(c *cfg) { if successCount == 0 { fatalOnErr(errors.New("could not listen to any gRPC endpoints")) } + c.cfgGRPC.reconnectTimeout = grpcconfig.ReconnectTimeout(c.appCfg) for _, endpoint := range endpointsToReconnect { scheduleReconnect(endpoint, c) @@ -60,49 +61,13 @@ func scheduleReconnect(endpoint string, c *cfg) { go func() { defer c.wg.Done() - timeout := grpcconfig.ReconnectTimeout(c.appCfg) - t := time.NewTicker(timeout) + t := time.NewTicker(c.cfgGRPC.reconnectTimeout) for { select { case <-t.C: - c.log.Info(logs.FrostFSNodeGRPCReconnecting, zap.String("endpoint", endpoint)) - var success, found bool - grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) { - if sc.Endpoint() != endpoint { - return - } - found = true - serverOpts, ok := getGrpcServerOpts(c, sc) - if !ok { - return - } - lis, err := net.Listen("tcp", sc.Endpoint()) - if err != nil { - c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(sc.Endpoint()) - c.log.Error(logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) - return - } - c.metricsCollector.GrpcServerMetrics().MarkHealthy(sc.Endpoint()) - - srv := grpc.NewServer(serverOpts...) - - c.onShutdown(func() { - stopGRPC("FrostFS Public API", srv, c.log) - }) - - c.cfgGRPC.appendAndHandle(sc.Endpoint(), lis, srv) - success = true - }) - if !found { - c.log.Warn(logs.FrostFSNodeGRPCServerConfigNotFound, zap.String("endpoint", endpoint)) + if tryReconnect(endpoint, c) { return } - - if success { - c.log.Info(logs.FrostFSNodeGRPCReconnectedSuccessfully, zap.String("endpoint", endpoint)) - return - } - c.log.Warn(logs.FrostFSNodeGRPCReconnectFailed, zap.Duration("next_try_in", timeout)) case <-c.done: return } @@ -110,6 +75,56 @@ func scheduleReconnect(endpoint string, c *cfg) { }() } +func tryReconnect(endpoint string, c *cfg) bool { + c.log.Info(logs.FrostFSNodeGRPCReconnecting, zap.String("endpoint", endpoint)) + + serverOpts, found := getGRPCEndpointOpts(endpoint, c) + if !found { + c.log.Warn(logs.FrostFSNodeGRPCServerConfigNotFound, zap.String("endpoint", endpoint)) + return true + } + + lis, err := net.Listen("tcp", endpoint) + if err != nil { + c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(endpoint) + c.log.Error(logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) + c.log.Warn(logs.FrostFSNodeGRPCReconnectFailed, zap.Duration("next_try_in", c.cfgGRPC.reconnectTimeout)) + return false + } + c.metricsCollector.GrpcServerMetrics().MarkHealthy(endpoint) + + srv := grpc.NewServer(serverOpts...) + + c.onShutdown(func() { + stopGRPC("FrostFS Public API", srv, c.log) + }) + + c.cfgGRPC.appendAndHandle(endpoint, lis, srv) + + c.log.Info(logs.FrostFSNodeGRPCReconnectedSuccessfully, zap.String("endpoint", endpoint)) + return true +} + +func getGRPCEndpointOpts(endpoint string, c *cfg) (result []grpc.ServerOption, found bool) { + unlock := c.LockAppConfigShared() + defer unlock() + grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) { + if found { + return + } + if sc.Endpoint() != endpoint { + return + } + var ok bool + result, ok = getGrpcServerOpts(c, sc) + if !ok { + return + } + found = true + }) + return +} + func getGrpcServerOpts(c *cfg, sc *grpcconfig.Config) ([]grpc.ServerOption, bool) { serverOpts := []grpc.ServerOption{ grpc.MaxRecvMsgSize(maxRecvMsgSize), From 7d7cf05575db445620a137d568f491309363bfc1 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 13 Dec 2023 15:46:31 +0300 Subject: [PATCH 0757/1943] [#851] ape: Initialize and use policy contract interface * Replace inmemory policy contract mock by initialized policy contract interface. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/config.go | 8 ++++++-- cmd/frostfs-node/main.go | 5 +++-- cmd/frostfs-node/morph.go | 1 + go.mod | 2 +- go.sum | 4 ++-- pkg/morph/client/client.go | 7 +++++++ pkg/morph/client/nns.go | 2 ++ 7 files changed, 22 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e6c30376f..e36081ba6 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -70,7 +70,7 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" - "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" + policy_client "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" neogoutil "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" @@ -623,6 +623,8 @@ type cfgLocalStorage struct { } type cfgAccessPolicyEngine struct { + policyContractHash neogoutil.Uint160 + accessPolicyEngine *accessPolicyEngine } @@ -1077,7 +1079,9 @@ func initAccessPolicyEngine(_ context.Context, c *cfg) { ) } - morphRuleStorage := inmemory.NewInmemoryMorphRuleChainStorage() + morphRuleStorage := policy_client.NewContractStorage( + c.cfgMorph.client.GetActor(), + c.cfgObject.cfgAccessPolicyEngine.policyContractHash) ape := newAccessPolicyEngine(morphRuleStorage, localOverrideDB) c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine = ape diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index d4dfb7e60..2fe3a0a25 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -98,14 +98,15 @@ func initApp(ctx context.Context, c *cfg) { fatalOnErr(c.cfgObject.cfgLocalStorage.localStorage.Init(ctx)) }) + initAndLog(c, "gRPC", initGRPC) + initAndLog(c, "netmap", func(c *cfg) { initNetmapService(ctx, c) }) + initAccessPolicyEngine(ctx, c) initAndLog(c, "access policy engine", func(c *cfg) { fatalOnErr(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalOverrideDatabaseCore().Open(ctx)) fatalOnErr(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalOverrideDatabaseCore().Init()) }) - initAndLog(c, "gRPC", initGRPC) - initAndLog(c, "netmap", func(c *cfg) { initNetmapService(ctx, c) }) initAndLog(c, "accounting", func(c *cfg) { initAccountingService(ctx, c) }) initAndLog(c, "container", func(c *cfg) { initContainerService(ctx, c) }) initAndLog(c, "session", initSessionService) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index f7100d0bf..d26142370 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -289,6 +289,7 @@ func lookupScriptHashesInNNS(c *cfg) { {&c.cfgAccounting.scriptHash, client.NNSBalanceContractName}, {&c.cfgContainer.scriptHash, client.NNSContainerContractName}, {&c.cfgMorph.proxyScriptHash, client.NNSProxyContractName}, + {&c.cfgObject.cfgAccessPolicyEngine.policyContractHash, client.NNSPolicyContractName}, } ) diff --git a/go.mod b/go.mod index 3e79f7ff7..6463c3b2b 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231211080303-8c673ee4f4af + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231214122253-62ea96b82ce3 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 diff --git a/go.sum b/go.sum index 15ae22422..c18e291f2 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231211080303-8c673ee4f4af h1:QSgejckGChrry5waJqf2votsOY0J7Sfh8tPqos0rCXA= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231211080303-8c673ee4f4af/go.mod h1:iJMX6qk9aIHIu3WVSd4puF5CHsNk5eOi++MaJJfNbXM= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231214122253-62ea96b82ce3 h1:xnS/YalLqCRplvpF3E9/PRGevj4cDe7qFNZT0mkjZcs= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231214122253-62ea96b82ce3/go.mod h1:v43imcuSmDwSNrePe4UTQh8jaE8FmsiKN3FcaEzmRzc= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 93a4176f9..e52adfa8e 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -539,3 +539,10 @@ func (c *Client) setActor(act *actor.Actor) { c.gasToken = nep17.New(act, gas.Hash) c.rolemgmt = rolemgmt.New(act) } + +func (c *Client) GetActor() *actor.Actor { + c.switchLock.RLock() + defer c.switchLock.RUnlock() + + return c.rpcActor +} diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index 758b220a2..218f7ad8e 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -33,6 +33,8 @@ const ( NNSProxyContractName = "proxy.frostfs" // NNSGroupKeyName is a name for the FrostFS group key record in NNS. NNSGroupKeyName = "group.frostfs" + // NNSPolicyContractName is a name of the policy contract in NNS. + NNSPolicyContractName = "policy.frostfs" ) var ( From be8f499b91d426f8435da6e9be0574f237b61b82 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 18 Dec 2023 14:53:00 +0300 Subject: [PATCH 0758/1943] [#717] Fix mixing CID and container ID All container ID outputs that may be used in scripts were replaced with CID. Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/modules/container/create.go | 2 +- cmd/frostfs-cli/modules/container/get.go | 2 +- cmd/frostfs-cli/modules/control/add_rule.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 37827b617..f670043ea 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -113,7 +113,7 @@ It will be stored in sidechain when inner ring will accepts it.`, id := res.ID() - cmd.Println("container ID:", id) + cmd.Println("CID:", id) if containerAwait { cmd.Println("awaiting...") diff --git a/cmd/frostfs-cli/modules/container/get.go b/cmd/frostfs-cli/modules/container/get.go index cd920f0c4..1681dc7f3 100644 --- a/cmd/frostfs-cli/modules/container/get.go +++ b/cmd/frostfs-cli/modules/container/get.go @@ -83,7 +83,7 @@ func prettyPrintContainer(cmd *cobra.Command, cnr container.Container, jsonEncod var id cid.ID container.CalculateID(&id, cnr) - cmd.Println("container ID:", id) + cmd.Println("CID:", id) cmd.Println("owner ID:", cnr.Owner()) diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index 48101c570..47233678d 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -57,7 +57,7 @@ func addRule(cmd *cobra.Command, _ []string) { chain.ID = apechain.ID(chainID) serializedChain := chain.Bytes() - cmd.Println("Container ID: " + cidStr) + cmd.Println("CID: " + cidStr) cmd.Println("Parsed chain:\n" + prettyJSONFormat(cmd, serializedChain)) req := &control.AddChainLocalOverrideRequest{ From d9cbb16bd3cec547ba9621a51ac3d8b0d75b211b Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 18 Dec 2023 12:47:32 +0300 Subject: [PATCH 0759/1943] [#866] Use TTL for blobovnicza tree cache Signed-off-by: Anton Nikiforov --- .../internal/modules/storagecfg/config.go | 1 + cmd/frostfs-node/config.go | 3 +++ .../shard/blobstor/blobovnicza/config.go | 21 +++++++++++++++++++ config/example/node.env | 2 ++ config/example/node.json | 2 ++ config/example/node.yaml | 1 + docs/storage-node-configuration.md | 18 +++++++++------- .../blobstor/blobovniczatree/blobovnicza.go | 2 +- .../blobstor/blobovniczatree/cache.go | 13 +++++------- .../blobstor/blobovniczatree/option.go | 9 ++++++++ 10 files changed, 55 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/config.go b/cmd/frostfs-adm/internal/modules/storagecfg/config.go index a07ce32c6..9f7efe2df 100644 --- a/cmd/frostfs-adm/internal/modules/storagecfg/config.go +++ b/cmd/frostfs-adm/internal/modules/storagecfg/config.go @@ -61,6 +61,7 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files + opened_cache_ttl: 5m # ttl for opened database file gc: remover_batch_size: 200 # number of objects to be removed by the garbage collector diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e36081ba6..88b93e6aa 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -185,6 +185,7 @@ type subStorageCfg struct { width uint64 leafWidth uint64 openedCacheSize int + openedCacheTTL time.Duration initWorkerCount int initInAdvance bool } @@ -305,6 +306,7 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.width = sub.ShallowWidth() sCfg.leafWidth = sub.LeafWidth() sCfg.openedCacheSize = sub.OpenedCacheSize() + sCfg.openedCacheTTL = sub.OpenedCacheTTL() sCfg.initWorkerCount = sub.InitWorkerCount() sCfg.initInAdvance = sub.InitInAdvance() case fstree.Type: @@ -898,6 +900,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobovniczatree.WithOpenedCacheTTL(sRead.openedCacheTTL), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), blobovniczatree.WithInitInAdvance(sRead.initInAdvance), blobovniczatree.WithLogger(c.log), diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index 37191cc72..0e007b425 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -1,6 +1,8 @@ package blobovniczaconfig import ( + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" @@ -23,6 +25,9 @@ const ( // OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's. OpenedCacheSizeDefault = 16 + // OpenedCacheTTLDefault is a default cache ttl of opened Blobovnicza's. + OpenedCacheTTLDefault = 0 // means expiring is off + // InitWorkerCountDefault is a default workers count to initialize Blobovnicza's. InitWorkerCountDefault = 5 ) @@ -101,6 +106,22 @@ func (x *Config) OpenedCacheSize() int { return OpenedCacheSizeDefault } +// OpenedCacheTTL returns the value of "opened_cache_ttl" config parameter. +// +// Returns OpenedCacheTTLDefault if the value is not a positive number. +func (x *Config) OpenedCacheTTL() time.Duration { + d := config.DurationSafe( + (*config.Config)(x), + "opened_cache_ttl", + ) + + if d > 0 { + return d + } + + return OpenedCacheTTLDefault +} + // BoltDB returns config instance for querying bolt db specific parameters. func (x *Config) BoltDB() *boltdbconfig.Config { return (*boltdbconfig.Config)(x) diff --git a/config/example/node.env b/config/example/node.env index be16f62bf..2972f56e9 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -125,6 +125,7 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 +FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=5m FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE @@ -174,6 +175,7 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 +FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_TTL=5m FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10 ### FSTree config FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree diff --git a/config/example/node.json b/config/example/node.json index ff38e3894..c803e11fb 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -173,6 +173,7 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, + "opened_cache_ttl": "5m", "leaf_width": 10, "init_worker_count": 10, "init_in_advance": true @@ -226,6 +227,7 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, + "opened_cache_ttl": "5m", "leaf_width": 10 }, { diff --git a/config/example/node.yaml b/config/example/node.yaml index c6a2be9ac..d4c3516bc 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -150,6 +150,7 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files + opened_cache_ttl: 5m # ttl for opened database file leaf_width: 10 # max count of key-value DB on leafs of object tree storage - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) depth: 5 # max depth of object tree storage in FS diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 71eabc042..613be1288 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -209,6 +209,7 @@ blobstor: depth: 1 width: 4 opened_cache_capacity: 50 + opened_cache_ttl: 5m ``` #### Common options for sub-storages @@ -225,14 +226,15 @@ blobstor: | `depth` | `int` | `4` | File-system tree depth. | #### `blobovnicza` type options -| Parameter | Type | Default value | Description | -|-------------------------|-----------|---------------|-------------------------------------------------------| -| `path` | `string` | | Path to the root of the blobstor. | -| `perm` | file mode | `0660` | Default permission for created files and directories. | -| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | -| `depth` | `int` | `2` | Blobovnicza tree depth. | -| `width` | `int` | `16` | Blobovnicza tree width. | -| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | +| Parameter | Type | Default value | Description | +|-------------------------|------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `path` | `string` | | Path to the root of the blobstor. | +| `perm` | file mode | `0660` | Default permission for created files and directories. | +| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | +| `depth` | `int` | `2` | Blobovnicza tree depth. | +| `width` | `int` | `16` | Blobovnicza tree width. | +| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | +| `opened_cache_ttl` | `duration` | `0` | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. | ### `gc` subsection diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 9f2950094..460aebbba 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -87,7 +87,7 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) - blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager) + blz.dbCache = newDBCache(blz.openedCacheSize, blz.openedCacheTTL, blz.commondbManager) blz.deleteProtectedObjects = newAddressMap() blz.dbFilesGuard = &sync.RWMutex{} blz.rebuildGuard = &sync.RWMutex{} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go index 317bac5ff..d8fe6e8b0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -1,10 +1,11 @@ package blobovniczatree import ( - "fmt" "sync" + "time" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" + "github.com/hashicorp/golang-lru/v2/expirable" "github.com/hashicorp/golang-lru/v2/simplelru" ) @@ -22,14 +23,10 @@ type dbCache struct { dbManager *dbManager } -func newDBCache(size int, dbManager *dbManager) *dbCache { - cache, err := simplelru.NewLRU[string, *sharedDB](size, func(_ string, evictedDB *sharedDB) { +func newDBCache(size int, ttl time.Duration, dbManager *dbManager) *dbCache { + cache := expirable.NewLRU[string, *sharedDB](size, func(_ string, evictedDB *sharedDB) { evictedDB.Close() - }) - if err != nil { - // occurs only if the size is not positive - panic(fmt.Errorf("could not create LRU cache of size %d: %w", size, err)) - } + }, ttl) return &dbCache{ cacheGuard: &sync.RWMutex{}, cache: cache, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index d36074dc8..e93640fae 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -16,6 +16,7 @@ type cfg struct { readOnly bool rootPath string openedCacheSize int + openedCacheTTL time.Duration blzShallowDepth uint64 blzShallowWidth uint64 blzLeafWidth uint64 @@ -34,6 +35,7 @@ type Option func(*cfg) const ( defaultPerm = 0o700 defaultOpenedCacheSize = 50 + defaultOpenedCacheTTL = 0 // means expiring is off defaultBlzShallowDepth = 2 defaultBlzShallowWidth = 16 defaultWaitBeforeDropDB = 10 * time.Second @@ -46,6 +48,7 @@ func initConfig(c *cfg) { log: &logger.Logger{Logger: zap.L()}, perm: defaultPerm, openedCacheSize: defaultOpenedCacheSize, + openedCacheTTL: defaultOpenedCacheTTL, blzShallowDepth: defaultBlzShallowDepth, blzShallowWidth: defaultBlzShallowWidth, reportError: func(string, error) {}, @@ -105,6 +108,12 @@ func WithOpenedCacheSize(sz int) Option { } } +func WithOpenedCacheTTL(ttl time.Duration) Option { + return func(c *cfg) { + c.openedCacheTTL = ttl + } +} + func WithObjectSizeLimit(sz uint64) Option { return func(c *cfg) { c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz)) From d69d318cb0d3ba8ce2d652606fa1192ac9a42b97 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 18 Dec 2023 17:55:24 +0300 Subject: [PATCH 0760/1943] [#878] node: Drain internal error's channel This fixes shutdown panic: 1. Some morph connection gets error and passes it to internalErr channel. 2. Storage node starts to shutdow and closes internalErr channel. 3. Other morph connection gets error and tries to pass it to internalErr channel. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 - cmd/frostfs-node/main.go | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 88b93e6aa..82e203788 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1328,5 +1328,4 @@ func (c *cfg) shutdown() { for i := range c.closers { c.closers[len(c.closers)-1-i].fn() } - close(c.internalErr) } diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 2fe3a0a25..f91a6decc 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "os" + "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -155,9 +156,22 @@ func wait(c *cfg) { <-c.done // graceful shutdown + drain := &sync.WaitGroup{} + drain.Add(1) + go func() { + defer drain.Done() + for err := range c.internalErr { + c.log.Warn(logs.FrostFSNodeInternalApplicationError, + zap.String("message", err.Error())) + } + }() + c.log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop) c.wg.Wait() + + close(c.internalErr) + drain.Wait() } func (c *cfg) onShutdown(f func()) { From 7ade11922e2d8bfe014ba34125f619c87ffe423f Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 20 Dec 2023 15:22:17 +0300 Subject: [PATCH 0761/1943] [#834] adm: Refactor `morph` module Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/deploy.go | 4 +- .../internal/modules/morph/initialize.go | 56 +----------- .../internal/modules/morph/n3client.go | 8 +- .../internal/modules/morph/root.go | 57 ++++++------ .../internal/modules/morph/util.go | 86 +++++++++++++++++++ 5 files changed, 120 insertions(+), 91 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/util.go diff --git a/cmd/frostfs-adm/internal/modules/morph/deploy.go b/cmd/frostfs-adm/internal/modules/morph/deploy.go index 54b506715..1facd6d10 100644 --- a/cmd/frostfs-adm/internal/modules/morph/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/deploy.go @@ -46,10 +46,10 @@ NNS name is taken by stripping '_contract.nef' from the NEF file (similar to fro func init() { ff := deployCmd.Flags() - ff.String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") + ff.String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) _ = deployCmd.MarkFlagFilename(alphabetWalletsFlag) - ff.StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + ff.StringP(endpointFlag, "r", "", endpointFlagDesc) ff.String(contractPathFlag, "", "Path to the contract directory") _ = deployCmd.MarkFlagFilename(contractPathFlag) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 96100ffb5..78356e6b4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -4,12 +4,10 @@ import ( "errors" "fmt" "os" - "path/filepath" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/core/native/nativenames" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -117,15 +115,11 @@ func (c *initializeContext) close() { func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContext, error) { walletDir := config.ResolveHomePath(viper.GetString(alphabetWalletsFlag)) - wallets, err := openAlphabetWallets(v, walletDir) + wallets, err := getAlphabetWallets(v, walletDir) if err != nil { return nil, err } - if len(wallets) > maxAlphabetNodes { - return nil, ErrTooManyAlphabetNodes - } - needContracts := cmd.Name() == "update-contracts" || cmd.Name() == "init" var w *wallet.Wallet @@ -257,54 +251,6 @@ func createWalletAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) { return accounts, nil } -func openAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, error) { - walletFiles, err := os.ReadDir(walletDir) - if err != nil { - return nil, fmt.Errorf("can't read alphabet wallets dir: %w", err) - } - - var size int -loop: - for i := 0; i < len(walletFiles); i++ { - name := innerring.GlagoliticLetter(i).String() + ".json" - for j := range walletFiles { - if walletFiles[j].Name() == name { - size++ - continue loop - } - } - break - } - if size == 0 { - return nil, errors.New("alphabet wallets dir is empty (run `generate-alphabet` command first)") - } - - wallets := make([]*wallet.Wallet, size) - for i := 0; i < size; i++ { - letter := innerring.GlagoliticLetter(i).String() - p := filepath.Join(walletDir, letter+".json") - w, err := wallet.NewWalletFromFile(p) - if err != nil { - return nil, fmt.Errorf("can't open wallet: %w", err) - } - - password, err := config.GetPassword(v, letter) - if err != nil { - return nil, fmt.Errorf("can't fetch password: %w", err) - } - - for i := range w.Accounts { - if err := w.Accounts[i].Decrypt(password, keys.NEP2ScryptParams()); err != nil { - return nil, fmt.Errorf("can't unlock wallet: %w", err) - } - } - - wallets[i] = w - } - - return wallets, nil -} - func (c *initializeContext) awaitTx() error { return c.clientContext.awaitTx(c.Command) } diff --git a/cmd/frostfs-adm/internal/modules/morph/n3client.go b/cmd/frostfs-adm/internal/modules/morph/n3client.go index 3550895c3..584bb8524 100644 --- a/cmd/frostfs-adm/internal/modules/morph/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/n3client.go @@ -73,13 +73,7 @@ func getN3Client(v *viper.Viper) (Client, error) { } func defaultClientContext(c Client, committeeAcc *wallet.Account) (*clientContext, error) { - commAct, err := actor.New(c, []actor.SignerAccount{{ - Signer: transaction.Signer{ - Account: committeeAcc.Contract.ScriptHash(), - Scopes: transaction.Global, - }, - Account: committeeAcc, - }}) + commAct, err := newActor(c, committeeAcc) if err != nil { return nil, err } diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 707ae733d..1a0cd8c8b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -7,8 +7,11 @@ import ( const ( alphabetWalletsFlag = "alphabet-wallets" + alphabetWalletsFlagDesc = "Path to alphabet wallets dir" alphabetSizeFlag = "size" endpointFlag = "rpc-endpoint" + endpointFlagDesc = "N3 RPC node endpoint" + endpointFlagShort = "r" storageWalletFlag = "storage-wallet" storageWalletLabelFlag = "label" storageGasCLIFlag = "initial-gas" @@ -264,12 +267,12 @@ func init() { func initNetmapCandidatesCmd() { RootCmd.AddCommand(netmapCandidatesCmd) - netmapCandidatesCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + netmapCandidatesCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) } func initDepositoryNotaryCmd() { RootCmd.AddCommand(depositNotaryCmd) - depositNotaryCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + depositNotaryCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) depositNotaryCmd.Flags().String(storageWalletFlag, "", "Path to storage node wallet") depositNotaryCmd.Flags().String(walletAccountFlag, "", "Wallet account address") depositNotaryCmd.Flags().String(refillGasAmountFlag, "", "Amount of GAS to deposit") @@ -278,8 +281,8 @@ func initDepositoryNotaryCmd() { func initRefillGasCmd() { RootCmd.AddCommand(refillGasCmd) - refillGasCmd.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") - refillGasCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + refillGasCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + refillGasCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) refillGasCmd.Flags().String(storageWalletFlag, "", "Path to storage node wallet") refillGasCmd.Flags().String(walletAddressFlag, "", "Address of wallet") refillGasCmd.Flags().String(refillGasAmountFlag, "", "Additional amount of GAS to transfer") @@ -288,21 +291,21 @@ func initRefillGasCmd() { func initListContainersCmd() { RootCmd.AddCommand(listContainersCmd) - listContainersCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + listContainersCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) listContainersCmd.Flags().String(containerContractFlag, "", "Container contract hash (for networks without NNS)") } func initRestoreContainersCmd() { RootCmd.AddCommand(restoreContainersCmd) - restoreContainersCmd.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") - restoreContainersCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + restoreContainersCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + restoreContainersCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) restoreContainersCmd.Flags().String(containerDumpFlag, "", "File to restore containers from") restoreContainersCmd.Flags().StringSlice(containerIDsFlag, nil, "Containers to restore") } func initDumpContainersCmd() { RootCmd.AddCommand(dumpContainersCmd) - dumpContainersCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + dumpContainersCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) dumpContainersCmd.Flags().String(containerDumpFlag, "", "File where to save dumped containers") dumpContainersCmd.Flags().String(containerContractFlag, "", "Container contract hash (for networks without NNS)") dumpContainersCmd.Flags().StringSlice(containerIDsFlag, nil, "Containers to dump") @@ -310,15 +313,15 @@ func initDumpContainersCmd() { func initUpdateContractsCmd() { RootCmd.AddCommand(updateContractsCmd) - updateContractsCmd.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") - updateContractsCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + updateContractsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + updateContractsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) updateContractsCmd.Flags().String(contractsInitFlag, "", "Path to archive with compiled FrostFS contracts") _ = updateContractsCmd.MarkFlagRequired(contractsInitFlag) } func initDumpBalancesCmd() { RootCmd.AddCommand(dumpBalancesCmd) - dumpBalancesCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + dumpBalancesCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) dumpBalancesCmd.Flags().BoolP(dumpBalancesStorageFlag, "s", false, "Dump balances of storage nodes from the current netmap") dumpBalancesCmd.Flags().BoolP(dumpBalancesAlphabetFlag, "a", false, "Dump balances of alphabet contracts") dumpBalancesCmd.Flags().BoolP(dumpBalancesProxyFlag, "p", false, "Dump balances of the proxy contract") @@ -327,53 +330,53 @@ func initDumpBalancesCmd() { func initSetConfigCmd() { RootCmd.AddCommand(setConfig) - setConfig.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") - setConfig.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + setConfig.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + setConfig.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) setConfig.Flags().Bool(forceConfigSet, false, "Force setting not well-known configuration key") setConfig.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initDumpNetworkConfigCmd() { RootCmd.AddCommand(dumpNetworkConfigCmd) - dumpNetworkConfigCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + dumpNetworkConfigCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) } func initDumpContractHashesCmd() { RootCmd.AddCommand(dumpContractHashesCmd) - dumpContractHashesCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + dumpContractHashesCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) dumpContractHashesCmd.Flags().String(customZoneFlag, "", "Custom zone to search.") } func initSetPolicyCmd() { RootCmd.AddCommand(setPolicy) - setPolicy.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") - setPolicy.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + setPolicy.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + setPolicy.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) setPolicy.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initDumpPolicyCmd() { RootCmd.AddCommand(dumpPolicy) - dumpPolicy.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + dumpPolicy.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) } func initRemoveNodesCmd() { RootCmd.AddCommand(removeNodes) - removeNodes.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") - removeNodes.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + removeNodes.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + removeNodes.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) removeNodes.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initForceNewEpochCmd() { RootCmd.AddCommand(forceNewEpoch) - forceNewEpoch.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") - forceNewEpoch.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + forceNewEpoch.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + forceNewEpoch.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) forceNewEpoch.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initGenerateStorageCmd() { RootCmd.AddCommand(generateStorageCmd) - generateStorageCmd.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") - generateStorageCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + generateStorageCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + generateStorageCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) generateStorageCmd.Flags().String(storageWalletFlag, "", "Path to new storage node wallet") generateStorageCmd.Flags().String(storageGasCLIFlag, "", "Initial amount of GAS to transfer") generateStorageCmd.Flags().StringP(storageWalletLabelFlag, "l", "", "Wallet label") @@ -381,8 +384,8 @@ func initGenerateStorageCmd() { func initInitCmd() { RootCmd.AddCommand(initCmd) - initCmd.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") - initCmd.Flags().StringP(endpointFlag, "r", "", "N3 RPC node endpoint") + initCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + initCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) initCmd.Flags().String(contractsInitFlag, "", "Path to archive with compiled FrostFS contracts") _ = initCmd.MarkFlagRequired(contractsInitFlag) initCmd.Flags().Uint(epochDurationCLIFlag, 240, "Amount of side chain blocks in one FrostFS epoch") @@ -397,7 +400,7 @@ func initInitCmd() { func initGenerateAlphabetCmd() { RootCmd.AddCommand(generateAlphabetCmd) - generateAlphabetCmd.Flags().String(alphabetWalletsFlag, "", "Path to alphabet wallets dir") + generateAlphabetCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) generateAlphabetCmd.Flags().Uint(alphabetSizeFlag, 7, "Amount of alphabet wallets to generate") } diff --git a/cmd/frostfs-adm/internal/modules/morph/util.go b/cmd/frostfs-adm/internal/modules/morph/util.go new file mode 100644 index 000000000..8b5489ab8 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/util.go @@ -0,0 +1,86 @@ +package morph + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/wallet" + "github.com/spf13/viper" +) + +func getAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, error) { + wallets, err := openAlphabetWallets(v, walletDir) + if err != nil { + return nil, err + } + + if len(wallets) > maxAlphabetNodes { + return nil, ErrTooManyAlphabetNodes + } + return wallets, nil +} + +func openAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, error) { + walletFiles, err := os.ReadDir(walletDir) + if err != nil { + return nil, fmt.Errorf("can't read alphabet wallets dir: %w", err) + } + + var size int +loop: + for i := 0; i < len(walletFiles); i++ { + name := innerring.GlagoliticLetter(i).String() + ".json" + for j := range walletFiles { + if walletFiles[j].Name() == name { + size++ + continue loop + } + } + break + } + if size == 0 { + return nil, errors.New("alphabet wallets dir is empty (run `generate-alphabet` command first)") + } + + wallets := make([]*wallet.Wallet, size) + for i := 0; i < size; i++ { + letter := innerring.GlagoliticLetter(i).String() + p := filepath.Join(walletDir, letter+".json") + w, err := wallet.NewWalletFromFile(p) + if err != nil { + return nil, fmt.Errorf("can't open wallet: %w", err) + } + + password, err := config.GetPassword(v, letter) + if err != nil { + return nil, fmt.Errorf("can't fetch password: %w", err) + } + + for i := range w.Accounts { + if err := w.Accounts[i].Decrypt(password, keys.NEP2ScryptParams()); err != nil { + return nil, fmt.Errorf("can't unlock wallet: %w", err) + } + } + + wallets[i] = w + } + + return wallets, nil +} + +func newActor(c actor.RPCActor, committeeAcc *wallet.Account) (*actor.Actor, error) { + return actor.New(c, []actor.SignerAccount{{ + Signer: transaction.Signer{ + Account: committeeAcc.Contract.ScriptHash(), + Scopes: transaction.Global, + }, + Account: committeeAcc, + }}) +} From 32f4e72e6a2c0fcc5569b131021d1ada5f26dcfb Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 20 Dec 2023 15:44:12 +0300 Subject: [PATCH 0762/1943] [#834] adm: Add commands to invoke methods of `policy` contract Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/internal/modules/morph/ape.go | 162 ++++++++++++++++++ .../internal/modules/morph/ape_util.go | 103 +++++++++++ .../internal/modules/morph/root.go | 4 + cmd/frostfs-cli/modules/util/ape.go | 33 ++++ 4 files changed, 302 insertions(+) create mode 100644 cmd/frostfs-adm/internal/modules/morph/ape.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/ape_util.go diff --git a/cmd/frostfs-adm/internal/modules/morph/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape.go new file mode 100644 index 000000000..6dd2bd185 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/ape.go @@ -0,0 +1,162 @@ +package morph + +import ( + "bytes" + "encoding/json" + + parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + namespaceTarget = "namespace" + containerTarget = "container" + jsonFlag = "json" + jsonFlagDesc = "Output rule chains in JSON format" + chainIDFlag = "chain-id" + chainIDDesc = "Rule chain ID" + ruleFlag = "rule" + ruleFlagDesc = "Rule chain in text format" + ruleJSONFlag = "rule-json" + ruleJSONFlagDesc = "Chain rule in JSON format or path to the file" + targetNameFlag = "target-name" + targetNameDesc = "Resource name in APE resource name format" + targetTypeFlag = "target-type" + targetTypeDesc = "Resource type(container/namespace)" +) + +var ( + apeCmd = &cobra.Command{ + Use: "ape", + Short: "Section for APE configuration commands", + } + + addRuleChainCmd = &cobra.Command{ + Use: "add-rule-chain", + Short: "Add rule chain", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + }, + Run: addRuleChain, + } + + removeRuleChainCmd = &cobra.Command{ + Use: "rm-rule-chain", + Short: "Remove rule chain", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + }, + Run: removeRuleChain, + } + + listRuleChainsCmd = &cobra.Command{ + Use: "list-rule-chains", + Short: "List rule chains", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + }, + Run: listRuleChains, + } +) + +func initAddRuleChainCmd() { + apeCmd.AddCommand(addRuleChainCmd) + + addRuleChainCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + addRuleChainCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + + addRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc) + _ = addRuleChainCmd.MarkFlagRequired(targetTypeFlag) + addRuleChainCmd.Flags().String(targetNameFlag, "", targetNameDesc) + _ = addRuleChainCmd.MarkFlagRequired(targetNameFlag) + + addRuleChainCmd.Flags().String(chainIDFlag, "", chainIDDesc) + _ = addRuleChainCmd.MarkFlagRequired(chainIDFlag) + addRuleChainCmd.Flags().String(ruleFlag, "", ruleFlagDesc) + addRuleChainCmd.Flags().String(ruleJSONFlag, "", ruleJSONFlagDesc) + + addRuleChainCmd.MarkFlagsMutuallyExclusive(ruleFlag, ruleJSONFlag) +} + +func initRemoveRuleChainCmd() { + apeCmd.AddCommand(removeRuleChainCmd) + + removeRuleChainCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + removeRuleChainCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + + removeRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc) + _ = removeRuleChainCmd.MarkFlagRequired(targetTypeFlag) + removeRuleChainCmd.Flags().String(targetNameFlag, "", targetNameDesc) + _ = removeRuleChainCmd.MarkFlagRequired(targetNameFlag) + removeRuleChainCmd.Flags().String(chainIDFlag, "", chainIDDesc) + _ = removeRuleChainCmd.MarkFlagRequired(chainIDFlag) +} + +func initListRuleChainsCmd() { + apeCmd.AddCommand(listRuleChainsCmd) + + listRuleChainsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + listRuleChainsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + listRuleChainsCmd.Flags().StringP(targetTypeFlag, "t", "", targetTypeDesc) + _ = listRuleChainsCmd.MarkFlagRequired(targetTypeFlag) + listRuleChainsCmd.Flags().String(targetNameFlag, "", targetNameDesc) + _ = listRuleChainsCmd.MarkFlagRequired(targetNameFlag) + listRuleChainsCmd.Flags().Bool(jsonFlag, false, jsonFlagDesc) +} + +func addRuleChain(cmd *cobra.Command, _ []string) { + chain := parseChain(cmd) + target := parseTarget(cmd) + pci, ac := newPolicyContractInterface(cmd) + h, vub, err := pci.AddMorphRuleChain(apechain.Ingress, target, chain) + cmd.Println("Waiting for transaction to persist...") + _, err = ac.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "add rule chain error: %w", err) + cmd.Println("Rule chain added successfully") +} + +func removeRuleChain(cmd *cobra.Command, _ []string) { + chainID := parseChainID(cmd) + target := parseTarget(cmd) + pci, ac := newPolicyContractInterface(cmd) + h, vub, err := pci.RemoveMorphRuleChain(apechain.Ingress, target, chainID) + cmd.Println("Waiting for transaction to persist...") + _, err = ac.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "remove rule chain error: %w", err) + cmd.Println("Rule chain removed successfully") +} + +func listRuleChains(cmd *cobra.Command, _ []string) { + target := parseTarget(cmd) + pci, _ := newPolicyContractInterface(cmd) + chains, err := pci.ListMorphRuleChains(apechain.Ingress, target) + commonCmd.ExitOnErr(cmd, "list rule chains error: %w", err) + if len(chains) == 0 { + return + } + + toJSON, _ := cmd.Flags().GetBool(jsonFlag) + if toJSON { + prettyJSONFormat(cmd, chains) + } else { + for _, c := range chains { + parseutil.PrintHumanReadableAPEChain(cmd, c) + } + } +} + +func prettyJSONFormat(cmd *cobra.Command, chains []*apechain.Chain) { + wr := bytes.NewBufferString("") + data, err := json.Marshal(chains) + if err == nil { + err = json.Indent(wr, data, "", " ") + } + commonCmd.ExitOnErr(cmd, "print rule chain error: %w", err) + cmd.Println(wr) +} diff --git a/cmd/frostfs-adm/internal/modules/morph/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape_util.go new file mode 100644 index 000000000..e50522f2d --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/ape_util.go @@ -0,0 +1,103 @@ +package morph + +import ( + "encoding/json" + "fmt" + "os" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + morph "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func parseTarget(cmd *cobra.Command) policyengine.Target { + var targetType policyengine.TargetType + typ, _ := cmd.Flags().GetString(targetTypeFlag) + switch typ { + case namespaceTarget: + targetType = policyengine.Namespace + case containerTarget: + targetType = policyengine.Container + default: + commonCmd.ExitOnErr(cmd, "read target type error: %w", fmt.Errorf("unknown target type")) + } + name, _ := cmd.Flags().GetString(targetNameFlag) + + return policyengine.Target{ + Name: name, + Type: targetType, + } +} + +func parseChainID(cmd *cobra.Command) apechain.ID { + chainID, _ := cmd.Flags().GetString(chainIDFlag) + if chainID == "" { + commonCmd.ExitOnErr(cmd, "read chain id error: %w", + fmt.Errorf("chain id cannot be empty")) + } + return apechain.ID(chainID) +} + +func parseChain(cmd *cobra.Command) *apechain.Chain { + chain := new(apechain.Chain) + + if ruleStmt, _ := cmd.Flags().GetString(ruleFlag); ruleStmt != "" { + parseErr := parseutil.ParseAPEChain(chain, []string{ruleStmt}) + commonCmd.ExitOnErr(cmd, "ape chain parser error: %w", parseErr) + } else if ruleJSON, _ := cmd.Flags().GetString(ruleJSONFlag); ruleJSON != "" { + var rule []byte + if _, err := os.Stat(ruleJSON); err == nil { + rule, err = os.ReadFile(ruleJSON) + commonCmd.ExitOnErr(cmd, "read file error: %w", err) + } else { + rule = []byte(ruleJSON) + if !json.Valid(rule) { + commonCmd.ExitOnErr(cmd, "read raw rule error: %w", + fmt.Errorf("invalid JSON")) + } + } + err := chain.DecodeBytes(rule) + commonCmd.ExitOnErr(cmd, "chain decode error: %w", err) + } else { + commonCmd.ExitOnErr(cmd, "", fmt.Errorf("rule is not passed")) + } + + chain.ID = parseChainID(cmd) + + return chain +} + +func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *actor.Actor) { + v := viper.GetViper() + c, err := getN3Client(v) + commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) + + walletDir := config.ResolveHomePath(viper.GetString(alphabetWalletsFlag)) + wallets, err := getAlphabetWallets(v, walletDir) + commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) + + committeeAcc, err := getWalletAccount(wallets[0], committeeAccountName) + commonCmd.ExitOnErr(cmd, "can't find committee account: %w", err) + + ac, err := newActor(c, committeeAcc) + commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) + + inv := &ac.Invoker + var ch util.Uint160 + r := management.NewReader(inv) + nnsCs, err := r.GetContractByID(1) + commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) + + ch, err = nnsResolveHash(inv, nnsCs.Hash, policyContract+".frostfs") + commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err) + + return morph.NewContractStorage(ac, ch), ac +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 1a0cd8c8b..475bf4b06 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -263,6 +263,10 @@ func init() { initRefillGasCmd() initDepositoryNotaryCmd() initNetmapCandidatesCmd() + RootCmd.AddCommand(apeCmd) + initAddRuleChainCmd() + initRemoveRuleChainCmd() + initListRuleChainsCmd() } func initNetmapCandidatesCmd() { diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index c5f8526e2..79966613d 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -3,11 +3,13 @@ package util import ( "errors" "fmt" + "strconv" "strings" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/flynn-archive/go-shlex" + "github.com/spf13/cobra" ) var ( @@ -20,6 +22,37 @@ var ( errUnknownCondObjectType = errors.New("condition object type is not recognized") ) +// PrintHumanReadableAPEChain print APE chain rules. +func PrintHumanReadableAPEChain(cmd *cobra.Command, chain *apechain.Chain) { + cmd.Println("ChainID: " + string(chain.ID)) + cmd.Println("Rules:") + for _, rule := range chain.Rules { + cmd.Println("\tStatus: " + rule.Status.String()) + cmd.Println("\tAny: " + strconv.FormatBool(rule.Any)) + cmd.Println("\tConditions:") + for _, c := range rule.Condition { + var ot string + switch c.Object { + case apechain.ObjectResource: + ot = "Resource" + case apechain.ObjectRequest: + ot = "Request" + default: + panic("unknown object type") + } + cmd.Println(fmt.Sprintf("\t\t%s %s %s %s", ot, c.Key, c.Op, c.Value)) + } + cmd.Println("\tActions:\tInverted:" + strconv.FormatBool(rule.Actions.Inverted)) + for _, name := range rule.Actions.Names { + cmd.Println("\t\t" + name) + } + cmd.Println("\tResources:\tInverted:" + strconv.FormatBool(rule.Resources.Inverted)) + for _, name := range rule.Resources.Names { + cmd.Println("\t\t" + name) + } + } +} + // ParseAPEChain parses APE chain rules. func ParseAPEChain(chain *apechain.Chain, rules []string) error { if len(rules) == 0 { From 419b07e8b881db1b351e2f5197a2543bda76208e Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 21 Dec 2023 12:16:55 +0300 Subject: [PATCH 0763/1943] [#883] adm: Use proxy contract as admin for frostfsid and policy contracts Signed-off-by: Alex Vanin --- cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 533fcad4d..436ba93d2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -546,7 +546,7 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an if found { items = append(items, h) } else { - items = append(items, nil) + items = append(items, c.Contracts[proxyContract].Hash) } case netmapContract: md := getDefaultNetmapContractConfigMap() @@ -582,7 +582,7 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an case proxyContract: items = nil case policyContract: - items = []any{nil} + items = append(items, c.Contracts[proxyContract].Hash) default: panic(fmt.Sprintf("invalid contract name: %s", ctrName)) } From ac0821a1a544bfeeecff1f4a0d32047b809be617 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 21 Dec 2023 12:26:47 +0300 Subject: [PATCH 0764/1943] [#883] adm: Support proxy accounts Signed-off-by: Alex Vanin --- .../internal/modules/morph/proxy.go | 68 +++++++++++++++++++ .../internal/modules/morph/root.go | 34 ++++++++++ 2 files changed, 102 insertions(+) create mode 100644 cmd/frostfs-adm/internal/modules/morph/proxy.go diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy.go new file mode 100644 index 000000000..effcf0e27 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/proxy.go @@ -0,0 +1,68 @@ +package morph + +import ( + "fmt" + + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/emit" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + accountAddressFlag = "account" +) + +func addProxyAccount(cmd *cobra.Command, _ []string) { + acc, _ := cmd.Flags().GetString(accountAddressFlag) + addr, err := address.StringToUint160(acc) + commonCmd.ExitOnErr(cmd, "invalid account: %w", err) + err = processAccount(cmd, addr, "addAccount") + commonCmd.ExitOnErr(cmd, "processing error: %w", err) +} + +func removeProxyAccount(cmd *cobra.Command, _ []string) { + acc, _ := cmd.Flags().GetString(accountAddressFlag) + addr, err := address.StringToUint160(acc) + commonCmd.ExitOnErr(cmd, "invalid account: %w", err) + err = processAccount(cmd, addr, "removeAccount") + commonCmd.ExitOnErr(cmd, "processing error: %w", err) +} + +func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error { + wCtx, err := newInitializeContext(cmd, viper.GetViper()) + if err != nil { + return fmt.Errorf("can't to initialize context: %w", err) + } + + r := management.NewReader(wCtx.ReadOnlyInvoker) + cs, err := r.GetContractByID(1) + if err != nil { + return fmt.Errorf("can't get NNS contract info: %w", err) + } + + proxyHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, proxyContract+".frostfs") + if err != nil { + return fmt.Errorf("can't get proxy contract hash: %w", err) + } + + bw := io.NewBufBinWriter() + emit.AppCall(bw.BinWriter, proxyHash, method, callflag.All, addr) + + if err := wCtx.sendConsensusTx(bw.Bytes()); err != nil { + return err + } + + if err = wCtx.awaitTx(); err != nil { + return err + } + + cmd.Println("Proxy contract has been updated") + + return nil +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 475bf4b06..75409421a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -241,6 +241,26 @@ var ( }, Run: listNetmapCandidatesNodes, } + + proxyAddAccountCmd = &cobra.Command{ + Use: "proxy-add-account", + Short: "Adds account to proxy contract", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: addProxyAccount, + } + + proxyRemoveAccountCmd = &cobra.Command{ + Use: "proxy-remove-account", + Short: "Remove from proxy contract", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: removeProxyAccount, + } ) func init() { @@ -267,6 +287,20 @@ func init() { initAddRuleChainCmd() initRemoveRuleChainCmd() initListRuleChainsCmd() + initProxyAddAccount() + initProxyRemoveAccount() +} + +func initProxyAddAccount() { + RootCmd.AddCommand(proxyAddAccountCmd) + proxyAddAccountCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + proxyAddAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") +} + +func initProxyRemoveAccount() { + RootCmd.AddCommand(proxyRemoveAccountCmd) + proxyRemoveAccountCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + proxyRemoveAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") } func initNetmapCandidatesCmd() { From b1eab1de54505281ee55eebaf23dd7f40d3cc53d Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 21 Dec 2023 13:59:46 +0300 Subject: [PATCH 0765/1943] [#883] adm: Add domainOf function to resolve NNS domain Signed-off-by: Alex Vanin --- cmd/frostfs-adm/internal/modules/morph/ape_util.go | 2 +- cmd/frostfs-adm/internal/modules/morph/balance.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/config.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/container.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/dump_hashes.go | 2 +- cmd/frostfs-adm/internal/modules/morph/epoch.go | 2 +- .../internal/modules/morph/initialize_deploy.go | 8 ++++++-- .../internal/modules/morph/netmap_candidates.go | 2 +- cmd/frostfs-adm/internal/modules/morph/proxy.go | 2 +- cmd/frostfs-adm/internal/modules/morph/remove_node.go | 2 +- 10 files changed, 18 insertions(+), 14 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape_util.go index e50522f2d..f64a7e108 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape_util.go @@ -96,7 +96,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *ac nnsCs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) - ch, err = nnsResolveHash(inv, nnsCs.Hash, policyContract+".frostfs") + ch, err = nnsResolveHash(inv, nnsCs.Hash, domainOf(policyContract)) commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err) return morph.NewContractStorage(ac, ch), ac diff --git a/cmd/frostfs-adm/internal/modules/morph/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance.go index 9b706c7bf..0fee45a8f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance.go @@ -63,7 +63,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err = nnsResolveHash(inv, nnsCs.Hash, netmapContract+".frostfs") + nmHash, err = nnsResolveHash(inv, nnsCs.Hash, domainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -136,7 +136,7 @@ func printStorageNodeBalances(cmd *cobra.Command, inv *invoker.Invoker, nmHash u } func printProxyContractBalance(cmd *cobra.Command, inv *invoker.Invoker, nnsHash util.Uint160) error { - h, err := nnsResolveHash(inv, nnsHash, proxyContract+".frostfs") + h, err := nnsResolveHash(inv, nnsHash, domainOf(proxyContract)) if err != nil { return fmt.Errorf("can't get hash of the proxy contract: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index 3c4029bca..db7d2ab8e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -37,7 +37,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := nnsResolveHash(inv, cs.Hash, netmapContract+".frostfs") + nmHash, err := nnsResolveHash(inv, cs.Hash, domainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -95,7 +95,7 @@ func setConfigCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, netmapContract+".frostfs") + nmHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/container.go b/cmd/frostfs-adm/internal/modules/morph/container.go index f3faa3044..e7a7f5466 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container.go @@ -35,7 +35,7 @@ func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker) (util.Ui if err != nil { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err = nnsResolveHash(inv, nnsCs.Hash, containerContract+".frostfs") + ch, err = nnsResolveHash(inv, nnsCs.Hash, domainOf(containerContract)) if err != nil { return util.Uint160{}, err } @@ -306,7 +306,7 @@ func fetchContainerContractHash(wCtx *initializeContext) (util.Uint160, error) { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err := nnsResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, containerContract+".frostfs") + ch, err := nnsResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, domainOf(containerContract)) if err != nil { return util.Uint160{}, fmt.Errorf("can't fetch container contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go index 585de6d6d..0a112f35c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go @@ -87,7 +87,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { for _, ctrName := range contractList { bw.Reset() emit.AppCall(bw.BinWriter, cs.Hash, "resolve", callflag.ReadOnly, - ctrName+".frostfs", int64(nns.TXT)) + domainOf(ctrName), int64(nns.TXT)) res, err := c.InvokeScript(bw.Bytes(), nil) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/epoch.go index 453e984b0..39dbb0caf 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/epoch.go @@ -27,7 +27,7 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, netmapContract+".frostfs") + nmHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 436ba93d2..61536b4d5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -91,6 +91,10 @@ const ( deployMethodName = "deploy" ) +func domainOf(contract string) string { + return contract + ".frostfs" +} + func (c *initializeContext) deployNNS(method string) error { cs := c.getContract(nnsContract) h := cs.Hash @@ -198,7 +202,7 @@ func (c *initializeContext) deployOrUpdateContracts(w *io2.BufBinWriter, nnsHash cs := c.getContract(ctrName) method := updateMethodName - ctrHash, err := nnsResolveHash(c.ReadOnlyInvoker, nnsHash, ctrName+".frostfs") + ctrHash, err := nnsResolveHash(c.ReadOnlyInvoker, nnsHash, domainOf(ctrName)) if err != nil { if errors.Is(err, errMissingNNSRecord) { // if contract not found we deploy it instead of update @@ -595,7 +599,7 @@ func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, if err != nil { return nil, fmt.Errorf("NNS is not yet deployed: %w", err) } - nmHash, err := nnsResolveHash(c.ReadOnlyInvoker, cs.Hash, netmapContract+".frostfs") + nmHash, err := nnsResolveHash(c.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) if err != nil { return nil, fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go index 30f7c69c4..bb1de09ca 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go @@ -20,7 +20,7 @@ func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) { cs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract info: %w", err) - nmHash, err := nnsResolveHash(inv, cs.Hash, netmapContract+".frostfs") + nmHash, err := nnsResolveHash(inv, cs.Hash, domainOf(netmapContract)) commonCmd.ExitOnErr(cmd, "can't get netmap contract hash: %w", err) res, err := inv.Call(nmHash, "netmapCandidates") diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy.go index effcf0e27..802a61246 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy.go @@ -46,7 +46,7 @@ func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error return fmt.Errorf("can't get NNS contract info: %w", err) } - proxyHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, proxyContract+".frostfs") + proxyHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(proxyContract)) if err != nil { return fmt.Errorf("can't get proxy contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/remove_node.go b/cmd/frostfs-adm/internal/modules/morph/remove_node.go index ba7bb90ff..ea958f2f7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/remove_node.go +++ b/cmd/frostfs-adm/internal/modules/morph/remove_node.go @@ -40,7 +40,7 @@ func removeNodesCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, netmapContract+".frostfs") + nmHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } From 825f65f79ee4c8934961a7b040f7d42a42f7e946 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 20 Dec 2023 16:20:36 +0300 Subject: [PATCH 0766/1943] [#873] node: Start metrics and pprof as soon as possible Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/httpcomponent.go | 21 +++++++++------------ cmd/frostfs-node/worker.go | 9 --------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/cmd/frostfs-node/httpcomponent.go b/cmd/frostfs-node/httpcomponent.go index 6e8fa8f2c..373a5e5c8 100644 --- a/cmd/frostfs-node/httpcomponent.go +++ b/cmd/frostfs-node/httpcomponent.go @@ -36,21 +36,19 @@ func (cmp *httpComponent) init(c *cfg) { cmp.shutdownDur, ), ) + c.wg.Add(1) + go func() { + defer c.wg.Done() + + c.log.Info(logs.FrostFSNodeStartListeningEndpoint, + zap.String("service", cmp.name), + zap.String("endpoint", cmp.address)) + fatalOnErr(srv.Serve()) + }() c.closers = append(c.closers, closer{ cmp.name, func() { stopAndLog(c, cmp.name, srv.Shutdown) }, }) - c.workers = append(c.workers, worker{ - cmp.name, - func(ctx context.Context) { - runAndLog(ctx, c, cmp.name, false, func(context.Context, *cfg) { - c.log.Info(logs.FrostFSNodeStartListeningEndpoint, - zap.String("service", cmp.name), - zap.String("endpoint", cmp.address)) - fatalOnErr(srv.Serve()) - }) - }, - }) } func (cmp *httpComponent) reload(ctx context.Context) error { @@ -64,7 +62,6 @@ func (cmp *httpComponent) reload(ctx context.Context) error { } // Cleanup delCloser(cmp.cfg, cmp.name) - delWorker(cmp.cfg, cmp.name) // Init server with new parameters cmp.init(cmp.cfg) // Start worker diff --git a/cmd/frostfs-node/worker.go b/cmd/frostfs-node/worker.go index bea235c48..c5649073b 100644 --- a/cmd/frostfs-node/worker.go +++ b/cmd/frostfs-node/worker.go @@ -30,15 +30,6 @@ func startWorker(ctx context.Context, c *cfg, wrk worker) { }(wrk) } -func delWorker(c *cfg, name string) { - for i, worker := range c.workers { - if worker.name == name { - c.workers = append(c.workers[:i], c.workers[i+1:]...) - return - } - } -} - func getWorker(c *cfg, name string) *worker { for _, wrk := range c.workers { if wrk.name == name { From eab981bf1a661cb4773ef53a57d5ae2324a211cd Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 21 Dec 2023 16:33:24 +0300 Subject: [PATCH 0767/1943] [#876] Fix linter error Signed-off-by: Anton Nikiforov --- pkg/services/control/types.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto index d1bc05e29..00fcb98d1 100644 --- a/pkg/services/control/types.proto +++ b/pkg/services/control/types.proto @@ -178,11 +178,11 @@ message ChainTarget { UNDEFINED = 0; NAMESPACE = 1; - + CONTAINER = 2; } TargetType type = 1; string Name = 2; -} \ No newline at end of file +} From 5b672fb3927e4a709d857a1da4f6deb72fd351ba Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 21 Dec 2023 16:33:37 +0300 Subject: [PATCH 0768/1943] [#876] adm: Add `morph ape get/set-admin` commands Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/internal/modules/morph/ape.go | 58 +++++++++++++++++++ .../internal/modules/morph/root.go | 4 ++ go.mod | 2 +- go.sum | 4 +- 4 files changed, 65 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape.go index 6dd2bd185..9f1d75adb 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape.go @@ -7,6 +7,7 @@ import ( parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -26,6 +27,8 @@ const ( targetNameDesc = "Resource name in APE resource name format" targetTypeFlag = "target-type" targetTypeDesc = "Resource type(container/namespace)" + addrAdminFlag = "addr" + addrAdminDesc = "The address of the admins wallet" ) var ( @@ -63,6 +66,26 @@ var ( }, Run: listRuleChains, } + + setAdminCmd = &cobra.Command{ + Use: "set-admin", + Short: "Set admin", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + }, + Run: setAdmin, + } + + getAdminCmd = &cobra.Command{ + Use: "get-admin", + Short: "Get admin", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + }, + Run: getAdmin, + } ) func initAddRuleChainCmd() { @@ -110,6 +133,22 @@ func initListRuleChainsCmd() { listRuleChainsCmd.Flags().Bool(jsonFlag, false, jsonFlagDesc) } +func initSetAdminCmd() { + apeCmd.AddCommand(setAdminCmd) + + setAdminCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + setAdminCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + setAdminCmd.Flags().String(addrAdminFlag, "", addrAdminDesc) + _ = setAdminCmd.MarkFlagRequired(addrAdminFlag) +} + +func initGetAdminCmd() { + apeCmd.AddCommand(getAdminCmd) + + getAdminCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + getAdminCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) +} + func addRuleChain(cmd *cobra.Command, _ []string) { chain := parseChain(cmd) target := parseTarget(cmd) @@ -151,6 +190,25 @@ func listRuleChains(cmd *cobra.Command, _ []string) { } } +func setAdmin(cmd *cobra.Command, _ []string) { + s, _ := cmd.Flags().GetString(addrAdminFlag) + addr, err := util.Uint160DecodeStringLE(s) + commonCmd.ExitOnErr(cmd, "can't decode admin addr: %w", err) + pci, ac := newPolicyContractInterface(cmd) + h, vub, err := pci.SetAdmin(addr) + cmd.Println("Waiting for transaction to persist...") + _, err = ac.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "can't set admin: %w", err) + cmd.Println("Admin set successfully") +} + +func getAdmin(cmd *cobra.Command, _ []string) { + pci, _ := newPolicyContractInterface(cmd) + addr, err := pci.GetAdmin() + commonCmd.ExitOnErr(cmd, "unable to get admin: %w", err) + cmd.Println(addr.StringLE()) +} + func prettyJSONFormat(cmd *cobra.Command, chains []*apechain.Chain) { wr := bytes.NewBufferString("") data, err := json.Marshal(chains) diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 75409421a..4176a6470 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -283,10 +283,14 @@ func init() { initRefillGasCmd() initDepositoryNotaryCmd() initNetmapCandidatesCmd() + RootCmd.AddCommand(apeCmd) initAddRuleChainCmd() initRemoveRuleChainCmd() initListRuleChainsCmd() + initSetAdminCmd() + initGetAdminCmd() + initProxyAddAccount() initProxyRemoveAccount() } diff --git a/go.mod b/go.mod index 6463c3b2b..2b8ffea31 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231214122253-62ea96b82ce3 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231221121354-ed93bb5cc574 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 diff --git a/go.sum b/go.sum index c18e291f2..56f4cba5a 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231214122253-62ea96b82ce3 h1:xnS/YalLqCRplvpF3E9/PRGevj4cDe7qFNZT0mkjZcs= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231214122253-62ea96b82ce3/go.mod h1:v43imcuSmDwSNrePe4UTQh8jaE8FmsiKN3FcaEzmRzc= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231221121354-ed93bb5cc574 h1:PmPaHc4P/Rjn1WQcDF5gcwILzWjkMOuRHLqzm9OZCdA= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231221121354-ed93bb5cc574/go.mod h1:ps6oKO0mxaPJzK3admTB3iwoBXKkHnS73n4PCrqpHBg= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= From 8180a0664f05b5adedf399cbd8ad90f0f37115aa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 22 Dec 2023 12:58:20 +0300 Subject: [PATCH 0769/1943] [#887] node: Drop badger writecache implementation Badger implementation isn't tested and works not well, but requires human resources to maintain. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-lens/internal/flags.go | 7 - .../internal/writecache/inspect.go | 32 +- cmd/frostfs-lens/internal/writecache/list.go | 30 +- cmd/frostfs-lens/internal/writecache/root.go | 1 - cmd/frostfs-node/config.go | 50 +-- .../config/engine/shard/writecache/config.go | 36 --- config/example/node.json | 1 - config/example/node.yaml | 1 - docs/storage-node-configuration.md | 2 - go.mod | 9 - go.sum | 74 ----- internal/logs/logs.go | 3 - .../engine/control_test.go | 10 +- .../shard/control_test.go | 10 +- pkg/local_object_storage/shard/range_test.go | 10 +- pkg/local_object_storage/shard/shard.go | 36 +-- pkg/local_object_storage/shard/shard_test.go | 22 +- .../shard/shutdown_test.go | 10 +- .../writecache/benchmark/writecache_test.go | 40 +-- .../{writecachebbolt => }/cachebbolt.go | 11 +- .../writecache/config/config.go | 22 -- .../{writecachebbolt => }/delete.go | 11 +- .../writecache/{writecachebbolt => }/flush.go | 13 +- .../flush.go => flush_test.go} | 110 ++++++- .../{writecachebbolt => }/generic_test.go | 2 +- .../writecache/{writecachebbolt => }/get.go | 11 +- .../{writecachebbolt => }/iterate.go | 2 +- .../writecache/{writecachebbolt => }/mode.go | 2 +- .../{writecachebbolt => }/options.go | 15 +- .../writecache/{writecachebbolt => }/put.go | 19 +- .../writecache/{writecachebbolt => }/state.go | 2 +- .../{writecachebbolt => }/storage.go | 7 +- .../writecache/{writecachebbolt => }/util.go | 2 +- .../writecachebadger/cachebadger.go | 138 --------- .../writecache/writecachebadger/delete.go | 70 ----- .../writecache/writecachebadger/flush.go | 286 ------------------ .../writecache/writecachebadger/flush_test.go | 69 ----- .../writecache/writecachebadger/gc.go | 45 --- .../writecachebadger/generic_test.go | 19 -- .../writecache/writecachebadger/get.go | 95 ------ .../writecache/writecachebadger/iterate.go | 32 -- .../writecache/writecachebadger/mode.go | 78 ----- .../writecache/writecachebadger/options.go | 119 -------- .../writecache/writecachebadger/put.go | 82 ----- .../writecache/writecachebadger/state.go | 67 ---- .../writecache/writecachebadger/storage.go | 91 ------ .../writecache/writecachebadger/util.go | 39 --- .../writecache/writecachebbolt/flush_test.go | 109 ------- 48 files changed, 202 insertions(+), 1750 deletions(-) rename pkg/local_object_storage/writecache/{writecachebbolt => }/cachebbolt.go (92%) delete mode 100644 pkg/local_object_storage/writecache/config/config.go rename pkg/local_object_storage/writecache/{writecachebbolt => }/delete.go (88%) rename pkg/local_object_storage/writecache/{writecachebbolt => }/flush.go (93%) rename pkg/local_object_storage/writecache/{writecachetest/flush.go => flush_test.go} (62%) rename pkg/local_object_storage/writecache/{writecachebbolt => }/generic_test.go (94%) rename pkg/local_object_storage/writecache/{writecachebbolt => }/get.go (90%) rename pkg/local_object_storage/writecache/{writecachebbolt => }/iterate.go (97%) rename pkg/local_object_storage/writecache/{writecachebbolt => }/mode.go (98%) rename pkg/local_object_storage/writecache/{writecachebbolt => }/options.go (92%) rename pkg/local_object_storage/writecache/{writecachebbolt => }/put.go (84%) rename pkg/local_object_storage/writecache/{writecachebbolt => }/state.go (98%) rename pkg/local_object_storage/writecache/{writecachebbolt => }/storage.go (93%) rename pkg/local_object_storage/writecache/{writecachebbolt => }/util.go (95%) delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/cachebadger.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/delete.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/flush.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/flush_test.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/gc.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/generic_test.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/get.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/iterate.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/mode.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/options.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/put.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/state.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/storage.go delete mode 100644 pkg/local_object_storage/writecache/writecachebadger/util.go delete mode 100644 pkg/local_object_storage/writecache/writecachebbolt/flush_test.go diff --git a/cmd/frostfs-lens/internal/flags.go b/cmd/frostfs-lens/internal/flags.go index 004519167..8a987a2d4 100644 --- a/cmd/frostfs-lens/internal/flags.go +++ b/cmd/frostfs-lens/internal/flags.go @@ -8,7 +8,6 @@ const ( flagAddress = "address" flagEnginePath = "path" flagOutFile = "out" - flagDBType = "dbtype" ) // AddAddressFlag adds the address flag to the passed cobra command. @@ -34,9 +33,3 @@ func AddOutputFileFlag(cmd *cobra.Command, v *string) { "File to save object payload") _ = cmd.MarkFlagFilename(flagOutFile) } - -// AddDBTypeFlag adds the DB type flag to the passed cobra command. -func AddDBTypeFlag(cmd *cobra.Command, v *string) { - cmd.Flags().StringVar(v, flagDBType, "bbolt", - "Type of DB used by write cache (default: bbolt)") -} diff --git a/cmd/frostfs-lens/internal/writecache/inspect.go b/cmd/frostfs-lens/internal/writecache/inspect.go index 1a733513b..afc986c8b 100644 --- a/cmd/frostfs-lens/internal/writecache/inspect.go +++ b/cmd/frostfs-lens/internal/writecache/inspect.go @@ -1,13 +1,10 @@ package writecache import ( - "fmt" "os" common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/spf13/cobra" ) @@ -23,34 +20,17 @@ func init() { common.AddAddressFlag(inspectCMD, &vAddress) common.AddComponentPathFlag(inspectCMD, &vPath) common.AddOutputFileFlag(inspectCMD, &vOut) - common.AddDBTypeFlag(inspectCMD, &vDBType) } func inspectFunc(cmd *cobra.Command, _ []string) { var data []byte - switch vDBType { - case "bbolt": - db, err := writecachebbolt.OpenDB(vPath, true, os.OpenFile) - common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) - defer db.Close() + db, err := writecache.OpenDB(vPath, true, os.OpenFile) + common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) + defer db.Close() - data, err = writecachebbolt.Get(db, []byte(vAddress)) - common.ExitOnErr(cmd, common.Errf("could not fetch object: %w", err)) - - case "badger": - log, err := logger.NewLogger(&logger.Prm{}) - common.ExitOnErr(cmd, common.Errf("could not create logger: %w", err)) - - db, err := writecachebadger.OpenDB(vPath, true, log) - common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) - - data, err = writecachebadger.Get(db, []byte(vAddress)) - common.ExitOnErr(cmd, common.Errf("could not fetch object: %w", err)) - - default: - common.ExitOnErr(cmd, fmt.Errorf("invalid dbtype: %q (possible values: bbolt, badger)", vDBType)) - } + data, err = writecache.Get(db, []byte(vAddress)) + common.ExitOnErr(cmd, common.Errf("could not fetch object: %w", err)) var o objectSDK.Object common.ExitOnErr(cmd, common.Errf("could not unmarshal object: %w", o.Unmarshal(data))) diff --git a/cmd/frostfs-lens/internal/writecache/list.go b/cmd/frostfs-lens/internal/writecache/list.go index df02a82f7..bcbae0ec9 100644 --- a/cmd/frostfs-lens/internal/writecache/list.go +++ b/cmd/frostfs-lens/internal/writecache/list.go @@ -6,9 +6,7 @@ import ( "os" common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" ) @@ -33,26 +31,10 @@ func listFunc(cmd *cobra.Command, _ []string) { return err } - switch vDBType { - case "bbolt": - db, err := writecachebbolt.OpenDB(vPath, true, os.OpenFile) - common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) - defer db.Close() + db, err := writecache.OpenDB(vPath, true, os.OpenFile) + common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) + defer db.Close() - err = writecachebbolt.IterateDB(db, wAddr) - common.ExitOnErr(cmd, common.Errf("write-cache iterator failure: %w", err)) - - case "badger": - log, err := logger.NewLogger(&logger.Prm{}) - common.ExitOnErr(cmd, common.Errf("could not create logger: %w", err)) - - db, err := writecachebadger.OpenDB(vPath, true, log) - common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) - - err = writecachebadger.IterateDB(db, wAddr) - common.ExitOnErr(cmd, common.Errf("write-cache iterator failure: %w", err)) - - default: - common.ExitOnErr(cmd, fmt.Errorf("invalid dbtype: %q (possible values: bbolt, badger)", vDBType)) - } + err = writecache.IterateDB(db, wAddr) + common.ExitOnErr(cmd, common.Errf("write-cache iterator failure: %w", err)) } diff --git a/cmd/frostfs-lens/internal/writecache/root.go b/cmd/frostfs-lens/internal/writecache/root.go index 11a8bb96b..eb3b325b6 100644 --- a/cmd/frostfs-lens/internal/writecache/root.go +++ b/cmd/frostfs-lens/internal/writecache/root.go @@ -8,7 +8,6 @@ var ( vAddress string vPath string vOut string - vDBType string ) // Root contains `write-cache` command definition. diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 82e203788..ffe49a7ef 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -41,9 +41,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" shardmode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" containerClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" @@ -138,7 +136,6 @@ type shardCfg struct { writecacheCfg struct { enabled bool - typ writecacheconfig.Type path string maxBatchSize int maxBatchDelay time.Duration @@ -147,7 +144,6 @@ type shardCfg struct { flushWorkerCount int sizeLimit uint64 noSync bool - gcInterval time.Duration } piloramaCfg struct { @@ -258,7 +254,6 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc := &newConfig.writecacheCfg wc.enabled = true - wc.typ = writeCacheCfg.Type() wc.path = writeCacheCfg.Path() wc.maxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize() wc.maxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay() @@ -267,7 +262,6 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc.flushWorkerCount = writeCacheCfg.WorkerCount() wc.sizeLimit = writeCacheCfg.SizeLimit() wc.noSync = writeCacheCfg.NoSync() - wc.gcInterval = writeCacheCfg.GCInterval() } } @@ -836,36 +830,20 @@ func (c *cfg) shardOpts() []shardOptsWithID { return shards } -func (c *cfg) getWriteCacheOpts(shCfg shardCfg) writecacheconfig.Options { - var writeCacheOpts writecacheconfig.Options +func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { + var writeCacheOpts []writecache.Option if wcRead := shCfg.writecacheCfg; wcRead.enabled { - switch wcRead.typ { - case writecacheconfig.TypeBBolt: - writeCacheOpts.Type = writecacheconfig.TypeBBolt - writeCacheOpts.BBoltOptions = append(writeCacheOpts.BBoltOptions, - writecachebbolt.WithPath(wcRead.path), - writecachebbolt.WithMaxBatchSize(wcRead.maxBatchSize), - writecachebbolt.WithMaxBatchDelay(wcRead.maxBatchDelay), - writecachebbolt.WithMaxObjectSize(wcRead.maxObjSize), - writecachebbolt.WithSmallObjectSize(wcRead.smallObjectSize), - writecachebbolt.WithFlushWorkersCount(wcRead.flushWorkerCount), - writecachebbolt.WithMaxCacheSize(wcRead.sizeLimit), - writecachebbolt.WithNoSync(wcRead.noSync), - writecachebbolt.WithLogger(c.log), - ) - case writecacheconfig.TypeBadger: - writeCacheOpts.Type = writecacheconfig.TypeBadger - writeCacheOpts.BadgerOptions = append(writeCacheOpts.BadgerOptions, - writecachebadger.WithPath(wcRead.path), - writecachebadger.WithMaxObjectSize(wcRead.maxObjSize), - writecachebadger.WithFlushWorkersCount(wcRead.flushWorkerCount), - writecachebadger.WithMaxCacheSize(wcRead.sizeLimit), - writecachebadger.WithLogger(c.log), - writecachebadger.WithGCInterval(wcRead.gcInterval), - ) - default: - panic(fmt.Sprintf("unknown writecache type: %q", wcRead.typ)) - } + writeCacheOpts = append(writeCacheOpts, + writecache.WithPath(wcRead.path), + writecache.WithMaxBatchSize(wcRead.maxBatchSize), + writecache.WithMaxBatchDelay(wcRead.maxBatchDelay), + writecache.WithMaxObjectSize(wcRead.maxObjSize), + writecache.WithSmallObjectSize(wcRead.smallObjectSize), + writecache.WithFlushWorkersCount(wcRead.flushWorkerCount), + writecache.WithMaxCacheSize(wcRead.sizeLimit), + writecache.WithNoSync(wcRead.noSync), + writecache.WithLogger(c.log), + ) } return writeCacheOpts } diff --git a/cmd/frostfs-node/config/engine/shard/writecache/config.go b/cmd/frostfs-node/config/engine/shard/writecache/config.go index d0fd4bf75..5e31e04ad 100644 --- a/cmd/frostfs-node/config/engine/shard/writecache/config.go +++ b/cmd/frostfs-node/config/engine/shard/writecache/config.go @@ -1,12 +1,8 @@ package writecacheconfig import ( - "fmt" - "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb" - writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" ) // Config is a wrapper over the config section @@ -25,9 +21,6 @@ const ( // SizeLimitDefault is a default write-cache size limit. SizeLimitDefault = 1 << 30 - - // DefaultGCInterval is the default duration of the GC cycle interval. - DefaultGCInterval = 1 * time.Minute ) // From wraps config section into Config. @@ -42,22 +35,6 @@ func (x *Config) Enabled() bool { return config.Bool((*config.Config)(x), "enabled") } -// Type returns the writecache implementation type to use. -// -// Panics if the type is not recognized. -func (x *Config) Type() writecacheconfig.Type { - t := config.String((*config.Config)(x), "type") - - switch t { - case "bbolt", "": - return writecacheconfig.TypeBBolt - case "badger": - return writecacheconfig.TypeBadger - } - - panic(fmt.Sprintf("invalid writecache type: %q", t)) -} - // Path returns the value of "path" config parameter. // // Panics if the value is not a non-empty string. @@ -149,16 +126,3 @@ func (x *Config) NoSync() bool { func (x *Config) BoltDB() *boltdbconfig.Config { return (*boltdbconfig.Config)(x) } - -// GCInterval returns the value of "gc_interval" config parameter. -// -// Returns DefaultGCInterval if the value is not a positive duration. -func (x *Config) GCInterval() time.Duration { - d := config.DurationSafe((*config.Config)(x), "gc_interval") - - if d > 0 { - return d - } - - return DefaultGCInterval -} diff --git a/config/example/node.json b/config/example/node.json index c803e11fb..8dda5e606 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -202,7 +202,6 @@ "resync_metabase": true, "writecache": { "enabled": true, - "type": "bbolt", "path": "tmp/1/cache", "memcache_capacity": 2147483648, "small_object_size": 16384, diff --git a/config/example/node.yaml b/config/example/node.yaml index d4c3516bc..171515164 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -127,7 +127,6 @@ storage: writecache: enabled: true - type: bbolt small_object_size: 16k # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes max_object_size: 134217728 # size threshold for "big" objects which bypass write-cache and go to the storage directly, bytes flush_worker_count: 30 # number of write-cache flusher threads diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 613be1288..e2334f8fb 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -277,7 +277,6 @@ metabase: ```yaml writecache: enabled: true - type: bbolt path: /path/to/writecache capacity: 4294967296 small_object_size: 16384 @@ -287,7 +286,6 @@ writecache: | Parameter | Type | Default value | Description | |----------------------|------------|---------------|----------------------------------------------------------------------------------------------------------------------| -| `type` | `string` | | Type of write cache backing implementation to use (`bbolt`, `badger`). | | `path` | `string` | | Path to the metabase file. | | `capacity` | `size` | unrestricted | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | | `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | diff --git a/go.mod b/go.mod index 2b8ffea31..c4266fe21 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 - github.com/dgraph-io/ristretto v0.1.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/uuid v1.4.0 github.com/hashicorp/golang-lru/v2 v2.0.7 @@ -44,18 +43,11 @@ require ( ) require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.2.0 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect go.mongodb.org/mongo-driver v1.11.4 // indirect - go.opencensus.io v0.24.0 // indirect ) require ( @@ -71,7 +63,6 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect - github.com/dgraph-io/badger/v4 v4.2.0 github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index 56f4cba5a..490cd4aec 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd h1:MxiIXu4vUATX9bNYFnM1dXAYkx5dnPOGCMzwNNXJEvM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231129062201-a1b61d394958 h1:X9yPizADIhD3K/gdKVCthlAnf9aQ3UJJGnZgIwwixRQ= @@ -17,7 +16,6 @@ git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9m git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= @@ -27,8 +25,6 @@ github.com/bits-and-blooms/bitset v1.11.0 h1:RMyy2mBBShArUAhfVRZJ2xyBO58KCBCtZFS github.com/bits-and-blooms/bitset v1.11.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= @@ -42,8 +38,6 @@ github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObk github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.2-0.20231117165148-e77308824822 h1:PvEjRgB/U4bv0jl9w65Wy9g0nIdkkW7vkNoR8Vq/als= @@ -57,19 +51,6 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= -github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= -github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= -github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= @@ -86,25 +67,14 @@ github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= -github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -112,20 +82,14 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= -github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -237,14 +201,12 @@ github.com/paulmach/orb v0.9.2/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/Enu github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= @@ -278,14 +240,12 @@ github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -311,8 +271,6 @@ go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= @@ -340,35 +298,24 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -376,7 +323,6 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -397,7 +343,6 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -412,10 +357,6 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -426,21 +367,11 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f h1:Vn+VyHU5guc9KjB5KrjI2q0wCOWEOIh0OEsleqakHJg= google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 h1:ZcOkrmX74HbKFYnpPY8Qsw93fC29TbJXspYKaBkSXDQ= google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4/go.mod h1:k2dtGpRrbsSyKcNPKKI5sstZkrNCZwpU/ns96JoHbGg= google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE= google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -448,10 +379,7 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= @@ -474,8 +402,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= diff --git a/internal/logs/logs.go b/internal/logs/logs.go index c3f4fdc7f..6ead83700 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -291,7 +291,6 @@ const ( ShardFailureToMarkLockersAsGarbage = "failure to mark lockers as garbage" ShardFailureToGetExpiredUnlockedObjects = "failure to get expired unlocked objects" ShardCouldNotMarkObjectToDeleteInMetabase = "could not mark object to delete in metabase" - WritecacheBadgerInitExperimental = "initializing badger-backed experimental writecache" WritecacheTriedToFlushItemsFromWritecache = "tried to flush items from write-cache" WritecacheWaitingForChannelsToFlush = "waiting for channels to flush" WritecacheFillingFlushMarksForObjectsInFSTree = "filling flush marks for objects in FSTree" @@ -300,8 +299,6 @@ const ( WritecacheFinishedUpdatingFlushMarks = "finished updating flush marks" WritecacheCantRemoveObjectsFromTheDatabase = "can't remove objects from the database" WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" - WritecacheDBValueLogGCRunCompleted = "value log GC run completed" - WritecacheBadgerObjAlreadyScheduled = "object already scheduled for flush" BlobovniczatreeCouldNotGetObjectFromLevel = "could not get object from level" BlobovniczatreeCouldNotReadPayloadRangeFromOpenedBlobovnicza = "could not read payload range from opened blobovnicza" BlobovniczatreeCouldNotReadPayloadRangeFromActiveBlobovnicza = "could not read payload range from active blobovnicza" diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 9cfb311ec..21652970b 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -20,8 +20,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" @@ -44,11 +43,8 @@ func TestInitializationFailure(t *testing.T) { storages, smallFileStorage, largeFileStorage := newTestStorages(t.TempDir(), 1<<20) - wcOpts := writecacheconfig.Options{ - Type: writecacheconfig.TypeBBolt, - BBoltOptions: []writecachebbolt.Option{ - writecachebbolt.WithPath(t.TempDir()), - }, + wcOpts := []writecache.Option{ + writecache.WithPath(t.TempDir()), } return []shard.Option{ diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index f08af6fc7..8a650db0c 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -17,8 +17,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -58,11 +57,8 @@ func TestShardOpen(t *testing.T) { return nil, fs.ErrPermission } - wcOpts := writecacheconfig.Options{ - Type: writecacheconfig.TypeBBolt, - BBoltOptions: []writecachebbolt.Option{ - writecachebbolt.WithPath(filepath.Join(dir, "wc")), - }, + wcOpts := []writecache.Option{ + writecache.WithPath(filepath.Join(dir, "wc")), } newShard := func() *Shard { diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 098a584ad..35dc0c6a5 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -12,8 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -69,11 +68,8 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { testCase{true, "object in write-cache, out of range, big offset", 100, newRange(101, math.MaxUint64-10)}) } - wcOpts := writecacheconfig.Options{ - Type: writecacheconfig.TypeBBolt, - BBoltOptions: []writecachebbolt.Option{ - writecachebbolt.WithMaxObjectSize(writeCacheMaxSize), - }, + wcOpts := []writecache.Option{ + writecache.WithMaxObjectSize(writeCacheMaxSize), } sh := newCustomShard(t, hasWriteCache, shardOptions{ diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 56c6aee1f..641b487ed 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -2,7 +2,6 @@ package shard import ( "context" - "fmt" "sync" "sync/atomic" "time" @@ -13,9 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -108,7 +104,7 @@ type cfg struct { metaOpts []meta.Option - writeCacheOpts writecacheconfig.Options + writeCacheOpts []writecache.Option piloramaOpts []pilorama.Option @@ -166,22 +162,11 @@ func New(opts ...Option) *Shard { s.blobStor.SetReportErrorFunc(reportFunc) if c.useWriteCache { - switch c.writeCacheOpts.Type { - case writecacheconfig.TypeBBolt: - s.writeCache = writecachebbolt.New( - append(c.writeCacheOpts.BBoltOptions, - writecachebbolt.WithReportErrorFunc(reportFunc), - writecachebbolt.WithBlobstor(bs), - writecachebbolt.WithMetabase(mb))...) - case writecacheconfig.TypeBadger: - s.writeCache = writecachebadger.New( - append(c.writeCacheOpts.BadgerOptions, - writecachebadger.WithReportErrorFunc(reportFunc), - writecachebadger.WithBlobstor(bs), - writecachebadger.WithMetabase(mb))...) - default: - panic(fmt.Sprintf("invalid writecache type: %v", c.writeCacheOpts.Type)) - } + s.writeCache = writecache.New( + append(c.writeCacheOpts, + writecache.WithReportErrorFunc(reportFunc), + writecache.WithBlobstor(bs), + writecache.WithMetabase(mb))...) } if s.piloramaOpts != nil { @@ -215,7 +200,7 @@ func WithMetaBaseOptions(opts ...meta.Option) Option { } // WithWriteCacheOptions returns option to set internal write cache options. -func WithWriteCacheOptions(opts writecacheconfig.Options) Option { +func WithWriteCacheOptions(opts []writecache.Option) Option { return func(c *cfg) { c.writeCacheOpts = opts } @@ -224,12 +209,7 @@ func WithWriteCacheOptions(opts writecacheconfig.Options) Option { // WithWriteCacheMetrics returns an option to set the metrics register used by the write cache. func WithWriteCacheMetrics(wcMetrics writecache.Metrics) Option { return func(c *cfg) { - switch c.writeCacheOpts.Type { - case writecacheconfig.TypeBBolt: - c.writeCacheOpts.BBoltOptions = append(c.writeCacheOpts.BBoltOptions, writecachebbolt.WithMetrics(wcMetrics)) - case writecacheconfig.TypeBadger: - c.writeCacheOpts.BadgerOptions = append(c.writeCacheOpts.BadgerOptions, writecachebadger.WithMetrics(wcMetrics)) - } + c.writeCacheOpts = append(c.writeCacheOpts, writecache.WithMetrics(wcMetrics)) } } diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 811c7f3ab..807633cd5 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -11,9 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" - writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -33,7 +31,7 @@ func (s epochState) CurrentEpoch() uint64 { type shardOptions struct { rootPath string dontRelease bool - wcOpts writecacheconfig.Options + wcOpts []writecache.Option bsOpts []blobstor.Option metaOptions []meta.Option @@ -48,22 +46,12 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard if o.rootPath == "" { o.rootPath = t.TempDir() } - if enableWriteCache && o.wcOpts.Type == 0 { - o.wcOpts.Type = writecacheconfig.TypeBBolt - } var sh *Shard if enableWriteCache { - switch o.wcOpts.Type { - case writecacheconfig.TypeBBolt: - o.wcOpts.BBoltOptions = append( - []writecachebbolt.Option{writecachebbolt.WithPath(filepath.Join(o.rootPath, "wcache"))}, - o.wcOpts.BBoltOptions...) - case writecacheconfig.TypeBadger: - o.wcOpts.BadgerOptions = append( - []writecachebadger.Option{writecachebadger.WithPath(filepath.Join(o.rootPath, "wcache"))}, - o.wcOpts.BadgerOptions...) - } + o.wcOpts = append( + []writecache.Option{writecache.WithPath(filepath.Join(o.rootPath, "wcache"))}, + o.wcOpts...) } if o.bsOpts == nil { diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index 163c3a4ae..b94ea50dc 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -7,8 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" @@ -36,11 +35,8 @@ func TestWriteCacheObjectLoss(t *testing.T) { } dir := t.TempDir() - wcOpts := writecacheconfig.Options{ - Type: writecacheconfig.TypeBBolt, - BBoltOptions: []writecachebbolt.Option{ - writecachebbolt.WithMaxObjectSize(smallSize * 2), - }, + wcOpts := []writecache.Option{ + writecache.WithMaxObjectSize(smallSize * 2), } sh := newCustomShard(t, true, shardOptions{dontRelease: true, rootPath: dir, wcOpts: wcOpts}) diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index ea4bf0d33..727d0fc7a 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -3,35 +3,26 @@ package benchmark import ( "context" "testing" - "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" "github.com/stretchr/testify/require" ) func BenchmarkWritecacheSeq(b *testing.B) { const payloadSize = 8 << 10 b.Run("bbolt_seq", func(b *testing.B) { - benchmarkPutSeq(b, newBBoltCache(b), payloadSize) - }) - b.Run("badger_seq", func(b *testing.B) { - benchmarkPutSeq(b, newBadgerCache(b), payloadSize) + benchmarkPutSeq(b, newCache(b), payloadSize) }) } func BenchmarkWritecachePar(b *testing.B) { const payloadSize = 8 << 10 b.Run("bbolt_par", func(b *testing.B) { - benchmarkPutPar(b, newBBoltCache(b), payloadSize) - }) - b.Run("badger_par", func(b *testing.B) { - benchmarkPutPar(b, newBadgerCache(b), payloadSize) + benchmarkPutPar(b, newCache(b), payloadSize) }) } @@ -95,28 +86,15 @@ func (testMetabase) UpdateStorageID(context.Context, meta.UpdateStorageIDPrm) (m return meta.UpdateStorageIDRes{}, nil } -func newBBoltCache(b *testing.B) writecache.Cache { +func newCache(b *testing.B) writecache.Cache { bs := teststore.New( teststore.WithPut(func(pp common.PutPrm) (common.PutRes, error) { return common.PutRes{}, nil }), ) - return writecachebbolt.New( - writecachebbolt.WithPath(b.TempDir()), - writecachebbolt.WithBlobstor(bs), - writecachebbolt.WithMetabase(testMetabase{}), - writecachebbolt.WithMaxCacheSize(256<<30), - writecachebbolt.WithSmallObjectSize(128<<10), - ) -} - -func newBadgerCache(b *testing.B) writecache.Cache { - bs := teststore.New( - teststore.WithPut(func(pp common.PutPrm) (common.PutRes, error) { return common.PutRes{}, nil }), - ) - return writecachebadger.New( - writecachebadger.WithPath(b.TempDir()), - writecachebadger.WithBlobstor(bs), - writecachebadger.WithMetabase(testMetabase{}), - writecachebadger.WithMaxCacheSize(256<<30), - writecachebadger.WithGCInterval(10*time.Second), + return writecache.New( + writecache.WithPath(b.TempDir()), + writecache.WithBlobstor(bs), + writecache.WithMetabase(testMetabase{}), + writecache.WithMaxCacheSize(256<<30), + writecache.WithSmallObjectSize(128<<10), ) } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go b/pkg/local_object_storage/writecache/cachebbolt.go similarity index 92% rename from pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go rename to pkg/local_object_storage/writecache/cachebbolt.go index 9bbf5b7df..5eeda33a6 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/cachebbolt.go +++ b/pkg/local_object_storage/writecache/cachebbolt.go @@ -1,4 +1,4 @@ -package writecachebbolt +package writecache import ( "context" @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.etcd.io/bbolt" @@ -58,7 +57,7 @@ const ( var defaultBucket = []byte{0} // New creates new writecache instance. -func New(opts ...Option) writecache.Cache { +func New(opts ...Option) Cache { c := &cache{ flushCh: make(chan objectInfo), mode: mode.ReadWrite, @@ -73,7 +72,7 @@ func New(opts ...Option) writecache.Cache { maxBatchSize: bbolt.DefaultMaxBatchSize, maxBatchDelay: bbolt.DefaultMaxBatchDelay, openFile: os.OpenFile, - metrics: writecache.DefaultMetrics(), + metrics: DefaultMetrics(), }, } @@ -89,8 +88,8 @@ func (c *cache) SetLogger(l *logger.Logger) { c.log = l } -func (c *cache) DumpInfo() writecache.Info { - return writecache.Info{ +func (c *cache) DumpInfo() Info { + return Info{ Path: c.path, } } diff --git a/pkg/local_object_storage/writecache/config/config.go b/pkg/local_object_storage/writecache/config/config.go deleted file mode 100644 index 91f097e17..000000000 --- a/pkg/local_object_storage/writecache/config/config.go +++ /dev/null @@ -1,22 +0,0 @@ -// Package config provides the common configuration options for write cache implementations. -package config - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt" -) - -// Type is the write cache implementation type. -type Type int - -const ( - TypeBBolt Type = iota - TypeBadger -) - -// Options are the configuration options for the write cache. -type Options struct { - Type Type - BBoltOptions []writecachebbolt.Option - BadgerOptions []writecachebadger.Option -} diff --git a/pkg/local_object_storage/writecache/writecachebbolt/delete.go b/pkg/local_object_storage/writecache/delete.go similarity index 88% rename from pkg/local_object_storage/writecache/writecachebbolt/delete.go rename to pkg/local_object_storage/writecache/delete.go index 15c83eed1..0a4f4d658 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -1,4 +1,4 @@ -package writecachebbolt +package writecache import ( "context" @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -27,7 +26,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { defer span.End() deleted := false - storageType := writecache.StorageTypeUndefined + storageType := StorageTypeUndefined startedAt := time.Now() defer func() { c.metrics.Delete(time.Since(startedAt), deleted, storageType) @@ -36,7 +35,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { - return writecache.ErrReadOnly + return ErrReadOnly } saddr := addr.EncodeToString() @@ -49,7 +48,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { }) if dataSize > 0 { - storageType = writecache.StorageTypeDB + storageType = StorageTypeDB var recordDeleted bool err := c.db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) @@ -74,7 +73,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { return nil } - storageType = writecache.StorageTypeFSTree + storageType = StorageTypeFSTree _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err == nil { storagelog.Write(c.log, diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush.go b/pkg/local_object_storage/writecache/flush.go similarity index 93% rename from pkg/local_object_storage/writecache/writecachebbolt/flush.go rename to pkg/local_object_storage/writecache/flush.go index d73e374f5..f4ceec8c8 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -1,4 +1,4 @@ -package writecachebbolt +package writecache import ( "bytes" @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -195,7 +194,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - err = c.flushObject(ctx, &obj, e.ObjectData, writecache.StorageTypeFSTree) + err = c.flushObject(ctx, &obj, e.ObjectData, StorageTypeFSTree) if err != nil { if ignoreErrors { return nil @@ -224,7 +223,7 @@ func (c *cache) workerFlushSmall(ctx context.Context) { return } - err := c.flushObject(ctx, objInfo.obj, objInfo.data, writecache.StorageTypeDB) + err := c.flushObject(ctx, objInfo.obj, objInfo.data, StorageTypeDB) if err != nil { // Error is handled in flushObject. continue @@ -235,7 +234,7 @@ func (c *cache) workerFlushSmall(ctx context.Context) { } // flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []byte, st writecache.StorageType) error { +func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []byte, st StorageType) error { var err error defer func() { @@ -274,7 +273,7 @@ func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []b // Write-cache must be in readonly mode to ensure correctness of an operation and // to prevent interference with background flush workers. func (c *cache) Flush(ctx context.Context, ignoreErrors bool) error { - ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Flush", + ctx, span := tracing.StartSpanFromContext(ctx, "Flush", trace.WithAttributes( attribute.Bool("ignore_errors", ignoreErrors), )) @@ -315,7 +314,7 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { return err } - if err := c.flushObject(ctx, &obj, data, writecache.StorageTypeDB); err != nil { + if err := c.flushObject(ctx, &obj, data, StorageTypeDB); err != nil { return err } } diff --git a/pkg/local_object_storage/writecache/writecachetest/flush.go b/pkg/local_object_storage/writecache/flush_test.go similarity index 62% rename from pkg/local_object_storage/writecache/writecachetest/flush.go rename to pkg/local_object_storage/writecache/flush_test.go index 2c495df52..20db1de95 100644 --- a/pkg/local_object_storage/writecache/writecachetest/flush.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -1,7 +1,8 @@ -package writecachetest +package writecache import ( "context" + "os" "path/filepath" "sync/atomic" "testing" @@ -13,12 +14,103 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" + "go.uber.org/zap" ) +func TestFlush(t *testing.T) { + testlogger := test.NewLogger(t, true) + + createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs MainStorage, opts ...Option) Cache { + return New( + append([]Option{ + WithLogger(testlogger), + WithPath(filepath.Join(t.TempDir(), "writecache")), + WithSmallObjectSize(smallSize), + WithMetabase(mb), + WithBlobstor(bs), + WithDisableBackgroundFlush(), + }, opts...)...) + } + + errCountOpt := func() (Option, *atomic.Uint32) { + cnt := &atomic.Uint32{} + return WithReportErrorFunc(func(msg string, err error) { + cnt.Add(1) + testlogger.Warn(msg, zap.Uint32("error_count", cnt.Load()), zap.Error(err)) + }), cnt + } + + failures := []TestFailureInjector[Option]{ + { + Desc: "db, invalid address", + InjectFn: func(t *testing.T, wc Cache) { + c := wc.(*cache) + obj := testutil.GenerateObject() + data, err := obj.Marshal() + require.NoError(t, err) + require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error { + b := tx.Bucket(defaultBucket) + return b.Put([]byte{1, 2, 3}, data) + })) + }, + }, + { + Desc: "db, invalid object", + InjectFn: func(t *testing.T, wc Cache) { + c := wc.(*cache) + require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error { + b := tx.Bucket(defaultBucket) + k := []byte(oidtest.Address().EncodeToString()) + v := []byte{1, 2, 3} + return b.Put(k, v) + })) + }, + }, + { + Desc: "fs, read error", + InjectFn: func(t *testing.T, wc Cache) { + c := wc.(*cache) + obj := testutil.GenerateObject() + data, err := obj.Marshal() + require.NoError(t, err) + + var prm common.PutPrm + prm.Address = objectCore.AddressOf(obj) + prm.RawData = data + + _, err = c.fsTree.Put(context.Background(), prm) + require.NoError(t, err) + + p := prm.Address.Object().EncodeToString() + "." + prm.Address.Container().EncodeToString() + p = filepath.Join(c.fsTree.RootPath, p[:1], p[1:]) + + _, err = os.Stat(p) // sanity check + require.NoError(t, err) + require.NoError(t, os.Truncate(p, 0)) // corrupt the file contents, so that it can't be unmarshalled + }, + }, + { + Desc: "fs, invalid object", + InjectFn: func(t *testing.T, wc Cache) { + c := wc.(*cache) + var prm common.PutPrm + prm.Address = oidtest.Address() + prm.RawData = []byte{1, 2, 3} + _, err := c.fsTree.Put(context.Background(), prm) + require.NoError(t, err) + }, + }, + } + + runFlushTest(t, createCacheFn, errCountOpt, failures...) +} + const ( objCount = 4 smallSize = 256 @@ -28,13 +120,13 @@ type CreateCacheFunc[Option any] func( t *testing.T, smallSize uint64, meta *meta.DB, - bs writecache.MainStorage, + bs MainStorage, opts ...Option, -) writecache.Cache +) Cache type TestFailureInjector[Option any] struct { Desc string - InjectFn func(*testing.T, writecache.Cache) + InjectFn func(*testing.T, Cache) } type objectPair struct { @@ -42,7 +134,7 @@ type objectPair struct { obj *objectSDK.Object } -func TestFlush[Option any]( +func runFlushTest[Option any]( t *testing.T, createCacheFn CreateCacheFunc[Option], errCountOption func() (Option, *atomic.Uint32), @@ -105,7 +197,7 @@ func newCache[Option any]( createCacheFn CreateCacheFunc[Option], smallSize uint64, opts ...Option, -) (writecache.Cache, *blobstor.BlobStor, *meta.DB) { +) (Cache, *blobstor.BlobStor, *meta.DB) { dir := t.TempDir() mb := meta.New( meta.WithPath(filepath.Join(dir, "meta")), @@ -136,7 +228,7 @@ func newCache[Option any]( return wc, bs, mb } -func putObject(t *testing.T, c writecache.Cache, size int) objectPair { +func putObject(t *testing.T, c Cache, size int) objectPair { obj := testutil.GenerateObjectWithSize(size) data, err := obj.Marshal() require.NoError(t, err) @@ -152,7 +244,7 @@ func putObject(t *testing.T, c writecache.Cache, size int) objectPair { return objectPair{prm.Address, prm.Object} } -func putObjects(t *testing.T, c writecache.Cache) []objectPair { +func putObjects(t *testing.T, c Cache) []objectPair { objects := make([]objectPair, objCount) for i := range objects { objects[i] = putObject(t, c, 1+(i%2)*smallSize) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go b/pkg/local_object_storage/writecache/generic_test.go similarity index 94% rename from pkg/local_object_storage/writecache/writecachebbolt/generic_test.go rename to pkg/local_object_storage/writecache/generic_test.go index 7eadd1afc..a6d9e4799 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/generic_test.go +++ b/pkg/local_object_storage/writecache/generic_test.go @@ -1,4 +1,4 @@ -package writecachebbolt +package writecache import ( "testing" diff --git a/pkg/local_object_storage/writecache/writecachebbolt/get.go b/pkg/local_object_storage/writecache/get.go similarity index 90% rename from pkg/local_object_storage/writecache/writecachebbolt/get.go rename to pkg/local_object_storage/writecache/get.go index 838b207be..d2496e447 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -1,4 +1,4 @@ -package writecachebbolt +package writecache import ( "bytes" @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -36,7 +35,7 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) (*objectSDK.Object, error) { found := false - storageType := writecache.StorageTypeUndefined + storageType := StorageTypeUndefined startedAt := time.Now() defer func() { c.metrics.Get(time.Since(startedAt), found, storageType) @@ -46,7 +45,7 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) if err == nil { obj := objectSDK.New() found = true - storageType = writecache.StorageTypeDB + storageType = StorageTypeDB return obj, obj.Unmarshal(value) } @@ -56,7 +55,7 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) } found = true - storageType = writecache.StorageTypeFSTree + storageType = StorageTypeFSTree return res.Object, nil } @@ -66,7 +65,7 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { saddr := addr.EncodeToString() - ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Head", + ctx, span := tracing.StartSpanFromContext(ctx, "Head", trace.WithAttributes( attribute.String("address", saddr), )) diff --git a/pkg/local_object_storage/writecache/writecachebbolt/iterate.go b/pkg/local_object_storage/writecache/iterate.go similarity index 97% rename from pkg/local_object_storage/writecache/writecachebbolt/iterate.go rename to pkg/local_object_storage/writecache/iterate.go index 530db42a6..5349c069c 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/iterate.go +++ b/pkg/local_object_storage/writecache/iterate.go @@ -1,4 +1,4 @@ -package writecachebbolt +package writecache import ( "errors" diff --git a/pkg/local_object_storage/writecache/writecachebbolt/mode.go b/pkg/local_object_storage/writecache/mode.go similarity index 98% rename from pkg/local_object_storage/writecache/writecachebbolt/mode.go rename to pkg/local_object_storage/writecache/mode.go index b187996a1..e3ff2286b 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -1,4 +1,4 @@ -package writecachebbolt +package writecache import ( "context" diff --git a/pkg/local_object_storage/writecache/writecachebbolt/options.go b/pkg/local_object_storage/writecache/options.go similarity index 92% rename from pkg/local_object_storage/writecache/writecachebbolt/options.go rename to pkg/local_object_storage/writecache/options.go index 3ea329192..c8eb1bc45 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -1,11 +1,10 @@ -package writecachebbolt +package writecache import ( "io/fs" "os" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -18,9 +17,9 @@ type options struct { // path is a path to a directory for write-cache. path string // blobstor is the main persistent storage. - blobstor writecache.MainStorage + blobstor MainStorage // metabase is the metabase instance. - metabase writecache.Metabase + metabase Metabase // maxObjectSize is the maximum size of the object stored in the write-cache. maxObjectSize uint64 // smallObjectSize is the maximum size of the object stored in the database. @@ -43,7 +42,7 @@ type options struct { // openFile is the function called internally by bbolt to open database files. Useful for hermetic testing. openFile func(string, int, fs.FileMode) (*os.File, error) // metrics is metrics implementation - metrics writecache.Metrics + metrics Metrics // disableBackgroundFlush is for testing purposes only. disableBackgroundFlush bool } @@ -63,14 +62,14 @@ func WithPath(path string) Option { } // WithBlobstor sets main object storage. -func WithBlobstor(bs writecache.MainStorage) Option { +func WithBlobstor(bs MainStorage) Option { return func(o *options) { o.blobstor = bs } } // WithMetabase sets metabase. -func WithMetabase(db writecache.Metabase) Option { +func WithMetabase(db Metabase) Option { return func(o *options) { o.metabase = db } @@ -152,7 +151,7 @@ func WithOpenFile(f func(string, int, fs.FileMode) (*os.File, error)) Option { } // WithMetrics sets metrics implementation. -func WithMetrics(metrics writecache.Metrics) Option { +func WithMetrics(metrics Metrics) Option { return func(o *options) { o.metrics = metrics } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/put.go b/pkg/local_object_storage/writecache/put.go similarity index 84% rename from pkg/local_object_storage/writecache/writecachebbolt/put.go rename to pkg/local_object_storage/writecache/put.go index 63fa544ed..6fc655c65 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -1,27 +1,18 @@ -package writecachebbolt +package writecache import ( "context" - "errors" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) -var ( - // ErrBigObject is returned when object is too big to be placed in cache. - ErrBigObject = errors.New("too big object") - // ErrOutOfSpace is returned when there is no space left to put a new object. - ErrOutOfSpace = errors.New("no space left in the write cache") -) - // Put puts object to write-cache. // // Returns ErrReadOnly if write-cache is in R/O mode. @@ -38,7 +29,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro startedAt := time.Now() added := false - storageType := writecache.StorageTypeUndefined + storageType := StorageTypeUndefined defer func() { c.metrics.Put(time.Since(startedAt), added, storageType) }() @@ -46,7 +37,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro c.modeMtx.RLock() defer c.modeMtx.RUnlock() if c.readOnly() { - return common.PutRes{}, writecache.ErrReadOnly + return common.PutRes{}, ErrReadOnly } sz := uint64(len(prm.RawData)) @@ -61,7 +52,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro } if sz <= c.smallObjectSize { - storageType = writecache.StorageTypeDB + storageType = StorageTypeDB err := c.putSmall(oi) if err == nil { added = true @@ -69,7 +60,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro return common.PutRes{}, err } - storageType = writecache.StorageTypeFSTree + storageType = StorageTypeFSTree err := c.putBig(ctx, oi.addr, prm) if err == nil { added = true diff --git a/pkg/local_object_storage/writecache/writecachebbolt/state.go b/pkg/local_object_storage/writecache/state.go similarity index 98% rename from pkg/local_object_storage/writecache/writecachebbolt/state.go rename to pkg/local_object_storage/writecache/state.go index 9261b2606..bc75aaf27 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -1,4 +1,4 @@ -package writecachebbolt +package writecache import ( "fmt" diff --git a/pkg/local_object_storage/writecache/writecachebbolt/storage.go b/pkg/local_object_storage/writecache/storage.go similarity index 93% rename from pkg/local_object_storage/writecache/writecachebbolt/storage.go rename to pkg/local_object_storage/writecache/storage.go index 7a5030625..5c25a3b33 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -1,4 +1,4 @@ -package writecachebbolt +package writecache import ( "context" @@ -10,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -78,7 +77,7 @@ func (c *cache) deleteFromDB(key string) { }) if err == nil { - c.metrics.Evict(writecache.StorageTypeDB) + c.metrics.Evict(StorageTypeDB) storagelog.Write(c.log, storagelog.AddressField(key), storagelog.StorageTypeField(wcStorageType), @@ -103,7 +102,7 @@ func (c *cache) deleteFromDisk(ctx context.Context, addr oid.Address) { storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree DELETE"), ) - c.metrics.Evict(writecache.StorageTypeFSTree) + c.metrics.Evict(StorageTypeFSTree) // counter changed by fstree c.estimateCacheSize() } diff --git a/pkg/local_object_storage/writecache/writecachebbolt/util.go b/pkg/local_object_storage/writecache/util.go similarity index 95% rename from pkg/local_object_storage/writecache/writecachebbolt/util.go rename to pkg/local_object_storage/writecache/util.go index fe225583c..0ed4a954e 100644 --- a/pkg/local_object_storage/writecache/writecachebbolt/util.go +++ b/pkg/local_object_storage/writecache/util.go @@ -1,4 +1,4 @@ -package writecachebbolt +package writecache import ( "io/fs" diff --git a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go b/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go deleted file mode 100644 index 484f01815..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/cachebadger.go +++ /dev/null @@ -1,138 +0,0 @@ -package writecachebadger - -import ( - "context" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.uber.org/zap" -) - -type cache struct { - options - - mode mode.Mode - modeMtx sync.RWMutex - - // flushCh is a channel with objects to flush. - flushCh chan objectInfo - // scheduled4Flush contains objects scheduled for flush via flushCh - // helps to avoid multiple flushing of one object - scheduled4Flush map[oid.Address]struct{} - scheduled4FlushMtx sync.RWMutex - // wg is a wait group for flush workers. - wg sync.WaitGroup - // store contains underlying database. - store - // cancel is cancel function, protected by modeMtx in Close. - cancel func() -} - -// wcStorageType is used for write-cache operations logging. -const wcStorageType = "write-cache" - -type objectInfo struct { - addr oid.Address - data []byte - obj *objectSDK.Object -} - -const ( - defaultMaxObjectSize = 64 << 20 // 64 MiB - defaultSmallObjectSize = 32 << 10 // 32 KiB - defaultMaxCacheSize = 1 << 30 // 1 GiB -) - -// New creates new writecache instance. -func New(opts ...Option) writecache.Cache { - c := &cache{ - flushCh: make(chan objectInfo), - mode: mode.ReadWrite, - scheduled4Flush: map[oid.Address]struct{}{}, - - options: options{ - log: &logger.Logger{Logger: zap.NewNop()}, - maxObjectSize: defaultMaxObjectSize, - workersCount: defaultFlushWorkersCount, - maxCacheSize: defaultMaxCacheSize, - metrics: writecache.DefaultMetrics(), - }, - } - - for i := range opts { - opts[i](&c.options) - } - - return c -} - -// SetLogger sets logger. It is used after the shard ID was generated to use it in logs. -func (c *cache) SetLogger(l *logger.Logger) { - c.log = l -} - -func (c *cache) DumpInfo() writecache.Info { - return writecache.Info{ - Path: c.path, - } -} - -// Open opens and initializes database. Reads object counters from the ObjectCounters instance. -func (c *cache) Open(_ context.Context, readOnly bool) error { - c.modeMtx.Lock() - defer c.modeMtx.Unlock() - - err := c.openStore(readOnly) - if err != nil { - return metaerr.Wrap(err) - } - return metaerr.Wrap(c.initCounters()) -} - -// Init runs necessary services. -func (c *cache) Init() error { - c.modeMtx.Lock() - defer c.modeMtx.Unlock() - - c.log.Info(logs.WritecacheBadgerInitExperimental) - c.metrics.SetMode(c.mode) - ctx, cancel := context.WithCancel(context.Background()) - c.cancel = cancel - c.runFlushLoop(ctx) - c.runGCLoop(ctx) - return nil -} - -// Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. -func (c *cache) Close() error { - // We cannot lock mutex for the whole operation duration - // because it is taken by some background workers, so `wg.Wait()` is done without modeMtx. - c.modeMtx.Lock() - if c.cancel != nil { - c.cancel() - c.cancel = nil - } - c.mode = mode.DegradedReadOnly // prevent new operations from being processed - c.modeMtx.Unlock() - - c.wg.Wait() - - c.modeMtx.Lock() - defer c.modeMtx.Unlock() - - var err error - if c.db != nil { - err = c.db.Close() - if err != nil { - c.db = nil - } - } - c.metrics.Close() - return nil -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/delete.go b/pkg/local_object_storage/writecache/writecachebadger/delete.go deleted file mode 100644 index f96bf2707..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/delete.go +++ /dev/null @@ -1,70 +0,0 @@ -package writecachebadger - -import ( - "context" - "time" - - storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/dgraph-io/badger/v4" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -// Delete removes object from write-cache. -// -// Returns an error of type apistatus.ObjectNotFound if object is missing in write-cache. -func (c *cache) Delete(ctx context.Context, addr oid.Address) error { - _, span := tracing.StartSpanFromContext(ctx, "writecache.Delete", - trace.WithAttributes( - attribute.String("address", addr.EncodeToString()), - )) - defer span.End() - - deleted := false - storageType := writecache.StorageTypeUndefined - startedAt := time.Now() - defer func() { - c.metrics.Delete(time.Since(startedAt), deleted, storageType) - }() - - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - if c.readOnly() { - return writecache.ErrReadOnly - } - - key := addr2key(addr) - - err := c.db.Update(func(tx *badger.Txn) error { - it, err := tx.Get(key[:]) - if err != nil { - if err == badger.ErrKeyNotFound { - return logicerr.Wrap(new(apistatus.ObjectNotFound)) - } - return err - } - if it.ValueSize() > 0 { - storageType = writecache.StorageTypeDB - err := tx.Delete(key[:]) - if err == nil { - storagelog.Write(c.log, - storagelog.AddressField(addr.EncodeToString()), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("db DELETE"), - ) - deleted = true - c.decDB() - } - return err - } - return nil - }) - - return metaerr.Wrap(err) -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush.go b/pkg/local_object_storage/writecache/writecachebadger/flush.go deleted file mode 100644 index 48e31dee6..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/flush.go +++ /dev/null @@ -1,286 +0,0 @@ -package writecachebadger - -import ( - "bytes" - "context" - "encoding/hex" - "errors" - "fmt" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - "github.com/dgraph-io/badger/v4" - "github.com/dgraph-io/ristretto/z" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - "go.uber.org/zap" -) - -const ( - // flushBatchSize is amount of keys which will be read from cache to be flushed - // to the main storage. It is used to reduce contention between cache put - // and cache persist. - flushBatchSize = 512 - // defaultFlushWorkersCount is number of workers for putting objects in main storage. - defaultFlushWorkersCount = 20 - // defaultFlushInterval is default time interval between successive flushes. - defaultFlushInterval = time.Second -) - -type collector struct { - cache *cache - scheduled int - processed int -} - -func (c *collector) send(ctx context.Context, cancel func(), buf *z.Buffer) error { - list, err := badger.BufferToKVList(buf) - if err != nil { - return err - } - for _, kv := range list.Kv { - select { - case <-ctx.Done(): - return nil - default: - } - if kv.StreamDone { - return nil - } - if c.scheduled >= flushBatchSize { - cancel() - return nil - } - if got, want := len(kv.Key), len(internalKey{}); got != want { - c.cache.log.Debug( - fmt.Sprintf("not expected db key len: got %d, want %d", got, want)) - continue - } - c.processed++ - obj := objectSDK.New() - val := bytes.Clone(kv.Value) - if err = obj.Unmarshal(val); err != nil { - continue - } - addr := objectCore.AddressOf(obj) - c.cache.scheduled4FlushMtx.RLock() - _, ok := c.cache.scheduled4Flush[addr] - c.cache.scheduled4FlushMtx.RUnlock() - if ok { - c.cache.log.Debug(logs.WritecacheBadgerObjAlreadyScheduled, zap.Stringer("obj", addr)) - continue - } - c.cache.scheduled4FlushMtx.Lock() - c.cache.scheduled4Flush[addr] = struct{}{} - c.cache.scheduled4FlushMtx.Unlock() - c.scheduled++ - select { - case c.cache.flushCh <- objectInfo{ - addr: addr, - data: val, - obj: obj, - }: - case <-ctx.Done(): - return nil - } - } - return nil -} - -// runFlushLoop starts background workers which periodically flush objects to the blobstor. -func (c *cache) runFlushLoop(ctx context.Context) { - if c.disableBackgroundFlush { - return - } - for i := 0; i < c.workersCount; i++ { - c.wg.Add(1) - go c.workerFlushSmall(ctx) - } - - c.wg.Add(1) - go func() { - defer c.wg.Done() - - tt := time.NewTimer(defaultFlushInterval) - defer tt.Stop() - - for { - select { - case <-tt.C: - c.flushSmallObjects(ctx) - tt.Reset(defaultFlushInterval) - case <-ctx.Done(): - return - } - } - }() -} - -func (c *cache) flushSmallObjects(ctx context.Context) { - for { - select { - case <-ctx.Done(): - return - default: - } - c.modeMtx.RLock() - if c.readOnly() { - c.modeMtx.RUnlock() - time.Sleep(time.Second) - continue - } - - // Using the db after Close will panic and badger won't wait for outstanding txs, - // so we need to check manually. - if c.db.IsClosed() { - c.modeMtx.RUnlock() - return - } - ctx, cancel := context.WithCancel(ctx) - coll := collector{ - cache: c, - } - stream := c.db.NewStream() - // All calls to Send are done by a single goroutine - stream.Send = func(buf *z.Buffer) error { - return coll.send(ctx, cancel, buf) - } - if err := stream.Orchestrate(ctx); err != nil { - c.log.Debug(fmt.Sprintf( - "error during flushing object from wc: %s", err)) - } - c.modeMtx.RUnlock() - if coll.scheduled == 0 { - break - } - c.log.Debug(logs.WritecacheTriedToFlushItemsFromWritecache, - zap.Int("scheduled", coll.scheduled), zap.Int("processed", coll.processed)) - } -} - -func (c *cache) reportFlushError(msg string, addr string, err error) { - if c.reportError != nil { - c.reportError(msg, err) - } else { - c.log.Error(msg, - zap.String("address", addr), - zap.Error(err)) - } -} - -// workerFlushSmall writes small objects to the main storage. -func (c *cache) workerFlushSmall(ctx context.Context) { - defer c.wg.Done() - - var objInfo objectInfo - for { - // Give priority to direct put. - select { - case objInfo = <-c.flushCh: - case <-ctx.Done(): - return - } - - err := c.flushObject(ctx, objInfo.obj, objInfo.data, writecache.StorageTypeDB) - if err == nil { - c.deleteFromDB([]internalKey{addr2key(objInfo.addr)}) - } - c.scheduled4FlushMtx.Lock() - delete(c.scheduled4Flush, objInfo.addr) - c.scheduled4FlushMtx.Unlock() - } -} - -// flushObject is used to write object directly to the main storage. -func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []byte, st writecache.StorageType) error { - var err error - - defer func() { - c.metrics.Flush(err == nil, st) - }() - - addr := objectCore.AddressOf(obj) - - var prm common.PutPrm - prm.Object = obj - prm.RawData = data - - res, err := c.blobstor.Put(ctx, prm) - if err != nil { - if !errors.Is(err, common.ErrNoSpace) && !errors.Is(err, common.ErrReadOnly) && - !errors.Is(err, blobstor.ErrNoPlaceFound) { - c.reportFlushError(logs.FrostFSNodeCantFlushObjectToBlobstor, - addr.EncodeToString(), err) - } - return err - } - - var updPrm meta.UpdateStorageIDPrm - updPrm.SetAddress(addr) - updPrm.SetStorageID(res.StorageID) - - _, err = c.metabase.UpdateStorageID(ctx, updPrm) - if err != nil { - c.reportFlushError(logs.FrostFSNodeCantUpdateObjectStorageID, - addr.EncodeToString(), err) - } - return err -} - -// Flush flushes all objects from the write-cache to the main storage. -// Write-cache must be in readonly mode to ensure correctness of an operation and -// to prevent interference with background flush workers. -func (c *cache) Flush(ctx context.Context, ignoreErrors bool) error { - ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Flush", - trace.WithAttributes( - attribute.Bool("ignore_errors", ignoreErrors), - )) - defer span.End() - - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - - return c.flush(ctx, ignoreErrors) -} - -func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { - return c.db.View(func(tx *badger.Txn) error { - it := tx.NewIterator(badger.DefaultIteratorOptions) - defer it.Close() - var key internalKey - for it.Rewind(); it.Valid(); it.Next() { - if got, want := int(it.Item().KeySize()), len(key); got != want { - err := fmt.Errorf("invalid db key len: got %d, want %d", got, want) - c.reportFlushError(logs.FrostFSNodeCantDecodeObjectAddressFromDB, hex.EncodeToString(it.Item().Key()), metaerr.Wrap(err)) - if ignoreErrors { - continue - } - return err - } - if err := it.Item().Value(func(data []byte) error { - var obj objectSDK.Object - if err := obj.Unmarshal(data); err != nil { - copy(key[:], it.Item().Key()) - c.reportFlushError(logs.FrostFSNodeCantUnmarshalObjectFromDB, key.address().EncodeToString(), metaerr.Wrap(err)) - if ignoreErrors { - return nil - } - return err - } - - return c.flushObject(ctx, &obj, data, writecache.StorageTypeDB) - }); err != nil { - return err - } - } - return nil - }) -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go b/pkg/local_object_storage/writecache/writecachebadger/flush_test.go deleted file mode 100644 index 5e3f60eab..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/flush_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package writecachebadger - -import ( - "path/filepath" - "sync/atomic" - "testing" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachetest" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" - oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - "github.com/dgraph-io/badger/v4" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestFlush(t *testing.T) { - testlogger := test.NewLogger(t, true) - - createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { - return New( - append([]Option{ - WithLogger(test.NewLogger(t, true)), - WithPath(filepath.Join(t.TempDir(), "writecache")), - WithMetabase(mb), - WithBlobstor(bs), - WithGCInterval(1 * time.Second), - WithDisableBackgroundFlush(), - }, opts...)...) - } - - errCountOpt := func() (Option, *atomic.Uint32) { - cnt := &atomic.Uint32{} - return WithReportErrorFunc(func(msg string, err error) { - cnt.Add(1) - testlogger.Warn(msg, zap.Uint32("error_count", cnt.Load()), zap.Error(err)) - }), cnt - } - - failures := []writecachetest.TestFailureInjector[Option]{ - { - Desc: "db, invalid address", - InjectFn: func(t *testing.T, wc writecache.Cache) { - c := wc.(*cache) - obj := testutil.GenerateObject() - data, err := obj.Marshal() - require.NoError(t, err) - require.NoError(t, c.db.Update(func(tx *badger.Txn) error { - return tx.Set([]byte{1, 2, 3}, data) - })) - }, - }, - { - Desc: "db, invalid object", - InjectFn: func(t *testing.T, wc writecache.Cache) { - c := wc.(*cache) - key := addr2key(oidtest.Address()) - require.NoError(t, c.db.Update(func(tx *badger.Txn) error { - return tx.Set(key[:], []byte{1, 2, 3}) - })) - }, - }, - } - - writecachetest.TestFlush(t, createCacheFn, errCountOpt, failures...) -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/gc.go b/pkg/local_object_storage/writecache/writecachebadger/gc.go deleted file mode 100644 index 8937ff295..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/gc.go +++ /dev/null @@ -1,45 +0,0 @@ -package writecachebadger - -import ( - "context" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" -) - -func (c *cache) runGCLoop(ctx context.Context) { - c.wg.Add(1) - - go func() { - defer c.wg.Done() - - t := time.NewTicker(c.gcInterval) - defer t.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-t.C: - c.runGC() - } - } - }() -} - -func (c *cache) runGC() { - // This serves to synchronize the c.db field when changing mode as well. - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - - ro := c.readOnly() - if ro { - return - } - - // 0.5 is the recommended value so that write amplification of the value log is 2. - // See https://pkg.go.dev/github.com/dgraph-io/badger/v4#DB.RunValueLogGC for more info. - for c.db.RunValueLogGC(0.5) == nil { - c.log.Debug(logs.WritecacheDBValueLogGCRunCompleted) - } -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/generic_test.go b/pkg/local_object_storage/writecache/writecachebadger/generic_test.go deleted file mode 100644 index 08845665f..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/generic_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package writecachebadger - -import ( - "testing" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/storagetest" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" -) - -func TestGeneric(t *testing.T) { - storagetest.TestAll(t, func(t *testing.T) storagetest.Component { - return New( - WithLogger(test.NewLogger(t, true)), - WithFlushWorkersCount(2), - WithPath(t.TempDir()), - WithGCInterval(1*time.Second)) - }) -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/get.go b/pkg/local_object_storage/writecache/writecachebadger/get.go deleted file mode 100644 index 42403e551..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/get.go +++ /dev/null @@ -1,95 +0,0 @@ -package writecachebadger - -import ( - "context" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/dgraph-io/badger/v4" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -// Get returns object from write-cache. -// -// Returns an error of type apistatus.ObjectNotFound if the requested object is missing in write-cache. -func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { - _, span := tracing.StartSpanFromContext(ctx, "writecache.Get", - trace.WithAttributes( - attribute.String("address", addr.EncodeToString()), - )) - defer span.End() - - obj, err := c.getInternal(addr) - return obj, metaerr.Wrap(err) -} - -func (c *cache) getInternal(addr oid.Address) (*objectSDK.Object, error) { - found := false - storageType := writecache.StorageTypeUndefined - startedAt := time.Now() - defer func() { - c.metrics.Get(time.Since(startedAt), found, storageType) - }() - - k := addr2key(addr) - value, err := Get(c.db, k[:]) - if err == nil { - obj := objectSDK.New() - found = true - storageType = writecache.StorageTypeDB - return obj, obj.Unmarshal(value) - } - - return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) -} - -// Head returns object header from write-cache. -// -// Returns an error of type apistatus.ObjectNotFound if the requested object is missing in write-cache. -func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { - _, span := tracing.StartSpanFromContext(ctx, "writecache.Head", - trace.WithAttributes( - attribute.String("address", addr.EncodeToString()), - )) - defer span.End() - - obj, err := c.getInternal(addr) - if err != nil { - return nil, metaerr.Wrap(err) - } - - return obj.CutPayload(), nil -} - -// Get fetches object from the underlying database. -// Key should be a stringified address. -// -// Returns an error of type apistatus.ObjectNotFound if the requested object is missing in db. -func Get(db *badger.DB, key []byte) ([]byte, error) { - var value []byte - - err := db.View(func(tx *badger.Txn) error { - it, err := tx.Get(key) - if err != nil { - if err == badger.ErrKeyNotFound { - return logicerr.Wrap(new(apistatus.ObjectNotFound)) - } - return err - } - v, err := it.ValueCopy(nil) - if err != nil { - return err - } - value = v - return nil - }) - - return value, metaerr.Wrap(err) -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/iterate.go b/pkg/local_object_storage/writecache/writecachebadger/iterate.go deleted file mode 100644 index 111242048..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/iterate.go +++ /dev/null @@ -1,32 +0,0 @@ -package writecachebadger - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/dgraph-io/badger/v4" -) - -// IterateDB iterates over all objects stored in badger.DB instance and passes them to f until error return. -// It is assumed that db is an underlying database of some WriteCache instance. -// -// DB must not be nil and should be opened. -func IterateDB(db *badger.DB, f func(oid.Address) error) error { - return metaerr.Wrap(db.View(func(tx *badger.Txn) error { - opts := badger.DefaultIteratorOptions - opts.PrefetchValues = false - it := tx.NewIterator(opts) - for it.Rewind(); it.Valid(); it.Next() { - var key internalKey - if got, want := len(it.Item().Key()), len(key); got != want { - return fmt.Errorf("invalid db key len: got %d, want %d", got, want) - } - copy(key[:], it.Item().Key()) - if err := f(key.address()); err != nil { - return err - } - } - return nil - })) -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/mode.go b/pkg/local_object_storage/writecache/writecachebadger/mode.go deleted file mode 100644 index 03d861831..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/mode.go +++ /dev/null @@ -1,78 +0,0 @@ -package writecachebadger - -import ( - "context" - "fmt" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -// SetMode sets write-cache mode of operation. -// When shard is put in read-only mode all objects in memory are flushed to disk -// and all background jobs are suspended. -func (c *cache) SetMode(m mode.Mode) error { - ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.SetMode", - trace.WithAttributes( - attribute.String("mode", m.String()), - )) - defer span.End() - - c.modeMtx.Lock() - defer c.modeMtx.Unlock() - - err := c.setMode(ctx, m) - if err == nil { - c.metrics.SetMode(m) - } - return err -} - -// setMode applies new mode. Must be called with cache.modeMtx lock taken. -func (c *cache) setMode(ctx context.Context, m mode.Mode) error { - var err error - turnOffMeta := m.NoMetabase() - - if turnOffMeta && !c.mode.NoMetabase() { - err = c.flush(ctx, true) - if err != nil { - return err - } - } - - if c.db != nil { - if err = c.db.Close(); err != nil { - return fmt.Errorf("can't close write-cache database: %w", err) - } - } - - // Suspend producers to ensure there are channel send operations in fly. - // flushCh is populated by `flush` with `modeMtx` taken, thus waiting until it is empty - // guarantees that there are no in-fly operations. - for len(c.flushCh) != 0 { - c.log.Info(logs.WritecacheWaitingForChannelsToFlush) - time.Sleep(time.Second) - } - - if turnOffMeta { - c.mode = m - return nil - } - - if err = c.openStore(m.ReadOnly()); err != nil { - return err - } - - c.mode = m - return nil -} - -// readOnly returns true if current mode is read-only. -// `c.modeMtx` must be taken. -func (c *cache) readOnly() bool { - return c.mode.ReadOnly() -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/options.go b/pkg/local_object_storage/writecache/writecachebadger/options.go deleted file mode 100644 index d041a9b88..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/options.go +++ /dev/null @@ -1,119 +0,0 @@ -package writecachebadger - -import ( - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" -) - -// Option represents write-cache configuration option. -type Option func(*options) - -type options struct { - log *logger.Logger - // path is a path to a directory for write-cache. - path string - // blobstor is the main persistent storage. - blobstor writecache.MainStorage - // metabase is the metabase instance. - metabase writecache.Metabase - // maxObjectSize is the maximum size of the object stored in the write-cache. - maxObjectSize uint64 - // workersCount is the number of workers flushing objects in parallel. - workersCount int - // maxCacheSize is the maximum total size of all objects saved in cache (DB + FS). - // 1 GiB by default. - maxCacheSize uint64 - // objCounters contains atomic counters for the number of objects stored in cache. - objCounters counters - // reportError is the function called when encountering disk errors in background workers. - reportError func(string, error) - // metrics is metrics implementation - metrics writecache.Metrics - // gcInterval is the interval duration to run the GC cycle. - gcInterval time.Duration - // disableBackgroundFlush is for testing purposes only. - disableBackgroundFlush bool -} - -// WithLogger sets logger. -func WithLogger(log *logger.Logger) Option { - return func(o *options) { - o.log = &logger.Logger{Logger: log.With(zap.String("component", "WriteCache"))} - } -} - -// WithPath sets path to writecache db. -func WithPath(path string) Option { - return func(o *options) { - o.path = path - } -} - -// WithBlobstor sets main object storage. -func WithBlobstor(bs writecache.MainStorage) Option { - return func(o *options) { - o.blobstor = bs - } -} - -// WithMetabase sets metabase. -func WithMetabase(db writecache.Metabase) Option { - return func(o *options) { - o.metabase = db - } -} - -// WithMaxObjectSize sets maximum object size to be stored in write-cache. -func WithMaxObjectSize(sz uint64) Option { - return func(o *options) { - if sz > 0 { - o.maxObjectSize = sz - } - } -} - -func WithFlushWorkersCount(c int) Option { - return func(o *options) { - if c > 0 { - o.workersCount = c - } - } -} - -// WithMaxCacheSize sets maximum write-cache size in bytes. -func WithMaxCacheSize(sz uint64) Option { - return func(o *options) { - o.maxCacheSize = sz - } -} - -// WithReportErrorFunc sets error reporting function. -func WithReportErrorFunc(f func(string, error)) Option { - return func(o *options) { - o.reportError = f - } -} - -// WithMetrics sets metrics implementation. -func WithMetrics(metrics writecache.Metrics) Option { - return func(o *options) { - o.metrics = metrics - } -} - -// WithGCInterval sets the duration of the interval to run GC cycles. -func WithGCInterval(d time.Duration) Option { - return func(o *options) { - o.gcInterval = d - } -} - -// WithDisableBackgroundFlush disables background flush, for testing purposes only. -func WithDisableBackgroundFlush() Option { - return func(o *options) { - o.disableBackgroundFlush = true - } -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/put.go b/pkg/local_object_storage/writecache/writecachebadger/put.go deleted file mode 100644 index 2071ba1d0..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/put.go +++ /dev/null @@ -1,82 +0,0 @@ -package writecachebadger - -import ( - "context" - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -// Put puts object to write-cache. -// -// Returns ErrReadOnly if write-cache is in R/O mode. -// Returns ErrNotInitialized if write-cache has not been initialized yet. -// Returns ErrOutOfSpace if saving an object leads to WC's size overflow. -// Returns ErrBigObject if an objects exceeds maximum object size. -func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { - _, span := tracing.StartSpanFromContext(ctx, "writecache.Put", - trace.WithAttributes( - attribute.String("address", prm.Address.EncodeToString()), - attribute.Bool("dont_compress", prm.DontCompress), - )) - defer span.End() - - startedAt := time.Now() - added := false - storageType := writecache.StorageTypeUndefined - defer func() { - c.metrics.Put(time.Since(startedAt), added, storageType) - }() - - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() - if c.readOnly() { - return common.PutRes{}, writecache.ErrReadOnly - } - - sz := uint64(len(prm.RawData)) - if sz > c.maxObjectSize { - return common.PutRes{}, writecache.ErrBigObject - } - - oi := objectInfo{ - addr: prm.Address, - obj: prm.Object, - data: prm.RawData, - } - - storageType = writecache.StorageTypeDB - err := c.put(oi) - if err == nil { - added = true - } - return common.PutRes{}, err -} - -// put persists objects to the write-cache database and -// pushes the to the flush workers queue. -func (c *cache) put(obj objectInfo) error { - cacheSize := c.estimateCacheSize() - if c.maxCacheSize < c.incSizeDB(cacheSize) { - return writecache.ErrOutOfSpace - } - - wb := c.db.NewWriteBatch() - k := addr2key(obj.addr) - _ = wb.Set(k[:], obj.data) - err := wb.Flush() - if err == nil { - storagelog.Write(c.log, - storagelog.AddressField(obj.addr), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("db PUT"), - ) - c.incDB() - } - return err -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/state.go b/pkg/local_object_storage/writecache/writecachebadger/state.go deleted file mode 100644 index e098eb060..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/state.go +++ /dev/null @@ -1,67 +0,0 @@ -package writecachebadger - -import ( - "fmt" - "math" - "sync/atomic" - - "github.com/dgraph-io/badger/v4" -) - -func (c *cache) estimateCacheSize() uint64 { - onDiskSize, _ := c.db.EstimateSize(nil) - c.metrics.SetEstimateSize(onDiskSize, 0) - return onDiskSize -} - -func (c *cache) incSizeDB(sz uint64) uint64 { - return sz + c.maxObjectSize -} - -type counters struct { - cDB atomic.Uint64 -} - -func (x *counters) IncDB() { - x.cDB.Add(1) -} - -func (x *counters) DecDB() { - x.cDB.Add(math.MaxUint64) -} - -func (x *counters) DB() uint64 { - return x.cDB.Load() -} - -func (c *cache) initCounters() error { - var inDB uint64 - err := c.db.View(func(tx *badger.Txn) error { - opts := badger.DefaultIteratorOptions - opts.PrefetchValues = false - it := tx.NewIterator(opts) - defer it.Close() - for it.Rewind(); it.Valid(); it.Next() { - inDB++ - } - return nil - }) - if err != nil { - return fmt.Errorf("could not read write-cache DB counter: %w", err) - } - - c.objCounters.cDB.Store(inDB) - c.metrics.SetActualCounters(inDB, 0) - - return nil -} - -func (c *cache) incDB() { - c.objCounters.IncDB() - c.metrics.SetActualCounters(c.objCounters.DB(), 0) -} - -func (c *cache) decDB() { - c.objCounters.DecDB() - c.metrics.SetActualCounters(c.objCounters.DB(), 0) -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/storage.go b/pkg/local_object_storage/writecache/writecachebadger/storage.go deleted file mode 100644 index 04337b7a6..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/storage.go +++ /dev/null @@ -1,91 +0,0 @@ -package writecachebadger - -import ( - "fmt" - "os" - "path/filepath" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/dgraph-io/badger/v4" - "go.uber.org/zap" -) - -// store represents persistent storage with in-memory LRU cache -// for flushed items on top of it. -type store struct { - db *badger.DB -} - -type internalKey [len(cid.ID{}) + len(oid.ID{})]byte - -func (k internalKey) address() oid.Address { - var addr oid.Address - var cnr cid.ID - var obj oid.ID - copy(cnr[:], k[:len(cnr)]) - copy(obj[:], k[len(cnr):]) - addr.SetContainer(cnr) - addr.SetObject(obj) - return addr -} - -func addr2key(addr oid.Address) internalKey { - var key internalKey - cnr, obj := addr.Container(), addr.Object() - copy(key[:len(cnr)], cnr[:]) - copy(key[len(cnr):], obj[:]) - return key -} - -const dbName = "small.badger" - -func (c *cache) openStore(readOnly bool) error { - err := util.MkdirAllX(c.path, os.ModePerm) - if err != nil { - return err - } - - c.db, err = OpenDB(filepath.Join(c.path, dbName), readOnly, c.log) - if err != nil { - return fmt.Errorf("could not open database: %w", err) - } - - return nil -} - -func (c *cache) deleteFromDB(keys []internalKey) []internalKey { - if len(keys) == 0 { - return keys - } - - wb := c.db.NewWriteBatch() - - var errorIndex int - for errorIndex = range keys { - if err := wb.Delete(keys[errorIndex][:]); err != nil { - break - } - } - - for i := 0; i < errorIndex; i++ { - c.decDB() - c.metrics.Evict(writecache.StorageTypeDB) - storagelog.Write(c.log, - storagelog.AddressField(keys[i]), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("db DELETE"), - ) - } - - if err := wb.Flush(); err != nil { - c.log.Error(logs.WritecacheCantRemoveObjectsFromTheDatabase, zap.Error(err)) - } - - copy(keys, keys[errorIndex:]) - return keys[:len(keys)-errorIndex] -} diff --git a/pkg/local_object_storage/writecache/writecachebadger/util.go b/pkg/local_object_storage/writecache/writecachebadger/util.go deleted file mode 100644 index e6079e370..000000000 --- a/pkg/local_object_storage/writecache/writecachebadger/util.go +++ /dev/null @@ -1,39 +0,0 @@ -package writecachebadger - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/dgraph-io/badger/v4" - badgeroptions "github.com/dgraph-io/badger/v4/options" -) - -// OpenDB opens a badger instance for write-cache. Opens in read-only mode if ro is true. -func OpenDB(p string, ro bool, l *logger.Logger) (*badger.DB, error) { - return badger.Open(badger.DefaultOptions(p). - WithReadOnly(ro). - WithSyncWrites(true). - WithCompression(badgeroptions.None). - WithLoggingLevel(badger.ERROR). - WithLogger(badgerLoggerWrapper{l})) -} - -type badgerLoggerWrapper struct { - l *logger.Logger -} - -func (w badgerLoggerWrapper) Errorf(msg string, args ...any) { - w.l.Error(fmt.Sprintf(msg, args...)) -} - -func (w badgerLoggerWrapper) Warningf(msg string, args ...any) { - w.l.Warn(fmt.Sprintf(msg, args...)) -} - -func (w badgerLoggerWrapper) Infof(msg string, args ...any) { - w.l.Info(fmt.Sprintf(msg, args...)) -} - -func (w badgerLoggerWrapper) Debugf(msg string, args ...any) { - w.l.Debug(fmt.Sprintf(msg, args...)) -} diff --git a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go b/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go deleted file mode 100644 index 89add8115..000000000 --- a/pkg/local_object_storage/writecache/writecachebbolt/flush_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package writecachebbolt - -import ( - "context" - "os" - "path/filepath" - "sync/atomic" - "testing" - - objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachetest" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" - oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - "github.com/stretchr/testify/require" - "go.etcd.io/bbolt" - "go.uber.org/zap" -) - -func TestFlush(t *testing.T) { - testlogger := test.NewLogger(t, true) - - createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs writecache.MainStorage, opts ...Option) writecache.Cache { - return New( - append([]Option{ - WithLogger(testlogger), - WithPath(filepath.Join(t.TempDir(), "writecache")), - WithSmallObjectSize(smallSize), - WithMetabase(mb), - WithBlobstor(bs), - WithDisableBackgroundFlush(), - }, opts...)...) - } - - errCountOpt := func() (Option, *atomic.Uint32) { - cnt := &atomic.Uint32{} - return WithReportErrorFunc(func(msg string, err error) { - cnt.Add(1) - testlogger.Warn(msg, zap.Uint32("error_count", cnt.Load()), zap.Error(err)) - }), cnt - } - - failures := []writecachetest.TestFailureInjector[Option]{ - { - Desc: "db, invalid address", - InjectFn: func(t *testing.T, wc writecache.Cache) { - c := wc.(*cache) - obj := testutil.GenerateObject() - data, err := obj.Marshal() - require.NoError(t, err) - require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - return b.Put([]byte{1, 2, 3}, data) - })) - }, - }, - { - Desc: "db, invalid object", - InjectFn: func(t *testing.T, wc writecache.Cache) { - c := wc.(*cache) - require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - k := []byte(oidtest.Address().EncodeToString()) - v := []byte{1, 2, 3} - return b.Put(k, v) - })) - }, - }, - { - Desc: "fs, read error", - InjectFn: func(t *testing.T, wc writecache.Cache) { - c := wc.(*cache) - obj := testutil.GenerateObject() - data, err := obj.Marshal() - require.NoError(t, err) - - var prm common.PutPrm - prm.Address = objectCore.AddressOf(obj) - prm.RawData = data - - _, err = c.fsTree.Put(context.Background(), prm) - require.NoError(t, err) - - p := prm.Address.Object().EncodeToString() + "." + prm.Address.Container().EncodeToString() - p = filepath.Join(c.fsTree.RootPath, p[:1], p[1:]) - - _, err = os.Stat(p) // sanity check - require.NoError(t, err) - require.NoError(t, os.Truncate(p, 0)) // corrupt the file contents, so that it can't be unmarshalled - }, - }, - { - Desc: "fs, invalid object", - InjectFn: func(t *testing.T, wc writecache.Cache) { - c := wc.(*cache) - var prm common.PutPrm - prm.Address = oidtest.Address() - prm.RawData = []byte{1, 2, 3} - _, err := c.fsTree.Put(context.Background(), prm) - require.NoError(t, err) - }, - }, - } - - writecachetest.TestFlush(t, createCacheFn, errCountOpt, failures...) -} From 764f70634d24868eddf247cd4fec10341d16967f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 19 Dec 2023 18:03:57 +0300 Subject: [PATCH 0770/1943] [#881] containerSvc: Add APE validation Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/container.go | 5 +- pkg/services/container/ape.go | 514 +++++++++++++++++++++++++++++ pkg/services/container/ape_test.go | 491 +++++++++++++++++++++++++++ 3 files changed, 1009 insertions(+), 1 deletion(-) create mode 100644 pkg/services/container/ape.go create mode 100644 pkg/services/container/ape_test.go diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 4fc7f5649..2b9bdd71b 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -35,7 +35,10 @@ func initContainerService(_ context.Context, c *cfg) { server := containerTransportGRPC.New( containerService.NewSignService( &c.key.PrivateKey, - containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), + containerService.NewAPEServer(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine, cnrRdr, + newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, + containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), + ), ), ) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go new file mode 100644 index 000000000..e1ef43c63 --- /dev/null +++ b/pkg/services/container/ape.go @@ -0,0 +1,514 @@ +package container + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/sha256" + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +var ( + errMissingContainerID = errors.New("missing container ID") + errSessionContainerMissmatch = errors.New("requested container is not related to the session") + errMissingVerificationHeader = errors.New("malformed request: empty verification header") + errInvalidSessionTokenSignature = errors.New("malformed request: invalid session token signature") + errInvalidSessionTokenOwner = errors.New("malformed request: invalid session token owner") + errEmptyBodySignature = errors.New("malformed request: empty body signature") + errMissingOwnerID = errors.New("malformed request: missing owner ID") +) + +type ir interface { + InnerRingKeys() ([][]byte, error) +} + +type containers interface { + Get(cid.ID) (*containercore.Container, error) +} + +type apeChecker struct { + router policyengine.ChainRouter + reader containers + ir ir + nm netmap.Source + + next Server +} + +func NewAPEServer(router policyengine.ChainRouter, reader containers, ir ir, nm netmap.Source, srv Server) Server { + return &apeChecker{ + router: router, + reader: reader, + ir: ir, + next: srv, + nm: nm, + } +} + +func (ac *apeChecker) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.AnnounceUsedSpace") + defer span.End() + + // this method is not used, so not checked + + return ac.next.AnnounceUsedSpace(ctx, req) +} + +func (ac *apeChecker) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.Delete") + defer span.End() + + if err := ac.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), + nativeschema.MethodDeleteContainer); err != nil { + return nil, err + } + + return ac.next.Delete(ctx, req) +} + +func (ac *apeChecker) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.Get") + defer span.End() + + if err := ac.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), + nativeschema.MethodGetContainer); err != nil { + return nil, err + } + + return ac.next.Get(ctx, req) +} + +func (ac *apeChecker) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.GetExtendedACL") + defer span.End() + + if err := ac.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), + nativeschema.MethodGetContainerEACL); err != nil { + return nil, err + } + + return ac.next.GetExtendedACL(ctx, req) +} + +func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.List") + defer span.End() + + role, pk, err := ac.getRoleWithoutContainerID(req.GetBody().GetOwnerID(), req.GetMetaHeader(), req.GetVerificationHeader()) + if err != nil { + return nil, err + } + + reqProps := map[string]string{ + nativeschema.PropertyKeyActorPublicKey: pk.String(), + nativeschema.PropertyKeyActorRole: role, + } + + request := &apeRequest{ + resource: &apeResource{ + name: nativeschema.ResourceFormatRootContainers, + props: make(map[string]string), + }, + op: nativeschema.MethodListContainers, + props: reqProps, + } + + s, found, err := ac.router.IsAllowed(apechain.Ingress, + policyengine.NewRequestTargetWithNamespace(""), + request) + if err != nil { + return nil, err + } + + if !found || s == apechain.Allow { + return ac.next.List(ctx, req) + } + + return nil, apeErr(nativeschema.MethodListContainers, s) +} + +func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.Put") + defer span.End() + + role, pk, err := ac.getRoleWithoutContainerID(req.GetBody().GetContainer().GetOwnerID(), req.GetMetaHeader(), req.GetVerificationHeader()) + if err != nil { + return nil, err + } + + reqProps := map[string]string{ + nativeschema.PropertyKeyActorPublicKey: pk.String(), + nativeschema.PropertyKeyActorRole: role, + } + + request := &apeRequest{ + resource: &apeResource{ + name: nativeschema.ResourceFormatRootContainers, + props: make(map[string]string), + }, + op: nativeschema.MethodPutContainer, + props: reqProps, + } + + s, found, err := ac.router.IsAllowed(apechain.Ingress, + policyengine.NewRequestTargetWithNamespace(""), + request) + if err != nil { + return nil, err + } + + if !found || s == apechain.Allow { + return ac.next.Put(ctx, req) + } + + return nil, apeErr(nativeschema.MethodPutContainer, s) +} + +func (ac *apeChecker) getRoleWithoutContainerID(oID *refs.OwnerID, mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader) (string, *keys.PublicKey, error) { + if vh == nil { + return "", nil, errMissingVerificationHeader + } + + if oID == nil { + return "", nil, errMissingOwnerID + } + var ownerID user.ID + if err := ownerID.ReadFromV2(*oID); err != nil { + return "", nil, err + } + + actor, pk, err := ac.getActorAndPublicKey(mh, vh, cid.ID{}) + if err != nil { + return "", nil, err + } + + if actor.Equals(ownerID) { + return nativeschema.PropertyValueContainerRoleOwner, pk, nil + } + + pkBytes := pk.Bytes() + isIR, err := ac.isInnerRingKey(pkBytes) + if err != nil { + return "", nil, err + } + if isIR { + return nativeschema.PropertyValueContainerRoleIR, pk, nil + } + + return nativeschema.PropertyValueContainerRoleOthers, pk, nil +} + +func (ac *apeChecker) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.SetExtendedACL") + defer span.End() + + if err := ac.validateContainerBoundedOperation(req.GetBody().GetEACL().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), + nativeschema.MethodSetContainerEACL); err != nil { + return nil, err + } + + return ac.next.SetExtendedACL(ctx, req) +} + +func (ac *apeChecker) validateContainerBoundedOperation(containerID *refs.ContainerID, mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader, op string) error { + if vh == nil { + return errMissingVerificationHeader + } + + id, err := getContainerID(containerID) + if err != nil { + return err + } + + cont, err := ac.reader.Get(id) + if err != nil { + return err + } + + reqProps, err := ac.getRequestProps(mh, vh, cont, id) + if err != nil { + return err + } + + request := &apeRequest{ + resource: &apeResource{ + name: fmt.Sprintf(nativeschema.ResourceFormatRootContainer, id.EncodeToString()), + props: ac.getContainerProps(cont), + }, + op: op, + props: reqProps, + } + + s, found, err := ac.router.IsAllowed(apechain.Ingress, + policyengine.NewRequestTargetWithContainer(id.EncodeToString()), + request) + if err != nil { + return err + } + + if !found || s == apechain.Allow { + return nil + } + + return apeErr(op, s) +} + +func apeErr(operation string, status apechain.Status) error { + errAccessDenied := &apistatus.ObjectAccessDenied{} + errAccessDenied.WriteReason(fmt.Sprintf("access to container operation %s is denied by access policy engine: %s", operation, status.String())) + return errAccessDenied +} + +func getContainerID(reqContID *refs.ContainerID) (cid.ID, error) { + if reqContID == nil { + return cid.ID{}, errMissingContainerID + } + var id cid.ID + err := id.ReadFromV2(*reqContID) + if err != nil { + return cid.ID{}, fmt.Errorf("invalid container ID: %w", err) + } + return id, nil +} + +type apeRequest struct { + resource *apeResource + op string + props map[string]string +} + +// Operation implements resource.Request. +func (r *apeRequest) Operation() string { + return r.op +} + +// Property implements resource.Request. +func (r *apeRequest) Property(key string) string { + return r.props[key] +} + +// Resource implements resource.Request. +func (r *apeRequest) Resource() resource.Resource { + return r.resource +} + +type apeResource struct { + name string + props map[string]string +} + +func (r *apeResource) Name() string { + return r.name +} + +func (r *apeResource) Property(key string) string { + return r.props[key] +} + +func (ac *apeChecker) getContainerProps(c *containercore.Container) map[string]string { + return map[string]string{ + nativeschema.PropertyKeyContainerOwnerID: c.Value.Owner().EncodeToString(), + } +} + +func (ac *apeChecker) getRequestProps(mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader, + cont *containercore.Container, cnrID cid.ID, +) (map[string]string, error) { + actor, pk, err := ac.getActorAndPublicKey(mh, vh, cnrID) + if err != nil { + return nil, err + } + role, err := ac.getRole(actor, pk, cont, cnrID) + if err != nil { + return nil, err + } + return map[string]string{ + nativeschema.PropertyKeyActorPublicKey: pk.String(), + nativeschema.PropertyKeyActorRole: role, + }, nil +} + +func (ac *apeChecker) getRole(actor *user.ID, pk *keys.PublicKey, cont *containercore.Container, cnrID cid.ID) (string, error) { + if cont.Value.Owner().Equals(*actor) { + return nativeschema.PropertyValueContainerRoleOwner, nil + } + + pkBytes := pk.Bytes() + isIR, err := ac.isInnerRingKey(pkBytes) + if err != nil { + return "", err + } + if isIR { + return nativeschema.PropertyValueContainerRoleIR, nil + } + + isContainer, err := ac.isContainerKey(pkBytes, cnrID, cont) + if err != nil { + return "", err + } + if isContainer { + return nativeschema.PropertyValueContainerRoleContainer, nil + } + + return nativeschema.PropertyValueContainerRoleOthers, nil +} + +func (ac *apeChecker) getActorAndPublicKey(mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader, cnrID cid.ID) (*user.ID, *keys.PublicKey, error) { + st, err := ac.getSessionToken(mh) + if err != nil { + return nil, nil, err + } + + if st != nil { + return ac.getActorAndPKFromSessionToken(st, cnrID) + } + return ac.getActorAndPKFromSignature(vh) +} + +func (ac *apeChecker) getActorAndPKFromSignature(vh *session.RequestVerificationHeader) (*user.ID, *keys.PublicKey, error) { + for vh.GetOrigin() != nil { + vh = vh.GetOrigin() + } + sig := vh.GetBodySignature() + if sig == nil { + return nil, nil, errEmptyBodySignature + } + key, err := keys.NewPublicKeyFromBytes(sig.GetKey(), elliptic.P256()) + if err != nil { + return nil, nil, fmt.Errorf("invalid signature key: %w", err) + } + + var userID user.ID + user.IDFromKey(&userID, (ecdsa.PublicKey)(*key)) + + return &userID, key, nil +} + +func (ac *apeChecker) getSessionToken(mh *session.RequestMetaHeader) (*sessionSDK.Object, error) { + for mh.GetOrigin() != nil { + mh = mh.GetOrigin() + } + st := mh.GetSessionToken() + if st == nil { + return nil, nil + } + + var tok sessionSDK.Object + err := tok.ReadFromV2(*st) + if err != nil { + return nil, fmt.Errorf("invalid session token: %w", err) + } + + return &tok, nil +} + +func (ac *apeChecker) getActorAndPKFromSessionToken(st *sessionSDK.Object, cnrID cid.ID) (*user.ID, *keys.PublicKey, error) { + if !st.AssertContainer(cnrID) { + return nil, nil, errSessionContainerMissmatch + } + if !st.VerifySignature() { + return nil, nil, errInvalidSessionTokenSignature + } + var tok session.Token + st.WriteToV2(&tok) + + signaturePublicKey, err := keys.NewPublicKeyFromBytes(tok.GetSignature().GetKey(), elliptic.P256()) + if err != nil { + return nil, nil, fmt.Errorf("invalid key in session token signature: %w", err) + } + + tokenIssuer := st.Issuer() + if !isOwnerFromKey(tokenIssuer, signaturePublicKey) { + return nil, nil, errInvalidSessionTokenOwner + } + + return &tokenIssuer, signaturePublicKey, nil +} + +func isOwnerFromKey(id user.ID, key *keys.PublicKey) bool { + if key == nil { + return false + } + + var id2 user.ID + user.IDFromKey(&id2, (ecdsa.PublicKey)(*key)) + + return id2.Equals(id) +} + +func (ac *apeChecker) isInnerRingKey(pk []byte) (bool, error) { + innerRingKeys, err := ac.ir.InnerRingKeys() + if err != nil { + return false, err + } + + for i := range innerRingKeys { + if bytes.Equal(innerRingKeys[i], pk) { + return true, nil + } + } + + return false, nil +} + +func (ac *apeChecker) isContainerKey(pk []byte, cnrID cid.ID, cont *containercore.Container) (bool, error) { + binCnrID := make([]byte, sha256.Size) + cnrID.Encode(binCnrID) + + nm, err := netmap.GetLatestNetworkMap(ac.nm) + if err != nil { + return false, err + } + + in, err := isContainerNode(nm, pk, binCnrID, cont) + if err != nil { + return false, err + } else if in { + return true, nil + } + + // then check previous netmap, this can happen in-between epoch change + // when node migrates data from last epoch container + nm, err = netmap.GetPreviousNetworkMap(ac.nm) + if err != nil { + return false, err + } + + return isContainerNode(nm, pk, binCnrID, cont) +} + +func isContainerNode(nm *netmapSDK.NetMap, pk, binCnrID []byte, cont *containercore.Container) (bool, error) { + cnrVectors, err := nm.ContainerNodes(cont.Value.PlacementPolicy(), binCnrID) + if err != nil { + return false, err + } + + for i := range cnrVectors { + for j := range cnrVectors[i] { + if bytes.Equal(cnrVectors[i][j].PublicKey(), pk) { + return true, nil + } + } + } + + return false, nil +} diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go new file mode 100644 index 000000000..b61e68eb5 --- /dev/null +++ b/pkg/services/container/ape_test.go @@ -0,0 +1,491 @@ +package container + +import ( + "context" + "errors" + "fmt" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" + containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + containertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + sessiontest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestAPE(t *testing.T) { + t.Parallel() + t.Run("deny get container for others", testDenyGetContainerForOthers) + t.Run("deny set container eACL for IR", testDenySetContainerEACLForIR) + t.Run("deny get container eACL for IR with session token", testDenyGetContainerEACLForIRSessionToken) + t.Run("deny put container for others with session token", testDenyPutContainerForOthersSessionToken) + t.Run("deny list containers for owner with PK", testDenyListContainersForPK) +} + +func testDenyGetContainerForOthers(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + apeSrv := NewAPEServer(router, contRdr, ir, nm, srv) + + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + contRdr.c[contID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(nm.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + nm.netmaps[nm.currentEpoch] = &testNetmap + nm.netmaps[nm.currentEpoch-1] = &testNetmap + + _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOthers, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := &container.GetRequest{} + req.SetBody(&container.GetRequestBody{}) + var refContID refs.ContainerID + contID.WriteToV2(&refContID) + req.GetBody().SetContainerID(&refContID) + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + resp, err := apeSrv.Get(context.Background(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) +} + +func testDenySetContainerEACLForIR(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + apeSrv := NewAPEServer(router, contRdr, ir, nm, srv) + + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + contRdr.c[contID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(nm.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + nm.netmaps[nm.currentEpoch] = &testNetmap + nm.netmaps[nm.currentEpoch-1] = &testNetmap + + _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodSetContainerEACL, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleIR, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := &container.SetExtendedACLRequest{} + req.SetBody(&container.SetExtendedACLRequestBody{}) + var refContID refs.ContainerID + contID.WriteToV2(&refContID) + req.GetBody().SetEACL(&acl.Table{}) + req.GetBody().GetEACL().SetContainerID(&refContID) + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + ir.keys = append(ir.keys, pk.PublicKey().Bytes()) + + resp, err := apeSrv.SetExtendedACL(context.Background(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) +} + +func testDenyGetContainerEACLForIRSessionToken(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + apeSrv := NewAPEServer(router, contRdr, ir, nm, srv) + + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + contRdr.c[contID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(nm.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + nm.netmaps[nm.currentEpoch] = &testNetmap + nm.netmaps[nm.currentEpoch-1] = &testNetmap + + _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainerEACL, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleIR, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := &container.GetExtendedACLRequest{} + req.SetBody(&container.GetExtendedACLRequestBody{}) + var refContID refs.ContainerID + contID.WriteToV2(&refContID) + req.GetBody().SetContainerID(&refContID) + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + sessionPK, err := keys.NewPrivateKey() + require.NoError(t, err) + sToken := sessiontest.ObjectSigned() + sToken.BindContainer(contID) + require.NoError(t, sToken.Sign(sessionPK.PrivateKey)) + var sTokenV2 session.Token + sToken.WriteToV2(&sTokenV2) + metaHeader := new(session.RequestMetaHeader) + metaHeader.SetSessionToken(&sTokenV2) + req.SetMetaHeader(metaHeader) + + ir.keys = append(ir.keys, sessionPK.PublicKey().Bytes()) + + resp, err := apeSrv.GetExtendedACL(context.Background(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) +} + +func testDenyPutContainerForOthersSessionToken(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + apeSrv := NewAPEServer(router, contRdr, ir, nm, srv) + + testContainer := containertest.Container() + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + + _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(""), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodPutContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + nativeschema.ResourceFormatRootContainers, + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOthers, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := &container.PutRequest{} + req.SetBody(&container.PutRequestBody{}) + var reqCont container.Container + testContainer.WriteToV2(&reqCont) + req.GetBody().SetContainer(&reqCont) + + sessionPK, err := keys.NewPrivateKey() + require.NoError(t, err) + sToken := sessiontest.ObjectSigned() + sToken.BindContainer(cid.ID{}) + require.NoError(t, sToken.Sign(sessionPK.PrivateKey)) + var sTokenV2 session.Token + sToken.WriteToV2(&sTokenV2) + metaHeader := new(session.RequestMetaHeader) + metaHeader.SetSessionToken(&sTokenV2) + req.SetMetaHeader(metaHeader) + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + resp, err := apeSrv.Put(context.Background(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) +} + +func testDenyListContainersForPK(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + apeSrv := NewAPEServer(router, contRdr, ir, nm, srv) + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + _, _, err = router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(""), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodListContainers, + }, + }, + Resources: chain.Resources{ + Names: []string{ + nativeschema.ResourceFormatRootContainers, + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorPublicKey, + Value: pk.PublicKey().String(), + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + var userID user.ID + user.IDFromKey(&userID, pk.PrivateKey.PublicKey) + + req := &container.ListRequest{} + req.SetBody(&container.ListRequestBody{}) + var ownerID refs.OwnerID + userID.WriteToV2(&ownerID) + req.GetBody().SetOwnerID(&ownerID) + + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + resp, err := apeSrv.List(context.Background(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) +} + +type srvStub struct { + calls map[string]int +} + +func (s *srvStub) AnnounceUsedSpace(context.Context, *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { + s.calls["AnnounceUsedSpace"]++ + return &container.AnnounceUsedSpaceResponse{}, nil +} + +func (s *srvStub) Delete(context.Context, *container.DeleteRequest) (*container.DeleteResponse, error) { + s.calls["Delete"]++ + return &container.DeleteResponse{}, nil +} + +func (s *srvStub) Get(context.Context, *container.GetRequest) (*container.GetResponse, error) { + s.calls["Get"]++ + return &container.GetResponse{}, nil +} + +func (s *srvStub) GetExtendedACL(context.Context, *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { + s.calls["GetExtendedACL"]++ + return &container.GetExtendedACLResponse{}, nil +} + +func (s *srvStub) List(context.Context, *container.ListRequest) (*container.ListResponse, error) { + s.calls["List"]++ + return &container.ListResponse{}, nil +} + +func (s *srvStub) Put(context.Context, *container.PutRequest) (*container.PutResponse, error) { + s.calls["Put"]++ + return &container.PutResponse{}, nil +} + +func (s *srvStub) SetExtendedACL(context.Context, *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { + s.calls["SetExtendedACL"]++ + return &container.SetExtendedACLResponse{}, nil +} + +type irStub struct { + keys [][]byte +} + +func (s *irStub) InnerRingKeys() ([][]byte, error) { + return s.keys, nil +} + +type containerStub struct { + c map[cid.ID]*containercore.Container +} + +func (s *containerStub) Get(id cid.ID) (*containercore.Container, error) { + if v, ok := s.c[id]; ok { + return v, nil + } + return nil, errors.New("container not found") +} + +type netmapStub struct { + netmaps map[uint64]*netmap.NetMap + currentEpoch uint64 +} + +func (s *netmapStub) GetNetMap(diff uint64) (*netmap.NetMap, error) { + if diff >= s.currentEpoch { + return nil, errors.New("invalid diff") + } + return s.GetNetMapByEpoch(s.currentEpoch - diff) +} + +func (s *netmapStub) GetNetMapByEpoch(epoch uint64) (*netmap.NetMap, error) { + if nm, found := s.netmaps[epoch]; found { + return nm, nil + } + return nil, errors.New("netmap not found") +} + +func (s *netmapStub) Epoch() (uint64, error) { + return s.currentEpoch, nil +} From b118734909bd22402eb2192426d6470ff5520931 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 28 Dec 2023 13:04:29 +0300 Subject: [PATCH 0771/1943] [#890] getsvc: Log node PK Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/remote.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index 4434f036a..cd94434cf 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -2,6 +2,7 @@ package getsvc import ( "context" + "encoding/hex" "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -16,7 +17,7 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { ctx, span := tracing.StartSpanFromContext(ctx, "getService.processNode") defer span.End() - r.log.Debug(logs.ProcessingNode) + r.log.Debug(logs.ProcessingNode, zap.String("node_key", hex.EncodeToString(info.PublicKey()))) rs, ok := r.getRemoteStorage(info) if !ok { From 0cb0fc17355f7c26e9229f157673330664cea59b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Dec 2023 08:20:15 +0300 Subject: [PATCH 0772/1943] [#569] writecache: Allow to seal writecache after flush Signed-off-by: Dmitrii Stepanov --- .../modules/control/flush_cache.go | 8 +- pkg/local_object_storage/engine/writecache.go | 10 +- pkg/local_object_storage/shard/writecache.go | 9 +- pkg/local_object_storage/writecache/flush.go | 91 ++- .../writecache/flush_test.go | 9 +- pkg/local_object_storage/writecache/mode.go | 6 +- .../writecache/storage.go | 24 +- .../writecache/writecache.go | 2 +- pkg/services/control/server/flush_cache.go | 1 + pkg/services/control/service.pb.go | 616 +++++++++--------- pkg/services/control/service.proto | 2 + pkg/services/control/service_frostfs.pb.go | 2 + 12 files changed, 436 insertions(+), 344 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/flush_cache.go b/cmd/frostfs-cli/modules/control/flush_cache.go index 48be393dc..7f632e9fc 100644 --- a/cmd/frostfs-cli/modules/control/flush_cache.go +++ b/cmd/frostfs-cli/modules/control/flush_cache.go @@ -8,6 +8,8 @@ import ( "github.com/spf13/cobra" ) +const sealFlag = "seal" + var flushCacheCmd = &cobra.Command{ Use: "flush-cache", Short: "Flush objects from the write-cache to the main storage", @@ -18,7 +20,10 @@ var flushCacheCmd = &cobra.Command{ func flushCache(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) - req := &control.FlushCacheRequest{Body: new(control.FlushCacheRequest_Body)} + seal, _ := cmd.Flags().GetBool(sealFlag) + req := &control.FlushCacheRequest{Body: &control.FlushCacheRequest_Body{ + Seal: seal, + }} req.Body.Shard_ID = getShardIDList(cmd) signRequest(cmd, pk, req) @@ -44,6 +49,7 @@ func initControlFlushCacheCmd() { ff := flushCacheCmd.Flags() ff.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") ff.Bool(shardAllFlag, false, "Process all shards") + ff.Bool(sealFlag, false, "Writecache will be left in read-only mode after flush completed") flushCacheCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) } diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 00a40105e..d92a86f5d 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -17,6 +17,7 @@ import ( type FlushWriteCachePrm struct { shardID *shard.ID ignoreErrors bool + seal bool } // SetShardID is an option to set shard ID. @@ -26,11 +27,16 @@ func (p *FlushWriteCachePrm) SetShardID(id *shard.ID) { p.shardID = id } -// SetIgnoreErrors sets errors ignore flag.. +// SetIgnoreErrors sets errors ignore flag. func (p *FlushWriteCachePrm) SetIgnoreErrors(ignore bool) { p.ignoreErrors = ignore } +// SetSeal sets seal flag. +func (p *FlushWriteCachePrm) SetSeal(v bool) { + p.seal = v +} + // FlushWriteCacheRes groups the resulting values of FlushWriteCache operation. type FlushWriteCacheRes struct{} @@ -40,6 +46,7 @@ func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePr trace.WithAttributes( attribute.String("shard)id", p.shardID.String()), attribute.Bool("ignore_errors", p.ignoreErrors), + attribute.Bool("seal", p.seal), )) defer span.End() @@ -53,6 +60,7 @@ func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePr var prm shard.FlushWriteCachePrm prm.SetIgnoreErrors(p.ignoreErrors) + prm.SetSeal(p.seal) return FlushWriteCacheRes{}, sh.FlushWriteCache(ctx, prm) } diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index 7ce279c54..4e57a0497 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -12,6 +12,7 @@ import ( // FlushWriteCachePrm represents parameters of a `FlushWriteCache` operation. type FlushWriteCachePrm struct { ignoreErrors bool + seal bool } // SetIgnoreErrors sets the flag to ignore read-errors during flush. @@ -19,6 +20,11 @@ func (p *FlushWriteCachePrm) SetIgnoreErrors(ignore bool) { p.ignoreErrors = ignore } +// SetSeal sets the flag to left writecache in read-only mode after flush. +func (p *FlushWriteCachePrm) SetSeal(v bool) { + p.seal = v +} + // errWriteCacheDisabled is returned when an operation on write-cache is performed, // but write-cache is disabled. var errWriteCacheDisabled = errors.New("write-cache is disabled") @@ -29,6 +35,7 @@ func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error trace.WithAttributes( attribute.String("shard_id", s.ID().String()), attribute.Bool("ignore_errors", p.ignoreErrors), + attribute.Bool("seal", p.seal), )) defer span.End() @@ -47,5 +54,5 @@ func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error return ErrDegradedMode } - return s.writeCache.Flush(ctx, p.ignoreErrors) + return s.writeCache.Flush(ctx, p.ignoreErrors, p.seal) } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index f4ceec8c8..17dcc1107 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -12,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -33,6 +34,8 @@ const ( defaultFlushInterval = time.Second ) +var errIterationCompleted = errors.New("iteration completed") + // runFlushLoop starts background workers which periodically flush objects to the blobstor. func (c *cache) runFlushLoop(ctx context.Context) { if c.disableBackgroundFlush { @@ -229,7 +232,7 @@ func (c *cache) workerFlushSmall(ctx context.Context) { continue } - c.deleteFromDB(objInfo.addr) + c.deleteFromDB(objInfo.addr, true) } } @@ -270,19 +273,29 @@ func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []b } // Flush flushes all objects from the write-cache to the main storage. -// Write-cache must be in readonly mode to ensure correctness of an operation and -// to prevent interference with background flush workers. -func (c *cache) Flush(ctx context.Context, ignoreErrors bool) error { - ctx, span := tracing.StartSpanFromContext(ctx, "Flush", +func (c *cache) Flush(ctx context.Context, ignoreErrors, seal bool) error { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Flush", trace.WithAttributes( attribute.Bool("ignore_errors", ignoreErrors), + attribute.Bool("seal", seal), )) defer span.End() - c.modeMtx.RLock() - defer c.modeMtx.RUnlock() + c.modeMtx.Lock() // exclusive lock to not to conflict with background flush + defer c.modeMtx.Unlock() - return c.flush(ctx, ignoreErrors) + if err := c.flush(ctx, ignoreErrors); err != nil { + return err + } + + if seal { + m := c.mode | mode.ReadOnly + if err := c.setMode(ctx, m, ignoreErrors); err != nil { + return err + } + c.metrics.SetMode(m) + } + return nil } func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { @@ -290,13 +303,53 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { return err } - return c.db.View(func(tx *bbolt.Tx) error { + var last string + for { + batch, err := c.readNextDBBatch(ignoreErrors, last) + if err != nil { + return err + } + if len(batch) == 0 { + break + } + for _, item := range batch { + var obj objectSDK.Object + if err := obj.Unmarshal(item.data); err != nil { + c.reportFlushError(logs.FSTreeCantDecodeDBObjectAddress, item.address, metaerr.Wrap(err)) + if ignoreErrors { + continue + } + return err + } + + if err := c.flushObject(ctx, &obj, item.data, StorageTypeDB); err != nil { + return err + } + c.deleteFromDB(item.address, false) + } + last = batch[len(batch)-1].address + } + return nil +} + +type batchItem struct { + data []byte + address string +} + +func (c *cache) readNextDBBatch(ignoreErrors bool, last string) ([]batchItem, error) { + const batchSize = 100 + var batch []batchItem + err := c.db.View(func(tx *bbolt.Tx) error { var addr oid.Address b := tx.Bucket(defaultBucket) cs := b.Cursor() - for k, data := cs.Seek(nil); k != nil; k, data = cs.Next() { + for k, data := cs.Seek([]byte(last)); k != nil; k, data = cs.Next() { sa := string(k) + if sa == last { + continue + } if err := addr.DecodeString(sa); err != nil { c.reportFlushError(logs.FSTreeCantDecodeDBObjectAddress, sa, metaerr.Wrap(err)) if ignoreErrors { @@ -305,19 +358,15 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { return err } - var obj objectSDK.Object - if err := obj.Unmarshal(data); err != nil { - c.reportFlushError(logs.FSTreeCantDecodeDBObjectAddress, sa, metaerr.Wrap(err)) - if ignoreErrors { - continue - } - return err - } - - if err := c.flushObject(ctx, &obj, data, StorageTypeDB); err != nil { - return err + batch = append(batch, batchItem{data: bytes.Clone(data), address: sa}) + if len(batch) == batchSize { + return errIterationCompleted } } return nil }) + if err == nil || errors.Is(err, errIterationCompleted) { + return batch, nil + } + return nil, err } diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 20db1de95..4a243c5ec 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -147,7 +147,7 @@ func runFlushTest[Option any]( require.NoError(t, bs.SetMode(mode.ReadWrite)) require.NoError(t, mb.SetMode(mode.ReadWrite)) - require.NoError(t, wc.Flush(context.Background(), false)) + require.NoError(t, wc.Flush(context.Background(), false, false)) check(t, mb, bs, objects) }) @@ -159,8 +159,6 @@ func runFlushTest[Option any]( // Blobstor is read-only, so we expect en error from `flush` here. require.Error(t, wc.SetMode(mode.Degraded)) - // First move to read-only mode to close background workers. - require.NoError(t, wc.SetMode(mode.ReadOnly)) require.NoError(t, bs.SetMode(mode.ReadWrite)) require.NoError(t, mb.SetMode(mode.ReadWrite)) require.NoError(t, wc.SetMode(mode.Degraded)) @@ -177,14 +175,13 @@ func runFlushTest[Option any]( objects := putObjects(t, wc) f.InjectFn(t, wc) - require.NoError(t, wc.SetMode(mode.ReadOnly)) require.NoError(t, bs.SetMode(mode.ReadWrite)) require.NoError(t, mb.SetMode(mode.ReadWrite)) require.Equal(t, uint32(0), errCount.Load()) - require.Error(t, wc.Flush(context.Background(), false)) + require.Error(t, wc.Flush(context.Background(), false, false)) require.Greater(t, errCount.Load(), uint32(0)) - require.NoError(t, wc.Flush(context.Background(), true)) + require.NoError(t, wc.Flush(context.Background(), true, false)) check(t, mb, bs, objects) }) diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index e3ff2286b..7c6439fe9 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -25,7 +25,7 @@ func (c *cache) SetMode(m mode.Mode) error { c.modeMtx.Lock() defer c.modeMtx.Unlock() - err := c.setMode(ctx, m) + err := c.setMode(ctx, m, true) if err == nil { c.metrics.SetMode(m) } @@ -33,12 +33,12 @@ func (c *cache) SetMode(m mode.Mode) error { } // setMode applies new mode. Must be called with cache.modeMtx lock taken. -func (c *cache) setMode(ctx context.Context, m mode.Mode) error { +func (c *cache) setMode(ctx context.Context, m mode.Mode, ignoreErrors bool) error { var err error turnOffMeta := m.NoMetabase() if turnOffMeta && !c.mode.NoMetabase() { - err = c.flush(ctx, true) + err = c.flush(ctx, ignoreErrors) if err != nil { return err } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index 5c25a3b33..6cc3b06d0 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -67,14 +67,24 @@ func (c *cache) openStore(readOnly bool) error { return nil } -func (c *cache) deleteFromDB(key string) { +func (c *cache) deleteFromDB(key string, batched bool) { var recordDeleted bool - err := c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - key := []byte(key) - recordDeleted = b.Get(key) != nil - return b.Delete(key) - }) + var err error + if batched { + err = c.db.Batch(func(tx *bbolt.Tx) error { + b := tx.Bucket(defaultBucket) + key := []byte(key) + recordDeleted = b.Get(key) != nil + return b.Delete(key) + }) + } else { + err = c.db.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket(defaultBucket) + key := []byte(key) + recordDeleted = b.Get(key) != nil + return b.Delete(key) + }) + } if err == nil { c.metrics.Evict(StorageTypeDB) diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 3d3501969..0549c27f7 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -35,7 +35,7 @@ type Cache interface { SetMode(mode.Mode) error SetLogger(*logger.Logger) DumpInfo() Info - Flush(context.Context, bool) error + Flush(context.Context, bool, bool) error Init() error Open(ctx context.Context, readOnly bool) error diff --git a/pkg/services/control/server/flush_cache.go b/pkg/services/control/server/flush_cache.go index 9ead530db..67ffa1f2c 100644 --- a/pkg/services/control/server/flush_cache.go +++ b/pkg/services/control/server/flush_cache.go @@ -18,6 +18,7 @@ func (s *Server) FlushCache(ctx context.Context, req *control.FlushCacheRequest) for _, shardID := range s.getShardIDList(req.GetBody().GetShard_ID()) { var prm engine.FlushWriteCachePrm prm.SetShardID(shardID) + prm.SetSeal(req.GetBody().GetSeal()) _, err = s.s.FlushWriteCache(ctx, prm) if err != nil { diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 8b8739aba..90e643e29 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -2585,6 +2585,8 @@ type FlushCacheRequest_Body struct { // ID of the shard. Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` + // If true, then writecache will be left in read-only mode after flush completed. + Seal bool `protobuf:"varint,2,opt,name=seal,proto3" json:"seal,omitempty"` } func (x *FlushCacheRequest_Body) Reset() { @@ -2626,6 +2628,13 @@ func (x *FlushCacheRequest_Body) GetShard_ID() [][]byte { return nil } +func (x *FlushCacheRequest_Body) GetSeal() bool { + if x != nil { + return x.Seal + } + return false +} + // Response body structure. type FlushCacheResponse_Body struct { state protoimpl.MessageState @@ -3772,7 +3781,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x9d, 0x01, 0x0a, 0x11, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xb1, 0x01, 0x0a, 0x11, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x33, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, @@ -3780,333 +3789,334 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, + 0x72, 0x65, 0x1a, 0x35, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, - 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x84, 0x01, 0x0a, 0x12, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xc9, 0x01, 0x0a, - 0x0d, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, - 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x55, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, - 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, - 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x72, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x64, 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x44, 0x75, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x22, 0x7c, 0x0a, 0x0e, 0x44, 0x6f, 0x63, 0x74, - 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xd6, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, - 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, - 0x98, 0x01, 0x0a, 0x1c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x65, 0x61, 0x6c, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x04, 0x73, 0x65, 0x61, 0x6c, 0x22, 0x84, 0x01, 0x0a, 0x12, 0x46, 0x6c, + 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x22, 0xc9, 0x01, 0x0a, 0x0d, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x55, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x20, 0x0a, + 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, + 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x64, 0x75, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x44, 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x22, 0x7c, 0x0a, 0x0e, + 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x9e, 0x01, 0x0a, 0x1f, 0x47, - 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xe0, 0x05, 0x0a, 0x20, - 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xc5, 0x04, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, - 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x4d, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x35, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5b, 0x0a, 0x0a, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x55, - 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, - 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x73, - 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x1a, 0x25, 0x0a, 0x0d, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x24, 0x0a, - 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x63, - 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73, 0x65, 0x63, 0x6f, - 0x6e, 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x23, 0x0a, - 0x1f, 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, - 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, - 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, - 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0x94, - 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, - 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xd6, 0x01, 0x0a, 0x1b, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, + 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, + 0x6f, 0x72, 0x73, 0x22, 0x98, 0x01, 0x0a, 0x1c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xdc, - 0x01, 0x0a, 0x1c, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x9e, + 0x01, 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, + 0xe0, 0x05, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xc5, 0x04, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x65, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, + 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x4d, 0x0a, 0x06, 0x73, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x35, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, 0x08, 0x64, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x44, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5b, + 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x12, 0x18, 0x0a, 0x07, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x07, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x1a, 0x25, 0x0a, 0x0d, 0x55, 0x6e, + 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x1a, 0x24, 0x0a, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, + 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, + 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x23, 0x0a, 0x1f, 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x48, + 0x41, 0x52, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, + 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, + 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, + 0x10, 0x02, 0x22, 0x94, 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x4a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, - 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xb5, 0x01, - 0x0a, 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, 0x53, 0x74, + 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, + 0x64, 0x79, 0x22, 0xdc, 0x01, 0x0a, 0x1c, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, + 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, + 0x68, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, + 0x6e, 0x22, 0xb5, 0x01, 0x0a, 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, + 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xe1, 0x01, 0x0a, 0x1c, 0x47, 0x65, + 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, + 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xe1, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, - 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, - 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xca, 0x01, 0x0a, - 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x40, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, - 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, - 0x6e, 0x73, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xba, 0x01, 0x0a, - 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, + 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, + 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x22, 0xca, 0x01, 0x0a, 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x20, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x32, 0x9d, 0x0b, 0x0a, 0x0e, 0x43, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, - 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, - 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, - 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, - 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, - 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, - 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, - 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, - 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, - 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, - 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, - 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, - 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, - 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb6, 0x01, + 0x0a, 0x1f, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, + 0x0a, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, - 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, - 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, - 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, - 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, + 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, + 0x22, 0xba, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x20, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x32, 0x9d, 0x0b, + 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, + 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, + 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, + 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, + 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, + 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, + 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, + 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, + 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, + 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, + 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, + 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, + 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, + 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, + 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, + 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, + 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, + 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index cbc3aaf54..e3b507387 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -280,6 +280,8 @@ message FlushCacheRequest { message Body { // ID of the shard. repeated bytes shard_ID = 1; + // If true, then writecache will be left in read-only mode after flush completed. + bool seal = 2; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index e163e54b7..3ace081e4 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -1180,6 +1180,7 @@ func (x *FlushCacheRequest_Body) StableSize() (size int) { return 0 } size += proto.RepeatedBytesSize(1, x.Shard_ID) + size += proto.BoolSize(2, x.Seal) return size } @@ -1200,6 +1201,7 @@ func (x *FlushCacheRequest_Body) StableMarshal(buf []byte) []byte { } var offset int offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) + offset += proto.BoolMarshal(2, buf[offset:], x.Seal) return buf } From 32c282ca10ba53d0fd2e923caf457ab4febf206c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Dec 2023 11:38:10 +0300 Subject: [PATCH 0773/1943] [#569] writecache: Refactor flush Make single RUnlock call instead of two. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/writecache/flush.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 17dcc1107..c4e4b3b34 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -138,13 +138,11 @@ func (c *cache) flushSmallObjects(ctx context.Context) { } } + c.modeMtx.RUnlock() if count == 0 { - c.modeMtx.RUnlock() break } - c.modeMtx.RUnlock() - c.log.Debug(logs.WritecacheTriedToFlushItemsFromWritecache, zap.Int("count", count), zap.String("start", base58.Encode(lastKey))) From 7a9db5bcdd2fd846252fa1388d76be2db8a9458d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Dec 2023 11:40:55 +0300 Subject: [PATCH 0774/1943] [#569] writecache: Do not wait modeMtx if mode changes Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/writecache/delete.go | 5 ++++- pkg/local_object_storage/writecache/put.go | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index 0a4f4d658..cedb9f4ae 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -18,6 +18,7 @@ import ( // Delete removes object from write-cache. // // Returns an error of type apistatus.ObjectNotFound if object is missing in write-cache. +// Returns ErrNotInitialized if write-cache has not been initialized yet. func (c *cache) Delete(ctx context.Context, addr oid.Address) error { ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Delete", trace.WithAttributes( @@ -32,7 +33,9 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { c.metrics.Delete(time.Since(startedAt), deleted, storageType) }() - c.modeMtx.RLock() + if !c.modeMtx.TryRLock() { + return ErrNotInitialized + } defer c.modeMtx.RUnlock() if c.readOnly() { return ErrReadOnly diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 6fc655c65..8b6c09e9a 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -34,7 +34,9 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro c.metrics.Put(time.Since(startedAt), added, storageType) }() - c.modeMtx.RLock() + if !c.modeMtx.TryRLock() { + return common.PutRes{}, ErrNotInitialized + } defer c.modeMtx.RUnlock() if c.readOnly() { return common.PutRes{}, ErrReadOnly From 581887148a8971434a3becbc4e561c8a859f5fd7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Dec 2023 14:37:22 +0300 Subject: [PATCH 0775/1943] [#569] cli: Add `control shards writecache seal` command It does the same as `control shards flush-writecache --seal`, but has better name. Signed-off-by: Dmitrii Stepanov --- .../modules/control/flush_cache.go | 9 +- cmd/frostfs-cli/modules/control/shards.go | 2 + cmd/frostfs-cli/modules/control/writecache.go | 73 ++ pkg/local_object_storage/engine/writecache.go | 77 +- pkg/local_object_storage/shard/writecache.go | 30 + pkg/local_object_storage/writecache/seal.go | 28 + .../writecache/writecache.go | 1 + pkg/services/control/rpc.go | 14 + .../control/server/seal_writecache.go | 48 + pkg/services/control/service.pb.go | 1085 +++++++++++------ pkg/services/control/service.proto | 32 + pkg/services/control/service_frostfs.pb.go | 213 ++++ pkg/services/control/service_grpc.pb.go | 39 + 13 files changed, 1297 insertions(+), 354 deletions(-) create mode 100644 cmd/frostfs-cli/modules/control/writecache.go create mode 100644 pkg/local_object_storage/writecache/seal.go create mode 100644 pkg/services/control/server/seal_writecache.go diff --git a/cmd/frostfs-cli/modules/control/flush_cache.go b/cmd/frostfs-cli/modules/control/flush_cache.go index 7f632e9fc..541961903 100644 --- a/cmd/frostfs-cli/modules/control/flush_cache.go +++ b/cmd/frostfs-cli/modules/control/flush_cache.go @@ -11,10 +11,11 @@ import ( const sealFlag = "seal" var flushCacheCmd = &cobra.Command{ - Use: "flush-cache", - Short: "Flush objects from the write-cache to the main storage", - Long: "Flush objects from the write-cache to the main storage", - Run: flushCache, + Use: "flush-cache", + Short: "Flush objects from the write-cache to the main storage", + Long: "Flush objects from the write-cache to the main storage", + Run: flushCache, + Deprecated: "Flushing objects from writecache to the main storage is performed by writecache automatically. To flush and seal writecache use `frostfs-cli control shards writecache seal`.", } func flushCache(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index 6208c560b..6d3ef420c 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -17,6 +17,7 @@ func initControlShardsCmd() { shardsCmd.AddCommand(evacuationShardCmd) shardsCmd.AddCommand(flushCacheCmd) shardsCmd.AddCommand(doctorCmd) + shardsCmd.AddCommand(writecacheShardCmd) initControlShardsListCmd() initControlSetShardModeCmd() @@ -24,4 +25,5 @@ func initControlShardsCmd() { initControlEvacuationShardCmd() initControlFlushCacheCmd() initControlDoctorCmd() + initControlShardsWritecacheCmd() } diff --git a/cmd/frostfs-cli/modules/control/writecache.go b/cmd/frostfs-cli/modules/control/writecache.go new file mode 100644 index 000000000..abc4ed2e6 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/writecache.go @@ -0,0 +1,73 @@ +package control + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "github.com/mr-tron/base58" + "github.com/spf13/cobra" +) + +var writecacheShardCmd = &cobra.Command{ + Use: "writecache", + Short: "Operations with storage node's write-cache", + Long: "Operations with storage node's write-cache", +} + +var sealWritecacheShardCmd = &cobra.Command{ + Use: "seal", + Short: "Flush objects from write-cache and move write-cache to degraded read only mode.", + Long: "Flush all the objects from the write-cache to the main storage and move the write-cache to the degraded read only mode: write-cache will be empty and no objects will be put in it.", + Run: sealWritecache, +} + +func sealWritecache(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag) + + req := &control.SealWriteCacheRequest{Body: &control.SealWriteCacheRequest_Body{ + Shard_ID: getShardIDList(cmd), + IgnoreErrors: ignoreErrors, + }} + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.SealWriteCacheResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.SealWriteCache(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + var success, failed uint + for _, res := range resp.GetBody().GetResults() { + if res.GetSuccess() { + success++ + cmd.Printf("Shard %s: OK\n", base58.Encode(res.GetShard_ID())) + } else { + failed++ + cmd.Printf("Shard %s: failed with error %q\n", base58.Encode(res.GetShard_ID()), res.GetError()) + } + } + cmd.Printf("Total: %d success, %d failed\n", success, failed) +} + +func initControlShardsWritecacheCmd() { + writecacheShardCmd.AddCommand(sealWritecacheShardCmd) + + initControlFlags(sealWritecacheShardCmd) + + ff := sealWritecacheShardCmd.Flags() + ff.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") + ff.Bool(shardAllFlag, false, "Process all shards") + ff.Bool(ignoreErrorsFlag, true, "Skip invalid/unreadable objects") + + sealWritecacheShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) +} diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index d92a86f5d..0eca018f8 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -2,6 +2,7 @@ package engine import ( "context" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -11,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/errgroup" ) // FlushWriteCachePrm groups the parameters of FlushWriteCache operation. @@ -44,7 +46,7 @@ type FlushWriteCacheRes struct{} func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) (FlushWriteCacheRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.FlushWriteCache", trace.WithAttributes( - attribute.String("shard)id", p.shardID.String()), + attribute.String("shard_id", p.shardID.String()), attribute.Bool("ignore_errors", p.ignoreErrors), attribute.Bool("seal", p.seal), )) @@ -65,6 +67,79 @@ func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePr return FlushWriteCacheRes{}, sh.FlushWriteCache(ctx, prm) } +type SealWriteCachePrm struct { + ShardIDs []*shard.ID + IgnoreErrors bool +} + +type ShardSealResult struct { + ShardID *shard.ID + Success bool + ErrorMsg string +} + +type SealWriteCacheRes struct { + ShardResults []ShardSealResult +} + +// SealWriteCache flushed all data to blobstore and moves write-cache to degraded read only mode. +func (e *StorageEngine) SealWriteCache(ctx context.Context, prm SealWriteCachePrm) (SealWriteCacheRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.SealWriteCache", + trace.WithAttributes( + attribute.Int("shard_id_count", len(prm.ShardIDs)), + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + + res := SealWriteCacheRes{ + ShardResults: make([]ShardSealResult, 0, len(prm.ShardIDs)), + } + resGuard := &sync.Mutex{} + + eg, egCtx := errgroup.WithContext(ctx) + for _, shardID := range prm.ShardIDs { + shardID := shardID + eg.Go(func() error { + e.mtx.RLock() + sh, ok := e.shards[shardID.String()] + e.mtx.RUnlock() + + if !ok { + resGuard.Lock() + defer resGuard.Unlock() + res.ShardResults = append(res.ShardResults, ShardSealResult{ + ShardID: shardID, + ErrorMsg: errShardNotFound.Error(), + }) + return nil + } + + err := sh.SealWriteCache(egCtx, shard.SealWriteCachePrm{IgnoreErrors: prm.IgnoreErrors}) + + resGuard.Lock() + defer resGuard.Unlock() + + if err != nil { + res.ShardResults = append(res.ShardResults, ShardSealResult{ + ShardID: shardID, + ErrorMsg: err.Error(), + }) + } else { + res.ShardResults = append(res.ShardResults, ShardSealResult{ + ShardID: shardID, + Success: true, + }) + } + return nil + }) + } + + if err := eg.Wait(); err != nil { + return SealWriteCacheRes{}, err + } + return res, nil +} + type writeCacheMetrics struct { shardID string metrics metrics.WriteCacheMetrics diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index 4e57a0497..05e014d29 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -56,3 +56,33 @@ func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error return s.writeCache.Flush(ctx, p.ignoreErrors, p.seal) } + +type SealWriteCachePrm struct { + IgnoreErrors bool +} + +// SealWriteCache flushes all data from the write-cache and moves it to degraded read only mode. +func (s *Shard) SealWriteCache(ctx context.Context, p SealWriteCachePrm) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.SealWriteCache", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.Bool("ignore_errors", p.IgnoreErrors), + )) + defer span.End() + + if !s.hasWriteCache() { + return errWriteCacheDisabled + } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.ReadOnly() { + return ErrReadOnlyMode + } + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } + + return s.writeCache.Seal(ctx, p.IgnoreErrors) +} diff --git a/pkg/local_object_storage/writecache/seal.go b/pkg/local_object_storage/writecache/seal.go new file mode 100644 index 000000000..be4fec367 --- /dev/null +++ b/pkg/local_object_storage/writecache/seal.go @@ -0,0 +1,28 @@ +package writecache + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +func (c *cache) Seal(ctx context.Context, ignoreErrors bool) error { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Seal", + trace.WithAttributes( + attribute.Bool("ignore_errors", ignoreErrors), + )) + defer span.End() + + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + + // flush will be done by setMode + err := c.setMode(ctx, mode.DegradedReadOnly, ignoreErrors) + if err == nil { + c.metrics.SetMode(mode.DegradedReadOnly) + } + return err +} diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 0549c27f7..99fc5390a 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -36,6 +36,7 @@ type Cache interface { SetLogger(*logger.Logger) DumpInfo() Info Flush(context.Context, bool, bool) error + Seal(context.Context, bool) error Init() error Open(ctx context.Context, readOnly bool) error diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index b9ec05e71..3fd4d4ae9 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -24,6 +24,7 @@ const ( rpcGetChainLocalOverride = "GetChainLocalOverride" rpcListChainLocalOverrides = "ListChainLocalOverrides" rpcRemoveChainLocalOverride = "RemoveChainLocalOverride" + rpcSealWriteCache = "SealWriteCache" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -264,3 +265,16 @@ func RemoveChainLocalOverride(cli *client.Client, req *RemoveChainLocalOverrideR return wResp.message, nil } + +// SealWriteCache executes ControlService.SealWriteCache RPC. +func SealWriteCache(cli *client.Client, req *SealWriteCacheRequest, opts ...client.CallOption) (*SealWriteCacheResponse, error) { + wResp := newResponseWrapper[SealWriteCacheResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcSealWriteCache), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} diff --git a/pkg/services/control/server/seal_writecache.go b/pkg/services/control/server/seal_writecache.go new file mode 100644 index 000000000..5c39cf484 --- /dev/null +++ b/pkg/services/control/server/seal_writecache.go @@ -0,0 +1,48 @@ +package control + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *Server) SealWriteCache(ctx context.Context, req *control.SealWriteCacheRequest) (*control.SealWriteCacheResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + prm := engine.SealWriteCachePrm{ + ShardIDs: s.getShardIDList(req.GetBody().GetShard_ID()), + IgnoreErrors: req.GetBody().GetIgnoreErrors(), + } + + res, err := s.s.SealWriteCache(ctx, prm) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.SealWriteCacheResponse{Body: &control.SealWriteCacheResponse_Body{}} + for _, r := range res.ShardResults { + if r.Success { + resp.Body.Results = append(resp.GetBody().GetResults(), &control.SealWriteCacheResponse_Body_Status{ + Shard_ID: *r.ShardID, + Success: true, + }) + } else { + resp.Body.Results = append(resp.GetBody().GetResults(), &control.SealWriteCacheResponse_Body_Status{ + Shard_ID: *r.ShardID, + Error: r.ErrorMsg, + }) + } + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, err + } + return resp, nil +} diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 90e643e29..33ed898ba 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -1884,6 +1884,116 @@ func (x *RemoveChainLocalOverrideResponse) GetSignature() *Signature { return nil } +type SealWriteCacheRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *SealWriteCacheRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *SealWriteCacheRequest) Reset() { + *x = SealWriteCacheRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SealWriteCacheRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SealWriteCacheRequest) ProtoMessage() {} + +func (x *SealWriteCacheRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[32] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SealWriteCacheRequest.ProtoReflect.Descriptor instead. +func (*SealWriteCacheRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32} +} + +func (x *SealWriteCacheRequest) GetBody() *SealWriteCacheRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *SealWriteCacheRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +type SealWriteCacheResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *SealWriteCacheResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *SealWriteCacheResponse) Reset() { + *x = SealWriteCacheResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SealWriteCacheResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SealWriteCacheResponse) ProtoMessage() {} + +func (x *SealWriteCacheResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[33] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SealWriteCacheResponse.ProtoReflect.Descriptor instead. +func (*SealWriteCacheResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33} +} + +func (x *SealWriteCacheResponse) GetBody() *SealWriteCacheResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *SealWriteCacheResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + // Health check request body. type HealthCheckRequest_Body struct { state protoimpl.MessageState @@ -1894,7 +2004,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1907,7 +2017,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1938,7 +2048,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1951,7 +2061,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2002,7 +2112,7 @@ type SetNetmapStatusRequest_Body struct { func (x *SetNetmapStatusRequest_Body) Reset() { *x = SetNetmapStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2015,7 +2125,7 @@ func (x *SetNetmapStatusRequest_Body) String() string { func (*SetNetmapStatusRequest_Body) ProtoMessage() {} func (x *SetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2055,7 +2165,7 @@ type SetNetmapStatusResponse_Body struct { func (x *SetNetmapStatusResponse_Body) Reset() { *x = SetNetmapStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2068,7 +2178,7 @@ func (x *SetNetmapStatusResponse_Body) String() string { func (*SetNetmapStatusResponse_Body) ProtoMessage() {} func (x *SetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2098,7 +2208,7 @@ type DropObjectsRequest_Body struct { func (x *DropObjectsRequest_Body) Reset() { *x = DropObjectsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2111,7 +2221,7 @@ func (x *DropObjectsRequest_Body) String() string { func (*DropObjectsRequest_Body) ProtoMessage() {} func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2144,7 +2254,7 @@ type DropObjectsResponse_Body struct { func (x *DropObjectsResponse_Body) Reset() { *x = DropObjectsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2157,7 +2267,7 @@ func (x *DropObjectsResponse_Body) String() string { func (*DropObjectsResponse_Body) ProtoMessage() {} func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2183,7 +2293,7 @@ type ListShardsRequest_Body struct { func (x *ListShardsRequest_Body) Reset() { *x = ListShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2196,7 +2306,7 @@ func (x *ListShardsRequest_Body) String() string { func (*ListShardsRequest_Body) ProtoMessage() {} func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2225,7 +2335,7 @@ type ListShardsResponse_Body struct { func (x *ListShardsResponse_Body) Reset() { *x = ListShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2238,7 +2348,7 @@ func (x *ListShardsResponse_Body) String() string { func (*ListShardsResponse_Body) ProtoMessage() {} func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2278,7 +2388,7 @@ type SetShardModeRequest_Body struct { func (x *SetShardModeRequest_Body) Reset() { *x = SetShardModeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2291,7 +2401,7 @@ func (x *SetShardModeRequest_Body) String() string { func (*SetShardModeRequest_Body) ProtoMessage() {} func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2338,7 +2448,7 @@ type SetShardModeResponse_Body struct { func (x *SetShardModeResponse_Body) Reset() { *x = SetShardModeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2351,7 +2461,7 @@ func (x *SetShardModeResponse_Body) String() string { func (*SetShardModeResponse_Body) ProtoMessage() {} func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2382,7 +2492,7 @@ type SynchronizeTreeRequest_Body struct { func (x *SynchronizeTreeRequest_Body) Reset() { *x = SynchronizeTreeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2395,7 +2505,7 @@ func (x *SynchronizeTreeRequest_Body) String() string { func (*SynchronizeTreeRequest_Body) ProtoMessage() {} func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2442,7 +2552,7 @@ type SynchronizeTreeResponse_Body struct { func (x *SynchronizeTreeResponse_Body) Reset() { *x = SynchronizeTreeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2455,7 +2565,7 @@ func (x *SynchronizeTreeResponse_Body) String() string { func (*SynchronizeTreeResponse_Body) ProtoMessage() {} func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2486,7 +2596,7 @@ type EvacuateShardRequest_Body struct { func (x *EvacuateShardRequest_Body) Reset() { *x = EvacuateShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2499,7 +2609,7 @@ func (x *EvacuateShardRequest_Body) String() string { func (*EvacuateShardRequest_Body) ProtoMessage() {} func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2541,7 +2651,7 @@ type EvacuateShardResponse_Body struct { func (x *EvacuateShardResponse_Body) Reset() { *x = EvacuateShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2554,7 +2664,7 @@ func (x *EvacuateShardResponse_Body) String() string { func (*EvacuateShardResponse_Body) ProtoMessage() {} func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2592,7 +2702,7 @@ type FlushCacheRequest_Body struct { func (x *FlushCacheRequest_Body) Reset() { *x = FlushCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2605,7 +2715,7 @@ func (x *FlushCacheRequest_Body) String() string { func (*FlushCacheRequest_Body) ProtoMessage() {} func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2645,7 +2755,7 @@ type FlushCacheResponse_Body struct { func (x *FlushCacheResponse_Body) Reset() { *x = FlushCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2658,7 +2768,7 @@ func (x *FlushCacheResponse_Body) String() string { func (*FlushCacheResponse_Body) ProtoMessage() {} func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2689,7 +2799,7 @@ type DoctorRequest_Body struct { func (x *DoctorRequest_Body) Reset() { *x = DoctorRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2702,7 +2812,7 @@ func (x *DoctorRequest_Body) String() string { func (*DoctorRequest_Body) ProtoMessage() {} func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2742,7 +2852,7 @@ type DoctorResponse_Body struct { func (x *DoctorResponse_Body) Reset() { *x = DoctorResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2755,7 +2865,7 @@ func (x *DoctorResponse_Body) String() string { func (*DoctorResponse_Body) ProtoMessage() {} func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2786,7 +2896,7 @@ type StartShardEvacuationRequest_Body struct { func (x *StartShardEvacuationRequest_Body) Reset() { *x = StartShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2799,7 +2909,7 @@ func (x *StartShardEvacuationRequest_Body) String() string { func (*StartShardEvacuationRequest_Body) ProtoMessage() {} func (x *StartShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2839,7 +2949,7 @@ type StartShardEvacuationResponse_Body struct { func (x *StartShardEvacuationResponse_Body) Reset() { *x = StartShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2852,7 +2962,7 @@ func (x *StartShardEvacuationResponse_Body) String() string { func (*StartShardEvacuationResponse_Body) ProtoMessage() {} func (x *StartShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2878,7 +2988,7 @@ type GetShardEvacuationStatusRequest_Body struct { func (x *GetShardEvacuationStatusRequest_Body) Reset() { *x = GetShardEvacuationStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2891,7 +3001,7 @@ func (x *GetShardEvacuationStatusRequest_Body) String() string { func (*GetShardEvacuationStatusRequest_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2936,7 +3046,7 @@ type GetShardEvacuationStatusResponse_Body struct { func (x *GetShardEvacuationStatusResponse_Body) Reset() { *x = GetShardEvacuationStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2949,7 +3059,7 @@ func (x *GetShardEvacuationStatusResponse_Body) String() string { func (*GetShardEvacuationStatusResponse_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3040,7 +3150,7 @@ type GetShardEvacuationStatusResponse_Body_UnixTimestamp struct { func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) Reset() { *x = GetShardEvacuationStatusResponse_Body_UnixTimestamp{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3053,7 +3163,7 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) String() string { func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3088,7 +3198,7 @@ type GetShardEvacuationStatusResponse_Body_Duration struct { func (x *GetShardEvacuationStatusResponse_Body_Duration) Reset() { *x = GetShardEvacuationStatusResponse_Body_Duration{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3101,7 +3211,7 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) String() string { func (*GetShardEvacuationStatusResponse_Body_Duration) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_Duration) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3134,7 +3244,7 @@ type StopShardEvacuationRequest_Body struct { func (x *StopShardEvacuationRequest_Body) Reset() { *x = StopShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3147,7 +3257,7 @@ func (x *StopShardEvacuationRequest_Body) String() string { func (*StopShardEvacuationRequest_Body) ProtoMessage() {} func (x *StopShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3173,7 +3283,7 @@ type StopShardEvacuationResponse_Body struct { func (x *StopShardEvacuationResponse_Body) Reset() { *x = StopShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3186,7 +3296,7 @@ func (x *StopShardEvacuationResponse_Body) String() string { func (*StopShardEvacuationResponse_Body) ProtoMessage() {} func (x *StopShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3218,7 +3328,7 @@ type AddChainLocalOverrideRequest_Body struct { func (x *AddChainLocalOverrideRequest_Body) Reset() { *x = AddChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3231,7 +3341,7 @@ func (x *AddChainLocalOverrideRequest_Body) String() string { func (*AddChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *AddChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3275,7 +3385,7 @@ type AddChainLocalOverrideResponse_Body struct { func (x *AddChainLocalOverrideResponse_Body) Reset() { *x = AddChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3288,7 +3398,7 @@ func (x *AddChainLocalOverrideResponse_Body) String() string { func (*AddChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *AddChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3325,7 +3435,7 @@ type GetChainLocalOverrideRequest_Body struct { func (x *GetChainLocalOverrideRequest_Body) Reset() { *x = GetChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3338,7 +3448,7 @@ func (x *GetChainLocalOverrideRequest_Body) String() string { func (*GetChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *GetChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3380,7 +3490,7 @@ type GetChainLocalOverrideResponse_Body struct { func (x *GetChainLocalOverrideResponse_Body) Reset() { *x = GetChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3393,7 +3503,7 @@ func (x *GetChainLocalOverrideResponse_Body) String() string { func (*GetChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *GetChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3428,7 +3538,7 @@ type ListChainLocalOverridesRequest_Body struct { func (x *ListChainLocalOverridesRequest_Body) Reset() { *x = ListChainLocalOverridesRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3441,7 +3551,7 @@ func (x *ListChainLocalOverridesRequest_Body) String() string { func (*ListChainLocalOverridesRequest_Body) ProtoMessage() {} func (x *ListChainLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[64] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3476,7 +3586,7 @@ type ListChainLocalOverridesResponse_Body struct { func (x *ListChainLocalOverridesResponse_Body) Reset() { *x = ListChainLocalOverridesResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3489,7 +3599,7 @@ func (x *ListChainLocalOverridesResponse_Body) String() string { func (*ListChainLocalOverridesResponse_Body) ProtoMessage() {} func (x *ListChainLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[65] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3526,7 +3636,7 @@ type RemoveChainLocalOverrideRequest_Body struct { func (x *RemoveChainLocalOverrideRequest_Body) Reset() { *x = RemoveChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3539,7 +3649,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) String() string { func (*RemoveChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[66] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3580,7 +3690,7 @@ type RemoveChainLocalOverrideResponse_Body struct { func (x *RemoveChainLocalOverrideResponse_Body) Reset() { *x = RemoveChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3593,7 +3703,7 @@ func (x *RemoveChainLocalOverrideResponse_Body) String() string { func (*RemoveChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[67] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3616,6 +3726,174 @@ func (x *RemoveChainLocalOverrideResponse_Body) GetRemoved() bool { return false } +// Request body structure. +type SealWriteCacheRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // ID of the shard. + Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` + // Flag indicating whether object read errors should be ignored. + IgnoreErrors bool `protobuf:"varint,2,opt,name=ignore_errors,json=ignoreErrors,proto3" json:"ignore_errors,omitempty"` +} + +func (x *SealWriteCacheRequest_Body) Reset() { + *x = SealWriteCacheRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[68] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SealWriteCacheRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SealWriteCacheRequest_Body) ProtoMessage() {} + +func (x *SealWriteCacheRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[68] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SealWriteCacheRequest_Body.ProtoReflect.Descriptor instead. +func (*SealWriteCacheRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32, 0} +} + +func (x *SealWriteCacheRequest_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} + +func (x *SealWriteCacheRequest_Body) GetIgnoreErrors() bool { + if x != nil { + return x.IgnoreErrors + } + return false +} + +type SealWriteCacheResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Results []*SealWriteCacheResponse_Body_Status `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` +} + +func (x *SealWriteCacheResponse_Body) Reset() { + *x = SealWriteCacheResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[69] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SealWriteCacheResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SealWriteCacheResponse_Body) ProtoMessage() {} + +func (x *SealWriteCacheResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[69] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SealWriteCacheResponse_Body.ProtoReflect.Descriptor instead. +func (*SealWriteCacheResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33, 0} +} + +func (x *SealWriteCacheResponse_Body) GetResults() []*SealWriteCacheResponse_Body_Status { + if x != nil { + return x.Results + } + return nil +} + +type SealWriteCacheResponse_Body_Status struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Shard_ID []byte `protobuf:"bytes,1,opt,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` + Success bool `protobuf:"varint,2,opt,name=success,proto3" json:"success,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` +} + +func (x *SealWriteCacheResponse_Body_Status) Reset() { + *x = SealWriteCacheResponse_Body_Status{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[70] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SealWriteCacheResponse_Body_Status) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SealWriteCacheResponse_Body_Status) ProtoMessage() {} + +func (x *SealWriteCacheResponse_Body_Status) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[70] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SealWriteCacheResponse_Body_Status.ProtoReflect.Descriptor instead. +func (*SealWriteCacheResponse_Body_Status) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33, 0, 0} +} + +func (x *SealWriteCacheResponse_Body_Status) GetShard_ID() []byte { + if x != nil { + return x.Shard_ID + } + return nil +} + +func (x *SealWriteCacheResponse_Body_Status) GetSuccess() bool { + if x != nil { + return x.Success + } + return false +} + +func (x *SealWriteCacheResponse_Body_Status) GetError() string { + if x != nil { + return x.Error + } + return "" +} + var File_pkg_services_control_service_proto protoreflect.FileDescriptor var file_pkg_services_control_service_proto_rawDesc = []byte{ @@ -4021,102 +4299,139 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x20, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x32, 0x9d, 0x0b, - 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, - 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, - 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, - 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, - 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, - 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, - 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, - 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, - 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, - 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, - 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, - 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, - 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, - 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, - 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, - 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x22, 0xca, 0x01, + 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, + 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, + 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, + 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, + 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, + 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, + 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, + 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0xf0, 0x0b, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, + 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, + 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, + 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, + 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, + 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, + 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, + 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, + 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, - 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, - 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, - 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, - 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, - 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, + 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, + 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, + 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, + 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4132,7 +4447,7 @@ func file_pkg_services_control_service_proto_rawDescGZIP() []byte { } var file_pkg_services_control_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 66) +var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 71) var file_pkg_services_control_service_proto_goTypes = []interface{}{ (GetShardEvacuationStatusResponse_Body_Status)(0), // 0: control.GetShardEvacuationStatusResponse.Body.Status (*HealthCheckRequest)(nil), // 1: control.HealthCheckRequest @@ -4167,161 +4482,173 @@ var file_pkg_services_control_service_proto_goTypes = []interface{}{ (*ListChainLocalOverridesResponse)(nil), // 30: control.ListChainLocalOverridesResponse (*RemoveChainLocalOverrideRequest)(nil), // 31: control.RemoveChainLocalOverrideRequest (*RemoveChainLocalOverrideResponse)(nil), // 32: control.RemoveChainLocalOverrideResponse - (*HealthCheckRequest_Body)(nil), // 33: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 34: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 35: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 36: control.SetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 37: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 38: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 39: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 40: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 41: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 42: control.SetShardModeResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 43: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 44: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 45: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 46: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 47: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 48: control.FlushCacheResponse.Body - (*DoctorRequest_Body)(nil), // 49: control.DoctorRequest.Body - (*DoctorResponse_Body)(nil), // 50: control.DoctorResponse.Body - (*StartShardEvacuationRequest_Body)(nil), // 51: control.StartShardEvacuationRequest.Body - (*StartShardEvacuationResponse_Body)(nil), // 52: control.StartShardEvacuationResponse.Body - (*GetShardEvacuationStatusRequest_Body)(nil), // 53: control.GetShardEvacuationStatusRequest.Body - (*GetShardEvacuationStatusResponse_Body)(nil), // 54: control.GetShardEvacuationStatusResponse.Body - (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 55: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 56: control.GetShardEvacuationStatusResponse.Body.Duration - (*StopShardEvacuationRequest_Body)(nil), // 57: control.StopShardEvacuationRequest.Body - (*StopShardEvacuationResponse_Body)(nil), // 58: control.StopShardEvacuationResponse.Body - (*AddChainLocalOverrideRequest_Body)(nil), // 59: control.AddChainLocalOverrideRequest.Body - (*AddChainLocalOverrideResponse_Body)(nil), // 60: control.AddChainLocalOverrideResponse.Body - (*GetChainLocalOverrideRequest_Body)(nil), // 61: control.GetChainLocalOverrideRequest.Body - (*GetChainLocalOverrideResponse_Body)(nil), // 62: control.GetChainLocalOverrideResponse.Body - (*ListChainLocalOverridesRequest_Body)(nil), // 63: control.ListChainLocalOverridesRequest.Body - (*ListChainLocalOverridesResponse_Body)(nil), // 64: control.ListChainLocalOverridesResponse.Body - (*RemoveChainLocalOverrideRequest_Body)(nil), // 65: control.RemoveChainLocalOverrideRequest.Body - (*RemoveChainLocalOverrideResponse_Body)(nil), // 66: control.RemoveChainLocalOverrideResponse.Body - (*Signature)(nil), // 67: control.Signature - (NetmapStatus)(0), // 68: control.NetmapStatus - (HealthStatus)(0), // 69: control.HealthStatus - (*ShardInfo)(nil), // 70: control.ShardInfo - (ShardMode)(0), // 71: control.ShardMode - (*ChainTarget)(nil), // 72: control.ChainTarget + (*SealWriteCacheRequest)(nil), // 33: control.SealWriteCacheRequest + (*SealWriteCacheResponse)(nil), // 34: control.SealWriteCacheResponse + (*HealthCheckRequest_Body)(nil), // 35: control.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 36: control.HealthCheckResponse.Body + (*SetNetmapStatusRequest_Body)(nil), // 37: control.SetNetmapStatusRequest.Body + (*SetNetmapStatusResponse_Body)(nil), // 38: control.SetNetmapStatusResponse.Body + (*DropObjectsRequest_Body)(nil), // 39: control.DropObjectsRequest.Body + (*DropObjectsResponse_Body)(nil), // 40: control.DropObjectsResponse.Body + (*ListShardsRequest_Body)(nil), // 41: control.ListShardsRequest.Body + (*ListShardsResponse_Body)(nil), // 42: control.ListShardsResponse.Body + (*SetShardModeRequest_Body)(nil), // 43: control.SetShardModeRequest.Body + (*SetShardModeResponse_Body)(nil), // 44: control.SetShardModeResponse.Body + (*SynchronizeTreeRequest_Body)(nil), // 45: control.SynchronizeTreeRequest.Body + (*SynchronizeTreeResponse_Body)(nil), // 46: control.SynchronizeTreeResponse.Body + (*EvacuateShardRequest_Body)(nil), // 47: control.EvacuateShardRequest.Body + (*EvacuateShardResponse_Body)(nil), // 48: control.EvacuateShardResponse.Body + (*FlushCacheRequest_Body)(nil), // 49: control.FlushCacheRequest.Body + (*FlushCacheResponse_Body)(nil), // 50: control.FlushCacheResponse.Body + (*DoctorRequest_Body)(nil), // 51: control.DoctorRequest.Body + (*DoctorResponse_Body)(nil), // 52: control.DoctorResponse.Body + (*StartShardEvacuationRequest_Body)(nil), // 53: control.StartShardEvacuationRequest.Body + (*StartShardEvacuationResponse_Body)(nil), // 54: control.StartShardEvacuationResponse.Body + (*GetShardEvacuationStatusRequest_Body)(nil), // 55: control.GetShardEvacuationStatusRequest.Body + (*GetShardEvacuationStatusResponse_Body)(nil), // 56: control.GetShardEvacuationStatusResponse.Body + (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 57: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 58: control.GetShardEvacuationStatusResponse.Body.Duration + (*StopShardEvacuationRequest_Body)(nil), // 59: control.StopShardEvacuationRequest.Body + (*StopShardEvacuationResponse_Body)(nil), // 60: control.StopShardEvacuationResponse.Body + (*AddChainLocalOverrideRequest_Body)(nil), // 61: control.AddChainLocalOverrideRequest.Body + (*AddChainLocalOverrideResponse_Body)(nil), // 62: control.AddChainLocalOverrideResponse.Body + (*GetChainLocalOverrideRequest_Body)(nil), // 63: control.GetChainLocalOverrideRequest.Body + (*GetChainLocalOverrideResponse_Body)(nil), // 64: control.GetChainLocalOverrideResponse.Body + (*ListChainLocalOverridesRequest_Body)(nil), // 65: control.ListChainLocalOverridesRequest.Body + (*ListChainLocalOverridesResponse_Body)(nil), // 66: control.ListChainLocalOverridesResponse.Body + (*RemoveChainLocalOverrideRequest_Body)(nil), // 67: control.RemoveChainLocalOverrideRequest.Body + (*RemoveChainLocalOverrideResponse_Body)(nil), // 68: control.RemoveChainLocalOverrideResponse.Body + (*SealWriteCacheRequest_Body)(nil), // 69: control.SealWriteCacheRequest.Body + (*SealWriteCacheResponse_Body)(nil), // 70: control.SealWriteCacheResponse.Body + (*SealWriteCacheResponse_Body_Status)(nil), // 71: control.SealWriteCacheResponse.Body.Status + (*Signature)(nil), // 72: control.Signature + (NetmapStatus)(0), // 73: control.NetmapStatus + (HealthStatus)(0), // 74: control.HealthStatus + (*ShardInfo)(nil), // 75: control.ShardInfo + (ShardMode)(0), // 76: control.ShardMode + (*ChainTarget)(nil), // 77: control.ChainTarget } var file_pkg_services_control_service_proto_depIdxs = []int32{ - 33, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 67, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 34, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 67, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 35, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 67, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 36, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 67, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 37, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 67, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature - 38, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 67, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature - 39, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 67, // 13: control.ListShardsRequest.signature:type_name -> control.Signature - 40, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 67, // 15: control.ListShardsResponse.signature:type_name -> control.Signature - 41, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 67, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature - 42, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 67, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature - 43, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 67, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 44, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 67, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 45, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 67, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature - 46, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 67, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature - 47, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 67, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature - 48, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 67, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature - 49, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body - 67, // 33: control.DoctorRequest.signature:type_name -> control.Signature - 50, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body - 67, // 35: control.DoctorResponse.signature:type_name -> control.Signature - 51, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body - 67, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature - 52, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body - 67, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature - 53, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body - 67, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 54, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body - 67, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 57, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body - 67, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature - 58, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body - 67, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature - 59, // 48: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body - 67, // 49: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature - 60, // 50: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body - 67, // 51: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature - 61, // 52: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body - 67, // 53: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature - 62, // 54: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body - 67, // 55: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature - 63, // 56: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body - 67, // 57: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature - 64, // 58: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body - 67, // 59: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature - 65, // 60: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body - 67, // 61: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature - 66, // 62: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body - 67, // 63: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature - 68, // 64: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 69, // 65: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 68, // 66: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 70, // 67: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 71, // 68: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 0, // 69: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status - 56, // 70: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration - 55, // 71: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - 72, // 72: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 72, // 73: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 72, // 74: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget - 72, // 75: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 1, // 76: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 3, // 77: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 5, // 78: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 7, // 79: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 9, // 80: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 11, // 81: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 13, // 82: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 19, // 83: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest - 21, // 84: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest - 23, // 85: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest - 15, // 86: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 17, // 87: control.ControlService.Doctor:input_type -> control.DoctorRequest - 25, // 88: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest - 27, // 89: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest - 29, // 90: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest - 31, // 91: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest - 2, // 92: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 4, // 93: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 6, // 94: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 8, // 95: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 10, // 96: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 12, // 97: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 14, // 98: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 20, // 99: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse - 22, // 100: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse - 24, // 101: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse - 16, // 102: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 18, // 103: control.ControlService.Doctor:output_type -> control.DoctorResponse - 26, // 104: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse - 28, // 105: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse - 30, // 106: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse - 32, // 107: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse - 92, // [92:108] is the sub-list for method output_type - 76, // [76:92] is the sub-list for method input_type - 76, // [76:76] is the sub-list for extension type_name - 76, // [76:76] is the sub-list for extension extendee - 0, // [0:76] is the sub-list for field type_name + 35, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body + 72, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature + 36, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body + 72, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature + 37, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body + 72, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature + 38, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body + 72, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature + 39, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body + 72, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature + 40, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body + 72, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature + 41, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body + 72, // 13: control.ListShardsRequest.signature:type_name -> control.Signature + 42, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body + 72, // 15: control.ListShardsResponse.signature:type_name -> control.Signature + 43, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body + 72, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature + 44, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body + 72, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature + 45, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body + 72, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature + 46, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body + 72, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature + 47, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body + 72, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature + 48, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body + 72, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature + 49, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body + 72, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature + 50, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body + 72, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature + 51, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body + 72, // 33: control.DoctorRequest.signature:type_name -> control.Signature + 52, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body + 72, // 35: control.DoctorResponse.signature:type_name -> control.Signature + 53, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body + 72, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature + 54, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body + 72, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature + 55, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body + 72, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 56, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body + 72, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 59, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body + 72, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature + 60, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body + 72, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature + 61, // 48: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body + 72, // 49: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature + 62, // 50: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body + 72, // 51: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature + 63, // 52: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body + 72, // 53: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature + 64, // 54: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body + 72, // 55: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature + 65, // 56: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body + 72, // 57: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature + 66, // 58: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body + 72, // 59: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature + 67, // 60: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body + 72, // 61: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature + 68, // 62: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body + 72, // 63: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature + 69, // 64: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body + 72, // 65: control.SealWriteCacheRequest.signature:type_name -> control.Signature + 70, // 66: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body + 72, // 67: control.SealWriteCacheResponse.signature:type_name -> control.Signature + 73, // 68: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus + 74, // 69: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus + 73, // 70: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus + 75, // 71: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo + 76, // 72: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode + 0, // 73: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status + 58, // 74: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration + 57, // 75: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + 77, // 76: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 77, // 77: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 77, // 78: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget + 77, // 79: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 71, // 80: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status + 1, // 81: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 3, // 82: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 5, // 83: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 7, // 84: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 9, // 85: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 11, // 86: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 13, // 87: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 19, // 88: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest + 21, // 89: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest + 23, // 90: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest + 15, // 91: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 17, // 92: control.ControlService.Doctor:input_type -> control.DoctorRequest + 25, // 93: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest + 27, // 94: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest + 29, // 95: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest + 31, // 96: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest + 33, // 97: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest + 2, // 98: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 4, // 99: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 6, // 100: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 8, // 101: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 10, // 102: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 12, // 103: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 14, // 104: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 20, // 105: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse + 22, // 106: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse + 24, // 107: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse + 16, // 108: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 18, // 109: control.ControlService.Doctor:output_type -> control.DoctorResponse + 26, // 110: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse + 28, // 111: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse + 30, // 112: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse + 32, // 113: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse + 34, // 114: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse + 98, // [98:115] is the sub-list for method output_type + 81, // [81:98] is the sub-list for method input_type + 81, // [81:81] is the sub-list for extension type_name + 81, // [81:81] is the sub-list for extension extendee + 0, // [0:81] is the sub-list for field type_name } func init() { file_pkg_services_control_service_proto_init() } @@ -4716,7 +5043,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*SealWriteCacheRequest); i { case 0: return &v.state case 1: @@ -4728,7 +5055,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { + switch v := v.(*SealWriteCacheResponse); i { case 0: return &v.state case 1: @@ -4740,7 +5067,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusRequest_Body); i { + switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state case 1: @@ -4752,7 +5079,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusResponse_Body); i { + switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state case 1: @@ -4764,7 +5091,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest_Body); i { + switch v := v.(*SetNetmapStatusRequest_Body); i { case 0: return &v.state case 1: @@ -4776,7 +5103,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse_Body); i { + switch v := v.(*SetNetmapStatusResponse_Body); i { case 0: return &v.state case 1: @@ -4788,7 +5115,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest_Body); i { + switch v := v.(*DropObjectsRequest_Body); i { case 0: return &v.state case 1: @@ -4800,7 +5127,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse_Body); i { + switch v := v.(*DropObjectsResponse_Body); i { case 0: return &v.state case 1: @@ -4812,7 +5139,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest_Body); i { + switch v := v.(*ListShardsRequest_Body); i { case 0: return &v.state case 1: @@ -4824,7 +5151,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse_Body); i { + switch v := v.(*ListShardsResponse_Body); i { case 0: return &v.state case 1: @@ -4836,7 +5163,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest_Body); i { + switch v := v.(*SetShardModeRequest_Body); i { case 0: return &v.state case 1: @@ -4848,7 +5175,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse_Body); i { + switch v := v.(*SetShardModeResponse_Body); i { case 0: return &v.state case 1: @@ -4860,7 +5187,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest_Body); i { + switch v := v.(*SynchronizeTreeRequest_Body); i { case 0: return &v.state case 1: @@ -4872,7 +5199,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse_Body); i { + switch v := v.(*SynchronizeTreeResponse_Body); i { case 0: return &v.state case 1: @@ -4884,7 +5211,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest_Body); i { + switch v := v.(*EvacuateShardRequest_Body); i { case 0: return &v.state case 1: @@ -4896,7 +5223,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheResponse_Body); i { + switch v := v.(*EvacuateShardResponse_Body); i { case 0: return &v.state case 1: @@ -4908,7 +5235,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorRequest_Body); i { + switch v := v.(*FlushCacheRequest_Body); i { case 0: return &v.state case 1: @@ -4920,7 +5247,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorResponse_Body); i { + switch v := v.(*FlushCacheResponse_Body); i { case 0: return &v.state case 1: @@ -4932,7 +5259,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationRequest_Body); i { + switch v := v.(*DoctorRequest_Body); i { case 0: return &v.state case 1: @@ -4944,7 +5271,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationResponse_Body); i { + switch v := v.(*DoctorResponse_Body); i { case 0: return &v.state case 1: @@ -4956,7 +5283,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusRequest_Body); i { + switch v := v.(*StartShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -4968,7 +5295,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body); i { + switch v := v.(*StartShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -4980,7 +5307,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { + switch v := v.(*GetShardEvacuationStatusRequest_Body); i { case 0: return &v.state case 1: @@ -4992,7 +5319,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body); i { case 0: return &v.state case 1: @@ -5004,7 +5331,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationRequest_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { case 0: return &v.state case 1: @@ -5016,7 +5343,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationResponse_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { case 0: return &v.state case 1: @@ -5028,7 +5355,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideRequest_Body); i { + switch v := v.(*StopShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -5040,7 +5367,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideResponse_Body); i { + switch v := v.(*StopShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -5052,7 +5379,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideRequest_Body); i { + switch v := v.(*AddChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -5064,7 +5391,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideResponse_Body); i { + switch v := v.(*AddChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -5076,7 +5403,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesRequest_Body); i { + switch v := v.(*GetChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -5088,7 +5415,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesResponse_Body); i { + switch v := v.(*GetChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -5100,7 +5427,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { + switch v := v.(*ListChainLocalOverridesRequest_Body); i { case 0: return &v.state case 1: @@ -5112,6 +5439,30 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListChainLocalOverridesResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { case 0: return &v.state @@ -5123,6 +5474,42 @@ func file_pkg_services_control_service_proto_init() { return nil } } + file_pkg_services_control_service_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheResponse_Body_Status); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -5130,7 +5517,7 @@ func file_pkg_services_control_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_service_proto_rawDesc, NumEnums: 1, - NumMessages: 66, + NumMessages: 71, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index e3b507387..34c040f2a 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -56,6 +56,9 @@ service ControlService { // Remove local access policy engine overrides stored in the node by chaind id. rpc RemoveChainLocalOverride (RemoveChainLocalOverrideRequest) returns (RemoveChainLocalOverrideResponse); + + // Flush objects from write-cache and move it to degraded read only mode. + rpc SealWriteCache(SealWriteCacheRequest) returns (SealWriteCacheResponse); } // Health check request. @@ -525,3 +528,32 @@ message RemoveChainLocalOverrideResponse { Signature signature = 2; } + +message SealWriteCacheRequest { + // Request body structure. + message Body { + // ID of the shard. + repeated bytes shard_ID = 1; + + // Flag indicating whether object read errors should be ignored. + bool ignore_errors = 2; + } + + Body body = 1; + Signature signature = 2; +} + +message SealWriteCacheResponse { + message Body { + message Status { + bytes shard_ID = 1; + bool success = 2; + string error = 3; + } + repeated Status results = 1; + } + + Body body = 1; + + Signature signature = 2; +} \ No newline at end of file diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 3ace081e4..4ce6791f8 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -2755,3 +2755,216 @@ func (x *RemoveChainLocalOverrideResponse) ReadSignedData(buf []byte) ([]byte, e func (x *RemoveChainLocalOverrideResponse) SetSignature(sig *Signature) { x.Signature = sig } + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *SealWriteCacheRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.RepeatedBytesSize(1, x.Shard_ID) + size += proto.BoolSize(2, x.IgnoreErrors) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *SealWriteCacheRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) + offset += proto.BoolMarshal(2, buf[offset:], x.IgnoreErrors) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *SealWriteCacheRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *SealWriteCacheRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *SealWriteCacheRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *SealWriteCacheRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *SealWriteCacheRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *SealWriteCacheResponse_Body_Status) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.BytesSize(1, x.Shard_ID) + size += proto.BoolSize(2, x.Success) + size += proto.StringSize(3, x.Error) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *SealWriteCacheResponse_Body_Status) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.BytesMarshal(1, buf[offset:], x.Shard_ID) + offset += proto.BoolMarshal(2, buf[offset:], x.Success) + offset += proto.StringMarshal(3, buf[offset:], x.Error) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *SealWriteCacheResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + for i := range x.Results { + size += proto.NestedStructureSize(1, x.Results[i]) + } + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *SealWriteCacheResponse_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + for i := range x.Results { + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Results[i]) + } + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *SealWriteCacheResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *SealWriteCacheResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *SealWriteCacheResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *SealWriteCacheResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *SealWriteCacheResponse) SetSignature(sig *Signature) { + x.Signature = sig +} diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index c3976c54a..e09f6750b 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -35,6 +35,7 @@ const ( ControlService_GetChainLocalOverride_FullMethodName = "/control.ControlService/GetChainLocalOverride" ControlService_ListChainLocalOverrides_FullMethodName = "/control.ControlService/ListChainLocalOverrides" ControlService_RemoveChainLocalOverride_FullMethodName = "/control.ControlService/RemoveChainLocalOverride" + ControlService_SealWriteCache_FullMethodName = "/control.ControlService/SealWriteCache" ) // ControlServiceClient is the client API for ControlService service. @@ -74,6 +75,8 @@ type ControlServiceClient interface { ListChainLocalOverrides(ctx context.Context, in *ListChainLocalOverridesRequest, opts ...grpc.CallOption) (*ListChainLocalOverridesResponse, error) // Remove local access policy engine overrides stored in the node by chaind id. RemoveChainLocalOverride(ctx context.Context, in *RemoveChainLocalOverrideRequest, opts ...grpc.CallOption) (*RemoveChainLocalOverrideResponse, error) + // Flush objects from write-cache and move it to degraded read only mode. + SealWriteCache(ctx context.Context, in *SealWriteCacheRequest, opts ...grpc.CallOption) (*SealWriteCacheResponse, error) } type controlServiceClient struct { @@ -228,6 +231,15 @@ func (c *controlServiceClient) RemoveChainLocalOverride(ctx context.Context, in return out, nil } +func (c *controlServiceClient) SealWriteCache(ctx context.Context, in *SealWriteCacheRequest, opts ...grpc.CallOption) (*SealWriteCacheResponse, error) { + out := new(SealWriteCacheResponse) + err := c.cc.Invoke(ctx, ControlService_SealWriteCache_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // ControlServiceServer is the server API for ControlService service. // All implementations should embed UnimplementedControlServiceServer // for forward compatibility @@ -265,6 +277,8 @@ type ControlServiceServer interface { ListChainLocalOverrides(context.Context, *ListChainLocalOverridesRequest) (*ListChainLocalOverridesResponse, error) // Remove local access policy engine overrides stored in the node by chaind id. RemoveChainLocalOverride(context.Context, *RemoveChainLocalOverrideRequest) (*RemoveChainLocalOverrideResponse, error) + // Flush objects from write-cache and move it to degraded read only mode. + SealWriteCache(context.Context, *SealWriteCacheRequest) (*SealWriteCacheResponse, error) } // UnimplementedControlServiceServer should be embedded to have forward compatible implementations. @@ -319,6 +333,9 @@ func (UnimplementedControlServiceServer) ListChainLocalOverrides(context.Context func (UnimplementedControlServiceServer) RemoveChainLocalOverride(context.Context, *RemoveChainLocalOverrideRequest) (*RemoveChainLocalOverrideResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method RemoveChainLocalOverride not implemented") } +func (UnimplementedControlServiceServer) SealWriteCache(context.Context, *SealWriteCacheRequest) (*SealWriteCacheResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SealWriteCache not implemented") +} // UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ControlServiceServer will @@ -619,6 +636,24 @@ func _ControlService_RemoveChainLocalOverride_Handler(srv interface{}, ctx conte return interceptor(ctx, in, info, handler) } +func _ControlService_SealWriteCache_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SealWriteCacheRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).SealWriteCache(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_SealWriteCache_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).SealWriteCache(ctx, req.(*SealWriteCacheRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -690,6 +725,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "RemoveChainLocalOverride", Handler: _ControlService_RemoveChainLocalOverride_Handler, }, + { + MethodName: "SealWriteCache", + Handler: _ControlService_SealWriteCache_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "pkg/services/control/service.proto", From 530249e3bd7ae7f8eff5ac22ecf27782a0470831 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 29 Dec 2023 17:11:36 +0300 Subject: [PATCH 0776/1943] [#893] go.mod: Update neo-go Signed-off-by: Evgenii Stratonikov --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/morph/client/notifications.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index c4266fe21..82ad0212d 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.12.0 github.com/nats-io/nats.go v1.31.0 - github.com/nspcc-dev/neo-go v0.104.0 + github.com/nspcc-dev/neo-go v0.105.0 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 github.com/paulmach/orb v0.9.2 @@ -113,7 +113,7 @@ require ( go.opentelemetry.io/otel/sdk v1.21.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.16.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 490cd4aec..75d2487c4 100644 --- a/go.sum +++ b/go.sum @@ -172,8 +172,8 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nspcc-dev/go-ordered-json v0.0.0-20231123160306-3374ff1e7a3c h1:OOQeE613BH93ICPq3eke5N78gWNeMjcBWkmD2NKyXVg= github.com/nspcc-dev/go-ordered-json v0.0.0-20231123160306-3374ff1e7a3c/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= -github.com/nspcc-dev/neo-go v0.104.0 h1:FGj3Z46yABcFIAI1SCLd1jQSoh+B00h/2VAgEgY1JKQ= -github.com/nspcc-dev/neo-go v0.104.0/go.mod h1:omsUK5PAtG2/nQ3/evs95QEg3wtkj3LH53e0NKtXVwQ= +github.com/nspcc-dev/neo-go v0.105.0 h1:vtNZYFEFySK8zRDhLzQYha849VzWrcKezlnq/oNQg/w= +github.com/nspcc-dev/neo-go v0.105.0/go.mod h1:6pchIHg5okeZO955RxpTh5q0sUI0vtpgPM6Q+no1rlI= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231201072346-6c0c2a6a98b0 h1:GYeU/mq2qfq6fNSuTzgxvlQQUydeCBicjzdnggwtuz0= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231201072346-6c0c2a6a98b0/go.mod h1:J/Mk6+nKeKSW4wygkZQFLQ6SkLOSGX5Ga0RuuuktEag= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= @@ -296,8 +296,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= diff --git a/pkg/morph/client/notifications.go b/pkg/morph/client/notifications.go index a013631df..35204bb36 100644 --- a/pkg/morph/client/notifications.go +++ b/pkg/morph/client/notifications.go @@ -73,7 +73,7 @@ func (c *Client) ReceiveNotaryRequests(txSigner util.Uint160, ch chan<- *result. return "", ErrConnectionLost } - return c.client.ReceiveNotaryRequests(&neorpc.TxFilter{Signer: &txSigner}, ch) + return c.client.ReceiveNotaryRequests(&neorpc.NotaryRequestFilter{Signer: &txSigner}, ch) } // Unsubscribe performs unsubscription for the given subscription ID. From 225fe2d4d5781b69bfe8222dd0a109d97181ad63 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 29 Dec 2023 18:54:38 +0300 Subject: [PATCH 0777/1943] [#894] blobovniczatree: Speedup rebuild test Down from 3s to 300ms. Signed-off-by: Evgenii Stratonikov --- .../blobstor/blobovniczatree/rebuild_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 921515a6a..904971d25 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -46,7 +46,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta WithBlobovniczaShallowWidth(sourceWidth), WithBlobovniczaShallowDepth(sourceDepth), WithRootPath(dir), - WithBlobovniczaSize(100*1024*1024), + WithBlobovniczaSize(100*1024), WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000), WithMoveBatchSize(3)) @@ -56,7 +56,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta eg, egCtx := errgroup.WithContext(context.Background()) storageIDs := make(map[oid.Address][]byte) storageIDsGuard := &sync.Mutex{} - for i := 0; i < 1000; i++ { + for i := 0; i < 100; i++ { eg.Go(func() error { obj := blobstortest.NewObject(1024) data, err := obj.Marshal() @@ -86,10 +86,10 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta WithBlobovniczaShallowWidth(targetWidth), WithBlobovniczaShallowDepth(targetDepth), WithRootPath(dir), - WithBlobovniczaSize(100*1024*1024), + WithBlobovniczaSize(100*1024), WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000), - WithMoveBatchSize(3)) + WithMoveBatchSize(50)) require.NoError(t, b.Open(false)) require.NoError(t, b.Init()) From d5d3d8badb45bb09a13fc4ca9f2c57d920f14616 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 29 Dec 2023 14:16:40 +0300 Subject: [PATCH 0778/1943] [#892] node: Add VSCode debug example Signed-off-by: Dmitrii Stepanov --- Makefile | 26 ++++++++++ README.md | 39 +++++++++++++++ dev/.vscode-example/launch.json | 89 +++++++++++++++++++++++++++++++++ dev/.vscode-example/tasks.json | 19 +++++++ dev/adm/frostfs-adm.yml | 17 +++++++ dev/docker-compose.yml | 16 ++++++ dev/ir/az.json | 69 +++++++++++++++++++++++++ dev/ir/contract.json | 30 +++++++++++ dev/neo-go/config.yml | 4 ++ dev/neo-go/node-wallet.json | 68 +++++++++++++++++++++++++ dev/neo-go/protocol.privnet.yml | 48 ++++++++++++++++++ dev/neo-go/wallet.json | 30 +++++++++++ dev/storage/wallet.json | 32 ++++++++++++ dev/wallet.json | 30 +++++++++++ 14 files changed, 517 insertions(+) create mode 100644 dev/.vscode-example/launch.json create mode 100644 dev/.vscode-example/tasks.json create mode 100644 dev/adm/frostfs-adm.yml create mode 100644 dev/docker-compose.yml create mode 100644 dev/ir/az.json create mode 100644 dev/ir/contract.json create mode 100644 dev/neo-go/config.yml create mode 100644 dev/neo-go/node-wallet.json create mode 100644 dev/neo-go/protocol.privnet.yml create mode 100644 dev/neo-go/wallet.json create mode 100644 dev/storage/wallet.json create mode 100644 dev/wallet.json diff --git a/Makefile b/Makefile index 153640da8..84653a03a 100755 --- a/Makefile +++ b/Makefile @@ -44,6 +44,10 @@ PROTOGEN_FROSTFS_DIR ?= $(PROTOBUF_DIR)/protogen-$(PROTOGEN_FROSTFS_VERSION) STATICCHECK_DIR ?= $(abspath $(BIN))/staticcheck STATICCHECK_VERSION_DIR ?= $(STATICCHECK_DIR)/$(STATICCHECK_VERSION) +FROSTFS_CONTRACTS_PATH=$(abspath ./../frostfs-contract) +LOCODE_DB_PATH=$(abspath ./.cache/locode_db) +LOCODE_DB_VERSION=v0.4.0 + .PHONY: help all images dep clean fmts fumpt imports test lint docker/lint prepare-release debpackage pre-commit unpre-commit @@ -234,3 +238,25 @@ debpackage: debclean: dh clean + +locode-download: + @wget -q -O ./.cache/locode_db.gz 'https://git.frostfs.info/TrueCloudLab/frostfs-locode-db/releases/download/${LOCODE_DB_VERSION}/locode_db.gz' + gzip -dfk ./.cache/locode_db.gz + +env-up: all + docker compose -f dev/docker-compose.yml up -d + @if [ ! -d "$(FROSTFS_CONTRACTS_PATH)" ]; then \ + echo "Frostfs contracts not found"; exit 1; \ + fi + ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph init --contracts ${FROSTFS_CONTRACTS_PATH} + ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet.json --gas 10.0 + @if [ ! -f "$(LOCODE_DB_PATH)" ]; then \ + make locode-download; \ + fi + +env-down: + docker compose -f dev/docker-compose.yml down + docker volume rm -f frostfs-node_neo-go + rm -f ./.cache/.frostfs-ir-state + rm -f ./.cache/.frostfs-node-state + rm -rf ./.cache/storage \ No newline at end of file diff --git a/README.md b/README.md index c3a9bf09c..ff816f2c7 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,45 @@ To make docker images suitable for use in [frostfs-dev-env](https://github.com/T make images ``` +# Debugging + +## VSCode + +To run and debug single node cluster with VSCode: + +1. Clone and build [frostfs-contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract) repository to the same directory level as `frostfs-node`. For example: + +``` +/ +├── src + ├── frostfs-node + └── frostfs-contract +``` +See `frostfs-contract`'s README.md for build instructions. + +2. Copy `launch.json` and `tasks.json` from `dev/.vscode-example` directory to `.vscode` directory. If you already have such files in `.vscode` directory, then merge them manually. + +3. Go to **Run and Debug** (`Ctrl+Shift+D`) and start `IR+Storage node` configuration. + +4. To create container and put object into it run (container and object IDs will be different): + +``` +./bin/frostfs-cli container create -r 127.0.0.1:8080 --wallet ./dev/wallet.json --policy "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" --basic-acl public-read-write --await +Enter password > <- press ENTER, the is no password for wallet +CID: CfPhEuHQ2PRvM4gfBQDC4dWZY3NccovyfcnEdiq2ixju + +./bin/frostfs-cli object put -r 127.0.0.1:8080 --wallet ./dev/wallet.json --file README.md --cid CfPhEuHQ2PRvM4gfBQDC4dWZY3NccovyfcnEdiq2ixju +Enter password > + 4300 / 4300 [===========================================================================================================================================================================================================] 100.00% 0s +[README.md] Object successfully stored + OID: 78sohnudVMnPsczXqsTUcvezosan2YDNVZwDE8Kq5YwU + CID: CfPhEuHQ2PRvM4gfBQDC4dWZY3NccovyfcnEdiq2ixju + +./bin/frostfs-cli object get -r 127.0.0.1:8080 --wallet ./dev/wallet.json --cid CfPhEuHQ2PRvM4gfBQDC4dWZY3NccovyfcnEdiq2ixju --oid 78sohnudVMnPsczXqsTUcvezosan2YDNVZwDE8Kq5YwU +... + +``` + # Contributing Feel free to contribute to this project after reading the [contributing diff --git a/dev/.vscode-example/launch.json b/dev/.vscode-example/launch.json new file mode 100644 index 000000000..cf91c0d24 --- /dev/null +++ b/dev/.vscode-example/launch.json @@ -0,0 +1,89 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "IR", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "cmd/frostfs-ir", + "env": { + "FROSTFS_IR_LOGGER_LEVEL":"info", + "FROSTFS_IR_WALLET_PATH":"${workspaceFolder}/dev/ir/az.json", + "FROSTFS_IR_WALLET_ADDRESS":"Nhfg3TbpwogLvDGVvAvqyThbsHgoSUKwtn", + "FROSTFS_IR_WALLET_PASSWORD":"one", + "FROSTFS_IR_WITHOUT_MAINNET":"true", + "FROSTFS_IR_MORPH_ENDPOINT_CLIENT_0_ADDRESS":"ws://127.0.0.1:30333/ws", + "FROSTFS_IR_MORPH_VALIDATORS":"02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2", + "FROSTFS_IR_TIMERS_EMIT":"50", + "FROSTFS_IR_TIMERS_STOP_ESTIMATION_MUL":"1", + "FROSTFS_IR_TIMERS_STOP_ESTIMATION_DIV":"4", + "FROSTFS_IR_TIMERS_COLLECT_BASIC_INCOME_MUL":"1", + "FROSTFS_IR_TIMERS_COLLECT_BASIC_INCOME_DIV":"2", + "FROSTFS_IR_TIMERS_DISTRIBUTE_BASIC_INCOME_MUL":"3", + "FROSTFS_IR_TIMERS_DISTRIBUTE_BASIC_INCOME_DIV":"4", + "FROSTFS_IR_EMIT_STORAGE_AMOUNT":"1000000000", + "FROSTFS_IR_NETMAP_CLEANER_ENABLED":"true", + "FROSTFS_IR_NETMAP_CLEANER_THRESHOLD":"3", + "FROSTFS_IR_LOCODE_DB_PATH":"${workspaceFolder}/.cache/locode_db", + "FROSTFS_IR_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8090", + "FROSTFS_IR_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/.frostfs-ir-state" + }, + "postDebugTask": "env-down" + }, + { + "name": "Storage node", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "cmd/frostfs-node", + "env": { + "FROSTFS_LOGGER_LEVEL":"debug", + "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", + "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", + "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", + "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet.json", + "FROSTFS_NODE_WALLET_PASSWORD":"", + "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8080", + "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8080", + "FROSTFS_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8081", + "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", + "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", + "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/.frostfs-node-state", + "FROSTFS_TREE_ENABLED":"true", + "FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME":"10", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/wc0", + "FROSTFS_STORAGE_SHARD_0_METABASE_PATH":"${workspaceFolder}/.cache/storage/meta0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/blobovnicza0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/fstree0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/pilorama0", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/wc1", + "FROSTFS_STORAGE_SHARD_1_METABASE_PATH":"${workspaceFolder}/.cache/storage/meta1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/blobovnicza1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/fstree1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/pilorama1" + }, + "postDebugTask": "env-down" + } + ], + "compounds": [ + { + "name": "IR+Storage node", + "configurations": ["IR", "Storage node"], + "preLaunchTask": "env-up", + "stopAll": true + } + ] +} \ No newline at end of file diff --git a/dev/.vscode-example/tasks.json b/dev/.vscode-example/tasks.json new file mode 100644 index 000000000..8c6099a05 --- /dev/null +++ b/dev/.vscode-example/tasks.json @@ -0,0 +1,19 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "shell", + "label": "env-up", + "command": "make env-up", + "group": "build", + "detail": "Up debug environment" + }, + { + "type": "shell", + "label": "env-down", + "command": "make env-down", + "group": "build", + "detail": "Down debug environment" + } + ] +} \ No newline at end of file diff --git a/dev/adm/frostfs-adm.yml b/dev/adm/frostfs-adm.yml new file mode 100644 index 000000000..5dc87e70c --- /dev/null +++ b/dev/adm/frostfs-adm.yml @@ -0,0 +1,17 @@ +rpc-endpoint: http://127.0.0.1:30333 +alphabet-wallets: ./dev/ir +network: + max_object_size: 5242880 + epoch_duration: 60 + basic_income_rate: 100000000 + homomorphic_hash_disabled: false + maintenance_mode_allowed: true + fee: + audit: 10000 + candidate: 10000000000 + container: 0 + container_alias: 0 + withdraw: 100000000 +credentials: + az: "one" + contract: "one" diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml new file mode 100644 index 000000000..7e68ea02e --- /dev/null +++ b/dev/docker-compose.yml @@ -0,0 +1,16 @@ +--- + +version: "2.4" +services: + neo-go: + image: nspccdev/neo-go:0.103.0 + container_name: neo-go + command: ["node", "--config-path", "/config", "--privnet", "--debug"] + stop_signal: SIGKILL + ports: + - 30333:30333 + volumes: + - ./neo-go/protocol.privnet.yml:/config/protocol.privnet.yml + - ./neo-go/node-wallet.json:/wallets/node-wallet.json + - ./neo-go/config.yml:/wallets/config.yml + - ./neo-go/wallet.json:/wallets/wallet.json diff --git a/dev/ir/az.json b/dev/ir/az.json new file mode 100644 index 000000000..a16aec295 --- /dev/null +++ b/dev/ir/az.json @@ -0,0 +1,69 @@ +{ + "version": "3.0", + "name":null, + "accounts": [ + { + "address": "Nhfg3TbpwogLvDGVvAvqyThbsHgoSUKwtn", + "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", + "label": "single", + "contract": { + "script": "DCECs2Ir9AF73+MXxYrtX0x1PyBrfbiWBG+n13S7xL9/jcJBVuezJw==", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "extra":null, + "isDefault": false + }, + { + "address": "NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP", + "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", + "label": "consensus", + "contract": { + "script": "EQwhArNiK/QBe9/jF8WK7V9MdT8ga324lgRvp9d0u8S/f43CEUGe0Nw6", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "extra":null, + "isDefault": false + }, + { + "address": "NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP", + "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", + "label": "committee", + "contract": { + "script": "EQwhArNiK/QBe9/jF8WK7V9MdT8ga324lgRvp9d0u8S/f43CEUGe0Nw6", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "extra":null, + "isDefault": true + } + ], + "scrypt": { + "n": 16384, + "r": 8, + "p": 8 + }, + "extra": { + "Tokens": null + } +} + diff --git a/dev/ir/contract.json b/dev/ir/contract.json new file mode 100644 index 000000000..310b77bd7 --- /dev/null +++ b/dev/ir/contract.json @@ -0,0 +1,30 @@ +{ + "version": "3.0", + "accounts": [ + { + "address": "Nhfg3TbpwogLvDGVvAvqyThbsHgoSUKwtn", + "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", + "label": "", + "contract": { + "script": "DCECs2Ir9AF73+MXxYrtX0x1PyBrfbiWBG+n13S7xL9/jcJBVuezJw==", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "isDefault": false + } + ], + "scrypt": { + "n": 16384, + "r": 8, + "p": 8 + }, + "extra": { + "Tokens": null + } +} diff --git a/dev/neo-go/config.yml b/dev/neo-go/config.yml new file mode 100644 index 000000000..7b4bb29d7 --- /dev/null +++ b/dev/neo-go/config.yml @@ -0,0 +1,4 @@ +--- + +Path: "/wallets/node-wallet.json" +Password: "one" diff --git a/dev/neo-go/node-wallet.json b/dev/neo-go/node-wallet.json new file mode 100644 index 000000000..8e88b432c --- /dev/null +++ b/dev/neo-go/node-wallet.json @@ -0,0 +1,68 @@ +{ + "version": "3.0", + "name":null, + "accounts": [ + { + "address": "Nhfg3TbpwogLvDGVvAvqyThbsHgoSUKwtn", + "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", + "label": "single", + "contract": { + "script": "DCECs2Ir9AF73+MXxYrtX0x1PyBrfbiWBG+n13S7xL9/jcJBVuezJw==", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "extra":null, + "isDefault": false + }, + { + "address": "NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP", + "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", + "label": "consensus", + "contract": { + "script": "EQwhArNiK/QBe9/jF8WK7V9MdT8ga324lgRvp9d0u8S/f43CEUGe0Nw6", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "extra":null, + "isDefault": false + }, + { + "address": "NfgHwwTi3wHAS8aFAN243C5vGbkYDpqLHP", + "key": "6PYM8VdX2BSm7BSXKzV4Fz6S3R9cDLLWNrD9nMjxW352jEv3fsC8N3wNLY", + "label": "committee", + "contract": { + "script": "EQwhArNiK/QBe9/jF8WK7V9MdT8ga324lgRvp9d0u8S/f43CEUGe0Nw6", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "extra":null, + "isDefault": true + } + ], + "scrypt": { + "n": 16384, + "r": 8, + "p": 8 + }, + "extra": { + "Tokens": null + } +} diff --git a/dev/neo-go/protocol.privnet.yml b/dev/neo-go/protocol.privnet.yml new file mode 100644 index 000000000..8aaf774a3 --- /dev/null +++ b/dev/neo-go/protocol.privnet.yml @@ -0,0 +1,48 @@ +ProtocolConfiguration: + Magic: 15405 + MaxTraceableBlocks: 200000 + TimePerBlock: 1s + MemPoolSize: 50000 + StandbyCommittee: + - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 + ValidatorsCount: 1 + SeedList: + - 0.0.0.0:20333 + VerifyTransactions: true + StateRootInHeader: true + P2PSigExtensions: true + +ApplicationConfiguration: + SkipBlockVerification: false + DBConfiguration: + Type: "boltdb" + BoltDBOptions: + FilePath: "./db/morph.bolt" + P2P: + Addresses: + - ":20333" + DialTimeout: 3s + ProtoTickInterval: 2s + PingInterval: 30s + PingTimeout: 90s + MaxPeers: 10 + AttemptConnPeers: 5 + MinPeers: 0 + Relay: true + Consensus: + Enabled: true + UnlockWallet: + Path: "./wallets/node-wallet.json" + Password: "one" + RPC: + Addresses: + - "0.0.0.0:30333" + Enabled: true + SessionEnabled: true + EnableCORSWorkaround: false + MaxGasInvoke: 100 + P2PNotary: + Enabled: true + UnlockWallet: + Path: "./wallets/node-wallet.json" + Password: "one" diff --git a/dev/neo-go/wallet.json b/dev/neo-go/wallet.json new file mode 100644 index 000000000..ce68d604c --- /dev/null +++ b/dev/neo-go/wallet.json @@ -0,0 +1,30 @@ +{ + "version": "3.0", + "accounts": [ + { + "address": "NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM", + "key": "6PYP7YrwGnLuu4WYQbEe3WJiC44aKmqwqawLsp7H3oh5vocS9xTv2ZfTp3", + "label": "", + "contract": { + "script": "DCEDGmxvu98CyjUXRfqGubpalFLXhaxPf8K3VIyipGxPz0pBVuezJw==", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "isdefault": false + } + ], + "scrypt": { + "n": 16384, + "r": 8, + "p": 8 + }, + "extra": { + "Tokens": null + } +} diff --git a/dev/storage/wallet.json b/dev/storage/wallet.json new file mode 100644 index 000000000..e5b6bb371 --- /dev/null +++ b/dev/storage/wallet.json @@ -0,0 +1,32 @@ +{ + "version":"3.0", + "accounts":[ + { + "address":"NejLbQpojKJWec4NQRMBhzsrmCyhXfGJJe", + "key":"6PYSS8ccmBcttfcw2YJh8VcNSoeQbQLuJLQ7HoKeYF5roRmGs9LUvmKcWz", + "label":"", + "contract":{ + "script":"DCECK7QEHFDWB/+HHex+TNd3g4jg6mhJ2EzL2aqPMuFqgTFBVuezJw==", + "parameters":[ + { + "name":"parameter0", + "type":"Signature" + } + ], + "deployed":false + }, + "lock":false, + "extra":null, + "isDefault":true + } + ], + "name":null, + "scrypt":{ + "n":16384, + "r":8, + "p":8 + }, + "extra":{ + "Tokens":null + } + } diff --git a/dev/wallet.json b/dev/wallet.json new file mode 100644 index 000000000..ce68d604c --- /dev/null +++ b/dev/wallet.json @@ -0,0 +1,30 @@ +{ + "version": "3.0", + "accounts": [ + { + "address": "NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM", + "key": "6PYP7YrwGnLuu4WYQbEe3WJiC44aKmqwqawLsp7H3oh5vocS9xTv2ZfTp3", + "label": "", + "contract": { + "script": "DCEDGmxvu98CyjUXRfqGubpalFLXhaxPf8K3VIyipGxPz0pBVuezJw==", + "parameters": [ + { + "name": "parameter0", + "type": "Signature" + } + ], + "deployed": false + }, + "lock": false, + "isdefault": false + } + ], + "scrypt": { + "n": 16384, + "r": 8, + "p": 8 + }, + "extra": { + "Tokens": null + } +} From dfd62ca6b1e20ee3a091da36fd77f7c262d6af5d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 25 Dec 2023 10:15:26 +0300 Subject: [PATCH 0779/1943] [#864] metabase: Refactor delete/inhume Available -> Logic, Raw -> Phy for delete/inhume results. Use single counter instead of vectors. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/counter.go | 2 +- .../metabase/counter_test.go | 14 ++-- pkg/local_object_storage/metabase/delete.go | 84 +++++++++---------- pkg/local_object_storage/metabase/inhume.go | 20 ++--- pkg/local_object_storage/shard/delete.go | 14 ++-- pkg/local_object_storage/shard/gc.go | 12 +-- pkg/local_object_storage/shard/inhume.go | 2 +- 7 files changed, 71 insertions(+), 77 deletions(-) diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index 96d80077d..581df7a59 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -239,7 +239,7 @@ func (db *DB) updateShardObjectCounter(tx *bbolt.Tx, typ objectType, delta uint6 return b.Put(counterKey, newCounter) } -func (db *DB) updateContainerCounter(tx *bbolt.Tx, delta map[cid.ID]ObjectCounters, inc bool) error { // TODO #838 +func (db *DB) updateContainerCounter(tx *bbolt.Tx, delta map[cid.ID]ObjectCounters, inc bool) error { b := tx.Bucket(containerCounterBucketName) if b == nil { return nil diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 9241c97a0..2f40daaae 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -91,7 +91,7 @@ func TestCounters(t *testing.T) { res, err := db.Delete(context.Background(), prm) require.NoError(t, err) - require.Equal(t, uint64(1), res.AvailableObjectsRemoved()) + require.Equal(t, uint64(1), res.LogicCount()) c, err := db.ObjectCounters() require.NoError(t, err) @@ -161,7 +161,7 @@ func TestCounters(t *testing.T) { res, err := db.Inhume(context.Background(), prm) require.NoError(t, err) - require.Equal(t, uint64(len(inhumedObjs)), res.AvailableInhumed()) + require.Equal(t, uint64(len(inhumedObjs)), res.LogicInhumed()) require.Equal(t, uint64(len(inhumedObjs)), res.UserInhumed()) c, err := db.ObjectCounters() @@ -389,7 +389,7 @@ func TestCounters_Expired(t *testing.T) { inhumeRes, err := db.Inhume(context.Background(), inhumePrm) require.NoError(t, err) - require.Equal(t, uint64(1), inhumeRes.AvailableInhumed()) + require.Equal(t, uint64(1), inhumeRes.LogicInhumed()) require.Equal(t, uint64(1), inhumeRes.UserInhumed()) c, err = db.ObjectCounters() @@ -423,8 +423,8 @@ func TestCounters_Expired(t *testing.T) { deleteRes, err := db.Delete(context.Background(), deletePrm) require.NoError(t, err) - require.Zero(t, deleteRes.AvailableObjectsRemoved()) - require.Zero(t, deleteRes.UserObjectsRemoved()) + require.Zero(t, deleteRes.LogicCount()) + require.Zero(t, deleteRes.UserCount()) if v, ok := exp[oo[0].Container()]; ok { v.Phy-- @@ -456,8 +456,8 @@ func TestCounters_Expired(t *testing.T) { deleteRes, err = db.Delete(context.Background(), deletePrm) require.NoError(t, err) - require.Equal(t, uint64(1), deleteRes.AvailableObjectsRemoved()) - require.Equal(t, uint64(1), deleteRes.UserObjectsRemoved()) + require.Equal(t, uint64(1), deleteRes.LogicCount()) + require.Equal(t, uint64(1), deleteRes.UserCount()) if v, ok := exp[oo[0].Container()]; ok { v.Phy-- diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 6a5661ed1..b2210b55c 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -27,22 +27,22 @@ type DeletePrm struct { // DeleteRes groups the resulting values of Delete operation. type DeleteRes struct { - rawRemoved uint64 - availableRemoved uint64 - userRemoved uint64 - sizes []uint64 - availableSizes []uint64 - removedByCnrID map[cid.ID]ObjectCounters + phyCount uint64 + logicCount uint64 + userCount uint64 + phySize uint64 + logicSize uint64 + removedByCnrID map[cid.ID]ObjectCounters } -// AvailableObjectsRemoved returns the number of removed available +// LogicCount returns the number of removed logic // objects. -func (d DeleteRes) AvailableObjectsRemoved() uint64 { - return d.availableRemoved +func (d DeleteRes) LogicCount() uint64 { + return d.logicCount } -func (d DeleteRes) UserObjectsRemoved() uint64 { - return d.userRemoved +func (d DeleteRes) UserCount() uint64 { + return d.userCount } // RemovedByCnrID returns the number of removed objects by container ID. @@ -50,19 +50,19 @@ func (d DeleteRes) RemovedByCnrID() map[cid.ID]ObjectCounters { return d.removedByCnrID } -// RawObjectsRemoved returns the number of removed raw objects. -func (d DeleteRes) RawObjectsRemoved() uint64 { - return d.rawRemoved +// PhyCount returns the number of removed physical objects. +func (d DeleteRes) PhyCount() uint64 { + return d.phyCount } -// RemovedPhysicalObjectSizes returns the sizes of removed physical objects. -func (d DeleteRes) RemovedPhysicalObjectSizes() []uint64 { - return d.sizes +// PhySize returns the size of removed physical objects. +func (d DeleteRes) PhySize() uint64 { + return d.phySize } -// RemovedLogicalObjectSizes returns the sizes of removed logical objects. -func (d DeleteRes) RemovedLogicalObjectSizes() []uint64 { - return d.availableSizes +// LogicSize returns the size of removed logical objects. +func (d DeleteRes) LogicSize() uint64 { + return d.logicSize } // SetAddresses is a Delete option to set the addresses of the objects to delete. @@ -129,8 +129,6 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { // references of the split objects. func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) { res := DeleteRes{ - sizes: make([]uint64, len(addrs)), - availableSizes: make([]uint64, len(addrs)), removedByCnrID: make(map[cid.ID]ObjectCounters), } refCounter := make(referenceCounter, len(addrs)) @@ -162,22 +160,22 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) } func (db *DB) updateCountersDelete(tx *bbolt.Tx, res DeleteRes) error { - if res.rawRemoved > 0 { - err := db.updateShardObjectCounter(tx, phy, res.rawRemoved, false) + if res.phyCount > 0 { + err := db.updateShardObjectCounter(tx, phy, res.phyCount, false) if err != nil { return fmt.Errorf("could not decrease phy object counter: %w", err) } } - if res.availableRemoved > 0 { - err := db.updateShardObjectCounter(tx, logical, res.availableRemoved, false) + if res.logicCount > 0 { + err := db.updateShardObjectCounter(tx, logical, res.logicCount, false) if err != nil { return fmt.Errorf("could not decrease logical object counter: %w", err) } } - if res.userRemoved > 0 { - err := db.updateShardObjectCounter(tx, user, res.userRemoved, false) + if res.userCount > 0 { + err := db.updateShardObjectCounter(tx, user, res.userCount, false) if err != nil { return fmt.Errorf("could not decrease user object counter: %w", err) } @@ -190,7 +188,7 @@ func (db *DB) updateCountersDelete(tx *bbolt.Tx, res DeleteRes) error { } func applyDeleteSingleResult(r deleteSingleResult, res *DeleteRes, addrs []oid.Address, i int) { - if r.Removed { + if r.Phy { if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { v.Phy++ res.removedByCnrID[addrs[i].Container()] = v @@ -200,11 +198,11 @@ func applyDeleteSingleResult(r deleteSingleResult, res *DeleteRes, addrs []oid.A } } - res.rawRemoved++ - res.sizes[i] = r.Size + res.phyCount++ + res.phySize += r.Size } - if r.Available { + if r.Logic { if v, ok := res.removedByCnrID[addrs[i].Container()]; ok { v.Logic++ res.removedByCnrID[addrs[i].Container()] = v @@ -214,8 +212,8 @@ func applyDeleteSingleResult(r deleteSingleResult, res *DeleteRes, addrs []oid.A } } - res.availableRemoved++ - res.availableSizes[i] = r.Size + res.logicCount++ + res.logicSize += r.Size } if r.User { @@ -228,15 +226,15 @@ func applyDeleteSingleResult(r deleteSingleResult, res *DeleteRes, addrs []oid.A } } - res.userRemoved++ + res.userCount++ } } type deleteSingleResult struct { - Removed bool - Available bool - User bool - Size uint64 + Phy bool + Logic bool + User bool + Size uint64 } // delete removes object indexes from the metabase. Counts the references @@ -302,10 +300,10 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter } return deleteSingleResult{ - Removed: true, - Available: removeAvailableObject, - User: isUserObject && removeAvailableObject, - Size: obj.PayloadSize(), + Phy: true, + Logic: removeAvailableObject, + User: isUserObject && removeAvailableObject, + Size: obj.PayloadSize(), }, nil } diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 21db9a629..3b2f01673 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -37,17 +37,17 @@ type DeletionInfo struct { // InhumeRes encapsulates results of Inhume operation. type InhumeRes struct { - deletedLockObj []oid.Address - availableInhumed uint64 - userInhumed uint64 - inhumedByCnrID map[cid.ID]ObjectCounters - deletionDetails []DeletionInfo + deletedLockObj []oid.Address + logicInhumed uint64 + userInhumed uint64 + inhumedByCnrID map[cid.ID]ObjectCounters + deletionDetails []DeletionInfo } -// AvailableInhumed return number of available object +// LogicInhumed return number of logic object // that have been inhumed. -func (i InhumeRes) AvailableInhumed() uint64 { - return i.availableInhumed +func (i InhumeRes) LogicInhumed() uint64 { + return i.logicInhumed } func (i InhumeRes) UserInhumed() uint64 { @@ -87,7 +87,7 @@ func (i *InhumeRes) storeDeletionInfo(containerID cid.ID, deletedSize uint64, is CID: containerID, IsUser: isUser, }) - i.availableInhumed++ + i.logicInhumed++ if isUser { i.userInhumed++ } @@ -265,7 +265,7 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes } func (db *DB) applyInhumeResToCounters(tx *bbolt.Tx, res *InhumeRes) error { - if err := db.updateShardObjectCounter(tx, logical, res.AvailableInhumed(), false); err != nil { + if err := db.updateShardObjectCounter(tx, logical, res.LogicInhumed(), false); err != nil { return err } if err := db.updateShardObjectCounter(tx, user, res.UserInhumed(), false); err != nil { diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 1c510bdb7..44f6c6b48 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -141,16 +141,12 @@ func (s *Shard) deleteFromMetabase(ctx context.Context, addr oid.Address) error if err != nil { return err } - s.decObjectCounterBy(physical, res.RawObjectsRemoved()) - s.decObjectCounterBy(logical, res.AvailableObjectsRemoved()) - s.decObjectCounterBy(user, res.UserObjectsRemoved()) + s.decObjectCounterBy(physical, res.PhyCount()) + s.decObjectCounterBy(logical, res.LogicCount()) + s.decObjectCounterBy(user, res.UserCount()) s.decContainerObjectCounter(res.RemovedByCnrID()) - removedPayload := res.RemovedPhysicalObjectSizes()[0] - logicalRemovedPayload := res.RemovedLogicalObjectSizes()[0] - if logicalRemovedPayload > 0 { - s.addToContainerSize(addr.Container().EncodeToString(), -int64(logicalRemovedPayload)) - } - s.addToPayloadSize(-int64(removedPayload)) + s.addToContainerSize(addr.Container().EncodeToString(), -int64(res.LogicSize())) + s.addToPayloadSize(-int64(res.PhySize())) return nil } diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 0f978f26e..d82db82b9 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -414,8 +414,8 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } - s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeRegular) - s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.gc.metrics.AddInhumedObjectCount(res.LogicInhumed(), objectTypeRegular) + s.decObjectCounterBy(logical, res.LogicInhumed()) s.decObjectCounterBy(user, res.UserInhumed()) s.decContainerObjectCounter(res.InhumedByCnrID()) @@ -629,8 +629,8 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston return } - s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeTombstone) - s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.gc.metrics.AddInhumedObjectCount(res.LogicInhumed(), objectTypeTombstone) + s.decObjectCounterBy(logical, res.LogicInhumed()) s.decObjectCounterBy(user, res.UserInhumed()) s.decContainerObjectCounter(res.InhumedByCnrID()) @@ -677,8 +677,8 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] return } - s.gc.metrics.AddInhumedObjectCount(res.AvailableInhumed(), objectTypeLock) - s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.gc.metrics.AddInhumedObjectCount(res.LogicInhumed(), objectTypeLock) + s.decObjectCounterBy(logical, res.LogicInhumed()) s.decObjectCounterBy(user, res.UserInhumed()) s.decContainerObjectCounter(res.InhumedByCnrID()) diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 48cfaa98e..746177c3a 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -121,7 +121,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { s.m.RUnlock() - s.decObjectCounterBy(logical, res.AvailableInhumed()) + s.decObjectCounterBy(logical, res.LogicInhumed()) s.decObjectCounterBy(user, res.UserInhumed()) s.decContainerObjectCounter(res.InhumedByCnrID()) From d75e7e9a21bf883870d4c39f8abf325976a32b5f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Dec 2023 18:58:36 +0300 Subject: [PATCH 0780/1943] [#864] engine: Drop container size metric if container deleted Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/container.go | 2 + internal/logs/logs.go | 7 + pkg/local_object_storage/engine/engine.go | 35 ++++- pkg/local_object_storage/engine/inhume.go | 97 ++++++++++++++ pkg/local_object_storage/engine/metrics.go | 1 + pkg/local_object_storage/engine/shards.go | 1 + pkg/local_object_storage/metabase/counter.go | 130 +++++++++++++++++++ pkg/local_object_storage/shard/container.go | 26 ++++ pkg/local_object_storage/shard/control.go | 1 + pkg/local_object_storage/shard/gc.go | 25 ++++ pkg/local_object_storage/shard/shard.go | 23 +++- pkg/metrics/engine.go | 5 + 12 files changed, 347 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 2b9bdd71b..898a4eef9 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -45,6 +45,8 @@ func initContainerService(_ context.Context, c *cfg) { c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { containerGRPC.RegisterContainerServiceServer(s, server) }) + + c.cfgObject.cfgLocalStorage.localStorage.SetContainerSource(cnrRdr) } func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc containerCore.Source) (*morphContainerReader, *morphContainerWriter) { diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 6ead83700..3075e66e9 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -557,4 +557,11 @@ const ( BlobovniczaSavingCountersToMetaFailed = "saving counters to blobovnicza's meta failed" ObjectRemovalFailureExistsInWritecache = "can't remove object: object must be flushed from writecache" FailedToReportStatusToSystemd = "failed to report status to systemd" + ShardGCCollectingExpiredMetricsStarted = "collecting expired metrics started" + ShardGCCollectingExpiredMetricsCompleted = "collecting expired metrics completed" + ShardGCFailedToCollectZeroSizeContainers = "failed to collect zero-size containers" + EngineFailedToCheckContainerAvailability = "failed to check container availability" + EngineFailedToGetContainerSize = "failed to get container size" + EngineFailedToDeleteContainerSize = "failed to delete container size" + EngineInterruptProcessingZeroSizeContainers = "interrupt processing zero-size containers" ) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 026a7c336..e03a08abc 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -7,12 +7,14 @@ import ( "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" ) @@ -218,14 +220,18 @@ type cfg struct { lowMem bool rebuildWorkersCount uint32 + + containerSource atomic.Pointer[containerSource] } func defaultCfg() *cfg { - return &cfg{ + res := &cfg{ log: &logger.Logger{Logger: zap.L()}, shardPoolSize: 20, rebuildWorkersCount: 100, } + res.containerSource.Store(&containerSource{}) + return res } // New creates, initializes and returns new StorageEngine instance. @@ -288,3 +294,30 @@ func WithRebuildWorkersCount(count uint32) Option { c.rebuildWorkersCount = count } } + +// SetContainerSource sets container source. +func (e *StorageEngine) SetContainerSource(cs container.Source) { + e.containerSource.Store(&containerSource{cs: cs}) +} + +type containerSource struct { + cs container.Source +} + +func (s *containerSource) IsContainerAvailable(ctx context.Context, id cid.ID) (bool, error) { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + + if s == nil || s.cs == nil { + return true, nil + } + + wasRemoved, err := container.WasRemoved(s.cs, id) + if err != nil { + return false, err + } + return !wasRemoved, nil +} diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 293746f70..6750adec2 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" @@ -259,3 +260,99 @@ func (e *StorageEngine) processDeletedLocks(ctx context.Context, lockers []oid.A } }) } + +func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid.ID) { + if len(ids) == 0 { + return + } + + idMap, err := e.selectNonExistedIDs(ctx, ids) + if err != nil { + return + } + + if len(idMap) == 0 { + return + } + + var failed bool + var prm shard.ContainerSizePrm + e.iterateOverUnsortedShards(func(sh hashedShard) bool { + select { + case <-ctx.Done(): + e.log.Info(logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(ctx.Err())) + failed = true + return true + default: + } + + var drop []cid.ID + for id := range idMap { + prm.SetContainerID(id) + s, err := sh.ContainerSize(prm) + if err != nil { + e.log.Warn(logs.EngineFailedToGetContainerSize, zap.Stringer("container_id", id), zap.Error(err)) + failed = true + return true + } + if s.Size() > 0 { + drop = append(drop, id) + } + } + for _, id := range drop { + delete(idMap, id) + } + + return len(idMap) == 0 + }) + + if failed || len(idMap) == 0 { + return + } + + e.iterateOverUnsortedShards(func(sh hashedShard) bool { + select { + case <-ctx.Done(): + e.log.Info(logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(ctx.Err())) + failed = true + return true + default: + } + + for id := range idMap { + if err := sh.DeleteContainerSize(ctx, id); err != nil { + e.log.Warn(logs.EngineFailedToDeleteContainerSize, zap.Stringer("container_id", id), zap.Error(err)) + failed = true + return true + } + } + + return false + }) + + if failed { + return + } + + for id := range idMap { + e.metrics.DeleteContainerSize(id.EncodeToString()) + } +} + +func (e *StorageEngine) selectNonExistedIDs(ctx context.Context, ids []cid.ID) (map[cid.ID]struct{}, error) { + cs := e.containerSource.Load() + + idMap := make(map[cid.ID]struct{}) + for _, id := range ids { + isAvailable, err := cs.IsContainerAvailable(ctx, id) + if err != nil { + e.log.Warn(logs.EngineFailedToCheckContainerAvailability, zap.Stringer("container_id", id), zap.Error(err)) + return nil, err + } + if isAvailable { + continue + } + idMap[id] = struct{}{} + } + return idMap, nil +} diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 363d098f6..e4cc504fa 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -16,6 +16,7 @@ type MetricRegister interface { SetMode(shardID string, mode mode.Mode) AddToContainerSize(cnrID string, size int64) + DeleteContainerSize(cnrID string) AddToPayloadCounter(shardID string, size int64) IncErrorCounter(shardID string) ClearErrorCounter(shardID string) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 7e3e85e81..cb3a4eca5 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -119,6 +119,7 @@ func (e *StorageEngine) createShard(ctx context.Context, opts []shard.Option) (* shard.WithDeletedLockCallback(e.processDeletedLocks), shard.WithReportErrorFunc(e.reportShardErrorBackground), shard.WithRebuildWorkerLimiter(e.rebuildLimiter), + shard.WithZeroSizeCallback(e.processZeroSizeContainers), )...) if err := sh.UpdateID(ctx); err != nil { diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index 581df7a59..271e78fe4 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -14,6 +14,8 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) var ( @@ -480,3 +482,131 @@ func IsUserObject(obj *objectSDK.Object) bool { (obj.SplitID() == nil || (hasParentID && len(obj.Children()) == 0)) } + +// ZeroSizeContainers returns containers with size = 0. +func (db *DB) ZeroSizeContainers(ctx context.Context) ([]cid.ID, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("ZeroSizeContainers", time.Since(startedAt), success) + }() + + ctx, span := tracing.StartSpanFromContext(ctx, "metabase.ZeroSizeContainers") + defer span.End() + + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + var result []cid.ID + lastKey := make([]byte, cidSize) + + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + completed, err := db.containerSizesNextBatch(lastKey, func(contID cid.ID, size uint64) { + if size == 0 { + result = append(result, contID) + } + }) + if err != nil { + return nil, err + } + if completed { + break + } + } + + success = true + return result, nil +} + +func (db *DB) containerSizesNextBatch(lastKey []byte, f func(cid.ID, uint64)) (bool, error) { + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return false, ErrDegradedMode + } + + counter := 0 + const batchSize = 1000 + + err := db.boltDB.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(containerVolumeBucketName) + c := b.Cursor() + var key, value []byte + for key, value = c.Seek(lastKey); key != nil; key, value = c.Next() { + if bytes.Equal(lastKey, key) { + continue + } + copy(lastKey, key) + + size := parseContainerSize(value) + var id cid.ID + if err := id.Decode(key); err != nil { + return err + } + f(id, size) + + counter++ + if counter == batchSize { + break + } + } + + if counter < batchSize { + return ErrInterruptIterator + } + return nil + }) + if err != nil { + if errors.Is(err, ErrInterruptIterator) { + return true, nil + } + return false, metaerr.Wrap(err) + } + return false, nil +} + +func (db *DB) DeleteContainerSize(ctx context.Context, id cid.ID) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("DeleteContainerSize", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.DeleteContainerSize", + trace.WithAttributes( + attribute.Stringer("container_id", id), + )) + defer span.End() + + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return ErrDegradedMode + } + + if db.mode.ReadOnly() { + return ErrReadOnlyMode + } + + err := db.boltDB.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket(containerVolumeBucketName) + + key := make([]byte, cidSize) + id.Encode(key) + return b.Delete(key) + }) + success = err == nil + return metaerr.Wrap(err) +} diff --git a/pkg/local_object_storage/shard/container.go b/pkg/local_object_storage/shard/container.go index 24090e8d8..856885486 100644 --- a/pkg/local_object_storage/shard/container.go +++ b/pkg/local_object_storage/shard/container.go @@ -1,9 +1,13 @@ package shard import ( + "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) type ContainerSizePrm struct { @@ -39,3 +43,25 @@ func (s *Shard) ContainerSize(prm ContainerSizePrm) (ContainerSizeRes, error) { size: size, }, nil } + +func (s *Shard) DeleteContainerSize(ctx context.Context, id cid.ID) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.DeleteContainerSize", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.Stringer("container_id", id), + )) + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.ReadOnly() { + return ErrReadOnlyMode + } + + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } + + return s.metaBase.DeleteContainerSize(ctx, id) +} diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index f103ebc2b..574bc1430 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -155,6 +155,7 @@ func (s *Shard) Init(ctx context.Context) error { s.collectExpiredLocks, s.collectExpiredObjects, s.collectExpiredTombstones, + s.collectExpiredMetrics, }, }, }, diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index d82db82b9..3017dce1f 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" @@ -726,3 +727,27 @@ func (s *Shard) HandleDeletedLocks(lockers []oid.Address) { func (s *Shard) NotificationChannel() chan<- Event { return s.gc.eventChan } + +func (s *Shard) collectExpiredMetrics(ctx context.Context, e Event) { + ctx, span := tracing.StartSpanFromContext(ctx, "shard.collectExpiredMetrics") + defer span.End() + + epoch := e.(newEpoch).epoch + + s.log.Debug(logs.ShardGCCollectingExpiredMetricsStarted, zap.Uint64("epoch", epoch)) + defer s.log.Debug(logs.ShardGCCollectingExpiredMetricsCompleted, zap.Uint64("epoch", epoch)) + + s.collectExpiredContainerSizeMetrics(ctx, epoch) +} + +func (s *Shard) collectExpiredContainerSizeMetrics(ctx context.Context, epoch uint64) { + ids, err := s.metaBase.ZeroSizeContainers(ctx) + if err != nil { + s.log.Warn(logs.ShardGCFailedToCollectZeroSizeContainers, zap.Uint64("epoch", epoch), zap.Error(err)) + return + } + if len(ids) == 0 { + return + } + s.zeroSizeContainersCallback(ctx, ids) +} diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 641b487ed..a83445cfe 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -53,6 +53,9 @@ type ExpiredObjectsCallback func(context.Context, uint64, []oid.Address) // DeletedLockCallback is a callback handling list of deleted LOCK objects. type DeletedLockCallback func(context.Context, []oid.Address) +// ZeroSizeContainersCallback is a callback hanfling list of zero-size containers. +type ZeroSizeContainersCallback func(context.Context, []cid.ID) + // MetricsWriter is an interface that must store shard's metrics. type MetricsWriter interface { // SetObjectCounter must set object counter taking into account object type. @@ -118,6 +121,8 @@ type cfg struct { deletedLockCallBack DeletedLockCallback + zeroSizeContainersCallback ZeroSizeContainersCallback + tsSource TombstoneSource metricsWriter MetricsWriter @@ -129,11 +134,12 @@ type cfg struct { func defaultCfg() *cfg { return &cfg{ - rmBatchSize: 100, - log: &logger.Logger{Logger: zap.L()}, - gcCfg: defaultGCCfg(), - reportErrorFunc: func(string, string, error) {}, - rebuildLimiter: &noopRebuildLimiter{}, + rmBatchSize: 100, + log: &logger.Logger{Logger: zap.L()}, + gcCfg: defaultGCCfg(), + reportErrorFunc: func(string, string, error) {}, + rebuildLimiter: &noopRebuildLimiter{}, + zeroSizeContainersCallback: func(context.Context, []cid.ID) {}, } } @@ -363,6 +369,13 @@ func WithRebuildWorkerLimiter(l RebuildWorkerLimiter) Option { } } +// WithZeroSizeCallback returns option to set zero-size containers callback. +func WithZeroSizeCallback(cb ZeroSizeContainersCallback) Option { + return func(c *cfg) { + c.zeroSizeContainersCallback = cb + } +} + func (s *Shard) fillInfo() { s.cfg.info.MetaBaseInfo = s.metaBase.DumpInfo() s.cfg.info.BlobStorInfo = s.blobStor.DumpInfo() diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 609c30f97..80e912048 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -11,6 +11,7 @@ import ( type EngineMetrics interface { AddMethodDuration(method string, d time.Duration) AddToContainerSize(cnrID string, size int64) + DeleteContainerSize(cnrID string) IncErrorCounter(shardID string) ClearErrorCounter(shardID string) DeleteShardMetrics(shardID string) @@ -79,6 +80,10 @@ func (m *engineMetrics) AddToContainerSize(cnrID string, size int64) { m.containerSize.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) } +func (m *engineMetrics) DeleteContainerSize(cnrID string) { + m.containerSize.DeletePartialMatch(prometheus.Labels{containerIDLabelKey: cnrID}) +} + func (m *engineMetrics) AddToPayloadCounter(shardID string, size int64) { m.payloadSize.With(prometheus.Labels{shardIDLabel: shardID}).Add(float64(size)) } From 4b8b4da6818ecce7700dd432f8a6ed5d8be86106 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Dec 2023 18:23:12 +0300 Subject: [PATCH 0781/1943] [#864] engine: Drop container count metric if container removed Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 3 + pkg/local_object_storage/engine/inhume.go | 82 ++++++++++- pkg/local_object_storage/engine/metrics.go | 1 + pkg/local_object_storage/engine/shards.go | 1 + pkg/local_object_storage/metabase/counter.go | 131 +++++++++++++++++- .../metabase/counter_test.go | 18 +-- pkg/local_object_storage/shard/container.go | 59 ++++++++ pkg/local_object_storage/shard/gc.go | 13 ++ .../shard/metrics_test.go | 6 +- pkg/local_object_storage/shard/shard.go | 29 ++-- pkg/metrics/engine.go | 5 + 11 files changed, 310 insertions(+), 38 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 3075e66e9..420c6612a 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -560,8 +560,11 @@ const ( ShardGCCollectingExpiredMetricsStarted = "collecting expired metrics started" ShardGCCollectingExpiredMetricsCompleted = "collecting expired metrics completed" ShardGCFailedToCollectZeroSizeContainers = "failed to collect zero-size containers" + ShardGCFailedToCollectZeroCountContainers = "failed to collect zero-count containers" EngineFailedToCheckContainerAvailability = "failed to check container availability" EngineFailedToGetContainerSize = "failed to get container size" EngineFailedToDeleteContainerSize = "failed to delete container size" EngineInterruptProcessingZeroSizeContainers = "interrupt processing zero-size containers" + EngineInterruptProcessingZeroCountContainers = "interrupt processing zero-count containers" + EngineFailedToGetContainerCounters = "failed to get container counters" ) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 6750adec2..fc6ee2430 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -266,7 +266,7 @@ func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid return } - idMap, err := e.selectNonExistedIDs(ctx, ids) + idMap, err := e.selectNonExistentIDs(ctx, ids) if err != nil { return } @@ -339,7 +339,85 @@ func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid } } -func (e *StorageEngine) selectNonExistedIDs(ctx context.Context, ids []cid.ID) (map[cid.ID]struct{}, error) { +func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []cid.ID) { + if len(ids) == 0 { + return + } + + idMap, err := e.selectNonExistentIDs(ctx, ids) + if err != nil { + return + } + + if len(idMap) == 0 { + return + } + + var failed bool + var prm shard.ContainerCountPrm + e.iterateOverUnsortedShards(func(sh hashedShard) bool { + select { + case <-ctx.Done(): + e.log.Info(logs.EngineInterruptProcessingZeroCountContainers, zap.Error(ctx.Err())) + failed = true + return true + default: + } + + var drop []cid.ID + for id := range idMap { + prm.ContainerID = id + s, err := sh.ContainerCount(ctx, prm) + if err != nil { + e.log.Warn(logs.EngineFailedToGetContainerCounters, zap.Stringer("container_id", id), zap.Error(err)) + failed = true + return true + } + if s.User > 0 || s.Logic > 0 || s.Phy > 0 { + drop = append(drop, id) + } + } + for _, id := range drop { + delete(idMap, id) + } + + return len(idMap) == 0 + }) + + if failed || len(idMap) == 0 { + return + } + + e.iterateOverUnsortedShards(func(sh hashedShard) bool { + select { + case <-ctx.Done(): + e.log.Info(logs.EngineInterruptProcessingZeroCountContainers, zap.Error(ctx.Err())) + failed = true + return true + default: + } + + for id := range idMap { + if err := sh.DeleteContainerSize(ctx, id); err != nil { + e.log.Warn(logs.EngineFailedToDeleteContainerSize, zap.Stringer("container_id", id), zap.Error(err)) + failed = true + return true + } + } + + return false + }) + + if failed { + return + } + + for id := range idMap { + e.metrics.DeleteContainerCount(id.EncodeToString()) + } +} + +func (e *StorageEngine) selectNonExistentIDs(ctx context.Context, ids []cid.ID) (map[cid.ID]struct{}, error) { cs := e.containerSource.Load() idMap := make(map[cid.ID]struct{}) diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index e4cc504fa..e4117bb1a 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -17,6 +17,7 @@ type MetricRegister interface { AddToContainerSize(cnrID string, size int64) DeleteContainerSize(cnrID string) + DeleteContainerCount(cnrID string) AddToPayloadCounter(shardID string, size int64) IncErrorCounter(shardID string) ClearErrorCounter(shardID string) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index cb3a4eca5..0455471e2 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -120,6 +120,7 @@ func (e *StorageEngine) createShard(ctx context.Context, opts []shard.Option) (* shard.WithReportErrorFunc(e.reportShardErrorBackground), shard.WithRebuildWorkerLimiter(e.rebuildLimiter), shard.WithZeroSizeCallback(e.processZeroSizeContainers), + shard.WithZeroCountCallback(e.processZeroCountContainers), )...) if err := sh.UpdateID(ctx); err != nil { diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index 271e78fe4..915fbd957 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -120,7 +120,9 @@ func (db *DB) ContainerCounters(ctx context.Context) (ContainerCounters, error) default: } - completed, err := db.containerCountersNextBatch(lastKey, &cc) + completed, err := db.containerCountersNextBatch(lastKey, func(id cid.ID, entity ObjectCounters) { + cc.Counts[id] = entity + }) if err != nil { return cc, err } @@ -133,7 +135,7 @@ func (db *DB) ContainerCounters(ctx context.Context) (ContainerCounters, error) return cc, nil } -func (db *DB) containerCountersNextBatch(lastKey []byte, cc *ContainerCounters) (bool, error) { +func (db *DB) containerCountersNextBatch(lastKey []byte, f func(id cid.ID, entity ObjectCounters)) (bool, error) { db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -165,7 +167,7 @@ func (db *DB) containerCountersNextBatch(lastKey []byte, cc *ContainerCounters) if err != nil { return err } - cc.Counts[cnrID] = ent + f(cnrID, ent) counter++ if counter == batchSize { @@ -187,6 +189,43 @@ func (db *DB) containerCountersNextBatch(lastKey []byte, cc *ContainerCounters) return false, nil } +func (db *DB) ContainerCount(ctx context.Context, id cid.ID) (ObjectCounters, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("ContainerCount", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.ContainerCount") + defer span.End() + + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return ObjectCounters{}, ErrDegradedMode + } + + var result ObjectCounters + + err := db.boltDB.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(containerCounterBucketName) + key := make([]byte, cidSize) + id.Encode(key) + v := b.Get(key) + if v == nil { + return nil + } + var err error + result, err = parseContainerCounterValue(v) + return err + }) + + return result, metaerr.Wrap(err) +} + func (db *DB) incCounters(tx *bbolt.Tx, cnrID cid.ID, isUserObject bool) error { if err := db.updateShardObjectCounter(tx, phy, 1, true); err != nil { return fmt.Errorf("could not increase phy object counter: %w", err) @@ -270,9 +309,6 @@ func (*DB) editContainerCounterValue(b *bbolt.Bucket, key []byte, delta ObjectCo entity.Phy = nextValue(entity.Phy, delta.Phy, inc) entity.Logic = nextValue(entity.Logic, delta.Logic, inc) entity.User = nextValue(entity.User, delta.User, inc) - if entity.IsZero() { - return b.Delete(key) - } value := containerCounterValue(entity) return b.Put(key, value) } @@ -610,3 +646,86 @@ func (db *DB) DeleteContainerSize(ctx context.Context, id cid.ID) error { success = err == nil return metaerr.Wrap(err) } + +// ZeroCountContainers returns containers with objects count = 0 in metabase. +func (db *DB) ZeroCountContainers(ctx context.Context) ([]cid.ID, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("ZeroCountContainers", time.Since(startedAt), success) + }() + + ctx, span := tracing.StartSpanFromContext(ctx, "metabase.ZeroCountContainers") + defer span.End() + + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return nil, ErrDegradedMode + } + + var result []cid.ID + + lastKey := make([]byte, cidSize) + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + completed, err := db.containerCountersNextBatch(lastKey, func(id cid.ID, entity ObjectCounters) { + if entity.IsZero() { + result = append(result, id) + } + }) + if err != nil { + return nil, metaerr.Wrap(err) + } + if completed { + break + } + } + success = true + return result, nil +} + +func (db *DB) DeleteContainerCount(ctx context.Context, id cid.ID) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("DeleteContainerCount", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.DeleteContainerCount", + trace.WithAttributes( + attribute.Stringer("container_id", id), + )) + defer span.End() + + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return ErrDegradedMode + } + + if db.mode.ReadOnly() { + return ErrReadOnlyMode + } + + err := db.boltDB.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket(containerCounterBucketName) + + key := make([]byte, cidSize) + id.Encode(key) + return b.Delete(key) + }) + success = err == nil + return metaerr.Wrap(err) +} diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 2f40daaae..4b7b565b3 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -105,11 +105,7 @@ func TestCounters(t *testing.T) { v.Phy-- v.Logic-- v.User-- - if v.IsZero() { - delete(exp, cnrID) - } else { - exp[cnrID] = v - } + exp[cnrID] = v } cc, err := db.ContainerCounters(context.Background()) @@ -428,11 +424,7 @@ func TestCounters_Expired(t *testing.T) { if v, ok := exp[oo[0].Container()]; ok { v.Phy-- - if v.IsZero() { - delete(exp, oo[0].Container()) - } else { - exp[oo[0].Container()] = v - } + exp[oo[0].Container()] = v } oo = oo[1:] @@ -463,11 +455,7 @@ func TestCounters_Expired(t *testing.T) { v.Phy-- v.Logic-- v.User-- - if v.IsZero() { - delete(exp, oo[0].Container()) - } else { - exp[oo[0].Container()] = v - } + exp[oo[0].Container()] = v } oo = oo[1:] diff --git a/pkg/local_object_storage/shard/container.go b/pkg/local_object_storage/shard/container.go index 856885486..364649b50 100644 --- a/pkg/local_object_storage/shard/container.go +++ b/pkg/local_object_storage/shard/container.go @@ -44,6 +44,43 @@ func (s *Shard) ContainerSize(prm ContainerSizePrm) (ContainerSizeRes, error) { }, nil } +type ContainerCountPrm struct { + ContainerID cid.ID +} + +type ContainerCountRes struct { + Phy uint64 + Logic uint64 + User uint64 +} + +func (s *Shard) ContainerCount(ctx context.Context, prm ContainerCountPrm) (ContainerCountRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.ContainerCount", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.Stringer("container_id", prm.ContainerID), + )) + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return ContainerCountRes{}, ErrDegradedMode + } + + counters, err := s.metaBase.ContainerCount(ctx, prm.ContainerID) + if err != nil { + return ContainerCountRes{}, fmt.Errorf("could not get container counters: %w", err) + } + + return ContainerCountRes{ + Phy: counters.Phy, + Logic: counters.Logic, + User: counters.User, + }, nil +} + func (s *Shard) DeleteContainerSize(ctx context.Context, id cid.ID) error { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.DeleteContainerSize", trace.WithAttributes( @@ -65,3 +102,25 @@ func (s *Shard) DeleteContainerSize(ctx context.Context, id cid.ID) error { return s.metaBase.DeleteContainerSize(ctx, id) } + +func (s *Shard) DeleteContainerCount(ctx context.Context, id cid.ID) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.DeleteContainerCount", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.Stringer("container_id", id), + )) + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.ReadOnly() { + return ErrReadOnlyMode + } + + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } + + return s.metaBase.DeleteContainerCount(ctx, id) +} diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 3017dce1f..bfd1f1b11 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -738,6 +738,7 @@ func (s *Shard) collectExpiredMetrics(ctx context.Context, e Event) { defer s.log.Debug(logs.ShardGCCollectingExpiredMetricsCompleted, zap.Uint64("epoch", epoch)) s.collectExpiredContainerSizeMetrics(ctx, epoch) + s.collectExpiredContainerCountMetrics(ctx, epoch) } func (s *Shard) collectExpiredContainerSizeMetrics(ctx context.Context, epoch uint64) { @@ -751,3 +752,15 @@ func (s *Shard) collectExpiredContainerSizeMetrics(ctx context.Context, epoch ui } s.zeroSizeContainersCallback(ctx, ids) } + +func (s *Shard) collectExpiredContainerCountMetrics(ctx context.Context, epoch uint64) { + ids, err := s.metaBase.ZeroCountContainers(ctx) + if err != nil { + s.log.Warn(logs.ShardGCFailedToCollectZeroCountContainers, zap.Uint64("epoch", epoch), zap.Error(err)) + return + } + if len(ids) == 0 { + return + } + s.zeroCountContainersCallback(ctx, ids) +} diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index f59565cb3..9c81c747d 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -336,11 +336,7 @@ func TestCounters(t *testing.T) { v.Logic-- v.Phy-- v.User-- - if v.IsZero() { - delete(expected, cnr) - } else { - expected[cnr] = v - } + expected[cnr] = v } } require.Equal(t, expectedLogicalSizes, mm.containerSizes()) diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index a83445cfe..13f7f689b 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -53,8 +53,8 @@ type ExpiredObjectsCallback func(context.Context, uint64, []oid.Address) // DeletedLockCallback is a callback handling list of deleted LOCK objects. type DeletedLockCallback func(context.Context, []oid.Address) -// ZeroSizeContainersCallback is a callback hanfling list of zero-size containers. -type ZeroSizeContainersCallback func(context.Context, []cid.ID) +// EmptyContainersCallback is a callback hanfling list of zero-size and zero-count containers. +type EmptyContainersCallback func(context.Context, []cid.ID) // MetricsWriter is an interface that must store shard's metrics. type MetricsWriter interface { @@ -121,7 +121,8 @@ type cfg struct { deletedLockCallBack DeletedLockCallback - zeroSizeContainersCallback ZeroSizeContainersCallback + zeroSizeContainersCallback EmptyContainersCallback + zeroCountContainersCallback EmptyContainersCallback tsSource TombstoneSource @@ -134,12 +135,13 @@ type cfg struct { func defaultCfg() *cfg { return &cfg{ - rmBatchSize: 100, - log: &logger.Logger{Logger: zap.L()}, - gcCfg: defaultGCCfg(), - reportErrorFunc: func(string, string, error) {}, - rebuildLimiter: &noopRebuildLimiter{}, - zeroSizeContainersCallback: func(context.Context, []cid.ID) {}, + rmBatchSize: 100, + log: &logger.Logger{Logger: zap.L()}, + gcCfg: defaultGCCfg(), + reportErrorFunc: func(string, string, error) {}, + rebuildLimiter: &noopRebuildLimiter{}, + zeroSizeContainersCallback: func(context.Context, []cid.ID) {}, + zeroCountContainersCallback: func(context.Context, []cid.ID) {}, } } @@ -370,12 +372,19 @@ func WithRebuildWorkerLimiter(l RebuildWorkerLimiter) Option { } // WithZeroSizeCallback returns option to set zero-size containers callback. -func WithZeroSizeCallback(cb ZeroSizeContainersCallback) Option { +func WithZeroSizeCallback(cb EmptyContainersCallback) Option { return func(c *cfg) { c.zeroSizeContainersCallback = cb } } +// WithZeroCountCallback returns option to set zero-count containers callback. +func WithZeroCountCallback(cb EmptyContainersCallback) Option { + return func(c *cfg) { + c.zeroCountContainersCallback = cb + } +} + func (s *Shard) fillInfo() { s.cfg.info.MetaBaseInfo = s.metaBase.DumpInfo() s.cfg.info.BlobStorInfo = s.blobStor.DumpInfo() diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 80e912048..4cc542470 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -12,6 +12,7 @@ type EngineMetrics interface { AddMethodDuration(method string, d time.Duration) AddToContainerSize(cnrID string, size int64) DeleteContainerSize(cnrID string) + DeleteContainerCount(cnrID string) IncErrorCounter(shardID string) ClearErrorCounter(shardID string) DeleteShardMetrics(shardID string) @@ -84,6 +85,10 @@ func (m *engineMetrics) DeleteContainerSize(cnrID string) { m.containerSize.DeletePartialMatch(prometheus.Labels{containerIDLabelKey: cnrID}) } +func (m *engineMetrics) DeleteContainerCount(cnrID string) { + m.contObjCounter.DeletePartialMatch(prometheus.Labels{containerIDLabelKey: cnrID}) +} + func (m *engineMetrics) AddToPayloadCounter(shardID string, size int64) { m.payloadSize.With(prometheus.Labels{shardIDLabel: shardID}).Add(float64(size)) } From 79bebe4a683ceab420742577a46bfc536bc2e118 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 9 Jan 2024 18:34:25 +0300 Subject: [PATCH 0782/1943] [#884] cli: Fix error message for undefined endpoint Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/internal/client/sdk.go | 4 ++++ pkg/network/address.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/cmd/frostfs-cli/internal/client/sdk.go b/cmd/frostfs-cli/internal/client/sdk.go index b3e04e5f5..f7c48b871 100644 --- a/cmd/frostfs-cli/internal/client/sdk.go +++ b/cmd/frostfs-cli/internal/client/sdk.go @@ -34,6 +34,10 @@ func GetSDKClientByFlag(cmd *cobra.Command, key *ecdsa.PrivateKey, endpointFlag func getSDKClientByFlag(cmd *cobra.Command, key *ecdsa.PrivateKey, endpointFlag string) (*client.Client, error) { var addr network.Address + if len(viper.GetString(endpointFlag)) == 0 { + return nil, fmt.Errorf("%s is not defined", endpointFlag) + } + err := addr.FromString(viper.GetString(endpointFlag)) if err != nil { return nil, fmt.Errorf("%v: %w", errInvalidEndpoint, err) diff --git a/pkg/network/address.go b/pkg/network/address.go index 8ad285725..aeb647885 100644 --- a/pkg/network/address.go +++ b/pkg/network/address.go @@ -88,6 +88,10 @@ func (a *Address) FromString(s string) error { // multiaddrStringFromHostAddr converts "localhost:8080" to "/dns4/localhost/tcp/8080". func multiaddrStringFromHostAddr(host string) (string, error) { + if len(host) == 0 { + return "", fmt.Errorf("host is empty") + } + endpoint, port, err := net.SplitHostPort(host) if err != nil { return "", err From 5c0a736a25ed2e30f19a1ecd86796390277b6a50 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 10 Jan 2024 18:37:54 +0300 Subject: [PATCH 0783/1943] [#899] containerSvc: Fix invalid session token type Signed-off-by: Dmitrii Stepanov --- pkg/services/container/ape.go | 12 +++++++----- pkg/services/container/ape_test.go | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index e1ef43c63..36edf4516 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -35,6 +35,8 @@ var ( errInvalidSessionTokenOwner = errors.New("malformed request: invalid session token owner") errEmptyBodySignature = errors.New("malformed request: empty body signature") errMissingOwnerID = errors.New("malformed request: missing owner ID") + + undefinedContainerID = cid.ID{} ) type ir interface { @@ -196,7 +198,7 @@ func (ac *apeChecker) getRoleWithoutContainerID(oID *refs.OwnerID, mh *session.R return "", nil, err } - actor, pk, err := ac.getActorAndPublicKey(mh, vh, cid.ID{}) + actor, pk, err := ac.getActorAndPublicKey(mh, vh, undefinedContainerID) if err != nil { return "", nil, err } @@ -403,7 +405,7 @@ func (ac *apeChecker) getActorAndPKFromSignature(vh *session.RequestVerification return &userID, key, nil } -func (ac *apeChecker) getSessionToken(mh *session.RequestMetaHeader) (*sessionSDK.Object, error) { +func (ac *apeChecker) getSessionToken(mh *session.RequestMetaHeader) (*sessionSDK.Container, error) { for mh.GetOrigin() != nil { mh = mh.GetOrigin() } @@ -412,7 +414,7 @@ func (ac *apeChecker) getSessionToken(mh *session.RequestMetaHeader) (*sessionSD return nil, nil } - var tok sessionSDK.Object + var tok sessionSDK.Container err := tok.ReadFromV2(*st) if err != nil { return nil, fmt.Errorf("invalid session token: %w", err) @@ -421,8 +423,8 @@ func (ac *apeChecker) getSessionToken(mh *session.RequestMetaHeader) (*sessionSD return &tok, nil } -func (ac *apeChecker) getActorAndPKFromSessionToken(st *sessionSDK.Object, cnrID cid.ID) (*user.ID, *keys.PublicKey, error) { - if !st.AssertContainer(cnrID) { +func (ac *apeChecker) getActorAndPKFromSessionToken(st *sessionSDK.Container, cnrID cid.ID) (*user.ID, *keys.PublicKey, error) { + if cnrID != undefinedContainerID && !st.AppliedTo(cnrID) { return nil, nil, errSessionContainerMissmatch } if !st.VerifySignature() { diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index b61e68eb5..f4dfbe50f 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -253,8 +253,8 @@ func testDenyGetContainerEACLForIRSessionToken(t *testing.T) { sessionPK, err := keys.NewPrivateKey() require.NoError(t, err) - sToken := sessiontest.ObjectSigned() - sToken.BindContainer(contID) + sToken := sessiontest.ContainerSigned() + sToken.ApplyOnlyTo(contID) require.NoError(t, sToken.Sign(sessionPK.PrivateKey)) var sTokenV2 session.Token sToken.WriteToV2(&sTokenV2) @@ -325,8 +325,8 @@ func testDenyPutContainerForOthersSessionToken(t *testing.T) { sessionPK, err := keys.NewPrivateKey() require.NoError(t, err) - sToken := sessiontest.ObjectSigned() - sToken.BindContainer(cid.ID{}) + sToken := sessiontest.ContainerSigned() + sToken.ApplyOnlyTo(cid.ID{}) require.NoError(t, sToken.Sign(sessionPK.PrivateKey)) var sTokenV2 session.Token sToken.WriteToV2(&sTokenV2) From c19396d203449c56271ff43d85f0a0b361f806d2 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 21 Dec 2023 15:05:45 +0300 Subject: [PATCH 0784/1943] [#885] control: Make chain id bytes in grpc Signed-off-by: Denis Kirillov --- cmd/frostfs-cli/modules/control/get_rule.go | 2 +- .../modules/control/remove_rule.go | 2 +- pkg/services/control/ir/service_grpc.pb.go | 25 ++++----- pkg/services/control/server/policy_engine.go | 4 +- pkg/services/control/service.pb.go | 24 ++++----- pkg/services/control/service.proto | 6 +-- pkg/services/control/service_frostfs.pb.go | 12 ++--- pkg/services/tree/service_grpc.pb.go | 54 +++++++------------ 8 files changed, 53 insertions(+), 76 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/get_rule.go b/cmd/frostfs-cli/modules/control/get_rule.go index 6d86d5d74..2bb6a7da2 100644 --- a/cmd/frostfs-cli/modules/control/get_rule.go +++ b/cmd/frostfs-cli/modules/control/get_rule.go @@ -38,7 +38,7 @@ func getRule(cmd *cobra.Command, _ []string) { Name: cidStr, Type: control.ChainTarget_CONTAINER, }, - ChainId: chainID, + ChainId: []byte(chainID), }, } diff --git a/cmd/frostfs-cli/modules/control/remove_rule.go b/cmd/frostfs-cli/modules/control/remove_rule.go index 929d0b1a1..60e5376a1 100644 --- a/cmd/frostfs-cli/modules/control/remove_rule.go +++ b/cmd/frostfs-cli/modules/control/remove_rule.go @@ -41,7 +41,7 @@ func removeRule(cmd *cobra.Command, _ []string) { Name: cidStr, Type: control.ChainTarget_CONTAINER, }, - ChainId: chainID, + ChainId: []byte(chainID), }, } diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index 724149c44..0f93c0e19 100644 --- a/pkg/services/control/ir/service_grpc.pb.go +++ b/pkg/services/control/ir/service_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.3.0 +// - protoc-gen-go-grpc v1.2.0 // - protoc v4.25.0 // source: pkg/services/control/ir/service.proto @@ -18,13 +18,6 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -const ( - ControlService_HealthCheck_FullMethodName = "/ircontrol.ControlService/HealthCheck" - ControlService_TickEpoch_FullMethodName = "/ircontrol.ControlService/TickEpoch" - ControlService_RemoveNode_FullMethodName = "/ircontrol.ControlService/RemoveNode" - ControlService_RemoveContainer_FullMethodName = "/ircontrol.ControlService/RemoveContainer" -) - // ControlServiceClient is the client API for ControlService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -49,7 +42,7 @@ func NewControlServiceClient(cc grpc.ClientConnInterface) ControlServiceClient { func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { out := new(HealthCheckResponse) - err := c.cc.Invoke(ctx, ControlService_HealthCheck_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/ircontrol.ControlService/HealthCheck", in, out, opts...) if err != nil { return nil, err } @@ -58,7 +51,7 @@ func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckR func (c *controlServiceClient) TickEpoch(ctx context.Context, in *TickEpochRequest, opts ...grpc.CallOption) (*TickEpochResponse, error) { out := new(TickEpochResponse) - err := c.cc.Invoke(ctx, ControlService_TickEpoch_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/ircontrol.ControlService/TickEpoch", in, out, opts...) if err != nil { return nil, err } @@ -67,7 +60,7 @@ func (c *controlServiceClient) TickEpoch(ctx context.Context, in *TickEpochReque func (c *controlServiceClient) RemoveNode(ctx context.Context, in *RemoveNodeRequest, opts ...grpc.CallOption) (*RemoveNodeResponse, error) { out := new(RemoveNodeResponse) - err := c.cc.Invoke(ctx, ControlService_RemoveNode_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/ircontrol.ControlService/RemoveNode", in, out, opts...) if err != nil { return nil, err } @@ -76,7 +69,7 @@ func (c *controlServiceClient) RemoveNode(ctx context.Context, in *RemoveNodeReq func (c *controlServiceClient) RemoveContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*RemoveContainerResponse, error) { out := new(RemoveContainerResponse) - err := c.cc.Invoke(ctx, ControlService_RemoveContainer_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/ircontrol.ControlService/RemoveContainer", in, out, opts...) if err != nil { return nil, err } @@ -135,7 +128,7 @@ func _ControlService_HealthCheck_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_HealthCheck_FullMethodName, + FullMethod: "/ircontrol.ControlService/HealthCheck", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).HealthCheck(ctx, req.(*HealthCheckRequest)) @@ -153,7 +146,7 @@ func _ControlService_TickEpoch_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_TickEpoch_FullMethodName, + FullMethod: "/ircontrol.ControlService/TickEpoch", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).TickEpoch(ctx, req.(*TickEpochRequest)) @@ -171,7 +164,7 @@ func _ControlService_RemoveNode_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_RemoveNode_FullMethodName, + FullMethod: "/ircontrol.ControlService/RemoveNode", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).RemoveNode(ctx, req.(*RemoveNodeRequest)) @@ -189,7 +182,7 @@ func _ControlService_RemoveContainer_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: ControlService_RemoveContainer_FullMethodName, + FullMethod: "/ircontrol.ControlService/RemoveContainer", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).RemoveContainer(ctx, req.(*RemoveContainerRequest)) diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 7ffa392a8..805c669a0 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -7,7 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" - engine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -51,7 +51,7 @@ func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainL resp := &control.AddChainLocalOverrideResponse{ Body: &control.AddChainLocalOverrideResponse_Body{ - ChainId: string(chain.ID), + ChainId: []byte(chain.ID), }, } err = SignMessage(s.key, resp) diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 33ed898ba..9fb6f865e 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -3379,7 +3379,7 @@ type AddChainLocalOverrideResponse_Body struct { // Chain ID assigned for the added rule chain. // If chain ID is left empty in the request, then // it will be generated. - ChainId string `protobuf:"bytes,1,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` + ChainId []byte `protobuf:"bytes,1,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` } func (x *AddChainLocalOverrideResponse_Body) Reset() { @@ -3414,11 +3414,11 @@ func (*AddChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25, 0} } -func (x *AddChainLocalOverrideResponse_Body) GetChainId() string { +func (x *AddChainLocalOverrideResponse_Body) GetChainId() []byte { if x != nil { return x.ChainId } - return "" + return nil } type GetChainLocalOverrideRequest_Body struct { @@ -3429,7 +3429,7 @@ type GetChainLocalOverrideRequest_Body struct { // Target for which the overrides are applied. Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` // Chain ID assigned for the added rule chain. - ChainId string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` + ChainId []byte `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` } func (x *GetChainLocalOverrideRequest_Body) Reset() { @@ -3471,11 +3471,11 @@ func (x *GetChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { return nil } -func (x *GetChainLocalOverrideRequest_Body) GetChainId() string { +func (x *GetChainLocalOverrideRequest_Body) GetChainId() []byte { if x != nil { return x.ChainId } - return "" + return nil } type GetChainLocalOverrideResponse_Body struct { @@ -3630,7 +3630,7 @@ type RemoveChainLocalOverrideRequest_Body struct { // Target for which the overrides are applied. Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` // Chain ID assigned for the added rule chain. - ChainId string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` + ChainId []byte `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` } func (x *RemoveChainLocalOverrideRequest_Body) Reset() { @@ -3672,11 +3672,11 @@ func (x *RemoveChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { return nil } -func (x *RemoveChainLocalOverrideRequest_Body) GetChainId() string { +func (x *RemoveChainLocalOverrideRequest_Body) GetChainId() []byte { if x != nil { return x.ChainId } - return "" + return nil } type RemoveChainLocalOverrideResponse_Body struct { @@ -4222,7 +4222,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, - 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xe1, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, @@ -4237,7 +4237,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, @@ -4287,7 +4287,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xba, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 34c040f2a..b76645b61 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -446,7 +446,7 @@ message AddChainLocalOverrideResponse { // Chain ID assigned for the added rule chain. // If chain ID is left empty in the request, then // it will be generated. - string chain_id = 1; + bytes chain_id = 1; } Body body = 1; @@ -461,7 +461,7 @@ message GetChainLocalOverrideRequest { ChainTarget target = 1; // Chain ID assigned for the added rule chain. - string chain_id = 2; + bytes chain_id = 2; } Body body = 1; @@ -511,7 +511,7 @@ message RemoveChainLocalOverrideRequest { ChainTarget target = 1; // Chain ID assigned for the added rule chain. - string chain_id = 2; + bytes chain_id = 2; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 4ce6791f8..574ab13c4 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -2157,7 +2157,7 @@ func (x *AddChainLocalOverrideResponse_Body) StableSize() (size int) { if x == nil { return 0 } - size += proto.StringSize(1, x.ChainId) + size += proto.BytesSize(1, x.ChainId) return size } @@ -2177,7 +2177,7 @@ func (x *AddChainLocalOverrideResponse_Body) StableMarshal(buf []byte) []byte { buf = make([]byte, x.StableSize()) } var offset int - offset += proto.StringMarshal(1, buf[offset:], x.ChainId) + offset += proto.BytesMarshal(1, buf[offset:], x.ChainId) return buf } @@ -2244,7 +2244,7 @@ func (x *GetChainLocalOverrideRequest_Body) StableSize() (size int) { return 0 } size += proto.NestedStructureSize(1, x.Target) - size += proto.StringSize(2, x.ChainId) + size += proto.BytesSize(2, x.ChainId) return size } @@ -2265,7 +2265,7 @@ func (x *GetChainLocalOverrideRequest_Body) StableMarshal(buf []byte) []byte { } var offset int offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) - offset += proto.StringMarshal(2, buf[offset:], x.ChainId) + offset += proto.BytesMarshal(2, buf[offset:], x.ChainId) return buf } @@ -2590,7 +2590,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) StableSize() (size int) { return 0 } size += proto.NestedStructureSize(1, x.Target) - size += proto.StringSize(2, x.ChainId) + size += proto.BytesSize(2, x.ChainId) return size } @@ -2611,7 +2611,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) StableMarshal(buf []byte) []byte } var offset int offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) - offset += proto.StringMarshal(2, buf[offset:], x.ChainId) + offset += proto.BytesMarshal(2, buf[offset:], x.ChainId) return buf } diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index 63f96e11a..66850c90e 100644 --- a/pkg/services/tree/service_grpc.pb.go +++ b/pkg/services/tree/service_grpc.pb.go @@ -1,9 +1,6 @@ -//* -// Service for working with CRDT tree. - // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.3.0 +// - protoc-gen-go-grpc v1.2.0 // - protoc v4.25.0 // source: pkg/services/tree/service.proto @@ -21,19 +18,6 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -const ( - TreeService_Add_FullMethodName = "/tree.TreeService/Add" - TreeService_AddByPath_FullMethodName = "/tree.TreeService/AddByPath" - TreeService_Remove_FullMethodName = "/tree.TreeService/Remove" - TreeService_Move_FullMethodName = "/tree.TreeService/Move" - TreeService_GetNodeByPath_FullMethodName = "/tree.TreeService/GetNodeByPath" - TreeService_GetSubTree_FullMethodName = "/tree.TreeService/GetSubTree" - TreeService_TreeList_FullMethodName = "/tree.TreeService/TreeList" - TreeService_Apply_FullMethodName = "/tree.TreeService/Apply" - TreeService_GetOpLog_FullMethodName = "/tree.TreeService/GetOpLog" - TreeService_Healthcheck_FullMethodName = "/tree.TreeService/Healthcheck" -) - // TreeServiceClient is the client API for TreeService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -71,7 +55,7 @@ func NewTreeServiceClient(cc grpc.ClientConnInterface) TreeServiceClient { func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*AddResponse, error) { out := new(AddResponse) - err := c.cc.Invoke(ctx, TreeService_Add_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/Add", in, out, opts...) if err != nil { return nil, err } @@ -80,7 +64,7 @@ func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grp func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, opts ...grpc.CallOption) (*AddByPathResponse, error) { out := new(AddByPathResponse) - err := c.cc.Invoke(ctx, TreeService_AddByPath_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/AddByPath", in, out, opts...) if err != nil { return nil, err } @@ -89,7 +73,7 @@ func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts ...grpc.CallOption) (*RemoveResponse, error) { out := new(RemoveResponse) - err := c.cc.Invoke(ctx, TreeService_Remove_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/Remove", in, out, opts...) if err != nil { return nil, err } @@ -98,7 +82,7 @@ func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...grpc.CallOption) (*MoveResponse, error) { out := new(MoveResponse) - err := c.cc.Invoke(ctx, TreeService_Move_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/Move", in, out, opts...) if err != nil { return nil, err } @@ -107,7 +91,7 @@ func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...g func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPathRequest, opts ...grpc.CallOption) (*GetNodeByPathResponse, error) { out := new(GetNodeByPathResponse) - err := c.cc.Invoke(ctx, TreeService_GetNodeByPath_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/GetNodeByPath", in, out, opts...) if err != nil { return nil, err } @@ -115,7 +99,7 @@ func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPath } func (c *treeServiceClient) GetSubTree(ctx context.Context, in *GetSubTreeRequest, opts ...grpc.CallOption) (TreeService_GetSubTreeClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], TreeService_GetSubTree_FullMethodName, opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], "/tree.TreeService/GetSubTree", opts...) if err != nil { return nil, err } @@ -148,7 +132,7 @@ func (x *treeServiceGetSubTreeClient) Recv() (*GetSubTreeResponse, error) { func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, opts ...grpc.CallOption) (*TreeListResponse, error) { out := new(TreeListResponse) - err := c.cc.Invoke(ctx, TreeService_TreeList_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/TreeList", in, out, opts...) if err != nil { return nil, err } @@ -157,7 +141,7 @@ func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, o func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts ...grpc.CallOption) (*ApplyResponse, error) { out := new(ApplyResponse) - err := c.cc.Invoke(ctx, TreeService_Apply_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/Apply", in, out, opts...) if err != nil { return nil, err } @@ -165,7 +149,7 @@ func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts .. } func (c *treeServiceClient) GetOpLog(ctx context.Context, in *GetOpLogRequest, opts ...grpc.CallOption) (TreeService_GetOpLogClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], TreeService_GetOpLog_FullMethodName, opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], "/tree.TreeService/GetOpLog", opts...) if err != nil { return nil, err } @@ -198,7 +182,7 @@ func (x *treeServiceGetOpLogClient) Recv() (*GetOpLogResponse, error) { func (c *treeServiceClient) Healthcheck(ctx context.Context, in *HealthcheckRequest, opts ...grpc.CallOption) (*HealthcheckResponse, error) { out := new(HealthcheckResponse) - err := c.cc.Invoke(ctx, TreeService_Healthcheck_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, "/tree.TreeService/Healthcheck", in, out, opts...) if err != nil { return nil, err } @@ -288,7 +272,7 @@ func _TreeService_Add_Handler(srv interface{}, ctx context.Context, dec func(int } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_Add_FullMethodName, + FullMethod: "/tree.TreeService/Add", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Add(ctx, req.(*AddRequest)) @@ -306,7 +290,7 @@ func _TreeService_AddByPath_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_AddByPath_FullMethodName, + FullMethod: "/tree.TreeService/AddByPath", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).AddByPath(ctx, req.(*AddByPathRequest)) @@ -324,7 +308,7 @@ func _TreeService_Remove_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_Remove_FullMethodName, + FullMethod: "/tree.TreeService/Remove", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Remove(ctx, req.(*RemoveRequest)) @@ -342,7 +326,7 @@ func _TreeService_Move_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_Move_FullMethodName, + FullMethod: "/tree.TreeService/Move", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Move(ctx, req.(*MoveRequest)) @@ -360,7 +344,7 @@ func _TreeService_GetNodeByPath_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_GetNodeByPath_FullMethodName, + FullMethod: "/tree.TreeService/GetNodeByPath", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).GetNodeByPath(ctx, req.(*GetNodeByPathRequest)) @@ -399,7 +383,7 @@ func _TreeService_TreeList_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_TreeList_FullMethodName, + FullMethod: "/tree.TreeService/TreeList", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).TreeList(ctx, req.(*TreeListRequest)) @@ -417,7 +401,7 @@ func _TreeService_Apply_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_Apply_FullMethodName, + FullMethod: "/tree.TreeService/Apply", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Apply(ctx, req.(*ApplyRequest)) @@ -456,7 +440,7 @@ func _TreeService_Healthcheck_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: TreeService_Healthcheck_FullMethodName, + FullMethod: "/tree.TreeService/Healthcheck", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Healthcheck(ctx, req.(*HealthcheckRequest)) From 4a4c790ec1d0ee2d72200c2fc9baf4df0aa67673 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 21 Dec 2023 15:55:09 +0300 Subject: [PATCH 0785/1943] [#885] cli: Support hex chain id in control API Signed-off-by: Denis Kirillov --- cmd/frostfs-cli/modules/control/add_rule.go | 16 ++++++++++++++-- cmd/frostfs-cli/modules/control/get_rule.go | 11 ++++++++++- cmd/frostfs-cli/modules/control/list_rules.go | 2 +- cmd/frostfs-cli/modules/control/remove_rule.go | 16 ++++++++++++++-- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index 47233678d..4fe8194bb 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -3,6 +3,7 @@ package control import ( "bytes" "crypto/sha256" + "encoding/hex" "encoding/json" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -42,6 +43,15 @@ func addRule(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) chainID, _ := cmd.Flags().GetString(chainIDFlag) + hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) + + chainIDRaw := []byte(chainID) + + if hexEncoded { + var err error + chainIDRaw, err = hex.DecodeString(chainID) + commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err) + } var cnr cid.ID cidStr, _ := cmd.Flags().GetString(commonflags.CIDFlag) @@ -54,7 +64,7 @@ func addRule(cmd *cobra.Command, _ []string) { chain := new(apechain.Chain) commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, []string{rule})) - chain.ID = apechain.ID(chainID) + chain.ID = apechain.ID(chainIDRaw) serializedChain := chain.Bytes() cmd.Println("CID: " + cidStr) @@ -84,7 +94,8 @@ func addRule(cmd *cobra.Command, _ []string) { verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - cmd.Println("Rule has been added. Chain id: ", resp.GetBody().GetChainId()) + chainIDRaw = resp.GetBody().GetChainId() + cmd.Printf("Rule has been added.\nChain id: '%s'\nChain id hex: '%x'\n", string(chainIDRaw), chainIDRaw) } func initControlAddRuleCmd() { @@ -94,4 +105,5 @@ func initControlAddRuleCmd() { ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) ff.String(ruleFlag, "", "Rule statement") ff.String(chainIDFlag, "", "Assign ID to the parsed chain") + ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") } diff --git a/cmd/frostfs-cli/modules/control/get_rule.go b/cmd/frostfs-cli/modules/control/get_rule.go index 2bb6a7da2..9508850ee 100644 --- a/cmd/frostfs-cli/modules/control/get_rule.go +++ b/cmd/frostfs-cli/modules/control/get_rule.go @@ -2,6 +2,7 @@ package control import ( "crypto/sha256" + "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -31,6 +32,13 @@ func getRule(cmd *cobra.Command, _ []string) { cnr.Encode(rawCID) chainID, _ := cmd.Flags().GetString(chainIDFlag) + hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) + + if hexEncoded { + chainIDBytes, err := hex.DecodeString(chainID) + commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err) + chainID = string(chainIDBytes) + } req := &control.GetChainLocalOverrideRequest{ Body: &control.GetChainLocalOverrideRequest_Body{ @@ -60,7 +68,7 @@ func getRule(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(resp.GetBody().GetChain())) // TODO (aarifullin): make pretty-formatted output for chains. - cmd.Println("Parsed chain:\n" + prettyJSONFormat(cmd, chain.Bytes())) + cmd.Printf("Parsed chain (chain id hex: '%x'):\n%s\n", chain.ID, prettyJSONFormat(cmd, chain.Bytes())) } func initControGetRuleCmd() { @@ -69,4 +77,5 @@ func initControGetRuleCmd() { ff := getRuleCmd.Flags() ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) ff.String(chainIDFlag, "", "Chain id") + ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") } diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index f087f5367..ef494b135 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -63,7 +63,7 @@ func listRules(cmd *cobra.Command, _ []string) { // TODO (aarifullin): make pretty-formatted output for chains. var chain apechain.Chain commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(c)) - cmd.Println("Parsed chain:\n" + prettyJSONFormat(cmd, chain.Bytes())) + cmd.Printf("Parsed chain (chain id hex: '%x'):\n%s\n", chain.ID, prettyJSONFormat(cmd, chain.Bytes())) } } diff --git a/cmd/frostfs-cli/modules/control/remove_rule.go b/cmd/frostfs-cli/modules/control/remove_rule.go index 60e5376a1..612ab7d0a 100644 --- a/cmd/frostfs-cli/modules/control/remove_rule.go +++ b/cmd/frostfs-cli/modules/control/remove_rule.go @@ -2,6 +2,7 @@ package control import ( "crypto/sha256" + "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -13,7 +14,8 @@ import ( ) const ( - chainIDFlag = "chain-id" + chainIDFlag = "chain-id" + chainIDHexFlag = "chain-id-hex" ) var removeRuleCmd = &cobra.Command{ @@ -34,6 +36,15 @@ func removeRule(cmd *cobra.Command, _ []string) { cnr.Encode(rawCID) chainID, _ := cmd.Flags().GetString(chainIDFlag) + hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) + + chainIDRaw := []byte(chainID) + + if hexEncoded { + var err error + chainIDRaw, err = hex.DecodeString(chainID) + commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err) + } req := &control.RemoveChainLocalOverrideRequest{ Body: &control.RemoveChainLocalOverrideRequest_Body{ @@ -41,7 +52,7 @@ func removeRule(cmd *cobra.Command, _ []string) { Name: cidStr, Type: control.ChainTarget_CONTAINER, }, - ChainId: []byte(chainID), + ChainId: chainIDRaw, }, } @@ -72,4 +83,5 @@ func initControlRemoveRuleCmd() { ff := removeRuleCmd.Flags() ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) ff.String(chainIDFlag, "", "Chain id") + ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") } From 5d982976fd7d2c34589f5a1858f8e608c5bd3f36 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 11 Jan 2024 11:00:45 +0300 Subject: [PATCH 0786/1943] [#898] ape: Fix bug with creating type bucket in chainbase Signed-off-by: Airat Arifullin --- pkg/ape/chainbase/boltdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ape/chainbase/boltdb.go b/pkg/ape/chainbase/boltdb.go index e54d01ea0..34a5c7d2d 100644 --- a/pkg/ape/chainbase/boltdb.go +++ b/pkg/ape/chainbase/boltdb.go @@ -140,7 +140,7 @@ func getTargetBucketCreateIfEmpty(tx *bbolt.Tx, name chain.Name, target policyen typeBucket := nbucket.Bucket([]byte{byte(target.Type)}) if typeBucket == nil { var err error - typeBucket, err = cbucket.CreateBucket([]byte{byte(target.Type)}) + typeBucket, err = nbucket.CreateBucket([]byte{byte(target.Type)}) if err != nil { return nil, fmt.Errorf("could not create a bucket for the target type '%c': %w", target.Type, err) } From a8e52ef7aaf41995e644ac92925db22f1295ccd0 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 11 Jan 2024 11:51:29 +0300 Subject: [PATCH 0787/1943] [#898] control: Fix codes for returning APE errors Signed-off-by: Airat Arifullin --- pkg/ape/chainbase/boltdb.go | 36 +++++++++++++------- pkg/services/control/server/policy_engine.go | 12 +++++-- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/pkg/ape/chainbase/boltdb.go b/pkg/ape/chainbase/boltdb.go index 34a5c7d2d..1047ad42d 100644 --- a/pkg/ape/chainbase/boltdb.go +++ b/pkg/ape/chainbase/boltdb.go @@ -25,15 +25,16 @@ type boltLocalOverrideStorage struct { var chainBucket = []byte{0} var ( - ErrChainBucketNotFound = logicerr.New("chain root bucket has not been found") + // ErrRootBucketNotFound signals the database has not been properly initialized. + ErrRootBucketNotFound = logicerr.New("root bucket not found") - ErrChainNotFound = logicerr.New("chain has not been found") + ErrGlobalNamespaceBucketNotFound = logicerr.New("global namespace bucket not found") - ErrGlobalNamespaceBucketNotFound = logicerr.New("global namespace bucket has not been found") + ErrTargetTypeBucketNotFound = logicerr.New("target type bucket not found") - ErrTargetTypeBucketNotFound = logicerr.New("target type bucket has not been found") + ErrTargetNameBucketNotFound = logicerr.New("target name bucket not found") - ErrTargetNameBucketNotFound = logicerr.New("target name bucket has not been found") + ErrBucketNotContainsChainID = logicerr.New("chain id not found in bucket") ) // NewBoltLocalOverrideDatabase returns storage wrapper for storing access policy engine @@ -101,31 +102,30 @@ func (cs *boltLocalOverrideStorage) Close() error { func getTargetBucket(tx *bbolt.Tx, name chain.Name, target policyengine.Target) (*bbolt.Bucket, error) { cbucket := tx.Bucket(chainBucket) if cbucket == nil { - return nil, ErrChainBucketNotFound + return nil, ErrRootBucketNotFound } nbucket := cbucket.Bucket([]byte(name)) if nbucket == nil { - return nil, fmt.Errorf("global namespace %s: %w", name, ErrGlobalNamespaceBucketNotFound) + return nil, fmt.Errorf("%w: %w: %s", policyengine.ErrChainNotFound, ErrGlobalNamespaceBucketNotFound, name) } typeBucket := nbucket.Bucket([]byte{byte(target.Type)}) if typeBucket == nil { - return nil, fmt.Errorf("type bucket '%c': %w", target.Type, ErrTargetTypeBucketNotFound) + return nil, fmt.Errorf("%w: %w: %c", policyengine.ErrChainNotFound, ErrTargetTypeBucketNotFound, target.Type) } rbucket := typeBucket.Bucket([]byte(target.Name)) if rbucket == nil { - return nil, fmt.Errorf("target name bucket %s: %w", target.Name, ErrTargetNameBucketNotFound) + return nil, fmt.Errorf("%w: %w: %s", policyengine.ErrChainNotFound, ErrTargetNameBucketNotFound, target.Name) } - return rbucket, nil } func getTargetBucketCreateIfEmpty(tx *bbolt.Tx, name chain.Name, target policyengine.Target) (*bbolt.Bucket, error) { cbucket := tx.Bucket(chainBucket) if cbucket == nil { - return nil, ErrChainBucketNotFound + return nil, ErrRootBucketNotFound } nbucket := cbucket.Bucket([]byte(name)) @@ -186,7 +186,7 @@ func (cs *boltLocalOverrideStorage) GetOverride(name chain.Name, target policyen } serializedChain = rbuck.Get([]byte(chainID)) if serializedChain == nil { - return ErrChainNotFound + return fmt.Errorf("%w: %w: %s", policyengine.ErrChainNotFound, ErrBucketNotContainsChainID, chainID) } serializedChain = slice.Copy(serializedChain) return nil @@ -225,7 +225,7 @@ func (cs *boltLocalOverrideStorage) ListOverrides(name chain.Name, target policy return nil }) }); err != nil { - if errors.Is(err, ErrGlobalNamespaceBucketNotFound) || errors.Is(err, ErrTargetNameBucketNotFound) { + if errors.Is(err, policyengine.ErrChainNotFound) { return []*chain.Chain{}, nil } return nil, err @@ -243,6 +243,16 @@ func (cs *boltLocalOverrideStorage) ListOverrides(name chain.Name, target policy func (cs *boltLocalOverrideStorage) DropAllOverrides(name chain.Name) error { return cs.db.Update(func(tx *bbolt.Tx) error { + cbucket := tx.Bucket(chainBucket) + if cbucket == nil { + return ErrRootBucketNotFound + } + + nbucket := cbucket.Bucket([]byte(name)) + if nbucket == nil { + return fmt.Errorf("%w: %w: global namespace %s", policyengine.ErrChainNotFound, ErrGlobalNamespaceBucketNotFound, name) + } + return tx.DeleteBucket([]byte(name)) }) } diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 805c669a0..519b21035 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -128,12 +128,18 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove return nil, err } + removed := true if err = s.localOverrideStorage.LocalStorage().RemoveOverride(apechain.Ingress, target, apechain.ID(req.GetBody().GetChainId())); err != nil { - return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) + code := getCodeByLocalStorageErr(err) + if code == codes.NotFound { + removed = false + } else { + return nil, status.Error(code, err.Error()) + } } resp := &control.RemoveChainLocalOverrideResponse{ Body: &control.RemoveChainLocalOverrideResponse_Body{ - Removed: true, + Removed: removed, }, } err = SignMessage(s.key, resp) @@ -144,7 +150,7 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove } func getCodeByLocalStorageErr(err error) codes.Code { - if errors.Is(err, engine.ErrChainNotFound) { + if errors.Is(err, engine.ErrChainNotFound) || errors.Is(err, engine.ErrChainNameNotFound) { return codes.NotFound } return codes.Internal From f1b2b8bffaf1c4d613d4e9afb5065aec4d8d1c74 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 9 Jan 2024 11:37:41 +0300 Subject: [PATCH 0788/1943] [#895] test: Fix NewLogger arguments list `debug` is always true. Signed-off-by: Dmitrii Stepanov --- pkg/innerring/blocktimer_test.go | 2 +- .../processors/alphabet/handlers_test.go | 6 +++--- .../processors/balance/handlers_test.go | 4 ++-- .../processors/container/handlers_test.go | 6 +++--- .../processors/frostfs/handlers_test.go | 2 +- .../processors/governance/handlers_test.go | 4 ++-- .../processors/netmap/handlers_test.go | 2 +- .../blobovnicza/blobovnicza_test.go | 2 +- .../blobovniczatree/concurrency_test.go | 2 +- .../blobstor/blobovniczatree/exists_test.go | 2 +- .../blobstor/blobovniczatree/generic_test.go | 4 ++-- .../blobovniczatree/rebuild_failover_test.go | 2 +- .../blobstor/blobovniczatree/rebuild_test.go | 4 ++-- .../blobstor/memstore/memstore_test.go | 2 +- .../engine/control_test.go | 4 ++-- .../engine/delete_test.go | 2 +- .../engine/engine_test.go | 4 ++-- pkg/local_object_storage/engine/error_test.go | 2 +- .../engine/evacuate_test.go | 2 +- pkg/local_object_storage/engine/list_test.go | 2 +- .../shard/control_test.go | 2 +- .../shard/gc_internal_test.go | 6 +++--- pkg/local_object_storage/shard/range_test.go | 2 +- pkg/local_object_storage/shard/reload_test.go | 2 +- pkg/local_object_storage/shard/shard_test.go | 6 +++--- .../writecache/flush_test.go | 2 +- .../writecache/generic_test.go | 2 +- pkg/morph/event/listener_test.go | 6 +++--- pkg/services/object/get/get_test.go | 6 +++--- pkg/services/object/search/search_test.go | 8 +++---- pkg/services/tree/signature_test.go | 2 +- pkg/util/logger/test/logger.go | 21 +++++++------------ 32 files changed, 60 insertions(+), 65 deletions(-) diff --git a/pkg/innerring/blocktimer_test.go b/pkg/innerring/blocktimer_test.go index 9d02721c2..242c0903b 100644 --- a/pkg/innerring/blocktimer_test.go +++ b/pkg/innerring/blocktimer_test.go @@ -19,7 +19,7 @@ func TestEpochTimer(t *testing.T) { } args := &epochTimerArgs{ - l: test.NewLogger(t, true), + l: test.NewLogger(t), alphabetState: alphaState, newEpochHandlers: []newEpochHandler{neh.Handle}, cnrWrapper: cnrStopper, diff --git a/pkg/innerring/processors/alphabet/handlers_test.go b/pkg/innerring/processors/alphabet/handlers_test.go index f94c8e78d..346901949 100644 --- a/pkg/innerring/processors/alphabet/handlers_test.go +++ b/pkg/innerring/processors/alphabet/handlers_test.go @@ -48,7 +48,7 @@ func TestProcessorEmitsGasToNetmapAndAlphabet(t *testing.T) { params := &alphabet.Params{ ParsedWallets: parsedWallets, - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), PoolSize: 2, StorageEmission: emission, IRList: &testIndexer{index: index}, @@ -125,7 +125,7 @@ func TestProcessorEmitsGasToNetmapIfNoParsedWallets(t *testing.T) { params := &alphabet.Params{ ParsedWallets: parsedWallets, - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), PoolSize: 2, StorageEmission: emission, IRList: &testIndexer{index: index}, @@ -186,7 +186,7 @@ func TestProcessorDoesntEmitGasIfNoNetmapOrParsedWallets(t *testing.T) { params := &alphabet.Params{ ParsedWallets: parsedWallets, - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), PoolSize: 2, StorageEmission: emission, IRList: &testIndexer{index: index}, diff --git a/pkg/innerring/processors/balance/handlers_test.go b/pkg/innerring/processors/balance/handlers_test.go index 10ae8c60e..86a9e15d0 100644 --- a/pkg/innerring/processors/balance/handlers_test.go +++ b/pkg/innerring/processors/balance/handlers_test.go @@ -21,7 +21,7 @@ func TestProcessorCallsFrostFSContractForLockEvent(t *testing.T) { bsc := util.Uint160{100} processor, err := New(&Params{ - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), PoolSize: 2, FrostFSClient: cl, BalanceSC: bsc, @@ -47,7 +47,7 @@ func TestProcessorDoesntCallFrostFSContractIfNotAlphabet(t *testing.T) { bsc := util.Uint160{100} processor, err := New(&Params{ - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), PoolSize: 2, FrostFSClient: cl, BalanceSC: bsc, diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index 9fbfdf811..a69d0e4c4 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -38,7 +38,7 @@ func TestPutEvent(t *testing.T) { mc := &testMorphClient{} proc, err := New(&Params{ - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, NetworkState: nst, @@ -99,7 +99,7 @@ func TestDeleteEvent(t *testing.T) { mc := &testMorphClient{} proc, err := New(&Params{ - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, NetworkState: nst, @@ -171,7 +171,7 @@ func TestSetEACLEvent(t *testing.T) { mc := &testMorphClient{} proc, err := New(&Params{ - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), PoolSize: 2, AlphabetState: &testAlphabetState{isAlphabet: true}, NetworkState: nst, diff --git a/pkg/innerring/processors/frostfs/handlers_test.go b/pkg/innerring/processors/frostfs/handlers_test.go index 2685808ed..6425172bd 100644 --- a/pkg/innerring/processors/frostfs/handlers_test.go +++ b/pkg/innerring/processors/frostfs/handlers_test.go @@ -193,7 +193,7 @@ func TestHandleConfig(t *testing.T) { func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { p := &Params{ - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), PoolSize: 1, FrostFSContract: util.Uint160{0}, BalanceClient: &testBalaceClient{}, diff --git a/pkg/innerring/processors/governance/handlers_test.go b/pkg/innerring/processors/governance/handlers_test.go index adf674173..2a505f8d1 100644 --- a/pkg/innerring/processors/governance/handlers_test.go +++ b/pkg/innerring/processors/governance/handlers_test.go @@ -42,7 +42,7 @@ func TestHandleAlphabetSyncEvent(t *testing.T) { proc, err := New( &Params{ - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), EpochState: es, AlphabetState: as, Voter: v, @@ -123,7 +123,7 @@ func TestHandleAlphabetDesignateEvent(t *testing.T) { proc, err := New( &Params{ - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), EpochState: es, AlphabetState: as, Voter: v, diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index 6a66fe323..b34abb78c 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -279,7 +279,7 @@ func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { eh := &testEventHandler{} p := &Params{ - Log: test.NewLogger(t, true), + Log: test.NewLogger(t), PoolSize: 1, CleanupEnabled: false, CleanupThreshold: 3, diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index 8d701ae5c..caee770e8 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -63,7 +63,7 @@ func TestBlobovnicza(t *testing.T) { WithPath(p), WithObjectSizeLimit(objSizeLim), WithFullSizeLimit(sizeLim), - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), ) defer os.Remove(p) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go index 24cb93865..350ebef54 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go @@ -17,7 +17,7 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) { const n = 1000 st := NewBlobovniczaTree( - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), WithObjectSizeLimit(1024), WithBlobovniczaShallowWidth(10), WithBlobovniczaShallowDepth(1), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 63df97595..d50fb7983 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -17,7 +17,7 @@ import ( func TestExistsInvalidStorageID(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), WithObjectSizeLimit(1024), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go index 6eb8b81ae..1a52eddf8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go @@ -13,7 +13,7 @@ func TestGeneric(t *testing.T) { helper := func(t *testing.T, dir string) common.Storage { return NewBlobovniczaTree( - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), @@ -40,7 +40,7 @@ func TestControl(t *testing.T) { newTree := func(t *testing.T) common.Storage { return NewBlobovniczaTree( - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index ff86c036a..855229405 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -129,7 +129,7 @@ func testRebuildFailoverObjectDeletedFromSource(t *testing.T) { func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object, mustUpdateStorageID bool) { b := NewBlobovniczaTree( - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 904971d25..0ac50f36a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -41,7 +41,7 @@ func TestBlobovniczaTreeRebuild(t *testing.T) { func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, targetDepth, targetWidth uint64, shouldMigrate bool) { dir := t.TempDir() b := NewBlobovniczaTree( - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(sourceWidth), WithBlobovniczaShallowDepth(sourceDepth), @@ -81,7 +81,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta require.NoError(t, b.Close()) b = NewBlobovniczaTree( - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(targetWidth), WithBlobovniczaShallowDepth(targetDepth), diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index eaa2a4b61..cabf7f5ad 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -15,7 +15,7 @@ import ( func TestSimpleLifecycle(t *testing.T) { s := New( WithRootPath("memstore"), - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), ) t.Cleanup(func() { _ = s.Close() }) require.NoError(t, s.Open(false)) diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 21652970b..41b7db91c 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -49,7 +49,7 @@ func TestInitializationFailure(t *testing.T) { return []shard.Option{ shard.WithID(sid), - shard.WithLogger(test.NewLogger(t, true)), + shard.WithLogger(test.NewLogger(t)), shard.WithBlobStorOptions( blobstor.WithStorages(storages)), shard.WithMetaBaseOptions( @@ -289,7 +289,7 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str te := testNewEngine(t). setShardsNumOpts(t, num, func(id int) []shard.Option { return []shard.Option{ - shard.WithLogger(test.NewLogger(t, true)), + shard.WithLogger(test.NewLogger(t)), shard.WithBlobStorOptions( blobstor.WithStorages(newStorages(filepath.Join(addPath, strconv.Itoa(id)), errSmallSize))), shard.WithMetaBaseOptions( diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 8e94732ae..f3e4a0976 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -53,7 +53,7 @@ func TestDeleteBigObject(t *testing.T) { s3 := testNewShard(t, 3) e := testNewEngine(t).setInitializedShards(t, s1, s2, s3).engine - e.log = test.NewLogger(t, true) + e.log = test.NewLogger(t) defer e.Close(context.Background()) for i := range children { diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 21bf37dc7..063ea451e 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -79,7 +79,7 @@ type testEngineWrapper struct { } func testNewEngine(t testing.TB, opts ...Option) *testEngineWrapper { - engine := New(WithLogger(test.NewLogger(t, true))) + engine := New(WithLogger(test.NewLogger(t))) for _, opt := range opts { opt(engine.cfg) } @@ -198,7 +198,7 @@ func testNewShard(t testing.TB, id int) *shard.Shard { func testDefaultShardOptions(t testing.TB, id int) []shard.Option { return []shard.Option{ - shard.WithLogger(test.NewLogger(t, true)), + shard.WithLogger(test.NewLogger(t)), shard.WithBlobStorOptions( blobstor.WithStorages( newStorages(t.TempDir(), 1<<20))), diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index cf62302f2..3f343e5b9 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -55,7 +55,7 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) largeFileStorage: largeFileStorage, } return []shard.Option{ - shard.WithLogger(test.NewLogger(t, true)), + shard.WithLogger(test.NewLogger(t)), shard.WithBlobStorOptions(blobstor.WithStorages(storages)), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", id))), diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 13ba94f5b..ca86ef544 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -30,7 +30,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng te := testNewEngine(t). setShardsNumOpts(t, shardNum, func(id int) []shard.Option { return []shard.Option{ - shard.WithLogger(test.NewLogger(t, true)), + shard.WithLogger(test.NewLogger(t)), shard.WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{{ Storage: fstree.New( diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 246118687..27c6c819f 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -65,7 +65,7 @@ func TestListWithCursor(t *testing.T) { t.Parallel() e := testNewEngine(t).setShardsNumOpts(t, tt.shardNum, func(id int) []shard.Option { return []shard.Option{ - shard.WithLogger(test.NewLogger(t, true)), + shard.WithLogger(test.NewLogger(t)), shard.WithBlobStorOptions( blobstor.WithStorages( newStorages(t.TempDir(), 1<<20))), diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 8a650db0c..4d7354595 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -64,7 +64,7 @@ func TestShardOpen(t *testing.T) { newShard := func() *Shard { return New( WithID(NewIDFromBytes([]byte{})), - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{ {Storage: st}, diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index b64cb9c40..a82fc3ffe 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -30,13 +30,13 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { rootPath := t.TempDir() var sh *Shard - l := test.NewLogger(t, true) + l := test.NewLogger(t) blobOpts := []blobstor.Option{ - blobstor.WithLogger(test.NewLogger(t, true)), + blobstor.WithLogger(test.NewLogger(t)), blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(test.NewLogger(t, true)), + blobovniczatree.WithLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 35dc0c6a5..28bbd30e5 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -78,7 +78,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(test.NewLogger(t, true)), + blobovniczatree.WithLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index fc1fc2d20..c76fc1a00 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -27,7 +27,7 @@ func TestShardReload(t *testing.T) { p := t.Name() defer os.RemoveAll(p) - l := test.NewLogger(t, true) + l := test.NewLogger(t) blobOpts := []blobstor.Option{ blobstor.WithLogger(l), blobstor.WithStorages([]blobstor.SubStorage{ diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 807633cd5..98d1c7b8e 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -56,11 +56,11 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard if o.bsOpts == nil { o.bsOpts = []blobstor.Option{ - blobstor.WithLogger(test.NewLogger(t, true)), + blobstor.WithLogger(test.NewLogger(t)), blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithLogger(test.NewLogger(t, true)), + blobovniczatree.WithLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(o.rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), @@ -78,7 +78,7 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard opts := []Option{ WithID(NewIDFromBytes([]byte{})), - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), WithBlobStorOptions(o.bsOpts...), WithMetaBaseOptions( append([]meta.Option{ diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 4a243c5ec..a358028e7 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -24,7 +24,7 @@ import ( ) func TestFlush(t *testing.T) { - testlogger := test.NewLogger(t, true) + testlogger := test.NewLogger(t) createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs MainStorage, opts ...Option) Cache { return New( diff --git a/pkg/local_object_storage/writecache/generic_test.go b/pkg/local_object_storage/writecache/generic_test.go index a6d9e4799..2913ed13b 100644 --- a/pkg/local_object_storage/writecache/generic_test.go +++ b/pkg/local_object_storage/writecache/generic_test.go @@ -10,7 +10,7 @@ import ( func TestGeneric(t *testing.T) { storagetest.TestAll(t, func(t *testing.T) storagetest.Component { return New( - WithLogger(test.NewLogger(t, true)), + WithLogger(test.NewLogger(t)), WithFlushWorkersCount(2), WithPath(t.TempDir())) }) diff --git a/pkg/morph/event/listener_test.go b/pkg/morph/event/listener_test.go index ae92bd580..5f7cf9f43 100644 --- a/pkg/morph/event/listener_test.go +++ b/pkg/morph/event/listener_test.go @@ -20,7 +20,7 @@ func TestEventHandling(t *testing.T) { notaryRequestsCh := make(chan *result.NotaryRequestEvent) l, err := NewListener(ListenerParams{ - Logger: test.NewLogger(t, true), + Logger: test.NewLogger(t), Subscriber: &testSubscriber{ blockCh: blockCh, notificationCh: notificationCh, @@ -102,7 +102,7 @@ func TestErrorPassing(t *testing.T) { t.Run("notification error", func(t *testing.T) { nErr := fmt.Errorf("notification error") l, err := NewListener(ListenerParams{ - Logger: test.NewLogger(t, true), + Logger: test.NewLogger(t), Subscriber: &testSubscriber{ blockCh: blockCh, notificationCh: notificationCh, @@ -126,7 +126,7 @@ func TestErrorPassing(t *testing.T) { t.Run("block error", func(t *testing.T) { bErr := fmt.Errorf("notification error") l, err := NewListener(ListenerParams{ - Logger: test.NewLogger(t, true), + Logger: test.NewLogger(t), Subscriber: &testSubscriber{ blockCh: blockCh, notificationCh: notificationCh, diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index f4d4f7372..64614d8d8 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -270,7 +270,7 @@ func TestGetLocalOnly(t *testing.T) { newSvc := func(storage *testStorage) *Service { return &Service{ - log: test.NewLogger(t, true), + log: test.NewLogger(t), localStorage: storage, } } @@ -532,7 +532,7 @@ func TestGetRemoteSmall(t *testing.T) { const curEpoch = 13 return &Service{ - log: test.NewLogger(t, true), + log: test.NewLogger(t), localStorage: newTestStorage(), traverserGenerator: &testTraverserGenerator{ c: cnr, @@ -1663,7 +1663,7 @@ func TestGetFromPastEpoch(t *testing.T) { const curEpoch = 13 svc := &Service{ - log: test.NewLogger(t, true), + log: test.NewLogger(t), localStorage: newTestStorage(), epochSource: testEpochReceiver(curEpoch), traverserGenerator: &testTraverserGenerator{ diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index 09d98eff2..802fa33ef 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -161,7 +161,7 @@ func TestGetLocalOnly(t *testing.T) { newSvc := func(storage *testStorage) *Service { svc := &Service{cfg: new(cfg)} - svc.log = test.NewLogger(t, true) + svc.log = test.NewLogger(t) svc.localStorage = storage return svc @@ -277,7 +277,7 @@ func TestGetRemoteSmall(t *testing.T) { newSvc := func(b *testPlacementBuilder, c *testClientCache) *Service { svc := &Service{cfg: new(cfg)} - svc.log = test.NewLogger(t, true) + svc.log = test.NewLogger(t) svc.localStorage = newTestStorage() const curEpoch = 13 @@ -430,7 +430,7 @@ func TestGetFromPastEpoch(t *testing.T) { c22.addResult(idCnr, ids22, nil) svc := &Service{cfg: new(cfg)} - svc.log = test.NewLogger(t, true) + svc.log = test.NewLogger(t) svc.localStorage = newTestStorage() const curEpoch = 13 @@ -543,7 +543,7 @@ func TestGetWithSessionToken(t *testing.T) { w := new(simpleIDWriter) svc := &Service{cfg: new(cfg)} - svc.log = test.NewLogger(t, true) + svc.log = test.NewLogger(t) svc.localStorage = localStorage const curEpoch = 13 diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 1449d5756..b55f4d4fc 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -100,7 +100,7 @@ func TestMessageSign(t *testing.T) { s := &Service{ cfg: cfg{ - log: test.NewLogger(t, true), + log: test.NewLogger(t), key: &privs[0].PrivateKey, nmSource: dummyNetmapSource{}, cnrSource: dummyContainerSource{ diff --git a/pkg/util/logger/test/logger.go b/pkg/util/logger/test/logger.go index 4a2870870..dd07e6170 100644 --- a/pkg/util/logger/test/logger.go +++ b/pkg/util/logger/test/logger.go @@ -10,20 +10,15 @@ import ( ) // NewLogger creates a new logger. -// -// If debug, development logger is created. -func NewLogger(t testing.TB, debug bool) *logger.Logger { +func NewLogger(t testing.TB) *logger.Logger { + cfg := zap.NewDevelopmentConfig() + cfg.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + + log, err := cfg.Build() + require.NoError(t, err, "could not prepare logger") + var l logger.Logger - l.Logger = zap.L() - - if debug { - cfg := zap.NewDevelopmentConfig() - cfg.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder - - log, err := cfg.Build() - require.NoError(t, err, "could not prepare logger") - l.Logger = log - } + l.Logger = log return &l } From 836818fb751f79ac129121d0eeea44de0193efc4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 9 Jan 2024 12:32:13 +0300 Subject: [PATCH 0789/1943] [#895] test: Use zaptest logger Previous implementation mixes logs from different tests, now logs are separeted by test. Signed-off-by: Dmitrii Stepanov --- pkg/util/logger/test/logger.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pkg/util/logger/test/logger.go b/pkg/util/logger/test/logger.go index dd07e6170..f93756d17 100644 --- a/pkg/util/logger/test/logger.go +++ b/pkg/util/logger/test/logger.go @@ -4,21 +4,16 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest" ) // NewLogger creates a new logger. func NewLogger(t testing.TB) *logger.Logger { - cfg := zap.NewDevelopmentConfig() - cfg.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder - - log, err := cfg.Build() - require.NoError(t, err, "could not prepare logger") - var l logger.Logger - l.Logger = log - + l.Logger = zaptest.NewLogger(t, + zaptest.Level(zapcore.DebugLevel), + zaptest.WrapOptions(zap.Development(), zap.AddCaller())) return &l } From 47dcfa20f32d9b9032f76b99d452d9fc42da7dfa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 9 Jan 2024 16:26:43 +0300 Subject: [PATCH 0790/1943] [#895] test: Use t.Cleanup only for external resources Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/get_test.go | 7 +- .../blobovniczatree/concurrency_test.go | 4 +- .../blobstor/blobovniczatree/exists_test.go | 2 +- .../blobstor/internal/blobstortest/delete.go | 2 +- .../blobstor/internal/blobstortest/exists.go | 2 +- .../blobstor/internal/blobstortest/get.go | 2 +- .../internal/blobstortest/get_range.go | 2 +- .../blobstor/internal/blobstortest/iterate.go | 2 +- .../blobstor/memstore/memstore_test.go | 2 +- .../blobstor/perf_test.go | 7 +- .../engine/control_test.go | 7 ++ .../engine/engine_test.go | 6 +- pkg/local_object_storage/engine/error_test.go | 2 + pkg/local_object_storage/engine/list_test.go | 6 +- pkg/local_object_storage/engine/lock_test.go | 12 +-- .../engine/shards_test.go | 4 +- .../metabase/containers_test.go | 3 + .../metabase/control_test.go | 1 + .../metabase/counter_test.go | 8 ++ pkg/local_object_storage/metabase/db_test.go | 6 -- .../metabase/delete_test.go | 4 + .../metabase/exists_test.go | 1 + .../metabase/expired_test.go | 1 + pkg/local_object_storage/metabase/get_test.go | 2 + .../metabase/graveyard_test.go | 6 ++ .../metabase/inhume_test.go | 3 + .../metabase/iterators_test.go | 2 + .../metabase/list_test.go | 3 + .../metabase/lock_test.go | 3 + .../metabase/movable_test.go | 1 + pkg/local_object_storage/metabase/put_test.go | 3 + .../metabase/select_test.go | 11 +++ .../metabase/storage_id_test.go | 1 + .../pilorama/bench_test.go | 3 +- .../pilorama/forest_test.go | 79 +++++++++++++------ pkg/local_object_storage/shard/delete_test.go | 1 + .../shard/gc_internal_test.go | 5 +- pkg/local_object_storage/shard/gc_test.go | 2 + pkg/local_object_storage/shard/get_test.go | 1 + pkg/local_object_storage/shard/head_test.go | 1 + pkg/local_object_storage/shard/inhume_test.go | 1 + pkg/local_object_storage/shard/list_test.go | 2 + pkg/local_object_storage/shard/lock_test.go | 5 +- .../shard/metrics_test.go | 5 +- pkg/local_object_storage/shard/range_test.go | 1 + pkg/local_object_storage/shard/reload_test.go | 4 + pkg/local_object_storage/shard/shard_test.go | 9 --- .../shard/shutdown_test.go | 3 +- .../writecache/benchmark/writecache_test.go | 5 +- .../writecache/flush_test.go | 4 +- 50 files changed, 164 insertions(+), 95 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/get_test.go b/pkg/local_object_storage/blobovnicza/get_test.go index 40c434eb7..c464abc87 100644 --- a/pkg/local_object_storage/blobovnicza/get_test.go +++ b/pkg/local_object_storage/blobovnicza/get_test.go @@ -2,7 +2,6 @@ package blobovnicza import ( "context" - "os" "path/filepath" "testing" @@ -15,11 +14,7 @@ func TestBlobovnicza_Get(t *testing.T) { filename := filepath.Join(t.TempDir(), "blob") var blz *Blobovnicza - - t.Cleanup(func() { - blz.Close() - os.RemoveAll(filename) - }) + defer func() { require.NoError(t, blz.Close()) }() fnInit := func(szLimit uint64) { if blz != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go index 350ebef54..e8b148bff 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go @@ -24,9 +24,9 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) { WithRootPath(t.TempDir())) require.NoError(t, st.Open(false)) require.NoError(t, st.Init()) - t.Cleanup(func() { + defer func() { require.NoError(t, st.Close()) - }) + }() objGen := &testutil.SeqObjGenerator{ObjSize: 1} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index d50fb7983..048f9a2e5 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -25,7 +25,7 @@ func TestExistsInvalidStorageID(t *testing.T) { WithBlobovniczaSize(1<<20)) require.NoError(t, b.Open(false)) require.NoError(t, b.Init()) - t.Cleanup(func() { _ = b.Close() }) + defer func() { require.NoError(t, b.Close()) }() obj := blobstortest.NewObject(1024) addr := object.AddressOf(obj) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go index 0bfac2772..f551cbf5f 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go @@ -14,7 +14,7 @@ func TestDelete(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) require.NoError(t, s.Open(false)) require.NoError(t, s.Init()) - t.Cleanup(func() { require.NoError(t, s.Close()) }) + defer func() { require.NoError(t, s.Close()) }() objects := prepare(t, 4, s, min, max) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go index 99f6a79e8..29d8bafee 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go @@ -13,7 +13,7 @@ func TestExists(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) require.NoError(t, s.Open(false)) require.NoError(t, s.Init()) - t.Cleanup(func() { require.NoError(t, s.Close()) }) + defer func() { require.NoError(t, s.Close()) }() objects := prepare(t, 1, s, min, max) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go index 9a7ebed09..65c0c3133 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go @@ -14,7 +14,7 @@ func TestGet(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) require.NoError(t, s.Open(false)) require.NoError(t, s.Init()) - t.Cleanup(func() { require.NoError(t, s.Close()) }) + defer func() { require.NoError(t, s.Close()) }() objects := prepare(t, 2, s, min, max) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go index 1a9da0726..9b99c12d5 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go @@ -16,7 +16,7 @@ func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) require.NoError(t, s.Open(false)) require.NoError(t, s.Init()) - t.Cleanup(func() { require.NoError(t, s.Close()) }) + defer func() { require.NoError(t, s.Close()) }() objects := prepare(t, 1, s, min, max) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index 985ca0173..45c68cebc 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -13,7 +13,7 @@ func TestIterate(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) require.NoError(t, s.Open(false)) require.NoError(t, s.Init()) - t.Cleanup(func() { require.NoError(t, s.Close()) }) + defer func() { require.NoError(t, s.Close()) }() objects := prepare(t, 10, s, min, max) diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index cabf7f5ad..bc9436350 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -17,7 +17,7 @@ func TestSimpleLifecycle(t *testing.T) { WithRootPath("memstore"), WithLogger(test.NewLogger(t)), ) - t.Cleanup(func() { _ = s.Close() }) + defer func() { require.NoError(t, s.Close()) }() require.NoError(t, s.Open(false)) require.NoError(t, s.Init()) diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index c773ea0ee..72079acba 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -25,10 +25,6 @@ func (s storage) open(b *testing.B) common.Storage { require.NoError(b, st.Open(false)) require.NoError(b, st.Init()) - b.Cleanup(func() { - require.NoError(b, st.Close()) - }) - return st } @@ -108,6 +104,7 @@ func BenchmarkSubstorageReadPerf(b *testing.B) { b.Run(fmt.Sprintf("%s-%s", stEntry.desc, tt.desc), func(b *testing.B) { objGen := tt.objGen() st := stEntry.open(b) + defer func() { require.NoError(b, st.Close()) }() // Fill database var errG errgroup.Group @@ -162,6 +159,7 @@ func BenchmarkSubstorageWritePerf(b *testing.B) { b.Run(fmt.Sprintf("%s-%s", stEntry.desc, genEntry.desc), func(b *testing.B) { gen := genEntry.create() st := stEntry.open(b) + defer func() { require.NoError(b, st.Close()) }() b.ResetTimer() b.RunParallel(func(pb *testing.PB) { @@ -200,6 +198,7 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) { b.Run(fmt.Sprintf("%s-%s", stEntry.desc, tt.desc), func(b *testing.B) { objGen := tt.objGen() st := stEntry.open(b) + defer func() { require.NoError(b, st.Close()) }() // Fill database for i := 0; i < tt.size; i++ { diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 41b7db91c..cdaa41a51 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -122,6 +122,9 @@ func testEngineFailInitAndReload(t *testing.T, errOnAdd bool, opts []shard.Optio var configID string e := New() + defer func() { + require.NoError(t, e.Close(context.Background())) + }() _, err := e.AddShard(context.Background(), opts...) if errOnAdd { require.Error(t, err) @@ -258,6 +261,8 @@ func TestReload(t *testing.T) { require.Equal(t, shardNum+1, len(e.shards)) require.Equal(t, shardNum+1, len(e.shardPools)) + + require.NoError(t, e.Close(context.Background())) }) t.Run("remove shards", func(t *testing.T) { @@ -276,6 +281,8 @@ func TestReload(t *testing.T) { // removed one require.Equal(t, shardNum-1, len(e.shards)) require.Equal(t, shardNum-1, len(e.shardPools)) + + require.NoError(t, e.Close(context.Background())) }) } diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 063ea451e..2b9cdcf84 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -2,7 +2,6 @@ package engine import ( "context" - "os" "path/filepath" "sync/atomic" "testing" @@ -49,10 +48,7 @@ func benchmarkExists(b *testing.B, shardNum int) { } e := testNewEngine(b).setInitializedShards(b, shards...).engine - b.Cleanup(func() { - _ = e.Close(context.Background()) - _ = os.RemoveAll(b.Name()) - }) + defer func() { require.NoError(b, e.Close(context.Background())) }() addr := oidtest.Address() for i := 0; i < 100; i++ { diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 3f343e5b9..77dbc9b7e 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -114,6 +114,7 @@ func TestErrorReporting(t *testing.T) { checkShardState(t, te.ng, te.shards[0].id, i, mode.ReadWrite) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } + require.NoError(t, te.ng.Close(context.Background())) }) t.Run("with error threshold", func(t *testing.T) { const errThreshold = 3 @@ -161,6 +162,7 @@ func TestErrorReporting(t *testing.T) { require.NoError(t, te.ng.SetShardMode(te.shards[0].id, mode.ReadWrite, true)) checkShardState(t, te.ng, te.shards[0].id, 0, mode.ReadWrite) + require.NoError(t, te.ng.Close(context.Background())) }) } diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 27c6c819f..186c3e094 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -80,9 +80,9 @@ func TestListWithCursor(t *testing.T) { require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) - t.Cleanup(func() { - e.Close(context.Background()) - }) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() expected := make([]object.AddressWithType, 0, tt.objectNum) got := make([]object.AddressWithType, 0, tt.objectNum) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index ec18fc5e3..69b74f4a4 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -62,9 +62,7 @@ func TestLockUserScenario(t *testing.T) { require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) - t.Cleanup(func() { - _ = e.Close(context.Background()) - }) + defer func() { require.NoError(t, e.Close(context.Background())) }() lockerID := oidtest.ID() tombID := oidtest.ID() @@ -169,9 +167,7 @@ func TestLockExpiration(t *testing.T) { require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) - t.Cleanup(func() { - _ = e.Close(context.Background()) - }) + defer func() { require.NoError(t, e.Close(context.Background())) }() const lockerExpiresAfter = 13 @@ -246,9 +242,7 @@ func TestLockForceRemoval(t *testing.T) { }).engine require.NoError(t, e.Open(context.Background())) require.NoError(t, e.Init(context.Background())) - t.Cleanup(func() { - _ = e.Close(context.Background()) - }) + defer func() { require.NoError(t, e.Close(context.Background())) }() cnr := cidtest.ID() var err error diff --git a/pkg/local_object_storage/engine/shards_test.go b/pkg/local_object_storage/engine/shards_test.go index e13017e24..f2896d557 100644 --- a/pkg/local_object_storage/engine/shards_test.go +++ b/pkg/local_object_storage/engine/shards_test.go @@ -12,9 +12,7 @@ func TestRemoveShard(t *testing.T) { te := testNewEngine(t).setShardsNum(t, numOfShards) e, ids := te.engine, te.shardIDs - t.Cleanup(func() { - e.Close(context.Background()) - }) + defer func() { require.NoError(t, e.Close(context.Background())) }() require.Equal(t, numOfShards, len(e.shardPools)) require.Equal(t, numOfShards, len(e.shards)) diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index 4e2dd550d..e7762ded6 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -19,6 +19,7 @@ func TestDB_Containers(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() const N = 10 @@ -96,6 +97,7 @@ func TestDB_ContainersCount(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() const R, T, SG, L = 10, 11, 12, 13 // amount of object per type @@ -141,6 +143,7 @@ func TestDB_ContainerSize(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() const ( C = 3 diff --git a/pkg/local_object_storage/metabase/control_test.go b/pkg/local_object_storage/metabase/control_test.go index 8f36423fd..0354a5eb6 100644 --- a/pkg/local_object_storage/metabase/control_test.go +++ b/pkg/local_object_storage/metabase/control_test.go @@ -15,6 +15,7 @@ import ( func TestReset(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() err := db.Reset() require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 4b7b565b3..306bb0cc0 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -22,6 +22,7 @@ func TestCounters(t *testing.T) { t.Run("defaults", func(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() c, err := db.ObjectCounters() require.NoError(t, err) require.Zero(t, c.Phy) @@ -36,6 +37,7 @@ func TestCounters(t *testing.T) { t.Run("put", func(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() oo := make([]*objectSDK.Object, 0, objCount) for i := 0; i < objCount; i++ { oo = append(oo, testutil.GenerateObject()) @@ -73,6 +75,7 @@ func TestCounters(t *testing.T) { t.Run("delete", func(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() oo := putObjs(t, db, objCount, false) exp := make(map[cid.ID]meta.ObjectCounters) @@ -117,6 +120,7 @@ func TestCounters(t *testing.T) { t.Run("inhume", func(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() oo := putObjs(t, db, objCount, false) exp := make(map[cid.ID]meta.ObjectCounters) @@ -176,6 +180,7 @@ func TestCounters(t *testing.T) { t.Run("put_split", func(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() parObj := testutil.GenerateObject() exp := make(map[cid.ID]meta.ObjectCounters) @@ -213,6 +218,7 @@ func TestCounters(t *testing.T) { t.Run("delete_split", func(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() oo := putObjs(t, db, objCount, true) exp := make(map[cid.ID]meta.ObjectCounters) @@ -254,6 +260,7 @@ func TestCounters(t *testing.T) { t.Run("inhume_split", func(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() oo := putObjs(t, db, objCount, true) exp := make(map[cid.ID]meta.ObjectCounters) @@ -321,6 +328,7 @@ func TestCounters_Expired(t *testing.T) { es := &epochState{epoch} db := newDB(t, meta.WithEpochState(es)) + defer func() { require.NoError(t, db.Close()) }() oo := make([]oid.Address, objCount) for i := range oo { diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index b3c38f3d7..fc20e2aad 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -2,7 +2,6 @@ package meta_test import ( "context" - "os" "path/filepath" "strconv" "testing" @@ -53,11 +52,6 @@ func newDB(t testing.TB, opts ...meta.Option) *meta.DB { require.NoError(t, bdb.Open(context.Background(), false)) require.NoError(t, bdb.Init()) - t.Cleanup(func() { - bdb.Close() - os.Remove(bdb.DumpInfo().Path) - }) - return bdb } diff --git a/pkg/local_object_storage/metabase/delete_test.go b/pkg/local_object_storage/metabase/delete_test.go index 9b68f0bf8..2ff6ba89f 100644 --- a/pkg/local_object_storage/metabase/delete_test.go +++ b/pkg/local_object_storage/metabase/delete_test.go @@ -18,6 +18,7 @@ import ( func TestDB_Delete(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() cnr := cidtest.ID() parent := testutil.GenerateObjectWithCID(cnr) @@ -78,6 +79,7 @@ func TestDB_Delete(t *testing.T) { func TestDeleteAllChildren(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() cnr := cidtest.ID() @@ -115,6 +117,7 @@ func TestDeleteAllChildren(t *testing.T) { func TestGraveOnlyDelete(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() addr := oidtest.Address() @@ -127,6 +130,7 @@ func TestGraveOnlyDelete(t *testing.T) { func TestExpiredObject(t *testing.T) { db := newDB(t, meta.WithEpochState(epochState{currEpoch})) + defer func() { require.NoError(t, db.Close()) }() checkExpiredObjects(t, db, func(exp, nonExp *objectSDK.Object) { // removing expired object should be error-free diff --git a/pkg/local_object_storage/metabase/exists_test.go b/pkg/local_object_storage/metabase/exists_test.go index 06394339a..0087c1e31 100644 --- a/pkg/local_object_storage/metabase/exists_test.go +++ b/pkg/local_object_storage/metabase/exists_test.go @@ -18,6 +18,7 @@ const currEpoch = 1000 func TestDB_Exists(t *testing.T) { db := newDB(t, meta.WithEpochState(epochState{currEpoch})) + defer func() { require.NoError(t, db.Close()) }() t.Run("no object", func(t *testing.T) { nonExist := testutil.GenerateObject() diff --git a/pkg/local_object_storage/metabase/expired_test.go b/pkg/local_object_storage/metabase/expired_test.go index 9a6bcc5db..bb98745ee 100644 --- a/pkg/local_object_storage/metabase/expired_test.go +++ b/pkg/local_object_storage/metabase/expired_test.go @@ -13,6 +13,7 @@ import ( func TestDB_SelectExpired(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() containerID1 := cidtest.ID() diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index 98a4bd960..af6b41327 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -22,6 +22,7 @@ import ( func TestDB_Get(t *testing.T) { db := newDB(t, meta.WithEpochState(epochState{currEpoch})) + defer func() { require.NoError(t, db.Close()) }() raw := testutil.GenerateObject() @@ -180,6 +181,7 @@ func benchmarkGet(b *testing.B, numOfObj int) { meta.WithMaxBatchSize(batchSize), meta.WithMaxBatchDelay(10*time.Millisecond), ) + defer func() { require.NoError(b, db.Close()) }() addrs := make([]oid.Address, 0, numOfObj) for i := 0; i < numOfObj; i++ { diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index 7476608f2..75c7e2852 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -14,6 +14,7 @@ import ( func TestDB_IterateDeletedObjects_EmptyDB(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() var counter int var iterGravePRM meta.GraveyardIterationPrm @@ -40,6 +41,7 @@ func TestDB_IterateDeletedObjects_EmptyDB(t *testing.T) { func TestDB_Iterate_OffsetNotFound(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() obj1 := testutil.GenerateObject() obj2 := testutil.GenerateObject() @@ -110,6 +112,7 @@ func TestDB_Iterate_OffsetNotFound(t *testing.T) { func TestDB_IterateDeletedObjects(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() // generate and put 4 objects obj1 := testutil.GenerateObject() @@ -196,6 +199,7 @@ func TestDB_IterateDeletedObjects(t *testing.T) { func TestDB_IterateOverGraveyard_Offset(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() // generate and put 4 objects obj1 := testutil.GenerateObject() @@ -294,6 +298,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { func TestDB_IterateOverGarbage_Offset(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() // generate and put 4 objects obj1 := testutil.GenerateObject() @@ -385,6 +390,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { func TestDB_DropGraves(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() // generate and put 2 objects obj1 := testutil.GenerateObject() diff --git a/pkg/local_object_storage/metabase/inhume_test.go b/pkg/local_object_storage/metabase/inhume_test.go index 378e38e79..163fbec2a 100644 --- a/pkg/local_object_storage/metabase/inhume_test.go +++ b/pkg/local_object_storage/metabase/inhume_test.go @@ -16,6 +16,7 @@ import ( func TestDB_Inhume(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() raw := testutil.GenerateObject() testutil.AddAttribute(raw, "foo", "bar") @@ -37,6 +38,7 @@ func TestDB_Inhume(t *testing.T) { func TestInhumeTombOnTomb(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() var ( err error @@ -99,6 +101,7 @@ func TestInhumeTombOnTomb(t *testing.T) { func TestInhumeLocked(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() locked := oidtest.Address() diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 034a931d2..54d56d923 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -17,6 +17,7 @@ import ( func TestDB_IterateExpired(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() const epoch = 13 @@ -68,6 +69,7 @@ func putWithExpiration(t *testing.T, db *meta.DB, typ objectSDK.Type, expiresAt func TestDB_IterateCoveredByTombstones(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() ts := oidtest.Address() protected1 := oidtest.Address() diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 3e598ed16..25c0e35bd 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -33,6 +33,7 @@ func listWithCursorPrepareDB(b *testing.B) *meta.DB { db := newDB(b, meta.WithMaxBatchSize(1), meta.WithBoltDBOptions(&bbolt.Options{ NoSync: true, })) // faster single-thread generation + defer func() { require.NoError(b, db.Close()) }() obj := testutil.GenerateObject() for i := 0; i < 100_000; i++ { // should be a multiple of all batch sizes @@ -70,6 +71,7 @@ func TestLisObjectsWithCursor(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() const ( containers = 5 @@ -165,6 +167,7 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() const total = 5 diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 834ab07a7..2d7bfc1cc 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -21,6 +21,7 @@ func TestDB_Lock(t *testing.T) { cnr := cidtest.ID() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() t.Run("empty locked list", func(t *testing.T) { require.Panics(t, func() { _ = db.Lock(context.Background(), cnr, oid.ID{}, nil) }) @@ -182,6 +183,7 @@ func TestDB_Lock_Expired(t *testing.T) { es := &epochState{e: 123} db := newDB(t, meta.WithEpochState(es)) + defer func() { require.NoError(t, db.Close()) }() // put an object addr := putWithExpiration(t, db, objectSDK.TypeRegular, 124) @@ -203,6 +205,7 @@ func TestDB_IsLocked(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() // existing and locked objs diff --git a/pkg/local_object_storage/metabase/movable_test.go b/pkg/local_object_storage/metabase/movable_test.go index 51e7e6d74..d66d92867 100644 --- a/pkg/local_object_storage/metabase/movable_test.go +++ b/pkg/local_object_storage/metabase/movable_test.go @@ -13,6 +13,7 @@ import ( func TestDB_Movable(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() raw1 := testutil.GenerateObject() raw2 := testutil.GenerateObject() diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index a4cb2edc1..28467199d 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -46,6 +46,7 @@ func BenchmarkPut(b *testing.B) { db := newDB(b, meta.WithMaxBatchDelay(time.Millisecond*10), meta.WithMaxBatchSize(runtime.NumCPU())) + defer func() { require.NoError(b, db.Close()) }() // Ensure the benchmark is bound by CPU and not waiting batch-delay time. b.SetParallelism(1) @@ -67,6 +68,7 @@ func BenchmarkPut(b *testing.B) { db := newDB(b, meta.WithMaxBatchDelay(time.Millisecond*10), meta.WithMaxBatchSize(1)) + defer func() { require.NoError(b, db.Close()) }() var index atomic.Int64 index.Store(-1) objs := prepareObjects(b, b.N) @@ -82,6 +84,7 @@ func BenchmarkPut(b *testing.B) { func TestDB_PutBlobovniczaUpdate(t *testing.T) { db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() raw1 := testutil.GenerateObject() storageID := []byte{1, 2, 3, 4} diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index e107085ab..9ff2fd65e 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -23,6 +23,7 @@ func TestDB_SelectUserAttributes(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() cnr := cidtest.ID() @@ -147,6 +148,7 @@ func TestDB_SelectRootPhyParent(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() cnr := cidtest.ID() @@ -300,6 +302,7 @@ func TestDB_SelectInhume(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() cnr := cidtest.ID() @@ -334,6 +337,7 @@ func TestDB_SelectPayloadHash(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() cnr := cidtest.ID() @@ -404,6 +408,7 @@ func TestDB_SelectWithSlowFilters(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() cnr := cidtest.ID() @@ -511,6 +516,7 @@ func TestDB_SelectObjectID(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() cnr := cidtest.ID() @@ -626,6 +632,7 @@ func TestDB_SelectSplitID(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() cnr := cidtest.ID() @@ -682,6 +689,7 @@ func TestDB_SelectContainerID(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() cnr := cidtest.ID() @@ -729,6 +737,8 @@ func TestDB_SelectContainerID(t *testing.T) { func BenchmarkSelect(b *testing.B) { const objCount = 1000 db := newDB(b) + defer func() { require.NoError(b, db.Close()) }() + cid := cidtest.ID() for i := 0; i < objCount; i++ { @@ -769,6 +779,7 @@ func TestExpiredObjects(t *testing.T) { t.Parallel() db := newDB(t, meta.WithEpochState(epochState{currEpoch})) + defer func() { require.NoError(t, db.Close()) }() checkExpiredObjects(t, db, func(exp, nonExp *objectSDK.Object) { cidExp, _ := exp.ContainerID() diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go index 63dbc2f3f..063dfb270 100644 --- a/pkg/local_object_storage/metabase/storage_id_test.go +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -15,6 +15,7 @@ func TestDB_StorageID(t *testing.T) { t.Parallel() db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() raw1 := testutil.GenerateObject() raw2 := testutil.GenerateObject() diff --git a/pkg/local_object_storage/pilorama/bench_test.go b/pkg/local_object_storage/pilorama/bench_test.go index 3d5ff1a7c..e3f3afd99 100644 --- a/pkg/local_object_storage/pilorama/bench_test.go +++ b/pkg/local_object_storage/pilorama/bench_test.go @@ -28,8 +28,9 @@ func BenchmarkCreate(b *testing.B) { WithMaxBatchSize(runtime.GOMAXPROCS(0))) require.NoError(b, f.Open(context.Background(), false)) require.NoError(b, f.Init()) + defer func() { require.NoError(b, f.Close()) }() + b.Cleanup(func() { - require.NoError(b, f.Close()) require.NoError(b, os.RemoveAll(tmpDir)) }) diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 67454b686..5813395f0 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -20,19 +20,15 @@ import ( var providers = []struct { name string - construct func(t testing.TB, opts ...Option) Forest + construct func(t testing.TB, opts ...Option) ForestStorage }{ - {"inmemory", func(t testing.TB, _ ...Option) Forest { + {"inmemory", func(t testing.TB, _ ...Option) ForestStorage { f := NewMemoryForest() require.NoError(t, f.Open(context.Background(), false)) require.NoError(t, f.Init()) - t.Cleanup(func() { - require.NoError(t, f.Close()) - }) - return f }}, - {"bbolt", func(t testing.TB, opts ...Option) Forest { + {"bbolt", func(t testing.TB, opts ...Option) ForestStorage { f := NewBoltForest( append([]Option{ WithPath(filepath.Join(t.TempDir(), "test.db")), @@ -40,9 +36,6 @@ var providers = []struct { }, opts...)...) require.NoError(t, f.Open(context.Background(), false)) require.NoError(t, f.Init()) - t.Cleanup(func() { - require.NoError(t, f.Close()) - }) return f }}, } @@ -62,7 +55,9 @@ func TestForest_TreeMove(t *testing.T) { } } -func testForestTreeMove(t *testing.T, s Forest) { +func testForestTreeMove(t *testing.T, s ForestStorage) { + defer func() { require.NoError(t, s.Close()) }() + cid := cidtest.ID() d := CIDDescriptor{cid, 0, 1} treeID := "version" @@ -124,7 +119,9 @@ func TestMemoryForest_TreeGetChildren(t *testing.T) { } } -func testForestTreeGetChildren(t *testing.T, s Forest) { +func testForestTreeGetChildren(t *testing.T, s ForestStorage) { + defer func() { require.NoError(t, s.Close()) }() + cid := cidtest.ID() d := CIDDescriptor{cid, 0, 1} treeID := "version" @@ -188,7 +185,9 @@ func TestForest_TreeDrop(t *testing.T) { } } -func testForestTreeDrop(t *testing.T, s Forest) { +func testForestTreeDrop(t *testing.T, s ForestStorage) { + defer func() { require.NoError(t, s.Close()) }() + const cidsSize = 3 var cids [cidsSize]cidSDK.ID @@ -256,7 +255,9 @@ func TestForest_TreeAdd(t *testing.T) { } } -func testForestTreeAdd(t *testing.T, s Forest) { +func testForestTreeAdd(t *testing.T, s ForestStorage) { + defer func() { require.NoError(t, s.Close()) }() + cid := cidtest.ID() d := CIDDescriptor{cid, 0, 1} treeID := "version" @@ -302,7 +303,9 @@ func TestForest_TreeAddByPath(t *testing.T) { } } -func testForestTreeAddByPath(t *testing.T, s Forest) { +func testForestTreeAddByPath(t *testing.T, s ForestStorage) { + defer func() { require.NoError(t, s.Close()) }() + cid := cidtest.ID() d := CIDDescriptor{cid, 0, 1} treeID := "version" @@ -425,7 +428,7 @@ func TestForest_Apply(t *testing.T) { } } -func testForestTreeApply(t *testing.T, constructor func(t testing.TB, _ ...Option) Forest) { +func testForestTreeApply(t *testing.T, constructor func(t testing.TB, _ ...Option) ForestStorage) { cid := cidtest.ID() treeID := "version" @@ -439,6 +442,8 @@ func testForestTreeApply(t *testing.T, constructor func(t testing.TB, _ ...Optio t.Run("add a child, then insert a parent removal", func(t *testing.T) { s := constructor(t) + defer func() { require.NoError(t, s.Close()) }() + testApply(t, s, 10, 0, Meta{Time: 1, Items: []KeyValue{{"grand", []byte{1}}}}) meta := Meta{Time: 3, Items: []KeyValue{{"child", []byte{3}}}} @@ -450,6 +455,7 @@ func testForestTreeApply(t *testing.T, constructor func(t testing.TB, _ ...Optio }) t.Run("add a child to non-existent parent, then add a parent", func(t *testing.T) { s := constructor(t) + defer func() { require.NoError(t, s.Close()) }() meta := Meta{Time: 1, Items: []KeyValue{{"child", []byte{3}}}} testApply(t, s, 11, 10, meta) @@ -469,7 +475,7 @@ func TestForest_ApplySameOperation(t *testing.T) { } } -func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ ...Option) Forest, parallel bool) { +func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ ...Option) ForestStorage, parallel bool) { cid := cidtest.ID() treeID := "version" @@ -519,6 +525,8 @@ func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ t.Run("expected", func(t *testing.T) { s := constructor(t) + defer func() { require.NoError(t, s.Close()) }() + for i := range logs { require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[i], false)) } @@ -526,6 +534,8 @@ func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ }) s := constructor(t, WithMaxBatchSize(batchSize)) + defer func() { require.NoError(t, s.Close()) }() + require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[0], false)) for i := 0; i < batchSize; i++ { errG.Go(func() error { @@ -545,7 +555,7 @@ func TestForest_GetOpLog(t *testing.T) { } } -func testForestTreeGetOpLog(t *testing.T, constructor func(t testing.TB, _ ...Option) Forest) { +func testForestTreeGetOpLog(t *testing.T, constructor func(t testing.TB, _ ...Option) ForestStorage) { cid := cidtest.ID() treeID := "version" logs := []Move{ @@ -565,6 +575,7 @@ func testForestTreeGetOpLog(t *testing.T, constructor func(t testing.TB, _ ...Op } s := constructor(t) + defer func() { require.NoError(t, s.Close()) }() t.Run("empty log, no panic", func(t *testing.T) { _, err := s.TreeGetOpLog(context.Background(), cid, treeID, 0) @@ -603,8 +614,9 @@ func TestForest_TreeExists(t *testing.T) { } } -func testForestTreeExists(t *testing.T, constructor func(t testing.TB, opts ...Option) Forest) { +func testForestTreeExists(t *testing.T, constructor func(t testing.TB, opts ...Option) ForestStorage) { s := constructor(t) + defer func() { require.NoError(t, s.Close()) }() checkExists := func(t *testing.T, expected bool, cid cidSDK.ID, treeID string) { actual, err := s.TreeExists(context.Background(), cid, treeID) @@ -663,6 +675,8 @@ func TestApplyTricky1(t *testing.T) { for i := range providers { t.Run(providers[i].name, func(t *testing.T) { s := providers[i].construct(t) + defer func() { require.NoError(t, s.Close()) }() + for i := range ops { require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } @@ -724,6 +738,8 @@ func TestApplyTricky2(t *testing.T) { for i := range providers { t.Run(providers[i].name, func(t *testing.T) { s := providers[i].construct(t) + defer func() { require.NoError(t, s.Close()) }() + for i := range ops { require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } @@ -821,7 +837,7 @@ func compareForests(t *testing.T, expected, actual Forest, cid cidSDK.ID, treeID } } -func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ ...Option) Forest, batchSize, opCount, iterCount int) { +func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ ...Option) ForestStorage, batchSize, opCount, iterCount int) { r := mrand.New(mrand.NewSource(42)) const nodeCount = 5 @@ -832,6 +848,8 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ treeID := "version" expected := constructor(t, WithNoSync(true)) + defer func() { require.NoError(t, expected.Close()) }() + for i := range ops { require.NoError(t, expected.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } @@ -860,10 +878,11 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ wg.Wait() compareForests(t, expected, actual, cid, treeID, nodeCount) + require.NoError(t, actual.Close()) } } -func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ ...Option) Forest) { +func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ ...Option) ForestStorage) { r := mrand.New(mrand.NewSource(42)) const ( @@ -877,6 +896,8 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ .. treeID := "version" expected := constructor(t, WithNoSync(true)) + defer func() { require.NoError(t, expected.Close()) }() + for i := range ops { require.NoError(t, expected.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } @@ -891,6 +912,7 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ .. require.NoError(t, actual.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } compareForests(t, expected, actual, cid, treeID, nodeCount) + require.NoError(t, actual.Close()) } } @@ -908,6 +930,8 @@ func BenchmarkApplySequential(b *testing.B) { b.Run("batchsize="+strconv.Itoa(bs), func(b *testing.B) { r := mrand.New(mrand.NewSource(time.Now().Unix())) s := providers[i].construct(b, WithMaxBatchSize(bs)) + defer func() { require.NoError(b, s.Close()) }() + benchmarkApply(b, s, func(opCount int) []Move { ops := make([]Move, opCount) for i := range ops { @@ -942,6 +966,8 @@ func BenchmarkApplyReorderLast(b *testing.B) { b.Run("batchsize="+strconv.Itoa(bs), func(b *testing.B) { r := mrand.New(mrand.NewSource(time.Now().Unix())) s := providers[i].construct(b, WithMaxBatchSize(bs)) + defer func() { require.NoError(b, s.Close()) }() + benchmarkApply(b, s, func(opCount int) []Move { ops := make([]Move, opCount) for i := range ops { @@ -996,7 +1022,8 @@ func TestTreeGetByPath(t *testing.T) { } } -func testTreeGetByPath(t *testing.T, s Forest) { +func testTreeGetByPath(t *testing.T, s ForestStorage) { + defer func() { require.NoError(t, s.Close()) }() cid := cidtest.ID() treeID := "version" @@ -1074,7 +1101,9 @@ func TestGetTrees(t *testing.T) { } } -func testTreeGetTrees(t *testing.T, s Forest) { +func testTreeGetTrees(t *testing.T, s ForestStorage) { + defer func() { require.NoError(t, s.Close()) }() + cids := []cidSDK.ID{cidtest.ID(), cidtest.ID()} d := CIDDescriptor{Position: 0, Size: 1} @@ -1118,7 +1147,9 @@ func TestTreeLastSyncHeight(t *testing.T) { } } -func testTreeLastSyncHeight(t *testing.T, f Forest) { +func testTreeLastSyncHeight(t *testing.T, f ForestStorage) { + defer func() { require.NoError(t, f.Close()) }() + cnr := cidtest.ID() treeID := "someTree" diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index f7687e961..9f205fa5d 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -38,6 +38,7 @@ func TestShard_Delete_BigObject(t *testing.T) { func testShard(t *testing.T, hasWriteCache bool, payloadSize int) { sh := newShard(t, hasWriteCache) + defer func() { require.NoError(t, sh.Close()) }() cnr := cidtest.ID() diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index a82fc3ffe..6a7941e04 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -78,10 +78,7 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { sh.gcCfg.testHookRemover = func(context.Context) gcRunResult { return gcRunResult{} } require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) - - t.Cleanup(func() { - require.NoError(t, sh.Close()) - }) + defer func() { require.NoError(t, sh.Close()) }() cnr := cidtest.ID() obj := testutil.GenerateObjectWithCID(cnr) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 8b535200d..bd8e0ac58 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -31,6 +31,7 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { return util.NewPseudoWorkerPool() // synchronous event processing })}, }) + defer func() { require.NoError(t, sh.Close()) }() cnr := cidtest.ID() @@ -127,6 +128,7 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { return util.NewPseudoWorkerPool() // synchronous event processing })}, }) + defer func() { require.NoError(t, sh.Close()) }() lock := testutil.GenerateObjectWithCID(cnr) lock.SetType(objectSDK.TypeLock) diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index 19a5e8d70..8a7c6972d 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -32,6 +32,7 @@ func TestShard_Get(t *testing.T) { func testShardGet(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) + defer func() { require.NoError(t, sh.Close()) }() var putPrm PutPrm var getPrm GetPrm diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index dfae48e84..1f4631993 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -30,6 +30,7 @@ func TestShard_Head(t *testing.T) { func testShardHead(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) + defer func() { require.NoError(t, sh.Close()) }() var putPrm PutPrm var headPrm HeadPrm diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 6c8e46faf..82754568f 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -27,6 +27,7 @@ func TestShard_Inhume(t *testing.T) { func testShardInhume(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) + defer func() { require.NoError(t, sh.Close()) }() cnr := cidtest.ID() diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index 9ca1753c4..8a49a36fd 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -18,12 +18,14 @@ func TestShard_List(t *testing.T) { t.Run("without write cache", func(t *testing.T) { t.Parallel() sh := newShard(t, false) + defer func() { require.NoError(t, sh.Close()) }() testShardList(t, sh) }) t.Run("with write cache", func(t *testing.T) { t.Parallel() shWC := newShard(t, true) + defer func() { require.NoError(t, shWC.Close()) }() testShardList(t, shWC) }) } diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index ca6b0ca38..91033efb7 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -61,9 +61,7 @@ func TestShard_Lock(t *testing.T) { require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) - t.Cleanup(func() { - releaseShard(sh, t) - }) + defer func() { require.NoError(t, sh.Close()) }() cnr := cidtest.ID() obj := testutil.GenerateObjectWithCID(cnr) @@ -149,6 +147,7 @@ func TestShard_Lock(t *testing.T) { func TestShard_IsLocked(t *testing.T) { sh := newShard(t, false) + defer func() { require.NoError(t, sh.Close()) }() cnr := cidtest.ID() obj := testutil.GenerateObjectWithCID(cnr) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 9c81c747d..2ab99eed3 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -160,6 +160,7 @@ func TestCounters(t *testing.T) { dir := t.TempDir() sh, mm := shardWithMetrics(t, dir) + defer func() { require.NoError(t, sh.Close()) }() sh.SetMode(mode.ReadOnly) require.Equal(t, mode.ReadOnly, mm.mode) @@ -382,10 +383,6 @@ func shardWithMetrics(t *testing.T, path string) (*Shard, *metricsStore) { require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) - t.Cleanup(func() { - sh.Close() - }) - return sh, mm } diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 28bbd30e5..5a8a29e0c 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -93,6 +93,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { }), }, }) + defer func() { require.NoError(t, sh.Close()) }() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index c76fc1a00..511ce0721 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -57,6 +57,10 @@ func TestShardReload(t *testing.T) { require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) + defer func() { + require.NoError(t, sh.Close()) + }() + objects := make([]objAddr, 5) for i := range objects { objects[i].obj = newObject() diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 98d1c7b8e..a5483f632 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -30,7 +30,6 @@ func (s epochState) CurrentEpoch() uint64 { type shardOptions struct { rootPath string - dontRelease bool wcOpts []writecache.Option bsOpts []blobstor.Option metaOptions []meta.Option @@ -109,13 +108,5 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) - if !o.dontRelease { - t.Cleanup(func() { releaseShard(sh, t) }) - } - return sh } - -func releaseShard(s *Shard, t testing.TB) { - require.NoError(t, s.Close()) -} diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index b94ea50dc..de00eabd1 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -39,7 +39,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { writecache.WithMaxObjectSize(smallSize * 2), } - sh := newCustomShard(t, true, shardOptions{dontRelease: true, rootPath: dir, wcOpts: wcOpts}) + sh := newCustomShard(t, true, shardOptions{rootPath: dir, wcOpts: wcOpts}) var errG errgroup.Group for i := range objects { @@ -55,6 +55,7 @@ func TestWriteCacheObjectLoss(t *testing.T) { require.NoError(t, sh.Close()) sh = newCustomShard(t, true, shardOptions{rootPath: dir, wcOpts: wcOpts}) + defer func() { require.NoError(t, sh.Close()) }() var getPrm GetPrm diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index 727d0fc7a..f7895a935 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -28,6 +28,7 @@ func BenchmarkWritecachePar(b *testing.B) { func benchmarkPutSeq(b *testing.B, cache writecache.Cache, size uint64) { benchmarkPutPrepare(b, cache) + defer func() { require.NoError(b, cache.Close()) }() ctx := context.Background() objGen := testutil.RandObjGenerator{ObjSize: size} @@ -50,6 +51,7 @@ func benchmarkPutSeq(b *testing.B, cache writecache.Cache, size uint64) { func benchmarkPutPar(b *testing.B, cache writecache.Cache, size uint64) { benchmarkPutPrepare(b, cache) + defer func() { require.NoError(b, cache.Close()) }() ctx := context.Background() @@ -75,9 +77,6 @@ func benchmarkPutPar(b *testing.B, cache writecache.Cache, size uint64) { func benchmarkPutPrepare(b *testing.B, cache writecache.Cache) { require.NoError(b, cache.Open(context.Background(), false), "opening") require.NoError(b, cache.Init(), "initializing") - b.Cleanup(func() { - require.NoError(b, cache.Close(), "closing") - }) } type testMetabase struct{} diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index a358028e7..bf68765d8 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -142,6 +142,7 @@ func runFlushTest[Option any]( ) { t.Run("no errors", func(t *testing.T) { wc, bs, mb := newCache(t, createCacheFn, smallSize) + defer func() { require.NoError(t, wc.Close()) }() objects := putObjects(t, wc) require.NoError(t, bs.SetMode(mode.ReadWrite)) @@ -154,6 +155,7 @@ func runFlushTest[Option any]( t.Run("flush on moving to degraded mode", func(t *testing.T) { wc, bs, mb := newCache(t, createCacheFn, smallSize) + defer func() { require.NoError(t, wc.Close()) }() objects := putObjects(t, wc) // Blobstor is read-only, so we expect en error from `flush` here. @@ -172,6 +174,7 @@ func runFlushTest[Option any]( t.Run(f.Desc, func(t *testing.T) { errCountOpt, errCount := errCountOption() wc, bs, mb := newCache(t, createCacheFn, smallSize, errCountOpt) + defer func() { require.NoError(t, wc.Close()) }() objects := putObjects(t, wc) f.InjectFn(t, wc) @@ -214,7 +217,6 @@ func newCache[Option any]( require.NoError(t, bs.Init()) wc := createCacheFn(t, smallSize, mb, bs, opts...) - t.Cleanup(func() { require.NoError(t, wc.Close()) }) require.NoError(t, wc.Open(context.Background(), false)) require.NoError(t, wc.Init()) From 7166e77c2bc3c623ce990e39413ac8ef50e352f1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 9 Jan 2024 17:27:54 +0300 Subject: [PATCH 0791/1943] [#895] test: Add logger to test shard Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/control_test.go | 2 +- pkg/local_object_storage/engine/engine_test.go | 13 +++++++++---- pkg/local_object_storage/engine/list_test.go | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index cdaa41a51..0f36f3203 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -298,7 +298,7 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str return []shard.Option{ shard.WithLogger(test.NewLogger(t)), shard.WithBlobStorOptions( - blobstor.WithStorages(newStorages(filepath.Join(addPath, strconv.Itoa(id)), errSmallSize))), + blobstor.WithStorages(newStorages(t, filepath.Join(addPath, strconv.Itoa(id)), errSmallSize))), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(addPath, fmt.Sprintf("%d.metabase", id))), meta.WithPermissions(0o700), diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 2b9cdcf84..eb15a9900 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -133,14 +133,15 @@ func (te *testEngineWrapper) setShardsNumAdditionalOpts(t testing.TB, num int, s return te } -func newStorages(root string, smallSize uint64) []blobstor.SubStorage { +func newStorages(t testing.TB, root string, smallSize uint64) []blobstor.SubStorage { return []blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( blobovniczatree.WithRootPath(filepath.Join(root, "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1), - blobovniczatree.WithPermissions(0o700)), + blobovniczatree.WithPermissions(0o700), + blobovniczatree.WithLogger(test.NewLogger(t))), Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < smallSize }, @@ -148,7 +149,8 @@ func newStorages(root string, smallSize uint64) []blobstor.SubStorage { { Storage: fstree.New( fstree.WithPath(root), - fstree.WithDepth(1)), + fstree.WithDepth(1), + fstree.WithLogger(test.NewLogger(t))), }, } } @@ -197,12 +199,15 @@ func testDefaultShardOptions(t testing.TB, id int) []shard.Option { shard.WithLogger(test.NewLogger(t)), shard.WithBlobStorOptions( blobstor.WithStorages( - newStorages(t.TempDir(), 1<<20))), + newStorages(t, t.TempDir(), 1<<20)), + blobstor.WithLogger(test.NewLogger(t)), + ), shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(t.TempDir(), "metabase")), meta.WithPermissions(0o700), meta.WithEpochState(epochState{}), + meta.WithLogger(test.NewLogger(t)), ), } } diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 186c3e094..7bb1ac0fa 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -68,7 +68,7 @@ func TestListWithCursor(t *testing.T) { shard.WithLogger(test.NewLogger(t)), shard.WithBlobStorOptions( blobstor.WithStorages( - newStorages(t.TempDir(), 1<<20))), + newStorages(t, t.TempDir(), 1<<20))), shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))), shard.WithMetaBaseOptions( meta.WithPath(filepath.Join(t.TempDir(), "metabase")), From a2ab373a0ac5480b1400a4a0e95d920adc541447 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 10 Jan 2024 11:56:24 +0300 Subject: [PATCH 0792/1943] [#895] metabase: Do not delete GC mark for virtual objects Signed-off-by: Dmitrii Stepanov --- .../engine/delete_test.go | 92 +++++++++++++++++++ .../engine/engine_test.go | 4 +- pkg/local_object_storage/metabase/delete.go | 30 ++++-- pkg/local_object_storage/shard/shard.go | 8 ++ 4 files changed, 124 insertions(+), 10 deletions(-) diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index f3e4a0976..32d07809a 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" @@ -85,6 +86,97 @@ func TestDeleteBigObject(t *testing.T) { } } +func TestDeleteBigObjectWithoutGC(t *testing.T) { + t.Parallel() + + cnr := cidtest.ID() + parentID := oidtest.ID() + splitID := objectSDK.NewSplitID() + + parent := testutil.GenerateObjectWithCID(cnr) + parent.SetID(parentID) + parent.SetPayload(nil) + + const childCount = 3 + children := make([]*objectSDK.Object, childCount) + childIDs := make([]oid.ID, childCount) + for i := range children { + children[i] = testutil.GenerateObjectWithCID(cnr) + if i != 0 { + children[i].SetPreviousID(childIDs[i-1]) + } + if i == len(children)-1 { + children[i].SetParent(parent) + } + children[i].SetSplitID(splitID) + children[i].SetPayload([]byte{byte(i), byte(i + 1), byte(i + 2)}) + childIDs[i], _ = children[i].ID() + } + + link := testutil.GenerateObjectWithCID(cnr) + link.SetParent(parent) + link.SetParentID(parentID) + link.SetSplitID(splitID) + link.SetChildren(childIDs...) + + s1 := testNewShard(t, 1, shard.WithDisabledGC()) + + e := testNewEngine(t).setInitializedShards(t, s1).engine + e.log = test.NewLogger(t) + defer e.Close(context.Background()) + + for i := range children { + require.NoError(t, Put(context.Background(), e, children[i])) + } + require.NoError(t, Put(context.Background(), e, link)) + + addrParent := object.AddressOf(parent) + checkGetError[*objectSDK.SplitInfoError](t, e, addrParent, true) + + addrLink := object.AddressOf(link) + checkGetError[error](t, e, addrLink, false) + + for i := range children { + checkGetError[error](t, e, object.AddressOf(children[i]), false) + } + + // delete logical + var deletePrm DeletePrm + deletePrm.WithForceRemoval() + deletePrm.WithAddress(addrParent) + + _, err := e.Delete(context.Background(), deletePrm) + require.NoError(t, err) + + checkGetError[*apistatus.ObjectNotFound](t, e, addrParent, true) + checkGetError[*apistatus.ObjectNotFound](t, e, addrLink, true) + for i := range children { + checkGetError[*apistatus.ObjectNotFound](t, e, object.AddressOf(children[i]), true) + } + + // delete physical + var delPrm shard.DeletePrm + delPrm.SetAddresses(addrParent) + _, err = s1.Delete(context.Background(), delPrm) + require.NoError(t, err) + + delPrm.SetAddresses(addrLink) + _, err = s1.Delete(context.Background(), delPrm) + require.NoError(t, err) + + for i := range children { + delPrm.SetAddresses(object.AddressOf(children[i])) + _, err = s1.Delete(context.Background(), delPrm) + require.NoError(t, err) + } + + checkGetError[*apistatus.ObjectNotFound](t, e, addrParent, true) + checkGetError[*apistatus.ObjectNotFound](t, e, addrLink, true) + for i := range children { + checkGetError[*apistatus.ObjectNotFound](t, e, object.AddressOf(children[i]), true) + } +} + func checkGetError[E error](t *testing.T, e *StorageEngine, addr oid.Address, shouldFail bool) { var getPrm GetPrm getPrm.WithAddress(addr) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index eb15a9900..a62734c0a 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -181,12 +181,12 @@ func newTestStorages(root string, smallSize uint64) ([]blobstor.SubStorage, *tes }, smallFileStorage, largeFileStorage } -func testNewShard(t testing.TB, id int) *shard.Shard { +func testNewShard(t testing.TB, id int, opts ...shard.Option) *shard.Shard { sid, err := generateShardID() require.NoError(t, err) shardOpts := append([]shard.Option{shard.WithID(sid)}, testDefaultShardOptions(t, id)...) - s := shard.New(shardOpts...) + s := shard.New(append(shardOpts, opts...)...) require.NoError(t, s.Open(context.Background())) require.NoError(t, s.Init(context.Background())) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index b2210b55c..8ca0dea61 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -251,26 +251,27 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter removeAvailableObject := inGraveyardWithKey(addrKey, graveyardBKT, garbageBKT) == 0 - // remove record from the garbage bucket - if garbageBKT != nil { - err := garbageBKT.Delete(addrKey) - if err != nil { - return deleteSingleResult{}, fmt.Errorf("could not remove from garbage bucket: %w", err) - } - } - // unmarshal object, work only with physically stored (raw == true) objects obj, err := db.get(tx, addr, key, false, true, currEpoch) if err != nil { var siErr *objectSDK.SplitInfoError if client.IsErrObjectNotFound(err) || errors.As(err, &siErr) { + // if object is virtual (parent) then do nothing, it will be deleted with last child return deleteSingleResult{}, nil } return deleteSingleResult{}, err } + // remove record from the garbage bucket + if garbageBKT != nil { + err := garbageBKT.Delete(addrKey) + if err != nil { + return deleteSingleResult{}, fmt.Errorf("could not remove from garbage bucket: %w", err) + } + } + // if object is an only link to a parent, then remove parent if parent := obj.Parent(); parent != nil { parAddr := object.AddressOf(parent) @@ -327,6 +328,19 @@ func (db *DB) deleteObject( return fmt.Errorf("can't remove fake bucket tree indexes: %w", err) } + if isParent { + // remove record from the garbage bucket, because regular object deletion does nothing for virtual object + garbageBKT := tx.Bucket(garbageBucketName) + if garbageBKT != nil { + key := make([]byte, addressKeySize) + addrKey := addressKey(object.AddressOf(obj), key) + err := garbageBKT.Delete(addrKey) + if err != nil { + return fmt.Errorf("could not remove from garbage bucket: %w", err) + } + } + } + return nil } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 13f7f689b..136f5463a 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -371,6 +371,14 @@ func WithRebuildWorkerLimiter(l RebuildWorkerLimiter) Option { } } +// WithDisabledGC disables GC. +// For testing purposes only. +func WithDisabledGC() Option { + return func(c *cfg) { + c.gcCfg.testHookRemover = func(ctx context.Context) gcRunResult { return gcRunResult{} } + } +} + // WithZeroSizeCallback returns option to set zero-size containers callback. func WithZeroSizeCallback(cb EmptyContainersCallback) Option { return func(c *cfg) { From be8607a1f6039e782535015a428c578945e12a7d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 Jan 2024 13:47:17 +0300 Subject: [PATCH 0793/1943] [#902] morph: Avoid creating session in TestInvokeIterator When the number of items to iterate over is less than 2048, there is no need to create a session and consume resources. Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 53 +++++++++++++++++---- pkg/morph/client/container/containers_of.go | 2 +- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index e52adfa8e..0c8544bd0 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -13,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "github.com/google/uuid" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -24,8 +25,10 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/rpcclient/rolemgmt" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" + "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" @@ -204,16 +207,27 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, return vub, nil } +// defaultPrefetchBatchSize is the default number of items to prefetch. +// It is dependent on VM limits (2048 items on stack), the default works for simple items. +// For example, to iterate over 2 field structs, the limit should be divided by 3 = 1 (struct itself) + 2 (fields). +const defaultPrefetchBatchSize = vm.MaxStackSize - 16 + // TestInvokeIterator invokes contract method returning an iterator and executes cb on each element. // If cb returns an error, the session is closed and this error is returned as-is. // If the remove neo-go node does not support sessions, `unwrap.ErrNoSessionID` is returned. -func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, contract util.Uint160, method string, args ...interface{}) error { +// batchSize is the number of items to prefetch: if the number of items in the iterator is less than batchSize, no session will be created. +// The default batchSize is 2000 (VM is limited by having 2048 items on stack, so if each iterator item is simple, 2000 items won't hit the limit). +func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int, contract util.Uint160, method string, args ...interface{}) error { start := time.Now() success := false defer func() { c.metrics.ObserveInvoke("TestInvokeIterator", contract.String(), method, success, time.Since(start)) }() + if batchSize <= 0 { + batchSize = defaultPrefetchBatchSize + } + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -221,34 +235,53 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, contract util return ErrConnectionLost } - val, err := c.rpcActor.Call(contract, method, args...) + script, err := smartcontract.CreateCallAndPrefetchIteratorScript(contract, method, batchSize, args...) + if err != nil { + return err + } + + val, err := c.rpcActor.Run(script) if err != nil { return err } else if val.State != HaltState { return wrapFrostFSError(¬HaltStateError{state: val.State, exception: val.FaultException}) } - sid, r, err := unwrap.SessionIterator(val, err) + arr, sid, r, err := unwrap.ArrayAndSessionIterator(val, err) if err != nil { return err } + for i := range arr { + if err := cb(arr[i]); err != nil { + return err + } + } + if (sid == uuid.UUID{}) { + success = true + return nil + } defer func() { _ = c.rpcActor.TerminateSession(sid) }() - items, err := c.rpcActor.TraverseIterator(sid, &r, 0) - for err == nil && len(items) != 0 { + for { + items, err := c.rpcActor.TraverseIterator(sid, &r, batchSize) + if err != nil { + return err + } + for i := range items { - if err = cb(items[i]); err != nil { + if err := cb(items[i]); err != nil { return err } } - items, err = c.rpcActor.TraverseIterator(sid, &r, 0) + if len(items) < batchSize { + break + } } - - success = err == nil - return err + success = true + return nil } // TestInvoke invokes contract method locally in neo-go node. This method should diff --git a/pkg/morph/client/container/containers_of.go b/pkg/morph/client/container/containers_of.go index 8a3c7220f..140047eb2 100644 --- a/pkg/morph/client/container/containers_of.go +++ b/pkg/morph/client/container/containers_of.go @@ -41,7 +41,7 @@ func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) { } cnrHash := c.client.ContractAddress() - err := c.client.Morph().TestInvokeIterator(cb, cnrHash, containersOfMethod, rawID) + err := c.client.Morph().TestInvokeIterator(cb, 0, cnrHash, containersOfMethod, rawID) if err != nil { if errors.Is(err, unwrap.ErrNoSessionID) { return c.List(idUser) From a6013917194960d1f44bf322877e6d9f99d21958 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 Jan 2024 14:04:56 +0300 Subject: [PATCH 0794/1943] [#902] adm: Do not create session when dumping hashes Unless the total number of domains is too big, there is no need to consume resources in neo-go. Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/dump_hashes.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go index 0a112f35c..14a492c5f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go @@ -13,6 +13,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" + "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/emit" @@ -143,7 +144,12 @@ func dumpCustomZoneHashes(cmd *cobra.Command, nnsHash util.Uint160, zone string, }) } - sessionID, iter, err := unwrap.SessionIterator(inv.Call(nnsHash, "tokens")) + script, err := smartcontract.CreateCallAndPrefetchIteratorScript(nnsHash, "tokens", nnsMaxTokens) + if err != nil { + return fmt.Errorf("create prefetch script: %w", err) + } + + arr, sessionID, iter, err := unwrap.ArrayAndSessionIterator(inv.Run(script)) if err != nil { if errors.Is(err, unwrap.ErrNoSessionID) { items, err := unwrap.Array(inv.CallAndExpandIterator(nnsHash, "tokens", nnsMaxTokens)) @@ -160,6 +166,10 @@ func dumpCustomZoneHashes(cmd *cobra.Command, nnsHash util.Uint160, zone string, return err } } else { + for i := range arr { + processItem(arr[i]) + } + defer func() { _ = inv.TerminateSession(sessionID) }() From 394f086fe2c7e56731b13ba1123bc12e71e779bb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 12 Jan 2024 09:56:13 +0300 Subject: [PATCH 0795/1943] [#891] getSvc: Fix get range hash implementation Get range can perform GET request, so this request must be done from container node to not to get access denied error. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 10 +- internal/logs/logs.go | 4 + pkg/services/object/get/v2/get_range_hash.go | 215 ++++++++++++++++++ pkg/services/object/get/v2/service.go | 76 +++++-- .../object_manager/placement/traverser.go | 9 + 5 files changed, 297 insertions(+), 17 deletions(-) create mode 100644 pkg/services/object/get/v2/get_range_hash.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index e84696e6b..a47ea2654 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -176,7 +176,7 @@ func initObjectService(c *cfg) { *c.cfgObject.getSvc = *sGet // need smth better - sGetV2 := createGetServiceV2(sGet, keyStorage) + sGetV2 := createGetServiceV2(c, sGet, keyStorage) sDelete := createDeleteService(c, keyStorage, sGet, sSearch, sPut) @@ -370,10 +370,16 @@ func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Tra getsvc.WithLogger(c.log)) } -func createGetServiceV2(sGet *getsvc.Service, keyStorage *util.KeyStorage) *getsvcV2.Service { +func createGetServiceV2(c *cfg, sGet *getsvc.Service, keyStorage *util.KeyStorage) *getsvcV2.Service { return getsvcV2.NewService( getsvcV2.WithInternalService(sGet), getsvcV2.WithKeyStorage(keyStorage), + getsvcV2.WithEpochSource(c.netMapSource), + getsvcV2.WithClientSource(c.clientCache), + getsvcV2.WithContainerSource(c.cfgObject.cnrSource), + getsvcV2.WithNetmapSource(c.netMapSource), + getsvcV2.WithNetmapAnnouncedKeys(c), + getsvcV2.WithLogger(c.log), ) } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 420c6612a..606b5cb9a 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -567,4 +567,8 @@ const ( EngineInterruptProcessingZeroSizeContainers = "interrupt processing zero-size containers" EngineInterruptProcessingZeroCountContainers = "interrupt processing zero-count containers" EngineFailedToGetContainerCounters = "failed to get container counters" + GetSvcV2FailedToParseNodeEndpoints = "failed to parse node endpoints" + GetSvcV2FailedToParseNodeExternalAddresses = "failed to parse node external addresses" + GetSvcV2FailedToGetRangeHashFromNode = "failed to get range hash from node" + GetSvcV2FailedToGetRangeHashFromAllOfContainerNodes = "failed to get range hash from all of container nodes" ) diff --git a/pkg/services/object/get/v2/get_range_hash.go b/pkg/services/object/get/v2/get_range_hash.go new file mode 100644 index 000000000..d2ed16ebf --- /dev/null +++ b/pkg/services/object/get/v2/get_range_hash.go @@ -0,0 +1,215 @@ +package getsvc + +import ( + "context" + "encoding/hex" + "fmt" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" + rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + clientCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" +) + +// GetRangeHash calls internal service and returns v2 response. +func (s *Service) GetRangeHash(ctx context.Context, req *objectV2.GetRangeHashRequest) (*objectV2.GetRangeHashResponse, error) { + forward, err := s.needToForwardGetRangeHashRequest(req) + if err != nil { + return nil, err + } + if forward.needToForward { + return s.forwardGetRangeHashRequest(ctx, req, forward) + } + p, err := s.toHashRangePrm(req) + if err != nil { + return nil, err + } + + res, err := s.svc.GetRangeHash(ctx, *p) + if err != nil { + return nil, err + } + + return toHashResponse(req.GetBody().GetType(), res), nil +} + +type getRangeForwardParams struct { + needToForward bool + containerNodes []netmapSDK.NodeInfo + address oid.Address +} + +func (s *Service) needToForwardGetRangeHashRequest(req *objectV2.GetRangeHashRequest) (getRangeForwardParams, error) { + if req.GetMetaHeader().GetTTL() <= 1 { + return getRangeForwardParams{}, nil + } + + var result getRangeForwardParams + addrV2 := req.GetBody().GetAddress() + if addrV2 == nil { + return result, errMissingObjAddress + } + + var addr oid.Address + err := addr.ReadFromV2(*addrV2) + if err != nil { + return result, errInvalidObjAddress(err) + } + result.address = addr + + cont, err := s.contSource.Get(addr.Container()) + if err != nil { + return result, fmt.Errorf("(%T) could not get container: %w", s, err) + } + + epoch, err := s.epochSource.Epoch() + if err != nil { + return result, fmt.Errorf("(%T) could not get epoch: %w", s, err) + } + + nm, err := s.netmapSource.GetNetMapByEpoch(epoch) + if err != nil { + return result, fmt.Errorf("(%T) could not get netmap: %w", s, err) + } + + builder := placement.NewNetworkMapBuilder(nm) + + objectID := addr.Object() + nodesVector, err := builder.BuildPlacement(addr.Container(), &objectID, cont.Value.PlacementPolicy()) + if err != nil { + return result, fmt.Errorf("(%T) could not build object placement: %w", s, err) + } + result.containerNodes = distinctBy(placement.FlattenNodes(nodesVector), func(n netmapSDK.NodeInfo) string { return hex.EncodeToString(n.PublicKey()) }) + + for _, node := range result.containerNodes { + if s.announcedKeys.IsLocalKey(node.PublicKey()) { + return result, nil + } + } + result.needToForward = true + return result, nil +} + +func (s *Service) forwardGetRangeHashRequest(ctx context.Context, req *objectV2.GetRangeHashRequest, params getRangeForwardParams) (*objectV2.GetRangeHashResponse, error) { + key, err := s.keyStorage.GetKey(nil) + if err != nil { + return nil, err + } + + metaHdr := new(session.RequestMetaHeader) + metaHdr.SetTTL(req.GetMetaHeader().GetTTL() - 1) + metaHdr.SetOrigin(req.GetMetaHeader()) + writeCurrentVersion(metaHdr) + req.SetMetaHeader(metaHdr) + + if err := signature.SignServiceMessage(key, req); err != nil { + return nil, err + } + + var firstErr error + for _, node := range params.containerNodes { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + var addrGr network.AddressGroup + if err := addrGr.FromIterator(network.NodeEndpointsIterator(node)); err != nil { + s.log.Warn(logs.GetSvcV2FailedToParseNodeEndpoints, zap.String("node_public_key", hex.EncodeToString(node.PublicKey()))) + continue + } + + var extAddr network.AddressGroup + if len(node.ExternalAddresses()) > 0 { + if err := extAddr.FromStringSlice(node.ExternalAddresses()); err != nil { + s.log.Warn(logs.GetSvcV2FailedToParseNodeExternalAddresses, zap.String("node_public_key", hex.EncodeToString(node.PublicKey()))) + continue + } + } + + var info clientCore.NodeInfo + clientCore.NodeInfoFromNetmapElement(&info, placement.NewNode(addrGr, extAddr, node.PublicKey())) + + resp, err := s.performGetRangeHashOnNode(ctx, req, info) + if err == nil { + return resp, nil + } + if firstErr == nil { + firstErr = err + } + s.log.Debug(logs.GetSvcV2FailedToGetRangeHashFromNode, + zap.String("node_public_key", hex.EncodeToString(node.PublicKey())), + zap.Stringer("address", params.address), + zap.Error(err)) + } + s.log.Debug(logs.GetSvcV2FailedToGetRangeHashFromAllOfContainerNodes, zap.Stringer("address", params.address), zap.Error(firstErr)) + if firstErr != nil { + return nil, firstErr + } + return nil, new(apistatus.ObjectNotFound) +} + +func (s *Service) performGetRangeHashOnNode(ctx context.Context, req *objectV2.GetRangeHashRequest, info clientCore.NodeInfo) (*objectV2.GetRangeHashResponse, error) { + cl, err := s.clientSource.Get(info) + if err != nil { + return nil, err + } + + var firstErr error + var resp *objectV2.GetRangeHashResponse + info.AddressGroup().IterateAddresses(func(a network.Address) bool { + resp, err = s.performGetRangeHashOnAddress(ctx, req, cl, a) + if err != nil { + if firstErr == nil { + firstErr = err + } + return false + } + return true + }) + if firstErr != nil { + return nil, firstErr + } + if resp == nil { + return nil, new(apistatus.ObjectNotFound) + } + return resp, nil +} + +func (s *Service) performGetRangeHashOnAddress(ctx context.Context, req *objectV2.GetRangeHashRequest, cl clientCore.MultiAddressClient, + a network.Address, +) (*objectV2.GetRangeHashResponse, error) { + var resp *objectV2.GetRangeHashResponse + var rpcErr error + err := cl.RawForAddress(ctx, a, func(cli *rpcclient.Client) error { + resp, rpcErr = rpc.HashObjectRange(cli, req, rpcclient.WithContext(ctx)) + return rpcErr + }) + if err != nil { + return nil, err + } + return resp, err +} + +func distinctBy[T any, K comparable](source []T, keySelector func(v T) K) []T { + var result []T + dict := make(map[K]struct{}) + for _, v := range source { + key := keySelector(v) + if _, exists := dict[key]; !exists { + result = append(result, v) + dict[key] = struct{}{} + } + } + return result +} diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index d4bce178a..07fe9b42a 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -5,10 +5,15 @@ import ( "errors" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" objutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.uber.org/zap" ) // Service implements Get operation of Object service v2. @@ -19,10 +24,30 @@ type Service struct { // Option represents Service constructor option. type Option func(*cfg) +type epochSource interface { + Epoch() (uint64, error) +} + +type clientSource interface { + Get(info clientcore.NodeInfo) (clientcore.MultiAddressClient, error) +} + type cfg struct { svc *getsvc.Service keyStorage *objutil.KeyStorage + + epochSource epochSource + + clientSource clientSource + + netmapSource netmap.Source + + announcedKeys netmap.AnnouncedKeys + + contSource container.Source + + log *logger.Logger } // NewService constructs Service instance from provided options. @@ -76,21 +101,6 @@ func (s *Service) GetRange(req *objectV2.GetRangeRequest, stream objectSvc.GetOb } } -// GetRangeHash calls internal service and returns v2 response. -func (s *Service) GetRangeHash(ctx context.Context, req *objectV2.GetRangeHashRequest) (*objectV2.GetRangeHashResponse, error) { - p, err := s.toHashRangePrm(req) - if err != nil { - return nil, err - } - - res, err := s.svc.GetRangeHash(ctx, *p) - if err != nil { - return nil, err - } - - return toHashResponse(req.GetBody().GetType(), res), nil -} - // Head serves ForstFS API v2 compatible HEAD requests. func (s *Service) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { resp := new(objectV2.HeadResponse) @@ -125,3 +135,39 @@ func WithKeyStorage(ks *objutil.KeyStorage) Option { c.keyStorage = ks } } + +func WithEpochSource(es epochSource) Option { + return func(c *cfg) { + c.epochSource = es + } +} + +func WithClientSource(cs clientSource) Option { + return func(c *cfg) { + c.clientSource = cs + } +} + +func WithNetmapSource(ns netmap.Source) Option { + return func(c *cfg) { + c.netmapSource = ns + } +} + +func WithNetmapAnnouncedKeys(ak netmap.AnnouncedKeys) Option { + return func(c *cfg) { + c.announcedKeys = ak + } +} + +func WithContainerSource(cs container.Source) Option { + return func(c *cfg) { + c.contSource = cs + } +} + +func WithLogger(l *logger.Logger) Option { + return func(c *cfg) { + c.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Get V2 service"))} + } +} diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index dc9ab5e7a..53da186e8 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -181,6 +181,15 @@ func (x Node) PublicKey() []byte { return x.key } +// NewNode creates new Node. +func NewNode(addresses network.AddressGroup, externalAddresses network.AddressGroup, key []byte) Node { + return Node{ + addresses: addresses, + externalAddresses: externalAddresses, + key: key, + } +} + // Next returns next unprocessed address of the object placement. // // Returns nil if no nodes left or traversal operation succeeded. From 52ffa9f1642fdee6c94a677c3202c55c9a654845 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 12 Jan 2024 10:05:29 +0300 Subject: [PATCH 0796/1943] [#891] getSvc: Refactor Get service V2 creation Use arguments for mandatory fields. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 13 ++-- pkg/services/object/get/v2/get_range_hash.go | 2 +- pkg/services/object/get/v2/service.go | 68 +++++--------------- 3 files changed, 24 insertions(+), 59 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index a47ea2654..2d18f82a3 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -372,13 +372,12 @@ func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Tra func createGetServiceV2(c *cfg, sGet *getsvc.Service, keyStorage *util.KeyStorage) *getsvcV2.Service { return getsvcV2.NewService( - getsvcV2.WithInternalService(sGet), - getsvcV2.WithKeyStorage(keyStorage), - getsvcV2.WithEpochSource(c.netMapSource), - getsvcV2.WithClientSource(c.clientCache), - getsvcV2.WithContainerSource(c.cfgObject.cnrSource), - getsvcV2.WithNetmapSource(c.netMapSource), - getsvcV2.WithNetmapAnnouncedKeys(c), + sGet, + keyStorage, + c.clientCache, + c.netMapSource, + c, + c.cfgObject.cnrSource, getsvcV2.WithLogger(c.log), ) } diff --git a/pkg/services/object/get/v2/get_range_hash.go b/pkg/services/object/get/v2/get_range_hash.go index d2ed16ebf..0054f0e9f 100644 --- a/pkg/services/object/get/v2/get_range_hash.go +++ b/pkg/services/object/get/v2/get_range_hash.go @@ -71,7 +71,7 @@ func (s *Service) needToForwardGetRangeHashRequest(req *objectV2.GetRangeHashReq return result, fmt.Errorf("(%T) could not get container: %w", s, err) } - epoch, err := s.epochSource.Epoch() + epoch, err := s.netmapSource.Epoch() if err != nil { return result, fmt.Errorf("(%T) could not get epoch: %w", s, err) } diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index 07fe9b42a..bcdc4120e 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -24,10 +24,6 @@ type Service struct { // Option represents Service constructor option. type Option func(*cfg) -type epochSource interface { - Epoch() (uint64, error) -} - type clientSource interface { Get(info clientcore.NodeInfo) (clientcore.MultiAddressClient, error) } @@ -37,8 +33,6 @@ type cfg struct { keyStorage *objutil.KeyStorage - epochSource epochSource - clientSource clientSource netmapSource netmap.Source @@ -51,8 +45,23 @@ type cfg struct { } // NewService constructs Service instance from provided options. -func NewService(opts ...Option) *Service { - c := new(cfg) +func NewService(svc *getsvc.Service, + keyStorage *objutil.KeyStorage, + clientSource clientSource, + netmapSource netmap.Source, + announcedKeys netmap.AnnouncedKeys, + contSource container.Source, + opts ...Option, +) *Service { + c := &cfg{ + svc: svc, + keyStorage: keyStorage, + clientSource: clientSource, + netmapSource: netmapSource, + announcedKeys: announcedKeys, + contSource: contSource, + log: &logger.Logger{Logger: zap.L()}, + } for i := range opts { opts[i](c) @@ -123,49 +132,6 @@ func (s *Service) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV return resp, err } -func WithInternalService(v *getsvc.Service) Option { - return func(c *cfg) { - c.svc = v - } -} - -// WithKeyStorage returns option to set local private key storage. -func WithKeyStorage(ks *objutil.KeyStorage) Option { - return func(c *cfg) { - c.keyStorage = ks - } -} - -func WithEpochSource(es epochSource) Option { - return func(c *cfg) { - c.epochSource = es - } -} - -func WithClientSource(cs clientSource) Option { - return func(c *cfg) { - c.clientSource = cs - } -} - -func WithNetmapSource(ns netmap.Source) Option { - return func(c *cfg) { - c.netmapSource = ns - } -} - -func WithNetmapAnnouncedKeys(ak netmap.AnnouncedKeys) Option { - return func(c *cfg) { - c.announcedKeys = ak - } -} - -func WithContainerSource(cs container.Source) Option { - return func(c *cfg) { - c.contSource = cs - } -} - func WithLogger(l *logger.Logger) Option { return func(c *cfg) { c.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Get V2 service"))} From e43609c61622af953028d2c7743e06c0e3b688e8 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 12 Jan 2024 12:02:11 +0300 Subject: [PATCH 0797/1943] [#885] go.mod: Update APE Signed-off-by: Denis Kirillov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 82ad0212d..ddf587ad5 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231221121354-ed93bb5cc574 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240112075604-c80c99b13e53 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 diff --git a/go.sum b/go.sum index 75d2487c4..9be0e55cd 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231221121354-ed93bb5cc574 h1:PmPaHc4P/Rjn1WQcDF5gcwILzWjkMOuRHLqzm9OZCdA= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20231221121354-ed93bb5cc574/go.mod h1:ps6oKO0mxaPJzK3admTB3iwoBXKkHnS73n4PCrqpHBg= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240112075604-c80c99b13e53 h1:LNfUfiGx3cECl15Q5ZspILWZriCeevH4Ue+eynKgGYQ= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240112075604-c80c99b13e53/go.mod h1:ps6oKO0mxaPJzK3admTB3iwoBXKkHnS73n4PCrqpHBg= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= From c8baf76fae5e0bedd49d66fcd8fe003996766f49 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 27 Dec 2023 17:18:15 +0300 Subject: [PATCH 0798/1943] [#872] object: Introduce APE middlewar for object service Signed-off-by: Airat Arifullin --- cmd/frostfs-node/object.go | 23 +- pkg/services/object/acl/ape.go | 49 --- pkg/services/object/acl/ape_request.go | 100 ------ pkg/services/object/acl/v2/request.go | 7 + pkg/services/object/acl/v2/service.go | 104 +++++- pkg/services/object/acl/v2/types.go | 6 - pkg/services/object/ape/checker.go | 82 +++++ pkg/services/object/ape/checker_test.go | 348 ++++++++++++++++++++ pkg/services/object/ape/request.go | 159 ++++++++++ pkg/services/object/ape/request_test.go | 324 +++++++++++++++++++ pkg/services/object/ape/service.go | 404 ++++++++++++++++++++++++ pkg/services/object/ape/types.go | 9 + pkg/services/object/request_context.go | 16 + 13 files changed, 1456 insertions(+), 175 deletions(-) delete mode 100644 pkg/services/object/acl/ape.go delete mode 100644 pkg/services/object/acl/ape_request.go create mode 100644 pkg/services/object/ape/checker.go create mode 100644 pkg/services/object/ape/checker_test.go create mode 100644 pkg/services/object/ape/request.go create mode 100644 pkg/services/object/ape/request_test.go create mode 100644 pkg/services/object/ape/service.go create mode 100644 pkg/services/object/ape/types.go create mode 100644 pkg/services/object/request_context.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 2d18f82a3..5c82f0dcc 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -24,6 +24,7 @@ import ( objectService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl" v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" + objectAPE "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/ape" deletesvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/delete" deletesvcV2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/delete/v2" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" @@ -183,11 +184,13 @@ func initObjectService(c *cfg) { sDeleteV2 := createDeleteServiceV2(sDelete) // build service pipeline - // grpc | | signature | response | acl | split + // grpc | | signature | response | acl | ape | split splitSvc := createSplitService(c, sPutV2, sGetV2, sSearchV2, sDeleteV2) - aclSvc := createACLServiceV2(c, splitSvc, &irFetcher) + apeSvc := createAPEService(c, splitSvc) + + aclSvc := createACLServiceV2(c, apeSvc, &irFetcher) var commonSvc objectService.Common commonSvc.Init(&c.internals, aclSvc) @@ -419,11 +422,11 @@ func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Servi ) } -func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter, irFetcher *cachedIRFetcher) v2.Service { +func createACLServiceV2(c *cfg, apeSvc *objectAPE.Service, irFetcher *cachedIRFetcher) v2.Service { ls := c.cfgObject.cfgLocalStorage.localStorage return v2.New( - splitSvc, + apeSvc, c.netMapSource, irFetcher, acl.NewChecker( @@ -431,12 +434,22 @@ func createACLServiceV2(c *cfg, splitSvc *objectService.TransportSplitter, irFet c.cfgObject.eaclSource, eaclSDK.NewValidator(), ls), - acl.NewAPEChecker(c.log, c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.chainRouter), c.cfgObject.cnrSource, v2.WithLogger(c.log), ) } +func createAPEService(c *cfg, splitSvc *objectService.TransportSplitter) *objectAPE.Service { + return objectAPE.NewService( + c.log, + objectAPE.NewChecker( + c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.chainRouter, + objectAPE.NewStorageEngineHeaderProvider(c.cfgObject.cfgLocalStorage.localStorage), + ), + splitSvc, + ) +} + type morphEACLFetcher struct { w *cntClient.Client } diff --git a/pkg/services/object/acl/ape.go b/pkg/services/object/acl/ape.go deleted file mode 100644 index a431ad252..000000000 --- a/pkg/services/object/acl/ape.go +++ /dev/null @@ -1,49 +0,0 @@ -package acl - -import ( - "fmt" - - v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" - policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" -) - -type apeCheckerImpl struct { - log *logger.Logger - chainRouter policyengine.ChainRouter -} - -func NewAPEChecker(log *logger.Logger, chainRouter policyengine.ChainRouter) v2.APEChainChecker { - return &apeCheckerImpl{ - log: log, - chainRouter: chainRouter, - } -} - -func (c *apeCheckerImpl) CheckIfRequestPermitted(reqInfo v2.RequestInfo) error { - request := new(Request) - request.FromRequestInfo(reqInfo) - - cnrTarget := reqInfo.ContainerID().EncodeToString() - - status, ruleFound, err := c.chainRouter.IsAllowed(apechain.Ingress, policyengine.NewRequestTargetWithContainer(cnrTarget), request) - if err != nil { - return err - } - - if !ruleFound || status == apechain.Allow { - return nil - } - - return apeErr(reqInfo, status) -} - -const accessDeniedAPEReasonFmt = "access to operation %s is denied by access policy engine: %s" - -func apeErr(req v2.RequestInfo, status apechain.Status) error { - errAccessDenied := &apistatus.ObjectAccessDenied{} - errAccessDenied.WriteReason(fmt.Sprintf(accessDeniedAPEReasonFmt, req.Operation(), status.String())) - return errAccessDenied -} diff --git a/pkg/services/object/acl/ape_request.go b/pkg/services/object/acl/ape_request.go deleted file mode 100644 index accbdce5f..000000000 --- a/pkg/services/object/acl/ape_request.go +++ /dev/null @@ -1,100 +0,0 @@ -package acl - -import ( - "fmt" - - v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" - aclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" - aperesource "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" - nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" -) - -type Request struct { - operation string - resource *resource - properties map[string]string -} - -var _ aperesource.Request = (*Request)(nil) - -type resource struct { - name string - properties map[string]string -} - -var _ aperesource.Resource = (*resource)(nil) - -func (r *resource) Name() string { - return r.name -} - -func (r *resource) Property(key string) string { - return r.properties[key] -} - -func getResource(reqInfo v2.RequestInfo) *resource { - var name string - cid := reqInfo.ContainerID() - if oid := reqInfo.ObjectID(); oid != nil { - name = fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, cid.EncodeToString(), oid.EncodeToString()) - } else { - name = fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) - } - return &resource{ - name: name, - properties: make(map[string]string), - } -} - -func getProperties(_ v2.RequestInfo) map[string]string { - return map[string]string{ - nativeschema.PropertyKeyActorPublicKey: "", - nativeschema.PropertyKeyActorRole: "", - } -} - -func getOperation(reqInfo v2.RequestInfo) string { - switch op := reqInfo.Operation(); op { - case aclSDK.OpObjectGet: - return nativeschema.MethodGetObject - case aclSDK.OpObjectHead: - return nativeschema.MethodHeadObject - case aclSDK.OpObjectPut: - return nativeschema.MethodPutObject - case aclSDK.OpObjectDelete: - return nativeschema.MethodDeleteObject - case aclSDK.OpObjectSearch: - return nativeschema.MethodSearchObject - case aclSDK.OpObjectRange: - return nativeschema.MethodRangeObject - case aclSDK.OpObjectHash: - return nativeschema.MethodHashObject - default: - return "" - } -} - -func NewRequest() *Request { - return &Request{ - resource: new(resource), - properties: map[string]string{}, - } -} - -func (r *Request) FromRequestInfo(ri v2.RequestInfo) { - r.operation = getOperation(ri) - r.resource = getResource(ri) - r.properties = getProperties(ri) -} - -func (r *Request) Operation() string { - return r.operation -} - -func (r *Request) Property(key string) string { - return r.properties[key] -} - -func (r *Request) Resource() aperesource.Resource { - return r.resource -} diff --git a/pkg/services/object/acl/v2/request.go b/pkg/services/object/acl/v2/request.go index 675768969..c58c00e0e 100644 --- a/pkg/services/object/acl/v2/request.go +++ b/pkg/services/object/acl/v2/request.go @@ -22,6 +22,9 @@ type RequestInfo struct { operation acl.Op // put, get, head, etc. cnrOwner user.ID // container owner + // cnrNamespace defined to which namespace a container is belonged. + cnrNamespace string + idCnr cid.ID // optional for some request @@ -57,6 +60,10 @@ func (r RequestInfo) ContainerOwner() user.ID { return r.cnrOwner } +func (r RequestInfo) ContainerNamespace() string { + return r.cnrNamespace +} + // ObjectID return object ID. func (r RequestInfo) ObjectID() *oid.ID { return r.obj diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index ee167d0e4..ed077411c 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + containerV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" @@ -67,10 +68,6 @@ type cfg struct { checker ACLChecker - // TODO(aarifullin): apeCheck is temporarily the part of - // acl service and must be standalone. - apeChecker APEChainChecker - irFetcher InnerRingFetcher nm netmap.Source @@ -83,7 +80,6 @@ func New(next object.ServiceServer, nm netmap.Source, irf InnerRingFetcher, acl ACLChecker, - apeChecker APEChainChecker, cs container.Source, opts ...Option, ) Service { @@ -93,7 +89,6 @@ func New(next object.ServiceServer, nm: nm, irFetcher: irf, checker: acl, - apeChecker: apeChecker, containers: cs, } @@ -107,6 +102,75 @@ func New(next object.ServiceServer, } } +// wrappedGetObjectStream propagates RequestContext into GetObjectStream's context. +// This allows to retrieve already calculated immutable request-specific values in next handler invocation. +type wrappedGetObjectStream struct { + object.GetObjectStream + + requestInfo RequestInfo +} + +func (w *wrappedGetObjectStream) Context() context.Context { + return context.WithValue(w.GetObjectStream.Context(), object.RequestContextKey, &object.RequestContext{ + Namespace: w.requestInfo.ContainerNamespace(), + SenderKey: w.requestInfo.SenderKey(), + Role: w.requestInfo.RequestRole(), + }) +} + +func newWrappedGetObjectStreamStream(getObjectStream object.GetObjectStream, reqInfo RequestInfo) object.GetObjectStream { + return &wrappedGetObjectStream{ + GetObjectStream: getObjectStream, + requestInfo: reqInfo, + } +} + +// wrappedRangeStream propagates RequestContext into GetObjectRangeStream's context. +// This allows to retrieve already calculated immutable request-specific values in next handler invocation. +type wrappedRangeStream struct { + object.GetObjectRangeStream + + requestInfo RequestInfo +} + +func (w *wrappedRangeStream) Context() context.Context { + return context.WithValue(w.GetObjectRangeStream.Context(), object.RequestContextKey, &object.RequestContext{ + Namespace: w.requestInfo.ContainerNamespace(), + SenderKey: w.requestInfo.SenderKey(), + Role: w.requestInfo.RequestRole(), + }) +} + +func newWrappedRangeStream(rangeStream object.GetObjectRangeStream, reqInfo RequestInfo) object.GetObjectRangeStream { + return &wrappedRangeStream{ + GetObjectRangeStream: rangeStream, + requestInfo: reqInfo, + } +} + +// wrappedSearchStream propagates RequestContext into SearchStream's context. +// This allows to retrieve already calculated immutable request-specific values in next handler invocation. +type wrappedSearchStream struct { + object.SearchStream + + requestInfo RequestInfo +} + +func (w *wrappedSearchStream) Context() context.Context { + return context.WithValue(w.SearchStream.Context(), object.RequestContextKey, &object.RequestContext{ + Namespace: w.requestInfo.ContainerNamespace(), + SenderKey: w.requestInfo.SenderKey(), + Role: w.requestInfo.RequestRole(), + }) +} + +func newWrappedSearchStream(searchStream object.SearchStream, reqInfo RequestInfo) object.SearchStream { + return &wrappedSearchStream{ + SearchStream: searchStream, + requestInfo: reqInfo, + } +} + // Get implements ServiceServer interface, makes ACL checks and calls // next Get method in the ServiceServer pipeline. func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream) error { @@ -158,7 +222,7 @@ func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream } return b.next.Get(request, &getStreamBasicChecker{ - GetObjectStream: stream, + GetObjectStream: newWrappedGetObjectStreamStream(stream, reqInfo), info: reqInfo, checker: b.checker, }) @@ -224,7 +288,7 @@ func (b Service) Head( return nil, eACLErr(reqInfo, err) } - resp, err := b.next.Head(ctx, request) + resp, err := b.next.Head(requestContext(ctx, reqInfo), request) if err == nil { if err = b.checker.CheckEACL(resp, reqInfo); err != nil { err = eACLErr(reqInfo, err) @@ -277,7 +341,7 @@ func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStr return b.next.Search(request, &searchStreamBasicChecker{ checker: b.checker, - SearchStream: stream, + SearchStream: newWrappedSearchStream(stream, reqInfo), info: reqInfo, }) } @@ -333,7 +397,7 @@ func (b Service) Delete( return nil, eACLErr(reqInfo, err) } - return b.next.Delete(ctx, request) + return b.next.Delete(requestContext(ctx, reqInfo), request) } func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetObjectRangeStream) error { @@ -386,11 +450,18 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb return b.next.GetRange(request, &rangeStreamBasicChecker{ checker: b.checker, - GetObjectRangeStream: stream, + GetObjectRangeStream: newWrappedRangeStream(stream, reqInfo), info: reqInfo, }) } +func requestContext(ctx context.Context, reqInfo RequestInfo) context.Context { + return context.WithValue(ctx, object.RequestContextKey, &object.RequestContext{ + SenderKey: reqInfo.SenderKey(), + Role: reqInfo.RequestRole(), + }) +} + func (b Service) GetRangeHash( ctx context.Context, request *objectV2.GetRangeHashRequest, @@ -442,7 +513,7 @@ func (b Service) GetRangeHash( return nil, eACLErr(reqInfo, err) } - return b.next.GetRangeHash(ctx, request) + return b.next.GetRangeHash(requestContext(ctx, reqInfo), request) } func (b Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { @@ -501,7 +572,7 @@ func (b Service) PutSingle(ctx context.Context, request *objectV2.PutSingleReque return nil, eACLErr(reqInfo, err) } - return b.next.PutSingle(ctx, request) + return b.next.PutSingle(requestContext(ctx, reqInfo), request) } func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { @@ -566,9 +637,11 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe reqInfo.obj = obj - if err := p.source.apeChecker.CheckIfRequestPermitted(reqInfo); err != nil { - return err + if !p.source.checker.CheckBasicACL(reqInfo) || !p.source.checker.StickyBitCheck(reqInfo, idOwner) { + return basicACLErr(reqInfo) } + + ctx = requestContext(ctx, reqInfo) } return p.next.Send(ctx, request) @@ -671,6 +744,7 @@ func (b Service) findRequestInfo(req MetaWithToken, idCnr cid.ID, op acl.Op) (in info.operation = op info.cnrOwner = cnr.Value.Owner() info.idCnr = idCnr + info.cnrNamespace = cnr.Value.Attribute(containerV2.SysAttributeZone) // it is assumed that at the moment the key will be valid, // otherwise the request would not pass validation diff --git a/pkg/services/object/acl/v2/types.go b/pkg/services/object/acl/v2/types.go index a113c4693..061cd26b6 100644 --- a/pkg/services/object/acl/v2/types.go +++ b/pkg/services/object/acl/v2/types.go @@ -26,9 +26,3 @@ type InnerRingFetcher interface { // the actual inner ring. InnerRingKeys() ([][]byte, error) } - -// APEChainChecker is the interface that provides methods to -// check if the access policy engine permits to perform the request. -type APEChainChecker interface { - CheckIfRequestPermitted(RequestInfo) error -} diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go new file mode 100644 index 000000000..fdaa9df85 --- /dev/null +++ b/pkg/services/object/ape/checker.go @@ -0,0 +1,82 @@ +package ape + +import ( + "context" + "fmt" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" +) + +type checkerImpl struct { + chainRouter policyengine.ChainRouter + + headerProvider HeaderProvider +} + +func NewChecker(chainRouter policyengine.ChainRouter, headerProvider HeaderProvider) Checker { + return &checkerImpl{ + chainRouter: chainRouter, + + headerProvider: headerProvider, + } +} + +type Prm struct { + Namespace string + + Container cid.ID + + // Object ID is omitted for some methods. + Object *oid.ID + + // If Header is set, then object attributes and properties will be parsed from + // a request/response's header. + Header *objectV2.Header + + // Method must be represented only as a constant represented in native schema. + Method string + + // Role must be representedonly as a constant represented in native schema. + Role string + + // An encoded sender's public key string. + SenderKey string +} + +var ( + errMissingOID = fmt.Errorf("object ID is not set") +) + +// CheckAPE checks if a request or a response is permitted creating an ape request and passing +// it to chain router. +func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { + r, err := c.newAPERequest(ctx, prm) + if err != nil { + return fmt.Errorf("failed to create ape request: %w", err) + } + + status, ruleFound, err := c.chainRouter.IsAllowed(apechain.Ingress, + policyengine.NewRequestTarget(prm.Namespace, prm.Container.EncodeToString()), r) + if err != nil { + return err + } + + if !ruleFound || status == apechain.Allow { + return nil + } + + return apeErr(prm.Method, status) +} + +const accessDeniedAPEReasonFmt = "access to operation %s is denied by access policy engine: %s" + +func apeErr(op string, status apechain.Status) error { + errAccessDenied := &apistatus.ObjectAccessDenied{} + errAccessDenied.WriteReason(fmt.Sprintf(accessDeniedAPEReasonFmt, op, status.String())) + return errAccessDenied +} diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go new file mode 100644 index 000000000..9d548480b --- /dev/null +++ b/pkg/services/object/ape/checker_test.go @@ -0,0 +1,348 @@ +package ape + +import ( + "context" + "encoding/hex" + "fmt" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/stretchr/testify/require" +) + +type headerProviderMock struct { + m map[oid.Address]*objectSDK.Object +} + +var _ HeaderProvider = (*headerProviderMock)(nil) + +func (h *headerProviderMock) addHeader(c cid.ID, o oid.ID, header *objectSDK.Object) { + var addr oid.Address + addr.SetContainer(c) + addr.SetObject(o) + h.m[addr] = header +} + +func (h *headerProviderMock) GetHeader(_ context.Context, c cid.ID, o oid.ID) (*objectSDK.Object, error) { + var addr oid.Address + addr.SetContainer(c) + addr.SetObject(o) + obj, ok := h.m[addr] + if !ok { + return nil, fmt.Errorf("address not found") + } + return obj, nil +} + +func newHeaderProviderMock() *headerProviderMock { + return &headerProviderMock{ + m: make(map[oid.Address]*objectSDK.Object), + } +} + +func newContainerIDSDK(t *testing.T, encodedCID string) cid.ID { + var cnr cid.ID + require.NoError(t, cnr.DecodeString(encodedCID)) + return cnr +} + +func newObjectIDSDK(t *testing.T, encodedOID *string) *oid.ID { + if encodedOID == nil { + return nil + } + obj := new(oid.ID) + require.NoError(t, obj.DecodeString(*encodedOID)) + return obj +} + +type headerObjectSDKParams struct { + majorVersion, minorVersion uint32 + owner user.ID + epoch uint64 + payloadSize uint64 + typ objectSDK.Type + payloadChecksum checksum.Checksum + payloadHomomorphicHash checksum.Checksum + attributes []struct { + key string + val string + } +} + +func stringPtr(s string) *string { + return &s +} + +func newHeaderObjectSDK(t *testing.T, cnr cid.ID, oid *oid.ID, headerObjSDK *headerObjectSDKParams) *objectSDK.Object { + objSDK := objectSDK.New() + objSDK.SetContainerID(cnr) + if oid != nil { + objSDK.SetID(*oid) + } + if headerObjSDK == nil { + return objSDK + } + ver := new(version.Version) + ver.SetMajor(headerObjSDK.majorVersion) + ver.SetMinor(headerObjSDK.minorVersion) + objSDK.SetVersion(ver) + objSDK.SetCreationEpoch(headerObjSDK.epoch) + objSDK.SetOwnerID(headerObjSDK.owner) + objSDK.SetPayloadSize(headerObjSDK.payloadSize) + objSDK.SetType(headerObjSDK.typ) + objSDK.SetPayloadChecksum(headerObjSDK.payloadChecksum) + objSDK.SetPayloadHomomorphicHash(headerObjSDK.payloadHomomorphicHash) + + var attrs []objectSDK.Attribute + for _, attr := range headerObjSDK.attributes { + attrSDK := objectSDK.NewAttribute() + attrSDK.SetKey(attr.key) + attrSDK.SetValue(attr.val) + attrs = append(attrs, *attrSDK) + } + objSDK.SetAttributes(attrs...) + + return objSDK +} + +type testHeader struct { + headerObjSDK *headerObjectSDKParams + + // If fromHeaderProvider is set, then running test should + // consider that a header is recieved from a header provider. + fromHeaderProvider bool + + // If fromHeaderProvider is set, then running test should + // consider that a header is recieved from a message header. + fromRequestResponseHeader bool +} + +var ( + methodsRequiredOID = []string{ + nativeschema.MethodGetObject, + nativeschema.MethodHeadObject, + nativeschema.MethodRangeObject, + nativeschema.MethodHashObject, + nativeschema.MethodDeleteObject, + } + + methodsOptionalOID = []string{ + nativeschema.MethodSearchObject, nativeschema.MethodPutObject, + } + + namespace = "test_namespace" + + containerID = "73tQMTYyUkTgmvPR1HWib6pndbhSoBovbnMF7Pws8Rcy" + + objectID = "BzQw5HH3feoxFDD5tCT87Y1726qzgLfxEE7wgtoRzB3R" + + role = "Container" + + senderKey = hex.EncodeToString([]byte{1, 0, 0, 1}) +) + +func TestAPECheck(t *testing.T) { + for _, test := range []struct { + name string + container string + object *string + methods []string + header testHeader + containerRules []chain.Rule + expectAPEErr bool + }{ + { + name: "oid required requests are allowed", + container: containerID, + object: stringPtr(objectID), + methods: methodsRequiredOID, + }, + { + name: "oid optional requests are allowed", + container: containerID, + methods: methodsOptionalOID, + }, + { + name: "oid required requests are denied", + container: containerID, + object: stringPtr(objectID), + methods: methodsRequiredOID, + containerRules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{Names: methodsRequiredOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, + }, + }, + }, + expectAPEErr: true, + }, + { + name: "oid required requests are denied by an attribute", + container: containerID, + object: stringPtr(objectID), + methods: methodsRequiredOID, + header: testHeader{ + headerObjSDK: &headerObjectSDKParams{ + attributes: []struct { + key string + val string + }{ + { + key: "attr1", + val: "attribute_value", + }, + }, + }, + fromHeaderProvider: true, + }, + containerRules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{Names: methodsRequiredOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, + }, + Any: true, + Condition: []chain.Condition{ + { + Op: chain.CondStringLike, + Object: chain.ObjectResource, + Key: "attr1", + Value: "attribute*", + }, + }, + }, + }, + expectAPEErr: true, + }, + { + name: "oid required requests are denied by sender", + container: containerID, + object: stringPtr(objectID), + methods: methodsRequiredOID, + header: testHeader{ + headerObjSDK: &headerObjectSDKParams{ + attributes: []struct { + key string + val string + }{ + { + key: "attr1", + val: "attribute_value", + }, + }, + }, + fromHeaderProvider: true, + }, + containerRules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{Names: methodsRequiredOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, + }, + Any: true, + Condition: []chain.Condition{ + { + Op: chain.CondStringLike, + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorPublicKey, + Value: senderKey, + }, + }, + }, + }, + expectAPEErr: true, + }, + { + name: "optional oid requests reached quota limit by an attribute", + container: containerID, + methods: methodsOptionalOID, + header: testHeader{ + headerObjSDK: &headerObjectSDKParams{ + payloadSize: 1000, + }, + fromRequestResponseHeader: true, + }, + containerRules: []chain.Rule{ + { + Status: chain.QuotaLimitReached, + Actions: chain.Actions{Names: methodsOptionalOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + Any: true, + Condition: []chain.Condition{ + { + Op: chain.CondStringEquals, + Object: chain.ObjectResource, + Key: nativeschema.PropertyKeyObjectPayloadLength, + Value: "1000", + }, + }, + }, + }, + expectAPEErr: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + for _, method := range test.methods { + t.Run(method, func(t *testing.T) { + headerProvider := newHeaderProviderMock() + + cnr := newContainerIDSDK(t, test.container) + obj := newObjectIDSDK(t, test.object) + + ls := inmemory.NewInmemoryLocalStorage() + ms := inmemory.NewInmemoryMorphRuleChainStorage() + + ls.AddOverride(chain.Ingress, policyengine.ContainerTarget(test.container), &chain.Chain{ + Rules: test.containerRules, + }) + + router := policyengine.NewDefaultChainRouterWithLocalOverrides(ms, ls) + + checker := NewChecker(router, headerProvider) + + prm := Prm{ + Method: method, + Container: cnr, + Object: obj, + Role: role, + SenderKey: senderKey, + } + + var headerObjSDK *objectSDK.Object + if test.header.headerObjSDK != nil { + headerObjSDK = newHeaderObjectSDK(t, cnr, obj, test.header.headerObjSDK) + if test.header.fromHeaderProvider { + require.NotNil(t, obj, "oid is required if a header is expected to be found in header provider") + headerProvider.addHeader(cnr, *obj, headerObjSDK) + } else if test.header.fromRequestResponseHeader { + prm.Header = headerObjSDK.ToV2().GetHeader() + } + } + + err := checker.CheckAPE(context.Background(), prm) + if test.expectAPEErr { + aErr := apeErr(method, chain.AccessDenied) + require.ErrorAs(t, err, &aErr) + } else { + require.NoError(t, err) + } + }) + } + }) + } +} diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go new file mode 100644 index 000000000..caf52645c --- /dev/null +++ b/pkg/services/object/ape/request.go @@ -0,0 +1,159 @@ +package ape + +import ( + "context" + "fmt" + "strconv" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + aperesource "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" +) + +type request struct { + operation string + resource *resource + properties map[string]string +} + +var _ aperesource.Request = (*request)(nil) + +type resource struct { + name string + properties map[string]string +} + +var _ aperesource.Resource = (*resource)(nil) + +func (r *resource) Name() string { + return r.name +} + +func (r *resource) Property(key string) string { + return r.properties[key] +} + +func (r *request) Operation() string { + return r.operation +} + +func (r *request) Property(key string) string { + return r.properties[key] +} + +func (r *request) Resource() aperesource.Resource { + return r.resource +} + +func nativeSchemaRole(role acl.Role) string { + switch role { + case acl.RoleOwner: + return nativeschema.PropertyValueContainerRoleOwner + case acl.RoleContainer: + return nativeschema.PropertyValueContainerRoleContainer + case acl.RoleInnerRing: + return nativeschema.PropertyValueContainerRoleIR + case acl.RoleOthers: + return nativeschema.PropertyValueContainerRoleOthers + default: + return "" + } +} + +func resourceName(cid cid.ID, oid *oid.ID, namespace string) string { + if namespace == "root" || namespace == "" { + if oid != nil { + return fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, cid.EncodeToString(), oid.EncodeToString()) + } + return fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) + } + if oid != nil { + return fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainerObject, namespace, cid.EncodeToString(), oid.EncodeToString()) + } + return fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainerObjects, namespace, cid.EncodeToString()) +} + +// objectProperties collects object properties from address parameters and a header if it is passed. +func objectProperties(cnr cid.ID, oid *oid.ID, header *objectV2.Header) map[string]string { + objectProps := map[string]string{ + nativeschema.PropertyKeyObjectContainerID: cnr.EncodeToString(), + } + + if oid != nil { + objectProps[nativeschema.PropertyKeyObjectID] = oid.String() + } + + if header == nil { + return objectProps + } + + objV2 := new(objectV2.Object) + objV2.SetHeader(header) + objSDK := objectSDK.NewFromV2(objV2) + + objectProps[nativeschema.PropertyKeyObjectVersion] = objSDK.Version().String() + objectProps[nativeschema.PropertyKeyObjectOwnerID] = objSDK.OwnerID().EncodeToString() + objectProps[nativeschema.PropertyKeyObjectCreationEpoch] = strconv.Itoa(int(objSDK.CreationEpoch())) + objectProps[nativeschema.PropertyKeyObjectPayloadLength] = strconv.Itoa(int(objSDK.PayloadSize())) + objectProps[nativeschema.PropertyKeyObjectType] = objSDK.Type().String() + + pcs, isSet := objSDK.PayloadChecksum() + if isSet { + objectProps[nativeschema.PropertyKeyObjectPayloadHash] = pcs.String() + } + hcs, isSet := objSDK.PayloadHomomorphicHash() + if isSet { + objectProps[nativeschema.PropertyKeyObjectHomomorphicHash] = hcs.String() + } + + for _, attr := range header.GetAttributes() { + objectProps[attr.GetKey()] = attr.GetValue() + } + + return objectProps +} + +// newAPERequest creates an APE request to be passed to a chain router. It collects resource properties from +// header provided by headerProvider. If it cannot be found in headerProvider, then properties are +// initialized from header given in prm (if it is set). Otherwise, just CID and OID are set to properties. +func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (*request, error) { + switch prm.Method { + case nativeschema.MethodGetObject, + nativeschema.MethodHeadObject, + nativeschema.MethodRangeObject, + nativeschema.MethodHashObject, + nativeschema.MethodDeleteObject: + if prm.Object == nil { + return nil, fmt.Errorf("method %s: %w", prm.Method, errMissingOID) + } + case nativeschema.MethodSearchObject, nativeschema.MethodPutObject: + default: + return nil, fmt.Errorf("unknown method: %s", prm.Method) + } + + var header *objectV2.Header + if prm.Header != nil { + header = prm.Header + } else if prm.Object != nil { + headerObjSDK, err := c.headerProvider.GetHeader(ctx, prm.Container, *prm.Object) + if err == nil { + header = headerObjSDK.ToV2().GetHeader() + } + } + + return &request{ + operation: prm.Method, + resource: &resource{ + name: resourceName(prm.Container, prm.Object, prm.Namespace), + properties: objectProperties(prm.Container, prm.Object, header), + }, + properties: map[string]string{ + nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, + nativeschema.PropertyKeyActorRole: prm.Role, + }, + }, nil +} diff --git a/pkg/services/object/ape/request_test.go b/pkg/services/object/ape/request_test.go new file mode 100644 index 000000000..55aa691b4 --- /dev/null +++ b/pkg/services/object/ape/request_test.go @@ -0,0 +1,324 @@ +package ape + +import ( + "context" + "fmt" + "testing" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/stretchr/testify/require" +) + +func TestObjectProperties(t *testing.T) { + for _, test := range []struct { + name string + container string + object *string + header *headerObjectSDKParams + }{ + { + name: "fully filled header", + container: containerID, + object: stringPtr(objectID), + header: &headerObjectSDKParams{ + majorVersion: 1, + minorVersion: 1, + owner: usertest.ID(), + epoch: 3, + payloadSize: 1000, + typ: objectSDK.TypeRegular, + payloadChecksum: checksumtest.Checksum(), + payloadHomomorphicHash: checksumtest.Checksum(), + attributes: []struct { + key string + val string + }{ + { + key: "attr1", + val: "val1", + }, + { + key: "attr2", + val: "val2", + }, + }, + }, + }, + { + name: "partially filled header", + container: containerID, + header: &headerObjectSDKParams{ + majorVersion: 1, + minorVersion: 1, + owner: usertest.ID(), + epoch: 3, + attributes: []struct { + key string + val string + }{ + { + key: "attr1", + val: "val1", + }, + }, + }, + }, + { + name: "only address paramaters set in header", + container: containerID, + object: stringPtr(objectID), + }, + { + name: "only container set in header", + container: containerID, + }, + } { + t.Run(test.name, func(t *testing.T) { + cnr := newContainerIDSDK(t, test.container) + obj := newObjectIDSDK(t, test.object) + header := newHeaderObjectSDK(t, cnr, obj, test.header) + + props := objectProperties(cnr, obj, header.ToV2().GetHeader()) + require.Equal(t, test.container, props[nativeschema.PropertyKeyObjectContainerID]) + + if obj != nil { + require.Equal(t, *test.object, props[nativeschema.PropertyKeyObjectID]) + } + + if test.header != nil { + require.Equal(t, + fmt.Sprintf("v%d.%d", test.header.majorVersion, test.header.minorVersion), + props[nativeschema.PropertyKeyObjectVersion], + ) + require.Equal(t, test.header.owner.EncodeToString(), props[nativeschema.PropertyKeyObjectOwnerID]) + require.Equal(t, fmt.Sprintf("%d", test.header.epoch), props[nativeschema.PropertyKeyObjectCreationEpoch]) + require.Equal(t, fmt.Sprintf("%d", test.header.payloadSize), props[nativeschema.PropertyKeyObjectPayloadLength]) + require.Equal(t, test.header.typ.String(), props[nativeschema.PropertyKeyObjectType]) + require.Equal(t, test.header.payloadChecksum.String(), props[nativeschema.PropertyKeyObjectPayloadHash]) + require.Equal(t, test.header.payloadHomomorphicHash.String(), props[nativeschema.PropertyKeyObjectHomomorphicHash]) + + for _, attr := range test.header.attributes { + require.Equal(t, attr.val, props[attr.key]) + } + } + }) + } +} + +func TestNewAPERequest(t *testing.T) { + tests := []struct { + name string + methods []string + namespace string + container string + object *string + header testHeader + expectErr error + }{ + { + name: "oid required requests", + methods: methodsRequiredOID, + namespace: namespace, + container: containerID, + object: stringPtr(objectID), + header: testHeader{ + headerObjSDK: &headerObjectSDKParams{ + majorVersion: 1, + minorVersion: 1, + owner: usertest.ID(), + epoch: 3, + payloadSize: 1000, + typ: objectSDK.TypeRegular, + payloadChecksum: checksumtest.Checksum(), + payloadHomomorphicHash: checksumtest.Checksum(), + }, + fromHeaderProvider: true, + }, + }, + { + name: "oid required requests but header cannot be found locally", + methods: methodsRequiredOID, + namespace: namespace, + container: containerID, + object: stringPtr(objectID), + header: testHeader{}, + }, + { + name: "oid required requests missed oid", + methods: methodsRequiredOID, + namespace: namespace, + container: containerID, + object: nil, + header: testHeader{}, + expectErr: errMissingOID, + }, + { + name: "response for oid required requests", + methods: methodsRequiredOID, + namespace: namespace, + container: containerID, + object: stringPtr(objectID), + header: testHeader{ + headerObjSDK: &headerObjectSDKParams{ + majorVersion: 1, + minorVersion: 1, + owner: usertest.ID(), + epoch: 3, + payloadSize: 1000, + typ: objectSDK.TypeRegular, + payloadChecksum: checksumtest.Checksum(), + payloadHomomorphicHash: checksumtest.Checksum(), + }, + fromRequestResponseHeader: true, + }, + }, + { + name: "oid not required methods request", + methods: methodsOptionalOID, + namespace: namespace, + container: containerID, + object: nil, + header: testHeader{ + headerObjSDK: &headerObjectSDKParams{ + majorVersion: 6, + minorVersion: 66, + owner: usertest.ID(), + epoch: 3, + typ: objectSDK.TypeLock, + }, + fromRequestResponseHeader: true, + }, + }, + { + name: "oid not required methods request but no header", + methods: methodsOptionalOID, + namespace: namespace, + container: containerID, + object: nil, + header: testHeader{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for _, method := range test.methods { + t.Run(method, func(t *testing.T) { + cnr := newContainerIDSDK(t, test.container) + obj := newObjectIDSDK(t, test.object) + + prm := Prm{ + Namespace: test.namespace, + Method: method, + Container: cnr, + Object: obj, + Role: role, + SenderKey: senderKey, + } + + headerSource := newHeaderProviderMock() + + var headerObjSDK *objectSDK.Object + if test.header.headerObjSDK != nil { + headerObjSDK = newHeaderObjectSDK(t, cnr, obj, test.header.headerObjSDK) + if test.header.fromHeaderProvider { + require.NotNil(t, obj, "oid is required if a header is expected to be found in header provider") + headerSource.addHeader(cnr, *obj, headerObjSDK) + } else if test.header.fromRequestResponseHeader { + prm.Header = headerObjSDK.ToV2().GetHeader() + } + } + + c := checkerImpl{ + headerProvider: headerSource, + } + + r, err := c.newAPERequest(context.TODO(), prm) + if test.expectErr != nil { + require.Error(t, err) + require.ErrorIs(t, err, test.expectErr) + return + } + + expectedRequest := request{ + operation: method, + resource: &resource{ + name: resourceName(cnr, obj, prm.Namespace), + properties: objectProperties(cnr, obj, func() *objectV2.Header { + if headerObjSDK != nil { + return headerObjSDK.ToV2().GetHeader() + } + return prm.Header + }()), + }, + properties: map[string]string{ + nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, + nativeschema.PropertyKeyActorRole: prm.Role, + }, + } + + require.Equal(t, expectedRequest, *r) + }) + } + }) + } +} + +func TestResourceName(t *testing.T) { + for _, test := range []struct { + name string + namespace string + container string + object *string + expected string + }{ + { + name: "non-root namespace, CID", + namespace: namespace, + container: containerID, + expected: fmt.Sprintf("native:object/%s/%s/*", namespace, containerID), + }, + { + name: "non-root namespace, CID, OID", + namespace: namespace, + container: containerID, + object: stringPtr(objectID), + expected: fmt.Sprintf("native:object/%s/%s/%s", namespace, containerID, objectID), + }, + { + name: "empty namespace, CID", + namespace: "", + container: containerID, + expected: fmt.Sprintf("native:object//%s/*", containerID), + }, + { + name: "empty namespace, CID, OID", + namespace: "", + container: containerID, + object: stringPtr(objectID), + expected: fmt.Sprintf("native:object//%s/%s", containerID, objectID), + }, + { + name: "root namespace, CID", + namespace: "root", + container: containerID, + expected: fmt.Sprintf("native:object//%s/*", containerID), + }, + { + name: "root namespace, CID, OID", + namespace: "root", + container: containerID, + object: stringPtr(objectID), + expected: fmt.Sprintf("native:object//%s/%s", containerID, objectID), + }, + } { + t.Run(test.name, func(t *testing.T) { + cnr := newContainerIDSDK(t, test.container) + obj := newObjectIDSDK(t, test.object) + require.Equal(t, test.expected, resourceName(cnr, obj, test.namespace)) + }) + } +} diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go new file mode 100644 index 000000000..df4dcc306 --- /dev/null +++ b/pkg/services/object/ape/service.go @@ -0,0 +1,404 @@ +package ape + +import ( + "context" + "encoding/hex" + "fmt" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" +) + +type Service struct { + log *logger.Logger + + apeChecker Checker + + next objectSvc.ServiceServer +} + +var _ objectSvc.ServiceServer = (*Service)(nil) + +type HeaderProvider interface { + GetHeader(ctx context.Context, cnr cid.ID, oid oid.ID) (*objectSDK.Object, error) +} + +type storageEngineHeaderProvider struct { + storageEngine *engine.StorageEngine +} + +func (p storageEngineHeaderProvider) GetHeader(ctx context.Context, cnr cid.ID, objID oid.ID) (*objectSDK.Object, error) { + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(objID) + return engine.Head(ctx, p.storageEngine, addr) +} + +func NewStorageEngineHeaderProvider(e *engine.StorageEngine) HeaderProvider { + return storageEngineHeaderProvider{ + storageEngine: e, + } +} + +func NewService(log *logger.Logger, apeChecker Checker, next objectSvc.ServiceServer) *Service { + return &Service{ + log: log, + apeChecker: apeChecker, + next: next, + } +} + +type getStreamBasicChecker struct { + objectSvc.GetObjectStream + + apeChecker Checker + + senderKey []byte + + role string +} + +func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { + if partInit, ok := resp.GetBody().GetObjectPart().(*objectV2.GetObjectPartInit); ok { + cnrID, objID, err := getAddressParamsSDK(partInit.GetHeader().GetContainerID(), partInit.GetObjectID()) + if err != nil { + return err + } + + prm := Prm{ + Container: cnrID, + Object: objID, + Header: partInit.GetHeader(), + Method: nativeschema.MethodGetContainer, + SenderKey: hex.EncodeToString(g.senderKey), + Role: g.role, + } + + if err := g.apeChecker.CheckAPE(g.Context(), prm); err != nil { + return err + } + } + return g.GetObjectStream.Send(resp) +} + +func requestContext(ctx context.Context) (*objectSvc.RequestContext, error) { + untyped := ctx.Value(objectSvc.RequestContextKey) + if untyped == nil { + return nil, fmt.Errorf("no key %s in context", objectSvc.RequestContextKey) + } + rc, ok := untyped.(*objectSvc.RequestContext) + if !ok { + return nil, fmt.Errorf("failed cast to RequestContext") + } + return rc, nil +} + +func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectStream) error { + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) + if err != nil { + return err + } + + reqCtx, err := requestContext(stream.Context()) + if err != nil { + return err + } + + err = c.apeChecker.CheckAPE(stream.Context(), Prm{ + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodGetObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + }) + if err != nil { + return err + } + + return c.next.Get(request, &getStreamBasicChecker{ + GetObjectStream: stream, + apeChecker: c.apeChecker, + }) +} + +type putStreamBasicChecker struct { + apeChecker Checker + + next objectSvc.PutObjectStream +} + +func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { + if partInit, ok := request.GetBody().GetObjectPart().(*objectV2.PutObjectPartInit); ok { + reqCtx, err := requestContext(ctx) + if err != nil { + return err + } + + cnrID, objID, err := getAddressParamsSDK(partInit.GetHeader().GetContainerID(), partInit.GetObjectID()) + if err != nil { + return err + } + + prm := Prm{ + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Header: partInit.GetHeader(), + Method: nativeschema.MethodPutObject, + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + Role: nativeSchemaRole(reqCtx.Role), + } + + if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { + return err + } + } + + return p.next.Send(ctx, request) +} + +func (p putStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PutResponse, error) { + return p.next.CloseAndRecv(ctx) +} + +func (c *Service) Put() (objectSvc.PutObjectStream, error) { + streamer, err := c.next.Put() + + return &putStreamBasicChecker{ + apeChecker: c.apeChecker, + next: streamer, + }, err +} + +func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) + if err != nil { + return nil, err + } + + reqCtx, err := requestContext(ctx) + if err != nil { + return nil, err + } + + err = c.apeChecker.CheckAPE(ctx, Prm{ + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodHeadObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + }) + if err != nil { + return nil, err + } + + resp, err := c.next.Head(ctx, request) + if err != nil { + return nil, err + } + + header := new(objectV2.Header) + switch headerPart := resp.GetBody().GetHeaderPart().(type) { + case *objectV2.ShortHeader: + cidV2 := new(refs.ContainerID) + cnrID.WriteToV2(cidV2) + header.SetContainerID(cidV2) + header.SetVersion(headerPart.GetVersion()) + header.SetCreationEpoch(headerPart.GetCreationEpoch()) + header.SetOwnerID(headerPart.GetOwnerID()) + header.SetObjectType(headerPart.GetObjectType()) + header.SetHomomorphicHash(header.GetHomomorphicHash()) + header.SetPayloadLength(headerPart.GetPayloadLength()) + header.SetPayloadHash(headerPart.GetPayloadHash()) + case *objectV2.HeaderWithSignature: + header = headerPart.GetHeader() + default: + return resp, nil + } + + err = c.apeChecker.CheckAPE(ctx, Prm{ + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Header: header, + Method: nativeschema.MethodHeadObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + }) + if err != nil { + return nil, err + } + return resp, nil +} + +func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.SearchStream) error { + var cnrID cid.ID + if cnrV2 := request.GetBody().GetContainerID(); cnrV2 != nil { + if err := cnrID.ReadFromV2(*cnrV2); err != nil { + return err + } + } + + reqCtx, err := requestContext(stream.Context()) + if err != nil { + return err + } + + err = c.apeChecker.CheckAPE(stream.Context(), Prm{ + Namespace: reqCtx.Namespace, + Container: cnrID, + Method: nativeschema.MethodSearchObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + }) + if err != nil { + return err + } + + return c.next.Search(request, stream) +} + +func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) (*objectV2.DeleteResponse, error) { + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) + if err != nil { + return nil, err + } + + reqCtx, err := requestContext(ctx) + if err != nil { + return nil, err + } + + err = c.apeChecker.CheckAPE(ctx, Prm{ + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodDeleteObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + }) + if err != nil { + return nil, err + } + + resp, err := c.next.Delete(ctx, request) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.GetObjectRangeStream) error { + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) + if err != nil { + return err + } + + reqCtx, err := requestContext(stream.Context()) + if err != nil { + return err + } + + err = c.apeChecker.CheckAPE(stream.Context(), Prm{ + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodRangeObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + }) + if err != nil { + return err + } + + return c.next.GetRange(request, stream) +} + +func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHashRequest) (*objectV2.GetRangeHashResponse, error) { + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) + if err != nil { + return nil, err + } + + reqCtx, err := requestContext(ctx) + if err != nil { + return nil, err + } + + prm := Prm{ + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodHashObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + } + + if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { + return nil, err + } + + resp, err := c.next.GetRangeHash(ctx, request) + if err != nil { + return nil, err + } + + if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { + return nil, err + } + return resp, nil +} + +func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetObject().GetHeader().GetContainerID(), request.GetBody().GetObject().GetObjectID()) + if err != nil { + return nil, err + } + + reqCtx, err := requestContext(ctx) + if err != nil { + return nil, err + } + + prm := Prm{ + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Header: request.GetBody().GetObject().GetHeader(), + Method: nativeschema.MethodPutObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + } + + if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { + return nil, err + } + + return c.next.PutSingle(ctx, request) +} + +func getAddressParamsSDK(cidV2 *refs.ContainerID, objV2 *refs.ObjectID) (cnrID cid.ID, objID *oid.ID, err error) { + if cidV2 != nil { + if err = cnrID.ReadFromV2(*cidV2); err != nil { + return + } + } + + if objV2 != nil { + objID = new(oid.ID) + if err = objID.ReadFromV2(*objV2); err != nil { + return + } + } + return +} diff --git a/pkg/services/object/ape/types.go b/pkg/services/object/ape/types.go new file mode 100644 index 000000000..46e55360d --- /dev/null +++ b/pkg/services/object/ape/types.go @@ -0,0 +1,9 @@ +package ape + +import "context" + +// Checker provides methods to check requests and responses +// with access policy engine. +type Checker interface { + CheckAPE(context.Context, Prm) error +} diff --git a/pkg/services/object/request_context.go b/pkg/services/object/request_context.go new file mode 100644 index 000000000..4b9aa04d1 --- /dev/null +++ b/pkg/services/object/request_context.go @@ -0,0 +1,16 @@ +package object + +import "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + +type RequestContextKeyT struct{} + +var RequestContextKey = RequestContextKeyT{} + +// RequestContext is a context passed between middleware handlers. +type RequestContext struct { + Namespace string + + SenderKey []byte + + Role acl.Role +} From 96b020626f8088dbf0796c6ca817b7a0240365fc Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 16 Jan 2024 23:52:37 +0300 Subject: [PATCH 0799/1943] [#915] ape: Fix method name in getStreamBasicChecker * Replace incorrect MethodGetContainer by MethodGetObject constant. Signed-off-by: Airat Arifullin --- pkg/services/object/ape/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index df4dcc306..d28ad4aeb 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -76,7 +76,7 @@ func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { Container: cnrID, Object: objID, Header: partInit.GetHeader(), - Method: nativeschema.MethodGetContainer, + Method: nativeschema.MethodGetObject, SenderKey: hex.EncodeToString(g.senderKey), Role: g.role, } From c1a80235db2a876fd32a8b51c58bb9d15dc7c535 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 15 Jan 2024 08:52:52 +0300 Subject: [PATCH 0800/1943] [#904] metabase: Log Inhume operation It will be very useful for troubleshooting. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/inhume.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 3b2f01673..cd83a6159 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -7,6 +7,7 @@ import ( "fmt" "time" + storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -184,6 +185,13 @@ func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { return db.inhumeTx(tx, currEpoch, prm, &res) }) success = err == nil + if success { + for _, addr := range prm.target { + storagelog.Write(db.log, + storagelog.AddressField(addr), + storagelog.OpField("metabase INHUME")) + } + } return res, metaerr.Wrap(err) } From 57171907e3485dc29d4b375ce5139758fdeef718 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 15 Jan 2024 09:04:25 +0300 Subject: [PATCH 0801/1943] [#904] metabase: Return if object was actuall inserted This requires to count metrics properly. Signed-off-by: Dmitrii Stepanov --- .../metabase/counter_test.go | 49 +++++++++++++++++++ pkg/local_object_storage/metabase/put.go | 20 +++++--- pkg/local_object_storage/shard/put.go | 11 +++-- 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 306bb0cc0..1797fc0aa 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -316,6 +316,55 @@ func TestCounters(t *testing.T) { }) } +func TestDoublePut(t *testing.T) { + t.Parallel() + db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() + obj := testutil.GenerateObject() + + exp := make(map[cid.ID]meta.ObjectCounters) + cnrID, _ := obj.ContainerID() + exp[cnrID] = meta.ObjectCounters{ + Logic: 1, + Phy: 1, + User: 1, + } + + var prm meta.PutPrm + prm.SetObject(obj) + pr, err := db.Put(context.Background(), prm) + require.NoError(t, err) + require.True(t, pr.Inserted) + + c, err := db.ObjectCounters() + require.NoError(t, err) + + require.Equal(t, uint64(1), c.Phy) + require.Equal(t, uint64(1), c.Logic) + require.Equal(t, uint64(1), c.User) + + cc, err := db.ContainerCounters(context.Background()) + require.NoError(t, err) + + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) + + pr, err = db.Put(context.Background(), prm) + require.NoError(t, err) + require.False(t, pr.Inserted) + + c, err = db.ObjectCounters() + require.NoError(t, err) + + require.Equal(t, uint64(1), c.Phy) + require.Equal(t, uint64(1), c.Logic) + require.Equal(t, uint64(1), c.User) + + cc, err = db.ContainerCounters(context.Background()) + require.NoError(t, err) + + require.Equal(t, meta.ContainerCounters{Counts: exp}, cc) +} + func TestCounters_Expired(t *testing.T) { // That test is about expired objects without // GCMark yet. Such objects should be treated as diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 2f5108953..1f0022389 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -36,7 +36,9 @@ type PutPrm struct { } // PutRes groups the resulting values of Put operation. -type PutRes struct{} +type PutRes struct { + Inserted bool +} // SetObject is a Put option to set object to save. func (p *PutPrm) SetObject(obj *objectSDK.Object) { @@ -85,7 +87,9 @@ func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { currEpoch := db.epochState.CurrentEpoch() err = db.boltDB.Batch(func(tx *bbolt.Tx) error { - return db.put(tx, prm.obj, prm.id, nil, currEpoch) + var e error + res, e = db.put(tx, prm.obj, prm.id, nil, currEpoch) + return e }) if err == nil { success = true @@ -102,10 +106,10 @@ func (db *DB) put(tx *bbolt.Tx, id []byte, si *objectSDK.SplitInfo, currEpoch uint64, -) error { +) (PutRes, error) { cnr, ok := obj.ContainerID() if !ok { - return errors.New("missing container in object") + return PutRes{}, errors.New("missing container in object") } isParent := si != nil @@ -116,14 +120,14 @@ func (db *DB) put(tx *bbolt.Tx, if errors.As(err, &splitInfoError) { exists = true // object exists, however it is virtual } else if err != nil { - return err // return any error besides SplitInfoError + return PutRes{}, err // return any error besides SplitInfoError } if exists { - return db.updateObj(tx, obj, id, si, isParent) + return PutRes{}, db.updateObj(tx, obj, id, si, isParent) } - return db.insertObject(tx, obj, id, si, isParent, cnr, currEpoch) + return PutRes{Inserted: true}, db.insertObject(tx, obj, id, si, isParent, cnr, currEpoch) } func (db *DB) updateObj(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *objectSDK.SplitInfo, isParent bool) error { @@ -153,7 +157,7 @@ func (db *DB) insertObject(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *o return err } - err = db.put(tx, par, id, parentSI, currEpoch) + _, err = db.put(tx, par, id, parentSI, currEpoch) if err != nil { return err } diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index 09a98260d..d7a9e7012 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -84,15 +84,18 @@ func (s *Shard) Put(ctx context.Context, prm PutPrm) (PutRes, error) { var pPrm meta.PutPrm pPrm.SetObject(prm.obj) pPrm.SetStorageID(res.StorageID) - if _, err := s.metaBase.Put(ctx, pPrm); err != nil { + res, err := s.metaBase.Put(ctx, pPrm) + if err != nil { // may we need to handle this case in a special way // since the object has been successfully written to BlobStor return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err) } - s.incObjectCounter(putPrm.Address.Container(), meta.IsUserObject(prm.obj)) - s.addToPayloadSize(int64(prm.obj.PayloadSize())) - s.addToContainerSize(putPrm.Address.Container().EncodeToString(), int64(prm.obj.PayloadSize())) + if res.Inserted { + s.incObjectCounter(putPrm.Address.Container(), meta.IsUserObject(prm.obj)) + s.addToPayloadSize(int64(prm.obj.PayloadSize())) + s.addToContainerSize(putPrm.Address.Container().EncodeToString(), int64(prm.obj.PayloadSize())) + } } return PutRes{}, nil From 63d3ed1ad86ed8fa527e1705762c64e80e4ec300 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 16 Jan 2024 09:09:59 +0300 Subject: [PATCH 0802/1943] [#904] tests: Close test engine after test Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/error_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 77dbc9b7e..7f48da08a 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -221,6 +221,7 @@ func TestBlobstorFailback(t *testing.T) { checkShardState(t, te.ng, te.shards[0].id, 0, mode.ReadWrite) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) + require.NoError(t, te.ng.Close(context.Background())) } func checkShardState(t *testing.T, e *StorageEngine, id *shard.ID, errCount uint32, mode mode.Mode) { From be330705509ff3e061d40719d8ce478e19c71155 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 12 Jan 2024 17:56:35 +0300 Subject: [PATCH 0803/1943] [#910] control: Take empty namespace for 'root' Signed-off-by: Airat Arifullin --- pkg/services/control/server/policy_engine.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 519b21035..9b42f3a98 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -17,7 +17,12 @@ func apeTarget(chainTarget *control.ChainTarget) (engine.Target, error) { case control.ChainTarget_CONTAINER: return engine.ContainerTarget(chainTarget.GetName()), nil case control.ChainTarget_NAMESPACE: - return engine.NamespaceTarget(chainTarget.GetName()), nil + namespace := chainTarget.GetName() + // If namespace is empty, we take it for root namespace. + if namespace == "" { + namespace = "root" + } + return engine.NamespaceTarget(namespace), nil default: } return engine.Target{}, status.Error(codes.InvalidArgument, From 0e3d144695ad7f6da169f5010aed71b1e64ebde2 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 12 Jan 2024 13:54:00 +0300 Subject: [PATCH 0804/1943] [#908] adm: Support frostfsid basic operations Signed-off-by: Denis Kirillov --- .../internal/modules/morph/frostfsid.go | 327 ++++++++++++++++++ .../internal/modules/morph/frostfsid_util.go | 27 ++ .../internal/modules/morph/root.go | 10 + go.mod | 2 +- go.sum | 4 +- 5 files changed, 367 insertions(+), 3 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/frostfsid.go diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go new file mode 100644 index 000000000..79ff0cdd0 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -0,0 +1,327 @@ +package morph + +import ( + "fmt" + "sort" + + frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" + "github.com/nspcc-dev/neo-go/pkg/vm/emit" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + namespaceFlag = "namespace" + subjectNameFlag = "subject-name" + subjectKeyFlag = "subject-key" + includeNamesFlag = "include-names" + groupNameFlag = "group-name" +) + +var ( + frostfsidCmd = &cobra.Command{ + Use: "frostfsid", + Short: "Section for frostfsid interactions commands", + } + + frostfsidCreateNamespaceCmd = &cobra.Command{ + Use: "create-namespace", + Short: "Create new namespace in frostfsid contract", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: frostfsidCreateNamespace, + } + + frostfsidListNamespacesCmd = &cobra.Command{ + Use: "list-namespaces", + Short: "List all namespaces in frostfsid", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: frostfsidListNamespaces, + } + + frostfsidCreateSubjectCmd = &cobra.Command{ + Use: "create-subject", + Short: "Create subject in frostfsid contract", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: frostfsidCreateSubject, + } + + frostfsidDeleteSubjectCmd = &cobra.Command{ + Use: "delete-subject", + Short: "Delete subject from frostfsid contract", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: frostfsidDeleteSubject, + } + + frostfsidListSubjectsCmd = &cobra.Command{ + Use: "list-subjects", + Short: "List subjects in namespace", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: frostfsidListSubjects, + } + + frostfsidCreateGroupCmd = &cobra.Command{ + Use: "create-group", + Short: "Create group in frostfsid contract", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: frostfsidCreateGroup, + } + + frostfsidDeleteGroupCmd = &cobra.Command{ + Use: "delete-group", + Short: "Delete group from frostfsid contract", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: frostfsidDeleteGroup, + } + + frostfsidListGroupsCmd = &cobra.Command{ + Use: "list-groups", + Short: "List groups in namespace", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: frostfsidListGroups, + } +) + +func initFrostfsIDCreateNamespaceCmd() { + frostfsidCmd.AddCommand(frostfsidCreateNamespaceCmd) + frostfsidCreateNamespaceCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidCreateNamespaceCmd.Flags().String(namespaceFlag, "", "Namespace name to create") +} + +func initFrostfsIDListNamespacesCmd() { + frostfsidCmd.AddCommand(frostfsidListNamespacesCmd) + frostfsidListNamespacesCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) +} + +func initFrostfsIDCreateSubjectCmd() { + frostfsidCmd.AddCommand(frostfsidCreateSubjectCmd) + frostfsidCreateSubjectCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidCreateSubjectCmd.Flags().String(namespaceFlag, "", "Namespace where create subject") + frostfsidCreateSubjectCmd.Flags().String(subjectNameFlag, "", "Subject name, must be unique in namespace") + frostfsidCreateSubjectCmd.Flags().String(subjectKeyFlag, "", "Subject hex-encoded public key") +} + +func initFrostfsIDDeleteSubjectCmd() { + frostfsidCmd.AddCommand(frostfsidDeleteSubjectCmd) + frostfsidDeleteSubjectCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidDeleteSubjectCmd.Flags().String(subjectAddressFlag, "", "Subject address") +} + +func initFrostfsIDListSubjectsCmd() { + frostfsidCmd.AddCommand(frostfsidListSubjectsCmd) + frostfsidListSubjectsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidListSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace to list subjects") + frostfsidListSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") +} + +func initFrostfsIDCreateGroupCmd() { + frostfsidCmd.AddCommand(frostfsidCreateGroupCmd) + frostfsidCreateGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidCreateGroupCmd.Flags().String(namespaceFlag, "", "Namespace where create group") + frostfsidCreateGroupCmd.Flags().String(groupNameFlag, "", "Group name, must be unique in namespace") +} + +func initFrostfsIDDeleteGroupCmd() { + frostfsidCmd.AddCommand(frostfsidDeleteGroupCmd) + frostfsidDeleteGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidDeleteGroupCmd.Flags().String(namespaceFlag, "", "Namespace to delete group") + frostfsidDeleteGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") +} + +func initFrostfsIDListGroupsCmd() { + frostfsidCmd.AddCommand(frostfsidListGroupsCmd) + frostfsidListGroupsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidListGroupsCmd.Flags().String(namespaceFlag, "", "Namespace to list groups") +} + +type ffsidMethodArgs struct { + name string + args []any +} + +func frostfsidCreateNamespace(cmd *cobra.Command, _ []string) { + ns, _ := cmd.Flags().GetString(namespaceFlag) + + err := sendFrostfsIDTx(cmd, ffsidMethodArgs{name: "createNamespace", args: []any{ns}}) + commonCmd.ExitOnErr(cmd, "processing error: %w", err) +} + +func frostfsidListNamespaces(cmd *cobra.Command, _ []string) { + ffsid, err := frostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract invoker: %w", err) + + namespaces, err := ffsid.ListNamespaces() + commonCmd.ExitOnErr(cmd, "list namespaces: %w", err) + + sort.Slice(namespaces, func(i, j int) bool { return namespaces[i].Name < namespaces[j].Name }) + + for _, namespace := range namespaces { + cmd.Printf("%q\n", namespace.Name) + } +} + +func frostfsidCreateSubject(cmd *cobra.Command, _ []string) { + ns, _ := cmd.Flags().GetString(namespaceFlag) + subjName, _ := cmd.Flags().GetString(subjectNameFlag) + subjKey := getFrostfsIDSubjectKey(cmd) + + args := []ffsidMethodArgs{ + {name: "createSubject", args: []any{ns, subjKey.Bytes()}}, + } + + if subjName != "" { + args = append(args, ffsidMethodArgs{name: "setSubjectName", args: []any{subjKey.GetScriptHash(), subjName}}) + } + + err := sendFrostfsIDTx(cmd, args...) + commonCmd.ExitOnErr(cmd, "processing error: %w", err) +} + +func frostfsidDeleteSubject(cmd *cobra.Command, _ []string) { + subjectAddress := getFrostfsIDSubjectAddress(cmd) + + err := sendFrostfsIDTx(cmd, ffsidMethodArgs{name: "deleteSubject", args: []any{subjectAddress}}) + commonCmd.ExitOnErr(cmd, "delete subject error: %w", err) +} + +func frostfsidListSubjects(cmd *cobra.Command, _ []string) { + ns, _ := cmd.Flags().GetString(namespaceFlag) + includeNames, _ := cmd.Flags().GetBool(includeNamesFlag) + + ffsid, err := frostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract invoker: %w", err) + + subAddresses, err := ffsid.ListNamespaceSubjects(ns) + commonCmd.ExitOnErr(cmd, "list subjects: %w", err) + + sort.Slice(subAddresses, func(i, j int) bool { return subAddresses[i].Less(subAddresses[j]) }) + + for _, addr := range subAddresses { + if !includeNames { + cmd.Println(address.Uint160ToString(addr)) + continue + } + + subj, err := ffsid.GetSubject(addr) + commonCmd.ExitOnErr(cmd, "get subject: %w", err) + + cmd.Printf("%s (%s)\n", address.Uint160ToString(addr), subj.Name) + } +} + +func frostfsidCreateGroup(cmd *cobra.Command, _ []string) { + ns, _ := cmd.Flags().GetString(namespaceFlag) + groupName, _ := cmd.Flags().GetString(groupNameFlag) + + ffsid, err := frostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + + groupID, err := ffsid.ParseGroupID(ffsid.Wait(ffsid.CreateGroup(ns, groupName))) + commonCmd.ExitOnErr(cmd, "create group: %w", err) + + cmd.Printf("group '%s' created with id: %d\n", groupName, groupID) +} + +func frostfsidDeleteGroup(cmd *cobra.Command, _ []string) { + ns, _ := cmd.Flags().GetString(namespaceFlag) + groupID, _ := cmd.Flags().GetInt64(groupIDFlag) + + ffsid, err := frostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + + _, err = ffsid.Wait(ffsid.DeleteGroup(ns, groupID)) + commonCmd.ExitOnErr(cmd, "delete group error: %w", err) +} + +func frostfsidListGroups(cmd *cobra.Command, _ []string) { + ns, _ := cmd.Flags().GetString(namespaceFlag) + + ffsid, err := frostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract invoker: %w", err) + + groups, err := ffsid.ListGroups(ns) + commonCmd.ExitOnErr(cmd, "list groups: %w", err) + + sort.Slice(groups, func(i, j int) bool { return groups[i].Name < groups[j].Name }) + + for _, group := range groups { + cmd.Printf("%q (%d)\n", group.Name, group.ID) + } +} + +func sendFrostfsIDTx(cmd *cobra.Command, methodArgs ...ffsidMethodArgs) error { + wCtx, err := newInitializeContext(cmd, viper.GetViper()) + if err != nil { + return fmt.Errorf("can't to initialize context: %w", err) + } + + r := management.NewReader(wCtx.ReadOnlyInvoker) + cs, err := r.GetContractByID(1) + if err != nil { + return fmt.Errorf("can't get NNS contract info: %w", err) + } + + ffsidHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(frostfsIDContract)) + if err != nil { + return fmt.Errorf("can't get proxy contract hash: %w", err) + } + + bw := io.NewBufBinWriter() + for _, method := range methodArgs { + emit.AppCall(bw.BinWriter, ffsidHash, method.name, callflag.All, method.args...) + } + + if err = wCtx.sendConsensusTx(bw.Bytes()); err != nil { + return err + } + + return wCtx.awaitTx() +} + +func frostfsIDClient(cmd *cobra.Command) (*frostfsidclient.Client, error) { + wCtx, err := newInitializeContext(cmd, viper.GetViper()) + if err != nil { + return nil, fmt.Errorf("can't to initialize context: %w", err) + } + + r := management.NewReader(wCtx.ReadOnlyInvoker) + cs, err := r.GetContractByID(1) + if err != nil { + return nil, fmt.Errorf("can't get NNS contract info: %w", err) + } + + ffsidHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(frostfsIDContract)) + if err != nil { + return nil, fmt.Errorf("can't get proxy contract hash: %w", err) + } + + return frostfsidclient.NewSimple(wCtx.CommitteeAct, ffsidHash), nil +} diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go index cfe3c1ee9..529d98dde 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go @@ -3,9 +3,11 @@ package morph import ( "fmt" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -31,3 +33,28 @@ func getFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) { } return util.Uint160{}, true, fmt.Errorf("frostfsid: admin is invalid: '%s'", admin) } + +func getFrostfsIDSubjectKey(cmd *cobra.Command) *keys.PublicKey { + subjKeyHex, _ := cmd.Flags().GetString(subjectKeyFlag) + subjKey, err := keys.NewPublicKeyFromString(subjKeyHex) + commonCmd.ExitOnErr(cmd, "invalid subject key: %w", err) + return subjKey +} + +//func decodeFrostfsIDNamespaces(resStack []stackitem.Item) ([]string, error) { +// var res []string +// +// if len(resStack) == 0 { +// return res, nil +// } +// +// if len(resStack) > 0 { +// nodes, err := decodeNodeList(resStack[0]) +// if err != nil { +// return nil, err +// } +// +// nm.SetNodes(nodes) +// } +// +//} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 4176a6470..6db262175 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -293,6 +293,16 @@ func init() { initProxyAddAccount() initProxyRemoveAccount() + + RootCmd.AddCommand(frostfsidCmd) + initFrostfsIDCreateNamespaceCmd() + initFrostfsIDListNamespacesCmd() + initFrostfsIDCreateSubjectCmd() + initFrostfsIDDeleteSubjectCmd() + initFrostfsIDListSubjectsCmd() + initFrostfsIDCreateGroupCmd() + initFrostfsIDDeleteGroupCmd() + initFrostfsIDListGroupsCmd() } func initProxyAddAccount() { diff --git a/go.mod b/go.mod index ddf587ad5..8b48cd788 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd - git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231129062201-a1b61d394958 + git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e git.frostfs.info/TrueCloudLab/hrw v1.2.1 diff --git a/go.sum b/go.sum index 9be0e55cd..11b208d32 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd h1:MxiIXu4vUATX9bNYFnM1dXAYkx5dnPOGCMzwNNXJEvM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231129062201-a1b61d394958 h1:X9yPizADIhD3K/gdKVCthlAnf9aQ3UJJGnZgIwwixRQ= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20231129062201-a1b61d394958/go.mod h1:rQWdsG18NaiFvkJpMguJev913KD/yleHaniRBkUyt0o= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa h1:uuN90s74idThHXi/jF/IlTdFimOIs+IBU44rdTE/Zdo= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= From 6ebd61298e1b744db925e908672c2a6781ba7bfe Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Fri, 12 Jan 2024 18:03:18 +0300 Subject: [PATCH 0805/1943] [#908] adm/frostfsid: Support subject/group management Signed-off-by: Denis Kirillov --- .../internal/modules/morph/frostfsid.go | 106 +++++++++++++++++- .../internal/modules/morph/frostfsid_util.go | 23 +--- .../internal/modules/morph/root.go | 3 + 3 files changed, 110 insertions(+), 22 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go index 79ff0cdd0..a40ba4d4c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -16,11 +16,13 @@ import ( ) const ( - namespaceFlag = "namespace" - subjectNameFlag = "subject-name" - subjectKeyFlag = "subject-key" - includeNamesFlag = "include-names" - groupNameFlag = "group-name" + namespaceFlag = "namespace" + subjectNameFlag = "subject-name" + subjectKeyFlag = "subject-key" + subjectAddressFlag = "subject-address" + includeNamesFlag = "include-names" + groupNameFlag = "group-name" + groupIDFlag = "group-id" ) var ( @@ -108,6 +110,36 @@ var ( }, Run: frostfsidListGroups, } + + frostfsidAddSubjectToGroupCmd = &cobra.Command{ + Use: "add-subject-to-group", + Short: "Add subject to group", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: frostfsidAddSubjectToGroup, + } + + frostfsidRemoveSubjectFromGroupCmd = &cobra.Command{ + Use: "remove-subject-from-group", + Short: "Remove subject from group", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: frostfsidRemoveSubjectFromGroup, + } + + frostfsidListGroupSubjectsCmd = &cobra.Command{ + Use: "list-group-subjects", + Short: "List subjects in group", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + }, + Run: frostfsidListGroupSubjects, + } ) func initFrostfsIDCreateNamespaceCmd() { @@ -162,6 +194,28 @@ func initFrostfsIDListGroupsCmd() { frostfsidListGroupsCmd.Flags().String(namespaceFlag, "", "Namespace to list groups") } +func initFrostfsIDAddSubjectToGroupCmd() { + frostfsidCmd.AddCommand(frostfsidAddSubjectToGroupCmd) + frostfsidAddSubjectToGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidAddSubjectToGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") + frostfsidAddSubjectToGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") +} + +func initFrostfsIDRemoveSubjectFromGroupCmd() { + frostfsidCmd.AddCommand(frostfsidRemoveSubjectFromGroupCmd) + frostfsidRemoveSubjectFromGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidRemoveSubjectFromGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") + frostfsidRemoveSubjectFromGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") +} + +func initFrostfsIDListGroupSubjectsCmd() { + frostfsidCmd.AddCommand(frostfsidListGroupSubjectsCmd) + frostfsidListGroupSubjectsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidListGroupSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace name") + frostfsidListGroupSubjectsCmd.Flags().Int64(groupIDFlag, 0, "Group id") + frostfsidListGroupSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") +} + type ffsidMethodArgs struct { name string args []any @@ -277,6 +331,48 @@ func frostfsidListGroups(cmd *cobra.Command, _ []string) { } } +func frostfsidAddSubjectToGroup(cmd *cobra.Command, _ []string) { + subjectAddress := getFrostfsIDSubjectAddress(cmd) + groupID, _ := cmd.Flags().GetInt64(groupIDFlag) + + err := sendFrostfsIDTx(cmd, ffsidMethodArgs{name: "addSubjectToGroup", args: []any{subjectAddress, groupID}}) + commonCmd.ExitOnErr(cmd, "add subject to namespace error: %w", err) +} + +func frostfsidRemoveSubjectFromGroup(cmd *cobra.Command, _ []string) { + subjectAddress := getFrostfsIDSubjectAddress(cmd) + groupID, _ := cmd.Flags().GetInt64(groupIDFlag) + + err := sendFrostfsIDTx(cmd, ffsidMethodArgs{name: "removeSubjectFromGroup", args: []any{subjectAddress, groupID}}) + commonCmd.ExitOnErr(cmd, "remove subject to namespace error: %w", err) +} + +func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { + ns, _ := cmd.Flags().GetString(namespaceFlag) + groupID, _ := cmd.Flags().GetInt64(groupIDFlag) + includeNames, _ := cmd.Flags().GetBool(includeNamesFlag) + + ffsid, err := frostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + + subjects, err := ffsid.ListGroupSubjects(ns, groupID) + commonCmd.ExitOnErr(cmd, "list group subjects: %w", err) + + sort.Slice(subjects, func(i, j int) bool { return subjects[i].Less(subjects[j]) }) + + for _, subjAddr := range subjects { + if !includeNames { + cmd.Println(address.Uint160ToString(subjAddr)) + continue + } + + subj, err := ffsid.GetSubject(subjAddr) + commonCmd.ExitOnErr(cmd, "get subject: %w", err) + + cmd.Printf("%s (%s)\n", address.Uint160ToString(subjAddr), subj.Name) + } +} + func sendFrostfsIDTx(cmd *cobra.Command, methodArgs ...ffsidMethodArgs) error { wCtx, err := newInitializeContext(cmd, viper.GetViper()) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go index 529d98dde..dc7e581eb 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go @@ -41,20 +41,9 @@ func getFrostfsIDSubjectKey(cmd *cobra.Command) *keys.PublicKey { return subjKey } -//func decodeFrostfsIDNamespaces(resStack []stackitem.Item) ([]string, error) { -// var res []string -// -// if len(resStack) == 0 { -// return res, nil -// } -// -// if len(resStack) > 0 { -// nodes, err := decodeNodeList(resStack[0]) -// if err != nil { -// return nil, err -// } -// -// nm.SetNodes(nodes) -// } -// -//} +func getFrostfsIDSubjectAddress(cmd *cobra.Command) util.Uint160 { + subjAddress, _ := cmd.Flags().GetString(subjectAddressFlag) + subjAddr, err := address.StringToUint160(subjAddress) + commonCmd.ExitOnErr(cmd, "invalid subject address: %w", err) + return subjAddr +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 6db262175..600fe21c8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -303,6 +303,9 @@ func init() { initFrostfsIDCreateGroupCmd() initFrostfsIDDeleteGroupCmd() initFrostfsIDListGroupsCmd() + initFrostfsIDAddSubjectToGroupCmd() + initFrostfsIDRemoveSubjectFromGroupCmd() + initFrostfsIDListGroupSubjectsCmd() } func initProxyAddAccount() { From 136acdba21608c36099709cdaf3cd7c73c5b6551 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 15 Jan 2024 11:48:44 +0300 Subject: [PATCH 0806/1943] [#908] adm/frostfsid: Use client for write operations Signed-off-by: Denis Kirillov --- .../internal/modules/morph/frostfsid.go | 154 +++++++++++------- 1 file changed, 93 insertions(+), 61 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go index a40ba4d4c..938fb1b67 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -1,15 +1,18 @@ package morph import ( + "errors" "fmt" "sort" frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -216,23 +219,23 @@ func initFrostfsIDListGroupSubjectsCmd() { frostfsidListGroupSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") } -type ffsidMethodArgs struct { - name string - args []any -} - func frostfsidCreateNamespace(cmd *cobra.Command, _ []string) { ns, _ := cmd.Flags().GetString(namespaceFlag) - err := sendFrostfsIDTx(cmd, ffsidMethodArgs{name: "createNamespace", args: []any{ns}}) - commonCmd.ExitOnErr(cmd, "processing error: %w", err) + ffsid, err := newFrostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + + ffsid.addCall(ffsid.roCli.CreateNamespaceCall(ns)) + + err = ffsid.sendWait() + commonCmd.ExitOnErr(cmd, "create namespace error: %w", err) } func frostfsidListNamespaces(cmd *cobra.Command, _ []string) { - ffsid, err := frostfsIDClient(cmd) + ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract invoker: %w", err) - namespaces, err := ffsid.ListNamespaces() + namespaces, err := ffsid.roCli.ListNamespaces() commonCmd.ExitOnErr(cmd, "list namespaces: %w", err) sort.Slice(namespaces, func(i, j int) bool { return namespaces[i].Name < namespaces[j].Name }) @@ -247,22 +250,27 @@ func frostfsidCreateSubject(cmd *cobra.Command, _ []string) { subjName, _ := cmd.Flags().GetString(subjectNameFlag) subjKey := getFrostfsIDSubjectKey(cmd) - args := []ffsidMethodArgs{ - {name: "createSubject", args: []any{ns, subjKey.Bytes()}}, - } + ffsid, err := newFrostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + ffsid.addCall(ffsid.roCli.CreateSubjectCall(ns, subjKey)) if subjName != "" { - args = append(args, ffsidMethodArgs{name: "setSubjectName", args: []any{subjKey.GetScriptHash(), subjName}}) + ffsid.addCall(ffsid.roCli.SetSubjectNameCall(subjKey.GetScriptHash(), subjName)) } - err := sendFrostfsIDTx(cmd, args...) - commonCmd.ExitOnErr(cmd, "processing error: %w", err) + err = ffsid.sendWait() + commonCmd.ExitOnErr(cmd, "create subject: %w", err) } func frostfsidDeleteSubject(cmd *cobra.Command, _ []string) { subjectAddress := getFrostfsIDSubjectAddress(cmd) - err := sendFrostfsIDTx(cmd, ffsidMethodArgs{name: "deleteSubject", args: []any{subjectAddress}}) + ffsid, err := newFrostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + + ffsid.addCall(ffsid.roCli.DeleteSubjectCall(subjectAddress)) + + err = ffsid.sendWait() commonCmd.ExitOnErr(cmd, "delete subject error: %w", err) } @@ -270,10 +278,10 @@ func frostfsidListSubjects(cmd *cobra.Command, _ []string) { ns, _ := cmd.Flags().GetString(namespaceFlag) includeNames, _ := cmd.Flags().GetBool(includeNamesFlag) - ffsid, err := frostfsIDClient(cmd) + ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract invoker: %w", err) - subAddresses, err := ffsid.ListNamespaceSubjects(ns) + subAddresses, err := ffsid.roCli.ListNamespaceSubjects(ns) commonCmd.ExitOnErr(cmd, "list subjects: %w", err) sort.Slice(subAddresses, func(i, j int) bool { return subAddresses[i].Less(subAddresses[j]) }) @@ -284,7 +292,7 @@ func frostfsidListSubjects(cmd *cobra.Command, _ []string) { continue } - subj, err := ffsid.GetSubject(addr) + subj, err := ffsid.roCli.GetSubject(addr) commonCmd.ExitOnErr(cmd, "get subject: %w", err) cmd.Printf("%s (%s)\n", address.Uint160ToString(addr), subj.Name) @@ -295,10 +303,12 @@ func frostfsidCreateGroup(cmd *cobra.Command, _ []string) { ns, _ := cmd.Flags().GetString(namespaceFlag) groupName, _ := cmd.Flags().GetString(groupNameFlag) - ffsid, err := frostfsIDClient(cmd) + ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract client: %w", err) - groupID, err := ffsid.ParseGroupID(ffsid.Wait(ffsid.CreateGroup(ns, groupName))) + ffsid.addCall(ffsid.roCli.CreateGroupCall(ns, groupName)) + + groupID, err := ffsid.roCli.ParseGroupID(ffsid.sendWaitRes()) commonCmd.ExitOnErr(cmd, "create group: %w", err) cmd.Printf("group '%s' created with id: %d\n", groupName, groupID) @@ -308,20 +318,22 @@ func frostfsidDeleteGroup(cmd *cobra.Command, _ []string) { ns, _ := cmd.Flags().GetString(namespaceFlag) groupID, _ := cmd.Flags().GetInt64(groupIDFlag) - ffsid, err := frostfsIDClient(cmd) + ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract client: %w", err) - _, err = ffsid.Wait(ffsid.DeleteGroup(ns, groupID)) + ffsid.addCall(ffsid.roCli.DeleteGroupCall(ns, groupID)) + + err = ffsid.sendWait() commonCmd.ExitOnErr(cmd, "delete group error: %w", err) } func frostfsidListGroups(cmd *cobra.Command, _ []string) { ns, _ := cmd.Flags().GetString(namespaceFlag) - ffsid, err := frostfsIDClient(cmd) + ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract invoker: %w", err) - groups, err := ffsid.ListGroups(ns) + groups, err := ffsid.roCli.ListGroups(ns) commonCmd.ExitOnErr(cmd, "list groups: %w", err) sort.Slice(groups, func(i, j int) bool { return groups[i].Name < groups[j].Name }) @@ -335,16 +347,26 @@ func frostfsidAddSubjectToGroup(cmd *cobra.Command, _ []string) { subjectAddress := getFrostfsIDSubjectAddress(cmd) groupID, _ := cmd.Flags().GetInt64(groupIDFlag) - err := sendFrostfsIDTx(cmd, ffsidMethodArgs{name: "addSubjectToGroup", args: []any{subjectAddress, groupID}}) - commonCmd.ExitOnErr(cmd, "add subject to namespace error: %w", err) + ffsid, err := newFrostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + + ffsid.addCall(ffsid.roCli.AddSubjectToGroupCall(subjectAddress, groupID)) + + err = ffsid.sendWait() + commonCmd.ExitOnErr(cmd, "add subject to group error: %w", err) } func frostfsidRemoveSubjectFromGroup(cmd *cobra.Command, _ []string) { subjectAddress := getFrostfsIDSubjectAddress(cmd) groupID, _ := cmd.Flags().GetInt64(groupIDFlag) - err := sendFrostfsIDTx(cmd, ffsidMethodArgs{name: "removeSubjectFromGroup", args: []any{subjectAddress, groupID}}) - commonCmd.ExitOnErr(cmd, "remove subject to namespace error: %w", err) + ffsid, err := newFrostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + + ffsid.addCall(ffsid.roCli.RemoveSubjectFromGroupCall(subjectAddress, groupID)) + + err = ffsid.sendWait() + commonCmd.ExitOnErr(cmd, "remove subject from group error: %w", err) } func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { @@ -352,10 +374,10 @@ func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { groupID, _ := cmd.Flags().GetInt64(groupIDFlag) includeNames, _ := cmd.Flags().GetBool(includeNamesFlag) - ffsid, err := frostfsIDClient(cmd) + ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract client: %w", err) - subjects, err := ffsid.ListGroupSubjects(ns, groupID) + subjects, err := ffsid.roCli.ListGroupSubjects(ns, groupID) commonCmd.ExitOnErr(cmd, "list group subjects: %w", err) sort.Slice(subjects, func(i, j int) bool { return subjects[i].Less(subjects[j]) }) @@ -366,43 +388,21 @@ func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { continue } - subj, err := ffsid.GetSubject(subjAddr) + subj, err := ffsid.roCli.GetSubject(subjAddr) commonCmd.ExitOnErr(cmd, "get subject: %w", err) cmd.Printf("%s (%s)\n", address.Uint160ToString(subjAddr), subj.Name) } } -func sendFrostfsIDTx(cmd *cobra.Command, methodArgs ...ffsidMethodArgs) error { - wCtx, err := newInitializeContext(cmd, viper.GetViper()) - if err != nil { - return fmt.Errorf("can't to initialize context: %w", err) - } - - r := management.NewReader(wCtx.ReadOnlyInvoker) - cs, err := r.GetContractByID(1) - if err != nil { - return fmt.Errorf("can't get NNS contract info: %w", err) - } - - ffsidHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(frostfsIDContract)) - if err != nil { - return fmt.Errorf("can't get proxy contract hash: %w", err) - } - - bw := io.NewBufBinWriter() - for _, method := range methodArgs { - emit.AppCall(bw.BinWriter, ffsidHash, method.name, callflag.All, method.args...) - } - - if err = wCtx.sendConsensusTx(bw.Bytes()); err != nil { - return err - } - - return wCtx.awaitTx() +type frostfsidClient struct { + bw *io.BufBinWriter + contractHash util.Uint160 + roCli *frostfsidclient.Client // client can be used only for waiting tx, parsing and forming method params + wCtx *initializeContext } -func frostfsIDClient(cmd *cobra.Command) (*frostfsidclient.Client, error) { +func newFrostfsIDClient(cmd *cobra.Command) (*frostfsidClient, error) { wCtx, err := newInitializeContext(cmd, viper.GetViper()) if err != nil { return nil, fmt.Errorf("can't to initialize context: %w", err) @@ -419,5 +419,37 @@ func frostfsIDClient(cmd *cobra.Command) (*frostfsidclient.Client, error) { return nil, fmt.Errorf("can't get proxy contract hash: %w", err) } - return frostfsidclient.NewSimple(wCtx.CommitteeAct, ffsidHash), nil + return &frostfsidClient{ + bw: io.NewBufBinWriter(), + contractHash: ffsidHash, + roCli: frostfsidclient.NewSimple(wCtx.CommitteeAct, ffsidHash), + wCtx: wCtx, + }, nil +} + +func (f *frostfsidClient) addCall(method string, args []any) { + emit.AppCall(f.bw.BinWriter, f.contractHash, method, callflag.All, args...) +} + +func (f *frostfsidClient) sendWait() error { + if err := f.wCtx.sendConsensusTx(f.bw.Bytes()); err != nil { + return err + } + f.bw.Reset() + + return f.wCtx.awaitTx() +} + +func (f *frostfsidClient) sendWaitRes() (*state.AppExecResult, error) { + if err := f.wCtx.sendConsensusTx(f.bw.Bytes()); err != nil { + return nil, err + } + f.bw.Reset() + + if len(f.wCtx.SentTxs) == 0 { + return nil, errors.New("no transactions to wait") + } + + f.wCtx.Command.Println("Waiting for transactions to persist...") + return f.roCli.Wait(f.wCtx.SentTxs[0].hash, f.wCtx.SentTxs[0].vub, nil) } From e42262a8632a6e636f94e8387a8f0349654380f9 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 17 Jan 2024 11:35:09 +0300 Subject: [PATCH 0807/1943] [#908] adm/frostfsid: Add parameter validations Signed-off-by: Denis Kirillov --- .../internal/modules/morph/frostfsid.go | 35 +++--- .../internal/modules/morph/frostfsid_util.go | 60 +++++++++ .../modules/morph/frostfsid_util_test.go | 119 ++++++++++++++++++ 3 files changed, 199 insertions(+), 15 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go index 938fb1b67..2a6043b7c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -28,6 +28,8 @@ const ( groupIDFlag = "group-id" ) +const rootNamespacePlaceholder = "" + var ( frostfsidCmd = &cobra.Command{ Use: "frostfsid", @@ -220,7 +222,7 @@ func initFrostfsIDListGroupSubjectsCmd() { } func frostfsidCreateNamespace(cmd *cobra.Command, _ []string) { - ns, _ := cmd.Flags().GetString(namespaceFlag) + ns := getFrostfsIDNamespace(cmd) ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract client: %w", err) @@ -241,13 +243,16 @@ func frostfsidListNamespaces(cmd *cobra.Command, _ []string) { sort.Slice(namespaces, func(i, j int) bool { return namespaces[i].Name < namespaces[j].Name }) for _, namespace := range namespaces { - cmd.Printf("%q\n", namespace.Name) + if namespace.Name == "" { + namespace.Name = rootNamespacePlaceholder + } + cmd.Printf("%s\n", namespace.Name) } } func frostfsidCreateSubject(cmd *cobra.Command, _ []string) { - ns, _ := cmd.Flags().GetString(namespaceFlag) - subjName, _ := cmd.Flags().GetString(subjectNameFlag) + ns := getFrostfsIDNamespace(cmd) + subjName := getFrostfsIDSubjectName(cmd) subjKey := getFrostfsIDSubjectKey(cmd) ffsid, err := newFrostfsIDClient(cmd) @@ -275,7 +280,7 @@ func frostfsidDeleteSubject(cmd *cobra.Command, _ []string) { } func frostfsidListSubjects(cmd *cobra.Command, _ []string) { - ns, _ := cmd.Flags().GetString(namespaceFlag) + ns := getFrostfsIDNamespace(cmd) includeNames, _ := cmd.Flags().GetBool(includeNamesFlag) ffsid, err := newFrostfsIDClient(cmd) @@ -300,8 +305,8 @@ func frostfsidListSubjects(cmd *cobra.Command, _ []string) { } func frostfsidCreateGroup(cmd *cobra.Command, _ []string) { - ns, _ := cmd.Flags().GetString(namespaceFlag) - groupName, _ := cmd.Flags().GetString(groupNameFlag) + ns := getFrostfsIDNamespace(cmd) + groupName := getFrostfsIDGroupName(cmd) ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract client: %w", err) @@ -315,8 +320,8 @@ func frostfsidCreateGroup(cmd *cobra.Command, _ []string) { } func frostfsidDeleteGroup(cmd *cobra.Command, _ []string) { - ns, _ := cmd.Flags().GetString(namespaceFlag) - groupID, _ := cmd.Flags().GetInt64(groupIDFlag) + ns := getFrostfsIDNamespace(cmd) + groupID := getFrostfsIDGroupID(cmd) ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract client: %w", err) @@ -328,7 +333,7 @@ func frostfsidDeleteGroup(cmd *cobra.Command, _ []string) { } func frostfsidListGroups(cmd *cobra.Command, _ []string) { - ns, _ := cmd.Flags().GetString(namespaceFlag) + ns := getFrostfsIDNamespace(cmd) ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract invoker: %w", err) @@ -339,13 +344,13 @@ func frostfsidListGroups(cmd *cobra.Command, _ []string) { sort.Slice(groups, func(i, j int) bool { return groups[i].Name < groups[j].Name }) for _, group := range groups { - cmd.Printf("%q (%d)\n", group.Name, group.ID) + cmd.Printf("%s (%d)\n", group.Name, group.ID) } } func frostfsidAddSubjectToGroup(cmd *cobra.Command, _ []string) { subjectAddress := getFrostfsIDSubjectAddress(cmd) - groupID, _ := cmd.Flags().GetInt64(groupIDFlag) + groupID := getFrostfsIDGroupID(cmd) ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract client: %w", err) @@ -358,7 +363,7 @@ func frostfsidAddSubjectToGroup(cmd *cobra.Command, _ []string) { func frostfsidRemoveSubjectFromGroup(cmd *cobra.Command, _ []string) { subjectAddress := getFrostfsIDSubjectAddress(cmd) - groupID, _ := cmd.Flags().GetInt64(groupIDFlag) + groupID := getFrostfsIDGroupID(cmd) ffsid, err := newFrostfsIDClient(cmd) commonCmd.ExitOnErr(cmd, "init contract client: %w", err) @@ -370,8 +375,8 @@ func frostfsidRemoveSubjectFromGroup(cmd *cobra.Command, _ []string) { } func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { - ns, _ := cmd.Flags().GetString(namespaceFlag) - groupID, _ := cmd.Flags().GetInt64(groupIDFlag) + ns := getFrostfsIDNamespace(cmd) + groupID := getFrostfsIDGroupID(cmd) includeNames, _ := cmd.Flags().GetBool(includeNamesFlag) ffsid, err := newFrostfsIDClient(cmd) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go index dc7e581eb..f88e4edfb 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go @@ -1,7 +1,9 @@ package morph import ( + "errors" "fmt" + "regexp" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -11,6 +13,14 @@ import ( "github.com/spf13/viper" ) +var ( + frostfsidSubjectNameRegexp = regexp.MustCompile(`^[\w+=,.@-]{1,64}$`) + frostfsidGroupNameRegexp = regexp.MustCompile(`^[\w+=,.@-]{1,128}$`) + + // frostfsidNamespaceNameRegexp similar to https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/commit/f2a82aa635aa57d9b05092d8cf15b170b53cc324/nns/nns_contract.go#L690 + frostfsidNamespaceNameRegexp = regexp.MustCompile(`(^$)|(^[a-z0-9]{1,2}$)|(^[a-z0-9][a-z0-9-]{1,48}[a-z0-9]$)`) +) + func getFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) { admin := v.GetString(frostfsIDAdminConfigKey) if admin == "" { @@ -47,3 +57,53 @@ func getFrostfsIDSubjectAddress(cmd *cobra.Command) util.Uint160 { commonCmd.ExitOnErr(cmd, "invalid subject address: %w", err) return subjAddr } + +func getFrostfsIDSubjectName(cmd *cobra.Command) string { + subjectName, _ := cmd.Flags().GetString(subjectNameFlag) + + if subjectName == "" { + return "" + } + + if !frostfsidSubjectNameRegexp.MatchString(subjectName) { + commonCmd.ExitOnErr(cmd, "invalid subject name: %w", + fmt.Errorf("name must match regexp: %s", frostfsidSubjectNameRegexp.String())) + } + + return subjectName +} + +func getFrostfsIDGroupName(cmd *cobra.Command) string { + groupName, _ := cmd.Flags().GetString(groupNameFlag) + + if !frostfsidGroupNameRegexp.MatchString(groupName) { + commonCmd.ExitOnErr(cmd, "invalid group name: %w", + fmt.Errorf("name must match regexp: %s", frostfsidGroupNameRegexp.String())) + } + + return groupName +} + +func getFrostfsIDGroupID(cmd *cobra.Command) int64 { + groupID, _ := cmd.Flags().GetInt64(groupIDFlag) + if groupID <= 0 { + commonCmd.ExitOnErr(cmd, "invalid group id: %w", + errors.New("group id must be positive integer")) + } + + return groupID +} + +func getFrostfsIDNamespace(cmd *cobra.Command) string { + ns, _ := cmd.Flags().GetString(namespaceFlag) + if ns == rootNamespacePlaceholder { + ns = "" + } + + if !frostfsidNamespaceNameRegexp.MatchString(ns) { + commonCmd.ExitOnErr(cmd, "invalid namespace: %w", + fmt.Errorf("name must match regexp: %s", frostfsidNamespaceNameRegexp.String())) + } + + return ns +} diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go index 192dc9f12..f73ebad08 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go @@ -51,3 +51,122 @@ func TestFrostfsIDConfig(t *testing.T) { require.False(t, found) }) } + +func TestNamespaceRegexp(t *testing.T) { + for _, tc := range []struct { + name string + namespace string + matched bool + }{ + { + name: "root empty ns", + namespace: "", + matched: true, + }, + { + name: "simple valid ns", + namespace: "my-namespace-123", + matched: true, + }, + { + name: "root placeholder", + namespace: "", + matched: false, + }, + { + name: "too long", + namespace: "abcdefghijklmnopkrstuvwxyzabcdefghijklmnopkrstuvwxyz", + matched: false, + }, + { + name: "start with hyphen", + namespace: "-ns", + matched: false, + }, + { + name: "end with hyphen", + namespace: "ns-", + matched: false, + }, + { + name: "with spaces", + namespace: "ns ns", + matched: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.matched, frostfsidNamespaceNameRegexp.MatchString(tc.namespace)) + }) + } +} + +func TestSubjectNameRegexp(t *testing.T) { + for _, tc := range []struct { + name string + subject string + matched bool + }{ + { + name: "empty", + subject: "", + matched: false, + }, + { + name: "invalid", + subject: "invalid{name}", + matched: false, + }, + { + name: "too long", + subject: "abcdefghijklmnopkrstuvwxyzabcdefghijklmnopkrstuvwxyzabcdefghijklmnopkrstuvwxyz", + matched: false, + }, + { + name: "valid", + subject: "valid_name.012345@6789", + matched: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.matched, frostfsidSubjectNameRegexp.MatchString(tc.subject)) + }) + } +} + +func TestSubjectGroupRegexp(t *testing.T) { + for _, tc := range []struct { + name string + subject string + matched bool + }{ + { + name: "empty", + subject: "", + matched: false, + }, + { + name: "invalid", + subject: "invalid{name}", + matched: false, + }, + { + name: "too long", + subject: "abcdefghijklmnopkrstuvwxyzabcdefghijklmnopkrstuvwxyzabcdefghijklmnopkrstuvwxyzabcdefghijklmnopkrstuvwxyzabcdefghijklmnopkrstuvwxyz", + matched: false, + }, + { + name: "long", + subject: "abcdefghijklmnopkrstuvwxyzabcdefghijklmnopkrstuvwxyzabcdefghijklmnopkrstuvwxyz", + matched: true, + }, + { + name: "valid", + subject: "valid_name.012345@6789", + matched: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.matched, frostfsidGroupNameRegexp.MatchString(tc.subject)) + }) + } +} From f5160b27fc94333cfe5cca3d04750f7980bf68d0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 18 Jan 2024 22:24:34 +0300 Subject: [PATCH 0808/1943] [#920] tests: Fix data races Signed-off-by: Dmitrii Stepanov --- .../engine/evacuate_test.go | 23 +++++++++++++++++++ .../engine/remove_copies_test.go | 9 ++++++++ pkg/local_object_storage/engine/tree_test.go | 5 ++++ 3 files changed, 37 insertions(+) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index ca86ef544..d874734d3 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -83,6 +83,9 @@ func TestEvacuateShard(t *testing.T) { const objPerShard = 3 e, ids, objects := newEngineEvacuate(t, 3, objPerShard) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() evacuateShardID := ids[2].String() @@ -161,6 +164,10 @@ func TestEvacuateNetwork(t *testing.T) { t.Run("single shard", func(t *testing.T) { t.Parallel() e, ids, objects := newEngineEvacuate(t, 1, 3) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() + evacuateShardID := ids[0].String() require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly)) @@ -181,6 +188,9 @@ func TestEvacuateNetwork(t *testing.T) { t.Run("multiple shards, evacuate one", func(t *testing.T) { t.Parallel() e, ids, objects := newEngineEvacuate(t, 2, 3) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) @@ -204,6 +214,10 @@ func TestEvacuateNetwork(t *testing.T) { t.Run("multiple shards, evacuate many", func(t *testing.T) { t.Parallel() e, ids, objects := newEngineEvacuate(t, 4, 5) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() + evacuateIDs := ids[0:3] var totalCount uint64 @@ -239,6 +253,9 @@ func TestEvacuateNetwork(t *testing.T) { func TestEvacuateCancellation(t *testing.T) { t.Parallel() e, ids, _ := newEngineEvacuate(t, 2, 3) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) @@ -264,6 +281,9 @@ func TestEvacuateCancellation(t *testing.T) { func TestEvacuateSingleProcess(t *testing.T) { e, ids, _ := newEngineEvacuate(t, 2, 3) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) @@ -303,6 +323,9 @@ func TestEvacuateSingleProcess(t *testing.T) { func TestEvacuateAsync(t *testing.T) { e, ids, _ := newEngineEvacuate(t, 2, 3) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) diff --git a/pkg/local_object_storage/engine/remove_copies_test.go b/pkg/local_object_storage/engine/remove_copies_test.go index 8131fcf0d..dfb2423ff 100644 --- a/pkg/local_object_storage/engine/remove_copies_test.go +++ b/pkg/local_object_storage/engine/remove_copies_test.go @@ -20,6 +20,9 @@ func TestRebalance(t *testing.T) { t.Parallel() te := newEngineWithErrorThreshold(t, "", 0) + defer func() { + require.NoError(t, te.ng.Close(context.Background())) + }() const ( objCount = 20 @@ -106,6 +109,9 @@ func TestRebalanceSingleThread(t *testing.T) { t.Parallel() te := newEngineWithErrorThreshold(t, "", 0) + defer func() { + require.NoError(t, te.ng.Close(context.Background())) + }() obj := testutil.GenerateObjectWithCID(cidtest.ID()) obj.SetPayload(make([]byte, errSmallSize)) @@ -153,6 +159,9 @@ type deleteEvent struct { func TestRebalanceExitByContext(t *testing.T) { te := newEngineWithErrorThreshold(t, "", 0) + defer func() { + require.NoError(t, te.ng.Close(context.Background())) + }() objects := make([]*objectSDK.Object, 4) for i := range objects { diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index f1650b5ae..2739058e9 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "github.com/stretchr/testify/require" ) func BenchmarkTreeVsSearch(b *testing.B) { @@ -25,6 +26,10 @@ func BenchmarkTreeVsSearch(b *testing.B) { func benchmarkTreeVsSearch(b *testing.B, objCount int) { te := newEngineWithErrorThreshold(b, "", 0) + defer func() { + require.NoError(b, te.ng.Close(context.Background())) + }() + cid := cidtest.ID() d := pilorama.CIDDescriptor{CID: cid, Position: 0, Size: 1} treeID := "someTree" From f526f49995575541626135e7d4542c0e014b63f2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 20 Dec 2023 17:18:28 +0300 Subject: [PATCH 0809/1943] [#874] engine: Check object existance concurrently Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/exists.go | 12 ++++ pkg/local_object_storage/blobovnicza/get.go | 12 ++++ .../blobstor/blobovniczatree/get.go | 5 ++ .../blobstor/blobovniczatree/iterate.go | 18 +++++ pkg/local_object_storage/blobstor/exists.go | 14 ++-- .../blobstor/fstree/fstree.go | 6 ++ .../engine/engine_test.go | 6 ++ pkg/local_object_storage/engine/exists.go | 72 +++++++++++-------- pkg/local_object_storage/engine/put.go | 2 - pkg/local_object_storage/metabase/exists.go | 6 ++ pkg/local_object_storage/shard/exists.go | 6 ++ 11 files changed, 124 insertions(+), 35 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/exists.go b/pkg/local_object_storage/blobovnicza/exists.go index f7bc84d4a..b5d723ee6 100644 --- a/pkg/local_object_storage/blobovnicza/exists.go +++ b/pkg/local_object_storage/blobovnicza/exists.go @@ -21,10 +21,22 @@ func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error )) defer span.End() + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + addrKey := addressKey(addr) err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if isNonDataBucket(bucketName) { return nil } diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index 600323f55..36cf69d54 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -51,6 +51,12 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { )) defer span.End() + select { + case <-ctx.Done(): + return GetRes{}, ctx.Err() + default: + } + var ( data []byte addrKey = addressKey(prm.addr) @@ -58,6 +64,12 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { if err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if isNonDataBucket(bucketName) { return nil } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 08cacda8a..5f1856673 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -94,6 +94,11 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G // // returns error if object could not be read from any blobovnicza of the same level. func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.GetPrm, blzPath string) (common.GetRes, error) { + select { + case <-ctx.Done(): + return common.GetRes{}, ctx.Err() + default: + } // open blobovnicza (cached inside) shBlz := b.getBlobovnicza(blzPath) blz, err := shBlz.Open() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 92014fd55..942d73a12 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -230,6 +230,12 @@ func (b *Blobovniczas) iterateSortedDBPaths(ctx context.Context, addr oid.Addres } func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path string, addr oid.Address, f func(string) (bool, error)) (bool, error) { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + sysPath := filepath.Join(b.rootPath, path) entries, err := os.ReadDir(sysPath) if os.IsNotExist(err) && b.readOnly && path == "" { // non initialized tree in read only mode @@ -252,6 +258,12 @@ func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path st if len(dbIdxs) > 0 { for _, dbIdx := range dbIdxs { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + dbPath := filepath.Join(path, u64ToHexStringExt(dbIdx)) stop, err := f(dbPath) if err != nil { @@ -266,6 +278,12 @@ func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path st if len(dirIdxs) > 0 { hrw.SortSliceByValue(dirIdxs, addressHash(&addr, path)) for _, dirIdx := range dirIdxs { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + dirPath := filepath.Join(path, u64ToHexString(dirIdx)) stop, err := b.iterateSordedDBPathsInternal(ctx, dirPath, addr, f) if err != nil { diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 43feec7c9..21b4016d2 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -3,6 +3,7 @@ package blobstor import ( "context" "encoding/hex" + "errors" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -57,27 +58,30 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi // error | found | log the error, return true, nil // error | not found | return the error // error | error | log the first error, return the second - var errors []error + var storageErrors []error for i := range b.storage { res, err := b.storage[i].Storage.Exists(ctx, prm) if err == nil && res.Exists { exists = true return res, nil } else if err != nil { - errors = append(errors, err) + if errors.Is(err, context.Canceled) { + return common.ExistsRes{}, err + } + storageErrors = append(storageErrors, err) } } - if len(errors) == 0 { + if len(storageErrors) == 0 { return common.ExistsRes{}, nil } - for _, err := range errors[:len(errors)-1] { + for _, err := range storageErrors[:len(storageErrors)-1] { b.log.Warn(logs.BlobstorErrorOccurredDuringObjectExistenceChecking, zap.Stringer("address", prm.Address), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } - return common.ExistsRes{}, errors[len(errors)-1] + return common.ExistsRes{}, storageErrors[len(storageErrors)-1] } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 948872fd2..af241becf 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -302,6 +302,12 @@ func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exist )) defer span.End() + select { + case <-ctx.Done(): + return common.ExistsRes{}, ctx.Err() + default: + } + p := t.treePath(prm.Address) _, err := os.Stat(p) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index a62734c0a..b20f45be5 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -39,6 +39,12 @@ func BenchmarkExists(b *testing.B) { b.Run("8 shards", func(b *testing.B) { benchmarkExists(b, 8) }) + b.Run("12 shards", func(b *testing.B) { + benchmarkExists(b, 12) + }) + b.Run("16 shards", func(b *testing.B) { + benchmarkExists(b, 12) + }) } func benchmarkExists(b *testing.B, shardNum int) { diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index ef6292768..ee4dad341 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -3,54 +3,70 @@ package engine import ( "context" "errors" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "golang.org/x/sync/errgroup" ) func (e *StorageEngine) exists(ctx context.Context, addr oid.Address) (bool, error) { var shPrm shard.ExistsPrm shPrm.SetAddress(addr) - alreadyRemoved := false - exists := false + + var exists atomic.Bool + eg, egCtx := errgroup.WithContext(ctx) + egCtx, cancel := context.WithCancel(egCtx) + defer cancel() e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { - res, err := sh.Exists(ctx, shPrm) - if err != nil { - if client.IsErrObjectAlreadyRemoved(err) { - alreadyRemoved = true - - return true - } - - var siErr *objectSDK.SplitInfoError - if errors.As(err, &siErr) { - return true - } - - if shard.IsErrObjectExpired(err) { - return true - } - - if !client.IsErrObjectNotFound(err) { - e.reportShardError(sh, "could not check existence of object in shard", err) - } - return false + select { + case <-egCtx.Done(): + return true + default: } - if !exists { - exists = res.Exists() - } + eg.Go(func() error { + res, err := sh.Exists(egCtx, shPrm) + if err != nil { + if errors.Is(err, context.Canceled) { + return err + } + if client.IsErrObjectAlreadyRemoved(err) { + return err + } + + var siErr *objectSDK.SplitInfoError + if errors.As(err, &siErr) { + return err + } + + if shard.IsErrObjectExpired(err) { + return err + } + + if !client.IsErrObjectNotFound(err) { + e.reportShardError(sh, "could not check existence of object in shard", err) + } + return nil + } + if res.Exists() { + exists.Store(true) + cancel() + } + return nil + }) return false }) - if alreadyRemoved { + err := eg.Wait() + if client.IsErrObjectAlreadyRemoved(err) { return false, new(apistatus.ObjectAlreadyRemoved) } - return exists, nil + return exists.Load(), nil } diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 79ee3a997..7ce915ad8 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -63,8 +63,6 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { addr := object.AddressOf(prm.obj) - // In #1146 this check was parallelized, however, it became - // much slower on fast machines for 4 shards. _, err := e.exists(ctx, addr) if err != nil { return err diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index aa9aba106..bca18e21f 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -67,6 +67,12 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err return res, ErrDegradedMode } + select { + case <-ctx.Done(): + return res, ctx.Err() + default: + } + currEpoch := db.epochState.CurrentEpoch() err = db.boltDB.View(func(tx *bbolt.Tx) error { diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 2cdb8dfa8..7296426a6 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -47,6 +47,12 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { )) defer span.End() + select { + case <-ctx.Done(): + return ExistsRes{}, ctx.Err() + default: + } + var exists bool var err error From f2f3294fc3e827ba8faecdf6ab121ba5cbe85231 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 18 Jan 2024 14:35:52 +0300 Subject: [PATCH 0810/1943] [#919] ape: Improve error messages in ape service * Wrap all APE middleware errors in apeErr that makes errors more explicit with status AccessDenied. * Use denyingRuleErr for denying status from chain router. Signed-off-by: Airat Arifullin --- pkg/services/object/ape/checker.go | 11 +-------- pkg/services/object/ape/checker_test.go | 6 ++--- pkg/services/object/ape/errors.go | 13 +++++++++++ pkg/services/object/ape/service.go | 30 ++++++++++++------------- 4 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 pkg/services/object/ape/errors.go diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index fdaa9df85..ccd23bf06 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -5,7 +5,6 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -70,13 +69,5 @@ func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { return nil } - return apeErr(prm.Method, status) -} - -const accessDeniedAPEReasonFmt = "access to operation %s is denied by access policy engine: %s" - -func apeErr(op string, status apechain.Status) error { - errAccessDenied := &apistatus.ObjectAccessDenied{} - errAccessDenied.WriteReason(fmt.Sprintf(accessDeniedAPEReasonFmt, op, status.String())) - return errAccessDenied + return fmt.Errorf("found denying rule for %s: %s", prm.Method, status) } diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index 9d548480b..08d097907 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -308,7 +308,8 @@ func TestAPECheck(t *testing.T) { ms := inmemory.NewInmemoryMorphRuleChainStorage() ls.AddOverride(chain.Ingress, policyengine.ContainerTarget(test.container), &chain.Chain{ - Rules: test.containerRules, + Rules: test.containerRules, + MatchType: chain.MatchTypeFirstMatch, }) router := policyengine.NewDefaultChainRouterWithLocalOverrides(ms, ls) @@ -336,8 +337,7 @@ func TestAPECheck(t *testing.T) { err := checker.CheckAPE(context.Background(), prm) if test.expectAPEErr { - aErr := apeErr(method, chain.AccessDenied) - require.ErrorAs(t, err, &aErr) + require.Error(t, err) } else { require.NoError(t, err) } diff --git a/pkg/services/object/ape/errors.go b/pkg/services/object/ape/errors.go new file mode 100644 index 000000000..7c2a82dfd --- /dev/null +++ b/pkg/services/object/ape/errors.go @@ -0,0 +1,13 @@ +package ape + +import ( + "fmt" + + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" +) + +func toStatusErr(err error) error { + errAccessDenied := &apistatus.ObjectAccessDenied{} + errAccessDenied.WriteReason(fmt.Sprintf("ape denied request: %s", err.Error())) + return errAccessDenied +} diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index d28ad4aeb..781f9df4b 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -69,7 +69,7 @@ func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { if partInit, ok := resp.GetBody().GetObjectPart().(*objectV2.GetObjectPartInit); ok { cnrID, objID, err := getAddressParamsSDK(partInit.GetHeader().GetContainerID(), partInit.GetObjectID()) if err != nil { - return err + return toStatusErr(err) } prm := Prm{ @@ -82,7 +82,7 @@ func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { } if err := g.apeChecker.CheckAPE(g.Context(), prm); err != nil { - return err + return toStatusErr(err) } } return g.GetObjectStream.Send(resp) @@ -103,12 +103,12 @@ func requestContext(ctx context.Context) (*objectSvc.RequestContext, error) { func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectStream) error { cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { - return err + return toStatusErr(err) } reqCtx, err := requestContext(stream.Context()) if err != nil { - return err + return toStatusErr(err) } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ @@ -120,7 +120,7 @@ func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectSt SenderKey: hex.EncodeToString(reqCtx.SenderKey), }) if err != nil { - return err + return toStatusErr(err) } return c.next.Get(request, &getStreamBasicChecker{ @@ -139,12 +139,12 @@ func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutR if partInit, ok := request.GetBody().GetObjectPart().(*objectV2.PutObjectPartInit); ok { reqCtx, err := requestContext(ctx) if err != nil { - return err + return toStatusErr(err) } cnrID, objID, err := getAddressParamsSDK(partInit.GetHeader().GetContainerID(), partInit.GetObjectID()) if err != nil { - return err + return toStatusErr(err) } prm := Prm{ @@ -158,7 +158,7 @@ func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutR } if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { - return err + return toStatusErr(err) } } @@ -244,13 +244,13 @@ func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.Searc var cnrID cid.ID if cnrV2 := request.GetBody().GetContainerID(); cnrV2 != nil { if err := cnrID.ReadFromV2(*cnrV2); err != nil { - return err + return toStatusErr(err) } } reqCtx, err := requestContext(stream.Context()) if err != nil { - return err + return toStatusErr(err) } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ @@ -261,7 +261,7 @@ func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.Searc SenderKey: hex.EncodeToString(reqCtx.SenderKey), }) if err != nil { - return err + return toStatusErr(err) } return c.next.Search(request, stream) @@ -301,12 +301,12 @@ func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) ( func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.GetObjectRangeStream) error { cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { - return err + return toStatusErr(err) } reqCtx, err := requestContext(stream.Context()) if err != nil { - return err + return toStatusErr(err) } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ @@ -318,7 +318,7 @@ func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.G SenderKey: hex.EncodeToString(reqCtx.SenderKey), }) if err != nil { - return err + return toStatusErr(err) } return c.next.GetRange(request, stream) @@ -381,7 +381,7 @@ func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequ } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { - return nil, err + return nil, toStatusErr(err) } return c.next.PutSingle(ctx, request) From 931a5e9aaf5e69afc1e71f256879a031d9c32a26 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 18 Jan 2024 13:39:21 +0300 Subject: [PATCH 0811/1943] [#918] engine: Move shard to degraded mode if metabase open failed Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + .../engine/control_test.go | 49 +++++++++---------- pkg/local_object_storage/engine/shards.go | 4 +- pkg/local_object_storage/shard/id.go | 46 ++++++++++------- 4 files changed, 53 insertions(+), 47 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 606b5cb9a..34ca0ae93 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -571,4 +571,5 @@ const ( GetSvcV2FailedToParseNodeExternalAddresses = "failed to parse node external addresses" GetSvcV2FailedToGetRangeHashFromNode = "failed to get range hash from node" GetSvcV2FailedToGetRangeHashFromAllOfContainerNodes = "failed to get range hash from all of container nodes" + FailedToUpdateShardID = "failed to update shard id" ) diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 0f36f3203..6bf6beac6 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -118,7 +118,7 @@ func TestInitializationFailure(t *testing.T) { }) } -func testEngineFailInitAndReload(t *testing.T, errOnAdd bool, opts []shard.Option, beforeReload func()) { +func testEngineFailInitAndReload(t *testing.T, degradedMode bool, opts []shard.Option, beforeReload func()) { var configID string e := New() @@ -126,33 +126,30 @@ func testEngineFailInitAndReload(t *testing.T, errOnAdd bool, opts []shard.Optio require.NoError(t, e.Close(context.Background())) }() _, err := e.AddShard(context.Background(), opts...) - if errOnAdd { - require.Error(t, err) - // This branch is only taken when we cannot update shard ID in the metabase. - // The id cannot be encountered during normal operation, but it is ok for tests: - // it is only compared for equality with other ids and we have 0 shards here. - configID = "id" - } else { - require.NoError(t, err) - - e.mtx.RLock() - var id string - for id = range e.shards { - break - } - configID = calculateShardID(e.shards[id].Shard.DumpInfo()) - e.mtx.RUnlock() - - err = e.Open(context.Background()) - if err == nil { - require.Error(t, e.Init(context.Background())) - } - } + require.NoError(t, err) e.mtx.RLock() - shardCount := len(e.shards) + var id string + for id = range e.shards { + break + } + configID = calculateShardID(e.shards[id].Shard.DumpInfo()) e.mtx.RUnlock() - require.Equal(t, 0, shardCount) + + err = e.Open(context.Background()) + require.NoError(t, err) + if degradedMode { + require.NoError(t, e.Init(context.Background())) + require.Equal(t, mode.DegradedReadOnly, e.DumpInfo().Shards[0].Mode) + return + } else { + require.Error(t, e.Init(context.Background())) + + e.mtx.RLock() + shardCount := len(e.shards) + e.mtx.RUnlock() + require.Equal(t, 0, shardCount) + } beforeReload() @@ -161,7 +158,7 @@ func testEngineFailInitAndReload(t *testing.T, errOnAdd bool, opts []shard.Optio })) e.mtx.RLock() - shardCount = len(e.shards) + shardCount := len(e.shards) e.mtx.RUnlock() require.Equal(t, 1, shardCount) } diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 0455471e2..37a857c9d 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -124,10 +124,10 @@ func (e *StorageEngine) createShard(ctx context.Context, opts []shard.Option) (* )...) if err := sh.UpdateID(ctx); err != nil { - return nil, fmt.Errorf("could not update shard ID: %w", err) + e.log.Warn(logs.FailedToUpdateShardID, zap.Stringer("shard_id", sh.ID()), zap.String("metabase_path", sh.DumpInfo().MetaBaseInfo.Path), zap.Error(err)) } - return sh, err + return sh, nil } func (e *StorageEngine) appendMetrics(id *shard.ID, opts []shard.Option) []shard.Option { diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index b0d95e54c..e3c209907 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -2,6 +2,7 @@ package shard import ( "context" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/mr-tron/base58" @@ -30,27 +31,32 @@ func (s *Shard) ID() *ID { // UpdateID reads shard ID saved in the metabase and updates it if it is missing. func (s *Shard) UpdateID(ctx context.Context) (err error) { + var metabaseOpened bool + var idFromMetabase []byte if err = s.metaBase.Open(ctx, false); err != nil { - return err + err = fmt.Errorf("failed to open metabase: %w", err) + } else { + metabaseOpened = true } - defer func() { - cErr := s.metaBase.Close() - if err == nil { - err = cErr - } - }() - id, err := s.metaBase.ReadShardID() - if err != nil { - return err - } - if len(id) != 0 { - s.info.ID = NewIDFromBytes(id) - - if s.cfg.metricsWriter != nil { - s.cfg.metricsWriter.SetShardID(s.info.ID.String()) + if metabaseOpened { + defer func() { + cErr := s.metaBase.Close() + if cErr != nil { + err = fmt.Errorf("failed to close metabase: %w", cErr) + } + }() + if idFromMetabase, err = s.metaBase.ReadShardID(); err != nil { + err = fmt.Errorf("failed to read shard id from metabase: %w", err) } } + if len(idFromMetabase) != 0 { + s.info.ID = NewIDFromBytes(idFromMetabase) + } + + if s.cfg.metricsWriter != nil { + s.cfg.metricsWriter.SetShardID(s.info.ID.String()) + } s.log = &logger.Logger{Logger: s.log.With(zap.Stringer("shard_id", s.info.ID))} s.metaBase.SetLogger(s.log) s.blobStor.SetLogger(s.log) @@ -63,8 +69,10 @@ func (s *Shard) UpdateID(ctx context.Context) (err error) { s.pilorama.SetParentID(s.info.ID.String()) } - if len(id) != 0 { - return nil + if len(idFromMetabase) == 0 && metabaseOpened { + if err = s.metaBase.WriteShardID(*s.info.ID); err != nil { + err = fmt.Errorf("failed to write shard id to metabase: %w", err) + } } - return s.metaBase.WriteShardID(*s.info.ID) + return } From 5ed330e43649ba3be8d894b7a2990214a7aed05b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 24 Jan 2024 18:41:02 +0300 Subject: [PATCH 0812/1943] [#927] metabase: Delete GC marks `key` is changed inside `db.get`, so encode address again after get. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/delete.go | 1 + .../metabase/delete_test.go | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 8ca0dea61..484dbe41b 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -264,6 +264,7 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter return deleteSingleResult{}, err } + addrKey = addressKey(addr, key) // remove record from the garbage bucket if garbageBKT != nil { err := garbageBKT.Delete(addrKey) diff --git a/pkg/local_object_storage/metabase/delete_test.go b/pkg/local_object_storage/metabase/delete_test.go index 2ff6ba89f..5a52868d3 100644 --- a/pkg/local_object_storage/metabase/delete_test.go +++ b/pkg/local_object_storage/metabase/delete_test.go @@ -140,6 +140,48 @@ func TestExpiredObject(t *testing.T) { }) } +func TestDelete(t *testing.T) { + db := newDB(t, meta.WithEpochState(epochState{currEpoch})) + defer func() { require.NoError(t, db.Close()) }() + + cnr := cidtest.ID() + for i := 0; i < 10; i++ { + obj := testutil.GenerateObjectWithCID(cnr) + + var prm meta.PutPrm + prm.SetObject(obj) + prm.SetStorageID([]byte("0/0")) + _, err := db.Put(context.Background(), prm) + require.NoError(t, err) + + var inhumePrm meta.InhumePrm + inhumePrm.SetAddresses(object.AddressOf(obj)) + _, err = db.Inhume(context.Background(), inhumePrm) + require.NoError(t, err) + } + + var addrs []oid.Address + var iprm meta.GarbageIterationPrm + iprm.SetHandler(func(o meta.GarbageObject) error { + addrs = append(addrs, o.Address()) + return nil + }) + require.NoError(t, db.IterateOverGarbage(context.Background(), iprm)) + require.Equal(t, 10, len(addrs)) + var deletePrm meta.DeletePrm + deletePrm.SetAddresses(addrs...) + _, err := db.Delete(context.Background(), deletePrm) + require.NoError(t, err) + + addrs = nil + iprm.SetHandler(func(o meta.GarbageObject) error { + addrs = append(addrs, o.Address()) + return nil + }) + require.NoError(t, db.IterateOverGarbage(context.Background(), iprm)) + require.Equal(t, 0, len(addrs)) +} + func metaDelete(db *meta.DB, addrs ...oid.Address) error { var deletePrm meta.DeletePrm deletePrm.SetAddresses(addrs...) From cc2da73b20a9aefd9fcab5ce9bb9312ffd41153d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 24 Jan 2024 20:09:20 +0300 Subject: [PATCH 0813/1943] [#929] go.mod: Update dependencies Signed-off-by: Evgenii Stratonikov --- go.mod | 77 +++++++++++++------------- go.sum | 170 ++++++++++++++++++++++++++++++--------------------------- 2 files changed, 128 insertions(+), 119 deletions(-) diff --git a/go.mod b/go.mod index 8b48cd788..3b269db34 100644 --- a/go.mod +++ b/go.mod @@ -3,51 +3,50 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.20 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240112150928-72885aae835c git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240117145620-110b7e41706e git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240112075604-c80c99b13e53 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240122104724-06cbfe8691ad git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 - github.com/google/uuid v1.4.0 + github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/klauspost/compress v1.17.4 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 - github.com/multiformats/go-multiaddr v0.12.0 - github.com/nats-io/nats.go v1.31.0 - github.com/nspcc-dev/neo-go v0.105.0 + github.com/multiformats/go-multiaddr v0.12.1 + github.com/nats-io/nats.go v1.32.0 + github.com/nspcc-dev/neo-go v0.105.1 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 - github.com/paulmach/orb v0.9.2 - github.com/prometheus/client_golang v1.17.0 + github.com/paulmach/orb v0.11.0 + github.com/prometheus/client_golang v1.18.0 github.com/spf13/cast v1.6.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.17.0 + github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.8.4 go.etcd.io/bbolt v1.3.8 - go.opentelemetry.io/otel v1.21.0 - go.opentelemetry.io/otel/trace v1.21.0 + go.opentelemetry.io/otel v1.22.0 + go.opentelemetry.io/otel/trace v1.22.0 go.uber.org/zap v1.26.0 - golang.org/x/exp v0.0.0-20231127185646-65229373498e - golang.org/x/sync v0.5.0 - golang.org/x/term v0.15.0 - google.golang.org/grpc v1.59.0 - google.golang.org/protobuf v1.31.0 + golang.org/x/exp v0.0.0-20240119083558-1b970713d09a + golang.org/x/sync v0.6.0 + golang.org/x/term v0.16.0 + google.golang.org/grpc v1.61.0 + google.golang.org/protobuf v1.32.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - go.mongodb.org/mongo-driver v1.11.4 // indirect + go.mongodb.org/mongo-driver v1.13.1 // indirect ) require ( @@ -55,23 +54,23 @@ require ( git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.11.0 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.12.2-0.20231117165148-e77308824822 // indirect + github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.4 // indirect @@ -88,15 +87,15 @@ require ( github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/nats-io/nkeys v0.4.6 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect - github.com/nspcc-dev/go-ordered-json v0.0.0-20231123160306-3374ff1e7a3c // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231201072346-6c0c2a6a98b0 // indirect + github.com/nspcc-dev/go-ordered-json v0.0.0-20240112074137-296698a162ae // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240122090917-ef99a7a9e33f // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -106,19 +105,19 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/twmb/murmur3 v1.1.8 // indirect github.com/urfave/cli v1.22.14 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 // indirect - go.opentelemetry.io/otel/metric v1.21.0 // indirect - go.opentelemetry.io/otel/sdk v1.21.0 // indirect - go.opentelemetry.io/proto/otlp v1.0.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0 // indirect + go.opentelemetry.io/otel/metric v1.22.0 // indirect + go.opentelemetry.io/otel/sdk v1.22.0 // indirect + go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.19.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/crypto v0.18.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 11b208d32..ca28fdaec 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,17 @@ -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd h1:MxiIXu4vUATX9bNYFnM1dXAYkx5dnPOGCMzwNNXJEvM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231031104748-498877e378fd/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240112150928-72885aae835c h1:dO2VLtoTQre4W28i1gSuW1MO80qFd0Lq9Q46Pu3q3TA= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240112150928-72885aae835c/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa h1:uuN90s74idThHXi/jF/IlTdFimOIs+IBU44rdTE/Zdo= git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e h1:RoGxgSNyVxDaYgjb40P+Xy6val+rs7b5QQtGkIUpUoY= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20231122162120-56debcfa569e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240117145620-110b7e41706e h1:0t+3iEb2wFJjJryqGnh5wkr9DEiwRNb61KaxC27U+po= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240117145620-110b7e41706e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240112075604-c80c99b13e53 h1:LNfUfiGx3cECl15Q5ZspILWZriCeevH4Ue+eynKgGYQ= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240112075604-c80c99b13e53/go.mod h1:ps6oKO0mxaPJzK3admTB3iwoBXKkHnS73n4PCrqpHBg= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240122104724-06cbfe8691ad h1:+D96Uu0Pw+55CrxP+srazthtZAh0Q19BtJo1Pm4hOP0= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240122104724-06cbfe8691ad/go.mod h1:ps6oKO0mxaPJzK3admTB3iwoBXKkHnS73n4PCrqpHBg= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= @@ -21,8 +21,8 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8 github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.11.0 h1:RMyy2mBBShArUAhfVRZJ2xyBO58KCBCtZFShw3umo6k= -github.com/bits-and-blooms/bitset v1.11.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -40,8 +40,8 @@ github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark-crypto v0.12.2-0.20231117165148-e77308824822 h1:PvEjRgB/U4bv0jl9w65Wy9g0nIdkkW7vkNoR8Vq/als= -github.com/consensys/gnark-crypto v0.12.2-0.20231117165148-e77308824822/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 h1:tYj5Ydh5D7Xg2R1tJnoG36Yta7NVB8C0vx36oPA3Bbw= +github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2/go.mod h1:wKqwsieaKPThcFkHe0d0zMsbHEUWFmZcG7KBCse210o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -62,8 +62,8 @@ github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmV github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -90,16 +90,16 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 h1:f4tggROQKKcnh4eItay6z/HbHLqghBxS8g7pyMhmDio= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0/go.mod h1:hKAkSgNkL0FII46ZkJcpVEAai4KV+swlIWCKfekd1pA= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 h1:HcUWd006luQPljE73d5sk+/VgYPGUReEVz2y1/qylwY= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -138,8 +138,6 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -156,26 +154,26 @@ github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aG github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.12.0 h1:1QlibTFkoXJuDjjYsMHhE73TnzJQl8FSWatk/0gxGzE= -github.com/multiformats/go-multiaddr v0.12.0/go.mod h1:WmZXgObOQOYp9r3cslLlppkrz1FYSHmE834dfz/lWu8= +github.com/multiformats/go-multiaddr v0.12.1 h1:vm+BA/WZA8QZDp1pF1FWhi5CT3g1tbi5GJmqpb6wnlk= +github.com/multiformats/go-multiaddr v0.12.1/go.mod h1:7mPkiBMmLeFipt+nNSq9pHZUeJSt8lHBgH6yhj0YQzE= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E= -github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8= -github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= -github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= +github.com/nats-io/nats.go v1.32.0 h1:Bx9BZS+aXYlxW08k8Gd3yR2s73pV5XSoAQUyp1Kwvp0= +github.com/nats-io/nats.go v1.32.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nspcc-dev/go-ordered-json v0.0.0-20231123160306-3374ff1e7a3c h1:OOQeE613BH93ICPq3eke5N78gWNeMjcBWkmD2NKyXVg= -github.com/nspcc-dev/go-ordered-json v0.0.0-20231123160306-3374ff1e7a3c/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= -github.com/nspcc-dev/neo-go v0.105.0 h1:vtNZYFEFySK8zRDhLzQYha849VzWrcKezlnq/oNQg/w= -github.com/nspcc-dev/neo-go v0.105.0/go.mod h1:6pchIHg5okeZO955RxpTh5q0sUI0vtpgPM6Q+no1rlI= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231201072346-6c0c2a6a98b0 h1:GYeU/mq2qfq6fNSuTzgxvlQQUydeCBicjzdnggwtuz0= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231201072346-6c0c2a6a98b0/go.mod h1:J/Mk6+nKeKSW4wygkZQFLQ6SkLOSGX5Ga0RuuuktEag= +github.com/nspcc-dev/go-ordered-json v0.0.0-20240112074137-296698a162ae h1:UFgMXcZthqiCqCyr3dOAtGICJ10gM8q0mFHyLR0UPQU= +github.com/nspcc-dev/go-ordered-json v0.0.0-20240112074137-296698a162ae/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= +github.com/nspcc-dev/neo-go v0.105.1 h1:r0b2yIwLBi+ARBKU94gHL9oTFEB/XMJ0YlS2HN9Qw34= +github.com/nspcc-dev/neo-go v0.105.1/go.mod h1:GNh0cRALV/cuj+/xg2ZHDsrFbqcInqG7jjhqsLEnlNc= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240122090917-ef99a7a9e33f h1:6h0zgQGKdStROXRlWhz2dZmrKkSmOJ4oiDs06nO39XI= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240122090917-ef99a7a9e33f/go.mod h1:J/Mk6+nKeKSW4wygkZQFLQ6SkLOSGX5Ga0RuuuktEag= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -196,21 +194,21 @@ github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/panjf2000/ants/v2 v2.9.0 h1:SztCLkVxBRigbg+vt0S5QvF5vxAbxbKt09/YfAJ0tEo= github.com/panjf2000/ants/v2 v2.9.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= -github.com/paulmach/orb v0.9.2 h1:p/YWV2uJwamAynnDOJGNbPBVtDHj3vG51k9tR1rFwJE= -github.com/paulmach/orb v0.9.2/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/orb v0.11.0 h1:JfVXJUBeH9ifc/OrhBY0lL16QsmPgpCHMlqSSYhcgAA= +github.com/paulmach/orb v0.11.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= -github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= -github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= +github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -235,8 +233,8 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI= -github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -253,7 +251,6 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= @@ -263,30 +260,34 @@ github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtb github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= -go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 h1:VhlEQAPp9R1ktYfrPk5SOryw1e9LDDTZCbIPFrho0ec= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0/go.mod h1:kB3ufRbfU+CQ4MlUcqtW8Z7YEOBeK2DJ6CmR5rYYF3E= -go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= -go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= -go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= -go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk= +go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= +go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= +go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 h1:H2JFgRcGiyHg7H7bwcwaQJYrNFqCqrbTQ8K4p1OvDu8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0/go.mod h1:WfCWp1bGoYK8MeULtI15MmQVczfR+bFkk0DF3h06QmQ= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0 h1:zr8ymM5OWWjjiWRzwTfZ67c905+2TMHYp2lMJ52QTyM= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0/go.mod h1:sQs7FT2iLVJ+67vYngGJkPe1qr39IzaBzaj9IDNNY8k= +go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= +go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= +go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= +go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= +go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -295,13 +296,15 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= -golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= +golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -309,20 +312,23 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -343,17 +349,20 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -361,19 +370,20 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f h1:Vn+VyHU5guc9KjB5KrjI2q0wCOWEOIh0OEsleqakHJg= -google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 h1:ZcOkrmX74HbKFYnpPY8Qsw93fC29TbJXspYKaBkSXDQ= -google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4/go.mod h1:k2dtGpRrbsSyKcNPKKI5sstZkrNCZwpU/ns96JoHbGg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= +google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -383,8 +393,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From c00eb7cceec6f4e598333588bf043141a4ccce0d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 24 Jan 2024 11:02:05 +0300 Subject: [PATCH 0814/1943] [#768] adm: Return error from getContractDeployData() Make code more idiomatic and unify in style with other error reporting. Retain panics for code errors. Signed-off-by: Evgenii Stratonikov --- .../modules/morph/initialize_deploy.go | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 61536b4d5..dd304d8af 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -222,7 +222,11 @@ func (c *initializeContext) deployOrUpdateContracts(w *io2.BufBinWriter, nnsHash invokeHash = ctrHash } - params := getContractDeployParameters(cs, c.getContractDeployData(ctrName, keysParam, updateMethodName)) + args, err := c.getContractDeployData(ctrName, keysParam, updateMethodName) + if err != nil { + return fmt.Errorf("%s: getting update params: %v", ctrName, err) + } + params := getContractDeployParameters(cs, args) res, err := c.CommitteeAct.MakeCall(invokeHash, method, params...) if err != nil { if method != updateMethodName || !strings.Contains(err.Error(), common.ErrAlreadyUpdated) { @@ -345,7 +349,11 @@ func (c *initializeContext) deployContracts() error { return fmt.Errorf("can't sign manifest group: %v", err) } - params := getContractDeployParameters(cs, c.getContractDeployData(ctrName, keysParam, deployMethodName)) + args, err := c.getContractDeployData(ctrName, keysParam, deployMethodName) + if err != nil { + return fmt.Errorf("%s: getting deploy params: %v", ctrName, err) + } + params := getContractDeployParameters(cs, args) res, err := c.CommitteeAct.MakeCall(management.Hash, deployMethodName, params...) if err != nil { return fmt.Errorf("can't deploy %s contract: %w", ctrName, err) @@ -511,7 +519,7 @@ func getContractDeployParameters(cs *contractState, deployData []any) []any { return []any{cs.RawNEF, cs.RawManifest, deployData} } -func (c *initializeContext) getContractDeployData(ctrName string, keysParam []any, method string) []any { +func (c *initializeContext) getContractDeployData(ctrName string, keysParam []any, method string) ([]any, error) { items := make([]any, 0, 6) switch ctrName { @@ -522,7 +530,7 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an smartcontract.Parameter{}) case processingContract: items = append(items, c.Contracts[frostfsContract].Hash) - return items[1:] // no notary info + return items[1:], nil // no notary info case balanceContract: items = append(items, c.Contracts[netmapContract].Hash, @@ -533,7 +541,7 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an r := management.NewReader(c.ReadOnlyInvoker) nnsCs, err := r.GetContractByID(1) if err != nil { - panic("NNS is not yet deployed") + return nil, errors.New("NNS is not yet deployed") } items = append(items, c.Contracts[netmapContract].Hash, @@ -544,7 +552,7 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an case frostfsIDContract: h, found, err := getFrostfsIDAdmin(viper.GetViper()) if err != nil { - panic(err) + return nil, err } if found { @@ -557,11 +565,11 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an if method == updateMethodName { arr, err := c.getNetConfigFromNetmapContract() if err != nil { - panic(err) + return nil, err } m, err := parseConfigFromNetmapContract(arr) if err != nil { - panic(err) + return nil, err } for k, v := range m { for _, key := range netmapConfigKeys { @@ -590,7 +598,7 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an default: panic(fmt.Sprintf("invalid contract name: %s", ctrName)) } - return items + return items, nil } func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, error) { From 61a3afbf9b1039b6663b2ecf88786491d7ed0d3f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 24 Jan 2024 11:04:37 +0300 Subject: [PATCH 0815/1943] [#768] adm: Rephrase error text for NNS fetch Error could come from the network, we must not assume everything. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index dd304d8af..20e44a81a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -541,7 +541,7 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an r := management.NewReader(c.ReadOnlyInvoker) nnsCs, err := r.GetContractByID(1) if err != nil { - return nil, errors.New("NNS is not yet deployed") + return nil, fmt.Errorf("get nns contract: %w", err) } items = append(items, c.Contracts[netmapContract].Hash, @@ -605,7 +605,7 @@ func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, r := management.NewReader(c.ReadOnlyInvoker) cs, err := r.GetContractByID(1) if err != nil { - return nil, fmt.Errorf("NNS is not yet deployed: %w", err) + return nil, fmt.Errorf("get nns contract: %w", err) } nmHash, err := nnsResolveHash(c.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) if err != nil { From d2d850786d83b8da0ef8a0c36dfcd98efed5678c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 24 Jan 2024 19:43:17 +0300 Subject: [PATCH 0816/1943] [#923] adm: Move netconfig merge to a separate function Signed-off-by: Evgenii Stratonikov --- .../modules/morph/initialize_deploy.go | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 20e44a81a..ca3a9599f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -563,22 +563,9 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an case netmapContract: md := getDefaultNetmapContractConfigMap() if method == updateMethodName { - arr, err := c.getNetConfigFromNetmapContract() - if err != nil { + if err := c.mergeNetmapConfig(md); err != nil { return nil, err } - m, err := parseConfigFromNetmapContract(arr) - if err != nil { - return nil, err - } - for k, v := range m { - for _, key := range netmapConfigKeys { - if k == key { - md[k] = v - break - } - } - } } var configParam []any @@ -618,6 +605,26 @@ func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, return arr, err } +func (c *initializeContext) mergeNetmapConfig(md map[string]any) error { + arr, err := c.getNetConfigFromNetmapContract() + if err != nil { + return err + } + m, err := parseConfigFromNetmapContract(arr) + if err != nil { + return err + } + for k, v := range m { + for _, key := range netmapConfigKeys { + if k == key { + md[k] = v + break + } + } + } + return nil +} + func (c *initializeContext) getAlphabetDeployItems(i, n int) []any { items := make([]any, 5) items[0] = c.Contracts[netmapContract].Hash From d13e37f70b44dfe7fe002ebf327c0d293093c0e5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 24 Jan 2024 11:19:18 +0300 Subject: [PATCH 0817/1943] [#768] adm: Do not change frostfsid admin on update Behave similarly to the netmap contract. Close #768 Signed-off-by: Evgenii Stratonikov --- .../modules/morph/initialize_deploy.go | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index ca3a9599f..6cf75c5f9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -550,7 +550,17 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an nnsCs.Hash, "container") case frostfsIDContract: - h, found, err := getFrostfsIDAdmin(viper.GetViper()) + var ( + h util.Uint160 + found bool + err error + ) + if method == updateMethodName { + h, found, err = c.getFrostfsIDAdminFromContract() + } + if method != updateMethodName || err == nil && !found { + h, found, err = getFrostfsIDAdmin(viper.GetViper()) + } if err != nil { return nil, err } @@ -588,6 +598,35 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an return items, nil } +func (c *initializeContext) getFrostfsIDAdminFromContract() (util.Uint160, bool, error) { + r := management.NewReader(c.ReadOnlyInvoker) + cs, err := r.GetContractByID(1) + if err != nil { + return util.Uint160{}, false, fmt.Errorf("get nns contract: %w", err) + } + fidHash, err := nnsResolveHash(c.ReadOnlyInvoker, cs.Hash, domainOf(frostfsIDContract)) + if err != nil { + return util.Uint160{}, false, fmt.Errorf("resolve frostfsid contract hash: %w", err) + } + item, err := unwrap.Item(c.ReadOnlyInvoker.Call(fidHash, "getAdmin")) + if err != nil { + return util.Uint160{}, false, fmt.Errorf("getAdmin: %w", err) + } + if _, ok := item.(stackitem.Null); ok { + return util.Uint160{}, false, nil + } + + bs, err := item.TryBytes() + if err != nil { + return util.Uint160{}, true, fmt.Errorf("getAdmin: decode result: %w", err) + } + h, err := util.Uint160DecodeBytesBE(bs) + if err != nil { + return util.Uint160{}, true, fmt.Errorf("getAdmin: decode result: %w", err) + } + return h, true, nil +} + func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, error) { r := management.NewReader(c.ReadOnlyInvoker) cs, err := r.GetContractByID(1) From 1fe7736d921a06411f8076cedb8edf68519aca62 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 24 Jan 2024 15:55:04 +0300 Subject: [PATCH 0818/1943] [#925] morph: Introduce switch rpc guarded rpc actor * Introduce switch rpc guarded rpc actor in morph client to prevent using invalidated rpc actor when RPC switch happens. * Initialize NewContractStorage with SwitchRPCGuardedActor. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/config.go | 2 +- pkg/morph/client/actor.go | 120 +++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 pkg/morph/client/actor.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index ffe49a7ef..1b3e094d9 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1061,7 +1061,7 @@ func initAccessPolicyEngine(_ context.Context, c *cfg) { } morphRuleStorage := policy_client.NewContractStorage( - c.cfgMorph.client.GetActor(), + client.NewSwitchRPCGuardedActor(c.cfgMorph.client), c.cfgObject.cfgAccessPolicyEngine.policyContractHash) ape := newAccessPolicyEngine(morphRuleStorage, localOverrideDB) diff --git a/pkg/morph/client/actor.go b/pkg/morph/client/actor.go new file mode 100644 index 000000000..ea6bc974c --- /dev/null +++ b/pkg/morph/client/actor.go @@ -0,0 +1,120 @@ +package client + +import ( + "github.com/nspcc-dev/neo-go/pkg/config/netmode" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/util" +) + +type actorProvider interface { + GetActor() *actor.Actor +} + +// Client switches an established connection with neo-go if it is broken. +// This leads to an invalidation of an rpc actor within Client. That means the +// components that are initilized with the rpc actor may unintentionally use +// it when it is already invalidated. SwitchRPCGuardedActor is used to prevent +// this situation, getting the rpc actor from Client. +type SwitchRPCGuardedActor struct { + actorProvider actorProvider +} + +func NewSwitchRPCGuardedActor(c *Client) *SwitchRPCGuardedActor { + return &SwitchRPCGuardedActor{ + actorProvider: c, + } +} + +func (a *SwitchRPCGuardedActor) Call(contract util.Uint160, operation string, params ...any) (*result.Invoke, error) { + return a.actorProvider.GetActor().Call(contract, operation, params...) +} + +func (a *SwitchRPCGuardedActor) CalculateNetworkFee(tx *transaction.Transaction) (int64, error) { + return a.actorProvider.GetActor().CalculateNetworkFee(tx) +} + +func (a *SwitchRPCGuardedActor) CalculateValidUntilBlock() (uint32, error) { + return a.actorProvider.GetActor().CalculateValidUntilBlock() +} + +func (a *SwitchRPCGuardedActor) GetBlockCount() (uint32, error) { + return a.actorProvider.GetActor().GetBlockCount() +} + +func (a *SwitchRPCGuardedActor) GetNetwork() netmode.Magic { + return a.actorProvider.GetActor().GetNetwork() +} + +func (a *SwitchRPCGuardedActor) GetVersion() result.Version { + return a.actorProvider.GetActor().GetVersion() +} + +func (a *SwitchRPCGuardedActor) MakeCall(contract util.Uint160, method string, params ...any) (*transaction.Transaction, error) { + return a.actorProvider.GetActor().MakeCall(contract, method, params...) +} + +func (a *SwitchRPCGuardedActor) MakeRun(script []byte) (*transaction.Transaction, error) { + return a.actorProvider.GetActor().MakeRun(script) +} + +func (a *SwitchRPCGuardedActor) MakeTunedCall(contract util.Uint160, method string, attrs []transaction.Attribute, txHook actor.TransactionCheckerModifier, params ...any) (*transaction.Transaction, error) { + return a.actorProvider.GetActor().MakeTunedCall(contract, method, attrs, txHook, params...) +} + +func (a *SwitchRPCGuardedActor) MakeTunedRun(script []byte, attrs []transaction.Attribute, txHook actor.TransactionCheckerModifier) (*transaction.Transaction, error) { + return a.actorProvider.GetActor().MakeTunedRun(script, attrs, txHook) +} + +func (a *SwitchRPCGuardedActor) MakeUncheckedRun(script []byte, sysfee int64, attrs []transaction.Attribute, txHook actor.TransactionModifier) (*transaction.Transaction, error) { + return a.actorProvider.GetActor().MakeUncheckedRun(script, sysfee, attrs, txHook) +} + +func (a *SwitchRPCGuardedActor) MakeUnsignedCall(contract util.Uint160, method string, attrs []transaction.Attribute, params ...any) (*transaction.Transaction, error) { + return a.actorProvider.GetActor().MakeUnsignedCall(contract, method, attrs, params...) +} + +func (a *SwitchRPCGuardedActor) MakeUnsignedRun(script []byte, attrs []transaction.Attribute) (*transaction.Transaction, error) { + return a.actorProvider.GetActor().MakeUnsignedRun(script, attrs) +} + +func (a *SwitchRPCGuardedActor) MakeUnsignedUncheckedRun(script []byte, sysFee int64, attrs []transaction.Attribute) (*transaction.Transaction, error) { + return a.actorProvider.GetActor().MakeUnsignedUncheckedRun(script, sysFee, attrs) +} + +func (a *SwitchRPCGuardedActor) Send(tx *transaction.Transaction) (util.Uint256, uint32, error) { + return a.actorProvider.GetActor().Send(tx) +} + +func (a *SwitchRPCGuardedActor) SendCall(contract util.Uint160, method string, params ...any) (util.Uint256, uint32, error) { + return a.actorProvider.GetActor().SendCall(contract, method, params...) +} + +func (a *SwitchRPCGuardedActor) SendRun(script []byte) (util.Uint256, uint32, error) { + return a.actorProvider.GetActor().SendRun(script) +} + +func (a *SwitchRPCGuardedActor) SendTunedCall(contract util.Uint160, method string, attrs []transaction.Attribute, txHook actor.TransactionCheckerModifier, params ...any) (util.Uint256, uint32, error) { + return a.actorProvider.GetActor().SendTunedCall(contract, method, attrs, txHook, params...) +} + +func (a *SwitchRPCGuardedActor) SendTunedRun(script []byte, attrs []transaction.Attribute, txHook actor.TransactionCheckerModifier) (util.Uint256, uint32, error) { + return a.actorProvider.GetActor().SendTunedRun(script, attrs, txHook) +} + +func (a *SwitchRPCGuardedActor) SendUncheckedRun(script []byte, sysfee int64, attrs []transaction.Attribute, txHook actor.TransactionModifier) (util.Uint256, uint32, error) { + return a.actorProvider.GetActor().SendUncheckedRun(script, sysfee, attrs, txHook) +} + +func (a *SwitchRPCGuardedActor) Sender() util.Uint160 { + return a.actorProvider.GetActor().Sender() +} + +func (a *SwitchRPCGuardedActor) Sign(tx *transaction.Transaction) error { + return a.actorProvider.GetActor().Sign(tx) +} + +func (a *SwitchRPCGuardedActor) SignAndSend(tx *transaction.Transaction) (util.Uint256, uint32, error) { + return a.actorProvider.GetActor().SignAndSend(tx) +} From b6fc3321c52055ca976a61158c33b47669fc77a7 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 22 Jan 2024 13:39:57 +0300 Subject: [PATCH 0819/1943] [#876] Fix linters Signed-off-by: Anton Nikiforov --- pkg/services/object/ape/checker.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index ccd23bf06..76ada3d2b 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -47,9 +47,7 @@ type Prm struct { SenderKey string } -var ( - errMissingOID = fmt.Errorf("object ID is not set") -) +var errMissingOID = fmt.Errorf("object ID is not set") // CheckAPE checks if a request or a response is permitted creating an ape request and passing // it to chain router. From 51d1d935ef60660cfdd32afccd1ac97dfbfc35b8 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 25 Jan 2024 20:25:23 +0300 Subject: [PATCH 0820/1943] [#876] cli: Add support for `container` in local rules Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/control/add_rule.go | 29 +-- cmd/frostfs-cli/modules/control/get_rule.go | 5 +- cmd/frostfs-cli/modules/control/list_rules.go | 4 +- cmd/frostfs-cli/modules/util/ape.go | 216 ++++++++++++------ cmd/frostfs-cli/modules/util/ape_test.go | 214 +++++++++++++++-- 5 files changed, 351 insertions(+), 117 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index 4fe8194bb..7e0805cd1 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -1,10 +1,8 @@ package control import ( - "bytes" "crypto/sha256" "encoding/hex" - "encoding/json" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -25,20 +23,13 @@ var addRuleCmd = &cobra.Command{ Use: "add-rule", Short: "Add local override", Long: "Add local APE rule to a node with following format:\n[:action_detail] [ ...] ", - Example: `allow Object.Get * -deny Object.Get EbxzAdz5LB4uqxuz6crWKAumBNtZyK2rKsqQP7TdZvwr/* -deny:QuotaLimitReached Object.Put Object.Resource:Department=HR * + Example: `control add-rule --endpoint ... -w ... --address ... --chain-id ChainID --cid ... --rule "allow Object.Get *" +--rule "deny Object.Get EbxzAdz5LB4uqxuz6crWKAumBNtZyK2rKsqQP7TdZvwr/*" +--rule "deny:QuotaLimitReached Object.Put Object.Resource:Department=HR *" `, Run: addRule, } -func prettyJSONFormat(cmd *cobra.Command, serializedChain []byte) string { - wr := bytes.NewBufferString("") - err := json.Indent(wr, serializedChain, "", " ") - commonCmd.ExitOnErr(cmd, "%w", err) - return wr.String() -} - func addRule(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) @@ -60,15 +51,15 @@ func addRule(cmd *cobra.Command, _ []string) { rawCID := make([]byte, sha256.Size) cnr.Encode(rawCID) - rule, _ := cmd.Flags().GetString(ruleFlag) + rule, _ := cmd.Flags().GetStringArray(ruleFlag) chain := new(apechain.Chain) - commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, []string{rule})) + commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, rule)) chain.ID = apechain.ID(chainIDRaw) serializedChain := chain.Bytes() - cmd.Println("CID: " + cidStr) - cmd.Println("Parsed chain:\n" + prettyJSONFormat(cmd, serializedChain)) + cmd.Println("Parsed chain:") + util.PrintHumanReadableAPEChain(cmd, chain) req := &control.AddChainLocalOverrideRequest{ Body: &control.AddChainLocalOverrideRequest_Body{ @@ -93,9 +84,7 @@ func addRule(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "rpc error: %w", err) verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - - chainIDRaw = resp.GetBody().GetChainId() - cmd.Printf("Rule has been added.\nChain id: '%s'\nChain id hex: '%x'\n", string(chainIDRaw), chainIDRaw) + cmd.Println("\nRule has been added.") } func initControlAddRuleCmd() { @@ -103,7 +92,7 @@ func initControlAddRuleCmd() { ff := addRuleCmd.Flags() ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) - ff.String(ruleFlag, "", "Rule statement") + ff.StringArray(ruleFlag, []string{}, "Rule statement") ff.String(chainIDFlag, "", "Assign ID to the parsed chain") ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") } diff --git a/cmd/frostfs-cli/modules/control/get_rule.go b/cmd/frostfs-cli/modules/control/get_rule.go index 9508850ee..e2271c446 100644 --- a/cmd/frostfs-cli/modules/control/get_rule.go +++ b/cmd/frostfs-cli/modules/control/get_rule.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -66,9 +67,7 @@ func getRule(cmd *cobra.Command, _ []string) { var chain apechain.Chain commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(resp.GetBody().GetChain())) - - // TODO (aarifullin): make pretty-formatted output for chains. - cmd.Printf("Parsed chain (chain id hex: '%x'):\n%s\n", chain.ID, prettyJSONFormat(cmd, chain.Bytes())) + util.PrintHumanReadableAPEChain(cmd, &chain) } func initControGetRuleCmd() { diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index ef494b135..c62651767 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -60,10 +61,9 @@ func listRules(cmd *cobra.Command, _ []string) { } for _, c := range chains { - // TODO (aarifullin): make pretty-formatted output for chains. var chain apechain.Chain commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(c)) - cmd.Printf("Parsed chain (chain id hex: '%x'):\n%s\n", chain.ID, prettyJSONFormat(cmd, chain.Bytes())) + util.PrintHumanReadableAPEChain(cmd, &chain) } } diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index 79966613d..6a7eca948 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -13,21 +13,25 @@ import ( ) var ( - errInvalidStatementFormat = errors.New("invalid statement format") - errInvalidConditionFormat = errors.New("invalid condition format") - errUnknownAction = errors.New("action is not recognized") - errUnknownOperation = errors.New("operation is not recognized") - errUnknownActionDetail = errors.New("action detail is not recognized") - errUnknownBinaryOperator = errors.New("binary operator is not recognized") - errUnknownCondObjectType = errors.New("condition object type is not recognized") + errInvalidStatementFormat = errors.New("invalid statement format") + errInvalidConditionFormat = errors.New("invalid condition format") + errUnknownStatus = errors.New("status is not recognized") + errUnknownStatusDetail = errors.New("status detail is not recognized") + errUnknownAction = errors.New("action is not recognized") + errUnknownBinaryOperator = errors.New("binary operator is not recognized") + errUnknownCondObjectType = errors.New("condition object type is not recognized") + errMixedTypesInRule = errors.New("found mixed type of actions and conditions in rule") + errNoActionsInRule = errors.New("there are no actions in rule") + errUnsupportedResourceFormat = errors.New("unsupported resource format") ) // PrintHumanReadableAPEChain print APE chain rules. func PrintHumanReadableAPEChain(cmd *cobra.Command, chain *apechain.Chain) { - cmd.Println("ChainID: " + string(chain.ID)) + cmd.Println("Chain ID: " + string(chain.ID)) + cmd.Printf(" HEX: %x\n", chain.ID) cmd.Println("Rules:") for _, rule := range chain.Rules { - cmd.Println("\tStatus: " + rule.Status.String()) + cmd.Println("\n\tStatus: " + rule.Status.String()) cmd.Println("\tAny: " + strconv.FormatBool(rule.Any)) cmd.Println("\tConditions:") for _, c := range rule.Condition { @@ -71,7 +75,7 @@ func ParseAPEChain(chain *apechain.Chain, rules []string) error { } // ParseAPERule parses access-policy-engine statement from the following form: -// [:action_detail] [ ...] +// [:status_detail] ... [...] ... // // Examples: // deny Object.Put * @@ -99,116 +103,178 @@ func parseRuleLexemes(r *apechain.Rule, lexemes []string) error { return err } - r.Actions, err = parseAction(lexemes[1]) - if err != nil { - return err + var isObject *bool + for i, lexeme := range lexemes[1:] { + var name string + var actionType bool + name, actionType, err = parseAction(lexeme) + if err != nil { + condition, errCond := parseCondition(lexeme) + if errCond != nil { + err = fmt.Errorf("%w:%w", err, errCond) + lexemes = lexemes[i+1:] + break + } + actionType = condition.Object == apechain.ObjectResource || condition.Object == apechain.ObjectRequest + r.Condition = append(r.Condition, *condition) + } else { + r.Actions.Names = append(r.Actions.Names, name) + } + if isObject == nil { + isObject = &actionType + } else if actionType != *isObject { + return errMixedTypesInRule + } + } + if len(r.Actions.Names) == 0 { + return fmt.Errorf("%w:%w", err, errNoActionsInRule) + } + for _, lexeme := range lexemes { + resource, errRes := parseResource(lexeme, *isObject) + if errRes != nil { + return fmt.Errorf("%w:%w", err, errRes) + } + r.Resources.Names = append(r.Resources.Names, resource) } - r.Condition, err = parseConditions(lexemes[2 : len(lexemes)-1]) - if err != nil { - return err - } - - r.Resources, err = parseResource(lexemes[len(lexemes)-1]) - return err + return nil } func parseStatus(lexeme string) (apechain.Status, error) { action, expression, found := strings.Cut(lexeme, ":") - switch action = strings.ToLower(action); action { + switch strings.ToLower(action) { case "deny": if !found { return apechain.AccessDenied, nil } else if strings.EqualFold(expression, "QuotaLimitReached") { return apechain.QuotaLimitReached, nil } else { - return 0, fmt.Errorf("%w: %s", errUnknownActionDetail, expression) + return 0, fmt.Errorf("%w: %s", errUnknownStatusDetail, expression) } case "allow": if found { - return 0, errUnknownActionDetail + return 0, errUnknownStatusDetail } return apechain.Allow, nil default: - return 0, errUnknownAction + return 0, errUnknownStatus } } -func parseAction(lexeme string) (apechain.Actions, error) { +func parseAction(lexeme string) (string, bool, error) { switch strings.ToLower(lexeme) { case "object.put": - return apechain.Actions{Names: []string{nativeschema.MethodPutObject}}, nil + return nativeschema.MethodPutObject, true, nil case "object.get": - return apechain.Actions{Names: []string{nativeschema.MethodGetObject}}, nil + return nativeschema.MethodGetObject, true, nil case "object.head": - return apechain.Actions{Names: []string{nativeschema.MethodHeadObject}}, nil + return nativeschema.MethodHeadObject, true, nil case "object.delete": - return apechain.Actions{Names: []string{nativeschema.MethodDeleteObject}}, nil + return nativeschema.MethodDeleteObject, true, nil case "object.search": - return apechain.Actions{Names: []string{nativeschema.MethodSearchObject}}, nil + return nativeschema.MethodSearchObject, true, nil case "object.range": - return apechain.Actions{Names: []string{nativeschema.MethodRangeObject}}, nil + return nativeschema.MethodRangeObject, true, nil case "object.hash": - return apechain.Actions{Names: []string{nativeschema.MethodHashObject}}, nil + return nativeschema.MethodHashObject, true, nil + case "container.put": + return nativeschema.MethodPutContainer, false, nil + case "container.delete": + return nativeschema.MethodDeleteContainer, false, nil + case "container.get": + return nativeschema.MethodGetContainer, false, nil + case "container.setcontainereacl": + return nativeschema.MethodSetContainerEACL, false, nil + case "container.getcontainereacl": + return nativeschema.MethodGetContainerEACL, false, nil default: } - return apechain.Actions{}, fmt.Errorf("%w: %s", errUnknownOperation, lexeme) + return "", false, fmt.Errorf("%w: %s", errUnknownAction, lexeme) } -func parseResource(lexeme string) (apechain.Resources, error) { - if lexeme == "*" { - return apechain.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, nil +func parseResource(lexeme string, isObj bool) (string, error) { + if len(lexeme) > 0 && !strings.HasSuffix(lexeme, "/") { + if isObj { + if lexeme == "*" { + return nativeschema.ResourceFormatAllObjects, nil + } else if lexeme == "/*" { + return nativeschema.ResourceFormatRootObjects, nil + } else if strings.HasPrefix(lexeme, "/") { + lexeme = lexeme[1:] + delimCount := strings.Count(lexeme, "/") + if delimCount == 1 && len(lexeme) >= 3 { // container/object + return nativeschema.ObjectPrefix + "//" + lexeme, nil + } + } else { + delimCount := strings.Count(lexeme, "/") + if delimCount == 1 && len(lexeme) >= 3 || + delimCount == 2 && len(lexeme) >= 5 { // namespace/container/object + return nativeschema.ObjectPrefix + "/" + lexeme, nil + } + } + } else { + if lexeme == "*" { + return nativeschema.ResourceFormatAllContainers, nil + } else if lexeme == "/*" { + return nativeschema.ResourceFormatRootContainers, nil + } else if strings.HasPrefix(lexeme, "/") && len(lexeme) > 1 { + lexeme = lexeme[1:] + delimCount := strings.Count(lexeme, "/") + if delimCount == 0 { + return nativeschema.ContainerPrefix + "//" + lexeme, nil + } + } else { + delimCount := strings.Count(lexeme, "/") + if delimCount == 1 && len(lexeme) > 3 { // namespace/container + return nativeschema.ContainerPrefix + "/" + lexeme, nil + } + } + } } - return apechain.Resources{Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, lexeme)}}, nil + return "", errUnsupportedResourceFormat } const ( ObjectResource = "object.resource" ObjectRequest = "object.request" + + ContainerResource = "container.resource" + ContainerRequest = "container.request" ) var typeToCondObject = map[string]apechain.ObjectType{ - ObjectResource: apechain.ObjectResource, - ObjectRequest: apechain.ObjectRequest, + ObjectResource: apechain.ObjectResource, + ObjectRequest: apechain.ObjectRequest, + ContainerResource: apechain.ContainerResource, + ContainerRequest: apechain.ContainerRequest, } -func parseConditions(lexemes []string) ([]apechain.Condition, error) { - conds := make([]apechain.Condition, 0) +func parseCondition(lexeme string) (*apechain.Condition, error) { + typ, expression, found := strings.Cut(lexeme, ":") + typ = strings.ToLower(typ) - for _, lexeme := range lexemes { - typ, expression, found := strings.Cut(lexeme, ":") - typ = strings.ToLower(typ) - - objType, ok := typeToCondObject[typ] - if ok { - if !found { - return nil, fmt.Errorf("%w: %s", errInvalidConditionFormat, lexeme) - } - - var lhs, rhs string - var binExpFound bool - - var cond apechain.Condition - cond.Object = objType - - lhs, rhs, binExpFound = strings.Cut(expression, "!=") - if !binExpFound { - lhs, rhs, binExpFound = strings.Cut(expression, "=") - if !binExpFound { - return nil, fmt.Errorf("%w: %s", errUnknownBinaryOperator, expression) - } - cond.Op = apechain.CondStringEquals - } else { - cond.Op = apechain.CondStringNotEquals - } - - cond.Key, cond.Value = lhs, rhs - - conds = append(conds, cond) - } else { - return nil, fmt.Errorf("%w: %s", errUnknownCondObjectType, typ) + objType, ok := typeToCondObject[typ] + if ok { + if !found { + return nil, fmt.Errorf("%w: %s", errInvalidConditionFormat, lexeme) } - } - return conds, nil + var cond apechain.Condition + cond.Object = objType + + lhs, rhs, binExpFound := strings.Cut(expression, "!=") + if !binExpFound { + lhs, rhs, binExpFound = strings.Cut(expression, "=") + if !binExpFound { + return nil, fmt.Errorf("%w: %s", errUnknownBinaryOperator, expression) + } + cond.Op = apechain.CondStringEquals + } else { + cond.Op = apechain.CondStringNotEquals + } + + cond.Key, cond.Value = lhs, rhs + return &cond, nil + } + return nil, fmt.Errorf("%w: %s", errUnknownCondObjectType, typ) } diff --git a/cmd/frostfs-cli/modules/util/ape_test.go b/cmd/frostfs-cli/modules/util/ape_test.go index 1cab8e6ae..3e4766098 100644 --- a/cmd/frostfs-cli/modules/util/ape_test.go +++ b/cmd/frostfs-cli/modules/util/ape_test.go @@ -1,6 +1,7 @@ package util import ( + "fmt" "testing" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -16,13 +17,76 @@ func TestParseAPERule(t *testing.T) { expectRule policyengine.Rule }{ { - name: "Valid allow rule", + name: "Valid allow rule for all objects", rule: "allow Object.Put *", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatAllObjects}}, + }, + }, + { + name: "Valid rule for all objects in root namespace", + rule: "allow Object.Put /*", expectRule: policyengine.Rule{ Status: policyengine.Allow, Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, - Condition: []policyengine.Condition{}, + }, + }, + { + name: "Valid rule for all objects in root namespace and container", + rule: "allow Object.Put /cid/*", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: policyengine.Resources{Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, "cid"), + }}, + }, + }, + { + name: "Valid rule for object in root namespace and container", + rule: "allow Object.Put /cid/oid", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: policyengine.Resources{Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, "cid", "oid"), + }}, + }, + }, + { + name: "Valid rule for all objects in namespace", + rule: "allow Object.Put ns/*", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: policyengine.Resources{Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatNamespaceObjects, "ns"), + }}, + }, + }, + { + name: "Valid rule for all objects in namespace and container", + rule: "allow Object.Put ns/cid/*", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: policyengine.Resources{Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainerObjects, "ns", "cid"), + }}, + }, + }, + { + name: "Valid rule for object in namespace and container", + rule: "allow Object.Put ns/cid/oid", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: policyengine.Resources{Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainerObject, "ns", "cid", "oid"), + }}, }, }, { @@ -31,8 +95,7 @@ func TestParseAPERule(t *testing.T) { expectRule: policyengine.Rule{ Status: policyengine.AccessDenied, Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, - Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, - Condition: []policyengine.Condition{}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatAllObjects}}, }, }, { @@ -41,8 +104,7 @@ func TestParseAPERule(t *testing.T) { expectRule: policyengine.Rule{ Status: policyengine.QuotaLimitReached, Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, - Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, - Condition: []policyengine.Condition{}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatAllObjects}}, }, }, { @@ -51,7 +113,7 @@ func TestParseAPERule(t *testing.T) { expectRule: policyengine.Rule{ Status: policyengine.Allow, Actions: policyengine.Actions{Names: []string{nativeschema.MethodGetObject}}, - Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatAllObjects}}, Condition: []policyengine.Condition{ { Op: policyengine.CondStringEquals, @@ -69,12 +131,12 @@ func TestParseAPERule(t *testing.T) { }, }, { - name: "Valid rule with conditions with action detail", + name: "Valid rule for object with conditions with action detail", rule: "deny:QuotaLimitReached Object.Get Object.Resource:Department=HR Object.Request:Actor!=ownerA *", expectRule: policyengine.Rule{ Status: policyengine.QuotaLimitReached, Actions: policyengine.Actions{Names: []string{nativeschema.MethodGetObject}}, - Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatAllObjects}}, Condition: []policyengine.Condition{ { Op: policyengine.CondStringEquals, @@ -92,19 +154,19 @@ func TestParseAPERule(t *testing.T) { }, }, { - name: "Invalid rule with unknown action", + name: "Invalid rule with unknown status", rule: "permit Object.Put *", + expectErr: errUnknownStatus, + }, + { + name: "Invalid rule with unknown action", + rule: "allow Object.PutOut *", expectErr: errUnknownAction, }, { - name: "Invalid rule with unknown operation", - rule: "allow Object.PutOut *", - expectErr: errUnknownOperation, - }, - { - name: "Invalid rule with unknown action detail", + name: "Invalid rule with unknown status detail", rule: "deny:UnknownActionDetail Object.Put *", - expectErr: errUnknownActionDetail, + expectErr: errUnknownStatusDetail, }, { name: "Invalid rule with unknown condition binary operator", @@ -116,6 +178,124 @@ func TestParseAPERule(t *testing.T) { rule: "deny Object.Put Object.ResourZe:Department=HR *", expectErr: errUnknownCondObjectType, }, + { + name: "Invalid rule with mixed types of actions", + rule: "allow Object.Put Container.Put *", + expectErr: errMixedTypesInRule, + }, + { + name: "Invalid rule with no actions", + rule: "allow Container.Resource:A=B *", + expectErr: errNoActionsInRule, + }, + { + name: "Invalid rule with invalid resource for object nm/cnt/obj/err", + rule: "allow Object.Put nm/cnt/obj/err", + expectErr: errUnsupportedResourceFormat, + }, + { + name: "Invalid rule with invalid resource for container nm/cnt/err", + rule: "allow Container.Put nm/cnt/err", + expectErr: errUnsupportedResourceFormat, + }, + { + name: "Invalid rule with invalid resource for container /nm/cnt/err", + rule: "allow Container.Put /nm/cnt/err", + expectErr: errUnsupportedResourceFormat, + }, + { + name: "Invalid rule with invalid resource for container /nm/cnt/", + rule: "allow Container.Put /nm/cnt/", + expectErr: errUnsupportedResourceFormat, + }, + { + name: "Invalid rule with invalid resource for container /nm/cnt", + rule: "allow Container.Put /nm/cnt", + expectErr: errUnsupportedResourceFormat, + }, + { + name: "Invalid rule with invalid resource for container /nm/", + rule: "allow Container.Put /nm/", + expectErr: errUnsupportedResourceFormat, + }, + { + name: "Valid rule for all containers", + rule: "allow Container.Put *", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutContainer}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatAllContainers}}, + }, + }, + { + name: "Valid rule for all containers in root namespace", + rule: "allow Container.Put /*", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutContainer}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootContainers}}, + }, + }, + { + name: "Valid rule for container in root namespace", + rule: "allow Container.Put /cid", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutContainer}}, + Resources: policyengine.Resources{Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, "cid"), + }}, + }, + }, + { + name: "Valid rule for all container in namespace", + rule: "allow Container.Put ns/*", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutContainer}}, + Resources: policyengine.Resources{Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainers, "ns"), + }}, + }, + }, + { + name: "Valid rule for container in namespace", + rule: "allow Container.Put ns/cid", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutContainer}}, + Resources: policyengine.Resources{Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainer, "ns", "cid"), + }}, + }, + }, + { + name: "Valid rule for container with conditions with action detail", + rule: "allow Container.Get Container.Resource:A=B Container.Put Container.Request:C!=D " + + "* /cnt_id", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodGetContainer, nativeschema.MethodPutContainer}}, + Resources: policyengine.Resources{Names: []string{ + nativeschema.ResourceFormatAllContainers, + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, "cnt_id"), + }}, + Condition: []policyengine.Condition{ + { + Op: policyengine.CondStringEquals, + Object: policyengine.ContainerResource, + Key: "A", + Value: "B", + }, + { + Op: policyengine.CondStringNotEquals, + Object: policyengine.ContainerRequest, + Key: "C", + Value: "D", + }, + }, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { From 417f8fc2c24cbe601a73c13c28d4c83ce211bc31 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 22 Jan 2024 13:41:26 +0300 Subject: [PATCH 0821/1943] [#876] cli: Add doc for commands `control *-rule` Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/docs/policy.md | 115 +++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 cmd/frostfs-cli/docs/policy.md diff --git a/cmd/frostfs-cli/docs/policy.md b/cmd/frostfs-cli/docs/policy.md new file mode 100644 index 000000000..06c325c62 --- /dev/null +++ b/cmd/frostfs-cli/docs/policy.md @@ -0,0 +1,115 @@ +# How manage local Access Policy Engine (APE) override of the node + +## Overview +APE is a replacement for eACL. Each rule can restrict somehow access to the object/container or list of them. +Here is a simple representation for the rule: +`[:status_detail] ... ... ...` + +Rule start with `status`(with or without details), contains list of actions(which this rule regulate) or conditions +(which can be under resource or request) and ends with list of resources. + +Resource is the combination of namespace, identificator of the FrostFS container/object and wildcard `*`. + +For object it can be represented as: +- `namespace/cid/oid` object in the container of the namespace +- `namespace/cid/*` all objects in the container of the namespace +- `namespace/*` all objects in the namespace +- `*` all objects +- `/*` all object in the `root` namespace +- `/cid/*` all objects in the container of the `root` namespace +- `/cid/oid` object in the container of the `root` namespace + +For container it can be represented as: +- `namespace/cid` container in the namespace +- `namespace/*` all containers in the namespace +- `*` all containers +- `/cid` container in the `root` namespace +- `/*` all containers in the `root` namespace + +Actions is a regular operations upon FrostFS containers/objects. Like `Object.Put`, `Container.Get` etc. + +In status section it is possible to use `allow`, `deny` or `deny:QuotaLimitReached` actions. + +It is prohibited to mix operation under FrostFS container and object in one rule. +The same statement is equal for conditions and resources - one rule is for one type of items. + +## Add rule +Local rule can be added with the command `frostfs-cli control add-rule`: +```shell +@:~$ frostfs-cli control add-rule --endpoint s04.frostfs.devenv:8081 -c cnt_create_cfg.yml \ +--address NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM --cid SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH \ +--chain-id TestPolicy \ +--rule "allow Object.Get Object.Head /*" --rule "deny Container.Put *" +Parsed chain: +Chain ID: TestPolicy + HEX: 54657374506f6c696379 +Rules: + + Status: Allowed + Any: false + Conditions: + Actions: Inverted:false + GetObject + HeadObject + Resources: Inverted:false + native:object//* + + Status: Access denied + Any: false + Conditions: + Actions: Inverted:false + PutContainer + Resources: Inverted:false + native:container/* + +Rule has been added. +@:~$ +``` +## List rules +Local rules can be listed with command `frostfs-cli control list-rules`: +```shell +@:~$ frostfs-cli control list-rules --endpoint s04.frostfs.devenv:8081 --address NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM \ +--cid SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH -w wallets/wallet.json +Enter password > +Chain ID: TestPolicy + HEX: 54657374506f6c696379 +Rules: + + Status: Allowed + Any: false +... +@:~$ +``` + +## Get rule +Rules can be retrieved with `frostfs-cli control get-rule`: +```shell +@:~$ frostfs-cli control get-rule --endpoint s04.frostfs.devenv:8081 -c cnt_create_cfg.yml \ +--address NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM --cid SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH \ +--chain-id TestPolicy +Parsed chain (chain id hex: '54657374506f6c696379'): +Chain ID: TestPolicy + HEX: 54657374506f6c696379 +Rules: + + Status: Allowed + Any: false +... +@:~$ +``` + +## Remove rule +To remove rule need to use command `frostfs-cli control remove-rule`: +```shell +@:~$ frostfs-cli control remove-rule --endpoint s04.frostfs.devenv:8081 -c cnt_create_cfg.yml \ +--address NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM --cid SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH --chain-id TestPolicy +Rule has been removed. +@:~$ frostfs-cli control get-rule --endpoint s04.frostfs.devenv:8081 -c cnt_create_cfg.yml \ +--address NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM --cid SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH --chain-id TestPolicy +rpc error: rpc error: code = NotFound desc = chain not found +@:~$ frostfs-cli control list-rules --endpoint s04.frostfs.devenv:8081 \ +--address NbUgTSFvPmsRxmGeWpuuGeJUoRoi6PErcM --cid SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH -w wallets/wallet.json +Enter password > +Local overrides are not defined for the container. +@:~$ +``` From 6e2cc327688e8faaed855bf7d14d93d13bbb1748 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 24 Jan 2024 13:56:12 +0300 Subject: [PATCH 0822/1943] [#681] objsvc: Validate session token owner for local sessions Previously, the check was in place only when session token was missing. Format validator checks are applied only to fully-prepared object, so this lead to the following situation: 1. Object is put locally with malformed token, because there are no checks. 2. Object cannot be replicated, because the token is malformed. This is now fixed and token check is done before any payload receival. Signed-off-by: Evgenii Stratonikov --- pkg/services/object/put/streamer.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index a9abe407d..90d580a1c 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -110,18 +110,22 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { // In case session token is missing, the line above returns the default key. // If it isn't owner key, replication attempts will fail, thus this check. - if sToken == nil { - ownerObj := prm.hdr.OwnerID() - if ownerObj.IsEmpty() { - return errors.New("missing object owner") - } + ownerObj := prm.hdr.OwnerID() + if ownerObj.IsEmpty() { + return errors.New("missing object owner") + } + if sToken == nil { var ownerSession user.ID user.IDFromKey(&ownerSession, sessionKey.PublicKey) if !ownerObj.Equals(ownerSession) { return fmt.Errorf("(%T) session token is missing but object owner id is different from the default key", p) } + } else { + if !ownerObj.Equals(sessionInfo.Owner) { + return fmt.Errorf("(%T) different token issuer and object owner identifiers %s/%s", p, sessionInfo.Owner, ownerObj) + } } p.sessionKey = sessionKey From c916a75948d0795afc38676dc8d506177a31476b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 26 Jan 2024 12:18:51 +0300 Subject: [PATCH 0823/1943] [#935] .forgejo: Update dco-go to v3 Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/dco.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 6746408aa..9aa0d3106 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -16,6 +16,6 @@ jobs: go-version: '1.21' - name: Run commit format checker - uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2 + uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 with: from: 'origin/${{ github.event.pull_request.base.ref }}' From df055fead56e6acef84f34da61bdca0c5b321f28 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 25 Jan 2024 11:21:00 +0300 Subject: [PATCH 0824/1943] [#931] morph: Provide batch size for container listing explicitly Besides VM stack item limit we also have restrictions on the size of JSON for a stackitem, which is 128k. This limit is much harder to calculate, because JSON representation includes type and the encoding is different for different items. Thus is makes no sense to invent our own default, so use the one provided by neo-go. But for container listing we know exactly what we process, so use big enough value, which is tested. Introduced in be8607a1f6. Refs #902 Refs https://github.com/nspcc-dev/neo-go/blob/v0.105.0/pkg/vm/stackitem/json.go#L353 Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 11 +++-------- pkg/morph/client/container/containers_of.go | 11 ++++++++++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 0c8544bd0..88f9703e7 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -22,13 +22,13 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/gas" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/rpcclient/rolemgmt" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" @@ -207,16 +207,11 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, return vub, nil } -// defaultPrefetchBatchSize is the default number of items to prefetch. -// It is dependent on VM limits (2048 items on stack), the default works for simple items. -// For example, to iterate over 2 field structs, the limit should be divided by 3 = 1 (struct itself) + 2 (fields). -const defaultPrefetchBatchSize = vm.MaxStackSize - 16 - // TestInvokeIterator invokes contract method returning an iterator and executes cb on each element. // If cb returns an error, the session is closed and this error is returned as-is. // If the remove neo-go node does not support sessions, `unwrap.ErrNoSessionID` is returned. // batchSize is the number of items to prefetch: if the number of items in the iterator is less than batchSize, no session will be created. -// The default batchSize is 2000 (VM is limited by having 2048 items on stack, so if each iterator item is simple, 2000 items won't hit the limit). +// The default batchSize is 100, the default limit from neo-go. func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int, contract util.Uint160, method string, args ...interface{}) error { start := time.Now() success := false @@ -225,7 +220,7 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int }() if batchSize <= 0 { - batchSize = defaultPrefetchBatchSize + batchSize = invoker.DefaultIteratorResultItems } c.switchLock.RLock() diff --git a/pkg/morph/client/container/containers_of.go b/pkg/morph/client/container/containers_of.go index 140047eb2..ce127335d 100644 --- a/pkg/morph/client/container/containers_of.go +++ b/pkg/morph/client/container/containers_of.go @@ -40,8 +40,17 @@ func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) { return nil } + // We would like to have batch size as big as possible, + // to reduce the number of round-trips and avoid creating sessions. + // The limit depends on 2 things: + // 1. VM limits: max 2048 items on stack. + // 2. JSON encoded size for the item with type = 128k. + // It turns out, that for container ID the second limit is hit first, + // 512 is big enough value and it is beautiful. + const batchSize = 512 + cnrHash := c.client.ContractAddress() - err := c.client.Morph().TestInvokeIterator(cb, 0, cnrHash, containersOfMethod, rawID) + err := c.client.Morph().TestInvokeIterator(cb, batchSize, cnrHash, containersOfMethod, rawID) if err != nil { if errors.Is(err, unwrap.ErrNoSessionID) { return c.List(idUser) From c681354afd6332e9ecc35623d2f53d7ebbe5c6a8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 30 Jan 2024 11:13:29 +0300 Subject: [PATCH 0825/1943] [#938] engine: Fix container count removal Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/inhume.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index fc6ee2430..80eda2204 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -398,7 +398,7 @@ func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []ci } for id := range idMap { - if err := sh.DeleteContainerSize(ctx, id); err != nil { + if err := sh.DeleteContainerCount(ctx, id); err != nil { e.log.Warn(logs.EngineFailedToDeleteContainerSize, zap.Stringer("container_id", id), zap.Error(err)) failed = true return true From 675eec91f34bd689191b6eeb9a5bf4dbd466237e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 30 Jan 2024 11:37:06 +0300 Subject: [PATCH 0826/1943] [#938] shard: Update only changed counters If metric value hasn't changed, but we update metric, then non existed metric will apear with zero value. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/shard.go | 46 +++++++++++++++++-------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 136f5463a..e8f1978d2 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -433,9 +433,9 @@ func (s *Shard) updateMetrics(ctx context.Context) { return } - s.cfg.metricsWriter.SetObjectCounter(physical, cc.Phy) - s.cfg.metricsWriter.SetObjectCounter(logical, cc.Logic) - s.cfg.metricsWriter.SetObjectCounter(user, cc.User) + s.setObjectCounterBy(physical, cc.Phy) + s.setObjectCounterBy(logical, cc.Logic) + s.setObjectCounterBy(user, cc.User) cnrList, err := s.metaBase.Containers(ctx) if err != nil { @@ -453,11 +453,11 @@ func (s *Shard) updateMetrics(ctx context.Context) { zap.Error(err)) continue } - s.metricsWriter.AddToContainerSize(cnrList[i].EncodeToString(), int64(size)) + s.addToContainerSize(cnrList[i].EncodeToString(), int64(size)) totalPayload += size } - s.metricsWriter.AddToPayloadSize(int64(totalPayload)) + s.addToPayloadSize(int64(totalPayload)) contCount, err := s.metaBase.ContainerCounters(ctx) if err != nil { @@ -465,9 +465,9 @@ func (s *Shard) updateMetrics(ctx context.Context) { return } for contID, count := range contCount.Counts { - s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), physical, count.Phy) - s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), logical, count.Logic) - s.metricsWriter.SetContainerObjectsCount(contID.EncodeToString(), user, count.User) + s.setContainerObjectsCount(contID.EncodeToString(), physical, count.Phy) + s.setContainerObjectsCount(contID.EncodeToString(), logical, count.Logic) + s.setContainerObjectsCount(contID.EncodeToString(), user, count.User) } } @@ -487,35 +487,53 @@ func (s *Shard) incObjectCounter(cnrID cid.ID, isUser bool) { } func (s *Shard) decObjectCounterBy(typ string, v uint64) { - if s.cfg.metricsWriter != nil { + if s.cfg.metricsWriter != nil && v > 0 { s.cfg.metricsWriter.AddToObjectCounter(typ, -int(v)) } } +func (s *Shard) setObjectCounterBy(typ string, v uint64) { + if s.cfg.metricsWriter != nil && v > 0 { + s.cfg.metricsWriter.SetObjectCounter(typ, v) + } +} + func (s *Shard) decContainerObjectCounter(byCnr map[cid.ID]meta.ObjectCounters) { if s.cfg.metricsWriter == nil { return } for cnrID, count := range byCnr { - s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), physical, count.Phy) - s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), logical, count.Logic) - s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), user, count.User) + if count.Phy > 0 { + s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), physical, count.Phy) + } + if count.Logic > 0 { + s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), logical, count.Logic) + } + if count.User > 0 { + s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), user, count.User) + } } } func (s *Shard) addToContainerSize(cnr string, size int64) { - if s.cfg.metricsWriter != nil { + if s.cfg.metricsWriter != nil && size != 0 { s.cfg.metricsWriter.AddToContainerSize(cnr, size) } } func (s *Shard) addToPayloadSize(size int64) { - if s.cfg.metricsWriter != nil { + if s.cfg.metricsWriter != nil && size != 0 { s.cfg.metricsWriter.AddToPayloadSize(size) } } +func (s *Shard) setContainerObjectsCount(cnr string, typ string, v uint64) { + if s.cfg.metricsWriter != nil && v > 0 { + s.metricsWriter.SetContainerObjectsCount(cnr, typ, v) + } +} + func (s *Shard) IncErrorCounter() { if s.cfg.metricsWriter != nil { s.cfg.metricsWriter.IncErrorCounter() From c441296592cdaca7152122a2ef1d8f85b02f241c Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 24 Jan 2024 14:48:47 +0300 Subject: [PATCH 0827/1943] [#930] policer: Release task pool when context cancelled Signed-off-by: Anton Nikiforov --- pkg/services/policer/process.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 873a1efb0..c4c820200 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -19,6 +19,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { for { select { case <-ctx.Done(): + p.taskPool.Release() return default: } @@ -36,6 +37,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { for i := range addrs { select { case <-ctx.Done(): + p.taskPool.Release() return default: addr := addrs[i] From e3573de6db6cd677023715b51046f9cecdee7e08 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 25 Jan 2024 11:02:57 +0300 Subject: [PATCH 0828/1943] [#930] gc: Stop internal activity by context Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config.go | 2 +- internal/logs/logs.go | 4 +- pkg/local_object_storage/engine/lock_test.go | 4 +- pkg/local_object_storage/engine/shards.go | 8 +++- pkg/local_object_storage/shard/gc.go | 50 +++++++++++++++----- 5 files changed, 50 insertions(+), 18 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 1b3e094d9..e1e228325 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1009,7 +1009,7 @@ func initLocalStorage(ctx context.Context, c *cfg) { ls := engine.New(c.engineOpts()...) addNewEpochAsyncNotificationHandler(c, func(ev event.Event) { - ls.HandleNewEpoch(ev.(netmap2.NewEpoch).EpochNumber()) + ls.HandleNewEpoch(ctx, ev.(netmap2.NewEpoch).EpochNumber()) }) // allocate memory for the service; diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 34ca0ae93..e81976a32 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -270,7 +270,9 @@ const ( ShardCantOpenMetabaseMoveToADegradedMode = "can't open metabase, move to a degraded mode" ShardCantInitializeMetabaseMoveToADegradedreadonlyMode = "can't initialize metabase, move to a degraded-read-only mode" ShardTryingToRestoreReadwriteMode = "trying to restore read-write mode" - ShardStopEventListenerByClosedChannel = "stop event listener by closed channel" + ShardStopEventListenerByClosedEventChannel = "stop event listener by closed `event` channel" + ShardStopEventListenerByClosedStopChannel = "stop event listener by closed `stop` channel" + ShardStopEventListenerByContext = "stop event listener by context" ShardCouldNotSubmitGCJobToWorkerPool = "could not submit GC job to worker pool" ShardGCIsStopped = "GC is stopped" ShardWaitingForGCWorkersToStop = "waiting for GC workers to stop..." diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 69b74f4a4..7fa7c27ef 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -133,7 +133,7 @@ func TestLockUserScenario(t *testing.T) { require.ErrorIs(t, err, meta.ErrLockObjectRemoval) // 5. - e.HandleNewEpoch(lockerExpiresAfter + 1) + e.HandleNewEpoch(context.Background(), lockerExpiresAfter+1) inhumePrm.WithTarget(tombAddr, objAddr) @@ -206,7 +206,7 @@ func TestLockExpiration(t *testing.T) { require.ErrorAs(t, err, &objLockedErr) // 3. - e.HandleNewEpoch(lockerExpiresAfter + 1) + e.HandleNewEpoch(context.Background(), lockerExpiresAfter+1) // 4. inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj)) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 37a857c9d..bd25dde59 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -329,14 +329,18 @@ func (e *StorageEngine) SetShardMode(id *shard.ID, m mode.Mode, resetErrorCounte } // HandleNewEpoch notifies every shard about NewEpoch event. -func (e *StorageEngine) HandleNewEpoch(epoch uint64) { +func (e *StorageEngine) HandleNewEpoch(ctx context.Context, epoch uint64) { ev := shard.EventNewEpoch(epoch) e.mtx.RLock() defer e.mtx.RUnlock() for _, sh := range e.shards { - sh.NotificationChannel() <- ev + select { + case <-ctx.Done(): + return + case sh.NotificationChannel() <- ev: + } } } diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index bfd1f1b11..24c3a337a 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -105,6 +105,8 @@ type gc struct { remover func(context.Context) gcRunResult + // eventChan is used only for listening for the new epoch event. + // It is ok to keep opened, we are listening for context done when writing in it. eventChan chan Event mEventHandler map[eventType]*eventHandlers } @@ -155,13 +157,21 @@ func (gc *gc) listenEvents(ctx context.Context) { defer gc.wg.Done() for { - event, ok := <-gc.eventChan - if !ok { - gc.log.Warn(logs.ShardStopEventListenerByClosedChannel) + select { + case <-gc.stopChannel: + gc.log.Warn(logs.ShardStopEventListenerByClosedStopChannel) return - } + case <-ctx.Done(): + gc.log.Warn(logs.ShardStopEventListenerByContext) + return + case event, ok := <-gc.eventChan: + if !ok { + gc.log.Warn(logs.ShardStopEventListenerByClosedEventChannel) + return + } - gc.handleEvent(ctx, event) + gc.handleEvent(ctx, event) + } } } @@ -180,6 +190,11 @@ func (gc *gc) handleEvent(ctx context.Context, event Event) { v.prevGroup.Add(len(v.handlers)) for i := range v.handlers { + select { + case <-ctx.Done(): + return + default: + } h := v.handlers[i] err := gc.workerPool.Submit(func() { @@ -196,6 +211,18 @@ func (gc *gc) handleEvent(ctx context.Context, event Event) { } } +func (gc *gc) releaseResources() { + if gc.workerPool != nil { + gc.workerPool.Release() + } + + // Avoid to close gc.eventChan here, + // because it is possible that we are close it earlier than stop writing. + // It is ok to keep it opened. + + gc.log.Debug(logs.ShardGCIsStopped) +} + func (gc *gc) tickRemover(ctx context.Context) { defer gc.wg.Done() @@ -204,14 +231,13 @@ func (gc *gc) tickRemover(ctx context.Context) { for { select { + case <-ctx.Done(): + // Context canceled earlier than we start to close shards. + // It make sense to stop collecting garbage by context too. + gc.releaseResources() + return case <-gc.stopChannel: - if gc.workerPool != nil { - gc.workerPool.Release() - } - - close(gc.eventChan) - - gc.log.Debug(logs.ShardGCIsStopped) + gc.releaseResources() return case <-timer.C: startedAt := time.Now() From 483a67b17064990d69e39e23d7b3fd6c72d9bfb3 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 30 Jan 2024 17:07:05 +0300 Subject: [PATCH 0829/1943] [#937] ape: Validate chain resource name Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/frostfsid_util.go | 22 +-- .../modules/morph/frostfsid_util_test.go | 7 +- internal/ape/util.go | 11 ++ pkg/services/control/server/ape/validate.go | 97 +++++++++++++ .../control/server/ape/validate_test.go | 132 ++++++++++++++++++ pkg/services/control/server/policy_engine.go | 8 ++ 6 files changed, 259 insertions(+), 18 deletions(-) create mode 100644 internal/ape/util.go create mode 100644 pkg/services/control/server/ape/validate.go create mode 100644 pkg/services/control/server/ape/validate_test.go diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go index f88e4edfb..6b6bdffa9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go @@ -3,9 +3,9 @@ package morph import ( "errors" "fmt" - "regexp" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/ape" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/util" @@ -13,14 +13,6 @@ import ( "github.com/spf13/viper" ) -var ( - frostfsidSubjectNameRegexp = regexp.MustCompile(`^[\w+=,.@-]{1,64}$`) - frostfsidGroupNameRegexp = regexp.MustCompile(`^[\w+=,.@-]{1,128}$`) - - // frostfsidNamespaceNameRegexp similar to https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/commit/f2a82aa635aa57d9b05092d8cf15b170b53cc324/nns/nns_contract.go#L690 - frostfsidNamespaceNameRegexp = regexp.MustCompile(`(^$)|(^[a-z0-9]{1,2}$)|(^[a-z0-9][a-z0-9-]{1,48}[a-z0-9]$)`) -) - func getFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) { admin := v.GetString(frostfsIDAdminConfigKey) if admin == "" { @@ -65,9 +57,9 @@ func getFrostfsIDSubjectName(cmd *cobra.Command) string { return "" } - if !frostfsidSubjectNameRegexp.MatchString(subjectName) { + if !ape.SubjectNameRegexp.MatchString(subjectName) { commonCmd.ExitOnErr(cmd, "invalid subject name: %w", - fmt.Errorf("name must match regexp: %s", frostfsidSubjectNameRegexp.String())) + fmt.Errorf("name must match regexp: %s", ape.SubjectNameRegexp.String())) } return subjectName @@ -76,9 +68,9 @@ func getFrostfsIDSubjectName(cmd *cobra.Command) string { func getFrostfsIDGroupName(cmd *cobra.Command) string { groupName, _ := cmd.Flags().GetString(groupNameFlag) - if !frostfsidGroupNameRegexp.MatchString(groupName) { + if !ape.GroupNameRegexp.MatchString(groupName) { commonCmd.ExitOnErr(cmd, "invalid group name: %w", - fmt.Errorf("name must match regexp: %s", frostfsidGroupNameRegexp.String())) + fmt.Errorf("name must match regexp: %s", ape.GroupNameRegexp.String())) } return groupName @@ -100,9 +92,9 @@ func getFrostfsIDNamespace(cmd *cobra.Command) string { ns = "" } - if !frostfsidNamespaceNameRegexp.MatchString(ns) { + if !ape.NamespaceNameRegexp.MatchString(ns) { commonCmd.ExitOnErr(cmd, "invalid namespace: %w", - fmt.Errorf("name must match regexp: %s", frostfsidNamespaceNameRegexp.String())) + fmt.Errorf("name must match regexp: %s", ape.NamespaceNameRegexp.String())) } return ns diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go index f73ebad08..79a0845f9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/ape" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/spf13/viper" @@ -95,7 +96,7 @@ func TestNamespaceRegexp(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.matched, frostfsidNamespaceNameRegexp.MatchString(tc.namespace)) + require.Equal(t, tc.matched, ape.NamespaceNameRegexp.MatchString(tc.namespace)) }) } } @@ -128,7 +129,7 @@ func TestSubjectNameRegexp(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.matched, frostfsidSubjectNameRegexp.MatchString(tc.subject)) + require.Equal(t, tc.matched, ape.SubjectNameRegexp.MatchString(tc.subject)) }) } } @@ -166,7 +167,7 @@ func TestSubjectGroupRegexp(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.matched, frostfsidGroupNameRegexp.MatchString(tc.subject)) + require.Equal(t, tc.matched, ape.GroupNameRegexp.MatchString(tc.subject)) }) } } diff --git a/internal/ape/util.go b/internal/ape/util.go new file mode 100644 index 000000000..99eba95ba --- /dev/null +++ b/internal/ape/util.go @@ -0,0 +1,11 @@ +package ape + +import "regexp" + +var ( + SubjectNameRegexp = regexp.MustCompile(`^[\w+=,.@-]{1,64}$`) + GroupNameRegexp = regexp.MustCompile(`^[\w+=,.@-]{1,128}$`) + + // NamespaceNameRegexp similar to https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/commit/f2a82aa635aa57d9b05092d8cf15b170b53cc324/nns/nns_contract.go#L690 + NamespaceNameRegexp = regexp.MustCompile(`(^$)|(^[a-z0-9]{1,2}$)|(^[a-z0-9][a-z0-9-]{1,48}[a-z0-9]$)`) +) diff --git a/pkg/services/control/server/ape/validate.go b/pkg/services/control/server/ape/validate.go new file mode 100644 index 000000000..f4aa0399f --- /dev/null +++ b/pkg/services/control/server/ape/validate.go @@ -0,0 +1,97 @@ +package ape + +import ( + "errors" + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/ape" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" +) + +var ( + ErrInvalidResource = errors.New("invalid resource name") + ErrUnsupportedPrefix = errors.New("unsupported resource name prefix") + ErrInvalidContainerID = errors.New("invalid container id") + ErrInvalidObjectID = errors.New("invalid object id") + ErrInvalidNamespace = fmt.Errorf("namespace must match regexp: %s", ape.NamespaceNameRegexp.String()) +) + +// ValidateResourceName validates resource name components - container and object id, namespace. +// Also validates matching resource name to templates of policy engine's native scheme. +func ValidateResourceName(name string) error { + if after, found := strings.CutPrefix(name, native.ObjectPrefix+"/"); found { + return validateObjectResourceName(after) + } else if after, found = strings.CutPrefix(name, native.ContainerPrefix+"/"); found { + return validateContainerResourceName(after) + } + return ErrUnsupportedPrefix +} + +// validateObjectResourceName validate name for object. +// Name should be without prefix `native.ObjectPrefix`. +func validateObjectResourceName(name string) error { + if name == "*" { + return nil + } + lexems := strings.Split(name, "/") + if len(lexems) == 1 && lexems[0] == "*" { + return nil + } else if len(lexems) == 2 { + // len == 2 means format `namespace(root_namespace)/*` + if lexems[0] != "" && !ape.NamespaceNameRegexp.MatchString(lexems[0]) { + return ErrInvalidNamespace + } + if lexems[1] == "*" { + return nil + } + } else if len(lexems) == 3 { + // len == 3 means format `namespace(root_namespace)/CID/OID(*)` + if lexems[0] != "" && !ape.NamespaceNameRegexp.MatchString(lexems[0]) { + return ErrInvalidNamespace + } + var cnr cid.ID + err := cnr.DecodeString(lexems[1]) + if err != nil { + return fmt.Errorf("%w: %w", ErrInvalidContainerID, err) + } + if lexems[2] == "*" { + return nil + } + var objID oid.ID + err = objID.DecodeString(lexems[2]) + if err != nil { + return fmt.Errorf("%w: %w", ErrInvalidObjectID, err) + } + return nil + } + return ErrInvalidResource +} + +// validateContainerResourceName validate resource name for container. +// Name should be without prefix `native.ContainerPrefix`. +func validateContainerResourceName(name string) error { + if name == "*" { + return nil + } + lexems := strings.Split(name, "/") + if len(lexems) == 1 && lexems[0] == "*" { + return nil + } else if len(lexems) == 2 { + // len == 2 means format `namespace(root_namespace)/CID(*)` + if lexems[0] != "" && !ape.NamespaceNameRegexp.MatchString(lexems[0]) { + return ErrInvalidNamespace + } + if lexems[1] != "*" { + var cnr cid.ID + err := cnr.DecodeString(lexems[1]) + if err != nil { + return fmt.Errorf("%w: %w", ErrInvalidContainerID, err) + } + } + return nil + } + return ErrInvalidResource +} diff --git a/pkg/services/control/server/ape/validate_test.go b/pkg/services/control/server/ape/validate_test.go new file mode 100644 index 000000000..af811efed --- /dev/null +++ b/pkg/services/control/server/ape/validate_test.go @@ -0,0 +1,132 @@ +package ape + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/stretchr/testify/require" +) + +func TestValidationOfChainResources(t *testing.T) { + tests := [...]struct { + testName string + resourceName string + expectErr error + }{ + { + testName: "native object: all objects", + resourceName: native.ObjectPrefix + "/*", + }, + { + testName: "native object: all objects in namespace", + resourceName: native.ObjectPrefix + "/ns/*", + }, + { + testName: "native object: all objects in root namespace", + resourceName: native.ObjectPrefix + "//*", + }, + { + testName: "native object: all objects in namespace/container", + resourceName: native.ObjectPrefix + "/ns/SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH/*", + }, + { + testName: "native object: all objects in root namespace/container", + resourceName: native.ObjectPrefix + "//SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH/*", + }, + { + testName: "native object: object in namespace/container", + resourceName: native.ObjectPrefix + "/ns/SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH/BCGsUu6o92oG1UALVox1sV6YbBUKUL2xSCtAFkrsuvWY", + }, + { + testName: "native object: object in root namespace/container", + resourceName: native.ObjectPrefix + "//SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH/BCGsUu6o92oG1UALVox1sV6YbBUKUL2xSCtAFkrsuvWY", + }, + { + testName: "native object: invalid all objects", + resourceName: native.ObjectPrefix + "/*12313", + expectErr: ErrInvalidResource, + }, + { + testName: "native object: all objects in invalid namespace", + resourceName: native.ObjectPrefix + "/qwe_123123/*", + expectErr: ErrInvalidNamespace, + }, + { + testName: "native object: invalid all objects in root namespace", + resourceName: native.ObjectPrefix + "//qwe", + expectErr: ErrInvalidResource, + }, + { + testName: "native object: invalid cid in all objects in root namespace", + resourceName: native.ObjectPrefix + "//SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytHqwe/*", + expectErr: ErrInvalidContainerID, + }, + { + testName: "native object: invalid cid in all objects in namespace", + resourceName: native.ObjectPrefix + "/ns/SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytHqwe/*", + expectErr: ErrInvalidContainerID, + }, + { + testName: "native object: invalid object in namespace/container", + resourceName: native.ObjectPrefix + "/ns/SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH/BCGsUu6o92oG1UALVox1sV6YbBUKUL2xSCtAFkrsuvWY111", + expectErr: ErrInvalidObjectID, + }, + { + testName: "native object: invalid resource", + resourceName: native.ObjectPrefix + "/ns/SeHNpifD/AFkrsuvWY111/AFkrsuvWY222", + expectErr: ErrInvalidResource, + }, + { + testName: "native container: all containers", + resourceName: native.ContainerPrefix + "/*", + }, + { + testName: "native container: all containers in namespace", + resourceName: native.ContainerPrefix + "/ns/*", + }, + { + testName: "native container: all containers in root namespace", + resourceName: native.ContainerPrefix + "//*", + }, + { + testName: "native container: container in namespace", + resourceName: native.ContainerPrefix + "/ns/SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH", + }, + { + testName: "native container: container in root namespace", + resourceName: native.ContainerPrefix + "//SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH", + }, + { + testName: "native container: invalid all containers", + resourceName: native.ContainerPrefix + "/*asd", + expectErr: ErrInvalidResource, + }, + { + testName: "native container: invalid resource", + resourceName: native.ContainerPrefix + "/ns/cid/cid", + expectErr: ErrInvalidResource, + }, + { + testName: "native container: invalid container in root namespace", + resourceName: native.ContainerPrefix + "//*asd", + expectErr: ErrInvalidContainerID, + }, + { + testName: "native container: container in invalid namespace", + resourceName: native.ContainerPrefix + "/ns_111/SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH", + expectErr: ErrInvalidNamespace, + }, + { + testName: "unsupported prefix", + resourceName: "native:test/ns_111/SeHNpifDH2Fc4scNBphrbmrKi96QXj2HzYJkhSGuytH", + expectErr: ErrUnsupportedPrefix, + }, + } + + for _, test := range tests { + t.Run(test.testName, func(t *testing.T) { + err := ValidateResourceName(test.resourceName) + require.ErrorIs(t, err, test.expectErr) + }) + } +} diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 9b42f3a98..bc85c4356 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -6,6 +6,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ape" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "google.golang.org/grpc/codes" @@ -38,6 +39,13 @@ func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainL if err := chain.DecodeBytes(req.GetBody().GetChain()); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } + for _, rule := range chain.Rules { + for _, name := range rule.Resources.Names { + if err := ape.ValidateResourceName(name); err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Errorf("invalid resource: %w", err).Error()) + } + } + } s.apeChainCounter.Add(1) // TODO (aarifullin): the such chain id is not well-designed yet. From 4352bd0e8eab50c4e51128ee0415e9d7324664b3 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 26 Jan 2024 18:45:10 +0300 Subject: [PATCH 0830/1943] [#934] ape: Transform empty namespace within chainbase Signed-off-by: Airat Arifullin --- pkg/ape/chainbase/boltdb.go | 8 ++++++++ pkg/services/control/server/policy_engine.go | 7 +------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/ape/chainbase/boltdb.go b/pkg/ape/chainbase/boltdb.go index 1047ad42d..4e15050b3 100644 --- a/pkg/ape/chainbase/boltdb.go +++ b/pkg/ape/chainbase/boltdb.go @@ -115,6 +115,10 @@ func getTargetBucket(tx *bbolt.Tx, name chain.Name, target policyengine.Target) return nil, fmt.Errorf("%w: %w: %c", policyengine.ErrChainNotFound, ErrTargetTypeBucketNotFound, target.Type) } + if target.Type == policyengine.Namespace && target.Name == "" { + target.Name = "root" + } + rbucket := typeBucket.Bucket([]byte(target.Name)) if rbucket == nil { return nil, fmt.Errorf("%w: %w: %s", policyengine.ErrChainNotFound, ErrTargetNameBucketNotFound, target.Name) @@ -146,6 +150,10 @@ func getTargetBucketCreateIfEmpty(tx *bbolt.Tx, name chain.Name, target policyen } } + if target.Type == policyengine.Namespace && target.Name == "" { + target.Name = "root" + } + rbucket := typeBucket.Bucket([]byte(target.Name)) if rbucket == nil { var err error diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index bc85c4356..6bf600748 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -18,12 +18,7 @@ func apeTarget(chainTarget *control.ChainTarget) (engine.Target, error) { case control.ChainTarget_CONTAINER: return engine.ContainerTarget(chainTarget.GetName()), nil case control.ChainTarget_NAMESPACE: - namespace := chainTarget.GetName() - // If namespace is empty, we take it for root namespace. - if namespace == "" { - namespace = "root" - } - return engine.NamespaceTarget(namespace), nil + return engine.NamespaceTarget(chainTarget.GetName()), nil default: } return engine.Target{}, status.Error(codes.InvalidArgument, From 96c86c4637a0e7de45c2e7266c90120ef7650584 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 26 Jan 2024 18:57:47 +0300 Subject: [PATCH 0831/1943] [#934] adm: Make frostfsid commands read alphabet wallets Signed-off-by: Airat Arifullin --- cmd/frostfs-adm/internal/modules/morph/frostfsid.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go index 2a6043b7c..50cd47744 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -151,11 +151,13 @@ func initFrostfsIDCreateNamespaceCmd() { frostfsidCmd.AddCommand(frostfsidCreateNamespaceCmd) frostfsidCreateNamespaceCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) frostfsidCreateNamespaceCmd.Flags().String(namespaceFlag, "", "Namespace name to create") + frostfsidCreateNamespaceCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDListNamespacesCmd() { frostfsidCmd.AddCommand(frostfsidListNamespacesCmd) frostfsidListNamespacesCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidListNamespacesCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDCreateSubjectCmd() { @@ -164,12 +166,14 @@ func initFrostfsIDCreateSubjectCmd() { frostfsidCreateSubjectCmd.Flags().String(namespaceFlag, "", "Namespace where create subject") frostfsidCreateSubjectCmd.Flags().String(subjectNameFlag, "", "Subject name, must be unique in namespace") frostfsidCreateSubjectCmd.Flags().String(subjectKeyFlag, "", "Subject hex-encoded public key") + frostfsidCreateSubjectCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDDeleteSubjectCmd() { frostfsidCmd.AddCommand(frostfsidDeleteSubjectCmd) frostfsidDeleteSubjectCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) frostfsidDeleteSubjectCmd.Flags().String(subjectAddressFlag, "", "Subject address") + frostfsidDeleteSubjectCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDListSubjectsCmd() { @@ -177,6 +181,7 @@ func initFrostfsIDListSubjectsCmd() { frostfsidListSubjectsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) frostfsidListSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace to list subjects") frostfsidListSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") + frostfsidListSubjectsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDCreateGroupCmd() { @@ -184,6 +189,7 @@ func initFrostfsIDCreateGroupCmd() { frostfsidCreateGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) frostfsidCreateGroupCmd.Flags().String(namespaceFlag, "", "Namespace where create group") frostfsidCreateGroupCmd.Flags().String(groupNameFlag, "", "Group name, must be unique in namespace") + frostfsidCreateGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDDeleteGroupCmd() { @@ -191,12 +197,14 @@ func initFrostfsIDDeleteGroupCmd() { frostfsidDeleteGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) frostfsidDeleteGroupCmd.Flags().String(namespaceFlag, "", "Namespace to delete group") frostfsidDeleteGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") + frostfsidDeleteGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDListGroupsCmd() { frostfsidCmd.AddCommand(frostfsidListGroupsCmd) frostfsidListGroupsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) frostfsidListGroupsCmd.Flags().String(namespaceFlag, "", "Namespace to list groups") + frostfsidListGroupsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDAddSubjectToGroupCmd() { @@ -204,6 +212,7 @@ func initFrostfsIDAddSubjectToGroupCmd() { frostfsidAddSubjectToGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) frostfsidAddSubjectToGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidAddSubjectToGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") + frostfsidAddSubjectToGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDRemoveSubjectFromGroupCmd() { @@ -211,6 +220,7 @@ func initFrostfsIDRemoveSubjectFromGroupCmd() { frostfsidRemoveSubjectFromGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) frostfsidRemoveSubjectFromGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidRemoveSubjectFromGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") + frostfsidRemoveSubjectFromGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDListGroupSubjectsCmd() { @@ -219,6 +229,7 @@ func initFrostfsIDListGroupSubjectsCmd() { frostfsidListGroupSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace name") frostfsidListGroupSubjectsCmd.Flags().Int64(groupIDFlag, 0, "Group id") frostfsidListGroupSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") + frostfsidListGroupSubjectsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func frostfsidCreateNamespace(cmd *cobra.Command, _ []string) { From 5be2af881abb92c20723d9f1cc5b34f5c0770cad Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 30 Jan 2024 00:03:18 +0300 Subject: [PATCH 0832/1943] [#934] container: Make container APE middleware read namespaces * Those methods that can access already existing containers and thus can get container properties should read namespace from Zone property. If Zone is not set, take a namespace for root. * Otherwise, define namespaces by owner ID via frostfs-id contract. * Improve unit-tests, consider more cases. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/config.go | 13 + cmd/frostfs-node/config/contracts/config.go | 4 + cmd/frostfs-node/container.go | 6 +- cmd/frostfs-node/morph.go | 1 + go.mod | 2 +- go.sum | 4 +- pkg/morph/client/frostfsid/subject.go | 17 +- pkg/services/container/ape.go | 138 +++- pkg/services/container/ape_test.go | 690 +++++++++++++++++++- pkg/services/object/acl/v2/service.go | 9 +- 10 files changed, 832 insertions(+), 52 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e1e228325..29582c782 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -449,6 +449,7 @@ type cfg struct { cfgMorph cfgMorph cfgAccounting cfgAccounting cfgContainer cfgContainer + cfgFrostfsID cfgFrostfsID cfgNodeInfo cfgNodeInfo cfgNetmap cfgNetmap cfgControlService cfgControlService @@ -569,6 +570,10 @@ type cfgContainer struct { workerPool util.WorkerPool // pool for asynchronous handlers } +type cfgFrostfsID struct { + scriptHash neogoutil.Uint160 +} + type cfgNetmap struct { scriptHash neogoutil.Uint160 wrapper *nmClient.Client @@ -681,6 +686,8 @@ func initCfg(appCfg *config.Config) *cfg { } c.cfgContainer = initContainer(appCfg) + c.cfgFrostfsID = initFrostfsID(appCfg) + c.cfgNetmap = initNetmap(appCfg, netState, relayOnly) c.cfgGRPC = initCfgGRPC() @@ -779,6 +786,12 @@ func initContainer(appCfg *config.Config) cfgContainer { } } +func initFrostfsID(appCfg *config.Config) cfgFrostfsID { + return cfgFrostfsID{ + scriptHash: contractsconfig.FrostfsID(appCfg), + } +} + func initCfgGRPC() cfgGRPC { maxChunkSize := uint64(maxMsgSize) * 3 / 4 // 25% to meta, 75% to payload maxAddrAmount := uint64(maxChunkSize) / addressSize // each address is about 72 bytes diff --git a/cmd/frostfs-node/config/contracts/config.go b/cmd/frostfs-node/config/contracts/config.go index c5f14f3ca..df0c0b958 100644 --- a/cmd/frostfs-node/config/contracts/config.go +++ b/cmd/frostfs-node/config/contracts/config.go @@ -38,6 +38,10 @@ func Container(c *config.Config) util.Uint160 { return contractAddress(c, "container") } +func FrostfsID(c *config.Config) util.Uint160 { + return contractAddress(c, "frostfsid") +} + // Proxy returnsthe value of "proxy" config parameter // from "contracts" section. // diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 898a4eef9..28f271075 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" containerTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/container/grpc" @@ -32,11 +33,14 @@ func initContainerService(_ context.Context, c *cfg) { cnrRdr, cnrWrt := configureEACLAndContainerSources(c, wrap, cnrSrc) + frostFSIDClient, err := frostfsid.NewFromMorph(c.cfgMorph.client, c.cfgFrostfsID.scriptHash, 0) + fatalOnErr(err) + server := containerTransportGRPC.New( containerService.NewSignService( &c.key.PrivateKey, containerService.NewAPEServer(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine, cnrRdr, - newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, + newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, frostFSIDClient, containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), ), ), diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index d26142370..698fb3b83 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -288,6 +288,7 @@ func lookupScriptHashesInNNS(c *cfg) { {&c.cfgNetmap.scriptHash, client.NNSNetmapContractName}, {&c.cfgAccounting.scriptHash, client.NNSBalanceContractName}, {&c.cfgContainer.scriptHash, client.NNSContainerContractName}, + {&c.cfgFrostfsID.scriptHash, client.NNSFrostFSIDContractName}, {&c.cfgMorph.proxyScriptHash, client.NNSProxyContractName}, {&c.cfgObject.cfgAccessPolicyEngine.policyContractHash, client.NNSPolicyContractName}, } diff --git a/go.mod b/go.mod index 3b269db34..a5cffc28c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240112150928-72885aae835c git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240117145620-110b7e41706e + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240126141009-65b4525b3bf0 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240122104724-06cbfe8691ad git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index ca28fdaec..4462c3575 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240117145620-110b7e41706e h1:0t+3iEb2wFJjJryqGnh5wkr9DEiwRNb61KaxC27U+po= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240117145620-110b7e41706e/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240126141009-65b4525b3bf0 h1:jp/Cg6CF3ip0IVRCjt48qKyv1cbMUmNYTn2z1jFLWqc= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240126141009-65b4525b3bf0/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240122104724-06cbfe8691ad h1:+D96Uu0Pw+55CrxP+srazthtZAh0Q19BtJo1Pm4hOP0= diff --git a/pkg/morph/client/frostfsid/subject.go b/pkg/morph/client/frostfsid/subject.go index b14675d58..52ae98ba5 100644 --- a/pkg/morph/client/frostfsid/subject.go +++ b/pkg/morph/client/frostfsid/subject.go @@ -33,16 +33,17 @@ func (c *Client) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) return subj, nil } -// parseSubject from https://git.frostfs.info/TrueCloudLab/frostfs-contract/src/commit/dd5919348da9731f24504e7bc485516c2ba5f11c/frostfsid/client/client.go#L592 -func parseSubject(structArr []stackitem.Item) (*frostfsidclient.Subject, error) { - if len(structArr) < 5 { - return nil, errors.New("invalid response subject struct") +func parseSubject(res []stackitem.Item) (*frostfsidclient.Subject, error) { + if ln := len(res); ln != 1 { + return nil, fmt.Errorf("unexpected stack item count (%s): %d", methodGetSubject, ln) } - var ( - err error - subj frostfsidclient.Subject - ) + structArr, err := client.ArrayFromStackItem(res[0]) + if err != nil { + return nil, fmt.Errorf("could not get item array of container (%s): %w", methodGetSubject, err) + } + + var subj frostfsidclient.Subject subj.PrimaryKey, err = unwrap.PublicKey(makeValidRes(structArr[0])) if err != nil { diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 36edf4516..c57e54d08 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -8,14 +8,17 @@ import ( "crypto/sha256" "errors" "fmt" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" @@ -25,6 +28,7 @@ import ( "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" ) var ( @@ -35,6 +39,7 @@ var ( errInvalidSessionTokenOwner = errors.New("malformed request: invalid session token owner") errEmptyBodySignature = errors.New("malformed request: empty body signature") errMissingOwnerID = errors.New("malformed request: missing owner ID") + errSubjectNotFound = errors.New("subject not found") undefinedContainerID = cid.ID{} ) @@ -47,22 +52,29 @@ type containers interface { Get(cid.ID) (*containercore.Container, error) } +type frostfsidSubjectProvider interface { + GetSubject(util.Uint160) (*client.Subject, error) +} + type apeChecker struct { router policyengine.ChainRouter reader containers ir ir nm netmap.Source + frostFSIDClient frostfsidSubjectProvider + next Server } -func NewAPEServer(router policyengine.ChainRouter, reader containers, ir ir, nm netmap.Source, srv Server) Server { +func NewAPEServer(router policyengine.ChainRouter, reader containers, ir ir, nm netmap.Source, frostFSIDClient frostfsidSubjectProvider, srv Server) Server { return &apeChecker{ - router: router, - reader: reader, - ir: ir, - next: srv, - nm: nm, + router: router, + reader: reader, + ir: ir, + next: srv, + nm: nm, + frostFSIDClient: frostFSIDClient, } } @@ -125,9 +137,17 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co nativeschema.PropertyKeyActorRole: role, } + namespace, err := ac.namespaceByOwner(req.GetBody().GetOwnerID()) + if err != nil { + return nil, fmt.Errorf("could not get owner namespace: %w", err) + } + if err := ac.validateNamespaceByPublicKey(pk, namespace); err != nil { + return nil, err + } + request := &apeRequest{ resource: &apeResource{ - name: nativeschema.ResourceFormatRootContainers, + name: resourceName(namespace, ""), props: make(map[string]string), }, op: nativeschema.MethodListContainers, @@ -135,7 +155,7 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co } s, found, err := ac.router.IsAllowed(apechain.Ingress, - policyengine.NewRequestTargetWithNamespace(""), + policyengine.NewRequestTargetWithNamespace(namespace), request) if err != nil { return nil, err @@ -162,9 +182,17 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont nativeschema.PropertyKeyActorRole: role, } + namespace, err := ac.namespaceByOwner(req.GetBody().GetContainer().GetOwnerID()) + if err != nil { + return nil, fmt.Errorf("get namespace error: %w", err) + } + if err = validateNamespace(req.GetBody().GetContainer(), namespace); err != nil { + return nil, err + } + request := &apeRequest{ resource: &apeResource{ - name: nativeschema.ResourceFormatRootContainers, + name: resourceName(namespace, ""), props: make(map[string]string), }, op: nativeschema.MethodPutContainer, @@ -172,7 +200,7 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont } s, found, err := ac.router.IsAllowed(apechain.Ingress, - policyengine.NewRequestTargetWithNamespace(""), + policyengine.NewRequestTargetWithNamespace(namespace), request) if err != nil { return nil, err @@ -251,9 +279,15 @@ func (ac *apeChecker) validateContainerBoundedOperation(containerID *refs.Contai return err } + namespace := "" + cntNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(cont.Value).Zone(), ".ns") + if hasNamespace { + namespace = cntNamespace + } + request := &apeRequest{ resource: &apeResource{ - name: fmt.Sprintf(nativeschema.ResourceFormatRootContainer, id.EncodeToString()), + name: resourceName(namespace, id.EncodeToString()), props: ac.getContainerProps(cont), }, op: op, @@ -261,7 +295,7 @@ func (ac *apeChecker) validateContainerBoundedOperation(containerID *refs.Contai } s, found, err := ac.router.IsAllowed(apechain.Ingress, - policyengine.NewRequestTargetWithContainer(id.EncodeToString()), + policyengine.NewRequestTarget(cntNamespace, id.EncodeToString()), request) if err != nil { return err @@ -326,6 +360,19 @@ func (r *apeResource) Property(key string) string { return r.props[key] } +func resourceName(namespace string, container string) string { + if namespace == "" && container == "" { + return nativeschema.ResourceFormatRootContainers + } + if namespace == "" && container != "" { + return fmt.Sprintf(nativeschema.ResourceFormatRootContainer, container) + } + if namespace != "" && container == "" { + return fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainers, namespace) + } + return fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainer, namespace, container) +} + func (ac *apeChecker) getContainerProps(c *containercore.Container) map[string]string { return map[string]string{ nativeschema.PropertyKeyContainerOwnerID: c.Value.Owner().EncodeToString(), @@ -514,3 +561,70 @@ func isContainerNode(nm *netmapSDK.NetMap, pk, binCnrID []byte, cont *containerc return false, nil } + +func (ac *apeChecker) namespaceByOwner(owner *refs.OwnerID) (string, error) { + var ownerSDK user.ID + if owner == nil { + return "", fmt.Errorf("owner id is not set") + } + if err := ownerSDK.ReadFromV2(*owner); err != nil { + return "", err + } + addr, err := ownerSDK.ScriptHash() + if err != nil { + return "", err + } + + namespace := "" + subject, err := ac.frostFSIDClient.GetSubject(addr) + if err == nil { + namespace = subject.Namespace + } else { + if !strings.Contains(err.Error(), errSubjectNotFound.Error()) { + return "", fmt.Errorf("get subject error: %w", err) + } + } + return namespace, nil +} + +// validateNamespace validates a namespace set in a container. +// If frostfs-id contract stores a namespace N1 for an owner ID and a container within a request +// is set with namespace N2 (via Zone() property), then N2 is invalid and the request is denied. +func validateNamespace(cnrV2 *container.Container, ownerIDNamespace string) error { + if cnrV2 == nil { + return nil + } + var cnr cnrSDK.Container + if err := cnr.ReadFromV2(*cnrV2); err != nil { + return err + } + cntNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(cnr).Zone(), ".ns") + if hasNamespace { + if cntNamespace != ownerIDNamespace { + if ownerIDNamespace == "" { + return fmt.Errorf("invalid domain zone: no namespace is expected") + } + return fmt.Errorf("invalid domain zone: expected namespace %s, but got %s", ownerIDNamespace, cntNamespace) + } + } else if ownerIDNamespace != "" { + return fmt.Errorf("invalid domain zone: expected namespace %s, but got invalid or empty", ownerIDNamespace) + } + return nil +} + +// validateNamespace validates if a namespace of a request actor equals to owner's namespace. +// An actor's namespace is calculated by a public key. +func (ac *apeChecker) validateNamespaceByPublicKey(pk *keys.PublicKey, ownerIDNamespace string) error { + var actor user.ID + user.IDFromKey(&actor, (ecdsa.PublicKey)(*pk)) + actorOwnerID := new(refs.OwnerID) + actor.WriteToV2(actorOwnerID) + actorNamespace, err := ac.namespaceByOwner(actorOwnerID) + if err != nil { + return fmt.Errorf("could not get actor namespace: %w", err) + } + if actorNamespace != ownerIDNamespace { + return fmt.Errorf("actor namespace %s differs from owner: %s", actorNamespace, ownerIDNamespace) + } + return nil +} diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index f4dfbe50f..5344d9f23 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -2,6 +2,7 @@ package container import ( "context" + "crypto/ecdsa" "errors" "fmt" "testing" @@ -9,10 +10,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" + "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" containertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/test" @@ -24,16 +27,25 @@ import ( "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/stretchr/testify/require" ) +const ( + testDomainName = "testdomainname" + testDomainZone = "testdomainname.ns" +) + func TestAPE(t *testing.T) { t.Parallel() t.Run("deny get container for others", testDenyGetContainerForOthers) t.Run("deny set container eACL for IR", testDenySetContainerEACLForIR) t.Run("deny get container eACL for IR with session token", testDenyGetContainerEACLForIRSessionToken) t.Run("deny put container for others with session token", testDenyPutContainerForOthersSessionToken) + t.Run("deny put container, read namespace from frostfsID", testDenyPutContainerReadNamespaceFromFrostfsID) + t.Run("deny put container with invlaid namespace", testDenyPutContainerInvalidNamespace) t.Run("deny list containers for owner with PK", testDenyListContainersForPK) + t.Run("deny list containers by namespace invalidation", testDenyListContainersValidationNamespaceError) } func testDenyGetContainerForOthers(t *testing.T) { @@ -49,7 +61,10 @@ func testDenyGetContainerForOthers(t *testing.T) { keys: [][]byte{}, } nm := &netmapStub{} - apeSrv := NewAPEServer(router, contRdr, ir, nm, srv) + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{}, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) contID := cidtest.ID() testContainer := containertest.Container() @@ -122,7 +137,10 @@ func testDenySetContainerEACLForIR(t *testing.T) { keys: [][]byte{}, } nm := &netmapStub{} - apeSrv := NewAPEServer(router, contRdr, ir, nm, srv) + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{}, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) contID := cidtest.ID() testContainer := containertest.Container() @@ -197,7 +215,10 @@ func testDenyGetContainerEACLForIRSessionToken(t *testing.T) { keys: [][]byte{}, } nm := &netmapStub{} - apeSrv := NewAPEServer(router, contRdr, ir, nm, srv) + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{}, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) contID := cidtest.ID() testContainer := containertest.Container() @@ -283,7 +304,10 @@ func testDenyPutContainerForOthersSessionToken(t *testing.T) { keys: [][]byte{}, } nm := &netmapStub{} - apeSrv := NewAPEServer(router, contRdr, ir, nm, srv) + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{}, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) testContainer := containertest.Container() @@ -317,26 +341,7 @@ func testDenyPutContainerForOthersSessionToken(t *testing.T) { }) require.NoError(t, err) - req := &container.PutRequest{} - req.SetBody(&container.PutRequestBody{}) - var reqCont container.Container - testContainer.WriteToV2(&reqCont) - req.GetBody().SetContainer(&reqCont) - - sessionPK, err := keys.NewPrivateKey() - require.NoError(t, err) - sToken := sessiontest.ContainerSigned() - sToken.ApplyOnlyTo(cid.ID{}) - require.NoError(t, sToken.Sign(sessionPK.PrivateKey)) - var sTokenV2 session.Token - sToken.WriteToV2(&sTokenV2) - metaHeader := new(session.RequestMetaHeader) - metaHeader.SetSessionToken(&sTokenV2) - req.SetMetaHeader(metaHeader) - - pk, err := keys.NewPrivateKey() - require.NoError(t, err) - require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + req := initPutRequest(t, testContainer) resp, err := apeSrv.Put(context.Background(), req) require.Nil(t, resp) @@ -344,6 +349,139 @@ func testDenyPutContainerForOthersSessionToken(t *testing.T) { require.ErrorAs(t, err, &errAccessDenied) } +func testDenyPutContainerReadNamespaceFromFrostfsID(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + + cnrID, testContainer := initTestContainer(t, true) + contRdr.c[cnrID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + + _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(testDomainName), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodPutContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainers, testDomainName), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOthers, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := initPutRequest(t, testContainer) + ownerScriptHash := initOwnerIDScriptHash(t, testContainer) + + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{ + ownerScriptHash: { + Namespace: testDomainName, + Name: testDomainName, + }, + }, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) + resp, err := apeSrv.Put(context.Background(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) +} + +func testDenyPutContainerInvalidNamespace(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + + cnrID, testContainer := initTestContainer(t, false) + var domain cnrSDK.Domain + domain.SetName("incorrect" + testDomainName) + domain.SetZone("incorrect" + testDomainZone) + cnrSDK.WriteDomain(&testContainer, domain) + contRdr.c[cnrID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + + _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(testDomainName), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodPutContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainers, testDomainName), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOthers, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := initPutRequest(t, testContainer) + ownerScriptHash := initOwnerIDScriptHash(t, testContainer) + + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{ + ownerScriptHash: { + Namespace: testDomainName, + Name: testDomainName, + }, + }, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) + resp, err := apeSrv.Put(context.Background(), req) + require.Nil(t, resp) + require.ErrorContains(t, err, "invalid domain zone") +} + func testDenyListContainersForPK(t *testing.T) { t.Parallel() srv := &srvStub{ @@ -357,7 +495,10 @@ func testDenyListContainersForPK(t *testing.T) { keys: [][]byte{}, } nm := &netmapStub{} - apeSrv := NewAPEServer(router, contRdr, ir, nm, srv) + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{}, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) nm.currentEpoch = 100 nm.netmaps = map[uint64]*netmap.NetMap{} @@ -409,6 +550,82 @@ func testDenyListContainersForPK(t *testing.T) { require.ErrorAs(t, err, &errAccessDenied) } +func testDenyListContainersValidationNamespaceError(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + + actorPK, err := keys.NewPrivateKey() + require.NoError(t, err) + + ownerPK, err := keys.NewPrivateKey() + require.NoError(t, err) + + actorScriptHash, ownerScriptHash := initActorOwnerScriptHashes(t, actorPK, ownerPK) + + const actorDomain = "actor" + testDomainName + + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{ + actorScriptHash: { + Namespace: actorDomain, + Name: actorDomain, + }, + ownerScriptHash: { + Namespace: testDomainName, + Name: testDomainName, + }, + }, + } + + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + + _, _, err = router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(""), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodListContainers, + }, + }, + Resources: chain.Resources{ + Names: []string{ + nativeschema.ResourceFormatRootContainers, + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorPublicKey, + Value: actorPK.PublicKey().String(), + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := initListRequest(t, actorPK, ownerPK) + + resp, err := apeSrv.List(context.Background(), req) + require.Nil(t, resp) + require.ErrorContains(t, err, "actor namespace "+actorDomain+" differs") +} + type srvStub struct { calls map[string]int } @@ -489,3 +706,424 @@ func (s *netmapStub) GetNetMapByEpoch(epoch uint64) (*netmap.NetMap, error) { func (s *netmapStub) Epoch() (uint64, error) { return s.currentEpoch, nil } + +type frostfsidStub struct { + subjects map[util.Uint160]*client.Subject +} + +func (f *frostfsidStub) GetSubject(owner util.Uint160) (*client.Subject, error) { + s, ok := f.subjects[owner] + if !ok { + return nil, errSubjectNotFound + } + return s, nil +} + +type testAPEServer struct { + engine engine.Engine + + containerReader *containerStub + + ir *irStub + + netmap *netmapStub + + frostfsIDSubjectReader *frostfsidStub + + apeChecker *apeChecker +} + +func newTestAPEServer() testAPEServer { + srv := &srvStub{ + calls: map[string]int{}, + } + + engine := inmemory.NewInMemory() + + containerReader := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + + ir := &irStub{ + keys: [][]byte{}, + } + + netmap := &netmapStub{} + + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{}, + } + + apeChecker := &apeChecker{ + router: engine, + reader: containerReader, + ir: ir, + nm: netmap, + frostFSIDClient: frostfsIDSubjectReader, + next: srv, + } + + return testAPEServer{ + engine: engine, + containerReader: containerReader, + ir: ir, + netmap: netmap, + frostfsIDSubjectReader: frostfsIDSubjectReader, + apeChecker: apeChecker, + } +} + +func TestValidateContainerBoundedOperation(t *testing.T) { + t.Parallel() + + t.Run("check root-defined container in root-defined container target rule", func(t *testing.T) { + t.Parallel() + + components := newTestAPEServer() + contID, testContainer := initTestContainer(t, false) + components.containerReader.c[contID] = &containercore.Container{Value: testContainer} + initTestNetmap(components.netmap) + + _, _, err := components.engine.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOthers, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := initTestGetContainerRequest(t, contID) + + err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + aErr := apeErr(nativeschema.MethodGetContainer, chain.AccessDenied) + require.ErrorContains(t, err, aErr.Error()) + }) + + t.Run("check root-defined container in testdomain-defined container target rule", func(t *testing.T) { + t.Parallel() + + components := newTestAPEServer() + contID, testContainer := initTestContainer(t, false) + components.containerReader.c[contID] = &containercore.Container{Value: testContainer} + initTestNetmap(components.netmap) + + _, _, err := components.engine.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainer, testDomainName, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOthers, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := initTestGetContainerRequest(t, contID) + + err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + require.NoError(t, err) + }) + + t.Run("check root-defined container in testdomain namespace target rule", func(t *testing.T) { + t.Parallel() + + components := newTestAPEServer() + contID, testContainer := initTestContainer(t, false) + components.containerReader.c[contID] = &containercore.Container{Value: testContainer} + initTestNetmap(components.netmap) + + _, _, err := components.engine.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(testDomainName), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainers, testDomainName), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOthers, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := initTestGetContainerRequest(t, contID) + + err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + require.NoError(t, err) + }) + + t.Run("check testdomain-defined container in root-defined container target rule", func(t *testing.T) { + t.Parallel() + + components := newTestAPEServer() + contID, testContainer := initTestContainer(t, true) + components.containerReader.c[contID] = &containercore.Container{Value: testContainer} + initTestNetmap(components.netmap) + + _, _, err := components.engine.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOthers, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := initTestGetContainerRequest(t, contID) + + err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + require.NoError(t, err) + }) + + t.Run("check testdomain-defined container in testdomain-defined container target rule", func(t *testing.T) { + t.Parallel() + + components := newTestAPEServer() + contID, testContainer := initTestContainer(t, true) + components.containerReader.c[contID] = &containercore.Container{Value: testContainer} + initTestNetmap(components.netmap) + + _, _, err := components.engine.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainer, testDomainName, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOthers, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := initTestGetContainerRequest(t, contID) + + err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + aErr := apeErr(nativeschema.MethodGetContainer, chain.AccessDenied) + require.ErrorContains(t, err, aErr.Error()) + }) + + t.Run("check testdomain-defined container in testdomain namespace target rule", func(t *testing.T) { + t.Parallel() + + components := newTestAPEServer() + contID, testContainer := initTestContainer(t, true) + components.containerReader.c[contID] = &containercore.Container{Value: testContainer} + initTestNetmap(components.netmap) + + _, _, err := components.engine.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(testDomainName), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainers, testDomainName), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOthers, + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := initTestGetContainerRequest(t, contID) + + err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + aErr := apeErr(nativeschema.MethodGetContainer, chain.AccessDenied) + require.ErrorContains(t, err, aErr.Error()) + }) +} + +func initTestGetContainerRequest(t *testing.T, contID cid.ID) *container.GetRequest { + req := &container.GetRequest{} + req.SetBody(&container.GetRequestBody{}) + var refContID refs.ContainerID + contID.WriteToV2(&refContID) + req.GetBody().SetContainerID(&refContID) + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + return req +} + +func initTestNetmap(netmapStub *netmapStub) { + netmapStub.currentEpoch = 100 + netmapStub.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(netmapStub.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + netmapStub.netmaps[netmapStub.currentEpoch] = &testNetmap + netmapStub.netmaps[netmapStub.currentEpoch-1] = &testNetmap +} + +func initTestContainer(t *testing.T, isDomainSet bool) (cid.ID, cnrSDK.Container) { + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + if isDomainSet { + // no domain defined -> container is defined in root namespace + var domain cnrSDK.Domain + domain.SetName(testDomainName) + domain.SetZone(testDomainZone) + cnrSDK.WriteDomain(&testContainer, domain) + } + return contID, testContainer +} + +func initPutRequest(t *testing.T, testContainer cnrSDK.Container) *container.PutRequest { + req := &container.PutRequest{} + req.SetBody(&container.PutRequestBody{}) + var reqCont container.Container + testContainer.WriteToV2(&reqCont) + req.GetBody().SetContainer(&reqCont) + + sessionPK, err := keys.NewPrivateKey() + require.NoError(t, err) + sToken := sessiontest.ContainerSigned() + sToken.ApplyOnlyTo(cid.ID{}) + require.NoError(t, sToken.Sign(sessionPK.PrivateKey)) + var sTokenV2 session.Token + sToken.WriteToV2(&sTokenV2) + metaHeader := new(session.RequestMetaHeader) + metaHeader.SetSessionToken(&sTokenV2) + req.SetMetaHeader(metaHeader) + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + return req +} + +func initOwnerIDScriptHash(t *testing.T, testContainer cnrSDK.Container) util.Uint160 { + var ownerSDK *user.ID + owner := testContainer.Owner() + ownerSDK = &owner + sc, err := ownerSDK.ScriptHash() + require.NoError(t, err) + return sc +} + +func initActorOwnerScriptHashes(t *testing.T, actorPK *keys.PrivateKey, ownerPK *keys.PrivateKey) (actorScriptHash util.Uint160, ownerScriptHash util.Uint160) { + var actorUserID user.ID + user.IDFromKey(&actorUserID, ecdsa.PublicKey(*actorPK.PublicKey())) + var err error + actorScriptHash, err = actorUserID.ScriptHash() + require.NoError(t, err) + + var ownerUserID user.ID + user.IDFromKey(&ownerUserID, ecdsa.PublicKey(*ownerPK.PublicKey())) + ownerScriptHash, err = ownerUserID.ScriptHash() + require.NoError(t, err) + require.NotEqual(t, ownerScriptHash.String(), actorScriptHash.String()) + return +} + +func initListRequest(t *testing.T, actorPK *keys.PrivateKey, ownerPK *keys.PrivateKey) *container.ListRequest { + var ownerUserID user.ID + user.IDFromKey(&ownerUserID, ownerPK.PrivateKey.PublicKey) + + req := &container.ListRequest{} + req.SetBody(&container.ListRequestBody{}) + var ownerID refs.OwnerID + ownerUserID.WriteToV2(&ownerID) + req.GetBody().SetOwnerID(&ownerID) + + require.NoError(t, signature.SignServiceMessage(&actorPK.PrivateKey, req)) + return req +} diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index ed077411c..59535dc8a 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -4,8 +4,8 @@ import ( "context" "errors" "fmt" + "strings" - containerV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -744,7 +745,11 @@ func (b Service) findRequestInfo(req MetaWithToken, idCnr cid.ID, op acl.Op) (in info.operation = op info.cnrOwner = cnr.Value.Owner() info.idCnr = idCnr - info.cnrNamespace = cnr.Value.Attribute(containerV2.SysAttributeZone) + + cnrNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(cnr.Value).Zone(), ".ns") + if hasNamespace { + info.cnrNamespace = cnrNamespace + } // it is assumed that at the moment the key will be valid, // otherwise the request would not pass validation From 6534252c226a47603a7224d74d40ef943c51a926 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 10 Jan 2024 15:17:05 +0300 Subject: [PATCH 0833/1943] [#799] policer: Refactor method `processNodes` Signed-off-by: Anton Nikiforov --- pkg/services/policer/check.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 73f83b2a7..bb6313d07 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -132,21 +132,21 @@ func (p *Policer) processNodes(ctx context.Context, requirements *placementRequi cancel() - if client.IsErrObjectNotFound(err) { - checkedNodes.submitReplicaCandidate(nodes[i]) - continue - } - - if isClientErrMaintenance(err) { - shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) - } else if err != nil { - p.log.Error(logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, - zap.Stringer("object", addr), - zap.String("error", err.Error()), - ) - } else { + if err == nil { shortage-- checkedNodes.submitReplicaHolder(nodes[i]) + } else { + if client.IsErrObjectNotFound(err) { + checkedNodes.submitReplicaCandidate(nodes[i]) + continue + } else if isClientErrMaintenance(err) { + shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) + } else { + p.log.Error(logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, + zap.Stringer("object", addr), + zap.String("error", err.Error()), + ) + } } } From d0eadf7ea2638a89e754592322bd2f1012c4756d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 21 Nov 2023 15:06:54 +0300 Subject: [PATCH 0834/1943] [#799] engine: Skip put when object removed from shard Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/engine/evacuate.go | 28 ++++----- pkg/local_object_storage/engine/put.go | 66 ++++++++++++++------- 2 files changed, 59 insertions(+), 35 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 7bb37ef61..f522e6fbd 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -406,21 +406,21 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add if _, ok := shardsToEvacuate[shards[j].ID().String()]; ok { continue } - putDone, exists := e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object) - if putDone || exists { - if putDone { - res.evacuated.Add(1) - e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, - zap.Stringer("from", sh.ID()), - zap.Stringer("to", shards[j].ID()), - zap.Stringer("addr", addr), - evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) - } - if exists { - res.skipped.Add(1) - } + switch e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object).status { + case putToShardSuccess: + res.evacuated.Add(1) + e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, + zap.Stringer("from", sh.ID()), + zap.Stringer("to", shards[j].ID()), + zap.Stringer("addr", addr), + evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return true, nil + case putToShardExists, putToShardRemoved: + res.skipped.Add(1) + return true, nil + default: + continue } } diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 7ce915ad8..2b1712f3f 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -12,6 +12,7 @@ import ( tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" @@ -26,6 +27,20 @@ type PutPrm struct { var errPutShard = errors.New("could not put object to any shard") +type putToShardStatus byte + +const ( + putToShardUnknown putToShardStatus = iota + putToShardSuccess + putToShardExists + putToShardRemoved +) + +type putToShardRes struct { + status putToShardStatus + err error +} + // WithObject is a Put option to set object to save. // // Option is required. @@ -68,8 +83,7 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { return err } - finished := false - + var shRes putToShardRes e.iterateOverSortedShards(addr, func(ind int, sh hashedShard) (stop bool) { e.mtx.RLock() pool, ok := e.shardPools[sh.ID().String()] @@ -78,25 +92,26 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { // Shard was concurrently removed, skip. return false } - - putDone, exists := e.putToShard(ctx, sh, ind, pool, addr, prm.obj) - finished = putDone || exists - return finished + shRes = e.putToShard(ctx, sh, ind, pool, addr, prm.obj) + return shRes.status != putToShardUnknown }) - - if !finished { - err = errPutShard + switch shRes.status { + case putToShardUnknown: + return errPutShard + case putToShardRemoved: + return shRes.err + case putToShardExists, putToShardSuccess: + return nil + default: + return errPutShard } - - return err } // putToShard puts object to sh. -// First return value is true iff put has been successfully done. -// Second return value is true iff object already exists. -func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, pool util.WorkerPool, addr oid.Address, obj *objectSDK.Object) (bool, bool) { - var putSuccess, alreadyExists bool - +// Return putToShardStatus and error if it is necessary to propagate an error upper. +func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, pool util.WorkerPool, + addr oid.Address, obj *objectSDK.Object, +) (res putToShardRes) { exitCh := make(chan struct{}) if err := pool.Submit(func() { @@ -110,14 +125,13 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, if shard.IsErrObjectExpired(err) { // object is already found but // expired => do nothing with it - alreadyExists = true + res.status = putToShardExists } return // this is not ErrAlreadyRemoved error so we can go to the next shard } - alreadyExists = exists.Exists() - if alreadyExists { + if exists.Exists() { if ind != 0 { var toMoveItPrm shard.ToMoveItPrm toMoveItPrm.SetAddress(addr) @@ -132,6 +146,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, } } + res.status = putToShardExists return } @@ -148,12 +163,21 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return } + if client.IsErrObjectAlreadyRemoved(err) { + e.log.Warn(logs.EngineCouldNotPutObjectToShard, + zap.Stringer("shard_id", sh.ID()), + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + res.status = putToShardRemoved + res.err = err + return + } e.reportShardError(sh, "could not put object to shard", err) return } - putSuccess = true + res.status = putToShardSuccess }); err != nil { e.log.Warn(logs.EngineCouldNotPutObjectToShard, zap.Error(err)) close(exitCh) @@ -161,7 +185,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, <-exitCh - return putSuccess, alreadyExists + return } // Put writes provided object to local storage. From a5446bc17d28475d7aa07c2aa3548de2d649cab8 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 2 Feb 2024 14:46:12 +0300 Subject: [PATCH 0835/1943] [#952] object: Pass namespace within context in ACL service Signed-off-by: Airat Arifullin --- pkg/services/object/acl/v2/service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 59535dc8a..af26a6fd1 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -458,6 +458,7 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb func requestContext(ctx context.Context, reqInfo RequestInfo) context.Context { return context.WithValue(ctx, object.RequestContextKey, &object.RequestContext{ + Namespace: reqInfo.ContainerNamespace(), SenderKey: reqInfo.SenderKey(), Role: reqInfo.RequestRole(), }) From 2cb04379a418028dd6d7d6bd7b77e796886cce00 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 29 Jan 2024 11:23:56 +0300 Subject: [PATCH 0836/1943] [#922] go.mod: Update APE Signed-off-by: Anton Nikiforov --- go.mod | 4 +- go.sum | 8 +++- pkg/ape/chainbase/boltdb.go | 49 +++++++++++++++++++- pkg/services/control/server/policy_engine.go | 2 +- 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a5cffc28c..07097c547 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240126141009-65b4525b3bf0 git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240122104724-06cbfe8691ad + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240129064140-8d21ab2d99d9 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 @@ -76,8 +76,10 @@ require ( github.com/holiman/uint256 v1.2.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect diff --git a/go.sum b/go.sum index 4462c3575..5243ac435 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240126141009-65b4525b3bf0 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240126141009-65b4525b3bf0/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240122104724-06cbfe8691ad h1:+D96Uu0Pw+55CrxP+srazthtZAh0Q19BtJo1Pm4hOP0= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240122104724-06cbfe8691ad/go.mod h1:ps6oKO0mxaPJzK3admTB3iwoBXKkHnS73n4PCrqpHBg= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240129064140-8d21ab2d99d9 h1:hkDIg6vzV1c1pyt5uj9EiBikHqKczfoNpPhwWvXaNf8= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240129064140-8d21ab2d99d9/go.mod h1:YVL7yFaT0QNSpA0z+RHudLvrLwT+lsFYGyBSVc1ustI= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= @@ -114,6 +114,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -129,6 +131,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= diff --git a/pkg/ape/chainbase/boltdb.go b/pkg/ape/chainbase/boltdb.go index 4e15050b3..c26156c7a 100644 --- a/pkg/ape/chainbase/boltdb.go +++ b/pkg/ape/chainbase/boltdb.go @@ -167,8 +167,8 @@ func getTargetBucketCreateIfEmpty(tx *bbolt.Tx, name chain.Name, target policyen } func (cs *boltLocalOverrideStorage) AddOverride(name chain.Name, target policyengine.Target, c *chain.Chain) (chain.ID, error) { - if c.ID == "" { - return "", fmt.Errorf("chain ID is not set") + if len(c.ID) == 0 { + return chain.ID{}, fmt.Errorf("chain ID is not set") } serializedChain := c.Bytes() @@ -264,3 +264,48 @@ func (cs *boltLocalOverrideStorage) DropAllOverrides(name chain.Name) error { return tx.DeleteBucket([]byte(name)) }) } + +func (cs *boltLocalOverrideStorage) ListOverrideDefinedTargets(name chain.Name) ([]policyengine.Target, error) { + var targets []policyengine.Target + if err := cs.db.View(func(tx *bbolt.Tx) error { + var err error + targets, err = getTargets(tx, name) + if err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + return targets, nil +} + +func getTargets(tx *bbolt.Tx, name chain.Name) ([]policyengine.Target, error) { + var targets []policyengine.Target + cbucket := tx.Bucket(chainBucket) + if cbucket == nil { + return nil, ErrRootBucketNotFound + } + + nbucket := cbucket.Bucket([]byte(name)) + if nbucket == nil { + return nil, fmt.Errorf("%w: %w: %s", policyengine.ErrChainNotFound, ErrGlobalNamespaceBucketNotFound, name) + } + + if err := nbucket.ForEachBucket(func(k []byte) error { + ttype := policyengine.TargetType(k[0]) + if err := nbucket.Bucket(k).ForEachBucket(func(k []byte) error { + targets = append(targets, policyengine.Target{ + Type: ttype, + Name: string(slice.Copy(k)), + }) + return nil + }); err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + return targets, nil +} diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 6bf600748..bd7f35f46 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -44,7 +44,7 @@ func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainL s.apeChainCounter.Add(1) // TODO (aarifullin): the such chain id is not well-designed yet. - if chain.ID == "" { + if len(chain.ID) == 0 { chain.ID = apechain.ID(fmt.Sprintf("%s:%d", apechain.Ingress, s.apeChainCounter.Load())) } From 95e15f499f3cf068a38358c2527d78ba33b1b6e9 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 29 Jan 2024 11:16:59 +0300 Subject: [PATCH 0837/1943] [#922] Update files generated by protoc Signed-off-by: Anton Nikiforov --- pkg/services/control/ir/service.pb.go | 2 +- pkg/services/control/ir/service_grpc.pb.go | 25 ++++++---- pkg/services/control/ir/types.pb.go | 2 +- pkg/services/control/service.pb.go | 2 +- pkg/services/control/types.pb.go | 2 +- pkg/services/tree/service.pb.go | 2 +- pkg/services/tree/service_grpc.pb.go | 54 ++++++++++++++-------- pkg/services/tree/types.pb.go | 2 +- 8 files changed, 57 insertions(+), 34 deletions(-) diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index a9be56cd9..7dc8f3aff 100644 --- a/pkg/services/control/ir/service.pb.go +++ b/pkg/services/control/ir/service.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 +// protoc-gen-go v1.32.0 // protoc v4.25.0 // source: pkg/services/control/ir/service.proto diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index 0f93c0e19..724149c44 100644 --- a/pkg/services/control/ir/service_grpc.pb.go +++ b/pkg/services/control/ir/service_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.0 // source: pkg/services/control/ir/service.proto @@ -18,6 +18,13 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + ControlService_HealthCheck_FullMethodName = "/ircontrol.ControlService/HealthCheck" + ControlService_TickEpoch_FullMethodName = "/ircontrol.ControlService/TickEpoch" + ControlService_RemoveNode_FullMethodName = "/ircontrol.ControlService/RemoveNode" + ControlService_RemoveContainer_FullMethodName = "/ircontrol.ControlService/RemoveContainer" +) + // ControlServiceClient is the client API for ControlService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -42,7 +49,7 @@ func NewControlServiceClient(cc grpc.ClientConnInterface) ControlServiceClient { func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { out := new(HealthCheckResponse) - err := c.cc.Invoke(ctx, "/ircontrol.ControlService/HealthCheck", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_HealthCheck_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -51,7 +58,7 @@ func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckR func (c *controlServiceClient) TickEpoch(ctx context.Context, in *TickEpochRequest, opts ...grpc.CallOption) (*TickEpochResponse, error) { out := new(TickEpochResponse) - err := c.cc.Invoke(ctx, "/ircontrol.ControlService/TickEpoch", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_TickEpoch_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -60,7 +67,7 @@ func (c *controlServiceClient) TickEpoch(ctx context.Context, in *TickEpochReque func (c *controlServiceClient) RemoveNode(ctx context.Context, in *RemoveNodeRequest, opts ...grpc.CallOption) (*RemoveNodeResponse, error) { out := new(RemoveNodeResponse) - err := c.cc.Invoke(ctx, "/ircontrol.ControlService/RemoveNode", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_RemoveNode_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -69,7 +76,7 @@ func (c *controlServiceClient) RemoveNode(ctx context.Context, in *RemoveNodeReq func (c *controlServiceClient) RemoveContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*RemoveContainerResponse, error) { out := new(RemoveContainerResponse) - err := c.cc.Invoke(ctx, "/ircontrol.ControlService/RemoveContainer", in, out, opts...) + err := c.cc.Invoke(ctx, ControlService_RemoveContainer_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -128,7 +135,7 @@ func _ControlService_HealthCheck_Handler(srv interface{}, ctx context.Context, d } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/ircontrol.ControlService/HealthCheck", + FullMethod: ControlService_HealthCheck_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).HealthCheck(ctx, req.(*HealthCheckRequest)) @@ -146,7 +153,7 @@ func _ControlService_TickEpoch_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/ircontrol.ControlService/TickEpoch", + FullMethod: ControlService_TickEpoch_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).TickEpoch(ctx, req.(*TickEpochRequest)) @@ -164,7 +171,7 @@ func _ControlService_RemoveNode_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/ircontrol.ControlService/RemoveNode", + FullMethod: ControlService_RemoveNode_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).RemoveNode(ctx, req.(*RemoveNodeRequest)) @@ -182,7 +189,7 @@ func _ControlService_RemoveContainer_Handler(srv interface{}, ctx context.Contex } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/ircontrol.ControlService/RemoveContainer", + FullMethod: ControlService_RemoveContainer_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ControlServiceServer).RemoveContainer(ctx, req.(*RemoveContainerRequest)) diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go index cc39ad572..819cedf6a 100644 --- a/pkg/services/control/ir/types.pb.go +++ b/pkg/services/control/ir/types.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 +// protoc-gen-go v1.32.0 // protoc v4.25.0 // source: pkg/services/control/ir/types.proto diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 9fb6f865e..f6b9ef608 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 +// protoc-gen-go v1.32.0 // protoc v4.25.0 // source: pkg/services/control/service.proto diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index 8f083d695..4556e8a55 100644 --- a/pkg/services/control/types.pb.go +++ b/pkg/services/control/types.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 +// protoc-gen-go v1.32.0 // protoc v4.25.0 // source: pkg/services/control/types.proto diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go index 534661389..441ae7927 100644 --- a/pkg/services/tree/service.pb.go +++ b/pkg/services/tree/service.pb.go @@ -3,7 +3,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 +// protoc-gen-go v1.32.0 // protoc v4.25.0 // source: pkg/services/tree/service.proto diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index 66850c90e..63f96e11a 100644 --- a/pkg/services/tree/service_grpc.pb.go +++ b/pkg/services/tree/service_grpc.pb.go @@ -1,6 +1,9 @@ +//* +// Service for working with CRDT tree. + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.0 // source: pkg/services/tree/service.proto @@ -18,6 +21,19 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + TreeService_Add_FullMethodName = "/tree.TreeService/Add" + TreeService_AddByPath_FullMethodName = "/tree.TreeService/AddByPath" + TreeService_Remove_FullMethodName = "/tree.TreeService/Remove" + TreeService_Move_FullMethodName = "/tree.TreeService/Move" + TreeService_GetNodeByPath_FullMethodName = "/tree.TreeService/GetNodeByPath" + TreeService_GetSubTree_FullMethodName = "/tree.TreeService/GetSubTree" + TreeService_TreeList_FullMethodName = "/tree.TreeService/TreeList" + TreeService_Apply_FullMethodName = "/tree.TreeService/Apply" + TreeService_GetOpLog_FullMethodName = "/tree.TreeService/GetOpLog" + TreeService_Healthcheck_FullMethodName = "/tree.TreeService/Healthcheck" +) + // TreeServiceClient is the client API for TreeService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -55,7 +71,7 @@ func NewTreeServiceClient(cc grpc.ClientConnInterface) TreeServiceClient { func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*AddResponse, error) { out := new(AddResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Add", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Add_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -64,7 +80,7 @@ func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grp func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, opts ...grpc.CallOption) (*AddByPathResponse, error) { out := new(AddByPathResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/AddByPath", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_AddByPath_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -73,7 +89,7 @@ func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts ...grpc.CallOption) (*RemoveResponse, error) { out := new(RemoveResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Remove", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Remove_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -82,7 +98,7 @@ func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...grpc.CallOption) (*MoveResponse, error) { out := new(MoveResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Move", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Move_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -91,7 +107,7 @@ func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...g func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPathRequest, opts ...grpc.CallOption) (*GetNodeByPathResponse, error) { out := new(GetNodeByPathResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/GetNodeByPath", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_GetNodeByPath_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -99,7 +115,7 @@ func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPath } func (c *treeServiceClient) GetSubTree(ctx context.Context, in *GetSubTreeRequest, opts ...grpc.CallOption) (TreeService_GetSubTreeClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], "/tree.TreeService/GetSubTree", opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], TreeService_GetSubTree_FullMethodName, opts...) if err != nil { return nil, err } @@ -132,7 +148,7 @@ func (x *treeServiceGetSubTreeClient) Recv() (*GetSubTreeResponse, error) { func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, opts ...grpc.CallOption) (*TreeListResponse, error) { out := new(TreeListResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/TreeList", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_TreeList_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -141,7 +157,7 @@ func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, o func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts ...grpc.CallOption) (*ApplyResponse, error) { out := new(ApplyResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Apply", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Apply_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -149,7 +165,7 @@ func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts .. } func (c *treeServiceClient) GetOpLog(ctx context.Context, in *GetOpLogRequest, opts ...grpc.CallOption) (TreeService_GetOpLogClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], "/tree.TreeService/GetOpLog", opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], TreeService_GetOpLog_FullMethodName, opts...) if err != nil { return nil, err } @@ -182,7 +198,7 @@ func (x *treeServiceGetOpLogClient) Recv() (*GetOpLogResponse, error) { func (c *treeServiceClient) Healthcheck(ctx context.Context, in *HealthcheckRequest, opts ...grpc.CallOption) (*HealthcheckResponse, error) { out := new(HealthcheckResponse) - err := c.cc.Invoke(ctx, "/tree.TreeService/Healthcheck", in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Healthcheck_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -272,7 +288,7 @@ func _TreeService_Add_Handler(srv interface{}, ctx context.Context, dec func(int } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Add", + FullMethod: TreeService_Add_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Add(ctx, req.(*AddRequest)) @@ -290,7 +306,7 @@ func _TreeService_AddByPath_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/AddByPath", + FullMethod: TreeService_AddByPath_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).AddByPath(ctx, req.(*AddByPathRequest)) @@ -308,7 +324,7 @@ func _TreeService_Remove_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Remove", + FullMethod: TreeService_Remove_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Remove(ctx, req.(*RemoveRequest)) @@ -326,7 +342,7 @@ func _TreeService_Move_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Move", + FullMethod: TreeService_Move_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Move(ctx, req.(*MoveRequest)) @@ -344,7 +360,7 @@ func _TreeService_GetNodeByPath_Handler(srv interface{}, ctx context.Context, de } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/GetNodeByPath", + FullMethod: TreeService_GetNodeByPath_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).GetNodeByPath(ctx, req.(*GetNodeByPathRequest)) @@ -383,7 +399,7 @@ func _TreeService_TreeList_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/TreeList", + FullMethod: TreeService_TreeList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).TreeList(ctx, req.(*TreeListRequest)) @@ -401,7 +417,7 @@ func _TreeService_Apply_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Apply", + FullMethod: TreeService_Apply_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Apply(ctx, req.(*ApplyRequest)) @@ -440,7 +456,7 @@ func _TreeService_Healthcheck_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/tree.TreeService/Healthcheck", + FullMethod: TreeService_Healthcheck_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(TreeServiceServer).Healthcheck(ctx, req.(*HealthcheckRequest)) diff --git a/pkg/services/tree/types.pb.go b/pkg/services/tree/types.pb.go index f6b19c20f..e44ac5566 100644 --- a/pkg/services/tree/types.pb.go +++ b/pkg/services/tree/types.pb.go @@ -3,7 +3,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 +// protoc-gen-go v1.32.0 // protoc v4.25.0 // source: pkg/services/tree/types.proto From 9916598dfb8ef534bf8f0c4124f59491f2f4f578 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 26 Jan 2024 17:10:26 +0300 Subject: [PATCH 0838/1943] [#922] control: Extend api with `ListOverrideDefinedTargets` Signed-off-by: Anton Nikiforov --- pkg/services/control/rpc.go | 48 +- pkg/services/control/server/policy_engine.go | 54 + pkg/services/control/service.pb.go | 1197 +++++++++++------- pkg/services/control/service.proto | 29 +- pkg/services/control/service_frostfs.pb.go | 176 +++ pkg/services/control/service_grpc.pb.go | 73 +- 6 files changed, 1093 insertions(+), 484 deletions(-) diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 3fd4d4ae9..24b20f873 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -8,23 +8,24 @@ import ( const serviceName = "control.ControlService" const ( - rpcHealthCheck = "HealthCheck" - rpcSetNetmapStatus = "SetNetmapStatus" - rpcDropObjects = "DropObjects" - rpcListShards = "ListShards" - rpcSetShardMode = "SetShardMode" - rpcSynchronizeTree = "SynchronizeTree" - rpcEvacuateShard = "EvacuateShard" - rpcStartShardEvacuation = "StartShardEvacuation" - rpcGetShardEvacuationStatus = "GetShardEvacuationStatus" - rpcStopShardEvacuation = "StopShardEvacuation" - rpcFlushCache = "FlushCache" - rpcDoctor = "Doctor" - rpcAddChainLocalOverride = "AddChainLocalOverride" - rpcGetChainLocalOverride = "GetChainLocalOverride" - rpcListChainLocalOverrides = "ListChainLocalOverrides" - rpcRemoveChainLocalOverride = "RemoveChainLocalOverride" - rpcSealWriteCache = "SealWriteCache" + rpcHealthCheck = "HealthCheck" + rpcSetNetmapStatus = "SetNetmapStatus" + rpcDropObjects = "DropObjects" + rpcListShards = "ListShards" + rpcSetShardMode = "SetShardMode" + rpcSynchronizeTree = "SynchronizeTree" + rpcEvacuateShard = "EvacuateShard" + rpcStartShardEvacuation = "StartShardEvacuation" + rpcGetShardEvacuationStatus = "GetShardEvacuationStatus" + rpcStopShardEvacuation = "StopShardEvacuation" + rpcFlushCache = "FlushCache" + rpcDoctor = "Doctor" + rpcAddChainLocalOverride = "AddChainLocalOverride" + rpcGetChainLocalOverride = "GetChainLocalOverride" + rpcListChainLocalOverrides = "ListChainLocalOverrides" + rpcRemoveChainLocalOverride = "RemoveChainLocalOverride" + rpcSealWriteCache = "SealWriteCache" + rpcListTargetsLocalOverrides = "ListTargetsLocalOverrides" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -240,6 +241,19 @@ func ListChainLocalOverrides(cli *client.Client, req *ListChainLocalOverridesReq return wResp.message, nil } +// ListTargetsLocalOverrides executes ControlService.ListTargetsLocalOverrides RPC. +func ListTargetsLocalOverrides(cli *client.Client, req *ListTargetsLocalOverridesRequest, opts ...client.CallOption) (*ListTargetsLocalOverridesResponse, error) { + wResp := newResponseWrapper[ListTargetsLocalOverridesResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcListTargetsLocalOverrides), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + // RemoveChainLocalOverride executes ControlService.RemoveChainLocalOverride RPC. func GetChainLocalOverride(cli *client.Client, req *GetChainLocalOverrideRequest, opts ...client.CallOption) (*GetChainLocalOverrideResponse, error) { wResp := newResponseWrapper[GetChainLocalOverrideResponse]() diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index bd7f35f46..079b605ac 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -25,6 +25,29 @@ func apeTarget(chainTarget *control.ChainTarget) (engine.Target, error) { fmt.Errorf("target type is not supported: %s", chainTarget.GetType().String()).Error()) } +func controlTarget(chainTarget *engine.Target) (control.ChainTarget, error) { + switch chainTarget.Type { + case engine.Container: + return control.ChainTarget{ + Name: chainTarget.Name, + Type: control.ChainTarget_CONTAINER, + }, nil + case engine.Namespace: + // If namespace is empty, we take it for root namespace. + nm := chainTarget.Name + if nm == "root" { + nm = "" + } + return control.ChainTarget{ + Name: nm, + Type: control.ChainTarget_NAMESPACE, + }, nil + default: + } + return control.ChainTarget{}, status.Error(codes.InvalidArgument, + fmt.Errorf("target type is not supported: %c", chainTarget.Type).Error()) +} + func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainLocalOverrideRequest) (*control.AddChainLocalOverrideResponse, error) { if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) @@ -157,6 +180,37 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove return resp, nil } +func (s *Server) ListTargetsLocalOverrides(_ context.Context, req *control.ListTargetsLocalOverridesRequest) (*control.ListTargetsLocalOverridesResponse, error) { + if err := s.isValidRequest(req); err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + apeChainName := apechain.Name(req.GetBody().GetChainName()) + apeTargets, err := s.localOverrideStorage.LocalStorage().ListOverrideDefinedTargets(apeChainName) + if err != nil { + return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) + } + targets := make([]*control.ChainTarget, 0, len(apeTargets)) + for i := range apeTargets { + target, err := controlTarget(&apeTargets[i]) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + targets = append(targets, &target) + } + + resp := &control.ListTargetsLocalOverridesResponse{ + Body: &control.ListTargetsLocalOverridesResponse_Body{ + Targets: targets, + }, + } + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil +} + func getCodeByLocalStorageErr(err error) codes.Code { if errors.Is(err, engine.ErrChainNotFound) || errors.Is(err, engine.ErrChainNameNotFound) { return codes.NotFound diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index f6b9ef608..e0c6e0622 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -1774,6 +1774,118 @@ func (x *ListChainLocalOverridesResponse) GetSignature() *Signature { return nil } +// ListTargetsLocalOverrides request. +type ListTargetsLocalOverridesRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *ListTargetsLocalOverridesRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *ListTargetsLocalOverridesRequest) Reset() { + *x = ListTargetsLocalOverridesRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListTargetsLocalOverridesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListTargetsLocalOverridesRequest) ProtoMessage() {} + +func (x *ListTargetsLocalOverridesRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[30] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListTargetsLocalOverridesRequest.ProtoReflect.Descriptor instead. +func (*ListTargetsLocalOverridesRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30} +} + +func (x *ListTargetsLocalOverridesRequest) GetBody() *ListTargetsLocalOverridesRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *ListTargetsLocalOverridesRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// ListTargetsLocalOverrides response. +type ListTargetsLocalOverridesResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *ListTargetsLocalOverridesResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *ListTargetsLocalOverridesResponse) Reset() { + *x = ListTargetsLocalOverridesResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListTargetsLocalOverridesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListTargetsLocalOverridesResponse) ProtoMessage() {} + +func (x *ListTargetsLocalOverridesResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[31] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListTargetsLocalOverridesResponse.ProtoReflect.Descriptor instead. +func (*ListTargetsLocalOverridesResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31} +} + +func (x *ListTargetsLocalOverridesResponse) GetBody() *ListTargetsLocalOverridesResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *ListTargetsLocalOverridesResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + type RemoveChainLocalOverrideRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1786,7 +1898,7 @@ type RemoveChainLocalOverrideRequest struct { func (x *RemoveChainLocalOverrideRequest) Reset() { *x = RemoveChainLocalOverrideRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1799,7 +1911,7 @@ func (x *RemoveChainLocalOverrideRequest) String() string { func (*RemoveChainLocalOverrideRequest) ProtoMessage() {} func (x *RemoveChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1812,7 +1924,7 @@ func (x *RemoveChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveChainLocalOverrideRequest.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32} } func (x *RemoveChainLocalOverrideRequest) GetBody() *RemoveChainLocalOverrideRequest_Body { @@ -1841,7 +1953,7 @@ type RemoveChainLocalOverrideResponse struct { func (x *RemoveChainLocalOverrideResponse) Reset() { *x = RemoveChainLocalOverrideResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1854,7 +1966,7 @@ func (x *RemoveChainLocalOverrideResponse) String() string { func (*RemoveChainLocalOverrideResponse) ProtoMessage() {} func (x *RemoveChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1867,7 +1979,7 @@ func (x *RemoveChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveChainLocalOverrideResponse.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33} } func (x *RemoveChainLocalOverrideResponse) GetBody() *RemoveChainLocalOverrideResponse_Body { @@ -1896,7 +2008,7 @@ type SealWriteCacheRequest struct { func (x *SealWriteCacheRequest) Reset() { *x = SealWriteCacheRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1909,7 +2021,7 @@ func (x *SealWriteCacheRequest) String() string { func (*SealWriteCacheRequest) ProtoMessage() {} func (x *SealWriteCacheRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1922,7 +2034,7 @@ func (x *SealWriteCacheRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheRequest.ProtoReflect.Descriptor instead. func (*SealWriteCacheRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34} } func (x *SealWriteCacheRequest) GetBody() *SealWriteCacheRequest_Body { @@ -1951,7 +2063,7 @@ type SealWriteCacheResponse struct { func (x *SealWriteCacheResponse) Reset() { *x = SealWriteCacheResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1964,7 +2076,7 @@ func (x *SealWriteCacheResponse) String() string { func (*SealWriteCacheResponse) ProtoMessage() {} func (x *SealWriteCacheResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1977,7 +2089,7 @@ func (x *SealWriteCacheResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheResponse.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35} } func (x *SealWriteCacheResponse) GetBody() *SealWriteCacheResponse_Body { @@ -2004,7 +2116,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2017,7 +2129,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2048,7 +2160,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2061,7 +2173,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2112,7 +2224,7 @@ type SetNetmapStatusRequest_Body struct { func (x *SetNetmapStatusRequest_Body) Reset() { *x = SetNetmapStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2125,7 +2237,7 @@ func (x *SetNetmapStatusRequest_Body) String() string { func (*SetNetmapStatusRequest_Body) ProtoMessage() {} func (x *SetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2165,7 +2277,7 @@ type SetNetmapStatusResponse_Body struct { func (x *SetNetmapStatusResponse_Body) Reset() { *x = SetNetmapStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2178,7 +2290,7 @@ func (x *SetNetmapStatusResponse_Body) String() string { func (*SetNetmapStatusResponse_Body) ProtoMessage() {} func (x *SetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2208,7 +2320,7 @@ type DropObjectsRequest_Body struct { func (x *DropObjectsRequest_Body) Reset() { *x = DropObjectsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2221,7 +2333,7 @@ func (x *DropObjectsRequest_Body) String() string { func (*DropObjectsRequest_Body) ProtoMessage() {} func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2254,7 +2366,7 @@ type DropObjectsResponse_Body struct { func (x *DropObjectsResponse_Body) Reset() { *x = DropObjectsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2267,7 +2379,7 @@ func (x *DropObjectsResponse_Body) String() string { func (*DropObjectsResponse_Body) ProtoMessage() {} func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2293,7 +2405,7 @@ type ListShardsRequest_Body struct { func (x *ListShardsRequest_Body) Reset() { *x = ListShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2306,7 +2418,7 @@ func (x *ListShardsRequest_Body) String() string { func (*ListShardsRequest_Body) ProtoMessage() {} func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2335,7 +2447,7 @@ type ListShardsResponse_Body struct { func (x *ListShardsResponse_Body) Reset() { *x = ListShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2348,7 +2460,7 @@ func (x *ListShardsResponse_Body) String() string { func (*ListShardsResponse_Body) ProtoMessage() {} func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2388,7 +2500,7 @@ type SetShardModeRequest_Body struct { func (x *SetShardModeRequest_Body) Reset() { *x = SetShardModeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2401,7 +2513,7 @@ func (x *SetShardModeRequest_Body) String() string { func (*SetShardModeRequest_Body) ProtoMessage() {} func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2448,7 +2560,7 @@ type SetShardModeResponse_Body struct { func (x *SetShardModeResponse_Body) Reset() { *x = SetShardModeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2461,7 +2573,7 @@ func (x *SetShardModeResponse_Body) String() string { func (*SetShardModeResponse_Body) ProtoMessage() {} func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2492,7 +2604,7 @@ type SynchronizeTreeRequest_Body struct { func (x *SynchronizeTreeRequest_Body) Reset() { *x = SynchronizeTreeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2505,7 +2617,7 @@ func (x *SynchronizeTreeRequest_Body) String() string { func (*SynchronizeTreeRequest_Body) ProtoMessage() {} func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2552,7 +2664,7 @@ type SynchronizeTreeResponse_Body struct { func (x *SynchronizeTreeResponse_Body) Reset() { *x = SynchronizeTreeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2565,7 +2677,7 @@ func (x *SynchronizeTreeResponse_Body) String() string { func (*SynchronizeTreeResponse_Body) ProtoMessage() {} func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2596,7 +2708,7 @@ type EvacuateShardRequest_Body struct { func (x *EvacuateShardRequest_Body) Reset() { *x = EvacuateShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2609,7 +2721,7 @@ func (x *EvacuateShardRequest_Body) String() string { func (*EvacuateShardRequest_Body) ProtoMessage() {} func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2651,7 +2763,7 @@ type EvacuateShardResponse_Body struct { func (x *EvacuateShardResponse_Body) Reset() { *x = EvacuateShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2664,7 +2776,7 @@ func (x *EvacuateShardResponse_Body) String() string { func (*EvacuateShardResponse_Body) ProtoMessage() {} func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2702,7 +2814,7 @@ type FlushCacheRequest_Body struct { func (x *FlushCacheRequest_Body) Reset() { *x = FlushCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2715,7 +2827,7 @@ func (x *FlushCacheRequest_Body) String() string { func (*FlushCacheRequest_Body) ProtoMessage() {} func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2755,7 +2867,7 @@ type FlushCacheResponse_Body struct { func (x *FlushCacheResponse_Body) Reset() { *x = FlushCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2768,7 +2880,7 @@ func (x *FlushCacheResponse_Body) String() string { func (*FlushCacheResponse_Body) ProtoMessage() {} func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2799,7 +2911,7 @@ type DoctorRequest_Body struct { func (x *DoctorRequest_Body) Reset() { *x = DoctorRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2812,7 +2924,7 @@ func (x *DoctorRequest_Body) String() string { func (*DoctorRequest_Body) ProtoMessage() {} func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2852,7 +2964,7 @@ type DoctorResponse_Body struct { func (x *DoctorResponse_Body) Reset() { *x = DoctorResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2865,7 +2977,7 @@ func (x *DoctorResponse_Body) String() string { func (*DoctorResponse_Body) ProtoMessage() {} func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2896,7 +3008,7 @@ type StartShardEvacuationRequest_Body struct { func (x *StartShardEvacuationRequest_Body) Reset() { *x = StartShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2909,7 +3021,7 @@ func (x *StartShardEvacuationRequest_Body) String() string { func (*StartShardEvacuationRequest_Body) ProtoMessage() {} func (x *StartShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2949,7 +3061,7 @@ type StartShardEvacuationResponse_Body struct { func (x *StartShardEvacuationResponse_Body) Reset() { *x = StartShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2962,7 +3074,7 @@ func (x *StartShardEvacuationResponse_Body) String() string { func (*StartShardEvacuationResponse_Body) ProtoMessage() {} func (x *StartShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2988,7 +3100,7 @@ type GetShardEvacuationStatusRequest_Body struct { func (x *GetShardEvacuationStatusRequest_Body) Reset() { *x = GetShardEvacuationStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3001,7 +3113,7 @@ func (x *GetShardEvacuationStatusRequest_Body) String() string { func (*GetShardEvacuationStatusRequest_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3046,7 +3158,7 @@ type GetShardEvacuationStatusResponse_Body struct { func (x *GetShardEvacuationStatusResponse_Body) Reset() { *x = GetShardEvacuationStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3059,7 +3171,7 @@ func (x *GetShardEvacuationStatusResponse_Body) String() string { func (*GetShardEvacuationStatusResponse_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3150,7 +3262,7 @@ type GetShardEvacuationStatusResponse_Body_UnixTimestamp struct { func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) Reset() { *x = GetShardEvacuationStatusResponse_Body_UnixTimestamp{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3163,7 +3275,7 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) String() string { func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3198,7 +3310,7 @@ type GetShardEvacuationStatusResponse_Body_Duration struct { func (x *GetShardEvacuationStatusResponse_Body_Duration) Reset() { *x = GetShardEvacuationStatusResponse_Body_Duration{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3211,7 +3323,7 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) String() string { func (*GetShardEvacuationStatusResponse_Body_Duration) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_Duration) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3244,7 +3356,7 @@ type StopShardEvacuationRequest_Body struct { func (x *StopShardEvacuationRequest_Body) Reset() { *x = StopShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3257,7 +3369,7 @@ func (x *StopShardEvacuationRequest_Body) String() string { func (*StopShardEvacuationRequest_Body) ProtoMessage() {} func (x *StopShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3283,7 +3395,7 @@ type StopShardEvacuationResponse_Body struct { func (x *StopShardEvacuationResponse_Body) Reset() { *x = StopShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3296,7 +3408,7 @@ func (x *StopShardEvacuationResponse_Body) String() string { func (*StopShardEvacuationResponse_Body) ProtoMessage() {} func (x *StopShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3328,7 +3440,7 @@ type AddChainLocalOverrideRequest_Body struct { func (x *AddChainLocalOverrideRequest_Body) Reset() { *x = AddChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3341,7 +3453,7 @@ func (x *AddChainLocalOverrideRequest_Body) String() string { func (*AddChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *AddChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3385,7 +3497,7 @@ type AddChainLocalOverrideResponse_Body struct { func (x *AddChainLocalOverrideResponse_Body) Reset() { *x = AddChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3398,7 +3510,7 @@ func (x *AddChainLocalOverrideResponse_Body) String() string { func (*AddChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *AddChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3435,7 +3547,7 @@ type GetChainLocalOverrideRequest_Body struct { func (x *GetChainLocalOverrideRequest_Body) Reset() { *x = GetChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3448,7 +3560,7 @@ func (x *GetChainLocalOverrideRequest_Body) String() string { func (*GetChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *GetChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[64] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3490,7 +3602,7 @@ type GetChainLocalOverrideResponse_Body struct { func (x *GetChainLocalOverrideResponse_Body) Reset() { *x = GetChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3503,7 +3615,7 @@ func (x *GetChainLocalOverrideResponse_Body) String() string { func (*GetChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *GetChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[65] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3538,7 +3650,7 @@ type ListChainLocalOverridesRequest_Body struct { func (x *ListChainLocalOverridesRequest_Body) Reset() { *x = ListChainLocalOverridesRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3551,7 +3663,7 @@ func (x *ListChainLocalOverridesRequest_Body) String() string { func (*ListChainLocalOverridesRequest_Body) ProtoMessage() {} func (x *ListChainLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[66] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3586,7 +3698,7 @@ type ListChainLocalOverridesResponse_Body struct { func (x *ListChainLocalOverridesResponse_Body) Reset() { *x = ListChainLocalOverridesResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3599,7 +3711,7 @@ func (x *ListChainLocalOverridesResponse_Body) String() string { func (*ListChainLocalOverridesResponse_Body) ProtoMessage() {} func (x *ListChainLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[67] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3622,6 +3734,102 @@ func (x *ListChainLocalOverridesResponse_Body) GetChains() [][]byte { return nil } +type ListTargetsLocalOverridesRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Target for which the overrides are applied. + ChainName string `protobuf:"bytes,1,opt,name=chainName,proto3" json:"chainName,omitempty"` +} + +func (x *ListTargetsLocalOverridesRequest_Body) Reset() { + *x = ListTargetsLocalOverridesRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[68] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListTargetsLocalOverridesRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListTargetsLocalOverridesRequest_Body) ProtoMessage() {} + +func (x *ListTargetsLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[68] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListTargetsLocalOverridesRequest_Body.ProtoReflect.Descriptor instead. +func (*ListTargetsLocalOverridesRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30, 0} +} + +func (x *ListTargetsLocalOverridesRequest_Body) GetChainName() string { + if x != nil { + return x.ChainName + } + return "" +} + +type ListTargetsLocalOverridesResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The list of chain targets. + Targets []*ChainTarget `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty"` +} + +func (x *ListTargetsLocalOverridesResponse_Body) Reset() { + *x = ListTargetsLocalOverridesResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[69] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListTargetsLocalOverridesResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListTargetsLocalOverridesResponse_Body) ProtoMessage() {} + +func (x *ListTargetsLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[69] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListTargetsLocalOverridesResponse_Body.ProtoReflect.Descriptor instead. +func (*ListTargetsLocalOverridesResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31, 0} +} + +func (x *ListTargetsLocalOverridesResponse_Body) GetTargets() []*ChainTarget { + if x != nil { + return x.Targets + } + return nil +} + type RemoveChainLocalOverrideRequest_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3636,7 +3844,7 @@ type RemoveChainLocalOverrideRequest_Body struct { func (x *RemoveChainLocalOverrideRequest_Body) Reset() { *x = RemoveChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[66] + mi := &file_pkg_services_control_service_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3649,7 +3857,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) String() string { func (*RemoveChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[66] + mi := &file_pkg_services_control_service_proto_msgTypes[70] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3662,7 +3870,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Messa // Deprecated: Use RemoveChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32, 0} } func (x *RemoveChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { @@ -3690,7 +3898,7 @@ type RemoveChainLocalOverrideResponse_Body struct { func (x *RemoveChainLocalOverrideResponse_Body) Reset() { *x = RemoveChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[67] + mi := &file_pkg_services_control_service_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3703,7 +3911,7 @@ func (x *RemoveChainLocalOverrideResponse_Body) String() string { func (*RemoveChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[67] + mi := &file_pkg_services_control_service_proto_msgTypes[71] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3716,7 +3924,7 @@ func (x *RemoveChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Mess // Deprecated: Use RemoveChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33, 0} } func (x *RemoveChainLocalOverrideResponse_Body) GetRemoved() bool { @@ -3741,7 +3949,7 @@ type SealWriteCacheRequest_Body struct { func (x *SealWriteCacheRequest_Body) Reset() { *x = SealWriteCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[68] + mi := &file_pkg_services_control_service_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3754,7 +3962,7 @@ func (x *SealWriteCacheRequest_Body) String() string { func (*SealWriteCacheRequest_Body) ProtoMessage() {} func (x *SealWriteCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[68] + mi := &file_pkg_services_control_service_proto_msgTypes[72] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3767,7 +3975,7 @@ func (x *SealWriteCacheRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheRequest_Body.ProtoReflect.Descriptor instead. func (*SealWriteCacheRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34, 0} } func (x *SealWriteCacheRequest_Body) GetShard_ID() [][]byte { @@ -3795,7 +4003,7 @@ type SealWriteCacheResponse_Body struct { func (x *SealWriteCacheResponse_Body) Reset() { *x = SealWriteCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[69] + mi := &file_pkg_services_control_service_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3808,7 +4016,7 @@ func (x *SealWriteCacheResponse_Body) String() string { func (*SealWriteCacheResponse_Body) ProtoMessage() {} func (x *SealWriteCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[69] + mi := &file_pkg_services_control_service_proto_msgTypes[73] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3821,7 +4029,7 @@ func (x *SealWriteCacheResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheResponse_Body.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35, 0} } func (x *SealWriteCacheResponse_Body) GetResults() []*SealWriteCacheResponse_Body_Status { @@ -3844,7 +4052,7 @@ type SealWriteCacheResponse_Body_Status struct { func (x *SealWriteCacheResponse_Body_Status) Reset() { *x = SealWriteCacheResponse_Body_Status{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[70] + mi := &file_pkg_services_control_service_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3857,7 +4065,7 @@ func (x *SealWriteCacheResponse_Body_Status) String() string { func (*SealWriteCacheResponse_Body_Status) ProtoMessage() {} func (x *SealWriteCacheResponse_Body_Status) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[70] + mi := &file_pkg_services_control_service_proto_msgTypes[74] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3870,7 +4078,7 @@ func (x *SealWriteCacheResponse_Body_Status) ProtoReflect() protoreflect.Message // Deprecated: Use SealWriteCacheResponse_Body_Status.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse_Body_Status) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33, 0, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35, 0, 0} } func (x *SealWriteCacheResponse_Body_Status) GetShard_ID() []byte { @@ -4273,165 +4481,197 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, - 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x20, 0x4c, 0x69, 0x73, 0x74, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x24, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, + 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xd2, 0x01, 0x0a, 0x21, 0x4c, 0x69, 0x73, 0x74, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x1a, 0x36, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2e, 0x0a, 0x07, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x22, 0xe7, 0x01, 0x0a, + 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, + 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, + 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, + 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xba, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, - 0x22, 0xba, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x20, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x22, 0xca, 0x01, - 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, - 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, - 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, - 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, - 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, - 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, + 0x65, 0x1a, 0x20, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x64, 0x22, 0xca, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, + 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, - 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, - 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0xf0, 0x0b, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, - 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, - 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, - 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, - 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, - 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, + 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, + 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, + 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, + 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, + 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, + 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, + 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, + 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0xe4, 0x0c, 0x0a, + 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, + 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, + 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, + 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, + 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, + 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, + 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, + 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, - 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, - 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, + 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, + 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, + 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, + 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, - 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, - 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, - 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, - 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, + 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, + 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, + 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, + 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, + 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, + 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, + 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, + 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4447,7 +4687,7 @@ func file_pkg_services_control_service_proto_rawDescGZIP() []byte { } var file_pkg_services_control_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 71) +var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 75) var file_pkg_services_control_service_proto_goTypes = []interface{}{ (GetShardEvacuationStatusResponse_Body_Status)(0), // 0: control.GetShardEvacuationStatusResponse.Body.Status (*HealthCheckRequest)(nil), // 1: control.HealthCheckRequest @@ -4480,175 +4720,186 @@ var file_pkg_services_control_service_proto_goTypes = []interface{}{ (*GetChainLocalOverrideResponse)(nil), // 28: control.GetChainLocalOverrideResponse (*ListChainLocalOverridesRequest)(nil), // 29: control.ListChainLocalOverridesRequest (*ListChainLocalOverridesResponse)(nil), // 30: control.ListChainLocalOverridesResponse - (*RemoveChainLocalOverrideRequest)(nil), // 31: control.RemoveChainLocalOverrideRequest - (*RemoveChainLocalOverrideResponse)(nil), // 32: control.RemoveChainLocalOverrideResponse - (*SealWriteCacheRequest)(nil), // 33: control.SealWriteCacheRequest - (*SealWriteCacheResponse)(nil), // 34: control.SealWriteCacheResponse - (*HealthCheckRequest_Body)(nil), // 35: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 36: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 37: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 38: control.SetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 39: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 40: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 41: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 42: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 43: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 44: control.SetShardModeResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 45: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 46: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 47: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 48: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 49: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 50: control.FlushCacheResponse.Body - (*DoctorRequest_Body)(nil), // 51: control.DoctorRequest.Body - (*DoctorResponse_Body)(nil), // 52: control.DoctorResponse.Body - (*StartShardEvacuationRequest_Body)(nil), // 53: control.StartShardEvacuationRequest.Body - (*StartShardEvacuationResponse_Body)(nil), // 54: control.StartShardEvacuationResponse.Body - (*GetShardEvacuationStatusRequest_Body)(nil), // 55: control.GetShardEvacuationStatusRequest.Body - (*GetShardEvacuationStatusResponse_Body)(nil), // 56: control.GetShardEvacuationStatusResponse.Body - (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 57: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 58: control.GetShardEvacuationStatusResponse.Body.Duration - (*StopShardEvacuationRequest_Body)(nil), // 59: control.StopShardEvacuationRequest.Body - (*StopShardEvacuationResponse_Body)(nil), // 60: control.StopShardEvacuationResponse.Body - (*AddChainLocalOverrideRequest_Body)(nil), // 61: control.AddChainLocalOverrideRequest.Body - (*AddChainLocalOverrideResponse_Body)(nil), // 62: control.AddChainLocalOverrideResponse.Body - (*GetChainLocalOverrideRequest_Body)(nil), // 63: control.GetChainLocalOverrideRequest.Body - (*GetChainLocalOverrideResponse_Body)(nil), // 64: control.GetChainLocalOverrideResponse.Body - (*ListChainLocalOverridesRequest_Body)(nil), // 65: control.ListChainLocalOverridesRequest.Body - (*ListChainLocalOverridesResponse_Body)(nil), // 66: control.ListChainLocalOverridesResponse.Body - (*RemoveChainLocalOverrideRequest_Body)(nil), // 67: control.RemoveChainLocalOverrideRequest.Body - (*RemoveChainLocalOverrideResponse_Body)(nil), // 68: control.RemoveChainLocalOverrideResponse.Body - (*SealWriteCacheRequest_Body)(nil), // 69: control.SealWriteCacheRequest.Body - (*SealWriteCacheResponse_Body)(nil), // 70: control.SealWriteCacheResponse.Body - (*SealWriteCacheResponse_Body_Status)(nil), // 71: control.SealWriteCacheResponse.Body.Status - (*Signature)(nil), // 72: control.Signature - (NetmapStatus)(0), // 73: control.NetmapStatus - (HealthStatus)(0), // 74: control.HealthStatus - (*ShardInfo)(nil), // 75: control.ShardInfo - (ShardMode)(0), // 76: control.ShardMode - (*ChainTarget)(nil), // 77: control.ChainTarget + (*ListTargetsLocalOverridesRequest)(nil), // 31: control.ListTargetsLocalOverridesRequest + (*ListTargetsLocalOverridesResponse)(nil), // 32: control.ListTargetsLocalOverridesResponse + (*RemoveChainLocalOverrideRequest)(nil), // 33: control.RemoveChainLocalOverrideRequest + (*RemoveChainLocalOverrideResponse)(nil), // 34: control.RemoveChainLocalOverrideResponse + (*SealWriteCacheRequest)(nil), // 35: control.SealWriteCacheRequest + (*SealWriteCacheResponse)(nil), // 36: control.SealWriteCacheResponse + (*HealthCheckRequest_Body)(nil), // 37: control.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 38: control.HealthCheckResponse.Body + (*SetNetmapStatusRequest_Body)(nil), // 39: control.SetNetmapStatusRequest.Body + (*SetNetmapStatusResponse_Body)(nil), // 40: control.SetNetmapStatusResponse.Body + (*DropObjectsRequest_Body)(nil), // 41: control.DropObjectsRequest.Body + (*DropObjectsResponse_Body)(nil), // 42: control.DropObjectsResponse.Body + (*ListShardsRequest_Body)(nil), // 43: control.ListShardsRequest.Body + (*ListShardsResponse_Body)(nil), // 44: control.ListShardsResponse.Body + (*SetShardModeRequest_Body)(nil), // 45: control.SetShardModeRequest.Body + (*SetShardModeResponse_Body)(nil), // 46: control.SetShardModeResponse.Body + (*SynchronizeTreeRequest_Body)(nil), // 47: control.SynchronizeTreeRequest.Body + (*SynchronizeTreeResponse_Body)(nil), // 48: control.SynchronizeTreeResponse.Body + (*EvacuateShardRequest_Body)(nil), // 49: control.EvacuateShardRequest.Body + (*EvacuateShardResponse_Body)(nil), // 50: control.EvacuateShardResponse.Body + (*FlushCacheRequest_Body)(nil), // 51: control.FlushCacheRequest.Body + (*FlushCacheResponse_Body)(nil), // 52: control.FlushCacheResponse.Body + (*DoctorRequest_Body)(nil), // 53: control.DoctorRequest.Body + (*DoctorResponse_Body)(nil), // 54: control.DoctorResponse.Body + (*StartShardEvacuationRequest_Body)(nil), // 55: control.StartShardEvacuationRequest.Body + (*StartShardEvacuationResponse_Body)(nil), // 56: control.StartShardEvacuationResponse.Body + (*GetShardEvacuationStatusRequest_Body)(nil), // 57: control.GetShardEvacuationStatusRequest.Body + (*GetShardEvacuationStatusResponse_Body)(nil), // 58: control.GetShardEvacuationStatusResponse.Body + (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 59: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 60: control.GetShardEvacuationStatusResponse.Body.Duration + (*StopShardEvacuationRequest_Body)(nil), // 61: control.StopShardEvacuationRequest.Body + (*StopShardEvacuationResponse_Body)(nil), // 62: control.StopShardEvacuationResponse.Body + (*AddChainLocalOverrideRequest_Body)(nil), // 63: control.AddChainLocalOverrideRequest.Body + (*AddChainLocalOverrideResponse_Body)(nil), // 64: control.AddChainLocalOverrideResponse.Body + (*GetChainLocalOverrideRequest_Body)(nil), // 65: control.GetChainLocalOverrideRequest.Body + (*GetChainLocalOverrideResponse_Body)(nil), // 66: control.GetChainLocalOverrideResponse.Body + (*ListChainLocalOverridesRequest_Body)(nil), // 67: control.ListChainLocalOverridesRequest.Body + (*ListChainLocalOverridesResponse_Body)(nil), // 68: control.ListChainLocalOverridesResponse.Body + (*ListTargetsLocalOverridesRequest_Body)(nil), // 69: control.ListTargetsLocalOverridesRequest.Body + (*ListTargetsLocalOverridesResponse_Body)(nil), // 70: control.ListTargetsLocalOverridesResponse.Body + (*RemoveChainLocalOverrideRequest_Body)(nil), // 71: control.RemoveChainLocalOverrideRequest.Body + (*RemoveChainLocalOverrideResponse_Body)(nil), // 72: control.RemoveChainLocalOverrideResponse.Body + (*SealWriteCacheRequest_Body)(nil), // 73: control.SealWriteCacheRequest.Body + (*SealWriteCacheResponse_Body)(nil), // 74: control.SealWriteCacheResponse.Body + (*SealWriteCacheResponse_Body_Status)(nil), // 75: control.SealWriteCacheResponse.Body.Status + (*Signature)(nil), // 76: control.Signature + (NetmapStatus)(0), // 77: control.NetmapStatus + (HealthStatus)(0), // 78: control.HealthStatus + (*ShardInfo)(nil), // 79: control.ShardInfo + (ShardMode)(0), // 80: control.ShardMode + (*ChainTarget)(nil), // 81: control.ChainTarget } var file_pkg_services_control_service_proto_depIdxs = []int32{ - 35, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 72, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 36, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 72, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 37, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 72, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 38, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 72, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 39, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 72, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature - 40, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 72, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature - 41, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 72, // 13: control.ListShardsRequest.signature:type_name -> control.Signature - 42, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 72, // 15: control.ListShardsResponse.signature:type_name -> control.Signature - 43, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 72, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature - 44, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 72, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature - 45, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 72, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 46, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 72, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 47, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 72, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature - 48, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 72, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature - 49, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 72, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature - 50, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 72, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature - 51, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body - 72, // 33: control.DoctorRequest.signature:type_name -> control.Signature - 52, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body - 72, // 35: control.DoctorResponse.signature:type_name -> control.Signature - 53, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body - 72, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature - 54, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body - 72, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature - 55, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body - 72, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 56, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body - 72, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 59, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body - 72, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature - 60, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body - 72, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature - 61, // 48: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body - 72, // 49: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature - 62, // 50: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body - 72, // 51: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature - 63, // 52: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body - 72, // 53: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature - 64, // 54: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body - 72, // 55: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature - 65, // 56: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body - 72, // 57: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature - 66, // 58: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body - 72, // 59: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature - 67, // 60: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body - 72, // 61: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature - 68, // 62: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body - 72, // 63: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature - 69, // 64: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body - 72, // 65: control.SealWriteCacheRequest.signature:type_name -> control.Signature - 70, // 66: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body - 72, // 67: control.SealWriteCacheResponse.signature:type_name -> control.Signature - 73, // 68: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 74, // 69: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 73, // 70: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 75, // 71: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 76, // 72: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 0, // 73: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status - 58, // 74: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration - 57, // 75: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - 77, // 76: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 77, // 77: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 77, // 78: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget - 77, // 79: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 71, // 80: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status - 1, // 81: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 3, // 82: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 5, // 83: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 7, // 84: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 9, // 85: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 11, // 86: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 13, // 87: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 19, // 88: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest - 21, // 89: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest - 23, // 90: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest - 15, // 91: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 17, // 92: control.ControlService.Doctor:input_type -> control.DoctorRequest - 25, // 93: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest - 27, // 94: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest - 29, // 95: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest - 31, // 96: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest - 33, // 97: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest - 2, // 98: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 4, // 99: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 6, // 100: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 8, // 101: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 10, // 102: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 12, // 103: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 14, // 104: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 20, // 105: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse - 22, // 106: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse - 24, // 107: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse - 16, // 108: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 18, // 109: control.ControlService.Doctor:output_type -> control.DoctorResponse - 26, // 110: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse - 28, // 111: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse - 30, // 112: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse - 32, // 113: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse - 34, // 114: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse - 98, // [98:115] is the sub-list for method output_type - 81, // [81:98] is the sub-list for method input_type - 81, // [81:81] is the sub-list for extension type_name - 81, // [81:81] is the sub-list for extension extendee - 0, // [0:81] is the sub-list for field type_name + 37, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body + 76, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature + 38, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body + 76, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature + 39, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body + 76, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature + 40, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body + 76, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature + 41, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body + 76, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature + 42, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body + 76, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature + 43, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body + 76, // 13: control.ListShardsRequest.signature:type_name -> control.Signature + 44, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body + 76, // 15: control.ListShardsResponse.signature:type_name -> control.Signature + 45, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body + 76, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature + 46, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body + 76, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature + 47, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body + 76, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature + 48, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body + 76, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature + 49, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body + 76, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature + 50, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body + 76, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature + 51, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body + 76, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature + 52, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body + 76, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature + 53, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body + 76, // 33: control.DoctorRequest.signature:type_name -> control.Signature + 54, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body + 76, // 35: control.DoctorResponse.signature:type_name -> control.Signature + 55, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body + 76, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature + 56, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body + 76, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature + 57, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body + 76, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 58, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body + 76, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 61, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body + 76, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature + 62, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body + 76, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature + 63, // 48: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body + 76, // 49: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature + 64, // 50: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body + 76, // 51: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature + 65, // 52: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body + 76, // 53: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature + 66, // 54: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body + 76, // 55: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature + 67, // 56: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body + 76, // 57: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature + 68, // 58: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body + 76, // 59: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature + 69, // 60: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body + 76, // 61: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature + 70, // 62: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body + 76, // 63: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature + 71, // 64: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body + 76, // 65: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature + 72, // 66: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body + 76, // 67: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature + 73, // 68: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body + 76, // 69: control.SealWriteCacheRequest.signature:type_name -> control.Signature + 74, // 70: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body + 76, // 71: control.SealWriteCacheResponse.signature:type_name -> control.Signature + 77, // 72: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus + 78, // 73: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus + 77, // 74: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus + 79, // 75: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo + 80, // 76: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode + 0, // 77: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status + 60, // 78: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration + 59, // 79: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + 81, // 80: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 81, // 81: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 81, // 82: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget + 81, // 83: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget + 81, // 84: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 75, // 85: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status + 1, // 86: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 3, // 87: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 5, // 88: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 7, // 89: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 9, // 90: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 11, // 91: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 13, // 92: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 19, // 93: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest + 21, // 94: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest + 23, // 95: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest + 15, // 96: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 17, // 97: control.ControlService.Doctor:input_type -> control.DoctorRequest + 25, // 98: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest + 27, // 99: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest + 29, // 100: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest + 33, // 101: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest + 31, // 102: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest + 35, // 103: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest + 2, // 104: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 4, // 105: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 6, // 106: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 8, // 107: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 10, // 108: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 12, // 109: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 14, // 110: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 20, // 111: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse + 22, // 112: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse + 24, // 113: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse + 16, // 114: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 18, // 115: control.ControlService.Doctor:output_type -> control.DoctorResponse + 26, // 116: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse + 28, // 117: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse + 30, // 118: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse + 34, // 119: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse + 32, // 120: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse + 36, // 121: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse + 104, // [104:122] is the sub-list for method output_type + 86, // [86:104] is the sub-list for method input_type + 86, // [86:86] is the sub-list for extension type_name + 86, // [86:86] is the sub-list for extension extendee + 0, // [0:86] is the sub-list for field type_name } func init() { file_pkg_services_control_service_proto_init() } @@ -5019,7 +5270,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideRequest); i { + switch v := v.(*ListTargetsLocalOverridesRequest); i { case 0: return &v.state case 1: @@ -5031,7 +5282,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideResponse); i { + switch v := v.(*ListTargetsLocalOverridesResponse); i { case 0: return &v.state case 1: @@ -5043,7 +5294,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheRequest); i { + switch v := v.(*RemoveChainLocalOverrideRequest); i { case 0: return &v.state case 1: @@ -5055,7 +5306,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse); i { + switch v := v.(*RemoveChainLocalOverrideResponse); i { case 0: return &v.state case 1: @@ -5067,7 +5318,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*SealWriteCacheRequest); i { case 0: return &v.state case 1: @@ -5079,7 +5330,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { + switch v := v.(*SealWriteCacheResponse); i { case 0: return &v.state case 1: @@ -5091,7 +5342,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusRequest_Body); i { + switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state case 1: @@ -5103,7 +5354,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusResponse_Body); i { + switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state case 1: @@ -5115,7 +5366,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest_Body); i { + switch v := v.(*SetNetmapStatusRequest_Body); i { case 0: return &v.state case 1: @@ -5127,7 +5378,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse_Body); i { + switch v := v.(*SetNetmapStatusResponse_Body); i { case 0: return &v.state case 1: @@ -5139,7 +5390,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest_Body); i { + switch v := v.(*DropObjectsRequest_Body); i { case 0: return &v.state case 1: @@ -5151,7 +5402,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse_Body); i { + switch v := v.(*DropObjectsResponse_Body); i { case 0: return &v.state case 1: @@ -5163,7 +5414,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest_Body); i { + switch v := v.(*ListShardsRequest_Body); i { case 0: return &v.state case 1: @@ -5175,7 +5426,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse_Body); i { + switch v := v.(*ListShardsResponse_Body); i { case 0: return &v.state case 1: @@ -5187,7 +5438,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest_Body); i { + switch v := v.(*SetShardModeRequest_Body); i { case 0: return &v.state case 1: @@ -5199,7 +5450,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse_Body); i { + switch v := v.(*SetShardModeResponse_Body); i { case 0: return &v.state case 1: @@ -5211,7 +5462,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest_Body); i { + switch v := v.(*SynchronizeTreeRequest_Body); i { case 0: return &v.state case 1: @@ -5223,7 +5474,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse_Body); i { + switch v := v.(*SynchronizeTreeResponse_Body); i { case 0: return &v.state case 1: @@ -5235,7 +5486,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest_Body); i { + switch v := v.(*EvacuateShardRequest_Body); i { case 0: return &v.state case 1: @@ -5247,7 +5498,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheResponse_Body); i { + switch v := v.(*EvacuateShardResponse_Body); i { case 0: return &v.state case 1: @@ -5259,7 +5510,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorRequest_Body); i { + switch v := v.(*FlushCacheRequest_Body); i { case 0: return &v.state case 1: @@ -5271,7 +5522,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorResponse_Body); i { + switch v := v.(*FlushCacheResponse_Body); i { case 0: return &v.state case 1: @@ -5283,7 +5534,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationRequest_Body); i { + switch v := v.(*DoctorRequest_Body); i { case 0: return &v.state case 1: @@ -5295,7 +5546,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationResponse_Body); i { + switch v := v.(*DoctorResponse_Body); i { case 0: return &v.state case 1: @@ -5307,7 +5558,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusRequest_Body); i { + switch v := v.(*StartShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -5319,7 +5570,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body); i { + switch v := v.(*StartShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -5331,7 +5582,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { + switch v := v.(*GetShardEvacuationStatusRequest_Body); i { case 0: return &v.state case 1: @@ -5343,7 +5594,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body); i { case 0: return &v.state case 1: @@ -5355,7 +5606,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationRequest_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { case 0: return &v.state case 1: @@ -5367,7 +5618,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationResponse_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { case 0: return &v.state case 1: @@ -5379,7 +5630,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideRequest_Body); i { + switch v := v.(*StopShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -5391,7 +5642,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideResponse_Body); i { + switch v := v.(*StopShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -5403,7 +5654,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideRequest_Body); i { + switch v := v.(*AddChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -5415,7 +5666,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideResponse_Body); i { + switch v := v.(*AddChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -5427,7 +5678,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesRequest_Body); i { + switch v := v.(*GetChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -5439,7 +5690,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesResponse_Body); i { + switch v := v.(*GetChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -5451,7 +5702,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { + switch v := v.(*ListChainLocalOverridesRequest_Body); i { case 0: return &v.state case 1: @@ -5463,7 +5714,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { + switch v := v.(*ListChainLocalOverridesResponse_Body); i { case 0: return &v.state case 1: @@ -5475,7 +5726,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheRequest_Body); i { + switch v := v.(*ListTargetsLocalOverridesRequest_Body); i { case 0: return &v.state case 1: @@ -5487,7 +5738,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse_Body); i { + switch v := v.(*ListTargetsLocalOverridesResponse_Body); i { case 0: return &v.state case 1: @@ -5499,6 +5750,54 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SealWriteCacheResponse_Body_Status); i { case 0: return &v.state @@ -5517,7 +5816,7 @@ func file_pkg_services_control_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_service_proto_rawDesc, NumEnums: 1, - NumMessages: 71, + NumMessages: 75, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index b76645b61..dc63a7188 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -57,6 +57,9 @@ service ControlService { // Remove local access policy engine overrides stored in the node by chaind id. rpc RemoveChainLocalOverride (RemoveChainLocalOverrideRequest) returns (RemoveChainLocalOverrideResponse); + // List targets of the local APE overrides stored in the node. + rpc ListTargetsLocalOverrides (ListTargetsLocalOverridesRequest) returns (ListTargetsLocalOverridesResponse); + // Flush objects from write-cache and move it to degraded read only mode. rpc SealWriteCache(SealWriteCacheRequest) returns (SealWriteCacheResponse); } @@ -505,6 +508,30 @@ message ListChainLocalOverridesResponse { Signature signature = 2; } +// ListTargetsLocalOverrides request. +message ListTargetsLocalOverridesRequest { + message Body { + // Target for which the overrides are applied. + string chainName = 1; + } + + Body body = 1; + + Signature signature = 2; +} + +// ListTargetsLocalOverrides response. +message ListTargetsLocalOverridesResponse { + message Body { + // The list of chain targets. + repeated ChainTarget targets = 1; + } + + Body body = 1; + + Signature signature = 2; +} + message RemoveChainLocalOverrideRequest { message Body { // Target for which the overrides are applied. @@ -556,4 +583,4 @@ message SealWriteCacheResponse { Body body = 1; Signature signature = 2; -} \ No newline at end of file +} diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 574ab13c4..b72fc0f12 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -2582,6 +2582,182 @@ func (x *ListChainLocalOverridesResponse) SetSignature(sig *Signature) { x.Signature = sig } +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListTargetsLocalOverridesRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.StringSize(1, x.ChainName) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ListTargetsLocalOverridesRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.StringMarshal(1, buf[offset:], x.ChainName) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListTargetsLocalOverridesRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ListTargetsLocalOverridesRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *ListTargetsLocalOverridesRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *ListTargetsLocalOverridesRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *ListTargetsLocalOverridesRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListTargetsLocalOverridesResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + for i := range x.Targets { + size += proto.NestedStructureSize(1, x.Targets[i]) + } + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ListTargetsLocalOverridesResponse_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + for i := range x.Targets { + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Targets[i]) + } + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListTargetsLocalOverridesResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ListTargetsLocalOverridesResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *ListTargetsLocalOverridesResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *ListTargetsLocalOverridesResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *ListTargetsLocalOverridesResponse) SetSignature(sig *Signature) { + x.Signature = sig +} + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index e09f6750b..89337323f 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -19,23 +19,24 @@ import ( const _ = grpc.SupportPackageIsVersion7 const ( - ControlService_HealthCheck_FullMethodName = "/control.ControlService/HealthCheck" - ControlService_SetNetmapStatus_FullMethodName = "/control.ControlService/SetNetmapStatus" - ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects" - ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" - ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" - ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree" - ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard" - ControlService_StartShardEvacuation_FullMethodName = "/control.ControlService/StartShardEvacuation" - ControlService_GetShardEvacuationStatus_FullMethodName = "/control.ControlService/GetShardEvacuationStatus" - ControlService_StopShardEvacuation_FullMethodName = "/control.ControlService/StopShardEvacuation" - ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache" - ControlService_Doctor_FullMethodName = "/control.ControlService/Doctor" - ControlService_AddChainLocalOverride_FullMethodName = "/control.ControlService/AddChainLocalOverride" - ControlService_GetChainLocalOverride_FullMethodName = "/control.ControlService/GetChainLocalOverride" - ControlService_ListChainLocalOverrides_FullMethodName = "/control.ControlService/ListChainLocalOverrides" - ControlService_RemoveChainLocalOverride_FullMethodName = "/control.ControlService/RemoveChainLocalOverride" - ControlService_SealWriteCache_FullMethodName = "/control.ControlService/SealWriteCache" + ControlService_HealthCheck_FullMethodName = "/control.ControlService/HealthCheck" + ControlService_SetNetmapStatus_FullMethodName = "/control.ControlService/SetNetmapStatus" + ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects" + ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" + ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" + ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree" + ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard" + ControlService_StartShardEvacuation_FullMethodName = "/control.ControlService/StartShardEvacuation" + ControlService_GetShardEvacuationStatus_FullMethodName = "/control.ControlService/GetShardEvacuationStatus" + ControlService_StopShardEvacuation_FullMethodName = "/control.ControlService/StopShardEvacuation" + ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache" + ControlService_Doctor_FullMethodName = "/control.ControlService/Doctor" + ControlService_AddChainLocalOverride_FullMethodName = "/control.ControlService/AddChainLocalOverride" + ControlService_GetChainLocalOverride_FullMethodName = "/control.ControlService/GetChainLocalOverride" + ControlService_ListChainLocalOverrides_FullMethodName = "/control.ControlService/ListChainLocalOverrides" + ControlService_RemoveChainLocalOverride_FullMethodName = "/control.ControlService/RemoveChainLocalOverride" + ControlService_ListTargetsLocalOverrides_FullMethodName = "/control.ControlService/ListTargetsLocalOverrides" + ControlService_SealWriteCache_FullMethodName = "/control.ControlService/SealWriteCache" ) // ControlServiceClient is the client API for ControlService service. @@ -75,6 +76,8 @@ type ControlServiceClient interface { ListChainLocalOverrides(ctx context.Context, in *ListChainLocalOverridesRequest, opts ...grpc.CallOption) (*ListChainLocalOverridesResponse, error) // Remove local access policy engine overrides stored in the node by chaind id. RemoveChainLocalOverride(ctx context.Context, in *RemoveChainLocalOverrideRequest, opts ...grpc.CallOption) (*RemoveChainLocalOverrideResponse, error) + // List targets of the local APE overrides stored in the node. + ListTargetsLocalOverrides(ctx context.Context, in *ListTargetsLocalOverridesRequest, opts ...grpc.CallOption) (*ListTargetsLocalOverridesResponse, error) // Flush objects from write-cache and move it to degraded read only mode. SealWriteCache(ctx context.Context, in *SealWriteCacheRequest, opts ...grpc.CallOption) (*SealWriteCacheResponse, error) } @@ -231,6 +234,15 @@ func (c *controlServiceClient) RemoveChainLocalOverride(ctx context.Context, in return out, nil } +func (c *controlServiceClient) ListTargetsLocalOverrides(ctx context.Context, in *ListTargetsLocalOverridesRequest, opts ...grpc.CallOption) (*ListTargetsLocalOverridesResponse, error) { + out := new(ListTargetsLocalOverridesResponse) + err := c.cc.Invoke(ctx, ControlService_ListTargetsLocalOverrides_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *controlServiceClient) SealWriteCache(ctx context.Context, in *SealWriteCacheRequest, opts ...grpc.CallOption) (*SealWriteCacheResponse, error) { out := new(SealWriteCacheResponse) err := c.cc.Invoke(ctx, ControlService_SealWriteCache_FullMethodName, in, out, opts...) @@ -277,6 +289,8 @@ type ControlServiceServer interface { ListChainLocalOverrides(context.Context, *ListChainLocalOverridesRequest) (*ListChainLocalOverridesResponse, error) // Remove local access policy engine overrides stored in the node by chaind id. RemoveChainLocalOverride(context.Context, *RemoveChainLocalOverrideRequest) (*RemoveChainLocalOverrideResponse, error) + // List targets of the local APE overrides stored in the node. + ListTargetsLocalOverrides(context.Context, *ListTargetsLocalOverridesRequest) (*ListTargetsLocalOverridesResponse, error) // Flush objects from write-cache and move it to degraded read only mode. SealWriteCache(context.Context, *SealWriteCacheRequest) (*SealWriteCacheResponse, error) } @@ -333,6 +347,9 @@ func (UnimplementedControlServiceServer) ListChainLocalOverrides(context.Context func (UnimplementedControlServiceServer) RemoveChainLocalOverride(context.Context, *RemoveChainLocalOverrideRequest) (*RemoveChainLocalOverrideResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method RemoveChainLocalOverride not implemented") } +func (UnimplementedControlServiceServer) ListTargetsLocalOverrides(context.Context, *ListTargetsLocalOverridesRequest) (*ListTargetsLocalOverridesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListTargetsLocalOverrides not implemented") +} func (UnimplementedControlServiceServer) SealWriteCache(context.Context, *SealWriteCacheRequest) (*SealWriteCacheResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method SealWriteCache not implemented") } @@ -636,6 +653,24 @@ func _ControlService_RemoveChainLocalOverride_Handler(srv interface{}, ctx conte return interceptor(ctx, in, info, handler) } +func _ControlService_ListTargetsLocalOverrides_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListTargetsLocalOverridesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).ListTargetsLocalOverrides(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_ListTargetsLocalOverrides_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).ListTargetsLocalOverrides(ctx, req.(*ListTargetsLocalOverridesRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _ControlService_SealWriteCache_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(SealWriteCacheRequest) if err := dec(in); err != nil { @@ -725,6 +760,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "RemoveChainLocalOverride", Handler: _ControlService_RemoveChainLocalOverride_Handler, }, + { + MethodName: "ListTargetsLocalOverrides", + Handler: _ControlService_ListTargetsLocalOverrides_Handler, + }, { MethodName: "SealWriteCache", Handler: _ControlService_SealWriteCache_Handler, From f64c48b1578f8726c83a405be7fed670fc06bc30 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 26 Jan 2024 17:13:40 +0300 Subject: [PATCH 0839/1943] [#922] Fix linter issue Signed-off-by: Anton Nikiforov --- Makefile | 2 +- README.md | 2 +- dev/.vscode-example/launch.json | 2 +- dev/.vscode-example/tasks.json | 2 +- dev/ir/az.json | 1 - 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 84653a03a..4ff42b7e4 100755 --- a/Makefile +++ b/Makefile @@ -259,4 +259,4 @@ env-down: docker volume rm -f frostfs-node_neo-go rm -f ./.cache/.frostfs-ir-state rm -f ./.cache/.frostfs-node-state - rm -rf ./.cache/storage \ No newline at end of file + rm -rf ./.cache/storage diff --git a/README.md b/README.md index ff816f2c7..ecd162a85 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ Enter password > <- press ENTER, the is no password for wallet CID: CfPhEuHQ2PRvM4gfBQDC4dWZY3NccovyfcnEdiq2ixju ./bin/frostfs-cli object put -r 127.0.0.1:8080 --wallet ./dev/wallet.json --file README.md --cid CfPhEuHQ2PRvM4gfBQDC4dWZY3NccovyfcnEdiq2ixju -Enter password > +Enter password > 4300 / 4300 [===========================================================================================================================================================================================================] 100.00% 0s [README.md] Object successfully stored OID: 78sohnudVMnPsczXqsTUcvezosan2YDNVZwDE8Kq5YwU diff --git a/dev/.vscode-example/launch.json b/dev/.vscode-example/launch.json index cf91c0d24..07d39482e 100644 --- a/dev/.vscode-example/launch.json +++ b/dev/.vscode-example/launch.json @@ -86,4 +86,4 @@ "stopAll": true } ] -} \ No newline at end of file +} diff --git a/dev/.vscode-example/tasks.json b/dev/.vscode-example/tasks.json index 8c6099a05..0ccd9d110 100644 --- a/dev/.vscode-example/tasks.json +++ b/dev/.vscode-example/tasks.json @@ -16,4 +16,4 @@ "detail": "Down debug environment" } ] -} \ No newline at end of file +} diff --git a/dev/ir/az.json b/dev/ir/az.json index a16aec295..8e88b432c 100644 --- a/dev/ir/az.json +++ b/dev/ir/az.json @@ -66,4 +66,3 @@ "Tokens": null } } - From befbaf9d56e734b224e3aed0b118daa96865ad48 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 26 Jan 2024 17:15:28 +0300 Subject: [PATCH 0840/1943] [#922] cli: Add new command `control list-targets` Signed-off-by: Anton Nikiforov --- .../modules/control/list_targets.go | 82 +++++++++++++++++++ cmd/frostfs-cli/modules/control/root.go | 2 + 2 files changed, 84 insertions(+) create mode 100644 cmd/frostfs-cli/modules/control/list_targets.go diff --git a/cmd/frostfs-cli/modules/control/list_targets.go b/cmd/frostfs-cli/modules/control/list_targets.go new file mode 100644 index 000000000..29033d017 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/list_targets.go @@ -0,0 +1,82 @@ +package control + +import ( + "bytes" + "crypto/sha256" + "fmt" + "strconv" + "text/tabwriter" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/spf13/cobra" +) + +const ( + chainNameFlag = "chain-name" + chainNameFlagUsage = "Chain name(ingress|s3)" +) + +var listTargetsCmd = &cobra.Command{ + Use: "list-targets", + Short: "List local targets", + Long: "List local APE overrides of the node", + Run: listTargets, +} + +func listTargets(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + var cnr cid.ID + chainName, _ := cmd.Flags().GetString(chainNameFlag) + + rawCID := make([]byte, sha256.Size) + cnr.Encode(rawCID) + + req := &control.ListTargetsLocalOverridesRequest{ + Body: &control.ListTargetsLocalOverridesRequest_Body{ + ChainName: chainName, + }, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.ListTargetsLocalOverridesResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.ListTargetsLocalOverrides(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + targets := resp.GetBody().GetTargets() + if len(targets) == 0 { + cmd.Println("Local overrides are not defined for the container.") + return + } + + buf := bytes.NewBuffer(nil) + tw := tabwriter.NewWriter(buf, 0, 2, 2, ' ', 0) + _, _ = tw.Write([]byte("#\tName\tType\n")) + for i, t := range targets { + _, _ = tw.Write([]byte(fmt.Sprintf("%s\t%s\t%s\n", strconv.Itoa(i), t.GetName(), t.GetType()))) + } + _ = tw.Flush() + cmd.Print(buf.String()) +} + +func initControlListTargetsCmd() { + initControlFlags(listTargetsCmd) + + ff := listTargetsCmd.Flags() + ff.String(chainNameFlag, "", chainNameFlagUsage) + + _ = cobra.MarkFlagRequired(ff, chainNameFlag) +} diff --git a/cmd/frostfs-cli/modules/control/root.go b/cmd/frostfs-cli/modules/control/root.go index f7c846eba..41222fc00 100644 --- a/cmd/frostfs-cli/modules/control/root.go +++ b/cmd/frostfs-cli/modules/control/root.go @@ -38,6 +38,7 @@ func init() { removeRuleCmd, listRulesCmd, getRuleCmd, + listTargetsCmd, ) initControlHealthCheckCmd() @@ -50,4 +51,5 @@ func init() { initControlRemoveRuleCmd() initControlListRulesCmd() initControGetRuleCmd() + initControlListTargetsCmd() } From c1a5b831b61e8c5004a7861f229906cf94dd33c4 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 2 Feb 2024 20:26:14 +0300 Subject: [PATCH 0841/1943] [#955] chainbase: Fix rule chain unmarshalling * Use correct way DecodeBytes instead unmarshalling by json. Signed-off-by: Airat Arifullin --- pkg/ape/chainbase/boltdb.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/ape/chainbase/boltdb.go b/pkg/ape/chainbase/boltdb.go index c26156c7a..a1dfc2567 100644 --- a/pkg/ape/chainbase/boltdb.go +++ b/pkg/ape/chainbase/boltdb.go @@ -2,7 +2,6 @@ package chainbase import ( "context" - "encoding/json" "errors" "fmt" "path/filepath" @@ -203,7 +202,7 @@ func (cs *boltLocalOverrideStorage) GetOverride(name chain.Name, target policyen } c := &chain.Chain{} - if err := json.Unmarshal(serializedChain, c); err != nil { + if err := c.DecodeBytes(serializedChain); err != nil { return nil, err } return c, nil @@ -241,7 +240,7 @@ func (cs *boltLocalOverrideStorage) ListOverrides(name chain.Name, target policy chains := make([]*chain.Chain, 0, len(serializedChains)) for _, serializedChain = range serializedChains { c := &chain.Chain{} - if err := json.Unmarshal(serializedChain, c); err != nil { + if err := c.DecodeBytes(serializedChain); err != nil { return nil, err } chains = append(chains, c) From 602ee11123b46dd8131903928c34c5f014d6d601 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Feb 2024 17:56:40 +0300 Subject: [PATCH 0842/1943] [#934] containersvc: Marhal public key in short format for APE Signed-off-by: Evgenii Stratonikov --- pkg/services/container/ape.go | 7 ++++--- pkg/services/container/ape_test.go | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index c57e54d08..d1b490bdf 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -6,6 +6,7 @@ import ( "crypto/ecdsa" "crypto/elliptic" "crypto/sha256" + "encoding/hex" "errors" "fmt" "strings" @@ -133,7 +134,7 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co } reqProps := map[string]string{ - nativeschema.PropertyKeyActorPublicKey: pk.String(), + nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(pk.Bytes()), nativeschema.PropertyKeyActorRole: role, } @@ -178,7 +179,7 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont } reqProps := map[string]string{ - nativeschema.PropertyKeyActorPublicKey: pk.String(), + nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(pk.Bytes()), nativeschema.PropertyKeyActorRole: role, } @@ -391,7 +392,7 @@ func (ac *apeChecker) getRequestProps(mh *session.RequestMetaHeader, vh *session return nil, err } return map[string]string{ - nativeschema.PropertyKeyActorPublicKey: pk.String(), + nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(pk.Bytes()), nativeschema.PropertyKeyActorRole: role, }, nil } diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index 5344d9f23..daaacb031 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -3,6 +3,7 @@ package container import ( "context" "crypto/ecdsa" + "encoding/hex" "errors" "fmt" "testing" @@ -524,7 +525,7 @@ func testDenyListContainersForPK(t *testing.T) { { Object: chain.ObjectRequest, Key: nativeschema.PropertyKeyActorPublicKey, - Value: pk.PublicKey().String(), + Value: hex.EncodeToString(pk.PublicKey().Bytes()), Op: chain.CondStringEquals, }, }, From afd2ba9a6664bb2e2e875959b29aa4afd5f4df5b Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 30 Jan 2024 08:39:34 +0300 Subject: [PATCH 0843/1943] [#110] Add check for repeated error log in policer processObject() returns 3 types of errors: container not found errors, could not get container error and placement vector building error. Every error will occur for all objects in container simultaneously, so we can log each error once and safely ignore the rest. Signed-off-by: Ekaterina Lebedeva --- pkg/services/policer/process.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index c4c820200..df4f0c4c7 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" ) @@ -34,6 +35,9 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { p.log.Warn(logs.PolicerFailureAtObjectSelectForReplication, zap.Error(err)) } + // contains all errors logged in this iteration for each container + cnrErrSkip := make(map[cid.ID][]error) + for i := range addrs { select { case <-ctx.Done(): @@ -55,10 +59,12 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { if p.objsInWork.add(addr.Address) { err := p.processObject(ctx, addr) - if err != nil { + if err != nil && !skipLog(cnrErrSkip[addr.Address.Container()], err) { p.log.Error(logs.PolicerUnableToProcessObj, zap.Stringer("object", addr.Address), zap.String("error", err.Error())) + + cnrErrSkip[addr.Address.Container()] = append(cnrErrSkip[addr.Address.Container()], err) } p.cache.Add(addr.Address, time.Now()) p.objsInWork.remove(addr.Address) @@ -72,3 +78,12 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { } } } + +func skipLog(errs []error, err error) bool { + for _, e := range errs { + if errors.Is(err, e) { + return true + } + } + return false +} From 4358d3c42321222296f9d834517a90a1d3ad5e7a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 30 Jan 2024 17:09:23 +0300 Subject: [PATCH 0844/1943] [#917] controlSvc: Add DetachShards handler Signed-off-by: Dmitrii Stepanov --- pkg/services/control/service.pb.go | 1067 ++++++++++++-------- pkg/services/control/service.proto | 21 + pkg/services/control/service_frostfs.pb.go | 163 +++ pkg/services/control/service_grpc.pb.go | 39 + 4 files changed, 895 insertions(+), 395 deletions(-) diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index e0c6e0622..c9645941b 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -2106,6 +2106,116 @@ func (x *SealWriteCacheResponse) GetSignature() *Signature { return nil } +type DetachShardsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *DetachShardsRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *DetachShardsRequest) Reset() { + *x = DetachShardsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DetachShardsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DetachShardsRequest) ProtoMessage() {} + +func (x *DetachShardsRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[36] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DetachShardsRequest.ProtoReflect.Descriptor instead. +func (*DetachShardsRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36} +} + +func (x *DetachShardsRequest) GetBody() *DetachShardsRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *DetachShardsRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +type DetachShardsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *DetachShardsResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *DetachShardsResponse) Reset() { + *x = DetachShardsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DetachShardsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DetachShardsResponse) ProtoMessage() {} + +func (x *DetachShardsResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[37] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DetachShardsResponse.ProtoReflect.Descriptor instead. +func (*DetachShardsResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37} +} + +func (x *DetachShardsResponse) GetBody() *DetachShardsResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *DetachShardsResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + // Health check request body. type HealthCheckRequest_Body struct { state protoimpl.MessageState @@ -2116,7 +2226,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2129,7 +2239,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2160,7 +2270,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2173,7 +2283,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2224,7 +2334,7 @@ type SetNetmapStatusRequest_Body struct { func (x *SetNetmapStatusRequest_Body) Reset() { *x = SetNetmapStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2237,7 +2347,7 @@ func (x *SetNetmapStatusRequest_Body) String() string { func (*SetNetmapStatusRequest_Body) ProtoMessage() {} func (x *SetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2277,7 +2387,7 @@ type SetNetmapStatusResponse_Body struct { func (x *SetNetmapStatusResponse_Body) Reset() { *x = SetNetmapStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2290,7 +2400,7 @@ func (x *SetNetmapStatusResponse_Body) String() string { func (*SetNetmapStatusResponse_Body) ProtoMessage() {} func (x *SetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2320,7 +2430,7 @@ type DropObjectsRequest_Body struct { func (x *DropObjectsRequest_Body) Reset() { *x = DropObjectsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2333,7 +2443,7 @@ func (x *DropObjectsRequest_Body) String() string { func (*DropObjectsRequest_Body) ProtoMessage() {} func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2366,7 +2476,7 @@ type DropObjectsResponse_Body struct { func (x *DropObjectsResponse_Body) Reset() { *x = DropObjectsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2379,7 +2489,7 @@ func (x *DropObjectsResponse_Body) String() string { func (*DropObjectsResponse_Body) ProtoMessage() {} func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2405,7 +2515,7 @@ type ListShardsRequest_Body struct { func (x *ListShardsRequest_Body) Reset() { *x = ListShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2418,7 +2528,7 @@ func (x *ListShardsRequest_Body) String() string { func (*ListShardsRequest_Body) ProtoMessage() {} func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2447,7 +2557,7 @@ type ListShardsResponse_Body struct { func (x *ListShardsResponse_Body) Reset() { *x = ListShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2460,7 +2570,7 @@ func (x *ListShardsResponse_Body) String() string { func (*ListShardsResponse_Body) ProtoMessage() {} func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2500,7 +2610,7 @@ type SetShardModeRequest_Body struct { func (x *SetShardModeRequest_Body) Reset() { *x = SetShardModeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2513,7 +2623,7 @@ func (x *SetShardModeRequest_Body) String() string { func (*SetShardModeRequest_Body) ProtoMessage() {} func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2560,7 +2670,7 @@ type SetShardModeResponse_Body struct { func (x *SetShardModeResponse_Body) Reset() { *x = SetShardModeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2573,7 +2683,7 @@ func (x *SetShardModeResponse_Body) String() string { func (*SetShardModeResponse_Body) ProtoMessage() {} func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2604,7 +2714,7 @@ type SynchronizeTreeRequest_Body struct { func (x *SynchronizeTreeRequest_Body) Reset() { *x = SynchronizeTreeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2617,7 +2727,7 @@ func (x *SynchronizeTreeRequest_Body) String() string { func (*SynchronizeTreeRequest_Body) ProtoMessage() {} func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2664,7 +2774,7 @@ type SynchronizeTreeResponse_Body struct { func (x *SynchronizeTreeResponse_Body) Reset() { *x = SynchronizeTreeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2677,7 +2787,7 @@ func (x *SynchronizeTreeResponse_Body) String() string { func (*SynchronizeTreeResponse_Body) ProtoMessage() {} func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2708,7 +2818,7 @@ type EvacuateShardRequest_Body struct { func (x *EvacuateShardRequest_Body) Reset() { *x = EvacuateShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2721,7 +2831,7 @@ func (x *EvacuateShardRequest_Body) String() string { func (*EvacuateShardRequest_Body) ProtoMessage() {} func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2763,7 +2873,7 @@ type EvacuateShardResponse_Body struct { func (x *EvacuateShardResponse_Body) Reset() { *x = EvacuateShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2776,7 +2886,7 @@ func (x *EvacuateShardResponse_Body) String() string { func (*EvacuateShardResponse_Body) ProtoMessage() {} func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2814,7 +2924,7 @@ type FlushCacheRequest_Body struct { func (x *FlushCacheRequest_Body) Reset() { *x = FlushCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2827,7 +2937,7 @@ func (x *FlushCacheRequest_Body) String() string { func (*FlushCacheRequest_Body) ProtoMessage() {} func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2867,7 +2977,7 @@ type FlushCacheResponse_Body struct { func (x *FlushCacheResponse_Body) Reset() { *x = FlushCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2880,7 +2990,7 @@ func (x *FlushCacheResponse_Body) String() string { func (*FlushCacheResponse_Body) ProtoMessage() {} func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2911,7 +3021,7 @@ type DoctorRequest_Body struct { func (x *DoctorRequest_Body) Reset() { *x = DoctorRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2924,7 +3034,7 @@ func (x *DoctorRequest_Body) String() string { func (*DoctorRequest_Body) ProtoMessage() {} func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2964,7 +3074,7 @@ type DoctorResponse_Body struct { func (x *DoctorResponse_Body) Reset() { *x = DoctorResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2977,7 +3087,7 @@ func (x *DoctorResponse_Body) String() string { func (*DoctorResponse_Body) ProtoMessage() {} func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3008,7 +3118,7 @@ type StartShardEvacuationRequest_Body struct { func (x *StartShardEvacuationRequest_Body) Reset() { *x = StartShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3021,7 +3131,7 @@ func (x *StartShardEvacuationRequest_Body) String() string { func (*StartShardEvacuationRequest_Body) ProtoMessage() {} func (x *StartShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3061,7 +3171,7 @@ type StartShardEvacuationResponse_Body struct { func (x *StartShardEvacuationResponse_Body) Reset() { *x = StartShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3074,7 +3184,7 @@ func (x *StartShardEvacuationResponse_Body) String() string { func (*StartShardEvacuationResponse_Body) ProtoMessage() {} func (x *StartShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3100,7 +3210,7 @@ type GetShardEvacuationStatusRequest_Body struct { func (x *GetShardEvacuationStatusRequest_Body) Reset() { *x = GetShardEvacuationStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3113,7 +3223,7 @@ func (x *GetShardEvacuationStatusRequest_Body) String() string { func (*GetShardEvacuationStatusRequest_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3158,7 +3268,7 @@ type GetShardEvacuationStatusResponse_Body struct { func (x *GetShardEvacuationStatusResponse_Body) Reset() { *x = GetShardEvacuationStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3171,7 +3281,7 @@ func (x *GetShardEvacuationStatusResponse_Body) String() string { func (*GetShardEvacuationStatusResponse_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3262,7 +3372,7 @@ type GetShardEvacuationStatusResponse_Body_UnixTimestamp struct { func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) Reset() { *x = GetShardEvacuationStatusResponse_Body_UnixTimestamp{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3275,7 +3385,7 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) String() string { func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3310,7 +3420,7 @@ type GetShardEvacuationStatusResponse_Body_Duration struct { func (x *GetShardEvacuationStatusResponse_Body_Duration) Reset() { *x = GetShardEvacuationStatusResponse_Body_Duration{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3323,7 +3433,7 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) String() string { func (*GetShardEvacuationStatusResponse_Body_Duration) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_Duration) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3356,7 +3466,7 @@ type StopShardEvacuationRequest_Body struct { func (x *StopShardEvacuationRequest_Body) Reset() { *x = StopShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3369,7 +3479,7 @@ func (x *StopShardEvacuationRequest_Body) String() string { func (*StopShardEvacuationRequest_Body) ProtoMessage() {} func (x *StopShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3395,7 +3505,7 @@ type StopShardEvacuationResponse_Body struct { func (x *StopShardEvacuationResponse_Body) Reset() { *x = StopShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3408,7 +3518,7 @@ func (x *StopShardEvacuationResponse_Body) String() string { func (*StopShardEvacuationResponse_Body) ProtoMessage() {} func (x *StopShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3440,7 +3550,7 @@ type AddChainLocalOverrideRequest_Body struct { func (x *AddChainLocalOverrideRequest_Body) Reset() { *x = AddChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3453,7 +3563,7 @@ func (x *AddChainLocalOverrideRequest_Body) String() string { func (*AddChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *AddChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[64] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3497,7 +3607,7 @@ type AddChainLocalOverrideResponse_Body struct { func (x *AddChainLocalOverrideResponse_Body) Reset() { *x = AddChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3510,7 +3620,7 @@ func (x *AddChainLocalOverrideResponse_Body) String() string { func (*AddChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *AddChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[65] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3547,7 +3657,7 @@ type GetChainLocalOverrideRequest_Body struct { func (x *GetChainLocalOverrideRequest_Body) Reset() { *x = GetChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3560,7 +3670,7 @@ func (x *GetChainLocalOverrideRequest_Body) String() string { func (*GetChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *GetChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[66] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3602,7 +3712,7 @@ type GetChainLocalOverrideResponse_Body struct { func (x *GetChainLocalOverrideResponse_Body) Reset() { *x = GetChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3615,7 +3725,7 @@ func (x *GetChainLocalOverrideResponse_Body) String() string { func (*GetChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *GetChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[67] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3650,7 +3760,7 @@ type ListChainLocalOverridesRequest_Body struct { func (x *ListChainLocalOverridesRequest_Body) Reset() { *x = ListChainLocalOverridesRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[66] + mi := &file_pkg_services_control_service_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3663,7 +3773,7 @@ func (x *ListChainLocalOverridesRequest_Body) String() string { func (*ListChainLocalOverridesRequest_Body) ProtoMessage() {} func (x *ListChainLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[66] + mi := &file_pkg_services_control_service_proto_msgTypes[68] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3698,7 +3808,7 @@ type ListChainLocalOverridesResponse_Body struct { func (x *ListChainLocalOverridesResponse_Body) Reset() { *x = ListChainLocalOverridesResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[67] + mi := &file_pkg_services_control_service_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3711,7 +3821,7 @@ func (x *ListChainLocalOverridesResponse_Body) String() string { func (*ListChainLocalOverridesResponse_Body) ProtoMessage() {} func (x *ListChainLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[67] + mi := &file_pkg_services_control_service_proto_msgTypes[69] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3746,7 +3856,7 @@ type ListTargetsLocalOverridesRequest_Body struct { func (x *ListTargetsLocalOverridesRequest_Body) Reset() { *x = ListTargetsLocalOverridesRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[68] + mi := &file_pkg_services_control_service_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3759,7 +3869,7 @@ func (x *ListTargetsLocalOverridesRequest_Body) String() string { func (*ListTargetsLocalOverridesRequest_Body) ProtoMessage() {} func (x *ListTargetsLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[68] + mi := &file_pkg_services_control_service_proto_msgTypes[70] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3794,7 +3904,7 @@ type ListTargetsLocalOverridesResponse_Body struct { func (x *ListTargetsLocalOverridesResponse_Body) Reset() { *x = ListTargetsLocalOverridesResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[69] + mi := &file_pkg_services_control_service_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3807,7 +3917,7 @@ func (x *ListTargetsLocalOverridesResponse_Body) String() string { func (*ListTargetsLocalOverridesResponse_Body) ProtoMessage() {} func (x *ListTargetsLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[69] + mi := &file_pkg_services_control_service_proto_msgTypes[71] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3844,7 +3954,7 @@ type RemoveChainLocalOverrideRequest_Body struct { func (x *RemoveChainLocalOverrideRequest_Body) Reset() { *x = RemoveChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[70] + mi := &file_pkg_services_control_service_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3857,7 +3967,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) String() string { func (*RemoveChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[70] + mi := &file_pkg_services_control_service_proto_msgTypes[72] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3898,7 +4008,7 @@ type RemoveChainLocalOverrideResponse_Body struct { func (x *RemoveChainLocalOverrideResponse_Body) Reset() { *x = RemoveChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[71] + mi := &file_pkg_services_control_service_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3911,7 +4021,7 @@ func (x *RemoveChainLocalOverrideResponse_Body) String() string { func (*RemoveChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[71] + mi := &file_pkg_services_control_service_proto_msgTypes[73] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3949,7 +4059,7 @@ type SealWriteCacheRequest_Body struct { func (x *SealWriteCacheRequest_Body) Reset() { *x = SealWriteCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[72] + mi := &file_pkg_services_control_service_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3962,7 +4072,7 @@ func (x *SealWriteCacheRequest_Body) String() string { func (*SealWriteCacheRequest_Body) ProtoMessage() {} func (x *SealWriteCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[72] + mi := &file_pkg_services_control_service_proto_msgTypes[74] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4003,7 +4113,7 @@ type SealWriteCacheResponse_Body struct { func (x *SealWriteCacheResponse_Body) Reset() { *x = SealWriteCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[73] + mi := &file_pkg_services_control_service_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4016,7 +4126,7 @@ func (x *SealWriteCacheResponse_Body) String() string { func (*SealWriteCacheResponse_Body) ProtoMessage() {} func (x *SealWriteCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[73] + mi := &file_pkg_services_control_service_proto_msgTypes[75] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4052,7 +4162,7 @@ type SealWriteCacheResponse_Body_Status struct { func (x *SealWriteCacheResponse_Body_Status) Reset() { *x = SealWriteCacheResponse_Body_Status{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[74] + mi := &file_pkg_services_control_service_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4065,7 +4175,7 @@ func (x *SealWriteCacheResponse_Body_Status) String() string { func (*SealWriteCacheResponse_Body_Status) ProtoMessage() {} func (x *SealWriteCacheResponse_Body_Status) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[74] + mi := &file_pkg_services_control_service_proto_msgTypes[76] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4102,6 +4212,91 @@ func (x *SealWriteCacheResponse_Body_Status) GetError() string { return "" } +type DetachShardsRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` +} + +func (x *DetachShardsRequest_Body) Reset() { + *x = DetachShardsRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[77] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DetachShardsRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DetachShardsRequest_Body) ProtoMessage() {} + +func (x *DetachShardsRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[77] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DetachShardsRequest_Body.ProtoReflect.Descriptor instead. +func (*DetachShardsRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36, 0} +} + +func (x *DetachShardsRequest_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} + +type DetachShardsResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *DetachShardsResponse_Body) Reset() { + *x = DetachShardsResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[78] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DetachShardsResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DetachShardsResponse_Body) ProtoMessage() {} + +func (x *DetachShardsResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[78] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DetachShardsResponse_Body.ProtoReflect.Descriptor instead. +func (*DetachShardsResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37, 0} +} + var File_pkg_services_control_service_proto protoreflect.FileDescriptor var file_pkg_services_control_service_proto_rawDesc = []byte{ @@ -4564,114 +4759,138 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0xe4, 0x0c, 0x0a, - 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, - 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, - 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, + 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, + 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, + 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, + 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb1, 0x0d, 0x0a, 0x0e, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, + 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, + 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, - 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, - 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, - 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, - 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, - 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, + 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, + 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, + 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, + 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, + 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, - 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, - 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, - 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, - 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, - 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, - 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, + 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, + 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, + 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, + 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, + 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, - 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, - 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, - 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, - 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, - 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, - 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, + 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, + 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, + 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, + 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, + 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, + 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, + 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, + 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4687,7 +4906,7 @@ func file_pkg_services_control_service_proto_rawDescGZIP() []byte { } var file_pkg_services_control_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 75) +var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 79) var file_pkg_services_control_service_proto_goTypes = []interface{}{ (GetShardEvacuationStatusResponse_Body_Status)(0), // 0: control.GetShardEvacuationStatusResponse.Body.Status (*HealthCheckRequest)(nil), // 1: control.HealthCheckRequest @@ -4726,180 +4945,190 @@ var file_pkg_services_control_service_proto_goTypes = []interface{}{ (*RemoveChainLocalOverrideResponse)(nil), // 34: control.RemoveChainLocalOverrideResponse (*SealWriteCacheRequest)(nil), // 35: control.SealWriteCacheRequest (*SealWriteCacheResponse)(nil), // 36: control.SealWriteCacheResponse - (*HealthCheckRequest_Body)(nil), // 37: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 38: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 39: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 40: control.SetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 41: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 42: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 43: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 44: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 45: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 46: control.SetShardModeResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 47: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 48: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 49: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 50: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 51: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 52: control.FlushCacheResponse.Body - (*DoctorRequest_Body)(nil), // 53: control.DoctorRequest.Body - (*DoctorResponse_Body)(nil), // 54: control.DoctorResponse.Body - (*StartShardEvacuationRequest_Body)(nil), // 55: control.StartShardEvacuationRequest.Body - (*StartShardEvacuationResponse_Body)(nil), // 56: control.StartShardEvacuationResponse.Body - (*GetShardEvacuationStatusRequest_Body)(nil), // 57: control.GetShardEvacuationStatusRequest.Body - (*GetShardEvacuationStatusResponse_Body)(nil), // 58: control.GetShardEvacuationStatusResponse.Body - (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 59: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 60: control.GetShardEvacuationStatusResponse.Body.Duration - (*StopShardEvacuationRequest_Body)(nil), // 61: control.StopShardEvacuationRequest.Body - (*StopShardEvacuationResponse_Body)(nil), // 62: control.StopShardEvacuationResponse.Body - (*AddChainLocalOverrideRequest_Body)(nil), // 63: control.AddChainLocalOverrideRequest.Body - (*AddChainLocalOverrideResponse_Body)(nil), // 64: control.AddChainLocalOverrideResponse.Body - (*GetChainLocalOverrideRequest_Body)(nil), // 65: control.GetChainLocalOverrideRequest.Body - (*GetChainLocalOverrideResponse_Body)(nil), // 66: control.GetChainLocalOverrideResponse.Body - (*ListChainLocalOverridesRequest_Body)(nil), // 67: control.ListChainLocalOverridesRequest.Body - (*ListChainLocalOverridesResponse_Body)(nil), // 68: control.ListChainLocalOverridesResponse.Body - (*ListTargetsLocalOverridesRequest_Body)(nil), // 69: control.ListTargetsLocalOverridesRequest.Body - (*ListTargetsLocalOverridesResponse_Body)(nil), // 70: control.ListTargetsLocalOverridesResponse.Body - (*RemoveChainLocalOverrideRequest_Body)(nil), // 71: control.RemoveChainLocalOverrideRequest.Body - (*RemoveChainLocalOverrideResponse_Body)(nil), // 72: control.RemoveChainLocalOverrideResponse.Body - (*SealWriteCacheRequest_Body)(nil), // 73: control.SealWriteCacheRequest.Body - (*SealWriteCacheResponse_Body)(nil), // 74: control.SealWriteCacheResponse.Body - (*SealWriteCacheResponse_Body_Status)(nil), // 75: control.SealWriteCacheResponse.Body.Status - (*Signature)(nil), // 76: control.Signature - (NetmapStatus)(0), // 77: control.NetmapStatus - (HealthStatus)(0), // 78: control.HealthStatus - (*ShardInfo)(nil), // 79: control.ShardInfo - (ShardMode)(0), // 80: control.ShardMode - (*ChainTarget)(nil), // 81: control.ChainTarget + (*DetachShardsRequest)(nil), // 37: control.DetachShardsRequest + (*DetachShardsResponse)(nil), // 38: control.DetachShardsResponse + (*HealthCheckRequest_Body)(nil), // 39: control.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 40: control.HealthCheckResponse.Body + (*SetNetmapStatusRequest_Body)(nil), // 41: control.SetNetmapStatusRequest.Body + (*SetNetmapStatusResponse_Body)(nil), // 42: control.SetNetmapStatusResponse.Body + (*DropObjectsRequest_Body)(nil), // 43: control.DropObjectsRequest.Body + (*DropObjectsResponse_Body)(nil), // 44: control.DropObjectsResponse.Body + (*ListShardsRequest_Body)(nil), // 45: control.ListShardsRequest.Body + (*ListShardsResponse_Body)(nil), // 46: control.ListShardsResponse.Body + (*SetShardModeRequest_Body)(nil), // 47: control.SetShardModeRequest.Body + (*SetShardModeResponse_Body)(nil), // 48: control.SetShardModeResponse.Body + (*SynchronizeTreeRequest_Body)(nil), // 49: control.SynchronizeTreeRequest.Body + (*SynchronizeTreeResponse_Body)(nil), // 50: control.SynchronizeTreeResponse.Body + (*EvacuateShardRequest_Body)(nil), // 51: control.EvacuateShardRequest.Body + (*EvacuateShardResponse_Body)(nil), // 52: control.EvacuateShardResponse.Body + (*FlushCacheRequest_Body)(nil), // 53: control.FlushCacheRequest.Body + (*FlushCacheResponse_Body)(nil), // 54: control.FlushCacheResponse.Body + (*DoctorRequest_Body)(nil), // 55: control.DoctorRequest.Body + (*DoctorResponse_Body)(nil), // 56: control.DoctorResponse.Body + (*StartShardEvacuationRequest_Body)(nil), // 57: control.StartShardEvacuationRequest.Body + (*StartShardEvacuationResponse_Body)(nil), // 58: control.StartShardEvacuationResponse.Body + (*GetShardEvacuationStatusRequest_Body)(nil), // 59: control.GetShardEvacuationStatusRequest.Body + (*GetShardEvacuationStatusResponse_Body)(nil), // 60: control.GetShardEvacuationStatusResponse.Body + (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 61: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 62: control.GetShardEvacuationStatusResponse.Body.Duration + (*StopShardEvacuationRequest_Body)(nil), // 63: control.StopShardEvacuationRequest.Body + (*StopShardEvacuationResponse_Body)(nil), // 64: control.StopShardEvacuationResponse.Body + (*AddChainLocalOverrideRequest_Body)(nil), // 65: control.AddChainLocalOverrideRequest.Body + (*AddChainLocalOverrideResponse_Body)(nil), // 66: control.AddChainLocalOverrideResponse.Body + (*GetChainLocalOverrideRequest_Body)(nil), // 67: control.GetChainLocalOverrideRequest.Body + (*GetChainLocalOverrideResponse_Body)(nil), // 68: control.GetChainLocalOverrideResponse.Body + (*ListChainLocalOverridesRequest_Body)(nil), // 69: control.ListChainLocalOverridesRequest.Body + (*ListChainLocalOverridesResponse_Body)(nil), // 70: control.ListChainLocalOverridesResponse.Body + (*ListTargetsLocalOverridesRequest_Body)(nil), // 71: control.ListTargetsLocalOverridesRequest.Body + (*ListTargetsLocalOverridesResponse_Body)(nil), // 72: control.ListTargetsLocalOverridesResponse.Body + (*RemoveChainLocalOverrideRequest_Body)(nil), // 73: control.RemoveChainLocalOverrideRequest.Body + (*RemoveChainLocalOverrideResponse_Body)(nil), // 74: control.RemoveChainLocalOverrideResponse.Body + (*SealWriteCacheRequest_Body)(nil), // 75: control.SealWriteCacheRequest.Body + (*SealWriteCacheResponse_Body)(nil), // 76: control.SealWriteCacheResponse.Body + (*SealWriteCacheResponse_Body_Status)(nil), // 77: control.SealWriteCacheResponse.Body.Status + (*DetachShardsRequest_Body)(nil), // 78: control.DetachShardsRequest.Body + (*DetachShardsResponse_Body)(nil), // 79: control.DetachShardsResponse.Body + (*Signature)(nil), // 80: control.Signature + (NetmapStatus)(0), // 81: control.NetmapStatus + (HealthStatus)(0), // 82: control.HealthStatus + (*ShardInfo)(nil), // 83: control.ShardInfo + (ShardMode)(0), // 84: control.ShardMode + (*ChainTarget)(nil), // 85: control.ChainTarget } var file_pkg_services_control_service_proto_depIdxs = []int32{ - 37, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 76, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 38, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 76, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 39, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 76, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 40, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 76, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 41, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 76, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature - 42, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 76, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature - 43, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 76, // 13: control.ListShardsRequest.signature:type_name -> control.Signature - 44, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 76, // 15: control.ListShardsResponse.signature:type_name -> control.Signature - 45, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 76, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature - 46, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 76, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature - 47, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 76, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 48, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 76, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 49, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 76, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature - 50, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 76, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature - 51, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 76, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature - 52, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 76, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature - 53, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body - 76, // 33: control.DoctorRequest.signature:type_name -> control.Signature - 54, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body - 76, // 35: control.DoctorResponse.signature:type_name -> control.Signature - 55, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body - 76, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature - 56, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body - 76, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature - 57, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body - 76, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 58, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body - 76, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 61, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body - 76, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature - 62, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body - 76, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature - 63, // 48: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body - 76, // 49: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature - 64, // 50: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body - 76, // 51: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature - 65, // 52: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body - 76, // 53: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature - 66, // 54: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body - 76, // 55: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature - 67, // 56: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body - 76, // 57: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature - 68, // 58: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body - 76, // 59: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature - 69, // 60: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body - 76, // 61: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature - 70, // 62: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body - 76, // 63: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature - 71, // 64: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body - 76, // 65: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature - 72, // 66: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body - 76, // 67: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature - 73, // 68: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body - 76, // 69: control.SealWriteCacheRequest.signature:type_name -> control.Signature - 74, // 70: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body - 76, // 71: control.SealWriteCacheResponse.signature:type_name -> control.Signature - 77, // 72: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 78, // 73: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 77, // 74: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 79, // 75: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 80, // 76: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 0, // 77: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status - 60, // 78: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration - 59, // 79: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - 81, // 80: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 81, // 81: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 81, // 82: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget - 81, // 83: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget - 81, // 84: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 75, // 85: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status - 1, // 86: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 3, // 87: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 5, // 88: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 7, // 89: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 9, // 90: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 11, // 91: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 13, // 92: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 19, // 93: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest - 21, // 94: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest - 23, // 95: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest - 15, // 96: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 17, // 97: control.ControlService.Doctor:input_type -> control.DoctorRequest - 25, // 98: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest - 27, // 99: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest - 29, // 100: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest - 33, // 101: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest - 31, // 102: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest - 35, // 103: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest - 2, // 104: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 4, // 105: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 6, // 106: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 8, // 107: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 10, // 108: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 12, // 109: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 14, // 110: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 20, // 111: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse - 22, // 112: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse - 24, // 113: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse - 16, // 114: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 18, // 115: control.ControlService.Doctor:output_type -> control.DoctorResponse - 26, // 116: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse - 28, // 117: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse - 30, // 118: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse - 34, // 119: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse - 32, // 120: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse - 36, // 121: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse - 104, // [104:122] is the sub-list for method output_type - 86, // [86:104] is the sub-list for method input_type - 86, // [86:86] is the sub-list for extension type_name - 86, // [86:86] is the sub-list for extension extendee - 0, // [0:86] is the sub-list for field type_name + 39, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body + 80, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature + 40, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body + 80, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature + 41, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body + 80, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature + 42, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body + 80, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature + 43, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body + 80, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature + 44, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body + 80, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature + 45, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body + 80, // 13: control.ListShardsRequest.signature:type_name -> control.Signature + 46, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body + 80, // 15: control.ListShardsResponse.signature:type_name -> control.Signature + 47, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body + 80, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature + 48, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body + 80, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature + 49, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body + 80, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature + 50, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body + 80, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature + 51, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body + 80, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature + 52, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body + 80, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature + 53, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body + 80, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature + 54, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body + 80, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature + 55, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body + 80, // 33: control.DoctorRequest.signature:type_name -> control.Signature + 56, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body + 80, // 35: control.DoctorResponse.signature:type_name -> control.Signature + 57, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body + 80, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature + 58, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body + 80, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature + 59, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body + 80, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 60, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body + 80, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 63, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body + 80, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature + 64, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body + 80, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature + 65, // 48: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body + 80, // 49: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature + 66, // 50: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body + 80, // 51: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature + 67, // 52: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body + 80, // 53: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature + 68, // 54: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body + 80, // 55: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature + 69, // 56: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body + 80, // 57: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature + 70, // 58: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body + 80, // 59: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature + 71, // 60: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body + 80, // 61: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature + 72, // 62: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body + 80, // 63: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature + 73, // 64: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body + 80, // 65: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature + 74, // 66: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body + 80, // 67: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature + 75, // 68: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body + 80, // 69: control.SealWriteCacheRequest.signature:type_name -> control.Signature + 76, // 70: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body + 80, // 71: control.SealWriteCacheResponse.signature:type_name -> control.Signature + 78, // 72: control.DetachShardsRequest.body:type_name -> control.DetachShardsRequest.Body + 80, // 73: control.DetachShardsRequest.signature:type_name -> control.Signature + 79, // 74: control.DetachShardsResponse.body:type_name -> control.DetachShardsResponse.Body + 80, // 75: control.DetachShardsResponse.signature:type_name -> control.Signature + 81, // 76: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus + 82, // 77: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus + 81, // 78: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus + 83, // 79: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo + 84, // 80: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode + 0, // 81: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status + 62, // 82: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration + 61, // 83: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + 85, // 84: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 85, // 85: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 85, // 86: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget + 85, // 87: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget + 85, // 88: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 77, // 89: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status + 1, // 90: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 3, // 91: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 5, // 92: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 7, // 93: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 9, // 94: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 11, // 95: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 13, // 96: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 19, // 97: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest + 21, // 98: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest + 23, // 99: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest + 15, // 100: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 17, // 101: control.ControlService.Doctor:input_type -> control.DoctorRequest + 25, // 102: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest + 27, // 103: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest + 29, // 104: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest + 33, // 105: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest + 31, // 106: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest + 35, // 107: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest + 37, // 108: control.ControlService.DetachShards:input_type -> control.DetachShardsRequest + 2, // 109: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 4, // 110: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 6, // 111: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 8, // 112: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 10, // 113: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 12, // 114: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 14, // 115: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 20, // 116: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse + 22, // 117: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse + 24, // 118: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse + 16, // 119: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 18, // 120: control.ControlService.Doctor:output_type -> control.DoctorResponse + 26, // 121: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse + 28, // 122: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse + 30, // 123: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse + 34, // 124: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse + 32, // 125: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse + 36, // 126: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse + 38, // 127: control.ControlService.DetachShards:output_type -> control.DetachShardsResponse + 109, // [109:128] is the sub-list for method output_type + 90, // [90:109] is the sub-list for method input_type + 90, // [90:90] is the sub-list for extension type_name + 90, // [90:90] is the sub-list for extension extendee + 0, // [0:90] is the sub-list for field type_name } func init() { file_pkg_services_control_service_proto_init() } @@ -5342,7 +5571,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*DetachShardsRequest); i { case 0: return &v.state case 1: @@ -5354,7 +5583,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { + switch v := v.(*DetachShardsResponse); i { case 0: return &v.state case 1: @@ -5366,7 +5595,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusRequest_Body); i { + switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state case 1: @@ -5378,7 +5607,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusResponse_Body); i { + switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state case 1: @@ -5390,7 +5619,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest_Body); i { + switch v := v.(*SetNetmapStatusRequest_Body); i { case 0: return &v.state case 1: @@ -5402,7 +5631,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse_Body); i { + switch v := v.(*SetNetmapStatusResponse_Body); i { case 0: return &v.state case 1: @@ -5414,7 +5643,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest_Body); i { + switch v := v.(*DropObjectsRequest_Body); i { case 0: return &v.state case 1: @@ -5426,7 +5655,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse_Body); i { + switch v := v.(*DropObjectsResponse_Body); i { case 0: return &v.state case 1: @@ -5438,7 +5667,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest_Body); i { + switch v := v.(*ListShardsRequest_Body); i { case 0: return &v.state case 1: @@ -5450,7 +5679,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse_Body); i { + switch v := v.(*ListShardsResponse_Body); i { case 0: return &v.state case 1: @@ -5462,7 +5691,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest_Body); i { + switch v := v.(*SetShardModeRequest_Body); i { case 0: return &v.state case 1: @@ -5474,7 +5703,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse_Body); i { + switch v := v.(*SetShardModeResponse_Body); i { case 0: return &v.state case 1: @@ -5486,7 +5715,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest_Body); i { + switch v := v.(*SynchronizeTreeRequest_Body); i { case 0: return &v.state case 1: @@ -5498,7 +5727,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse_Body); i { + switch v := v.(*SynchronizeTreeResponse_Body); i { case 0: return &v.state case 1: @@ -5510,7 +5739,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest_Body); i { + switch v := v.(*EvacuateShardRequest_Body); i { case 0: return &v.state case 1: @@ -5522,7 +5751,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheResponse_Body); i { + switch v := v.(*EvacuateShardResponse_Body); i { case 0: return &v.state case 1: @@ -5534,7 +5763,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorRequest_Body); i { + switch v := v.(*FlushCacheRequest_Body); i { case 0: return &v.state case 1: @@ -5546,7 +5775,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorResponse_Body); i { + switch v := v.(*FlushCacheResponse_Body); i { case 0: return &v.state case 1: @@ -5558,7 +5787,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationRequest_Body); i { + switch v := v.(*DoctorRequest_Body); i { case 0: return &v.state case 1: @@ -5570,7 +5799,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationResponse_Body); i { + switch v := v.(*DoctorResponse_Body); i { case 0: return &v.state case 1: @@ -5582,7 +5811,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusRequest_Body); i { + switch v := v.(*StartShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -5594,7 +5823,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body); i { + switch v := v.(*StartShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -5606,7 +5835,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { + switch v := v.(*GetShardEvacuationStatusRequest_Body); i { case 0: return &v.state case 1: @@ -5618,7 +5847,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body); i { case 0: return &v.state case 1: @@ -5630,7 +5859,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationRequest_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { case 0: return &v.state case 1: @@ -5642,7 +5871,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationResponse_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { case 0: return &v.state case 1: @@ -5654,7 +5883,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideRequest_Body); i { + switch v := v.(*StopShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -5666,7 +5895,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideResponse_Body); i { + switch v := v.(*StopShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -5678,7 +5907,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideRequest_Body); i { + switch v := v.(*AddChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -5690,7 +5919,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideResponse_Body); i { + switch v := v.(*AddChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -5702,7 +5931,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesRequest_Body); i { + switch v := v.(*GetChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -5714,7 +5943,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesResponse_Body); i { + switch v := v.(*GetChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -5726,7 +5955,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesRequest_Body); i { + switch v := v.(*ListChainLocalOverridesRequest_Body); i { case 0: return &v.state case 1: @@ -5738,7 +5967,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesResponse_Body); i { + switch v := v.(*ListChainLocalOverridesResponse_Body); i { case 0: return &v.state case 1: @@ -5750,7 +5979,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { + switch v := v.(*ListTargetsLocalOverridesRequest_Body); i { case 0: return &v.state case 1: @@ -5762,7 +5991,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { + switch v := v.(*ListTargetsLocalOverridesResponse_Body); i { case 0: return &v.state case 1: @@ -5774,7 +6003,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheRequest_Body); i { + switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -5786,7 +6015,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse_Body); i { + switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -5798,6 +6027,30 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SealWriteCacheResponse_Body_Status); i { case 0: return &v.state @@ -5809,6 +6062,30 @@ func file_pkg_services_control_service_proto_init() { return nil } } + file_pkg_services_control_service_proto_msgTypes[77].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DetachShardsRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[78].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DetachShardsResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -5816,7 +6093,7 @@ func file_pkg_services_control_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_service_proto_rawDesc, NumEnums: 1, - NumMessages: 75, + NumMessages: 79, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index dc63a7188..4cb1dee33 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -62,6 +62,9 @@ service ControlService { // Flush objects from write-cache and move it to degraded read only mode. rpc SealWriteCache(SealWriteCacheRequest) returns (SealWriteCacheResponse); + + // DetachShards detaches and closes shards. + rpc DetachShards(DetachShardsRequest) returns (DetachShardsResponse); } // Health check request. @@ -584,3 +587,21 @@ message SealWriteCacheResponse { Signature signature = 2; } + +message DetachShardsRequest { + message Body { + repeated bytes shard_ID = 1; + } + + Body body = 1; + Signature signature = 2; +} + +message DetachShardsResponse { + message Body { + } + + Body body = 1; + + Signature signature = 2; +} diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index b72fc0f12..125a12d00 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -3144,3 +3144,166 @@ func (x *SealWriteCacheResponse) ReadSignedData(buf []byte) ([]byte, error) { func (x *SealWriteCacheResponse) SetSignature(sig *Signature) { x.Signature = sig } + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *DetachShardsRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.RepeatedBytesSize(1, x.Shard_ID) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *DetachShardsRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *DetachShardsRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *DetachShardsRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *DetachShardsRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *DetachShardsRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *DetachShardsRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *DetachShardsResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *DetachShardsResponse_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *DetachShardsResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *DetachShardsResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *DetachShardsResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *DetachShardsResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *DetachShardsResponse) SetSignature(sig *Signature) { + x.Signature = sig +} diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 89337323f..69029ccb3 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -37,6 +37,7 @@ const ( ControlService_RemoveChainLocalOverride_FullMethodName = "/control.ControlService/RemoveChainLocalOverride" ControlService_ListTargetsLocalOverrides_FullMethodName = "/control.ControlService/ListTargetsLocalOverrides" ControlService_SealWriteCache_FullMethodName = "/control.ControlService/SealWriteCache" + ControlService_DetachShards_FullMethodName = "/control.ControlService/DetachShards" ) // ControlServiceClient is the client API for ControlService service. @@ -80,6 +81,8 @@ type ControlServiceClient interface { ListTargetsLocalOverrides(ctx context.Context, in *ListTargetsLocalOverridesRequest, opts ...grpc.CallOption) (*ListTargetsLocalOverridesResponse, error) // Flush objects from write-cache and move it to degraded read only mode. SealWriteCache(ctx context.Context, in *SealWriteCacheRequest, opts ...grpc.CallOption) (*SealWriteCacheResponse, error) + // DetachShards detaches and closes shards. + DetachShards(ctx context.Context, in *DetachShardsRequest, opts ...grpc.CallOption) (*DetachShardsResponse, error) } type controlServiceClient struct { @@ -252,6 +255,15 @@ func (c *controlServiceClient) SealWriteCache(ctx context.Context, in *SealWrite return out, nil } +func (c *controlServiceClient) DetachShards(ctx context.Context, in *DetachShardsRequest, opts ...grpc.CallOption) (*DetachShardsResponse, error) { + out := new(DetachShardsResponse) + err := c.cc.Invoke(ctx, ControlService_DetachShards_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // ControlServiceServer is the server API for ControlService service. // All implementations should embed UnimplementedControlServiceServer // for forward compatibility @@ -293,6 +305,8 @@ type ControlServiceServer interface { ListTargetsLocalOverrides(context.Context, *ListTargetsLocalOverridesRequest) (*ListTargetsLocalOverridesResponse, error) // Flush objects from write-cache and move it to degraded read only mode. SealWriteCache(context.Context, *SealWriteCacheRequest) (*SealWriteCacheResponse, error) + // DetachShards detaches and closes shards. + DetachShards(context.Context, *DetachShardsRequest) (*DetachShardsResponse, error) } // UnimplementedControlServiceServer should be embedded to have forward compatible implementations. @@ -353,6 +367,9 @@ func (UnimplementedControlServiceServer) ListTargetsLocalOverrides(context.Conte func (UnimplementedControlServiceServer) SealWriteCache(context.Context, *SealWriteCacheRequest) (*SealWriteCacheResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method SealWriteCache not implemented") } +func (UnimplementedControlServiceServer) DetachShards(context.Context, *DetachShardsRequest) (*DetachShardsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DetachShards not implemented") +} // UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ControlServiceServer will @@ -689,6 +706,24 @@ func _ControlService_SealWriteCache_Handler(srv interface{}, ctx context.Context return interceptor(ctx, in, info, handler) } +func _ControlService_DetachShards_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DetachShardsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).DetachShards(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_DetachShards_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).DetachShards(ctx, req.(*DetachShardsRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -768,6 +803,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "SealWriteCache", Handler: _ControlService_SealWriteCache_Handler, }, + { + MethodName: "DetachShards", + Handler: _ControlService_DetachShards_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "pkg/services/control/service.proto", From 9ba48c582dd803c8252a7e8b63f77eed5054acf6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 30 Jan 2024 18:18:58 +0300 Subject: [PATCH 0845/1943] [#917] engine: Allow to detach shards Signed-off-by: Dmitrii Stepanov --- .../modules/control/detach_shards.go | 49 ++++++++++ cmd/frostfs-cli/modules/control/shards.go | 2 + .../modules/control/shards_set_mode.go | 10 +- pkg/local_object_storage/engine/shards.go | 97 +++++++++++++++++++ .../engine/shards_test.go | 20 ++++ pkg/services/control/rpc.go | 20 ++++ pkg/services/control/server/detach_shards.go | 37 +++++++ 7 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-cli/modules/control/detach_shards.go create mode 100644 pkg/services/control/server/detach_shards.go diff --git a/cmd/frostfs-cli/modules/control/detach_shards.go b/cmd/frostfs-cli/modules/control/detach_shards.go new file mode 100644 index 000000000..5e5b60c3d --- /dev/null +++ b/cmd/frostfs-cli/modules/control/detach_shards.go @@ -0,0 +1,49 @@ +package control + +import ( + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "github.com/spf13/cobra" +) + +var shardsDetachCmd = &cobra.Command{ + Use: "detach", + Short: "Detach and close the shards", + Long: "Detach and close the shards", + Run: shardsDetach, +} + +func shardsDetach(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + req := &control.DetachShardsRequest{ + Body: &control.DetachShardsRequest_Body{ + Shard_ID: getShardIDListFromIDFlag(cmd, false), + }, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.DetachShardsResponse + var err error + err = cli.ExecRaw(func(client *rawclient.Client) error { + resp, err = control.DetachShards(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Shard mode update request successfully sent.") +} + +func initControlShardsDetachCmd() { + initControlFlags(shardsDetachCmd) + + flags := shardsDetachCmd.Flags() + flags.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") +} diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index 6d3ef420c..d8198c426 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -18,6 +18,7 @@ func initControlShardsCmd() { shardsCmd.AddCommand(flushCacheCmd) shardsCmd.AddCommand(doctorCmd) shardsCmd.AddCommand(writecacheShardCmd) + shardsCmd.AddCommand(shardsDetachCmd) initControlShardsListCmd() initControlSetShardModeCmd() @@ -26,4 +27,5 @@ func initControlShardsCmd() { initControlFlushCacheCmd() initControlDoctorCmd() initControlShardsWritecacheCmd() + initControlShardsDetachCmd() } diff --git a/cmd/frostfs-cli/modules/control/shards_set_mode.go b/cmd/frostfs-cli/modules/control/shards_set_mode.go index 78f768965..1c87b405b 100644 --- a/cmd/frostfs-cli/modules/control/shards_set_mode.go +++ b/cmd/frostfs-cli/modules/control/shards_set_mode.go @@ -145,9 +145,17 @@ func getShardIDList(cmd *cobra.Command) [][]byte { return nil } + return getShardIDListFromIDFlag(cmd, true) +} + +func getShardIDListFromIDFlag(cmd *cobra.Command, withAllFlag bool) [][]byte { sidList, _ := cmd.Flags().GetStringSlice(shardIDFlag) if len(sidList) == 0 { - commonCmd.ExitOnErr(cmd, "", fmt.Errorf("either --%s or --%s flag must be provided", shardIDFlag, shardAllFlag)) + if withAllFlag { + commonCmd.ExitOnErr(cmd, "", fmt.Errorf("either --%s or --%s flag must be provided", shardIDFlag, shardAllFlag)) + } else { + commonCmd.ExitOnErr(cmd, "", fmt.Errorf("--%s flag value must be provided", shardIDFlag)) + } } // We can sort the ID list and perform this check without additional allocations, diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index bd25dde59..83fa4f5cb 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -2,7 +2,9 @@ package engine import ( "context" + "errors" "fmt" + "sync" "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -14,6 +16,7 @@ import ( "github.com/google/uuid" "github.com/panjf2000/ants/v2" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) var errShardNotFound = logicerr.New("shard not found") @@ -344,6 +347,100 @@ func (e *StorageEngine) HandleNewEpoch(ctx context.Context, epoch uint64) { } } +func (e *StorageEngine) DetachShards(ids []*shard.ID) error { + if len(ids) == 0 { + return logicerr.New("ids must be non-empty") + } + + deletedShards, err := e.deleteShards(ids) + if err != nil { + return err + } + + return e.closeShards(deletedShards) +} + +// closeShards closes deleted shards. Tries to close all shards. +// Returns single error with joined shard errors. +func (e *StorageEngine) closeShards(deletedShards []hashedShard) error { + var multiErr error + var multiErrGuard sync.Mutex + var eg errgroup.Group + for _, sh := range deletedShards { + sh := sh + eg.Go(func() error { + err := sh.SetMode(mode.Disabled) + if err != nil { + e.log.Error(logs.EngineCouldNotChangeShardModeToDisabled, + zap.Stringer("id", sh.ID()), + zap.Error(err), + ) + multiErrGuard.Lock() + multiErr = errors.Join(multiErr, fmt.Errorf("could not change shard (id:%s) mode to disabled: %w", sh.ID(), err)) + multiErrGuard.Unlock() + } + + err = sh.Close() + if err != nil { + e.log.Error(logs.EngineCouldNotCloseRemovedShard, + zap.Stringer("id", sh.ID()), + zap.Error(err), + ) + multiErrGuard.Lock() + multiErr = errors.Join(multiErr, fmt.Errorf("could not close removed shard (id:%s): %w", sh.ID(), err)) + multiErrGuard.Unlock() + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return err + } + return multiErr +} + +// deleteShards deletes shards with specified ids from engine shard list +// and releases all engine resources associated with shards. +// Returns deleted shards or error if some shard could not be deleted. +func (e *StorageEngine) deleteShards(ids []*shard.ID) ([]hashedShard, error) { + ss := make([]hashedShard, 0, len(ids)) + + e.mtx.Lock() + defer e.mtx.Unlock() + + for _, id := range ids { + idStr := id.String() + sh, found := e.shards[idStr] + if !found { + return nil, errShardNotFound + } + ss = append(ss, sh) + } + + if len(ss) == len(e.shards) { + return nil, logicerr.New("could not delete all the shards") + } + + for _, sh := range ss { + idStr := sh.ID().String() + + sh.DeleteShardMetrics() + + delete(e.shards, idStr) + + pool, ok := e.shardPools[idStr] + if ok { + pool.Release() + delete(e.shardPools, idStr) + } + + e.log.Info(logs.EngineShardHasBeenRemoved, + zap.String("id", idStr)) + } + + return ss, nil +} + func (s hashedShard) Hash() uint64 { return s.hash } diff --git a/pkg/local_object_storage/engine/shards_test.go b/pkg/local_object_storage/engine/shards_test.go index f2896d557..3bb602ce4 100644 --- a/pkg/local_object_storage/engine/shards_test.go +++ b/pkg/local_object_storage/engine/shards_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "github.com/stretchr/testify/require" ) @@ -42,3 +44,21 @@ func TestRemoveShard(t *testing.T) { require.True(t, ok != removed) } } + +func TestDisableShards(t *testing.T) { + t.Parallel() + + const numOfShards = 2 + + te := testNewEngine(t).setShardsNum(t, numOfShards) + e, ids := te.engine, te.shardIDs + defer func() { require.NoError(t, e.Close(context.Background())) }() + + require.ErrorAs(t, e.DetachShards(ids), new(logicerr.Logical)) + require.ErrorAs(t, e.DetachShards(nil), new(logicerr.Logical)) + require.ErrorAs(t, e.DetachShards([]*shard.ID{}), new(logicerr.Logical)) + + require.NoError(t, e.DetachShards([]*shard.ID{ids[0]})) + + require.Equal(t, 1, len(e.shards)) +} diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 24b20f873..877bb63c2 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -26,6 +26,7 @@ const ( rpcRemoveChainLocalOverride = "RemoveChainLocalOverride" rpcSealWriteCache = "SealWriteCache" rpcListTargetsLocalOverrides = "ListTargetsLocalOverrides" + rpcDetachShards = "DetachShards" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -292,3 +293,22 @@ func SealWriteCache(cli *client.Client, req *SealWriteCacheRequest, opts ...clie return wResp.message, nil } + +// DetachShards executes ControlService.DetachShards RPC. +func DetachShards( + cli *client.Client, + req *DetachShardsRequest, + opts ...client.CallOption, +) (*DetachShardsResponse, error) { + wResp := newResponseWrapper[DetachShardsResponse]() + + wReq := &requestWrapper{ + m: req, + } + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcDetachShards), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} diff --git a/pkg/services/control/server/detach_shards.go b/pkg/services/control/server/detach_shards.go new file mode 100644 index 000000000..c8bea97b4 --- /dev/null +++ b/pkg/services/control/server/detach_shards.go @@ -0,0 +1,37 @@ +package control + +import ( + "context" + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *Server) DetachShards(_ context.Context, req *control.DetachShardsRequest) (*control.DetachShardsResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + shardIDs := s.getShardIDList(req.GetBody().GetShard_ID()) + + if err := s.s.DetachShards(shardIDs); err != nil { + if errors.As(err, new(logicerr.Logical)) { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.DetachShardsResponse{ + Body: &control.DetachShardsResponse_Body{}, + } + + if err = SignMessage(s.key, resp); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return resp, nil +} From 20b4447df77d03d8468442febdee24632013d3b6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 30 Jan 2024 18:32:55 +0300 Subject: [PATCH 0846/1943] [#917] docs: Extend shard mode description Add shards detach details. Signed-off-by: Dmitrii Stepanov --- docs/shard-modes.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/shard-modes.md b/docs/shard-modes.md index 7fca33d09..3abc26a60 100644 --- a/docs/shard-modes.md +++ b/docs/shard-modes.md @@ -46,3 +46,9 @@ Shard can automatically switch to a `degraded-read-only` mode in 3 cases: 1. If the metabase was not available or couldn't be opened/initialized during shard startup. 2. If shard error counter exceeds threshold. 3. If the metabase couldn't be reopened during SIGHUP handling. + +# Detach shard + +To detach a shard use `frostfs-cli control shards detach` command. This command removes the shards from the storage +engine and closes all resources associated with the shards. +Limitation: `SIGHUP` or storage node restart lead to detached shard will be again online. \ No newline at end of file From d7838790c654da64cbafd2a7017e7812108a94e1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 Jan 2024 13:29:26 +0300 Subject: [PATCH 0847/1943] [#917] dev: Extend launch.json example Add storage and dev wallets to control authorized keys list. Signed-off-by: Dmitrii Stepanov --- dev/.vscode-example/launch.json | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/.vscode-example/launch.json b/dev/.vscode-example/launch.json index 07d39482e..459350145 100644 --- a/dev/.vscode-example/launch.json +++ b/dev/.vscode-example/launch.json @@ -47,6 +47,7 @@ "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8080", "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8080", "FROSTFS_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8081", + "FROSTFS_CONTROL_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/.frostfs-node-state", From b1a1b2107dafb4b76cc9d1595f21a5b2037be158 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 12 Jan 2024 17:44:27 +0300 Subject: [PATCH 0848/1943] [#909] cli: Make add-rule and list-rules recieve namespace param Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/control/add_rule.go | 23 ++----- cmd/frostfs-cli/modules/control/get_rule.go | 19 ++---- cmd/frostfs-cli/modules/control/list_rules.go | 65 +++++++++++++++---- .../modules/control/remove_rule.go | 19 ++---- 4 files changed, 67 insertions(+), 59 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index 7e0805cd1..0f5ac11f3 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -1,16 +1,13 @@ package control import ( - "crypto/sha256" "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" ) @@ -33,6 +30,8 @@ var addRuleCmd = &cobra.Command{ func addRule(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) + target := parseTarget(cmd) + chainID, _ := cmd.Flags().GetString(chainIDFlag) hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) @@ -44,13 +43,6 @@ func addRule(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err) } - var cnr cid.ID - cidStr, _ := cmd.Flags().GetString(commonflags.CIDFlag) - commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(cidStr)) - - rawCID := make([]byte, sha256.Size) - cnr.Encode(rawCID) - rule, _ := cmd.Flags().GetStringArray(ruleFlag) chain := new(apechain.Chain) @@ -63,11 +55,8 @@ func addRule(cmd *cobra.Command, _ []string) { req := &control.AddChainLocalOverrideRequest{ Body: &control.AddChainLocalOverrideRequest_Body{ - Target: &control.ChainTarget{ - Type: control.ChainTarget_CONTAINER, - Name: cidStr, - }, - Chain: serializedChain, + Target: target, + Chain: serializedChain, }, } @@ -91,8 +80,10 @@ func initControlAddRuleCmd() { initControlFlags(addRuleCmd) ff := addRuleCmd.Flags() - ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) ff.StringArray(ruleFlag, []string{}, "Rule statement") ff.String(chainIDFlag, "", "Assign ID to the parsed chain") + ff.String(targetNameFlag, "", targetNameDesc) + ff.String(targetTypeFlag, "", targetTypeDesc) + _ = addRuleCmd.MarkFlagRequired(targetTypeFlag) ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") } diff --git a/cmd/frostfs-cli/modules/control/get_rule.go b/cmd/frostfs-cli/modules/control/get_rule.go index e2271c446..050cf165c 100644 --- a/cmd/frostfs-cli/modules/control/get_rule.go +++ b/cmd/frostfs-cli/modules/control/get_rule.go @@ -1,16 +1,13 @@ package control import ( - "crypto/sha256" "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" ) @@ -25,12 +22,7 @@ var getRuleCmd = &cobra.Command{ func getRule(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) - var cnr cid.ID - cidStr, _ := cmd.Flags().GetString(commonflags.CIDFlag) - commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(cidStr)) - - rawCID := make([]byte, sha256.Size) - cnr.Encode(rawCID) + target := parseTarget(cmd) chainID, _ := cmd.Flags().GetString(chainIDFlag) hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) @@ -43,10 +35,7 @@ func getRule(cmd *cobra.Command, _ []string) { req := &control.GetChainLocalOverrideRequest{ Body: &control.GetChainLocalOverrideRequest_Body{ - Target: &control.ChainTarget{ - Name: cidStr, - Type: control.ChainTarget_CONTAINER, - }, + Target: target, ChainId: []byte(chainID), }, } @@ -74,7 +63,9 @@ func initControGetRuleCmd() { initControlFlags(getRuleCmd) ff := getRuleCmd.Flags() - ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + ff.String(targetNameFlag, "", targetNameDesc) + ff.String(targetTypeFlag, "", targetTypeDesc) + _ = getRuleCmd.MarkFlagRequired(targetTypeFlag) ff.String(chainIDFlag, "", "Chain id") ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") } diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index c62651767..31f5ff959 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -1,16 +1,17 @@ package control import ( - "crypto/sha256" + "fmt" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "github.com/nspcc-dev/neo-go/cli/input" "github.com/spf13/cobra" ) @@ -21,22 +22,56 @@ var listRulesCmd = &cobra.Command{ Run: listRules, } +const ( + defaultNamespace = "root" + namespaceTarget = "namespace" + containerTarget = "container" +) + +const ( + targetNameFlag = "target-name" + targetNameDesc = "Resource name in APE resource name format" + targetTypeFlag = "target-type" + targetTypeDesc = "Resource type(container/namespace)" +) + +func parseTarget(cmd *cobra.Command) *control.ChainTarget { + typ, _ := cmd.Flags().GetString(targetTypeFlag) + name, _ := cmd.Flags().GetString(targetNameFlag) + switch typ { + case namespaceTarget: + if name == "" { + ln, err := input.ReadLine(fmt.Sprintf("Target name is not set. Confirm to use %s namespace (n|Y)> ", defaultNamespace)) + commonCmd.ExitOnErr(cmd, "read line error: %w", err) + ln = strings.ToLower(ln) + if len(ln) > 0 && (ln[0] == 'n') { + commonCmd.ExitOnErr(cmd, "read namespace error: %w", fmt.Errorf("setting default value was declined")) + } + name = defaultNamespace + } + return &control.ChainTarget{ + Name: name, + Type: control.ChainTarget_NAMESPACE, + } + case containerTarget: + var cnr cid.ID + commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(name)) + return &control.ChainTarget{ + Name: name, + Type: control.ChainTarget_CONTAINER, + } + default: + commonCmd.ExitOnErr(cmd, "read target type error: %w", fmt.Errorf("unknown target type")) + } + return nil +} + func listRules(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) - var cnr cid.ID - cidStr, _ := cmd.Flags().GetString(commonflags.CIDFlag) - commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(cidStr)) - - rawCID := make([]byte, sha256.Size) - cnr.Encode(rawCID) - req := &control.ListChainLocalOverridesRequest{ Body: &control.ListChainLocalOverridesRequest_Body{ - Target: &control.ChainTarget{ - Name: cidStr, - Type: control.ChainTarget_CONTAINER, - }, + Target: parseTarget(cmd), }, } @@ -71,5 +106,7 @@ func initControlListRulesCmd() { initControlFlags(listRulesCmd) ff := listRulesCmd.Flags() - ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + ff.String(targetNameFlag, "", targetNameDesc) + ff.String(targetTypeFlag, "", targetTypeDesc) + _ = listRulesCmd.MarkFlagRequired(targetTypeFlag) } diff --git a/cmd/frostfs-cli/modules/control/remove_rule.go b/cmd/frostfs-cli/modules/control/remove_rule.go index 612ab7d0a..1c3be375f 100644 --- a/cmd/frostfs-cli/modules/control/remove_rule.go +++ b/cmd/frostfs-cli/modules/control/remove_rule.go @@ -1,15 +1,12 @@ package control import ( - "crypto/sha256" "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/spf13/cobra" ) @@ -28,13 +25,6 @@ var removeRuleCmd = &cobra.Command{ func removeRule(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) - var cnr cid.ID - cidStr, _ := cmd.Flags().GetString(commonflags.CIDFlag) - commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(cidStr)) - - rawCID := make([]byte, sha256.Size) - cnr.Encode(rawCID) - chainID, _ := cmd.Flags().GetString(chainIDFlag) hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) @@ -48,10 +38,7 @@ func removeRule(cmd *cobra.Command, _ []string) { req := &control.RemoveChainLocalOverrideRequest{ Body: &control.RemoveChainLocalOverrideRequest_Body{ - Target: &control.ChainTarget{ - Name: cidStr, - Type: control.ChainTarget_CONTAINER, - }, + Target: parseTarget(cmd), ChainId: chainIDRaw, }, } @@ -81,7 +68,9 @@ func initControlRemoveRuleCmd() { initControlFlags(removeRuleCmd) ff := removeRuleCmd.Flags() - ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + ff.String(targetNameFlag, "", targetNameDesc) + ff.String(targetTypeFlag, "", targetTypeDesc) + _ = removeRuleCmd.MarkFlagRequired(targetTypeFlag) ff.String(chainIDFlag, "", "Chain id") ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") } From a6c9a337cd1177ac07a066b6b29de0ae5ef88e5c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 6 Feb 2024 20:29:39 +0300 Subject: [PATCH 0849/1943] [#965] morph: Get rid of container.List invocations ContainersOf() is better in almost every aspect, besides creating a session when the containers number is between 1024 and 2048 (prefetch script does limited unwrapping). Making List() private helps to ensure it is no longer used and can be safely removed in future. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/container.go | 6 +++--- pkg/morph/client/container/containers_of.go | 2 +- pkg/morph/client/container/list.go | 4 ++-- pkg/services/container/morph/executor.go | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 28f271075..d566c0af3 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -210,7 +210,7 @@ type morphContainerReader struct { src containerCore.Source lister interface { - List(*user.ID) ([]cid.ID, error) + ContainersOf(*user.ID) ([]cid.ID, error) } } @@ -226,8 +226,8 @@ func (x *morphContainerReader) GetEACL(id cid.ID) (*containerCore.EACL, error) { return x.eacl.GetEACL(id) } -func (x *morphContainerReader) List(id *user.ID) ([]cid.ID, error) { - return x.lister.List(id) +func (x *morphContainerReader) ContainersOf(id *user.ID) ([]cid.ID, error) { + return x.lister.ContainersOf(id) } type morphContainerWriter struct { diff --git a/pkg/morph/client/container/containers_of.go b/pkg/morph/client/container/containers_of.go index ce127335d..c4db0fe6e 100644 --- a/pkg/morph/client/container/containers_of.go +++ b/pkg/morph/client/container/containers_of.go @@ -53,7 +53,7 @@ func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) { err := c.client.Morph().TestInvokeIterator(cb, batchSize, cnrHash, containersOfMethod, rawID) if err != nil { if errors.Is(err, unwrap.ErrNoSessionID) { - return c.List(idUser) + return c.list(idUser) } return nil, err } diff --git a/pkg/morph/client/container/list.go b/pkg/morph/client/container/list.go index 8f165f4b1..6fed46c1a 100644 --- a/pkg/morph/client/container/list.go +++ b/pkg/morph/client/container/list.go @@ -8,13 +8,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) -// List returns a list of container identifiers belonging +// list returns a list of container identifiers belonging // to the specified user of FrostFS system. The list is composed // through Container contract call. // // Returns the identifiers of all FrostFS containers if pointer // to user identifier is nil. -func (c *Client) List(idUser *user.ID) ([]cid.ID, error) { +func (c *Client) list(idUser *user.ID) ([]cid.ID, error) { var rawID []byte if idUser != nil { diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index dec022219..f1217705e 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -26,10 +26,10 @@ type Reader interface { containercore.Source containercore.EACLSource - // List returns a list of container identifiers belonging + // ContainersOf returns a list of container identifiers belonging // to the specified user of FrostFS system. Returns the identifiers // of all FrostFS containers if pointer to owner identifier is nil. - List(*user.ID) ([]cid.ID, error) + ContainersOf(*user.ID) ([]cid.ID, error) } // Writer is an interface of container storage updater. @@ -187,7 +187,7 @@ func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) return nil, fmt.Errorf("invalid user ID: %w", err) } - cnrs, err := s.rdr.List(&id) + cnrs, err := s.rdr.ContainersOf(&id) if err != nil { return nil, err } From 6010dfdf3de79a81e70adfad5cd4e0d2c678904b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 10:01:27 +0300 Subject: [PATCH 0850/1943] [#969] policer: Make error skip thread-safe Introduces in afd2ba9a66. Refs #914 Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/process.go | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index df4f0c4c7..0f1198f8d 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -3,6 +3,7 @@ package policer import ( "context" "errors" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -35,9 +36,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { p.log.Warn(logs.PolicerFailureAtObjectSelectForReplication, zap.Error(err)) } - // contains all errors logged in this iteration for each container - cnrErrSkip := make(map[cid.ID][]error) - + skipMap := newSkipMap() for i := range addrs { select { case <-ctx.Done(): @@ -59,12 +58,10 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { if p.objsInWork.add(addr.Address) { err := p.processObject(ctx, addr) - if err != nil && !skipLog(cnrErrSkip[addr.Address.Container()], err) { + if err != nil && !skipMap.addSeenError(addr.Address.Container(), err) { p.log.Error(logs.PolicerUnableToProcessObj, zap.Stringer("object", addr.Address), zap.String("error", err.Error())) - - cnrErrSkip[addr.Address.Container()] = append(cnrErrSkip[addr.Address.Container()], err) } p.cache.Add(addr.Address, time.Now()) p.objsInWork.remove(addr.Address) @@ -79,11 +76,28 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { } } -func skipLog(errs []error, err error) bool { - for _, e := range errs { +type errMap struct { + sync.Mutex + skipMap map[cid.ID][]error +} + +func newSkipMap() *errMap { + return &errMap{ + skipMap: make(map[cid.ID][]error), + } +} + +// addSeenError marks err as seen error for the container. +// Returns true is the error has already been added. +func (m *errMap) addSeenError(cnr cid.ID, err error) bool { + m.Lock() + defer m.Unlock() + + for _, e := range m.skipMap[cnr] { if errors.Is(err, e) { return true } } + m.skipMap[cnr] = append(m.skipMap[cnr], err) return false } From c3fa902780975b129e1d85991016634403575c88 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 10:05:02 +0300 Subject: [PATCH 0851/1943] [#969] policer: Restrict the number of remembered errors Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/process.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 0f1198f8d..a5ebb0010 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -98,6 +98,13 @@ func (m *errMap) addSeenError(cnr cid.ID, err error) bool { return true } } - m.skipMap[cnr] = append(m.skipMap[cnr], err) + + // Restrict list length to avoid possible OOM if some random error is added in future. + const maxErrListLength = 10 + + lst := m.skipMap[cnr] + if len(lst) < maxErrListLength { + m.skipMap[cnr] = append(lst, err) + } return false } From cfc5ce7853276b0d86ba645eaa2ad186b406b98c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Feb 2024 19:25:24 +0300 Subject: [PATCH 0852/1943] [#964] metabase: Drop GC marks if object not found GC inhumes expired locks and tombstones on all the shards. So it could be GC mark without object. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/delete.go | 13 ++++++-- .../metabase/delete_test.go | 31 +++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 484dbe41b..8e5557636 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -254,9 +254,18 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter // unmarshal object, work only with physically stored (raw == true) objects obj, err := db.get(tx, addr, key, false, true, currEpoch) if err != nil { + if client.IsErrObjectNotFound(err) { + addrKey = addressKey(addr, key) + if garbageBKT != nil { + err := garbageBKT.Delete(addrKey) + if err != nil { + return deleteSingleResult{}, fmt.Errorf("could not remove from garbage bucket: %w", err) + } + } + return deleteSingleResult{}, nil + } var siErr *objectSDK.SplitInfoError - - if client.IsErrObjectNotFound(err) || errors.As(err, &siErr) { + if errors.As(err, &siErr) { // if object is virtual (parent) then do nothing, it will be deleted with last child return deleteSingleResult{}, nil } diff --git a/pkg/local_object_storage/metabase/delete_test.go b/pkg/local_object_storage/metabase/delete_test.go index 5a52868d3..649c0ebe3 100644 --- a/pkg/local_object_storage/metabase/delete_test.go +++ b/pkg/local_object_storage/metabase/delete_test.go @@ -182,6 +182,37 @@ func TestDelete(t *testing.T) { require.Equal(t, 0, len(addrs)) } +func TestDeleteDropsGCMarkIfObjectNotFound(t *testing.T) { + db := newDB(t, meta.WithEpochState(epochState{currEpoch})) + defer func() { require.NoError(t, db.Close()) }() + + addr := oidtest.Address() + + var prm meta.InhumePrm + prm.SetAddresses(addr) + prm.SetGCMark() + _, err := db.Inhume(context.Background(), prm) + require.NoError(t, err) + + var garbageCount int + var itPrm meta.GarbageIterationPrm + itPrm.SetHandler(func(g meta.GarbageObject) error { + garbageCount++ + return nil + }) + require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm)) + require.Equal(t, 1, garbageCount) + + var delPrm meta.DeletePrm + delPrm.SetAddresses(addr) + _, err = db.Delete(context.Background(), delPrm) + require.NoError(t, err) + + garbageCount = 0 + require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm)) + require.Equal(t, 0, garbageCount) +} + func metaDelete(db *meta.DB, addrs ...oid.Address) error { var deletePrm meta.DeletePrm deletePrm.SetAddresses(addrs...) From 053a195ac2812b2b3d6c1c865cf564c027a30f9e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 7 Feb 2024 19:44:28 +0300 Subject: [PATCH 0853/1943] [#968] adm: Allow concurrent epoch ticks Previous fix was incomplete, there are two possible places for this error to occur. Refs #592 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/epoch.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/epoch.go index 39dbb0caf..e491e8b8f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/epoch.go @@ -37,18 +37,14 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { return err } - if err := wCtx.sendConsensusTx(bw.Bytes()); err != nil { - return err + if err = wCtx.sendConsensusTx(bw.Bytes()); err == nil { + err = wCtx.awaitTx() } - - if err := wCtx.awaitTx(); err != nil { - if strings.Contains(err.Error(), "invalid epoch") { - cmd.Println("Epoch has already ticked.") - return nil - } - return err + if err != nil && strings.Contains(err.Error(), "invalid epoch") { + cmd.Println("Epoch has already ticked.") + return nil } - return nil + return err } func emitNewEpochCall(bw *io.BufBinWriter, wCtx *initializeContext, nmHash util.Uint160) error { From cbfeb724662ccc8c041286cacbf8ca18df2eedf7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Feb 2024 20:14:47 +0300 Subject: [PATCH 0854/1943] [#956] policer: Remove WithMaxCapacity option We already provide the pool and this argument is used only for preallocation. No functional changes. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/object.go | 1 - pkg/services/policer/option.go | 10 ---------- pkg/services/policer/policer.go | 2 +- pkg/services/policer/policer_test.go | 8 ++++++++ 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 5c82f0dcc..7f1d094fd 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -263,7 +263,6 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl ) } }), - policer.WithMaxCapacity(c.cfgObject.pool.replicatorPoolSize), policer.WithPool(c.cfgObject.pool.replication), policer.WithMetrics(c.metricsCollector.PolicerMetrics()), ) diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index fe47555c4..92cf83a84 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -64,8 +64,6 @@ type cfg struct { taskPool *ants.Pool - maxCapacity int - batchSize, cacheSize uint32 rebalanceFreq, evictDuration, sleepDuration time.Duration @@ -158,14 +156,6 @@ func WithRedundantCopyCallback(cb RedundantCopyCallback) Option { } } -// WithMaxCapacity returns option to set max capacity -// that can be set to the pool. -func WithMaxCapacity(capacity int) Option { - return func(c *cfg) { - c.maxCapacity = capacity - } -} - // WithPool returns option to set pool for // policy and replication operations. func WithPool(p *ants.Pool) Option { diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index a68b194d4..363c0b922 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -66,7 +66,7 @@ func New(opts ...Option) *Policer { cfg: c, cache: cache, objsInWork: &objectsInWork{ - objs: make(map[oid.Address]struct{}, c.maxCapacity), + objs: make(map[oid.Address]struct{}, c.taskPool.Cap()), }, } } diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index c73d33629..41abeb384 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -239,6 +239,7 @@ func TestProcessObject(t *testing.T) { gotReplicateTo = append(gotReplicateTo, int(node.PublicKey()[0])) } })), + WithPool(testPool(t)), ) addrWithType := objectcore.AddressWithType{ @@ -276,6 +277,7 @@ func TestProcessObjectError(t *testing.T) { p := New( WithContainerSource(source), WithBuryFunc(buryFn), + WithPool(testPool(t)), ) addrWithType := objectcore.AddressWithType{ @@ -348,6 +350,12 @@ func TestIteratorContract(t *testing.T) { }, it.calls) } +func testPool(t *testing.T) *ants.Pool { + pool, err := ants.NewPool(4) + require.NoError(t, err) + return pool +} + type nextResult struct { objs []objectcore.AddressWithType err error From edbe06e07ea0fab2bd3edace784aa338dcaf48d1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Feb 2024 20:57:49 +0300 Subject: [PATCH 0855/1943] [#956] policer/test: Reuse testPool helper Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/policer_test.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 41abeb384..50f206fd9 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -48,16 +48,12 @@ func TestBuryObjectWithoutContainer(t *testing.T) { return nil } - // Task pool - pool, err := ants.NewPool(4) - require.NoError(t, err) - // Policer instance p := New( WithKeySpaceIterator(&sliceKeySpaceIterator{objs: objs}), WithContainerSource(containerSrc), WithBuryFunc(buryFn), - WithPool(pool), + WithPool(testPool(t)), ) ctx, cancel := context.WithCancel(context.Background()) @@ -298,9 +294,6 @@ func TestIteratorContract(t *testing.T) { return nil } - pool, err := ants.NewPool(4) - require.NoError(t, err) - it := &predefinedIterator{ scenario: []nextResult{ {objs, nil}, @@ -326,7 +319,7 @@ func TestIteratorContract(t *testing.T) { WithKeySpaceIterator(it), WithContainerSource(containerSrc), WithBuryFunc(buryFn), - WithPool(pool), + WithPool(testPool(t)), func(c *cfg) { c.sleepDuration = time.Millisecond }, From 426cf58b986dbf6c11e370fbdd9ca2788d0832d8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Feb 2024 20:17:04 +0300 Subject: [PATCH 0856/1943] [#956] node: Remove pool sizes from config struct They are available through the pool methods and unused outside of the function that sets them. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 29582c782..e5881cc2c 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -632,14 +632,8 @@ type cfgAccessPolicyEngine struct { type cfgObjectRoutines struct { putRemote *ants.Pool - putRemoteCapacity int - putLocal *ants.Pool - putLocalCapacity int - - replicatorPoolSize int - replication *ants.Pool } @@ -1094,20 +1088,20 @@ func initObjectPool(cfg *config.Config) (pool cfgObjectRoutines) { optNonBlocking := ants.WithNonblocking(true) - pool.putRemoteCapacity = objectconfig.Put(cfg).PoolSizeRemote() - pool.putRemote, err = ants.NewPool(pool.putRemoteCapacity, optNonBlocking) + putRemoteCapacity := objectconfig.Put(cfg).PoolSizeRemote() + pool.putRemote, err = ants.NewPool(putRemoteCapacity, optNonBlocking) fatalOnErr(err) - pool.putLocalCapacity = objectconfig.Put(cfg).PoolSizeLocal() - pool.putLocal, err = ants.NewPool(pool.putLocalCapacity, optNonBlocking) + putLocalCapacity := objectconfig.Put(cfg).PoolSizeLocal() + pool.putLocal, err = ants.NewPool(putLocalCapacity, optNonBlocking) fatalOnErr(err) - pool.replicatorPoolSize = replicatorconfig.PoolSize(cfg) - if pool.replicatorPoolSize <= 0 { - pool.replicatorPoolSize = pool.putRemoteCapacity + replicatorPoolSize := replicatorconfig.PoolSize(cfg) + if replicatorPoolSize <= 0 { + replicatorPoolSize = putRemoteCapacity } - pool.replication, err = ants.NewPool(pool.replicatorPoolSize) + pool.replication, err = ants.NewPool(replicatorPoolSize) fatalOnErr(err) return pool From 805862f4b726c13e57225c472e4992eab29a37aa Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 2 Feb 2024 20:28:05 +0300 Subject: [PATCH 0857/1943] [#956] node: Allow to reload goroutine pool sizes Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config.go | 23 +++++++++++++++++++++++ internal/logs/logs.go | 1 + 2 files changed, 24 insertions(+) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e5881cc2c..c41863ecd 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1235,6 +1235,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { setRuntimeParameters(c) return nil }}) + components = append(components, dCmp{"pools", c.reloadPools}) components = append(components, dCmp{"tracing", func() error { updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg)) if updated { @@ -1279,6 +1280,28 @@ func (c *cfg) reloadConfig(ctx context.Context) { c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } +func (c *cfg) reloadPools() error { + newSize := objectconfig.Put(c.appCfg).PoolSizeLocal() + c.reloadPool(c.cfgObject.pool.putLocal, newSize, "object.put.local_pool_size") + + newSize = objectconfig.Put(c.appCfg).PoolSizeRemote() + c.reloadPool(c.cfgObject.pool.putRemote, newSize, "object.put.remote_pool_size") + + newSize = replicatorconfig.PoolSize(c.appCfg) + c.reloadPool(c.cfgObject.pool.replication, newSize, "replicator.pool_size") + + return nil +} + +func (c *cfg) reloadPool(p *ants.Pool, newSize int, name string) { + oldSize := p.Cap() + if oldSize != newSize { + c.log.Info(logs.FrostFSNodePoolConfigurationUpdate, zap.String("field", name), + zap.Int("old", oldSize), zap.Int("new", newSize)) + p.Tune(newSize) + } +} + func (c *cfg) reloadAppConfig() error { unlock := c.LockAppConfigExclusive() defer unlock() diff --git a/internal/logs/logs.go b/internal/logs/logs.go index e81976a32..c3d2e5150 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -451,6 +451,7 @@ const ( FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" FrostFSNodeTracingConfigationUpdated = "tracing configation updated" FrostFSNodeStorageEngineConfigurationUpdate = "storage engine configuration update" + FrostFSNodePoolConfigurationUpdate = "adjust pool configuration" FrostFSNodeUpdatedConfigurationApplying = "updated configuration applying" FrostFSNodeConfigurationHasBeenReloadedSuccessfully = "configuration has been reloaded successfully" FrostFSNodeReadNewlyCreatedContainerAfterTheNotification = "read newly created container after the notification" From 80b581d49967178e927d1575de759dcf37a9d519 Mon Sep 17 00:00:00 2001 From: Olga Konstantinova Date: Sat, 3 Feb 2024 20:17:12 +0300 Subject: [PATCH 0858/1943] [#466] adm: Allow to download contracts from Gitea Signed-off-by: Olga Konstantinova --- .../internal/modules/morph/download.go | 81 +++++++++++++++++++ .../internal/modules/morph/initialize.go | 7 ++ .../modules/morph/initialize_deploy.go | 10 ++- .../internal/modules/morph/root.go | 13 ++- go.mod | 4 + go.sum | 21 +++++ 6 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/download.go diff --git a/cmd/frostfs-adm/internal/modules/morph/download.go b/cmd/frostfs-adm/internal/modules/morph/download.go new file mode 100644 index 000000000..5bd2d98bd --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/download.go @@ -0,0 +1,81 @@ +package morph + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" + + "code.gitea.io/sdk/gitea" + "github.com/spf13/cobra" +) + +func downloadContracts(cmd *cobra.Command, url string) (io.ReadCloser, error) { + cmd.Printf("Downloading contracts archive from '%s'\n", url) + + // HTTP client with connect timeout + client := http.Client{ + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + }).DialContext, + }, + } + + ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("can't create request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("can't fetch contracts archive: %w", err) + } + return resp.Body, nil +} + +func downloadContractsFromRepository(cmd *cobra.Command) (io.ReadCloser, error) { + client, err := gitea.NewClient("https://git.frostfs.info") + if err != nil { + return nil, fmt.Errorf("can't initialize repository client: %w", err) + } + + releases, _, err := client.ListReleases("TrueCloudLab", "frostfs-contract", gitea.ListReleasesOptions{}) + if err != nil { + return nil, fmt.Errorf("can't fetch release information: %w", err) + } + + var latestRelease *gitea.Release + for _, r := range releases { + if !r.IsDraft && !r.IsPrerelease { + latestRelease = r + break + } + } + + if latestRelease == nil { + return nil, fmt.Errorf("attempt to fetch contracts archive from the offitial repository failed: no releases found") + } + + cmd.Printf("Found release %s (%s)\n", latestRelease.TagName, latestRelease.Title) + + var url string + for _, a := range latestRelease.Attachments { + if strings.HasPrefix(a.Name, "frostfs-contract") { + url = a.DownloadURL + break + } + } + if url == "" { + return nil, errors.New("can't find contracts archive in the latest release") + } + + return downloadContracts(cmd, url) +} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 78356e6b4..04e8620cc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -50,6 +50,7 @@ type initializeContext struct { Contracts map[string]*contractState Command *cobra.Command ContractPath string + ContractURL string } var ErrTooManyAlphabetNodes = fmt.Errorf("too many alphabet nodes (maximum allowed is %d)", maxAlphabetNodes) @@ -152,6 +153,11 @@ func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContex return nil, err } + var ctrURL string + if needContracts { + ctrURL, _ = cmd.Flags().GetString(contractsURLFlag) + } + if err := checkNotaryEnabled(c); err != nil { return nil, err } @@ -176,6 +182,7 @@ func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContex Command: cmd, Contracts: make(map[string]*contractState), ContractPath: ctrPath, + ContractURL: ctrURL, } if needContracts { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 6cf75c5f9..f715f0e03 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -399,10 +399,14 @@ func (c *initializeContext) readContracts(names []string) error { } } else { var r io.ReadCloser - if c.ContractPath == "" { - return errors.New("contracts flag is missing") + if c.ContractPath != "" { + r, err = os.Open(c.ContractPath) + } else if c.ContractURL != "" { + r, err = downloadContracts(c.Command, c.ContractURL) + } else { + r, err = downloadContractsFromRepository(c.Command) } - r, err = os.Open(c.ContractPath) + if err != nil { return fmt.Errorf("can't open contracts archive: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 600fe21c8..6cc2d5a96 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -17,6 +17,9 @@ const ( storageGasCLIFlag = "initial-gas" storageGasConfigFlag = "storage.initial_gas" contractsInitFlag = "contracts" + contractsInitFlagDesc = "Path to archive with compiled FrostFS contracts (the default is to fetch the latest release from the official repository)" + contractsURLFlag = "contracts-url" + contractsURLFlagDesc = "URL to archive with compiled FrostFS contracts" maxObjectSizeInitFlag = "network.max_object_size" maxObjectSizeCLIFlag = "max-object-size" epochDurationInitFlag = "network.epoch_duration" @@ -370,8 +373,9 @@ func initUpdateContractsCmd() { RootCmd.AddCommand(updateContractsCmd) updateContractsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) updateContractsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) - updateContractsCmd.Flags().String(contractsInitFlag, "", "Path to archive with compiled FrostFS contracts") - _ = updateContractsCmd.MarkFlagRequired(contractsInitFlag) + updateContractsCmd.Flags().String(contractsInitFlag, "", contractsInitFlagDesc) + updateContractsCmd.Flags().String(contractsURLFlag, "", contractsURLFlagDesc) + updateContractsCmd.MarkFlagsMutuallyExclusive(contractsInitFlag, contractsURLFlag) } func initDumpBalancesCmd() { @@ -441,8 +445,8 @@ func initInitCmd() { RootCmd.AddCommand(initCmd) initCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) initCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) - initCmd.Flags().String(contractsInitFlag, "", "Path to archive with compiled FrostFS contracts") - _ = initCmd.MarkFlagRequired(contractsInitFlag) + initCmd.Flags().String(contractsInitFlag, "", contractsInitFlagDesc) + initCmd.Flags().String(contractsURLFlag, "", contractsURLFlagDesc) initCmd.Flags().Uint(epochDurationCLIFlag, 240, "Amount of side chain blocks in one FrostFS epoch") initCmd.Flags().Uint(maxObjectSizeCLIFlag, 67108864, "Max single object size in bytes") initCmd.Flags().Bool(homomorphicHashDisabledCLIFlag, false, "Disable object homomorphic hashing") @@ -451,6 +455,7 @@ func initInitCmd() { initCmd.Flags().Uint64(containerAliasFeeCLIFlag, 500, "Container alias fee") initCmd.Flags().String(protoConfigPath, "", "Path to the consensus node configuration") initCmd.Flags().String(localDumpFlag, "", "Path to the blocks dump file") + initCmd.MarkFlagsMutuallyExclusive(contractsInitFlag, contractsURLFlag) } func initGenerateAlphabetCmd() { diff --git a/go.mod b/go.mod index 07097c547..5d6edc222 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-node go 1.20 require ( + code.gitea.io/sdk/gitea v0.17.1 git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240112150928-72885aae835c git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 @@ -61,8 +62,10 @@ require ( github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/davidmz/go-pageant v1.0.2 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -71,6 +74,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.4 // indirect diff --git a/go.sum b/go.sum index 5243ac435..b9052efde 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= +code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240112150928-72885aae835c h1:dO2VLtoTQre4W28i1gSuW1MO80qFd0Lq9Q46Pu3q3TA= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240112150928-72885aae835c/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa h1:uuN90s74idThHXi/jF/IlTdFimOIs+IBU44rdTE/Zdo= @@ -49,6 +51,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= +github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -61,6 +65,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= +github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -100,6 +106,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 h1:HcUWd006luQPljE73d5sk+ github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -300,8 +308,10 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= @@ -309,6 +319,7 @@ golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/i golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -322,6 +333,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -330,6 +343,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= @@ -355,10 +369,15 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -367,6 +386,7 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -375,6 +395,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 8e2a0611f4a93ec580122bb18920a3a49325f3b3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Feb 2024 14:09:58 +0300 Subject: [PATCH 0859/1943] [#947] tree: Add method to list all trees Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/pilorama/boltdb.go | 62 +++++++++++++++++++ pkg/local_object_storage/pilorama/forest.go | 51 +++++++++++++++ .../pilorama/forest_test.go | 56 +++++++++++++++++ .../pilorama/interface.go | 44 +++++++++++++ pkg/local_object_storage/shard/tree.go | 21 +++++++ 5 files changed, 234 insertions(+) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 52cdec586..39400391f 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1134,6 +1134,68 @@ func (t *boltForest) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) return err } +// TreeListTrees implements ForestStorage. +func (t *boltForest) TreeListTrees(ctx context.Context, prm TreeListTreesPrm) (*TreeListTreesResult, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeListTrees", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeListTrees") + defer span.End() + + t.modeMtx.RLock() + defer t.modeMtx.RUnlock() + + if t.mode.NoMetabase() { + return nil, ErrDegradedMode + } + + batchSize := prm.BatchSize + if batchSize <= 0 { + batchSize = treeListTreesBatchSizeDefault + } + var res TreeListTreesResult + err := metaerr.Wrap(t.db.View(func(tx *bbolt.Tx) error { + c := tx.Cursor() + checkNextPageToken := true + for k, _ := c.Seek(prm.NextPageToken); k != nil; k, _ = c.Next() { + if bytes.Equal(k, dataBucket) || bytes.Equal(k, logBucket) { + continue + } + + if checkNextPageToken && bytes.Equal(k, prm.NextPageToken) { + checkNextPageToken = false + continue + } + + var contID cidSDK.ID + if err := contID.Decode(k[:32]); err != nil { + return fmt.Errorf("failed to decode containerID: %w", err) + } + res.Items = append(res.Items, ContainerIDTreeID{ + CID: contID, + TreeID: string(k[32:]), + }) + + if len(res.Items) == batchSize { + res.NextPageToken = make([]byte, len(k)) + copy(res.NextPageToken, k) + break + } + } + return nil + })) + success = err == nil + if err != nil { + return nil, err + } + return &res, nil +} + func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node, error) { c := bTree.Cursor() diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index b82e5b3bd..8a1e86147 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -2,6 +2,7 @@ package pilorama import ( "context" + "fmt" "sort" "strings" @@ -260,3 +261,53 @@ func (f *memoryForest) TreeLastSyncHeight(_ context.Context, cid cid.ID, treeID } return t.syncHeight, nil } + +// TreeListTrees implements Forest. +func (f *memoryForest) TreeListTrees(_ context.Context, prm TreeListTreesPrm) (*TreeListTreesResult, error) { + batchSize := prm.BatchSize + if batchSize <= 0 { + batchSize = treeListTreesBatchSizeDefault + } + tmpSlice := make([]string, 0, len(f.treeMap)) + for k := range f.treeMap { + tmpSlice = append(tmpSlice, k) + } + sort.Strings(tmpSlice) + var idx int + if len(prm.NextPageToken) > 0 { + last := string(prm.NextPageToken) + idx, _ = sort.Find(len(tmpSlice), func(i int) int { + return -1 * strings.Compare(tmpSlice[i], last) + }) + if idx == len(tmpSlice) { + return &TreeListTreesResult{}, nil + } + if tmpSlice[idx] == last { + idx++ + } + } + + var result TreeListTreesResult + for idx < len(tmpSlice) { + cidAndTree := strings.Split(tmpSlice[idx], "/") + if len(cidAndTree) != 2 { + return nil, fmt.Errorf("invalid format: key must be cid and treeID") + } + var contID cid.ID + if err := contID.DecodeString(cidAndTree[0]); err != nil { + return nil, fmt.Errorf("invalid format: %w", err) + } + + result.Items = append(result.Items, ContainerIDTreeID{ + CID: contID, + TreeID: cidAndTree[1], + }) + + if len(result.Items) == batchSize { + result.NextPageToken = []byte(tmpSlice[idx]) + break + } + idx++ + } + return &result, nil +} diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 5813395f0..2e7c1f529 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -1189,3 +1189,59 @@ func testTreeLastSyncHeight(t *testing.T, f ForestStorage) { require.ErrorIs(t, err, ErrTreeNotFound) }) } + +func TestForest_ListTrees(t *testing.T) { + for i := range providers { + i := i + t.Run(providers[i].name, func(t *testing.T) { + testTreeListTrees(t, providers[i].construct) + }) + } +} + +func testTreeListTrees(t *testing.T, constructor func(t testing.TB, _ ...Option) ForestStorage) { + batchSize := 10 + t.Run("empty", func(t *testing.T) { + testTreeListTreesCount(t, constructor, batchSize, 0) + }) + t.Run("count lower than batch size", func(t *testing.T) { + testTreeListTreesCount(t, constructor, batchSize, batchSize-1) + }) + t.Run("count equals batch size", func(t *testing.T) { + testTreeListTreesCount(t, constructor, batchSize, batchSize) + }) + t.Run("count greater than batch size", func(t *testing.T) { + testTreeListTreesCount(t, constructor, batchSize, batchSize+1) + }) + t.Run("count equals multiplied batch size", func(t *testing.T) { + testTreeListTreesCount(t, constructor, batchSize, 3*batchSize) + }) + t.Run("count equals multiplied batch size with addition", func(t *testing.T) { + testTreeListTreesCount(t, constructor, batchSize, 3*batchSize+batchSize/2) + }) +} + +func testTreeListTreesCount(t *testing.T, constructor func(t testing.TB, _ ...Option) ForestStorage, batchSize, count int) { + f := constructor(t) + var expected []ContainerIDTreeID + + treeIDs := []string{"version", "system", "s", "avada kedavra"} + for i := 0; i < count; i++ { + cid := cidtest.ID() + treeID := treeIDs[i%len(treeIDs)] + expected = append(expected, ContainerIDTreeID{ + CID: cid, + TreeID: treeID, + }) + + ops := prepareRandomTree(5, 5) + for _, op := range ops { + require.NoError(t, f.TreeApply(context.Background(), cid, treeID, &op, false)) + } + } + + actual, err := treeListAll(context.Background(), f, batchSize) + require.NoError(t, err) + + require.ElementsMatch(t, expected, actual) +} diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index e7f7eb512..3efd1a68b 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -63,6 +63,9 @@ type ForestStorage interface { SetMode(m mode.Mode) error SetParentID(id string) Forest + + // TreeListTrees returns all pairs "containerID:treeID". + TreeListTrees(ctx context.Context, prm TreeListTreesPrm) (*TreeListTreesResult, error) } const ( @@ -85,3 +88,44 @@ var ErrInvalidCIDDescriptor = logicerr.New("cid descriptor is invalid") func (d CIDDescriptor) checkValid() bool { return 0 <= d.Position && d.Position < d.Size } + +var treeListTreesBatchSizeDefault = 1000 + +type ContainerIDTreeID struct { + CID cidSDK.ID + TreeID string +} + +type TreeListTreesPrm struct { + NextPageToken []byte + // BatchSize is batch size to list trees. If not lower or equals zero, than treeListTreesBatchSizeDefault is used. + BatchSize int +} + +type TreeListTreesResult struct { + NextPageToken []byte + Items []ContainerIDTreeID +} + +func TreeListAll(ctx context.Context, f ForestStorage) ([]ContainerIDTreeID, error) { + return treeListAll(ctx, f, treeListTreesBatchSizeDefault) +} + +func treeListAll(ctx context.Context, f ForestStorage, batchSize int) ([]ContainerIDTreeID, error) { + var prm TreeListTreesPrm + var result []ContainerIDTreeID + first := true + + for len(prm.NextPageToken) > 0 || first { + first = false + + res, err := f.TreeListTrees(ctx, prm) + if err != nil { + return nil, err + } + prm.NextPageToken = res.NextPageToken + result = append(result, res.Items...) + } + + return result, nil +} diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 7795b820d..017b3450d 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -353,3 +353,24 @@ func (s *Shard) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID st } return s.pilorama.TreeLastSyncHeight(ctx, cid, treeID) } + +func (s *Shard) TreeListTrees(ctx context.Context, prm pilorama.TreeListTreesPrm) (*pilorama.TreeListTreesResult, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeListTrees", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + ), + ) + defer span.End() + + if s.pilorama == nil { + return nil, ErrPiloramaDisabled + } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return nil, ErrDegradedMode + } + return s.pilorama.TreeListTrees(ctx, prm) +} From a6eb66bf9cc932f5f42a491386d6a62e402fe0cf Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Feb 2024 15:42:30 +0300 Subject: [PATCH 0860/1943] [#947] evacuate: Refactor evacuate parameters Drop methods to make it easier to extend. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 54 ++++++------------- .../engine/evacuate_test.go | 30 +++++------ pkg/services/control/server/evacuate.go | 9 ++-- pkg/services/control/server/evacuate_async.go | 11 ++-- 4 files changed, 43 insertions(+), 61 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index f522e6fbd..d95098fbf 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -30,10 +30,10 @@ var ( // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { - shardID []*shard.ID - handler func(context.Context, oid.Address, *objectSDK.Object) error - ignoreErrors bool - async bool + ShardID []*shard.ID + Handler func(context.Context, oid.Address, *objectSDK.Object) error + IgnoreErrors bool + Async bool } // EvacuateShardRes represents result of the EvacuateShard operation. @@ -54,26 +54,6 @@ func NewEvacuateShardRes() *EvacuateShardRes { } } -// WithShardIDList sets shard ID. -func (p *EvacuateShardPrm) WithShardIDList(id []*shard.ID) { - p.shardID = id -} - -// WithIgnoreErrors sets flag to ignore errors. -func (p *EvacuateShardPrm) WithIgnoreErrors(ignore bool) { - p.ignoreErrors = ignore -} - -// WithFaultHandler sets handler to call for objects which cannot be saved on other shards. -func (p *EvacuateShardPrm) WithFaultHandler(f func(context.Context, oid.Address, *objectSDK.Object) error) { - p.handler = f -} - -// WithAsync sets flag to run evacuate async. -func (p *EvacuateShardPrm) WithAsync(async bool) { - p.async = async -} - // Evacuated returns amount of evacuated objects. // Objects for which handler returned no error are also assumed evacuated. func (p *EvacuateShardRes) Evacuated() uint64 { @@ -145,20 +125,20 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev default: } - shardIDs := make([]string, len(prm.shardID)) - for i := range prm.shardID { - shardIDs[i] = prm.shardID[i].String() + shardIDs := make([]string, len(prm.ShardID)) + for i := range prm.ShardID { + shardIDs[i] = prm.ShardID[i].String() } ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Evacuate", trace.WithAttributes( attribute.StringSlice("shardIDs", shardIDs), - attribute.Bool("async", prm.async), - attribute.Bool("ignoreErrors", prm.ignoreErrors), + attribute.Bool("async", prm.Async), + attribute.Bool("ignoreErrors", prm.IgnoreErrors), )) defer span.End() - shards, weights, err := e.getActualShards(shardIDs, prm.handler != nil) + shards, weights, err := e.getActualShards(shardIDs, prm.Handler != nil) if err != nil { return nil, err } @@ -173,7 +153,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev } res := NewEvacuateShardRes() - ctx = ctxOrBackground(ctx, prm.async) + ctx = ctxOrBackground(ctx, prm.Async) eg, egCtx, err := e.evacuateLimiter.TryStart(ctx, shardIDs, res) if err != nil { return nil, err @@ -183,7 +163,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev return e.evacuateShards(egCtx, shardIDs, prm, res, shards, weights, shardsToEvacuate) }) - if prm.async { + if prm.Async { return nil, nil } @@ -204,8 +184,8 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShards", trace.WithAttributes( attribute.StringSlice("shardIDs", shardIDs), - attribute.Bool("async", prm.async), - attribute.Bool("ignoreErrors", prm.ignoreErrors), + attribute.Bool("async", prm.Async), + attribute.Bool("ignoreErrors", prm.IgnoreErrors), )) defer func() { @@ -357,7 +337,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to getRes, err := sh.Get(ctx, getPrm) if err != nil { - if prm.ignoreErrors { + if prm.IgnoreErrors { res.failed.Add(1) continue } @@ -375,13 +355,13 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to continue } - if prm.handler == nil { + if prm.Handler == nil { // Do not check ignoreErrors flag here because // ignoring errors on put make this command kinda useless. return fmt.Errorf("%w: %s", errPutShard, toEvacuate[i]) } - err = prm.handler(ctx, addr, getRes.Object()) + err = prm.Handler(ctx, addr, getRes.Object()) if err != nil { e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index d874734d3..2dc4a177c 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -102,7 +102,7 @@ func TestEvacuateShard(t *testing.T) { checkHasObjects(t) var prm EvacuateShardPrm - prm.WithShardIDList(ids[2:3]) + prm.ShardID = ids[2:3] t.Run("must be read-only", func(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) @@ -173,13 +173,13 @@ func TestEvacuateNetwork(t *testing.T) { require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly)) var prm EvacuateShardPrm - prm.shardID = ids[0:1] + prm.ShardID = ids[0:1] res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errMustHaveTwoShards) require.Equal(t, uint64(0), res.Evacuated()) - prm.handler = acceptOneOf(objects, 2) + prm.Handler = acceptOneOf(objects, 2) res, err = e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) @@ -196,15 +196,15 @@ func TestEvacuateNetwork(t *testing.T) { require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) var prm EvacuateShardPrm - prm.shardID = ids[1:2] - prm.handler = acceptOneOf(objects, 2) + prm.ShardID = ids[1:2] + prm.Handler = acceptOneOf(objects, 2) res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) require.Equal(t, uint64(2), res.Evacuated()) t.Run("no errors", func(t *testing.T) { - prm.handler = acceptOneOf(objects, 3) + prm.Handler = acceptOneOf(objects, 3) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) @@ -233,15 +233,15 @@ func TestEvacuateNetwork(t *testing.T) { } var prm EvacuateShardPrm - prm.shardID = evacuateIDs - prm.handler = acceptOneOf(objects, totalCount-1) + prm.ShardID = evacuateIDs + prm.Handler = acceptOneOf(objects, totalCount-1) res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) require.Equal(t, totalCount-1, res.Evacuated()) t.Run("no errors", func(t *testing.T) { - prm.handler = acceptOneOf(objects, totalCount) + prm.Handler = acceptOneOf(objects, totalCount) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) @@ -261,8 +261,8 @@ func TestEvacuateCancellation(t *testing.T) { require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) var prm EvacuateShardPrm - prm.shardID = ids[1:2] - prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ShardID = ids[1:2] + prm.Handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { select { case <-ctx.Done(): return ctx.Err() @@ -292,8 +292,8 @@ func TestEvacuateSingleProcess(t *testing.T) { running := make(chan interface{}) var prm EvacuateShardPrm - prm.shardID = ids[1:2] - prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ShardID = ids[1:2] + prm.Handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { select { case <-running: default: @@ -334,8 +334,8 @@ func TestEvacuateAsync(t *testing.T) { running := make(chan interface{}) var prm EvacuateShardPrm - prm.shardID = ids[1:2] - prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ShardID = ids[1:2] + prm.Handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { select { case <-running: default: diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index 8f62c3489..52ef083c4 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -25,10 +25,11 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe return nil, status.Error(codes.PermissionDenied, err.Error()) } - var prm engine.EvacuateShardPrm - prm.WithShardIDList(s.getShardIDList(req.GetBody().GetShard_ID())) - prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) - prm.WithFaultHandler(s.replicate) + prm := engine.EvacuateShardPrm{ + ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), + IgnoreErrors: req.GetBody().GetIgnoreErrors(), + Handler: s.replicate, + } res, err := s.s.Evacuate(ctx, prm) if err != nil { diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index cdf3656e2..112d4449f 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -17,11 +17,12 @@ func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartSha return nil, status.Error(codes.PermissionDenied, err.Error()) } - var prm engine.EvacuateShardPrm - prm.WithShardIDList(s.getShardIDList(req.GetBody().GetShard_ID())) - prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) - prm.WithFaultHandler(s.replicate) - prm.WithAsync(true) + prm := engine.EvacuateShardPrm{ + ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), + IgnoreErrors: req.GetBody().GetIgnoreErrors(), + Handler: s.replicate, + Async: true, + } _, err = s.s.Evacuate(ctx, prm) if err != nil { From b3f3505adaca55ec9aeca1a1a6d6a0a2e584f3f9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Feb 2024 16:33:09 +0300 Subject: [PATCH 0861/1943] [#947] cli: Allow to specify evacuation scope It may be required to evacuate only objects or only tree or all, so now it spossible to specify. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/evacuation.go | 23 + pkg/local_object_storage/engine/evacuate.go | 30 + pkg/services/control/server/evacuate.go | 1 + pkg/services/control/server/evacuate_async.go | 5 + pkg/services/control/service.pb.go | 1211 +++++++++-------- pkg/services/control/service.proto | 8 + pkg/services/control/service_frostfs.pb.go | 2 + 7 files changed, 706 insertions(+), 574 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 821dd1a6c..9d4cb64ac 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -19,6 +19,11 @@ import ( const ( awaitFlag = "await" noProgressFlag = "no-progress" + scopeFlag = "scope" + + scopeAll = "all" + scopeObjects = "objects" + scopeTrees = "trees" ) var evacuationShardCmd = &cobra.Command{ @@ -57,6 +62,7 @@ func startEvacuateShard(cmd *cobra.Command, _ []string) { Body: &control.StartShardEvacuationRequest_Body{ Shard_ID: getShardIDList(cmd), IgnoreErrors: ignoreErrors, + Scope: getEvacuationScope(cmd), }, } @@ -82,6 +88,22 @@ func startEvacuateShard(cmd *cobra.Command, _ []string) { } } +func getEvacuationScope(cmd *cobra.Command) uint32 { + rawScope, err := cmd.Flags().GetString(scopeFlag) + commonCmd.ExitOnErr(cmd, "Invalid scope value: %w", err) + switch rawScope { + case scopeAll: + return uint32(control.StartShardEvacuationRequest_Body_OBJECTS) | uint32(control.StartShardEvacuationRequest_Body_TREES) + case scopeObjects: + return uint32(control.StartShardEvacuationRequest_Body_OBJECTS) + case scopeTrees: + return uint32(control.StartShardEvacuationRequest_Body_TREES) + default: + commonCmd.ExitOnErr(cmd, "Invalid scope value: %w", fmt.Errorf("unknown scope %s", rawScope)) + } + return uint32(control.StartShardEvacuationRequest_Body_NONE) +} + func getEvacuateShardStatus(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) req := &control.GetShardEvacuationStatusRequest{ @@ -309,6 +331,7 @@ func initControlStartEvacuationShardCmd() { flags.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") flags.Bool(shardAllFlag, false, "Process all shards") flags.Bool(ignoreErrorsFlag, true, "Skip invalid/unreadable objects") + flags.String(scopeFlag, scopeAll, fmt.Sprintf("Evacuation scope; possible values: %s, %s, %s", scopeTrees, scopeObjects, scopeAll)) flags.Bool(awaitFlag, false, "Block execution until evacuation is completed") flags.Bool(noProgressFlag, false, fmt.Sprintf("Print progress if %s provided", awaitFlag)) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index d95098fbf..ad432e400 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -28,12 +29,40 @@ var ( evacuationOperationLogField = zap.String("operation", "evacuation") ) +// EvacuateScope is an evacuation scope. Keep in sync with pkg/services/control/service.proto. +type EvacuateScope uint32 + +var ( + EvacuateScopeObjects EvacuateScope = 1 + EvacuateScopeTrees EvacuateScope = 2 +) + +func (s EvacuateScope) String() string { + var sb strings.Builder + first := true + if s&EvacuateScopeObjects == EvacuateScopeObjects { + if !first { + sb.WriteString(";") + } + sb.WriteString("objects") + first = false + } + if s&EvacuateScopeTrees == EvacuateScopeTrees { + if !first { + sb.WriteString(";") + } + sb.WriteString("trees") + } + return sb.String() +} + // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { ShardID []*shard.ID Handler func(context.Context, oid.Address, *objectSDK.Object) error IgnoreErrors bool Async bool + Scope EvacuateScope } // EvacuateShardRes represents result of the EvacuateShard operation. @@ -135,6 +164,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev attribute.StringSlice("shardIDs", shardIDs), attribute.Bool("async", prm.Async), attribute.Bool("ignoreErrors", prm.IgnoreErrors), + attribute.Stringer("scope", prm.Scope), )) defer span.End() diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index 52ef083c4..6cba72d71 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -29,6 +29,7 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), IgnoreErrors: req.GetBody().GetIgnoreErrors(), Handler: s.replicate, + Scope: engine.EvacuateScopeObjects, } res, err := s.s.Evacuate(ctx, prm) diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index 112d4449f..0b2851275 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -17,11 +17,16 @@ func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartSha return nil, status.Error(codes.PermissionDenied, err.Error()) } + if req.GetBody().GetScope() == uint32(control.StartShardEvacuationRequest_Body_NONE) { + return nil, status.Error(codes.InvalidArgument, "no evacuation scope") + } + prm := engine.EvacuateShardPrm{ ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), IgnoreErrors: req.GetBody().GetIgnoreErrors(), Handler: s.replicate, Async: true, + Scope: engine.EvacuateScope(req.GetBody().GetScope()), } _, err = s.s.Evacuate(ctx, prm) diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index c9645941b..9c1a87415 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -20,6 +20,55 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type StartShardEvacuationRequest_Body_Scope int32 + +const ( + StartShardEvacuationRequest_Body_NONE StartShardEvacuationRequest_Body_Scope = 0 + StartShardEvacuationRequest_Body_OBJECTS StartShardEvacuationRequest_Body_Scope = 1 + StartShardEvacuationRequest_Body_TREES StartShardEvacuationRequest_Body_Scope = 2 +) + +// Enum value maps for StartShardEvacuationRequest_Body_Scope. +var ( + StartShardEvacuationRequest_Body_Scope_name = map[int32]string{ + 0: "NONE", + 1: "OBJECTS", + 2: "TREES", + } + StartShardEvacuationRequest_Body_Scope_value = map[string]int32{ + "NONE": 0, + "OBJECTS": 1, + "TREES": 2, + } +) + +func (x StartShardEvacuationRequest_Body_Scope) Enum() *StartShardEvacuationRequest_Body_Scope { + p := new(StartShardEvacuationRequest_Body_Scope) + *p = x + return p +} + +func (x StartShardEvacuationRequest_Body_Scope) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (StartShardEvacuationRequest_Body_Scope) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_services_control_service_proto_enumTypes[0].Descriptor() +} + +func (StartShardEvacuationRequest_Body_Scope) Type() protoreflect.EnumType { + return &file_pkg_services_control_service_proto_enumTypes[0] +} + +func (x StartShardEvacuationRequest_Body_Scope) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use StartShardEvacuationRequest_Body_Scope.Descriptor instead. +func (StartShardEvacuationRequest_Body_Scope) EnumDescriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18, 0, 0} +} + // Evacuate status enum. type GetShardEvacuationStatusResponse_Body_Status int32 @@ -54,11 +103,11 @@ func (x GetShardEvacuationStatusResponse_Body_Status) String() string { } func (GetShardEvacuationStatusResponse_Body_Status) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_services_control_service_proto_enumTypes[0].Descriptor() + return file_pkg_services_control_service_proto_enumTypes[1].Descriptor() } func (GetShardEvacuationStatusResponse_Body_Status) Type() protoreflect.EnumType { - return &file_pkg_services_control_service_proto_enumTypes[0] + return &file_pkg_services_control_service_proto_enumTypes[1] } func (x GetShardEvacuationStatusResponse_Body_Status) Number() protoreflect.EnumNumber { @@ -3113,6 +3162,8 @@ type StartShardEvacuationRequest_Body struct { Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` // Flag indicating whether object read errors should be ignored. IgnoreErrors bool `protobuf:"varint,2,opt,name=ignore_errors,json=ignoreErrors,proto3" json:"ignore_errors,omitempty"` + // Evacuation scope. + Scope uint32 `protobuf:"varint,3,opt,name=scope,proto3" json:"scope,omitempty"` } func (x *StartShardEvacuationRequest_Body) Reset() { @@ -3161,6 +3212,13 @@ func (x *StartShardEvacuationRequest_Body) GetIgnoreErrors() bool { return false } +func (x *StartShardEvacuationRequest_Body) GetScope() uint32 { + if x != nil { + return x.Scope + } + return 0 +} + // Response body structure. type StartShardEvacuationResponse_Body struct { state protoimpl.MessageState @@ -4502,7 +4560,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xd6, 0x01, 0x0a, 0x1b, 0x53, + 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x98, 0x02, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, @@ -4511,386 +4569,390 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, - 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, - 0x6f, 0x72, 0x73, 0x22, 0x98, 0x01, 0x0a, 0x1c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, - 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x9e, - 0x01, 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, - 0xe0, 0x05, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x87, 0x01, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, + 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, + 0x72, 0x6f, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x22, 0x29, 0x0a, 0x05, 0x53, 0x63, + 0x6f, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x0b, 0x0a, + 0x07, 0x4f, 0x42, 0x4a, 0x45, 0x43, 0x54, 0x53, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, + 0x45, 0x45, 0x53, 0x10, 0x02, 0x22, 0x98, 0x01, 0x0a, 0x1c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x22, 0x9e, 0x01, 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xc5, 0x04, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x65, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, - 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x4d, 0x0a, 0x06, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x35, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, 0x08, 0x64, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, + 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, + 0x79, 0x22, 0xe0, 0x05, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x44, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5b, - 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x12, 0x18, 0x0a, 0x07, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x07, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x1a, 0x25, 0x0a, 0x0d, 0x55, 0x6e, - 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x1a, 0x24, 0x0a, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, - 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, - 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x23, 0x0a, 0x1f, 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x48, - 0x41, 0x52, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, - 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, - 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, - 0x10, 0x02, 0x22, 0x94, 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, - 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, 0x53, 0x74, - 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x22, 0xdc, 0x01, 0x0a, 0x1c, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, - 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, - 0x6e, 0x22, 0xb5, 0x01, 0x0a, 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, - 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xe1, 0x01, 0x0a, 0x1c, 0x47, 0x65, - 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, - 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x22, 0xca, 0x01, 0x0a, 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x2c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xc5, 0x04, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x65, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, + 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x61, 0x69, + 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, + 0x64, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x4d, 0x0a, 0x06, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x35, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, 0x08, 0x64, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x44, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x5b, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x23, 0x0a, + 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x07, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x1a, 0x25, 0x0a, 0x0d, + 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x1a, 0x24, 0x0a, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x18, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x23, 0x0a, 0x1f, 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, + 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, + 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, + 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, + 0x45, 0x44, 0x10, 0x02, 0x22, 0x94, 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, + 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x22, 0xdc, 0x01, 0x0a, 0x1c, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, + 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb6, 0x01, - 0x0a, 0x1f, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, - 0x0a, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, - 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x20, 0x4c, 0x69, 0x73, 0x74, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, - 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x24, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x68, 0x61, - 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xd2, 0x01, 0x0a, 0x21, 0x4c, 0x69, 0x73, 0x74, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, + 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, + 0x61, 0x69, 0x6e, 0x22, 0xb5, 0x01, 0x0a, 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, + 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xe1, 0x01, 0x0a, 0x1c, + 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, + 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x1a, 0x36, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2e, 0x0a, 0x07, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x22, 0xe7, 0x01, 0x0a, - 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, - 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xba, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, - 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x20, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x64, 0x22, 0xca, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, - 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x22, 0xca, 0x01, 0x0a, 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, + 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, + 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, + 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, - 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, - 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, - 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, + 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x20, 0x4c, 0x69, 0x73, + 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x24, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x63, + 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xd2, 0x01, 0x0a, 0x21, 0x4c, 0x69, + 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x36, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2e, + 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x22, 0xe7, + 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, - 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, - 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, - 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, - 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, - 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, - 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, - 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, - 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, + 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xba, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb1, 0x0d, 0x0a, 0x0e, - 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, - 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, - 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, - 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, - 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, - 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, - 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, - 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x20, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x22, 0xca, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, + 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, + 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, + 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, + 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, + 0x72, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, + 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, + 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, + 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, + 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, + 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, + 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, + 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, + 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, + 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, + 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb1, 0x0d, + 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, + 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, + 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, + 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, + 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, + 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, + 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, + 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, + 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, + 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, + 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, + 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, - 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, - 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, - 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, - 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, - 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, 0x4c, - 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, - 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, - 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, - 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, - 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, - 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, - 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, - 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, + 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, + 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, + 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, + 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, + 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, + 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, + 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, + 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, + 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, + 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, + 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4905,225 +4967,226 @@ func file_pkg_services_control_service_proto_rawDescGZIP() []byte { return file_pkg_services_control_service_proto_rawDescData } -var file_pkg_services_control_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_pkg_services_control_service_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 79) var file_pkg_services_control_service_proto_goTypes = []interface{}{ - (GetShardEvacuationStatusResponse_Body_Status)(0), // 0: control.GetShardEvacuationStatusResponse.Body.Status - (*HealthCheckRequest)(nil), // 1: control.HealthCheckRequest - (*HealthCheckResponse)(nil), // 2: control.HealthCheckResponse - (*SetNetmapStatusRequest)(nil), // 3: control.SetNetmapStatusRequest - (*SetNetmapStatusResponse)(nil), // 4: control.SetNetmapStatusResponse - (*DropObjectsRequest)(nil), // 5: control.DropObjectsRequest - (*DropObjectsResponse)(nil), // 6: control.DropObjectsResponse - (*ListShardsRequest)(nil), // 7: control.ListShardsRequest - (*ListShardsResponse)(nil), // 8: control.ListShardsResponse - (*SetShardModeRequest)(nil), // 9: control.SetShardModeRequest - (*SetShardModeResponse)(nil), // 10: control.SetShardModeResponse - (*SynchronizeTreeRequest)(nil), // 11: control.SynchronizeTreeRequest - (*SynchronizeTreeResponse)(nil), // 12: control.SynchronizeTreeResponse - (*EvacuateShardRequest)(nil), // 13: control.EvacuateShardRequest - (*EvacuateShardResponse)(nil), // 14: control.EvacuateShardResponse - (*FlushCacheRequest)(nil), // 15: control.FlushCacheRequest - (*FlushCacheResponse)(nil), // 16: control.FlushCacheResponse - (*DoctorRequest)(nil), // 17: control.DoctorRequest - (*DoctorResponse)(nil), // 18: control.DoctorResponse - (*StartShardEvacuationRequest)(nil), // 19: control.StartShardEvacuationRequest - (*StartShardEvacuationResponse)(nil), // 20: control.StartShardEvacuationResponse - (*GetShardEvacuationStatusRequest)(nil), // 21: control.GetShardEvacuationStatusRequest - (*GetShardEvacuationStatusResponse)(nil), // 22: control.GetShardEvacuationStatusResponse - (*StopShardEvacuationRequest)(nil), // 23: control.StopShardEvacuationRequest - (*StopShardEvacuationResponse)(nil), // 24: control.StopShardEvacuationResponse - (*AddChainLocalOverrideRequest)(nil), // 25: control.AddChainLocalOverrideRequest - (*AddChainLocalOverrideResponse)(nil), // 26: control.AddChainLocalOverrideResponse - (*GetChainLocalOverrideRequest)(nil), // 27: control.GetChainLocalOverrideRequest - (*GetChainLocalOverrideResponse)(nil), // 28: control.GetChainLocalOverrideResponse - (*ListChainLocalOverridesRequest)(nil), // 29: control.ListChainLocalOverridesRequest - (*ListChainLocalOverridesResponse)(nil), // 30: control.ListChainLocalOverridesResponse - (*ListTargetsLocalOverridesRequest)(nil), // 31: control.ListTargetsLocalOverridesRequest - (*ListTargetsLocalOverridesResponse)(nil), // 32: control.ListTargetsLocalOverridesResponse - (*RemoveChainLocalOverrideRequest)(nil), // 33: control.RemoveChainLocalOverrideRequest - (*RemoveChainLocalOverrideResponse)(nil), // 34: control.RemoveChainLocalOverrideResponse - (*SealWriteCacheRequest)(nil), // 35: control.SealWriteCacheRequest - (*SealWriteCacheResponse)(nil), // 36: control.SealWriteCacheResponse - (*DetachShardsRequest)(nil), // 37: control.DetachShardsRequest - (*DetachShardsResponse)(nil), // 38: control.DetachShardsResponse - (*HealthCheckRequest_Body)(nil), // 39: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 40: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 41: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 42: control.SetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 43: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 44: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 45: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 46: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 47: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 48: control.SetShardModeResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 49: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 50: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 51: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 52: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 53: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 54: control.FlushCacheResponse.Body - (*DoctorRequest_Body)(nil), // 55: control.DoctorRequest.Body - (*DoctorResponse_Body)(nil), // 56: control.DoctorResponse.Body - (*StartShardEvacuationRequest_Body)(nil), // 57: control.StartShardEvacuationRequest.Body - (*StartShardEvacuationResponse_Body)(nil), // 58: control.StartShardEvacuationResponse.Body - (*GetShardEvacuationStatusRequest_Body)(nil), // 59: control.GetShardEvacuationStatusRequest.Body - (*GetShardEvacuationStatusResponse_Body)(nil), // 60: control.GetShardEvacuationStatusResponse.Body - (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 61: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 62: control.GetShardEvacuationStatusResponse.Body.Duration - (*StopShardEvacuationRequest_Body)(nil), // 63: control.StopShardEvacuationRequest.Body - (*StopShardEvacuationResponse_Body)(nil), // 64: control.StopShardEvacuationResponse.Body - (*AddChainLocalOverrideRequest_Body)(nil), // 65: control.AddChainLocalOverrideRequest.Body - (*AddChainLocalOverrideResponse_Body)(nil), // 66: control.AddChainLocalOverrideResponse.Body - (*GetChainLocalOverrideRequest_Body)(nil), // 67: control.GetChainLocalOverrideRequest.Body - (*GetChainLocalOverrideResponse_Body)(nil), // 68: control.GetChainLocalOverrideResponse.Body - (*ListChainLocalOverridesRequest_Body)(nil), // 69: control.ListChainLocalOverridesRequest.Body - (*ListChainLocalOverridesResponse_Body)(nil), // 70: control.ListChainLocalOverridesResponse.Body - (*ListTargetsLocalOverridesRequest_Body)(nil), // 71: control.ListTargetsLocalOverridesRequest.Body - (*ListTargetsLocalOverridesResponse_Body)(nil), // 72: control.ListTargetsLocalOverridesResponse.Body - (*RemoveChainLocalOverrideRequest_Body)(nil), // 73: control.RemoveChainLocalOverrideRequest.Body - (*RemoveChainLocalOverrideResponse_Body)(nil), // 74: control.RemoveChainLocalOverrideResponse.Body - (*SealWriteCacheRequest_Body)(nil), // 75: control.SealWriteCacheRequest.Body - (*SealWriteCacheResponse_Body)(nil), // 76: control.SealWriteCacheResponse.Body - (*SealWriteCacheResponse_Body_Status)(nil), // 77: control.SealWriteCacheResponse.Body.Status - (*DetachShardsRequest_Body)(nil), // 78: control.DetachShardsRequest.Body - (*DetachShardsResponse_Body)(nil), // 79: control.DetachShardsResponse.Body - (*Signature)(nil), // 80: control.Signature - (NetmapStatus)(0), // 81: control.NetmapStatus - (HealthStatus)(0), // 82: control.HealthStatus - (*ShardInfo)(nil), // 83: control.ShardInfo - (ShardMode)(0), // 84: control.ShardMode - (*ChainTarget)(nil), // 85: control.ChainTarget + (StartShardEvacuationRequest_Body_Scope)(0), // 0: control.StartShardEvacuationRequest.Body.Scope + (GetShardEvacuationStatusResponse_Body_Status)(0), // 1: control.GetShardEvacuationStatusResponse.Body.Status + (*HealthCheckRequest)(nil), // 2: control.HealthCheckRequest + (*HealthCheckResponse)(nil), // 3: control.HealthCheckResponse + (*SetNetmapStatusRequest)(nil), // 4: control.SetNetmapStatusRequest + (*SetNetmapStatusResponse)(nil), // 5: control.SetNetmapStatusResponse + (*DropObjectsRequest)(nil), // 6: control.DropObjectsRequest + (*DropObjectsResponse)(nil), // 7: control.DropObjectsResponse + (*ListShardsRequest)(nil), // 8: control.ListShardsRequest + (*ListShardsResponse)(nil), // 9: control.ListShardsResponse + (*SetShardModeRequest)(nil), // 10: control.SetShardModeRequest + (*SetShardModeResponse)(nil), // 11: control.SetShardModeResponse + (*SynchronizeTreeRequest)(nil), // 12: control.SynchronizeTreeRequest + (*SynchronizeTreeResponse)(nil), // 13: control.SynchronizeTreeResponse + (*EvacuateShardRequest)(nil), // 14: control.EvacuateShardRequest + (*EvacuateShardResponse)(nil), // 15: control.EvacuateShardResponse + (*FlushCacheRequest)(nil), // 16: control.FlushCacheRequest + (*FlushCacheResponse)(nil), // 17: control.FlushCacheResponse + (*DoctorRequest)(nil), // 18: control.DoctorRequest + (*DoctorResponse)(nil), // 19: control.DoctorResponse + (*StartShardEvacuationRequest)(nil), // 20: control.StartShardEvacuationRequest + (*StartShardEvacuationResponse)(nil), // 21: control.StartShardEvacuationResponse + (*GetShardEvacuationStatusRequest)(nil), // 22: control.GetShardEvacuationStatusRequest + (*GetShardEvacuationStatusResponse)(nil), // 23: control.GetShardEvacuationStatusResponse + (*StopShardEvacuationRequest)(nil), // 24: control.StopShardEvacuationRequest + (*StopShardEvacuationResponse)(nil), // 25: control.StopShardEvacuationResponse + (*AddChainLocalOverrideRequest)(nil), // 26: control.AddChainLocalOverrideRequest + (*AddChainLocalOverrideResponse)(nil), // 27: control.AddChainLocalOverrideResponse + (*GetChainLocalOverrideRequest)(nil), // 28: control.GetChainLocalOverrideRequest + (*GetChainLocalOverrideResponse)(nil), // 29: control.GetChainLocalOverrideResponse + (*ListChainLocalOverridesRequest)(nil), // 30: control.ListChainLocalOverridesRequest + (*ListChainLocalOverridesResponse)(nil), // 31: control.ListChainLocalOverridesResponse + (*ListTargetsLocalOverridesRequest)(nil), // 32: control.ListTargetsLocalOverridesRequest + (*ListTargetsLocalOverridesResponse)(nil), // 33: control.ListTargetsLocalOverridesResponse + (*RemoveChainLocalOverrideRequest)(nil), // 34: control.RemoveChainLocalOverrideRequest + (*RemoveChainLocalOverrideResponse)(nil), // 35: control.RemoveChainLocalOverrideResponse + (*SealWriteCacheRequest)(nil), // 36: control.SealWriteCacheRequest + (*SealWriteCacheResponse)(nil), // 37: control.SealWriteCacheResponse + (*DetachShardsRequest)(nil), // 38: control.DetachShardsRequest + (*DetachShardsResponse)(nil), // 39: control.DetachShardsResponse + (*HealthCheckRequest_Body)(nil), // 40: control.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 41: control.HealthCheckResponse.Body + (*SetNetmapStatusRequest_Body)(nil), // 42: control.SetNetmapStatusRequest.Body + (*SetNetmapStatusResponse_Body)(nil), // 43: control.SetNetmapStatusResponse.Body + (*DropObjectsRequest_Body)(nil), // 44: control.DropObjectsRequest.Body + (*DropObjectsResponse_Body)(nil), // 45: control.DropObjectsResponse.Body + (*ListShardsRequest_Body)(nil), // 46: control.ListShardsRequest.Body + (*ListShardsResponse_Body)(nil), // 47: control.ListShardsResponse.Body + (*SetShardModeRequest_Body)(nil), // 48: control.SetShardModeRequest.Body + (*SetShardModeResponse_Body)(nil), // 49: control.SetShardModeResponse.Body + (*SynchronizeTreeRequest_Body)(nil), // 50: control.SynchronizeTreeRequest.Body + (*SynchronizeTreeResponse_Body)(nil), // 51: control.SynchronizeTreeResponse.Body + (*EvacuateShardRequest_Body)(nil), // 52: control.EvacuateShardRequest.Body + (*EvacuateShardResponse_Body)(nil), // 53: control.EvacuateShardResponse.Body + (*FlushCacheRequest_Body)(nil), // 54: control.FlushCacheRequest.Body + (*FlushCacheResponse_Body)(nil), // 55: control.FlushCacheResponse.Body + (*DoctorRequest_Body)(nil), // 56: control.DoctorRequest.Body + (*DoctorResponse_Body)(nil), // 57: control.DoctorResponse.Body + (*StartShardEvacuationRequest_Body)(nil), // 58: control.StartShardEvacuationRequest.Body + (*StartShardEvacuationResponse_Body)(nil), // 59: control.StartShardEvacuationResponse.Body + (*GetShardEvacuationStatusRequest_Body)(nil), // 60: control.GetShardEvacuationStatusRequest.Body + (*GetShardEvacuationStatusResponse_Body)(nil), // 61: control.GetShardEvacuationStatusResponse.Body + (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 62: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 63: control.GetShardEvacuationStatusResponse.Body.Duration + (*StopShardEvacuationRequest_Body)(nil), // 64: control.StopShardEvacuationRequest.Body + (*StopShardEvacuationResponse_Body)(nil), // 65: control.StopShardEvacuationResponse.Body + (*AddChainLocalOverrideRequest_Body)(nil), // 66: control.AddChainLocalOverrideRequest.Body + (*AddChainLocalOverrideResponse_Body)(nil), // 67: control.AddChainLocalOverrideResponse.Body + (*GetChainLocalOverrideRequest_Body)(nil), // 68: control.GetChainLocalOverrideRequest.Body + (*GetChainLocalOverrideResponse_Body)(nil), // 69: control.GetChainLocalOverrideResponse.Body + (*ListChainLocalOverridesRequest_Body)(nil), // 70: control.ListChainLocalOverridesRequest.Body + (*ListChainLocalOverridesResponse_Body)(nil), // 71: control.ListChainLocalOverridesResponse.Body + (*ListTargetsLocalOverridesRequest_Body)(nil), // 72: control.ListTargetsLocalOverridesRequest.Body + (*ListTargetsLocalOverridesResponse_Body)(nil), // 73: control.ListTargetsLocalOverridesResponse.Body + (*RemoveChainLocalOverrideRequest_Body)(nil), // 74: control.RemoveChainLocalOverrideRequest.Body + (*RemoveChainLocalOverrideResponse_Body)(nil), // 75: control.RemoveChainLocalOverrideResponse.Body + (*SealWriteCacheRequest_Body)(nil), // 76: control.SealWriteCacheRequest.Body + (*SealWriteCacheResponse_Body)(nil), // 77: control.SealWriteCacheResponse.Body + (*SealWriteCacheResponse_Body_Status)(nil), // 78: control.SealWriteCacheResponse.Body.Status + (*DetachShardsRequest_Body)(nil), // 79: control.DetachShardsRequest.Body + (*DetachShardsResponse_Body)(nil), // 80: control.DetachShardsResponse.Body + (*Signature)(nil), // 81: control.Signature + (NetmapStatus)(0), // 82: control.NetmapStatus + (HealthStatus)(0), // 83: control.HealthStatus + (*ShardInfo)(nil), // 84: control.ShardInfo + (ShardMode)(0), // 85: control.ShardMode + (*ChainTarget)(nil), // 86: control.ChainTarget } var file_pkg_services_control_service_proto_depIdxs = []int32{ - 39, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 80, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 40, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 80, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 41, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 80, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 42, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 80, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 43, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 80, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature - 44, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 80, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature - 45, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 80, // 13: control.ListShardsRequest.signature:type_name -> control.Signature - 46, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 80, // 15: control.ListShardsResponse.signature:type_name -> control.Signature - 47, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 80, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature - 48, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 80, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature - 49, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 80, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 50, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 80, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 51, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 80, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature - 52, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 80, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature - 53, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 80, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature - 54, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 80, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature - 55, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body - 80, // 33: control.DoctorRequest.signature:type_name -> control.Signature - 56, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body - 80, // 35: control.DoctorResponse.signature:type_name -> control.Signature - 57, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body - 80, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature - 58, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body - 80, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature - 59, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body - 80, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 60, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body - 80, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 63, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body - 80, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature - 64, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body - 80, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature - 65, // 48: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body - 80, // 49: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature - 66, // 50: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body - 80, // 51: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature - 67, // 52: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body - 80, // 53: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature - 68, // 54: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body - 80, // 55: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature - 69, // 56: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body - 80, // 57: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature - 70, // 58: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body - 80, // 59: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature - 71, // 60: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body - 80, // 61: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature - 72, // 62: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body - 80, // 63: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature - 73, // 64: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body - 80, // 65: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature - 74, // 66: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body - 80, // 67: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature - 75, // 68: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body - 80, // 69: control.SealWriteCacheRequest.signature:type_name -> control.Signature - 76, // 70: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body - 80, // 71: control.SealWriteCacheResponse.signature:type_name -> control.Signature - 78, // 72: control.DetachShardsRequest.body:type_name -> control.DetachShardsRequest.Body - 80, // 73: control.DetachShardsRequest.signature:type_name -> control.Signature - 79, // 74: control.DetachShardsResponse.body:type_name -> control.DetachShardsResponse.Body - 80, // 75: control.DetachShardsResponse.signature:type_name -> control.Signature - 81, // 76: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 82, // 77: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 81, // 78: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 83, // 79: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 84, // 80: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 0, // 81: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status - 62, // 82: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration - 61, // 83: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - 85, // 84: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 85, // 85: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 85, // 86: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget - 85, // 87: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget - 85, // 88: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 77, // 89: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status - 1, // 90: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 3, // 91: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 5, // 92: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 7, // 93: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 9, // 94: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 11, // 95: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 13, // 96: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 19, // 97: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest - 21, // 98: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest - 23, // 99: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest - 15, // 100: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 17, // 101: control.ControlService.Doctor:input_type -> control.DoctorRequest - 25, // 102: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest - 27, // 103: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest - 29, // 104: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest - 33, // 105: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest - 31, // 106: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest - 35, // 107: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest - 37, // 108: control.ControlService.DetachShards:input_type -> control.DetachShardsRequest - 2, // 109: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 4, // 110: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 6, // 111: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 8, // 112: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 10, // 113: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 12, // 114: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 14, // 115: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 20, // 116: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse - 22, // 117: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse - 24, // 118: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse - 16, // 119: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 18, // 120: control.ControlService.Doctor:output_type -> control.DoctorResponse - 26, // 121: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse - 28, // 122: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse - 30, // 123: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse - 34, // 124: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse - 32, // 125: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse - 36, // 126: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse - 38, // 127: control.ControlService.DetachShards:output_type -> control.DetachShardsResponse + 40, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body + 81, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature + 41, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body + 81, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature + 42, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body + 81, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature + 43, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body + 81, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature + 44, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body + 81, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature + 45, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body + 81, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature + 46, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body + 81, // 13: control.ListShardsRequest.signature:type_name -> control.Signature + 47, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body + 81, // 15: control.ListShardsResponse.signature:type_name -> control.Signature + 48, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body + 81, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature + 49, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body + 81, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature + 50, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body + 81, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature + 51, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body + 81, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature + 52, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body + 81, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature + 53, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body + 81, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature + 54, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body + 81, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature + 55, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body + 81, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature + 56, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body + 81, // 33: control.DoctorRequest.signature:type_name -> control.Signature + 57, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body + 81, // 35: control.DoctorResponse.signature:type_name -> control.Signature + 58, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body + 81, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature + 59, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body + 81, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature + 60, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body + 81, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 61, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body + 81, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 64, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body + 81, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature + 65, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body + 81, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature + 66, // 48: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body + 81, // 49: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature + 67, // 50: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body + 81, // 51: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature + 68, // 52: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body + 81, // 53: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature + 69, // 54: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body + 81, // 55: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature + 70, // 56: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body + 81, // 57: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature + 71, // 58: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body + 81, // 59: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature + 72, // 60: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body + 81, // 61: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature + 73, // 62: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body + 81, // 63: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature + 74, // 64: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body + 81, // 65: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature + 75, // 66: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body + 81, // 67: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature + 76, // 68: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body + 81, // 69: control.SealWriteCacheRequest.signature:type_name -> control.Signature + 77, // 70: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body + 81, // 71: control.SealWriteCacheResponse.signature:type_name -> control.Signature + 79, // 72: control.DetachShardsRequest.body:type_name -> control.DetachShardsRequest.Body + 81, // 73: control.DetachShardsRequest.signature:type_name -> control.Signature + 80, // 74: control.DetachShardsResponse.body:type_name -> control.DetachShardsResponse.Body + 81, // 75: control.DetachShardsResponse.signature:type_name -> control.Signature + 82, // 76: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus + 83, // 77: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus + 82, // 78: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus + 84, // 79: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo + 85, // 80: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode + 1, // 81: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status + 63, // 82: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration + 62, // 83: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + 86, // 84: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 86, // 85: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 86, // 86: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget + 86, // 87: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget + 86, // 88: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 78, // 89: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status + 2, // 90: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 4, // 91: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 6, // 92: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 8, // 93: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 10, // 94: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 12, // 95: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 14, // 96: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 20, // 97: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest + 22, // 98: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest + 24, // 99: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest + 16, // 100: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 18, // 101: control.ControlService.Doctor:input_type -> control.DoctorRequest + 26, // 102: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest + 28, // 103: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest + 30, // 104: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest + 34, // 105: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest + 32, // 106: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest + 36, // 107: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest + 38, // 108: control.ControlService.DetachShards:input_type -> control.DetachShardsRequest + 3, // 109: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 5, // 110: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 7, // 111: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 9, // 112: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 11, // 113: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 13, // 114: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 15, // 115: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 21, // 116: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse + 23, // 117: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse + 25, // 118: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse + 17, // 119: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 19, // 120: control.ControlService.Doctor:output_type -> control.DoctorResponse + 27, // 121: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse + 29, // 122: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse + 31, // 123: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse + 35, // 124: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse + 33, // 125: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse + 37, // 126: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse + 39, // 127: control.ControlService.DetachShards:output_type -> control.DetachShardsResponse 109, // [109:128] is the sub-list for method output_type 90, // [90:109] is the sub-list for method input_type 90, // [90:90] is the sub-list for extension type_name @@ -6092,7 +6155,7 @@ func file_pkg_services_control_service_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_service_proto_rawDesc, - NumEnums: 1, + NumEnums: 2, NumMessages: 79, NumExtensions: 0, NumServices: 1, diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 4cb1dee33..73e8ca90f 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -336,10 +336,18 @@ message DoctorResponse { message StartShardEvacuationRequest { // Request body structure. message Body { + enum Scope { + NONE = 0; + OBJECTS = 1; + TREES = 2; + } + // IDs of the shards. repeated bytes shard_ID = 1; // Flag indicating whether object read errors should be ignored. bool ignore_errors = 2; + // Evacuation scope. + uint32 scope = 3; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 125a12d00..b18d30dba 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -1511,6 +1511,7 @@ func (x *StartShardEvacuationRequest_Body) StableSize() (size int) { } size += proto.RepeatedBytesSize(1, x.Shard_ID) size += proto.BoolSize(2, x.IgnoreErrors) + size += proto.UInt32Size(3, x.Scope) return size } @@ -1532,6 +1533,7 @@ func (x *StartShardEvacuationRequest_Body) StableMarshal(buf []byte) []byte { var offset int offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) offset += proto.BoolMarshal(2, buf[offset:], x.IgnoreErrors) + offset += proto.UInt32Marshal(3, buf[offset:], x.Scope) return buf } From 15d853ea22115c5345cbd0a616d2879b06f883e0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Feb 2024 17:48:43 +0300 Subject: [PATCH 0862/1943] [#947] controlSvc: Return tree evacuation stat Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 142 ++-- .../engine/evacuate_limiter.go | 37 +- .../engine/evacuate_test.go | 48 +- pkg/services/control/server/convert.go | 21 +- pkg/services/control/server/evacuate.go | 10 +- pkg/services/control/server/evacuate_async.go | 10 +- pkg/services/control/service.pb.go | 716 +++++++++--------- pkg/services/control/service.proto | 15 +- pkg/services/control/service_frostfs.pb.go | 22 +- 9 files changed, 569 insertions(+), 452 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index ad432e400..3853c4e3a 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -58,62 +58,93 @@ func (s EvacuateScope) String() string { // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { - ShardID []*shard.ID - Handler func(context.Context, oid.Address, *objectSDK.Object) error - IgnoreErrors bool - Async bool - Scope EvacuateScope + ShardID []*shard.ID + ObjectsHandler func(context.Context, oid.Address, *objectSDK.Object) error + IgnoreErrors bool + Async bool + Scope EvacuateScope } // EvacuateShardRes represents result of the EvacuateShard operation. type EvacuateShardRes struct { - evacuated *atomic.Uint64 - total *atomic.Uint64 - failed *atomic.Uint64 - skipped *atomic.Uint64 + objEvacuated *atomic.Uint64 + objTotal *atomic.Uint64 + objFailed *atomic.Uint64 + objSkipped *atomic.Uint64 + + trEvacuated *atomic.Uint64 + trTotal *atomic.Uint64 + trFailed *atomic.Uint64 } // NewEvacuateShardRes creates new EvacuateShardRes instance. func NewEvacuateShardRes() *EvacuateShardRes { return &EvacuateShardRes{ - evacuated: new(atomic.Uint64), - total: new(atomic.Uint64), - failed: new(atomic.Uint64), - skipped: new(atomic.Uint64), + objEvacuated: new(atomic.Uint64), + objTotal: new(atomic.Uint64), + objFailed: new(atomic.Uint64), + objSkipped: new(atomic.Uint64), + trEvacuated: new(atomic.Uint64), + trTotal: new(atomic.Uint64), + trFailed: new(atomic.Uint64), } } -// Evacuated returns amount of evacuated objects. +// ObjectsEvacuated returns amount of evacuated objects. // Objects for which handler returned no error are also assumed evacuated. -func (p *EvacuateShardRes) Evacuated() uint64 { +func (p *EvacuateShardRes) ObjectsEvacuated() uint64 { if p == nil { return 0 } - return p.evacuated.Load() + return p.objEvacuated.Load() } -// Total returns total count objects to evacuate. -func (p *EvacuateShardRes) Total() uint64 { +// ObjectsTotal returns total count objects to evacuate. +func (p *EvacuateShardRes) ObjectsTotal() uint64 { if p == nil { return 0 } - return p.total.Load() + return p.objTotal.Load() } -// Failed returns count of failed objects to evacuate. -func (p *EvacuateShardRes) Failed() uint64 { +// ObjectsFailed returns count of failed objects to evacuate. +func (p *EvacuateShardRes) ObjectsFailed() uint64 { if p == nil { return 0 } - return p.failed.Load() + return p.objFailed.Load() } -// Skipped returns count of skipped objects. -func (p *EvacuateShardRes) Skipped() uint64 { +// ObjectsSkipped returns count of skipped objects. +func (p *EvacuateShardRes) ObjectsSkipped() uint64 { if p == nil { return 0 } - return p.skipped.Load() + return p.objSkipped.Load() +} + +// TreesEvacuated returns amount of evacuated trees. +func (p *EvacuateShardRes) TreesEvacuated() uint64 { + if p == nil { + return 0 + } + return p.trEvacuated.Load() +} + +// TreesTotal returns total count trees to evacuate. +func (p *EvacuateShardRes) TreesTotal() uint64 { + if p == nil { + return 0 + } + return p.trTotal.Load() +} + +// TreesFailed returns count of failed trees to evacuate. +func (p *EvacuateShardRes) TreesFailed() uint64 { + if p == nil { + return 0 + } + return p.trFailed.Load() } // DeepCopy returns deep copy of result instance. @@ -123,16 +154,22 @@ func (p *EvacuateShardRes) DeepCopy() *EvacuateShardRes { } res := &EvacuateShardRes{ - evacuated: new(atomic.Uint64), - total: new(atomic.Uint64), - failed: new(atomic.Uint64), - skipped: new(atomic.Uint64), + objEvacuated: new(atomic.Uint64), + objTotal: new(atomic.Uint64), + objFailed: new(atomic.Uint64), + objSkipped: new(atomic.Uint64), + trEvacuated: new(atomic.Uint64), + trTotal: new(atomic.Uint64), + trFailed: new(atomic.Uint64), } - res.evacuated.Store(p.evacuated.Load()) - res.total.Store(p.total.Load()) - res.failed.Store(p.failed.Load()) - res.skipped.Store(p.skipped.Load()) + res.objEvacuated.Store(p.objEvacuated.Load()) + res.objTotal.Store(p.objTotal.Load()) + res.objFailed.Store(p.objFailed.Load()) + res.objSkipped.Store(p.objSkipped.Load()) + res.trTotal.Store(p.trTotal.Load()) + res.trEvacuated.Store(p.trEvacuated.Load()) + res.trFailed.Store(p.trFailed.Load()) return res } @@ -168,7 +205,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev )) defer span.End() - shards, weights, err := e.getActualShards(shardIDs, prm.Handler != nil) + shards, weights, err := e.getActualShards(shardIDs, prm) if err != nil { return nil, err } @@ -216,6 +253,7 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p attribute.StringSlice("shardIDs", shardIDs), attribute.Bool("async", prm.Async), attribute.Bool("ignoreErrors", prm.IgnoreErrors), + attribute.Stringer("scope", prm.Scope), )) defer func() { @@ -224,18 +262,19 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p }() e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) err = e.getTotalObjectsCount(ctx, shardsToEvacuate, res) if err != nil { e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) return err } for _, shardID := range shardIDs { if err = e.evacuateShard(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { - e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField) + e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) return err } } @@ -243,10 +282,13 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, - zap.Uint64("total", res.Total()), - zap.Uint64("evacuated", res.Evacuated()), - zap.Uint64("failed", res.Failed()), - zap.Uint64("skipped", res.Skipped()), + zap.Uint64("total_objects", res.ObjectsTotal()), + zap.Uint64("evacuated_objects", res.ObjectsEvacuated()), + zap.Uint64("failed_objects", res.ObjectsFailed()), + zap.Uint64("skipped_objects", res.ObjectsSkipped()), + zap.Uint64("total_trees", res.TreesTotal()), + zap.Uint64("evacuated_trees", res.TreesEvacuated()), + zap.Uint64("failed_trees", res.TreesFailed()), ) return nil } @@ -263,7 +305,7 @@ func (e *StorageEngine) getTotalObjectsCount(ctx context.Context, shardsToEvacua } return err } - res.total.Add(cnt) + res.objTotal.Add(cnt) } return nil } @@ -307,7 +349,7 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E return nil } -func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) ([]pooledShard, []float64, error) { +func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) ([]pooledShard, []float64, error) { e.mtx.RLock() defer e.mtx.RUnlock() @@ -322,7 +364,7 @@ func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) } } - if len(e.shards)-len(shardIDs) < 1 && !handlerDefined { + if len(e.shards)-len(shardIDs) < 1 && prm.ObjectsHandler == nil { return nil, nil, errMustHaveTwoShards } @@ -368,7 +410,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to getRes, err := sh.Get(ctx, getPrm) if err != nil { if prm.IgnoreErrors { - res.failed.Add(1) + res.objFailed.Add(1) continue } e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, @@ -385,19 +427,19 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to continue } - if prm.Handler == nil { + if prm.ObjectsHandler == nil { // Do not check ignoreErrors flag here because // ignoring errors on put make this command kinda useless. return fmt.Errorf("%w: %s", errPutShard, toEvacuate[i]) } - err = prm.Handler(ctx, addr, getRes.Object()) + err = prm.ObjectsHandler(ctx, addr, getRes.Object()) if err != nil { e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } - res.evacuated.Add(1) + res.objEvacuated.Add(1) } return nil } @@ -418,7 +460,7 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add } switch e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object).status { case putToShardSuccess: - res.evacuated.Add(1) + res.objEvacuated.Add(1) e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), @@ -427,7 +469,7 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return true, nil case putToShardExists, putToShardRemoved: - res.skipped.Add(1) + res.objSkipped.Add(1) return true, nil default: continue diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go index 83acd30d9..63960e238 100644 --- a/pkg/local_object_storage/engine/evacuate_limiter.go +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -34,32 +34,53 @@ func (s *EvacuationState) ShardIDs() []string { return s.shardIDs } -func (s *EvacuationState) Evacuated() uint64 { +func (s *EvacuationState) ObjectsEvacuated() uint64 { if s == nil { return 0 } - return s.result.Evacuated() + return s.result.ObjectsEvacuated() } -func (s *EvacuationState) Total() uint64 { +func (s *EvacuationState) ObjectsTotal() uint64 { if s == nil { return 0 } - return s.result.Total() + return s.result.ObjectsTotal() } -func (s *EvacuationState) Failed() uint64 { +func (s *EvacuationState) ObjectsFailed() uint64 { if s == nil { return 0 } - return s.result.Failed() + return s.result.ObjectsFailed() } -func (s *EvacuationState) Skipped() uint64 { +func (s *EvacuationState) ObjectsSkipped() uint64 { if s == nil { return 0 } - return s.result.Skipped() + return s.result.ObjectsSkipped() +} + +func (s *EvacuationState) TreesEvacuated() uint64 { + if s == nil { + return 0 + } + return s.result.TreesEvacuated() +} + +func (s *EvacuationState) TreesTotal() uint64 { + if s == nil { + return 0 + } + return s.result.TreesTotal() +} + +func (s *EvacuationState) TreesFailed() uint64 { + if s == nil { + return 0 + } + return s.result.TreesFailed() } func (s *EvacuationState) ProcessingStatus() EvacuateProcessState { diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 2dc4a177c..cadcc2933 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -107,14 +107,14 @@ func TestEvacuateShard(t *testing.T) { t.Run("must be read-only", func(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, ErrMustBeReadOnly) - require.Equal(t, uint64(0), res.Evacuated()) + require.Equal(t, uint64(0), res.ObjectsEvacuated()) }) require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly)) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, uint64(objPerShard), res.Evacuated()) + require.Equal(t, uint64(objPerShard), res.ObjectsEvacuated()) // We check that all objects are available both before and after shard removal. // First case is a real-world use-case. It ensures that an object can be put in presense @@ -125,7 +125,7 @@ func TestEvacuateShard(t *testing.T) { // Calling it again is OK, but all objects are already moved, so no new PUTs should be done. res, err = e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, uint64(0), res.Evacuated()) + require.Equal(t, uint64(0), res.ObjectsEvacuated()) checkHasObjects(t) @@ -177,13 +177,13 @@ func TestEvacuateNetwork(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errMustHaveTwoShards) - require.Equal(t, uint64(0), res.Evacuated()) + require.Equal(t, uint64(0), res.ObjectsEvacuated()) - prm.Handler = acceptOneOf(objects, 2) + prm.ObjectsHandler = acceptOneOf(objects, 2) res, err = e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, uint64(2), res.Evacuated()) + require.Equal(t, uint64(2), res.ObjectsEvacuated()) }) t.Run("multiple shards, evacuate one", func(t *testing.T) { t.Parallel() @@ -197,18 +197,18 @@ func TestEvacuateNetwork(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] - prm.Handler = acceptOneOf(objects, 2) + prm.ObjectsHandler = acceptOneOf(objects, 2) res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, uint64(2), res.Evacuated()) + require.Equal(t, uint64(2), res.ObjectsEvacuated()) t.Run("no errors", func(t *testing.T) { - prm.Handler = acceptOneOf(objects, 3) + prm.ObjectsHandler = acceptOneOf(objects, 3) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, uint64(3), res.Evacuated()) + require.Equal(t, uint64(3), res.ObjectsEvacuated()) }) }) t.Run("multiple shards, evacuate many", func(t *testing.T) { @@ -234,18 +234,18 @@ func TestEvacuateNetwork(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = evacuateIDs - prm.Handler = acceptOneOf(objects, totalCount-1) + prm.ObjectsHandler = acceptOneOf(objects, totalCount-1) res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, totalCount-1, res.Evacuated()) + require.Equal(t, totalCount-1, res.ObjectsEvacuated()) t.Run("no errors", func(t *testing.T) { - prm.Handler = acceptOneOf(objects, totalCount) + prm.ObjectsHandler = acceptOneOf(objects, totalCount) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, totalCount, res.Evacuated()) + require.Equal(t, totalCount, res.ObjectsEvacuated()) }) }) } @@ -262,7 +262,7 @@ func TestEvacuateCancellation(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] - prm.Handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { select { case <-ctx.Done(): return ctx.Err() @@ -276,7 +276,7 @@ func TestEvacuateCancellation(t *testing.T) { res, err := e.Evacuate(ctx, prm) require.ErrorContains(t, err, "context canceled") - require.Equal(t, uint64(0), res.Evacuated()) + require.Equal(t, uint64(0), res.ObjectsEvacuated()) } func TestEvacuateSingleProcess(t *testing.T) { @@ -293,7 +293,7 @@ func TestEvacuateSingleProcess(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] - prm.Handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { select { case <-running: default: @@ -307,14 +307,14 @@ func TestEvacuateSingleProcess(t *testing.T) { eg.Go(func() error { res, err := e.Evacuate(egCtx, prm) require.NoError(t, err, "first evacuation failed") - require.Equal(t, uint64(3), res.Evacuated()) + require.Equal(t, uint64(3), res.ObjectsEvacuated()) return nil }) eg.Go(func() error { <-running res, err := e.Evacuate(egCtx, prm) require.ErrorContains(t, err, "evacuate is already running for shard ids", "second evacuation not failed") - require.Equal(t, uint64(0), res.Evacuated()) + require.Equal(t, uint64(0), res.ObjectsEvacuated()) close(blocker) return nil }) @@ -335,7 +335,7 @@ func TestEvacuateAsync(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] - prm.Handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { select { case <-running: default: @@ -348,7 +348,7 @@ func TestEvacuateAsync(t *testing.T) { st, err := e.GetEvacuationState(context.Background()) require.NoError(t, err, "get init state failed") require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state") - require.Equal(t, uint64(0), st.Evacuated(), "invalid init count") + require.Equal(t, uint64(0), st.ObjectsEvacuated(), "invalid init count") require.Nil(t, st.StartedAt(), "invalid init started at") require.Nil(t, st.FinishedAt(), "invalid init finished at") require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids") @@ -358,7 +358,7 @@ func TestEvacuateAsync(t *testing.T) { eg.Go(func() error { res, err := e.Evacuate(egCtx, prm) require.NoError(t, err, "first evacuation failed") - require.Equal(t, uint64(3), res.Evacuated()) + require.Equal(t, uint64(3), res.ObjectsEvacuated()) return nil }) @@ -367,7 +367,7 @@ func TestEvacuateAsync(t *testing.T) { st, err = e.GetEvacuationState(context.Background()) require.NoError(t, err, "get running state failed") require.Equal(t, EvacuateProcessStateRunning, st.ProcessingStatus(), "invalid running state") - require.Equal(t, uint64(0), st.Evacuated(), "invalid running count") + require.Equal(t, uint64(0), st.ObjectsEvacuated(), "invalid running count") require.NotNil(t, st.StartedAt(), "invalid running started at") require.Nil(t, st.FinishedAt(), "invalid init finished at") expectedShardIDs := make([]string, 0, 2) @@ -385,7 +385,7 @@ func TestEvacuateAsync(t *testing.T) { }, 3*time.Second, 10*time.Millisecond, "invalid final state") require.NoError(t, err, "get final state failed") - require.Equal(t, uint64(3), st.Evacuated(), "invalid final count") + require.Equal(t, uint64(3), st.ObjectsEvacuated(), "invalid final count") require.NotNil(t, st.StartedAt(), "invalid final started at") require.NotNil(t, st.FinishedAt(), "invalid final finished at") require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids") diff --git a/pkg/services/control/server/convert.go b/pkg/services/control/server/convert.go index f922d7278..4f3132356 100644 --- a/pkg/services/control/server/convert.go +++ b/pkg/services/control/server/convert.go @@ -47,15 +47,18 @@ func stateToResponse(state *engine.EvacuationState) (*control.GetShardEvacuation } return &control.GetShardEvacuationStatusResponse{ Body: &control.GetShardEvacuationStatusResponse_Body{ - Shard_ID: shardIDs, - Evacuated: state.Evacuated(), - Total: state.Total(), - Failed: state.Failed(), - Status: evacStatus, - StartedAt: startedAt, - Duration: duration, - ErrorMessage: state.ErrorMessage(), - Skipped: state.Skipped(), + Shard_ID: shardIDs, + EvacuatedObjects: state.ObjectsEvacuated(), + TotalObjects: state.ObjectsTotal(), + FailedObjects: state.ObjectsFailed(), + Status: evacStatus, + StartedAt: startedAt, + Duration: duration, + ErrorMessage: state.ErrorMessage(), + SkippedObjects: state.ObjectsSkipped(), + TotalTrees: state.TreesTotal(), + EvacuatedTrees: state.TreesEvacuated(), + FailedTrees: state.TreesFailed(), }, }, nil } diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index 6cba72d71..ac8b3d548 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -26,10 +26,10 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe } prm := engine.EvacuateShardPrm{ - ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), - IgnoreErrors: req.GetBody().GetIgnoreErrors(), - Handler: s.replicate, - Scope: engine.EvacuateScopeObjects, + ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), + IgnoreErrors: req.GetBody().GetIgnoreErrors(), + ObjectsHandler: s.replicate, + Scope: engine.EvacuateScopeObjects, } res, err := s.s.Evacuate(ctx, prm) @@ -39,7 +39,7 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe resp := &control.EvacuateShardResponse{ Body: &control.EvacuateShardResponse_Body{ - Count: uint32(res.Evacuated()), + Count: uint32(res.ObjectsEvacuated()), }, } diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index 0b2851275..91f0731c4 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -22,11 +22,11 @@ func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartSha } prm := engine.EvacuateShardPrm{ - ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), - IgnoreErrors: req.GetBody().GetIgnoreErrors(), - Handler: s.replicate, - Async: true, - Scope: engine.EvacuateScope(req.GetBody().GetScope()), + ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), + IgnoreErrors: req.GetBody().GetIgnoreErrors(), + ObjectsHandler: s.replicate, + Async: true, + Scope: engine.EvacuateScope(req.GetBody().GetScope()), } _, err = s.s.Evacuate(ctx, prm) diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 9c1a87415..4e95ac617 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -3304,11 +3304,11 @@ type GetShardEvacuationStatusResponse_Body struct { unknownFields protoimpl.UnknownFields // Total objects to evacuate count. The value is approximate, so evacuated + failed + skipped == total is not guaranteed after completion. - Total uint64 `protobuf:"varint,1,opt,name=total,proto3" json:"total,omitempty"` + TotalObjects uint64 `protobuf:"varint,1,opt,name=total_objects,json=totalObjects,proto3" json:"total_objects,omitempty"` // Evacuated objects count. - Evacuated uint64 `protobuf:"varint,2,opt,name=evacuated,proto3" json:"evacuated,omitempty"` + EvacuatedObjects uint64 `protobuf:"varint,2,opt,name=evacuated_objects,json=evacuatedObjects,proto3" json:"evacuated_objects,omitempty"` // Failed objects count. - Failed uint64 `protobuf:"varint,3,opt,name=failed,proto3" json:"failed,omitempty"` + FailedObjects uint64 `protobuf:"varint,3,opt,name=failed_objects,json=failedObjects,proto3" json:"failed_objects,omitempty"` // Shard IDs. Shard_ID [][]byte `protobuf:"bytes,4,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` // Evacuation process status. @@ -3320,7 +3320,13 @@ type GetShardEvacuationStatusResponse_Body struct { // Error message if evacuation failed. ErrorMessage string `protobuf:"bytes,8,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` // Skipped objects count. - Skipped uint64 `protobuf:"varint,9,opt,name=skipped,proto3" json:"skipped,omitempty"` + SkippedObjects uint64 `protobuf:"varint,9,opt,name=skipped_objects,json=skippedObjects,proto3" json:"skipped_objects,omitempty"` + // Total trees to evacuate count. + TotalTrees uint64 `protobuf:"varint,10,opt,name=total_trees,json=totalTrees,proto3" json:"total_trees,omitempty"` + // Evacuated trees count. + EvacuatedTrees uint64 `protobuf:"varint,11,opt,name=evacuated_trees,json=evacuatedTrees,proto3" json:"evacuated_trees,omitempty"` + // Failed trees count. + FailedTrees uint64 `protobuf:"varint,12,opt,name=failed_trees,json=failedTrees,proto3" json:"failed_trees,omitempty"` } func (x *GetShardEvacuationStatusResponse_Body) Reset() { @@ -3355,23 +3361,23 @@ func (*GetShardEvacuationStatusResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0} } -func (x *GetShardEvacuationStatusResponse_Body) GetTotal() uint64 { +func (x *GetShardEvacuationStatusResponse_Body) GetTotalObjects() uint64 { if x != nil { - return x.Total + return x.TotalObjects } return 0 } -func (x *GetShardEvacuationStatusResponse_Body) GetEvacuated() uint64 { +func (x *GetShardEvacuationStatusResponse_Body) GetEvacuatedObjects() uint64 { if x != nil { - return x.Evacuated + return x.EvacuatedObjects } return 0 } -func (x *GetShardEvacuationStatusResponse_Body) GetFailed() uint64 { +func (x *GetShardEvacuationStatusResponse_Body) GetFailedObjects() uint64 { if x != nil { - return x.Failed + return x.FailedObjects } return 0 } @@ -3411,9 +3417,30 @@ func (x *GetShardEvacuationStatusResponse_Body) GetErrorMessage() string { return "" } -func (x *GetShardEvacuationStatusResponse_Body) GetSkipped() uint64 { +func (x *GetShardEvacuationStatusResponse_Body) GetSkippedObjects() uint64 { if x != nil { - return x.Skipped + return x.SkippedObjects + } + return 0 +} + +func (x *GetShardEvacuationStatusResponse_Body) GetTotalTrees() uint64 { + if x != nil { + return x.TotalTrees + } + return 0 +} + +func (x *GetShardEvacuationStatusResponse_Body) GetEvacuatedTrees() uint64 { + if x != nil { + return x.EvacuatedTrees + } + return 0 +} + +func (x *GetShardEvacuationStatusResponse_Body) GetFailedTrees() uint64 { + if x != nil { + return x.FailedTrees } return 0 } @@ -4598,7 +4625,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, - 0x79, 0x22, 0xe0, 0x05, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x79, 0x22, 0x89, 0x07, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, @@ -4607,168 +4634,103 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xc5, 0x04, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x65, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, - 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x61, 0x69, - 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, - 0x64, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x4d, 0x0a, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x35, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, 0x08, 0x64, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x44, 0x75, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x5b, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x23, 0x0a, - 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x18, 0x09, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x07, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x1a, 0x25, 0x0a, 0x0d, - 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x1a, 0x24, 0x0a, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x18, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x23, 0x0a, 0x1f, 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, - 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, - 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, - 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, - 0x45, 0x44, 0x10, 0x02, 0x22, 0x94, 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, - 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xee, 0x05, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6f, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x74, 0x6f, + 0x74, 0x61, 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x65, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, + 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x65, + 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0d, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x19, + 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, + 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x4d, 0x0a, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x35, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5b, 0x0a, + 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x3c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, + 0x79, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, + 0x27, 0x0a, 0x0f, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, + 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, + 0x6c, 0x5f, 0x74, 0x72, 0x65, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x74, + 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x72, 0x65, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x72, 0x65, 0x65, 0x73, 0x18, 0x0b, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0e, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x54, 0x72, 0x65, + 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x74, 0x72, 0x65, + 0x65, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, + 0x54, 0x72, 0x65, 0x65, 0x73, 0x1a, 0x25, 0x0a, 0x0d, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x24, 0x0a, 0x08, + 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x6f, + 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, + 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x23, 0x0a, 0x1f, + 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, + 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0x94, 0x01, + 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x22, 0xdc, 0x01, 0x0a, 0x1c, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, + 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, + 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xdc, 0x01, + 0x0a, 0x1c, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, + 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x1a, 0x4a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xb5, 0x01, 0x0a, + 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x49, 0x64, 0x22, 0xe1, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, - 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, - 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x22, 0xb5, 0x01, 0x0a, 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, - 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xe1, 0x01, 0x0a, 0x1c, - 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, - 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, - 0x69, 0x6e, 0x22, 0xca, 0x01, 0x0a, 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, - 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, - 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, - 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, - 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x20, 0x4c, 0x69, 0x73, - 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x24, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x63, - 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xd2, 0x01, 0x0a, 0x21, 0x4c, 0x69, - 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x36, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2e, - 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x22, 0xe7, - 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, + 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, @@ -4778,181 +4740,257 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xba, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x20, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x22, 0xca, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, - 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, - 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, - 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, - 0x72, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, - 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, - 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, - 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, - 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, - 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, + 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xca, 0x01, 0x0a, 0x1e, + 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, 0x73, + 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x20, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x24, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, + 0x6d, 0x65, 0x22, 0xd2, 0x01, 0x0a, 0x21, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, - 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, - 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb1, 0x0d, - 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, - 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, - 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, - 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, - 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, - 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, - 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, - 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, - 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, - 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, - 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, - 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, - 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, - 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, - 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, - 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, + 0x36, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2e, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, + 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, + 0x64, 0x22, 0xba, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, - 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, - 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, - 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, - 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, - 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, - 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x20, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x22, 0xca, + 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, + 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, + 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, + 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, + 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, + 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, + 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, + 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, + 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, - 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, - 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb1, 0x0d, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, + 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, + 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, + 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, + 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, + 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, + 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, + 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, + 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, + 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, + 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, + 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, + 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, + 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 73e8ca90f..16e0f707d 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -394,11 +394,11 @@ message GetShardEvacuationStatusResponse { } // Total objects to evacuate count. The value is approximate, so evacuated + failed + skipped == total is not guaranteed after completion. - uint64 total = 1; + uint64 total_objects = 1; // Evacuated objects count. - uint64 evacuated = 2; + uint64 evacuated_objects = 2; // Failed objects count. - uint64 failed = 3; + uint64 failed_objects = 3; // Shard IDs. repeated bytes shard_ID = 4; @@ -412,7 +412,14 @@ message GetShardEvacuationStatusResponse { string error_message = 8; // Skipped objects count. - uint64 skipped = 9; + uint64 skipped_objects = 9; + + // Total trees to evacuate count. + uint64 total_trees = 10; + // Evacuated trees count. + uint64 evacuated_trees = 11; + // Failed trees count. + uint64 failed_trees = 12; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index b18d30dba..37527aa8c 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -1815,15 +1815,18 @@ func (x *GetShardEvacuationStatusResponse_Body) StableSize() (size int) { if x == nil { return 0 } - size += proto.UInt64Size(1, x.Total) - size += proto.UInt64Size(2, x.Evacuated) - size += proto.UInt64Size(3, x.Failed) + size += proto.UInt64Size(1, x.TotalObjects) + size += proto.UInt64Size(2, x.EvacuatedObjects) + size += proto.UInt64Size(3, x.FailedObjects) size += proto.RepeatedBytesSize(4, x.Shard_ID) size += proto.EnumSize(5, int32(x.Status)) size += proto.NestedStructureSize(6, x.Duration) size += proto.NestedStructureSize(7, x.StartedAt) size += proto.StringSize(8, x.ErrorMessage) - size += proto.UInt64Size(9, x.Skipped) + size += proto.UInt64Size(9, x.SkippedObjects) + size += proto.UInt64Size(10, x.TotalTrees) + size += proto.UInt64Size(11, x.EvacuatedTrees) + size += proto.UInt64Size(12, x.FailedTrees) return size } @@ -1843,15 +1846,18 @@ func (x *GetShardEvacuationStatusResponse_Body) StableMarshal(buf []byte) []byte buf = make([]byte, x.StableSize()) } var offset int - offset += proto.UInt64Marshal(1, buf[offset:], x.Total) - offset += proto.UInt64Marshal(2, buf[offset:], x.Evacuated) - offset += proto.UInt64Marshal(3, buf[offset:], x.Failed) + offset += proto.UInt64Marshal(1, buf[offset:], x.TotalObjects) + offset += proto.UInt64Marshal(2, buf[offset:], x.EvacuatedObjects) + offset += proto.UInt64Marshal(3, buf[offset:], x.FailedObjects) offset += proto.RepeatedBytesMarshal(4, buf[offset:], x.Shard_ID) offset += proto.EnumMarshal(5, buf[offset:], int32(x.Status)) offset += proto.NestedStructureMarshal(6, buf[offset:], x.Duration) offset += proto.NestedStructureMarshal(7, buf[offset:], x.StartedAt) offset += proto.StringMarshal(8, buf[offset:], x.ErrorMessage) - offset += proto.UInt64Marshal(9, buf[offset:], x.Skipped) + offset += proto.UInt64Marshal(9, buf[offset:], x.SkippedObjects) + offset += proto.UInt64Marshal(10, buf[offset:], x.TotalTrees) + offset += proto.UInt64Marshal(11, buf[offset:], x.EvacuatedTrees) + offset += proto.UInt64Marshal(12, buf[offset:], x.FailedTrees) return buf } From e4064c43941098ee639d8efc0b7f75480732830d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Feb 2024 17:49:09 +0300 Subject: [PATCH 0863/1943] [#947] cli: Print tree evacuation stat Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/evacuation.go | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 9d4cb64ac..1d192aa50 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -241,15 +241,17 @@ func printStatus(cmd *cobra.Command, resp *control.GetShardEvacuationStatusRespo func appendEstimation(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { if resp.GetBody().GetStatus() != control.GetShardEvacuationStatusResponse_Body_RUNNING || resp.GetBody().GetDuration() == nil || - resp.GetBody().GetTotal() == 0 || - resp.GetBody().GetEvacuated()+resp.GetBody().GetFailed()+resp.GetBody().GetSkipped() == 0 { + (resp.GetBody().GetTotalObjects() == 0 && resp.GetBody().GetTotalTrees() == 0) || + (resp.GetBody().GetEvacuatedObjects()+resp.GetBody().GetFailedObjects()+resp.GetBody().GetSkippedObjects() == 0 && + resp.GetBody().GetEvacuatedTrees()+resp.GetBody().GetFailedTrees() == 0) { return } durationSeconds := float64(resp.GetBody().GetDuration().GetSeconds()) - evacuated := float64(resp.GetBody().GetEvacuated() + resp.GetBody().GetFailed() + resp.GetBody().GetSkipped()) + evacuated := float64(resp.GetBody().GetEvacuatedObjects() + resp.GetBody().GetFailedObjects() + resp.GetBody().GetSkippedObjects() + + resp.GetBody().GetEvacuatedTrees() + resp.GetBody().GetFailedTrees()) avgObjEvacuationTimeSeconds := durationSeconds / evacuated - objectsLeft := float64(resp.GetBody().GetTotal()) - evacuated + objectsLeft := float64(resp.GetBody().GetTotalObjects()+resp.GetBody().GetTotalTrees()) - evacuated leftSeconds := avgObjEvacuationTimeSeconds * objectsLeft leftMinutes := int(leftSeconds / 60) @@ -307,11 +309,14 @@ func appendShardIDs(sb *strings.Builder, resp *control.GetShardEvacuationStatusR } func appendCounts(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { - sb.WriteString(fmt.Sprintf(" Evacuated %d objects out of %d, failed to evacuate: %d, skipped: %d.", - resp.GetBody().GetEvacuated(), - resp.GetBody().GetTotal(), - resp.GetBody().GetFailed(), - resp.GetBody().GetSkipped())) + sb.WriteString(fmt.Sprintf(" Evacuated %d objects out of %d, failed to evacuate: %d, skipped: %d; evacuated %d trees out of %d, failed to evacuate: %d.", + resp.GetBody().GetEvacuatedObjects(), + resp.GetBody().GetTotalObjects(), + resp.GetBody().GetFailedObjects(), + resp.GetBody().GetSkippedObjects(), + resp.GetBody().GetEvacuatedTrees(), + resp.GetBody().GetTotalTrees(), + resp.GetBody().GetFailedTrees())) } func initControlEvacuationShardCmd() { From 728150d1d292fe00dccabff22b97151ef20409cc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Feb 2024 13:59:50 +0300 Subject: [PATCH 0864/1943] [#947] engine: Evacuate trees to local shards Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 231 +++++++++++++++++- .../engine/evacuate_test.go | 133 ++++++++-- pkg/local_object_storage/pilorama/boltdb.go | 52 ++++ pkg/local_object_storage/pilorama/forest.go | 24 ++ .../pilorama/interface.go | 29 ++- pkg/local_object_storage/shard/tree.go | 29 +++ 6 files changed, 463 insertions(+), 35 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 3853c4e3a..15fb69c2f 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -5,11 +5,13 @@ import ( "errors" "fmt" "strings" + "sync" "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" @@ -56,6 +58,18 @@ func (s EvacuateScope) String() string { return sb.String() } +func (s EvacuateScope) WithObjects() bool { + return s&EvacuateScopeObjects == EvacuateScopeObjects +} + +func (s EvacuateScope) WithTrees() bool { + return s&EvacuateScopeTrees == EvacuateScopeTrees +} + +func (s EvacuateScope) TreesOnly() bool { + return s == EvacuateScopeTrees +} + // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { ShardID []*shard.ID @@ -264,7 +278,7 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) - err = e.getTotalObjectsCount(ctx, shardsToEvacuate, res) + err = e.getTotals(ctx, prm, shardsToEvacuate, res) if err != nil { e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) @@ -293,19 +307,28 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p return nil } -func (e *StorageEngine) getTotalObjectsCount(ctx context.Context, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) error { - ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.getTotalObjectsCount") +func (e *StorageEngine) getTotals(ctx context.Context, prm EvacuateShardPrm, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.getTotals") defer span.End() for _, sh := range shardsToEvacuate { - cnt, err := sh.LogicalObjectsCount(ctx) - if err != nil { - if errors.Is(err, shard.ErrDegradedMode) { - continue + if prm.Scope.WithObjects() { + cnt, err := sh.LogicalObjectsCount(ctx) + if err != nil { + if errors.Is(err, shard.ErrDegradedMode) { + continue + } + return err } - return err + res.objTotal.Add(cnt) + } + if prm.Scope.WithTrees() && sh.PiloramaEnabled() { + cnt, err := pilorama.TreeCountAll(ctx, sh) + if err != nil { + return err + } + res.trTotal.Add(cnt) } - res.objTotal.Add(cnt) } return nil } @@ -319,6 +342,24 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E )) defer span.End() + if prm.Scope.WithObjects() { + if err := e.evacuateShardObjects(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { + return err + } + } + + if prm.Scope.WithTrees() && shardsToEvacuate[shardID].PiloramaEnabled() { + if err := e.evacuateShardTrees(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { + return err + } + } + + return nil +} + +func (e *StorageEngine) evacuateShardObjects(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, +) error { var listPrm shard.ListWithCursorPrm listPrm.WithCount(defaultEvacuateBatchSize) @@ -349,6 +390,172 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E return nil } +func (e *StorageEngine) evacuateShardTrees(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, +) error { + sh := shardsToEvacuate[shardID] + + var listPrm pilorama.TreeListTreesPrm + first := true + + for len(listPrm.NextPageToken) > 0 || first { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + first = false + + listRes, err := sh.TreeListTrees(ctx, listPrm) + if err != nil { + return err + } + listPrm.NextPageToken = listRes.NextPageToken + if err := e.evacuateTrees(ctx, sh, listRes.Items, prm, res, shards, weights, shardsToEvacuate); err != nil { + return err + } + } + return nil +} + +func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, trees []pilorama.ContainerIDTreeID, + prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, weights []float64, + shardsToEvacuate map[string]*shard.Shard, +) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateTrees", + trace.WithAttributes( + attribute.Int("trees_count", len(trees)), + )) + defer span.End() + + for _, contTree := range trees { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + success, _, err := e.tryEvacuateTreeLocal(ctx, sh, contTree, prm, shards, weights, shardsToEvacuate) + if err != nil { + return err + } + if success { + res.trEvacuated.Add(1) + } else { + res.trFailed.Add(1) + } + } + return nil +} + +func (e *StorageEngine) tryEvacuateTreeLocal(ctx context.Context, sh *shard.Shard, tree pilorama.ContainerIDTreeID, + prm EvacuateShardPrm, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, +) (bool, string, error) { + target, found, err := e.findShardToEvacuateTree(ctx, tree, shards, weights, shardsToEvacuate) + if err != nil { + return false, "", err + } + if !found { + return false, "", nil + } + const readBatchSize = 1000 + source := make(chan *pilorama.Move, readBatchSize) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + var wg sync.WaitGroup + + wg.Add(1) + var applyErr error + go func() { + defer wg.Done() + + applyErr = target.TreeApplyStream(ctx, tree.CID, tree.TreeID, source) + if applyErr != nil { + cancel() + } + }() + + var height uint64 + for { + op, err := sh.TreeGetOpLog(ctx, tree.CID, tree.TreeID, height) + if err != nil { + cancel() + wg.Wait() + close(source) // close after cancel to ctx.Done() hits first + if prm.IgnoreErrors { + return false, "", nil + } + return false, "", err + } + + if op.Time == 0 { // completed get op log + close(source) + wg.Wait() + if applyErr == nil { + return true, target.ID().String(), nil + } + if prm.IgnoreErrors { + return false, "", nil + } + return false, "", applyErr + } + + select { + case <-ctx.Done(): // apply stream failed or operation cancelled + wg.Wait() + if prm.IgnoreErrors { + return false, "", nil + } + if applyErr != nil { + return false, "", applyErr + } + return false, "", ctx.Err() + case source <- &op: + } + + height = op.Time + 1 + } +} + +// findShardToEvacuateTree returns first shard according HRW or first shard with tree exists. +func (e *StorageEngine) findShardToEvacuateTree(ctx context.Context, tree pilorama.ContainerIDTreeID, + shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, +) (pooledShard, bool, error) { + hrw.SortHasherSliceByWeightValue(shards, weights, hrw.StringHash(tree.CID.EncodeToString())) + var result pooledShard + var found bool + for _, target := range shards { + select { + case <-ctx.Done(): + return pooledShard{}, false, ctx.Err() + default: + } + + if _, ok := shardsToEvacuate[target.ID().String()]; ok { + continue + } + + if !target.PiloramaEnabled() || target.GetMode().ReadOnly() { + continue + } + + if !found { + result = target + found = true + } + + exists, err := target.TreeExists(ctx, tree.CID, tree.TreeID) + if err != nil { + continue + } + if exists { + return target, true, nil + } + } + return result, found, nil +} + func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) ([]pooledShard, []float64, error) { e.mtx.RLock() defer e.mtx.RUnlock() @@ -362,9 +569,13 @@ func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) if !sh.GetMode().ReadOnly() { return nil, nil, ErrMustBeReadOnly } + + if prm.Scope.TreesOnly() && !sh.PiloramaEnabled() { + return nil, nil, fmt.Errorf("shard %s doesn't have pilorama enabled", sh.ID()) + } } - if len(e.shards)-len(shardIDs) < 1 && prm.ObjectsHandler == nil { + if len(e.shards)-len(shardIDs) < 1 && prm.ObjectsHandler == nil && prm.Scope.WithObjects() { return nil, nil, errMustHaveTwoShards } diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index cadcc2933..d234f8ee3 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" @@ -41,6 +42,10 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", id))), meta.WithPermissions(0o700), meta.WithEpochState(epochState{})), + shard.WithPiloramaOptions( + pilorama.WithPath(filepath.Join(dir, fmt.Sprintf("%d.pilorama", id))), + pilorama.WithPerm(0o700), + ), } }) e, ids := te.engine, te.shardIDs @@ -48,36 +53,32 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng require.NoError(t, e.Init(context.Background())) objects := make([]*objectSDK.Object, 0, objPerShard*len(ids)) - - for _, sh := range ids { - obj := testutil.GenerateObjectWithCID(cidtest.ID()) - objects = append(objects, obj) - - var putPrm shard.PutPrm - putPrm.SetObject(obj) - _, err := e.shards[sh.String()].Put(context.Background(), putPrm) - require.NoError(t, err) + treeID := "version" + meta := []pilorama.KeyValue{ + {Key: pilorama.AttributeVersion, Value: []byte("XXX")}, + {Key: pilorama.AttributeFilename, Value: []byte("file.txt")}, } - for i := 0; ; i++ { - objects = append(objects, testutil.GenerateObjectWithCID(cidtest.ID())) + for _, sh := range ids { + for i := 0; i < objPerShard; i++ { + contID := cidtest.ID() + obj := testutil.GenerateObjectWithCID(contID) + objects = append(objects, obj) - var putPrm PutPrm - putPrm.WithObject(objects[len(objects)-1]) + var putPrm shard.PutPrm + putPrm.SetObject(obj) + _, err := e.shards[sh.String()].Put(context.Background(), putPrm) + require.NoError(t, err) - err := e.Put(context.Background(), putPrm) - require.NoError(t, err) - - res, err := e.shards[ids[len(ids)-1].String()].List(context.Background()) - require.NoError(t, err) - if len(res.AddressList()) == objPerShard { - break + _, err = e.shards[sh.String()].TreeAddByPath(context.Background(), pilorama.CIDDescriptor{CID: contID, Position: 0, Size: 1}, + treeID, pilorama.AttributeFilename, []string{"path", "to", "the", "file"}, meta) + require.NoError(t, err) } } return e, ids, objects } -func TestEvacuateShard(t *testing.T) { +func TestEvacuateShardObjects(t *testing.T) { t.Parallel() const objPerShard = 3 @@ -103,6 +104,7 @@ func TestEvacuateShard(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[2:3] + prm.Scope = EvacuateScopeObjects t.Run("must be read-only", func(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) @@ -137,7 +139,7 @@ func TestEvacuateShard(t *testing.T) { checkHasObjects(t) } -func TestEvacuateNetwork(t *testing.T) { +func TestEvacuateObjectsNetwork(t *testing.T) { t.Parallel() errReplication := errors.New("handler error") @@ -174,6 +176,7 @@ func TestEvacuateNetwork(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[0:1] + prm.Scope = EvacuateScopeObjects res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errMustHaveTwoShards) @@ -198,6 +201,7 @@ func TestEvacuateNetwork(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] prm.ObjectsHandler = acceptOneOf(objects, 2) + prm.Scope = EvacuateScopeObjects res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) @@ -235,6 +239,7 @@ func TestEvacuateNetwork(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = evacuateIDs prm.ObjectsHandler = acceptOneOf(objects, totalCount-1) + prm.Scope = EvacuateScopeObjects res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) @@ -270,6 +275,7 @@ func TestEvacuateCancellation(t *testing.T) { } return nil } + prm.Scope = EvacuateScopeObjects ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -293,6 +299,7 @@ func TestEvacuateSingleProcess(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] + prm.Scope = EvacuateScopeObjects prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { select { case <-running: @@ -321,7 +328,7 @@ func TestEvacuateSingleProcess(t *testing.T) { require.NoError(t, eg.Wait()) } -func TestEvacuateAsync(t *testing.T) { +func TestEvacuateObjectsAsync(t *testing.T) { e, ids, _ := newEngineEvacuate(t, 2, 3) defer func() { require.NoError(t, e.Close(context.Background())) @@ -335,6 +342,7 @@ func TestEvacuateAsync(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] + prm.Scope = EvacuateScopeObjects prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { select { case <-running: @@ -393,3 +401,82 @@ func TestEvacuateAsync(t *testing.T) { require.NoError(t, eg.Wait()) } + +func TestEvacuateTreesLocal(t *testing.T) { + e, ids, _ := newEngineEvacuate(t, 2, 3) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + + var prm EvacuateShardPrm + prm.ShardID = ids[0:1] + prm.Scope = EvacuateScopeTrees + + expectedShardIDs := make([]string, 0, 1) + for _, id := range ids[0:1] { + expectedShardIDs = append(expectedShardIDs, id.String()) + } + + st, err := e.GetEvacuationState(context.Background()) + require.NoError(t, err, "get init state failed") + require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state") + require.Equal(t, uint64(0), st.TreesEvacuated(), "invalid init count") + require.Nil(t, st.StartedAt(), "invalid init started at") + require.Nil(t, st.FinishedAt(), "invalid init finished at") + require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid init error message") + + res, err := e.Evacuate(context.Background(), prm) + require.NotNil(t, res, "sync evacuation result must be not nil") + require.NoError(t, err, "evacuation failed") + + st, err = e.GetEvacuationState(context.Background()) + require.NoError(t, err, "get evacuation state failed") + require.Equal(t, EvacuateProcessStateCompleted, st.ProcessingStatus()) + + require.Equal(t, uint64(3), st.TreesTotal(), "invalid trees total count") + require.Equal(t, uint64(3), st.TreesEvacuated(), "invalid trees evacuated count") + require.Equal(t, uint64(0), st.TreesFailed(), "invalid trees failed count") + require.NotNil(t, st.StartedAt(), "invalid final started at") + require.NotNil(t, st.FinishedAt(), "invalid final finished at") + require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid final error message") + + sourceTrees, err := pilorama.TreeListAll(context.Background(), e.shards[ids[0].String()]) + require.NoError(t, err, "list source trees failed") + require.Len(t, sourceTrees, 3) + + for _, tr := range sourceTrees { + exists, err := e.shards[ids[1].String()].TreeExists(context.Background(), tr.CID, tr.TreeID) + require.NoError(t, err, "failed to check tree existance") + require.True(t, exists, "tree doesn't exists on target shard") + + var height uint64 + var sourceOps []pilorama.Move + for { + op, err := e.shards[ids[0].String()].TreeGetOpLog(context.Background(), tr.CID, tr.TreeID, height) + require.NoError(t, err) + if op.Time == 0 { + break + } + sourceOps = append(sourceOps, op) + height = op.Time + 1 + } + + height = 0 + var targetOps []pilorama.Move + for { + op, err := e.shards[ids[1].String()].TreeGetOpLog(context.Background(), tr.CID, tr.TreeID, height) + require.NoError(t, err) + if op.Time == 0 { + break + } + targetOps = append(targetOps, op) + height = op.Time + 1 + } + + require.Equal(t, sourceOps, targetOps) + } +} diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 39400391f..61318fffc 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -549,6 +549,58 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string return metaerr.Wrap(err) } +// TreeApplyStream should be used with caution: this method locks other write transactions while `source` is not closed. +func (t *boltForest) TreeApplyStream(ctx context.Context, cnr cidSDK.ID, treeID string, source <-chan *Move) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeApplyStream", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeApplyStream", + trace.WithAttributes( + attribute.String("container_id", cnr.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + t.modeMtx.RLock() + defer t.modeMtx.RUnlock() + + if t.mode.NoMetabase() { + return ErrDegradedMode + } else if t.mode.ReadOnly() { + return ErrReadOnlyMode + } + + fullID := bucketName(cnr, treeID) + err := metaerr.Wrap(t.db.Update(func(tx *bbolt.Tx) error { + bLog, bTree, err := t.getTreeBuckets(tx, fullID) + if err != nil { + return err + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + case m, ok := <-source: + if !ok { + return nil + } + var lm Move + if e := t.applyOperation(bLog, bTree, []*Move{m}, &lm); e != nil { + return e + } + } + } + })) + success = err == nil + return err +} + func (t *boltForest) addBatch(cnr cidSDK.ID, treeID string, m *Move, ch chan error) { t.mtx.Lock() for i := 0; i < len(t.batches); i++ { diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 8a1e86147..ab6174002 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -311,3 +311,27 @@ func (f *memoryForest) TreeListTrees(_ context.Context, prm TreeListTreesPrm) (* } return &result, nil } + +// TreeApplyStream implements ForestStorage. +func (f *memoryForest) TreeApplyStream(ctx context.Context, cnr cid.ID, treeID string, source <-chan *Move) error { + fullID := cnr.String() + "/" + treeID + s, ok := f.treeMap[fullID] + if !ok { + s = newMemoryTree() + f.treeMap[fullID] = s + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case m, ok := <-source: + if !ok { + return nil + } + if e := s.Apply(m); e != nil { + return e + } + } + } +} diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 3efd1a68b..b3391407c 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -66,6 +66,7 @@ type ForestStorage interface { // TreeListTrees returns all pairs "containerID:treeID". TreeListTrees(ctx context.Context, prm TreeListTreesPrm) (*TreeListTreesResult, error) + TreeApplyStream(ctx context.Context, cnr cidSDK.ID, treeID string, source <-chan *Move) error } const ( @@ -107,12 +108,17 @@ type TreeListTreesResult struct { Items []ContainerIDTreeID } -func TreeListAll(ctx context.Context, f ForestStorage) ([]ContainerIDTreeID, error) { +type treeList interface { + TreeListTrees(ctx context.Context, prm TreeListTreesPrm) (*TreeListTreesResult, error) +} + +func TreeListAll(ctx context.Context, f treeList) ([]ContainerIDTreeID, error) { return treeListAll(ctx, f, treeListTreesBatchSizeDefault) } -func treeListAll(ctx context.Context, f ForestStorage, batchSize int) ([]ContainerIDTreeID, error) { +func treeListAll(ctx context.Context, f treeList, batchSize int) ([]ContainerIDTreeID, error) { var prm TreeListTreesPrm + prm.BatchSize = batchSize var result []ContainerIDTreeID first := true @@ -129,3 +135,22 @@ func treeListAll(ctx context.Context, f ForestStorage, batchSize int) ([]Contain return result, nil } + +func TreeCountAll(ctx context.Context, f treeList) (uint64, error) { + var prm TreeListTreesPrm + var result uint64 + first := true + + for len(prm.NextPageToken) > 0 || first { + first = false + + res, err := f.TreeListTrees(ctx, prm) + if err != nil { + return 0, err + } + prm.NextPageToken = res.NextPageToken + result += uint64(len(res.Items)) + } + + return result, nil +} diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 017b3450d..e78f29277 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -374,3 +374,32 @@ func (s *Shard) TreeListTrees(ctx context.Context, prm pilorama.TreeListTreesPrm } return s.pilorama.TreeListTrees(ctx, prm) } + +func (s *Shard) PiloramaEnabled() bool { + return s.pilorama != nil +} + +func (s *Shard) TreeApplyStream(ctx context.Context, cnr cidSDK.ID, treeID string, source <-chan *pilorama.Move) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeApplyStream", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cnr.EncodeToString()), + attribute.String("tree_id", treeID)), + ) + defer span.End() + + if s.pilorama == nil { + return ErrPiloramaDisabled + } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.ReadOnly() { + return ErrReadOnlyMode + } + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } + return s.pilorama.TreeApplyStream(ctx, cnr, treeID, source) +} From db67c21d5578729c2a329e3a475ac6ca08fd1f80 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Feb 2024 17:34:32 +0300 Subject: [PATCH 0865/1943] [#947] engine: Evacuate trees to remote nodes Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/control.go | 14 +- internal/logs/logs.go | 3 + pkg/local_object_storage/engine/evacuate.go | 37 +++++- .../engine/evacuate_test.go | 85 ++++++++++++ pkg/services/control/server/evacuate.go | 121 ++++++++++++++---- pkg/services/control/server/evacuate_async.go | 3 +- .../control/server/syncronize_tree.go | 7 +- pkg/services/tree/replicator.go | 108 ++++++++-------- 8 files changed, 283 insertions(+), 95 deletions(-) diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index ee2a2952a..e1e6e3ac9 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -9,23 +9,13 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" controlSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sdnotify" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" "google.golang.org/grpc" ) const serviceNameControl = "control" -type treeSynchronizer struct { - treeSvc *tree.Service -} - -func (t treeSynchronizer) Synchronize(ctx context.Context, cnr cid.ID, treeID string) error { - return t.treeSvc.SynchronizeTree(ctx, cnr, treeID) -} - func initControlService(c *cfg) { endpoint := controlconfig.GRPC(c.appCfg).Endpoint() if endpoint == controlconfig.GRPCEndpointDefault { @@ -50,9 +40,7 @@ func initControlService(c *cfg) { controlSvc.WithReplicator(c.replicator), controlSvc.WithNodeState(c), controlSvc.WithLocalStorage(c.cfgObject.cfgLocalStorage.localStorage), - controlSvc.WithTreeService(treeSynchronizer{ - c.treeService, - }), + controlSvc.WithTreeService(c.treeService), controlSvc.WithLocalOverrideStorage(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine), ) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index c3d2e5150..d839915d8 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -575,4 +575,7 @@ const ( GetSvcV2FailedToGetRangeHashFromNode = "failed to get range hash from node" GetSvcV2FailedToGetRangeHashFromAllOfContainerNodes = "failed to get range hash from all of container nodes" FailedToUpdateShardID = "failed to update shard id" + EngineShardsEvacuationFailedToMoveTree = "failed to evacuate tree to other node" + EngineShardsEvacuationTreeEvacuatedLocal = "tree evacuated to local node" + EngineShardsEvacuationTreeEvacuatedRemote = "tree evacuated to other node" ) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 15fb69c2f..becddfa8b 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -17,6 +17,7 @@ import ( tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" @@ -74,6 +75,7 @@ func (s EvacuateScope) TreesOnly() bool { type EvacuateShardPrm struct { ShardID []*shard.ID ObjectsHandler func(context.Context, oid.Address, *objectSDK.Object) error + TreeHandler func(context.Context, cid.ID, string, pilorama.Forest) (string, error) IgnoreErrors bool Async bool Scope EvacuateScope @@ -436,19 +438,44 @@ func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, tree default: } - success, _, err := e.tryEvacuateTreeLocal(ctx, sh, contTree, prm, shards, weights, shardsToEvacuate) + success, shardID, err := e.tryEvacuateTreeLocal(ctx, sh, contTree, prm, shards, weights, shardsToEvacuate) if err != nil { return err } if success { + e.log.Debug(logs.EngineShardsEvacuationTreeEvacuatedLocal, + zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), + zap.String("from_shard_id", sh.ID().String()), zap.String("to_shard_id", shardID), + evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) res.trEvacuated.Add(1) - } else { - res.trFailed.Add(1) + continue } + + nodePK, err := e.evacuateTreeToOtherNode(ctx, sh, contTree, prm) + if err != nil { + e.log.Error(logs.EngineShardsEvacuationFailedToMoveTree, + zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), + zap.String("from_shard_id", sh.ID().String()), evacuationOperationLogField, + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + return err + } + e.log.Debug(logs.EngineShardsEvacuationTreeEvacuatedRemote, + zap.String("cid", contTree.CID.EncodeToString()), zap.String("treeID", contTree.TreeID), + zap.String("from_shardID", sh.ID().String()), zap.String("to_node", nodePK), + evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + res.trEvacuated.Add(1) } return nil } +func (e *StorageEngine) evacuateTreeToOtherNode(ctx context.Context, sh *shard.Shard, tree pilorama.ContainerIDTreeID, prm EvacuateShardPrm) (string, error) { + if prm.TreeHandler == nil { + return "", fmt.Errorf("failed to evacuate tree '%s' for container %s from shard %s: local evacuation failed, but no remote evacuation available", tree.TreeID, tree.CID, sh.ID()) + } + + return prm.TreeHandler(ctx, tree.CID, tree.TreeID, sh) +} + func (e *StorageEngine) tryEvacuateTreeLocal(ctx context.Context, sh *shard.Shard, tree pilorama.ContainerIDTreeID, prm EvacuateShardPrm, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, ) (bool, string, error) { @@ -579,6 +606,10 @@ func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) return nil, nil, errMustHaveTwoShards } + if len(e.shards)-len(shardIDs) < 1 && prm.TreeHandler == nil && prm.Scope.WithTrees() { + return nil, nil, errMustHaveTwoShards + } + // We must have all shards, to have correct information about their // indexes in a sorted slice and set appropriate marks in the metabase. // Evacuated shard is skipped during put. diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index d234f8ee3..32582cd2a 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -18,6 +18,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -480,3 +481,87 @@ func TestEvacuateTreesLocal(t *testing.T) { require.Equal(t, sourceOps, targetOps) } } + +func TestEvacuateTreesRemote(t *testing.T) { + e, ids, _ := newEngineEvacuate(t, 2, 3) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + + evacuatedTreeOps := make(map[string][]*pilorama.Move) + var prm EvacuateShardPrm + prm.ShardID = ids + prm.Scope = EvacuateScopeTrees + prm.TreeHandler = func(ctx context.Context, contID cid.ID, treeID string, f pilorama.Forest) (string, error) { + key := contID.String() + treeID + var height uint64 + for { + op, err := f.TreeGetOpLog(ctx, contID, treeID, height) + require.NoError(t, err) + + if op.Time == 0 { + return "", nil + } + evacuatedTreeOps[key] = append(evacuatedTreeOps[key], &op) + height = op.Time + 1 + } + } + + expectedShardIDs := make([]string, 0, len(ids)) + for _, id := range ids { + expectedShardIDs = append(expectedShardIDs, id.String()) + } + + st, err := e.GetEvacuationState(context.Background()) + require.NoError(t, err, "get init state failed") + require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state") + require.Equal(t, uint64(0), st.TreesEvacuated(), "invalid init count") + require.Nil(t, st.StartedAt(), "invalid init started at") + require.Nil(t, st.FinishedAt(), "invalid init finished at") + require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid init error message") + + res, err := e.Evacuate(context.Background(), prm) + require.NotNil(t, res, "sync evacuation must return not nil") + require.NoError(t, err, "evacuation failed") + + st, err = e.GetEvacuationState(context.Background()) + require.NoError(t, err, "get evacuation state failed") + require.Equal(t, EvacuateProcessStateCompleted, st.ProcessingStatus()) + + require.NoError(t, err, "get final state failed") + require.Equal(t, uint64(6), st.TreesTotal(), "invalid trees total count") + require.Equal(t, uint64(6), st.TreesEvacuated(), "invalid trees evacuated count") + require.Equal(t, uint64(0), st.TreesFailed(), "invalid trees failed count") + require.NotNil(t, st.StartedAt(), "invalid final started at") + require.NotNil(t, st.FinishedAt(), "invalid final finished at") + require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids") + require.Equal(t, "", st.ErrorMessage(), "invalid final error message") + + expectedTreeOps := make(map[string][]*pilorama.Move) + for i := 0; i < len(e.shards); i++ { + sourceTrees, err := pilorama.TreeListAll(context.Background(), e.shards[ids[i].String()]) + require.NoError(t, err, "list source trees failed") + require.Len(t, sourceTrees, 3) + + for _, tr := range sourceTrees { + key := tr.CID.String() + tr.TreeID + var height uint64 + for { + op, err := e.shards[ids[i].String()].TreeGetOpLog(context.Background(), tr.CID, tr.TreeID, height) + require.NoError(t, err) + + if op.Time == 0 { + break + } + expectedTreeOps[key] = append(expectedTreeOps[key], &op) + height = op.Time + 1 + } + } + } + + require.Equal(t, expectedTreeOps, evacuatedTreeOps) +} diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index ac8b3d548..99082b30d 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -4,13 +4,17 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/hex" "errors" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -28,7 +32,7 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe prm := engine.EvacuateShardPrm{ ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), IgnoreErrors: req.GetBody().GetIgnoreErrors(), - ObjectsHandler: s.replicate, + ObjectsHandler: s.replicateObject, Scope: engine.EvacuateScopeObjects, } @@ -50,7 +54,7 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe return resp, nil } -func (s *Server) replicate(ctx context.Context, addr oid.Address, obj *objectSDK.Object) error { +func (s *Server) replicateObject(ctx context.Context, addr oid.Address, obj *objectSDK.Object) error { cid, ok := obj.ContainerID() if !ok { // Return nil to prevent situations where a shard can't be evacuated @@ -58,33 +62,11 @@ func (s *Server) replicate(ctx context.Context, addr oid.Address, obj *objectSDK return nil } - nm, err := s.netMapSrc.GetNetMap(0) + nodes, err := s.getContainerNodes(cid) if err != nil { return err } - c, err := s.cnrSrc.Get(cid) - if err != nil { - return err - } - - binCnr := make([]byte, sha256.Size) - cid.Encode(binCnr) - - ns, err := nm.ContainerNodes(c.Value.PlacementPolicy(), binCnr) - if err != nil { - return fmt.Errorf("can't build a list of container nodes") - } - - nodes := placement.FlattenNodes(ns) - bs := (*keys.PublicKey)(&s.key.PublicKey).Bytes() - for i := 0; i < len(nodes); i++ { - if bytes.Equal(nodes[i].PublicKey(), bs) { - copy(nodes[i:], nodes[i+1:]) - nodes = nodes[:len(nodes)-1] - } - } - var res replicatorResult task := replicator.Task{ NumCopies: 1, @@ -100,6 +82,95 @@ func (s *Server) replicate(ctx context.Context, addr oid.Address, obj *objectSDK return nil } +func (s *Server) replicateTree(ctx context.Context, contID cid.ID, treeID string, forest pilorama.Forest) (string, error) { + nodes, err := s.getContainerNodes(contID) + if err != nil { + return "", err + } + if len(nodes) == 0 { + return "", fmt.Errorf("no remote nodes available to replicate tree '%s' of container %s", treeID, contID) + } + + for _, node := range nodes { + err = s.replicateTreeToNode(ctx, forest, contID, treeID, node) + if err == nil { + return hex.EncodeToString(node.PublicKey()), nil + } + } + return "", err +} + +func (s *Server) replicateTreeToNode(ctx context.Context, forest pilorama.Forest, contID cid.ID, treeID string, node netmap.NodeInfo) error { + rawCID := make([]byte, sha256.Size) + contID.Encode(rawCID) + + var height uint64 + for { + op, err := forest.TreeGetOpLog(ctx, contID, treeID, height) + if err != nil { + return err + } + + if op.Time == 0 { + return nil + } + + req := &tree.ApplyRequest{ + Body: &tree.ApplyRequest_Body{ + ContainerId: rawCID, + TreeId: treeID, + Operation: &tree.LogMove{ + ParentId: op.Parent, + Meta: op.Meta.Bytes(), + ChildId: op.Child, + }, + }, + } + + err = tree.SignMessage(req, s.key) + if err != nil { + return fmt.Errorf("can't sign apply request: %w", err) + } + + err = s.treeService.ReplicateTreeOp(ctx, node, req) + if err != nil { + return err + } + + height = op.Time + 1 + } +} + +func (s *Server) getContainerNodes(contID cid.ID) ([]netmap.NodeInfo, error) { + nm, err := s.netMapSrc.GetNetMap(0) + if err != nil { + return nil, err + } + + c, err := s.cnrSrc.Get(contID) + if err != nil { + return nil, err + } + + binCnr := make([]byte, sha256.Size) + contID.Encode(binCnr) + + ns, err := nm.ContainerNodes(c.Value.PlacementPolicy(), binCnr) + if err != nil { + return nil, fmt.Errorf("can't build a list of container nodes") + } + + nodes := placement.FlattenNodes(ns) + bs := (*keys.PublicKey)(&s.key.PublicKey).Bytes() + for i := 0; i < len(nodes); i++ { + if bytes.Equal(nodes[i].PublicKey(), bs) { + copy(nodes[i:], nodes[i+1:]) + nodes = nodes[:len(nodes)-1] + } + } + return nodes, nil +} + type replicatorResult struct { count int } diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index 91f0731c4..cd7e8a2c7 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -24,7 +24,8 @@ func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartSha prm := engine.EvacuateShardPrm{ ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), IgnoreErrors: req.GetBody().GetIgnoreErrors(), - ObjectsHandler: s.replicate, + ObjectsHandler: s.replicateObject, + TreeHandler: s.replicateTree, Async: true, Scope: engine.EvacuateScope(req.GetBody().GetScope()), } diff --git a/pkg/services/control/server/syncronize_tree.go b/pkg/services/control/server/syncronize_tree.go index dce3e8831..678f87d0a 100644 --- a/pkg/services/control/server/syncronize_tree.go +++ b/pkg/services/control/server/syncronize_tree.go @@ -4,14 +4,17 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // TreeService represents a tree service instance. type TreeService interface { - Synchronize(ctx context.Context, cnr cid.ID, treeID string) error + SynchronizeTree(ctx context.Context, cnr cid.ID, treeID string) error + ReplicateTreeOp(ctx context.Context, n netmapSDK.NodeInfo, req *tree.ApplyRequest) error } func (s *Server) SynchronizeTree(ctx context.Context, req *control.SynchronizeTreeRequest) (*control.SynchronizeTreeResponse, error) { @@ -31,7 +34,7 @@ func (s *Server) SynchronizeTree(ctx context.Context, req *control.SynchronizeTr return nil, status.Error(codes.Internal, err.Error()) } - err = s.treeService.Synchronize(ctx, cnr, b.GetTreeId()) + err = s.treeService.SynchronizeTree(ctx, cnr, b.GetTreeId()) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 49a37b8be..346198b3c 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -71,61 +71,67 @@ func (s *Service) replicationWorker(ctx context.Context) { case <-s.closeCh: return case task := <-s.replicationTasks: - ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.HandleReplicationTask", - trace.WithAttributes( - attribute.String("public_key", hex.EncodeToString(task.n.PublicKey())), - ), - ) - start := time.Now() - - var lastErr error - var lastAddr string - - task.n.IterateNetworkEndpoints(func(addr string) bool { - ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.HandleReplicationTaskOnEndpoint", - trace.WithAttributes( - attribute.String("public_key", hex.EncodeToString(task.n.PublicKey())), - attribute.String("address", addr), - ), - ) - defer span.End() - - lastAddr = addr - - c, err := s.cache.get(ctx, addr) - if err != nil { - lastErr = fmt.Errorf("can't create client: %w", err) - return false - } - - ctx, cancel := context.WithTimeout(ctx, s.replicatorTimeout) - _, lastErr = c.Apply(ctx, task.req) - cancel() - - return lastErr == nil - }) - - if lastErr != nil { - if errors.Is(lastErr, errRecentlyFailed) { - s.log.Debug(logs.TreeDoNotSendUpdateToTheNode, - zap.String("last_error", lastErr.Error()), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) - } else { - s.log.Warn(logs.TreeFailedToSentUpdateToTheNode, - zap.String("last_error", lastErr.Error()), - zap.String("address", lastAddr), - zap.String("key", hex.EncodeToString(task.n.PublicKey())), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) - } - s.metrics.AddReplicateTaskDuration(time.Since(start), false) - } else { - s.metrics.AddReplicateTaskDuration(time.Since(start), true) - } - span.End() + _ = s.ReplicateTreeOp(ctx, task.n, task.req) } } } +func (s *Service) ReplicateTreeOp(ctx context.Context, n netmapSDK.NodeInfo, req *ApplyRequest) error { + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.HandleReplicationTask", + trace.WithAttributes( + attribute.String("public_key", hex.EncodeToString(n.PublicKey())), + ), + ) + defer span.End() + + start := time.Now() + + var lastErr error + var lastAddr string + + n.IterateNetworkEndpoints(func(addr string) bool { + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.HandleReplicationTaskOnEndpoint", + trace.WithAttributes( + attribute.String("public_key", hex.EncodeToString(n.PublicKey())), + attribute.String("address", addr), + ), + ) + defer span.End() + + lastAddr = addr + + c, err := s.cache.get(ctx, addr) + if err != nil { + lastErr = fmt.Errorf("can't create client: %w", err) + return false + } + + ctx, cancel := context.WithTimeout(ctx, s.replicatorTimeout) + _, lastErr = c.Apply(ctx, req) + cancel() + + return lastErr == nil + }) + + if lastErr != nil { + if errors.Is(lastErr, errRecentlyFailed) { + s.log.Debug(logs.TreeDoNotSendUpdateToTheNode, + zap.String("last_error", lastErr.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } else { + s.log.Warn(logs.TreeFailedToSentUpdateToTheNode, + zap.String("last_error", lastErr.Error()), + zap.String("address", lastAddr), + zap.String("key", hex.EncodeToString(n.PublicKey())), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } + s.metrics.AddReplicateTaskDuration(time.Since(start), false) + return lastErr + } + s.metrics.AddReplicateTaskDuration(time.Since(start), true) + return nil +} + func (s *Service) replicateLoop(ctx context.Context) { for i := 0; i < s.replicatorWorkerCount; i++ { go s.replicationWorker(ctx) From 60527abb65efe99ec85949d5a8f71d813d9ba386 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 8 Feb 2024 11:45:36 +0300 Subject: [PATCH 0866/1943] [#947] docs: Extend evacuation docs Add pilorama evacuation description and examples. Signed-off-by: Dmitrii Stepanov --- docs/evacuation.md | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/docs/evacuation.md b/docs/evacuation.md index 9bfa0e214..1bccc87bf 100644 --- a/docs/evacuation.md +++ b/docs/evacuation.md @@ -14,7 +14,7 @@ Only one running evacuation process is allowed on the node at a time. ## Commands -`frostfs-cli control shards evacuation start` starts evacuation process for shards specified. To start evacuating all node shards, use the `--all` flag. +`frostfs-cli control shards evacuation start` starts evacuation process for shards specified. To start evacuating all node shards, use the `--all` flag. By default, objects and trees are evacuated. To limit the evacuation scope, use `--scope` flag (possible values are `all`, `trees`, `objects`). `frostfs-cli control shards evacuation stop` stops running evacuation process. @@ -39,15 +39,15 @@ Shard evacuation has been successfully started. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 14 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:03. Estimated time left: 2 minutes. +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 131 objects out of 618, failed to evacuate: 0, skipped: 0; evacuated 0 trees out of 19, failed to evacuate: 0. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:00:03. Estimated time left: 2 minutes. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 23 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:01:05. Estimated time left: 1 minutes. +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: running. Evacuated 260 objects out of 618, failed to evacuate: 0, skipped: 0; evacuated 0 trees out of 19, failed to evacuate: 0. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:01:05. Estimated time left: 1 minutes. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: completed. Evacuated 61 object out of 61, failed to evacuate 0 objects. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:02:13. +Shard IDs: 8kEBwtvKLU3Hva3PaaodUi. Status: completed. Evacuated 618 objects out of 618, failed to evacuate: 0, skipped: 0; evacuated 19 trees out of 19, failed to evacuate: 0. Started at: 2023-05-10T10:13:06Z UTC. Duration: 00:02:13. ``` ### Stop running evacuation process @@ -58,7 +58,7 @@ Shard evacuation has been successfully started. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 15 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:03. Estimated time left: 0 minutes. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 131 objects out of 618, failed to evacuate: 0, skipped: 0; evacuated 0 trees out of 19, failed to evacuate: 0. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:03. Estimated time left: 0 minutes. frostfs-cli control shards evacuation stop --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > @@ -66,7 +66,7 @@ Evacuation stopped. frostfs-cli control shards evacuation status --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json Enter password > -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: completed. Evacuated 31 object out of 73, failed to evacuate 0 objects. Error: context canceled. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:07. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: completed. Evacuated 131 objects out of 618, failed to evacuate: 0, skipped: 0; evacuated 0 trees out of 19, failed to evacuate: 0. Error: context canceled. Started at: 2023-05-10T10:15:47Z UTC. Duration: 00:00:07. ``` ### Start evacuation and await it completes @@ -75,11 +75,11 @@ frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 - Enter password > Shard evacuation has been successfully started. Progress will be reported every 5 seconds. -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 18 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:04. Estimated time left: 0 minutes. -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 43 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:09. Estimated time left: 0 minutes. -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 68 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. Estimated time left: 0 minutes. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 131 objects out of 618, failed to evacuate: 0, skipped: 0; evacuated 0 trees out of 19, failed to evacuate: 0. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:04. Estimated time left: 0 minutes. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 343 objects out of 618, failed to evacuate: 0, skipped: 0; evacuated 0 trees out of 19, failed to evacuate: 0. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:09. Estimated time left: 0 minutes. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Status: running. Evacuated 545 objects out of 618, failed to evacuate: 0, skipped: 0; evacuated 0 trees out of 19, failed to evacuate: 0. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. Estimated time left: 0 minutes. Shard evacuation has been completed. -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 618 objects out of 618, failed to evacuate: 0, skipped: 0; evacuated 19 trees out of 19, failed to evacuate: 0. Started at: 2023-05-10T10:18:42Z UTC. Duration: 00:00:14. ``` ### Start evacuation and await it completes without progress notifications @@ -88,5 +88,15 @@ frostfs-cli control shards evacuation start --endpoint s01.frostfs.devenv:8081 - Enter password > Shard evacuation has been successfully started. Shard evacuation has been completed. -Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 73 object out of 73, failed to evacuate 0 objects. Started at: 2023-05-10T10:20:00Z UTC. Duration: 00:00:14. +Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 618 objects out of 618, failed to evacuate: 0, skipped: 0; evacuated 19 trees out of 19, failed to evacuate: 0. Started at: 2023-05-10T10:20:00Z UTC. Duration: 00:00:14. ``` + +### Start trees evacuation and await it completes +```bash +frostfs-cli control shards evacuation start --id FxR6QujButNCHn7jjdhxGP --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --await --scope trees +Enter password > +Shard evacuation has been successfully started. +Progress will be reported every 5 seconds. +Shard evacuation has been completed. +Shard IDs: FxR6QujButNCHn7jjdhxGP. Evacuated 0 objects out of 0, failed to evacuate: 0, skipped: 0; evacuated 2 trees out of 2, failed to evacuate: 0. Started at: 2024-02-08T08:44:17Z UTC. Duration: 00:00:00. +``` \ No newline at end of file From d19ade23c8fa0ab5496e94c739efb7ec40d35e64 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 9 Feb 2024 09:17:17 +0300 Subject: [PATCH 0867/1943] [#959] node: Set mode to shard's components when open it Avoid opening database for `metabase` and `cache` in `Degraded` mode. Signed-off-by: Anton Nikiforov --- cmd/frostfs-lens/internal/meta/root.go | 3 +- .../blobstor/blobstor_test.go | 9 +++-- pkg/local_object_storage/blobstor/control.go | 19 +++++++++- .../blobstor/exists_test.go | 3 +- .../blobstor/iterate_test.go | 3 +- pkg/local_object_storage/blobstor/mode.go | 2 +- .../internal/storagetest/storage.go | 18 ++++----- pkg/local_object_storage/metabase/control.go | 15 +++++++- pkg/local_object_storage/metabase/db_test.go | 3 +- pkg/local_object_storage/metabase/mode.go | 24 +++++------- .../metabase/mode_test.go | 37 +++++++++++++++++++ .../metabase/version_test.go | 15 ++++---- .../pilorama/bench_test.go | 3 +- pkg/local_object_storage/pilorama/boltdb.go | 21 +++++++---- pkg/local_object_storage/pilorama/forest.go | 2 +- .../pilorama/forest_test.go | 5 ++- .../pilorama/interface.go | 2 +- .../pilorama/mode_test.go | 31 ++++++++++++++++ pkg/local_object_storage/shard/control.go | 20 ++++++---- pkg/local_object_storage/shard/id.go | 7 ++-- .../writecache/benchmark/writecache_test.go | 3 +- .../writecache/cachebbolt.go | 10 ++++- .../writecache/flush_test.go | 6 +-- .../writecache/mode_test.go | 30 +++++++++++++++ .../writecache/writecache.go | 2 +- 25 files changed, 219 insertions(+), 74 deletions(-) create mode 100644 pkg/local_object_storage/metabase/mode_test.go create mode 100644 pkg/local_object_storage/pilorama/mode_test.go create mode 100644 pkg/local_object_storage/writecache/mode_test.go diff --git a/cmd/frostfs-lens/internal/meta/root.go b/cmd/frostfs-lens/internal/meta/root.go index a59574b6c..6741abd0c 100644 --- a/cmd/frostfs-lens/internal/meta/root.go +++ b/cmd/frostfs-lens/internal/meta/root.go @@ -5,6 +5,7 @@ import ( common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/spf13/cobra" "go.etcd.io/bbolt" ) @@ -43,7 +44,7 @@ func openMeta(cmd *cobra.Command) *meta.DB { }), meta.WithEpochState(epochState{}), ) - common.ExitOnErr(cmd, common.Errf("could not open metabase: %w", db.Open(cmd.Context(), true))) + common.ExitOnErr(cmd, common.Errf("could not open metabase: %w", db.Open(cmd.Context(), mode.ReadOnly))) return db } diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index f1d567da7..3dac3bfa4 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" ) @@ -51,7 +52,7 @@ func TestCompression(t *testing.T) { bs := New( WithCompressObjects(compress), WithStorages(defaultStorages(dir, smallSizeLimit))) - require.NoError(t, bs.Open(context.Background(), false)) + require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) require.NoError(t, bs.Init()) return bs } @@ -126,7 +127,7 @@ func TestBlobstor_needsCompression(t *testing.T) { Storage: fstree.New(fstree.WithPath(dir)), }, })) - require.NoError(t, bs.Open(context.Background(), false)) + require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) require.NoError(t, bs.Init()) return bs } @@ -188,7 +189,7 @@ func TestConcurrentPut(t *testing.T) { blobStor := New( WithStorages(defaultStorages(dir, smallSizeLimit))) - require.NoError(t, blobStor.Open(context.Background(), false)) + require.NoError(t, blobStor.Open(context.Background(), mode.ReadWrite)) require.NoError(t, blobStor.Init()) testGet := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { @@ -268,7 +269,7 @@ func TestConcurrentDelete(t *testing.T) { blobStor := New( WithStorages(defaultStorages(dir, smallSizeLimit))) - require.NoError(t, blobStor.Open(context.Background(), false)) + require.NoError(t, blobStor.Open(context.Background(), mode.ReadWrite)) require.NoError(t, blobStor.Init()) testPut := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 4b8a36de8..45b0c3f10 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -6,13 +6,29 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "go.uber.org/zap" ) // Open opens BlobStor. -func (b *BlobStor) Open(ctx context.Context, readOnly bool) error { +func (b *BlobStor) Open(ctx context.Context, mode mode.Mode) error { b.log.Debug(logs.BlobstorOpening) + b.modeMtx.Lock() + defer b.modeMtx.Unlock() + b.mode = mode + + err := b.openBlobStor(ctx, mode) + if err != nil { + return err + } + b.metrics.SetMode(mode.ReadOnly()) + + return nil +} + +func (b *BlobStor) openBlobStor(ctx context.Context, mode mode.Mode) error { + readOnly := mode.ReadOnly() for i := range b.storage { select { case <-ctx.Done(): @@ -24,7 +40,6 @@ func (b *BlobStor) Open(ctx context.Context, readOnly bool) error { return err } } - b.metrics.SetMode(readOnly) return nil } diff --git a/pkg/local_object_storage/blobstor/exists_test.go b/pkg/local_object_storage/blobstor/exists_test.go index 367b63af1..783c198b2 100644 --- a/pkg/local_object_storage/blobstor/exists_test.go +++ b/pkg/local_object_storage/blobstor/exists_test.go @@ -7,6 +7,7 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -20,7 +21,7 @@ func TestExists(t *testing.T) { b := New(WithStorages(storages)) - require.NoError(t, b.Open(context.Background(), false)) + require.NoError(t, b.Open(context.Background(), mode.ReadWrite)) require.NoError(t, b.Init()) objects := []*objectSDK.Object{ diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index ef3fda991..079728380 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -7,6 +7,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -26,7 +27,7 @@ func TestIterateObjects(t *testing.T) { defer os.RemoveAll(p) // open Blobstor - require.NoError(t, blobStor.Open(context.Background(), false)) + require.NoError(t, blobStor.Open(context.Background(), mode.ReadWrite)) // initialize Blobstor require.NoError(t, blobStor.Init()) diff --git a/pkg/local_object_storage/blobstor/mode.go b/pkg/local_object_storage/blobstor/mode.go index 2f4473bd8..dc7dad687 100644 --- a/pkg/local_object_storage/blobstor/mode.go +++ b/pkg/local_object_storage/blobstor/mode.go @@ -22,7 +22,7 @@ func (b *BlobStor) SetMode(m mode.Mode) error { err := b.Close() if err == nil { - if err = b.Open(context.TODO(), m.ReadOnly()); err == nil { + if err = b.openBlobStor(context.TODO(), m); err == nil { err = b.Init() } } diff --git a/pkg/local_object_storage/internal/storagetest/storage.go b/pkg/local_object_storage/internal/storagetest/storage.go index ec60a2d0e..586b3dcc6 100644 --- a/pkg/local_object_storage/internal/storagetest/storage.go +++ b/pkg/local_object_storage/internal/storagetest/storage.go @@ -10,7 +10,7 @@ import ( // Component represents single storage component. type Component interface { - Open(context.Context, bool) error + Open(context.Context, mode.Mode) error SetMode(mode.Mode) error Init() error Close() error @@ -58,18 +58,18 @@ func TestCloseAfterOpen(t *testing.T, cons Constructor) { t.Run("RW", func(t *testing.T) { // Use-case: irrecoverable error on some components, close everything. s := cons(t) - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Close()) }) t.Run("RO", func(t *testing.T) { // Use-case: irrecoverable error on some components, close everything. // Open in read-only must be done after the db is here. s := cons(t) - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Init()) require.NoError(t, s.Close()) - require.NoError(t, s.Open(context.Background(), true)) + require.NoError(t, s.Open(context.Background(), mode.ReadOnly)) require.NoError(t, s.Close()) }) } @@ -78,7 +78,7 @@ func TestCloseAfterOpen(t *testing.T, cons Constructor) { func TestCloseTwice(t *testing.T, cons Constructor) { // Use-case: move to maintenance mode twice, first time failed. s := cons(t) - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Init()) require.NoError(t, s.Close()) require.NoError(t, s.Close()) // already closed, no-op @@ -90,12 +90,12 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { // Use-case: metabase `Init` failed, // call `SetMode` on all not-yet-initialized components. s := cons(t) - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.SetMode(m)) t.Run("after open in RO", func(t *testing.T) { require.NoError(t, s.Close()) - require.NoError(t, s.Open(context.Background(), true)) + require.NoError(t, s.Open(context.Background(), mode.ReadOnly)) require.NoError(t, s.SetMode(m)) }) @@ -104,7 +104,7 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { t.Run("after init", func(t *testing.T) { s := cons(t) // Use-case: notmal node operation. - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Init()) require.NoError(t, s.SetMode(m)) require.NoError(t, s.Close()) @@ -114,7 +114,7 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { func TestModeTransition(t *testing.T, cons Constructor, from, to mode.Mode) { // Use-case: normal node operation. s := cons(t) - require.NoError(t, s.Open(context.Background(), false)) + require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Init()) require.NoError(t, s.SetMode(from)) require.NoError(t, s.SetMode(to)) diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 00fe1e02b..c1a86819d 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -22,7 +22,18 @@ var ErrDegradedMode = logicerr.New("metabase is in a degraded mode") var ErrReadOnlyMode = logicerr.New("metabase is in a read-only mode") // Open boltDB instance for metabase. -func (db *DB) Open(_ context.Context, readOnly bool) error { +func (db *DB) Open(_ context.Context, mode mode.Mode) error { + db.modeMtx.Lock() + defer db.modeMtx.Unlock() + db.mode = mode + + if mode.NoMetabase() { + return nil + } + return db.openDB(mode) +} + +func (db *DB) openDB(mode mode.Mode) error { err := util.MkdirAllX(filepath.Dir(db.info.Path), db.info.Permission) if err != nil { return fmt.Errorf("can't create dir %s for metabase: %w", db.info.Path, err) @@ -34,7 +45,7 @@ func (db *DB) Open(_ context.Context, readOnly bool) error { opts := *bbolt.DefaultOptions db.boltOptions = &opts } - db.boltOptions.ReadOnly = readOnly + db.boltOptions.ReadOnly = mode.ReadOnly() return metaerr.Wrap(db.openBolt()) } diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index fc20e2aad..01e1ed2bc 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -9,6 +9,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -49,7 +50,7 @@ func newDB(t testing.TB, opts ...meta.Option) *meta.DB { }, opts...)..., ) - require.NoError(t, bdb.Open(context.Background(), false)) + require.NoError(t, bdb.Open(context.Background(), mode.ReadWrite)) require.NoError(t, bdb.Init()) return bdb diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index a18095f3e..b382e99c2 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -1,7 +1,6 @@ package meta import ( - "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -23,21 +22,16 @@ func (db *DB) SetMode(m mode.Mode) error { } } - var err error - switch { - case m.NoMetabase(): + if m.NoMetabase() { db.boltDB = nil - case m.ReadOnly(): - err = db.Open(context.TODO(), true) - default: - err = db.Open(context.TODO(), false) - } - if err == nil && !m.NoMetabase() && !m.ReadOnly() { - err = db.Init() - } - - if err != nil { - return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err) + } else { + err := db.openDB(m) + if err == nil && !m.ReadOnly() { + err = db.Init() + } + if err != nil { + return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err) + } } db.mode = m diff --git a/pkg/local_object_storage/metabase/mode_test.go b/pkg/local_object_storage/metabase/mode_test.go new file mode 100644 index 000000000..1b9f60055 --- /dev/null +++ b/pkg/local_object_storage/metabase/mode_test.go @@ -0,0 +1,37 @@ +package meta + +import ( + "context" + "path/filepath" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "github.com/stretchr/testify/require" +) + +type epochStateTest struct{} + +func (s epochStateTest) CurrentEpoch() uint64 { + return 0 +} + +func Test_Mode(t *testing.T) { + t.Parallel() + bdb := New([]Option{ + WithPath(filepath.Join(t.TempDir(), "metabase")), + WithPermissions(0o600), + WithEpochState(epochStateTest{}), + }...) + + require.NoError(t, bdb.Open(context.Background(), mode.DegradedReadOnly)) + require.Nil(t, bdb.boltDB) + require.NoError(t, bdb.Init()) + require.Nil(t, bdb.boltDB) + require.NoError(t, bdb.Close()) + + require.NoError(t, bdb.Open(context.Background(), mode.Degraded)) + require.Nil(t, bdb.boltDB) + require.NoError(t, bdb.Init()) + require.Nil(t, bdb.boltDB) + require.NoError(t, bdb.Close()) +} diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index ddeddd189..cea42da0f 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" ) @@ -43,13 +44,13 @@ func TestVersion(t *testing.T) { } t.Run("simple", func(t *testing.T) { db := newDB(t) - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Init()) check(t, db) require.NoError(t, db.Close()) t.Run("reopen", func(t *testing.T) { - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Init()) check(t, db) require.NoError(t, db.Close()) @@ -57,29 +58,29 @@ func TestVersion(t *testing.T) { }) t.Run("old data", func(t *testing.T) { db := newDB(t) - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.WriteShardID([]byte{1, 2, 3, 4})) require.NoError(t, db.Close()) - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Init()) check(t, db) require.NoError(t, db.Close()) }) t.Run("invalid version", func(t *testing.T) { db := newDB(t) - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.boltDB.Update(func(tx *bbolt.Tx) error { return updateVersion(tx, version+1) })) require.NoError(t, db.Close()) - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.Error(t, db.Init()) require.NoError(t, db.Close()) t.Run("reset", func(t *testing.T) { - require.NoError(t, db.Open(context.Background(), false)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Reset()) check(t, db) require.NoError(t, db.Close()) diff --git a/pkg/local_object_storage/pilorama/bench_test.go b/pkg/local_object_storage/pilorama/bench_test.go index e3f3afd99..22b951a41 100644 --- a/pkg/local_object_storage/pilorama/bench_test.go +++ b/pkg/local_object_storage/pilorama/bench_test.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" ) @@ -26,7 +27,7 @@ func BenchmarkCreate(b *testing.B) { f := NewBoltForest( WithPath(filepath.Join(tmpDir, "test.db")), WithMaxBatchSize(runtime.GOMAXPROCS(0))) - require.NoError(b, f.Open(context.Background(), false)) + require.NoError(b, f.Open(context.Background(), mode.ReadWrite)) require.NoError(b, f.Init()) defer func() { require.NoError(b, f.Close()) }() diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 61318fffc..17133e8f3 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -99,7 +99,7 @@ func (t *boltForest) SetMode(m mode.Mode) error { err := t.Close() if err == nil && !m.NoMetabase() { - if err = t.Open(context.TODO(), m.ReadOnly()); err == nil { + if err = t.openBolt(m); err == nil { err = t.Init() } } @@ -112,7 +112,18 @@ func (t *boltForest) SetMode(m mode.Mode) error { return nil } -func (t *boltForest) Open(_ context.Context, readOnly bool) error { +func (t *boltForest) Open(_ context.Context, mode mode.Mode) error { + t.modeMtx.Lock() + defer t.modeMtx.Unlock() + t.mode = mode + if mode.NoMetabase() { + return nil + } + return t.openBolt(mode) +} + +func (t *boltForest) openBolt(mode mode.Mode) error { + readOnly := mode.ReadOnly() err := util.MkdirAllX(filepath.Dir(t.path), t.perm) if err != nil { return metaerr.Wrap(fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)) @@ -131,11 +142,7 @@ func (t *boltForest) Open(_ context.Context, readOnly bool) error { t.db.MaxBatchSize = t.maxBatchSize t.db.MaxBatchDelay = t.maxBatchDelay - m := mode.ReadWrite - if readOnly { - m = mode.ReadOnly - } - t.metrics.SetMode(m) + t.metrics.SetMode(mode) return nil } diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index ab6174002..8276490b3 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -112,7 +112,7 @@ func (f *memoryForest) Init() error { return nil } -func (f *memoryForest) Open(context.Context, bool) error { +func (f *memoryForest) Open(context.Context, mode.Mode) error { return nil } diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 2e7c1f529..383320eb9 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -24,7 +25,7 @@ var providers = []struct { }{ {"inmemory", func(t testing.TB, _ ...Option) ForestStorage { f := NewMemoryForest() - require.NoError(t, f.Open(context.Background(), false)) + require.NoError(t, f.Open(context.Background(), mode.ReadWrite)) require.NoError(t, f.Init()) return f }}, @@ -34,7 +35,7 @@ var providers = []struct { WithPath(filepath.Join(t.TempDir(), "test.db")), WithMaxBatchSize(1), }, opts...)...) - require.NoError(t, f.Open(context.Background(), false)) + require.NoError(t, f.Open(context.Background(), mode.ReadWrite)) require.NoError(t, f.Init()) return f }}, diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index b3391407c..5fefb2e27 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -58,7 +58,7 @@ type ForestStorage interface { // DumpInfo returns information about the pilorama. DumpInfo() Info Init() error - Open(context.Context, bool) error + Open(context.Context, mode.Mode) error Close() error SetMode(m mode.Mode) error SetParentID(id string) diff --git a/pkg/local_object_storage/pilorama/mode_test.go b/pkg/local_object_storage/pilorama/mode_test.go new file mode 100644 index 000000000..01d3da9f0 --- /dev/null +++ b/pkg/local_object_storage/pilorama/mode_test.go @@ -0,0 +1,31 @@ +package pilorama + +import ( + "context" + "path/filepath" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "github.com/stretchr/testify/require" +) + +func Test_Mode(t *testing.T) { + t.Parallel() + f := NewBoltForest( + []Option{ + WithPath(filepath.Join(t.TempDir(), "test.db")), + WithMaxBatchSize(1), + }...) + + require.NoError(t, f.Open(context.Background(), mode.DegradedReadOnly)) + require.Nil(t, f.(*boltForest).db) + require.NoError(t, f.Init()) + require.Nil(t, f.(*boltForest).db) + require.NoError(t, f.Close()) + + require.NoError(t, f.Open(context.Background(), mode.Degraded)) + require.Nil(t, f.(*boltForest).db) + require.NoError(t, f.Init()) + require.Nil(t, f.(*boltForest).db) + require.NoError(t, f.Close()) +} diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 574bc1430..8d1007ffd 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -43,12 +43,17 @@ func (s *Shard) handleMetabaseFailure(stage string, err error) error { // Open opens all Shard's components. func (s *Shard) Open(ctx context.Context) error { components := []interface { - Open(context.Context, bool) error + Open(context.Context, mode.Mode) error }{ - s.blobStor, s.metaBase, + s.blobStor, + } + m := s.GetMode() + + if !m.NoMetabase() { + components = append(components, s.metaBase) } - if s.hasWriteCache() { + if s.hasWriteCache() && !m.NoMetabase() { components = append(components, s.writeCache) } @@ -57,12 +62,12 @@ func (s *Shard) Open(ctx context.Context) error { } for i, component := range components { - if err := component.Open(ctx, false); err != nil { + if err := component.Open(ctx, m); err != nil { if component == s.metaBase { // We must first open all other components to avoid // opening non-existent DB in read-only mode. for j := i + 1; j < len(components); j++ { - if err := components[j].Open(ctx, false); err != nil { + if err := components[j].Open(ctx, m); err != nil { // Other components must be opened, fail. return fmt.Errorf("could not open %T: %w", components[j], err) } @@ -97,8 +102,9 @@ func (s *Shard) Init(ctx context.Context) error { } var components []initializer + m := s.GetMode() - if !s.GetMode().NoMetabase() { + if !m.NoMetabase() { var initMetabase initializer if s.NeedRefillMetabase() { @@ -114,7 +120,7 @@ func (s *Shard) Init(ctx context.Context) error { components = []initializer{s.blobStor} } - if s.hasWriteCache() { + if s.hasWriteCache() && !m.NoMetabase() { components = append(components, s.writeCache) } diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index e3c209907..b67fd59cd 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -31,12 +31,11 @@ func (s *Shard) ID() *ID { // UpdateID reads shard ID saved in the metabase and updates it if it is missing. func (s *Shard) UpdateID(ctx context.Context) (err error) { - var metabaseOpened bool var idFromMetabase []byte - if err = s.metaBase.Open(ctx, false); err != nil { + metabaseOpened := !s.GetMode().NoMetabase() + if err = s.metaBase.Open(ctx, s.GetMode()); err != nil { err = fmt.Errorf("failed to open metabase: %w", err) - } else { - metabaseOpened = true + metabaseOpened = false } if metabaseOpened { defer func() { diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index f7895a935..c1c0e88b3 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "github.com/stretchr/testify/require" ) @@ -75,7 +76,7 @@ func benchmarkPutPar(b *testing.B, cache writecache.Cache, size uint64) { } func benchmarkPutPrepare(b *testing.B, cache writecache.Cache) { - require.NoError(b, cache.Open(context.Background(), false), "opening") + require.NoError(b, cache.Open(context.Background(), mode.ReadWrite), "opening") require.NoError(b, cache.Init(), "initializing") } diff --git a/pkg/local_object_storage/writecache/cachebbolt.go b/pkg/local_object_storage/writecache/cachebbolt.go index 5eeda33a6..2ae62c461 100644 --- a/pkg/local_object_storage/writecache/cachebbolt.go +++ b/pkg/local_object_storage/writecache/cachebbolt.go @@ -95,8 +95,14 @@ func (c *cache) DumpInfo() Info { } // Open opens and initializes database. Reads object counters from the ObjectCounters instance. -func (c *cache) Open(_ context.Context, readOnly bool) error { - err := c.openStore(readOnly) +func (c *cache) Open(_ context.Context, mode mode.Mode) error { + c.modeMtx.Lock() + defer c.modeMtx.Unlock() + c.mode = mode + if mode.NoMetabase() { + return nil + } + err := c.openStore(mode.ReadOnly()) if err != nil { return metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index bf68765d8..3c951bebe 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -202,7 +202,7 @@ func newCache[Option any]( mb := meta.New( meta.WithPath(filepath.Join(dir, "meta")), meta.WithEpochState(dummyEpoch{})) - require.NoError(t, mb.Open(context.Background(), false)) + require.NoError(t, mb.Open(context.Background(), mode.ReadWrite)) require.NoError(t, mb.Init()) bs := blobstor.New(blobstor.WithStorages([]blobstor.SubStorage{ @@ -213,11 +213,11 @@ func newCache[Option any]( fstree.WithDirNameLen(1)), }, })) - require.NoError(t, bs.Open(context.Background(), false)) + require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) require.NoError(t, bs.Init()) wc := createCacheFn(t, smallSize, mb, bs, opts...) - require.NoError(t, wc.Open(context.Background(), false)) + require.NoError(t, wc.Open(context.Background(), mode.ReadWrite)) require.NoError(t, wc.Init()) // First set mode for metabase and blobstor to prevent background flushes. diff --git a/pkg/local_object_storage/writecache/mode_test.go b/pkg/local_object_storage/writecache/mode_test.go new file mode 100644 index 000000000..f684c15bc --- /dev/null +++ b/pkg/local_object_storage/writecache/mode_test.go @@ -0,0 +1,30 @@ +package writecache + +import ( + "context" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "github.com/stretchr/testify/require" +) + +func TestMode(t *testing.T) { + t.Parallel() + wc := New( + WithLogger(test.NewLogger(t)), + WithFlushWorkersCount(2), + WithPath(t.TempDir())) + + require.NoError(t, wc.Open(context.Background(), mode.DegradedReadOnly)) + require.Nil(t, wc.(*cache).db) + require.NoError(t, wc.Init()) + require.Nil(t, wc.(*cache).db) + require.NoError(t, wc.Close()) + + require.NoError(t, wc.Open(context.Background(), mode.Degraded)) + require.Nil(t, wc.(*cache).db) + require.NoError(t, wc.Init()) + require.Nil(t, wc.(*cache).db) + require.NoError(t, wc.Close()) +} diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 99fc5390a..69dfee953 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -39,7 +39,7 @@ type Cache interface { Seal(context.Context, bool) error Init() error - Open(ctx context.Context, readOnly bool) error + Open(ctx context.Context, mode mode.Mode) error Close() error } From 9a622a750dfbe159332ddf4869ab3e21e467a0fc Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 1 Feb 2024 10:26:29 +0300 Subject: [PATCH 0868/1943] [#970] fstree: Move temporary path handling in a separate function Allow to easier test different implementations. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobstor/fstree/fstree.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index af241becf..3da5568e2 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -357,9 +357,12 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err } size = len(prm.RawData) + return common.PutRes{StorageID: []byte{}}, t.writeData(p, prm.RawData) +} + +func (t *FSTree) writeData(p string, data []byte) error { tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10) - err = t.writeAndRename(tmpPath, p, prm.RawData) - return common.PutRes{StorageID: []byte{}}, err + return t.writeAndRename(tmpPath, p, data) } // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. From ff488b53a11be2e91bdf821e87d0ec6c70686ca8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 1 Feb 2024 10:28:31 +0300 Subject: [PATCH 0869/1943] [#970] fstree: Move write functions to a separate file Signed-off-by: Evgenii Stratonikov --- .../blobstor/fstree/fstree.go | 63 ---------------- .../blobstor/fstree/fstree_write_generic.go | 74 +++++++++++++++++++ 2 files changed, 74 insertions(+), 63 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 3da5568e2..7bb8db5da 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -360,69 +360,6 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err return common.PutRes{StorageID: []byte{}}, t.writeData(p, prm.RawData) } -func (t *FSTree) writeData(p string, data []byte) error { - tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10) - return t.writeAndRename(tmpPath, p, data) -} - -// writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. -func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { - if t.fileCounterEnabled { - t.fileGuard.Lock(p) - defer t.fileGuard.Unlock(p) - } - - err := t.writeFile(tmpPath, data) - if err != nil { - var pe *fs.PathError - if errors.As(err, &pe) { - switch pe.Err { - case syscall.ENOSPC: - err = common.ErrNoSpace - _ = os.RemoveAll(tmpPath) - } - } - return err - } - - if t.fileCounterEnabled { - t.fileCounter.Inc() - var targetFileExists bool - if _, e := os.Stat(p); e == nil { - targetFileExists = true - } - err = os.Rename(tmpPath, p) - if err == nil && targetFileExists { - t.fileCounter.Dec() - } - } else { - err = os.Rename(tmpPath, p) - } - return err -} - -func (t *FSTree) writeFlags() int { - flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL - if t.noSync { - return flags - } - return flags | os.O_SYNC -} - -// writeFile writes data to a file with path p. -// The code is copied from `os.WriteFile` with minor corrections for flags. -func (t *FSTree) writeFile(p string, data []byte) error { - f, err := os.OpenFile(p, t.writeFlags(), t.Permissions) - if err != nil { - return err - } - _, err = f.Write(data) - if err1 := f.Close(); err1 != nil && err == nil { - err = err1 - } - return err -} - // Get returns an object from the storage by address. func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { var ( diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go new file mode 100644 index 000000000..411914034 --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -0,0 +1,74 @@ +package fstree + +import ( + "errors" + "io/fs" + "os" + "strconv" + "syscall" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" +) + +func (t *FSTree) writeData(p string, data []byte) error { + tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10) + return t.writeAndRename(tmpPath, p, data) +} + +// writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. +func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { + if t.fileCounterEnabled { + t.fileGuard.Lock(p) + defer t.fileGuard.Unlock(p) + } + + err := t.writeFile(tmpPath, data) + if err != nil { + var pe *fs.PathError + if errors.As(err, &pe) { + switch pe.Err { + case syscall.ENOSPC: + err = common.ErrNoSpace + _ = os.RemoveAll(tmpPath) + } + } + return err + } + + if t.fileCounterEnabled { + t.fileCounter.Inc() + var targetFileExists bool + if _, e := os.Stat(p); e == nil { + targetFileExists = true + } + err = os.Rename(tmpPath, p) + if err == nil && targetFileExists { + t.fileCounter.Dec() + } + } else { + err = os.Rename(tmpPath, p) + } + return err +} + +func (t *FSTree) writeFlags() int { + flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL + if t.noSync { + return flags + } + return flags | os.O_SYNC +} + +// writeFile writes data to a file with path p. +// The code is copied from `os.WriteFile` with minor corrections for flags. +func (t *FSTree) writeFile(p string, data []byte) error { + f, err := os.OpenFile(p, t.writeFlags(), t.Permissions) + if err != nil { + return err + } + _, err = f.Write(data) + if err1 := f.Close(); err1 != nil && err == nil { + err = err1 + } + return err +} From fc31b9c947bf7185ec252e2e63d615a6c59b16ab Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Sat, 9 Sep 2023 11:16:06 +0300 Subject: [PATCH 0870/1943] [#970] fstree: Add linux-specific file writer using O_TMPFILE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit O_TMPFILE is implemented for all modern FSes and it's much easier and safer to use. If application crashes in the middle of writing this file would be gone and won't leave any garbage. Notice that this implementation makes a different choice wrt EEXIST handling, generic one always overwrites, while this one keeps the old data. There is no real performance difference. SSD (slow&old), XFS, Core i7-8565U: Sync ``` name old time/op new time/op delta Put/size=1024,thread=1/fstree-8 1.74ms ± 3% 0.06ms ± 7% -96.31% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 10.0ms ±41% 1.1ms ±18% -88.95% (p=0.000 n=9+10) Put/size=1024,thread=100/fstree-8 32.3ms ±60% 6.5ms ±14% -79.97% (p=0.000 n=10+10) Put/size=1048576,thread=1/fstree-8 17.8ms ±90% 3.4ms ±70% -81.08% (p=0.000 n=10+10) Put/size=1048576,thread=20/fstree-8 103ms ±174% 112ms ±158% ~ (p=0.971 n=10+10) Put/size=1048576,thread=100/fstree-8 949ms ±78% 583ms ±132% ~ (p=0.089 n=10+10) name old alloc/op new alloc/op delta Put/size=1024,thread=1/fstree-8 3.17kB ± 1% 1.96kB ± 0% -38.09% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 59.6kB ± 1% 39.2kB ± 1% -34.30% (p=0.000 n=8+10) Put/size=1024,thread=100/fstree-8 299kB ± 0% 198kB ± 0% -33.90% (p=0.000 n=7+9) Put/size=1048576,thread=1/fstree-8 3.38kB ± 1% 2.36kB ± 1% -30.22% (p=0.000 n=10+10) Put/size=1048576,thread=20/fstree-8 65.7kB ± 4% 47.7kB ± 6% -27.27% (p=0.000 n=10+10) Put/size=1048576,thread=100/fstree-8 351kB ± 8% 245kB ± 8% -30.22% (p=0.000 n=10+10) name old allocs/op new allocs/op delta Put/size=1024,thread=1/fstree-8 30.3 ± 2% 21.0 ± 0% -30.69% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 554 ± 1% 413 ± 0% -25.35% (p=0.000 n=8+10) Put/size=1024,thread=100/fstree-8 2.77k ± 0% 2.07k ± 0% -25.27% (p=0.000 n=7+10) Put/size=1048576,thread=1/fstree-8 32.0 ± 0% 25.0 ± 0% -21.88% (p=0.000 n=9+8) Put/size=1048576,thread=20/fstree-8 609 ± 5% 494 ± 6% -18.93% (p=0.000 n=10+10) Put/size=1048576,thread=100/fstree-8 3.25k ± 9% 2.50k ± 8% -23.21% (p=0.000 n=10+10) ``` No sync ``` name old time/op new time/op delta Put/size=1024,thread=1/fstree-8 71.3µs ±10% 59.8µs ±10% -16.21% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 1.43ms ± 6% 1.22ms ±13% -14.53% (p=0.000 n=10+10) Put/size=1024,thread=100/fstree-8 8.12ms ± 3% 6.36ms ± 2% -21.67% (p=0.000 n=8+9) Put/size=1048576,thread=1/fstree-8 1.88ms ±70% 1.61ms ±78% ~ (p=0.393 n=10+10) Put/size=1048576,thread=20/fstree-8 32.7ms ±28% 34.2ms ±112% ~ (p=0.968 n=9+10) Put/size=1048576,thread=100/fstree-8 262ms ±56% 226ms ±34% ~ (p=0.447 n=10+9) name old alloc/op new alloc/op delta Put/size=1024,thread=1/fstree-8 2.89kB ± 0% 1.96kB ± 0% -32.28% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 58.2kB ± 0% 39.5kB ± 0% -32.09% (p=0.000 n=8+8) Put/size=1024,thread=100/fstree-8 291kB ± 0% 198kB ± 0% -32.19% (p=0.000 n=9+9) Put/size=1048576,thread=1/fstree-8 3.05kB ± 1% 2.13kB ± 1% -30.16% (p=0.000 n=10+9) Put/size=1048576,thread=20/fstree-8 62.6kB ± 0% 44.3kB ± 0% -29.23% (p=0.000 n=9+9) Put/size=1048576,thread=100/fstree-8 302kB ± 0% 210kB ± 1% -30.39% (p=0.000 n=9+9) name old allocs/op new allocs/op delta Put/size=1024,thread=1/fstree-8 27.0 ± 0% 21.0 ± 0% -22.22% (p=0.000 n=10+10) Put/size=1024,thread=20/fstree-8 539 ± 0% 415 ± 0% -22.98% (p=0.000 n=10+10) Put/size=1024,thread=100/fstree-8 2.69k ± 0% 2.07k ± 0% -23.09% (p=0.000 n=9+9) Put/size=1048576,thread=1/fstree-8 28.0 ± 0% 22.3 ± 3% -20.36% (p=0.000 n=8+10) Put/size=1048576,thread=20/fstree-8 577 ± 0% 458 ± 0% -20.72% (p=0.000 n=9+9) Put/size=1048576,thread=100/fstree-8 2.76k ± 0% 2.15k ± 0% -22.05% (p=0.000 n=9+8) ``` HDD (LVM), ext4, Ryzen 5 1600: Sync ``` │ fs.sync-generic │ fs.sync-linux │ │ sec/op │ sec/op vs base │ Put/size=1024,thread=1/fstree-12 34.70m ± 19% 33.59m ± 16% ~ (p=0.529 n=10) Put/size=1024,thread=20/fstree-12 188.8m ± 8% 189.2m ± 16% ~ (p=0.739 n=10) Put/size=1024,thread=100/fstree-12 264.8m ± 22% 273.6m ± 28% ~ (p=0.353 n=10) Put/size=1048576,thread=1/fstree-12 54.90m ± 14% 47.08m ± 18% ~ (p=0.063 n=10) Put/size=1048576,thread=20/fstree-12 244.1m ± 14% 220.4m ± 22% ~ (p=0.579 n=10) Put/size=1048576,thread=100/fstree-12 847.2m ± 5% 893.6m ± 3% +5.48% (p=0.000 n=10) geomean 164.3m 158.9m -3.29% │ fs.sync-generic │ fs.sync-linux │ │ B/op │ B/op vs base │ Put/size=1024,thread=1/fstree-12 3.375Ki ± 1% 2.471Ki ± 1% -26.80% (p=0.000 n=10) Put/size=1024,thread=20/fstree-12 66.62Ki ± 6% 49.21Ki ± 6% -26.15% (p=0.000 n=10) Put/size=1024,thread=100/fstree-12 319.2Ki ± 1% 230.9Ki ± 2% -27.64% (p=0.000 n=10) Put/size=1048576,thread=1/fstree-12 3.457Ki ± 1% 2.559Ki ± 1% -25.97% (p=0.000 n=10) Put/size=1048576,thread=20/fstree-12 66.91Ki ± 1% 49.16Ki ± 1% -26.52% (p=0.000 n=10) Put/size=1048576,thread=100/fstree-12 338.8Ki ± 2% 252.3Ki ± 3% -25.54% (p=0.000 n=10) geomean 42.17Ki 31.02Ki -26.44% │ fs.sync-generic │ fs.sync-linux │ │ allocs/op │ allocs/op vs base │ Put/size=1024,thread=1/fstree-12 33.00 ± 0% 27.00 ± 0% -18.18% (p=0.000 n=10) Put/size=1024,thread=20/fstree-12 639.5 ± 1% 519.0 ± 2% -18.84% (p=0.000 n=10) Put/size=1024,thread=100/fstree-12 3.059k ± 1% 2.478k ± 2% -18.99% (p=0.000 n=10) Put/size=1048576,thread=1/fstree-12 33.50 ± 1% 28.00 ± 4% -16.42% (p=0.000 n=10) Put/size=1048576,thread=20/fstree-12 638.5 ± 1% 520.0 ± 1% -18.56% (p=0.000 n=10) Put/size=1048576,thread=100/fstree-12 3.209k ± 2% 2.655k ± 2% -17.28% (p=0.000 n=10) geomean 405.3 332.1 -18.05% ``` No sync ``` │ fs.nosync-generic │ fs.nosync-linux │ │ sec/op │ sec/op vs base │ Put/size=1024,thread=1/fstree-12 148.2µ ± 20% 136.6µ ± 19% -7.89% (p=0.029 n=10) Put/size=1024,thread=20/fstree-12 1.140m ± 26% 1.364m ± 16% ~ (p=0.143 n=10) Put/size=1024,thread=100/fstree-12 11.93m ± 68% 26.89m ± 62% ~ (p=0.123 n=10) Put/size=1048576,thread=1/fstree-12 1.302m ± 3% 1.287m ± 5% ~ (p=0.481 n=10) Put/size=1048576,thread=20/fstree-12 77.52m ± 8% 74.07m ± 7% ~ (p=0.278 n=10+9) Put/size=1048576,thread=100/fstree-12 226.1m ± ∞ ¹ geomean 5.986m 3.434m +18.60% ² ¹ need >= 6 samples for confidence interval at level 0.95 ² benchmark set differs from baseline; geomeans may not be comparable │ fs.nosync-generic │ fs.nosync-linux │ │ B/op │ B/op vs base │ Put/size=1024,thread=1/fstree-12 2.879Ki ± 0% 1.972Ki ± 0% -31.51% (p=0.000 n=10) Put/size=1024,thread=20/fstree-12 55.94Ki ± 1% 37.90Ki ± 1% -32.25% (p=0.000 n=10) Put/size=1024,thread=100/fstree-12 272.6Ki ± 0% 182.1Ki ± 9% -33.21% (p=0.000 n=10) Put/size=1048576,thread=1/fstree-12 3.158Ki ± 0% 2.259Ki ± 0% -28.46% (p=0.000 n=10) Put/size=1048576,thread=20/fstree-12 58.87Ki ± 0% 41.03Ki ± 0% -30.30% (p=0.000 n=10+9) Put/size=1048576,thread=100/fstree-12 299.8Ki ± ∞ ¹ geomean 36.71Ki 16.60Ki -31.17% ² ¹ need >= 6 samples for confidence interval at level 0.95 ² benchmark set differs from baseline; geomeans may not be comparable │ fs.nosync-generic │ fs.nosync-linux │ │ allocs/op │ allocs/op vs base │ Put/size=1024,thread=1/fstree-12 28.00 ± 0% 22.00 ± 0% -21.43% (p=0.000 n=10) Put/size=1024,thread=20/fstree-12 530.0 ± 0% 407.5 ± 1% -23.11% (p=0.000 n=10) Put/size=1024,thread=100/fstree-12 2.567k ± 0% 1.956k ± 9% -23.77% (p=0.000 n=10) Put/size=1048576,thread=1/fstree-12 30.00 ± 0% 24.00 ± 0% -20.00% (p=0.000 n=10) Put/size=1048576,thread=20/fstree-12 553.5 ± 0% 434.0 ± 0% -21.59% (n=10+9) Put/size=1048576,thread=100/fstree-12 2.803k ± ∞ ¹ geomean 347.9 178.8 -21.99% ² ¹ need >= 6 samples for confidence interval at level 0.95 ² benchmark set differs from baseline; geomeans may not be comparable ``` Signed-off-by: Roman Khimov Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- .../blobstor/fstree/fstree_write_generic.go | 2 + .../blobstor/fstree/fstree_write_linux.go | 48 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go diff --git a/go.mod b/go.mod index 5d6edc222..06d2d3299 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,7 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/sync v0.6.0 + golang.org/x/sys v0.16.0 golang.org/x/term v0.16.0 google.golang.org/grpc v1.61.0 google.golang.org/protobuf v1.32.0 @@ -120,7 +121,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.18.0 // indirect golang.org/x/net v0.20.0 // indirect - golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 411914034..0882be0dd 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -1,3 +1,5 @@ +//go:build !linux + package fstree import ( diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go new file mode 100644 index 000000000..edb7f0dce --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -0,0 +1,48 @@ +//go:build linux + +package fstree + +import ( + "errors" + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "golang.org/x/sys/unix" +) + +func (t *FSTree) writeData(p string, data []byte) error { + err := t.writeFile(p, data) + if errors.Is(err, unix.ENOSPC) { + return common.ErrNoSpace + } + return err +} + +func (t *FSTree) writeFile(p string, data []byte) error { + flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC + if !t.noSync { + flags |= unix.O_DSYNC + } + fd, err := unix.Open(t.RootPath, flags, uint32(t.Permissions)) + if err != nil { + return err + } + tmpPath := "/proc/self/fd/" + strconv.FormatUint(uint64(fd), 10) + n, err := unix.Write(fd, data) + if err == nil { + if n == len(data) { + err = unix.Linkat(unix.AT_FDCWD, tmpPath, unix.AT_FDCWD, p, unix.AT_SYMLINK_FOLLOW) + if errors.Is(err, unix.EEXIST) { + // https://github.com/nspcc-dev/neofs-node/issues/2563 + err = nil + } + } else { + err = errors.New("incomplete write") + } + } + errClose := unix.Close(fd) + if err != nil { + return err // Close() error is ignored, we have a better one. + } + return errClose +} From 7f692409cf1a25167446f9094d8ba7b8bd678f8e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 11:57:23 +0300 Subject: [PATCH 0871/1943] [#970] fstree: Handle unsupported O_TMPFILE Metabase test relied on this behaviour, so fix the test too. Cherry-picking was hard and did too many conflicts, here is an original PR: https://github.com/nspcc-dev/neofs-node/pull/2624 Signed-off-by: Evgenii Stratonikov --- .../blobstor/fstree/control.go | 7 +++ .../blobstor/fstree/fstree.go | 4 +- .../blobstor/fstree/fstree_write_generic.go | 58 +++++++++++-------- .../blobstor/fstree/fstree_write_linux.go | 46 ++++++++++++--- .../blobstor/fstree/fstree_write_specific.go | 9 +++ .../shard/control_test.go | 10 +++- 6 files changed, 97 insertions(+), 37 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index c56312d38..b126f8d29 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -16,6 +16,13 @@ func (t *FSTree) Init() error { if err := util.MkdirAllX(t.RootPath, t.Permissions); err != nil { return err } + if !t.readOnly { + f := newSpecificWriteData(t.fileCounter, t.RootPath, t.Permissions, t.noSync) + if f != nil { + t.writeData = f + } + } + return t.initFileCounter() } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 7bb8db5da..fc29d4eaa 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -10,7 +10,6 @@ import ( "path/filepath" "strconv" "strings" - "sync/atomic" "syscall" "time" @@ -58,7 +57,7 @@ type FSTree struct { fileCounter FileCounter fileCounterEnabled bool - suffix atomic.Uint64 + writeData func(string, []byte) error } // Info groups the information about file storage. @@ -96,6 +95,7 @@ func New(opts ...Option) *FSTree { for i := range opts { opts[i](f) } + f.writeData = newGenericWriteData(f) return f } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 0882be0dd..3d65808ec 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -1,5 +1,3 @@ -//go:build !linux - package fstree import ( @@ -7,24 +5,46 @@ import ( "io/fs" "os" "strconv" + "sync/atomic" "syscall" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" ) -func (t *FSTree) writeData(p string, data []byte) error { - tmpPath := p + "#" + strconv.FormatUint(t.suffix.Add(1), 10) - return t.writeAndRename(tmpPath, p, data) +type genericWriter struct { + perm fs.FileMode + flags int + + t *FSTree + suffix atomic.Uint64 +} + +func newGenericWriteData(t *FSTree) func(string, []byte) error { + flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL + if !t.noSync { + flags |= os.O_SYNC + } + var w = &genericWriter{ + perm: t.Permissions, + flags: flags, + t: t, + } + return w.writeData +} + +func (w *genericWriter) writeData(p string, data []byte) error { + tmpPath := p + "#" + strconv.FormatUint(w.suffix.Add(1), 10) + return w.writeAndRename(tmpPath, p, data) } // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. -func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { - if t.fileCounterEnabled { - t.fileGuard.Lock(p) - defer t.fileGuard.Unlock(p) +func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error { + if w.t.fileCounterEnabled { + w.t.fileGuard.Lock(p) + defer w.t.fileGuard.Unlock(p) } - err := t.writeFile(tmpPath, data) + err := w.writeFile(tmpPath, data) if err != nil { var pe *fs.PathError if errors.As(err, &pe) { @@ -37,15 +57,15 @@ func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { return err } - if t.fileCounterEnabled { - t.fileCounter.Inc() + if w.t.fileCounterEnabled { + w.t.fileCounter.Inc() var targetFileExists bool if _, e := os.Stat(p); e == nil { targetFileExists = true } err = os.Rename(tmpPath, p) if err == nil && targetFileExists { - t.fileCounter.Dec() + w.t.fileCounter.Dec() } } else { err = os.Rename(tmpPath, p) @@ -53,18 +73,10 @@ func (t *FSTree) writeAndRename(tmpPath, p string, data []byte) error { return err } -func (t *FSTree) writeFlags() int { - flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL - if t.noSync { - return flags - } - return flags | os.O_SYNC -} - // writeFile writes data to a file with path p. // The code is copied from `os.WriteFile` with minor corrections for flags. -func (t *FSTree) writeFile(p string, data []byte) error { - f, err := os.OpenFile(p, t.writeFlags(), t.Permissions) +func (w *genericWriter) writeFile(p string, data []byte) error { + f, err := os.OpenFile(p, w.flags, w.perm) if err != nil { return err } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go index edb7f0dce..3416cf7ac 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -4,26 +4,52 @@ package fstree import ( "errors" + "io/fs" "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "golang.org/x/sys/unix" ) -func (t *FSTree) writeData(p string, data []byte) error { - err := t.writeFile(p, data) +type linuxWriter struct { + root string + perm uint32 + flags int + + counter FileCounter +} + +func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync bool) func(string, []byte) error { + flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC + if !noSync { + flags |= unix.O_DSYNC + } + fd, err := unix.Open(root, flags, uint32(perm)) + if err != nil { + // Which means that OS-specific writeData can't be created + // and FSTree should use the generic one. + return nil + } + _ = unix.Close(fd) // Don't care about error. + w := &linuxWriter{ + root: root, + perm: uint32(perm), + flags: flags, + counter: c, + } + return w.writeData +} + +func (w *linuxWriter) writeData(p string, data []byte) error { + err := w.writeFile(p, data) if errors.Is(err, unix.ENOSPC) { return common.ErrNoSpace } return err } -func (t *FSTree) writeFile(p string, data []byte) error { - flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC - if !t.noSync { - flags |= unix.O_DSYNC - } - fd, err := unix.Open(t.RootPath, flags, uint32(t.Permissions)) +func (w *linuxWriter) writeFile(p string, data []byte) error { + fd, err := unix.Open(w.root, w.flags, w.perm) if err != nil { return err } @@ -32,8 +58,10 @@ func (t *FSTree) writeFile(p string, data []byte) error { if err == nil { if n == len(data) { err = unix.Linkat(unix.AT_FDCWD, tmpPath, unix.AT_FDCWD, p, unix.AT_SYMLINK_FOLLOW) + if err == nil { + w.counter.Inc() + } if errors.Is(err, unix.EEXIST) { - // https://github.com/nspcc-dev/neofs-node/issues/2563 err = nil } } else { diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go new file mode 100644 index 000000000..16b630b6a --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go @@ -0,0 +1,9 @@ +//go:build !linux + +package fstree + +import "io/fs" + +func newSpecificWriteData(_ FileCounter, _ string, _ fs.FileMode, _ bool) func(string, []byte) error { + return nil +} diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 4d7354595..d08747e13 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -2,6 +2,7 @@ package shard import ( "context" + "fmt" "io/fs" "math" "os" @@ -11,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -145,8 +145,12 @@ func TestRefillMetabaseCorrupted(t *testing.T) { require.NoError(t, sh.Close()) addr := object.AddressOf(obj) - _, err = fsTree.Put(context.Background(), common.PutPrm{Address: addr, RawData: []byte("not an object")}) - require.NoError(t, err) + // This is copied from `fstree.treePath()` to avoid exporting function just for tests. + { + saddr := addr.Object().EncodeToString() + "." + addr.Container().EncodeToString() + p := fmt.Sprintf("%s/%s/%s", fsTree.RootPath, saddr[:2], saddr[2:]) + require.NoError(t, os.WriteFile(p, []byte("not an object"), fsTree.Permissions)) + } sh = New( WithID(NewIDFromBytes([]byte{})), From fb74524ac73eaa1f46b63c06389035ae9b4b9bed Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 17:53:27 +0300 Subject: [PATCH 0872/1943] [#970] fstree: Move delete implementation to a separate file Signed-off-by: Evgenii Stratonikov --- .../blobstor/fstree/control.go | 2 +- .../blobstor/fstree/fstree.go | 22 +++----------- .../blobstor/fstree/fstree_write_generic.go | 30 +++++++++++++++++-- .../blobstor/fstree/fstree_write_linux.go | 17 +++++++++-- .../blobstor/fstree/fstree_write_specific.go | 6 ++-- 5 files changed, 52 insertions(+), 25 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index b126f8d29..cec531f2e 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -19,7 +19,7 @@ func (t *FSTree) Init() error { if !t.readOnly { f := newSpecificWriteData(t.fileCounter, t.RootPath, t.Permissions, t.noSync) if f != nil { - t.writeData = f + t.writer = f } } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index fc29d4eaa..ba691b056 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -57,7 +57,7 @@ type FSTree struct { fileCounter FileCounter fileCounterEnabled bool - writeData func(string, []byte) error + writer writer } // Info groups the information about file storage. @@ -95,7 +95,7 @@ func New(opts ...Option) *FSTree { for i := range opts { opts[i](f) } - f.writeData = newGenericWriteData(f) + f.writer = newGenericWriteData(f) return f } @@ -266,21 +266,7 @@ func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Delet } p := t.treePath(prm.Address) - - if t.fileCounterEnabled { - t.fileGuard.Lock(p) - err = os.Remove(p) - t.fileGuard.Unlock(p) - if err == nil { - t.fileCounter.Dec() - } - } else { - err = os.Remove(p) - } - - if err != nil && os.IsNotExist(err) { - err = logicerr.Wrap(new(apistatus.ObjectNotFound)) - } + err = t.writer.removeFile(p) return common.DeleteRes{}, err } @@ -357,7 +343,7 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err } size = len(prm.RawData) - return common.PutRes{StorageID: []byte{}}, t.writeData(p, prm.RawData) + return common.PutRes{StorageID: []byte{}}, t.writer.writeData(p, prm.RawData) } // Get returns an object from the storage by address. diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 3d65808ec..e1e3a53bb 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -9,8 +9,15 @@ import ( "syscall" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) +type writer interface { + writeData(string, []byte) error + removeFile(string) error +} + type genericWriter struct { perm fs.FileMode flags int @@ -19,7 +26,7 @@ type genericWriter struct { suffix atomic.Uint64 } -func newGenericWriteData(t *FSTree) func(string, []byte) error { +func newGenericWriteData(t *FSTree) writer { flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL if !t.noSync { flags |= os.O_SYNC @@ -29,7 +36,7 @@ func newGenericWriteData(t *FSTree) func(string, []byte) error { flags: flags, t: t, } - return w.writeData + return w } func (w *genericWriter) writeData(p string, data []byte) error { @@ -86,3 +93,22 @@ func (w *genericWriter) writeFile(p string, data []byte) error { } return err } + +func (w *genericWriter) removeFile(p string) error { + var err error + if w.t.fileCounterEnabled { + w.t.fileGuard.Lock(p) + err = os.Remove(p) + w.t.fileGuard.Unlock(p) + if err == nil { + w.t.fileCounter.Dec() + } + } else { + err = os.Remove(p) + } + + if err != nil && os.IsNotExist(err) { + err = logicerr.Wrap(new(apistatus.ObjectNotFound)) + } + return err +} diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go index 3416cf7ac..a77c9d1b2 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -8,6 +8,8 @@ import ( "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "golang.org/x/sys/unix" ) @@ -19,7 +21,7 @@ type linuxWriter struct { counter FileCounter } -func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync bool) func(string, []byte) error { +func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync bool) writer { flags := unix.O_WRONLY | unix.O_TMPFILE | unix.O_CLOEXEC if !noSync { flags |= unix.O_DSYNC @@ -37,7 +39,7 @@ func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync b flags: flags, counter: c, } - return w.writeData + return w } func (w *linuxWriter) writeData(p string, data []byte) error { @@ -74,3 +76,14 @@ func (w *linuxWriter) writeFile(p string, data []byte) error { } return errClose } + +func (w *linuxWriter) removeFile(p string) error { + err := unix.Unlink(p) + if err != nil && err == unix.ENOENT { + return logicerr.Wrap(new(apistatus.ObjectNotFound)) + } + if err == nil { + w.counter.Dec() + } + return err +} diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go index 16b630b6a..9bd559366 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go @@ -2,8 +2,10 @@ package fstree -import "io/fs" +import ( + "io/fs" +) -func newSpecificWriteData(_ FileCounter, _ string, _ fs.FileMode, _ bool) func(string, []byte) error { +func newSpecificWriteData(_ FileCounter, _ string, _ fs.FileMode, _ bool) writer { return nil } From abd502215f0fc0c8e6454eeeeb54b9c83c680dc4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 18:04:18 +0300 Subject: [PATCH 0873/1943] [#970] fstree: Move file locking to the generic writer It is not a part of FSTree itself, but rather a way to solve concurrent counter update on non-linux implementations. New linux implementations is pretty simple: link fails when the file exists, unlink fails when the file doesn't exist. Signed-off-by: Evgenii Stratonikov --- .../blobstor/fstree/counter.go | 5 +++ .../blobstor/fstree/fstree.go | 9 ++-- .../blobstor/fstree/fstree_write_generic.go | 45 ++++++++++++------- .../blobstor/fstree/option.go | 3 -- 4 files changed, 37 insertions(+), 25 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/counter.go b/pkg/local_object_storage/blobstor/fstree/counter.go index 70b346093..718104e2e 100644 --- a/pkg/local_object_storage/blobstor/fstree/counter.go +++ b/pkg/local_object_storage/blobstor/fstree/counter.go @@ -18,6 +18,11 @@ func (c *noopCounter) Set(uint64) {} func (c *noopCounter) Inc() {} func (c *noopCounter) Dec() {} +func counterEnabled(c FileCounter) bool { + _, noop := c.(*noopCounter) + return !noop +} + type SimpleCounter struct { v atomic.Uint64 } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index ba691b056..76ce5bf17 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -53,9 +53,7 @@ type FSTree struct { readOnly bool metrics Metrics - fileGuard keyLock - fileCounter FileCounter - fileCounterEnabled bool + fileCounter FileCounter writer writer } @@ -88,14 +86,13 @@ func New(opts ...Option) *FSTree { Depth: 4, DirNameLen: DirNameLen, metrics: &noopMetrics{}, - fileGuard: &noopKeyLock{}, fileCounter: &noopCounter{}, log: &logger.Logger{Logger: zap.L()}, } for i := range opts { opts[i](f) } - f.writer = newGenericWriteData(f) + f.writer = newGenericWriteData(f.fileCounter, f.Permissions, f.noSync) return f } @@ -444,7 +441,7 @@ func (t *FSTree) GetRange(ctx context.Context, prm common.GetRangePrm) (common.G // initFileCounter walks the file tree rooted at FSTree's root, // counts total items count, inits counter and returns number of stored objects. func (t *FSTree) initFileCounter() error { - if !t.fileCounterEnabled { + if !counterEnabled(t.fileCounter) { return nil } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index e1e3a53bb..808076959 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) @@ -22,19 +23,31 @@ type genericWriter struct { perm fs.FileMode flags int - t *FSTree - suffix atomic.Uint64 + fileGuard keyLock + fileCounter FileCounter + fileCounterEnabled bool + suffix atomic.Uint64 } -func newGenericWriteData(t *FSTree) writer { +func newGenericWriteData(c FileCounter, perm fs.FileMode, noSync bool) writer { flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC | os.O_EXCL - if !t.noSync { + if !noSync { flags |= os.O_SYNC } + + var fileGuard keyLock = &noopKeyLock{} + fileCounterEnabled := counterEnabled(c) + if fileCounterEnabled { + fileGuard = utilSync.NewKeyLocker[string]() + } + var w = &genericWriter{ - perm: t.Permissions, + perm: perm, flags: flags, - t: t, + + fileCounterEnabled: fileCounterEnabled, + fileGuard: fileGuard, + fileCounter: c, } return w } @@ -46,9 +59,9 @@ func (w *genericWriter) writeData(p string, data []byte) error { // writeAndRename opens tmpPath exclusively, writes data to it and renames it to p. func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error { - if w.t.fileCounterEnabled { - w.t.fileGuard.Lock(p) - defer w.t.fileGuard.Unlock(p) + if w.fileCounterEnabled { + w.fileGuard.Lock(p) + defer w.fileGuard.Unlock(p) } err := w.writeFile(tmpPath, data) @@ -64,15 +77,15 @@ func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error { return err } - if w.t.fileCounterEnabled { - w.t.fileCounter.Inc() + if w.fileCounterEnabled { + w.fileCounter.Inc() var targetFileExists bool if _, e := os.Stat(p); e == nil { targetFileExists = true } err = os.Rename(tmpPath, p) if err == nil && targetFileExists { - w.t.fileCounter.Dec() + w.fileCounter.Dec() } } else { err = os.Rename(tmpPath, p) @@ -96,12 +109,12 @@ func (w *genericWriter) writeFile(p string, data []byte) error { func (w *genericWriter) removeFile(p string) error { var err error - if w.t.fileCounterEnabled { - w.t.fileGuard.Lock(p) + if w.fileCounterEnabled { + w.fileGuard.Lock(p) err = os.Remove(p) - w.t.fileGuard.Unlock(p) + w.fileGuard.Unlock(p) if err == nil { - w.t.fileCounter.Dec() + w.fileCounter.Dec() } } else { err = os.Remove(p) diff --git a/pkg/local_object_storage/blobstor/fstree/option.go b/pkg/local_object_storage/blobstor/fstree/option.go index e6c7a2608..4d1f8fc22 100644 --- a/pkg/local_object_storage/blobstor/fstree/option.go +++ b/pkg/local_object_storage/blobstor/fstree/option.go @@ -4,7 +4,6 @@ import ( "io/fs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" "go.uber.org/zap" ) @@ -48,9 +47,7 @@ func WithMetrics(m Metrics) Option { func WithFileCounter(c FileCounter) Option { return func(f *FSTree) { - f.fileCounterEnabled = true f.fileCounter = c - f.fileGuard = utilSync.NewKeyLocker[string]() } } From b36a45323859ffb2c19c56ce5253967609f008db Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 21:07:02 +0300 Subject: [PATCH 0874/1943] [#970] fstree: Add build tag to enable generic version on linux Unless tested, generic version can start gaining bugs. With a separate build tag we can have the best of both worlds: 1. Use optimized implementation for linux by default. 2. Run tests or benchmarks for both. Note that they are not actually run automatically now, but this is at leas possible. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go | 2 +- .../blobstor/fstree/fstree_write_specific.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go index a77c9d1b2..efc5a3d3d 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -1,4 +1,4 @@ -//go:build linux +//go:build linux && !fstree_generic package fstree diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go index 9bd559366..67052d947 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_specific.go @@ -1,4 +1,4 @@ -//go:build !linux +//go:build !linux || fstree_generic package fstree From 962e5a9c19be08398ac2d84a590b553d0f8cd4c8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 8 Feb 2024 17:04:58 +0300 Subject: [PATCH 0875/1943] [#736] logger: Add journald support Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/main.go | 4 ++ cmd/frostfs-node/config.go | 9 ++- cmd/frostfs-node/config/logger/config.go | 18 ++++++ cmd/frostfs-node/config/logger/config_test.go | 8 +-- cmd/frostfs-node/validate.go | 5 ++ config/example/node.env | 1 + config/example/node.json | 3 +- config/example/node.yaml | 1 + go.mod | 2 + go.sum | 4 ++ pkg/util/logger/logger.go | 59 +++++++++++++++++++ 11 files changed, 108 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 2b39f78e3..9879342b7 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -68,6 +68,10 @@ func main() { cfg.GetString("logger.level"), ) exitErr(err) + err = logPrm.SetDestination( + cfg.GetString("logger.destination"), + ) + exitErr(err) logPrm.SamplingHook = metrics.LogMetrics().GetSamplingHook() log, err = logger.NewLogger(logPrm) exitErr(err) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c41863ecd..50219a8c7 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -96,7 +96,8 @@ type applicationConfiguration struct { _read bool LoggerCfg struct { - level string + level string + destination string } EngineCfg struct { @@ -209,6 +210,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { // Logger a.LoggerCfg.level = loggerconfig.Level(c) + a.LoggerCfg.destination = loggerconfig.Destination(c) // Storage Engine @@ -1004,6 +1006,11 @@ func (c *cfg) loggerPrm() (*logger.Prm, error) { // not expected since validation should be performed before panic(fmt.Sprintf("incorrect log level format: %s", c.LoggerCfg.level)) } + err = c.dynamicConfiguration.logger.SetDestination(c.LoggerCfg.destination) + if err != nil { + // not expected since validation should be performed before + panic(fmt.Sprintf("incorrect log destination format: %s", c.LoggerCfg.destination)) + } return c.dynamicConfiguration.logger, nil } diff --git a/cmd/frostfs-node/config/logger/config.go b/cmd/frostfs-node/config/logger/config.go index 02645c543..378b9d793 100644 --- a/cmd/frostfs-node/config/logger/config.go +++ b/cmd/frostfs-node/config/logger/config.go @@ -5,12 +5,14 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/logging/lokicore/loki" ) const ( // LevelDefault is a default logger level. LevelDefault = "info" + DestinationDefault = logger.DestinationStdout subsection = "logger" lokiSubsection = "loki" AddressDefault = "localhost:3100" @@ -34,6 +36,22 @@ func Level(c *config.Config) string { return LevelDefault } +// Destination returns the value of "destination" config parameter +// from "logger" section. +// +// Returns DestinationDefault if the value is not a non-empty string. +func Destination(c *config.Config) string { + v := config.StringSafe( + c.Sub(subsection), + "destination", + ) + if v != "" { + return v + } + + return DestinationDefault +} + // ToLokiConfig extracts loki config. func ToLokiConfig(c *config.Config) loki.Config { hostname, _ := os.Hostname() diff --git a/cmd/frostfs-node/config/logger/config_test.go b/cmd/frostfs-node/config/logger/config_test.go index 8a0acbd85..3587a0ddb 100644 --- a/cmd/frostfs-node/config/logger/config_test.go +++ b/cmd/frostfs-node/config/logger/config_test.go @@ -11,15 +11,15 @@ import ( func TestLoggerSection_Level(t *testing.T) { t.Run("defaults", func(t *testing.T) { - v := loggerconfig.Level(configtest.EmptyConfig()) - require.Equal(t, loggerconfig.LevelDefault, v) + require.Equal(t, loggerconfig.LevelDefault, loggerconfig.Level(configtest.EmptyConfig())) + require.Equal(t, loggerconfig.DestinationDefault, loggerconfig.Destination(configtest.EmptyConfig())) }) const path = "../../../../config/example/node" fileConfigTest := func(c *config.Config) { - v := loggerconfig.Level(c) - require.Equal(t, "debug", v) + require.Equal(t, "debug", loggerconfig.Level(c)) + require.Equal(t, "journald", loggerconfig.Destination(c)) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/validate.go b/cmd/frostfs-node/validate.go index 80c90ec44..ae52b9e4a 100644 --- a/cmd/frostfs-node/validate.go +++ b/cmd/frostfs-node/validate.go @@ -25,6 +25,11 @@ func validateConfig(c *config.Config) error { return fmt.Errorf("invalid logger level: %w", err) } + err = loggerPrm.SetDestination(loggerconfig.Destination(c)) + if err != nil { + return fmt.Errorf("invalid logger destination: %w", err) + } + // shard configuration validation shardNum := 0 diff --git a/config/example/node.env b/config/example/node.env index 2972f56e9..a1db0c876 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -1,4 +1,5 @@ FROSTFS_LOGGER_LEVEL=debug +FROSTFS_LOGGER_DESTINATION=journald FROSTFS_PPROF_ENABLED=true FROSTFS_PPROF_ADDRESS=localhost:6060 diff --git a/config/example/node.json b/config/example/node.json index 8dda5e606..a7d7a3651 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -1,6 +1,7 @@ { "logger": { - "level": "debug" + "level": "debug", + "destination": "journald" }, "pprof": { "enabled": true, diff --git a/config/example/node.yaml b/config/example/node.yaml index 171515164..678ee1a87 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -1,5 +1,6 @@ logger: level: debug # logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" + destination: journald # logger destination: one of "stdout" (default), "journald" systemdnotify: enabled: true diff --git a/go.mod b/go.mod index 06d2d3299..aa771f1e7 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240129064140-8d21ab2d99d9 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 + git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 @@ -30,6 +31,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 + github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.8.4 go.etcd.io/bbolt v1.3.8 go.opentelemetry.io/otel v1.22.0 diff --git a/go.sum b/go.sum index b9052efde..e15236331 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9m git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= +git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 h1:HeY8n27VyPRQe49l/fzyVMkWEB2fsLJYKp64pwA7tz4= +git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02/go.mod h1:rQFJJdEOV7KbbMtQYR2lNfiZk+ONRDJSbMCTWxKt8Fw= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= @@ -247,6 +249,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/ssgreg/journald v1.0.0 h1:0YmTDPJXxcWDPba12qNMdO6TxvfkFSYpFIJ31CwmLcU= +github.com/ssgreg/journald v1.0.0/go.mod h1:RUckwmTM8ghGWPslq2+ZBZzbb9/2KgjzYZ4JEP+oRt0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index f9a5fe38f..c6db04df9 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -1,6 +1,10 @@ package logger import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/zapjournald" + "github.com/ssgreg/journald" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -35,8 +39,15 @@ type Prm struct { SamplingHook func(e zapcore.Entry, sd zapcore.SamplingDecision) // do not support runtime rereading + dest string } +const ( + DestinationUndefined = "" + DestinationStdout = "stdout" + DestinationJournald = "journald" +) + // SetLevelString sets the minimum logging level. Default is // "info". // @@ -48,6 +59,16 @@ func (p *Prm) SetLevelString(s string) error { return p.level.UnmarshalText([]byte(s)) } +func (p *Prm) SetDestination(d string) error { + if d != DestinationStdout && d != DestinationJournald { + return fmt.Errorf("invalid logger destination %s", d) + } + if p != nil { + p.dest = d + } + return nil +} + // Reload reloads configuration of a connected instance of the Logger. // Returns ErrLoggerNotConnected if no connection has been performed. // Returns any reconfiguration error from the Logger directly. @@ -79,7 +100,17 @@ func NewLogger(prm *Prm) (*Logger, error) { if prm == nil { prm = defaultPrm() } + switch prm.dest { + case DestinationUndefined, DestinationStdout: + return newConsoleLogger(prm) + case DestinationJournald: + return newJournaldLogger(prm) + default: + return nil, fmt.Errorf("unknown destination %s", prm.dest) + } +} +func newConsoleLogger(prm *Prm) (*Logger, error) { lvl := zap.NewAtomicLevelAt(prm.level) c := zap.NewProductionConfig() @@ -103,6 +134,34 @@ func NewLogger(prm *Prm) (*Logger, error) { return l, nil } +func newJournaldLogger(prm *Prm) (*Logger, error) { + lvl := zap.NewAtomicLevelAt(prm.level) + + c := zap.NewProductionConfig() + c.Level = lvl + c.Encoding = "console" + c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + if prm.SamplingHook != nil { + c.Sampling.Hook = prm.SamplingHook + } + + encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields) + + core := zapjournald.NewCore(zap.NewAtomicLevelAt(prm.level), encoder, &journald.Journal{}, zapjournald.SyslogFields) + coreWithContext := core.With([]zapcore.Field{ + zapjournald.SyslogFacility(zapjournald.LogDaemon), + zapjournald.SyslogIdentifier(), + zapjournald.SyslogPid(), + }) + + lZap := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) + + l := &Logger{Logger: lZap, lvl: lvl} + prm._log = l + + return l, nil +} + func (l *Logger) reload(prm Prm) error { l.lvl.SetLevel(prm.level) return nil From 3a41858a0f2b02f426d37abe9d9a39ca1835436f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 12 Feb 2024 13:28:37 +0300 Subject: [PATCH 0876/1943] [#975] ir: Add default logger destination Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/defaults.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index a7fe8d563..127a68b29 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -8,6 +8,7 @@ import ( func defaultConfiguration(cfg *viper.Viper) { cfg.SetDefault("logger.level", "info") + cfg.SetDefault("logger.destination", "stdout") setPprofDefaults(cfg) From 6a5769d1da019edef19c9e23aaa9c55e80281199 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 13 Feb 2024 09:09:09 +0300 Subject: [PATCH 0877/1943] [#948] Fix `gofumpt` issue Signed-off-by: Anton Nikiforov --- .../blobstor/fstree/fstree_write_generic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 808076959..8b2622885 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -41,7 +41,7 @@ func newGenericWriteData(c FileCounter, perm fs.FileMode, noSync bool) writer { fileGuard = utilSync.NewKeyLocker[string]() } - var w = &genericWriter{ + w := &genericWriter{ perm: perm, flags: flags, From 0bd030507ef7f0d6a47a8f310d3de03804a8d416 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 13 Feb 2024 09:09:47 +0300 Subject: [PATCH 0878/1943] [#948] metrics: Set actual value for `shard_id` after restart Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/engine/metrics.go | 4 ++++ pkg/local_object_storage/engine/writecache.go | 4 ++++ pkg/local_object_storage/shard/control.go | 3 +++ pkg/local_object_storage/shard/gc.go | 2 ++ pkg/local_object_storage/shard/id.go | 7 ++++++- pkg/local_object_storage/writecache/cachebbolt.go | 4 ++++ pkg/local_object_storage/writecache/metrics.go | 3 +++ pkg/local_object_storage/writecache/writecache.go | 1 + 8 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index e4117bb1a..91dfa8762 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -44,6 +44,10 @@ type gcMetrics struct { shardID string } +func (m *gcMetrics) SetShardID(id string) { + m.shardID = id +} + func (m *gcMetrics) AddRunDuration(d time.Duration, success bool) { m.storage.AddRunDuration(m.shardID, d, success) } diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 0eca018f8..f30bce00d 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -145,6 +145,10 @@ type writeCacheMetrics struct { metrics metrics.WriteCacheMetrics } +func (m *writeCacheMetrics) SetShardID(id string) { + m.shardID = id +} + func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.StorageType) { m.metrics.AddMethodDuration(m.shardID, "Get", success, d, st.String()) } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 8d1007ffd..408354b55 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -166,6 +166,9 @@ func (s *Shard) Init(ctx context.Context) error { }, }, } + if s.gc.metrics != nil { + s.gc.metrics.SetShardID(s.info.ID.String()) + } s.gc.init(ctx) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 24c3a337a..83be3259a 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -81,6 +81,7 @@ const ( ) type GCMectrics interface { + SetShardID(string) AddRunDuration(d time.Duration, success bool) AddDeletedCount(deleted, failed uint64) AddExpiredObjectCollectionDuration(d time.Duration, success bool, objectType string) @@ -89,6 +90,7 @@ type GCMectrics interface { type noopGCMetrics struct{} +func (m *noopGCMetrics) SetShardID(string) {} func (m *noopGCMetrics) AddRunDuration(time.Duration, bool) {} func (m *noopGCMetrics) AddDeletedCount(uint64, uint64) {} func (m *noopGCMetrics) AddExpiredObjectCollectionDuration(time.Duration, bool, string) {} diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index b67fd59cd..be474e0fa 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -53,9 +53,14 @@ func (s *Shard) UpdateID(ctx context.Context) (err error) { s.info.ID = NewIDFromBytes(idFromMetabase) } + shardID := s.info.ID.String() if s.cfg.metricsWriter != nil { - s.cfg.metricsWriter.SetShardID(s.info.ID.String()) + s.cfg.metricsWriter.SetShardID(shardID) } + if s.writeCache != nil && s.writeCache.GetMetrics() != nil { + s.writeCache.GetMetrics().SetShardID(shardID) + } + s.log = &logger.Logger{Logger: s.log.With(zap.Stringer("shard_id", s.info.ID))} s.metaBase.SetLogger(s.log) s.blobStor.SetLogger(s.log) diff --git a/pkg/local_object_storage/writecache/cachebbolt.go b/pkg/local_object_storage/writecache/cachebbolt.go index 2ae62c461..fdba8d409 100644 --- a/pkg/local_object_storage/writecache/cachebbolt.go +++ b/pkg/local_object_storage/writecache/cachebbolt.go @@ -146,3 +146,7 @@ func (c *cache) Close() error { c.metrics.Close() return nil } + +func (c *cache) GetMetrics() Metrics { + return c.metrics +} diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go index 5eac06698..492e0973a 100644 --- a/pkg/local_object_storage/writecache/metrics.go +++ b/pkg/local_object_storage/writecache/metrics.go @@ -19,6 +19,7 @@ const ( ) type Metrics interface { + SetShardID(string) Get(d time.Duration, success bool, st StorageType) Delete(d time.Duration, success bool, st StorageType) Put(d time.Duration, success bool, st StorageType) @@ -35,6 +36,8 @@ func DefaultMetrics() Metrics { return metricsStub{} } type metricsStub struct{} +func (metricsStub) SetShardID(string) {} + func (metricsStub) Get(time.Duration, bool, StorageType) {} func (metricsStub) Delete(time.Duration, bool, StorageType) {} diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 69dfee953..112594ec0 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -41,6 +41,7 @@ type Cache interface { Init() error Open(ctx context.Context, mode mode.Mode) error Close() error + GetMetrics() Metrics } // MainStorage is the interface of the underlying storage of Cache implementations. From cda3a3d83440ffab9ec415376cc085bea802c027 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 31 Jan 2024 17:26:26 +0300 Subject: [PATCH 0879/1943] [#932] adm: Refactor command `morph` Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/internal/modules/morph/ape.go | 21 +-- .../internal/modules/morph/ape_util.go | 9 +- .../internal/modules/morph/balance.go | 9 +- .../internal/modules/morph/config.go | 3 +- .../internal/modules/morph/container.go | 5 +- .../internal/modules/morph/deploy.go | 5 +- .../internal/modules/morph/dump_hashes.go | 7 +- .../internal/modules/morph/frostfsid.go | 47 +++--- .../internal/modules/morph/generate.go | 8 +- .../internal/modules/morph/generate_test.go | 3 +- .../internal/modules/morph/initialize.go | 144 +++--------------- .../modules/morph/initialize_deploy.go | 3 +- .../internal/modules/morph/initialize_nns.go | 5 +- .../modules/morph/initialize_register.go | 5 +- .../modules/morph/initialize_roles.go | 3 +- .../internal/modules/morph/initialize_test.go | 19 +-- .../modules/morph/initialize_transfer.go | 11 +- .../modules/morph/netmap_candidates.go | 3 +- .../internal/modules/morph/notary.go | 5 +- .../internal/modules/morph/policy.go | 3 +- .../internal/modules/morph/root.go | 88 ++++++----- .../internal/modules/morph/util/const.go | 15 ++ .../internal/modules/morph/util/initialize.go | 90 +++++++++++ .../modules/morph/{ => util}/local_client.go | 46 +++--- .../modules/morph/{ => util}/n3client.go | 45 ++++-- .../internal/modules/morph/{ => util}/util.go | 12 +- 26 files changed, 324 insertions(+), 290 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/util/const.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/util/initialize.go rename cmd/frostfs-adm/internal/modules/morph/{ => util}/local_client.go (89%) rename cmd/frostfs-adm/internal/modules/morph/{ => util}/n3client.go (72%) rename cmd/frostfs-adm/internal/modules/morph/{ => util}/util.go (86%) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape.go index 9f1d75adb..5ff6e7ab8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -41,7 +42,7 @@ var ( Use: "add-rule-chain", Short: "Add rule chain", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) }, Run: addRuleChain, @@ -51,7 +52,7 @@ var ( Use: "rm-rule-chain", Short: "Remove rule chain", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) }, Run: removeRuleChain, @@ -61,7 +62,7 @@ var ( Use: "list-rule-chains", Short: "List rule chains", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) }, Run: listRuleChains, @@ -71,7 +72,7 @@ var ( Use: "set-admin", Short: "Set admin", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) }, Run: setAdmin, @@ -81,7 +82,7 @@ var ( Use: "get-admin", Short: "Get admin", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) }, Run: getAdmin, @@ -91,7 +92,7 @@ var ( func initAddRuleChainCmd() { apeCmd.AddCommand(addRuleChainCmd) - addRuleChainCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + addRuleChainCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) addRuleChainCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) addRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc) @@ -110,7 +111,7 @@ func initAddRuleChainCmd() { func initRemoveRuleChainCmd() { apeCmd.AddCommand(removeRuleChainCmd) - removeRuleChainCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + removeRuleChainCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) removeRuleChainCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) removeRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc) @@ -124,7 +125,7 @@ func initRemoveRuleChainCmd() { func initListRuleChainsCmd() { apeCmd.AddCommand(listRuleChainsCmd) - listRuleChainsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + listRuleChainsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) listRuleChainsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) listRuleChainsCmd.Flags().StringP(targetTypeFlag, "t", "", targetTypeDesc) _ = listRuleChainsCmd.MarkFlagRequired(targetTypeFlag) @@ -136,7 +137,7 @@ func initListRuleChainsCmd() { func initSetAdminCmd() { apeCmd.AddCommand(setAdminCmd) - setAdminCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + setAdminCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) setAdminCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) setAdminCmd.Flags().String(addrAdminFlag, "", addrAdminDesc) _ = setAdminCmd.MarkFlagRequired(addrAdminFlag) @@ -145,7 +146,7 @@ func initSetAdminCmd() { func initGetAdminCmd() { apeCmd.AddCommand(getAdminCmd) - getAdminCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + getAdminCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) getAdminCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } diff --git a/cmd/frostfs-adm/internal/modules/morph/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape_util.go index f64a7e108..30668532b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape_util.go @@ -6,6 +6,7 @@ import ( "os" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -77,17 +78,17 @@ func parseChain(cmd *cobra.Command) *apechain.Chain { func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *actor.Actor) { v := viper.GetViper() - c, err := getN3Client(v) + c, err := morphUtil.GetN3Client(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) walletDir := config.ResolveHomePath(viper.GetString(alphabetWalletsFlag)) - wallets, err := getAlphabetWallets(v, walletDir) + wallets, err := morphUtil.GetAlphabetWallets(v, walletDir) commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) - committeeAcc, err := getWalletAccount(wallets[0], committeeAccountName) + committeeAcc, err := morphUtil.GetWalletAccount(wallets[0], committeeAccountName) commonCmd.ExitOnErr(cmd, "can't find committee account: %w", err) - ac, err := newActor(c, committeeAcc) + ac, err := morphUtil.NewActor(c, committeeAcc) commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) inv := &ac.Invoker diff --git a/cmd/frostfs-adm/internal/modules/morph/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance.go index 0fee45a8f..b4259e11e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance.go @@ -7,6 +7,7 @@ import ( "math/big" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -49,7 +50,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { nmHash util.Uint160 ) - c, err := getN3Client(viper.GetViper()) + c, err := morphUtil.GetN3Client(viper.GetViper()) if err != nil { return err } @@ -150,7 +151,7 @@ func printProxyContractBalance(cmd *cobra.Command, inv *invoker.Invoker, nnsHash return nil } -func printAlphabetContractBalances(cmd *cobra.Command, c Client, inv *invoker.Invoker, count int, nnsHash util.Uint160) error { +func printAlphabetContractBalances(cmd *cobra.Command, c morphUtil.Client, inv *invoker.Invoker, count int, nnsHash util.Uint160) error { alphaList := make([]accBalancePair, count) w := io.NewBufBinWriter() @@ -184,7 +185,7 @@ func printAlphabetContractBalances(cmd *cobra.Command, c Client, inv *invoker.In return nil } -func fetchIRNodes(c Client, desigHash util.Uint160) ([]accBalancePair, error) { +func fetchIRNodes(c morphUtil.Client, desigHash util.Uint160) ([]accBalancePair, error) { inv := invoker.New(c, nil) height, err := c.GetBlockCount() @@ -192,7 +193,7 @@ func fetchIRNodes(c Client, desigHash util.Uint160) ([]accBalancePair, error) { return nil, fmt.Errorf("can't get block height: %w", err) } - arr, err := getDesignatedByRole(inv, desigHash, noderoles.NeoFSAlphabet, height) + arr, err := morphUtil.GetDesignatedByRole(inv, desigHash, noderoles.NeoFSAlphabet, height) if err != nil { return nil, errors.New("can't fetch list of IR nodes from the netmap contract") } diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index db7d2ab8e..d9ddcb8ad 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -10,6 +10,7 @@ import ( "strings" "text/tabwriter" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -24,7 +25,7 @@ import ( const forceConfigSet = "force" func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { - c, err := getN3Client(viper.GetViper()) + c, err := util.GetN3Client(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/container.go b/cmd/frostfs-adm/internal/modules/morph/container.go index e7a7f5466..b9db4c4e5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container.go @@ -7,6 +7,7 @@ import ( "os" "sort" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" "github.com/nspcc-dev/neo-go/pkg/io" @@ -73,7 +74,7 @@ func dumpContainers(cmd *cobra.Command, _ []string) error { return fmt.Errorf("invalid filename: %w", err) } - c, err := getN3Client(viper.GetViper()) + c, err := morphUtil.GetN3Client(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } @@ -163,7 +164,7 @@ func dumpSingleContainer(bw *io.BufBinWriter, ch util.Uint160, inv *invoker.Invo } func listContainers(cmd *cobra.Command, _ []string) error { - c, err := getN3Client(viper.GetViper()) + c, err := morphUtil.GetN3Client(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/deploy.go b/cmd/frostfs-adm/internal/modules/morph/deploy.go index 1facd6d10..db54a8db5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/deploy.go @@ -7,6 +7,7 @@ import ( "strings" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/cli/cmdargs" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -38,7 +39,7 @@ Contract's manifest file name must be 'config.json'. NNS name is taken by stripping '_contract.nef' from the NEF file (similar to frostfs contracts).`, PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: deployContractCmd, } @@ -49,7 +50,7 @@ func init() { ff.String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) _ = deployCmd.MarkFlagFilename(alphabetWalletsFlag) - ff.StringP(endpointFlag, "r", "", endpointFlagDesc) + ff.StringP(util.EndpointFlag, "r", "", util.EndpointFlagDesc) ff.String(contractPathFlag, "", "Path to the contract directory") _ = deployCmd.MarkFlagFilename(contractPathFlag) diff --git a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go index 14a492c5f..60e9de9ca 100644 --- a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go @@ -8,6 +8,7 @@ import ( "text/tabwriter" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -33,7 +34,7 @@ type contractDumpInfo struct { } func dumpContractHashes(cmd *cobra.Command, _ []string) error { - c, err := getN3Client(viper.GetViper()) + c, err := morphUtil.GetN3Client(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } @@ -110,7 +111,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { return nil } -func dumpCustomZoneHashes(cmd *cobra.Command, nnsHash util.Uint160, zone string, c Client) error { +func dumpCustomZoneHashes(cmd *cobra.Command, nnsHash util.Uint160, zone string, c morphUtil.Client) error { const nnsMaxTokens = 100 inv := invoker.New(c, nil) @@ -224,7 +225,7 @@ func printContractInfo(cmd *cobra.Command, infos []contractDumpInfo) { cmd.Print(buf.String()) } -func fillContractVersion(cmd *cobra.Command, c Client, infos []contractDumpInfo) { +func fillContractVersion(cmd *cobra.Command, c morphUtil.Client, infos []contractDumpInfo) { bw := io.NewBufBinWriter() sub := io.NewBufBinWriter() for i := range infos { diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go index 50cd47744..8150dcb5f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -6,6 +6,7 @@ import ( "sort" frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -41,7 +42,7 @@ var ( Short: "Create new namespace in frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidCreateNamespace, } @@ -51,7 +52,7 @@ var ( Short: "List all namespaces in frostfsid", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidListNamespaces, } @@ -61,7 +62,7 @@ var ( Short: "Create subject in frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidCreateSubject, } @@ -71,7 +72,7 @@ var ( Short: "Delete subject from frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidDeleteSubject, } @@ -81,7 +82,7 @@ var ( Short: "List subjects in namespace", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidListSubjects, } @@ -91,7 +92,7 @@ var ( Short: "Create group in frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidCreateGroup, } @@ -101,7 +102,7 @@ var ( Short: "Delete group from frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidDeleteGroup, } @@ -111,7 +112,7 @@ var ( Short: "List groups in namespace", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidListGroups, } @@ -121,7 +122,7 @@ var ( Short: "Add subject to group", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidAddSubjectToGroup, } @@ -131,7 +132,7 @@ var ( Short: "Remove subject from group", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidRemoveSubjectFromGroup, } @@ -141,7 +142,7 @@ var ( Short: "List subjects in group", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidListGroupSubjects, } @@ -149,20 +150,20 @@ var ( func initFrostfsIDCreateNamespaceCmd() { frostfsidCmd.AddCommand(frostfsidCreateNamespaceCmd) - frostfsidCreateNamespaceCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidCreateNamespaceCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidCreateNamespaceCmd.Flags().String(namespaceFlag, "", "Namespace name to create") frostfsidCreateNamespaceCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDListNamespacesCmd() { frostfsidCmd.AddCommand(frostfsidListNamespacesCmd) - frostfsidListNamespacesCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidListNamespacesCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidListNamespacesCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDCreateSubjectCmd() { frostfsidCmd.AddCommand(frostfsidCreateSubjectCmd) - frostfsidCreateSubjectCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidCreateSubjectCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidCreateSubjectCmd.Flags().String(namespaceFlag, "", "Namespace where create subject") frostfsidCreateSubjectCmd.Flags().String(subjectNameFlag, "", "Subject name, must be unique in namespace") frostfsidCreateSubjectCmd.Flags().String(subjectKeyFlag, "", "Subject hex-encoded public key") @@ -171,14 +172,14 @@ func initFrostfsIDCreateSubjectCmd() { func initFrostfsIDDeleteSubjectCmd() { frostfsidCmd.AddCommand(frostfsidDeleteSubjectCmd) - frostfsidDeleteSubjectCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidDeleteSubjectCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidDeleteSubjectCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidDeleteSubjectCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDListSubjectsCmd() { frostfsidCmd.AddCommand(frostfsidListSubjectsCmd) - frostfsidListSubjectsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidListSubjectsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidListSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace to list subjects") frostfsidListSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") frostfsidListSubjectsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) @@ -186,7 +187,7 @@ func initFrostfsIDListSubjectsCmd() { func initFrostfsIDCreateGroupCmd() { frostfsidCmd.AddCommand(frostfsidCreateGroupCmd) - frostfsidCreateGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidCreateGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidCreateGroupCmd.Flags().String(namespaceFlag, "", "Namespace where create group") frostfsidCreateGroupCmd.Flags().String(groupNameFlag, "", "Group name, must be unique in namespace") frostfsidCreateGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) @@ -194,7 +195,7 @@ func initFrostfsIDCreateGroupCmd() { func initFrostfsIDDeleteGroupCmd() { frostfsidCmd.AddCommand(frostfsidDeleteGroupCmd) - frostfsidDeleteGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidDeleteGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidDeleteGroupCmd.Flags().String(namespaceFlag, "", "Namespace to delete group") frostfsidDeleteGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") frostfsidDeleteGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) @@ -202,14 +203,14 @@ func initFrostfsIDDeleteGroupCmd() { func initFrostfsIDListGroupsCmd() { frostfsidCmd.AddCommand(frostfsidListGroupsCmd) - frostfsidListGroupsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidListGroupsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidListGroupsCmd.Flags().String(namespaceFlag, "", "Namespace to list groups") frostfsidListGroupsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) } func initFrostfsIDAddSubjectToGroupCmd() { frostfsidCmd.AddCommand(frostfsidAddSubjectToGroupCmd) - frostfsidAddSubjectToGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidAddSubjectToGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidAddSubjectToGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidAddSubjectToGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") frostfsidAddSubjectToGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) @@ -217,7 +218,7 @@ func initFrostfsIDAddSubjectToGroupCmd() { func initFrostfsIDRemoveSubjectFromGroupCmd() { frostfsidCmd.AddCommand(frostfsidRemoveSubjectFromGroupCmd) - frostfsidRemoveSubjectFromGroupCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidRemoveSubjectFromGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidRemoveSubjectFromGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidRemoveSubjectFromGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") frostfsidRemoveSubjectFromGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) @@ -225,7 +226,7 @@ func initFrostfsIDRemoveSubjectFromGroupCmd() { func initFrostfsIDListGroupSubjectsCmd() { frostfsidCmd.AddCommand(frostfsidListGroupSubjectsCmd) - frostfsidListGroupSubjectsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + frostfsidListGroupSubjectsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidListGroupSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace name") frostfsidListGroupSubjectsCmd.Flags().Int64(groupIDFlag, 0, "Group id") frostfsidListGroupSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") @@ -467,5 +468,5 @@ func (f *frostfsidClient) sendWaitRes() (*state.AppExecResult, error) { } f.wCtx.Command.Println("Waiting for transactions to persist...") - return f.roCli.Wait(f.wCtx.SentTxs[0].hash, f.wCtx.SentTxs[0].vub, nil) + return f.roCli.Wait(f.wCtx.SentTxs[0].Hash, f.wCtx.SentTxs[0].Vub, nil) } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index b8e006962..2d82c1237 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -7,6 +7,7 @@ import ( "path/filepath" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -27,7 +28,6 @@ import ( const ( singleAccountName = "single" committeeAccountName = "committee" - consensusAccountName = "consensus" ) func generateAlphabetCreds(cmd *cobra.Command, _ []string) error { @@ -39,8 +39,8 @@ func generateAlphabetCreds(cmd *cobra.Command, _ []string) error { if size == 0 { return errors.New("size must be > 0") } - if size > maxAlphabetNodes { - return ErrTooManyAlphabetNodes + if size > morphUtil.MaxAlphabetNodes { + return morphUtil.ErrTooManyAlphabetNodes } v := viper.GetViper() @@ -109,7 +109,7 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er if err := addMultisigAccount(wallets[i], majCount, committeeAccountName, passwords[i], ps); err != nil { return fmt.Errorf("can't create committee account: %w", err) } - if err := addMultisigAccount(wallets[i], bftCount, consensusAccountName, passwords[i], ps); err != nil { + if err := addMultisigAccount(wallets[i], bftCount, morphUtil.ConsensusAccountName, passwords[i], ps); err != nil { return fmt.Errorf("can't create consentus account: %w", err) } if err := wallets[i].SavePretty(); err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate_test.go index 7f0a2708c..2386b94e2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate_test.go @@ -10,6 +10,7 @@ import ( "sync" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/cli/input" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -87,7 +88,7 @@ func TestGenerateAlphabet(t *testing.T) { err := a.Decrypt(strconv.FormatUint(i, 10), keys.NEP2ScryptParams()) require.NoError(t, err, "can't decrypt account") switch a.Label { - case consensusAccountName: + case util.ConsensusAccountName: require.Equal(t, smartcontract.GetDefaultHonestNodeCount(size), len(a.Contract.Parameters)) case committeeAccountName: require.Equal(t, smartcontract.GetMajorityHonestNodeCount(size), len(a.Contract.Parameters)) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 04e8620cc..2dff2f06f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -4,10 +4,9 @@ import ( "errors" "fmt" "os" - "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/core/native/nativenames" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -15,28 +14,19 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" - "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" "github.com/spf13/viper" ) -const ( - // maxAlphabetNodes is the maximum number of candidates allowed, which is currently limited by the size - // of the invocation script. - // See: https://github.com/nspcc-dev/neo-go/blob/740488f7f35e367eaa99a71c0a609c315fe2b0fc/pkg/core/transaction/witness.go#L10 - maxAlphabetNodes = 22 -) - type cache struct { nnsCs *state.Contract groupKey *keys.PublicKey } type initializeContext struct { - clientContext + morphUtil.ClientContext cache // CommitteeAcc is used for retrieving the committee address and the verification script. CommitteeAcc *wallet.Account @@ -53,8 +43,6 @@ type initializeContext struct { ContractURL string } -var ErrTooManyAlphabetNodes = fmt.Errorf("too many alphabet nodes (maximum allowed is %d)", maxAlphabetNodes) - func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { initCtx, err := newInitializeContext(cmd, viper.GetViper()) if err != nil { @@ -105,8 +93,8 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { } func (c *initializeContext) close() { - if local, ok := c.Client.(*localClient); ok { - err := local.dump() + if local, ok := c.Client.(*morphUtil.LocalClient); ok { + err := local.Dump() if err != nil { c.Command.PrintErrf("Can't write dump: %v\n", err) os.Exit(1) @@ -116,7 +104,7 @@ func (c *initializeContext) close() { func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContext, error) { walletDir := config.ResolveHomePath(viper.GetString(alphabetWalletsFlag)) - wallets, err := getAlphabetWallets(v, walletDir) + wallets, err := morphUtil.GetAlphabetWallets(v, walletDir) if err != nil { return nil, err } @@ -134,12 +122,12 @@ func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContex return nil, err } - committeeAcc, err := getWalletAccount(wallets[0], committeeAccountName) + committeeAcc, err := morphUtil.GetWalletAccount(wallets[0], committeeAccountName) if err != nil { return nil, fmt.Errorf("can't find committee account: %w", err) } - consensusAcc, err := getWalletAccount(wallets[0], consensusAccountName) + consensusAcc, err := morphUtil.GetWalletAccount(wallets[0], morphUtil.ConsensusAccountName) if err != nil { return nil, fmt.Errorf("can't find consensus account: %w", err) } @@ -167,13 +155,13 @@ func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContex return nil, err } - cliCtx, err := defaultClientContext(c, committeeAcc) + cliCtx, err := morphUtil.DefaultClientContext(c, committeeAcc) if err != nil { return nil, fmt.Errorf("client context: %w", err) } initCtx := &initializeContext{ - clientContext: *cliCtx, + ClientContext: *cliCtx, ConsensusAcc: consensusAcc, CommitteeAcc: committeeAcc, ContractWallet: w, @@ -210,16 +198,16 @@ func validateInit(cmd *cobra.Command) error { return nil } -func createClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) (Client, error) { - var c Client +func createClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) (morphUtil.Client, error) { + var c morphUtil.Client var err error if ldf := cmd.Flags().Lookup(localDumpFlag); ldf != nil && ldf.Changed { - if cmd.Flags().Changed(endpointFlag) { - return nil, fmt.Errorf("`%s` and `%s` flags are mutually exclusive", endpointFlag, localDumpFlag) + if cmd.Flags().Changed(morphUtil.EndpointFlag) { + return nil, fmt.Errorf("`%s` and `%s` flags are mutually exclusive", morphUtil.EndpointFlag, localDumpFlag) } - c, err = newLocalClient(cmd, v, wallets, ldf.Value.String()) + c, err = morphUtil.NewLocalClient(cmd, v, wallets, ldf.Value.String()) } else { - c, err = getN3Client(v) + c, err = morphUtil.GetN3Client(v) } if err != nil { return nil, fmt.Errorf("can't create N3 client: %w", err) @@ -249,7 +237,7 @@ func getContractsPath(cmd *cobra.Command, needContracts bool) (string, error) { func createWalletAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) { accounts := make([]*wallet.Account, len(wallets)) for i, w := range wallets { - acc, err := getWalletAccount(w, singleAccountName) + acc, err := morphUtil.GetWalletAccount(w, singleAccountName) if err != nil { return nil, fmt.Errorf("wallet %s is invalid (no single account): %w", w.Path(), err) } @@ -259,7 +247,7 @@ func createWalletAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) { } func (c *initializeContext) awaitTx() error { - return c.clientContext.awaitTx(c.Command) + return c.ClientContext.AwaitTx(c.Command) } func (c *initializeContext) nnsContractState() (*state.Contract, error) { @@ -310,89 +298,6 @@ func (c *initializeContext) getSigner(tryGroup bool, acc *wallet.Account) transa return signer } -func (c *clientContext) awaitTx(cmd *cobra.Command) error { - if len(c.SentTxs) == 0 { - return nil - } - - if local, ok := c.Client.(*localClient); ok { - if err := local.putTransactions(); err != nil { - return fmt.Errorf("can't persist transactions: %w", err) - } - } - - err := awaitTx(cmd, c.Client, c.SentTxs) - c.SentTxs = c.SentTxs[:0] - - return err -} - -func awaitTx(cmd *cobra.Command, c Client, txs []hashVUBPair) error { - cmd.Println("Waiting for transactions to persist...") - - at := trigger.Application - - var retErr error - -loop: - for i := range txs { - var it int - var pollInterval time.Duration - var pollIntervalChanged bool - for { - // We must fetch current height before application log, to avoid race condition. - currBlock, err := c.GetBlockCount() - if err != nil { - return fmt.Errorf("can't fetch current block height: %w", err) - } - res, err := c.GetApplicationLog(txs[i].hash, &at) - if err == nil { - if retErr == nil && len(res.Executions) > 0 && res.Executions[0].VMState != vmstate.Halt { - retErr = fmt.Errorf("tx %d persisted in %s state: %s", - i, res.Executions[0].VMState, res.Executions[0].FaultException) - } - continue loop - } - if txs[i].vub < currBlock { - return fmt.Errorf("tx was not persisted: vub=%d, height=%d", txs[i].vub, currBlock) - } - - pollInterval, pollIntervalChanged = nextPollInterval(it, pollInterval) - if pollIntervalChanged && viper.GetBool(commonflags.Verbose) { - cmd.Printf("Pool interval to check transaction persistence changed: %s\n", pollInterval.String()) - } - - timer := time.NewTimer(pollInterval) - select { - case <-cmd.Context().Done(): - return cmd.Context().Err() - case <-timer.C: - } - - it++ - } - } - - return retErr -} - -func nextPollInterval(it int, previous time.Duration) (time.Duration, bool) { - const minPollInterval = 1 * time.Second - const maxPollInterval = 16 * time.Second - const changeAfter = 5 - if it == 0 { - return minPollInterval, true - } - if it%changeAfter != 0 { - return previous, false - } - nextInterval := previous * 2 - if nextInterval > maxPollInterval { - return maxPollInterval, previous != maxPollInterval - } - return nextInterval, true -} - // sendCommitteeTx creates transaction from script, signs it by committee nodes and sends it to RPC. // If tryGroup is false, global scope is used for the signer (useful when // working with native contracts). @@ -445,24 +350,15 @@ func (c *initializeContext) sendMultiTx(script []byte, tryGroup bool, withConsen return err } if withConsensus { - if err := c.multiSign(tx, consensusAccountName); err != nil { + if err := c.multiSign(tx, morphUtil.ConsensusAccountName); err != nil { return err } } - return c.sendTx(tx, c.Command, false) + return c.SendTx(tx, c.Command, false) } -func getWalletAccount(w *wallet.Wallet, typ string) (*wallet.Account, error) { - for i := range w.Accounts { - if w.Accounts[i].Label == typ { - return w.Accounts[i], nil - } - } - return nil, fmt.Errorf("account for '%s' not found", typ) -} - -func checkNotaryEnabled(c Client) error { +func checkNotaryEnabled(c morphUtil.Client) error { ns, err := c.GetNativeContracts() if err != nil { return fmt.Errorf("can't get native contract hashes: %w", err) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index f715f0e03..9f0e929a8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/common" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -332,7 +333,7 @@ func (c *initializeContext) deployContracts() error { return fmt.Errorf("can't deploy alphabet #%d contract: %w", i, err) } - c.SentTxs = append(c.SentTxs, hashVUBPair{hash: txHash, vub: vub}) + c.SentTxs = append(c.SentTxs, morphUtil.HashVUBPair{Hash: txHash, Vub: vub}) } for _, ctrName := range contractList { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go index 1bf4c4bee..7364ba0e3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go @@ -8,6 +8,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -288,14 +289,14 @@ func parseNNSResolveResult(res stackitem.Item) (util.Uint160, error) { return util.Uint160{}, errors.New("no valid hashes are found") } -func nnsIsAvailable(c Client, nnsHash util.Uint160, name string) (bool, error) { +func nnsIsAvailable(c morphUtil.Client, nnsHash util.Uint160, name string) (bool, error) { switch c.(type) { case *rpcclient.Client: inv := invoker.New(c, nil) reader := nnsClient.NewReader(inv, nnsHash) return reader.IsAvailable(name) default: - b, err := unwrap.Bool(invokeFunction(c, nnsHash, "isAvailable", []any{name}, nil)) + b, err := unwrap.Bool(morphUtil.InvokeFunction(c, nnsHash, "isAvailable", []any{name}, nil)) if err != nil { return false, fmt.Errorf("`isAvailable`: invalid response: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index 92145a796..e8e73ae73 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -5,6 +5,7 @@ import ( "fmt" "math/big" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -78,7 +79,7 @@ func (c *initializeContext) registerCandidateRange(start, end int) error { } } - return c.sendTx(tx, c.Command, true) + return c.SendTx(tx, c.Command, true) } func (c *initializeContext) registerCandidates() error { @@ -156,7 +157,7 @@ func (c *initializeContext) getCandidateRegisterPrice() (int64, error) { return reader.GetRegisterPrice() default: neoHash := neo.Hash - res, err := invokeFunction(c.Client, neoHash, "getRegisterPrice", nil, nil) + res, err := morphUtil.InvokeFunction(c.Client, neoHash, "getRegisterPrice", nil, nil) if err != nil { return 0, err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go b/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go index 46466602e..aed35e230 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go @@ -1,6 +1,7 @@ package morph import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/rolemgmt" @@ -40,6 +41,6 @@ func (c *initializeContext) setRolesFinished() (bool, error) { return false, err } - pubs, err := getDesignatedByRole(c.ReadOnlyInvoker, rolemgmt.Hash, noderoles.NeoFSAlphabet, height) + pubs, err := util.GetDesignatedByRole(c.ReadOnlyInvoker, rolemgmt.Hash, noderoles.NeoFSAlphabet, height) return len(pubs) == len(c.Wallets), err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index 613c63894..cf71a01d6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -42,10 +43,10 @@ func TestInitialize(t *testing.T) { testInitialize(t, 16) }) t.Run("max nodes", func(t *testing.T) { - testInitialize(t, maxAlphabetNodes) + testInitialize(t, util.MaxAlphabetNodes) }) t.Run("too many nodes", func(t *testing.T) { - require.ErrorIs(t, generateTestData(t, t.TempDir(), maxAlphabetNodes+1), ErrTooManyAlphabetNodes) + require.ErrorIs(t, generateTestData(t, t.TempDir(), util.MaxAlphabetNodes+1), util.ErrTooManyAlphabetNodes) }) } @@ -54,7 +55,7 @@ func testInitialize(t *testing.T, committeeSize int) { v := viper.GetViper() require.NoError(t, generateTestData(t, testdataDir, committeeSize)) - v.Set(protoConfigPath, filepath.Join(testdataDir, protoFileName)) + v.Set(util.ProtoConfigPath, filepath.Join(testdataDir, protoFileName)) // Set to the path or remove the next statement to download from the network. require.NoError(t, initCmd.Flags().Set(contractsInitFlag, contractsPath)) @@ -150,33 +151,33 @@ func TestNextPollInterval(t *testing.T) { var pollInterval time.Duration var iteration int - pollInterval, hasChanged := nextPollInterval(iteration, pollInterval) + pollInterval, hasChanged := util.NextPollInterval(iteration, pollInterval) require.True(t, hasChanged) require.Equal(t, time.Second, pollInterval) iteration = 4 - pollInterval, hasChanged = nextPollInterval(iteration, pollInterval) + pollInterval, hasChanged = util.NextPollInterval(iteration, pollInterval) require.False(t, hasChanged) require.Equal(t, time.Second, pollInterval) iteration = 5 - pollInterval, hasChanged = nextPollInterval(iteration, pollInterval) + pollInterval, hasChanged = util.NextPollInterval(iteration, pollInterval) require.True(t, hasChanged) require.Equal(t, 2*time.Second, pollInterval) iteration = 10 - pollInterval, hasChanged = nextPollInterval(iteration, pollInterval) + pollInterval, hasChanged = util.NextPollInterval(iteration, pollInterval) require.True(t, hasChanged) require.Equal(t, 4*time.Second, pollInterval) iteration = 20 pollInterval = 32 * time.Second - pollInterval, hasChanged = nextPollInterval(iteration, pollInterval) + pollInterval, hasChanged = util.NextPollInterval(iteration, pollInterval) require.True(t, hasChanged) // from 32s to 16s require.Equal(t, 16*time.Second, pollInterval) pollInterval = 16 * time.Second - pollInterval, hasChanged = nextPollInterval(iteration, pollInterval) + pollInterval, hasChanged = util.NextPollInterval(iteration, pollInterval) require.False(t, hasChanged) require.Equal(t, 16*time.Second, pollInterval) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go index 12b94a2ed..e9945cee6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go @@ -4,6 +4,7 @@ import ( "fmt" "math/big" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/io" @@ -67,7 +68,7 @@ func (c *initializeContext) transferFunds() error { return fmt.Errorf("can't create transfer transaction: %w", err) } - if err := c.multiSignAndSend(tx, consensusAccountName); err != nil { + if err := c.multiSignAndSend(tx, morphUtil.ConsensusAccountName); err != nil { return fmt.Errorf("can't send transfer transaction: %w", err) } @@ -87,7 +88,7 @@ func (c *initializeContext) multiSignAndSend(tx *transaction.Transaction, accTyp return err } - return c.sendTx(tx, c.Command, false) + return c.SendTx(tx, c.Command, false) } func (c *initializeContext) multiSign(tx *transaction.Transaction, accType string) error { @@ -102,11 +103,11 @@ func (c *initializeContext) multiSign(tx *transaction.Transaction, accType strin // Use parameter context to avoid dealing with signature order. pc := scContext.NewParameterContext("", network, tx) h := c.CommitteeAcc.Contract.ScriptHash() - if accType == consensusAccountName { + if accType == morphUtil.ConsensusAccountName { h = c.ConsensusAcc.Contract.ScriptHash() } for _, w := range c.Wallets { - acc, err := getWalletAccount(w, accType) + acc, err := morphUtil.GetWalletAccount(w, accType) if err != nil { return fmt.Errorf("can't find %s wallet account: %w", accType, err) } @@ -174,7 +175,7 @@ type transferTarget struct { Data any } -func createNEP17MultiTransferTx(c Client, acc *wallet.Account, recipients []transferTarget) (*transaction.Transaction, error) { +func createNEP17MultiTransferTx(c morphUtil.Client, acc *wallet.Account, recipients []transferTarget) (*transaction.Transaction, error) { from := acc.Contract.ScriptHash() w := io.NewBufBinWriter() diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go index bb1de09ca..7d31d1816 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go @@ -2,6 +2,7 @@ package morph import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -11,7 +12,7 @@ import ( ) func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) { - c, err := getN3Client(viper.GetViper()) + c, err := util.GetN3Client(viper.GetViper()) commonCmd.ExitOnErr(cmd, "can't create N3 client: %w", err) inv := invoker.New(c, nil) diff --git a/cmd/frostfs-adm/internal/modules/morph/notary.go b/cmd/frostfs-adm/internal/modules/morph/notary.go index 2459f127b..3e7afdbf2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary.go @@ -5,6 +5,7 @@ import ( "math/big" "strconv" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/cli/input" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -79,7 +80,7 @@ func depositNotary(cmd *cobra.Command, _ []string) error { } func transferGas(cmd *cobra.Command, acc *wallet.Account, accHash util.Uint160, gasAmount fixedn.Fixed8, till int64) error { - c, err := getN3Client(viper.GetViper()) + c, err := morphUtil.GetN3Client(viper.GetViper()) if err != nil { return err } @@ -116,7 +117,7 @@ func transferGas(cmd *cobra.Command, acc *wallet.Account, accHash util.Uint160, return fmt.Errorf("could not send tx: %w", err) } - return awaitTx(cmd, c, []hashVUBPair{{hash: txHash, vub: vub}}) + return morphUtil.AwaitTx(cmd, c, []morphUtil.HashVUBPair{{Hash: txHash, Vub: vub}}) } func openWallet(cmd *cobra.Command) (*wallet.Wallet, error) { diff --git a/cmd/frostfs-adm/internal/modules/morph/policy.go b/cmd/frostfs-adm/internal/modules/morph/policy.go index 005a02711..ba73d132c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy.go @@ -7,6 +7,7 @@ import ( "strings" "text/tabwriter" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -58,7 +59,7 @@ func setPolicyCmd(cmd *cobra.Command, args []string) error { } func dumpPolicyCmd(cmd *cobra.Command, _ []string) error { - c, err := getN3Client(viper.GetViper()) + c, err := util.GetN3Client(viper.GetViper()) commonCmd.ExitOnErr(cmd, "can't create N3 client:", err) inv := invoker.New(c, nil) diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 6cc2d5a96..dad8c4ee4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -1,6 +1,7 @@ package morph import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -9,9 +10,6 @@ const ( alphabetWalletsFlag = "alphabet-wallets" alphabetWalletsFlagDesc = "Path to alphabet wallets dir" alphabetSizeFlag = "size" - endpointFlag = "rpc-endpoint" - endpointFlagDesc = "N3 RPC node endpoint" - endpointFlagShort = "r" storageWalletFlag = "storage-wallet" storageWalletLabelFlag = "label" storageGasCLIFlag = "initial-gas" @@ -42,8 +40,8 @@ const ( walletAccountFlag = "account" notaryDepositTillFlag = "till" localDumpFlag = "local-dump" - protoConfigPath = "protocol" - walletAddressFlag = "wallet-address" + + walletAddressFlag = "wallet-address" ) var ( @@ -68,7 +66,7 @@ var ( Short: "Initialize side chain network with smart-contracts and network settings", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) _ = viper.BindPFlag(epochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) _ = viper.BindPFlag(maxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) _ = viper.BindPFlag(homomorphicHashDisabledInitFlag, cmd.Flags().Lookup(homomorphicHashDisabledCLIFlag)) @@ -76,7 +74,7 @@ var ( _ = viper.BindPFlag(containerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) _ = viper.BindPFlag(containerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) _ = viper.BindPFlag(withdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) - _ = viper.BindPFlag(protoConfigPath, cmd.Flags().Lookup(protoConfigPath)) + _ = viper.BindPFlag(util.ProtoConfigPath, cmd.Flags().Lookup(util.ProtoConfigPath)) }, RunE: initializeSideChainCmd, } @@ -86,7 +84,7 @@ var ( Short: "Generate storage node wallet for the morph network", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) _ = viper.BindPFlag(storageGasConfigFlag, cmd.Flags().Lookup(storageGasCLIFlag)) }, RunE: generateStorageCreds, @@ -97,7 +95,7 @@ var ( Short: "Refill GAS of storage node's wallet in the morph network", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) _ = viper.BindPFlag(refillGasAmountFlag, cmd.Flags().Lookup(refillGasAmountFlag)) }, RunE: func(cmd *cobra.Command, args []string) error { @@ -110,7 +108,7 @@ var ( Short: "Create new FrostFS epoch event in the side chain", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: forceNewEpochCmd, } @@ -121,7 +119,7 @@ var ( Long: `Move nodes to the Offline state in the candidates list and tick an epoch to update the netmap`, PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: removeNodesCmd, } @@ -132,7 +130,7 @@ var ( Short: "Add/update global config value in the FrostFS network", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, Args: cobra.MinimumNArgs(1), RunE: setConfigCmd, @@ -144,7 +142,7 @@ var ( Short: "Set global policy values", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: setPolicyCmd, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -156,7 +154,7 @@ var ( Use: "dump-policy", Short: "Dump FrostFS policy", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: dumpPolicyCmd, } @@ -165,7 +163,7 @@ var ( Use: "dump-hashes", Short: "Dump deployed contract hashes", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: dumpContractHashes, } @@ -174,7 +172,7 @@ var ( Use: "dump-config", Short: "Dump FrostFS network config", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: dumpNetworkConfig, } @@ -183,7 +181,7 @@ var ( Use: "dump-balances", Short: "Dump GAS balances", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: dumpBalances, } @@ -193,7 +191,7 @@ var ( Short: "Update FrostFS contracts", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: updateContracts, } @@ -202,7 +200,7 @@ var ( Use: "dump-containers", Short: "Dump FrostFS containers to file", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: dumpContainers, } @@ -212,7 +210,7 @@ var ( Short: "Restore FrostFS containers from file", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: restoreContainers, } @@ -221,7 +219,7 @@ var ( Use: "list-containers", Short: "List FrostFS containers", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: listContainers, } @@ -230,7 +228,7 @@ var ( Use: "deposit-notary", Short: "Deposit GAS for notary service", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: depositNotary, } @@ -239,7 +237,7 @@ var ( Use: "netmap-candidates", Short: "List netmap candidates nodes", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) }, Run: listNetmapCandidatesNodes, @@ -250,7 +248,7 @@ var ( Short: "Adds account to proxy contract", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, Run: addProxyAccount, } @@ -260,7 +258,7 @@ var ( Short: "Remove from proxy contract", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) - _ = viper.BindPFlag(endpointFlag, cmd.Flags().Lookup(endpointFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, Run: removeProxyAccount, } @@ -313,24 +311,24 @@ func init() { func initProxyAddAccount() { RootCmd.AddCommand(proxyAddAccountCmd) - proxyAddAccountCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + proxyAddAccountCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) proxyAddAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") } func initProxyRemoveAccount() { RootCmd.AddCommand(proxyRemoveAccountCmd) - proxyRemoveAccountCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + proxyRemoveAccountCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) proxyRemoveAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") } func initNetmapCandidatesCmd() { RootCmd.AddCommand(netmapCandidatesCmd) - netmapCandidatesCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + netmapCandidatesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) } func initDepositoryNotaryCmd() { RootCmd.AddCommand(depositNotaryCmd) - depositNotaryCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + depositNotaryCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) depositNotaryCmd.Flags().String(storageWalletFlag, "", "Path to storage node wallet") depositNotaryCmd.Flags().String(walletAccountFlag, "", "Wallet account address") depositNotaryCmd.Flags().String(refillGasAmountFlag, "", "Amount of GAS to deposit") @@ -340,7 +338,7 @@ func initDepositoryNotaryCmd() { func initRefillGasCmd() { RootCmd.AddCommand(refillGasCmd) refillGasCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) - refillGasCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + refillGasCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) refillGasCmd.Flags().String(storageWalletFlag, "", "Path to storage node wallet") refillGasCmd.Flags().String(walletAddressFlag, "", "Address of wallet") refillGasCmd.Flags().String(refillGasAmountFlag, "", "Additional amount of GAS to transfer") @@ -349,21 +347,21 @@ func initRefillGasCmd() { func initListContainersCmd() { RootCmd.AddCommand(listContainersCmd) - listContainersCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + listContainersCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) listContainersCmd.Flags().String(containerContractFlag, "", "Container contract hash (for networks without NNS)") } func initRestoreContainersCmd() { RootCmd.AddCommand(restoreContainersCmd) restoreContainersCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) - restoreContainersCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + restoreContainersCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) restoreContainersCmd.Flags().String(containerDumpFlag, "", "File to restore containers from") restoreContainersCmd.Flags().StringSlice(containerIDsFlag, nil, "Containers to restore") } func initDumpContainersCmd() { RootCmd.AddCommand(dumpContainersCmd) - dumpContainersCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + dumpContainersCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) dumpContainersCmd.Flags().String(containerDumpFlag, "", "File where to save dumped containers") dumpContainersCmd.Flags().String(containerContractFlag, "", "Container contract hash (for networks without NNS)") dumpContainersCmd.Flags().StringSlice(containerIDsFlag, nil, "Containers to dump") @@ -372,7 +370,7 @@ func initDumpContainersCmd() { func initUpdateContractsCmd() { RootCmd.AddCommand(updateContractsCmd) updateContractsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) - updateContractsCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + updateContractsCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) updateContractsCmd.Flags().String(contractsInitFlag, "", contractsInitFlagDesc) updateContractsCmd.Flags().String(contractsURLFlag, "", contractsURLFlagDesc) updateContractsCmd.MarkFlagsMutuallyExclusive(contractsInitFlag, contractsURLFlag) @@ -380,7 +378,7 @@ func initUpdateContractsCmd() { func initDumpBalancesCmd() { RootCmd.AddCommand(dumpBalancesCmd) - dumpBalancesCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + dumpBalancesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) dumpBalancesCmd.Flags().BoolP(dumpBalancesStorageFlag, "s", false, "Dump balances of storage nodes from the current netmap") dumpBalancesCmd.Flags().BoolP(dumpBalancesAlphabetFlag, "a", false, "Dump balances of alphabet contracts") dumpBalancesCmd.Flags().BoolP(dumpBalancesProxyFlag, "p", false, "Dump balances of the proxy contract") @@ -390,52 +388,52 @@ func initDumpBalancesCmd() { func initSetConfigCmd() { RootCmd.AddCommand(setConfig) setConfig.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) - setConfig.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + setConfig.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) setConfig.Flags().Bool(forceConfigSet, false, "Force setting not well-known configuration key") setConfig.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initDumpNetworkConfigCmd() { RootCmd.AddCommand(dumpNetworkConfigCmd) - dumpNetworkConfigCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + dumpNetworkConfigCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) } func initDumpContractHashesCmd() { RootCmd.AddCommand(dumpContractHashesCmd) - dumpContractHashesCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + dumpContractHashesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) dumpContractHashesCmd.Flags().String(customZoneFlag, "", "Custom zone to search.") } func initSetPolicyCmd() { RootCmd.AddCommand(setPolicy) setPolicy.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) - setPolicy.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + setPolicy.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) setPolicy.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initDumpPolicyCmd() { RootCmd.AddCommand(dumpPolicy) - dumpPolicy.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + dumpPolicy.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) } func initRemoveNodesCmd() { RootCmd.AddCommand(removeNodes) removeNodes.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) - removeNodes.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + removeNodes.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) removeNodes.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initForceNewEpochCmd() { RootCmd.AddCommand(forceNewEpoch) forceNewEpoch.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) - forceNewEpoch.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + forceNewEpoch.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) forceNewEpoch.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initGenerateStorageCmd() { RootCmd.AddCommand(generateStorageCmd) generateStorageCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) - generateStorageCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + generateStorageCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) generateStorageCmd.Flags().String(storageWalletFlag, "", "Path to new storage node wallet") generateStorageCmd.Flags().String(storageGasCLIFlag, "", "Initial amount of GAS to transfer") generateStorageCmd.Flags().StringP(storageWalletLabelFlag, "l", "", "Wallet label") @@ -444,7 +442,7 @@ func initGenerateStorageCmd() { func initInitCmd() { RootCmd.AddCommand(initCmd) initCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) - initCmd.Flags().StringP(endpointFlag, endpointFlagShort, "", endpointFlagDesc) + initCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) initCmd.Flags().String(contractsInitFlag, "", contractsInitFlagDesc) initCmd.Flags().String(contractsURLFlag, "", contractsURLFlagDesc) initCmd.Flags().Uint(epochDurationCLIFlag, 240, "Amount of side chain blocks in one FrostFS epoch") @@ -453,7 +451,7 @@ func initInitCmd() { // Defaults are taken from neo-preodolenie. initCmd.Flags().Uint64(containerFeeCLIFlag, 1000, "Container registration fee") initCmd.Flags().Uint64(containerAliasFeeCLIFlag, 500, "Container alias fee") - initCmd.Flags().String(protoConfigPath, "", "Path to the consensus node configuration") + initCmd.Flags().String(util.ProtoConfigPath, "", "Path to the consensus node configuration") initCmd.Flags().String(localDumpFlag, "", "Path to the blocks dump file") initCmd.MarkFlagsMutuallyExclusive(contractsInitFlag, contractsURLFlag) } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/util/const.go new file mode 100644 index 000000000..7278d8357 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/util/const.go @@ -0,0 +1,15 @@ +package util + +const ( + ConsensusAccountName = "consensus" + ProtoConfigPath = "protocol" + + // MaxAlphabetNodes is the maximum number of candidates allowed, which is currently limited by the size + // of the invocation script. + // See: https://github.com/nspcc-dev/neo-go/blob/740488f7f35e367eaa99a71c0a609c315fe2b0fc/pkg/core/transaction/witness.go#L10 + MaxAlphabetNodes = 22 + + EndpointFlag = "rpc-endpoint" + EndpointFlagDesc = "N3 RPC node endpoint" + EndpointFlagShort = "r" +) diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go new file mode 100644 index 000000000..d15d99b93 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go @@ -0,0 +1,90 @@ +package util + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" + "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" + "github.com/nspcc-dev/neo-go/pkg/wallet" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ErrTooManyAlphabetNodes = fmt.Errorf("too many alphabet nodes (maximum allowed is %d)", MaxAlphabetNodes) + +func AwaitTx(cmd *cobra.Command, c Client, txs []HashVUBPair) error { + cmd.Println("Waiting for transactions to persist...") + + at := trigger.Application + + var retErr error + +loop: + for i := range txs { + var it int + var pollInterval time.Duration + var pollIntervalChanged bool + for { + // We must fetch current height before application log, to avoid race condition. + currBlock, err := c.GetBlockCount() + if err != nil { + return fmt.Errorf("can't fetch current block height: %w", err) + } + res, err := c.GetApplicationLog(txs[i].Hash, &at) + if err == nil { + if retErr == nil && len(res.Executions) > 0 && res.Executions[0].VMState != vmstate.Halt { + retErr = fmt.Errorf("tx %d persisted in %s state: %s", + i, res.Executions[0].VMState, res.Executions[0].FaultException) + } + continue loop + } + if txs[i].Vub < currBlock { + return fmt.Errorf("tx was not persisted: Vub=%d, height=%d", txs[i].Vub, currBlock) + } + + pollInterval, pollIntervalChanged = NextPollInterval(it, pollInterval) + if pollIntervalChanged && viper.GetBool(commonflags.Verbose) { + cmd.Printf("Pool interval to check transaction persistence changed: %s\n", pollInterval.String()) + } + + timer := time.NewTimer(pollInterval) + select { + case <-cmd.Context().Done(): + return cmd.Context().Err() + case <-timer.C: + } + + it++ + } + } + + return retErr +} + +func NextPollInterval(it int, previous time.Duration) (time.Duration, bool) { + const minPollInterval = 1 * time.Second + const maxPollInterval = 16 * time.Second + const changeAfter = 5 + if it == 0 { + return minPollInterval, true + } + if it%changeAfter != 0 { + return previous, false + } + nextInterval := previous * 2 + if nextInterval > maxPollInterval { + return maxPollInterval, previous != maxPollInterval + } + return nextInterval, true +} + +func GetWalletAccount(w *wallet.Wallet, typ string) (*wallet.Account, error) { + for i := range w.Accounts { + if w.Accounts[i].Label == typ { + return w.Accounts[i], nil + } + } + return nil, fmt.Errorf("account for '%s' not found", typ) +} diff --git a/cmd/frostfs-adm/internal/modules/morph/local_client.go b/cmd/frostfs-adm/internal/modules/morph/util/local_client.go similarity index 89% rename from cmd/frostfs-adm/internal/modules/morph/local_client.go rename to cmd/frostfs-adm/internal/modules/morph/util/local_client.go index 1a723042c..4b001ca84 100644 --- a/cmd/frostfs-adm/internal/modules/morph/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/local_client.go @@ -1,4 +1,4 @@ -package morph +package util import ( "crypto/elliptic" @@ -38,7 +38,7 @@ import ( "go.uber.org/zap" ) -type localClient struct { +type LocalClient struct { bc *core.Blockchain transactions []*transaction.Transaction dumpPath string @@ -46,8 +46,8 @@ type localClient struct { maxGasInvoke int64 } -func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet, dumpPath string) (*localClient, error) { - cfg, err := config.LoadFile(v.GetString(protoConfigPath)) +func NewLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet, dumpPath string) (*LocalClient, error) { + cfg, err := config.LoadFile(v.GetString(ProtoConfigPath)) if err != nil { return nil, err } @@ -60,7 +60,7 @@ func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet m := smartcontract.GetDefaultHonestNodeCount(int(cfg.ProtocolConfiguration.ValidatorsCount)) accounts := make([]*wallet.Account, len(wallets)) for i := range accounts { - accounts[i], err = getWalletAccount(wallets[i], consensusAccountName) + accounts[i], err = GetWalletAccount(wallets[i], ConsensusAccountName) if err != nil { return nil, err } @@ -102,7 +102,7 @@ func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet } } - return &localClient{ + return &LocalClient{ bc: bc, dumpPath: dumpPath, accounts: accounts[:m], @@ -110,15 +110,15 @@ func newLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet }, nil } -func (l *localClient) GetBlockCount() (uint32, error) { +func (l *LocalClient) GetBlockCount() (uint32, error) { return l.bc.BlockHeight(), nil } -func (l *localClient) GetNativeContracts() ([]state.NativeContract, error) { +func (l *LocalClient) GetNativeContracts() ([]state.NativeContract, error) { return l.bc.GetNatives(), nil } -func (l *localClient) GetApplicationLog(h util.Uint256, t *trigger.Type) (*result.ApplicationLog, error) { +func (l *LocalClient) GetApplicationLog(h util.Uint256, t *trigger.Type) (*result.ApplicationLog, error) { aer, err := l.bc.GetAppExecResults(h, *t) if err != nil { return nil, err @@ -128,13 +128,13 @@ func (l *localClient) GetApplicationLog(h util.Uint256, t *trigger.Type) (*resul return &a, nil } -func (l *localClient) GetCommittee() (keys.PublicKeys, error) { +func (l *LocalClient) GetCommittee() (keys.PublicKeys, error) { // not used by `morph init` command panic("unexpected call") } // InvokeFunction is implemented via `InvokeScript`. -func (l *localClient) InvokeFunction(h util.Uint160, method string, sPrm []smartcontract.Parameter, ss []transaction.Signer) (*result.Invoke, error) { +func (l *LocalClient) InvokeFunction(h util.Uint160, method string, sPrm []smartcontract.Parameter, ss []transaction.Signer) (*result.Invoke, error) { var err error pp := make([]any, len(sPrm)) @@ -145,21 +145,21 @@ func (l *localClient) InvokeFunction(h util.Uint160, method string, sPrm []smart } } - return invokeFunction(l, h, method, pp, ss) + return InvokeFunction(l, h, method, pp, ss) } -func (l *localClient) TerminateSession(_ uuid.UUID) (bool, error) { +func (l *LocalClient) TerminateSession(_ uuid.UUID) (bool, error) { // not used by `morph init` command panic("unexpected call") } -func (l *localClient) TraverseIterator(_, _ uuid.UUID, _ int) ([]stackitem.Item, error) { +func (l *LocalClient) TraverseIterator(_, _ uuid.UUID, _ int) ([]stackitem.Item, error) { // not used by `morph init` command panic("unexpected call") } // GetVersion return default version. -func (l *localClient) GetVersion() (*result.Version, error) { +func (l *LocalClient) GetVersion() (*result.Version, error) { c := l.bc.GetConfig() return &result.Version{ Protocol: result.Protocol{ @@ -180,7 +180,7 @@ func (l *localClient) GetVersion() (*result.Version, error) { }, nil } -func (l *localClient) InvokeContractVerify(util.Uint160, []smartcontract.Parameter, []transaction.Signer, ...transaction.Witness) (*result.Invoke, error) { +func (l *LocalClient) InvokeContractVerify(util.Uint160, []smartcontract.Parameter, []transaction.Signer, ...transaction.Witness) (*result.Invoke, error) { // not used by `morph init` command panic("unexpected call") } @@ -188,7 +188,7 @@ func (l *localClient) InvokeContractVerify(util.Uint160, []smartcontract.Paramet // CalculateNetworkFee calculates network fee for the given transaction. // Copied from neo-go with minor corrections (no need to support non-notary mode): // https://github.com/nspcc-dev/neo-go/blob/v0.103.0/pkg/services/rpcsrv/server.go#L911 -func (l *localClient) CalculateNetworkFee(tx *transaction.Transaction) (int64, error) { +func (l *LocalClient) CalculateNetworkFee(tx *transaction.Transaction) (int64, error) { // Avoid setting hash for this tx: server code doesn't touch client transaction. data := tx.Bytes() tx, err := transaction.NewTransactionFromBytes(data) @@ -259,7 +259,7 @@ func (l *localClient) CalculateNetworkFee(tx *transaction.Transaction) (int64, e return netFee, nil } -func (l *localClient) InvokeScript(script []byte, signers []transaction.Signer) (*result.Invoke, error) { +func (l *LocalClient) InvokeScript(script []byte, signers []transaction.Signer) (*result.Invoke, error) { lastBlock, err := l.bc.GetBlock(l.bc.CurrentBlockHash()) if err != nil { return nil, err @@ -295,7 +295,7 @@ func (l *localClient) InvokeScript(script []byte, signers []transaction.Signer) }, nil } -func (l *localClient) SendRawTransaction(tx *transaction.Transaction) (util.Uint256, error) { +func (l *LocalClient) SendRawTransaction(tx *transaction.Transaction) (util.Uint256, error) { // We need to test that transaction was formed correctly to catch as many errors as we can. bs := tx.Bytes() _, err := transaction.NewTransactionFromBytes(bs) @@ -307,7 +307,7 @@ func (l *localClient) SendRawTransaction(tx *transaction.Transaction) (util.Uint return tx.Hash(), nil } -func (l *localClient) putTransactions() error { +func (l *LocalClient) putTransactions() error { // 1. Prepare new block. lastBlock, err := l.bc.GetBlock(l.bc.CurrentBlockHash()) if err != nil { @@ -349,7 +349,7 @@ func (l *localClient) putTransactions() error { return l.bc.AddBlock(b) } -func invokeFunction(c Client, h util.Uint160, method string, parameters []any, signers []transaction.Signer) (*result.Invoke, error) { +func InvokeFunction(c Client, h util.Uint160, method string, parameters []any, signers []transaction.Signer) (*result.Invoke, error) { w := io.NewBufBinWriter() emit.Array(w.BinWriter, parameters...) emit.AppCallNoArgs(w.BinWriter, h, method, callflag.All) @@ -361,7 +361,7 @@ func invokeFunction(c Client, h util.Uint160, method string, parameters []any, s var errGetDesignatedByRoleResponse = errors.New("`getDesignatedByRole`: invalid response") -func getDesignatedByRole(inv *invoker.Invoker, h util.Uint160, role noderoles.Role, u uint32) (keys.PublicKeys, error) { +func GetDesignatedByRole(inv *invoker.Invoker, h util.Uint160, role noderoles.Role, u uint32) (keys.PublicKeys, error) { arr, err := unwrap.Array(inv.Call(h, "getDesignatedByRole", int64(role), int64(u))) if err != nil { return nil, errGetDesignatedByRoleResponse @@ -382,7 +382,7 @@ func getDesignatedByRole(inv *invoker.Invoker, h util.Uint160, role noderoles.Ro return pubs, nil } -func (l *localClient) dump() (err error) { +func (l *LocalClient) Dump() (err error) { defer l.bc.Close() f, err := os.Create(l.dumpPath) diff --git a/cmd/frostfs-adm/internal/modules/morph/n3client.go b/cmd/frostfs-adm/internal/modules/morph/util/n3client.go similarity index 72% rename from cmd/frostfs-adm/internal/modules/morph/n3client.go rename to cmd/frostfs-adm/internal/modules/morph/util/n3client.go index 584bb8524..0cc751cf1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/n3client.go @@ -1,4 +1,4 @@ -package morph +package util import ( "context" @@ -34,19 +34,19 @@ type Client interface { CalculateNetworkFee(tx *transaction.Transaction) (int64, error) } -type hashVUBPair struct { - hash util.Uint256 - vub uint32 +type HashVUBPair struct { + Hash util.Uint256 + Vub uint32 } -type clientContext struct { +type ClientContext struct { Client Client // a raw neo-go client OR a local chain implementation CommitteeAct *actor.Actor // committee actor with the Global witness scope ReadOnlyInvoker *invoker.Invoker // R/O contract invoker, does not contain any signer - SentTxs []hashVUBPair + SentTxs []HashVUBPair } -func getN3Client(v *viper.Viper) (Client, error) { +func GetN3Client(v *viper.Viper) (Client, error) { // number of opened connections // by neo-go client per one host const ( @@ -55,7 +55,7 @@ func getN3Client(v *viper.Viper) (Client, error) { ) ctx := context.Background() - endpoint := v.GetString(endpointFlag) + endpoint := v.GetString(EndpointFlag) if endpoint == "" { return nil, errors.New("missing endpoint") } @@ -72,20 +72,20 @@ func getN3Client(v *viper.Viper) (Client, error) { return c, nil } -func defaultClientContext(c Client, committeeAcc *wallet.Account) (*clientContext, error) { - commAct, err := newActor(c, committeeAcc) +func DefaultClientContext(c Client, committeeAcc *wallet.Account) (*ClientContext, error) { + commAct, err := NewActor(c, committeeAcc) if err != nil { return nil, err } - return &clientContext{ + return &ClientContext{ Client: c, CommitteeAct: commAct, ReadOnlyInvoker: invoker.New(c, nil), }, nil } -func (c *clientContext) sendTx(tx *transaction.Transaction, cmd *cobra.Command, await bool) error { +func (c *ClientContext) SendTx(tx *transaction.Transaction, cmd *cobra.Command, await bool) error { h, err := c.Client.SendRawTransaction(tx) if err != nil { return err @@ -95,10 +95,27 @@ func (c *clientContext) sendTx(tx *transaction.Transaction, cmd *cobra.Command, return fmt.Errorf("sent and actual tx hashes mismatch:\n\tsent: %v\n\tactual: %v", tx.Hash().StringLE(), h.StringLE()) } - c.SentTxs = append(c.SentTxs, hashVUBPair{hash: h, vub: tx.ValidUntilBlock}) + c.SentTxs = append(c.SentTxs, HashVUBPair{Hash: h, Vub: tx.ValidUntilBlock}) if await { - return c.awaitTx(cmd) + return c.AwaitTx(cmd) } return nil } + +func (c *ClientContext) AwaitTx(cmd *cobra.Command) error { + if len(c.SentTxs) == 0 { + return nil + } + + if local, ok := c.Client.(*LocalClient); ok { + if err := local.putTransactions(); err != nil { + return fmt.Errorf("can't persist transactions: %w", err) + } + } + + err := AwaitTx(cmd, c.Client, c.SentTxs) + c.SentTxs = c.SentTxs[:0] + + return err +} diff --git a/cmd/frostfs-adm/internal/modules/morph/util.go b/cmd/frostfs-adm/internal/modules/morph/util/util.go similarity index 86% rename from cmd/frostfs-adm/internal/modules/morph/util.go rename to cmd/frostfs-adm/internal/modules/morph/util/util.go index 8b5489ab8..bbe635a40 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/util.go @@ -1,4 +1,4 @@ -package morph +package util import ( "errors" @@ -15,19 +15,19 @@ import ( "github.com/spf13/viper" ) -func getAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, error) { - wallets, err := openAlphabetWallets(v, walletDir) +func GetAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, error) { + wallets, err := OpenAlphabetWallets(v, walletDir) if err != nil { return nil, err } - if len(wallets) > maxAlphabetNodes { + if len(wallets) > MaxAlphabetNodes { return nil, ErrTooManyAlphabetNodes } return wallets, nil } -func openAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, error) { +func OpenAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, error) { walletFiles, err := os.ReadDir(walletDir) if err != nil { return nil, fmt.Errorf("can't read alphabet wallets dir: %w", err) @@ -75,7 +75,7 @@ loop: return wallets, nil } -func newActor(c actor.RPCActor, committeeAcc *wallet.Account) (*actor.Actor, error) { +func NewActor(c actor.RPCActor, committeeAcc *wallet.Account) (*actor.Actor, error) { return actor.New(c, []actor.SignerAccount{{ Signer: transaction.Signer{ Account: committeeAcc.Contract.ScriptHash(), From 7954c7f8aff7b5b53983e617a2f8c99e06596ca6 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 09:43:51 +0300 Subject: [PATCH 0880/1943] [#932] adm: Move `NNS` methods to `util` package Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/ape_util.go | 2 +- .../internal/modules/morph/balance.go | 6 +-- .../internal/modules/morph/config.go | 4 +- .../internal/modules/morph/container.go | 4 +- .../internal/modules/morph/deploy.go | 2 +- .../internal/modules/morph/dump_hashes.go | 6 +-- .../internal/modules/morph/epoch.go | 3 +- .../internal/modules/morph/frostfsid.go | 2 +- .../modules/morph/initialize_deploy.go | 8 +-- .../internal/modules/morph/initialize_nns.go | 49 +---------------- .../modules/morph/netmap_candidates.go | 2 +- .../internal/modules/morph/proxy.go | 3 +- .../internal/modules/morph/remove_node.go | 3 +- .../internal/modules/morph/util/initialize.go | 52 +++++++++++++++++++ 14 files changed, 78 insertions(+), 68 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape_util.go index 30668532b..6b174232a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape_util.go @@ -97,7 +97,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *ac nnsCs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) - ch, err = nnsResolveHash(inv, nnsCs.Hash, domainOf(policyContract)) + ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, domainOf(policyContract)) commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err) return morph.NewContractStorage(ac, ch), ac diff --git a/cmd/frostfs-adm/internal/modules/morph/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance.go index b4259e11e..71bdebc70 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance.go @@ -64,7 +64,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err = nnsResolveHash(inv, nnsCs.Hash, domainOf(netmapContract)) + nmHash, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, domainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -137,7 +137,7 @@ func printStorageNodeBalances(cmd *cobra.Command, inv *invoker.Invoker, nmHash u } func printProxyContractBalance(cmd *cobra.Command, inv *invoker.Invoker, nnsHash util.Uint160) error { - h, err := nnsResolveHash(inv, nnsHash, domainOf(proxyContract)) + h, err := morphUtil.NNSResolveHash(inv, nnsHash, domainOf(proxyContract)) if err != nil { return fmt.Errorf("can't get hash of the proxy contract: %w", err) } @@ -170,7 +170,7 @@ func printAlphabetContractBalances(cmd *cobra.Command, c morphUtil.Client, inv * } for i := range alphaList { - h, err := parseNNSResolveResult(alphaRes.Stack[i]) + h, err := morphUtil.ParseNNSResolveResult(alphaRes.Stack[i]) if err != nil { return fmt.Errorf("can't fetch the alphabet contract #%d hash: %w", i, err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index d9ddcb8ad..a8762ac15 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -38,7 +38,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := nnsResolveHash(inv, cs.Hash, domainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(inv, cs.Hash, domainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -96,7 +96,7 @@ func setConfigCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/container.go b/cmd/frostfs-adm/internal/modules/morph/container.go index b9db4c4e5..b93bff8e6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container.go @@ -36,7 +36,7 @@ func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker) (util.Ui if err != nil { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err = nnsResolveHash(inv, nnsCs.Hash, domainOf(containerContract)) + ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, domainOf(containerContract)) if err != nil { return util.Uint160{}, err } @@ -307,7 +307,7 @@ func fetchContainerContractHash(wCtx *initializeContext) (util.Uint160, error) { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err := nnsResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, domainOf(containerContract)) + ch, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, domainOf(containerContract)) if err != nil { return util.Uint160{}, fmt.Errorf("can't fetch container contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/deploy.go b/cmd/frostfs-adm/internal/modules/morph/deploy.go index db54a8db5..a29002d25 100644 --- a/cmd/frostfs-adm/internal/modules/morph/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/deploy.go @@ -89,7 +89,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { domain := ctrName + "." + zone isUpdate, _ := cmd.Flags().GetBool(updateFlag) if isUpdate { - cs.Hash, err = nnsResolveHash(c.ReadOnlyInvoker, nnsCs.Hash, domain) + cs.Hash, err = util.NNSResolveHash(c.ReadOnlyInvoker, nnsCs.Hash, domain) if err != nil { return fmt.Errorf("can't fetch contract hash from NNS: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go index 60e9de9ca..f4c276f51 100644 --- a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go @@ -79,7 +79,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { for i := 0; i < irSize; i++ { info := contractDumpInfo{name: fmt.Sprintf("alphabet %d", i)} - if h, err := parseNNSResolveResult(alphaRes.Stack[i]); err == nil { + if h, err := morphUtil.ParseNNSResolveResult(alphaRes.Stack[i]); err == nil { info.hash = h } infos = append(infos, info) @@ -98,7 +98,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { info := contractDumpInfo{name: ctrName} if len(res.Stack) != 0 { - if h, err := parseNNSResolveResult(res.Stack[0]); err == nil { + if h, err := morphUtil.ParseNNSResolveResult(res.Stack[0]); err == nil { info.hash = h } } @@ -133,7 +133,7 @@ func dumpCustomZoneHashes(cmd *cobra.Command, nnsHash util.Uint160, zone string, return } - h, err := nnsResolveHash(inv, nnsHash, string(bs)) + h, err := morphUtil.NNSResolveHash(inv, nnsHash, string(bs)) if err != nil { cmd.PrintErrf("Could not resolve name %s: %v\n", string(bs), err) return diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/epoch.go index e491e8b8f..86f4f0fea 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/epoch.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + util2 "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" @@ -27,7 +28,7 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) + nmHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go index 8150dcb5f..a420b0775 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -431,7 +431,7 @@ func newFrostfsIDClient(cmd *cobra.Command) (*frostfsidClient, error) { return nil, fmt.Errorf("can't get NNS contract info: %w", err) } - ffsidHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(frostfsIDContract)) + ffsidHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(frostfsIDContract)) if err != nil { return nil, fmt.Errorf("can't get proxy contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 9f0e929a8..fe65a2849 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -203,7 +203,7 @@ func (c *initializeContext) deployOrUpdateContracts(w *io2.BufBinWriter, nnsHash cs := c.getContract(ctrName) method := updateMethodName - ctrHash, err := nnsResolveHash(c.ReadOnlyInvoker, nnsHash, domainOf(ctrName)) + ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, domainOf(ctrName)) if err != nil { if errors.Is(err, errMissingNNSRecord) { // if contract not found we deploy it instead of update @@ -267,7 +267,7 @@ func (c *initializeContext) deployAlphabetAccounts(nnsHash util.Uint160, w *io2. // alphabet contracts should be deployed by individual nodes to get different hashes. for i, acc := range c.Accounts { - ctrHash, err := nnsResolveHash(c.ReadOnlyInvoker, nnsHash, getAlphabetNNSDomain(i)) + ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, getAlphabetNNSDomain(i)) if err != nil { return nil, fmt.Errorf("can't resolve hash for contract update: %w", err) } @@ -609,7 +609,7 @@ func (c *initializeContext) getFrostfsIDAdminFromContract() (util.Uint160, bool, if err != nil { return util.Uint160{}, false, fmt.Errorf("get nns contract: %w", err) } - fidHash, err := nnsResolveHash(c.ReadOnlyInvoker, cs.Hash, domainOf(frostfsIDContract)) + fidHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, domainOf(frostfsIDContract)) if err != nil { return util.Uint160{}, false, fmt.Errorf("resolve frostfsid contract hash: %w", err) } @@ -638,7 +638,7 @@ func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, if err != nil { return nil, fmt.Errorf("get nns contract: %w", err) } - nmHash, err := nnsResolveHash(c.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) + nmHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) if err != nil { return nil, fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go index 7364ba0e3..8267fd04d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go @@ -184,7 +184,7 @@ func (c *initializeContext) nnsRegisterDomainScript(nnsHash, expectedHash util.U return bw.Bytes(), false, nil } - s, err := nnsResolveHash(c.ReadOnlyInvoker, nnsHash, domain) + s, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, domain) if err != nil { return nil, false, err } @@ -220,21 +220,8 @@ func (c *initializeContext) nnsRootRegistered(nnsHash util.Uint160, zone string) var errMissingNNSRecord = errors.New("missing NNS record") -// Returns errMissingNNSRecord if invocation fault exception contains "token not found". -func nnsResolveHash(inv *invoker.Invoker, nnsHash util.Uint160, domain string) (util.Uint160, error) { - item, err := nnsResolve(inv, nnsHash, domain) - if err != nil { - return util.Uint160{}, err - } - return parseNNSResolveResult(item) -} - -func nnsResolve(inv *invoker.Invoker, nnsHash util.Uint160, domain string) (stackitem.Item, error) { - return unwrap.Item(inv.Call(nnsHash, "resolve", domain, int64(nns.TXT))) -} - func nnsResolveKey(inv *invoker.Invoker, nnsHash util.Uint160, domain string) (*keys.PublicKey, error) { - res, err := nnsResolve(inv, nnsHash, domain) + res, err := morphUtil.NNSResolve(inv, nnsHash, domain) if err != nil { return nil, err } @@ -257,38 +244,6 @@ func nnsResolveKey(inv *invoker.Invoker, nnsHash util.Uint160, domain string) (* return nil, errors.New("no valid keys are found") } -// parseNNSResolveResult parses the result of resolving NNS record. -// It works with multiple formats (corresponding to multiple NNS versions). -// If array of hashes is provided, it returns only the first one. -func parseNNSResolveResult(res stackitem.Item) (util.Uint160, error) { - arr, ok := res.Value().([]stackitem.Item) - if !ok { - arr = []stackitem.Item{res} - } - if _, ok := res.Value().(stackitem.Null); ok || len(arr) == 0 { - return util.Uint160{}, errors.New("NNS record is missing") - } - for i := range arr { - bs, err := arr[i].TryBytes() - if err != nil { - continue - } - - // We support several formats for hash encoding, this logic should be maintained in sync - // with nnsResolve from pkg/morph/client/nns.go - h, err := util.Uint160DecodeStringLE(string(bs)) - if err == nil { - return h, nil - } - - h, err = address.StringToUint160(string(bs)) - if err == nil { - return h, nil - } - } - return util.Uint160{}, errors.New("no valid hashes are found") -} - func nnsIsAvailable(c morphUtil.Client, nnsHash util.Uint160, name string) (bool, error) { switch c.(type) { case *rpcclient.Client: diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go index 7d31d1816..cf961cb4d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go @@ -21,7 +21,7 @@ func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) { cs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract info: %w", err) - nmHash, err := nnsResolveHash(inv, cs.Hash, domainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(inv, cs.Hash, domainOf(netmapContract)) commonCmd.ExitOnErr(cmd, "can't get netmap contract hash: %w", err) res, err := inv.Call(nmHash, "netmapCandidates") diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy.go index 802a61246..5a70bfe61 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy.go @@ -3,6 +3,7 @@ package morph import ( "fmt" + util2 "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" @@ -46,7 +47,7 @@ func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error return fmt.Errorf("can't get NNS contract info: %w", err) } - proxyHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(proxyContract)) + proxyHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(proxyContract)) if err != nil { return fmt.Errorf("can't get proxy contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/remove_node.go b/cmd/frostfs-adm/internal/modules/morph/remove_node.go index ea958f2f7..0ae9a1ef7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/remove_node.go +++ b/cmd/frostfs-adm/internal/modules/morph/remove_node.go @@ -5,6 +5,7 @@ import ( "fmt" netmapcontract "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" @@ -40,7 +41,7 @@ func removeNodesCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := nnsResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go index d15d99b93..7b195aebd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go @@ -1,11 +1,18 @@ package util import ( + "errors" "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" @@ -88,3 +95,48 @@ func GetWalletAccount(w *wallet.Wallet, typ string) (*wallet.Account, error) { } return nil, fmt.Errorf("account for '%s' not found", typ) } + +func NNSResolve(inv *invoker.Invoker, nnsHash util.Uint160, domain string) (stackitem.Item, error) { + return unwrap.Item(inv.Call(nnsHash, "resolve", domain, int64(nns.TXT))) +} + +// ParseNNSResolveResult parses the result of resolving NNS record. +// It works with multiple formats (corresponding to multiple NNS versions). +// If array of hashes is provided, it returns only the first one. +func ParseNNSResolveResult(res stackitem.Item) (util.Uint160, error) { + arr, ok := res.Value().([]stackitem.Item) + if !ok { + arr = []stackitem.Item{res} + } + if _, ok := res.Value().(stackitem.Null); ok || len(arr) == 0 { + return util.Uint160{}, errors.New("NNS record is missing") + } + for i := range arr { + bs, err := arr[i].TryBytes() + if err != nil { + continue + } + + // We support several formats for hash encoding, this logic should be maintained in sync + // with NNSResolve from pkg/morph/client/nns.go + h, err := util.Uint160DecodeStringLE(string(bs)) + if err == nil { + return h, nil + } + + h, err = address.StringToUint160(string(bs)) + if err == nil { + return h, nil + } + } + return util.Uint160{}, errors.New("no valid hashes are found") +} + +// NNSResolveHash Returns errMissingNNSRecord if invocation fault exception contains "token not found". +func NNSResolveHash(inv *invoker.Invoker, nnsHash util.Uint160, domain string) (util.Uint160, error) { + item, err := NNSResolve(inv, nnsHash, domain) + if err != nil { + return util.Uint160{}, err + } + return ParseNNSResolveResult(item) +} From 34fcab3498035be3b3fef924d40480327f51ca05 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 09:51:24 +0300 Subject: [PATCH 0881/1943] [#932] adm: Move `alphabet-wallets` flag to `util` package Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/internal/modules/morph/ape.go | 20 ++++---- .../internal/modules/morph/ape_util.go | 2 +- .../internal/modules/morph/deploy.go | 6 +-- .../internal/modules/morph/frostfsid.go | 44 ++++++++--------- .../internal/modules/morph/generate.go | 2 +- .../internal/modules/morph/generate_test.go | 10 ++-- .../internal/modules/morph/initialize.go | 2 +- .../internal/modules/morph/initialize_test.go | 4 +- .../internal/modules/morph/root.go | 48 +++++++++---------- .../internal/modules/morph/util/const.go | 8 ++-- 10 files changed, 73 insertions(+), 73 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape.go index 5ff6e7ab8..f778a6944 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape.go @@ -43,7 +43,7 @@ var ( Short: "Add rule chain", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) }, Run: addRuleChain, } @@ -53,7 +53,7 @@ var ( Short: "Remove rule chain", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) }, Run: removeRuleChain, } @@ -63,7 +63,7 @@ var ( Short: "List rule chains", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) }, Run: listRuleChains, } @@ -73,7 +73,7 @@ var ( Short: "Set admin", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) }, Run: setAdmin, } @@ -83,7 +83,7 @@ var ( Short: "Get admin", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) }, Run: getAdmin, } @@ -93,7 +93,7 @@ func initAddRuleChainCmd() { apeCmd.AddCommand(addRuleChainCmd) addRuleChainCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - addRuleChainCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + addRuleChainCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) addRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc) _ = addRuleChainCmd.MarkFlagRequired(targetTypeFlag) @@ -112,7 +112,7 @@ func initRemoveRuleChainCmd() { apeCmd.AddCommand(removeRuleChainCmd) removeRuleChainCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - removeRuleChainCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + removeRuleChainCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) removeRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc) _ = removeRuleChainCmd.MarkFlagRequired(targetTypeFlag) @@ -126,7 +126,7 @@ func initListRuleChainsCmd() { apeCmd.AddCommand(listRuleChainsCmd) listRuleChainsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - listRuleChainsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + listRuleChainsCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) listRuleChainsCmd.Flags().StringP(targetTypeFlag, "t", "", targetTypeDesc) _ = listRuleChainsCmd.MarkFlagRequired(targetTypeFlag) listRuleChainsCmd.Flags().String(targetNameFlag, "", targetNameDesc) @@ -138,7 +138,7 @@ func initSetAdminCmd() { apeCmd.AddCommand(setAdminCmd) setAdminCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - setAdminCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + setAdminCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) setAdminCmd.Flags().String(addrAdminFlag, "", addrAdminDesc) _ = setAdminCmd.MarkFlagRequired(addrAdminFlag) } @@ -147,7 +147,7 @@ func initGetAdminCmd() { apeCmd.AddCommand(getAdminCmd) getAdminCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - getAdminCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + getAdminCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func addRuleChain(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-adm/internal/modules/morph/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape_util.go index 6b174232a..e89c833db 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape_util.go @@ -81,7 +81,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *ac c, err := morphUtil.GetN3Client(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) - walletDir := config.ResolveHomePath(viper.GetString(alphabetWalletsFlag)) + walletDir := config.ResolveHomePath(viper.GetString(morphUtil.AlphabetWalletsFlag)) wallets, err := morphUtil.GetAlphabetWallets(v, walletDir) commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) diff --git a/cmd/frostfs-adm/internal/modules/morph/deploy.go b/cmd/frostfs-adm/internal/modules/morph/deploy.go index a29002d25..85c207d79 100644 --- a/cmd/frostfs-adm/internal/modules/morph/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/deploy.go @@ -38,7 +38,7 @@ Compiled contract file name must contain '_contract.nef' suffix. Contract's manifest file name must be 'config.json'. NNS name is taken by stripping '_contract.nef' from the NEF file (similar to frostfs contracts).`, PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: deployContractCmd, @@ -47,8 +47,8 @@ NNS name is taken by stripping '_contract.nef' from the NEF file (similar to fro func init() { ff := deployCmd.Flags() - ff.String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) - _ = deployCmd.MarkFlagFilename(alphabetWalletsFlag) + ff.String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) + _ = deployCmd.MarkFlagFilename(util.AlphabetWalletsFlag) ff.StringP(util.EndpointFlag, "r", "", util.EndpointFlagDesc) ff.String(contractPathFlag, "", "Path to the contract directory") diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go index a420b0775..c114e48e9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -41,7 +41,7 @@ var ( Use: "create-namespace", Short: "Create new namespace in frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidCreateNamespace, @@ -51,7 +51,7 @@ var ( Use: "list-namespaces", Short: "List all namespaces in frostfsid", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidListNamespaces, @@ -61,7 +61,7 @@ var ( Use: "create-subject", Short: "Create subject in frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidCreateSubject, @@ -71,7 +71,7 @@ var ( Use: "delete-subject", Short: "Delete subject from frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidDeleteSubject, @@ -81,7 +81,7 @@ var ( Use: "list-subjects", Short: "List subjects in namespace", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidListSubjects, @@ -91,7 +91,7 @@ var ( Use: "create-group", Short: "Create group in frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidCreateGroup, @@ -101,7 +101,7 @@ var ( Use: "delete-group", Short: "Delete group from frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidDeleteGroup, @@ -111,7 +111,7 @@ var ( Use: "list-groups", Short: "List groups in namespace", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidListGroups, @@ -121,7 +121,7 @@ var ( Use: "add-subject-to-group", Short: "Add subject to group", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidAddSubjectToGroup, @@ -131,7 +131,7 @@ var ( Use: "remove-subject-from-group", Short: "Remove subject from group", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidRemoveSubjectFromGroup, @@ -141,7 +141,7 @@ var ( Use: "list-group-subjects", Short: "List subjects in group", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) }, Run: frostfsidListGroupSubjects, @@ -152,13 +152,13 @@ func initFrostfsIDCreateNamespaceCmd() { frostfsidCmd.AddCommand(frostfsidCreateNamespaceCmd) frostfsidCreateNamespaceCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidCreateNamespaceCmd.Flags().String(namespaceFlag, "", "Namespace name to create") - frostfsidCreateNamespaceCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + frostfsidCreateNamespaceCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDListNamespacesCmd() { frostfsidCmd.AddCommand(frostfsidListNamespacesCmd) frostfsidListNamespacesCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - frostfsidListNamespacesCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + frostfsidListNamespacesCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDCreateSubjectCmd() { @@ -167,14 +167,14 @@ func initFrostfsIDCreateSubjectCmd() { frostfsidCreateSubjectCmd.Flags().String(namespaceFlag, "", "Namespace where create subject") frostfsidCreateSubjectCmd.Flags().String(subjectNameFlag, "", "Subject name, must be unique in namespace") frostfsidCreateSubjectCmd.Flags().String(subjectKeyFlag, "", "Subject hex-encoded public key") - frostfsidCreateSubjectCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + frostfsidCreateSubjectCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDDeleteSubjectCmd() { frostfsidCmd.AddCommand(frostfsidDeleteSubjectCmd) frostfsidDeleteSubjectCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidDeleteSubjectCmd.Flags().String(subjectAddressFlag, "", "Subject address") - frostfsidDeleteSubjectCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + frostfsidDeleteSubjectCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDListSubjectsCmd() { @@ -182,7 +182,7 @@ func initFrostfsIDListSubjectsCmd() { frostfsidListSubjectsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidListSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace to list subjects") frostfsidListSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") - frostfsidListSubjectsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + frostfsidListSubjectsCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDCreateGroupCmd() { @@ -190,7 +190,7 @@ func initFrostfsIDCreateGroupCmd() { frostfsidCreateGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidCreateGroupCmd.Flags().String(namespaceFlag, "", "Namespace where create group") frostfsidCreateGroupCmd.Flags().String(groupNameFlag, "", "Group name, must be unique in namespace") - frostfsidCreateGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + frostfsidCreateGroupCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDDeleteGroupCmd() { @@ -198,14 +198,14 @@ func initFrostfsIDDeleteGroupCmd() { frostfsidDeleteGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidDeleteGroupCmd.Flags().String(namespaceFlag, "", "Namespace to delete group") frostfsidDeleteGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") - frostfsidDeleteGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + frostfsidDeleteGroupCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDListGroupsCmd() { frostfsidCmd.AddCommand(frostfsidListGroupsCmd) frostfsidListGroupsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidListGroupsCmd.Flags().String(namespaceFlag, "", "Namespace to list groups") - frostfsidListGroupsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + frostfsidListGroupsCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDAddSubjectToGroupCmd() { @@ -213,7 +213,7 @@ func initFrostfsIDAddSubjectToGroupCmd() { frostfsidAddSubjectToGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidAddSubjectToGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidAddSubjectToGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") - frostfsidAddSubjectToGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + frostfsidAddSubjectToGroupCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDRemoveSubjectFromGroupCmd() { @@ -221,7 +221,7 @@ func initFrostfsIDRemoveSubjectFromGroupCmd() { frostfsidRemoveSubjectFromGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidRemoveSubjectFromGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidRemoveSubjectFromGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") - frostfsidRemoveSubjectFromGroupCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + frostfsidRemoveSubjectFromGroupCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDListGroupSubjectsCmd() { @@ -230,7 +230,7 @@ func initFrostfsIDListGroupSubjectsCmd() { frostfsidListGroupSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace name") frostfsidListGroupSubjectsCmd.Flags().Int64(groupIDFlag, 0, "Group id") frostfsidListGroupSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") - frostfsidListGroupSubjectsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + frostfsidListGroupSubjectsCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func frostfsidCreateNamespace(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index 2d82c1237..fd27a3c2c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -44,7 +44,7 @@ func generateAlphabetCreds(cmd *cobra.Command, _ []string) error { } v := viper.GetViper() - walletDir := config.ResolveHomePath(viper.GetString(alphabetWalletsFlag)) + walletDir := config.ResolveHomePath(viper.GetString(morphUtil.AlphabetWalletsFlag)) pwds, err := initializeWallets(v, walletDir, int(size)) if err != nil { return err diff --git a/cmd/frostfs-adm/internal/modules/morph/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate_test.go index 2386b94e2..79a0ea7de 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate_test.go @@ -34,28 +34,28 @@ func TestGenerateAlphabet(t *testing.T) { t.Run("zero size", func(t *testing.T) { buf.Reset() - v.Set(alphabetWalletsFlag, walletDir) + v.Set(util.AlphabetWalletsFlag, walletDir) require.NoError(t, cmd.Flags().Set(alphabetSizeFlag, "0")) buf.WriteString("pass\r") require.Error(t, generateAlphabetCreds(cmd, nil)) }) t.Run("no password provided", func(t *testing.T) { buf.Reset() - v.Set(alphabetWalletsFlag, walletDir) + v.Set(util.AlphabetWalletsFlag, walletDir) require.NoError(t, cmd.Flags().Set(alphabetSizeFlag, "1")) require.Error(t, generateAlphabetCreds(cmd, nil)) }) t.Run("missing directory", func(t *testing.T) { buf.Reset() dir := filepath.Join(os.TempDir(), "notexist."+strconv.FormatUint(rand.Uint64(), 10)) - v.Set(alphabetWalletsFlag, dir) + v.Set(util.AlphabetWalletsFlag, dir) require.NoError(t, cmd.Flags().Set(alphabetSizeFlag, "1")) buf.WriteString("pass\r") require.Error(t, generateAlphabetCreds(cmd, nil)) }) t.Run("no password for contract group wallet", func(t *testing.T) { buf.Reset() - v.Set(alphabetWalletsFlag, walletDir) + v.Set(util.AlphabetWalletsFlag, walletDir) require.NoError(t, cmd.Flags().Set(alphabetSizeFlag, strconv.FormatUint(size, 10))) for i := uint64(0); i < size; i++ { buf.WriteString(strconv.FormatUint(i, 10) + "\r") @@ -64,7 +64,7 @@ func TestGenerateAlphabet(t *testing.T) { }) buf.Reset() - v.Set(alphabetWalletsFlag, walletDir) + v.Set(util.AlphabetWalletsFlag, walletDir) require.NoError(t, generateAlphabetCmd.Flags().Set(alphabetSizeFlag, strconv.FormatUint(size, 10))) for i := uint64(0); i < size; i++ { buf.WriteString(strconv.FormatUint(i, 10) + "\r") diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 2dff2f06f..51d00f9f4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -103,7 +103,7 @@ func (c *initializeContext) close() { } func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContext, error) { - walletDir := config.ResolveHomePath(viper.GetString(alphabetWalletsFlag)) + walletDir := config.ResolveHomePath(viper.GetString(morphUtil.AlphabetWalletsFlag)) wallets, err := morphUtil.GetAlphabetWallets(v, walletDir) if err != nil { return nil, err diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index cf71a01d6..bd7c8ee1f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -62,7 +62,7 @@ func testInitialize(t *testing.T, committeeSize int) { dumpPath := filepath.Join(testdataDir, "out") require.NoError(t, initCmd.Flags().Set(localDumpFlag, dumpPath)) - v.Set(alphabetWalletsFlag, testdataDir) + v.Set(util.AlphabetWalletsFlag, testdataDir) v.Set(epochDurationInitFlag, 1) v.Set(maxObjectSizeInitFlag, 1024) @@ -93,7 +93,7 @@ func testInitialize(t *testing.T, committeeSize int) { func generateTestData(t *testing.T, dir string, size int) error { v := viper.GetViper() - v.Set(alphabetWalletsFlag, dir) + v.Set(util.AlphabetWalletsFlag, dir) sizeStr := strconv.FormatUint(uint64(size), 10) if err := generateAlphabetCmd.Flags().Set(alphabetSizeFlag, sizeStr); err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index dad8c4ee4..5a4fa0063 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -7,8 +7,6 @@ import ( ) const ( - alphabetWalletsFlag = "alphabet-wallets" - alphabetWalletsFlagDesc = "Path to alphabet wallets dir" alphabetSizeFlag = "size" storageWalletFlag = "storage-wallet" storageWalletLabelFlag = "label" @@ -56,7 +54,7 @@ var ( Short: "Generate alphabet wallets for consensus nodes of the morph network", PreRun: func(cmd *cobra.Command, _ []string) { // PreRun fixes https://github.com/spf13/viper/issues/233 - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) }, RunE: generateAlphabetCreds, } @@ -65,7 +63,7 @@ var ( Use: "init", Short: "Initialize side chain network with smart-contracts and network settings", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) _ = viper.BindPFlag(epochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) _ = viper.BindPFlag(maxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) @@ -83,7 +81,7 @@ var ( Use: "generate-storage-wallet", Short: "Generate storage node wallet for the morph network", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) _ = viper.BindPFlag(storageGasConfigFlag, cmd.Flags().Lookup(storageGasCLIFlag)) }, @@ -94,7 +92,7 @@ var ( Use: "refill-gas", Short: "Refill GAS of storage node's wallet in the morph network", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) _ = viper.BindPFlag(refillGasAmountFlag, cmd.Flags().Lookup(refillGasAmountFlag)) }, @@ -107,7 +105,7 @@ var ( Use: "force-new-epoch", Short: "Create new FrostFS epoch event in the side chain", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: forceNewEpochCmd, @@ -118,7 +116,7 @@ var ( Short: "Remove storage nodes from the netmap", Long: `Move nodes to the Offline state in the candidates list and tick an epoch to update the netmap`, PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: removeNodesCmd, @@ -129,7 +127,7 @@ var ( DisableFlagsInUseLine: true, Short: "Add/update global config value in the FrostFS network", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, Args: cobra.MinimumNArgs(1), @@ -141,7 +139,7 @@ var ( DisableFlagsInUseLine: true, Short: "Set global policy values", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: setPolicyCmd, @@ -190,7 +188,7 @@ var ( Use: "update-contracts", Short: "Update FrostFS contracts", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: updateContracts, @@ -209,7 +207,7 @@ var ( Use: "restore-containers", Short: "Restore FrostFS containers from file", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, RunE: restoreContainers, @@ -238,7 +236,7 @@ var ( Short: "List netmap candidates nodes", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) }, Run: listNetmapCandidatesNodes, } @@ -247,7 +245,7 @@ var ( Use: "proxy-add-account", Short: "Adds account to proxy contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, Run: addProxyAccount, @@ -257,7 +255,7 @@ var ( Use: "proxy-remove-account", Short: "Remove from proxy contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(alphabetWalletsFlag, cmd.Flags().Lookup(alphabetWalletsFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) }, Run: removeProxyAccount, @@ -337,7 +335,7 @@ func initDepositoryNotaryCmd() { func initRefillGasCmd() { RootCmd.AddCommand(refillGasCmd) - refillGasCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + refillGasCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) refillGasCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) refillGasCmd.Flags().String(storageWalletFlag, "", "Path to storage node wallet") refillGasCmd.Flags().String(walletAddressFlag, "", "Address of wallet") @@ -353,7 +351,7 @@ func initListContainersCmd() { func initRestoreContainersCmd() { RootCmd.AddCommand(restoreContainersCmd) - restoreContainersCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + restoreContainersCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) restoreContainersCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) restoreContainersCmd.Flags().String(containerDumpFlag, "", "File to restore containers from") restoreContainersCmd.Flags().StringSlice(containerIDsFlag, nil, "Containers to restore") @@ -369,7 +367,7 @@ func initDumpContainersCmd() { func initUpdateContractsCmd() { RootCmd.AddCommand(updateContractsCmd) - updateContractsCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + updateContractsCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) updateContractsCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) updateContractsCmd.Flags().String(contractsInitFlag, "", contractsInitFlagDesc) updateContractsCmd.Flags().String(contractsURLFlag, "", contractsURLFlagDesc) @@ -387,7 +385,7 @@ func initDumpBalancesCmd() { func initSetConfigCmd() { RootCmd.AddCommand(setConfig) - setConfig.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + setConfig.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) setConfig.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) setConfig.Flags().Bool(forceConfigSet, false, "Force setting not well-known configuration key") setConfig.Flags().String(localDumpFlag, "", "Path to the blocks dump file") @@ -406,7 +404,7 @@ func initDumpContractHashesCmd() { func initSetPolicyCmd() { RootCmd.AddCommand(setPolicy) - setPolicy.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + setPolicy.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) setPolicy.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) setPolicy.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } @@ -418,21 +416,21 @@ func initDumpPolicyCmd() { func initRemoveNodesCmd() { RootCmd.AddCommand(removeNodes) - removeNodes.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + removeNodes.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) removeNodes.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) removeNodes.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initForceNewEpochCmd() { RootCmd.AddCommand(forceNewEpoch) - forceNewEpoch.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + forceNewEpoch.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) forceNewEpoch.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) forceNewEpoch.Flags().String(localDumpFlag, "", "Path to the blocks dump file") } func initGenerateStorageCmd() { RootCmd.AddCommand(generateStorageCmd) - generateStorageCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + generateStorageCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) generateStorageCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) generateStorageCmd.Flags().String(storageWalletFlag, "", "Path to new storage node wallet") generateStorageCmd.Flags().String(storageGasCLIFlag, "", "Initial amount of GAS to transfer") @@ -441,7 +439,7 @@ func initGenerateStorageCmd() { func initInitCmd() { RootCmd.AddCommand(initCmd) - initCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + initCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) initCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) initCmd.Flags().String(contractsInitFlag, "", contractsInitFlagDesc) initCmd.Flags().String(contractsURLFlag, "", contractsURLFlagDesc) @@ -458,7 +456,7 @@ func initInitCmd() { func initGenerateAlphabetCmd() { RootCmd.AddCommand(generateAlphabetCmd) - generateAlphabetCmd.Flags().String(alphabetWalletsFlag, "", alphabetWalletsFlagDesc) + generateAlphabetCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) generateAlphabetCmd.Flags().Uint(alphabetSizeFlag, 7, "Amount of alphabet wallets to generate") } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/util/const.go index 7278d8357..4f11fd04d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/const.go @@ -9,7 +9,9 @@ const ( // See: https://github.com/nspcc-dev/neo-go/blob/740488f7f35e367eaa99a71c0a609c315fe2b0fc/pkg/core/transaction/witness.go#L10 MaxAlphabetNodes = 22 - EndpointFlag = "rpc-endpoint" - EndpointFlagDesc = "N3 RPC node endpoint" - EndpointFlagShort = "r" + EndpointFlag = "rpc-endpoint" + EndpointFlagDesc = "N3 RPC node endpoint" + EndpointFlagShort = "r" + AlphabetWalletsFlag = "alphabet-wallets" + AlphabetWalletsFlagDesc = "Path to alphabet wallets dir" ) From fdeb99c52f1f88a794eb5e1bce0a6a725bccad60 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 09:58:12 +0300 Subject: [PATCH 0882/1943] [#932] adm: Move `DomainOf` to `util` package Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/internal/modules/morph/ape_util.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/balance.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/config.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/container.go | 4 ++-- .../internal/modules/morph/dump_hashes.go | 2 +- cmd/frostfs-adm/internal/modules/morph/epoch.go | 2 +- cmd/frostfs-adm/internal/modules/morph/frostfsid.go | 2 +- cmd/frostfs-adm/internal/modules/morph/generate.go | 11 +++-------- .../internal/modules/morph/generate_test.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/initialize.go | 6 +++--- .../internal/modules/morph/initialize_deploy.go | 12 ++++-------- .../internal/modules/morph/initialize_register.go | 2 +- .../internal/modules/morph/initialize_test.go | 2 +- .../internal/modules/morph/initialize_transfer.go | 2 +- .../internal/modules/morph/netmap_candidates.go | 2 +- cmd/frostfs-adm/internal/modules/morph/proxy.go | 2 +- .../internal/modules/morph/remove_node.go | 2 +- cmd/frostfs-adm/internal/modules/morph/util/const.go | 3 +++ .../internal/modules/morph/util/initialize.go | 4 ++++ 19 files changed, 36 insertions(+), 38 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape_util.go index e89c833db..59e85dda1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape_util.go @@ -85,7 +85,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *ac wallets, err := morphUtil.GetAlphabetWallets(v, walletDir) commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) - committeeAcc, err := morphUtil.GetWalletAccount(wallets[0], committeeAccountName) + committeeAcc, err := morphUtil.GetWalletAccount(wallets[0], morphUtil.CommitteeAccountName) commonCmd.ExitOnErr(cmd, "can't find committee account: %w", err) ac, err := morphUtil.NewActor(c, committeeAcc) @@ -97,7 +97,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *ac nnsCs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) - ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, domainOf(policyContract)) + ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(policyContract)) commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err) return morph.NewContractStorage(ac, ch), ac diff --git a/cmd/frostfs-adm/internal/modules/morph/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance.go index 71bdebc70..8ee7aeeed 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance.go @@ -64,7 +64,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, domainOf(netmapContract)) + nmHash, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -137,7 +137,7 @@ func printStorageNodeBalances(cmd *cobra.Command, inv *invoker.Invoker, nmHash u } func printProxyContractBalance(cmd *cobra.Command, inv *invoker.Invoker, nnsHash util.Uint160) error { - h, err := morphUtil.NNSResolveHash(inv, nnsHash, domainOf(proxyContract)) + h, err := morphUtil.NNSResolveHash(inv, nnsHash, morphUtil.DomainOf(proxyContract)) if err != nil { return fmt.Errorf("can't get hash of the proxy contract: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index a8762ac15..3892d31b1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -38,7 +38,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(inv, cs.Hash, domainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -96,7 +96,7 @@ func setConfigCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/container.go b/cmd/frostfs-adm/internal/modules/morph/container.go index b93bff8e6..5f59f7db5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container.go @@ -36,7 +36,7 @@ func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker) (util.Ui if err != nil { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, domainOf(containerContract)) + ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(containerContract)) if err != nil { return util.Uint160{}, err } @@ -307,7 +307,7 @@ func fetchContainerContractHash(wCtx *initializeContext) (util.Uint160, error) { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, domainOf(containerContract)) + ch, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, morphUtil.DomainOf(containerContract)) if err != nil { return util.Uint160{}, fmt.Errorf("can't fetch container contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go index f4c276f51..05ef72843 100644 --- a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go @@ -89,7 +89,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { for _, ctrName := range contractList { bw.Reset() emit.AppCall(bw.BinWriter, cs.Hash, "resolve", callflag.ReadOnly, - domainOf(ctrName), int64(nns.TXT)) + morphUtil.DomainOf(ctrName), int64(nns.TXT)) res, err := c.InvokeScript(bw.Bytes(), nil) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/epoch.go index 86f4f0fea..274568afc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/epoch.go @@ -28,7 +28,7 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) + nmHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util2.DomainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go index c114e48e9..083fc9ed9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -431,7 +431,7 @@ func newFrostfsIDClient(cmd *cobra.Command) (*frostfsidClient, error) { return nil, fmt.Errorf("can't get NNS contract info: %w", err) } - ffsidHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(frostfsIDContract)) + ffsidHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(frostfsIDContract)) if err != nil { return nil, fmt.Errorf("can't get proxy contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index fd27a3c2c..a7561b266 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -25,11 +25,6 @@ import ( "golang.org/x/sync/errgroup" ) -const ( - singleAccountName = "single" - committeeAccountName = "committee" -) - func generateAlphabetCreds(cmd *cobra.Command, _ []string) error { // alphabet size is not part of the config size, err := cmd.Flags().GetUint(alphabetSizeFlag) @@ -87,7 +82,7 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er if err != nil { return nil, fmt.Errorf("can't create wallet: %w", err) } - if err := w.CreateAccount(singleAccountName, password); err != nil { + if err := w.CreateAccount(morphUtil.SingleAccountName, password); err != nil { return nil, fmt.Errorf("can't create account: %w", err) } @@ -106,7 +101,7 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er i := i ps := pubs.Copy() errG.Go(func() error { - if err := addMultisigAccount(wallets[i], majCount, committeeAccountName, passwords[i], ps); err != nil { + if err := addMultisigAccount(wallets[i], majCount, morphUtil.CommitteeAccountName, passwords[i], ps); err != nil { return fmt.Errorf("can't create committee account: %w", err) } if err := addMultisigAccount(wallets[i], bftCount, morphUtil.ConsensusAccountName, passwords[i], ps); err != nil { @@ -182,7 +177,7 @@ func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error } if label == "" { - label = singleAccountName + label = morphUtil.SingleAccountName } if err := w.CreateAccount(label, password); err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate_test.go index 79a0ea7de..207366ec7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate_test.go @@ -90,10 +90,10 @@ func TestGenerateAlphabet(t *testing.T) { switch a.Label { case util.ConsensusAccountName: require.Equal(t, smartcontract.GetDefaultHonestNodeCount(size), len(a.Contract.Parameters)) - case committeeAccountName: + case util.CommitteeAccountName: require.Equal(t, smartcontract.GetMajorityHonestNodeCount(size), len(a.Contract.Parameters)) default: - require.Equal(t, singleAccountName, a.Label) + require.Equal(t, util.SingleAccountName, a.Label) } } }() diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 51d00f9f4..a432fbc7e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -122,7 +122,7 @@ func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContex return nil, err } - committeeAcc, err := morphUtil.GetWalletAccount(wallets[0], committeeAccountName) + committeeAcc, err := morphUtil.GetWalletAccount(wallets[0], morphUtil.CommitteeAccountName) if err != nil { return nil, fmt.Errorf("can't find committee account: %w", err) } @@ -237,7 +237,7 @@ func getContractsPath(cmd *cobra.Command, needContracts bool) (string, error) { func createWalletAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) { accounts := make([]*wallet.Account, len(wallets)) for i, w := range wallets { - acc, err := morphUtil.GetWalletAccount(w, singleAccountName) + acc, err := morphUtil.GetWalletAccount(w, morphUtil.SingleAccountName) if err != nil { return nil, fmt.Errorf("wallet %s is invalid (no single account): %w", w.Path(), err) } @@ -346,7 +346,7 @@ func (c *initializeContext) sendMultiTx(script []byte, tryGroup bool, withConsen return fmt.Errorf("could not perform test invocation: %w", err) } - if err := c.multiSign(tx, committeeAccountName); err != nil { + if err := c.multiSign(tx, morphUtil.CommitteeAccountName); err != nil { return err } if withConsensus { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index fe65a2849..66caf8c19 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -92,10 +92,6 @@ const ( deployMethodName = "deploy" ) -func domainOf(contract string) string { - return contract + ".frostfs" -} - func (c *initializeContext) deployNNS(method string) error { cs := c.getContract(nnsContract) h := cs.Hash @@ -133,7 +129,7 @@ func (c *initializeContext) deployNNS(method string) error { return fmt.Errorf("failed to create deploy tx for %s: %w", nnsContract, err) } - if err := c.multiSignAndSend(tx, committeeAccountName); err != nil { + if err := c.multiSignAndSend(tx, morphUtil.CommitteeAccountName); err != nil { return fmt.Errorf("can't send deploy transaction: %w", err) } @@ -203,7 +199,7 @@ func (c *initializeContext) deployOrUpdateContracts(w *io2.BufBinWriter, nnsHash cs := c.getContract(ctrName) method := updateMethodName - ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, domainOf(ctrName)) + ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, morphUtil.DomainOf(ctrName)) if err != nil { if errors.Is(err, errMissingNNSRecord) { // if contract not found we deploy it instead of update @@ -609,7 +605,7 @@ func (c *initializeContext) getFrostfsIDAdminFromContract() (util.Uint160, bool, if err != nil { return util.Uint160{}, false, fmt.Errorf("get nns contract: %w", err) } - fidHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, domainOf(frostfsIDContract)) + fidHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(frostfsIDContract)) if err != nil { return util.Uint160{}, false, fmt.Errorf("resolve frostfsid contract hash: %w", err) } @@ -638,7 +634,7 @@ func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, if err != nil { return nil, fmt.Errorf("get nns contract: %w", err) } - nmHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) + nmHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(netmapContract)) if err != nil { return nil, fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index e8e73ae73..78c160ec2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -68,7 +68,7 @@ func (c *initializeContext) registerCandidateRange(start, end int) error { if err != nil { return fmt.Errorf("can't create tx: %w", err) } - if err := c.multiSign(tx, committeeAccountName); err != nil { + if err := c.multiSign(tx, morphUtil.CommitteeAccountName); err != nil { return fmt.Errorf("can't sign a transaction: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index bd7c8ee1f..d5d50eca8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -113,7 +113,7 @@ func generateTestData(t *testing.T, dir string, size int) error { return fmt.Errorf("wallet doesn't exist: %w", err) } for _, acc := range w.Accounts { - if acc.Label == singleAccountName { + if acc.Label == util.SingleAccountName { pub, ok := vm.ParseSignatureContract(acc.Contract.Script) if !ok { return fmt.Errorf("could not parse signature script for %s", acc.Address) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go index e9945cee6..90e216486 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go @@ -161,7 +161,7 @@ func (c *initializeContext) transferGASToProxy() error { return err } - if err := c.multiSignAndSend(tx, committeeAccountName); err != nil { + if err := c.multiSignAndSend(tx, morphUtil.CommitteeAccountName); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go index cf961cb4d..623f1d9ae 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go @@ -21,7 +21,7 @@ func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) { cs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract info: %w", err) - nmHash, err := util.NNSResolveHash(inv, cs.Hash, domainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(netmapContract)) commonCmd.ExitOnErr(cmd, "can't get netmap contract hash: %w", err) res, err := inv.Call(nmHash, "netmapCandidates") diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy.go index 5a70bfe61..5ae4ae041 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy.go @@ -47,7 +47,7 @@ func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error return fmt.Errorf("can't get NNS contract info: %w", err) } - proxyHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(proxyContract)) + proxyHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util2.DomainOf(proxyContract)) if err != nil { return fmt.Errorf("can't get proxy contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/remove_node.go b/cmd/frostfs-adm/internal/modules/morph/remove_node.go index 0ae9a1ef7..75c486e03 100644 --- a/cmd/frostfs-adm/internal/modules/morph/remove_node.go +++ b/cmd/frostfs-adm/internal/modules/morph/remove_node.go @@ -41,7 +41,7 @@ func removeNodesCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, domainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(netmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/util/const.go index 4f11fd04d..d6e5dbc98 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/const.go @@ -14,4 +14,7 @@ const ( EndpointFlagShort = "r" AlphabetWalletsFlag = "alphabet-wallets" AlphabetWalletsFlagDesc = "Path to alphabet wallets dir" + + SingleAccountName = "single" + CommitteeAccountName = "committee" ) diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go index 7b195aebd..923061a09 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go @@ -140,3 +140,7 @@ func NNSResolveHash(inv *invoker.Invoker, nnsHash util.Uint160, domain string) ( } return ParseNNSResolveResult(item) } + +func DomainOf(contract string) string { + return contract + ".frostfs" +} From ba00fc4971170b9d610674a3bb100a688e13e0a5 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 10:06:05 +0300 Subject: [PATCH 0883/1943] [#932] adm: Move contract name constants to `util` package Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/ape_util.go | 2 +- .../internal/modules/morph/balance.go | 4 +- .../internal/modules/morph/config.go | 4 +- .../internal/modules/morph/container.go | 4 +- .../internal/modules/morph/dump_hashes.go | 2 +- .../internal/modules/morph/epoch.go | 2 +- .../internal/modules/morph/frostfsid.go | 2 +- .../modules/morph/initialize_deploy.go | 89 ++++++++----------- .../internal/modules/morph/initialize_nns.go | 4 +- .../modules/morph/initialize_register.go | 2 +- .../modules/morph/initialize_transfer.go | 2 +- .../modules/morph/netmap_candidates.go | 2 +- .../internal/modules/morph/proxy.go | 2 +- .../internal/modules/morph/remove_node.go | 2 +- .../internal/modules/morph/util/const.go | 11 +++ 15 files changed, 66 insertions(+), 68 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape_util.go index 59e85dda1..f9dcdffe8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape_util.go @@ -97,7 +97,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *ac nnsCs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) - ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(policyContract)) + ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(morphUtil.PolicyContract)) commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err) return morph.NewContractStorage(ac, ch), ac diff --git a/cmd/frostfs-adm/internal/modules/morph/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance.go index 8ee7aeeed..451d57954 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance.go @@ -64,7 +64,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(netmapContract)) + nmHash, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(morphUtil.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -137,7 +137,7 @@ func printStorageNodeBalances(cmd *cobra.Command, inv *invoker.Invoker, nmHash u } func printProxyContractBalance(cmd *cobra.Command, inv *invoker.Invoker, nnsHash util.Uint160) error { - h, err := morphUtil.NNSResolveHash(inv, nnsHash, morphUtil.DomainOf(proxyContract)) + h, err := morphUtil.NNSResolveHash(inv, nnsHash, morphUtil.DomainOf(morphUtil.ProxyContract)) if err != nil { return fmt.Errorf("can't get hash of the proxy contract: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index 3892d31b1..0d042d94a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -38,7 +38,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(util.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -96,7 +96,7 @@ func setConfigCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(util.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/container.go b/cmd/frostfs-adm/internal/modules/morph/container.go index 5f59f7db5..5400a51c9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container.go @@ -36,7 +36,7 @@ func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker) (util.Ui if err != nil { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(containerContract)) + ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(morphUtil.ContainerContract)) if err != nil { return util.Uint160{}, err } @@ -307,7 +307,7 @@ func fetchContainerContractHash(wCtx *initializeContext) (util.Uint160, error) { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, morphUtil.DomainOf(containerContract)) + ch, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, morphUtil.DomainOf(morphUtil.ContainerContract)) if err != nil { return util.Uint160{}, fmt.Errorf("can't fetch container contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go index 05ef72843..ce3585c67 100644 --- a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go @@ -50,7 +50,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { return dumpCustomZoneHashes(cmd, cs.Hash, zone, c) } - infos := []contractDumpInfo{{name: nnsContract, hash: cs.Hash}} + infos := []contractDumpInfo{{name: morphUtil.NNSContract, hash: cs.Hash}} irSize := 0 for ; irSize < lastGlagoliticLetter; irSize++ { diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/epoch.go index 274568afc..7cb73e264 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/epoch.go @@ -28,7 +28,7 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util2.DomainOf(netmapContract)) + nmHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util2.DomainOf(util2.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go index 083fc9ed9..0ad8fe763 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -431,7 +431,7 @@ func newFrostfsIDClient(cmd *cobra.Command) (*frostfsidClient, error) { return nil, fmt.Errorf("can't get NNS contract info: %w", err) } - ffsidHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(frostfsIDContract)) + ffsidHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.FrostfsIDContract)) if err != nil { return nil, fmt.Errorf("can't get proxy contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 66caf8c19..e76a9f0d3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -35,36 +35,23 @@ import ( "github.com/spf13/viper" ) -const ( - nnsContract = "nns" - frostfsContract = "frostfs" // not deployed in side-chain. - processingContract = "processing" // not deployed in side-chain. - alphabetContract = "alphabet" - balanceContract = "balance" - containerContract = "container" - frostfsIDContract = "frostfsid" - netmapContract = "netmap" - policyContract = "policy" - proxyContract = "proxy" -) - const frostfsIDAdminConfigKey = "frostfsid.admin" var ( contractList = []string{ - balanceContract, - containerContract, - frostfsIDContract, - netmapContract, - policyContract, - proxyContract, + morphUtil.BalanceContract, + morphUtil.ContainerContract, + morphUtil.FrostfsIDContract, + morphUtil.NetmapContract, + morphUtil.PolicyContract, + morphUtil.ProxyContract, } fullContractList = append([]string{ - frostfsContract, - processingContract, - nnsContract, - alphabetContract, + morphUtil.FrostfsContract, + morphUtil.ProcessingContract, + morphUtil.NNSContract, + morphUtil.AlphabetContract, }, contractList...) netmapConfigKeys = []string{ @@ -93,7 +80,7 @@ const ( ) func (c *initializeContext) deployNNS(method string) error { - cs := c.getContract(nnsContract) + cs := c.getContract(morphUtil.NNSContract) h := cs.Hash nnsCs, err := c.nnsContractState() @@ -126,7 +113,7 @@ func (c *initializeContext) deployNNS(method string) error { tx, err := c.CommitteeAct.MakeCall(invokeHash, method, params...) if err != nil { - return fmt.Errorf("failed to create deploy tx for %s: %w", nnsContract, err) + return fmt.Errorf("failed to create deploy tx for %s: %w", morphUtil.NNSContract, err) } if err := c.multiSignAndSend(tx, morphUtil.CommitteeAccountName); err != nil { @@ -137,7 +124,7 @@ func (c *initializeContext) deployNNS(method string) error { } func (c *initializeContext) updateContracts() error { - alphaCs := c.getContract(alphabetContract) + alphaCs := c.getContract(morphUtil.AlphabetContract) nnsCs, err := c.nnsContractState() if err != nil { @@ -296,7 +283,7 @@ func (c *initializeContext) deployAlphabetAccounts(nnsHash util.Uint160, w *io2. } func (c *initializeContext) deployContracts() error { - alphaCs := c.getContract(alphabetContract) + alphaCs := c.getContract(morphUtil.AlphabetContract) var keysParam []any @@ -422,7 +409,7 @@ func (c *initializeContext) readContracts(names []string) error { } for _, ctrName := range names { - if ctrName != alphabetContract { + if ctrName != morphUtil.AlphabetContract { cs := c.Contracts[ctrName] cs.Hash = state.CreateContractHash(c.CommitteeAcc.Contract.ScriptHash(), cs.NEF.Checksum, cs.Manifest.Name) @@ -524,19 +511,19 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an items := make([]any, 0, 6) switch ctrName { - case frostfsContract: + case morphUtil.FrostfsContract: items = append(items, - c.Contracts[processingContract].Hash, + c.Contracts[morphUtil.ProcessingContract].Hash, keysParam, smartcontract.Parameter{}) - case processingContract: - items = append(items, c.Contracts[frostfsContract].Hash) + case morphUtil.ProcessingContract: + items = append(items, c.Contracts[morphUtil.FrostfsContract].Hash) return items[1:], nil // no notary info - case balanceContract: + case morphUtil.BalanceContract: items = append(items, - c.Contracts[netmapContract].Hash, - c.Contracts[containerContract].Hash) - case containerContract: + c.Contracts[morphUtil.NetmapContract].Hash, + c.Contracts[morphUtil.ContainerContract].Hash) + case morphUtil.ContainerContract: // In case if NNS is updated multiple times, we can't calculate // it's actual hash based on local data, thus query chain. r := management.NewReader(c.ReadOnlyInvoker) @@ -545,12 +532,12 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an return nil, fmt.Errorf("get nns contract: %w", err) } items = append(items, - c.Contracts[netmapContract].Hash, - c.Contracts[balanceContract].Hash, - c.Contracts[frostfsIDContract].Hash, + c.Contracts[morphUtil.NetmapContract].Hash, + c.Contracts[morphUtil.BalanceContract].Hash, + c.Contracts[morphUtil.FrostfsIDContract].Hash, nnsCs.Hash, "container") - case frostfsIDContract: + case morphUtil.FrostfsIDContract: var ( h util.Uint160 found bool @@ -569,9 +556,9 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an if found { items = append(items, h) } else { - items = append(items, c.Contracts[proxyContract].Hash) + items = append(items, c.Contracts[morphUtil.ProxyContract].Hash) } - case netmapContract: + case morphUtil.NetmapContract: md := getDefaultNetmapContractConfigMap() if method == updateMethodName { if err := c.mergeNetmapConfig(md); err != nil { @@ -585,14 +572,14 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an } items = append(items, - c.Contracts[balanceContract].Hash, - c.Contracts[containerContract].Hash, + c.Contracts[morphUtil.BalanceContract].Hash, + c.Contracts[morphUtil.ContainerContract].Hash, keysParam, configParam) - case proxyContract: + case morphUtil.ProxyContract: items = nil - case policyContract: - items = append(items, c.Contracts[proxyContract].Hash) + case morphUtil.PolicyContract: + items = append(items, c.Contracts[morphUtil.ProxyContract].Hash) default: panic(fmt.Sprintf("invalid contract name: %s", ctrName)) } @@ -605,7 +592,7 @@ func (c *initializeContext) getFrostfsIDAdminFromContract() (util.Uint160, bool, if err != nil { return util.Uint160{}, false, fmt.Errorf("get nns contract: %w", err) } - fidHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(frostfsIDContract)) + fidHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.FrostfsIDContract)) if err != nil { return util.Uint160{}, false, fmt.Errorf("resolve frostfsid contract hash: %w", err) } @@ -634,7 +621,7 @@ func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, if err != nil { return nil, fmt.Errorf("get nns contract: %w", err) } - nmHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(netmapContract)) + nmHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.NetmapContract)) if err != nil { return nil, fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -667,8 +654,8 @@ func (c *initializeContext) mergeNetmapConfig(md map[string]any) error { func (c *initializeContext) getAlphabetDeployItems(i, n int) []any { items := make([]any, 5) - items[0] = c.Contracts[netmapContract].Hash - items[1] = c.Contracts[proxyContract].Hash + items[0] = c.Contracts[morphUtil.NetmapContract].Hash + items[1] = c.Contracts[morphUtil.ProxyContract].Hash items[2] = innerring.GlagoliticLetter(i).String() items[3] = int64(i) items[4] = int64(n) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go index 8267fd04d..7b0bc8197 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go @@ -55,7 +55,7 @@ func (c *initializeContext) setNNS() error { } } - alphaCs := c.getContract(alphabetContract) + alphaCs := c.getContract(morphUtil.AlphabetContract) for i, acc := range c.Accounts { alphaCs.Hash = state.CreateContractHash(acc.Contract.ScriptHash(), alphaCs.NEF.Checksum, alphaCs.Manifest.Name) @@ -139,7 +139,7 @@ func (c *initializeContext) emitUpdateNNSGroupScript(bw *io.BufBinWriter, nnsHas } func getAlphabetNNSDomain(i int) string { - return alphabetContract + strconv.FormatUint(uint64(i), 10) + ".frostfs" + return morphUtil.AlphabetContract + strconv.FormatUint(uint64(i), 10) + ".frostfs" } // wrapRegisterScriptWithPrice wraps a given script with `getPrice`/`setPrice` calls for NNS. diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index 78c160ec2..04afd7c77 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -123,7 +123,7 @@ func (c *initializeContext) transferNEOToAlphabetContracts() error { return err } - cs := c.getContract(alphabetContract) + cs := c.getContract(morphUtil.AlphabetContract) amount := initialAlphabetNEOAmount / len(c.Wallets) bw := io.NewBufBinWriter() diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go index 90e216486..a24d3a6f6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go @@ -144,7 +144,7 @@ func (c *initializeContext) multiSign(tx *transaction.Transaction, accType strin } func (c *initializeContext) transferGASToProxy() error { - proxyCs := c.getContract(proxyContract) + proxyCs := c.getContract(morphUtil.ProxyContract) r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) bal, err := r.BalanceOf(proxyCs.Hash) diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go index 623f1d9ae..c09ba8b8d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go @@ -21,7 +21,7 @@ func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) { cs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract info: %w", err) - nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(util.NetmapContract)) commonCmd.ExitOnErr(cmd, "can't get netmap contract hash: %w", err) res, err := inv.Call(nmHash, "netmapCandidates") diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy.go index 5ae4ae041..9e973b0de 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy.go @@ -47,7 +47,7 @@ func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error return fmt.Errorf("can't get NNS contract info: %w", err) } - proxyHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util2.DomainOf(proxyContract)) + proxyHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util2.DomainOf(util2.ProxyContract)) if err != nil { return fmt.Errorf("can't get proxy contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/remove_node.go b/cmd/frostfs-adm/internal/modules/morph/remove_node.go index 75c486e03..678bcc7ee 100644 --- a/cmd/frostfs-adm/internal/modules/morph/remove_node.go +++ b/cmd/frostfs-adm/internal/modules/morph/remove_node.go @@ -41,7 +41,7 @@ func removeNodesCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(netmapContract)) + nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(util.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/util/const.go index d6e5dbc98..bf71b067f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/const.go @@ -17,4 +17,15 @@ const ( SingleAccountName = "single" CommitteeAccountName = "committee" + + NNSContract = "nns" + FrostfsContract = "frostfs" // not deployed in side-chain. + ProcessingContract = "processing" // not deployed in side-chain. + AlphabetContract = "alphabet" + BalanceContract = "balance" + ContainerContract = "container" + FrostfsIDContract = "frostfsid" + NetmapContract = "netmap" + PolicyContract = "policy" + ProxyContract = "proxy" ) From bee3741f4e6895b7dc0ee13258afcc15bc76ec44 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 10:20:17 +0300 Subject: [PATCH 0884/1943] [#932] adm: Move cmd `morph ape` to `ape` package Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/{ => ape}/ape.go | 17 ++++++----------- .../modules/morph/{ => ape}/ape_util.go | 2 +- .../internal/modules/morph/ape/root.go | 16 ++++++++++++++++ cmd/frostfs-adm/internal/modules/morph/root.go | 8 ++------ 4 files changed, 25 insertions(+), 18 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => ape}/ape.go (96%) rename cmd/frostfs-adm/internal/modules/morph/{ => ape}/ape_util.go (99%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/ape/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go similarity index 96% rename from cmd/frostfs-adm/internal/modules/morph/ape.go rename to cmd/frostfs-adm/internal/modules/morph/ape/ape.go index f778a6944..de3765042 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go @@ -1,4 +1,4 @@ -package morph +package ape import ( "bytes" @@ -33,11 +33,6 @@ const ( ) var ( - apeCmd = &cobra.Command{ - Use: "ape", - Short: "Section for APE configuration commands", - } - addRuleChainCmd = &cobra.Command{ Use: "add-rule-chain", Short: "Add rule chain", @@ -90,7 +85,7 @@ var ( ) func initAddRuleChainCmd() { - apeCmd.AddCommand(addRuleChainCmd) + Cmd.AddCommand(addRuleChainCmd) addRuleChainCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) addRuleChainCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) @@ -109,7 +104,7 @@ func initAddRuleChainCmd() { } func initRemoveRuleChainCmd() { - apeCmd.AddCommand(removeRuleChainCmd) + Cmd.AddCommand(removeRuleChainCmd) removeRuleChainCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) removeRuleChainCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) @@ -123,7 +118,7 @@ func initRemoveRuleChainCmd() { } func initListRuleChainsCmd() { - apeCmd.AddCommand(listRuleChainsCmd) + Cmd.AddCommand(listRuleChainsCmd) listRuleChainsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) listRuleChainsCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) @@ -135,7 +130,7 @@ func initListRuleChainsCmd() { } func initSetAdminCmd() { - apeCmd.AddCommand(setAdminCmd) + Cmd.AddCommand(setAdminCmd) setAdminCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) setAdminCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) @@ -144,7 +139,7 @@ func initSetAdminCmd() { } func initGetAdminCmd() { - apeCmd.AddCommand(getAdminCmd) + Cmd.AddCommand(getAdminCmd) getAdminCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) getAdminCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/ape_util.go rename to cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index f9dcdffe8..21f27bfd7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -1,4 +1,4 @@ -package morph +package ape import ( "encoding/json" diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/root.go b/cmd/frostfs-adm/internal/modules/morph/ape/root.go new file mode 100644 index 000000000..11b6894cf --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/ape/root.go @@ -0,0 +1,16 @@ +package ape + +import "github.com/spf13/cobra" + +var Cmd = &cobra.Command{ + Use: "ape", + Short: "Section for APE configuration commands", +} + +func init() { + initAddRuleChainCmd() + initRemoveRuleChainCmd() + initListRuleChainsCmd() + initSetAdminCmd() + initGetAdminCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 5a4fa0063..328877967 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -1,6 +1,7 @@ package morph import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -283,12 +284,7 @@ func init() { initDepositoryNotaryCmd() initNetmapCandidatesCmd() - RootCmd.AddCommand(apeCmd) - initAddRuleChainCmd() - initRemoveRuleChainCmd() - initListRuleChainsCmd() - initSetAdminCmd() - initGetAdminCmd() + RootCmd.AddCommand(ape.Cmd) initProxyAddAccount() initProxyRemoveAccount() From b68f7be0b6c7c0f58f5c972a309005c811fa9e74 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 14:30:59 +0300 Subject: [PATCH 0885/1943] [#932] adm: Prepare to move `InitializeContext` to `util` package Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/config.go | 6 +- .../internal/modules/morph/container.go | 14 +- .../internal/modules/morph/deploy.go | 14 +- .../internal/modules/morph/dump_hashes.go | 2 +- .../internal/modules/morph/epoch.go | 8 +- .../internal/modules/morph/frostfsid.go | 10 +- .../internal/modules/morph/generate.go | 6 +- .../internal/modules/morph/group.go | 5 +- .../internal/modules/morph/initialize.go | 80 +++++----- .../modules/morph/initialize_deploy.go | 148 ++++++++---------- .../internal/modules/morph/initialize_nns.go | 91 +++-------- .../modules/morph/initialize_register.go | 26 +-- .../modules/morph/initialize_roles.go | 10 +- .../modules/morph/initialize_transfer.go | 24 +-- .../internal/modules/morph/policy.go | 6 +- .../internal/modules/morph/proxy.go | 6 +- .../internal/modules/morph/remove_node.go | 8 +- .../internal/modules/morph/update.go | 6 +- .../internal/modules/morph/util/initialize.go | 43 +++++ .../modules/morph/util/initialize_ctx.go | 34 ++++ 20 files changed, 277 insertions(+), 270 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index 0d042d94a..2d0f12afe 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -85,7 +85,7 @@ func setConfigCmd(cmd *cobra.Command, args []string) error { return errors.New("empty config pairs") } - wCtx, err := newInitializeContext(cmd, viper.GetViper()) + wCtx, err := NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't initialize context: %w", err) } @@ -119,12 +119,12 @@ func setConfigCmd(cmd *cobra.Command, args []string) error { } } - err = wCtx.sendConsensusTx(bw.Bytes()) + err = wCtx.SendConsensusTx(bw.Bytes()) if err != nil { return err } - return wCtx.awaitTx() + return wCtx.AwaitTx() } func parseConfigPair(kvStr string, force bool) (key string, val any, err error) { diff --git a/cmd/frostfs-adm/internal/modules/morph/container.go b/cmd/frostfs-adm/internal/modules/morph/container.go index 5400a51c9..ab32dcabb 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container.go @@ -193,11 +193,11 @@ func restoreContainers(cmd *cobra.Command, _ []string) error { return fmt.Errorf("invalid filename: %w", err) } - wCtx, err := newInitializeContext(cmd, viper.GetViper()) + wCtx, err := NewInitializeContext(cmd, viper.GetViper()) if err != nil { return err } - defer wCtx.close() + defer wCtx.Close() containers, err := parseContainers(filename) if err != nil { @@ -219,10 +219,10 @@ func restoreContainers(cmd *cobra.Command, _ []string) error { return err } - return wCtx.awaitTx() + return wCtx.AwaitTx() } -func restoreOrPutContainers(containers []Container, isOK func([]byte) bool, cmd *cobra.Command, wCtx *initializeContext, ch util.Uint160) error { +func restoreOrPutContainers(containers []Container, isOK func([]byte) bool, cmd *cobra.Command, wCtx *InitializeContext, ch util.Uint160) error { bw := io.NewBufBinWriter() for _, cnt := range containers { hv := hash.Sha256(cnt.Value) @@ -246,7 +246,7 @@ func restoreOrPutContainers(containers []Container, isOK func([]byte) bool, cmd panic(bw.Err) } - if err := wCtx.sendConsensusTx(bw.Bytes()); err != nil { + if err := wCtx.SendConsensusTx(bw.Bytes()); err != nil { return err } } @@ -262,7 +262,7 @@ func putContainer(bw *io.BufBinWriter, ch util.Uint160, cnt Container) { } } -func isContainerRestored(cmd *cobra.Command, wCtx *initializeContext, containerHash util.Uint160, bw *io.BufBinWriter, hashValue util.Uint256) (bool, error) { +func isContainerRestored(cmd *cobra.Command, wCtx *InitializeContext, containerHash util.Uint160, bw *io.BufBinWriter, hashValue util.Uint256) (bool, error) { emit.AppCall(bw.BinWriter, containerHash, "get", callflag.All, hashValue.BytesBE()) res, err := wCtx.Client.InvokeScript(bw.Bytes(), nil) if err != nil { @@ -300,7 +300,7 @@ func parseContainers(filename string) ([]Container, error) { return containers, nil } -func fetchContainerContractHash(wCtx *initializeContext) (util.Uint160, error) { +func fetchContainerContractHash(wCtx *InitializeContext) (util.Uint160, error) { r := management.NewReader(wCtx.ReadOnlyInvoker) nnsCs, err := r.GetContractByID(1) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/deploy.go b/cmd/frostfs-adm/internal/modules/morph/deploy.go index 85c207d79..7f11362e0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/deploy.go @@ -60,11 +60,11 @@ func init() { func deployContractCmd(cmd *cobra.Command, args []string) error { v := viper.GetViper() - c, err := newInitializeContext(cmd, v) + c, err := NewInitializeContext(cmd, v) if err != nil { return fmt.Errorf("initialization error: %w", err) } - defer c.close() + defer c.Close() ctrPath, _ := cmd.Flags().GetString(contractPathFlag) ctrName, err := probeContractName(ctrPath) @@ -123,13 +123,13 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { panic(fmt.Errorf("BUG: can't create deployment script: %w", writer.Err)) } - if err := c.sendCommitteeTx(writer.Bytes(), false); err != nil { + if err := c.SendCommitteeTx(writer.Bytes(), false); err != nil { return err } - return c.awaitTx() + return c.AwaitTx() } -func registerNNS(nnsCs *state.Contract, c *initializeContext, zone string, domain string, cs *contractState, writer *io.BufBinWriter) error { +func registerNNS(nnsCs *state.Contract, c *InitializeContext, zone string, domain string, cs *util.ContractState, writer *io.BufBinWriter) error { bw := io.NewBufBinWriter() emit.Instruction(bw.BinWriter, opcode.INITSSLOT, []byte{1}) emit.AppCall(bw.BinWriter, nnsCs.Hash, "getPrice", callflag.All) @@ -139,7 +139,7 @@ func registerNNS(nnsCs *state.Contract, c *initializeContext, zone string, domai start := bw.Len() needRecord := false - ok, err := c.nnsRootRegistered(nnsCs.Hash, zone) + ok, err := c.NNSRootRegistered(nnsCs.Hash, zone) if err != nil { return err } else if !ok { @@ -155,7 +155,7 @@ func registerNNS(nnsCs *state.Contract, c *initializeContext, zone string, domai frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) } else { - s, ok, err := c.nnsRegisterDomainScript(nnsCs.Hash, cs.Hash, domain) + s, ok, err := c.NNSRegisterDomainScript(nnsCs.Hash, cs.Hash, domain) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go index ce3585c67..79b82ee47 100644 --- a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go @@ -54,7 +54,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { irSize := 0 for ; irSize < lastGlagoliticLetter; irSize++ { - ok, err := nnsIsAvailable(c, cs.Hash, getAlphabetNNSDomain(irSize)) + ok, err := morphUtil.NNSIsAvailable(c, cs.Hash, getAlphabetNNSDomain(irSize)) if err != nil { return err } else if ok { diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/epoch.go index 7cb73e264..0fc25d01d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/epoch.go @@ -17,7 +17,7 @@ import ( ) func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { - wCtx, err := newInitializeContext(cmd, viper.GetViper()) + wCtx, err := NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) } @@ -38,8 +38,8 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { return err } - if err = wCtx.sendConsensusTx(bw.Bytes()); err == nil { - err = wCtx.awaitTx() + if err = wCtx.SendConsensusTx(bw.Bytes()); err == nil { + err = wCtx.AwaitTx() } if err != nil && strings.Contains(err.Error(), "invalid epoch") { cmd.Println("Epoch has already ticked.") @@ -48,7 +48,7 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { return err } -func emitNewEpochCall(bw *io.BufBinWriter, wCtx *initializeContext, nmHash util.Uint160) error { +func emitNewEpochCall(bw *io.BufBinWriter, wCtx *InitializeContext, nmHash util.Uint160) error { curr, err := unwrap.Int64(wCtx.ReadOnlyInvoker.Call(nmHash, "epoch")) if err != nil { return errors.New("can't fetch current epoch from the netmap contract") diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go index 0ad8fe763..4adde815d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -416,11 +416,11 @@ type frostfsidClient struct { bw *io.BufBinWriter contractHash util.Uint160 roCli *frostfsidclient.Client // client can be used only for waiting tx, parsing and forming method params - wCtx *initializeContext + wCtx *InitializeContext } func newFrostfsIDClient(cmd *cobra.Command) (*frostfsidClient, error) { - wCtx, err := newInitializeContext(cmd, viper.GetViper()) + wCtx, err := NewInitializeContext(cmd, viper.GetViper()) if err != nil { return nil, fmt.Errorf("can't to initialize context: %w", err) } @@ -449,16 +449,16 @@ func (f *frostfsidClient) addCall(method string, args []any) { } func (f *frostfsidClient) sendWait() error { - if err := f.wCtx.sendConsensusTx(f.bw.Bytes()); err != nil { + if err := f.wCtx.SendConsensusTx(f.bw.Bytes()); err != nil { return err } f.bw.Reset() - return f.wCtx.awaitTx() + return f.wCtx.AwaitTx() } func (f *frostfsidClient) sendWaitRes() (*state.AppExecResult, error) { - if err := f.wCtx.sendConsensusTx(f.bw.Bytes()); err != nil { + if err := f.wCtx.SendConsensusTx(f.bw.Bytes()); err != nil { return nil, err } f.bw.Reset() diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index a7561b266..f8e155f5b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -195,7 +195,7 @@ func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error return err } - wCtx, err := newInitializeContext(cmd, viper.GetViper()) + wCtx, err := NewInitializeContext(cmd, viper.GetViper()) if err != nil { return err } @@ -208,11 +208,11 @@ func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error return fmt.Errorf("BUG: invalid transfer arguments: %w", bw.Err) } - if err := wCtx.sendCommitteeTx(bw.Bytes(), false); err != nil { + if err := wCtx.SendCommitteeTx(bw.Bytes(), false); err != nil { return err } - return wCtx.awaitTx() + return wCtx.AwaitTx() } func parseGASAmount(s string) (fixedn.Fixed8, error) { diff --git a/cmd/frostfs-adm/internal/modules/morph/group.go b/cmd/frostfs-adm/internal/modules/morph/group.go index 3fdffd4e6..0e6c511da 100644 --- a/cmd/frostfs-adm/internal/modules/morph/group.go +++ b/cmd/frostfs-adm/internal/modules/morph/group.go @@ -7,6 +7,7 @@ import ( "path/filepath" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + util2 "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest" "github.com/nspcc-dev/neo-go/pkg/util" @@ -75,8 +76,8 @@ func openContractWallet(v *viper.Viper, cmd *cobra.Command, walletDir string) (* return w, nil } -func (c *initializeContext) addManifestGroup(h util.Uint160, cs *contractState) error { - priv := c.ContractWallet.Accounts[0].PrivateKey() +func addManifestGroup(cw *wallet.Wallet, h util.Uint160, cs *util2.ContractState) error { + priv := cw.Accounts[0].PrivateKey() pub := priv.PublicKey() sig := priv.Sign(h.BytesBE()) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index a432fbc7e..2057b4a24 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -20,14 +20,14 @@ import ( "github.com/spf13/viper" ) -type cache struct { - nnsCs *state.Contract - groupKey *keys.PublicKey +type Cache struct { + NNSCs *state.Contract + GroupKey *keys.PublicKey } -type initializeContext struct { +type InitializeContext struct { morphUtil.ClientContext - cache + Cache // CommitteeAcc is used for retrieving the committee address and the verification script. CommitteeAcc *wallet.Account // ConsensusAcc is used for retrieving the committee address and the verification script. @@ -37,62 +37,62 @@ type initializeContext struct { ContractWallet *wallet.Wallet // Accounts contains simple signature accounts in the same order as in Wallets. Accounts []*wallet.Account - Contracts map[string]*contractState + Contracts map[string]*morphUtil.ContractState Command *cobra.Command ContractPath string ContractURL string } func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { - initCtx, err := newInitializeContext(cmd, viper.GetViper()) + initCtx, err := NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("initialization error: %w", err) } - defer initCtx.close() + defer initCtx.Close() // 1. Transfer funds to committee accounts. cmd.Println("Stage 1: transfer GAS to alphabet nodes.") - if err := initCtx.transferFunds(); err != nil { + if err := transferFunds(initCtx); err != nil { return err } cmd.Println("Stage 2: set notary and alphabet nodes in designate contract.") - if err := initCtx.setNotaryAndAlphabetNodes(); err != nil { + if err := setNotaryAndAlphabetNodes(initCtx); err != nil { return err } // 3. Deploy NNS contract. cmd.Println("Stage 3: deploy NNS contract.") - if err := initCtx.deployNNS(deployMethodName); err != nil { + if err := deployNNS(initCtx, deployMethodName); err != nil { return err } // 4. Deploy NeoFS contracts. cmd.Println("Stage 4: deploy NeoFS contracts.") - if err := initCtx.deployContracts(); err != nil { + if err := deployContracts(initCtx); err != nil { return err } cmd.Println("Stage 4.1: Transfer GAS to proxy contract.") - if err := initCtx.transferGASToProxy(); err != nil { + if err := transferGASToProxy(initCtx); err != nil { return err } cmd.Println("Stage 5: register candidates.") - if err := initCtx.registerCandidates(); err != nil { + if err := registerCandidates(initCtx); err != nil { return err } cmd.Println("Stage 6: transfer NEO to alphabet contracts.") - if err := initCtx.transferNEOToAlphabetContracts(); err != nil { + if err := transferNEOToAlphabetContracts(initCtx); err != nil { return err } cmd.Println("Stage 7: set addresses in NNS.") - return initCtx.setNNS() + return setNNS(initCtx) } -func (c *initializeContext) close() { +func (c *InitializeContext) Close() { if local, ok := c.Client.(*morphUtil.LocalClient); ok { err := local.Dump() if err != nil { @@ -102,7 +102,7 @@ func (c *initializeContext) close() { } } -func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContext, error) { +func NewInitializeContext(cmd *cobra.Command, v *viper.Viper) (*InitializeContext, error) { walletDir := config.ResolveHomePath(viper.GetString(morphUtil.AlphabetWalletsFlag)) wallets, err := morphUtil.GetAlphabetWallets(v, walletDir) if err != nil { @@ -160,7 +160,7 @@ func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContex return nil, fmt.Errorf("client context: %w", err) } - initCtx := &initializeContext{ + initCtx := &InitializeContext{ ClientContext: *cliCtx, ConsensusAcc: consensusAcc, CommitteeAcc: committeeAcc, @@ -168,7 +168,7 @@ func newInitializeContext(cmd *cobra.Command, v *viper.Viper) (*initializeContex Wallets: wallets, Accounts: accounts, Command: cmd, - Contracts: make(map[string]*contractState), + Contracts: make(map[string]*morphUtil.ContractState), ContractPath: ctrPath, ContractURL: ctrURL, } @@ -246,13 +246,13 @@ func createWalletAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) { return accounts, nil } -func (c *initializeContext) awaitTx() error { +func (c *InitializeContext) AwaitTx() error { return c.ClientContext.AwaitTx(c.Command) } -func (c *initializeContext) nnsContractState() (*state.Contract, error) { - if c.nnsCs != nil { - return c.nnsCs, nil +func (c *InitializeContext) NNSContractState() (*state.Contract, error) { + if c.NNSCs != nil { + return c.NNSCs, nil } r := management.NewReader(c.ReadOnlyInvoker) @@ -261,16 +261,16 @@ func (c *initializeContext) nnsContractState() (*state.Contract, error) { return nil, err } - c.nnsCs = cs + c.NNSCs = cs return cs, nil } -func (c *initializeContext) getSigner(tryGroup bool, acc *wallet.Account) transaction.Signer { - if tryGroup && c.groupKey != nil { +func (c *InitializeContext) GetSigner(tryGroup bool, acc *wallet.Account) transaction.Signer { + if tryGroup && c.GroupKey != nil { return transaction.Signer{ Account: acc.Contract.ScriptHash(), Scopes: transaction.CustomGroups, - AllowedGroups: keys.PublicKeys{c.groupKey}, + AllowedGroups: keys.PublicKeys{c.GroupKey}, } } @@ -283,14 +283,14 @@ func (c *initializeContext) getSigner(tryGroup bool, acc *wallet.Account) transa return signer } - nnsCs, err := c.nnsContractState() + nnsCs, err := c.NNSContractState() if err != nil { return signer } - groupKey, err := nnsResolveKey(c.ReadOnlyInvoker, nnsCs.Hash, morphClient.NNSGroupKeyName) + groupKey, err := morphUtil.NNSResolveKey(c.ReadOnlyInvoker, nnsCs.Hash, morphClient.NNSGroupKeyName) if err == nil { - c.groupKey = groupKey + c.GroupKey = groupKey signer.Scopes = transaction.CustomGroups signer.AllowedGroups = keys.PublicKeys{groupKey} @@ -298,21 +298,21 @@ func (c *initializeContext) getSigner(tryGroup bool, acc *wallet.Account) transa return signer } -// sendCommitteeTx creates transaction from script, signs it by committee nodes and sends it to RPC. +// SendCommitteeTx creates transaction from script, signs it by committee nodes and sends it to RPC. // If tryGroup is false, global scope is used for the signer (useful when // working with native contracts). -func (c *initializeContext) sendCommitteeTx(script []byte, tryGroup bool) error { +func (c *InitializeContext) SendCommitteeTx(script []byte, tryGroup bool) error { return c.sendMultiTx(script, tryGroup, false) } -// sendConsensusTx creates transaction from script, signs it by alphabet nodes and sends it to RPC. +// SendConsensusTx creates transaction from script, signs it by alphabet nodes and sends it to RPC. // Not that because this is used only after the contracts were initialized and deployed, // we always try to have a group scope. -func (c *initializeContext) sendConsensusTx(script []byte) error { +func (c *InitializeContext) SendConsensusTx(script []byte) error { return c.sendMultiTx(script, true, true) } -func (c *initializeContext) sendMultiTx(script []byte, tryGroup bool, withConsensus bool) error { +func (c *InitializeContext) sendMultiTx(script []byte, tryGroup bool, withConsensus bool) error { var act *actor.Actor var err error @@ -321,12 +321,12 @@ func (c *initializeContext) sendMultiTx(script []byte, tryGroup bool, withConsen // Even for consensus signatures we need the committee to pay. signers := make([]actor.SignerAccount, 1, 2) signers[0] = actor.SignerAccount{ - Signer: c.getSigner(tryGroup, c.CommitteeAcc), + Signer: c.GetSigner(tryGroup, c.CommitteeAcc), Account: c.CommitteeAcc, } if withConsensus { signers = append(signers, actor.SignerAccount{ - Signer: c.getSigner(tryGroup, c.ConsensusAcc), + Signer: c.GetSigner(tryGroup, c.ConsensusAcc), Account: c.ConsensusAcc, }) } @@ -346,11 +346,11 @@ func (c *initializeContext) sendMultiTx(script []byte, tryGroup bool, withConsen return fmt.Errorf("could not perform test invocation: %w", err) } - if err := c.multiSign(tx, morphUtil.CommitteeAccountName); err != nil { + if err := c.MultiSign(tx, morphUtil.CommitteeAccountName); err != nil { return err } if withConsensus { - if err := c.multiSign(tx, morphUtil.ConsensusAccountName); err != nil { + if err := c.MultiSign(tx, morphUtil.ConsensusAccountName); err != nil { return err } } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index e76a9f0d3..fd27eda6c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -4,7 +4,6 @@ import ( "archive/tar" "compress/gzip" "encoding/hex" - "encoding/json" "errors" "fmt" "io" @@ -22,12 +21,11 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/address" io2 "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" - "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest" - "github.com/nspcc-dev/neo-go/pkg/smartcontract/nef" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" @@ -66,24 +64,16 @@ var ( } ) -type contractState struct { - NEF *nef.File - RawNEF []byte - Manifest *manifest.Manifest - RawManifest []byte - Hash util.Uint160 -} - const ( updateMethodName = "update" deployMethodName = "deploy" ) -func (c *initializeContext) deployNNS(method string) error { - cs := c.getContract(morphUtil.NNSContract) +func deployNNS(c *InitializeContext, method string) error { + cs := c.GetContract(morphUtil.NNSContract) h := cs.Hash - nnsCs, err := c.nnsContractState() + nnsCs, err := c.NNSContractState() if err != nil { return err } @@ -99,7 +89,7 @@ func (c *initializeContext) deployNNS(method string) error { h = nnsCs.Hash } - err = c.addManifestGroup(h, cs) + err = addManifestGroup(c.ContractWallet, h, cs) if err != nil { return fmt.Errorf("can't sign manifest group: %v", err) } @@ -116,17 +106,17 @@ func (c *initializeContext) deployNNS(method string) error { return fmt.Errorf("failed to create deploy tx for %s: %w", morphUtil.NNSContract, err) } - if err := c.multiSignAndSend(tx, morphUtil.CommitteeAccountName); err != nil { + if err := c.MultiSignAndSend(tx, morphUtil.CommitteeAccountName); err != nil { return fmt.Errorf("can't send deploy transaction: %w", err) } - return c.awaitTx() + return c.AwaitTx() } -func (c *initializeContext) updateContracts() error { - alphaCs := c.getContract(morphUtil.AlphabetContract) +func updateContractsInternal(c *InitializeContext) error { + alphaCs := c.GetContract(morphUtil.AlphabetContract) - nnsCs, err := c.nnsContractState() + nnsCs, err := c.NNSContractState() if err != nil { return err } @@ -147,19 +137,19 @@ func (c *initializeContext) updateContracts() error { emit.Bytes(w.BinWriter, alphaCs.RawNEF) emit.Opcodes(w.BinWriter, opcode.STSFLD0) - keysParam, err := c.deployAlphabetAccounts(nnsHash, w, alphaCs) + keysParam, err := deployAlphabetAccounts(c, nnsHash, w, alphaCs) if err != nil { return err } w.Reset() - if err = c.deployOrUpdateContracts(w, nnsHash, keysParam); err != nil { + if err = deployOrUpdateContracts(c, w, nnsHash, keysParam); err != nil { return err } groupKey := c.ContractWallet.Accounts[0].PrivateKey().PublicKey() - _, _, err = c.emitUpdateNNSGroupScript(w, nnsHash, groupKey) + _, _, err = c.EmitUpdateNNSGroupScript(w, nnsHash, groupKey) if err != nil { return err } @@ -170,20 +160,20 @@ func (c *initializeContext) updateContracts() error { emit.Opcodes(w.BinWriter, opcode.PACK) emit.AppCallNoArgs(w.BinWriter, nnsHash, "setPrice", callflag.All) - if err := c.sendCommitteeTx(w.Bytes(), false); err != nil { + if err := c.SendCommitteeTx(w.Bytes(), false); err != nil { return err } - return c.awaitTx() + return c.AwaitTx() } -func (c *initializeContext) deployOrUpdateContracts(w *io2.BufBinWriter, nnsHash util.Uint160, keysParam []any) error { +func deployOrUpdateContracts(c *InitializeContext, w *io2.BufBinWriter, nnsHash util.Uint160, keysParam []any) error { emit.Instruction(w.BinWriter, opcode.INITSSLOT, []byte{1}) emit.AppCall(w.BinWriter, nnsHash, "getPrice", callflag.All) emit.Opcodes(w.BinWriter, opcode.STSFLD0) emit.AppCall(w.BinWriter, nnsHash, "setPrice", callflag.All, 1) for _, ctrName := range contractList { - cs := c.getContract(ctrName) + cs := c.GetContract(ctrName) method := updateMethodName ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, morphUtil.DomainOf(ctrName)) @@ -196,7 +186,7 @@ func (c *initializeContext) deployOrUpdateContracts(w *io2.BufBinWriter, nnsHash } } - err = c.addManifestGroup(ctrHash, cs) + err = addManifestGroup(c.ContractWallet, ctrHash, cs) if err != nil { return fmt.Errorf("can't sign manifest group: %v", err) } @@ -206,7 +196,7 @@ func (c *initializeContext) deployOrUpdateContracts(w *io2.BufBinWriter, nnsHash invokeHash = ctrHash } - args, err := c.getContractDeployData(ctrName, keysParam, updateMethodName) + args, err := getContractDeployData(c, ctrName, keysParam, updateMethodName) if err != nil { return fmt.Errorf("%s: getting update params: %v", ctrName, err) } @@ -223,9 +213,9 @@ func (c *initializeContext) deployOrUpdateContracts(w *io2.BufBinWriter, nnsHash w.WriteBytes(res.Script) if method == deployMethodName { - // same actions are done in initializeContext.setNNS, can be unified + // same actions are done in InitializeContext.setNNS, can be unified domain := ctrName + ".frostfs" - script, ok, err := c.nnsRegisterDomainScript(nnsHash, cs.Hash, domain) + script, ok, err := c.NNSRegisterDomainScript(nnsHash, cs.Hash, domain) if err != nil { return err } @@ -243,7 +233,7 @@ func (c *initializeContext) deployOrUpdateContracts(w *io2.BufBinWriter, nnsHash return nil } -func (c *initializeContext) deployAlphabetAccounts(nnsHash util.Uint160, w *io2.BufBinWriter, alphaCs *contractState) ([]any, error) { +func deployAlphabetAccounts(c *InitializeContext, nnsHash util.Uint160, w *io2.BufBinWriter, alphaCs *morphUtil.ContractState) ([]any, error) { var keysParam []any baseGroups := alphaCs.Manifest.Groups @@ -257,11 +247,11 @@ func (c *initializeContext) deployAlphabetAccounts(nnsHash util.Uint160, w *io2. keysParam = append(keysParam, acc.PrivateKey().PublicKey().Bytes()) - params := c.getAlphabetDeployItems(i, len(c.Wallets)) + params := c.GetAlphabetDeployItems(i, len(c.Wallets)) emit.Array(w.BinWriter, params...) alphaCs.Manifest.Groups = baseGroups - err = c.addManifestGroup(ctrHash, alphaCs) + err = addManifestGroup(c.ContractWallet, ctrHash, alphaCs) if err != nil { return nil, fmt.Errorf("can't sign manifest group: %v", err) } @@ -272,7 +262,7 @@ func (c *initializeContext) deployAlphabetAccounts(nnsHash util.Uint160, w *io2. emit.Opcodes(w.BinWriter, opcode.PACK) emit.AppCallNoArgs(w.BinWriter, ctrHash, updateMethodName, callflag.All) } - if err := c.sendCommitteeTx(w.Bytes(), false); err != nil { + if err := c.SendCommitteeTx(w.Bytes(), false); err != nil { if !strings.Contains(err.Error(), common.ErrAlreadyUpdated) { return nil, err } @@ -282,8 +272,8 @@ func (c *initializeContext) deployAlphabetAccounts(nnsHash util.Uint160, w *io2. return keysParam, nil } -func (c *initializeContext) deployContracts() error { - alphaCs := c.getContract(morphUtil.AlphabetContract) +func deployContracts(c *InitializeContext) error { + alphaCs := c.GetContract(morphUtil.AlphabetContract) var keysParam []any @@ -292,19 +282,19 @@ func (c *initializeContext) deployContracts() error { // alphabet contracts should be deployed by individual nodes to get different hashes. for i, acc := range c.Accounts { ctrHash := state.CreateContractHash(acc.Contract.ScriptHash(), alphaCs.NEF.Checksum, alphaCs.Manifest.Name) - if c.isUpdated(ctrHash, alphaCs) { + if c.IsUpdated(ctrHash, alphaCs) { c.Command.Printf("Alphabet contract #%d is already deployed.\n", i) continue } alphaCs.Manifest.Groups = baseGroups - err := c.addManifestGroup(ctrHash, alphaCs) + err := addManifestGroup(c.ContractWallet, ctrHash, alphaCs) if err != nil { return fmt.Errorf("can't sign manifest group: %v", err) } keysParam = append(keysParam, acc.PrivateKey().PublicKey().Bytes()) - params := getContractDeployParameters(alphaCs, c.getAlphabetDeployItems(i, len(c.Wallets))) + params := getContractDeployParameters(alphaCs, c.GetAlphabetDeployItems(i, len(c.Wallets))) act, err := actor.NewSimple(c.Client, acc) if err != nil { @@ -320,20 +310,20 @@ func (c *initializeContext) deployContracts() error { } for _, ctrName := range contractList { - cs := c.getContract(ctrName) + cs := c.GetContract(ctrName) ctrHash := cs.Hash - if c.isUpdated(ctrHash, cs) { + if c.IsUpdated(ctrHash, cs) { c.Command.Printf("%s contract is already deployed.\n", ctrName) continue } - err := c.addManifestGroup(ctrHash, cs) + err := addManifestGroup(c.ContractWallet, ctrHash, cs) if err != nil { return fmt.Errorf("can't sign manifest group: %v", err) } - args, err := c.getContractDeployData(ctrName, keysParam, deployMethodName) + args, err := getContractDeployData(c, ctrName, keysParam, deployMethodName) if err != nil { return fmt.Errorf("%s: getting deploy params: %v", ctrName, err) } @@ -343,25 +333,25 @@ func (c *initializeContext) deployContracts() error { return fmt.Errorf("can't deploy %s contract: %w", ctrName, err) } - if err := c.sendCommitteeTx(res.Script, false); err != nil { + if err := c.SendCommitteeTx(res.Script, false); err != nil { return err } } - return c.awaitTx() + return c.AwaitTx() } -func (c *initializeContext) isUpdated(ctrHash util.Uint160, cs *contractState) bool { +func (c *InitializeContext) IsUpdated(ctrHash util.Uint160, cs *morphUtil.ContractState) bool { r := management.NewReader(c.ReadOnlyInvoker) realCs, err := r.GetContract(ctrHash) return err == nil && realCs != nil && realCs.NEF.Checksum == cs.NEF.Checksum } -func (c *initializeContext) getContract(ctrName string) *contractState { +func (c *InitializeContext) GetContract(ctrName string) *morphUtil.ContractState { return c.Contracts[ctrName] } -func (c *initializeContext) readContracts(names []string) error { +func (c *InitializeContext) readContracts(names []string) error { var ( fi os.FileInfo err error @@ -401,7 +391,7 @@ func (c *initializeContext) readContracts(names []string) error { return err } for _, name := range names { - if err := m[name].parse(); err != nil { + if err := m[name].Parse(); err != nil { return err } c.Contracts[name] = m[name] @@ -418,7 +408,7 @@ func (c *initializeContext) readContracts(names []string) error { return nil } -func readContract(ctrPath, ctrName string) (*contractState, error) { +func readContract(ctrPath, ctrName string) (*morphUtil.ContractState, error) { rawNef, err := os.ReadFile(filepath.Join(ctrPath, ctrName+"_contract.nef")) if err != nil { return nil, fmt.Errorf("can't read NEF file for %s contract: %w", ctrName, err) @@ -428,34 +418,18 @@ func readContract(ctrPath, ctrName string) (*contractState, error) { return nil, fmt.Errorf("can't read manifest file for %s contract: %w", ctrName, err) } - cs := &contractState{ + cs := &morphUtil.ContractState{ RawNEF: rawNef, RawManifest: rawManif, } - return cs, cs.parse() + return cs, cs.Parse() } -func (cs *contractState) parse() error { - nf, err := nef.FileFromBytes(cs.RawNEF) - if err != nil { - return fmt.Errorf("can't parse NEF file: %w", err) - } - - m := new(manifest.Manifest) - if err := json.Unmarshal(cs.RawManifest, m); err != nil { - return fmt.Errorf("can't parse manifest file: %w", err) - } - - cs.NEF = &nf - cs.Manifest = m - return nil -} - -func readContractsFromArchive(file io.Reader, names []string) (map[string]*contractState, error) { - m := make(map[string]*contractState, len(names)) +func readContractsFromArchive(file io.Reader, names []string) (map[string]*morphUtil.ContractState, error) { + m := make(map[string]*morphUtil.ContractState, len(names)) for i := range names { - m[names[i]] = new(contractState) + m[names[i]] = new(morphUtil.ContractState) } gr, err := gzip.NewReader(file) @@ -503,11 +477,11 @@ func readContractsFromArchive(file io.Reader, names []string) (map[string]*contr return m, nil } -func getContractDeployParameters(cs *contractState, deployData []any) []any { +func getContractDeployParameters(cs *morphUtil.ContractState, deployData []any) []any { return []any{cs.RawNEF, cs.RawManifest, deployData} } -func (c *initializeContext) getContractDeployData(ctrName string, keysParam []any, method string) ([]any, error) { +func getContractDeployData(c *InitializeContext, ctrName string, keysParam []any, method string) ([]any, error) { items := make([]any, 0, 6) switch ctrName { @@ -544,7 +518,7 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an err error ) if method == updateMethodName { - h, found, err = c.getFrostfsIDAdminFromContract() + h, found, err = getFrostfsIDAdminFromContract(c.ReadOnlyInvoker) } if method != updateMethodName || err == nil && !found { h, found, err = getFrostfsIDAdmin(viper.GetViper()) @@ -561,7 +535,7 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an case morphUtil.NetmapContract: md := getDefaultNetmapContractConfigMap() if method == updateMethodName { - if err := c.mergeNetmapConfig(md); err != nil { + if err := mergeNetmapConfig(c.ReadOnlyInvoker, md); err != nil { return nil, err } } @@ -586,17 +560,17 @@ func (c *initializeContext) getContractDeployData(ctrName string, keysParam []an return items, nil } -func (c *initializeContext) getFrostfsIDAdminFromContract() (util.Uint160, bool, error) { - r := management.NewReader(c.ReadOnlyInvoker) +func getFrostfsIDAdminFromContract(roInvoker *invoker.Invoker) (util.Uint160, bool, error) { + r := management.NewReader(roInvoker) cs, err := r.GetContractByID(1) if err != nil { return util.Uint160{}, false, fmt.Errorf("get nns contract: %w", err) } - fidHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.FrostfsIDContract)) + fidHash, err := morphUtil.NNSResolveHash(roInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.FrostfsIDContract)) if err != nil { return util.Uint160{}, false, fmt.Errorf("resolve frostfsid contract hash: %w", err) } - item, err := unwrap.Item(c.ReadOnlyInvoker.Call(fidHash, "getAdmin")) + item, err := unwrap.Item(roInvoker.Call(fidHash, "getAdmin")) if err != nil { return util.Uint160{}, false, fmt.Errorf("getAdmin: %w", err) } @@ -615,25 +589,25 @@ func (c *initializeContext) getFrostfsIDAdminFromContract() (util.Uint160, bool, return h, true, nil } -func (c *initializeContext) getNetConfigFromNetmapContract() ([]stackitem.Item, error) { - r := management.NewReader(c.ReadOnlyInvoker) +func getNetConfigFromNetmapContract(roInvoker *invoker.Invoker) ([]stackitem.Item, error) { + r := management.NewReader(roInvoker) cs, err := r.GetContractByID(1) if err != nil { return nil, fmt.Errorf("get nns contract: %w", err) } - nmHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.NetmapContract)) + nmHash, err := morphUtil.NNSResolveHash(roInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.NetmapContract)) if err != nil { return nil, fmt.Errorf("can't get netmap contract hash: %w", err) } - arr, err := unwrap.Array(c.ReadOnlyInvoker.Call(nmHash, "listConfig")) + arr, err := unwrap.Array(roInvoker.Call(nmHash, "listConfig")) if err != nil { return nil, fmt.Errorf("can't fetch list of network config keys from the netmap contract") } return arr, err } -func (c *initializeContext) mergeNetmapConfig(md map[string]any) error { - arr, err := c.getNetConfigFromNetmapContract() +func mergeNetmapConfig(roInvoker *invoker.Invoker, md map[string]any) error { + arr, err := getNetConfigFromNetmapContract(roInvoker) if err != nil { return err } @@ -652,7 +626,7 @@ func (c *initializeContext) mergeNetmapConfig(md map[string]any) error { return nil } -func (c *initializeContext) getAlphabetDeployItems(i, n int) []any { +func (c *InitializeContext) GetAlphabetDeployItems(i, n int) []any { items := make([]any, 5) items[0] = c.Contracts[morphUtil.NetmapContract].Hash items[1] = c.Contracts[morphUtil.ProxyContract].Hash diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go index 7b0bc8197..2a80077c1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go @@ -14,16 +14,11 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" - nnsClient "github.com/nspcc-dev/neo-go/pkg/rpcclient/nns" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" ) @@ -31,14 +26,14 @@ const defaultExpirationTime = 10 * 365 * 24 * time.Hour / time.Second const frostfsOpsEmail = "ops@frostfs.info" -func (c *initializeContext) setNNS() error { +func setNNS(c *InitializeContext) error { r := management.NewReader(c.ReadOnlyInvoker) nnsCs, err := r.GetContractByID(1) if err != nil { return err } - ok, err := c.nnsRootRegistered(nnsCs.Hash, "frostfs") + ok, err := c.NNSRootRegistered(nnsCs.Hash, "frostfs") if err != nil { return err } else if !ok { @@ -47,48 +42,48 @@ func (c *initializeContext) setNNS() error { "frostfs", c.CommitteeAcc.Contract.ScriptHash(), frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) - if err := c.sendCommitteeTx(bw.Bytes(), true); err != nil { + if err := c.SendCommitteeTx(bw.Bytes(), true); err != nil { return fmt.Errorf("can't add domain root to NNS: %w", err) } - if err := c.awaitTx(); err != nil { + if err := c.AwaitTx(); err != nil { return err } } - alphaCs := c.getContract(morphUtil.AlphabetContract) + alphaCs := c.GetContract(morphUtil.AlphabetContract) for i, acc := range c.Accounts { alphaCs.Hash = state.CreateContractHash(acc.Contract.ScriptHash(), alphaCs.NEF.Checksum, alphaCs.Manifest.Name) domain := getAlphabetNNSDomain(i) - if err := c.nnsRegisterDomain(nnsCs.Hash, alphaCs.Hash, domain); err != nil { + if err := nnsRegisterDomain(c, nnsCs.Hash, alphaCs.Hash, domain); err != nil { return err } c.Command.Printf("NNS: Set %s -> %s\n", domain, alphaCs.Hash.StringLE()) } for _, ctrName := range contractList { - cs := c.getContract(ctrName) + cs := c.GetContract(ctrName) domain := ctrName + ".frostfs" - if err := c.nnsRegisterDomain(nnsCs.Hash, cs.Hash, domain); err != nil { + if err := nnsRegisterDomain(c, nnsCs.Hash, cs.Hash, domain); err != nil { return err } c.Command.Printf("NNS: Set %s -> %s\n", domain, cs.Hash.StringLE()) } groupKey := c.ContractWallet.Accounts[0].PrivateKey().PublicKey() - err = c.updateNNSGroup(nnsCs.Hash, groupKey) + err = updateNNSGroup(c, nnsCs.Hash, groupKey) if err != nil { return err } c.Command.Printf("NNS: Set %s -> %s\n", morphClient.NNSGroupKeyName, hex.EncodeToString(groupKey.Bytes())) - return c.awaitTx() + return c.AwaitTx() } -func (c *initializeContext) updateNNSGroup(nnsHash util.Uint160, pub *keys.PublicKey) error { +func updateNNSGroup(c *InitializeContext, nnsHash util.Uint160, pub *keys.PublicKey) error { bw := io.NewBufBinWriter() - keyAlreadyAdded, domainRegCodeEmitted, err := c.emitUpdateNNSGroupScript(bw, nnsHash, pub) + keyAlreadyAdded, domainRegCodeEmitted, err := c.EmitUpdateNNSGroupScript(bw, nnsHash, pub) if keyAlreadyAdded || err != nil { return err } @@ -101,20 +96,20 @@ func (c *initializeContext) updateNNSGroup(nnsHash util.Uint160, pub *keys.Publi script = w.Bytes() } - return c.sendCommitteeTx(script, true) + return c.SendCommitteeTx(script, true) } -// emitUpdateNNSGroupScript emits script for updating group key stored in NNS. +// EmitUpdateNNSGroupScript emits script for updating group key stored in NNS. // First return value is true iff the key is already there and nothing should be done. // Second return value is true iff a domain registration code was emitted. -func (c *initializeContext) emitUpdateNNSGroupScript(bw *io.BufBinWriter, nnsHash util.Uint160, pub *keys.PublicKey) (bool, bool, error) { - isAvail, err := nnsIsAvailable(c.Client, nnsHash, morphClient.NNSGroupKeyName) +func (c *InitializeContext) EmitUpdateNNSGroupScript(bw *io.BufBinWriter, nnsHash util.Uint160, pub *keys.PublicKey) (bool, bool, error) { + isAvail, err := morphUtil.NNSIsAvailable(c.Client, nnsHash, morphClient.NNSGroupKeyName) if err != nil { return false, false, err } if !isAvail { - currentPub, err := nnsResolveKey(c.ReadOnlyInvoker, nnsHash, morphClient.NNSGroupKeyName) + currentPub, err := morphUtil.NNSResolveKey(c.ReadOnlyInvoker, nnsHash, morphClient.NNSGroupKeyName) if err != nil { return false, false, err } @@ -165,8 +160,8 @@ func wrapRegisterScriptWithPrice(w *io.BufBinWriter, nnsHash util.Uint160, s []b } } -func (c *initializeContext) nnsRegisterDomainScript(nnsHash, expectedHash util.Uint160, domain string) ([]byte, bool, error) { - ok, err := nnsIsAvailable(c.Client, nnsHash, domain) +func (c *InitializeContext) NNSRegisterDomainScript(nnsHash, expectedHash util.Uint160, domain string) ([]byte, bool, error) { + ok, err := morphUtil.NNSIsAvailable(c.Client, nnsHash, domain) if err != nil { return nil, false, err } @@ -191,8 +186,8 @@ func (c *initializeContext) nnsRegisterDomainScript(nnsHash, expectedHash util.U return nil, s == expectedHash, nil } -func (c *initializeContext) nnsRegisterDomain(nnsHash, expectedHash util.Uint160, domain string) error { - script, ok, err := c.nnsRegisterDomainScript(nnsHash, expectedHash, domain) +func nnsRegisterDomain(c *InitializeContext, nnsHash, expectedHash util.Uint160, domain string) error { + script, ok, err := c.NNSRegisterDomainScript(nnsHash, expectedHash, domain) if ok || err != nil { return err } @@ -206,10 +201,10 @@ func (c *initializeContext) nnsRegisterDomain(nnsHash, expectedHash util.Uint160 domain, int64(nns.TXT), expectedHash.StringLE()) emit.AppCall(w.BinWriter, nnsHash, "addRecord", callflag.All, domain, int64(nns.TXT), address.Uint160ToString(expectedHash)) - return c.sendCommitteeTx(w.Bytes(), true) + return c.SendCommitteeTx(w.Bytes(), true) } -func (c *initializeContext) nnsRootRegistered(nnsHash util.Uint160, zone string) (bool, error) { +func (c *InitializeContext) NNSRootRegistered(nnsHash util.Uint160, zone string) (bool, error) { res, err := c.CommitteeAct.Call(nnsHash, "isAvailable", "name."+zone) if err != nil { return false, err @@ -219,43 +214,3 @@ func (c *initializeContext) nnsRootRegistered(nnsHash util.Uint160, zone string) } var errMissingNNSRecord = errors.New("missing NNS record") - -func nnsResolveKey(inv *invoker.Invoker, nnsHash util.Uint160, domain string) (*keys.PublicKey, error) { - res, err := morphUtil.NNSResolve(inv, nnsHash, domain) - if err != nil { - return nil, err - } - if _, ok := res.Value().(stackitem.Null); ok { - return nil, errors.New("NNS record is missing") - } - arr, ok := res.Value().([]stackitem.Item) - if !ok { - return nil, errors.New("API of the NNS contract method `resolve` has changed") - } - for i := range arr { - var bs []byte - bs, err = arr[i].TryBytes() - if err != nil { - continue - } - - return keys.NewPublicKeyFromString(string(bs)) - } - return nil, errors.New("no valid keys are found") -} - -func nnsIsAvailable(c morphUtil.Client, nnsHash util.Uint160, name string) (bool, error) { - switch c.(type) { - case *rpcclient.Client: - inv := invoker.New(c, nil) - reader := nnsClient.NewReader(inv, nnsHash) - return reader.IsAvailable(name) - default: - b, err := unwrap.Bool(morphUtil.InvokeFunction(c, nnsHash, "isAvailable", []any{name}, nil)) - if err != nil { - return false, fmt.Errorf("`isAvailable`: invalid response: %w", err) - } - - return b, nil - } -} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index 04afd7c77..9d3bcff09 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -28,8 +28,8 @@ const ( registerBatchSize = transaction.MaxAttributes - 1 ) -func (c *initializeContext) registerCandidateRange(start, end int) error { - regPrice, err := c.getCandidateRegisterPrice() +func registerCandidateRange(c *InitializeContext, start, end int) error { + regPrice, err := getCandidateRegisterPrice(c) if err != nil { return fmt.Errorf("can't fetch registration price: %w", err) } @@ -46,7 +46,7 @@ func (c *initializeContext) registerCandidateRange(start, end int) error { } signers := []actor.SignerAccount{{ - Signer: c.getSigner(false, c.CommitteeAcc), + Signer: c.GetSigner(false, c.CommitteeAcc), Account: c.CommitteeAcc, }} for _, acc := range c.Accounts[start:end] { @@ -68,7 +68,7 @@ func (c *initializeContext) registerCandidateRange(start, end int) error { if err != nil { return fmt.Errorf("can't create tx: %w", err) } - if err := c.multiSign(tx, morphUtil.CommitteeAccountName); err != nil { + if err := c.MultiSign(tx, morphUtil.CommitteeAccountName); err != nil { return fmt.Errorf("can't sign a transaction: %w", err) } @@ -82,7 +82,7 @@ func (c *initializeContext) registerCandidateRange(start, end int) error { return c.SendTx(tx, c.Command, true) } -func (c *initializeContext) registerCandidates() error { +func registerCandidates(c *InitializeContext) error { cc, err := unwrap.Array(c.ReadOnlyInvoker.Call(neo.Hash, "getCandidates")) if err != nil { return fmt.Errorf("`getCandidates`: %w", err) @@ -107,7 +107,7 @@ func (c *initializeContext) registerCandidates() error { if have >= end { continue } - if err := c.registerCandidateRange(start, end); err != nil { + if err := registerCandidateRange(c, start, end); err != nil { return fmt.Errorf("registering candidates %d..%d: %q", start, end-1, err) } } @@ -115,15 +115,15 @@ func (c *initializeContext) registerCandidates() error { return nil } -func (c *initializeContext) transferNEOToAlphabetContracts() error { +func transferNEOToAlphabetContracts(c *InitializeContext) error { neoHash := neo.Hash - ok, err := c.transferNEOFinished(neoHash) + ok, err := transferNEOFinished(c, neoHash) if ok || err != nil { return err } - cs := c.getContract(morphUtil.AlphabetContract) + cs := c.GetContract(morphUtil.AlphabetContract) amount := initialAlphabetNEOAmount / len(c.Wallets) bw := io.NewBufBinWriter() @@ -134,14 +134,14 @@ func (c *initializeContext) transferNEOToAlphabetContracts() error { emit.Opcodes(bw.BinWriter, opcode.ASSERT) } - if err := c.sendCommitteeTx(bw.Bytes(), false); err != nil { + if err := c.SendCommitteeTx(bw.Bytes(), false); err != nil { return err } - return c.awaitTx() + return c.AwaitTx() } -func (c *initializeContext) transferNEOFinished(neoHash util.Uint160) (bool, error) { +func transferNEOFinished(c *InitializeContext, neoHash util.Uint160) (bool, error) { r := nep17.NewReader(c.ReadOnlyInvoker, neoHash) bal, err := r.BalanceOf(c.CommitteeAcc.Contract.ScriptHash()) return bal.Cmp(big.NewInt(native.NEOTotalSupply)) == -1, err @@ -149,7 +149,7 @@ func (c *initializeContext) transferNEOFinished(neoHash util.Uint160) (bool, err var errGetPriceInvalid = errors.New("`getRegisterPrice`: invalid response") -func (c *initializeContext) getCandidateRegisterPrice() (int64, error) { +func getCandidateRegisterPrice(c *InitializeContext) (int64, error) { switch c.Client.(type) { case *rpcclient.Client: inv := invoker.New(c.Client, nil) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go b/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go index aed35e230..35750d602 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go @@ -9,8 +9,8 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/emit" ) -func (c *initializeContext) setNotaryAndAlphabetNodes() error { - if ok, err := c.setRolesFinished(); ok || err != nil { +func setNotaryAndAlphabetNodes(c *InitializeContext) error { + if ok, err := setRolesFinished(c); ok || err != nil { if err == nil { c.Command.Println("Stage 2: already performed.") } @@ -28,14 +28,14 @@ func (c *initializeContext) setNotaryAndAlphabetNodes() error { emit.AppCall(w.BinWriter, rolemgmt.Hash, "designateAsRole", callflag.States|callflag.AllowNotify, int64(noderoles.NeoFSAlphabet), pubs) - if err := c.sendCommitteeTx(w.Bytes(), false); err != nil { + if err := c.SendCommitteeTx(w.Bytes(), false); err != nil { return err } - return c.awaitTx() + return c.AwaitTx() } -func (c *initializeContext) setRolesFinished() (bool, error) { +func setRolesFinished(c *InitializeContext) (bool, error) { height, err := c.Client.GetBlockCount() if err != nil { return false, err diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go index a24d3a6f6..634b47892 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go @@ -28,8 +28,8 @@ const ( initialProxyGASAmount = 50_000 * native.GASFactor ) -func (c *initializeContext) transferFunds() error { - ok, err := c.transferFundsFinished() +func transferFunds(c *InitializeContext) error { + ok, err := transferFundsFinished(c) if ok || err != nil { if err == nil { c.Command.Println("Stage 1: already performed.") @@ -68,14 +68,14 @@ func (c *initializeContext) transferFunds() error { return fmt.Errorf("can't create transfer transaction: %w", err) } - if err := c.multiSignAndSend(tx, morphUtil.ConsensusAccountName); err != nil { + if err := c.MultiSignAndSend(tx, morphUtil.ConsensusAccountName); err != nil { return fmt.Errorf("can't send transfer transaction: %w", err) } - return c.awaitTx() + return c.AwaitTx() } -func (c *initializeContext) transferFundsFinished() (bool, error) { +func transferFundsFinished(c *InitializeContext) (bool, error) { acc := c.Accounts[0] r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) @@ -83,15 +83,15 @@ func (c *initializeContext) transferFundsFinished() (bool, error) { return res.Cmp(big.NewInt(initialAlphabetGASAmount/2)) == 1, err } -func (c *initializeContext) multiSignAndSend(tx *transaction.Transaction, accType string) error { - if err := c.multiSign(tx, accType); err != nil { +func (c *InitializeContext) MultiSignAndSend(tx *transaction.Transaction, accType string) error { + if err := c.MultiSign(tx, accType); err != nil { return err } return c.SendTx(tx, c.Command, false) } -func (c *initializeContext) multiSign(tx *transaction.Transaction, accType string) error { +func (c *InitializeContext) MultiSign(tx *transaction.Transaction, accType string) error { version, err := c.Client.GetVersion() if err != nil { // error appears only if client @@ -143,8 +143,8 @@ func (c *initializeContext) multiSign(tx *transaction.Transaction, accType strin return fmt.Errorf("%s account was not found among transaction signers", accType) } -func (c *initializeContext) transferGASToProxy() error { - proxyCs := c.getContract(morphUtil.ProxyContract) +func transferGASToProxy(c *InitializeContext) error { + proxyCs := c.GetContract(morphUtil.ProxyContract) r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) bal, err := r.BalanceOf(proxyCs.Hash) @@ -161,11 +161,11 @@ func (c *initializeContext) transferGASToProxy() error { return err } - if err := c.multiSignAndSend(tx, morphUtil.CommitteeAccountName); err != nil { + if err := c.MultiSignAndSend(tx, morphUtil.CommitteeAccountName); err != nil { return err } - return c.awaitTx() + return c.AwaitTx() } type transferTarget struct { diff --git a/cmd/frostfs-adm/internal/modules/morph/policy.go b/cmd/frostfs-adm/internal/modules/morph/policy.go index ba73d132c..97cc76ab7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy.go @@ -25,7 +25,7 @@ const ( ) func setPolicyCmd(cmd *cobra.Command, args []string) error { - wCtx, err := newInitializeContext(cmd, viper.GetViper()) + wCtx, err := NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) } @@ -51,11 +51,11 @@ func setPolicyCmd(cmd *cobra.Command, args []string) error { emit.AppCall(bw.BinWriter, policy.Hash, "set"+k, callflag.All, int64(value)) } - if err := wCtx.sendCommitteeTx(bw.Bytes(), false); err != nil { + if err := wCtx.SendCommitteeTx(bw.Bytes(), false); err != nil { return err } - return wCtx.awaitTx() + return wCtx.AwaitTx() } func dumpPolicyCmd(cmd *cobra.Command, _ []string) error { diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy.go index 9e973b0de..4bd461b97 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy.go @@ -36,7 +36,7 @@ func removeProxyAccount(cmd *cobra.Command, _ []string) { } func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error { - wCtx, err := newInitializeContext(cmd, viper.GetViper()) + wCtx, err := NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) } @@ -55,11 +55,11 @@ func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error bw := io.NewBufBinWriter() emit.AppCall(bw.BinWriter, proxyHash, method, callflag.All, addr) - if err := wCtx.sendConsensusTx(bw.Bytes()); err != nil { + if err := wCtx.SendConsensusTx(bw.Bytes()); err != nil { return err } - if err = wCtx.awaitTx(); err != nil { + if err = wCtx.AwaitTx(); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/remove_node.go b/cmd/frostfs-adm/internal/modules/morph/remove_node.go index 678bcc7ee..42464e731 100644 --- a/cmd/frostfs-adm/internal/modules/morph/remove_node.go +++ b/cmd/frostfs-adm/internal/modules/morph/remove_node.go @@ -29,11 +29,11 @@ func removeNodesCmd(cmd *cobra.Command, args []string) error { } } - wCtx, err := newInitializeContext(cmd, viper.GetViper()) + wCtx, err := NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't initialize context: %w", err) } - defer wCtx.close() + defer wCtx.Close() r := management.NewReader(wCtx.ReadOnlyInvoker) cs, err := r.GetContractByID(1) @@ -56,9 +56,9 @@ func removeNodesCmd(cmd *cobra.Command, args []string) error { return err } - if err := wCtx.sendConsensusTx(bw.Bytes()); err != nil { + if err := wCtx.SendConsensusTx(bw.Bytes()); err != nil { return err } - return wCtx.awaitTx() + return wCtx.AwaitTx() } diff --git a/cmd/frostfs-adm/internal/modules/morph/update.go b/cmd/frostfs-adm/internal/modules/morph/update.go index 90b6d6558..b6fcbb5b7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/update.go +++ b/cmd/frostfs-adm/internal/modules/morph/update.go @@ -8,14 +8,14 @@ import ( ) func updateContracts(cmd *cobra.Command, _ []string) error { - wCtx, err := newInitializeContext(cmd, viper.GetViper()) + wCtx, err := NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("initialization error: %w", err) } - if err := wCtx.deployNNS(updateMethodName); err != nil { + if err := deployNNS(wCtx, updateMethodName); err != nil { return err } - return wCtx.updateContracts() + return updateContractsInternal(wCtx) } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go index 923061a09..725181ec7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go @@ -7,8 +7,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + nns2 "github.com/nspcc-dev/neo-go/pkg/rpcclient/nns" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" @@ -144,3 +147,43 @@ func NNSResolveHash(inv *invoker.Invoker, nnsHash util.Uint160, domain string) ( func DomainOf(contract string) string { return contract + ".frostfs" } + +func NNSResolveKey(inv *invoker.Invoker, nnsHash util.Uint160, domain string) (*keys.PublicKey, error) { + res, err := NNSResolve(inv, nnsHash, domain) + if err != nil { + return nil, err + } + if _, ok := res.Value().(stackitem.Null); ok { + return nil, errors.New("NNS record is missing") + } + arr, ok := res.Value().([]stackitem.Item) + if !ok { + return nil, errors.New("API of the NNS contract method `resolve` has changed") + } + for i := range arr { + var bs []byte + bs, err = arr[i].TryBytes() + if err != nil { + continue + } + + return keys.NewPublicKeyFromString(string(bs)) + } + return nil, errors.New("no valid keys are found") +} + +func NNSIsAvailable(c Client, nnsHash util.Uint160, name string) (bool, error) { + switch c.(type) { + case *rpcclient.Client: + inv := invoker.New(c, nil) + reader := nns2.NewReader(inv, nnsHash) + return reader.IsAvailable(name) + default: + b, err := unwrap.Bool(InvokeFunction(c, nnsHash, "isAvailable", []any{name}, nil)) + if err != nil { + return false, fmt.Errorf("`isAvailable`: invalid response: %w", err) + } + + return b, nil + } +} diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go new file mode 100644 index 000000000..be5dcf2db --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go @@ -0,0 +1,34 @@ +package util + +import ( + "encoding/json" + "fmt" + + "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/nef" + "github.com/nspcc-dev/neo-go/pkg/util" +) + +func (cs *ContractState) Parse() error { + nf, err := nef.FileFromBytes(cs.RawNEF) + if err != nil { + return fmt.Errorf("can't parse NEF file: %w", err) + } + + m := new(manifest.Manifest) + if err := json.Unmarshal(cs.RawManifest, m); err != nil { + return fmt.Errorf("can't parse manifest file: %w", err) + } + + cs.NEF = &nf + cs.Manifest = m + return nil +} + +type ContractState struct { + NEF *nef.File + RawNEF []byte + Manifest *manifest.Manifest + RawManifest []byte + Hash util.Uint160 +} From 77694a2f3b4aebf1fd8b19ecd949b96c1608a6f6 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 15:21:51 +0300 Subject: [PATCH 0886/1943] [#932] adm: Move `InitializeContext` to `util` package Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/config.go | 2 +- .../internal/modules/morph/container.go | 8 +- .../internal/modules/morph/deploy.go | 10 +- .../internal/modules/morph/dump_hashes.go | 2 +- .../internal/modules/morph/epoch.go | 4 +- .../internal/modules/morph/frostfsid.go | 4 +- .../internal/modules/morph/generate.go | 4 +- .../internal/modules/morph/generate_test.go | 2 +- .../internal/modules/morph/group.go | 67 --- .../internal/modules/morph/initialize.go | 323 +---------- .../modules/morph/initialize_deploy.go | 206 +------ .../internal/modules/morph/initialize_nns.go | 85 +-- .../modules/morph/initialize_register.go | 10 +- .../modules/morph/initialize_roles.go | 4 +- .../internal/modules/morph/initialize_test.go | 16 +- .../modules/morph/initialize_transfer.go | 67 +-- .../internal/modules/morph/netmap_util.go | 5 +- .../internal/modules/morph/notary.go | 2 +- .../internal/modules/morph/policy.go | 2 +- .../internal/modules/morph/proxy.go | 2 +- .../internal/modules/morph/remove_node.go | 2 +- .../internal/modules/morph/root.go | 47 +- .../internal/modules/morph/update.go | 3 +- .../internal/modules/morph/util/const.go | 34 ++ .../modules/morph/{ => util}/download.go | 2 +- .../internal/modules/morph/util/initialize.go | 21 + .../modules/morph/util/initialize_ctx.go | 521 +++++++++++++++++- .../internal/modules/morph/util/util.go | 73 +++ .../internal/modules/morph/util/wallet.go | 75 +++ 29 files changed, 809 insertions(+), 794 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => util}/download.go (99%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/util/wallet.go diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config.go index 2d0f12afe..2cff322d1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config.go @@ -85,7 +85,7 @@ func setConfigCmd(cmd *cobra.Command, args []string) error { return errors.New("empty config pairs") } - wCtx, err := NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := util.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't initialize context: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/container.go b/cmd/frostfs-adm/internal/modules/morph/container.go index ab32dcabb..b18fa79ed 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container.go @@ -193,7 +193,7 @@ func restoreContainers(cmd *cobra.Command, _ []string) error { return fmt.Errorf("invalid filename: %w", err) } - wCtx, err := NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return err } @@ -222,7 +222,7 @@ func restoreContainers(cmd *cobra.Command, _ []string) error { return wCtx.AwaitTx() } -func restoreOrPutContainers(containers []Container, isOK func([]byte) bool, cmd *cobra.Command, wCtx *InitializeContext, ch util.Uint160) error { +func restoreOrPutContainers(containers []Container, isOK func([]byte) bool, cmd *cobra.Command, wCtx *morphUtil.InitializeContext, ch util.Uint160) error { bw := io.NewBufBinWriter() for _, cnt := range containers { hv := hash.Sha256(cnt.Value) @@ -262,7 +262,7 @@ func putContainer(bw *io.BufBinWriter, ch util.Uint160, cnt Container) { } } -func isContainerRestored(cmd *cobra.Command, wCtx *InitializeContext, containerHash util.Uint160, bw *io.BufBinWriter, hashValue util.Uint256) (bool, error) { +func isContainerRestored(cmd *cobra.Command, wCtx *morphUtil.InitializeContext, containerHash util.Uint160, bw *io.BufBinWriter, hashValue util.Uint256) (bool, error) { emit.AppCall(bw.BinWriter, containerHash, "get", callflag.All, hashValue.BytesBE()) res, err := wCtx.Client.InvokeScript(bw.Bytes(), nil) if err != nil { @@ -300,7 +300,7 @@ func parseContainers(filename string) ([]Container, error) { return containers, nil } -func fetchContainerContractHash(wCtx *InitializeContext) (util.Uint160, error) { +func fetchContainerContractHash(wCtx *morphUtil.InitializeContext) (util.Uint160, error) { r := management.NewReader(wCtx.ReadOnlyInvoker) nnsCs, err := r.GetContractByID(1) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/deploy.go b/cmd/frostfs-adm/internal/modules/morph/deploy.go index 7f11362e0..e6fdbbcf7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/deploy.go @@ -60,7 +60,7 @@ func init() { func deployContractCmd(cmd *cobra.Command, args []string) error { v := viper.GetViper() - c, err := NewInitializeContext(cmd, v) + c, err := util.NewInitializeContext(cmd, v) if err != nil { return fmt.Errorf("initialization error: %w", err) } @@ -72,7 +72,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { return err } - cs, err := readContract(ctrPath, ctrName) + cs, err := util.ReadContract(ctrPath, ctrName) if err != nil { return err } @@ -129,7 +129,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { return c.AwaitTx() } -func registerNNS(nnsCs *state.Contract, c *InitializeContext, zone string, domain string, cs *util.ContractState, writer *io.BufBinWriter) error { +func registerNNS(nnsCs *state.Contract, c *util.InitializeContext, zone string, domain string, cs *util.ContractState, writer *io.BufBinWriter) error { bw := io.NewBufBinWriter() emit.Instruction(bw.BinWriter, opcode.INITSSLOT, []byte{1}) emit.AppCall(bw.BinWriter, nnsCs.Hash, "getPrice", callflag.All) @@ -147,12 +147,12 @@ func registerNNS(nnsCs *state.Contract, c *InitializeContext, zone string, domai emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, zone, c.CommitteeAcc.Contract.ScriptHash(), - frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) + util.FrostfsOpsEmail, int64(3600), int64(600), int64(util.DefaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, domain, c.CommitteeAcc.Contract.ScriptHash(), - frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) + util.FrostfsOpsEmail, int64(3600), int64(600), int64(util.DefaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) } else { s, ok, err := c.NNSRegisterDomainScript(nnsCs.Hash, cs.Hash, domain) diff --git a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go index 79b82ee47..d33311a8b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go @@ -86,7 +86,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { } } - for _, ctrName := range contractList { + for _, ctrName := range morphUtil.ContractList { bw.Reset() emit.AppCall(bw.BinWriter, cs.Hash, "resolve", callflag.ReadOnly, morphUtil.DomainOf(ctrName), int64(nns.TXT)) diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/epoch.go index 0fc25d01d..851f757b0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/epoch.go @@ -17,7 +17,7 @@ import ( ) func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { - wCtx, err := NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := util2.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) } @@ -48,7 +48,7 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { return err } -func emitNewEpochCall(bw *io.BufBinWriter, wCtx *InitializeContext, nmHash util.Uint160) error { +func emitNewEpochCall(bw *io.BufBinWriter, wCtx *util2.InitializeContext, nmHash util.Uint160) error { curr, err := unwrap.Int64(wCtx.ReadOnlyInvoker.Call(nmHash, "epoch")) if err != nil { return errors.New("can't fetch current epoch from the netmap contract") diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go index 4adde815d..3e39c3019 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid.go @@ -416,11 +416,11 @@ type frostfsidClient struct { bw *io.BufBinWriter contractHash util.Uint160 roCli *frostfsidclient.Client // client can be used only for waiting tx, parsing and forming method params - wCtx *InitializeContext + wCtx *morphUtil.InitializeContext } func newFrostfsIDClient(cmd *cobra.Command) (*frostfsidClient, error) { - wCtx, err := NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return nil, fmt.Errorf("can't to initialize context: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index f8e155f5b..2258eb832 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -45,7 +45,7 @@ func generateAlphabetCreds(cmd *cobra.Command, _ []string) error { return err } - _, err = initializeContractWallet(v, walletDir) + _, err = morphUtil.InitializeContractWallet(v, walletDir) if err != nil { return err } @@ -195,7 +195,7 @@ func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error return err } - wCtx, err := NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate_test.go index 207366ec7..e9e942d74 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate_test.go @@ -101,7 +101,7 @@ func TestGenerateAlphabet(t *testing.T) { wg.Wait() t.Run("check contract group wallet", func(t *testing.T) { - p := filepath.Join(walletDir, contractWalletFilename) + p := filepath.Join(walletDir, util.ContractWalletFilename) w, err := wallet.NewWalletFromFile(p) require.NoError(t, err, "contract wallet doesn't exist") require.Equal(t, 1, len(w.Accounts), "contract wallet must have 1 accout") diff --git a/cmd/frostfs-adm/internal/modules/morph/group.go b/cmd/frostfs-adm/internal/modules/morph/group.go index 0e6c511da..655b8b2c3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/group.go +++ b/cmd/frostfs-adm/internal/modules/morph/group.go @@ -2,80 +2,13 @@ package morph import ( "encoding/json" - "fmt" - "os" - "path/filepath" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" util2 "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" - "github.com/spf13/cobra" - "github.com/spf13/viper" ) -const ( - contractWalletFilename = "contract.json" - contractWalletPasswordKey = "contract" -) - -func initializeContractWallet(v *viper.Viper, walletDir string) (*wallet.Wallet, error) { - password, err := config.GetPassword(v, contractWalletPasswordKey) - if err != nil { - return nil, err - } - - w, err := wallet.NewWallet(filepath.Join(walletDir, contractWalletFilename)) - if err != nil { - return nil, err - } - - acc, err := wallet.NewAccount() - if err != nil { - return nil, err - } - - err = acc.Encrypt(password, keys.NEP2ScryptParams()) - if err != nil { - return nil, err - } - - w.AddAccount(acc) - if err := w.SavePretty(); err != nil { - return nil, err - } - - return w, nil -} - -func openContractWallet(v *viper.Viper, cmd *cobra.Command, walletDir string) (*wallet.Wallet, error) { - p := filepath.Join(walletDir, contractWalletFilename) - w, err := wallet.NewWalletFromFile(p) - if err != nil { - if !os.IsNotExist(err) { - return nil, fmt.Errorf("can't open wallet: %w", err) - } - - cmd.Printf("Contract group wallet is missing, initialize at %s\n", p) - return initializeContractWallet(v, walletDir) - } - - password, err := config.GetPassword(v, contractWalletPasswordKey) - if err != nil { - return nil, err - } - - for i := range w.Accounts { - if err := w.Accounts[i].Decrypt(password, keys.NEP2ScryptParams()); err != nil { - return nil, fmt.Errorf("can't unlock wallet: %w", err) - } - } - - return w, nil -} - func addManifestGroup(cw *wallet.Wallet, h util.Uint160, cs *util2.ContractState) error { priv := cw.Accounts[0].PrivateKey() pub := priv.PublicKey() diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index 2057b4a24..d4219294c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -1,50 +1,15 @@ package morph import ( - "errors" "fmt" - "os" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" - morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "github.com/nspcc-dev/neo-go/pkg/core/native/nativenames" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/core/transaction" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" - "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" "github.com/spf13/viper" ) -type Cache struct { - NNSCs *state.Contract - GroupKey *keys.PublicKey -} - -type InitializeContext struct { - morphUtil.ClientContext - Cache - // CommitteeAcc is used for retrieving the committee address and the verification script. - CommitteeAcc *wallet.Account - // ConsensusAcc is used for retrieving the committee address and the verification script. - ConsensusAcc *wallet.Account - Wallets []*wallet.Wallet - // ContractWallet is a wallet for providing the contract group signature. - ContractWallet *wallet.Wallet - // Accounts contains simple signature accounts in the same order as in Wallets. - Accounts []*wallet.Account - Contracts map[string]*morphUtil.ContractState - Command *cobra.Command - ContractPath string - ContractURL string -} - func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { - initCtx, err := NewInitializeContext(cmd, viper.GetViper()) + initCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("initialization error: %w", err) } @@ -91,289 +56,3 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { cmd.Println("Stage 7: set addresses in NNS.") return setNNS(initCtx) } - -func (c *InitializeContext) Close() { - if local, ok := c.Client.(*morphUtil.LocalClient); ok { - err := local.Dump() - if err != nil { - c.Command.PrintErrf("Can't write dump: %v\n", err) - os.Exit(1) - } - } -} - -func NewInitializeContext(cmd *cobra.Command, v *viper.Viper) (*InitializeContext, error) { - walletDir := config.ResolveHomePath(viper.GetString(morphUtil.AlphabetWalletsFlag)) - wallets, err := morphUtil.GetAlphabetWallets(v, walletDir) - if err != nil { - return nil, err - } - - needContracts := cmd.Name() == "update-contracts" || cmd.Name() == "init" - - var w *wallet.Wallet - w, err = getWallet(cmd, v, needContracts, walletDir) - if err != nil { - return nil, err - } - - c, err := createClient(cmd, v, wallets) - if err != nil { - return nil, err - } - - committeeAcc, err := morphUtil.GetWalletAccount(wallets[0], morphUtil.CommitteeAccountName) - if err != nil { - return nil, fmt.Errorf("can't find committee account: %w", err) - } - - consensusAcc, err := morphUtil.GetWalletAccount(wallets[0], morphUtil.ConsensusAccountName) - if err != nil { - return nil, fmt.Errorf("can't find consensus account: %w", err) - } - - if err := validateInit(cmd); err != nil { - return nil, err - } - - ctrPath, err := getContractsPath(cmd, needContracts) - if err != nil { - return nil, err - } - - var ctrURL string - if needContracts { - ctrURL, _ = cmd.Flags().GetString(contractsURLFlag) - } - - if err := checkNotaryEnabled(c); err != nil { - return nil, err - } - - accounts, err := createWalletAccounts(wallets) - if err != nil { - return nil, err - } - - cliCtx, err := morphUtil.DefaultClientContext(c, committeeAcc) - if err != nil { - return nil, fmt.Errorf("client context: %w", err) - } - - initCtx := &InitializeContext{ - ClientContext: *cliCtx, - ConsensusAcc: consensusAcc, - CommitteeAcc: committeeAcc, - ContractWallet: w, - Wallets: wallets, - Accounts: accounts, - Command: cmd, - Contracts: make(map[string]*morphUtil.ContractState), - ContractPath: ctrPath, - ContractURL: ctrURL, - } - - if needContracts { - err := initCtx.readContracts(fullContractList) - if err != nil { - return nil, err - } - } - - return initCtx, nil -} - -func validateInit(cmd *cobra.Command) error { - if cmd.Name() != "init" { - return nil - } - if viper.GetInt64(epochDurationInitFlag) <= 0 { - return fmt.Errorf("epoch duration must be positive") - } - - if viper.GetInt64(maxObjectSizeInitFlag) <= 0 { - return fmt.Errorf("max object size must be positive") - } - - return nil -} - -func createClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) (morphUtil.Client, error) { - var c morphUtil.Client - var err error - if ldf := cmd.Flags().Lookup(localDumpFlag); ldf != nil && ldf.Changed { - if cmd.Flags().Changed(morphUtil.EndpointFlag) { - return nil, fmt.Errorf("`%s` and `%s` flags are mutually exclusive", morphUtil.EndpointFlag, localDumpFlag) - } - c, err = morphUtil.NewLocalClient(cmd, v, wallets, ldf.Value.String()) - } else { - c, err = morphUtil.GetN3Client(v) - } - if err != nil { - return nil, fmt.Errorf("can't create N3 client: %w", err) - } - return c, nil -} - -func getWallet(cmd *cobra.Command, v *viper.Viper, needContracts bool, walletDir string) (*wallet.Wallet, error) { - if !needContracts { - return nil, nil - } - return openContractWallet(v, cmd, walletDir) -} - -func getContractsPath(cmd *cobra.Command, needContracts bool) (string, error) { - if !needContracts { - return "", nil - } - - ctrPath, err := cmd.Flags().GetString(contractsInitFlag) - if err != nil { - return "", fmt.Errorf("invalid contracts path: %w", err) - } - return ctrPath, nil -} - -func createWalletAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) { - accounts := make([]*wallet.Account, len(wallets)) - for i, w := range wallets { - acc, err := morphUtil.GetWalletAccount(w, morphUtil.SingleAccountName) - if err != nil { - return nil, fmt.Errorf("wallet %s is invalid (no single account): %w", w.Path(), err) - } - accounts[i] = acc - } - return accounts, nil -} - -func (c *InitializeContext) AwaitTx() error { - return c.ClientContext.AwaitTx(c.Command) -} - -func (c *InitializeContext) NNSContractState() (*state.Contract, error) { - if c.NNSCs != nil { - return c.NNSCs, nil - } - - r := management.NewReader(c.ReadOnlyInvoker) - cs, err := r.GetContractByID(1) - if err != nil { - return nil, err - } - - c.NNSCs = cs - return cs, nil -} - -func (c *InitializeContext) GetSigner(tryGroup bool, acc *wallet.Account) transaction.Signer { - if tryGroup && c.GroupKey != nil { - return transaction.Signer{ - Account: acc.Contract.ScriptHash(), - Scopes: transaction.CustomGroups, - AllowedGroups: keys.PublicKeys{c.GroupKey}, - } - } - - signer := transaction.Signer{ - Account: acc.Contract.ScriptHash(), - Scopes: transaction.Global, // Scope is important, as we have nested call to container contract. - } - - if !tryGroup { - return signer - } - - nnsCs, err := c.NNSContractState() - if err != nil { - return signer - } - - groupKey, err := morphUtil.NNSResolveKey(c.ReadOnlyInvoker, nnsCs.Hash, morphClient.NNSGroupKeyName) - if err == nil { - c.GroupKey = groupKey - - signer.Scopes = transaction.CustomGroups - signer.AllowedGroups = keys.PublicKeys{groupKey} - } - return signer -} - -// SendCommitteeTx creates transaction from script, signs it by committee nodes and sends it to RPC. -// If tryGroup is false, global scope is used for the signer (useful when -// working with native contracts). -func (c *InitializeContext) SendCommitteeTx(script []byte, tryGroup bool) error { - return c.sendMultiTx(script, tryGroup, false) -} - -// SendConsensusTx creates transaction from script, signs it by alphabet nodes and sends it to RPC. -// Not that because this is used only after the contracts were initialized and deployed, -// we always try to have a group scope. -func (c *InitializeContext) SendConsensusTx(script []byte) error { - return c.sendMultiTx(script, true, true) -} - -func (c *InitializeContext) sendMultiTx(script []byte, tryGroup bool, withConsensus bool) error { - var act *actor.Actor - var err error - - withConsensus = withConsensus && !c.ConsensusAcc.Contract.ScriptHash().Equals(c.CommitteeAcc.ScriptHash()) - if tryGroup { - // Even for consensus signatures we need the committee to pay. - signers := make([]actor.SignerAccount, 1, 2) - signers[0] = actor.SignerAccount{ - Signer: c.GetSigner(tryGroup, c.CommitteeAcc), - Account: c.CommitteeAcc, - } - if withConsensus { - signers = append(signers, actor.SignerAccount{ - Signer: c.GetSigner(tryGroup, c.ConsensusAcc), - Account: c.ConsensusAcc, - }) - } - act, err = actor.New(c.Client, signers) - } else { - if withConsensus { - panic("BUG: should never happen") - } - act, err = c.CommitteeAct, nil - } - if err != nil { - return fmt.Errorf("could not create actor: %w", err) - } - - tx, err := act.MakeUnsignedRun(script, []transaction.Attribute{{Type: transaction.HighPriority}}) - if err != nil { - return fmt.Errorf("could not perform test invocation: %w", err) - } - - if err := c.MultiSign(tx, morphUtil.CommitteeAccountName); err != nil { - return err - } - if withConsensus { - if err := c.MultiSign(tx, morphUtil.ConsensusAccountName); err != nil { - return err - } - } - - return c.SendTx(tx, c.Command, false) -} - -func checkNotaryEnabled(c morphUtil.Client) error { - ns, err := c.GetNativeContracts() - if err != nil { - return fmt.Errorf("can't get native contract hashes: %w", err) - } - - notaryEnabled := false - nativeHashes := make(map[string]util.Uint160, len(ns)) - for i := range ns { - if ns[i].Manifest.Name == nativenames.Notary { - notaryEnabled = true - } - nativeHashes[ns[i].Manifest.Name] = ns[i].Hash - } - if !notaryEnabled { - return errors.New("notary contract must be enabled") - } - return nil -} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index fd27eda6c..322033abc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -1,20 +1,14 @@ package morph import ( - "archive/tar" - "compress/gzip" "encoding/hex" "errors" "fmt" - "io" - "os" - "path/filepath" "strings" "git.frostfs.info/TrueCloudLab/frostfs-contract/common" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -35,41 +29,23 @@ import ( const frostfsIDAdminConfigKey = "frostfsid.admin" -var ( - contractList = []string{ - morphUtil.BalanceContract, - morphUtil.ContainerContract, - morphUtil.FrostfsIDContract, - morphUtil.NetmapContract, - morphUtil.PolicyContract, - morphUtil.ProxyContract, - } - - fullContractList = append([]string{ - morphUtil.FrostfsContract, - morphUtil.ProcessingContract, - morphUtil.NNSContract, - morphUtil.AlphabetContract, - }, contractList...) - - netmapConfigKeys = []string{ - netmap.EpochDurationConfig, - netmap.MaxObjectSizeConfig, - netmap.ContainerFeeConfig, - netmap.ContainerAliasFeeConfig, - netmap.IrCandidateFeeConfig, - netmap.WithdrawFeeConfig, - netmap.HomomorphicHashingDisabledKey, - netmap.MaintenanceModeAllowedConfig, - } -) +var netmapConfigKeys = []string{ + netmap.EpochDurationConfig, + netmap.MaxObjectSizeConfig, + netmap.ContainerFeeConfig, + netmap.ContainerAliasFeeConfig, + netmap.IrCandidateFeeConfig, + netmap.WithdrawFeeConfig, + netmap.HomomorphicHashingDisabledKey, + netmap.MaintenanceModeAllowedConfig, +} const ( updateMethodName = "update" deployMethodName = "deploy" ) -func deployNNS(c *InitializeContext, method string) error { +func deployNNS(c *morphUtil.InitializeContext, method string) error { cs := c.GetContract(morphUtil.NNSContract) h := cs.Hash @@ -113,7 +89,7 @@ func deployNNS(c *InitializeContext, method string) error { return c.AwaitTx() } -func updateContractsInternal(c *InitializeContext) error { +func updateContractsInternal(c *morphUtil.InitializeContext) error { alphaCs := c.GetContract(morphUtil.AlphabetContract) nnsCs, err := c.NNSContractState() @@ -166,13 +142,13 @@ func updateContractsInternal(c *InitializeContext) error { return c.AwaitTx() } -func deployOrUpdateContracts(c *InitializeContext, w *io2.BufBinWriter, nnsHash util.Uint160, keysParam []any) error { +func deployOrUpdateContracts(c *morphUtil.InitializeContext, w *io2.BufBinWriter, nnsHash util.Uint160, keysParam []any) error { emit.Instruction(w.BinWriter, opcode.INITSSLOT, []byte{1}) emit.AppCall(w.BinWriter, nnsHash, "getPrice", callflag.All) emit.Opcodes(w.BinWriter, opcode.STSFLD0) emit.AppCall(w.BinWriter, nnsHash, "setPrice", callflag.All, 1) - for _, ctrName := range contractList { + for _, ctrName := range morphUtil.ContractList { cs := c.GetContract(ctrName) method := updateMethodName @@ -233,7 +209,7 @@ func deployOrUpdateContracts(c *InitializeContext, w *io2.BufBinWriter, nnsHash return nil } -func deployAlphabetAccounts(c *InitializeContext, nnsHash util.Uint160, w *io2.BufBinWriter, alphaCs *morphUtil.ContractState) ([]any, error) { +func deployAlphabetAccounts(c *morphUtil.InitializeContext, nnsHash util.Uint160, w *io2.BufBinWriter, alphaCs *morphUtil.ContractState) ([]any, error) { var keysParam []any baseGroups := alphaCs.Manifest.Groups @@ -272,7 +248,7 @@ func deployAlphabetAccounts(c *InitializeContext, nnsHash util.Uint160, w *io2.B return keysParam, nil } -func deployContracts(c *InitializeContext) error { +func deployContracts(c *morphUtil.InitializeContext) error { alphaCs := c.GetContract(morphUtil.AlphabetContract) var keysParam []any @@ -309,7 +285,7 @@ func deployContracts(c *InitializeContext) error { c.SentTxs = append(c.SentTxs, morphUtil.HashVUBPair{Hash: txHash, Vub: vub}) } - for _, ctrName := range contractList { + for _, ctrName := range morphUtil.ContractList { cs := c.GetContract(ctrName) ctrHash := cs.Hash @@ -341,147 +317,11 @@ func deployContracts(c *InitializeContext) error { return c.AwaitTx() } -func (c *InitializeContext) IsUpdated(ctrHash util.Uint160, cs *morphUtil.ContractState) bool { - r := management.NewReader(c.ReadOnlyInvoker) - realCs, err := r.GetContract(ctrHash) - return err == nil && realCs != nil && realCs.NEF.Checksum == cs.NEF.Checksum -} - -func (c *InitializeContext) GetContract(ctrName string) *morphUtil.ContractState { - return c.Contracts[ctrName] -} - -func (c *InitializeContext) readContracts(names []string) error { - var ( - fi os.FileInfo - err error - ) - if c.ContractPath != "" { - fi, err = os.Stat(c.ContractPath) - if err != nil { - return fmt.Errorf("invalid contracts path: %w", err) - } - } - - if c.ContractPath != "" && fi.IsDir() { - for _, ctrName := range names { - cs, err := readContract(filepath.Join(c.ContractPath, ctrName), ctrName) - if err != nil { - return err - } - c.Contracts[ctrName] = cs - } - } else { - var r io.ReadCloser - if c.ContractPath != "" { - r, err = os.Open(c.ContractPath) - } else if c.ContractURL != "" { - r, err = downloadContracts(c.Command, c.ContractURL) - } else { - r, err = downloadContractsFromRepository(c.Command) - } - - if err != nil { - return fmt.Errorf("can't open contracts archive: %w", err) - } - defer r.Close() - - m, err := readContractsFromArchive(r, names) - if err != nil { - return err - } - for _, name := range names { - if err := m[name].Parse(); err != nil { - return err - } - c.Contracts[name] = m[name] - } - } - - for _, ctrName := range names { - if ctrName != morphUtil.AlphabetContract { - cs := c.Contracts[ctrName] - cs.Hash = state.CreateContractHash(c.CommitteeAcc.Contract.ScriptHash(), - cs.NEF.Checksum, cs.Manifest.Name) - } - } - return nil -} - -func readContract(ctrPath, ctrName string) (*morphUtil.ContractState, error) { - rawNef, err := os.ReadFile(filepath.Join(ctrPath, ctrName+"_contract.nef")) - if err != nil { - return nil, fmt.Errorf("can't read NEF file for %s contract: %w", ctrName, err) - } - rawManif, err := os.ReadFile(filepath.Join(ctrPath, "config.json")) - if err != nil { - return nil, fmt.Errorf("can't read manifest file for %s contract: %w", ctrName, err) - } - - cs := &morphUtil.ContractState{ - RawNEF: rawNef, - RawManifest: rawManif, - } - - return cs, cs.Parse() -} - -func readContractsFromArchive(file io.Reader, names []string) (map[string]*morphUtil.ContractState, error) { - m := make(map[string]*morphUtil.ContractState, len(names)) - for i := range names { - m[names[i]] = new(morphUtil.ContractState) - } - - gr, err := gzip.NewReader(file) - if err != nil { - return nil, fmt.Errorf("contracts file must be tar.gz archive: %w", err) - } - - r := tar.NewReader(gr) - for h, err := r.Next(); ; h, err = r.Next() { - if err != nil { - break - } - - dir, _ := filepath.Split(h.Name) - ctrName := filepath.Base(dir) - - cs, ok := m[ctrName] - if !ok { - continue - } - - switch { - case strings.HasSuffix(h.Name, filepath.Join(ctrName, ctrName+"_contract.nef")): - cs.RawNEF, err = io.ReadAll(r) - if err != nil { - return nil, fmt.Errorf("can't read NEF file for %s contract: %w", ctrName, err) - } - case strings.HasSuffix(h.Name, "config.json"): - cs.RawManifest, err = io.ReadAll(r) - if err != nil { - return nil, fmt.Errorf("can't read manifest file for %s contract: %w", ctrName, err) - } - } - m[ctrName] = cs - } - - for ctrName, cs := range m { - if cs.RawNEF == nil { - return nil, fmt.Errorf("NEF for %s contract wasn't found", ctrName) - } - if cs.RawManifest == nil { - return nil, fmt.Errorf("manifest for %s contract wasn't found", ctrName) - } - } - return m, nil -} - func getContractDeployParameters(cs *morphUtil.ContractState, deployData []any) []any { return []any{cs.RawNEF, cs.RawManifest, deployData} } -func getContractDeployData(c *InitializeContext, ctrName string, keysParam []any, method string) ([]any, error) { +func getContractDeployData(c *morphUtil.InitializeContext, ctrName string, keysParam []any, method string) ([]any, error) { items := make([]any, 0, 6) switch ctrName { @@ -625,13 +465,3 @@ func mergeNetmapConfig(roInvoker *invoker.Invoker, md map[string]any) error { } return nil } - -func (c *InitializeContext) GetAlphabetDeployItems(i, n int) []any { - items := make([]any, 5) - items[0] = c.Contracts[morphUtil.NetmapContract].Hash - items[1] = c.Contracts[morphUtil.ProxyContract].Hash - items[2] = innerring.GlagoliticLetter(i).String() - items[3] = int64(i) - items[4] = int64(n) - return items -} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go index 2a80077c1..8e5223098 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "strconv" - "time" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" @@ -19,14 +18,9 @@ import ( "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" - "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" ) -const defaultExpirationTime = 10 * 365 * 24 * time.Hour / time.Second - -const frostfsOpsEmail = "ops@frostfs.info" - -func setNNS(c *InitializeContext) error { +func setNNS(c *morphUtil.InitializeContext) error { r := management.NewReader(c.ReadOnlyInvoker) nnsCs, err := r.GetContractByID(1) if err != nil { @@ -40,7 +34,7 @@ func setNNS(c *InitializeContext) error { bw := io.NewBufBinWriter() emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, "frostfs", c.CommitteeAcc.Contract.ScriptHash(), - frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) + morphUtil.FrostfsOpsEmail, int64(3600), int64(600), int64(morphUtil.DefaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) if err := c.SendCommitteeTx(bw.Bytes(), true); err != nil { return fmt.Errorf("can't add domain root to NNS: %w", err) @@ -61,7 +55,7 @@ func setNNS(c *InitializeContext) error { c.Command.Printf("NNS: Set %s -> %s\n", domain, alphaCs.Hash.StringLE()) } - for _, ctrName := range contractList { + for _, ctrName := range morphUtil.ContractList { cs := c.GetContract(ctrName) domain := ctrName + ".frostfs" @@ -81,7 +75,7 @@ func setNNS(c *InitializeContext) error { return c.AwaitTx() } -func updateNNSGroup(c *InitializeContext, nnsHash util.Uint160, pub *keys.PublicKey) error { +func updateNNSGroup(c *morphUtil.InitializeContext, nnsHash util.Uint160, pub *keys.PublicKey) error { bw := io.NewBufBinWriter() keyAlreadyAdded, domainRegCodeEmitted, err := c.EmitUpdateNNSGroupScript(bw, nnsHash, pub) if keyAlreadyAdded || err != nil { @@ -99,40 +93,6 @@ func updateNNSGroup(c *InitializeContext, nnsHash util.Uint160, pub *keys.Public return c.SendCommitteeTx(script, true) } -// EmitUpdateNNSGroupScript emits script for updating group key stored in NNS. -// First return value is true iff the key is already there and nothing should be done. -// Second return value is true iff a domain registration code was emitted. -func (c *InitializeContext) EmitUpdateNNSGroupScript(bw *io.BufBinWriter, nnsHash util.Uint160, pub *keys.PublicKey) (bool, bool, error) { - isAvail, err := morphUtil.NNSIsAvailable(c.Client, nnsHash, morphClient.NNSGroupKeyName) - if err != nil { - return false, false, err - } - - if !isAvail { - currentPub, err := morphUtil.NNSResolveKey(c.ReadOnlyInvoker, nnsHash, morphClient.NNSGroupKeyName) - if err != nil { - return false, false, err - } - - if pub.Equal(currentPub) { - return true, false, nil - } - } - - if isAvail { - emit.AppCall(bw.BinWriter, nnsHash, "register", callflag.All, - morphClient.NNSGroupKeyName, c.CommitteeAcc.Contract.ScriptHash(), - frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) - emit.Opcodes(bw.BinWriter, opcode.ASSERT) - } - - emit.AppCall(bw.BinWriter, nnsHash, "deleteRecords", callflag.All, "group.frostfs", int64(nns.TXT)) - emit.AppCall(bw.BinWriter, nnsHash, "addRecord", callflag.All, - "group.frostfs", int64(nns.TXT), hex.EncodeToString(pub.Bytes())) - - return false, isAvail, nil -} - func getAlphabetNNSDomain(i int) string { return morphUtil.AlphabetContract + strconv.FormatUint(uint64(i), 10) + ".frostfs" } @@ -160,33 +120,7 @@ func wrapRegisterScriptWithPrice(w *io.BufBinWriter, nnsHash util.Uint160, s []b } } -func (c *InitializeContext) NNSRegisterDomainScript(nnsHash, expectedHash util.Uint160, domain string) ([]byte, bool, error) { - ok, err := morphUtil.NNSIsAvailable(c.Client, nnsHash, domain) - if err != nil { - return nil, false, err - } - - if ok { - bw := io.NewBufBinWriter() - emit.AppCall(bw.BinWriter, nnsHash, "register", callflag.All, - domain, c.CommitteeAcc.Contract.ScriptHash(), - frostfsOpsEmail, int64(3600), int64(600), int64(defaultExpirationTime), int64(3600)) - emit.Opcodes(bw.BinWriter, opcode.ASSERT) - - if bw.Err != nil { - panic(bw.Err) - } - return bw.Bytes(), false, nil - } - - s, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, domain) - if err != nil { - return nil, false, err - } - return nil, s == expectedHash, nil -} - -func nnsRegisterDomain(c *InitializeContext, nnsHash, expectedHash util.Uint160, domain string) error { +func nnsRegisterDomain(c *morphUtil.InitializeContext, nnsHash, expectedHash util.Uint160, domain string) error { script, ok, err := c.NNSRegisterDomainScript(nnsHash, expectedHash, domain) if ok || err != nil { return err @@ -204,13 +138,4 @@ func nnsRegisterDomain(c *InitializeContext, nnsHash, expectedHash util.Uint160, return c.SendCommitteeTx(w.Bytes(), true) } -func (c *InitializeContext) NNSRootRegistered(nnsHash util.Uint160, zone string) (bool, error) { - res, err := c.CommitteeAct.Call(nnsHash, "isAvailable", "name."+zone) - if err != nil { - return false, err - } - - return res.State == vmstate.Halt.String(), nil -} - var errMissingNNSRecord = errors.New("missing NNS record") diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go index 9d3bcff09..d6cbd2d56 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_register.go @@ -28,7 +28,7 @@ const ( registerBatchSize = transaction.MaxAttributes - 1 ) -func registerCandidateRange(c *InitializeContext, start, end int) error { +func registerCandidateRange(c *morphUtil.InitializeContext, start, end int) error { regPrice, err := getCandidateRegisterPrice(c) if err != nil { return fmt.Errorf("can't fetch registration price: %w", err) @@ -82,7 +82,7 @@ func registerCandidateRange(c *InitializeContext, start, end int) error { return c.SendTx(tx, c.Command, true) } -func registerCandidates(c *InitializeContext) error { +func registerCandidates(c *morphUtil.InitializeContext) error { cc, err := unwrap.Array(c.ReadOnlyInvoker.Call(neo.Hash, "getCandidates")) if err != nil { return fmt.Errorf("`getCandidates`: %w", err) @@ -115,7 +115,7 @@ func registerCandidates(c *InitializeContext) error { return nil } -func transferNEOToAlphabetContracts(c *InitializeContext) error { +func transferNEOToAlphabetContracts(c *morphUtil.InitializeContext) error { neoHash := neo.Hash ok, err := transferNEOFinished(c, neoHash) @@ -141,7 +141,7 @@ func transferNEOToAlphabetContracts(c *InitializeContext) error { return c.AwaitTx() } -func transferNEOFinished(c *InitializeContext, neoHash util.Uint160) (bool, error) { +func transferNEOFinished(c *morphUtil.InitializeContext, neoHash util.Uint160) (bool, error) { r := nep17.NewReader(c.ReadOnlyInvoker, neoHash) bal, err := r.BalanceOf(c.CommitteeAcc.Contract.ScriptHash()) return bal.Cmp(big.NewInt(native.NEOTotalSupply)) == -1, err @@ -149,7 +149,7 @@ func transferNEOFinished(c *InitializeContext, neoHash util.Uint160) (bool, erro var errGetPriceInvalid = errors.New("`getRegisterPrice`: invalid response") -func getCandidateRegisterPrice(c *InitializeContext) (int64, error) { +func getCandidateRegisterPrice(c *morphUtil.InitializeContext) (int64, error) { switch c.Client.(type) { case *rpcclient.Client: inv := invoker.New(c.Client, nil) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go b/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go index 35750d602..9e885d5f8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go @@ -9,7 +9,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/emit" ) -func setNotaryAndAlphabetNodes(c *InitializeContext) error { +func setNotaryAndAlphabetNodes(c *util.InitializeContext) error { if ok, err := setRolesFinished(c); ok || err != nil { if err == nil { c.Command.Println("Stage 2: already performed.") @@ -35,7 +35,7 @@ func setNotaryAndAlphabetNodes(c *InitializeContext) error { return c.AwaitTx() } -func setRolesFinished(c *InitializeContext) (bool, error) { +func setRolesFinished(c *util.InitializeContext) (bool, error) { height, err := c.Client.GetBlockCount() if err != nil { return false, err diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index d5d50eca8..63e9887c0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -58,27 +58,27 @@ func testInitialize(t *testing.T, committeeSize int) { v.Set(util.ProtoConfigPath, filepath.Join(testdataDir, protoFileName)) // Set to the path or remove the next statement to download from the network. - require.NoError(t, initCmd.Flags().Set(contractsInitFlag, contractsPath)) + require.NoError(t, initCmd.Flags().Set(util.ContractsInitFlag, contractsPath)) dumpPath := filepath.Join(testdataDir, "out") - require.NoError(t, initCmd.Flags().Set(localDumpFlag, dumpPath)) + require.NoError(t, initCmd.Flags().Set(util.LocalDumpFlag, dumpPath)) v.Set(util.AlphabetWalletsFlag, testdataDir) - v.Set(epochDurationInitFlag, 1) - v.Set(maxObjectSizeInitFlag, 1024) + v.Set(util.EpochDurationInitFlag, 1) + v.Set(util.MaxObjectSizeInitFlag, 1024) setTestCredentials(v, committeeSize) require.NoError(t, initializeSideChainCmd(initCmd, nil)) t.Run("force-new-epoch", func(t *testing.T) { - require.NoError(t, forceNewEpoch.Flags().Set(localDumpFlag, dumpPath)) + require.NoError(t, forceNewEpoch.Flags().Set(util.LocalDumpFlag, dumpPath)) require.NoError(t, forceNewEpochCmd(forceNewEpoch, nil)) }) t.Run("set-config", func(t *testing.T) { - require.NoError(t, setConfig.Flags().Set(localDumpFlag, dumpPath)) + require.NoError(t, setConfig.Flags().Set(util.LocalDumpFlag, dumpPath)) require.NoError(t, setConfigCmd(setConfig, []string{"MaintenanceModeAllowed=true"})) }) t.Run("set-policy", func(t *testing.T) { - require.NoError(t, setPolicy.Flags().Set(localDumpFlag, dumpPath)) + require.NoError(t, setPolicy.Flags().Set(util.LocalDumpFlag, dumpPath)) require.NoError(t, setPolicyCmd(setPolicy, []string{"ExecFeeFactor=1"})) }) t.Run("remove-node", func(t *testing.T) { @@ -86,7 +86,7 @@ func testInitialize(t *testing.T, committeeSize int) { require.NoError(t, err) pub := hex.EncodeToString(pk.PublicKey().Bytes()) - require.NoError(t, removeNodes.Flags().Set(localDumpFlag, dumpPath)) + require.NoError(t, removeNodes.Flags().Set(util.LocalDumpFlag, dumpPath)) require.NoError(t, removeNodesCmd(removeNodes, []string{pub})) }) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go index 634b47892..ab4acefee 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go @@ -13,7 +13,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/rpcclient/neo" "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" - scContext "github.com/nspcc-dev/neo-go/pkg/smartcontract/context" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" @@ -28,7 +27,7 @@ const ( initialProxyGASAmount = 50_000 * native.GASFactor ) -func transferFunds(c *InitializeContext) error { +func transferFunds(c *morphUtil.InitializeContext) error { ok, err := transferFundsFinished(c) if ok || err != nil { if err == nil { @@ -75,7 +74,7 @@ func transferFunds(c *InitializeContext) error { return c.AwaitTx() } -func transferFundsFinished(c *InitializeContext) (bool, error) { +func transferFundsFinished(c *morphUtil.InitializeContext) (bool, error) { acc := c.Accounts[0] r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) @@ -83,67 +82,7 @@ func transferFundsFinished(c *InitializeContext) (bool, error) { return res.Cmp(big.NewInt(initialAlphabetGASAmount/2)) == 1, err } -func (c *InitializeContext) MultiSignAndSend(tx *transaction.Transaction, accType string) error { - if err := c.MultiSign(tx, accType); err != nil { - return err - } - - return c.SendTx(tx, c.Command, false) -} - -func (c *InitializeContext) MultiSign(tx *transaction.Transaction, accType string) error { - version, err := c.Client.GetVersion() - if err != nil { - // error appears only if client - // has not been initialized - panic(err) - } - network := version.Protocol.Network - - // Use parameter context to avoid dealing with signature order. - pc := scContext.NewParameterContext("", network, tx) - h := c.CommitteeAcc.Contract.ScriptHash() - if accType == morphUtil.ConsensusAccountName { - h = c.ConsensusAcc.Contract.ScriptHash() - } - for _, w := range c.Wallets { - acc, err := morphUtil.GetWalletAccount(w, accType) - if err != nil { - return fmt.Errorf("can't find %s wallet account: %w", accType, err) - } - - priv := acc.PrivateKey() - sign := priv.SignHashable(uint32(network), tx) - if err := pc.AddSignature(h, acc.Contract, priv.PublicKey(), sign); err != nil { - return fmt.Errorf("can't add signature: %w", err) - } - if len(pc.Items[h].Signatures) == len(acc.Contract.Parameters) { - break - } - } - - w, err := pc.GetWitness(h) - if err != nil { - return fmt.Errorf("incomplete signature: %w", err) - } - - for i := range tx.Signers { - if tx.Signers[i].Account == h { - if i < len(tx.Scripts) { - tx.Scripts[i] = *w - } else if i == len(tx.Scripts) { - tx.Scripts = append(tx.Scripts, *w) - } else { - panic("BUG: invalid signing order") - } - return nil - } - } - - return fmt.Errorf("%s account was not found among transaction signers", accType) -} - -func transferGASToProxy(c *InitializeContext) error { +func transferGASToProxy(c *morphUtil.InitializeContext) error { proxyCs := c.GetContract(morphUtil.ProxyContract) r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_util.go b/cmd/frostfs-adm/internal/modules/morph/netmap_util.go index fa7aa0af3..3cf853cca 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap_util.go @@ -3,6 +3,7 @@ package morph import ( "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/spf13/viper" @@ -10,8 +11,8 @@ import ( func getDefaultNetmapContractConfigMap() map[string]any { m := make(map[string]any) - m[netmap.EpochDurationConfig] = viper.GetInt64(epochDurationInitFlag) - m[netmap.MaxObjectSizeConfig] = viper.GetInt64(maxObjectSizeInitFlag) + m[netmap.EpochDurationConfig] = viper.GetInt64(util.EpochDurationInitFlag) + m[netmap.MaxObjectSizeConfig] = viper.GetInt64(util.MaxObjectSizeInitFlag) m[netmap.ContainerFeeConfig] = viper.GetInt64(containerFeeInitFlag) m[netmap.ContainerAliasFeeConfig] = viper.GetInt64(containerAliasFeeInitFlag) m[netmap.IrCandidateFeeConfig] = viper.GetInt64(candidateFeeInitFlag) diff --git a/cmd/frostfs-adm/internal/modules/morph/notary.go b/cmd/frostfs-adm/internal/modules/morph/notary.go index 3e7afdbf2..d05307705 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary.go @@ -85,7 +85,7 @@ func transferGas(cmd *cobra.Command, acc *wallet.Account, accHash util.Uint160, return err } - if err := checkNotaryEnabled(c); err != nil { + if err := morphUtil.CheckNotaryEnabled(c); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/policy.go b/cmd/frostfs-adm/internal/modules/morph/policy.go index 97cc76ab7..d01c5e1a5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy.go @@ -25,7 +25,7 @@ const ( ) func setPolicyCmd(cmd *cobra.Command, args []string) error { - wCtx, err := NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := util.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy.go index 4bd461b97..aa98fb70d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy.go @@ -36,7 +36,7 @@ func removeProxyAccount(cmd *cobra.Command, _ []string) { } func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error { - wCtx, err := NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := util2.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/remove_node.go b/cmd/frostfs-adm/internal/modules/morph/remove_node.go index 42464e731..e63a5dee9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/remove_node.go +++ b/cmd/frostfs-adm/internal/modules/morph/remove_node.go @@ -29,7 +29,7 @@ func removeNodesCmd(cmd *cobra.Command, args []string) error { } } - wCtx, err := NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := util.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't initialize context: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 328877967..7585f5845 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -8,18 +8,14 @@ import ( ) const ( - alphabetSizeFlag = "size" - storageWalletFlag = "storage-wallet" - storageWalletLabelFlag = "label" - storageGasCLIFlag = "initial-gas" - storageGasConfigFlag = "storage.initial_gas" - contractsInitFlag = "contracts" - contractsInitFlagDesc = "Path to archive with compiled FrostFS contracts (the default is to fetch the latest release from the official repository)" - contractsURLFlag = "contracts-url" - contractsURLFlagDesc = "URL to archive with compiled FrostFS contracts" - maxObjectSizeInitFlag = "network.max_object_size" - maxObjectSizeCLIFlag = "max-object-size" - epochDurationInitFlag = "network.epoch_duration" + alphabetSizeFlag = "size" + storageWalletFlag = "storage-wallet" + storageWalletLabelFlag = "label" + storageGasCLIFlag = "initial-gas" + storageGasConfigFlag = "storage.initial_gas" + + maxObjectSizeCLIFlag = "max-object-size" + epochDurationCLIFlag = "epoch-duration" containerFeeInitFlag = "network.fee.container" containerAliasFeeInitFlag = "network.fee.container_alias" @@ -38,7 +34,6 @@ const ( refillGasAmountFlag = "gas" walletAccountFlag = "account" notaryDepositTillFlag = "till" - localDumpFlag = "local-dump" walletAddressFlag = "wallet-address" ) @@ -66,8 +61,8 @@ var ( PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - _ = viper.BindPFlag(epochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) - _ = viper.BindPFlag(maxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) + _ = viper.BindPFlag(util.EpochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) + _ = viper.BindPFlag(util.MaxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) _ = viper.BindPFlag(homomorphicHashDisabledInitFlag, cmd.Flags().Lookup(homomorphicHashDisabledCLIFlag)) _ = viper.BindPFlag(candidateFeeInitFlag, cmd.Flags().Lookup(candidateFeeCLIFlag)) _ = viper.BindPFlag(containerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) @@ -365,9 +360,9 @@ func initUpdateContractsCmd() { RootCmd.AddCommand(updateContractsCmd) updateContractsCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) updateContractsCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - updateContractsCmd.Flags().String(contractsInitFlag, "", contractsInitFlagDesc) - updateContractsCmd.Flags().String(contractsURLFlag, "", contractsURLFlagDesc) - updateContractsCmd.MarkFlagsMutuallyExclusive(contractsInitFlag, contractsURLFlag) + updateContractsCmd.Flags().String(util.ContractsInitFlag, "", util.ContractsInitFlagDesc) + updateContractsCmd.Flags().String(util.ContractsURLFlag, "", util.ContractsURLFlagDesc) + updateContractsCmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) } func initDumpBalancesCmd() { @@ -384,7 +379,7 @@ func initSetConfigCmd() { setConfig.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) setConfig.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) setConfig.Flags().Bool(forceConfigSet, false, "Force setting not well-known configuration key") - setConfig.Flags().String(localDumpFlag, "", "Path to the blocks dump file") + setConfig.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") } func initDumpNetworkConfigCmd() { @@ -402,7 +397,7 @@ func initSetPolicyCmd() { RootCmd.AddCommand(setPolicy) setPolicy.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) setPolicy.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - setPolicy.Flags().String(localDumpFlag, "", "Path to the blocks dump file") + setPolicy.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") } func initDumpPolicyCmd() { @@ -414,14 +409,14 @@ func initRemoveNodesCmd() { RootCmd.AddCommand(removeNodes) removeNodes.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) removeNodes.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - removeNodes.Flags().String(localDumpFlag, "", "Path to the blocks dump file") + removeNodes.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") } func initForceNewEpochCmd() { RootCmd.AddCommand(forceNewEpoch) forceNewEpoch.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) forceNewEpoch.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - forceNewEpoch.Flags().String(localDumpFlag, "", "Path to the blocks dump file") + forceNewEpoch.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") } func initGenerateStorageCmd() { @@ -437,8 +432,8 @@ func initInitCmd() { RootCmd.AddCommand(initCmd) initCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) initCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - initCmd.Flags().String(contractsInitFlag, "", contractsInitFlagDesc) - initCmd.Flags().String(contractsURLFlag, "", contractsURLFlagDesc) + initCmd.Flags().String(util.ContractsInitFlag, "", util.ContractsInitFlagDesc) + initCmd.Flags().String(util.ContractsURLFlag, "", util.ContractsURLFlagDesc) initCmd.Flags().Uint(epochDurationCLIFlag, 240, "Amount of side chain blocks in one FrostFS epoch") initCmd.Flags().Uint(maxObjectSizeCLIFlag, 67108864, "Max single object size in bytes") initCmd.Flags().Bool(homomorphicHashDisabledCLIFlag, false, "Disable object homomorphic hashing") @@ -446,8 +441,8 @@ func initInitCmd() { initCmd.Flags().Uint64(containerFeeCLIFlag, 1000, "Container registration fee") initCmd.Flags().Uint64(containerAliasFeeCLIFlag, 500, "Container alias fee") initCmd.Flags().String(util.ProtoConfigPath, "", "Path to the consensus node configuration") - initCmd.Flags().String(localDumpFlag, "", "Path to the blocks dump file") - initCmd.MarkFlagsMutuallyExclusive(contractsInitFlag, contractsURLFlag) + initCmd.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") + initCmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) } func initGenerateAlphabetCmd() { diff --git a/cmd/frostfs-adm/internal/modules/morph/update.go b/cmd/frostfs-adm/internal/modules/morph/update.go index b6fcbb5b7..2d5b24712 100644 --- a/cmd/frostfs-adm/internal/modules/morph/update.go +++ b/cmd/frostfs-adm/internal/modules/morph/update.go @@ -3,12 +3,13 @@ package morph import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/spf13/cobra" "github.com/spf13/viper" ) func updateContracts(cmd *cobra.Command, _ []string) error { - wCtx, err := NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := util.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("initialization error: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/util/const.go index bf71b067f..f100ebc61 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/const.go @@ -1,5 +1,7 @@ package util +import "time" + const ( ConsensusAccountName = "consensus" ProtoConfigPath = "protocol" @@ -14,6 +16,13 @@ const ( EndpointFlagShort = "r" AlphabetWalletsFlag = "alphabet-wallets" AlphabetWalletsFlagDesc = "Path to alphabet wallets dir" + LocalDumpFlag = "local-dump" + ContractsInitFlag = "contracts" + ContractsInitFlagDesc = "Path to archive with compiled FrostFS contracts (the default is to fetch the latest release from the official repository)" + ContractsURLFlag = "contracts-url" + ContractsURLFlagDesc = "URL to archive with compiled FrostFS contracts" + EpochDurationInitFlag = "network.epoch_duration" + MaxObjectSizeInitFlag = "network.max_object_size" SingleAccountName = "single" CommitteeAccountName = "committee" @@ -28,4 +37,29 @@ const ( NetmapContract = "netmap" PolicyContract = "policy" ProxyContract = "proxy" + + ContractWalletFilename = "contract.json" + ContractWalletPasswordKey = "contract" + + FrostfsOpsEmail = "ops@frostfs.info" + + DefaultExpirationTime = 10 * 365 * 24 * time.Hour / time.Second +) + +var ( + ContractList = []string{ + BalanceContract, + ContainerContract, + FrostfsIDContract, + NetmapContract, + PolicyContract, + ProxyContract, + } + + FullContractList = append([]string{ + FrostfsContract, + ProcessingContract, + NNSContract, + AlphabetContract, + }, ContractList...) ) diff --git a/cmd/frostfs-adm/internal/modules/morph/download.go b/cmd/frostfs-adm/internal/modules/morph/util/download.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/download.go rename to cmd/frostfs-adm/internal/modules/morph/util/download.go index 5bd2d98bd..ff97a7f4a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/download.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/download.go @@ -1,4 +1,4 @@ -package morph +package util import ( "context" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go index 725181ec7..0e4c22644 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "github.com/nspcc-dev/neo-go/pkg/core/native/nativenames" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/rpcclient" @@ -187,3 +188,23 @@ func NNSIsAvailable(c Client, nnsHash util.Uint160, name string) (bool, error) { return b, nil } } + +func CheckNotaryEnabled(c Client) error { + ns, err := c.GetNativeContracts() + if err != nil { + return fmt.Errorf("can't get native contract hashes: %w", err) + } + + notaryEnabled := false + nativeHashes := make(map[string]util.Uint160, len(ns)) + for i := range ns { + if ns[i].Manifest.Name == nativenames.Notary { + notaryEnabled = true + } + nativeHashes[ns[i].Manifest.Name] = ns[i].Hash + } + if !notaryEnabled { + return errors.New("notary contract must be enabled") + } + return nil +} diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go index be5dcf2db..e7ced2ae6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go @@ -1,14 +1,67 @@ package util import ( + "encoding/hex" "encoding/json" "fmt" + io2 "io" + "os" + "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/context" "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest" "github.com/nspcc-dev/neo-go/pkg/smartcontract/nef" "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/emit" + "github.com/nspcc-dev/neo-go/pkg/vm/opcode" + "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" + "github.com/nspcc-dev/neo-go/pkg/wallet" + "github.com/spf13/cobra" + "github.com/spf13/viper" ) +type ContractState struct { + NEF *nef.File + RawNEF []byte + Manifest *manifest.Manifest + RawManifest []byte + Hash util.Uint160 +} + +type Cache struct { + NNSCs *state.Contract + GroupKey *keys.PublicKey +} + +type InitializeContext struct { + ClientContext + Cache + // CommitteeAcc is used for retrieving the committee address and the verification script. + CommitteeAcc *wallet.Account + // ConsensusAcc is used for retrieving the committee address and the verification script. + ConsensusAcc *wallet.Account + Wallets []*wallet.Wallet + // ContractWallet is a wallet for providing the contract group signature. + ContractWallet *wallet.Wallet + // Accounts contains simple signature accounts in the same order as in Wallets. + Accounts []*wallet.Account + Contracts map[string]*ContractState + Command *cobra.Command + ContractPath string + ContractURL string +} + func (cs *ContractState) Parse() error { nf, err := nef.FileFromBytes(cs.RawNEF) if err != nil { @@ -25,10 +78,466 @@ func (cs *ContractState) Parse() error { return nil } -type ContractState struct { - NEF *nef.File - RawNEF []byte - Manifest *manifest.Manifest - RawManifest []byte - Hash util.Uint160 +func NewInitializeContext(cmd *cobra.Command, v *viper.Viper) (*InitializeContext, error) { + walletDir := config.ResolveHomePath(viper.GetString(AlphabetWalletsFlag)) + wallets, err := GetAlphabetWallets(v, walletDir) + if err != nil { + return nil, err + } + + needContracts := cmd.Name() == "update-contracts" || cmd.Name() == "init" + + var w *wallet.Wallet + w, err = GetWallet(cmd, v, needContracts, walletDir) + if err != nil { + return nil, err + } + + c, err := createClient(cmd, v, wallets) + if err != nil { + return nil, err + } + + committeeAcc, err := GetWalletAccount(wallets[0], CommitteeAccountName) + if err != nil { + return nil, fmt.Errorf("can't find committee account: %w", err) + } + + consensusAcc, err := GetWalletAccount(wallets[0], ConsensusAccountName) + if err != nil { + return nil, fmt.Errorf("can't find consensus account: %w", err) + } + + if err := validateInit(cmd); err != nil { + return nil, err + } + + ctrPath, err := getContractsPath(cmd, needContracts) + if err != nil { + return nil, err + } + + var ctrURL string + if needContracts { + ctrURL, _ = cmd.Flags().GetString(ContractsURLFlag) + } + + if err := CheckNotaryEnabled(c); err != nil { + return nil, err + } + + accounts, err := createWalletAccounts(wallets) + if err != nil { + return nil, err + } + + cliCtx, err := DefaultClientContext(c, committeeAcc) + if err != nil { + return nil, fmt.Errorf("client context: %w", err) + } + + initCtx := &InitializeContext{ + ClientContext: *cliCtx, + ConsensusAcc: consensusAcc, + CommitteeAcc: committeeAcc, + ContractWallet: w, + Wallets: wallets, + Accounts: accounts, + Command: cmd, + Contracts: make(map[string]*ContractState), + ContractPath: ctrPath, + ContractURL: ctrURL, + } + + if needContracts { + err := readContracts(initCtx, FullContractList) + if err != nil { + return nil, err + } + } + + return initCtx, nil +} + +func validateInit(cmd *cobra.Command) error { + if cmd.Name() != "init" { + return nil + } + if viper.GetInt64(EpochDurationInitFlag) <= 0 { + return fmt.Errorf("epoch duration must be positive") + } + + if viper.GetInt64(MaxObjectSizeInitFlag) <= 0 { + return fmt.Errorf("max object size must be positive") + } + + return nil +} + +func createClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) (Client, error) { + var c Client + var err error + if ldf := cmd.Flags().Lookup(LocalDumpFlag); ldf != nil && ldf.Changed { + if cmd.Flags().Changed(EndpointFlag) { + return nil, fmt.Errorf("`%s` and `%s` flags are mutually exclusive", EndpointFlag, LocalDumpFlag) + } + c, err = NewLocalClient(cmd, v, wallets, ldf.Value.String()) + } else { + c, err = GetN3Client(v) + } + if err != nil { + return nil, fmt.Errorf("can't create N3 client: %w", err) + } + return c, nil +} + +func getContractsPath(cmd *cobra.Command, needContracts bool) (string, error) { + if !needContracts { + return "", nil + } + + ctrPath, err := cmd.Flags().GetString(ContractsInitFlag) + if err != nil { + return "", fmt.Errorf("invalid contracts path: %w", err) + } + return ctrPath, nil +} + +func createWalletAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) { + accounts := make([]*wallet.Account, len(wallets)) + for i, w := range wallets { + acc, err := GetWalletAccount(w, SingleAccountName) + if err != nil { + return nil, fmt.Errorf("wallet %s is invalid (no single account): %w", w.Path(), err) + } + accounts[i] = acc + } + return accounts, nil +} + +func readContracts(c *InitializeContext, names []string) error { + var ( + fi os.FileInfo + err error + ) + if c.ContractPath != "" { + fi, err = os.Stat(c.ContractPath) + if err != nil { + return fmt.Errorf("invalid contracts path: %w", err) + } + } + + if c.ContractPath != "" && fi.IsDir() { + for _, ctrName := range names { + cs, err := ReadContract(filepath.Join(c.ContractPath, ctrName), ctrName) + if err != nil { + return err + } + c.Contracts[ctrName] = cs + } + } else { + var r io2.ReadCloser + if c.ContractPath != "" { + r, err = os.Open(c.ContractPath) + } else if c.ContractURL != "" { + r, err = downloadContracts(c.Command, c.ContractURL) + } else { + r, err = downloadContractsFromRepository(c.Command) + } + if err != nil { + return fmt.Errorf("can't open contracts archive: %w", err) + } + defer r.Close() + + m, err := ReadContractsFromArchive(r, names) + if err != nil { + return err + } + for _, name := range names { + if err := m[name].Parse(); err != nil { + return err + } + c.Contracts[name] = m[name] + } + } + + for _, ctrName := range names { + if ctrName != AlphabetContract { + cs := c.Contracts[ctrName] + cs.Hash = state.CreateContractHash(c.CommitteeAcc.Contract.ScriptHash(), + cs.NEF.Checksum, cs.Manifest.Name) + } + } + return nil +} + +func (c *InitializeContext) Close() { + if local, ok := c.Client.(*LocalClient); ok { + err := local.Dump() + if err != nil { + c.Command.PrintErrf("Can't write dump: %v\n", err) + os.Exit(1) + } + } +} + +func (c *InitializeContext) AwaitTx() error { + return c.ClientContext.AwaitTx(c.Command) +} + +func (c *InitializeContext) NNSContractState() (*state.Contract, error) { + if c.NNSCs != nil { + return c.NNSCs, nil + } + + r := management.NewReader(c.ReadOnlyInvoker) + cs, err := r.GetContractByID(1) + if err != nil { + return nil, err + } + + c.NNSCs = cs + return cs, nil +} + +func (c *InitializeContext) GetSigner(tryGroup bool, acc *wallet.Account) transaction.Signer { + if tryGroup && c.GroupKey != nil { + return transaction.Signer{ + Account: acc.Contract.ScriptHash(), + Scopes: transaction.CustomGroups, + AllowedGroups: keys.PublicKeys{c.GroupKey}, + } + } + + signer := transaction.Signer{ + Account: acc.Contract.ScriptHash(), + Scopes: transaction.Global, // Scope is important, as we have nested call to container contract. + } + + if !tryGroup { + return signer + } + + nnsCs, err := c.NNSContractState() + if err != nil { + return signer + } + + groupKey, err := NNSResolveKey(c.ReadOnlyInvoker, nnsCs.Hash, client.NNSGroupKeyName) + if err == nil { + c.GroupKey = groupKey + + signer.Scopes = transaction.CustomGroups + signer.AllowedGroups = keys.PublicKeys{groupKey} + } + return signer +} + +// SendCommitteeTx creates transaction from script, signs it by committee nodes and sends it to RPC. +// If tryGroup is false, global scope is used for the signer (useful when +// working with native contracts). +func (c *InitializeContext) SendCommitteeTx(script []byte, tryGroup bool) error { + return c.sendMultiTx(script, tryGroup, false) +} + +// SendConsensusTx creates transaction from script, signs it by alphabet nodes and sends it to RPC. +// Not that because this is used only after the contracts were initialized and deployed, +// we always try to have a group scope. +func (c *InitializeContext) SendConsensusTx(script []byte) error { + return c.sendMultiTx(script, true, true) +} + +func (c *InitializeContext) sendMultiTx(script []byte, tryGroup bool, withConsensus bool) error { + var act *actor.Actor + var err error + + withConsensus = withConsensus && !c.ConsensusAcc.Contract.ScriptHash().Equals(c.CommitteeAcc.ScriptHash()) + if tryGroup { + // Even for consensus signatures we need the committee to pay. + signers := make([]actor.SignerAccount, 1, 2) + signers[0] = actor.SignerAccount{ + Signer: c.GetSigner(tryGroup, c.CommitteeAcc), + Account: c.CommitteeAcc, + } + if withConsensus { + signers = append(signers, actor.SignerAccount{ + Signer: c.GetSigner(tryGroup, c.ConsensusAcc), + Account: c.ConsensusAcc, + }) + } + act, err = actor.New(c.Client, signers) + } else { + if withConsensus { + panic("BUG: should never happen") + } + act, err = c.CommitteeAct, nil + } + if err != nil { + return fmt.Errorf("could not create actor: %w", err) + } + + tx, err := act.MakeUnsignedRun(script, []transaction.Attribute{{Type: transaction.HighPriority}}) + if err != nil { + return fmt.Errorf("could not perform test invocation: %w", err) + } + + if err := c.MultiSign(tx, CommitteeAccountName); err != nil { + return err + } + if withConsensus { + if err := c.MultiSign(tx, ConsensusAccountName); err != nil { + return err + } + } + + return c.SendTx(tx, c.Command, false) +} + +func (c *InitializeContext) MultiSignAndSend(tx *transaction.Transaction, accType string) error { + if err := c.MultiSign(tx, accType); err != nil { + return err + } + + return c.SendTx(tx, c.Command, false) +} + +func (c *InitializeContext) MultiSign(tx *transaction.Transaction, accType string) error { + version, err := c.Client.GetVersion() + if err != nil { + // error appears only if client + // has not been initialized + panic(err) + } + network := version.Protocol.Network + + // Use parameter context to avoid dealing with signature order. + pc := context.NewParameterContext("", network, tx) + h := c.CommitteeAcc.Contract.ScriptHash() + if accType == ConsensusAccountName { + h = c.ConsensusAcc.Contract.ScriptHash() + } + for _, w := range c.Wallets { + acc, err := GetWalletAccount(w, accType) + if err != nil { + return fmt.Errorf("can't find %s wallet account: %w", accType, err) + } + + priv := acc.PrivateKey() + sign := priv.SignHashable(uint32(network), tx) + if err := pc.AddSignature(h, acc.Contract, priv.PublicKey(), sign); err != nil { + return fmt.Errorf("can't add signature: %w", err) + } + if len(pc.Items[h].Signatures) == len(acc.Contract.Parameters) { + break + } + } + + w, err := pc.GetWitness(h) + if err != nil { + return fmt.Errorf("incomplete signature: %w", err) + } + + for i := range tx.Signers { + if tx.Signers[i].Account == h { + if i < len(tx.Scripts) { + tx.Scripts[i] = *w + } else if i == len(tx.Scripts) { + tx.Scripts = append(tx.Scripts, *w) + } else { + panic("BUG: invalid signing order") + } + return nil + } + } + + return fmt.Errorf("%s account was not found among transaction signers", accType) +} + +// EmitUpdateNNSGroupScript emits script for updating group key stored in NNS. +// First return value is true iff the key is already there and nothing should be done. +// Second return value is true iff a domain registration code was emitted. +func (c *InitializeContext) EmitUpdateNNSGroupScript(bw *io.BufBinWriter, nnsHash util.Uint160, pub *keys.PublicKey) (bool, bool, error) { + isAvail, err := NNSIsAvailable(c.Client, nnsHash, client.NNSGroupKeyName) + if err != nil { + return false, false, err + } + + if !isAvail { + currentPub, err := NNSResolveKey(c.ReadOnlyInvoker, nnsHash, client.NNSGroupKeyName) + if err != nil { + return false, false, err + } + + if pub.Equal(currentPub) { + return true, false, nil + } + } + + if isAvail { + emit.AppCall(bw.BinWriter, nnsHash, "register", callflag.All, + client.NNSGroupKeyName, c.CommitteeAcc.Contract.ScriptHash(), + FrostfsOpsEmail, int64(3600), int64(600), int64(DefaultExpirationTime), int64(3600)) + emit.Opcodes(bw.BinWriter, opcode.ASSERT) + } + + emit.AppCall(bw.BinWriter, nnsHash, "deleteRecords", callflag.All, "group.frostfs", int64(nns.TXT)) + emit.AppCall(bw.BinWriter, nnsHash, "addRecord", callflag.All, + "group.frostfs", int64(nns.TXT), hex.EncodeToString(pub.Bytes())) + + return false, isAvail, nil +} + +func (c *InitializeContext) NNSRegisterDomainScript(nnsHash, expectedHash util.Uint160, domain string) ([]byte, bool, error) { + ok, err := NNSIsAvailable(c.Client, nnsHash, domain) + if err != nil { + return nil, false, err + } + + if ok { + bw := io.NewBufBinWriter() + emit.AppCall(bw.BinWriter, nnsHash, "register", callflag.All, + domain, c.CommitteeAcc.Contract.ScriptHash(), + FrostfsOpsEmail, int64(3600), int64(600), int64(DefaultExpirationTime), int64(3600)) + emit.Opcodes(bw.BinWriter, opcode.ASSERT) + + if bw.Err != nil { + panic(bw.Err) + } + return bw.Bytes(), false, nil + } + + s, err := NNSResolveHash(c.ReadOnlyInvoker, nnsHash, domain) + if err != nil { + return nil, false, err + } + return nil, s == expectedHash, nil +} + +func (c *InitializeContext) NNSRootRegistered(nnsHash util.Uint160, zone string) (bool, error) { + res, err := c.CommitteeAct.Call(nnsHash, "isAvailable", "name."+zone) + if err != nil { + return false, err + } + + return res.State == vmstate.Halt.String(), nil +} + +func (c *InitializeContext) IsUpdated(ctrHash util.Uint160, cs *ContractState) bool { + r := management.NewReader(c.ReadOnlyInvoker) + realCs, err := r.GetContract(ctrHash) + return err == nil && realCs != nil && realCs.NEF.Checksum == cs.NEF.Checksum +} + +func (c *InitializeContext) GetContract(ctrName string) *ContractState { + return c.Contracts[ctrName] +} + +func (c *InitializeContext) GetAlphabetDeployItems(i, n int) []any { + items := make([]any, 5) + items[0] = c.Contracts[NetmapContract].Hash + items[1] = c.Contracts[ProxyContract].Hash + items[2] = innerring.GlagoliticLetter(i).String() + items[3] = int64(i) + items[4] = int64(n) + return items } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/util.go b/cmd/frostfs-adm/internal/modules/morph/util/util.go index bbe635a40..abd72b04d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/util.go @@ -1,10 +1,14 @@ package util import ( + "archive/tar" + "compress/gzip" "errors" "fmt" + "io" "os" "path/filepath" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" @@ -84,3 +88,72 @@ func NewActor(c actor.RPCActor, committeeAcc *wallet.Account) (*actor.Actor, err Account: committeeAcc, }}) } + +func ReadContract(ctrPath, ctrName string) (*ContractState, error) { + rawNef, err := os.ReadFile(filepath.Join(ctrPath, ctrName+"_contract.nef")) + if err != nil { + return nil, fmt.Errorf("can't read NEF file for %s contract: %w", ctrName, err) + } + rawManif, err := os.ReadFile(filepath.Join(ctrPath, "config.json")) + if err != nil { + return nil, fmt.Errorf("can't read manifest file for %s contract: %w", ctrName, err) + } + + cs := &ContractState{ + RawNEF: rawNef, + RawManifest: rawManif, + } + + return cs, cs.Parse() +} + +func ReadContractsFromArchive(file io.Reader, names []string) (map[string]*ContractState, error) { + m := make(map[string]*ContractState, len(names)) + for i := range names { + m[names[i]] = new(ContractState) + } + + gr, err := gzip.NewReader(file) + if err != nil { + return nil, fmt.Errorf("contracts file must be tar.gz archive: %w", err) + } + + r := tar.NewReader(gr) + for h, err := r.Next(); ; h, err = r.Next() { + if err != nil { + break + } + + dir, _ := filepath.Split(h.Name) + ctrName := filepath.Base(dir) + + cs, ok := m[ctrName] + if !ok { + continue + } + + switch { + case strings.HasSuffix(h.Name, filepath.Join(ctrName, ctrName+"_contract.nef")): + cs.RawNEF, err = io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("can't read NEF file for %s contract: %w", ctrName, err) + } + case strings.HasSuffix(h.Name, "config.json"): + cs.RawManifest, err = io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("can't read manifest file for %s contract: %w", ctrName, err) + } + } + m[ctrName] = cs + } + + for ctrName, cs := range m { + if cs.RawNEF == nil { + return nil, fmt.Errorf("NEF for %s contract wasn't found", ctrName) + } + if cs.RawManifest == nil { + return nil, fmt.Errorf("manifest for %s contract wasn't found", ctrName) + } + } + return m, nil +} diff --git a/cmd/frostfs-adm/internal/modules/morph/util/wallet.go b/cmd/frostfs-adm/internal/modules/morph/util/wallet.go new file mode 100644 index 000000000..2110adb10 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/util/wallet.go @@ -0,0 +1,75 @@ +package util + +import ( + "fmt" + "os" + "path/filepath" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/wallet" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func InitializeContractWallet(v *viper.Viper, walletDir string) (*wallet.Wallet, error) { + password, err := config.GetPassword(v, ContractWalletPasswordKey) + if err != nil { + return nil, err + } + + w, err := wallet.NewWallet(filepath.Join(walletDir, ContractWalletFilename)) + if err != nil { + return nil, err + } + + acc, err := wallet.NewAccount() + if err != nil { + return nil, err + } + + err = acc.Encrypt(password, keys.NEP2ScryptParams()) + if err != nil { + return nil, err + } + + w.AddAccount(acc) + if err := w.SavePretty(); err != nil { + return nil, err + } + + return w, nil +} + +func OpenContractWallet(v *viper.Viper, cmd *cobra.Command, walletDir string) (*wallet.Wallet, error) { + p := filepath.Join(walletDir, ContractWalletFilename) + w, err := wallet.NewWalletFromFile(p) + if err != nil { + if !os.IsNotExist(err) { + return nil, fmt.Errorf("can't open wallet: %w", err) + } + + cmd.Printf("Contract group wallet is missing, initialize at %s\n", p) + return InitializeContractWallet(v, walletDir) + } + + password, err := config.GetPassword(v, ContractWalletPasswordKey) + if err != nil { + return nil, err + } + + for i := range w.Accounts { + if err := w.Accounts[i].Decrypt(password, keys.NEP2ScryptParams()); err != nil { + return nil, fmt.Errorf("can't unlock wallet: %w", err) + } + } + + return w, nil +} + +func GetWallet(cmd *cobra.Command, v *viper.Viper, needContracts bool, walletDir string) (*wallet.Wallet, error) { + if !needContracts { + return nil, nil + } + return OpenContractWallet(v, cmd, walletDir) +} From f6ff3de0ae5ddf53b77a7d244ccdb0ca7ed432cf Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 15:55:33 +0300 Subject: [PATCH 0887/1943] [#932] adm: Move `set-/dump-policy` to `policy` package Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/initialize_test.go | 5 +- .../modules/morph/{ => policy}/policy.go | 4 +- .../internal/modules/morph/policy/root.go | 47 +++++++++++++++++++ .../internal/modules/morph/root.go | 40 ++-------------- 4 files changed, 55 insertions(+), 41 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => policy}/policy.go (97%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/policy/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index 63e9887c0..e9571e50c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/config" @@ -78,8 +79,8 @@ func testInitialize(t *testing.T, committeeSize int) { require.NoError(t, setConfigCmd(setConfig, []string{"MaintenanceModeAllowed=true"})) }) t.Run("set-policy", func(t *testing.T) { - require.NoError(t, setPolicy.Flags().Set(util.LocalDumpFlag, dumpPath)) - require.NoError(t, setPolicyCmd(setPolicy, []string{"ExecFeeFactor=1"})) + require.NoError(t, policy.Set.Flags().Set(util.LocalDumpFlag, dumpPath)) + require.NoError(t, policy.SetPolicyCmd(policy.Set, []string{"ExecFeeFactor=1"})) }) t.Run("remove-node", func(t *testing.T) { pk, err := keys.NewPrivateKey() diff --git a/cmd/frostfs-adm/internal/modules/morph/policy.go b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go similarity index 97% rename from cmd/frostfs-adm/internal/modules/morph/policy.go rename to cmd/frostfs-adm/internal/modules/morph/policy/policy.go index d01c5e1a5..7f4a8d326 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go @@ -1,4 +1,4 @@ -package morph +package policy import ( "bytes" @@ -24,7 +24,7 @@ const ( setFeeParam = "FeePerByte" ) -func setPolicyCmd(cmd *cobra.Command, args []string) error { +func SetPolicyCmd(cmd *cobra.Command, args []string) error { wCtx, err := util.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) diff --git a/cmd/frostfs-adm/internal/modules/morph/policy/root.go b/cmd/frostfs-adm/internal/modules/morph/policy/root.go new file mode 100644 index 000000000..99ff56ee8 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/policy/root.go @@ -0,0 +1,47 @@ +package policy + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + Set = &cobra.Command{ + Use: "set-policy [ExecFeeFactor=] [StoragePrice=] [FeePerByte=]", + DisableFlagsInUseLine: true, + Short: "Set global policy values", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: SetPolicyCmd, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return []string{"ExecFeeFactor=", "StoragePrice=", "FeePerByte="}, cobra.ShellCompDirectiveNoSpace + }, + } + + Dump = &cobra.Command{ + Use: "dump-policy", + Short: "Dump FrostFS policy", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: dumpPolicyCmd, + } +) + +func initSetPolicyCmd() { + Set.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) + Set.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + Set.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") +} + +func initDumpPolicyCmd() { + Dump.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) +} + +func init() { + initSetPolicyCmd() + initDumpPolicyCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 7585f5845..b53a64486 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -2,6 +2,7 @@ package morph import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/ape" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -130,29 +131,6 @@ var ( RunE: setConfigCmd, } - setPolicy = &cobra.Command{ - Use: "set-policy [ExecFeeFactor=] [StoragePrice=] [FeePerByte=]", - DisableFlagsInUseLine: true, - Short: "Set global policy values", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: setPolicyCmd, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return []string{"ExecFeeFactor=", "StoragePrice=", "FeePerByte="}, cobra.ShellCompDirectiveNoSpace - }, - } - - dumpPolicy = &cobra.Command{ - Use: "dump-policy", - Short: "Dump FrostFS policy", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: dumpPolicyCmd, - } - dumpContractHashesCmd = &cobra.Command{ Use: "dump-hashes", Short: "Dump deployed contract hashes", @@ -265,8 +243,8 @@ func init() { initGenerateStorageCmd() initForceNewEpochCmd() initRemoveNodesCmd() - initSetPolicyCmd() - initDumpPolicyCmd() + RootCmd.AddCommand(policy.Set) + RootCmd.AddCommand(policy.Dump) initDumpContractHashesCmd() initDumpNetworkConfigCmd() initSetConfigCmd() @@ -393,18 +371,6 @@ func initDumpContractHashesCmd() { dumpContractHashesCmd.Flags().String(customZoneFlag, "", "Custom zone to search.") } -func initSetPolicyCmd() { - RootCmd.AddCommand(setPolicy) - setPolicy.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - setPolicy.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - setPolicy.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") -} - -func initDumpPolicyCmd() { - RootCmd.AddCommand(dumpPolicy) - dumpPolicy.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) -} - func initRemoveNodesCmd() { RootCmd.AddCommand(removeNodes) removeNodes.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) From a92188e5f9bef9c426c5d09826afd4ee1bcc01c2 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 16:00:45 +0300 Subject: [PATCH 0888/1943] [#932] adm: Reduce methods visibility in `util` package Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/util/initialize_ctx.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/util/util.go | 6 +++--- cmd/frostfs-adm/internal/modules/morph/util/wallet.go | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go index e7ced2ae6..4305df1e5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go @@ -88,7 +88,7 @@ func NewInitializeContext(cmd *cobra.Command, v *viper.Viper) (*InitializeContex needContracts := cmd.Name() == "update-contracts" || cmd.Name() == "init" var w *wallet.Wallet - w, err = GetWallet(cmd, v, needContracts, walletDir) + w, err = getWallet(cmd, v, needContracts, walletDir) if err != nil { return nil, err } @@ -249,7 +249,7 @@ func readContracts(c *InitializeContext, names []string) error { } defer r.Close() - m, err := ReadContractsFromArchive(r, names) + m, err := readContractsFromArchive(r, names) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/util.go b/cmd/frostfs-adm/internal/modules/morph/util/util.go index abd72b04d..4a963d3d8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/util.go @@ -20,7 +20,7 @@ import ( ) func GetAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, error) { - wallets, err := OpenAlphabetWallets(v, walletDir) + wallets, err := openAlphabetWallets(v, walletDir) if err != nil { return nil, err } @@ -31,7 +31,7 @@ func GetAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, err return wallets, nil } -func OpenAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, error) { +func openAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, error) { walletFiles, err := os.ReadDir(walletDir) if err != nil { return nil, fmt.Errorf("can't read alphabet wallets dir: %w", err) @@ -107,7 +107,7 @@ func ReadContract(ctrPath, ctrName string) (*ContractState, error) { return cs, cs.Parse() } -func ReadContractsFromArchive(file io.Reader, names []string) (map[string]*ContractState, error) { +func readContractsFromArchive(file io.Reader, names []string) (map[string]*ContractState, error) { m := make(map[string]*ContractState, len(names)) for i := range names { m[names[i]] = new(ContractState) diff --git a/cmd/frostfs-adm/internal/modules/morph/util/wallet.go b/cmd/frostfs-adm/internal/modules/morph/util/wallet.go index 2110adb10..e0e553d06 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/wallet.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/wallet.go @@ -41,7 +41,7 @@ func InitializeContractWallet(v *viper.Viper, walletDir string) (*wallet.Wallet, return w, nil } -func OpenContractWallet(v *viper.Viper, cmd *cobra.Command, walletDir string) (*wallet.Wallet, error) { +func openContractWallet(v *viper.Viper, cmd *cobra.Command, walletDir string) (*wallet.Wallet, error) { p := filepath.Join(walletDir, ContractWalletFilename) w, err := wallet.NewWalletFromFile(p) if err != nil { @@ -67,9 +67,9 @@ func OpenContractWallet(v *viper.Viper, cmd *cobra.Command, walletDir string) (* return w, nil } -func GetWallet(cmd *cobra.Command, v *viper.Viper, needContracts bool, walletDir string) (*wallet.Wallet, error) { +func getWallet(cmd *cobra.Command, v *viper.Viper, needContracts bool, walletDir string) (*wallet.Wallet, error) { if !needContracts { return nil, nil } - return OpenContractWallet(v, cmd, walletDir) + return openContractWallet(v, cmd, walletDir) } From 218bd72f9abbab228f72896b4fb8cbedf944a709 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 16:11:17 +0300 Subject: [PATCH 0889/1943] [#932] adm: Move command `frostfsid` to package `frostfsid` Signed-off-by: Anton Nikiforov --- .../morph/{ => frostfsid}/frostfsid.go | 31 ++++++++++--------- .../morph/{ => frostfsid}/frostfsid_util.go | 4 +-- .../{ => frostfsid}/frostfsid_util_test.go | 8 ++--- .../internal/modules/morph/frostfsid/root.go | 15 +++++++++ .../modules/morph/initialize_deploy.go | 5 ++- .../internal/modules/morph/root.go | 14 ++------- 6 files changed, 41 insertions(+), 36 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => frostfsid}/frostfsid.go (96%) rename cmd/frostfs-adm/internal/modules/morph/{ => frostfsid}/frostfsid_util.go (96%) rename cmd/frostfs-adm/internal/modules/morph/{ => frostfsid}/frostfsid_util_test.go (96%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go similarity index 96% rename from cmd/frostfs-adm/internal/modules/morph/frostfsid.go rename to cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 3e39c3019..76bbc7148 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -1,4 +1,4 @@ -package morph +package frostfsid import ( "errors" @@ -27,12 +27,13 @@ const ( includeNamesFlag = "include-names" groupNameFlag = "group-name" groupIDFlag = "group-id" + + frostfsIDAdminConfigKey = "frostfsid.admin" + rootNamespacePlaceholder = "" ) -const rootNamespacePlaceholder = "" - var ( - frostfsidCmd = &cobra.Command{ + Cmd = &cobra.Command{ Use: "frostfsid", Short: "Section for frostfsid interactions commands", } @@ -149,20 +150,20 @@ var ( ) func initFrostfsIDCreateNamespaceCmd() { - frostfsidCmd.AddCommand(frostfsidCreateNamespaceCmd) + Cmd.AddCommand(frostfsidCreateNamespaceCmd) frostfsidCreateNamespaceCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidCreateNamespaceCmd.Flags().String(namespaceFlag, "", "Namespace name to create") frostfsidCreateNamespaceCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDListNamespacesCmd() { - frostfsidCmd.AddCommand(frostfsidListNamespacesCmd) + Cmd.AddCommand(frostfsidListNamespacesCmd) frostfsidListNamespacesCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidListNamespacesCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDCreateSubjectCmd() { - frostfsidCmd.AddCommand(frostfsidCreateSubjectCmd) + Cmd.AddCommand(frostfsidCreateSubjectCmd) frostfsidCreateSubjectCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidCreateSubjectCmd.Flags().String(namespaceFlag, "", "Namespace where create subject") frostfsidCreateSubjectCmd.Flags().String(subjectNameFlag, "", "Subject name, must be unique in namespace") @@ -171,14 +172,14 @@ func initFrostfsIDCreateSubjectCmd() { } func initFrostfsIDDeleteSubjectCmd() { - frostfsidCmd.AddCommand(frostfsidDeleteSubjectCmd) + Cmd.AddCommand(frostfsidDeleteSubjectCmd) frostfsidDeleteSubjectCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidDeleteSubjectCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidDeleteSubjectCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDListSubjectsCmd() { - frostfsidCmd.AddCommand(frostfsidListSubjectsCmd) + Cmd.AddCommand(frostfsidListSubjectsCmd) frostfsidListSubjectsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidListSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace to list subjects") frostfsidListSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") @@ -186,7 +187,7 @@ func initFrostfsIDListSubjectsCmd() { } func initFrostfsIDCreateGroupCmd() { - frostfsidCmd.AddCommand(frostfsidCreateGroupCmd) + Cmd.AddCommand(frostfsidCreateGroupCmd) frostfsidCreateGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidCreateGroupCmd.Flags().String(namespaceFlag, "", "Namespace where create group") frostfsidCreateGroupCmd.Flags().String(groupNameFlag, "", "Group name, must be unique in namespace") @@ -194,7 +195,7 @@ func initFrostfsIDCreateGroupCmd() { } func initFrostfsIDDeleteGroupCmd() { - frostfsidCmd.AddCommand(frostfsidDeleteGroupCmd) + Cmd.AddCommand(frostfsidDeleteGroupCmd) frostfsidDeleteGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidDeleteGroupCmd.Flags().String(namespaceFlag, "", "Namespace to delete group") frostfsidDeleteGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") @@ -202,14 +203,14 @@ func initFrostfsIDDeleteGroupCmd() { } func initFrostfsIDListGroupsCmd() { - frostfsidCmd.AddCommand(frostfsidListGroupsCmd) + Cmd.AddCommand(frostfsidListGroupsCmd) frostfsidListGroupsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidListGroupsCmd.Flags().String(namespaceFlag, "", "Namespace to list groups") frostfsidListGroupsCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) } func initFrostfsIDAddSubjectToGroupCmd() { - frostfsidCmd.AddCommand(frostfsidAddSubjectToGroupCmd) + Cmd.AddCommand(frostfsidAddSubjectToGroupCmd) frostfsidAddSubjectToGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidAddSubjectToGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidAddSubjectToGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") @@ -217,7 +218,7 @@ func initFrostfsIDAddSubjectToGroupCmd() { } func initFrostfsIDRemoveSubjectFromGroupCmd() { - frostfsidCmd.AddCommand(frostfsidRemoveSubjectFromGroupCmd) + Cmd.AddCommand(frostfsidRemoveSubjectFromGroupCmd) frostfsidRemoveSubjectFromGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidRemoveSubjectFromGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidRemoveSubjectFromGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") @@ -225,7 +226,7 @@ func initFrostfsIDRemoveSubjectFromGroupCmd() { } func initFrostfsIDListGroupSubjectsCmd() { - frostfsidCmd.AddCommand(frostfsidListGroupSubjectsCmd) + Cmd.AddCommand(frostfsidListGroupSubjectsCmd) frostfsidListGroupSubjectsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) frostfsidListGroupSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace name") frostfsidListGroupSubjectsCmd.Flags().Int64(groupIDFlag, 0, "Group id") diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util.go similarity index 96% rename from cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go rename to cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util.go index 6b6bdffa9..99c04f57d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util.go @@ -1,4 +1,4 @@ -package morph +package frostfsid import ( "errors" @@ -13,7 +13,7 @@ import ( "github.com/spf13/viper" ) -func getFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) { +func GetFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) { admin := v.GetString(frostfsIDAdminConfigKey) if admin == "" { return util.Uint160{}, false, nil diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go similarity index 96% rename from cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go rename to cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go index 79a0845f9..119e10d15 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid_util_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go @@ -1,4 +1,4 @@ -package morph +package frostfsid import ( "encoding/hex" @@ -30,7 +30,7 @@ func TestFrostfsIDConfig(t *testing.T) { v := viper.New() v.Set("frostfsid.admin", fmts[i]) - actual, found, err := getFrostfsIDAdmin(v) + actual, found, err := GetFrostfsIDAdmin(v) require.NoError(t, err) require.True(t, found) require.Equal(t, pks[i].GetScriptHash(), actual) @@ -40,14 +40,14 @@ func TestFrostfsIDConfig(t *testing.T) { v := viper.New() v.Set("frostfsid.admin", "abc") - _, found, err := getFrostfsIDAdmin(v) + _, found, err := GetFrostfsIDAdmin(v) require.Error(t, err) require.True(t, found) }) t.Run("missing key", func(t *testing.T) { v := viper.New() - _, found, err := getFrostfsIDAdmin(v) + _, found, err := GetFrostfsIDAdmin(v) require.NoError(t, err) require.False(t, found) }) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go new file mode 100644 index 000000000..850474794 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go @@ -0,0 +1,15 @@ +package frostfsid + +func init() { + initFrostfsIDCreateNamespaceCmd() + initFrostfsIDListNamespacesCmd() + initFrostfsIDCreateSubjectCmd() + initFrostfsIDDeleteSubjectCmd() + initFrostfsIDListSubjectsCmd() + initFrostfsIDCreateGroupCmd() + initFrostfsIDDeleteGroupCmd() + initFrostfsIDListGroupsCmd() + initFrostfsIDAddSubjectToGroupCmd() + initFrostfsIDRemoveSubjectFromGroupCmd() + initFrostfsIDListGroupSubjectsCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 322033abc..5cd7b3ba0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/common" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -27,8 +28,6 @@ import ( "github.com/spf13/viper" ) -const frostfsIDAdminConfigKey = "frostfsid.admin" - var netmapConfigKeys = []string{ netmap.EpochDurationConfig, netmap.MaxObjectSizeConfig, @@ -361,7 +360,7 @@ func getContractDeployData(c *morphUtil.InitializeContext, ctrName string, keysP h, found, err = getFrostfsIDAdminFromContract(c.ReadOnlyInvoker) } if method != updateMethodName || err == nil && !found { - h, found, err = getFrostfsIDAdmin(viper.GetViper()) + h, found, err = frostfsid.GetFrostfsIDAdmin(viper.GetViper()) } if err != nil { return nil, err diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index b53a64486..0d6f48d98 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -2,6 +2,7 @@ package morph import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/ape" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/spf13/cobra" @@ -262,18 +263,7 @@ func init() { initProxyAddAccount() initProxyRemoveAccount() - RootCmd.AddCommand(frostfsidCmd) - initFrostfsIDCreateNamespaceCmd() - initFrostfsIDListNamespacesCmd() - initFrostfsIDCreateSubjectCmd() - initFrostfsIDDeleteSubjectCmd() - initFrostfsIDListSubjectsCmd() - initFrostfsIDCreateGroupCmd() - initFrostfsIDDeleteGroupCmd() - initFrostfsIDListGroupsCmd() - initFrostfsIDAddSubjectToGroupCmd() - initFrostfsIDRemoveSubjectFromGroupCmd() - initFrostfsIDListGroupSubjectsCmd() + RootCmd.AddCommand(frostfsid.Cmd) } func initProxyAddAccount() { From 8148c9dc19e8b7fd29d4de916ec66b1772a025ac Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 16:31:46 +0300 Subject: [PATCH 0890/1943] [#932] adm: Move command `morph proxy` to package `proxy` Signed-off-by: Anton Nikiforov --- .../modules/morph/{ => proxy}/proxy.go | 2 +- .../internal/modules/morph/proxy/root.go | 43 +++++++++++++++++++ .../internal/modules/morph/root.go | 38 ++-------------- 3 files changed, 47 insertions(+), 36 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => proxy}/proxy.go (99%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/proxy/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/proxy.go rename to cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go index aa98fb70d..7455c0bd0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go @@ -1,4 +1,4 @@ -package morph +package proxy import ( "fmt" diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy/root.go b/cmd/frostfs-adm/internal/modules/morph/proxy/root.go new file mode 100644 index 000000000..14820d5ab --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/root.go @@ -0,0 +1,43 @@ +package proxy + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + AddAccountCmd = &cobra.Command{ + Use: "proxy-add-account", + Short: "Adds account to proxy contract", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + Run: addProxyAccount, + } + RemoveAccountCmd = &cobra.Command{ + Use: "proxy-remove-account", + Short: "Remove from proxy contract", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + Run: removeProxyAccount, + } +) + +func initProxyAddAccount() { + AddAccountCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + AddAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") +} + +func initProxyRemoveAccount() { + RemoveAccountCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + RemoveAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") +} + +func init() { + initProxyAddAccount() + initProxyRemoveAccount() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 0d6f48d98..7eb41ddc2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -4,6 +4,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/proxy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -215,26 +216,6 @@ var ( }, Run: listNetmapCandidatesNodes, } - - proxyAddAccountCmd = &cobra.Command{ - Use: "proxy-add-account", - Short: "Adds account to proxy contract", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - Run: addProxyAccount, - } - - proxyRemoveAccountCmd = &cobra.Command{ - Use: "proxy-remove-account", - Short: "Remove from proxy contract", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - Run: removeProxyAccount, - } ) func init() { @@ -259,25 +240,12 @@ func init() { initNetmapCandidatesCmd() RootCmd.AddCommand(ape.Cmd) - - initProxyAddAccount() - initProxyRemoveAccount() + RootCmd.AddCommand(proxy.AddAccountCmd) + RootCmd.AddCommand(proxy.RemoveAccountCmd) RootCmd.AddCommand(frostfsid.Cmd) } -func initProxyAddAccount() { - RootCmd.AddCommand(proxyAddAccountCmd) - proxyAddAccountCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - proxyAddAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") -} - -func initProxyRemoveAccount() { - RootCmd.AddCommand(proxyRemoveAccountCmd) - proxyRemoveAccountCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - proxyRemoveAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") -} - func initNetmapCandidatesCmd() { RootCmd.AddCommand(netmapCandidatesCmd) netmapCandidatesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) From beb9d80e348330855c5a943645b666f70004d690 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 16:38:20 +0300 Subject: [PATCH 0891/1943] [#932] adm: Move command `dump-balances` to package `balance` Signed-off-by: Anton Nikiforov --- .../modules/morph/{ => balance}/balance.go | 4 +-- .../internal/modules/morph/balance/root.go | 28 +++++++++++++++++++ .../internal/modules/morph/dump_hashes.go | 4 +-- .../modules/morph/initialize_deploy.go | 2 +- .../internal/modules/morph/initialize_nns.go | 7 +---- .../internal/modules/morph/root.go | 21 ++------------ .../internal/modules/morph/util/util.go | 5 ++++ 7 files changed, 41 insertions(+), 30 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => balance}/balance.go (99%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/balance/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/balance.go rename to cmd/frostfs-adm/internal/modules/morph/balance/balance.go index 451d57954..9d16ada46 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go @@ -1,4 +1,4 @@ -package morph +package balance import ( "crypto/elliptic" @@ -157,7 +157,7 @@ func printAlphabetContractBalances(cmd *cobra.Command, c morphUtil.Client, inv * w := io.NewBufBinWriter() for i := range alphaList { emit.AppCall(w.BinWriter, nnsHash, "resolve", callflag.ReadOnly, - getAlphabetNNSDomain(i), + morphUtil.GetAlphabetNNSDomain(i), int64(nns.TXT)) } if w.Err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/balance/root.go b/cmd/frostfs-adm/internal/modules/morph/balance/root.go new file mode 100644 index 000000000..757637e5d --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/balance/root.go @@ -0,0 +1,28 @@ +package balance + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var DumpCmd = &cobra.Command{ + Use: "dump-balances", + Short: "Dump GAS balances", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: dumpBalances, +} + +func initDumpBalancesCmd() { + DumpCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + DumpCmd.Flags().BoolP(dumpBalancesStorageFlag, "s", false, "Dump balances of storage nodes from the current netmap") + DumpCmd.Flags().BoolP(dumpBalancesAlphabetFlag, "a", false, "Dump balances of alphabet contracts") + DumpCmd.Flags().BoolP(dumpBalancesProxyFlag, "p", false, "Dump balances of the proxy contract") + DumpCmd.Flags().Bool(dumpBalancesUseScriptHashFlag, false, "Use script-hash format for addresses") +} + +func init() { + initDumpBalancesCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go index d33311a8b..c92c25690 100644 --- a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go @@ -54,7 +54,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { irSize := 0 for ; irSize < lastGlagoliticLetter; irSize++ { - ok, err := morphUtil.NNSIsAvailable(c, cs.Hash, getAlphabetNNSDomain(irSize)) + ok, err := morphUtil.NNSIsAvailable(c, cs.Hash, morphUtil.GetAlphabetNNSDomain(irSize)) if err != nil { return err } else if ok { @@ -68,7 +68,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { bw.Reset() for i := 0; i < irSize; i++ { emit.AppCall(bw.BinWriter, cs.Hash, "resolve", callflag.ReadOnly, - getAlphabetNNSDomain(i), + morphUtil.GetAlphabetNNSDomain(i), int64(nns.TXT)) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 5cd7b3ba0..50427c93f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -215,7 +215,7 @@ func deployAlphabetAccounts(c *morphUtil.InitializeContext, nnsHash util.Uint160 // alphabet contracts should be deployed by individual nodes to get different hashes. for i, acc := range c.Accounts { - ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, getAlphabetNNSDomain(i)) + ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, morphUtil.GetAlphabetNNSDomain(i)) if err != nil { return nil, fmt.Errorf("can't resolve hash for contract update: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go index 8e5223098..53435e67b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go @@ -4,7 +4,6 @@ import ( "encoding/hex" "errors" "fmt" - "strconv" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" @@ -48,7 +47,7 @@ func setNNS(c *morphUtil.InitializeContext) error { for i, acc := range c.Accounts { alphaCs.Hash = state.CreateContractHash(acc.Contract.ScriptHash(), alphaCs.NEF.Checksum, alphaCs.Manifest.Name) - domain := getAlphabetNNSDomain(i) + domain := morphUtil.GetAlphabetNNSDomain(i) if err := nnsRegisterDomain(c, nnsCs.Hash, alphaCs.Hash, domain); err != nil { return err } @@ -93,10 +92,6 @@ func updateNNSGroup(c *morphUtil.InitializeContext, nnsHash util.Uint160, pub *k return c.SendCommitteeTx(script, true) } -func getAlphabetNNSDomain(i int) string { - return morphUtil.AlphabetContract + strconv.FormatUint(uint64(i), 10) + ".frostfs" -} - // wrapRegisterScriptWithPrice wraps a given script with `getPrice`/`setPrice` calls for NNS. // It is intended to be used for a single transaction, and not as a part of other scripts. // It is assumed that script already contains static slot initialization code, the first one diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 7eb41ddc2..ac757428a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -2,6 +2,7 @@ package morph import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/ape" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/proxy" @@ -151,15 +152,6 @@ var ( RunE: dumpNetworkConfig, } - dumpBalancesCmd = &cobra.Command{ - Use: "dump-balances", - Short: "Dump GAS balances", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: dumpBalances, - } - updateContractsCmd = &cobra.Command{ Use: "update-contracts", Short: "Update FrostFS contracts", @@ -230,7 +222,7 @@ func init() { initDumpContractHashesCmd() initDumpNetworkConfigCmd() initSetConfigCmd() - initDumpBalancesCmd() + RootCmd.AddCommand(balance.DumpCmd) initUpdateContractsCmd() initDumpContainersCmd() initRestoreContainersCmd() @@ -301,15 +293,6 @@ func initUpdateContractsCmd() { updateContractsCmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) } -func initDumpBalancesCmd() { - RootCmd.AddCommand(dumpBalancesCmd) - dumpBalancesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - dumpBalancesCmd.Flags().BoolP(dumpBalancesStorageFlag, "s", false, "Dump balances of storage nodes from the current netmap") - dumpBalancesCmd.Flags().BoolP(dumpBalancesAlphabetFlag, "a", false, "Dump balances of alphabet contracts") - dumpBalancesCmd.Flags().BoolP(dumpBalancesProxyFlag, "p", false, "Dump balances of the proxy contract") - dumpBalancesCmd.Flags().Bool(dumpBalancesUseScriptHashFlag, false, "Use script-hash format for addresses") -} - func initSetConfigCmd() { RootCmd.AddCommand(setConfig) setConfig.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) diff --git a/cmd/frostfs-adm/internal/modules/morph/util/util.go b/cmd/frostfs-adm/internal/modules/morph/util/util.go index 4a963d3d8..a6f94049b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/util.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "strconv" "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" @@ -157,3 +158,7 @@ func readContractsFromArchive(file io.Reader, names []string) (map[string]*Contr } return m, nil } + +func GetAlphabetNNSDomain(i int) string { + return AlphabetContract + strconv.FormatUint(uint64(i), 10) + ".frostfs" +} From f7a8f51c669968a6db1dd5caa49be36d45b5663a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 16:58:48 +0300 Subject: [PATCH 0892/1943] [#932] adm: Move command `deposit-notary` to package `notary` Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/generate.go | 18 ++------- .../modules/morph/{ => notary}/notary.go | 21 ++++++---- .../internal/modules/morph/notary/root.go | 28 +++++++++++++ .../internal/modules/morph/root.go | 40 +++++-------------- .../internal/modules/morph/util/const.go | 2 + .../internal/modules/morph/util/util.go | 12 ++++++ 6 files changed, 68 insertions(+), 53 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => notary}/notary.go (86%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/notary/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate.go index 2258eb832..57c40ec9f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate.go @@ -11,7 +11,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" - "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/gas" "github.com/nspcc-dev/neo-go/pkg/smartcontract" @@ -139,7 +138,7 @@ func generateStorageCreds(cmd *cobra.Command, _ []string) error { func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error) { // storage wallet path is not part of the config - storageWalletPath, _ := cmd.Flags().GetString(storageWalletFlag) + storageWalletPath, _ := cmd.Flags().GetString(morphUtil.StorageWalletFlag) // wallet address is not part of the config walletAddress, _ := cmd.Flags().GetString(walletAddressFlag) @@ -152,7 +151,7 @@ func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error } } else { if storageWalletPath == "" { - return fmt.Errorf("missing wallet path (use '--%s ')", storageWalletFlag) + return fmt.Errorf("missing wallet path (use '--%s ')", morphUtil.StorageWalletFlag) } var w *wallet.Wallet @@ -190,7 +189,7 @@ func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error gasStr := viper.GetString(gasFlag) - gasAmount, err := parseGASAmount(gasStr) + gasAmount, err := morphUtil.ParseGASAmount(gasStr) if err != nil { return err } @@ -214,14 +213,3 @@ func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error return wCtx.AwaitTx() } - -func parseGASAmount(s string) (fixedn.Fixed8, error) { - gasAmount, err := fixedn.Fixed8FromString(s) - if err != nil { - return 0, fmt.Errorf("invalid GAS amount %s: %w", s, err) - } - if gasAmount <= 0 { - return 0, fmt.Errorf("GAS amount must be positive (got %d)", gasAmount) - } - return gasAmount, nil -} diff --git a/cmd/frostfs-adm/internal/modules/morph/notary.go b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go similarity index 86% rename from cmd/frostfs-adm/internal/modules/morph/notary.go rename to cmd/frostfs-adm/internal/modules/morph/notary/notary.go index d05307705..ecf9331b5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go @@ -1,4 +1,4 @@ -package morph +package notary import ( "fmt" @@ -21,9 +21,14 @@ import ( "github.com/spf13/viper" ) -// defaultNotaryDepositLifetime is an amount of blocks notary deposit stays valid. -// https://github.com/nspcc-dev/neo-go/blob/master/pkg/core/native/notary.go#L48 -const defaultNotaryDepositLifetime = 5760 +const ( + // defaultNotaryDepositLifetime is an amount of blocks notary deposit stays valid. + // https://github.com/nspcc-dev/neo-go/blob/master/pkg/core/native/notary.go#L48 + defaultNotaryDepositLifetime = 5760 + + walletAccountFlag = "account" + notaryDepositTillFlag = "till" +) func depositNotary(cmd *cobra.Command, _ []string) error { w, err := openWallet(cmd) @@ -55,11 +60,11 @@ func depositNotary(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't unlock account: %v", err) } - gasStr, err := cmd.Flags().GetString(refillGasAmountFlag) + gasStr, err := cmd.Flags().GetString(morphUtil.RefillGasAmountFlag) if err != nil { return err } - gasAmount, err := parseGASAmount(gasStr) + gasAmount, err := morphUtil.ParseGASAmount(gasStr) if err != nil { return err } @@ -121,11 +126,11 @@ func transferGas(cmd *cobra.Command, acc *wallet.Account, accHash util.Uint160, } func openWallet(cmd *cobra.Command) (*wallet.Wallet, error) { - p, err := cmd.Flags().GetString(storageWalletFlag) + p, err := cmd.Flags().GetString(morphUtil.StorageWalletFlag) if err != nil { return nil, err } else if p == "" { - return nil, fmt.Errorf("missing wallet path (use '--%s ')", storageWalletFlag) + return nil, fmt.Errorf("missing wallet path (use '--%s ')", morphUtil.StorageWalletFlag) } w, err := wallet.NewWalletFromFile(p) diff --git a/cmd/frostfs-adm/internal/modules/morph/notary/root.go b/cmd/frostfs-adm/internal/modules/morph/notary/root.go new file mode 100644 index 000000000..9a4164c7b --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/notary/root.go @@ -0,0 +1,28 @@ +package notary + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var DepositCmd = &cobra.Command{ + Use: "deposit-notary", + Short: "Deposit GAS for notary service", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: depositNotary, +} + +func initDepositoryNotaryCmd() { + DepositCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + DepositCmd.Flags().String(util.StorageWalletFlag, "", "Path to storage node wallet") + DepositCmd.Flags().String(walletAccountFlag, "", "Wallet account address") + DepositCmd.Flags().String(util.RefillGasAmountFlag, "", "Amount of GAS to deposit") + DepositCmd.Flags().String(notaryDepositTillFlag, "", "Notary deposit duration in blocks") +} + +func init() { + initDepositoryNotaryCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index ac757428a..fdc9bb32c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -4,6 +4,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/notary" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/proxy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" @@ -12,8 +13,8 @@ import ( ) const ( - alphabetSizeFlag = "size" - storageWalletFlag = "storage-wallet" + alphabetSizeFlag = "size" + storageWalletLabelFlag = "label" storageGasCLIFlag = "initial-gas" storageGasConfigFlag = "storage.initial_gas" @@ -35,9 +36,6 @@ const ( containerDumpFlag = "dump" containerContractFlag = "container-contract" containerIDsFlag = "cid" - refillGasAmountFlag = "gas" - walletAccountFlag = "account" - notaryDepositTillFlag = "till" walletAddressFlag = "wallet-address" ) @@ -94,10 +92,10 @@ var ( PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - _ = viper.BindPFlag(refillGasAmountFlag, cmd.Flags().Lookup(refillGasAmountFlag)) + _ = viper.BindPFlag(util.RefillGasAmountFlag, cmd.Flags().Lookup(util.RefillGasAmountFlag)) }, RunE: func(cmd *cobra.Command, args []string) error { - return refillGas(cmd, refillGasAmountFlag, false) + return refillGas(cmd, util.RefillGasAmountFlag, false) }, } @@ -190,15 +188,6 @@ var ( RunE: listContainers, } - depositNotaryCmd = &cobra.Command{ - Use: "deposit-notary", - Short: "Deposit GAS for notary service", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: depositNotary, - } - netmapCandidatesCmd = &cobra.Command{ Use: "netmap-candidates", Short: "List netmap candidates nodes", @@ -228,7 +217,7 @@ func init() { initRestoreContainersCmd() initListContainersCmd() initRefillGasCmd() - initDepositoryNotaryCmd() + RootCmd.AddCommand(notary.DepositCmd) initNetmapCandidatesCmd() RootCmd.AddCommand(ape.Cmd) @@ -243,23 +232,14 @@ func initNetmapCandidatesCmd() { netmapCandidatesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) } -func initDepositoryNotaryCmd() { - RootCmd.AddCommand(depositNotaryCmd) - depositNotaryCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - depositNotaryCmd.Flags().String(storageWalletFlag, "", "Path to storage node wallet") - depositNotaryCmd.Flags().String(walletAccountFlag, "", "Wallet account address") - depositNotaryCmd.Flags().String(refillGasAmountFlag, "", "Amount of GAS to deposit") - depositNotaryCmd.Flags().String(notaryDepositTillFlag, "", "Notary deposit duration in blocks") -} - func initRefillGasCmd() { RootCmd.AddCommand(refillGasCmd) refillGasCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) refillGasCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - refillGasCmd.Flags().String(storageWalletFlag, "", "Path to storage node wallet") + refillGasCmd.Flags().String(util.StorageWalletFlag, "", "Path to storage node wallet") refillGasCmd.Flags().String(walletAddressFlag, "", "Address of wallet") - refillGasCmd.Flags().String(refillGasAmountFlag, "", "Additional amount of GAS to transfer") - refillGasCmd.MarkFlagsMutuallyExclusive(walletAddressFlag, storageWalletFlag) + refillGasCmd.Flags().String(util.RefillGasAmountFlag, "", "Additional amount of GAS to transfer") + refillGasCmd.MarkFlagsMutuallyExclusive(walletAddressFlag, util.StorageWalletFlag) } func initListContainersCmd() { @@ -330,7 +310,7 @@ func initGenerateStorageCmd() { RootCmd.AddCommand(generateStorageCmd) generateStorageCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) generateStorageCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - generateStorageCmd.Flags().String(storageWalletFlag, "", "Path to new storage node wallet") + generateStorageCmd.Flags().String(util.StorageWalletFlag, "", "Path to new storage node wallet") generateStorageCmd.Flags().String(storageGasCLIFlag, "", "Initial amount of GAS to transfer") generateStorageCmd.Flags().StringP(storageWalletLabelFlag, "l", "", "Wallet label") } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/util/const.go index f100ebc61..a532eb2ab 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/const.go @@ -23,6 +23,8 @@ const ( ContractsURLFlagDesc = "URL to archive with compiled FrostFS contracts" EpochDurationInitFlag = "network.epoch_duration" MaxObjectSizeInitFlag = "network.max_object_size" + RefillGasAmountFlag = "gas" + StorageWalletFlag = "storage-wallet" SingleAccountName = "single" CommitteeAccountName = "committee" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/util.go b/cmd/frostfs-adm/internal/modules/morph/util/util.go index a6f94049b..1fd9695c3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/util.go @@ -15,6 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/viper" @@ -162,3 +163,14 @@ func readContractsFromArchive(file io.Reader, names []string) (map[string]*Contr func GetAlphabetNNSDomain(i int) string { return AlphabetContract + strconv.FormatUint(uint64(i), 10) + ".frostfs" } + +func ParseGASAmount(s string) (fixedn.Fixed8, error) { + gasAmount, err := fixedn.Fixed8FromString(s) + if err != nil { + return 0, fmt.Errorf("invalid GAS amount %s: %w", s, err) + } + if gasAmount <= 0 { + return 0, fmt.Errorf("GAS amount must be positive (got %d)", gasAmount) + } + return gasAmount, nil +} From be15eab82a66d0d10989658231a805c2a7eaaae3 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 17:28:10 +0300 Subject: [PATCH 0893/1943] [#932] adm: Move `dump/set-config` to package `config` Signed-off-by: Anton Nikiforov --- .../modules/morph/{ => config}/config.go | 12 +-- .../internal/modules/morph/config/root.go | 46 +++++++++++ .../modules/morph/initialize_deploy.go | 4 +- .../internal/modules/morph/initialize_test.go | 5 +- .../internal/modules/morph/netmap_util.go | 45 ----------- .../internal/modules/morph/root.go | 77 +++++-------------- .../internal/modules/morph/util/const.go | 28 ++++--- .../internal/modules/morph/util/netmap.go | 49 ++++++++++++ 8 files changed, 142 insertions(+), 124 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => config}/config.go (94%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/config/root.go delete mode 100644 cmd/frostfs-adm/internal/modules/morph/netmap_util.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/util/netmap.go diff --git a/cmd/frostfs-adm/internal/modules/morph/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go similarity index 94% rename from cmd/frostfs-adm/internal/modules/morph/config.go rename to cmd/frostfs-adm/internal/modules/morph/config/config.go index 2cff322d1..b1ed1cc21 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -1,4 +1,4 @@ -package morph +package config import ( "bytes" @@ -51,7 +51,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { buf := bytes.NewBuffer(nil) tw := tabwriter.NewWriter(buf, 0, 2, 2, ' ', 0) - m, err := parseConfigFromNetmapContract(arr) + m, err := util.ParseConfigFromNetmapContract(arr) if err != nil { return err } @@ -66,7 +66,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%d (int)\n", k, n))) case netmap.HomomorphicHashingDisabledKey, netmap.MaintenanceModeAllowedConfig: if len(v) == 0 || len(v) > 1 { - return invalidConfigValueErr(k) + return util.InvalidConfigValueErr(k) } _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%t (bool)\n", k, v[0] == 1))) default: @@ -80,7 +80,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { return nil } -func setConfigCmd(cmd *cobra.Command, args []string) error { +func SetConfigCmd(cmd *cobra.Command, args []string) error { if len(args) == 0 { return errors.New("empty config pairs") } @@ -162,7 +162,3 @@ func parseConfigPair(kvStr string, force bool) (key string, val any, err error) return } - -func invalidConfigValueErr(key string) error { - return fmt.Errorf("invalid %s config value from netmap contract", key) -} diff --git a/cmd/frostfs-adm/internal/modules/morph/config/root.go b/cmd/frostfs-adm/internal/modules/morph/config/root.go new file mode 100644 index 000000000..4b5366226 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/config/root.go @@ -0,0 +1,46 @@ +package config + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + SetCmd = &cobra.Command{ + Use: "set-config key1=val1 [key2=val2 ...]", + DisableFlagsInUseLine: true, + Short: "Add/update global config value in the FrostFS network", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + Args: cobra.MinimumNArgs(1), + RunE: SetConfigCmd, + } + + DumpCmd = &cobra.Command{ + Use: "dump-config", + Short: "Dump FrostFS network config", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: dumpNetworkConfig, + } +) + +func initSetConfigCmd() { + SetCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) + SetCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + SetCmd.Flags().Bool(forceConfigSet, false, "Force setting not well-known configuration key") + SetCmd.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") +} + +func initDumpNetworkConfigCmd() { + DumpCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) +} + +func init() { + initSetConfigCmd() + initDumpNetworkConfigCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 50427c93f..3a78f6042 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -372,7 +372,7 @@ func getContractDeployData(c *morphUtil.InitializeContext, ctrName string, keysP items = append(items, c.Contracts[morphUtil.ProxyContract].Hash) } case morphUtil.NetmapContract: - md := getDefaultNetmapContractConfigMap() + md := morphUtil.GetDefaultNetmapContractConfigMap() if method == updateMethodName { if err := mergeNetmapConfig(c.ReadOnlyInvoker, md); err != nil { return nil, err @@ -450,7 +450,7 @@ func mergeNetmapConfig(roInvoker *invoker.Invoker, md map[string]any) error { if err != nil { return err } - m, err := parseConfigFromNetmapContract(arr) + m, err := morphUtil.ParseConfigFromNetmapContract(arr) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index e9571e50c..f0ad8044c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + cmdConfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" @@ -75,8 +76,8 @@ func testInitialize(t *testing.T, committeeSize int) { require.NoError(t, forceNewEpochCmd(forceNewEpoch, nil)) }) t.Run("set-config", func(t *testing.T) { - require.NoError(t, setConfig.Flags().Set(util.LocalDumpFlag, dumpPath)) - require.NoError(t, setConfigCmd(setConfig, []string{"MaintenanceModeAllowed=true"})) + require.NoError(t, cmdConfig.SetCmd.Flags().Set(util.LocalDumpFlag, dumpPath)) + require.NoError(t, cmdConfig.SetConfigCmd(cmdConfig.SetCmd, []string{"MaintenanceModeAllowed=true"})) }) t.Run("set-policy", func(t *testing.T) { require.NoError(t, policy.Set.Flags().Set(util.LocalDumpFlag, dumpPath)) diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_util.go b/cmd/frostfs-adm/internal/modules/morph/netmap_util.go deleted file mode 100644 index 3cf853cca..000000000 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_util.go +++ /dev/null @@ -1,45 +0,0 @@ -package morph - -import ( - "errors" - - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/spf13/viper" -) - -func getDefaultNetmapContractConfigMap() map[string]any { - m := make(map[string]any) - m[netmap.EpochDurationConfig] = viper.GetInt64(util.EpochDurationInitFlag) - m[netmap.MaxObjectSizeConfig] = viper.GetInt64(util.MaxObjectSizeInitFlag) - m[netmap.ContainerFeeConfig] = viper.GetInt64(containerFeeInitFlag) - m[netmap.ContainerAliasFeeConfig] = viper.GetInt64(containerAliasFeeInitFlag) - m[netmap.IrCandidateFeeConfig] = viper.GetInt64(candidateFeeInitFlag) - m[netmap.WithdrawFeeConfig] = viper.GetInt64(withdrawFeeInitFlag) - m[netmap.HomomorphicHashingDisabledKey] = viper.GetBool(homomorphicHashDisabledInitFlag) - m[netmap.MaintenanceModeAllowedConfig] = viper.GetBool(maintenanceModeAllowedInitFlag) - return m -} - -func parseConfigFromNetmapContract(arr []stackitem.Item) (map[string][]byte, error) { - m := make(map[string][]byte, len(arr)) - for _, param := range arr { - tuple, ok := param.Value().([]stackitem.Item) - if !ok || len(tuple) != 2 { - return nil, errors.New("invalid ListConfig response from netmap contract") - } - - k, err := tuple[0].TryBytes() - if err != nil { - return nil, errors.New("invalid config key from netmap contract") - } - - v, err := tuple[1].TryBytes() - if err != nil { - return nil, invalidConfigValueErr(string(k)) - } - m[string(k)] = v - } - return m, nil -} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index fdc9bb32c..f0e47bc74 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -3,6 +3,7 @@ package morph import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/balance" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/notary" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" @@ -21,21 +22,19 @@ const ( maxObjectSizeCLIFlag = "max-object-size" - epochDurationCLIFlag = "epoch-duration" - containerFeeInitFlag = "network.fee.container" - containerAliasFeeInitFlag = "network.fee.container_alias" - containerFeeCLIFlag = "container-fee" - containerAliasFeeCLIFlag = "container-alias-fee" - candidateFeeInitFlag = "network.fee.candidate" - candidateFeeCLIFlag = "candidate-fee" - homomorphicHashDisabledInitFlag = "network.homomorphic_hash_disabled" - maintenanceModeAllowedInitFlag = "network.maintenance_mode_allowed" - homomorphicHashDisabledCLIFlag = "homomorphic-disabled" - withdrawFeeInitFlag = "network.fee.withdraw" - withdrawFeeCLIFlag = "withdraw-fee" - containerDumpFlag = "dump" - containerContractFlag = "container-contract" - containerIDsFlag = "cid" + epochDurationCLIFlag = "epoch-duration" + + containerFeeCLIFlag = "container-fee" + containerAliasFeeCLIFlag = "container-alias-fee" + + candidateFeeCLIFlag = "candidate-fee" + + homomorphicHashDisabledCLIFlag = "homomorphic-disabled" + + withdrawFeeCLIFlag = "withdraw-fee" + containerDumpFlag = "dump" + containerContractFlag = "container-contract" + containerIDsFlag = "cid" walletAddressFlag = "wallet-address" ) @@ -65,11 +64,11 @@ var ( _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) _ = viper.BindPFlag(util.EpochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) _ = viper.BindPFlag(util.MaxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) - _ = viper.BindPFlag(homomorphicHashDisabledInitFlag, cmd.Flags().Lookup(homomorphicHashDisabledCLIFlag)) - _ = viper.BindPFlag(candidateFeeInitFlag, cmd.Flags().Lookup(candidateFeeCLIFlag)) - _ = viper.BindPFlag(containerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) - _ = viper.BindPFlag(containerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) - _ = viper.BindPFlag(withdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) + _ = viper.BindPFlag(util.HomomorphicHashDisabledInitFlag, cmd.Flags().Lookup(homomorphicHashDisabledCLIFlag)) + _ = viper.BindPFlag(util.CandidateFeeInitFlag, cmd.Flags().Lookup(candidateFeeCLIFlag)) + _ = viper.BindPFlag(util.ContainerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) + _ = viper.BindPFlag(util.ContainerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) + _ = viper.BindPFlag(util.WithdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) _ = viper.BindPFlag(util.ProtoConfigPath, cmd.Flags().Lookup(util.ProtoConfigPath)) }, RunE: initializeSideChainCmd, @@ -120,18 +119,6 @@ var ( RunE: removeNodesCmd, } - setConfig = &cobra.Command{ - Use: "set-config key1=val1 [key2=val2 ...]", - DisableFlagsInUseLine: true, - Short: "Add/update global config value in the FrostFS network", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - Args: cobra.MinimumNArgs(1), - RunE: setConfigCmd, - } - dumpContractHashesCmd = &cobra.Command{ Use: "dump-hashes", Short: "Dump deployed contract hashes", @@ -141,15 +128,6 @@ var ( RunE: dumpContractHashes, } - dumpNetworkConfigCmd = &cobra.Command{ - Use: "dump-config", - Short: "Dump FrostFS network config", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: dumpNetworkConfig, - } - updateContractsCmd = &cobra.Command{ Use: "update-contracts", Short: "Update FrostFS contracts", @@ -209,8 +187,8 @@ func init() { RootCmd.AddCommand(policy.Set) RootCmd.AddCommand(policy.Dump) initDumpContractHashesCmd() - initDumpNetworkConfigCmd() - initSetConfigCmd() + RootCmd.AddCommand(config.SetCmd) + RootCmd.AddCommand(config.DumpCmd) RootCmd.AddCommand(balance.DumpCmd) initUpdateContractsCmd() initDumpContainersCmd() @@ -273,19 +251,6 @@ func initUpdateContractsCmd() { updateContractsCmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) } -func initSetConfigCmd() { - RootCmd.AddCommand(setConfig) - setConfig.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - setConfig.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - setConfig.Flags().Bool(forceConfigSet, false, "Force setting not well-known configuration key") - setConfig.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") -} - -func initDumpNetworkConfigCmd() { - RootCmd.AddCommand(dumpNetworkConfigCmd) - dumpNetworkConfigCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) -} - func initDumpContractHashesCmd() { RootCmd.AddCommand(dumpContractHashesCmd) dumpContractHashesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/util/const.go index a532eb2ab..c4a674bd6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/const.go @@ -11,20 +11,26 @@ const ( // See: https://github.com/nspcc-dev/neo-go/blob/740488f7f35e367eaa99a71c0a609c315fe2b0fc/pkg/core/transaction/witness.go#L10 MaxAlphabetNodes = 22 - EndpointFlag = "rpc-endpoint" - EndpointFlagDesc = "N3 RPC node endpoint" - EndpointFlagShort = "r" - AlphabetWalletsFlag = "alphabet-wallets" - AlphabetWalletsFlagDesc = "Path to alphabet wallets dir" - LocalDumpFlag = "local-dump" - ContractsInitFlag = "contracts" + EndpointFlag = "rpc-endpoint" + EndpointFlagDesc = "N3 RPC node endpoint" + EndpointFlagShort = "r" + AlphabetWalletsFlag = "alphabet-wallets" + AlphabetWalletsFlagDesc = "Path to alphabet wallets dir" + LocalDumpFlag = "local-dump" + ContractsInitFlag = "contracts" ContractsInitFlagDesc = "Path to archive with compiled FrostFS contracts (the default is to fetch the latest release from the official repository)" ContractsURLFlag = "contracts-url" ContractsURLFlagDesc = "URL to archive with compiled FrostFS contracts" - EpochDurationInitFlag = "network.epoch_duration" - MaxObjectSizeInitFlag = "network.max_object_size" - RefillGasAmountFlag = "gas" - StorageWalletFlag = "storage-wallet" + EpochDurationInitFlag = "network.epoch_duration" + MaxObjectSizeInitFlag = "network.max_object_size" + RefillGasAmountFlag = "gas" + StorageWalletFlag = "storage-wallet" + ContainerFeeInitFlag = "network.fee.container" + ContainerAliasFeeInitFlag = "network.fee.container_alias" + CandidateFeeInitFlag = "network.fee.candidate" + WithdrawFeeInitFlag = "network.fee.withdraw" + MaintenanceModeAllowedInitFlag = "network.maintenance_mode_allowed" + HomomorphicHashDisabledInitFlag = "network.homomorphic_hash_disabled" SingleAccountName = "single" CommitteeAccountName = "committee" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/netmap.go b/cmd/frostfs-adm/internal/modules/morph/util/netmap.go new file mode 100644 index 000000000..c9dc2e3b5 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/util/netmap.go @@ -0,0 +1,49 @@ +package util + +import ( + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "github.com/spf13/viper" +) + +func GetDefaultNetmapContractConfigMap() map[string]any { + m := make(map[string]any) + m[netmap.EpochDurationConfig] = viper.GetInt64(EpochDurationInitFlag) + m[netmap.MaxObjectSizeConfig] = viper.GetInt64(MaxObjectSizeInitFlag) + m[netmap.ContainerFeeConfig] = viper.GetInt64(ContainerFeeInitFlag) + m[netmap.ContainerAliasFeeConfig] = viper.GetInt64(ContainerAliasFeeInitFlag) + m[netmap.IrCandidateFeeConfig] = viper.GetInt64(CandidateFeeInitFlag) + m[netmap.WithdrawFeeConfig] = viper.GetInt64(WithdrawFeeInitFlag) + m[netmap.HomomorphicHashingDisabledKey] = viper.GetBool(HomomorphicHashDisabledInitFlag) + m[netmap.MaintenanceModeAllowedConfig] = viper.GetBool(MaintenanceModeAllowedInitFlag) + return m +} + +func ParseConfigFromNetmapContract(arr []stackitem.Item) (map[string][]byte, error) { + m := make(map[string][]byte, len(arr)) + for _, param := range arr { + tuple, ok := param.Value().([]stackitem.Item) + if !ok || len(tuple) != 2 { + return nil, errors.New("invalid ListConfig response from netmap contract") + } + + k, err := tuple[0].TryBytes() + if err != nil { + return nil, errors.New("invalid config key from netmap contract") + } + + v, err := tuple[1].TryBytes() + if err != nil { + return nil, InvalidConfigValueErr(string(k)) + } + m[string(k)] = v + } + return m, nil +} + +func InvalidConfigValueErr(key string) error { + return fmt.Errorf("invalid %s config value from netmap contract", key) +} From 36fd6c663c6030fb283f540f2d7c31dab878aeef Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 17:36:26 +0300 Subject: [PATCH 0894/1943] [#932] adm: Move `dump/restore/list-containers` to package `container` Signed-off-by: Anton Nikiforov --- .../morph/{ => container}/container.go | 2 +- .../internal/modules/morph/container/root.go | 68 +++++++++++++++++++ .../internal/modules/morph/root.go | 62 ++--------------- 3 files changed, 74 insertions(+), 58 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => container}/container.go (99%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/container/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/container.go rename to cmd/frostfs-adm/internal/modules/morph/container/container.go index b18fa79ed..46d4921fe 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -1,4 +1,4 @@ -package morph +package container import ( "encoding/json" diff --git a/cmd/frostfs-adm/internal/modules/morph/container/root.go b/cmd/frostfs-adm/internal/modules/morph/container/root.go new file mode 100644 index 000000000..4cae8cf4f --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/container/root.go @@ -0,0 +1,68 @@ +package container + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + containerDumpFlag = "dump" + containerContractFlag = "container-contract" + containerIDsFlag = "cid" +) + +var ( + DumpCmd = &cobra.Command{ + Use: "dump-containers", + Short: "Dump FrostFS containers to file", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: dumpContainers, + } + + RestoreCmd = &cobra.Command{ + Use: "restore-containers", + Short: "Restore FrostFS containers from file", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: restoreContainers, + } + + ListCmd = &cobra.Command{ + Use: "list-containers", + Short: "List FrostFS containers", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: listContainers, + } +) + +func initListContainersCmd() { + ListCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + ListCmd.Flags().String(containerContractFlag, "", "Container contract hash (for networks without NNS)") +} + +func initRestoreContainersCmd() { + RestoreCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) + RestoreCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + RestoreCmd.Flags().String(containerDumpFlag, "", "File to restore containers from") + RestoreCmd.Flags().StringSlice(containerIDsFlag, nil, "Containers to restore") +} + +func initDumpContainersCmd() { + DumpCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + DumpCmd.Flags().String(containerDumpFlag, "", "File where to save dumped containers") + DumpCmd.Flags().String(containerContractFlag, "", "Container contract hash (for networks without NNS)") + DumpCmd.Flags().StringSlice(containerIDsFlag, nil, "Containers to dump") +} + +func init() { + initDumpContainersCmd() + initRestoreContainersCmd() + initListContainersCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index f0e47bc74..98e72fa51 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -4,6 +4,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/container" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/notary" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" @@ -31,10 +32,7 @@ const ( homomorphicHashDisabledCLIFlag = "homomorphic-disabled" - withdrawFeeCLIFlag = "withdraw-fee" - containerDumpFlag = "dump" - containerContractFlag = "container-contract" - containerIDsFlag = "cid" + withdrawFeeCLIFlag = "withdraw-fee" walletAddressFlag = "wallet-address" ) @@ -138,34 +136,6 @@ var ( RunE: updateContracts, } - dumpContainersCmd = &cobra.Command{ - Use: "dump-containers", - Short: "Dump FrostFS containers to file", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: dumpContainers, - } - - restoreContainersCmd = &cobra.Command{ - Use: "restore-containers", - Short: "Restore FrostFS containers from file", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: restoreContainers, - } - - listContainersCmd = &cobra.Command{ - Use: "list-containers", - Short: "List FrostFS containers", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: listContainers, - } - netmapCandidatesCmd = &cobra.Command{ Use: "netmap-candidates", Short: "List netmap candidates nodes", @@ -191,9 +161,9 @@ func init() { RootCmd.AddCommand(config.DumpCmd) RootCmd.AddCommand(balance.DumpCmd) initUpdateContractsCmd() - initDumpContainersCmd() - initRestoreContainersCmd() - initListContainersCmd() + RootCmd.AddCommand(container.ListCmd) + RootCmd.AddCommand(container.RestoreCmd) + RootCmd.AddCommand(container.DumpCmd) initRefillGasCmd() RootCmd.AddCommand(notary.DepositCmd) initNetmapCandidatesCmd() @@ -220,28 +190,6 @@ func initRefillGasCmd() { refillGasCmd.MarkFlagsMutuallyExclusive(walletAddressFlag, util.StorageWalletFlag) } -func initListContainersCmd() { - RootCmd.AddCommand(listContainersCmd) - listContainersCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - listContainersCmd.Flags().String(containerContractFlag, "", "Container contract hash (for networks without NNS)") -} - -func initRestoreContainersCmd() { - RootCmd.AddCommand(restoreContainersCmd) - restoreContainersCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - restoreContainersCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - restoreContainersCmd.Flags().String(containerDumpFlag, "", "File to restore containers from") - restoreContainersCmd.Flags().StringSlice(containerIDsFlag, nil, "Containers to restore") -} - -func initDumpContainersCmd() { - RootCmd.AddCommand(dumpContainersCmd) - dumpContainersCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - dumpContainersCmd.Flags().String(containerDumpFlag, "", "File where to save dumped containers") - dumpContainersCmd.Flags().String(containerContractFlag, "", "Container contract hash (for networks without NNS)") - dumpContainersCmd.Flags().StringSlice(containerIDsFlag, nil, "Containers to dump") -} - func initUpdateContractsCmd() { RootCmd.AddCommand(updateContractsCmd) updateContractsCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) From 9690bd02aad0c767740890fd4b5d0f1ef82bed39 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 17:41:44 +0300 Subject: [PATCH 0895/1943] [#932] adm: Move `netmap-candidates` to package `netmap` Signed-off-by: Anton Nikiforov --- .../morph/{ => netmap}/netmap_candidates.go | 2 +- .../internal/modules/morph/netmap/root.go | 25 +++++++++++++++++++ .../internal/modules/morph/root.go | 18 ++----------- 3 files changed, 28 insertions(+), 17 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => netmap}/netmap_candidates.go (98%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/netmap/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go b/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go similarity index 98% rename from cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go rename to cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go index c09ba8b8d..f105fe911 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap_candidates.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go @@ -1,4 +1,4 @@ -package morph +package netmap import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go new file mode 100644 index 000000000..b8f9a61da --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go @@ -0,0 +1,25 @@ +package netmap + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var CandidatesCmd = &cobra.Command{ + Use: "netmap-candidates", + Short: "List netmap candidates nodes", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + }, + Run: listNetmapCandidatesNodes, +} + +func initNetmapCandidatesCmd() { + CandidatesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) +} + +func init() { + initNetmapCandidatesCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 98e72fa51..908c4183f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/container" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/notary" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/proxy" @@ -135,16 +136,6 @@ var ( }, RunE: updateContracts, } - - netmapCandidatesCmd = &cobra.Command{ - Use: "netmap-candidates", - Short: "List netmap candidates nodes", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - }, - Run: listNetmapCandidatesNodes, - } ) func init() { @@ -166,7 +157,7 @@ func init() { RootCmd.AddCommand(container.DumpCmd) initRefillGasCmd() RootCmd.AddCommand(notary.DepositCmd) - initNetmapCandidatesCmd() + RootCmd.AddCommand(netmap.CandidatesCmd) RootCmd.AddCommand(ape.Cmd) RootCmd.AddCommand(proxy.AddAccountCmd) @@ -175,11 +166,6 @@ func init() { RootCmd.AddCommand(frostfsid.Cmd) } -func initNetmapCandidatesCmd() { - RootCmd.AddCommand(netmapCandidatesCmd) - netmapCandidatesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) -} - func initRefillGasCmd() { RootCmd.AddCommand(refillGasCmd) refillGasCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) From ce4254798032d3cc616cf05f2925a69f9a319a2c Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 17:53:00 +0300 Subject: [PATCH 0896/1943] [#932] adm: Move `remove-nodes` to package `node` Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/epoch.go | 22 +-------------- .../internal/modules/morph/initialize_test.go | 5 ++-- .../morph/{remove_node.go => node/remove.go} | 6 ++-- .../internal/modules/morph/node/root.go | 28 +++++++++++++++++++ .../internal/modules/morph/root.go | 21 ++------------ .../internal/modules/morph/util/netmap.go | 20 +++++++++++++ 6 files changed, 57 insertions(+), 45 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{remove_node.go => node/remove.go} (92%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/node/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/epoch.go index 851f757b0..69d4d3151 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/epoch.go @@ -1,17 +1,12 @@ package morph import ( - "errors" "fmt" "strings" util2 "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" - "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" - "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -34,7 +29,7 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { } bw := io.NewBufBinWriter() - if err := emitNewEpochCall(bw, wCtx, nmHash); err != nil { + if err := util2.EmitNewEpochCall(bw, wCtx, nmHash); err != nil { return err } @@ -47,18 +42,3 @@ func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { } return err } - -func emitNewEpochCall(bw *io.BufBinWriter, wCtx *util2.InitializeContext, nmHash util.Uint160) error { - curr, err := unwrap.Int64(wCtx.ReadOnlyInvoker.Call(nmHash, "epoch")) - if err != nil { - return errors.New("can't fetch current epoch from the netmap contract") - } - - newEpoch := curr + 1 - wCtx.Command.Printf("Current epoch: %d, increase to %d.\n", curr, newEpoch) - - // In NeoFS this is done via Notary contract. Here, however, we can form the - // transaction locally. - emit.AppCall(bw.BinWriter, nmHash, "newEpoch", callflag.All, newEpoch) - return bw.Err -} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index f0ad8044c..02af6458c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -10,6 +10,7 @@ import ( "time" cmdConfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/node" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" @@ -88,8 +89,8 @@ func testInitialize(t *testing.T, committeeSize int) { require.NoError(t, err) pub := hex.EncodeToString(pk.PublicKey().Bytes()) - require.NoError(t, removeNodes.Flags().Set(util.LocalDumpFlag, dumpPath)) - require.NoError(t, removeNodesCmd(removeNodes, []string{pub})) + require.NoError(t, node.RemoveCmd.Flags().Set(util.LocalDumpFlag, dumpPath)) + require.NoError(t, node.RemoveNodesCmd(node.RemoveCmd, []string{pub})) }) } diff --git a/cmd/frostfs-adm/internal/modules/morph/remove_node.go b/cmd/frostfs-adm/internal/modules/morph/node/remove.go similarity index 92% rename from cmd/frostfs-adm/internal/modules/morph/remove_node.go rename to cmd/frostfs-adm/internal/modules/morph/node/remove.go index e63a5dee9..0e305c1f4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/remove_node.go +++ b/cmd/frostfs-adm/internal/modules/morph/node/remove.go @@ -1,4 +1,4 @@ -package morph +package node import ( "errors" @@ -15,7 +15,7 @@ import ( "github.com/spf13/viper" ) -func removeNodesCmd(cmd *cobra.Command, args []string) error { +func RemoveNodesCmd(cmd *cobra.Command, args []string) error { if len(args) == 0 { return errors.New("at least one node key must be provided") } @@ -52,7 +52,7 @@ func removeNodesCmd(cmd *cobra.Command, args []string) error { int64(netmapcontract.NodeStateOffline), nodeKeys[i].Bytes()) } - if err := emitNewEpochCall(bw, wCtx, nmHash); err != nil { + if err := util.EmitNewEpochCall(bw, wCtx, nmHash); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/node/root.go b/cmd/frostfs-adm/internal/modules/morph/node/root.go new file mode 100644 index 000000000..e301a1bc3 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/node/root.go @@ -0,0 +1,28 @@ +package node + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var RemoveCmd = &cobra.Command{ + Use: "remove-nodes key1 [key2 [...]]", + Short: "Remove storage nodes from the netmap", + Long: `Move nodes to the Offline state in the candidates list and tick an epoch to update the netmap`, + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: RemoveNodesCmd, +} + +func initRemoveNodesCmd() { + RemoveCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) + RemoveCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + RemoveCmd.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") +} + +func init() { + initRemoveNodesCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 908c4183f..23c413bff 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/container" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/node" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/notary" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/proxy" @@ -107,17 +108,6 @@ var ( RunE: forceNewEpochCmd, } - removeNodes = &cobra.Command{ - Use: "remove-nodes key1 [key2 [...]]", - Short: "Remove storage nodes from the netmap", - Long: `Move nodes to the Offline state in the candidates list and tick an epoch to update the netmap`, - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: removeNodesCmd, - } - dumpContractHashesCmd = &cobra.Command{ Use: "dump-hashes", Short: "Dump deployed contract hashes", @@ -144,7 +134,7 @@ func init() { initDeployCmd() initGenerateStorageCmd() initForceNewEpochCmd() - initRemoveNodesCmd() + RootCmd.AddCommand(node.RemoveCmd) RootCmd.AddCommand(policy.Set) RootCmd.AddCommand(policy.Dump) initDumpContractHashesCmd() @@ -191,13 +181,6 @@ func initDumpContractHashesCmd() { dumpContractHashesCmd.Flags().String(customZoneFlag, "", "Custom zone to search.") } -func initRemoveNodesCmd() { - RootCmd.AddCommand(removeNodes) - removeNodes.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - removeNodes.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - removeNodes.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") -} - func initForceNewEpochCmd() { RootCmd.AddCommand(forceNewEpoch) forceNewEpoch.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) diff --git a/cmd/frostfs-adm/internal/modules/morph/util/netmap.go b/cmd/frostfs-adm/internal/modules/morph/util/netmap.go index c9dc2e3b5..34c527d0b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/netmap.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/netmap.go @@ -5,6 +5,11 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/spf13/viper" ) @@ -47,3 +52,18 @@ func ParseConfigFromNetmapContract(arr []stackitem.Item) (map[string][]byte, err func InvalidConfigValueErr(key string) error { return fmt.Errorf("invalid %s config value from netmap contract", key) } + +func EmitNewEpochCall(bw *io.BufBinWriter, wCtx *InitializeContext, nmHash util.Uint160) error { + curr, err := unwrap.Int64(wCtx.ReadOnlyInvoker.Call(nmHash, "epoch")) + if err != nil { + return errors.New("can't fetch current epoch from the netmap contract") + } + + newEpoch := curr + 1 + wCtx.Command.Printf("Current epoch: %d, increase to %d.\n", curr, newEpoch) + + // In NeoFS this is done via Notary contract. Here, however, we can form the + // transaction locally. + emit.AppCall(bw.BinWriter, nmHash, "newEpoch", callflag.All, newEpoch) + return bw.Err +} From 9b65f1595aef0ebd43fedf419a3a8b96acfb2132 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 1 Feb 2024 17:59:53 +0300 Subject: [PATCH 0897/1943] [#932] adm: Move `force-new-epoch` to package `netmap` Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/initialize_test.go | 5 +-- .../modules/morph/{ => netmap}/epoch.go | 4 +-- .../internal/modules/morph/netmap/root.go | 36 ++++++++++++++----- .../internal/modules/morph/root.go | 19 +--------- 4 files changed, 33 insertions(+), 31 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => netmap}/epoch.go (93%) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index 02af6458c..8b0fddf3d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -10,6 +10,7 @@ import ( "time" cmdConfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/node" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" @@ -73,8 +74,8 @@ func testInitialize(t *testing.T, committeeSize int) { require.NoError(t, initializeSideChainCmd(initCmd, nil)) t.Run("force-new-epoch", func(t *testing.T) { - require.NoError(t, forceNewEpoch.Flags().Set(util.LocalDumpFlag, dumpPath)) - require.NoError(t, forceNewEpochCmd(forceNewEpoch, nil)) + require.NoError(t, netmap.ForceNewEpoch.Flags().Set(util.LocalDumpFlag, dumpPath)) + require.NoError(t, netmap.ForceNewEpochCmd(netmap.ForceNewEpoch, nil)) }) t.Run("set-config", func(t *testing.T) { require.NoError(t, cmdConfig.SetCmd.Flags().Set(util.LocalDumpFlag, dumpPath)) diff --git a/cmd/frostfs-adm/internal/modules/morph/epoch.go b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go similarity index 93% rename from cmd/frostfs-adm/internal/modules/morph/epoch.go rename to cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go index 69d4d3151..4d64f190f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go @@ -1,4 +1,4 @@ -package morph +package netmap import ( "fmt" @@ -11,7 +11,7 @@ import ( "github.com/spf13/viper" ) -func forceNewEpochCmd(cmd *cobra.Command, _ []string) error { +func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { wCtx, err := util2.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go index b8f9a61da..888b994e9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go @@ -6,20 +6,38 @@ import ( "github.com/spf13/viper" ) -var CandidatesCmd = &cobra.Command{ - Use: "netmap-candidates", - Short: "List netmap candidates nodes", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - }, - Run: listNetmapCandidatesNodes, -} +var ( + CandidatesCmd = &cobra.Command{ + Use: "netmap-candidates", + Short: "List netmap candidates nodes", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + }, + Run: listNetmapCandidatesNodes, + } + ForceNewEpoch = &cobra.Command{ + Use: "force-new-epoch", + Short: "Create new FrostFS epoch event in the side chain", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: ForceNewEpochCmd, + } +) func initNetmapCandidatesCmd() { CandidatesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) } +func initForceNewEpochCmd() { + ForceNewEpoch.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) + ForceNewEpoch.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + ForceNewEpoch.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") +} + func init() { initNetmapCandidatesCmd() + initForceNewEpochCmd() } diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 23c413bff..6546b6dee 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -98,16 +98,6 @@ var ( }, } - forceNewEpoch = &cobra.Command{ - Use: "force-new-epoch", - Short: "Create new FrostFS epoch event in the side chain", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: forceNewEpochCmd, - } - dumpContractHashesCmd = &cobra.Command{ Use: "dump-hashes", Short: "Dump deployed contract hashes", @@ -133,7 +123,7 @@ func init() { initInitCmd() initDeployCmd() initGenerateStorageCmd() - initForceNewEpochCmd() + RootCmd.AddCommand(netmap.ForceNewEpoch) RootCmd.AddCommand(node.RemoveCmd) RootCmd.AddCommand(policy.Set) RootCmd.AddCommand(policy.Dump) @@ -181,13 +171,6 @@ func initDumpContractHashesCmd() { dumpContractHashesCmd.Flags().String(customZoneFlag, "", "Custom zone to search.") } -func initForceNewEpochCmd() { - RootCmd.AddCommand(forceNewEpoch) - forceNewEpoch.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - forceNewEpoch.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - forceNewEpoch.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") -} - func initGenerateStorageCmd() { RootCmd.AddCommand(generateStorageCmd) generateStorageCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) From e2557b2f0b23b0c3163b274d524f9cefb74e77b0 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 2 Feb 2024 09:06:24 +0300 Subject: [PATCH 0898/1943] [#932] adm: Move `dump-hashes` to package `contract` Signed-off-by: Anton Nikiforov --- .../morph/{ => contract}/dump_hashes.go | 4 +-- .../internal/modules/morph/contract/root.go | 25 +++++++++++++++++++ .../internal/modules/morph/deploy.go | 5 ++-- .../internal/modules/morph/root.go | 18 ++----------- .../internal/modules/morph/util/const.go | 1 + 5 files changed, 32 insertions(+), 21 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => contract}/dump_hashes.go (98%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/contract/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go similarity index 98% rename from cmd/frostfs-adm/internal/modules/morph/dump_hashes.go rename to cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index c92c25690..8f7fbe6e0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -1,4 +1,4 @@ -package morph +package contract import ( "bytes" @@ -45,7 +45,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { return err } - zone, _ := cmd.Flags().GetString(customZoneFlag) + zone, _ := cmd.Flags().GetString(morphUtil.CustomZoneFlag) if zone != "" { return dumpCustomZoneHashes(cmd, cs.Hash, zone, c) } diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/root.go b/cmd/frostfs-adm/internal/modules/morph/contract/root.go new file mode 100644 index 000000000..8decfd4a3 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/contract/root.go @@ -0,0 +1,25 @@ +package contract + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var DumpHashesCmd = &cobra.Command{ + Use: "dump-hashes", + Short: "Dump deployed contract hashes", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: dumpContractHashes, +} + +func initDumpContractHashesCmd() { + DumpHashesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + DumpHashesCmd.Flags().String(util.CustomZoneFlag, "", "Custom zone to search.") +} + +func init() { + initDumpContractHashesCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/deploy.go b/cmd/frostfs-adm/internal/modules/morph/deploy.go index e6fdbbcf7..a312b34cd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/deploy.go @@ -24,7 +24,6 @@ import ( const ( contractPathFlag = "contract" updateFlag = "update" - customZoneFlag = "domain" ) var deployCmd = &cobra.Command{ @@ -55,7 +54,7 @@ func init() { _ = deployCmd.MarkFlagFilename(contractPathFlag) ff.Bool(updateFlag, false, "Update an existing contract") - ff.String(customZoneFlag, "frostfs", "Custom zone for NNS") + ff.String(util.CustomZoneFlag, "frostfs", "Custom zone for NNS") } func deployContractCmd(cmd *cobra.Command, args []string) error { @@ -85,7 +84,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { callHash := management.Hash method := deployMethodName - zone, _ := cmd.Flags().GetString(customZoneFlag) + zone, _ := cmd.Flags().GetString(util.CustomZoneFlag) domain := ctrName + "." + zone isUpdate, _ := cmd.Flags().GetBool(updateFlag) if isUpdate { diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 6546b6dee..66dd77556 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/contract" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/node" @@ -98,15 +99,6 @@ var ( }, } - dumpContractHashesCmd = &cobra.Command{ - Use: "dump-hashes", - Short: "Dump deployed contract hashes", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: dumpContractHashes, - } - updateContractsCmd = &cobra.Command{ Use: "update-contracts", Short: "Update FrostFS contracts", @@ -127,7 +119,7 @@ func init() { RootCmd.AddCommand(node.RemoveCmd) RootCmd.AddCommand(policy.Set) RootCmd.AddCommand(policy.Dump) - initDumpContractHashesCmd() + RootCmd.AddCommand(contract.DumpHashesCmd) RootCmd.AddCommand(config.SetCmd) RootCmd.AddCommand(config.DumpCmd) RootCmd.AddCommand(balance.DumpCmd) @@ -165,12 +157,6 @@ func initUpdateContractsCmd() { updateContractsCmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) } -func initDumpContractHashesCmd() { - RootCmd.AddCommand(dumpContractHashesCmd) - dumpContractHashesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - dumpContractHashesCmd.Flags().String(customZoneFlag, "", "Custom zone to search.") -} - func initGenerateStorageCmd() { RootCmd.AddCommand(generateStorageCmd) generateStorageCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/util/const.go index c4a674bd6..cdf1cd443 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/const.go @@ -31,6 +31,7 @@ const ( WithdrawFeeInitFlag = "network.fee.withdraw" MaintenanceModeAllowedInitFlag = "network.maintenance_mode_allowed" HomomorphicHashDisabledInitFlag = "network.homomorphic_hash_disabled" + CustomZoneFlag = "domain" SingleAccountName = "single" CommitteeAccountName = "committee" From 76343f19e569f01949396a7a442f0004970cd67a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 2 Feb 2024 09:49:03 +0300 Subject: [PATCH 0899/1943] [#932] adm: Move `update-contracts` to package `contract` Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/contract/root.go | 36 +- .../internal/modules/morph/contract/update.go | 196 +++++++++ .../internal/modules/morph/deploy.go | 4 +- .../modules/morph/frostfsid/frostfsid.go | 1 - .../modules/morph/frostfsid/frostfsid_util.go | 24 -- .../morph/frostfsid/frostfsid_util_test.go | 7 +- .../internal/modules/morph/initialize.go | 2 +- .../modules/morph/initialize_deploy.go | 401 +----------------- .../internal/modules/morph/initialize_nns.go | 3 - .../internal/modules/morph/root.go | 21 +- .../internal/modules/morph/update.go | 22 - .../internal/modules/morph/util/const.go | 3 + .../internal/modules/morph/util/contract.go | 169 ++++++++ .../internal/modules/morph/util/frostfsid.go | 35 ++ .../modules/morph/{ => util}/group.go | 5 +- .../internal/modules/morph/util/netmap.go | 50 +++ 16 files changed, 498 insertions(+), 481 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/contract/update.go delete mode 100644 cmd/frostfs-adm/internal/modules/morph/update.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/util/contract.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/util/frostfsid.go rename cmd/frostfs-adm/internal/modules/morph/{ => util}/group.go (77%) diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/root.go b/cmd/frostfs-adm/internal/modules/morph/contract/root.go index 8decfd4a3..71feb31cd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/root.go @@ -6,20 +6,40 @@ import ( "github.com/spf13/viper" ) -var DumpHashesCmd = &cobra.Command{ - Use: "dump-hashes", - Short: "Dump deployed contract hashes", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: dumpContractHashes, -} +var ( + DumpHashesCmd = &cobra.Command{ + Use: "dump-hashes", + Short: "Dump deployed contract hashes", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: dumpContractHashes, + } + UpdateCmd = &cobra.Command{ + Use: "update-contracts", + Short: "Update FrostFS contracts", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + }, + RunE: updateContracts, + } +) func initDumpContractHashesCmd() { DumpHashesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) DumpHashesCmd.Flags().String(util.CustomZoneFlag, "", "Custom zone to search.") } +func initUpdateContractsCmd() { + UpdateCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) + UpdateCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + UpdateCmd.Flags().String(util.ContractsInitFlag, "", util.ContractsInitFlagDesc) + UpdateCmd.Flags().String(util.ContractsURLFlag, "", util.ContractsURLFlagDesc) + UpdateCmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) +} + func init() { initDumpContractHashesCmd() + initUpdateContractsCmd() } diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/update.go b/cmd/frostfs-adm/internal/modules/morph/contract/update.go new file mode 100644 index 000000000..b1b0bf1b3 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/contract/update.go @@ -0,0 +1,196 @@ +package contract + +import ( + "encoding/hex" + "errors" + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-contract/common" + "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + io2 "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" + neoUtil "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/emit" + "github.com/nspcc-dev/neo-go/pkg/vm/opcode" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var errMissingNNSRecord = errors.New("missing NNS record") + +func updateContracts(cmd *cobra.Command, _ []string) error { + wCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) + if err != nil { + return fmt.Errorf("initialization error: %w", err) + } + + if err := morphUtil.DeployNNS(wCtx, morphUtil.UpdateMethodName); err != nil { + return err + } + + return updateContractsInternal(wCtx) +} + +func updateContractsInternal(c *morphUtil.InitializeContext) error { + alphaCs := c.GetContract(morphUtil.AlphabetContract) + + nnsCs, err := c.NNSContractState() + if err != nil { + return err + } + nnsHash := nnsCs.Hash + + w := io2.NewBufBinWriter() + + // Update script size for a single-node committee is close to the maximum allowed size of 65535. + // Because of this we want to reuse alphabet contract NEF and manifest for different updates. + // The generated script is as following. + // 1. Initialize static slot for alphabet NEF. + // 2. Store NEF into the static slot. + // 3. Push parameters for each alphabet contract on stack. + // 4. Add contract group to the manifest. + // 5. For each alphabet contract, invoke `update` using parameters on stack and + // NEF from step 2 and manifest from step 4. + emit.Instruction(w.BinWriter, opcode.INITSSLOT, []byte{1}) + emit.Bytes(w.BinWriter, alphaCs.RawNEF) + emit.Opcodes(w.BinWriter, opcode.STSFLD0) + + keysParam, err := deployAlphabetAccounts(c, nnsHash, w, alphaCs) + if err != nil { + return err + } + + w.Reset() + + if err = deployOrUpdateContracts(c, w, nnsHash, keysParam); err != nil { + return err + } + + groupKey := c.ContractWallet.Accounts[0].PrivateKey().PublicKey() + _, _, err = c.EmitUpdateNNSGroupScript(w, nnsHash, groupKey) + if err != nil { + return err + } + c.Command.Printf("NNS: Set %s -> %s\n", morphClient.NNSGroupKeyName, hex.EncodeToString(groupKey.Bytes())) + + emit.Opcodes(w.BinWriter, opcode.LDSFLD0) + emit.Int(w.BinWriter, 1) + emit.Opcodes(w.BinWriter, opcode.PACK) + emit.AppCallNoArgs(w.BinWriter, nnsHash, "setPrice", callflag.All) + + if err := c.SendCommitteeTx(w.Bytes(), false); err != nil { + return err + } + return c.AwaitTx() +} + +func deployAlphabetAccounts(c *morphUtil.InitializeContext, nnsHash neoUtil.Uint160, w *io2.BufBinWriter, alphaCs *morphUtil.ContractState) ([]any, error) { + var keysParam []any + + baseGroups := alphaCs.Manifest.Groups + + // alphabet contracts should be deployed by individual nodes to get different hashes. + for i, acc := range c.Accounts { + ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, morphUtil.GetAlphabetNNSDomain(i)) + if err != nil { + return nil, fmt.Errorf("can't resolve hash for contract update: %w", err) + } + + keysParam = append(keysParam, acc.PrivateKey().PublicKey().Bytes()) + + params := c.GetAlphabetDeployItems(i, len(c.Wallets)) + emit.Array(w.BinWriter, params...) + + alphaCs.Manifest.Groups = baseGroups + err = morphUtil.AddManifestGroup(c.ContractWallet, ctrHash, alphaCs) + if err != nil { + return nil, fmt.Errorf("can't sign manifest group: %v", err) + } + + emit.Bytes(w.BinWriter, alphaCs.RawManifest) + emit.Opcodes(w.BinWriter, opcode.LDSFLD0) + emit.Int(w.BinWriter, 3) + emit.Opcodes(w.BinWriter, opcode.PACK) + emit.AppCallNoArgs(w.BinWriter, ctrHash, morphUtil.UpdateMethodName, callflag.All) + } + if err := c.SendCommitteeTx(w.Bytes(), false); err != nil { + if !strings.Contains(err.Error(), common.ErrAlreadyUpdated) { + return nil, err + } + c.Command.Println("Alphabet contracts are already updated.") + } + + return keysParam, nil +} + +func deployOrUpdateContracts(c *morphUtil.InitializeContext, w *io2.BufBinWriter, nnsHash neoUtil.Uint160, keysParam []any) error { + emit.Instruction(w.BinWriter, opcode.INITSSLOT, []byte{1}) + emit.AppCall(w.BinWriter, nnsHash, "getPrice", callflag.All) + emit.Opcodes(w.BinWriter, opcode.STSFLD0) + emit.AppCall(w.BinWriter, nnsHash, "setPrice", callflag.All, 1) + + for _, ctrName := range morphUtil.ContractList { + cs := c.GetContract(ctrName) + + method := morphUtil.UpdateMethodName + ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, morphUtil.DomainOf(ctrName)) + if err != nil { + if errors.Is(err, errMissingNNSRecord) { + // if contract not found we deploy it instead of update + method = morphUtil.DeployMethodName + } else { + return fmt.Errorf("can't resolve hash for contract update: %w", err) + } + } + + err = morphUtil.AddManifestGroup(c.ContractWallet, ctrHash, cs) + if err != nil { + return fmt.Errorf("can't sign manifest group: %v", err) + } + + invokeHash := management.Hash + if method == morphUtil.UpdateMethodName { + invokeHash = ctrHash + } + + args, err := morphUtil.GetContractDeployData(c, ctrName, keysParam, morphUtil.UpdateMethodName) + if err != nil { + return fmt.Errorf("%s: getting update params: %v", ctrName, err) + } + params := morphUtil.GetContractDeployParameters(cs, args) + res, err := c.CommitteeAct.MakeCall(invokeHash, method, params...) + if err != nil { + if method != morphUtil.UpdateMethodName || !strings.Contains(err.Error(), common.ErrAlreadyUpdated) { + return fmt.Errorf("deploy contract: %w", err) + } + c.Command.Printf("%s contract is already updated.\n", ctrName) + continue + } + + w.WriteBytes(res.Script) + + if method == morphUtil.DeployMethodName { + // same actions are done in InitializeContext.setNNS, can be unified + domain := ctrName + ".frostfs" + script, ok, err := c.NNSRegisterDomainScript(nnsHash, cs.Hash, domain) + if err != nil { + return err + } + if !ok { + w.WriteBytes(script) + emit.AppCall(w.BinWriter, nnsHash, "deleteRecords", callflag.All, domain, int64(nns.TXT)) + emit.AppCall(w.BinWriter, nnsHash, "addRecord", callflag.All, + domain, int64(nns.TXT), cs.Hash.StringLE()) + emit.AppCall(w.BinWriter, nnsHash, "addRecord", callflag.All, + domain, int64(nns.TXT), address.Uint160ToString(cs.Hash)) + } + c.Command.Printf("NNS: Set %s -> %s\n", domain, cs.Hash.StringLE()) + } + } + return nil +} diff --git a/cmd/frostfs-adm/internal/modules/morph/deploy.go b/cmd/frostfs-adm/internal/modules/morph/deploy.go index a312b34cd..b5b2794ca 100644 --- a/cmd/frostfs-adm/internal/modules/morph/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/deploy.go @@ -83,7 +83,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { } callHash := management.Hash - method := deployMethodName + method := util.DeployMethodName zone, _ := cmd.Flags().GetString(util.CustomZoneFlag) domain := ctrName + "." + zone isUpdate, _ := cmd.Flags().GetBool(updateFlag) @@ -93,7 +93,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't fetch contract hash from NNS: %w", err) } callHash = cs.Hash - method = updateMethodName + method = util.UpdateMethodName } else { cs.Hash = state.CreateContractHash( c.CommitteeAcc.Contract.ScriptHash(), diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 76bbc7148..d40b09da3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -28,7 +28,6 @@ const ( groupNameFlag = "group-name" groupIDFlag = "group-id" - frostfsIDAdminConfigKey = "frostfsid.admin" rootNamespacePlaceholder = "" ) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util.go index 99c04f57d..541a459c1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util.go @@ -10,32 +10,8 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/cobra" - "github.com/spf13/viper" ) -func GetFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) { - admin := v.GetString(frostfsIDAdminConfigKey) - if admin == "" { - return util.Uint160{}, false, nil - } - - h, err := address.StringToUint160(admin) - if err == nil { - return h, true, nil - } - - h, err = util.Uint160DecodeStringLE(admin) - if err == nil { - return h, true, nil - } - - pk, err := keys.NewPublicKeyFromString(admin) - if err == nil { - return pk.GetScriptHash(), true, nil - } - return util.Uint160{}, true, fmt.Errorf("frostfsid: admin is invalid: '%s'", admin) -} - func getFrostfsIDSubjectKey(cmd *cobra.Command) *keys.PublicKey { subjKeyHex, _ := cmd.Flags().GetString(subjectKeyFlag) subjKey, err := keys.NewPublicKeyFromString(subjKeyHex) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go index 119e10d15..fa6d07bf2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/ape" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -30,7 +31,7 @@ func TestFrostfsIDConfig(t *testing.T) { v := viper.New() v.Set("frostfsid.admin", fmts[i]) - actual, found, err := GetFrostfsIDAdmin(v) + actual, found, err := util.GetFrostfsIDAdmin(v) require.NoError(t, err) require.True(t, found) require.Equal(t, pks[i].GetScriptHash(), actual) @@ -40,14 +41,14 @@ func TestFrostfsIDConfig(t *testing.T) { v := viper.New() v.Set("frostfsid.admin", "abc") - _, found, err := GetFrostfsIDAdmin(v) + _, found, err := util.GetFrostfsIDAdmin(v) require.Error(t, err) require.True(t, found) }) t.Run("missing key", func(t *testing.T) { v := viper.New() - _, found, err := GetFrostfsIDAdmin(v) + _, found, err := util.GetFrostfsIDAdmin(v) require.NoError(t, err) require.False(t, found) }) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize.go index d4219294c..30f7248cd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize.go @@ -28,7 +28,7 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { // 3. Deploy NNS contract. cmd.Println("Stage 3: deploy NNS contract.") - if err := deployNNS(initCtx, deployMethodName); err != nil { + if err := morphUtil.DeployNNS(initCtx, morphUtil.DeployMethodName); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go index 3a78f6042..4d5487419 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go @@ -1,252 +1,14 @@ package morph import ( - "encoding/hex" - "errors" "fmt" - "strings" - "git.frostfs.info/TrueCloudLab/frostfs-contract/common" - "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" - morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/encoding/address" - io2 "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" - "github.com/nspcc-dev/neo-go/pkg/smartcontract" - "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" - "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm/emit" - "github.com/nspcc-dev/neo-go/pkg/vm/opcode" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/spf13/viper" ) -var netmapConfigKeys = []string{ - netmap.EpochDurationConfig, - netmap.MaxObjectSizeConfig, - netmap.ContainerFeeConfig, - netmap.ContainerAliasFeeConfig, - netmap.IrCandidateFeeConfig, - netmap.WithdrawFeeConfig, - netmap.HomomorphicHashingDisabledKey, - netmap.MaintenanceModeAllowedConfig, -} - -const ( - updateMethodName = "update" - deployMethodName = "deploy" -) - -func deployNNS(c *morphUtil.InitializeContext, method string) error { - cs := c.GetContract(morphUtil.NNSContract) - h := cs.Hash - - nnsCs, err := c.NNSContractState() - if err != nil { - return err - } - if nnsCs != nil { - if nnsCs.NEF.Checksum == cs.NEF.Checksum { - if method == deployMethodName { - c.Command.Println("NNS contract is already deployed.") - } else { - c.Command.Println("NNS contract is already updated.") - } - return nil - } - h = nnsCs.Hash - } - - err = addManifestGroup(c.ContractWallet, h, cs) - if err != nil { - return fmt.Errorf("can't sign manifest group: %v", err) - } - - params := getContractDeployParameters(cs, nil) - - invokeHash := management.Hash - if method == updateMethodName { - invokeHash = nnsCs.Hash - } - - tx, err := c.CommitteeAct.MakeCall(invokeHash, method, params...) - if err != nil { - return fmt.Errorf("failed to create deploy tx for %s: %w", morphUtil.NNSContract, err) - } - - if err := c.MultiSignAndSend(tx, morphUtil.CommitteeAccountName); err != nil { - return fmt.Errorf("can't send deploy transaction: %w", err) - } - - return c.AwaitTx() -} - -func updateContractsInternal(c *morphUtil.InitializeContext) error { - alphaCs := c.GetContract(morphUtil.AlphabetContract) - - nnsCs, err := c.NNSContractState() - if err != nil { - return err - } - nnsHash := nnsCs.Hash - - w := io2.NewBufBinWriter() - - // Update script size for a single-node committee is close to the maximum allowed size of 65535. - // Because of this we want to reuse alphabet contract NEF and manifest for different updates. - // The generated script is as following. - // 1. Initialize static slot for alphabet NEF. - // 2. Store NEF into the static slot. - // 3. Push parameters for each alphabet contract on stack. - // 4. Add contract group to the manifest. - // 5. For each alphabet contract, invoke `update` using parameters on stack and - // NEF from step 2 and manifest from step 4. - emit.Instruction(w.BinWriter, opcode.INITSSLOT, []byte{1}) - emit.Bytes(w.BinWriter, alphaCs.RawNEF) - emit.Opcodes(w.BinWriter, opcode.STSFLD0) - - keysParam, err := deployAlphabetAccounts(c, nnsHash, w, alphaCs) - if err != nil { - return err - } - - w.Reset() - - if err = deployOrUpdateContracts(c, w, nnsHash, keysParam); err != nil { - return err - } - - groupKey := c.ContractWallet.Accounts[0].PrivateKey().PublicKey() - _, _, err = c.EmitUpdateNNSGroupScript(w, nnsHash, groupKey) - if err != nil { - return err - } - c.Command.Printf("NNS: Set %s -> %s\n", morphClient.NNSGroupKeyName, hex.EncodeToString(groupKey.Bytes())) - - emit.Opcodes(w.BinWriter, opcode.LDSFLD0) - emit.Int(w.BinWriter, 1) - emit.Opcodes(w.BinWriter, opcode.PACK) - emit.AppCallNoArgs(w.BinWriter, nnsHash, "setPrice", callflag.All) - - if err := c.SendCommitteeTx(w.Bytes(), false); err != nil { - return err - } - return c.AwaitTx() -} - -func deployOrUpdateContracts(c *morphUtil.InitializeContext, w *io2.BufBinWriter, nnsHash util.Uint160, keysParam []any) error { - emit.Instruction(w.BinWriter, opcode.INITSSLOT, []byte{1}) - emit.AppCall(w.BinWriter, nnsHash, "getPrice", callflag.All) - emit.Opcodes(w.BinWriter, opcode.STSFLD0) - emit.AppCall(w.BinWriter, nnsHash, "setPrice", callflag.All, 1) - - for _, ctrName := range morphUtil.ContractList { - cs := c.GetContract(ctrName) - - method := updateMethodName - ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, morphUtil.DomainOf(ctrName)) - if err != nil { - if errors.Is(err, errMissingNNSRecord) { - // if contract not found we deploy it instead of update - method = deployMethodName - } else { - return fmt.Errorf("can't resolve hash for contract update: %w", err) - } - } - - err = addManifestGroup(c.ContractWallet, ctrHash, cs) - if err != nil { - return fmt.Errorf("can't sign manifest group: %v", err) - } - - invokeHash := management.Hash - if method == updateMethodName { - invokeHash = ctrHash - } - - args, err := getContractDeployData(c, ctrName, keysParam, updateMethodName) - if err != nil { - return fmt.Errorf("%s: getting update params: %v", ctrName, err) - } - params := getContractDeployParameters(cs, args) - res, err := c.CommitteeAct.MakeCall(invokeHash, method, params...) - if err != nil { - if method != updateMethodName || !strings.Contains(err.Error(), common.ErrAlreadyUpdated) { - return fmt.Errorf("deploy contract: %w", err) - } - c.Command.Printf("%s contract is already updated.\n", ctrName) - continue - } - - w.WriteBytes(res.Script) - - if method == deployMethodName { - // same actions are done in InitializeContext.setNNS, can be unified - domain := ctrName + ".frostfs" - script, ok, err := c.NNSRegisterDomainScript(nnsHash, cs.Hash, domain) - if err != nil { - return err - } - if !ok { - w.WriteBytes(script) - emit.AppCall(w.BinWriter, nnsHash, "deleteRecords", callflag.All, domain, int64(nns.TXT)) - emit.AppCall(w.BinWriter, nnsHash, "addRecord", callflag.All, - domain, int64(nns.TXT), cs.Hash.StringLE()) - emit.AppCall(w.BinWriter, nnsHash, "addRecord", callflag.All, - domain, int64(nns.TXT), address.Uint160ToString(cs.Hash)) - } - c.Command.Printf("NNS: Set %s -> %s\n", domain, cs.Hash.StringLE()) - } - } - return nil -} - -func deployAlphabetAccounts(c *morphUtil.InitializeContext, nnsHash util.Uint160, w *io2.BufBinWriter, alphaCs *morphUtil.ContractState) ([]any, error) { - var keysParam []any - - baseGroups := alphaCs.Manifest.Groups - - // alphabet contracts should be deployed by individual nodes to get different hashes. - for i, acc := range c.Accounts { - ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, morphUtil.GetAlphabetNNSDomain(i)) - if err != nil { - return nil, fmt.Errorf("can't resolve hash for contract update: %w", err) - } - - keysParam = append(keysParam, acc.PrivateKey().PublicKey().Bytes()) - - params := c.GetAlphabetDeployItems(i, len(c.Wallets)) - emit.Array(w.BinWriter, params...) - - alphaCs.Manifest.Groups = baseGroups - err = addManifestGroup(c.ContractWallet, ctrHash, alphaCs) - if err != nil { - return nil, fmt.Errorf("can't sign manifest group: %v", err) - } - - emit.Bytes(w.BinWriter, alphaCs.RawManifest) - emit.Opcodes(w.BinWriter, opcode.LDSFLD0) - emit.Int(w.BinWriter, 3) - emit.Opcodes(w.BinWriter, opcode.PACK) - emit.AppCallNoArgs(w.BinWriter, ctrHash, updateMethodName, callflag.All) - } - if err := c.SendCommitteeTx(w.Bytes(), false); err != nil { - if !strings.Contains(err.Error(), common.ErrAlreadyUpdated) { - return nil, err - } - c.Command.Println("Alphabet contracts are already updated.") - } - - return keysParam, nil -} - func deployContracts(c *morphUtil.InitializeContext) error { alphaCs := c.GetContract(morphUtil.AlphabetContract) @@ -263,20 +25,20 @@ func deployContracts(c *morphUtil.InitializeContext) error { } alphaCs.Manifest.Groups = baseGroups - err := addManifestGroup(c.ContractWallet, ctrHash, alphaCs) + err := morphUtil.AddManifestGroup(c.ContractWallet, ctrHash, alphaCs) if err != nil { return fmt.Errorf("can't sign manifest group: %v", err) } keysParam = append(keysParam, acc.PrivateKey().PublicKey().Bytes()) - params := getContractDeployParameters(alphaCs, c.GetAlphabetDeployItems(i, len(c.Wallets))) + params := morphUtil.GetContractDeployParameters(alphaCs, c.GetAlphabetDeployItems(i, len(c.Wallets))) act, err := actor.NewSimple(c.Client, acc) if err != nil { return fmt.Errorf("could not create actor: %w", err) } - txHash, vub, err := act.SendCall(management.Hash, deployMethodName, params...) + txHash, vub, err := act.SendCall(management.Hash, morphUtil.DeployMethodName, params...) if err != nil { return fmt.Errorf("can't deploy alphabet #%d contract: %w", i, err) } @@ -293,17 +55,17 @@ func deployContracts(c *morphUtil.InitializeContext) error { continue } - err := addManifestGroup(c.ContractWallet, ctrHash, cs) + err := morphUtil.AddManifestGroup(c.ContractWallet, ctrHash, cs) if err != nil { return fmt.Errorf("can't sign manifest group: %v", err) } - args, err := getContractDeployData(c, ctrName, keysParam, deployMethodName) + args, err := morphUtil.GetContractDeployData(c, ctrName, keysParam, morphUtil.DeployMethodName) if err != nil { return fmt.Errorf("%s: getting deploy params: %v", ctrName, err) } - params := getContractDeployParameters(cs, args) - res, err := c.CommitteeAct.MakeCall(management.Hash, deployMethodName, params...) + params := morphUtil.GetContractDeployParameters(cs, args) + res, err := c.CommitteeAct.MakeCall(management.Hash, morphUtil.DeployMethodName, params...) if err != nil { return fmt.Errorf("can't deploy %s contract: %w", ctrName, err) } @@ -315,152 +77,3 @@ func deployContracts(c *morphUtil.InitializeContext) error { return c.AwaitTx() } - -func getContractDeployParameters(cs *morphUtil.ContractState, deployData []any) []any { - return []any{cs.RawNEF, cs.RawManifest, deployData} -} - -func getContractDeployData(c *morphUtil.InitializeContext, ctrName string, keysParam []any, method string) ([]any, error) { - items := make([]any, 0, 6) - - switch ctrName { - case morphUtil.FrostfsContract: - items = append(items, - c.Contracts[morphUtil.ProcessingContract].Hash, - keysParam, - smartcontract.Parameter{}) - case morphUtil.ProcessingContract: - items = append(items, c.Contracts[morphUtil.FrostfsContract].Hash) - return items[1:], nil // no notary info - case morphUtil.BalanceContract: - items = append(items, - c.Contracts[morphUtil.NetmapContract].Hash, - c.Contracts[morphUtil.ContainerContract].Hash) - case morphUtil.ContainerContract: - // In case if NNS is updated multiple times, we can't calculate - // it's actual hash based on local data, thus query chain. - r := management.NewReader(c.ReadOnlyInvoker) - nnsCs, err := r.GetContractByID(1) - if err != nil { - return nil, fmt.Errorf("get nns contract: %w", err) - } - items = append(items, - c.Contracts[morphUtil.NetmapContract].Hash, - c.Contracts[morphUtil.BalanceContract].Hash, - c.Contracts[morphUtil.FrostfsIDContract].Hash, - nnsCs.Hash, - "container") - case morphUtil.FrostfsIDContract: - var ( - h util.Uint160 - found bool - err error - ) - if method == updateMethodName { - h, found, err = getFrostfsIDAdminFromContract(c.ReadOnlyInvoker) - } - if method != updateMethodName || err == nil && !found { - h, found, err = frostfsid.GetFrostfsIDAdmin(viper.GetViper()) - } - if err != nil { - return nil, err - } - - if found { - items = append(items, h) - } else { - items = append(items, c.Contracts[morphUtil.ProxyContract].Hash) - } - case morphUtil.NetmapContract: - md := morphUtil.GetDefaultNetmapContractConfigMap() - if method == updateMethodName { - if err := mergeNetmapConfig(c.ReadOnlyInvoker, md); err != nil { - return nil, err - } - } - - var configParam []any - for k, v := range md { - configParam = append(configParam, k, v) - } - - items = append(items, - c.Contracts[morphUtil.BalanceContract].Hash, - c.Contracts[morphUtil.ContainerContract].Hash, - keysParam, - configParam) - case morphUtil.ProxyContract: - items = nil - case morphUtil.PolicyContract: - items = append(items, c.Contracts[morphUtil.ProxyContract].Hash) - default: - panic(fmt.Sprintf("invalid contract name: %s", ctrName)) - } - return items, nil -} - -func getFrostfsIDAdminFromContract(roInvoker *invoker.Invoker) (util.Uint160, bool, error) { - r := management.NewReader(roInvoker) - cs, err := r.GetContractByID(1) - if err != nil { - return util.Uint160{}, false, fmt.Errorf("get nns contract: %w", err) - } - fidHash, err := morphUtil.NNSResolveHash(roInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.FrostfsIDContract)) - if err != nil { - return util.Uint160{}, false, fmt.Errorf("resolve frostfsid contract hash: %w", err) - } - item, err := unwrap.Item(roInvoker.Call(fidHash, "getAdmin")) - if err != nil { - return util.Uint160{}, false, fmt.Errorf("getAdmin: %w", err) - } - if _, ok := item.(stackitem.Null); ok { - return util.Uint160{}, false, nil - } - - bs, err := item.TryBytes() - if err != nil { - return util.Uint160{}, true, fmt.Errorf("getAdmin: decode result: %w", err) - } - h, err := util.Uint160DecodeBytesBE(bs) - if err != nil { - return util.Uint160{}, true, fmt.Errorf("getAdmin: decode result: %w", err) - } - return h, true, nil -} - -func getNetConfigFromNetmapContract(roInvoker *invoker.Invoker) ([]stackitem.Item, error) { - r := management.NewReader(roInvoker) - cs, err := r.GetContractByID(1) - if err != nil { - return nil, fmt.Errorf("get nns contract: %w", err) - } - nmHash, err := morphUtil.NNSResolveHash(roInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.NetmapContract)) - if err != nil { - return nil, fmt.Errorf("can't get netmap contract hash: %w", err) - } - arr, err := unwrap.Array(roInvoker.Call(nmHash, "listConfig")) - if err != nil { - return nil, fmt.Errorf("can't fetch list of network config keys from the netmap contract") - } - return arr, err -} - -func mergeNetmapConfig(roInvoker *invoker.Invoker, md map[string]any) error { - arr, err := getNetConfigFromNetmapContract(roInvoker) - if err != nil { - return err - } - m, err := morphUtil.ParseConfigFromNetmapContract(arr) - if err != nil { - return err - } - for k, v := range m { - for _, key := range netmapConfigKeys { - if k == key { - md[k] = v - break - } - } - } - return nil -} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go index 53435e67b..f943ac4a1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go @@ -2,7 +2,6 @@ package morph import ( "encoding/hex" - "errors" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" @@ -132,5 +131,3 @@ func nnsRegisterDomain(c *morphUtil.InitializeContext, nnsHash, expectedHash uti domain, int64(nns.TXT), address.Uint160ToString(expectedHash)) return c.SendCommitteeTx(w.Bytes(), true) } - -var errMissingNNSRecord = errors.New("missing NNS record") diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 66dd77556..3af6f78a5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -98,16 +98,6 @@ var ( return refillGas(cmd, util.RefillGasAmountFlag, false) }, } - - updateContractsCmd = &cobra.Command{ - Use: "update-contracts", - Short: "Update FrostFS contracts", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - }, - RunE: updateContracts, - } ) func init() { @@ -123,7 +113,7 @@ func init() { RootCmd.AddCommand(config.SetCmd) RootCmd.AddCommand(config.DumpCmd) RootCmd.AddCommand(balance.DumpCmd) - initUpdateContractsCmd() + RootCmd.AddCommand(contract.UpdateCmd) RootCmd.AddCommand(container.ListCmd) RootCmd.AddCommand(container.RestoreCmd) RootCmd.AddCommand(container.DumpCmd) @@ -148,15 +138,6 @@ func initRefillGasCmd() { refillGasCmd.MarkFlagsMutuallyExclusive(walletAddressFlag, util.StorageWalletFlag) } -func initUpdateContractsCmd() { - RootCmd.AddCommand(updateContractsCmd) - updateContractsCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - updateContractsCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - updateContractsCmd.Flags().String(util.ContractsInitFlag, "", util.ContractsInitFlagDesc) - updateContractsCmd.Flags().String(util.ContractsURLFlag, "", util.ContractsURLFlagDesc) - updateContractsCmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) -} - func initGenerateStorageCmd() { RootCmd.AddCommand(generateStorageCmd) generateStorageCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) diff --git a/cmd/frostfs-adm/internal/modules/morph/update.go b/cmd/frostfs-adm/internal/modules/morph/update.go deleted file mode 100644 index 2d5b24712..000000000 --- a/cmd/frostfs-adm/internal/modules/morph/update.go +++ /dev/null @@ -1,22 +0,0 @@ -package morph - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -func updateContracts(cmd *cobra.Command, _ []string) error { - wCtx, err := util.NewInitializeContext(cmd, viper.GetViper()) - if err != nil { - return fmt.Errorf("initialization error: %w", err) - } - - if err := deployNNS(wCtx, updateMethodName); err != nil { - return err - } - - return updateContractsInternal(wCtx) -} diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/util/const.go index cdf1cd443..a37afc3da 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/const.go @@ -53,6 +53,9 @@ const ( FrostfsOpsEmail = "ops@frostfs.info" DefaultExpirationTime = 10 * 365 * 24 * time.Hour / time.Second + + DeployMethodName = "deploy" + UpdateMethodName = "update" ) var ( diff --git a/cmd/frostfs-adm/internal/modules/morph/util/contract.go b/cmd/frostfs-adm/internal/modules/morph/util/contract.go new file mode 100644 index 000000000..8faa49d1c --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/util/contract.go @@ -0,0 +1,169 @@ +package util + +import ( + "fmt" + + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" + "github.com/nspcc-dev/neo-go/pkg/smartcontract" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "github.com/spf13/viper" +) + +func getFrostfsIDAdminFromContract(roInvoker *invoker.Invoker) (util.Uint160, bool, error) { + r := management.NewReader(roInvoker) + cs, err := r.GetContractByID(1) + if err != nil { + return util.Uint160{}, false, fmt.Errorf("get nns contract: %w", err) + } + fidHash, err := NNSResolveHash(roInvoker, cs.Hash, DomainOf(FrostfsIDContract)) + if err != nil { + return util.Uint160{}, false, fmt.Errorf("resolve frostfsid contract hash: %w", err) + } + item, err := unwrap.Item(roInvoker.Call(fidHash, "getAdmin")) + if err != nil { + return util.Uint160{}, false, fmt.Errorf("getAdmin: %w", err) + } + if _, ok := item.(stackitem.Null); ok { + return util.Uint160{}, false, nil + } + + bs, err := item.TryBytes() + if err != nil { + return util.Uint160{}, true, fmt.Errorf("getAdmin: decode result: %w", err) + } + h, err := util.Uint160DecodeBytesBE(bs) + if err != nil { + return util.Uint160{}, true, fmt.Errorf("getAdmin: decode result: %w", err) + } + return h, true, nil +} + +func GetContractDeployData(c *InitializeContext, ctrName string, keysParam []any, method string) ([]any, error) { + items := make([]any, 0, 6) + + switch ctrName { + case FrostfsContract: + items = append(items, + c.Contracts[ProcessingContract].Hash, + keysParam, + smartcontract.Parameter{}) + case ProcessingContract: + items = append(items, c.Contracts[FrostfsContract].Hash) + return items[1:], nil // no notary info + case BalanceContract: + items = append(items, + c.Contracts[NetmapContract].Hash, + c.Contracts[ContainerContract].Hash) + case ContainerContract: + // In case if NNS is updated multiple times, we can't calculate + // it's actual hash based on local data, thus query chain. + r := management.NewReader(c.ReadOnlyInvoker) + nnsCs, err := r.GetContractByID(1) + if err != nil { + return nil, fmt.Errorf("get nns contract: %w", err) + } + items = append(items, + c.Contracts[NetmapContract].Hash, + c.Contracts[BalanceContract].Hash, + c.Contracts[FrostfsIDContract].Hash, + nnsCs.Hash, + "container") + case FrostfsIDContract: + var ( + h util.Uint160 + found bool + err error + ) + if method == UpdateMethodName { + h, found, err = getFrostfsIDAdminFromContract(c.ReadOnlyInvoker) + } + if method != UpdateMethodName || err == nil && !found { + h, found, err = GetFrostfsIDAdmin(viper.GetViper()) + } + if err != nil { + return nil, err + } + + if found { + items = append(items, h) + } else { + items = append(items, c.Contracts[ProxyContract].Hash) + } + case NetmapContract: + md := GetDefaultNetmapContractConfigMap() + if method == UpdateMethodName { + if err := MergeNetmapConfig(c.ReadOnlyInvoker, md); err != nil { + return nil, err + } + } + + var configParam []any + for k, v := range md { + configParam = append(configParam, k, v) + } + + items = append(items, + c.Contracts[BalanceContract].Hash, + c.Contracts[ContainerContract].Hash, + keysParam, + configParam) + case ProxyContract: + items = nil + case PolicyContract: + items = append(items, c.Contracts[ProxyContract].Hash) + default: + panic(fmt.Sprintf("invalid contract name: %s", ctrName)) + } + return items, nil +} + +func GetContractDeployParameters(cs *ContractState, deployData []any) []any { + return []any{cs.RawNEF, cs.RawManifest, deployData} +} + +func DeployNNS(c *InitializeContext, method string) error { + cs := c.GetContract(NNSContract) + h := cs.Hash + + nnsCs, err := c.NNSContractState() + if err != nil { + return err + } + if nnsCs != nil { + if nnsCs.NEF.Checksum == cs.NEF.Checksum { + if method == DeployMethodName { + c.Command.Println("NNS contract is already deployed.") + } else { + c.Command.Println("NNS contract is already updated.") + } + return nil + } + h = nnsCs.Hash + } + + err = AddManifestGroup(c.ContractWallet, h, cs) + if err != nil { + return fmt.Errorf("can't sign manifest group: %v", err) + } + + params := GetContractDeployParameters(cs, nil) + + invokeHash := management.Hash + if method == UpdateMethodName { + invokeHash = nnsCs.Hash + } + + tx, err := c.CommitteeAct.MakeCall(invokeHash, method, params...) + if err != nil { + return fmt.Errorf("failed to create deploy tx for %s: %w", NNSContract, err) + } + + if err := c.MultiSignAndSend(tx, CommitteeAccountName); err != nil { + return fmt.Errorf("can't send deploy transaction: %w", err) + } + + return c.AwaitTx() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/util/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/util/frostfsid.go new file mode 100644 index 000000000..7aea1c492 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/util/frostfsid.go @@ -0,0 +1,35 @@ +package util + +import ( + "fmt" + + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/spf13/viper" +) + +const frostfsIDAdminConfigKey = "frostfsid.admin" + +func GetFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) { + admin := v.GetString(frostfsIDAdminConfigKey) + if admin == "" { + return util.Uint160{}, false, nil + } + + h, err := address.StringToUint160(admin) + if err == nil { + return h, true, nil + } + + h, err = util.Uint160DecodeStringLE(admin) + if err == nil { + return h, true, nil + } + + pk, err := keys.NewPublicKeyFromString(admin) + if err == nil { + return pk.GetScriptHash(), true, nil + } + return util.Uint160{}, true, fmt.Errorf("frostfsid: admin is invalid: '%s'", admin) +} diff --git a/cmd/frostfs-adm/internal/modules/morph/group.go b/cmd/frostfs-adm/internal/modules/morph/util/group.go similarity index 77% rename from cmd/frostfs-adm/internal/modules/morph/group.go rename to cmd/frostfs-adm/internal/modules/morph/util/group.go index 655b8b2c3..a0306f3bc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/group.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/group.go @@ -1,15 +1,14 @@ -package morph +package util import ( "encoding/json" - util2 "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" ) -func addManifestGroup(cw *wallet.Wallet, h util.Uint160, cs *util2.ContractState) error { +func AddManifestGroup(cw *wallet.Wallet, h util.Uint160, cs *ContractState) error { priv := cw.Accounts[0].PrivateKey() pub := priv.PublicKey() diff --git a/cmd/frostfs-adm/internal/modules/morph/util/netmap.go b/cmd/frostfs-adm/internal/modules/morph/util/netmap.go index 34c527d0b..bfe8ea165 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/netmap.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/netmap.go @@ -6,6 +6,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/util" @@ -14,6 +16,17 @@ import ( "github.com/spf13/viper" ) +var NetmapConfigKeys = []string{ + netmap.EpochDurationConfig, + netmap.MaxObjectSizeConfig, + netmap.ContainerFeeConfig, + netmap.ContainerAliasFeeConfig, + netmap.IrCandidateFeeConfig, + netmap.WithdrawFeeConfig, + netmap.HomomorphicHashingDisabledKey, + netmap.MaintenanceModeAllowedConfig, +} + func GetDefaultNetmapContractConfigMap() map[string]any { m := make(map[string]any) m[netmap.EpochDurationConfig] = viper.GetInt64(EpochDurationInitFlag) @@ -67,3 +80,40 @@ func EmitNewEpochCall(bw *io.BufBinWriter, wCtx *InitializeContext, nmHash util. emit.AppCall(bw.BinWriter, nmHash, "newEpoch", callflag.All, newEpoch) return bw.Err } + +func GetNetConfigFromNetmapContract(roInvoker *invoker.Invoker) ([]stackitem.Item, error) { + r := management.NewReader(roInvoker) + cs, err := r.GetContractByID(1) + if err != nil { + return nil, fmt.Errorf("get nns contract: %w", err) + } + nmHash, err := NNSResolveHash(roInvoker, cs.Hash, DomainOf(NetmapContract)) + if err != nil { + return nil, fmt.Errorf("can't get netmap contract hash: %w", err) + } + arr, err := unwrap.Array(roInvoker.Call(nmHash, "listConfig")) + if err != nil { + return nil, fmt.Errorf("can't fetch list of network config keys from the netmap contract") + } + return arr, err +} + +func MergeNetmapConfig(roInvoker *invoker.Invoker, md map[string]any) error { + arr, err := GetNetConfigFromNetmapContract(roInvoker) + if err != nil { + return err + } + m, err := ParseConfigFromNetmapContract(arr) + if err != nil { + return err + } + for k, v := range m { + for _, key := range NetmapConfigKeys { + if k == key { + md[k] = v + break + } + } + } + return nil +} From b8cf0a6b88faff7dc36368a5edbed638f54c09ec Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 2 Feb 2024 09:54:47 +0300 Subject: [PATCH 0900/1943] [#932] adm: Move `deploy` to package `contract` Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/{ => contract}/deploy.go | 10 +++++----- cmd/frostfs-adm/internal/modules/morph/root.go | 6 +----- 2 files changed, 6 insertions(+), 10 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => contract}/deploy.go (97%) diff --git a/cmd/frostfs-adm/internal/modules/morph/deploy.go b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go similarity index 97% rename from cmd/frostfs-adm/internal/modules/morph/deploy.go rename to cmd/frostfs-adm/internal/modules/morph/contract/deploy.go index b5b2794ca..35007a273 100644 --- a/cmd/frostfs-adm/internal/modules/morph/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go @@ -1,4 +1,4 @@ -package morph +package contract import ( "encoding/json" @@ -26,7 +26,7 @@ const ( updateFlag = "update" ) -var deployCmd = &cobra.Command{ +var DeployCmd = &cobra.Command{ Use: "deploy", Short: "Deploy additional smart-contracts", Long: `Deploy additional smart-contract which are not related to core. @@ -44,14 +44,14 @@ NNS name is taken by stripping '_contract.nef' from the NEF file (similar to fro } func init() { - ff := deployCmd.Flags() + ff := DeployCmd.Flags() ff.String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - _ = deployCmd.MarkFlagFilename(util.AlphabetWalletsFlag) + _ = DeployCmd.MarkFlagFilename(util.AlphabetWalletsFlag) ff.StringP(util.EndpointFlag, "r", "", util.EndpointFlagDesc) ff.String(contractPathFlag, "", "Path to the contract directory") - _ = deployCmd.MarkFlagFilename(contractPathFlag) + _ = DeployCmd.MarkFlagFilename(contractPathFlag) ff.Bool(updateFlag, false, "Update an existing contract") ff.String(util.CustomZoneFlag, "frostfs", "Custom zone for NNS") diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 3af6f78a5..44ff543d9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -103,7 +103,7 @@ var ( func init() { initGenerateAlphabetCmd() initInitCmd() - initDeployCmd() + RootCmd.AddCommand(contract.DeployCmd) initGenerateStorageCmd() RootCmd.AddCommand(netmap.ForceNewEpoch) RootCmd.AddCommand(node.RemoveCmd) @@ -169,7 +169,3 @@ func initGenerateAlphabetCmd() { generateAlphabetCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) generateAlphabetCmd.Flags().Uint(alphabetSizeFlag, 7, "Amount of alphabet wallets to generate") } - -func initDeployCmd() { - RootCmd.AddCommand(deployCmd) -} From 86b2515744671f1ecf665c97c5a2765d50a9f5a7 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 2 Feb 2024 10:15:55 +0300 Subject: [PATCH 0901/1943] [#932] adm: Move `generate.go` to package `generate` Signed-off-by: Anton Nikiforov --- .../modules/morph/{ => generate}/generate.go | 6 +- .../morph/{ => generate}/generate_test.go | 30 ++++---- .../internal/modules/morph/generate/root.go | 76 +++++++++++++++++++ .../internal/modules/morph/initialize_test.go | 7 +- .../internal/modules/morph/root.go | 74 +----------------- .../internal/modules/morph/util/const.go | 3 + 6 files changed, 104 insertions(+), 92 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => generate}/generate.go (97%) rename cmd/frostfs-adm/internal/modules/morph/{ => generate}/generate_test.go (77%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/generate/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go similarity index 97% rename from cmd/frostfs-adm/internal/modules/morph/generate.go rename to cmd/frostfs-adm/internal/modules/morph/generate/generate.go index 57c40ec9f..364ce2c6e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go @@ -1,4 +1,4 @@ -package morph +package generate import ( "errors" @@ -24,9 +24,9 @@ import ( "golang.org/x/sync/errgroup" ) -func generateAlphabetCreds(cmd *cobra.Command, _ []string) error { +func AlphabetCreds(cmd *cobra.Command, _ []string) error { // alphabet size is not part of the config - size, err := cmd.Flags().GetUint(alphabetSizeFlag) + size, err := cmd.Flags().GetUint(morphUtil.AlphabetSizeFlag) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go similarity index 77% rename from cmd/frostfs-adm/internal/modules/morph/generate_test.go rename to cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go index e9e942d74..49379ee26 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go @@ -1,4 +1,4 @@ -package morph +package generate import ( "bytes" @@ -21,57 +21,55 @@ import ( "golang.org/x/term" ) -const testContractPassword = "grouppass" - func TestGenerateAlphabet(t *testing.T) { const size = 4 walletDir := t.TempDir() buf := setupTestTerminal(t) - cmd := generateAlphabetCmd + cmd := GenerateAlphabetCmd v := viper.GetViper() t.Run("zero size", func(t *testing.T) { buf.Reset() v.Set(util.AlphabetWalletsFlag, walletDir) - require.NoError(t, cmd.Flags().Set(alphabetSizeFlag, "0")) + require.NoError(t, cmd.Flags().Set(util.AlphabetSizeFlag, "0")) buf.WriteString("pass\r") - require.Error(t, generateAlphabetCreds(cmd, nil)) + require.Error(t, AlphabetCreds(cmd, nil)) }) t.Run("no password provided", func(t *testing.T) { buf.Reset() v.Set(util.AlphabetWalletsFlag, walletDir) - require.NoError(t, cmd.Flags().Set(alphabetSizeFlag, "1")) - require.Error(t, generateAlphabetCreds(cmd, nil)) + require.NoError(t, cmd.Flags().Set(util.AlphabetSizeFlag, "1")) + require.Error(t, AlphabetCreds(cmd, nil)) }) t.Run("missing directory", func(t *testing.T) { buf.Reset() dir := filepath.Join(os.TempDir(), "notexist."+strconv.FormatUint(rand.Uint64(), 10)) v.Set(util.AlphabetWalletsFlag, dir) - require.NoError(t, cmd.Flags().Set(alphabetSizeFlag, "1")) + require.NoError(t, cmd.Flags().Set(util.AlphabetSizeFlag, "1")) buf.WriteString("pass\r") - require.Error(t, generateAlphabetCreds(cmd, nil)) + require.Error(t, AlphabetCreds(cmd, nil)) }) t.Run("no password for contract group wallet", func(t *testing.T) { buf.Reset() v.Set(util.AlphabetWalletsFlag, walletDir) - require.NoError(t, cmd.Flags().Set(alphabetSizeFlag, strconv.FormatUint(size, 10))) + require.NoError(t, cmd.Flags().Set(util.AlphabetSizeFlag, strconv.FormatUint(size, 10))) for i := uint64(0); i < size; i++ { buf.WriteString(strconv.FormatUint(i, 10) + "\r") } - require.Error(t, generateAlphabetCreds(cmd, nil)) + require.Error(t, AlphabetCreds(cmd, nil)) }) buf.Reset() v.Set(util.AlphabetWalletsFlag, walletDir) - require.NoError(t, generateAlphabetCmd.Flags().Set(alphabetSizeFlag, strconv.FormatUint(size, 10))) + require.NoError(t, GenerateAlphabetCmd.Flags().Set(util.AlphabetSizeFlag, strconv.FormatUint(size, 10))) for i := uint64(0); i < size; i++ { buf.WriteString(strconv.FormatUint(i, 10) + "\r") } - buf.WriteString(testContractPassword + "\r") - require.NoError(t, generateAlphabetCreds(generateAlphabetCmd, nil)) + buf.WriteString(util.TestContractPassword + "\r") + require.NoError(t, AlphabetCreds(GenerateAlphabetCmd, nil)) var wg sync.WaitGroup for i := uint64(0); i < size; i++ { @@ -105,7 +103,7 @@ func TestGenerateAlphabet(t *testing.T) { w, err := wallet.NewWalletFromFile(p) require.NoError(t, err, "contract wallet doesn't exist") require.Equal(t, 1, len(w.Accounts), "contract wallet must have 1 accout") - require.NoError(t, w.Accounts[0].Decrypt(testContractPassword, keys.NEP2ScryptParams())) + require.NoError(t, w.Accounts[0].Decrypt(util.TestContractPassword, keys.NEP2ScryptParams())) }) } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/root.go b/cmd/frostfs-adm/internal/modules/morph/generate/root.go new file mode 100644 index 000000000..e5f5b38cd --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/generate/root.go @@ -0,0 +1,76 @@ +package generate + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + storageWalletLabelFlag = "label" + storageGasCLIFlag = "initial-gas" + storageGasConfigFlag = "storage.initial_gas" + walletAddressFlag = "wallet-address" +) + +var ( + GenerateStorageCmd = &cobra.Command{ + Use: "generate-storage-wallet", + Short: "Generate storage node wallet for the morph network", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(storageGasConfigFlag, cmd.Flags().Lookup(storageGasCLIFlag)) + }, + RunE: generateStorageCreds, + } + RefillGasCmd = &cobra.Command{ + Use: "refill-gas", + Short: "Refill GAS of storage node's wallet in the morph network", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(util.RefillGasAmountFlag, cmd.Flags().Lookup(util.RefillGasAmountFlag)) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return refillGas(cmd, util.RefillGasAmountFlag, false) + }, + } + GenerateAlphabetCmd = &cobra.Command{ + Use: "generate-alphabet", + Short: "Generate alphabet wallets for consensus nodes of the morph network", + PreRun: func(cmd *cobra.Command, _ []string) { + // PreRun fixes https://github.com/spf13/viper/issues/233 + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + }, + RunE: AlphabetCreds, + } +) + +func initRefillGasCmd() { + RefillGasCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) + RefillGasCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + RefillGasCmd.Flags().String(util.StorageWalletFlag, "", "Path to storage node wallet") + RefillGasCmd.Flags().String(walletAddressFlag, "", "Address of wallet") + RefillGasCmd.Flags().String(util.RefillGasAmountFlag, "", "Additional amount of GAS to transfer") + RefillGasCmd.MarkFlagsMutuallyExclusive(walletAddressFlag, util.StorageWalletFlag) +} + +func initGenerateStorageCmd() { + GenerateStorageCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) + GenerateStorageCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + GenerateStorageCmd.Flags().String(util.StorageWalletFlag, "", "Path to new storage node wallet") + GenerateStorageCmd.Flags().String(storageGasCLIFlag, "", "Initial amount of GAS to transfer") + GenerateStorageCmd.Flags().StringP(storageWalletLabelFlag, "l", "", "Wallet label") +} + +func initGenerateAlphabetCmd() { + GenerateAlphabetCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) + GenerateAlphabetCmd.Flags().Uint(util.AlphabetSizeFlag, 7, "Amount of alphabet wallets to generate") +} + +func init() { + initRefillGasCmd() + initGenerateStorageCmd() + initGenerateAlphabetCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go index 8b0fddf3d..3315b5d33 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize_test.go @@ -10,6 +10,7 @@ import ( "time" cmdConfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/generate" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/node" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" @@ -100,12 +101,12 @@ func generateTestData(t *testing.T, dir string, size int) error { v.Set(util.AlphabetWalletsFlag, dir) sizeStr := strconv.FormatUint(uint64(size), 10) - if err := generateAlphabetCmd.Flags().Set(alphabetSizeFlag, sizeStr); err != nil { + if err := generate.GenerateAlphabetCmd.Flags().Set(util.AlphabetSizeFlag, sizeStr); err != nil { return err } setTestCredentials(v, size) - if err := generateAlphabetCreds(generateAlphabetCmd, nil); err != nil { + if err := generate.AlphabetCreds(generate.GenerateAlphabetCmd, nil); err != nil { return err } @@ -148,7 +149,7 @@ func setTestCredentials(v *viper.Viper, size int) { for i := 0; i < size; i++ { v.Set("credentials."+innerring.GlagoliticLetter(i).String(), strconv.FormatUint(uint64(i), 10)) } - v.Set("credentials.contract", testContractPassword) + v.Set("credentials.contract", util.TestContractPassword) } func TestNextPollInterval(t *testing.T) { diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 44ff543d9..e2a4517f3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/container" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/contract" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/generate" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/node" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/notary" @@ -18,12 +19,6 @@ import ( ) const ( - alphabetSizeFlag = "size" - - storageWalletLabelFlag = "label" - storageGasCLIFlag = "initial-gas" - storageGasConfigFlag = "storage.initial_gas" - maxObjectSizeCLIFlag = "max-object-size" epochDurationCLIFlag = "epoch-duration" @@ -36,8 +31,6 @@ const ( homomorphicHashDisabledCLIFlag = "homomorphic-disabled" withdrawFeeCLIFlag = "withdraw-fee" - - walletAddressFlag = "wallet-address" ) var ( @@ -47,16 +40,6 @@ var ( Short: "Section for morph network configuration commands", } - generateAlphabetCmd = &cobra.Command{ - Use: "generate-alphabet", - Short: "Generate alphabet wallets for consensus nodes of the morph network", - PreRun: func(cmd *cobra.Command, _ []string) { - // PreRun fixes https://github.com/spf13/viper/issues/233 - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - }, - RunE: generateAlphabetCreds, - } - initCmd = &cobra.Command{ Use: "init", Short: "Initialize side chain network with smart-contracts and network settings", @@ -74,37 +57,13 @@ var ( }, RunE: initializeSideChainCmd, } - - generateStorageCmd = &cobra.Command{ - Use: "generate-storage-wallet", - Short: "Generate storage node wallet for the morph network", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - _ = viper.BindPFlag(storageGasConfigFlag, cmd.Flags().Lookup(storageGasCLIFlag)) - }, - RunE: generateStorageCreds, - } - - refillGasCmd = &cobra.Command{ - Use: "refill-gas", - Short: "Refill GAS of storage node's wallet in the morph network", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - _ = viper.BindPFlag(util.RefillGasAmountFlag, cmd.Flags().Lookup(util.RefillGasAmountFlag)) - }, - RunE: func(cmd *cobra.Command, args []string) error { - return refillGas(cmd, util.RefillGasAmountFlag, false) - }, - } ) func init() { - initGenerateAlphabetCmd() + RootCmd.AddCommand(generate.RefillGasCmd) initInitCmd() RootCmd.AddCommand(contract.DeployCmd) - initGenerateStorageCmd() + RootCmd.AddCommand(generate.GenerateStorageCmd) RootCmd.AddCommand(netmap.ForceNewEpoch) RootCmd.AddCommand(node.RemoveCmd) RootCmd.AddCommand(policy.Set) @@ -117,7 +76,7 @@ func init() { RootCmd.AddCommand(container.ListCmd) RootCmd.AddCommand(container.RestoreCmd) RootCmd.AddCommand(container.DumpCmd) - initRefillGasCmd() + RootCmd.AddCommand(generate.GenerateAlphabetCmd) RootCmd.AddCommand(notary.DepositCmd) RootCmd.AddCommand(netmap.CandidatesCmd) @@ -128,25 +87,6 @@ func init() { RootCmd.AddCommand(frostfsid.Cmd) } -func initRefillGasCmd() { - RootCmd.AddCommand(refillGasCmd) - refillGasCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - refillGasCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - refillGasCmd.Flags().String(util.StorageWalletFlag, "", "Path to storage node wallet") - refillGasCmd.Flags().String(walletAddressFlag, "", "Address of wallet") - refillGasCmd.Flags().String(util.RefillGasAmountFlag, "", "Additional amount of GAS to transfer") - refillGasCmd.MarkFlagsMutuallyExclusive(walletAddressFlag, util.StorageWalletFlag) -} - -func initGenerateStorageCmd() { - RootCmd.AddCommand(generateStorageCmd) - generateStorageCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - generateStorageCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - generateStorageCmd.Flags().String(util.StorageWalletFlag, "", "Path to new storage node wallet") - generateStorageCmd.Flags().String(storageGasCLIFlag, "", "Initial amount of GAS to transfer") - generateStorageCmd.Flags().StringP(storageWalletLabelFlag, "l", "", "Wallet label") -} - func initInitCmd() { RootCmd.AddCommand(initCmd) initCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) @@ -163,9 +103,3 @@ func initInitCmd() { initCmd.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") initCmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) } - -func initGenerateAlphabetCmd() { - RootCmd.AddCommand(generateAlphabetCmd) - generateAlphabetCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - generateAlphabetCmd.Flags().Uint(alphabetSizeFlag, 7, "Amount of alphabet wallets to generate") -} diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/util/const.go index a37afc3da..1128adee9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/const.go @@ -32,6 +32,7 @@ const ( MaintenanceModeAllowedInitFlag = "network.maintenance_mode_allowed" HomomorphicHashDisabledInitFlag = "network.homomorphic_hash_disabled" CustomZoneFlag = "domain" + AlphabetSizeFlag = "size" SingleAccountName = "single" CommitteeAccountName = "committee" @@ -56,6 +57,8 @@ const ( DeployMethodName = "deploy" UpdateMethodName = "update" + + TestContractPassword = "grouppass" ) var ( From 63c34ea707b6aa3f0cc4ca7640a631bbac182009 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 2 Feb 2024 10:25:04 +0300 Subject: [PATCH 0902/1943] [#932] adm: Move command `init` to package `initialize` Signed-off-by: Anton Nikiforov --- .../morph/{ => initialize}/initialize.go | 2 +- .../{ => initialize}/initialize_deploy.go | 2 +- .../morph/{ => initialize}/initialize_nns.go | 2 +- .../{ => initialize}/initialize_register.go | 2 +- .../{ => initialize}/initialize_roles.go | 2 +- .../morph/{ => initialize}/initialize_test.go | 8 +-- .../{ => initialize}/initialize_transfer.go | 2 +- .../internal/modules/morph/initialize/root.go | 55 +++++++++++++++ .../internal/modules/morph/root.go | 67 ++----------------- 9 files changed, 72 insertions(+), 70 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{ => initialize}/initialize.go (98%) rename cmd/frostfs-adm/internal/modules/morph/{ => initialize}/initialize_deploy.go (99%) rename cmd/frostfs-adm/internal/modules/morph/{ => initialize}/initialize_nns.go (99%) rename cmd/frostfs-adm/internal/modules/morph/{ => initialize}/initialize_register.go (99%) rename cmd/frostfs-adm/internal/modules/morph/{ => initialize}/initialize_roles.go (98%) rename cmd/frostfs-adm/internal/modules/morph/{ => initialize}/initialize_test.go (96%) rename cmd/frostfs-adm/internal/modules/morph/{ => initialize}/initialize_transfer.go (99%) create mode 100644 cmd/frostfs-adm/internal/modules/morph/initialize/root.go diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go similarity index 98% rename from cmd/frostfs-adm/internal/modules/morph/initialize.go rename to cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go index 30f7248cd..d1b766019 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go @@ -1,4 +1,4 @@ -package morph +package initialize import ( "fmt" diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_deploy.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go rename to cmd/frostfs-adm/internal/modules/morph/initialize/initialize_deploy.go index 4d5487419..9374b9f7f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_deploy.go @@ -1,4 +1,4 @@ -package morph +package initialize import ( "fmt" diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/initialize_nns.go rename to cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go index f943ac4a1..a9df7127f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go @@ -1,4 +1,4 @@ -package morph +package initialize import ( "encoding/hex" diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/initialize_register.go rename to cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go index d6cbd2d56..cb33aa985 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go @@ -1,4 +1,4 @@ -package morph +package initialize import ( "errors" diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_roles.go similarity index 98% rename from cmd/frostfs-adm/internal/modules/morph/initialize_roles.go rename to cmd/frostfs-adm/internal/modules/morph/initialize/initialize_roles.go index 9e885d5f8..d7f17ccfb 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_roles.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_roles.go @@ -1,4 +1,4 @@ -package morph +package initialize import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go similarity index 96% rename from cmd/frostfs-adm/internal/modules/morph/initialize_test.go rename to cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go index 3315b5d33..d6b180609 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go @@ -1,4 +1,4 @@ -package morph +package initialize import ( "encoding/hex" @@ -63,16 +63,16 @@ func testInitialize(t *testing.T, committeeSize int) { v.Set(util.ProtoConfigPath, filepath.Join(testdataDir, protoFileName)) // Set to the path or remove the next statement to download from the network. - require.NoError(t, initCmd.Flags().Set(util.ContractsInitFlag, contractsPath)) + require.NoError(t, Cmd.Flags().Set(util.ContractsInitFlag, contractsPath)) dumpPath := filepath.Join(testdataDir, "out") - require.NoError(t, initCmd.Flags().Set(util.LocalDumpFlag, dumpPath)) + require.NoError(t, Cmd.Flags().Set(util.LocalDumpFlag, dumpPath)) v.Set(util.AlphabetWalletsFlag, testdataDir) v.Set(util.EpochDurationInitFlag, 1) v.Set(util.MaxObjectSizeInitFlag, 1024) setTestCredentials(v, committeeSize) - require.NoError(t, initializeSideChainCmd(initCmd, nil)) + require.NoError(t, initializeSideChainCmd(Cmd, nil)) t.Run("force-new-epoch", func(t *testing.T) { require.NoError(t, netmap.ForceNewEpoch.Flags().Set(util.LocalDumpFlag, dumpPath)) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go rename to cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go index ab4acefee..8e501953b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go @@ -1,4 +1,4 @@ -package morph +package initialize import ( "fmt" diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/root.go b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go new file mode 100644 index 000000000..7dd6d17ca --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go @@ -0,0 +1,55 @@ +package initialize + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + maxObjectSizeCLIFlag = "max-object-size" + epochDurationCLIFlag = "epoch-duration" + containerFeeCLIFlag = "container-fee" + containerAliasFeeCLIFlag = "container-alias-fee" + candidateFeeCLIFlag = "candidate-fee" + homomorphicHashDisabledCLIFlag = "homomorphic-disabled" + withdrawFeeCLIFlag = "withdraw-fee" +) + +var Cmd = &cobra.Command{ + Use: "init", + Short: "Initialize side chain network with smart-contracts and network settings", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(util.EpochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) + _ = viper.BindPFlag(util.MaxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) + _ = viper.BindPFlag(util.HomomorphicHashDisabledInitFlag, cmd.Flags().Lookup(homomorphicHashDisabledCLIFlag)) + _ = viper.BindPFlag(util.CandidateFeeInitFlag, cmd.Flags().Lookup(candidateFeeCLIFlag)) + _ = viper.BindPFlag(util.ContainerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) + _ = viper.BindPFlag(util.ContainerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) + _ = viper.BindPFlag(util.WithdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) + _ = viper.BindPFlag(util.ProtoConfigPath, cmd.Flags().Lookup(util.ProtoConfigPath)) + }, + RunE: initializeSideChainCmd, +} + +func initInitCmd() { + Cmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) + Cmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + Cmd.Flags().String(util.ContractsInitFlag, "", util.ContractsInitFlagDesc) + Cmd.Flags().String(util.ContractsURLFlag, "", util.ContractsURLFlagDesc) + Cmd.Flags().Uint(epochDurationCLIFlag, 240, "Amount of side chain blocks in one FrostFS epoch") + Cmd.Flags().Uint(maxObjectSizeCLIFlag, 67108864, "Max single object size in bytes") + Cmd.Flags().Bool(homomorphicHashDisabledCLIFlag, false, "Disable object homomorphic hashing") + // Defaults are taken from neo-preodolenie. + Cmd.Flags().Uint64(containerFeeCLIFlag, 1000, "Container registration fee") + Cmd.Flags().Uint64(containerAliasFeeCLIFlag, 500, "Container alias fee") + Cmd.Flags().String(util.ProtoConfigPath, "", "Path to the consensus node configuration") + Cmd.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") + Cmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) +} + +func init() { + initInitCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index e2a4517f3..9d2b25ceb 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -8,60 +8,24 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/contract" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/generate" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/initialize" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/node" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/notary" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/proxy" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/spf13/cobra" - "github.com/spf13/viper" ) -const ( - maxObjectSizeCLIFlag = "max-object-size" - - epochDurationCLIFlag = "epoch-duration" - - containerFeeCLIFlag = "container-fee" - containerAliasFeeCLIFlag = "container-alias-fee" - - candidateFeeCLIFlag = "candidate-fee" - - homomorphicHashDisabledCLIFlag = "homomorphic-disabled" - - withdrawFeeCLIFlag = "withdraw-fee" -) - -var ( - // RootCmd is a root command of config section. - RootCmd = &cobra.Command{ - Use: "morph", - Short: "Section for morph network configuration commands", - } - - initCmd = &cobra.Command{ - Use: "init", - Short: "Initialize side chain network with smart-contracts and network settings", - PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - _ = viper.BindPFlag(util.EpochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) - _ = viper.BindPFlag(util.MaxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) - _ = viper.BindPFlag(util.HomomorphicHashDisabledInitFlag, cmd.Flags().Lookup(homomorphicHashDisabledCLIFlag)) - _ = viper.BindPFlag(util.CandidateFeeInitFlag, cmd.Flags().Lookup(candidateFeeCLIFlag)) - _ = viper.BindPFlag(util.ContainerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) - _ = viper.BindPFlag(util.ContainerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) - _ = viper.BindPFlag(util.WithdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) - _ = viper.BindPFlag(util.ProtoConfigPath, cmd.Flags().Lookup(util.ProtoConfigPath)) - }, - RunE: initializeSideChainCmd, - } -) +// RootCmd is a root command of config section. +var RootCmd = &cobra.Command{ + Use: "morph", + Short: "Section for morph network configuration commands", +} func init() { RootCmd.AddCommand(generate.RefillGasCmd) - initInitCmd() + RootCmd.AddCommand(initialize.Cmd) RootCmd.AddCommand(contract.DeployCmd) RootCmd.AddCommand(generate.GenerateStorageCmd) RootCmd.AddCommand(netmap.ForceNewEpoch) @@ -86,20 +50,3 @@ func init() { RootCmd.AddCommand(frostfsid.Cmd) } - -func initInitCmd() { - RootCmd.AddCommand(initCmd) - initCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - initCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - initCmd.Flags().String(util.ContractsInitFlag, "", util.ContractsInitFlagDesc) - initCmd.Flags().String(util.ContractsURLFlag, "", util.ContractsURLFlagDesc) - initCmd.Flags().Uint(epochDurationCLIFlag, 240, "Amount of side chain blocks in one FrostFS epoch") - initCmd.Flags().Uint(maxObjectSizeCLIFlag, 67108864, "Max single object size in bytes") - initCmd.Flags().Bool(homomorphicHashDisabledCLIFlag, false, "Disable object homomorphic hashing") - // Defaults are taken from neo-preodolenie. - initCmd.Flags().Uint64(containerFeeCLIFlag, 1000, "Container registration fee") - initCmd.Flags().Uint64(containerAliasFeeCLIFlag, 500, "Container alias fee") - initCmd.Flags().String(util.ProtoConfigPath, "", "Path to the consensus node configuration") - initCmd.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") - initCmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) -} From 7b0e3f5010b353987f9d00a41c7e3f409204f588 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 2 Feb 2024 14:30:32 +0300 Subject: [PATCH 0903/1943] [#932] adm: Remove unnecessary import alias usage Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go | 8 ++++---- cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go index 4d64f190f..45e29c029 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - util2 "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/spf13/cobra" @@ -12,7 +12,7 @@ import ( ) func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { - wCtx, err := util2.NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := util.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) } @@ -23,13 +23,13 @@ func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util2.DomainOf(util2.NetmapContract)) + nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(util.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } bw := io.NewBufBinWriter() - if err := util2.EmitNewEpochCall(bw, wCtx, nmHash); err != nil { + if err := util.EmitNewEpochCall(bw, wCtx, nmHash); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go index 7455c0bd0..a8fd97be2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go @@ -3,7 +3,7 @@ package proxy import ( "fmt" - util2 "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" @@ -36,7 +36,7 @@ func removeProxyAccount(cmd *cobra.Command, _ []string) { } func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error { - wCtx, err := util2.NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) } @@ -47,7 +47,7 @@ func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error return fmt.Errorf("can't get NNS contract info: %w", err) } - proxyHash, err := util2.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util2.DomainOf(util2.ProxyContract)) + proxyHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.ProxyContract)) if err != nil { return fmt.Errorf("can't get proxy contract hash: %w", err) } From 814c411f4a2129453da58db6acb72a878a8746f7 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 2 Feb 2024 15:04:48 +0300 Subject: [PATCH 0904/1943] [#932] adm: Move flags to package `commonflags` Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/internal/commonflags/flags.go | 25 ++++++ .../internal/modules/morph/ape/ape.go | 42 ++++----- .../internal/modules/morph/ape/ape_util.go | 3 +- .../internal/modules/morph/balance/root.go | 6 +- .../internal/modules/morph/config/root.go | 16 ++-- .../internal/modules/morph/container/root.go | 18 ++-- .../internal/modules/morph/contract/deploy.go | 15 ++-- .../modules/morph/contract/dump_hashes.go | 3 +- .../internal/modules/morph/contract/root.go | 22 ++--- .../modules/morph/frostfsid/frostfsid.go | 89 ++++++++++--------- .../modules/morph/generate/generate.go | 9 +- .../modules/morph/generate/generate_test.go | 21 ++--- .../internal/modules/morph/generate/root.go | 36 ++++---- .../morph/initialize/initialize_test.go | 23 ++--- .../internal/modules/morph/initialize/root.go | 31 +++---- .../internal/modules/morph/netmap/root.go | 18 ++-- .../internal/modules/morph/node/root.go | 12 +-- .../internal/modules/morph/notary/notary.go | 7 +- .../internal/modules/morph/notary/root.go | 10 +-- .../internal/modules/morph/policy/root.go | 16 ++-- .../internal/modules/morph/proxy/root.go | 14 +-- .../internal/modules/morph/util/const.go | 23 ----- .../modules/morph/util/initialize_ctx.go | 17 ++-- .../internal/modules/morph/util/n3client.go | 3 +- .../internal/modules/morph/util/netmap.go | 17 ++-- 25 files changed, 255 insertions(+), 241 deletions(-) diff --git a/cmd/frostfs-adm/internal/commonflags/flags.go b/cmd/frostfs-adm/internal/commonflags/flags.go index 2f1ae4cc5..71c36b71a 100644 --- a/cmd/frostfs-adm/internal/commonflags/flags.go +++ b/cmd/frostfs-adm/internal/commonflags/flags.go @@ -11,4 +11,29 @@ const ( Verbose = "verbose" VerboseShorthand = "v" VerboseUsage = "Verbose output" + + EndpointFlag = "rpc-endpoint" + EndpointFlagDesc = "N3 RPC node endpoint" + EndpointFlagShort = "r" + + AlphabetWalletsFlag = "alphabet-wallets" + AlphabetWalletsFlagDesc = "Path to alphabet wallets dir" + + LocalDumpFlag = "local-dump" + ContractsInitFlag = "contracts" + ContractsInitFlagDesc = "Path to archive with compiled FrostFS contracts (the default is to fetch the latest release from the official repository)" + ContractsURLFlag = "contracts-url" + ContractsURLFlagDesc = "URL to archive with compiled FrostFS contracts" + EpochDurationInitFlag = "network.epoch_duration" + MaxObjectSizeInitFlag = "network.max_object_size" + RefillGasAmountFlag = "gas" + StorageWalletFlag = "storage-wallet" + ContainerFeeInitFlag = "network.fee.container" + ContainerAliasFeeInitFlag = "network.fee.container_alias" + CandidateFeeInitFlag = "network.fee.candidate" + WithdrawFeeInitFlag = "network.fee.withdraw" + MaintenanceModeAllowedInitFlag = "network.maintenance_mode_allowed" + HomomorphicHashDisabledInitFlag = "network.homomorphic_hash_disabled" + CustomZoneFlag = "domain" + AlphabetSizeFlag = "size" ) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go index de3765042..8395fd4ba 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -37,8 +37,8 @@ var ( Use: "add-rule-chain", Short: "Add rule chain", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, Run: addRuleChain, } @@ -47,8 +47,8 @@ var ( Use: "rm-rule-chain", Short: "Remove rule chain", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, Run: removeRuleChain, } @@ -57,8 +57,8 @@ var ( Use: "list-rule-chains", Short: "List rule chains", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, Run: listRuleChains, } @@ -67,8 +67,8 @@ var ( Use: "set-admin", Short: "Set admin", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, Run: setAdmin, } @@ -77,8 +77,8 @@ var ( Use: "get-admin", Short: "Get admin", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, Run: getAdmin, } @@ -87,8 +87,8 @@ var ( func initAddRuleChainCmd() { Cmd.AddCommand(addRuleChainCmd) - addRuleChainCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - addRuleChainCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + addRuleChainCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + addRuleChainCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) addRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc) _ = addRuleChainCmd.MarkFlagRequired(targetTypeFlag) @@ -106,8 +106,8 @@ func initAddRuleChainCmd() { func initRemoveRuleChainCmd() { Cmd.AddCommand(removeRuleChainCmd) - removeRuleChainCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - removeRuleChainCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + removeRuleChainCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + removeRuleChainCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) removeRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc) _ = removeRuleChainCmd.MarkFlagRequired(targetTypeFlag) @@ -120,8 +120,8 @@ func initRemoveRuleChainCmd() { func initListRuleChainsCmd() { Cmd.AddCommand(listRuleChainsCmd) - listRuleChainsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - listRuleChainsCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + listRuleChainsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + listRuleChainsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) listRuleChainsCmd.Flags().StringP(targetTypeFlag, "t", "", targetTypeDesc) _ = listRuleChainsCmd.MarkFlagRequired(targetTypeFlag) listRuleChainsCmd.Flags().String(targetNameFlag, "", targetNameDesc) @@ -132,8 +132,8 @@ func initListRuleChainsCmd() { func initSetAdminCmd() { Cmd.AddCommand(setAdminCmd) - setAdminCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - setAdminCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + setAdminCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + setAdminCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) setAdminCmd.Flags().String(addrAdminFlag, "", addrAdminDesc) _ = setAdminCmd.MarkFlagRequired(addrAdminFlag) } @@ -141,8 +141,8 @@ func initSetAdminCmd() { func initGetAdminCmd() { Cmd.AddCommand(getAdminCmd) - getAdminCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - getAdminCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + getAdminCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + getAdminCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func addRuleChain(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 21f27bfd7..7dc5fa082 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" @@ -81,7 +82,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *ac c, err := morphUtil.GetN3Client(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) - walletDir := config.ResolveHomePath(viper.GetString(morphUtil.AlphabetWalletsFlag)) + walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) wallets, err := morphUtil.GetAlphabetWallets(v, walletDir) commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) diff --git a/cmd/frostfs-adm/internal/modules/morph/balance/root.go b/cmd/frostfs-adm/internal/modules/morph/balance/root.go index 757637e5d..3be712367 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance/root.go @@ -1,7 +1,7 @@ package balance import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -10,13 +10,13 @@ var DumpCmd = &cobra.Command{ Use: "dump-balances", Short: "Dump GAS balances", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: dumpBalances, } func initDumpBalancesCmd() { - DumpCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + DumpCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) DumpCmd.Flags().BoolP(dumpBalancesStorageFlag, "s", false, "Dump balances of storage nodes from the current netmap") DumpCmd.Flags().BoolP(dumpBalancesAlphabetFlag, "a", false, "Dump balances of alphabet contracts") DumpCmd.Flags().BoolP(dumpBalancesProxyFlag, "p", false, "Dump balances of the proxy contract") diff --git a/cmd/frostfs-adm/internal/modules/morph/config/root.go b/cmd/frostfs-adm/internal/modules/morph/config/root.go index 4b5366226..6b9094de0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/root.go @@ -1,7 +1,7 @@ package config import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -12,8 +12,8 @@ var ( DisableFlagsInUseLine: true, Short: "Add/update global config value in the FrostFS network", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Args: cobra.MinimumNArgs(1), RunE: SetConfigCmd, @@ -23,21 +23,21 @@ var ( Use: "dump-config", Short: "Dump FrostFS network config", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: dumpNetworkConfig, } ) func initSetConfigCmd() { - SetCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - SetCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + SetCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + SetCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) SetCmd.Flags().Bool(forceConfigSet, false, "Force setting not well-known configuration key") - SetCmd.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") + SetCmd.Flags().String(commonflags.LocalDumpFlag, "", "Path to the blocks dump file") } func initDumpNetworkConfigCmd() { - DumpCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + DumpCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) } func init() { diff --git a/cmd/frostfs-adm/internal/modules/morph/container/root.go b/cmd/frostfs-adm/internal/modules/morph/container/root.go index 4cae8cf4f..2b314ab09 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/root.go @@ -1,7 +1,7 @@ package container import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -17,7 +17,7 @@ var ( Use: "dump-containers", Short: "Dump FrostFS containers to file", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: dumpContainers, } @@ -26,8 +26,8 @@ var ( Use: "restore-containers", Short: "Restore FrostFS containers from file", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: restoreContainers, } @@ -36,26 +36,26 @@ var ( Use: "list-containers", Short: "List FrostFS containers", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: listContainers, } ) func initListContainersCmd() { - ListCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + ListCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) ListCmd.Flags().String(containerContractFlag, "", "Container contract hash (for networks without NNS)") } func initRestoreContainersCmd() { - RestoreCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - RestoreCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + RestoreCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + RestoreCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) RestoreCmd.Flags().String(containerDumpFlag, "", "File to restore containers from") RestoreCmd.Flags().StringSlice(containerIDsFlag, nil, "Containers to restore") } func initDumpContainersCmd() { - DumpCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + DumpCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) DumpCmd.Flags().String(containerDumpFlag, "", "File where to save dumped containers") DumpCmd.Flags().String(containerContractFlag, "", "Container contract hash (for networks without NNS)") DumpCmd.Flags().StringSlice(containerIDsFlag, nil, "Containers to dump") diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go index 35007a273..67a8d2fac 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go @@ -7,6 +7,7 @@ import ( "strings" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/cli/cmdargs" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -37,8 +38,8 @@ Compiled contract file name must contain '_contract.nef' suffix. Contract's manifest file name must be 'config.json'. NNS name is taken by stripping '_contract.nef' from the NEF file (similar to frostfs contracts).`, PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: deployContractCmd, } @@ -46,15 +47,15 @@ NNS name is taken by stripping '_contract.nef' from the NEF file (similar to fro func init() { ff := DeployCmd.Flags() - ff.String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - _ = DeployCmd.MarkFlagFilename(util.AlphabetWalletsFlag) + ff.String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + _ = DeployCmd.MarkFlagFilename(commonflags.AlphabetWalletsFlag) - ff.StringP(util.EndpointFlag, "r", "", util.EndpointFlagDesc) + ff.StringP(commonflags.EndpointFlag, "r", "", commonflags.EndpointFlagDesc) ff.String(contractPathFlag, "", "Path to the contract directory") _ = DeployCmd.MarkFlagFilename(contractPathFlag) ff.Bool(updateFlag, false, "Update an existing contract") - ff.String(util.CustomZoneFlag, "frostfs", "Custom zone for NNS") + ff.String(commonflags.CustomZoneFlag, "frostfs", "Custom zone for NNS") } func deployContractCmd(cmd *cobra.Command, args []string) error { @@ -84,7 +85,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { callHash := management.Hash method := util.DeployMethodName - zone, _ := cmd.Flags().GetString(util.CustomZoneFlag) + zone, _ := cmd.Flags().GetString(commonflags.CustomZoneFlag) domain := ctrName + "." + zone isUpdate, _ := cmd.Flags().GetBool(updateFlag) if isUpdate { diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index 8f7fbe6e0..ca1e5433b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -8,6 +8,7 @@ import ( "text/tabwriter" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/io" @@ -45,7 +46,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { return err } - zone, _ := cmd.Flags().GetString(morphUtil.CustomZoneFlag) + zone, _ := cmd.Flags().GetString(commonflags.CustomZoneFlag) if zone != "" { return dumpCustomZoneHashes(cmd, cs.Hash, zone, c) } diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/root.go b/cmd/frostfs-adm/internal/modules/morph/contract/root.go index 71feb31cd..9bad2bd66 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/root.go @@ -1,7 +1,7 @@ package contract import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -11,7 +11,7 @@ var ( Use: "dump-hashes", Short: "Dump deployed contract hashes", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: dumpContractHashes, } @@ -19,24 +19,24 @@ var ( Use: "update-contracts", Short: "Update FrostFS contracts", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: updateContracts, } ) func initDumpContractHashesCmd() { - DumpHashesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - DumpHashesCmd.Flags().String(util.CustomZoneFlag, "", "Custom zone to search.") + DumpHashesCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + DumpHashesCmd.Flags().String(commonflags.CustomZoneFlag, "", "Custom zone to search.") } func initUpdateContractsCmd() { - UpdateCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - UpdateCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - UpdateCmd.Flags().String(util.ContractsInitFlag, "", util.ContractsInitFlagDesc) - UpdateCmd.Flags().String(util.ContractsURLFlag, "", util.ContractsURLFlagDesc) - UpdateCmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) + UpdateCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + UpdateCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + UpdateCmd.Flags().String(commonflags.ContractsInitFlag, "", commonflags.ContractsInitFlagDesc) + UpdateCmd.Flags().String(commonflags.ContractsURLFlag, "", commonflags.ContractsURLFlagDesc) + UpdateCmd.MarkFlagsMutuallyExclusive(commonflags.ContractsInitFlag, commonflags.ContractsURLFlag) } func init() { diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index d40b09da3..0a64b9e16 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -6,6 +6,7 @@ import ( "sort" frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -41,8 +42,8 @@ var ( Use: "create-namespace", Short: "Create new namespace in frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidCreateNamespace, } @@ -51,8 +52,8 @@ var ( Use: "list-namespaces", Short: "List all namespaces in frostfsid", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidListNamespaces, } @@ -61,8 +62,8 @@ var ( Use: "create-subject", Short: "Create subject in frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidCreateSubject, } @@ -71,8 +72,8 @@ var ( Use: "delete-subject", Short: "Delete subject from frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidDeleteSubject, } @@ -81,8 +82,8 @@ var ( Use: "list-subjects", Short: "List subjects in namespace", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidListSubjects, } @@ -91,8 +92,8 @@ var ( Use: "create-group", Short: "Create group in frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidCreateGroup, } @@ -101,8 +102,8 @@ var ( Use: "delete-group", Short: "Delete group from frostfsid contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidDeleteGroup, } @@ -111,8 +112,8 @@ var ( Use: "list-groups", Short: "List groups in namespace", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidListGroups, } @@ -121,8 +122,8 @@ var ( Use: "add-subject-to-group", Short: "Add subject to group", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidAddSubjectToGroup, } @@ -131,8 +132,8 @@ var ( Use: "remove-subject-from-group", Short: "Remove subject from group", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidRemoveSubjectFromGroup, } @@ -141,8 +142,8 @@ var ( Use: "list-group-subjects", Short: "List subjects in group", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(morphUtil.AlphabetWalletsFlag, cmd.Flags().Lookup(morphUtil.AlphabetWalletsFlag)) - _ = viper.BindPFlag(morphUtil.EndpointFlag, cmd.Flags().Lookup(morphUtil.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidListGroupSubjects, } @@ -150,87 +151,87 @@ var ( func initFrostfsIDCreateNamespaceCmd() { Cmd.AddCommand(frostfsidCreateNamespaceCmd) - frostfsidCreateNamespaceCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) + frostfsidCreateNamespaceCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidCreateNamespaceCmd.Flags().String(namespaceFlag, "", "Namespace name to create") - frostfsidCreateNamespaceCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + frostfsidCreateNamespaceCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDListNamespacesCmd() { Cmd.AddCommand(frostfsidListNamespacesCmd) - frostfsidListNamespacesCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) - frostfsidListNamespacesCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + frostfsidListNamespacesCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + frostfsidListNamespacesCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDCreateSubjectCmd() { Cmd.AddCommand(frostfsidCreateSubjectCmd) - frostfsidCreateSubjectCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) + frostfsidCreateSubjectCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidCreateSubjectCmd.Flags().String(namespaceFlag, "", "Namespace where create subject") frostfsidCreateSubjectCmd.Flags().String(subjectNameFlag, "", "Subject name, must be unique in namespace") frostfsidCreateSubjectCmd.Flags().String(subjectKeyFlag, "", "Subject hex-encoded public key") - frostfsidCreateSubjectCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + frostfsidCreateSubjectCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDDeleteSubjectCmd() { Cmd.AddCommand(frostfsidDeleteSubjectCmd) - frostfsidDeleteSubjectCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) + frostfsidDeleteSubjectCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidDeleteSubjectCmd.Flags().String(subjectAddressFlag, "", "Subject address") - frostfsidDeleteSubjectCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + frostfsidDeleteSubjectCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDListSubjectsCmd() { Cmd.AddCommand(frostfsidListSubjectsCmd) - frostfsidListSubjectsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) + frostfsidListSubjectsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidListSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace to list subjects") frostfsidListSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") - frostfsidListSubjectsCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + frostfsidListSubjectsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDCreateGroupCmd() { Cmd.AddCommand(frostfsidCreateGroupCmd) - frostfsidCreateGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) + frostfsidCreateGroupCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidCreateGroupCmd.Flags().String(namespaceFlag, "", "Namespace where create group") frostfsidCreateGroupCmd.Flags().String(groupNameFlag, "", "Group name, must be unique in namespace") - frostfsidCreateGroupCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + frostfsidCreateGroupCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDDeleteGroupCmd() { Cmd.AddCommand(frostfsidDeleteGroupCmd) - frostfsidDeleteGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) + frostfsidDeleteGroupCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidDeleteGroupCmd.Flags().String(namespaceFlag, "", "Namespace to delete group") frostfsidDeleteGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") - frostfsidDeleteGroupCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + frostfsidDeleteGroupCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDListGroupsCmd() { Cmd.AddCommand(frostfsidListGroupsCmd) - frostfsidListGroupsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) + frostfsidListGroupsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidListGroupsCmd.Flags().String(namespaceFlag, "", "Namespace to list groups") - frostfsidListGroupsCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + frostfsidListGroupsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDAddSubjectToGroupCmd() { Cmd.AddCommand(frostfsidAddSubjectToGroupCmd) - frostfsidAddSubjectToGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) + frostfsidAddSubjectToGroupCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidAddSubjectToGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidAddSubjectToGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") - frostfsidAddSubjectToGroupCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + frostfsidAddSubjectToGroupCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDRemoveSubjectFromGroupCmd() { Cmd.AddCommand(frostfsidRemoveSubjectFromGroupCmd) - frostfsidRemoveSubjectFromGroupCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) + frostfsidRemoveSubjectFromGroupCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidRemoveSubjectFromGroupCmd.Flags().String(subjectAddressFlag, "", "Subject address") frostfsidRemoveSubjectFromGroupCmd.Flags().Int64(groupIDFlag, 0, "Group id") - frostfsidRemoveSubjectFromGroupCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + frostfsidRemoveSubjectFromGroupCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDListGroupSubjectsCmd() { Cmd.AddCommand(frostfsidListGroupSubjectsCmd) - frostfsidListGroupSubjectsCmd.Flags().StringP(morphUtil.EndpointFlag, morphUtil.EndpointFlagShort, "", morphUtil.EndpointFlagDesc) + frostfsidListGroupSubjectsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidListGroupSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace name") frostfsidListGroupSubjectsCmd.Flags().Int64(groupIDFlag, 0, "Group id") frostfsidListGroupSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") - frostfsidListGroupSubjectsCmd.Flags().String(morphUtil.AlphabetWalletsFlag, "", morphUtil.AlphabetWalletsFlagDesc) + frostfsidListGroupSubjectsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func frostfsidCreateNamespace(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go index 364ce2c6e..8db0aecd2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" @@ -26,7 +27,7 @@ import ( func AlphabetCreds(cmd *cobra.Command, _ []string) error { // alphabet size is not part of the config - size, err := cmd.Flags().GetUint(morphUtil.AlphabetSizeFlag) + size, err := cmd.Flags().GetUint(commonflags.AlphabetSizeFlag) if err != nil { return err } @@ -38,7 +39,7 @@ func AlphabetCreds(cmd *cobra.Command, _ []string) error { } v := viper.GetViper() - walletDir := config.ResolveHomePath(viper.GetString(morphUtil.AlphabetWalletsFlag)) + walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) pwds, err := initializeWallets(v, walletDir, int(size)) if err != nil { return err @@ -138,7 +139,7 @@ func generateStorageCreds(cmd *cobra.Command, _ []string) error { func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error) { // storage wallet path is not part of the config - storageWalletPath, _ := cmd.Flags().GetString(morphUtil.StorageWalletFlag) + storageWalletPath, _ := cmd.Flags().GetString(commonflags.StorageWalletFlag) // wallet address is not part of the config walletAddress, _ := cmd.Flags().GetString(walletAddressFlag) @@ -151,7 +152,7 @@ func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error } } else { if storageWalletPath == "" { - return fmt.Errorf("missing wallet path (use '--%s ')", morphUtil.StorageWalletFlag) + return fmt.Errorf("missing wallet path (use '--%s ')", commonflags.StorageWalletFlag) } var w *wallet.Wallet diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go index 49379ee26..add00b5ce 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go @@ -10,6 +10,7 @@ import ( "sync" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/cli/input" @@ -32,29 +33,29 @@ func TestGenerateAlphabet(t *testing.T) { t.Run("zero size", func(t *testing.T) { buf.Reset() - v.Set(util.AlphabetWalletsFlag, walletDir) - require.NoError(t, cmd.Flags().Set(util.AlphabetSizeFlag, "0")) + v.Set(commonflags.AlphabetWalletsFlag, walletDir) + require.NoError(t, cmd.Flags().Set(commonflags.AlphabetSizeFlag, "0")) buf.WriteString("pass\r") require.Error(t, AlphabetCreds(cmd, nil)) }) t.Run("no password provided", func(t *testing.T) { buf.Reset() - v.Set(util.AlphabetWalletsFlag, walletDir) - require.NoError(t, cmd.Flags().Set(util.AlphabetSizeFlag, "1")) + v.Set(commonflags.AlphabetWalletsFlag, walletDir) + require.NoError(t, cmd.Flags().Set(commonflags.AlphabetSizeFlag, "1")) require.Error(t, AlphabetCreds(cmd, nil)) }) t.Run("missing directory", func(t *testing.T) { buf.Reset() dir := filepath.Join(os.TempDir(), "notexist."+strconv.FormatUint(rand.Uint64(), 10)) - v.Set(util.AlphabetWalletsFlag, dir) - require.NoError(t, cmd.Flags().Set(util.AlphabetSizeFlag, "1")) + v.Set(commonflags.AlphabetWalletsFlag, dir) + require.NoError(t, cmd.Flags().Set(commonflags.AlphabetSizeFlag, "1")) buf.WriteString("pass\r") require.Error(t, AlphabetCreds(cmd, nil)) }) t.Run("no password for contract group wallet", func(t *testing.T) { buf.Reset() - v.Set(util.AlphabetWalletsFlag, walletDir) - require.NoError(t, cmd.Flags().Set(util.AlphabetSizeFlag, strconv.FormatUint(size, 10))) + v.Set(commonflags.AlphabetWalletsFlag, walletDir) + require.NoError(t, cmd.Flags().Set(commonflags.AlphabetSizeFlag, strconv.FormatUint(size, 10))) for i := uint64(0); i < size; i++ { buf.WriteString(strconv.FormatUint(i, 10) + "\r") } @@ -62,8 +63,8 @@ func TestGenerateAlphabet(t *testing.T) { }) buf.Reset() - v.Set(util.AlphabetWalletsFlag, walletDir) - require.NoError(t, GenerateAlphabetCmd.Flags().Set(util.AlphabetSizeFlag, strconv.FormatUint(size, 10))) + v.Set(commonflags.AlphabetWalletsFlag, walletDir) + require.NoError(t, GenerateAlphabetCmd.Flags().Set(commonflags.AlphabetSizeFlag, strconv.FormatUint(size, 10))) for i := uint64(0); i < size; i++ { buf.WriteString(strconv.FormatUint(i, 10) + "\r") } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/root.go b/cmd/frostfs-adm/internal/modules/morph/generate/root.go index e5f5b38cd..6b04848ae 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/root.go @@ -1,7 +1,7 @@ package generate import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -18,8 +18,8 @@ var ( Use: "generate-storage-wallet", Short: "Generate storage node wallet for the morph network", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) _ = viper.BindPFlag(storageGasConfigFlag, cmd.Flags().Lookup(storageGasCLIFlag)) }, RunE: generateStorageCreds, @@ -28,12 +28,12 @@ var ( Use: "refill-gas", Short: "Refill GAS of storage node's wallet in the morph network", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - _ = viper.BindPFlag(util.RefillGasAmountFlag, cmd.Flags().Lookup(util.RefillGasAmountFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.RefillGasAmountFlag, cmd.Flags().Lookup(commonflags.RefillGasAmountFlag)) }, RunE: func(cmd *cobra.Command, args []string) error { - return refillGas(cmd, util.RefillGasAmountFlag, false) + return refillGas(cmd, commonflags.RefillGasAmountFlag, false) }, } GenerateAlphabetCmd = &cobra.Command{ @@ -41,32 +41,32 @@ var ( Short: "Generate alphabet wallets for consensus nodes of the morph network", PreRun: func(cmd *cobra.Command, _ []string) { // PreRun fixes https://github.com/spf13/viper/issues/233 - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, RunE: AlphabetCreds, } ) func initRefillGasCmd() { - RefillGasCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - RefillGasCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - RefillGasCmd.Flags().String(util.StorageWalletFlag, "", "Path to storage node wallet") + RefillGasCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + RefillGasCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + RefillGasCmd.Flags().String(commonflags.StorageWalletFlag, "", "Path to storage node wallet") RefillGasCmd.Flags().String(walletAddressFlag, "", "Address of wallet") - RefillGasCmd.Flags().String(util.RefillGasAmountFlag, "", "Additional amount of GAS to transfer") - RefillGasCmd.MarkFlagsMutuallyExclusive(walletAddressFlag, util.StorageWalletFlag) + RefillGasCmd.Flags().String(commonflags.RefillGasAmountFlag, "", "Additional amount of GAS to transfer") + RefillGasCmd.MarkFlagsMutuallyExclusive(walletAddressFlag, commonflags.StorageWalletFlag) } func initGenerateStorageCmd() { - GenerateStorageCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - GenerateStorageCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - GenerateStorageCmd.Flags().String(util.StorageWalletFlag, "", "Path to new storage node wallet") + GenerateStorageCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + GenerateStorageCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + GenerateStorageCmd.Flags().String(commonflags.StorageWalletFlag, "", "Path to new storage node wallet") GenerateStorageCmd.Flags().String(storageGasCLIFlag, "", "Initial amount of GAS to transfer") GenerateStorageCmd.Flags().StringP(storageWalletLabelFlag, "l", "", "Wallet label") } func initGenerateAlphabetCmd() { - GenerateAlphabetCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - GenerateAlphabetCmd.Flags().Uint(util.AlphabetSizeFlag, 7, "Amount of alphabet wallets to generate") + GenerateAlphabetCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + GenerateAlphabetCmd.Flags().Uint(commonflags.AlphabetSizeFlag, 7, "Amount of alphabet wallets to generate") } func init() { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go index d6b180609..88bf0f45f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" cmdConfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/generate" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/netmap" @@ -63,27 +64,27 @@ func testInitialize(t *testing.T, committeeSize int) { v.Set(util.ProtoConfigPath, filepath.Join(testdataDir, protoFileName)) // Set to the path or remove the next statement to download from the network. - require.NoError(t, Cmd.Flags().Set(util.ContractsInitFlag, contractsPath)) + require.NoError(t, Cmd.Flags().Set(commonflags.ContractsInitFlag, contractsPath)) dumpPath := filepath.Join(testdataDir, "out") - require.NoError(t, Cmd.Flags().Set(util.LocalDumpFlag, dumpPath)) - v.Set(util.AlphabetWalletsFlag, testdataDir) - v.Set(util.EpochDurationInitFlag, 1) - v.Set(util.MaxObjectSizeInitFlag, 1024) + require.NoError(t, Cmd.Flags().Set(commonflags.LocalDumpFlag, dumpPath)) + v.Set(commonflags.AlphabetWalletsFlag, testdataDir) + v.Set(commonflags.EpochDurationInitFlag, 1) + v.Set(commonflags.MaxObjectSizeInitFlag, 1024) setTestCredentials(v, committeeSize) require.NoError(t, initializeSideChainCmd(Cmd, nil)) t.Run("force-new-epoch", func(t *testing.T) { - require.NoError(t, netmap.ForceNewEpoch.Flags().Set(util.LocalDumpFlag, dumpPath)) + require.NoError(t, netmap.ForceNewEpoch.Flags().Set(commonflags.LocalDumpFlag, dumpPath)) require.NoError(t, netmap.ForceNewEpochCmd(netmap.ForceNewEpoch, nil)) }) t.Run("set-config", func(t *testing.T) { - require.NoError(t, cmdConfig.SetCmd.Flags().Set(util.LocalDumpFlag, dumpPath)) + require.NoError(t, cmdConfig.SetCmd.Flags().Set(commonflags.LocalDumpFlag, dumpPath)) require.NoError(t, cmdConfig.SetConfigCmd(cmdConfig.SetCmd, []string{"MaintenanceModeAllowed=true"})) }) t.Run("set-policy", func(t *testing.T) { - require.NoError(t, policy.Set.Flags().Set(util.LocalDumpFlag, dumpPath)) + require.NoError(t, policy.Set.Flags().Set(commonflags.LocalDumpFlag, dumpPath)) require.NoError(t, policy.SetPolicyCmd(policy.Set, []string{"ExecFeeFactor=1"})) }) t.Run("remove-node", func(t *testing.T) { @@ -91,17 +92,17 @@ func testInitialize(t *testing.T, committeeSize int) { require.NoError(t, err) pub := hex.EncodeToString(pk.PublicKey().Bytes()) - require.NoError(t, node.RemoveCmd.Flags().Set(util.LocalDumpFlag, dumpPath)) + require.NoError(t, node.RemoveCmd.Flags().Set(commonflags.LocalDumpFlag, dumpPath)) require.NoError(t, node.RemoveNodesCmd(node.RemoveCmd, []string{pub})) }) } func generateTestData(t *testing.T, dir string, size int) error { v := viper.GetViper() - v.Set(util.AlphabetWalletsFlag, dir) + v.Set(commonflags.AlphabetWalletsFlag, dir) sizeStr := strconv.FormatUint(uint64(size), 10) - if err := generate.GenerateAlphabetCmd.Flags().Set(util.AlphabetSizeFlag, sizeStr); err != nil { + if err := generate.GenerateAlphabetCmd.Flags().Set(commonflags.AlphabetSizeFlag, sizeStr); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/root.go b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go index 7dd6d17ca..54958991e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go @@ -1,6 +1,7 @@ package initialize import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -20,25 +21,25 @@ var Cmd = &cobra.Command{ Use: "init", Short: "Initialize side chain network with smart-contracts and network settings", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - _ = viper.BindPFlag(util.EpochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) - _ = viper.BindPFlag(util.MaxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) - _ = viper.BindPFlag(util.HomomorphicHashDisabledInitFlag, cmd.Flags().Lookup(homomorphicHashDisabledCLIFlag)) - _ = viper.BindPFlag(util.CandidateFeeInitFlag, cmd.Flags().Lookup(candidateFeeCLIFlag)) - _ = viper.BindPFlag(util.ContainerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) - _ = viper.BindPFlag(util.ContainerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) - _ = viper.BindPFlag(util.WithdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.EpochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) + _ = viper.BindPFlag(commonflags.MaxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) + _ = viper.BindPFlag(commonflags.HomomorphicHashDisabledInitFlag, cmd.Flags().Lookup(homomorphicHashDisabledCLIFlag)) + _ = viper.BindPFlag(commonflags.CandidateFeeInitFlag, cmd.Flags().Lookup(candidateFeeCLIFlag)) + _ = viper.BindPFlag(commonflags.ContainerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) + _ = viper.BindPFlag(commonflags.ContainerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) + _ = viper.BindPFlag(commonflags.WithdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) _ = viper.BindPFlag(util.ProtoConfigPath, cmd.Flags().Lookup(util.ProtoConfigPath)) }, RunE: initializeSideChainCmd, } func initInitCmd() { - Cmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - Cmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - Cmd.Flags().String(util.ContractsInitFlag, "", util.ContractsInitFlagDesc) - Cmd.Flags().String(util.ContractsURLFlag, "", util.ContractsURLFlagDesc) + Cmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + Cmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + Cmd.Flags().String(commonflags.ContractsInitFlag, "", commonflags.ContractsInitFlagDesc) + Cmd.Flags().String(commonflags.ContractsURLFlag, "", commonflags.ContractsURLFlagDesc) Cmd.Flags().Uint(epochDurationCLIFlag, 240, "Amount of side chain blocks in one FrostFS epoch") Cmd.Flags().Uint(maxObjectSizeCLIFlag, 67108864, "Max single object size in bytes") Cmd.Flags().Bool(homomorphicHashDisabledCLIFlag, false, "Disable object homomorphic hashing") @@ -46,8 +47,8 @@ func initInitCmd() { Cmd.Flags().Uint64(containerFeeCLIFlag, 1000, "Container registration fee") Cmd.Flags().Uint64(containerAliasFeeCLIFlag, 500, "Container alias fee") Cmd.Flags().String(util.ProtoConfigPath, "", "Path to the consensus node configuration") - Cmd.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") - Cmd.MarkFlagsMutuallyExclusive(util.ContractsInitFlag, util.ContractsURLFlag) + Cmd.Flags().String(commonflags.LocalDumpFlag, "", "Path to the blocks dump file") + Cmd.MarkFlagsMutuallyExclusive(commonflags.ContractsInitFlag, commonflags.ContractsURLFlag) } func init() { diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go index 888b994e9..31fda860e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go @@ -1,7 +1,7 @@ package netmap import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -11,8 +11,8 @@ var ( Use: "netmap-candidates", Short: "List netmap candidates nodes", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, Run: listNetmapCandidatesNodes, } @@ -20,21 +20,21 @@ var ( Use: "force-new-epoch", Short: "Create new FrostFS epoch event in the side chain", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: ForceNewEpochCmd, } ) func initNetmapCandidatesCmd() { - CandidatesCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + CandidatesCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) } func initForceNewEpochCmd() { - ForceNewEpoch.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - ForceNewEpoch.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - ForceNewEpoch.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") + ForceNewEpoch.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + ForceNewEpoch.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + ForceNewEpoch.Flags().String(commonflags.LocalDumpFlag, "", "Path to the blocks dump file") } func init() { diff --git a/cmd/frostfs-adm/internal/modules/morph/node/root.go b/cmd/frostfs-adm/internal/modules/morph/node/root.go index e301a1bc3..1c38ae8bc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/node/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/node/root.go @@ -1,7 +1,7 @@ package node import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -11,16 +11,16 @@ var RemoveCmd = &cobra.Command{ Short: "Remove storage nodes from the netmap", Long: `Move nodes to the Offline state in the candidates list and tick an epoch to update the netmap`, PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: RemoveNodesCmd, } func initRemoveNodesCmd() { - RemoveCmd.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - RemoveCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - RemoveCmd.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") + RemoveCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + RemoveCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + RemoveCmd.Flags().String(commonflags.LocalDumpFlag, "", "Path to the blocks dump file") } func init() { diff --git a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go index ecf9331b5..3b1bb179f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go @@ -5,6 +5,7 @@ import ( "math/big" "strconv" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/cli/input" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -60,7 +61,7 @@ func depositNotary(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't unlock account: %v", err) } - gasStr, err := cmd.Flags().GetString(morphUtil.RefillGasAmountFlag) + gasStr, err := cmd.Flags().GetString(commonflags.RefillGasAmountFlag) if err != nil { return err } @@ -126,11 +127,11 @@ func transferGas(cmd *cobra.Command, acc *wallet.Account, accHash util.Uint160, } func openWallet(cmd *cobra.Command) (*wallet.Wallet, error) { - p, err := cmd.Flags().GetString(morphUtil.StorageWalletFlag) + p, err := cmd.Flags().GetString(commonflags.StorageWalletFlag) if err != nil { return nil, err } else if p == "" { - return nil, fmt.Errorf("missing wallet path (use '--%s ')", morphUtil.StorageWalletFlag) + return nil, fmt.Errorf("missing wallet path (use '--%s ')", commonflags.StorageWalletFlag) } w, err := wallet.NewWalletFromFile(p) diff --git a/cmd/frostfs-adm/internal/modules/morph/notary/root.go b/cmd/frostfs-adm/internal/modules/morph/notary/root.go index 9a4164c7b..497ff8ea1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary/root.go @@ -1,7 +1,7 @@ package notary import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -10,16 +10,16 @@ var DepositCmd = &cobra.Command{ Use: "deposit-notary", Short: "Deposit GAS for notary service", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: depositNotary, } func initDepositoryNotaryCmd() { - DepositCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - DepositCmd.Flags().String(util.StorageWalletFlag, "", "Path to storage node wallet") + DepositCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + DepositCmd.Flags().String(commonflags.StorageWalletFlag, "", "Path to storage node wallet") DepositCmd.Flags().String(walletAccountFlag, "", "Wallet account address") - DepositCmd.Flags().String(util.RefillGasAmountFlag, "", "Amount of GAS to deposit") + DepositCmd.Flags().String(commonflags.RefillGasAmountFlag, "", "Amount of GAS to deposit") DepositCmd.Flags().String(notaryDepositTillFlag, "", "Notary deposit duration in blocks") } diff --git a/cmd/frostfs-adm/internal/modules/morph/policy/root.go b/cmd/frostfs-adm/internal/modules/morph/policy/root.go index 99ff56ee8..db50279dc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy/root.go @@ -1,7 +1,7 @@ package policy import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -12,8 +12,8 @@ var ( DisableFlagsInUseLine: true, Short: "Set global policy values", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: SetPolicyCmd, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { @@ -25,20 +25,20 @@ var ( Use: "dump-policy", Short: "Dump FrostFS policy", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: dumpPolicyCmd, } ) func initSetPolicyCmd() { - Set.Flags().String(util.AlphabetWalletsFlag, "", util.AlphabetWalletsFlagDesc) - Set.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) - Set.Flags().String(util.LocalDumpFlag, "", "Path to the blocks dump file") + Set.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + Set.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + Set.Flags().String(commonflags.LocalDumpFlag, "", "Path to the blocks dump file") } func initDumpPolicyCmd() { - Dump.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + Dump.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) } func init() { diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy/root.go b/cmd/frostfs-adm/internal/modules/morph/proxy/root.go index 14820d5ab..082bc57d1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/root.go @@ -1,7 +1,7 @@ package proxy import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -11,8 +11,8 @@ var ( Use: "proxy-add-account", Short: "Adds account to proxy contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: addProxyAccount, } @@ -20,20 +20,20 @@ var ( Use: "proxy-remove-account", Short: "Remove from proxy contract", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(util.AlphabetWalletsFlag, cmd.Flags().Lookup(util.AlphabetWalletsFlag)) - _ = viper.BindPFlag(util.EndpointFlag, cmd.Flags().Lookup(util.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: removeProxyAccount, } ) func initProxyAddAccount() { - AddAccountCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + AddAccountCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) AddAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") } func initProxyRemoveAccount() { - RemoveAccountCmd.Flags().StringP(util.EndpointFlag, util.EndpointFlagShort, "", util.EndpointFlagDesc) + RemoveAccountCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) RemoveAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/util/const.go index 1128adee9..7c867db13 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/const.go @@ -11,29 +11,6 @@ const ( // See: https://github.com/nspcc-dev/neo-go/blob/740488f7f35e367eaa99a71c0a609c315fe2b0fc/pkg/core/transaction/witness.go#L10 MaxAlphabetNodes = 22 - EndpointFlag = "rpc-endpoint" - EndpointFlagDesc = "N3 RPC node endpoint" - EndpointFlagShort = "r" - AlphabetWalletsFlag = "alphabet-wallets" - AlphabetWalletsFlagDesc = "Path to alphabet wallets dir" - LocalDumpFlag = "local-dump" - ContractsInitFlag = "contracts" - ContractsInitFlagDesc = "Path to archive with compiled FrostFS contracts (the default is to fetch the latest release from the official repository)" - ContractsURLFlag = "contracts-url" - ContractsURLFlagDesc = "URL to archive with compiled FrostFS contracts" - EpochDurationInitFlag = "network.epoch_duration" - MaxObjectSizeInitFlag = "network.max_object_size" - RefillGasAmountFlag = "gas" - StorageWalletFlag = "storage-wallet" - ContainerFeeInitFlag = "network.fee.container" - ContainerAliasFeeInitFlag = "network.fee.container_alias" - CandidateFeeInitFlag = "network.fee.candidate" - WithdrawFeeInitFlag = "network.fee.withdraw" - MaintenanceModeAllowedInitFlag = "network.maintenance_mode_allowed" - HomomorphicHashDisabledInitFlag = "network.homomorphic_hash_disabled" - CustomZoneFlag = "domain" - AlphabetSizeFlag = "size" - SingleAccountName = "single" CommitteeAccountName = "committee" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go index 4305df1e5..19f8bbb68 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go @@ -9,6 +9,7 @@ import ( "path/filepath" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" @@ -79,7 +80,7 @@ func (cs *ContractState) Parse() error { } func NewInitializeContext(cmd *cobra.Command, v *viper.Viper) (*InitializeContext, error) { - walletDir := config.ResolveHomePath(viper.GetString(AlphabetWalletsFlag)) + walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) wallets, err := GetAlphabetWallets(v, walletDir) if err != nil { return nil, err @@ -119,7 +120,7 @@ func NewInitializeContext(cmd *cobra.Command, v *viper.Viper) (*InitializeContex var ctrURL string if needContracts { - ctrURL, _ = cmd.Flags().GetString(ContractsURLFlag) + ctrURL, _ = cmd.Flags().GetString(commonflags.ContractsURLFlag) } if err := CheckNotaryEnabled(c); err != nil { @@ -163,11 +164,11 @@ func validateInit(cmd *cobra.Command) error { if cmd.Name() != "init" { return nil } - if viper.GetInt64(EpochDurationInitFlag) <= 0 { + if viper.GetInt64(commonflags.EpochDurationInitFlag) <= 0 { return fmt.Errorf("epoch duration must be positive") } - if viper.GetInt64(MaxObjectSizeInitFlag) <= 0 { + if viper.GetInt64(commonflags.MaxObjectSizeInitFlag) <= 0 { return fmt.Errorf("max object size must be positive") } @@ -177,9 +178,9 @@ func validateInit(cmd *cobra.Command) error { func createClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) (Client, error) { var c Client var err error - if ldf := cmd.Flags().Lookup(LocalDumpFlag); ldf != nil && ldf.Changed { - if cmd.Flags().Changed(EndpointFlag) { - return nil, fmt.Errorf("`%s` and `%s` flags are mutually exclusive", EndpointFlag, LocalDumpFlag) + if ldf := cmd.Flags().Lookup(commonflags.LocalDumpFlag); ldf != nil && ldf.Changed { + if cmd.Flags().Changed(commonflags.EndpointFlag) { + return nil, fmt.Errorf("`%s` and `%s` flags are mutually exclusive", commonflags.EndpointFlag, commonflags.LocalDumpFlag) } c, err = NewLocalClient(cmd, v, wallets, ldf.Value.String()) } else { @@ -196,7 +197,7 @@ func getContractsPath(cmd *cobra.Command, needContracts bool) (string, error) { return "", nil } - ctrPath, err := cmd.Flags().GetString(ContractsInitFlag) + ctrPath, err := cmd.Flags().GetString(commonflags.ContractsInitFlag) if err != nil { return "", fmt.Errorf("invalid contracts path: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/n3client.go b/cmd/frostfs-adm/internal/modules/morph/util/n3client.go index 0cc751cf1..56a96c410 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/n3client.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -55,7 +56,7 @@ func GetN3Client(v *viper.Viper) (Client, error) { ) ctx := context.Background() - endpoint := v.GetString(EndpointFlag) + endpoint := v.GetString(commonflags.EndpointFlag) if endpoint == "" { return nil, errors.New("missing endpoint") } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/netmap.go b/cmd/frostfs-adm/internal/modules/morph/util/netmap.go index bfe8ea165..5aa304732 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/netmap.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/netmap.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -29,14 +30,14 @@ var NetmapConfigKeys = []string{ func GetDefaultNetmapContractConfigMap() map[string]any { m := make(map[string]any) - m[netmap.EpochDurationConfig] = viper.GetInt64(EpochDurationInitFlag) - m[netmap.MaxObjectSizeConfig] = viper.GetInt64(MaxObjectSizeInitFlag) - m[netmap.ContainerFeeConfig] = viper.GetInt64(ContainerFeeInitFlag) - m[netmap.ContainerAliasFeeConfig] = viper.GetInt64(ContainerAliasFeeInitFlag) - m[netmap.IrCandidateFeeConfig] = viper.GetInt64(CandidateFeeInitFlag) - m[netmap.WithdrawFeeConfig] = viper.GetInt64(WithdrawFeeInitFlag) - m[netmap.HomomorphicHashingDisabledKey] = viper.GetBool(HomomorphicHashDisabledInitFlag) - m[netmap.MaintenanceModeAllowedConfig] = viper.GetBool(MaintenanceModeAllowedInitFlag) + m[netmap.EpochDurationConfig] = viper.GetInt64(commonflags.EpochDurationInitFlag) + m[netmap.MaxObjectSizeConfig] = viper.GetInt64(commonflags.MaxObjectSizeInitFlag) + m[netmap.ContainerFeeConfig] = viper.GetInt64(commonflags.ContainerFeeInitFlag) + m[netmap.ContainerAliasFeeConfig] = viper.GetInt64(commonflags.ContainerAliasFeeInitFlag) + m[netmap.IrCandidateFeeConfig] = viper.GetInt64(commonflags.CandidateFeeInitFlag) + m[netmap.WithdrawFeeConfig] = viper.GetInt64(commonflags.WithdrawFeeInitFlag) + m[netmap.HomomorphicHashingDisabledKey] = viper.GetBool(commonflags.HomomorphicHashDisabledInitFlag) + m[netmap.MaintenanceModeAllowedConfig] = viper.GetBool(commonflags.MaintenanceModeAllowedInitFlag) return m } From e2cee4cf09474f757f64958e25f73e3f605e526d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 2 Feb 2024 15:26:57 +0300 Subject: [PATCH 0905/1943] [#932] adm: Move `const` to package `constants` Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/ape/ape_util.go | 5 +- .../internal/modules/morph/balance/balance.go | 5 +- .../internal/modules/morph/config/config.go | 5 +- .../morph/{util => constants}/const.go | 2 +- .../modules/morph/container/container.go | 5 +- .../internal/modules/morph/contract/deploy.go | 9 +-- .../modules/morph/contract/dump_hashes.go | 5 +- .../internal/modules/morph/contract/update.go | 21 +++---- .../modules/morph/frostfsid/frostfsid.go | 3 +- .../modules/morph/generate/generate.go | 11 ++-- .../modules/morph/generate/generate_test.go | 14 ++--- .../modules/morph/initialize/initialize.go | 3 +- .../morph/initialize/initialize_deploy.go | 11 ++-- .../morph/initialize/initialize_nns.go | 7 ++- .../morph/initialize/initialize_register.go | 5 +- .../morph/initialize/initialize_test.go | 11 ++-- .../morph/initialize/initialize_transfer.go | 7 ++- .../internal/modules/morph/initialize/root.go | 6 +- .../internal/modules/morph/netmap/epoch.go | 3 +- .../modules/morph/netmap/netmap_candidates.go | 3 +- .../internal/modules/morph/node/remove.go | 3 +- .../internal/modules/morph/proxy/proxy.go | 3 +- .../internal/modules/morph/util/contract.go | 57 ++++++++++--------- .../internal/modules/morph/util/initialize.go | 3 +- .../modules/morph/util/initialize_ctx.go | 25 ++++---- .../modules/morph/util/local_client.go | 5 +- .../internal/modules/morph/util/netmap.go | 3 +- .../internal/modules/morph/util/util.go | 5 +- .../internal/modules/morph/util/wallet.go | 9 +-- 29 files changed, 140 insertions(+), 114 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{util => constants}/const.go (98%) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 7dc5fa082..13392b1ce 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -86,7 +87,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *ac wallets, err := morphUtil.GetAlphabetWallets(v, walletDir) commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) - committeeAcc, err := morphUtil.GetWalletAccount(wallets[0], morphUtil.CommitteeAccountName) + committeeAcc, err := morphUtil.GetWalletAccount(wallets[0], constants.CommitteeAccountName) commonCmd.ExitOnErr(cmd, "can't find committee account: %w", err) ac, err := morphUtil.NewActor(c, committeeAcc) @@ -98,7 +99,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *ac nnsCs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) - ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(morphUtil.PolicyContract)) + ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(constants.PolicyContract)) commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err) return morph.NewContractStorage(ac, ch), ac diff --git a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go index 9d16ada46..019c62346 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go @@ -7,6 +7,7 @@ import ( "math/big" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" @@ -64,7 +65,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(morphUtil.NetmapContract)) + nmHash, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(constants.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -137,7 +138,7 @@ func printStorageNodeBalances(cmd *cobra.Command, inv *invoker.Invoker, nmHash u } func printProxyContractBalance(cmd *cobra.Command, inv *invoker.Invoker, nnsHash util.Uint160) error { - h, err := morphUtil.NNSResolveHash(inv, nnsHash, morphUtil.DomainOf(morphUtil.ProxyContract)) + h, err := morphUtil.NNSResolveHash(inv, nnsHash, morphUtil.DomainOf(constants.ProxyContract)) if err != nil { return fmt.Errorf("can't get hash of the proxy contract: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index b1ed1cc21..f1845c9ce 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -10,6 +10,7 @@ import ( "strings" "text/tabwriter" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/io" @@ -38,7 +39,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(util.NetmapContract)) + nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(constants.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -96,7 +97,7 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(util.NetmapContract)) + nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(constants.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/const.go b/cmd/frostfs-adm/internal/modules/morph/constants/const.go similarity index 98% rename from cmd/frostfs-adm/internal/modules/morph/util/const.go rename to cmd/frostfs-adm/internal/modules/morph/constants/const.go index 7c867db13..5a7f8f922 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/constants/const.go @@ -1,4 +1,4 @@ -package util +package constants import "time" diff --git a/cmd/frostfs-adm/internal/modules/morph/container/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go index 46d4921fe..6ba6a3168 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -7,6 +7,7 @@ import ( "os" "sort" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" @@ -36,7 +37,7 @@ func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker) (util.Ui if err != nil { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(morphUtil.ContainerContract)) + ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(constants.ContainerContract)) if err != nil { return util.Uint160{}, err } @@ -307,7 +308,7 @@ func fetchContainerContractHash(wCtx *morphUtil.InitializeContext) (util.Uint160 return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, morphUtil.DomainOf(morphUtil.ContainerContract)) + ch, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, morphUtil.DomainOf(constants.ContainerContract)) if err != nil { return util.Uint160{}, fmt.Errorf("can't fetch container contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go index 67a8d2fac..e7c4255b5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/cli/cmdargs" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -84,7 +85,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { } callHash := management.Hash - method := util.DeployMethodName + method := constants.DeployMethodName zone, _ := cmd.Flags().GetString(commonflags.CustomZoneFlag) domain := ctrName + "." + zone isUpdate, _ := cmd.Flags().GetBool(updateFlag) @@ -94,7 +95,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't fetch contract hash from NNS: %w", err) } callHash = cs.Hash - method = util.UpdateMethodName + method = constants.UpdateMethodName } else { cs.Hash = state.CreateContractHash( c.CommitteeAcc.Contract.ScriptHash(), @@ -147,12 +148,12 @@ func registerNNS(nnsCs *state.Contract, c *util.InitializeContext, zone string, emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, zone, c.CommitteeAcc.Contract.ScriptHash(), - util.FrostfsOpsEmail, int64(3600), int64(600), int64(util.DefaultExpirationTime), int64(3600)) + constants.FrostfsOpsEmail, int64(3600), int64(600), int64(constants.DefaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, domain, c.CommitteeAcc.Contract.ScriptHash(), - util.FrostfsOpsEmail, int64(3600), int64(600), int64(util.DefaultExpirationTime), int64(3600)) + constants.FrostfsOpsEmail, int64(3600), int64(600), int64(constants.DefaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) } else { s, ok, err := c.NNSRegisterDomainScript(nnsCs.Hash, cs.Hash, domain) diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index ca1e5433b..ae7264405 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/io" @@ -51,7 +52,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { return dumpCustomZoneHashes(cmd, cs.Hash, zone, c) } - infos := []contractDumpInfo{{name: morphUtil.NNSContract, hash: cs.Hash}} + infos := []contractDumpInfo{{name: constants.NNSContract, hash: cs.Hash}} irSize := 0 for ; irSize < lastGlagoliticLetter; irSize++ { @@ -87,7 +88,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { } } - for _, ctrName := range morphUtil.ContractList { + for _, ctrName := range constants.ContractList { bw.Reset() emit.AppCall(bw.BinWriter, cs.Hash, "resolve", callflag.ReadOnly, morphUtil.DomainOf(ctrName), int64(nns.TXT)) diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/update.go b/cmd/frostfs-adm/internal/modules/morph/contract/update.go index b1b0bf1b3..33e49fad8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/update.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/update.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/common" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -29,7 +30,7 @@ func updateContracts(cmd *cobra.Command, _ []string) error { return fmt.Errorf("initialization error: %w", err) } - if err := morphUtil.DeployNNS(wCtx, morphUtil.UpdateMethodName); err != nil { + if err := morphUtil.DeployNNS(wCtx, constants.UpdateMethodName); err != nil { return err } @@ -37,7 +38,7 @@ func updateContracts(cmd *cobra.Command, _ []string) error { } func updateContractsInternal(c *morphUtil.InitializeContext) error { - alphaCs := c.GetContract(morphUtil.AlphabetContract) + alphaCs := c.GetContract(constants.AlphabetContract) nnsCs, err := c.NNSContractState() if err != nil { @@ -116,7 +117,7 @@ func deployAlphabetAccounts(c *morphUtil.InitializeContext, nnsHash neoUtil.Uint emit.Opcodes(w.BinWriter, opcode.LDSFLD0) emit.Int(w.BinWriter, 3) emit.Opcodes(w.BinWriter, opcode.PACK) - emit.AppCallNoArgs(w.BinWriter, ctrHash, morphUtil.UpdateMethodName, callflag.All) + emit.AppCallNoArgs(w.BinWriter, ctrHash, constants.UpdateMethodName, callflag.All) } if err := c.SendCommitteeTx(w.Bytes(), false); err != nil { if !strings.Contains(err.Error(), common.ErrAlreadyUpdated) { @@ -134,15 +135,15 @@ func deployOrUpdateContracts(c *morphUtil.InitializeContext, w *io2.BufBinWriter emit.Opcodes(w.BinWriter, opcode.STSFLD0) emit.AppCall(w.BinWriter, nnsHash, "setPrice", callflag.All, 1) - for _, ctrName := range morphUtil.ContractList { + for _, ctrName := range constants.ContractList { cs := c.GetContract(ctrName) - method := morphUtil.UpdateMethodName + method := constants.UpdateMethodName ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, morphUtil.DomainOf(ctrName)) if err != nil { if errors.Is(err, errMissingNNSRecord) { // if contract not found we deploy it instead of update - method = morphUtil.DeployMethodName + method = constants.DeployMethodName } else { return fmt.Errorf("can't resolve hash for contract update: %w", err) } @@ -154,18 +155,18 @@ func deployOrUpdateContracts(c *morphUtil.InitializeContext, w *io2.BufBinWriter } invokeHash := management.Hash - if method == morphUtil.UpdateMethodName { + if method == constants.UpdateMethodName { invokeHash = ctrHash } - args, err := morphUtil.GetContractDeployData(c, ctrName, keysParam, morphUtil.UpdateMethodName) + args, err := morphUtil.GetContractDeployData(c, ctrName, keysParam, constants.UpdateMethodName) if err != nil { return fmt.Errorf("%s: getting update params: %v", ctrName, err) } params := morphUtil.GetContractDeployParameters(cs, args) res, err := c.CommitteeAct.MakeCall(invokeHash, method, params...) if err != nil { - if method != morphUtil.UpdateMethodName || !strings.Contains(err.Error(), common.ErrAlreadyUpdated) { + if method != constants.UpdateMethodName || !strings.Contains(err.Error(), common.ErrAlreadyUpdated) { return fmt.Errorf("deploy contract: %w", err) } c.Command.Printf("%s contract is already updated.\n", ctrName) @@ -174,7 +175,7 @@ func deployOrUpdateContracts(c *morphUtil.InitializeContext, w *io2.BufBinWriter w.WriteBytes(res.Script) - if method == morphUtil.DeployMethodName { + if method == constants.DeployMethodName { // same actions are done in InitializeContext.setNNS, can be unified domain := ctrName + ".frostfs" script, ok, err := c.NNSRegisterDomainScript(nnsHash, cs.Hash, domain) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 0a64b9e16..e5d246b4e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -7,6 +7,7 @@ import ( frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -432,7 +433,7 @@ func newFrostfsIDClient(cmd *cobra.Command) (*frostfsidClient, error) { return nil, fmt.Errorf("can't get NNS contract info: %w", err) } - ffsidHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.FrostfsIDContract)) + ffsidHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(constants.FrostfsIDContract)) if err != nil { return nil, fmt.Errorf("can't get proxy contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go index 8db0aecd2..f032ee708 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -34,7 +35,7 @@ func AlphabetCreds(cmd *cobra.Command, _ []string) error { if size == 0 { return errors.New("size must be > 0") } - if size > morphUtil.MaxAlphabetNodes { + if size > constants.MaxAlphabetNodes { return morphUtil.ErrTooManyAlphabetNodes } @@ -82,7 +83,7 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er if err != nil { return nil, fmt.Errorf("can't create wallet: %w", err) } - if err := w.CreateAccount(morphUtil.SingleAccountName, password); err != nil { + if err := w.CreateAccount(constants.SingleAccountName, password); err != nil { return nil, fmt.Errorf("can't create account: %w", err) } @@ -101,10 +102,10 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er i := i ps := pubs.Copy() errG.Go(func() error { - if err := addMultisigAccount(wallets[i], majCount, morphUtil.CommitteeAccountName, passwords[i], ps); err != nil { + if err := addMultisigAccount(wallets[i], majCount, constants.CommitteeAccountName, passwords[i], ps); err != nil { return fmt.Errorf("can't create committee account: %w", err) } - if err := addMultisigAccount(wallets[i], bftCount, morphUtil.ConsensusAccountName, passwords[i], ps); err != nil { + if err := addMultisigAccount(wallets[i], bftCount, constants.ConsensusAccountName, passwords[i], ps); err != nil { return fmt.Errorf("can't create consentus account: %w", err) } if err := wallets[i].SavePretty(); err != nil { @@ -177,7 +178,7 @@ func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error } if label == "" { - label = morphUtil.SingleAccountName + label = constants.SingleAccountName } if err := w.CreateAccount(label, password); err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go index add00b5ce..d4a4fee5d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go @@ -11,7 +11,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/cli/input" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -69,7 +69,7 @@ func TestGenerateAlphabet(t *testing.T) { buf.WriteString(strconv.FormatUint(i, 10) + "\r") } - buf.WriteString(util.TestContractPassword + "\r") + buf.WriteString(constants.TestContractPassword + "\r") require.NoError(t, AlphabetCreds(GenerateAlphabetCmd, nil)) var wg sync.WaitGroup @@ -87,12 +87,12 @@ func TestGenerateAlphabet(t *testing.T) { err := a.Decrypt(strconv.FormatUint(i, 10), keys.NEP2ScryptParams()) require.NoError(t, err, "can't decrypt account") switch a.Label { - case util.ConsensusAccountName: + case constants.ConsensusAccountName: require.Equal(t, smartcontract.GetDefaultHonestNodeCount(size), len(a.Contract.Parameters)) - case util.CommitteeAccountName: + case constants.CommitteeAccountName: require.Equal(t, smartcontract.GetMajorityHonestNodeCount(size), len(a.Contract.Parameters)) default: - require.Equal(t, util.SingleAccountName, a.Label) + require.Equal(t, constants.SingleAccountName, a.Label) } } }() @@ -100,11 +100,11 @@ func TestGenerateAlphabet(t *testing.T) { wg.Wait() t.Run("check contract group wallet", func(t *testing.T) { - p := filepath.Join(walletDir, util.ContractWalletFilename) + p := filepath.Join(walletDir, constants.ContractWalletFilename) w, err := wallet.NewWalletFromFile(p) require.NoError(t, err, "contract wallet doesn't exist") require.Equal(t, 1, len(w.Accounts), "contract wallet must have 1 accout") - require.NoError(t, w.Accounts[0].Decrypt(util.TestContractPassword, keys.NEP2ScryptParams())) + require.NoError(t, w.Accounts[0].Decrypt(constants.TestContractPassword, keys.NEP2ScryptParams())) }) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go index d1b766019..d3903e226 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go @@ -3,6 +3,7 @@ package initialize import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -28,7 +29,7 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { // 3. Deploy NNS contract. cmd.Println("Stage 3: deploy NNS contract.") - if err := morphUtil.DeployNNS(initCtx, morphUtil.DeployMethodName); err != nil { + if err := morphUtil.DeployNNS(initCtx, constants.DeployMethodName); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_deploy.go index 9374b9f7f..417cab269 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_deploy.go @@ -3,6 +3,7 @@ package initialize import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" @@ -10,7 +11,7 @@ import ( ) func deployContracts(c *morphUtil.InitializeContext) error { - alphaCs := c.GetContract(morphUtil.AlphabetContract) + alphaCs := c.GetContract(constants.AlphabetContract) var keysParam []any @@ -38,7 +39,7 @@ func deployContracts(c *morphUtil.InitializeContext) error { return fmt.Errorf("could not create actor: %w", err) } - txHash, vub, err := act.SendCall(management.Hash, morphUtil.DeployMethodName, params...) + txHash, vub, err := act.SendCall(management.Hash, constants.DeployMethodName, params...) if err != nil { return fmt.Errorf("can't deploy alphabet #%d contract: %w", i, err) } @@ -46,7 +47,7 @@ func deployContracts(c *morphUtil.InitializeContext) error { c.SentTxs = append(c.SentTxs, morphUtil.HashVUBPair{Hash: txHash, Vub: vub}) } - for _, ctrName := range morphUtil.ContractList { + for _, ctrName := range constants.ContractList { cs := c.GetContract(ctrName) ctrHash := cs.Hash @@ -60,12 +61,12 @@ func deployContracts(c *morphUtil.InitializeContext) error { return fmt.Errorf("can't sign manifest group: %v", err) } - args, err := morphUtil.GetContractDeployData(c, ctrName, keysParam, morphUtil.DeployMethodName) + args, err := morphUtil.GetContractDeployData(c, ctrName, keysParam, constants.DeployMethodName) if err != nil { return fmt.Errorf("%s: getting deploy params: %v", ctrName, err) } params := morphUtil.GetContractDeployParameters(cs, args) - res, err := c.CommitteeAct.MakeCall(management.Hash, morphUtil.DeployMethodName, params...) + res, err := c.CommitteeAct.MakeCall(management.Hash, constants.DeployMethodName, params...) if err != nil { return fmt.Errorf("can't deploy %s contract: %w", ctrName, err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go index a9df7127f..c7c1d5b67 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -32,7 +33,7 @@ func setNNS(c *morphUtil.InitializeContext) error { bw := io.NewBufBinWriter() emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, "frostfs", c.CommitteeAcc.Contract.ScriptHash(), - morphUtil.FrostfsOpsEmail, int64(3600), int64(600), int64(morphUtil.DefaultExpirationTime), int64(3600)) + constants.FrostfsOpsEmail, int64(3600), int64(600), int64(constants.DefaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) if err := c.SendCommitteeTx(bw.Bytes(), true); err != nil { return fmt.Errorf("can't add domain root to NNS: %w", err) @@ -42,7 +43,7 @@ func setNNS(c *morphUtil.InitializeContext) error { } } - alphaCs := c.GetContract(morphUtil.AlphabetContract) + alphaCs := c.GetContract(constants.AlphabetContract) for i, acc := range c.Accounts { alphaCs.Hash = state.CreateContractHash(acc.Contract.ScriptHash(), alphaCs.NEF.Checksum, alphaCs.Manifest.Name) @@ -53,7 +54,7 @@ func setNNS(c *morphUtil.InitializeContext) error { c.Command.Printf("NNS: Set %s -> %s\n", domain, alphaCs.Hash.StringLE()) } - for _, ctrName := range morphUtil.ContractList { + for _, ctrName := range constants.ContractList { cs := c.GetContract(ctrName) domain := ctrName + ".frostfs" diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go index cb33aa985..d3a30485e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go @@ -5,6 +5,7 @@ import ( "fmt" "math/big" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -68,7 +69,7 @@ func registerCandidateRange(c *morphUtil.InitializeContext, start, end int) erro if err != nil { return fmt.Errorf("can't create tx: %w", err) } - if err := c.MultiSign(tx, morphUtil.CommitteeAccountName); err != nil { + if err := c.MultiSign(tx, constants.CommitteeAccountName); err != nil { return fmt.Errorf("can't sign a transaction: %w", err) } @@ -123,7 +124,7 @@ func transferNEOToAlphabetContracts(c *morphUtil.InitializeContext) error { return err } - cs := c.GetContract(morphUtil.AlphabetContract) + cs := c.GetContract(constants.AlphabetContract) amount := initialAlphabetNEOAmount / len(c.Wallets) bw := io.NewBufBinWriter() diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go index 88bf0f45f..0114410c7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" cmdConfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/generate" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/node" @@ -49,10 +50,10 @@ func TestInitialize(t *testing.T) { testInitialize(t, 16) }) t.Run("max nodes", func(t *testing.T) { - testInitialize(t, util.MaxAlphabetNodes) + testInitialize(t, constants.MaxAlphabetNodes) }) t.Run("too many nodes", func(t *testing.T) { - require.ErrorIs(t, generateTestData(t, t.TempDir(), util.MaxAlphabetNodes+1), util.ErrTooManyAlphabetNodes) + require.ErrorIs(t, generateTestData(t, t.TempDir(), constants.MaxAlphabetNodes+1), util.ErrTooManyAlphabetNodes) }) } @@ -61,7 +62,7 @@ func testInitialize(t *testing.T, committeeSize int) { v := viper.GetViper() require.NoError(t, generateTestData(t, testdataDir, committeeSize)) - v.Set(util.ProtoConfigPath, filepath.Join(testdataDir, protoFileName)) + v.Set(constants.ProtoConfigPath, filepath.Join(testdataDir, protoFileName)) // Set to the path or remove the next statement to download from the network. require.NoError(t, Cmd.Flags().Set(commonflags.ContractsInitFlag, contractsPath)) @@ -119,7 +120,7 @@ func generateTestData(t *testing.T, dir string, size int) error { return fmt.Errorf("wallet doesn't exist: %w", err) } for _, acc := range w.Accounts { - if acc.Label == util.SingleAccountName { + if acc.Label == constants.SingleAccountName { pub, ok := vm.ParseSignatureContract(acc.Contract.Script) if !ok { return fmt.Errorf("could not parse signature script for %s", acc.Address) @@ -150,7 +151,7 @@ func setTestCredentials(v *viper.Viper, size int) { for i := 0; i < size; i++ { v.Set("credentials."+innerring.GlagoliticLetter(i).String(), strconv.FormatUint(uint64(i), 10)) } - v.Set("credentials.contract", util.TestContractPassword) + v.Set("credentials.contract", constants.TestContractPassword) } func TestNextPollInterval(t *testing.T) { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go index 8e501953b..cad5f53c2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go @@ -4,6 +4,7 @@ import ( "fmt" "math/big" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -67,7 +68,7 @@ func transferFunds(c *morphUtil.InitializeContext) error { return fmt.Errorf("can't create transfer transaction: %w", err) } - if err := c.MultiSignAndSend(tx, morphUtil.ConsensusAccountName); err != nil { + if err := c.MultiSignAndSend(tx, constants.ConsensusAccountName); err != nil { return fmt.Errorf("can't send transfer transaction: %w", err) } @@ -83,7 +84,7 @@ func transferFundsFinished(c *morphUtil.InitializeContext) (bool, error) { } func transferGASToProxy(c *morphUtil.InitializeContext) error { - proxyCs := c.GetContract(morphUtil.ProxyContract) + proxyCs := c.GetContract(constants.ProxyContract) r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) bal, err := r.BalanceOf(proxyCs.Hash) @@ -100,7 +101,7 @@ func transferGASToProxy(c *morphUtil.InitializeContext) error { return err } - if err := c.MultiSignAndSend(tx, morphUtil.CommitteeAccountName); err != nil { + if err := c.MultiSignAndSend(tx, constants.CommitteeAccountName); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/root.go b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go index 54958991e..581bef532 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go @@ -2,7 +2,7 @@ package initialize import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -30,7 +30,7 @@ var Cmd = &cobra.Command{ _ = viper.BindPFlag(commonflags.ContainerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) _ = viper.BindPFlag(commonflags.ContainerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) _ = viper.BindPFlag(commonflags.WithdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) - _ = viper.BindPFlag(util.ProtoConfigPath, cmd.Flags().Lookup(util.ProtoConfigPath)) + _ = viper.BindPFlag(constants.ProtoConfigPath, cmd.Flags().Lookup(constants.ProtoConfigPath)) }, RunE: initializeSideChainCmd, } @@ -46,7 +46,7 @@ func initInitCmd() { // Defaults are taken from neo-preodolenie. Cmd.Flags().Uint64(containerFeeCLIFlag, 1000, "Container registration fee") Cmd.Flags().Uint64(containerAliasFeeCLIFlag, 500, "Container alias fee") - Cmd.Flags().String(util.ProtoConfigPath, "", "Path to the consensus node configuration") + Cmd.Flags().String(constants.ProtoConfigPath, "", "Path to the consensus node configuration") Cmd.Flags().String(commonflags.LocalDumpFlag, "", "Path to the blocks dump file") Cmd.MarkFlagsMutuallyExclusive(commonflags.ContractsInitFlag, commonflags.ContractsURLFlag) } diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go index 45e29c029..8e681cc3b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" @@ -23,7 +24,7 @@ func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(util.NetmapContract)) + nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(constants.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go b/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go index f105fe911..d7ad601dd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go @@ -2,6 +2,7 @@ package netmap import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -21,7 +22,7 @@ func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) { cs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract info: %w", err) - nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(util.NetmapContract)) + nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(constants.NetmapContract)) commonCmd.ExitOnErr(cmd, "can't get netmap contract hash: %w", err) res, err := inv.Call(nmHash, "netmapCandidates") diff --git a/cmd/frostfs-adm/internal/modules/morph/node/remove.go b/cmd/frostfs-adm/internal/modules/morph/node/remove.go index 0e305c1f4..e25cb6efb 100644 --- a/cmd/frostfs-adm/internal/modules/morph/node/remove.go +++ b/cmd/frostfs-adm/internal/modules/morph/node/remove.go @@ -5,6 +5,7 @@ import ( "fmt" netmapcontract "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/io" @@ -41,7 +42,7 @@ func RemoveNodesCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(util.NetmapContract)) + nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(constants.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go index a8fd97be2..2549958ef 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go @@ -3,6 +3,7 @@ package proxy import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -47,7 +48,7 @@ func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error return fmt.Errorf("can't get NNS contract info: %w", err) } - proxyHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(morphUtil.ProxyContract)) + proxyHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(constants.ProxyContract)) if err != nil { return fmt.Errorf("can't get proxy contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/contract.go b/cmd/frostfs-adm/internal/modules/morph/util/contract.go index 8faa49d1c..8f3b20e83 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/contract.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/contract.go @@ -3,6 +3,7 @@ package util import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" @@ -18,7 +19,7 @@ func getFrostfsIDAdminFromContract(roInvoker *invoker.Invoker) (util.Uint160, bo if err != nil { return util.Uint160{}, false, fmt.Errorf("get nns contract: %w", err) } - fidHash, err := NNSResolveHash(roInvoker, cs.Hash, DomainOf(FrostfsIDContract)) + fidHash, err := NNSResolveHash(roInvoker, cs.Hash, DomainOf(constants.FrostfsIDContract)) if err != nil { return util.Uint160{}, false, fmt.Errorf("resolve frostfsid contract hash: %w", err) } @@ -45,19 +46,19 @@ func GetContractDeployData(c *InitializeContext, ctrName string, keysParam []any items := make([]any, 0, 6) switch ctrName { - case FrostfsContract: + case constants.FrostfsContract: items = append(items, - c.Contracts[ProcessingContract].Hash, + c.Contracts[constants.ProcessingContract].Hash, keysParam, smartcontract.Parameter{}) - case ProcessingContract: - items = append(items, c.Contracts[FrostfsContract].Hash) + case constants.ProcessingContract: + items = append(items, c.Contracts[constants.FrostfsContract].Hash) return items[1:], nil // no notary info - case BalanceContract: + case constants.BalanceContract: items = append(items, - c.Contracts[NetmapContract].Hash, - c.Contracts[ContainerContract].Hash) - case ContainerContract: + c.Contracts[constants.NetmapContract].Hash, + c.Contracts[constants.ContainerContract].Hash) + case constants.ContainerContract: // In case if NNS is updated multiple times, we can't calculate // it's actual hash based on local data, thus query chain. r := management.NewReader(c.ReadOnlyInvoker) @@ -66,21 +67,21 @@ func GetContractDeployData(c *InitializeContext, ctrName string, keysParam []any return nil, fmt.Errorf("get nns contract: %w", err) } items = append(items, - c.Contracts[NetmapContract].Hash, - c.Contracts[BalanceContract].Hash, - c.Contracts[FrostfsIDContract].Hash, + c.Contracts[constants.NetmapContract].Hash, + c.Contracts[constants.BalanceContract].Hash, + c.Contracts[constants.FrostfsIDContract].Hash, nnsCs.Hash, "container") - case FrostfsIDContract: + case constants.FrostfsIDContract: var ( h util.Uint160 found bool err error ) - if method == UpdateMethodName { + if method == constants.UpdateMethodName { h, found, err = getFrostfsIDAdminFromContract(c.ReadOnlyInvoker) } - if method != UpdateMethodName || err == nil && !found { + if method != constants.UpdateMethodName || err == nil && !found { h, found, err = GetFrostfsIDAdmin(viper.GetViper()) } if err != nil { @@ -90,11 +91,11 @@ func GetContractDeployData(c *InitializeContext, ctrName string, keysParam []any if found { items = append(items, h) } else { - items = append(items, c.Contracts[ProxyContract].Hash) + items = append(items, c.Contracts[constants.ProxyContract].Hash) } - case NetmapContract: + case constants.NetmapContract: md := GetDefaultNetmapContractConfigMap() - if method == UpdateMethodName { + if method == constants.UpdateMethodName { if err := MergeNetmapConfig(c.ReadOnlyInvoker, md); err != nil { return nil, err } @@ -106,14 +107,14 @@ func GetContractDeployData(c *InitializeContext, ctrName string, keysParam []any } items = append(items, - c.Contracts[BalanceContract].Hash, - c.Contracts[ContainerContract].Hash, + c.Contracts[constants.BalanceContract].Hash, + c.Contracts[constants.ContainerContract].Hash, keysParam, configParam) - case ProxyContract: + case constants.ProxyContract: items = nil - case PolicyContract: - items = append(items, c.Contracts[ProxyContract].Hash) + case constants.PolicyContract: + items = append(items, c.Contracts[constants.ProxyContract].Hash) default: panic(fmt.Sprintf("invalid contract name: %s", ctrName)) } @@ -125,7 +126,7 @@ func GetContractDeployParameters(cs *ContractState, deployData []any) []any { } func DeployNNS(c *InitializeContext, method string) error { - cs := c.GetContract(NNSContract) + cs := c.GetContract(constants.NNSContract) h := cs.Hash nnsCs, err := c.NNSContractState() @@ -134,7 +135,7 @@ func DeployNNS(c *InitializeContext, method string) error { } if nnsCs != nil { if nnsCs.NEF.Checksum == cs.NEF.Checksum { - if method == DeployMethodName { + if method == constants.DeployMethodName { c.Command.Println("NNS contract is already deployed.") } else { c.Command.Println("NNS contract is already updated.") @@ -152,16 +153,16 @@ func DeployNNS(c *InitializeContext, method string) error { params := GetContractDeployParameters(cs, nil) invokeHash := management.Hash - if method == UpdateMethodName { + if method == constants.UpdateMethodName { invokeHash = nnsCs.Hash } tx, err := c.CommitteeAct.MakeCall(invokeHash, method, params...) if err != nil { - return fmt.Errorf("failed to create deploy tx for %s: %w", NNSContract, err) + return fmt.Errorf("failed to create deploy tx for %s: %w", constants.NNSContract, err) } - if err := c.MultiSignAndSend(tx, CommitteeAccountName); err != nil { + if err := c.MultiSignAndSend(tx, constants.CommitteeAccountName); err != nil { return fmt.Errorf("can't send deploy transaction: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go index 0e4c22644..2d676327d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/initialize.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "github.com/nspcc-dev/neo-go/pkg/core/native/nativenames" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -23,7 +24,7 @@ import ( "github.com/spf13/viper" ) -var ErrTooManyAlphabetNodes = fmt.Errorf("too many alphabet nodes (maximum allowed is %d)", MaxAlphabetNodes) +var ErrTooManyAlphabetNodes = fmt.Errorf("too many alphabet nodes (maximum allowed is %d)", constants.MaxAlphabetNodes) func AwaitTx(cmd *cobra.Command, c Client, txs []HashVUBPair) error { cmd.Println("Waiting for transactions to persist...") diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go index 19f8bbb68..907ba57fe 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -99,12 +100,12 @@ func NewInitializeContext(cmd *cobra.Command, v *viper.Viper) (*InitializeContex return nil, err } - committeeAcc, err := GetWalletAccount(wallets[0], CommitteeAccountName) + committeeAcc, err := GetWalletAccount(wallets[0], constants.CommitteeAccountName) if err != nil { return nil, fmt.Errorf("can't find committee account: %w", err) } - consensusAcc, err := GetWalletAccount(wallets[0], ConsensusAccountName) + consensusAcc, err := GetWalletAccount(wallets[0], constants.ConsensusAccountName) if err != nil { return nil, fmt.Errorf("can't find consensus account: %w", err) } @@ -151,7 +152,7 @@ func NewInitializeContext(cmd *cobra.Command, v *viper.Viper) (*InitializeContex } if needContracts { - err := readContracts(initCtx, FullContractList) + err := readContracts(initCtx, constants.FullContractList) if err != nil { return nil, err } @@ -207,7 +208,7 @@ func getContractsPath(cmd *cobra.Command, needContracts bool) (string, error) { func createWalletAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) { accounts := make([]*wallet.Account, len(wallets)) for i, w := range wallets { - acc, err := GetWalletAccount(w, SingleAccountName) + acc, err := GetWalletAccount(w, constants.SingleAccountName) if err != nil { return nil, fmt.Errorf("wallet %s is invalid (no single account): %w", w.Path(), err) } @@ -263,7 +264,7 @@ func readContracts(c *InitializeContext, names []string) error { } for _, ctrName := range names { - if ctrName != AlphabetContract { + if ctrName != constants.AlphabetContract { cs := c.Contracts[ctrName] cs.Hash = state.CreateContractHash(c.CommitteeAcc.Contract.ScriptHash(), cs.NEF.Checksum, cs.Manifest.Name) @@ -382,11 +383,11 @@ func (c *InitializeContext) sendMultiTx(script []byte, tryGroup bool, withConsen return fmt.Errorf("could not perform test invocation: %w", err) } - if err := c.MultiSign(tx, CommitteeAccountName); err != nil { + if err := c.MultiSign(tx, constants.CommitteeAccountName); err != nil { return err } if withConsensus { - if err := c.MultiSign(tx, ConsensusAccountName); err != nil { + if err := c.MultiSign(tx, constants.ConsensusAccountName); err != nil { return err } } @@ -414,7 +415,7 @@ func (c *InitializeContext) MultiSign(tx *transaction.Transaction, accType strin // Use parameter context to avoid dealing with signature order. pc := context.NewParameterContext("", network, tx) h := c.CommitteeAcc.Contract.ScriptHash() - if accType == ConsensusAccountName { + if accType == constants.ConsensusAccountName { h = c.ConsensusAcc.Contract.ScriptHash() } for _, w := range c.Wallets { @@ -477,7 +478,7 @@ func (c *InitializeContext) EmitUpdateNNSGroupScript(bw *io.BufBinWriter, nnsHas if isAvail { emit.AppCall(bw.BinWriter, nnsHash, "register", callflag.All, client.NNSGroupKeyName, c.CommitteeAcc.Contract.ScriptHash(), - FrostfsOpsEmail, int64(3600), int64(600), int64(DefaultExpirationTime), int64(3600)) + constants.FrostfsOpsEmail, int64(3600), int64(600), int64(constants.DefaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) } @@ -498,7 +499,7 @@ func (c *InitializeContext) NNSRegisterDomainScript(nnsHash, expectedHash util.U bw := io.NewBufBinWriter() emit.AppCall(bw.BinWriter, nnsHash, "register", callflag.All, domain, c.CommitteeAcc.Contract.ScriptHash(), - FrostfsOpsEmail, int64(3600), int64(600), int64(DefaultExpirationTime), int64(3600)) + constants.FrostfsOpsEmail, int64(3600), int64(600), int64(constants.DefaultExpirationTime), int64(3600)) emit.Opcodes(bw.BinWriter, opcode.ASSERT) if bw.Err != nil { @@ -535,8 +536,8 @@ func (c *InitializeContext) GetContract(ctrName string) *ContractState { func (c *InitializeContext) GetAlphabetDeployItems(i, n int) []any { items := make([]any, 5) - items[0] = c.Contracts[NetmapContract].Hash - items[1] = c.Contracts[ProxyContract].Hash + items[0] = c.Contracts[constants.NetmapContract].Hash + items[1] = c.Contracts[constants.ProxyContract].Hash items[2] = innerring.GlagoliticLetter(i).String() items[3] = int64(i) items[4] = int64(n) diff --git a/cmd/frostfs-adm/internal/modules/morph/util/local_client.go b/cmd/frostfs-adm/internal/modules/morph/util/local_client.go index 4b001ca84..35fb07cc4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/local_client.go @@ -8,6 +8,7 @@ import ( "sort" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/core" @@ -47,7 +48,7 @@ type LocalClient struct { } func NewLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet, dumpPath string) (*LocalClient, error) { - cfg, err := config.LoadFile(v.GetString(ProtoConfigPath)) + cfg, err := config.LoadFile(v.GetString(constants.ProtoConfigPath)) if err != nil { return nil, err } @@ -60,7 +61,7 @@ func NewLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet m := smartcontract.GetDefaultHonestNodeCount(int(cfg.ProtocolConfiguration.ValidatorsCount)) accounts := make([]*wallet.Account, len(wallets)) for i := range accounts { - accounts[i], err = GetWalletAccount(wallets[i], ConsensusAccountName) + accounts[i], err = GetWalletAccount(wallets[i], constants.ConsensusAccountName) if err != nil { return nil, err } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/netmap.go b/cmd/frostfs-adm/internal/modules/morph/util/netmap.go index 5aa304732..fbce8a0fa 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/netmap.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/netmap.go @@ -5,6 +5,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -88,7 +89,7 @@ func GetNetConfigFromNetmapContract(roInvoker *invoker.Invoker) ([]stackitem.Ite if err != nil { return nil, fmt.Errorf("get nns contract: %w", err) } - nmHash, err := NNSResolveHash(roInvoker, cs.Hash, DomainOf(NetmapContract)) + nmHash, err := NNSResolveHash(roInvoker, cs.Hash, DomainOf(constants.NetmapContract)) if err != nil { return nil, fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/util.go b/cmd/frostfs-adm/internal/modules/morph/util/util.go index 1fd9695c3..d85fa698e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/util.go @@ -12,6 +12,7 @@ import ( "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -27,7 +28,7 @@ func GetAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, err return nil, err } - if len(wallets) > MaxAlphabetNodes { + if len(wallets) > constants.MaxAlphabetNodes { return nil, ErrTooManyAlphabetNodes } return wallets, nil @@ -161,7 +162,7 @@ func readContractsFromArchive(file io.Reader, names []string) (map[string]*Contr } func GetAlphabetNNSDomain(i int) string { - return AlphabetContract + strconv.FormatUint(uint64(i), 10) + ".frostfs" + return constants.AlphabetContract + strconv.FormatUint(uint64(i), 10) + ".frostfs" } func ParseGASAmount(s string) (fixedn.Fixed8, error) { diff --git a/cmd/frostfs-adm/internal/modules/morph/util/wallet.go b/cmd/frostfs-adm/internal/modules/morph/util/wallet.go index e0e553d06..ca722f241 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/wallet.go +++ b/cmd/frostfs-adm/internal/modules/morph/util/wallet.go @@ -6,6 +6,7 @@ import ( "path/filepath" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" @@ -13,12 +14,12 @@ import ( ) func InitializeContractWallet(v *viper.Viper, walletDir string) (*wallet.Wallet, error) { - password, err := config.GetPassword(v, ContractWalletPasswordKey) + password, err := config.GetPassword(v, constants.ContractWalletPasswordKey) if err != nil { return nil, err } - w, err := wallet.NewWallet(filepath.Join(walletDir, ContractWalletFilename)) + w, err := wallet.NewWallet(filepath.Join(walletDir, constants.ContractWalletFilename)) if err != nil { return nil, err } @@ -42,7 +43,7 @@ func InitializeContractWallet(v *viper.Viper, walletDir string) (*wallet.Wallet, } func openContractWallet(v *viper.Viper, cmd *cobra.Command, walletDir string) (*wallet.Wallet, error) { - p := filepath.Join(walletDir, ContractWalletFilename) + p := filepath.Join(walletDir, constants.ContractWalletFilename) w, err := wallet.NewWalletFromFile(p) if err != nil { if !os.IsNotExist(err) { @@ -53,7 +54,7 @@ func openContractWallet(v *viper.Viper, cmd *cobra.Command, walletDir string) (* return InitializeContractWallet(v, walletDir) } - password, err := config.GetPassword(v, ContractWalletPasswordKey) + password, err := config.GetPassword(v, constants.ContractWalletPasswordKey) if err != nil { return nil, err } From 802192cfef65640f5de193270384d58a6ec1117d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 2 Feb 2024 15:36:14 +0300 Subject: [PATCH 0906/1943] [#932] adm: Rename `util` to `helper` To avoid conflicts with `util` packages in other imports. Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/ape/ape_util.go | 12 +++++----- .../internal/modules/morph/balance/balance.go | 18 +++++++------- .../internal/modules/morph/config/config.go | 14 +++++------ .../modules/morph/container/container.go | 18 +++++++------- .../internal/modules/morph/contract/deploy.go | 10 ++++---- .../modules/morph/contract/dump_hashes.go | 20 ++++++++-------- .../internal/modules/morph/contract/update.go | 24 +++++++++---------- .../modules/morph/frostfsid/frostfsid.go | 8 +++---- .../morph/frostfsid/frostfsid_util_test.go | 8 +++---- .../modules/morph/generate/generate.go | 10 ++++---- .../morph/{util => helper}/contract.go | 2 +- .../morph/{util => helper}/download.go | 2 +- .../morph/{util => helper}/frostfsid.go | 2 +- .../modules/morph/{util => helper}/group.go | 2 +- .../morph/{util => helper}/initialize.go | 2 +- .../morph/{util => helper}/initialize_ctx.go | 2 +- .../morph/{util => helper}/local_client.go | 2 +- .../morph/{util => helper}/n3client.go | 2 +- .../modules/morph/{util => helper}/netmap.go | 2 +- .../modules/morph/{util => helper}/util.go | 2 +- .../modules/morph/{util => helper}/wallet.go | 2 +- .../modules/morph/initialize/initialize.go | 6 ++--- .../morph/initialize/initialize_deploy.go | 16 ++++++------- .../morph/initialize/initialize_nns.go | 10 ++++---- .../morph/initialize/initialize_register.go | 14 +++++------ .../morph/initialize/initialize_roles.go | 8 +++---- .../morph/initialize/initialize_test.go | 16 ++++++------- .../morph/initialize/initialize_transfer.go | 10 ++++---- .../internal/modules/morph/netmap/epoch.go | 8 +++---- .../modules/morph/netmap/netmap_candidates.go | 6 ++--- .../internal/modules/morph/node/remove.go | 8 +++---- .../internal/modules/morph/notary/notary.go | 10 ++++---- .../internal/modules/morph/policy/policy.go | 6 ++--- .../internal/modules/morph/proxy/proxy.go | 6 ++--- 34 files changed, 144 insertions(+), 144 deletions(-) rename cmd/frostfs-adm/internal/modules/morph/{util => helper}/contract.go (99%) rename cmd/frostfs-adm/internal/modules/morph/{util => helper}/download.go (99%) rename cmd/frostfs-adm/internal/modules/morph/{util => helper}/frostfsid.go (98%) rename cmd/frostfs-adm/internal/modules/morph/{util => helper}/group.go (98%) rename cmd/frostfs-adm/internal/modules/morph/{util => helper}/initialize.go (99%) rename cmd/frostfs-adm/internal/modules/morph/{util => helper}/initialize_ctx.go (99%) rename cmd/frostfs-adm/internal/modules/morph/{util => helper}/local_client.go (99%) rename cmd/frostfs-adm/internal/modules/morph/{util => helper}/n3client.go (99%) rename cmd/frostfs-adm/internal/modules/morph/{util => helper}/netmap.go (99%) rename cmd/frostfs-adm/internal/modules/morph/{util => helper}/util.go (99%) rename cmd/frostfs-adm/internal/modules/morph/{util => helper}/wallet.go (99%) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 13392b1ce..bbbdf7805 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -8,7 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -80,17 +80,17 @@ func parseChain(cmd *cobra.Command) *apechain.Chain { func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *actor.Actor) { v := viper.GetViper() - c, err := morphUtil.GetN3Client(v) + c, err := helper.GetN3Client(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) - wallets, err := morphUtil.GetAlphabetWallets(v, walletDir) + wallets, err := helper.GetAlphabetWallets(v, walletDir) commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) - committeeAcc, err := morphUtil.GetWalletAccount(wallets[0], constants.CommitteeAccountName) + committeeAcc, err := helper.GetWalletAccount(wallets[0], constants.CommitteeAccountName) commonCmd.ExitOnErr(cmd, "can't find committee account: %w", err) - ac, err := morphUtil.NewActor(c, committeeAcc) + ac, err := helper.NewActor(c, committeeAcc) commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) inv := &ac.Invoker @@ -99,7 +99,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *ac nnsCs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) - ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(constants.PolicyContract)) + ch, err = helper.NNSResolveHash(inv, nnsCs.Hash, helper.DomainOf(constants.PolicyContract)) commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err) return morph.NewContractStorage(ac, ch), ac diff --git a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go index 019c62346..428fb98b9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go @@ -8,7 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -51,7 +51,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { nmHash util.Uint160 ) - c, err := morphUtil.GetN3Client(viper.GetViper()) + c, err := helper.GetN3Client(viper.GetViper()) if err != nil { return err } @@ -65,7 +65,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(constants.NetmapContract)) + nmHash, err = helper.NNSResolveHash(inv, nnsCs.Hash, helper.DomainOf(constants.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -138,7 +138,7 @@ func printStorageNodeBalances(cmd *cobra.Command, inv *invoker.Invoker, nmHash u } func printProxyContractBalance(cmd *cobra.Command, inv *invoker.Invoker, nnsHash util.Uint160) error { - h, err := morphUtil.NNSResolveHash(inv, nnsHash, morphUtil.DomainOf(constants.ProxyContract)) + h, err := helper.NNSResolveHash(inv, nnsHash, helper.DomainOf(constants.ProxyContract)) if err != nil { return fmt.Errorf("can't get hash of the proxy contract: %w", err) } @@ -152,13 +152,13 @@ func printProxyContractBalance(cmd *cobra.Command, inv *invoker.Invoker, nnsHash return nil } -func printAlphabetContractBalances(cmd *cobra.Command, c morphUtil.Client, inv *invoker.Invoker, count int, nnsHash util.Uint160) error { +func printAlphabetContractBalances(cmd *cobra.Command, c helper.Client, inv *invoker.Invoker, count int, nnsHash util.Uint160) error { alphaList := make([]accBalancePair, count) w := io.NewBufBinWriter() for i := range alphaList { emit.AppCall(w.BinWriter, nnsHash, "resolve", callflag.ReadOnly, - morphUtil.GetAlphabetNNSDomain(i), + helper.GetAlphabetNNSDomain(i), int64(nns.TXT)) } if w.Err != nil { @@ -171,7 +171,7 @@ func printAlphabetContractBalances(cmd *cobra.Command, c morphUtil.Client, inv * } for i := range alphaList { - h, err := morphUtil.ParseNNSResolveResult(alphaRes.Stack[i]) + h, err := helper.ParseNNSResolveResult(alphaRes.Stack[i]) if err != nil { return fmt.Errorf("can't fetch the alphabet contract #%d hash: %w", i, err) } @@ -186,7 +186,7 @@ func printAlphabetContractBalances(cmd *cobra.Command, c morphUtil.Client, inv * return nil } -func fetchIRNodes(c morphUtil.Client, desigHash util.Uint160) ([]accBalancePair, error) { +func fetchIRNodes(c helper.Client, desigHash util.Uint160) ([]accBalancePair, error) { inv := invoker.New(c, nil) height, err := c.GetBlockCount() @@ -194,7 +194,7 @@ func fetchIRNodes(c morphUtil.Client, desigHash util.Uint160) ([]accBalancePair, return nil, fmt.Errorf("can't get block height: %w", err) } - arr, err := morphUtil.GetDesignatedByRole(inv, desigHash, noderoles.NeoFSAlphabet, height) + arr, err := helper.GetDesignatedByRole(inv, desigHash, noderoles.NeoFSAlphabet, height) if err != nil { return nil, errors.New("can't fetch list of IR nodes from the netmap contract") } diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index f1845c9ce..3414a8f7f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -11,7 +11,7 @@ import ( "text/tabwriter" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -26,7 +26,7 @@ import ( const forceConfigSet = "force" func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { - c, err := util.GetN3Client(viper.GetViper()) + c, err := helper.GetN3Client(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } @@ -39,7 +39,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(constants.NetmapContract)) + nmHash, err := helper.NNSResolveHash(inv, cs.Hash, helper.DomainOf(constants.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -52,7 +52,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { buf := bytes.NewBuffer(nil) tw := tabwriter.NewWriter(buf, 0, 2, 2, ' ', 0) - m, err := util.ParseConfigFromNetmapContract(arr) + m, err := helper.ParseConfigFromNetmapContract(arr) if err != nil { return err } @@ -67,7 +67,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%d (int)\n", k, n))) case netmap.HomomorphicHashingDisabledKey, netmap.MaintenanceModeAllowedConfig: if len(v) == 0 || len(v) > 1 { - return util.InvalidConfigValueErr(k) + return helper.InvalidConfigValueErr(k) } _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%t (bool)\n", k, v[0] == 1))) default: @@ -86,7 +86,7 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { return errors.New("empty config pairs") } - wCtx, err := util.NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't initialize context: %w", err) } @@ -97,7 +97,7 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(constants.NetmapContract)) + nmHash, err := helper.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, helper.DomainOf(constants.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/container/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go index 6ba6a3168..690db2f4f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -8,7 +8,7 @@ import ( "sort" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" "github.com/nspcc-dev/neo-go/pkg/io" @@ -37,7 +37,7 @@ func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker) (util.Ui if err != nil { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err = morphUtil.NNSResolveHash(inv, nnsCs.Hash, morphUtil.DomainOf(constants.ContainerContract)) + ch, err = helper.NNSResolveHash(inv, nnsCs.Hash, helper.DomainOf(constants.ContainerContract)) if err != nil { return util.Uint160{}, err } @@ -75,7 +75,7 @@ func dumpContainers(cmd *cobra.Command, _ []string) error { return fmt.Errorf("invalid filename: %w", err) } - c, err := morphUtil.GetN3Client(viper.GetViper()) + c, err := helper.GetN3Client(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } @@ -165,7 +165,7 @@ func dumpSingleContainer(bw *io.BufBinWriter, ch util.Uint160, inv *invoker.Invo } func listContainers(cmd *cobra.Command, _ []string) error { - c, err := morphUtil.GetN3Client(viper.GetViper()) + c, err := helper.GetN3Client(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } @@ -194,7 +194,7 @@ func restoreContainers(cmd *cobra.Command, _ []string) error { return fmt.Errorf("invalid filename: %w", err) } - wCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return err } @@ -223,7 +223,7 @@ func restoreContainers(cmd *cobra.Command, _ []string) error { return wCtx.AwaitTx() } -func restoreOrPutContainers(containers []Container, isOK func([]byte) bool, cmd *cobra.Command, wCtx *morphUtil.InitializeContext, ch util.Uint160) error { +func restoreOrPutContainers(containers []Container, isOK func([]byte) bool, cmd *cobra.Command, wCtx *helper.InitializeContext, ch util.Uint160) error { bw := io.NewBufBinWriter() for _, cnt := range containers { hv := hash.Sha256(cnt.Value) @@ -263,7 +263,7 @@ func putContainer(bw *io.BufBinWriter, ch util.Uint160, cnt Container) { } } -func isContainerRestored(cmd *cobra.Command, wCtx *morphUtil.InitializeContext, containerHash util.Uint160, bw *io.BufBinWriter, hashValue util.Uint256) (bool, error) { +func isContainerRestored(cmd *cobra.Command, wCtx *helper.InitializeContext, containerHash util.Uint160, bw *io.BufBinWriter, hashValue util.Uint256) (bool, error) { emit.AppCall(bw.BinWriter, containerHash, "get", callflag.All, hashValue.BytesBE()) res, err := wCtx.Client.InvokeScript(bw.Bytes(), nil) if err != nil { @@ -301,14 +301,14 @@ func parseContainers(filename string) ([]Container, error) { return containers, nil } -func fetchContainerContractHash(wCtx *morphUtil.InitializeContext) (util.Uint160, error) { +func fetchContainerContractHash(wCtx *helper.InitializeContext) (util.Uint160, error) { r := management.NewReader(wCtx.ReadOnlyInvoker) nnsCs, err := r.GetContractByID(1) if err != nil { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } - ch, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, morphUtil.DomainOf(constants.ContainerContract)) + ch, err := helper.NNSResolveHash(wCtx.ReadOnlyInvoker, nnsCs.Hash, helper.DomainOf(constants.ContainerContract)) if err != nil { return util.Uint160{}, fmt.Errorf("can't fetch container contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go index e7c4255b5..9a92e8f19 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/nspcc-dev/neo-go/cli/cmdargs" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -61,7 +61,7 @@ func init() { func deployContractCmd(cmd *cobra.Command, args []string) error { v := viper.GetViper() - c, err := util.NewInitializeContext(cmd, v) + c, err := helper.NewInitializeContext(cmd, v) if err != nil { return fmt.Errorf("initialization error: %w", err) } @@ -73,7 +73,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { return err } - cs, err := util.ReadContract(ctrPath, ctrName) + cs, err := helper.ReadContract(ctrPath, ctrName) if err != nil { return err } @@ -90,7 +90,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { domain := ctrName + "." + zone isUpdate, _ := cmd.Flags().GetBool(updateFlag) if isUpdate { - cs.Hash, err = util.NNSResolveHash(c.ReadOnlyInvoker, nnsCs.Hash, domain) + cs.Hash, err = helper.NNSResolveHash(c.ReadOnlyInvoker, nnsCs.Hash, domain) if err != nil { return fmt.Errorf("can't fetch contract hash from NNS: %w", err) } @@ -130,7 +130,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { return c.AwaitTx() } -func registerNNS(nnsCs *state.Contract, c *util.InitializeContext, zone string, domain string, cs *util.ContractState, writer *io.BufBinWriter) error { +func registerNNS(nnsCs *state.Contract, c *helper.InitializeContext, zone string, domain string, cs *helper.ContractState, writer *io.BufBinWriter) error { bw := io.NewBufBinWriter() emit.Instruction(bw.BinWriter, opcode.INITSSLOT, []byte{1}) emit.AppCall(bw.BinWriter, nnsCs.Hash, "getPrice", callflag.All) diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index ae7264405..6e406be56 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -36,7 +36,7 @@ type contractDumpInfo struct { } func dumpContractHashes(cmd *cobra.Command, _ []string) error { - c, err := morphUtil.GetN3Client(viper.GetViper()) + c, err := helper.GetN3Client(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } @@ -56,7 +56,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { irSize := 0 for ; irSize < lastGlagoliticLetter; irSize++ { - ok, err := morphUtil.NNSIsAvailable(c, cs.Hash, morphUtil.GetAlphabetNNSDomain(irSize)) + ok, err := helper.NNSIsAvailable(c, cs.Hash, helper.GetAlphabetNNSDomain(irSize)) if err != nil { return err } else if ok { @@ -70,7 +70,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { bw.Reset() for i := 0; i < irSize; i++ { emit.AppCall(bw.BinWriter, cs.Hash, "resolve", callflag.ReadOnly, - morphUtil.GetAlphabetNNSDomain(i), + helper.GetAlphabetNNSDomain(i), int64(nns.TXT)) } @@ -81,7 +81,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { for i := 0; i < irSize; i++ { info := contractDumpInfo{name: fmt.Sprintf("alphabet %d", i)} - if h, err := morphUtil.ParseNNSResolveResult(alphaRes.Stack[i]); err == nil { + if h, err := helper.ParseNNSResolveResult(alphaRes.Stack[i]); err == nil { info.hash = h } infos = append(infos, info) @@ -91,7 +91,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { for _, ctrName := range constants.ContractList { bw.Reset() emit.AppCall(bw.BinWriter, cs.Hash, "resolve", callflag.ReadOnly, - morphUtil.DomainOf(ctrName), int64(nns.TXT)) + helper.DomainOf(ctrName), int64(nns.TXT)) res, err := c.InvokeScript(bw.Bytes(), nil) if err != nil { @@ -100,7 +100,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { info := contractDumpInfo{name: ctrName} if len(res.Stack) != 0 { - if h, err := morphUtil.ParseNNSResolveResult(res.Stack[0]); err == nil { + if h, err := helper.ParseNNSResolveResult(res.Stack[0]); err == nil { info.hash = h } } @@ -113,7 +113,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { return nil } -func dumpCustomZoneHashes(cmd *cobra.Command, nnsHash util.Uint160, zone string, c morphUtil.Client) error { +func dumpCustomZoneHashes(cmd *cobra.Command, nnsHash util.Uint160, zone string, c helper.Client) error { const nnsMaxTokens = 100 inv := invoker.New(c, nil) @@ -135,7 +135,7 @@ func dumpCustomZoneHashes(cmd *cobra.Command, nnsHash util.Uint160, zone string, return } - h, err := morphUtil.NNSResolveHash(inv, nnsHash, string(bs)) + h, err := helper.NNSResolveHash(inv, nnsHash, string(bs)) if err != nil { cmd.PrintErrf("Could not resolve name %s: %v\n", string(bs), err) return @@ -227,7 +227,7 @@ func printContractInfo(cmd *cobra.Command, infos []contractDumpInfo) { cmd.Print(buf.String()) } -func fillContractVersion(cmd *cobra.Command, c morphUtil.Client, infos []contractDumpInfo) { +func fillContractVersion(cmd *cobra.Command, c helper.Client, infos []contractDumpInfo) { bw := io.NewBufBinWriter() sub := io.NewBufBinWriter() for i := range infos { diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/update.go b/cmd/frostfs-adm/internal/modules/morph/contract/update.go index 33e49fad8..109849aab 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/update.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/update.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/common" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/encoding/address" io2 "github.com/nspcc-dev/neo-go/pkg/io" @@ -25,19 +25,19 @@ import ( var errMissingNNSRecord = errors.New("missing NNS record") func updateContracts(cmd *cobra.Command, _ []string) error { - wCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("initialization error: %w", err) } - if err := morphUtil.DeployNNS(wCtx, constants.UpdateMethodName); err != nil { + if err := helper.DeployNNS(wCtx, constants.UpdateMethodName); err != nil { return err } return updateContractsInternal(wCtx) } -func updateContractsInternal(c *morphUtil.InitializeContext) error { +func updateContractsInternal(c *helper.InitializeContext) error { alphaCs := c.GetContract(constants.AlphabetContract) nnsCs, err := c.NNSContractState() @@ -90,14 +90,14 @@ func updateContractsInternal(c *morphUtil.InitializeContext) error { return c.AwaitTx() } -func deployAlphabetAccounts(c *morphUtil.InitializeContext, nnsHash neoUtil.Uint160, w *io2.BufBinWriter, alphaCs *morphUtil.ContractState) ([]any, error) { +func deployAlphabetAccounts(c *helper.InitializeContext, nnsHash neoUtil.Uint160, w *io2.BufBinWriter, alphaCs *helper.ContractState) ([]any, error) { var keysParam []any baseGroups := alphaCs.Manifest.Groups // alphabet contracts should be deployed by individual nodes to get different hashes. for i, acc := range c.Accounts { - ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, morphUtil.GetAlphabetNNSDomain(i)) + ctrHash, err := helper.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, helper.GetAlphabetNNSDomain(i)) if err != nil { return nil, fmt.Errorf("can't resolve hash for contract update: %w", err) } @@ -108,7 +108,7 @@ func deployAlphabetAccounts(c *morphUtil.InitializeContext, nnsHash neoUtil.Uint emit.Array(w.BinWriter, params...) alphaCs.Manifest.Groups = baseGroups - err = morphUtil.AddManifestGroup(c.ContractWallet, ctrHash, alphaCs) + err = helper.AddManifestGroup(c.ContractWallet, ctrHash, alphaCs) if err != nil { return nil, fmt.Errorf("can't sign manifest group: %v", err) } @@ -129,7 +129,7 @@ func deployAlphabetAccounts(c *morphUtil.InitializeContext, nnsHash neoUtil.Uint return keysParam, nil } -func deployOrUpdateContracts(c *morphUtil.InitializeContext, w *io2.BufBinWriter, nnsHash neoUtil.Uint160, keysParam []any) error { +func deployOrUpdateContracts(c *helper.InitializeContext, w *io2.BufBinWriter, nnsHash neoUtil.Uint160, keysParam []any) error { emit.Instruction(w.BinWriter, opcode.INITSSLOT, []byte{1}) emit.AppCall(w.BinWriter, nnsHash, "getPrice", callflag.All) emit.Opcodes(w.BinWriter, opcode.STSFLD0) @@ -139,7 +139,7 @@ func deployOrUpdateContracts(c *morphUtil.InitializeContext, w *io2.BufBinWriter cs := c.GetContract(ctrName) method := constants.UpdateMethodName - ctrHash, err := morphUtil.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, morphUtil.DomainOf(ctrName)) + ctrHash, err := helper.NNSResolveHash(c.ReadOnlyInvoker, nnsHash, helper.DomainOf(ctrName)) if err != nil { if errors.Is(err, errMissingNNSRecord) { // if contract not found we deploy it instead of update @@ -149,7 +149,7 @@ func deployOrUpdateContracts(c *morphUtil.InitializeContext, w *io2.BufBinWriter } } - err = morphUtil.AddManifestGroup(c.ContractWallet, ctrHash, cs) + err = helper.AddManifestGroup(c.ContractWallet, ctrHash, cs) if err != nil { return fmt.Errorf("can't sign manifest group: %v", err) } @@ -159,11 +159,11 @@ func deployOrUpdateContracts(c *morphUtil.InitializeContext, w *io2.BufBinWriter invokeHash = ctrHash } - args, err := morphUtil.GetContractDeployData(c, ctrName, keysParam, constants.UpdateMethodName) + args, err := helper.GetContractDeployData(c, ctrName, keysParam, constants.UpdateMethodName) if err != nil { return fmt.Errorf("%s: getting update params: %v", ctrName, err) } - params := morphUtil.GetContractDeployParameters(cs, args) + params := helper.GetContractDeployParameters(cs, args) res, err := c.CommitteeAct.MakeCall(invokeHash, method, params...) if err != nil { if method != constants.UpdateMethodName || !strings.Contains(err.Error(), common.ErrAlreadyUpdated) { diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index e5d246b4e..0c791d391 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -8,7 +8,7 @@ import ( frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -418,11 +418,11 @@ type frostfsidClient struct { bw *io.BufBinWriter contractHash util.Uint160 roCli *frostfsidclient.Client // client can be used only for waiting tx, parsing and forming method params - wCtx *morphUtil.InitializeContext + wCtx *helper.InitializeContext } func newFrostfsIDClient(cmd *cobra.Command) (*frostfsidClient, error) { - wCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return nil, fmt.Errorf("can't to initialize context: %w", err) } @@ -433,7 +433,7 @@ func newFrostfsIDClient(cmd *cobra.Command) (*frostfsidClient, error) { return nil, fmt.Errorf("can't get NNS contract info: %w", err) } - ffsidHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(constants.FrostfsIDContract)) + ffsidHash, err := helper.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, helper.DomainOf(constants.FrostfsIDContract)) if err != nil { return nil, fmt.Errorf("can't get proxy contract hash: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go index fa6d07bf2..cce859d2f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go @@ -4,7 +4,7 @@ import ( "encoding/hex" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/ape" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -31,7 +31,7 @@ func TestFrostfsIDConfig(t *testing.T) { v := viper.New() v.Set("frostfsid.admin", fmts[i]) - actual, found, err := util.GetFrostfsIDAdmin(v) + actual, found, err := helper.GetFrostfsIDAdmin(v) require.NoError(t, err) require.True(t, found) require.Equal(t, pks[i].GetScriptHash(), actual) @@ -41,14 +41,14 @@ func TestFrostfsIDConfig(t *testing.T) { v := viper.New() v.Set("frostfsid.admin", "abc") - _, found, err := util.GetFrostfsIDAdmin(v) + _, found, err := helper.GetFrostfsIDAdmin(v) require.Error(t, err) require.True(t, found) }) t.Run("missing key", func(t *testing.T) { v := viper.New() - _, found, err := util.GetFrostfsIDAdmin(v) + _, found, err := helper.GetFrostfsIDAdmin(v) require.NoError(t, err) require.False(t, found) }) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go index f032ee708..8ecc9f1b6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -36,7 +36,7 @@ func AlphabetCreds(cmd *cobra.Command, _ []string) error { return errors.New("size must be > 0") } if size > constants.MaxAlphabetNodes { - return morphUtil.ErrTooManyAlphabetNodes + return helper.ErrTooManyAlphabetNodes } v := viper.GetViper() @@ -46,7 +46,7 @@ func AlphabetCreds(cmd *cobra.Command, _ []string) error { return err } - _, err = morphUtil.InitializeContractWallet(v, walletDir) + _, err = helper.InitializeContractWallet(v, walletDir) if err != nil { return err } @@ -191,12 +191,12 @@ func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error gasStr := viper.GetString(gasFlag) - gasAmount, err := morphUtil.ParseGASAmount(gasStr) + gasAmount, err := helper.ParseGASAmount(gasStr) if err != nil { return err } - wCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/util/contract.go b/cmd/frostfs-adm/internal/modules/morph/helper/contract.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/util/contract.go rename to cmd/frostfs-adm/internal/modules/morph/helper/contract.go index 8f3b20e83..a740f3f09 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/contract.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/contract.go @@ -1,4 +1,4 @@ -package util +package helper import ( "fmt" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/download.go b/cmd/frostfs-adm/internal/modules/morph/helper/download.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/util/download.go rename to cmd/frostfs-adm/internal/modules/morph/helper/download.go index ff97a7f4a..08d427a46 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/download.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/download.go @@ -1,4 +1,4 @@ -package util +package helper import ( "context" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/helper/frostfsid.go similarity index 98% rename from cmd/frostfs-adm/internal/modules/morph/util/frostfsid.go rename to cmd/frostfs-adm/internal/modules/morph/helper/frostfsid.go index 7aea1c492..f29042b82 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/frostfsid.go @@ -1,4 +1,4 @@ -package util +package helper import ( "fmt" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/group.go b/cmd/frostfs-adm/internal/modules/morph/helper/group.go similarity index 98% rename from cmd/frostfs-adm/internal/modules/morph/util/group.go rename to cmd/frostfs-adm/internal/modules/morph/helper/group.go index a0306f3bc..10a164651 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/group.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/group.go @@ -1,4 +1,4 @@ -package util +package helper import ( "encoding/json" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/util/initialize.go rename to cmd/frostfs-adm/internal/modules/morph/helper/initialize.go index 2d676327d..00c2fae6d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go @@ -1,4 +1,4 @@ -package util +package helper import ( "errors" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go rename to cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go index 907ba57fe..8269f19dc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go @@ -1,4 +1,4 @@ -package util +package helper import ( "encoding/hex" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/local_client.go b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/util/local_client.go rename to cmd/frostfs-adm/internal/modules/morph/helper/local_client.go index 35fb07cc4..d309e7e5b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go @@ -1,4 +1,4 @@ -package util +package helper import ( "crypto/elliptic" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/n3client.go b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/util/n3client.go rename to cmd/frostfs-adm/internal/modules/morph/helper/n3client.go index 56a96c410..84dd69c79 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go @@ -1,4 +1,4 @@ -package util +package helper import ( "context" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/netmap.go b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/util/netmap.go rename to cmd/frostfs-adm/internal/modules/morph/helper/netmap.go index fbce8a0fa..732a5e11f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/netmap.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go @@ -1,4 +1,4 @@ -package util +package helper import ( "errors" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/util.go b/cmd/frostfs-adm/internal/modules/morph/helper/util.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/util/util.go rename to cmd/frostfs-adm/internal/modules/morph/helper/util.go index d85fa698e..f0399ab37 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/util.go @@ -1,4 +1,4 @@ -package util +package helper import ( "archive/tar" diff --git a/cmd/frostfs-adm/internal/modules/morph/util/wallet.go b/cmd/frostfs-adm/internal/modules/morph/helper/wallet.go similarity index 99% rename from cmd/frostfs-adm/internal/modules/morph/util/wallet.go rename to cmd/frostfs-adm/internal/modules/morph/helper/wallet.go index ca722f241..bd01cd59e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/util/wallet.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/wallet.go @@ -1,4 +1,4 @@ -package util +package helper import ( "fmt" diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go index d3903e226..cdaf7d3bc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go @@ -4,13 +4,13 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/spf13/cobra" "github.com/spf13/viper" ) func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { - initCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) + initCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("initialization error: %w", err) } @@ -29,7 +29,7 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { // 3. Deploy NNS contract. cmd.Println("Stage 3: deploy NNS contract.") - if err := morphUtil.DeployNNS(initCtx, constants.DeployMethodName); err != nil { + if err := helper.DeployNNS(initCtx, constants.DeployMethodName); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_deploy.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_deploy.go index 417cab269..f40ea732c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_deploy.go @@ -4,13 +4,13 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" ) -func deployContracts(c *morphUtil.InitializeContext) error { +func deployContracts(c *helper.InitializeContext) error { alphaCs := c.GetContract(constants.AlphabetContract) var keysParam []any @@ -26,13 +26,13 @@ func deployContracts(c *morphUtil.InitializeContext) error { } alphaCs.Manifest.Groups = baseGroups - err := morphUtil.AddManifestGroup(c.ContractWallet, ctrHash, alphaCs) + err := helper.AddManifestGroup(c.ContractWallet, ctrHash, alphaCs) if err != nil { return fmt.Errorf("can't sign manifest group: %v", err) } keysParam = append(keysParam, acc.PrivateKey().PublicKey().Bytes()) - params := morphUtil.GetContractDeployParameters(alphaCs, c.GetAlphabetDeployItems(i, len(c.Wallets))) + params := helper.GetContractDeployParameters(alphaCs, c.GetAlphabetDeployItems(i, len(c.Wallets))) act, err := actor.NewSimple(c.Client, acc) if err != nil { @@ -44,7 +44,7 @@ func deployContracts(c *morphUtil.InitializeContext) error { return fmt.Errorf("can't deploy alphabet #%d contract: %w", i, err) } - c.SentTxs = append(c.SentTxs, morphUtil.HashVUBPair{Hash: txHash, Vub: vub}) + c.SentTxs = append(c.SentTxs, helper.HashVUBPair{Hash: txHash, Vub: vub}) } for _, ctrName := range constants.ContractList { @@ -56,16 +56,16 @@ func deployContracts(c *morphUtil.InitializeContext) error { continue } - err := morphUtil.AddManifestGroup(c.ContractWallet, ctrHash, cs) + err := helper.AddManifestGroup(c.ContractWallet, ctrHash, cs) if err != nil { return fmt.Errorf("can't sign manifest group: %v", err) } - args, err := morphUtil.GetContractDeployData(c, ctrName, keysParam, constants.DeployMethodName) + args, err := helper.GetContractDeployData(c, ctrName, keysParam, constants.DeployMethodName) if err != nil { return fmt.Errorf("%s: getting deploy params: %v", ctrName, err) } - params := morphUtil.GetContractDeployParameters(cs, args) + params := helper.GetContractDeployParameters(cs, args) res, err := c.CommitteeAct.MakeCall(management.Hash, constants.DeployMethodName, params...) if err != nil { return fmt.Errorf("can't deploy %s contract: %w", ctrName, err) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go index c7c1d5b67..7a628d5a3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -19,7 +19,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/opcode" ) -func setNNS(c *morphUtil.InitializeContext) error { +func setNNS(c *helper.InitializeContext) error { r := management.NewReader(c.ReadOnlyInvoker) nnsCs, err := r.GetContractByID(1) if err != nil { @@ -47,7 +47,7 @@ func setNNS(c *morphUtil.InitializeContext) error { for i, acc := range c.Accounts { alphaCs.Hash = state.CreateContractHash(acc.Contract.ScriptHash(), alphaCs.NEF.Checksum, alphaCs.Manifest.Name) - domain := morphUtil.GetAlphabetNNSDomain(i) + domain := helper.GetAlphabetNNSDomain(i) if err := nnsRegisterDomain(c, nnsCs.Hash, alphaCs.Hash, domain); err != nil { return err } @@ -74,7 +74,7 @@ func setNNS(c *morphUtil.InitializeContext) error { return c.AwaitTx() } -func updateNNSGroup(c *morphUtil.InitializeContext, nnsHash util.Uint160, pub *keys.PublicKey) error { +func updateNNSGroup(c *helper.InitializeContext, nnsHash util.Uint160, pub *keys.PublicKey) error { bw := io.NewBufBinWriter() keyAlreadyAdded, domainRegCodeEmitted, err := c.EmitUpdateNNSGroupScript(bw, nnsHash, pub) if keyAlreadyAdded || err != nil { @@ -115,7 +115,7 @@ func wrapRegisterScriptWithPrice(w *io.BufBinWriter, nnsHash util.Uint160, s []b } } -func nnsRegisterDomain(c *morphUtil.InitializeContext, nnsHash, expectedHash util.Uint160, domain string) error { +func nnsRegisterDomain(c *helper.InitializeContext, nnsHash, expectedHash util.Uint160, domain string) error { script, ok, err := c.NNSRegisterDomainScript(nnsHash, expectedHash, domain) if ok || err != nil { return err diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go index d3a30485e..6a0a88016 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go @@ -6,7 +6,7 @@ import ( "math/big" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -29,7 +29,7 @@ const ( registerBatchSize = transaction.MaxAttributes - 1 ) -func registerCandidateRange(c *morphUtil.InitializeContext, start, end int) error { +func registerCandidateRange(c *helper.InitializeContext, start, end int) error { regPrice, err := getCandidateRegisterPrice(c) if err != nil { return fmt.Errorf("can't fetch registration price: %w", err) @@ -83,7 +83,7 @@ func registerCandidateRange(c *morphUtil.InitializeContext, start, end int) erro return c.SendTx(tx, c.Command, true) } -func registerCandidates(c *morphUtil.InitializeContext) error { +func registerCandidates(c *helper.InitializeContext) error { cc, err := unwrap.Array(c.ReadOnlyInvoker.Call(neo.Hash, "getCandidates")) if err != nil { return fmt.Errorf("`getCandidates`: %w", err) @@ -116,7 +116,7 @@ func registerCandidates(c *morphUtil.InitializeContext) error { return nil } -func transferNEOToAlphabetContracts(c *morphUtil.InitializeContext) error { +func transferNEOToAlphabetContracts(c *helper.InitializeContext) error { neoHash := neo.Hash ok, err := transferNEOFinished(c, neoHash) @@ -142,7 +142,7 @@ func transferNEOToAlphabetContracts(c *morphUtil.InitializeContext) error { return c.AwaitTx() } -func transferNEOFinished(c *morphUtil.InitializeContext, neoHash util.Uint160) (bool, error) { +func transferNEOFinished(c *helper.InitializeContext, neoHash util.Uint160) (bool, error) { r := nep17.NewReader(c.ReadOnlyInvoker, neoHash) bal, err := r.BalanceOf(c.CommitteeAcc.Contract.ScriptHash()) return bal.Cmp(big.NewInt(native.NEOTotalSupply)) == -1, err @@ -150,7 +150,7 @@ func transferNEOFinished(c *morphUtil.InitializeContext, neoHash util.Uint160) ( var errGetPriceInvalid = errors.New("`getRegisterPrice`: invalid response") -func getCandidateRegisterPrice(c *morphUtil.InitializeContext) (int64, error) { +func getCandidateRegisterPrice(c *helper.InitializeContext) (int64, error) { switch c.Client.(type) { case *rpcclient.Client: inv := invoker.New(c.Client, nil) @@ -158,7 +158,7 @@ func getCandidateRegisterPrice(c *morphUtil.InitializeContext) (int64, error) { return reader.GetRegisterPrice() default: neoHash := neo.Hash - res, err := morphUtil.InvokeFunction(c.Client, neoHash, "getRegisterPrice", nil, nil) + res, err := helper.InvokeFunction(c.Client, neoHash, "getRegisterPrice", nil, nil) if err != nil { return 0, err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_roles.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_roles.go index d7f17ccfb..a6815ee13 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_roles.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_roles.go @@ -1,7 +1,7 @@ package initialize import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/rolemgmt" @@ -9,7 +9,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/emit" ) -func setNotaryAndAlphabetNodes(c *util.InitializeContext) error { +func setNotaryAndAlphabetNodes(c *helper.InitializeContext) error { if ok, err := setRolesFinished(c); ok || err != nil { if err == nil { c.Command.Println("Stage 2: already performed.") @@ -35,12 +35,12 @@ func setNotaryAndAlphabetNodes(c *util.InitializeContext) error { return c.AwaitTx() } -func setRolesFinished(c *util.InitializeContext) (bool, error) { +func setRolesFinished(c *helper.InitializeContext) (bool, error) { height, err := c.Client.GetBlockCount() if err != nil { return false, err } - pubs, err := util.GetDesignatedByRole(c.ReadOnlyInvoker, rolemgmt.Hash, noderoles.NeoFSAlphabet, height) + pubs, err := helper.GetDesignatedByRole(c.ReadOnlyInvoker, rolemgmt.Hash, noderoles.NeoFSAlphabet, height) return len(pubs) == len(c.Wallets), err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go index 0114410c7..a05e995a7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go @@ -13,10 +13,10 @@ import ( cmdConfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/generate" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/node" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -53,7 +53,7 @@ func TestInitialize(t *testing.T) { testInitialize(t, constants.MaxAlphabetNodes) }) t.Run("too many nodes", func(t *testing.T) { - require.ErrorIs(t, generateTestData(t, t.TempDir(), constants.MaxAlphabetNodes+1), util.ErrTooManyAlphabetNodes) + require.ErrorIs(t, generateTestData(t, t.TempDir(), constants.MaxAlphabetNodes+1), helper.ErrTooManyAlphabetNodes) }) } @@ -158,33 +158,33 @@ func TestNextPollInterval(t *testing.T) { var pollInterval time.Duration var iteration int - pollInterval, hasChanged := util.NextPollInterval(iteration, pollInterval) + pollInterval, hasChanged := helper.NextPollInterval(iteration, pollInterval) require.True(t, hasChanged) require.Equal(t, time.Second, pollInterval) iteration = 4 - pollInterval, hasChanged = util.NextPollInterval(iteration, pollInterval) + pollInterval, hasChanged = helper.NextPollInterval(iteration, pollInterval) require.False(t, hasChanged) require.Equal(t, time.Second, pollInterval) iteration = 5 - pollInterval, hasChanged = util.NextPollInterval(iteration, pollInterval) + pollInterval, hasChanged = helper.NextPollInterval(iteration, pollInterval) require.True(t, hasChanged) require.Equal(t, 2*time.Second, pollInterval) iteration = 10 - pollInterval, hasChanged = util.NextPollInterval(iteration, pollInterval) + pollInterval, hasChanged = helper.NextPollInterval(iteration, pollInterval) require.True(t, hasChanged) require.Equal(t, 4*time.Second, pollInterval) iteration = 20 pollInterval = 32 * time.Second - pollInterval, hasChanged = util.NextPollInterval(iteration, pollInterval) + pollInterval, hasChanged = helper.NextPollInterval(iteration, pollInterval) require.True(t, hasChanged) // from 32s to 16s require.Equal(t, 16*time.Second, pollInterval) pollInterval = 16 * time.Second - pollInterval, hasChanged = util.NextPollInterval(iteration, pollInterval) + pollInterval, hasChanged = helper.NextPollInterval(iteration, pollInterval) require.False(t, hasChanged) require.Equal(t, 16*time.Second, pollInterval) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go index cad5f53c2..8bcd7eb58 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go @@ -5,7 +5,7 @@ import ( "math/big" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/io" @@ -28,7 +28,7 @@ const ( initialProxyGASAmount = 50_000 * native.GASFactor ) -func transferFunds(c *morphUtil.InitializeContext) error { +func transferFunds(c *helper.InitializeContext) error { ok, err := transferFundsFinished(c) if ok || err != nil { if err == nil { @@ -75,7 +75,7 @@ func transferFunds(c *morphUtil.InitializeContext) error { return c.AwaitTx() } -func transferFundsFinished(c *morphUtil.InitializeContext) (bool, error) { +func transferFundsFinished(c *helper.InitializeContext) (bool, error) { acc := c.Accounts[0] r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) @@ -83,7 +83,7 @@ func transferFundsFinished(c *morphUtil.InitializeContext) (bool, error) { return res.Cmp(big.NewInt(initialAlphabetGASAmount/2)) == 1, err } -func transferGASToProxy(c *morphUtil.InitializeContext) error { +func transferGASToProxy(c *helper.InitializeContext) error { proxyCs := c.GetContract(constants.ProxyContract) r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) @@ -115,7 +115,7 @@ type transferTarget struct { Data any } -func createNEP17MultiTransferTx(c morphUtil.Client, acc *wallet.Account, recipients []transferTarget) (*transaction.Transaction, error) { +func createNEP17MultiTransferTx(c helper.Client, acc *wallet.Account, recipients []transferTarget) (*transaction.Transaction, error) { from := acc.Contract.ScriptHash() w := io.NewBufBinWriter() diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go index 8e681cc3b..4fd671f2a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go @@ -5,7 +5,7 @@ import ( "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/spf13/cobra" @@ -13,7 +13,7 @@ import ( ) func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { - wCtx, err := util.NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) } @@ -24,13 +24,13 @@ func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(constants.NetmapContract)) + nmHash, err := helper.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, helper.DomainOf(constants.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } bw := io.NewBufBinWriter() - if err := util.EmitNewEpochCall(bw, wCtx, nmHash); err != nil { + if err := helper.EmitNewEpochCall(bw, wCtx, nmHash); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go b/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go index d7ad601dd..e9404e910 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go @@ -3,7 +3,7 @@ package netmap import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -13,7 +13,7 @@ import ( ) func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) { - c, err := util.GetN3Client(viper.GetViper()) + c, err := helper.GetN3Client(viper.GetViper()) commonCmd.ExitOnErr(cmd, "can't create N3 client: %w", err) inv := invoker.New(c, nil) @@ -22,7 +22,7 @@ func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) { cs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract info: %w", err) - nmHash, err := util.NNSResolveHash(inv, cs.Hash, util.DomainOf(constants.NetmapContract)) + nmHash, err := helper.NNSResolveHash(inv, cs.Hash, helper.DomainOf(constants.NetmapContract)) commonCmd.ExitOnErr(cmd, "can't get netmap contract hash: %w", err) res, err := inv.Call(nmHash, "netmapCandidates") diff --git a/cmd/frostfs-adm/internal/modules/morph/node/remove.go b/cmd/frostfs-adm/internal/modules/morph/node/remove.go index e25cb6efb..0d14bc646 100644 --- a/cmd/frostfs-adm/internal/modules/morph/node/remove.go +++ b/cmd/frostfs-adm/internal/modules/morph/node/remove.go @@ -6,7 +6,7 @@ import ( netmapcontract "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" @@ -30,7 +30,7 @@ func RemoveNodesCmd(cmd *cobra.Command, args []string) error { } } - wCtx, err := util.NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't initialize context: %w", err) } @@ -42,7 +42,7 @@ func RemoveNodesCmd(cmd *cobra.Command, args []string) error { return fmt.Errorf("can't get NNS contract info: %w", err) } - nmHash, err := util.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, util.DomainOf(constants.NetmapContract)) + nmHash, err := helper.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, helper.DomainOf(constants.NetmapContract)) if err != nil { return fmt.Errorf("can't get netmap contract hash: %w", err) } @@ -53,7 +53,7 @@ func RemoveNodesCmd(cmd *cobra.Command, args []string) error { int64(netmapcontract.NodeStateOffline), nodeKeys[i].Bytes()) } - if err := util.EmitNewEpochCall(bw, wCtx, nmHash); err != nil { + if err := helper.EmitNewEpochCall(bw, wCtx, nmHash); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go index 3b1bb179f..6d65a8284 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go @@ -6,7 +6,7 @@ import ( "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/nspcc-dev/neo-go/cli/input" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -65,7 +65,7 @@ func depositNotary(cmd *cobra.Command, _ []string) error { if err != nil { return err } - gasAmount, err := morphUtil.ParseGASAmount(gasStr) + gasAmount, err := helper.ParseGASAmount(gasStr) if err != nil { return err } @@ -86,12 +86,12 @@ func depositNotary(cmd *cobra.Command, _ []string) error { } func transferGas(cmd *cobra.Command, acc *wallet.Account, accHash util.Uint160, gasAmount fixedn.Fixed8, till int64) error { - c, err := morphUtil.GetN3Client(viper.GetViper()) + c, err := helper.GetN3Client(viper.GetViper()) if err != nil { return err } - if err := morphUtil.CheckNotaryEnabled(c); err != nil { + if err := helper.CheckNotaryEnabled(c); err != nil { return err } @@ -123,7 +123,7 @@ func transferGas(cmd *cobra.Command, acc *wallet.Account, accHash util.Uint160, return fmt.Errorf("could not send tx: %w", err) } - return morphUtil.AwaitTx(cmd, c, []morphUtil.HashVUBPair{{Hash: txHash, Vub: vub}}) + return helper.AwaitTx(cmd, c, []helper.HashVUBPair{{Hash: txHash, Vub: vub}}) } func openWallet(cmd *cobra.Command) (*wallet.Wallet, error) { diff --git a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go index 7f4a8d326..440e73a12 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go @@ -7,7 +7,7 @@ import ( "strings" "text/tabwriter" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -25,7 +25,7 @@ const ( ) func SetPolicyCmd(cmd *cobra.Command, args []string) error { - wCtx, err := util.NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) } @@ -59,7 +59,7 @@ func SetPolicyCmd(cmd *cobra.Command, args []string) error { } func dumpPolicyCmd(cmd *cobra.Command, _ []string) error { - c, err := util.GetN3Client(viper.GetViper()) + c, err := helper.GetN3Client(viper.GetViper()) commonCmd.ExitOnErr(cmd, "can't create N3 client:", err) inv := invoker.New(c, nil) diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go index 2549958ef..7e173ac49 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go @@ -4,7 +4,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" - morphUtil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" @@ -37,7 +37,7 @@ func removeProxyAccount(cmd *cobra.Command, _ []string) { } func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error { - wCtx, err := morphUtil.NewInitializeContext(cmd, viper.GetViper()) + wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't to initialize context: %w", err) } @@ -48,7 +48,7 @@ func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error return fmt.Errorf("can't get NNS contract info: %w", err) } - proxyHash, err := morphUtil.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, morphUtil.DomainOf(constants.ProxyContract)) + proxyHash, err := helper.NNSResolveHash(wCtx.ReadOnlyInvoker, cs.Hash, helper.DomainOf(constants.ProxyContract)) if err != nil { return fmt.Errorf("can't get proxy contract hash: %w", err) } From 35370283ba1f6ee815102c33938407b4bd029c03 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 13 Feb 2024 10:08:49 +0300 Subject: [PATCH 0907/1943] [#948] adm: Move `TestNextPollInterval` to package `helper` Signed-off-by: Anton Nikiforov --- .../modules/morph/helper/initialize_test.go | 43 +++++++++++++++++++ .../morph/initialize/initialize_test.go | 35 --------------- 2 files changed, 43 insertions(+), 35 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/helper/initialize_test.go diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_test.go new file mode 100644 index 000000000..f3ce42f51 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_test.go @@ -0,0 +1,43 @@ +package helper + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestNextPollInterval(t *testing.T) { + var pollInterval time.Duration + var iteration int + + pollInterval, hasChanged := NextPollInterval(iteration, pollInterval) + require.True(t, hasChanged) + require.Equal(t, time.Second, pollInterval) + + iteration = 4 + pollInterval, hasChanged = NextPollInterval(iteration, pollInterval) + require.False(t, hasChanged) + require.Equal(t, time.Second, pollInterval) + + iteration = 5 + pollInterval, hasChanged = NextPollInterval(iteration, pollInterval) + require.True(t, hasChanged) + require.Equal(t, 2*time.Second, pollInterval) + + iteration = 10 + pollInterval, hasChanged = NextPollInterval(iteration, pollInterval) + require.True(t, hasChanged) + require.Equal(t, 4*time.Second, pollInterval) + + iteration = 20 + pollInterval = 32 * time.Second + pollInterval, hasChanged = NextPollInterval(iteration, pollInterval) + require.True(t, hasChanged) // from 32s to 16s + require.Equal(t, 16*time.Second, pollInterval) + + pollInterval = 16 * time.Second + pollInterval, hasChanged = NextPollInterval(iteration, pollInterval) + require.False(t, hasChanged) + require.Equal(t, 16*time.Second, pollInterval) +} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go index a05e995a7..e39c7356f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go @@ -153,38 +153,3 @@ func setTestCredentials(v *viper.Viper, size int) { } v.Set("credentials.contract", constants.TestContractPassword) } - -func TestNextPollInterval(t *testing.T) { - var pollInterval time.Duration - var iteration int - - pollInterval, hasChanged := helper.NextPollInterval(iteration, pollInterval) - require.True(t, hasChanged) - require.Equal(t, time.Second, pollInterval) - - iteration = 4 - pollInterval, hasChanged = helper.NextPollInterval(iteration, pollInterval) - require.False(t, hasChanged) - require.Equal(t, time.Second, pollInterval) - - iteration = 5 - pollInterval, hasChanged = helper.NextPollInterval(iteration, pollInterval) - require.True(t, hasChanged) - require.Equal(t, 2*time.Second, pollInterval) - - iteration = 10 - pollInterval, hasChanged = helper.NextPollInterval(iteration, pollInterval) - require.True(t, hasChanged) - require.Equal(t, 4*time.Second, pollInterval) - - iteration = 20 - pollInterval = 32 * time.Second - pollInterval, hasChanged = helper.NextPollInterval(iteration, pollInterval) - require.True(t, hasChanged) // from 32s to 16s - require.Equal(t, 16*time.Second, pollInterval) - - pollInterval = 16 * time.Second - pollInterval, hasChanged = helper.NextPollInterval(iteration, pollInterval) - require.False(t, hasChanged) - require.Equal(t, 16*time.Second, pollInterval) -} From 5cbf57081f044d843a704b5d2c5da7367c3a040a Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 13 Feb 2024 16:22:08 +0300 Subject: [PATCH 0908/1943] [#980] adm: Introduce flag `chain-name` for APE managing commands Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/ape/ape.go | 44 ++++++++++--------- .../internal/modules/morph/ape/ape_util.go | 20 +++++++++ 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go index 8395fd4ba..c08dc512e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go @@ -14,22 +14,24 @@ import ( ) const ( - namespaceTarget = "namespace" - containerTarget = "container" - jsonFlag = "json" - jsonFlagDesc = "Output rule chains in JSON format" - chainIDFlag = "chain-id" - chainIDDesc = "Rule chain ID" - ruleFlag = "rule" - ruleFlagDesc = "Rule chain in text format" - ruleJSONFlag = "rule-json" - ruleJSONFlagDesc = "Chain rule in JSON format or path to the file" - targetNameFlag = "target-name" - targetNameDesc = "Resource name in APE resource name format" - targetTypeFlag = "target-type" - targetTypeDesc = "Resource type(container/namespace)" - addrAdminFlag = "addr" - addrAdminDesc = "The address of the admins wallet" + namespaceTarget = "namespace" + containerTarget = "container" + jsonFlag = "json" + jsonFlagDesc = "Output rule chains in JSON format" + chainIDFlag = "chain-id" + chainIDDesc = "Rule chain ID" + ruleFlag = "rule" + ruleFlagDesc = "Rule chain in text format" + ruleJSONFlag = "rule-json" + ruleJSONFlagDesc = "Chain rule in JSON format or path to the file" + targetNameFlag = "target-name" + targetNameDesc = "Resource name in APE resource name format" + targetTypeFlag = "target-type" + targetTypeDesc = "Resource type(container/namespace)" + addrAdminFlag = "addr" + addrAdminDesc = "The address of the admins wallet" + chainNameFlag = "chain-name" + chainNameFlagDesc = "Chain name(ingress|s3)" ) var ( @@ -99,7 +101,7 @@ func initAddRuleChainCmd() { _ = addRuleChainCmd.MarkFlagRequired(chainIDFlag) addRuleChainCmd.Flags().String(ruleFlag, "", ruleFlagDesc) addRuleChainCmd.Flags().String(ruleJSONFlag, "", ruleJSONFlagDesc) - + addRuleChainCmd.Flags().String(chainNameFlag, ingress, chainNameFlagDesc) addRuleChainCmd.MarkFlagsMutuallyExclusive(ruleFlag, ruleJSONFlag) } @@ -115,6 +117,7 @@ func initRemoveRuleChainCmd() { _ = removeRuleChainCmd.MarkFlagRequired(targetNameFlag) removeRuleChainCmd.Flags().String(chainIDFlag, "", chainIDDesc) _ = removeRuleChainCmd.MarkFlagRequired(chainIDFlag) + removeRuleChainCmd.Flags().String(chainNameFlag, ingress, chainNameFlagDesc) } func initListRuleChainsCmd() { @@ -127,6 +130,7 @@ func initListRuleChainsCmd() { listRuleChainsCmd.Flags().String(targetNameFlag, "", targetNameDesc) _ = listRuleChainsCmd.MarkFlagRequired(targetNameFlag) listRuleChainsCmd.Flags().Bool(jsonFlag, false, jsonFlagDesc) + listRuleChainsCmd.Flags().String(chainNameFlag, ingress, chainNameFlagDesc) } func initSetAdminCmd() { @@ -149,7 +153,7 @@ func addRuleChain(cmd *cobra.Command, _ []string) { chain := parseChain(cmd) target := parseTarget(cmd) pci, ac := newPolicyContractInterface(cmd) - h, vub, err := pci.AddMorphRuleChain(apechain.Ingress, target, chain) + h, vub, err := pci.AddMorphRuleChain(parseChainName(cmd), target, chain) cmd.Println("Waiting for transaction to persist...") _, err = ac.Wait(h, vub, err) commonCmd.ExitOnErr(cmd, "add rule chain error: %w", err) @@ -160,7 +164,7 @@ func removeRuleChain(cmd *cobra.Command, _ []string) { chainID := parseChainID(cmd) target := parseTarget(cmd) pci, ac := newPolicyContractInterface(cmd) - h, vub, err := pci.RemoveMorphRuleChain(apechain.Ingress, target, chainID) + h, vub, err := pci.RemoveMorphRuleChain(parseChainName(cmd), target, chainID) cmd.Println("Waiting for transaction to persist...") _, err = ac.Wait(h, vub, err) commonCmd.ExitOnErr(cmd, "remove rule chain error: %w", err) @@ -170,7 +174,7 @@ func removeRuleChain(cmd *cobra.Command, _ []string) { func listRuleChains(cmd *cobra.Command, _ []string) { target := parseTarget(cmd) pci, _ := newPolicyContractInterface(cmd) - chains, err := pci.ListMorphRuleChains(apechain.Ingress, target) + chains, err := pci.ListMorphRuleChains(parseChainName(cmd), target) commonCmd.ExitOnErr(cmd, "list rule chains error: %w", err) if len(chains) == 0 { return diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index bbbdf7805..7f23e3754 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" @@ -21,6 +22,16 @@ import ( "github.com/spf13/viper" ) +const ( + ingress = "ingress" + s3 = "s3" +) + +var mChainName = map[string]apechain.Name{ + ingress: apechain.Ingress, + s3: apechain.S3, +} + func parseTarget(cmd *cobra.Command) policyengine.Target { var targetType policyengine.TargetType typ, _ := cmd.Flags().GetString(targetTypeFlag) @@ -78,6 +89,15 @@ func parseChain(cmd *cobra.Command) *apechain.Chain { return chain } +func parseChainName(cmd *cobra.Command) apechain.Name { + chainName, _ := cmd.Flags().GetString(chainNameFlag) + apeChainName, ok := mChainName[strings.ToLower(chainName)] + if !ok { + commonCmd.ExitOnErr(cmd, "", fmt.Errorf("unsupported chain name")) + } + return apeChainName +} + func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *actor.Actor) { v := viper.GetViper() c, err := helper.GetN3Client(v) From 2429508ac5df680bebc63c339d1e24a527ef1739 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 14 Feb 2024 15:39:28 +0300 Subject: [PATCH 0909/1943] [#959] shard: Skip rebuild in `DEGRADED` mode Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/shard/control.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 408354b55..774fe8045 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -173,8 +173,9 @@ func (s *Shard) Init(ctx context.Context) error { s.gc.init(ctx) s.rb = newRebuilder(s.rebuildLimiter) - s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) - + if !m.NoMetabase() { + s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) + } return nil } @@ -327,9 +328,11 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { defer unlock() s.rb.Stop(s.log) - defer func() { - s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) - }() + if !s.info.Mode.NoMetabase() { + defer func() { + s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) + }() + } ok, err := s.metaBase.Reload(c.metaOpts...) if err != nil { From 05b5f5ca851a07c4994608e40dd134ff05042822 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 14 Feb 2024 16:10:33 +0300 Subject: [PATCH 0910/1943] [#959] writecache: Fix panic on `Get` when it is not initialized Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/writecache/get.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index d2496e447..5194446ec 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -84,6 +84,9 @@ func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, // // Returns an error of type apistatus.ObjectNotFound if the requested object is missing in db. func Get(db *bbolt.DB, key []byte) ([]byte, error) { + if db == nil { + return nil, ErrNotInitialized + } var value []byte err := db.View(func(tx *bbolt.Tx) error { b := tx.Bucket(defaultBucket) From 6bafdab004304c445c701f64c3e2d90636bb2ceb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 15 Feb 2024 15:30:42 +0300 Subject: [PATCH 0911/1943] [#988] go.mod: Update api-go and sdk-go versions Signed-off-by: Dmitrii Stepanov --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index aa771f1e7..d86676dd3 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.20 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240112150928-72885aae835c + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240215124401-634e24aba715 git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240126141009-65b4525b3bf0 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240215122347-a86170f53af7 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240129064140-8d21ab2d99d9 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index e15236331..fa5da98cc 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,15 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240112150928-72885aae835c h1:dO2VLtoTQre4W28i1gSuW1MO80qFd0Lq9Q46Pu3q3TA= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240112150928-72885aae835c/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240215124401-634e24aba715 h1:EDtL9OJcdeevV/jmNMtHugulAQprdOnknNPvLB3LRgE= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240215124401-634e24aba715/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa h1:uuN90s74idThHXi/jF/IlTdFimOIs+IBU44rdTE/Zdo= git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240126141009-65b4525b3bf0 h1:jp/Cg6CF3ip0IVRCjt48qKyv1cbMUmNYTn2z1jFLWqc= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240126141009-65b4525b3bf0/go.mod h1:t1akKcUH7iBrFHX8rSXScYMP17k2kYQXMbZooiL5Juw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240215122347-a86170f53af7 h1:3a5g14oC+aMQ1p0bvuqG3JZKGLOcqRvcHzKAFV4eS0w= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240215122347-a86170f53af7/go.mod h1:XcgrbZ88XfvhAMxmZCQJ0dv6FyRSq6Mg2J7nN8uuO0k= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240129064140-8d21ab2d99d9 h1:hkDIg6vzV1c1pyt5uj9EiBikHqKczfoNpPhwWvXaNf8= From 2680192ba036c518445c5a2d19d5cc49a5edfdaa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 15 Feb 2024 17:13:29 +0300 Subject: [PATCH 0912/1943] [#988] objectSvc: Fix `SetMarshalData` for PutSingle After api-go update it is required to pass marshal data to `SetMarshalData`. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/sign.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 2f4f9cb9a..4bf581b78 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -122,7 +122,7 @@ func (s *SignService) Head(ctx context.Context, req *object.HeadRequest) (*objec } func (s *SignService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { - req.GetBody().SetMarshalData(nil) + req.GetBody().SetMarshalData(req.GetBody().StableMarshal(nil)) if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.PutSingleResponse) return resp, s.sigSvc.SignResponse(resp, err) From 15fc5bac26fb9bf0f31dc293078acdfb6f620a56 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 14 Feb 2024 12:34:25 +0300 Subject: [PATCH 0913/1943] [#972] keyer: Use UncompressedBytes() for marshaling elliptic.Marshal() becomes deprecated in go1.21 Signed-off-by: Evgenii Stratonikov --- pkg/util/keyer/dashboard.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/util/keyer/dashboard.go b/pkg/util/keyer/dashboard.go index bc5f2e76e..b2942b52a 100644 --- a/pkg/util/keyer/dashboard.go +++ b/pkg/util/keyer/dashboard.go @@ -1,7 +1,6 @@ package keyer import ( - "crypto/elliptic" "encoding/hex" "fmt" "os" @@ -45,7 +44,7 @@ func (d Dashboard) PrettyPrint(uncompressed, useHex bool) { if d.pubKey != nil { if uncompressed { - data = elliptic.Marshal(elliptic.P256(), d.pubKey.X, d.pubKey.Y) + data = d.pubKey.UncompressedBytes() } else { data = d.pubKey.Bytes() } From e39a714c257248bfaca9086f38e7b1dc15761a6b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 19:14:10 +0300 Subject: [PATCH 0914/1943] [#972] go.mod: Bump go version to go1.21 Signed-off-by: Evgenii Stratonikov --- .docker/Dockerfile.adm | 2 +- .docker/Dockerfile.ci | 2 +- .docker/Dockerfile.cli | 2 +- .docker/Dockerfile.ir | 2 +- .docker/Dockerfile.storage | 2 +- .forgejo/workflows/build.yml | 2 +- .forgejo/workflows/dco.yml | 2 +- .forgejo/workflows/tests.yml | 6 +++--- .forgejo/workflows/vulncheck.yml | 2 +- Makefile | 2 +- README.md | 2 +- go.mod | 2 +- 12 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.docker/Dockerfile.adm b/.docker/Dockerfile.adm index eeccaab79..b3dad06d3 100644 --- a/.docker/Dockerfile.adm +++ b/.docker/Dockerfile.adm @@ -1,4 +1,4 @@ -FROM golang:1.21 as builder +FROM golang:1.22 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ci b/.docker/Dockerfile.ci index ef6586a64..e9077c831 100644 --- a/.docker/Dockerfile.ci +++ b/.docker/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM golang:1.21 +FROM golang:1.22 WORKDIR /tmp diff --git a/.docker/Dockerfile.cli b/.docker/Dockerfile.cli index 0dd4cebcf..5adedc140 100644 --- a/.docker/Dockerfile.cli +++ b/.docker/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM golang:1.21 as builder +FROM golang:1.22 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ir b/.docker/Dockerfile.ir index 4015df673..25025bb2f 100644 --- a/.docker/Dockerfile.ir +++ b/.docker/Dockerfile.ir @@ -1,4 +1,4 @@ -FROM golang:1.21 as builder +FROM golang:1.22 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage b/.docker/Dockerfile.storage index ced6ea538..a16005516 100644 --- a/.docker/Dockerfile.storage +++ b/.docker/Dockerfile.storage @@ -1,4 +1,4 @@ -FROM golang:1.21 as builder +FROM golang:1.22 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 3c74d9434..86943fe88 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.20', '1.21' ] + go_versions: [ '1.21', '1.22' ] steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 9aa0d3106..7c5af8410 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' - name: Run commit format checker uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index f66a2c401..8b0d9220b 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -11,7 +11,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' cache: true - name: Install linters @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.20', '1.21' ] + go_versions: [ '1.21', '1.22' ] fail-fast: false steps: - uses: actions/checkout@v3 @@ -63,7 +63,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' cache: true - name: Install staticcheck diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 8ea01749b..3af564c4b 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/Makefile b/Makefile index 4ff42b7e4..e8f61a874 100755 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 HUB_IMAGE ?= truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" -GO_VERSION ?= 1.21 +GO_VERSION ?= 1.22 LINT_VERSION ?= 1.55.2 TRUECLOUDLAB_LINT_VERSION ?= 0.0.3 PROTOC_VERSION ?= 25.0 diff --git a/README.md b/README.md index ecd162a85..413010372 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The latest version of frostfs-node works with frostfs-contract # Building -To make all binaries you need Go 1.20+ and `make`: +To make all binaries you need Go 1.21+ and `make`: ``` make all ``` diff --git a/go.mod b/go.mod index d86676dd3..e0c0185cd 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-node -go 1.20 +go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 From 45fd4e4ff16dd63d60afe816c41ee71e1a16db28 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 8 Feb 2024 19:14:34 +0300 Subject: [PATCH 0915/1943] [#972] pilorama: Remove removeDuplicatesInPlace() Also, check that slices.CompareFunc() indeed passes all the tests before removal. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/batch.go | 15 +--- .../pilorama/batch_test.go | 70 ------------------- 2 files changed, 2 insertions(+), 83 deletions(-) delete mode 100644 pkg/local_object_storage/pilorama/batch_test.go diff --git a/pkg/local_object_storage/pilorama/batch.go b/pkg/local_object_storage/pilorama/batch.go index c65488b74..520c6dfb4 100644 --- a/pkg/local_object_storage/pilorama/batch.go +++ b/pkg/local_object_storage/pilorama/batch.go @@ -2,6 +2,7 @@ package pilorama import ( "encoding/binary" + "slices" "sort" "sync" "time" @@ -50,7 +51,7 @@ func (b *batch) run() { sort.Slice(b.operations, func(i, j int) bool { return b.operations[i].Time < b.operations[j].Time }) - b.operations = removeDuplicatesInPlace(b.operations) + b.operations = slices.CompactFunc(b.operations, func(x, y *Move) bool { return x.Time == y.Time }) // Our main use-case is addition of new items. In this case, // we do not need to perform undo()/redo(), just do(). @@ -115,15 +116,3 @@ func (b *batch) run() { b.results[i] <- err } } - -func removeDuplicatesInPlace(a []*Move) []*Move { - equalCount := 0 - for i := 1; i < len(a); i++ { - if a[i].Time == a[i-1].Time { - equalCount++ - } else { - a[i-equalCount] = a[i] - } - } - return a[:len(a)-equalCount] -} diff --git a/pkg/local_object_storage/pilorama/batch_test.go b/pkg/local_object_storage/pilorama/batch_test.go deleted file mode 100644 index 931fce18c..000000000 --- a/pkg/local_object_storage/pilorama/batch_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package pilorama - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func Test_removeDuplicatesInPlace(t *testing.T) { - testCases := []struct { - before []int - after []int - }{ - { - before: []int{}, - after: []int{}, - }, - { - before: []int{1}, - after: []int{1}, - }, - { - before: []int{1, 2}, - after: []int{1, 2}, - }, - { - before: []int{1, 2, 3}, - after: []int{1, 2, 3}, - }, - { - before: []int{1, 1, 2}, - after: []int{1, 2}, - }, - { - before: []int{1, 2, 2}, - after: []int{1, 2}, - }, - { - before: []int{1, 2, 2, 3}, - after: []int{1, 2, 3}, - }, - { - before: []int{1, 1, 1}, - after: []int{1}, - }, - { - before: []int{1, 1, 2, 2}, - after: []int{1, 2}, - }, - { - before: []int{1, 1, 1, 2, 3, 3, 3}, - after: []int{1, 2, 3}, - }, - } - - for _, tc := range testCases { - ops := make([]*Move, len(tc.before)) - for i := range ops { - ops[i] = &Move{Meta: Meta{Time: Timestamp(tc.before[i])}} - } - - expected := make([]*Move, len(tc.after)) - for i := range expected { - expected[i] = &Move{Meta: Meta{Time: Timestamp(tc.after[i])}} - } - - actual := removeDuplicatesInPlace(ops) - require.Equal(t, expected, actual, "%d", tc.before) - } -} From 89784b2e0ab573ffd220c9689dd9911b83984caa Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 14 Feb 2024 10:03:07 +0300 Subject: [PATCH 0916/1943] [#972] Use min/max builtins Signed-off-by: Evgenii Stratonikov --- .../modules/morph/initialize/initialize_register.go | 5 +---- pkg/innerring/processors/netmap/cleanup_table.go | 5 +---- pkg/local_object_storage/blobovnicza/iterate.go | 8 -------- pkg/local_object_storage/engine/list.go | 5 +---- pkg/local_object_storage/shard/gc.go | 12 ++---------- pkg/morph/client/notary.go | 6 +----- pkg/services/object/internal/client/client.go | 5 +---- pkg/services/object/transport_splitter.go | 5 +---- pkg/services/policer/policer_test.go | 5 +---- pkg/services/tree/sync.go | 8 ++------ 10 files changed, 11 insertions(+), 53 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go index 6a0a88016..4c6607f9a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go @@ -100,10 +100,7 @@ func registerCandidates(c *helper.InitializeContext) error { // Register candidates in batches in order to overcome the signers amount limit. // See: https://github.com/nspcc-dev/neo-go/blob/master/pkg/core/transaction/transaction.go#L27 for i := 0; i < need; i += registerBatchSize { - start, end := i, i+registerBatchSize - if end > need { - end = need - } + start, end := i, min(i+registerBatchSize, need) // This check is sound because transactions are accepted/rejected atomically. if have >= end { continue diff --git a/pkg/innerring/processors/netmap/cleanup_table.go b/pkg/innerring/processors/netmap/cleanup_table.go index 705e21d99..c18611569 100644 --- a/pkg/innerring/processors/netmap/cleanup_table.go +++ b/pkg/innerring/processors/netmap/cleanup_table.go @@ -82,10 +82,7 @@ func (c *cleanupTable) touch(keyString string, now uint64, binNodeInfo []byte) b result := !ok || access.removeFlag || !bytes.Equal(access.binNodeInfo, binNodeInfo) access.removeFlag = false // reset remove flag on each touch - if now > access.epoch { - access.epoch = now - } - + access.epoch = max(access.epoch, now) access.binNodeInfo = binNodeInfo // update binary node info c.lastAccess[keyString] = access diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index 32b0ccea7..e87da751d 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -56,14 +56,6 @@ func (b *Blobovnicza) iterateBounds(useObjLimitBound bool, f func(uint64, uint64 return nil } -func max(a, b uint64) uint64 { - if a > b { - return a - } - - return b -} - // IterationElement represents a unit of elements through which Iterate operation passes. type IterationElement struct { addr oid.Address diff --git a/pkg/local_object_storage/engine/list.go b/pkg/local_object_storage/engine/list.go index f9229a2b1..7245caeeb 100644 --- a/pkg/local_object_storage/engine/list.go +++ b/pkg/local_object_storage/engine/list.go @@ -134,10 +134,7 @@ func (e *StorageEngine) ListWithCursor(ctx context.Context, prm ListWithCursorPr continue } - count := prm.count - uint32(len(result)) - if count > batchSize { - count = batchSize - } + count := min(prm.count-uint32(len(result)), batchSize) var shardPrm shard.ListWithCursorPrm shardPrm.WithCount(count) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 83be3259a..ef8e97d34 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -343,16 +343,8 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { } func (s *Shard) getExpiredObjectsParameters() (workerCount, batchSize int) { - workerCount = minExpiredWorkers - batchSize = minExpiredBatchSize - - if s.gc.gcCfg.expiredCollectorBatchSize > batchSize { - batchSize = s.gc.gcCfg.expiredCollectorBatchSize - } - - if s.gc.gcCfg.expiredCollectorWorkerCount > workerCount { - workerCount = s.gc.gcCfg.expiredCollectorWorkerCount - } + workerCount = max(minExpiredWorkers, s.gc.gcCfg.expiredCollectorWorkerCount) + batchSize = max(minExpiredBatchSize, s.gc.gcCfg.expiredCollectorBatchSize) return } diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 5b817a805..febdef4d0 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -162,11 +162,7 @@ func (c *Client) DepositNotary(amount fixedn.Fixed8, delta uint32) (res util.Uin return util.Uint256{}, fmt.Errorf("can't get previous expiration value: %w", err) } - till := int64(bc + delta) - if till < currentTill { - till = currentTill - } - + till := max(int64(bc+delta), currentTill) return c.depositNotary(amount, till) } diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index dd2de0fd1..6477faed1 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -343,10 +343,7 @@ func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, e return nil, new(apistatus.ObjectOutOfRange) } - ln := prm.ln - if ln > maxInitialBufferSize { - ln = maxInitialBufferSize - } + ln := min(prm.ln, maxInitialBufferSize) w := bytes.NewBuffer(make([]byte, ln)) _, err = io.CopyN(w, rdr, int64(prm.ln)) diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index 2d9810cd3..54e49cb12 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -164,10 +164,7 @@ func (s *searchStreamMsgSizeCtrl) Send(resp *object.SearchResponse) error { newResp.SetBody(body) } - cut := s.addrAmount - if cut > ln { - cut = ln - } + cut := min(s.addrAmount, ln) body.SetIDList(ids[:cut]) newResp.SetMetaHeader(resp.GetMetaHeader()) diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 50f206fd9..be0974c39 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -388,10 +388,7 @@ func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectc if it.cur >= len(it.objs) { return nil, engine.ErrEndOfListing } - end := it.cur + int(size) - if end > len(it.objs) { - end = len(it.objs) - } + end := min(it.cur+int(size), len(it.objs)) ret := it.objs[it.cur:end] it.cur = end return ret, nil diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 2a19ae18a..1fa9a2b6c 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -167,9 +167,7 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram merged <- ms[minTimeMoveIndex] height := ms[minTimeMoveIndex].Time if ms[minTimeMoveIndex] = <-streams[minTimeMoveIndex]; ms[minTimeMoveIndex] == nil { - if minStreamedLastHeight > height { - minStreamedLastHeight = height - } + minStreamedLastHeight = min(minStreamedLastHeight, height) } } @@ -203,9 +201,7 @@ func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID s errGroup.Go(func() error { if err := s.forest.TreeApply(ctx, cid, treeID, m, true); err != nil { heightMtx.Lock() - if m.Time < unappliedOperationHeight { - unappliedOperationHeight = m.Time - } + unappliedOperationHeight = min(unappliedOperationHeight, m.Time) heightMtx.Unlock() return err } From d2f13a29de7c6e758ac03e29f90c7566ced14b11 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 14 Feb 2024 10:14:32 +0300 Subject: [PATCH 0917/1943] [#972] Adopt slices.BinarySearch() Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/container/container.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/container/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go index 690db2f4f..eda388d37 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "slices" "sort" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" @@ -446,7 +447,7 @@ func getCIDFilterFunc(cmd *cobra.Command) (func([]byte) bool, error) { var id cid.ID id.SetSHA256(v) idStr := id.EncodeToString() - n := sort.Search(len(rawIDs), func(i int) bool { return rawIDs[i] >= idStr }) - return n < len(rawIDs) && rawIDs[n] == idStr + _, found := slices.BinarySearch(rawIDs, idStr) + return found }, nil } From 6d9707ff1fcb70b5e6eaf6c1c7cd0a1af597f541 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 14 Feb 2024 10:42:16 +0300 Subject: [PATCH 0918/1943] [#972] Use require.ElementsMatch() where possible Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/list_test.go | 12 +----------- .../metabase/containers_test.go | 12 +----------- pkg/local_object_storage/metabase/list_test.go | 14 +------------- 3 files changed, 3 insertions(+), 35 deletions(-) diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 7bb1ac0fa..4fc9569c7 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -3,7 +3,6 @@ package engine import ( "context" "path/filepath" - "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -18,13 +17,6 @@ import ( "github.com/stretchr/testify/require" ) -func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithType { - sort.Slice(addrWithType, func(i, j int) bool { - return addrWithType[i].Address.EncodeToString() < addrWithType[j].Address.EncodeToString() - }) - return addrWithType -} - func TestListWithCursor(t *testing.T) { t.Parallel() @@ -98,7 +90,6 @@ func TestListWithCursor(t *testing.T) { require.NoError(t, err) expected = append(expected, object.AddressWithType{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)}) } - expected = sortAddresses(expected) var prm ListWithCursorPrm prm.count = tt.batchSize @@ -113,8 +104,7 @@ func TestListWithCursor(t *testing.T) { prm.cursor = res.Cursor() } - got = sortAddresses(got) - require.Equal(t, expected, got) + require.ElementsMatch(t, expected, got) }) } } diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index e7762ded6..66c562565 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -3,7 +3,6 @@ package meta_test import ( "context" "math/rand" - "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -125,18 +124,9 @@ func TestDB_ContainersCount(t *testing.T) { } } - sort.Slice(expected, func(i, j int) bool { - return expected[i].EncodeToString() < expected[j].EncodeToString() - }) - got, err := db.Containers(context.Background()) require.NoError(t, err) - - sort.Slice(got, func(i, j int) bool { - return got[i].EncodeToString() < got[j].EncodeToString() - }) - - require.Equal(t, expected, got) + require.ElementsMatch(t, expected, got) } func TestDB_ContainerSize(t *testing.T) { diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 25c0e35bd..e1ccb4e06 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -3,7 +3,6 @@ package meta_test import ( "context" "errors" - "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -128,8 +127,6 @@ func TestLisObjectsWithCursor(t *testing.T) { expected = append(expected, object.AddressWithType{Address: object.AddressOf(child), Type: objectSDK.TypeRegular}) } - expected = sortAddresses(expected) - t.Run("success with various count", func(t *testing.T) { for countPerReq := 1; countPerReq <= total; countPerReq++ { got := make([]object.AddressWithType, 0, total) @@ -151,9 +148,7 @@ func TestLisObjectsWithCursor(t *testing.T) { _, _, err = metaListWithCursor(db, uint32(countPerReq), cursor) require.ErrorIs(t, err, meta.ErrEndOfListing, "count:%d", countPerReq, cursor) - - got = sortAddresses(got) - require.Equal(t, expected, got, "count:%d", countPerReq) + require.ElementsMatch(t, expected, got, "count:%d", countPerReq) } }) @@ -216,13 +211,6 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) { } } -func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithType { - sort.Slice(addrWithType, func(i, j int) bool { - return addrWithType[i].Address.EncodeToString() < addrWithType[j].Address.EncodeToString() - }) - return addrWithType -} - func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]object.AddressWithType, *meta.Cursor, error) { var listPrm meta.ListPrm listPrm.SetCount(count) From b871d7a5e83a616e4a769d08705ba7ba9394503f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 14 Feb 2024 10:50:27 +0300 Subject: [PATCH 0919/1943] [#972] Use slices.Sort* when useful Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/shards_set_mode.go | 7 ++----- pkg/services/tree/getsubtree_test.go | 6 ++---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/shards_set_mode.go b/cmd/frostfs-cli/modules/control/shards_set_mode.go index 1c87b405b..e73f15178 100644 --- a/cmd/frostfs-cli/modules/control/shards_set_mode.go +++ b/cmd/frostfs-cli/modules/control/shards_set_mode.go @@ -3,7 +3,7 @@ package control import ( "bytes" "fmt" - "sort" + "slices" "strings" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -177,9 +177,6 @@ func getShardIDListFromIDFlag(cmd *cobra.Command, withAllFlag bool) [][]byte { res = append(res, raw) } - sort.Slice(res, func(i, j int) bool { - return bytes.Compare(res[i], res[j]) < 0 - }) - + slices.SortFunc(res, bytes.Compare) return res } diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index 63cdc73a2..d220f1e2b 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -4,7 +4,7 @@ import ( "context" "errors" "path" - "sort" + "slices" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -177,9 +177,7 @@ func TestGetSubTreeOrderAsc(t *testing.T) { require.True(t, found, "unknown node") } - require.True(t, sort.SliceIsSorted(paths, func(i, j int) bool { - return paths[i] < paths[j] - })) + require.True(t, slices.IsSorted(paths)) } var ( From f3e50772fd929a39e9fefc1b2a15c49beabdc3a7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 14 Feb 2024 10:52:52 +0300 Subject: [PATCH 0920/1943] [#972] Drop x/exp/slices dependency Signed-off-by: Evgenii Stratonikov --- go.sum | 14 ++++++++++++++ pkg/innerring/processors/frostfs/handlers.go | 2 +- .../internal/testutil/generators_test.go | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index fa5da98cc..8785ca975 100644 --- a/go.sum +++ b/go.sum @@ -59,9 +59,11 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etly github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= @@ -96,6 +98,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -135,19 +138,24 @@ github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/4 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= @@ -229,6 +237,7 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -305,6 +314,7 @@ go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40 go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= @@ -325,6 +335,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -401,12 +412,14 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= @@ -427,6 +440,7 @@ google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index 5782dc744..c80f9fdc5 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -3,13 +3,13 @@ package frostfs import ( "bytes" "encoding/hex" + "slices" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "go.uber.org/zap" - "golang.org/x/exp/slices" ) func (np *Processor) handleDeposit(ev event.Event) { diff --git a/pkg/local_object_storage/internal/testutil/generators_test.go b/pkg/local_object_storage/internal/testutil/generators_test.go index f2c60f9b4..f7be6014d 100644 --- a/pkg/local_object_storage/internal/testutil/generators_test.go +++ b/pkg/local_object_storage/internal/testutil/generators_test.go @@ -2,10 +2,10 @@ package testutil import ( "encoding/binary" + "slices" "testing" "github.com/stretchr/testify/require" - "golang.org/x/exp/slices" ) func TestOverwriteObjGenerator(t *testing.T) { From 13d5cd3e21e1ecac4db7f780fe2a5b2b245ffaf7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 19 Feb 2024 13:08:18 +0300 Subject: [PATCH 0921/1943] [#991] logger: Fix journald logger Allow to change logger level. Signed-off-by: Dmitrii Stepanov --- pkg/util/logger/logger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index c6db04df9..e67afb36b 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -147,7 +147,7 @@ func newJournaldLogger(prm *Prm) (*Logger, error) { encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields) - core := zapjournald.NewCore(zap.NewAtomicLevelAt(prm.level), encoder, &journald.Journal{}, zapjournald.SyslogFields) + core := zapjournald.NewCore(lvl, encoder, &journald.Journal{}, zapjournald.SyslogFields) coreWithContext := core.With([]zapcore.Field{ zapjournald.SyslogFacility(zapjournald.LogDaemon), zapjournald.SyslogIdentifier(), From 9e55836da583f02f64fd002199b127c76ced3b3d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 19 Feb 2024 15:53:38 +0300 Subject: [PATCH 0922/1943] [#994] cli: Output pilorama path in `shards list` Do it for JSON too, not only for human output. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/shards_list.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-cli/modules/control/shards_list.go b/cmd/frostfs-cli/modules/control/shards_list.go index 0b5d6b0a3..3eb74ec82 100644 --- a/cmd/frostfs-cli/modules/control/shards_list.go +++ b/cmd/frostfs-cli/modules/control/shards_list.go @@ -70,6 +70,7 @@ func prettyPrintShardsJSON(cmd *cobra.Command, ii []*control.ShardInfo) { "metabase": i.GetMetabasePath(), "blobstor": i.GetBlobstor(), "writecache": i.GetWritecachePath(), + "pilorama": i.GetPiloramaPath(), "error_count": i.GetErrorCount(), }) } From 946f2ec2bf4b5e99726811ac8bcb225bbca0708f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 19 Feb 2024 17:02:39 +0300 Subject: [PATCH 0923/1943] Revert "[#972] Drop x/exp/slices dependency" This reverts commit f3e50772fd929a39e9fefc1b2a15c49beabdc3a7. Signed-off-by: Evgenii Stratonikov --- go.sum | 14 -------------- pkg/innerring/processors/frostfs/handlers.go | 2 +- .../internal/testutil/generators_test.go | 2 +- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/go.sum b/go.sum index 8785ca975..fa5da98cc 100644 --- a/go.sum +++ b/go.sum @@ -59,11 +59,9 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etly github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= -github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= @@ -98,7 +96,6 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -138,24 +135,19 @@ github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/4 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= @@ -237,7 +229,6 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -314,7 +305,6 @@ go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40 go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= @@ -335,7 +325,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -412,14 +401,12 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= -google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= @@ -440,7 +427,6 @@ google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index c80f9fdc5..5782dc744 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -3,13 +3,13 @@ package frostfs import ( "bytes" "encoding/hex" - "slices" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "go.uber.org/zap" + "golang.org/x/exp/slices" ) func (np *Processor) handleDeposit(ev event.Event) { diff --git a/pkg/local_object_storage/internal/testutil/generators_test.go b/pkg/local_object_storage/internal/testutil/generators_test.go index f7be6014d..f2c60f9b4 100644 --- a/pkg/local_object_storage/internal/testutil/generators_test.go +++ b/pkg/local_object_storage/internal/testutil/generators_test.go @@ -2,10 +2,10 @@ package testutil import ( "encoding/binary" - "slices" "testing" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" ) func TestOverwriteObjGenerator(t *testing.T) { From 3359349acbbfb0aa8e208c5286ff7bd6537df2be Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 19 Feb 2024 17:02:47 +0300 Subject: [PATCH 0924/1943] Revert "[#972] Use slices.Sort* when useful" This reverts commit b871d7a5e83a616e4a769d08705ba7ba9394503f. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/shards_set_mode.go | 7 +++++-- pkg/services/tree/getsubtree_test.go | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/shards_set_mode.go b/cmd/frostfs-cli/modules/control/shards_set_mode.go index e73f15178..1c87b405b 100644 --- a/cmd/frostfs-cli/modules/control/shards_set_mode.go +++ b/cmd/frostfs-cli/modules/control/shards_set_mode.go @@ -3,7 +3,7 @@ package control import ( "bytes" "fmt" - "slices" + "sort" "strings" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -177,6 +177,9 @@ func getShardIDListFromIDFlag(cmd *cobra.Command, withAllFlag bool) [][]byte { res = append(res, raw) } - slices.SortFunc(res, bytes.Compare) + sort.Slice(res, func(i, j int) bool { + return bytes.Compare(res[i], res[j]) < 0 + }) + return res } diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index d220f1e2b..63cdc73a2 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -4,7 +4,7 @@ import ( "context" "errors" "path" - "slices" + "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -177,7 +177,9 @@ func TestGetSubTreeOrderAsc(t *testing.T) { require.True(t, found, "unknown node") } - require.True(t, slices.IsSorted(paths)) + require.True(t, sort.SliceIsSorted(paths, func(i, j int) bool { + return paths[i] < paths[j] + })) } var ( From 7627d08914ab4cfa6fd73f17e586f4b0f1a0bbc1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 19 Feb 2024 17:02:56 +0300 Subject: [PATCH 0925/1943] Revert "[#972] Use require.ElementsMatch() where possible" This reverts commit 6d9707ff1fcb70b5e6eaf6c1c7cd0a1af597f541. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/list_test.go | 12 +++++++++++- .../metabase/containers_test.go | 12 +++++++++++- pkg/local_object_storage/metabase/list_test.go | 14 +++++++++++++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 4fc9569c7..7bb1ac0fa 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -3,6 +3,7 @@ package engine import ( "context" "path/filepath" + "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -17,6 +18,13 @@ import ( "github.com/stretchr/testify/require" ) +func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithType { + sort.Slice(addrWithType, func(i, j int) bool { + return addrWithType[i].Address.EncodeToString() < addrWithType[j].Address.EncodeToString() + }) + return addrWithType +} + func TestListWithCursor(t *testing.T) { t.Parallel() @@ -90,6 +98,7 @@ func TestListWithCursor(t *testing.T) { require.NoError(t, err) expected = append(expected, object.AddressWithType{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)}) } + expected = sortAddresses(expected) var prm ListWithCursorPrm prm.count = tt.batchSize @@ -104,7 +113,8 @@ func TestListWithCursor(t *testing.T) { prm.cursor = res.Cursor() } - require.ElementsMatch(t, expected, got) + got = sortAddresses(got) + require.Equal(t, expected, got) }) } } diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index 66c562565..e7762ded6 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -3,6 +3,7 @@ package meta_test import ( "context" "math/rand" + "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -124,9 +125,18 @@ func TestDB_ContainersCount(t *testing.T) { } } + sort.Slice(expected, func(i, j int) bool { + return expected[i].EncodeToString() < expected[j].EncodeToString() + }) + got, err := db.Containers(context.Background()) require.NoError(t, err) - require.ElementsMatch(t, expected, got) + + sort.Slice(got, func(i, j int) bool { + return got[i].EncodeToString() < got[j].EncodeToString() + }) + + require.Equal(t, expected, got) } func TestDB_ContainerSize(t *testing.T) { diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index e1ccb4e06..25c0e35bd 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -3,6 +3,7 @@ package meta_test import ( "context" "errors" + "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -127,6 +128,8 @@ func TestLisObjectsWithCursor(t *testing.T) { expected = append(expected, object.AddressWithType{Address: object.AddressOf(child), Type: objectSDK.TypeRegular}) } + expected = sortAddresses(expected) + t.Run("success with various count", func(t *testing.T) { for countPerReq := 1; countPerReq <= total; countPerReq++ { got := make([]object.AddressWithType, 0, total) @@ -148,7 +151,9 @@ func TestLisObjectsWithCursor(t *testing.T) { _, _, err = metaListWithCursor(db, uint32(countPerReq), cursor) require.ErrorIs(t, err, meta.ErrEndOfListing, "count:%d", countPerReq, cursor) - require.ElementsMatch(t, expected, got, "count:%d", countPerReq) + + got = sortAddresses(got) + require.Equal(t, expected, got, "count:%d", countPerReq) } }) @@ -211,6 +216,13 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) { } } +func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithType { + sort.Slice(addrWithType, func(i, j int) bool { + return addrWithType[i].Address.EncodeToString() < addrWithType[j].Address.EncodeToString() + }) + return addrWithType +} + func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]object.AddressWithType, *meta.Cursor, error) { var listPrm meta.ListPrm listPrm.SetCount(count) From 4bfc6d29b910bc8e85852180ca0caabbf2eff2f4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 19 Feb 2024 17:03:04 +0300 Subject: [PATCH 0926/1943] Revert "[#972] Adopt slices.BinarySearch()" This reverts commit d2f13a29de7c6e758ac03e29f90c7566ced14b11. Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/container/container.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/container/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go index eda388d37..690db2f4f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "os" - "slices" "sort" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" @@ -447,7 +446,7 @@ func getCIDFilterFunc(cmd *cobra.Command) (func([]byte) bool, error) { var id cid.ID id.SetSHA256(v) idStr := id.EncodeToString() - _, found := slices.BinarySearch(rawIDs, idStr) - return found + n := sort.Search(len(rawIDs), func(i int) bool { return rawIDs[i] >= idStr }) + return n < len(rawIDs) && rawIDs[n] == idStr }, nil } From dad56d2e98179904a9b263103ad04794202a7c1f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 19 Feb 2024 17:03:11 +0300 Subject: [PATCH 0927/1943] Revert "[#972] Use min/max builtins" This reverts commit 89784b2e0ab573ffd220c9689dd9911b83984caa. Signed-off-by: Evgenii Stratonikov --- .../modules/morph/initialize/initialize_register.go | 5 ++++- pkg/innerring/processors/netmap/cleanup_table.go | 5 ++++- pkg/local_object_storage/blobovnicza/iterate.go | 8 ++++++++ pkg/local_object_storage/engine/list.go | 5 ++++- pkg/local_object_storage/shard/gc.go | 12 ++++++++++-- pkg/morph/client/notary.go | 6 +++++- pkg/services/object/internal/client/client.go | 5 ++++- pkg/services/object/transport_splitter.go | 5 ++++- pkg/services/policer/policer_test.go | 5 ++++- pkg/services/tree/sync.go | 8 ++++++-- 10 files changed, 53 insertions(+), 11 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go index 4c6607f9a..6a0a88016 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go @@ -100,7 +100,10 @@ func registerCandidates(c *helper.InitializeContext) error { // Register candidates in batches in order to overcome the signers amount limit. // See: https://github.com/nspcc-dev/neo-go/blob/master/pkg/core/transaction/transaction.go#L27 for i := 0; i < need; i += registerBatchSize { - start, end := i, min(i+registerBatchSize, need) + start, end := i, i+registerBatchSize + if end > need { + end = need + } // This check is sound because transactions are accepted/rejected atomically. if have >= end { continue diff --git a/pkg/innerring/processors/netmap/cleanup_table.go b/pkg/innerring/processors/netmap/cleanup_table.go index c18611569..705e21d99 100644 --- a/pkg/innerring/processors/netmap/cleanup_table.go +++ b/pkg/innerring/processors/netmap/cleanup_table.go @@ -82,7 +82,10 @@ func (c *cleanupTable) touch(keyString string, now uint64, binNodeInfo []byte) b result := !ok || access.removeFlag || !bytes.Equal(access.binNodeInfo, binNodeInfo) access.removeFlag = false // reset remove flag on each touch - access.epoch = max(access.epoch, now) + if now > access.epoch { + access.epoch = now + } + access.binNodeInfo = binNodeInfo // update binary node info c.lastAccess[keyString] = access diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index e87da751d..32b0ccea7 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -56,6 +56,14 @@ func (b *Blobovnicza) iterateBounds(useObjLimitBound bool, f func(uint64, uint64 return nil } +func max(a, b uint64) uint64 { + if a > b { + return a + } + + return b +} + // IterationElement represents a unit of elements through which Iterate operation passes. type IterationElement struct { addr oid.Address diff --git a/pkg/local_object_storage/engine/list.go b/pkg/local_object_storage/engine/list.go index 7245caeeb..f9229a2b1 100644 --- a/pkg/local_object_storage/engine/list.go +++ b/pkg/local_object_storage/engine/list.go @@ -134,7 +134,10 @@ func (e *StorageEngine) ListWithCursor(ctx context.Context, prm ListWithCursorPr continue } - count := min(prm.count-uint32(len(result)), batchSize) + count := prm.count - uint32(len(result)) + if count > batchSize { + count = batchSize + } var shardPrm shard.ListWithCursorPrm shardPrm.WithCount(count) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index ef8e97d34..83be3259a 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -343,8 +343,16 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { } func (s *Shard) getExpiredObjectsParameters() (workerCount, batchSize int) { - workerCount = max(minExpiredWorkers, s.gc.gcCfg.expiredCollectorWorkerCount) - batchSize = max(minExpiredBatchSize, s.gc.gcCfg.expiredCollectorBatchSize) + workerCount = minExpiredWorkers + batchSize = minExpiredBatchSize + + if s.gc.gcCfg.expiredCollectorBatchSize > batchSize { + batchSize = s.gc.gcCfg.expiredCollectorBatchSize + } + + if s.gc.gcCfg.expiredCollectorWorkerCount > workerCount { + workerCount = s.gc.gcCfg.expiredCollectorWorkerCount + } return } diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index febdef4d0..5b817a805 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -162,7 +162,11 @@ func (c *Client) DepositNotary(amount fixedn.Fixed8, delta uint32) (res util.Uin return util.Uint256{}, fmt.Errorf("can't get previous expiration value: %w", err) } - till := max(int64(bc+delta), currentTill) + till := int64(bc + delta) + if till < currentTill { + till = currentTill + } + return c.depositNotary(amount, till) } diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 6477faed1..dd2de0fd1 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -343,7 +343,10 @@ func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, e return nil, new(apistatus.ObjectOutOfRange) } - ln := min(prm.ln, maxInitialBufferSize) + ln := prm.ln + if ln > maxInitialBufferSize { + ln = maxInitialBufferSize + } w := bytes.NewBuffer(make([]byte, ln)) _, err = io.CopyN(w, rdr, int64(prm.ln)) diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index 54e49cb12..2d9810cd3 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -164,7 +164,10 @@ func (s *searchStreamMsgSizeCtrl) Send(resp *object.SearchResponse) error { newResp.SetBody(body) } - cut := min(s.addrAmount, ln) + cut := s.addrAmount + if cut > ln { + cut = ln + } body.SetIDList(ids[:cut]) newResp.SetMetaHeader(resp.GetMetaHeader()) diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index be0974c39..50f206fd9 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -388,7 +388,10 @@ func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectc if it.cur >= len(it.objs) { return nil, engine.ErrEndOfListing } - end := min(it.cur+int(size), len(it.objs)) + end := it.cur + int(size) + if end > len(it.objs) { + end = len(it.objs) + } ret := it.objs[it.cur:end] it.cur = end return ret, nil diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 1fa9a2b6c..2a19ae18a 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -167,7 +167,9 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram merged <- ms[minTimeMoveIndex] height := ms[minTimeMoveIndex].Time if ms[minTimeMoveIndex] = <-streams[minTimeMoveIndex]; ms[minTimeMoveIndex] == nil { - minStreamedLastHeight = min(minStreamedLastHeight, height) + if minStreamedLastHeight > height { + minStreamedLastHeight = height + } } } @@ -201,7 +203,9 @@ func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID s errGroup.Go(func() error { if err := s.forest.TreeApply(ctx, cid, treeID, m, true); err != nil { heightMtx.Lock() - unappliedOperationHeight = min(unappliedOperationHeight, m.Time) + if m.Time < unappliedOperationHeight { + unappliedOperationHeight = m.Time + } heightMtx.Unlock() return err } From 9f68305c2e88b1aea294ac3e75b75038151534e1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 19 Feb 2024 17:03:25 +0300 Subject: [PATCH 0928/1943] Revert "[#972] pilorama: Remove removeDuplicatesInPlace()" This reverts commit 45fd4e4ff16dd63d60afe816c41ee71e1a16db28. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/batch.go | 15 +++- .../pilorama/batch_test.go | 70 +++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 pkg/local_object_storage/pilorama/batch_test.go diff --git a/pkg/local_object_storage/pilorama/batch.go b/pkg/local_object_storage/pilorama/batch.go index 520c6dfb4..c65488b74 100644 --- a/pkg/local_object_storage/pilorama/batch.go +++ b/pkg/local_object_storage/pilorama/batch.go @@ -2,7 +2,6 @@ package pilorama import ( "encoding/binary" - "slices" "sort" "sync" "time" @@ -51,7 +50,7 @@ func (b *batch) run() { sort.Slice(b.operations, func(i, j int) bool { return b.operations[i].Time < b.operations[j].Time }) - b.operations = slices.CompactFunc(b.operations, func(x, y *Move) bool { return x.Time == y.Time }) + b.operations = removeDuplicatesInPlace(b.operations) // Our main use-case is addition of new items. In this case, // we do not need to perform undo()/redo(), just do(). @@ -116,3 +115,15 @@ func (b *batch) run() { b.results[i] <- err } } + +func removeDuplicatesInPlace(a []*Move) []*Move { + equalCount := 0 + for i := 1; i < len(a); i++ { + if a[i].Time == a[i-1].Time { + equalCount++ + } else { + a[i-equalCount] = a[i] + } + } + return a[:len(a)-equalCount] +} diff --git a/pkg/local_object_storage/pilorama/batch_test.go b/pkg/local_object_storage/pilorama/batch_test.go new file mode 100644 index 000000000..931fce18c --- /dev/null +++ b/pkg/local_object_storage/pilorama/batch_test.go @@ -0,0 +1,70 @@ +package pilorama + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_removeDuplicatesInPlace(t *testing.T) { + testCases := []struct { + before []int + after []int + }{ + { + before: []int{}, + after: []int{}, + }, + { + before: []int{1}, + after: []int{1}, + }, + { + before: []int{1, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 3}, + after: []int{1, 2, 3}, + }, + { + before: []int{1, 1, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 2, 2, 3}, + after: []int{1, 2, 3}, + }, + { + before: []int{1, 1, 1}, + after: []int{1}, + }, + { + before: []int{1, 1, 2, 2}, + after: []int{1, 2}, + }, + { + before: []int{1, 1, 1, 2, 3, 3, 3}, + after: []int{1, 2, 3}, + }, + } + + for _, tc := range testCases { + ops := make([]*Move, len(tc.before)) + for i := range ops { + ops[i] = &Move{Meta: Meta{Time: Timestamp(tc.before[i])}} + } + + expected := make([]*Move, len(tc.after)) + for i := range expected { + expected[i] = &Move{Meta: Meta{Time: Timestamp(tc.after[i])}} + } + + actual := removeDuplicatesInPlace(ops) + require.Equal(t, expected, actual, "%d", tc.before) + } +} From 9adcb253be772bfc4f716307c564ac1f0545f85c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 19 Feb 2024 17:03:32 +0300 Subject: [PATCH 0929/1943] Revert "[#972] go.mod: Bump go version to go1.21" This reverts commit e39a714c257248bfaca9086f38e7b1dc15761a6b. Signed-off-by: Evgenii Stratonikov --- .docker/Dockerfile.adm | 2 +- .docker/Dockerfile.ci | 2 +- .docker/Dockerfile.cli | 2 +- .docker/Dockerfile.ir | 2 +- .docker/Dockerfile.storage | 2 +- .forgejo/workflows/build.yml | 2 +- .forgejo/workflows/dco.yml | 2 +- .forgejo/workflows/tests.yml | 6 +++--- .forgejo/workflows/vulncheck.yml | 2 +- Makefile | 2 +- README.md | 2 +- go.mod | 2 +- 12 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.docker/Dockerfile.adm b/.docker/Dockerfile.adm index b3dad06d3..eeccaab79 100644 --- a/.docker/Dockerfile.adm +++ b/.docker/Dockerfile.adm @@ -1,4 +1,4 @@ -FROM golang:1.22 as builder +FROM golang:1.21 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ci b/.docker/Dockerfile.ci index e9077c831..ef6586a64 100644 --- a/.docker/Dockerfile.ci +++ b/.docker/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM golang:1.22 +FROM golang:1.21 WORKDIR /tmp diff --git a/.docker/Dockerfile.cli b/.docker/Dockerfile.cli index 5adedc140..0dd4cebcf 100644 --- a/.docker/Dockerfile.cli +++ b/.docker/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM golang:1.22 as builder +FROM golang:1.21 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ir b/.docker/Dockerfile.ir index 25025bb2f..4015df673 100644 --- a/.docker/Dockerfile.ir +++ b/.docker/Dockerfile.ir @@ -1,4 +1,4 @@ -FROM golang:1.22 as builder +FROM golang:1.21 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage b/.docker/Dockerfile.storage index a16005516..ced6ea538 100644 --- a/.docker/Dockerfile.storage +++ b/.docker/Dockerfile.storage @@ -1,4 +1,4 @@ -FROM golang:1.22 as builder +FROM golang:1.21 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 86943fe88..3c74d9434 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.21', '1.22' ] + go_versions: [ '1.20', '1.21' ] steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 7c5af8410..9aa0d3106 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.21' - name: Run commit format checker uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 8b0d9220b..f66a2c401 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -11,7 +11,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.21' cache: true - name: Install linters @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.21', '1.22' ] + go_versions: [ '1.20', '1.21' ] fail-fast: false steps: - uses: actions/checkout@v3 @@ -63,7 +63,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.21' cache: true - name: Install staticcheck diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 3af564c4b..8ea01749b 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.21' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/Makefile b/Makefile index e8f61a874..4ff42b7e4 100755 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 HUB_IMAGE ?= truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" -GO_VERSION ?= 1.22 +GO_VERSION ?= 1.21 LINT_VERSION ?= 1.55.2 TRUECLOUDLAB_LINT_VERSION ?= 0.0.3 PROTOC_VERSION ?= 25.0 diff --git a/README.md b/README.md index 413010372..ecd162a85 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The latest version of frostfs-node works with frostfs-contract # Building -To make all binaries you need Go 1.21+ and `make`: +To make all binaries you need Go 1.20+ and `make`: ``` make all ``` diff --git a/go.mod b/go.mod index e0c0185cd..d86676dd3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-node -go 1.21 +go 1.20 require ( code.gitea.io/sdk/gitea v0.17.1 From 9611710e190b260a9067d57f64f3678da721e78c Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 16 Feb 2024 12:36:42 +0300 Subject: [PATCH 0930/1943] [#989] cli: Read and parse chains from file * Introduce path flag to make add-rule command read and parse chain from file. File is binary/JSON-encoded chain. Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/control/add_rule.go | 37 +++++++++++++++------ cmd/frostfs-cli/modules/util/ape.go | 18 ++++++++++ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index 0f5ac11f3..45b36bfb1 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -2,6 +2,7 @@ package control import ( "encoding/hex" + "errors" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" @@ -14,6 +15,7 @@ import ( const ( ruleFlag = "rule" + pathFlag = "path" ) var addRuleCmd = &cobra.Command{ @@ -23,15 +25,13 @@ var addRuleCmd = &cobra.Command{ Example: `control add-rule --endpoint ... -w ... --address ... --chain-id ChainID --cid ... --rule "allow Object.Get *" --rule "deny Object.Get EbxzAdz5LB4uqxuz6crWKAumBNtZyK2rKsqQP7TdZvwr/*" --rule "deny:QuotaLimitReached Object.Put Object.Resource:Department=HR *" + +control add-rule --endpoint ... -w ... --address ... --chain-id ChainID --cid ... --path some_chain.json `, Run: addRule, } -func addRule(cmd *cobra.Command, _ []string) { - pk := key.Get(cmd) - - target := parseTarget(cmd) - +func parseChain(cmd *cobra.Command) *apechain.Chain { chainID, _ := cmd.Flags().GetString(chainIDFlag) hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) @@ -43,20 +43,34 @@ func addRule(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err) } - rule, _ := cmd.Flags().GetStringArray(ruleFlag) - chain := new(apechain.Chain) - commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, rule)) chain.ID = apechain.ID(chainIDRaw) - serializedChain := chain.Bytes() + + if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 { + commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, rules)) + } else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" { + commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", util.ParseAPEChainBinaryOrJSON(chain, encPath)) + } else { + commonCmd.ExitOnErr(cmd, "parser error", errors.New("rule is not passed")) + } cmd.Println("Parsed chain:") util.PrintHumanReadableAPEChain(cmd, chain) + return chain +} + +func addRule(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + target := parseTarget(cmd) + + parsed := parseChain(cmd) + req := &control.AddChainLocalOverrideRequest{ Body: &control.AddChainLocalOverrideRequest_Body{ Target: target, - Chain: serializedChain, + Chain: parsed.Bytes(), }, } @@ -81,9 +95,12 @@ func initControlAddRuleCmd() { ff := addRuleCmd.Flags() ff.StringArray(ruleFlag, []string{}, "Rule statement") + ff.String(pathFlag, "", "Path to encoded chain in JSON or binary format") ff.String(chainIDFlag, "", "Assign ID to the parsed chain") ff.String(targetNameFlag, "", targetNameDesc) ff.String(targetTypeFlag, "", targetTypeDesc) _ = addRuleCmd.MarkFlagRequired(targetTypeFlag) ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") + + addRuleCmd.MarkFlagsMutuallyExclusive(pathFlag, ruleFlag) } diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index 6a7eca948..dfe70e7da 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -3,6 +3,7 @@ package util import ( "errors" "fmt" + "os" "strconv" "strings" @@ -57,6 +58,23 @@ func PrintHumanReadableAPEChain(cmd *cobra.Command, chain *apechain.Chain) { } } +func ParseAPEChainBinaryOrJSON(chain *apechain.Chain, path string) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read file <%s>: %w", path, err) + } + + err = chain.UnmarshalBinary(data) + if err != nil { + err = chain.UnmarshalJSON(data) + if err != nil { + return fmt.Errorf("invalid format: %w", err) + } + } + + return nil +} + // ParseAPEChain parses APE chain rules. func ParseAPEChain(chain *apechain.Chain, rules []string) error { if len(rules) == 0 { From 613e11c4d2dcc0190b0b80b6040ea8d257349c58 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 16 Feb 2024 12:59:48 +0300 Subject: [PATCH 0931/1943] [#989] adm: Read and parse chains from file * Slightly fix the approach to read encoded chain from file in frostfs-adm. Signed-off-by: Airat Arifullin --- .../internal/modules/morph/ape/ape.go | 10 +++---- .../internal/modules/morph/ape/ape_util.go | 28 ++++++------------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go index c08dc512e..957c7a15f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go @@ -22,8 +22,8 @@ const ( chainIDDesc = "Rule chain ID" ruleFlag = "rule" ruleFlagDesc = "Rule chain in text format" - ruleJSONFlag = "rule-json" - ruleJSONFlagDesc = "Chain rule in JSON format or path to the file" + pathFlag = "path" + pathFlagDesc = "path to encoded chain in JSON or binary format" targetNameFlag = "target-name" targetNameDesc = "Resource name in APE resource name format" targetTypeFlag = "target-type" @@ -99,10 +99,10 @@ func initAddRuleChainCmd() { addRuleChainCmd.Flags().String(chainIDFlag, "", chainIDDesc) _ = addRuleChainCmd.MarkFlagRequired(chainIDFlag) - addRuleChainCmd.Flags().String(ruleFlag, "", ruleFlagDesc) - addRuleChainCmd.Flags().String(ruleJSONFlag, "", ruleJSONFlagDesc) + addRuleChainCmd.Flags().StringArray(ruleFlag, []string{}, ruleFlagDesc) + addRuleChainCmd.Flags().String(pathFlag, "", pathFlagDesc) addRuleChainCmd.Flags().String(chainNameFlag, ingress, chainNameFlagDesc) - addRuleChainCmd.MarkFlagsMutuallyExclusive(ruleFlag, ruleJSONFlag) + addRuleChainCmd.MarkFlagsMutuallyExclusive(ruleFlag, pathFlag) } func initRemoveRuleChainCmd() { diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 7f23e3754..fe09776ed 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -1,9 +1,7 @@ package ape import ( - "encoding/json" "fmt" - "os" "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" @@ -63,29 +61,19 @@ func parseChainID(cmd *cobra.Command) apechain.ID { func parseChain(cmd *cobra.Command) *apechain.Chain { chain := new(apechain.Chain) - if ruleStmt, _ := cmd.Flags().GetString(ruleFlag); ruleStmt != "" { - parseErr := parseutil.ParseAPEChain(chain, []string{ruleStmt}) - commonCmd.ExitOnErr(cmd, "ape chain parser error: %w", parseErr) - } else if ruleJSON, _ := cmd.Flags().GetString(ruleJSONFlag); ruleJSON != "" { - var rule []byte - if _, err := os.Stat(ruleJSON); err == nil { - rule, err = os.ReadFile(ruleJSON) - commonCmd.ExitOnErr(cmd, "read file error: %w", err) - } else { - rule = []byte(ruleJSON) - if !json.Valid(rule) { - commonCmd.ExitOnErr(cmd, "read raw rule error: %w", - fmt.Errorf("invalid JSON")) - } - } - err := chain.DecodeBytes(rule) - commonCmd.ExitOnErr(cmd, "chain decode error: %w", err) + if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 { + commonCmd.ExitOnErr(cmd, "parser error: %w", parseutil.ParseAPEChain(chain, rules)) + } else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" { + commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", parseutil.ParseAPEChainBinaryOrJSON(chain, encPath)) } else { - commonCmd.ExitOnErr(cmd, "", fmt.Errorf("rule is not passed")) + commonCmd.ExitOnErr(cmd, "parser error: %w", fmt.Errorf("rule is not passed")) } chain.ID = parseChainID(cmd) + cmd.Println("Parsed chain:") + parseutil.PrintHumanReadableAPEChain(cmd, chain) + return chain } From 0f064b79628c52779d6fbdebdbcb2e74597240ce Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 16 Feb 2024 13:13:54 +0300 Subject: [PATCH 0932/1943] [#989] util: Introduce any and all statements for ape rule parsing Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/docs/policy.md | 3 +++ cmd/frostfs-cli/modules/util/ape.go | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/cmd/frostfs-cli/docs/policy.md b/cmd/frostfs-cli/docs/policy.md index 06c325c62..43dd97fcf 100644 --- a/cmd/frostfs-cli/docs/policy.md +++ b/cmd/frostfs-cli/docs/policy.md @@ -30,6 +30,9 @@ Actions is a regular operations upon FrostFS containers/objects. Like `Object.Pu In status section it is possible to use `allow`, `deny` or `deny:QuotaLimitReached` actions. +If a statement does not contain lexeme `any`, field `Any` is set to `false` by default. Otherwise, it is set +to `true`. Optionally, `all` can be used - it also sets `Any=false`. + It is prohibited to mix operation under FrostFS container and object in one rule. The same statement is equal for conditions and resources - one rule is for one type of items. diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index dfe70e7da..41d186b0c 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -100,6 +100,8 @@ func ParseAPEChain(chain *apechain.Chain, rules []string) error { // deny:QuotaLimitReached Object.Put * // allow Object.Put * // allow Object.Get Object.Resource:Department=HR Object.Request:Actor=ownerA * +// allow Object.Get any Object.Resource:Department=HR Object.Request:Actor=ownerA * +// allow Object.Get all Object.Resource:Department=HR Object.Request:Actor=ownerA * // //nolint:godot func ParseAPERule(r *apechain.Rule, rule string) error { @@ -123,6 +125,12 @@ func parseRuleLexemes(r *apechain.Rule, lexemes []string) error { var isObject *bool for i, lexeme := range lexemes[1:] { + anyExpr, anyErr := parseAnyAll(lexeme) + if anyErr == nil { + r.Any = anyExpr + continue + } + var name string var actionType bool name, actionType, err = parseAction(lexeme) @@ -158,6 +166,17 @@ func parseRuleLexemes(r *apechain.Rule, lexemes []string) error { return nil } +func parseAnyAll(lexeme string) (bool, error) { + switch strings.ToLower(lexeme) { + case "any": + return true, nil + case "all": + return false, nil + default: + return false, fmt.Errorf("any/all is not parsed") + } +} + func parseStatus(lexeme string) (apechain.Status, error) { action, expression, found := strings.Cut(lexeme, ":") switch strings.ToLower(action) { From 47d9ce71be9e9b85a1da639ee2ad152b76659ea3 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 14 Feb 2024 17:57:19 +0300 Subject: [PATCH 0933/1943] [#986] cli: Allow add-rule command to parse new actions * Introduce Object.* and Container.* actions that span all methods for services. Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/docs/policy.md | 1 + cmd/frostfs-cli/modules/util/ape.go | 69 ++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-cli/docs/policy.md b/cmd/frostfs-cli/docs/policy.md index 43dd97fcf..1d51818ac 100644 --- a/cmd/frostfs-cli/docs/policy.md +++ b/cmd/frostfs-cli/docs/policy.md @@ -27,6 +27,7 @@ For container it can be represented as: - `/*` all containers in the `root` namespace Actions is a regular operations upon FrostFS containers/objects. Like `Object.Put`, `Container.Get` etc. +You can use `Object.*`, `Container.*` that implies all actions. In status section it is possible to use `allow`, `deny` or `deny:QuotaLimitReached` actions. diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index 41d186b0c..1500f0c5c 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -102,6 +102,8 @@ func ParseAPEChain(chain *apechain.Chain, rules []string) error { // allow Object.Get Object.Resource:Department=HR Object.Request:Actor=ownerA * // allow Object.Get any Object.Resource:Department=HR Object.Request:Actor=ownerA * // allow Object.Get all Object.Resource:Department=HR Object.Request:Actor=ownerA * +// allow Object.* * +// allow Container.* * // //nolint:godot func ParseAPERule(r *apechain.Rule, rule string) error { @@ -112,6 +114,18 @@ func ParseAPERule(r *apechain.Rule, rule string) error { return parseRuleLexemes(r, lexemes) } +func unique(inputSlice []string) []string { + uniqueSlice := make([]string, 0, len(inputSlice)) + seen := make(map[string]bool, len(inputSlice)) + for _, element := range inputSlice { + if !seen[element] { + uniqueSlice = append(uniqueSlice, element) + seen[element] = true + } + } + return uniqueSlice +} + func parseRuleLexemes(r *apechain.Rule, lexemes []string) error { if len(lexemes) < 2 { return errInvalidStatementFormat @@ -131,9 +145,9 @@ func parseRuleLexemes(r *apechain.Rule, lexemes []string) error { continue } - var name string + var names []string var actionType bool - name, actionType, err = parseAction(lexeme) + names, actionType, err = parseAction(lexeme) if err != nil { condition, errCond := parseCondition(lexeme) if errCond != nil { @@ -144,7 +158,7 @@ func parseRuleLexemes(r *apechain.Rule, lexemes []string) error { actionType = condition.Object == apechain.ObjectResource || condition.Object == apechain.ObjectRequest r.Condition = append(r.Condition, *condition) } else { - r.Actions.Names = append(r.Actions.Names, name) + r.Actions.Names = append(r.Actions.Names, names...) } if isObject == nil { isObject = &actionType @@ -152,6 +166,7 @@ func parseRuleLexemes(r *apechain.Rule, lexemes []string) error { return errMixedTypesInRule } } + r.Actions.Names = unique(r.Actions.Names) if len(r.Actions.Names) == 0 { return fmt.Errorf("%w:%w", err, errNoActionsInRule) } @@ -198,35 +213,55 @@ func parseStatus(lexeme string) (apechain.Status, error) { } } -func parseAction(lexeme string) (string, bool, error) { +func parseAction(lexeme string) ([]string, bool, error) { switch strings.ToLower(lexeme) { case "object.put": - return nativeschema.MethodPutObject, true, nil + return []string{nativeschema.MethodPutObject}, true, nil case "object.get": - return nativeschema.MethodGetObject, true, nil + return []string{nativeschema.MethodGetObject}, true, nil case "object.head": - return nativeschema.MethodHeadObject, true, nil + return []string{nativeschema.MethodHeadObject}, true, nil case "object.delete": - return nativeschema.MethodDeleteObject, true, nil + return []string{nativeschema.MethodDeleteObject}, true, nil case "object.search": - return nativeschema.MethodSearchObject, true, nil + return []string{nativeschema.MethodSearchObject}, true, nil case "object.range": - return nativeschema.MethodRangeObject, true, nil + return []string{nativeschema.MethodRangeObject}, true, nil case "object.hash": - return nativeschema.MethodHashObject, true, nil + return []string{nativeschema.MethodHashObject}, true, nil + case "object.*": + return []string{ + nativeschema.MethodPutObject, + nativeschema.MethodGetObject, + nativeschema.MethodHeadObject, + nativeschema.MethodDeleteObject, + nativeschema.MethodSearchObject, + nativeschema.MethodHashObject, + }, true, nil case "container.put": - return nativeschema.MethodPutContainer, false, nil + return []string{nativeschema.MethodPutContainer}, false, nil case "container.delete": - return nativeschema.MethodDeleteContainer, false, nil + return []string{nativeschema.MethodDeleteContainer}, false, nil case "container.get": - return nativeschema.MethodGetContainer, false, nil + return []string{nativeschema.MethodGetContainer}, false, nil case "container.setcontainereacl": - return nativeschema.MethodSetContainerEACL, false, nil + return []string{nativeschema.MethodSetContainerEACL}, false, nil case "container.getcontainereacl": - return nativeschema.MethodGetContainerEACL, false, nil + return []string{nativeschema.MethodGetContainerEACL}, false, nil + case "container.list": + return []string{nativeschema.MethodListContainers}, false, nil + case "container.*": + return []string{ + nativeschema.MethodPutContainer, + nativeschema.MethodDeleteContainer, + nativeschema.MethodGetContainer, + nativeschema.MethodSetContainerEACL, + nativeschema.MethodGetContainerEACL, + nativeschema.MethodListContainers, + }, false, nil default: } - return "", false, fmt.Errorf("%w: %s", errUnknownAction, lexeme) + return nil, false, fmt.Errorf("%w: %s", errUnknownAction, lexeme) } func parseResource(lexeme string, isObj bool) (string, error) { From adf7ebab5b295070adb3c866058c218f53d07d42 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 16 Feb 2024 22:49:53 +0300 Subject: [PATCH 0934/1943] [#996] metabase: Speed up bucket creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most of the time it exits, e.g. when it is per-container and use on each object PUT. Bbolt implementation first tries to create bucket and then returns it if it exists. Create operation uses cursor and thus is not very lightweight, we can avoid it. ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ old │ new │ │ sec/op │ sec/op vs base │ Put/parallel-8 174.4µ ± 3% 163.3µ ± 3% -6.39% (p=0.000 n=10) Put/sequential-8 263.3µ ± 2% 259.0µ ± 1% -1.64% (p=0.000 n=10) geomean 214.3µ 205.6µ -4.05% │ old │ new │ │ B/op │ B/op vs base │ Put/parallel-8 275.3Ki ± 3% 281.1Ki ± 4% ~ (p=0.063 n=10) Put/sequential-8 413.0Ki ± 2% 426.6Ki ± 2% +3.29% (p=0.003 n=10) geomean 337.2Ki 346.3Ki +2.70% │ old │ new │ │ allocs/op │ allocs/op vs base │ Put/parallel-8 678.0 ± 1% 524.5 ± 2% -22.64% (p=0.000 n=10) Put/sequential-8 1.329k ± 0% 1.183k ± 0% -10.91% (p=0.000 n=10) geomean 949.1 787.9 -16.98% ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/counter.go | 4 ++-- pkg/local_object_storage/metabase/put.go | 23 +++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index 915fbd957..ee1ab41de 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -346,7 +346,7 @@ func (db *DB) incContainerObjectCounter(tx *bbolt.Tx, cnrID cid.ID, isUserObject // Does nothing if counters are not empty and force is false. If force is // true, updates the counters anyway. func syncCounter(tx *bbolt.Tx, force bool) error { - shardInfoB, err := tx.CreateBucketIfNotExists(shardInfoBucket) + shardInfoB, err := createBucketLikelyExists(tx, shardInfoBucket) if err != nil { return fmt.Errorf("could not get shard info bucket: %w", err) } @@ -359,7 +359,7 @@ func syncCounter(tx *bbolt.Tx, force bool) error { return nil } - containerCounterB, err := tx.CreateBucketIfNotExists(containerCounterBucketName) + containerCounterB, err := createBucketLikelyExists(tx, containerCounterBucketName) if err != nil { return fmt.Errorf("could not get container counter bucket: %w", err) } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 1f0022389..5d8e4d263 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -356,8 +356,21 @@ func updateFKBTIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun return nil } +type bucketContainer interface { + Bucket([]byte) *bbolt.Bucket + CreateBucket([]byte) (*bbolt.Bucket, error) + CreateBucketIfNotExists([]byte) (*bbolt.Bucket, error) +} + +func createBucketLikelyExists[T bucketContainer](tx T, name []byte) (*bbolt.Bucket, error) { + if bkt := tx.Bucket(name); bkt != nil { + return bkt, nil + } + return tx.CreateBucket(name) +} + func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error { - bkt, err := tx.CreateBucketIfNotExists(item.name) + bkt, err := createBucketLikelyExists(tx, item.name) if err != nil { return fmt.Errorf("can't create index %v: %w", item.name, err) } @@ -366,12 +379,12 @@ func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error { } func putFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { - bkt, err := tx.CreateBucketIfNotExists(item.name) + bkt, err := createBucketLikelyExists(tx, item.name) if err != nil { return fmt.Errorf("can't create index %v: %w", item.name, err) } - fkbtRoot, err := bkt.CreateBucketIfNotExists(item.key) + fkbtRoot, err := createBucketLikelyExists(bkt, item.key) if err != nil { return fmt.Errorf("can't create fake bucket tree index %v: %w", item.key, err) } @@ -380,7 +393,7 @@ func putFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { } func putListIndexItem(tx *bbolt.Tx, item namedBucketItem) error { - bkt, err := tx.CreateBucketIfNotExists(item.name) + bkt, err := createBucketLikelyExists(tx, item.name) if err != nil { return fmt.Errorf("can't create index %v: %w", item.name, err) } @@ -474,7 +487,7 @@ func getVarUint(data []byte) (uint64, int, error) { // storage location to another. func updateStorageID(tx *bbolt.Tx, addr oid.Address, id []byte) error { key := make([]byte, bucketKeySize) - bkt, err := tx.CreateBucketIfNotExists(smallBucketName(addr.Container(), key)) + bkt, err := createBucketLikelyExists(tx, smallBucketName(addr.Container(), key)) if err != nil { return err } From 7470c383dda3cdb86e6c5c4761f67a98a88c7383 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Feb 2024 15:55:18 +0300 Subject: [PATCH 0935/1943] [#997] metabase: Drop toMoveIt bucket It is not used. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 - pkg/local_object_storage/engine/evacuate.go | 2 +- pkg/local_object_storage/engine/put.go | 20 +-- .../metabase/containers_test.go | 17 --- pkg/local_object_storage/metabase/control.go | 13 +- pkg/local_object_storage/metabase/delete.go | 5 - .../metabase/delete_test.go | 14 -- pkg/local_object_storage/metabase/movable.go | 144 ------------------ .../metabase/movable_test.go | 85 ----------- pkg/local_object_storage/shard/move.go | 62 -------- 10 files changed, 16 insertions(+), 347 deletions(-) delete mode 100644 pkg/local_object_storage/metabase/movable.go delete mode 100644 pkg/local_object_storage/metabase/movable_test.go delete mode 100644 pkg/local_object_storage/shard/move.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index d839915d8..6b48940b7 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -220,7 +220,6 @@ const ( EngineCouldNotCloseShard = "could not close shard" EngineCouldNotReloadAShard = "could not reload a shard" EngineAddedNewShard = "added new shard" - EngineCouldNotMarkObjectForShardRelocation = "could not mark object for shard relocation" EngineCouldNotPutObjectToShard = "could not put object to shard" EngineErrorDuringSearchingForObjectChildren = "error during searching for object children" EngineCouldNotInhumeObjectInShard = "could not inhume object in shard" diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index becddfa8b..67db518c6 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -700,7 +700,7 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add if _, ok := shardsToEvacuate[shards[j].ID().String()]; ok { continue } - switch e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object).status { + switch e.putToShard(ctx, shards[j].hashedShard, shards[j].pool, addr, object).status { case putToShardSuccess: res.objEvacuated.Add(1) e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 2b1712f3f..c1cd29cb8 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -84,7 +84,7 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { } var shRes putToShardRes - e.iterateOverSortedShards(addr, func(ind int, sh hashedShard) (stop bool) { + e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { e.mtx.RLock() pool, ok := e.shardPools[sh.ID().String()] e.mtx.RUnlock() @@ -92,7 +92,7 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { // Shard was concurrently removed, skip. return false } - shRes = e.putToShard(ctx, sh, ind, pool, addr, prm.obj) + shRes = e.putToShard(ctx, sh, pool, addr, prm.obj) return shRes.status != putToShardUnknown }) switch shRes.status { @@ -109,7 +109,7 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { // putToShard puts object to sh. // Return putToShardStatus and error if it is necessary to propagate an error upper. -func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, pool util.WorkerPool, +func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool util.WorkerPool, addr oid.Address, obj *objectSDK.Object, ) (res putToShardRes) { exitCh := make(chan struct{}) @@ -132,20 +132,6 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, ind int, } if exists.Exists() { - if ind != 0 { - var toMoveItPrm shard.ToMoveItPrm - toMoveItPrm.SetAddress(addr) - - _, err = sh.ToMoveIt(ctx, toMoveItPrm) - if err != nil { - e.log.Warn(logs.EngineCouldNotMarkObjectForShardRelocation, - zap.Stringer("shard", sh.ID()), - zap.String("error", err.Error()), - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), - ) - } - } - res.status = putToShardExists return } diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index e7762ded6..a90cd9d97 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -74,23 +74,6 @@ func TestDB_Containers(t *testing.T) { require.NoError(t, err) assertContains(cnrs, cnr) }) - - t.Run("ToMoveIt", func(t *testing.T) { - obj := testutil.GenerateObject() - - require.NoError(t, putBig(db, obj)) - - cnrs, err := db.Containers(context.Background()) - require.NoError(t, err) - cnr, _ := obj.ContainerID() - assertContains(cnrs, cnr) - - require.NoError(t, metaToMoveIt(db, object.AddressOf(obj))) - - cnrs, err = db.Containers(context.Background()) - require.NoError(t, err) - assertContains(cnrs, cnr) - }) } func TestDB_ContainersCount(t *testing.T) { diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index c1a86819d..ec4c29c4b 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -117,12 +117,16 @@ func (db *DB) init(reset bool) error { string(containerVolumeBucketName): {}, string(containerCounterBucketName): {}, string(graveyardBucketName): {}, - string(toMoveItBucketName): {}, string(garbageBucketName): {}, string(shardInfoBucket): {}, string(bucketNameLocked): {}, } + // buckets that are not used anymore + deprecatedBuckets := [][]byte{ + toMoveItBucketName, + } + return db.boltDB.Update(func(tx *bbolt.Tx) error { var err error if !reset { @@ -147,6 +151,13 @@ func (db *DB) init(reset bool) error { } } + for _, b := range deprecatedBuckets { + err := tx.DeleteBucket(b) + if err != nil && !errors.Is(err, bbolt.ErrBucketNotFound) { + return fmt.Errorf("could not delete deprecated bucket %s: %w", string(b), err) + } + } + if !reset { // counters will be recalculated by refill metabase err = syncCounter(tx, false) if err != nil { diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 8e5557636..e5a98ff01 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -434,7 +434,6 @@ func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error addr := object.AddressOf(obj) objKey := objectKey(addr.Object(), make([]byte, objectKeySize)) - addrKey := addressKey(addr, make([]byte, addressKeySize)) cnr := addr.Container() bucketName := make([]byte, bucketKeySize) @@ -470,10 +469,6 @@ func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error name: rootBucketName(cnr, bucketName), key: objKey, }) - delUniqueIndexItem(tx, namedBucketItem{ // remove from ToMoveIt index - name: toMoveItBucketName, - key: addrKey, - }) return nil } diff --git a/pkg/local_object_storage/metabase/delete_test.go b/pkg/local_object_storage/metabase/delete_test.go index 649c0ebe3..2053874d0 100644 --- a/pkg/local_object_storage/metabase/delete_test.go +++ b/pkg/local_object_storage/metabase/delete_test.go @@ -33,15 +33,6 @@ func TestDB_Delete(t *testing.T) { err := putBig(db, child) require.NoError(t, err) - // fill ToMoveIt index - err = metaToMoveIt(db, object.AddressOf(child)) - require.NoError(t, err) - - // check if Movable list is not empty - l, err := metaMovable(db) - require.NoError(t, err) - require.Len(t, l, 1) - // try to remove parent, should be no-op, error-free err = metaDelete(db, object.AddressOf(parent)) require.NoError(t, err) @@ -61,11 +52,6 @@ func TestDB_Delete(t *testing.T) { err = metaDelete(db, object.AddressOf(child)) require.NoError(t, err) - // check if there is no data in Movable index - l, err = metaMovable(db) - require.NoError(t, err) - require.Len(t, l, 0) - // check if they marked as already removed ok, err := metaExists(db, object.AddressOf(child)) diff --git a/pkg/local_object_storage/metabase/movable.go b/pkg/local_object_storage/metabase/movable.go deleted file mode 100644 index 763e49a5d..000000000 --- a/pkg/local_object_storage/metabase/movable.go +++ /dev/null @@ -1,144 +0,0 @@ -package meta - -import ( - "context" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" - "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.etcd.io/bbolt" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -// ToMoveItPrm groups the parameters of ToMoveIt operation. -type ToMoveItPrm struct { - addr oid.Address -} - -// ToMoveItRes groups the resulting values of ToMoveIt operation. -type ToMoveItRes struct{} - -// SetAddress sets address of the object to move into another shard. -func (p *ToMoveItPrm) SetAddress(addr oid.Address) { - p.addr = addr -} - -// DoNotMovePrm groups the parameters of DoNotMove operation. -type DoNotMovePrm struct { - addr oid.Address -} - -// DoNotMoveRes groups the resulting values of DoNotMove operation. -type DoNotMoveRes struct{} - -// SetAddress sets address of the object to prevent moving into another shard. -func (p *DoNotMovePrm) SetAddress(addr oid.Address) { - p.addr = addr -} - -// MovablePrm groups the parameters of Movable operation. -type MovablePrm struct{} - -// MovableRes groups the resulting values of Movable operation. -type MovableRes struct { - addrList []oid.Address -} - -// AddressList returns resulting addresses of Movable operation. -func (p MovableRes) AddressList() []oid.Address { - return p.addrList -} - -// ToMoveIt marks objects to move it into another shard. This useful for -// faster HRW fetching. -func (db *DB) ToMoveIt(ctx context.Context, prm ToMoveItPrm) (res ToMoveItRes, err error) { - _, span := tracing.StartSpanFromContext(ctx, "metabase.ToMoveIt", - trace.WithAttributes( - attribute.String("address", prm.addr.EncodeToString()), - )) - defer span.End() - - db.modeMtx.RLock() - defer db.modeMtx.RUnlock() - - if db.mode.NoMetabase() { - return res, ErrDegradedMode - } else if db.mode.ReadOnly() { - return res, ErrReadOnlyMode - } - - key := make([]byte, addressKeySize) - key = addressKey(prm.addr, key) - - err = db.boltDB.Update(func(tx *bbolt.Tx) error { - toMoveIt := tx.Bucket(toMoveItBucketName) - return toMoveIt.Put(key, zeroValue) - }) - - return res, metaerr.Wrap(err) -} - -// DoNotMove removes `MoveIt` mark from the object. -func (db *DB) DoNotMove(prm DoNotMovePrm) (res DoNotMoveRes, err error) { - db.modeMtx.RLock() - defer db.modeMtx.RUnlock() - - if db.mode.NoMetabase() { - return res, ErrDegradedMode - } else if db.mode.ReadOnly() { - return res, ErrReadOnlyMode - } - - key := make([]byte, addressKeySize) - key = addressKey(prm.addr, key) - - err = db.boltDB.Update(func(tx *bbolt.Tx) error { - toMoveIt := tx.Bucket(toMoveItBucketName) - return toMoveIt.Delete(key) - }) - - return res, metaerr.Wrap(err) -} - -// Movable returns list of marked objects to move into other shard. -func (db *DB) Movable(_ MovablePrm) (MovableRes, error) { - db.modeMtx.RLock() - defer db.modeMtx.RUnlock() - - if db.mode.NoMetabase() { - return MovableRes{}, ErrDegradedMode - } - - var strAddrs []string - - err := db.boltDB.View(func(tx *bbolt.Tx) error { - toMoveIt := tx.Bucket(toMoveItBucketName) - return toMoveIt.ForEach(func(k, v []byte) error { - strAddrs = append(strAddrs, string(k)) - - return nil - }) - }) - if err != nil { - return MovableRes{}, metaerr.Wrap(err) - } - - // we can parse strings to structures in-place, but probably it seems - // more efficient to keep bolt db TX code smaller because it might be - // bottleneck. - addrs := make([]oid.Address, len(strAddrs)) - - for i := range strAddrs { - err = decodeAddressFromKey(&addrs[i], []byte(strAddrs[i])) - if err != nil { - return MovableRes{}, metaerr.Wrap(fmt.Errorf("can't parse object address %v: %w", - strAddrs[i], err)) - } - } - - return MovableRes{ - addrList: addrs, - }, nil -} diff --git a/pkg/local_object_storage/metabase/movable_test.go b/pkg/local_object_storage/metabase/movable_test.go deleted file mode 100644 index d66d92867..000000000 --- a/pkg/local_object_storage/metabase/movable_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package meta_test - -import ( - "context" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/stretchr/testify/require" -) - -func TestDB_Movable(t *testing.T) { - db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() - - raw1 := testutil.GenerateObject() - raw2 := testutil.GenerateObject() - - // put two objects in metabase - err := putBig(db, raw1) - require.NoError(t, err) - - err = putBig(db, raw2) - require.NoError(t, err) - - // check if toMoveIt index empty - toMoveList, err := metaMovable(db) - require.NoError(t, err) - require.Len(t, toMoveList, 0) - - // mark to move object2 - err = metaToMoveIt(db, object.AddressOf(raw2)) - require.NoError(t, err) - - // check if toMoveIt index contains address of object 2 - toMoveList, err = metaMovable(db) - require.NoError(t, err) - require.Len(t, toMoveList, 1) - require.Contains(t, toMoveList, object.AddressOf(raw2)) - - // remove from toMoveIt index non existing address - err = metaDoNotMove(db, object.AddressOf(raw1)) - require.NoError(t, err) - - // check if toMoveIt index hasn't changed - toMoveList, err = metaMovable(db) - require.NoError(t, err) - require.Len(t, toMoveList, 1) - - // remove from toMoveIt index existing address - err = metaDoNotMove(db, object.AddressOf(raw2)) - require.NoError(t, err) - - // check if toMoveIt index is empty now - toMoveList, err = metaMovable(db) - require.NoError(t, err) - require.Len(t, toMoveList, 0) -} - -func metaToMoveIt(db *meta.DB, addr oid.Address) error { - var toMovePrm meta.ToMoveItPrm - toMovePrm.SetAddress(addr) - - _, err := db.ToMoveIt(context.Background(), toMovePrm) - return err -} - -func metaMovable(db *meta.DB) ([]oid.Address, error) { - r, err := db.Movable(meta.MovablePrm{}) - if err != nil { - return nil, err - } - - return r.AddressList(), nil -} - -func metaDoNotMove(db *meta.DB, addr oid.Address) error { - var doNotMovePrm meta.DoNotMovePrm - doNotMovePrm.SetAddress(addr) - - _, err := db.DoNotMove(doNotMovePrm) - return err -} diff --git a/pkg/local_object_storage/shard/move.go b/pkg/local_object_storage/shard/move.go deleted file mode 100644 index 539dbc97f..000000000 --- a/pkg/local_object_storage/shard/move.go +++ /dev/null @@ -1,62 +0,0 @@ -package shard - -import ( - "context" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" - "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - "go.uber.org/zap" -) - -// ToMoveItPrm encapsulates parameters for ToMoveIt operation. -type ToMoveItPrm struct { - addr oid.Address -} - -// ToMoveItRes encapsulates results of ToMoveIt operation. -type ToMoveItRes struct{} - -// SetAddress sets object address that should be marked to move into another -// shard. -func (p *ToMoveItPrm) SetAddress(addr oid.Address) { - p.addr = addr -} - -// ToMoveIt calls metabase.ToMoveIt method to mark object as relocatable to -// another shard. -func (s *Shard) ToMoveIt(ctx context.Context, prm ToMoveItPrm) (ToMoveItRes, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "Shard.ToMoveIt", - trace.WithAttributes( - attribute.String("shard_id", s.ID().String()), - attribute.String("address", prm.addr.EncodeToString()), - )) - defer span.End() - - s.m.RLock() - defer s.m.RUnlock() - - m := s.info.Mode - if m.ReadOnly() { - return ToMoveItRes{}, ErrReadOnlyMode - } else if m.NoMetabase() { - return ToMoveItRes{}, ErrDegradedMode - } - - var toMovePrm meta.ToMoveItPrm - toMovePrm.SetAddress(prm.addr) - - _, err := s.metaBase.ToMoveIt(ctx, toMovePrm) - if err != nil { - s.log.Debug(logs.ShardCouldNotMarkObjectForShardRelocationInMetabase, - zap.String("error", err.Error()), - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), - ) - } - - return ToMoveItRes{}, nil -} From e18f0f517856668e35bb8753fa89214803d77f7c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 20 Feb 2024 18:59:35 +0300 Subject: [PATCH 0936/1943] [#999] morph: Use Global scope for proxy contract Proxy contract can now be used as an owner of NNS domains, thus we need it not only to pay for the transaction but also to check domain ownership. CalledByEntry is not enough, because we may register NNS domains owned by proxy indirectly from the container contract. Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/notary.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 5b817a805..1665fec1d 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -569,7 +569,11 @@ func (c *Client) notaryCosigners(invokedByAlpha bool, ir []*keys.PublicKey, comm s[0] = actor.SignerAccount{ Signer: transaction.Signer{ Account: c.notary.proxy, - Scopes: transaction.None, + // Do not change this: + // We must be able to call NNS contract indirectly from the Container contract. + // Thus, CalledByEntry is not sufficient. + // In future we may restrict this to all the usecases we have. + Scopes: transaction.Global, }, Account: notary.FakeContractAccount(c.notary.proxy), } From 4b13b85173402860f41a2d9aede7c13a4ccc8375 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 21 Feb 2024 15:22:50 +0300 Subject: [PATCH 0937/1943] [#1000] morph: Fix batch size in TraverseIterator() Initial prefetch size can be arbitrary an restricted only by VM/RPC limits. For TraverseIterator() there is an explicit check on the server-side, though. Introduced in df055fead56e6ac. Refs #931. Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 88f9703e7..2570bc2c9 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -260,8 +260,13 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int _ = c.rpcActor.TerminateSession(sid) }() + // Batch size for TraverseIterator() can restricted on the server-side. + traverseBatchSize := batchSize + if invoker.DefaultIteratorResultItems < traverseBatchSize { + traverseBatchSize = invoker.DefaultIteratorResultItems + } for { - items, err := c.rpcActor.TraverseIterator(sid, &r, batchSize) + items, err := c.rpcActor.TraverseIterator(sid, &r, traverseBatchSize) if err != nil { return err } @@ -271,7 +276,7 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int return err } } - if len(items) < batchSize { + if len(items) < traverseBatchSize { break } } From abea258b657cb5208afae58b08f79e9385fe1825 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 21 Feb 2024 15:29:08 +0300 Subject: [PATCH 0938/1943] [#1000] adm: Use default batch size for TraverseIterator() Nothing is broken now, but will easily become if we change nnsMaxTokens, thus this change. Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/contract/dump_hashes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index 6e406be56..11cc3bd04 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -177,12 +177,12 @@ func dumpCustomZoneHashes(cmd *cobra.Command, nnsHash util.Uint160, zone string, _ = inv.TerminateSession(sessionID) }() - items, err := inv.TraverseIterator(sessionID, &iter, nnsMaxTokens) + items, err := inv.TraverseIterator(sessionID, &iter, 0) for err == nil && len(items) != 0 { for i := range items { processItem(items[i]) } - items, err = inv.TraverseIterator(sessionID, &iter, nnsMaxTokens) + items, err = inv.TraverseIterator(sessionID, &iter, 0) } if err != nil { return fmt.Errorf("error during NNS domains iteration: %w", err) From 2ad433dbcbab64caea243dbbae4553c7c0931719 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 26 Feb 2024 11:19:52 +0300 Subject: [PATCH 0939/1943] [#1005] engine: Drop shards weights Unused. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 63 +++++++++---------- .../engine/remove_copies.go | 2 +- .../engine/remove_copies_test.go | 2 +- pkg/local_object_storage/engine/shards.go | 21 +------ .../engine/shards_test.go | 25 ++++++++ pkg/local_object_storage/engine/tree.go | 14 ++--- pkg/local_object_storage/shard/info.go | 3 - pkg/local_object_storage/shard/weight.go | 12 ---- 8 files changed, 65 insertions(+), 77 deletions(-) delete mode 100644 pkg/local_object_storage/shard/weight.go diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 67db518c6..45600fe66 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -221,7 +221,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev )) defer span.End() - shards, weights, err := e.getActualShards(shardIDs, prm) + shards, err := e.getActualShards(shardIDs, prm) if err != nil { return nil, err } @@ -243,7 +243,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev } eg.Go(func() error { - return e.evacuateShards(egCtx, shardIDs, prm, res, shards, weights, shardsToEvacuate) + return e.evacuateShards(egCtx, shardIDs, prm, res, shards, shardsToEvacuate) }) if prm.Async { @@ -261,7 +261,7 @@ func ctxOrBackground(ctx context.Context, background bool) context.Context { } func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, + shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) error { var err error ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShards", @@ -288,7 +288,7 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p } for _, shardID := range shardIDs { - if err = e.evacuateShard(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { + if err = e.evacuateShard(ctx, shardID, prm, res, shards, shardsToEvacuate); err != nil { e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) return err @@ -336,7 +336,7 @@ func (e *StorageEngine) getTotals(ctx context.Context, prm EvacuateShardPrm, sha } func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, + shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShard", trace.WithAttributes( @@ -345,13 +345,13 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E defer span.End() if prm.Scope.WithObjects() { - if err := e.evacuateShardObjects(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { + if err := e.evacuateShardObjects(ctx, shardID, prm, res, shards, shardsToEvacuate); err != nil { return err } } if prm.Scope.WithTrees() && shardsToEvacuate[shardID].PiloramaEnabled() { - if err := e.evacuateShardTrees(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { + if err := e.evacuateShardTrees(ctx, shardID, prm, res, shards, shardsToEvacuate); err != nil { return err } } @@ -360,7 +360,7 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E } func (e *StorageEngine) evacuateShardObjects(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, + shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) error { var listPrm shard.ListWithCursorPrm listPrm.WithCount(defaultEvacuateBatchSize) @@ -383,7 +383,7 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, shardID string return err } - if err = e.evacuateObjects(ctx, sh, listRes.AddressList(), prm, res, shards, weights, shardsToEvacuate); err != nil { + if err = e.evacuateObjects(ctx, sh, listRes.AddressList(), prm, res, shards, shardsToEvacuate); err != nil { return err } @@ -393,7 +393,7 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, shardID string } func (e *StorageEngine) evacuateShardTrees(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, + shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) error { sh := shardsToEvacuate[shardID] @@ -414,7 +414,7 @@ func (e *StorageEngine) evacuateShardTrees(ctx context.Context, shardID string, return err } listPrm.NextPageToken = listRes.NextPageToken - if err := e.evacuateTrees(ctx, sh, listRes.Items, prm, res, shards, weights, shardsToEvacuate); err != nil { + if err := e.evacuateTrees(ctx, sh, listRes.Items, prm, res, shards, shardsToEvacuate); err != nil { return err } } @@ -422,8 +422,7 @@ func (e *StorageEngine) evacuateShardTrees(ctx context.Context, shardID string, } func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, trees []pilorama.ContainerIDTreeID, - prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, weights []float64, - shardsToEvacuate map[string]*shard.Shard, + prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateTrees", trace.WithAttributes( @@ -438,7 +437,7 @@ func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, tree default: } - success, shardID, err := e.tryEvacuateTreeLocal(ctx, sh, contTree, prm, shards, weights, shardsToEvacuate) + success, shardID, err := e.tryEvacuateTreeLocal(ctx, sh, contTree, prm, shards, shardsToEvacuate) if err != nil { return err } @@ -477,9 +476,9 @@ func (e *StorageEngine) evacuateTreeToOtherNode(ctx context.Context, sh *shard.S } func (e *StorageEngine) tryEvacuateTreeLocal(ctx context.Context, sh *shard.Shard, tree pilorama.ContainerIDTreeID, - prm EvacuateShardPrm, shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, + prm EvacuateShardPrm, shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) (bool, string, error) { - target, found, err := e.findShardToEvacuateTree(ctx, tree, shards, weights, shardsToEvacuate) + target, found, err := e.findShardToEvacuateTree(ctx, tree, shards, shardsToEvacuate) if err != nil { return false, "", err } @@ -547,9 +546,9 @@ func (e *StorageEngine) tryEvacuateTreeLocal(ctx context.Context, sh *shard.Shar // findShardToEvacuateTree returns first shard according HRW or first shard with tree exists. func (e *StorageEngine) findShardToEvacuateTree(ctx context.Context, tree pilorama.ContainerIDTreeID, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, + shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) (pooledShard, bool, error) { - hrw.SortHasherSliceByWeightValue(shards, weights, hrw.StringHash(tree.CID.EncodeToString())) + hrw.SortHasherSliceByValue(shards, hrw.StringHash(tree.CID.EncodeToString())) var result pooledShard var found bool for _, target := range shards { @@ -583,31 +582,31 @@ func (e *StorageEngine) findShardToEvacuateTree(ctx context.Context, tree pilora return result, found, nil } -func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) ([]pooledShard, []float64, error) { +func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) ([]pooledShard, error) { e.mtx.RLock() defer e.mtx.RUnlock() for i := range shardIDs { sh, ok := e.shards[shardIDs[i]] if !ok { - return nil, nil, errShardNotFound + return nil, errShardNotFound } if !sh.GetMode().ReadOnly() { - return nil, nil, ErrMustBeReadOnly + return nil, ErrMustBeReadOnly } if prm.Scope.TreesOnly() && !sh.PiloramaEnabled() { - return nil, nil, fmt.Errorf("shard %s doesn't have pilorama enabled", sh.ID()) + return nil, fmt.Errorf("shard %s doesn't have pilorama enabled", sh.ID()) } } if len(e.shards)-len(shardIDs) < 1 && prm.ObjectsHandler == nil && prm.Scope.WithObjects() { - return nil, nil, errMustHaveTwoShards + return nil, errMustHaveTwoShards } if len(e.shards)-len(shardIDs) < 1 && prm.TreeHandler == nil && prm.Scope.WithTrees() { - return nil, nil, errMustHaveTwoShards + return nil, errMustHaveTwoShards } // We must have all shards, to have correct information about their @@ -620,17 +619,11 @@ func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) pool: e.shardPools[id], }) } - - weights := make([]float64, 0, len(shards)) - for i := range shards { - weights = append(weights, e.shardWeight(shards[i].Shard)) - } - - return shards, weights, nil + return shards, nil } func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, toEvacuate []object.AddressWithType, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, + shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateObjects", trace.WithAttributes( @@ -660,7 +653,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to return err } - evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), sh, shards, weights, shardsToEvacuate, res) + evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), sh, shards, shardsToEvacuate, res) if err != nil { return err } @@ -687,9 +680,9 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to } func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, - shards []pooledShard, weights []float64, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes, + shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes, ) (bool, error) { - hrw.SortHasherSliceByWeightValue(shards, weights, hrw.StringHash(addr.EncodeToString())) + hrw.SortHasherSliceByValue(shards, hrw.StringHash(addr.EncodeToString())) for j := range shards { select { case <-ctx.Done(): diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index 573153268..00562e4cb 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -111,7 +111,7 @@ func (e *StorageEngine) removeObjects(ctx context.Context, ch <-chan oid.Address for addr := range ch { h := hrw.StringHash(addr.EncodeToString()) - shards := sortShardsByWeight(shards, h) + hrw.SortHasherSliceByValue(shards, h) found := false for i := range shards { var existsPrm shard.ExistsPrm diff --git a/pkg/local_object_storage/engine/remove_copies_test.go b/pkg/local_object_storage/engine/remove_copies_test.go index dfb2423ff..99963576c 100644 --- a/pkg/local_object_storage/engine/remove_copies_test.go +++ b/pkg/local_object_storage/engine/remove_copies_test.go @@ -41,7 +41,7 @@ func TestRebalance(t *testing.T) { obj.SetPayload(make([]byte, errSmallSize)) objects[i].object = obj - shards := te.ng.sortShardsByWeight(object.AddressOf(obj)) + shards := te.ng.sortShards(object.AddressOf(obj)) objects[i].bestShard = *shards[0].Shard.ID() objects[i].worstShard = *shards[1].Shard.ID() } diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 83fa4f5cb..4bbf7eff0 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -254,13 +254,7 @@ func generateShardID() (*shard.ID, error) { return shard.NewIDFromBytes(bin), nil } -func (e *StorageEngine) shardWeight(sh *shard.Shard) float64 { - weightValues := sh.WeightValues() - - return float64(weightValues.FreeSpace) -} - -func (e *StorageEngine) sortShardsByWeight(objAddr interface{ EncodeToString() string }) []hashedShard { +func (e *StorageEngine) sortShards(objAddr interface{ EncodeToString() string }) []hashedShard { e.mtx.RLock() defer e.mtx.RUnlock() @@ -269,16 +263,7 @@ func (e *StorageEngine) sortShardsByWeight(objAddr interface{ EncodeToString() s for _, sh := range e.shards { shards = append(shards, hashedShard(sh)) } - return sortShardsByWeight(shards, h) -} - -func sortShardsByWeight(shards []hashedShard, h uint64) []hashedShard { - weights := make([]float64, 0, len(shards)) - for _, sh := range shards { - weights = append(weights, float64(sh.Shard.WeightValues().FreeSpace)) - } - - hrw.SortHasherSliceByWeightValue(shards, weights, h) + hrw.SortHasherSliceByValue(shards, h) return shards } @@ -296,7 +281,7 @@ func (e *StorageEngine) unsortedShards() []hashedShard { } func (e *StorageEngine) iterateOverSortedShards(addr oid.Address, handler func(int, hashedShard) (stop bool)) { - for i, sh := range e.sortShardsByWeight(addr) { + for i, sh := range e.sortShards(addr) { if handler(i, sh) { break } diff --git a/pkg/local_object_storage/engine/shards_test.go b/pkg/local_object_storage/engine/shards_test.go index 3bb602ce4..f4c7a4309 100644 --- a/pkg/local_object_storage/engine/shards_test.go +++ b/pkg/local_object_storage/engine/shards_test.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/hrw" "github.com/stretchr/testify/require" ) @@ -62,3 +63,27 @@ func TestDisableShards(t *testing.T) { require.Equal(t, 1, len(e.shards)) } + +func TestSortShardsByWeight(t *testing.T) { + t.Parallel() + + const numOfShards = 500 + + var shards1 []hashedShard + var weights1 []float64 + var shards2 []hashedShard + for i := 0; i < numOfShards; i++ { + shards1 = append(shards1, hashedShard{ + hash: uint64(i), + }) + weights1 = append(weights1, 0) + shards2 = append(shards2, hashedShard{ + hash: uint64(i), + }) + } + + hrw.SortHasherSliceByWeightValue(shards1, weights1, 0) + hrw.SortHasherSliceByValue(shards2, 0) + + require.Equal(t, shards1, shards2) +} diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 6c3594bab..f50b9cb68 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -125,7 +125,7 @@ func (e *StorageEngine) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID var err error var nodes []pilorama.Node - for _, sh := range e.sortShardsByWeight(cid) { + for _, sh := range e.sortShards(cid) { nodes, err = sh.TreeGetByPath(ctx, cid, treeID, attr, path, latest) if err != nil { if err == shard.ErrPiloramaDisabled { @@ -158,7 +158,7 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s var err error var m pilorama.Meta var p uint64 - for _, sh := range e.sortShardsByWeight(cid) { + for _, sh := range e.sortShards(cid) { m, p, err = sh.TreeGetMeta(ctx, cid, treeID, nodeID) if err != nil { if err == shard.ErrPiloramaDisabled { @@ -190,7 +190,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree var err error var nodes []pilorama.NodeInfo - for _, sh := range e.sortShardsByWeight(cid) { + for _, sh := range e.sortShards(cid) { nodes, err = sh.TreeGetChildren(ctx, cid, treeID, nodeID) if err != nil { if err == shard.ErrPiloramaDisabled { @@ -222,7 +222,7 @@ func (e *StorageEngine) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID var err error var lm pilorama.Move - for _, sh := range e.sortShardsByWeight(cid) { + for _, sh := range e.sortShards(cid) { lm, err = sh.TreeGetOpLog(ctx, cid, treeID, height) if err != nil { if err == shard.ErrPiloramaDisabled { @@ -252,7 +252,7 @@ func (e *StorageEngine) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID stri defer span.End() var err error - for _, sh := range e.sortShardsByWeight(cid) { + for _, sh := range e.sortShards(cid) { err = sh.TreeDrop(ctx, cid, treeID) if err != nil { if err == shard.ErrPiloramaDisabled { @@ -375,7 +375,7 @@ func (e *StorageEngine) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, t var err error var height uint64 - for _, sh := range e.sortShardsByWeight(cid) { + for _, sh := range e.sortShards(cid) { height, err = sh.TreeLastSyncHeight(ctx, cid, treeID) if err != nil { if err == shard.ErrPiloramaDisabled { @@ -395,7 +395,7 @@ func (e *StorageEngine) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, t } func (e *StorageEngine) getTreeShard(ctx context.Context, cid cidSDK.ID, treeID string) (int, []hashedShard, error) { - lst := e.sortShardsByWeight(cid) + lst := e.sortShards(cid) for i, sh := range lst { exists, err := sh.TreeExists(ctx, cid, treeID) if err != nil { diff --git a/pkg/local_object_storage/shard/info.go b/pkg/local_object_storage/shard/info.go index a5b9c1a7d..1051ab3db 100644 --- a/pkg/local_object_storage/shard/info.go +++ b/pkg/local_object_storage/shard/info.go @@ -25,9 +25,6 @@ type Info struct { // Information about the Write Cache. WriteCacheInfo writecache.Info - // Weight parameters of the shard. - WeightValues WeightValues - // ErrorCount contains amount of errors occurred in shard operations. ErrorCount uint32 diff --git a/pkg/local_object_storage/shard/weight.go b/pkg/local_object_storage/shard/weight.go deleted file mode 100644 index 0ab5ad61d..000000000 --- a/pkg/local_object_storage/shard/weight.go +++ /dev/null @@ -1,12 +0,0 @@ -package shard - -// WeightValues groups values of Shard weight parameters. -type WeightValues struct { - // Amount of free disk space. Measured in kilobytes. - FreeSpace uint64 -} - -// WeightValues returns current weight values of the Shard. -func (s *Shard) WeightValues() WeightValues { - return s.info.WeightValues -} From 918613546f0812234869b4f6906bb74adbba9e15 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 26 Feb 2024 16:30:03 +0300 Subject: [PATCH 0940/1943] [#1008] metabase: Do not update storageID on put There may be a race condition between put an object and flushing the writecache: 1. Put object to the writecache 2. Writecache flushes object to the blobstore and sets blobstore's storageID 3. Put object to the metabase, set writecache's storageID Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/put.go | 20 +++++++------- .../metabase/storage_id.go | 13 +--------- .../metabase/storage_id_test.go | 26 +++++++++++++++++++ 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 5d8e4d263..429d981fe 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -137,7 +137,7 @@ func (db *DB) updateObj(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *obje // When storage engine moves objects between different sub-storages, // it calls metabase.Put method with new storage ID, thus triggering this code. if !isParent && id != nil { - return updateStorageID(tx, objectCore.AddressOf(obj), id) + return setStorageID(tx, objectCore.AddressOf(obj), id, true) } // when storage already has last object in split hierarchy and there is @@ -236,12 +236,7 @@ func putUniqueIndexes( // index storageID if it is present if id != nil { - err = putUniqueIndexItem(tx, namedBucketItem{ - name: smallBucketName(cnr, bucketName), - key: objKey, - val: id, - }) - if err != nil { + if err = setStorageID(tx, objectCore.AddressOf(obj), id, false); err != nil { return err } } @@ -483,16 +478,19 @@ func getVarUint(data []byte) (uint64, int, error) { } } -// updateStorageID for existing objects if they were moved from one +// setStorageID for existing objects if they were moved from one // storage location to another. -func updateStorageID(tx *bbolt.Tx, addr oid.Address, id []byte) error { +func setStorageID(tx *bbolt.Tx, addr oid.Address, id []byte, override bool) error { key := make([]byte, bucketKeySize) bkt, err := createBucketLikelyExists(tx, smallBucketName(addr.Container(), key)) if err != nil { return err } - - return bkt.Put(objectKey(addr.Object(), key), id) + key = objectKey(addr.Object(), key) + if override || bkt.Get(key) == nil { + return bkt.Put(key, id) + } + return nil } // updateSpliInfo for existing objects if storage filled with extra information diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index ef6b44dfd..f9767935c 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -3,11 +3,9 @@ package meta import ( "bytes" "context" - "errors" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -120,17 +118,8 @@ func (db *DB) UpdateStorageID(ctx context.Context, prm UpdateStorageIDPrm) (res return res, ErrReadOnlyMode } - currEpoch := db.epochState.CurrentEpoch() - err = db.boltDB.Batch(func(tx *bbolt.Tx) error { - exists, err := db.exists(tx, prm.addr, currEpoch) - if err == nil && exists { - err = updateStorageID(tx, prm.addr, prm.id) - } else if errors.As(err, new(logicerr.Logical)) { - err = updateStorageID(tx, prm.addr, prm.id) - } - - return err + return setStorageID(tx, prm.addr, prm.id, true) }) success = err == nil return res, metaerr.Wrap(err) diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go index 063dfb270..aaf6480ab 100644 --- a/pkg/local_object_storage/metabase/storage_id_test.go +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -75,6 +75,32 @@ func TestDB_StorageID(t *testing.T) { }) } +func TestPutWritecacheDataRace(t *testing.T) { + t.Parallel() + + db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() + + putStorageID := []byte{1, 2, 3} + wcStorageID := []byte{1, 2, 3, 4, 5} + o := testutil.GenerateObject() + + fetchedStorageID, err := metaStorageID(db, object.AddressOf(o)) + require.NoError(t, err) + require.Nil(t, fetchedStorageID) + + // writecache flushes object and updates storageID before object actually saved to the metabase + metaUpdateStorageID(db, object.AddressOf(o), wcStorageID) + + // put object completes with writecache's storageID + err = metaPut(db, o, putStorageID) + require.NoError(t, err) + + fetchedStorageID, err = metaStorageID(db, object.AddressOf(o)) + require.NoError(t, err) + require.Equal(t, wcStorageID, fetchedStorageID) +} + func metaUpdateStorageID(db *meta.DB, addr oid.Address, id []byte) error { var sidPrm meta.UpdateStorageIDPrm sidPrm.SetAddress(addr) From 9801d084380f1a5be664ad97e527bce458c7291b Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 13 Feb 2024 14:56:25 +0300 Subject: [PATCH 0941/1943] [#932] adm: Move defaults for NNS to package `constants` Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/internal/modules/morph/constants/const.go | 5 ++++- cmd/frostfs-adm/internal/modules/morph/contract/deploy.go | 6 ++++-- .../internal/modules/morph/helper/initialize_ctx.go | 6 ++++-- .../internal/modules/morph/initialize/initialize_nns.go | 3 ++- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/constants/const.go b/cmd/frostfs-adm/internal/modules/morph/constants/const.go index 5a7f8f922..a3b4f129a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/constants/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/constants/const.go @@ -28,7 +28,10 @@ const ( ContractWalletFilename = "contract.json" ContractWalletPasswordKey = "contract" - FrostfsOpsEmail = "ops@frostfs.info" + FrostfsOpsEmail = "ops@frostfs.info" + NNSRefreshDefVal = int64(3600) + NNSRetryDefVal = int64(600) + NNSTtlDefVal = int64(3600) DefaultExpirationTime = 10 * 365 * 24 * time.Hour / time.Second diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go index 9a92e8f19..7a8f34a5d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go @@ -148,12 +148,14 @@ func registerNNS(nnsCs *state.Contract, c *helper.InitializeContext, zone string emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, zone, c.CommitteeAcc.Contract.ScriptHash(), - constants.FrostfsOpsEmail, int64(3600), int64(600), int64(constants.DefaultExpirationTime), int64(3600)) + constants.FrostfsOpsEmail, constants.NNSRefreshDefVal, constants.NNSRetryDefVal, + int64(constants.DefaultExpirationTime), constants.NNSTtlDefVal) emit.Opcodes(bw.BinWriter, opcode.ASSERT) emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, domain, c.CommitteeAcc.Contract.ScriptHash(), - constants.FrostfsOpsEmail, int64(3600), int64(600), int64(constants.DefaultExpirationTime), int64(3600)) + constants.FrostfsOpsEmail, constants.NNSRefreshDefVal, constants.NNSRetryDefVal, + int64(constants.DefaultExpirationTime), constants.NNSTtlDefVal) emit.Opcodes(bw.BinWriter, opcode.ASSERT) } else { s, ok, err := c.NNSRegisterDomainScript(nnsCs.Hash, cs.Hash, domain) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go index 8269f19dc..09d64b352 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go @@ -478,7 +478,8 @@ func (c *InitializeContext) EmitUpdateNNSGroupScript(bw *io.BufBinWriter, nnsHas if isAvail { emit.AppCall(bw.BinWriter, nnsHash, "register", callflag.All, client.NNSGroupKeyName, c.CommitteeAcc.Contract.ScriptHash(), - constants.FrostfsOpsEmail, int64(3600), int64(600), int64(constants.DefaultExpirationTime), int64(3600)) + constants.FrostfsOpsEmail, constants.NNSRefreshDefVal, constants.NNSRetryDefVal, + int64(constants.DefaultExpirationTime), constants.NNSTtlDefVal) emit.Opcodes(bw.BinWriter, opcode.ASSERT) } @@ -499,7 +500,8 @@ func (c *InitializeContext) NNSRegisterDomainScript(nnsHash, expectedHash util.U bw := io.NewBufBinWriter() emit.AppCall(bw.BinWriter, nnsHash, "register", callflag.All, domain, c.CommitteeAcc.Contract.ScriptHash(), - constants.FrostfsOpsEmail, int64(3600), int64(600), int64(constants.DefaultExpirationTime), int64(3600)) + constants.FrostfsOpsEmail, constants.NNSRefreshDefVal, constants.NNSRetryDefVal, + int64(constants.DefaultExpirationTime), constants.NNSTtlDefVal) emit.Opcodes(bw.BinWriter, opcode.ASSERT) if bw.Err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go index 7a628d5a3..14f12cf8e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go @@ -33,7 +33,8 @@ func setNNS(c *helper.InitializeContext) error { bw := io.NewBufBinWriter() emit.AppCall(bw.BinWriter, nnsCs.Hash, "register", callflag.All, "frostfs", c.CommitteeAcc.Contract.ScriptHash(), - constants.FrostfsOpsEmail, int64(3600), int64(600), int64(constants.DefaultExpirationTime), int64(3600)) + constants.FrostfsOpsEmail, constants.NNSRefreshDefVal, constants.NNSRetryDefVal, + int64(constants.DefaultExpirationTime), constants.NNSTtlDefVal) emit.Opcodes(bw.BinWriter, opcode.ASSERT) if err := c.SendCommitteeTx(bw.Bytes(), true); err != nil { return fmt.Errorf("can't add domain root to NNS: %w", err) From 61c58e2f927af812473a772bcf1589375784f0d9 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 14 Feb 2024 08:58:43 +0300 Subject: [PATCH 0942/1943] [#932] adm: Add commands to manipulate with `NNS` contract Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/ape/ape_util.go | 10 +- .../modules/morph/helper/initialize.go | 12 ++ .../internal/modules/morph/nns/helper.go | 28 ++++ .../internal/modules/morph/nns/record.go | 148 ++++++++++++++++++ .../internal/modules/morph/nns/register.go | 44 ++++++ .../internal/modules/morph/nns/renew.go | 26 +++ .../internal/modules/morph/nns/root.go | 101 ++++++++++++ .../internal/modules/morph/nns/tokens.go | 24 +++ .../internal/modules/morph/nns/update.go | 50 ++++++ .../internal/modules/morph/root.go | 2 + 10 files changed, 436 insertions(+), 9 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/nns/helper.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/nns/record.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/nns/register.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/nns/renew.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/nns/root.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/nns/tokens.go create mode 100644 cmd/frostfs-adm/internal/modules/morph/nns/update.go diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index fe09776ed..dccd8909f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -4,8 +4,6 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" @@ -91,13 +89,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *ac c, err := helper.GetN3Client(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) - walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) - wallets, err := helper.GetAlphabetWallets(v, walletDir) - commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) - - committeeAcc, err := helper.GetWalletAccount(wallets[0], constants.CommitteeAccountName) - commonCmd.ExitOnErr(cmd, "can't find committee account: %w", err) - + committeeAcc := helper.GetComitteAcc(cmd, v) ac, err := helper.NewActor(c, committeeAcc) commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go index 00c2fae6d..961ceba53 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go @@ -7,7 +7,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/core/native/nativenames" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -101,6 +103,16 @@ func GetWalletAccount(w *wallet.Wallet, typ string) (*wallet.Account, error) { return nil, fmt.Errorf("account for '%s' not found", typ) } +func GetComitteAcc(cmd *cobra.Command, v *viper.Viper) *wallet.Account { + walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) + wallets, err := GetAlphabetWallets(v, walletDir) + commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) + + committeeAcc, err := GetWalletAccount(wallets[0], constants.CommitteeAccountName) + commonCmd.ExitOnErr(cmd, "can't find committee account: %w", err) + return committeeAcc +} + func NNSResolve(inv *invoker.Invoker, nnsHash util.Uint160, domain string) (stackitem.Item, error) { return unwrap.Item(inv.Call(nnsHash, "resolve", domain, int64(nns.TXT))) } diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go new file mode 100644 index 000000000..4c9c9e576 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go @@ -0,0 +1,28 @@ +package nns + +import ( + client "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func getRPCClient(cmd *cobra.Command) (*client.Contract, *actor.Actor, util.Uint160) { + v := viper.GetViper() + c, err := helper.GetN3Client(v) + commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) + + committeeAcc := helper.GetComitteAcc(cmd, v) + ac, err := helper.NewActor(c, committeeAcc) + commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) + + inv := &ac.Invoker + r := management.NewReader(inv) + nnsCs, err := r.GetContractByID(1) + commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) + return client.New(ac, nnsCs.Hash), ac, nnsCs.Hash +} diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/record.go b/cmd/frostfs-adm/internal/modules/morph/nns/record.go new file mode 100644 index 000000000..0e217eb61 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/nns/record.go @@ -0,0 +1,148 @@ +package nns + +import ( + "errors" + "math/big" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "github.com/spf13/cobra" +) + +func initAddRecordCmd() { + Cmd.AddCommand(addRecordCmd) + addRecordCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + addRecordCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + addRecordCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) + addRecordCmd.Flags().String(nnsRecordTypeFlag, "", nnsRecordTypeFlagDesc) + addRecordCmd.Flags().String(nnsRecordDataFlag, "", nnsRecordDataFlagDesc) + + _ = cobra.MarkFlagRequired(addRecordCmd.Flags(), nnsNameFlag) + _ = cobra.MarkFlagRequired(addRecordCmd.Flags(), nnsRecordTypeFlag) + _ = cobra.MarkFlagRequired(addRecordCmd.Flags(), nnsRecordDataFlag) +} + +func initGetRecordsCmd() { + Cmd.AddCommand(getRecordsCmd) + getRecordsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + getRecordsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + getRecordsCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) + getRecordsCmd.Flags().String(nnsRecordTypeFlag, "", nnsRecordTypeFlagDesc) + + _ = cobra.MarkFlagRequired(getRecordsCmd.Flags(), nnsNameFlag) +} + +func initDelRecordsCmd() { + Cmd.AddCommand(delRecordsCmd) + delRecordsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + delRecordsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + delRecordsCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) + delRecordsCmd.Flags().String(nnsRecordTypeFlag, "", nnsRecordTypeFlagDesc) + + _ = cobra.MarkFlagRequired(delRecordsCmd.Flags(), nnsNameFlag) + _ = cobra.MarkFlagRequired(delRecordsCmd.Flags(), nnsRecordTypeFlag) +} + +func addRecord(cmd *cobra.Command, _ []string) { + c, actor, _ := getRPCClient(cmd) + name, _ := cmd.Flags().GetString(nnsNameFlag) + data, _ := cmd.Flags().GetString(nnsRecordDataFlag) + recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) + typ, err := getRecordType(recordType) + commonCmd.ExitOnErr(cmd, "unable to parse record type: %w", err) + h, vub, err := c.AddRecord(name, typ, data) + commonCmd.ExitOnErr(cmd, "unable to add record: %w", err) + + cmd.Println("Waiting for transaction to persist...") + _, err = actor.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "renew domain error: %w", err) + cmd.Println("Record added successfully") +} + +func getRecords(cmd *cobra.Command, _ []string) { + c, act, hash := getRPCClient(cmd) + name, _ := cmd.Flags().GetString(nnsNameFlag) + recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) + if recordType == "" { + sid, r, err := unwrap.SessionIterator(act.Invoker.Call(hash, "getAllRecords", name)) + commonCmd.ExitOnErr(cmd, "unable to get records: %w", err) + defer func() { + _ = act.Invoker.TerminateSession(sid) + }() + items, err := act.Invoker.TraverseIterator(sid, &r, 0) + commonCmd.ExitOnErr(cmd, "unable to get records: %w", err) + for len(items) != 0 { + for j := range items { + rs := items[j].Value().([]stackitem.Item) + bs, err := rs[2].TryBytes() + commonCmd.ExitOnErr(cmd, "unable to parse record state: %w", err) + cmd.Printf("%s %s\n", + recordTypeToString(nns.RecordType(rs[1].Value().(*big.Int).Int64())), + string(bs)) + } + items, err = act.Invoker.TraverseIterator(sid, &r, 0) + commonCmd.ExitOnErr(cmd, "unable to get records: %w", err) + } + } else { + typ, err := getRecordType(recordType) + commonCmd.ExitOnErr(cmd, "unable to parse record type: %w", err) + items, err := c.GetRecords(name, typ) + commonCmd.ExitOnErr(cmd, "unable to get records: %w", err) + for _, item := range items { + record, err := item.TryBytes() + commonCmd.ExitOnErr(cmd, "unable to parse response: %w", err) + cmd.Println(string(record)) + } + } +} + +func delRecords(cmd *cobra.Command, _ []string) { + c, actor, _ := getRPCClient(cmd) + name, _ := cmd.Flags().GetString(nnsNameFlag) + recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) + typ, err := getRecordType(recordType) + commonCmd.ExitOnErr(cmd, "unable to parse record type: %w", err) + h, vub, err := c.DeleteRecords(name, typ) + commonCmd.ExitOnErr(cmd, "unable to delete records: %w", err) + + cmd.Println("Waiting for transaction to persist...") + _, err = actor.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "delete records error: %w", err) + cmd.Println("Records removed successfully") +} + +func getRecordType(recordType string) (*big.Int, error) { + switch strings.ToUpper(recordType) { + case "A": + return big.NewInt(int64(nns.A)), nil + case "CNAME": + return big.NewInt(int64(nns.CNAME)), nil + case "SOA": + return big.NewInt(int64(nns.SOA)), nil + case "TXT": + return big.NewInt(int64(nns.TXT)), nil + case "AAAA": + return big.NewInt(int64(nns.AAAA)), nil + } + return nil, errors.New("unsupported record type") +} + +func recordTypeToString(rt nns.RecordType) string { + switch rt { + case nns.A: + return "A" + case nns.CNAME: + return "CNAME" + case nns.SOA: + return "SOA" + case nns.TXT: + return "TXT" + case nns.AAAA: + return "AAAA" + } + return "" +} diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/register.go b/cmd/frostfs-adm/internal/modules/morph/nns/register.go new file mode 100644 index 000000000..d05d9f171 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/nns/register.go @@ -0,0 +1,44 @@ +package nns + +import ( + "math/big" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/spf13/cobra" +) + +func initRegisterCmd() { + Cmd.AddCommand(registerCmd) + registerCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + registerCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + registerCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) + registerCmd.Flags().String(nnsEmailFlag, constants.FrostfsOpsEmail, "Domain owner email") + registerCmd.Flags().Int64(nnsRefreshFlag, constants.NNSRefreshDefVal, "SOA record REFRESH parameter") + registerCmd.Flags().Int64(nnsRetryFlag, constants.NNSRetryDefVal, "SOA record RETRY parameter") + registerCmd.Flags().Int64(nnsExpireFlag, int64(constants.DefaultExpirationTime), "SOA record EXPIRE parameter") + registerCmd.Flags().Int64(nnsTTLFlag, constants.NNSTtlDefVal, "SOA record TTL parameter") + + _ = cobra.MarkFlagRequired(registerCmd.Flags(), nnsNameFlag) +} + +func registerDomain(cmd *cobra.Command, _ []string) { + c, actor, _ := getRPCClient(cmd) + + name, _ := cmd.Flags().GetString(nnsNameFlag) + email, _ := cmd.Flags().GetString(nnsEmailFlag) + refresh, _ := cmd.Flags().GetInt64(nnsRefreshFlag) + retry, _ := cmd.Flags().GetInt64(nnsRetryFlag) + expire, _ := cmd.Flags().GetInt64(nnsExpireFlag) + ttl, _ := cmd.Flags().GetInt64(nnsTTLFlag) + + h, vub, err := c.Register(name, actor.Sender(), email, big.NewInt(refresh), + big.NewInt(retry), big.NewInt(expire), big.NewInt(ttl)) + commonCmd.ExitOnErr(cmd, "unable to register domain: %w", err) + + cmd.Println("Waiting for transaction to persist...") + _, err = actor.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "register domain error: %w", err) + cmd.Println("Domain registered successfully") +} diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/renew.go b/cmd/frostfs-adm/internal/modules/morph/nns/renew.go new file mode 100644 index 000000000..b13092240 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/nns/renew.go @@ -0,0 +1,26 @@ +package nns + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/spf13/cobra" +) + +func initRenewCmd() { + Cmd.AddCommand(renewCmd) + renewCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + renewCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + renewCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) +} + +func renewDomain(cmd *cobra.Command, _ []string) { + c, actor, _ := getRPCClient(cmd) + name, _ := cmd.Flags().GetString(nnsNameFlag) + h, vub, err := c.Renew(name) + commonCmd.ExitOnErr(cmd, "unable to renew domain: %w", err) + + cmd.Println("Waiting for transaction to persist...") + _, err = actor.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "renew domain error: %w", err) + cmd.Println("Domain renewed successfully") +} diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/root.go b/cmd/frostfs-adm/internal/modules/morph/nns/root.go new file mode 100644 index 000000000..09133f932 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/nns/root.go @@ -0,0 +1,101 @@ +package nns + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + nnsNameFlag = "name" + nnsNameFlagDesc = "Domain name" + nnsEmailFlag = "email" + nnsRefreshFlag = "refresh" + nnsRetryFlag = "retry" + nnsExpireFlag = "expire" + nnsTTLFlag = "ttl" + nnsRecordTypeFlag = "type" + nnsRecordTypeFlagDesc = "Domain name service record type(A|CNAME|SOA|TXT)" + nnsRecordDataFlag = "data" + nnsRecordDataFlagDesc = "Domain name service record data" +) + +var ( + Cmd = &cobra.Command{ + Use: "nns", + Short: "Section for Neo Name Service (NNS)", + } + tokensCmd = &cobra.Command{ + Use: "tokens", + Short: "List all registered domain names", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + }, + Run: listTokens, + } + registerCmd = &cobra.Command{ + Use: "register", + Short: "Registers a new domain", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + }, + Run: registerDomain, + } + renewCmd = &cobra.Command{ + Use: "renew", + Short: "Increases domain expiration date", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + }, + Run: renewDomain, + } + updateCmd = &cobra.Command{ + Use: "update", + Short: "Updates soa record", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + }, + Run: updateSOA, + } + addRecordCmd = &cobra.Command{ + Use: "add-record", + Short: "Adds a new record of the specified type to the provided domain", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + }, + Run: addRecord, + } + getRecordsCmd = &cobra.Command{ + Use: "get-records", + Short: "Returns domain record of the specified type", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + }, + Run: getRecords, + } + delRecordsCmd = &cobra.Command{ + Use: "delete-records", + Short: "Removes domain records with the specified type", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + }, + Run: delRecords, + } +) + +func init() { + initTokensCmd() + initRegisterCmd() + initRenewCmd() + initUpdateCmd() + initAddRecordCmd() + initGetRecordsCmd() + initDelRecordsCmd() +} diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/tokens.go b/cmd/frostfs-adm/internal/modules/morph/nns/tokens.go new file mode 100644 index 000000000..6e8ffb40a --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/nns/tokens.go @@ -0,0 +1,24 @@ +package nns + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/spf13/cobra" +) + +func initTokensCmd() { + Cmd.AddCommand(tokensCmd) + tokensCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + tokensCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) +} + +func listTokens(cmd *cobra.Command, _ []string) { + c, _, _ := getRPCClient(cmd) + it, err := c.Tokens() + commonCmd.ExitOnErr(cmd, "unable to get tokens: %w", err) + for toks, err := it.Next(10); err == nil && len(toks) > 0; toks, err = it.Next(10) { + for _, token := range toks { + cmd.Println(string(token)) + } + } +} diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/update.go b/cmd/frostfs-adm/internal/modules/morph/nns/update.go new file mode 100644 index 000000000..3437316e3 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/nns/update.go @@ -0,0 +1,50 @@ +package nns + +import ( + "math/big" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/spf13/cobra" +) + +func initUpdateCmd() { + Cmd.AddCommand(updateCmd) + updateCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + updateCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + updateCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) + updateCmd.Flags().String(nnsEmailFlag, constants.FrostfsOpsEmail, "Domain owner email") + updateCmd.Flags().Int64(nnsRefreshFlag, constants.NNSRefreshDefVal, + "The number of seconds between update requests from secondary and slave name servers") + updateCmd.Flags().Int64(nnsRetryFlag, constants.NNSRetryDefVal, + "The number of seconds the secondary or slave will wait before retrying when the last attempt has failed") + updateCmd.Flags().Int64(nnsExpireFlag, int64(constants.DefaultExpirationTime), + "The number of seconds a master or slave will wait before considering the data stale "+ + "if it cannot reach the primary name server") + updateCmd.Flags().Int64(nnsTTLFlag, constants.NNSTtlDefVal, + "The number of seconds a domain name is cached locally before expiration and return to authoritative "+ + "nameservers for updated information") + + _ = cobra.MarkFlagRequired(updateCmd.Flags(), nnsNameFlag) +} + +func updateSOA(cmd *cobra.Command, _ []string) { + c, actor, _ := getRPCClient(cmd) + + name, _ := cmd.Flags().GetString(nnsNameFlag) + email, _ := cmd.Flags().GetString(nnsEmailFlag) + refresh, _ := cmd.Flags().GetInt64(nnsRefreshFlag) + retry, _ := cmd.Flags().GetInt64(nnsRetryFlag) + expire, _ := cmd.Flags().GetInt64(nnsExpireFlag) + ttl, _ := cmd.Flags().GetInt64(nnsTTLFlag) + + h, vub, err := c.UpdateSOA(name, email, big.NewInt(refresh), + big.NewInt(retry), big.NewInt(expire), big.NewInt(ttl)) + commonCmd.ExitOnErr(cmd, "unable to send transaction: %w", err) + + cmd.Println("Waiting for transaction to persist...") + _, err = actor.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "register domain error: %w", err) + cmd.Println("SOA records updated successfully") +} diff --git a/cmd/frostfs-adm/internal/modules/morph/root.go b/cmd/frostfs-adm/internal/modules/morph/root.go index 9d2b25ceb..e8426d56e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/root.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/generate" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/initialize" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/node" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/notary" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/policy" @@ -49,4 +50,5 @@ func init() { RootCmd.AddCommand(proxy.RemoveAccountCmd) RootCmd.AddCommand(frostfsid.Cmd) + RootCmd.AddCommand(nns.Cmd) } From bc9dbb26ec0ca191b2b41e3f71caf6dbd7563346 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 20 Feb 2024 11:36:36 +0300 Subject: [PATCH 0943/1943] [#932] adm: Add custom `Actor` to sign tx by all committee accounts Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/ape/ape_util.go | 14 +- .../internal/modules/morph/helper/actor.go | 169 ++++++++++++++++++ .../internal/modules/morph/nns/helper.go | 9 +- .../internal/modules/morph/nns/root.go | 2 - 4 files changed, 177 insertions(+), 17 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/helper/actor.go diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index dccd8909f..3a990fe17 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -11,7 +11,6 @@ import ( apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" morph "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/cobra" @@ -84,22 +83,19 @@ func parseChainName(cmd *cobra.Command) apechain.Name { return apeChainName } -func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *actor.Actor) { - v := viper.GetViper() - c, err := helper.GetN3Client(v) +func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *helper.LocalActor) { + c, err := helper.GetN3Client(viper.GetViper()) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) - committeeAcc := helper.GetComitteAcc(cmd, v) - ac, err := helper.NewActor(c, committeeAcc) + ac, err := helper.NewLocalActor(cmd, c) commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) - inv := &ac.Invoker var ch util.Uint160 - r := management.NewReader(inv) + r := management.NewReader(ac.Invoker) nnsCs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) - ch, err = helper.NNSResolveHash(inv, nnsCs.Hash, helper.DomainOf(constants.PolicyContract)) + ch, err = helper.NNSResolveHash(ac.Invoker, nnsCs.Hash, helper.DomainOf(constants.PolicyContract)) commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err) return morph.NewContractStorage(ac, ch), ac diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/actor.go b/cmd/frostfs-adm/internal/modules/morph/helper/actor.go new file mode 100644 index 000000000..f920aa5ba --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/helper/actor.go @@ -0,0 +1,169 @@ +package helper + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/google/uuid" + "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/context" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "github.com/nspcc-dev/neo-go/pkg/wallet" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// LocalActor is a kludge, do not use it outside of the morph commands. +type LocalActor struct { + neoActor *actor.Actor + accounts []*wallet.Account + Invoker *invoker.Invoker +} + +// NewLocalActor create LocalActor with accounts form provided wallets. +// In case of empty wallets provided created actor with dummy account only for read operation. +func NewLocalActor(cmd *cobra.Command, c actor.RPCActor) (*LocalActor, error) { + walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) + var act *actor.Actor + var accounts []*wallet.Account + if walletDir == "" { + account, err := wallet.NewAccount() + commonCmd.ExitOnErr(cmd, "unable to create dummy account: %w", err) + act, err = actor.New(c, []actor.SignerAccount{{ + Signer: transaction.Signer{ + Account: account.Contract.ScriptHash(), + Scopes: transaction.Global, + }, + Account: account, + }}) + if err != nil { + return nil, err + } + } else { + wallets, err := GetAlphabetWallets(viper.GetViper(), walletDir) + commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) + + for _, w := range wallets { + acc, err := GetWalletAccount(w, constants.CommitteeAccountName) + commonCmd.ExitOnErr(cmd, "can't find committee account: %w", err) + accounts = append(accounts, acc) + } + act, err = actor.New(c, []actor.SignerAccount{{ + Signer: transaction.Signer{ + Account: accounts[0].Contract.ScriptHash(), + Scopes: transaction.Global, + }, + Account: accounts[0], + }}) + if err != nil { + return nil, err + } + } + return &LocalActor{ + neoActor: act, + accounts: accounts, + Invoker: &act.Invoker, + }, nil +} + +func (a *LocalActor) SendCall(contract util.Uint160, method string, params ...any) (util.Uint256, uint32, error) { + tx, err := a.neoActor.MakeCall(contract, method, params...) + if err != nil { + return util.Uint256{}, 0, err + } + err = a.resign(tx) + if err != nil { + return util.Uint256{}, 0, err + } + return a.neoActor.Send(tx) +} + +func (a *LocalActor) SendRun(script []byte) (util.Uint256, uint32, error) { + tx, err := a.neoActor.MakeRun(script) + if err != nil { + return util.Uint256{}, 0, err + } + err = a.resign(tx) + if err != nil { + return util.Uint256{}, 0, err + } + return a.neoActor.Send(tx) +} + +// resign is used to sign tx with committee accounts. +// Inside the methods `MakeCall` and `SendRun` of the NeoGO's actor transaction is signing by committee account, +// because actor uses committee wallet. +// But it is not enough, need to sign with another committee accounts. +func (a *LocalActor) resign(tx *transaction.Transaction) error { + if len(a.accounts[0].Contract.Parameters) > 1 { + // Use parameter context to avoid dealing with signature order. + network := a.neoActor.GetNetwork() + pc := context.NewParameterContext("", network, tx) + h := a.accounts[0].Contract.ScriptHash() + for _, acc := range a.accounts { + priv := acc.PrivateKey() + sign := priv.SignHashable(uint32(network), tx) + if err := pc.AddSignature(h, acc.Contract, priv.PublicKey(), sign); err != nil { + return fmt.Errorf("can't add signature: %w", err) + } + if len(pc.Items[h].Signatures) == len(acc.Contract.Parameters) { + break + } + } + + w, err := pc.GetWitness(h) + if err != nil { + return fmt.Errorf("incomplete signature: %w", err) + } + tx.Scripts[0] = *w + } + return nil +} + +func (a *LocalActor) Wait(h util.Uint256, vub uint32, err error) (*state.AppExecResult, error) { + return a.neoActor.Wait(h, vub, err) +} + +func (a *LocalActor) Sender() util.Uint160 { + return a.neoActor.Sender() +} + +func (a *LocalActor) Call(contract util.Uint160, operation string, params ...any) (*result.Invoke, error) { + return a.neoActor.Call(contract, operation, params...) +} + +func (a *LocalActor) CallAndExpandIterator(_ util.Uint160, _ string, _ int, _ ...any) (*result.Invoke, error) { + panic("unimplemented") +} + +func (a *LocalActor) TerminateSession(_ uuid.UUID) error { + panic("unimplemented") +} + +func (a *LocalActor) TraverseIterator(sessionID uuid.UUID, iterator *result.Iterator, num int) ([]stackitem.Item, error) { + return a.neoActor.TraverseIterator(sessionID, iterator, num) +} + +func (a *LocalActor) MakeRun(_ []byte) (*transaction.Transaction, error) { + panic("unimplemented") +} + +func (a *LocalActor) MakeUnsignedCall(_ util.Uint160, _ string, _ []transaction.Attribute, _ ...any) (*transaction.Transaction, error) { + panic("unimplemented") +} + +func (a *LocalActor) MakeUnsignedRun(_ []byte, _ []transaction.Attribute) (*transaction.Transaction, error) { + panic("unimplemented") +} + +func (a *LocalActor) MakeCall(_ util.Uint160, _ string, _ ...any) (*transaction.Transaction, error) { + panic("unimplemented") +} diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go index 4c9c9e576..36c3dd2fd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go @@ -4,24 +4,21 @@ import ( client "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/cobra" "github.com/spf13/viper" ) -func getRPCClient(cmd *cobra.Command) (*client.Contract, *actor.Actor, util.Uint160) { +func getRPCClient(cmd *cobra.Command) (*client.Contract, *helper.LocalActor, util.Uint160) { v := viper.GetViper() c, err := helper.GetN3Client(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) - committeeAcc := helper.GetComitteAcc(cmd, v) - ac, err := helper.NewActor(c, committeeAcc) + ac, err := helper.NewLocalActor(cmd, c) commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) - inv := &ac.Invoker - r := management.NewReader(inv) + r := management.NewReader(ac.Invoker) nnsCs, err := r.GetContractByID(1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) return client.New(ac, nnsCs.Hash), ac, nnsCs.Hash diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/root.go b/cmd/frostfs-adm/internal/modules/morph/nns/root.go index 09133f932..e528e4b7b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/root.go @@ -30,7 +30,6 @@ var ( Short: "List all registered domain names", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) - _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, Run: listTokens, } @@ -75,7 +74,6 @@ var ( Short: "Returns domain record of the specified type", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) - _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, Run: getRecords, } From 7cc368e188dc03e761cad0801e7f269a81b1ed83 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 14 Feb 2024 14:14:07 +0300 Subject: [PATCH 0944/1943] [#986] object: Introduce soft ape checks * Soft APE check means that APE should allow request even it gets status NoRuleFound for a request. Otherwise, it is interpreted as Deny. * Soft APE check is performed if basic ACL mask is not set. Signed-off-by: Airat Arifullin --- pkg/services/object/acl/v2/request.go | 7 ++ pkg/services/object/acl/v2/service.go | 107 ++++++++++-------- pkg/services/object/ape/checker.go | 7 +- pkg/services/object/ape/checker_test.go | 18 +++ pkg/services/object/ape/service.go | 143 ++++++++++++++---------- pkg/services/object/request_context.go | 2 + 6 files changed, 176 insertions(+), 108 deletions(-) diff --git a/pkg/services/object/acl/v2/request.go b/pkg/services/object/acl/v2/request.go index c58c00e0e..74279e453 100644 --- a/pkg/services/object/acl/v2/request.go +++ b/pkg/services/object/acl/v2/request.go @@ -104,6 +104,13 @@ func (r RequestInfo) RequestRole() acl.Role { return r.requestRole } +// IsSoftAPECheck states if APE should perform soft checks. +// Soft APE check allows a request if CheckAPE returns NoRuleFound for it, +// otherwise it denies the request. +func (r RequestInfo) IsSoftAPECheck() bool { + return r.BasicACL().Bits() != 0 +} + // MetaWithToken groups session and bearer tokens, // verification header and raw API request. type MetaWithToken struct { diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index af26a6fd1..b51386e0c 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -113,9 +113,10 @@ type wrappedGetObjectStream struct { func (w *wrappedGetObjectStream) Context() context.Context { return context.WithValue(w.GetObjectStream.Context(), object.RequestContextKey, &object.RequestContext{ - Namespace: w.requestInfo.ContainerNamespace(), - SenderKey: w.requestInfo.SenderKey(), - Role: w.requestInfo.RequestRole(), + Namespace: w.requestInfo.ContainerNamespace(), + SenderKey: w.requestInfo.SenderKey(), + Role: w.requestInfo.RequestRole(), + SoftAPECheck: w.requestInfo.IsSoftAPECheck(), }) } @@ -136,9 +137,10 @@ type wrappedRangeStream struct { func (w *wrappedRangeStream) Context() context.Context { return context.WithValue(w.GetObjectRangeStream.Context(), object.RequestContextKey, &object.RequestContext{ - Namespace: w.requestInfo.ContainerNamespace(), - SenderKey: w.requestInfo.SenderKey(), - Role: w.requestInfo.RequestRole(), + Namespace: w.requestInfo.ContainerNamespace(), + SenderKey: w.requestInfo.SenderKey(), + Role: w.requestInfo.RequestRole(), + SoftAPECheck: w.requestInfo.IsSoftAPECheck(), }) } @@ -159,9 +161,10 @@ type wrappedSearchStream struct { func (w *wrappedSearchStream) Context() context.Context { return context.WithValue(w.SearchStream.Context(), object.RequestContextKey, &object.RequestContext{ - Namespace: w.requestInfo.ContainerNamespace(), - SenderKey: w.requestInfo.SenderKey(), - Role: w.requestInfo.RequestRole(), + Namespace: w.requestInfo.ContainerNamespace(), + SenderKey: w.requestInfo.SenderKey(), + Role: w.requestInfo.RequestRole(), + SoftAPECheck: w.requestInfo.IsSoftAPECheck(), }) } @@ -216,10 +219,12 @@ func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream reqInfo.obj = obj - if !b.checker.CheckBasicACL(reqInfo) { - return basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return eACLErr(reqInfo, err) + if reqInfo.IsSoftAPECheck() { + if !b.checker.CheckBasicACL(reqInfo) { + return basicACLErr(reqInfo) + } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { + return eACLErr(reqInfo, err) + } } return b.next.Get(request, &getStreamBasicChecker{ @@ -283,10 +288,12 @@ func (b Service) Head( reqInfo.obj = obj - if !b.checker.CheckBasicACL(reqInfo) { - return nil, basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return nil, eACLErr(reqInfo, err) + if reqInfo.IsSoftAPECheck() { + if !b.checker.CheckBasicACL(reqInfo) { + return nil, basicACLErr(reqInfo) + } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { + return nil, eACLErr(reqInfo, err) + } } resp, err := b.next.Head(requestContext(ctx, reqInfo), request) @@ -334,10 +341,12 @@ func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStr return err } - if !b.checker.CheckBasicACL(reqInfo) { - return basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return eACLErr(reqInfo, err) + if reqInfo.IsSoftAPECheck() { + if !b.checker.CheckBasicACL(reqInfo) { + return basicACLErr(reqInfo) + } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { + return eACLErr(reqInfo, err) + } } return b.next.Search(request, &searchStreamBasicChecker{ @@ -392,10 +401,12 @@ func (b Service) Delete( reqInfo.obj = obj - if !b.checker.CheckBasicACL(reqInfo) { - return nil, basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return nil, eACLErr(reqInfo, err) + if reqInfo.IsSoftAPECheck() { + if !b.checker.CheckBasicACL(reqInfo) { + return nil, basicACLErr(reqInfo) + } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { + return nil, eACLErr(reqInfo, err) + } } return b.next.Delete(requestContext(ctx, reqInfo), request) @@ -443,10 +454,12 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb reqInfo.obj = obj - if !b.checker.CheckBasicACL(reqInfo) { - return basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return eACLErr(reqInfo, err) + if reqInfo.IsSoftAPECheck() { + if !b.checker.CheckBasicACL(reqInfo) { + return basicACLErr(reqInfo) + } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { + return eACLErr(reqInfo, err) + } } return b.next.GetRange(request, &rangeStreamBasicChecker{ @@ -458,9 +471,10 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb func requestContext(ctx context.Context, reqInfo RequestInfo) context.Context { return context.WithValue(ctx, object.RequestContextKey, &object.RequestContext{ - Namespace: reqInfo.ContainerNamespace(), - SenderKey: reqInfo.SenderKey(), - Role: reqInfo.RequestRole(), + Namespace: reqInfo.ContainerNamespace(), + SenderKey: reqInfo.SenderKey(), + Role: reqInfo.RequestRole(), + SoftAPECheck: reqInfo.IsSoftAPECheck(), }) } @@ -509,10 +523,12 @@ func (b Service) GetRangeHash( reqInfo.obj = obj - if !b.checker.CheckBasicACL(reqInfo) { - return nil, basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return nil, eACLErr(reqInfo, err) + if reqInfo.IsSoftAPECheck() { + if !b.checker.CheckBasicACL(reqInfo) { + return nil, basicACLErr(reqInfo) + } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { + return nil, eACLErr(reqInfo, err) + } } return b.next.GetRangeHash(requestContext(ctx, reqInfo), request) @@ -566,12 +582,13 @@ func (b Service) PutSingle(ctx context.Context, request *objectV2.PutSingleReque reqInfo.obj = obj - if !b.checker.CheckBasicACL(reqInfo) || !b.checker.StickyBitCheck(reqInfo, idOwner) { - return nil, basicACLErr(reqInfo) - } - - if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return nil, eACLErr(reqInfo, err) + if reqInfo.IsSoftAPECheck() { + if !b.checker.CheckBasicACL(reqInfo) || !b.checker.StickyBitCheck(reqInfo, idOwner) { + return nil, basicACLErr(reqInfo) + } + if err := b.checker.CheckEACL(request, reqInfo); err != nil { + return nil, eACLErr(reqInfo, err) + } } return b.next.PutSingle(requestContext(ctx, reqInfo), request) @@ -639,8 +656,10 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe reqInfo.obj = obj - if !p.source.checker.CheckBasicACL(reqInfo) || !p.source.checker.StickyBitCheck(reqInfo, idOwner) { - return basicACLErr(reqInfo) + if reqInfo.IsSoftAPECheck() { + if !p.source.checker.CheckBasicACL(reqInfo) || !p.source.checker.StickyBitCheck(reqInfo, idOwner) { + return basicACLErr(reqInfo) + } } ctx = requestContext(ctx, reqInfo) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 76ada3d2b..13b2729e9 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -45,6 +45,9 @@ type Prm struct { // An encoded sender's public key string. SenderKey string + + // If SoftAPECheck is set to true, then NoRuleFound is interpreted as allow. + SoftAPECheck bool } var errMissingOID = fmt.Errorf("object ID is not set") @@ -63,9 +66,9 @@ func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { return err } - if !ruleFound || status == apechain.Allow { + if !ruleFound && prm.SoftAPECheck || status == apechain.Allow { return nil } - return fmt.Errorf("found denying rule for %s: %s", prm.Method, status) + return fmt.Errorf("method %s: %s", prm.Method, status) } diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index 08d097907..d7e97064b 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -165,11 +165,29 @@ func TestAPECheck(t *testing.T) { container: containerID, object: stringPtr(objectID), methods: methodsRequiredOID, + containerRules: []chain.Rule{ + { + Status: chain.Allow, + Actions: chain.Actions{Names: methodsRequiredOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, + }, + }, + }, }, { name: "oid optional requests are allowed", container: containerID, methods: methodsOptionalOID, + containerRules: []chain.Rule{ + { + Status: chain.Allow, + Actions: chain.Actions{Names: methodsOptionalOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + }, + }, }, { name: "oid required requests are denied", diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 781f9df4b..f05bdaa83 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -60,9 +60,13 @@ type getStreamBasicChecker struct { apeChecker Checker + namespace string + senderKey []byte role string + + softAPECheck bool } func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { @@ -73,12 +77,14 @@ func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { } prm := Prm{ - Container: cnrID, - Object: objID, - Header: partInit.GetHeader(), - Method: nativeschema.MethodGetObject, - SenderKey: hex.EncodeToString(g.senderKey), - Role: g.role, + Namespace: g.namespace, + Container: cnrID, + Object: objID, + Header: partInit.GetHeader(), + Method: nativeschema.MethodGetObject, + SenderKey: hex.EncodeToString(g.senderKey), + Role: g.role, + SoftAPECheck: g.softAPECheck, } if err := g.apeChecker.CheckAPE(g.Context(), prm); err != nil { @@ -112,12 +118,13 @@ func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectSt } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodGetObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodGetObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return toStatusErr(err) @@ -126,6 +133,10 @@ func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectSt return c.next.Get(request, &getStreamBasicChecker{ GetObjectStream: stream, apeChecker: c.apeChecker, + namespace: reqCtx.Namespace, + senderKey: reqCtx.SenderKey, + role: nativeSchemaRole(reqCtx.Role), + softAPECheck: reqCtx.SoftAPECheck, }) } @@ -148,13 +159,14 @@ func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutR } prm := Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Header: partInit.GetHeader(), - Method: nativeschema.MethodPutObject, - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - Role: nativeSchemaRole(reqCtx.Role), + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Header: partInit.GetHeader(), + Method: nativeschema.MethodPutObject, + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + Role: nativeSchemaRole(reqCtx.Role), + SoftAPECheck: reqCtx.SoftAPECheck, } if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -190,12 +202,13 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj } err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodHeadObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodHeadObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return nil, err @@ -226,13 +239,14 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj } err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Header: header, - Method: nativeschema.MethodHeadObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Header: header, + Method: nativeschema.MethodHeadObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return nil, err @@ -254,11 +268,12 @@ func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.Searc } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Method: nativeschema.MethodSearchObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), + Namespace: reqCtx.Namespace, + Container: cnrID, + Method: nativeschema.MethodSearchObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return toStatusErr(err) @@ -279,12 +294,13 @@ func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) ( } err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodDeleteObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodDeleteObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return nil, err @@ -310,12 +326,13 @@ func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.G } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodRangeObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodRangeObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return toStatusErr(err) @@ -336,12 +353,13 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa } prm := Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodHashObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodHashObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + SoftAPECheck: reqCtx.SoftAPECheck, } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -371,13 +389,14 @@ func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequ } prm := Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Header: request.GetBody().GetObject().GetHeader(), - Method: nativeschema.MethodPutObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Header: request.GetBody().GetObject().GetHeader(), + Method: nativeschema.MethodPutObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + SoftAPECheck: reqCtx.SoftAPECheck, } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { diff --git a/pkg/services/object/request_context.go b/pkg/services/object/request_context.go index 4b9aa04d1..e0e6c9ed9 100644 --- a/pkg/services/object/request_context.go +++ b/pkg/services/object/request_context.go @@ -13,4 +13,6 @@ type RequestContext struct { SenderKey []byte Role acl.Role + + SoftAPECheck bool } From b1d171c261a644f538299cbd7a544683e6a4dfe7 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 14 Feb 2024 18:14:06 +0300 Subject: [PATCH 0945/1943] [#986] container: Interpret APE NoRuleFound as request deny * If APE check returns NoRuleFound, then it is taken for request deny. * Add more unit-test for ape container middleware. Signed-off-by: Airat Arifullin --- pkg/services/container/ape.go | 8 +- pkg/services/container/ape_test.go | 155 +++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 4 deletions(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index d1b490bdf..c01ad8a1d 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -162,7 +162,7 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co return nil, err } - if !found || s == apechain.Allow { + if found && s == apechain.Allow { return ac.next.List(ctx, req) } @@ -207,7 +207,7 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont return nil, err } - if !found || s == apechain.Allow { + if found && s == apechain.Allow { return ac.next.Put(ctx, req) } @@ -296,13 +296,13 @@ func (ac *apeChecker) validateContainerBoundedOperation(containerID *refs.Contai } s, found, err := ac.router.IsAllowed(apechain.Ingress, - policyengine.NewRequestTarget(cntNamespace, id.EncodeToString()), + policyengine.NewRequestTarget(namespace, id.EncodeToString()), request) if err != nil { return err } - if !found || s == apechain.Allow { + if found && s == apechain.Allow { return nil } diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index daaacb031..1af59d1c6 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -39,6 +39,8 @@ const ( func TestAPE(t *testing.T) { t.Parallel() + t.Run("allow then deny get container", testAllowThenDenyGetContainerRuleDefined) + t.Run("deny get container no rule found", testDenyGetContainerNoRuleFound) t.Run("deny get container for others", testDenyGetContainerForOthers) t.Run("deny set container eACL for IR", testDenySetContainerEACLForIR) t.Run("deny get container eACL for IR with session token", testDenyGetContainerEACLForIRSessionToken) @@ -49,6 +51,130 @@ func TestAPE(t *testing.T) { t.Run("deny list containers by namespace invalidation", testDenyListContainersValidationNamespaceError) } +func testAllowThenDenyGetContainerRuleDefined(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{}, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) + + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + contRdr.c[contID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(nm.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + nm.netmaps[nm.currentEpoch] = &testNetmap + nm.netmaps[nm.currentEpoch-1] = &testNetmap + + addDefaultAllowGetPolicy(t, router, contID) + + req := &container.GetRequest{} + req.SetBody(&container.GetRequestBody{}) + var refContID refs.ContainerID + contID.WriteToV2(&refContID) + req.GetBody().SetContainerID(&refContID) + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + _, err = apeSrv.Get(context.Background(), req) + require.NoError(t, err) + + _, _, err = router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), + }, + }, + }, + }, + }) + require.NoError(t, err) + + resp, err := apeSrv.Get(context.Background(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) + require.Contains(t, errAccessDenied.Reason(), chain.AccessDenied.String()) +} + +func testDenyGetContainerNoRuleFound(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{}, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) + + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + contRdr.c[contID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(nm.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + nm.netmaps[nm.currentEpoch] = &testNetmap + nm.netmaps[nm.currentEpoch-1] = &testNetmap + + req := &container.GetRequest{} + req.SetBody(&container.GetRequestBody{}) + var refContID refs.ContainerID + contID.WriteToV2(&refContID) + req.GetBody().SetContainerID(&refContID) + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + resp, err := apeSrv.Get(context.Background(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) + require.Contains(t, errAccessDenied.Reason(), chain.NoRuleFound.String()) +} + func testDenyGetContainerForOthers(t *testing.T) { t.Parallel() srv := &srvStub{ @@ -854,6 +980,8 @@ func TestValidateContainerBoundedOperation(t *testing.T) { }) require.NoError(t, err) + addDefaultAllowGetPolicy(t, components.engine, contID) + req := initTestGetContainerRequest(t, contID) err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) @@ -895,6 +1023,8 @@ func TestValidateContainerBoundedOperation(t *testing.T) { }) require.NoError(t, err) + addDefaultAllowGetPolicy(t, components.engine, contID) + req := initTestGetContainerRequest(t, contID) err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) @@ -936,6 +1066,8 @@ func TestValidateContainerBoundedOperation(t *testing.T) { }) require.NoError(t, err) + addDefaultAllowGetPolicy(t, components.engine, contID) + req := initTestGetContainerRequest(t, contID) err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) @@ -977,6 +1109,8 @@ func TestValidateContainerBoundedOperation(t *testing.T) { }) require.NoError(t, err) + addDefaultAllowGetPolicy(t, components.engine, contID) + req := initTestGetContainerRequest(t, contID) err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) @@ -1128,3 +1262,24 @@ func initListRequest(t *testing.T, actorPK *keys.PrivateKey, ownerPK *keys.Priva require.NoError(t, signature.SignServiceMessage(&actorPK.PrivateKey, req)) return req } + +func addDefaultAllowGetPolicy(t *testing.T, e engine.Engine, contID cid.ID) { + _, _, err := e.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.Allow, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + nativeschema.ResourceFormatAllContainers, + }, + }, + }, + }, + }) + require.NoError(t, err) +} From 75a1a95c2caefb12c91ce08a0b1ea5d5d992af6b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 15 Feb 2024 13:59:52 +0300 Subject: [PATCH 0946/1943] [#986] tree: Skip ACL checks if basicACL mask is unset Signed-off-by: Airat Arifullin --- pkg/services/tree/signature.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 15067d3cd..985e1ad94 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -77,6 +77,11 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op } basicACL := cnr.Value.BasicACL() + // Basic ACL mask can be unset, if a container operations are performed + // with strict APE checks only. + if basicACL == 0x0 { + return nil + } if !basicACL.IsOpAllowed(op, role) { return basicACLErr(op) From 93bf9acbc2e0d2572a9f47299bbd3ea408d6d004 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 27 Feb 2024 12:27:09 +0300 Subject: [PATCH 0947/1943] [#898] control: Remove removed flag from RemoveChainLocalOverrideResponse * Remove removed flag in service.proto for RemoveChainLocalOverrideResponse. * Regenerate control API. * Return error only if RemoveOverride returns non-NotFound code. Signed-off-by: Airat Arifullin --- .../modules/control/remove_rule.go | 6 +- pkg/services/control/server/policy_engine.go | 10 +- pkg/services/control/service.pb.go | 321 +++++++++--------- pkg/services/control/service.proto | 1 - pkg/services/control/service_frostfs.pb.go | 9 - 5 files changed, 158 insertions(+), 189 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/remove_rule.go b/cmd/frostfs-cli/modules/control/remove_rule.go index 1c3be375f..7684ac1dc 100644 --- a/cmd/frostfs-cli/modules/control/remove_rule.go +++ b/cmd/frostfs-cli/modules/control/remove_rule.go @@ -57,11 +57,7 @@ func removeRule(cmd *cobra.Command, _ []string) { verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - if resp.GetBody().GetRemoved() { - cmd.Println("Rule has been removed.") - } else { - cmd.Println("Rule has not been removed.") - } + cmd.Println("Rule has been removed.") } func initControlRemoveRuleCmd() { diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 079b605ac..4ea863f79 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -159,19 +159,13 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove return nil, err } - removed := true if err = s.localOverrideStorage.LocalStorage().RemoveOverride(apechain.Ingress, target, apechain.ID(req.GetBody().GetChainId())); err != nil { - code := getCodeByLocalStorageErr(err) - if code == codes.NotFound { - removed = false - } else { + if code := getCodeByLocalStorageErr(err); code != codes.NotFound { return nil, status.Error(code, err.Error()) } } resp := &control.RemoveChainLocalOverrideResponse{ - Body: &control.RemoveChainLocalOverrideResponse_Body{ - Removed: removed, - }, + Body: &control.RemoveChainLocalOverrideResponse_Body{}, } err = SignMessage(s.key, resp) if err != nil { diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 4e95ac617..c24bc1597 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -4086,8 +4086,6 @@ type RemoveChainLocalOverrideResponse_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - - Removed bool `protobuf:"varint,1,opt,name=removed,proto3" json:"removed,omitempty"` } func (x *RemoveChainLocalOverrideResponse_Body) Reset() { @@ -4122,13 +4120,6 @@ func (*RemoveChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33, 0} } -func (x *RemoveChainLocalOverrideResponse_Body) GetRemoved() bool { - if x != nil { - return x.Removed - } - return false -} - // Request body structure. type SealWriteCacheRequest_Body struct { state protoimpl.MessageState @@ -4816,7 +4807,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, - 0x64, 0x22, 0xba, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x64, 0x22, 0xa0, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, @@ -4825,172 +4816,170 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x20, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x22, 0xca, - 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, - 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, - 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, - 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x22, 0xca, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, + 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, - 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, - 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, - 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, + 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, + 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, + 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, + 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, + 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, - 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, - 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, + 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, + 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, + 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, + 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, + 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, + 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb1, 0x0d, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, - 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, - 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, - 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, - 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, - 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, + 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, + 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb1, 0x0d, 0x0a, + 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, + 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, + 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, + 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, + 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, + 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, + 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, + 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, - 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, - 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, + 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, + 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, + 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, + 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, - 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, - 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, - 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, - 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, - 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, + 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, + 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, + 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, + 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, + 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, + 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, + 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, + 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, + 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, + 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 16e0f707d..1377965e6 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -566,7 +566,6 @@ message RemoveChainLocalOverrideRequest { message RemoveChainLocalOverrideResponse { message Body { - bool removed = 1; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 37527aa8c..56b2bd54c 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -2861,7 +2861,6 @@ func (x *RemoveChainLocalOverrideResponse_Body) StableSize() (size int) { if x == nil { return 0 } - size += proto.BoolSize(1, x.Removed) return size } @@ -2874,14 +2873,6 @@ func (x *RemoveChainLocalOverrideResponse_Body) StableSize() (size int) { // // Structures with the same field values have the same binary format. func (x *RemoveChainLocalOverrideResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BoolMarshal(1, buf[offset:], x.Removed) return buf } From 6f25c790aa20a43949fe64cf90d8fe79b437d9b6 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 29 Feb 2024 14:22:10 +0300 Subject: [PATCH 0948/1943] [#1012] cli: Fix messages for list commands Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/control/list_rules.go | 5 +++-- cmd/frostfs-cli/modules/control/list_targets.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index 31f5ff959..4598320d7 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -69,9 +69,10 @@ func parseTarget(cmd *cobra.Command) *control.ChainTarget { func listRules(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) + target := parseTarget(cmd) req := &control.ListChainLocalOverridesRequest{ Body: &control.ListChainLocalOverridesRequest_Body{ - Target: parseTarget(cmd), + Target: target, }, } @@ -91,7 +92,7 @@ func listRules(cmd *cobra.Command, _ []string) { chains := resp.GetBody().GetChains() if len(chains) == 0 { - cmd.Println("Local overrides are not defined for the container.") + cmd.Printf("Local overrides are not defined for the %s.\n", strings.ToLower(target.GetType().String())) return } diff --git a/cmd/frostfs-cli/modules/control/list_targets.go b/cmd/frostfs-cli/modules/control/list_targets.go index 29033d017..ddd80b31b 100644 --- a/cmd/frostfs-cli/modules/control/list_targets.go +++ b/cmd/frostfs-cli/modules/control/list_targets.go @@ -58,7 +58,7 @@ func listTargets(cmd *cobra.Command, _ []string) { targets := resp.GetBody().GetTargets() if len(targets) == 0 { - cmd.Println("Local overrides are not defined for the container.") + cmd.Println("Local overrides are not defined for any target.") return } From 6dbb61caf4f9b99176dc0351cf42c9ed3b3a1db0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 1 Mar 2024 11:44:03 +0300 Subject: [PATCH 0949/1943] [#1016] Makefile: Add gopls check target Signed-off-by: Dmitrii Stepanov --- Makefile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Makefile b/Makefile index 4ff42b7e4..14738272e 100755 --- a/Makefile +++ b/Makefile @@ -44,6 +44,10 @@ PROTOGEN_FROSTFS_DIR ?= $(PROTOBUF_DIR)/protogen-$(PROTOGEN_FROSTFS_VERSION) STATICCHECK_DIR ?= $(abspath $(BIN))/staticcheck STATICCHECK_VERSION_DIR ?= $(STATICCHECK_DIR)/$(STATICCHECK_VERSION) +GOPLS_VERSION ?= v0.15.1 +GOPLS_DIR ?= $(abspath $(BIN))/gopls +GOPLS_VERSION_DIR ?= $(GOPLS_DIR)/$(GOPLS_VERSION) + FROSTFS_CONTRACTS_PATH=$(abspath ./../frostfs-contract) LOCODE_DB_PATH=$(abspath ./.cache/locode_db) LOCODE_DB_VERSION=v0.4.0 @@ -201,6 +205,19 @@ staticcheck-run: fi @$(STATICCHECK_VERSION_DIR)/staticcheck ./... +gopls-install: + @rm -rf $(GOPLS_DIR) + @mkdir $(GOPLS_DIR) + @GOBIN=$(GOPLS_VERSION_DIR) go install golang.org/x/tools/gopls@$(GOPLS_VERSION) + +gopls-run: + @if [ ! -d "$(GOPLS_VERSION_DIR)" ]; then \ + make gopls-install; \ + fi + @if [[ $$(find . -type f -name "*.go" -print | xargs $(GOPLS_VERSION_DIR)/gopls check | tee /dev/tty | wc -l) -ne 0 ]]; then \ + exit 1; \ + fi + # Run linters in Docker docker/lint: docker run --rm -t \ From d6534fd755c22fd23ed471e2440d95d640903f98 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 1 Mar 2024 11:58:17 +0300 Subject: [PATCH 0950/1943] [#1016] frostfs-node: Fix gopls issues Signed-off-by: Dmitrii Stepanov --- .../morph/initialize/initialize_test.go | 6 ++--- cmd/frostfs-node/cache_test.go | 2 +- cmd/frostfs-node/config/morph/config_test.go | 10 +++++++-- internal/ape/converter_test.go | 22 +++++++++---------- pkg/innerring/initialization.go | 5 ----- .../blobstor/blobovniczatree/cache.go | 2 +- .../engine/delete_test.go | 8 +++---- .../engine/engine_test.go | 12 +++++----- pkg/local_object_storage/engine/head_test.go | 4 ++-- .../engine/inhume_test.go | 4 ++-- pkg/local_object_storage/metabase/put_test.go | 6 ++--- pkg/services/object/ape/checker_test.go | 4 ++-- pkg/services/object/ape/request_test.go | 4 ++-- .../placement/traverser_test.go | 12 +++++----- pkg/services/tree/cache.go | 2 +- pkg/services/tree/sync.go | 2 +- 16 files changed, 53 insertions(+), 52 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go index e39c7356f..6c52aa2ab 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go @@ -53,7 +53,7 @@ func TestInitialize(t *testing.T) { testInitialize(t, constants.MaxAlphabetNodes) }) t.Run("too many nodes", func(t *testing.T) { - require.ErrorIs(t, generateTestData(t, t.TempDir(), constants.MaxAlphabetNodes+1), helper.ErrTooManyAlphabetNodes) + require.ErrorIs(t, generateTestData(t.TempDir(), constants.MaxAlphabetNodes+1), helper.ErrTooManyAlphabetNodes) }) } @@ -61,7 +61,7 @@ func testInitialize(t *testing.T, committeeSize int) { testdataDir := t.TempDir() v := viper.GetViper() - require.NoError(t, generateTestData(t, testdataDir, committeeSize)) + require.NoError(t, generateTestData(testdataDir, committeeSize)) v.Set(constants.ProtoConfigPath, filepath.Join(testdataDir, protoFileName)) // Set to the path or remove the next statement to download from the network. @@ -98,7 +98,7 @@ func testInitialize(t *testing.T, committeeSize int) { }) } -func generateTestData(t *testing.T, dir string, size int) error { +func generateTestData(dir string, size int) error { v := viper.GetViper() v.Set(commonflags.AlphabetWalletsFlag, dir) diff --git a/cmd/frostfs-node/cache_test.go b/cmd/frostfs-node/cache_test.go index 6e076abfc..6981ac721 100644 --- a/cmd/frostfs-node/cache_test.go +++ b/cmd/frostfs-node/cache_test.go @@ -10,7 +10,7 @@ import ( func TestTTLNetCache(t *testing.T) { ttlDuration := time.Millisecond * 50 - cache := newNetworkTTLCache[string, time.Time](10, ttlDuration, testNetValueReader) + cache := newNetworkTTLCache(10, ttlDuration, testNetValueReader) key := "key" diff --git a/cmd/frostfs-node/config/morph/config_test.go b/cmd/frostfs-node/config/morph/config_test.go index 192140446..a30733cd0 100644 --- a/cmd/frostfs-node/config/morph/config_test.go +++ b/cmd/frostfs-node/config/morph/config_test.go @@ -24,8 +24,14 @@ func TestMorphSection(t *testing.T) { const path = "../../../../config/example/node" rpcs := []client.Endpoint{ - {"wss://rpc1.morph.frostfs.info:40341/ws", 1}, - {"wss://rpc2.morph.frostfs.info:40341/ws", 2}, + { + Address: "wss://rpc1.morph.frostfs.info:40341/ws", + Priority: 1, + }, + { + Address: "wss://rpc2.morph.frostfs.info:40341/ws", + Priority: 2, + }, } fileConfigTest := func(c *config.Config) { diff --git a/internal/ape/converter_test.go b/internal/ape/converter_test.go index de72408b1..b5f8939c4 100644 --- a/internal/ape/converter_test.go +++ b/internal/ape/converter_test.go @@ -27,7 +27,7 @@ func TestEACLTableWithoutRecords(t *testing.T) { res: &testResource{name: nativeschema.ResourceFormatRootObjects}, } - compare(t, tb, vu, ch, req) + compare(t, vu, ch, req) cnrID := cidtest.ID() tb.SetCID(cnrID) @@ -37,7 +37,7 @@ func TestEACLTableWithoutRecords(t *testing.T) { ch, err = ConvertEACLToAPE(tb) require.NoError(t, err) - compare(t, tb, vu, ch, req) + compare(t, vu, ch, req) } func TestNoTargets(t *testing.T) { @@ -69,7 +69,7 @@ func TestNoTargets(t *testing.T) { }, res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, } - compare(t, tb, vu, ch, req) + compare(t, vu, ch, req) } } @@ -109,7 +109,7 @@ func TestNoFilters(t *testing.T) { }, res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, } - compare(t, tb, vu, ch, req) + compare(t, vu, ch, req) } }) @@ -155,7 +155,7 @@ func TestNoFilters(t *testing.T) { }, res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, } - compare(t, tb, vu, ch, req) + compare(t, vu, ch, req) } }) @@ -199,7 +199,7 @@ func TestNoFilters(t *testing.T) { }, res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, } - compare(t, tb, vu, ch, req) + compare(t, vu, ch, req) } }) @@ -236,7 +236,7 @@ func TestNoFilters(t *testing.T) { }, res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, } - compare(t, tb, vu, ch, req) + compare(t, vu, ch, req) } }) } @@ -292,7 +292,7 @@ func TestWithFilters(t *testing.T) { }, }, } - compare(t, tb, vu, ch, req) + compare(t, vu, ch, req) } }) @@ -342,7 +342,7 @@ func TestWithFilters(t *testing.T) { name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString()), }, } - compare(t, tb, vu, ch, req) + compare(t, vu, ch, req) } }) } @@ -391,10 +391,10 @@ func TestNoHeader(t *testing.T) { name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString()), }, } - compare(t, tb, vu, ch, req) + compare(t, vu, ch, req) } -func compare(t *testing.T, eaclTable *eacl.Table, vu *eacl.ValidationUnit, ch *apechain.Chain, req *testRequest) { +func compare(t *testing.T, vu *eacl.ValidationUnit, ch *apechain.Chain, req *testRequest) { validator := eacl.NewValidator() for eaclOp, apeOp := range eaclOperationToEngineAction { vu.WithOperation(eaclOp) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index f4d9b4169..1a4174289 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -42,10 +42,6 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, return err } - if err != nil { - return err - } - netSettings := (*networkSettings)(s.netmapClient) var netMapCandidateStateValidator statevalidation.NetMapCandidateValidator @@ -76,7 +72,6 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, NodeStateSettings: netSettings, }) - if err != nil { return err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go index d8fe6e8b0..ef1793edf 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -24,7 +24,7 @@ type dbCache struct { } func newDBCache(size int, ttl time.Duration, dbManager *dbManager) *dbCache { - cache := expirable.NewLRU[string, *sharedDB](size, func(_ string, evictedDB *sharedDB) { + cache := expirable.NewLRU(size, func(_ string, evictedDB *sharedDB) { evictedDB.Close() }, ttl) return &dbCache{ diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 32d07809a..4a6758012 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -49,9 +49,9 @@ func TestDeleteBigObject(t *testing.T) { link.SetSplitID(splitID) link.SetChildren(childIDs...) - s1 := testNewShard(t, 1) - s2 := testNewShard(t, 2) - s3 := testNewShard(t, 3) + s1 := testNewShard(t) + s2 := testNewShard(t) + s3 := testNewShard(t) e := testNewEngine(t).setInitializedShards(t, s1, s2, s3).engine e.log = test.NewLogger(t) @@ -119,7 +119,7 @@ func TestDeleteBigObjectWithoutGC(t *testing.T) { link.SetSplitID(splitID) link.SetChildren(childIDs...) - s1 := testNewShard(t, 1, shard.WithDisabledGC()) + s1 := testNewShard(t, shard.WithDisabledGC()) e := testNewEngine(t).setInitializedShards(t, s1).engine e.log = test.NewLogger(t) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index b20f45be5..b6858df49 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -50,7 +50,7 @@ func BenchmarkExists(b *testing.B) { func benchmarkExists(b *testing.B, shardNum int) { shards := make([]*shard.Shard, shardNum) for i := 0; i < shardNum; i++ { - shards[i] = testNewShard(b, i) + shards[i] = testNewShard(b) } e := testNewEngine(b).setInitializedShards(b, shards...).engine @@ -112,7 +112,7 @@ func (te *testEngineWrapper) setShardsNum(t testing.TB, num int) *testEngineWrap shards := make([]*shard.Shard, 0, num) for i := 0; i < num; i++ { - shards = append(shards, testNewShard(t, i)) + shards = append(shards, testNewShard(t)) } return te.setInitializedShards(t, shards...) @@ -130,7 +130,7 @@ func (te *testEngineWrapper) setShardsNumOpts(t testing.TB, num int, shardOpts f func (te *testEngineWrapper) setShardsNumAdditionalOpts(t testing.TB, num int, shardOpts func(id int) []shard.Option) *testEngineWrapper { for i := 0; i < num; i++ { - defaultOpts := testDefaultShardOptions(t, i) + defaultOpts := testDefaultShardOptions(t) opts := append(defaultOpts, shardOpts(i)...) id, err := te.engine.AddShard(context.Background(), opts...) require.NoError(t, err) @@ -187,11 +187,11 @@ func newTestStorages(root string, smallSize uint64) ([]blobstor.SubStorage, *tes }, smallFileStorage, largeFileStorage } -func testNewShard(t testing.TB, id int, opts ...shard.Option) *shard.Shard { +func testNewShard(t testing.TB, opts ...shard.Option) *shard.Shard { sid, err := generateShardID() require.NoError(t, err) - shardOpts := append([]shard.Option{shard.WithID(sid)}, testDefaultShardOptions(t, id)...) + shardOpts := append([]shard.Option{shard.WithID(sid)}, testDefaultShardOptions(t)...) s := shard.New(append(shardOpts, opts...)...) require.NoError(t, s.Open(context.Background())) @@ -200,7 +200,7 @@ func testNewShard(t testing.TB, id int, opts ...shard.Option) *shard.Shard { return s } -func testDefaultShardOptions(t testing.TB, id int) []shard.Option { +func testDefaultShardOptions(t testing.TB) []shard.Option { return []shard.Option{ shard.WithLogger(test.NewLogger(t)), shard.WithBlobStorOptions( diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go index 5c123d617..5afc50f07 100644 --- a/pkg/local_object_storage/engine/head_test.go +++ b/pkg/local_object_storage/engine/head_test.go @@ -39,8 +39,8 @@ func TestHeadRaw(t *testing.T) { link.SetSplitID(splitID) t.Run("virtual object split in different shards", func(t *testing.T) { - s1 := testNewShard(t, 1) - s2 := testNewShard(t, 2) + s1 := testNewShard(t) + s2 := testNewShard(t) e := testNewEngine(t).setInitializedShards(t, s1, s2).engine defer e.Close(context.Background()) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 4bb128bd7..9daa113f8 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -56,8 +56,8 @@ func TestStorageEngine_Inhume(t *testing.T) { t.Run("delete big object", func(t *testing.T) { t.Parallel() - s1 := testNewShard(t, 1) - s2 := testNewShard(t, 2) + s1 := testNewShard(t) + s2 := testNewShard(t) e := testNewEngine(t).setInitializedShards(t, s1, s2).engine defer e.Close(context.Background()) diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index 28467199d..84e4029f2 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/require" ) -func prepareObjects(t testing.TB, n int) []*objectSDK.Object { +func prepareObjects(n int) []*objectSDK.Object { cnr := cidtest.ID() parentID := objecttest.ID() objs := make([]*objectSDK.Object, n) @@ -53,7 +53,7 @@ func BenchmarkPut(b *testing.B) { var index atomic.Int64 index.Store(-1) - objs := prepareObjects(b, b.N) + objs := prepareObjects(b.N) b.ResetTimer() b.ReportAllocs() b.RunParallel(func(pb *testing.PB) { @@ -71,7 +71,7 @@ func BenchmarkPut(b *testing.B) { defer func() { require.NoError(b, db.Close()) }() var index atomic.Int64 index.Store(-1) - objs := prepareObjects(b, b.N) + objs := prepareObjects(b.N) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index d7e97064b..443414959 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -82,7 +82,7 @@ func stringPtr(s string) *string { return &s } -func newHeaderObjectSDK(t *testing.T, cnr cid.ID, oid *oid.ID, headerObjSDK *headerObjectSDKParams) *objectSDK.Object { +func newHeaderObjectSDK(cnr cid.ID, oid *oid.ID, headerObjSDK *headerObjectSDKParams) *objectSDK.Object { objSDK := objectSDK.New() objSDK.SetContainerID(cnr) if oid != nil { @@ -344,7 +344,7 @@ func TestAPECheck(t *testing.T) { var headerObjSDK *objectSDK.Object if test.header.headerObjSDK != nil { - headerObjSDK = newHeaderObjectSDK(t, cnr, obj, test.header.headerObjSDK) + headerObjSDK = newHeaderObjectSDK(cnr, obj, test.header.headerObjSDK) if test.header.fromHeaderProvider { require.NotNil(t, obj, "oid is required if a header is expected to be found in header provider") headerProvider.addHeader(cnr, *obj, headerObjSDK) diff --git a/pkg/services/object/ape/request_test.go b/pkg/services/object/ape/request_test.go index 55aa691b4..0dcc0b84b 100644 --- a/pkg/services/object/ape/request_test.go +++ b/pkg/services/object/ape/request_test.go @@ -80,7 +80,7 @@ func TestObjectProperties(t *testing.T) { t.Run(test.name, func(t *testing.T) { cnr := newContainerIDSDK(t, test.container) obj := newObjectIDSDK(t, test.object) - header := newHeaderObjectSDK(t, cnr, obj, test.header) + header := newHeaderObjectSDK(cnr, obj, test.header) props := objectProperties(cnr, obj, header.ToV2().GetHeader()) require.Equal(t, test.container, props[nativeschema.PropertyKeyObjectContainerID]) @@ -223,7 +223,7 @@ func TestNewAPERequest(t *testing.T) { var headerObjSDK *objectSDK.Object if test.header.headerObjSDK != nil { - headerObjSDK = newHeaderObjectSDK(t, cnr, obj, test.header.headerObjSDK) + headerObjSDK = newHeaderObjectSDK(cnr, obj, test.header.headerObjSDK) if test.header.fromHeaderProvider { require.NotNil(t, obj, "oid is required if a header is expected to be found in header provider") headerSource.addHeader(cnr, *obj, headerObjSDK) diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index 9b70efc73..f5731c81e 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -40,7 +40,7 @@ func copyVectors(v [][]netmap.NodeInfo) [][]netmap.NodeInfo { return vc } -func testPlacement(t *testing.T, ss, rs []int) ([][]netmap.NodeInfo, container.Container) { +func testPlacement(ss, rs []int) ([][]netmap.NodeInfo, container.Container) { nodes := make([][]netmap.NodeInfo, 0, len(rs)) replicas := make([]netmap.ReplicaDescriptor, 0, len(rs)) num := uint32(0) @@ -83,7 +83,7 @@ func TestTraverserObjectScenarios(t *testing.T) { selectors := []int{2, 3} replicas := []int{1, 2} - nodes, cnr := testPlacement(t, selectors, replicas) + nodes, cnr := testPlacement(selectors, replicas) nodesCopy := copyVectors(nodes) @@ -112,7 +112,7 @@ func TestTraverserObjectScenarios(t *testing.T) { selectors := []int{5, 3} replicas := []int{2, 2} - nodes, cnr := testPlacement(t, selectors, replicas) + nodes, cnr := testPlacement(selectors, replicas) nodesCopy := copyVectors(nodes) @@ -141,7 +141,7 @@ func TestTraverserObjectScenarios(t *testing.T) { selectors := []int{5, 3} replicas := []int{2, 2} - nodes, cnr := testPlacement(t, selectors, replicas) + nodes, cnr := testPlacement(selectors, replicas) nodesCopy := copyVectors(nodes) @@ -184,7 +184,7 @@ func TestTraverserObjectScenarios(t *testing.T) { selectors := []int{2, 3} replicas := []int{1, 2} - nodes, cnr := testPlacement(t, selectors, replicas) + nodes, cnr := testPlacement(selectors, replicas) tr, err := NewTraverser( ForContainer(cnr), @@ -213,7 +213,7 @@ func TestTraverserRemValues(t *testing.T) { selectors := []int{3, 4, 5} replicas := []int{2, 3, 4} - nodes, cnr := testPlacement(t, selectors, replicas) + nodes, cnr := testPlacement(selectors, replicas) nodesCopy := copyVectors(nodes) testCases := [...]struct { diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index f50aa0b0d..1be1c2f83 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -35,7 +35,7 @@ const ( var errRecentlyFailed = errors.New("client has recently failed") func (c *clientCache) init() { - l, _ := simplelru.NewLRU[string, cacheItem](defaultClientCacheSize, func(_ string, value cacheItem) { + l, _ := simplelru.NewLRU(defaultClientCacheSize, func(_ string, value cacheItem) { if conn := value.cc; conn != nil { _ = conn.Close() } diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 2a19ae18a..064ee5900 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -251,7 +251,7 @@ func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, } opsCh <- m } - if err != nil && !errors.Is(err, io.EOF) { + if !errors.Is(err, io.EOF) { return err } return nil From 46a04463b2c733e331f8a10a561241e08272856a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 1 Mar 2024 12:00:27 +0300 Subject: [PATCH 0951/1943] [#1016] forgejo: Add gopls check step Signed-off-by: Dmitrii Stepanov --- .forgejo/workflows/tests.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index f66a2c401..449af3f51 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -71,3 +71,21 @@ jobs: - name: Run staticcheck run: make staticcheck-run + + gopls: + name: gopls check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.21' + cache: true + + - name: Install gopls + run: make gopls-install + + - name: Run gopls + run: make gopls-run From ae5bb87e7087245d775352eb2c1a74dd5d38468e Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 1 Mar 2024 19:29:33 +0300 Subject: [PATCH 0952/1943] Revert "[#866] Use TTL for blobovnicza tree cache" This reverts commit d9cbb16bd3cec547ba9621a51ac3d8b0d75b211b. Signed-off-by: Anton Nikiforov --- .../internal/modules/storagecfg/config.go | 1 - cmd/frostfs-node/config.go | 3 --- .../shard/blobstor/blobovnicza/config.go | 21 ------------------- config/example/node.env | 2 -- config/example/node.json | 2 -- config/example/node.yaml | 1 - docs/storage-node-configuration.md | 18 +++++++--------- .../blobstor/blobovniczatree/blobovnicza.go | 2 +- .../blobstor/blobovniczatree/cache.go | 13 +++++++----- .../blobstor/blobovniczatree/option.go | 9 -------- 10 files changed, 17 insertions(+), 55 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/config.go b/cmd/frostfs-adm/internal/modules/storagecfg/config.go index 9f7efe2df..a07ce32c6 100644 --- a/cmd/frostfs-adm/internal/modules/storagecfg/config.go +++ b/cmd/frostfs-adm/internal/modules/storagecfg/config.go @@ -61,7 +61,6 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files - opened_cache_ttl: 5m # ttl for opened database file gc: remover_batch_size: 200 # number of objects to be removed by the garbage collector diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 50219a8c7..360fdbec6 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -182,7 +182,6 @@ type subStorageCfg struct { width uint64 leafWidth uint64 openedCacheSize int - openedCacheTTL time.Duration initWorkerCount int initInAdvance bool } @@ -302,7 +301,6 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.width = sub.ShallowWidth() sCfg.leafWidth = sub.LeafWidth() sCfg.openedCacheSize = sub.OpenedCacheSize() - sCfg.openedCacheTTL = sub.OpenedCacheTTL() sCfg.initWorkerCount = sub.InitWorkerCount() sCfg.initInAdvance = sub.InitInAdvance() case fstree.Type: @@ -887,7 +885,6 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), - blobovniczatree.WithOpenedCacheTTL(sRead.openedCacheTTL), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), blobovniczatree.WithInitInAdvance(sRead.initInAdvance), blobovniczatree.WithLogger(c.log), diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index 0e007b425..37191cc72 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -1,8 +1,6 @@ package blobovniczaconfig import ( - "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" @@ -25,9 +23,6 @@ const ( // OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's. OpenedCacheSizeDefault = 16 - // OpenedCacheTTLDefault is a default cache ttl of opened Blobovnicza's. - OpenedCacheTTLDefault = 0 // means expiring is off - // InitWorkerCountDefault is a default workers count to initialize Blobovnicza's. InitWorkerCountDefault = 5 ) @@ -106,22 +101,6 @@ func (x *Config) OpenedCacheSize() int { return OpenedCacheSizeDefault } -// OpenedCacheTTL returns the value of "opened_cache_ttl" config parameter. -// -// Returns OpenedCacheTTLDefault if the value is not a positive number. -func (x *Config) OpenedCacheTTL() time.Duration { - d := config.DurationSafe( - (*config.Config)(x), - "opened_cache_ttl", - ) - - if d > 0 { - return d - } - - return OpenedCacheTTLDefault -} - // BoltDB returns config instance for querying bolt db specific parameters. func (x *Config) BoltDB() *boltdbconfig.Config { return (*boltdbconfig.Config)(x) diff --git a/config/example/node.env b/config/example/node.env index a1db0c876..b0364b741 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -126,7 +126,6 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 -FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=5m FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE @@ -176,7 +175,6 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 -FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_TTL=5m FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10 ### FSTree config FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree diff --git a/config/example/node.json b/config/example/node.json index a7d7a3651..495fcb8ce 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -174,7 +174,6 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, - "opened_cache_ttl": "5m", "leaf_width": 10, "init_worker_count": 10, "init_in_advance": true @@ -227,7 +226,6 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, - "opened_cache_ttl": "5m", "leaf_width": 10 }, { diff --git a/config/example/node.yaml b/config/example/node.yaml index 678ee1a87..96af3b208 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -150,7 +150,6 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files - opened_cache_ttl: 5m # ttl for opened database file leaf_width: 10 # max count of key-value DB on leafs of object tree storage - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) depth: 5 # max depth of object tree storage in FS diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index e2334f8fb..6573e3ea2 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -209,7 +209,6 @@ blobstor: depth: 1 width: 4 opened_cache_capacity: 50 - opened_cache_ttl: 5m ``` #### Common options for sub-storages @@ -226,15 +225,14 @@ blobstor: | `depth` | `int` | `4` | File-system tree depth. | #### `blobovnicza` type options -| Parameter | Type | Default value | Description | -|-------------------------|------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `path` | `string` | | Path to the root of the blobstor. | -| `perm` | file mode | `0660` | Default permission for created files and directories. | -| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | -| `depth` | `int` | `2` | Blobovnicza tree depth. | -| `width` | `int` | `16` | Blobovnicza tree width. | -| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | -| `opened_cache_ttl` | `duration` | `0` | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. | +| Parameter | Type | Default value | Description | +|-------------------------|-----------|---------------|-------------------------------------------------------| +| `path` | `string` | | Path to the root of the blobstor. | +| `perm` | file mode | `0660` | Default permission for created files and directories. | +| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | +| `depth` | `int` | `2` | Blobovnicza tree depth. | +| `width` | `int` | `16` | Blobovnicza tree width. | +| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | ### `gc` subsection diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 460aebbba..9f2950094 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -87,7 +87,7 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) - blz.dbCache = newDBCache(blz.openedCacheSize, blz.openedCacheTTL, blz.commondbManager) + blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager) blz.deleteProtectedObjects = newAddressMap() blz.dbFilesGuard = &sync.RWMutex{} blz.rebuildGuard = &sync.RWMutex{} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go index ef1793edf..766f5ef0a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -1,11 +1,10 @@ package blobovniczatree import ( + "fmt" "sync" - "time" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" - "github.com/hashicorp/golang-lru/v2/expirable" "github.com/hashicorp/golang-lru/v2/simplelru" ) @@ -23,10 +22,14 @@ type dbCache struct { dbManager *dbManager } -func newDBCache(size int, ttl time.Duration, dbManager *dbManager) *dbCache { - cache := expirable.NewLRU(size, func(_ string, evictedDB *sharedDB) { +func newDBCache(size int, dbManager *dbManager) *dbCache { + cache, err := simplelru.NewLRU(size, func(_ string, evictedDB *sharedDB) { evictedDB.Close() - }, ttl) + }) + if err != nil { + // occurs only if the size is not positive + panic(fmt.Errorf("could not create LRU cache of size %d: %w", size, err)) + } return &dbCache{ cacheGuard: &sync.RWMutex{}, cache: cache, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index e93640fae..d36074dc8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -16,7 +16,6 @@ type cfg struct { readOnly bool rootPath string openedCacheSize int - openedCacheTTL time.Duration blzShallowDepth uint64 blzShallowWidth uint64 blzLeafWidth uint64 @@ -35,7 +34,6 @@ type Option func(*cfg) const ( defaultPerm = 0o700 defaultOpenedCacheSize = 50 - defaultOpenedCacheTTL = 0 // means expiring is off defaultBlzShallowDepth = 2 defaultBlzShallowWidth = 16 defaultWaitBeforeDropDB = 10 * time.Second @@ -48,7 +46,6 @@ func initConfig(c *cfg) { log: &logger.Logger{Logger: zap.L()}, perm: defaultPerm, openedCacheSize: defaultOpenedCacheSize, - openedCacheTTL: defaultOpenedCacheTTL, blzShallowDepth: defaultBlzShallowDepth, blzShallowWidth: defaultBlzShallowWidth, reportError: func(string, error) {}, @@ -108,12 +105,6 @@ func WithOpenedCacheSize(sz int) Option { } } -func WithOpenedCacheTTL(ttl time.Duration) Option { - return func(c *cfg) { - c.openedCacheTTL = ttl - } -} - func WithObjectSizeLimit(sz uint64) Option { return func(c *cfg) { c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz)) From b38effd799671eebaf56e11e91404e92ed491f86 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 4 Mar 2024 09:37:31 +0300 Subject: [PATCH 0953/1943] [#1019] go.mod: Update sdk-go Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d86676dd3..0b0b03d0a 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240215124401-634e24aba715 git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240215122347-a86170f53af7 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240129064140-8d21ab2d99d9 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index fa5da98cc..51d68321e 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240215122347-a86170f53af7 h1:3a5g14oC+aMQ1p0bvuqG3JZKGLOcqRvcHzKAFV4eS0w= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240215122347-a86170f53af7/go.mod h1:XcgrbZ88XfvhAMxmZCQJ0dv6FyRSq6Mg2J7nN8uuO0k= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b h1:nLIWYXe4e1fWgpKeMfVke/CNBn388egh4fArFdvhfHw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b/go.mod h1:XcgrbZ88XfvhAMxmZCQJ0dv6FyRSq6Mg2J7nN8uuO0k= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240129064140-8d21ab2d99d9 h1:hkDIg6vzV1c1pyt5uj9EiBikHqKczfoNpPhwWvXaNf8= From 1c504dca5c09827d331632b1684104f052aba2e7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 4 Mar 2024 16:14:09 +0300 Subject: [PATCH 0954/1943] [#1021] dev: Up neo-go version Current version of neo-go is 0.105. Signed-off-by: Dmitrii Stepanov --- dev/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 7e68ea02e..9d026797c 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -3,7 +3,7 @@ version: "2.4" services: neo-go: - image: nspccdev/neo-go:0.103.0 + image: nspccdev/neo-go:0.105.0 container_name: neo-go command: ["node", "--config-path", "/config", "--privnet", "--debug"] stop_signal: SIGKILL From 702351a5d1b422a12815cbe4598c81e0bf1b503c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 4 Mar 2024 18:01:39 +0300 Subject: [PATCH 0955/1943] [#983] blobstor: Allow to specify wait before drop time Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 15 +++++++++------ cmd/frostfs-node/config/engine/config_test.go | 2 ++ .../shard/blobstor/blobovnicza/config.go | 19 +++++++++++++++++++ config/example/node.env | 1 + config/example/node.json | 3 ++- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 19 +++++++++++-------- 7 files changed, 45 insertions(+), 15 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 360fdbec6..e5d3f7311 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -178,12 +178,13 @@ type subStorageCfg struct { noSync bool // blobovnicza-specific - size uint64 - width uint64 - leafWidth uint64 - openedCacheSize int - initWorkerCount int - initInAdvance bool + size uint64 + width uint64 + leafWidth uint64 + openedCacheSize int + initWorkerCount int + initInAdvance bool + rebuildDropTimeout time.Duration } // readConfig fills applicationConfiguration with raw configuration values @@ -303,6 +304,7 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.openedCacheSize = sub.OpenedCacheSize() sCfg.initWorkerCount = sub.InitWorkerCount() sCfg.initInAdvance = sub.InitInAdvance() + sCfg.rebuildDropTimeout = sub.RebuildDropTimeout() case fstree.Type: sub := fstreeconfig.From((*config.Config)(storagesCfg[i])) sCfg.depth = sub.Depth() @@ -887,6 +889,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), blobovniczatree.WithInitInAdvance(sRead.initInAdvance), + blobovniczatree.WithWaitBeforeDropDB(sRead.rebuildDropTimeout), blobovniczatree.WithLogger(c.log), blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit), } diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 6c7b23cc4..86c938309 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -101,6 +101,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 10, blz.LeafWidth()) require.EqualValues(t, 10, blz.InitWorkerCount()) require.EqualValues(t, true, blz.InitInAdvance()) + require.EqualValues(t, 30*time.Second, blz.RebuildDropTimeout()) require.Equal(t, "tmp/0/blob", ss[1].Path()) require.EqualValues(t, 0o644, ss[1].Perm()) @@ -151,6 +152,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 50, blz.OpenedCacheSize()) require.EqualValues(t, 10, blz.LeafWidth()) require.EqualValues(t, blobovniczaconfig.InitWorkerCountDefault, blz.InitWorkerCount()) + require.EqualValues(t, blobovniczaconfig.RebuildDropTimeoutDefault, blz.RebuildDropTimeout()) require.Equal(t, "tmp/1/blob", ss[1].Path()) require.EqualValues(t, 0o644, ss[1].Perm()) diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index 37191cc72..9619a1027 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -1,6 +1,8 @@ package blobovniczaconfig import ( + "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" @@ -25,6 +27,9 @@ const ( // InitWorkerCountDefault is a default workers count to initialize Blobovnicza's. InitWorkerCountDefault = 5 + + // RebuildDropTimeoutDefault is a default timeout value to wait before drop single blobovnicza. + RebuildDropTimeoutDefault = 10 * time.Second ) // From wraps config section into Config. @@ -141,3 +146,17 @@ func (x *Config) InitInAdvance() bool { "init_in_advance", ) } + +// RebuildDropTimeout returns the value of "rebuild_drop_timeout" config parameter. +// +// Returns RebuildDropTimeoutDefault if the value is not defined or invalid. +func (x *Config) RebuildDropTimeout() time.Duration { + d := config.DurationSafe( + (*config.Config)(x), + "rebuild_drop_timeout", + ) + if d > 0 { + return d + } + return RebuildDropTimeoutDefault +} diff --git a/config/example/node.env b/config/example/node.env index b0364b741..810a2842b 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -129,6 +129,7 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE +FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_REBUILD_DROP_TIMEOUT=30s ### FSTree config FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH=tmp/0/blob diff --git a/config/example/node.json b/config/example/node.json index 495fcb8ce..e24a12f5f 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -176,7 +176,8 @@ "opened_cache_capacity": 50, "leaf_width": 10, "init_worker_count": 10, - "init_in_advance": true + "init_in_advance": true, + "rebuild_drop_timeout": "30s" }, { "type": "fstree", diff --git a/config/example/node.yaml b/config/example/node.yaml index 96af3b208..c9886355f 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -190,6 +190,7 @@ storage: path: tmp/0/blob/blobovnicza init_worker_count: 10 #count of workers to initialize blobovniczas init_in_advance: true + rebuild_drop_timeout: 30s # timeout before drop single blobovnicza - type: fstree path: tmp/0/blob # blobstor path diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 6573e3ea2..15071f23d 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -225,14 +225,17 @@ blobstor: | `depth` | `int` | `4` | File-system tree depth. | #### `blobovnicza` type options -| Parameter | Type | Default value | Description | -|-------------------------|-----------|---------------|-------------------------------------------------------| -| `path` | `string` | | Path to the root of the blobstor. | -| `perm` | file mode | `0660` | Default permission for created files and directories. | -| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | -| `depth` | `int` | `2` | Blobovnicza tree depth. | -| `width` | `int` | `16` | Blobovnicza tree width. | -| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | +| Parameter | Type | Default value | Description | +| ----------------------- | ---------- | ------------- | --------------------------------------------------------------------- | +| `path` | `string` | | Path to the root of the blobstor. | +| `perm` | file mode | `0660` | Default permission for created files and directories. | +| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | +| `depth` | `int` | `2` | Blobovnicza tree depth. | +| `width` | `int` | `16` | Blobovnicza tree width. | +| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | +| `init_worker_count` | `int` | `5` | Maximum number of concurrent initialization workers. | +| `init_in_advance` | `bool` | `false` | If `true`, than all the blobovnicza files will be created on startup. | +| `rebuild_drop_timeout` | `duration` | `10s` | Timeout before drop empty blobovnicza file during rebuild. | ### `gc` subsection From c6f0545298ca0dffffeb5168f0d4aafb58e3cea9 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 5 Mar 2024 11:14:21 +0300 Subject: [PATCH 0956/1943] [#1025] Fill last releases changelog Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00837c79b..bc2902230 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,36 @@ Changelog for FrostFS Node ## [Unreleased] +### Added +### Changed +### Fixed +### Removed +### Updated + +## [v0.38.0-rc.2] + +### Added +- Add `trace_id` to logs in `frostfs-node` (#146) +- Allow to forcefully remove container from IR (#733) +- LOKI support (#740) +- Allow sealing writecache (#569) +- Support tree service in data evacuation (#947) +- Use new policy engine mechanism for access control (#770, #804) + +### Changed +- Sort output in `frostfs-cli` subcommands (#333) +- Send bootstrap query at each epoch tick (#721) +- Do not retain garbage in fstree on systems supporting O_TMPFILE (#970) + +### Fixed +- Handle synchronization failures better in tree service (#741) +- Fix invalid batch size for iterator traversal in morph (#1000) + +### Updated +- `neo-go` to `v0.105.0` + +## [v0.37.0] + ### Added - Support impersonate bearer token (#229) - Change log level on SIGHUP for ir (#125) From b84cf91f734627ddf3889910c162a41214af3af4 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 26 Feb 2024 19:23:54 +0300 Subject: [PATCH 0957/1943] [#1009] adm: Make workaround for get-admin and list-rule-chains * Inroduce workaround to create actor for contract storage interface without passing a real alphabet wallet. This is made by creating a dummy account. Signed-off-by: Airat Arifullin --- .../internal/modules/morph/ape/ape.go | 8 ++------ .../internal/modules/morph/ape/ape_util.go | 17 +++++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go index 957c7a15f..e3990da07 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go @@ -60,7 +60,6 @@ var ( Short: "List rule chains", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) - _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, Run: listRuleChains, } @@ -80,7 +79,6 @@ var ( Short: "Get admin", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) - _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, Run: getAdmin, } @@ -124,7 +122,6 @@ func initListRuleChainsCmd() { Cmd.AddCommand(listRuleChainsCmd) listRuleChainsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) - listRuleChainsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) listRuleChainsCmd.Flags().StringP(targetTypeFlag, "t", "", targetTypeDesc) _ = listRuleChainsCmd.MarkFlagRequired(targetTypeFlag) listRuleChainsCmd.Flags().String(targetNameFlag, "", targetNameDesc) @@ -146,7 +143,6 @@ func initGetAdminCmd() { Cmd.AddCommand(getAdminCmd) getAdminCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) - getAdminCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func addRuleChain(cmd *cobra.Command, _ []string) { @@ -173,7 +169,7 @@ func removeRuleChain(cmd *cobra.Command, _ []string) { func listRuleChains(cmd *cobra.Command, _ []string) { target := parseTarget(cmd) - pci, _ := newPolicyContractInterface(cmd) + pci, _ := newPolicyContractReaderInterface(cmd) chains, err := pci.ListMorphRuleChains(parseChainName(cmd), target) commonCmd.ExitOnErr(cmd, "list rule chains error: %w", err) if len(chains) == 0 { @@ -203,7 +199,7 @@ func setAdmin(cmd *cobra.Command, _ []string) { } func getAdmin(cmd *cobra.Command, _ []string) { - pci, _ := newPolicyContractInterface(cmd) + pci, _ := newPolicyContractReaderInterface(cmd) addr, err := pci.GetAdmin() commonCmd.ExitOnErr(cmd, "unable to get admin: %w", err) cmd.Println(addr.StringLE()) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 3a990fe17..97f748da0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -11,6 +11,7 @@ import ( apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" morph "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/cobra" @@ -83,6 +84,22 @@ func parseChainName(cmd *cobra.Command) apechain.Name { return apeChainName } +func newPolicyContractReaderInterface(cmd *cobra.Command) (*morph.ContractStorageReader, *invoker.Invoker) { + c, err := helper.GetN3Client(viper.GetViper()) + commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) + + inv := invoker.New(c, nil) + var ch util.Uint160 + r := management.NewReader(inv) + nnsCs, err := r.GetContractByID(1) + commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) + + ch, err = helper.NNSResolveHash(inv, nnsCs.Hash, helper.DomainOf(constants.PolicyContract)) + commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err) + + return morph.NewContractStorageReader(inv, ch), inv +} + func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *helper.LocalActor) { c, err := helper.GetN3Client(viper.GetViper()) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) diff --git a/go.mod b/go.mod index 0b0b03d0a..926c1b64e 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240129064140-8d21ab2d99d9 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240304151211-839f22e1a36d git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 51d68321e..2b09f5312 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b/go.mod h1:XcgrbZ88XfvhAMxmZCQJ0dv6FyRSq6Mg2J7nN8uuO0k= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240129064140-8d21ab2d99d9 h1:hkDIg6vzV1c1pyt5uj9EiBikHqKczfoNpPhwWvXaNf8= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240129064140-8d21ab2d99d9/go.mod h1:YVL7yFaT0QNSpA0z+RHudLvrLwT+lsFYGyBSVc1ustI= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240304151211-839f22e1a36d h1:y7KP7vrX5Cbx9R2nzCFj7Ljjsa/KkWZS4nQIXHEeKdw= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240304151211-839f22e1a36d/go.mod h1:YVL7yFaT0QNSpA0z+RHudLvrLwT+lsFYGyBSVc1ustI= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= From e5d18e7a85b5f96c5ad57fc2eda38275d1b44123 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 5 Mar 2024 16:01:49 +0300 Subject: [PATCH 0958/1943] [#1023] adm: Make --namespace flag required Signed-off-by: Alexander Chuprov --- cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 0c791d391..d2e45696e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -155,6 +155,7 @@ func initFrostfsIDCreateNamespaceCmd() { frostfsidCreateNamespaceCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidCreateNamespaceCmd.Flags().String(namespaceFlag, "", "Namespace name to create") frostfsidCreateNamespaceCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + _ = frostfsidCreateNamespaceCmd.MarkFlagRequired(namespaceFlag) } func initFrostfsIDListNamespacesCmd() { From 0882840bf5258233fbc5ba412eb739fcab7d548c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 6 Mar 2024 12:33:30 +0300 Subject: [PATCH 0959/1943] [#634] shard: Add writecache inhume tests Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc_test.go | 126 ++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index bd8e0ac58..a438b5def 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -7,10 +7,13 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -167,3 +170,126 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { _, err = sh.Get(context.Background(), getPrm) require.True(t, client.IsErrObjectNotFound(err), "expired complex object must be deleted on epoch after lock expires") } + +func TestGCDropsObjectInhumedFromWritecache(t *testing.T) { + t.Parallel() + + t.Run("flush write-cache before inhume", func(t *testing.T) { + t.Parallel() + testGCDropsObjectInhumedFromWritecache(t, true) + }) + + t.Run("don't flush write-cache before inhume", func(t *testing.T) { + t.Parallel() + testGCDropsObjectInhumedFromWritecache(t, false) + }) +} + +func testGCDropsObjectInhumedFromWritecache(t *testing.T, flushbeforeInhume bool) { + sh := newCustomShard(t, true, shardOptions{ + additionalShardOptions: []Option{WithDisabledGC()}, + wcOpts: []writecache.Option{writecache.WithDisableBackgroundFlush()}, + }) + defer func() { require.NoError(t, sh.Close()) }() + + obj := testutil.GenerateObjectWithSize(1024) + + var putPrm PutPrm + putPrm.SetObject(obj) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + // writecache stores object + wcObj, err := sh.writeCache.Head(context.Background(), objectCore.AddressOf(obj)) + require.NoError(t, err) + require.Equal(t, objectCore.AddressOf(obj), objectCore.AddressOf(wcObj)) + + // blobstore doesn't store object + bsRes, err := sh.blobStor.Get(context.Background(), common.GetPrm{ + Address: objectCore.AddressOf(obj), + }) + require.ErrorAs(t, err, new(*apistatus.ObjectNotFound)) + require.Nil(t, bsRes.Object) + require.Nil(t, bsRes.RawData) + + if flushbeforeInhume { + sh.writeCache.Flush(context.Background(), false, false) + } + + var inhumePrm InhumePrm + inhumePrm.MarkAsGarbage(objectCore.AddressOf(obj)) + _, err = sh.Inhume(context.Background(), inhumePrm) + require.NoError(t, err) + + // writecache doesn't store object + wcObj, err = sh.writeCache.Head(context.Background(), objectCore.AddressOf(obj)) + require.Error(t, err) + require.Nil(t, wcObj) + + if flushbeforeInhume { + // blobstore store object + bsRes, err = sh.blobStor.Get(context.Background(), common.GetPrm{ + Address: objectCore.AddressOf(obj), + }) + require.NoError(t, err) + require.Equal(t, objectCore.AddressOf(obj), objectCore.AddressOf(bsRes.Object)) + } else { + + // blobstore doesn't store object + bsRes, err = sh.blobStor.Get(context.Background(), common.GetPrm{ + Address: objectCore.AddressOf(obj), + }) + require.ErrorAs(t, err, new(*apistatus.ObjectNotFound)) + require.Nil(t, bsRes.Object) + require.Nil(t, bsRes.RawData) + } + + gcRes := sh.removeGarbage(context.Background()) + require.True(t, gcRes.success) + require.Equal(t, uint64(1), gcRes.deleted) +} + +func TestGCDontDeleteObjectFromWritecache(t *testing.T) { + sh := newCustomShard(t, true, shardOptions{ + additionalShardOptions: []Option{WithDisabledGC()}, + wcOpts: []writecache.Option{writecache.WithDisableBackgroundFlush()}, + }) + defer func() { require.NoError(t, sh.Close()) }() + + obj := testutil.GenerateObjectWithSize(1024) + + var putPrm PutPrm + putPrm.SetObject(obj) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + // writecache stores object + wcObj, err := sh.writeCache.Head(context.Background(), objectCore.AddressOf(obj)) + require.NoError(t, err) + require.Equal(t, objectCore.AddressOf(obj), objectCore.AddressOf(wcObj)) + + // blobstore doesn't store object + bsRes, err := sh.blobStor.Get(context.Background(), common.GetPrm{ + Address: objectCore.AddressOf(obj), + }) + require.ErrorAs(t, err, new(*apistatus.ObjectNotFound)) + require.Nil(t, bsRes.Object) + require.Nil(t, bsRes.RawData) + + var metaInhumePrm meta.InhumePrm + metaInhumePrm.SetAddresses(objectCore.AddressOf(obj)) + metaInhumePrm.SetLockObjectHandling() + metaInhumePrm.SetGCMark() + _, err = sh.metaBase.Inhume(context.Background(), metaInhumePrm) + require.NoError(t, err) + + // logs: WARN shard/delete.go:98 can't remove object: object must be flushed from writecache + gcRes := sh.removeGarbage(context.Background()) + require.True(t, gcRes.success) + require.Equal(t, uint64(0), gcRes.deleted) + + // writecache stores object + wcObj, err = sh.writeCache.Head(context.Background(), objectCore.AddressOf(obj)) + require.NoError(t, err) + require.Equal(t, objectCore.AddressOf(obj), objectCore.AddressOf(wcObj)) +} From 63a29110ee79dba1e16764cf1350f4d4b3f328a0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 6 Mar 2024 12:54:56 +0300 Subject: [PATCH 0960/1943] [#634] go.mod: Bump protobuf version Found by vulncheck: Vulnerability #1: GO-2024-2611 Infinite loop in JSON unmarshaling in google.golang.org/protobuf More info: https://pkg.go.dev/vuln/GO-2024-2611 Module: google.golang.org/protobuf Found in: google.golang.org/protobuf@v1.32.0 Fixed in: google.golang.org/protobuf@v1.33.0 Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 926c1b64e..6ba1ed73d 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( golang.org/x/sys v0.16.0 golang.org/x/term v0.16.0 google.golang.org/grpc v1.61.0 - google.golang.org/protobuf v1.32.0 + google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 2b09f5312..24120b20a 100644 --- a/go.sum +++ b/go.sum @@ -422,8 +422,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From dacf580b8777250523f4999c3bb6beebd4800336 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Mar 2024 17:02:23 +0300 Subject: [PATCH 0961/1943] [#973] Makefile: Up golangci-lint version Signed-off-by: Dmitrii Stepanov --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 14738272e..493ef25e5 100755 --- a/Makefile +++ b/Makefile @@ -8,8 +8,8 @@ HUB_IMAGE ?= truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.21 -LINT_VERSION ?= 1.55.2 -TRUECLOUDLAB_LINT_VERSION ?= 0.0.3 +LINT_VERSION ?= 1.56.1 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.5 PROTOC_VERSION ?= 25.0 PROTOC_GEN_GO_VERSION ?= $(shell go list -f '{{.Version}}' -m google.golang.org/protobuf) PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2) From 66a26b7775106ec0d679ca04899f4ecdac7e98de Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Mar 2024 17:11:49 +0300 Subject: [PATCH 0962/1943] [#973] node: Resolve revive: unused-parameter linter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-adm/internal/modules/morph/generate/root.go | 2 +- cmd/frostfs-adm/internal/modules/morph/policy/root.go | 2 +- cmd/frostfs-cli/internal/client/client.go | 6 +++--- cmd/frostfs-cli/modules/accounting/balance.go | 2 +- cmd/frostfs-cli/modules/accounting/root.go | 2 +- cmd/frostfs-cli/modules/container/create.go | 2 +- cmd/frostfs-cli/modules/container/delete.go | 2 +- cmd/frostfs-cli/modules/container/get.go | 2 +- cmd/frostfs-cli/modules/container/get_eacl.go | 2 +- cmd/frostfs-cli/modules/container/list.go | 2 +- cmd/frostfs-cli/modules/container/list_objects.go | 2 +- cmd/frostfs-cli/modules/container/nodes.go | 2 +- cmd/frostfs-cli/modules/container/root.go | 2 +- cmd/frostfs-cli/modules/container/set_eacl.go | 2 +- cmd/frostfs-cli/modules/control/drop_objects.go | 2 +- cmd/frostfs-cli/modules/control/root.go | 2 +- cmd/frostfs-cli/modules/netmap/get_epoch.go | 2 +- cmd/frostfs-cli/modules/netmap/netinfo.go | 2 +- cmd/frostfs-cli/modules/netmap/nodeinfo.go | 2 +- cmd/frostfs-cli/modules/netmap/root.go | 2 +- cmd/frostfs-cli/modules/netmap/snapshot.go | 2 +- cmd/frostfs-cli/modules/object/put.go | 4 ++-- cmd/frostfs-cli/modules/object/root.go | 2 +- cmd/frostfs-cli/modules/session/create.go | 2 +- cmd/frostfs-cli/modules/util/root.go | 2 +- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/netmap.go | 4 ++-- pkg/local_object_storage/blobovnicza/control.go | 2 +- pkg/local_object_storage/metabase/control.go | 2 +- pkg/local_object_storage/metabase/inhume.go | 2 +- pkg/local_object_storage/metabase/select.go | 6 +++--- pkg/local_object_storage/shard/shard.go | 2 +- pkg/local_object_storage/writecache/iterate.go | 2 +- pkg/services/notificator/nats/service.go | 4 ++-- 34 files changed, 41 insertions(+), 41 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/root.go b/cmd/frostfs-adm/internal/modules/morph/generate/root.go index 6b04848ae..3633d9a8e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/root.go @@ -32,7 +32,7 @@ var ( _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) _ = viper.BindPFlag(commonflags.RefillGasAmountFlag, cmd.Flags().Lookup(commonflags.RefillGasAmountFlag)) }, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return refillGas(cmd, commonflags.RefillGasAmountFlag, false) }, } diff --git a/cmd/frostfs-adm/internal/modules/morph/policy/root.go b/cmd/frostfs-adm/internal/modules/morph/policy/root.go index db50279dc..a8a356207 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy/root.go @@ -16,7 +16,7 @@ var ( _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, RunE: SetPolicyCmd, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + ValidArgsFunction: func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { return []string{"ExecFeeFactor=", "StoragePrice=", "FeePerByte="}, cobra.ShellCompDirectiveNoSpace }, } diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 9807e111c..f5ccabbe6 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -353,7 +353,7 @@ type PutObjectPrm struct { rdr io.Reader - headerCallback func(*objectSDK.Object) + headerCallback func() prepareLocally bool } @@ -370,7 +370,7 @@ func (x *PutObjectPrm) SetPayloadReader(rdr io.Reader) { // SetHeaderCallback sets callback which is called on the object after the header is received // but before the payload is written. -func (x *PutObjectPrm) SetHeaderCallback(f func(*objectSDK.Object)) { +func (x *PutObjectPrm) SetHeaderCallback(f func()) { x.headerCallback = f } @@ -439,7 +439,7 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { if wrt.WriteHeader(ctx, *prm.hdr) { if prm.headerCallback != nil { - prm.headerCallback(prm.hdr) + prm.headerCallback() } sz := prm.hdr.PayloadSize() diff --git a/cmd/frostfs-cli/modules/accounting/balance.go b/cmd/frostfs-cli/modules/accounting/balance.go index d10b9eaa1..1364b5e8e 100644 --- a/cmd/frostfs-cli/modules/accounting/balance.go +++ b/cmd/frostfs-cli/modules/accounting/balance.go @@ -23,7 +23,7 @@ var accountingBalanceCmd = &cobra.Command{ Use: "balance", Short: "Get internal balance of FrostFS account", Long: `Get internal balance of FrostFS account`, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { var idUser user.ID pk := key.GetOrGenerate(cmd) diff --git a/cmd/frostfs-cli/modules/accounting/root.go b/cmd/frostfs-cli/modules/accounting/root.go index 8ab8aa125..afeca7626 100644 --- a/cmd/frostfs-cli/modules/accounting/root.go +++ b/cmd/frostfs-cli/modules/accounting/root.go @@ -11,7 +11,7 @@ var Cmd = &cobra.Command{ Use: "accounting", Short: "Operations with accounts and balances", Long: `Operations with accounts and balances`, - PersistentPreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRun: func(cmd *cobra.Command, _ []string) { flags := cmd.Flags() _ = viper.BindPFlag(commonflags.WalletPath, flags.Lookup(commonflags.WalletPath)) diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index f670043ea..728332b35 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -38,7 +38,7 @@ var createContainerCmd = &cobra.Command{ Short: "Create new container", Long: `Create new container and register it in the FrostFS. It will be stored in sidechain when inner ring will accepts it.`, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { placementPolicy, err := parseContainerPolicy(cmd, containerPolicy) commonCmd.ExitOnErr(cmd, "", err) diff --git a/cmd/frostfs-cli/modules/container/delete.go b/cmd/frostfs-cli/modules/container/delete.go index 20de84f74..e5425bf25 100644 --- a/cmd/frostfs-cli/modules/container/delete.go +++ b/cmd/frostfs-cli/modules/container/delete.go @@ -20,7 +20,7 @@ var deleteContainerCmd = &cobra.Command{ Short: "Delete existing container", Long: `Delete existing container. Only owner of the container has a permission to remove container.`, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { id := parseContainerID(cmd) tok := getSession(cmd) diff --git a/cmd/frostfs-cli/modules/container/get.go b/cmd/frostfs-cli/modules/container/get.go index 1681dc7f3..8c4ab14f8 100644 --- a/cmd/frostfs-cli/modules/container/get.go +++ b/cmd/frostfs-cli/modules/container/get.go @@ -33,7 +33,7 @@ var getContainerInfoCmd = &cobra.Command{ Use: "get", Short: "Get container field info", Long: `Get container field info`, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { cnr, _ := getContainer(cmd) prettyPrintContainer(cmd, cnr, containerJSON) diff --git a/cmd/frostfs-cli/modules/container/get_eacl.go b/cmd/frostfs-cli/modules/container/get_eacl.go index 8896db769..4ed1c82e1 100644 --- a/cmd/frostfs-cli/modules/container/get_eacl.go +++ b/cmd/frostfs-cli/modules/container/get_eacl.go @@ -16,7 +16,7 @@ var getExtendedACLCmd = &cobra.Command{ Use: "get-eacl", Short: "Get extended ACL table of container", Long: `Get extended ACL table of container`, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { id := parseContainerID(cmd) pk := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 86c1e5fba..8f5db49df 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -31,7 +31,7 @@ var listContainersCmd = &cobra.Command{ Use: "list", Short: "List all created containers", Long: "List all created containers", - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { var idUser user.ID key := key.GetOrGenerate(cmd) diff --git a/cmd/frostfs-cli/modules/container/list_objects.go b/cmd/frostfs-cli/modules/container/list_objects.go index 1e0aeb4db..ff2f8cf45 100644 --- a/cmd/frostfs-cli/modules/container/list_objects.go +++ b/cmd/frostfs-cli/modules/container/list_objects.go @@ -28,7 +28,7 @@ var listContainerObjectsCmd = &cobra.Command{ Use: "list-objects", Short: "List existing objects in container", Long: `List existing objects in container`, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { id := parseContainerID(cmd) filters := new(objectSDK.SearchFilters) diff --git a/cmd/frostfs-cli/modules/container/nodes.go b/cmd/frostfs-cli/modules/container/nodes.go index 9273dd651..d785fa23d 100644 --- a/cmd/frostfs-cli/modules/container/nodes.go +++ b/cmd/frostfs-cli/modules/container/nodes.go @@ -19,7 +19,7 @@ var containerNodesCmd = &cobra.Command{ Use: "nodes", Short: "Show nodes for container", Long: "Show nodes taking part in a container at the current epoch.", - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { cnr, pkey := getContainer(cmd) if pkey == nil { diff --git a/cmd/frostfs-cli/modules/container/root.go b/cmd/frostfs-cli/modules/container/root.go index ab7f5fb90..99d1a4231 100644 --- a/cmd/frostfs-cli/modules/container/root.go +++ b/cmd/frostfs-cli/modules/container/root.go @@ -10,7 +10,7 @@ var Cmd = &cobra.Command{ Use: "container", Short: "Operations with containers", Long: "Operations with containers", - PersistentPreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRun: func(cmd *cobra.Command, _ []string) { // bind exactly that cmd's flags to // the viper before execution commonflags.Bind(cmd) diff --git a/cmd/frostfs-cli/modules/container/set_eacl.go b/cmd/frostfs-cli/modules/container/set_eacl.go index f2f60f5c3..86aa50a57 100644 --- a/cmd/frostfs-cli/modules/container/set_eacl.go +++ b/cmd/frostfs-cli/modules/container/set_eacl.go @@ -25,7 +25,7 @@ var setExtendedACLCmd = &cobra.Command{ Short: "Set new extended ACL table for container", Long: `Set new extended ACL table for container. Container ID in EACL table will be substituted with ID from the CLI.`, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { id := parseContainerID(cmd) eaclTable := common.ReadEACL(cmd, flagVarsSetEACL.srcPath) diff --git a/cmd/frostfs-cli/modules/control/drop_objects.go b/cmd/frostfs-cli/modules/control/drop_objects.go index 9de3ae8a3..8c0bb2332 100644 --- a/cmd/frostfs-cli/modules/control/drop_objects.go +++ b/cmd/frostfs-cli/modules/control/drop_objects.go @@ -14,7 +14,7 @@ var dropObjectsCmd = &cobra.Command{ Use: "drop-objects", Short: "Drop objects from the node's local storage", Long: "Drop objects from the node's local storage", - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) dropObjectsList, _ := cmd.Flags().GetStringSlice(dropObjectsFlag) diff --git a/cmd/frostfs-cli/modules/control/root.go b/cmd/frostfs-cli/modules/control/root.go index 41222fc00..b20d3618e 100644 --- a/cmd/frostfs-cli/modules/control/root.go +++ b/cmd/frostfs-cli/modules/control/root.go @@ -10,7 +10,7 @@ var Cmd = &cobra.Command{ Use: "control", Short: "Operations with storage node", Long: `Operations with storage node`, - PersistentPreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRun: func(cmd *cobra.Command, _ []string) { ff := cmd.Flags() _ = viper.BindPFlag(commonflags.WalletPath, ff.Lookup(commonflags.WalletPath)) diff --git a/cmd/frostfs-cli/modules/netmap/get_epoch.go b/cmd/frostfs-cli/modules/netmap/get_epoch.go index 1670f16c0..8e60e69bf 100644 --- a/cmd/frostfs-cli/modules/netmap/get_epoch.go +++ b/cmd/frostfs-cli/modules/netmap/get_epoch.go @@ -12,7 +12,7 @@ var getEpochCmd = &cobra.Command{ Use: "epoch", Short: "Get current epoch number", Long: "Get current epoch number", - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { p := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, p, commonflags.RPC) diff --git a/cmd/frostfs-cli/modules/netmap/netinfo.go b/cmd/frostfs-cli/modules/netmap/netinfo.go index ceea6a890..cd43ba68d 100644 --- a/cmd/frostfs-cli/modules/netmap/netinfo.go +++ b/cmd/frostfs-cli/modules/netmap/netinfo.go @@ -16,7 +16,7 @@ var netInfoCmd = &cobra.Command{ Use: "netinfo", Short: "Get information about FrostFS network", Long: "Get information about FrostFS network", - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { p := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, p, commonflags.RPC) diff --git a/cmd/frostfs-cli/modules/netmap/nodeinfo.go b/cmd/frostfs-cli/modules/netmap/nodeinfo.go index bf73286ed..b6ec48f35 100644 --- a/cmd/frostfs-cli/modules/netmap/nodeinfo.go +++ b/cmd/frostfs-cli/modules/netmap/nodeinfo.go @@ -18,7 +18,7 @@ var nodeInfoCmd = &cobra.Command{ Use: "nodeinfo", Short: "Get target node info", Long: `Get target node info`, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { p := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, p, commonflags.RPC) diff --git a/cmd/frostfs-cli/modules/netmap/root.go b/cmd/frostfs-cli/modules/netmap/root.go index aaa83f12f..b4f5897e5 100644 --- a/cmd/frostfs-cli/modules/netmap/root.go +++ b/cmd/frostfs-cli/modules/netmap/root.go @@ -9,7 +9,7 @@ var Cmd = &cobra.Command{ Use: "netmap", Short: "Operations with Network Map", Long: `Operations with Network Map`, - PersistentPreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRun: func(cmd *cobra.Command, _ []string) { // bind exactly that cmd's flags to // the viper before execution commonflags.Bind(cmd) diff --git a/cmd/frostfs-cli/modules/netmap/snapshot.go b/cmd/frostfs-cli/modules/netmap/snapshot.go index eaaf598b9..650d8a1b9 100644 --- a/cmd/frostfs-cli/modules/netmap/snapshot.go +++ b/cmd/frostfs-cli/modules/netmap/snapshot.go @@ -12,7 +12,7 @@ var snapshotCmd = &cobra.Command{ Use: "snapshot", Short: "Request current local snapshot of the network map", Long: `Request current local snapshot of the network map`, - Run: func(cmd *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, _ []string) { p := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, p, commonflags.RPC) diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index a41d3c0aa..45e02edb3 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -174,7 +174,7 @@ func setFilePayloadReader(cmd *cobra.Command, f *os.File, prm *internalclient.Pu p := pb.New64(fi.Size()) p.Output = cmd.OutOrStdout() prm.SetPayloadReader(p.NewProxyReader(f)) - prm.SetHeaderCallback(func(o *objectSDK.Object) { p.Start() }) + prm.SetHeaderCallback(func() { p.Start() }) return p } @@ -182,7 +182,7 @@ func setBinaryPayloadReader(cmd *cobra.Command, obj *objectSDK.Object, prm *inte p := pb.New(len(obj.Payload())) p.Output = cmd.OutOrStdout() prm.SetPayloadReader(p.NewProxyReader(payloadReader)) - prm.SetHeaderCallback(func(o *objectSDK.Object) { p.Start() }) + prm.SetHeaderCallback(func() { p.Start() }) return p } diff --git a/cmd/frostfs-cli/modules/object/root.go b/cmd/frostfs-cli/modules/object/root.go index e04366f4d..7d8008b10 100644 --- a/cmd/frostfs-cli/modules/object/root.go +++ b/cmd/frostfs-cli/modules/object/root.go @@ -10,7 +10,7 @@ var Cmd = &cobra.Command{ Use: "object", Short: "Operations with Objects", Long: `Operations with Objects`, - PersistentPreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRun: func(cmd *cobra.Command, _ []string) { // bind exactly that cmd's flags to // the viper before execution commonflags.Bind(cmd) diff --git a/cmd/frostfs-cli/modules/session/create.go b/cmd/frostfs-cli/modules/session/create.go index e92e4baa6..e13200a5d 100644 --- a/cmd/frostfs-cli/modules/session/create.go +++ b/cmd/frostfs-cli/modules/session/create.go @@ -29,7 +29,7 @@ var createCmd = &cobra.Command{ Use: "create", Short: "Create session token", Run: createSession, - PersistentPreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.WalletPath, cmd.Flags().Lookup(commonflags.WalletPath)) _ = viper.BindPFlag(commonflags.Account, cmd.Flags().Lookup(commonflags.Account)) }, diff --git a/cmd/frostfs-cli/modules/util/root.go b/cmd/frostfs-cli/modules/util/root.go index 3cdd236d4..4a6b4403b 100644 --- a/cmd/frostfs-cli/modules/util/root.go +++ b/cmd/frostfs-cli/modules/util/root.go @@ -9,7 +9,7 @@ import ( var Cmd = &cobra.Command{ Use: "util", Short: "Utility operations", - PersistentPreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRun: func(cmd *cobra.Command, _ []string) { flags := cmd.Flags() _ = viper.BindPFlag(commonflags.GenerateKey, flags.Lookup(commonflags.GenerateKey)) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e5d3f7311..166d36b0c 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -925,7 +925,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { ss = append(ss, blobstor.SubStorage{ Storage: fstree.New(fstreeOpts...), - Policy: func(_ *objectSDK.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, _ []byte) bool { return true }, }) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index b21e842c5..b1d854da4 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -176,7 +176,7 @@ func addNewEpochNotificationHandlers(c *cfg) { c.cfgNetmap.state.setCurrentEpoch(ev.(netmapEvent.NewEpoch).EpochNumber()) }) - addNewEpochAsyncNotificationHandler(c, func(ev event.Event) { + addNewEpochAsyncNotificationHandler(c, func(_ event.Event) { if !c.needBootstrap() || c.cfgNetmap.reBoostrapTurnedOff.Load() { // fixes #470 return } @@ -203,7 +203,7 @@ func addNewEpochNotificationHandlers(c *cfg) { }) if c.cfgMorph.notaryEnabled { - addNewEpochAsyncNotificationHandler(c, func(ev event.Event) { + addNewEpochAsyncNotificationHandler(c, func(_ event.Event) { _, err := makeNotaryDeposit(c) if err != nil { c.log.Error(logs.FrostFSNodeCouldNotMakeNotaryDeposit, diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index dfc4f884f..dab301647 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -115,7 +115,7 @@ func (b *Blobovnicza) initializeCounters() error { return nil } - return b.iterateAllDataBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) { + return b.iterateAllDataBuckets(tx, func(_, _ uint64, b *bbolt.Bucket) (bool, error) { return false, b.ForEach(func(k, v []byte) error { size += uint64(len(k) + len(v)) items++ diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index ec4c29c4b..a275f4560 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -167,7 +167,7 @@ func (db *DB) init(reset bool) error { return nil } - err = tx.ForEach(func(name []byte, b *bbolt.Bucket) error { + err = tx.ForEach(func(name []byte, _ *bbolt.Bucket) error { if _, ok := mStaticBuckets[string(name)]; !ok { return tx.DeleteBucket(name) } diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index cd83a6159..250504120 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -354,7 +354,7 @@ func isTomb(graveyardBucket *bbolt.Bucket, key []byte) (bool, error) { // iterate over graveyard and check if target address // is the address of tombstone in graveyard. - err := graveyardBucket.ForEach(func(k, v []byte) error { + err := graveyardBucket.ForEach(func(_, v []byte) error { // check if graveyard has record with key corresponding // to tombstone address (at least one) targetIsTomb = bytes.Equal(v, key) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 8b086a89f..e5321c794 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -174,7 +174,7 @@ func selectAllFromBucket(tx *bbolt.Tx, name []byte, to map[string]int, fNum int) return } - _ = bkt.ForEach(func(k, v []byte) error { + _ = bkt.ForEach(func(k, _ []byte) error { markAddressInCache(to, fNum, string(k)) return nil @@ -386,7 +386,7 @@ func (db *DB) selectFromList( return } - if err = fMatch.matchBucket(bkt, f.Header(), f.Value(), func(key, val []byte) error { + if err = fMatch.matchBucket(bkt, f.Header(), f.Value(), func(_, val []byte) error { l, err := decodeList(val) if err != nil { db.log.Debug(logs.MetabaseCantDecodeListBucketLeaf, @@ -459,7 +459,7 @@ func (db *DB) selectObjectID( return } - err := fMatch.matchBucket(bkt, f.Header(), f.Value(), func(k, v []byte) error { + err := fMatch.matchBucket(bkt, f.Header(), f.Value(), func(k, _ []byte) error { var id oid.ID if err := id.Decode(k); err == nil { appendOID(id) diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index e8f1978d2..8368f6db4 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -375,7 +375,7 @@ func WithRebuildWorkerLimiter(l RebuildWorkerLimiter) Option { // For testing purposes only. func WithDisabledGC() Option { return func(c *cfg) { - c.gcCfg.testHookRemover = func(ctx context.Context) gcRunResult { return gcRunResult{} } + c.gcCfg.testHookRemover = func(_ context.Context) gcRunResult { return gcRunResult{} } } } diff --git a/pkg/local_object_storage/writecache/iterate.go b/pkg/local_object_storage/writecache/iterate.go index 5349c069c..9ec039f91 100644 --- a/pkg/local_object_storage/writecache/iterate.go +++ b/pkg/local_object_storage/writecache/iterate.go @@ -27,7 +27,7 @@ func IterateDB(db *bbolt.DB, f func(oid.Address) error) error { var addr oid.Address - return b.ForEach(func(k, v []byte) error { + return b.ForEach(func(k, _ []byte) error { err := addr.DecodeString(string(k)) if err != nil { return fmt.Errorf("could not parse object address: %w", err) diff --git a/pkg/services/notificator/nats/service.go b/pkg/services/notificator/nats/service.go index 7f525df95..9de538f2b 100644 --- a/pkg/services/notificator/nats/service.go +++ b/pkg/services/notificator/nats/service.go @@ -93,10 +93,10 @@ func New(oo ...Option) *Writer { w.opts.nOpts = append(w.opts.nOpts, nats.NoCallbacksAfterClientClose(), // do not call callbacks when it was planned writer stop - nats.DisconnectErrHandler(func(conn *nats.Conn, err error) { + nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { w.log.Error(logs.NatsNatsConnectionWasLost, zap.Error(err)) }), - nats.ReconnectHandler(func(conn *nats.Conn) { + nats.ReconnectHandler(func(_ *nats.Conn) { w.log.Warn(logs.NatsNatsReconnectedToTheServer) }), ) From d433b492658fe50d87a6bb51fde80180fd804d47 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Mar 2024 17:55:50 +0300 Subject: [PATCH 0963/1943] [#973] node: Resolve perfsprint linter `fmt.Errorf can be replaced with errors.New` and `fmt.Sprintf can be replaced with string addition` Signed-off-by: Dmitrii Stepanov --- .../internal/modules/morph/ape/ape_util.go | 17 ++++++++++++----- .../internal/modules/morph/helper/contract.go | 2 +- .../internal/modules/morph/helper/download.go | 4 +++- .../modules/morph/helper/initialize_ctx.go | 10 ++++++++-- .../internal/modules/morph/helper/netmap.go | 4 +++- .../internal/modules/morph/notary/notary.go | 5 ++++- .../internal/modules/morph/policy/policy.go | 5 ++++- cmd/frostfs-cli/internal/client/client.go | 4 +++- cmd/frostfs-cli/modules/control/list_rules.go | 10 ++++++++-- cmd/frostfs-cli/modules/object/nodes.go | 4 +++- cmd/frostfs-cli/modules/util/ape.go | 3 ++- cmd/frostfs-ir/httpcomponent.go | 13 ++++++------- cmd/frostfs-ir/pprof.go | 5 ++--- cmd/frostfs-node/config.go | 4 ++-- cmd/frostfs-node/config/engine/shard/config.go | 4 +--- cmd/frostfs-node/config/morph/config.go | 6 ++++-- cmd/frostfs-node/httpcomponent.go | 3 +-- cmd/frostfs-node/main.go | 6 +++--- cmd/internal/common/config/viper.go | 5 ++++- internal/ape/converter.go | 2 +- pkg/ape/chainbase/boltdb.go | 4 +++- .../processors/container/process_container.go | 5 ++++- .../blobstor/blobovniczatree/blobovnicza.go | 3 +-- pkg/local_object_storage/metabase/counter.go | 9 +++++++-- pkg/local_object_storage/metabase/delete.go | 4 +++- pkg/local_object_storage/metabase/util.go | 6 ++++-- pkg/local_object_storage/pilorama/forest.go | 5 ++++- pkg/morph/client/netmap/peer.go | 5 ++++- pkg/network/address.go | 5 ++++- pkg/services/container/ape.go | 6 ++++-- pkg/services/container/morph/executor.go | 4 +++- pkg/services/control/ir/server/calls.go | 6 +++--- pkg/services/control/server/convert.go | 3 +-- pkg/services/control/server/evacuate.go | 4 +++- pkg/services/object/ape/checker.go | 3 ++- pkg/services/object/ape/errors.go | 4 +--- pkg/services/object/ape/service.go | 5 ++++- pkg/services/object/put/single.go | 4 +++- pkg/util/sdnotify/sdnotify.go | 14 ++++++++++---- 39 files changed, 143 insertions(+), 72 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 97f748da0..253e130a8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -1,7 +1,7 @@ package ape import ( - "fmt" + "errors" "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" @@ -28,6 +28,13 @@ var mChainName = map[string]apechain.Name{ s3: apechain.S3, } +var ( + errUnknownTargetType = errors.New("unknown target type") + errChainIDCannotBeEmpty = errors.New("chain id cannot be empty") + errRuleIsNotParsed = errors.New("rule is not passed") + errUnsupportedChainName = errors.New("unsupported chain name") +) + func parseTarget(cmd *cobra.Command) policyengine.Target { var targetType policyengine.TargetType typ, _ := cmd.Flags().GetString(targetTypeFlag) @@ -37,7 +44,7 @@ func parseTarget(cmd *cobra.Command) policyengine.Target { case containerTarget: targetType = policyengine.Container default: - commonCmd.ExitOnErr(cmd, "read target type error: %w", fmt.Errorf("unknown target type")) + commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType) } name, _ := cmd.Flags().GetString(targetNameFlag) @@ -51,7 +58,7 @@ func parseChainID(cmd *cobra.Command) apechain.ID { chainID, _ := cmd.Flags().GetString(chainIDFlag) if chainID == "" { commonCmd.ExitOnErr(cmd, "read chain id error: %w", - fmt.Errorf("chain id cannot be empty")) + errChainIDCannotBeEmpty) } return apechain.ID(chainID) } @@ -64,7 +71,7 @@ func parseChain(cmd *cobra.Command) *apechain.Chain { } else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" { commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", parseutil.ParseAPEChainBinaryOrJSON(chain, encPath)) } else { - commonCmd.ExitOnErr(cmd, "parser error: %w", fmt.Errorf("rule is not passed")) + commonCmd.ExitOnErr(cmd, "parser error: %w", errRuleIsNotParsed) } chain.ID = parseChainID(cmd) @@ -79,7 +86,7 @@ func parseChainName(cmd *cobra.Command) apechain.Name { chainName, _ := cmd.Flags().GetString(chainNameFlag) apeChainName, ok := mChainName[strings.ToLower(chainName)] if !ok { - commonCmd.ExitOnErr(cmd, "", fmt.Errorf("unsupported chain name")) + commonCmd.ExitOnErr(cmd, "", errUnsupportedChainName) } return apeChainName } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/contract.go b/cmd/frostfs-adm/internal/modules/morph/helper/contract.go index a740f3f09..7326ec1f5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/contract.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/contract.go @@ -116,7 +116,7 @@ func GetContractDeployData(c *InitializeContext, ctrName string, keysParam []any case constants.PolicyContract: items = append(items, c.Contracts[constants.ProxyContract].Hash) default: - panic(fmt.Sprintf("invalid contract name: %s", ctrName)) + panic("invalid contract name: " + ctrName) } return items, nil } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/download.go b/cmd/frostfs-adm/internal/modules/morph/helper/download.go index 08d427a46..71528a5db 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/download.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/download.go @@ -14,6 +14,8 @@ import ( "github.com/spf13/cobra" ) +var errNoReleasesFound = errors.New("attempt to fetch contracts archive from the offitial repository failed: no releases found") + func downloadContracts(cmd *cobra.Command, url string) (io.ReadCloser, error) { cmd.Printf("Downloading contracts archive from '%s'\n", url) @@ -61,7 +63,7 @@ func downloadContractsFromRepository(cmd *cobra.Command) (io.ReadCloser, error) } if latestRelease == nil { - return nil, fmt.Errorf("attempt to fetch contracts archive from the offitial repository failed: no releases found") + return nil, errNoReleasesFound } cmd.Printf("Found release %s (%s)\n", latestRelease.TagName, latestRelease.Title) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go index 09d64b352..b5b6adf05 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go @@ -3,6 +3,7 @@ package helper import ( "encoding/hex" "encoding/json" + "errors" "fmt" io2 "io" "os" @@ -33,6 +34,11 @@ import ( "github.com/spf13/viper" ) +var ( + errNegativeDuration = errors.New("epoch duration must be positive") + errNegativeSize = errors.New("max object size must be positive") +) + type ContractState struct { NEF *nef.File RawNEF []byte @@ -166,11 +172,11 @@ func validateInit(cmd *cobra.Command) error { return nil } if viper.GetInt64(commonflags.EpochDurationInitFlag) <= 0 { - return fmt.Errorf("epoch duration must be positive") + return errNegativeDuration } if viper.GetInt64(commonflags.MaxObjectSizeInitFlag) <= 0 { - return fmt.Errorf("max object size must be positive") + return errNegativeSize } return nil diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go index 732a5e11f..f68fbefaf 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go @@ -29,6 +29,8 @@ var NetmapConfigKeys = []string{ netmap.MaintenanceModeAllowedConfig, } +var errFailedToFetchListOfNetworkKeys = errors.New("can't fetch list of network config keys from the netmap contract") + func GetDefaultNetmapContractConfigMap() map[string]any { m := make(map[string]any) m[netmap.EpochDurationConfig] = viper.GetInt64(commonflags.EpochDurationInitFlag) @@ -95,7 +97,7 @@ func GetNetConfigFromNetmapContract(roInvoker *invoker.Invoker) ([]stackitem.Ite } arr, err := unwrap.Array(roInvoker.Call(nmHash, "listConfig")) if err != nil { - return nil, fmt.Errorf("can't fetch list of network config keys from the netmap contract") + return nil, errFailedToFetchListOfNetworkKeys } return arr, err } diff --git a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go index 6d65a8284..9b213da4e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go @@ -1,6 +1,7 @@ package notary import ( + "errors" "fmt" "math/big" "strconv" @@ -31,6 +32,8 @@ const ( notaryDepositTillFlag = "till" ) +var errInvalidNotaryDepositLifetime = errors.New("notary deposit lifetime must be a positive integer") + func depositNotary(cmd *cobra.Command, _ []string) error { w, err := openWallet(cmd) if err != nil { @@ -78,7 +81,7 @@ func depositNotary(cmd *cobra.Command, _ []string) error { if tillStr != "" { till, err = strconv.ParseInt(tillStr, 10, 64) if err != nil || till <= 0 { - return fmt.Errorf("notary deposit lifetime must be a positive integer") + return errInvalidNotaryDepositLifetime } } diff --git a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go index 440e73a12..a11ca2704 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go @@ -2,6 +2,7 @@ package policy import ( "bytes" + "errors" "fmt" "strconv" "strings" @@ -24,6 +25,8 @@ const ( setFeeParam = "FeePerByte" ) +var errInvalidParameterFormat = errors.New("invalid parameter format, must be Parameter=Value") + func SetPolicyCmd(cmd *cobra.Command, args []string) error { wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { @@ -34,7 +37,7 @@ func SetPolicyCmd(cmd *cobra.Command, args []string) error { for i := range args { k, v, found := strings.Cut(args[i], "=") if !found { - return fmt.Errorf("invalid parameter format, must be Parameter=Value") + return errInvalidParameterFormat } switch k { diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index f5ccabbe6..a6d9968c5 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -21,6 +21,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" ) +var errMissingHeaderInResponse = errors.New("missing header in response") + // BalanceOfPrm groups parameters of BalanceOf operation. type BalanceOfPrm struct { commonPrm @@ -654,7 +656,7 @@ func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) var hdr objectSDK.Object if !res.ReadHeader(&hdr) { - return nil, fmt.Errorf("missing header in response") + return nil, errMissingHeaderInResponse } return &HeadObjectRes{ diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index 4598320d7..b345ecbfb 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -1,6 +1,7 @@ package control import ( + "errors" "fmt" "strings" @@ -35,6 +36,11 @@ const ( targetTypeDesc = "Resource type(container/namespace)" ) +var ( + errSettingDefaultValueWasDeclined = errors.New("setting default value was declined") + errUnknownTargetType = errors.New("unknown target type") +) + func parseTarget(cmd *cobra.Command) *control.ChainTarget { typ, _ := cmd.Flags().GetString(targetTypeFlag) name, _ := cmd.Flags().GetString(targetNameFlag) @@ -45,7 +51,7 @@ func parseTarget(cmd *cobra.Command) *control.ChainTarget { commonCmd.ExitOnErr(cmd, "read line error: %w", err) ln = strings.ToLower(ln) if len(ln) > 0 && (ln[0] == 'n') { - commonCmd.ExitOnErr(cmd, "read namespace error: %w", fmt.Errorf("setting default value was declined")) + commonCmd.ExitOnErr(cmd, "read namespace error: %w", errSettingDefaultValueWasDeclined) } name = defaultNamespace } @@ -61,7 +67,7 @@ func parseTarget(cmd *cobra.Command) *control.ChainTarget { Type: control.ChainTarget_CONTAINER, } default: - commonCmd.ExitOnErr(cmd, "read target type error: %w", fmt.Errorf("unknown target type")) + commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType) } return nil } diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 70a3eb75f..50c47e822 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -30,6 +30,8 @@ const ( verifyPresenceAllFlag = "verify-presence-all" ) +var errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint") + type objectNodesInfo struct { containerID cid.ID objectID oid.ID @@ -299,7 +301,7 @@ func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.N return nil, lastErr } if cli == nil { - return nil, fmt.Errorf("failed to create client: no available endpoint") + return nil, errNoAvailableEndpoint } return cli, nil } diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index 1500f0c5c..d2dd0ced2 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -24,6 +24,7 @@ var ( errMixedTypesInRule = errors.New("found mixed type of actions and conditions in rule") errNoActionsInRule = errors.New("there are no actions in rule") errUnsupportedResourceFormat = errors.New("unsupported resource format") + errFailedToParseAllAny = errors.New("any/all is not parsed") ) // PrintHumanReadableAPEChain print APE chain rules. @@ -188,7 +189,7 @@ func parseAnyAll(lexeme string) (bool, error) { case "all": return false, nil default: - return false, fmt.Errorf("any/all is not parsed") + return false, errFailedToParseAllAny } } diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index 3a6d77d84..2792c3548 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net/http" "time" @@ -26,7 +25,7 @@ const ( ) func (c *httpComponent) init() { - log.Info(fmt.Sprintf("init %s", c.name)) + log.Info("init " + c.name) c.enabled = cfg.GetBool(c.name + enabledKeyPostfix) c.address = cfg.GetString(c.name + addressKeyPostfix) c.shutdownDur = cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) @@ -40,14 +39,14 @@ func (c *httpComponent) init() { httputil.WithShutdownTimeout(c.shutdownDur), ) } else { - log.Info(fmt.Sprintf("%s is disabled, skip", c.name)) + log.Info(c.name + " is disabled, skip") c.srv = nil } } func (c *httpComponent) start() { if c.srv != nil { - log.Info(fmt.Sprintf("start %s", c.name)) + log.Info("start " + c.name) wg.Add(1) go func() { defer wg.Done() @@ -58,7 +57,7 @@ func (c *httpComponent) start() { func (c *httpComponent) shutdown() error { if c.srv != nil { - log.Info(fmt.Sprintf("shutdown %s", c.name)) + log.Info("shutdown " + c.name) return c.srv.Shutdown() } return nil @@ -72,9 +71,9 @@ func (c *httpComponent) needReload() bool { } func (c *httpComponent) reload() { - log.Info(fmt.Sprintf("reload %s", c.name)) + log.Info("reload " + c.name) if c.needReload() { - log.Info(fmt.Sprintf("%s config updated", c.name)) + log.Info(c.name + " config updated") if err := c.shutdown(); err != nil { log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error()), diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go index d67c463fc..ff5642008 100644 --- a/cmd/frostfs-ir/pprof.go +++ b/cmd/frostfs-ir/pprof.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "runtime" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -53,9 +52,9 @@ func (c *pprofComponent) needReload() bool { } func (c *pprofComponent) reload() { - log.Info(fmt.Sprintf("reload %s", c.name)) + log.Info("reload " + c.name) if c.needReload() { - log.Info(fmt.Sprintf("%s config updated", c.name)) + log.Info(c.name + " config updated") if err := c.shutdown(); err != nil { log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error())) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 166d36b0c..48b76ac0f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1004,12 +1004,12 @@ func (c *cfg) loggerPrm() (*logger.Prm, error) { err := c.dynamicConfiguration.logger.SetLevelString(c.LoggerCfg.level) if err != nil { // not expected since validation should be performed before - panic(fmt.Sprintf("incorrect log level format: %s", c.LoggerCfg.level)) + panic("incorrect log level format: " + c.LoggerCfg.level) } err = c.dynamicConfiguration.logger.SetDestination(c.LoggerCfg.destination) if err != nil { // not expected since validation should be performed before - panic(fmt.Sprintf("incorrect log destination format: %s", c.LoggerCfg.destination)) + panic("incorrect log destination format: " + c.LoggerCfg.destination) } return c.dynamicConfiguration.logger, nil diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go index 16100c3a7..f9cfdf2a8 100644 --- a/cmd/frostfs-node/config/engine/shard/config.go +++ b/cmd/frostfs-node/config/engine/shard/config.go @@ -1,8 +1,6 @@ package shardconfig import ( - "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" blobstorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor" gcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/gc" @@ -158,7 +156,7 @@ func (x *Config) Mode() (m mode.Mode) { case "disabled": m = mode.Disabled default: - panic(fmt.Sprintf("unknown shard mode: %s", s)) + panic("unknown shard mode: " + s) } return diff --git a/cmd/frostfs-node/config/morph/config.go b/cmd/frostfs-node/config/morph/config.go index 4ab608ef3..866223073 100644 --- a/cmd/frostfs-node/config/morph/config.go +++ b/cmd/frostfs-node/config/morph/config.go @@ -1,7 +1,7 @@ package morphconfig import ( - "fmt" + "errors" "strconv" "time" @@ -27,6 +27,8 @@ const ( SwitchIntervalDefault = 2 * time.Minute ) +var errNoMorphEndpoints = errors.New("no morph chain RPC endpoints, see `morph.rpc_endpoint` section") + // RPCEndpoint returns list of the values of "rpc_endpoint" config parameter // from "morph" section. // @@ -54,7 +56,7 @@ func RPCEndpoint(c *config.Config) []client.Endpoint { } if len(es) == 0 { - panic(fmt.Errorf("no morph chain RPC endpoints, see `morph.rpc_endpoint` section")) + panic(errNoMorphEndpoints) } return es } diff --git a/cmd/frostfs-node/httpcomponent.go b/cmd/frostfs-node/httpcomponent.go index 373a5e5c8..2ec20d848 100644 --- a/cmd/frostfs-node/httpcomponent.go +++ b/cmd/frostfs-node/httpcomponent.go @@ -2,7 +2,6 @@ package main import ( "context" - "fmt" "net/http" "time" @@ -23,7 +22,7 @@ type httpComponent struct { func (cmp *httpComponent) init(c *cfg) { if !cmp.enabled { - c.log.Info(fmt.Sprintf("%s is disabled", cmp.name)) + c.log.Info(cmp.name + " is disabled") return } // Init server with parameters diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index f91a6decc..e1eeb6861 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -75,7 +75,7 @@ func main() { func initAndLog(c *cfg, name string, initializer func(*cfg)) { c.log.Info(fmt.Sprintf("initializing %s service...", name)) initializer(c) - c.log.Info(fmt.Sprintf("%s service has been successfully initialized", name)) + c.log.Info(name + " service has been successfully initialized") } func initApp(ctx context.Context, c *cfg) { @@ -124,7 +124,7 @@ func runAndLog(ctx context.Context, c *cfg, name string, logSuccess bool, starte starter(ctx, c) if logSuccess { - c.log.Info(fmt.Sprintf("%s service started successfully", name)) + c.log.Info(name + " service started successfully") } } @@ -138,7 +138,7 @@ func stopAndLog(c *cfg, name string, stopper func() error) { ) } - c.log.Debug(fmt.Sprintf("%s service has been stopped", name)) + c.log.Debug(name + " service has been stopped") } func bootUp(ctx context.Context, c *cfg) { diff --git a/cmd/internal/common/config/viper.go b/cmd/internal/common/config/viper.go index 41b8831ff..f06d407c1 100644 --- a/cmd/internal/common/config/viper.go +++ b/cmd/internal/common/config/viper.go @@ -1,6 +1,7 @@ package config import ( + "errors" "fmt" "strings" @@ -15,6 +16,8 @@ const ( EnvSeparator = "_" ) +var errProvideViperInOpts = errors.New("provide viper in opts") + func CreateViper(opts ...Option) (*viper.Viper, error) { o := defaultOpts() for i := range opts { @@ -59,7 +62,7 @@ func ReloadViper(opts ...Option) error { } if o.v == nil { - return fmt.Errorf("provide viper in opts") + return errProvideViperInOpts } if o.path != "" { diff --git a/internal/ape/converter.go b/internal/ape/converter.go index f629e262d..2adc7645a 100644 --- a/internal/ape/converter.go +++ b/internal/ape/converter.go @@ -18,7 +18,7 @@ func (e *ConvertEACLError) Error() string { if e == nil { return "" } - return fmt.Sprintf("failed to convert eACL table to policy engine chain: %s", e.nested.Error()) + return "failed to convert eACL table to policy engine chain: " + e.nested.Error() } func (e *ConvertEACLError) Unwrap() error { diff --git a/pkg/ape/chainbase/boltdb.go b/pkg/ape/chainbase/boltdb.go index a1dfc2567..27bfd8468 100644 --- a/pkg/ape/chainbase/boltdb.go +++ b/pkg/ape/chainbase/boltdb.go @@ -34,6 +34,8 @@ var ( ErrTargetNameBucketNotFound = logicerr.New("target name bucket not found") ErrBucketNotContainsChainID = logicerr.New("chain id not found in bucket") + + errChainIDIsNotSet = errors.New("chain ID is not set") ) // NewBoltLocalOverrideDatabase returns storage wrapper for storing access policy engine @@ -167,7 +169,7 @@ func getTargetBucketCreateIfEmpty(tx *bbolt.Tx, name chain.Name, target policyen func (cs *boltLocalOverrideStorage) AddOverride(name chain.Name, target policyengine.Target, c *chain.Chain) (chain.ID, error) { if len(c.ID) == 0 { - return chain.ID{}, fmt.Errorf("chain ID is not set") + return chain.ID{}, errChainIDIsNotSet } serializedChain := c.Bytes() diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 2ad863b27..a950997fd 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -1,6 +1,7 @@ package container import ( + "errors" "fmt" "strings" @@ -31,6 +32,8 @@ type putContainerContext struct { d containerSDK.Domain } +var errContainerAndOwnerNamespaceDontMatch = errors.New("container and owner namespaces do not match") + // Process a new container from the user by checking the container sanity // and sending approve tx back to the morph. func (cp *Processor) processContainerPut(put putEvent) bool { @@ -193,7 +196,7 @@ func (cp *Processor) checkNNS(ctx *putContainerContext, cnr containerSDK.Contain } if subject.Namespace != namespace { - return fmt.Errorf("container and owner namespaces do not match") + return errContainerAndOwnerNamespaceDontMatch } return nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 9f2950094..be51509b6 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -2,7 +2,6 @@ package blobovniczatree import ( "errors" - "fmt" "strconv" "strings" "sync" @@ -117,7 +116,7 @@ func u64ToHexStringExt(ind uint64) string { func u64FromHexString(str string) uint64 { v, err := strconv.ParseUint(strings.TrimSuffix(str, dbExtension), 16, 64) if err != nil { - panic(fmt.Sprintf("blobovnicza name is not an index %s", str)) + panic("blobovnicza name is not an index " + str) } return v diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index ee1ab41de..dc85b4697 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -24,6 +24,11 @@ var ( objectUserCounterKey = []byte("user_counter") ) +var ( + errInvalidKeyLenght = errors.New("invalid key length") + errInvalidValueLenght = errors.New("invalid value length") +) + type objectType uint8 const ( @@ -474,7 +479,7 @@ func containerCounterValue(entity ObjectCounters) []byte { func parseContainerCounterKey(buf []byte) (cid.ID, error) { if len(buf) != cidSize { - return cid.ID{}, fmt.Errorf("invalid key length") + return cid.ID{}, errInvalidKeyLenght } var cnrID cid.ID if err := cnrID.Decode(buf); err != nil { @@ -486,7 +491,7 @@ func parseContainerCounterKey(buf []byte) (cid.ID, error) { // parseContainerCounterValue return phy, logic values. func parseContainerCounterValue(buf []byte) (ObjectCounters, error) { if len(buf) != 24 { - return ObjectCounters{}, fmt.Errorf("invalid value length") + return ObjectCounters{}, errInvalidValueLenght } return ObjectCounters{ Phy: binary.LittleEndian.Uint64(buf), diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index e5a98ff01..e591b8661 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -20,6 +20,8 @@ import ( "go.opentelemetry.io/otel/trace" ) +var errFailedToRemoveUniqueIndexes = errors.New("can't remove unique indexes") + // DeletePrm groups the parameters of Delete operation. type DeletePrm struct { addrs []oid.Address @@ -325,7 +327,7 @@ func (db *DB) deleteObject( ) error { err := delUniqueIndexes(tx, obj, isParent) if err != nil { - return fmt.Errorf("can't remove unique indexes") + return errFailedToRemoveUniqueIndexes } err = updateListIndexes(tx, obj, delListIndexItem) diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index c04aff61c..d46a421a3 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -3,7 +3,7 @@ package meta import ( "bytes" "crypto/sha256" - "fmt" + "errors" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -25,6 +25,8 @@ var ( containerCounterBucketName = []byte{containerCountersPrefix} zeroValue = []byte{0xFF} + + errInvalidLength = errors.New("invalid length") ) // Prefix bytes for database keys. All ids and addresses are encoded in binary @@ -198,7 +200,7 @@ func addressKey(addr oid.Address, key []byte) []byte { // parses object address formed by addressKey. func decodeAddressFromKey(dst *oid.Address, k []byte) error { if len(k) != addressKeySize { - return fmt.Errorf("invalid length") + return errInvalidLength } var cnr cid.ID diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 8276490b3..dbfc9dccd 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -2,6 +2,7 @@ package pilorama import ( "context" + "errors" "fmt" "sort" "strings" @@ -10,6 +11,8 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) +var errInvalidKeyFormat = errors.New("invalid format: key must be cid and treeID") + // memoryForest represents multiple replicating trees sharing a single storage. type memoryForest struct { // treeMap maps tree identifier (container ID + name) to the replicated log. @@ -291,7 +294,7 @@ func (f *memoryForest) TreeListTrees(_ context.Context, prm TreeListTreesPrm) (* for idx < len(tmpSlice) { cidAndTree := strings.Split(tmpSlice[idx], "/") if len(cidAndTree) != 2 { - return nil, fmt.Errorf("invalid format: key must be cid and treeID") + return nil, errInvalidKeyFormat } var contID cid.ID if err := contID.DecodeString(cidAndTree[0]); err != nil { diff --git a/pkg/morph/client/netmap/peer.go b/pkg/morph/client/netmap/peer.go index 30f51f699..764bbc899 100644 --- a/pkg/morph/client/netmap/peer.go +++ b/pkg/morph/client/netmap/peer.go @@ -1,12 +1,15 @@ package netmap import ( + "errors" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) +var errFailedToRemovePeerWithoutNotary = errors.New("peer can be forcefully removed only in notary environment") + // AddPeerPrm groups parameters of AddPeer operation. type AddPeerPrm struct { nodeInfo netmap.NodeInfo @@ -46,7 +49,7 @@ func (c *Client) AddPeer(p AddPeerPrm) error { // If vub > 0, vub will be used as valid until block value. func (c *Client) ForceRemovePeer(nodeInfo netmap.NodeInfo, vub uint32) (uint32, error) { if !c.client.WithNotary() { - return 0, fmt.Errorf("peer can be forcefully removed only in notary environment") + return 0, errFailedToRemovePeerWithoutNotary } prm := UpdatePeerPrm{} diff --git a/pkg/network/address.go b/pkg/network/address.go index aeb647885..88f4a571d 100644 --- a/pkg/network/address.go +++ b/pkg/network/address.go @@ -1,6 +1,7 @@ package network import ( + "errors" "fmt" "net" "net/url" @@ -18,6 +19,8 @@ import ( URIAddr strings: "127.0.0.1:8080" */ +var errHostIsEmpty = errors.New("host is empty") + // Address represents the FrostFS node // network address. type Address struct { @@ -89,7 +92,7 @@ func (a *Address) FromString(s string) error { // multiaddrStringFromHostAddr converts "localhost:8080" to "/dns4/localhost/tcp/8080". func multiaddrStringFromHostAddr(host string) (string, error) { if len(host) == 0 { - return "", fmt.Errorf("host is empty") + return "", errHostIsEmpty } endpoint, port, err := net.SplitHostPort(host) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index c01ad8a1d..83361257a 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -41,6 +41,8 @@ var ( errEmptyBodySignature = errors.New("malformed request: empty body signature") errMissingOwnerID = errors.New("malformed request: missing owner ID") errSubjectNotFound = errors.New("subject not found") + errOwnerIDIsNotSet = errors.New("owner id is not set") + errInvalidDomainZone = errors.New("invalid domain zone: no namespace is expected") undefinedContainerID = cid.ID{} ) @@ -566,7 +568,7 @@ func isContainerNode(nm *netmapSDK.NetMap, pk, binCnrID []byte, cont *containerc func (ac *apeChecker) namespaceByOwner(owner *refs.OwnerID) (string, error) { var ownerSDK user.ID if owner == nil { - return "", fmt.Errorf("owner id is not set") + return "", errOwnerIDIsNotSet } if err := ownerSDK.ReadFromV2(*owner); err != nil { return "", err @@ -603,7 +605,7 @@ func validateNamespace(cnrV2 *container.Container, ownerIDNamespace string) erro if hasNamespace { if cntNamespace != ownerIDNamespace { if ownerIDNamespace == "" { - return fmt.Errorf("invalid domain zone: no namespace is expected") + return errInvalidDomainZone } return fmt.Errorf("invalid domain zone: expected namespace %s, but got %s", ownerIDNamespace, cntNamespace) } diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index f1217705e..a653fa066 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -16,6 +16,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) +var errMissingUserID = errors.New("missing user ID") + type morphExecutor struct { rdr Reader wrt Writer @@ -177,7 +179,7 @@ func (s *morphExecutor) Get(_ context.Context, body *container.GetRequestBody) ( func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) (*container.ListResponseBody, error) { idV2 := body.GetOwnerID() if idV2 == nil { - return nil, fmt.Errorf("missing user ID") + return nil, errMissingUserID } var id user.ID diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 58610b0d1..2447a8a74 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -121,7 +121,7 @@ func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainer if len(req.GetBody().GetContainerId()) > 0 { var containerID cid.ID if err := containerID.Decode(req.GetBody().GetContainerId()); err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed to parse container ID: %s", err.Error())) + return nil, status.Error(codes.InvalidArgument, "failed to parse container ID: "+err.Error()) } var err error vub, err = s.removeContainer(containerID, req.GetBody().GetVub()) @@ -131,11 +131,11 @@ func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainer } else { var ownerID refs.OwnerID if err := ownerID.Unmarshal(req.GetBody().GetOwner()); err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed to parse ownerID: %s", err.Error())) + return nil, status.Error(codes.InvalidArgument, "failed to parse ownerID: %s"+err.Error()) } var owner user.ID if err := owner.ReadFromV2(ownerID); err != nil { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed to read owner: %s", err.Error())) + return nil, status.Error(codes.InvalidArgument, "failed to read owner: "+err.Error()) } cids, err := s.containerClient.ContainersOf(&owner) diff --git a/pkg/services/control/server/convert.go b/pkg/services/control/server/convert.go index 4f3132356..61d7e41c1 100644 --- a/pkg/services/control/server/convert.go +++ b/pkg/services/control/server/convert.go @@ -1,7 +1,6 @@ package control import ( - "fmt" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" @@ -16,7 +15,7 @@ func stateToResponse(state *engine.EvacuationState) (*control.GetShardEvacuation for _, shID := range state.ShardIDs() { id, err := base58.Decode(shID) if err != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("invalid shard id format: %s", shID)) + return nil, status.Error(codes.Internal, "invalid shard id format: "+shID) } shardIDs = append(shardIDs, id) } diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index 99082b30d..462ab211e 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -23,6 +23,8 @@ import ( "google.golang.org/grpc/status" ) +var errFailedToBuildListOfContainerNodes = errors.New("can't build a list of container nodes") + func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRequest) (*control.EvacuateShardResponse, error) { err := s.isValidRequest(req) if err != nil { @@ -157,7 +159,7 @@ func (s *Server) getContainerNodes(contID cid.ID) ([]netmap.NodeInfo, error) { ns, err := nm.ContainerNodes(c.Value.PlacementPolicy(), binCnr) if err != nil { - return nil, fmt.Errorf("can't build a list of container nodes") + return nil, errFailedToBuildListOfContainerNodes } nodes := placement.FlattenNodes(ns) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 13b2729e9..13dab0ec7 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -2,6 +2,7 @@ package ape import ( "context" + "errors" "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" @@ -50,7 +51,7 @@ type Prm struct { SoftAPECheck bool } -var errMissingOID = fmt.Errorf("object ID is not set") +var errMissingOID = errors.New("object ID is not set") // CheckAPE checks if a request or a response is permitted creating an ape request and passing // it to chain router. diff --git a/pkg/services/object/ape/errors.go b/pkg/services/object/ape/errors.go index 7c2a82dfd..1b2024ed5 100644 --- a/pkg/services/object/ape/errors.go +++ b/pkg/services/object/ape/errors.go @@ -1,13 +1,11 @@ package ape import ( - "fmt" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) func toStatusErr(err error) error { errAccessDenied := &apistatus.ObjectAccessDenied{} - errAccessDenied.WriteReason(fmt.Sprintf("ape denied request: %s", err.Error())) + errAccessDenied.WriteReason("ape denied request: " + err.Error()) return errAccessDenied } diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index f05bdaa83..cc18a0bcf 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -3,6 +3,7 @@ package ape import ( "context" "encoding/hex" + "errors" "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" @@ -16,6 +17,8 @@ import ( nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" ) +var errFailedToCastToRequestContext = errors.New("failed cast to RequestContext") + type Service struct { log *logger.Logger @@ -101,7 +104,7 @@ func requestContext(ctx context.Context) (*objectSvc.RequestContext, error) { } rc, ok := untyped.(*objectSvc.RequestContext) if !ok { - return nil, fmt.Errorf("failed cast to RequestContext") + return nil, errFailedToCastToRequestContext } return rc, nil } diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 183b2310d..09e1eb092 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -32,6 +32,8 @@ import ( "go.uber.org/zap" ) +var errInvalidPayloadChecksum = errors.New("incorrect payload checksum") + type putSingleRequestSigner struct { req *objectAPI.PutSingleRequest keyStorage *svcutil.KeyStorage @@ -125,7 +127,7 @@ func (s *Service) validatePutSingleChecksum(obj *objectSDK.Object) error { } if !bytes.Equal(hash.Sum(nil), cs.Value()) { - return fmt.Errorf("incorrect payload checksum") + return errInvalidPayloadChecksum } return nil diff --git a/pkg/util/sdnotify/sdnotify.go b/pkg/util/sdnotify/sdnotify.go index 7c3d2f1d2..fe4ad1c3e 100644 --- a/pkg/util/sdnotify/sdnotify.go +++ b/pkg/util/sdnotify/sdnotify.go @@ -1,6 +1,7 @@ package sdnotify import ( + "errors" "fmt" "net" "os" @@ -13,14 +14,19 @@ const ( ReloadingEnabled = "RELOADING=1" ) -var socket *net.UnixAddr +var ( + socket *net.UnixAddr + + errSocketVariableIsNotPresent = errors.New("\"NOTIFY_SOCKET\" environment variable is not present") + errSocketIsNotInitialized = errors.New("socket is not initialized") +) // Initializes socket with provided name of // environment variable. func InitSocket() error { notifySocket := os.Getenv("NOTIFY_SOCKET") if notifySocket == "" { - return fmt.Errorf("\"NOTIFY_SOCKET\" environment variable is not present") + return errSocketVariableIsNotPresent } socket = &net.UnixAddr{ Name: notifySocket, @@ -38,14 +44,14 @@ func FlagAndStatus(status string) error { // Status sends systemd notify STATUS=%s{status}. func Status(status string) error { - return Send(fmt.Sprintf("STATUS=%s", status)) + return Send("STATUS=" + status) } // Send state through the notify socket if any. // If the notify socket was not detected, it returns an error. func Send(state string) error { if socket == nil { - return fmt.Errorf("socket is not initialized") + return errSocketIsNotInitialized } conn, err := net.DialUnix(socket.Net, nil, socket) if err != nil { From 3195142d671a38ba4b4080f4369fbb4b66f6bb26 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 20 Feb 2024 17:24:57 +0300 Subject: [PATCH 0964/1943] [#959] writecache: Avoid manipulation with cache in `DEGRADED` mode Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/writecache/delete.go | 4 ++++ pkg/local_object_storage/writecache/flush.go | 5 ++++- pkg/local_object_storage/writecache/get.go | 16 ++++++++++++++++ pkg/local_object_storage/writecache/mode.go | 6 ++++++ pkg/local_object_storage/writecache/put.go | 3 +++ .../writecache/writecache.go | 2 ++ 6 files changed, 35 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index cedb9f4ae..b1a0511ee 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -19,6 +19,7 @@ import ( // // Returns an error of type apistatus.ObjectNotFound if object is missing in write-cache. // Returns ErrNotInitialized if write-cache has not been initialized yet. +// Returns ErrDegraded if write-cache is in DEGRADED mode. func (c *cache) Delete(ctx context.Context, addr oid.Address) error { ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Delete", trace.WithAttributes( @@ -40,6 +41,9 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { if c.readOnly() { return ErrReadOnly } + if c.noMetabase() { + return ErrDegraded + } saddr := addr.EncodeToString() diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index c4e4b3b34..c3e19c4d2 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -155,7 +155,7 @@ func (c *cache) workerFlushBig(ctx context.Context) { select { case <-tick.C: c.modeMtx.RLock() - if c.readOnly() { + if c.readOnly() || c.noMetabase() { c.modeMtx.RUnlock() break } @@ -281,6 +281,9 @@ func (c *cache) Flush(ctx context.Context, ignoreErrors, seal bool) error { c.modeMtx.Lock() // exclusive lock to not to conflict with background flush defer c.modeMtx.Unlock() + if c.noMetabase() { + return ErrDegraded + } if err := c.flush(ctx, ignoreErrors); err != nil { return err diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index 5194446ec..bf26833bd 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -29,6 +29,14 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e )) defer span.End() + if !c.modeMtx.TryRLock() { + return nil, ErrNotInitialized + } + defer c.modeMtx.RUnlock() + if c.mode.NoMetabase() { + return nil, ErrDegraded + } + obj, err := c.getInternal(ctx, saddr, addr) return obj, metaerr.Wrap(err) } @@ -71,6 +79,14 @@ func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, )) defer span.End() + if !c.modeMtx.TryRLock() { + return nil, ErrNotInitialized + } + defer c.modeMtx.RUnlock() + if c.mode.NoMetabase() { + return nil, ErrDegraded + } + obj, err := c.getInternal(ctx, saddr, addr) if err != nil { return nil, metaerr.Wrap(err) diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 7c6439fe9..a10e593aa 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -76,3 +76,9 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode, ignoreErrors bool) err func (c *cache) readOnly() bool { return c.mode.ReadOnly() } + +// noMetabase returns true if c is operating without the metabase. +// `c.modeMtx` must be taken. +func (c *cache) noMetabase() bool { + return c.mode.NoMetabase() +} diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 8b6c09e9a..0e419f95b 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -41,6 +41,9 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro if c.readOnly() { return common.PutRes{}, ErrReadOnly } + if c.noMetabase() { + return common.PutRes{}, ErrDegraded + } sz := uint64(len(prm.RawData)) if sz > c.maxObjectSize { diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 112594ec0..71dba61cf 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -59,6 +59,8 @@ type Metabase interface { var ( // ErrReadOnly is returned when Put/Write is performed in a read-only mode. ErrReadOnly = logicerr.New("write-cache is in read-only mode") + // ErrDegraded is returned when writecache is in degraded mode. + ErrDegraded = logicerr.New("write-cache is in degraded mode") // ErrNotInitialized is returned when write-cache is initializing. ErrNotInitialized = logicerr.New("write-cache is not initialized yet") // ErrBigObject is returned when object is too big to be placed in cache. From d9fe63ee03388d80480f8c5ecdb0a747cb01e136 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 12 Mar 2024 10:16:18 +0300 Subject: [PATCH 0965/1943] [#1036] help.mk: Fix target regexp pattern We use no space before the colon, but we use it when declaring a variable. Signed-off-by: Evgenii Stratonikov --- help.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/help.mk b/help.mk index c065ec862..21408cf17 100644 --- a/help.mk +++ b/help.mk @@ -8,4 +8,4 @@ help: @echo '' @echo ' Targets:' @echo '' - @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+ ?:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq + @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq From aa27596d779b195631684a00a4293386b7dc1af7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 12 Mar 2024 10:19:41 +0300 Subject: [PATCH 0966/1943] [#1036] help.mk: Print target without comments Make it easy to notice missing comments and avoid reusing comment from some other place. Signed-off-by: Evgenii Stratonikov --- help.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/help.mk b/help.mk index 21408cf17..a2ac989dc 100644 --- a/help.mk +++ b/help.mk @@ -8,4 +8,4 @@ help: @echo '' @echo ' Targets:' @echo '' - @awk '/^#/{ comment = substr($$0,3) } comment && /^[a-zA-Z][a-zA-Z0-9_-]+:/{ print " ", $$1, comment }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq + @awk '/^#/{ comment = substr($$0,3) } /^[a-zA-Z][a-zA-Z0-9_-]+:/{ print " ", $$1, comment; comment = "" }' $(MAKEFILE_LIST) | column -t -s ':' | grep -v 'IGNORE' | sort | uniq From 6eb63cf5c7b1a2d32c63cddb51a6d67074fa4852 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 12 Mar 2024 10:24:35 +0300 Subject: [PATCH 0967/1943] [#1036] Makefile: Add missing comments Signed-off-by: Evgenii Stratonikov --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index 493ef25e5..21d7d4275 100755 --- a/Makefile +++ b/Makefile @@ -115,6 +115,7 @@ protoc: --go-grpc_out=. --go-grpc_opt=paths=source_relative $$f; \ done +# Install protoc protoc-install: @rm -rf $(PROTOBUF_DIR) @mkdir $(PROTOBUF_DIR) @@ -161,6 +162,7 @@ imports: @echo "⇒ Processing goimports check" @goimports -w cmd/ pkg/ misc/ +# Run gofumpt fumpt: @echo "⇒ Processing gofumpt check" @gofumpt -l -w cmd/ pkg/ misc/ @@ -170,6 +172,7 @@ test: @echo "⇒ Running go test" @go test ./... -count=1 +# Run pre-commit pre-commit-run: @pre-commit run -a --hook-stage manual @@ -205,11 +208,13 @@ staticcheck-run: fi @$(STATICCHECK_VERSION_DIR)/staticcheck ./... +# Install gopls gopls-install: @rm -rf $(GOPLS_DIR) @mkdir $(GOPLS_DIR) @GOBIN=$(GOPLS_VERSION_DIR) go install golang.org/x/tools/gopls@$(GOPLS_VERSION) +# Run gopls gopls-run: @if [ ! -d "$(GOPLS_VERSION_DIR)" ]; then \ make gopls-install; \ @@ -253,13 +258,16 @@ debpackage: "Please see CHANGELOG.md for code changes for $(VERSION)" dpkg-buildpackage --no-sign -b +# Cleanup deb package build directories debclean: dh clean +# Download locode database locode-download: @wget -q -O ./.cache/locode_db.gz 'https://git.frostfs.info/TrueCloudLab/frostfs-locode-db/releases/download/${LOCODE_DB_VERSION}/locode_db.gz' gzip -dfk ./.cache/locode_db.gz +# Start dev environment env-up: all docker compose -f dev/docker-compose.yml up -d @if [ ! -d "$(FROSTFS_CONTRACTS_PATH)" ]; then \ @@ -271,6 +279,7 @@ env-up: all make locode-download; \ fi +# Shutdown dev environment env-down: docker compose -f dev/docker-compose.yml down docker volume rm -f frostfs-node_neo-go From b4cb54e7ed83c1595129aa29e89db2d003707567 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 29 Feb 2024 18:31:04 +0300 Subject: [PATCH 0968/1943] [#963] node: Go on initialization even deposit notary is hung * Make makeAndWaitNotaryDeposit run asynchronously as worker during application boot-up. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/config.go | 6 ++++++ cmd/frostfs-node/main.go | 16 ++++++++++++++-- cmd/frostfs-node/morph.go | 2 ++ internal/logs/logs.go | 2 ++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 48b76ac0f..40fa34a95 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -352,6 +352,12 @@ type internals struct { log *logger.Logger + // Some methods that initialize node components may launch an asynchronous job, + // appending it to workers. That means despite application is already running, + // not all components completely initialized yet. + // initAppWG is used to control this situation. + initAppWG sync.WaitGroup + wg sync.WaitGroup workers []worker closers []closer diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index e1eeb6861..f82b2f171 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -67,7 +67,10 @@ func main() { bootUp(ctx, c) - c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY) + go func() { + c.initAppWG.Wait() + c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY) + }() wait(c) } @@ -144,7 +147,16 @@ func stopAndLog(c *cfg, name string, stopper func() error) { func bootUp(ctx context.Context, c *cfg) { runAndLog(ctx, c, "NATS", true, connectNats) runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) }) - runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit) + + // It may happen that boot-up waits for the execution of a notary deposit transaction + // and waiting loop may hang for an indefinite time. In this case, we need to let + // frostfs-node go on initialization, although its functionality will be available partially. + // That's why makeAndWaitNotaryDeposit is run asynchroniosly. + c.initAppWG.Add(1) + c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { + defer c.initAppWG.Done() + runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit) + })) bootstrapNode(c) startWorkers(ctx, c) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 698fb3b83..1b148095b 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -158,6 +158,7 @@ var ( func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error { for i := 0; i < notaryDepositRetriesAmount; i++ { + c.log.Debug(logs.ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted) select { case <-ctx.Done(): return ctx.Err() @@ -167,6 +168,7 @@ func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error { ok, err := c.cfgMorph.client.TxHalt(tx) if err == nil { if ok { + c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted) return nil } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 6b48940b7..bd8b8ed01 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -158,6 +158,8 @@ const ( ClientNotaryDepositInvoke = "notary deposit invoke" ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked" ClientNotaryRequestInvoked = "notary request invoked" + ClientNotaryDepositTransactionWasSuccessfullyPersisted = "notary deposit transaction was successfully persisted" + ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted = "attempt to wait for notary deposit transaction to get persisted" ClientNeoClientInvoke = "neo client invoke" ClientNativeGasTransferInvoke = "native gas transfer invoke" ClientBatchGasTransferInvoke = "batch gas transfer invoke" From 5c252c9193f638eacce54b580fa7a6493e1651c7 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 12 Mar 2024 15:09:55 +0300 Subject: [PATCH 0969/1943] [#1039] object: Skip APE check for certain request roles * Skip APE check if a role is Container. * Skip APE check if a role is IR and methods are get-like. Signed-off-by: Airat Arifullin --- pkg/services/object/ape/checker.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 13dab0ec7..1d42d21d4 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -10,6 +10,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" ) type checkerImpl struct { @@ -56,6 +57,21 @@ var errMissingOID = errors.New("object ID is not set") // CheckAPE checks if a request or a response is permitted creating an ape request and passing // it to chain router. func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { + // APE check is ignored for some inter-node requests. + if prm.Role == nativeschema.PropertyValueContainerRoleContainer { + return nil + } else if prm.Role == nativeschema.PropertyValueContainerRoleIR { + switch prm.Method { + case nativeschema.MethodGetObject, + nativeschema.MethodHeadObject, + nativeschema.MethodSearchObject, + nativeschema.MethodRangeObject, + nativeschema.MethodHashObject: + return nil + default: + } + } + r, err := c.newAPERequest(ctx, prm) if err != nil { return fmt.Errorf("failed to create ape request: %w", err) From e3579922d83eef1560779ba02addd2d1cf6ca021 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 12 Mar 2024 12:51:05 +0300 Subject: [PATCH 0970/1943] [#1038] pre-commit: Remove gitlint Signed-off-by: Evgenii Stratonikov --- .gitlint | 11 ----------- .pre-commit-config.yaml | 7 ------- 2 files changed, 18 deletions(-) delete mode 100644 .gitlint diff --git a/.gitlint b/.gitlint deleted file mode 100644 index e7218ac53..000000000 --- a/.gitlint +++ /dev/null @@ -1,11 +0,0 @@ -[general] -fail-without-commits=True -regex-style-search=True -contrib=CC1 - -[title-match-regex] -regex=^\[\#[0-9Xx]+\]\s - -[ignore-by-title] -regex=^Release(.*) -ignore=title-match-regex diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f66adda0b..f1d43ab0a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,13 +2,6 @@ ci: autofix_prs: false repos: - - repo: https://github.com/jorisroovers/gitlint - rev: v0.19.1 - hooks: - - id: gitlint - stages: [commit-msg] - - id: gitlint-ci - - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: From 179b6e64faf8f4177c765321114eb620bce4e1e5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 12 Mar 2024 13:07:08 +0300 Subject: [PATCH 0971/1943] [#1038] Makefile: Allow to override testflags Signed-off-by: Evgenii Stratonikov --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 21d7d4275..4d3c121bf 100755 --- a/Makefile +++ b/Makefile @@ -168,9 +168,10 @@ fumpt: @gofumpt -l -w cmd/ pkg/ misc/ # Run Unit Test with go test +test: GOFLAGS ?= "-count=1" test: @echo "⇒ Running go test" - @go test ./... -count=1 + @GOFLAGS="$(GOFLAGS)" go test ./... # Run pre-commit pre-commit-run: From f42a529f49b7b19f5b360cc4057c3f5c365978b5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 12 Mar 2024 13:08:09 +0300 Subject: [PATCH 0972/1943] [#1038] pre-commit: Use cached tests in hook Signed-off-by: Evgenii Stratonikov --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f1d43ab0a..13284b9c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: hooks: - id: go-unit-tests name: go unit tests - entry: make test + entry: make test GOFLAGS='' pass_filenames: false types: [go] language: system From 7a4b6057cebb85ac93cd5b2f0bd60563c229db9b Mon Sep 17 00:00:00 2001 From: aarifullin Date: Wed, 13 Mar 2024 11:53:59 +0300 Subject: [PATCH 0973/1943] Revert "[#963] node: Go on initialization even deposit notary is hung" This reverts commit b4cb54e7ed83c1595129aa29e89db2d003707567. Signed-off-by: aarifullin --- cmd/frostfs-node/config.go | 6 ------ cmd/frostfs-node/main.go | 16 ++-------------- cmd/frostfs-node/morph.go | 2 -- internal/logs/logs.go | 2 -- 4 files changed, 2 insertions(+), 24 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 40fa34a95..48b76ac0f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -352,12 +352,6 @@ type internals struct { log *logger.Logger - // Some methods that initialize node components may launch an asynchronous job, - // appending it to workers. That means despite application is already running, - // not all components completely initialized yet. - // initAppWG is used to control this situation. - initAppWG sync.WaitGroup - wg sync.WaitGroup workers []worker closers []closer diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index f82b2f171..e1eeb6861 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -67,10 +67,7 @@ func main() { bootUp(ctx, c) - go func() { - c.initAppWG.Wait() - c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY) - }() + c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY) wait(c) } @@ -147,16 +144,7 @@ func stopAndLog(c *cfg, name string, stopper func() error) { func bootUp(ctx context.Context, c *cfg) { runAndLog(ctx, c, "NATS", true, connectNats) runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) }) - - // It may happen that boot-up waits for the execution of a notary deposit transaction - // and waiting loop may hang for an indefinite time. In this case, we need to let - // frostfs-node go on initialization, although its functionality will be available partially. - // That's why makeAndWaitNotaryDeposit is run asynchroniosly. - c.initAppWG.Add(1) - c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { - defer c.initAppWG.Done() - runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit) - })) + runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit) bootstrapNode(c) startWorkers(ctx, c) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 1b148095b..698fb3b83 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -158,7 +158,6 @@ var ( func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error { for i := 0; i < notaryDepositRetriesAmount; i++ { - c.log.Debug(logs.ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted) select { case <-ctx.Done(): return ctx.Err() @@ -168,7 +167,6 @@ func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error { ok, err := c.cfgMorph.client.TxHalt(tx) if err == nil { if ok { - c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted) return nil } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index bd8b8ed01..6b48940b7 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -158,8 +158,6 @@ const ( ClientNotaryDepositInvoke = "notary deposit invoke" ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked" ClientNotaryRequestInvoked = "notary request invoked" - ClientNotaryDepositTransactionWasSuccessfullyPersisted = "notary deposit transaction was successfully persisted" - ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted = "attempt to wait for notary deposit transaction to get persisted" ClientNeoClientInvoke = "neo client invoke" ClientNativeGasTransferInvoke = "native gas transfer invoke" ClientBatchGasTransferInvoke = "batch gas transfer invoke" From d1d53d2bb686b081a038f3ae11db294df3abdadc Mon Sep 17 00:00:00 2001 From: aarifullin Date: Wed, 13 Mar 2024 11:57:51 +0300 Subject: [PATCH 0974/1943] [#963] node: Add logging for waitNotaryDeposit Signed-off-by: Airat Arifullin --- cmd/frostfs-node/morph.go | 2 ++ internal/logs/logs.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 698fb3b83..1b148095b 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -158,6 +158,7 @@ var ( func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error { for i := 0; i < notaryDepositRetriesAmount; i++ { + c.log.Debug(logs.ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted) select { case <-ctx.Done(): return ctx.Err() @@ -167,6 +168,7 @@ func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error { ok, err := c.cfgMorph.client.TxHalt(tx) if err == nil { if ok { + c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted) return nil } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 6b48940b7..bd8b8ed01 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -158,6 +158,8 @@ const ( ClientNotaryDepositInvoke = "notary deposit invoke" ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked" ClientNotaryRequestInvoked = "notary request invoked" + ClientNotaryDepositTransactionWasSuccessfullyPersisted = "notary deposit transaction was successfully persisted" + ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted = "attempt to wait for notary deposit transaction to get persisted" ClientNeoClientInvoke = "neo client invoke" ClientNativeGasTransferInvoke = "native gas transfer invoke" ClientBatchGasTransferInvoke = "batch gas transfer invoke" From 926cdeb0728a1982e96447642093b08a6ef62a4a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 12 Mar 2024 18:41:52 +0300 Subject: [PATCH 0975/1943] [#1043] services: Regenerate proto Signed-off-by: Dmitrii Stepanov --- pkg/services/control/ir/service.pb.go | 2 +- pkg/services/control/ir/types.pb.go | 2 +- pkg/services/control/service.pb.go | 2 +- pkg/services/control/types.pb.go | 2 +- pkg/services/tree/service.pb.go | 2 +- pkg/services/tree/types.pb.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index 7dc8f3aff..d1e253bf5 100644 --- a/pkg/services/control/ir/service.pb.go +++ b/pkg/services/control/ir/service.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.32.0 +// protoc-gen-go v1.33.0 // protoc v4.25.0 // source: pkg/services/control/ir/service.proto diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go index 819cedf6a..828814b25 100644 --- a/pkg/services/control/ir/types.pb.go +++ b/pkg/services/control/ir/types.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.32.0 +// protoc-gen-go v1.33.0 // protoc v4.25.0 // source: pkg/services/control/ir/types.proto diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index c24bc1597..261065b58 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.32.0 +// protoc-gen-go v1.33.0 // protoc v4.25.0 // source: pkg/services/control/service.proto diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index 4556e8a55..2fc16a926 100644 --- a/pkg/services/control/types.pb.go +++ b/pkg/services/control/types.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.32.0 +// protoc-gen-go v1.33.0 // protoc v4.25.0 // source: pkg/services/control/types.proto diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go index 441ae7927..7930361e1 100644 --- a/pkg/services/tree/service.pb.go +++ b/pkg/services/tree/service.pb.go @@ -3,7 +3,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.32.0 +// protoc-gen-go v1.33.0 // protoc v4.25.0 // source: pkg/services/tree/service.proto diff --git a/pkg/services/tree/types.pb.go b/pkg/services/tree/types.pb.go index e44ac5566..6464ccb77 100644 --- a/pkg/services/tree/types.pb.go +++ b/pkg/services/tree/types.pb.go @@ -3,7 +3,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.32.0 +// protoc-gen-go v1.33.0 // protoc v4.25.0 // source: pkg/services/tree/types.proto From 31e2396a5f59990fa5a135dcd6b28e23f603b693 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 12 Mar 2024 18:57:38 +0300 Subject: [PATCH 0976/1943] [#1043] control: Add ResetEvacuationStatus implementation Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 10 + .../engine/evacuate_limiter.go | 15 + .../engine/evacuate_test.go | 12 + pkg/services/control/server/evacuate_async.go | 26 + pkg/services/control/service.pb.go | 1646 ++++++++++------- pkg/services/control/service.proto | 19 + pkg/services/control/service_frostfs.pb.go | 154 ++ pkg/services/control/service_grpc.pb.go | 77 +- 8 files changed, 1254 insertions(+), 705 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 45600fe66..81fe47e65 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -733,3 +733,13 @@ func (e *StorageEngine) EnqueRunningEvacuationStop(ctx context.Context) error { return e.evacuateLimiter.CancelIfRunning() } + +func (e *StorageEngine) ResetEvacuationStatus(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + return e.evacuateLimiter.ResetEvacuationStatus() +} diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go index 63960e238..1e6b9ccb1 100644 --- a/pkg/local_object_storage/engine/evacuate_limiter.go +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -204,3 +204,18 @@ func (l *evacuationLimiter) CancelIfRunning() error { l.cancel() return nil } + +func (l *evacuationLimiter) ResetEvacuationStatus() error { + l.guard.Lock() + defer l.guard.Unlock() + + if l.state.processState == EvacuateProcessStateRunning { + return logicerr.New("there is running evacuation task") + } + + l.state = EvacuationState{} + l.eg = nil + l.cancel = nil + + return nil +} diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 32582cd2a..e8d9a449f 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -386,6 +386,8 @@ func TestEvacuateObjectsAsync(t *testing.T) { require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid running shard ids") require.Equal(t, "", st.ErrorMessage(), "invalid init error message") + require.Error(t, e.ResetEvacuationStatus(context.Background())) + close(blocker) require.Eventually(t, func() bool { @@ -401,6 +403,16 @@ func TestEvacuateObjectsAsync(t *testing.T) { require.Equal(t, "", st.ErrorMessage(), "invalid final error message") require.NoError(t, eg.Wait()) + + require.NoError(t, e.ResetEvacuationStatus(context.Background())) + st, err = e.GetEvacuationState(context.Background()) + require.NoError(t, err, "get state after reset failed") + require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid state after reset") + require.Equal(t, uint64(0), st.ObjectsEvacuated(), "invalid count after reset") + require.Nil(t, st.StartedAt(), "invalid started at after reset") + require.Nil(t, st.FinishedAt(), "invalid finished at after reset") + require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid shard ids after reset") + require.Equal(t, "", st.ErrorMessage(), "invalid error message after reset") } func TestEvacuateTreesLocal(t *testing.T) { diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index cd7e8a2c7..42ae2635b 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -102,3 +102,29 @@ func (s *Server) StopShardEvacuation(ctx context.Context, req *control.StopShard } return resp, nil } + +func (s *Server) ResetShardEvacuationStatus(ctx context.Context, req *control.ResetShardEvacuationStatusRequest) (*control.ResetShardEvacuationStatusResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + err = s.s.ResetEvacuationStatus(ctx) + if err != nil { + var logicalErr logicerr.Logical + if errors.As(err, &logicalErr) { + return nil, status.Error(codes.Aborted, err.Error()) + } + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.ResetShardEvacuationStatusResponse{ + Body: &control.ResetShardEvacuationStatusResponse_Body{}, + } + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil +} diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 261065b58..586619767 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -1375,6 +1375,118 @@ func (x *GetShardEvacuationStatusResponse) GetSignature() *Signature { return nil } +// ResetShardEvacuationStatus request. +type ResetShardEvacuationStatusRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *ResetShardEvacuationStatusRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *ResetShardEvacuationStatusRequest) Reset() { + *x = ResetShardEvacuationStatusRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ResetShardEvacuationStatusRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResetShardEvacuationStatusRequest) ProtoMessage() {} + +func (x *ResetShardEvacuationStatusRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[22] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResetShardEvacuationStatusRequest.ProtoReflect.Descriptor instead. +func (*ResetShardEvacuationStatusRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22} +} + +func (x *ResetShardEvacuationStatusRequest) GetBody() *ResetShardEvacuationStatusRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *ResetShardEvacuationStatusRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// ResetShardEvacuationStatus response. +type ResetShardEvacuationStatusResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *ResetShardEvacuationStatusResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *ResetShardEvacuationStatusResponse) Reset() { + *x = ResetShardEvacuationStatusResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ResetShardEvacuationStatusResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResetShardEvacuationStatusResponse) ProtoMessage() {} + +func (x *ResetShardEvacuationStatusResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[23] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResetShardEvacuationStatusResponse.ProtoReflect.Descriptor instead. +func (*ResetShardEvacuationStatusResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23} +} + +func (x *ResetShardEvacuationStatusResponse) GetBody() *ResetShardEvacuationStatusResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *ResetShardEvacuationStatusResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + // StopShardEvacuation request. type StopShardEvacuationRequest struct { state protoimpl.MessageState @@ -1388,7 +1500,7 @@ type StopShardEvacuationRequest struct { func (x *StopShardEvacuationRequest) Reset() { *x = StopShardEvacuationRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[22] + mi := &file_pkg_services_control_service_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1401,7 +1513,7 @@ func (x *StopShardEvacuationRequest) String() string { func (*StopShardEvacuationRequest) ProtoMessage() {} func (x *StopShardEvacuationRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[22] + mi := &file_pkg_services_control_service_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1414,7 +1526,7 @@ func (x *StopShardEvacuationRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShardEvacuationRequest.ProtoReflect.Descriptor instead. func (*StopShardEvacuationRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24} } func (x *StopShardEvacuationRequest) GetBody() *StopShardEvacuationRequest_Body { @@ -1444,7 +1556,7 @@ type StopShardEvacuationResponse struct { func (x *StopShardEvacuationResponse) Reset() { *x = StopShardEvacuationResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[23] + mi := &file_pkg_services_control_service_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1457,7 +1569,7 @@ func (x *StopShardEvacuationResponse) String() string { func (*StopShardEvacuationResponse) ProtoMessage() {} func (x *StopShardEvacuationResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[23] + mi := &file_pkg_services_control_service_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1470,7 +1582,7 @@ func (x *StopShardEvacuationResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShardEvacuationResponse.ProtoReflect.Descriptor instead. func (*StopShardEvacuationResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25} } func (x *StopShardEvacuationResponse) GetBody() *StopShardEvacuationResponse_Body { @@ -1500,7 +1612,7 @@ type AddChainLocalOverrideRequest struct { func (x *AddChainLocalOverrideRequest) Reset() { *x = AddChainLocalOverrideRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1513,7 +1625,7 @@ func (x *AddChainLocalOverrideRequest) String() string { func (*AddChainLocalOverrideRequest) ProtoMessage() {} func (x *AddChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1526,7 +1638,7 @@ func (x *AddChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AddChainLocalOverrideRequest.ProtoReflect.Descriptor instead. func (*AddChainLocalOverrideRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26} } func (x *AddChainLocalOverrideRequest) GetBody() *AddChainLocalOverrideRequest_Body { @@ -1556,7 +1668,7 @@ type AddChainLocalOverrideResponse struct { func (x *AddChainLocalOverrideResponse) Reset() { *x = AddChainLocalOverrideResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1569,7 +1681,7 @@ func (x *AddChainLocalOverrideResponse) String() string { func (*AddChainLocalOverrideResponse) ProtoMessage() {} func (x *AddChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1582,7 +1694,7 @@ func (x *AddChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AddChainLocalOverrideResponse.ProtoReflect.Descriptor instead. func (*AddChainLocalOverrideResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27} } func (x *AddChainLocalOverrideResponse) GetBody() *AddChainLocalOverrideResponse_Body { @@ -1612,7 +1724,7 @@ type GetChainLocalOverrideRequest struct { func (x *GetChainLocalOverrideRequest) Reset() { *x = GetChainLocalOverrideRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1625,7 +1737,7 @@ func (x *GetChainLocalOverrideRequest) String() string { func (*GetChainLocalOverrideRequest) ProtoMessage() {} func (x *GetChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1638,7 +1750,7 @@ func (x *GetChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetChainLocalOverrideRequest.ProtoReflect.Descriptor instead. func (*GetChainLocalOverrideRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28} } func (x *GetChainLocalOverrideRequest) GetBody() *GetChainLocalOverrideRequest_Body { @@ -1668,7 +1780,7 @@ type GetChainLocalOverrideResponse struct { func (x *GetChainLocalOverrideResponse) Reset() { *x = GetChainLocalOverrideResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1681,7 +1793,7 @@ func (x *GetChainLocalOverrideResponse) String() string { func (*GetChainLocalOverrideResponse) ProtoMessage() {} func (x *GetChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1694,7 +1806,7 @@ func (x *GetChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetChainLocalOverrideResponse.ProtoReflect.Descriptor instead. func (*GetChainLocalOverrideResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29} } func (x *GetChainLocalOverrideResponse) GetBody() *GetChainLocalOverrideResponse_Body { @@ -1724,7 +1836,7 @@ type ListChainLocalOverridesRequest struct { func (x *ListChainLocalOverridesRequest) Reset() { *x = ListChainLocalOverridesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1737,7 +1849,7 @@ func (x *ListChainLocalOverridesRequest) String() string { func (*ListChainLocalOverridesRequest) ProtoMessage() {} func (x *ListChainLocalOverridesRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1750,7 +1862,7 @@ func (x *ListChainLocalOverridesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListChainLocalOverridesRequest.ProtoReflect.Descriptor instead. func (*ListChainLocalOverridesRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30} } func (x *ListChainLocalOverridesRequest) GetBody() *ListChainLocalOverridesRequest_Body { @@ -1780,7 +1892,7 @@ type ListChainLocalOverridesResponse struct { func (x *ListChainLocalOverridesResponse) Reset() { *x = ListChainLocalOverridesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1793,7 +1905,7 @@ func (x *ListChainLocalOverridesResponse) String() string { func (*ListChainLocalOverridesResponse) ProtoMessage() {} func (x *ListChainLocalOverridesResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1806,7 +1918,7 @@ func (x *ListChainLocalOverridesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListChainLocalOverridesResponse.ProtoReflect.Descriptor instead. func (*ListChainLocalOverridesResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31} } func (x *ListChainLocalOverridesResponse) GetBody() *ListChainLocalOverridesResponse_Body { @@ -1836,7 +1948,7 @@ type ListTargetsLocalOverridesRequest struct { func (x *ListTargetsLocalOverridesRequest) Reset() { *x = ListTargetsLocalOverridesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1849,7 +1961,7 @@ func (x *ListTargetsLocalOverridesRequest) String() string { func (*ListTargetsLocalOverridesRequest) ProtoMessage() {} func (x *ListTargetsLocalOverridesRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1862,7 +1974,7 @@ func (x *ListTargetsLocalOverridesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListTargetsLocalOverridesRequest.ProtoReflect.Descriptor instead. func (*ListTargetsLocalOverridesRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32} } func (x *ListTargetsLocalOverridesRequest) GetBody() *ListTargetsLocalOverridesRequest_Body { @@ -1892,7 +2004,7 @@ type ListTargetsLocalOverridesResponse struct { func (x *ListTargetsLocalOverridesResponse) Reset() { *x = ListTargetsLocalOverridesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1905,7 +2017,7 @@ func (x *ListTargetsLocalOverridesResponse) String() string { func (*ListTargetsLocalOverridesResponse) ProtoMessage() {} func (x *ListTargetsLocalOverridesResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1918,7 +2030,7 @@ func (x *ListTargetsLocalOverridesResponse) ProtoReflect() protoreflect.Message // Deprecated: Use ListTargetsLocalOverridesResponse.ProtoReflect.Descriptor instead. func (*ListTargetsLocalOverridesResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33} } func (x *ListTargetsLocalOverridesResponse) GetBody() *ListTargetsLocalOverridesResponse_Body { @@ -1947,7 +2059,7 @@ type RemoveChainLocalOverrideRequest struct { func (x *RemoveChainLocalOverrideRequest) Reset() { *x = RemoveChainLocalOverrideRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1960,7 +2072,7 @@ func (x *RemoveChainLocalOverrideRequest) String() string { func (*RemoveChainLocalOverrideRequest) ProtoMessage() {} func (x *RemoveChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1973,7 +2085,7 @@ func (x *RemoveChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveChainLocalOverrideRequest.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34} } func (x *RemoveChainLocalOverrideRequest) GetBody() *RemoveChainLocalOverrideRequest_Body { @@ -2002,7 +2114,7 @@ type RemoveChainLocalOverrideResponse struct { func (x *RemoveChainLocalOverrideResponse) Reset() { *x = RemoveChainLocalOverrideResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2015,7 +2127,7 @@ func (x *RemoveChainLocalOverrideResponse) String() string { func (*RemoveChainLocalOverrideResponse) ProtoMessage() {} func (x *RemoveChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2028,7 +2140,7 @@ func (x *RemoveChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveChainLocalOverrideResponse.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35} } func (x *RemoveChainLocalOverrideResponse) GetBody() *RemoveChainLocalOverrideResponse_Body { @@ -2057,7 +2169,7 @@ type SealWriteCacheRequest struct { func (x *SealWriteCacheRequest) Reset() { *x = SealWriteCacheRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2070,7 +2182,7 @@ func (x *SealWriteCacheRequest) String() string { func (*SealWriteCacheRequest) ProtoMessage() {} func (x *SealWriteCacheRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2083,7 +2195,7 @@ func (x *SealWriteCacheRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheRequest.ProtoReflect.Descriptor instead. func (*SealWriteCacheRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36} } func (x *SealWriteCacheRequest) GetBody() *SealWriteCacheRequest_Body { @@ -2112,7 +2224,7 @@ type SealWriteCacheResponse struct { func (x *SealWriteCacheResponse) Reset() { *x = SealWriteCacheResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2125,7 +2237,7 @@ func (x *SealWriteCacheResponse) String() string { func (*SealWriteCacheResponse) ProtoMessage() {} func (x *SealWriteCacheResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2138,7 +2250,7 @@ func (x *SealWriteCacheResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheResponse.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37} } func (x *SealWriteCacheResponse) GetBody() *SealWriteCacheResponse_Body { @@ -2167,7 +2279,7 @@ type DetachShardsRequest struct { func (x *DetachShardsRequest) Reset() { *x = DetachShardsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2180,7 +2292,7 @@ func (x *DetachShardsRequest) String() string { func (*DetachShardsRequest) ProtoMessage() {} func (x *DetachShardsRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2193,7 +2305,7 @@ func (x *DetachShardsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsRequest.ProtoReflect.Descriptor instead. func (*DetachShardsRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38} } func (x *DetachShardsRequest) GetBody() *DetachShardsRequest_Body { @@ -2222,7 +2334,7 @@ type DetachShardsResponse struct { func (x *DetachShardsResponse) Reset() { *x = DetachShardsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2235,7 +2347,7 @@ func (x *DetachShardsResponse) String() string { func (*DetachShardsResponse) ProtoMessage() {} func (x *DetachShardsResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2248,7 +2360,7 @@ func (x *DetachShardsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsResponse.ProtoReflect.Descriptor instead. func (*DetachShardsResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39} } func (x *DetachShardsResponse) GetBody() *DetachShardsResponse_Body { @@ -2275,7 +2387,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2288,7 +2400,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2319,7 +2431,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2332,7 +2444,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2383,7 +2495,7 @@ type SetNetmapStatusRequest_Body struct { func (x *SetNetmapStatusRequest_Body) Reset() { *x = SetNetmapStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2396,7 +2508,7 @@ func (x *SetNetmapStatusRequest_Body) String() string { func (*SetNetmapStatusRequest_Body) ProtoMessage() {} func (x *SetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2436,7 +2548,7 @@ type SetNetmapStatusResponse_Body struct { func (x *SetNetmapStatusResponse_Body) Reset() { *x = SetNetmapStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2449,7 +2561,7 @@ func (x *SetNetmapStatusResponse_Body) String() string { func (*SetNetmapStatusResponse_Body) ProtoMessage() {} func (x *SetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2479,7 +2591,7 @@ type DropObjectsRequest_Body struct { func (x *DropObjectsRequest_Body) Reset() { *x = DropObjectsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2492,7 +2604,7 @@ func (x *DropObjectsRequest_Body) String() string { func (*DropObjectsRequest_Body) ProtoMessage() {} func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2525,7 +2637,7 @@ type DropObjectsResponse_Body struct { func (x *DropObjectsResponse_Body) Reset() { *x = DropObjectsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2538,7 +2650,7 @@ func (x *DropObjectsResponse_Body) String() string { func (*DropObjectsResponse_Body) ProtoMessage() {} func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2564,7 +2676,7 @@ type ListShardsRequest_Body struct { func (x *ListShardsRequest_Body) Reset() { *x = ListShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2577,7 +2689,7 @@ func (x *ListShardsRequest_Body) String() string { func (*ListShardsRequest_Body) ProtoMessage() {} func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2606,7 +2718,7 @@ type ListShardsResponse_Body struct { func (x *ListShardsResponse_Body) Reset() { *x = ListShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2619,7 +2731,7 @@ func (x *ListShardsResponse_Body) String() string { func (*ListShardsResponse_Body) ProtoMessage() {} func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2659,7 +2771,7 @@ type SetShardModeRequest_Body struct { func (x *SetShardModeRequest_Body) Reset() { *x = SetShardModeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2672,7 +2784,7 @@ func (x *SetShardModeRequest_Body) String() string { func (*SetShardModeRequest_Body) ProtoMessage() {} func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2719,7 +2831,7 @@ type SetShardModeResponse_Body struct { func (x *SetShardModeResponse_Body) Reset() { *x = SetShardModeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2732,7 +2844,7 @@ func (x *SetShardModeResponse_Body) String() string { func (*SetShardModeResponse_Body) ProtoMessage() {} func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2763,7 +2875,7 @@ type SynchronizeTreeRequest_Body struct { func (x *SynchronizeTreeRequest_Body) Reset() { *x = SynchronizeTreeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2776,7 +2888,7 @@ func (x *SynchronizeTreeRequest_Body) String() string { func (*SynchronizeTreeRequest_Body) ProtoMessage() {} func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2823,7 +2935,7 @@ type SynchronizeTreeResponse_Body struct { func (x *SynchronizeTreeResponse_Body) Reset() { *x = SynchronizeTreeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2836,7 +2948,7 @@ func (x *SynchronizeTreeResponse_Body) String() string { func (*SynchronizeTreeResponse_Body) ProtoMessage() {} func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2867,7 +2979,7 @@ type EvacuateShardRequest_Body struct { func (x *EvacuateShardRequest_Body) Reset() { *x = EvacuateShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2880,7 +2992,7 @@ func (x *EvacuateShardRequest_Body) String() string { func (*EvacuateShardRequest_Body) ProtoMessage() {} func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2922,7 +3034,7 @@ type EvacuateShardResponse_Body struct { func (x *EvacuateShardResponse_Body) Reset() { *x = EvacuateShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2935,7 +3047,7 @@ func (x *EvacuateShardResponse_Body) String() string { func (*EvacuateShardResponse_Body) ProtoMessage() {} func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2973,7 +3085,7 @@ type FlushCacheRequest_Body struct { func (x *FlushCacheRequest_Body) Reset() { *x = FlushCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2986,7 +3098,7 @@ func (x *FlushCacheRequest_Body) String() string { func (*FlushCacheRequest_Body) ProtoMessage() {} func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3026,7 +3138,7 @@ type FlushCacheResponse_Body struct { func (x *FlushCacheResponse_Body) Reset() { *x = FlushCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3039,7 +3151,7 @@ func (x *FlushCacheResponse_Body) String() string { func (*FlushCacheResponse_Body) ProtoMessage() {} func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3070,7 +3182,7 @@ type DoctorRequest_Body struct { func (x *DoctorRequest_Body) Reset() { *x = DoctorRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3083,7 +3195,7 @@ func (x *DoctorRequest_Body) String() string { func (*DoctorRequest_Body) ProtoMessage() {} func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3123,7 +3235,7 @@ type DoctorResponse_Body struct { func (x *DoctorResponse_Body) Reset() { *x = DoctorResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3136,7 +3248,7 @@ func (x *DoctorResponse_Body) String() string { func (*DoctorResponse_Body) ProtoMessage() {} func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3169,7 +3281,7 @@ type StartShardEvacuationRequest_Body struct { func (x *StartShardEvacuationRequest_Body) Reset() { *x = StartShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3182,7 +3294,7 @@ func (x *StartShardEvacuationRequest_Body) String() string { func (*StartShardEvacuationRequest_Body) ProtoMessage() {} func (x *StartShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3229,7 +3341,7 @@ type StartShardEvacuationResponse_Body struct { func (x *StartShardEvacuationResponse_Body) Reset() { *x = StartShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3242,7 +3354,7 @@ func (x *StartShardEvacuationResponse_Body) String() string { func (*StartShardEvacuationResponse_Body) ProtoMessage() {} func (x *StartShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3268,7 +3380,7 @@ type GetShardEvacuationStatusRequest_Body struct { func (x *GetShardEvacuationStatusRequest_Body) Reset() { *x = GetShardEvacuationStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3281,7 +3393,7 @@ func (x *GetShardEvacuationStatusRequest_Body) String() string { func (*GetShardEvacuationStatusRequest_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3332,7 +3444,7 @@ type GetShardEvacuationStatusResponse_Body struct { func (x *GetShardEvacuationStatusResponse_Body) Reset() { *x = GetShardEvacuationStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3345,7 +3457,7 @@ func (x *GetShardEvacuationStatusResponse_Body) String() string { func (*GetShardEvacuationStatusResponse_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3457,7 +3569,7 @@ type GetShardEvacuationStatusResponse_Body_UnixTimestamp struct { func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) Reset() { *x = GetShardEvacuationStatusResponse_Body_UnixTimestamp{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3470,7 +3582,7 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) String() string { func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3505,7 +3617,7 @@ type GetShardEvacuationStatusResponse_Body_Duration struct { func (x *GetShardEvacuationStatusResponse_Body_Duration) Reset() { *x = GetShardEvacuationStatusResponse_Body_Duration{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3518,7 +3630,7 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) String() string { func (*GetShardEvacuationStatusResponse_Body_Duration) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_Duration) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3541,6 +3653,82 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) GetSeconds() int64 { return 0 } +type ResetShardEvacuationStatusRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ResetShardEvacuationStatusRequest_Body) Reset() { + *x = ResetShardEvacuationStatusRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[64] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ResetShardEvacuationStatusRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResetShardEvacuationStatusRequest_Body) ProtoMessage() {} + +func (x *ResetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[64] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResetShardEvacuationStatusRequest_Body.ProtoReflect.Descriptor instead. +func (*ResetShardEvacuationStatusRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22, 0} +} + +type ResetShardEvacuationStatusResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ResetShardEvacuationStatusResponse_Body) Reset() { + *x = ResetShardEvacuationStatusResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[65] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ResetShardEvacuationStatusResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResetShardEvacuationStatusResponse_Body) ProtoMessage() {} + +func (x *ResetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[65] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResetShardEvacuationStatusResponse_Body.ProtoReflect.Descriptor instead. +func (*ResetShardEvacuationStatusResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0} +} + // Request body structure. type StopShardEvacuationRequest_Body struct { state protoimpl.MessageState @@ -3551,7 +3739,7 @@ type StopShardEvacuationRequest_Body struct { func (x *StopShardEvacuationRequest_Body) Reset() { *x = StopShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3564,7 +3752,7 @@ func (x *StopShardEvacuationRequest_Body) String() string { func (*StopShardEvacuationRequest_Body) ProtoMessage() {} func (x *StopShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[66] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3577,7 +3765,7 @@ func (x *StopShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShardEvacuationRequest_Body.ProtoReflect.Descriptor instead. func (*StopShardEvacuationRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24, 0} } // Response body structure. @@ -3590,7 +3778,7 @@ type StopShardEvacuationResponse_Body struct { func (x *StopShardEvacuationResponse_Body) Reset() { *x = StopShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3603,7 +3791,7 @@ func (x *StopShardEvacuationResponse_Body) String() string { func (*StopShardEvacuationResponse_Body) ProtoMessage() {} func (x *StopShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[67] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3616,7 +3804,7 @@ func (x *StopShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShardEvacuationResponse_Body.ProtoReflect.Descriptor instead. func (*StopShardEvacuationResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25, 0} } type AddChainLocalOverrideRequest_Body struct { @@ -3635,7 +3823,7 @@ type AddChainLocalOverrideRequest_Body struct { func (x *AddChainLocalOverrideRequest_Body) Reset() { *x = AddChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3648,7 +3836,7 @@ func (x *AddChainLocalOverrideRequest_Body) String() string { func (*AddChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *AddChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[68] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3661,7 +3849,7 @@ func (x *AddChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message // Deprecated: Use AddChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. func (*AddChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26, 0} } func (x *AddChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { @@ -3692,7 +3880,7 @@ type AddChainLocalOverrideResponse_Body struct { func (x *AddChainLocalOverrideResponse_Body) Reset() { *x = AddChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3705,7 +3893,7 @@ func (x *AddChainLocalOverrideResponse_Body) String() string { func (*AddChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *AddChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[69] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3718,7 +3906,7 @@ func (x *AddChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message // Deprecated: Use AddChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. func (*AddChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27, 0} } func (x *AddChainLocalOverrideResponse_Body) GetChainId() []byte { @@ -3742,7 +3930,7 @@ type GetChainLocalOverrideRequest_Body struct { func (x *GetChainLocalOverrideRequest_Body) Reset() { *x = GetChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[66] + mi := &file_pkg_services_control_service_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3755,7 +3943,7 @@ func (x *GetChainLocalOverrideRequest_Body) String() string { func (*GetChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *GetChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[66] + mi := &file_pkg_services_control_service_proto_msgTypes[70] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3768,7 +3956,7 @@ func (x *GetChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message // Deprecated: Use GetChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. func (*GetChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28, 0} } func (x *GetChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { @@ -3797,7 +3985,7 @@ type GetChainLocalOverrideResponse_Body struct { func (x *GetChainLocalOverrideResponse_Body) Reset() { *x = GetChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[67] + mi := &file_pkg_services_control_service_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3810,7 +3998,7 @@ func (x *GetChainLocalOverrideResponse_Body) String() string { func (*GetChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *GetChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[67] + mi := &file_pkg_services_control_service_proto_msgTypes[71] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3823,7 +4011,7 @@ func (x *GetChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message // Deprecated: Use GetChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. func (*GetChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29, 0} } func (x *GetChainLocalOverrideResponse_Body) GetChain() []byte { @@ -3845,7 +4033,7 @@ type ListChainLocalOverridesRequest_Body struct { func (x *ListChainLocalOverridesRequest_Body) Reset() { *x = ListChainLocalOverridesRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[68] + mi := &file_pkg_services_control_service_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3858,7 +4046,7 @@ func (x *ListChainLocalOverridesRequest_Body) String() string { func (*ListChainLocalOverridesRequest_Body) ProtoMessage() {} func (x *ListChainLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[68] + mi := &file_pkg_services_control_service_proto_msgTypes[72] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3871,7 +4059,7 @@ func (x *ListChainLocalOverridesRequest_Body) ProtoReflect() protoreflect.Messag // Deprecated: Use ListChainLocalOverridesRequest_Body.ProtoReflect.Descriptor instead. func (*ListChainLocalOverridesRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30, 0} } func (x *ListChainLocalOverridesRequest_Body) GetTarget() *ChainTarget { @@ -3893,7 +4081,7 @@ type ListChainLocalOverridesResponse_Body struct { func (x *ListChainLocalOverridesResponse_Body) Reset() { *x = ListChainLocalOverridesResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[69] + mi := &file_pkg_services_control_service_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3906,7 +4094,7 @@ func (x *ListChainLocalOverridesResponse_Body) String() string { func (*ListChainLocalOverridesResponse_Body) ProtoMessage() {} func (x *ListChainLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[69] + mi := &file_pkg_services_control_service_proto_msgTypes[73] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3919,7 +4107,7 @@ func (x *ListChainLocalOverridesResponse_Body) ProtoReflect() protoreflect.Messa // Deprecated: Use ListChainLocalOverridesResponse_Body.ProtoReflect.Descriptor instead. func (*ListChainLocalOverridesResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31, 0} } func (x *ListChainLocalOverridesResponse_Body) GetChains() [][]byte { @@ -3941,7 +4129,7 @@ type ListTargetsLocalOverridesRequest_Body struct { func (x *ListTargetsLocalOverridesRequest_Body) Reset() { *x = ListTargetsLocalOverridesRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[70] + mi := &file_pkg_services_control_service_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3954,7 +4142,7 @@ func (x *ListTargetsLocalOverridesRequest_Body) String() string { func (*ListTargetsLocalOverridesRequest_Body) ProtoMessage() {} func (x *ListTargetsLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[70] + mi := &file_pkg_services_control_service_proto_msgTypes[74] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3967,7 +4155,7 @@ func (x *ListTargetsLocalOverridesRequest_Body) ProtoReflect() protoreflect.Mess // Deprecated: Use ListTargetsLocalOverridesRequest_Body.ProtoReflect.Descriptor instead. func (*ListTargetsLocalOverridesRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32, 0} } func (x *ListTargetsLocalOverridesRequest_Body) GetChainName() string { @@ -3989,7 +4177,7 @@ type ListTargetsLocalOverridesResponse_Body struct { func (x *ListTargetsLocalOverridesResponse_Body) Reset() { *x = ListTargetsLocalOverridesResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[71] + mi := &file_pkg_services_control_service_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4002,7 +4190,7 @@ func (x *ListTargetsLocalOverridesResponse_Body) String() string { func (*ListTargetsLocalOverridesResponse_Body) ProtoMessage() {} func (x *ListTargetsLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[71] + mi := &file_pkg_services_control_service_proto_msgTypes[75] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4015,7 +4203,7 @@ func (x *ListTargetsLocalOverridesResponse_Body) ProtoReflect() protoreflect.Mes // Deprecated: Use ListTargetsLocalOverridesResponse_Body.ProtoReflect.Descriptor instead. func (*ListTargetsLocalOverridesResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33, 0} } func (x *ListTargetsLocalOverridesResponse_Body) GetTargets() []*ChainTarget { @@ -4039,7 +4227,7 @@ type RemoveChainLocalOverrideRequest_Body struct { func (x *RemoveChainLocalOverrideRequest_Body) Reset() { *x = RemoveChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[72] + mi := &file_pkg_services_control_service_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4052,7 +4240,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) String() string { func (*RemoveChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[72] + mi := &file_pkg_services_control_service_proto_msgTypes[76] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4065,7 +4253,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Messa // Deprecated: Use RemoveChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34, 0} } func (x *RemoveChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { @@ -4091,7 +4279,7 @@ type RemoveChainLocalOverrideResponse_Body struct { func (x *RemoveChainLocalOverrideResponse_Body) Reset() { *x = RemoveChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[73] + mi := &file_pkg_services_control_service_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4104,7 +4292,7 @@ func (x *RemoveChainLocalOverrideResponse_Body) String() string { func (*RemoveChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[73] + mi := &file_pkg_services_control_service_proto_msgTypes[77] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4117,7 +4305,7 @@ func (x *RemoveChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Mess // Deprecated: Use RemoveChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35, 0} } // Request body structure. @@ -4135,7 +4323,7 @@ type SealWriteCacheRequest_Body struct { func (x *SealWriteCacheRequest_Body) Reset() { *x = SealWriteCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[74] + mi := &file_pkg_services_control_service_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4148,7 +4336,7 @@ func (x *SealWriteCacheRequest_Body) String() string { func (*SealWriteCacheRequest_Body) ProtoMessage() {} func (x *SealWriteCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[74] + mi := &file_pkg_services_control_service_proto_msgTypes[78] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4161,7 +4349,7 @@ func (x *SealWriteCacheRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheRequest_Body.ProtoReflect.Descriptor instead. func (*SealWriteCacheRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36, 0} } func (x *SealWriteCacheRequest_Body) GetShard_ID() [][]byte { @@ -4189,7 +4377,7 @@ type SealWriteCacheResponse_Body struct { func (x *SealWriteCacheResponse_Body) Reset() { *x = SealWriteCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[75] + mi := &file_pkg_services_control_service_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4202,7 +4390,7 @@ func (x *SealWriteCacheResponse_Body) String() string { func (*SealWriteCacheResponse_Body) ProtoMessage() {} func (x *SealWriteCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[75] + mi := &file_pkg_services_control_service_proto_msgTypes[79] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4215,7 +4403,7 @@ func (x *SealWriteCacheResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheResponse_Body.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37, 0} } func (x *SealWriteCacheResponse_Body) GetResults() []*SealWriteCacheResponse_Body_Status { @@ -4238,7 +4426,7 @@ type SealWriteCacheResponse_Body_Status struct { func (x *SealWriteCacheResponse_Body_Status) Reset() { *x = SealWriteCacheResponse_Body_Status{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[76] + mi := &file_pkg_services_control_service_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4251,7 +4439,7 @@ func (x *SealWriteCacheResponse_Body_Status) String() string { func (*SealWriteCacheResponse_Body_Status) ProtoMessage() {} func (x *SealWriteCacheResponse_Body_Status) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[76] + mi := &file_pkg_services_control_service_proto_msgTypes[80] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4264,7 +4452,7 @@ func (x *SealWriteCacheResponse_Body_Status) ProtoReflect() protoreflect.Message // Deprecated: Use SealWriteCacheResponse_Body_Status.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse_Body_Status) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35, 0, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37, 0, 0} } func (x *SealWriteCacheResponse_Body_Status) GetShard_ID() []byte { @@ -4299,7 +4487,7 @@ type DetachShardsRequest_Body struct { func (x *DetachShardsRequest_Body) Reset() { *x = DetachShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[77] + mi := &file_pkg_services_control_service_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4312,7 +4500,7 @@ func (x *DetachShardsRequest_Body) String() string { func (*DetachShardsRequest_Body) ProtoMessage() {} func (x *DetachShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[77] + mi := &file_pkg_services_control_service_proto_msgTypes[81] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4325,7 +4513,7 @@ func (x *DetachShardsRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsRequest_Body.ProtoReflect.Descriptor instead. func (*DetachShardsRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38, 0} } func (x *DetachShardsRequest_Body) GetShard_ID() [][]byte { @@ -4344,7 +4532,7 @@ type DetachShardsResponse_Body struct { func (x *DetachShardsResponse_Body) Reset() { *x = DetachShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[78] + mi := &file_pkg_services_control_service_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4357,7 +4545,7 @@ func (x *DetachShardsResponse_Body) String() string { func (*DetachShardsResponse_Body) ProtoMessage() {} func (x *DetachShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[78] + mi := &file_pkg_services_control_service_proto_msgTypes[82] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4370,7 +4558,7 @@ func (x *DetachShardsResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsResponse_Body.ProtoReflect.Descriptor instead. func (*DetachShardsResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39, 0} } var File_pkg_services_control_service_proto protoreflect.FileDescriptor @@ -4672,314 +4860,342 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, - 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0x94, 0x01, - 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, - 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xdc, 0x01, - 0x0a, 0x1c, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, - 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x4a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xb5, 0x01, 0x0a, - 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0xa2, 0x01, + 0x0a, 0x21, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, + 0x64, 0x79, 0x22, 0xa4, 0x01, 0x0a, 0x22, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, - 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, - 0x69, 0x6e, 0x49, 0x64, 0x22, 0xe1, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, - 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, - 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, + 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x94, 0x01, 0x0a, 0x1a, 0x53, 0x74, + 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x22, 0x96, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xdc, 0x01, 0x0a, 0x1c, 0x41, 0x64, + 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4a, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xb5, 0x01, 0x0a, 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xca, 0x01, 0x0a, 0x1e, - 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, 0x73, - 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, - 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x61, - 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x20, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x24, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, - 0x6d, 0x65, 0x22, 0xd2, 0x01, 0x0a, 0x21, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x36, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2e, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, - 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, - 0x64, 0x22, 0xa0, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, + 0x22, 0xe1, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, + 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x22, 0xca, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, - 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, - 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, - 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, - 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, - 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, - 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, - 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, - 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, - 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, - 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x64, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xca, 0x01, 0x0a, 0x1e, 0x4c, 0x69, 0x73, 0x74, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, - 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, - 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x34, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x22, 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1e, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x22, 0xbe, 0x01, + 0x0a, 0x20, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x24, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xd2, + 0x01, 0x0a, 0x21, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x36, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x2e, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x73, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xa0, 0x01, + 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x22, 0xca, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, + 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, + 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, + 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0xa9, 0x02, + 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb1, 0x0d, 0x0a, - 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, - 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, - 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, - 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, - 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, - 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, - 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, - 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, - 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, - 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, - 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, - 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, - 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, - 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, - 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, - 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, - 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, + 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, + 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, + 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, + 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, - 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, - 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, - 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, - 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, + 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, + 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, + 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, + 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xa8, 0x0e, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, + 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, + 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, + 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, + 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, + 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, + 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, + 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, + 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, + 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, + 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, + 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, + 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, + 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, + 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, + 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, + 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, + 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4995,7 +5211,7 @@ func file_pkg_services_control_service_proto_rawDescGZIP() []byte { } var file_pkg_services_control_service_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 79) +var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 83) var file_pkg_services_control_service_proto_goTypes = []interface{}{ (StartShardEvacuationRequest_Body_Scope)(0), // 0: control.StartShardEvacuationRequest.Body.Scope (GetShardEvacuationStatusResponse_Body_Status)(0), // 1: control.GetShardEvacuationStatusResponse.Body.Status @@ -5021,204 +5237,214 @@ var file_pkg_services_control_service_proto_goTypes = []interface{}{ (*StartShardEvacuationResponse)(nil), // 21: control.StartShardEvacuationResponse (*GetShardEvacuationStatusRequest)(nil), // 22: control.GetShardEvacuationStatusRequest (*GetShardEvacuationStatusResponse)(nil), // 23: control.GetShardEvacuationStatusResponse - (*StopShardEvacuationRequest)(nil), // 24: control.StopShardEvacuationRequest - (*StopShardEvacuationResponse)(nil), // 25: control.StopShardEvacuationResponse - (*AddChainLocalOverrideRequest)(nil), // 26: control.AddChainLocalOverrideRequest - (*AddChainLocalOverrideResponse)(nil), // 27: control.AddChainLocalOverrideResponse - (*GetChainLocalOverrideRequest)(nil), // 28: control.GetChainLocalOverrideRequest - (*GetChainLocalOverrideResponse)(nil), // 29: control.GetChainLocalOverrideResponse - (*ListChainLocalOverridesRequest)(nil), // 30: control.ListChainLocalOverridesRequest - (*ListChainLocalOverridesResponse)(nil), // 31: control.ListChainLocalOverridesResponse - (*ListTargetsLocalOverridesRequest)(nil), // 32: control.ListTargetsLocalOverridesRequest - (*ListTargetsLocalOverridesResponse)(nil), // 33: control.ListTargetsLocalOverridesResponse - (*RemoveChainLocalOverrideRequest)(nil), // 34: control.RemoveChainLocalOverrideRequest - (*RemoveChainLocalOverrideResponse)(nil), // 35: control.RemoveChainLocalOverrideResponse - (*SealWriteCacheRequest)(nil), // 36: control.SealWriteCacheRequest - (*SealWriteCacheResponse)(nil), // 37: control.SealWriteCacheResponse - (*DetachShardsRequest)(nil), // 38: control.DetachShardsRequest - (*DetachShardsResponse)(nil), // 39: control.DetachShardsResponse - (*HealthCheckRequest_Body)(nil), // 40: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 41: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 42: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 43: control.SetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 44: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 45: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 46: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 47: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 48: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 49: control.SetShardModeResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 50: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 51: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 52: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 53: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 54: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 55: control.FlushCacheResponse.Body - (*DoctorRequest_Body)(nil), // 56: control.DoctorRequest.Body - (*DoctorResponse_Body)(nil), // 57: control.DoctorResponse.Body - (*StartShardEvacuationRequest_Body)(nil), // 58: control.StartShardEvacuationRequest.Body - (*StartShardEvacuationResponse_Body)(nil), // 59: control.StartShardEvacuationResponse.Body - (*GetShardEvacuationStatusRequest_Body)(nil), // 60: control.GetShardEvacuationStatusRequest.Body - (*GetShardEvacuationStatusResponse_Body)(nil), // 61: control.GetShardEvacuationStatusResponse.Body - (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 62: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 63: control.GetShardEvacuationStatusResponse.Body.Duration - (*StopShardEvacuationRequest_Body)(nil), // 64: control.StopShardEvacuationRequest.Body - (*StopShardEvacuationResponse_Body)(nil), // 65: control.StopShardEvacuationResponse.Body - (*AddChainLocalOverrideRequest_Body)(nil), // 66: control.AddChainLocalOverrideRequest.Body - (*AddChainLocalOverrideResponse_Body)(nil), // 67: control.AddChainLocalOverrideResponse.Body - (*GetChainLocalOverrideRequest_Body)(nil), // 68: control.GetChainLocalOverrideRequest.Body - (*GetChainLocalOverrideResponse_Body)(nil), // 69: control.GetChainLocalOverrideResponse.Body - (*ListChainLocalOverridesRequest_Body)(nil), // 70: control.ListChainLocalOverridesRequest.Body - (*ListChainLocalOverridesResponse_Body)(nil), // 71: control.ListChainLocalOverridesResponse.Body - (*ListTargetsLocalOverridesRequest_Body)(nil), // 72: control.ListTargetsLocalOverridesRequest.Body - (*ListTargetsLocalOverridesResponse_Body)(nil), // 73: control.ListTargetsLocalOverridesResponse.Body - (*RemoveChainLocalOverrideRequest_Body)(nil), // 74: control.RemoveChainLocalOverrideRequest.Body - (*RemoveChainLocalOverrideResponse_Body)(nil), // 75: control.RemoveChainLocalOverrideResponse.Body - (*SealWriteCacheRequest_Body)(nil), // 76: control.SealWriteCacheRequest.Body - (*SealWriteCacheResponse_Body)(nil), // 77: control.SealWriteCacheResponse.Body - (*SealWriteCacheResponse_Body_Status)(nil), // 78: control.SealWriteCacheResponse.Body.Status - (*DetachShardsRequest_Body)(nil), // 79: control.DetachShardsRequest.Body - (*DetachShardsResponse_Body)(nil), // 80: control.DetachShardsResponse.Body - (*Signature)(nil), // 81: control.Signature - (NetmapStatus)(0), // 82: control.NetmapStatus - (HealthStatus)(0), // 83: control.HealthStatus - (*ShardInfo)(nil), // 84: control.ShardInfo - (ShardMode)(0), // 85: control.ShardMode - (*ChainTarget)(nil), // 86: control.ChainTarget + (*ResetShardEvacuationStatusRequest)(nil), // 24: control.ResetShardEvacuationStatusRequest + (*ResetShardEvacuationStatusResponse)(nil), // 25: control.ResetShardEvacuationStatusResponse + (*StopShardEvacuationRequest)(nil), // 26: control.StopShardEvacuationRequest + (*StopShardEvacuationResponse)(nil), // 27: control.StopShardEvacuationResponse + (*AddChainLocalOverrideRequest)(nil), // 28: control.AddChainLocalOverrideRequest + (*AddChainLocalOverrideResponse)(nil), // 29: control.AddChainLocalOverrideResponse + (*GetChainLocalOverrideRequest)(nil), // 30: control.GetChainLocalOverrideRequest + (*GetChainLocalOverrideResponse)(nil), // 31: control.GetChainLocalOverrideResponse + (*ListChainLocalOverridesRequest)(nil), // 32: control.ListChainLocalOverridesRequest + (*ListChainLocalOverridesResponse)(nil), // 33: control.ListChainLocalOverridesResponse + (*ListTargetsLocalOverridesRequest)(nil), // 34: control.ListTargetsLocalOverridesRequest + (*ListTargetsLocalOverridesResponse)(nil), // 35: control.ListTargetsLocalOverridesResponse + (*RemoveChainLocalOverrideRequest)(nil), // 36: control.RemoveChainLocalOverrideRequest + (*RemoveChainLocalOverrideResponse)(nil), // 37: control.RemoveChainLocalOverrideResponse + (*SealWriteCacheRequest)(nil), // 38: control.SealWriteCacheRequest + (*SealWriteCacheResponse)(nil), // 39: control.SealWriteCacheResponse + (*DetachShardsRequest)(nil), // 40: control.DetachShardsRequest + (*DetachShardsResponse)(nil), // 41: control.DetachShardsResponse + (*HealthCheckRequest_Body)(nil), // 42: control.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 43: control.HealthCheckResponse.Body + (*SetNetmapStatusRequest_Body)(nil), // 44: control.SetNetmapStatusRequest.Body + (*SetNetmapStatusResponse_Body)(nil), // 45: control.SetNetmapStatusResponse.Body + (*DropObjectsRequest_Body)(nil), // 46: control.DropObjectsRequest.Body + (*DropObjectsResponse_Body)(nil), // 47: control.DropObjectsResponse.Body + (*ListShardsRequest_Body)(nil), // 48: control.ListShardsRequest.Body + (*ListShardsResponse_Body)(nil), // 49: control.ListShardsResponse.Body + (*SetShardModeRequest_Body)(nil), // 50: control.SetShardModeRequest.Body + (*SetShardModeResponse_Body)(nil), // 51: control.SetShardModeResponse.Body + (*SynchronizeTreeRequest_Body)(nil), // 52: control.SynchronizeTreeRequest.Body + (*SynchronizeTreeResponse_Body)(nil), // 53: control.SynchronizeTreeResponse.Body + (*EvacuateShardRequest_Body)(nil), // 54: control.EvacuateShardRequest.Body + (*EvacuateShardResponse_Body)(nil), // 55: control.EvacuateShardResponse.Body + (*FlushCacheRequest_Body)(nil), // 56: control.FlushCacheRequest.Body + (*FlushCacheResponse_Body)(nil), // 57: control.FlushCacheResponse.Body + (*DoctorRequest_Body)(nil), // 58: control.DoctorRequest.Body + (*DoctorResponse_Body)(nil), // 59: control.DoctorResponse.Body + (*StartShardEvacuationRequest_Body)(nil), // 60: control.StartShardEvacuationRequest.Body + (*StartShardEvacuationResponse_Body)(nil), // 61: control.StartShardEvacuationResponse.Body + (*GetShardEvacuationStatusRequest_Body)(nil), // 62: control.GetShardEvacuationStatusRequest.Body + (*GetShardEvacuationStatusResponse_Body)(nil), // 63: control.GetShardEvacuationStatusResponse.Body + (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 64: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 65: control.GetShardEvacuationStatusResponse.Body.Duration + (*ResetShardEvacuationStatusRequest_Body)(nil), // 66: control.ResetShardEvacuationStatusRequest.Body + (*ResetShardEvacuationStatusResponse_Body)(nil), // 67: control.ResetShardEvacuationStatusResponse.Body + (*StopShardEvacuationRequest_Body)(nil), // 68: control.StopShardEvacuationRequest.Body + (*StopShardEvacuationResponse_Body)(nil), // 69: control.StopShardEvacuationResponse.Body + (*AddChainLocalOverrideRequest_Body)(nil), // 70: control.AddChainLocalOverrideRequest.Body + (*AddChainLocalOverrideResponse_Body)(nil), // 71: control.AddChainLocalOverrideResponse.Body + (*GetChainLocalOverrideRequest_Body)(nil), // 72: control.GetChainLocalOverrideRequest.Body + (*GetChainLocalOverrideResponse_Body)(nil), // 73: control.GetChainLocalOverrideResponse.Body + (*ListChainLocalOverridesRequest_Body)(nil), // 74: control.ListChainLocalOverridesRequest.Body + (*ListChainLocalOverridesResponse_Body)(nil), // 75: control.ListChainLocalOverridesResponse.Body + (*ListTargetsLocalOverridesRequest_Body)(nil), // 76: control.ListTargetsLocalOverridesRequest.Body + (*ListTargetsLocalOverridesResponse_Body)(nil), // 77: control.ListTargetsLocalOverridesResponse.Body + (*RemoveChainLocalOverrideRequest_Body)(nil), // 78: control.RemoveChainLocalOverrideRequest.Body + (*RemoveChainLocalOverrideResponse_Body)(nil), // 79: control.RemoveChainLocalOverrideResponse.Body + (*SealWriteCacheRequest_Body)(nil), // 80: control.SealWriteCacheRequest.Body + (*SealWriteCacheResponse_Body)(nil), // 81: control.SealWriteCacheResponse.Body + (*SealWriteCacheResponse_Body_Status)(nil), // 82: control.SealWriteCacheResponse.Body.Status + (*DetachShardsRequest_Body)(nil), // 83: control.DetachShardsRequest.Body + (*DetachShardsResponse_Body)(nil), // 84: control.DetachShardsResponse.Body + (*Signature)(nil), // 85: control.Signature + (NetmapStatus)(0), // 86: control.NetmapStatus + (HealthStatus)(0), // 87: control.HealthStatus + (*ShardInfo)(nil), // 88: control.ShardInfo + (ShardMode)(0), // 89: control.ShardMode + (*ChainTarget)(nil), // 90: control.ChainTarget } var file_pkg_services_control_service_proto_depIdxs = []int32{ - 40, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 81, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 41, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 81, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 42, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 81, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 43, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 81, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 44, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 81, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature - 45, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 81, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature - 46, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 81, // 13: control.ListShardsRequest.signature:type_name -> control.Signature - 47, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 81, // 15: control.ListShardsResponse.signature:type_name -> control.Signature - 48, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 81, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature - 49, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 81, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature - 50, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 81, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 51, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 81, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 52, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 81, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature - 53, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 81, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature - 54, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 81, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature - 55, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 81, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature - 56, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body - 81, // 33: control.DoctorRequest.signature:type_name -> control.Signature - 57, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body - 81, // 35: control.DoctorResponse.signature:type_name -> control.Signature - 58, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body - 81, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature - 59, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body - 81, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature - 60, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body - 81, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 61, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body - 81, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 64, // 44: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body - 81, // 45: control.StopShardEvacuationRequest.signature:type_name -> control.Signature - 65, // 46: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body - 81, // 47: control.StopShardEvacuationResponse.signature:type_name -> control.Signature - 66, // 48: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body - 81, // 49: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature - 67, // 50: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body - 81, // 51: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature - 68, // 52: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body - 81, // 53: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature - 69, // 54: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body - 81, // 55: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature - 70, // 56: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body - 81, // 57: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature - 71, // 58: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body - 81, // 59: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature - 72, // 60: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body - 81, // 61: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature - 73, // 62: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body - 81, // 63: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature - 74, // 64: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body - 81, // 65: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature - 75, // 66: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body - 81, // 67: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature - 76, // 68: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body - 81, // 69: control.SealWriteCacheRequest.signature:type_name -> control.Signature - 77, // 70: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body - 81, // 71: control.SealWriteCacheResponse.signature:type_name -> control.Signature - 79, // 72: control.DetachShardsRequest.body:type_name -> control.DetachShardsRequest.Body - 81, // 73: control.DetachShardsRequest.signature:type_name -> control.Signature - 80, // 74: control.DetachShardsResponse.body:type_name -> control.DetachShardsResponse.Body - 81, // 75: control.DetachShardsResponse.signature:type_name -> control.Signature - 82, // 76: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 83, // 77: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 82, // 78: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 84, // 79: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 85, // 80: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 1, // 81: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status - 63, // 82: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration - 62, // 83: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - 86, // 84: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 86, // 85: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 86, // 86: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget - 86, // 87: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget - 86, // 88: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 78, // 89: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status - 2, // 90: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 4, // 91: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 6, // 92: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 8, // 93: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 10, // 94: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 12, // 95: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 14, // 96: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 20, // 97: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest - 22, // 98: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest - 24, // 99: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest - 16, // 100: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 18, // 101: control.ControlService.Doctor:input_type -> control.DoctorRequest - 26, // 102: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest - 28, // 103: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest - 30, // 104: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest - 34, // 105: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest - 32, // 106: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest - 36, // 107: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest - 38, // 108: control.ControlService.DetachShards:input_type -> control.DetachShardsRequest - 3, // 109: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 5, // 110: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 7, // 111: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 9, // 112: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 11, // 113: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 13, // 114: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 15, // 115: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 21, // 116: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse - 23, // 117: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse - 25, // 118: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse - 17, // 119: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 19, // 120: control.ControlService.Doctor:output_type -> control.DoctorResponse - 27, // 121: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse - 29, // 122: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse - 31, // 123: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse - 35, // 124: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse - 33, // 125: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse - 37, // 126: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse - 39, // 127: control.ControlService.DetachShards:output_type -> control.DetachShardsResponse - 109, // [109:128] is the sub-list for method output_type - 90, // [90:109] is the sub-list for method input_type - 90, // [90:90] is the sub-list for extension type_name - 90, // [90:90] is the sub-list for extension extendee - 0, // [0:90] is the sub-list for field type_name + 42, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body + 85, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature + 43, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body + 85, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature + 44, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body + 85, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature + 45, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body + 85, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature + 46, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body + 85, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature + 47, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body + 85, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature + 48, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body + 85, // 13: control.ListShardsRequest.signature:type_name -> control.Signature + 49, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body + 85, // 15: control.ListShardsResponse.signature:type_name -> control.Signature + 50, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body + 85, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature + 51, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body + 85, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature + 52, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body + 85, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature + 53, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body + 85, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature + 54, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body + 85, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature + 55, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body + 85, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature + 56, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body + 85, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature + 57, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body + 85, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature + 58, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body + 85, // 33: control.DoctorRequest.signature:type_name -> control.Signature + 59, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body + 85, // 35: control.DoctorResponse.signature:type_name -> control.Signature + 60, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body + 85, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature + 61, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body + 85, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature + 62, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body + 85, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 63, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body + 85, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 66, // 44: control.ResetShardEvacuationStatusRequest.body:type_name -> control.ResetShardEvacuationStatusRequest.Body + 85, // 45: control.ResetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 67, // 46: control.ResetShardEvacuationStatusResponse.body:type_name -> control.ResetShardEvacuationStatusResponse.Body + 85, // 47: control.ResetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 68, // 48: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body + 85, // 49: control.StopShardEvacuationRequest.signature:type_name -> control.Signature + 69, // 50: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body + 85, // 51: control.StopShardEvacuationResponse.signature:type_name -> control.Signature + 70, // 52: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body + 85, // 53: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature + 71, // 54: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body + 85, // 55: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature + 72, // 56: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body + 85, // 57: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature + 73, // 58: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body + 85, // 59: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature + 74, // 60: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body + 85, // 61: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature + 75, // 62: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body + 85, // 63: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature + 76, // 64: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body + 85, // 65: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature + 77, // 66: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body + 85, // 67: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature + 78, // 68: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body + 85, // 69: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature + 79, // 70: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body + 85, // 71: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature + 80, // 72: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body + 85, // 73: control.SealWriteCacheRequest.signature:type_name -> control.Signature + 81, // 74: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body + 85, // 75: control.SealWriteCacheResponse.signature:type_name -> control.Signature + 83, // 76: control.DetachShardsRequest.body:type_name -> control.DetachShardsRequest.Body + 85, // 77: control.DetachShardsRequest.signature:type_name -> control.Signature + 84, // 78: control.DetachShardsResponse.body:type_name -> control.DetachShardsResponse.Body + 85, // 79: control.DetachShardsResponse.signature:type_name -> control.Signature + 86, // 80: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus + 87, // 81: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus + 86, // 82: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus + 88, // 83: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo + 89, // 84: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode + 1, // 85: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status + 65, // 86: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration + 64, // 87: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + 90, // 88: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 90, // 89: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 90, // 90: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget + 90, // 91: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget + 90, // 92: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 82, // 93: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status + 2, // 94: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 4, // 95: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 6, // 96: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 8, // 97: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 10, // 98: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 12, // 99: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 14, // 100: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 20, // 101: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest + 22, // 102: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest + 24, // 103: control.ControlService.ResetShardEvacuationStatus:input_type -> control.ResetShardEvacuationStatusRequest + 26, // 104: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest + 16, // 105: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 18, // 106: control.ControlService.Doctor:input_type -> control.DoctorRequest + 28, // 107: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest + 30, // 108: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest + 32, // 109: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest + 36, // 110: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest + 34, // 111: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest + 38, // 112: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest + 40, // 113: control.ControlService.DetachShards:input_type -> control.DetachShardsRequest + 3, // 114: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 5, // 115: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 7, // 116: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 9, // 117: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 11, // 118: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 13, // 119: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 15, // 120: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 21, // 121: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse + 23, // 122: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse + 25, // 123: control.ControlService.ResetShardEvacuationStatus:output_type -> control.ResetShardEvacuationStatusResponse + 27, // 124: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse + 17, // 125: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 19, // 126: control.ControlService.Doctor:output_type -> control.DoctorResponse + 29, // 127: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse + 31, // 128: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse + 33, // 129: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse + 37, // 130: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse + 35, // 131: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse + 39, // 132: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse + 41, // 133: control.ControlService.DetachShards:output_type -> control.DetachShardsResponse + 114, // [114:134] is the sub-list for method output_type + 94, // [94:114] is the sub-list for method input_type + 94, // [94:94] is the sub-list for extension type_name + 94, // [94:94] is the sub-list for extension extendee + 0, // [0:94] is the sub-list for field type_name } func init() { file_pkg_services_control_service_proto_init() } @@ -5493,7 +5719,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationRequest); i { + switch v := v.(*ResetShardEvacuationStatusRequest); i { case 0: return &v.state case 1: @@ -5505,7 +5731,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationResponse); i { + switch v := v.(*ResetShardEvacuationStatusResponse); i { case 0: return &v.state case 1: @@ -5517,7 +5743,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideRequest); i { + switch v := v.(*StopShardEvacuationRequest); i { case 0: return &v.state case 1: @@ -5529,7 +5755,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideResponse); i { + switch v := v.(*StopShardEvacuationResponse); i { case 0: return &v.state case 1: @@ -5541,7 +5767,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideRequest); i { + switch v := v.(*AddChainLocalOverrideRequest); i { case 0: return &v.state case 1: @@ -5553,7 +5779,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideResponse); i { + switch v := v.(*AddChainLocalOverrideResponse); i { case 0: return &v.state case 1: @@ -5565,7 +5791,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesRequest); i { + switch v := v.(*GetChainLocalOverrideRequest); i { case 0: return &v.state case 1: @@ -5577,7 +5803,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesResponse); i { + switch v := v.(*GetChainLocalOverrideResponse); i { case 0: return &v.state case 1: @@ -5589,7 +5815,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesRequest); i { + switch v := v.(*ListChainLocalOverridesRequest); i { case 0: return &v.state case 1: @@ -5601,7 +5827,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesResponse); i { + switch v := v.(*ListChainLocalOverridesResponse); i { case 0: return &v.state case 1: @@ -5613,7 +5839,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideRequest); i { + switch v := v.(*ListTargetsLocalOverridesRequest); i { case 0: return &v.state case 1: @@ -5625,7 +5851,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideResponse); i { + switch v := v.(*ListTargetsLocalOverridesResponse); i { case 0: return &v.state case 1: @@ -5637,7 +5863,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheRequest); i { + switch v := v.(*RemoveChainLocalOverrideRequest); i { case 0: return &v.state case 1: @@ -5649,7 +5875,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse); i { + switch v := v.(*RemoveChainLocalOverrideResponse); i { case 0: return &v.state case 1: @@ -5661,7 +5887,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsRequest); i { + switch v := v.(*SealWriteCacheRequest); i { case 0: return &v.state case 1: @@ -5673,7 +5899,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsResponse); i { + switch v := v.(*SealWriteCacheResponse); i { case 0: return &v.state case 1: @@ -5685,7 +5911,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*DetachShardsRequest); i { case 0: return &v.state case 1: @@ -5697,7 +5923,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { + switch v := v.(*DetachShardsResponse); i { case 0: return &v.state case 1: @@ -5709,7 +5935,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusRequest_Body); i { + switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state case 1: @@ -5721,7 +5947,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusResponse_Body); i { + switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state case 1: @@ -5733,7 +5959,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest_Body); i { + switch v := v.(*SetNetmapStatusRequest_Body); i { case 0: return &v.state case 1: @@ -5745,7 +5971,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse_Body); i { + switch v := v.(*SetNetmapStatusResponse_Body); i { case 0: return &v.state case 1: @@ -5757,7 +5983,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest_Body); i { + switch v := v.(*DropObjectsRequest_Body); i { case 0: return &v.state case 1: @@ -5769,7 +5995,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse_Body); i { + switch v := v.(*DropObjectsResponse_Body); i { case 0: return &v.state case 1: @@ -5781,7 +6007,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest_Body); i { + switch v := v.(*ListShardsRequest_Body); i { case 0: return &v.state case 1: @@ -5793,7 +6019,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse_Body); i { + switch v := v.(*ListShardsResponse_Body); i { case 0: return &v.state case 1: @@ -5805,7 +6031,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest_Body); i { + switch v := v.(*SetShardModeRequest_Body); i { case 0: return &v.state case 1: @@ -5817,7 +6043,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse_Body); i { + switch v := v.(*SetShardModeResponse_Body); i { case 0: return &v.state case 1: @@ -5829,7 +6055,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest_Body); i { + switch v := v.(*SynchronizeTreeRequest_Body); i { case 0: return &v.state case 1: @@ -5841,7 +6067,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse_Body); i { + switch v := v.(*SynchronizeTreeResponse_Body); i { case 0: return &v.state case 1: @@ -5853,7 +6079,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest_Body); i { + switch v := v.(*EvacuateShardRequest_Body); i { case 0: return &v.state case 1: @@ -5865,7 +6091,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheResponse_Body); i { + switch v := v.(*EvacuateShardResponse_Body); i { case 0: return &v.state case 1: @@ -5877,7 +6103,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorRequest_Body); i { + switch v := v.(*FlushCacheRequest_Body); i { case 0: return &v.state case 1: @@ -5889,7 +6115,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorResponse_Body); i { + switch v := v.(*FlushCacheResponse_Body); i { case 0: return &v.state case 1: @@ -5901,7 +6127,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationRequest_Body); i { + switch v := v.(*DoctorRequest_Body); i { case 0: return &v.state case 1: @@ -5913,7 +6139,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationResponse_Body); i { + switch v := v.(*DoctorResponse_Body); i { case 0: return &v.state case 1: @@ -5925,7 +6151,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusRequest_Body); i { + switch v := v.(*StartShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -5937,7 +6163,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body); i { + switch v := v.(*StartShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -5949,7 +6175,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { + switch v := v.(*GetShardEvacuationStatusRequest_Body); i { case 0: return &v.state case 1: @@ -5961,7 +6187,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body); i { case 0: return &v.state case 1: @@ -5973,7 +6199,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationRequest_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { case 0: return &v.state case 1: @@ -5985,7 +6211,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationResponse_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { case 0: return &v.state case 1: @@ -5997,7 +6223,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideRequest_Body); i { + switch v := v.(*ResetShardEvacuationStatusRequest_Body); i { case 0: return &v.state case 1: @@ -6009,7 +6235,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideResponse_Body); i { + switch v := v.(*ResetShardEvacuationStatusResponse_Body); i { case 0: return &v.state case 1: @@ -6021,7 +6247,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideRequest_Body); i { + switch v := v.(*StopShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -6033,7 +6259,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideResponse_Body); i { + switch v := v.(*StopShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -6045,7 +6271,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesRequest_Body); i { + switch v := v.(*AddChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -6057,7 +6283,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesResponse_Body); i { + switch v := v.(*AddChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -6069,7 +6295,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesRequest_Body); i { + switch v := v.(*GetChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -6081,7 +6307,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesResponse_Body); i { + switch v := v.(*GetChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -6093,7 +6319,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { + switch v := v.(*ListChainLocalOverridesRequest_Body); i { case 0: return &v.state case 1: @@ -6105,7 +6331,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { + switch v := v.(*ListChainLocalOverridesResponse_Body); i { case 0: return &v.state case 1: @@ -6117,7 +6343,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheRequest_Body); i { + switch v := v.(*ListTargetsLocalOverridesRequest_Body); i { case 0: return &v.state case 1: @@ -6129,7 +6355,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse_Body); i { + switch v := v.(*ListTargetsLocalOverridesResponse_Body); i { case 0: return &v.state case 1: @@ -6141,7 +6367,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse_Body_Status); i { + switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -6153,7 +6379,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[77].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsRequest_Body); i { + switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -6165,6 +6391,54 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[78].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[79].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[80].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheResponse_Body_Status); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[81].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DetachShardsRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[82].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DetachShardsResponse_Body); i { case 0: return &v.state @@ -6183,7 +6457,7 @@ func file_pkg_services_control_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_service_proto_rawDesc, NumEnums: 2, - NumMessages: 79, + NumMessages: 83, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 1377965e6..8501e952f 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -36,6 +36,9 @@ service ControlService { // GetShardEvacuationStatus returns evacuation status. rpc GetShardEvacuationStatus (GetShardEvacuationStatusRequest) returns (GetShardEvacuationStatusResponse); + // ResetShardEvacuationStatus resets evacuation status if there is no running evacuation process. + rpc ResetShardEvacuationStatus (ResetShardEvacuationStatusRequest) returns (ResetShardEvacuationStatusResponse); + // StopShardEvacuation stops moving all data from one shard to the others. rpc StopShardEvacuation (StopShardEvacuationRequest) returns (StopShardEvacuationResponse); @@ -426,6 +429,22 @@ message GetShardEvacuationStatusResponse { Signature signature = 2; } +// ResetShardEvacuationStatus request. +message ResetShardEvacuationStatusRequest { + message Body {} + + Body body = 1; + Signature signature = 2; +} + +// ResetShardEvacuationStatus response. +message ResetShardEvacuationStatusResponse { + message Body {} + + Body body = 1; + Signature signature = 2; +} + // StopShardEvacuation request. message StopShardEvacuationRequest { // Request body structure. diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 56b2bd54c..b90749269 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -1916,6 +1916,160 @@ func (x *GetShardEvacuationStatusResponse) SetSignature(sig *Signature) { x.Signature = sig } +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ResetShardEvacuationStatusRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ResetShardEvacuationStatusRequest_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ResetShardEvacuationStatusRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ResetShardEvacuationStatusRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *ResetShardEvacuationStatusRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *ResetShardEvacuationStatusRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *ResetShardEvacuationStatusRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ResetShardEvacuationStatusResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ResetShardEvacuationStatusResponse_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ResetShardEvacuationStatusResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *ResetShardEvacuationStatusResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *ResetShardEvacuationStatusResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *ResetShardEvacuationStatusResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *ResetShardEvacuationStatusResponse) SetSignature(sig *Signature) { + x.Signature = sig +} + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 69029ccb3..ad33e330a 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -19,25 +19,26 @@ import ( const _ = grpc.SupportPackageIsVersion7 const ( - ControlService_HealthCheck_FullMethodName = "/control.ControlService/HealthCheck" - ControlService_SetNetmapStatus_FullMethodName = "/control.ControlService/SetNetmapStatus" - ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects" - ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" - ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" - ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree" - ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard" - ControlService_StartShardEvacuation_FullMethodName = "/control.ControlService/StartShardEvacuation" - ControlService_GetShardEvacuationStatus_FullMethodName = "/control.ControlService/GetShardEvacuationStatus" - ControlService_StopShardEvacuation_FullMethodName = "/control.ControlService/StopShardEvacuation" - ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache" - ControlService_Doctor_FullMethodName = "/control.ControlService/Doctor" - ControlService_AddChainLocalOverride_FullMethodName = "/control.ControlService/AddChainLocalOverride" - ControlService_GetChainLocalOverride_FullMethodName = "/control.ControlService/GetChainLocalOverride" - ControlService_ListChainLocalOverrides_FullMethodName = "/control.ControlService/ListChainLocalOverrides" - ControlService_RemoveChainLocalOverride_FullMethodName = "/control.ControlService/RemoveChainLocalOverride" - ControlService_ListTargetsLocalOverrides_FullMethodName = "/control.ControlService/ListTargetsLocalOverrides" - ControlService_SealWriteCache_FullMethodName = "/control.ControlService/SealWriteCache" - ControlService_DetachShards_FullMethodName = "/control.ControlService/DetachShards" + ControlService_HealthCheck_FullMethodName = "/control.ControlService/HealthCheck" + ControlService_SetNetmapStatus_FullMethodName = "/control.ControlService/SetNetmapStatus" + ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects" + ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" + ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" + ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree" + ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard" + ControlService_StartShardEvacuation_FullMethodName = "/control.ControlService/StartShardEvacuation" + ControlService_GetShardEvacuationStatus_FullMethodName = "/control.ControlService/GetShardEvacuationStatus" + ControlService_ResetShardEvacuationStatus_FullMethodName = "/control.ControlService/ResetShardEvacuationStatus" + ControlService_StopShardEvacuation_FullMethodName = "/control.ControlService/StopShardEvacuation" + ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache" + ControlService_Doctor_FullMethodName = "/control.ControlService/Doctor" + ControlService_AddChainLocalOverride_FullMethodName = "/control.ControlService/AddChainLocalOverride" + ControlService_GetChainLocalOverride_FullMethodName = "/control.ControlService/GetChainLocalOverride" + ControlService_ListChainLocalOverrides_FullMethodName = "/control.ControlService/ListChainLocalOverrides" + ControlService_RemoveChainLocalOverride_FullMethodName = "/control.ControlService/RemoveChainLocalOverride" + ControlService_ListTargetsLocalOverrides_FullMethodName = "/control.ControlService/ListTargetsLocalOverrides" + ControlService_SealWriteCache_FullMethodName = "/control.ControlService/SealWriteCache" + ControlService_DetachShards_FullMethodName = "/control.ControlService/DetachShards" ) // ControlServiceClient is the client API for ControlService service. @@ -63,6 +64,8 @@ type ControlServiceClient interface { StartShardEvacuation(ctx context.Context, in *StartShardEvacuationRequest, opts ...grpc.CallOption) (*StartShardEvacuationResponse, error) // GetShardEvacuationStatus returns evacuation status. GetShardEvacuationStatus(ctx context.Context, in *GetShardEvacuationStatusRequest, opts ...grpc.CallOption) (*GetShardEvacuationStatusResponse, error) + // ResetShardEvacuationStatus resets evacuation status if there is no running evacuation process. + ResetShardEvacuationStatus(ctx context.Context, in *ResetShardEvacuationStatusRequest, opts ...grpc.CallOption) (*ResetShardEvacuationStatusResponse, error) // StopShardEvacuation stops moving all data from one shard to the others. StopShardEvacuation(ctx context.Context, in *StopShardEvacuationRequest, opts ...grpc.CallOption) (*StopShardEvacuationResponse, error) // FlushCache moves all data from one shard to the others. @@ -174,6 +177,15 @@ func (c *controlServiceClient) GetShardEvacuationStatus(ctx context.Context, in return out, nil } +func (c *controlServiceClient) ResetShardEvacuationStatus(ctx context.Context, in *ResetShardEvacuationStatusRequest, opts ...grpc.CallOption) (*ResetShardEvacuationStatusResponse, error) { + out := new(ResetShardEvacuationStatusResponse) + err := c.cc.Invoke(ctx, ControlService_ResetShardEvacuationStatus_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *controlServiceClient) StopShardEvacuation(ctx context.Context, in *StopShardEvacuationRequest, opts ...grpc.CallOption) (*StopShardEvacuationResponse, error) { out := new(StopShardEvacuationResponse) err := c.cc.Invoke(ctx, ControlService_StopShardEvacuation_FullMethodName, in, out, opts...) @@ -287,6 +299,8 @@ type ControlServiceServer interface { StartShardEvacuation(context.Context, *StartShardEvacuationRequest) (*StartShardEvacuationResponse, error) // GetShardEvacuationStatus returns evacuation status. GetShardEvacuationStatus(context.Context, *GetShardEvacuationStatusRequest) (*GetShardEvacuationStatusResponse, error) + // ResetShardEvacuationStatus resets evacuation status if there is no running evacuation process. + ResetShardEvacuationStatus(context.Context, *ResetShardEvacuationStatusRequest) (*ResetShardEvacuationStatusResponse, error) // StopShardEvacuation stops moving all data from one shard to the others. StopShardEvacuation(context.Context, *StopShardEvacuationRequest) (*StopShardEvacuationResponse, error) // FlushCache moves all data from one shard to the others. @@ -340,6 +354,9 @@ func (UnimplementedControlServiceServer) StartShardEvacuation(context.Context, * func (UnimplementedControlServiceServer) GetShardEvacuationStatus(context.Context, *GetShardEvacuationStatusRequest) (*GetShardEvacuationStatusResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetShardEvacuationStatus not implemented") } +func (UnimplementedControlServiceServer) ResetShardEvacuationStatus(context.Context, *ResetShardEvacuationStatusRequest) (*ResetShardEvacuationStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ResetShardEvacuationStatus not implemented") +} func (UnimplementedControlServiceServer) StopShardEvacuation(context.Context, *StopShardEvacuationRequest) (*StopShardEvacuationResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method StopShardEvacuation not implemented") } @@ -544,6 +561,24 @@ func _ControlService_GetShardEvacuationStatus_Handler(srv interface{}, ctx conte return interceptor(ctx, in, info, handler) } +func _ControlService_ResetShardEvacuationStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ResetShardEvacuationStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).ResetShardEvacuationStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_ResetShardEvacuationStatus_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).ResetShardEvacuationStatus(ctx, req.(*ResetShardEvacuationStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _ControlService_StopShardEvacuation_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(StopShardEvacuationRequest) if err := dec(in); err != nil { @@ -767,6 +802,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetShardEvacuationStatus", Handler: _ControlService_GetShardEvacuationStatus_Handler, }, + { + MethodName: "ResetShardEvacuationStatus", + Handler: _ControlService_ResetShardEvacuationStatus_Handler, + }, { MethodName: "StopShardEvacuation", Handler: _ControlService_StopShardEvacuation_Handler, From 17f5463389f54f083b6ead323e80d61f137101b7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 13 Mar 2024 10:40:44 +0300 Subject: [PATCH 0977/1943] [#1043] cli: Add reset evacuation status command Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/evacuation.go | 32 ++++++++++++ docs/evacuation.md | 2 + pkg/services/control/rpc.go | 52 ++++++++++++------- 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 1d192aa50..6fa5ed75c 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -53,6 +53,13 @@ var stopEvacuationShardCmd = &cobra.Command{ Run: stopEvacuateShardStatus, } +var resetEvacuationStatusShardCmd = &cobra.Command{ + Use: "reset", + Short: "Reset evacuate objects from shard status", + Long: "Reset evacuate objects from shard to other shards status", + Run: resetEvacuateShardStatus, +} + func startEvacuateShard(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) @@ -152,6 +159,29 @@ func stopEvacuateShardStatus(cmd *cobra.Command, _ []string) { cmd.Println("Evacuation stopped.") } +func resetEvacuateShardStatus(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + req := &control.ResetShardEvacuationStatusRequest{ + Body: &control.ResetShardEvacuationStatusRequest_Body{}, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.ResetShardEvacuationStatusResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.ResetShardEvacuationStatus(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "Reset shards evacuation status failed, rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Shards evacuation status has been reset.") +} + func waitEvacuateCompletion(cmd *cobra.Command, pk *ecdsa.PrivateKey, cli *clientSDK.Client, printProgress, printCompleted bool) { const statusPollingInterval = 1 * time.Second const reportIntervalSeconds = 5 @@ -323,10 +353,12 @@ func initControlEvacuationShardCmd() { evacuationShardCmd.AddCommand(startEvacuationShardCmd) evacuationShardCmd.AddCommand(getEvacuationShardStatusCmd) evacuationShardCmd.AddCommand(stopEvacuationShardCmd) + evacuationShardCmd.AddCommand(resetEvacuationStatusShardCmd) initControlStartEvacuationShardCmd() initControlFlags(getEvacuationShardStatusCmd) initControlFlags(stopEvacuationShardCmd) + initControlFlags(resetEvacuationStatusShardCmd) } func initControlStartEvacuationShardCmd() { diff --git a/docs/evacuation.md b/docs/evacuation.md index 1bccc87bf..602e32e9b 100644 --- a/docs/evacuation.md +++ b/docs/evacuation.md @@ -20,6 +20,8 @@ Only one running evacuation process is allowed on the node at a time. `frostfs-cli control shards evacuation status` prints evacuation process status. +`frostfs-cli control shards evacuation reset` resets evacuation process status. + See commands `--help` output for detailed description. ## Examples diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 877bb63c2..b07b10274 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -8,25 +8,26 @@ import ( const serviceName = "control.ControlService" const ( - rpcHealthCheck = "HealthCheck" - rpcSetNetmapStatus = "SetNetmapStatus" - rpcDropObjects = "DropObjects" - rpcListShards = "ListShards" - rpcSetShardMode = "SetShardMode" - rpcSynchronizeTree = "SynchronizeTree" - rpcEvacuateShard = "EvacuateShard" - rpcStartShardEvacuation = "StartShardEvacuation" - rpcGetShardEvacuationStatus = "GetShardEvacuationStatus" - rpcStopShardEvacuation = "StopShardEvacuation" - rpcFlushCache = "FlushCache" - rpcDoctor = "Doctor" - rpcAddChainLocalOverride = "AddChainLocalOverride" - rpcGetChainLocalOverride = "GetChainLocalOverride" - rpcListChainLocalOverrides = "ListChainLocalOverrides" - rpcRemoveChainLocalOverride = "RemoveChainLocalOverride" - rpcSealWriteCache = "SealWriteCache" - rpcListTargetsLocalOverrides = "ListTargetsLocalOverrides" - rpcDetachShards = "DetachShards" + rpcHealthCheck = "HealthCheck" + rpcSetNetmapStatus = "SetNetmapStatus" + rpcDropObjects = "DropObjects" + rpcListShards = "ListShards" + rpcSetShardMode = "SetShardMode" + rpcSynchronizeTree = "SynchronizeTree" + rpcEvacuateShard = "EvacuateShard" + rpcStartShardEvacuation = "StartShardEvacuation" + rpcGetShardEvacuationStatus = "GetShardEvacuationStatus" + rpcResetShardEvacuationStatus = "ResetShardEvacuationStatus" + rpcStopShardEvacuation = "StopShardEvacuation" + rpcFlushCache = "FlushCache" + rpcDoctor = "Doctor" + rpcAddChainLocalOverride = "AddChainLocalOverride" + rpcGetChainLocalOverride = "GetChainLocalOverride" + rpcListChainLocalOverrides = "ListChainLocalOverrides" + rpcRemoveChainLocalOverride = "RemoveChainLocalOverride" + rpcSealWriteCache = "SealWriteCache" + rpcListTargetsLocalOverrides = "ListTargetsLocalOverrides" + rpcDetachShards = "DetachShards" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -190,6 +191,19 @@ func StopShardEvacuation(cli *client.Client, req *StopShardEvacuationRequest, op return wResp.message, nil } +// ResetShardEvacuationStatus executes ControlService.ResetShardEvacuationStatus RPC. +func ResetShardEvacuationStatus(cli *client.Client, req *ResetShardEvacuationStatusRequest, opts ...client.CallOption) (*ResetShardEvacuationStatusResponse, error) { + wResp := newResponseWrapper[ResetShardEvacuationStatusResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcResetShardEvacuationStatus), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + // FlushCache executes ControlService.FlushCache RPC. func FlushCache(cli *client.Client, req *FlushCacheRequest, opts ...client.CallOption) (*FlushCacheResponse, error) { wResp := newResponseWrapper[FlushCacheResponse]() From 0a600521ad4529ee841ba619fb50c9cb34707f1e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 13 Mar 2024 12:01:06 +0300 Subject: [PATCH 0978/1943] [#1043] dev: Add empty pass config Signed-off-by: Dmitrii Stepanov --- dev/empty_pass.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 dev/empty_pass.yml diff --git a/dev/empty_pass.yml b/dev/empty_pass.yml new file mode 100644 index 000000000..2a088f952 --- /dev/null +++ b/dev/empty_pass.yml @@ -0,0 +1 @@ +password: "" \ No newline at end of file From 7ed07d2dfd2a3db38c043232f93525a3b40c640e Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 7 Mar 2024 19:16:40 +0300 Subject: [PATCH 0979/1943] [#976] morph: Implement missing methods for `SwitchRPCGuardedActor` Signed-off-by: Anton Nikiforov --- pkg/morph/client/actor.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/morph/client/actor.go b/pkg/morph/client/actor.go index ea6bc974c..8c283a672 100644 --- a/pkg/morph/client/actor.go +++ b/pkg/morph/client/actor.go @@ -1,11 +1,13 @@ package client import ( + "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/config/netmode" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" ) type actorProvider interface { @@ -118,3 +120,15 @@ func (a *SwitchRPCGuardedActor) Sign(tx *transaction.Transaction) error { func (a *SwitchRPCGuardedActor) SignAndSend(tx *transaction.Transaction) (util.Uint256, uint32, error) { return a.actorProvider.GetActor().SignAndSend(tx) } + +func (a *SwitchRPCGuardedActor) CallAndExpandIterator(contract util.Uint160, method string, maxItems int, params ...any) (*result.Invoke, error) { + return a.actorProvider.GetActor().CallAndExpandIterator(contract, method, maxItems, params...) +} + +func (a *SwitchRPCGuardedActor) TerminateSession(sessionID uuid.UUID) error { + return a.actorProvider.GetActor().TerminateSession(sessionID) +} + +func (a *SwitchRPCGuardedActor) TraverseIterator(sessionID uuid.UUID, iterator *result.Iterator, num int) ([]stackitem.Item, error) { + return a.actorProvider.GetActor().TraverseIterator(sessionID, iterator, num) +} From 2d595ec15f516dda5b66bd85d8c08c6136e514ad Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 7 Mar 2024 19:21:18 +0300 Subject: [PATCH 0980/1943] [#976] ape: Update dependency Signed-off-by: Anton Nikiforov --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/ape/chainbase/boltdb.go | 37 +++++++++++++++++++++++++++---------- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 6ba1ed73d..2c2eac68f 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,11 @@ go 1.20 require ( code.gitea.io/sdk/gitea v0.17.1 git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240215124401-634e24aba715 - git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa + git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240304151211-839f22e1a36d + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240307151106-2ec958cbfdfd git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 24120b20a..d90bf0f6a 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240215124401-634e24aba715 h1:EDtL9OJcdeevV/jmNMtHugulAQprdOnknNPvLB3LRgE= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240215124401-634e24aba715/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa h1:uuN90s74idThHXi/jF/IlTdFimOIs+IBU44rdTE/Zdo= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.1-0.20240115082915-f2a82aa635aa/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0 h1:FzurjElUwC7InY9v5rzXReKbfBL5yRJKSWJPq6BKhH0= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= @@ -12,8 +12,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b/go.mod h1:XcgrbZ88XfvhAMxmZCQJ0dv6FyRSq6Mg2J7nN8uuO0k= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240304151211-839f22e1a36d h1:y7KP7vrX5Cbx9R2nzCFj7Ljjsa/KkWZS4nQIXHEeKdw= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240304151211-839f22e1a36d/go.mod h1:YVL7yFaT0QNSpA0z+RHudLvrLwT+lsFYGyBSVc1ustI= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240307151106-2ec958cbfdfd h1:pyIl9f4nIr7ekJ73W9keLIQ5dpoKb8o6xNmodsXY5+o= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240307151106-2ec958cbfdfd/go.mod h1:H/AW85RtYxVTbcgwHW76DqXeKlsiCIOeNXHPqyDBrfQ= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/pkg/ape/chainbase/boltdb.go b/pkg/ape/chainbase/boltdb.go index 27bfd8468..5f3c342c1 100644 --- a/pkg/ape/chainbase/boltdb.go +++ b/pkg/ape/chainbase/boltdb.go @@ -100,7 +100,7 @@ func (cs *boltLocalOverrideStorage) Close() error { return err } -func getTargetBucket(tx *bbolt.Tx, name chain.Name, target policyengine.Target) (*bbolt.Bucket, error) { +func getTypeBucket(tx *bbolt.Tx, name chain.Name, target policyengine.Target) (*bbolt.Bucket, error) { cbucket := tx.Bucket(chainBucket) if cbucket == nil { return nil, ErrRootBucketNotFound @@ -110,16 +110,25 @@ func getTargetBucket(tx *bbolt.Tx, name chain.Name, target policyengine.Target) if nbucket == nil { return nil, fmt.Errorf("%w: %w: %s", policyengine.ErrChainNotFound, ErrGlobalNamespaceBucketNotFound, name) } + return nbucket.Bucket([]byte{byte(target.Type)}), nil +} - typeBucket := nbucket.Bucket([]byte{byte(target.Type)}) +func normalizeTargetName(target *policyengine.Target) { + if target.Type == policyengine.Namespace && target.Name == "" { + target.Name = "root" + } +} + +func getTargetBucket(tx *bbolt.Tx, name chain.Name, target policyengine.Target) (*bbolt.Bucket, error) { + typeBucket, err := getTypeBucket(tx, name, target) + if err != nil { + return nil, err + } if typeBucket == nil { return nil, fmt.Errorf("%w: %w: %c", policyengine.ErrChainNotFound, ErrTargetTypeBucketNotFound, target.Type) } - if target.Type == policyengine.Namespace && target.Name == "" { - target.Name = "root" - } - + normalizeTargetName(&target) rbucket := typeBucket.Bucket([]byte(target.Name)) if rbucket == nil { return nil, fmt.Errorf("%w: %w: %s", policyengine.ErrChainNotFound, ErrTargetNameBucketNotFound, target.Name) @@ -151,10 +160,7 @@ func getTargetBucketCreateIfEmpty(tx *bbolt.Tx, name chain.Name, target policyen } } - if target.Type == policyengine.Namespace && target.Name == "" { - target.Name = "root" - } - + normalizeTargetName(&target) rbucket := typeBucket.Bucket([]byte(target.Name)) if rbucket == nil { var err error @@ -220,6 +226,17 @@ func (cs *boltLocalOverrideStorage) RemoveOverride(name chain.Name, target polic }) } +func (cs *boltLocalOverrideStorage) RemoveOverridesByTarget(name chain.Name, target policyengine.Target) error { + return cs.db.Update(func(tx *bbolt.Tx) error { + typeBucket, err := getTypeBucket(tx, name, target) + if err != nil { + return err + } + normalizeTargetName(&target) + return typeBucket.DeleteBucket([]byte(target.Name)) + }) +} + func (cs *boltLocalOverrideStorage) ListOverrides(name chain.Name, target policyengine.Target) ([]*chain.Chain, error) { var serializedChains [][]byte var serializedChain []byte From 5ee5f1df42aa778c2ad2fb0142147ce7ad632e59 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 7 Mar 2024 14:22:52 +0300 Subject: [PATCH 0981/1943] [#976] control: Introduce new method `RemoveChainLocalOverridesByTarget` Signed-off-by: Anton Nikiforov --- pkg/services/control/rpc.go | 54 +- pkg/services/control/server/policy_engine.go | 25 + pkg/services/control/service.pb.go | 1294 +++++++++++------- pkg/services/control/service.proto | 23 + pkg/services/control/service_frostfs.pb.go | 163 +++ pkg/services/control/service_grpc.pb.go | 79 +- 6 files changed, 1096 insertions(+), 542 deletions(-) diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index b07b10274..f9397c128 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -8,26 +8,27 @@ import ( const serviceName = "control.ControlService" const ( - rpcHealthCheck = "HealthCheck" - rpcSetNetmapStatus = "SetNetmapStatus" - rpcDropObjects = "DropObjects" - rpcListShards = "ListShards" - rpcSetShardMode = "SetShardMode" - rpcSynchronizeTree = "SynchronizeTree" - rpcEvacuateShard = "EvacuateShard" - rpcStartShardEvacuation = "StartShardEvacuation" - rpcGetShardEvacuationStatus = "GetShardEvacuationStatus" - rpcResetShardEvacuationStatus = "ResetShardEvacuationStatus" - rpcStopShardEvacuation = "StopShardEvacuation" - rpcFlushCache = "FlushCache" - rpcDoctor = "Doctor" - rpcAddChainLocalOverride = "AddChainLocalOverride" - rpcGetChainLocalOverride = "GetChainLocalOverride" - rpcListChainLocalOverrides = "ListChainLocalOverrides" - rpcRemoveChainLocalOverride = "RemoveChainLocalOverride" - rpcSealWriteCache = "SealWriteCache" - rpcListTargetsLocalOverrides = "ListTargetsLocalOverrides" - rpcDetachShards = "DetachShards" + rpcHealthCheck = "HealthCheck" + rpcSetNetmapStatus = "SetNetmapStatus" + rpcDropObjects = "DropObjects" + rpcListShards = "ListShards" + rpcSetShardMode = "SetShardMode" + rpcSynchronizeTree = "SynchronizeTree" + rpcEvacuateShard = "EvacuateShard" + rpcStartShardEvacuation = "StartShardEvacuation" + rpcGetShardEvacuationStatus = "GetShardEvacuationStatus" + rpcResetShardEvacuationStatus = "ResetShardEvacuationStatus" + rpcStopShardEvacuation = "StopShardEvacuation" + rpcFlushCache = "FlushCache" + rpcDoctor = "Doctor" + rpcAddChainLocalOverride = "AddChainLocalOverride" + rpcGetChainLocalOverride = "GetChainLocalOverride" + rpcListChainLocalOverrides = "ListChainLocalOverrides" + rpcRemoveChainLocalOverride = "RemoveChainLocalOverride" + rpcRemoveChainLocalOverridesByTarget = "RemoveChainLocalOverridesByTarget" + rpcSealWriteCache = "SealWriteCache" + rpcListTargetsLocalOverrides = "ListTargetsLocalOverrides" + rpcDetachShards = "DetachShards" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -295,6 +296,19 @@ func RemoveChainLocalOverride(cli *client.Client, req *RemoveChainLocalOverrideR return wResp.message, nil } +// RemoveChainLocalOverridesByTarget executes ControlService.RemoveChainLocalOverridesByTarget RPC. +func RemoveChainLocalOverridesByTarget(cli *client.Client, req *RemoveChainLocalOverridesByTargetRequest, opts ...client.CallOption) (*RemoveChainLocalOverridesByTargetResponse, error) { + wResp := newResponseWrapper[RemoveChainLocalOverridesByTargetResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcRemoveChainLocalOverridesByTarget), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + // SealWriteCache executes ControlService.SealWriteCache RPC. func SealWriteCache(cli *client.Client, req *SealWriteCacheRequest, opts ...client.CallOption) (*SealWriteCacheResponse, error) { wResp := newResponseWrapper[SealWriteCacheResponse]() diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 4ea863f79..16b365b21 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -174,6 +174,31 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove return resp, nil } +func (s *Server) RemoveChainLocalOverridesByTarget(_ context.Context, req *control.RemoveChainLocalOverridesByTargetRequest) (*control.RemoveChainLocalOverridesByTargetResponse, error) { + if err := s.isValidRequest(req); err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + target, err := apeTarget(req.GetBody().GetTarget()) + if err != nil { + return nil, err + } + + if err = s.localOverrideStorage.LocalStorage().RemoveOverridesByTarget(apechain.Ingress, target); err != nil { + if code := getCodeByLocalStorageErr(err); code != codes.NotFound { + return nil, status.Error(code, err.Error()) + } + } + resp := &control.RemoveChainLocalOverridesByTargetResponse{ + Body: &control.RemoveChainLocalOverridesByTargetResponse_Body{}, + } + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil +} + func (s *Server) ListTargetsLocalOverrides(_ context.Context, req *control.ListTargetsLocalOverridesRequest) (*control.ListTargetsLocalOverridesResponse, error) { if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 586619767..345110bab 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -2157,6 +2157,116 @@ func (x *RemoveChainLocalOverrideResponse) GetSignature() *Signature { return nil } +type RemoveChainLocalOverridesByTargetRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *RemoveChainLocalOverridesByTargetRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *RemoveChainLocalOverridesByTargetRequest) Reset() { + *x = RemoveChainLocalOverridesByTargetRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveChainLocalOverridesByTargetRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveChainLocalOverridesByTargetRequest) ProtoMessage() {} + +func (x *RemoveChainLocalOverridesByTargetRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[36] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveChainLocalOverridesByTargetRequest.ProtoReflect.Descriptor instead. +func (*RemoveChainLocalOverridesByTargetRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36} +} + +func (x *RemoveChainLocalOverridesByTargetRequest) GetBody() *RemoveChainLocalOverridesByTargetRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *RemoveChainLocalOverridesByTargetRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +type RemoveChainLocalOverridesByTargetResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body *RemoveChainLocalOverridesByTargetResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *RemoveChainLocalOverridesByTargetResponse) Reset() { + *x = RemoveChainLocalOverridesByTargetResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveChainLocalOverridesByTargetResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveChainLocalOverridesByTargetResponse) ProtoMessage() {} + +func (x *RemoveChainLocalOverridesByTargetResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[37] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveChainLocalOverridesByTargetResponse.ProtoReflect.Descriptor instead. +func (*RemoveChainLocalOverridesByTargetResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37} +} + +func (x *RemoveChainLocalOverridesByTargetResponse) GetBody() *RemoveChainLocalOverridesByTargetResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *RemoveChainLocalOverridesByTargetResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + type SealWriteCacheRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2169,7 +2279,7 @@ type SealWriteCacheRequest struct { func (x *SealWriteCacheRequest) Reset() { *x = SealWriteCacheRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2182,7 +2292,7 @@ func (x *SealWriteCacheRequest) String() string { func (*SealWriteCacheRequest) ProtoMessage() {} func (x *SealWriteCacheRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2195,7 +2305,7 @@ func (x *SealWriteCacheRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheRequest.ProtoReflect.Descriptor instead. func (*SealWriteCacheRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38} } func (x *SealWriteCacheRequest) GetBody() *SealWriteCacheRequest_Body { @@ -2224,7 +2334,7 @@ type SealWriteCacheResponse struct { func (x *SealWriteCacheResponse) Reset() { *x = SealWriteCacheResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2237,7 +2347,7 @@ func (x *SealWriteCacheResponse) String() string { func (*SealWriteCacheResponse) ProtoMessage() {} func (x *SealWriteCacheResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2250,7 +2360,7 @@ func (x *SealWriteCacheResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheResponse.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39} } func (x *SealWriteCacheResponse) GetBody() *SealWriteCacheResponse_Body { @@ -2279,7 +2389,7 @@ type DetachShardsRequest struct { func (x *DetachShardsRequest) Reset() { *x = DetachShardsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2292,7 +2402,7 @@ func (x *DetachShardsRequest) String() string { func (*DetachShardsRequest) ProtoMessage() {} func (x *DetachShardsRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2305,7 +2415,7 @@ func (x *DetachShardsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsRequest.ProtoReflect.Descriptor instead. func (*DetachShardsRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{40} } func (x *DetachShardsRequest) GetBody() *DetachShardsRequest_Body { @@ -2334,7 +2444,7 @@ type DetachShardsResponse struct { func (x *DetachShardsResponse) Reset() { *x = DetachShardsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2347,7 +2457,7 @@ func (x *DetachShardsResponse) String() string { func (*DetachShardsResponse) ProtoMessage() {} func (x *DetachShardsResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2360,7 +2470,7 @@ func (x *DetachShardsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsResponse.ProtoReflect.Descriptor instead. func (*DetachShardsResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{41} } func (x *DetachShardsResponse) GetBody() *DetachShardsResponse_Body { @@ -2387,7 +2497,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2400,7 +2510,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2431,7 +2541,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2444,7 +2554,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2495,7 +2605,7 @@ type SetNetmapStatusRequest_Body struct { func (x *SetNetmapStatusRequest_Body) Reset() { *x = SetNetmapStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2508,7 +2618,7 @@ func (x *SetNetmapStatusRequest_Body) String() string { func (*SetNetmapStatusRequest_Body) ProtoMessage() {} func (x *SetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2548,7 +2658,7 @@ type SetNetmapStatusResponse_Body struct { func (x *SetNetmapStatusResponse_Body) Reset() { *x = SetNetmapStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2561,7 +2671,7 @@ func (x *SetNetmapStatusResponse_Body) String() string { func (*SetNetmapStatusResponse_Body) ProtoMessage() {} func (x *SetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2591,7 +2701,7 @@ type DropObjectsRequest_Body struct { func (x *DropObjectsRequest_Body) Reset() { *x = DropObjectsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2604,7 +2714,7 @@ func (x *DropObjectsRequest_Body) String() string { func (*DropObjectsRequest_Body) ProtoMessage() {} func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2637,7 +2747,7 @@ type DropObjectsResponse_Body struct { func (x *DropObjectsResponse_Body) Reset() { *x = DropObjectsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2650,7 +2760,7 @@ func (x *DropObjectsResponse_Body) String() string { func (*DropObjectsResponse_Body) ProtoMessage() {} func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2676,7 +2786,7 @@ type ListShardsRequest_Body struct { func (x *ListShardsRequest_Body) Reset() { *x = ListShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2689,7 +2799,7 @@ func (x *ListShardsRequest_Body) String() string { func (*ListShardsRequest_Body) ProtoMessage() {} func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2718,7 +2828,7 @@ type ListShardsResponse_Body struct { func (x *ListShardsResponse_Body) Reset() { *x = ListShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2731,7 +2841,7 @@ func (x *ListShardsResponse_Body) String() string { func (*ListShardsResponse_Body) ProtoMessage() {} func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2771,7 +2881,7 @@ type SetShardModeRequest_Body struct { func (x *SetShardModeRequest_Body) Reset() { *x = SetShardModeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2784,7 +2894,7 @@ func (x *SetShardModeRequest_Body) String() string { func (*SetShardModeRequest_Body) ProtoMessage() {} func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2831,7 +2941,7 @@ type SetShardModeResponse_Body struct { func (x *SetShardModeResponse_Body) Reset() { *x = SetShardModeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2844,7 +2954,7 @@ func (x *SetShardModeResponse_Body) String() string { func (*SetShardModeResponse_Body) ProtoMessage() {} func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2875,7 +2985,7 @@ type SynchronizeTreeRequest_Body struct { func (x *SynchronizeTreeRequest_Body) Reset() { *x = SynchronizeTreeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2888,7 +2998,7 @@ func (x *SynchronizeTreeRequest_Body) String() string { func (*SynchronizeTreeRequest_Body) ProtoMessage() {} func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2935,7 +3045,7 @@ type SynchronizeTreeResponse_Body struct { func (x *SynchronizeTreeResponse_Body) Reset() { *x = SynchronizeTreeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2948,7 +3058,7 @@ func (x *SynchronizeTreeResponse_Body) String() string { func (*SynchronizeTreeResponse_Body) ProtoMessage() {} func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2979,7 +3089,7 @@ type EvacuateShardRequest_Body struct { func (x *EvacuateShardRequest_Body) Reset() { *x = EvacuateShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2992,7 +3102,7 @@ func (x *EvacuateShardRequest_Body) String() string { func (*EvacuateShardRequest_Body) ProtoMessage() {} func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3034,7 +3144,7 @@ type EvacuateShardResponse_Body struct { func (x *EvacuateShardResponse_Body) Reset() { *x = EvacuateShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3047,7 +3157,7 @@ func (x *EvacuateShardResponse_Body) String() string { func (*EvacuateShardResponse_Body) ProtoMessage() {} func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3085,7 +3195,7 @@ type FlushCacheRequest_Body struct { func (x *FlushCacheRequest_Body) Reset() { *x = FlushCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3098,7 +3208,7 @@ func (x *FlushCacheRequest_Body) String() string { func (*FlushCacheRequest_Body) ProtoMessage() {} func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3138,7 +3248,7 @@ type FlushCacheResponse_Body struct { func (x *FlushCacheResponse_Body) Reset() { *x = FlushCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3151,7 +3261,7 @@ func (x *FlushCacheResponse_Body) String() string { func (*FlushCacheResponse_Body) ProtoMessage() {} func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3182,7 +3292,7 @@ type DoctorRequest_Body struct { func (x *DoctorRequest_Body) Reset() { *x = DoctorRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3195,7 +3305,7 @@ func (x *DoctorRequest_Body) String() string { func (*DoctorRequest_Body) ProtoMessage() {} func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3235,7 +3345,7 @@ type DoctorResponse_Body struct { func (x *DoctorResponse_Body) Reset() { *x = DoctorResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3248,7 +3358,7 @@ func (x *DoctorResponse_Body) String() string { func (*DoctorResponse_Body) ProtoMessage() {} func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3281,7 +3391,7 @@ type StartShardEvacuationRequest_Body struct { func (x *StartShardEvacuationRequest_Body) Reset() { *x = StartShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3294,7 +3404,7 @@ func (x *StartShardEvacuationRequest_Body) String() string { func (*StartShardEvacuationRequest_Body) ProtoMessage() {} func (x *StartShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3341,7 +3451,7 @@ type StartShardEvacuationResponse_Body struct { func (x *StartShardEvacuationResponse_Body) Reset() { *x = StartShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3354,7 +3464,7 @@ func (x *StartShardEvacuationResponse_Body) String() string { func (*StartShardEvacuationResponse_Body) ProtoMessage() {} func (x *StartShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3380,7 +3490,7 @@ type GetShardEvacuationStatusRequest_Body struct { func (x *GetShardEvacuationStatusRequest_Body) Reset() { *x = GetShardEvacuationStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3393,7 +3503,7 @@ func (x *GetShardEvacuationStatusRequest_Body) String() string { func (*GetShardEvacuationStatusRequest_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3444,7 +3554,7 @@ type GetShardEvacuationStatusResponse_Body struct { func (x *GetShardEvacuationStatusResponse_Body) Reset() { *x = GetShardEvacuationStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3457,7 +3567,7 @@ func (x *GetShardEvacuationStatusResponse_Body) String() string { func (*GetShardEvacuationStatusResponse_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3569,7 +3679,7 @@ type GetShardEvacuationStatusResponse_Body_UnixTimestamp struct { func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) Reset() { *x = GetShardEvacuationStatusResponse_Body_UnixTimestamp{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3582,7 +3692,7 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) String() string { func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[64] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3617,7 +3727,7 @@ type GetShardEvacuationStatusResponse_Body_Duration struct { func (x *GetShardEvacuationStatusResponse_Body_Duration) Reset() { *x = GetShardEvacuationStatusResponse_Body_Duration{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3630,7 +3740,7 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) String() string { func (*GetShardEvacuationStatusResponse_Body_Duration) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_Duration) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[65] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3662,7 +3772,7 @@ type ResetShardEvacuationStatusRequest_Body struct { func (x *ResetShardEvacuationStatusRequest_Body) Reset() { *x = ResetShardEvacuationStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3675,7 +3785,7 @@ func (x *ResetShardEvacuationStatusRequest_Body) String() string { func (*ResetShardEvacuationStatusRequest_Body) ProtoMessage() {} func (x *ResetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[66] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3700,7 +3810,7 @@ type ResetShardEvacuationStatusResponse_Body struct { func (x *ResetShardEvacuationStatusResponse_Body) Reset() { *x = ResetShardEvacuationStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3713,7 +3823,7 @@ func (x *ResetShardEvacuationStatusResponse_Body) String() string { func (*ResetShardEvacuationStatusResponse_Body) ProtoMessage() {} func (x *ResetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[67] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3739,7 +3849,7 @@ type StopShardEvacuationRequest_Body struct { func (x *StopShardEvacuationRequest_Body) Reset() { *x = StopShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[66] + mi := &file_pkg_services_control_service_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3752,7 +3862,7 @@ func (x *StopShardEvacuationRequest_Body) String() string { func (*StopShardEvacuationRequest_Body) ProtoMessage() {} func (x *StopShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[66] + mi := &file_pkg_services_control_service_proto_msgTypes[68] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3778,7 +3888,7 @@ type StopShardEvacuationResponse_Body struct { func (x *StopShardEvacuationResponse_Body) Reset() { *x = StopShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[67] + mi := &file_pkg_services_control_service_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3791,7 +3901,7 @@ func (x *StopShardEvacuationResponse_Body) String() string { func (*StopShardEvacuationResponse_Body) ProtoMessage() {} func (x *StopShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[67] + mi := &file_pkg_services_control_service_proto_msgTypes[69] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3823,7 +3933,7 @@ type AddChainLocalOverrideRequest_Body struct { func (x *AddChainLocalOverrideRequest_Body) Reset() { *x = AddChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[68] + mi := &file_pkg_services_control_service_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3836,7 +3946,7 @@ func (x *AddChainLocalOverrideRequest_Body) String() string { func (*AddChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *AddChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[68] + mi := &file_pkg_services_control_service_proto_msgTypes[70] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3880,7 +3990,7 @@ type AddChainLocalOverrideResponse_Body struct { func (x *AddChainLocalOverrideResponse_Body) Reset() { *x = AddChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[69] + mi := &file_pkg_services_control_service_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3893,7 +4003,7 @@ func (x *AddChainLocalOverrideResponse_Body) String() string { func (*AddChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *AddChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[69] + mi := &file_pkg_services_control_service_proto_msgTypes[71] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3930,7 +4040,7 @@ type GetChainLocalOverrideRequest_Body struct { func (x *GetChainLocalOverrideRequest_Body) Reset() { *x = GetChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[70] + mi := &file_pkg_services_control_service_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3943,7 +4053,7 @@ func (x *GetChainLocalOverrideRequest_Body) String() string { func (*GetChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *GetChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[70] + mi := &file_pkg_services_control_service_proto_msgTypes[72] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3985,7 +4095,7 @@ type GetChainLocalOverrideResponse_Body struct { func (x *GetChainLocalOverrideResponse_Body) Reset() { *x = GetChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[71] + mi := &file_pkg_services_control_service_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3998,7 +4108,7 @@ func (x *GetChainLocalOverrideResponse_Body) String() string { func (*GetChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *GetChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[71] + mi := &file_pkg_services_control_service_proto_msgTypes[73] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4033,7 +4143,7 @@ type ListChainLocalOverridesRequest_Body struct { func (x *ListChainLocalOverridesRequest_Body) Reset() { *x = ListChainLocalOverridesRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[72] + mi := &file_pkg_services_control_service_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4046,7 +4156,7 @@ func (x *ListChainLocalOverridesRequest_Body) String() string { func (*ListChainLocalOverridesRequest_Body) ProtoMessage() {} func (x *ListChainLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[72] + mi := &file_pkg_services_control_service_proto_msgTypes[74] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4081,7 +4191,7 @@ type ListChainLocalOverridesResponse_Body struct { func (x *ListChainLocalOverridesResponse_Body) Reset() { *x = ListChainLocalOverridesResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[73] + mi := &file_pkg_services_control_service_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4094,7 +4204,7 @@ func (x *ListChainLocalOverridesResponse_Body) String() string { func (*ListChainLocalOverridesResponse_Body) ProtoMessage() {} func (x *ListChainLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[73] + mi := &file_pkg_services_control_service_proto_msgTypes[75] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4129,7 +4239,7 @@ type ListTargetsLocalOverridesRequest_Body struct { func (x *ListTargetsLocalOverridesRequest_Body) Reset() { *x = ListTargetsLocalOverridesRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[74] + mi := &file_pkg_services_control_service_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4142,7 +4252,7 @@ func (x *ListTargetsLocalOverridesRequest_Body) String() string { func (*ListTargetsLocalOverridesRequest_Body) ProtoMessage() {} func (x *ListTargetsLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[74] + mi := &file_pkg_services_control_service_proto_msgTypes[76] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4177,7 +4287,7 @@ type ListTargetsLocalOverridesResponse_Body struct { func (x *ListTargetsLocalOverridesResponse_Body) Reset() { *x = ListTargetsLocalOverridesResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[75] + mi := &file_pkg_services_control_service_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4190,7 +4300,7 @@ func (x *ListTargetsLocalOverridesResponse_Body) String() string { func (*ListTargetsLocalOverridesResponse_Body) ProtoMessage() {} func (x *ListTargetsLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[75] + mi := &file_pkg_services_control_service_proto_msgTypes[77] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4227,7 +4337,7 @@ type RemoveChainLocalOverrideRequest_Body struct { func (x *RemoveChainLocalOverrideRequest_Body) Reset() { *x = RemoveChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[76] + mi := &file_pkg_services_control_service_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4240,7 +4350,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) String() string { func (*RemoveChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[76] + mi := &file_pkg_services_control_service_proto_msgTypes[78] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4279,7 +4389,7 @@ type RemoveChainLocalOverrideResponse_Body struct { func (x *RemoveChainLocalOverrideResponse_Body) Reset() { *x = RemoveChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[77] + mi := &file_pkg_services_control_service_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4292,7 +4402,7 @@ func (x *RemoveChainLocalOverrideResponse_Body) String() string { func (*RemoveChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[77] + mi := &file_pkg_services_control_service_proto_msgTypes[79] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4308,6 +4418,92 @@ func (*RemoveChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35, 0} } +type RemoveChainLocalOverridesByTargetRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Target for which the overrides are applied. + Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` +} + +func (x *RemoveChainLocalOverridesByTargetRequest_Body) Reset() { + *x = RemoveChainLocalOverridesByTargetRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[80] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveChainLocalOverridesByTargetRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveChainLocalOverridesByTargetRequest_Body) ProtoMessage() {} + +func (x *RemoveChainLocalOverridesByTargetRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[80] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveChainLocalOverridesByTargetRequest_Body.ProtoReflect.Descriptor instead. +func (*RemoveChainLocalOverridesByTargetRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36, 0} +} + +func (x *RemoveChainLocalOverridesByTargetRequest_Body) GetTarget() *ChainTarget { + if x != nil { + return x.Target + } + return nil +} + +type RemoveChainLocalOverridesByTargetResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *RemoveChainLocalOverridesByTargetResponse_Body) Reset() { + *x = RemoveChainLocalOverridesByTargetResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[81] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoveChainLocalOverridesByTargetResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoveChainLocalOverridesByTargetResponse_Body) ProtoMessage() {} + +func (x *RemoveChainLocalOverridesByTargetResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[81] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoveChainLocalOverridesByTargetResponse_Body.ProtoReflect.Descriptor instead. +func (*RemoveChainLocalOverridesByTargetResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37, 0} +} + // Request body structure. type SealWriteCacheRequest_Body struct { state protoimpl.MessageState @@ -4323,7 +4519,7 @@ type SealWriteCacheRequest_Body struct { func (x *SealWriteCacheRequest_Body) Reset() { *x = SealWriteCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[78] + mi := &file_pkg_services_control_service_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4336,7 +4532,7 @@ func (x *SealWriteCacheRequest_Body) String() string { func (*SealWriteCacheRequest_Body) ProtoMessage() {} func (x *SealWriteCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[78] + mi := &file_pkg_services_control_service_proto_msgTypes[82] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4349,7 +4545,7 @@ func (x *SealWriteCacheRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheRequest_Body.ProtoReflect.Descriptor instead. func (*SealWriteCacheRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38, 0} } func (x *SealWriteCacheRequest_Body) GetShard_ID() [][]byte { @@ -4377,7 +4573,7 @@ type SealWriteCacheResponse_Body struct { func (x *SealWriteCacheResponse_Body) Reset() { *x = SealWriteCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[79] + mi := &file_pkg_services_control_service_proto_msgTypes[83] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4390,7 +4586,7 @@ func (x *SealWriteCacheResponse_Body) String() string { func (*SealWriteCacheResponse_Body) ProtoMessage() {} func (x *SealWriteCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[79] + mi := &file_pkg_services_control_service_proto_msgTypes[83] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4403,7 +4599,7 @@ func (x *SealWriteCacheResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheResponse_Body.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39, 0} } func (x *SealWriteCacheResponse_Body) GetResults() []*SealWriteCacheResponse_Body_Status { @@ -4426,7 +4622,7 @@ type SealWriteCacheResponse_Body_Status struct { func (x *SealWriteCacheResponse_Body_Status) Reset() { *x = SealWriteCacheResponse_Body_Status{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[80] + mi := &file_pkg_services_control_service_proto_msgTypes[84] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4439,7 +4635,7 @@ func (x *SealWriteCacheResponse_Body_Status) String() string { func (*SealWriteCacheResponse_Body_Status) ProtoMessage() {} func (x *SealWriteCacheResponse_Body_Status) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[80] + mi := &file_pkg_services_control_service_proto_msgTypes[84] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4452,7 +4648,7 @@ func (x *SealWriteCacheResponse_Body_Status) ProtoReflect() protoreflect.Message // Deprecated: Use SealWriteCacheResponse_Body_Status.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse_Body_Status) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37, 0, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39, 0, 0} } func (x *SealWriteCacheResponse_Body_Status) GetShard_ID() []byte { @@ -4487,7 +4683,7 @@ type DetachShardsRequest_Body struct { func (x *DetachShardsRequest_Body) Reset() { *x = DetachShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[81] + mi := &file_pkg_services_control_service_proto_msgTypes[85] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4500,7 +4696,7 @@ func (x *DetachShardsRequest_Body) String() string { func (*DetachShardsRequest_Body) ProtoMessage() {} func (x *DetachShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[81] + mi := &file_pkg_services_control_service_proto_msgTypes[85] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4513,7 +4709,7 @@ func (x *DetachShardsRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsRequest_Body.ProtoReflect.Descriptor instead. func (*DetachShardsRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{40, 0} } func (x *DetachShardsRequest_Body) GetShard_ID() [][]byte { @@ -4532,7 +4728,7 @@ type DetachShardsResponse_Body struct { func (x *DetachShardsResponse_Body) Reset() { *x = DetachShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[82] + mi := &file_pkg_services_control_service_proto_msgTypes[86] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4545,7 +4741,7 @@ func (x *DetachShardsResponse_Body) String() string { func (*DetachShardsResponse_Body) ProtoMessage() {} func (x *DetachShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[82] + mi := &file_pkg_services_control_service_proto_msgTypes[86] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4558,7 +4754,7 @@ func (x *DetachShardsResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsResponse_Body.ProtoReflect.Descriptor instead. func (*DetachShardsResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{41, 0} } var File_pkg_services_control_service_proto protoreflect.FileDescriptor @@ -5026,176 +5222,211 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x22, 0xca, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, - 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, - 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, - 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0xa9, 0x02, - 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, - 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, + 0x22, 0xde, 0x01, 0x0a, 0x28, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4a, 0x0a, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, + 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x22, 0xb2, 0x01, 0x0a, 0x29, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, + 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x4b, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xca, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, + 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, - 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, - 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, - 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, + 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, + 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, + 0x6f, 0x72, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, + 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, - 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, - 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, + 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, + 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xa8, 0x0e, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, - 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, - 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, - 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, - 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, - 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, - 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, - 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, + 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, + 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, + 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb5, + 0x0f, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, + 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, + 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, + 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, + 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, + 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, + 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, + 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, + 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, + 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, + 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, + 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, - 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, - 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, - 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, - 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, - 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, + 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, + 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, + 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, + 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, + 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, - 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, - 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, - 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, - 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, - 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8a, 0x01, 0x0a, 0x21, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x31, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, + 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, + 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, + 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, + 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, + 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, + 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, + 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, + 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -5211,7 +5442,7 @@ func file_pkg_services_control_service_proto_rawDescGZIP() []byte { } var file_pkg_services_control_service_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 83) +var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 87) var file_pkg_services_control_service_proto_goTypes = []interface{}{ (StartShardEvacuationRequest_Body_Scope)(0), // 0: control.StartShardEvacuationRequest.Body.Scope (GetShardEvacuationStatusResponse_Body_Status)(0), // 1: control.GetShardEvacuationStatusResponse.Body.Status @@ -5251,200 +5482,211 @@ var file_pkg_services_control_service_proto_goTypes = []interface{}{ (*ListTargetsLocalOverridesResponse)(nil), // 35: control.ListTargetsLocalOverridesResponse (*RemoveChainLocalOverrideRequest)(nil), // 36: control.RemoveChainLocalOverrideRequest (*RemoveChainLocalOverrideResponse)(nil), // 37: control.RemoveChainLocalOverrideResponse - (*SealWriteCacheRequest)(nil), // 38: control.SealWriteCacheRequest - (*SealWriteCacheResponse)(nil), // 39: control.SealWriteCacheResponse - (*DetachShardsRequest)(nil), // 40: control.DetachShardsRequest - (*DetachShardsResponse)(nil), // 41: control.DetachShardsResponse - (*HealthCheckRequest_Body)(nil), // 42: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 43: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 44: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 45: control.SetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 46: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 47: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 48: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 49: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 50: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 51: control.SetShardModeResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 52: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 53: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 54: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 55: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 56: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 57: control.FlushCacheResponse.Body - (*DoctorRequest_Body)(nil), // 58: control.DoctorRequest.Body - (*DoctorResponse_Body)(nil), // 59: control.DoctorResponse.Body - (*StartShardEvacuationRequest_Body)(nil), // 60: control.StartShardEvacuationRequest.Body - (*StartShardEvacuationResponse_Body)(nil), // 61: control.StartShardEvacuationResponse.Body - (*GetShardEvacuationStatusRequest_Body)(nil), // 62: control.GetShardEvacuationStatusRequest.Body - (*GetShardEvacuationStatusResponse_Body)(nil), // 63: control.GetShardEvacuationStatusResponse.Body - (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 64: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 65: control.GetShardEvacuationStatusResponse.Body.Duration - (*ResetShardEvacuationStatusRequest_Body)(nil), // 66: control.ResetShardEvacuationStatusRequest.Body - (*ResetShardEvacuationStatusResponse_Body)(nil), // 67: control.ResetShardEvacuationStatusResponse.Body - (*StopShardEvacuationRequest_Body)(nil), // 68: control.StopShardEvacuationRequest.Body - (*StopShardEvacuationResponse_Body)(nil), // 69: control.StopShardEvacuationResponse.Body - (*AddChainLocalOverrideRequest_Body)(nil), // 70: control.AddChainLocalOverrideRequest.Body - (*AddChainLocalOverrideResponse_Body)(nil), // 71: control.AddChainLocalOverrideResponse.Body - (*GetChainLocalOverrideRequest_Body)(nil), // 72: control.GetChainLocalOverrideRequest.Body - (*GetChainLocalOverrideResponse_Body)(nil), // 73: control.GetChainLocalOverrideResponse.Body - (*ListChainLocalOverridesRequest_Body)(nil), // 74: control.ListChainLocalOverridesRequest.Body - (*ListChainLocalOverridesResponse_Body)(nil), // 75: control.ListChainLocalOverridesResponse.Body - (*ListTargetsLocalOverridesRequest_Body)(nil), // 76: control.ListTargetsLocalOverridesRequest.Body - (*ListTargetsLocalOverridesResponse_Body)(nil), // 77: control.ListTargetsLocalOverridesResponse.Body - (*RemoveChainLocalOverrideRequest_Body)(nil), // 78: control.RemoveChainLocalOverrideRequest.Body - (*RemoveChainLocalOverrideResponse_Body)(nil), // 79: control.RemoveChainLocalOverrideResponse.Body - (*SealWriteCacheRequest_Body)(nil), // 80: control.SealWriteCacheRequest.Body - (*SealWriteCacheResponse_Body)(nil), // 81: control.SealWriteCacheResponse.Body - (*SealWriteCacheResponse_Body_Status)(nil), // 82: control.SealWriteCacheResponse.Body.Status - (*DetachShardsRequest_Body)(nil), // 83: control.DetachShardsRequest.Body - (*DetachShardsResponse_Body)(nil), // 84: control.DetachShardsResponse.Body - (*Signature)(nil), // 85: control.Signature - (NetmapStatus)(0), // 86: control.NetmapStatus - (HealthStatus)(0), // 87: control.HealthStatus - (*ShardInfo)(nil), // 88: control.ShardInfo - (ShardMode)(0), // 89: control.ShardMode - (*ChainTarget)(nil), // 90: control.ChainTarget + (*RemoveChainLocalOverridesByTargetRequest)(nil), // 38: control.RemoveChainLocalOverridesByTargetRequest + (*RemoveChainLocalOverridesByTargetResponse)(nil), // 39: control.RemoveChainLocalOverridesByTargetResponse + (*SealWriteCacheRequest)(nil), // 40: control.SealWriteCacheRequest + (*SealWriteCacheResponse)(nil), // 41: control.SealWriteCacheResponse + (*DetachShardsRequest)(nil), // 42: control.DetachShardsRequest + (*DetachShardsResponse)(nil), // 43: control.DetachShardsResponse + (*HealthCheckRequest_Body)(nil), // 44: control.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 45: control.HealthCheckResponse.Body + (*SetNetmapStatusRequest_Body)(nil), // 46: control.SetNetmapStatusRequest.Body + (*SetNetmapStatusResponse_Body)(nil), // 47: control.SetNetmapStatusResponse.Body + (*DropObjectsRequest_Body)(nil), // 48: control.DropObjectsRequest.Body + (*DropObjectsResponse_Body)(nil), // 49: control.DropObjectsResponse.Body + (*ListShardsRequest_Body)(nil), // 50: control.ListShardsRequest.Body + (*ListShardsResponse_Body)(nil), // 51: control.ListShardsResponse.Body + (*SetShardModeRequest_Body)(nil), // 52: control.SetShardModeRequest.Body + (*SetShardModeResponse_Body)(nil), // 53: control.SetShardModeResponse.Body + (*SynchronizeTreeRequest_Body)(nil), // 54: control.SynchronizeTreeRequest.Body + (*SynchronizeTreeResponse_Body)(nil), // 55: control.SynchronizeTreeResponse.Body + (*EvacuateShardRequest_Body)(nil), // 56: control.EvacuateShardRequest.Body + (*EvacuateShardResponse_Body)(nil), // 57: control.EvacuateShardResponse.Body + (*FlushCacheRequest_Body)(nil), // 58: control.FlushCacheRequest.Body + (*FlushCacheResponse_Body)(nil), // 59: control.FlushCacheResponse.Body + (*DoctorRequest_Body)(nil), // 60: control.DoctorRequest.Body + (*DoctorResponse_Body)(nil), // 61: control.DoctorResponse.Body + (*StartShardEvacuationRequest_Body)(nil), // 62: control.StartShardEvacuationRequest.Body + (*StartShardEvacuationResponse_Body)(nil), // 63: control.StartShardEvacuationResponse.Body + (*GetShardEvacuationStatusRequest_Body)(nil), // 64: control.GetShardEvacuationStatusRequest.Body + (*GetShardEvacuationStatusResponse_Body)(nil), // 65: control.GetShardEvacuationStatusResponse.Body + (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 66: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 67: control.GetShardEvacuationStatusResponse.Body.Duration + (*ResetShardEvacuationStatusRequest_Body)(nil), // 68: control.ResetShardEvacuationStatusRequest.Body + (*ResetShardEvacuationStatusResponse_Body)(nil), // 69: control.ResetShardEvacuationStatusResponse.Body + (*StopShardEvacuationRequest_Body)(nil), // 70: control.StopShardEvacuationRequest.Body + (*StopShardEvacuationResponse_Body)(nil), // 71: control.StopShardEvacuationResponse.Body + (*AddChainLocalOverrideRequest_Body)(nil), // 72: control.AddChainLocalOverrideRequest.Body + (*AddChainLocalOverrideResponse_Body)(nil), // 73: control.AddChainLocalOverrideResponse.Body + (*GetChainLocalOverrideRequest_Body)(nil), // 74: control.GetChainLocalOverrideRequest.Body + (*GetChainLocalOverrideResponse_Body)(nil), // 75: control.GetChainLocalOverrideResponse.Body + (*ListChainLocalOverridesRequest_Body)(nil), // 76: control.ListChainLocalOverridesRequest.Body + (*ListChainLocalOverridesResponse_Body)(nil), // 77: control.ListChainLocalOverridesResponse.Body + (*ListTargetsLocalOverridesRequest_Body)(nil), // 78: control.ListTargetsLocalOverridesRequest.Body + (*ListTargetsLocalOverridesResponse_Body)(nil), // 79: control.ListTargetsLocalOverridesResponse.Body + (*RemoveChainLocalOverrideRequest_Body)(nil), // 80: control.RemoveChainLocalOverrideRequest.Body + (*RemoveChainLocalOverrideResponse_Body)(nil), // 81: control.RemoveChainLocalOverrideResponse.Body + (*RemoveChainLocalOverridesByTargetRequest_Body)(nil), // 82: control.RemoveChainLocalOverridesByTargetRequest.Body + (*RemoveChainLocalOverridesByTargetResponse_Body)(nil), // 83: control.RemoveChainLocalOverridesByTargetResponse.Body + (*SealWriteCacheRequest_Body)(nil), // 84: control.SealWriteCacheRequest.Body + (*SealWriteCacheResponse_Body)(nil), // 85: control.SealWriteCacheResponse.Body + (*SealWriteCacheResponse_Body_Status)(nil), // 86: control.SealWriteCacheResponse.Body.Status + (*DetachShardsRequest_Body)(nil), // 87: control.DetachShardsRequest.Body + (*DetachShardsResponse_Body)(nil), // 88: control.DetachShardsResponse.Body + (*Signature)(nil), // 89: control.Signature + (NetmapStatus)(0), // 90: control.NetmapStatus + (HealthStatus)(0), // 91: control.HealthStatus + (*ShardInfo)(nil), // 92: control.ShardInfo + (ShardMode)(0), // 93: control.ShardMode + (*ChainTarget)(nil), // 94: control.ChainTarget } var file_pkg_services_control_service_proto_depIdxs = []int32{ - 42, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 85, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 43, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 85, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 44, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 85, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 45, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 85, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 46, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 85, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature - 47, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 85, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature - 48, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 85, // 13: control.ListShardsRequest.signature:type_name -> control.Signature - 49, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 85, // 15: control.ListShardsResponse.signature:type_name -> control.Signature - 50, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 85, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature - 51, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 85, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature - 52, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 85, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 53, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 85, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 54, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 85, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature - 55, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 85, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature - 56, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 85, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature - 57, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 85, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature - 58, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body - 85, // 33: control.DoctorRequest.signature:type_name -> control.Signature - 59, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body - 85, // 35: control.DoctorResponse.signature:type_name -> control.Signature - 60, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body - 85, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature - 61, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body - 85, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature - 62, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body - 85, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 63, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body - 85, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 66, // 44: control.ResetShardEvacuationStatusRequest.body:type_name -> control.ResetShardEvacuationStatusRequest.Body - 85, // 45: control.ResetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 67, // 46: control.ResetShardEvacuationStatusResponse.body:type_name -> control.ResetShardEvacuationStatusResponse.Body - 85, // 47: control.ResetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 68, // 48: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body - 85, // 49: control.StopShardEvacuationRequest.signature:type_name -> control.Signature - 69, // 50: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body - 85, // 51: control.StopShardEvacuationResponse.signature:type_name -> control.Signature - 70, // 52: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body - 85, // 53: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature - 71, // 54: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body - 85, // 55: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature - 72, // 56: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body - 85, // 57: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature - 73, // 58: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body - 85, // 59: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature - 74, // 60: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body - 85, // 61: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature - 75, // 62: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body - 85, // 63: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature - 76, // 64: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body - 85, // 65: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature - 77, // 66: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body - 85, // 67: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature - 78, // 68: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body - 85, // 69: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature - 79, // 70: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body - 85, // 71: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature - 80, // 72: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body - 85, // 73: control.SealWriteCacheRequest.signature:type_name -> control.Signature - 81, // 74: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body - 85, // 75: control.SealWriteCacheResponse.signature:type_name -> control.Signature - 83, // 76: control.DetachShardsRequest.body:type_name -> control.DetachShardsRequest.Body - 85, // 77: control.DetachShardsRequest.signature:type_name -> control.Signature - 84, // 78: control.DetachShardsResponse.body:type_name -> control.DetachShardsResponse.Body - 85, // 79: control.DetachShardsResponse.signature:type_name -> control.Signature - 86, // 80: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 87, // 81: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 86, // 82: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 88, // 83: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 89, // 84: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 1, // 85: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status - 65, // 86: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration - 64, // 87: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - 90, // 88: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 90, // 89: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 90, // 90: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget - 90, // 91: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget - 90, // 92: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 82, // 93: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status - 2, // 94: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 4, // 95: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 6, // 96: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 8, // 97: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 10, // 98: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 12, // 99: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 14, // 100: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 20, // 101: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest - 22, // 102: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest - 24, // 103: control.ControlService.ResetShardEvacuationStatus:input_type -> control.ResetShardEvacuationStatusRequest - 26, // 104: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest - 16, // 105: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 18, // 106: control.ControlService.Doctor:input_type -> control.DoctorRequest - 28, // 107: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest - 30, // 108: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest - 32, // 109: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest - 36, // 110: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest - 34, // 111: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest - 38, // 112: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest - 40, // 113: control.ControlService.DetachShards:input_type -> control.DetachShardsRequest - 3, // 114: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 5, // 115: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 7, // 116: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 9, // 117: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 11, // 118: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 13, // 119: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 15, // 120: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 21, // 121: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse - 23, // 122: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse - 25, // 123: control.ControlService.ResetShardEvacuationStatus:output_type -> control.ResetShardEvacuationStatusResponse - 27, // 124: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse - 17, // 125: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 19, // 126: control.ControlService.Doctor:output_type -> control.DoctorResponse - 29, // 127: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse - 31, // 128: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse - 33, // 129: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse - 37, // 130: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse - 35, // 131: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse - 39, // 132: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse - 41, // 133: control.ControlService.DetachShards:output_type -> control.DetachShardsResponse - 114, // [114:134] is the sub-list for method output_type - 94, // [94:114] is the sub-list for method input_type - 94, // [94:94] is the sub-list for extension type_name - 94, // [94:94] is the sub-list for extension extendee - 0, // [0:94] is the sub-list for field type_name + 44, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body + 89, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature + 45, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body + 89, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature + 46, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body + 89, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature + 47, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body + 89, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature + 48, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body + 89, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature + 49, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body + 89, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature + 50, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body + 89, // 13: control.ListShardsRequest.signature:type_name -> control.Signature + 51, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body + 89, // 15: control.ListShardsResponse.signature:type_name -> control.Signature + 52, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body + 89, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature + 53, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body + 89, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature + 54, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body + 89, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature + 55, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body + 89, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature + 56, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body + 89, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature + 57, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body + 89, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature + 58, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body + 89, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature + 59, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body + 89, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature + 60, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body + 89, // 33: control.DoctorRequest.signature:type_name -> control.Signature + 61, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body + 89, // 35: control.DoctorResponse.signature:type_name -> control.Signature + 62, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body + 89, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature + 63, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body + 89, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature + 64, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body + 89, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 65, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body + 89, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 68, // 44: control.ResetShardEvacuationStatusRequest.body:type_name -> control.ResetShardEvacuationStatusRequest.Body + 89, // 45: control.ResetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 69, // 46: control.ResetShardEvacuationStatusResponse.body:type_name -> control.ResetShardEvacuationStatusResponse.Body + 89, // 47: control.ResetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 70, // 48: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body + 89, // 49: control.StopShardEvacuationRequest.signature:type_name -> control.Signature + 71, // 50: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body + 89, // 51: control.StopShardEvacuationResponse.signature:type_name -> control.Signature + 72, // 52: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body + 89, // 53: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature + 73, // 54: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body + 89, // 55: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature + 74, // 56: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body + 89, // 57: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature + 75, // 58: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body + 89, // 59: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature + 76, // 60: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body + 89, // 61: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature + 77, // 62: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body + 89, // 63: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature + 78, // 64: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body + 89, // 65: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature + 79, // 66: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body + 89, // 67: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature + 80, // 68: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body + 89, // 69: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature + 81, // 70: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body + 89, // 71: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature + 82, // 72: control.RemoveChainLocalOverridesByTargetRequest.body:type_name -> control.RemoveChainLocalOverridesByTargetRequest.Body + 89, // 73: control.RemoveChainLocalOverridesByTargetRequest.signature:type_name -> control.Signature + 83, // 74: control.RemoveChainLocalOverridesByTargetResponse.body:type_name -> control.RemoveChainLocalOverridesByTargetResponse.Body + 89, // 75: control.RemoveChainLocalOverridesByTargetResponse.signature:type_name -> control.Signature + 84, // 76: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body + 89, // 77: control.SealWriteCacheRequest.signature:type_name -> control.Signature + 85, // 78: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body + 89, // 79: control.SealWriteCacheResponse.signature:type_name -> control.Signature + 87, // 80: control.DetachShardsRequest.body:type_name -> control.DetachShardsRequest.Body + 89, // 81: control.DetachShardsRequest.signature:type_name -> control.Signature + 88, // 82: control.DetachShardsResponse.body:type_name -> control.DetachShardsResponse.Body + 89, // 83: control.DetachShardsResponse.signature:type_name -> control.Signature + 90, // 84: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus + 91, // 85: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus + 90, // 86: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus + 92, // 87: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo + 93, // 88: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode + 1, // 89: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status + 67, // 90: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration + 66, // 91: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + 94, // 92: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 94, // 93: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 94, // 94: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget + 94, // 95: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget + 94, // 96: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 94, // 97: control.RemoveChainLocalOverridesByTargetRequest.Body.target:type_name -> control.ChainTarget + 86, // 98: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status + 2, // 99: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 4, // 100: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 6, // 101: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 8, // 102: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 10, // 103: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 12, // 104: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 14, // 105: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 20, // 106: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest + 22, // 107: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest + 24, // 108: control.ControlService.ResetShardEvacuationStatus:input_type -> control.ResetShardEvacuationStatusRequest + 26, // 109: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest + 16, // 110: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 18, // 111: control.ControlService.Doctor:input_type -> control.DoctorRequest + 28, // 112: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest + 30, // 113: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest + 32, // 114: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest + 36, // 115: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest + 38, // 116: control.ControlService.RemoveChainLocalOverridesByTarget:input_type -> control.RemoveChainLocalOverridesByTargetRequest + 34, // 117: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest + 40, // 118: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest + 42, // 119: control.ControlService.DetachShards:input_type -> control.DetachShardsRequest + 3, // 120: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 5, // 121: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 7, // 122: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 9, // 123: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 11, // 124: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 13, // 125: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 15, // 126: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 21, // 127: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse + 23, // 128: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse + 25, // 129: control.ControlService.ResetShardEvacuationStatus:output_type -> control.ResetShardEvacuationStatusResponse + 27, // 130: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse + 17, // 131: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 19, // 132: control.ControlService.Doctor:output_type -> control.DoctorResponse + 29, // 133: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse + 31, // 134: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse + 33, // 135: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse + 37, // 136: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse + 39, // 137: control.ControlService.RemoveChainLocalOverridesByTarget:output_type -> control.RemoveChainLocalOverridesByTargetResponse + 35, // 138: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse + 41, // 139: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse + 43, // 140: control.ControlService.DetachShards:output_type -> control.DetachShardsResponse + 120, // [120:141] is the sub-list for method output_type + 99, // [99:120] is the sub-list for method input_type + 99, // [99:99] is the sub-list for extension type_name + 99, // [99:99] is the sub-list for extension extendee + 0, // [0:99] is the sub-list for field type_name } func init() { file_pkg_services_control_service_proto_init() } @@ -5887,7 +6129,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheRequest); i { + switch v := v.(*RemoveChainLocalOverridesByTargetRequest); i { case 0: return &v.state case 1: @@ -5899,7 +6141,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse); i { + switch v := v.(*RemoveChainLocalOverridesByTargetResponse); i { case 0: return &v.state case 1: @@ -5911,7 +6153,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsRequest); i { + switch v := v.(*SealWriteCacheRequest); i { case 0: return &v.state case 1: @@ -5923,7 +6165,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsResponse); i { + switch v := v.(*SealWriteCacheResponse); i { case 0: return &v.state case 1: @@ -5935,7 +6177,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*DetachShardsRequest); i { case 0: return &v.state case 1: @@ -5947,7 +6189,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { + switch v := v.(*DetachShardsResponse); i { case 0: return &v.state case 1: @@ -5959,7 +6201,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusRequest_Body); i { + switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state case 1: @@ -5971,7 +6213,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusResponse_Body); i { + switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state case 1: @@ -5983,7 +6225,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest_Body); i { + switch v := v.(*SetNetmapStatusRequest_Body); i { case 0: return &v.state case 1: @@ -5995,7 +6237,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse_Body); i { + switch v := v.(*SetNetmapStatusResponse_Body); i { case 0: return &v.state case 1: @@ -6007,7 +6249,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest_Body); i { + switch v := v.(*DropObjectsRequest_Body); i { case 0: return &v.state case 1: @@ -6019,7 +6261,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse_Body); i { + switch v := v.(*DropObjectsResponse_Body); i { case 0: return &v.state case 1: @@ -6031,7 +6273,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest_Body); i { + switch v := v.(*ListShardsRequest_Body); i { case 0: return &v.state case 1: @@ -6043,7 +6285,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse_Body); i { + switch v := v.(*ListShardsResponse_Body); i { case 0: return &v.state case 1: @@ -6055,7 +6297,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest_Body); i { + switch v := v.(*SetShardModeRequest_Body); i { case 0: return &v.state case 1: @@ -6067,7 +6309,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse_Body); i { + switch v := v.(*SetShardModeResponse_Body); i { case 0: return &v.state case 1: @@ -6079,7 +6321,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest_Body); i { + switch v := v.(*SynchronizeTreeRequest_Body); i { case 0: return &v.state case 1: @@ -6091,7 +6333,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse_Body); i { + switch v := v.(*SynchronizeTreeResponse_Body); i { case 0: return &v.state case 1: @@ -6103,7 +6345,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest_Body); i { + switch v := v.(*EvacuateShardRequest_Body); i { case 0: return &v.state case 1: @@ -6115,7 +6357,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheResponse_Body); i { + switch v := v.(*EvacuateShardResponse_Body); i { case 0: return &v.state case 1: @@ -6127,7 +6369,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorRequest_Body); i { + switch v := v.(*FlushCacheRequest_Body); i { case 0: return &v.state case 1: @@ -6139,7 +6381,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorResponse_Body); i { + switch v := v.(*FlushCacheResponse_Body); i { case 0: return &v.state case 1: @@ -6151,7 +6393,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationRequest_Body); i { + switch v := v.(*DoctorRequest_Body); i { case 0: return &v.state case 1: @@ -6163,7 +6405,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationResponse_Body); i { + switch v := v.(*DoctorResponse_Body); i { case 0: return &v.state case 1: @@ -6175,7 +6417,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusRequest_Body); i { + switch v := v.(*StartShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -6187,7 +6429,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body); i { + switch v := v.(*StartShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -6199,7 +6441,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { + switch v := v.(*GetShardEvacuationStatusRequest_Body); i { case 0: return &v.state case 1: @@ -6211,7 +6453,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body); i { case 0: return &v.state case 1: @@ -6223,7 +6465,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResetShardEvacuationStatusRequest_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { case 0: return &v.state case 1: @@ -6235,7 +6477,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResetShardEvacuationStatusResponse_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { case 0: return &v.state case 1: @@ -6247,7 +6489,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationRequest_Body); i { + switch v := v.(*ResetShardEvacuationStatusRequest_Body); i { case 0: return &v.state case 1: @@ -6259,7 +6501,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationResponse_Body); i { + switch v := v.(*ResetShardEvacuationStatusResponse_Body); i { case 0: return &v.state case 1: @@ -6271,7 +6513,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideRequest_Body); i { + switch v := v.(*StopShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -6283,7 +6525,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideResponse_Body); i { + switch v := v.(*StopShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -6295,7 +6537,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideRequest_Body); i { + switch v := v.(*AddChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -6307,7 +6549,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideResponse_Body); i { + switch v := v.(*AddChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -6319,7 +6561,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesRequest_Body); i { + switch v := v.(*GetChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -6331,7 +6573,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesResponse_Body); i { + switch v := v.(*GetChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -6343,7 +6585,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesRequest_Body); i { + switch v := v.(*ListChainLocalOverridesRequest_Body); i { case 0: return &v.state case 1: @@ -6355,7 +6597,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesResponse_Body); i { + switch v := v.(*ListChainLocalOverridesResponse_Body); i { case 0: return &v.state case 1: @@ -6367,7 +6609,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { + switch v := v.(*ListTargetsLocalOverridesRequest_Body); i { case 0: return &v.state case 1: @@ -6379,7 +6621,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[77].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { + switch v := v.(*ListTargetsLocalOverridesResponse_Body); i { case 0: return &v.state case 1: @@ -6391,7 +6633,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[78].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheRequest_Body); i { + switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -6403,7 +6645,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[79].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse_Body); i { + switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -6415,7 +6657,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[80].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse_Body_Status); i { + switch v := v.(*RemoveChainLocalOverridesByTargetRequest_Body); i { case 0: return &v.state case 1: @@ -6427,7 +6669,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[81].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsRequest_Body); i { + switch v := v.(*RemoveChainLocalOverridesByTargetResponse_Body); i { case 0: return &v.state case 1: @@ -6439,6 +6681,54 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[82].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[83].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[84].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheResponse_Body_Status); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[85].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DetachShardsRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[86].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DetachShardsResponse_Body); i { case 0: return &v.state @@ -6457,7 +6747,7 @@ func file_pkg_services_control_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_service_proto_rawDesc, NumEnums: 2, - NumMessages: 83, + NumMessages: 87, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 8501e952f..04ea62e0e 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -60,6 +60,9 @@ service ControlService { // Remove local access policy engine overrides stored in the node by chaind id. rpc RemoveChainLocalOverride (RemoveChainLocalOverrideRequest) returns (RemoveChainLocalOverrideResponse); + // Remove local access policy engine overrides stored in the node by chaind id. + rpc RemoveChainLocalOverridesByTarget (RemoveChainLocalOverridesByTargetRequest) returns (RemoveChainLocalOverridesByTargetResponse); + // List targets of the local APE overrides stored in the node. rpc ListTargetsLocalOverrides (ListTargetsLocalOverridesRequest) returns (ListTargetsLocalOverridesResponse); @@ -592,6 +595,26 @@ message RemoveChainLocalOverrideResponse { Signature signature = 2; } +message RemoveChainLocalOverridesByTargetRequest { + message Body { + // Target for which the overrides are applied. + ChainTarget target = 1; + } + + Body body = 1; + + Signature signature = 2; +} + +message RemoveChainLocalOverridesByTargetResponse { + message Body { + } + + Body body = 1; + + Signature signature = 2; +} + message SealWriteCacheRequest { // Request body structure. message Body { diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index b90749269..9d8d6376c 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -3085,6 +3085,169 @@ func (x *RemoveChainLocalOverrideResponse) SetSignature(sig *Signature) { x.Signature = sig } +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveChainLocalOverridesByTargetRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Target) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveChainLocalOverridesByTargetRequest_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveChainLocalOverridesByTargetRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveChainLocalOverridesByTargetRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *RemoveChainLocalOverridesByTargetRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *RemoveChainLocalOverridesByTargetRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *RemoveChainLocalOverridesByTargetRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveChainLocalOverridesByTargetResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveChainLocalOverridesByTargetResponse_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *RemoveChainLocalOverridesByTargetResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *RemoveChainLocalOverridesByTargetResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *RemoveChainLocalOverridesByTargetResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *RemoveChainLocalOverridesByTargetResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *RemoveChainLocalOverridesByTargetResponse) SetSignature(sig *Signature) { + x.Signature = sig +} + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index ad33e330a..95264fcd3 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -19,26 +19,27 @@ import ( const _ = grpc.SupportPackageIsVersion7 const ( - ControlService_HealthCheck_FullMethodName = "/control.ControlService/HealthCheck" - ControlService_SetNetmapStatus_FullMethodName = "/control.ControlService/SetNetmapStatus" - ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects" - ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" - ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" - ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree" - ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard" - ControlService_StartShardEvacuation_FullMethodName = "/control.ControlService/StartShardEvacuation" - ControlService_GetShardEvacuationStatus_FullMethodName = "/control.ControlService/GetShardEvacuationStatus" - ControlService_ResetShardEvacuationStatus_FullMethodName = "/control.ControlService/ResetShardEvacuationStatus" - ControlService_StopShardEvacuation_FullMethodName = "/control.ControlService/StopShardEvacuation" - ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache" - ControlService_Doctor_FullMethodName = "/control.ControlService/Doctor" - ControlService_AddChainLocalOverride_FullMethodName = "/control.ControlService/AddChainLocalOverride" - ControlService_GetChainLocalOverride_FullMethodName = "/control.ControlService/GetChainLocalOverride" - ControlService_ListChainLocalOverrides_FullMethodName = "/control.ControlService/ListChainLocalOverrides" - ControlService_RemoveChainLocalOverride_FullMethodName = "/control.ControlService/RemoveChainLocalOverride" - ControlService_ListTargetsLocalOverrides_FullMethodName = "/control.ControlService/ListTargetsLocalOverrides" - ControlService_SealWriteCache_FullMethodName = "/control.ControlService/SealWriteCache" - ControlService_DetachShards_FullMethodName = "/control.ControlService/DetachShards" + ControlService_HealthCheck_FullMethodName = "/control.ControlService/HealthCheck" + ControlService_SetNetmapStatus_FullMethodName = "/control.ControlService/SetNetmapStatus" + ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects" + ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" + ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" + ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree" + ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard" + ControlService_StartShardEvacuation_FullMethodName = "/control.ControlService/StartShardEvacuation" + ControlService_GetShardEvacuationStatus_FullMethodName = "/control.ControlService/GetShardEvacuationStatus" + ControlService_ResetShardEvacuationStatus_FullMethodName = "/control.ControlService/ResetShardEvacuationStatus" + ControlService_StopShardEvacuation_FullMethodName = "/control.ControlService/StopShardEvacuation" + ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache" + ControlService_Doctor_FullMethodName = "/control.ControlService/Doctor" + ControlService_AddChainLocalOverride_FullMethodName = "/control.ControlService/AddChainLocalOverride" + ControlService_GetChainLocalOverride_FullMethodName = "/control.ControlService/GetChainLocalOverride" + ControlService_ListChainLocalOverrides_FullMethodName = "/control.ControlService/ListChainLocalOverrides" + ControlService_RemoveChainLocalOverride_FullMethodName = "/control.ControlService/RemoveChainLocalOverride" + ControlService_RemoveChainLocalOverridesByTarget_FullMethodName = "/control.ControlService/RemoveChainLocalOverridesByTarget" + ControlService_ListTargetsLocalOverrides_FullMethodName = "/control.ControlService/ListTargetsLocalOverrides" + ControlService_SealWriteCache_FullMethodName = "/control.ControlService/SealWriteCache" + ControlService_DetachShards_FullMethodName = "/control.ControlService/DetachShards" ) // ControlServiceClient is the client API for ControlService service. @@ -80,6 +81,8 @@ type ControlServiceClient interface { ListChainLocalOverrides(ctx context.Context, in *ListChainLocalOverridesRequest, opts ...grpc.CallOption) (*ListChainLocalOverridesResponse, error) // Remove local access policy engine overrides stored in the node by chaind id. RemoveChainLocalOverride(ctx context.Context, in *RemoveChainLocalOverrideRequest, opts ...grpc.CallOption) (*RemoveChainLocalOverrideResponse, error) + // Remove local access policy engine overrides stored in the node by chaind id. + RemoveChainLocalOverridesByTarget(ctx context.Context, in *RemoveChainLocalOverridesByTargetRequest, opts ...grpc.CallOption) (*RemoveChainLocalOverridesByTargetResponse, error) // List targets of the local APE overrides stored in the node. ListTargetsLocalOverrides(ctx context.Context, in *ListTargetsLocalOverridesRequest, opts ...grpc.CallOption) (*ListTargetsLocalOverridesResponse, error) // Flush objects from write-cache and move it to degraded read only mode. @@ -249,6 +252,15 @@ func (c *controlServiceClient) RemoveChainLocalOverride(ctx context.Context, in return out, nil } +func (c *controlServiceClient) RemoveChainLocalOverridesByTarget(ctx context.Context, in *RemoveChainLocalOverridesByTargetRequest, opts ...grpc.CallOption) (*RemoveChainLocalOverridesByTargetResponse, error) { + out := new(RemoveChainLocalOverridesByTargetResponse) + err := c.cc.Invoke(ctx, ControlService_RemoveChainLocalOverridesByTarget_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *controlServiceClient) ListTargetsLocalOverrides(ctx context.Context, in *ListTargetsLocalOverridesRequest, opts ...grpc.CallOption) (*ListTargetsLocalOverridesResponse, error) { out := new(ListTargetsLocalOverridesResponse) err := c.cc.Invoke(ctx, ControlService_ListTargetsLocalOverrides_FullMethodName, in, out, opts...) @@ -315,6 +327,8 @@ type ControlServiceServer interface { ListChainLocalOverrides(context.Context, *ListChainLocalOverridesRequest) (*ListChainLocalOverridesResponse, error) // Remove local access policy engine overrides stored in the node by chaind id. RemoveChainLocalOverride(context.Context, *RemoveChainLocalOverrideRequest) (*RemoveChainLocalOverrideResponse, error) + // Remove local access policy engine overrides stored in the node by chaind id. + RemoveChainLocalOverridesByTarget(context.Context, *RemoveChainLocalOverridesByTargetRequest) (*RemoveChainLocalOverridesByTargetResponse, error) // List targets of the local APE overrides stored in the node. ListTargetsLocalOverrides(context.Context, *ListTargetsLocalOverridesRequest) (*ListTargetsLocalOverridesResponse, error) // Flush objects from write-cache and move it to degraded read only mode. @@ -378,6 +392,9 @@ func (UnimplementedControlServiceServer) ListChainLocalOverrides(context.Context func (UnimplementedControlServiceServer) RemoveChainLocalOverride(context.Context, *RemoveChainLocalOverrideRequest) (*RemoveChainLocalOverrideResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method RemoveChainLocalOverride not implemented") } +func (UnimplementedControlServiceServer) RemoveChainLocalOverridesByTarget(context.Context, *RemoveChainLocalOverridesByTargetRequest) (*RemoveChainLocalOverridesByTargetResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RemoveChainLocalOverridesByTarget not implemented") +} func (UnimplementedControlServiceServer) ListTargetsLocalOverrides(context.Context, *ListTargetsLocalOverridesRequest) (*ListTargetsLocalOverridesResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListTargetsLocalOverrides not implemented") } @@ -705,6 +722,24 @@ func _ControlService_RemoveChainLocalOverride_Handler(srv interface{}, ctx conte return interceptor(ctx, in, info, handler) } +func _ControlService_RemoveChainLocalOverridesByTarget_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RemoveChainLocalOverridesByTargetRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).RemoveChainLocalOverridesByTarget(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_RemoveChainLocalOverridesByTarget_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).RemoveChainLocalOverridesByTarget(ctx, req.(*RemoveChainLocalOverridesByTargetRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _ControlService_ListTargetsLocalOverrides_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ListTargetsLocalOverridesRequest) if err := dec(in); err != nil { @@ -834,6 +869,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "RemoveChainLocalOverride", Handler: _ControlService_RemoveChainLocalOverride_Handler, }, + { + MethodName: "RemoveChainLocalOverridesByTarget", + Handler: _ControlService_RemoveChainLocalOverridesByTarget_Handler, + }, { MethodName: "ListTargetsLocalOverrides", Handler: _ControlService_ListTargetsLocalOverrides_Handler, From 11fde3cde4d19a274a953a5ea02d5f3aea7c3041 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 12 Mar 2024 13:43:16 +0300 Subject: [PATCH 0982/1943] [#976] cli: Allow to remove all chains by target Signed-off-by: Anton Nikiforov --- .../modules/control/remove_rule.go | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/remove_rule.go b/cmd/frostfs-cli/modules/control/remove_rule.go index 7684ac1dc..4189ea76b 100644 --- a/cmd/frostfs-cli/modules/control/remove_rule.go +++ b/cmd/frostfs-cli/modules/control/remove_rule.go @@ -2,6 +2,7 @@ package control import ( "encoding/hex" + "errors" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" @@ -13,21 +14,48 @@ import ( const ( chainIDFlag = "chain-id" chainIDHexFlag = "chain-id-hex" + allFlag = "all" ) -var removeRuleCmd = &cobra.Command{ - Use: "remove-rule", - Short: "Remove local override", - Long: "Remove local APE override of the node", - Run: removeRule, -} +var ( + errEmptyChainID = errors.New("chain id cannot be empty") + + removeRuleCmd = &cobra.Command{ + Use: "remove-rule", + Short: "Remove local override", + Long: "Remove local APE override of the node", + Run: removeRule, + } +) func removeRule(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) + hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) + removeAll, _ := cmd.Flags().GetBool(allFlag) + if removeAll { + req := &control.RemoveChainLocalOverridesByTargetRequest{ + Body: &control.RemoveChainLocalOverridesByTargetRequest_Body{ + Target: parseTarget(cmd), + }, + } + signRequest(cmd, pk, req) + cli := getClient(cmd, pk) + var resp *control.RemoveChainLocalOverridesByTargetResponse + var err error + err = cli.ExecRaw(func(client *client.Client) error { + resp, err = control.RemoveChainLocalOverridesByTarget(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + cmd.Println("All rules have been removed.") + return + } chainID, _ := cmd.Flags().GetString(chainIDFlag) - hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) - + if chainID == "" { + commonCmd.ExitOnErr(cmd, "read chain id error: %w", errEmptyChainID) + } chainIDRaw := []byte(chainID) if hexEncoded { @@ -69,4 +97,6 @@ func initControlRemoveRuleCmd() { _ = removeRuleCmd.MarkFlagRequired(targetTypeFlag) ff.String(chainIDFlag, "", "Chain id") ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") + ff.Bool(allFlag, false, "Remove all chains") + removeRuleCmd.MarkFlagsMutuallyExclusive(allFlag, chainIDFlag) } From bf70d7784457442b30669b123eacb7aa5c24de04 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 7 Mar 2024 14:33:07 +0300 Subject: [PATCH 0983/1943] [#976] adm: Allow to remove all chains by target Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/internal/commonflags/flags.go | 1 + .../internal/modules/morph/ape/ape.go | 24 +++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-adm/internal/commonflags/flags.go b/cmd/frostfs-adm/internal/commonflags/flags.go index 71c36b71a..bf6c17bd1 100644 --- a/cmd/frostfs-adm/internal/commonflags/flags.go +++ b/cmd/frostfs-adm/internal/commonflags/flags.go @@ -36,4 +36,5 @@ const ( HomomorphicHashDisabledInitFlag = "network.homomorphic_hash_disabled" CustomZoneFlag = "domain" AlphabetSizeFlag = "size" + AllFlag = "all" ) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go index e3990da07..fc572f5ff 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go @@ -114,8 +114,9 @@ func initRemoveRuleChainCmd() { removeRuleChainCmd.Flags().String(targetNameFlag, "", targetNameDesc) _ = removeRuleChainCmd.MarkFlagRequired(targetNameFlag) removeRuleChainCmd.Flags().String(chainIDFlag, "", chainIDDesc) - _ = removeRuleChainCmd.MarkFlagRequired(chainIDFlag) removeRuleChainCmd.Flags().String(chainNameFlag, ingress, chainNameFlagDesc) + removeRuleChainCmd.Flags().Bool(commonflags.AllFlag, false, "Remove all chains for target") + removeRuleChainCmd.MarkFlagsMutuallyExclusive(commonflags.AllFlag, chainIDFlag) } func initListRuleChainsCmd() { @@ -157,14 +158,23 @@ func addRuleChain(cmd *cobra.Command, _ []string) { } func removeRuleChain(cmd *cobra.Command, _ []string) { - chainID := parseChainID(cmd) target := parseTarget(cmd) pci, ac := newPolicyContractInterface(cmd) - h, vub, err := pci.RemoveMorphRuleChain(parseChainName(cmd), target, chainID) - cmd.Println("Waiting for transaction to persist...") - _, err = ac.Wait(h, vub, err) - commonCmd.ExitOnErr(cmd, "remove rule chain error: %w", err) - cmd.Println("Rule chain removed successfully") + removeAll, _ := cmd.Flags().GetBool(commonflags.AllFlag) + if removeAll { + h, vub, err := pci.RemoveMorphRuleChainsByTarget(parseChainName(cmd), target) + cmd.Println("Waiting for transaction to persist...") + _, err = ac.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "remove rule chain error: %w", err) + cmd.Println("All chains for target removed successfully") + } else { + chainID := parseChainID(cmd) + h, vub, err := pci.RemoveMorphRuleChain(parseChainName(cmd), target, chainID) + cmd.Println("Waiting for transaction to persist...") + _, err = ac.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "remove rule chain error: %w", err) + cmd.Println("Rule chain removed successfully") + } } func listRuleChains(cmd *cobra.Command, _ []string) { From fb9219af3961aa69c69e80a4526622ce637b0ddb Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 12 Mar 2024 11:02:14 +0300 Subject: [PATCH 0984/1943] [#976] Fix trailing whitespace and end of file Signed-off-by: Anton Nikiforov --- dev/empty_pass.yml | 2 +- docs/evacuation.md | 4 ++-- docs/shard-modes.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/empty_pass.yml b/dev/empty_pass.yml index 2a088f952..1986cf9e4 100644 --- a/dev/empty_pass.yml +++ b/dev/empty_pass.yml @@ -1 +1 @@ -password: "" \ No newline at end of file +password: "" diff --git a/docs/evacuation.md b/docs/evacuation.md index 602e32e9b..9db514a9e 100644 --- a/docs/evacuation.md +++ b/docs/evacuation.md @@ -96,9 +96,9 @@ Shard IDs: 54Y8aot9uc7BSadw2XtYr3. Evacuated 618 objects out of 618, failed to e ### Start trees evacuation and await it completes ```bash frostfs-cli control shards evacuation start --id FxR6QujButNCHn7jjdhxGP --endpoint s01.frostfs.devenv:8081 --wallet ./../frostfs-dev-env/services/storage/wallet01.json --await --scope trees -Enter password > +Enter password > Shard evacuation has been successfully started. Progress will be reported every 5 seconds. Shard evacuation has been completed. Shard IDs: FxR6QujButNCHn7jjdhxGP. Evacuated 0 objects out of 0, failed to evacuate: 0, skipped: 0; evacuated 2 trees out of 2, failed to evacuate: 0. Started at: 2024-02-08T08:44:17Z UTC. Duration: 00:00:00. -``` \ No newline at end of file +``` diff --git a/docs/shard-modes.md b/docs/shard-modes.md index 3abc26a60..789a1c241 100644 --- a/docs/shard-modes.md +++ b/docs/shard-modes.md @@ -51,4 +51,4 @@ Shard can automatically switch to a `degraded-read-only` mode in 3 cases: To detach a shard use `frostfs-cli control shards detach` command. This command removes the shards from the storage engine and closes all resources associated with the shards. -Limitation: `SIGHUP` or storage node restart lead to detached shard will be again online. \ No newline at end of file +Limitation: `SIGHUP` or storage node restart lead to detached shard will be again online. From d7be70e93f851df42d573f7ef736359745e75f0f Mon Sep 17 00:00:00 2001 From: aarifullin Date: Tue, 12 Mar 2024 17:08:09 +0300 Subject: [PATCH 0985/1943] [#1040] object: Wrap CheckAPE errors to status errors * All methods should wrap CheckAPE error, if it occurs, to status error. Signed-off-by: Airat Arifullin --- pkg/services/object/ape/service.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index cc18a0bcf..8f45a8ffa 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -214,7 +214,7 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { - return nil, err + return nil, toStatusErr(err) } resp, err := c.next.Head(ctx, request) @@ -252,7 +252,7 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { - return nil, err + return nil, toStatusErr(err) } return resp, nil } @@ -306,7 +306,7 @@ func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) ( SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { - return nil, err + return nil, toStatusErr(err) } resp, err := c.next.Delete(ctx, request) @@ -366,7 +366,7 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { - return nil, err + return nil, toStatusErr(err) } resp, err := c.next.GetRangeHash(ctx, request) @@ -375,7 +375,7 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { - return nil, err + return nil, toStatusErr(err) } return resp, nil } From bd216b79cb30e560165095172c44b7e48e2ce362 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 15 Mar 2024 09:54:36 +0300 Subject: [PATCH 0986/1943] Release v0.38.0 Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 3 ++- VERSION | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc2902230..7c69b69d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ Changelog for FrostFS Node ### Removed ### Updated -## [v0.38.0-rc.2] +## [v0.38.0] ### Added - Add `trace_id` to logs in `frostfs-node` (#146) @@ -18,6 +18,7 @@ Changelog for FrostFS Node - Allow sealing writecache (#569) - Support tree service in data evacuation (#947) - Use new policy engine mechanism for access control (#770, #804) +- Log about active notary deposit waiting (#963) ### Changed - Sort output in `frostfs-cli` subcommands (#333) diff --git a/VERSION b/VERSION index e1d6235d3..765098dc4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.36.0 +v0.38.0 From 7278201753e2e0abc3fd6013d8c3d6085dcfb34d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 11 Mar 2024 11:22:56 +0300 Subject: [PATCH 0987/1943] [#1030] adm: Add command `morph ape list-targets` Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/ape/ape.go | 40 +++++++++++++++++++ .../internal/modules/morph/ape/ape_util.go | 25 +++++++----- .../internal/modules/morph/ape/root.go | 1 + 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go index fc572f5ff..710595fe8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go @@ -82,6 +82,15 @@ var ( }, Run: getAdmin, } + + listTargetsCmd = &cobra.Command{ + Use: "list-targets", + Short: "List targets", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + }, + Run: listTargets, + } ) func initAddRuleChainCmd() { @@ -146,6 +155,14 @@ func initGetAdminCmd() { getAdminCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) } +func initListTargetsCmd() { + Cmd.AddCommand(listTargetsCmd) + + listTargetsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + listTargetsCmd.Flags().StringP(targetTypeFlag, "t", "", targetTypeDesc) + _ = listTargetsCmd.MarkFlagRequired(targetTypeFlag) +} + func addRuleChain(cmd *cobra.Command, _ []string) { chain := parseChain(cmd) target := parseTarget(cmd) @@ -215,6 +232,29 @@ func getAdmin(cmd *cobra.Command, _ []string) { cmd.Println(addr.StringLE()) } +func listTargets(cmd *cobra.Command, _ []string) { + typ, err := parseTargetType(cmd) + commonCmd.ExitOnErr(cmd, "parse target type error: %w", err) + pci, inv := newPolicyContractReaderInterface(cmd) + + sid, it, err := pci.ListTargetsIterator(typ) + commonCmd.ExitOnErr(cmd, "list targets error: %w", err) + items, err := inv.TraverseIterator(sid, &it, 0) + for err == nil && len(items) != 0 { + for _, item := range items { + bts, err := item.TryBytes() + commonCmd.ExitOnErr(cmd, "list targets error: %w", err) + if len(bts) == 0 { + cmd.Println("(no name)") + } else { + cmd.Println(string(bts)) + } + } + items, err = inv.TraverseIterator(sid, &it, 0) + commonCmd.ExitOnErr(cmd, "unable to list targets: %w", err) + } +} + func prettyJSONFormat(cmd *cobra.Command, chains []*apechain.Chain) { wr := bytes.NewBufferString("") data, err := json.Marshal(chains) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 253e130a8..b3610e7f7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -36,24 +36,27 @@ var ( ) func parseTarget(cmd *cobra.Command) policyengine.Target { - var targetType policyengine.TargetType - typ, _ := cmd.Flags().GetString(targetTypeFlag) - switch typ { - case namespaceTarget: - targetType = policyengine.Namespace - case containerTarget: - targetType = policyengine.Container - default: - commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType) - } name, _ := cmd.Flags().GetString(targetNameFlag) + typ, err := parseTargetType(cmd) + commonCmd.ExitOnErr(cmd, "read target type error: %w", err) return policyengine.Target{ Name: name, - Type: targetType, + Type: typ, } } +func parseTargetType(cmd *cobra.Command) (policyengine.TargetType, error) { + typ, _ := cmd.Flags().GetString(targetTypeFlag) + switch typ { + case namespaceTarget: + return policyengine.Namespace, nil + case containerTarget: + return policyengine.Container, nil + } + return -1, errUnknownTargetType +} + func parseChainID(cmd *cobra.Command) apechain.ID { chainID, _ := cmd.Flags().GetString(chainIDFlag) if chainID == "" { diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/root.go b/cmd/frostfs-adm/internal/modules/morph/ape/root.go index 11b6894cf..a4746cd2c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/root.go @@ -13,4 +13,5 @@ func init() { initListRuleChainsCmd() initSetAdminCmd() initGetAdminCmd() + initListTargetsCmd() } From 6959e617c49d5b4f9f66a0c88223cd317c82bb70 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Fri, 15 Mar 2024 15:16:52 +0300 Subject: [PATCH 0988/1943] [#1047] object: Set container owner ID property to ape request * Introduce ContainerOwner field in RequestContext. * Set ContainerOwner in aclv2 middleware. * Set PropertyKeyContainerOwnerID for object ape request. Signed-off-by: Airat Arifullin --- pkg/services/object/acl/v2/service.go | 36 +++--- pkg/services/object/ape/checker.go | 4 + pkg/services/object/ape/request.go | 7 +- pkg/services/object/ape/request_test.go | 29 +++-- pkg/services/object/ape/service.go | 159 +++++++++++++----------- pkg/services/object/request_context.go | 7 +- 6 files changed, 142 insertions(+), 100 deletions(-) diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index b51386e0c..1f1c275cf 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -113,10 +113,11 @@ type wrappedGetObjectStream struct { func (w *wrappedGetObjectStream) Context() context.Context { return context.WithValue(w.GetObjectStream.Context(), object.RequestContextKey, &object.RequestContext{ - Namespace: w.requestInfo.ContainerNamespace(), - SenderKey: w.requestInfo.SenderKey(), - Role: w.requestInfo.RequestRole(), - SoftAPECheck: w.requestInfo.IsSoftAPECheck(), + Namespace: w.requestInfo.ContainerNamespace(), + ContainerOwner: w.requestInfo.ContainerOwner(), + SenderKey: w.requestInfo.SenderKey(), + Role: w.requestInfo.RequestRole(), + SoftAPECheck: w.requestInfo.IsSoftAPECheck(), }) } @@ -137,10 +138,11 @@ type wrappedRangeStream struct { func (w *wrappedRangeStream) Context() context.Context { return context.WithValue(w.GetObjectRangeStream.Context(), object.RequestContextKey, &object.RequestContext{ - Namespace: w.requestInfo.ContainerNamespace(), - SenderKey: w.requestInfo.SenderKey(), - Role: w.requestInfo.RequestRole(), - SoftAPECheck: w.requestInfo.IsSoftAPECheck(), + Namespace: w.requestInfo.ContainerNamespace(), + ContainerOwner: w.requestInfo.ContainerOwner(), + SenderKey: w.requestInfo.SenderKey(), + Role: w.requestInfo.RequestRole(), + SoftAPECheck: w.requestInfo.IsSoftAPECheck(), }) } @@ -161,10 +163,11 @@ type wrappedSearchStream struct { func (w *wrappedSearchStream) Context() context.Context { return context.WithValue(w.SearchStream.Context(), object.RequestContextKey, &object.RequestContext{ - Namespace: w.requestInfo.ContainerNamespace(), - SenderKey: w.requestInfo.SenderKey(), - Role: w.requestInfo.RequestRole(), - SoftAPECheck: w.requestInfo.IsSoftAPECheck(), + Namespace: w.requestInfo.ContainerNamespace(), + ContainerOwner: w.requestInfo.ContainerOwner(), + SenderKey: w.requestInfo.SenderKey(), + Role: w.requestInfo.RequestRole(), + SoftAPECheck: w.requestInfo.IsSoftAPECheck(), }) } @@ -471,10 +474,11 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb func requestContext(ctx context.Context, reqInfo RequestInfo) context.Context { return context.WithValue(ctx, object.RequestContextKey, &object.RequestContext{ - Namespace: reqInfo.ContainerNamespace(), - SenderKey: reqInfo.SenderKey(), - Role: reqInfo.RequestRole(), - SoftAPECheck: reqInfo.IsSoftAPECheck(), + Namespace: reqInfo.ContainerNamespace(), + ContainerOwner: reqInfo.ContainerOwner(), + SenderKey: reqInfo.SenderKey(), + Role: reqInfo.RequestRole(), + SoftAPECheck: reqInfo.IsSoftAPECheck(), }) } diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 1d42d21d4..45efce5a7 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -8,6 +8,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" @@ -48,6 +49,9 @@ type Prm struct { // An encoded sender's public key string. SenderKey string + // An encoded container's owner user ID. + ContainerOwner user.ID + // If SoftAPECheck is set to true, then NoRuleFound is interpreted as allow. SoftAPECheck bool } diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index caf52645c..3689bd178 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -10,6 +10,7 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" aperesource "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" ) @@ -78,11 +79,13 @@ func resourceName(cid cid.ID, oid *oid.ID, namespace string) string { } // objectProperties collects object properties from address parameters and a header if it is passed. -func objectProperties(cnr cid.ID, oid *oid.ID, header *objectV2.Header) map[string]string { +func objectProperties(cnr cid.ID, oid *oid.ID, cnrOwner user.ID, header *objectV2.Header) map[string]string { objectProps := map[string]string{ nativeschema.PropertyKeyObjectContainerID: cnr.EncodeToString(), } + objectProps[nativeschema.PropertyKeyContainerOwnerID] = cnrOwner.EncodeToString() + if oid != nil { objectProps[nativeschema.PropertyKeyObjectID] = oid.String() } @@ -149,7 +152,7 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (*request, err operation: prm.Method, resource: &resource{ name: resourceName(prm.Container, prm.Object, prm.Namespace), - properties: objectProperties(prm.Container, prm.Object, header), + properties: objectProperties(prm.Container, prm.Object, prm.ContainerOwner, header), }, properties: map[string]string{ nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, diff --git a/pkg/services/object/ape/request_test.go b/pkg/services/object/ape/request_test.go index 0dcc0b84b..c5b43fa8b 100644 --- a/pkg/services/object/ape/request_test.go +++ b/pkg/services/object/ape/request_test.go @@ -8,11 +8,16 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/stretchr/testify/require" ) +const ( + testOwnerID = "FPPtmAi9TCX329" +) + func TestObjectProperties(t *testing.T) { for _, test := range []struct { name string @@ -82,8 +87,12 @@ func TestObjectProperties(t *testing.T) { obj := newObjectIDSDK(t, test.object) header := newHeaderObjectSDK(cnr, obj, test.header) - props := objectProperties(cnr, obj, header.ToV2().GetHeader()) + var testCnrOwner user.ID + require.NoError(t, testCnrOwner.DecodeString(testOwnerID)) + + props := objectProperties(cnr, obj, testCnrOwner, header.ToV2().GetHeader()) require.Equal(t, test.container, props[nativeschema.PropertyKeyObjectContainerID]) + require.Equal(t, testOwnerID, props[nativeschema.PropertyKeyContainerOwnerID]) if obj != nil { require.Equal(t, *test.object, props[nativeschema.PropertyKeyObjectID]) @@ -210,13 +219,17 @@ func TestNewAPERequest(t *testing.T) { cnr := newContainerIDSDK(t, test.container) obj := newObjectIDSDK(t, test.object) + var testCnrOwner user.ID + require.NoError(t, testCnrOwner.DecodeString(testOwnerID)) + prm := Prm{ - Namespace: test.namespace, - Method: method, - Container: cnr, - Object: obj, - Role: role, - SenderKey: senderKey, + Namespace: test.namespace, + Method: method, + Container: cnr, + Object: obj, + Role: role, + SenderKey: senderKey, + ContainerOwner: testCnrOwner, } headerSource := newHeaderProviderMock() @@ -247,7 +260,7 @@ func TestNewAPERequest(t *testing.T) { operation: method, resource: &resource{ name: resourceName(cnr, obj, prm.Namespace), - properties: objectProperties(cnr, obj, func() *objectV2.Header { + properties: objectProperties(cnr, obj, testCnrOwner, func() *objectV2.Header { if headerObjSDK != nil { return headerObjSDK.ToV2().GetHeader() } diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 8f45a8ffa..0c203209d 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -14,6 +14,7 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" ) @@ -67,6 +68,8 @@ type getStreamBasicChecker struct { senderKey []byte + containerOwner user.ID + role string softAPECheck bool @@ -80,14 +83,15 @@ func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { } prm := Prm{ - Namespace: g.namespace, - Container: cnrID, - Object: objID, - Header: partInit.GetHeader(), - Method: nativeschema.MethodGetObject, - SenderKey: hex.EncodeToString(g.senderKey), - Role: g.role, - SoftAPECheck: g.softAPECheck, + Namespace: g.namespace, + Container: cnrID, + Object: objID, + Header: partInit.GetHeader(), + Method: nativeschema.MethodGetObject, + SenderKey: hex.EncodeToString(g.senderKey), + ContainerOwner: g.containerOwner, + Role: g.role, + SoftAPECheck: g.softAPECheck, } if err := g.apeChecker.CheckAPE(g.Context(), prm); err != nil { @@ -121,13 +125,14 @@ func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectSt } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodGetObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - SoftAPECheck: reqCtx.SoftAPECheck, + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodGetObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return toStatusErr(err) @@ -162,14 +167,15 @@ func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutR } prm := Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Header: partInit.GetHeader(), - Method: nativeschema.MethodPutObject, - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - Role: nativeSchemaRole(reqCtx.Role), - SoftAPECheck: reqCtx.SoftAPECheck, + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Header: partInit.GetHeader(), + Method: nativeschema.MethodPutObject, + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + Role: nativeSchemaRole(reqCtx.Role), + SoftAPECheck: reqCtx.SoftAPECheck, } if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -205,13 +211,14 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj } err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodHeadObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - SoftAPECheck: reqCtx.SoftAPECheck, + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodHeadObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return nil, toStatusErr(err) @@ -242,14 +249,15 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj } err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Header: header, - Method: nativeschema.MethodHeadObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - SoftAPECheck: reqCtx.SoftAPECheck, + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Header: header, + Method: nativeschema.MethodHeadObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return nil, toStatusErr(err) @@ -271,12 +279,13 @@ func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.Searc } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Method: nativeschema.MethodSearchObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - SoftAPECheck: reqCtx.SoftAPECheck, + Namespace: reqCtx.Namespace, + Container: cnrID, + Method: nativeschema.MethodSearchObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return toStatusErr(err) @@ -297,13 +306,14 @@ func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) ( } err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodDeleteObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - SoftAPECheck: reqCtx.SoftAPECheck, + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodDeleteObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return nil, toStatusErr(err) @@ -329,13 +339,14 @@ func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.G } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodRangeObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - SoftAPECheck: reqCtx.SoftAPECheck, + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodRangeObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + SoftAPECheck: reqCtx.SoftAPECheck, }) if err != nil { return toStatusErr(err) @@ -356,13 +367,14 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa } prm := Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodHashObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - SoftAPECheck: reqCtx.SoftAPECheck, + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodHashObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + SoftAPECheck: reqCtx.SoftAPECheck, } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -392,14 +404,15 @@ func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequ } prm := Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Header: request.GetBody().GetObject().GetHeader(), - Method: nativeschema.MethodPutObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - SoftAPECheck: reqCtx.SoftAPECheck, + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Header: request.GetBody().GetObject().GetHeader(), + Method: nativeschema.MethodPutObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + SoftAPECheck: reqCtx.SoftAPECheck, } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { diff --git a/pkg/services/object/request_context.go b/pkg/services/object/request_context.go index e0e6c9ed9..6a0965b40 100644 --- a/pkg/services/object/request_context.go +++ b/pkg/services/object/request_context.go @@ -1,6 +1,9 @@ package object -import "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" +) type RequestContextKeyT struct{} @@ -12,6 +15,8 @@ type RequestContext struct { SenderKey []byte + ContainerOwner user.ID + Role acl.Role SoftAPECheck bool From 6bf77cabd452bad85bb3418969a38b8fc24acb59 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 15 Mar 2024 10:10:03 +0300 Subject: [PATCH 0989/1943] [#1044] ape: Add morph chain cache Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 10 ++- cmd/frostfs-node/config/morph/config.go | 15 +++++ cmd/frostfs-node/config/morph/config_test.go | 2 + cmd/frostfs-node/policy_engine.go | 67 ++++++++++++++++++++ config/example/node.env | 1 + config/example/node.json | 3 +- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 40 ++++++------ 8 files changed, 118 insertions(+), 21 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 48b76ac0f..424970b8f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -24,6 +24,7 @@ import ( blobovniczaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza" fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger" + morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" @@ -68,6 +69,7 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + policy_engine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" policy_client "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" neogoutil "github.com/nspcc-dev/neo-go/pkg/util" @@ -1074,10 +1076,16 @@ func initAccessPolicyEngine(_ context.Context, c *cfg) { ) } - morphRuleStorage := policy_client.NewContractStorage( + var morphRuleStorage policy_engine.MorphRuleChainStorage + morphRuleStorage = policy_client.NewContractStorage( client.NewSwitchRPCGuardedActor(c.cfgMorph.client), c.cfgObject.cfgAccessPolicyEngine.policyContractHash) + cacheSize := morphconfig.APEChainCacheSize(c.appCfg) + if cacheSize > 0 { + morphRuleStorage = newMorphCache(morphRuleStorage, int(cacheSize), morphconfig.CacheTTL(c.appCfg)) + } + ape := newAccessPolicyEngine(morphRuleStorage, localOverrideDB) c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine = ape diff --git a/cmd/frostfs-node/config/morph/config.go b/cmd/frostfs-node/config/morph/config.go index 866223073..c32940625 100644 --- a/cmd/frostfs-node/config/morph/config.go +++ b/cmd/frostfs-node/config/morph/config.go @@ -25,6 +25,9 @@ const ( // SwitchIntervalDefault is a default Neo RPCs switch interval. SwitchIntervalDefault = 2 * time.Minute + + // APEChainCacheSizeDefault is a default value of APE chain cache. + APEChainCacheSizeDefault = 10_000 ) var errNoMorphEndpoints = errors.New("no morph chain RPC endpoints, see `morph.rpc_endpoint` section") @@ -99,3 +102,15 @@ func SwitchInterval(c *config.Config) time.Duration { return SwitchIntervalDefault } + +// APEChainCacheSize returns the value of "ape_chain_cache_size" config parameter +// from "morph" section. +// +// Returns 0 if the value is not positive integer. +// Returns APEChainCacheSizeDefault if the value is missing. +func APEChainCacheSize(c *config.Config) uint32 { + if c.Sub(subsection).Value("ape_chain_cache_size") == nil { + return APEChainCacheSizeDefault + } + return config.Uint32Safe(c.Sub(subsection), "ape_chain_cache_size") +} diff --git a/cmd/frostfs-node/config/morph/config_test.go b/cmd/frostfs-node/config/morph/config_test.go index a30733cd0..13da3d70d 100644 --- a/cmd/frostfs-node/config/morph/config_test.go +++ b/cmd/frostfs-node/config/morph/config_test.go @@ -19,6 +19,7 @@ func TestMorphSection(t *testing.T) { require.Equal(t, morphconfig.DialTimeoutDefault, morphconfig.DialTimeout(empty)) require.Equal(t, morphconfig.CacheTTLDefault, morphconfig.CacheTTL(empty)) require.Equal(t, morphconfig.SwitchIntervalDefault, morphconfig.SwitchInterval(empty)) + require.Equal(t, uint32(morphconfig.APEChainCacheSizeDefault), morphconfig.APEChainCacheSize(empty)) }) const path = "../../../../config/example/node" @@ -39,6 +40,7 @@ func TestMorphSection(t *testing.T) { require.Equal(t, 30*time.Second, morphconfig.DialTimeout(c)) require.Equal(t, 15*time.Second, morphconfig.CacheTTL(c)) require.Equal(t, 3*time.Minute, morphconfig.SwitchInterval(c)) + require.Equal(t, uint32(100000), morphconfig.APEChainCacheSize(c)) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/policy_engine.go b/cmd/frostfs-node/policy_engine.go index 74cbf841c..100520f50 100644 --- a/cmd/frostfs-node/policy_engine.go +++ b/cmd/frostfs-node/policy_engine.go @@ -2,11 +2,16 @@ package main import ( "sync" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" + "github.com/google/uuid" + "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/util" ) type accessPolicyEngine struct { @@ -21,6 +26,68 @@ type accessPolicyEngine struct { var _ engine.LocalOverrideEngine = (*accessPolicyEngine)(nil) +var _ engine.MorphRuleChainStorage = (*morphAPEChainCache)(nil) + +type morphAPEChainCacheKey struct { + name chain.Name + target engine.Target +} + +type morphAPEChainCache struct { + source engine.MorphRuleChainStorage + cache *expirable.LRU[morphAPEChainCacheKey, []*chain.Chain] +} + +func newMorphCache(source engine.MorphRuleChainStorage, size int, ttl time.Duration) engine.MorphRuleChainStorage { + return &morphAPEChainCache{ + source: source, + cache: expirable.NewLRU(size, func(morphAPEChainCacheKey, []*chain.Chain) {}, ttl), + } +} + +func (m *morphAPEChainCache) AddMorphRuleChain(name chain.Name, target engine.Target, c *chain.Chain) (util.Uint256, uint32, error) { + m.cache.Remove(morphAPEChainCacheKey{name: name, target: target}) + return m.source.AddMorphRuleChain(name, target, c) +} + +func (m *morphAPEChainCache) GetAdmin() (util.Uint160, error) { + return m.source.GetAdmin() +} + +func (m *morphAPEChainCache) ListMorphRuleChains(name chain.Name, target engine.Target) ([]*chain.Chain, error) { + key := morphAPEChainCacheKey{name: name, target: target} + result, found := m.cache.Get(key) + if found { + return result, nil + } + + result, err := m.source.ListMorphRuleChains(name, target) + if err != nil { + return nil, err + } + + m.cache.Add(key, result) + return result, nil +} + +func (m *morphAPEChainCache) RemoveMorphRuleChain(name chain.Name, target engine.Target, chainID chain.ID) (util.Uint256, uint32, error) { + m.cache.Remove(morphAPEChainCacheKey{name: name, target: target}) + return m.source.RemoveMorphRuleChain(name, target, chainID) +} + +func (m *morphAPEChainCache) SetAdmin(addr util.Uint160) (util.Uint256, uint32, error) { + return m.source.SetAdmin(addr) +} + +func (m *morphAPEChainCache) ListTargetsIterator(targetType engine.TargetType) (uuid.UUID, result.Iterator, error) { + return m.source.ListTargetsIterator(targetType) +} + +func (m *morphAPEChainCache) RemoveMorphRuleChainsByTarget(name chain.Name, target engine.Target) (util.Uint256, uint32, error) { + m.cache.Remove(morphAPEChainCacheKey{name: name, target: target}) + return m.source.RemoveMorphRuleChainsByTarget(name, target) +} + func newAccessPolicyEngine( morphChainStorage engine.MorphRuleChainStorage, localOverrideDatabase chainbase.LocalOverrideDatabase, diff --git a/config/example/node.env b/config/example/node.env index 810a2842b..976c42629 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -70,6 +70,7 @@ FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS="wss://rpc1.morph.frostfs.info:40341/ws" FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY=0 FROSTFS_MORPH_RPC_ENDPOINT_1_ADDRESS="wss://rpc2.morph.frostfs.info:40341/ws" FROSTFS_MORPH_RPC_ENDPOINT_1_PRIORITY=2 +FROSTFS_MORPH_APE_CHAIN_CACHE_SIZE=100000 # API Client section FROSTFS_APICLIENT_DIAL_TIMEOUT=15s diff --git a/config/example/node.json b/config/example/node.json index e24a12f5f..648fb77b0 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -110,7 +110,8 @@ "address": "wss://rpc2.morph.frostfs.info:40341/ws", "priority": 2 } - ] + ], + "ape_chain_cache_size": 100000 }, "apiclient": { "dial_timeout": "15s", diff --git a/config/example/node.yaml b/config/example/node.yaml index c9886355f..2dcf7c4d9 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -94,6 +94,7 @@ morph: priority: 0 - address: wss://rpc2.morph.frostfs.info:40341/ws priority: 2 + ape_chain_cache_size: 100000 apiclient: dial_timeout: 15s # timeout for FrostFS API client connection diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 15071f23d..02ead3020 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -12,19 +12,19 @@ There are some custom types used for brevity: # Structure -| Section | Description | -|--------------|---------------------------------------------------------| -| `logger` | [Logging parameters](#logger-section) | -| `pprof` | [PProf configuration](#pprof-section) | -| `prometheus` | [Prometheus metrics configuration](#prometheus-section) | -| `control` | [Control service configuration](#control-section) | -| `contracts` | [Override FrostFS contracts hashes](#contracts-section) | -| `morph` | [N3 blockchain client configuration](#morph-section) | -| `apiclient` | [FrostFS API client configuration](#apiclient-section) | -| `policer` | [Policer service configuration](#policer-section) | -| `replicator` | [Replicator service configuration](#replicator-section) | -| `storage` | [Storage engine configuration](#storage-section) | -| `runtime` | [Runtime configuration](#runtime-section) | +| Section | Description | +|------------------------|---------------------------------------------------------------------| +| `logger` | [Logging parameters](#logger-section) | +| `pprof` | [PProf configuration](#pprof-section) | +| `prometheus` | [Prometheus metrics configuration](#prometheus-section) | +| `control` | [Control service configuration](#control-section) | +| `contracts` | [Override FrostFS contracts hashes](#contracts-section) | +| `morph` | [N3 blockchain client configuration](#morph-section) | +| `apiclient` | [FrostFS API client configuration](#apiclient-section) | +| `policer` | [Policer service configuration](#policer-section) | +| `replicator` | [Replicator service configuration](#replicator-section) | +| `storage` | [Storage engine configuration](#storage-section) | +| `runtime` | [Runtime configuration](#runtime-section) | # `control` section @@ -139,6 +139,7 @@ contracts: morph: dial_timeout: 30s cache_ttl: 15s + ape_chain_cache_size: 10000 rpc_endpoint: - address: wss://rpc1.morph.frostfs.info:40341/ws priority: 1 @@ -147,12 +148,13 @@ morph: switch_interval: 2m ``` -| Parameter | Type | Default value | Description | -|-------------------|-----------------------------------------------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `dial_timeout` | `duration` | `5s` | Timeout for dialing connections to N3 RPCs. | -| `cache_ttl` | `duration` | Morph block time | Sidechain cache TTL value (min interval between similar calls).
Negative value disables caching.
Cached entities: containers, container lists, eACL tables. | -| `rpc_endpoint` | list of [endpoint descriptions](#rpc_endpoint-subsection) | | Array of endpoint descriptions. | -| `switch_interval` | `duration` | `2m` | Time interval between the attempts to connect to the highest priority RPC node if the connection is not established yet. | +| Parameter | Type | Default value | Description | +| ---------------------- | --------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `dial_timeout` | `duration` | `5s` | Timeout for dialing connections to N3 RPCs. | +| `cache_ttl` | `duration` | Morph block time | Sidechain cache TTL value (min interval between similar calls).
Negative value disables caching.
Cached entities: containers, container lists, eACL tables. | +| `rpc_endpoint` | list of [endpoint descriptions](#rpc_endpoint-subsection) | | Array of endpoint descriptions. | +| `switch_interval` | `duration` | `2m` | Time interval between the attempts to connect to the highest priority RPC node if the connection is not established yet. | +| `ape_chain_cache_size` | `int` | `10000` | Size of the morph cache for APE chains. | ## `rpc_endpoint` subsection | Parameter | Type | Default value | Description | From 5d58b44bc8895350d5aa70da6b71f0ade8847960 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 15 Mar 2024 10:14:17 +0300 Subject: [PATCH 0990/1943] [#1044] node: Drop unused methods from APE implementation Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/policy_engine.go | 40 ++++--------------------------- 2 files changed, 5 insertions(+), 37 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 424970b8f..a1f7bba40 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1076,7 +1076,7 @@ func initAccessPolicyEngine(_ context.Context, c *cfg) { ) } - var morphRuleStorage policy_engine.MorphRuleChainStorage + var morphRuleStorage policy_engine.MorphRuleChainStorageReader morphRuleStorage = policy_client.NewContractStorage( client.NewSwitchRPCGuardedActor(c.cfgMorph.client), c.cfgObject.cfgAccessPolicyEngine.policyContractHash) diff --git a/cmd/frostfs-node/policy_engine.go b/cmd/frostfs-node/policy_engine.go index 100520f50..13e30a17e 100644 --- a/cmd/frostfs-node/policy_engine.go +++ b/cmd/frostfs-node/policy_engine.go @@ -19,14 +19,10 @@ type accessPolicyEngine struct { chainRouter engine.ChainRouter - morphChainStorage engine.MorphRuleChainStorage - localOverrideDatabase chainbase.LocalOverrideDatabase } -var _ engine.LocalOverrideEngine = (*accessPolicyEngine)(nil) - -var _ engine.MorphRuleChainStorage = (*morphAPEChainCache)(nil) +var _ engine.MorphRuleChainStorageReader = (*morphAPEChainCache)(nil) type morphAPEChainCacheKey struct { name chain.Name @@ -34,22 +30,17 @@ type morphAPEChainCacheKey struct { } type morphAPEChainCache struct { - source engine.MorphRuleChainStorage + source engine.MorphRuleChainStorageReader cache *expirable.LRU[morphAPEChainCacheKey, []*chain.Chain] } -func newMorphCache(source engine.MorphRuleChainStorage, size int, ttl time.Duration) engine.MorphRuleChainStorage { +func newMorphCache(source engine.MorphRuleChainStorageReader, size int, ttl time.Duration) engine.MorphRuleChainStorageReader { return &morphAPEChainCache{ source: source, cache: expirable.NewLRU(size, func(morphAPEChainCacheKey, []*chain.Chain) {}, ttl), } } -func (m *morphAPEChainCache) AddMorphRuleChain(name chain.Name, target engine.Target, c *chain.Chain) (util.Uint256, uint32, error) { - m.cache.Remove(morphAPEChainCacheKey{name: name, target: target}) - return m.source.AddMorphRuleChain(name, target, c) -} - func (m *morphAPEChainCache) GetAdmin() (util.Uint160, error) { return m.source.GetAdmin() } @@ -70,26 +61,12 @@ func (m *morphAPEChainCache) ListMorphRuleChains(name chain.Name, target engine. return result, nil } -func (m *morphAPEChainCache) RemoveMorphRuleChain(name chain.Name, target engine.Target, chainID chain.ID) (util.Uint256, uint32, error) { - m.cache.Remove(morphAPEChainCacheKey{name: name, target: target}) - return m.source.RemoveMorphRuleChain(name, target, chainID) -} - -func (m *morphAPEChainCache) SetAdmin(addr util.Uint160) (util.Uint256, uint32, error) { - return m.source.SetAdmin(addr) -} - func (m *morphAPEChainCache) ListTargetsIterator(targetType engine.TargetType) (uuid.UUID, result.Iterator, error) { return m.source.ListTargetsIterator(targetType) } -func (m *morphAPEChainCache) RemoveMorphRuleChainsByTarget(name chain.Name, target engine.Target) (util.Uint256, uint32, error) { - m.cache.Remove(morphAPEChainCacheKey{name: name, target: target}) - return m.source.RemoveMorphRuleChainsByTarget(name, target) -} - func newAccessPolicyEngine( - morphChainStorage engine.MorphRuleChainStorage, + morphChainStorage engine.MorphRuleChainStorageReader, localOverrideDatabase chainbase.LocalOverrideDatabase, ) *accessPolicyEngine { return &accessPolicyEngine{ @@ -98,8 +75,6 @@ func newAccessPolicyEngine( localOverrideDatabase, ), - morphChainStorage: morphChainStorage, - localOverrideDatabase: localOverrideDatabase, } } @@ -111,13 +86,6 @@ func (a *accessPolicyEngine) IsAllowed(name chain.Name, target engine.RequestTar return a.chainRouter.IsAllowed(name, target, r) } -func (a *accessPolicyEngine) MorphRuleChainStorage() engine.MorphRuleChainStorage { - a.mtx.Lock() - defer a.mtx.Unlock() - - return a.morphChainStorage -} - func (a *accessPolicyEngine) LocalStorage() engine.LocalOverrideStorage { a.mtx.Lock() defer a.mtx.Unlock() From c09c70161395b5d34dc4ea128fc63877e5d1f773 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 19 Mar 2024 11:49:19 +0300 Subject: [PATCH 0991/1943] [#1048] metabase: Fix drop buckets during resync Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/control.go | 46 ++++++++------- .../metabase/reset_test.go | 57 +++++++++++++++++++ 2 files changed, 82 insertions(+), 21 deletions(-) create mode 100644 pkg/local_object_storage/metabase/reset_test.go diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index a275f4560..83b0b11b6 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -21,6 +21,22 @@ var ErrDegradedMode = logicerr.New("metabase is in a degraded mode") // ErrReadOnlyMode is returned when metabase is in a read-only mode. var ErrReadOnlyMode = logicerr.New("metabase is in a read-only mode") +var ( + mStaticBuckets = map[string]struct{}{ + string(containerVolumeBucketName): {}, + string(containerCounterBucketName): {}, + string(graveyardBucketName): {}, + string(garbageBucketName): {}, + string(shardInfoBucket): {}, + string(bucketNameLocked): {}, + } + + // deprecatedBuckets buckets that are not used anymore. + deprecatedBuckets = [][]byte{ + toMoveItBucketName, + } +) + // Open boltDB instance for metabase. func (db *DB) Open(_ context.Context, mode mode.Mode) error { db.modeMtx.Lock() @@ -113,20 +129,6 @@ func (db *DB) init(reset bool) error { return nil } - mStaticBuckets := map[string]struct{}{ - string(containerVolumeBucketName): {}, - string(containerCounterBucketName): {}, - string(graveyardBucketName): {}, - string(garbageBucketName): {}, - string(shardInfoBucket): {}, - string(bucketNameLocked): {}, - } - - // buckets that are not used anymore - deprecatedBuckets := [][]byte{ - toMoveItBucketName, - } - return db.boltDB.Update(func(tx *bbolt.Tx) error { var err error if !reset { @@ -167,15 +169,17 @@ func (db *DB) init(reset bool) error { return nil } - err = tx.ForEach(func(name []byte, _ *bbolt.Bucket) error { + bucketCursor := tx.Cursor() + name, _ := bucketCursor.First() + for name != nil { if _, ok := mStaticBuckets[string(name)]; !ok { - return tx.DeleteBucket(name) + if err := tx.DeleteBucket(name); err != nil { + return err + } + name, _ = bucketCursor.Seek(name) + continue } - - return nil - }) - if err != nil { - return err + name, _ = bucketCursor.Next() } return updateVersion(tx, version) }) diff --git a/pkg/local_object_storage/metabase/reset_test.go b/pkg/local_object_storage/metabase/reset_test.go new file mode 100644 index 000000000..66f5eefc6 --- /dev/null +++ b/pkg/local_object_storage/metabase/reset_test.go @@ -0,0 +1,57 @@ +package meta + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" +) + +type epochState struct{ e uint64 } + +func (s epochState) CurrentEpoch() uint64 { + return s.e +} + +func TestResetDropsContainerBuckets(t *testing.T) { + t.Parallel() + + db := New( + []Option{ + WithPath(filepath.Join(t.TempDir(), "metabase")), + WithPermissions(0o600), + WithEpochState(epochState{}), + }..., + ) + + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + require.NoError(t, db.Init()) + + defer func() { require.NoError(t, db.Close()) }() + + for idx := 0; idx < 100; idx++ { + var putPrm PutPrm + putPrm.SetObject(testutil.GenerateObject()) + putPrm.SetStorageID([]byte(fmt.Sprintf("0/%d", idx))) + _, err := db.Put(context.Background(), putPrm) + require.NoError(t, err) + } + + require.NoError(t, db.Reset()) + + var bucketCount int + require.NoError(t, db.boltDB.Update(func(tx *bbolt.Tx) error { + return tx.ForEach(func(name []byte, b *bbolt.Bucket) error { + _, exists := mStaticBuckets[string(name)] + require.True(t, exists, "unexpected bucket:"+string(name)) + bucketCount++ + return nil + }) + })) + require.Equal(t, len(mStaticBuckets), bucketCount) +} From c7a12ca3d8b5b238d8b7fef93470b1d620b5dad1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 21 Mar 2024 18:46:07 +0300 Subject: [PATCH 0992/1943] [#1054] network: Optimize IsTLSEnabled() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No big deal, but it is called multiple times in sorting routine, this easily results in 20 allocations per group traversal. ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ old │ new │ │ sec/op │ sec/op vs base │ AddressTLSEnabled-8 184.6n ± 1% 103.3n ± 6% -44.04% (p=0.000 n=10) │ old │ new │ │ B/op │ B/op vs base │ AddressTLSEnabled-8 704.0 ± 0% 0.0 ± 0% -100.00% (p=0.000 n=10) │ old │ new │ │ allocs/op │ allocs/op vs base │ AddressTLSEnabled-8 1.000 ± 0% 0.000 ± 0% -100.00% (p=0.000 n=10) ``` Signed-off-by: Evgenii Stratonikov --- pkg/network/tls.go | 9 ++------- pkg/network/tls_test.go | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pkg/network/tls.go b/pkg/network/tls.go index 9aac89c47..544dc8240 100644 --- a/pkg/network/tls.go +++ b/pkg/network/tls.go @@ -13,11 +13,6 @@ var tls, _ = multiaddr.NewMultiaddr("/" + tlsProtocolName) // IsTLSEnabled searches for wrapped TLS protocol in multiaddr. func (a Address) IsTLSEnabled() bool { - for _, protoc := range a.ma.Protocols() { - if protoc.Code == multiaddr.P_TLS { - return true - } - } - - return false + _, err := a.ma.ValueForProtocol(multiaddr.P_TLS) + return err == nil } diff --git a/pkg/network/tls_test.go b/pkg/network/tls_test.go index d93ea6a12..6c352484b 100644 --- a/pkg/network/tls_test.go +++ b/pkg/network/tls_test.go @@ -27,3 +27,18 @@ func TestAddress_TLSEnabled(t *testing.T) { require.Equal(t, test.wantTLS, addr.IsTLSEnabled(), test.input) } } + +func BenchmarkAddressTLSEnabled(b *testing.B) { + var addr Address + err := addr.FromString("/dns4/localhost/tcp/8080/tls") + require.NoError(b, err) + + b.ResetTimer() + b.ReportAllocs() + + var enabled bool + for i := 0; i < b.N; i++ { + enabled = addr.IsTLSEnabled() + } + require.True(b, enabled) +} From 8690db697c40926e3e77601fde5977caa4f03b2e Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 26 Mar 2024 14:07:25 +0300 Subject: [PATCH 0993/1943] [#1056] Makefile: fix target locode-download wget fails if .cache dir does not exist Signed-off-by: Ekaterina Lebedeva --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4d3c121bf..20c84d041 100755 --- a/Makefile +++ b/Makefile @@ -265,8 +265,9 @@ debclean: # Download locode database locode-download: - @wget -q -O ./.cache/locode_db.gz 'https://git.frostfs.info/TrueCloudLab/frostfs-locode-db/releases/download/${LOCODE_DB_VERSION}/locode_db.gz' - gzip -dfk ./.cache/locode_db.gz + mkdir -p $(TMP_DIR) + @wget -q -O ./$(TMP_DIR)/locode_db.gz 'https://git.frostfs.info/TrueCloudLab/frostfs-locode-db/releases/download/${LOCODE_DB_VERSION}/locode_db.gz' + gzip -dfk ./$(TMP_DIR)/locode_db.gz # Start dev environment env-up: all From 0990a9b0bd8592e8e49f750779197b9c77197095 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Thu, 28 Mar 2024 16:12:33 +0300 Subject: [PATCH 0994/1943] [#1055] blobstor: fix mode metric It used to always show CLOSED after setting shard mode to read-only regardless of actual mode. Now metric represents actual blobstor mode of operations. Signed-off-by: Ekaterina Lebedeva --- pkg/local_object_storage/blobstor/mode.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/local_object_storage/blobstor/mode.go b/pkg/local_object_storage/blobstor/mode.go index dc7dad687..a579a6f92 100644 --- a/pkg/local_object_storage/blobstor/mode.go +++ b/pkg/local_object_storage/blobstor/mode.go @@ -31,5 +31,6 @@ func (b *BlobStor) SetMode(m mode.Mode) error { } b.mode = m + b.metrics.SetMode(m.ReadOnly()) return nil } From fd8cdb96714b75c90f26a428cb96ed1e68204b63 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 27 Mar 2024 10:18:40 +0300 Subject: [PATCH 0995/1943] [#1057] netmap: Do not iterate over external addresses in Node Signed-off-by: Evgenii Stratonikov --- pkg/core/netmap/nodes.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/core/netmap/nodes.go b/pkg/core/netmap/nodes.go index 17fccc620..b0c9e1f9e 100644 --- a/pkg/core/netmap/nodes.go +++ b/pkg/core/netmap/nodes.go @@ -18,11 +18,6 @@ func (x Node) PublicKey() []byte { // and passes them into f. Handler MUST NOT be nil. func (x Node) IterateAddresses(f func(string) bool) { (netmap.NodeInfo)(x).IterateNetworkEndpoints(f) - for _, addr := range (netmap.NodeInfo)(x).ExternalAddresses() { - if f(addr) { - return - } - } } // NumberOfAddresses returns number of announced network addresses. From 942d83611bbebb503a462522c288883ef415ead1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 1 Apr 2024 09:54:42 +0300 Subject: [PATCH 0996/1943] [#874] engine: Revert Check object existance concurrently This reverts commit f526f49995575541626135e7d4542c0e014b63f2. Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/exists.go | 12 ---- pkg/local_object_storage/blobovnicza/get.go | 12 ---- .../blobstor/blobovniczatree/get.go | 5 -- .../blobstor/blobovniczatree/iterate.go | 18 ----- pkg/local_object_storage/blobstor/exists.go | 14 ++-- .../blobstor/fstree/fstree.go | 6 -- .../engine/engine_test.go | 6 -- pkg/local_object_storage/engine/exists.go | 72 ++++++++----------- pkg/local_object_storage/engine/put.go | 2 + pkg/local_object_storage/metabase/exists.go | 6 -- pkg/local_object_storage/shard/exists.go | 6 -- 11 files changed, 35 insertions(+), 124 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/exists.go b/pkg/local_object_storage/blobovnicza/exists.go index b5d723ee6..f7bc84d4a 100644 --- a/pkg/local_object_storage/blobovnicza/exists.go +++ b/pkg/local_object_storage/blobovnicza/exists.go @@ -21,22 +21,10 @@ func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error )) defer span.End() - select { - case <-ctx.Done(): - return false, ctx.Err() - default: - } - addrKey := addressKey(addr) err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if isNonDataBucket(bucketName) { return nil } diff --git a/pkg/local_object_storage/blobovnicza/get.go b/pkg/local_object_storage/blobovnicza/get.go index 36cf69d54..600323f55 100644 --- a/pkg/local_object_storage/blobovnicza/get.go +++ b/pkg/local_object_storage/blobovnicza/get.go @@ -51,12 +51,6 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { )) defer span.End() - select { - case <-ctx.Done(): - return GetRes{}, ctx.Err() - default: - } - var ( data []byte addrKey = addressKey(prm.addr) @@ -64,12 +58,6 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) { if err := b.boltDB.View(func(tx *bbolt.Tx) error { return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - if isNonDataBucket(bucketName) { return nil } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 5f1856673..08cacda8a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -94,11 +94,6 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G // // returns error if object could not be read from any blobovnicza of the same level. func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.GetPrm, blzPath string) (common.GetRes, error) { - select { - case <-ctx.Done(): - return common.GetRes{}, ctx.Err() - default: - } // open blobovnicza (cached inside) shBlz := b.getBlobovnicza(blzPath) blz, err := shBlz.Open() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 942d73a12..92014fd55 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -230,12 +230,6 @@ func (b *Blobovniczas) iterateSortedDBPaths(ctx context.Context, addr oid.Addres } func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path string, addr oid.Address, f func(string) (bool, error)) (bool, error) { - select { - case <-ctx.Done(): - return false, ctx.Err() - default: - } - sysPath := filepath.Join(b.rootPath, path) entries, err := os.ReadDir(sysPath) if os.IsNotExist(err) && b.readOnly && path == "" { // non initialized tree in read only mode @@ -258,12 +252,6 @@ func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path st if len(dbIdxs) > 0 { for _, dbIdx := range dbIdxs { - select { - case <-ctx.Done(): - return false, ctx.Err() - default: - } - dbPath := filepath.Join(path, u64ToHexStringExt(dbIdx)) stop, err := f(dbPath) if err != nil { @@ -278,12 +266,6 @@ func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path st if len(dirIdxs) > 0 { hrw.SortSliceByValue(dirIdxs, addressHash(&addr, path)) for _, dirIdx := range dirIdxs { - select { - case <-ctx.Done(): - return false, ctx.Err() - default: - } - dirPath := filepath.Join(path, u64ToHexString(dirIdx)) stop, err := b.iterateSordedDBPathsInternal(ctx, dirPath, addr, f) if err != nil { diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 21b4016d2..43feec7c9 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -3,7 +3,6 @@ package blobstor import ( "context" "encoding/hex" - "errors" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -58,30 +57,27 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi // error | found | log the error, return true, nil // error | not found | return the error // error | error | log the first error, return the second - var storageErrors []error + var errors []error for i := range b.storage { res, err := b.storage[i].Storage.Exists(ctx, prm) if err == nil && res.Exists { exists = true return res, nil } else if err != nil { - if errors.Is(err, context.Canceled) { - return common.ExistsRes{}, err - } - storageErrors = append(storageErrors, err) + errors = append(errors, err) } } - if len(storageErrors) == 0 { + if len(errors) == 0 { return common.ExistsRes{}, nil } - for _, err := range storageErrors[:len(storageErrors)-1] { + for _, err := range errors[:len(errors)-1] { b.log.Warn(logs.BlobstorErrorOccurredDuringObjectExistenceChecking, zap.Stringer("address", prm.Address), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } - return common.ExistsRes{}, storageErrors[len(storageErrors)-1] + return common.ExistsRes{}, errors[len(errors)-1] } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 76ce5bf17..420f341a4 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -285,12 +285,6 @@ func (t *FSTree) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exist )) defer span.End() - select { - case <-ctx.Done(): - return common.ExistsRes{}, ctx.Err() - default: - } - p := t.treePath(prm.Address) _, err := os.Stat(p) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index b6858df49..c41804e07 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -39,12 +39,6 @@ func BenchmarkExists(b *testing.B) { b.Run("8 shards", func(b *testing.B) { benchmarkExists(b, 8) }) - b.Run("12 shards", func(b *testing.B) { - benchmarkExists(b, 12) - }) - b.Run("16 shards", func(b *testing.B) { - benchmarkExists(b, 12) - }) } func benchmarkExists(b *testing.B, shardNum int) { diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index ee4dad341..ef6292768 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -3,70 +3,54 @@ package engine import ( "context" "errors" - "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "golang.org/x/sync/errgroup" ) func (e *StorageEngine) exists(ctx context.Context, addr oid.Address) (bool, error) { var shPrm shard.ExistsPrm shPrm.SetAddress(addr) - - var exists atomic.Bool - eg, egCtx := errgroup.WithContext(ctx) - egCtx, cancel := context.WithCancel(egCtx) - defer cancel() + alreadyRemoved := false + exists := false e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { - select { - case <-egCtx.Done(): - return true - default: + res, err := sh.Exists(ctx, shPrm) + if err != nil { + if client.IsErrObjectAlreadyRemoved(err) { + alreadyRemoved = true + + return true + } + + var siErr *objectSDK.SplitInfoError + if errors.As(err, &siErr) { + return true + } + + if shard.IsErrObjectExpired(err) { + return true + } + + if !client.IsErrObjectNotFound(err) { + e.reportShardError(sh, "could not check existence of object in shard", err) + } + return false } - eg.Go(func() error { - res, err := sh.Exists(egCtx, shPrm) - if err != nil { - if errors.Is(err, context.Canceled) { - return err - } + if !exists { + exists = res.Exists() + } - if client.IsErrObjectAlreadyRemoved(err) { - return err - } - - var siErr *objectSDK.SplitInfoError - if errors.As(err, &siErr) { - return err - } - - if shard.IsErrObjectExpired(err) { - return err - } - - if !client.IsErrObjectNotFound(err) { - e.reportShardError(sh, "could not check existence of object in shard", err) - } - return nil - } - if res.Exists() { - exists.Store(true) - cancel() - } - return nil - }) return false }) - err := eg.Wait() - if client.IsErrObjectAlreadyRemoved(err) { + if alreadyRemoved { return false, new(apistatus.ObjectAlreadyRemoved) } - return exists.Load(), nil + return exists, nil } diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index c1cd29cb8..777f728b7 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -78,6 +78,8 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { addr := object.AddressOf(prm.obj) + // In #1146 this check was parallelized, however, it became + // much slower on fast machines for 4 shards. _, err := e.exists(ctx, addr) if err != nil { return err diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index bca18e21f..aa9aba106 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -67,12 +67,6 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err return res, ErrDegradedMode } - select { - case <-ctx.Done(): - return res, ctx.Err() - default: - } - currEpoch := db.epochState.CurrentEpoch() err = db.boltDB.View(func(tx *bbolt.Tx) error { diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 7296426a6..2cdb8dfa8 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -47,12 +47,6 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { )) defer span.End() - select { - case <-ctx.Done(): - return ExistsRes{}, ctx.Err() - default: - } - var exists bool var err error From f23e38c28569716e2a770ca5f84df723449d597c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 1 Apr 2024 12:48:30 +0300 Subject: [PATCH 0997/1943] Revert "[#446] engine: Move to read-only on blobstor errors" This reverts commit 69df0d21c2d5f14a28d45951c32702ceec9da759. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/engine.go | 34 ++++++++----------- pkg/local_object_storage/engine/error_test.go | 2 +- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index e03a08abc..558fe92ed 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -50,7 +49,6 @@ type shardWrapper struct { type setModeRequest struct { sh *shard.Shard - isMeta bool errorCount uint32 } @@ -76,7 +74,7 @@ func (e *StorageEngine) setModeLoop() { if !ok { inProgress[sid] = struct{}{} go func() { - e.moveToDegraded(r.sh, r.errorCount, r.isMeta) + e.moveToDegraded(r.sh, r.errorCount) mtx.Lock() delete(inProgress, sid) @@ -88,7 +86,7 @@ func (e *StorageEngine) setModeLoop() { } } -func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) { +func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { sid := sh.ID() log := e.log.With( zap.Stringer("shard_id", sid), @@ -97,23 +95,21 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta e.mtx.RLock() defer e.mtx.RUnlock() - if isMeta { - err := sh.SetMode(mode.DegradedReadOnly) - if err == nil { - log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) - return - } + err := sh.SetMode(mode.DegradedReadOnly) + if err != nil { log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, zap.Error(err)) - } - err := sh.SetMode(mode.ReadOnly) - if err != nil { - log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) - return + err = sh.SetMode(mode.ReadOnly) + if err != nil { + log.Error(logs.EngineFailedToMoveShardInReadonlyMode, + zap.Error(err)) + } else { + log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) + } + } else { + log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) } - - log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } // reportShardErrorBackground increases shard error counter and logs an error. @@ -179,13 +175,11 @@ func (e *StorageEngine) reportShardErrorWithFlags( return } - isMeta := errors.As(err, new(metaerr.Error)) if block { - e.moveToDegraded(sh, errCount, isMeta) + e.moveToDegraded(sh, errCount) } else { req := setModeRequest{ errorCount: errCount, - isMeta: isMeta, sh: sh, } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 7f48da08a..ec4287bdd 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -153,7 +153,7 @@ func TestErrorReporting(t *testing.T) { for i := uint32(0); i < 2; i++ { _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) - checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly) + checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } From e12fcc041d421c4847b027db3ba0c7525610a321 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 28 Mar 2024 15:53:26 +0300 Subject: [PATCH 0998/1943] [#1059] services/tree: Fast sorted listing Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- pkg/local_object_storage/engine/tree.go | 33 +++ pkg/local_object_storage/pilorama/boltdb.go | 206 +++++++++++++++--- pkg/local_object_storage/pilorama/forest.go | 43 ++++ .../pilorama/forest_test.go | 202 +++++++++++++++++ pkg/local_object_storage/pilorama/heap.go | 64 ++++++ .../pilorama/interface.go | 3 + pkg/local_object_storage/shard/tree.go | 25 +++ pkg/services/tree/getsubtree_test.go | 2 +- pkg/services/tree/service.go | 87 ++++++++ 10 files changed, 638 insertions(+), 29 deletions(-) create mode 100644 pkg/local_object_storage/pilorama/heap.go diff --git a/go.mod b/go.mod index 2c2eac68f..1d18bb9d3 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 @@ -64,7 +65,6 @@ require ( github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index f50b9cb68..52ea399fe 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -209,6 +209,39 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree return nil, err } +// TreeSortedByFilename implements the pilorama.Forest interface. +func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node, last string, count int) ([]pilorama.NodeInfo, string, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeSortedByFilename", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("node_id", strconv.FormatUint(nodeID, 10)), + ), + ) + defer span.End() + + var err error + var nodes []pilorama.NodeInfo + var cursor string + for _, sh := range e.sortShards(cid) { + nodes, cursor, err = sh.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count) + if err != nil { + if err == shard.ErrPiloramaDisabled { + break + } + if !errors.Is(err, pilorama.ErrTreeNotFound) { + e.reportShardError(sh, "can't perform `TreeSortedByFilename`", err, + zap.Stringer("cid", cid), + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } + continue + } + return nodes, cursor, nil + } + return nil, last, err +} + // TreeGetOpLog implements the pilorama.Forest interface. func (e *StorageEngine) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (pilorama.Move, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetOpLog", diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 17133e8f3..1487296a4 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -9,6 +9,7 @@ import ( "math/rand" "os" "path/filepath" + "sort" "strconv" "sync" "time" @@ -986,6 +987,149 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri return m, parentID, metaerr.Wrap(err) } +func (t *boltForest) hasFewChildren(b *bbolt.Bucket, nodeID Node, threshold int) bool { + key := make([]byte, 9) + key[0] = 'c' + binary.LittleEndian.PutUint64(key[1:], nodeID) + + count := 0 + c := b.Cursor() + for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { + if count++; count > threshold { + return false + } + } + return true +} + +// TreeSortedByFilename implements the Forest interface. +func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node, last string, count int) ([]NodeInfo, string, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeSortedByFilename", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeSortedByFilename", + trace.WithAttributes( + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("node_id", strconv.FormatUint(nodeID, 10)), + ), + ) + defer span.End() + + t.modeMtx.RLock() + defer t.modeMtx.RUnlock() + + if t.mode.NoMetabase() { + return nil, "", ErrDegradedMode + } + + h := newHeap(last, count) + key := make([]byte, 9) + + var result []NodeInfo + var fewChildren bool + + err := t.db.View(func(tx *bbolt.Tx) error { + treeRoot := tx.Bucket(bucketName(cid, treeID)) + if treeRoot == nil { + return ErrTreeNotFound + } + + b := treeRoot.Bucket(dataBucket) + + // If the node is a leaf, we could scan all filenames in the tree. + // To prevent this we first count the number of children: if it is less than + // the number of nodes we need to return, fallback to TreeGetChildren() implementation. + if fewChildren = t.hasFewChildren(b, nodeID, count); fewChildren { + var err error + result, err = t.getChildren(b, nodeID) + return err + } + + t.fillSortedChildren(b, nodeID, h) + + for info, ok := h.pop(); ok; info, ok = h.pop() { + childInfo, err := t.getChildInfo(b, key, info.id) + if err != nil { + return err + } + result = append(result, childInfo) + } + return nil + }) + + success = err == nil + if err != nil { + return nil, last, metaerr.Wrap(err) + } + + if fewChildren { + sort.Slice(result, func(i, j int) bool { + return bytes.Compare(result[i].Meta.GetAttr(AttributeFilename), result[j].Meta.GetAttr(AttributeFilename)) == -1 + }) + for i := range result { + if bytes.Compare([]byte(last), result[i].Meta.GetAttr(AttributeFilename)) == -1 { + result = result[i:] + break + } + } + } + if len(result) != 0 { + last = string(result[len(result)-1].Meta.GetAttr(AttributeFilename)) + } + return result, last, metaerr.Wrap(err) +} + +func (t *boltForest) getChildInfo(b *bbolt.Bucket, key []byte, childID Node) (NodeInfo, error) { + childInfo := NodeInfo{ID: childID} + parentID, _, metaBytes, found := t.getState(b, stateKey(key, childID)) + if found { + childInfo.ParentID = parentID + if err := childInfo.Meta.FromBytes(metaBytes); err != nil { + return NodeInfo{}, err + } + } + return childInfo, nil +} + +func (t *boltForest) fillSortedChildren(b *bbolt.Bucket, nodeID Node, h *fixedHeap) { + c := b.Cursor() + prefix := internalKeyPrefix(nil, AttributeFilename) + + length := uint16(0) + count := 0 + for k, _ := c.Seek(prefix); len(k) > 0 && k[0] == 'i'; k, _ = c.Next() { + if len(k) < len(prefix)+2+16 { + continue + } + + parentID := binary.LittleEndian.Uint64(k[len(k)-16:]) + if parentID != nodeID { + continue + } + + actualLength := binary.LittleEndian.Uint16(k[len(prefix):]) + childID := binary.LittleEndian.Uint64(k[len(k)-8:]) + filename := string(k[len(prefix)+2 : len(k)-16]) + processed := h.push(childID, filename) + if actualLength != length { + length = actualLength + count = 1 + } else if processed { + if count++; count > h.count { + length = actualLength + 1 + c.Seek(append(prefix, byte(length), byte(length>>8))) + c.Prev() // c.Next() will be performed by for loop + } + } + } +} + // TreeGetChildren implements the Forest interface. func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) { var ( @@ -1012,10 +1156,6 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID return nil, ErrDegradedMode } - key := make([]byte, 9) - key[0] = 'c' - binary.LittleEndian.PutUint64(key[1:], nodeID) - var result []NodeInfo err := t.db.View(func(tx *bbolt.Tx) error { @@ -1025,27 +1165,34 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID } b := treeRoot.Bucket(dataBucket) - c := b.Cursor() - for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { - childID := binary.LittleEndian.Uint64(k[9:]) - childInfo := NodeInfo{ - ID: childID, - } - parentID, _, metaBytes, found := t.getState(b, stateKey(key, childID)) - if found { - childInfo.ParentID = parentID - if err := childInfo.Meta.FromBytes(metaBytes); err != nil { - return err - } - } - result = append(result, childInfo) - } - return nil + + var err error + result, err = t.getChildren(b, nodeID) + return err }) success = err == nil return result, metaerr.Wrap(err) } +func (t *boltForest) getChildren(b *bbolt.Bucket, nodeID Node) ([]NodeInfo, error) { + var result []NodeInfo + + key := make([]byte, 9) + key[0] = 'c' + binary.LittleEndian.PutUint64(key[1:], nodeID) + + c := b.Cursor() + for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { + childID := binary.LittleEndian.Uint64(k[9:]) + childInfo, err := t.getChildInfo(b, key, childID) + if err != nil { + return nil, err + } + result = append(result, childInfo) + } + return result, nil +} + // TreeList implements the Forest interface. func (t *boltForest) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) { var ( @@ -1358,6 +1505,16 @@ func childrenKey(key []byte, child, parent Node) []byte { return key[:childrenKeySize] } +func internalKeyPrefix(key []byte, k string) []byte { + key = key[:0] + key = append(key, 'i') + + l := len(k) + key = append(key, byte(l), byte(l>>8)) + key = append(key, k...) + return key +} + // 'i' + attribute name (string) + attribute value (string) + parent (id) + node (id) -> 0/1. func internalKey(key []byte, k, v string, parent, node Node) []byte { size := 1 /* prefix */ + 2*2 /* len */ + 2*8 /* nodes */ + len(k) + len(v) @@ -1365,14 +1522,9 @@ func internalKey(key []byte, k, v string, parent, node Node) []byte { key = make([]byte, 0, size) } - key = key[:0] - key = append(key, 'i') + key = internalKeyPrefix(key, k) - l := len(k) - key = append(key, byte(l), byte(l>>8)) - key = append(key, k...) - - l = len(v) + l := len(v) key = append(key, byte(l), byte(l>>8)) key = append(key, v...) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index dbfc9dccd..fedd3f90d 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -1,6 +1,7 @@ package pilorama import ( + "bytes" "context" "errors" "fmt" @@ -154,6 +155,48 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string, return s.infoMap[nodeID].Meta, s.infoMap[nodeID].Parent, nil } +// TreeSortedByFilename implements the Forest interface. +func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeID Node, start string, count int) ([]NodeInfo, string, error) { + fullID := cid.String() + "/" + treeID + s, ok := f.treeMap[fullID] + if !ok { + return nil, "", ErrTreeNotFound + } + if count == 0 { + return nil, start, nil + } + + children := s.tree.getChildren(nodeID) + res := make([]NodeInfo, 0, len(children)) + for _, childID := range children { + if len(s.infoMap[childID].Meta.GetAttr(AttributeFilename)) == 0 { + continue + } + res = append(res, NodeInfo{ + ID: childID, + Meta: s.infoMap[childID].Meta, + ParentID: s.infoMap[childID].Parent, + }) + } + if len(res) == 0 { + return res, "", nil + } + + sort.Slice(res, func(i, j int) bool { + return bytes.Compare(res[i].Meta.GetAttr(AttributeFilename), res[j].Meta.GetAttr(AttributeFilename)) == -1 + }) + for i := range res { + if string(res[i].Meta.GetAttr(AttributeFilename)) > start { + finish := i + count + if len(res) < finish { + finish = len(res) + } + return res[i:finish], string(res[finish-1].Meta.GetAttr(AttributeFilename)), nil + } + } + return nil, string(res[len(res)-1].Meta.GetAttr(AttributeFilename)), nil +} + // TreeGetChildren implements the Forest interface. func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID string, nodeID Node) ([]NodeInfo, error) { fullID := cid.String() + "/" + treeID diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 383320eb9..6c1bc343f 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -7,6 +7,7 @@ import ( mrand "math/rand" "path/filepath" "strconv" + "strings" "sync" "testing" "time" @@ -15,6 +16,8 @@ import ( cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "github.com/davecgh/go-spew/spew" + "github.com/google/uuid" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) @@ -178,6 +181,205 @@ func testForestTreeGetChildren(t *testing.T, s ForestStorage) { }) } +func BenchmarkForestSortedIteration(b *testing.B) { + for i := range providers { + if providers[i].name == "inmemory" { + continue + } + + cnr := cidtest.ID() + treeID := "version" + f := providers[i].construct(b) + + const total = 100_000 + d := CIDDescriptor{cnr, 0, 1} + for i := 0; i < total; i++ { + u, err := uuid.NewRandom() + if err != nil { + b.FailNow() + } + + _, err = f.TreeMove(context.Background(), d, treeID, &Move{ + Parent: RootID, + Child: RootID + Node(i+1), + Meta: Meta{ + Time: Timestamp(i + 1), + Items: []KeyValue{{ + Key: AttributeFilename, Value: []byte(u.String()), + }}, + }, + }) + if err != nil { + b.FailNow() + } + } + + b.Run(providers[i].name+",root", func(b *testing.B) { + for i := 0; i < b.N; i++ { + res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, RootID, "", 100) + if err != nil || len(res) != 100 { + b.Fatalf("err %v, count %d", err, len(res)) + } + } + }) + b.Run(providers[i].name+",leaf", func(b *testing.B) { + for i := 0; i < b.N; i++ { + res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, 1, "", 100) + if err != nil || len(res) != 0 { + b.FailNow() + } + } + }) + } +} + +func TestForest_TreeSortedIteration(t *testing.T) { + for i := range providers { + t.Run(providers[i].name, func(t *testing.T) { + testForestTreeSortedIteration(t, providers[i].construct(t)) + }) + } +} + +func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { + defer func() { require.NoError(t, s.Close()) }() + + cid := cidtest.ID() + d := CIDDescriptor{cid, 0, 1} + treeID := "version" + treeAdd := func(t *testing.T, ts int) { + _, err := s.TreeMove(context.Background(), d, treeID, &Move{ + Child: RootID + uint64(ts), + Parent: RootID, + Meta: Meta{ + Time: Timestamp(ts), + Items: []KeyValue{ + {Key: AttributeFilename, Value: []byte(strconv.Itoa(ts))}, + }, + }, + }) + require.NoError(t, err) + } + + const count = 9 + for i := 0; i < count; i++ { + treeAdd(t, i+1) + } + + var result []NodeInfo + treeAppend := func(t *testing.T, last string, count int) string { + res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, RootID, last, count) + require.NoError(t, err) + result = append(result, res...) + spew.Dump(last, res) + return cursor + } + + last := treeAppend(t, "", 2) + last = treeAppend(t, last, 3) + last = treeAppend(t, last, 0) + last = treeAppend(t, last, 1) + _ = treeAppend(t, last, 10) + + require.Len(t, result, count) + for i := range result { + require.Equal(t, RootID+uint64(i+1), result[i].ID) + require.Equal(t, strconv.Itoa(RootID+i+1), string(result[i].Meta.GetAttr(AttributeFilename))) + } +} + +func TestForest_TreeSortedFilename(t *testing.T) { + for i := range providers { + t.Run(providers[i].name, func(t *testing.T) { + testForestTreeSortedByFilename(t, providers[i].construct(t)) + }) + } +} + +func testForestTreeSortedByFilename(t *testing.T, s ForestStorage) { + defer func() { require.NoError(t, s.Close()) }() + + const controlAttr = "control_attr" + cid := cidtest.ID() + d := CIDDescriptor{cid, 0, 1} + treeID := "version" + + treeAddByPath := func(t *testing.T, filename string) { + path := strings.Split(filename, "/") + _, err := s.TreeAddByPath(context.Background(), d, treeID, AttributeFilename, path[:len(path)-1], + []KeyValue{ + {Key: AttributeFilename, Value: []byte(path[len(path)-1])}, + {Key: controlAttr, Value: []byte(filename)}, + }, + ) + require.NoError(t, err) + } + + expectAttributes := func(t *testing.T, attr string, expected []string, res []NodeInfo) { + require.Equal(t, len(expected), len(res)) + + actual := make([]string, len(res)) + for i := range actual { + actual[i] = string(res[i].Meta.GetAttr(attr)) + } + require.Equal(t, expected, actual) + } + + items := []string{ + "a/bbb/ccc", + "a/bbb/xxx", + "a/bbb/z", + "b/bbb/ccc", + "b/xxx/z", + "c", + } + + // Ensure we do not depend on insertion order in any way. + mrand.Shuffle(len(items), func(i, j int) { + items[i], items[j] = items[j], items[i] + }) + for i := range items { + treeAddByPath(t, items[i]) + } + + getChildren := func(t *testing.T, id Node) []NodeInfo { + res, _, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, id, "", len(items)) + require.NoError(t, err) + return res + } + + res := getChildren(t, RootID) + expectAttributes(t, AttributeFilename, []string{"a", "b", "c"}, res) + expectAttributes(t, controlAttr, []string{"", "", "c"}, res) + + { + ra := getChildren(t, res[0].ID) + expectAttributes(t, AttributeFilename, []string{"bbb"}, ra) + expectAttributes(t, controlAttr, []string{""}, ra) + + rabbb := getChildren(t, ra[0].ID) + expectAttributes(t, AttributeFilename, []string{"ccc", "xxx", "z"}, rabbb) + expectAttributes(t, controlAttr, []string{"a/bbb/ccc", "a/bbb/xxx", "a/bbb/z"}, rabbb) + } + { + rb := getChildren(t, res[1].ID) + expectAttributes(t, AttributeFilename, []string{"bbb", "xxx"}, rb) + expectAttributes(t, controlAttr, []string{"", ""}, rb) + + rbbbb := getChildren(t, rb[0].ID) + expectAttributes(t, AttributeFilename, []string{"ccc"}, rbbbb) + expectAttributes(t, controlAttr, []string{"b/bbb/ccc"}, rbbbb) + + rbxxx := getChildren(t, rb[1].ID) + expectAttributes(t, AttributeFilename, []string{"z"}, rbxxx) + expectAttributes(t, controlAttr, []string{"b/xxx/z"}, rbxxx) + } + { + rc := getChildren(t, res[2].ID) + require.Len(t, rc, 0) + } +} + func TestForest_TreeDrop(t *testing.T) { for i := range providers { t.Run(providers[i].name, func(t *testing.T) { diff --git a/pkg/local_object_storage/pilorama/heap.go b/pkg/local_object_storage/pilorama/heap.go new file mode 100644 index 000000000..667283c2a --- /dev/null +++ b/pkg/local_object_storage/pilorama/heap.go @@ -0,0 +1,64 @@ +package pilorama + +import ( + "container/heap" +) + +type heapInfo struct { + id Node + filename string +} + +type filenameHeap []heapInfo + +func (h filenameHeap) Len() int { return len(h) } +func (h filenameHeap) Less(i, j int) bool { return h[i].filename < h[j].filename } +func (h filenameHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *filenameHeap) Push(x any) { + *h = append(*h, x.(heapInfo)) +} +func (h *filenameHeap) Pop() any { + old := *h + n := len(old) + x := old[n-1] + *h = old[0 : n-1] + return x +} + +// fixedHeap maintains a fixed number of smallest elements started at some point. +type fixedHeap struct { + start string + max string + count int + h *filenameHeap +} + +func newHeap(start string, count int) *fixedHeap { + h := new(filenameHeap) + heap.Init(h) + + return &fixedHeap{ + start: start, + max: "", + count: count, + h: h, + } +} + +func (h *fixedHeap) push(id Node, filename string) bool { + if filename == "" || filename <= h.start { + return false + } + heap.Push(h.h, heapInfo{id: id, filename: filename}) + if h.h.Len() > h.count { + heap.Remove(h.h, h.h.Len()-1) + } + return true +} + +func (h *fixedHeap) pop() (heapInfo, bool) { + if h.h.Len() != 0 { + return heap.Pop(h.h).(heapInfo), true + } + return heapInfo{}, false +} diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 5fefb2e27..79449e080 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -33,6 +33,9 @@ type Forest interface { // TreeGetChildren returns children of the node with the specified ID. The order is arbitrary. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) + // TreeSortedByFilename returns children of the node with the specified ID. The nodes are sorted by the filename attribute.. + // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. + TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node, last string, count int) ([]NodeInfo, string, error) // TreeGetOpLog returns first log operation stored at or above the height. // In case no such operation is found, empty Move and nil error should be returned. TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index e78f29277..9a78f99ea 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -183,6 +183,31 @@ func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID strin return s.pilorama.TreeGetChildren(ctx, cid, treeID, nodeID) } +// TreeSortedByFilename implements the pilorama.Forest interface. +func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node, last string, count int) ([]pilorama.NodeInfo, string, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeSortedByFilename", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cid.EncodeToString()), + attribute.String("tree_id", treeID), + attribute.String("node_id", strconv.FormatUint(nodeID, 10)), + ), + ) + defer span.End() + + if s.pilorama == nil { + return nil, "", ErrPiloramaDisabled + } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return nil, "", ErrDegradedMode + } + return s.pilorama.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count) +} + // TreeGetOpLog implements the pilorama.Forest interface. func (s *Shard) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (pilorama.Move, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetOpLog", diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index 63cdc73a2..4f009adb6 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -174,7 +174,7 @@ func TestGetSubTreeOrderAsc(t *testing.T) { paths = append(paths, path.Join(tree[j].path...)) } } - require.True(t, found, "unknown node") + require.True(t, found, "unknown node %d %v", i, acc.seen[i].GetBody().GetNodeId()) } require.True(t, sort.SliceIsSorted(paths, func(i, j int) bool { diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 59b01796e..f715d9249 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -440,7 +440,94 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS return getSubTree(srv.Context(), srv, cid, b, s.forest) } +func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { + const batchSize = 1000 + + type stackItem struct { + values []pilorama.NodeInfo + parent pilorama.Node + last string + } + + // Traverse the tree in a DFS manner. Because we need to support arbitrary depth, + // recursive implementation is not suitable here, so we maintain explicit stack. + m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId()) + if err != nil { + return err + } + + err = srv.Send(&GetSubTreeResponse{ + Body: &GetSubTreeResponse_Body{ + NodeId: b.GetRootId(), + ParentId: p, + Timestamp: m.Time, + Meta: metaToProto(m.Items), + }, + }) + if err != nil { + return err + } + + stack := []stackItem{{ + values: nil, + parent: b.GetRootId(), + last: "", + }} + + for { + if len(stack) == 0 { + break + } else if item := &stack[len(stack)-1]; len(item.values) == 0 { + nodes, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), item.parent, item.last, batchSize) + if err != nil { + return err + } + item.values = nodes + item.last = last + + if len(nodes) == 0 { + stack = stack[:len(stack)-1] + continue + } + } + + node := stack[len(stack)-1].values[0] + stack[len(stack)-1].values = stack[len(stack)-1].values[1:] + + err = srv.Send(&GetSubTreeResponse{ + Body: &GetSubTreeResponse_Body{ + NodeId: node.ID, + ParentId: node.ParentID, + Timestamp: node.Meta.Time, + Meta: metaToProto(node.Meta.Items), + }, + }) + if err != nil { + return err + } + + if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() { + children, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), node.ID, "", batchSize) + if err != nil { + return err + } + if len(children) != 0 { + stack = append(stack, stackItem{ + values: children, + parent: node.ID, + last: last, + }) + } + } + } + return nil +} + func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { + if b.GetOrderBy().GetDirection() == GetSubTreeRequest_Body_Order_Asc { + return getSortedSubTree(ctx, srv, cid, b, forest) + } + // Traverse the tree in a DFS manner. Because we need to support arbitrary depth, // recursive implementation is not suitable here, so we maintain explicit stack. m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId()) From 81a0346a960e49f54aa6a032d8bbcbb23d49b0bd Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 19 Mar 2024 01:03:46 +0300 Subject: [PATCH 0999/1943] [#949] metabase: fix metabase mode metric It used to always show CLOSED regardless of actual mode. Now metric represents actual metabase mode of operations. Signed-off-by: Ekaterina Lebedeva --- pkg/local_object_storage/metabase/control.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 83b0b11b6..f6eb35ee8 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -42,6 +42,7 @@ func (db *DB) Open(_ context.Context, mode mode.Mode) error { db.modeMtx.Lock() defer db.modeMtx.Unlock() db.mode = mode + db.metrics.SetMode(mode) if mode.NoMetabase() { return nil From d5194ab2a6dacd533daf9bc601622c4ef19b77fa Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 19 Mar 2024 12:40:00 +0300 Subject: [PATCH 1000/1943] [#949] metabase: fix shard.UpdateID() metabase.Open() now reports metabase mode metric. shard.UpdateID() needs to read shard ID from metabase => needs to open metabase. It caused reporting 'shard undefined' metrics. To avoid reporting wrong metrics metabase.GetShardID() was added which also opens metabase and does not report metrics. Signed-off-by: Ekaterina Lebedeva --- pkg/local_object_storage/engine/shards.go | 4 +- pkg/local_object_storage/metabase/control.go | 9 ++- pkg/local_object_storage/metabase/shard_id.go | 63 +++++++++++++++---- .../metabase/version_test.go | 4 +- pkg/local_object_storage/shard/id.go | 27 +++----- 5 files changed, 69 insertions(+), 38 deletions(-) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 4bbf7eff0..ccf2e9dac 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -107,7 +107,7 @@ func (e *StorageEngine) AddShard(ctx context.Context, opts ...shard.Option) (*sh return sh.ID(), nil } -func (e *StorageEngine) createShard(ctx context.Context, opts []shard.Option) (*shard.Shard, error) { +func (e *StorageEngine) createShard(_ context.Context, opts []shard.Option) (*shard.Shard, error) { id, err := generateShardID() if err != nil { return nil, fmt.Errorf("could not generate shard ID: %w", err) @@ -126,7 +126,7 @@ func (e *StorageEngine) createShard(ctx context.Context, opts []shard.Option) (* shard.WithZeroCountCallback(e.processZeroCountContainers), )...) - if err := sh.UpdateID(ctx); err != nil { + if err := sh.UpdateID(); err != nil { e.log.Warn(logs.FailedToUpdateShardID, zap.Stringer("shard_id", sh.ID()), zap.String("metabase_path", sh.DumpInfo().MetaBaseInfo.Path), zap.Error(err)) } diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index f6eb35ee8..cd53f0cd2 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -202,11 +202,12 @@ func (db *DB) SyncCounters() error { })) } -// Close closes boltDB instance. +// Close closes boltDB instance +// and reports metabase metric. func (db *DB) Close() error { var err error if db.boltDB != nil { - err = metaerr.Wrap(db.boltDB.Close()) + err = db.close() } if err == nil { db.metrics.Close() @@ -214,6 +215,10 @@ func (db *DB) Close() error { return err } +func (db *DB) close() error { + return metaerr.Wrap(db.boltDB.Close()) +} + // Reload reloads part of the configuration. // It returns true iff database was reopened. // If a config option is invalid, it logs an error and returns nil. diff --git a/pkg/local_object_storage/metabase/shard_id.go b/pkg/local_object_storage/metabase/shard_id.go index c1ec8c67e..7ae336a6c 100644 --- a/pkg/local_object_storage/metabase/shard_id.go +++ b/pkg/local_object_storage/metabase/shard_id.go @@ -2,8 +2,11 @@ package meta import ( "bytes" + "errors" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + metamode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "go.etcd.io/bbolt" ) @@ -12,16 +15,32 @@ var ( shardIDKey = []byte("id") ) -// ReadShardID reads shard id from db. +// GetShardID sets metabase operation mode +// and reads shard id from db. // If id is missing, returns nil, nil. -func (db *DB) ReadShardID() ([]byte, error) { - db.modeMtx.RLock() - defer db.modeMtx.RUnlock() +// +// GetShardID does not report any metrics. +func (db *DB) GetShardID(mode metamode.Mode) ([]byte, error) { + db.modeMtx.Lock() + defer db.modeMtx.Unlock() + db.mode = mode - if db.mode.NoMetabase() { - return nil, ErrDegradedMode + if err := db.openDB(mode); err != nil { + return nil, fmt.Errorf("failed to open metabase: %w", err) } + id, err := db.readShardID() + + if cErr := db.close(); cErr != nil { + err = errors.Join(err, fmt.Errorf("failed to close metabase: %w", cErr)) + } + + return id, metaerr.Wrap(err) +} + +// ReadShardID reads shard id from db. +// If id is missing, returns nil, nil. +func (db *DB) readShardID() ([]byte, error) { var id []byte err := db.boltDB.View(func(tx *bbolt.Tx) error { b := tx.Bucket(shardInfoBucket) @@ -33,17 +52,35 @@ func (db *DB) ReadShardID() ([]byte, error) { return id, metaerr.Wrap(err) } -// WriteShardID writes shard it to db. -func (db *DB) WriteShardID(id []byte) error { - db.modeMtx.RLock() - defer db.modeMtx.RUnlock() +// SetShardID sets metabase operation mode +// and writes shard id to db. +func (db *DB) SetShardID(id []byte, mode metamode.Mode) error { + db.modeMtx.Lock() + defer db.modeMtx.Unlock() + db.mode = mode - if db.mode.NoMetabase() { - return ErrDegradedMode - } else if db.mode.ReadOnly() { + if mode.ReadOnly() { return ErrReadOnlyMode } + if err := db.openDB(mode); err != nil { + return fmt.Errorf("failed to open metabase: %w", err) + } + + err := db.writeShardID(id) + if err == nil { + db.metrics.SetMode(mode) + } + + if cErr := db.close(); cErr != nil { + err = errors.Join(err, fmt.Errorf("failed to close metabase: %w", cErr)) + } + + return metaerr.Wrap(err) +} + +// writeShardID writes shard id to db. +func (db *DB) writeShardID(id []byte) error { return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { b, err := tx.CreateBucketIfNotExists(shardInfoBucket) if err != nil { diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index cea42da0f..b2af428ff 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -58,9 +58,7 @@ func TestVersion(t *testing.T) { }) t.Run("old data", func(t *testing.T) { db := newDB(t) - require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, db.WriteShardID([]byte{1, 2, 3, 4})) - require.NoError(t, db.Close()) + require.NoError(t, db.SetShardID([]byte{1, 2, 3, 4}, mode.ReadWrite)) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Init()) diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index be474e0fa..2fe68d270 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -1,9 +1,10 @@ package shard import ( - "context" + "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/mr-tron/base58" "go.uber.org/zap" @@ -30,21 +31,11 @@ func (s *Shard) ID() *ID { } // UpdateID reads shard ID saved in the metabase and updates it if it is missing. -func (s *Shard) UpdateID(ctx context.Context) (err error) { +func (s *Shard) UpdateID() (err error) { var idFromMetabase []byte - metabaseOpened := !s.GetMode().NoMetabase() - if err = s.metaBase.Open(ctx, s.GetMode()); err != nil { - err = fmt.Errorf("failed to open metabase: %w", err) - metabaseOpened = false - } - if metabaseOpened { - defer func() { - cErr := s.metaBase.Close() - if cErr != nil { - err = fmt.Errorf("failed to close metabase: %w", cErr) - } - }() - if idFromMetabase, err = s.metaBase.ReadShardID(); err != nil { + modeDegraded := s.GetMode().NoMetabase() + if !modeDegraded { + if idFromMetabase, err = s.metaBase.GetShardID(mode.ReadOnly); err != nil { err = fmt.Errorf("failed to read shard id from metabase: %w", err) } } @@ -73,9 +64,9 @@ func (s *Shard) UpdateID(ctx context.Context) (err error) { s.pilorama.SetParentID(s.info.ID.String()) } - if len(idFromMetabase) == 0 && metabaseOpened { - if err = s.metaBase.WriteShardID(*s.info.ID); err != nil { - err = fmt.Errorf("failed to write shard id to metabase: %w", err) + if len(idFromMetabase) == 0 && !modeDegraded { + if setErr := s.metaBase.SetShardID(*s.info.ID, s.GetMode()); setErr != nil { + err = errors.Join(err, fmt.Errorf("failed to write shard id to metabase: %w", setErr)) } } return From 4080b99310df4170deadfa3160033304ddd9d442 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 2 Apr 2024 12:40:56 +0300 Subject: [PATCH 1001/1943] [#1061] node: Set TTL for morph rule cache from morph config Signed-off-by: Airat Arifullin --- cmd/frostfs-node/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index a1f7bba40..45ef771f3 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1083,7 +1083,7 @@ func initAccessPolicyEngine(_ context.Context, c *cfg) { cacheSize := morphconfig.APEChainCacheSize(c.appCfg) if cacheSize > 0 { - morphRuleStorage = newMorphCache(morphRuleStorage, int(cacheSize), morphconfig.CacheTTL(c.appCfg)) + morphRuleStorage = newMorphCache(morphRuleStorage, int(cacheSize), c.cfgMorph.cacheTTL) } ape := newAccessPolicyEngine(morphRuleStorage, localOverrideDB) From 17af91619a297354e901aaa1d8cd5be0680a9da5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 2 Apr 2024 15:02:14 +0300 Subject: [PATCH 1002/1943] [#1070] pilorama: Fix cycling behaviour for sorted listing In case there are no items left, return empty slice. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/boltdb.go | 22 ++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 1487296a4..29a9306b6 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1069,15 +1069,7 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr } if fewChildren { - sort.Slice(result, func(i, j int) bool { - return bytes.Compare(result[i].Meta.GetAttr(AttributeFilename), result[j].Meta.GetAttr(AttributeFilename)) == -1 - }) - for i := range result { - if bytes.Compare([]byte(last), result[i].Meta.GetAttr(AttributeFilename)) == -1 { - result = result[i:] - break - } - } + result = sortAndCut(result, []byte(last)) } if len(result) != 0 { last = string(result[len(result)-1].Meta.GetAttr(AttributeFilename)) @@ -1085,6 +1077,18 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr return result, last, metaerr.Wrap(err) } +func sortAndCut(result []NodeInfo, last []byte) []NodeInfo { + sort.Slice(result, func(i, j int) bool { + return bytes.Compare(result[i].Meta.GetAttr(AttributeFilename), result[j].Meta.GetAttr(AttributeFilename)) == -1 + }) + for i := range result { + if bytes.Compare(last, result[i].Meta.GetAttr(AttributeFilename)) == -1 { + return result[i:] + } + } + return nil +} + func (t *boltForest) getChildInfo(b *bbolt.Bucket, key []byte, childID Node) (NodeInfo, error) { childInfo := NodeInfo{ID: childID} parentID, _, metaBytes, found := t.getState(b, stateKey(key, childID)) From ff4c23f59af4f9d59ac682f99b10abf5efc4914c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 2 Apr 2024 10:41:38 +0300 Subject: [PATCH 1003/1943] [#1070] services/tree: Fix fast listing depth processing For unsorted `GetSubTree()` we return a single node for depth=1. Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/getsubtree_test.go | 96 ++++++++++++++++++++-------- pkg/services/tree/service.go | 25 +++----- 2 files changed, 80 insertions(+), 41 deletions(-) diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index 4f009adb6..4bf679984 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "path" + "path/filepath" "sort" "testing" @@ -123,9 +124,21 @@ func TestGetSubTree(t *testing.T) { } func TestGetSubTreeOrderAsc(t *testing.T) { + t.Run("memory forest", func(t *testing.T) { + testGetSubTreeOrderAsc(t, pilorama.NewMemoryForest()) + }) + + t.Run("boltdb forest", func(t *testing.T) { + p := pilorama.NewBoltForest(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))) + require.NoError(t, p.Open(context.Background(), 0644)) + require.NoError(t, p.Init()) + testGetSubTreeOrderAsc(t, p) + }) +} + +func testGetSubTreeOrderAsc(t *testing.T, p pilorama.ForestStorage) { d := pilorama.CIDDescriptor{CID: cidtest.ID(), Size: 1} treeID := "sometree" - p := pilorama.NewMemoryForest() tree := []struct { path []string @@ -151,35 +164,66 @@ func TestGetSubTreeOrderAsc(t *testing.T) { tree[i].id = lm[0].Child } - acc := subTreeAcc{errIndex: -1} - err := getSubTree(context.Background(), &acc, d.CID, &GetSubTreeRequest_Body{ - TreeId: treeID, - OrderBy: &GetSubTreeRequest_Body_Order{ - Direction: GetSubTreeRequest_Body_Order_Asc, - }, - }, p) - require.NoError(t, err) - // GetSubTree must return child only after is has returned the parent. - require.Equal(t, uint64(0), acc.seen[0].Body.NodeId) + t.Run("total", func(t *testing.T) { + t.Skip() + acc := subTreeAcc{errIndex: -1} + err := getSubTree(context.Background(), &acc, d.CID, &GetSubTreeRequest_Body{ + TreeId: treeID, + OrderBy: &GetSubTreeRequest_Body_Order{ + Direction: GetSubTreeRequest_Body_Order_Asc, + }, + }, p) + require.NoError(t, err) + // GetSubTree must return child only after is has returned the parent. + require.Equal(t, uint64(0), acc.seen[0].Body.NodeId) - paths := make([]string, 0, len(acc.seen)) - for i := range acc.seen { - if i == 0 { - continue - } - found := false - for j := range tree { - if acc.seen[i].Body.NodeId == tree[j].id { - found = true - paths = append(paths, path.Join(tree[j].path...)) + paths := make([]string, 0, len(acc.seen)) + for i := range acc.seen { + if i == 0 { + continue } + found := false + for j := range tree { + if acc.seen[i].Body.NodeId == tree[j].id { + found = true + paths = append(paths, path.Join(tree[j].path...)) + } + } + require.True(t, found, "unknown node %d %v", i, acc.seen[i].GetBody().GetNodeId()) } - require.True(t, found, "unknown node %d %v", i, acc.seen[i].GetBody().GetNodeId()) - } - require.True(t, sort.SliceIsSorted(paths, func(i, j int) bool { - return paths[i] < paths[j] - })) + require.True(t, sort.SliceIsSorted(paths, func(i, j int) bool { + return paths[i] < paths[j] + })) + }) + t.Run("depth=1", func(t *testing.T) { + acc := subTreeAcc{errIndex: -1} + err := getSubTree(context.Background(), &acc, d.CID, &GetSubTreeRequest_Body{ + TreeId: treeID, + Depth: 1, + OrderBy: &GetSubTreeRequest_Body_Order{ + Direction: GetSubTreeRequest_Body_Order_Asc, + }, + }, p) + require.NoError(t, err) + require.Len(t, acc.seen, 1) + require.Equal(t, uint64(0), acc.seen[0].Body.NodeId) + }) + t.Run("depth=2", func(t *testing.T) { + acc := subTreeAcc{errIndex: -1} + err := getSubTree(context.Background(), &acc, d.CID, &GetSubTreeRequest_Body{ + TreeId: treeID, + Depth: 2, + OrderBy: &GetSubTreeRequest_Body_Order{ + Direction: GetSubTreeRequest_Body_Order_Asc, + }, + }, p) + require.NoError(t, err) + require.Len(t, acc.seen, 3) + require.Equal(t, uint64(0), acc.seen[0].Body.NodeId) + require.Equal(t, uint64(0), acc.seen[1].GetBody().GetParentId()) + require.Equal(t, uint64(0), acc.seen[2].GetBody().GetParentId()) + }) } var ( diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index f715d9249..903db4455 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -456,28 +456,23 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid return err } - err = srv.Send(&GetSubTreeResponse{ - Body: &GetSubTreeResponse_Body{ - NodeId: b.GetRootId(), - ParentId: p, - Timestamp: m.Time, - Meta: metaToProto(m.Items), - }, - }) - if err != nil { - return err - } - stack := []stackItem{{ - values: nil, - parent: b.GetRootId(), - last: "", + values: []pilorama.NodeInfo{{ + ID: b.GetRootId(), + Meta: m, + ParentID: p, + }}, + parent: p, }} for { if len(stack) == 0 { break } else if item := &stack[len(stack)-1]; len(item.values) == 0 { + if len(stack) == 1 { + break + } + nodes, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), item.parent, item.last, batchSize) if err != nil { return err From 4738508ce231fab8b10e3151056474a087c6a382 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Fri, 29 Mar 2024 15:31:37 +0300 Subject: [PATCH 1004/1943] [#1063] go.mod: Update SDK version * Update frostfs-sdk and frostfs-api-go versions. * Refactor depreacted method ReplicaNumberByIndex. Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/container/create.go | 4 ++-- cmd/frostfs-cli/modules/container/nodes.go | 2 +- cmd/frostfs-cli/modules/object/nodes.go | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/services/object_manager/placement/traverser.go | 2 +- pkg/services/policer/check.go | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 728332b35..6d0ed750e 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -58,12 +58,12 @@ It will be stored in sidechain when inner ring will accepts it.`, "use --force option to skip this check: %w", err) for i, nodes := range nodesByRep { - if placementPolicy.ReplicaNumberByIndex(i) > uint32(len(nodes)) { + if placementPolicy.ReplicaDescriptor(i).NumberOfObjects() > uint32(len(nodes)) { commonCmd.ExitOnErr(cmd, "", fmt.Errorf( "the number of nodes '%d' in selector is not enough for the number of replicas '%d', "+ "use --force option to skip this check", len(nodes), - placementPolicy.ReplicaNumberByIndex(i), + placementPolicy.ReplicaDescriptor(i).NumberOfObjects(), )) } } diff --git a/cmd/frostfs-cli/modules/container/nodes.go b/cmd/frostfs-cli/modules/container/nodes.go index d785fa23d..b8765061c 100644 --- a/cmd/frostfs-cli/modules/container/nodes.go +++ b/cmd/frostfs-cli/modules/container/nodes.go @@ -46,7 +46,7 @@ var containerNodesCmd = &cobra.Command{ commonCmd.ExitOnErr(cmd, "could not build container nodes for given container: %w", err) for i := range cnrNodes { - cmd.Printf("Descriptor #%d, REP %d:\n", i+1, policy.ReplicaNumberByIndex(i)) + cmd.Printf("Descriptor #%d, REP %d:\n", i+1, policy.ReplicaDescriptor(i).NumberOfObjects()) for j := range cnrNodes[i] { commonCmd.PrettyPrintNodeInfo(cmd, cnrNodes[i][j], j, "\t", short) } diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 50c47e822..38b9dec82 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -190,7 +190,7 @@ func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placemen placement, err := placementBuilder.BuildPlacement(objInfo.containerID, &objInfo.objectID, placementPolicy) commonCmd.ExitOnErr(cmd, "failed to get required placement: %w", err) for repIdx, rep := range placement { - numOfReplicas := placementPolicy.ReplicaNumberByIndex(repIdx) + numOfReplicas := placementPolicy.ReplicaDescriptor(repIdx).NumberOfObjects() var nodeIdx uint32 for _, n := range rep { if !objInfo.isLockOrTombstone && nodeIdx == numOfReplicas { // lock and tombstone objects should be on all container nodes diff --git a/go.mod b/go.mod index 1d18bb9d3..3159b9c0a 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.20 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240215124401-634e24aba715 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240327095603-491a47e7fe24 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240307151106-2ec958cbfdfd git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index d90bf0f6a..e4bd28600 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,15 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240215124401-634e24aba715 h1:EDtL9OJcdeevV/jmNMtHugulAQprdOnknNPvLB3LRgE= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240215124401-634e24aba715/go.mod h1:uY0AYmCznjZdghDnAk7THFIe1Vlg531IxUcus7ZfUJI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240327095603-491a47e7fe24 h1:uIkl0mKWwDICUZTbNWZ38HLYDBI9rMgdAhYQWZ0C9iQ= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240327095603-491a47e7fe24/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0 h1:FzurjElUwC7InY9v5rzXReKbfBL5yRJKSWJPq6BKhH0= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b h1:nLIWYXe4e1fWgpKeMfVke/CNBn388egh4fArFdvhfHw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240301150205-6fe4e2541d0b/go.mod h1:XcgrbZ88XfvhAMxmZCQJ0dv6FyRSq6Mg2J7nN8uuO0k= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92 h1:hSyM52d8yIaOpYQlLlVYdrGbgCsvIDjwl3AJaJUlYPU= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92/go.mod h1:i0RKqiF4z3UOxLSNwhHw+cUz/JyYWuTRpnn9ere4Y3w= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240307151106-2ec958cbfdfd h1:pyIl9f4nIr7ekJ73W9keLIQ5dpoKb8o6xNmodsXY5+o= diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 53da186e8..a699b4454 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -137,7 +137,7 @@ func defaultCopiesVector(policy netmap.PlacementPolicy) []int { copyVector := make([]int, 0, replNum) for i := 0; i < replNum; i++ { - copyVector = append(copyVector, int(policy.ReplicaNumberByIndex(i))) + copyVector = append(copyVector, int(policy.ReplicaDescriptor(i).NumberOfObjects())) } return copyVector diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index bb6313d07..d2297fed7 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -63,7 +63,7 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add default: } - p.processNodes(ctx, c, addrWithType, nn[i], policy.ReplicaNumberByIndex(i), checkedNodes) + p.processNodes(ctx, c, addrWithType, nn[i], policy.ReplicaDescriptor(i).NumberOfObjects(), checkedNodes) } if !c.needLocalCopy && c.removeLocalCopy { From 531542ce60a90a01ae04b63c54a92fede1b881c3 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Fri, 29 Mar 2024 15:59:50 +0300 Subject: [PATCH 1005/1943] [#1063] cli: Validate container creation for EC policy Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/container/create.go | 28 +++++++++++++++------ cmd/frostfs-cli/modules/container/nodes.go | 10 +++++++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 6d0ed750e..c6f576908 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -58,13 +58,27 @@ It will be stored in sidechain when inner ring will accepts it.`, "use --force option to skip this check: %w", err) for i, nodes := range nodesByRep { - if placementPolicy.ReplicaDescriptor(i).NumberOfObjects() > uint32(len(nodes)) { - commonCmd.ExitOnErr(cmd, "", fmt.Errorf( - "the number of nodes '%d' in selector is not enough for the number of replicas '%d', "+ - "use --force option to skip this check", - len(nodes), - placementPolicy.ReplicaDescriptor(i).NumberOfObjects(), - )) + if repNum := placementPolicy.ReplicaDescriptor(i).NumberOfObjects(); repNum > 0 { + if repNum > uint32(len(nodes)) { + commonCmd.ExitOnErr(cmd, "", fmt.Errorf( + "the number of nodes '%d' in selector is not enough for the number of replicas '%d', "+ + "use --force option to skip this check", + len(nodes), + repNum, + )) + } + } else if ecParts := placementPolicy.ReplicaDescriptor(i).TotalECPartCount(); ecParts > 0 { + if ecParts > uint32(len(nodes)) { + commonCmd.ExitOnErr(cmd, "", fmt.Errorf( + "the number of nodes '%d' in selector is not enough for EC placement '%d.%d', "+ + "use --force option to skip this check", + len(nodes), + placementPolicy.ReplicaDescriptor(i).GetECDataCount(), + placementPolicy.ReplicaDescriptor(i).GetECParityCount(), + )) + } + } else { + commonCmd.ExitOnErr(cmd, "%w", errors.New("no replication policy is set")) } } } diff --git a/cmd/frostfs-cli/modules/container/nodes.go b/cmd/frostfs-cli/modules/container/nodes.go index b8765061c..1ae8ab604 100644 --- a/cmd/frostfs-cli/modules/container/nodes.go +++ b/cmd/frostfs-cli/modules/container/nodes.go @@ -2,6 +2,7 @@ package container import ( "crypto/sha256" + "errors" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -46,7 +47,14 @@ var containerNodesCmd = &cobra.Command{ commonCmd.ExitOnErr(cmd, "could not build container nodes for given container: %w", err) for i := range cnrNodes { - cmd.Printf("Descriptor #%d, REP %d:\n", i+1, policy.ReplicaDescriptor(i).NumberOfObjects()) + if repNum := policy.ReplicaDescriptor(i).NumberOfObjects(); repNum > 0 { + cmd.Printf("Descriptor #%d, REP %d:\n", i+1, repNum) + } else if ecParts := policy.ReplicaDescriptor(i).TotalECPartCount(); ecParts > 0 { + cmd.Printf("Descriptor #%d, EC %d.%d:\n", i+1, policy.ReplicaDescriptor(i).GetECDataCount(), + policy.ReplicaDescriptor(i).GetECParityCount()) + } else { + commonCmd.ExitOnErr(cmd, "%w", errors.New("no replication policy is set")) + } for j := range cnrNodes[i] { commonCmd.PrettyPrintNodeInfo(cmd, cnrNodes[i][j], j, "\t", short) } From d614f04a0a29504d82073fc4d69b50b029a7315c Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 3 Apr 2024 11:44:02 +0300 Subject: [PATCH 1006/1943] [#1072] Fix gofumpt issues Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/pilorama/heap.go | 1 + pkg/services/tree/getsubtree_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/pilorama/heap.go b/pkg/local_object_storage/pilorama/heap.go index 667283c2a..50abb663c 100644 --- a/pkg/local_object_storage/pilorama/heap.go +++ b/pkg/local_object_storage/pilorama/heap.go @@ -17,6 +17,7 @@ func (h filenameHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } func (h *filenameHeap) Push(x any) { *h = append(*h, x.(heapInfo)) } + func (h *filenameHeap) Pop() any { old := *h n := len(old) diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index 4bf679984..9a4223e30 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -130,7 +130,7 @@ func TestGetSubTreeOrderAsc(t *testing.T) { t.Run("boltdb forest", func(t *testing.T) { p := pilorama.NewBoltForest(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))) - require.NoError(t, p.Open(context.Background(), 0644)) + require.NoError(t, p.Open(context.Background(), 0o644)) require.NoError(t, p.Init()) testGetSubTreeOrderAsc(t, p) }) From 9aa533e59a700455fc5aaf47701729767f12241b Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 4 Apr 2024 11:05:25 +0300 Subject: [PATCH 1007/1943] [#1072] node, ir: Add new config option `kludge_compatibility_mode` Signed-off-by: Anton Nikiforov --- cmd/frostfs-ir/config.go | 1 + cmd/frostfs-ir/defaults.go | 2 ++ cmd/frostfs-ir/main.go | 6 +++++- cmd/frostfs-node/config.go | 12 +++++++++++- cmd/frostfs-node/config/node/config.go | 5 +++++ pkg/innerring/innerring.go | 5 ++++- 6 files changed, 28 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index be870052c..c73b68eb5 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -34,6 +34,7 @@ func reloadConfig() error { if err != nil { return err } + cmode.Store(cfg.GetBool("node.kludge_compatibility_mode")) err = logPrm.SetLevelString(cfg.GetString("logger.level")) if err != nil { return err diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index 127a68b29..23a475591 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -43,6 +43,8 @@ func defaultConfiguration(cfg *viper.Viper) { setControlDefaults(cfg) cfg.SetDefault("governance.disable", false) + + cfg.SetDefault("node.kludge_compatibility_mode", false) } func setControlDefaults(cfg *viper.Viper) { diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 9879342b7..31390dd74 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "sync" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" @@ -37,6 +38,7 @@ var ( cfg *viper.Viper configFile *string configDir *string + cmode = &atomic.Bool{} ) func exitErr(err error) { @@ -62,6 +64,8 @@ func main() { cfg, err = newConfig() exitErr(err) + cmode.Store(cfg.GetBool("node.kludge_compatibility_mode")) + metrics := irMetrics.NewInnerRingMetrics() err = logPrm.SetLevelString( @@ -84,7 +88,7 @@ func main() { metricsCmp = newMetricsComponent() metricsCmp.init() - innerRing, err = innerring.New(ctx, log, cfg, intErr, metrics) + innerRing, err = innerring.New(ctx, log, cfg, intErr, metrics, cmode) exitErr(err) pprofCmp.start() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 45ef771f3..d78a90cfc 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -109,6 +109,9 @@ type applicationConfiguration struct { lowMem bool rebuildWorkers uint32 } + + // if need to run node in compatibility with other versions mode + cmode *atomic.Bool } type shardCfg struct { @@ -204,10 +207,13 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { } // clear if it is rereading + cmode := a.cmode *a = applicationConfiguration{} + a.cmode = cmode } a._read = true + a.cmode.Store(nodeconfig.CompatibilityMode(c)) // Logger @@ -648,7 +654,11 @@ type cfgControlService struct { var persistateSideChainLastBlockKey = []byte("side_chain_last_processed_block") func initCfg(appCfg *config.Config) *cfg { - c := &cfg{} + c := &cfg{ + applicationConfiguration: applicationConfiguration{ + cmode: &atomic.Bool{}, + }, + } err := c.readConfig(appCfg) if err != nil { diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index ac76ad47e..90338556e 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -292,3 +292,8 @@ func (l PersistentPolicyRulesConfig) Perm() fs.FileMode { func (l PersistentPolicyRulesConfig) NoSync() bool { return config.BoolSafe((*config.Config)(l.cfg), "no_sync") } + +// CompatibilityMode returns true if need to run node in compatibility with previous versions mode. +func CompatibilityMode(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "kludge_compatibility_mode") +} diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 5d7dc5a52..11d43f845 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -103,6 +103,8 @@ type ( // should report start errors // to the application. runners []func(chan<- error) error + + cmode *atomic.Bool } chainParams struct { @@ -330,12 +332,13 @@ func (s *Server) registerStarter(f func() error) { // New creates instance of inner ring sever structure. func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error, - metrics *metrics.InnerRingServiceMetrics, + metrics *metrics.InnerRingServiceMetrics, cmode *atomic.Bool, ) (*Server, error) { var err error server := &Server{ log: log, irMetrics: metrics, + cmode: cmode, } server.sdNotify, err = server.initSdNotify(cfg) From ffb1a6f81a4ea8f249af58de66b2543ed6366a16 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 4 Apr 2024 11:36:38 +0300 Subject: [PATCH 1008/1943] [#1072] Fix issue from `govulncheck` Signed-off-by: Anton Nikiforov --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 3159b9c0a..2544f0fb9 100644 --- a/go.mod +++ b/go.mod @@ -40,8 +40,8 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/sync v0.6.0 - golang.org/x/sys v0.16.0 - golang.org/x/term v0.16.0 + golang.org/x/sys v0.18.0 + golang.org/x/term v0.18.0 google.golang.org/grpc v1.61.0 google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v3 v3.0.1 @@ -121,8 +121,8 @@ require ( go.opentelemetry.io/otel/sdk v1.22.0 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.18.0 // indirect - golang.org/x/net v0.20.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect diff --git a/go.sum b/go.sum index e4bd28600..8dbaa5373 100644 --- a/go.sum +++ b/go.sum @@ -316,8 +316,8 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -339,8 +339,8 @@ golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -375,15 +375,15 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 0290f86579bc92cce094287a6bd00affb51b0f0b Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sun, 31 Mar 2024 21:36:57 +0300 Subject: [PATCH 1009/1943] [#1065] adm: refactor dump-config Signed-off-by: Alexander Chuprov --- .../internal/modules/morph/config/config.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index 3414a8f7f..2238a38df 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -105,12 +105,22 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { forceFlag, _ := cmd.Flags().GetBool(forceConfigSet) bw := io.NewBufBinWriter() + prm := make(map[string]any) + for _, arg := range args { k, v, err := parseConfigPair(arg, forceFlag) if err != nil { return err } + prm[k] = v + } + + if err := validateConfig(prm); err != nil { + return err + } + + for k, v := range prm { // In NeoFS this is done via Notary contract. Here, however, we can form the // transaction locally. The first `nil` argument is required only for notary // disabled environment which is not supported by that command. @@ -128,6 +138,16 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { return wCtx.AwaitTx() } +func validateConfig(args map[string]any) error { + for k, v := range args { + value, ok := v.(int64) + if !ok || value < 0 { + return fmt.Errorf("%s must be >= 0, got %v", k, v) + } + } + return nil +} + func parseConfigPair(kvStr string, force bool) (key string, val any, err error) { k, v, found := strings.Cut(kvStr, "=") if !found { From 17f7adb640a18fef40c835acfaa225f26e3da481 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 8 Apr 2024 12:27:30 +0300 Subject: [PATCH 1010/1943] [#1065] adm: Add support EC parameters Signed-off-by: Alexander Chuprov --- cmd/frostfs-adm/docs/deploy.md | 2 ++ cmd/frostfs-adm/internal/commonflags/flags.go | 2 ++ .../internal/modules/config/config.go | 6 ++++ .../internal/modules/config/config_test.go | 2 ++ .../internal/modules/morph/config/config.go | 29 +++++++++++++++---- .../internal/modules/morph/helper/netmap.go | 2 ++ .../internal/modules/morph/initialize/root.go | 2 ++ dev/adm/frostfs-adm.yml | 2 ++ pkg/morph/client/netmap/config.go | 2 ++ 9 files changed, 43 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-adm/docs/deploy.md b/cmd/frostfs-adm/docs/deploy.md index 3b9f4c98c..e510f478d 100644 --- a/cmd/frostfs-adm/docs/deploy.md +++ b/cmd/frostfs-adm/docs/deploy.md @@ -34,6 +34,8 @@ alphabet-wallets: /home/user/deploy/alphabet-wallets network: max_object_size: 67108864 epoch_duration: 240 + max_ec_data_count: 12 + max_ec_parity_count: 4 fee: candidate: 0 container: 0 diff --git a/cmd/frostfs-adm/internal/commonflags/flags.go b/cmd/frostfs-adm/internal/commonflags/flags.go index bf6c17bd1..81395edb0 100644 --- a/cmd/frostfs-adm/internal/commonflags/flags.go +++ b/cmd/frostfs-adm/internal/commonflags/flags.go @@ -26,6 +26,8 @@ const ( ContractsURLFlagDesc = "URL to archive with compiled FrostFS contracts" EpochDurationInitFlag = "network.epoch_duration" MaxObjectSizeInitFlag = "network.max_object_size" + MaxECDataCountFlag = "network.max_ec_data_count" + MaxECParityCounFlag = "network.max_ec_parity_count" RefillGasAmountFlag = "gas" StorageWalletFlag = "storage-wallet" ContainerFeeInitFlag = "network.fee.container" diff --git a/cmd/frostfs-adm/internal/modules/config/config.go b/cmd/frostfs-adm/internal/modules/config/config.go index dd98a98b2..a98245d01 100644 --- a/cmd/frostfs-adm/internal/modules/config/config.go +++ b/cmd/frostfs-adm/internal/modules/config/config.go @@ -21,6 +21,8 @@ type configTemplate struct { CandidateFee int ContainerFee int ContainerAliasFee int + MaxECDataCount int + MaxECParityCount int WithdrawFee int Glagolitics []string HomomorphicHashDisabled bool @@ -31,6 +33,8 @@ alphabet-wallets: {{ .AlphabetDir}} network: max_object_size: {{ .MaxObjectSize}} epoch_duration: {{ .EpochDuration}} + max_ec_data_count: {{ .MaxECDataCount}} + max_ec_parity_count: {{ .MaxECParityCount}} homomorphic_hash_disabled: {{ .HomomorphicHashDisabled}} fee: candidate: {{ .CandidateFee}} @@ -106,6 +110,8 @@ func generateConfigExample(appDir string, credSize int) (string, error) { tmpl := configTemplate{ Endpoint: "https://neo.rpc.node:30333", MaxObjectSize: 67108864, // 64 MiB + MaxECDataCount: 12, // Tested with 16-node networks, assuming 12 data + 4 parity nodes. + MaxECParityCount: 4, // Maximum 4 parity chunks, typically <= 3 for most policies. EpochDuration: 240, // 1 hour with 15s per block HomomorphicHashDisabled: false, // object homomorphic hash is enabled CandidateFee: 100_0000_0000, // 100.0 GAS (Fixed8) diff --git a/cmd/frostfs-adm/internal/modules/config/config_test.go b/cmd/frostfs-adm/internal/modules/config/config_test.go index 3fa800a06..beb1210e1 100644 --- a/cmd/frostfs-adm/internal/modules/config/config_test.go +++ b/cmd/frostfs-adm/internal/modules/config/config_test.go @@ -27,6 +27,8 @@ func TestGenerateConfigExample(t *testing.T) { require.Equal(t, "https://neo.rpc.node:30333", v.GetString("rpc-endpoint")) require.Equal(t, filepath.Join(appDir, "alphabet-wallets"), v.GetString("alphabet-wallets")) require.Equal(t, 67108864, v.GetInt("network.max_object_size")) + require.Equal(t, 12, v.GetInt("network.max_ec_data_count")) + require.Equal(t, 4, v.GetInt("network.max_ec_parity_count")) require.Equal(t, 240, v.GetInt("network.epoch_duration")) require.Equal(t, 10000000000, v.GetInt("network.fee.candidate")) require.Equal(t, 1000, v.GetInt("network.fee.container")) diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index 2238a38df..f35e7aa03 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -60,7 +60,8 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { switch k { case netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, netmap.EpochDurationConfig, netmap.IrCandidateFeeConfig, - netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig: + netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig, + netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig: nbuf := make([]byte, 8) copy(nbuf[:], v) n := binary.LittleEndian.Uint64(nbuf) @@ -103,10 +104,8 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { } forceFlag, _ := cmd.Flags().GetBool(forceConfigSet) - bw := io.NewBufBinWriter() prm := make(map[string]any) - for _, arg := range args { k, v, err := parseConfigPair(arg, forceFlag) if err != nil { @@ -116,7 +115,7 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { prm[k] = v } - if err := validateConfig(prm); err != nil { + if err := validateConfig(prm, forceFlag); err != nil { return err } @@ -138,12 +137,29 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { return wCtx.AwaitTx() } -func validateConfig(args map[string]any) error { +func validateConfig(args map[string]any, forceFlag bool) error { + var sumEC int64 + _, okData := args[netmap.MaxECDataCountConfig] + _, okParity := args[netmap.MaxECParityCountConfig] + if okData != okParity { + return fmt.Errorf("both %s and %s must be present in the configuration", + netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig) + } + for k, v := range args { value, ok := v.(int64) if !ok || value < 0 { return fmt.Errorf("%s must be >= 0, got %v", k, v) } + + if k == netmap.MaxECDataCountConfig || k == netmap.MaxECParityCountConfig { + sumEC += value + } + } + + if sumEC > 256 && !forceFlag { + return fmt.Errorf("the sum of %s and %s must be <= 256, got %d", + netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig, sumEC) } return nil } @@ -160,7 +176,8 @@ func parseConfigPair(kvStr string, force bool) (key string, val any, err error) switch key { case netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, netmap.EpochDurationConfig, netmap.IrCandidateFeeConfig, - netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig: + netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig, + netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig: val, err = strconv.ParseInt(valRaw, 10, 64) if err != nil { err = fmt.Errorf("could not parse %s's value '%s' as int: %w", key, valRaw, err) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go index f68fbefaf..9ea27d3a3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go @@ -35,6 +35,8 @@ func GetDefaultNetmapContractConfigMap() map[string]any { m := make(map[string]any) m[netmap.EpochDurationConfig] = viper.GetInt64(commonflags.EpochDurationInitFlag) m[netmap.MaxObjectSizeConfig] = viper.GetInt64(commonflags.MaxObjectSizeInitFlag) + m[netmap.MaxECDataCountConfig] = viper.GetInt64(commonflags.MaxECDataCountFlag) + m[netmap.MaxECParityCountConfig] = viper.GetInt64(commonflags.MaxECParityCounFlag) m[netmap.ContainerFeeConfig] = viper.GetInt64(commonflags.ContainerFeeInitFlag) m[netmap.ContainerAliasFeeConfig] = viper.GetInt64(commonflags.ContainerAliasFeeInitFlag) m[netmap.IrCandidateFeeConfig] = viper.GetInt64(commonflags.CandidateFeeInitFlag) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/root.go b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go index 581bef532..b7885c512 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go @@ -25,6 +25,8 @@ var Cmd = &cobra.Command{ _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) _ = viper.BindPFlag(commonflags.EpochDurationInitFlag, cmd.Flags().Lookup(epochDurationCLIFlag)) _ = viper.BindPFlag(commonflags.MaxObjectSizeInitFlag, cmd.Flags().Lookup(maxObjectSizeCLIFlag)) + _ = viper.BindPFlag(commonflags.MaxECDataCountFlag, cmd.Flags().Lookup(commonflags.MaxECDataCountFlag)) + _ = viper.BindPFlag(commonflags.MaxECParityCounFlag, cmd.Flags().Lookup(commonflags.MaxECParityCounFlag)) _ = viper.BindPFlag(commonflags.HomomorphicHashDisabledInitFlag, cmd.Flags().Lookup(homomorphicHashDisabledCLIFlag)) _ = viper.BindPFlag(commonflags.CandidateFeeInitFlag, cmd.Flags().Lookup(candidateFeeCLIFlag)) _ = viper.BindPFlag(commonflags.ContainerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) diff --git a/dev/adm/frostfs-adm.yml b/dev/adm/frostfs-adm.yml index 5dc87e70c..e75cc27f0 100644 --- a/dev/adm/frostfs-adm.yml +++ b/dev/adm/frostfs-adm.yml @@ -6,6 +6,8 @@ network: basic_income_rate: 100000000 homomorphic_hash_disabled: false maintenance_mode_allowed: true + max_ec_data_count: 12 + max_ec_parity_count: 4 fee: audit: 10000 candidate: 10000000000 diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 21ba145d4..2d19a8193 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -11,6 +11,8 @@ import ( const ( MaxObjectSizeConfig = "MaxObjectSize" + MaxECParityCountConfig = "MaxECParityCount" + MaxECDataCountConfig = "MaxECDataCount" EpochDurationConfig = "EpochDuration" ContainerFeeConfig = "ContainerFee" ContainerAliasFeeConfig = "ContainerAliasFee" From 92569b9bbf7982a802cb3f8fb36406eaf897d701 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 8 Apr 2024 12:27:51 +0300 Subject: [PATCH 1011/1943] [#1065] cli: Add support EC parameters Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/modules/netmap/netinfo.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/frostfs-cli/modules/netmap/netinfo.go b/cmd/frostfs-cli/modules/netmap/netinfo.go index cd43ba68d..2dbd72355 100644 --- a/cmd/frostfs-cli/modules/netmap/netinfo.go +++ b/cmd/frostfs-cli/modules/netmap/netinfo.go @@ -43,6 +43,8 @@ var netInfoCmd = &cobra.Command{ cmd.Printf(format, "Epoch duration", netInfo.EpochDuration()) cmd.Printf(format, "Inner Ring candidate fee", netInfo.IRCandidateFee()) cmd.Printf(format, "Maximum object size", netInfo.MaxObjectSize()) + cmd.Printf(format, "Maximum count of data shards", netInfo.MaxECDataCount()) + cmd.Printf(format, "Maximum count of parity shards", netInfo.MaxECParityCount()) cmd.Printf(format, "Withdrawal fee", netInfo.WithdrawalFee()) cmd.Printf(format, "Homomorphic hashing disabled", netInfo.HomomorphicHashingDisabled()) cmd.Printf(format, "Maintenance mode allowed", netInfo.MaintenanceModeAllowed()) From 39da6433544be0fd70660181482b5f8247c92e21 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Mar 2024 11:36:58 +0300 Subject: [PATCH 1012/1943] [#1064] putsvc: Refactor distributed target Extract object builder. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/builder.go | 54 ++++++++++++++++++++++++++ pkg/services/object/put/distributed.go | 36 +---------------- pkg/services/object/put/streamer.go | 18 +++------ 3 files changed, 60 insertions(+), 48 deletions(-) create mode 100644 pkg/services/object/put/builder.go diff --git a/pkg/services/object/put/builder.go b/pkg/services/object/put/builder.go new file mode 100644 index 000000000..64baf4e05 --- /dev/null +++ b/pkg/services/object/put/builder.go @@ -0,0 +1,54 @@ +package putsvc + +import ( + "context" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" +) + +var _ transformer.ChunkedObjectWriter = (*inMemoryObjectBuilder)(nil) + +type inMemoryObjectBuilder struct { + objectWriter transformer.ObjectWriter + payload *payload + + obj *objectSDK.Object +} + +func newInMemoryObjectBuilder(objectWriter transformer.ObjectWriter) *inMemoryObjectBuilder { + return &inMemoryObjectBuilder{ + objectWriter: objectWriter, + payload: getPayload(), + } +} + +func (b *inMemoryObjectBuilder) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { + defer func() { + putPayload(b.payload) + b.payload = nil + }() + + b.obj.SetPayload(b.payload.Data) + + if err := b.objectWriter.WriteObject(ctx, b.obj); err != nil { + return nil, err + } + + id, _ := b.obj.ID() + return &transformer.AccessIdentifiers{ + SelfID: id, + }, nil +} + +func (b *inMemoryObjectBuilder) Write(_ context.Context, p []byte) (int, error) { + b.payload.Data = append(b.payload.Data, p...) + + return len(p), nil +} + +func (b *inMemoryObjectBuilder) WriteHeader(_ context.Context, obj *objectSDK.Object) error { + b.obj = obj + + return nil +} diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index 509f4aee0..c71427b67 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" ) type preparedObjectTarget interface { @@ -15,16 +14,13 @@ type preparedObjectTarget interface { } type distributedTarget struct { - placementOpts []placement.Option - extraBroadcastEnabled bool + placementOpts []placement.Option obj *objectSDK.Object objMeta object.ContentMeta *cfg - payload *payload - nodeTargetInitializer func(nodeDesc) preparedObjectTarget relay func(context.Context, nodeDesc) error @@ -91,36 +87,6 @@ func (x errIncompletePut) Error() string { return commonMsg } -func (t *distributedTarget) WriteHeader(_ context.Context, obj *objectSDK.Object) error { - t.obj = obj - - return nil -} - -func (t *distributedTarget) Write(_ context.Context, p []byte) (n int, err error) { - t.payload.Data = append(t.payload.Data, p...) - - return len(p), nil -} - -func (t *distributedTarget) Close(ctx context.Context) (*transformer.AccessIdentifiers, error) { - defer func() { - putPayload(t.payload) - t.payload = nil - }() - - t.obj.SetPayload(t.payload.Data) - - if err := t.WriteObject(ctx, t.obj); err != nil { - return nil, err - } - - id, _ := t.obj.ID() - return &transformer.AccessIdentifiers{ - SelfID: id, - }, nil -} - // WriteObject implements the transformer.ObjectWriter interface. func (t *distributedTarget) WriteObject(ctx context.Context, obj *objectSDK.Object) error { t.obj = obj diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 90d580a1c..f32b2ab99 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -79,7 +78,7 @@ func (p *Streamer) initUntrustedTarget(prm *PutInitPrm) error { // prepare untrusted-Put object target p.target = &validatingPreparedTarget{ - nextTarget: p.newCommonTarget(prm), + nextTarget: newInMemoryObjectBuilder(p.newObjectWriter(prm)), fmt: p.fmtValidator, maxPayloadSz: p.maxPayloadSz, @@ -133,7 +132,7 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { fmt: p.fmtValidator, nextTarget: transformer.NewPayloadSizeLimiter(transformer.Params{ Key: sessionKey, - NextTargetInit: func() transformer.ObjectWriter { return p.newCommonTarget(prm) }, + NextTargetInit: func() transformer.ObjectWriter { return p.newObjectWriter(prm) }, NetworkState: p.networkState, MaxSize: p.maxPayloadSz, WithoutHomomorphicHash: containerSDK.IsHomomorphicHashingDisabled(prm.cnr), @@ -196,7 +195,7 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error { return nil } -func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { +func (p *Streamer) newObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { var relay func(context.Context, nodeDesc) error if p.relay != nil { relay = func(ctx context.Context, node nodeDesc) error { @@ -213,16 +212,9 @@ func (p *Streamer) newCommonTarget(prm *PutInitPrm) *distributedTarget { } } - // enable additional container broadcast on non-local operation - // if object has TOMBSTONE or LOCK type. - typ := prm.hdr.Type() - withBroadcast := !prm.common.LocalOnly() && (typ == objectSDK.TypeTombstone || typ == objectSDK.TypeLock) - return &distributedTarget{ - cfg: p.cfg, - placementOpts: prm.traverseOpts, - extraBroadcastEnabled: withBroadcast, - payload: getPayload(), + cfg: p.cfg, + placementOpts: prm.traverseOpts, nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { return localTarget{ From 1c5e0f90aabaee60f2275b7fb3fead8bedabc019 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 28 Mar 2024 13:46:19 +0300 Subject: [PATCH 1013/1943] [#1064] putsvc: Add EC put Signed-off-by: Dmitrii Stepanov --- go.mod | 1 + go.sum | 2 + internal/logs/logs.go | 2 + pkg/core/container/ec.go | 11 + pkg/core/object/ec.go | 13 + pkg/core/policy/ec.go | 20 ++ pkg/services/object/put/ec.go | 265 ++++++++++++++++++ pkg/services/object/put/single.go | 88 +++++- pkg/services/object/put/streamer.go | 48 +++- pkg/services/object/put/writer.go | 23 ++ .../object_manager/placement/traverser.go | 2 +- 11 files changed, 452 insertions(+), 23 deletions(-) create mode 100644 pkg/core/container/ec.go create mode 100644 pkg/core/object/ec.go create mode 100644 pkg/core/policy/ec.go create mode 100644 pkg/services/object/put/ec.go create mode 100644 pkg/services/object/put/writer.go diff --git a/go.mod b/go.mod index 2544f0fb9..f940b306a 100644 --- a/go.mod +++ b/go.mod @@ -85,6 +85,7 @@ require ( github.com/ipfs/go-cid v0.4.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/klauspost/reedsolomon v1.12.1 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect diff --git a/go.sum b/go.sum index 8dbaa5373..7f7e84398 100644 --- a/go.sum +++ b/go.sum @@ -133,6 +133,8 @@ github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/reedsolomon v1.12.1 h1:NhWgum1efX1x58daOBGCFWcxtEhOhXKKl1HAPQUp03Q= +github.com/klauspost/reedsolomon v1.12.1/go.mod h1:nEi5Kjb6QqtbofI6s+cbG/j1da11c96IBYBSnVGtuBs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/internal/logs/logs.go b/internal/logs/logs.go index bd8b8ed01..64e5f771e 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -579,4 +579,6 @@ const ( EngineShardsEvacuationFailedToMoveTree = "failed to evacuate tree to other node" EngineShardsEvacuationTreeEvacuatedLocal = "tree evacuated to local node" EngineShardsEvacuationTreeEvacuatedRemote = "tree evacuated to other node" + ECFailedToSendToContainerNode = "failed to send EC object to container node" + ECFailedToSaveECPart = "failed to save EC part" ) diff --git a/pkg/core/container/ec.go b/pkg/core/container/ec.go new file mode 100644 index 000000000..1acb87f2b --- /dev/null +++ b/pkg/core/container/ec.go @@ -0,0 +1,11 @@ +package container + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" +) + +// IsECContainer returns True if container has erasure coding policy. +func IsECContainer(cnr containerSDK.Container) bool { + return policy.IsECPlacement(cnr.PlacementPolicy()) +} diff --git a/pkg/core/object/ec.go b/pkg/core/object/ec.go new file mode 100644 index 000000000..549ff7cd3 --- /dev/null +++ b/pkg/core/object/ec.go @@ -0,0 +1,13 @@ +package object + +import ( + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" +) + +// IsECSupported returns True if EC supported for object. +// +// EC supported only for regular, not linking objects. +func IsECSupported(obj *objectSDK.Object) bool { + return obj.Type() == objectSDK.TypeRegular && + len(obj.Children()) == 0 +} diff --git a/pkg/core/policy/ec.go b/pkg/core/policy/ec.go new file mode 100644 index 000000000..846af775a --- /dev/null +++ b/pkg/core/policy/ec.go @@ -0,0 +1,20 @@ +package policy + +import ( + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" +) + +// IsECPlacement returns True if policy is erasure coding policy. +func IsECPlacement(policy netmapSDK.PlacementPolicy) bool { + return policy.NumberOfReplicas() == 1 && policy.ReplicaDescriptor(0).GetECDataCount() > 0 +} + +// ECDataCount returns EC data count for EC placement policy. +func ECDataCount(policy netmapSDK.PlacementPolicy) int { + return int(policy.ReplicaDescriptor(0).GetECDataCount()) +} + +// ECParityCount returns EC parity count for EC placement policy. +func ECParityCount(policy netmapSDK.PlacementPolicy) int { + return int(policy.ReplicaDescriptor(0).GetECParityCount()) +} diff --git a/pkg/services/object/put/ec.go b/pkg/services/object/put/ec.go new file mode 100644 index 000000000..74db3c31f --- /dev/null +++ b/pkg/services/object/put/ec.go @@ -0,0 +1,265 @@ +package putsvc + +import ( + "context" + "crypto/ecdsa" + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" + svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +var _ transformer.ObjectWriter = (*ecWriter)(nil) + +var errUnsupportedECObject = errors.New("object is not supported for erasure coding") + +type ecWriter struct { + cfg *cfg + placementOpts []placement.Option + container containerSDK.Container + key *ecdsa.PrivateKey + commonPrm *svcutil.CommonPrm + relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error + + objMeta object.ContentMeta + objMetaValid bool +} + +func (e *ecWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error { + relayed, err := e.relayIfNotContainerNode(ctx) + if err != nil { + return err + } + if relayed { + return nil + } + + if !object.IsECSupported(obj) { + // must be resolved by caller + return errUnsupportedECObject + } + + if !e.objMetaValid { + if e.objMeta, err = e.cfg.fmtValidator.ValidateContent(obj); err != nil { + return fmt.Errorf("(%T) could not validate payload content: %w", e, err) + } + e.objMetaValid = true + } + + if obj.ECHeader() != nil { + return e.writeECPart(ctx, obj) + } + return e.writeRawObject(ctx, obj) +} + +func (e *ecWriter) relayIfNotContainerNode(ctx context.Context) (bool, error) { + if e.relay == nil { + return false, nil + } + currentNodeIsContainerNode, err := e.currentNodeIsContainerNode() + if err != nil { + return false, err + } + if currentNodeIsContainerNode { + // object can be splitted or saved local + return false, nil + } + if err := e.relayToContainerNode(ctx); err != nil { + return false, err + } + return true, nil +} + +func (e *ecWriter) currentNodeIsContainerNode() (bool, error) { + t, err := placement.NewTraverser(e.placementOpts...) + if err != nil { + return false, err + } + for { + nodes := t.Next() + if len(nodes) == 0 { + break + } + for _, node := range nodes { + if e.cfg.netmapKeys.IsLocalKey(node.PublicKey()) { + return true, nil + } + } + } + return false, nil +} + +func (e *ecWriter) relayToContainerNode(ctx context.Context) error { + t, err := placement.NewTraverser(e.placementOpts...) + if err != nil { + return err + } + var lastErr error + for { + nodes := t.Next() + if len(nodes) == 0 { + break + } + for _, node := range nodes { + var info client.NodeInfo + client.NodeInfoFromNetmapElement(&info, node) + + c, err := e.cfg.clientConstructor.Get(info) + if err != nil { + return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) + } + + completed := make(chan interface{}) + if poolErr := e.cfg.remotePool.Submit(func() { + defer close(completed) + err = e.relay(ctx, info, c) + }); poolErr != nil { + close(completed) + svcutil.LogWorkerPoolError(e.cfg.log, "PUT", poolErr) + return poolErr + } + <-completed + + if err == nil { + return nil + } + e.cfg.log.Logger.Warn(logs.ECFailedToSendToContainerNode, zap.Stringers("address_group", info.AddressGroup())) + lastErr = err + } + } + if lastErr == nil { + return nil + } + return errIncompletePut{ + singleErr: lastErr, + } +} + +func (e *ecWriter) writeECPart(ctx context.Context, obj *objectSDK.Object) error { + t, err := placement.NewTraverser(e.placementOpts...) + if err != nil { + return err + } + + eg, egCtx := errgroup.WithContext(ctx) + for { + nodes := t.Next() + if len(nodes) == 0 { + break + } + + eg.Go(func() error { + return e.writePart(egCtx, obj, int(obj.ECHeader().Index()), nodes) + }) + } + if err := eg.Wait(); err != nil { + return errIncompletePut{ + singleErr: err, + } + } + return nil +} + +func (e *ecWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) error { + // now only single EC policy is supported + c, err := erasurecode.NewConstructor(policy.ECDataCount(e.container.PlacementPolicy()), policy.ECParityCount(e.container.PlacementPolicy())) + if err != nil { + return err + } + parts, err := c.Split(obj, e.key) + if err != nil { + return err + } + t, err := placement.NewTraverser(e.placementOpts...) + if err != nil { + return err + } + + eg, egCtx := errgroup.WithContext(ctx) + for { + nodes := t.Next() + if len(nodes) == 0 { + break + } + + for idx := range parts { + idx := idx + eg.Go(func() error { + return e.writePart(egCtx, parts[idx], idx, nodes) + }) + } + } + if err := eg.Wait(); err != nil { + return errIncompletePut{ + singleErr: err, + } + } + return nil +} + +func (e *ecWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx int, nodes []placement.Node) error { + var err error + node := nodes[partIdx%len(nodes)] + if e.cfg.netmapKeys.IsLocalKey(node.PublicKey()) { + err = e.writePartLocal(ctx, obj) + } else { + err = e.writePartRemote(ctx, obj, node) + } + if err == nil { + return nil + } + e.cfg.log.Warn(logs.ECFailedToSaveECPart, zap.Stringer("parent_object", object.AddressOf(obj)), zap.Error(err)) + return err +} + +func (e *ecWriter) writePartLocal(ctx context.Context, obj *objectSDK.Object) error { + var err error + localTarget := localTarget{ + storage: e.cfg.localStore, + } + completed := make(chan interface{}) + if poolErr := e.cfg.localPool.Submit(func() { + defer close(completed) + err = localTarget.WriteObject(ctx, obj, e.objMeta) + }); poolErr != nil { + close(completed) + return poolErr + } + <-completed + return err +} + +func (e *ecWriter) writePartRemote(ctx context.Context, obj *objectSDK.Object, node placement.Node) error { + var clientNodeInfo client.NodeInfo + client.NodeInfoFromNetmapElement(&clientNodeInfo, node) + + remoteTaget := remoteTarget{ + privateKey: e.key, + clientConstructor: e.cfg.clientConstructor, + commonPrm: e.commonPrm, + nodeInfo: clientNodeInfo, + } + + var err error + completed := make(chan interface{}) + if poolErr := e.cfg.remotePool.Submit(func() { + defer close(completed) + err = remoteTaget.WriteObject(ctx, obj, e.objMeta) + }); poolErr != nil { + close(completed) + return poolErr + } + <-completed + return err +} diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 09e1eb092..43b3b0ac1 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -16,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" @@ -25,6 +26,7 @@ import ( tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/tzhash/tz" "go.opentelemetry.io/otel/attribute" @@ -32,7 +34,10 @@ import ( "go.uber.org/zap" ) -var errInvalidPayloadChecksum = errors.New("incorrect payload checksum") +var ( + errInvalidPayloadChecksum = errors.New("incorrect payload checksum") + errInvalidECObject = errors.New("object must be splitted to EC parts") +) type putSingleRequestSigner struct { req *objectAPI.PutSingleRequest @@ -148,12 +153,20 @@ func (s *Service) validatePutSingleObject(ctx context.Context, obj *objectSDK.Ob func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { localOnly := req.GetMetaHeader().GetTTL() <= 1 - placementOptions, err := s.getPutSinglePlacementOptions(obj, req.GetBody().GetCopiesNumber(), localOnly) + placement, err := s.getPutSinglePlacementOptions(obj, req.GetBody().GetCopiesNumber(), localOnly) if err != nil { return err } - iter := s.cfg.newNodeIterator(placementOptions) + if placement.isEC { + return s.saveToECReplicas(ctx, placement, obj, req, meta) + } + + return s.saveToREPReplicas(ctx, placement, obj, localOnly, req, meta) +} + +func (s *Service) saveToREPReplicas(ctx context.Context, placement putSinglePlacement, obj *objectSDK.Object, localOnly bool, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { + iter := s.cfg.newNodeIterator(placement.placementOptions) iter.extraBroadcastEnabled = needAdditionalBroadcast(obj, localOnly) signer := &putSingleRequestSigner{ @@ -167,38 +180,83 @@ func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *o }) } -func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) ([]placement.Option, error) { - var result []placement.Option - if len(copiesNumber) > 0 { - result = append(result, placement.WithCopyNumbers(copiesNumber)) +func (s *Service) saveToECReplicas(ctx context.Context, placement putSinglePlacement, obj *objectSDK.Object, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { + if obj.Type() == objectSDK.TypeRegular && obj.ECHeader() == nil { + return errInvalidECObject } + commonPrm, err := svcutil.CommonPrmFromV2(req) + if err != nil { + return err + } + key, err := s.cfg.keyStorage.GetKey(nil) + if err != nil { + return err + } + signer := &putSingleRequestSigner{ + req: req, + keyStorage: s.keyStorage, + signer: &sync.Once{}, + } + + w := ecWriter{ + cfg: s.cfg, + placementOpts: placement.placementOptions, + objMeta: meta, + objMetaValid: true, + commonPrm: commonPrm, + container: placement.container, + key: key, + relay: func(ctx context.Context, ni client.NodeInfo, mac client.MultiAddressClient) error { + return s.redirectPutSingleRequest(ctx, signer, obj, ni, mac) + }, + } + return w.WriteObject(ctx, obj) +} + +type putSinglePlacement struct { + placementOptions []placement.Option + isEC bool + container containerSDK.Container +} + +func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) (putSinglePlacement, error) { + var result putSinglePlacement + cnrID, ok := obj.ContainerID() if !ok { - return nil, errors.New("missing container ID") + return result, errors.New("missing container ID") } cnrInfo, err := s.cnrSrc.Get(cnrID) if err != nil { - return nil, fmt.Errorf("could not get container by ID: %w", err) + return result, fmt.Errorf("could not get container by ID: %w", err) } - result = append(result, placement.ForContainer(cnrInfo.Value)) + result.container = cnrInfo.Value + result.isEC = container.IsECContainer(cnrInfo.Value) && object.IsECSupported(obj) + if len(copiesNumber) > 0 && !result.isEC { + result.placementOptions = append(result.placementOptions, placement.WithCopyNumbers(copiesNumber)) + } + result.placementOptions = append(result.placementOptions, placement.ForContainer(cnrInfo.Value)) objID, ok := obj.ID() if !ok { - return nil, errors.New("missing object ID") + return result, errors.New("missing object ID") } - result = append(result, placement.ForObject(objID)) + if obj.ECHeader() != nil { + objID = obj.ECHeader().Parent() + } + result.placementOptions = append(result.placementOptions, placement.ForObject(objID)) latestNetmap, err := netmap.GetLatestNetworkMap(s.netMapSrc) if err != nil { - return nil, fmt.Errorf("could not get latest network map: %w", err) + return result, fmt.Errorf("could not get latest network map: %w", err) } builder := placement.NewNetworkMapBuilder(latestNetmap) if localOnly { - result = append(result, placement.SuccessAfter(1)) + result.placementOptions = append(result.placementOptions, placement.SuccessAfter(1)) builder = svcutil.NewLocalPlacement(builder, s.netmapKeys) } - result = append(result, placement.UseBuilder(builder)) + result.placementOptions = append(result.placementOptions, placement.UseBuilder(builder)) return result, nil } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index f32b2ab99..14dae38d5 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -7,7 +7,9 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -19,7 +21,7 @@ import ( type Streamer struct { *cfg - sessionKey *ecdsa.PrivateKey + privateKey *ecdsa.PrivateKey target transformer.ChunkedObjectWriter @@ -76,6 +78,12 @@ func (p *Streamer) initTarget(prm *PutInitPrm) error { func (p *Streamer) initUntrustedTarget(prm *PutInitPrm) error { p.relay = prm.relay + nodeKey, err := p.cfg.keyStorage.GetKey(nil) + if err != nil { + return err + } + p.privateKey = nodeKey + // prepare untrusted-Put object target p.target = &validatingPreparedTarget{ nextTarget: newInMemoryObjectBuilder(p.newObjectWriter(prm)), @@ -102,7 +110,7 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { } } - sessionKey, err := p.keyStorage.GetKey(sessionInfo) + key, err := p.keyStorage.GetKey(sessionInfo) if err != nil { return fmt.Errorf("(%T) could not receive session key: %w", p, err) } @@ -116,7 +124,7 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { if sToken == nil { var ownerSession user.ID - user.IDFromKey(&ownerSession, sessionKey.PublicKey) + user.IDFromKey(&ownerSession, key.PublicKey) if !ownerObj.Equals(ownerSession) { return fmt.Errorf("(%T) session token is missing but object owner id is different from the default key", p) @@ -127,11 +135,11 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { } } - p.sessionKey = sessionKey + p.privateKey = key p.target = &validatingTarget{ fmt: p.fmtValidator, nextTarget: transformer.NewPayloadSizeLimiter(transformer.Params{ - Key: sessionKey, + Key: key, NextTargetInit: func() transformer.ObjectWriter { return p.newObjectWriter(prm) }, NetworkState: p.networkState, MaxSize: p.maxPayloadSz, @@ -171,7 +179,12 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error { placement.ForContainer(prm.cnr), ) - if id, ok := prm.hdr.ID(); ok { + if ech := prm.hdr.ECHeader(); ech != nil { + prm.traverseOpts = append(prm.traverseOpts, + // set identifier of the processing object + placement.ForObject(ech.Parent()), + ) + } else if id, ok := prm.hdr.ID(); ok { prm.traverseOpts = append(prm.traverseOpts, // set identifier of the processing object placement.ForObject(id), @@ -196,6 +209,13 @@ func (p *Streamer) preparePrm(prm *PutInitPrm) error { } func (p *Streamer) newObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { + if container.IsECContainer(prm.cnr) && object.IsECSupported(prm.hdr) { + return p.newECWriter(prm) + } + return p.newDefaultObjectWriter(prm) +} + +func (p *Streamer) newDefaultObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { var relay func(context.Context, nodeDesc) error if p.relay != nil { relay = func(ctx context.Context, node nodeDesc) error { @@ -223,7 +243,7 @@ func (p *Streamer) newObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { } rt := &remoteTarget{ - privateKey: p.sessionKey, + privateKey: p.privateKey, commonPrm: prm.common, clientConstructor: p.clientConstructor, } @@ -236,6 +256,20 @@ func (p *Streamer) newObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { } } +func (p *Streamer) newECWriter(prm *PutInitPrm) transformer.ObjectWriter { + return &objectWriterDispatcher{ + ecWriter: &ecWriter{ + cfg: p.cfg, + placementOpts: append(prm.traverseOpts, placement.WithCopyNumbers(nil)), // copies number ignored for EC + container: prm.cnr, + key: p.privateKey, + commonPrm: prm.common, + relay: p.relay, + }, + repWriter: p.newDefaultObjectWriter(prm), + } +} + func (p *Streamer) SendChunk(ctx context.Context, prm *PutChunkPrm) error { if p.target == nil { return errNotInit diff --git a/pkg/services/object/put/writer.go b/pkg/services/object/put/writer.go new file mode 100644 index 000000000..53eee6006 --- /dev/null +++ b/pkg/services/object/put/writer.go @@ -0,0 +1,23 @@ +package putsvc + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" +) + +var _ transformer.ObjectWriter = (*objectWriterDispatcher)(nil) + +type objectWriterDispatcher struct { + ecWriter transformer.ObjectWriter + repWriter transformer.ObjectWriter +} + +func (m *objectWriterDispatcher) WriteObject(ctx context.Context, obj *objectSDK.Object) error { + if object.IsECSupported(obj) { + return m.ecWriter.WriteObject(ctx, obj) + } + return m.repWriter.WriteObject(ctx, obj) +} diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index a699b4454..306169571 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -137,7 +137,7 @@ func defaultCopiesVector(policy netmap.PlacementPolicy) []int { copyVector := make([]int, 0, replNum) for i := 0; i < replNum; i++ { - copyVector = append(copyVector, int(policy.ReplicaDescriptor(i).NumberOfObjects())) + copyVector = append(copyVector, int(policy.ReplicaDescriptor(i).NumberOfObjects()+policy.ReplicaDescriptor(i).GetECDataCount()+policy.ReplicaDescriptor(i).GetECParityCount())) } return copyVector From 8668cbf147759c57c96a015174a51790a2982d1b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 28 Mar 2024 13:47:36 +0300 Subject: [PATCH 1014/1943] [#1064] dev: Add `IR + 4 storage nodes` configuration Signed-off-by: Dmitrii Stepanov --- Makefile | 12 +- dev/.vscode-example/launch.json | 197 +++++++++++++++++++-- dev/storage/{wallet.json => wallet01.json} | 0 dev/storage/wallet02.json | 30 ++++ dev/storage/wallet03.json | 30 ++++ dev/storage/wallet04.json | 30 ++++ 6 files changed, 278 insertions(+), 21 deletions(-) rename dev/storage/{wallet.json => wallet01.json} (100%) create mode 100644 dev/storage/wallet02.json create mode 100644 dev/storage/wallet03.json create mode 100644 dev/storage/wallet04.json diff --git a/Makefile b/Makefile index 20c84d041..fbf93facd 100755 --- a/Makefile +++ b/Makefile @@ -276,15 +276,19 @@ env-up: all echo "Frostfs contracts not found"; exit 1; \ fi ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph init --contracts ${FROSTFS_CONTRACTS_PATH} - ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet.json --gas 10.0 + ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet01.json --gas 10.0 + ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet02.json --gas 10.0 + ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet03.json --gas 10.0 + ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet04.json --gas 10.0 @if [ ! -f "$(LOCODE_DB_PATH)" ]; then \ make locode-download; \ fi + mkdir -p ./$(TMP_DIR)/state + mkdir -p ./$(TMP_DIR)/storage # Shutdown dev environment env-down: docker compose -f dev/docker-compose.yml down docker volume rm -f frostfs-node_neo-go - rm -f ./.cache/.frostfs-ir-state - rm -f ./.cache/.frostfs-node-state - rm -rf ./.cache/storage + rm -rf ./$(TMP_DIR)/state + rm -rf ./$(TMP_DIR)/storage diff --git a/dev/.vscode-example/launch.json b/dev/.vscode-example/launch.json index 459350145..6aedde85e 100644 --- a/dev/.vscode-example/launch.json +++ b/dev/.vscode-example/launch.json @@ -27,12 +27,12 @@ "FROSTFS_IR_NETMAP_CLEANER_THRESHOLD":"3", "FROSTFS_IR_LOCODE_DB_PATH":"${workspaceFolder}/.cache/locode_db", "FROSTFS_IR_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8090", - "FROSTFS_IR_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/.frostfs-ir-state" + "FROSTFS_IR_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-ir-state" }, "postDebugTask": "env-down" }, { - "name": "Storage node", + "name": "Storage node 1", "type": "go", "request": "launch", "mode": "debug", @@ -42,7 +42,8 @@ "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", - "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet.json", + "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", + "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet01.json", "FROSTFS_NODE_WALLET_PASSWORD":"", "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8080", "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8080", @@ -50,31 +51,187 @@ "FROSTFS_CONTROL_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", - "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/.frostfs-node-state", + "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-node-s1-state", "FROSTFS_TREE_ENABLED":"true", "FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME":"10", "FROSTFS_STORAGE_SHARD_0_WRITECACHE_ENABLED":"true", - "FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/wc0", - "FROSTFS_STORAGE_SHARD_0_METABASE_PATH":"${workspaceFolder}/.cache/storage/meta0", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s1/wc0", + "FROSTFS_STORAGE_SHARD_0_METABASE_PATH":"${workspaceFolder}/.cache/storage/s1/meta0", "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE":"blobovnicza", - "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/blobovnicza0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s1/blobovnicza0", "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH":"2", "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH":"4", "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE":"fstree", - "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/fstree0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s1/fstree0", "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH":"2", - "FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/pilorama0", + "FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s1/pilorama0", "FROSTFS_STORAGE_SHARD_1_WRITECACHE_ENABLED":"true", - "FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/wc1", - "FROSTFS_STORAGE_SHARD_1_METABASE_PATH":"${workspaceFolder}/.cache/storage/meta1", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s1/wc1", + "FROSTFS_STORAGE_SHARD_1_METABASE_PATH":"${workspaceFolder}/.cache/storage/s1/meta1", "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE":"blobovnicza", - "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/blobovnicza1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s1/blobovnicza1", "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH":"2", "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH":"4", "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE":"fstree", - "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/fstree1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s1/fstree1", "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH":"2", - "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/pilorama1" + "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s1/pilorama1", + "FROSTFS_PROMETHEUS_ENABLED":"true", + "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9090", + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" + }, + "postDebugTask": "env-down" + }, + { + "name": "Storage node 2", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "cmd/frostfs-node", + "env": { + "FROSTFS_LOGGER_LEVEL":"debug", + "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", + "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", + "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", + "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", + "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet02.json", + "FROSTFS_NODE_WALLET_PASSWORD":"", + "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8082", + "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8082", + "FROSTFS_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8083", + "FROSTFS_CONTROL_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", + "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", + "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", + "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-node-s2-state", + "FROSTFS_TREE_ENABLED":"true", + "FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME":"10", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s2/wc0", + "FROSTFS_STORAGE_SHARD_0_METABASE_PATH":"${workspaceFolder}/.cache/storage/s2/meta0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s2/blobovnicza0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s2/fstree0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s2/pilorama0", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s2/wc1", + "FROSTFS_STORAGE_SHARD_1_METABASE_PATH":"${workspaceFolder}/.cache/storage/s2/meta1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s2/blobovnicza1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s2/fstree1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s2/pilorama1", + "FROSTFS_PROMETHEUS_ENABLED":"true", + "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9091", + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" + }, + "postDebugTask": "env-down" + }, + { + "name": "Storage node 3", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "cmd/frostfs-node", + "env": { + "FROSTFS_LOGGER_LEVEL":"debug", + "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", + "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", + "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", + "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", + "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet03.json", + "FROSTFS_NODE_WALLET_PASSWORD":"", + "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8084", + "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8084", + "FROSTFS_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8085", + "FROSTFS_CONTROL_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", + "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", + "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", + "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-node-s3-state", + "FROSTFS_TREE_ENABLED":"true", + "FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME":"10", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s3/wc0", + "FROSTFS_STORAGE_SHARD_0_METABASE_PATH":"${workspaceFolder}/.cache/storage/s3/meta0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s3/blobovnicza0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s3/fstree0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s3/pilorama0", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s3/wc1", + "FROSTFS_STORAGE_SHARD_1_METABASE_PATH":"${workspaceFolder}/.cache/storage/s3/meta1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s3/blobovnicza1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s3/fstree1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s3/pilorama1", + "FROSTFS_PROMETHEUS_ENABLED":"true", + "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9092", + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" + }, + "postDebugTask": "env-down" + }, + { + "name": "Storage node 4", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "cmd/frostfs-node", + "env": { + "FROSTFS_LOGGER_LEVEL":"debug", + "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", + "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", + "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", + "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", + "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet04.json", + "FROSTFS_NODE_WALLET_PASSWORD":"", + "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8086", + "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8086", + "FROSTFS_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8087", + "FROSTFS_CONTROL_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", + "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", + "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", + "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-node-s4-state", + "FROSTFS_TREE_ENABLED":"true", + "FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME":"10", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_0_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s4/wc0", + "FROSTFS_STORAGE_SHARD_0_METABASE_PATH":"${workspaceFolder}/.cache/storage/s4/meta0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s4/blobovnicza0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s4/fstree0", + "FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_0_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s4/pilorama0", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_ENABLED":"true", + "FROSTFS_STORAGE_SHARD_1_WRITECACHE_PATH":"${workspaceFolder}/.cache/storage/s4/wc1", + "FROSTFS_STORAGE_SHARD_1_METABASE_PATH":"${workspaceFolder}/.cache/storage/s4/meta1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_TYPE":"blobovnicza", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_PATH":"${workspaceFolder}/.cache/storage/s4/blobovnicza1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH":"4", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE":"fstree", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH":"${workspaceFolder}/.cache/storage/s4/fstree1", + "FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_DEPTH":"2", + "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s4/pilorama1", + "FROSTFS_PROMETHEUS_ENABLED":"true", + "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9093", + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" }, "postDebugTask": "env-down" } @@ -82,9 +239,15 @@ "compounds": [ { "name": "IR+Storage node", - "configurations": ["IR", "Storage node"], + "configurations": ["IR", "Storage node 1"], "preLaunchTask": "env-up", "stopAll": true - } + }, + { + "name": "IR + 4 storage nodes", + "configurations": ["IR", "Storage node 1", "Storage node 2", "Storage node 3", "Storage node 4"], + "preLaunchTask": "env-up", + "stopAll": true + } ] -} +} \ No newline at end of file diff --git a/dev/storage/wallet.json b/dev/storage/wallet01.json similarity index 100% rename from dev/storage/wallet.json rename to dev/storage/wallet01.json diff --git a/dev/storage/wallet02.json b/dev/storage/wallet02.json new file mode 100644 index 000000000..9c073deef --- /dev/null +++ b/dev/storage/wallet02.json @@ -0,0 +1,30 @@ +{ + "version":"3.0", + "accounts":[ + { + "address":"NVXXy3hNTvwVEZa2dAibALyJB3Q86aiHvL", + "key":"6PYXd9hxMYfaCkgeZp3q1RoMB921RQFkRxYftcacTJ2S7MUwnivrxi6Yk5", + "label":"", + "contract":{ + "script":"DCED/2W2rnkTSk3OnQ0504Uem6tO6Xq/hugeHFu8UM0oJq5BVuezJw==", + "parameters":[ + { + "name":"parameter0", + "type":"Signature" + } + ], + "deployed":false + }, + "lock":false, + "isDefault":false + } + ], + "scrypt":{ + "n":16384, + "r":8, + "p":8 + }, + "extra":{ + "Tokens":null + } + } diff --git a/dev/storage/wallet03.json b/dev/storage/wallet03.json new file mode 100644 index 000000000..c054a3160 --- /dev/null +++ b/dev/storage/wallet03.json @@ -0,0 +1,30 @@ +{ + "version":"3.0", + "accounts":[ + { + "address":"NPTmih9X14Y7xLvmD6RVtDHdH1Y9qJwoTe", + "key":"6PYXNeQzge9fWztVnWYRbr5Mh9q1y4npKVARHYGb484Hct1iNd3vXGR1kk", + "label":"", + "contract":{ + "script":"DCECrJIM198LYbKJBy5rlG4tpOGjG5qxxiG7R14w+kqxAsNBVuezJw==", + "parameters":[ + { + "name":"parameter0", + "type":"Signature" + } + ], + "deployed":false + }, + "lock":false, + "isDefault":false + } + ], + "scrypt":{ + "n":16384, + "r":8, + "p":8 + }, + "extra":{ + "Tokens":null + } + } diff --git a/dev/storage/wallet04.json b/dev/storage/wallet04.json new file mode 100644 index 000000000..cb4676df6 --- /dev/null +++ b/dev/storage/wallet04.json @@ -0,0 +1,30 @@ +{ + "version":"3.0", + "accounts":[ + { + "address":"Ne2DAQbWvP1s7TbtFc7BStKMnjKJdBaVRm", + "key":"6PYWCsGWx8uSVYK94tvK7Ccit8x8Z3f3dHADTFTgLhT9NBXTBqBECL8AyC", + "label":"", + "contract":{ + "script":"DCEDjIYpWeVrQ+IPeRh8T+ngvHyMZsFgPmzw7H+Hq2sI3DVBVuezJw==", + "parameters":[ + { + "name":"parameter0", + "type":"Signature" + } + ], + "deployed":false + }, + "lock":false, + "isDefault":false + } + ], + "scrypt":{ + "n":16384, + "r":8, + "p":8 + }, + "extra":{ + "Tokens":null + } + } From bdf49909041342266bf2fa6ef548abfcf85652b6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 28 Mar 2024 17:27:22 +0300 Subject: [PATCH 1015/1943] [#1064] cli: Add EC header output to object head Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/head.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index db466e588..f97ef952d 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -171,6 +171,15 @@ func printHeader(cmd *cobra.Command, obj *objectSDK.Object) error { cmd.Printf(" signature: %s\n", hex.EncodeToString(sigV2.GetSign())) } + if ecHeader := obj.ECHeader(); ecHeader != nil { + cmd.Print("EC header:\n") + + cmd.Printf(" parent object ID: %s\n", ecHeader.Parent().EncodeToString()) + cmd.Printf(" index: %d\n", ecHeader.Index()) + cmd.Printf(" total: %d\n", ecHeader.Total()) + cmd.Printf(" header length: %d\n", ecHeader.HeaderLength()) + } + return printSplitHeader(cmd, obj) } From f5b67c67353cf20d8ab0d0f8dd5fdcca40d363c2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 28 Mar 2024 17:29:30 +0300 Subject: [PATCH 1016/1943] [#1064] policer: Disable EC processing Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/check.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index d2297fed7..d68d99d18 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + policycore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -40,6 +41,10 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add } policy := cnr.Value.PlacementPolicy() + if policycore.IsECPlacement(policy) { + // EC not supported yet by policer + return nil + } nn, err := p.placementBuilder.BuildPlacement(idCnr, &idObj, policy) if err != nil { From 2b8836184971997e24d4c1fa5e9af1b5f0db2740 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 9 Apr 2024 11:53:18 +0300 Subject: [PATCH 1017/1943] [#1062] object: Fix buffer allocation for `PayloadRange` Signed-off-by: Anton Nikiforov --- pkg/services/object/internal/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index dd2de0fd1..4634c96e1 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -348,7 +348,7 @@ func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, e ln = maxInitialBufferSize } - w := bytes.NewBuffer(make([]byte, ln)) + w := bytes.NewBuffer(make([]byte, 0, ln)) _, err = io.CopyN(w, rdr, int64(prm.ln)) if err != nil { return nil, fmt.Errorf("read payload: %w", err) From 5b8200de8800b1f239e63b0cf004428a19fb3339 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 4 Mar 2024 11:58:36 +0300 Subject: [PATCH 1018/1943] [#984] blobovnicza: Do not fail rebuild on big objects If blobovnicza contains objects larger than object size parameter value, then rebuild fails with an error, because there is no such bucket in database. This commit forces to create bucket on rebuild. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobovnicza/put.go | 17 ++++- .../blobstor/blobovniczatree/rebuild.go | 1 + .../blobstor/blobovniczatree/rebuild_test.go | 67 +++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index d15b6e2b2..9af8a93fe 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -17,6 +17,8 @@ type PutPrm struct { addr oid.Address objData []byte + + force bool } // PutRes groups the resulting values of Put operation. @@ -32,6 +34,11 @@ func (p *PutPrm) SetMarshaledObject(data []byte) { p.objData = data } +// SetForce sets force option. +func (p *PutPrm) SetForce(f bool) { + p.force = f +} + // Put saves an object in Blobovnicza. // // If binary representation of the object is not set, @@ -66,7 +73,15 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { // expected to happen: // - before initialization step (incorrect usage by design) // - if DB is corrupted (in future this case should be handled) - return logicerr.Wrap(fmt.Errorf("(%T) bucket for size %d not created", b, sz)) + // - blobovnicza's object size changed before rebuild (handled if prm.force flag specified) + if !prm.force { + return logicerr.Wrap(fmt.Errorf("(%T) bucket for size %d not created", b, sz)) + } + var err error + buck, err = tx.CreateBucket(bucketName) + if err != nil { + return fmt.Errorf("(%T) failed to create bucket for size %d: %w", b, sz, err) + } } // save the object in bucket diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index 2302661d5..93ef8ba2e 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -402,6 +402,7 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, var putPrm blobovnicza.PutPrm putPrm.SetAddress(i.Address) putPrm.SetMarshaledObject(i.ObjectData) + putPrm.SetForce(true) _, err = target.Blobovnicza().Put(ctx, putPrm) if err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 0ac50f36a..578cb1a7f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -38,6 +38,73 @@ func TestBlobovniczaTreeRebuild(t *testing.T) { }) } +func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + b := NewBlobovniczaTree( + WithLogger(test.NewLogger(t)), + WithObjectSizeLimit(64*1024), // 64KB object size limit + WithBlobovniczaShallowWidth(5), + WithBlobovniczaShallowDepth(2), // depth = 2 + WithRootPath(dir), + WithBlobovniczaSize(100*1024), + WithWaitBeforeDropDB(0), + WithOpenedCacheSize(1000), + WithMoveBatchSize(3)) + require.NoError(t, b.Open(false)) + require.NoError(t, b.Init()) + + obj := blobstortest.NewObject(64 * 1024) // 64KB object + data, err := obj.Marshal() + require.NoError(t, err) + var prm common.PutPrm + prm.Address = object.AddressOf(obj) + prm.RawData = data + res, err := b.Put(context.Background(), prm) + require.NoError(t, err) + + storageIDs := make(map[oid.Address][]byte) + storageIDs[prm.Address] = res.StorageID + + require.NoError(t, b.Close()) + + b = NewBlobovniczaTree( + WithLogger(test.NewLogger(t)), + WithObjectSizeLimit(32*1024), // 32KB object size limit + WithBlobovniczaShallowWidth(5), + WithBlobovniczaShallowDepth(3), // depth = 3 + WithRootPath(dir), + WithBlobovniczaSize(100*1024), + WithWaitBeforeDropDB(0), + WithOpenedCacheSize(1000), + WithMoveBatchSize(3)) + require.NoError(t, b.Open(false)) + require.NoError(t, b.Init()) + + metaStub := &storageIDUpdateStub{ + storageIDs: storageIDs, + guard: &sync.Mutex{}, + } + var rPrm common.RebuildPrm + rPrm.MetaStorage = metaStub + rPrm.WorkerLimiter = &rebuildLimiterStub{} + rRes, err := b.Rebuild(context.Background(), rPrm) + require.NoError(t, err) + dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 + require.True(t, dataMigrated) + + for addr, storageID := range storageIDs { + var gPrm common.GetPrm + gPrm.Address = addr + gPrm.StorageID = storageID + _, err := b.Get(context.Background(), gPrm) + require.NoError(t, err) + } + + require.NoError(t, b.Close()) +} + func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, targetDepth, targetWidth uint64, shouldMigrate bool) { dir := t.TempDir() b := NewBlobovniczaTree( From 338d8cbebd235a437250c9c4113fddc74bf9c474 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 4 Apr 2024 14:55:38 +0300 Subject: [PATCH 1019/1943] [#1080] ape: Do not read object headers before Head/Get Signed-off-by: Dmitrii Stepanov --- pkg/services/object/ape/checker.go | 3 +++ pkg/services/object/ape/request.go | 2 +- pkg/services/object/ape/service.go | 34 ++++++++++++++++-------------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 45efce5a7..119df5fc0 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -54,6 +54,9 @@ type Prm struct { // If SoftAPECheck is set to true, then NoRuleFound is interpreted as allow. SoftAPECheck bool + + // If true, object headers will not retrieved from storage engine. + WithoutHeaderRequest bool } var errMissingOID = errors.New("object ID is not set") diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index 3689bd178..bdce97676 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -141,7 +141,7 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (*request, err var header *objectV2.Header if prm.Header != nil { header = prm.Header - } else if prm.Object != nil { + } else if prm.Object != nil && !prm.WithoutHeaderRequest { headerObjSDK, err := c.headerProvider.GetHeader(ctx, prm.Container, *prm.Object) if err == nil { header = headerObjSDK.ToV2().GetHeader() diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 0c203209d..95f36be79 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -125,14 +125,15 @@ func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectSt } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodGetObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - SoftAPECheck: reqCtx.SoftAPECheck, + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodGetObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + SoftAPECheck: reqCtx.SoftAPECheck, + WithoutHeaderRequest: true, }) if err != nil { return toStatusErr(err) @@ -211,14 +212,15 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj } err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodHeadObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - SoftAPECheck: reqCtx.SoftAPECheck, + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodHeadObject, + Role: nativeSchemaRole(reqCtx.Role), + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + SoftAPECheck: reqCtx.SoftAPECheck, + WithoutHeaderRequest: true, }) if err != nil { return nil, toStatusErr(err) From e74bdaa5d5999774b5d55f006374d228ca3dd161 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 4 Apr 2024 15:42:52 +0300 Subject: [PATCH 1020/1943] [#1080] ape: Use value for APE request Signed-off-by: Dmitrii Stepanov --- pkg/services/object/ape/request.go | 28 +++++++++++++------------ pkg/services/object/ape/request_test.go | 4 ++-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index bdce97676..b81fe8c32 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -17,36 +17,38 @@ import ( type request struct { operation string - resource *resource + resource resource properties map[string]string } -var _ aperesource.Request = (*request)(nil) +var defaultRequest = request{} + +var _ aperesource.Request = request{} type resource struct { name string properties map[string]string } -var _ aperesource.Resource = (*resource)(nil) +var _ aperesource.Resource = resource{} -func (r *resource) Name() string { +func (r resource) Name() string { return r.name } -func (r *resource) Property(key string) string { +func (r resource) Property(key string) string { return r.properties[key] } -func (r *request) Operation() string { +func (r request) Operation() string { return r.operation } -func (r *request) Property(key string) string { +func (r request) Property(key string) string { return r.properties[key] } -func (r *request) Resource() aperesource.Resource { +func (r request) Resource() aperesource.Resource { return r.resource } @@ -123,7 +125,7 @@ func objectProperties(cnr cid.ID, oid *oid.ID, cnrOwner user.ID, header *objectV // newAPERequest creates an APE request to be passed to a chain router. It collects resource properties from // header provided by headerProvider. If it cannot be found in headerProvider, then properties are // initialized from header given in prm (if it is set). Otherwise, just CID and OID are set to properties. -func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (*request, error) { +func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (request, error) { switch prm.Method { case nativeschema.MethodGetObject, nativeschema.MethodHeadObject, @@ -131,11 +133,11 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (*request, err nativeschema.MethodHashObject, nativeschema.MethodDeleteObject: if prm.Object == nil { - return nil, fmt.Errorf("method %s: %w", prm.Method, errMissingOID) + return defaultRequest, fmt.Errorf("method %s: %w", prm.Method, errMissingOID) } case nativeschema.MethodSearchObject, nativeschema.MethodPutObject: default: - return nil, fmt.Errorf("unknown method: %s", prm.Method) + return defaultRequest, fmt.Errorf("unknown method: %s", prm.Method) } var header *objectV2.Header @@ -148,9 +150,9 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (*request, err } } - return &request{ + return request{ operation: prm.Method, - resource: &resource{ + resource: resource{ name: resourceName(prm.Container, prm.Object, prm.Namespace), properties: objectProperties(prm.Container, prm.Object, prm.ContainerOwner, header), }, diff --git a/pkg/services/object/ape/request_test.go b/pkg/services/object/ape/request_test.go index c5b43fa8b..71e234e78 100644 --- a/pkg/services/object/ape/request_test.go +++ b/pkg/services/object/ape/request_test.go @@ -258,7 +258,7 @@ func TestNewAPERequest(t *testing.T) { expectedRequest := request{ operation: method, - resource: &resource{ + resource: resource{ name: resourceName(cnr, obj, prm.Namespace), properties: objectProperties(cnr, obj, testCnrOwner, func() *objectV2.Header { if headerObjSDK != nil { @@ -273,7 +273,7 @@ func TestNewAPERequest(t *testing.T) { }, } - require.Equal(t, expectedRequest, *r) + require.Equal(t, expectedRequest, r) }) } }) From 7b1adfed3e7898400e715251c0e37f89bbb2c279 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 5 Apr 2024 16:42:48 +0300 Subject: [PATCH 1021/1943] [#1080] metabase: Open bucket for container counter once Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/counter.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index dc85b4697..4d860261c 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -232,14 +232,19 @@ func (db *DB) ContainerCount(ctx context.Context, id cid.ID) (ObjectCounters, er } func (db *DB) incCounters(tx *bbolt.Tx, cnrID cid.ID, isUserObject bool) error { - if err := db.updateShardObjectCounter(tx, phy, 1, true); err != nil { + b := tx.Bucket(shardInfoBucket) + if b == nil { + return db.incContainerObjectCounter(tx, cnrID, isUserObject) + } + + if err := db.updateShardObjectCounterBucket(b, phy, 1, true); err != nil { return fmt.Errorf("could not increase phy object counter: %w", err) } - if err := db.updateShardObjectCounter(tx, logical, 1, true); err != nil { + if err := db.updateShardObjectCounterBucket(b, logical, 1, true); err != nil { return fmt.Errorf("could not increase logical object counter: %w", err) } if isUserObject { - if err := db.updateShardObjectCounter(tx, user, 1, true); err != nil { + if err := db.updateShardObjectCounterBucket(b, user, 1, true); err != nil { return fmt.Errorf("could not increase user object counter: %w", err) } } @@ -252,6 +257,10 @@ func (db *DB) updateShardObjectCounter(tx *bbolt.Tx, typ objectType, delta uint6 return nil } + return db.updateShardObjectCounterBucket(b, typ, delta, inc) +} + +func (*DB) updateShardObjectCounterBucket(b *bbolt.Bucket, typ objectType, delta uint64, inc bool) error { var counter uint64 var counterKey []byte From 76398c06b00922c58363e707727ac227aa81fb13 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 9 Apr 2024 15:29:49 +0300 Subject: [PATCH 1022/1943] [#1080] metabase: Add StorageID metric Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/storage_id.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index f9767935c..6d620b41a 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -36,6 +36,14 @@ func (r StorageIDRes) StorageID() []byte { // StorageID returns storage descriptor for objects from the blobstor. // It is put together with the object can makes get/delete operation faster. func (db *DB) StorageID(ctx context.Context, prm StorageIDPrm) (res StorageIDRes, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("StorageID", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.StorageID", trace.WithAttributes( attribute.String("address", prm.addr.EncodeToString()), @@ -54,7 +62,7 @@ func (db *DB) StorageID(ctx context.Context, prm StorageIDPrm) (res StorageIDRes return err }) - + success = err == nil return res, metaerr.Wrap(err) } From 1005bf4f56d98cf09b285ee56ba1d447c508ffd8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 5 Mar 2024 13:41:15 +0300 Subject: [PATCH 1023/1943] [#1024] shard: Add refill metabase benchmark Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/refill_test.go | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 pkg/local_object_storage/shard/refill_test.go diff --git a/pkg/local_object_storage/shard/refill_test.go b/pkg/local_object_storage/shard/refill_test.go new file mode 100644 index 000000000..07ec9180c --- /dev/null +++ b/pkg/local_object_storage/shard/refill_test.go @@ -0,0 +1,72 @@ +package shard + +import ( + "context" + "os" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +func BenchmarkRefillMetabase(b *testing.B) { + b.Run("100 objects", func(b *testing.B) { + benchRefillMetabase(b, 100) + }) + + b.Run("1000 objects", func(b *testing.B) { + benchRefillMetabase(b, 1000) + }) + + b.Run("2000 objects", func(b *testing.B) { + benchRefillMetabase(b, 2000) + }) + + b.Run("5000 objects", func(b *testing.B) { + benchRefillMetabase(b, 5000) + }) +} + +func benchRefillMetabase(b *testing.B, objectsCount int) { + sh := newShard(b, false) + defer func() { require.NoError(b, sh.Close()) }() + + var putPrm PutPrm + + for i := 0; i < objectsCount/2; i++ { + obj := testutil.GenerateObject() + testutil.AddAttribute(obj, "foo", "bar") + testutil.AddPayload(obj, 1<<5) // blobvnicza tree obj + + putPrm.SetObject(obj) + + _, err := sh.Put(context.Background(), putPrm) + require.NoError(b, err) + } + + for i := 0; i < objectsCount/2; i++ { + obj := testutil.GenerateObject() + testutil.AddAttribute(obj, "foo", "bar") + obj.SetID(oidtest.ID()) + testutil.AddPayload(obj, 1<<20) // fstree obj + + putPrm.SetObject(obj) + + _, err := sh.Put(context.Background(), putPrm) + require.NoError(b, err) + } + + require.NoError(b, sh.Close()) + require.NoError(b, os.Remove(sh.metaBase.DumpInfo().Path)) + + require.NoError(b, sh.Open(context.Background())) + sh.cfg.refillMetabase = true + + b.ReportAllocs() + b.ResetTimer() + + require.NoError(b, sh.Init(context.Background())) + + require.NoError(b, sh.Close()) +} From 57466594fbef35d014479e7d96ccd17db438fef5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 5 Mar 2024 15:39:50 +0300 Subject: [PATCH 1024/1943] [#1024] shard: Resync metabase concurrently Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 11 ++- cmd/frostfs-node/config/engine/config_test.go | 2 + .../config/engine/shard/config.go | 15 ++++ config/example/node.env | 1 + config/example/node.json | 1 + config/example/node.yaml | 1 + docs/storage-node-configuration.md | 9 ++- pkg/local_object_storage/shard/control.go | 76 ++++++++++++------- pkg/local_object_storage/shard/refill_test.go | 6 +- pkg/local_object_storage/shard/shard.go | 10 ++- 10 files changed, 93 insertions(+), 39 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d78a90cfc..c5ed8b50c 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -119,10 +119,11 @@ type shardCfg struct { estimateCompressibility bool estimateCompressibilityThreshold float64 - smallSizeObjectLimit uint64 - uncompressableContentType []string - refillMetabase bool - mode shardmode.Mode + smallSizeObjectLimit uint64 + uncompressableContentType []string + refillMetabase bool + refillMetabaseWorkersCount int + mode shardmode.Mode metaCfg struct { path string @@ -234,6 +235,7 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, oldConfig var newConfig shardCfg newConfig.refillMetabase = oldConfig.RefillMetabase() + newConfig.refillMetabaseWorkersCount = oldConfig.RefillMetabaseWorkersCount() newConfig.mode = oldConfig.Mode() newConfig.compress = oldConfig.Compress() newConfig.estimateCompressibility = oldConfig.EstimateCompressibility() @@ -986,6 +988,7 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { sh.shOpts = []shard.Option{ shard.WithLogger(c.log), shard.WithRefillMetabase(shCfg.refillMetabase), + shard.WithRefillMetabaseWorkersCount(shCfg.refillMetabaseWorkersCount), shard.WithMode(shCfg.mode), shard.WithBlobStorOptions(blobstoreOpts...), shard.WithMetaBaseOptions(mbOptions...), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 86c938309..b5c926fc3 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -117,6 +117,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, false, sc.RefillMetabase()) require.Equal(t, mode.ReadOnly, sc.Mode()) + require.Equal(t, 100, sc.RefillMetabaseWorkersCount()) case 1: require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path()) require.Equal(t, fs.FileMode(0o644), pl.Perm()) @@ -168,6 +169,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, true, sc.RefillMetabase()) require.Equal(t, mode.ReadWrite, sc.Mode()) + require.Equal(t, shardconfig.RefillMetabaseWorkersCountDefault, sc.RefillMetabaseWorkersCount()) } return nil }) diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go index f9cfdf2a8..0620c9f63 100644 --- a/cmd/frostfs-node/config/engine/shard/config.go +++ b/cmd/frostfs-node/config/engine/shard/config.go @@ -18,6 +18,7 @@ const ( // SmallSizeLimitDefault is a default limit of small objects payload in bytes. SmallSizeLimitDefault = 1 << 20 EstimateCompressibilityThresholdDefault = 0.1 + RefillMetabaseWorkersCountDefault = 500 ) // From wraps config section into Config. @@ -134,6 +135,20 @@ func (x *Config) RefillMetabase() bool { ) } +// RefillMetabaseWorkersCount returns the value of "resync_metabase_worker_count" config parameter. +// +// Returns RefillMetabaseWorkersCountDefault if the value is not a positive number. +func (x *Config) RefillMetabaseWorkersCount() int { + v := config.IntSafe( + (*config.Config)(x), + "resync_metabase_worker_count", + ) + if v > 0 { + return int(v) + } + return RefillMetabaseWorkersCountDefault +} + // Mode return the value of "mode" config parameter. // // Panics if read the value is not one of predefined diff --git a/config/example/node.env b/config/example/node.env index 976c42629..9068886d3 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -98,6 +98,7 @@ FROSTFS_STORAGE_REBUILD_WORKERS_COUNT=1000 ## 0 shard ### Flag to refill Metabase from BlobStor FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE=false +FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE_WORKER_COUNT=100 ### Flag to set shard mode FROSTFS_STORAGE_SHARD_0_MODE=read-only ### Write cache config diff --git a/config/example/node.json b/config/example/node.json index 648fb77b0..bae28720b 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -144,6 +144,7 @@ "0": { "mode": "read-only", "resync_metabase": false, + "resync_metabase_worker_count": 100, "writecache": { "enabled": false, "no_sync": true, diff --git a/config/example/node.yaml b/config/example/node.yaml index 2dcf7c4d9..7a824adbe 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -167,6 +167,7 @@ storage: # degraded-read-only # disabled (do not work with the shard, allows to not remove it from the config) resync_metabase: false # sync metabase with blobstor on start, expensive, leave false until complete understanding + resync_metabase_worker_count: 100 writecache: enabled: false diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 02ead3020..aa3a17815 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -179,14 +179,15 @@ Contains configuration for each shard. Keys must be consecutive numbers starting `default` subsection has the same format and specifies defaults for missing values. The following table describes configuration for each shard. -| Parameter | Type | Default value | Description | -|-------------------------------------|---------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Parameter | Type | Default value | Description | +| ------------------------------------------------ | ------------------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `compress` | `bool` | `false` | Flag to enable compression. | | `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). | -| `compression_estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. | -| `compression_estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. | +| `compression_estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. | +| `compression_estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. | | `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` | | `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. | +| `resync_metabase_worker_count` | `int` | `1000` | Count of concurrent workers to resync metabase. | | `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. | | `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. | | `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 774fe8045..d080e2919 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -15,6 +15,7 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) func (s *Shard) handleMetabaseFailure(stage string, err error) error { @@ -185,39 +186,56 @@ func (s *Shard) refillMetabase(ctx context.Context) error { return fmt.Errorf("could not reset metabase: %w", err) } - obj := objectSDK.New() + eg, egCtx := errgroup.WithContext(ctx) + if s.cfg.refillMetabaseWorkersCount > 0 { + eg.SetLimit(s.cfg.refillMetabaseWorkersCount) + } - err = blobstor.IterateBinaryObjects(ctx, s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { - if err := obj.Unmarshal(data); err != nil { - s.log.Warn(logs.ShardCouldNotUnmarshalObject, - zap.Stringer("address", addr), - zap.String("err", err.Error())) + itErr := blobstor.IterateBinaryObjects(egCtx, s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { + eg.Go(func() error { + obj := objectSDK.New() + if err := obj.Unmarshal(data); err != nil { + s.log.Warn(logs.ShardCouldNotUnmarshalObject, + zap.Stringer("address", addr), + zap.String("err", err.Error())) + return nil + } + + var err error + switch obj.Type() { + case objectSDK.TypeTombstone: + err = s.refillTombstoneObject(egCtx, obj) + case objectSDK.TypeLock: + err = s.refillLockObject(egCtx, obj) + default: + } + if err != nil { + return err + } + + var mPrm meta.PutPrm + mPrm.SetObject(obj) + mPrm.SetStorageID(descriptor) + + _, err = s.metaBase.Put(egCtx, mPrm) + if err != nil && !client.IsErrObjectAlreadyRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) { + return err + } + + return nil + }) + + select { + case <-egCtx.Done(): + return egCtx.Err() + default: return nil } - - var err error - switch obj.Type() { - case objectSDK.TypeTombstone: - err = s.refillTombstoneObject(ctx, obj) - case objectSDK.TypeLock: - err = s.refillLockObject(ctx, obj) - default: - } - if err != nil { - return err - } - - var mPrm meta.PutPrm - mPrm.SetObject(obj) - mPrm.SetStorageID(descriptor) - - _, err = s.metaBase.Put(ctx, mPrm) - if err != nil && !client.IsErrObjectAlreadyRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) { - return err - } - - return nil }) + + egErr := eg.Wait() + + err = errors.Join(egErr, itErr) if err != nil { return fmt.Errorf("could not put objects to the meta: %w", err) } diff --git a/pkg/local_object_storage/shard/refill_test.go b/pkg/local_object_storage/shard/refill_test.go index 07ec9180c..509ccaaa6 100644 --- a/pkg/local_object_storage/shard/refill_test.go +++ b/pkg/local_object_storage/shard/refill_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" + shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -29,7 +30,10 @@ func BenchmarkRefillMetabase(b *testing.B) { } func benchRefillMetabase(b *testing.B, objectsCount int) { - sh := newShard(b, false) + sh := newCustomShard(b, false, shardOptions{ + additionalShardOptions: []Option{WithRefillMetabaseWorkersCount(shardconfig.RefillMetabaseWorkersCountDefault)}, + }) + defer func() { require.NoError(b, sh.Close()) }() var putPrm PutPrm diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 8368f6db4..81d854373 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -95,7 +95,8 @@ type MetricsWriter interface { type cfg struct { m sync.RWMutex - refillMetabase bool + refillMetabase bool + refillMetabaseWorkersCount int rmBatchSize int @@ -300,6 +301,13 @@ func WithRefillMetabase(v bool) Option { } } +// WithRefillMetabaseWorkersCount returns option to set count of workers to refill the Metabase on Shard's initialization step. +func WithRefillMetabaseWorkersCount(v int) Option { + return func(c *cfg) { + c.refillMetabaseWorkersCount = v + } +} + // WithMode returns option to set shard's mode. Mode must be one of the predefined: // - mode.ReadWrite; // - mode.ReadOnly. From e3d9dd6ee8134107926cf4cf58d41899b1eebf96 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 5 Mar 2024 16:08:49 +0300 Subject: [PATCH 1025/1943] [#1024] blobovnicza: Copy data on iterate DB value is only valid while the tx is alive. But handler may to run something in other goroutine. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobovnicza/iterate.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index 32b0ccea7..b1cb91920 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "bytes" "context" "fmt" "math" @@ -158,7 +159,7 @@ func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes, } if !prm.withoutData { - elem.data = v + elem.data = bytes.Clone(v) } return prm.handler(elem) From 1b17258c042d3b1f1b4d2bdb530cbb5cd1560139 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 12 Mar 2024 17:36:26 +0300 Subject: [PATCH 1026/1943] [#1029] metabase: Add refill metrics Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 2 + .../blobovnicza/control.go | 4 + .../blobstor/blobovniczatree/count.go | 38 ++++++++ .../blobstor/blobovniczatree/metrics.go | 2 + .../blobstor/common/storage.go | 1 + .../blobstor/fstree/fstree.go | 39 ++++++++ .../blobstor/fstree/metrics.go | 2 + pkg/local_object_storage/blobstor/info.go | 45 +++++++++ .../blobstor/memstore/memstore.go | 7 ++ pkg/local_object_storage/blobstor/metrics.go | 2 + .../blobstor/teststore/teststore.go | 7 ++ pkg/local_object_storage/engine/metrics.go | 4 + pkg/local_object_storage/engine/shards.go | 12 +++ .../metrics/blobovnicza.go | 4 + pkg/local_object_storage/metrics/blobstore.go | 4 + pkg/local_object_storage/metrics/fstree.go | 4 + pkg/local_object_storage/shard/control.go | 94 ++++++++++++++----- .../shard/control_test.go | 16 +++- .../shard/metrics_test.go | 60 +++++++++--- pkg/local_object_storage/shard/reload_test.go | 1 + pkg/local_object_storage/shard/shard.go | 6 ++ pkg/metrics/engine.go | 53 ++++++++++- pkg/metrics/mode.go | 6 ++ 23 files changed, 366 insertions(+), 47 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/count.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 64e5f771e..d56d425ce 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -579,6 +579,8 @@ const ( EngineShardsEvacuationFailedToMoveTree = "failed to evacuate tree to other node" EngineShardsEvacuationTreeEvacuatedLocal = "tree evacuated to local node" EngineShardsEvacuationTreeEvacuatedRemote = "tree evacuated to other node" + EngineRefillFailedToGetObjectsCount = "failed to get blobstor objects count, no resync percent estimation is available" + BlobstoreFailedToGetFileinfo = "failed to get file info" ECFailedToSendToContainerNode = "failed to send EC object to container node" ECFailedToSaveECPart = "failed to save EC part" ) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index dab301647..aeaa4e1d5 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -101,6 +101,10 @@ func (b *Blobovnicza) Init() error { return b.initializeCounters() } +func (b *Blobovnicza) ObjectsCount() uint64 { + return b.itemsCount.Load() +} + func (b *Blobovnicza) initializeCounters() error { var size uint64 var items uint64 diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/count.go b/pkg/local_object_storage/blobstor/blobovniczatree/count.go new file mode 100644 index 000000000..cf91637d7 --- /dev/null +++ b/pkg/local_object_storage/blobstor/blobovniczatree/count.go @@ -0,0 +1,38 @@ +package blobovniczatree + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" +) + +func (b *Blobovniczas) ObjectsCount(ctx context.Context) (uint64, error) { + var ( + success bool + startedAt = time.Now() + ) + defer func() { + b.metrics.ObjectsCount(time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.ObjectsCount") + defer span.End() + + var result uint64 + err := b.iterateExistingDBPaths(ctx, func(p string) (bool, error) { + shDB := b.getBlobovniczaWithoutCaching(p) + blz, err := shDB.Open() + if err != nil { + return true, err + } + defer shDB.Close() + + result += blz.ObjectsCount() + return false, nil + }) + if err != nil { + return 0, err + } + return result, nil +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index ed05f8772..28289c19e 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -24,6 +24,7 @@ type Metrics interface { SetRebuildStatus(status string) ObjectMoved(d time.Duration) SetRebuildPercent(value uint32) + ObjectsCount(d time.Duration, success bool) Delete(d time.Duration, success, withStorageID bool) Exists(d time.Duration, success, withStorageID bool) @@ -47,6 +48,7 @@ func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} func (m *noopMetrics) Iterate(time.Duration, bool) {} func (m *noopMetrics) Put(time.Duration, int, bool) {} +func (m *noopMetrics) ObjectsCount(time.Duration, bool) {} func (m *noopMetrics) Blobovnicza() blobovnicza.Metrics { return &blobovnicza.NoopMetrics{} } diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index e552fafea..8f629b1de 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -15,6 +15,7 @@ type Storage interface { Type() string Path() string + ObjectsCount(ctx context.Context) (uint64, error) SetCompressor(cc *compression.Config) Compressor() *compression.Config diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 420f341a4..7a064af6b 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -467,6 +467,45 @@ func (t *FSTree) countFiles() (uint64, error) { return counter, nil } +func (t *FSTree) ObjectsCount(ctx context.Context) (uint64, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.ObjectsCount(time.Since(startedAt), success) + }() + + ctx, span := tracing.StartSpanFromContext(ctx, "FSTree.ObjectsCount", + trace.WithAttributes( + attribute.String("path", t.RootPath), + )) + defer span.End() + + var result uint64 + + err := filepath.WalkDir(t.RootPath, + func(_ string, d fs.DirEntry, _ error) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if !d.IsDir() { + result++ + } + + return nil + }, + ) + if err != nil { + return 0, fmt.Errorf("could not walk through %s directory: %w", t.RootPath, err) + } + success = true + return result, nil +} + // Type is fstree storage type used in logs and configuration. const Type = "fstree" diff --git a/pkg/local_object_storage/blobstor/fstree/metrics.go b/pkg/local_object_storage/blobstor/fstree/metrics.go index ca6a54975..eeb686672 100644 --- a/pkg/local_object_storage/blobstor/fstree/metrics.go +++ b/pkg/local_object_storage/blobstor/fstree/metrics.go @@ -14,6 +14,7 @@ type Metrics interface { Put(d time.Duration, size int, success bool) Get(d time.Duration, size int, success bool) GetRange(d time.Duration, size int, success bool) + ObjectsCount(d time.Duration, success bool) } type noopMetrics struct{} @@ -27,3 +28,4 @@ func (m *noopMetrics) Exists(time.Duration, bool) {} func (m *noopMetrics) Put(time.Duration, int, bool) {} func (m *noopMetrics) Get(time.Duration, int, bool) {} func (m *noopMetrics) GetRange(time.Duration, int, bool) {} +func (m *noopMetrics) ObjectsCount(time.Duration, bool) {} diff --git a/pkg/local_object_storage/blobstor/info.go b/pkg/local_object_storage/blobstor/info.go index 2fd62af81..8a5bb870a 100644 --- a/pkg/local_object_storage/blobstor/info.go +++ b/pkg/local_object_storage/blobstor/info.go @@ -1,5 +1,14 @@ package blobstor +import ( + "context" + "sync/atomic" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "golang.org/x/sync/errgroup" +) + // DumpInfo returns information about blob stor. func (b *BlobStor) DumpInfo() Info { b.modeMtx.RLock() @@ -15,3 +24,39 @@ func (b *BlobStor) DumpInfo() Info { SubStorages: sub, } } + +// ObjectsCount returns Blobstore's total objects count. +func (b *BlobStor) ObjectsCount(ctx context.Context) (uint64, error) { + var err error + startedAt := time.Now() + defer func() { + b.metrics.ObjectsCount(time.Since(startedAt), err == nil) + }() + + ctx, span := tracing.StartSpanFromContext(ctx, "BlobStor.ObjectsCount") + defer span.End() + + b.modeMtx.RLock() + defer b.modeMtx.RUnlock() + + var result atomic.Uint64 + + eg, egCtx := errgroup.WithContext(ctx) + for i := range b.storage { + i := i + eg.Go(func() error { + v, e := b.storage[i].Storage.ObjectsCount(egCtx) + if e != nil { + return e + } + result.Add(v) + return nil + }) + } + + if err = eg.Wait(); err != nil { + return 0, err + } + + return result.Load(), nil +} diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index 39bed00bf..0252c7983 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -163,3 +163,10 @@ func (s *memstoreImpl) Iterate(_ context.Context, req common.IteratePrm) (common func (s *memstoreImpl) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { return common.RebuildRes{}, nil } + +func (s *memstoreImpl) ObjectsCount(_ context.Context) (uint64, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + return uint64(len(s.objs)), nil +} diff --git a/pkg/local_object_storage/blobstor/metrics.go b/pkg/local_object_storage/blobstor/metrics.go index 4a7b40092..aadc237af 100644 --- a/pkg/local_object_storage/blobstor/metrics.go +++ b/pkg/local_object_storage/blobstor/metrics.go @@ -13,6 +13,7 @@ type Metrics interface { Get(d time.Duration, size int, success, withStorageID bool) Iterate(d time.Duration, success bool) Put(d time.Duration, size int, success bool) + ObjectsCount(d time.Duration, success bool) } type noopMetrics struct{} @@ -26,3 +27,4 @@ func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {} func (m *noopMetrics) Get(time.Duration, int, bool, bool) {} func (m *noopMetrics) Iterate(time.Duration, bool) {} func (m *noopMetrics) Put(time.Duration, int, bool) {} +func (m *noopMetrics) ObjectsCount(time.Duration, bool) {} diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index c0cdfacf8..016fd520f 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -233,3 +233,10 @@ func (s *TestStore) SetParentID(string) {} func (s *TestStore) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) { return common.RebuildRes{}, nil } + +func (s *TestStore) ObjectsCount(ctx context.Context) (uint64, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.st.ObjectsCount(ctx) +} diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 91dfa8762..6b99dff7e 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -27,6 +27,10 @@ type MetricRegister interface { IncContainerObjectCounter(shardID, contID, objectType string) SubContainerObjectCounter(shardID, contID, objectType string, v uint64) + IncRefillObjectsCount(shardID, path string, size int, success bool) + SetRefillPercent(shardID, path string, percent uint32) + SetRefillStatus(shardID, path, status string) + WriteCache() metrics.WriteCacheMetrics GC() metrics.GCMetrics } diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index ccf2e9dac..54d3e7d65 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -85,6 +85,18 @@ func (m *metricsWithID) SubContainerObjectsCount(cnrID string, objectType string m.mw.SubContainerObjectCounter(m.id, cnrID, objectType, value) } +func (m *metricsWithID) IncRefillObjectsCount(path string, size int, success bool) { + m.mw.IncRefillObjectsCount(m.id, path, size, success) +} + +func (m *metricsWithID) SetRefillPercent(path string, percent uint32) { + m.mw.SetRefillPercent(m.id, path, percent) +} + +func (m *metricsWithID) SetRefillStatus(path string, status string) { + m.mw.SetRefillStatus(m.id, path, status) +} + // AddShard adds a new shard to the storage engine. // // Returns any error encountered that did not allow adding a shard. diff --git a/pkg/local_object_storage/metrics/blobovnicza.go b/pkg/local_object_storage/metrics/blobovnicza.go index 3a92a7bc7..1e294efa5 100644 --- a/pkg/local_object_storage/metrics/blobovnicza.go +++ b/pkg/local_object_storage/metrics/blobovnicza.go @@ -87,6 +87,10 @@ func (m *blobovniczaTreeMetrics) Put(d time.Duration, size int, success bool) { } } +func (m *blobovniczaTreeMetrics) ObjectsCount(d time.Duration, success bool) { + m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "ObjectsCount", d, success, metrics_impl.NullBool{}) +} + type blobovniczaMetrics struct { m metrics_impl.BlobobvnizcaMetrics shardID func() string diff --git a/pkg/local_object_storage/metrics/blobstore.go b/pkg/local_object_storage/metrics/blobstore.go index 48249e89c..b3871dfca 100644 --- a/pkg/local_object_storage/metrics/blobstore.go +++ b/pkg/local_object_storage/metrics/blobstore.go @@ -63,3 +63,7 @@ func (m *blobstoreMetrics) Put(d time.Duration, size int, success bool) { m.m.AddPut(m.shardID, size) } } + +func (m *blobstoreMetrics) ObjectsCount(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, "ObjectsCount", d, success, metrics_impl.NullBool{}) +} diff --git a/pkg/local_object_storage/metrics/fstree.go b/pkg/local_object_storage/metrics/fstree.go index 0def3210c..e035b3a46 100644 --- a/pkg/local_object_storage/metrics/fstree.go +++ b/pkg/local_object_storage/metrics/fstree.go @@ -65,3 +65,7 @@ func (m *fstreeMetrics) GetRange(d time.Duration, size int, success bool) { m.m.AddGet(m.shardID, m.path, size) } } + +func (m *fstreeMetrics) ObjectsCount(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "ObjectsCount", d, success) +} diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index d080e2919..6712822a0 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -181,47 +182,54 @@ func (s *Shard) Init(ctx context.Context) error { } func (s *Shard) refillMetabase(ctx context.Context) error { + path := s.metaBase.DumpInfo().Path + s.metricsWriter.SetRefillStatus(path, "running") + s.metricsWriter.SetRefillPercent(path, 0) + var success bool + defer func() { + if success { + s.metricsWriter.SetRefillStatus(path, "completed") + } else { + s.metricsWriter.SetRefillStatus(path, "failed") + } + }() + err := s.metaBase.Reset() if err != nil { return fmt.Errorf("could not reset metabase: %w", err) } + withCount := true + totalObjects, err := s.blobStor.ObjectsCount(ctx) + if err != nil { + s.log.Warn(logs.EngineRefillFailedToGetObjectsCount, zap.Error(err)) + withCount = false + } + eg, egCtx := errgroup.WithContext(ctx) if s.cfg.refillMetabaseWorkersCount > 0 { eg.SetLimit(s.cfg.refillMetabaseWorkersCount) } + var completedCount uint64 + var metricGuard sync.Mutex itErr := blobstor.IterateBinaryObjects(egCtx, s.blobStor, func(addr oid.Address, data []byte, descriptor []byte) error { eg.Go(func() error { - obj := objectSDK.New() - if err := obj.Unmarshal(data); err != nil { - s.log.Warn(logs.ShardCouldNotUnmarshalObject, - zap.Stringer("address", addr), - zap.String("err", err.Error())) - return nil - } + var success bool + defer func() { + s.metricsWriter.IncRefillObjectsCount(path, len(data), success) + if withCount { + metricGuard.Lock() + completedCount++ + s.metricsWriter.SetRefillPercent(path, uint32(completedCount*100/totalObjects)) + metricGuard.Unlock() + } + }() - var err error - switch obj.Type() { - case objectSDK.TypeTombstone: - err = s.refillTombstoneObject(egCtx, obj) - case objectSDK.TypeLock: - err = s.refillLockObject(egCtx, obj) - default: - } - if err != nil { + if err := s.refillObject(egCtx, data, addr, descriptor); err != nil { return err } - - var mPrm meta.PutPrm - mPrm.SetObject(obj) - mPrm.SetStorageID(descriptor) - - _, err = s.metaBase.Put(egCtx, mPrm) - if err != nil && !client.IsErrObjectAlreadyRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) { - return err - } - + success = true return nil }) @@ -245,6 +253,40 @@ func (s *Shard) refillMetabase(ctx context.Context) error { return fmt.Errorf("could not sync object counters: %w", err) } + success = true + s.metricsWriter.SetRefillPercent(path, 100) + return nil +} + +func (s *Shard) refillObject(ctx context.Context, data []byte, addr oid.Address, descriptor []byte) error { + obj := objectSDK.New() + if err := obj.Unmarshal(data); err != nil { + s.log.Warn(logs.ShardCouldNotUnmarshalObject, + zap.Stringer("address", addr), + zap.String("err", err.Error())) + return nil + } + + var err error + switch obj.Type() { + case objectSDK.TypeTombstone: + err = s.refillTombstoneObject(ctx, obj) + case objectSDK.TypeLock: + err = s.refillLockObject(ctx, obj) + default: + } + if err != nil { + return err + } + + var mPrm meta.PutPrm + mPrm.SetObject(obj) + mPrm.SetStorageID(descriptor) + + _, err = s.metaBase.Put(ctx, mPrm) + if err != nil && !client.IsErrObjectAlreadyRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) { + return err + } return nil } diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index d08747e13..44fee1636 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -126,11 +126,15 @@ func TestRefillMetabaseCorrupted(t *testing.T) { }), } + mm := NewMetricStore() + sh := New( WithID(NewIDFromBytes([]byte{})), WithBlobStorOptions(blobOpts...), WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))), - WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta")), meta.WithEpochState(epochState{}))) + WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta")), meta.WithEpochState(epochState{})), + WithMetricsWriter(mm), + ) require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) @@ -157,7 +161,8 @@ func TestRefillMetabaseCorrupted(t *testing.T) { WithBlobStorOptions(blobOpts...), WithPiloramaOptions(pilorama.WithPath(filepath.Join(dir, "pilorama"))), WithMetaBaseOptions(meta.WithPath(filepath.Join(dir, "meta_new")), meta.WithEpochState(epochState{})), - WithRefillMetabase(true)) + WithRefillMetabase(true), + WithMetricsWriter(mm)) require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) @@ -185,6 +190,8 @@ func TestRefillMetabase(t *testing.T) { }), } + mm := NewMetricStore() + sh := New( WithID(NewIDFromBytes([]byte{})), WithBlobStorOptions(blobOpts...), @@ -194,6 +201,7 @@ func TestRefillMetabase(t *testing.T) { ), WithPiloramaOptions( pilorama.WithPath(filepath.Join(p, "pilorama"))), + WithMetricsWriter(mm), ) // open Blobstor @@ -362,6 +370,7 @@ func TestRefillMetabase(t *testing.T) { ), WithPiloramaOptions( pilorama.WithPath(filepath.Join(p, "pilorama_another"))), + WithMetricsWriter(mm), ) // open Blobstor @@ -389,4 +398,7 @@ func TestRefillMetabase(t *testing.T) { checkObj(object.AddressOf(tombObj), tombObj) checkTombMembers(true) checkLocked(t, cnrLocked, locked) + require.Equal(t, int64(len(mObjs)+2), mm.refillCount) // 1 lock + 1 tomb + require.Equal(t, "completed", mm.refillStatus) + require.Equal(t, uint32(100), mm.refillPercent) } diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 2ab99eed3..38d465f31 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -21,13 +21,28 @@ import ( ) type metricsStore struct { - mtx sync.Mutex - objCounters map[string]uint64 - cnrSize map[string]int64 - cnrCount map[string]uint64 - pldSize int64 - mode mode.Mode - errCounter int64 + mtx sync.Mutex + objCounters map[string]uint64 + cnrSize map[string]int64 + cnrCount map[string]uint64 + pldSize int64 + mode mode.Mode + errCounter int64 + refillCount int64 + refillSize int64 + refillPercent uint32 + refillStatus string +} + +func NewMetricStore() *metricsStore { + return &metricsStore{ + objCounters: map[string]uint64{ + "phy": 0, + "logic": 0, + }, + cnrSize: make(map[string]int64), + cnrCount: make(map[string]uint64), + } } func (m *metricsStore) SetShardID(_ string) {} @@ -155,6 +170,28 @@ func (m *metricsStore) getContainerCount(cnrID, objectType string) (uint64, bool return v, ok } +func (m *metricsStore) IncRefillObjectsCount(_ string, size int, success bool) { + m.mtx.Lock() + defer m.mtx.Unlock() + + m.refillCount++ + m.refillSize += int64(size) +} + +func (m *metricsStore) SetRefillPercent(_ string, percent uint32) { + m.mtx.Lock() + defer m.mtx.Unlock() + + m.refillPercent = percent +} + +func (m *metricsStore) SetRefillStatus(_ string, status string) { + m.mtx.Lock() + defer m.mtx.Unlock() + + m.refillStatus = status +} + func TestCounters(t *testing.T) { t.Parallel() @@ -361,14 +398,7 @@ func shardWithMetrics(t *testing.T, path string) (*Shard, *metricsStore) { }), } - mm := &metricsStore{ - objCounters: map[string]uint64{ - "phy": 0, - "logic": 0, - }, - cnrSize: make(map[string]int64), - cnrCount: make(map[string]uint64), - } + mm := NewMetricStore() sh := New( WithID(NewIDFromBytes([]byte{})), diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 511ce0721..b5ea2fec7 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -51,6 +51,7 @@ func TestShardReload(t *testing.T) { WithMetaBaseOptions(metaOpts...), WithPiloramaOptions( pilorama.WithPath(filepath.Join(p, "pilorama"))), + WithMetricsWriter(NewMetricStore()), } sh := New(opts...) diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 81d854373..d9cd2b2f4 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -90,6 +90,12 @@ type MetricsWriter interface { IncContainerObjectsCount(cnrID string, objectType string) // SubContainerObjectsCount subtracts container object count. SubContainerObjectsCount(cnrID string, objectType string, value uint64) + // IncRefillObjectsCount increments refill objects count. + IncRefillObjectsCount(path string, size int, success bool) + // SetRefillPercent sets refill percent. + SetRefillPercent(path string, percent uint32) + // SetRefillStatus sets refill status. + SetRefillStatus(path string, status string) } type cfg struct { diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 4cc542470..e37777e40 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -1,6 +1,7 @@ package metrics import ( + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -23,6 +24,9 @@ type EngineMetrics interface { SetContainerObjectCounter(shardID, contID, objectType string, v uint64) IncContainerObjectCounter(shardID, contID, objectType string) SubContainerObjectCounter(shardID, contID, objectType string, v uint64) + IncRefillObjectsCount(shardID, path string, size int, success bool) + SetRefillPercent(shardID, path string, percent uint32) + SetRefillStatus(shardID, path, status string) WriteCache() WriteCacheMetrics GC() GCMetrics @@ -37,6 +41,11 @@ type engineMetrics struct { mode *shardIDModeValue contObjCounter *prometheus.GaugeVec + refillStatus *shardIDPathModeValue + refillObjCounter *prometheus.GaugeVec + refillPayloadCounter *prometheus.GaugeVec + refillPercentCounter *prometheus.GaugeVec + gc *gcMetrics writeCache *writeCacheMetrics } @@ -55,10 +64,14 @@ func newEngineMetrics() *engineMetrics { objectCounter: newEngineGaugeVector("objects_total", "Objects counters per shards. DEPRECATED: Will be deleted in next releasese, use frostfs_node_engine_container_objects_total metric.", []string{shardIDLabel, typeLabel}), - gc: newGCMetrics(), - writeCache: newWriteCacheMetrics(), - mode: newShardIDMode(engineSubsystem, "mode_info", "Shard mode"), - contObjCounter: newEngineGaugeVector("container_objects_total", "Count of objects for each container", []string{shardIDLabel, containerIDLabelKey, typeLabel}), + gc: newGCMetrics(), + writeCache: newWriteCacheMetrics(), + mode: newShardIDMode(engineSubsystem, "mode_info", "Shard mode"), + contObjCounter: newEngineGaugeVector("container_objects_total", "Count of objects for each container", []string{shardIDLabel, containerIDLabelKey, typeLabel}), + refillStatus: newShardIDPathMode(engineSubsystem, "resync_metabase_status", "Resync from blobstore to metabase status"), + refillObjCounter: newEngineGaugeVector("resync_metabase_objects_total", "Count of objects resynced from blobstore to metabase", []string{shardIDLabel, pathLabel, successLabel}), + refillPayloadCounter: newEngineGaugeVector("resync_metabase_objects_size_bytes", "Size of objects resynced from blobstore to metabase", []string{shardIDLabel, pathLabel, successLabel}), + refillPercentCounter: newEngineGaugeVector("resync_metabase_complete_percent", "Percent of resynced from blobstore to metabase completeness", []string{shardIDLabel, pathLabel}), } } @@ -106,7 +119,11 @@ func (m *engineMetrics) DeleteShardMetrics(shardID string) { m.payloadSize.Delete(prometheus.Labels{shardIDLabel: shardID}) m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) m.contObjCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.refillObjCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.refillPayloadCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.refillPercentCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) m.mode.Delete(shardID) + m.refillStatus.DeleteByShardID(shardID) } func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) { @@ -168,3 +185,31 @@ func (m *engineMetrics) WriteCache() WriteCacheMetrics { func (m *engineMetrics) GC() GCMetrics { return m.gc } + +func (m *engineMetrics) IncRefillObjectsCount(shardID, path string, size int, success bool) { + m.refillObjCounter.With( + prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + }, + ).Inc() + m.refillPayloadCounter.With( + prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + }, + ).Add(float64(size)) +} + +func (m *engineMetrics) SetRefillPercent(shardID, path string, percent uint32) { + m.refillPercentCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Set(float64(percent)) +} + +func (m *engineMetrics) SetRefillStatus(shardID, path, status string) { + m.refillStatus.SetMode(shardID, path, status) +} diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go index 312a6b33d..cf6bdac48 100644 --- a/pkg/metrics/mode.go +++ b/pkg/metrics/mode.go @@ -74,6 +74,12 @@ func (m *shardIDPathModeValue) Delete(shardID, path string) { }) } +func (m *shardIDPathModeValue) DeleteByShardID(shardID string) { + m.modeValue.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + }) +} + func modeFromBool(readOnly bool) string { modeValue := readWriteMode if readOnly { From 43fdb1da453cee28a5b6726d1d8bbc15708fe714 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 Apr 2024 14:15:42 +0300 Subject: [PATCH 1027/1943] Reapply "[#972] go.mod: Bump go version to go1.21" This reverts commit 9adcb253be772bfc4f716307c564ac1f0545f85c. Signed-off-by: Evgenii Stratonikov --- .docker/Dockerfile.adm | 2 +- .docker/Dockerfile.ci | 2 +- .docker/Dockerfile.cli | 2 +- .docker/Dockerfile.ir | 2 +- .docker/Dockerfile.storage | 2 +- .forgejo/workflows/build.yml | 2 +- .forgejo/workflows/dco.yml | 2 +- .forgejo/workflows/tests.yml | 6 +++--- .forgejo/workflows/vulncheck.yml | 2 +- Makefile | 2 +- README.md | 2 +- go.mod | 2 +- 12 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.docker/Dockerfile.adm b/.docker/Dockerfile.adm index eeccaab79..b3dad06d3 100644 --- a/.docker/Dockerfile.adm +++ b/.docker/Dockerfile.adm @@ -1,4 +1,4 @@ -FROM golang:1.21 as builder +FROM golang:1.22 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ci b/.docker/Dockerfile.ci index ef6586a64..e9077c831 100644 --- a/.docker/Dockerfile.ci +++ b/.docker/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM golang:1.21 +FROM golang:1.22 WORKDIR /tmp diff --git a/.docker/Dockerfile.cli b/.docker/Dockerfile.cli index 0dd4cebcf..5adedc140 100644 --- a/.docker/Dockerfile.cli +++ b/.docker/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM golang:1.21 as builder +FROM golang:1.22 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ir b/.docker/Dockerfile.ir index 4015df673..25025bb2f 100644 --- a/.docker/Dockerfile.ir +++ b/.docker/Dockerfile.ir @@ -1,4 +1,4 @@ -FROM golang:1.21 as builder +FROM golang:1.22 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage b/.docker/Dockerfile.storage index ced6ea538..a16005516 100644 --- a/.docker/Dockerfile.storage +++ b/.docker/Dockerfile.storage @@ -1,4 +1,4 @@ -FROM golang:1.21 as builder +FROM golang:1.22 as builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 3c74d9434..86943fe88 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.20', '1.21' ] + go_versions: [ '1.21', '1.22' ] steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 9aa0d3106..7c5af8410 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' - name: Run commit format checker uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 449af3f51..b0c9adbf2 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -11,7 +11,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' cache: true - name: Install linters @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.20', '1.21' ] + go_versions: [ '1.21', '1.22' ] fail-fast: false steps: - uses: actions/checkout@v3 @@ -63,7 +63,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' cache: true - name: Install staticcheck diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 8ea01749b..3af564c4b 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/Makefile b/Makefile index fbf93facd..710c12703 100755 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 HUB_IMAGE ?= truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" -GO_VERSION ?= 1.21 +GO_VERSION ?= 1.22 LINT_VERSION ?= 1.56.1 TRUECLOUDLAB_LINT_VERSION ?= 0.0.5 PROTOC_VERSION ?= 25.0 diff --git a/README.md b/README.md index ecd162a85..413010372 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The latest version of frostfs-node works with frostfs-contract # Building -To make all binaries you need Go 1.20+ and `make`: +To make all binaries you need Go 1.21+ and `make`: ``` make all ``` diff --git a/go.mod b/go.mod index f940b306a..b99799337 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-node -go 1.20 +go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 From d864945961e8d773e9123ce4faafaa33c3a7ee57 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 Apr 2024 14:15:48 +0300 Subject: [PATCH 1028/1943] Reapply "[#972] pilorama: Remove removeDuplicatesInPlace()" This reverts commit 9f68305c2e88b1aea294ac3e75b75038151534e1. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/batch.go | 15 +--- .../pilorama/batch_test.go | 70 ------------------- 2 files changed, 2 insertions(+), 83 deletions(-) delete mode 100644 pkg/local_object_storage/pilorama/batch_test.go diff --git a/pkg/local_object_storage/pilorama/batch.go b/pkg/local_object_storage/pilorama/batch.go index c65488b74..520c6dfb4 100644 --- a/pkg/local_object_storage/pilorama/batch.go +++ b/pkg/local_object_storage/pilorama/batch.go @@ -2,6 +2,7 @@ package pilorama import ( "encoding/binary" + "slices" "sort" "sync" "time" @@ -50,7 +51,7 @@ func (b *batch) run() { sort.Slice(b.operations, func(i, j int) bool { return b.operations[i].Time < b.operations[j].Time }) - b.operations = removeDuplicatesInPlace(b.operations) + b.operations = slices.CompactFunc(b.operations, func(x, y *Move) bool { return x.Time == y.Time }) // Our main use-case is addition of new items. In this case, // we do not need to perform undo()/redo(), just do(). @@ -115,15 +116,3 @@ func (b *batch) run() { b.results[i] <- err } } - -func removeDuplicatesInPlace(a []*Move) []*Move { - equalCount := 0 - for i := 1; i < len(a); i++ { - if a[i].Time == a[i-1].Time { - equalCount++ - } else { - a[i-equalCount] = a[i] - } - } - return a[:len(a)-equalCount] -} diff --git a/pkg/local_object_storage/pilorama/batch_test.go b/pkg/local_object_storage/pilorama/batch_test.go deleted file mode 100644 index 931fce18c..000000000 --- a/pkg/local_object_storage/pilorama/batch_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package pilorama - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func Test_removeDuplicatesInPlace(t *testing.T) { - testCases := []struct { - before []int - after []int - }{ - { - before: []int{}, - after: []int{}, - }, - { - before: []int{1}, - after: []int{1}, - }, - { - before: []int{1, 2}, - after: []int{1, 2}, - }, - { - before: []int{1, 2, 3}, - after: []int{1, 2, 3}, - }, - { - before: []int{1, 1, 2}, - after: []int{1, 2}, - }, - { - before: []int{1, 2, 2}, - after: []int{1, 2}, - }, - { - before: []int{1, 2, 2, 3}, - after: []int{1, 2, 3}, - }, - { - before: []int{1, 1, 1}, - after: []int{1}, - }, - { - before: []int{1, 1, 2, 2}, - after: []int{1, 2}, - }, - { - before: []int{1, 1, 1, 2, 3, 3, 3}, - after: []int{1, 2, 3}, - }, - } - - for _, tc := range testCases { - ops := make([]*Move, len(tc.before)) - for i := range ops { - ops[i] = &Move{Meta: Meta{Time: Timestamp(tc.before[i])}} - } - - expected := make([]*Move, len(tc.after)) - for i := range expected { - expected[i] = &Move{Meta: Meta{Time: Timestamp(tc.after[i])}} - } - - actual := removeDuplicatesInPlace(ops) - require.Equal(t, expected, actual, "%d", tc.before) - } -} From 3dc81cb4fcfb8080259115a4649afcec61810630 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 Apr 2024 14:16:18 +0300 Subject: [PATCH 1029/1943] Reapply "[#972] Use min/max builtins" This reverts commit dad56d2e98179904a9b263103ad04794202a7c1f. Signed-off-by: Evgenii Stratonikov --- .../modules/morph/initialize/initialize_register.go | 5 +---- pkg/innerring/processors/netmap/cleanup_table.go | 5 +---- pkg/local_object_storage/blobovnicza/iterate.go | 8 -------- pkg/local_object_storage/engine/list.go | 5 +---- pkg/local_object_storage/shard/gc.go | 12 ++---------- pkg/morph/client/notary.go | 6 +----- pkg/services/object/internal/client/client.go | 5 +---- pkg/services/object/transport_splitter.go | 5 +---- pkg/services/policer/policer_test.go | 5 +---- pkg/services/tree/sync.go | 8 ++------ 10 files changed, 11 insertions(+), 53 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go index 6a0a88016..4c6607f9a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go @@ -100,10 +100,7 @@ func registerCandidates(c *helper.InitializeContext) error { // Register candidates in batches in order to overcome the signers amount limit. // See: https://github.com/nspcc-dev/neo-go/blob/master/pkg/core/transaction/transaction.go#L27 for i := 0; i < need; i += registerBatchSize { - start, end := i, i+registerBatchSize - if end > need { - end = need - } + start, end := i, min(i+registerBatchSize, need) // This check is sound because transactions are accepted/rejected atomically. if have >= end { continue diff --git a/pkg/innerring/processors/netmap/cleanup_table.go b/pkg/innerring/processors/netmap/cleanup_table.go index 705e21d99..c18611569 100644 --- a/pkg/innerring/processors/netmap/cleanup_table.go +++ b/pkg/innerring/processors/netmap/cleanup_table.go @@ -82,10 +82,7 @@ func (c *cleanupTable) touch(keyString string, now uint64, binNodeInfo []byte) b result := !ok || access.removeFlag || !bytes.Equal(access.binNodeInfo, binNodeInfo) access.removeFlag = false // reset remove flag on each touch - if now > access.epoch { - access.epoch = now - } - + access.epoch = max(access.epoch, now) access.binNodeInfo = binNodeInfo // update binary node info c.lastAccess[keyString] = access diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index b1cb91920..01e5529da 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -57,14 +57,6 @@ func (b *Blobovnicza) iterateBounds(useObjLimitBound bool, f func(uint64, uint64 return nil } -func max(a, b uint64) uint64 { - if a > b { - return a - } - - return b -} - // IterationElement represents a unit of elements through which Iterate operation passes. type IterationElement struct { addr oid.Address diff --git a/pkg/local_object_storage/engine/list.go b/pkg/local_object_storage/engine/list.go index f9229a2b1..7245caeeb 100644 --- a/pkg/local_object_storage/engine/list.go +++ b/pkg/local_object_storage/engine/list.go @@ -134,10 +134,7 @@ func (e *StorageEngine) ListWithCursor(ctx context.Context, prm ListWithCursorPr continue } - count := prm.count - uint32(len(result)) - if count > batchSize { - count = batchSize - } + count := min(prm.count-uint32(len(result)), batchSize) var shardPrm shard.ListWithCursorPrm shardPrm.WithCount(count) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 83be3259a..ef8e97d34 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -343,16 +343,8 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { } func (s *Shard) getExpiredObjectsParameters() (workerCount, batchSize int) { - workerCount = minExpiredWorkers - batchSize = minExpiredBatchSize - - if s.gc.gcCfg.expiredCollectorBatchSize > batchSize { - batchSize = s.gc.gcCfg.expiredCollectorBatchSize - } - - if s.gc.gcCfg.expiredCollectorWorkerCount > workerCount { - workerCount = s.gc.gcCfg.expiredCollectorWorkerCount - } + workerCount = max(minExpiredWorkers, s.gc.gcCfg.expiredCollectorWorkerCount) + batchSize = max(minExpiredBatchSize, s.gc.gcCfg.expiredCollectorBatchSize) return } diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 1665fec1d..4865b43ef 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -162,11 +162,7 @@ func (c *Client) DepositNotary(amount fixedn.Fixed8, delta uint32) (res util.Uin return util.Uint256{}, fmt.Errorf("can't get previous expiration value: %w", err) } - till := int64(bc + delta) - if till < currentTill { - till = currentTill - } - + till := max(int64(bc+delta), currentTill) return c.depositNotary(amount, till) } diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 4634c96e1..2c405070d 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -343,10 +343,7 @@ func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, e return nil, new(apistatus.ObjectOutOfRange) } - ln := prm.ln - if ln > maxInitialBufferSize { - ln = maxInitialBufferSize - } + ln := min(prm.ln, maxInitialBufferSize) w := bytes.NewBuffer(make([]byte, 0, ln)) _, err = io.CopyN(w, rdr, int64(prm.ln)) diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index 2d9810cd3..54e49cb12 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -164,10 +164,7 @@ func (s *searchStreamMsgSizeCtrl) Send(resp *object.SearchResponse) error { newResp.SetBody(body) } - cut := s.addrAmount - if cut > ln { - cut = ln - } + cut := min(s.addrAmount, ln) body.SetIDList(ids[:cut]) newResp.SetMetaHeader(resp.GetMetaHeader()) diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 50f206fd9..be0974c39 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -388,10 +388,7 @@ func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectc if it.cur >= len(it.objs) { return nil, engine.ErrEndOfListing } - end := it.cur + int(size) - if end > len(it.objs) { - end = len(it.objs) - } + end := min(it.cur+int(size), len(it.objs)) ret := it.objs[it.cur:end] it.cur = end return ret, nil diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 064ee5900..0f85f50b1 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -167,9 +167,7 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram merged <- ms[minTimeMoveIndex] height := ms[minTimeMoveIndex].Time if ms[minTimeMoveIndex] = <-streams[minTimeMoveIndex]; ms[minTimeMoveIndex] == nil { - if minStreamedLastHeight > height { - minStreamedLastHeight = height - } + minStreamedLastHeight = min(minStreamedLastHeight, height) } } @@ -203,9 +201,7 @@ func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID s errGroup.Go(func() error { if err := s.forest.TreeApply(ctx, cid, treeID, m, true); err != nil { heightMtx.Lock() - if m.Time < unappliedOperationHeight { - unappliedOperationHeight = m.Time - } + unappliedOperationHeight = min(unappliedOperationHeight, m.Time) heightMtx.Unlock() return err } From 2ca5dfc2f60a5174c8077d4bbc3f5d7963b0f158 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 Apr 2024 14:15:56 +0300 Subject: [PATCH 1030/1943] Reapply "[#972] Adopt slices.BinarySearch()" This reverts commit 4bfc6d29b910bc8e85852180ca0caabbf2eff2f4. Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/container/container.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/container/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go index 690db2f4f..eda388d37 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "slices" "sort" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" @@ -446,7 +447,7 @@ func getCIDFilterFunc(cmd *cobra.Command) (func([]byte) bool, error) { var id cid.ID id.SetSHA256(v) idStr := id.EncodeToString() - n := sort.Search(len(rawIDs), func(i int) bool { return rawIDs[i] >= idStr }) - return n < len(rawIDs) && rawIDs[n] == idStr + _, found := slices.BinarySearch(rawIDs, idStr) + return found }, nil } From 2c4b50a71ed9eaf23a34fe0c327249fc60391cf7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 Apr 2024 14:15:57 +0300 Subject: [PATCH 1031/1943] Reapply "[#972] Use require.ElementsMatch() where possible" This reverts commit 7627d08914ab4cfa6fd73f17e586f4b0f1a0bbc1. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/list_test.go | 12 +----------- .../metabase/containers_test.go | 12 +----------- pkg/local_object_storage/metabase/list_test.go | 14 +------------- 3 files changed, 3 insertions(+), 35 deletions(-) diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 7bb1ac0fa..4fc9569c7 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -3,7 +3,6 @@ package engine import ( "context" "path/filepath" - "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -18,13 +17,6 @@ import ( "github.com/stretchr/testify/require" ) -func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithType { - sort.Slice(addrWithType, func(i, j int) bool { - return addrWithType[i].Address.EncodeToString() < addrWithType[j].Address.EncodeToString() - }) - return addrWithType -} - func TestListWithCursor(t *testing.T) { t.Parallel() @@ -98,7 +90,6 @@ func TestListWithCursor(t *testing.T) { require.NoError(t, err) expected = append(expected, object.AddressWithType{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)}) } - expected = sortAddresses(expected) var prm ListWithCursorPrm prm.count = tt.batchSize @@ -113,8 +104,7 @@ func TestListWithCursor(t *testing.T) { prm.cursor = res.Cursor() } - got = sortAddresses(got) - require.Equal(t, expected, got) + require.ElementsMatch(t, expected, got) }) } } diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index a90cd9d97..5d6788d7e 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -3,7 +3,6 @@ package meta_test import ( "context" "math/rand" - "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -108,18 +107,9 @@ func TestDB_ContainersCount(t *testing.T) { } } - sort.Slice(expected, func(i, j int) bool { - return expected[i].EncodeToString() < expected[j].EncodeToString() - }) - got, err := db.Containers(context.Background()) require.NoError(t, err) - - sort.Slice(got, func(i, j int) bool { - return got[i].EncodeToString() < got[j].EncodeToString() - }) - - require.Equal(t, expected, got) + require.ElementsMatch(t, expected, got) } func TestDB_ContainerSize(t *testing.T) { diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 25c0e35bd..e1ccb4e06 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -3,7 +3,6 @@ package meta_test import ( "context" "errors" - "sort" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -128,8 +127,6 @@ func TestLisObjectsWithCursor(t *testing.T) { expected = append(expected, object.AddressWithType{Address: object.AddressOf(child), Type: objectSDK.TypeRegular}) } - expected = sortAddresses(expected) - t.Run("success with various count", func(t *testing.T) { for countPerReq := 1; countPerReq <= total; countPerReq++ { got := make([]object.AddressWithType, 0, total) @@ -151,9 +148,7 @@ func TestLisObjectsWithCursor(t *testing.T) { _, _, err = metaListWithCursor(db, uint32(countPerReq), cursor) require.ErrorIs(t, err, meta.ErrEndOfListing, "count:%d", countPerReq, cursor) - - got = sortAddresses(got) - require.Equal(t, expected, got, "count:%d", countPerReq) + require.ElementsMatch(t, expected, got, "count:%d", countPerReq) } }) @@ -216,13 +211,6 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) { } } -func sortAddresses(addrWithType []object.AddressWithType) []object.AddressWithType { - sort.Slice(addrWithType, func(i, j int) bool { - return addrWithType[i].Address.EncodeToString() < addrWithType[j].Address.EncodeToString() - }) - return addrWithType -} - func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]object.AddressWithType, *meta.Cursor, error) { var listPrm meta.ListPrm listPrm.SetCount(count) From 669103a33efc1ddedee847689e74c82441dc9916 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 Apr 2024 14:15:59 +0300 Subject: [PATCH 1032/1943] Reapply "[#972] Use slices.Sort* when useful" This reverts commit 3359349acbbfb0aa8e208c5286ff7bd6537df2be. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/shards_set_mode.go | 7 ++----- pkg/services/tree/getsubtree_test.go | 6 ++---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/shards_set_mode.go b/cmd/frostfs-cli/modules/control/shards_set_mode.go index 1c87b405b..e73f15178 100644 --- a/cmd/frostfs-cli/modules/control/shards_set_mode.go +++ b/cmd/frostfs-cli/modules/control/shards_set_mode.go @@ -3,7 +3,7 @@ package control import ( "bytes" "fmt" - "sort" + "slices" "strings" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" @@ -177,9 +177,6 @@ func getShardIDListFromIDFlag(cmd *cobra.Command, withAllFlag bool) [][]byte { res = append(res, raw) } - sort.Slice(res, func(i, j int) bool { - return bytes.Compare(res[i], res[j]) < 0 - }) - + slices.SortFunc(res, bytes.Compare) return res } diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index 9a4223e30..305c2bac9 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -5,7 +5,7 @@ import ( "errors" "path" "path/filepath" - "sort" + "slices" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -192,9 +192,7 @@ func testGetSubTreeOrderAsc(t *testing.T, p pilorama.ForestStorage) { require.True(t, found, "unknown node %d %v", i, acc.seen[i].GetBody().GetNodeId()) } - require.True(t, sort.SliceIsSorted(paths, func(i, j int) bool { - return paths[i] < paths[j] - })) + require.True(t, slices.IsSorted(paths)) }) t.Run("depth=1", func(t *testing.T) { acc := subTreeAcc{errIndex: -1} From 5ef5734c4e2e0011506dca04086ab27c4028fafa Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 Apr 2024 14:16:00 +0300 Subject: [PATCH 1033/1943] Reapply "[#972] Drop x/exp/slices dependency" This reverts commit 946f2ec2bf4b5e99726811ac8bcb225bbca0708f. Signed-off-by: Evgenii Stratonikov --- go.sum | 14 ++++++++++++++ pkg/innerring/processors/frostfs/handlers.go | 2 +- .../internal/testutil/generators_test.go | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 7f7e84398..66273b174 100644 --- a/go.sum +++ b/go.sum @@ -59,9 +59,11 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etly github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= @@ -96,6 +98,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -137,19 +140,24 @@ github.com/klauspost/reedsolomon v1.12.1 h1:NhWgum1efX1x58daOBGCFWcxtEhOhXKKl1HA github.com/klauspost/reedsolomon v1.12.1/go.mod h1:nEi5Kjb6QqtbofI6s+cbG/j1da11c96IBYBSnVGtuBs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= @@ -231,6 +239,7 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -307,6 +316,7 @@ go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40 go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= @@ -327,6 +337,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -403,12 +414,14 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= @@ -429,6 +442,7 @@ google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index 5782dc744..c80f9fdc5 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -3,13 +3,13 @@ package frostfs import ( "bytes" "encoding/hex" + "slices" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" "go.uber.org/zap" - "golang.org/x/exp/slices" ) func (np *Processor) handleDeposit(ev event.Event) { diff --git a/pkg/local_object_storage/internal/testutil/generators_test.go b/pkg/local_object_storage/internal/testutil/generators_test.go index f2c60f9b4..f7be6014d 100644 --- a/pkg/local_object_storage/internal/testutil/generators_test.go +++ b/pkg/local_object_storage/internal/testutil/generators_test.go @@ -2,10 +2,10 @@ package testutil import ( "encoding/binary" + "slices" "testing" "github.com/stretchr/testify/require" - "golang.org/x/exp/slices" ) func TestOverwriteObjGenerator(t *testing.T) { From 40781b3a20240185b2b76e6fc676196ded1d9395 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 10 Apr 2024 11:49:31 +0300 Subject: [PATCH 1034/1943] [#1086] engine: Change mode in case of errors async Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/engine.go | 33 ++++++++----------- pkg/local_object_storage/engine/error_test.go | 7 ++-- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 558fe92ed..b8ac6cf97 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -133,7 +133,7 @@ func (e *StorageEngine) reportShardErrorBackground(id string, msg string, err er errCount := sh.errorCount.Add(1) sh.Shard.IncErrorCounter() - e.reportShardErrorWithFlags(sh.Shard, errCount, false, msg, err) + e.reportShardErrorWithFlags(sh.Shard, errCount, msg, err) } // reportShardError checks that the amount of errors doesn't exceed the configured threshold. @@ -153,13 +153,12 @@ func (e *StorageEngine) reportShardError( errCount := sh.errorCount.Add(1) sh.Shard.IncErrorCounter() - e.reportShardErrorWithFlags(sh.Shard, errCount, true, msg, err, fields...) + e.reportShardErrorWithFlags(sh.Shard, errCount, msg, err, fields...) } func (e *StorageEngine) reportShardErrorWithFlags( sh *shard.Shard, errCount uint32, - block bool, msg string, err error, fields ...zap.Field, @@ -175,23 +174,19 @@ func (e *StorageEngine) reportShardErrorWithFlags( return } - if block { - e.moveToDegraded(sh, errCount) - } else { - req := setModeRequest{ - errorCount: errCount, - sh: sh, - } + req := setModeRequest{ + errorCount: errCount, + sh: sh, + } - select { - case e.setModeCh <- req: - default: - // For background workers we can have a lot of such errors, - // thus logging is done with DEBUG level. - e.log.Debug(logs.EngineModeChangeIsInProgressIgnoringSetmodeRequest, - zap.Stringer("shard_id", sid), - zap.Uint32("error_count", errCount)) - } + select { + case e.setModeCh <- req: + default: + // For background workers we can have a lot of such errors, + // thus logging is done with DEBUG level. + e.log.Debug(logs.EngineModeChangeIsInProgressIgnoringSetmodeRequest, + zap.Stringer("shard_id", sid), + zap.Uint32("error_count", errCount)) } } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index ec4287bdd..6c44966f2 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strconv" "testing" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -229,6 +230,8 @@ func checkShardState(t *testing.T, e *StorageEngine, id *shard.ID, errCount uint sh := e.shards[id.String()] e.mtx.RUnlock() - require.Equal(t, errCount, sh.errorCount.Load()) - require.Equal(t, mode, sh.GetMode()) + require.Eventually(t, func() bool { + return errCount == sh.errorCount.Load() && + mode == sh.GetMode() + }, 10*time.Second, 10*time.Millisecond, "shard mode doesn't changed to expected state in 10 seconds") } From cdae227f82b6aeba684e3efdd66c881d66c2cd65 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 10 Apr 2024 15:26:39 +0300 Subject: [PATCH 1035/1943] [#1083] go.mod: Bump grpc version Signed-off-by: Anton Nikiforov --- go.mod | 7 +++---- go.sum | 18 ++++++++---------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index b99799337..4d497b0f9 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( golang.org/x/sync v0.6.0 golang.org/x/sys v0.18.0 golang.org/x/term v0.18.0 - google.golang.org/grpc v1.61.0 + google.golang.org/grpc v1.63.2 google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -71,7 +71,6 @@ require ( github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 // indirect @@ -125,8 +124,8 @@ require ( golang.org/x/crypto v0.21.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 66273b174..6c945172e 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,6 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -420,14 +418,14 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= -google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= -google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= -google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 661faa639e8c3d54dfd1888d0b2d0a9f8f83ce90 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Thu, 11 Apr 2024 11:53:01 +0300 Subject: [PATCH 1036/1943] [#1090] go.mod: Update policy-engine version Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4d497b0f9..7f9c9d457 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92 git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240307151106-2ec958cbfdfd + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240410114823-1f190e1668ec git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 6c945172e..83bb5818d 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92/go.mod h1:i0RKqiF4z3UOxLSNwhHw+cUz/JyYWuTRpnn9ere4Y3w= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240307151106-2ec958cbfdfd h1:pyIl9f4nIr7ekJ73W9keLIQ5dpoKb8o6xNmodsXY5+o= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240307151106-2ec958cbfdfd/go.mod h1:H/AW85RtYxVTbcgwHW76DqXeKlsiCIOeNXHPqyDBrfQ= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240410114823-1f190e1668ec h1:OG8tBs5CN2HKp10sAWdtiFaX8qSGFyLGWfQmf4FQ6bE= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240410114823-1f190e1668ec/go.mod h1:H/AW85RtYxVTbcgwHW76DqXeKlsiCIOeNXHPqyDBrfQ= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= From f4dcb418f2bf7d733247a6dfd751744309a5a79f Mon Sep 17 00:00:00 2001 From: aarifullin Date: Thu, 11 Apr 2024 12:35:49 +0300 Subject: [PATCH 1037/1943] [#1090] ape: Move ape request and resource implementations to common package Signed-off-by: Airat Arifullin --- pkg/ape/converter/converter.go | 44 +++++++++++++ pkg/ape/request/request.go | 55 ++++++++++++++++ pkg/services/container/ape.go | 84 ++++++++----------------- pkg/services/object/ape/request.go | 57 ++++------------- pkg/services/object/ape/request_test.go | 18 +++--- 5 files changed, 144 insertions(+), 114 deletions(-) create mode 100644 pkg/ape/converter/converter.go create mode 100644 pkg/ape/request/request.go diff --git a/pkg/ape/converter/converter.go b/pkg/ape/converter/converter.go new file mode 100644 index 000000000..9032680af --- /dev/null +++ b/pkg/ape/converter/converter.go @@ -0,0 +1,44 @@ +package converter + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" +) + +func SchemaRoleFromACLRole(role acl.Role) (string, error) { + switch role { + case acl.RoleOwner: + return nativeschema.PropertyValueContainerRoleOwner, nil + case acl.RoleContainer: + return nativeschema.PropertyValueContainerRoleContainer, nil + case acl.RoleInnerRing: + return nativeschema.PropertyValueContainerRoleIR, nil + case acl.RoleOthers: + return nativeschema.PropertyValueContainerRoleOthers, nil + default: + return "", fmt.Errorf("failed to convert %s", role.String()) + } +} + +func SchemaMethodFromACLOperation(op acl.Op) (string, error) { + switch op { + case acl.OpObjectGet: + return nativeschema.MethodGetObject, nil + case acl.OpObjectHead: + return nativeschema.MethodHeadObject, nil + case acl.OpObjectPut: + return nativeschema.MethodPutObject, nil + case acl.OpObjectDelete: + return nativeschema.MethodDeleteObject, nil + case acl.OpObjectSearch: + return nativeschema.MethodSearchObject, nil + case acl.OpObjectRange: + return nativeschema.MethodRangeObject, nil + case acl.OpObjectHash: + return nativeschema.MethodHashObject, nil + default: + return "", fmt.Errorf("operation cannot be converted: %d", op) + } +} diff --git a/pkg/ape/request/request.go b/pkg/ape/request/request.go new file mode 100644 index 000000000..6d62ef3d5 --- /dev/null +++ b/pkg/ape/request/request.go @@ -0,0 +1,55 @@ +package ape + +import ( + aperesource "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" +) + +type Request struct { + operation string + resource Resource + properties map[string]string +} + +func NewRequest(operation string, resource Resource, properties map[string]string) Request { + return Request{ + operation: operation, + resource: resource, + properties: properties, + } +} + +var _ aperesource.Request = Request{} + +func (r Request) Operation() string { + return r.operation +} + +func (r Request) Property(key string) string { + return r.properties[key] +} + +func (r Request) Resource() aperesource.Resource { + return r.resource +} + +type Resource struct { + name string + properties map[string]string +} + +var _ aperesource.Resource = Resource{} + +func NewResource(name string, properties map[string]string) Resource { + return Resource{ + name: name, + properties: properties, + } +} + +func (r Resource) Name() string { + return r.name +} + +func (r Resource) Property(key string) string { + return r.properties[key] +} diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 83361257a..7622a40bc 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -15,6 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -26,7 +27,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" - "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" @@ -148,14 +148,14 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co return nil, err } - request := &apeRequest{ - resource: &apeResource{ - name: resourceName(namespace, ""), - props: make(map[string]string), - }, - op: nativeschema.MethodListContainers, - props: reqProps, - } + request := aperequest.NewRequest( + nativeschema.MethodListContainers, + aperequest.NewResource( + resourceName(namespace, ""), + make(map[string]string), + ), + reqProps, + ) s, found, err := ac.router.IsAllowed(apechain.Ingress, policyengine.NewRequestTargetWithNamespace(namespace), @@ -193,14 +193,14 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont return nil, err } - request := &apeRequest{ - resource: &apeResource{ - name: resourceName(namespace, ""), - props: make(map[string]string), - }, - op: nativeschema.MethodPutContainer, - props: reqProps, - } + request := aperequest.NewRequest( + nativeschema.MethodPutContainer, + aperequest.NewResource( + resourceName(namespace, ""), + make(map[string]string), + ), + reqProps, + ) s, found, err := ac.router.IsAllowed(apechain.Ingress, policyengine.NewRequestTargetWithNamespace(namespace), @@ -288,14 +288,14 @@ func (ac *apeChecker) validateContainerBoundedOperation(containerID *refs.Contai namespace = cntNamespace } - request := &apeRequest{ - resource: &apeResource{ - name: resourceName(namespace, id.EncodeToString()), - props: ac.getContainerProps(cont), - }, - op: op, - props: reqProps, - } + request := aperequest.NewRequest( + op, + aperequest.NewResource( + resourceName(namespace, id.EncodeToString()), + ac.getContainerProps(cont), + ), + reqProps, + ) s, found, err := ac.router.IsAllowed(apechain.Ingress, policyengine.NewRequestTarget(namespace, id.EncodeToString()), @@ -329,40 +329,6 @@ func getContainerID(reqContID *refs.ContainerID) (cid.ID, error) { return id, nil } -type apeRequest struct { - resource *apeResource - op string - props map[string]string -} - -// Operation implements resource.Request. -func (r *apeRequest) Operation() string { - return r.op -} - -// Property implements resource.Request. -func (r *apeRequest) Property(key string) string { - return r.props[key] -} - -// Resource implements resource.Request. -func (r *apeRequest) Resource() resource.Resource { - return r.resource -} - -type apeResource struct { - name string - props map[string]string -} - -func (r *apeResource) Name() string { - return r.name -} - -func (r *apeResource) Property(key string) string { - return r.props[key] -} - func resourceName(namespace string, container string) string { if namespace == "" && container == "" { return nativeschema.ResourceFormatRootContainers diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index b81fe8c32..7bbee31c1 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -6,51 +6,16 @@ import ( "strconv" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - aperesource "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" ) -type request struct { - operation string - resource resource - properties map[string]string -} - -var defaultRequest = request{} - -var _ aperesource.Request = request{} - -type resource struct { - name string - properties map[string]string -} - -var _ aperesource.Resource = resource{} - -func (r resource) Name() string { - return r.name -} - -func (r resource) Property(key string) string { - return r.properties[key] -} - -func (r request) Operation() string { - return r.operation -} - -func (r request) Property(key string) string { - return r.properties[key] -} - -func (r request) Resource() aperesource.Resource { - return r.resource -} +var defaultRequest = aperequest.Request{} func nativeSchemaRole(role acl.Role) string { switch role { @@ -125,7 +90,7 @@ func objectProperties(cnr cid.ID, oid *oid.ID, cnrOwner user.ID, header *objectV // newAPERequest creates an APE request to be passed to a chain router. It collects resource properties from // header provided by headerProvider. If it cannot be found in headerProvider, then properties are // initialized from header given in prm (if it is set). Otherwise, just CID and OID are set to properties. -func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (request, error) { +func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Request, error) { switch prm.Method { case nativeschema.MethodGetObject, nativeschema.MethodHeadObject, @@ -150,15 +115,15 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (request, erro } } - return request{ - operation: prm.Method, - resource: resource{ - name: resourceName(prm.Container, prm.Object, prm.Namespace), - properties: objectProperties(prm.Container, prm.Object, prm.ContainerOwner, header), - }, - properties: map[string]string{ + return aperequest.NewRequest( + prm.Method, + aperequest.NewResource( + resourceName(prm.Container, prm.Object, prm.Namespace), + objectProperties(prm.Container, prm.Object, prm.ContainerOwner, header), + ), + map[string]string{ nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, nativeschema.PropertyKeyActorRole: prm.Role, }, - }, nil + ), nil } diff --git a/pkg/services/object/ape/request_test.go b/pkg/services/object/ape/request_test.go index 71e234e78..fdb7af219 100644 --- a/pkg/services/object/ape/request_test.go +++ b/pkg/services/object/ape/request_test.go @@ -6,6 +6,7 @@ import ( "testing" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -256,22 +257,21 @@ func TestNewAPERequest(t *testing.T) { return } - expectedRequest := request{ - operation: method, - resource: resource{ - name: resourceName(cnr, obj, prm.Namespace), - properties: objectProperties(cnr, obj, testCnrOwner, func() *objectV2.Header { + expectedRequest := aperequest.NewRequest( + method, + aperequest.NewResource( + resourceName(cnr, obj, prm.Namespace), + objectProperties(cnr, obj, testCnrOwner, func() *objectV2.Header { if headerObjSDK != nil { return headerObjSDK.ToV2().GetHeader() } return prm.Header - }()), - }, - properties: map[string]string{ + }())), + map[string]string{ nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, nativeschema.PropertyKeyActorRole: prm.Role, }, - } + ) require.Equal(t, expectedRequest, r) }) From 6a46c6d2294d6e6e994fe53c8e0347c8cbbbec88 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Thu, 11 Apr 2024 18:10:33 +0300 Subject: [PATCH 1038/1943] [#1090] tree: Make workaround for APE checks * Make `verifyClient` method perform APE check if a container was created with zero-filled basic ACL. * Object verbs are used in APE, until tree verbs are introduced. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/tree.go | 4 +- pkg/services/tree/ape.go | 69 ++++++++++++++++++++++++++++++++++ pkg/services/tree/options.go | 9 +++++ pkg/services/tree/signature.go | 13 ++++--- 4 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 pkg/services/tree/ape.go diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index dced05bc2..ff00b6fac 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -63,7 +63,9 @@ func initTreeService(c *cfg) { tree.WithReplicationChannelCapacity(treeConfig.ReplicationChannelCapacity()), tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount()), tree.WithAuthorizedKeys(treeConfig.AuthorizedKeys()), - tree.WithMetrics(c.metricsCollector.TreeService())) + tree.WithMetrics(c.metricsCollector.TreeService()), + tree.WithAPERouter(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine), + ) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { tree.RegisterTreeServiceServer(s, c.treeService) diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go new file mode 100644 index 000000000..52036074a --- /dev/null +++ b/pkg/services/tree/ape.go @@ -0,0 +1,69 @@ +package tree + +import ( + "encoding/hex" + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/converter" + aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" + core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +func (s *Service) checkAPE(container *core.Container, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey) error { + namespace := "" + cntNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(container.Value).Zone(), ".ns") + if hasNamespace { + namespace = cntNamespace + } + + schemaMethod, err := converter.SchemaMethodFromACLOperation(operation) + if err != nil { + return apeErr(err) + } + schemaRole, err := converter.SchemaRoleFromACLRole(role) + if err != nil { + return apeErr(err) + } + reqProps := map[string]string{ + nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(publicKey.Bytes()), + nativeschema.PropertyKeyActorRole: schemaRole, + } + + var resourceName string + if namespace == "root" || namespace == "" { + resourceName = fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cid.EncodeToString()) + } else { + resourceName = fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainerObjects, namespace, cid.EncodeToString()) + } + + request := aperequest.NewRequest( + schemaMethod, + aperequest.NewResource(resourceName, make(map[string]string)), + reqProps, + ) + + status, found, err := s.router.IsAllowed(apechain.Ingress, engine.NewRequestTarget(namespace, cid.EncodeToString()), request) + if err != nil { + return apeErr(err) + } + if found && status == apechain.Allow { + return nil + } + err = fmt.Errorf("access to operation %s is denied by access policy engine: %s", schemaMethod, status.String()) + return apeErr(err) +} + +func apeErr(err error) error { + errAccessDenied := &apistatus.ObjectAccessDenied{} + errAccessDenied.WriteReason(err.Error()) + return errAccessDenied +} diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index 043e12cb2..0e5f64274 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) @@ -38,6 +39,8 @@ type cfg struct { containerCacheSize int authorizedKeys [][]byte + router policyengine.ChainRouter + metrics MetricsRegister } @@ -139,3 +142,9 @@ func WithAuthorizedKeys(keys keys.PublicKeys) Option { } } } + +func WithAPERouter(router policyengine.ChainRouter) Option { + return func(c *cfg) { + c.router = router + } +} diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 985e1ad94..162b189e3 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -71,7 +71,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op return err } - role, err := roleFromReq(cnr, req, bt) + role, pubKey, err := roleAndPubKeyFromReq(cnr, req, bt) if err != nil { return fmt.Errorf("can't get request role: %w", err) } @@ -79,8 +79,11 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op basicACL := cnr.Value.BasicACL() // Basic ACL mask can be unset, if a container operations are performed // with strict APE checks only. + // + // FIXME(@aarifullin): tree service temporiraly performs APE checks on + // object verbs, because tree verbs have not been introduced yet. if basicACL == 0x0 { - return nil + return s.checkAPE(cnr, cid, op, role, pubKey) } if !basicACL.IsOpAllowed(op, role) { @@ -222,7 +225,7 @@ func SignMessage(m message, key *ecdsa.PrivateKey) error { return nil } -func roleFromReq(cnr *core.Container, req message, bt *bearer.Token) (acl.Role, error) { +func roleAndPubKeyFromReq(cnr *core.Container, req message, bt *bearer.Token) (acl.Role, *keys.PublicKey, error) { role := acl.RoleOthers owner := cnr.Value.Owner() @@ -233,7 +236,7 @@ func roleFromReq(cnr *core.Container, req message, bt *bearer.Token) (acl.Role, pub, err := keys.NewPublicKeyFromBytes(rawKey, elliptic.P256()) if err != nil { - return role, fmt.Errorf("invalid public key: %w", err) + return role, nil, fmt.Errorf("invalid public key: %w", err) } var reqSigner user.ID @@ -243,7 +246,7 @@ func roleFromReq(cnr *core.Container, req message, bt *bearer.Token) (acl.Role, role = acl.RoleOwner } - return role, nil + return role, pub, nil } func eACLOp(op acl.Op) eacl.Operation { From 5be36924e3ec84a7069ececa4b3e02212ce57c4e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 Apr 2024 11:25:51 +0300 Subject: [PATCH 1039/1943] [#41] log: Log storage operations in only in Debug They are mostly useless unless we need to _debug_ a specific issue. The amount of logs we produce is too big. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/internal/log/log.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/internal/log/log.go b/pkg/local_object_storage/internal/log/log.go index 82024ffa7..6272067d3 100644 --- a/pkg/local_object_storage/internal/log/log.go +++ b/pkg/local_object_storage/internal/log/log.go @@ -10,7 +10,7 @@ const headMsg = "local object storage operation" // Write writes message about storage engine's operation to logger. func Write(logger *logger.Logger, fields ...zap.Field) { - logger.Info(headMsg, fields...) + logger.Debug(headMsg, fields...) } // AddressField returns logger's field for object address. From e5e0542482998f0e65cd811ce3ff331def189001 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 Apr 2024 11:32:35 +0300 Subject: [PATCH 1040/1943] [#1085] log: Move storage log message to constants package Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 1 + pkg/local_object_storage/internal/log/log.go | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index d56d425ce..bd67217c4 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -201,6 +201,7 @@ const ( SubscriberCantCastBlockEventValueToBlock = "can't cast block event value to block" SubscriberCantCastNotifyEventValueToTheNotaryRequestStruct = "can't cast notify event value to the notary request struct" SubscriberUnsupportedNotificationFromTheChain = "unsupported notification from the chain" + StorageOperation = "local object storage operation" BlobovniczaCreatingDirectoryForBoltDB = "creating directory for BoltDB" BlobovniczaOpeningBoltDB = "opening BoltDB" BlobovniczaInitializing = "initializing..." diff --git a/pkg/local_object_storage/internal/log/log.go b/pkg/local_object_storage/internal/log/log.go index 6272067d3..23740868d 100644 --- a/pkg/local_object_storage/internal/log/log.go +++ b/pkg/local_object_storage/internal/log/log.go @@ -1,16 +1,14 @@ package storagelog import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) -// headMsg is a distinctive part of all messages. -const headMsg = "local object storage operation" - // Write writes message about storage engine's operation to logger. func Write(logger *logger.Logger, fields ...zap.Field) { - logger.Debug(headMsg, fields...) + logger.Debug(logs.StorageOperation, fields...) } // AddressField returns logger's field for object address. From 91e79c98ba5805b3da5358f131a0b301c58b4633 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 Apr 2024 13:51:39 +0300 Subject: [PATCH 1041/1943] [#1089] ape: Provide request actor as an additional target Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- pkg/services/container/ape.go | 30 +++++++++++++++---------- pkg/services/object/ape/checker.go | 10 +++++++-- pkg/services/object/ape/checker_test.go | 5 ++++- pkg/services/tree/ape.go | 3 ++- 6 files changed, 35 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 7f9c9d457..8cfea305d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92 git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240410114823-1f190e1668ec + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 83bb5818d..76ed7e5dc 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92/go.mod h1:i0RKqiF4z3UOxLSNwhHw+cUz/JyYWuTRpnn9ere4Y3w= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240410114823-1f190e1668ec h1:OG8tBs5CN2HKp10sAWdtiFaX8qSGFyLGWfQmf4FQ6bE= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240410114823-1f190e1668ec/go.mod h1:H/AW85RtYxVTbcgwHW76DqXeKlsiCIOeNXHPqyDBrfQ= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a h1:wbndKvHbwDQiSMQWL75RxiTZCeUyCi7NUj1lsfdAGkc= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a/go.mod h1:H/AW85RtYxVTbcgwHW76DqXeKlsiCIOeNXHPqyDBrfQ= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 7622a40bc..02549bded 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -157,9 +157,12 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co reqProps, ) - s, found, err := ac.router.IsAllowed(apechain.Ingress, - policyengine.NewRequestTargetWithNamespace(namespace), - request) + rt := policyengine.NewRequestTargetWithNamespace(namespace) + rt.User = &policyengine.Target{ + Type: policyengine.User, + Name: fmt.Sprintf("%s:%s", namespace, pk.Address()), + } + s, found, err := ac.router.IsAllowed(apechain.Ingress, rt, request) if err != nil { return nil, err } @@ -202,9 +205,12 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont reqProps, ) - s, found, err := ac.router.IsAllowed(apechain.Ingress, - policyengine.NewRequestTargetWithNamespace(namespace), - request) + rt := policyengine.NewRequestTargetWithNamespace(namespace) + rt.User = &policyengine.Target{ + Type: policyengine.User, + Name: fmt.Sprintf("%s:%s", namespace, pk.Address()), + } + s, found, err := ac.router.IsAllowed(apechain.Ingress, rt, request) if err != nil { return nil, err } @@ -277,7 +283,7 @@ func (ac *apeChecker) validateContainerBoundedOperation(containerID *refs.Contai return err } - reqProps, err := ac.getRequestProps(mh, vh, cont, id) + reqProps, pk, err := ac.getRequestProps(mh, vh, cont, id) if err != nil { return err } @@ -298,7 +304,7 @@ func (ac *apeChecker) validateContainerBoundedOperation(containerID *refs.Contai ) s, found, err := ac.router.IsAllowed(apechain.Ingress, - policyengine.NewRequestTarget(namespace, id.EncodeToString()), + policyengine.NewRequestTargetExtended(namespace, id.EncodeToString(), fmt.Sprintf("%s:%s", namespace, pk.Address()), nil), request) if err != nil { return err @@ -350,19 +356,19 @@ func (ac *apeChecker) getContainerProps(c *containercore.Container) map[string]s func (ac *apeChecker) getRequestProps(mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader, cont *containercore.Container, cnrID cid.ID, -) (map[string]string, error) { +) (map[string]string, *keys.PublicKey, error) { actor, pk, err := ac.getActorAndPublicKey(mh, vh, cnrID) if err != nil { - return nil, err + return nil, nil, err } role, err := ac.getRole(actor, pk, cont, cnrID) if err != nil { - return nil, err + return nil, nil, err } return map[string]string{ nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(pk.Bytes()), nativeschema.PropertyKeyActorRole: role, - }, nil + }, pk, nil } func (ac *apeChecker) getRole(actor *user.ID, pk *keys.PublicKey, cont *containercore.Container, cnrID cid.ID) (string, error) { diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 119df5fc0..1063bd901 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -12,6 +12,7 @@ import ( apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) type checkerImpl struct { @@ -84,8 +85,13 @@ func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { return fmt.Errorf("failed to create ape request: %w", err) } - status, ruleFound, err := c.chainRouter.IsAllowed(apechain.Ingress, - policyengine.NewRequestTarget(prm.Namespace, prm.Container.EncodeToString()), r) + pub, err := keys.NewPublicKeyFromString(prm.SenderKey) + if err != nil { + return err + } + + rt := policyengine.NewRequestTargetExtended(prm.Namespace, prm.Container.EncodeToString(), fmt.Sprintf("%s:%s", prm.Namespace, pub.Address()), nil) + status, ruleFound, err := c.chainRouter.IsAllowed(apechain.Ingress, rt, r) if err != nil { return err } diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index 443414959..fc915715c 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -16,6 +16,7 @@ import ( policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" ) @@ -147,7 +148,9 @@ var ( role = "Container" - senderKey = hex.EncodeToString([]byte{1, 0, 0, 1}) + senderPrivateKey, _ = keys.NewPrivateKey() + + senderKey = hex.EncodeToString(senderPrivateKey.PublicKey().Bytes()) ) func TestAPECheck(t *testing.T) { diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 52036074a..5da49a591 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -51,7 +51,8 @@ func (s *Service) checkAPE(container *core.Container, cid cid.ID, operation acl. reqProps, ) - status, found, err := s.router.IsAllowed(apechain.Ingress, engine.NewRequestTarget(namespace, cid.EncodeToString()), request) + rt := engine.NewRequestTargetExtended(namespace, cid.EncodeToString(), fmt.Sprintf("%s:%s", namespace, publicKey.Address()), nil) + status, found, err := s.router.IsAllowed(apechain.Ingress, rt, request) if err != nil { return apeErr(err) } From 00941862999ef79cc02fdeea720399bb89c3faaf Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 Apr 2024 17:01:19 +0300 Subject: [PATCH 1042/1943] [#1089] control: Format proto files with clang-format Signed-off-by: Evgenii Stratonikov --- pkg/services/control/service.proto | 786 ++++++++++++++--------------- pkg/services/control/types.proto | 223 ++++---- 2 files changed, 503 insertions(+), 506 deletions(-) diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 04ea62e0e..94032b346 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -6,658 +6,656 @@ import "pkg/services/control/types.proto"; option go_package = "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"; -// `ControlService` provides an interface for internal work with the storage node. +// `ControlService` provides an interface for internal work with the storage +// node. service ControlService { - // Performs health check of the storage node. - rpc HealthCheck (HealthCheckRequest) returns (HealthCheckResponse); + // Performs health check of the storage node. + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); - // Sets status of the storage node in FrostFS network map. - rpc SetNetmapStatus (SetNetmapStatusRequest) returns (SetNetmapStatusResponse); + // Sets status of the storage node in FrostFS network map. + rpc SetNetmapStatus(SetNetmapStatusRequest) returns (SetNetmapStatusResponse); - // Mark objects to be removed from node's local object storage. - rpc DropObjects (DropObjectsRequest) returns (DropObjectsResponse); + // Mark objects to be removed from node's local object storage. + rpc DropObjects(DropObjectsRequest) returns (DropObjectsResponse); - // Returns list that contains information about all shards of a node. - rpc ListShards (ListShardsRequest) returns (ListShardsResponse); + // Returns list that contains information about all shards of a node. + rpc ListShards(ListShardsRequest) returns (ListShardsResponse); - // Sets mode of the shard. - rpc SetShardMode (SetShardModeRequest) returns (SetShardModeResponse); + // Sets mode of the shard. + rpc SetShardMode(SetShardModeRequest) returns (SetShardModeResponse); - // Synchronizes all log operations for the specified tree. - rpc SynchronizeTree (SynchronizeTreeRequest) returns (SynchronizeTreeResponse); + // Synchronizes all log operations for the specified tree. + rpc SynchronizeTree(SynchronizeTreeRequest) returns (SynchronizeTreeResponse); - // EvacuateShard moves all data from one shard to the others. - // Deprecated: Use StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation - rpc EvacuateShard (EvacuateShardRequest) returns (EvacuateShardResponse); + // EvacuateShard moves all data from one shard to the others. + // Deprecated: Use + // StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation + rpc EvacuateShard(EvacuateShardRequest) returns (EvacuateShardResponse); - // StartShardEvacuation starts moving all data from one shard to the others. - rpc StartShardEvacuation (StartShardEvacuationRequest) returns (StartShardEvacuationResponse); + // StartShardEvacuation starts moving all data from one shard to the others. + rpc StartShardEvacuation(StartShardEvacuationRequest) + returns (StartShardEvacuationResponse); - // GetShardEvacuationStatus returns evacuation status. - rpc GetShardEvacuationStatus (GetShardEvacuationStatusRequest) returns (GetShardEvacuationStatusResponse); + // GetShardEvacuationStatus returns evacuation status. + rpc GetShardEvacuationStatus(GetShardEvacuationStatusRequest) + returns (GetShardEvacuationStatusResponse); - // ResetShardEvacuationStatus resets evacuation status if there is no running evacuation process. - rpc ResetShardEvacuationStatus (ResetShardEvacuationStatusRequest) returns (ResetShardEvacuationStatusResponse); + // ResetShardEvacuationStatus resets evacuation status if there is no running + // evacuation process. + rpc ResetShardEvacuationStatus(ResetShardEvacuationStatusRequest) + returns (ResetShardEvacuationStatusResponse); - // StopShardEvacuation stops moving all data from one shard to the others. - rpc StopShardEvacuation (StopShardEvacuationRequest) returns (StopShardEvacuationResponse); + // StopShardEvacuation stops moving all data from one shard to the others. + rpc StopShardEvacuation(StopShardEvacuationRequest) + returns (StopShardEvacuationResponse); - // FlushCache moves all data from one shard to the others. - rpc FlushCache (FlushCacheRequest) returns (FlushCacheResponse); + // FlushCache moves all data from one shard to the others. + rpc FlushCache(FlushCacheRequest) returns (FlushCacheResponse); - // Doctor performs storage restructuring operations on engine. - rpc Doctor (DoctorRequest) returns (DoctorResponse); + // Doctor performs storage restructuring operations on engine. + rpc Doctor(DoctorRequest) returns (DoctorResponse); - // Add local access policy engine overrides to a node. - rpc AddChainLocalOverride (AddChainLocalOverrideRequest) returns (AddChainLocalOverrideResponse); + // Add local access policy engine overrides to a node. + rpc AddChainLocalOverride(AddChainLocalOverrideRequest) + returns (AddChainLocalOverrideResponse); - // Get local access policy engine overrides stored in the node by chain id. - rpc GetChainLocalOverride (GetChainLocalOverrideRequest) returns (GetChainLocalOverrideResponse); + // Get local access policy engine overrides stored in the node by chain id. + rpc GetChainLocalOverride(GetChainLocalOverrideRequest) + returns (GetChainLocalOverrideResponse); - // List local access policy engine overrides stored in the node by container id. - rpc ListChainLocalOverrides (ListChainLocalOverridesRequest) returns (ListChainLocalOverridesResponse); + // List local access policy engine overrides stored in the node by container + // id. + rpc ListChainLocalOverrides(ListChainLocalOverridesRequest) + returns (ListChainLocalOverridesResponse); - // Remove local access policy engine overrides stored in the node by chaind id. - rpc RemoveChainLocalOverride (RemoveChainLocalOverrideRequest) returns (RemoveChainLocalOverrideResponse); + // Remove local access policy engine overrides stored in the node by chaind + // id. + rpc RemoveChainLocalOverride(RemoveChainLocalOverrideRequest) + returns (RemoveChainLocalOverrideResponse); - // Remove local access policy engine overrides stored in the node by chaind id. - rpc RemoveChainLocalOverridesByTarget (RemoveChainLocalOverridesByTargetRequest) returns (RemoveChainLocalOverridesByTargetResponse); + // Remove local access policy engine overrides stored in the node by chaind + // id. + rpc RemoveChainLocalOverridesByTarget( + RemoveChainLocalOverridesByTargetRequest) + returns (RemoveChainLocalOverridesByTargetResponse); - // List targets of the local APE overrides stored in the node. - rpc ListTargetsLocalOverrides (ListTargetsLocalOverridesRequest) returns (ListTargetsLocalOverridesResponse); + // List targets of the local APE overrides stored in the node. + rpc ListTargetsLocalOverrides(ListTargetsLocalOverridesRequest) + returns (ListTargetsLocalOverridesResponse); - // Flush objects from write-cache and move it to degraded read only mode. - rpc SealWriteCache(SealWriteCacheRequest) returns (SealWriteCacheResponse); + // Flush objects from write-cache and move it to degraded read only mode. + rpc SealWriteCache(SealWriteCacheRequest) returns (SealWriteCacheResponse); - // DetachShards detaches and closes shards. - rpc DetachShards(DetachShardsRequest) returns (DetachShardsResponse); + // DetachShards detaches and closes shards. + rpc DetachShards(DetachShardsRequest) returns (DetachShardsResponse); } // Health check request. message HealthCheckRequest { - // Health check request body. - message Body { - } + // Health check request body. + message Body {} - // Body of health check request message. - Body body = 1; + // Body of health check request message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } // Health check request. message HealthCheckResponse { - // Health check response body - message Body { - // Status of the storage node in FrostFS network map. - NetmapStatus netmap_status = 1; + // Health check response body + message Body { + // Status of the storage node in FrostFS network map. + NetmapStatus netmap_status = 1; - // Health status of storage node application. - HealthStatus health_status = 2; - } + // Health status of storage node application. + HealthStatus health_status = 2; + } - // Body of health check response message. - Body body = 1; + // Body of health check response message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } // Set netmap status request. message SetNetmapStatusRequest { - // Set netmap status request body. - message Body { - // New storage node status in FrostFS network map. - // If status is MAINTENANCE, the node checks whether maintenance is - // allowed in the network settings. In case of prohibition, the request - // is denied. Otherwise, node switches to local maintenance state. To - // force local maintenance, use `force_maintenance` flag. - NetmapStatus status = 1; + // Set netmap status request body. + message Body { + // New storage node status in FrostFS network map. + // If status is MAINTENANCE, the node checks whether maintenance is + // allowed in the network settings. In case of prohibition, the request + // is denied. Otherwise, node switches to local maintenance state. To + // force local maintenance, use `force_maintenance` flag. + NetmapStatus status = 1; - // MAINTENANCE status validation skip flag. If set, node starts local - // maintenance regardless of network settings. The flag MUST NOT be - // set for any other status. - bool force_maintenance = 2; - } + // MAINTENANCE status validation skip flag. If set, node starts local + // maintenance regardless of network settings. The flag MUST NOT be + // set for any other status. + bool force_maintenance = 2; + } - // Body of set netmap status request message. - Body body = 1; + // Body of set netmap status request message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } // Set netmap status response. message SetNetmapStatusResponse { - // Set netmap status response body - message Body { - } + // Set netmap status response body + message Body {} - // Body of set netmap status response message. - Body body = 1; + // Body of set netmap status response message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } // Request to drop the objects. message DropObjectsRequest { - // Request body structure. - message Body { - // List of object addresses to be removed. - // in FrostFS API binary format. - repeated bytes address_list = 1; - } + // Request body structure. + message Body { + // List of object addresses to be removed. + // in FrostFS API binary format. + repeated bytes address_list = 1; + } - // Body of the request message. - Body body = 1; + // Body of the request message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } // Response to request to drop the objects. message DropObjectsResponse { - // Response body structure. - message Body { - } + // Response body structure. + message Body {} - // Body of the response message. - Body body = 1; + // Body of the response message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } // Request to list all shards of the node. message ListShardsRequest { - // Request body structure. - message Body { - } + // Request body structure. + message Body {} - // Body of the request message. - Body body = 1; + // Body of the request message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } // ListShards response. message ListShardsResponse { - // Response body structure. - message Body { - // List of the node's shards. - repeated ShardInfo shards = 1; - } + // Response body structure. + message Body { + // List of the node's shards. + repeated ShardInfo shards = 1; + } - // Body of the response message. - Body body = 1; + // Body of the response message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } // Request to set mode of the shard. message SetShardModeRequest { - // Request body structure. - message Body { - // ID of the shard. - repeated bytes shard_ID = 1; + // Request body structure. + message Body { + // ID of the shard. + repeated bytes shard_ID = 1; - // Mode that requested to be set. - ShardMode mode = 2; + // Mode that requested to be set. + ShardMode mode = 2; - // Flag signifying whether error counter should be set to 0. - bool resetErrorCounter = 3; - } + // Flag signifying whether error counter should be set to 0. + bool resetErrorCounter = 3; + } - // Body of set shard mode request message. - Body body = 1; + // Body of set shard mode request message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } // SetShardMode response. message SetShardModeResponse { - // Response body structure. - message Body { - } + // Response body structure. + message Body {} - // Body of set shard mode response message. - Body body = 1; + // Body of set shard mode response message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } // SynchronizeTree request. message SynchronizeTreeRequest { - // Request body structure. - message Body { - bytes container_id = 1; - string tree_id = 2; - // Starting height for the synchronization. Can be omitted. - uint64 height = 3; - } + // Request body structure. + message Body { + bytes container_id = 1; + string tree_id = 2; + // Starting height for the synchronization. Can be omitted. + uint64 height = 3; + } - // Body of restore shard request message. - Body body = 1; + // Body of restore shard request message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } // SynchronizeTree response. message SynchronizeTreeResponse { - // Response body structure. - message Body { - } + // Response body structure. + message Body {} - // Body of restore shard response message. - Body body = 1; + // Body of restore shard response message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } - // EvacuateShard request. message EvacuateShardRequest { - // Request body structure. - message Body { - // ID of the shard. - repeated bytes shard_ID = 1; + // Request body structure. + message Body { + // ID of the shard. + repeated bytes shard_ID = 1; - // Flag indicating whether object read errors should be ignored. - bool ignore_errors = 2; - } + // Flag indicating whether object read errors should be ignored. + bool ignore_errors = 2; + } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } // EvacuateShard response. message EvacuateShardResponse { - // Response body structure. - message Body { - uint32 count = 1; - } + // Response body structure. + message Body { uint32 count = 1; } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } // FlushCache request. message FlushCacheRequest { - // Request body structure. - message Body { - // ID of the shard. - repeated bytes shard_ID = 1; - // If true, then writecache will be left in read-only mode after flush completed. - bool seal = 2; - } + // Request body structure. + message Body { + // ID of the shard. + repeated bytes shard_ID = 1; + // If true, then writecache will be left in read-only mode after flush + // completed. + bool seal = 2; + } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } // FlushCache response. message FlushCacheResponse { - // Response body structure. - message Body { - } + // Response body structure. + message Body {} - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } - // Doctor request. message DoctorRequest { - // Request body structure. - message Body { - // Number of threads to use for the operation. - uint32 concurrency = 1; - // Flag to search engine for duplicate objects and leave only one copy. - bool remove_duplicates = 2; - } + // Request body structure. + message Body { + // Number of threads to use for the operation. + uint32 concurrency = 1; + // Flag to search engine for duplicate objects and leave only one copy. + bool remove_duplicates = 2; + } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } // Doctor response. message DoctorResponse { - // Response body structure. - message Body { - } + // Response body structure. + message Body {} - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } // StartShardEvacuation request. message StartShardEvacuationRequest { - // Request body structure. - message Body { - enum Scope { - NONE = 0; - OBJECTS = 1; - TREES = 2; - } - - // IDs of the shards. - repeated bytes shard_ID = 1; - // Flag indicating whether object read errors should be ignored. - bool ignore_errors = 2; - // Evacuation scope. - uint32 scope = 3; + // Request body structure. + message Body { + enum Scope { + NONE = 0; + OBJECTS = 1; + TREES = 2; } - Body body = 1; - Signature signature = 2; + // IDs of the shards. + repeated bytes shard_ID = 1; + // Flag indicating whether object read errors should be ignored. + bool ignore_errors = 2; + // Evacuation scope. + uint32 scope = 3; + } + + Body body = 1; + Signature signature = 2; } // StartShardEvacuation response. message StartShardEvacuationResponse { - // Response body structure. - message Body {} + // Response body structure. + message Body {} - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } // GetShardEvacuationStatus request. message GetShardEvacuationStatusRequest { - // Request body structure. - message Body {} + // Request body structure. + message Body {} - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } // GetShardEvacuationStatus response. message GetShardEvacuationStatusResponse { - // Response body structure. - message Body { - // Evacuate status enum. - enum Status { - EVACUATE_SHARD_STATUS_UNDEFINED = 0; - RUNNING = 1; - COMPLETED = 2; - } - - // Unix timestamp value. - message UnixTimestamp { - int64 value = 1; - } - - // Duration in seconds. - message Duration { - int64 seconds = 1; - } - - // Total objects to evacuate count. The value is approximate, so evacuated + failed + skipped == total is not guaranteed after completion. - uint64 total_objects = 1; - // Evacuated objects count. - uint64 evacuated_objects = 2; - // Failed objects count. - uint64 failed_objects = 3; - - // Shard IDs. - repeated bytes shard_ID = 4; - // Evacuation process status. - Status status = 5; - // Evacuation process duration. - Duration duration = 6; - // Evacuation process started at timestamp. - UnixTimestamp started_at = 7; - // Error message if evacuation failed. - string error_message = 8; - - // Skipped objects count. - uint64 skipped_objects = 9; - - // Total trees to evacuate count. - uint64 total_trees = 10; - // Evacuated trees count. - uint64 evacuated_trees = 11; - // Failed trees count. - uint64 failed_trees = 12; + // Response body structure. + message Body { + // Evacuate status enum. + enum Status { + EVACUATE_SHARD_STATUS_UNDEFINED = 0; + RUNNING = 1; + COMPLETED = 2; } - Body body = 1; - Signature signature = 2; + // Unix timestamp value. + message UnixTimestamp { int64 value = 1; } + + // Duration in seconds. + message Duration { int64 seconds = 1; } + + // Total objects to evacuate count. The value is approximate, so evacuated + + // failed + skipped == total is not guaranteed after completion. + uint64 total_objects = 1; + // Evacuated objects count. + uint64 evacuated_objects = 2; + // Failed objects count. + uint64 failed_objects = 3; + + // Shard IDs. + repeated bytes shard_ID = 4; + // Evacuation process status. + Status status = 5; + // Evacuation process duration. + Duration duration = 6; + // Evacuation process started at timestamp. + UnixTimestamp started_at = 7; + // Error message if evacuation failed. + string error_message = 8; + + // Skipped objects count. + uint64 skipped_objects = 9; + + // Total trees to evacuate count. + uint64 total_trees = 10; + // Evacuated trees count. + uint64 evacuated_trees = 11; + // Failed trees count. + uint64 failed_trees = 12; + } + + Body body = 1; + Signature signature = 2; } // ResetShardEvacuationStatus request. message ResetShardEvacuationStatusRequest { - message Body {} + message Body {} - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } // ResetShardEvacuationStatus response. message ResetShardEvacuationStatusResponse { - message Body {} + message Body {} - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } // StopShardEvacuation request. message StopShardEvacuationRequest { - // Request body structure. - message Body {} + // Request body structure. + message Body {} - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } // StopShardEvacuation response. message StopShardEvacuationResponse { - // Response body structure. - message Body {} + // Response body structure. + message Body {} - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } // AddChainLocalOverride request. message AddChainLocalOverrideRequest { - message Body { - // Target for which the overrides are applied. - ChainTarget target = 1; + message Body { + // Target for which the overrides are applied. + ChainTarget target = 1; - // Serialized rule chain. If chain ID is left empty - // in the chain, then it will be generated and returned - // in the response. - bytes chain = 2; - } + // Serialized rule chain. If chain ID is left empty + // in the chain, then it will be generated and returned + // in the response. + bytes chain = 2; + } - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } // AddChainLocalOverride response. message AddChainLocalOverrideResponse { - message Body { - // Chain ID assigned for the added rule chain. - // If chain ID is left empty in the request, then - // it will be generated. - bytes chain_id = 1; - } + message Body { + // Chain ID assigned for the added rule chain. + // If chain ID is left empty in the request, then + // it will be generated. + bytes chain_id = 1; + } - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } // GetChainLocalOverride request. message GetChainLocalOverrideRequest { - message Body { - // Target for which the overrides are applied. - ChainTarget target = 1; + message Body { + // Target for which the overrides are applied. + ChainTarget target = 1; - // Chain ID assigned for the added rule chain. - bytes chain_id = 2; - } + // Chain ID assigned for the added rule chain. + bytes chain_id = 2; + } - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } // GetChainLocalOverride response. message GetChainLocalOverrideResponse { - message Body { - // Serialized rule chain. - bytes chain = 1; - } + message Body { + // Serialized rule chain. + bytes chain = 1; + } - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } // ListChainLocalOverrides request. message ListChainLocalOverridesRequest { - message Body { - // Target for which the overrides are applied. - ChainTarget target = 1; - } + message Body { + // Target for which the overrides are applied. + ChainTarget target = 1; + } - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } // ListChainLocalOverrides response. message ListChainLocalOverridesResponse { - message Body { - // The list of serialized rule chain. - repeated bytes chains = 1; - } + message Body { + // The list of serialized rule chain. + repeated bytes chains = 1; + } - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } // ListTargetsLocalOverrides request. message ListTargetsLocalOverridesRequest { - message Body { - // Target for which the overrides are applied. - string chainName = 1; - } + message Body { + // Target for which the overrides are applied. + string chainName = 1; + } - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } // ListTargetsLocalOverrides response. message ListTargetsLocalOverridesResponse { - message Body { - // The list of chain targets. - repeated ChainTarget targets = 1; - } + message Body { + // The list of chain targets. + repeated ChainTarget targets = 1; + } - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } message RemoveChainLocalOverrideRequest { - message Body { - // Target for which the overrides are applied. - ChainTarget target = 1; + message Body { + // Target for which the overrides are applied. + ChainTarget target = 1; - // Chain ID assigned for the added rule chain. - bytes chain_id = 2; - } + // Chain ID assigned for the added rule chain. + bytes chain_id = 2; + } - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } message RemoveChainLocalOverrideResponse { - message Body { - } + message Body {} - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } message RemoveChainLocalOverridesByTargetRequest { - message Body { - // Target for which the overrides are applied. - ChainTarget target = 1; - } + message Body { + // Target for which the overrides are applied. + ChainTarget target = 1; + } - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } -message RemoveChainLocalOverridesByTargetResponse { - message Body { - } +message RemoveChainLocalOverridesByTargetResponse { + message Body {} - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } message SealWriteCacheRequest { - // Request body structure. - message Body { - // ID of the shard. - repeated bytes shard_ID = 1; + // Request body structure. + message Body { + // ID of the shard. + repeated bytes shard_ID = 1; - // Flag indicating whether object read errors should be ignored. - bool ignore_errors = 2; - } + // Flag indicating whether object read errors should be ignored. + bool ignore_errors = 2; + } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } message SealWriteCacheResponse { - message Body { - message Status { - bytes shard_ID = 1; - bool success = 2; - string error = 3; - } - repeated Status results = 1; + message Body { + message Status { + bytes shard_ID = 1; + bool success = 2; + string error = 3; } + repeated Status results = 1; + } - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } message DetachShardsRequest { - message Body { - repeated bytes shard_ID = 1; - } + message Body { repeated bytes shard_ID = 1; } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } message DetachShardsResponse { - message Body { - } + message Body {} - Body body = 1; + Body body = 1; - Signature signature = 2; + Signature signature = 2; } diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto index 00fcb98d1..3306924ac 100644 --- a/pkg/services/control/types.proto +++ b/pkg/services/control/types.proto @@ -6,183 +6,182 @@ option go_package = "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/con // Signature of some message. message Signature { - // Public key used for signing. - bytes key = 1 [json_name = "key"]; + // Public key used for signing. + bytes key = 1 [ json_name = "key" ]; - // Binary signature. - bytes sign = 2 [json_name = "signature"]; + // Binary signature. + bytes sign = 2 [ json_name = "signature" ]; } // Status of the storage node in the FrostFS network map. enum NetmapStatus { - // Undefined status, default value. - STATUS_UNDEFINED = 0; + // Undefined status, default value. + STATUS_UNDEFINED = 0; - // Node is online. - ONLINE = 1; + // Node is online. + ONLINE = 1; - // Node is offline. - OFFLINE = 2; + // Node is offline. + OFFLINE = 2; - // Node is maintained by the owner. - MAINTENANCE = 3; + // Node is maintained by the owner. + MAINTENANCE = 3; } // FrostFS node description. message NodeInfo { - // Public key of the FrostFS node in a binary format. - bytes public_key = 1 [json_name = "publicKey"]; + // Public key of the FrostFS node in a binary format. + bytes public_key = 1 [ json_name = "publicKey" ]; - // Ways to connect to a node. - repeated string addresses = 2 [json_name = "addresses"]; + // Ways to connect to a node. + repeated string addresses = 2 [ json_name = "addresses" ]; - // Administrator-defined Attributes of the FrostFS Storage Node. - // - // `Attribute` is a Key-Value metadata pair. Key name must be a valid UTF-8 - // string. Value can't be empty. - // - // Node's attributes are mostly used during Storage Policy evaluation to - // calculate object's placement and find a set of nodes satisfying policy - // requirements. There are some "well-known" node attributes common to all the - // Storage Nodes in the network and used implicitly with default values if not - // explicitly set: - // - // * Capacity \ + // Administrator-defined Attributes of the FrostFS Storage Node. + // + // `Attribute` is a Key-Value metadata pair. Key name must be a valid UTF-8 + // string. Value can't be empty. + // + // Node's attributes are mostly used during Storage Policy evaluation to + // calculate object's placement and find a set of nodes satisfying policy + // requirements. There are some "well-known" node attributes common to all the + // Storage Nodes in the network and used implicitly with default values if not + // explicitly set: + // + // * Capacity \ // Total available disk space in Gigabytes. - // * Price \ + // * Price \ // Price in GAS tokens for storing one GB of data during one Epoch. In node - // attributes it's a string presenting floating point number with comma or - // point delimiter for decimal part. In the Network Map it will be saved as - // 64-bit unsigned integer representing number of minimal token fractions. - // * Locode \ + // attributes it's a string presenting floating point number with comma or + // point delimiter for decimal part. In the Network Map it will be saved as + // 64-bit unsigned integer representing number of minimal token fractions. + // * Locode \ // Node's geographic location in - // [UN/LOCODE](https://www.unece.org/cefact/codesfortrade/codes_index.html) - // format approximated to the nearest point defined in standard. - // * Country \ + // [UN/LOCODE](https://www.unece.org/cefact/codesfortrade/codes_index.html) + // format approximated to the nearest point defined in standard. + // * Country \ // Country code in - // [ISO 3166-1_alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) - // format. Calculated automatically from `Locode` attribute - // * Region \ + // [ISO 3166-1_alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) + // format. Calculated automatically from `Locode` attribute + // * Region \ // Country's administative subdivision where node is located. Calculated - // automatically from `Locode` attribute based on `SubDiv` field. Presented - // in [ISO 3166-2](https://en.wikipedia.org/wiki/ISO_3166-2) format. - // * City \ + // automatically from `Locode` attribute based on `SubDiv` field. Presented + // in [ISO 3166-2](https://en.wikipedia.org/wiki/ISO_3166-2) format. + // * City \ // City, town, village or rural area name where node is located written - // without diacritics . Calculated automatically from `Locode` attribute. - // - // For detailed description of each well-known attribute please see the - // corresponding section in FrostFS Technical specification. - message Attribute { - // Key of the node attribute. - string key = 1 [json_name = "key"]; + // without diacritics . Calculated automatically from `Locode` attribute. + // + // For detailed description of each well-known attribute please see the + // corresponding section in FrostFS Technical specification. + message Attribute { + // Key of the node attribute. + string key = 1 [ json_name = "key" ]; - // Value of the node attribute. - string value = 2 [json_name = "value"]; + // Value of the node attribute. + string value = 2 [ json_name = "value" ]; - // Parent keys, if any. For example for `City` it could be `Region` and - // `Country`. - repeated string parents = 3 [json_name = "parents"]; - } - // Carries list of the FrostFS node attributes in a key-value form. Key name - // must be a node-unique valid UTF-8 string. Value can't be empty. NodeInfo - // structures with duplicated attribute names or attributes with empty values - // will be considered invalid. - repeated Attribute attributes = 3 [json_name = "attributes"]; + // Parent keys, if any. For example for `City` it could be `Region` and + // `Country`. + repeated string parents = 3 [ json_name = "parents" ]; + } + // Carries list of the FrostFS node attributes in a key-value form. Key name + // must be a node-unique valid UTF-8 string. Value can't be empty. NodeInfo + // structures with duplicated attribute names or attributes with empty values + // will be considered invalid. + repeated Attribute attributes = 3 [ json_name = "attributes" ]; - // Carries state of the FrostFS node. - NetmapStatus state = 4 [json_name = "state"]; + // Carries state of the FrostFS node. + NetmapStatus state = 4 [ json_name = "state" ]; } // Network map structure. message Netmap { - // Network map revision number. - uint64 epoch = 1 [json_name = "epoch"]; + // Network map revision number. + uint64 epoch = 1 [ json_name = "epoch" ]; - // Nodes presented in network. - repeated NodeInfo nodes = 2 [json_name = "nodes"]; + // Nodes presented in network. + repeated NodeInfo nodes = 2 [ json_name = "nodes" ]; } // Health status of the storage node application. enum HealthStatus { - // Undefined status, default value. - HEALTH_STATUS_UNDEFINED = 0; + // Undefined status, default value. + HEALTH_STATUS_UNDEFINED = 0; - // Storage node application is starting. - STARTING = 1; + // Storage node application is starting. + STARTING = 1; - // Storage node application is started and serves all services. - READY = 2; + // Storage node application is started and serves all services. + READY = 2; - // Storage node application is shutting down. - SHUTTING_DOWN = 3; + // Storage node application is shutting down. + SHUTTING_DOWN = 3; - // Storage node application is reconfiguring. - RECONFIGURING = 4; + // Storage node application is reconfiguring. + RECONFIGURING = 4; } // Shard description. message ShardInfo { - // ID of the shard. - bytes shard_ID = 1 [json_name = "shardID"]; + // ID of the shard. + bytes shard_ID = 1 [ json_name = "shardID" ]; - // Path to shard's metabase. - string metabase_path = 2 [json_name = "metabasePath"]; + // Path to shard's metabase. + string metabase_path = 2 [ json_name = "metabasePath" ]; - // Shard's blobstor info. - repeated BlobstorInfo blobstor = 3 [json_name = "blobstor"]; + // Shard's blobstor info. + repeated BlobstorInfo blobstor = 3 [ json_name = "blobstor" ]; - // Path to shard's write-cache, empty if disabled. - string writecache_path = 4 [json_name = "writecachePath"]; + // Path to shard's write-cache, empty if disabled. + string writecache_path = 4 [ json_name = "writecachePath" ]; - // Work mode of the shard. - ShardMode mode = 5; + // Work mode of the shard. + ShardMode mode = 5; - // Amount of errors occured. - uint32 errorCount = 6; + // Amount of errors occured. + uint32 errorCount = 6; - // Path to shard's pilorama storage. - string pilorama_path = 7 [json_name = "piloramaPath"]; + // Path to shard's pilorama storage. + string pilorama_path = 7 [ json_name = "piloramaPath" ]; } // Blobstor component description. message BlobstorInfo { - // Path to the root. - string path = 1 [json_name = "path"]; - // Component type. - string type = 2 [json_name = "type"]; + // Path to the root. + string path = 1 [ json_name = "path" ]; + // Component type. + string type = 2 [ json_name = "type" ]; } // Work mode of the shard. enum ShardMode { - // Undefined mode, default value. - SHARD_MODE_UNDEFINED = 0; + // Undefined mode, default value. + SHARD_MODE_UNDEFINED = 0; - // Read-write. - READ_WRITE = 1; + // Read-write. + READ_WRITE = 1; - // Read-only. - READ_ONLY = 2; + // Read-only. + READ_ONLY = 2; - // Degraded. - DEGRADED = 3; + // Degraded. + DEGRADED = 3; - // DegradedReadOnly. - DEGRADED_READ_ONLY = 4; + // DegradedReadOnly. + DEGRADED_READ_ONLY = 4; } - // ChainTarget is an object to which local overrides // are applied. message ChainTarget { - enum TargetType { - UNDEFINED = 0; + enum TargetType { + UNDEFINED = 0; - NAMESPACE = 1; + NAMESPACE = 1; - CONTAINER = 2; - } + CONTAINER = 2; + } - TargetType type = 1; + TargetType type = 1; - string Name = 2; + string Name = 2; } From 3ea1d7b7290a2dad4fa2992772006921e848733a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 11 Apr 2024 17:06:09 +0300 Subject: [PATCH 1043/1943] [#1089] control: Add USER and GROUP targets for local override storage Signed-off-by: Evgenii Stratonikov --- pkg/services/control/server/policy_engine.go | 14 +++++ pkg/services/control/service.pb.go | 6 +- pkg/services/control/service_grpc.pb.go | 30 ++++++---- pkg/services/control/types.pb.go | 59 +++++++++++--------- pkg/services/control/types.proto | 4 ++ 5 files changed, 75 insertions(+), 38 deletions(-) diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 16b365b21..7ec3d58ac 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -19,6 +19,10 @@ func apeTarget(chainTarget *control.ChainTarget) (engine.Target, error) { return engine.ContainerTarget(chainTarget.GetName()), nil case control.ChainTarget_NAMESPACE: return engine.NamespaceTarget(chainTarget.GetName()), nil + case control.ChainTarget_USER: + return engine.UserTarget(chainTarget.GetName()), nil + case control.ChainTarget_GROUP: + return engine.GroupTarget(chainTarget.GetName()), nil default: } return engine.Target{}, status.Error(codes.InvalidArgument, @@ -42,6 +46,16 @@ func controlTarget(chainTarget *engine.Target) (control.ChainTarget, error) { Name: nm, Type: control.ChainTarget_NAMESPACE, }, nil + case engine.User: + return control.ChainTarget{ + Name: chainTarget.Name, + Type: control.ChainTarget_USER, + }, nil + case engine.Group: + return control.ChainTarget{ + Name: chainTarget.Name, + Type: control.ChainTarget_GROUP, + }, nil default: } return control.ChainTarget{}, status.Error(codes.InvalidArgument, diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 345110bab..9c597beec 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -3188,7 +3188,8 @@ type FlushCacheRequest_Body struct { // ID of the shard. Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` - // If true, then writecache will be left in read-only mode after flush completed. + // If true, then writecache will be left in read-only mode after flush + // completed. Seal bool `protobuf:"varint,2,opt,name=seal,proto3" json:"seal,omitempty"` } @@ -3525,7 +3526,8 @@ type GetShardEvacuationStatusResponse_Body struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Total objects to evacuate count. The value is approximate, so evacuated + failed + skipped == total is not guaranteed after completion. + // Total objects to evacuate count. The value is approximate, so evacuated + + // failed + skipped == total is not guaranteed after completion. TotalObjects uint64 `protobuf:"varint,1,opt,name=total_objects,json=totalObjects,proto3" json:"total_objects,omitempty"` // Evacuated objects count. EvacuatedObjects uint64 `protobuf:"varint,2,opt,name=evacuated_objects,json=evacuatedObjects,proto3" json:"evacuated_objects,omitempty"` diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 95264fcd3..feeee0006 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -59,13 +59,15 @@ type ControlServiceClient interface { // Synchronizes all log operations for the specified tree. SynchronizeTree(ctx context.Context, in *SynchronizeTreeRequest, opts ...grpc.CallOption) (*SynchronizeTreeResponse, error) // EvacuateShard moves all data from one shard to the others. - // Deprecated: Use StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation + // Deprecated: Use + // StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation EvacuateShard(ctx context.Context, in *EvacuateShardRequest, opts ...grpc.CallOption) (*EvacuateShardResponse, error) // StartShardEvacuation starts moving all data from one shard to the others. StartShardEvacuation(ctx context.Context, in *StartShardEvacuationRequest, opts ...grpc.CallOption) (*StartShardEvacuationResponse, error) // GetShardEvacuationStatus returns evacuation status. GetShardEvacuationStatus(ctx context.Context, in *GetShardEvacuationStatusRequest, opts ...grpc.CallOption) (*GetShardEvacuationStatusResponse, error) - // ResetShardEvacuationStatus resets evacuation status if there is no running evacuation process. + // ResetShardEvacuationStatus resets evacuation status if there is no running + // evacuation process. ResetShardEvacuationStatus(ctx context.Context, in *ResetShardEvacuationStatusRequest, opts ...grpc.CallOption) (*ResetShardEvacuationStatusResponse, error) // StopShardEvacuation stops moving all data from one shard to the others. StopShardEvacuation(ctx context.Context, in *StopShardEvacuationRequest, opts ...grpc.CallOption) (*StopShardEvacuationResponse, error) @@ -77,11 +79,14 @@ type ControlServiceClient interface { AddChainLocalOverride(ctx context.Context, in *AddChainLocalOverrideRequest, opts ...grpc.CallOption) (*AddChainLocalOverrideResponse, error) // Get local access policy engine overrides stored in the node by chain id. GetChainLocalOverride(ctx context.Context, in *GetChainLocalOverrideRequest, opts ...grpc.CallOption) (*GetChainLocalOverrideResponse, error) - // List local access policy engine overrides stored in the node by container id. + // List local access policy engine overrides stored in the node by container + // id. ListChainLocalOverrides(ctx context.Context, in *ListChainLocalOverridesRequest, opts ...grpc.CallOption) (*ListChainLocalOverridesResponse, error) - // Remove local access policy engine overrides stored in the node by chaind id. + // Remove local access policy engine overrides stored in the node by chaind + // id. RemoveChainLocalOverride(ctx context.Context, in *RemoveChainLocalOverrideRequest, opts ...grpc.CallOption) (*RemoveChainLocalOverrideResponse, error) - // Remove local access policy engine overrides stored in the node by chaind id. + // Remove local access policy engine overrides stored in the node by chaind + // id. RemoveChainLocalOverridesByTarget(ctx context.Context, in *RemoveChainLocalOverridesByTargetRequest, opts ...grpc.CallOption) (*RemoveChainLocalOverridesByTargetResponse, error) // List targets of the local APE overrides stored in the node. ListTargetsLocalOverrides(ctx context.Context, in *ListTargetsLocalOverridesRequest, opts ...grpc.CallOption) (*ListTargetsLocalOverridesResponse, error) @@ -305,13 +310,15 @@ type ControlServiceServer interface { // Synchronizes all log operations for the specified tree. SynchronizeTree(context.Context, *SynchronizeTreeRequest) (*SynchronizeTreeResponse, error) // EvacuateShard moves all data from one shard to the others. - // Deprecated: Use StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation + // Deprecated: Use + // StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation EvacuateShard(context.Context, *EvacuateShardRequest) (*EvacuateShardResponse, error) // StartShardEvacuation starts moving all data from one shard to the others. StartShardEvacuation(context.Context, *StartShardEvacuationRequest) (*StartShardEvacuationResponse, error) // GetShardEvacuationStatus returns evacuation status. GetShardEvacuationStatus(context.Context, *GetShardEvacuationStatusRequest) (*GetShardEvacuationStatusResponse, error) - // ResetShardEvacuationStatus resets evacuation status if there is no running evacuation process. + // ResetShardEvacuationStatus resets evacuation status if there is no running + // evacuation process. ResetShardEvacuationStatus(context.Context, *ResetShardEvacuationStatusRequest) (*ResetShardEvacuationStatusResponse, error) // StopShardEvacuation stops moving all data from one shard to the others. StopShardEvacuation(context.Context, *StopShardEvacuationRequest) (*StopShardEvacuationResponse, error) @@ -323,11 +330,14 @@ type ControlServiceServer interface { AddChainLocalOverride(context.Context, *AddChainLocalOverrideRequest) (*AddChainLocalOverrideResponse, error) // Get local access policy engine overrides stored in the node by chain id. GetChainLocalOverride(context.Context, *GetChainLocalOverrideRequest) (*GetChainLocalOverrideResponse, error) - // List local access policy engine overrides stored in the node by container id. + // List local access policy engine overrides stored in the node by container + // id. ListChainLocalOverrides(context.Context, *ListChainLocalOverridesRequest) (*ListChainLocalOverridesResponse, error) - // Remove local access policy engine overrides stored in the node by chaind id. + // Remove local access policy engine overrides stored in the node by chaind + // id. RemoveChainLocalOverride(context.Context, *RemoveChainLocalOverrideRequest) (*RemoveChainLocalOverrideResponse, error) - // Remove local access policy engine overrides stored in the node by chaind id. + // Remove local access policy engine overrides stored in the node by chaind + // id. RemoveChainLocalOverridesByTarget(context.Context, *RemoveChainLocalOverridesByTargetRequest) (*RemoveChainLocalOverridesByTargetResponse, error) // List targets of the local APE overrides stored in the node. ListTargetsLocalOverrides(context.Context, *ListTargetsLocalOverridesRequest) (*ListTargetsLocalOverridesResponse, error) diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index 2fc16a926..858755694 100644 --- a/pkg/services/control/types.pb.go +++ b/pkg/services/control/types.pb.go @@ -205,6 +205,8 @@ const ( ChainTarget_UNDEFINED ChainTarget_TargetType = 0 ChainTarget_NAMESPACE ChainTarget_TargetType = 1 ChainTarget_CONTAINER ChainTarget_TargetType = 2 + ChainTarget_USER ChainTarget_TargetType = 3 + ChainTarget_GROUP ChainTarget_TargetType = 4 ) // Enum value maps for ChainTarget_TargetType. @@ -213,11 +215,15 @@ var ( 0: "UNDEFINED", 1: "NAMESPACE", 2: "CONTAINER", + 3: "USER", + 4: "GROUP", } ChainTarget_TargetType_value = map[string]int32{ "UNDEFINED": 0, "NAMESPACE": 1, "CONTAINER": 2, + "USER": 3, + "GROUP": 4, } ) @@ -814,40 +820,41 @@ var file_pkg_services_control_types_proto_rawDesc = []byte{ 0x6d, 0x61, 0x50, 0x61, 0x74, 0x68, 0x22, 0x36, 0x0a, 0x0c, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x91, + 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xa6, 0x01, 0x0a, 0x0b, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x33, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x39, 0x0a, 0x0a, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x4e, 0x0a, 0x0a, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, - 0x10, 0x02, 0x2a, 0x4e, 0x0a, 0x0c, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, - 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x4e, 0x4c, 0x49, - 0x4e, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x46, 0x46, 0x4c, 0x49, 0x4e, 0x45, 0x10, - 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, - 0x10, 0x03, 0x2a, 0x6a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x17, 0x48, 0x45, 0x41, 0x4c, 0x54, 0x48, 0x5f, 0x53, 0x54, 0x41, - 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, - 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, - 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x48, 0x55, 0x54, - 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x52, - 0x45, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x55, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x04, 0x2a, 0x6a, - 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x53, - 0x48, 0x41, 0x52, 0x44, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, - 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x57, 0x52, - 0x49, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, - 0x4c, 0x59, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, - 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x5f, 0x52, - 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x04, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, - 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, - 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, - 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x55, 0x53, 0x45, 0x52, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, + 0x47, 0x52, 0x4f, 0x55, 0x50, 0x10, 0x04, 0x2a, 0x4e, 0x0a, 0x0c, 0x4e, 0x65, 0x74, 0x6d, 0x61, + 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, + 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, + 0x06, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x46, 0x46, + 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, + 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x2a, 0x6a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x17, 0x48, 0x45, 0x41, 0x4c, 0x54, + 0x48, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, + 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11, 0x0a, + 0x0d, 0x53, 0x48, 0x55, 0x54, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x03, + 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x55, 0x52, 0x49, 0x4e, + 0x47, 0x10, 0x04, 0x2a, 0x6a, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, + 0x12, 0x18, 0x0a, 0x14, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, + 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, + 0x41, 0x44, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, + 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, + 0x52, 0x41, 0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x47, 0x52, 0x41, + 0x44, 0x45, 0x44, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x04, 0x42, + 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, + 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, + 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, + 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto index 3306924ac..55636d88a 100644 --- a/pkg/services/control/types.proto +++ b/pkg/services/control/types.proto @@ -179,6 +179,10 @@ message ChainTarget { NAMESPACE = 1; CONTAINER = 2; + + USER = 3; + + GROUP = 4; } TargetType type = 1; From 46bc6a79308383482711845fa71a11be28b96465 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 15 Apr 2024 10:47:45 +0300 Subject: [PATCH 1044/1943] [#1095] cli: Support user/group target for local overrides Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/control/list_rules.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index b345ecbfb..f5fc27bda 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -27,6 +27,8 @@ const ( defaultNamespace = "root" namespaceTarget = "namespace" containerTarget = "container" + userTarget = "user" + groupTarget = "group" ) const ( @@ -66,6 +68,16 @@ func parseTarget(cmd *cobra.Command) *control.ChainTarget { Name: name, Type: control.ChainTarget_CONTAINER, } + case userTarget: + return &control.ChainTarget{ + Name: name, + Type: control.ChainTarget_USER, + } + case groupTarget: + return &control.ChainTarget{ + Name: name, + Type: control.ChainTarget_GROUP, + } default: commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType) } From 97e54066d03f5037a0c3d37f9fe737b5ca1d5eb0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 16 Apr 2024 13:19:16 +0300 Subject: [PATCH 1045/1943] [#1095] adm: Support user/group target for APE Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/ape/ape.go | 2 ++ cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go index 710595fe8..077e03737 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go @@ -16,6 +16,8 @@ import ( const ( namespaceTarget = "namespace" containerTarget = "container" + userTarget = "user" + groupTarget = "group" jsonFlag = "json" jsonFlagDesc = "Output rule chains in JSON format" chainIDFlag = "chain-id" diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index b3610e7f7..42307e78f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -53,6 +53,10 @@ func parseTargetType(cmd *cobra.Command) (policyengine.TargetType, error) { return policyengine.Namespace, nil case containerTarget: return policyengine.Container, nil + case userTarget: + return policyengine.User, nil + case groupTarget: + return policyengine.Group, nil } return -1, errUnknownTargetType } From 6772976657f7ec4a2263cc9f49abb65aa9943831 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 15 Apr 2024 15:54:16 +0300 Subject: [PATCH 1046/1943] [#1096] container: Make ape middleware fill request with user claim tags Signed-off-by: Airat Arifullin --- pkg/services/container/ape.go | 46 +++++++++++++-- pkg/services/container/ape_test.go | 89 +++++++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 5 deletions(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 02549bded..2ce427fa3 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -27,11 +27,16 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" ) +const ( + subjectNotFoundErrorMessage = "subject not found" +) + var ( errMissingContainerID = errors.New("missing container ID") errSessionContainerMissmatch = errors.New("requested container is not related to the session") @@ -40,7 +45,6 @@ var ( errInvalidSessionTokenOwner = errors.New("malformed request: invalid session token owner") errEmptyBodySignature = errors.New("malformed request: empty body signature") errMissingOwnerID = errors.New("malformed request: missing owner ID") - errSubjectNotFound = errors.New("subject not found") errOwnerIDIsNotSet = errors.New("owner id is not set") errInvalidDomainZone = errors.New("invalid domain zone: no namespace is expected") @@ -140,6 +144,11 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co nativeschema.PropertyKeyActorRole: role, } + reqProps, err = ac.fillWithUserClaimTags(reqProps, pk) + if err != nil { + return nil, err + } + namespace, err := ac.namespaceByOwner(req.GetBody().GetOwnerID()) if err != nil { return nil, fmt.Errorf("could not get owner namespace: %w", err) @@ -188,6 +197,11 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont nativeschema.PropertyKeyActorRole: role, } + reqProps, err = ac.fillWithUserClaimTags(reqProps, pk) + if err != nil { + return nil, err + } + namespace, err := ac.namespaceByOwner(req.GetBody().GetContainer().GetOwnerID()) if err != nil { return nil, fmt.Errorf("get namespace error: %w", err) @@ -365,10 +379,15 @@ func (ac *apeChecker) getRequestProps(mh *session.RequestMetaHeader, vh *session if err != nil { return nil, nil, err } - return map[string]string{ + reqProps := map[string]string{ nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(pk.Bytes()), nativeschema.PropertyKeyActorRole: role, - }, pk, nil + } + reqProps, err = ac.fillWithUserClaimTags(reqProps, pk) + if err != nil { + return nil, nil, err + } + return reqProps, pk, nil } func (ac *apeChecker) getRole(actor *user.ID, pk *keys.PublicKey, cont *containercore.Container, cnrID cid.ID) (string, error) { @@ -555,7 +574,7 @@ func (ac *apeChecker) namespaceByOwner(owner *refs.OwnerID) (string, error) { if err == nil { namespace = subject.Namespace } else { - if !strings.Contains(err.Error(), errSubjectNotFound.Error()) { + if !strings.Contains(err.Error(), subjectNotFoundErrorMessage) { return "", fmt.Errorf("get subject error: %w", err) } } @@ -603,3 +622,22 @@ func (ac *apeChecker) validateNamespaceByPublicKey(pk *keys.PublicKey, ownerIDNa } return nil } + +// fillWithUserClaimTags fills ape request properties with user claim tags getting them from frostfsid contract by actor public key. +func (ac *apeChecker) fillWithUserClaimTags(reqProps map[string]string, pk *keys.PublicKey) (map[string]string, error) { + if reqProps == nil { + reqProps = make(map[string]string) + } + subj, err := ac.frostFSIDClient.GetSubject(pk.GetScriptHash()) + if err != nil { + if !strings.Contains(err.Error(), subjectNotFoundErrorMessage) { + return nil, fmt.Errorf("get subject error: %w", err) + } + return reqProps, nil + } + for k, v := range subj.KV { + properyKey := fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, k) + reqProps[properyKey] = v + } + return reqProps, nil +} diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index 1af59d1c6..e26e82d43 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -26,6 +26,7 @@ import ( "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" + commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" @@ -42,6 +43,7 @@ func TestAPE(t *testing.T) { t.Run("allow then deny get container", testAllowThenDenyGetContainerRuleDefined) t.Run("deny get container no rule found", testDenyGetContainerNoRuleFound) t.Run("deny get container for others", testDenyGetContainerForOthers) + t.Run("deny get container by user claim tag", testDenyGetContainerByUserClaimTag) t.Run("deny set container eACL for IR", testDenySetContainerEACLForIR) t.Run("deny get container eACL for IR with session token", testDenyGetContainerEACLForIRSessionToken) t.Run("deny put container for others with session token", testDenyPutContainerForOthersSessionToken) @@ -251,6 +253,91 @@ func testDenyGetContainerForOthers(t *testing.T) { require.ErrorAs(t, err, &errAccessDenied) } +func testDenyGetContainerByUserClaimTag(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{ + pk.PublicKey().GetScriptHash(): { + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + }, + }, + } + + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) + + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + contRdr.c[contID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(nm.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + nm.netmaps[nm.currentEpoch] = &testNetmap + nm.netmaps[nm.currentEpoch-1] = &testNetmap + + _, _, err = router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, "tag-attr1"), + Value: "value100", + Op: chain.CondStringNotEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := &container.GetRequest{} + req.SetBody(&container.GetRequestBody{}) + var refContID refs.ContainerID + contID.WriteToV2(&refContID) + req.GetBody().SetContainerID(&refContID) + + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + resp, err := apeSrv.Get(context.Background(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) +} + func testDenySetContainerEACLForIR(t *testing.T) { t.Parallel() srv := &srvStub{ @@ -841,7 +928,7 @@ type frostfsidStub struct { func (f *frostfsidStub) GetSubject(owner util.Uint160) (*client.Subject, error) { s, ok := f.subjects[owner] if !ok { - return nil, errSubjectNotFound + return nil, fmt.Errorf("%s", subjectNotFoundErrorMessage) } return s, nil } From c21d72ac238c49f0e482e5ed4a19b45035c7de0d Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 15 Apr 2024 16:51:19 +0300 Subject: [PATCH 1047/1943] [#1096] object: Make ape middleware fill request with user claim tags Signed-off-by: Airat Arifullin --- cmd/frostfs-node/config.go | 3 ++ cmd/frostfs-node/container.go | 3 +- cmd/frostfs-node/object.go | 1 + pkg/services/object/ape/checker.go | 12 ++++++- pkg/services/object/ape/checker_test.go | 40 +++++++++++++++++++++- pkg/services/object/ape/request.go | 45 ++++++++++++++++++++++--- pkg/services/object/ape/request_test.go | 11 ++++-- 7 files changed, 105 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c5ed8b50c..066d63e65 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -46,6 +46,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" containerClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + frostfsidClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmap2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" @@ -415,6 +416,8 @@ type shared struct { cnrClient *containerClient.Client + frostfsidClient *frostfsidClient.Client + respSvc *response.Service replicator *replicator.Replicator diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index d566c0af3..e006648ed 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -35,12 +35,13 @@ func initContainerService(_ context.Context, c *cfg) { frostFSIDClient, err := frostfsid.NewFromMorph(c.cfgMorph.client, c.cfgFrostfsID.scriptHash, 0) fatalOnErr(err) + c.shared.frostfsidClient = frostFSIDClient server := containerTransportGRPC.New( containerService.NewSignService( &c.key.PrivateKey, containerService.NewAPEServer(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine, cnrRdr, - newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, frostFSIDClient, + newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, c.shared.frostfsidClient, containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), ), ), diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 7f1d094fd..6160bbc76 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -444,6 +444,7 @@ func createAPEService(c *cfg, splitSvc *objectService.TransportSplitter) *object objectAPE.NewChecker( c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.chainRouter, objectAPE.NewStorageEngineHeaderProvider(c.cfgObject.cfgLocalStorage.localStorage), + c.shared.frostfsidClient, ), splitSvc, ) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 1063bd901..170787317 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -6,6 +6,7 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -13,19 +14,28 @@ import ( policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" ) type checkerImpl struct { chainRouter policyengine.ChainRouter headerProvider HeaderProvider + + frostFSIDClient frostfsidSubjectProvider } -func NewChecker(chainRouter policyengine.ChainRouter, headerProvider HeaderProvider) Checker { +type frostfsidSubjectProvider interface { + GetSubject(util.Uint160) (*client.Subject, error) +} + +func NewChecker(chainRouter policyengine.ChainRouter, headerProvider HeaderProvider, frostFSIDClient frostfsidSubjectProvider) Checker { return &checkerImpl{ chainRouter: chainRouter, headerProvider: headerProvider, + + frostFSIDClient: frostFSIDClient, } } diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index fc915715c..9510bf039 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -6,6 +6,7 @@ import ( "fmt" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -17,6 +18,7 @@ import ( "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/stretchr/testify/require" ) @@ -153,6 +155,41 @@ var ( senderKey = hex.EncodeToString(senderPrivateKey.PublicKey().Bytes()) ) +type frostfsIDProviderMock struct { + m map[util.Uint160]*client.Subject +} + +var _ frostfsidSubjectProvider = (*frostfsIDProviderMock)(nil) + +func newFrostfsIDProviderMock(t *testing.T) *frostfsIDProviderMock { + return &frostfsIDProviderMock{ + m: map[util.Uint160]*client.Subject{ + scriptHashFromSenderKey(t, senderKey): { + Namespace: "testnamespace", + Name: "test", + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + }, + }, + } +} + +func scriptHashFromSenderKey(t *testing.T, senderKey string) util.Uint160 { + pk, err := keys.NewPublicKeyFromString(senderKey) + require.NoError(t, err) + return pk.GetScriptHash() +} + +func (f *frostfsIDProviderMock) GetSubject(key util.Uint160) (*client.Subject, error) { + v, ok := f.m[key] + if !ok { + return nil, fmt.Errorf("%s", subjectNotFoundErrorMessage) + } + return v, nil +} + func TestAPECheck(t *testing.T) { for _, test := range []struct { name string @@ -321,6 +358,7 @@ func TestAPECheck(t *testing.T) { for _, method := range test.methods { t.Run(method, func(t *testing.T) { headerProvider := newHeaderProviderMock() + frostfsidProvider := newFrostfsIDProviderMock(t) cnr := newContainerIDSDK(t, test.container) obj := newObjectIDSDK(t, test.object) @@ -335,7 +373,7 @@ func TestAPECheck(t *testing.T) { router := policyengine.NewDefaultChainRouterWithLocalOverrides(ms, ls) - checker := NewChecker(router, headerProvider) + checker := NewChecker(router, headerProvider, frostfsidProvider) prm := Prm{ Method: method, diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index 7bbee31c1..5daf8751b 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strconv" + "strings" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" @@ -12,7 +13,13 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +const ( + subjectNotFoundErrorMessage = "subject not found" ) var defaultRequest = aperequest.Request{} @@ -115,15 +122,45 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re } } + reqProps := map[string]string{ + nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, + nativeschema.PropertyKeyActorRole: prm.Role, + } + var err error + reqProps, err = c.fillWithUserClaimTags(reqProps, prm) + if err != nil { + return defaultRequest, err + } + return aperequest.NewRequest( prm.Method, aperequest.NewResource( resourceName(prm.Container, prm.Object, prm.Namespace), objectProperties(prm.Container, prm.Object, prm.ContainerOwner, header), ), - map[string]string{ - nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, - nativeschema.PropertyKeyActorRole: prm.Role, - }, + reqProps, ), nil } + +// fillWithUserClaimTags fills ape request properties with user claim tags getting them from frostfsid contract by actor public key. +func (c *checkerImpl) fillWithUserClaimTags(reqProps map[string]string, prm Prm) (map[string]string, error) { + if reqProps == nil { + reqProps = make(map[string]string) + } + pk, err := keys.NewPublicKeyFromString(prm.SenderKey) + if err != nil { + return nil, err + } + subj, err := c.frostFSIDClient.GetSubject(pk.GetScriptHash()) + if err != nil { + if !strings.Contains(err.Error(), subjectNotFoundErrorMessage) { + return nil, fmt.Errorf("get subject error: %w", err) + } + return reqProps, nil + } + for k, v := range subj.KV { + properyKey := fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, k) + reqProps[properyKey] = v + } + return reqProps, nil +} diff --git a/pkg/services/object/ape/request_test.go b/pkg/services/object/ape/request_test.go index fdb7af219..cda78e69c 100644 --- a/pkg/services/object/ape/request_test.go +++ b/pkg/services/object/ape/request_test.go @@ -11,6 +11,7 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" + commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/stretchr/testify/require" ) @@ -234,6 +235,7 @@ func TestNewAPERequest(t *testing.T) { } headerSource := newHeaderProviderMock() + ffidProvider := newFrostfsIDProviderMock(t) var headerObjSDK *objectSDK.Object if test.header.headerObjSDK != nil { @@ -247,7 +249,8 @@ func TestNewAPERequest(t *testing.T) { } c := checkerImpl{ - headerProvider: headerSource, + headerProvider: headerSource, + frostFSIDClient: ffidProvider, } r, err := c.newAPERequest(context.TODO(), prm) @@ -268,8 +271,10 @@ func TestNewAPERequest(t *testing.T) { return prm.Header }())), map[string]string{ - nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, - nativeschema.PropertyKeyActorRole: prm.Role, + nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, + nativeschema.PropertyKeyActorRole: prm.Role, + fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, "tag-attr1"): "value1", + fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, "tag-attr2"): "value2", }, ) From 10ee865e98e87106c42f764849faa1adb0a5c6dc Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 15 Apr 2024 17:04:34 +0300 Subject: [PATCH 1048/1943] [#1096] tree: Make `verifyClient` fill ape request with user claim tags Signed-off-by: Airat Arifullin --- cmd/frostfs-node/tree.go | 1 + pkg/services/tree/ape.go | 28 ++++++++++++++++++++++++++++ pkg/services/tree/options.go | 27 ++++++++++++++++++++------- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index ff00b6fac..49ecb6fdc 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -53,6 +53,7 @@ func initTreeService(c *cfg) { src: c.cfgObject.cnrSource, cli: c.shared.cnrClient, }), + tree.WithFrostfsidSubjectProvider(c.shared.frostfsidClient), tree.WithEACLSource(c.cfgObject.eaclSource), tree.WithNetmapSource(c.netMapSource), tree.WithPrivateKey(&c.key.PrivateKey), diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 5da49a591..f58721509 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -14,10 +14,15 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) +var ( + subjectNotFoundErrorMessage = "subject not found" +) + func (s *Service) checkAPE(container *core.Container, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey) error { namespace := "" cntNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(container.Value).Zone(), ".ns") @@ -37,6 +42,10 @@ func (s *Service) checkAPE(container *core.Container, cid cid.ID, operation acl. nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(publicKey.Bytes()), nativeschema.PropertyKeyActorRole: schemaRole, } + reqProps, err = s.fillWithUserClaimTags(reqProps, publicKey) + if err != nil { + return apeErr(err) + } var resourceName string if namespace == "root" || namespace == "" { @@ -68,3 +77,22 @@ func apeErr(err error) error { errAccessDenied.WriteReason(err.Error()) return errAccessDenied } + +// fillWithUserClaimTags fills ape request properties with user claim tags getting them from frostfsid contract by actor public key. +func (s *Service) fillWithUserClaimTags(reqProps map[string]string, publicKey *keys.PublicKey) (map[string]string, error) { + if reqProps == nil { + reqProps = make(map[string]string) + } + subj, err := s.frostfsidSubjectProvider.GetSubject(publicKey.GetScriptHash()) + if err != nil { + if !strings.Contains(err.Error(), subjectNotFoundErrorMessage) { + return nil, fmt.Errorf("get subject error: %w", err) + } + return reqProps, nil + } + for k, v := range subj.KV { + properyKey := fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, k) + reqProps[properyKey] = v + } + return reqProps, nil +} diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index 0e5f64274..c4ef80856 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -4,6 +4,7 @@ import ( "crypto/ecdsa" "time" + "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -11,8 +12,13 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" ) +type FrostfsidSubjectProvider interface { + GetSubject(util.Uint160) (*client.Subject, error) +} + type ContainerSource interface { container.Source @@ -25,13 +31,14 @@ type ContainerSource interface { } type cfg struct { - log *logger.Logger - key *ecdsa.PrivateKey - rawPub []byte - nmSource netmap.Source - cnrSource ContainerSource - eaclSource container.EACLSource - forest pilorama.Forest + log *logger.Logger + key *ecdsa.PrivateKey + rawPub []byte + nmSource netmap.Source + cnrSource ContainerSource + frostfsidSubjectProvider FrostfsidSubjectProvider + eaclSource container.EACLSource + forest pilorama.Forest // replication-related parameters replicatorChannelCapacity int replicatorWorkerCount int @@ -55,6 +62,12 @@ func WithContainerSource(src ContainerSource) Option { } } +func WithFrostfsidSubjectProvider(provider FrostfsidSubjectProvider) Option { + return func(c *cfg) { + c.frostfsidSubjectProvider = provider + } +} + // WithEACLSource sets a eACL table source for a tree service. // This option is required. func WithEACLSource(src container.EACLSource) Option { From 6d4583f5de0dea88e1e57704e2d630b34d745636 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 16 Apr 2024 14:01:25 +0300 Subject: [PATCH 1049/1943] [#1097] docs: Describe authentication mechanisms Signed-off-by: Evgenii Stratonikov --- docs/authentication.md | 57 ++++++++++++++++++++ docs/images/authentication/authoverview.puml | 28 ++++++++++ docs/images/authentication/authoverview.svg | 1 + 3 files changed, 86 insertions(+) create mode 100644 docs/authentication.md create mode 100644 docs/images/authentication/authoverview.puml create mode 100644 docs/images/authentication/authoverview.svg diff --git a/docs/authentication.md b/docs/authentication.md new file mode 100644 index 000000000..4efb03f93 --- /dev/null +++ b/docs/authentication.md @@ -0,0 +1,57 @@ +# Authentication and signatures + +## General overview + +![Auth general overview](images/authentication/authoverview.svg) + +## Signatures + +Every message in the FrostFS network is signed. +Each signature consists of: +1. Scheme +2. Public key +3. Signature + +If signature check fails, operation is aborted and the error is returned to the user. + +### Schemes +Currently, 3 schemes are defined in the [frostfs-api](https://git.frostfs.info/TrueCloudLab/frostfs-api/src/commit/4bae9dd78abcf1a358a65a45fe7303e37fd98099/refs/types.proto#L105): + +#### ECDSA + +Defined in section 6 of [FIPS 186](https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-5.pdf). +Implemented in the Go stdlib. +This is the primary algorithm used to sign and verify requests. +The hash algorithm used is SHA-512. + +#### RFC6979 + +[RFC 6979](https://www.rfc-editor.org/rfc/rfc6979) defines deterministic algorithm for ECDSA signatures. +It it used primarily used by neo-go and allows us to perform signature checks inside the contract, such as for container. +The hash algorithm used is SHA-256 + +### Public key + +ECDSA public key corresponding to the private key being used to sign a message. +It is the primary user identity and is used to determine the request originator. + +## Session token + +Session token can override the rules of determining request owner. +It is defined in the [frostfs-api](https://git.frostfs.info/TrueCloudLab/frostfs-api/src/branch/master/session/types.proto#L89). +If user A signs a session token for user B, then user B can sign request with its own key, while the node will still process the request as if it was originated from user A. +This is used, for example, when putting objects in system: +1. User creates a session with node, recevieving session token. +2. User signs session token for a freshly generated key, stored on a storage node. +3. User sends raw stream of bytes, while the node signs created objects with the session key. This way other nodes can validate the object owned by user, even though it is signed by a different key. + +Session token may have some restrictions: +1. Lifetime, effectively an epoch after which it becomes invalid. +2. Set of operations it applies to. +3. The entity it is given to. This is provided in `session_key` field containing the public key. + +## Bearer token + +## FrostFS ID + +## APE \ No newline at end of file diff --git a/docs/images/authentication/authoverview.puml b/docs/images/authentication/authoverview.puml new file mode 100644 index 000000000..1242fdfc6 --- /dev/null +++ b/docs/images/authentication/authoverview.puml @@ -0,0 +1,28 @@ +@startuml authoverview +!include +!include +AddElementTag("smart-contract", $bgColor=#0abab5) + +Person(user, "User", "User with private key") + +Container_Boundary(stor, "FrostFS Storage") { + Component(verify, "Sign Service", $descr="Check request signature") + Component(apesvc, "APE Service") + Component(objsvc, "Object service") +} + +Container_Boundary(neogo, "Blockchain") { + Interface "NeoGo" + Component(ffsid, "FrostFS ID", $tags="smart-contract", $descr="Stores namespaces and users") + Component(policy, "Policy", $tags="smart-contract", $descr="Stores APE rules") +} + +Rel_R(user, verify, "Requests", "gRPC") +Rel_R(verify, apesvc, "Access control") +Rel_R(apesvc, objsvc, "Operation") +Rel_D(apesvc, NeoGo, "Get data to validate request") +Rel("NeoGo", ffsid, "Fetch users") +Rel("NeoGo", policy, "Fetch policies") + +SHOW_LEGEND(true) +@enduml \ No newline at end of file diff --git a/docs/images/authentication/authoverview.svg b/docs/images/authentication/authoverview.svg new file mode 100644 index 000000000..a34a68da0 --- /dev/null +++ b/docs/images/authentication/authoverview.svg @@ -0,0 +1 @@ +FrostFS Storage[Container]Blockchain[Container]Sign Service Check request signatureAPE ServiceObject serviceNeoGoFrostFS ID Stores namespaces andusersPolicy Stores APE rulesUser User with private keyRequests[gRPC]Access controlOperationGet data to validaterequestFetch usersFetch policiesLegend  person  component  container boundary(dashed)  smart-contract(last text color)  \ No newline at end of file From 7bc30038032825dec6ed122384842ed152098892 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 23 Apr 2024 14:12:37 +0300 Subject: [PATCH 1050/1943] [#1104] docs: Add bearer token description to auth doc This is about authentication only and eACL is deprecated, so only mention `allow_impersonate` flag. Signed-off-by: Evgenii Stratonikov --- docs/authentication.md | 17 +++++++++++++++-- docs/images/authentication/impersonate.puml | 15 +++++++++++++++ docs/images/authentication/impersonate.svg | 1 + 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 docs/images/authentication/impersonate.puml create mode 100644 docs/images/authentication/impersonate.svg diff --git a/docs/authentication.md b/docs/authentication.md index 4efb03f93..544839b2c 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -35,7 +35,18 @@ The hash algorithm used is SHA-256 ECDSA public key corresponding to the private key being used to sign a message. It is the primary user identity and is used to determine the request originator. -## Session token +## Tokens + +Generally, the request owner, i.e. an account all access control checks are applied to +is taken from the request signature. +However, session and bearer tokens can alter authentication process by making "effective" request owner differ from the actual one. +The general scheme is given by the following picture: + +![Token processing](images/authentication/impersonate.svg) + +It is important to note, that the token is only valid when the request signature corresponds to the actor token is issued to. + +### Session token Session token can override the rules of determining request owner. It is defined in the [frostfs-api](https://git.frostfs.info/TrueCloudLab/frostfs-api/src/branch/master/session/types.proto#L89). @@ -50,7 +61,9 @@ Session token may have some restrictions: 2. Set of operations it applies to. 3. The entity it is given to. This is provided in `session_key` field containing the public key. -## Bearer token +### Bearer token + +Bearer token is generally used for access control but can also affect authentication if `allow_impersonate` flag is set. With this flag it behaves similarly to session token. ## FrostFS ID diff --git a/docs/images/authentication/impersonate.puml b/docs/images/authentication/impersonate.puml new file mode 100644 index 000000000..e9feae6e5 --- /dev/null +++ b/docs/images/authentication/impersonate.puml @@ -0,0 +1,15 @@ +@startuml impersonate +start + +if (The request has bearer token with allow_impersonate=true?) then (yes) + :Treat bearer token issuer as the request owner.; + end +(no) elseif (The request has session token?) then (yes) + :Treat session token issuer as the request owner.; + end +else (no) + :Determine request owner from the request signature.; + end +endif + +@enduml \ No newline at end of file diff --git a/docs/images/authentication/impersonate.svg b/docs/images/authentication/impersonate.svg new file mode 100644 index 000000000..add2c5439 --- /dev/null +++ b/docs/images/authentication/impersonate.svg @@ -0,0 +1 @@ +yesThe request has bearer token with allow_impersonate=true?Treat bearer token issuer as the request owner.yesThe request has session token?nonoTreat session token issuer as the request owner.Determine request owner from the request signature. \ No newline at end of file From 1f02ac25665af1c7d9c8b882e2c859147793ba02 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 24 Apr 2024 14:59:41 +0300 Subject: [PATCH 1051/1943] [#1103] pre-commit: Exclude `*.svg` Signed-off-by: Anton Nikiforov --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 13284b9c5..d2d90fa5c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - id: end-of-file-fixer - exclude: ".key$" + exclude: "(.key|.svg)$" - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.9.0.6 From 700e891b854f9a21f81614ef968dcc66a4a987da Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 24 Apr 2024 15:01:19 +0300 Subject: [PATCH 1052/1943] [#1103] Fix end of file and trim trailing whitespace Signed-off-by: Anton Nikiforov --- dev/.vscode-example/launch.json | 2 +- docs/authentication.md | 2 +- docs/images/authentication/authoverview.puml | 4 ++-- pkg/services/tree/ape.go | 4 +--- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/dev/.vscode-example/launch.json b/dev/.vscode-example/launch.json index 6aedde85e..990fd42a8 100644 --- a/dev/.vscode-example/launch.json +++ b/dev/.vscode-example/launch.json @@ -250,4 +250,4 @@ "stopAll": true } ] -} \ No newline at end of file +} diff --git a/docs/authentication.md b/docs/authentication.md index 544839b2c..3fe5ca512 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -67,4 +67,4 @@ Bearer token is generally used for access control but can also affect authentica ## FrostFS ID -## APE \ No newline at end of file +## APE diff --git a/docs/images/authentication/authoverview.puml b/docs/images/authentication/authoverview.puml index 1242fdfc6..20cac9f52 100644 --- a/docs/images/authentication/authoverview.puml +++ b/docs/images/authentication/authoverview.puml @@ -5,7 +5,7 @@ AddElementTag("smart-contract", $bgColor=#0abab5) Person(user, "User", "User with private key") -Container_Boundary(stor, "FrostFS Storage") { +Container_Boundary(stor, "FrostFS Storage") { Component(verify, "Sign Service", $descr="Check request signature") Component(apesvc, "APE Service") Component(objsvc, "Object service") @@ -25,4 +25,4 @@ Rel("NeoGo", ffsid, "Fetch users") Rel("NeoGo", policy, "Fetch policies") SHOW_LEGEND(true) -@enduml \ No newline at end of file +@enduml diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index f58721509..76ec254f0 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -19,9 +19,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) -var ( - subjectNotFoundErrorMessage = "subject not found" -) +var subjectNotFoundErrorMessage = "subject not found" func (s *Service) checkAPE(container *core.Container, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey) error { namespace := "" From 167c52a1a977301f9899d9b617aaa34545c7bd73 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 23 Apr 2024 15:21:29 +0300 Subject: [PATCH 1053/1943] [#1103] node: Reduce amount of lines for method `StorageEngine.head` Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/engine/head.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index ba5e7cc1d..f56e1b772 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -67,20 +67,17 @@ func (e *StorageEngine) Head(ctx context.Context, prm HeadPrm) (res HeadRes, err func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.head") defer span.End() - if e.metrics != nil { defer elapsed("Head", e.metrics.AddMethodDuration)() } var ( - head *objectSDK.Object - siErr *objectSDK.SplitInfoError - outSI *objectSDK.SplitInfo - + head *objectSDK.Object + siErr *objectSDK.SplitInfoError + outSI *objectSDK.SplitInfo outError error = new(apistatus.ObjectNotFound) + shPrm shard.HeadPrm ) - - var shPrm shard.HeadPrm shPrm.SetAddress(prm.addr) shPrm.SetRaw(prm.raw) @@ -94,44 +91,34 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) if outSI == nil { outSI = objectSDK.NewSplitInfo() } - util.MergeSplitInfo(siErr.SplitInfo(), outSI) - _, withLink := outSI.Link() _, withLast := outSI.LastPart() - // stop iterating over shards if SplitInfo structure is complete if withLink && withLast { return true } - return false case client.IsErrObjectAlreadyRemoved(err): outError = err - return true // stop, return it back case shard.IsErrObjectExpired(err): // object is found but should not // be returned outError = new(apistatus.ObjectNotFound) - return true default: e.reportShardError(sh, "could not head object from shard", err) return false } } - head = res.Object() - return true }) if outSI != nil { return HeadRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(outSI)) - } - - if head == nil { + } else if head == nil { return HeadRes{}, outError } From 112a7c690f55a334eb05ce2b219f4551d928f45f Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 22 Apr 2024 09:43:42 +0300 Subject: [PATCH 1054/1943] [#1103] node: Implement `Get\Head` requests for EC object Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/object/get.go | 4 + cmd/frostfs-cli/modules/object/head.go | 4 + cmd/frostfs-cli/modules/object/range.go | 44 +++++++ cmd/frostfs-node/object.go | 4 +- go.mod | 4 +- go.sum | 8 +- internal/logs/logs.go | 9 ++ pkg/local_object_storage/engine/get.go | 14 +++ pkg/local_object_storage/engine/head.go | 11 ++ pkg/local_object_storage/metabase/exists.go | 4 + pkg/local_object_storage/metabase/get.go | 29 +++++ pkg/local_object_storage/metabase/get_test.go | 39 ++++++ pkg/local_object_storage/metabase/put.go | 39 ++++++ pkg/local_object_storage/metabase/util.go | 10 ++ pkg/local_object_storage/util/ecinfo.go | 25 ++++ pkg/local_object_storage/util/ecinfo_test.go | 56 +++++++++ pkg/network/cache/multi.go | 6 +- pkg/services/object/get/assemble.go | 2 + pkg/services/object/get/assembleec.go | 79 ++++++++++++ pkg/services/object/get/assemblerec.go | 117 ++++++++++++++++++ pkg/services/object/get/get.go | 13 ++ pkg/services/object/get/local.go | 6 + pkg/services/object/get/remote.go | 25 +++- pkg/services/object/get/request.go | 4 + pkg/services/object/get/service.go | 4 + pkg/services/object/get/status.go | 1 + pkg/services/object/get/v2/get_forwarder.go | 3 + pkg/services/object/get/v2/head_forwarder.go | 3 + pkg/services/object/get/v2/service.go | 8 ++ pkg/services/object/get/v2/util.go | 15 +++ 30 files changed, 579 insertions(+), 11 deletions(-) create mode 100644 pkg/local_object_storage/util/ecinfo.go create mode 100644 pkg/local_object_storage/util/ecinfo_test.go create mode 100644 pkg/services/object/get/assembleec.go create mode 100644 pkg/services/object/get/assemblerec.go diff --git a/cmd/frostfs-cli/modules/object/get.go b/cmd/frostfs-cli/modules/object/get.go index 9a888ccd3..f1edccba2 100644 --- a/cmd/frostfs-cli/modules/object/get.go +++ b/cmd/frostfs-cli/modules/object/get.go @@ -99,6 +99,10 @@ func getObject(cmd *cobra.Command, _ []string) { return } + if ok := printECInfoErr(cmd, err); ok { + return + } + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) } diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index f97ef952d..14797dc41 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -70,6 +70,10 @@ func getObjectHeader(cmd *cobra.Command, _ []string) { return } + if ok := printECInfoErr(cmd, err); ok { + return + } + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) } diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index 0eee7bdba..9ba752237 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -146,6 +146,50 @@ func marshalSplitInfo(cmd *cobra.Command, info *objectSDK.SplitInfo) ([]byte, er } } +func printECInfoErr(cmd *cobra.Command, err error) bool { + var errECInfo *objectSDK.ECInfoError + + ok := errors.As(err, &errECInfo) + + if ok { + cmd.PrintErrln("Object is erasure-encoded, ec information received.") + printECInfo(cmd, errECInfo.ECInfo()) + } + + return ok +} + +func printECInfo(cmd *cobra.Command, info *objectSDK.ECInfo) { + bs, err := marshalECInfo(cmd, info) + commonCmd.ExitOnErr(cmd, "can't marshal split info: %w", err) + + cmd.Println(string(bs)) +} + +func marshalECInfo(cmd *cobra.Command, info *objectSDK.ECInfo) ([]byte, error) { + toJSON, _ := cmd.Flags().GetBool(commonflags.JSON) + toProto, _ := cmd.Flags().GetBool("proto") + switch { + case toJSON && toProto: + return nil, errors.New("'--json' and '--proto' flags are mutually exclusive") + case toJSON: + return info.MarshalJSON() + case toProto: + return info.Marshal() + default: + b := bytes.NewBuffer(nil) + b.WriteString("Total chunks: " + strconv.Itoa(int(info.Chunks[0].Total))) + for _, chunk := range info.Chunks { + var id oid.ID + if err := id.Decode(chunk.ID.GetValue()); err != nil { + return nil, fmt.Errorf("unable to decode chunk id: %w", err) + } + b.WriteString("\n Index: " + strconv.Itoa(int(chunk.Index)) + " ID: " + id.String()) + } + return b.Bytes(), nil + } +} + func getRangeList(cmd *cobra.Command) ([]objectSDK.Range, error) { v := cmd.Flag("range").Value.String() if len(v) == 0 { diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 6160bbc76..a7a084fd1 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -173,7 +173,7 @@ func initObjectService(c *cfg) { sSearchV2 := createSearchSvcV2(sSearch, keyStorage) - sGet := createGetService(c, keyStorage, traverseGen, c.clientCache) + sGet := createGetService(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource) *c.cfgObject.getSvc = *sGet // need smth better @@ -358,6 +358,7 @@ func createSearchSvcV2(sSearch *searchsvc.Service, keyStorage *util.KeyStorage) func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache, + containerSource containercore.Source, ) *getsvc.Service { ls := c.cfgObject.cfgLocalStorage.localStorage @@ -369,6 +370,7 @@ func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Tra placement.SuccessAfter(1), ), coreConstructor, + containerSource, getsvc.WithLogger(c.log)) } diff --git a/go.mod b/go.mod index 8cfea305d..f5469dee6 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240327095603-491a47e7fe24 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240422151450-df9b65324a4c git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240424080726-20ab57bf7ec3 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index 76ed7e5dc..0a885cb2f 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,15 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240327095603-491a47e7fe24 h1:uIkl0mKWwDICUZTbNWZ38HLYDBI9rMgdAhYQWZ0C9iQ= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240327095603-491a47e7fe24/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240422151450-df9b65324a4c h1:RFDrNsF2e+EJfaB8lZrRRxNjQkLfM09gnEyudvGuc10= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240422151450-df9b65324a4c/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0 h1:FzurjElUwC7InY9v5rzXReKbfBL5yRJKSWJPq6BKhH0= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92 h1:hSyM52d8yIaOpYQlLlVYdrGbgCsvIDjwl3AJaJUlYPU= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240329104804-ec0cb2169f92/go.mod h1:i0RKqiF4z3UOxLSNwhHw+cUz/JyYWuTRpnn9ere4Y3w= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240424080726-20ab57bf7ec3 h1:7Sd/J2IM0uGpmFKBgseUh6/JsdJN06b8W8UZMKAUDZg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240424080726-20ab57bf7ec3/go.mod h1:wDFmMP7l00Xd5VZVzF2MuhyJCnotyhfxHYnvrEEG/e4= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a h1:wbndKvHbwDQiSMQWL75RxiTZCeUyCi7NUj1lsfdAGkc= diff --git a/internal/logs/logs.go b/internal/logs/logs.go index bd67217c4..e76ab10b6 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -99,9 +99,17 @@ const ( GetRemoteCallFailed = "remote call failed" GetCanNotAssembleTheObject = "can not assemble the object" GetTryingToAssembleTheObject = "trying to assemble the object..." + GetTryingToAssembleTheECObject = "trying to assemble the ec object..." GetAssemblingSplittedObject = "assembling splitted object..." + GetAssemblingECObject = "assembling erasure-coded object..." + GetUnableToGetAllPartsECObject = "unable to get all parts, continue to reconstruct with existed" + GetUnableToGetPartECObject = "unable to get part of the erasure-encoded object" + GetUnableToHeadPartECObject = "unable to head part of the erasure-encoded object" + GetUnableToGetECObjectContainer = "unable to get container for erasure-coded object" GetAssemblingSplittedObjectCompleted = "assembling splitted object completed" + GetAssemblingECObjectCompleted = "assembling erasure-coded object completed" GetFailedToAssembleSplittedObject = "failed to assemble splitted object" + GetFailedToAssembleECObject = "failed to assemble erasure-coded object" GetCouldNotGenerateContainerTraverser = "could not generate container traverser" GetCouldNotConstructRemoteNodeClient = "could not construct remote node client" GetCouldNotWriteHeader = "could not write header" @@ -111,6 +119,7 @@ const ( GetCompletingTheOperation = "completing the operation" GetRequestedObjectWasMarkedAsRemoved = "requested object was marked as removed" GetRequestedObjectIsVirtual = "requested object is virtual" + GetRequestedObjectIsEC = "requested object is erasure-coded" GetRequestedRangeIsOutOfObjectBounds = "requested range is out of object bounds" PutAdditionalContainerBroadcastFailure = "additional container broadcast failure" SearchReturnResultDirectly = "return result directly" diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index f77c44226..991af3d1a 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -88,6 +88,10 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { return GetRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(it.SplitInfo)) } + if it.ECInfo != nil { + return GetRes{}, logicerr.Wrap(objectSDK.NewECInfoError(it.ECInfo)) + } + if it.ObjectExpired { return GetRes{}, errNotFound } @@ -119,6 +123,7 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { type getShardIterator struct { Object *objectSDK.Object SplitInfo *objectSDK.SplitInfo + ECInfo *objectSDK.ECInfo OutError error ShardWithMeta hashedShard MetaError error @@ -130,6 +135,7 @@ type getShardIterator struct { Engine *StorageEngine splitInfoErr *objectSDK.SplitInfoError + ecInfoErr *objectSDK.ECInfoError } func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { @@ -164,6 +170,14 @@ func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { // stop iterating over shards if SplitInfo structure is complete return withLink && withLast + case errors.As(err, &i.ecInfoErr): + if i.ECInfo == nil { + i.ECInfo = objectSDK.NewECInfo() + } + + util.MergeECInfo(i.ecInfoErr.ECInfo(), i.ECInfo) + // stop iterating over shards if ECInfo structure is complete + return len(i.ECInfo.Chunks) == int(i.ECInfo.Chunks[0].Total) case client.IsErrObjectAlreadyRemoved(err): i.OutError = err return true // stop, return it back diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index f56e1b772..1c61ca455 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -75,6 +75,8 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) head *objectSDK.Object siErr *objectSDK.SplitInfoError outSI *objectSDK.SplitInfo + eiErr *objectSDK.ECInfoError + outEI *objectSDK.ECInfo outError error = new(apistatus.ObjectNotFound) shPrm shard.HeadPrm ) @@ -99,6 +101,13 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) return true } return false + case errors.As(err, &eiErr): + if outEI == nil { + outEI = objectSDK.NewECInfo() + } + util.MergeECInfo(eiErr.ECInfo(), outEI) + // stop iterating over shards if ECInfo structure is complete + return len(outEI.Chunks) == int(outEI.Chunks[0].Total) case client.IsErrObjectAlreadyRemoved(err): outError = err return true // stop, return it back @@ -118,6 +127,8 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) if outSI != nil { return HeadRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(outSI)) + } else if outEI != nil { + return HeadRes{}, logicerr.Wrap(objectSDK.NewECInfoError(outEI)) } else if head == nil { return HeadRes{}, outError } diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index aa9aba106..bf6766c05 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -108,6 +108,10 @@ func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (exists b return false, logicerr.Wrap(objectSDK.NewSplitInfoError(splitInfo)) } + // if parent bucket is empty, then check if object exists in ec bucket + if data := getFromBucket(tx, ecInfoBucketName(cnr, key), objKey); len(data) != 0 { + return false, getECInfoError(tx, cnr, data) + } // if parent bucket is empty, then check if object exists in typed buckets return firstIrregularObjectType(tx, cnr, objKey) != objectSDK.TypeRegular, nil diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index d18331a3d..aa19502e8 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -110,6 +110,11 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b return obj, obj.Unmarshal(data) } + data = getFromBucket(tx, ecInfoBucketName(cnr, bucketName), key) + if len(data) != 0 { + return nil, getECInfoError(tx, cnr, data) + } + // if not found then check in tombstone index data = getFromBucket(tx, tombstoneBucketName(cnr, bucketName), key) if len(data) != 0 { @@ -185,3 +190,27 @@ func getSplitInfoError(tx *bbolt.Tx, cnr cid.ID, key []byte) error { return logicerr.Wrap(new(apistatus.ObjectNotFound)) } + +func getECInfoError(tx *bbolt.Tx, cnr cid.ID, data []byte) error { + offset := 0 + ecInfo := objectSDK.NewECInfo() + for offset < len(data) { + key := data[offset : offset+objectKeySize] + // check in primary index + ojbData := getFromBucket(tx, primaryBucketName(cnr, make([]byte, bucketKeySize)), key) + if len(data) != 0 { + obj := objectSDK.New() + if err := obj.Unmarshal(ojbData); err != nil { + return err + } + chunk := objectSDK.ECChunk{} + id, _ := obj.ID() + chunk.SetID(id) + chunk.Index = obj.ECHeader().Index() + chunk.Total = obj.ECHeader().Total() + ecInfo.AddChunk(chunk) + } + offset += objectKeySize + } + return logicerr.Wrap(objectSDK.NewECInfoError(ecInfo)) +} diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index af6b41327..247ddf9cd 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -3,6 +3,7 @@ package meta_test import ( "bytes" "context" + "errors" "fmt" "os" "runtime" @@ -15,8 +16,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" ) @@ -109,6 +112,42 @@ func TestDB_Get(t *testing.T) { require.True(t, binaryEqual(child.CutPayload(), newChild)) }) + t.Run("put erasure-coded object", func(t *testing.T) { + cnr := cidtest.ID() + virtual := testutil.GenerateObjectWithCID(cnr) + c, err := erasurecode.NewConstructor(3, 1) + require.NoError(t, err) + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + parts, err := c.Split(virtual, &pk.PrivateKey) + require.NoError(t, err) + for _, part := range parts { + err = putBig(db, part) + var eiError *objectSDK.ECInfoError + if err != nil && !errors.As(err, &eiError) { + require.NoError(t, err) + } + } + _, err = metaGet(db, object.AddressOf(virtual), true) + var eiError *objectSDK.ECInfoError + require.ErrorAs(t, err, &eiError) + require.Equal(t, len(eiError.ECInfo().Chunks), len(parts)) + for _, chunk := range eiError.ECInfo().Chunks { + var found bool + for _, part := range parts { + partID, _ := part.ID() + var chunkID oid.ID + require.NoError(t, chunkID.ReadFromV2(chunk.ID)) + if chunkID.Equals(partID) { + found = true + } + } + if !found { + require.Fail(t, "chunk not found") + } + } + }) + t.Run("get removed object", func(t *testing.T) { obj := oidtest.Address() ts := oidtest.Address() diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 429d981fe..2822303a0 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -1,6 +1,7 @@ package meta import ( + "bytes" "context" "encoding/binary" "errors" @@ -264,6 +265,13 @@ func putUniqueIndexes( if err != nil { return err } + + if ecHead := obj.GetECHeader(); ecHead != nil { + err = putECInfo(tx, cnr, objKey, ecHead) + if err != nil { + return err + } + } } return nil @@ -571,3 +579,34 @@ func isLinkObject(obj *objectSDK.Object) bool { func isLastObject(obj *objectSDK.Object) bool { return len(obj.Children()) == 0 && obj.Parent() != nil } + +func putECInfo(tx *bbolt.Tx, + cnr cid.ID, objKey []byte, + ecHead *objectSDK.ECHeader, +) error { + parentID := objectKey(ecHead.Parent(), make([]byte, objectKeySize)) + bucketName := make([]byte, bucketKeySize) + + val := getFromBucket(tx, ecInfoBucketName(cnr, bucketName), parentID) + if len(val) == 0 { + val = objKey + } else { + offset := 0 + found := false + for offset < len(val) { + if bytes.Equal(objKey, val[offset:offset+objectKeySize]) { + found = true + break + } + offset += objectKeySize + } + if !found { + val = append(val, objKey...) + } + } + return putUniqueIndexItem(tx, namedBucketItem{ + name: ecInfoBucketName(cnr, make([]byte, bucketKeySize)), + key: parentID, + val: val, + }) +} diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index d46a421a3..9249ae49b 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -119,6 +119,11 @@ const ( // Key: container ID + type // Value: container size in bytes as little-endian uint64 containerCountersPrefix + + // ecInfoPrefix is used for storing relation between EC parent id and chunk id. + // Key: container ID + type + // Value: Object id + ecInfoPrefix ) const ( @@ -190,6 +195,11 @@ func splitBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, splitPrefix, key) } +// ecInfoBucketName returns _ecinfo. +func ecInfoBucketName(cnr cid.ID, key []byte) []byte { + return bucketName(cnr, ecInfoPrefix, key) +} + // addressKey returns key for K-V tables when key is a whole address. func addressKey(addr oid.Address, key []byte) []byte { addr.Container().Encode(key) diff --git a/pkg/local_object_storage/util/ecinfo.go b/pkg/local_object_storage/util/ecinfo.go new file mode 100644 index 000000000..a92fbceea --- /dev/null +++ b/pkg/local_object_storage/util/ecinfo.go @@ -0,0 +1,25 @@ +package util + +import ( + "bytes" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" +) + +// MergeECInfo ignores conflicts and rewrites `to` with non empty values +// from `from`. +func MergeECInfo(from, to *objectSDK.ECInfo) *objectSDK.ECInfo { + for _, fchunk := range from.Chunks { + add := true + for _, tchunk := range to.Chunks { + if bytes.Equal(tchunk.ID.GetValue(), fchunk.ID.GetValue()) { + add = false + break + } + } + if add { + to.AddChunk(*objectSDK.NewECChunkFromV2(&fchunk)) + } + } + return to +} diff --git a/pkg/local_object_storage/util/ecinfo_test.go b/pkg/local_object_storage/util/ecinfo_test.go new file mode 100644 index 000000000..081006088 --- /dev/null +++ b/pkg/local_object_storage/util/ecinfo_test.go @@ -0,0 +1,56 @@ +package util + +import ( + "crypto/rand" + "testing" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/stretchr/testify/require" +) + +func TestMergeECInfo(t *testing.T) { + id := generateV2ID() + target := objectSDK.NewECInfo() + var chunk objectSDK.ECChunk + chunk.Total = 2 + chunk.Index = 0 + chunk.SetID(id) + target.AddChunk(chunk) + + t.Run("merge empty", func(t *testing.T) { + to := objectSDK.NewECInfo() + + result := MergeECInfo(target, to) + require.Equal(t, result, target) + }) + + t.Run("merge existed", func(t *testing.T) { + to := objectSDK.NewECInfo() + to.AddChunk(chunk) + + result := MergeECInfo(target, to) + require.Equal(t, result, target) + }) + t.Run("merge extend", func(t *testing.T) { + to := objectSDK.NewECInfo() + var chunk objectSDK.ECChunk + chunk.Total = 2 + chunk.Index = 1 + chunk.SetID(generateV2ID()) + to.AddChunk(chunk) + + result := MergeECInfo(target, to) + require.Equal(t, len(result.Chunks), 2) + }) +} + +func generateV2ID() oid.ID { + var buf [32]byte + _, _ = rand.Read(buf[:]) + + var id oid.ID + _ = id.Decode(buf[:]) + + return id +} diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 5dd206283..f19510d76 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -167,8 +167,9 @@ func (x *multiClient) iterateClients(ctx context.Context, f func(clientcore.Clie // from the SDK client; should not be considered // as a connection error var siErr *objectSDK.SplitInfoError + var eiErr *objectSDK.ECInfoError - success := err == nil || errors.Is(err, context.Canceled) || errors.As(err, &siErr) + success := err == nil || errors.Is(err, context.Canceled) || errors.As(err, &siErr) || errors.As(err, &eiErr) if success || firstErr == nil || errors.Is(firstErr, errRecentlyFailed) { firstErr = err } @@ -196,7 +197,8 @@ func (x *multiClient) ReportError(err error) { // from the SDK client; should not be considered // as a connection error var siErr *objectSDK.SplitInfoError - if errors.As(err, &siErr) { + var eiErr *objectSDK.ECInfoError + if errors.As(err, &siErr) || errors.As(err, &eiErr) { return } diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index 6a8c5c818..548e8e45d 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -139,9 +139,11 @@ func (r *request) getObjectWithIndependentRequest(ctx context.Context, prm Reque remoteStorageConstructor: r.remoteStorageConstructor, epochSource: r.epochSource, localStorage: r.localStorage, + containerSource: r.containerSource, prm: prm, infoSplit: objectSDK.NewSplitInfo(), + infoEC: objectSDK.NewECInfo(), log: r.log, } diff --git a/pkg/services/object/get/assembleec.go b/pkg/services/object/get/assembleec.go new file mode 100644 index 000000000..d288ef249 --- /dev/null +++ b/pkg/services/object/get/assembleec.go @@ -0,0 +1,79 @@ +package getsvc + +import ( + "context" + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "go.uber.org/zap" +) + +func (r *request) assembleEC(ctx context.Context) { + if r.isRaw() { + r.log.Debug(logs.GetCanNotAssembleTheObject) + return + } + + // Any access tokens are not expected to be used in the assembly process: + // - there is no requirement to specify child objects in session/bearer + // token for `GET`/`GETRANGE`/`RANGEHASH` requests in the API protocol, + // and, therefore, their missing in the original request should not be + // considered as error; on the other hand, without session for every child + // object, it is impossible to attach bearer token in the new generated + // requests correctly because the token has not been issued for that node's + // key; + // - the assembly process is expected to be handled on a container node + // only since the requests forwarding mechanism presentation; such the + // node should have enough rights for getting any child object by design. + r.prm.common.ForgetTokens() + + // Do not use forwarding during assembly stage. + // Request forwarding closure inherited in produced + // `execCtx` so it should be disabled there. + r.disableForwarding() + + r.log.Debug(logs.GetTryingToAssembleTheECObject) + + assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.containerSource, r.log) + + r.log.Debug(logs.GetAssemblingECObject, + zap.Stringer("address", r.address()), + zap.Uint64("range_offset", r.ctxRange().GetOffset()), + zap.Uint64("range_length", r.ctxRange().GetLength()), + ) + defer r.log.Debug(logs.GetAssemblingECObjectCompleted, + zap.Stringer("address", r.address()), + zap.Uint64("range_offset", r.ctxRange().GetOffset()), + zap.Uint64("range_length", r.ctxRange().GetLength()), + ) + + obj, err := assembler.Assemble(ctx, r.prm.objWriter, r.headOnly()) + if err != nil { + r.log.Warn(logs.GetFailedToAssembleECObject, + zap.Error(err), + zap.Stringer("address", r.address()), + zap.Uint64("range_offset", r.ctxRange().GetOffset()), + zap.Uint64("range_length", r.ctxRange().GetLength()), + ) + } + + var errRemoved *apistatus.ObjectAlreadyRemoved + var errOutOfRange *apistatus.ObjectOutOfRange + + switch { + default: + r.status = statusUndefined + r.err = err + case err == nil: + r.status = statusOK + r.err = nil + r.collectedObject = obj + case errors.As(err, &errRemoved): + r.status = statusINHUMED + r.err = errRemoved + case errors.As(err, &errOutOfRange): + r.status = statusOutOfRange + r.err = errOutOfRange + } +} diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go new file mode 100644 index 000000000..d73d771cf --- /dev/null +++ b/pkg/services/object/get/assemblerec.go @@ -0,0 +1,117 @@ +package getsvc + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type assemblerec struct { + addr oid.Address + ecInfo *objectSDK.ECInfo + rng *objectSDK.Range + objGetter objectGetter + cs container.Source + log *logger.Logger +} + +func newAssemblerEC( + addr oid.Address, + ecInfo *objectSDK.ECInfo, + rng *objectSDK.Range, + objGetter objectGetter, + cs container.Source, + log *logger.Logger, +) *assemblerec { + return &assemblerec{ + addr: addr, + rng: rng, + ecInfo: ecInfo, + objGetter: objGetter, + cs: cs, + log: log, + } +} + +// Assemble assembles erasure-coded object and writes it's content to ObjectWriter. +// It returns parent object. +func (a *assemblerec) Assemble(ctx context.Context, writer ObjectWriter, headOnly bool) (*objectSDK.Object, error) { + parts := a.retrieveParts(ctx, headOnly) + cnt, err := a.cs.Get(a.addr.Container()) + if err != nil { + return nil, err + } + c, err := erasurecode.NewConstructor( + policy.ECDataCount(cnt.Value.PlacementPolicy()), + policy.ECParityCount(cnt.Value.PlacementPolicy()), + ) + if err != nil { + return nil, err + } + if headOnly { + obj, err := c.ReconstructHeader(parts) + if err == nil { + return obj, writer.WriteHeader(ctx, obj) + } + return nil, err + } + obj, err := c.Reconstruct(parts) + if err == nil { + err = writer.WriteHeader(ctx, obj.CutPayload()) + if err == nil { + err = writer.WriteChunk(ctx, obj.Payload()) + if err != nil { + return nil, err + } + } + } + return obj, err +} + +func (a *assemblerec) retrieveParts(mainCtx context.Context, headOnly bool) []*objectSDK.Object { + parts := make([]*objectSDK.Object, int(a.ecInfo.Chunks[0].Total)) + errGroup, ctx := errgroup.WithContext(mainCtx) + + for i := range a.ecInfo.Chunks { + chunk := a.ecInfo.Chunks[i] + errGroup.Go(func() error { + objID := new(oid.ID) + err := objID.ReadFromV2(chunk.ID) + if err != nil { + return fmt.Errorf("invalid object ID: %w", err) + } + var obj *objectSDK.Object + if headOnly { + obj, err = a.objGetter.HeadObject(ctx, *objID) + if err != nil { + a.log.Debug(logs.GetUnableToHeadPartECObject, zap.Stringer("part_id", objID), zap.Error(err)) + return nil + } + } else { + sow := NewSimpleObjectWriter() + obj, err = a.objGetter.GetObjectAndWritePayload(ctx, *objID, nil, sow) + if err != nil { + a.log.Debug(logs.GetUnableToGetPartECObject, zap.Stringer("part_id", objID), zap.Error(err)) + return nil + } + obj.SetPayload(sow.pld) + } + parts[chunk.Index] = obj + return nil + }) + } + + if err := errGroup.Wait(); err != nil { + a.log.Debug(logs.GetUnableToGetAllPartsECObject, zap.Error(err)) + } + return parts +} diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 3d5547047..9738935d2 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -73,9 +73,12 @@ func (s *Service) get(ctx context.Context, prm RequestParameters) error { remoteStorageConstructor: s.remoteStorageConstructor, epochSource: s.epochSource, localStorage: s.localStorage, + containerSource: s.containerSource, prm: prm, infoSplit: objectSDK.NewSplitInfo(), + infoEC: objectSDK.NewECInfo(), + log: s.log, } exec.setLogger(s.log) @@ -106,6 +109,16 @@ func (exec *request) analyzeStatus(ctx context.Context, execCnr bool) { exec.assemble(ctx) case statusOutOfRange: exec.log.Debug(logs.GetRequestedRangeIsOutOfObjectBounds) + case statusEC: + if !exec.isLocal() { + if execCnr { + exec.executeOnContainer(ctx) + exec.analyzeStatus(ctx, false) + } else { + exec.log.Debug(logs.GetRequestedObjectIsEC) + exec.assembleEC(ctx) + } + } default: exec.log.Debug(logs.OperationFinishedWithError, zap.Error(exec.err), diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index 257465019..fcfc9befc 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -5,6 +5,7 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -22,6 +23,7 @@ func (r *request) executeLocal(ctx context.Context) { r.collectedObject, err = r.get(ctx) var errSplitInfo *objectSDK.SplitInfoError + var errECInfo *objectSDK.ECInfoError var errRemoved *apistatus.ObjectAlreadyRemoved var errOutOfRange *apistatus.ObjectOutOfRange @@ -42,6 +44,10 @@ func (r *request) executeLocal(ctx context.Context) { r.status = statusVIRTUAL mergeSplitInfo(r.splitInfo(), errSplitInfo.SplitInfo()) r.err = objectSDK.NewSplitInfoError(r.infoSplit) + case errors.As(err, &errECInfo): + r.status = statusEC + util.MergeECInfo(errECInfo.ECInfo(), r.infoEC) + r.err = objectSDK.NewECInfoError(r.infoEC) case errors.As(err, &errOutOfRange): r.status = statusOutOfRange r.err = errOutOfRange diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index cd94434cf..302a4a4bc 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -7,6 +7,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -27,15 +29,20 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { obj, err := r.getRemote(ctx, rs, info) var errSplitInfo *objectSDK.SplitInfoError + var errECInfo *objectSDK.ECInfoError var errRemoved *apistatus.ObjectAlreadyRemoved var errOutOfRange *apistatus.ObjectOutOfRange switch { default: + r.log.Debug(logs.GetRemoteCallFailed, zap.Error(err)) + if r.status == statusEC { + // we need to continue getting another chunks from another nodes + // in case of network issue + return false + } r.status = statusUndefined r.err = new(apistatus.ObjectNotFound) - - r.log.Debug(logs.GetRemoteCallFailed, zap.Error(err)) case err == nil: r.status = statusOK r.err = nil @@ -57,6 +64,20 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { r.status = statusVIRTUAL mergeSplitInfo(r.splitInfo(), errSplitInfo.SplitInfo()) r.err = objectSDK.NewSplitInfoError(r.infoSplit) + case errors.As(err, &errECInfo): + r.status = statusEC + util.MergeECInfo(r.infoEC, errECInfo.ECInfo()) + r.infoEC = errECInfo.ECInfo() + r.err = objectSDK.NewECInfoError(r.infoEC) + if r.isRaw() { + return len(r.infoEC.Chunks) == int(r.infoEC.Chunks[0].Total) + } + cnt, err := r.containerSource.Get(r.address().Container()) + if err == nil { + return len(r.infoEC.Chunks) == policy.ECDataCount(cnt.Value.PlacementPolicy()) + } + r.log.Debug(logs.GetUnableToGetECObjectContainer, zap.Error(err)) + return len(r.infoEC.Chunks) == int(r.infoEC.Chunks[0].Total) } return r.status != statusUndefined diff --git a/pkg/services/object/get/request.go b/pkg/services/object/get/request.go index b9223a637..d0b79e30c 100644 --- a/pkg/services/object/get/request.go +++ b/pkg/services/object/get/request.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -22,6 +23,8 @@ type request struct { infoSplit *objectSDK.SplitInfo + infoEC *objectSDK.ECInfo + log *logger.Logger collectedObject *objectSDK.Object @@ -33,6 +36,7 @@ type request struct { traverserGenerator traverserGenerator remoteStorageConstructor remoteStorageConstructor localStorage localStorage + containerSource container.Source } func (r *request) setLogger(l *logger.Logger) { diff --git a/pkg/services/object/get/service.go b/pkg/services/object/get/service.go index bdf01a977..3413abeb7 100644 --- a/pkg/services/object/get/service.go +++ b/pkg/services/object/get/service.go @@ -1,6 +1,7 @@ package getsvc import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -16,6 +17,7 @@ type Service struct { epochSource epochSource keyStore keyStorage remoteStorageConstructor remoteStorageConstructor + containerSource container.Source } // New creates, initializes and returns utility serving @@ -26,6 +28,7 @@ func New( e localStorageEngine, tg traverserGenerator, cc clientConstructor, + cs container.Source, opts ...Option, ) *Service { result := &Service{ @@ -39,6 +42,7 @@ func New( remoteStorageConstructor: &multiclientRemoteStorageConstructor{ clientConstructor: cc, }, + containerSource: cs, } for _, option := range opts { option(result) diff --git a/pkg/services/object/get/status.go b/pkg/services/object/get/status.go index 3a5eebe32..919338d7f 100644 --- a/pkg/services/object/get/status.go +++ b/pkg/services/object/get/status.go @@ -6,6 +6,7 @@ const ( statusINHUMED statusVIRTUAL statusOutOfRange + statusEC ) type statusError struct { diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index 40aa3f62e..774f98643 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -166,6 +166,9 @@ func (f *getRequestForwarder) readStream(ctx context.Context, c client.MultiAddr case *objectV2.SplitInfo: si := objectSDK.NewSplitInfoFromV2(v) return objectSDK.NewSplitInfoError(si) + case *objectV2.ECInfo: + ei := objectSDK.NewECInfoFromV2(v) + return objectSDK.NewECInfoError(ei) } } return nil diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index a1bce1517..11286321a 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -84,6 +84,9 @@ func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr ne case *objectV2.SplitInfo: si := objectSDK.NewSplitInfoFromV2(v) return nil, objectSDK.NewSplitInfoError(si) + case *objectV2.ECInfo: + ei := objectSDK.NewECInfoFromV2(v) + return nil, objectSDK.NewECInfoError(ei) } objv2 := new(objectV2.Object) diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index bcdc4120e..682128df6 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -82,10 +82,13 @@ func (s *Service) Get(req *objectV2.GetRequest, stream objectSvc.GetObjectStream err = s.svc.Get(stream.Context(), *p) var splitErr *objectSDK.SplitInfoError + var ecErr *objectSDK.ECInfoError switch { case errors.As(err, &splitErr): return stream.Send(splitInfoResponse(splitErr.SplitInfo())) + case errors.As(err, &ecErr): + return stream.Send(ecInfoResponse(ecErr.ECInfo())) default: return err } @@ -123,11 +126,16 @@ func (s *Service) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV err = s.svc.Head(ctx, *p) var splitErr *objectSDK.SplitInfoError + var ecErr *objectSDK.ECInfoError if errors.As(err, &splitErr) { setSplitInfoHeadResponse(splitErr.SplitInfo(), resp) err = nil } + if errors.As(err, &ecErr) { + setECInfoHeadResponse(ecErr.ECInfo(), resp) + err = nil + } return resp, err } diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 7f7dd7480..da6428985 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -270,6 +270,17 @@ func splitInfoResponse(info *objectSDK.SplitInfo) *objectV2.GetResponse { return resp } +func ecInfoResponse(info *objectSDK.ECInfo) *objectV2.GetResponse { + resp := new(objectV2.GetResponse) + + body := new(objectV2.GetResponseBody) + resp.SetBody(body) + + body.SetObjectPart(info.ToV2()) + + return resp +} + func splitInfoRangeResponse(info *objectSDK.SplitInfo) *objectV2.GetRangeResponse { resp := new(objectV2.GetRangeResponse) @@ -285,6 +296,10 @@ func setSplitInfoHeadResponse(info *objectSDK.SplitInfo, resp *objectV2.HeadResp resp.GetBody().SetHeaderPart(info.ToV2()) } +func setECInfoHeadResponse(info *objectSDK.ECInfo, resp *objectV2.HeadResponse) { + resp.GetBody().SetHeaderPart(info.ToV2()) +} + func toHashResponse(typ refs.ChecksumType, res *getsvc.RangeHashRes) *objectV2.GetRangeHashResponse { resp := new(objectV2.GetRangeHashResponse) From 411a8d0245e00accfc8c9d045f182f7f843faf79 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 1 Mar 2024 14:43:26 +0300 Subject: [PATCH 1055/1943] [#1004] blobovnicza: Use TTL for blobovnicza tree cache Signed-off-by: Anton Nikiforov --- .../internal/modules/storagecfg/config.go | 2 + cmd/frostfs-node/config.go | 36 ++++++---- .../shard/blobstor/blobovnicza/config.go | 38 ++++++++++ config/example/node.env | 4 ++ config/example/node.json | 4 ++ config/example/node.yaml | 2 + docs/storage-node-configuration.md | 26 ++++--- go.mod | 1 + go.sum | 2 + .../blobstor/blobovniczatree/blobovnicza.go | 6 +- .../blobstor/blobovniczatree/cache.go | 69 +++++++++++++------ .../blobovniczatree/concurrency_test.go | 1 + .../blobstor/blobovniczatree/control_test.go | 4 ++ .../blobstor/blobovniczatree/exists_test.go | 1 + .../blobstor/blobovniczatree/generic_test.go | 3 + .../blobstor/blobovniczatree/iterate_test.go | 1 + .../blobstor/blobovniczatree/option.go | 54 ++++++++++----- .../blobovniczatree/rebuild_failover_test.go | 1 + .../blobstor/blobovniczatree/rebuild_test.go | 4 ++ .../blobstor/blobstor_test.go | 2 + .../blobstor/perf_test.go | 1 + .../engine/engine_test.go | 2 + .../shard/gc_internal_test.go | 1 + pkg/local_object_storage/shard/lock_test.go | 1 + pkg/local_object_storage/shard/range_test.go | 1 + pkg/local_object_storage/shard/shard_test.go | 1 + 26 files changed, 203 insertions(+), 65 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/config.go b/cmd/frostfs-adm/internal/modules/storagecfg/config.go index a07ce32c6..77183fb49 100644 --- a/cmd/frostfs-adm/internal/modules/storagecfg/config.go +++ b/cmd/frostfs-adm/internal/modules/storagecfg/config.go @@ -61,6 +61,8 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files + opened_cache_ttl: 5m # ttl for opened database file + opened_cache_exp_interval: 15s # cache cleanup interval for expired blobovnicza's gc: remover_batch_size: 200 # number of objects to be removed by the garbage collector diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 066d63e65..eb9f9fb18 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -185,13 +185,15 @@ type subStorageCfg struct { noSync bool // blobovnicza-specific - size uint64 - width uint64 - leafWidth uint64 - openedCacheSize int - initWorkerCount int - initInAdvance bool - rebuildDropTimeout time.Duration + size uint64 + width uint64 + leafWidth uint64 + openedCacheSize int + initWorkerCount int + initInAdvance bool + rebuildDropTimeout time.Duration + openedCacheTTL time.Duration + openedCacheExpInterval time.Duration } // readConfig fills applicationConfiguration with raw configuration values @@ -313,6 +315,8 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.width = sub.ShallowWidth() sCfg.leafWidth = sub.LeafWidth() sCfg.openedCacheSize = sub.OpenedCacheSize() + sCfg.openedCacheTTL = sub.OpenedCacheTTL() + sCfg.openedCacheExpInterval = sub.OpenedCacheExpInterval() sCfg.initWorkerCount = sub.InitWorkerCount() sCfg.initInAdvance = sub.InitInAdvance() sCfg.rebuildDropTimeout = sub.RebuildDropTimeout() @@ -846,11 +850,11 @@ type shardOptsWithID struct { shOpts []shard.Option } -func (c *cfg) shardOpts() []shardOptsWithID { +func (c *cfg) shardOpts(ctx context.Context) []shardOptsWithID { shards := make([]shardOptsWithID, 0, len(c.EngineCfg.shards)) for _, shCfg := range c.EngineCfg.shards { - shards = append(shards, c.getShardOpts(shCfg)) + shards = append(shards, c.getShardOpts(ctx, shCfg)) } return shards @@ -891,7 +895,7 @@ func (c *cfg) getPiloramaOpts(shCfg shardCfg) []pilorama.Option { return piloramaOpts } -func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { +func (c *cfg) getSubstorageOpts(ctx context.Context, shCfg shardCfg) []blobstor.SubStorage { var ss []blobstor.SubStorage for _, sRead := range shCfg.subStorages { switch sRead.typ { @@ -904,6 +908,8 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobovniczatree.WithOpenedCacheTTL(sRead.openedCacheTTL), + blobovniczatree.WithOpenedCacheExpInterval(sRead.openedCacheExpInterval), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), blobovniczatree.WithInitInAdvance(sRead.initInAdvance), blobovniczatree.WithWaitBeforeDropDB(sRead.rebuildDropTimeout), @@ -919,7 +925,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { ) } ss = append(ss, blobstor.SubStorage{ - Storage: blobovniczatree.NewBlobovniczaTree(blobTreeOpts...), + Storage: blobovniczatree.NewBlobovniczaTree(ctx, blobTreeOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < shCfg.smallSizeObjectLimit }, @@ -954,10 +960,10 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { return ss } -func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { +func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID { writeCacheOpts := c.getWriteCacheOpts(shCfg) piloramaOpts := c.getPiloramaOpts(shCfg) - ss := c.getSubstorageOpts(shCfg) + ss := c.getSubstorageOpts(ctx, shCfg) blobstoreOpts := []blobstor.Option{ blobstor.WithCompressObjects(shCfg.compress), @@ -1049,7 +1055,7 @@ func initLocalStorage(ctx context.Context, c *cfg) { c.cfgObject.getSvc = new(getsvc.Service) var shardsAttached int - for _, optsWithMeta := range c.shardOpts() { + for _, optsWithMeta := range c.shardOpts(ctx) { id, err := ls.AddShard(ctx, append(optsWithMeta.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))...) if err != nil { c.log.Error(logs.FrostFSNodeFailedToAttachShardToEngine, zap.Error(err)) @@ -1289,7 +1295,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { // Storage Engine var rcfg engine.ReConfiguration - for _, optsWithID := range c.shardOpts() { + for _, optsWithID := range c.shardOpts(ctx) { rcfg.AddShard(optsWithID.configID, append(optsWithID.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))) } diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index 9619a1027..c8ea4c2cb 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -25,6 +25,12 @@ const ( // OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's. OpenedCacheSizeDefault = 16 + // OpenedCacheTTLDefault is a default cache ttl of opened Blobovnicza's. + OpenedCacheTTLDefault = 0 // means expiring is off + + // OpenedCacheExpIntervalDefault is a default cache cleanup interval for expired Blobovnicza's. + OpenedCacheExpIntervalDefault = 15 * time.Second + // InitWorkerCountDefault is a default workers count to initialize Blobovnicza's. InitWorkerCountDefault = 5 @@ -106,6 +112,38 @@ func (x *Config) OpenedCacheSize() int { return OpenedCacheSizeDefault } +// OpenedCacheTTL returns the value of "opened_cache_ttl" config parameter. +// +// Returns OpenedCacheTTLDefault if the value is not a positive number. +func (x *Config) OpenedCacheTTL() time.Duration { + d := config.DurationSafe( + (*config.Config)(x), + "opened_cache_ttl", + ) + + if d > 0 { + return d + } + + return OpenedCacheTTLDefault +} + +// OpenedCacheExpInterval returns the value of "opened_cache_exp_interval" config parameter. +// +// Returns OpenedCacheExpIntervalDefault if the value is not a positive number. +func (x *Config) OpenedCacheExpInterval() time.Duration { + d := config.DurationSafe( + (*config.Config)(x), + "opened_cache_exp_interval", + ) + + if d > 0 { + return d + } + + return OpenedCacheExpIntervalDefault +} + // BoltDB returns config instance for querying bolt db specific parameters. func (x *Config) BoltDB() *boltdbconfig.Config { return (*boltdbconfig.Config)(x) diff --git a/config/example/node.env b/config/example/node.env index 9068886d3..77733f568 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -128,6 +128,8 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 +FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=5m +FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_EXP_INTERVAL=15s FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE @@ -178,6 +180,8 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 +FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_TTL=5m +FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_EXP_INTERVAL=15s FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10 ### FSTree config FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree diff --git a/config/example/node.json b/config/example/node.json index bae28720b..2ac8697bb 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -176,6 +176,8 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, + "opened_cache_ttl": "5m", + "opened_cache_exp_interval": "15s", "leaf_width": 10, "init_worker_count": 10, "init_in_advance": true, @@ -229,6 +231,8 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, + "opened_cache_ttl": "5m", + "opened_cache_exp_interval": "15s", "leaf_width": 10 }, { diff --git a/config/example/node.yaml b/config/example/node.yaml index 7a824adbe..d2109fcc5 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -151,6 +151,8 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files + opened_cache_ttl: 5m # ttl for opened database file + opened_cache_exp_interval: 15s # cache cleanup interval for expired blobovnicza's leaf_width: 10 # max count of key-value DB on leafs of object tree storage - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) depth: 5 # max depth of object tree storage in FS diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index aa3a17815..ce9065ac2 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -212,6 +212,8 @@ blobstor: depth: 1 width: 4 opened_cache_capacity: 50 + opened_cache_ttl: 5m + opened_cache_exp_interval: 15s ``` #### Common options for sub-storages @@ -228,17 +230,19 @@ blobstor: | `depth` | `int` | `4` | File-system tree depth. | #### `blobovnicza` type options -| Parameter | Type | Default value | Description | -| ----------------------- | ---------- | ------------- | --------------------------------------------------------------------- | -| `path` | `string` | | Path to the root of the blobstor. | -| `perm` | file mode | `0660` | Default permission for created files and directories. | -| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | -| `depth` | `int` | `2` | Blobovnicza tree depth. | -| `width` | `int` | `16` | Blobovnicza tree width. | -| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | -| `init_worker_count` | `int` | `5` | Maximum number of concurrent initialization workers. | -| `init_in_advance` | `bool` | `false` | If `true`, than all the blobovnicza files will be created on startup. | -| `rebuild_drop_timeout` | `duration` | `10s` | Timeout before drop empty blobovnicza file during rebuild. | +| Parameter | Type | Default value | Description | +|-----------------------------| ---------- |---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `path` | `string` | | Path to the root of the blobstor. | +| `perm` | file mode | `0660` | Default permission for created files and directories. | +| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | +| `depth` | `int` | `2` | Blobovnicza tree depth. | +| `width` | `int` | `16` | Blobovnicza tree width. | +| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | +| `opened_cache_ttl` | `duration` | `0` | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. | +| `opened_cache_exp_interval` | `duration` | `15s` | Cache cleanup interval for expired blobovnicza's. | +| `init_worker_count` | `int` | `5` | Maximum number of concurrent initialization workers. | +| `init_in_advance` | `bool` | `false` | If `true`, than all the blobovnicza files will be created on startup. | +| `rebuild_drop_timeout` | `duration` | `10s` | Timeout before drop empty blobovnicza file during rebuild. | ### `gc` subsection diff --git a/go.mod b/go.mod index f5469dee6..1be2526f2 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/chzyer/readline v1.5.1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 + github.com/go-pkgz/expirable-cache/v3 v3.0.0 github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/klauspost/compress v1.17.4 diff --git a/go.sum b/go.sum index 0a885cb2f..0795c955b 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,8 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-pkgz/expirable-cache/v3 v3.0.0 h1:u3/gcu3sabLYiTCevoRKv+WzjIn5oo7P8XtiXBeRDLw= +github.com/go-pkgz/expirable-cache/v3 v3.0.0/go.mod h1:2OQiDyEGQalYecLWmXprm3maPXeVb5/6/X7yRPYTzec= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index be51509b6..58466f759 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "context" "errors" "strconv" "strings" @@ -72,7 +73,7 @@ const ( ) // NewBlobovniczaTree returns new instance of blobovniczas tree. -func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { +func NewBlobovniczaTree(ctx context.Context, opts ...Option) (blz *Blobovniczas) { blz = new(Blobovniczas) initConfig(&blz.cfg) @@ -86,7 +87,8 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) - blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager) + blz.dbCache = newDBCache(ctx, blz.openedCacheSize, + blz.openedCacheTTL, blz.openedCacheExpInterval, blz.commondbManager) blz.deleteProtectedObjects = newAddressMap() blz.dbFilesGuard = &sync.RWMutex{} blz.rebuildGuard = &sync.RWMutex{} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go index 766f5ef0a..4fce3e81a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -1,11 +1,12 @@ package blobovniczatree import ( - "fmt" + "context" "sync" + "time" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" - "github.com/hashicorp/golang-lru/v2/simplelru" + cache "github.com/go-pkgz/expirable-cache/v3" ) // dbCache caches sharedDB instances that are NOT open for Put. @@ -13,30 +14,55 @@ import ( // Uses dbManager for opening/closing sharedDB instances. // Stores a reference to an cached sharedDB, so dbManager does not close it. type dbCache struct { - cacheGuard *sync.RWMutex - cache simplelru.LRUCache[string, *sharedDB] + cacheGuard *sync.Mutex + cache cache.Cache[string, *sharedDB] pathLock *utilSync.KeyLocker[string] // the order of locks is important: pathLock first, cacheGuard second closed bool nonCached map[string]struct{} + wg sync.WaitGroup + cancel context.CancelFunc dbManager *dbManager } -func newDBCache(size int, dbManager *dbManager) *dbCache { - cache, err := simplelru.NewLRU(size, func(_ string, evictedDB *sharedDB) { - evictedDB.Close() - }) - if err != nil { - // occurs only if the size is not positive - panic(fmt.Errorf("could not create LRU cache of size %d: %w", size, err)) - } - return &dbCache{ - cacheGuard: &sync.RWMutex{}, - cache: cache, +func newDBCache(parentCtx context.Context, size int, + ttl time.Duration, expInterval time.Duration, + dbManager *dbManager, +) *dbCache { + ch := cache.NewCache[string, *sharedDB](). + WithTTL(ttl).WithLRU().WithMaxKeys(size). + WithOnEvicted(func(_ string, db *sharedDB) { + db.Close() + }) + ctx, cancel := context.WithCancel(parentCtx) + res := &dbCache{ + cacheGuard: &sync.Mutex{}, + wg: sync.WaitGroup{}, + cancel: cancel, + cache: ch, dbManager: dbManager, pathLock: utilSync.NewKeyLocker[string](), nonCached: make(map[string]struct{}), } + if ttl > 0 { + res.wg.Add(1) + go func() { + ticker := time.NewTicker(expInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + res.wg.Done() + return + case <-ticker.C: + res.cacheGuard.Lock() + res.cache.DeleteExpired() + res.cacheGuard.Unlock() + } + } + }() + } + return res } func (c *dbCache) Open() { @@ -49,6 +75,8 @@ func (c *dbCache) Open() { func (c *dbCache) Close() { c.cacheGuard.Lock() defer c.cacheGuard.Unlock() + c.cancel() + c.wg.Wait() c.cache.Purge() c.closed = true } @@ -88,6 +116,8 @@ func (c *dbCache) getExisted(path string) *sharedDB { if value, ok := c.cache.Get(path); ok { return value + } else if value != nil { + c.cache.Invalidate(path) } return nil } @@ -119,10 +149,9 @@ func (c *dbCache) put(path string, db *sharedDB) bool { _, isNonCached := c.nonCached[path] - if !isNonCached && !c.closed { - c.cache.Add(path, db) - return true + if isNonCached || c.closed { + return false } - - return false + c.cache.Add(path, db) + return true } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go index e8b148bff..97606df02 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go @@ -17,6 +17,7 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) { const n = 1000 st := NewBlobovniczaTree( + context.Background(), WithLogger(test.NewLogger(t)), WithObjectSizeLimit(1024), WithBlobovniczaShallowWidth(10), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go index 11d4e78fe..0356fbaae 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go @@ -49,6 +49,7 @@ func createTestTree(t *testing.T, currentDepth, depth, width uint64, path string func openAndCloseTestTree(t *testing.T, depth, width uint64, path string) { blz := NewBlobovniczaTree( + context.Background(), WithBlobovniczaShallowDepth(depth), WithBlobovniczaShallowWidth(width), WithRootPath(path), @@ -78,6 +79,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { rootDir := t.TempDir() blz := NewBlobovniczaTree( + context.Background(), WithBlobovniczaShallowDepth(3), WithBlobovniczaShallowWidth(5), WithRootPath(rootDir), @@ -115,6 +117,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { // change depth and width blz = NewBlobovniczaTree( + context.Background(), WithBlobovniczaShallowDepth(5), WithBlobovniczaShallowWidth(2), WithRootPath(rootDir), @@ -152,6 +155,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { // change depth and width back blz = NewBlobovniczaTree( + context.Background(), WithBlobovniczaShallowDepth(3), WithBlobovniczaShallowWidth(5), WithRootPath(rootDir), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 048f9a2e5..625e620b8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -17,6 +17,7 @@ import ( func TestExistsInvalidStorageID(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( + context.Background(), WithLogger(test.NewLogger(t)), WithObjectSizeLimit(1024), WithBlobovniczaShallowWidth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go index 1a52eddf8..d390ecf1d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -13,6 +14,7 @@ func TestGeneric(t *testing.T) { helper := func(t *testing.T, dir string) common.Storage { return NewBlobovniczaTree( + context.Background(), WithLogger(test.NewLogger(t)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), @@ -40,6 +42,7 @@ func TestControl(t *testing.T) { newTree := func(t *testing.T) common.Storage { return NewBlobovniczaTree( + context.Background(), WithLogger(test.NewLogger(t)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go index c322fa7a3..506dd9a4b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go @@ -12,6 +12,7 @@ func TestIterateSortedLeavesAndDBPathsAreSame(t *testing.T) { t.Parallel() blz := NewBlobovniczaTree( + context.Background(), WithBlobovniczaShallowDepth(3), WithBlobovniczaShallowWidth(5), WithRootPath(t.TempDir()), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index d36074dc8..634499275 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -27,32 +27,40 @@ type cfg struct { blzInitWorkerCount int blzMoveBatchSize int createDBInAdvance bool + // TTL for blobovnicza's cache + openedCacheTTL time.Duration + // Interval for deletion expired blobovnicza's + openedCacheExpInterval time.Duration } type Option func(*cfg) const ( - defaultPerm = 0o700 - defaultOpenedCacheSize = 50 - defaultBlzShallowDepth = 2 - defaultBlzShallowWidth = 16 - defaultWaitBeforeDropDB = 10 * time.Second - defaultBlzInitWorkerCount = 5 - defaulBlzMoveBatchSize = 10000 + defaultPerm = 0o700 + defaultOpenedCacheSize = 50 + defaultOpenedCacheTTL = 0 // means expiring is off + defaultOpenedCacheInterval = 15 * time.Second + defaultBlzShallowDepth = 2 + defaultBlzShallowWidth = 16 + defaultWaitBeforeDropDB = 10 * time.Second + defaultBlzInitWorkerCount = 5 + defaulBlzMoveBatchSize = 10000 ) func initConfig(c *cfg) { *c = cfg{ - log: &logger.Logger{Logger: zap.L()}, - perm: defaultPerm, - openedCacheSize: defaultOpenedCacheSize, - blzShallowDepth: defaultBlzShallowDepth, - blzShallowWidth: defaultBlzShallowWidth, - reportError: func(string, error) {}, - metrics: &noopMetrics{}, - waitBeforeDropDB: defaultWaitBeforeDropDB, - blzInitWorkerCount: defaultBlzInitWorkerCount, - blzMoveBatchSize: defaulBlzMoveBatchSize, + log: &logger.Logger{Logger: zap.L()}, + perm: defaultPerm, + openedCacheSize: defaultOpenedCacheSize, + openedCacheTTL: defaultOpenedCacheTTL, + openedCacheExpInterval: defaultOpenedCacheInterval, + blzShallowDepth: defaultBlzShallowDepth, + blzShallowWidth: defaultBlzShallowWidth, + reportError: func(string, error) {}, + metrics: &noopMetrics{}, + waitBeforeDropDB: defaultWaitBeforeDropDB, + blzInitWorkerCount: defaultBlzInitWorkerCount, + blzMoveBatchSize: defaulBlzMoveBatchSize, } } @@ -105,6 +113,18 @@ func WithOpenedCacheSize(sz int) Option { } } +func WithOpenedCacheTTL(ttl time.Duration) Option { + return func(c *cfg) { + c.openedCacheTTL = ttl + } +} + +func WithOpenedCacheExpInterval(expInterval time.Duration) Option { + return func(c *cfg) { + c.openedCacheExpInterval = expInterval + } +} + func WithObjectSizeLimit(sz uint64) Option { return func(c *cfg) { c.blzOpts = append(c.blzOpts, blobovnicza.WithObjectSizeLimit(sz)) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index 855229405..31dadb533 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -129,6 +129,7 @@ func testRebuildFailoverObjectDeletedFromSource(t *testing.T) { func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object, mustUpdateStorageID bool) { b := NewBlobovniczaTree( + context.Background(), WithLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 578cb1a7f..6b8ae4b5c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -43,6 +43,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( + context.Background(), WithLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), // 64KB object size limit WithBlobovniczaShallowWidth(5), @@ -70,6 +71,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { require.NoError(t, b.Close()) b = NewBlobovniczaTree( + context.Background(), WithLogger(test.NewLogger(t)), WithObjectSizeLimit(32*1024), // 32KB object size limit WithBlobovniczaShallowWidth(5), @@ -108,6 +110,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, targetDepth, targetWidth uint64, shouldMigrate bool) { dir := t.TempDir() b := NewBlobovniczaTree( + context.Background(), WithLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(sourceWidth), @@ -148,6 +151,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta require.NoError(t, b.Close()) b = NewBlobovniczaTree( + context.Background(), WithLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(targetWidth), diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index 3dac3bfa4..c7d80dc84 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -18,6 +18,7 @@ import ( func defaultTestStorages(p string, smallSizeLimit uint64) ([]SubStorage, *teststore.TestStore, *teststore.TestStore) { smallFileStorage := teststore.New(teststore.WithSubstorage(blobovniczatree.NewBlobovniczaTree( + context.Background(), blobovniczatree.WithRootPath(filepath.Join(p, "blobovniczas")), blobovniczatree.WithBlobovniczaShallowWidth(1)), // default width is 16, slow init )) @@ -117,6 +118,7 @@ func TestBlobstor_needsCompression(t *testing.T) { WithStorages([]SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( + context.Background(), blobovniczatree.WithRootPath(filepath.Join(dir, "blobovnicza")), blobovniczatree.WithBlobovniczaShallowWidth(1)), // default width is 16, slow init Policy: func(_ *objectSDK.Object, data []byte) bool { diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index 72079acba..fb5afb88b 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -73,6 +73,7 @@ var storages = []storage{ desc: "blobovniczatree", create: func(dir string) common.Storage { return blobovniczatree.NewBlobovniczaTree( + context.Background(), blobovniczatree.WithRootPath(dir), ) }, diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index c41804e07..b82700ed8 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -137,6 +137,7 @@ func newStorages(t testing.TB, root string, smallSize uint64) []blobstor.SubStor return []blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( + context.Background(), blobovniczatree.WithRootPath(filepath.Join(root, "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1), @@ -158,6 +159,7 @@ func newStorages(t testing.TB, root string, smallSize uint64) []blobstor.SubStor func newTestStorages(root string, smallSize uint64) ([]blobstor.SubStorage, *teststore.TestStore, *teststore.TestStore) { smallFileStorage := teststore.New( teststore.WithSubstorage(blobovniczatree.NewBlobovniczaTree( + context.Background(), blobovniczatree.WithRootPath(filepath.Join(root, "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1), diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index 6a7941e04..3993593ad 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -36,6 +36,7 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( + context.Background(), blobovniczatree.WithLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 91033efb7..9ce95feb1 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -35,6 +35,7 @@ func TestShard_Lock(t *testing.T) { blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( + context.Background(), blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(2), blobovniczatree.WithBlobovniczaShallowWidth(2)), diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 5a8a29e0c..cc73db316 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -78,6 +78,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( + context.Background(), blobovniczatree.WithLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index a5483f632..73ba2e82b 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -59,6 +59,7 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard blobstor.WithStorages([]blobstor.SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( + context.Background(), blobovniczatree.WithLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(o.rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), From 3b9d12b11c45d4029b0990f93b6efa1a91a12198 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 27 Apr 2024 14:17:22 +0300 Subject: [PATCH 1056/1943] [#1110] node: Fix comment about nodeInfo type Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/netmap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index b1d854da4..f3cdfe711 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -31,7 +31,7 @@ type networkState struct { controlNetStatus atomic.Int32 // control.NetmapStatus - nodeInfo atomic.Value // *netmapSDK.NodeInfo + nodeInfo atomic.Value // netmapSDK.NodeInfo metrics *metrics.NodeMetrics } From c8e2ca2ab4777f0bf830045368ae12c0fd99c6e4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 27 Apr 2024 14:20:06 +0300 Subject: [PATCH 1057/1943] [#1110] node: Rename handleLocalNodeInfo() It is used in "handler" only once, what we really do is set the variable. And we have another "local" node info in `cfgNodeInfo`, this one is not really local (node info), more like (local node) info, so use setContractNodeInfo to distinguish it from the local view on the node info. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config.go | 4 ++-- cmd/frostfs-node/netmap.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index eb9f9fb18..cf4a1e4c3 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1157,10 +1157,10 @@ func (c *cfg) LocalNodeInfo() (*netmapV2.NodeInfo, error) { return &res, nil } -// handleLocalNodeInfo rewrites local node info from the FrostFS network map. +// setContractNodeInfo rewrites local node info from the FrostFS network map. // Called with nil when storage node is outside the FrostFS network map // (before entering the network and after leaving it). -func (c *cfg) handleLocalNodeInfo(ni *netmap.NodeInfo) { +func (c *cfg) setContractNodeInfo(ni *netmap.NodeInfo) { c.cfgNetmap.state.setNodeInfo(ni) } diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index f3cdfe711..683c186fa 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -199,7 +199,7 @@ func addNewEpochNotificationHandlers(c *cfg) { return } - c.handleLocalNodeInfo(ni) + c.setContractNodeInfo(ni) }) if c.cfgMorph.notaryEnabled { @@ -270,7 +270,7 @@ func initNetmapState(c *cfg) { c.cfgNetmap.state.setCurrentEpoch(epoch) c.cfgNetmap.startEpoch = epoch - c.handleLocalNodeInfo(ni) + c.setContractNodeInfo(ni) } func nodeState(ni *netmapSDK.NodeInfo) string { From 4b514f5ba02f6252eb2c721657a87c0da201c71b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 27 Apr 2024 14:48:36 +0300 Subject: [PATCH 1058/1943] [#1110] node: Log maintenance stop only if it was enabled Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index cf4a1e4c3..7bf0077bd 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -388,8 +388,9 @@ func (c *cfg) startMaintenance() { // stops node's maintenance. func (c *internals) stopMaintenance() { - c.isMaintenance.Store(false) - c.log.Info(logs.FrostFSNodeStoppedLocalNodesMaintenance) + if c.isMaintenance.CompareAndSwap(true, false) { + c.log.Info(logs.FrostFSNodeStoppedLocalNodesMaintenance) + } } // IsMaintenance checks if storage node is under maintenance. From 52bebe9452a2db8239c9c88a8f865ce42786510f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 27 Apr 2024 15:06:19 +0300 Subject: [PATCH 1059/1943] [#1110] node: Use single handler for new epoch event Bootstrap logic depends on the netmap status, which in turn depends on the node info. Updating them in a single thread makes things more predictable. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config.go | 12 ++++++++++++ cmd/frostfs-node/netmap.go | 22 +++++----------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 7bf0077bd..5606ed24c 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1165,6 +1165,18 @@ func (c *cfg) setContractNodeInfo(ni *netmap.NodeInfo) { c.cfgNetmap.state.setNodeInfo(ni) } +func (c *cfg) updateContractNodeInfo(epoch uint64) { + ni, err := c.netmapLocalNodeState(epoch) + if err != nil { + c.log.Error(logs.FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch, + zap.Uint64("epoch", epoch), + zap.String("error", err.Error())) + return + } + + c.setContractNodeInfo(ni) +} + // bootstrapWithState calls "addPeer" method of the Sidechain Netmap contract // with the binary-encoded information from the current node's configuration. // The state is set using the provided setter which MUST NOT be nil. diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 683c186fa..56f2ca98f 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -176,7 +176,11 @@ func addNewEpochNotificationHandlers(c *cfg) { c.cfgNetmap.state.setCurrentEpoch(ev.(netmapEvent.NewEpoch).EpochNumber()) }) - addNewEpochAsyncNotificationHandler(c, func(_ event.Event) { + addNewEpochAsyncNotificationHandler(c, func(ev event.Event) { + e := ev.(netmapEvent.NewEpoch).EpochNumber() + + c.updateContractNodeInfo(e) + if !c.needBootstrap() || c.cfgNetmap.reBoostrapTurnedOff.Load() { // fixes #470 return } @@ -186,22 +190,6 @@ func addNewEpochNotificationHandlers(c *cfg) { } }) - addNewEpochAsyncNotificationHandler(c, func(ev event.Event) { - e := ev.(netmapEvent.NewEpoch).EpochNumber() - - ni, err := c.netmapLocalNodeState(e) - if err != nil { - c.log.Error(logs.FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch, - zap.Uint64("epoch", e), - zap.String("error", err.Error()), - ) - - return - } - - c.setContractNodeInfo(ni) - }) - if c.cfgMorph.notaryEnabled { addNewEpochAsyncNotificationHandler(c, func(_ event.Event) { _, err := makeNotaryDeposit(c) From 4730ecfdb83dd52c34479167eb3eb630c0cf1862 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 27 Apr 2024 15:34:29 +0300 Subject: [PATCH 1060/1943] [#966] node: Refactor WriteCacheMetrics interface Grouping common fields of methods will enhance the readability of the interface. Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/engine/writecache.go | 18 +++++++++--------- pkg/metrics/mode.go | 4 ++-- pkg/metrics/writecache.go | 18 +++++++++--------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index f30bce00d..850e10d17 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -150,20 +150,20 @@ func (m *writeCacheMetrics) SetShardID(id string) { } func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddMethodDuration(m.shardID, "Get", success, d, st.String()) + m.metrics.AddMethodDuration(m.shardID, st.String(), "Get", success, d) } func (m *writeCacheMetrics) Delete(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddMethodDuration(m.shardID, "Delete", success, d, st.String()) + m.metrics.AddMethodDuration(m.shardID, st.String(), "Delete", success, d) } func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddMethodDuration(m.shardID, "Put", success, d, st.String()) + m.metrics.AddMethodDuration(m.shardID, st.String(), "Put", success, d) } func (m *writeCacheMetrics) SetEstimateSize(db, fstree uint64) { - m.metrics.SetEstimateSize(m.shardID, db, writecache.StorageTypeDB.String()) - m.metrics.SetEstimateSize(m.shardID, fstree, writecache.StorageTypeFSTree.String()) + m.metrics.SetEstimateSize(m.shardID, writecache.StorageTypeDB.String(), db) + m.metrics.SetEstimateSize(m.shardID, writecache.StorageTypeFSTree.String(), fstree) } func (m *writeCacheMetrics) SetMode(mode mode.Mode) { @@ -171,16 +171,16 @@ func (m *writeCacheMetrics) SetMode(mode mode.Mode) { } func (m *writeCacheMetrics) SetActualCounters(db, fstree uint64) { - m.metrics.SetActualCount(m.shardID, db, writecache.StorageTypeDB.String()) - m.metrics.SetActualCount(m.shardID, fstree, writecache.StorageTypeFSTree.String()) + m.metrics.SetActualCount(m.shardID, writecache.StorageTypeDB.String(), db) + m.metrics.SetActualCount(m.shardID, writecache.StorageTypeFSTree.String(), fstree) } func (m *writeCacheMetrics) Flush(success bool, st writecache.StorageType) { - m.metrics.IncOperationCounter(m.shardID, "Flush", metrics.NullBool{Bool: success, Valid: true}, st.String()) + m.metrics.IncOperationCounter(m.shardID, st.String(), "Flush", metrics.NullBool{Bool: success, Valid: true}) } func (m *writeCacheMetrics) Evict(st writecache.StorageType) { - m.metrics.IncOperationCounter(m.shardID, "Evict", metrics.NullBool{}, st.String()) + m.metrics.IncOperationCounter(m.shardID, st.String(), "Evict", metrics.NullBool{}) } func (m *writeCacheMetrics) Close() { diff --git a/pkg/metrics/mode.go b/pkg/metrics/mode.go index cf6bdac48..a9ac47acd 100644 --- a/pkg/metrics/mode.go +++ b/pkg/metrics/mode.go @@ -21,7 +21,7 @@ func newShardIDMode(subsystem, name, help string) *shardIDModeValue { } } -func (m *shardIDModeValue) SetMode(shardID string, mode string) { +func (m *shardIDModeValue) SetMode(shardID, mode string) { m.modeValue.DeletePartialMatch(prometheus.Labels{ shardIDLabel: shardID, }) @@ -54,7 +54,7 @@ func newShardIDPathMode(subsystem, name, help string) *shardIDPathModeValue { } } -func (m *shardIDPathModeValue) SetMode(shardID, path string, mode string) { +func (m *shardIDPathModeValue) SetMode(shardID, path, mode string) { m.modeValue.DeletePartialMatch(prometheus.Labels{ shardIDLabel: shardID, pathLabel: path, diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index ea3d8742a..70181ee2b 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -9,11 +9,11 @@ import ( ) type WriteCacheMetrics interface { - AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) - SetActualCount(shardID string, count uint64, storageType string) - SetEstimateSize(shardID string, size uint64, storageType string) - SetMode(shardID string, mode string) - IncOperationCounter(shardID string, operation string, success NullBool, storageType string) + AddMethodDuration(shardID, storageType, method string, success bool, d time.Duration) + SetActualCount(shardID, storageType string, count uint64) + SetEstimateSize(shardID, storageType string, size uint64) + SetMode(shardID, mode string) + IncOperationCounter(shardID, storageType, operation string, success NullBool) Close(shardID string) } @@ -48,7 +48,7 @@ func newWriteCacheMetrics() *writeCacheMetrics { } } -func (m *writeCacheMetrics) AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string) { +func (m *writeCacheMetrics) AddMethodDuration(shardID, storageType, method string, success bool, d time.Duration) { m.methodDuration.With( prometheus.Labels{ shardIDLabel: shardID, @@ -59,14 +59,14 @@ func (m *writeCacheMetrics) AddMethodDuration(shardID string, method string, suc ).Observe(d.Seconds()) } -func (m *writeCacheMetrics) SetActualCount(shardID string, count uint64, storageType string) { +func (m *writeCacheMetrics) SetActualCount(shardID, storageType string, count uint64) { m.actualCount.With(prometheus.Labels{ shardIDLabel: shardID, storageLabel: storageType, }).Set(float64(count)) } -func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storageType string) { +func (m *writeCacheMetrics) SetEstimateSize(shardID, storageType string, size uint64) { m.estimatedSize.With(prometheus.Labels{ shardIDLabel: shardID, storageLabel: storageType, @@ -77,7 +77,7 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { m.mode.SetMode(shardID, mode) } -func (m *writeCacheMetrics) IncOperationCounter(shardID string, operation string, success NullBool, storageType string) { +func (m *writeCacheMetrics) IncOperationCounter(shardID, storageType, operation string, success NullBool) { m.operationCounter.With(prometheus.Labels{ shardIDLabel: shardID, storageLabel: storageType, From c9efaa5819d40352a9a6ce904876344517010cf5 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Sat, 27 Apr 2024 15:49:07 +0300 Subject: [PATCH 1061/1943] [#966] node: Add path of the write_cache to metric labels Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/engine/writecache.go | 25 +++++++----- pkg/local_object_storage/shard/shard.go | 1 + .../writecache/metrics.go | 3 ++ pkg/metrics/writecache.go | 38 ++++++++++--------- 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 850e10d17..670cfcbf5 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -143,6 +143,11 @@ func (e *StorageEngine) SealWriteCache(ctx context.Context, prm SealWriteCachePr type writeCacheMetrics struct { shardID string metrics metrics.WriteCacheMetrics + path string +} + +func (m *writeCacheMetrics) SetPath(path string) { + m.path = path } func (m *writeCacheMetrics) SetShardID(id string) { @@ -150,20 +155,20 @@ func (m *writeCacheMetrics) SetShardID(id string) { } func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddMethodDuration(m.shardID, st.String(), "Get", success, d) + m.metrics.AddMethodDuration(m.shardID, m.path, st.String(), "Get", success, d) } func (m *writeCacheMetrics) Delete(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddMethodDuration(m.shardID, st.String(), "Delete", success, d) + m.metrics.AddMethodDuration(m.shardID, m.path, st.String(), "Delete", success, d) } func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.StorageType) { - m.metrics.AddMethodDuration(m.shardID, st.String(), "Put", success, d) + m.metrics.AddMethodDuration(m.shardID, m.path, st.String(), "Put", success, d) } func (m *writeCacheMetrics) SetEstimateSize(db, fstree uint64) { - m.metrics.SetEstimateSize(m.shardID, writecache.StorageTypeDB.String(), db) - m.metrics.SetEstimateSize(m.shardID, writecache.StorageTypeFSTree.String(), fstree) + m.metrics.SetEstimateSize(m.shardID, m.path, writecache.StorageTypeDB.String(), db) + m.metrics.SetEstimateSize(m.shardID, m.path, writecache.StorageTypeFSTree.String(), fstree) } func (m *writeCacheMetrics) SetMode(mode mode.Mode) { @@ -171,18 +176,18 @@ func (m *writeCacheMetrics) SetMode(mode mode.Mode) { } func (m *writeCacheMetrics) SetActualCounters(db, fstree uint64) { - m.metrics.SetActualCount(m.shardID, writecache.StorageTypeDB.String(), db) - m.metrics.SetActualCount(m.shardID, writecache.StorageTypeFSTree.String(), fstree) + m.metrics.SetActualCount(m.shardID, m.path, writecache.StorageTypeDB.String(), db) + m.metrics.SetActualCount(m.shardID, m.path, writecache.StorageTypeFSTree.String(), fstree) } func (m *writeCacheMetrics) Flush(success bool, st writecache.StorageType) { - m.metrics.IncOperationCounter(m.shardID, st.String(), "Flush", metrics.NullBool{Bool: success, Valid: true}) + m.metrics.IncOperationCounter(m.shardID, m.path, st.String(), "Flush", metrics.NullBool{Bool: success, Valid: true}) } func (m *writeCacheMetrics) Evict(st writecache.StorageType) { - m.metrics.IncOperationCounter(m.shardID, st.String(), "Evict", metrics.NullBool{}) + m.metrics.IncOperationCounter(m.shardID, m.path, st.String(), "Evict", metrics.NullBool{}) } func (m *writeCacheMetrics) Close() { - m.metrics.Close(m.shardID) + m.metrics.Close(m.shardID, m.path) } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index d9cd2b2f4..cf59c0bfc 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -182,6 +182,7 @@ func New(opts ...Option) *Shard { writecache.WithReportErrorFunc(reportFunc), writecache.WithBlobstor(bs), writecache.WithMetabase(mb))...) + s.writeCache.GetMetrics().SetPath(s.writeCache.DumpInfo().Path) } if s.piloramaOpts != nil { diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go index 492e0973a..962e22eea 100644 --- a/pkg/local_object_storage/writecache/metrics.go +++ b/pkg/local_object_storage/writecache/metrics.go @@ -29,6 +29,7 @@ type Metrics interface { SetEstimateSize(db, fstree uint64) SetMode(m mode.Mode) SetActualCounters(db, fstree uint64) + SetPath(path string) Close() } @@ -38,6 +39,8 @@ type metricsStub struct{} func (metricsStub) SetShardID(string) {} +func (metricsStub) SetPath(string) {} + func (metricsStub) Get(time.Duration, bool, StorageType) {} func (metricsStub) Delete(time.Duration, bool, StorageType) {} diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 70181ee2b..2fe01526e 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -9,12 +9,12 @@ import ( ) type WriteCacheMetrics interface { - AddMethodDuration(shardID, storageType, method string, success bool, d time.Duration) - SetActualCount(shardID, storageType string, count uint64) - SetEstimateSize(shardID, storageType string, size uint64) + AddMethodDuration(shardID, path, storageType, method string, success bool, d time.Duration) + SetActualCount(shardID, path, storageType string, count uint64) + SetEstimateSize(shardID, path, storageType string, size uint64) SetMode(shardID, mode string) - IncOperationCounter(shardID, storageType, operation string, success NullBool) - Close(shardID string) + IncOperationCounter(shardID, path, storageType, operation string, success NullBool) + Close(shardID, path string) } type writeCacheMetrics struct { @@ -41,35 +41,38 @@ func newWriteCacheMetrics() *writeCacheMetrics { Subsystem: writeCacheSubsystem, Name: "operations_total", Help: "The number of writecache operations processed", - }, []string{shardIDLabel, storageLabel, successLabel, operationLabel}), - actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{shardIDLabel, storageLabel}), - estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{shardIDLabel, storageLabel}), + }, []string{shardIDLabel, storageLabel, successLabel, operationLabel, pathLabel}), + actualCount: newWCGaugeVec("actual_objects_total", "Actual objects count in writecache", []string{shardIDLabel, storageLabel, pathLabel}), + estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{shardIDLabel, storageLabel, pathLabel}), mode: newShardIDMode(writeCacheSubsystem, "mode_info", "Writecache mode value"), } } -func (m *writeCacheMetrics) AddMethodDuration(shardID, storageType, method string, success bool, d time.Duration) { +func (m *writeCacheMetrics) AddMethodDuration(shardID, path, storageType, method string, success bool, d time.Duration) { m.methodDuration.With( prometheus.Labels{ shardIDLabel: shardID, successLabel: strconv.FormatBool(success), storageLabel: storageType, methodLabel: method, + pathLabel: path, }, ).Observe(d.Seconds()) } -func (m *writeCacheMetrics) SetActualCount(shardID, storageType string, count uint64) { +func (m *writeCacheMetrics) SetActualCount(shardID, path, storageType string, count uint64) { m.actualCount.With(prometheus.Labels{ shardIDLabel: shardID, storageLabel: storageType, + pathLabel: path, }).Set(float64(count)) } -func (m *writeCacheMetrics) SetEstimateSize(shardID, storageType string, size uint64) { +func (m *writeCacheMetrics) SetEstimateSize(shardID, path, storageType string, size uint64) { m.estimatedSize.With(prometheus.Labels{ shardIDLabel: shardID, storageLabel: storageType, + pathLabel: path, }).Set(float64(size)) } @@ -77,21 +80,22 @@ func (m *writeCacheMetrics) SetMode(shardID string, mode string) { m.mode.SetMode(shardID, mode) } -func (m *writeCacheMetrics) IncOperationCounter(shardID, storageType, operation string, success NullBool) { +func (m *writeCacheMetrics) IncOperationCounter(shardID, path, storageType, operation string, success NullBool) { m.operationCounter.With(prometheus.Labels{ shardIDLabel: shardID, storageLabel: storageType, operationLabel: operation, successLabel: success.String(), + pathLabel: path, }).Inc() } -func (m *writeCacheMetrics) Close(shardID string) { +func (m *writeCacheMetrics) Close(shardID, path string) { m.mode.Delete(shardID) - m.methodDuration.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) - m.operationCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) - m.actualCount.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) - m.estimatedSize.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) + m.methodDuration.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID, pathLabel: path}) + m.operationCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID, pathLabel: path}) + m.actualCount.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID, pathLabel: path}) + m.estimatedSize.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID, pathLabel: path}) } func newWCGaugeVec(name, help string, labels []string) *prometheus.GaugeVec { From 71789676d5b17ea62dc64a6afa5260df90b4fd26 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 2 May 2024 11:56:46 +0300 Subject: [PATCH 1062/1943] [#1114] aclsvc: Add tests for request ownership Signed-off-by: Evgenii Stratonikov --- pkg/services/object/acl/v2/request_test.go | 163 +++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 pkg/services/object/acl/v2/request_test.go diff --git a/pkg/services/object/acl/v2/request_test.go b/pkg/services/object/acl/v2/request_test.go new file mode 100644 index 000000000..6c6d79adf --- /dev/null +++ b/pkg/services/object/acl/v2/request_test.go @@ -0,0 +1,163 @@ +package v2 + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + sigutilV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/signature" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/google/uuid" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestRequestOwner(t *testing.T) { + containerOwner, err := keys.NewPrivateKey() + require.NoError(t, err) + + userPk, err := keys.NewPrivateKey() + require.NoError(t, err) + + var userID user.ID + user.IDFromKey(&userID, userPk.PrivateKey.PublicKey) + + var userSignature refs.Signature + userSignature.SetKey(userPk.PublicKey().Bytes()) + + vh := new(sessionV2.RequestVerificationHeader) + vh.SetBodySignature(&userSignature) + + t.Run("empty verification header", func(t *testing.T) { + req := MetaWithToken{} + checkOwner(t, req, nil, errEmptyVerificationHeader) + }) + t.Run("empty verification header signature", func(t *testing.T) { + req := MetaWithToken{ + vheader: new(sessionV2.RequestVerificationHeader), + } + checkOwner(t, req, nil, errEmptyBodySig) + }) + t.Run("no tokens", func(t *testing.T) { + req := MetaWithToken{ + vheader: vh, + } + checkOwner(t, req, userPk.PublicKey(), nil) + }) + + t.Run("bearer without impersonate, no session", func(t *testing.T) { + req := MetaWithToken{ + vheader: vh, + bearer: newBearer(t, containerOwner, userID, false), + } + checkOwner(t, req, userPk.PublicKey(), nil) + }) + t.Run("bearer with impersonate, no session", func(t *testing.T) { + req := MetaWithToken{ + vheader: vh, + bearer: newBearer(t, containerOwner, userID, true), + } + checkOwner(t, req, containerOwner.PublicKey(), nil) + }) + t.Run("bearer with impersonate, with session", func(t *testing.T) { + // To check that bearer token takes priority, use different key to sign session token. + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + req := MetaWithToken{ + vheader: vh, + bearer: newBearer(t, containerOwner, userID, true), + token: newSession(t, pk), + } + checkOwner(t, req, containerOwner.PublicKey(), nil) + }) + t.Run("with session", func(t *testing.T) { + req := MetaWithToken{ + vheader: vh, + token: newSession(t, containerOwner), + } + checkOwner(t, req, containerOwner.PublicKey(), nil) + }) + t.Run("malformed session token", func(t *testing.T) { + // This test is tricky: session token has issuer field and signature, which must correspond to each other. + // SDK prevents constructing such token in the first place, but it is still possible via API. + // Thus, construct v2 token, convert it to SDK one and pass to our function. + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + var user1 user.ID + user.IDFromKey(&user1, pk.PrivateKey.PublicKey) + + var id refs.OwnerID + id.SetValue(user1.WalletBytes()) + + raw, err := uuid.New().MarshalBinary() + require.NoError(t, err) + + var cidV2 refs.ContainerID + cidtest.ID().WriteToV2(&cidV2) + + sessionCtx := new(sessionV2.ObjectSessionContext) + sessionCtx.SetTarget(&cidV2) + + var body sessionV2.TokenBody + body.SetOwnerID(&id) + body.SetID(raw) + body.SetLifetime(new(sessionV2.TokenLifetime)) + body.SetSessionKey(pk.PublicKey().Bytes()) + body.SetContext(sessionCtx) + + var tokV2 sessionV2.Token + tokV2.SetBody(&body) + require.NoError(t, sigutilV2.SignData(&containerOwner.PrivateKey, smWrapper{Token: &tokV2})) + require.NoError(t, sigutilV2.VerifyData(smWrapper{Token: &tokV2})) + + var tok sessionSDK.Object + require.NoError(t, tok.ReadFromV2(tokV2)) + + req := MetaWithToken{ + vheader: vh, + token: &tok, + } + checkOwner(t, req, nil, errInvalidSessionOwner) + }) +} + +type smWrapper struct { + *sessionV2.Token +} + +func (s smWrapper) ReadSignedData(data []byte) ([]byte, error) { + return s.Token.GetBody().StableMarshal(data), nil +} +func (s smWrapper) SignedDataSize() int { + return s.Token.GetBody().StableSize() +} + +func newSession(t *testing.T, pk *keys.PrivateKey) *sessionSDK.Object { + var tok sessionSDK.Object + require.NoError(t, tok.Sign(pk.PrivateKey)) + return &tok +} + +func newBearer(t *testing.T, pk *keys.PrivateKey, user user.ID, impersonate bool) *bearer.Token { + var tok bearer.Token + tok.SetImpersonate(impersonate) + tok.ForUser(user) + require.NoError(t, tok.Sign(pk.PrivateKey)) + return &tok +} + +func checkOwner(t *testing.T, req MetaWithToken, expected *keys.PublicKey, expectedErr error) { + _, actual, err := req.RequestOwner() + if expectedErr != nil { + require.ErrorIs(t, err, expectedErr) + return + } + + require.NoError(t, err) + require.Equal(t, expected, actual) +} From ec2873caa7c67777f5fe96a87818c048754f706e Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 2 May 2024 20:03:33 +0300 Subject: [PATCH 1063/1943] [#1116] node: Fix writecache metrics Signed-off-by: Alexander Chuprov --- pkg/metrics/writecache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/writecache.go b/pkg/metrics/writecache.go index 2fe01526e..1b708f710 100644 --- a/pkg/metrics/writecache.go +++ b/pkg/metrics/writecache.go @@ -35,7 +35,7 @@ func newWriteCacheMetrics() *writeCacheMetrics { Subsystem: writeCacheSubsystem, Name: "request_duration_seconds", Help: "Writecache request process duration", - }, []string{shardIDLabel, successLabel, storageLabel, methodLabel}), + }, []string{shardIDLabel, successLabel, storageLabel, methodLabel, pathLabel}), operationCounter: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: writeCacheSubsystem, From e07869a8cf918349acfdcfa5058931352b3a9e71 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Sat, 27 Apr 2024 15:24:11 +0300 Subject: [PATCH 1064/1943] [#1100] Remove unused fields Signed-off-by: Ekaterina Lebedeva --- .../internal/modules/morph/helper/local_client.go | 8 +++----- cmd/frostfs-cli/modules/container/policy_playground.go | 8 +++----- pkg/innerring/innerring.go | 3 --- pkg/local_object_storage/pilorama/heap.go | 2 -- pkg/services/object/acl/v2/request_test.go | 1 + pkg/services/policer/option.go | 3 +-- 6 files changed, 8 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go index d309e7e5b..31203dde6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go @@ -44,7 +44,6 @@ type LocalClient struct { transactions []*transaction.Transaction dumpPath string accounts []*wallet.Account - maxGasInvoke int64 } func NewLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet, dumpPath string) (*LocalClient, error) { @@ -104,10 +103,9 @@ func NewLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet } return &LocalClient{ - bc: bc, - dumpPath: dumpPath, - accounts: accounts[:m], - maxGasInvoke: 15_0000_0000, + bc: bc, + dumpPath: dumpPath, + accounts: accounts[:m], }, nil } diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 1ac41f08c..095ab6438 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -20,14 +20,12 @@ import ( type policyPlaygroundREPL struct { cmd *cobra.Command - args []string nodes map[string]netmap.NodeInfo } -func newPolicyPlaygroundREPL(cmd *cobra.Command, args []string) (*policyPlaygroundREPL, error) { +func newPolicyPlaygroundREPL(cmd *cobra.Command) (*policyPlaygroundREPL, error) { return &policyPlaygroundREPL{ cmd: cmd, - args: args, nodes: map[string]netmap.NodeInfo{}, }, nil } @@ -221,8 +219,8 @@ var policyPlaygroundCmd = &cobra.Command{ Short: "A REPL for testing placement policies", Long: `A REPL for testing placement policies. If a wallet and endpoint is provided, the initial netmap data will be loaded from the snapshot of the node. Otherwise, an empty playground is created.`, - Run: func(cmd *cobra.Command, args []string) { - repl, err := newPolicyPlaygroundREPL(cmd, args) + Run: func(cmd *cobra.Command, _ []string) { + repl, err := newPolicyPlaygroundREPL(cmd) commonCmd.ExitOnErr(cmd, "could not create policy playground: %w", err) commonCmd.ExitOnErr(cmd, "policy playground failed: %w", repl.run()) }, diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 11d43f845..5dea18f9d 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -69,7 +69,6 @@ type ( // internal variables key *keys.PrivateKey - pubKey []byte contracts *contracts predefinedValidators keys.PublicKeys initialEpochTickDelta uint32 @@ -391,8 +390,6 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, fmt.Errorf("ir: can't parse predefined validators list: %w", err) } - server.pubKey = server.key.PublicKey().Bytes() - var morphClients *serverMorphClients morphClients, err = server.initClientsFromMorph() if err != nil { diff --git a/pkg/local_object_storage/pilorama/heap.go b/pkg/local_object_storage/pilorama/heap.go index 50abb663c..d81ec735d 100644 --- a/pkg/local_object_storage/pilorama/heap.go +++ b/pkg/local_object_storage/pilorama/heap.go @@ -29,7 +29,6 @@ func (h *filenameHeap) Pop() any { // fixedHeap maintains a fixed number of smallest elements started at some point. type fixedHeap struct { start string - max string count int h *filenameHeap } @@ -40,7 +39,6 @@ func newHeap(start string, count int) *fixedHeap { return &fixedHeap{ start: start, - max: "", count: count, h: h, } diff --git a/pkg/services/object/acl/v2/request_test.go b/pkg/services/object/acl/v2/request_test.go index 6c6d79adf..980d1a2e5 100644 --- a/pkg/services/object/acl/v2/request_test.go +++ b/pkg/services/object/acl/v2/request_test.go @@ -133,6 +133,7 @@ type smWrapper struct { func (s smWrapper) ReadSignedData(data []byte) ([]byte, error) { return s.Token.GetBody().StableMarshal(data), nil } + func (s smWrapper) SignedDataSize() int { return s.Token.GetBody().StableSize() } diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 92cf83a84..58c74ee1a 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -66,7 +66,7 @@ type cfg struct { batchSize, cacheSize uint32 - rebalanceFreq, evictDuration, sleepDuration time.Duration + evictDuration, sleepDuration time.Duration metrics MetricsRegister } @@ -76,7 +76,6 @@ func defaultCfg() *cfg { log: &logger.Logger{Logger: zap.L()}, batchSize: 10, cacheSize: 1024, // 1024 * address size = 1024 * 64 = 64 MiB - rebalanceFreq: 1 * time.Second, sleepDuration: 1 * time.Second, evictDuration: 30 * time.Second, metrics: noopMetrics{}, From 45f4e6939d93925f5cdf3d8bed604ae515711ea6 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Thu, 2 May 2024 19:45:09 +0300 Subject: [PATCH 1065/1943] [#1117] morph: Make frostfsid client provide GetSubjectExtended method Signed-off-by: Airat Arifullin --- pkg/morph/client/frostfsid/subject.go | 132 +++++++++++++++++++++++++- 1 file changed, 131 insertions(+), 1 deletion(-) diff --git a/pkg/morph/client/frostfsid/subject.go b/pkg/morph/client/frostfsid/subject.go index 52ae98ba5..169be6bc5 100644 --- a/pkg/morph/client/frostfsid/subject.go +++ b/pkg/morph/client/frostfsid/subject.go @@ -13,7 +13,10 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" ) -const methodGetSubject = "getSubject" +const ( + methodGetSubject = "getSubject" + methodGetSubjectExtended = "getSubjectExtended" +) func (c *Client) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) { prm := client.TestInvokePrm{} @@ -33,6 +36,24 @@ func (c *Client) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) return subj, nil } +func (c *Client) GetSubjectExtended(addr util.Uint160) (*frostfsidclient.SubjectExtended, error) { + prm := client.TestInvokePrm{} + prm.SetMethod(methodGetSubjectExtended) + prm.SetArgs(addr) + + res, err := c.client.TestInvoke(prm) + if err != nil { + return nil, fmt.Errorf("could not perform test invocation (%s): %w", methodGetSubject, err) + } + + subj, err := parseSubjectExtended(res) + if err != nil { + return nil, fmt.Errorf("could not parse test invocation result (%s): %w", methodGetSubject, err) + } + + return subj, nil +} + func parseSubject(res []stackitem.Item) (*frostfsidclient.Subject, error) { if ln := len(res); ln != 1 { return nil, fmt.Errorf("unexpected stack item count (%s): %d", methodGetSubject, ln) @@ -79,6 +100,64 @@ func parseSubject(res []stackitem.Item) (*frostfsidclient.Subject, error) { return &subj, nil } +func parseSubjectExtended(res []stackitem.Item) (*frostfsidclient.SubjectExtended, error) { + if ln := len(res); ln != 1 { + return nil, fmt.Errorf("unexpected stack item count (%s): %d", methodGetSubject, ln) + } + + structArr, err := client.ArrayFromStackItem(res[0]) + if err != nil { + return nil, fmt.Errorf("could not get item array of container (%s): %w", methodGetSubject, err) + } + + var subj frostfsidclient.SubjectExtended + + subj.PrimaryKey, err = unwrap.PublicKey(makeValidRes(structArr[0])) + if err != nil { + return nil, err + } + + if !structArr[1].Equals(stackitem.Null{}) { + subj.AdditionalKeys, err = unwrap.ArrayOfPublicKeys(makeValidRes(structArr[1])) + if err != nil { + return nil, err + } + } + + if !structArr[2].Equals(stackitem.Null{}) { + subj.Namespace, err = unwrap.UTF8String(makeValidRes(structArr[2])) + if err != nil { + return nil, err + } + } + + if !structArr[3].Equals(stackitem.Null{}) { + subj.Name, err = unwrap.UTF8String(makeValidRes(structArr[3])) + if err != nil { + return nil, err + } + } + + subj.KV, err = parseMap(structArr[4]) + if err != nil { + return nil, err + } + + if !structArr[5].Equals(stackitem.Null{}) { + groupItems, ok := structArr[5].Value().([]stackitem.Item) + if !ok { + return nil, errors.New("invalid groups field") + } + + subj.Groups, err = parseGroups(groupItems) + if err != nil { + return nil, err + } + } + + return &subj, nil +} + func makeValidRes(item stackitem.Item) (*result.Invoke, error) { return &result.Invoke{ Stack: []stackitem.Item{item}, @@ -116,3 +195,54 @@ func parseMap(item stackitem.Item) (map[string]string, error) { return res, nil } + +func parseGroups(items []stackitem.Item) ([]*frostfsidclient.Group, error) { + var err error + res := make([]*frostfsidclient.Group, len(items)) + + for i := 0; i < len(items); i++ { + arr, ok := items[i].Value().([]stackitem.Item) + if !ok { + return nil, errors.New("invalid group type") + } + res[i], err = parseGroup(arr) + if err != nil { + return nil, err + } + } + + return res, nil +} + +func parseGroup(structArr []stackitem.Item) (*frostfsidclient.Group, error) { + if len(structArr) < 4 { + return nil, errors.New("invalid response group struct") + } + + groupID, err := structArr[0].TryInteger() + if err != nil { + return nil, err + } + + name, err := structArr[1].TryBytes() + if err != nil { + return nil, err + } + + namespace, err := structArr[2].TryBytes() + if err != nil { + return nil, err + } + + kvs, err := parseMap(structArr[3]) + if err != nil { + return nil, err + } + + return &frostfsidclient.Group{ + ID: groupID.Int64(), + Name: string(name), + Namespace: string(namespace), + KV: kvs, + }, nil +} From 6c76c9b457b24349d6fc1ff0d262f57006931b03 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Thu, 2 May 2024 20:26:06 +0300 Subject: [PATCH 1066/1943] [#1117] core: Introduce SubjectProvider interface for FrostfsID * Make tree, object and container services use SubjectProvider interface. * Fix unit-tests. Signed-off-by: Airat Arifullin --- pkg/ape/request/request.go | 2 +- pkg/core/frostfsid/subject_provider.go | 16 ++++++++++ pkg/morph/client/frostfsid/client.go | 3 ++ pkg/services/container/ape.go | 19 +++--------- pkg/services/container/ape_test.go | 17 ++++++++-- pkg/services/object/ape/checker.go | 11 ++----- pkg/services/object/ape/checker_test.go | 41 ++++++++++++++++++++++--- pkg/services/object/ape/request.go | 7 ++--- pkg/services/tree/ape.go | 5 ++- pkg/services/tree/options.go | 11 ++----- 10 files changed, 85 insertions(+), 47 deletions(-) create mode 100644 pkg/core/frostfsid/subject_provider.go diff --git a/pkg/ape/request/request.go b/pkg/ape/request/request.go index 6d62ef3d5..de67dea23 100644 --- a/pkg/ape/request/request.go +++ b/pkg/ape/request/request.go @@ -1,4 +1,4 @@ -package ape +package request import ( aperesource "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" diff --git a/pkg/core/frostfsid/subject_provider.go b/pkg/core/frostfsid/subject_provider.go new file mode 100644 index 000000000..ecfd0eb15 --- /dev/null +++ b/pkg/core/frostfsid/subject_provider.go @@ -0,0 +1,16 @@ +package frostfsid + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + "github.com/nspcc-dev/neo-go/pkg/util" +) + +const ( + SubjectNotFoundErrorMessage = "subject not found" +) + +// SubjectProvider interface provides methods to get subject from FrostfsID contract. +type SubjectProvider interface { + GetSubject(util.Uint160) (*client.Subject, error) + GetSubjectExtended(util.Uint160) (*client.SubjectExtended, error) +} diff --git a/pkg/morph/client/frostfsid/client.go b/pkg/morph/client/frostfsid/client.go index 3efa522bc..4c31f42de 100644 --- a/pkg/morph/client/frostfsid/client.go +++ b/pkg/morph/client/frostfsid/client.go @@ -3,6 +3,7 @@ package frostfsid import ( "fmt" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/util" @@ -20,6 +21,8 @@ type Client struct { client *client.StaticClient // static FrostFS ID contract client } +var _ frostfsidcore.SubjectProvider = (*Client)(nil) + // NewFromMorph wraps client to work with FrostFS ID contract. func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8) (*Client, error) { sc, err := client.NewStatic(cli, contract, fee, client.TryNotary(), client.AsAlphabet()) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 2ce427fa3..9c38d1bba 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -14,9 +14,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -30,11 +30,6 @@ import ( commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/util" -) - -const ( - subjectNotFoundErrorMessage = "subject not found" ) var ( @@ -59,22 +54,18 @@ type containers interface { Get(cid.ID) (*containercore.Container, error) } -type frostfsidSubjectProvider interface { - GetSubject(util.Uint160) (*client.Subject, error) -} - type apeChecker struct { router policyengine.ChainRouter reader containers ir ir nm netmap.Source - frostFSIDClient frostfsidSubjectProvider + frostFSIDClient frostfsidcore.SubjectProvider next Server } -func NewAPEServer(router policyengine.ChainRouter, reader containers, ir ir, nm netmap.Source, frostFSIDClient frostfsidSubjectProvider, srv Server) Server { +func NewAPEServer(router policyengine.ChainRouter, reader containers, ir ir, nm netmap.Source, frostFSIDClient frostfsidcore.SubjectProvider, srv Server) Server { return &apeChecker{ router: router, reader: reader, @@ -574,7 +565,7 @@ func (ac *apeChecker) namespaceByOwner(owner *refs.OwnerID) (string, error) { if err == nil { namespace = subject.Namespace } else { - if !strings.Contains(err.Error(), subjectNotFoundErrorMessage) { + if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { return "", fmt.Errorf("get subject error: %w", err) } } @@ -630,7 +621,7 @@ func (ac *apeChecker) fillWithUserClaimTags(reqProps map[string]string, pk *keys } subj, err := ac.frostFSIDClient.GetSubject(pk.GetScriptHash()) if err != nil { - if !strings.Contains(err.Error(), subjectNotFoundErrorMessage) { + if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { return nil, fmt.Errorf("get subject error: %w", err) } return reqProps, nil diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index e26e82d43..a2fe79956 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -15,6 +15,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -922,13 +923,22 @@ func (s *netmapStub) Epoch() (uint64, error) { } type frostfsidStub struct { - subjects map[util.Uint160]*client.Subject + subjects map[util.Uint160]*client.Subject + subjectsExt map[util.Uint160]*client.SubjectExtended } func (f *frostfsidStub) GetSubject(owner util.Uint160) (*client.Subject, error) { s, ok := f.subjects[owner] if !ok { - return nil, fmt.Errorf("%s", subjectNotFoundErrorMessage) + return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) + } + return s, nil +} + +func (f *frostfsidStub) GetSubjectExtended(owner util.Uint160) (*client.SubjectExtended, error) { + s, ok := f.subjectsExt[owner] + if !ok { + return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) } return s, nil } @@ -965,7 +975,8 @@ func newTestAPEServer() testAPEServer { netmap := &netmapStub{} frostfsIDSubjectReader := &frostfsidStub{ - subjects: map[util.Uint160]*client.Subject{}, + subjects: map[util.Uint160]*client.Subject{}, + subjectsExt: map[util.Uint160]*client.SubjectExtended{}, } apeChecker := &apeChecker{ diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 170787317..e74b05379 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -6,7 +6,7 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -14,7 +14,6 @@ import ( policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/util" ) type checkerImpl struct { @@ -22,14 +21,10 @@ type checkerImpl struct { headerProvider HeaderProvider - frostFSIDClient frostfsidSubjectProvider + frostFSIDClient frostfsidcore.SubjectProvider } -type frostfsidSubjectProvider interface { - GetSubject(util.Uint160) (*client.Subject, error) -} - -func NewChecker(chainRouter policyengine.ChainRouter, headerProvider HeaderProvider, frostFSIDClient frostfsidSubjectProvider) Checker { +func NewChecker(chainRouter policyengine.ChainRouter, headerProvider HeaderProvider, frostFSIDClient frostfsidcore.SubjectProvider) Checker { return &checkerImpl{ chainRouter: chainRouter, diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index 9510bf039..c591bc494 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -7,6 +7,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -156,14 +157,15 @@ var ( ) type frostfsIDProviderMock struct { - m map[util.Uint160]*client.Subject + subjects map[util.Uint160]*client.Subject + subjectsExtended map[util.Uint160]*client.SubjectExtended } -var _ frostfsidSubjectProvider = (*frostfsIDProviderMock)(nil) +var _ frostfsidcore.SubjectProvider = (*frostfsIDProviderMock)(nil) func newFrostfsIDProviderMock(t *testing.T) *frostfsIDProviderMock { return &frostfsIDProviderMock{ - m: map[util.Uint160]*client.Subject{ + subjects: map[util.Uint160]*client.Subject{ scriptHashFromSenderKey(t, senderKey): { Namespace: "testnamespace", Name: "test", @@ -173,6 +175,27 @@ func newFrostfsIDProviderMock(t *testing.T) *frostfsIDProviderMock { }, }, }, + subjectsExtended: map[util.Uint160]*client.SubjectExtended{ + scriptHashFromSenderKey(t, senderKey): { + Namespace: "testnamespace", + Name: "test", + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + Groups: []*client.Group{ + { + ID: 1, + Name: "test", + Namespace: "testnamespace", + KV: map[string]string{ + "attr1": "value1", + "attr2": "value2", + }, + }, + }, + }, + }, } } @@ -183,9 +206,17 @@ func scriptHashFromSenderKey(t *testing.T, senderKey string) util.Uint160 { } func (f *frostfsIDProviderMock) GetSubject(key util.Uint160) (*client.Subject, error) { - v, ok := f.m[key] + v, ok := f.subjects[key] if !ok { - return nil, fmt.Errorf("%s", subjectNotFoundErrorMessage) + return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) + } + return v, nil +} + +func (f *frostfsIDProviderMock) GetSubjectExtended(key util.Uint160) (*client.SubjectExtended, error) { + v, ok := f.subjectsExtended[key] + if !ok { + return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) } return v, nil } diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index 5daf8751b..82b6adb1f 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -8,6 +8,7 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -18,10 +19,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) -const ( - subjectNotFoundErrorMessage = "subject not found" -) - var defaultRequest = aperequest.Request{} func nativeSchemaRole(role acl.Role) string { @@ -153,7 +150,7 @@ func (c *checkerImpl) fillWithUserClaimTags(reqProps map[string]string, prm Prm) } subj, err := c.frostFSIDClient.GetSubject(pk.GetScriptHash()) if err != nil { - if !strings.Contains(err.Error(), subjectNotFoundErrorMessage) { + if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { return nil, fmt.Errorf("get subject error: %w", err) } return reqProps, nil diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 76ec254f0..47e7cb41d 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/converter" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" @@ -19,8 +20,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) -var subjectNotFoundErrorMessage = "subject not found" - func (s *Service) checkAPE(container *core.Container, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey) error { namespace := "" cntNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(container.Value).Zone(), ".ns") @@ -83,7 +82,7 @@ func (s *Service) fillWithUserClaimTags(reqProps map[string]string, publicKey *k } subj, err := s.frostfsidSubjectProvider.GetSubject(publicKey.GetScriptHash()) if err != nil { - if !strings.Contains(err.Error(), subjectNotFoundErrorMessage) { + if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { return nil, fmt.Errorf("get subject error: %w", err) } return reqProps, nil diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index c4ef80856..a99ba6046 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -4,21 +4,16 @@ import ( "crypto/ecdsa" "time" - "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/util" ) -type FrostfsidSubjectProvider interface { - GetSubject(util.Uint160) (*client.Subject, error) -} - type ContainerSource interface { container.Source @@ -36,7 +31,7 @@ type cfg struct { rawPub []byte nmSource netmap.Source cnrSource ContainerSource - frostfsidSubjectProvider FrostfsidSubjectProvider + frostfsidSubjectProvider frostfsidcore.SubjectProvider eaclSource container.EACLSource forest pilorama.Forest // replication-related parameters @@ -62,7 +57,7 @@ func WithContainerSource(src ContainerSource) Option { } } -func WithFrostfsidSubjectProvider(provider FrostfsidSubjectProvider) Option { +func WithFrostfsidSubjectProvider(provider frostfsidcore.SubjectProvider) Option { return func(c *cfg) { c.frostfsidSubjectProvider = provider } From b60a51b862eb699478cd5c59b3b9222a5da8c60d Mon Sep 17 00:00:00 2001 From: aarifullin Date: Thu, 2 May 2024 21:15:58 +0300 Subject: [PATCH 1067/1943] [#1117] ape: Introduce `FormFrostfsIDRequestProperties` method * `FormFrostfsIDRequestProperties` gets user claim tags and group id and sets them as ape request properties. * Make tree, container and object service use the method. * Fix unit-tests. Signed-off-by: Airat Arifullin --- pkg/ape/request/frostfsid.go | 36 +++++ pkg/services/container/ape.go | 13 +- pkg/services/container/ape_test.go | 170 ++++++++++++++++++++++++ pkg/services/object/ape/request.go | 15 +-- pkg/services/object/ape/request_test.go | 1 + pkg/services/tree/ape.go | 14 +- 6 files changed, 219 insertions(+), 30 deletions(-) create mode 100644 pkg/ape/request/frostfsid.go diff --git a/pkg/ape/request/frostfsid.go b/pkg/ape/request/frostfsid.go new file mode 100644 index 000000000..fa620e58c --- /dev/null +++ b/pkg/ape/request/frostfsid.go @@ -0,0 +1,36 @@ +package request + +import ( + "fmt" + "strconv" + "strings" + + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +// FormFrostfsIDRequestProperties forms frostfsid specific request properties like user-claim tags and group ID. +func FormFrostfsIDRequestProperties(frostFSIDClient frostfsidcore.SubjectProvider, pk *keys.PublicKey) (map[string]string, error) { + reqProps := make(map[string]string) + subj, err := frostFSIDClient.GetSubjectExtended(pk.GetScriptHash()) + if err != nil { + if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { + return nil, fmt.Errorf("get subject error: %w", err) + } + return reqProps, nil + } + for k, v := range subj.KV { + propertyKey := fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, k) + reqProps[propertyKey] = v + } + + groups := make([]string, len(subj.Groups)) + for i, group := range subj.Groups { + groups[i] = strconv.FormatInt(group.ID, 10) + } + reqProps[commonschema.PropertyKeyFrostFSIDGroupID] = apechain.FormCondSliceContainsValue(groups) + + return reqProps, nil +} diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 9c38d1bba..22440ab80 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -27,7 +27,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" - commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) @@ -619,16 +618,12 @@ func (ac *apeChecker) fillWithUserClaimTags(reqProps map[string]string, pk *keys if reqProps == nil { reqProps = make(map[string]string) } - subj, err := ac.frostFSIDClient.GetSubject(pk.GetScriptHash()) + props, err := aperequest.FormFrostfsIDRequestProperties(ac.frostFSIDClient, pk) if err != nil { - if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { - return nil, fmt.Errorf("get subject error: %w", err) - } - return reqProps, nil + return reqProps, err } - for k, v := range subj.KV { - properyKey := fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, k) - reqProps[properyKey] = v + for propertyName, properyValue := range props { + reqProps[propertyName] = properyValue } return reqProps, nil } diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index a2fe79956..94c3027c0 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -45,6 +45,7 @@ func TestAPE(t *testing.T) { t.Run("deny get container no rule found", testDenyGetContainerNoRuleFound) t.Run("deny get container for others", testDenyGetContainerForOthers) t.Run("deny get container by user claim tag", testDenyGetContainerByUserClaimTag) + t.Run("deny get container by group id", testDenyGetContainerByGroupID) t.Run("deny set container eACL for IR", testDenySetContainerEACLForIR) t.Run("deny get container eACL for IR with session token", testDenyGetContainerEACLForIRSessionToken) t.Run("deny put container for others with session token", testDenyPutContainerForOthersSessionToken) @@ -279,6 +280,19 @@ func testDenyGetContainerByUserClaimTag(t *testing.T) { }, }, }, + subjectsExt: map[util.Uint160]*client.SubjectExtended{ + pk.PublicKey().GetScriptHash(): { + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + Groups: []*client.Group{ + { + ID: 19888, + }, + }, + }, + }, } apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) @@ -339,6 +353,104 @@ func testDenyGetContainerByUserClaimTag(t *testing.T) { require.ErrorAs(t, err, &errAccessDenied) } +func testDenyGetContainerByGroupID(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{ + pk.PublicKey().GetScriptHash(): { + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + }, + }, + subjectsExt: map[util.Uint160]*client.SubjectExtended{ + pk.PublicKey().GetScriptHash(): { + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + Groups: []*client.Group{ + { + ID: 19888, + }, + }, + }, + }, + } + + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) + + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + contRdr.c[contID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(nm.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + nm.netmaps[nm.currentEpoch] = &testNetmap + nm.netmaps[nm.currentEpoch-1] = &testNetmap + + _, _, err = router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Object: chain.ObjectRequest, + Key: commonschema.PropertyKeyFrostFSIDGroupID, + Value: "19888", + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := &container.GetRequest{} + req.SetBody(&container.GetRequestBody{}) + var refContID refs.ContainerID + contID.WriteToV2(&refContID) + req.GetBody().SetContainerID(&refContID) + + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + resp, err := apeSrv.Get(context.Background(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) +} + func testDenySetContainerEACLForIR(t *testing.T) { t.Parallel() srv := &srvStub{ @@ -621,6 +733,21 @@ func testDenyPutContainerReadNamespaceFromFrostfsID(t *testing.T) { Name: testDomainName, }, }, + subjectsExt: map[util.Uint160]*client.SubjectExtended{ + ownerScriptHash: { + Namespace: testDomainName, + Name: testDomainName, + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + Groups: []*client.Group{ + { + ID: 19888, + }, + }, + }, + }, } apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) resp, err := apeSrv.Put(context.Background(), req) @@ -690,6 +817,21 @@ func testDenyPutContainerInvalidNamespace(t *testing.T) { Name: testDomainName, }, }, + subjectsExt: map[util.Uint160]*client.SubjectExtended{ + ownerScriptHash: { + Namespace: testDomainName, + Name: testDomainName, + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + Groups: []*client.Group{ + { + ID: 19888, + }, + }, + }, + }, } apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) resp, err := apeSrv.Put(context.Background(), req) @@ -800,6 +942,34 @@ func testDenyListContainersValidationNamespaceError(t *testing.T) { Name: testDomainName, }, }, + subjectsExt: map[util.Uint160]*client.SubjectExtended{ + actorScriptHash: { + Namespace: actorDomain, + Name: actorDomain, + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + Groups: []*client.Group{ + { + ID: 19777, + }, + }, + }, + ownerScriptHash: { + Namespace: testDomainName, + Name: testDomainName, + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + Groups: []*client.Group{ + { + ID: 19888, + }, + }, + }, + }, } apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index 82b6adb1f..cbecc73e3 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -4,17 +4,14 @@ import ( "context" "fmt" "strconv" - "strings" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" - frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) @@ -148,16 +145,12 @@ func (c *checkerImpl) fillWithUserClaimTags(reqProps map[string]string, prm Prm) if err != nil { return nil, err } - subj, err := c.frostFSIDClient.GetSubject(pk.GetScriptHash()) + props, err := aperequest.FormFrostfsIDRequestProperties(c.frostFSIDClient, pk) if err != nil { - if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { - return nil, fmt.Errorf("get subject error: %w", err) - } - return reqProps, nil + return reqProps, err } - for k, v := range subj.KV { - properyKey := fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, k) - reqProps[properyKey] = v + for propertyName, properyValue := range props { + reqProps[propertyName] = properyValue } return reqProps, nil } diff --git a/pkg/services/object/ape/request_test.go b/pkg/services/object/ape/request_test.go index cda78e69c..5c74e7d1d 100644 --- a/pkg/services/object/ape/request_test.go +++ b/pkg/services/object/ape/request_test.go @@ -275,6 +275,7 @@ func TestNewAPERequest(t *testing.T) { nativeschema.PropertyKeyActorRole: prm.Role, fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, "tag-attr1"): "value1", fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, "tag-attr2"): "value2", + commonschema.PropertyKeyFrostFSIDGroupID: "1", }, ) diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 47e7cb41d..622049d6a 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -8,14 +8,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/converter" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" - commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) @@ -80,16 +78,12 @@ func (s *Service) fillWithUserClaimTags(reqProps map[string]string, publicKey *k if reqProps == nil { reqProps = make(map[string]string) } - subj, err := s.frostfsidSubjectProvider.GetSubject(publicKey.GetScriptHash()) + props, err := aperequest.FormFrostfsIDRequestProperties(s.frostfsidSubjectProvider, publicKey) if err != nil { - if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { - return nil, fmt.Errorf("get subject error: %w", err) - } - return reqProps, nil + return reqProps, err } - for k, v := range subj.KV { - properyKey := fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, k) - reqProps[properyKey] = v + for propertyName, properyValue := range props { + reqProps[propertyName] = properyValue } return reqProps, nil } From 9cc51f86b7c4276a0bec623e494745c92f21fffa Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 6 May 2024 19:30:19 +0300 Subject: [PATCH 1068/1943] [#1117] node: Introduce cache for frostfsid contract client Signed-off-by: Airat Arifullin --- cmd/frostfs-node/config.go | 4 +- cmd/frostfs-node/config/morph/config.go | 15 ++++++ cmd/frostfs-node/container.go | 13 ++++- cmd/frostfs-node/frostfsid.go | 71 +++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 cmd/frostfs-node/frostfsid.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 5606ed24c..d713cf2fb 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -32,6 +32,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" @@ -46,7 +47,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" containerClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - frostfsidClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmap2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" @@ -421,7 +421,7 @@ type shared struct { cnrClient *containerClient.Client - frostfsidClient *frostfsidClient.Client + frostfsidClient frostfsidcore.SubjectProvider respSvc *response.Service diff --git a/cmd/frostfs-node/config/morph/config.go b/cmd/frostfs-node/config/morph/config.go index c32940625..a240c5d49 100644 --- a/cmd/frostfs-node/config/morph/config.go +++ b/cmd/frostfs-node/config/morph/config.go @@ -28,6 +28,9 @@ const ( // APEChainCacheSizeDefault is a default value of APE chain cache. APEChainCacheSizeDefault = 10_000 + + // FrostfsIDCacheSizeDefault is a default value of APE chain cache. + FrostfsIDCacheSizeDefault = 10_000 ) var errNoMorphEndpoints = errors.New("no morph chain RPC endpoints, see `morph.rpc_endpoint` section") @@ -114,3 +117,15 @@ func APEChainCacheSize(c *config.Config) uint32 { } return config.Uint32Safe(c.Sub(subsection), "ape_chain_cache_size") } + +// FrostfsIDCacheSize returns the value of "frostfsid_cache_size" config parameter +// from "morph" section. +// +// Returns 0 if the value is not positive integer. +// Returns FrostfsIDCacheSizeDefault if the value is missing. +func FrostfsIDCacheSize(c *config.Config) uint32 { + if c.Sub(subsection).Value("frostfsid_cache_size") == nil { + return FrostfsIDCacheSizeDefault + } + return config.Uint32Safe(c.Sub(subsection), "frostfsid_cache_size") +} diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index e006648ed..61600376b 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -6,8 +6,10 @@ import ( "net" containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc" + morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -33,9 +35,16 @@ func initContainerService(_ context.Context, c *cfg) { cnrRdr, cnrWrt := configureEACLAndContainerSources(c, wrap, cnrSrc) - frostFSIDClient, err := frostfsid.NewFromMorph(c.cfgMorph.client, c.cfgFrostfsID.scriptHash, 0) + var frostfsIDSubjectProvider frostfsidcore.SubjectProvider + frostfsIDSubjectProvider, err = frostfsid.NewFromMorph(c.cfgMorph.client, c.cfgFrostfsID.scriptHash, 0) fatalOnErr(err) - c.shared.frostfsidClient = frostFSIDClient + + cacheSize := morphconfig.FrostfsIDCacheSize(c.appCfg) + if cacheSize > 0 { + frostfsIDSubjectProvider = newMorphFrostfsIDCache(frostfsIDSubjectProvider, int(cacheSize), c.cfgMorph.cacheTTL) + } + + c.shared.frostfsidClient = frostfsIDSubjectProvider server := containerTransportGRPC.New( containerService.NewSignService( diff --git a/cmd/frostfs-node/frostfsid.go b/cmd/frostfs-node/frostfsid.go new file mode 100644 index 000000000..68cc1d040 --- /dev/null +++ b/cmd/frostfs-node/frostfsid.go @@ -0,0 +1,71 @@ +package main + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" + "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/nspcc-dev/neo-go/pkg/util" +) + +type morphFrostfsIDCache struct { + subjProvider frostfsidcore.SubjectProvider + + subjCache *expirable.LRU[util.Uint160, *client.Subject] + + subjExtCache *expirable.LRU[util.Uint160, *client.SubjectExtended] +} + +func newMorphFrostfsIDCache(subjProvider frostfsidcore.SubjectProvider, size int, ttl time.Duration) frostfsidcore.SubjectProvider { + return &morphFrostfsIDCache{ + subjProvider: subjProvider, + + subjCache: expirable.NewLRU(size, func(util.Uint160, *client.Subject) {}, ttl), + + subjExtCache: expirable.NewLRU(size, func(util.Uint160, *client.SubjectExtended) {}, ttl), + } +} + +func (m *morphFrostfsIDCache) GetSubject(addr util.Uint160) (*client.Subject, error) { + result, found := m.subjCache.Get(addr) + if found { + return result, nil + } + + result, err := m.subjProvider.GetSubject(addr) + if err != nil { + return nil, err + } + + m.subjCache.Add(addr, result) + return result, nil +} + +func (m *morphFrostfsIDCache) GetSubjectExtended(addr util.Uint160) (*client.SubjectExtended, error) { + subjExt, found := m.subjExtCache.Get(addr) + if found { + return subjExt, nil + } + + var err error + subjExt, err = m.subjProvider.GetSubjectExtended(addr) + if err != nil { + return nil, err + } + + m.subjExtCache.Add(addr, subjExt) + m.subjCache.Add(addr, subjectFromSubjectExtended(subjExt)) + + return subjExt, nil +} + +func subjectFromSubjectExtended(subjExt *client.SubjectExtended) *client.Subject { + return &client.Subject{ + PrimaryKey: subjExt.PrimaryKey, + AdditionalKeys: subjExt.AdditionalKeys, + Namespace: subjExt.Name, + Name: subjExt.Name, + KV: subjExt.KV, + } +} From fc7b07f3145c735a1cf7ab5d1b6b75f4f1aec3e9 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 6 May 2024 14:52:15 +0300 Subject: [PATCH 1069/1943] [#1118] docs: Fix pre-commit warnings Signed-off-by: Evgenii Stratonikov --- docs/images/authentication/impersonate.puml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/images/authentication/impersonate.puml b/docs/images/authentication/impersonate.puml index e9feae6e5..f0a5436f9 100644 --- a/docs/images/authentication/impersonate.puml +++ b/docs/images/authentication/impersonate.puml @@ -12,4 +12,4 @@ else (no) end endif -@enduml \ No newline at end of file +@enduml From a23d53b2d499ff87496ffd07b3fd2d267afb1550 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 6 May 2024 08:58:55 +0300 Subject: [PATCH 1070/1943] [#1118] .forgejo: Add pre-commit action Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/pre-commit.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .forgejo/workflows/pre-commit.yml diff --git a/.forgejo/workflows/pre-commit.yml b/.forgejo/workflows/pre-commit.yml new file mode 100644 index 000000000..6453b5029 --- /dev/null +++ b/.forgejo/workflows/pre-commit.yml @@ -0,0 +1,25 @@ +name: Pre-commit hooks +on: [pull_request] + +jobs: + precommit: + name: Pre-commit + env: + # Skip pre-commit hooks which are executed by other actions. + SKIP: make-lint,go-staticcheck-repo-mod,go-unit-tests,gofumpt + runs-on: ubuntu-22.04 + # If we use actions/setup-python from either Github or Gitea, + # the line above fails with a cryptic error about not being able to found python. + # So install everything manually. + steps: + - uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.22 + - name: Set up Python + run: | + apt update + apt install -y pre-commit + - name: Run pre-commit + run: pre-commit run --color=always --hook-stage manual --all-files From 00b2b77b261b195df583916660161d45df4a30ae Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 2 May 2024 14:29:59 +0300 Subject: [PATCH 1071/1943] [#1112] node: Implement `Range\RangeHash` requests for EC object Signed-off-by: Anton Nikiforov --- go.mod | 4 +- go.sum | 8 +-- pkg/local_object_storage/engine/range.go | 13 ++++ pkg/services/object/get/assemblerec.go | 62 ++++++++++++++++--- .../object/get/v2/get_range_forwarder.go | 3 + pkg/services/object/get/v2/service.go | 3 + pkg/services/object/get/v2/util.go | 11 ++++ 7 files changed, 88 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 1be2526f2..08a59581d 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240422151450-df9b65324a4c + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240427200446-67c6f305b21f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240424080726-20ab57bf7ec3 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240507063414-99e02858af12 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index 0795c955b..6200022c7 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,15 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240422151450-df9b65324a4c h1:RFDrNsF2e+EJfaB8lZrRRxNjQkLfM09gnEyudvGuc10= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240422151450-df9b65324a4c/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240427200446-67c6f305b21f h1:YyjsQNtrngQzIKOUtApXoi5r5pewatM+cXfpY19vZWo= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240427200446-67c6f305b21f/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0 h1:FzurjElUwC7InY9v5rzXReKbfBL5yRJKSWJPq6BKhH0= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240424080726-20ab57bf7ec3 h1:7Sd/J2IM0uGpmFKBgseUh6/JsdJN06b8W8UZMKAUDZg= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240424080726-20ab57bf7ec3/go.mod h1:wDFmMP7l00Xd5VZVzF2MuhyJCnotyhfxHYnvrEEG/e4= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240507063414-99e02858af12 h1:uEAn+TBXxgqXXygrYW2X8dFmGydGyJvBc5ysnfHFhYM= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240507063414-99e02858af12/go.mod h1:e7H9nNFpx1Tj3R20Zoxy0Vo6Srlb6zV5L7ZQXqg9rn4= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a h1:wbndKvHbwDQiSMQWL75RxiTZCeUyCi7NUj1lsfdAGkc= diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index e45845d6c..f5b33a251 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -102,6 +102,9 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error if it.SplitInfo != nil { return RngRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(it.SplitInfo)) } + if it.ECInfo != nil { + return RngRes{}, logicerr.Wrap(objectSDK.NewECInfoError(it.ECInfo)) + } if it.Object == nil { // If any shard is in a degraded mode, we should assume that metabase could store @@ -147,6 +150,8 @@ type getRangeShardIterator struct { Object *objectSDK.Object SplitInfoError *objectSDK.SplitInfoError SplitInfo *objectSDK.SplitInfo + ECInfoError *objectSDK.ECInfoError + ECInfo *objectSDK.ECInfo OutError error ShardWithMeta hashedShard MetaError error @@ -188,6 +193,14 @@ func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) { // stop iterating over shards if SplitInfo structure is complete return withLink && withLast + case errors.As(err, &i.ECInfoError): + if i.ECInfo == nil { + i.ECInfo = objectSDK.NewECInfo() + } + + util.MergeECInfo(i.ECInfoError.ECInfo(), i.ECInfo) + // stop iterating over shards if ECInfo structure is complete + return len(i.ECInfo.Chunks) == int(i.ECInfo.Chunks[0].Total) case client.IsErrObjectAlreadyRemoved(err), shard.IsErrOutOfRange(err): diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index d73d771cf..4a624e467 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -45,23 +46,64 @@ func newAssemblerEC( // Assemble assembles erasure-coded object and writes it's content to ObjectWriter. // It returns parent object. func (a *assemblerec) Assemble(ctx context.Context, writer ObjectWriter, headOnly bool) (*objectSDK.Object, error) { - parts := a.retrieveParts(ctx, headOnly) + if headOnly { + return a.reconstructHeader(ctx, writer) + } else if a.rng != nil { + return a.reconstructRange(ctx, writer) + } + return a.reconstructObject(ctx, writer) +} + +func (a *assemblerec) getConstructor() (*erasurecode.Constructor, error) { cnt, err := a.cs.Get(a.addr.Container()) if err != nil { return nil, err } - c, err := erasurecode.NewConstructor( - policy.ECDataCount(cnt.Value.PlacementPolicy()), - policy.ECParityCount(cnt.Value.PlacementPolicy()), - ) + dataCount := policy.ECDataCount(cnt.Value.PlacementPolicy()) + parityCount := policy.ECParityCount(cnt.Value.PlacementPolicy()) + return erasurecode.NewConstructor(dataCount, parityCount) +} + +func (a *assemblerec) reconstructHeader(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { + parts := a.retrieveParts(ctx, true) + c, err := a.getConstructor() if err != nil { return nil, err } - if headOnly { - obj, err := c.ReconstructHeader(parts) - if err == nil { - return obj, writer.WriteHeader(ctx, obj) - } + obj, err := c.ReconstructHeader(parts) + if err == nil { + return obj, writer.WriteHeader(ctx, obj) + } + return nil, err +} + +func (a *assemblerec) reconstructRange(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { + parts := a.retrieveParts(ctx, false) + c, err := a.getConstructor() + if err != nil { + return nil, err + } + obj, err := c.Reconstruct(parts) + if err != nil { + return nil, err + } + + from := a.rng.GetOffset() + to := from + a.rng.GetLength() + if pLen := uint64(len(obj.Payload())); to < from || pLen < from || pLen < to { + return nil, &apistatus.ObjectOutOfRange{} + } + err = writer.WriteChunk(ctx, obj.Payload()[from:to]) + if err != nil { + return nil, err + } + return obj, err +} + +func (a *assemblerec) reconstructObject(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { + parts := a.retrieveParts(ctx, false) + c, err := a.getConstructor() + if err != nil { return nil, err } obj, err := c.Reconstruct(parts) diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 8a56c59a6..5b05ec370 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -132,6 +132,9 @@ func (f *getRangeRequestForwarder) readStream(ctx context.Context, rangeStream * case *objectV2.SplitInfo: si := objectSDK.NewSplitInfoFromV2(v) return objectSDK.NewSplitInfoError(si) + case *objectV2.ECInfo: + ei := objectSDK.NewECInfoFromV2(v) + return objectSDK.NewECInfoError(ei) } } return nil diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index 682128df6..edd19b441 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -104,10 +104,13 @@ func (s *Service) GetRange(req *objectV2.GetRangeRequest, stream objectSvc.GetOb err = s.svc.GetRange(stream.Context(), *p) var splitErr *objectSDK.SplitInfoError + var ecErr *objectSDK.ECInfoError switch { case errors.As(err, &splitErr): return stream.Send(splitInfoRangeResponse(splitErr.SplitInfo())) + case errors.As(err, &ecErr): + return stream.Send(ecInfoRangeResponse(ecErr.ECInfo())) default: return err } diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index da6428985..610076c7a 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -292,6 +292,17 @@ func splitInfoRangeResponse(info *objectSDK.SplitInfo) *objectV2.GetRangeRespons return resp } +func ecInfoRangeResponse(info *objectSDK.ECInfo) *objectV2.GetRangeResponse { + resp := new(objectV2.GetRangeResponse) + + body := new(objectV2.GetRangeResponseBody) + resp.SetBody(body) + + body.SetRangePart(info.ToV2()) + + return resp +} + func setSplitInfoHeadResponse(info *objectSDK.SplitInfo, resp *objectV2.HeadResponse) { resp.GetBody().SetHeaderPart(info.ToV2()) } From 93c0ccad4ffbda4b5b4a9a99af8ac88c97a9df28 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 5 Apr 2024 15:54:36 +0300 Subject: [PATCH 1072/1943] [#1077] objectsvc: Fix possible panic in GetRange() Signed-off-by: Evgenii Stratonikov --- pkg/services/object/get/assembler.go | 2 +- pkg/services/object/get/get_test.go | 220 +++++++++++++++++++++++++-- 2 files changed, 206 insertions(+), 16 deletions(-) diff --git a/pkg/services/object/get/assembler.go b/pkg/services/object/get/assembler.go index f10c67e52..025296ec7 100644 --- a/pkg/services/object/get/assembler.go +++ b/pkg/services/object/get/assembler.go @@ -114,7 +114,7 @@ func (a *assembler) initializeFromSourceObjectID(ctx context.Context, id oid.ID) } to := uint64(0) - if seekOff+seekLen > a.currentOffset+from { + if seekOff+seekLen >= a.currentOffset+from { to = seekOff + seekLen - a.currentOffset } diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 64614d8d8..be1e96c29 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -1,6 +1,7 @@ package getsvc import ( + "bytes" "context" "crypto/ecdsa" "crypto/rand" @@ -25,6 +26,9 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" ) @@ -61,6 +65,9 @@ type testEpochReceiver uint64 func (e testEpochReceiver) Epoch() (uint64, error) { return uint64(e), nil } +func (e testEpochReceiver) CurrentEpoch() uint64 { + return uint64(e) +} func newTestStorage() *testStorage { return &testStorage{ @@ -555,21 +562,6 @@ func TestGetRemoteSmall(t *testing.T) { return p } - newRngPrm := func(raw bool, w ChunkWriter, off, ln uint64) RangePrm { - p := RangePrm{} - p.SetChunkWriter(w) - p.WithRawFlag(raw) - p.common = new(util.CommonPrm).WithLocalOnly(false) - - r := objectSDK.NewRange() - r.SetOffset(off) - r.SetLength(ln) - - p.SetRange(r) - - return p - } - newHeadPrm := func(raw bool, w ObjectWriter) HeadPrm { p := HeadPrm{} p.SetHeaderWriter(w) @@ -1628,6 +1620,204 @@ func TestGetRemoteSmall(t *testing.T) { }) } +type testTarget struct { + objects []*objectSDK.Object +} + +func (tt *testTarget) WriteObject(_ context.Context, obj *objectSDK.Object) error { + tt.objects = append(tt.objects, obj) + return nil +} + +func objectChain(t *testing.T, cnr cid.ID, singleSize, totalSize uint64) (oid.ID, []*objectSDK.Object, *objectSDK.Object, []byte) { + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + tt := new(testTarget) + p := transformer.NewPayloadSizeLimiter(transformer.Params{ + Key: &pk.PrivateKey, + NextTargetInit: func() transformer.ObjectWriter { return tt }, + NetworkState: testEpochReceiver(1), + MaxSize: singleSize, + }) + + payload := make([]byte, totalSize) + _, err = rand.Read(payload) + require.NoError(t, err) + + ver := version.Current() + hdr := objectSDK.New() + hdr.SetContainerID(cnr) + hdr.SetType(objectSDK.TypeRegular) + hdr.SetVersion(&ver) + + ctx := context.Background() + require.NoError(t, p.WriteHeader(ctx, hdr)) + + _, err = p.Write(ctx, payload) + require.NoError(t, err) + + res, err := p.Close(ctx) + require.NoError(t, err) + + if totalSize <= singleSize { + // Small object, no linking. + require.Len(t, tt.objects, 1) + return res.SelfID, tt.objects, nil, payload + } + + return *res.ParentID, tt.objects[:len(tt.objects)-1], tt.objects[len(tt.objects)-1], bytes.Clone(payload) +} + +func newRngPrm(raw bool, w ChunkWriter, off, ln uint64) RangePrm { + p := RangePrm{} + p.SetChunkWriter(w) + p.WithRawFlag(raw) + p.common = new(util.CommonPrm) + + r := objectSDK.NewRange() + r.SetOffset(off) + r.SetLength(ln) + + p.SetRange(r) + return p +} + +func TestGetRange(t *testing.T) { + var cnr container.Container + cnr.SetPlacementPolicy(netmaptest.PlacementPolicy()) + + var idCnr cid.ID + container.CalculateID(&idCnr, cnr) + + ns, as := testNodeMatrix(t, []int{2}) + + testGetRange := func(t *testing.T, svc *Service, addr oid.Address, from, to uint64, payload []byte) { + w := NewSimpleObjectWriter() + rngPrm := newRngPrm(false, w, from, to-from) + rngPrm.WithAddress(addr) + + err := svc.GetRange(context.Background(), rngPrm) + require.NoError(t, err) + if from == to { + require.Nil(t, w.Object().Payload()) + } else { + require.Equal(t, payload[from:to], w.Object().Payload()) + } + } + + newSvc := func(b *testPlacementBuilder, c *testClientCache) *Service { + const curEpoch = 13 + + return &Service{ + log: test.NewLogger(t), + localStorage: newTestStorage(), + traverserGenerator: &testTraverserGenerator{ + c: cnr, + b: map[uint64]placement.Builder{ + curEpoch: b, + }, + }, + epochSource: testEpochReceiver(curEpoch), + remoteStorageConstructor: c, + keyStore: &testKeyStorage{}, + } + } + + t.Run("small", func(t *testing.T) { + const totalSize = 5 + _, objs, _, payload := objectChain(t, idCnr, totalSize, totalSize) + require.Len(t, objs, 1) + require.Len(t, payload, totalSize) + + obj := objs[0] + addr := object.AddressOf(obj) + builder := &testPlacementBuilder{vectors: map[string][][]netmap.NodeInfo{addr.EncodeToString(): ns}} + + c1 := newTestClient() + c1.addResult(addr, obj, nil) + + svc := newSvc(builder, &testClientCache{ + clients: map[string]*testClient{ + as[0][0]: c1, + as[0][1]: c1, + }, + }) + + for from := 0; from < totalSize-1; from++ { + for to := from; to < totalSize; to++ { + t.Run(fmt.Sprintf("from=%d,to=%d", from, to), func(t *testing.T) { + testGetRange(t, svc, addr, uint64(from), uint64(to), payload) + }) + } + } + + }) + t.Run("big", func(t *testing.T) { + const totalSize = 9 + id, objs, link, payload := objectChain(t, idCnr, 3, totalSize) // 3 parts + require.Equal(t, totalSize, len(payload)) + + builder := &testPlacementBuilder{vectors: map[string][][]netmap.NodeInfo{}} + builder.vectors[idCnr.EncodeToString()+"/"+id.EncodeToString()] = ns + builder.vectors[object.AddressOf(link).EncodeToString()] = ns + for i := range objs { + builder.vectors[object.AddressOf(objs[i]).EncodeToString()] = ns + } + + var addr oid.Address + addr.SetContainer(idCnr) + addr.SetObject(id) + + const ( + linkingLast = "splitinfo=last" + linkingChildren = "splitinfo=children" + linkingBoth = "splitinfo=both" + ) + + lastID, _ := objs[len(objs)-1].ID() + linkID, _ := link.ID() + + for _, kind := range []string{linkingLast, linkingChildren, linkingBoth} { + t.Run(kind, func(t *testing.T) { + c1 := newTestClient() + for i := range objs { + c1.addResult(object.AddressOf(objs[i]), objs[i], nil) + } + + c1.addResult(object.AddressOf(link), link, nil) + + si := objectSDK.NewSplitInfo() + switch kind { + case linkingLast: + si.SetLastPart(lastID) + case linkingChildren: + si.SetLink(linkID) + case linkingBoth: + si.SetLastPart(lastID) + si.SetLink(linkID) + } + c1.addResult(addr, nil, objectSDK.NewSplitInfoError(si)) + + svc := newSvc(builder, &testClientCache{ + clients: map[string]*testClient{ + as[0][0]: c1, + as[0][1]: c1, + }, + }) + + for from := 0; from < totalSize-1; from++ { + for to := from; to < totalSize; to++ { + t.Run(fmt.Sprintf("from=%d,to=%d", from, to), func(t *testing.T) { + testGetRange(t, svc, addr, uint64(from), uint64(to), payload) + }) + } + } + }) + } + }) +} + func TestGetFromPastEpoch(t *testing.T) { ctx := context.Background() From 21a490da8f0399ee180b655c367a9b2a8565a0f8 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 7 May 2024 12:44:50 +0300 Subject: [PATCH 1073/1943] [#1112] Fix issue from `gofumpt` Signed-off-by: Anton Nikiforov --- pkg/services/object/get/get_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index be1e96c29..988cd6982 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -65,6 +65,7 @@ type testEpochReceiver uint64 func (e testEpochReceiver) Epoch() (uint64, error) { return uint64(e), nil } + func (e testEpochReceiver) CurrentEpoch() uint64 { return uint64(e) } @@ -1751,7 +1752,6 @@ func TestGetRange(t *testing.T) { }) } } - }) t.Run("big", func(t *testing.T) { const totalSize = 9 From 3e782527b87019795bbc9702dbbdd3b9c8c2f6cf Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 7 May 2024 12:45:13 +0300 Subject: [PATCH 1074/1943] [#1112] node: Add test for `Range` request for EC object Signed-off-by: Anton Nikiforov --- pkg/services/object/get/getrangeec_test.go | 182 +++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 pkg/services/object/get/getrangeec_test.go diff --git a/pkg/services/object/get/getrangeec_test.go b/pkg/services/object/get/getrangeec_test.go new file mode 100644 index 000000000..b8497d7d1 --- /dev/null +++ b/pkg/services/object/get/getrangeec_test.go @@ -0,0 +1,182 @@ +package getsvc + +import ( + "context" + "crypto/rand" + "fmt" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + coreContainer "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +type containerStorage struct { + cnt *container.Container +} + +func (cs *containerStorage) Get(cid.ID) (*coreContainer.Container, error) { + coreCnt := coreContainer.Container{ + Value: *cs.cnt, + } + return &coreCnt, nil +} + +func (cs *containerStorage) DeletionInfo(cid.ID) (*coreContainer.DelInfo, error) { + return nil, nil +} + +func TestGetRangeEC(t *testing.T) { + var dataCount uint32 = 3 + var parityCount uint32 = 1 + cnr := container.Container{} + p := netmap.PlacementPolicy{} + p.SetContainerBackupFactor(1) + x := netmap.ReplicaDescriptor{} + x.SetECDataCount(dataCount) + x.SetECParityCount(parityCount) + p.AddReplicas(x) + cnr.SetPlacementPolicy(p) + + var idCnr cid.ID + container.CalculateID(&idCnr, cnr) + + ns, as := testNodeMatrix(t, []int{4}) + + testGetRange := func(t *testing.T, svc *Service, addr oid.Address, from, to uint64, payload []byte) { + w := NewSimpleObjectWriter() + rngPrm := newRngPrm(false, w, from, to-from) + rngPrm.WithAddress(addr) + + err := svc.GetRange(context.Background(), rngPrm) + require.NoError(t, err) + if from == to { + require.Nil(t, w.Object().Payload()) + } else { + require.Equal(t, payload[from:to], w.Object().Payload()) + } + } + + newSvc := func(b *testPlacementBuilder, c *testClientCache) *Service { + const curEpoch = 13 + + return &Service{ + log: test.NewLogger(t), + localStorage: newTestStorage(), + traverserGenerator: &testTraverserGenerator{ + c: cnr, + b: map[uint64]placement.Builder{ + curEpoch: b, + }, + }, + epochSource: testEpochReceiver(curEpoch), + remoteStorageConstructor: c, + keyStore: &testKeyStorage{}, + containerSource: &containerStorage{ + cnt: &cnr, + }, + } + } + const totalSize = 5 + obj, parts := objectECChain(t, &idCnr, &cnr, totalSize, totalSize) + require.Len(t, parts, int(dataCount+parityCount)) + require.Len(t, obj.Payload(), totalSize) + + addr := object.AddressOf(obj) + builder := &testPlacementBuilder{ + vectors: map[string][][]netmap.NodeInfo{ + addr.EncodeToString(): ns, + }, + } + + clients := map[string]*testClient{} + for i, part := range parts { + builder.vectors[object.AddressOf(part).EncodeToString()] = ns + + tc := newTestClient() + + ecInfo := objectSDK.NewECInfo() + + chunk := objectSDK.ECChunk{} + chunk.Total = uint32(len(parts)) + chunk.Index = uint32(i) + id, _ := part.ID() + idv2 := refs.ObjectID{} + id.WriteToV2(&idv2) + chunk.ID = idv2 + + ecInfo.AddChunk(chunk) + errECInfo := objectSDK.NewECInfoError(ecInfo) + + tc.addResult(addr, nil, errECInfo) + tc.addResult(object.AddressOf(part), part, nil) + + clients[as[0][i]] = tc + } + + svc := newSvc(builder, &testClientCache{ + clients: clients, + }) + + for from := 0; from < totalSize-1; from++ { + for to := from; to < totalSize; to++ { + t.Run(fmt.Sprintf("from=%d,to=%d", from, to), func(t *testing.T) { + testGetRange(t, svc, addr, uint64(from), uint64(to), obj.Payload()) + }) + } + } +} + +func objectECChain(t *testing.T, cnrId *cid.ID, cnr *container.Container, singleSize, totalSize uint64) (*objectSDK.Object, []*objectSDK.Object) { + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + tt := new(testTarget) + p := transformer.NewPayloadSizeLimiter(transformer.Params{ + Key: &pk.PrivateKey, + NextTargetInit: func() transformer.ObjectWriter { return tt }, + NetworkState: testEpochReceiver(1), + MaxSize: singleSize, + }) + + payload := make([]byte, totalSize) + _, err = rand.Read(payload) + require.NoError(t, err) + + ver := version.Current() + hdr := objectSDK.New() + hdr.SetContainerID(*cnrId) + hdr.SetType(objectSDK.TypeRegular) + hdr.SetVersion(&ver) + + ctx := context.Background() + require.NoError(t, p.WriteHeader(ctx, hdr)) + + _, err = p.Write(ctx, payload) + require.NoError(t, err) + + _, err = p.Close(ctx) + require.NoError(t, err) + + require.Len(t, tt.objects, 1) + + c, err := erasurecode.NewConstructor(policy.ECDataCount(cnr.PlacementPolicy()), policy.ECParityCount(cnr.PlacementPolicy())) + require.NoError(t, err) + parts, err := c.Split(tt.objects[0], &pk.PrivateKey) + require.NoError(t, err) + + return tt.objects[0], parts +} From fe2c1c926ff890d64de326b1263ea118d187f797 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 7 May 2024 14:23:23 +0300 Subject: [PATCH 1075/1943] [#1112] node: Fix race warning for `GetObjectAndWritePayload` Signed-off-by: Anton Nikiforov --- pkg/services/object/get/assemble.go | 2 +- pkg/services/object/get/assembleec.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index 548e8e45d..3f4a02c02 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -37,6 +37,7 @@ func (r *request) assemble(ctx context.Context) { r.log.Debug(logs.GetTryingToAssembleTheObject) + r.prm.common = r.prm.common.WithLocalOnly(false) assembler := newAssembler(r.address(), r.splitInfo(), r.ctxRange(), r) r.log.Debug(logs.GetAssemblingSplittedObject, @@ -119,7 +120,6 @@ func (r *request) GetObjectAndWritePayload(ctx context.Context, id oid.ID, rng * } p := r.prm - p.common = p.common.WithLocalOnly(false) p.objWriter = w p.rng = rng diff --git a/pkg/services/object/get/assembleec.go b/pkg/services/object/get/assembleec.go index d288ef249..58641c975 100644 --- a/pkg/services/object/get/assembleec.go +++ b/pkg/services/object/get/assembleec.go @@ -35,6 +35,7 @@ func (r *request) assembleEC(ctx context.Context) { r.log.Debug(logs.GetTryingToAssembleTheECObject) + r.prm.common = r.prm.common.WithLocalOnly(false) assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.containerSource, r.log) r.log.Debug(logs.GetAssemblingECObject, From 854200a874919f0bfb6dcb20a790091a621f028c Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 7 May 2024 19:11:21 +0300 Subject: [PATCH 1076/1943] [#1115] node: Remove unused const Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/modules/tree/root.go | 3 +- cmd/frostfs-node/config/morph/config.go | 3 +- internal/logs/logs.go | 1088 +++++++++++------------ 3 files changed, 514 insertions(+), 580 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/root.go b/cmd/frostfs-cli/modules/tree/root.go index 701a78f2a..efd1c08b5 100644 --- a/cmd/frostfs-cli/modules/tree/root.go +++ b/cmd/frostfs-cli/modules/tree/root.go @@ -40,8 +40,7 @@ const ( metaFlagKey = "meta" - pathFlagKey = "path" - pathAttributeFlagKey = "pattr" + pathFlagKey = "path" latestOnlyFlagKey = "latest" diff --git a/cmd/frostfs-node/config/morph/config.go b/cmd/frostfs-node/config/morph/config.go index a240c5d49..b0c6527db 100644 --- a/cmd/frostfs-node/config/morph/config.go +++ b/cmd/frostfs-node/config/morph/config.go @@ -10,8 +10,7 @@ import ( ) const ( - subsection = "morph" - notarySubsection = "notary" + subsection = "morph" // DialTimeoutDefault is a default dial timeout of morph chain client connection. DialTimeoutDefault = 5 * time.Second diff --git a/internal/logs/logs.go b/internal/logs/logs.go index e76ab10b6..4576e2e59 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -17,580 +17,516 @@ const ( ) const ( - InnerringNonalphabetModeDoNotStopContainerEstimations = "non-alphabet mode, do not stop container estimations" - InnerringCantStopEpochEstimation = "can't stop epoch estimation" - InnerringCantMakeNotaryDepositInMainChain = "can't make notary deposit in main chain" - InnerringCantMakeNotaryDepositInSideChain = "can't make notary deposit in side chain" - InnerringNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" - InnerringCantGetInnerRingIndex = "can't get inner ring index" - InnerringCantGetInnerRingSize = "can't get inner ring size" - InnerringCantGetAlphabetIndex = "can't get alphabet index" - InnerringIgnoreValidatorVoteNodeNotInAlphabetRange = "ignore validator vote: node not in alphabet range" - InnerringIgnoreValidatorVoteEmptyValidatorsList = "ignore validator vote: empty validators list" - InnerringCantInvokeVoteMethodInAlphabetContract = "can't invoke vote method in alphabet contract" - InnerringCantGetLastProcessedMainChainBlockNumber = "can't get last processed main chain block number" - InnerringNotarySupport = "notary support" - InnerringAlphabetKeysSyncIsDisabled = "alphabet keys sync is disabled" - InnerringNoControlServerEndpointSpecified = "no Control server endpoint specified, service is disabled" - InnerringCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" - InnerringFailedToSetGroupSignerScope = "failed to set group signer scope, continue with Global" - InnerringCantVoteForPreparedValidators = "can't vote for prepared validators" - InnerringNewBlock = "new block" - InnerringCantUpdatePersistentState = "can't update persistent state" - InnerringCloserError = "closer error" - InnerringReadConfigFromBlockchain = "read config from blockchain" - NotificatorNotificatorStartProcessingObjectNotifications = "notificator: start processing object notifications" - NotificatorNotificatorProcessingObjectNotification = "notificator: processing object notification" - PolicerCouldNotGetContainer = "could not get container" - PolicerCouldNotConfirmContainerRemoval = "could not confirm container removal" - PolicerCouldNotInhumeObjectWithMissingContainer = "could not inhume object with missing container" - PolicerCouldNotBuildPlacementVectorForObject = "could not build placement vector for object" - PolicerRedundantLocalObjectCopyDetected = "redundant local object copy detected" - PolicerReceiveObjectHeaderToCheckPolicyCompliance = "receive object header to check policy compliance" - PolicerConsiderNodeUnderMaintenanceAsOK = "consider node under maintenance as OK" - PolicerShortageOfObjectCopiesDetected = "shortage of object copies detected" - PolicerSomeOfTheCopiesAreStoredOnNodesUnderMaintenance = "some of the copies are stored on nodes under maintenance, save local copy" - PolicerRoutineStopped = "routine stopped" - PolicerFailureAtObjectSelectForReplication = "failure at object select for replication" - PolicerPoolSubmission = "pool submission" - PolicerUnableToProcessObj = "unable to process object" - ReplicatorFinishWork = "finish work" - ReplicatorCouldNotGetObjectFromLocalStorage = "could not get object from local storage" - ReplicatorCouldNotReplicateObject = "could not replicate object" - ReplicatorObjectSuccessfullyReplicated = "object successfully replicated" - TreeRedirectingTreeServiceQuery = "redirecting tree service query" - TreeBearerPresentedButNotAllowedByACL = "bearer presented but not allowed by ACL" - TreeCouldNotGetLastSynchronizedHeightForATree = "could not get last synchronized height for a tree" - TreeCouldNotUpdateLastSynchronizedHeightForATree = "could not update last synchronized height for a tree" - TreeSynchronizeTree = "synchronize tree" - TreeFailedToRunTreeSynchronizationOverAllNodes = "failed to run tree synchronization over all nodes" - TreeFailedToRunTreeSynchronizationForSpecificNode = "failed to run tree synchronization for specific node" - TreeFailedToParseAddressForTreeSynchronization = "failed to parse address for tree synchronization" - TreeFailedToConnectForTreeSynchronization = "failed to connect for tree synchronization" - TreeSyncingTrees = "syncing trees..." - TreeCouldNotFetchContainers = "could not fetch containers" - TreeTreesHaveBeenSynchronized = "trees have been synchronized" - TreeSyncingContainerTrees = "syncing container trees..." - TreeCouldNotSyncTrees = "could not sync trees" - TreeContainerTreesHaveBeenSynced = "container trees have been synced" - TreeCouldNotQueryTreesForSynchronization = "could not query trees for synchronization" - TreeRemovingRedundantTrees = "removing redundant trees..." - TreeCouldNotCheckIfContainerExisted = "could not check if the container ever existed" - TreeCouldNotRemoveRedundantTree = "could not remove redundant tree" - TreeCouldNotCalculateContainerNodes = "could not calculate container nodes" - TreeFailedToApplyReplicatedOperation = "failed to apply replicated operation" - TreeDoNotSendUpdateToTheNode = "do not send update to the node" - TreeFailedToSentUpdateToTheNode = "failed to sent update to the node" - TreeErrorDuringReplication = "error during replication" - PersistentCouldNotGetSessionFromPersistentStorage = "could not get session from persistent storage" - PersistentCouldNotDeleteSToken = "could not delete token" - PersistentCouldNotCleanUpExpiredTokens = "could not clean up expired tokens" - ControllerReportIsAlreadyStarted = "report is already started" - TombstoneCouldNotGetTheTombstoneTheSource = "tombstone getter: could not get the tombstone the source" - DeleteNoSplitInfoObjectIsPHY = "no split info, object is PHY" - DeleteAssemblingChain = "assembling chain..." - DeleteCollectingChildren = "collecting children..." - DeleteSupplementBySplitID = "supplement by split ID" - DeleteFormingTombstoneStructure = "forming tombstone structure..." - DeleteTombstoneStructureSuccessfullyFormedSaving = "tombstone structure successfully formed, saving..." - DeleteFormingSplitInfo = "forming split info..." - DeleteSplitInfoSuccessfullyFormedCollectingMembers = "split info successfully formed, collecting members..." - DeleteMembersSuccessfullyCollected = "members successfully collected" - GetRemoteCallFailed = "remote call failed" - GetCanNotAssembleTheObject = "can not assemble the object" - GetTryingToAssembleTheObject = "trying to assemble the object..." - GetTryingToAssembleTheECObject = "trying to assemble the ec object..." - GetAssemblingSplittedObject = "assembling splitted object..." - GetAssemblingECObject = "assembling erasure-coded object..." - GetUnableToGetAllPartsECObject = "unable to get all parts, continue to reconstruct with existed" - GetUnableToGetPartECObject = "unable to get part of the erasure-encoded object" - GetUnableToHeadPartECObject = "unable to head part of the erasure-encoded object" - GetUnableToGetECObjectContainer = "unable to get container for erasure-coded object" - GetAssemblingSplittedObjectCompleted = "assembling splitted object completed" - GetAssemblingECObjectCompleted = "assembling erasure-coded object completed" - GetFailedToAssembleSplittedObject = "failed to assemble splitted object" - GetFailedToAssembleECObject = "failed to assemble erasure-coded object" - GetCouldNotGenerateContainerTraverser = "could not generate container traverser" - GetCouldNotConstructRemoteNodeClient = "could not construct remote node client" - GetCouldNotWriteHeader = "could not write header" - GetCouldNotWritePayloadChunk = "could not write payload chunk" - GetLocalGetFailed = "local get failed" - GetReturnResultDirectly = "return result directly" - GetCompletingTheOperation = "completing the operation" - GetRequestedObjectWasMarkedAsRemoved = "requested object was marked as removed" - GetRequestedObjectIsVirtual = "requested object is virtual" - GetRequestedObjectIsEC = "requested object is erasure-coded" - GetRequestedRangeIsOutOfObjectBounds = "requested range is out of object bounds" - PutAdditionalContainerBroadcastFailure = "additional container broadcast failure" - SearchReturnResultDirectly = "return result directly" - SearchCouldNotConstructRemoteNodeClient = "could not construct remote node client" - SearchRemoteOperationFailed = "remote operation failed" - SearchCouldNotGenerateContainerTraverser = "could not generate container traverser" - SearchCouldNotWriteObjectIdentifiers = "could not write object identifiers" - SearchLocalOperationFailed = "local operation failed" - UtilObjectServiceError = "object service error" - UtilCouldNotPushTaskToWorkerPool = "could not push task to worker pool" - V2CantCheckIfRequestFromInnerRing = "can't check if request from inner ring" - V2CantCheckIfRequestFromContainerNode = "can't check if request from container node" - NatsNatsConnectionWasLost = "nats: connection was lost" - NatsNatsReconnectedToTheServer = "nats: reconnected to the server" - NatsNatsClosingConnectionAsTheContextIsDone = "nats: closing connection as the context is done" - NatsConnectedToEndpoint = "nats: successfully connected to endpoint" - ControllerStartingToAnnounceTheValuesOfTheMetrics = "starting to announce the values of the metrics" - ControllerCouldNotInitializeIteratorOverLocallyCollectedMetrics = "could not initialize iterator over locally collected metrics" - ControllerCouldNotInitializeAnnouncementAccumulator = "could not initialize announcement accumulator" - ControllerIteratorOverLocallyCollectedMetricsAborted = "iterator over locally collected metrics aborted" - ControllerCouldNotFinishWritingLocalAnnouncements = "could not finish writing local announcements" - ControllerTrustAnnouncementSuccessfullyFinished = "trust announcement successfully finished" - ControllerAnnouncementIsAlreadyStarted = "announcement is already started" - ControllerAnnouncementSuccessfullyInterrupted = "announcement successfully interrupted" - ControllerAnnouncementIsNotStartedOrAlreadyInterrupted = "announcement is not started or already interrupted" - ControllerCouldNotInitializeIteratorOverLocallyAccumulatedAnnouncements = "could not initialize iterator over locally accumulated announcements" - ControllerCouldNotInitializeResultTarget = "could not initialize result target" - ControllerIteratorOverLocalAnnouncementsAborted = "iterator over local announcements aborted" - ControllerCouldNotFinishWritingLoadEstimations = "could not finish writing load estimations" - RouteCouldNotInitializeWriterProvider = "could not initialize writer provider" - RouteCouldNotInitializeWriter = "could not initialize writer" - RouteCouldNotPutTheValue = "could not put the value" - RouteCouldNotCloseRemoteServerWriter = "could not close remote server writer" - ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch = "could not restore block subscription after RPC switch" - ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch = "could not restore notification subscription after RPC switch" - ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch = "could not restore notary notification subscription after RPC switch" - ClientCouldNotEstablishConnectionToTheSwitchedRPCNode = "could not establish connection to the switched RPC node" - ClientConnectionToTheNewRPCNodeHasBeenEstablished = "connection to the new RPC node has been established" - ClientSwitchingToTheNextRPCNode = "switching to the next RPC node" - ClientCouldNotEstablishConnectionToAnyRPCNode = "could not establish connection to any RPC node" - ClientCouldNotCreateClientToTheHigherPriorityNode = "could not create client to the higher priority node" - ClientSwitchedToTheHigherPriorityRPC = "switched to the higher priority RPC" - ClientCouldNotRestoreSideChainSubscriptionsUsingNode = "could not restore side chain subscriptions using node" - ClientNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" - ClientNotaryDepositInvoke = "notary deposit invoke" - ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked" - ClientNotaryRequestInvoked = "notary request invoked" - ClientNotaryDepositTransactionWasSuccessfullyPersisted = "notary deposit transaction was successfully persisted" - ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted = "attempt to wait for notary deposit transaction to get persisted" - ClientNeoClientInvoke = "neo client invoke" - ClientNativeGasTransferInvoke = "native gas transfer invoke" - ClientBatchGasTransferInvoke = "batch gas transfer invoke" - ClientCantGetBlockchainHeight = "can't get blockchain height" - ClientCantGetBlockchainHeight243 = "can't get blockchain height" - EventCouldNotSubmitHandlerToWorkerPool = "could not Submit handler to worker pool" - EventCouldNotStartListenToEvents = "could not start listen to events" - EventStopEventListenerByError = "stop event listener by error" - EventStopEventListenerByContext = "stop event listener by context" - EventStopEventListenerByNotificationChannel = "stop event listener by notification channel" - EventNilNotificationEventWasCaught = "nil notification event was caught" - EventStopEventListenerByNotaryChannel = "stop event listener by notary channel" - EventNilNotaryEventWasCaught = "nil notary event was caught" - EventStopEventListenerByBlockChannel = "stop event listener by block channel" - EventNilBlockWasCaught = "nil block was caught" - EventListenerWorkerPoolDrained = "listener worker pool drained" - EventEventParserNotSet = "event parser not set" - EventCouldNotParseNotificationEvent = "could not parse notification event" - EventNotificationHandlersForParsedNotificationEventWereNotRegistered = "notification handlers for parsed notification event were not registered" - EventSkipExpiredMainTXNotaryEvent = "skip expired main TX notary event" - EventCouldNotPrepareAndValidateNotaryEvent = "could not prepare and validate notary event" - EventNotaryParserNotSet = "notary parser not set" - EventCouldNotParseNotaryEvent = "could not parse notary event" - EventNotaryHandlersForParsedNotificationEventWereNotRegistered = "notary handlers for parsed notification event were not registered" - EventIgnoreNilEventParser = "ignore nil event parser" - EventListenerHasBeenAlreadyStartedIgnoreParser = "listener has been already started, ignore parser" - EventRegisteredNewEventParser = "registered new event parser" - EventIgnoreNilEventHandler = "ignore nil event handler" - EventIgnoreHandlerOfEventWoParser = "ignore handler of event w/o parser" - EventRegisteredNewEventHandler = "registered new event handler" - EventIgnoreNilNotaryEventParser = "ignore nil notary event parser" - EventListenerHasBeenAlreadyStartedIgnoreNotaryParser = "listener has been already started, ignore notary parser" - EventIgnoreNilNotaryEventHandler = "ignore nil notary event handler" - EventIgnoreHandlerOfNotaryEventWoParser = "ignore handler of notary event w/o parser" - EventIgnoreNilBlockHandler = "ignore nil block handler" - SubscriberRemoteNotificationChannelHasBeenClosed = "remote notification channel has been closed" - SubscriberCantCastNotifyEventValueToTheNotifyStruct = "can't cast notify event value to the notify struct" - SubscriberNewNotificationEventFromSidechain = "new notification event from sidechain" - SubscriberCantCastBlockEventValueToBlock = "can't cast block event value to block" - SubscriberCantCastNotifyEventValueToTheNotaryRequestStruct = "can't cast notify event value to the notary request struct" - SubscriberUnsupportedNotificationFromTheChain = "unsupported notification from the chain" - StorageOperation = "local object storage operation" - BlobovniczaCreatingDirectoryForBoltDB = "creating directory for BoltDB" - BlobovniczaOpeningBoltDB = "opening BoltDB" - BlobovniczaInitializing = "initializing..." - BlobovniczaAlreadyInitialized = "already initialized" - BlobovniczaCreatingBucketForSizeRange = "creating bucket for size range" - BlobovniczaClosingBoltDB = "closing BoltDB" - BlobovniczaObjectWasRemovedFromBucket = "object was removed from bucket" - BlobstorOpening = "opening..." - BlobstorInitializing = "initializing..." - BlobstorClosing = "closing..." - BlobstorCouldntCloseStorage = "couldn't close storage" - BlobstorErrorOccurredDuringObjectExistenceChecking = "error occurred during object existence checking" - BlobstorErrorOccurredDuringTheIteration = "error occurred during the iteration" - EngineShardHasBeenRemoved = "shard has been removed" - EngineCouldNotCloseRemovedShard = "could not close removed shard" - EngineCouldNotOpenShardClosingAndSkipping = "could not open shard, closing and skipping" - EngineCouldNotClosePartiallyInitializedShard = "could not close partially initialized shard" - EngineCouldNotInitializeShardClosingAndSkipping = "could not initialize shard, closing and skipping" - EngineCouldNotCloseShard = "could not close shard" - EngineCouldNotReloadAShard = "could not reload a shard" - EngineAddedNewShard = "added new shard" - EngineCouldNotPutObjectToShard = "could not put object to shard" - EngineErrorDuringSearchingForObjectChildren = "error during searching for object children" - EngineCouldNotInhumeObjectInShard = "could not inhume object in shard" - EngineStartingRemovalOfLocallyredundantCopies = "starting removal of locally-redundant copies" - EngineStartedDuplicatesRemovalRoutine = "started duplicates removal routine" - EngineFinishedRemovalOfLocallyredundantCopies = "finished removal of locally-redundant copies" - EngineRemovingAnObjectWithoutFullLockingCheck = "removing an object without full locking check" - EngineInterruptProcessingTheExpiredLocks = "interrupt processing the expired locks" - EngineInterruptProcessingTheDeletedLocks = "interrupt processing the deleted locks" - EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly = "failed to move shard in degraded-read-only mode, moving to read-only" - EngineFailedToMoveShardInReadonlyMode = "failed to move shard in read-only mode" - EngineShardIsMovedInReadonlyModeDueToErrorThreshold = "shard is moved in read-only mode due to error threshold" - EngineShardIsMovedInDegradedModeDueToErrorThreshold = "shard is moved in degraded mode due to error threshold" - EngineModeChangeIsInProgressIgnoringSetmodeRequest = "mode change is in progress, ignoring set-mode request" - EngineStartedShardsEvacuation = "started shards evacuation" - EngineFinishedSuccessfullyShardsEvacuation = "shards evacuation finished successfully" - EngineFinishedWithErrorShardsEvacuation = "shards evacuation finished with error" - EngineObjectIsMovedToAnotherShard = "object is moved to another shard" - MetabaseMissingMatcher = "missing matcher" - MetabaseErrorInFKBTSelection = "error in FKBT selection" - MetabaseCantDecodeListBucketLeaf = "can't decode list bucket leaf" - MetabaseUnknownOperation = "unknown operation" - MetabaseCantIterateOverTheBucket = "can't iterate over the bucket" - MetabaseCouldNotIterateOverTheBuckets = "could not iterate over the buckets" - MetabaseCreatedDirectoryForMetabase = "created directory for Metabase" - MetabaseOpenedBoltDBInstanceForMetabase = "opened boltDB instance for Metabase" - MetabaseCheckingMetabaseVersion = "checking metabase version" - ShardCantSelectAllObjects = "can't select all objects" - ShardSettingShardMode = "setting shard mode" - ShardShardModeSetSuccessfully = "shard mode set successfully" - ShardCouldNotMarkObjectForShardRelocationInMetabase = "could not mark object for shard relocation in metabase" - ShardCantDeleteObjectFromWriteCache = "can't delete object from write cache" - ShardCantGetStorageIDFromMetabase = "can't get storage ID from metabase" - ShardCantRemoveObjectFromBlobStor = "can't remove object from blobStor" - ShardFetchingObjectWithoutMeta = "fetching object without meta" - ShardObjectIsMissingInWritecache = "object is missing in write-cache" - ShardFailedToFetchObjectFromWritecache = "failed to fetch object from write-cache" - ShardCantPutObjectToTheWritecacheTryingBlobstor = "can't put object to the write-cache, trying blobstor" - ShardMetaObjectCounterRead = "meta: object counter read" - ShardMetaCantReadContainerList = "meta: can't read container list" - ShardMetaCantReadContainerSize = "meta: can't read container size" - ShardMetaInfoPresentButObjectNotFound = "meta info was present, but the object is missing" - ShardMetabaseFailureSwitchingMode = "metabase failure, switching mode" - ShardCantMoveShardToReadonlySwitchMode = "can't move shard to readonly, switch mode" - ShardCouldNotUnmarshalObject = "could not unmarshal object" - ShardCouldNotCloseShardComponent = "could not close shard component" - ShardCantOpenMetabaseMoveToADegradedMode = "can't open metabase, move to a degraded mode" - ShardCantInitializeMetabaseMoveToADegradedreadonlyMode = "can't initialize metabase, move to a degraded-read-only mode" - ShardTryingToRestoreReadwriteMode = "trying to restore read-write mode" - ShardStopEventListenerByClosedEventChannel = "stop event listener by closed `event` channel" - ShardStopEventListenerByClosedStopChannel = "stop event listener by closed `stop` channel" - ShardStopEventListenerByContext = "stop event listener by context" - ShardCouldNotSubmitGCJobToWorkerPool = "could not submit GC job to worker pool" - ShardGCIsStopped = "GC is stopped" - ShardWaitingForGCWorkersToStop = "waiting for GC workers to stop..." - ShardIteratorOverMetabaseGraveyardFailed = "iterator over metabase graveyard failed" - ShardCouldNotDeleteTheObjects = "could not delete the objects" - ShardIteratorOverExpiredObjectsFailed = "iterator over expired objects failed" - ShardCouldNotInhumeTheObjects = "could not inhume the objects" - ShardStartedExpiredTombstonesHandling = "started expired tombstones handling" - ShardIteratingTombstones = "iterating tombstones" - ShardShardIsInADegradedModeSkipCollectingExpiredTombstones = "shard is in a degraded mode, skip collecting expired tombstones" - ShardIteratorOverGraveyardFailed = "iterator over graveyard failed" - ShardHandlingExpiredTombstonesBatch = "handling expired tombstones batch" - ShardFinishedExpiredTombstonesHandling = "finished expired tombstones handling" - ShardIteratorOverExpiredLocksFailed = "iterator over expired locks failed" - ShardCouldNotMarkTombstonesAsGarbage = "could not mark tombstones as garbage" - ShardCouldNotDropExpiredGraveRecords = "could not drop expired grave records" - ShardFailureToUnlockObjects = "failure to unlock objects" - ShardFailureToMarkLockersAsGarbage = "failure to mark lockers as garbage" - ShardFailureToGetExpiredUnlockedObjects = "failure to get expired unlocked objects" - ShardCouldNotMarkObjectToDeleteInMetabase = "could not mark object to delete in metabase" - WritecacheTriedToFlushItemsFromWritecache = "tried to flush items from write-cache" - WritecacheWaitingForChannelsToFlush = "waiting for channels to flush" - WritecacheFillingFlushMarksForObjectsInFSTree = "filling flush marks for objects in FSTree" - WritecacheFinishedUpdatingFSTreeFlushMarks = "finished updating FSTree flush marks" - WritecacheFillingFlushMarksForObjectsInDatabase = "filling flush marks for objects in database" - WritecacheFinishedUpdatingFlushMarks = "finished updating flush marks" - WritecacheCantRemoveObjectsFromTheDatabase = "can't remove objects from the database" - WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" - BlobovniczatreeCouldNotGetObjectFromLevel = "could not get object from level" - BlobovniczatreeCouldNotReadPayloadRangeFromOpenedBlobovnicza = "could not read payload range from opened blobovnicza" - BlobovniczatreeCouldNotReadPayloadRangeFromActiveBlobovnicza = "could not read payload range from active blobovnicza" - BlobovniczatreeCouldNotCloseBlobovnicza = "could not close Blobovnicza" - BlobovniczatreeBlobovniczaSuccessfullyClosedOnEvict = "blobovnicza successfully closed on evict" - BlobovniczatreeUpdatingActiveBlobovnicza = "updating active blobovnicza..." - BlobovniczatreeActiveBlobovniczaSuccessfullyUpdated = "active blobovnicza successfully updated" - BlobovniczatreeBlobovniczaSuccessfullyActivated = "blobovnicza successfully activated" - BlobovniczatreeCouldNotRemoveObjectFromLevel = "could not remove object from level" - BlobovniczatreeCouldNotRemoveObjectFromOpenedBlobovnicza = "could not remove object from opened blobovnicza" - BlobovniczatreeCouldNotRemoveObjectFromActiveBlobovnicza = "could not remove object from active blobovnicza" - BlobovniczatreeCouldNotGetActiveBlobovnicza = "could not get active blobovnicza" - BlobovniczatreeBlobovniczaOverflowed = "blobovnicza overflowed" - BlobovniczatreeCouldNotUpdateActiveBlobovnicza = "could not update active blobovnicza" - BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza = "could not put object to active blobovnicza" - BlobovniczatreeCouldNotReadObjectFromOpenedBlobovnicza = "could not read object from opened blobovnicza" - BlobovniczatreeCouldNotGetObjectFromActiveBlobovnicza = "could not get object from active blobovnicza" - BlobovniczatreeInitializingBlobovniczas = "initializing Blobovnicza's" - BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization = "read-only mode, skip blobovniczas initialization..." - BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing = "blobovnicza successfully initialized, closing..." - BlobovniczatreeCouldNotCloseActiveBlobovnicza = "could not close active blobovnicza" - AlphabetTick = "tick" - AlphabetAlphabetProcessorWorkerPoolDrained = "alphabet processor worker pool drained" - AlphabetNonAlphabetModeIgnoreGasEmissionEvent = "non alphabet mode, ignore gas emission event" - AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent = "node is out of alphabet range, ignore gas emission event" - AlphabetCantInvokeAlphabetEmitMethod = "can't invoke alphabet emit method" - AlphabetStorageNodeEmissionIsOff = "storage node emission is off" - AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes = "can't get netmap snapshot to emit gas to storage nodes" - AlphabetGasEmission = "gas emission" - AlphabetCantParseNodePublicKey = "can't parse node public key" - AlphabetCantTransferGas = "can't transfer gas" - AlphabetCantTransferGasToWallet = "can't transfer gas to wallet" - AlphabetAlphabetWorkerPool = "alphabet worker pool" - BalanceBalanceWorkerPoolDrained = "balance worker pool drained" - BalanceNonAlphabetModeIgnoreBalanceLock = "non alphabet mode, ignore balance lock" - BalanceCantSendLockAssetTx = "can't send lock asset tx" - BalanceBalanceWorkerPool = "balance worker pool" - ContainerContainerWorkerPool = "container worker pool" - ContainerContainerProcessorWorkerPoolDrained = "container processor worker pool drained" - ContainerNonAlphabetModeIgnoreContainerPut = "non alphabet mode, ignore container put" - ContainerPutContainerCheckFailed = "put container check failed" - ContainerCouldNotApprovePutContainer = "could not approve put container" - ContainerNonAlphabetModeIgnoreContainerDelete = "non alphabet mode, ignore container delete" - ContainerDeleteContainerCheckFailed = "delete container check failed" - ContainerCouldNotApproveDeleteContainer = "could not approve delete container" - ContainerNonAlphabetModeIgnoreSetEACL = "non alphabet mode, ignore set EACL" - ContainerSetEACLCheckFailed = "set EACL check failed" - ContainerCouldNotApproveSetEACL = "could not approve set EACL" - FrostFSNonAlphabetModeIgnoreBind = "non alphabet mode, ignore bind" - FrostFSInvalidManageKeyEvent = "invalid manage key event" - FrostFSCouldNotDecodeScriptHashFromBytes = "could not decode script hash from bytes" - FrostFSNonAlphabetModeIgnoreConfig = "non alphabet mode, ignore config" - FrostFSCantRelaySetConfigEvent = "can't relay set config event" - FrostFSFrostfsWorkerPool = "frostfs worker pool" - FrostFSFrostfsProcessorWorkerPoolDrained = "frostfs processor worker pool drained" - FrostFSNonAlphabetModeIgnoreDeposit = "non alphabet mode, ignore deposit" - FrostFSCantTransferAssetsToBalanceContract = "can't transfer assets to balance contract" - FrostFSDoubleMintEmissionDeclined = "double mint emission declined" - FrostFSCantGetGasBalanceOfTheNode = "can't get gas balance of the node" - FrostFSGasBalanceThresholdHasBeenReached = "gas balance threshold has been reached" - FrostFSCantTransferNativeGasToReceiver = "can't transfer native gas to receiver" - FrostFSNonAlphabetModeIgnoreWithdraw = "non alphabet mode, ignore withdraw" - FrostFSCantCreateLockAccount = "can't create lock account" - FrostFSCantLockAssetsForWithdraw = "can't lock assets for withdraw" - FrostFSNonAlphabetModeIgnoreCheque = "non alphabet mode, ignore cheque" - FrostFSCantTransferAssetsToFedContract = "can't transfer assets to fed contract" - GovernanceNewEvent = "new event" - GovernanceGovernanceWorkerPoolDrained = "governance worker pool drained" - GovernanceNonAlphabetModeIgnoreAlphabetSync = "non alphabet mode, ignore alphabet sync" - GovernanceCantFetchAlphabetListFromMainNet = "can't fetch alphabet list from main net" - GovernanceCantFetchAlphabetListFromSideChain = "can't fetch alphabet list from side chain" - GovernanceCantMergeAlphabetListsFromMainNetAndSideChain = "can't merge alphabet lists from main net and side chain" - GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged = "no governance update, alphabet list has not been changed" - GovernanceAlphabetListHasBeenChangedStartingUpdate = "alphabet list has been changed, starting update" - GovernanceCantVoteForSideChainCommittee = "can't vote for side chain committee" - GovernanceFinishedAlphabetListUpdate = "finished alphabet list update" - GovernanceCantFetchInnerRingListFromSideChain = "can't fetch inner ring list from side chain" - GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys = "can't create new inner ring list with new alphabet keys" - GovernanceUpdateOfTheInnerRingList = "update of the inner ring list" - GovernanceCantUpdateInnerRingListWithNewAlphabetKeys = "can't update inner ring list with new alphabet keys" - GovernanceCantUpdateListOfNotaryNodesInSideChain = "can't update list of notary nodes in side chain" - GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract = "can't update list of alphabet nodes in frostfs contract" - NetmapNetmapWorkerPool = "netmap worker pool" - NetmapTick = "tick" - NetmapNetmapWorkerPoolDrained = "netmap worker pool drained" - NetmapNetmapCleanUpRoutineIsDisabled518 = "netmap clean up routine is disabled" - NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick = "non alphabet mode, ignore new netmap cleanup tick" - NetmapCantDecodePublicKeyOfNetmapNode = "can't decode public key of netmap node" - NetmapVoteToRemoveNodeFromNetmap = "vote to remove node from netmap" - NetmapCantInvokeNetmapUpdateState = "can't invoke netmap.UpdateState" - NetmapCantIterateOnNetmapCleanerCache = "can't iterate on netmap cleaner cache" - NetmapCantGetEpochDuration = "can't get epoch duration" - NetmapCantGetTransactionHeight = "can't get transaction height" - NetmapCantResetEpochTimer = "can't reset epoch timer" - NetmapCantGetNetmapSnapshotToPerformCleanup = "can't get netmap snapshot to perform cleanup" - NetmapCantStartContainerSizeEstimation = "can't start container size estimation" - NetmapNonAlphabetModeIgnoreNewEpochTick = "non alphabet mode, ignore new epoch tick" - NetmapNextEpoch = "next epoch" - NetmapCantInvokeNetmapNewEpoch = "can't invoke netmap.NewEpoch" - NetmapNonAlphabetModeIgnoreNewPeerNotification = "non alphabet mode, ignore new peer notification" - NetmapNonhaltNotaryTransaction = "non-halt notary transaction" - NetmapCantParseNetworkMapCandidate = "can't parse network map candidate" - NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate = "could not verify and update information about network map candidate" - NetmapApprovingNetworkMapCandidate = "approving network map candidate" - NetmapCantInvokeNetmapAddPeer = "can't invoke netmap.AddPeer" - NetmapNonAlphabetModeIgnoreUpdatePeerNotification = "non alphabet mode, ignore update peer notification" - NetmapPreventSwitchingNodeToMaintenanceState = "prevent switching node to maintenance state" - NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" - FrostFSIRInternalError = "internal error" - FrostFSIRCouldNotShutdownHTTPServer = "could not shutdown HTTP server" - FrostFSIRApplicationStopped = "application stopped" - FrostFSIRCouldntCreateRPCClientForEndpoint = "could not create RPC client for endpoint" - FrostFSIRCreatedRPCClientForEndpoint = "created RPC client for endpoint" - FrostFSIRReloadExtraWallets = "reload extra wallets" - FrostFSNodeStartListeningEndpoint = "start listening endpoint" - FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" - FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" - FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" - FrostFSNodeStoppingGRPCServer = "stopping gRPC server..." - FrostFSNodeGRPCCannotShutdownGracefullyForcingStop = "gRPC cannot shutdown gracefully, forcing stop" - FrostFSNodeGRPCServerStoppedSuccessfully = "gRPC server stopped successfully" - FrostFSNodeGRPCServerError = "gRPC server error" - FrostFSNodeGRPCReconnecting = "reconnecting gRPC server..." - FrostFSNodeGRPCReconnectedSuccessfully = "gRPC server reconnected successfully" - FrostFSNodeGRPCServerConfigNotFound = "gRPC server config not found" - FrostFSNodeGRPCReconnectFailed = "failed to reconnect gRPC server" - FrostFSNodeWaitingForAllProcessesToStop = "waiting for all processes to stop" - FrostFSNodeStartedLocalNodesMaintenance = "started local node's maintenance" - FrostFSNodeStoppedLocalNodesMaintenance = "stopped local node's maintenance" - FrostFSNodeFailedToAttachShardToEngine = "failed to attach shard to engine" - FrostFSNodeShardAttachedToEngine = "shard attached to engine" - FrostFSNodeClosingComponentsOfTheStorageEngine = "closing components of the storage engine..." - FrostFSNodeAccessPolicyEngineClosingFailure = "ape closing failure" - FrostFSNodeStorageEngineClosingFailure = "storage engine closing failure" - FrostFSNodePersistentRuleStorageDBPathIsNotSetInmemoryWillBeUsed = "persistent rule storage db path is not set: in-memory will be used" - FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully = "all components of the storage engine closed successfully" - FrostFSNodeBootstrappingWithTheMaintenanceState = "bootstrapping with the maintenance state" - FrostFSNodeBootstrappingWithOnlineState = "bootstrapping with online state" - FrostFSNodeTerminationSignalHasBeenReceivedStopping = "termination signal has been received, stopping..." - FrostFSNodeTerminationSignalProcessingIsComplete = "termination signal processing is complete" - FrostFSNodeInternalApplicationError = "internal application error" - FrostFSNodeInternalErrorProcessingIsComplete = "internal error processing is complete" - FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration = "SIGHUP has been received, rereading configuration..." - FrostFSNodeSIGHUPSkip = "node not ready for reconfiguration, skipped SIGHUP" - FrostFSNodeShutdownSkip = "node already is going to shutting down, skipped shutdown" - FrostFSNodeShutdownWhenNotReady = "node is going to shutting down when subsystems still initializing" - FrostFSNodeConfigurationReading = "configuration reading" - FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" - FrostFSNodeTracingConfigationUpdated = "tracing configation updated" - FrostFSNodeStorageEngineConfigurationUpdate = "storage engine configuration update" - FrostFSNodePoolConfigurationUpdate = "adjust pool configuration" - FrostFSNodeUpdatedConfigurationApplying = "updated configuration applying" - FrostFSNodeConfigurationHasBeenReloadedSuccessfully = "configuration has been reloaded successfully" - FrostFSNodeReadNewlyCreatedContainerAfterTheNotification = "read newly created container after the notification" - FrostFSNodeContainerCreationEventsReceipt = "container creation event's receipt" - FrostFSNodeContainerRemovalEventsReceipt = "container removal event's receipt" - FrostFSNodeSaveUsedSpaceAnnouncementInContract = "save used space announcement in contract" - FrostFSNodeFailedToCalculateContainerSizeInStorageEngine = "failed to calculate container size in storage engine" - FrostFSNodeContainerSizeInStorageEngineCalculatedSuccessfully = "container size in storage engine calculated successfully" - FrostFSNodeNotificatorCouldNotListContainers = "notificator: could not list containers" - FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer = "notificator: could not select objects from container" - FrostFSNodeNotificatorCouldNotProcessObject = "notificator: could not process object" - FrostFSNodeNotificatorFinishedProcessingObjectNotifications = "notificator: finished processing object notifications" - FrostFSNodeCouldNotWriteObjectNotification = "could not write object notification" - FrostFSNodeCouldNotGetMaxObjectSizeValue = "could not get max object size value" - FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage = "could not inhume mark redundant copy as garbage" - FrostFSNodeFailedInitTracing = "failed init tracing" - FrostFSNodeFailedShutdownTracing = "failed shutdown tracing" - FrostFSNodeFailedToCreateNeoRPCClient = "failed to create neo RPC client" - FrostFSNodeClosingMorphComponents = "closing morph components..." - FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal = "failed to set group signer scope, continue with Global" - FrostFSNodeNotarySupport = "notary support" - FrostFSNodeMorphcacheTTLFetchedFromNetwork = "morph.cache_ttl fetched from network" - FrostFSNodeNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" - FrostFSNodeCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" - FrostFSNodeNewEpochEventFromSidechain = "new epoch event from sidechain" - FrostFSNodeNewBlock = "new block" - FrostFSNodeCantUpdatePersistentState = "can't update persistent state" - FrostFSNodeCantSendRebootstrapTx = "can't send re-bootstrap tx" - FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch = "could not update node state on new epoch" - FrostFSNodeCouldNotMakeNotaryDeposit = "could not make notary deposit" - FrostFSNodeInitialNetworkState = "initial network state" - FrostFSNodeTreeServiceIsNotEnabledSkipInitialization = "tree service is not enabled, skip initialization" - FrostFSNodeCouldNotSynchronizeTreeService = "could not synchronize Tree Service" - FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" - FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" - FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" - FrostFSNodePolicerIsDisabled = "policer is disabled" - CommonApplicationStarted = "application started" - ShardGCCollectingExpiredObjectsStarted = "collecting expired objects started" - ShardGCCollectingExpiredObjectsCompleted = "collecting expired objects completed" - ShardGCCollectingExpiredLocksStarted = "collecting expired locks started" - ShardGCCollectingExpiredLocksCompleted = "collecting expired locks completed" - ShardGCRemoveGarbageStarted = "garbage remove started" - ShardGCRemoveGarbageCompleted = "garbage remove completed" - EngineShardsEvacuationFailedToCount = "failed to get total objects count to evacuate" - EngineShardsEvacuationFailedToListObjects = "failed to list objects to evacuate" - EngineShardsEvacuationFailedToReadObject = "failed to read object to evacuate" - EngineShardsEvacuationFailedToMoveObject = "failed to evacuate object to other node" - ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" - ShardDeleteCantDeleteFromWriteCache = "can't delete object from write cache" - FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" - EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled" - NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap = "the node is already in candidate list with online state, skip initial bootstrap" - RPConnectionLost = "RPC connection lost, attempting reconnect" - RPCNodeSwitchFailure = "can't switch RPC node" - FSTreeCantReadFile = "can't read a file" - FSTreeCantUnmarshalObject = "can't unmarshal an object" - FSTreeCantFushObjectBlobstor = "can't flush an object to blobstor" - FSTreeCantUpdateID = "can't update object storage ID" - FSTreeCantDecodeDBObjectAddress = "can't decode object address from the DB" - PutSingleRedirectFailure = "failed to redirect PutSingle request" - StorageIDRetrievalFailure = "can't get storage ID from metabase" - ObjectRemovalFailureBlobStor = "can't remove object from blobStor" - CandidateStatusPriority = "candidate status is different from the netmap status, the former takes priority" - TombstoneExpirationParseFailure = "tombstone getter: could not parse tombstone expiration epoch" - FrostFSNodeCantUpdateObjectStorageID = "can't update object storage ID" - FrostFSNodeCantFlushObjectToBlobstor = "can't flush an object to blobstor" - FrostFSNodeCantDecodeObjectAddressFromDB = "can't decode object address from the DB" - FrostFSNodeCantUnmarshalObjectFromDB = "can't unmarshal an object from the DB" - RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" - RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" - FailedToCountWritecacheItems = "failed to count writecache items" - AttemtToCloseAlreadyClosedBlobovnicza = "attempt to close an already closed blobovnicza" - FailedToGetContainerCounters = "failed to get container counters values" - FailedToRebuildBlobstore = "failed to rebuild blobstore" - BlobstoreRebuildStarted = "blobstore rebuild started" - BlobstoreRebuildCompletedSuccessfully = "blobstore rebuild completed successfully" - BlobstoreRebuildStopped = "blobstore rebuild stopped" - BlobovniczaTreeFixingFileExtensions = "fixing blobovnicza tree file extensions..." - BlobovniczaTreeFixingFileExtensionsCompletedSuccessfully = "fixing blobovnicza tree file extensions completed successfully" - BlobovniczaTreeFixingFileExtensionsFailed = "failed to fix blobovnicza tree file extensions" - BlobovniczaTreeFixingFileExtensionForFile = "fixing blobovnicza file extension..." - BlobovniczaTreeFixingFileExtensionCompletedSuccessfully = "fixing blobovnicza file extension completed successfully" - BlobovniczaTreeFixingFileExtensionFailed = "failed to fix blobovnicza file extension" - BlobstorRebuildFailedToRebuildStorages = "failed to rebuild storages" - BlobstorRebuildRebuildStoragesCompleted = "storages rebuild completed" - BlobovniczaTreeCollectingDBToRebuild = "collecting blobovniczas to rebuild..." - BlobovniczaTreeCollectingDBToRebuildFailed = "collecting blobovniczas to rebuild failed" - BlobovniczaTreeCollectingDBToRebuildSuccess = "collecting blobovniczas to rebuild completed successfully" - BlobovniczaTreeRebuildingBlobovnicza = "rebuilding blobovnicza..." - BlobovniczaTreeRebuildingBlobovniczaFailed = "rebuilding blobovnicza failed" - BlobovniczaTreeRebuildingBlobovniczaSuccess = "rebuilding blobovnicza completed successfully" - BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza = "could not put move info to source blobovnicza" - BlobovniczatreeCouldNotUpdateStorageID = "could not update storage ID" - BlobovniczatreeCouldNotDropMoveInfo = "could not drop move info from source blobovnicza" - BlobovniczatreeCouldNotDeleteFromSource = "could not delete object from source blobovnicza" - BlobovniczaTreeCompletingPreviousRebuild = "completing previous rebuild if failed..." - BlobovniczaTreeCompletedPreviousRebuildSuccess = "previous rebuild completed successfully" - BlobovniczaTreeCompletedPreviousRebuildFailed = "failed to complete previous rebuild" - BlobovniczatreeCouldNotCheckExistenceInSourceDB = "could not check object existence in source blobovnicza" - BlobovniczatreeCouldNotCheckExistenceInTargetDB = "could not check object existence in target blobovnicza" - BlobovniczatreeCouldNotGetObjectFromSourceDB = "could not get object from source blobovnicza" - BlobovniczatreeCouldNotPutObjectToTargetDB = "could not put object to target blobovnicza" - BlobovniczaSavingCountersToMeta = "saving counters to blobovnicza's meta..." - BlobovniczaSavingCountersToMetaSuccess = "saving counters to blobovnicza's meta completed successfully" - BlobovniczaSavingCountersToMetaFailed = "saving counters to blobovnicza's meta failed" - ObjectRemovalFailureExistsInWritecache = "can't remove object: object must be flushed from writecache" - FailedToReportStatusToSystemd = "failed to report status to systemd" - ShardGCCollectingExpiredMetricsStarted = "collecting expired metrics started" - ShardGCCollectingExpiredMetricsCompleted = "collecting expired metrics completed" - ShardGCFailedToCollectZeroSizeContainers = "failed to collect zero-size containers" - ShardGCFailedToCollectZeroCountContainers = "failed to collect zero-count containers" - EngineFailedToCheckContainerAvailability = "failed to check container availability" - EngineFailedToGetContainerSize = "failed to get container size" - EngineFailedToDeleteContainerSize = "failed to delete container size" - EngineInterruptProcessingZeroSizeContainers = "interrupt processing zero-size containers" - EngineInterruptProcessingZeroCountContainers = "interrupt processing zero-count containers" - EngineFailedToGetContainerCounters = "failed to get container counters" - GetSvcV2FailedToParseNodeEndpoints = "failed to parse node endpoints" - GetSvcV2FailedToParseNodeExternalAddresses = "failed to parse node external addresses" - GetSvcV2FailedToGetRangeHashFromNode = "failed to get range hash from node" - GetSvcV2FailedToGetRangeHashFromAllOfContainerNodes = "failed to get range hash from all of container nodes" - FailedToUpdateShardID = "failed to update shard id" - EngineShardsEvacuationFailedToMoveTree = "failed to evacuate tree to other node" - EngineShardsEvacuationTreeEvacuatedLocal = "tree evacuated to local node" - EngineShardsEvacuationTreeEvacuatedRemote = "tree evacuated to other node" - EngineRefillFailedToGetObjectsCount = "failed to get blobstor objects count, no resync percent estimation is available" - BlobstoreFailedToGetFileinfo = "failed to get file info" - ECFailedToSendToContainerNode = "failed to send EC object to container node" - ECFailedToSaveECPart = "failed to save EC part" + InnerringNonalphabetModeDoNotStopContainerEstimations = "non-alphabet mode, do not stop container estimations" + InnerringCantStopEpochEstimation = "can't stop epoch estimation" + InnerringCantMakeNotaryDepositInMainChain = "can't make notary deposit in main chain" + InnerringCantMakeNotaryDepositInSideChain = "can't make notary deposit in side chain" + InnerringNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" + InnerringCantGetInnerRingIndex = "can't get inner ring index" + InnerringCantGetInnerRingSize = "can't get inner ring size" + InnerringCantGetAlphabetIndex = "can't get alphabet index" + InnerringIgnoreValidatorVoteNodeNotInAlphabetRange = "ignore validator vote: node not in alphabet range" + InnerringIgnoreValidatorVoteEmptyValidatorsList = "ignore validator vote: empty validators list" + InnerringCantInvokeVoteMethodInAlphabetContract = "can't invoke vote method in alphabet contract" + InnerringCantGetLastProcessedMainChainBlockNumber = "can't get last processed main chain block number" + InnerringNotarySupport = "notary support" + InnerringAlphabetKeysSyncIsDisabled = "alphabet keys sync is disabled" + InnerringNoControlServerEndpointSpecified = "no Control server endpoint specified, service is disabled" + InnerringCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" + InnerringFailedToSetGroupSignerScope = "failed to set group signer scope, continue with Global" + InnerringCantVoteForPreparedValidators = "can't vote for prepared validators" + InnerringNewBlock = "new block" + InnerringCantUpdatePersistentState = "can't update persistent state" + InnerringCloserError = "closer error" + InnerringReadConfigFromBlockchain = "read config from blockchain" + NotificatorNotificatorStartProcessingObjectNotifications = "notificator: start processing object notifications" + NotificatorNotificatorProcessingObjectNotification = "notificator: processing object notification" + PolicerCouldNotGetContainer = "could not get container" + PolicerCouldNotConfirmContainerRemoval = "could not confirm container removal" + PolicerCouldNotInhumeObjectWithMissingContainer = "could not inhume object with missing container" + PolicerCouldNotBuildPlacementVectorForObject = "could not build placement vector for object" + PolicerRedundantLocalObjectCopyDetected = "redundant local object copy detected" + PolicerReceiveObjectHeaderToCheckPolicyCompliance = "receive object header to check policy compliance" + PolicerConsiderNodeUnderMaintenanceAsOK = "consider node under maintenance as OK" + PolicerShortageOfObjectCopiesDetected = "shortage of object copies detected" + PolicerSomeOfTheCopiesAreStoredOnNodesUnderMaintenance = "some of the copies are stored on nodes under maintenance, save local copy" + PolicerRoutineStopped = "routine stopped" + PolicerFailureAtObjectSelectForReplication = "failure at object select for replication" + PolicerPoolSubmission = "pool submission" + PolicerUnableToProcessObj = "unable to process object" + ReplicatorFinishWork = "finish work" + ReplicatorCouldNotGetObjectFromLocalStorage = "could not get object from local storage" + ReplicatorCouldNotReplicateObject = "could not replicate object" + ReplicatorObjectSuccessfullyReplicated = "object successfully replicated" + TreeRedirectingTreeServiceQuery = "redirecting tree service query" + TreeBearerPresentedButNotAllowedByACL = "bearer presented but not allowed by ACL" + TreeCouldNotGetLastSynchronizedHeightForATree = "could not get last synchronized height for a tree" + TreeCouldNotUpdateLastSynchronizedHeightForATree = "could not update last synchronized height for a tree" + TreeSynchronizeTree = "synchronize tree" + TreeFailedToRunTreeSynchronizationOverAllNodes = "failed to run tree synchronization over all nodes" + TreeFailedToRunTreeSynchronizationForSpecificNode = "failed to run tree synchronization for specific node" + TreeFailedToParseAddressForTreeSynchronization = "failed to parse address for tree synchronization" + TreeFailedToConnectForTreeSynchronization = "failed to connect for tree synchronization" + TreeSyncingTrees = "syncing trees..." + TreeCouldNotFetchContainers = "could not fetch containers" + TreeTreesHaveBeenSynchronized = "trees have been synchronized" + TreeSyncingContainerTrees = "syncing container trees..." + TreeCouldNotSyncTrees = "could not sync trees" + TreeContainerTreesHaveBeenSynced = "container trees have been synced" + TreeCouldNotQueryTreesForSynchronization = "could not query trees for synchronization" + TreeRemovingRedundantTrees = "removing redundant trees..." + TreeCouldNotCheckIfContainerExisted = "could not check if the container ever existed" + TreeCouldNotRemoveRedundantTree = "could not remove redundant tree" + TreeCouldNotCalculateContainerNodes = "could not calculate container nodes" + TreeFailedToApplyReplicatedOperation = "failed to apply replicated operation" + TreeDoNotSendUpdateToTheNode = "do not send update to the node" + TreeFailedToSentUpdateToTheNode = "failed to sent update to the node" + TreeErrorDuringReplication = "error during replication" + PersistentCouldNotGetSessionFromPersistentStorage = "could not get session from persistent storage" + PersistentCouldNotDeleteSToken = "could not delete token" + PersistentCouldNotCleanUpExpiredTokens = "could not clean up expired tokens" + TombstoneCouldNotGetTheTombstoneTheSource = "tombstone getter: could not get the tombstone the source" + DeleteNoSplitInfoObjectIsPHY = "no split info, object is PHY" + DeleteAssemblingChain = "assembling chain..." + DeleteCollectingChildren = "collecting children..." + DeleteSupplementBySplitID = "supplement by split ID" + DeleteFormingTombstoneStructure = "forming tombstone structure..." + DeleteTombstoneStructureSuccessfullyFormedSaving = "tombstone structure successfully formed, saving..." + DeleteFormingSplitInfo = "forming split info..." + DeleteSplitInfoSuccessfullyFormedCollectingMembers = "split info successfully formed, collecting members..." + DeleteMembersSuccessfullyCollected = "members successfully collected" + GetRemoteCallFailed = "remote call failed" + GetCanNotAssembleTheObject = "can not assemble the object" + GetTryingToAssembleTheObject = "trying to assemble the object..." + GetTryingToAssembleTheECObject = "trying to assemble the ec object..." + GetAssemblingSplittedObject = "assembling splitted object..." + GetAssemblingECObject = "assembling erasure-coded object..." + GetUnableToGetAllPartsECObject = "unable to get all parts, continue to reconstruct with existed" + GetUnableToGetPartECObject = "unable to get part of the erasure-encoded object" + GetUnableToHeadPartECObject = "unable to head part of the erasure-encoded object" + GetUnableToGetECObjectContainer = "unable to get container for erasure-coded object" + GetAssemblingSplittedObjectCompleted = "assembling splitted object completed" + GetAssemblingECObjectCompleted = "assembling erasure-coded object completed" + GetFailedToAssembleSplittedObject = "failed to assemble splitted object" + GetFailedToAssembleECObject = "failed to assemble erasure-coded object" + GetCouldNotGenerateContainerTraverser = "could not generate container traverser" + GetCouldNotConstructRemoteNodeClient = "could not construct remote node client" + GetCouldNotWriteHeader = "could not write header" + GetCouldNotWritePayloadChunk = "could not write payload chunk" + GetLocalGetFailed = "local get failed" + GetReturnResultDirectly = "return result directly" + GetCompletingTheOperation = "completing the operation" + GetRequestedObjectWasMarkedAsRemoved = "requested object was marked as removed" + GetRequestedObjectIsVirtual = "requested object is virtual" + GetRequestedObjectIsEC = "requested object is erasure-coded" + GetRequestedRangeIsOutOfObjectBounds = "requested range is out of object bounds" + PutAdditionalContainerBroadcastFailure = "additional container broadcast failure" + SearchReturnResultDirectly = "return result directly" + SearchCouldNotConstructRemoteNodeClient = "could not construct remote node client" + SearchRemoteOperationFailed = "remote operation failed" + SearchCouldNotGenerateContainerTraverser = "could not generate container traverser" + SearchCouldNotWriteObjectIdentifiers = "could not write object identifiers" + SearchLocalOperationFailed = "local operation failed" + UtilObjectServiceError = "object service error" + UtilCouldNotPushTaskToWorkerPool = "could not push task to worker pool" + V2CantCheckIfRequestFromInnerRing = "can't check if request from inner ring" + V2CantCheckIfRequestFromContainerNode = "can't check if request from container node" + NatsNatsConnectionWasLost = "nats: connection was lost" + NatsNatsReconnectedToTheServer = "nats: reconnected to the server" + NatsNatsClosingConnectionAsTheContextIsDone = "nats: closing connection as the context is done" + NatsConnectedToEndpoint = "nats: successfully connected to endpoint" + ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch = "could not restore block subscription after RPC switch" + ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch = "could not restore notification subscription after RPC switch" + ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch = "could not restore notary notification subscription after RPC switch" + ClientCouldNotEstablishConnectionToTheSwitchedRPCNode = "could not establish connection to the switched RPC node" + ClientConnectionToTheNewRPCNodeHasBeenEstablished = "connection to the new RPC node has been established" + ClientCouldNotCreateClientToTheHigherPriorityNode = "could not create client to the higher priority node" + ClientSwitchedToTheHigherPriorityRPC = "switched to the higher priority RPC" + ClientNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" + ClientNotaryDepositInvoke = "notary deposit invoke" + ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked" + ClientNotaryRequestInvoked = "notary request invoked" + ClientNotaryDepositTransactionWasSuccessfullyPersisted = "notary deposit transaction was successfully persisted" + ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted = "attempt to wait for notary deposit transaction to get persisted" + ClientNeoClientInvoke = "neo client invoke" + ClientNativeGasTransferInvoke = "native gas transfer invoke" + ClientBatchGasTransferInvoke = "batch gas transfer invoke" + ClientCantGetBlockchainHeight = "can't get blockchain height" + ClientCantGetBlockchainHeight243 = "can't get blockchain height" + EventCouldNotSubmitHandlerToWorkerPool = "could not Submit handler to worker pool" + EventCouldNotStartListenToEvents = "could not start listen to events" + EventStopEventListenerByError = "stop event listener by error" + EventStopEventListenerByContext = "stop event listener by context" + EventStopEventListenerByNotificationChannel = "stop event listener by notification channel" + EventNilNotificationEventWasCaught = "nil notification event was caught" + EventStopEventListenerByNotaryChannel = "stop event listener by notary channel" + EventNilNotaryEventWasCaught = "nil notary event was caught" + EventStopEventListenerByBlockChannel = "stop event listener by block channel" + EventNilBlockWasCaught = "nil block was caught" + EventListenerWorkerPoolDrained = "listener worker pool drained" + EventEventParserNotSet = "event parser not set" + EventCouldNotParseNotificationEvent = "could not parse notification event" + EventNotificationHandlersForParsedNotificationEventWereNotRegistered = "notification handlers for parsed notification event were not registered" + EventSkipExpiredMainTXNotaryEvent = "skip expired main TX notary event" + EventCouldNotPrepareAndValidateNotaryEvent = "could not prepare and validate notary event" + EventNotaryParserNotSet = "notary parser not set" + EventCouldNotParseNotaryEvent = "could not parse notary event" + EventNotaryHandlersForParsedNotificationEventWereNotRegistered = "notary handlers for parsed notification event were not registered" + EventIgnoreNilEventParser = "ignore nil event parser" + EventListenerHasBeenAlreadyStartedIgnoreParser = "listener has been already started, ignore parser" + EventRegisteredNewEventParser = "registered new event parser" + EventIgnoreNilEventHandler = "ignore nil event handler" + EventIgnoreHandlerOfEventWoParser = "ignore handler of event w/o parser" + EventRegisteredNewEventHandler = "registered new event handler" + EventIgnoreNilNotaryEventParser = "ignore nil notary event parser" + EventListenerHasBeenAlreadyStartedIgnoreNotaryParser = "listener has been already started, ignore notary parser" + EventIgnoreNilNotaryEventHandler = "ignore nil notary event handler" + EventIgnoreHandlerOfNotaryEventWoParser = "ignore handler of notary event w/o parser" + EventIgnoreNilBlockHandler = "ignore nil block handler" + StorageOperation = "local object storage operation" + BlobovniczaCreatingDirectoryForBoltDB = "creating directory for BoltDB" + BlobovniczaOpeningBoltDB = "opening BoltDB" + BlobovniczaInitializing = "initializing..." + BlobovniczaAlreadyInitialized = "already initialized" + BlobovniczaCreatingBucketForSizeRange = "creating bucket for size range" + BlobovniczaClosingBoltDB = "closing BoltDB" + BlobovniczaObjectWasRemovedFromBucket = "object was removed from bucket" + BlobstorOpening = "opening..." + BlobstorInitializing = "initializing..." + BlobstorClosing = "closing..." + BlobstorCouldntCloseStorage = "couldn't close storage" + BlobstorErrorOccurredDuringObjectExistenceChecking = "error occurred during object existence checking" + BlobstorErrorOccurredDuringTheIteration = "error occurred during the iteration" + EngineShardHasBeenRemoved = "shard has been removed" + EngineCouldNotCloseRemovedShard = "could not close removed shard" + EngineCouldNotOpenShardClosingAndSkipping = "could not open shard, closing and skipping" + EngineCouldNotClosePartiallyInitializedShard = "could not close partially initialized shard" + EngineCouldNotInitializeShardClosingAndSkipping = "could not initialize shard, closing and skipping" + EngineCouldNotCloseShard = "could not close shard" + EngineCouldNotReloadAShard = "could not reload a shard" + EngineAddedNewShard = "added new shard" + EngineCouldNotPutObjectToShard = "could not put object to shard" + EngineErrorDuringSearchingForObjectChildren = "error during searching for object children" + EngineCouldNotInhumeObjectInShard = "could not inhume object in shard" + EngineStartingRemovalOfLocallyredundantCopies = "starting removal of locally-redundant copies" + EngineStartedDuplicatesRemovalRoutine = "started duplicates removal routine" + EngineFinishedRemovalOfLocallyredundantCopies = "finished removal of locally-redundant copies" + EngineRemovingAnObjectWithoutFullLockingCheck = "removing an object without full locking check" + EngineInterruptProcessingTheExpiredLocks = "interrupt processing the expired locks" + EngineInterruptProcessingTheDeletedLocks = "interrupt processing the deleted locks" + EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly = "failed to move shard in degraded-read-only mode, moving to read-only" + EngineFailedToMoveShardInReadonlyMode = "failed to move shard in read-only mode" + EngineShardIsMovedInReadonlyModeDueToErrorThreshold = "shard is moved in read-only mode due to error threshold" + EngineShardIsMovedInDegradedModeDueToErrorThreshold = "shard is moved in degraded mode due to error threshold" + EngineModeChangeIsInProgressIgnoringSetmodeRequest = "mode change is in progress, ignoring set-mode request" + EngineStartedShardsEvacuation = "started shards evacuation" + EngineFinishedSuccessfullyShardsEvacuation = "shards evacuation finished successfully" + EngineFinishedWithErrorShardsEvacuation = "shards evacuation finished with error" + EngineObjectIsMovedToAnotherShard = "object is moved to another shard" + MetabaseMissingMatcher = "missing matcher" + MetabaseErrorInFKBTSelection = "error in FKBT selection" + MetabaseCantDecodeListBucketLeaf = "can't decode list bucket leaf" + MetabaseUnknownOperation = "unknown operation" + MetabaseCantIterateOverTheBucket = "can't iterate over the bucket" + MetabaseCouldNotIterateOverTheBuckets = "could not iterate over the buckets" + MetabaseCreatedDirectoryForMetabase = "created directory for Metabase" + MetabaseOpenedBoltDBInstanceForMetabase = "opened boltDB instance for Metabase" + MetabaseCheckingMetabaseVersion = "checking metabase version" + ShardCantSelectAllObjects = "can't select all objects" + ShardSettingShardMode = "setting shard mode" + ShardShardModeSetSuccessfully = "shard mode set successfully" + ShardFetchingObjectWithoutMeta = "fetching object without meta" + ShardObjectIsMissingInWritecache = "object is missing in write-cache" + ShardFailedToFetchObjectFromWritecache = "failed to fetch object from write-cache" + ShardCantPutObjectToTheWritecacheTryingBlobstor = "can't put object to the write-cache, trying blobstor" + ShardMetaObjectCounterRead = "meta: object counter read" + ShardMetaCantReadContainerList = "meta: can't read container list" + ShardMetaCantReadContainerSize = "meta: can't read container size" + ShardMetaInfoPresentButObjectNotFound = "meta info was present, but the object is missing" + ShardMetabaseFailureSwitchingMode = "metabase failure, switching mode" + ShardCantMoveShardToReadonlySwitchMode = "can't move shard to readonly, switch mode" + ShardCouldNotUnmarshalObject = "could not unmarshal object" + ShardCouldNotCloseShardComponent = "could not close shard component" + ShardCantOpenMetabaseMoveToADegradedMode = "can't open metabase, move to a degraded mode" + ShardCantInitializeMetabaseMoveToADegradedreadonlyMode = "can't initialize metabase, move to a degraded-read-only mode" + ShardTryingToRestoreReadwriteMode = "trying to restore read-write mode" + ShardStopEventListenerByClosedEventChannel = "stop event listener by closed `event` channel" + ShardStopEventListenerByClosedStopChannel = "stop event listener by closed `stop` channel" + ShardStopEventListenerByContext = "stop event listener by context" + ShardCouldNotSubmitGCJobToWorkerPool = "could not submit GC job to worker pool" + ShardGCIsStopped = "GC is stopped" + ShardWaitingForGCWorkersToStop = "waiting for GC workers to stop..." + ShardIteratorOverMetabaseGraveyardFailed = "iterator over metabase graveyard failed" + ShardCouldNotDeleteTheObjects = "could not delete the objects" + ShardIteratorOverExpiredObjectsFailed = "iterator over expired objects failed" + ShardCouldNotInhumeTheObjects = "could not inhume the objects" + ShardStartedExpiredTombstonesHandling = "started expired tombstones handling" + ShardIteratingTombstones = "iterating tombstones" + ShardShardIsInADegradedModeSkipCollectingExpiredTombstones = "shard is in a degraded mode, skip collecting expired tombstones" + ShardIteratorOverGraveyardFailed = "iterator over graveyard failed" + ShardHandlingExpiredTombstonesBatch = "handling expired tombstones batch" + ShardFinishedExpiredTombstonesHandling = "finished expired tombstones handling" + ShardIteratorOverExpiredLocksFailed = "iterator over expired locks failed" + ShardCouldNotMarkTombstonesAsGarbage = "could not mark tombstones as garbage" + ShardCouldNotDropExpiredGraveRecords = "could not drop expired grave records" + ShardFailureToUnlockObjects = "failure to unlock objects" + ShardFailureToMarkLockersAsGarbage = "failure to mark lockers as garbage" + ShardFailureToGetExpiredUnlockedObjects = "failure to get expired unlocked objects" + ShardCouldNotMarkObjectToDeleteInMetabase = "could not mark object to delete in metabase" + WritecacheTriedToFlushItemsFromWritecache = "tried to flush items from write-cache" + WritecacheWaitingForChannelsToFlush = "waiting for channels to flush" + WritecacheCantRemoveObjectsFromTheDatabase = "can't remove objects from the database" + WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" + BlobovniczatreeCouldNotGetObjectFromLevel = "could not get object from level" + BlobovniczatreeCouldNotCloseBlobovnicza = "could not close Blobovnicza" + BlobovniczatreeCouldNotRemoveObjectFromLevel = "could not remove object from level" + BlobovniczatreeCouldNotGetActiveBlobovnicza = "could not get active blobovnicza" + BlobovniczatreeBlobovniczaOverflowed = "blobovnicza overflowed" + BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza = "could not put object to active blobovnicza" + BlobovniczatreeInitializingBlobovniczas = "initializing Blobovnicza's" + BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization = "read-only mode, skip blobovniczas initialization..." + BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing = "blobovnicza successfully initialized, closing..." + AlphabetTick = "tick" + AlphabetAlphabetProcessorWorkerPoolDrained = "alphabet processor worker pool drained" + AlphabetNonAlphabetModeIgnoreGasEmissionEvent = "non alphabet mode, ignore gas emission event" + AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent = "node is out of alphabet range, ignore gas emission event" + AlphabetCantInvokeAlphabetEmitMethod = "can't invoke alphabet emit method" + AlphabetStorageNodeEmissionIsOff = "storage node emission is off" + AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes = "can't get netmap snapshot to emit gas to storage nodes" + AlphabetGasEmission = "gas emission" + AlphabetCantParseNodePublicKey = "can't parse node public key" + AlphabetCantTransferGas = "can't transfer gas" + AlphabetCantTransferGasToWallet = "can't transfer gas to wallet" + AlphabetAlphabetWorkerPool = "alphabet worker pool" + BalanceBalanceWorkerPoolDrained = "balance worker pool drained" + BalanceNonAlphabetModeIgnoreBalanceLock = "non alphabet mode, ignore balance lock" + BalanceCantSendLockAssetTx = "can't send lock asset tx" + BalanceBalanceWorkerPool = "balance worker pool" + ContainerContainerWorkerPool = "container worker pool" + ContainerContainerProcessorWorkerPoolDrained = "container processor worker pool drained" + ContainerNonAlphabetModeIgnoreContainerPut = "non alphabet mode, ignore container put" + ContainerPutContainerCheckFailed = "put container check failed" + ContainerCouldNotApprovePutContainer = "could not approve put container" + ContainerNonAlphabetModeIgnoreContainerDelete = "non alphabet mode, ignore container delete" + ContainerDeleteContainerCheckFailed = "delete container check failed" + ContainerCouldNotApproveDeleteContainer = "could not approve delete container" + ContainerNonAlphabetModeIgnoreSetEACL = "non alphabet mode, ignore set EACL" + ContainerSetEACLCheckFailed = "set EACL check failed" + ContainerCouldNotApproveSetEACL = "could not approve set EACL" + FrostFSNonAlphabetModeIgnoreConfig = "non alphabet mode, ignore config" + FrostFSCantRelaySetConfigEvent = "can't relay set config event" + FrostFSFrostfsWorkerPool = "frostfs worker pool" + FrostFSFrostfsProcessorWorkerPoolDrained = "frostfs processor worker pool drained" + FrostFSNonAlphabetModeIgnoreDeposit = "non alphabet mode, ignore deposit" + FrostFSCantTransferAssetsToBalanceContract = "can't transfer assets to balance contract" + FrostFSDoubleMintEmissionDeclined = "double mint emission declined" + FrostFSCantGetGasBalanceOfTheNode = "can't get gas balance of the node" + FrostFSGasBalanceThresholdHasBeenReached = "gas balance threshold has been reached" + FrostFSCantTransferNativeGasToReceiver = "can't transfer native gas to receiver" + FrostFSNonAlphabetModeIgnoreWithdraw = "non alphabet mode, ignore withdraw" + FrostFSCantCreateLockAccount = "can't create lock account" + FrostFSCantLockAssetsForWithdraw = "can't lock assets for withdraw" + FrostFSNonAlphabetModeIgnoreCheque = "non alphabet mode, ignore cheque" + FrostFSCantTransferAssetsToFedContract = "can't transfer assets to fed contract" + GovernanceNewEvent = "new event" + GovernanceGovernanceWorkerPoolDrained = "governance worker pool drained" + GovernanceNonAlphabetModeIgnoreAlphabetSync = "non alphabet mode, ignore alphabet sync" + GovernanceCantFetchAlphabetListFromMainNet = "can't fetch alphabet list from main net" + GovernanceCantFetchAlphabetListFromSideChain = "can't fetch alphabet list from side chain" + GovernanceCantMergeAlphabetListsFromMainNetAndSideChain = "can't merge alphabet lists from main net and side chain" + GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged = "no governance update, alphabet list has not been changed" + GovernanceAlphabetListHasBeenChangedStartingUpdate = "alphabet list has been changed, starting update" + GovernanceCantVoteForSideChainCommittee = "can't vote for side chain committee" + GovernanceFinishedAlphabetListUpdate = "finished alphabet list update" + GovernanceCantFetchInnerRingListFromSideChain = "can't fetch inner ring list from side chain" + GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys = "can't create new inner ring list with new alphabet keys" + GovernanceUpdateOfTheInnerRingList = "update of the inner ring list" + GovernanceCantUpdateInnerRingListWithNewAlphabetKeys = "can't update inner ring list with new alphabet keys" + GovernanceCantUpdateListOfNotaryNodesInSideChain = "can't update list of notary nodes in side chain" + GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract = "can't update list of alphabet nodes in frostfs contract" + NetmapNetmapWorkerPool = "netmap worker pool" + NetmapTick = "tick" + NetmapNetmapWorkerPoolDrained = "netmap worker pool drained" + NetmapNetmapCleanUpRoutineIsDisabled518 = "netmap clean up routine is disabled" + NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick = "non alphabet mode, ignore new netmap cleanup tick" + NetmapCantDecodePublicKeyOfNetmapNode = "can't decode public key of netmap node" + NetmapVoteToRemoveNodeFromNetmap = "vote to remove node from netmap" + NetmapCantInvokeNetmapUpdateState = "can't invoke netmap.UpdateState" + NetmapCantIterateOnNetmapCleanerCache = "can't iterate on netmap cleaner cache" + NetmapCantGetEpochDuration = "can't get epoch duration" + NetmapCantGetTransactionHeight = "can't get transaction height" + NetmapCantResetEpochTimer = "can't reset epoch timer" + NetmapCantGetNetmapSnapshotToPerformCleanup = "can't get netmap snapshot to perform cleanup" + NetmapCantStartContainerSizeEstimation = "can't start container size estimation" + NetmapNonAlphabetModeIgnoreNewEpochTick = "non alphabet mode, ignore new epoch tick" + NetmapNextEpoch = "next epoch" + NetmapCantInvokeNetmapNewEpoch = "can't invoke netmap.NewEpoch" + NetmapNonAlphabetModeIgnoreNewPeerNotification = "non alphabet mode, ignore new peer notification" + NetmapNonhaltNotaryTransaction = "non-halt notary transaction" + NetmapCantParseNetworkMapCandidate = "can't parse network map candidate" + NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate = "could not verify and update information about network map candidate" + NetmapApprovingNetworkMapCandidate = "approving network map candidate" + NetmapCantInvokeNetmapAddPeer = "can't invoke netmap.AddPeer" + NetmapNonAlphabetModeIgnoreUpdatePeerNotification = "non alphabet mode, ignore update peer notification" + NetmapPreventSwitchingNodeToMaintenanceState = "prevent switching node to maintenance state" + NetmapCantInvokeNetmapUpdatePeer = "can't invoke netmap.UpdatePeer" + FrostFSIRInternalError = "internal error" + FrostFSIRCouldNotShutdownHTTPServer = "could not shutdown HTTP server" + FrostFSIRApplicationStopped = "application stopped" + FrostFSIRCouldntCreateRPCClientForEndpoint = "could not create RPC client for endpoint" + FrostFSIRCreatedRPCClientForEndpoint = "created RPC client for endpoint" + FrostFSIRReloadExtraWallets = "reload extra wallets" + FrostFSNodeStartListeningEndpoint = "start listening endpoint" + FrostFSNodeCouldNotReadCertificateFromFile = "could not read certificate from file" + FrostFSNodeCantListenGRPCEndpoint = "can't listen gRPC endpoint" + FrostFSNodeStopListeningGRPCEndpoint = "stop listening gRPC endpoint" + FrostFSNodeStoppingGRPCServer = "stopping gRPC server..." + FrostFSNodeGRPCCannotShutdownGracefullyForcingStop = "gRPC cannot shutdown gracefully, forcing stop" + FrostFSNodeGRPCServerStoppedSuccessfully = "gRPC server stopped successfully" + FrostFSNodeGRPCServerError = "gRPC server error" + FrostFSNodeGRPCReconnecting = "reconnecting gRPC server..." + FrostFSNodeGRPCReconnectedSuccessfully = "gRPC server reconnected successfully" + FrostFSNodeGRPCServerConfigNotFound = "gRPC server config not found" + FrostFSNodeGRPCReconnectFailed = "failed to reconnect gRPC server" + FrostFSNodeWaitingForAllProcessesToStop = "waiting for all processes to stop" + FrostFSNodeStartedLocalNodesMaintenance = "started local node's maintenance" + FrostFSNodeStoppedLocalNodesMaintenance = "stopped local node's maintenance" + FrostFSNodeFailedToAttachShardToEngine = "failed to attach shard to engine" + FrostFSNodeShardAttachedToEngine = "shard attached to engine" + FrostFSNodeClosingComponentsOfTheStorageEngine = "closing components of the storage engine..." + FrostFSNodeAccessPolicyEngineClosingFailure = "ape closing failure" + FrostFSNodeStorageEngineClosingFailure = "storage engine closing failure" + FrostFSNodePersistentRuleStorageDBPathIsNotSetInmemoryWillBeUsed = "persistent rule storage db path is not set: in-memory will be used" + FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully = "all components of the storage engine closed successfully" + FrostFSNodeBootstrappingWithTheMaintenanceState = "bootstrapping with the maintenance state" + FrostFSNodeBootstrappingWithOnlineState = "bootstrapping with online state" + FrostFSNodeTerminationSignalHasBeenReceivedStopping = "termination signal has been received, stopping..." + FrostFSNodeTerminationSignalProcessingIsComplete = "termination signal processing is complete" + FrostFSNodeInternalApplicationError = "internal application error" + FrostFSNodeInternalErrorProcessingIsComplete = "internal error processing is complete" + FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration = "SIGHUP has been received, rereading configuration..." + FrostFSNodeSIGHUPSkip = "node not ready for reconfiguration, skipped SIGHUP" + FrostFSNodeShutdownSkip = "node already is going to shutting down, skipped shutdown" + FrostFSNodeShutdownWhenNotReady = "node is going to shutting down when subsystems still initializing" + FrostFSNodeConfigurationReading = "configuration reading" + FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" + FrostFSNodeTracingConfigationUpdated = "tracing configation updated" + FrostFSNodeStorageEngineConfigurationUpdate = "storage engine configuration update" + FrostFSNodePoolConfigurationUpdate = "adjust pool configuration" + FrostFSNodeUpdatedConfigurationApplying = "updated configuration applying" + FrostFSNodeConfigurationHasBeenReloadedSuccessfully = "configuration has been reloaded successfully" + FrostFSNodeReadNewlyCreatedContainerAfterTheNotification = "read newly created container after the notification" + FrostFSNodeContainerCreationEventsReceipt = "container creation event's receipt" + FrostFSNodeContainerRemovalEventsReceipt = "container removal event's receipt" + FrostFSNodeNotificatorCouldNotListContainers = "notificator: could not list containers" + FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer = "notificator: could not select objects from container" + FrostFSNodeNotificatorCouldNotProcessObject = "notificator: could not process object" + FrostFSNodeNotificatorFinishedProcessingObjectNotifications = "notificator: finished processing object notifications" + FrostFSNodeCouldNotWriteObjectNotification = "could not write object notification" + FrostFSNodeCouldNotGetMaxObjectSizeValue = "could not get max object size value" + FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage = "could not inhume mark redundant copy as garbage" + FrostFSNodeFailedInitTracing = "failed init tracing" + FrostFSNodeFailedShutdownTracing = "failed shutdown tracing" + FrostFSNodeFailedToCreateNeoRPCClient = "failed to create neo RPC client" + FrostFSNodeClosingMorphComponents = "closing morph components..." + FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal = "failed to set group signer scope, continue with Global" + FrostFSNodeNotarySupport = "notary support" + FrostFSNodeMorphcacheTTLFetchedFromNetwork = "morph.cache_ttl fetched from network" + FrostFSNodeNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" + FrostFSNodeCantGetLastProcessedSideChainBlockNumber = "can't get last processed side chain block number" + FrostFSNodeNewEpochEventFromSidechain = "new epoch event from sidechain" + FrostFSNodeNewBlock = "new block" + FrostFSNodeCantUpdatePersistentState = "can't update persistent state" + FrostFSNodeCantSendRebootstrapTx = "can't send re-bootstrap tx" + FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch = "could not update node state on new epoch" + FrostFSNodeCouldNotMakeNotaryDeposit = "could not make notary deposit" + FrostFSNodeInitialNetworkState = "initial network state" + FrostFSNodeTreeServiceIsNotEnabledSkipInitialization = "tree service is not enabled, skip initialization" + FrostFSNodeCouldNotSynchronizeTreeService = "could not synchronize Tree Service" + FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" + FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" + FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" + FrostFSNodePolicerIsDisabled = "policer is disabled" + CommonApplicationStarted = "application started" + ShardGCCollectingExpiredObjectsStarted = "collecting expired objects started" + ShardGCCollectingExpiredObjectsCompleted = "collecting expired objects completed" + ShardGCCollectingExpiredLocksStarted = "collecting expired locks started" + ShardGCCollectingExpiredLocksCompleted = "collecting expired locks completed" + ShardGCRemoveGarbageStarted = "garbage remove started" + ShardGCRemoveGarbageCompleted = "garbage remove completed" + EngineShardsEvacuationFailedToCount = "failed to get total objects count to evacuate" + EngineShardsEvacuationFailedToListObjects = "failed to list objects to evacuate" + EngineShardsEvacuationFailedToReadObject = "failed to read object to evacuate" + EngineShardsEvacuationFailedToMoveObject = "failed to evacuate object to other node" + ShardGCFailedToGetExpiredWithLinked = "failed to get expired objects with linked" + FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" + EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled" + RPConnectionLost = "RPC connection lost, attempting reconnect" + RPCNodeSwitchFailure = "can't switch RPC node" + FSTreeCantUnmarshalObject = "can't unmarshal an object" + FSTreeCantFushObjectBlobstor = "can't flush an object to blobstor" + FSTreeCantUpdateID = "can't update object storage ID" + FSTreeCantDecodeDBObjectAddress = "can't decode object address from the DB" + PutSingleRedirectFailure = "failed to redirect PutSingle request" + StorageIDRetrievalFailure = "can't get storage ID from metabase" + ObjectRemovalFailureBlobStor = "can't remove object from blobStor" + CandidateStatusPriority = "candidate status is different from the netmap status, the former takes priority" + TombstoneExpirationParseFailure = "tombstone getter: could not parse tombstone expiration epoch" + RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" + RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" + AttemtToCloseAlreadyClosedBlobovnicza = "attempt to close an already closed blobovnicza" + FailedToGetContainerCounters = "failed to get container counters values" + FailedToRebuildBlobstore = "failed to rebuild blobstore" + BlobstoreRebuildStarted = "blobstore rebuild started" + BlobstoreRebuildCompletedSuccessfully = "blobstore rebuild completed successfully" + BlobstoreRebuildStopped = "blobstore rebuild stopped" + BlobovniczaTreeFixingFileExtensions = "fixing blobovnicza tree file extensions..." + BlobovniczaTreeFixingFileExtensionsCompletedSuccessfully = "fixing blobovnicza tree file extensions completed successfully" + BlobovniczaTreeFixingFileExtensionsFailed = "failed to fix blobovnicza tree file extensions" + BlobovniczaTreeFixingFileExtensionForFile = "fixing blobovnicza file extension..." + BlobovniczaTreeFixingFileExtensionCompletedSuccessfully = "fixing blobovnicza file extension completed successfully" + BlobovniczaTreeFixingFileExtensionFailed = "failed to fix blobovnicza file extension" + BlobstorRebuildFailedToRebuildStorages = "failed to rebuild storages" + BlobstorRebuildRebuildStoragesCompleted = "storages rebuild completed" + BlobovniczaTreeCollectingDBToRebuild = "collecting blobovniczas to rebuild..." + BlobovniczaTreeCollectingDBToRebuildFailed = "collecting blobovniczas to rebuild failed" + BlobovniczaTreeCollectingDBToRebuildSuccess = "collecting blobovniczas to rebuild completed successfully" + BlobovniczaTreeRebuildingBlobovnicza = "rebuilding blobovnicza..." + BlobovniczaTreeRebuildingBlobovniczaFailed = "rebuilding blobovnicza failed" + BlobovniczaTreeRebuildingBlobovniczaSuccess = "rebuilding blobovnicza completed successfully" + BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza = "could not put move info to source blobovnicza" + BlobovniczatreeCouldNotUpdateStorageID = "could not update storage ID" + BlobovniczatreeCouldNotDropMoveInfo = "could not drop move info from source blobovnicza" + BlobovniczatreeCouldNotDeleteFromSource = "could not delete object from source blobovnicza" + BlobovniczaTreeCompletingPreviousRebuild = "completing previous rebuild if failed..." + BlobovniczaTreeCompletedPreviousRebuildSuccess = "previous rebuild completed successfully" + BlobovniczaTreeCompletedPreviousRebuildFailed = "failed to complete previous rebuild" + BlobovniczatreeCouldNotCheckExistenceInTargetDB = "could not check object existence in target blobovnicza" + BlobovniczatreeCouldNotPutObjectToTargetDB = "could not put object to target blobovnicza" + BlobovniczaSavingCountersToMeta = "saving counters to blobovnicza's meta..." + BlobovniczaSavingCountersToMetaSuccess = "saving counters to blobovnicza's meta completed successfully" + BlobovniczaSavingCountersToMetaFailed = "saving counters to blobovnicza's meta failed" + ObjectRemovalFailureExistsInWritecache = "can't remove object: object must be flushed from writecache" + FailedToReportStatusToSystemd = "failed to report status to systemd" + ShardGCCollectingExpiredMetricsStarted = "collecting expired metrics started" + ShardGCCollectingExpiredMetricsCompleted = "collecting expired metrics completed" + ShardGCFailedToCollectZeroSizeContainers = "failed to collect zero-size containers" + ShardGCFailedToCollectZeroCountContainers = "failed to collect zero-count containers" + EngineFailedToCheckContainerAvailability = "failed to check container availability" + EngineFailedToGetContainerSize = "failed to get container size" + EngineFailedToDeleteContainerSize = "failed to delete container size" + EngineInterruptProcessingZeroSizeContainers = "interrupt processing zero-size containers" + EngineInterruptProcessingZeroCountContainers = "interrupt processing zero-count containers" + EngineFailedToGetContainerCounters = "failed to get container counters" + GetSvcV2FailedToParseNodeEndpoints = "failed to parse node endpoints" + GetSvcV2FailedToParseNodeExternalAddresses = "failed to parse node external addresses" + GetSvcV2FailedToGetRangeHashFromNode = "failed to get range hash from node" + GetSvcV2FailedToGetRangeHashFromAllOfContainerNodes = "failed to get range hash from all of container nodes" + FailedToUpdateShardID = "failed to update shard id" + EngineShardsEvacuationFailedToMoveTree = "failed to evacuate tree to other node" + EngineShardsEvacuationTreeEvacuatedLocal = "tree evacuated to local node" + EngineShardsEvacuationTreeEvacuatedRemote = "tree evacuated to other node" + EngineRefillFailedToGetObjectsCount = "failed to get blobstor objects count, no resync percent estimation is available" + ECFailedToSendToContainerNode = "failed to send EC object to container node" + ECFailedToSaveECPart = "failed to save EC part" ) From 5c730de96e1cd1dd45f6bd46b79ebbaff3532e18 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 3 May 2024 14:52:28 +0300 Subject: [PATCH 1077/1943] [#1120] cli: Add EC support to `object nodes` command Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 220 ++++++++++++++++-------- 1 file changed, 152 insertions(+), 68 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 38b9dec82..b33cbf17f 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -32,11 +33,16 @@ const ( var errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint") -type objectNodesInfo struct { - containerID cid.ID - objectID oid.ID - relatedObjectIDs []oid.ID - isLockOrTombstone bool +type phyObject struct { + containerID cid.ID + objectID oid.ID + storedOnAllContainerNodes bool + ecHeader *ecHeader +} + +type ecHeader struct { + index uint32 + parent oid.ID } type boolError struct { @@ -49,7 +55,7 @@ var objectNodesCmd = &cobra.Command{ Short: "List of nodes where the object is stored", Long: `List of nodes where the object should be stored and where it is actually stored. Lock objects must exist on all nodes of the container. - For complex objects, a node is considered to store an object if the node stores at least one part of the complex object. + For complex and EC objects, a node is considered to store an object if the node stores at least one part of the complex object or one chunk of the EC object. By default, the actual storage of the object is checked only on the nodes that should store the object. To check all nodes, use the flag --verify-presence-all.`, Run: objectNodes, } @@ -76,18 +82,18 @@ func objectNodes(cmd *cobra.Command, _ []string) { pk := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - objectInfo := getObjectInfo(cmd, cnrID, objID, cli, pk) + objects := getPhyObjects(cmd, cnrID, objID, cli, pk) placementPolicy, netmap := getPlacementPolicyAndNetmap(cmd, cnrID, cli) - requiredPlacement := getRequiredPlacement(cmd, objectInfo, placementPolicy, netmap) + requiredPlacement := getRequiredPlacement(cmd, objects, placementPolicy, netmap) - actualPlacement := getActualPlacement(cmd, netmap, requiredPlacement, pk, objectInfo) + actualPlacement := getActualPlacement(cmd, netmap, requiredPlacement, pk, objects) printPlacement(cmd, netmap, requiredPlacement, actualPlacement) } -func getObjectInfo(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) *objectNodesInfo { +func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) []phyObject { var addrObj oid.Address addrObj.SetContainer(cnrID) addrObj.SetObject(objID) @@ -102,44 +108,101 @@ func getObjectInfo(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.C res, err := internalclient.HeadObject(cmd.Context(), prmHead) if err == nil { - return &objectNodesInfo{ - containerID: cnrID, - objectID: objID, - isLockOrTombstone: res.Header().Type() == objectSDK.TypeLock || res.Header().Type() == objectSDK.TypeTombstone, + obj := phyObject{ + containerID: cnrID, + objectID: objID, + storedOnAllContainerNodes: res.Header().Type() == objectSDK.TypeLock || res.Header().Type() == objectSDK.TypeTombstone, } + if res.Header().ECHeader() != nil { + obj.ecHeader = &ecHeader{ + index: res.Header().ECHeader().Index(), + parent: res.Header().ECHeader().Parent(), + } + } + return []phyObject{obj} } var errSplitInfo *objectSDK.SplitInfoError - - if !errors.As(err, &errSplitInfo) { - commonCmd.ExitOnErr(cmd, "failed to get object info: %w", err) - return nil + if errors.As(err, &errSplitInfo) { + return getComplexObjectParts(cmd, cnrID, objID, cli, prmHead, errSplitInfo) } + var ecInfoError *objectSDK.ECInfoError + if errors.As(err, &ecInfoError) { + return getECObjectChunks(cmd, cnrID, objID, ecInfoError) + } + commonCmd.ExitOnErr(cmd, "failed to get object info: %w", err) + return nil +} + +func getComplexObjectParts(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) []phyObject { + members := getCompexObjectMembers(cmd, cnrID, objID, cli, prmHead, errSplitInfo) + return flattenComplexMembersIfECContainer(cmd, cnrID, members, prmHead) +} + +func getCompexObjectMembers(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) []oid.ID { splitInfo := errSplitInfo.SplitInfo() if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnrID); ok { - return &objectNodesInfo{ - containerID: cnrID, - objectID: objID, - relatedObjectIDs: members, - } + return members } if members, ok := tryGetSplitMembersBySplitID(cmd, splitInfo, cli, cnrID); ok { - return &objectNodesInfo{ - containerID: cnrID, - objectID: objID, - relatedObjectIDs: members, - } + return members } - members := tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnrID, objID) - return &objectNodesInfo{ - containerID: cnrID, - objectID: objID, - relatedObjectIDs: members, + return tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnrID, objID) +} + +func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, members []oid.ID, prmHead internalclient.HeadObjectPrm) []phyObject { + result := make([]phyObject, 0, len(members)) + var addrObj oid.Address + addrObj.SetContainer(cnrID) + prmHead.SetRawFlag(true) // to get an error instead of whole object + for _, partObjID := range members { + addrObj.SetObject(partObjID) + prmHead.SetAddress(addrObj) + + _, err := internalclient.HeadObject(cmd.Context(), prmHead) + var ecInfoError *objectSDK.ECInfoError + if errors.As(err, &ecInfoError) { + chunks := getECObjectChunks(cmd, cnrID, partObjID, ecInfoError) + result = append(result, chunks...) + continue + } else if err == nil { // not EC object, so all members must be phy objects + for _, member := range members { + result = append(result, phyObject{ + containerID: cnrID, + objectID: member, + }) + } + break + } + commonCmd.ExitOnErr(cmd, "failed to read EC chunk of complex object: %w", err) } + return result +} + +func getECObjectChunks(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, errECInfo *objectSDK.ECInfoError) []phyObject { + ecInfo := errECInfo.ECInfo() + result := make([]phyObject, 0, len(ecInfo.Chunks)) + for _, ch := range ecInfo.Chunks { + var chID oid.ID + err := chID.ReadFromV2(ch.ID) + if err != nil { + commonCmd.ExitOnErr(cmd, "failed to read EC chunk ID %w", err) + return nil + } + result = append(result, phyObject{ + containerID: cnrID, + objectID: chID, + ecHeader: &ecHeader{ + index: ch.Index, + parent: objID, + }, + }) + } + return result } func getPlacementPolicyAndNetmap(cmd *cobra.Command, cnrID cid.ID, cli *client.Client) (placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) { @@ -184,29 +247,28 @@ func getNetMap(ctx context.Context, cli *client.Client) (*netmapSDK.NetMap, erro return &nm, nil } -func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) map[uint64]netmapSDK.NodeInfo { +func getRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) map[uint64]netmapSDK.NodeInfo { + if policy.IsECPlacement(placementPolicy) { + return getECRequiredPlacement(cmd, objects, placementPolicy, netmap) + } + return getReplicaRequiredPlacement(cmd, objects, placementPolicy, netmap) +} + +func getReplicaRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) map[uint64]netmapSDK.NodeInfo { nodes := make(map[uint64]netmapSDK.NodeInfo) placementBuilder := placement.NewNetworkMapBuilder(netmap) - placement, err := placementBuilder.BuildPlacement(objInfo.containerID, &objInfo.objectID, placementPolicy) - commonCmd.ExitOnErr(cmd, "failed to get required placement: %w", err) - for repIdx, rep := range placement { - numOfReplicas := placementPolicy.ReplicaDescriptor(repIdx).NumberOfObjects() - var nodeIdx uint32 - for _, n := range rep { - if !objInfo.isLockOrTombstone && nodeIdx == numOfReplicas { // lock and tombstone objects should be on all container nodes - break - } - nodes[n.Hash()] = n - nodeIdx++ - } - } - - for _, relatedObjID := range objInfo.relatedObjectIDs { - placement, err = placementBuilder.BuildPlacement(objInfo.containerID, &relatedObjID, placementPolicy) - commonCmd.ExitOnErr(cmd, "failed to get required placement for related object: %w", err) - for _, rep := range placement { + for _, object := range objects { + placement, err := placementBuilder.BuildPlacement(object.containerID, &object.objectID, placementPolicy) + commonCmd.ExitOnErr(cmd, "failed to get required placement for object: %w", err) + for repIdx, rep := range placement { + numOfReplicas := placementPolicy.ReplicaDescriptor(repIdx).NumberOfObjects() + var nodeIdx uint32 for _, n := range rep { + if !object.storedOnAllContainerNodes && nodeIdx == numOfReplicas { + break + } nodes[n.Hash()] = n + nodeIdx++ } } } @@ -214,8 +276,42 @@ func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placemen return nodes } +func getECRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) map[uint64]netmapSDK.NodeInfo { + nodes := make(map[uint64]netmapSDK.NodeInfo) + for _, object := range objects { + getECRequiredPlacementInternal(cmd, object, placementPolicy, netmap, nodes) + } + return nodes +} + +func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap, nodes map[uint64]netmapSDK.NodeInfo) { + placementObjectID := object.objectID + if object.ecHeader != nil { + placementObjectID = object.ecHeader.parent + } + placementBuilder := placement.NewNetworkMapBuilder(netmap) + placement, err := placementBuilder.BuildPlacement(object.containerID, &placementObjectID, placementPolicy) + commonCmd.ExitOnErr(cmd, "failed to get required placement: %w", err) + + for _, vector := range placement { + if object.storedOnAllContainerNodes { + for _, node := range vector { + nodes[node.Hash()] = node + } + continue + } + + if object.ecHeader != nil { + chunkIdx := int(object.ecHeader.index) + nodeIdx := chunkIdx % len(vector) + node := vector[nodeIdx] + nodes[node.Hash()] = node + } + } +} + func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, - pk *ecdsa.PrivateKey, objInfo *objectNodesInfo, + pk *ecdsa.PrivateKey, objects []phyObject, ) map[uint64]boolError { result := make(map[uint64]boolError) resultMtx := &sync.Mutex{} @@ -243,23 +339,11 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPl return nil } - eg.Go(func() error { - var v boolError - v.value, v.err = isObjectStoredOnNode(egCtx, cmd, objInfo.containerID, objInfo.objectID, cli, pk) - resultMtx.Lock() - defer resultMtx.Unlock() - if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) { - return nil - } - result[cand.Hash()] = v - return nil - }) - - for _, rObjID := range objInfo.relatedObjectIDs { - rObjID := rObjID + for _, object := range objects { + object := object eg.Go(func() error { var v boolError - v.value, v.err = isObjectStoredOnNode(egCtx, cmd, objInfo.containerID, rObjID, cli, pk) + v.value, v.err = isObjectStoredOnNode(egCtx, cmd, object.containerID, object.objectID, cli, pk) resultMtx.Lock() defer resultMtx.Unlock() if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) { From ada1b9f737dce80d05ddb15bde9eb8339ccce4fc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 3 May 2024 17:20:41 +0300 Subject: [PATCH 1078/1943] [#1120] objectSvc: Fix EC put placement Use parent object ID to compute placement. Fix too many copies saving. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/ec.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/services/object/put/ec.go b/pkg/services/object/put/ec.go index 74db3c31f..6237872fb 100644 --- a/pkg/services/object/put/ec.go +++ b/pkg/services/object/put/ec.go @@ -147,7 +147,7 @@ func (e *ecWriter) relayToContainerNode(ctx context.Context) error { } func (e *ecWriter) writeECPart(ctx context.Context, obj *objectSDK.Object) error { - t, err := placement.NewTraverser(e.placementOpts...) + t, err := placement.NewTraverser(append(e.placementOpts, placement.ForObject(obj.ECHeader().Parent()))...) if err != nil { return err } @@ -162,6 +162,7 @@ func (e *ecWriter) writeECPart(ctx context.Context, obj *objectSDK.Object) error eg.Go(func() error { return e.writePart(egCtx, obj, int(obj.ECHeader().Index()), nodes) }) + t.SubmitSuccess() } if err := eg.Wait(); err != nil { return errIncompletePut{ @@ -181,7 +182,8 @@ func (e *ecWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) er if err != nil { return err } - t, err := placement.NewTraverser(e.placementOpts...) + objID, _ := obj.ID() + t, err := placement.NewTraverser(append(e.placementOpts, placement.ForObject(objID))...) if err != nil { return err } @@ -198,6 +200,7 @@ func (e *ecWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) er eg.Go(func() error { return e.writePart(egCtx, parts[idx], idx, nodes) }) + t.SubmitSuccess() } } if err := eg.Wait(); err != nil { From 654384990cc67024586f4a577c2251a1d8e53812 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 6 May 2024 12:37:00 +0300 Subject: [PATCH 1079/1943] [#1120] cli: Fix `object nodes` for linking objects Do not use linking objects to get placement for complex object. Linking objects should be stored on all container nodes, also they are not required. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 11 +++++++---- cmd/frostfs-cli/modules/object/util.go | 10 ++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index b33cbf17f..c3bf239cc 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -109,9 +109,11 @@ func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.C res, err := internalclient.HeadObject(cmd.Context(), prmHead) if err == nil { obj := phyObject{ - containerID: cnrID, - objectID: objID, - storedOnAllContainerNodes: res.Header().Type() == objectSDK.TypeLock || res.Header().Type() == objectSDK.TypeTombstone, + containerID: cnrID, + objectID: objID, + storedOnAllContainerNodes: res.Header().Type() == objectSDK.TypeLock || + res.Header().Type() == objectSDK.TypeTombstone || + len(res.Header().Children()) > 0, } if res.Header().ECHeader() != nil { obj.ecHeader = &ecHeader{ @@ -143,7 +145,7 @@ func getComplexObjectParts(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli * func getCompexObjectMembers(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) []oid.ID { splitInfo := errSplitInfo.SplitInfo() - if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnrID); ok { + if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnrID, false); ok { return members } @@ -412,6 +414,7 @@ func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, if errors.As(err, ¬Found) || errors.As(err, &removed) { return false, nil } + cmd.Printf("failed to get object %s from client\n", objID.EncodeToString()) return false, err } diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index 37e9f74e0..c8625eb94 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -370,7 +370,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, splitInfo := errSplit.SplitInfo() - if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnr); ok { + if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnr, true); ok { return members } @@ -381,7 +381,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, return tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnr, obj) } -func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, prmHead internal.HeadObjectPrm, cnr cid.ID) ([]oid.ID, bool) { +func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, prmHead internal.HeadObjectPrm, cnr cid.ID, withLinking bool) ([]oid.ID, bool) { // collect split chain by the descending ease of operations (ease is evaluated heuristically). // If any approach fails, we don't try the next since we assume that it will fail too. @@ -402,8 +402,10 @@ func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *objectSDK. common.PrintVerbose(cmd, "Received split members from the linking object: %v", children) - // include linking object - return append(children, idLinking), true + if withLinking { + return append(children, idLinking), true + } + return children, true } // linking object is not required for From a45b548a6f28e9ccde71a3b422497e88cc467aa9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 6 May 2024 15:52:41 +0300 Subject: [PATCH 1080/1943] [#1120] cli: Add explain to `object nodes` Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 95 ++++++++++++++++++++----- 1 file changed, 77 insertions(+), 18 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index c3bf239cc..2e84cc6a5 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -29,6 +29,7 @@ import ( const ( verifyPresenceAllFlag = "verify-presence-all" + explainFlag = "explain" ) var errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint") @@ -50,6 +51,11 @@ type boolError struct { err error } +type objectPlacement struct { + requiredNodes []netmapSDK.NodeInfo + confirmedNodes []netmapSDK.NodeInfo +} + var objectNodesCmd = &cobra.Command{ Use: "nodes", Short: "List of nodes where the object is stored", @@ -71,7 +77,8 @@ func initObjectNodesCmd() { flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage) _ = objectGetCmd.MarkFlagRequired(commonflags.OIDFlag) - flags.Bool("verify-presence-all", false, "Verify the actual presence of the object on all netmap nodes") + flags.Bool(verifyPresenceAllFlag, false, "Verify the actual presence of the object on all netmap nodes") + flags.Bool(explainFlag, false, "Show detailed information about the object placement") } func objectNodes(cmd *cobra.Command, _ []string) { @@ -86,11 +93,11 @@ func objectNodes(cmd *cobra.Command, _ []string) { placementPolicy, netmap := getPlacementPolicyAndNetmap(cmd, cnrID, cli) - requiredPlacement := getRequiredPlacement(cmd, objects, placementPolicy, netmap) + requiredNodes, objectsPlacement := getRequiredPlacement(cmd, objects, placementPolicy, netmap) - actualPlacement := getActualPlacement(cmd, netmap, requiredPlacement, pk, objects) + actualPlacement := getActualPlacement(cmd, netmap, requiredNodes, pk, objects, objectsPlacement) - printPlacement(cmd, netmap, requiredPlacement, actualPlacement) + printPlacement(cmd, netmap, requiredNodes, actualPlacement, objID, objects, objectsPlacement) } func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) []phyObject { @@ -249,15 +256,16 @@ func getNetMap(ctx context.Context, cli *client.Client) (*netmapSDK.NetMap, erro return &nm, nil } -func getRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) map[uint64]netmapSDK.NodeInfo { +func getRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) (map[uint64]netmapSDK.NodeInfo, map[oid.ID]objectPlacement) { if policy.IsECPlacement(placementPolicy) { return getECRequiredPlacement(cmd, objects, placementPolicy, netmap) } return getReplicaRequiredPlacement(cmd, objects, placementPolicy, netmap) } -func getReplicaRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) map[uint64]netmapSDK.NodeInfo { +func getReplicaRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) (map[uint64]netmapSDK.NodeInfo, map[oid.ID]objectPlacement) { nodes := make(map[uint64]netmapSDK.NodeInfo) + objectsNodes := make(map[oid.ID]objectPlacement) placementBuilder := placement.NewNetworkMapBuilder(netmap) for _, object := range objects { placement, err := placementBuilder.BuildPlacement(object.containerID, &object.objectID, placementPolicy) @@ -270,23 +278,29 @@ func getReplicaRequiredPlacement(cmd *cobra.Command, objects []phyObject, placem break } nodes[n.Hash()] = n + + op := objectsNodes[object.objectID] + op.requiredNodes = append(op.requiredNodes, n) + objectsNodes[object.objectID] = op + nodeIdx++ } } } - return nodes + return nodes, objectsNodes } -func getECRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) map[uint64]netmapSDK.NodeInfo { +func getECRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) (map[uint64]netmapSDK.NodeInfo, map[oid.ID]objectPlacement) { nodes := make(map[uint64]netmapSDK.NodeInfo) + objectsNodes := make(map[oid.ID]objectPlacement) for _, object := range objects { - getECRequiredPlacementInternal(cmd, object, placementPolicy, netmap, nodes) + getECRequiredPlacementInternal(cmd, object, placementPolicy, netmap, nodes, objectsNodes) } - return nodes + return nodes, objectsNodes } -func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap, nodes map[uint64]netmapSDK.NodeInfo) { +func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap, nodes map[uint64]netmapSDK.NodeInfo, objectNodes map[oid.ID]objectPlacement) { placementObjectID := object.objectID if object.ecHeader != nil { placementObjectID = object.ecHeader.parent @@ -299,6 +313,10 @@ func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placem if object.storedOnAllContainerNodes { for _, node := range vector { nodes[node.Hash()] = node + + op := objectNodes[object.objectID] + op.requiredNodes = append(op.requiredNodes, node) + objectNodes[object.objectID] = op } continue } @@ -308,12 +326,16 @@ func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placem nodeIdx := chunkIdx % len(vector) node := vector[nodeIdx] nodes[node.Hash()] = node + + op := objectNodes[object.objectID] + op.requiredNodes = append(op.requiredNodes, node) + objectNodes[object.objectID] = op } } } func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, - pk *ecdsa.PrivateKey, objects []phyObject, + pk *ecdsa.PrivateKey, objects []phyObject, objectNodes map[oid.ID]objectPlacement, ) map[uint64]boolError { result := make(map[uint64]boolError) resultMtx := &sync.Mutex{} @@ -348,6 +370,11 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPl v.value, v.err = isObjectStoredOnNode(egCtx, cmd, object.containerID, object.objectID, cli, pk) resultMtx.Lock() defer resultMtx.Unlock() + if v.err == nil && v.value { + op := objectNodes[object.objectID] + op.confirmedNodes = append(op.confirmedNodes, cand) + objectNodes[object.objectID] = op + } if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) { return nil } @@ -418,12 +445,12 @@ func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, return false, err } -func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, actualPlacement map[uint64]boolError) { +func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, + actualPlacement map[uint64]boolError, objID oid.ID, objects []phyObject, objectNodes map[oid.ID]objectPlacement, +) { w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug) - defer func() { - commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", w.Flush()) - }() - fmt.Fprintln(w, "Node ID\tShould contain object\tActually contains object\t") + _, err := fmt.Fprintln(w, "Node ID\tShould contain object\tActually contains object\t") + commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", err) for _, n := range netmap.Nodes() { nodeID := hex.EncodeToString(n.PublicKey()) _, required := requiredPlacement[n.Hash()] @@ -436,6 +463,38 @@ func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacem actualStr = strconv.FormatBool(actual.value) } } - fmt.Fprintf(w, "%s\t%s\t%s\t\n", nodeID, strconv.FormatBool(required), actualStr) + _, err := fmt.Fprintf(w, "%s\t%s\t%s\t\n", nodeID, strconv.FormatBool(required), actualStr) + commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", err) + } + commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", w.Flush()) + + if explain, _ := cmd.Flags().GetBool(explainFlag); !explain { + return + } + + fmt.Fprintf(cmd.OutOrStdout(), "Object %s stores payload in %d data objects:\n", objID.EncodeToString(), len(objects)) + + for _, object := range objects { + fmt.Fprintf(cmd.OutOrStdout(), "- %s\n", object.objectID) + if object.ecHeader != nil { + fmt.Fprintf(cmd.OutOrStdout(), "\tEC index: %d\n", object.ecHeader.index) + fmt.Fprintf(cmd.OutOrStdout(), "\tEC parent: %s\n", object.ecHeader.parent.EncodeToString()) + } + op, ok := objectNodes[object.objectID] + if !ok { + continue + } + if len(op.requiredNodes) > 0 { + fmt.Fprintf(cmd.OutOrStdout(), "\tRequired nodes:\n") + for _, node := range op.requiredNodes { + fmt.Fprintf(cmd.OutOrStdout(), "\t\t- %s\n", hex.EncodeToString(node.PublicKey())) + } + } + if len(op.confirmedNodes) > 0 { + fmt.Fprintf(cmd.OutOrStdout(), "\tActual nodes:\n") + for _, node := range op.confirmedNodes { + fmt.Fprintf(cmd.OutOrStdout(), "\t\t- %s\n", hex.EncodeToString(node.PublicKey())) + } + } } } From 368218f0cca4d436ff786eb0bf8d9fd1c7a1b7ac Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 7 May 2024 12:12:34 +0300 Subject: [PATCH 1081/1943] [#1120] cli: Edit `object nodes` output Print detailed information only. Allow to output to JSON. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 238 +++++++++++++++--------- 1 file changed, 154 insertions(+), 84 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 2e84cc6a5..b5f839eb8 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -1,14 +1,16 @@ package object import ( + "bytes" + "cmp" "context" "crypto/ecdsa" "encoding/hex" + "encoding/json" "errors" "fmt" - "strconv" + "slices" "sync" - "text/tabwriter" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -29,7 +31,6 @@ import ( const ( verifyPresenceAllFlag = "verify-presence-all" - explainFlag = "explain" ) var errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint") @@ -46,16 +47,30 @@ type ecHeader struct { parent oid.ID } -type boolError struct { - value bool - err error -} - type objectPlacement struct { requiredNodes []netmapSDK.NodeInfo confirmedNodes []netmapSDK.NodeInfo } +type objectNodesResult struct { + errors []error + placements map[oid.ID]objectPlacement +} + +type ObjNodesDataObject struct { + ObjectID string `json:"object_id"` + RequiredNodes []string `json:"required_nodes,omitempty"` + ConfirmedNodes []string `json:"confirmed_nodes,omitempty"` + ECParentObjectID *string `json:"ec_parent_object_id,omitempty"` + ECIndex *uint32 `json:"ec_index,omitempty"` +} + +type objNodesResultJSON struct { + ObjectID string `json:"object_id"` + DataObjects []ObjNodesDataObject `json:"data_objects,omitempty"` + Errors []string `json:"errors,omitempty"` +} + var objectNodesCmd = &cobra.Command{ Use: "nodes", Short: "List of nodes where the object is stored", @@ -77,8 +92,8 @@ func initObjectNodesCmd() { flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage) _ = objectGetCmd.MarkFlagRequired(commonflags.OIDFlag) - flags.Bool(verifyPresenceAllFlag, false, "Verify the actual presence of the object on all netmap nodes") - flags.Bool(explainFlag, false, "Show detailed information about the object placement") + flags.Bool(verifyPresenceAllFlag, false, "Verify the actual presence of the object on all netmap nodes.") + flags.Bool(commonflags.JSON, false, "Print information about the object placement as json.") } func objectNodes(cmd *cobra.Command, _ []string) { @@ -93,11 +108,11 @@ func objectNodes(cmd *cobra.Command, _ []string) { placementPolicy, netmap := getPlacementPolicyAndNetmap(cmd, cnrID, cli) - requiredNodes, objectsPlacement := getRequiredPlacement(cmd, objects, placementPolicy, netmap) + result := getRequiredPlacement(cmd, objects, placementPolicy, netmap) - actualPlacement := getActualPlacement(cmd, netmap, requiredNodes, pk, objects, objectsPlacement) + getActualPlacement(cmd, netmap, pk, objects, result) - printPlacement(cmd, netmap, requiredNodes, actualPlacement, objID, objects, objectsPlacement) + printPlacement(cmd, objID, objects, result) } func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) []phyObject { @@ -256,16 +271,17 @@ func getNetMap(ctx context.Context, cli *client.Client) (*netmapSDK.NetMap, erro return &nm, nil } -func getRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) (map[uint64]netmapSDK.NodeInfo, map[oid.ID]objectPlacement) { +func getRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) *objectNodesResult { if policy.IsECPlacement(placementPolicy) { return getECRequiredPlacement(cmd, objects, placementPolicy, netmap) } return getReplicaRequiredPlacement(cmd, objects, placementPolicy, netmap) } -func getReplicaRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) (map[uint64]netmapSDK.NodeInfo, map[oid.ID]objectPlacement) { - nodes := make(map[uint64]netmapSDK.NodeInfo) - objectsNodes := make(map[oid.ID]objectPlacement) +func getReplicaRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) *objectNodesResult { + result := &objectNodesResult{ + placements: make(map[oid.ID]objectPlacement), + } placementBuilder := placement.NewNetworkMapBuilder(netmap) for _, object := range objects { placement, err := placementBuilder.BuildPlacement(object.containerID, &object.objectID, placementPolicy) @@ -277,30 +293,30 @@ func getReplicaRequiredPlacement(cmd *cobra.Command, objects []phyObject, placem if !object.storedOnAllContainerNodes && nodeIdx == numOfReplicas { break } - nodes[n.Hash()] = n - op := objectsNodes[object.objectID] + op := result.placements[object.objectID] op.requiredNodes = append(op.requiredNodes, n) - objectsNodes[object.objectID] = op + result.placements[object.objectID] = op nodeIdx++ } } } - return nodes, objectsNodes + return result } -func getECRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) (map[uint64]netmapSDK.NodeInfo, map[oid.ID]objectPlacement) { - nodes := make(map[uint64]netmapSDK.NodeInfo) - objectsNodes := make(map[oid.ID]objectPlacement) - for _, object := range objects { - getECRequiredPlacementInternal(cmd, object, placementPolicy, netmap, nodes, objectsNodes) +func getECRequiredPlacement(cmd *cobra.Command, objects []phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap) *objectNodesResult { + result := &objectNodesResult{ + placements: make(map[oid.ID]objectPlacement), } - return nodes, objectsNodes + for _, object := range objects { + getECRequiredPlacementInternal(cmd, object, placementPolicy, netmap, result) + } + return result } -func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap, nodes map[uint64]netmapSDK.NodeInfo, objectNodes map[oid.ID]objectPlacement) { +func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placementPolicy netmapSDK.PlacementPolicy, netmap *netmapSDK.NetMap, result *objectNodesResult) { placementObjectID := object.objectID if object.ecHeader != nil { placementObjectID = object.ecHeader.parent @@ -312,11 +328,9 @@ func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placem for _, vector := range placement { if object.storedOnAllContainerNodes { for _, node := range vector { - nodes[node.Hash()] = node - - op := objectNodes[object.objectID] + op := result.placements[object.objectID] op.requiredNodes = append(op.requiredNodes, node) - objectNodes[object.objectID] = op + result.placements[object.objectID] = op } continue } @@ -325,30 +339,18 @@ func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placem chunkIdx := int(object.ecHeader.index) nodeIdx := chunkIdx % len(vector) node := vector[nodeIdx] - nodes[node.Hash()] = node - op := objectNodes[object.objectID] + op := result.placements[object.objectID] op.requiredNodes = append(op.requiredNodes, node) - objectNodes[object.objectID] = op + result.placements[object.objectID] = op } } } -func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, - pk *ecdsa.PrivateKey, objects []phyObject, objectNodes map[oid.ID]objectPlacement, -) map[uint64]boolError { - result := make(map[uint64]boolError) +func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa.PrivateKey, objects []phyObject, result *objectNodesResult) { resultMtx := &sync.Mutex{} - var candidates []netmapSDK.NodeInfo - checkAllNodes, _ := cmd.Flags().GetBool(verifyPresenceAllFlag) - if checkAllNodes { - candidates = netmap.Nodes() - } else { - for _, n := range requiredPlacement { - candidates = append(candidates, n) - } - } + candidates := getNodesToCheckObjectExistance(cmd, netmap, result) eg, egCtx := errgroup.WithContext(cmd.Context()) for _, cand := range candidates { @@ -359,26 +361,24 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPl if err != nil { resultMtx.Lock() defer resultMtx.Unlock() - result[cand.Hash()] = boolError{err: err} + result.errors = append(result.errors, fmt.Errorf("failed to connect to node %s: %w", hex.EncodeToString(cand.PublicKey()), err)) return nil } for _, object := range objects { object := object eg.Go(func() error { - var v boolError - v.value, v.err = isObjectStoredOnNode(egCtx, cmd, object.containerID, object.objectID, cli, pk) + stored, err := isObjectStoredOnNode(egCtx, cmd, object.containerID, object.objectID, cli, pk) resultMtx.Lock() defer resultMtx.Unlock() - if v.err == nil && v.value { - op := objectNodes[object.objectID] + if err == nil && stored { + op := result.placements[object.objectID] op.confirmedNodes = append(op.confirmedNodes, cand) - objectNodes[object.objectID] = op + result.placements[object.objectID] = op } - if prev, exists := result[cand.Hash()]; exists && (prev.err != nil || prev.value) { - return nil + if err != nil { + result.errors = append(result.errors, fmt.Errorf("failed to check object %s existence on node %s: %w", object.objectID.EncodeToString(), hex.EncodeToString(cand.PublicKey()), err)) } - result[cand.Hash()] = v return nil }) } @@ -387,7 +387,24 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPl } commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", eg.Wait()) - return result +} + +func getNodesToCheckObjectExistance(cmd *cobra.Command, netmap *netmapSDK.NetMap, result *objectNodesResult) []netmapSDK.NodeInfo { + checkAllNodes, _ := cmd.Flags().GetBool(verifyPresenceAllFlag) + if checkAllNodes { + return netmap.Nodes() + } + var nodes []netmapSDK.NodeInfo + visited := make(map[uint64]struct{}) + for _, p := range result.placements { + for _, node := range p.requiredNodes { + if _, ok := visited[node.Hash()]; !ok { + nodes = append(nodes, node) + visited[node.Hash()] = struct{}{} + } + } + } + return nodes } func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.NodeInfo, pk *ecdsa.PrivateKey) (*client.Client, error) { @@ -445,33 +462,44 @@ func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, return false, err } -func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo, - actualPlacement map[uint64]boolError, objID oid.ID, objects []phyObject, objectNodes map[oid.ID]objectPlacement, -) { - w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug) - _, err := fmt.Fprintln(w, "Node ID\tShould contain object\tActually contains object\t") - commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", err) - for _, n := range netmap.Nodes() { - nodeID := hex.EncodeToString(n.PublicKey()) - _, required := requiredPlacement[n.Hash()] - actual, actualExists := actualPlacement[n.Hash()] - actualStr := "" - if actualExists { - if actual.err != nil { - actualStr = fmt.Sprintf("error: %v", actual.err) - } else { - actualStr = strconv.FormatBool(actual.value) - } +func printPlacement(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) { + normilizeObjectNodesResult(objects, result) + if json, _ := cmd.Flags().GetBool(commonflags.JSON); json { + printObjectNodesAsJSON(cmd, objID, objects, result) + } else { + printObjectNodesAsText(cmd, objID, objects, result) + } +} + +func normilizeObjectNodesResult(objects []phyObject, result *objectNodesResult) { + slices.SortFunc(objects, func(lhs, rhs phyObject) int { + if lhs.ecHeader == nil && rhs.ecHeader == nil { + return bytes.Compare(lhs.objectID[:], rhs.objectID[:]) } - _, err := fmt.Fprintf(w, "%s\t%s\t%s\t\n", nodeID, strconv.FormatBool(required), actualStr) - commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", err) - } - commonCmd.ExitOnErr(cmd, "failed to print placement info: %w", w.Flush()) - - if explain, _ := cmd.Flags().GetBool(explainFlag); !explain { - return + if lhs.ecHeader == nil { + return -1 + } + if rhs.ecHeader == nil { + return 1 + } + if lhs.ecHeader.parent == rhs.ecHeader.parent { + return cmp.Compare(lhs.ecHeader.index, rhs.ecHeader.index) + } + return bytes.Compare(lhs.ecHeader.parent[:], rhs.ecHeader.parent[:]) + }) + for _, obj := range objects { + op := result.placements[obj.objectID] + slices.SortFunc(op.confirmedNodes, func(lhs, rhs netmapSDK.NodeInfo) int { + return bytes.Compare(lhs.PublicKey(), rhs.PublicKey()) + }) + slices.SortFunc(op.requiredNodes, func(lhs, rhs netmapSDK.NodeInfo) int { + return bytes.Compare(lhs.PublicKey(), rhs.PublicKey()) + }) + result.placements[obj.objectID] = op } +} +func printObjectNodesAsText(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) { fmt.Fprintf(cmd.OutOrStdout(), "Object %s stores payload in %d data objects:\n", objID.EncodeToString(), len(objects)) for _, object := range objects { @@ -480,7 +508,7 @@ func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacem fmt.Fprintf(cmd.OutOrStdout(), "\tEC index: %d\n", object.ecHeader.index) fmt.Fprintf(cmd.OutOrStdout(), "\tEC parent: %s\n", object.ecHeader.parent.EncodeToString()) } - op, ok := objectNodes[object.objectID] + op, ok := result.placements[object.objectID] if !ok { continue } @@ -491,10 +519,52 @@ func printPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacem } } if len(op.confirmedNodes) > 0 { - fmt.Fprintf(cmd.OutOrStdout(), "\tActual nodes:\n") + fmt.Fprintf(cmd.OutOrStdout(), "\tConfirmed nodes:\n") for _, node := range op.confirmedNodes { fmt.Fprintf(cmd.OutOrStdout(), "\t\t- %s\n", hex.EncodeToString(node.PublicKey())) } } } + + if len(result.errors) == 0 { + return + } + fmt.Fprintf(cmd.OutOrStdout(), "Errors:\n") + for _, err := range result.errors { + fmt.Fprintf(cmd.OutOrStdout(), "\t%s\n", err.Error()) + } +} + +func printObjectNodesAsJSON(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) { + jsonResult := &objNodesResultJSON{ + ObjectID: objID.EncodeToString(), + } + + for _, object := range objects { + do := ObjNodesDataObject{ + ObjectID: object.objectID.EncodeToString(), + } + if object.ecHeader != nil { + do.ECIndex = &object.ecHeader.index + ecParent := object.ecHeader.parent.EncodeToString() + do.ECParentObjectID = &ecParent + } + op, ok := result.placements[object.objectID] + if !ok { + continue + } + for _, rn := range op.requiredNodes { + do.RequiredNodes = append(do.RequiredNodes, hex.EncodeToString(rn.PublicKey())) + } + for _, cn := range op.confirmedNodes { + do.ConfirmedNodes = append(do.ConfirmedNodes, hex.EncodeToString(cn.PublicKey())) + } + jsonResult.DataObjects = append(jsonResult.DataObjects, do) + } + for _, err := range result.errors { + jsonResult.Errors = append(jsonResult.Errors, err.Error()) + } + b, err := json.Marshal(jsonResult) + commonCmd.ExitOnErr(cmd, "failed to marshal json: %w", err) + cmd.Println(string(b)) } From 0144117cc97f1221f7f018c84c0203b5a5ba9400 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 8 May 2024 10:02:54 +0300 Subject: [PATCH 1082/1943] [#1125] objectSvc: Add EC header APE check Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 5 +- pkg/core/object/sender_classifier.go | 10 +- pkg/services/object/ape/checker.go | 22 +-- pkg/services/object/ape/checker_test.go | 169 +++++++++++++++++++++++- pkg/services/object/ape/request.go | 66 ++++++++- pkg/services/object/ape/service.go | 23 +++- 6 files changed, 272 insertions(+), 23 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index a7a084fd1..833daf628 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -445,8 +445,11 @@ func createAPEService(c *cfg, splitSvc *objectService.TransportSplitter) *object c.log, objectAPE.NewChecker( c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.chainRouter, - objectAPE.NewStorageEngineHeaderProvider(c.cfgObject.cfgLocalStorage.localStorage), + objectAPE.NewStorageEngineHeaderProvider(c.cfgObject.cfgLocalStorage.localStorage, c.cfgObject.getSvc), c.shared.frostfsidClient, + c.netMapSource, + c.cfgObject.cnrSource, + c.binPublicKey, ), splitSvc, ) diff --git a/pkg/core/object/sender_classifier.go b/pkg/core/object/sender_classifier.go index ac881431e..13d0ebfb1 100644 --- a/pkg/core/object/sender_classifier.go +++ b/pkg/core/object/sender_classifier.go @@ -122,7 +122,7 @@ func (c SenderClassifier) isContainerKey( return false, err } - in, err := lookupKeyInContainer(nm, owner, idCnr, cnr) + in, err := LookupKeyInContainer(nm, owner, idCnr, cnr) if err != nil { return false, err } else if in { @@ -136,12 +136,12 @@ func (c SenderClassifier) isContainerKey( return false, err } - return lookupKeyInContainer(nm, owner, idCnr, cnr) + return LookupKeyInContainer(nm, owner, idCnr, cnr) } -func lookupKeyInContainer( +func LookupKeyInContainer( nm *netmap.NetMap, - owner, idCnr []byte, + pkey, idCnr []byte, cnr container.Container, ) (bool, error) { cnrVectors, err := nm.ContainerNodes(cnr.PlacementPolicy(), idCnr) @@ -151,7 +151,7 @@ func lookupKeyInContainer( for i := range cnrVectors { for j := range cnrVectors[i] { - if bytes.Equal(cnrVectors[i][j].PublicKey(), owner) { + if bytes.Equal(cnrVectors[i][j].PublicKey(), pkey) { return true, nil } } diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index e74b05379..c39f922f7 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -6,7 +6,9 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -17,20 +19,22 @@ import ( ) type checkerImpl struct { - chainRouter policyengine.ChainRouter - - headerProvider HeaderProvider - + chainRouter policyengine.ChainRouter + headerProvider HeaderProvider frostFSIDClient frostfsidcore.SubjectProvider + nm netmap.Source + cnrSource container.Source + nodePK []byte } -func NewChecker(chainRouter policyengine.ChainRouter, headerProvider HeaderProvider, frostFSIDClient frostfsidcore.SubjectProvider) Checker { +func NewChecker(chainRouter policyengine.ChainRouter, headerProvider HeaderProvider, frostFSIDClient frostfsidcore.SubjectProvider, nm netmap.Source, cnrSource container.Source, nodePK []byte) Checker { return &checkerImpl{ - chainRouter: chainRouter, - - headerProvider: headerProvider, - + chainRouter: chainRouter, + headerProvider: headerProvider, frostFSIDClient: frostFSIDClient, + nm: nm, + cnrSource: cnrSource, + nodePK: nodePK, } } diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index c591bc494..b5f064428 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -3,15 +3,22 @@ package ape import ( "context" "encoding/hex" + "errors" "fmt" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -36,7 +43,7 @@ func (h *headerProviderMock) addHeader(c cid.ID, o oid.ID, header *objectSDK.Obj h.m[addr] = header } -func (h *headerProviderMock) GetHeader(_ context.Context, c cid.ID, o oid.ID) (*objectSDK.Object, error) { +func (h *headerProviderMock) GetHeader(_ context.Context, c cid.ID, o oid.ID, _ bool) (*objectSDK.Object, error) { var addr oid.Address addr.SetContainer(c) addr.SetObject(o) @@ -404,7 +411,7 @@ func TestAPECheck(t *testing.T) { router := policyengine.NewDefaultChainRouterWithLocalOverrides(ms, ls) - checker := NewChecker(router, headerProvider, frostfsidProvider) + checker := NewChecker(router, headerProvider, frostfsidProvider, nil, nil, nil) prm := Prm{ Method: method, @@ -436,3 +443,161 @@ func TestAPECheck(t *testing.T) { }) } } + +type netmapStub struct { + netmaps map[uint64]*netmapSDK.NetMap + currentEpoch uint64 +} + +func (s *netmapStub) GetNetMap(diff uint64) (*netmapSDK.NetMap, error) { + if diff >= s.currentEpoch { + return nil, errors.New("invalid diff") + } + return s.GetNetMapByEpoch(s.currentEpoch - diff) +} + +func (s *netmapStub) GetNetMapByEpoch(epoch uint64) (*netmapSDK.NetMap, error) { + if nm, found := s.netmaps[epoch]; found { + return nm, nil + } + return nil, errors.New("netmap not found") +} + +func (s *netmapStub) Epoch() (uint64, error) { + return s.currentEpoch, nil +} + +type testContainerSource struct { + containers map[cid.ID]*container.Container +} + +func (s *testContainerSource) Get(cnrID cid.ID) (*container.Container, error) { + if cnr, found := s.containers[cnrID]; found { + return cnr, nil + } + return nil, fmt.Errorf("container not found") +} + +func (s *testContainerSource) DeletionInfo(cid.ID) (*container.DelInfo, error) { + return nil, nil +} + +func TestPutECChunk(t *testing.T) { + headerProvider := newHeaderProviderMock() + frostfsidProvider := newFrostfsIDProviderMock(t) + + cnr := newContainerIDSDK(t, containerID) + obj := newObjectIDSDK(t, &objectID) + + ls := inmemory.NewInmemoryLocalStorage() + ms := inmemory.NewInmemoryMorphRuleChainStorage() + + ls.AddOverride(chain.Ingress, policyengine.ContainerTarget(containerID), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{Names: methodsOptionalOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + Any: true, + Condition: []chain.Condition{ + { + Op: chain.CondStringEquals, + Object: chain.ObjectResource, + Key: "attr1", + Value: "value", + }, + }, + }, + }, + MatchType: chain.MatchTypeFirstMatch, + }) + + router := policyengine.NewDefaultChainRouterWithLocalOverrides(ms, ls) + + node1Key, err := keys.NewPrivateKey() + require.NoError(t, err) + node1 := netmapSDK.NodeInfo{} + node1.SetPublicKey(node1Key.PublicKey().Bytes()) + netmap := &netmapSDK.NetMap{} + netmap.SetEpoch(100) + netmap.SetNodes([]netmapSDK.NodeInfo{node1}) + + nm := &netmapStub{ + currentEpoch: 100, + netmaps: map[uint64]*netmapSDK.NetMap{ + 100: netmap, + }, + } + + cont := containerSDK.Container{} + cont.Init() + pp := netmapSDK.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + cont.SetPlacementPolicy(pp) + cs := &testContainerSource{ + containers: map[cid.ID]*container.Container{ + cnr: { + Value: cont, + }, + }, + } + + checker := NewChecker(router, headerProvider, frostfsidProvider, nm, cs, node1Key.PublicKey().Bytes()) + + ecParentID := oidtest.ID() + chunkHeader := newHeaderObjectSDK(cnr, obj, nil).ToV2().GetHeader() + ecHeader := object.ECHeader{ + Index: 1, + Total: 5, + Parent: &refs.ObjectID{}, + } + chunkHeader.SetEC(&ecHeader) + ecParentID.WriteToV2(ecHeader.Parent) + + parentHeader := newHeaderObjectSDK(cnr, &ecParentID, &headerObjectSDKParams{ + attributes: []struct { + key string + val string + }{ + { + key: "attr1", + val: "value", + }, + }, + }) + headerProvider.addHeader(cnr, ecParentID, parentHeader) + + t.Run("access denied for container node", func(t *testing.T) { + prm := Prm{ + Method: nativeschema.MethodPutObject, + Container: cnr, + Object: obj, + Role: role, + SenderKey: senderKey, + Header: chunkHeader, + SoftAPECheck: true, + } + + err = checker.CheckAPE(context.Background(), prm) + require.Error(t, err) + }) + t.Run("access allowed for non container node", func(t *testing.T) { + otherKey, err := keys.NewPrivateKey() + require.NoError(t, err) + checker = NewChecker(router, headerProvider, frostfsidProvider, nm, cs, otherKey.PublicKey().Bytes()) + prm := Prm{ + Method: nativeschema.MethodPutObject, + Container: cnr, + Object: obj, + Role: nativeschema.PropertyValueContainerRoleOthers, + SenderKey: senderKey, + Header: chunkHeader, + SoftAPECheck: true, + } + + err = checker.CheckAPE(context.Background(), prm) + require.NoError(t, err) + }) +} diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index cbecc73e3..71a9aec2c 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -2,11 +2,14 @@ package ape import ( "context" + "crypto/sha256" "fmt" "strconv" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -110,12 +113,12 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re if prm.Header != nil { header = prm.Header } else if prm.Object != nil && !prm.WithoutHeaderRequest { - headerObjSDK, err := c.headerProvider.GetHeader(ctx, prm.Container, *prm.Object) + headerObjSDK, err := c.headerProvider.GetHeader(ctx, prm.Container, *prm.Object, true) if err == nil { header = headerObjSDK.ToV2().GetHeader() } } - + header = c.fillHeaderWithECParent(ctx, prm, header) reqProps := map[string]string{ nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, nativeschema.PropertyKeyActorRole: prm.Role, @@ -136,6 +139,65 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re ), nil } +func (c *checkerImpl) fillHeaderWithECParent(ctx context.Context, prm Prm, header *objectV2.Header) *objectV2.Header { + if header == nil { + return header + } + if header.GetEC() == nil { + return header + } + if prm.Role == nativeschema.PropertyValueContainerRoleContainer || + prm.Role == nativeschema.PropertyValueContainerRoleIR { + return header + } + parentObjRefID := header.GetEC().Parent + if parentObjRefID == nil { + return header + } + var parentObjID oid.ID + if err := parentObjID.ReadFromV2(*parentObjRefID); err != nil { + return header + } + // only container node have access to collect parent object + contNode, err := c.currentNodeIsContainerNode(prm.Container) + if err != nil || !contNode { + return header + } + parentObj, err := c.headerProvider.GetHeader(ctx, prm.Container, parentObjID, false) + if err != nil { + return header + } + return parentObj.ToV2().GetHeader() +} + +func (c *checkerImpl) currentNodeIsContainerNode(cnrID cid.ID) (bool, error) { + cnr, err := c.cnrSource.Get(cnrID) + if err != nil { + return false, err + } + + nm, err := netmap.GetLatestNetworkMap(c.nm) + if err != nil { + return false, err + } + idCnr := make([]byte, sha256.Size) + cnrID.Encode(idCnr) + + in, err := object.LookupKeyInContainer(nm, c.nodePK, idCnr, cnr.Value) + if err != nil { + return false, err + } else if in { + return true, nil + } + + nm, err = netmap.GetPreviousNetworkMap(c.nm) + if err != nil { + return false, err + } + + return object.LookupKeyInContainer(nm, c.nodePK, idCnr, cnr.Value) +} + // fillWithUserClaimTags fills ape request properties with user claim tags getting them from frostfsid contract by actor public key. func (c *checkerImpl) fillWithUserClaimTags(reqProps map[string]string, prm Prm) (map[string]string, error) { if reqProps == nil { diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 95f36be79..63c2a9909 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -10,6 +10,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" + getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -31,23 +33,36 @@ type Service struct { var _ objectSvc.ServiceServer = (*Service)(nil) type HeaderProvider interface { - GetHeader(ctx context.Context, cnr cid.ID, oid oid.ID) (*objectSDK.Object, error) + GetHeader(ctx context.Context, cnr cid.ID, oid oid.ID, local bool) (*objectSDK.Object, error) } type storageEngineHeaderProvider struct { storageEngine *engine.StorageEngine + getSvc *getsvc.Service } -func (p storageEngineHeaderProvider) GetHeader(ctx context.Context, cnr cid.ID, objID oid.ID) (*objectSDK.Object, error) { +func (p storageEngineHeaderProvider) GetHeader(ctx context.Context, cnr cid.ID, objID oid.ID, local bool) (*objectSDK.Object, error) { var addr oid.Address addr.SetContainer(cnr) addr.SetObject(objID) - return engine.Head(ctx, p.storageEngine, addr) + if local { + return engine.Head(ctx, p.storageEngine, addr) + } + w := getsvc.NewSimpleObjectWriter() + var headPrm getsvc.HeadPrm + headPrm.WithAddress(addr) + headPrm.SetHeaderWriter(w) + headPrm.SetCommonParameters(&util.CommonPrm{}) // default values are ok + if err := p.getSvc.Head(ctx, headPrm); err != nil { + return nil, err + } + return w.Object(), nil } -func NewStorageEngineHeaderProvider(e *engine.StorageEngine) HeaderProvider { +func NewStorageEngineHeaderProvider(e *engine.StorageEngine, s *getsvc.Service) HeaderProvider { return storageEngineHeaderProvider{ storageEngine: e, + getSvc: s, } } From 74135776c744ef10da22c53cf57107b035b686a2 Mon Sep 17 00:00:00 2001 From: Anoke Date: Wed, 8 May 2024 01:08:50 +0300 Subject: [PATCH 1083/1943] [#1067] adm: Fix panic on negative value Signed-off-by: Anoke --- cmd/frostfs-adm/internal/modules/morph/policy/policy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go index a11ca2704..5ab05e429 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go @@ -48,7 +48,7 @@ func SetPolicyCmd(cmd *cobra.Command, args []string) error { value, err := strconv.ParseUint(v, 10, 32) if err != nil { - return fmt.Errorf("can't parse parameter value '%s': %w", args[1], err) + return fmt.Errorf("can't parse parameter value '%s': %w", args[i], err) } emit.AppCall(bw.BinWriter, policy.Hash, "set"+k, callflag.All, int64(value)) From 20baf6e112e8e91a06a372c78e0f4d5604f2b282 Mon Sep 17 00:00:00 2001 From: aarifullin Date: Fri, 26 Apr 2024 17:27:56 +0300 Subject: [PATCH 1084/1943] [#1108] ape: Update policy-engine version for listing by iteration * Update go.mod with a new version of policy-engine pacakge. * Adapt SwitchRPCGuardedActor to ContractStorage interface. * Fix `frostfs-adm` util. Signed-off-by: Airat Arifullin --- .../internal/modules/morph/ape/ape_util.go | 17 ++++++++++++++++- .../internal/modules/morph/helper/actor.go | 18 ++++++++++++------ go.mod | 4 ++-- go.sum | 8 ++++---- pkg/morph/client/actor.go | 10 ++++++++++ pkg/morph/client/client.go | 7 +++++++ 6 files changed, 51 insertions(+), 13 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 42307e78f..dfa7c6392 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -98,6 +98,16 @@ func parseChainName(cmd *cobra.Command) apechain.Name { return apeChainName } +// invokerAdapter adapats invoker.Invoker to ContractStorageInvoker interface. +type invokerAdapter struct { + *invoker.Invoker + rpcActor invoker.RPCInvoke +} + +func (n *invokerAdapter) GetRPCInvoker() invoker.RPCInvoke { + return n.rpcActor +} + func newPolicyContractReaderInterface(cmd *cobra.Command) (*morph.ContractStorageReader, *invoker.Invoker) { c, err := helper.GetN3Client(viper.GetViper()) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) @@ -111,7 +121,12 @@ func newPolicyContractReaderInterface(cmd *cobra.Command) (*morph.ContractStorag ch, err = helper.NNSResolveHash(inv, nnsCs.Hash, helper.DomainOf(constants.PolicyContract)) commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err) - return morph.NewContractStorageReader(inv, ch), inv + invokerAdapter := &invokerAdapter{ + Invoker: inv, + rpcActor: c, + } + + return morph.NewContractStorageReader(invokerAdapter, ch), inv } func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *helper.LocalActor) { diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/actor.go b/cmd/frostfs-adm/internal/modules/morph/helper/actor.go index f920aa5ba..1ca246f9f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/actor.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/actor.go @@ -23,9 +23,10 @@ import ( // LocalActor is a kludge, do not use it outside of the morph commands. type LocalActor struct { - neoActor *actor.Actor - accounts []*wallet.Account - Invoker *invoker.Invoker + neoActor *actor.Actor + accounts []*wallet.Account + Invoker *invoker.Invoker + rpcInvoker invoker.RPCInvoke } // NewLocalActor create LocalActor with accounts form provided wallets. @@ -68,9 +69,10 @@ func NewLocalActor(cmd *cobra.Command, c actor.RPCActor) (*LocalActor, error) { } } return &LocalActor{ - neoActor: act, - accounts: accounts, - Invoker: &act.Invoker, + neoActor: act, + accounts: accounts, + Invoker: &act.Invoker, + rpcInvoker: c, }, nil } @@ -167,3 +169,7 @@ func (a *LocalActor) MakeUnsignedRun(_ []byte, _ []transaction.Attribute) (*tran func (a *LocalActor) MakeCall(_ util.Uint160, _ string, _ ...any) (*transaction.Transaction, error) { panic("unimplemented") } + +func (a *LocalActor) GetRPCInvoker() invoker.RPCInvoke { + return a.rpcInvoker +} diff --git a/go.mod b/go.mod index 08a59581d..142ce02cc 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,11 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240427200446-67c6f305b21f - git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0 + git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240507063414-99e02858af12 git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240426062043-c5397286410f git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 6200022c7..241e25d8f 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240427200446-67c6f305b21f h1:YyjsQNtrngQzIKOUtApXoi5r5pewatM+cXfpY19vZWo= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240427200446-67c6f305b21f/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0 h1:FzurjElUwC7InY9v5rzXReKbfBL5yRJKSWJPq6BKhH0= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.0/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 h1:Tp4I+XOLp3VCJORfxSamQtj3RZNISbaLM4WD5iIzXxg= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= @@ -12,8 +12,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240507063414-99e02858af12 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240507063414-99e02858af12/go.mod h1:e7H9nNFpx1Tj3R20Zoxy0Vo6Srlb6zV5L7ZQXqg9rn4= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a h1:wbndKvHbwDQiSMQWL75RxiTZCeUyCi7NUj1lsfdAGkc= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a/go.mod h1:H/AW85RtYxVTbcgwHW76DqXeKlsiCIOeNXHPqyDBrfQ= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240426062043-c5397286410f h1:z+AqVpjWIZVh91eIt+lBTK1AwWtj2EBv+YE2PJKvvuk= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240426062043-c5397286410f/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/pkg/morph/client/actor.go b/pkg/morph/client/actor.go index 8c283a672..b6718dea5 100644 --- a/pkg/morph/client/actor.go +++ b/pkg/morph/client/actor.go @@ -6,12 +6,14 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" ) type actorProvider interface { GetActor() *actor.Actor + GetRPCActor() actor.RPCActor } // Client switches an established connection with neo-go if it is broken. @@ -132,3 +134,11 @@ func (a *SwitchRPCGuardedActor) TerminateSession(sessionID uuid.UUID) error { func (a *SwitchRPCGuardedActor) TraverseIterator(sessionID uuid.UUID, iterator *result.Iterator, num int) ([]stackitem.Item, error) { return a.actorProvider.GetActor().TraverseIterator(sessionID, iterator, num) } + +func (a *SwitchRPCGuardedActor) GetRPCActor() actor.RPCActor { + return a.actorProvider.GetRPCActor() +} + +func (a *SwitchRPCGuardedActor) GetRPCInvoker() invoker.RPCInvoke { + return a.actorProvider.GetRPCActor() +} diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 2570bc2c9..c9f819f04 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -579,3 +579,10 @@ func (c *Client) GetActor() *actor.Actor { return c.rpcActor } + +func (c *Client) GetRPCActor() actor.RPCActor { + c.switchLock.RLock() + defer c.switchLock.RUnlock() + + return c.client +} From 952d13cd2b078dad97bdd38a71896eefe9de0d99 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 14 May 2024 12:23:26 +0300 Subject: [PATCH 1085/1943] [#1124] cli: Improve APE rule parsing * Make APE rule parser to read condition's kind in unambiguous using lexemes `ResourceCondition`, `RequestCondition` instead confusing `Object.Request`, `Object.Resource`. * Fix unit-tests. Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/control/add_rule.go | 2 +- cmd/frostfs-cli/modules/util/ape.go | 58 ++++----- cmd/frostfs-cli/modules/util/ape_test.go | 60 ++++----- go.mod | 2 +- go.sum | 4 +- internal/ape/converter.go | 12 +- pkg/services/container/ape_test.go | 128 ++++++++++---------- pkg/services/object/ape/checker_test.go | 32 ++--- 8 files changed, 151 insertions(+), 147 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index 45b36bfb1..a22d0525d 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -24,7 +24,7 @@ var addRuleCmd = &cobra.Command{ Long: "Add local APE rule to a node with following format:\n[:action_detail] [ ...] ", Example: `control add-rule --endpoint ... -w ... --address ... --chain-id ChainID --cid ... --rule "allow Object.Get *" --rule "deny Object.Get EbxzAdz5LB4uqxuz6crWKAumBNtZyK2rKsqQP7TdZvwr/*" ---rule "deny:QuotaLimitReached Object.Put Object.Resource:Department=HR *" +--rule "deny:QuotaLimitReached Object.Put ResourceCondition:Department=HR *" control add-rule --endpoint ... -w ... --address ... --chain-id ChainID --cid ... --path some_chain.json `, diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index d2dd0ced2..50253b366 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -21,7 +21,7 @@ var ( errUnknownAction = errors.New("action is not recognized") errUnknownBinaryOperator = errors.New("binary operator is not recognized") errUnknownCondObjectType = errors.New("condition object type is not recognized") - errMixedTypesInRule = errors.New("found mixed type of actions and conditions in rule") + errMixedTypesInRule = errors.New("found mixed type of actions in rule") errNoActionsInRule = errors.New("there are no actions in rule") errUnsupportedResourceFormat = errors.New("unsupported resource format") errFailedToParseAllAny = errors.New("any/all is not parsed") @@ -38,10 +38,10 @@ func PrintHumanReadableAPEChain(cmd *cobra.Command, chain *apechain.Chain) { cmd.Println("\tConditions:") for _, c := range rule.Condition { var ot string - switch c.Object { - case apechain.ObjectResource: + switch c.Kind { + case apechain.KindResource: ot = "Resource" - case apechain.ObjectRequest: + case apechain.KindRequest: ot = "Request" default: panic("unknown object type") @@ -100,9 +100,9 @@ func ParseAPEChain(chain *apechain.Chain, rules []string) error { // deny Object.Put * // deny:QuotaLimitReached Object.Put * // allow Object.Put * -// allow Object.Get Object.Resource:Department=HR Object.Request:Actor=ownerA * -// allow Object.Get any Object.Resource:Department=HR Object.Request:Actor=ownerA * -// allow Object.Get all Object.Resource:Department=HR Object.Request:Actor=ownerA * +// allow Object.Get ResourceCondition:Department=HR RequestCondition:Actor=ownerA * +// allow Object.Get any ResourceCondition:Department=HR RequestCondition:Actor=ownerA * +// allow Object.Get all ResourceCondition:Department=HR RequestCondition:Actor=ownerA * // allow Object.* * // allow Container.* * // @@ -138,7 +138,9 @@ func parseRuleLexemes(r *apechain.Rule, lexemes []string) error { return err } - var isObject *bool + var objectTargeted bool + var containerTargeted bool + for i, lexeme := range lexemes[1:] { anyExpr, anyErr := parseAnyAll(lexeme) if anyErr == nil { @@ -156,23 +158,30 @@ func parseRuleLexemes(r *apechain.Rule, lexemes []string) error { lexemes = lexemes[i+1:] break } - actionType = condition.Object == apechain.ObjectResource || condition.Object == apechain.ObjectRequest r.Condition = append(r.Condition, *condition) } else { + if actionType { + objectTargeted = true + } else { + containerTargeted = true + } + if objectTargeted && containerTargeted { + // Actually, APE chain allows to define rules for several resources, for example, if + // chain target is namespace, but the parser primitevly compiles verbs, + // conditions and resources in one rule. So, for the parser, one rule relates only to + // one resource type - object or container. + return errMixedTypesInRule + } + r.Actions.Names = append(r.Actions.Names, names...) } - if isObject == nil { - isObject = &actionType - } else if actionType != *isObject { - return errMixedTypesInRule - } } r.Actions.Names = unique(r.Actions.Names) if len(r.Actions.Names) == 0 { return fmt.Errorf("%w:%w", err, errNoActionsInRule) } for _, lexeme := range lexemes { - resource, errRes := parseResource(lexeme, *isObject) + resource, errRes := parseResource(lexeme, objectTargeted) if errRes != nil { return fmt.Errorf("%w:%w", err, errRes) } @@ -308,32 +317,27 @@ func parseResource(lexeme string, isObj bool) (string, error) { } const ( - ObjectResource = "object.resource" - ObjectRequest = "object.request" - - ContainerResource = "container.resource" - ContainerRequest = "container.request" + ResourceCondition = "resourcecondition" + RequestCondition = "requestcondition" ) -var typeToCondObject = map[string]apechain.ObjectType{ - ObjectResource: apechain.ObjectResource, - ObjectRequest: apechain.ObjectRequest, - ContainerResource: apechain.ContainerResource, - ContainerRequest: apechain.ContainerRequest, +var typeToCondKindType = map[string]apechain.ConditionKindType{ + ResourceCondition: apechain.KindResource, + RequestCondition: apechain.KindRequest, } func parseCondition(lexeme string) (*apechain.Condition, error) { typ, expression, found := strings.Cut(lexeme, ":") typ = strings.ToLower(typ) - objType, ok := typeToCondObject[typ] + condKindType, ok := typeToCondKindType[typ] if ok { if !found { return nil, fmt.Errorf("%w: %s", errInvalidConditionFormat, lexeme) } var cond apechain.Condition - cond.Object = objType + cond.Kind = condKindType lhs, rhs, binExpFound := strings.Cut(expression, "!=") if !binExpFound { diff --git a/cmd/frostfs-cli/modules/util/ape_test.go b/cmd/frostfs-cli/modules/util/ape_test.go index 3e4766098..d93210f41 100644 --- a/cmd/frostfs-cli/modules/util/ape_test.go +++ b/cmd/frostfs-cli/modules/util/ape_test.go @@ -109,46 +109,46 @@ func TestParseAPERule(t *testing.T) { }, { name: "Valid allow rule with conditions", - rule: "allow Object.Get Object.Resource:Department=HR Object.Request:Actor!=ownerA *", + rule: "allow Object.Get ResourceCondition:Department=HR RequestCondition:Actor!=ownerA *", expectRule: policyengine.Rule{ Status: policyengine.Allow, Actions: policyengine.Actions{Names: []string{nativeschema.MethodGetObject}}, Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatAllObjects}}, Condition: []policyengine.Condition{ { - Op: policyengine.CondStringEquals, - Object: policyengine.ObjectResource, - Key: "Department", - Value: "HR", + Op: policyengine.CondStringEquals, + Kind: policyengine.KindResource, + Key: "Department", + Value: "HR", }, { - Op: policyengine.CondStringNotEquals, - Object: policyengine.ObjectRequest, - Key: "Actor", - Value: "ownerA", + Op: policyengine.CondStringNotEquals, + Kind: policyengine.KindRequest, + Key: "Actor", + Value: "ownerA", }, }, }, }, { name: "Valid rule for object with conditions with action detail", - rule: "deny:QuotaLimitReached Object.Get Object.Resource:Department=HR Object.Request:Actor!=ownerA *", + rule: "deny:QuotaLimitReached Object.Get ResourceCondition:Department=HR RequestCondition:Actor!=ownerA *", expectRule: policyengine.Rule{ Status: policyengine.QuotaLimitReached, Actions: policyengine.Actions{Names: []string{nativeschema.MethodGetObject}}, Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatAllObjects}}, Condition: []policyengine.Condition{ { - Op: policyengine.CondStringEquals, - Object: policyengine.ObjectResource, - Key: "Department", - Value: "HR", + Op: policyengine.CondStringEquals, + Kind: policyengine.KindResource, + Key: "Department", + Value: "HR", }, { - Op: policyengine.CondStringNotEquals, - Object: policyengine.ObjectRequest, - Key: "Actor", - Value: "ownerA", + Op: policyengine.CondStringNotEquals, + Kind: policyengine.KindRequest, + Key: "Actor", + Value: "ownerA", }, }, }, @@ -170,12 +170,12 @@ func TestParseAPERule(t *testing.T) { }, { name: "Invalid rule with unknown condition binary operator", - rule: "deny Object.Put Object.Resource:Department
Date: Mon, 13 May 2024 15:46:15 +0300 Subject: [PATCH 1086/1943] [#1128] util/test: Remove unused package Signed-off-by: Evgenii Stratonikov --- pkg/util/test/keys.go | 137 ------------------------------------------ 1 file changed, 137 deletions(-) delete mode 100644 pkg/util/test/keys.go diff --git a/pkg/util/test/keys.go b/pkg/util/test/keys.go deleted file mode 100644 index d233a633a..000000000 --- a/pkg/util/test/keys.go +++ /dev/null @@ -1,137 +0,0 @@ -package test - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/x509" - "encoding/hex" - "testing" - - "github.com/stretchr/testify/require" -) - -// Keys is a list of test private keys in hex format. -var Keys = []string{ - "307702010104203ee1fd84dd7199925f8d32f897aaa7f2d6484aa3738e5e0abd03f8240d7c6d8ca00a06082a8648ce3d030107a1440342000475099c302b77664a2508bec1cae47903857b762c62713f190e8d99912ef76737f36191e4c0ea50e47b0e0edbae24fd6529df84f9bd63f87219df3a086efe9195", - "3077020101042035f2b425109b17b1d8f3b5c50daea1091e27d2452bce1126080bd4b98de9bb67a00a06082a8648ce3d030107a144034200045188d33a3113ac77fea0c17137e434d704283c234400b9b70bcdf4829094374abb5818767e460a94f36046ffcef44576fa59ef0e5f31fb86351c06c3d84e156c", - "30770201010420f20cd67ed4ea58307945f5e89a5e016b463fbcad610ee9a7b5e0094a780c63afa00a06082a8648ce3d030107a14403420004c4c574d1bbe7efb2feaeed99e6c03924d6d3c9ad76530437d75c07bff3ddcc0f3f7ef209b4c5156b7395dfa4479dd6aca00d8b0419c2d0ff34de73fad4515694", - "30770201010420335cd4300acc9594cc9a0b8c5b3b3148b29061d019daac1b97d0fbc884f0281ea00a06082a8648ce3d030107a14403420004563eece0b9035e679d28e2d548072773c43ce44a53cb7f30d3597052210dbb70674d8eefac71ca17b3dc6499c9167e833b2c079b2abfe87a5564c2014c6132ca", - "30770201010420063a502c7127688e152ce705f626ca75bf0b62c5106018460f1b2a0d86567546a00a06082a8648ce3d030107a14403420004f8152966ad33b3c2622bdd032f5989fbd63a9a3af34e12eefee912c37defc8801ef16cc2c16120b3359b7426a7609af8f4185a05dcd42e115ae0df0758bc4b4c", - "30770201010420714c3ae55534a1d065ea1213f40a3b276ec50c75eb37ee5934780e1a48027fa2a00a06082a8648ce3d030107a1440342000452d9fd2376f6b3bcb4706cad54ec031d95a1a70414129286c247cd2bc521f73fa8874a6a6466b9d111631645d891e3692688d19c052c244e592a742173ea8984", - "30770201010420324b97d5f2c68e402b6723c600c3a7350559cc90018f9bfce0deed3d57890916a00a06082a8648ce3d030107a1440342000451ec65b2496b1d8ece3efe68a8b57ce7bc75b4171f07fa5b26c63a27fb4f92169c1b15150a8bace13f322b554127eca12155130c0b729872935fd714df05df5e", - "3077020101042086ebcc716545e69a52a7f9a41404583e17984a20d96fafe9a98de0ac420a2f88a00a06082a8648ce3d030107a144034200045f7d63e18e6b896730f45989b7a8d00c0b86c75c2b834d903bc681833592bdcc25cf189e6ddef7b22217fd442b9825f17a985e7e2020b20188486dd53be9073e", - "3077020101042021a5b7932133e23d4ebb7a39713defd99fc94edfc909cf24722754c9077f0d61a00a06082a8648ce3d030107a14403420004d351a4c87ec3b33e62610cb3fd197962c0081bbe1b1b888bc41844f4c6df9cd3fd4637a6f35aa3d4531fecc156b1707504f37f9ef154beebc622afc29ab3f896", - "3077020101042081ef410f78e459fa110908048fc8923fe1e84d7ce75f78f32b8c114c572bfb87a00a06082a8648ce3d030107a144034200046e3859e6ab43c0f45b7891761f0da86a7b62f931f3d963efd3103924920a73b32ce5bc8f14d8fb31e63ccd336b0016eeb951323c915339ca6c4c1ebc01bbeb2b", - "307702010104209dd827fa67faf3912e981b8dbccafb6ded908957ba67cf4c5f37c07d33abb6c5a00a06082a8648ce3d030107a14403420004e5cb5ae6a1bd3861a6b233c9e13fa0183319f601d0f4e99b27461e28f473e822de395d15c1e14d29a6bd4b597547e8c5d09a7dd3a722a739bb76936c1ad43c0e", - "3077020101042005a03e332e1aff5273c52c38ec6c5a1593170ddf8d13989a8a160d894566fc6ba00a06082a8648ce3d030107a144034200045a11611542f07f2d5666de502994ef61f069674513811df42290254c26f71134100fed43ea8ecd9833be9abb42d95be8661f790c15b41ca20db5b4df4f664fb4", - "307702010104206e833f66daf44696cafc63297ff88e16ba13feefa5b6ab3b92a771ff593e96d0a00a06082a8648ce3d030107a14403420004434e0e3ec85c1edaf614f91b7e3203ab4d8e7e1c8a2042223f882fc04da7b1f77f8f2ee3b290ecfa6470a1c416a22b368d05578beb25ec31bcf60aff2e3ffcd4", - "30770201010420937c4796b9fc62fde4521c18289f0e610cf9b5ebf976be8d292bc8306cee2011a00a06082a8648ce3d030107a14403420004ba5951adddf8eb9bc5dac2c03a33584d321f902353c0aadccd3158256b294f5aa9cd5215201d74de2906630d8cefb4f298ff89caa29b5c90f9d15294f8d785bc", - "307702010104204b002204533f9b2fb035087df7f4288e496fc84e09299765de7a6cd61e6a32bca00a06082a8648ce3d030107a1440342000441abcf37a4d0962156c549de8497120b87e5e370a967188ab1d2d7abce53711dfd692a37f30018e2d14030185b16a8e0b9ca61dca82bfe6d8fc55c836355b770", - "3077020101042093ffa35f1977b170a0343986537de367f59ea5a8bd4a8fdd01c5d9700a7282dba00a06082a8648ce3d030107a144034200040e01090b297cf536740b5c0abb15afba03139b0d4b647fdd0c01d457936499c19283cf7b1aee2899923e879c97ddeffe4a1fa2bffc59d331b55982972524b45b", - "307702010104201c1a2209a2b6f445fb63b9c6469d3edc01c99bab10957f0cbe5fad2b1c548975a00a06082a8648ce3d030107a144034200040c8fd2da7bad95b6b3782c0a742476ffcb35e5bc539ea19bbccb5ed05265da3ab51ec39afd01fbee800e05ec0eb94b68854cd9c3de6ab028d011c53085ffc1b3", - "30770201010420b524d8cba99619f1f9559e2fe38b2c6d84a484d38574a92e56977f79eac8b537a00a06082a8648ce3d030107a14403420004a6d7d0db0cc0a46860fb912a7ace42c801d8d693e2678f07c3f5b9ea3cb0311169cbd96b0b9fc78f81e73d2d432b2c224d8d84380125ecc126481ee322335740", - "307702010104207681725fec424a0c75985acfb7be7baed18b43ec7a18c0b47aa757849444557ca00a06082a8648ce3d030107a14403420004bd4453efc74d7dedf442b6fc249848c461a0c636bb6a85c86a194add1f8a5fac9bf0c04ece3f233c5aba2dee0d8a2a11b6a297edae60c0bc0536454ce0b5f9dd", - "30770201010420ae43929b14666baa934684c20a03358cda860b89208824fac56b48f80920edc4a00a06082a8648ce3d030107a14403420004d706b0d86743d6052375aa5aa1a3613c87dccfe704dc85b4ed4f49a84a248a94582202927ec0c082234919f3ce6617152ba0d02497b81c61284261ce86cef905", - "3077020101042089d600f43c47ab98e00225e9b2d4a6c7ab771490f856d4679d9e1e0cca3009d0a00a06082a8648ce3d030107a144034200048515055045543e429173fc8f9f56a070bd4314b2b3005437d8504e6b6885f85101409b933e27c0de11415aee516d0d1b474088a437ece496ceb4f1c131e9ea40", - "3077020101042015518dcf888c7b241dac1c8bfa19d99f7fdba7ba37ed57d69bbbd95bb376ea4ca00a06082a8648ce3d030107a1440342000459e88d92efaa5277d60948feaa0bcd14388da00e35f9bae8282985441788f8beb2b84b71b1ae8aa24d64bb83759b80e3f05c07a791ffe10079c0e1694d74618c", - "307702010104203e840868a96e59ca10f048202cce02e51655a932ff0ac98a7b5589a8df17f580a00a06082a8648ce3d030107a14403420004f296414e914dcefd29bc8a493f8aedc683e5514a8ec5160637bee40ebaa85a421a363c8f7ce3ed113e97d2c4b6d9cd31d21698a54fce8d8e280a6be9ee4fbca9", - "30770201010420aa746067891cf005286d56d53092f77961f828bf5bf11aade18c8a458090d39aa00a06082a8648ce3d030107a144034200044af5ad2dacbb32ab795ab734d26bae6c098bd2ba9ca607542174d61b49ca3c07786aeb0c96908793a63d4f20cd370a77b7ec65e6b285c6337764e7ae3cd5fa1c", - "307702010104207135cbd831d52e778622c21ed035df9e3c6e4128de38fbf4d165a0583b5b4a29a00a06082a8648ce3d030107a1440342000412e2b9e11f288d8db60fbb00456f5969e2816a214a295d8e4d38fbacab6b0a7e0cdb8557e53d408244083f192d8a604d5b764ab44b467e34664ca82e012b60ab", - "3077020101042064b839ca26c42e2e97e94da5589db2de18597a12d6167fdfe0d20e932de747a2a00a06082a8648ce3d030107a1440342000481e90c2173b720447ae28361149598a7245ed51c3881a89353da25b8e574b8c9b2d80b2563efe5d9a0184b57af2431116c8a4ad8071ef2764ca3d3744c638401", - "30770201010420a56df8e6349520d27c36eb1e9675720c702d562842c859cd54b3d866f2cada30a00a06082a8648ce3d030107a14403420004dc08beb5b857f6da13ae1116e40a6e4e4b5aaebc8040eae0b3037c243b1c24def39de670380472df7aa98cb9e0f1132bc4afc0629d80a24c54b8ad600cb24cd4", - "30770201010420bd2dd18485a9667673b2c38c2ad51cc756a199d18fe1100acf29b647a549171ea00a06082a8648ce3d030107a1440342000422825ffe8b3416b6755a7076a7dc6f746ff29ee0a4455dceb0f3262127d51c9bb53f2c204636da8d7a09961274d7c7ba2ef3c771e83fb996ffe3f9882c530ffd", - "307702010104203058a0c8de5c6d4a5c7f64883e7d3c9f5097c8bc073cc482421e903b37123c06a00a06082a8648ce3d030107a14403420004f959705673c2f4112673e43d1d876ca71c64153abb6c9f58d1c3b3c1f8c213ee346833fb695eb533664d596a68e42150a21b405e3a08ed70af5f568275a7a79f", - "307702010104202bd9035bf38e7c4580abc377a6e9c31aa9bdaff90af2ce688eda9a532c83875ea00a06082a8648ce3d030107a14403420004918010ea3387786c6a257996ec74d7ee4e1703b3b811118f4e89fabfef7c694495191848a0d590313a0be9784644ef98e0f0f7e50fed5bee3fa48d66edbcd2b5", - "30770201010420aa055d6cbe96e1cfbe39530bc4b7a976baff53ce399956f0d8241750d3379990a00a06082a8648ce3d030107a1440342000444e8b6deda76c12320a8c5b7a48141ebf5dc9288df79a0f418ab92d82061d10118b8bce9fb200e5009a19fb0e19036762b3ef85440405f43225d6ee3350bf96c", - "30770201010420b8712525a79c7bd3df2a9dbabde1a111078a7ef30687a2efe0f0c4b4a23f2aa0a00a06082a8648ce3d030107a144034200049dc9e3d836a834f6d14ae99dfc70ad9b65c84f351c8dbc4f9b1b61c238051fb1db23e43d4b6e17803e21ebc44fe2f66742e306daa8c4ca7d79c6dd01fc1a4e4e", - "3077020101042086c18b56c4a2264b37c18a7937f026ab07ca6076eeea1ab90376492efb7875d9a00a06082a8648ce3d030107a144034200042f169311f2fae406de3c4a64fec94a22c35972281922a69e7657185997ae59fb3f69ac94295e58681cfbd263f8e6fbce144cc7925b71d90f57de3f3e10588321", - "30770201010420f58221355e1b2da73d66de482ec1edcb8597f3967d00d1356f4678fea6ad67e6a00a06082a8648ce3d030107a14403420004238cc44f02fa566e249a9697a078b9d38eba06012d54a29a430843a18df7a0a4207d704a360399db95eca591f2f81b6c50390467f293a1623b4757bdb4138101", - "30770201010420b10888a0157d524667fd575683bdcded4628a65149fde59b7340781b0cf2e36ea00a06082a8648ce3d030107a14403420004222ba11430b8719929c726aec74e8e70893e2960bc2bbee70fbaa6d88fa2a346adf0c450ea9823f0ba77d334fcd476ea036a62199338d7aa32e56c708d7a8caa", - "30770201010420edf001bd24c92e4f65789aae228223e77df71ce9bbfd7ce4d236ea3648e1f7fea00a06082a8648ce3d030107a1440342000472693c95786ab9f4e7c923338ce98bd068e28b71f84b77e7adb378c2ce2d8f1a2e13833df1afe4569367d7a4eee3abf50124299a28045a0073ea324f5ddb45ea", - "30770201010420e2649e591fc9072dd55573e41fc4ebfdf1db118951e4b7b2a98027ac9a4f7702a00a06082a8648ce3d030107a144034200046e34c9dea1836671f1ef259d7c3ee678c2f92d092af2518413fe9ba153a07ca8e9938784876e90cfa2989a00a83b1ac599c87a8d15be8001e46dfbfe018156a2", - "3077020101042069cd9b710f25613794751aed951004c888d4611aefa45abc23abff218e608290a00a06082a8648ce3d030107a14403420004dcf8ff34ab841720ff8dc08b60a14f41689e65f979a1af69b5e106f4262a2cb0947c9619e980caf20b3e7c8f15e60fc31c5b611c8a58370ba8201c9b6b932bd4", - "307702010104202898cef1944aaf90fddf433390323a02a79938568cf99f6c25bc9aa9e5cddb0aa00a06082a8648ce3d030107a1440342000491a1c20420f5005f5761419e4dcd0d9da0cf2ea4733f6d98a3d0c124f284cabdc65eafd9d2cad9b1122fca791c8b37997feed130c5725ea797cf07c61fb82734", - "30770201010420e568bd3ffa639aa418e7d5bc9e83f3f56690ebf645015ff7f0e216d76045efd5a00a06082a8648ce3d030107a144034200042424b498297124037db950bf2a1e652ba7f977363f4f69d7308531d27bf392219d93cb78f4379b7ffb16f3e7be311e208af2409bd33000fd25a8707ac6bec76b", - "307702010104205163d5d5eea4db97fccc692871f257842fdaca0eca967d29924242f7a2c56ad7a00a06082a8648ce3d030107a144034200044e2ca8312122039c3374db08851710d3b9a2efcbd8f5df004ec7b60a348aee32466f799b5957d39845f451071bb1f3bb99f25bf43196e7c772f7b84f39221b3a", - "30770201010420301eb936d2737886ab2fbf670952f9ba0d324827b81801810bfd60c89e8ca862a00a06082a8648ce3d030107a14403420004455454b1f3828a2328a8925c4c98bd6e37dece276efb3299d8b7d78c9d7e6f978b14d021c07bae0c18a623fc52ab2fec1523a89b2fd0cda373e9c9442a3545f2", - "3077020101042032c12a9bca8070c131b0a46944c17adf35eb44079f3c887fc3b93740bb9c03fca00a06082a8648ce3d030107a14403420004e61da413c4d5dbc6c004089d96a3cb55f4b20b70c544f3823a7a6322c53e134fcb8a885729ef284d68d23e0a58009d48b369f9c4f5a665a8880a48606491dd8a", - "30770201010420aa2b40742722b81c6ffd5c47b94b8be747da259e172a82d27ebc525c8f46d17aa00a06082a8648ce3d030107a14403420004f87a863ed11592cf4f96e837038b105d155f5e09a31386ab4604234e8a975d49a9612b4597b7fb206087b70a26bce4aca31edb253530e6da83ce16beefa99f60", - "307702010104202a70a0c827b4ce8d433e800ab0818b1401b220fadea75feff655251ee4317556a00a06082a8648ce3d030107a14403420004a5c9209fd53dc1ce2c873782ec507db5e0f9cc78292a84ecafc5bab16c2e4d786a882ad77ad999f3d6ba676ad80354ad376dabc4fa03a6c15ead3aa16f213bc5", - "307702010104202787d04901f48c81774171ef2e2a4d440b81f7fa1f12ab93d8e79ffab3416a1ca00a06082a8648ce3d030107a14403420004010d32df4d50343609932a923f11422e3bea5fa1319fb8ce0cc800f66aa38b3f7fda1bc17c824278734baa3d9b7f52262eeacbca21304b74ba4795b5055b1e9f", - "3077020101042032423728a897144d4fb95090ca0ac67a23eb22e2f7f925cbddaf542eeaec8faaa00a06082a8648ce3d030107a14403420004c37f9fec5b1be5b0286300ace6a5d25df8189d29604145a77b6578a4e3956ed3d9af48f8ee1e39868bba9e359e5444984f0428755e29d2012f235c9a56749148", - "30770201010420d5bd2a3867937e0b903d19113e859ca9f6497f4af082894a6911cef3a3a12d35a00a06082a8648ce3d030107a14403420004435b2e891c46023f422119f18a04c75b9322ea4aaddd10a0568438310896388bf7037e98bd5979a6f0839acb07dead1f2f973640dcc11dcee1de8a07c0b3dd80", - "30770201010420590edcf1f2b6ee6c1b836ace33b934597883a00ce84fe812a4b3e22432846972a00a06082a8648ce3d030107a14403420004183d7cad633cb0f4ab774f4dc19b9db87e7ef97b0f4d43ac395d2409dabbe5339dbad661c7c2fd05606e2edb08f8ace660f73bf5232011262d563603f61d2353", - "30770201010420a0ea4e16cf8c7c641d70aea82192fb9303aab6e7b5cd72586ba287d50f4612d6a00a06082a8648ce3d030107a1440342000482a72d31e71f0aea778cb42b324abf853cb4e4e8d4b2ae0e5130480073e911f183134c047a7e1cd41a845a38057ea51a1527923518cbf47c3e195a9f44e1d242", - "307702010104209e04b00c8d0f96ddb2fbb48cfc199905bfbfcc894acb77b56bf16a945a7c7d08a00a06082a8648ce3d030107a1440342000405efd203dcddfb66d514be0de2b35050b83e3738096cd35398165bfdbe34d34c0d96a4e6df503903c75c2c06b66b02b15cd7bf74c147d7a9f0a5e53b83c5762d", - "30770201010420aa69f1cc2cb3482a12af4b1614d6dde01216f1cad1c9f03c681daa8648b75b37a00a06082a8648ce3d030107a1440342000474ffec1297420d0cf730b42942058699d803ab618e1e40ccf9cc17f71f62b3123d863fbf8fae37b6c958892af6151159f74e2a568917bfc2f4e00c55c32b52e7", - "3077020101042090a04300e8d6ed9f44422a2cf93817604bf1f6233c4333ba0db20ab726852fa4a00a06082a8648ce3d030107a144034200049e6f2001baf2b6fb25e3273907ed7320f494de6b5882c4c4b9bcee7ddc60274e064cc68c64325c001f07a505722062d1ca9774a2cc1e0cd28fe5f807865bfcc1", - "3077020101042088945c19c6ce3e63f8d8a421616391d83bec79a0c590f1607b247ffa0c677dd3a00a06082a8648ce3d030107a1440342000492d17d410f9eabf7ae4509a92494e9fe94a72947f24e60c5bb6e12b2cde3c1bfe5305a0d759138069d44268f174136971ecb752df602c282e48d40f43a8734e3", - "3077020101042079d14eacdc4f21dc5284bd8487dcb2c22e9e53e71909474f922bf695f49cf23ea00a06082a8648ce3d030107a1440342000428039292c5bcf3593639bf5835ec9411ffd3ac236c0186697623930b5ca63f32ff41df5217e7def770d9a0de87f61526497bd9aaa95d924e0a17d85958e7c095", - "30770201010420a6ac867ff8d00aaad23198415868a64e59217b4d22474752a146fcb52204dfa5a00a06082a8648ce3d030107a14403420004a5f37a779265c55cd4f5a7f3bffc4679395898046eb9d67d8670be39001de5a7bc010b0d218561626272989c5952e8e0d95d2590f78eec44dc62a46184956301", - "30770201010420df446014577f6081113cd7d33c6ba91b9ac3d083e76f8873358f83129e2d0111a00a06082a8648ce3d030107a14403420004da0c932759f50ad705507f876138c2c6e012764abc8764a6dd609e6ad06099952b120be71690bc091591f1aa8d7d6e9365deddbc958bc87ff150358ad33f7537", - "30770201010420b3351033eaaee3a9ea27cd7dc54aa2c8d787b14b7d428165f1a04a59c6d5b0f2a00a06082a8648ce3d030107a14403420004da3984fb8152403a9fb9068b16f9afb5c900f24230e205567b4405ee3cad2db3ff46968489d494b38d0c85fcc4aecccb61fc00dca54c8fd99ee5bf5e2616f1b7", - "30770201010420deedbcef7f6821f6aab2b15ce198f5eb2064f6eb461a6b7776b4da35c81b1506a00a06082a8648ce3d030107a1440342000405422b86ce66b18e68f0fb14f28e4ed9b1f7ee84f57957f4e4b4c6b0c392e6357e4698fb707f590be1b915622ec8da476071a56919211f6e5e888284d4e33f06", - "3077020101042078c3db0d3b1114cb99f1d0bea0d3aec9067b26964e2b85fe9df4789b24cb3da5a00a06082a8648ce3d030107a144034200046874e52d7d58b6697b407b0c0eea3cfeb528e34fca1589c5031e11aae1ad1f9280e7a4c37ddf28479cd07b4246ce9398e0e24f99946f87e08532fa26b8fb8016", - "30770201010420f0ba42553b146cf088d3a5a3645782fe675d23561897ced7f1270a8d05cfdaaaa00a06082a8648ce3d030107a14403420004c250e12f3aa1fb6261c57cdb091cd90d82917e103711425888477b9da4359d2803aaf0015638294c7c0baa4ec77ba8fceff5ee7f15ea087a4174f58d518006dd", - "307702010104207f2c0fc4b0e418b2d4c72a63fdc27f158f6ad44c26d161f489714525b6a13db1a00a06082a8648ce3d030107a144034200041d83885672021e783d8bd995d187f407bbda2c6bed5e8fabc7c6c5cb304a85eaffa12dad7ba874ac45f4258fffe07534843ff7fe76075470f2c77104d781688f", - "30770201010420d3de828ac9742704d4e6981ce1fc8c473e508eda3a121cda420dacbdf39d48e9a00a06082a8648ce3d030107a14403420004c78abfc4a5c0eb3ee0c9817d1790b7ca9fd528d0bc727f9daf63f4212097538b6888b9de2ae4dff29895500be456fe0ccbee340aecb546d1558b08c3718aaa4a", - "30770201010420d9c4e477b56f2ff0b211acd82b450336276534b350747315152a4923e6e65294a00a06082a8648ce3d030107a14403420004fbd540966b03fe2c2314f20248d345e3e9b92d6a7cfea22d1b5367f01b32d616f317e00cea1f659437b4302610abba8abb0f2bfce0a91b952e9565159c1e464e", - "30770201010420fb84f4a426fa12920c2cf7c2d821280530c0fa93960ded8c20120511dc1d5069a00a06082a8648ce3d030107a14403420004c0177f13c6e00bb9029df089006a332192bdf12a782c60a8d00d110c53db67c344584f22677695a7f1629db1600b0559ced49ac931b08cc6a58e5ea436bde2f8", - "30770201010420653ce060214028f7aa584910f0925d702bde18d52d8e530f07dd5004076eb614a00a06082a8648ce3d030107a1440342000433668d0c9085feae4b285fe260a316e24f24c0bb8e442583e23284bf5a962cd0357cd63ac4d1cdda58afb201bceee911ebe7cf134652dc4390f4e328f6cb5d65", - "307702010104206123b7d5b8c53b2a2a95dd2e42fe550617b7520fe9bd94a99045addb828ad847a00a06082a8648ce3d030107a1440342000487c10fdeaabf8072dcea0dc5b18be4d72f2b8298bc891ea0a11d202438b7598ac588f16a9cd697f8220434d4e15ff4c82daaae63955525633335843069434aea", - "3077020101042000b793c9b8553ee7bec21cd966f5aaff59a07d1fa3fa86e0164bcd2f7f4dd586a00a06082a8648ce3d030107a1440342000419d4179dbeae7fa87e356f0406c327239d34e540cd7db5174a81bd6197738bc72e46fe4bd1512dc4b35950b2c1e78e6f8f54980193be78d45e4d97a837455777", - "307702010104200fb1a771004f6be6300eccd603b9c9e269fbdd69e5eb183d7acad51b0b205b88a00a06082a8648ce3d030107a14403420004d3b7fa62bacff49714ef28a955cdc30f4aef323293ac3aebab824892dfa3306f2ec319f5bca1771b956b4a9b1c2f565dc08b29c07ec84623932a5d6fb59be6c7", - "30770201010420fe6907b91407619fdc95153cd59df061e88095678801008d3901f29c7c434243a00a06082a8648ce3d030107a14403420004796fcea7889128f8060b04e9000381fd3d80fe68f000063b182fe9d8984e740c387c4ed4c6729e8c715c576fe355a9b7dda6890c55b15ae6013fd51e8858b2f2", - "30770201010420111eaff6db3b279d014b45b3da091909f054f37c350c237fe9d51b4342811299a00a06082a8648ce3d030107a144034200047d51f9178725c4134579ac6d0cb84745e0d2068ccf72d30c02dd431547f868d1cb93b5774c7e1eb9582e2151521ff16cdf80b3ba4646d64f7982066f9eb679f0", - "30770201010420631d01e6aaa68e6c36e3425b984df02bc5b54e81951479f7cea8fd1b804bab57a00a06082a8648ce3d030107a14403420004fa1b1ed9ff904f1f050577e05b5175e897d462598fdd323c8ef25f6072dfa43034baa0119e64092fb44f7a04d59d16ba8645f52cfb7775a6536c00f7fc2ee2f1", - "307702010104201ec553d14d45acdf147dba5fcbc3a42a1f763411d5c206d03600ed810b0cf106a00a06082a8648ce3d030107a14403420004e9a309a24d1061204087de10e5bc64b6d45369399a5a402d630ca2d04b34ae9d27d491e5fadd5d082e14454e6b2a572a24904ba2a8dc7430b20d361134188589", - "307702010104206d31e401bb20968106a058f8df70cd5fb8e9aaca0b01a176649712aa594ff600a00a06082a8648ce3d030107a144034200048555a2f9e7256c57b406c729d2d8da12c009f219e81cecb522cb3c494dcc1c76ac6d2f641dafe816065482fb88916e1a719672c82406556e16c32cf90752a92f", - "307702010104208ada3d6ea6000cecbfcc3eafc5d1b0674fabece2b4ed8e9192200021b8861da0a00a06082a8648ce3d030107a14403420004a99e7ed75a2e28e30d8bad1a779f2a48bded02db32b22715c804d8eeadfbf453d063f099874cb170a10d613f6b6b3be0dbdb44c79fc34f81f68aeff570193e78", - "30770201010420d066dfb8f6ba957e19656d5b2362df0fb27075836ec7141ce344f76aa364c3cea00a06082a8648ce3d030107a14403420004597fd2183c21f6d04fa686e813cf7f838594e2e9c95b86ce34b8871674d78cc685b0918fd623e3019d8c7b67104395b1f94fc3338d0772e306572236bab59c39", - "307702010104202c291b04d43060f4c2fd896b7a9b6b4f847fb590f6774b78a0dff2513b32f55ca00a06082a8648ce3d030107a14403420004e80bd7e6445ee6947616e235f59bbecbaa0a49737be3b969363ee8d3cfccbbc42a0a1282de0f27c135c34afad7e5c563c674e3d18f8abcad4a73c8c79dad3efa", - "3077020101042029af306b5c8e677768355076ba86113411023024189e687d8b9c4dee12f156fda00a06082a8648ce3d030107a144034200049d7d21e6e1e586b5868853a3751618de597241215fb2328331d2f273299a11295fe6ccd5d990bf33cf0cdcda9944bf34094d5ffa4e5512ee4a55c9f5a8c25294", - "3077020101042022e65c9fc484173b9c931261d54d2cf34b70deccb19ce0a84ce3b08bc2e0648ba00a06082a8648ce3d030107a14403420004ea9ee4ab7475ebaff6ea2a290fc77aafa4b893447d1a033f40400b4d62ee923a31d06fe5f28dbc2ebec467ebd2e002a9ea72057f0b0c60fe564584a6539376ad", - "307702010104205000583dc21cb6fd26df1c7d6e4efb9b47ceff73c0d94ed453bae0c13a9e5795a00a06082a8648ce3d030107a144034200045a6a5b5886b01f54dfa0788f15d3542aec160843a57e723008d1b984dd572ecb8935662daaba53d756d45442efbae067f52b0b151899a645afb663205babddd3", - "30770201010420997431e73eae00f476bb1a221b4cc9dfd18d787be207b7069141627f61ba752da00a06082a8648ce3d030107a144034200047c89dc8c46a27e20c37b0ecf1150e8b92c2dd4dc534a25545f87a5f0c44fdbf4dee2af5bcdc4012f0acee168aeb55bb4d24738fac105fc056928ff5870491047", - "307702010104207dc10db95a597a80e916d7f8e4e419b609d767538fe9732bcc5f9d783c605a2ba00a06082a8648ce3d030107a144034200042e2ae4fae087a11fcdf9565670164c229337ed87b5056687c6bceeb84108db9a88b9e5d96a0cf121255ceefce0bb5239608768bb841e6687dbd9626222eb5187", - "307702010104209056e22b347f5f1839f1a53f1250d098616ff04db0b49b1fddb18b987930cec7a00a06082a8648ce3d030107a1440342000427cc4c7fb5d7ac047161aee78e812ad264ba25dd878684637308674ea693817b20a5e3672de6a92dfbf82f641268052fa742e6f35ff91c617334f09f89bd1218", - "30770201010420554ea6cfeb2cc4f1e29c08e65317d72731ee03940af9ff6a141b761d5d054db6a00a06082a8648ce3d030107a14403420004a6121746c0553ede0944da8a7f304831fcefb51b40acf78016d41cc45cc5f7e9a1b22bbea028daab5cb4c39cadf84da442749cbfc04536d6f85c3254ec7a0805", - "30770201010420f53ff1c7db3c4e7c734bf7396a1a5364ac2dfe4b794b118aada6bab72cde8969a00a06082a8648ce3d030107a1440342000414b11ec158e3f9d558bd1da1ed0e38c92b1ad55834f3ce08e456747279dd9ed1143cff4f5e8d70189f4b114e3cd609105d6eb8f431f392487e4c9e16a152dba1", - "30770201010420b3f394090547f5dcb2e77cef65e03a3b7d1c953cd0e069553da2795ab0adc950a00a06082a8648ce3d030107a14403420004a1a9dbe5d6dfa2dfb039aebabe96b12faf97c994e1430323d074ecbd90ef075e0fe9dc7d5eef2483d485ffb0b4a01b01e131754fb38059a1365d342d5175397a", - "30770201010420bf13c42fa84c409161f9d73ce20fd85b20c5381914aa2a2375452b34cd352022a00a06082a8648ce3d030107a14403420004e0134214a5349a235cee406ad942ca105ef871a7e4c922ef4769466d8495c78b82f6c49270c8cd913e0cf407cdab679dd9914090ea91122ca9fb654ebcfce57d", - "30770201010420440d975b65bf585d0813137fe041461de59221856eaf255479b5e69721cfb30da00a06082a8648ce3d030107a14403420004935a9626ddb7bd6fbcd2ad9d9333851bbc64b9997cb8e43b1a17f8e9968ed6b0e5d2edf105fbabc9bd745fa2120ac527bbfefb6e8ed96844f80b8e27b6d9a549", - "307702010104209ea2dc59260408165d6c42205aa52e275f81c39d9bf5b1b9c8187ade875e8068a00a06082a8648ce3d030107a14403420004bc570aa24df0306cb761ee9fb22e61f59ae4f11e8804491d8651084f191c800d1e6b16e4bc3693b88f9bef82849f3cd6914a15cae60322c1f4822a2bdf426782", - "30770201010420505b596fb71a2e36c0ba07da03442a721f3f1832dcac19631d6c11b36ab81986a00a06082a8648ce3d030107a1440342000472cfb26cf07faa4e6e9d328214677b5eb51cd2e35717ac661d732115e592a07482bf966a31792cc993bdf816a732069ed423871b53fb3c7eabab2f4d3d272013", - "3077020101042089a9d5b397c521db4bb4a5f3e8f2043e43bb5617a2070e7bfa30dd2dbf1815a1a00a06082a8648ce3d030107a1440342000468d2aeaf641b839095644cfd4b72ab97d0bf3fae1ed36e9f81d9aff333b0123f7b846f6ca61dbbd4e10988e740463addef793994a1498987883ecf237f18bc40", - "307702010104200919a89aedb4e20cfcd2cb568c8de18b1b60b5da17aaea3be9804eb5bc3280f5a00a06082a8648ce3d030107a14403420004139812ec6bd62fd3ce71040d87cc07671948ff82300fae5f3af80dcd4e22c870c0102c4add460b2cbbeeb298f58037fc645da20aa8f5531a5ff56d3e5b2d1944", - "30770201010420b145fc69cfabff378f390f0a99fb98ddc8ba9228cb1adf9c7099c6393a24567aa00a06082a8648ce3d030107a14403420004b660084cb05e005fb163011663fee6946f354714565069968f16e89e9a7aac45610f05502ff9d9e3cd0fdc88083bd8840a518b71135e59a0f0f235636d5eb7c4", - "3077020101042082d39168f289e784ace49bfdd523297b524c494f83fe7d04dd2f055b48d636b9a00a06082a8648ce3d030107a14403420004ea4021da5eec4e7f333059625ecbad3969676cf625cbf0da316f55f50ccd40e6174fdb7023c07abdb3ca91203acbcb5e78e1601f1a9aa616c5019ac5b2222ff4", - "3077020101042066a1ebc23e993674bfdc3b9721c280b7f3c1599903063ea7899b848b942a6169a00a06082a8648ce3d030107a144034200046bdb182c6c0c1f9ea898c3847bc4b46014cb8da6a02d75b7bed3c4a9a4e9c8836d4ce22fe68b68ae56a91fb435c7ea8f05bca8e8fcb1d6b77770d419f99e51da", - "30770201010420fa2cda21b761c46fcc5b54d47b045e24affdb95425e859bb367a07950119ab6ba00a06082a8648ce3d030107a144034200044b9e4cee102ad23fea3357f8f5f95ab9d60d34086ba4b39d5f37cbc61998ac9658ec56033ad72977d41e449d449f5aac2bc653ea8038fc04a011ff02ec49e088", - "3077020101042028acfb3c41b7be1d9d0506ac3702c363ffd767dd738dc8ab581ad7add2ec8872a00a06082a8648ce3d030107a144034200047467dedfb8c9a7d9496d4898d6ace0fba063545ab0d345d8b63b90871927ed269645a745a7335ca511d86a366f24e7832477842b4041a9ab564c5fbce49e4df8", - "307702010104202e57b8b867bd95a8dfcdd2cb8f82ea41bff21610019afd6e2367e755dec5b944a00a06082a8648ce3d030107a144034200048f97eb2d6ee2d3da8746d8d4f84469ea765fb0d1412b167b6d8a916b5f968b4d64ede5ea6d6e08ec0de192262fcb3ebed49e9d17858261affed84827b38c6cc9", - "3077020101042021a904281e4c31386ce34a5b52af3a068caa65819fbcf0ca76ab6041ecdaf454a00a06082a8648ce3d030107a1440342000405f9b7894a97fcddfc3285b8e974718606616fe07c70b7ab2bfb28a85fb3014c2610ab9e8e6da8ae3da032837d3a14b1e791d2633bdd8551b4817a080b9aa697", - "3077020101042089c2c73d08bd03da4c3111aa0b78bb1edc5243d8e119513035d3741e851dec1ca00a06082a8648ce3d030107a14403420004ec9ebc34f45150334fd1d8c92274fe43c5b3b059f15cb1963f6cf7d54bc6b1b0b4ef1c5d56d2d06ab54ce2e7606e0fa5d2f188a2d593b22d9cf6a0098aa00cb6", -} - -// DecodeKey creates a test private key. -func DecodeKey(t testing.TB, i int) *ecdsa.PrivateKey { - if i < 0 { - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - require.NoError(t, err, "could not generate uniq key") - - return key - } - - if current, size := i, len(Keys); current >= size { - t.Fatalf("add more test keys, used %d from %d", current, size) - } - - buf, err := hex.DecodeString(Keys[i]) - require.NoError(t, err, "could not to decode hex string") - - key, err := x509.ParseECPrivateKey(buf) - require.NoError(t, err, "could not to parse ec private key") - return key -} From 6e71ae3bda05fbfc6404dc72b10de6ce1926e0a7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 14 May 2024 16:04:09 +0300 Subject: [PATCH 1087/1943] [#1130] fstree: Remove useless Stat() call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ old │ new │ │ sec/op │ sec/op vs base │ SubstorageReadPerf/fstree_nosync-seq100-8 2.689µ ± 2% 2.428µ ± 4% -9.72% (p=0.000 n=10) SubstorageReadPerf/fstree_nosync-rand100-8 2.727µ ± 1% 2.497µ ± 2% -8.42% (p=0.000 n=10) geomean 2.708µ 2.462µ -9.07% ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/blobstor/fstree/fstree.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 7a064af6b..02580dbfa 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -358,10 +358,6 @@ func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, err p := t.treePath(prm.Address) - if _, err := os.Stat(p); os.IsNotExist(err) { - return common.GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) - } - var data []byte var err error { From d0f64c23a5d96937210c1b9aafbdd235674fab57 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 14 May 2024 17:02:00 +0300 Subject: [PATCH 1088/1943] Release v0.39.0 Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 19 +++++++++++++++++++ VERSION | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c69b69d6..53e2c0bc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,25 @@ Changelog for FrostFS Node ### Removed ### Updated +## [v0.39.0] + +### Added +- Preliminary erasure coding support (#1065, #1112, #1103, #1120) +- TTL cache for blobovnicza tree (#1004) +- Cache for frostfsid and policy contracts (#1117) +- Writecache path to metric labels (#966) +- Documentation for authentication mechanisms (#1097, #1104) +- Metrics for metabase resync status (#1029) + +### Changed +- Speed up metabase resync (#1024) + +### Fixed +- Possible panic in GET_RANGE (#1077) + +### Updated +- Minimum required Go version to 1.21 + ## [v0.38.0] ### Added diff --git a/VERSION b/VERSION index 765098dc4..2302e306c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.38.0 +v0.39.0 From b4cfc80579a2b21196f4a44ad45ac2dcc619ed5c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 15 May 2024 09:45:31 +0300 Subject: [PATCH 1089/1943] [#1133] node: Improve tests for shards.default config section Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config/engine/config_test.go | 4 ++++ config/example/node.env | 4 ++-- config/example/node.json | 4 ++-- config/example/node.yaml | 2 ++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index b5c926fc3..3f9c7ec71 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -98,6 +98,8 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 1, blz.ShallowDepth()) require.EqualValues(t, 4, blz.ShallowWidth()) require.EqualValues(t, 50, blz.OpenedCacheSize()) + require.EqualValues(t, time.Minute, blz.OpenedCacheTTL()) + require.EqualValues(t, 30*time.Second, blz.OpenedCacheExpInterval()) require.EqualValues(t, 10, blz.LeafWidth()) require.EqualValues(t, 10, blz.InitWorkerCount()) require.EqualValues(t, true, blz.InitInAdvance()) @@ -151,6 +153,8 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 1, blz.ShallowDepth()) require.EqualValues(t, 4, blz.ShallowWidth()) require.EqualValues(t, 50, blz.OpenedCacheSize()) + require.EqualValues(t, 5*time.Minute, blz.OpenedCacheTTL()) + require.EqualValues(t, 15*time.Second, blz.OpenedCacheExpInterval()) require.EqualValues(t, 10, blz.LeafWidth()) require.EqualValues(t, blobovniczaconfig.InitWorkerCountDefault, blz.InitWorkerCount()) require.EqualValues(t, blobovniczaconfig.RebuildDropTimeoutDefault, blz.RebuildDropTimeout()) diff --git a/config/example/node.env b/config/example/node.env index 77733f568..db31ae35a 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -128,8 +128,8 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 -FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=5m -FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_EXP_INTERVAL=15s +FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=1m +FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_EXP_INTERVAL=30s FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE diff --git a/config/example/node.json b/config/example/node.json index 2ac8697bb..80140e5a2 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -176,8 +176,8 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, - "opened_cache_ttl": "5m", - "opened_cache_exp_interval": "15s", + "opened_cache_ttl": "1m", + "opened_cache_exp_interval": "30s", "leaf_width": 10, "init_worker_count": 10, "init_in_advance": true, diff --git a/config/example/node.yaml b/config/example/node.yaml index d2109fcc5..13c1823d8 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -195,6 +195,8 @@ storage: init_worker_count: 10 #count of workers to initialize blobovniczas init_in_advance: true rebuild_drop_timeout: 30s # timeout before drop single blobovnicza + opened_cache_ttl: 1m + opened_cache_exp_interval: 30s - type: fstree path: tmp/0/blob # blobstor path From 300654b045dbd5ebcb05e601aa51c76321f2c30c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 15 May 2024 11:51:38 +0300 Subject: [PATCH 1090/1943] [#1083] objsvc/v2: Properly check response status after forwarding Previously we had cryptic error: ``` debug get/remote.go:38 remote call failed {"component": "Object.Get service", "request": "HEAD", "address": "9sTxoVrhJ7WBtXQfK2NJ7zDV5yCF7BPLKK1XTxYPdGsP/BbHV4KZZ8y2BPqAT5kyjdHRLkfbtY2xf5uYoMVqxACn1", "raw": false, "local": false, "with session": false, "with bearer": false, "error": "unexpected header type "} ``` Now we have and expected error: ``` debug get/remote.go:38 remote call failed {"component": "Object.Get service", "request": "HEAD", "address": "D2rqaMG4D2VHdv3HKky8UYSYmwQFH2v9oXXqtyRZPTMy/BbHV4KZZ8y2BPqAT5kyjdHRLkfbtY2xf5uYoMVqxACn1", "raw": false, "local": false, "with session": false, "with bearer": false, "error": "status: code = 2049 message = object not found"} ``` Signed-off-by: Evgenii Stratonikov --- pkg/services/object/get/v2/head_forwarder.go | 3 +-- pkg/services/object/get/v2/util.go | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index 11286321a..0c2925859 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -24,7 +24,6 @@ import ( type headRequestForwarder struct { Request *objectV2.HeadRequest - Response *objectV2.HeadResponse OnceResign sync.Once ObjectAddr oid.Address Key *ecdsa.PrivateKey @@ -172,5 +171,5 @@ func (f *headRequestForwarder) verifyResponse(headResp *objectV2.HeadResponse, p return errResponseVerificationFailed(err) } - return checkStatus(f.Response.GetMetaHeader().GetStatus()) + return checkStatus(headResp.GetMetaHeader().GetStatus()) } diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 610076c7a..d71b381e7 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -249,7 +249,6 @@ func (s *Service) toHeadPrm(req *objectV2.HeadRequest, resp *objectV2.HeadRespon forwarder := &headRequestForwarder{ Request: req, - Response: resp, ObjectAddr: objAddr, Key: key, } From 0924b62a95889e731890ff5725fc38871a2c3601 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 15 May 2024 11:56:17 +0300 Subject: [PATCH 1091/1943] [#1083] objsvc/v2: Unify response verification after forwarding 1. Use the same routine for HEAD/GET_RANGE methods. 2. Make error message similar. Signed-off-by: Evgenii Stratonikov --- pkg/services/object/get/v2/errors.go | 4 ---- .../object/get/v2/get_range_forwarder.go | 17 +---------------- pkg/services/object/get/v2/head_forwarder.go | 17 +---------------- pkg/services/object/get/v2/util.go | 19 +++++++++++++++++++ 4 files changed, 21 insertions(+), 36 deletions(-) diff --git a/pkg/services/object/get/v2/errors.go b/pkg/services/object/get/v2/errors.go index 01b57f1f2..213455e10 100644 --- a/pkg/services/object/get/v2/errors.go +++ b/pkg/services/object/get/v2/errors.go @@ -63,10 +63,6 @@ func errCouldNotWriteObjChunk(forwarder string, err error) error { return fmt.Errorf("could not write object chunk in %s forwarder: %w", forwarder, err) } -func errCouldNotVerifyRangeResponse(resp *objectV2.GetRangeResponse, err error) error { - return fmt.Errorf("could not verify %T: %w", resp, err) -} - func errCouldNotCreateGetRangeStream(err error) error { return fmt.Errorf("could not create Get payload range stream: %w", err) } diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 5b05ec370..10ecfc4a3 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -14,7 +14,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -64,20 +63,6 @@ func (f *getRangeRequestForwarder) forwardRequestToNode(ctx context.Context, add return nil, f.readStream(ctx, rangeStream, c, pubkey) } -func (f *getRangeRequestForwarder) verifyResponse(resp *objectV2.GetRangeResponse, pubkey []byte) error { - // verify response key - if err := internal.VerifyResponseKeyV2(pubkey, resp); err != nil { - return err - } - - // verify response structure - if err := signature.VerifyServiceMessage(resp); err != nil { - return errCouldNotVerifyRangeResponse(resp, err) - } - - return checkStatus(resp.GetMetaHeader().GetStatus()) -} - func (f *getRangeRequestForwarder) openStream(ctx context.Context, addr network.Address, c client.MultiAddressClient) (*rpc.ObjectRangeResponseReader, error) { // open stream var rangeStream *rpc.ObjectRangeResponseReader @@ -107,7 +92,7 @@ func (f *getRangeRequestForwarder) readStream(ctx context.Context, rangeStream * return errReadingResponseFailed(err) } - if err := f.verifyResponse(resp, pubkey); err != nil { + if err := verifyResponse(resp, pubkey); err != nil { return err } diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index 0c2925859..5e16008b8 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -60,7 +59,7 @@ func (f *headRequestForwarder) forwardRequestToNode(ctx context.Context, addr ne return nil, err } - if err := f.verifyResponse(headResp, pubkey); err != nil { + if err := verifyResponse(headResp, pubkey); err != nil { return nil, err } @@ -159,17 +158,3 @@ func (f *headRequestForwarder) sendHeadRequest(ctx context.Context, addr network } return headResp, nil } - -func (f *headRequestForwarder) verifyResponse(headResp *objectV2.HeadResponse, pubkey []byte) error { - // verify response key - if err := internal.VerifyResponseKeyV2(pubkey, headResp); err != nil { - return err - } - - // verify response structure - if err := signature.VerifyServiceMessage(headResp); err != nil { - return errResponseVerificationFailed(err) - } - - return checkStatus(headResp.GetMetaHeader().GetStatus()) -} diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index d71b381e7..852c2aec3 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -8,11 +8,13 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/status" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -408,3 +410,20 @@ func chunkToSend(global, local int, chunk []byte) []byte { return chunk[global-local:] } + +type apiResponse interface { + GetMetaHeader() *session.ResponseMetaHeader + GetVerificationHeader() *session.ResponseVerificationHeader +} + +func verifyResponse(resp apiResponse, pubkey []byte) error { + if err := internal.VerifyResponseKeyV2(pubkey, resp); err != nil { + return err + } + + if err := signature.VerifyServiceMessage(resp); err != nil { + return errResponseVerificationFailed(err) + } + + return checkStatus(resp.GetMetaHeader().GetStatus()) +} From b3eaa8a9bcf0933f511adbc7bfea6a8d67b2c679 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 15 May 2024 12:03:32 +0300 Subject: [PATCH 1092/1943] [#1083] objsvc/v2: Check response status in RANGE_HASH forwarder Fixes #1083 Signed-off-by: Evgenii Stratonikov --- pkg/services/object/get/v2/get_range_hash.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/services/object/get/v2/get_range_hash.go b/pkg/services/object/get/v2/get_range_hash.go index 0054f0e9f..e97b60f66 100644 --- a/pkg/services/object/get/v2/get_range_hash.go +++ b/pkg/services/object/get/v2/get_range_hash.go @@ -142,6 +142,9 @@ func (s *Service) forwardGetRangeHashRequest(ctx context.Context, req *objectV2. resp, err := s.performGetRangeHashOnNode(ctx, req, info) if err == nil { + if err := verifyResponse(resp, info.PublicKey()); err != nil { + return nil, err + } return resp, nil } if firstErr == nil { From 5c582e96fdb65fb37251dfddd21a4c8d92cc4043 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 15 May 2024 13:06:50 +0300 Subject: [PATCH 1093/1943] [#1136] metabase: Fix creation of `ECInfoError` Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/metabase/get.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index aa19502e8..355f62933 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -198,7 +198,7 @@ func getECInfoError(tx *bbolt.Tx, cnr cid.ID, data []byte) error { key := data[offset : offset+objectKeySize] // check in primary index ojbData := getFromBucket(tx, primaryBucketName(cnr, make([]byte, bucketKeySize)), key) - if len(data) != 0 { + if len(ojbData) != 0 { obj := objectSDK.New() if err := obj.Unmarshal(ojbData); err != nil { return err From f3e09cb09b78c08dab1fb904e22243f11ddd4731 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 15 May 2024 13:23:59 +0300 Subject: [PATCH 1094/1943] [#1135] sdnotify: Send MONOTONIC_USEC on reload Fixes #1135 Signed-off-by: Evgenii Stratonikov --- pkg/util/sdnotify/clock.go | 10 ++++++++++ pkg/util/sdnotify/clock.s | 2 ++ pkg/util/sdnotify/sdnotify.go | 15 +++++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 pkg/util/sdnotify/clock.go create mode 100644 pkg/util/sdnotify/clock.s diff --git a/pkg/util/sdnotify/clock.go b/pkg/util/sdnotify/clock.go new file mode 100644 index 000000000..f5419d027 --- /dev/null +++ b/pkg/util/sdnotify/clock.go @@ -0,0 +1,10 @@ +package sdnotify + +import ( + // For go:linkname to work. + _ "unsafe" +) + +//go:noescape +//go:linkname nanotime runtime.nanotime +func nanotime() int64 diff --git a/pkg/util/sdnotify/clock.s b/pkg/util/sdnotify/clock.s new file mode 100644 index 000000000..ad033ff4f --- /dev/null +++ b/pkg/util/sdnotify/clock.s @@ -0,0 +1,2 @@ +// The file is intentionally empty. +// It is a workaround for https://github.com/golang/go/issues/15006 diff --git a/pkg/util/sdnotify/sdnotify.go b/pkg/util/sdnotify/sdnotify.go index fe4ad1c3e..16a3f11c1 100644 --- a/pkg/util/sdnotify/sdnotify.go +++ b/pkg/util/sdnotify/sdnotify.go @@ -38,6 +38,21 @@ func InitSocket() error { // FlagAndStatus sends systemd a combination of a // well-known status and STATUS=%s{status}, separated by newline. func FlagAndStatus(status string) error { + if status == ReloadingEnabled { + // From https://www.man7.org/linux/man-pages/man5/systemd.service.5.html + // + // When initiating the reload process the service is + // expected to reply with a notification message via + // sd_notify(3) that contains the "RELOADING=1" field in + // combination with "MONOTONIC_USEC=" set to the current + // monotonic time (i.e. CLOCK_MONOTONIC in + // clock_gettime(2)) in μs, formatted as decimal string. + // Once reloading is complete another notification message + // must be sent, containing "READY=1". + // + // For MONOTONIC_USEC format refer to https://www.man7.org/linux/man-pages/man3/sd_notify.3.html + status += fmt.Sprintf("\nMONOTONIC_USEC=%d", uint64(nanotime())/1e3 /* microseconds in nanoseconds */) + } status += "\nSTATUS=" + strings.TrimSuffix(status, "=1") return Send(status) } From b078fe5ba10e110d6323b49fd48974d869cd3e66 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 16 May 2024 12:11:57 +0300 Subject: [PATCH 1095/1943] [#1092] control: Move SignMessage to separate package Signed-off-by: Alexander Chuprov --- .../modules/control/synchronize_tree.go | 4 +- cmd/frostfs-cli/modules/control/util.go | 6 +-- .../control/server/ctrlmessage/sign.go | 44 +++++++++++++++++++ pkg/services/control/server/detach_shards.go | 3 +- pkg/services/control/server/doctor.go | 3 +- pkg/services/control/server/evacuate.go | 5 ++- pkg/services/control/server/evacuate_async.go | 9 ++-- pkg/services/control/server/flush_cache.go | 3 +- pkg/services/control/server/gc.go | 3 +- pkg/services/control/server/healthcheck.go | 3 +- pkg/services/control/server/list_shards.go | 3 +- pkg/services/control/server/policy_engine.go | 13 +++--- .../control/server/seal_writecache.go | 3 +- .../control/server/set_netmap_status.go | 3 +- pkg/services/control/server/set_shard_mode.go | 3 +- pkg/services/control/server/sign.go | 40 +---------------- .../control/server/syncronize_tree.go | 3 +- 17 files changed, 86 insertions(+), 65 deletions(-) create mode 100644 pkg/services/control/server/ctrlmessage/sign.go diff --git a/cmd/frostfs-cli/modules/control/synchronize_tree.go b/cmd/frostfs-cli/modules/control/synchronize_tree.go index 2287344d6..5f2e4da96 100644 --- a/cmd/frostfs-cli/modules/control/synchronize_tree.go +++ b/cmd/frostfs-cli/modules/control/synchronize_tree.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - controlSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/spf13/cobra" ) @@ -60,7 +60,7 @@ func synchronizeTree(cmd *cobra.Command, _ []string) { }, } - err := controlSvc.SignMessage(pk, req) + err := ctrlmessage.Sign(pk, req) commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) cli := getClient(cmd, pk) diff --git a/cmd/frostfs-cli/modules/control/util.go b/cmd/frostfs-cli/modules/control/util.go index ffaceff13..c0577ac0c 100644 --- a/cmd/frostfs-cli/modules/control/util.go +++ b/cmd/frostfs-cli/modules/control/util.go @@ -8,7 +8,7 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - controlSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" "github.com/spf13/cobra" @@ -33,8 +33,8 @@ func initControlIRFlags(cmd *cobra.Command) { ff.Uint32(irFlagNameVUB, 0, "Valid until block value for notary transaction") } -func signRequest(cmd *cobra.Command, pk *ecdsa.PrivateKey, req controlSvc.SignedMessage) { - err := controlSvc.SignMessage(pk, req) +func signRequest(cmd *cobra.Command, pk *ecdsa.PrivateKey, req ctrlmessage.SignedMessage) { + err := ctrlmessage.Sign(pk, req) commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) } diff --git a/pkg/services/control/server/ctrlmessage/sign.go b/pkg/services/control/server/ctrlmessage/sign.go new file mode 100644 index 000000000..31425b337 --- /dev/null +++ b/pkg/services/control/server/ctrlmessage/sign.go @@ -0,0 +1,44 @@ +package ctrlmessage + +import ( + "crypto/ecdsa" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" + frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" +) + +type SignedMessage interface { + ReadSignedData([]byte) ([]byte, error) + GetSignature() *control.Signature + SetSignature(*control.Signature) +} + +// Sign signs Control service ctrlmessage with private key. +func Sign(key *ecdsa.PrivateKey, msg SignedMessage) error { + binBody, err := msg.ReadSignedData(nil) + if err != nil { + return fmt.Errorf("marshal request body: %w", err) + } + + var sig frostfscrypto.Signature + + err = sig.Calculate(frostfsecdsa.Signer(*key), binBody) + if err != nil { + return fmt.Errorf("calculate signature: %w", err) + } + + // TODO(@cthulhu-rider): #468 use Signature ctrlmessage from FrostFS API to avoid conversion + var sigV2 refs.Signature + sig.WriteToV2(&sigV2) + + var sigControl control.Signature + sigControl.SetKey(sigV2.GetKey()) + sigControl.SetSign(sigV2.GetSign()) + + msg.SetSignature(&sigControl) + + return nil +} diff --git a/pkg/services/control/server/detach_shards.go b/pkg/services/control/server/detach_shards.go index c8bea97b4..a4111bddb 100644 --- a/pkg/services/control/server/detach_shards.go +++ b/pkg/services/control/server/detach_shards.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -29,7 +30,7 @@ func (s *Server) DetachShards(_ context.Context, req *control.DetachShardsReques Body: &control.DetachShardsResponse_Body{}, } - if err = SignMessage(s.key, resp); err != nil { + if err = ctrlmessage.Sign(s.key, resp); err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/doctor.go b/pkg/services/control/server/doctor.go index 707a6c983..80041de44 100644 --- a/pkg/services/control/server/doctor.go +++ b/pkg/services/control/server/doctor.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -29,7 +30,7 @@ func (s *Server) Doctor(ctx context.Context, req *control.DoctorRequest) (*contr resp := &control.DoctorResponse{Body: &control.DoctorResponse_Body{}} - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index 462ab211e..3b65bff2c 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" @@ -49,7 +50,7 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe }, } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -131,7 +132,7 @@ func (s *Server) replicateTreeToNode(ctx context.Context, forest pilorama.Forest err = tree.SignMessage(req, s.key) if err != nil { - return fmt.Errorf("can't sign apply request: %w", err) + return fmt.Errorf("can't message apply request: %w", err) } err = s.treeService.ReplicateTreeOp(ctx, node, req) diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index 42ae2635b..b829573ec 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -43,7 +44,7 @@ func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartSha Body: &control.StartShardEvacuationResponse_Body{}, } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -70,7 +71,7 @@ func (s *Server) GetShardEvacuationStatus(ctx context.Context, req *control.GetS return nil, err } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -96,7 +97,7 @@ func (s *Server) StopShardEvacuation(ctx context.Context, req *control.StopShard Body: &control.StopShardEvacuationResponse_Body{}, } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -122,7 +123,7 @@ func (s *Server) ResetShardEvacuationStatus(ctx context.Context, req *control.Re Body: &control.ResetShardEvacuationStatusResponse_Body{}, } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/flush_cache.go b/pkg/services/control/server/flush_cache.go index 67ffa1f2c..031002d71 100644 --- a/pkg/services/control/server/flush_cache.go +++ b/pkg/services/control/server/flush_cache.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -28,7 +29,7 @@ func (s *Server) FlushCache(ctx context.Context, req *control.FlushCacheRequest) resp := &control.FlushCacheResponse{Body: &control.FlushCacheResponse_Body{}} - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/gc.go b/pkg/services/control/server/gc.go index d382dd7e5..d9fefc38e 100644 --- a/pkg/services/control/server/gc.go +++ b/pkg/services/control/server/gc.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -58,7 +59,7 @@ func (s *Server) DropObjects(ctx context.Context, req *control.DropObjectsReques resp.SetBody(body) // sign the response - if err := SignMessage(s.key, resp); err != nil { + if err := ctrlmessage.Sign(s.key, resp); err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/healthcheck.go b/pkg/services/control/server/healthcheck.go index 9e87caa49..121c51280 100644 --- a/pkg/services/control/server/healthcheck.go +++ b/pkg/services/control/server/healthcheck.go @@ -4,6 +4,7 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -27,7 +28,7 @@ func (s *Server) HealthCheck(_ context.Context, req *control.HealthCheckRequest) body.SetHealthStatus(s.healthChecker.HealthStatus()) // sign the response - if err := SignMessage(s.key, resp); err != nil { + if err := ctrlmessage.Sign(s.key, resp); err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/list_shards.go b/pkg/services/control/server/list_shards.go index a020547a2..d6531b947 100644 --- a/pkg/services/control/server/list_shards.go +++ b/pkg/services/control/server/list_shards.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -59,7 +60,7 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) ( body.SetShards(shardInfos) // sign the response - if err := SignMessage(s.key, resp); err != nil { + if err := ctrlmessage.Sign(s.key, resp); err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 7ec3d58ac..e2e1fd731 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ape" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "google.golang.org/grpc/codes" @@ -99,7 +100,7 @@ func (s *Server) AddChainLocalOverride(_ context.Context, req *control.AddChainL ChainId: []byte(chain.ID), }, } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -125,7 +126,7 @@ func (s *Server) GetChainLocalOverride(_ context.Context, req *control.GetChainL Chain: chain.Bytes(), }, } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -156,7 +157,7 @@ func (s *Server) ListChainLocalOverrides(_ context.Context, req *control.ListCha Chains: serializedChains, }, } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -181,7 +182,7 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove resp := &control.RemoveChainLocalOverrideResponse{ Body: &control.RemoveChainLocalOverrideResponse_Body{}, } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -206,7 +207,7 @@ func (s *Server) RemoveChainLocalOverridesByTarget(_ context.Context, req *contr resp := &control.RemoveChainLocalOverridesByTargetResponse{ Body: &control.RemoveChainLocalOverridesByTargetResponse_Body{}, } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -237,7 +238,7 @@ func (s *Server) ListTargetsLocalOverrides(_ context.Context, req *control.ListT Targets: targets, }, } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/seal_writecache.go b/pkg/services/control/server/seal_writecache.go index 5c39cf484..e3f8b8caf 100644 --- a/pkg/services/control/server/seal_writecache.go +++ b/pkg/services/control/server/seal_writecache.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -40,7 +41,7 @@ func (s *Server) SealWriteCache(ctx context.Context, req *control.SealWriteCache } } - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, err } diff --git a/pkg/services/control/server/set_netmap_status.go b/pkg/services/control/server/set_netmap_status.go index d4a856952..3fd69df12 100644 --- a/pkg/services/control/server/set_netmap_status.go +++ b/pkg/services/control/server/set_netmap_status.go @@ -4,6 +4,7 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -44,7 +45,7 @@ func (s *Server) SetNetmapStatus(_ context.Context, req *control.SetNetmapStatus resp.SetBody(body) // sign the response - if err := SignMessage(s.key, resp); err != nil { + if err := ctrlmessage.Sign(s.key, resp); err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/set_shard_mode.go b/pkg/services/control/server/set_shard_mode.go index 18a7eb327..52835c41d 100644 --- a/pkg/services/control/server/set_shard_mode.go +++ b/pkg/services/control/server/set_shard_mode.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -50,7 +51,7 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques resp.SetBody(body) // sign the response - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/server/sign.go b/pkg/services/control/server/sign.go index acc405821..514af273f 100644 --- a/pkg/services/control/server/sign.go +++ b/pkg/services/control/server/sign.go @@ -2,26 +2,17 @@ package control import ( "bytes" - "crypto/ecdsa" "errors" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" - frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" ) -// SignedMessage is an interface of Control service message. -type SignedMessage interface { - ReadSignedData([]byte) ([]byte, error) - GetSignature() *control.Signature - SetSignature(*control.Signature) -} - var errDisallowedKey = errors.New("key is not in the allowed list") -func (s *Server) isValidRequest(req SignedMessage) error { +func (s *Server) isValidRequest(req ctrlmessage.SignedMessage) error { sign := req.GetSignature() if sign == nil { // TODO(@cthulhu-rider): #468 use "const" error @@ -68,30 +59,3 @@ func (s *Server) isValidRequest(req SignedMessage) error { return nil } - -// SignMessage signs Control service message with private key. -func SignMessage(key *ecdsa.PrivateKey, msg SignedMessage) error { - binBody, err := msg.ReadSignedData(nil) - if err != nil { - return fmt.Errorf("marshal request body: %w", err) - } - - var sig frostfscrypto.Signature - - err = sig.Calculate(frostfsecdsa.Signer(*key), binBody) - if err != nil { - return fmt.Errorf("calculate signature: %w", err) - } - - // TODO(@cthulhu-rider): #468 use Signature message from FrostFS API to avoid conversion - var sigV2 refs.Signature - sig.WriteToV2(&sigV2) - - var sigControl control.Signature - sigControl.SetKey(sigV2.GetKey()) - sigControl.SetSign(sigV2.GetSign()) - - msg.SetSignature(&sigControl) - - return nil -} diff --git a/pkg/services/control/server/syncronize_tree.go b/pkg/services/control/server/syncronize_tree.go index 678f87d0a..b2a966b2c 100644 --- a/pkg/services/control/server/syncronize_tree.go +++ b/pkg/services/control/server/syncronize_tree.go @@ -4,6 +4,7 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -42,7 +43,7 @@ func (s *Server) SynchronizeTree(ctx context.Context, req *control.SynchronizeTr resp := new(control.SynchronizeTreeResponse) resp.SetBody(new(control.SynchronizeTreeResponse_Body)) - err = SignMessage(s.key, resp) + err = ctrlmessage.Sign(s.key, resp) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } From 2f04ce2f7946d1bcafe0280488f42806f106df8c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 15 May 2024 16:04:01 +0300 Subject: [PATCH 1096/1943] [#1138] adm: Parallelize `generate-alphabet` command This is the longest test in our suite, try to help CI a bit. Before: ``` $ go test -run=TestGenerateAlphabet -count=10 . ok git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/generate 45.400s ``` After: ``` $ go test -run=TestGenerateAlphabet -count=10 . ok git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/generate 33.267s ``` Signed-off-by: Evgenii Stratonikov --- .../modules/morph/generate/generate.go | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go index 8ecc9f1b6..c7de599e5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go @@ -65,34 +65,42 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er pubs := make(keys.PublicKeys, size) passwords := make([]string, size) + var errG errgroup.Group + for i := range wallets { password, err := config.GetPassword(v, innerring.GlagoliticLetter(i).String()) if err != nil { return nil, fmt.Errorf("can't fetch password: %w", err) } - p := filepath.Join(walletDir, innerring.GlagoliticLetter(i).String()+".json") - f, err := os.OpenFile(p, os.O_CREATE, 0o644) - if err != nil { - return nil, fmt.Errorf("can't create wallet file: %w", err) - } - if err := f.Close(); err != nil { - return nil, fmt.Errorf("can't close wallet file: %w", err) - } - w, err := wallet.NewWallet(p) - if err != nil { - return nil, fmt.Errorf("can't create wallet: %w", err) - } - if err := w.CreateAccount(constants.SingleAccountName, password); err != nil { - return nil, fmt.Errorf("can't create account: %w", err) - } + i := i + errG.Go(func() error { + p := filepath.Join(walletDir, innerring.GlagoliticLetter(i).String()+".json") + f, err := os.OpenFile(p, os.O_CREATE, 0o644) + if err != nil { + return fmt.Errorf("can't create wallet file: %w", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("can't close wallet file: %w", err) + } + w, err := wallet.NewWallet(p) + if err != nil { + return fmt.Errorf("can't create wallet: %w", err) + } + if err := w.CreateAccount(constants.SingleAccountName, password); err != nil { + return fmt.Errorf("can't create account: %w", err) + } - passwords[i] = password - wallets[i] = w - pubs[i] = w.Accounts[0].PrivateKey().PublicKey() + passwords[i] = password + wallets[i] = w + pubs[i] = w.Accounts[0].PrivateKey().PublicKey() + return nil + }) } - var errG errgroup.Group + if err := errG.Wait(); err != nil { + return nil, err + } // Create committee account with N/2+1 multi-signature. majCount := smartcontract.GetMajorityHonestNodeCount(size) From af57d5a6a12f1be58c836e079ff3fc73acca39b3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 15 May 2024 16:12:41 +0300 Subject: [PATCH 1097/1943] [#1138] adm/test: Use `--size=1` for negative tests Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/generate/generate_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go index d4a4fee5d..1dd6420eb 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go @@ -23,8 +23,6 @@ import ( ) func TestGenerateAlphabet(t *testing.T) { - const size = 4 - walletDir := t.TempDir() buf := setupTestTerminal(t) @@ -55,13 +53,13 @@ func TestGenerateAlphabet(t *testing.T) { t.Run("no password for contract group wallet", func(t *testing.T) { buf.Reset() v.Set(commonflags.AlphabetWalletsFlag, walletDir) - require.NoError(t, cmd.Flags().Set(commonflags.AlphabetSizeFlag, strconv.FormatUint(size, 10))) - for i := uint64(0); i < size; i++ { - buf.WriteString(strconv.FormatUint(i, 10) + "\r") - } + require.NoError(t, cmd.Flags().Set(commonflags.AlphabetSizeFlag, "1")) + buf.WriteString("pass\r") require.Error(t, AlphabetCreds(cmd, nil)) }) + const size = 4 + buf.Reset() v.Set(commonflags.AlphabetWalletsFlag, walletDir) require.NoError(t, GenerateAlphabetCmd.Flags().Set(commonflags.AlphabetSizeFlag, strconv.FormatUint(size, 10))) From d45d086acd0f7b60a288bc69fd70ec8513ea23b7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 13 May 2024 16:50:21 +0300 Subject: [PATCH 1098/1943] [#1129] policer: Add EC chunk replication Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/keyspaceiterator.go | 2 +- internal/logs/logs.go | 1 + pkg/core/object/{address.go => info.go} | 15 +- pkg/local_object_storage/engine/evacuate.go | 2 +- pkg/local_object_storage/engine/list.go | 6 +- pkg/local_object_storage/engine/list_test.go | 6 +- pkg/local_object_storage/metabase/list.go | 23 ++- .../metabase/list_test.go | 14 +- pkg/local_object_storage/shard/list.go | 4 +- pkg/services/control/server/evacuate.go | 2 +- pkg/services/policer/check.go | 41 +++--- pkg/services/policer/ec.go | 137 ++++++++++++++++++ pkg/services/policer/option.go | 4 +- pkg/services/policer/policer_test.go | 64 ++++++-- pkg/services/replicator/process.go | 4 +- 15 files changed, 263 insertions(+), 62 deletions(-) rename pkg/core/object/{address.go => info.go} (62%) create mode 100644 pkg/services/policer/ec.go diff --git a/cmd/frostfs-node/keyspaceiterator.go b/cmd/frostfs-node/keyspaceiterator.go index e7214aacb..09a8f7f73 100644 --- a/cmd/frostfs-node/keyspaceiterator.go +++ b/cmd/frostfs-node/keyspaceiterator.go @@ -13,7 +13,7 @@ type keySpaceIterator struct { cur *engine.Cursor } -func (it *keySpaceIterator) Next(ctx context.Context, batchSize uint32) ([]objectcore.AddressWithType, error) { +func (it *keySpaceIterator) Next(ctx context.Context, batchSize uint32) ([]objectcore.Info, error) { var prm engine.ListWithCursorPrm prm.WithCursor(it.cur) prm.WithCount(batchSize) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 4576e2e59..f51c72e73 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -529,4 +529,5 @@ const ( EngineRefillFailedToGetObjectsCount = "failed to get blobstor objects count, no resync percent estimation is available" ECFailedToSendToContainerNode = "failed to send EC object to container node" ECFailedToSaveECPart = "failed to save EC part" + PolicerNodeIsNotContainerNodeForECObject = "current node is not container node for EC object" ) diff --git a/pkg/core/object/address.go b/pkg/core/object/info.go similarity index 62% rename from pkg/core/object/address.go rename to pkg/core/object/info.go index a25d853eb..67c9a3188 100644 --- a/pkg/core/object/address.go +++ b/pkg/core/object/info.go @@ -7,14 +7,21 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -// AddressWithType groups object address with its FrostFS -// object type. -type AddressWithType struct { +type ECInfo struct { + ParentID oid.ID + Index uint32 + Total uint32 +} + +// Info groups object address with its FrostFS +// object info. +type Info struct { Address oid.Address Type objectSDK.Type IsLinkingObject bool + ECInfo *ECInfo } -func (v AddressWithType) String() string { +func (v Info) String() string { return fmt.Sprintf("address: %s, type: %s, is linking: %t", v.Address, v.Type, v.IsLinkingObject) } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 81fe47e65..87542eec1 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -622,7 +622,7 @@ func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) return shards, nil } -func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, toEvacuate []object.AddressWithType, prm EvacuateShardPrm, res *EvacuateShardRes, +func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, toEvacuate []object.Info, prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateObjects", diff --git a/pkg/local_object_storage/engine/list.go b/pkg/local_object_storage/engine/list.go index 7245caeeb..cb3830b7c 100644 --- a/pkg/local_object_storage/engine/list.go +++ b/pkg/local_object_storage/engine/list.go @@ -68,12 +68,12 @@ func (p *ListWithCursorPrm) WithCursor(cursor *Cursor) { // ListWithCursorRes contains values returned from ListWithCursor operation. type ListWithCursorRes struct { - addrList []objectcore.AddressWithType + addrList []objectcore.Info cursor *Cursor } // AddressList returns addresses selected by ListWithCursor operation. -func (l ListWithCursorRes) AddressList() []objectcore.AddressWithType { +func (l ListWithCursorRes) AddressList() []objectcore.Info { return l.addrList } @@ -98,7 +98,7 @@ func (l ListWithCursorRes) Cursor() *Cursor { // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. func (e *StorageEngine) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (ListWithCursorRes, error) { - result := make([]objectcore.AddressWithType, 0, prm.count) + result := make([]objectcore.Info, 0, prm.count) // Set initial cursors cursor := prm.cursor diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 4fc9569c7..dd8a2e8a0 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -76,8 +76,8 @@ func TestListWithCursor(t *testing.T) { require.NoError(t, e.Close(context.Background())) }() - expected := make([]object.AddressWithType, 0, tt.objectNum) - got := make([]object.AddressWithType, 0, tt.objectNum) + expected := make([]object.Info, 0, tt.objectNum) + got := make([]object.Info, 0, tt.objectNum) for i := 0; i < tt.objectNum; i++ { containerID := cidtest.ID() @@ -88,7 +88,7 @@ func TestListWithCursor(t *testing.T) { err := e.Put(context.Background(), prm) require.NoError(t, err) - expected = append(expected, object.AddressWithType{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)}) + expected = append(expected, object.Info{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)}) } var prm ListWithCursorPrm diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index f7a314452..544b2e666 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -47,12 +47,12 @@ func (l *ListPrm) SetCursor(cursor *Cursor) { // ListRes contains values returned from ListWithCursor operation. type ListRes struct { - addrList []objectcore.AddressWithType + addrList []objectcore.Info cursor *Cursor } // AddressList returns addresses selected by ListWithCursor operation. -func (l ListRes) AddressList() []objectcore.AddressWithType { +func (l ListRes) AddressList() []objectcore.Info { return l.addrList } @@ -89,7 +89,7 @@ func (db *DB) ListWithCursor(ctx context.Context, prm ListPrm) (res ListRes, err return res, ErrDegradedMode } - result := make([]objectcore.AddressWithType, 0, prm.count) + result := make([]objectcore.Info, 0, prm.count) err = db.boltDB.View(func(tx *bbolt.Tx) error { res.addrList, res.cursor, err = db.listWithCursor(tx, result, prm.count, prm.cursor) @@ -99,7 +99,7 @@ func (db *DB) ListWithCursor(ctx context.Context, prm ListPrm) (res ListRes, err return res, metaerr.Wrap(err) } -func (db *DB) listWithCursor(tx *bbolt.Tx, result []objectcore.AddressWithType, count int, cursor *Cursor) ([]objectcore.AddressWithType, *Cursor, error) { +func (db *DB) listWithCursor(tx *bbolt.Tx, result []objectcore.Info, count int, cursor *Cursor) ([]objectcore.Info, *Cursor, error) { threshold := cursor == nil // threshold is a flag to ignore cursor var bucketName []byte var err error @@ -183,11 +183,11 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket graveyardBkt, garbageBkt *bbolt.Bucket, // cached graveyard buckets cidRaw []byte, // container ID prefix, optimization cnt cid.ID, // container ID - to []objectcore.AddressWithType, // listing result + to []objectcore.Info, // listing result limit int, // stop listing at `limit` items in result cursor *Cursor, // start from cursor object threshold bool, // ignore cursor and start immediately -) ([]objectcore.AddressWithType, []byte, *Cursor, error) { +) ([]objectcore.Info, []byte, *Cursor, error) { if cursor == nil { cursor = new(Cursor) } @@ -219,18 +219,27 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket } var isLinkingObj bool + var ecInfo *objectcore.ECInfo if objType == objectSDK.TypeRegular { var o objectSDK.Object if err := o.Unmarshal(v); err != nil { return nil, nil, nil, err } isLinkingObj = isLinkObject(&o) + ecHeader := o.ECHeader() + if ecHeader != nil { + ecInfo = &objectcore.ECInfo{ + ParentID: ecHeader.Parent(), + Index: ecHeader.Index(), + Total: ecHeader.Total(), + } + } } var a oid.Address a.SetContainer(cnt) a.SetObject(obj) - to = append(to, objectcore.AddressWithType{Address: a, Type: objType, IsLinkingObject: isLinkingObj}) + to = append(to, objectcore.Info{Address: a, Type: objType, IsLinkingObject: isLinkingObj, ECInfo: ecInfo}) count++ } diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index e1ccb4e06..a92e2eff4 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -77,7 +77,7 @@ func TestLisObjectsWithCursor(t *testing.T) { total = containers * 4 // regular + ts + child + lock ) - expected := make([]object.AddressWithType, 0, total) + expected := make([]object.Info, 0, total) // fill metabase with objects for i := 0; i < containers; i++ { @@ -88,21 +88,21 @@ func TestLisObjectsWithCursor(t *testing.T) { obj.SetType(objectSDK.TypeRegular) err := putBig(db, obj) require.NoError(t, err) - expected = append(expected, object.AddressWithType{Address: object.AddressOf(obj), Type: objectSDK.TypeRegular}) + expected = append(expected, object.Info{Address: object.AddressOf(obj), Type: objectSDK.TypeRegular}) // add one tombstone obj = testutil.GenerateObjectWithCID(containerID) obj.SetType(objectSDK.TypeTombstone) err = putBig(db, obj) require.NoError(t, err) - expected = append(expected, object.AddressWithType{Address: object.AddressOf(obj), Type: objectSDK.TypeTombstone}) + expected = append(expected, object.Info{Address: object.AddressOf(obj), Type: objectSDK.TypeTombstone}) // add one lock obj = testutil.GenerateObjectWithCID(containerID) obj.SetType(objectSDK.TypeLock) err = putBig(db, obj) require.NoError(t, err) - expected = append(expected, object.AddressWithType{Address: object.AddressOf(obj), Type: objectSDK.TypeLock}) + expected = append(expected, object.Info{Address: object.AddressOf(obj), Type: objectSDK.TypeLock}) // add one inhumed (do not include into expected) obj = testutil.GenerateObjectWithCID(containerID) @@ -124,12 +124,12 @@ func TestLisObjectsWithCursor(t *testing.T) { child.SetSplitID(splitID) err = putBig(db, child) require.NoError(t, err) - expected = append(expected, object.AddressWithType{Address: object.AddressOf(child), Type: objectSDK.TypeRegular}) + expected = append(expected, object.Info{Address: object.AddressOf(child), Type: objectSDK.TypeRegular}) } t.Run("success with various count", func(t *testing.T) { for countPerReq := 1; countPerReq <= total; countPerReq++ { - got := make([]object.AddressWithType, 0, total) + got := make([]object.Info, 0, total) res, cursor, err := metaListWithCursor(db, uint32(countPerReq), nil) require.NoError(t, err, "count:%d", countPerReq) @@ -211,7 +211,7 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) { } } -func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]object.AddressWithType, *meta.Cursor, error) { +func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]object.Info, *meta.Cursor, error) { var listPrm meta.ListPrm listPrm.SetCount(count) listPrm.SetCursor(cursor) diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index d6e4d7e50..08ea81a0c 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -42,7 +42,7 @@ type ListWithCursorPrm struct { // ListWithCursorRes contains values returned from ListWithCursor operation. type ListWithCursorRes struct { - addrList []objectcore.AddressWithType + addrList []objectcore.Info cursor *Cursor } @@ -59,7 +59,7 @@ func (p *ListWithCursorPrm) WithCursor(cursor *Cursor) { } // AddressList returns addresses selected by ListWithCursor operation. -func (r ListWithCursorRes) AddressList() []objectcore.AddressWithType { +func (r ListWithCursorRes) AddressList() []objectcore.Info { return r.addrList } diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index 3b65bff2c..794bc199a 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -77,7 +77,7 @@ func (s *Server) replicateObject(ctx context.Context, addr oid.Address, obj *obj Obj: obj, Nodes: nodes, } - s.replicator.HandleTask(ctx, task, &res) + s.replicator.HandleReplicationTask(ctx, task, &res) if res.count == 0 { return errors.New("object was not replicated") diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index d68d99d18..2153275cc 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -18,19 +18,15 @@ import ( "go.uber.org/zap" ) -func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.AddressWithType) error { - addr := addrWithType.Address - idCnr := addr.Container() - idObj := addr.Object() - - cnr, err := p.cnrSrc.Get(idCnr) +func (p *Policer) processObject(ctx context.Context, objInfo objectcore.Info) error { + cnr, err := p.cnrSrc.Get(objInfo.Address.Container()) if err != nil { if client.IsErrContainerNotFound(err) { - existed, errWasRemoved := containercore.WasRemoved(p.cnrSrc, idCnr) + existed, errWasRemoved := containercore.WasRemoved(p.cnrSrc, objInfo.Address.Container()) if errWasRemoved != nil { return fmt.Errorf("%s: %w", logs.PolicerCouldNotConfirmContainerRemoval, errWasRemoved) } else if existed { - err := p.buryFn(ctx, addrWithType.Address) + err := p.buryFn(ctx, objInfo.Address) if err != nil { return fmt.Errorf("%s: %w", logs.PolicerCouldNotInhumeObjectWithMissingContainer, err) } @@ -41,11 +37,16 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add } policy := cnr.Value.PlacementPolicy() - if policycore.IsECPlacement(policy) { - // EC not supported yet by policer - return nil - } + if policycore.IsECPlacement(policy) { + return p.processECContainerObject(ctx, objInfo, policy) + } + return p.processRepContainerObject(ctx, objInfo, policy) +} + +func (p *Policer) processRepContainerObject(ctx context.Context, objInfo objectcore.Info, policy netmap.PlacementPolicy) error { + idObj := objInfo.Address.Object() + idCnr := objInfo.Address.Container() nn, err := p.placementBuilder.BuildPlacement(idCnr, &idObj, policy) if err != nil { return fmt.Errorf("%s: %w", logs.PolicerCouldNotBuildPlacementVectorForObject, err) @@ -68,15 +69,15 @@ func (p *Policer) processObject(ctx context.Context, addrWithType objectcore.Add default: } - p.processNodes(ctx, c, addrWithType, nn[i], policy.ReplicaDescriptor(i).NumberOfObjects(), checkedNodes) + p.processRepNodes(ctx, c, objInfo, nn[i], policy.ReplicaDescriptor(i).NumberOfObjects(), checkedNodes) } if !c.needLocalCopy && c.removeLocalCopy { p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, - zap.Stringer("object", addr), + zap.Stringer("object", objInfo.Address), ) - p.cbRedundantCopy(ctx, addr) + p.cbRedundantCopy(ctx, objInfo.Address) } return nil } @@ -89,16 +90,16 @@ type placementRequirements struct { removeLocalCopy bool } -func (p *Policer) processNodes(ctx context.Context, requirements *placementRequirements, addrWithType objectcore.AddressWithType, +func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRequirements, objInfo objectcore.Info, nodes []netmap.NodeInfo, shortage uint32, checkedNodes nodeCache, ) { - addr := addrWithType.Address - typ := addrWithType.Type + addr := objInfo.Address + typ := objInfo.Type // Number of copies that are stored on maintenance nodes. var uncheckedCopies int - if typ == objectSDK.TypeLock || typ == objectSDK.TypeTombstone || addrWithType.IsLinkingObject { + if typ == objectSDK.TypeLock || typ == objectSDK.TypeTombstone || objInfo.IsLinkingObject { // all nodes of a container must store the `LOCK`, `TOMBSTONE` and linking objects // for correct object removal protection: // - `LOCK`, `TOMBSTONE` and linking objects are broadcast on their PUT requests; @@ -196,7 +197,7 @@ func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address Nodes: nodes, } - p.replicator.HandleTask(ctx, task, checkedNodes) + p.replicator.HandleReplicationTask(ctx, task, checkedNodes) case uncheckedCopies > 0: // If we have more copies than needed, but some of them are from the maintenance nodes, diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go new file mode 100644 index 000000000..194b0ddbe --- /dev/null +++ b/pkg/services/policer/ec.go @@ -0,0 +1,137 @@ +package policer + +import ( + "context" + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "go.uber.org/zap" +) + +var errInvalidECPlacement = errors.New("invalid EC placement: EC placement must have one placement vector") + +func (p *Policer) processECContainerObject(ctx context.Context, objInfo objectcore.Info, policy netmap.PlacementPolicy) error { + if objInfo.ECInfo == nil { + return p.processECContainerRepObject(ctx, objInfo, policy) + } + return p.processECContainerECObject(ctx, objInfo, policy) +} + +// processECContainerRepObject processes non erasure coded objects in EC container: tombstones, locks and linking objects. +// All of them must be stored on all of the container nodes. +func (p *Policer) processECContainerRepObject(ctx context.Context, objInfo objectcore.Info, policy netmap.PlacementPolicy) error { + objID := objInfo.Address.Object() + nn, err := p.placementBuilder.BuildPlacement(objInfo.Address.Container(), &objID, policy) + if err != nil { + return fmt.Errorf("%s: %w", logs.PolicerCouldNotBuildPlacementVectorForObject, err) + } + if len(nn) != 1 { + return errInvalidECPlacement + } + + c := &placementRequirements{} + checkedNodes := newNodeCache() + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + rd := policy.ReplicaDescriptor(0) + // processNodes replaces rd.GetECDataCount() + rd.GetECParityCount() for len(nn[0]) for locks, tomstones and linking objects. + p.processRepNodes(ctx, c, objInfo, nn[0], rd.GetECDataCount()+rd.GetECParityCount(), checkedNodes) + + if !c.needLocalCopy && c.removeLocalCopy { + p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, + zap.Stringer("object", objInfo.Address), + ) + + p.cbRedundantCopy(ctx, objInfo.Address) + } + return nil +} + +func (p *Policer) processECContainerECObject(ctx context.Context, objInfo objectcore.Info, policy netmap.PlacementPolicy) error { + nn, err := p.placementBuilder.BuildPlacement(objInfo.Address.Container(), &objInfo.ECInfo.ParentID, policy) + if err != nil { + return fmt.Errorf("%s: %w", logs.PolicerCouldNotBuildPlacementVectorForObject, err) + } + if len(nn) != 1 { + return errInvalidECPlacement + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + validPlacement := p.processECChunk(ctx, objInfo, nn[0]) + if !validPlacement { + p.pullRequiredECChunks(ctx, objInfo, nn[0]) + } + return nil +} + +// processECChunk replicates EC chunk if needed. +// Returns True if current chunk should be stored on current node. +func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo) bool { + var removeLocalChunk bool + requiredNode := nodes[int(objInfo.ECInfo.Index)%(len(nodes))] + if p.cfg.netmapKeys.IsLocalKey(requiredNode.PublicKey()) { + // current node is required node, we are happy + return true + } + if requiredNode.IsMaintenance() { + // consider maintenance mode has object, but do not drop local copy + p.log.Debug(logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(requiredNode))) + return false + } + + callCtx, cancel := context.WithTimeout(ctx, p.headTimeout) + _, err := p.remoteHeader(callCtx, requiredNode, objInfo.Address) + cancel() + + if err == nil { + removeLocalChunk = true + } else if client.IsErrObjectNotFound(err) { + p.log.Debug(logs.PolicerShortageOfObjectCopiesDetected, zap.Stringer("object", objInfo.Address), zap.Uint32("shortage", 1)) + task := replicator.Task{ + NumCopies: 1, + Addr: objInfo.Address, + Nodes: []netmap.NodeInfo{requiredNode}, + } + p.replicator.HandleReplicationTask(ctx, task, newNodeCache()) + } else if isClientErrMaintenance(err) { + // consider maintenance mode has object, but do not drop local copy + p.log.Debug(logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(requiredNode))) + } else { + p.log.Error(logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, zap.Stringer("object", objInfo.Address), zap.String("error", err.Error())) + } + + if removeLocalChunk { + p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, zap.Stringer("object", objInfo.Address)) + p.cbRedundantCopy(ctx, objInfo.Address) + } + + return false +} + +func (p *Policer) pullRequiredECChunks(_ context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo) { + requiredChunkIndexes := make(map[uint32]struct{}) + for i, n := range nodes { + if p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) { + requiredChunkIndexes[uint32(i)] = struct{}{} + } + } + if len(requiredChunkIndexes) == 0 { + p.log.Info(logs.PolicerNodeIsNotContainerNodeForECObject, zap.Stringer("object", objInfo.ECInfo.ParentID)) + return + } +} diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 58c74ee1a..849e5ed8b 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -22,7 +22,7 @@ import ( // Note that the underlying implementation might be circular: i.e. it can restart // when the end of the key space is reached. type KeySpaceIterator interface { - Next(context.Context, uint32) ([]objectcore.AddressWithType, error) + Next(context.Context, uint32) ([]objectcore.Info, error) Rewind() } @@ -35,7 +35,7 @@ type BuryFunc func(context.Context, oid.Address) error // Replicator is the interface to a consumer of replication tasks. type Replicator interface { - HandleTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) + HandleReplicationTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) } // RemoteObjectHeaderFunc is the function to obtain HEAD info from a specific remote node. diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index be0974c39..2c70b1c04 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -26,7 +26,7 @@ import ( func TestBuryObjectWithoutContainer(t *testing.T) { // Key space addr := oidtest.Address() - objs := []objectcore.AddressWithType{ + objs := []objectcore.Info{ { Address: addr, Type: objectSDK.TypeRegular, @@ -78,6 +78,7 @@ func TestProcessObject(t *testing.T) { maintenanceNodes []int wantRemoveRedundant bool wantReplicateTo []int + ecInfo *objectcore.ECInfo }{ { desc: "1 copy already held by local node", @@ -144,6 +145,47 @@ func TestProcessObject(t *testing.T) { objHolders: []int{1}, maintenanceNodes: []int{2}, }, + { + desc: "lock object must be replicated to all EC nodes", + objType: objectSDK.TypeLock, + nodeCount: 3, + policy: `EC 1.1`, + placement: [][]int{{0, 1, 2}}, + wantReplicateTo: []int{1, 2}, + }, + { + desc: "tombstone object must be replicated to all EC nodes", + objType: objectSDK.TypeTombstone, + nodeCount: 3, + policy: `EC 1.1`, + placement: [][]int{{0, 1, 2}}, + wantReplicateTo: []int{1, 2}, + }, + { + desc: "EC chunk stored valid on current node", + objType: objectSDK.TypeRegular, + nodeCount: 2, + policy: `EC 1.1`, + placement: [][]int{{0}}, + ecInfo: &objectcore.ECInfo{ + ParentID: oidtest.ID(), + Index: 1, + Total: 2, + }, + }, + { + desc: "EC chunk must be replicated to other EC node", + objType: objectSDK.TypeRegular, + nodeCount: 2, + policy: `EC 1.1`, + placement: [][]int{{1}}, + wantReplicateTo: []int{1}, + ecInfo: &objectcore.ECInfo{ + ParentID: oidtest.ID(), + Index: 1, + Total: 2, + }, + }, } for i := range tests { @@ -173,6 +215,9 @@ func TestProcessObject(t *testing.T) { if cnr.Equals(addr.Container()) && obj != nil && obj.Equals(addr.Object()) { return placementVectors, nil } + if ti.ecInfo != nil && cnr.Equals(addr.Container()) && obj != nil && obj.Equals(ti.ecInfo.ParentID) { + return placementVectors, nil + } t.Errorf("unexpected placement build: cid=%v oid=%v", cnr, obj) return nil, errors.New("unexpected placement build") } @@ -238,9 +283,10 @@ func TestProcessObject(t *testing.T) { WithPool(testPool(t)), ) - addrWithType := objectcore.AddressWithType{ + addrWithType := objectcore.Info{ Address: addr, Type: ti.objType, + ECInfo: ti.ecInfo, } err := p.processObject(context.Background(), addrWithType) @@ -276,7 +322,7 @@ func TestProcessObjectError(t *testing.T) { WithPool(testPool(t)), ) - addrWithType := objectcore.AddressWithType{ + addrWithType := objectcore.Info{ Address: addr, } @@ -285,7 +331,7 @@ func TestProcessObjectError(t *testing.T) { func TestIteratorContract(t *testing.T) { addr := oidtest.Address() - objs := []objectcore.AddressWithType{{ + objs := []objectcore.Info{{ Address: addr, Type: objectSDK.TypeRegular, }} @@ -350,7 +396,7 @@ func testPool(t *testing.T) *ants.Pool { } type nextResult struct { - objs []objectcore.AddressWithType + objs []objectcore.Info err error } @@ -361,7 +407,7 @@ type predefinedIterator struct { calls []string } -func (it *predefinedIterator) Next(ctx context.Context, size uint32) ([]objectcore.AddressWithType, error) { +func (it *predefinedIterator) Next(ctx context.Context, size uint32) ([]objectcore.Info, error) { if it.pos == len(it.scenario) { close(it.finishCh) <-ctx.Done() @@ -380,11 +426,11 @@ func (it *predefinedIterator) Rewind() { // sliceKeySpaceIterator is a KeySpaceIterator backed by a slice. type sliceKeySpaceIterator struct { - objs []objectcore.AddressWithType + objs []objectcore.Info cur int } -func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectcore.AddressWithType, error) { +func (it *sliceKeySpaceIterator) Next(_ context.Context, size uint32) ([]objectcore.Info, error) { if it.cur >= len(it.objs) { return nil, engine.ErrEndOfListing } @@ -422,6 +468,6 @@ func (f announcedKeysFunc) IsLocalKey(k []byte) bool { return f(k) } // replicatorFunc is a Replicator backed by a function. type replicatorFunc func(context.Context, replicator.Task, replicator.TaskResult) -func (f replicatorFunc) HandleTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) { +func (f replicatorFunc) HandleReplicationTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) { f(ctx, task, res) } diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 3a46e5f04..06d41b74e 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -21,9 +21,9 @@ type TaskResult interface { SubmitSuccessfulReplication(netmap.NodeInfo) } -// HandleTask executes replication task inside invoking goroutine. +// HandleReplicationTask executes replication task inside invoking goroutine. // Passes all the nodes that accepted the replication to the TaskResult. -func (p *Replicator) HandleTask(ctx context.Context, task Task, res TaskResult) { +func (p *Replicator) HandleReplicationTask(ctx context.Context, task Task, res TaskResult) { p.metrics.IncInFlightRequest() defer p.metrics.DecInFlightRequest() defer func() { From cbe9757490a44436683a406f7917222190ab7dd6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 14 May 2024 14:43:21 +0300 Subject: [PATCH 1099/1943] [#1129] policer: Pull required EC chunks Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 16 +- internal/logs/logs.go | 8 +- pkg/services/object/get/remote_getter.go | 55 ++++ pkg/services/object/head/prm.go | 17 - pkg/services/object/head/remote.go | 18 +- pkg/services/policer/check.go | 2 +- pkg/services/policer/ec.go | 166 +++++++++- pkg/services/policer/ec_test.go | 402 +++++++++++++++++++++++ pkg/services/policer/option.go | 17 +- pkg/services/policer/policer_test.go | 31 +- pkg/services/replicator/process.go | 2 +- pkg/services/replicator/pull.go | 72 ++++ pkg/services/replicator/replicator.go | 9 + 13 files changed, 744 insertions(+), 71 deletions(-) create mode 100644 pkg/services/object/get/remote_getter.go delete mode 100644 pkg/services/object/head/prm.go create mode 100644 pkg/services/policer/ec_test.go create mode 100644 pkg/services/replicator/pull.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 833daf628..af185a8da 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -242,11 +242,20 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl placement.NewNetworkMapSourceBuilder(c.netMapSource), ), policer.WithRemoteObjectHeaderFunc( - func(ctx context.Context, ni netmapSDK.NodeInfo, a oid.Address) (*objectSDK.Object, error) { - prm := new(headsvc.RemoteHeadPrm).WithNodeInfo(ni).WithObjectAddress(a) + func(ctx context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) { + prm := new(headsvc.RemoteHeadPrm).WithNodeInfo(ni).WithObjectAddress(a).WithRaw(raw) return remoteHeader.Head(ctx, prm) }, ), + policer.WithLocalObjectHeaderFunc(func(ctx context.Context, a oid.Address) (*objectSDK.Object, error) { + var prm engine.HeadPrm + prm.WithAddress(a) + res, err := c.cfgObject.cfgLocalStorage.localStorage.Head(ctx, prm) + if err != nil { + return nil, err + } + return res.Header(), nil + }), policer.WithNetmapKeys(c), policer.WithHeadTimeout( policerconfig.HeadTimeout(c.appCfg), @@ -297,6 +306,9 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa replicator.WithRemoteSender( putsvc.NewRemoteSender(keyStorage, cache), ), + replicator.WithRemoteGetter( + getsvc.NewRemoteGetter(c.clientCache, c.netMapSource, keyStorage), + ), replicator.WithMetrics(c.metricsCollector.Replicator()), ) } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index f51c72e73..2e89e4304 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -529,5 +529,11 @@ const ( EngineRefillFailedToGetObjectsCount = "failed to get blobstor objects count, no resync percent estimation is available" ECFailedToSendToContainerNode = "failed to send EC object to container node" ECFailedToSaveECPart = "failed to save EC part" - PolicerNodeIsNotContainerNodeForECObject = "current node is not container node for EC object" + PolicerNodeIsNotECObjectNode = "current node is not EC object node" + PolicerFailedToGetLocalECChunks = "failed to get local EC chunks" + PolicerMissingECChunk = "failed to find EC chunk on any of the nodes" + PolicerFailedToDecodeECChunkID = "failed to decode EC chunk ID" + PolicerDifferentObjectIDForTheSameECChunk = "different object IDs for the same EC chunk" + ReplicatorCouldNotGetObjectFromRemoteStorage = "could not get object from remote storage" + ReplicatorCouldNotPutObjectToLocalStorage = "could not put object to local storage" ) diff --git a/pkg/services/object/get/remote_getter.go b/pkg/services/object/get/remote_getter.go new file mode 100644 index 000000000..0df67dec9 --- /dev/null +++ b/pkg/services/object/get/remote_getter.go @@ -0,0 +1,55 @@ +package getsvc + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +type RemoteGetPrm struct { + Address oid.Address + Node netmapSDK.NodeInfo +} + +type RemoteGetter struct { + s remoteStorageConstructor + es epochSource + ks keyStorage +} + +func (g *RemoteGetter) Get(ctx context.Context, prm RemoteGetPrm) (*objectSDK.Object, error) { + var nodeInfo client.NodeInfo + if err := client.NodeInfoFromRawNetmapElement(&nodeInfo, netmapCore.Node(prm.Node)); err != nil { + return nil, err + } + rs, err := g.s.Get(nodeInfo) + if err != nil { + return nil, err + } + epoch, err := g.es.Epoch() + if err != nil { + return nil, err + } + key, err := g.ks.GetKey(nil) + if err != nil { + return nil, err + } + r := RemoteRequestParams{ + Epoch: epoch, + TTL: 1, + PrivateKey: key, + } + return rs.Get(ctx, prm.Address, r) +} + +func NewRemoteGetter(cc clientConstructor, es epochSource, ks keyStorage) *RemoteGetter { + return &RemoteGetter{ + s: &multiclientRemoteStorageConstructor{clientConstructor: cc}, + es: es, + ks: ks, + } +} diff --git a/pkg/services/object/head/prm.go b/pkg/services/object/head/prm.go deleted file mode 100644 index 5566e48fe..000000000 --- a/pkg/services/object/head/prm.go +++ /dev/null @@ -1,17 +0,0 @@ -package headsvc - -import ( - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -type Prm struct { - addr oid.Address -} - -func (p *Prm) WithAddress(v oid.Address) *Prm { - if p != nil { - p.addr = v - } - - return p -} diff --git a/pkg/services/object/head/remote.go b/pkg/services/object/head/remote.go index c9c17d4d8..6d47e37eb 100644 --- a/pkg/services/object/head/remote.go +++ b/pkg/services/object/head/remote.go @@ -28,8 +28,8 @@ type RemoteHeader struct { // RemoteHeadPrm groups remote header operation parameters. type RemoteHeadPrm struct { - commonHeadPrm *Prm - + addr oid.Address + raw bool node netmap.NodeInfo } @@ -57,12 +57,19 @@ func (p *RemoteHeadPrm) WithNodeInfo(v netmap.NodeInfo) *RemoteHeadPrm { // WithObjectAddress sets object address. func (p *RemoteHeadPrm) WithObjectAddress(v oid.Address) *RemoteHeadPrm { if p != nil { - p.commonHeadPrm = new(Prm).WithAddress(v) + p.addr = v } return p } +func (p *RemoteHeadPrm) WithRaw(v bool) *RemoteHeadPrm { + if p != nil { + p.raw = v + } + return p +} + // Head requests object header from the remote node. func (h *RemoteHeader) Head(ctx context.Context, prm *RemoteHeadPrm) (*objectSDK.Object, error) { key, err := h.keyStorage.GetKey(nil) @@ -86,8 +93,11 @@ func (h *RemoteHeader) Head(ctx context.Context, prm *RemoteHeadPrm) (*objectSDK headPrm.SetClient(c) headPrm.SetPrivateKey(key) - headPrm.SetAddress(prm.commonHeadPrm.addr) + headPrm.SetAddress(prm.addr) headPrm.SetTTL(remoteOpTTL) + if prm.raw { + headPrm.SetRawFlag() + } res, err := internalclient.HeadObject(ctx, headPrm) if err != nil { diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 2153275cc..2cdb72505 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -134,7 +134,7 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe callCtx, cancel := context.WithTimeout(ctx, p.headTimeout) - _, err := p.remoteHeader(callCtx, nodes[i], addr) + _, err := p.remoteHeader(callCtx, nodes[i], addr, false) cancel() diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index 194b0ddbe..6eca3bc00 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -10,9 +10,18 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) +var errNoECinfoReturnded = errors.New("no EC info returned") + +type ecChunkProcessResult struct { + validPlacement bool + removeLocal bool +} + var errInvalidECPlacement = errors.New("invalid EC placement: EC placement must have one placement vector") func (p *Policer) processECContainerObject(ctx context.Context, objInfo objectcore.Info, policy netmap.PlacementPolicy) error { @@ -72,30 +81,38 @@ func (p *Policer) processECContainerECObject(ctx context.Context, objInfo object default: } - validPlacement := p.processECChunk(ctx, objInfo, nn[0]) - if !validPlacement { - p.pullRequiredECChunks(ctx, objInfo, nn[0]) + res := p.processECChunk(ctx, objInfo, nn[0]) + if !res.validPlacement { + // drop local chunk only if all required chunks are in place + res.removeLocal = res.removeLocal && p.pullRequiredECChunks(ctx, objInfo, nn[0]) + } + p.adjustECPlacement(ctx, objInfo, nn[0]) + + if res.removeLocal { + p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, zap.Stringer("object", objInfo.Address)) + p.cbRedundantCopy(ctx, objInfo.Address) } return nil } // processECChunk replicates EC chunk if needed. -// Returns True if current chunk should be stored on current node. -func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo) bool { +func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo) ecChunkProcessResult { var removeLocalChunk bool requiredNode := nodes[int(objInfo.ECInfo.Index)%(len(nodes))] if p.cfg.netmapKeys.IsLocalKey(requiredNode.PublicKey()) { // current node is required node, we are happy - return true + return ecChunkProcessResult{ + validPlacement: true, + } } if requiredNode.IsMaintenance() { // consider maintenance mode has object, but do not drop local copy p.log.Debug(logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(requiredNode))) - return false + return ecChunkProcessResult{} } callCtx, cancel := context.WithTimeout(ctx, p.headTimeout) - _, err := p.remoteHeader(callCtx, requiredNode, objInfo.Address) + _, err := p.remoteHeader(callCtx, requiredNode, objInfo.Address, false) cancel() if err == nil { @@ -115,23 +132,138 @@ func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, n p.log.Error(logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, zap.Stringer("object", objInfo.Address), zap.String("error", err.Error())) } - if removeLocalChunk { - p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, zap.Stringer("object", objInfo.Address)) - p.cbRedundantCopy(ctx, objInfo.Address) + return ecChunkProcessResult{ + removeLocal: removeLocalChunk, + } +} + +func (p *Policer) pullRequiredECChunks(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo) bool { + var parentAddress oid.Address + parentAddress.SetContainer(objInfo.Address.Container()) + parentAddress.SetObject(objInfo.ECInfo.ParentID) + + requiredChunkIndexes := p.collectRequiredECChunks(nodes, objInfo) + if len(requiredChunkIndexes) == 0 { + p.log.Info(logs.PolicerNodeIsNotECObjectNode, zap.Stringer("object", objInfo.ECInfo.ParentID)) + return true } + err := p.resolveLocalECChunks(ctx, parentAddress, requiredChunkIndexes) + if err != nil { + p.log.Error(logs.PolicerFailedToGetLocalECChunks, zap.Error(err), zap.Stringer("object", parentAddress)) + return false + } + if len(requiredChunkIndexes) == 0 { + return true + } + + indexToObjectID := make(map[uint32]oid.ID) + success := p.resolveRemoteECChunks(ctx, parentAddress, nodes, requiredChunkIndexes, indexToObjectID) + if !success { + return false + } + + for index, candidates := range requiredChunkIndexes { + var addr oid.Address + addr.SetContainer(objInfo.Address.Container()) + addr.SetObject(indexToObjectID[index]) + p.replicator.HandlePullTask(ctx, replicator.Task{ + Addr: addr, + Nodes: candidates, + }) + } + // there was some missing chunks, it's not ok return false } -func (p *Policer) pullRequiredECChunks(_ context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo) { - requiredChunkIndexes := make(map[uint32]struct{}) +func (p *Policer) collectRequiredECChunks(nodes []netmap.NodeInfo, objInfo objectcore.Info) map[uint32][]netmap.NodeInfo { + requiredChunkIndexes := make(map[uint32][]netmap.NodeInfo) for i, n := range nodes { + if uint32(i) == objInfo.ECInfo.Total { + break + } if p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) { - requiredChunkIndexes[uint32(i)] = struct{}{} + requiredChunkIndexes[uint32(i)] = []netmap.NodeInfo{} } } - if len(requiredChunkIndexes) == 0 { - p.log.Info(logs.PolicerNodeIsNotContainerNodeForECObject, zap.Stringer("object", objInfo.ECInfo.ParentID)) - return + return requiredChunkIndexes +} + +func (p *Policer) resolveLocalECChunks(ctx context.Context, parentAddress oid.Address, required map[uint32][]netmap.NodeInfo) error { + _, err := p.localHeader(ctx, parentAddress) + var eiErr *objectSDK.ECInfoError + if err == nil { // should not be happen + return errNoECinfoReturnded + } + if !errors.As(err, &eiErr) { + return err + } + for _, ch := range eiErr.ECInfo().Chunks { + delete(required, ch.Index) + } + return nil +} + +func (p *Policer) resolveRemoteECChunks(ctx context.Context, parentAddress oid.Address, nodes []netmap.NodeInfo, required map[uint32][]netmap.NodeInfo, indexToObjectID map[uint32]oid.ID) bool { + var eiErr *objectSDK.ECInfoError + for _, n := range nodes { + if p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) { + continue + } + _, err := p.remoteHeader(ctx, n, parentAddress, true) + if !errors.As(err, &eiErr) { + continue + } + for _, ch := range eiErr.ECInfo().Chunks { + if candidates, ok := required[ch.Index]; ok { + candidates = append(candidates, n) + required[ch.Index] = candidates + + var chunkID oid.ID + if err := chunkID.ReadFromV2(ch.ID); err != nil { + p.log.Error(logs.PolicerFailedToDecodeECChunkID, zap.Error(err), zap.Stringer("object", parentAddress)) + return false + } + if existed, ok := indexToObjectID[ch.Index]; ok && existed != chunkID { + p.log.Error(logs.PolicerDifferentObjectIDForTheSameECChunk, zap.Stringer("first", existed), + zap.Stringer("second", chunkID), zap.Stringer("object", parentAddress)) + return false + } + indexToObjectID[ch.Index] = chunkID + } + } + } + + for index, candidates := range required { + if len(candidates) == 0 { + p.log.Error(logs.PolicerMissingECChunk, zap.Stringer("object", parentAddress), zap.Uint32("index", index)) + return false + } + } + + return true +} + +func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo) { + var parentAddress oid.Address + parentAddress.SetContainer(objInfo.Address.Container()) + parentAddress.SetObject(objInfo.ECInfo.ParentID) + var eiErr *objectSDK.ECInfoError + for idx, n := range nodes { + if uint32(idx) == objInfo.ECInfo.Total { + return + } + if p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) { + continue + } + _, err := p.remoteHeader(ctx, n, parentAddress, true) + if errors.As(err, &eiErr) { + continue + } + p.replicator.HandleReplicationTask(ctx, replicator.Task{ + NumCopies: 1, + Addr: objInfo.Address, + Nodes: []netmap.NodeInfo{n}, + }, newNodeCache()) } } diff --git a/pkg/services/policer/ec_test.go b/pkg/services/policer/ec_test.go new file mode 100644 index 000000000..3957c112b --- /dev/null +++ b/pkg/services/policer/ec_test.go @@ -0,0 +1,402 @@ +package policer + +import ( + "bytes" + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +func TestECChunkHasValidPlacement(t *testing.T) { + t.Parallel() + chunkAddress := oidtest.Address() + parentID := oidtest.ID() + + var policy netmapSDK.PlacementPolicy + require.NoError(t, policy.DecodeString("EC 2.1")) + + cnr := &container.Container{} + cnr.Value.Init() + cnr.Value.SetPlacementPolicy(policy) + containerSrc := containerSrc{ + get: func(id cid.ID) (*container.Container, error) { + if id.Equals(chunkAddress.Container()) { + return cnr, nil + } + return nil, new(apistatus.ContainerNotFound) + }, + } + + nodes := make([]netmapSDK.NodeInfo, 4) + for i := range nodes { + nodes[i].SetPublicKey([]byte{byte(i)}) + } + + placementBuilder := func(cnr cid.ID, obj *oid.ID, p netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) { + if cnr.Equals(chunkAddress.Container()) && obj.Equals(parentID) { + return [][]netmapSDK.NodeInfo{nodes}, nil + } + return nil, errors.New("unexpected placement build") + } + + headFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) { + require.True(t, raw, "remote header for parent object must be called with raw flag") + index := int(ni.PublicKey()[0]) + require.True(t, index == 1 || index == 2, "invalid node to get parent header") + require.True(t, a.Container() == chunkAddress.Container() && a.Object() == parentID, "invalid address to get remote header") + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(oidtest.ID()) + ch.Index = uint32(index) + ch.Total = 3 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + + p := New( + WithContainerSource(containerSrc), + WithPlacementBuilder(placementBuilderFunc(placementBuilder)), + WithNetmapKeys(announcedKeysFunc(func(k []byte) bool { + return bytes.Equal(k, nodes[0].PublicKey()) + })), + WithRemoteObjectHeaderFunc(headFn), + WithPool(testPool(t)), + ) + + objInfo := objectcore.Info{ + Address: chunkAddress, + Type: objectSDK.TypeRegular, + ECInfo: &objectcore.ECInfo{ + ParentID: parentID, + Index: 0, + Total: 3, + }, + } + err := p.processObject(context.Background(), objInfo) + require.NoError(t, err) +} + +func TestECChunkHasInvalidPlacement(t *testing.T) { + t.Parallel() + chunkAddress := oidtest.Address() + parentID := oidtest.ID() + chunkObject := objectSDK.New() + chunkObject.SetContainerID(chunkAddress.Container()) + chunkObject.SetID(chunkAddress.Object()) + chunkObject.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + chunkObject.SetPayloadSize(uint64(10)) + chunkObject.SetECHeader(objectSDK.NewECHeader(parentID, 1, 3, []byte{}, 0)) + + var policy netmapSDK.PlacementPolicy + require.NoError(t, policy.DecodeString("EC 2.1")) + + cnr := &container.Container{} + cnr.Value.Init() + cnr.Value.SetPlacementPolicy(policy) + containerSrc := containerSrc{ + get: func(id cid.ID) (*container.Container, error) { + if id.Equals(chunkAddress.Container()) { + return cnr, nil + } + return nil, new(apistatus.ContainerNotFound) + }, + } + + nodes := make([]netmapSDK.NodeInfo, 4) + for i := range nodes { + nodes[i].SetPublicKey([]byte{byte(i)}) + } + + placementBuilder := func(cnr cid.ID, obj *oid.ID, p netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) { + if cnr.Equals(chunkAddress.Container()) && obj.Equals(parentID) { + return [][]netmapSDK.NodeInfo{nodes}, nil + } + return nil, errors.New("unexpected placement build") + } + + objInfo := objectcore.Info{ + Address: chunkAddress, + Type: objectSDK.TypeRegular, + ECInfo: &objectcore.ECInfo{ + ParentID: parentID, + Index: 1, + Total: 3, + }, + } + + t.Run("node0 has chunk1, node1 has chunk0 and chunk1", func(t *testing.T) { + // policer should pull chunk0 on first run and drop chunk1 on second run + var allowDrop bool + requiredChunkID := oidtest.ID() + headFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) { + if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a == chunkAddress && !raw { + return chunkObject, nil + } + if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a.Container() == chunkAddress.Container() && + a.Object() == parentID && raw { + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(oidtest.ID()) + ch.Index = 1 + ch.Total = 3 + ei.AddChunk(ch) + ch.Index = 0 + ch.SetID(requiredChunkID) + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + if bytes.Equal(ni.PublicKey(), nodes[2].PublicKey()) && a.Container() == chunkAddress.Container() && + a.Object() == parentID && raw { + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(oidtest.ID()) + ch.Index = 2 + ch.Total = 3 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + if bytes.Equal(ni.PublicKey(), nodes[3].PublicKey()) && a.Container() == chunkAddress.Container() && + a.Object() == parentID && raw { + return nil, new(apistatus.ObjectNotFound) + } + require.Fail(t, "unexpected remote HEAD") + return nil, fmt.Errorf("unexpected remote HEAD") + } + + localHeadF := func(_ context.Context, addr oid.Address) (*objectSDK.Object, error) { + require.True(t, addr.Container() == chunkAddress.Container() && addr.Object() == parentID, "unexpected local HEAD") + if allowDrop { + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(oidtest.ID()) + ch.Index = 1 + ch.Total = 3 + ei.AddChunk(ch) + ch.SetID(requiredChunkID) + ch.Index = 0 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(oidtest.ID()) + ch.Index = 1 + ch.Total = 3 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + + var pullCounter atomic.Int64 + var dropped []oid.Address + p := New( + WithContainerSource(containerSrc), + WithPlacementBuilder(placementBuilderFunc(placementBuilder)), + WithNetmapKeys(announcedKeysFunc(func(k []byte) bool { + return bytes.Equal(k, nodes[0].PublicKey()) + })), + WithRemoteObjectHeaderFunc(headFn), + WithLocalObjectHeaderFunc(localHeadF), + WithReplicator(pullFunc(func(ctx context.Context, r replicator.Task) { + require.True(t, r.Addr.Container() == chunkAddress.Container() && r.Addr.Object() == requiredChunkID && + len(r.Nodes) == 1 && bytes.Equal(r.Nodes[0].PublicKey(), nodes[1].PublicKey()), "invalid pull task") + pullCounter.Add(1) + })), + WithRedundantCopyCallback(func(ctx context.Context, a oid.Address) { + require.True(t, allowDrop, "invalid redundent copy call") + dropped = append(dropped, a) + }), + WithPool(testPool(t)), + ) + + err := p.processObject(context.Background(), objInfo) + require.NoError(t, err) + require.Equal(t, int64(1), pullCounter.Load(), "invalid pull count") + require.Equal(t, 0, len(dropped), "invalid dropped count") + allowDrop = true + err = p.processObject(context.Background(), objInfo) + require.NoError(t, err) + require.Equal(t, int64(1), pullCounter.Load(), "invalid pull count") + require.Equal(t, 1, len(dropped), "invalid dropped count") + require.True(t, chunkAddress.Equals(dropped[0]), "invalid dropped object") + }) + + t.Run("node0 has chunk0 and chunk1, node1 has chunk1", func(t *testing.T) { + // policer should drop chunk1 + headFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) { + if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a == chunkAddress && !raw { + return chunkObject, nil + } + if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a.Container() == chunkAddress.Container() && + a.Object() == parentID && raw { + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(chunkAddress.Object()) + ch.Index = 1 + ch.Total = 3 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + if bytes.Equal(ni.PublicKey(), nodes[2].PublicKey()) && a.Container() == chunkAddress.Container() && + a.Object() == parentID && raw { + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(oidtest.ID()) + ch.Index = 2 + ch.Total = 3 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + if bytes.Equal(ni.PublicKey(), nodes[3].PublicKey()) && a.Container() == chunkAddress.Container() && + a.Object() == parentID && raw { + return nil, new(apistatus.ObjectNotFound) + } + require.Fail(t, "unexpected remote HEAD") + return nil, fmt.Errorf("unexpected remote HEAD") + } + + localHeadF := func(_ context.Context, addr oid.Address) (*objectSDK.Object, error) { + require.True(t, addr.Container() == chunkAddress.Container() && addr.Object() == parentID, "unexpected local HEAD") + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(chunkAddress.Object()) + ch.Index = 1 + ch.Total = 3 + ei.AddChunk(ch) + ch.SetID(oidtest.ID()) + ch.Index = 0 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + + var dropped []oid.Address + p := New( + WithContainerSource(containerSrc), + WithPlacementBuilder(placementBuilderFunc(placementBuilder)), + WithNetmapKeys(announcedKeysFunc(func(k []byte) bool { + return bytes.Equal(k, nodes[0].PublicKey()) + })), + WithRemoteObjectHeaderFunc(headFn), + WithLocalObjectHeaderFunc(localHeadF), + WithRedundantCopyCallback(func(ctx context.Context, a oid.Address) { + dropped = append(dropped, a) + }), + WithPool(testPool(t)), + ) + + err := p.processObject(context.Background(), objInfo) + require.NoError(t, err) + require.Equal(t, 1, len(dropped), "invalid dropped count") + require.True(t, chunkAddress.Equals(dropped[0]), "invalid dropped object") + }) + + t.Run("node0 has chunk0 and chunk1, node1 has no chunks", func(t *testing.T) { + // policer should replicate chunk1 to node1 on first run and drop chunk1 on node0 on second run + var secondRun bool + headFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) { + if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a == chunkAddress && !raw { + if !secondRun { + return nil, new(apistatus.ObjectNotFound) + } + return chunkObject, nil + } + if bytes.Equal(ni.PublicKey(), nodes[1].PublicKey()) && a.Container() == chunkAddress.Container() && + a.Object() == parentID && raw { + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(chunkAddress.Object()) + ch.Index = 1 + ch.Total = 3 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + if bytes.Equal(ni.PublicKey(), nodes[2].PublicKey()) && a.Container() == chunkAddress.Container() && + a.Object() == parentID && raw { + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(oidtest.ID()) + ch.Index = 2 + ch.Total = 3 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + if bytes.Equal(ni.PublicKey(), nodes[3].PublicKey()) && a.Container() == chunkAddress.Container() && + a.Object() == parentID && raw { + return nil, new(apistatus.ObjectNotFound) + } + require.Fail(t, "unexpected remote HEAD") + return nil, fmt.Errorf("unexpected remote HEAD") + } + + localHeadF := func(_ context.Context, addr oid.Address) (*objectSDK.Object, error) { + require.True(t, addr.Container() == chunkAddress.Container() && addr.Object() == parentID, "unexpected local HEAD") + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(chunkAddress.Object()) + ch.Index = 1 + ch.Total = 3 + ei.AddChunk(ch) + ch.SetID(oidtest.ID()) + ch.Index = 0 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + + var dropped []oid.Address + var replicated []replicator.Task + p := New( + WithContainerSource(containerSrc), + WithPlacementBuilder(placementBuilderFunc(placementBuilder)), + WithNetmapKeys(announcedKeysFunc(func(k []byte) bool { + return bytes.Equal(k, nodes[0].PublicKey()) + })), + WithRemoteObjectHeaderFunc(headFn), + WithLocalObjectHeaderFunc(localHeadF), + WithRedundantCopyCallback(func(ctx context.Context, a oid.Address) { + dropped = append(dropped, a) + }), + WithReplicator(replicatorFunc(func(ctx context.Context, t replicator.Task, tr replicator.TaskResult) { + replicated = append(replicated, t) + })), + WithPool(testPool(t)), + ) + + err := p.processObject(context.Background(), objInfo) + require.NoError(t, err) + require.Equal(t, 0, len(dropped), "invalid dropped count") + require.Equal(t, 1, len(replicated), "invalid replicated count") + require.Equal(t, chunkAddress, replicated[0].Addr, "invalid replicated object") + require.True(t, bytes.Equal(replicated[0].Nodes[0].PublicKey(), nodes[1].PublicKey()), "invalid replicate target") + + secondRun = true + err = p.processObject(context.Background(), objInfo) + require.NoError(t, err) + require.Equal(t, 1, len(replicated), "invalid replicated count") + require.Equal(t, chunkAddress, replicated[0].Addr, "invalid replicated object") + require.True(t, bytes.Equal(replicated[0].Nodes[0].PublicKey(), nodes[1].PublicKey()), "invalid replicate target") + require.Equal(t, 1, len(dropped), "invalid dropped count") + require.True(t, chunkAddress.Equals(dropped[0]), "invalid dropped object") + }) +} + +type pullFunc func(context.Context, replicator.Task) + +func (f pullFunc) HandleReplicationTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) { + panic("not implemented") +} + +func (f pullFunc) HandlePullTask(ctx context.Context, task replicator.Task) { + f(ctx, task) +} diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 849e5ed8b..9a646eea6 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -36,10 +36,14 @@ type BuryFunc func(context.Context, oid.Address) error // Replicator is the interface to a consumer of replication tasks. type Replicator interface { HandleReplicationTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) + HandlePullTask(ctx context.Context, task replicator.Task) } // RemoteObjectHeaderFunc is the function to obtain HEAD info from a specific remote node. -type RemoteObjectHeaderFunc func(context.Context, netmapSDK.NodeInfo, oid.Address) (*objectSDK.Object, error) +type RemoteObjectHeaderFunc func(context.Context, netmapSDK.NodeInfo, oid.Address, bool) (*objectSDK.Object, error) + +// LocalObjectHeaderFunc is the function to obtain HEAD info from the current node. +type LocalObjectHeaderFunc func(context.Context, oid.Address) (*objectSDK.Object, error) type cfg struct { headTimeout time.Duration @@ -56,6 +60,8 @@ type cfg struct { remoteHeader RemoteObjectHeaderFunc + localHeader LocalObjectHeaderFunc + netmapKeys netmap.AnnouncedKeys replicator Replicator @@ -125,13 +131,20 @@ func WithPlacementBuilder(v placement.Builder) Option { } } -// WithRemoteObjectHeader returns option to set object header receiver of Policer. +// WithRemoteObjectHeader returns option to set remote object header receiver of Policer. func WithRemoteObjectHeaderFunc(v RemoteObjectHeaderFunc) Option { return func(c *cfg) { c.remoteHeader = v } } +// WithLocalObjectHeaderFunc returns option to set local object header receiver of Policer. +func WithLocalObjectHeaderFunc(v LocalObjectHeaderFunc) Option { + return func(c *cfg) { + c.localHeader = v + } +} + // WithNetmapKeys returns option to set tool to work with announced public keys. func WithNetmapKeys(v netmap.AnnouncedKeys) Option { return func(c *cfg) { diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 2c70b1c04..072f21a6f 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -161,31 +161,6 @@ func TestProcessObject(t *testing.T) { placement: [][]int{{0, 1, 2}}, wantReplicateTo: []int{1, 2}, }, - { - desc: "EC chunk stored valid on current node", - objType: objectSDK.TypeRegular, - nodeCount: 2, - policy: `EC 1.1`, - placement: [][]int{{0}}, - ecInfo: &objectcore.ECInfo{ - ParentID: oidtest.ID(), - Index: 1, - Total: 2, - }, - }, - { - desc: "EC chunk must be replicated to other EC node", - objType: objectSDK.TypeRegular, - nodeCount: 2, - policy: `EC 1.1`, - placement: [][]int{{1}}, - wantReplicateTo: []int{1}, - ecInfo: &objectcore.ECInfo{ - ParentID: oidtest.ID(), - Index: 1, - Total: 2, - }, - }, } for i := range tests { @@ -223,7 +198,7 @@ func TestProcessObject(t *testing.T) { } // Object remote header - headFn := func(_ context.Context, ni netmap.NodeInfo, a oid.Address) (*objectSDK.Object, error) { + headFn := func(_ context.Context, ni netmap.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) { index := int(ni.PublicKey()[0]) if a != addr || index < 1 || index >= ti.nodeCount { t.Errorf("unexpected remote object head: node=%+v addr=%v", ni, a) @@ -471,3 +446,7 @@ type replicatorFunc func(context.Context, replicator.Task, replicator.TaskResult func (f replicatorFunc) HandleReplicationTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) { f(ctx, task, res) } + +func (f replicatorFunc) HandlePullTask(ctx context.Context, task replicator.Task) { + panic("not implemented") +} diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 06d41b74e..3d04b7084 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -32,7 +32,7 @@ func (p *Replicator) HandleReplicationTask(ctx context.Context, task Task, res T ) }() - ctx, span := tracing.StartSpanFromContext(ctx, "Replicator.HandleTask", + ctx, span := tracing.StartSpanFromContext(ctx, "Replicator.HandleReplicateTask", trace.WithAttributes( attribute.Stringer("address", task.Addr), attribute.Int64("number_of_copies", int64(task.NumCopies)), diff --git a/pkg/services/replicator/pull.go b/pkg/services/replicator/pull.go new file mode 100644 index 000000000..d178700f6 --- /dev/null +++ b/pkg/services/replicator/pull.go @@ -0,0 +1,72 @@ +package replicator + +import ( + "context" + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +var errFailedToGetObjectFromAnyNode = errors.New("failed to get object from any node") + +func (p *Replicator) HandlePullTask(ctx context.Context, task Task) { + p.metrics.IncInFlightRequest() + defer p.metrics.DecInFlightRequest() + defer func() { + p.log.Debug(logs.ReplicatorFinishWork, zap.String("type", "pull")) + }() + + ctx, span := tracing.StartSpanFromContext(ctx, "Replicator.HandlePullTask", + trace.WithAttributes( + attribute.Stringer("address", task.Addr), + attribute.Int("nodes_count", len(task.Nodes)), + )) + defer span.End() + + var obj *objectSDK.Object + + for _, node := range task.Nodes { + var err error + obj, err = p.remoteGetter.Get(ctx, getsvc.RemoteGetPrm{ + Address: task.Addr, + Node: node, + }) + if err == nil { + break + } + var endpoints []string + node.IterateNetworkEndpoints(func(s string) bool { + endpoints = append(endpoints, s) + return false + }) + p.log.Error(logs.ReplicatorCouldNotGetObjectFromRemoteStorage, + zap.Stringer("object", task.Addr), + zap.Error(err), + zap.Strings("endpoints", endpoints), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } + + if obj == nil { + p.log.Error(logs.ReplicatorCouldNotGetObjectFromRemoteStorage, + zap.Stringer("object", task.Addr), + zap.Error(errFailedToGetObjectFromAnyNode), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + return + } + + err := engine.Put(ctx, p.localStorage, obj) + if err != nil { + p.log.Error(logs.ReplicatorCouldNotPutObjectToLocalStorage, + zap.Stringer("object", task.Addr), + zap.Error(err), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } +} diff --git a/pkg/services/replicator/replicator.go b/pkg/services/replicator/replicator.go index bb817cb32..a67f2e766 100644 --- a/pkg/services/replicator/replicator.go +++ b/pkg/services/replicator/replicator.go @@ -4,6 +4,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" @@ -25,6 +26,8 @@ type cfg struct { remoteSender *putsvc.RemoteSender + remoteGetter *getsvc.RemoteGetter + localStorage *engine.StorageEngine metrics MetricsRegister @@ -70,6 +73,12 @@ func WithRemoteSender(v *putsvc.RemoteSender) Option { } } +func WithRemoteGetter(v *getsvc.RemoteGetter) Option { + return func(c *cfg) { + c.remoteGetter = v + } +} + // WithLocalStorage returns option to set local object storage of Replicator. func WithLocalStorage(v *engine.StorageEngine) Option { return func(c *cfg) { From 4ab6c404f7cf401935af4fd1e1a7aa7c22a09dd0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 15 May 2024 15:50:16 +0300 Subject: [PATCH 1100/1943] [#1129] policer: Drop unused Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/check.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 2cdb72505..04456fb20 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -54,11 +54,6 @@ func (p *Policer) processRepContainerObject(ctx context.Context, objInfo objectc c := &placementRequirements{} - var numOfContainerNodes int - for i := range nn { - numOfContainerNodes += len(nn[i]) - } - // cached info about already checked nodes checkedNodes := newNodeCache() From 1cd8562db83b89134c1e4e17085c9713a911c97b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 15 May 2024 16:11:51 +0300 Subject: [PATCH 1101/1943] [#1129] policer: Refactor shortage Drop override inside method. Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/check.go | 21 ++++++++++----------- pkg/services/policer/ec.go | 4 +--- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 04456fb20..7120ba0b8 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -64,7 +64,16 @@ func (p *Policer) processRepContainerObject(ctx context.Context, objInfo objectc default: } - p.processRepNodes(ctx, c, objInfo, nn[i], policy.ReplicaDescriptor(i).NumberOfObjects(), checkedNodes) + shortage := policy.ReplicaDescriptor(i).NumberOfObjects() + if objInfo.Type == objectSDK.TypeLock || objInfo.Type == objectSDK.TypeTombstone || objInfo.IsLinkingObject { + // all nodes of a container must store the `LOCK`, `TOMBSTONE` and linking objects + // for correct object removal protection: + // - `LOCK`, `TOMBSTONE` and linking objects are broadcast on their PUT requests; + // - `LOCK` object removal is a prohibited action in the GC. + shortage = uint32(len(nn[i])) + } + + p.processRepNodes(ctx, c, objInfo, nn[i], shortage, checkedNodes) } if !c.needLocalCopy && c.removeLocalCopy { @@ -89,19 +98,9 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe nodes []netmap.NodeInfo, shortage uint32, checkedNodes nodeCache, ) { addr := objInfo.Address - typ := objInfo.Type // Number of copies that are stored on maintenance nodes. var uncheckedCopies int - - if typ == objectSDK.TypeLock || typ == objectSDK.TypeTombstone || objInfo.IsLinkingObject { - // all nodes of a container must store the `LOCK`, `TOMBSTONE` and linking objects - // for correct object removal protection: - // - `LOCK`, `TOMBSTONE` and linking objects are broadcast on their PUT requests; - // - `LOCK` object removal is a prohibited action in the GC. - shortage = uint32(len(nodes)) - } - for i := 0; shortage > 0 && i < len(nodes); i++ { select { case <-ctx.Done(): diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index 6eca3bc00..fbd389d71 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -52,9 +52,7 @@ func (p *Policer) processECContainerRepObject(ctx context.Context, objInfo objec default: } - rd := policy.ReplicaDescriptor(0) - // processNodes replaces rd.GetECDataCount() + rd.GetECParityCount() for len(nn[0]) for locks, tomstones and linking objects. - p.processRepNodes(ctx, c, objInfo, nn[0], rd.GetECDataCount()+rd.GetECParityCount(), checkedNodes) + p.processRepNodes(ctx, c, objInfo, nn[0], uint32(len(nn[0])), checkedNodes) if !c.needLocalCopy && c.removeLocalCopy { p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, From 0e42126ddcc7d42db01935cd7ba7a4ff4df712c9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 16 May 2024 10:02:41 +0300 Subject: [PATCH 1102/1943] [#1129] object: Fix check owner for EC part Do not validate EC part owner if request from container node. Signed-off-by: Dmitrii Stepanov --- pkg/core/object/fmt.go | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index e266800b2..fa1e40dd0 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -167,17 +167,29 @@ func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { token := obj.SessionToken() ownerID := obj.OwnerID() + if token == nil && obj.ECHeader() != nil { + role, err := v.isIROrContainerNode(obj, binKey) + if err != nil { + return err + } + if role == acl.RoleContainer { + // EC part could be restored or created by container node, so ownerID could not match object signature + return nil + } + return v.checkOwnerKey(ownerID, key) + } + if token == nil || !token.AssertAuthKey(&key) { return v.checkOwnerKey(ownerID, key) } if v.verifyTokenIssuer { - signerIsIROrContainerNode, err := v.isIROrContainerNode(obj, binKey) + role, err := v.isIROrContainerNode(obj, binKey) if err != nil { return err } - if signerIsIROrContainerNode { + if role == acl.RoleContainer || role == acl.RoleInnerRing { return nil } @@ -190,10 +202,10 @@ func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { return nil } -func (v *FormatValidator) isIROrContainerNode(obj *objectSDK.Object, signerKey []byte) (bool, error) { +func (v *FormatValidator) isIROrContainerNode(obj *objectSDK.Object, signerKey []byte) (acl.Role, error) { cnrID, containerIDSet := obj.ContainerID() if !containerIDSet { - return false, errNilCID + return acl.RoleOthers, errNilCID } cnrIDBin := make([]byte, sha256.Size) @@ -201,14 +213,14 @@ func (v *FormatValidator) isIROrContainerNode(obj *objectSDK.Object, signerKey [ cnr, err := v.containers.Get(cnrID) if err != nil { - return false, fmt.Errorf("failed to get container (id=%s): %w", cnrID.EncodeToString(), err) + return acl.RoleOthers, fmt.Errorf("failed to get container (id=%s): %w", cnrID.EncodeToString(), err) } res, err := v.senderClassifier.IsInnerRingOrContainerNode(signerKey, cnrID, cnr.Value) if err != nil { - return false, err + return acl.RoleOthers, err } - return res.Role == acl.RoleContainer || res.Role == acl.RoleInnerRing, nil + return res.Role, nil } func (v *FormatValidator) checkOwnerKey(id user.ID, key frostfsecdsa.PublicKey) error { From 44f2e8f27fcf6a10ae2b0dc0fdc8842026bcd386 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 16 May 2024 10:05:17 +0300 Subject: [PATCH 1103/1943] [#1129] putSvc: Allow to put single unprepared object to EC container Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/single.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 43b3b0ac1..6d2f3dba8 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -34,10 +34,7 @@ import ( "go.uber.org/zap" ) -var ( - errInvalidPayloadChecksum = errors.New("incorrect payload checksum") - errInvalidECObject = errors.New("object must be splitted to EC parts") -) +var errInvalidPayloadChecksum = errors.New("incorrect payload checksum") type putSingleRequestSigner struct { req *objectAPI.PutSingleRequest @@ -181,10 +178,6 @@ func (s *Service) saveToREPReplicas(ctx context.Context, placement putSinglePlac } func (s *Service) saveToECReplicas(ctx context.Context, placement putSinglePlacement, obj *objectSDK.Object, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { - if obj.Type() == objectSDK.TypeRegular && obj.ECHeader() == nil { - return errInvalidECObject - } - commonPrm, err := svcutil.CommonPrmFromV2(req) if err != nil { return err From 436c9f555803b67b31a4098a12c6854fad182ba7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 16 May 2024 12:26:49 +0300 Subject: [PATCH 1104/1943] [#1129] policer: Restore EC object Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 21 +- internal/logs/logs.go | 5 + .../{head/remote.go => remote_reader.go} | 62 ++++-- pkg/services/policer/ec.go | 145 ++++++++++++- pkg/services/policer/ec_test.go | 195 ++++++++++++++++-- pkg/services/policer/option.go | 30 +++ pkg/services/policer/policer_test.go | 35 ++-- pkg/services/replicator/put.go | 47 +++++ 8 files changed, 483 insertions(+), 57 deletions(-) rename pkg/services/object/{head/remote.go => remote_reader.go} (55%) create mode 100644 pkg/services/replicator/put.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index af185a8da..9f34896bc 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -29,7 +29,6 @@ import ( deletesvcV2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/delete/v2" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" getsvcV2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get/v2" - headsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/head" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" putsvcV2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put/v2" searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" @@ -231,7 +230,7 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl return err } - remoteHeader := headsvc.NewRemoteHeader(keyStorage, clientConstructor) + remoteReader := objectService.NewRemoteReader(keyStorage, clientConstructor) pol := policer.New( policer.WithLogger(c.log), @@ -243,8 +242,8 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl ), policer.WithRemoteObjectHeaderFunc( func(ctx context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) { - prm := new(headsvc.RemoteHeadPrm).WithNodeInfo(ni).WithObjectAddress(a).WithRaw(raw) - return remoteHeader.Head(ctx, prm) + prm := new(objectService.RemoteRequestPrm).WithNodeInfo(ni).WithObjectAddress(a).WithRaw(raw) + return remoteReader.Head(ctx, prm) }, ), policer.WithLocalObjectHeaderFunc(func(ctx context.Context, a oid.Address) (*objectSDK.Object, error) { @@ -256,6 +255,19 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl } return res.Header(), nil }), + policer.WithRemoteObjectGetFunc(func(ctx context.Context, ni netmapSDK.NodeInfo, a oid.Address) (*objectSDK.Object, error) { + prm := new(objectService.RemoteRequestPrm).WithNodeInfo(ni).WithObjectAddress(a) + return remoteReader.Get(ctx, prm) + }), + policer.WithLocalObjectGetFunc(func(ctx context.Context, a oid.Address) (*objectSDK.Object, error) { + var prm engine.GetPrm + prm.WithAddress(a) + res, err := c.cfgObject.cfgLocalStorage.localStorage.Get(ctx, prm) + if err != nil { + return nil, err + } + return res.Object(), nil + }), policer.WithNetmapKeys(c), policer.WithHeadTimeout( policerconfig.HeadTimeout(c.appCfg), @@ -274,6 +286,7 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl }), policer.WithPool(c.cfgObject.pool.replication), policer.WithMetrics(c.metricsCollector.PolicerMetrics()), + policer.WithKeyStorage(keyStorage), ) c.workers = append(c.workers, worker{ diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 2e89e4304..43fd77624 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -536,4 +536,9 @@ const ( PolicerDifferentObjectIDForTheSameECChunk = "different object IDs for the same EC chunk" ReplicatorCouldNotGetObjectFromRemoteStorage = "could not get object from remote storage" ReplicatorCouldNotPutObjectToLocalStorage = "could not put object to local storage" + PolicerCouldNotGetObjectFromNodeMoving = "could not get EC object from the node, moving current chunk to the node" + PolicerCouldNotRestoreObjectNotEnoughChunks = "could not restore EC object: not enough chunks" + PolicerFailedToRestoreObject = "failed to restore EC object" + PolicerCouldNotGetChunk = "could not get EC chunk" + PolicerCouldNotGetChunks = "could not get EC chunks" ) diff --git a/pkg/services/object/head/remote.go b/pkg/services/object/remote_reader.go similarity index 55% rename from pkg/services/object/head/remote.go rename to pkg/services/object/remote_reader.go index 6d47e37eb..18b6107cf 100644 --- a/pkg/services/object/head/remote.go +++ b/pkg/services/object/remote_reader.go @@ -1,4 +1,4 @@ -package headsvc +package object import ( "context" @@ -18,16 +18,16 @@ type ClientConstructor interface { Get(clientcore.NodeInfo) (clientcore.MultiAddressClient, error) } -// RemoteHeader represents utility for getting -// the object header from a remote host. -type RemoteHeader struct { +// RemoteReader represents utility for getting +// the object from a remote host. +type RemoteReader struct { keyStorage *util.KeyStorage clientCache ClientConstructor } -// RemoteHeadPrm groups remote header operation parameters. -type RemoteHeadPrm struct { +// RemoteRequestPrm groups remote operation parameters. +type RemoteRequestPrm struct { addr oid.Address raw bool node netmap.NodeInfo @@ -37,16 +37,16 @@ const remoteOpTTL = 1 var ErrNotFound = errors.New("object header not found") -// NewRemoteHeader creates, initializes and returns new RemoteHeader instance. -func NewRemoteHeader(keyStorage *util.KeyStorage, cache ClientConstructor) *RemoteHeader { - return &RemoteHeader{ +// NewRemoteReader creates, initializes and returns new RemoteHeader instance. +func NewRemoteReader(keyStorage *util.KeyStorage, cache ClientConstructor) *RemoteReader { + return &RemoteReader{ keyStorage: keyStorage, clientCache: cache, } } // WithNodeInfo sets information about the remote node. -func (p *RemoteHeadPrm) WithNodeInfo(v netmap.NodeInfo) *RemoteHeadPrm { +func (p *RemoteRequestPrm) WithNodeInfo(v netmap.NodeInfo) *RemoteRequestPrm { if p != nil { p.node = v } @@ -55,7 +55,7 @@ func (p *RemoteHeadPrm) WithNodeInfo(v netmap.NodeInfo) *RemoteHeadPrm { } // WithObjectAddress sets object address. -func (p *RemoteHeadPrm) WithObjectAddress(v oid.Address) *RemoteHeadPrm { +func (p *RemoteRequestPrm) WithObjectAddress(v oid.Address) *RemoteRequestPrm { if p != nil { p.addr = v } @@ -63,7 +63,7 @@ func (p *RemoteHeadPrm) WithObjectAddress(v oid.Address) *RemoteHeadPrm { return p } -func (p *RemoteHeadPrm) WithRaw(v bool) *RemoteHeadPrm { +func (p *RemoteRequestPrm) WithRaw(v bool) *RemoteRequestPrm { if p != nil { p.raw = v } @@ -71,7 +71,7 @@ func (p *RemoteHeadPrm) WithRaw(v bool) *RemoteHeadPrm { } // Head requests object header from the remote node. -func (h *RemoteHeader) Head(ctx context.Context, prm *RemoteHeadPrm) (*objectSDK.Object, error) { +func (h *RemoteReader) Head(ctx context.Context, prm *RemoteRequestPrm) (*objectSDK.Object, error) { key, err := h.keyStorage.GetKey(nil) if err != nil { return nil, fmt.Errorf("(%T) could not receive private key: %w", h, err) @@ -106,3 +106,39 @@ func (h *RemoteHeader) Head(ctx context.Context, prm *RemoteHeadPrm) (*objectSDK return res.Header(), nil } + +func (h *RemoteReader) Get(ctx context.Context, prm *RemoteRequestPrm) (*objectSDK.Object, error) { + key, err := h.keyStorage.GetKey(nil) + if err != nil { + return nil, fmt.Errorf("(%T) could not receive private key: %w", h, err) + } + + var info clientcore.NodeInfo + + err = clientcore.NodeInfoFromRawNetmapElement(&info, netmapCore.Node(prm.node)) + if err != nil { + return nil, fmt.Errorf("parse client node info: %w", err) + } + + c, err := h.clientCache.Get(info) + if err != nil { + return nil, fmt.Errorf("(%T) could not create SDK client %s: %w", h, info.AddressGroup(), err) + } + + var getPrm internalclient.GetObjectPrm + + getPrm.SetClient(c) + getPrm.SetPrivateKey(key) + getPrm.SetAddress(prm.addr) + getPrm.SetTTL(remoteOpTTL) + if prm.raw { + getPrm.SetRawFlag() + } + + res, err := internalclient.GetObject(ctx, getPrm) + if err != nil { + return nil, fmt.Errorf("(%T) could not head object in %s: %w", h, info.AddressGroup(), err) + } + + return res.Object(), nil +} diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index fbd389d71..b15efdd15 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -2,6 +2,7 @@ package policer import ( "context" + "encoding/hex" "errors" "fmt" @@ -11,8 +12,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) var errNoECinfoReturnded = errors.New("no EC info returned") @@ -84,7 +87,7 @@ func (p *Policer) processECContainerECObject(ctx context.Context, objInfo object // drop local chunk only if all required chunks are in place res.removeLocal = res.removeLocal && p.pullRequiredECChunks(ctx, objInfo, nn[0]) } - p.adjustECPlacement(ctx, objInfo, nn[0]) + p.adjustECPlacement(ctx, objInfo, nn[0], policy) if res.removeLocal { p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, zap.Stringer("object", objInfo.Address)) @@ -224,7 +227,7 @@ func (p *Policer) resolveRemoteECChunks(ctx context.Context, parentAddress oid.A } if existed, ok := indexToObjectID[ch.Index]; ok && existed != chunkID { p.log.Error(logs.PolicerDifferentObjectIDForTheSameECChunk, zap.Stringer("first", existed), - zap.Stringer("second", chunkID), zap.Stringer("object", parentAddress)) + zap.Stringer("second", chunkID), zap.Stringer("object", parentAddress), zap.Uint32("index", ch.Index)) return false } indexToObjectID[ch.Index] = chunkID @@ -242,26 +245,146 @@ func (p *Policer) resolveRemoteECChunks(ctx context.Context, parentAddress oid.A return true } -func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo) { +func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo, policy netmap.PlacementPolicy) { var parentAddress oid.Address parentAddress.SetContainer(objInfo.Address.Container()) parentAddress.SetObject(objInfo.ECInfo.ParentID) var eiErr *objectSDK.ECInfoError + resolved := make(map[uint32][]netmap.NodeInfo) + chunkIDs := make(map[uint32]oid.ID) + restore := true // do not restore EC chunks if some node returned error for idx, n := range nodes { - if uint32(idx) == objInfo.ECInfo.Total { + if uint32(idx) >= objInfo.ECInfo.Total && uint32(len(resolved)) == objInfo.ECInfo.Total { return } + var err error if p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) { - continue + _, err = p.localHeader(ctx, parentAddress) + } else { + _, err = p.remoteHeader(ctx, n, parentAddress, true) } - _, err := p.remoteHeader(ctx, n, parentAddress, true) + if errors.As(err, &eiErr) { + for _, ch := range eiErr.ECInfo().Chunks { + resolved[ch.Index] = append(resolved[ch.Index], n) + var ecInfoChunkID oid.ID + if err := ecInfoChunkID.ReadFromV2(ch.ID); err != nil { + p.log.Error(logs.PolicerFailedToDecodeECChunkID, zap.Error(err), zap.Stringer("object", parentAddress)) + return + } + if chunkID, exist := chunkIDs[ch.Index]; exist && chunkID != ecInfoChunkID { + p.log.Error(logs.PolicerDifferentObjectIDForTheSameECChunk, zap.Stringer("first", chunkID), + zap.Stringer("second", ecInfoChunkID), zap.Stringer("object", parentAddress), zap.Uint32("index", ch.Index)) + return + } + chunkIDs[ch.Index] = ecInfoChunkID + } + } else if !p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) && uint32(idx) < objInfo.ECInfo.Total { + p.log.Warn(logs.PolicerCouldNotGetObjectFromNodeMoving, zap.String("node", hex.EncodeToString(n.PublicKey())), zap.Stringer("object", parentAddress), zap.Error(err)) + p.replicator.HandleReplicationTask(ctx, replicator.Task{ + NumCopies: 1, + Addr: objInfo.Address, + Nodes: []netmap.NodeInfo{n}, + }, newNodeCache()) + restore = false + } + } + if !restore || uint32(len(resolved)) == objInfo.ECInfo.Total { + return + } + if objInfo.ECInfo.Total-uint32(len(resolved)) > policy.ReplicaDescriptor(0).GetECParityCount() { + var found []uint32 + for i := range resolved { + found = append(found, i) + } + p.log.Error(logs.PolicerCouldNotRestoreObjectNotEnoughChunks, zap.Stringer("object", parentAddress), zap.Uint32s("found_chunks", found)) + return + } + p.restoreECObject(ctx, objInfo, parentAddress, nodes, resolved, chunkIDs, policy) +} + +func (p *Policer) restoreECObject(ctx context.Context, objInfo objectcore.Info, parentAddress oid.Address, nodes []netmap.NodeInfo, existedChunks map[uint32][]netmap.NodeInfo, chunkIDs map[uint32]oid.ID, policy netmap.PlacementPolicy) { + c, err := erasurecode.NewConstructor(int(policy.ReplicaDescriptor(0).GetECDataCount()), int(policy.ReplicaDescriptor(0).GetECParityCount())) + if err != nil { + p.log.Error(logs.PolicerFailedToRestoreObject, zap.Stringer("object", parentAddress), zap.Error(err)) + return + } + parts := p.collectExistedChunks(ctx, objInfo, existedChunks, parentAddress, chunkIDs) + if parts == nil { + return + } + key, err := p.keyStorage.GetKey(nil) + if err != nil { + p.log.Error(logs.PolicerFailedToRestoreObject, zap.Stringer("object", parentAddress), zap.Error(err)) + return + } + required := make([]bool, len(parts)) + for i, p := range parts { + if p == nil { + required[i] = true + } + } + if err := c.ReconstructParts(parts, required, key); err != nil { + p.log.Error(logs.PolicerFailedToRestoreObject, zap.Stringer("object", parentAddress), zap.Error(err)) + return + } + for idx, part := range parts { + if _, exists := existedChunks[uint32(idx)]; exists { continue } - p.replicator.HandleReplicationTask(ctx, replicator.Task{ - NumCopies: 1, - Addr: objInfo.Address, - Nodes: []netmap.NodeInfo{n}, - }, newNodeCache()) + var addr oid.Address + addr.SetContainer(parentAddress.Container()) + pID, _ := part.ID() + addr.SetObject(pID) + targetNode := nodes[idx%len(nodes)] + if p.cfg.netmapKeys.IsLocalKey(targetNode.PublicKey()) { + p.replicator.HandleLocalPutTask(ctx, replicator.Task{ + Addr: addr, + Obj: part, + }) + } else { + p.replicator.HandleReplicationTask(ctx, replicator.Task{ + NumCopies: 1, + Addr: addr, + Nodes: []netmap.NodeInfo{targetNode}, + Obj: part, + }, newNodeCache()) + } } } + +func (p *Policer) collectExistedChunks(ctx context.Context, objInfo objectcore.Info, existedChunks map[uint32][]netmap.NodeInfo, parentAddress oid.Address, chunkIDs map[uint32]oid.ID) []*objectSDK.Object { + parts := make([]*objectSDK.Object, objInfo.ECInfo.Total) + errGroup, egCtx := errgroup.WithContext(ctx) + for idx, nodes := range existedChunks { + idx := idx + nodes := nodes + errGroup.Go(func() error { + var objID oid.Address + objID.SetContainer(parentAddress.Container()) + objID.SetObject(chunkIDs[idx]) + var obj *objectSDK.Object + var err error + for _, node := range nodes { + if p.cfg.netmapKeys.IsLocalKey(node.PublicKey()) { + obj, err = p.localObject(egCtx, objID) + } else { + obj, err = p.remoteObject(egCtx, node, objID) + } + if err == nil { + break + } + p.log.Warn(logs.PolicerCouldNotGetChunk, zap.Stringer("object", parentAddress), zap.Stringer("chunkID", objID), zap.Error(err), zap.String("node", hex.EncodeToString(node.PublicKey()))) + } + if obj != nil { + parts[idx] = obj + } + return nil + }) + } + if err := errGroup.Wait(); err != nil { + p.log.Error(logs.PolicerCouldNotGetChunks, zap.Stringer("object", parentAddress), zap.Error(err)) + return nil + } + return parts +} diff --git a/pkg/services/policer/ec_test.go b/pkg/services/policer/ec_test.go index 3957c112b..3ac9699f0 100644 --- a/pkg/services/policer/ec_test.go +++ b/pkg/services/policer/ec_test.go @@ -3,6 +3,7 @@ package policer import ( "bytes" "context" + "crypto/rand" "errors" "fmt" "sync/atomic" @@ -10,13 +11,16 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" ) @@ -52,7 +56,7 @@ func TestECChunkHasValidPlacement(t *testing.T) { return nil, errors.New("unexpected placement build") } - headFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) { + remoteHeadFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) { require.True(t, raw, "remote header for parent object must be called with raw flag") index := int(ni.PublicKey()[0]) require.True(t, index == 1 || index == 2, "invalid node to get parent header") @@ -66,13 +70,25 @@ func TestECChunkHasValidPlacement(t *testing.T) { return nil, objectSDK.NewECInfoError(ei) } + localHeadFn := func(_ context.Context, a oid.Address) (*objectSDK.Object, error) { + require.True(t, a.Container() == chunkAddress.Container() && a.Object() == parentID, "invalid address to get remote header") + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(oidtest.ID()) + ch.Index = uint32(0) + ch.Total = 3 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + p := New( WithContainerSource(containerSrc), WithPlacementBuilder(placementBuilderFunc(placementBuilder)), WithNetmapKeys(announcedKeysFunc(func(k []byte) bool { return bytes.Equal(k, nodes[0].PublicKey()) })), - WithRemoteObjectHeaderFunc(headFn), + WithRemoteObjectHeaderFunc(remoteHeadFn), + WithLocalObjectHeaderFunc(localHeadFn), WithPool(testPool(t)), ) @@ -209,11 +225,13 @@ func TestECChunkHasInvalidPlacement(t *testing.T) { })), WithRemoteObjectHeaderFunc(headFn), WithLocalObjectHeaderFunc(localHeadF), - WithReplicator(pullFunc(func(ctx context.Context, r replicator.Task) { - require.True(t, r.Addr.Container() == chunkAddress.Container() && r.Addr.Object() == requiredChunkID && - len(r.Nodes) == 1 && bytes.Equal(r.Nodes[0].PublicKey(), nodes[1].PublicKey()), "invalid pull task") - pullCounter.Add(1) - })), + WithReplicator(&testReplicator{ + handlePullTask: (func(ctx context.Context, r replicator.Task) { + require.True(t, r.Addr.Container() == chunkAddress.Container() && r.Addr.Object() == requiredChunkID && + len(r.Nodes) == 1 && bytes.Equal(r.Nodes[0].PublicKey(), nodes[1].PublicKey()), "invalid pull task") + pullCounter.Add(1) + }), + }), WithRedundantCopyCallback(func(ctx context.Context, a oid.Address) { require.True(t, allowDrop, "invalid redundent copy call") dropped = append(dropped, a) @@ -367,9 +385,11 @@ func TestECChunkHasInvalidPlacement(t *testing.T) { WithRedundantCopyCallback(func(ctx context.Context, a oid.Address) { dropped = append(dropped, a) }), - WithReplicator(replicatorFunc(func(ctx context.Context, t replicator.Task, tr replicator.TaskResult) { - replicated = append(replicated, t) - })), + WithReplicator(&testReplicator{ + handleReplicationTask: func(ctx context.Context, t replicator.Task, tr replicator.TaskResult) { + replicated = append(replicated, t) + }, + }), WithPool(testPool(t)), ) @@ -391,12 +411,155 @@ func TestECChunkHasInvalidPlacement(t *testing.T) { }) } -type pullFunc func(context.Context, replicator.Task) +func TestECChunkRestore(t *testing.T) { + // node0 has chunk0, node1 has chunk1 + // policer should replicate chunk0 to node2 on the first run + // then restore EC object and replicate chunk2 to node2 on the second run + t.Parallel() -func (f pullFunc) HandleReplicationTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) { - panic("not implemented") -} + payload := make([]byte, 64) + rand.Read(payload) + parentAddress := oidtest.Address() + parentObject := objectSDK.New() + parentObject.SetContainerID(parentAddress.Container()) + parentObject.SetPayload(payload) + parentObject.SetPayloadSize(64) + objectSDK.CalculateAndSetPayloadChecksum(parentObject) + err := objectSDK.CalculateAndSetID(parentObject) + require.NoError(t, err) + id, _ := parentObject.ID() + parentAddress.SetObject(id) -func (f pullFunc) HandlePullTask(ctx context.Context, task replicator.Task) { - f(ctx, task) + chunkIDs := make([]oid.ID, 3) + c, err := erasurecode.NewConstructor(2, 1) + require.NoError(t, err) + key, err := keys.NewPrivateKey() + require.NoError(t, err) + chunks, err := c.Split(parentObject, &key.PrivateKey) + require.NoError(t, err) + for i, ch := range chunks { + chunkIDs[i], _ = ch.ID() + } + + var policy netmapSDK.PlacementPolicy + require.NoError(t, policy.DecodeString("EC 2.1")) + + cnr := &container.Container{} + cnr.Value.Init() + cnr.Value.SetPlacementPolicy(policy) + containerSrc := containerSrc{ + get: func(id cid.ID) (*container.Container, error) { + if id.Equals(parentAddress.Container()) { + return cnr, nil + } + return nil, new(apistatus.ContainerNotFound) + }, + } + + nodes := make([]netmapSDK.NodeInfo, 4) + for i := range nodes { + nodes[i].SetPublicKey([]byte{byte(i)}) + } + + placementBuilder := func(cnr cid.ID, obj *oid.ID, p netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) { + if cnr.Equals(parentAddress.Container()) && obj.Equals(parentAddress.Object()) { + return [][]netmapSDK.NodeInfo{nodes}, nil + } + return nil, errors.New("unexpected placement build") + } + var secondRun bool + remoteHeadFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) { + require.True(t, raw, "remote header for parent object must be called with raw flag") + index := int(ni.PublicKey()[0]) + require.True(t, index == 1 || index == 2 || index == 3, "invalid node to get parent header") + require.True(t, a == parentAddress, "invalid address to get remote header") + if index == 1 { + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(chunkIDs[1]) + ch.Index = uint32(1) + ch.Total = 3 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + if index == 2 && secondRun { + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(chunkIDs[0]) + ch.Index = uint32(0) + ch.Total = 3 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + return nil, new(apistatus.ObjectNotFound) + } + + localHeadFn := func(_ context.Context, a oid.Address) (*objectSDK.Object, error) { + require.True(t, a == parentAddress, "invalid address to get remote header") + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(chunkIDs[0]) + ch.Index = uint32(0) + ch.Total = 3 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + + var replicatedObj []*objectSDK.Object + p := New( + WithContainerSource(containerSrc), + WithPlacementBuilder(placementBuilderFunc(placementBuilder)), + WithNetmapKeys(announcedKeysFunc(func(k []byte) bool { + return bytes.Equal(k, nodes[0].PublicKey()) + })), + WithRemoteObjectHeaderFunc(remoteHeadFn), + WithLocalObjectHeaderFunc(localHeadFn), + WithReplicator(&testReplicator{ + handleReplicationTask: func(ctx context.Context, t replicator.Task, tr replicator.TaskResult) { + if t.Obj != nil { + replicatedObj = append(replicatedObj, t.Obj) + } + }, + }), + WithLocalObjectGetFunc(func(ctx context.Context, a oid.Address) (*objectSDK.Object, error) { + require.True(t, a.Container() == parentAddress.Container() && a.Object() == chunkIDs[0], "invalid local object request") + return chunks[0], nil + }), + WithRemoteObjectGetFunc(func(ctx context.Context, ni netmapSDK.NodeInfo, a oid.Address) (*objectSDK.Object, error) { + index := ni.PublicKey()[0] + if index == 2 { + return nil, new(apistatus.ObjectNotFound) + } + return chunks[index], nil + }), + WithPool(testPool(t)), + WithKeyStorage(util.NewKeyStorage(&key.PrivateKey, nil, nil)), + ) + + var chunkAddress oid.Address + chunkAddress.SetContainer(parentAddress.Container()) + chunkAddress.SetObject(chunkIDs[0]) + objInfo := objectcore.Info{ + Address: chunkAddress, + Type: objectSDK.TypeRegular, + ECInfo: &objectcore.ECInfo{ + ParentID: parentAddress.Object(), + Index: 0, + Total: 3, + }, + } + err = p.processObject(context.Background(), objInfo) + require.NoError(t, err) + secondRun = true + err = p.processObject(context.Background(), objInfo) + require.NoError(t, err) + + require.Equal(t, 1, len(replicatedObj), "invalid replicated objects count") + chunks[2].SetSignature(nil) + expectedData, err := chunks[2].MarshalJSON() + require.NoError(t, err) + replicatedObj[0].SetSignature(nil) + actualData, err := replicatedObj[0].MarshalJSON() + require.NoError(t, err) + require.EqualValues(t, string(expectedData), string(actualData), "invalid restored objects") } diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 9a646eea6..9dbfd8b9f 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -37,6 +38,7 @@ type BuryFunc func(context.Context, oid.Address) error type Replicator interface { HandleReplicationTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) HandlePullTask(ctx context.Context, task replicator.Task) + HandleLocalPutTask(ctx context.Context, task replicator.Task) } // RemoteObjectHeaderFunc is the function to obtain HEAD info from a specific remote node. @@ -45,6 +47,10 @@ type RemoteObjectHeaderFunc func(context.Context, netmapSDK.NodeInfo, oid.Addres // LocalObjectHeaderFunc is the function to obtain HEAD info from the current node. type LocalObjectHeaderFunc func(context.Context, oid.Address) (*objectSDK.Object, error) +type RemoteObjectGetFunc func(context.Context, netmapSDK.NodeInfo, oid.Address) (*objectSDK.Object, error) + +type LocalObjectGetFunc func(context.Context, oid.Address) (*objectSDK.Object, error) + type cfg struct { headTimeout time.Duration @@ -75,6 +81,12 @@ type cfg struct { evictDuration, sleepDuration time.Duration metrics MetricsRegister + + remoteObject RemoteObjectGetFunc + + localObject LocalObjectGetFunc + + keyStorage *util.KeyStorage } func defaultCfg() *cfg { @@ -145,6 +157,18 @@ func WithLocalObjectHeaderFunc(v LocalObjectHeaderFunc) Option { } } +func WithRemoteObjectGetFunc(v RemoteObjectGetFunc) Option { + return func(c *cfg) { + c.remoteObject = v + } +} + +func WithLocalObjectGetFunc(v LocalObjectGetFunc) Option { + return func(c *cfg) { + c.localObject = v + } +} + // WithNetmapKeys returns option to set tool to work with announced public keys. func WithNetmapKeys(v netmap.AnnouncedKeys) Option { return func(c *cfg) { @@ -182,3 +206,9 @@ func WithMetrics(m MetricsRegister) Option { c.metrics = m } } + +func WithKeyStorage(ks *util.KeyStorage) Option { + return func(c *cfg) { + c.keyStorage = ks + } +} diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 072f21a6f..e353ea428 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -249,12 +249,14 @@ func TestProcessObject(t *testing.T) { require.True(t, a.Equals(addr), "unexpected redundant copy callback: a=%v", a) gotRemoveRedundant = true }), - WithReplicator(replicatorFunc(func(_ context.Context, task replicator.Task, res replicator.TaskResult) { - require.True(t, task.Addr.Equals(addr), "unexpected replicator task: %+v", task) - for _, node := range task.Nodes { - gotReplicateTo = append(gotReplicateTo, int(node.PublicKey()[0])) - } - })), + WithReplicator(&testReplicator{ + handleReplicationTask: func(_ context.Context, task replicator.Task, res replicator.TaskResult) { + require.True(t, task.Addr.Equals(addr), "unexpected replicator task: %+v", task) + for _, node := range task.Nodes { + gotReplicateTo = append(gotReplicateTo, int(node.PublicKey()[0])) + } + }, + }), WithPool(testPool(t)), ) @@ -440,13 +442,20 @@ type announcedKeysFunc func([]byte) bool func (f announcedKeysFunc) IsLocalKey(k []byte) bool { return f(k) } -// replicatorFunc is a Replicator backed by a function. -type replicatorFunc func(context.Context, replicator.Task, replicator.TaskResult) - -func (f replicatorFunc) HandleReplicationTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) { - f(ctx, task, res) +type testReplicator struct { + handleReplicationTask func(ctx context.Context, task replicator.Task, res replicator.TaskResult) + handleLocalPutTask func(ctx context.Context, task replicator.Task) + handlePullTask func(ctx context.Context, task replicator.Task) } -func (f replicatorFunc) HandlePullTask(ctx context.Context, task replicator.Task) { - panic("not implemented") +func (r *testReplicator) HandleReplicationTask(ctx context.Context, task replicator.Task, res replicator.TaskResult) { + r.handleReplicationTask(ctx, task, res) +} + +func (r *testReplicator) HandleLocalPutTask(ctx context.Context, task replicator.Task) { + r.handleLocalPutTask(ctx, task) +} + +func (r *testReplicator) HandlePullTask(ctx context.Context, task replicator.Task) { + r.handlePullTask(ctx, task) } diff --git a/pkg/services/replicator/put.go b/pkg/services/replicator/put.go new file mode 100644 index 000000000..c06ec3f65 --- /dev/null +++ b/pkg/services/replicator/put.go @@ -0,0 +1,47 @@ +package replicator + +import ( + "context" + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +var errObjectNotDefined = errors.New("object is not defined") + +func (p *Replicator) HandleLocalPutTask(ctx context.Context, task Task) { + p.metrics.IncInFlightRequest() + defer p.metrics.DecInFlightRequest() + defer func() { + p.log.Debug(logs.ReplicatorFinishWork, zap.String("type", "pull")) + }() + + ctx, span := tracing.StartSpanFromContext(ctx, "Replicator.HandleLocalPutTask", + trace.WithAttributes( + attribute.Stringer("address", task.Addr), + attribute.Int("nodes_count", len(task.Nodes)), + )) + defer span.End() + + if task.Obj == nil { + p.log.Error(logs.ReplicatorCouldNotPutObjectToLocalStorage, + zap.Stringer("object", task.Addr), + zap.Error(errObjectNotDefined), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + return + } + + err := engine.Put(ctx, p.localStorage, task.Obj) + if err != nil { + p.log.Error(logs.ReplicatorCouldNotPutObjectToLocalStorage, + zap.Stringer("object", task.Addr), + zap.Error(err), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } +} From 8fd678e269cda0808cc57fc628ef2d38ea67ca23 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 20 May 2024 11:45:40 +0300 Subject: [PATCH 1105/1943] [#1141] go.mod: Update frostfs-sdk-go and frostfs-api-go/v2 * Also fix unit-test. Signed-off-by: Airat Arifullin --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/services/policer/ec_test.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 875385203..81e2fa989 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240427200446-67c6f305b21f + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240516133103-0803bc6ded00 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240507063414-99e02858af12 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240517121628-3de256d05e32 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index 9bb97cbe1..5dd9b2423 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,15 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240427200446-67c6f305b21f h1:YyjsQNtrngQzIKOUtApXoi5r5pewatM+cXfpY19vZWo= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240427200446-67c6f305b21f/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240516133103-0803bc6ded00 h1:Q3B9WtFh05AXhUFs/2CLvhh9tuFs/Zd/XemWBbuzvg8= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240516133103-0803bc6ded00/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 h1:Tp4I+XOLp3VCJORfxSamQtj3RZNISbaLM4WD5iIzXxg= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240507063414-99e02858af12 h1:uEAn+TBXxgqXXygrYW2X8dFmGydGyJvBc5ysnfHFhYM= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240507063414-99e02858af12/go.mod h1:e7H9nNFpx1Tj3R20Zoxy0Vo6Srlb6zV5L7ZQXqg9rn4= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240517121628-3de256d05e32 h1:ebXBsKCAINoJLhBUqi1x3fToxUHUsUjEBGKo86JQoho= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240517121628-3de256d05e32/go.mod h1:Wu1jsGM6JhZwS5oMWeslKvMcvRbg8nGt/B/TfsQPoFA= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d h1:hHk8FWbWgEnwm2I045CaBIrZBjy/o81CehIVOySA/pQ= diff --git a/pkg/services/policer/ec_test.go b/pkg/services/policer/ec_test.go index 3ac9699f0..11e7a8cdf 100644 --- a/pkg/services/policer/ec_test.go +++ b/pkg/services/policer/ec_test.go @@ -114,7 +114,7 @@ func TestECChunkHasInvalidPlacement(t *testing.T) { chunkObject.SetID(chunkAddress.Object()) chunkObject.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) chunkObject.SetPayloadSize(uint64(10)) - chunkObject.SetECHeader(objectSDK.NewECHeader(parentID, 1, 3, []byte{}, 0)) + chunkObject.SetECHeader(objectSDK.NewECHeader(parentID, nil, nil, 1, 3, []byte{}, 0)) var policy netmapSDK.PlacementPolicy require.NoError(t, policy.DecodeString("EC 2.1")) From 89a80e9a0f007793bc08eb094c91576c7307827f Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 20 May 2024 13:14:27 +0300 Subject: [PATCH 1106/1943] [#1141] metabase: Fix `putUniqueIndexItem` * `GetECHeader` is not correct way to determine if an object's got EC-header: `ECHeader` must be used for that. Signed-off-by: Airat Arifullin --- pkg/local_object_storage/metabase/put.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 2822303a0..8b7b83016 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -266,7 +266,7 @@ func putUniqueIndexes( return err } - if ecHead := obj.GetECHeader(); ecHead != nil { + if ecHead := obj.ECHeader(); ecHead != nil { err = putECInfo(tx, cnr, objKey, ecHead) if err != nil { return err From 40b04c00efb305596f7db82ef771a5ac65b20bed Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 20 May 2024 13:19:51 +0300 Subject: [PATCH 1107/1943] [#1141] metabase: Fix `IsUserObject` method Signed-off-by: Airat Arifullin --- pkg/local_object_storage/metabase/counter.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index 4d860261c..275099ff2 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -527,10 +527,14 @@ func containerObjectCounterInitialized(tx *bbolt.Tx) bool { } func IsUserObject(obj *objectSDK.Object) bool { - _, hasParentID := obj.ParentID() - return obj.Type() == objectSDK.TypeRegular && - (obj.SplitID() == nil || - (hasParentID && len(obj.Children()) == 0)) + ech := obj.ECHeader() + if ech == nil { + _, hasParentID := obj.ParentID() + return obj.Type() == objectSDK.TypeRegular && + (obj.SplitID() == nil || + (hasParentID && len(obj.Children()) == 0)) + } + return ech.Index() == 0 && (ech.ParentSplitID() == nil || ech.ParentSplitParentID() != nil) } // ZeroSizeContainers returns containers with size = 0. From 2697d4d1fe0fae4f684521fdfbd6408b9fcd1099 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 20 May 2024 18:40:16 +0300 Subject: [PATCH 1108/1943] [#1143] node: Fix frostfsid cache * Fix `subjectFromSubjectExtended` converter. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/frostfsid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-node/frostfsid.go b/cmd/frostfs-node/frostfsid.go index 68cc1d040..fb55a6019 100644 --- a/cmd/frostfs-node/frostfsid.go +++ b/cmd/frostfs-node/frostfsid.go @@ -64,7 +64,7 @@ func subjectFromSubjectExtended(subjExt *client.SubjectExtended) *client.Subject return &client.Subject{ PrimaryKey: subjExt.PrimaryKey, AdditionalKeys: subjExt.AdditionalKeys, - Namespace: subjExt.Name, + Namespace: subjExt.Namespace, Name: subjExt.Name, KV: subjExt.KV, } From 51ade979e863af0bd20c5694f750633f9dd9b066 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 24 Apr 2024 19:46:59 +0300 Subject: [PATCH 1109/1943] [#1105] ape: Introduce contract storage with proxy contract verification * `ProxyVerificationContractStorage` uses Proxy contract as a cosigner. * `ProxyVerificationContractStorage` recreates a contract storage for each handler invocation because of an issue: rpc-actor from morph client may be expired. This way won't create a bottlenecks because it is expected that this contract storage implementation will be used not so often. * Make morph client return `RPCActor` (that is websocket client in fact). * Make `SwitchRPCGuardedActor` return `RPCActor` as it will be used for `ProxyVerificationContractStorage`. Signed-off-by: Airat Arifullin --- pkg/ape/contract_storage/proxy.go | 128 ++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 pkg/ape/contract_storage/proxy.go diff --git a/pkg/ape/contract_storage/proxy.go b/pkg/ape/contract_storage/proxy.go new file mode 100644 index 000000000..953b91a79 --- /dev/null +++ b/pkg/ape/contract_storage/proxy.go @@ -0,0 +1,128 @@ +package contractstorage + +import ( + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + policy_morph "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/notary" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/wallet" +) + +type ProxyAdaptedContractStorage interface { + AddMorphRuleChain(name chain.Name, target engine.Target, c *chain.Chain) (util.Uint256, uint32, error) + + RemoveMorphRuleChain(name chain.Name, target engine.Target, chainID chain.ID) (util.Uint256, uint32, error) + + ListMorphRuleChains(name chain.Name, target engine.Target) ([]*chain.Chain, error) +} + +var _ ProxyAdaptedContractStorage = (engine.MorphRuleChainStorage)(nil) + +type RPCActorProvider interface { + GetRPCActor() actor.RPCActor +} + +// ProxyVerificationContractStorage uses decorated MorphRuleChainStorage with actor where cosigner is a proxy contract. +type ProxyVerificationContractStorage struct { + rpcActorProvider RPCActorProvider + + acc *wallet.Account + + proxyScriptHash util.Uint160 + + policyScriptHash util.Uint160 +} + +var _ ProxyAdaptedContractStorage = (*ProxyVerificationContractStorage)(nil) + +func NewProxyVerificationContractStorage(rpcActorProvider RPCActorProvider, key *keys.PrivateKey, proxyScriptHash, policyScriptHash util.Uint160) *ProxyVerificationContractStorage { + return &ProxyVerificationContractStorage{ + rpcActorProvider: rpcActorProvider, + + acc: wallet.NewAccountFromPrivateKey(key), + + proxyScriptHash: proxyScriptHash, + + policyScriptHash: policyScriptHash, + } +} + +// contractStorageActorAdapter adapats *actor.Actor to policy_morph.ContractStorageActor interface. +type contractStorageActorAdapter struct { + *actor.Actor + rpcActor invoker.RPCInvoke +} + +func (n *contractStorageActorAdapter) GetRPCInvoker() invoker.RPCInvoke { + return n.rpcActor +} + +func (contractStorage *ProxyVerificationContractStorage) newContractStorageActor() (policy_morph.ContractStorageActor, error) { + rpcActor := contractStorage.rpcActorProvider.GetRPCActor() + act, err := actor.New(rpcActor, cosigners(contractStorage.acc, contractStorage.proxyScriptHash, contractStorage.policyScriptHash)) + if err != nil { + return nil, err + } + return &contractStorageActorAdapter{ + Actor: act, + rpcActor: rpcActor, + }, nil +} + +// AddMorphRuleChain add morph rule chain to Policy contract using both Proxy contract and storage account as consigners. +func (contractStorage *ProxyVerificationContractStorage) AddMorphRuleChain(name chain.Name, target engine.Target, c *chain.Chain) (util.Uint256, uint32, error) { + // contractStorageActor is reconstructed per each method invocation because RPCActor's (that is, basically, WSClient) connection may get invalidated, but + // ProxyVerificationContractStorage does not manage reconnections. + contractStorageActor, err := contractStorage.newContractStorageActor() + if err != nil { + return util.Uint256{}, 0, err + } + return policy_morph.NewContractStorage(contractStorageActor, contractStorage.policyScriptHash).AddMorphRuleChain(name, target, c) +} + +// RemoveMorphRuleChain removes morph rule chain from Policy contract using both Proxy contract and storage account as consigners. +func (contractStorage *ProxyVerificationContractStorage) RemoveMorphRuleChain(name chain.Name, target engine.Target, chainID chain.ID) (util.Uint256, uint32, error) { + // contractStorageActor is reconstructed per each method invocation because RPCActor's (that is, basically, WSClient) connection may get invalidated, but + // ProxyVerificationContractStorage does not manage reconnections. + contractStorageActor, err := contractStorage.newContractStorageActor() + if err != nil { + return util.Uint256{}, 0, err + } + return policy_morph.NewContractStorage(contractStorageActor, contractStorage.policyScriptHash).RemoveMorphRuleChain(name, target, chainID) +} + +// ListMorphRuleChains lists morph rule chains from Policy contract using both Proxy contract and storage account as consigners. +func (contractStorage *ProxyVerificationContractStorage) ListMorphRuleChains(name chain.Name, target engine.Target) ([]*chain.Chain, error) { + // contractStorageActor is reconstructed per each method invocation because RPCActor's (that is, basically, WSClient) connection may get invalidated, but + // ProxyVerificationContractStorage does not manage reconnections. + contractStorageActor, err := contractStorage.newContractStorageActor() + if err != nil { + return nil, err + } + return policy_morph.NewContractStorage(contractStorageActor, contractStorage.policyScriptHash).ListMorphRuleChains(name, target) +} + +func cosigners(acc *wallet.Account, proxyScriptHash, policyScriptHash util.Uint160) []actor.SignerAccount { + return []actor.SignerAccount{ + { + Signer: transaction.Signer{ + Account: proxyScriptHash, + Scopes: transaction.CustomContracts, + AllowedContracts: []util.Uint160{policyScriptHash}, + }, + Account: notary.FakeContractAccount(proxyScriptHash), + }, + { + Signer: transaction.Signer{ + Account: acc.Contract.ScriptHash(), + Scopes: transaction.CalledByEntry, + }, + Account: acc, + }, + } +} From 542d3adcb2b78ea1dee6f546a933802b2f1d5c56 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 24 Apr 2024 20:23:15 +0300 Subject: [PATCH 1110/1943] [#1105] apemanager: Implement apemanager service * Introduce grpc server for apemanager service and its implementation in `pkg/services/apemanager`. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/apemanager.go | 29 +++ cmd/frostfs-node/main.go | 1 + .../transport/apemanager/grpc/service.go | 63 +++++ pkg/services/apemanager/errors/errors.go | 11 + pkg/services/apemanager/executor.go | 245 ++++++++++++++++++ pkg/services/apemanager/server.go | 13 + pkg/services/apemanager/sign.go | 49 ++++ pkg/services/apemanager/validation.go | 23 ++ 8 files changed, 434 insertions(+) create mode 100644 cmd/frostfs-node/apemanager.go create mode 100644 pkg/network/transport/apemanager/grpc/service.go create mode 100644 pkg/services/apemanager/errors/errors.go create mode 100644 pkg/services/apemanager/executor.go create mode 100644 pkg/services/apemanager/server.go create mode 100644 pkg/services/apemanager/sign.go create mode 100644 pkg/services/apemanager/validation.go diff --git a/cmd/frostfs-node/apemanager.go b/cmd/frostfs-node/apemanager.go new file mode 100644 index 000000000..b9928faaf --- /dev/null +++ b/cmd/frostfs-node/apemanager.go @@ -0,0 +1,29 @@ +package main + +import ( + "net" + + apemanager_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager/grpc" + ape_contract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/contract_storage" + morph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + apemanager_transport "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/apemanager/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/apemanager" + "google.golang.org/grpc" +) + +func initAPEManagerService(c *cfg) { + contractStorage := ape_contract.NewProxyVerificationContractStorage( + morph.NewSwitchRPCGuardedActor(c.cfgMorph.client), + c.shared.key, + c.cfgMorph.proxyScriptHash, + c.cfgObject.cfgAccessPolicyEngine.policyContractHash) + + execsvc := apemanager.New(c.cfgObject.cnrSource, contractStorage, + apemanager.WithLogger(c.log)) + sigsvc := apemanager.NewSignService(&c.key.PrivateKey, execsvc) + server := apemanager_transport.New(sigsvc) + + c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { + apemanager_grpc.RegisterAPEManagerServiceServer(s, server) + }) +} diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index e1eeb6861..dbef1e494 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -114,6 +114,7 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(c, "notification", func(c *cfg) { initNotifications(ctx, c) }) initAndLog(c, "object", initObjectService) initAndLog(c, "tree", initTreeService) + initAndLog(c, "apemanager", initAPEManagerService) initAndLog(c, "control", initControlService) initAndLog(c, "morph notifications", func(c *cfg) { listenMorphNotifications(ctx, c) }) diff --git a/pkg/network/transport/apemanager/grpc/service.go b/pkg/network/transport/apemanager/grpc/service.go new file mode 100644 index 000000000..59783cfc0 --- /dev/null +++ b/pkg/network/transport/apemanager/grpc/service.go @@ -0,0 +1,63 @@ +package apemanager + +import ( + "context" + + apemanager_v2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" + apemanager_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager/grpc" + apemanager_svc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/apemanager" +) + +type Server struct { + srv apemanager_svc.Server +} + +var _ apemanager_grpc.APEManagerServiceServer = (*Server)(nil) + +func New(c apemanager_svc.Server) *Server { + return &Server{ + srv: c, + } +} + +func (s *Server) AddChain(ctx context.Context, req *apemanager_grpc.AddChainRequest) (*apemanager_grpc.AddChainResponse, error) { + v2req := new(apemanager_v2.AddChainRequest) + if err := v2req.FromGRPCMessage(req); err != nil { + return nil, err + } + + resp, err := s.srv.AddChain(ctx, v2req) + if err != nil { + return nil, err + } + + return resp.ToGRPCMessage().(*apemanager_grpc.AddChainResponse), nil +} + +func (s *Server) RemoveChain(ctx context.Context, req *apemanager_grpc.RemoveChainRequest) (*apemanager_grpc.RemoveChainResponse, error) { + v2req := new(apemanager_v2.RemoveChainRequest) + if err := v2req.FromGRPCMessage(req); err != nil { + return nil, err + } + + resp, err := s.srv.RemoveChain(ctx, v2req) + if err != nil { + return nil, err + } + + return resp.ToGRPCMessage().(*apemanager_grpc.RemoveChainResponse), nil +} + +func (s *Server) ListChains(ctx context.Context, req *apemanager_grpc.ListChainsRequest) (*apemanager_grpc.ListChainsResponse, error) { + v2req := new(apemanager_v2.ListChainsRequest) + if err := v2req.FromGRPCMessage(req); err != nil { + return nil, err + } + + resp, err := s.srv.ListChains(ctx, v2req) + if err != nil { + return nil, err + } + + return resp.ToGRPCMessage().(*apemanager_grpc.ListChainsResponse), nil +} diff --git a/pkg/services/apemanager/errors/errors.go b/pkg/services/apemanager/errors/errors.go new file mode 100644 index 000000000..e64f9a8d1 --- /dev/null +++ b/pkg/services/apemanager/errors/errors.go @@ -0,0 +1,11 @@ +package errors + +import ( + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" +) + +func ErrAPEManagerAccessDenied(reason string) error { + err := new(apistatus.APEManagerAccessDenied) + err.WriteReason(reason) + return err +} diff --git a/pkg/services/apemanager/executor.go b/pkg/services/apemanager/executor.go new file mode 100644 index 000000000..bad43b5fa --- /dev/null +++ b/pkg/services/apemanager/executor.go @@ -0,0 +1,245 @@ +package apemanager + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "errors" + "fmt" + + apemanager_v2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + ape_contract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/contract_storage" + containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + apemanager_errors "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/apemanager/errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + policy_engine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "github.com/mr-tron/base58/base58" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "go.uber.org/zap" +) + +var ( + errEmptyBodySignature = errors.New("malformed request: empty body signature") +) + +type cfg struct { + log *logger.Logger +} + +type Service struct { + cfg + + cnrSrc containercore.Source + + contractStorage ape_contract.ProxyAdaptedContractStorage +} + +type Option func(*cfg) + +func New(cnrSrc containercore.Source, contractStorage ape_contract.ProxyAdaptedContractStorage, opts ...Option) *Service { + s := &Service{ + cnrSrc: cnrSrc, + + contractStorage: contractStorage, + } + + for i := range opts { + opts[i](&s.cfg) + } + + if s.log == nil { + s.log = &logger.Logger{Logger: zap.NewNop()} + } + + return s +} + +func WithLogger(log *logger.Logger) Option { + return func(c *cfg) { + c.log = log + } +} + +var _ Server = (*Service)(nil) + +// validateContainerTargetRequest validates request for the container target. +// It checks if request actor is the owner of the container, otherwise it denies the request. +func (s *Service) validateContainerTargetRequest(cid string, pubKey *keys.PublicKey) error { + var cidSDK cidSDK.ID + if err := cidSDK.DecodeString(cid); err != nil { + return fmt.Errorf("invalid CID format: %w", err) + } + isOwner, err := s.isActorContainerOwner(cidSDK, pubKey) + if err != nil { + return fmt.Errorf("failed to check owner: %w", err) + } + if !isOwner { + return apemanager_errors.ErrAPEManagerAccessDenied("actor must be container owner") + } + return nil +} + +func (s *Service) AddChain(_ context.Context, req *apemanager_v2.AddChainRequest) (*apemanager_v2.AddChainResponse, error) { + pub, err := getSignaturePublicKey(req.GetVerificationHeader()) + if err != nil { + return nil, err + } + + chain, err := decodeAndValidateChain(req.GetBody().GetChain().GetKind().(*apemanager_v2.ChainRaw).GetRaw()) + if err != nil { + return nil, err + } + if len(chain.ID) == 0 { + const randomIDLength = 10 + randID, err := base58Str(randomIDLength) + if err != nil { + return nil, fmt.Errorf("randomize chain ID error: %w", err) + } + chain.ID = apechain.ID(randID) + } + + var target policy_engine.Target + + switch targetType := req.GetBody().GetTarget().GetTargetType(); targetType { + case apemanager_v2.TargetTypeContainer: + reqCID := req.GetBody().GetTarget().GetName() + if err = s.validateContainerTargetRequest(reqCID, pub); err != nil { + return nil, err + } + target = policy_engine.ContainerTarget(reqCID) + default: + return nil, fmt.Errorf("unsupported target type: %s", targetType) + } + + if _, _, err = s.contractStorage.AddMorphRuleChain(apechain.Ingress, target, &chain); err != nil { + return nil, err + } + + body := new(apemanager_v2.AddChainResponseBody) + body.SetChainID(chain.ID) + + resp := new(apemanager_v2.AddChainResponse) + resp.SetBody(body) + + return resp, nil +} + +func (s *Service) RemoveChain(_ context.Context, req *apemanager_v2.RemoveChainRequest) (*apemanager_v2.RemoveChainResponse, error) { + pub, err := getSignaturePublicKey(req.GetVerificationHeader()) + if err != nil { + return nil, err + } + + var target policy_engine.Target + + switch targetType := req.GetBody().GetTarget().GetTargetType(); targetType { + case apemanager_v2.TargetTypeContainer: + reqCID := req.GetBody().GetTarget().GetName() + if err = s.validateContainerTargetRequest(reqCID, pub); err != nil { + return nil, err + } + target = policy_engine.ContainerTarget(reqCID) + default: + return nil, fmt.Errorf("unsupported target type: %s", targetType) + } + + if _, _, err = s.contractStorage.RemoveMorphRuleChain(apechain.Ingress, target, req.GetBody().GetChainID()); err != nil { + return nil, err + } + + body := new(apemanager_v2.RemoveChainResponseBody) + + resp := new(apemanager_v2.RemoveChainResponse) + resp.SetBody(body) + + return resp, nil +} + +func (s *Service) ListChains(_ context.Context, req *apemanager_v2.ListChainsRequest) (*apemanager_v2.ListChainsResponse, error) { + pub, err := getSignaturePublicKey(req.GetVerificationHeader()) + if err != nil { + return nil, err + } + + var target policy_engine.Target + + switch targetType := req.GetBody().GetTarget().GetTargetType(); targetType { + case apemanager_v2.TargetTypeContainer: + reqCID := req.GetBody().GetTarget().GetName() + if err = s.validateContainerTargetRequest(reqCID, pub); err != nil { + return nil, err + } + target = policy_engine.ContainerTarget(reqCID) + default: + return nil, fmt.Errorf("unsupported target type: %s", targetType) + } + + chs, err := s.contractStorage.ListMorphRuleChains(apechain.Ingress, target) + if err != nil { + return nil, err + } + + res := make([]*apemanager_v2.Chain, 0, len(chs)) + for _, ch := range chs { + v2chraw := new(apemanager_v2.ChainRaw) + v2chraw.SetRaw(ch.Bytes()) + + v2ch := new(apemanager_v2.Chain) + v2ch.SetKind(v2chraw) + + res = append(res, v2ch) + } + + body := new(apemanager_v2.ListChainsResponseBody) + body.SetChains(res) + + resp := new(apemanager_v2.ListChainsResponse) + resp.SetBody(body) + + return resp, nil +} + +func getSignaturePublicKey(vh *session.RequestVerificationHeader) (*keys.PublicKey, error) { + for vh.GetOrigin() != nil { + vh = vh.GetOrigin() + } + sig := vh.GetBodySignature() + if sig == nil { + return nil, errEmptyBodySignature + } + key, err := keys.NewPublicKeyFromBytes(sig.GetKey(), elliptic.P256()) + if err != nil { + return nil, fmt.Errorf("invalid signature key: %w", err) + } + + return key, nil +} + +func (s *Service) isActorContainerOwner(cid cidSDK.ID, pk *keys.PublicKey) (bool, error) { + var actor user.ID + user.IDFromKey(&actor, (ecdsa.PublicKey)(*pk)) + actorOwnerID := new(refs.OwnerID) + actor.WriteToV2(actorOwnerID) + + cnr, err := s.cnrSrc.Get(cid) + if err != nil { + return false, fmt.Errorf("get container error: %w", err) + } + return cnr.Value.Owner().Equals(actor), nil +} + +// base58Str generates base58 string. +func base58Str(n int) (string, error) { + b := make([]byte, n) + _, err := rand.Read(b) + if err != nil { + return "", err + } + return base58.FastBase58Encoding(b), nil +} diff --git a/pkg/services/apemanager/server.go b/pkg/services/apemanager/server.go new file mode 100644 index 000000000..90b2d92ae --- /dev/null +++ b/pkg/services/apemanager/server.go @@ -0,0 +1,13 @@ +package apemanager + +import ( + "context" + + apemanager_v2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" +) + +type Server interface { + AddChain(context.Context, *apemanager_v2.AddChainRequest) (*apemanager_v2.AddChainResponse, error) + RemoveChain(context.Context, *apemanager_v2.RemoveChainRequest) (*apemanager_v2.RemoveChainResponse, error) + ListChains(context.Context, *apemanager_v2.ListChainsRequest) (*apemanager_v2.ListChainsResponse, error) +} diff --git a/pkg/services/apemanager/sign.go b/pkg/services/apemanager/sign.go new file mode 100644 index 000000000..eda2a7342 --- /dev/null +++ b/pkg/services/apemanager/sign.go @@ -0,0 +1,49 @@ +package apemanager + +import ( + "context" + "crypto/ecdsa" + + apemanager_v2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" +) + +type signService struct { + sigSvc *util.SignService + + next Server +} + +func NewSignService(key *ecdsa.PrivateKey, next Server) Server { + return &signService{ + sigSvc: util.NewUnarySignService(key), + next: next, + } +} + +func (s *signService) AddChain(ctx context.Context, req *apemanager_v2.AddChainRequest) (*apemanager_v2.AddChainResponse, error) { + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(apemanager_v2.AddChainResponse) + return resp, s.sigSvc.SignResponse(resp, err) + } + resp, err := util.EnsureNonNilResponse(s.next.AddChain(ctx, req)) + return resp, s.sigSvc.SignResponse(resp, err) +} + +func (s *signService) RemoveChain(ctx context.Context, req *apemanager_v2.RemoveChainRequest) (*apemanager_v2.RemoveChainResponse, error) { + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(apemanager_v2.RemoveChainResponse) + return resp, s.sigSvc.SignResponse(resp, err) + } + resp, err := util.EnsureNonNilResponse(s.next.RemoveChain(ctx, req)) + return resp, s.sigSvc.SignResponse(resp, err) +} + +func (s *signService) ListChains(ctx context.Context, req *apemanager_v2.ListChainsRequest) (*apemanager_v2.ListChainsResponse, error) { + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(apemanager_v2.ListChainsResponse) + return resp, s.sigSvc.SignResponse(resp, err) + } + resp, err := util.EnsureNonNilResponse(s.next.ListChains(ctx, req)) + return resp, s.sigSvc.SignResponse(resp, err) +} diff --git a/pkg/services/apemanager/validation.go b/pkg/services/apemanager/validation.go new file mode 100644 index 000000000..b26fcf8ee --- /dev/null +++ b/pkg/services/apemanager/validation.go @@ -0,0 +1,23 @@ +package apemanager + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ape" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" +) + +func decodeAndValidateChain(encodedChain []byte) (chain apechain.Chain, err error) { + if err = chain.DecodeBytes(encodedChain); err != nil { + return + } + for _, rule := range chain.Rules { + for _, name := range rule.Resources.Names { + if err = ape.ValidateResourceName(name); err != nil { + err = fmt.Errorf("invalid resource: %w", err) + return + } + } + } + return +} From 2b02f52cd9cdc7fd7e930dac16d7fff0ceebc129 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 13 May 2024 19:07:44 +0300 Subject: [PATCH 1111/1943] [#1105] cli: Add apemanager commands Signed-off-by: Airat Arifullin --- .../modules/ape_manager/add_chain.go | 141 ++++++++++++++++++ .../modules/ape_manager/list_chain.go | 49 ++++++ .../modules/ape_manager/remove_chain.go | 66 ++++++++ cmd/frostfs-cli/modules/ape_manager/root.go | 21 +++ cmd/frostfs-cli/modules/root.go | 2 + 5 files changed, 279 insertions(+) create mode 100644 cmd/frostfs-cli/modules/ape_manager/add_chain.go create mode 100644 cmd/frostfs-cli/modules/ape_manager/list_chain.go create mode 100644 cmd/frostfs-cli/modules/ape_manager/remove_chain.go create mode 100644 cmd/frostfs-cli/modules/ape_manager/root.go diff --git a/cmd/frostfs-cli/modules/ape_manager/add_chain.go b/cmd/frostfs-cli/modules/ape_manager/add_chain.go new file mode 100644 index 000000000..7e6614392 --- /dev/null +++ b/cmd/frostfs-cli/modules/ape_manager/add_chain.go @@ -0,0 +1,141 @@ +package apemanager + +import ( + "encoding/hex" + "errors" + + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apemanager_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/apemanager" + client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "github.com/spf13/cobra" +) + +const ( + chainIDFlag = "chain-id" + chainIDHexFlag = "chain-id-hex" + ruleFlag = "rule" + pathFlag = "path" +) + +const ( + targetNameFlag = "target-name" + targetNameDesc = "Resource name in APE resource name format" + targetTypeFlag = "target-type" + targetTypeDesc = "Resource type(container/namespace)" +) + +const ( + defaultNamespace = "" + namespaceTarget = "namespace" + containerTarget = "container" + userTarget = "user" + groupTarget = "group" +) + +var ( + errUnknownTargetType = errors.New("unknown target type") +) + +var addCmd = &cobra.Command{ + Use: "add", + Short: "Add rule chain for a target", + Run: add, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func parseTarget(cmd *cobra.Command) (ct apemanager_sdk.ChainTarget) { + typ, _ := cmd.Flags().GetString(targetTypeFlag) + name, _ := cmd.Flags().GetString(targetNameFlag) + + ct.Name = name + + switch typ { + case namespaceTarget: + ct.TargetType = apemanager_sdk.TargetTypeNamespace + case containerTarget: + var cnr cid.ID + commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(name)) + ct.TargetType = apemanager_sdk.TargetTypeContainer + case userTarget: + ct.TargetType = apemanager_sdk.TargetTypeUser + case groupTarget: + ct.TargetType = apemanager_sdk.TargetTypeGroup + default: + commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType) + } + return ct +} + +func parseChain(cmd *cobra.Command) apemanager_sdk.Chain { + chainID, _ := cmd.Flags().GetString(chainIDFlag) + hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) + + chainIDRaw := []byte(chainID) + + if hexEncoded { + var err error + chainIDRaw, err = hex.DecodeString(chainID) + commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err) + } + + chain := new(apechain.Chain) + chain.ID = apechain.ID(chainIDRaw) + + if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 { + commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, rules)) + } else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" { + commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", util.ParseAPEChainBinaryOrJSON(chain, encPath)) + } else { + commonCmd.ExitOnErr(cmd, "parser error: %w", errors.New("rule is not passed")) + } + + cmd.Println("Parsed chain:") + util.PrintHumanReadableAPEChain(cmd, chain) + + serialized := chain.Bytes() + return apemanager_sdk.Chain{ + Raw: serialized, + } +} + +func add(cmd *cobra.Command, _ []string) { + c := parseChain(cmd) + + target := parseTarget(cmd) + + key := key.Get(cmd) + cli := internalclient.GetSDKClientByFlag(cmd, key, commonflags.RPC) + + res, err := cli.APEManagerAddChain(cmd.Context(), client_sdk.PrmAPEManagerAddChain{ + ChainTarget: target, + Chain: c, + }) + + commonCmd.ExitOnErr(cmd, "add chain error: %w", err) + + cmd.Println("Rule has been added.") + cmd.Println("Chain ID: ", string(res.ChainID)) +} + +func initAddCmd() { + commonflags.Init(addCmd) + + ff := addCmd.Flags() + ff.StringArray(ruleFlag, []string{}, "Rule statement") + ff.String(pathFlag, "", "Path to encoded chain in JSON or binary format") + ff.String(chainIDFlag, "", "Assign ID to the parsed chain") + ff.String(targetNameFlag, "", targetNameDesc) + ff.String(targetTypeFlag, "", targetTypeDesc) + _ = addCmd.MarkFlagRequired(targetTypeFlag) + ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") + + addCmd.MarkFlagsMutuallyExclusive(pathFlag, ruleFlag) +} diff --git a/cmd/frostfs-cli/modules/ape_manager/list_chain.go b/cmd/frostfs-cli/modules/ape_manager/list_chain.go new file mode 100644 index 000000000..a5dd44614 --- /dev/null +++ b/cmd/frostfs-cli/modules/ape_manager/list_chain.go @@ -0,0 +1,49 @@ +package apemanager + +import ( + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + apeutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "github.com/spf13/cobra" +) + +var listCmd = &cobra.Command{ + Use: "list", + Short: "List rule chains defined on target", + Run: list, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} + +func list(cmd *cobra.Command, _ []string) { + target := parseTarget(cmd) + + key := key.Get(cmd) + cli := internalclient.GetSDKClientByFlag(cmd, key, commonflags.RPC) + + resp, err := cli.APEManagerListChains(cmd.Context(), + client_sdk.PrmAPEManagerListChains{ + ChainTarget: target, + }) + commonCmd.ExitOnErr(cmd, "list chains call error: %w", err) + + for _, respChain := range resp.Chains { + var chain apechain.Chain + commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(respChain.Raw)) + apeutil.PrintHumanReadableAPEChain(cmd, &chain) + } +} + +func initListCmd() { + commonflags.Init(listCmd) + + ff := listCmd.Flags() + ff.String(targetNameFlag, "", targetNameDesc) + ff.String(targetTypeFlag, "", targetTypeDesc) + _ = listCmd.MarkFlagRequired(targetTypeFlag) +} diff --git a/cmd/frostfs-cli/modules/ape_manager/remove_chain.go b/cmd/frostfs-cli/modules/ape_manager/remove_chain.go new file mode 100644 index 000000000..179bd5c9e --- /dev/null +++ b/cmd/frostfs-cli/modules/ape_manager/remove_chain.go @@ -0,0 +1,66 @@ +package apemanager + +import ( + "encoding/hex" + "errors" + + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + "github.com/spf13/cobra" +) + +var ( + errEmptyChainID = errors.New("chain id cannot be empty") + + removeCmd = &cobra.Command{ + Use: "remove", + Short: "Remove rule chain for a target", + Run: remove, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, + } +) + +func remove(cmd *cobra.Command, _ []string) { + target := parseTarget(cmd) + + key := key.Get(cmd) + cli := internalclient.GetSDKClientByFlag(cmd, key, commonflags.RPC) + + chainID, _ := cmd.Flags().GetString(chainIDFlag) + if chainID == "" { + commonCmd.ExitOnErr(cmd, "read chain id error: %w", errEmptyChainID) + } + chainIDRaw := []byte(chainID) + + hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) + if hexEncoded { + var err error + chainIDRaw, err = hex.DecodeString(chainID) + commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err) + } + + _, err := cli.APEManagerRemoveChain(cmd.Context(), client_sdk.PrmAPEManagerRemoveChain{ + ChainTarget: target, + ChainID: chainIDRaw, + }) + + commonCmd.ExitOnErr(cmd, "remove chain error: %w", err) + + cmd.Println("\nRule has been removed.") +} + +func initRemoveCmd() { + commonflags.Init(removeCmd) + + ff := removeCmd.Flags() + ff.String(targetNameFlag, "", targetNameDesc) + ff.String(targetTypeFlag, "", targetTypeDesc) + _ = removeCmd.MarkFlagRequired(targetTypeFlag) + ff.String(chainIDFlag, "", "Chain id") + ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") +} diff --git a/cmd/frostfs-cli/modules/ape_manager/root.go b/cmd/frostfs-cli/modules/ape_manager/root.go new file mode 100644 index 000000000..7b4f92921 --- /dev/null +++ b/cmd/frostfs-cli/modules/ape_manager/root.go @@ -0,0 +1,21 @@ +package apemanager + +import ( + "github.com/spf13/cobra" +) + +var Cmd = &cobra.Command{ + Use: "ape-manager", + Short: "Operations with APE manager", + Long: `Operations with APE manager`, +} + +func init() { + Cmd.AddCommand(addCmd) + Cmd.AddCommand(removeCmd) + Cmd.AddCommand(listCmd) + + initAddCmd() + initRemoveCmd() + initListCmd() +} diff --git a/cmd/frostfs-cli/modules/root.go b/cmd/frostfs-cli/modules/root.go index 87e6a905f..21c367d29 100644 --- a/cmd/frostfs-cli/modules/root.go +++ b/cmd/frostfs-cli/modules/root.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" accountingCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/accounting" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/acl" + apemanager "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/ape_manager" bearerCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/bearer" containerCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/container" controlCli "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/control" @@ -81,6 +82,7 @@ func init() { rootCmd.Flags().Bool("version", false, "Application version and FrostFS API compatibility") rootCmd.AddCommand(acl.Cmd) + rootCmd.AddCommand(apemanager.Cmd) rootCmd.AddCommand(bearerCli.Cmd) rootCmd.AddCommand(sessionCli.Cmd) rootCmd.AddCommand(accountingCli.Cmd) From 43625e7536a15cc625597a495858aea5a8812f09 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 20 May 2024 16:11:44 +0300 Subject: [PATCH 1112/1943] [#1142] container: Fill APE-request property with source IP Signed-off-by: Airat Arifullin --- pkg/services/container/ape.go | 32 ++++++-- pkg/services/container/ape_test.go | 127 +++++++++++++++++++++++++++-- 2 files changed, 146 insertions(+), 13 deletions(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 22440ab80..fb1cc8121 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "errors" "fmt" + "net" "strings" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" @@ -27,8 +28,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "google.golang.org/grpc/peer" ) var ( @@ -88,7 +91,7 @@ func (ac *apeChecker) Delete(ctx context.Context, req *container.DeleteRequest) ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.Delete") defer span.End() - if err := ac.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), + if err := ac.validateContainerBoundedOperation(ctx, req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodDeleteContainer); err != nil { return nil, err } @@ -100,7 +103,7 @@ func (ac *apeChecker) Get(ctx context.Context, req *container.GetRequest) (*cont ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.Get") defer span.End() - if err := ac.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), + if err := ac.validateContainerBoundedOperation(ctx, req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer); err != nil { return nil, err } @@ -112,7 +115,7 @@ func (ac *apeChecker) GetExtendedACL(ctx context.Context, req *container.GetExte ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.GetExtendedACL") defer span.End() - if err := ac.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), + if err := ac.validateContainerBoundedOperation(ctx, req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainerEACL); err != nil { return nil, err } @@ -138,6 +141,11 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co if err != nil { return nil, err } + if p, ok := peer.FromContext(ctx); ok { + if tcpAddr, ok := p.Addr.(*net.TCPAddr); ok { + reqProps[commonschema.PropertyKeyFrostFSSourceIP] = tcpAddr.IP.String() + } + } namespace, err := ac.namespaceByOwner(req.GetBody().GetOwnerID()) if err != nil { @@ -191,6 +199,11 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont if err != nil { return nil, err } + if p, ok := peer.FromContext(ctx); ok { + if tcpAddr, ok := p.Addr.(*net.TCPAddr); ok { + reqProps[commonschema.PropertyKeyFrostFSSourceIP] = tcpAddr.IP.String() + } + } namespace, err := ac.namespaceByOwner(req.GetBody().GetContainer().GetOwnerID()) if err != nil { @@ -264,7 +277,7 @@ func (ac *apeChecker) SetExtendedACL(ctx context.Context, req *container.SetExte ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.SetExtendedACL") defer span.End() - if err := ac.validateContainerBoundedOperation(req.GetBody().GetEACL().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), + if err := ac.validateContainerBoundedOperation(ctx, req.GetBody().GetEACL().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodSetContainerEACL); err != nil { return nil, err } @@ -272,7 +285,7 @@ func (ac *apeChecker) SetExtendedACL(ctx context.Context, req *container.SetExte return ac.next.SetExtendedACL(ctx, req) } -func (ac *apeChecker) validateContainerBoundedOperation(containerID *refs.ContainerID, mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader, op string) error { +func (ac *apeChecker) validateContainerBoundedOperation(ctx context.Context, containerID *refs.ContainerID, mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader, op string) error { if vh == nil { return errMissingVerificationHeader } @@ -287,7 +300,7 @@ func (ac *apeChecker) validateContainerBoundedOperation(containerID *refs.Contai return err } - reqProps, pk, err := ac.getRequestProps(mh, vh, cont, id) + reqProps, pk, err := ac.getRequestProps(ctx, mh, vh, cont, id) if err != nil { return err } @@ -358,7 +371,7 @@ func (ac *apeChecker) getContainerProps(c *containercore.Container) map[string]s } } -func (ac *apeChecker) getRequestProps(mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader, +func (ac *apeChecker) getRequestProps(ctx context.Context, mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader, cont *containercore.Container, cnrID cid.ID, ) (map[string]string, *keys.PublicKey, error) { actor, pk, err := ac.getActorAndPublicKey(mh, vh, cnrID) @@ -377,6 +390,11 @@ func (ac *apeChecker) getRequestProps(mh *session.RequestMetaHeader, vh *session if err != nil { return nil, nil, err } + if p, ok := peer.FromContext(ctx); ok { + if tcpAddr, ok := p.Addr.(*net.TCPAddr); ok { + reqProps[commonschema.PropertyKeyFrostFSSourceIP] = tcpAddr.IP.String() + } + } return reqProps, pk, nil } diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index 01a0fa4d2..90c0225dd 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "net" "testing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" @@ -32,6 +33,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/stretchr/testify/require" + "google.golang.org/grpc/peer" ) const ( @@ -45,6 +47,7 @@ func TestAPE(t *testing.T) { t.Run("deny get container no rule found", testDenyGetContainerNoRuleFound) t.Run("deny get container for others", testDenyGetContainerForOthers) t.Run("deny get container by user claim tag", testDenyGetContainerByUserClaimTag) + t.Run("deny get container by IP", testDenyGetContainerByIP) t.Run("deny get container by group id", testDenyGetContainerByGroupID) t.Run("deny set container eACL for IR", testDenySetContainerEACLForIR) t.Run("deny get container eACL for IR with session token", testDenyGetContainerEACLForIRSessionToken) @@ -55,6 +58,19 @@ func TestAPE(t *testing.T) { t.Run("deny list containers by namespace invalidation", testDenyListContainersValidationNamespaceError) } +const ( + incomingIP = "192.92.33.1" +) + +func ctxWithPeerInfo() context.Context { + return peer.NewContext(context.Background(), &peer.Peer{ + Addr: &net.TCPAddr{ + IP: net.ParseIP(incomingIP), + Port: 41111, + }, + }) +} + func testAllowThenDenyGetContainerRuleDefined(t *testing.T) { t.Parallel() srv := &srvStub{ @@ -353,6 +369,105 @@ func testDenyGetContainerByUserClaimTag(t *testing.T) { require.ErrorAs(t, err, &errAccessDenied) } +func testDenyGetContainerByIP(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{ + pk.PublicKey().GetScriptHash(): { + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + }, + }, + subjectsExt: map[util.Uint160]*client.SubjectExtended{ + pk.PublicKey().GetScriptHash(): { + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + Groups: []*client.Group{ + { + ID: 19888, + }, + }, + }, + }, + } + + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) + + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + contRdr.c[contID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(nm.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + nm.netmaps[nm.currentEpoch] = &testNetmap + nm.netmaps[nm.currentEpoch-1] = &testNetmap + + _, _, err = router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Kind: chain.KindRequest, + Key: commonschema.PropertyKeyFrostFSSourceIP, + Value: incomingIP + "/16", + Op: chain.CondIPAddress, + }, + }, + }, + }, + }) + require.NoError(t, err) + + req := &container.GetRequest{} + req.SetBody(&container.GetRequestBody{}) + var refContID refs.ContainerID + contID.WriteToV2(&refContID) + req.GetBody().SetContainerID(&refContID) + + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + resp, err := apeSrv.Get(ctxWithPeerInfo(), req) + require.Nil(t, resp) + var errAccessDenied *apistatus.ObjectAccessDenied + require.ErrorAs(t, err, &errAccessDenied) + require.Contains(t, errAccessDenied.Reason(), chain.AccessDenied.String()) +} + func testDenyGetContainerByGroupID(t *testing.T) { t.Parallel() srv := &srvStub{ @@ -1208,7 +1323,7 @@ func TestValidateContainerBoundedOperation(t *testing.T) { req := initTestGetContainerRequest(t, contID) - err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + err = components.apeChecker.validateContainerBoundedOperation(ctxWithPeerInfo(), req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) aErr := apeErr(nativeschema.MethodGetContainer, chain.AccessDenied) require.ErrorContains(t, err, aErr.Error()) }) @@ -1252,7 +1367,7 @@ func TestValidateContainerBoundedOperation(t *testing.T) { req := initTestGetContainerRequest(t, contID) - err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + err = components.apeChecker.validateContainerBoundedOperation(ctxWithPeerInfo(), req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) require.NoError(t, err) }) @@ -1295,7 +1410,7 @@ func TestValidateContainerBoundedOperation(t *testing.T) { req := initTestGetContainerRequest(t, contID) - err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + err = components.apeChecker.validateContainerBoundedOperation(ctxWithPeerInfo(), req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) require.NoError(t, err) }) @@ -1338,7 +1453,7 @@ func TestValidateContainerBoundedOperation(t *testing.T) { req := initTestGetContainerRequest(t, contID) - err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + err = components.apeChecker.validateContainerBoundedOperation(ctxWithPeerInfo(), req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) require.NoError(t, err) }) @@ -1381,7 +1496,7 @@ func TestValidateContainerBoundedOperation(t *testing.T) { req := initTestGetContainerRequest(t, contID) - err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + err = components.apeChecker.validateContainerBoundedOperation(ctxWithPeerInfo(), req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) aErr := apeErr(nativeschema.MethodGetContainer, chain.AccessDenied) require.ErrorContains(t, err, aErr.Error()) }) @@ -1423,7 +1538,7 @@ func TestValidateContainerBoundedOperation(t *testing.T) { req := initTestGetContainerRequest(t, contID) - err = components.apeChecker.validateContainerBoundedOperation(req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) + err = components.apeChecker.validateContainerBoundedOperation(ctxWithPeerInfo(), req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), nativeschema.MethodGetContainer) aErr := apeErr(nativeschema.MethodGetContainer, chain.AccessDenied) require.ErrorContains(t, err, aErr.Error()) }) From 482c5129ac37f1b816afbadfed4c231c3a53a24f Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 20 May 2024 16:19:00 +0300 Subject: [PATCH 1113/1943] [#1142] object: Fill APE-request with source IP property Signed-off-by: Airat Arifullin --- pkg/services/object/ape/request.go | 9 +++++++++ pkg/services/object/ape/request_test.go | 16 +++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index 71a9aec2c..02ead8c60 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "fmt" + "net" "strconv" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" @@ -15,8 +16,10 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "google.golang.org/grpc/peer" ) var defaultRequest = aperequest.Request{} @@ -129,6 +132,12 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re return defaultRequest, err } + if p, ok := peer.FromContext(ctx); ok { + if tcpAddr, ok := p.Addr.(*net.TCPAddr); ok { + reqProps[commonschema.PropertyKeyFrostFSSourceIP] = tcpAddr.IP.String() + } + } + return aperequest.NewRequest( prm.Method, aperequest.NewResource( diff --git a/pkg/services/object/ape/request_test.go b/pkg/services/object/ape/request_test.go index 5c74e7d1d..9dad69d17 100644 --- a/pkg/services/object/ape/request_test.go +++ b/pkg/services/object/ape/request_test.go @@ -3,6 +3,7 @@ package ape import ( "context" "fmt" + "net" "testing" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" @@ -14,12 +15,24 @@ import ( commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/stretchr/testify/require" + "google.golang.org/grpc/peer" ) const ( testOwnerID = "FPPtmAi9TCX329" + + incomingIP = "192.92.33.1" ) +func ctxWithPeerInfo() context.Context { + return peer.NewContext(context.Background(), &peer.Peer{ + Addr: &net.TCPAddr{ + IP: net.ParseIP(incomingIP), + Port: 41111, + }, + }) +} + func TestObjectProperties(t *testing.T) { for _, test := range []struct { name string @@ -253,7 +266,7 @@ func TestNewAPERequest(t *testing.T) { frostFSIDClient: ffidProvider, } - r, err := c.newAPERequest(context.TODO(), prm) + r, err := c.newAPERequest(ctxWithPeerInfo(), prm) if test.expectErr != nil { require.Error(t, err) require.ErrorIs(t, err, test.expectErr) @@ -276,6 +289,7 @@ func TestNewAPERequest(t *testing.T) { fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, "tag-attr1"): "value1", fmt.Sprintf(commonschema.PropertyKeyFormatFrostFSIDUserClaim, "tag-attr2"): "value2", commonschema.PropertyKeyFrostFSIDGroupID: "1", + commonschema.PropertyKeyFrostFSSourceIP: incomingIP, }, ) From 3627b44e92395d2be7eeda9790513021b9f345ca Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 20 May 2024 16:34:31 +0300 Subject: [PATCH 1114/1943] [#1142] tree: Fill APE-request with source IP property Signed-off-by: Airat Arifullin --- pkg/services/tree/ape.go | 11 ++++++++- pkg/services/tree/service.go | 12 +++++----- pkg/services/tree/signature.go | 5 +++-- pkg/services/tree/signature_test.go | 35 +++++++++++++++-------------- 4 files changed, 37 insertions(+), 26 deletions(-) diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 622049d6a..eabc02bd7 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -1,8 +1,10 @@ package tree import ( + "context" "encoding/hex" "fmt" + "net" "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/converter" @@ -14,11 +16,13 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "google.golang.org/grpc/peer" ) -func (s *Service) checkAPE(container *core.Container, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey) error { +func (s *Service) checkAPE(ctx context.Context, container *core.Container, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey) error { namespace := "" cntNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(container.Value).Zone(), ".ns") if hasNamespace { @@ -41,6 +45,11 @@ func (s *Service) checkAPE(container *core.Container, cid cid.ID, operation acl. if err != nil { return apeErr(err) } + if p, ok := peer.FromContext(ctx); ok { + if tcpAddr, ok := p.Addr.(*net.TCPAddr); ok { + reqProps[commonschema.PropertyKeyFrostFSSourceIP] = tcpAddr.IP.String() + } + } var resourceName string if namespace == "root" || namespace == "" { diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 903db4455..cce41e94d 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -105,7 +105,7 @@ func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error return nil, err } - err := s.verifyClient(req, cid, b.GetBearerToken(), acl.OpObjectPut) + err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectPut) if err != nil { return nil, err } @@ -157,7 +157,7 @@ func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByP return nil, err } - err := s.verifyClient(req, cid, b.GetBearerToken(), acl.OpObjectPut) + err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectPut) if err != nil { return nil, err } @@ -221,7 +221,7 @@ func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveRespon return nil, err } - err := s.verifyClient(req, cid, b.GetBearerToken(), acl.OpObjectPut) + err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectPut) if err != nil { return nil, err } @@ -274,7 +274,7 @@ func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, er return nil, err } - err := s.verifyClient(req, cid, b.GetBearerToken(), acl.OpObjectPut) + err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectPut) if err != nil { return nil, err } @@ -326,7 +326,7 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) return nil, err } - err := s.verifyClient(req, cid, b.GetBearerToken(), acl.OpObjectGet) + err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectGet) if err != nil { return nil, err } @@ -408,7 +408,7 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS return err } - err := s.verifyClient(req, cid, b.GetBearerToken(), acl.OpObjectGet) + err := s.verifyClient(srv.Context(), req, cid, b.GetBearerToken(), acl.OpObjectGet) if err != nil { return err } diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 162b189e3..0445ccfab 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -2,6 +2,7 @@ package tree import ( "bytes" + "context" "crypto/ecdsa" "crypto/elliptic" "errors" @@ -48,7 +49,7 @@ var ( // Operation must be one of: // - 1. ObjectPut; // - 2. ObjectGet. -func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op acl.Op) error { +func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, rawBearer []byte, op acl.Op) error { err := verifyMessage(req) if err != nil { return err @@ -83,7 +84,7 @@ func (s *Service) verifyClient(req message, cid cidSDK.ID, rawBearer []byte, op // FIXME(@aarifullin): tree service temporiraly performs APE checks on // object verbs, because tree verbs have not been introduced yet. if basicACL == 0x0 { - return s.checkAPE(cnr, cid, op, role, pubKey) + return s.checkAPE(ctx, cnr, cid, op, role, pubKey) } if !basicACL.IsOpAllowed(op, role) { diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index b55f4d4fc..ce5039f7c 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -1,6 +1,7 @@ package tree import ( + "context" "crypto/ecdsa" "crypto/sha256" "errors" @@ -132,26 +133,26 @@ func TestMessageSign(t *testing.T) { cnr.Value.SetBasicACL(acl.PublicRW) t.Run("missing signature, no panic", func(t *testing.T) { - require.Error(t, s.verifyClient(req, cid2, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid2, nil, op)) }) require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.NoError(t, s.verifyClient(req, cid1, nil, op)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, nil, op)) t.Run("invalid CID", func(t *testing.T) { - require.Error(t, s.verifyClient(req, cid2, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid2, nil, op)) }) cnr.Value.SetBasicACL(acl.Private) t.Run("extension disabled", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.Error(t, s.verifyClient(req, cid2, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid2, nil, op)) }) t.Run("invalid key", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(req, cid1, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, nil, op)) }) t.Run("bearer", func(t *testing.T) { @@ -164,7 +165,7 @@ func TestMessageSign(t *testing.T) { t.Run("invalid bearer", func(t *testing.T) { req.Body.BearerToken = []byte{0xFF} require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.Error(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("invalid bearer CID", func(t *testing.T) { @@ -173,7 +174,7 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bt.Marshal() require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("invalid bearer owner", func(t *testing.T) { bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) @@ -181,7 +182,7 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bt.Marshal() require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("invalid bearer signature", func(t *testing.T) { bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) @@ -193,7 +194,7 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bv2.StableMarshal(nil) require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("impersonate", func(t *testing.T) { @@ -205,8 +206,8 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bt.Marshal() require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.Error(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.NoError(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) @@ -216,18 +217,18 @@ func TestMessageSign(t *testing.T) { t.Run("put and get", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.NoError(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.NoError(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) t.Run("only get", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[2].PrivateKey)) - require.Error(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.NoError(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) t.Run("none", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[3].PrivateKey)) - require.Error(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.Error(t, s.verifyClient(req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) }) } From cc3f762cf29935bd06ee1c9c00ea1790703ba10d Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 28 May 2024 15:08:28 +0300 Subject: [PATCH 1115/1943] [#12] cli: clarify `-g` usage in `container list` Add usage replacement for `container list -g` and verbose warning when using `-g` without `--owner`. Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/internal/commonflags/flags.go | 8 ++++---- cmd/frostfs-cli/modules/container/list.go | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-cli/internal/commonflags/flags.go b/cmd/frostfs-cli/internal/commonflags/flags.go index 5049dc3b1..6915ef42b 100644 --- a/cmd/frostfs-cli/internal/commonflags/flags.go +++ b/cmd/frostfs-cli/internal/commonflags/flags.go @@ -11,9 +11,9 @@ import ( // values and their usage descriptions. const ( GenerateKey = "generate-key" - generateKeyShorthand = "g" - generateKeyDefault = false - generateKeyUsage = "Generate new private key" + GenerateKeyShorthand = "g" + GenerateKeyDefault = false + GenerateKeyUsage = "Generate new private key" WalletPath = "wallet" WalletPathShorthand = "w" @@ -72,7 +72,7 @@ func Init(cmd *cobra.Command) { func InitWithoutRPC(cmd *cobra.Command) { ff := cmd.Flags() - ff.BoolP(GenerateKey, generateKeyShorthand, generateKeyDefault, generateKeyUsage) + ff.BoolP(GenerateKey, GenerateKeyShorthand, GenerateKeyDefault, GenerateKeyUsage) ff.StringP(WalletPath, WalletPathShorthand, WalletPathDefault, WalletPathUsage) ff.StringP(Account, AccountShorthand, AccountDefault, AccountUsage) } diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 8f5db49df..a1410d7a0 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -18,6 +18,8 @@ const ( flagListPrintAttr = "with-attr" flagListContainerOwner = "owner" flagListName = "name" + + generateKeyContainerUsage = commonflags.GenerateKeyUsage + ", should be used with --owner flag" ) // flag vars of list command. @@ -34,6 +36,11 @@ var listContainersCmd = &cobra.Command{ Run: func(cmd *cobra.Command, _ []string) { var idUser user.ID + generateKey, _ := cmd.Flags().GetBool(commonflags.GenerateKey) + if flagVarListContainerOwner == "" && generateKey { + cmd.PrintErrln("WARN: using -g without --owner - output will be empty") + } + key := key.GetOrGenerate(cmd) if flagVarListContainerOwner == "" { @@ -104,4 +111,5 @@ func initContainerListContainersCmd() { flags.BoolVar(&flagVarListPrintAttr, flagListPrintAttr, false, "Request and print attributes of each container", ) + flags.Lookup(commonflags.GenerateKey).Usage = generateKeyContainerUsage } From 4a34d0d40e6dcedd8da9a64ac6d9a53613498697 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Wed, 29 May 2024 11:01:01 +0300 Subject: [PATCH 1116/1943] [#1149] go.mod: Bump neo-go up to v0.106.0 Required to work with neo-go v0.106.0 node with default hardfork configuration. Without neo-go client version bump, it throws error. failed to get network magic: unexpected hardfork: Cockatrice Signed-off-by: Alex Vanin --- .../modules/morph/helper/local_client.go | 2 +- .../internal/modules/morph/helper/n3client.go | 2 +- go.mod | 19 ++++---- go.sum | 47 +++++++++---------- pkg/ape/chainbase/boltdb.go | 8 ++-- 5 files changed, 37 insertions(+), 41 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go index 31203dde6..44d1b4ecf 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go @@ -113,7 +113,7 @@ func (l *LocalClient) GetBlockCount() (uint32, error) { return l.bc.BlockHeight(), nil } -func (l *LocalClient) GetNativeContracts() ([]state.NativeContract, error) { +func (l *LocalClient) GetNativeContracts() ([]state.Contract, error) { return l.bc.GetNatives(), nil } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go index 84dd69c79..55449b435 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go @@ -27,7 +27,7 @@ type Client interface { invoker.RPCInvoke GetBlockCount() (uint32, error) - GetNativeContracts() ([]state.NativeContract, error) + GetNativeContracts() ([]state.Contract, error) GetApplicationLog(util.Uint256, *trigger.Type) (*result.ApplicationLog, error) GetVersion() (*result.Version, error) SendRawTransaction(*transaction.Transaction) (util.Uint256, error) diff --git a/go.mod b/go.mod index 81e2fa989..1335747aa 100644 --- a/go.mod +++ b/go.mod @@ -24,21 +24,21 @@ require ( github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.12.1 github.com/nats-io/nats.go v1.32.0 - github.com/nspcc-dev/neo-go v0.105.1 + github.com/nspcc-dev/neo-go v0.106.0 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 github.com/paulmach/orb v0.11.0 - github.com/prometheus/client_golang v1.18.0 + github.com/prometheus/client_golang v1.19.0 github.com/spf13/cast v1.6.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 github.com/ssgreg/journald v1.0.0 - github.com/stretchr/testify v1.8.4 - go.etcd.io/bbolt v1.3.8 + github.com/stretchr/testify v1.9.0 + go.etcd.io/bbolt v1.3.9 go.opentelemetry.io/otel v1.22.0 go.opentelemetry.io/otel/trace v1.22.0 - go.uber.org/zap v1.26.0 + go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/sync v0.6.0 golang.org/x/sys v0.18.0 @@ -78,7 +78,6 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -99,13 +98,13 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect - github.com/nspcc-dev/go-ordered-json v0.0.0-20240112074137-296698a162ae // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240122090917-ef99a7a9e33f // indirect - github.com/nspcc-dev/rfc6979 v0.2.0 // indirect + github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect + github.com/nspcc-dev/rfc6979 v0.2.1 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.46.0 // indirect + github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index 5dd9b2423..ce68ddb22 100644 --- a/go.sum +++ b/go.sum @@ -113,8 +113,6 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= -github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -192,14 +190,14 @@ github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nspcc-dev/go-ordered-json v0.0.0-20240112074137-296698a162ae h1:UFgMXcZthqiCqCyr3dOAtGICJ10gM8q0mFHyLR0UPQU= -github.com/nspcc-dev/go-ordered-json v0.0.0-20240112074137-296698a162ae/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U= -github.com/nspcc-dev/neo-go v0.105.1 h1:r0b2yIwLBi+ARBKU94gHL9oTFEB/XMJ0YlS2HN9Qw34= -github.com/nspcc-dev/neo-go v0.105.1/go.mod h1:GNh0cRALV/cuj+/xg2ZHDsrFbqcInqG7jjhqsLEnlNc= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240122090917-ef99a7a9e33f h1:6h0zgQGKdStROXRlWhz2dZmrKkSmOJ4oiDs06nO39XI= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240122090917-ef99a7a9e33f/go.mod h1:J/Mk6+nKeKSW4wygkZQFLQ6SkLOSGX5Ga0RuuuktEag= -github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= -github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= +github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk= +github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc= +github.com/nspcc-dev/neo-go v0.106.0 h1:YiOdW/GcLmbVSvxMRfD5aytO6n3TDHrEA97VHMawy6g= +github.com/nspcc-dev/neo-go v0.106.0/go.mod h1:9k7vBqqQeePuj4c2CqMN9uPFsfxvwSk9vnQFrkE5eg8= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d h1:Vcb7YkZuUSSIC+WF/xV3UDfHbAxZgyT2zGleJP3Ig5k= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d/go.mod h1:/vrbWSHc7YS1KSYhVOyyeucXW/e+1DkVBOgnBEXUCeY= +github.com/nspcc-dev/rfc6979 v0.2.1 h1:8wWxkamHWFmO790GsewSoKUSJjVnL1fmdRpokU/RgRM= +github.com/nspcc-dev/rfc6979 v0.2.1/go.mod h1:Tk7h5kyUWkhjyO3zUgFFhy1v2vQv3BvQEntakdtqrWc= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -227,12 +225,12 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= -github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -272,8 +270,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= @@ -283,8 +282,6 @@ github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= -github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= -github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= @@ -294,8 +291,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= -go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= +go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk= go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= @@ -319,8 +316,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -336,8 +333,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -413,8 +410,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/ape/chainbase/boltdb.go b/pkg/ape/chainbase/boltdb.go index 5f3c342c1..005b3bd84 100644 --- a/pkg/ape/chainbase/boltdb.go +++ b/pkg/ape/chainbase/boltdb.go @@ -1,6 +1,7 @@ package chainbase import ( + "bytes" "context" "errors" "fmt" @@ -11,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" - "github.com/nspcc-dev/neo-go/pkg/util/slice" "go.etcd.io/bbolt" ) @@ -203,7 +203,7 @@ func (cs *boltLocalOverrideStorage) GetOverride(name chain.Name, target policyen if serializedChain == nil { return fmt.Errorf("%w: %w: %s", policyengine.ErrChainNotFound, ErrBucketNotContainsChainID, chainID) } - serializedChain = slice.Copy(serializedChain) + serializedChain = bytes.Clone(serializedChain) return nil }); err != nil { return nil, err @@ -246,7 +246,7 @@ func (cs *boltLocalOverrideStorage) ListOverrides(name chain.Name, target policy return err } return rbuck.ForEach(func(_, v []byte) error { - serializedChain = slice.Copy(v) + serializedChain = bytes.Clone(v) serializedChains = append(serializedChains, serializedChain) return nil }) @@ -315,7 +315,7 @@ func getTargets(tx *bbolt.Tx, name chain.Name) ([]policyengine.Target, error) { if err := nbucket.Bucket(k).ForEachBucket(func(k []byte) error { targets = append(targets, policyengine.Target{ Type: ttype, - Name: string(slice.Copy(k)), + Name: string(bytes.Clone(k)), }) return nil }); err != nil { From 50923ed81c085fa6556ac58c4dd7f7d6cc06d76c Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 27 May 2024 15:57:02 +0300 Subject: [PATCH 1117/1943] [#1147] Fix gofumpt issue Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/ape_manager/add_chain.go | 4 +--- pkg/services/apemanager/executor.go | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-cli/modules/ape_manager/add_chain.go b/cmd/frostfs-cli/modules/ape_manager/add_chain.go index 7e6614392..dea7fb36e 100644 --- a/cmd/frostfs-cli/modules/ape_manager/add_chain.go +++ b/cmd/frostfs-cli/modules/ape_manager/add_chain.go @@ -38,9 +38,7 @@ const ( groupTarget = "group" ) -var ( - errUnknownTargetType = errors.New("unknown target type") -) +var errUnknownTargetType = errors.New("unknown target type") var addCmd = &cobra.Command{ Use: "add", diff --git a/pkg/services/apemanager/executor.go b/pkg/services/apemanager/executor.go index bad43b5fa..9cfc97473 100644 --- a/pkg/services/apemanager/executor.go +++ b/pkg/services/apemanager/executor.go @@ -24,9 +24,7 @@ import ( "go.uber.org/zap" ) -var ( - errEmptyBodySignature = errors.New("malformed request: empty body signature") -) +var errEmptyBodySignature = errors.New("malformed request: empty body signature") type cfg struct { log *logger.Logger From e43e7bec3a9c8ea39e75e949c40a3996138b51d0 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 27 May 2024 15:57:48 +0300 Subject: [PATCH 1118/1943] [#1147] log: Remove redundant `address` field from log Filled when logger created for `request` object from package `getsvc`. Signed-off-by: Anton Nikiforov --- pkg/services/object/get/assemble.go | 3 --- pkg/services/object/get/assembleec.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index 3f4a02c02..66c4580b0 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -41,12 +41,10 @@ func (r *request) assemble(ctx context.Context) { assembler := newAssembler(r.address(), r.splitInfo(), r.ctxRange(), r) r.log.Debug(logs.GetAssemblingSplittedObject, - zap.Stringer("address", r.address()), zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), ) defer r.log.Debug(logs.GetAssemblingSplittedObjectCompleted, - zap.Stringer("address", r.address()), zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), ) @@ -55,7 +53,6 @@ func (r *request) assemble(ctx context.Context) { if err != nil { r.log.Warn(logs.GetFailedToAssembleSplittedObject, zap.Error(err), - zap.Stringer("address", r.address()), zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), ) diff --git a/pkg/services/object/get/assembleec.go b/pkg/services/object/get/assembleec.go index 58641c975..5c999929a 100644 --- a/pkg/services/object/get/assembleec.go +++ b/pkg/services/object/get/assembleec.go @@ -39,12 +39,10 @@ func (r *request) assembleEC(ctx context.Context) { assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.containerSource, r.log) r.log.Debug(logs.GetAssemblingECObject, - zap.Stringer("address", r.address()), zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), ) defer r.log.Debug(logs.GetAssemblingECObjectCompleted, - zap.Stringer("address", r.address()), zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), ) @@ -53,7 +51,6 @@ func (r *request) assembleEC(ctx context.Context) { if err != nil { r.log.Warn(logs.GetFailedToAssembleECObject, zap.Error(err), - zap.Stringer("address", r.address()), zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), ) From 3555c73225dfe87e0a7eb8a8a801f8880f0c23c3 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 27 May 2024 16:03:22 +0300 Subject: [PATCH 1119/1943] [#1147] object: Use methods on pointer for `deletesvc.execCtx` Signed-off-by: Anton Nikiforov --- pkg/services/object/delete/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index d48f8ab7b..d119328a6 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -40,7 +40,7 @@ func (exec *execCtx) setLogger(l *logger.Logger) { )} } -func (exec execCtx) isLocal() bool { +func (exec *execCtx) isLocal() bool { return exec.prm.common.LocalOnly() } From d355274cd0f708022382ddda78ccdfdb85d2854f Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 27 May 2024 16:05:20 +0300 Subject: [PATCH 1120/1943] [#1147] object: Use methods on pointer for `searchsvc.execCtx` Signed-off-by: Anton Nikiforov --- pkg/services/object/search/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index 2e0d48773..4a2c04ecd 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -28,7 +28,7 @@ func (exec *execCtx) setLogger(l *logger.Logger) { )} } -func (exec execCtx) isLocal() bool { +func (exec *execCtx) isLocal() bool { return exec.prm.common.LocalOnly() } From a82c8cc5b87c3511e75aec7f52289bea8175ad90 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 27 May 2024 16:11:57 +0300 Subject: [PATCH 1121/1943] [#1147] gc: Execute callback for expired tombstones when they exists Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/shard/gc.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index ef8e97d34..d605746e8 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -525,7 +525,9 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { } log.Debug(logs.ShardHandlingExpiredTombstonesBatch, zap.Int("number", len(tssExp))) - s.expiredTombstonesCallback(ctx, tssExp) + if len(tssExp) > 0 { + s.expiredTombstonesCallback(ctx, tssExp) + } iterPrm.SetOffset(tss[tssLen-1].Address()) tss = tss[:0] From 88b8ddd9023670893bb8f0950caf80de6f541f39 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 27 May 2024 21:31:59 +0300 Subject: [PATCH 1122/1943] [#1147] cli: Fix output when print EC info with flags `json` & `proto` Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/object/range.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index 9ba752237..ad4bc3d59 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -152,7 +152,11 @@ func printECInfoErr(cmd *cobra.Command, err error) bool { ok := errors.As(err, &errECInfo) if ok { - cmd.PrintErrln("Object is erasure-encoded, ec information received.") + toJSON, _ := cmd.Flags().GetBool(commonflags.JSON) + toProto, _ := cmd.Flags().GetBool("proto") + if !(toJSON || toProto) { + cmd.PrintErrln("Object is erasure-encoded, ec information received.") + } printECInfo(cmd, errECInfo.ECInfo()) } From 6130650bb6e7501f22b176b90eb3e5274b009f6e Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 27 May 2024 22:07:43 +0300 Subject: [PATCH 1123/1943] [#1147] node: Implement `Lock\Delete` requests for EC object Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/object/util.go | 28 +++++---- internal/logs/logs.go | 2 + pkg/local_object_storage/engine/delete.go | 38 +++++++++++- .../engine/engine_test.go | 3 +- pkg/local_object_storage/engine/exists.go | 11 +++- pkg/local_object_storage/engine/inhume.go | 30 +++++++++- pkg/local_object_storage/engine/lock.go | 21 ++++++- pkg/local_object_storage/engine/put.go | 20 ++++++- pkg/local_object_storage/metabase/delete.go | 44 +++++++++++++- pkg/local_object_storage/metabase/exists.go | 38 ++++++++---- pkg/local_object_storage/metabase/inhume.go | 43 ++++++++++++++ pkg/local_object_storage/metabase/lock.go | 58 +++++++++++++++++++ pkg/local_object_storage/metabase/put.go | 2 +- pkg/local_object_storage/metabase/select.go | 2 +- pkg/local_object_storage/shard/exists.go | 18 +++++- pkg/local_object_storage/shard/lock.go | 17 ++++++ pkg/services/object/delete/exec.go | 30 +++++++++- pkg/services/object/delete/local.go | 14 +---- pkg/services/object/delete/service.go | 2 +- pkg/services/object/delete/util.go | 16 +---- 20 files changed, 371 insertions(+), 66 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index c8625eb94..ff6b3219d 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -357,6 +357,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, _, err := internal.HeadObject(cmd.Context(), prmHead) var errSplit *objectSDK.SplitInfoError + var errEC *objectSDK.ECInfoError switch { default: @@ -366,19 +367,22 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, return nil case errors.As(err, &errSplit): common.PrintVerbose(cmd, "Split information received - object is virtual.") + splitInfo := errSplit.SplitInfo() + + if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnr, true); ok { + return members + } + + if members, ok := tryGetSplitMembersBySplitID(cmd, splitInfo, cli, cnr); ok { + return members + } + + return tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnr, obj) + case errors.As(err, &errEC): + common.PrintVerbose(cmd, "Object is erasure-coded.") + return nil } - - splitInfo := errSplit.SplitInfo() - - if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnr, true); ok { - return members - } - - if members, ok := tryGetSplitMembersBySplitID(cmd, splitInfo, cli, cnr); ok { - return members - } - - return tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnr, obj) + return nil } func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, prmHead internal.HeadObjectPrm, cnr cid.ID, withLinking bool) ([]oid.ID, bool) { diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 43fd77624..7dc63341d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -95,6 +95,7 @@ const ( DeleteFormingSplitInfo = "forming split info..." DeleteSplitInfoSuccessfullyFormedCollectingMembers = "split info successfully formed, collecting members..." DeleteMembersSuccessfullyCollected = "members successfully collected" + DeleteECObjectReceived = "erasure-coded object received, form tombstone" GetRemoteCallFailed = "remote call failed" GetCanNotAssembleTheObject = "can not assemble the object" GetTryingToAssembleTheObject = "trying to assemble the object..." @@ -213,6 +214,7 @@ const ( EngineFinishedRemovalOfLocallyredundantCopies = "finished removal of locally-redundant copies" EngineRemovingAnObjectWithoutFullLockingCheck = "removing an object without full locking check" EngineInterruptProcessingTheExpiredLocks = "interrupt processing the expired locks" + EngineInterruptGettingLockers = "can't get object's lockers" EngineInterruptProcessingTheDeletedLocks = "interrupt processing the deleted locks" EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly = "failed to move shard in degraded-read-only mode, moving to read-only" EngineFailedToMoveShardInReadonlyMode = "failed to move shard in read-only mode" diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 68a7325c6..44a612213 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -76,6 +76,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e is bool } var splitInfo *objectSDK.SplitInfo + var ecInfo *objectSDK.ECInfo // Removal of a big object is done in multiple stages: // 1. Remove the parent object. If it is locked or already removed, return immediately. @@ -91,13 +92,18 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e } var splitErr *objectSDK.SplitInfoError - if !errors.As(err, &splitErr) { + var ecErr *objectSDK.ECInfoError + if errors.As(err, &splitErr) { + splitInfo = splitErr.SplitInfo() + } else if errors.As(err, &ecErr) { + e.deleteChunks(ctx, sh, ecInfo, prm) + return false + } else { if !client.IsErrObjectNotFound(err) { e.reportShardError(sh, "could not check object existence", err) } return false } - splitInfo = splitErr.SplitInfo() } else if !resExists.Exists() { return false } @@ -171,3 +177,31 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo return false }) } + +func (e *StorageEngine) deleteChunks( + ctx context.Context, sh hashedShard, ecInfo *objectSDK.ECInfo, prm DeletePrm, +) { + var inhumePrm shard.InhumePrm + if prm.forceRemoval { + inhumePrm.ForceRemoval() + } + for _, chunk := range ecInfo.Chunks { + var addr oid.Address + addr.SetContainer(prm.addr.Container()) + var objID oid.ID + err := objID.ReadFromV2(chunk.ID) + if err != nil { + e.reportShardError(sh, "could not delete EC chunk", err) + } + addr.SetObject(objID) + inhumePrm.MarkAsGarbage(addr) + _, err = sh.Inhume(ctx, inhumePrm) + if err != nil { + e.log.Debug(logs.EngineCouldNotInhumeObjectInShard, + zap.Stringer("addr", addr), + zap.String("err", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + continue + } + } +} diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index b82700ed8..70c54590f 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -17,6 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "git.frostfs.info/TrueCloudLab/hrw" "github.com/panjf2000/ants/v2" @@ -62,7 +63,7 @@ func benchmarkExists(b *testing.B, shardNum int) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - ok, err := e.exists(context.Background(), addr) + ok, _, err := e.exists(context.Background(), addr, oid.Address{}) if err != nil || ok { b.Fatalf("%t %v", ok, err) } diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index ef6292768..ce669ec5e 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -11,11 +11,13 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -func (e *StorageEngine) exists(ctx context.Context, addr oid.Address) (bool, error) { +func (e *StorageEngine) exists(ctx context.Context, addr oid.Address, parentAddr oid.Address) (bool, bool, error) { var shPrm shard.ExistsPrm shPrm.SetAddress(addr) + shPrm.SetParentAddress(parentAddr) alreadyRemoved := false exists := false + locked := false e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { res, err := sh.Exists(ctx, shPrm) @@ -44,13 +46,16 @@ func (e *StorageEngine) exists(ctx context.Context, addr oid.Address) (bool, err if !exists { exists = res.Exists() } + if !locked { + locked = res.Locked() + } return false }) if alreadyRemoved { - return false, new(apistatus.ObjectAlreadyRemoved) + return false, false, new(apistatus.ObjectAlreadyRemoved) } - return exists, nil + return exists, locked, nil } diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 80eda2204..62e7be933 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -152,7 +152,8 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh } var siErr *objectSDK.SplitInfoError - if !errors.As(err, &siErr) { + var ecErr *objectSDK.ECInfoError + if !(errors.As(err, &siErr) || errors.As(err, &ecErr)) { e.reportShardError(sh, "could not check for presents in shard", err) return } @@ -220,6 +221,33 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e return locked, outErr } +// GetLocked return lock id's if object is locked according to StorageEngine's state. +func (e *StorageEngine) GetLocked(ctx context.Context, addr oid.Address) ([]oid.ID, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.GetLocked", + trace.WithAttributes( + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + + var locked []oid.ID + var outErr error + + e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { + ld, err := h.Shard.GetLocked(ctx, addr) + if err != nil { + e.reportShardError(h, logs.EngineInterruptGettingLockers, err, zap.Stringer("addr", addr), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + outErr = err + } + locked = append(locked, ld...) + return false + }) + if len(locked) > 0 { + return locked, nil + } + return locked, outErr +} + func (e *StorageEngine) processExpiredTombstones(ctx context.Context, addrs []meta.TombstonedObject) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { sh.HandleExpiredTombstones(ctx, addrs) diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 5ad603421..af56c3e1e 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -83,7 +83,26 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo exRes, err := sh.Exists(ctx, existsPrm) if err != nil { var siErr *objectSDK.SplitInfoError - if !errors.As(err, &siErr) { + var eiErr *objectSDK.ECInfoError + if errors.As(err, &eiErr) { + eclocked := []oid.ID{locked} + for _, chunk := range eiErr.ECInfo().Chunks { + var objID oid.ID + err = objID.ReadFromV2(chunk.ID) + if err != nil { + e.reportShardError(sh, "could not lock object in shard", err) + return false + } + eclocked = append(eclocked, objID) + } + err = sh.Lock(ctx, idCnr, locker, eclocked) + if err != nil { + e.reportShardError(sh, "could not lock object in shard", err) + return false + } + root = true + return false + } else if !errors.As(err, &siErr) { if shard.IsErrObjectExpired(err) { // object is already expired => // do not lock it diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 777f728b7..aa94ef22e 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -80,11 +80,29 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { // In #1146 this check was parallelized, however, it became // much slower on fast machines for 4 shards. - _, err := e.exists(ctx, addr) + var parent oid.Address + if prm.obj.ECHeader() != nil { + parent.SetObject(prm.obj.ECHeader().Parent()) + parent.SetContainer(addr.Container()) + } + existed, locked, err := e.exists(ctx, addr, parent) if err != nil { return err } + if !existed && locked { + lockers, err := e.GetLocked(ctx, parent) + if err != nil { + return err + } + for _, locker := range lockers { + err = e.lock(ctx, addr.Container(), locker, []oid.ID{addr.Object()}) + if err != nil { + return err + } + } + } + var shRes putToShardRes e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { e.mtx.RLock() diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index e591b8661..4addeae77 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -267,8 +267,10 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter return deleteSingleResult{}, nil } var siErr *objectSDK.SplitInfoError - if errors.As(err, &siErr) { + var ecErr *objectSDK.ECInfoError + if errors.As(err, &siErr) || errors.As(err, &ecErr) { // if object is virtual (parent) then do nothing, it will be deleted with last child + // if object is erasure-coded it will be deleted with the last chunk presented on the shard return deleteSingleResult{}, nil } @@ -471,6 +473,46 @@ func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error name: rootBucketName(cnr, bucketName), key: objKey, }) + if obj.ECHeader() != nil { + err := delECInfo(tx, cnr, objKey, obj.ECHeader()) + if err != nil { + return err + } + } return nil } + +func delECInfo(tx *bbolt.Tx, cnr cid.ID, objKey []byte, ecHead *objectSDK.ECHeader) error { + parentID := objectKey(ecHead.Parent(), make([]byte, objectKeySize)) + bucketName := make([]byte, bucketKeySize) + + val := getFromBucket(tx, ecInfoBucketName(cnr, bucketName), parentID) + if len(val) > 0 { + if bytes.Equal(val, objKey) { + delUniqueIndexItem(tx, namedBucketItem{ + name: ecInfoBucketName(cnr, bucketName), + key: parentID, + }) + } else { + val = bytes.Clone(val) + offset := 0 + for offset < len(val) { + if bytes.Equal(objKey, val[offset:offset+objectKeySize]) { + val = append(val[:offset], val[offset+objectKeySize:]...) + break + } + offset += objectKeySize + } + err := putUniqueIndexItem(tx, namedBucketItem{ + name: ecInfoBucketName(cnr, make([]byte, bucketKeySize)), + key: parentID, + val: val, + }) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index bf6766c05..153d92110 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -20,12 +20,14 @@ import ( // ExistsPrm groups the parameters of Exists operation. type ExistsPrm struct { - addr oid.Address + addr oid.Address + paddr oid.Address } // ExistsRes groups the resulting values of Exists operation. type ExistsRes struct { exists bool + locked bool } var ErrLackSplitInfo = logicerr.New("no split info on parent object") @@ -35,11 +37,21 @@ func (p *ExistsPrm) SetAddress(addr oid.Address) { p.addr = addr } +// SetParent is an Exists option to set objects parent. +func (p *ExistsPrm) SetParent(addr oid.Address) { + p.paddr = addr +} + // Exists returns the fact that the object is in the metabase. func (p ExistsRes) Exists() bool { return p.exists } +// Locked returns the fact that the object is locked. +func (p ExistsRes) Locked() bool { + return p.locked +} + // Exists returns ErrAlreadyRemoved if addr was marked as removed. Otherwise it // returns true if addr is in primary index or false if it is not. // @@ -70,7 +82,7 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err currEpoch := db.epochState.CurrentEpoch() err = db.boltDB.View(func(tx *bbolt.Tx) error { - res.exists, err = db.exists(tx, prm.addr, currEpoch) + res.exists, res.locked, err = db.exists(tx, prm.addr, prm.paddr, currEpoch) return err }) @@ -78,15 +90,19 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err return res, metaerr.Wrap(err) } -func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (exists bool, err error) { +func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, parent oid.Address, currEpoch uint64) (bool, bool, error) { + var locked bool + if !parent.Equals(oid.Address{}) { + locked = objectLocked(tx, parent.Container(), parent.Object()) + } // check graveyard and object expiration first switch objectStatus(tx, addr, currEpoch) { case 1: - return false, logicerr.Wrap(new(apistatus.ObjectNotFound)) + return false, locked, logicerr.Wrap(new(apistatus.ObjectNotFound)) case 2: - return false, logicerr.Wrap(new(apistatus.ObjectAlreadyRemoved)) + return false, locked, logicerr.Wrap(new(apistatus.ObjectAlreadyRemoved)) case 3: - return false, ErrObjectIsExpired + return false, locked, ErrObjectIsExpired } objKey := objectKey(addr.Object(), make([]byte, objectKeySize)) @@ -96,25 +112,25 @@ func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (exists b // if graveyard is empty, then check if object exists in primary bucket if inBucket(tx, primaryBucketName(cnr, key), objKey) { - return true, nil + return true, locked, nil } // if primary bucket is empty, then check if object exists in parent bucket if inBucket(tx, parentBucketName(cnr, key), objKey) { splitInfo, err := getSplitInfo(tx, cnr, objKey) if err != nil { - return false, err + return false, locked, err } - return false, logicerr.Wrap(objectSDK.NewSplitInfoError(splitInfo)) + return false, locked, logicerr.Wrap(objectSDK.NewSplitInfoError(splitInfo)) } // if parent bucket is empty, then check if object exists in ec bucket if data := getFromBucket(tx, ecInfoBucketName(cnr, key), objKey); len(data) != 0 { - return false, getECInfoError(tx, cnr, data) + return false, locked, getECInfoError(tx, cnr, data) } // if parent bucket is empty, then check if object exists in typed buckets - return firstIrregularObjectType(tx, cnr, objKey) != objectSDK.TypeRegular, nil + return firstIrregularObjectType(tx, cnr, objKey) != objectSDK.TypeRegular, locked, nil } // objectStatus returns: diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 250504120..c265fb217 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -229,11 +229,17 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes obj, err := db.get(tx, prm.target[i], buf, false, true, epoch) targetKey := addressKey(prm.target[i], buf) + var ecErr *objectSDK.ECInfoError if err == nil { err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, targetKey, cnr, obj, res) if err != nil { return err } + } else if errors.As(err, &ecErr) { + err = db.inhumeECInfo(tx, epoch, prm.tomb, res, garbageBKT, graveyardBKT, ecErr.ECInfo(), cnr, bkt, value, targetKey) + if err != nil { + return err + } } if prm.tomb != nil { @@ -272,6 +278,43 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes return db.applyInhumeResToCounters(tx, res) } +func (db *DB) inhumeECInfo(tx *bbolt.Tx, epoch uint64, tomb *oid.Address, res *InhumeRes, + garbageBKT *bbolt.Bucket, graveyardBKT *bbolt.Bucket, + ecInfo *objectSDK.ECInfo, cnr cid.ID, targetBucket *bbolt.Bucket, value []byte, targetKey []byte, +) error { + for _, chunk := range ecInfo.Chunks { + chunkBuf := make([]byte, addressKeySize) + var chunkAddr oid.Address + chunkAddr.SetContainer(cnr) + var chunkID oid.ID + err := chunkID.ReadFromV2(chunk.ID) + if err != nil { + return err + } + chunkAddr.SetObject(chunkID) + chunkObj, err := db.get(tx, chunkAddr, chunkBuf, false, true, epoch) + if err != nil { + return err + } + err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, targetKey, cnr, chunkObj, res) + if err != nil { + return err + } + chunkKey := addressKey(chunkAddr, chunkBuf) + if tomb != nil { + _, err = db.markAsGC(graveyardBKT, garbageBKT, chunkKey) + if err != nil { + return err + } + } + err = targetBucket.Put(chunkKey, value) + if err != nil { + return err + } + } + return nil +} + func (db *DB) applyInhumeResToCounters(tx *bbolt.Tx, res *InhumeRes) error { if err := db.updateShardObjectCounter(tx, logical, res.LogicInhumed(), false); err != nil { return err diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 0bc2b06f0..732ba426d 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -175,6 +175,31 @@ func objectLocked(tx *bbolt.Tx, idCnr cid.ID, idObj oid.ID) bool { return false } +// return `LOCK` id's if specified object is locked in the specified container. +func getLocked(tx *bbolt.Tx, idCnr cid.ID, idObj oid.ID) ([]oid.ID, error) { + var lockers []oid.ID + bucketLocked := tx.Bucket(bucketNameLocked) + if bucketLocked != nil { + key := make([]byte, cidSize) + idCnr.Encode(key) + bucketLockedContainer := bucketLocked.Bucket(key) + if bucketLockedContainer != nil { + binObjIDs, err := decodeList(bucketLockedContainer.Get(objectKey(idObj, key))) + if err != nil { + return nil, fmt.Errorf("decode list of object lockers: %w", err) + } + for _, binObjID := range binObjIDs { + var id oid.ID + if err = id.Decode(binObjID); err != nil { + return nil, err + } + lockers = append(lockers, id) + } + } + } + return lockers, nil +} + // releases all records about the objects locked by the locker. // Returns slice of unlocked object ID's or an error. // @@ -325,3 +350,36 @@ func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, e success = err == nil return res, err } + +// GetLocked return `LOCK` id's if provided object is locked by any `LOCK`. Not found +// object is considered as non-locked. +// +// Returns only non-logical errors related to underlying database. +func (db *DB) GetLocked(ctx context.Context, addr oid.Address) (res []oid.ID, err error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("GetLocked", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.GetLocked", + trace.WithAttributes( + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return res, ErrDegradedMode + } + err = metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { + res, err = getLocked(tx, addr.Container(), addr.Object()) + return nil + })) + success = err == nil + return res, err +} diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 8b7b83016..94e68d85a 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -115,7 +115,7 @@ func (db *DB) put(tx *bbolt.Tx, isParent := si != nil - exists, err := db.exists(tx, objectCore.AddressOf(obj), currEpoch) + exists, _, err := db.exists(tx, objectCore.AddressOf(obj), oid.Address{}, currEpoch) var splitInfoError *objectSDK.SplitInfoError if errors.As(err, &splitInfoError) { diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index e5321c794..65d0111c4 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -428,7 +428,7 @@ func (db *DB) selectObjectID( addr.SetObject(id) var splitInfoError *objectSDK.SplitInfoError - ok, err := db.exists(tx, addr, currEpoch) + ok, _, err := db.exists(tx, addr, oid.Address{}, currEpoch) if (err == nil && ok) || errors.As(err, &splitInfoError) { raw := make([]byte, objectKeySize) id.Encode(raw) diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 2cdb8dfa8..12c339c05 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -13,12 +13,14 @@ import ( // ExistsPrm groups the parameters of Exists operation. type ExistsPrm struct { - addr oid.Address + addr oid.Address + paddr oid.Address } // ExistsRes groups the resulting values of Exists operation. type ExistsRes struct { ex bool + lc bool } // SetAddress is an Exists option to set object checked for existence. @@ -26,11 +28,21 @@ func (p *ExistsPrm) SetAddress(addr oid.Address) { p.addr = addr } +// SetParentAddress is an Exists option to set parent object checked for existence. +func (p *ExistsPrm) SetParentAddress(addr oid.Address) { + p.paddr = addr +} + // Exists returns the fact that the object is in the shard. func (p ExistsRes) Exists() bool { return p.ex } +// Locked returns the fact that the object is locked. +func (p ExistsRes) Locked() bool { + return p.lc +} + // Exists checks if object is presented in shard. // // Returns any error encountered that does not allow to @@ -48,6 +60,7 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { defer span.End() var exists bool + var locked bool var err error s.m.RLock() @@ -65,13 +78,16 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { } else { var existsPrm meta.ExistsPrm existsPrm.SetAddress(prm.addr) + existsPrm.SetParent(prm.paddr) var res meta.ExistsRes res, err = s.metaBase.Exists(ctx, existsPrm) exists = res.Exists() + locked = res.Locked() } return ExistsRes{ ex: exists, + lc: locked, }, err } diff --git a/pkg/local_object_storage/shard/lock.go b/pkg/local_object_storage/shard/lock.go index 52186cbfd..4a8d89d63 100644 --- a/pkg/local_object_storage/shard/lock.go +++ b/pkg/local_object_storage/shard/lock.go @@ -71,3 +71,20 @@ func (s *Shard) IsLocked(ctx context.Context, addr oid.Address) (bool, error) { return res.Locked(), nil } + +// GetLocked return lock id's of the provided object. Not found object is +// considered as not locked. Requires healthy metabase, returns ErrDegradedMode otherwise. +func (s *Shard) GetLocked(ctx context.Context, addr oid.Address) ([]oid.ID, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.GetLocked", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("address", addr.EncodeToString()), + )) + defer span.End() + + m := s.GetMode() + if m.NoMetabase() { + return nil, ErrDegradedMode + } + return s.metaBase.GetLocked(ctx, addr) +} diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index d119328a6..45bd89d36 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -2,6 +2,7 @@ package deletesvc import ( "context" + "errors" "fmt" "strconv" @@ -64,9 +65,32 @@ func (exec *execCtx) newAddress(id oid.ID) oid.Address { return a } -func (exec *execCtx) formSplitInfo(ctx context.Context) error { - var err error - exec.splitInfo, err = exec.svc.header.splitInfo(ctx, exec) +func (exec *execCtx) formExtendedInfo(ctx context.Context) error { + _, err := exec.svc.header.head(ctx, exec) + + var errSplitInfo *objectSDK.SplitInfoError + var errECInfo *objectSDK.ECInfoError + + switch { + case err == nil: + return nil + case errors.As(err, &errSplitInfo): + exec.splitInfo = errSplitInfo.SplitInfo() + exec.tombstone.SetSplitID(exec.splitInfo.SplitID()) + + exec.log.Debug(logs.DeleteSplitInfoSuccessfullyFormedCollectingMembers) + + if err := exec.collectMembers(ctx); err != nil { + return err + } + + exec.log.Debug(logs.DeleteMembersSuccessfullyCollected) + return nil + case errors.As(err, &errECInfo): + exec.log.Debug(logs.DeleteECObjectReceived) + return nil + } + if err != nil && !apiclient.IsErrObjectAlreadyRemoved(err) { // IsErrObjectAlreadyRemoved check is required because splitInfo // implicitly performs Head request that may return ObjectAlreadyRemoved diff --git a/pkg/services/object/delete/local.go b/pkg/services/object/delete/local.go index 55ce4408d..2c3c47f49 100644 --- a/pkg/services/object/delete/local.go +++ b/pkg/services/object/delete/local.go @@ -35,19 +35,9 @@ func (exec *execCtx) formTombstone(ctx context.Context) error { exec.log.Debug(logs.DeleteFormingSplitInfo) - if err := exec.formSplitInfo(ctx); err != nil { - return fmt.Errorf("form split info: %w", err) + if err := exec.formExtendedInfo(ctx); err != nil { + return fmt.Errorf("form extended info: %w", err) } - exec.log.Debug(logs.DeleteSplitInfoSuccessfullyFormedCollectingMembers) - - exec.tombstone.SetSplitID(exec.splitInfo.SplitID()) - - if err := exec.collectMembers(ctx); err != nil { - return err - } - - exec.log.Debug(logs.DeleteMembersSuccessfullyCollected) - return exec.initTombstoneObject() } diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index 0ba4da437..0ba21eee3 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -41,7 +41,7 @@ type cfg struct { header interface { // must return (nil, nil) for PHY objects - splitInfo(context.Context, *execCtx) (*objectSDK.SplitInfo, error) + head(context.Context, *execCtx) (*objectSDK.Object, error) children(context.Context, *execCtx) ([]oid.ID, error) diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index 439abca2b..bb2b5f00b 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -2,7 +2,6 @@ package deletesvc import ( "context" - "errors" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" @@ -44,19 +43,8 @@ func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oi return wr.Object(), nil } -func (w *headSvcWrapper) splitInfo(ctx context.Context, exec *execCtx) (*objectSDK.SplitInfo, error) { - _, err := w.headAddress(ctx, exec, exec.address()) - - var errSplitInfo *objectSDK.SplitInfoError - - switch { - case err == nil: - return nil, nil - case errors.As(err, &errSplitInfo): - return errSplitInfo.SplitInfo(), nil - default: - return nil, err - } +func (w *headSvcWrapper) head(ctx context.Context, exec *execCtx) (*objectSDK.Object, error) { + return w.headAddress(ctx, exec, exec.address()) } func (w *headSvcWrapper) children(ctx context.Context, exec *execCtx) ([]oid.ID, error) { From 9f80d6d9a20e1d027f768f1b3c3737f960ebc40f Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 28 May 2024 12:14:13 +0300 Subject: [PATCH 1124/1943] [#1147] Makefile: Fix `gopls-run` target Signed-off-by: Anton Nikiforov --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 710c12703..c93d06aa8 100755 --- a/Makefile +++ b/Makefile @@ -44,9 +44,12 @@ PROTOGEN_FROSTFS_DIR ?= $(PROTOBUF_DIR)/protogen-$(PROTOGEN_FROSTFS_VERSION) STATICCHECK_DIR ?= $(abspath $(BIN))/staticcheck STATICCHECK_VERSION_DIR ?= $(STATICCHECK_DIR)/$(STATICCHECK_VERSION) +SOURCES = $(shell find . -type f -name "*.go" -print) + GOPLS_VERSION ?= v0.15.1 GOPLS_DIR ?= $(abspath $(BIN))/gopls GOPLS_VERSION_DIR ?= $(GOPLS_DIR)/$(GOPLS_VERSION) +GOPLS_TEMP_FILE := $(shell mktemp) FROSTFS_CONTRACTS_PATH=$(abspath ./../frostfs-contract) LOCODE_DB_PATH=$(abspath ./.cache/locode_db) @@ -220,9 +223,12 @@ gopls-run: @if [ ! -d "$(GOPLS_VERSION_DIR)" ]; then \ make gopls-install; \ fi - @if [[ $$(find . -type f -name "*.go" -print | xargs $(GOPLS_VERSION_DIR)/gopls check | tee /dev/tty | wc -l) -ne 0 ]]; then \ + $(GOPLS_VERSION_DIR)/gopls check $(SOURCES) 2>&1 >$(GOPLS_TEMP_FILE) + @if [[ $$(wc -l < $(GOPLS_TEMP_FILE)) -ne 0 ]]; then \ + cat $(GOPLS_TEMP_FILE); \ exit 1; \ fi + rm $(GOPLS_TEMP_FILE) # Run linters in Docker docker/lint: From c1af13b47e234a91cc8e1909920ac0ea2faf8a07 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 28 May 2024 12:21:08 +0300 Subject: [PATCH 1125/1943] [#1147] node: Fix issue from `gopls` Signed-off-by: Anton Nikiforov --- pkg/services/object/delete/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index 45bd89d36..22928dcd5 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -91,7 +91,7 @@ func (exec *execCtx) formExtendedInfo(ctx context.Context) error { return nil } - if err != nil && !apiclient.IsErrObjectAlreadyRemoved(err) { + if !apiclient.IsErrObjectAlreadyRemoved(err) { // IsErrObjectAlreadyRemoved check is required because splitInfo // implicitly performs Head request that may return ObjectAlreadyRemoved // status that is not specified for Delete. From 92e19feb577242aea36639e554b3cf71fc7085d1 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 30 May 2024 09:26:06 +0300 Subject: [PATCH 1126/1943] [#1147] node: Use public fields for `shard.ExistsPrm` Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/engine/delete.go | 2 +- .../engine/engine_test.go | 5 +++- pkg/local_object_storage/engine/exists.go | 10 ++++---- pkg/local_object_storage/engine/inhume.go | 2 +- pkg/local_object_storage/engine/lock.go | 2 +- pkg/local_object_storage/engine/put.go | 7 ++++-- .../engine/remove_copies.go | 2 +- pkg/local_object_storage/shard/exists.go | 24 +++++++------------ pkg/local_object_storage/shard/reload_test.go | 2 +- 9 files changed, 26 insertions(+), 30 deletions(-) diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 44a612213..096528967 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -83,7 +83,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e // 2. Otherwise, search for all objects with a particular SplitID and delete them too. e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { var existsPrm shard.ExistsPrm - existsPrm.SetAddress(prm.addr) + existsPrm.Address = prm.addr resExists, err := sh.Exists(ctx, existsPrm) if err != nil { diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 70c54590f..49976abbb 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -63,7 +63,10 @@ func benchmarkExists(b *testing.B, shardNum int) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - ok, _, err := e.exists(context.Background(), addr, oid.Address{}) + var shPrm shard.ExistsPrm + shPrm.Address = addr + shPrm.ParentAddress = oid.Address{} + ok, _, err := e.exists(context.Background(), shPrm) if err != nil || ok { b.Fatalf("%t %v", ok, err) } diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index ce669ec5e..c57f79691 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -8,18 +8,16 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) -func (e *StorageEngine) exists(ctx context.Context, addr oid.Address, parentAddr oid.Address) (bool, bool, error) { - var shPrm shard.ExistsPrm - shPrm.SetAddress(addr) - shPrm.SetParentAddress(parentAddr) +// exists return in the first value true if object exists. +// Second return value marks is parent object locked. +func (e *StorageEngine) exists(ctx context.Context, shPrm shard.ExistsPrm) (bool, bool, error) { alreadyRemoved := false exists := false locked := false - e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { + e.iterateOverSortedShards(shPrm.Address, func(_ int, sh hashedShard) (stop bool) { res, err := sh.Exists(ctx, shPrm) if err != nil { if client.IsErrObjectAlreadyRemoved(err) { diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 62e7be933..991305af0 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -142,7 +142,7 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh }() if checkExists { - existPrm.SetAddress(addr) + existPrm.Address = addr exRes, err := sh.Exists(ctx, existPrm) if err != nil { if client.IsErrObjectAlreadyRemoved(err) || shard.IsErrObjectExpired(err) { diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index af56c3e1e..5354c205f 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -78,7 +78,7 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo if checkExists { var existsPrm shard.ExistsPrm - existsPrm.SetAddress(addrLocked) + existsPrm.Address = addrLocked exRes, err := sh.Exists(ctx, existsPrm) if err != nil { diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index aa94ef22e..2a78febed 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -85,7 +85,10 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { parent.SetObject(prm.obj.ECHeader().Parent()) parent.SetContainer(addr.Container()) } - existed, locked, err := e.exists(ctx, addr, parent) + var shPrm shard.ExistsPrm + shPrm.Address = addr + shPrm.ParentAddress = parent + existed, locked, err := e.exists(ctx, shPrm) if err != nil { return err } @@ -138,7 +141,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti defer close(exitCh) var existPrm shard.ExistsPrm - existPrm.SetAddress(addr) + existPrm.Address = addr exists, err := sh.Exists(ctx, existPrm) if err != nil { diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index 00562e4cb..b99cf4f44 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -115,7 +115,7 @@ func (e *StorageEngine) removeObjects(ctx context.Context, ch <-chan oid.Address found := false for i := range shards { var existsPrm shard.ExistsPrm - existsPrm.SetAddress(addr) + existsPrm.Address = addr res, err := shards[i].Exists(ctx, existsPrm) if err != nil { diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 12c339c05..b5a9604b4 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -13,8 +13,10 @@ import ( // ExistsPrm groups the parameters of Exists operation. type ExistsPrm struct { - addr oid.Address - paddr oid.Address + // Exists option to set object checked for existence. + Address oid.Address + // Exists option to set parent object checked for existence. + ParentAddress oid.Address } // ExistsRes groups the resulting values of Exists operation. @@ -23,16 +25,6 @@ type ExistsRes struct { lc bool } -// SetAddress is an Exists option to set object checked for existence. -func (p *ExistsPrm) SetAddress(addr oid.Address) { - p.addr = addr -} - -// SetParentAddress is an Exists option to set parent object checked for existence. -func (p *ExistsPrm) SetParentAddress(addr oid.Address) { - p.paddr = addr -} - // Exists returns the fact that the object is in the shard. func (p ExistsRes) Exists() bool { return p.ex @@ -55,7 +47,7 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Exists", trace.WithAttributes( attribute.String("shard_id", s.ID().String()), - attribute.String("address", prm.addr.EncodeToString()), + attribute.String("address", prm.Address.EncodeToString()), )) defer span.End() @@ -70,15 +62,15 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { return ExistsRes{}, ErrShardDisabled } else if s.info.Mode.NoMetabase() { var p common.ExistsPrm - p.Address = prm.addr + p.Address = prm.Address var res common.ExistsRes res, err = s.blobStor.Exists(ctx, p) exists = res.Exists } else { var existsPrm meta.ExistsPrm - existsPrm.SetAddress(prm.addr) - existsPrm.SetParent(prm.paddr) + existsPrm.SetAddress(prm.Address) + existsPrm.SetParent(prm.ParentAddress) var res meta.ExistsRes res, err = s.metaBase.Exists(ctx, existsPrm) diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index b5ea2fec7..7dacbfa6c 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -72,7 +72,7 @@ func TestShardReload(t *testing.T) { checkHasObjects := func(t *testing.T, exists bool) { for i := range objects { var prm ExistsPrm - prm.SetAddress(objects[i].addr) + prm.Address = objects[i].addr res, err := sh.Exists(context.Background(), prm) require.NoError(t, err) From 0b367007fc0d37d009de7cd7f90ab2964c5ab85c Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 28 May 2024 16:12:14 +0300 Subject: [PATCH 1127/1943] [#1152] go.mod: Update api-go and sdk versions * Resolve conflicts for apemanager since api-go contains ape and apemanager packages and SDK only ape package. Signed-off-by: Airat Arifullin --- .../modules/ape_manager/add_chain.go | 16 ++++----- go.mod | 4 +-- go.sum | 8 ++--- pkg/services/apemanager/executor.go | 35 ++++++++++--------- 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/cmd/frostfs-cli/modules/ape_manager/add_chain.go b/cmd/frostfs-cli/modules/ape_manager/add_chain.go index dea7fb36e..c6622da25 100644 --- a/cmd/frostfs-cli/modules/ape_manager/add_chain.go +++ b/cmd/frostfs-cli/modules/ape_manager/add_chain.go @@ -9,7 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - apemanager_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/apemanager" + apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -49,7 +49,7 @@ var addCmd = &cobra.Command{ }, } -func parseTarget(cmd *cobra.Command) (ct apemanager_sdk.ChainTarget) { +func parseTarget(cmd *cobra.Command) (ct apeSDK.ChainTarget) { typ, _ := cmd.Flags().GetString(targetTypeFlag) name, _ := cmd.Flags().GetString(targetNameFlag) @@ -57,22 +57,22 @@ func parseTarget(cmd *cobra.Command) (ct apemanager_sdk.ChainTarget) { switch typ { case namespaceTarget: - ct.TargetType = apemanager_sdk.TargetTypeNamespace + ct.TargetType = apeSDK.TargetTypeNamespace case containerTarget: var cnr cid.ID commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(name)) - ct.TargetType = apemanager_sdk.TargetTypeContainer + ct.TargetType = apeSDK.TargetTypeContainer case userTarget: - ct.TargetType = apemanager_sdk.TargetTypeUser + ct.TargetType = apeSDK.TargetTypeUser case groupTarget: - ct.TargetType = apemanager_sdk.TargetTypeGroup + ct.TargetType = apeSDK.TargetTypeGroup default: commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType) } return ct } -func parseChain(cmd *cobra.Command) apemanager_sdk.Chain { +func parseChain(cmd *cobra.Command) apeSDK.Chain { chainID, _ := cmd.Flags().GetString(chainIDFlag) hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) @@ -99,7 +99,7 @@ func parseChain(cmd *cobra.Command) apemanager_sdk.Chain { util.PrintHumanReadableAPEChain(cmd, chain) serialized := chain.Bytes() - return apemanager_sdk.Chain{ + return apeSDK.Chain{ Raw: serialized, } } diff --git a/go.mod b/go.mod index 1335747aa..6edd305bb 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240516133103-0803bc6ded00 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240517121628-3de256d05e32 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531121442-717a7d00ef21 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index ce68ddb22..dcae21bc3 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,15 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240516133103-0803bc6ded00 h1:Q3B9WtFh05AXhUFs/2CLvhh9tuFs/Zd/XemWBbuzvg8= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240516133103-0803bc6ded00/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 h1:H5GvrVlowIMWfzqQkhY0p0myooJxQ1sMRVSFfXawwWg= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 h1:Tp4I+XOLp3VCJORfxSamQtj3RZNISbaLM4WD5iIzXxg= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240517121628-3de256d05e32 h1:ebXBsKCAINoJLhBUqi1x3fToxUHUsUjEBGKo86JQoho= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240517121628-3de256d05e32/go.mod h1:Wu1jsGM6JhZwS5oMWeslKvMcvRbg8nGt/B/TfsQPoFA= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531121442-717a7d00ef21 h1:ugo9k9s0+51BCMhD4mqHD+qD5P4BOTfDxcEMqTsM9+A= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531121442-717a7d00ef21/go.mod h1:dwBHqBoseOpU4EiIPSGxhNeQx2QOEao/1r8h26syswI= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d h1:hHk8FWbWgEnwm2I045CaBIrZBjy/o81CehIVOySA/pQ= diff --git a/pkg/services/apemanager/executor.go b/pkg/services/apemanager/executor.go index 9cfc97473..25f43486a 100644 --- a/pkg/services/apemanager/executor.go +++ b/pkg/services/apemanager/executor.go @@ -8,7 +8,8 @@ import ( "errors" "fmt" - apemanager_v2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" + apeV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/ape" + apemanagerV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" ape_contract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/contract_storage" @@ -83,13 +84,13 @@ func (s *Service) validateContainerTargetRequest(cid string, pubKey *keys.Public return nil } -func (s *Service) AddChain(_ context.Context, req *apemanager_v2.AddChainRequest) (*apemanager_v2.AddChainResponse, error) { +func (s *Service) AddChain(_ context.Context, req *apemanagerV2.AddChainRequest) (*apemanagerV2.AddChainResponse, error) { pub, err := getSignaturePublicKey(req.GetVerificationHeader()) if err != nil { return nil, err } - chain, err := decodeAndValidateChain(req.GetBody().GetChain().GetKind().(*apemanager_v2.ChainRaw).GetRaw()) + chain, err := decodeAndValidateChain(req.GetBody().GetChain().GetKind().(*apeV2.ChainRaw).GetRaw()) if err != nil { return nil, err } @@ -105,7 +106,7 @@ func (s *Service) AddChain(_ context.Context, req *apemanager_v2.AddChainRequest var target policy_engine.Target switch targetType := req.GetBody().GetTarget().GetTargetType(); targetType { - case apemanager_v2.TargetTypeContainer: + case apeV2.TargetTypeContainer: reqCID := req.GetBody().GetTarget().GetName() if err = s.validateContainerTargetRequest(reqCID, pub); err != nil { return nil, err @@ -119,16 +120,16 @@ func (s *Service) AddChain(_ context.Context, req *apemanager_v2.AddChainRequest return nil, err } - body := new(apemanager_v2.AddChainResponseBody) + body := new(apemanagerV2.AddChainResponseBody) body.SetChainID(chain.ID) - resp := new(apemanager_v2.AddChainResponse) + resp := new(apemanagerV2.AddChainResponse) resp.SetBody(body) return resp, nil } -func (s *Service) RemoveChain(_ context.Context, req *apemanager_v2.RemoveChainRequest) (*apemanager_v2.RemoveChainResponse, error) { +func (s *Service) RemoveChain(_ context.Context, req *apemanagerV2.RemoveChainRequest) (*apemanagerV2.RemoveChainResponse, error) { pub, err := getSignaturePublicKey(req.GetVerificationHeader()) if err != nil { return nil, err @@ -137,7 +138,7 @@ func (s *Service) RemoveChain(_ context.Context, req *apemanager_v2.RemoveChainR var target policy_engine.Target switch targetType := req.GetBody().GetTarget().GetTargetType(); targetType { - case apemanager_v2.TargetTypeContainer: + case apeV2.TargetTypeContainer: reqCID := req.GetBody().GetTarget().GetName() if err = s.validateContainerTargetRequest(reqCID, pub); err != nil { return nil, err @@ -151,15 +152,15 @@ func (s *Service) RemoveChain(_ context.Context, req *apemanager_v2.RemoveChainR return nil, err } - body := new(apemanager_v2.RemoveChainResponseBody) + body := new(apemanagerV2.RemoveChainResponseBody) - resp := new(apemanager_v2.RemoveChainResponse) + resp := new(apemanagerV2.RemoveChainResponse) resp.SetBody(body) return resp, nil } -func (s *Service) ListChains(_ context.Context, req *apemanager_v2.ListChainsRequest) (*apemanager_v2.ListChainsResponse, error) { +func (s *Service) ListChains(_ context.Context, req *apemanagerV2.ListChainsRequest) (*apemanagerV2.ListChainsResponse, error) { pub, err := getSignaturePublicKey(req.GetVerificationHeader()) if err != nil { return nil, err @@ -168,7 +169,7 @@ func (s *Service) ListChains(_ context.Context, req *apemanager_v2.ListChainsReq var target policy_engine.Target switch targetType := req.GetBody().GetTarget().GetTargetType(); targetType { - case apemanager_v2.TargetTypeContainer: + case apeV2.TargetTypeContainer: reqCID := req.GetBody().GetTarget().GetName() if err = s.validateContainerTargetRequest(reqCID, pub); err != nil { return nil, err @@ -183,21 +184,21 @@ func (s *Service) ListChains(_ context.Context, req *apemanager_v2.ListChainsReq return nil, err } - res := make([]*apemanager_v2.Chain, 0, len(chs)) + res := make([]*apeV2.Chain, 0, len(chs)) for _, ch := range chs { - v2chraw := new(apemanager_v2.ChainRaw) + v2chraw := new(apeV2.ChainRaw) v2chraw.SetRaw(ch.Bytes()) - v2ch := new(apemanager_v2.Chain) + v2ch := new(apeV2.Chain) v2ch.SetKind(v2chraw) res = append(res, v2ch) } - body := new(apemanager_v2.ListChainsResponseBody) + body := new(apemanagerV2.ListChainsResponseBody) body.SetChains(res) - resp := new(apemanager_v2.ListChainsResponse) + resp := new(apemanagerV2.ListChainsResponse) resp.SetBody(body) return resp, nil From f0edebea1820d96b9219917d34c7649f31db3782 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 21 May 2024 14:19:22 +0300 Subject: [PATCH 1128/1943] [#1144] metabase: Support ec parent filter for `Search` Signed-off-by: Airat Arifullin --- pkg/local_object_storage/metabase/db.go | 2 +- pkg/local_object_storage/metabase/put.go | 11 +++++++++++ pkg/local_object_storage/metabase/select.go | 3 +++ pkg/local_object_storage/metabase/util.go | 11 +++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index f5341ff2e..f2203f4d9 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -262,7 +262,7 @@ func unknownMatcherBucket(_ *bbolt.Bucket, _ string, _ string, _ func([]byte, [] // in boltDB. Useful for getting filter values from unique and list indexes. func bucketKeyHelper(hdr string, val string) []byte { switch hdr { - case v2object.FilterHeaderParent: + case v2object.FilterHeaderParent, v2object.FilterHeaderECParent: v, err := base58.Decode(val) if err != nil { return nil diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 94e68d85a..bb5976ec4 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -323,6 +323,17 @@ func updateListIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun } } + if ech := obj.ECHeader(); ech != nil { + err := f(tx, namedBucketItem{ + name: ecParentToChunksBucketName(cnr, bucketName), + key: objectKey(ech.Parent(), make([]byte, objectKeySize)), + val: objKey, + }) + if err != nil { + return err + } + } + return nil } diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 65d0111c4..f544f95e2 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -211,6 +211,9 @@ func (db *DB) selectFastFilter( case v2object.FilterHeaderSplitID: bucketName := splitBucketName(cnr, bucketName) db.selectFromList(tx, bucketName, f, to, fNum) + case v2object.FilterHeaderECParent: + bucketName := ecParentToChunksBucketName(cnr, bucketName) + db.selectFromList(tx, bucketName, f, to, fNum) case v2object.FilterPropertyRoot: selectAllFromBucket(tx, rootBucketName(cnr, bucketName), to, fNum) case v2object.FilterPropertyPhy: diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index 9249ae49b..cfccff394 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -124,6 +124,12 @@ const ( // Key: container ID + type // Value: Object id ecInfoPrefix + + // ecParentToChunksPrefix is used to store a relation between EC parent ID and chunks, + // but unlike for ecInfoPrefix the list of chunk IDs is encoded with encodeList. + // Key: EC parent ID + // Value: list of EC chunk IDs + ecParentToChunksPrefix ) const ( @@ -200,6 +206,11 @@ func ecInfoBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, ecInfoPrefix, key) } +// ecParentToChunksBucketName returns _ecParentToChunks. +func ecParentToChunksBucketName(cnr cid.ID, key []byte) []byte { + return bucketName(cnr, ecParentToChunksPrefix, key) +} + // addressKey returns key for K-V tables when key is a whole address. func addressKey(addr oid.Address, key []byte) []byte { addr.Container().Encode(key) From f8e33f8e3ad90ff01d85c33cbeb21a5c9a7dfa16 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 21 May 2024 14:25:15 +0300 Subject: [PATCH 1129/1943] [#1144] metabase: Proprely choose root OID for EC-splitted objects * If EC-parent is a part of Split itself, then save to root bucket its parent; * If EC-parent is not a part of Split itself, then save to root bucket OID of this EC-parent. Signed-off-by: Airat Arifullin --- pkg/local_object_storage/metabase/put.go | 39 ++++++++++++++++++------ 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index bb5976ec4..8a08adc36 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -257,17 +257,19 @@ func putUniqueIndexes( } } - err = putUniqueIndexItem(tx, namedBucketItem{ - name: rootBucketName(cnr, bucketName), - key: objKey, - val: splitInfo, - }) - if err != nil { - return err - } - + isObjKeySet := true if ecHead := obj.ECHeader(); ecHead != nil { - err = putECInfo(tx, cnr, objKey, ecHead) + if err = putECInfo(tx, cnr, objKey, ecHead); err != nil { + return err + } + objKey, isObjKeySet = objectKeyByECHeader(ecHead) + } + if isObjKeySet { + err = putUniqueIndexItem(tx, namedBucketItem{ + name: rootBucketName(cnr, bucketName), + key: objKey, + val: splitInfo, + }) if err != nil { return err } @@ -277,6 +279,23 @@ func putUniqueIndexes( return nil } +// objectKeyByECHeader returns objectKey for an object that has EC Header. +// If object's parent is in Split, then parent's non-nil Split parent ID is set to object key. +// If object's parent is not in Split, then its ID is set to object key. +// Otherwise, such object keys should be ignored -- they are not put to the root bucket. +func objectKeyByECHeader(ech *objectSDK.ECHeader) (objKey []byte, isSet bool) { + if ech.ParentSplitID() != nil { + if parentSplitParentID := ech.ParentSplitParentID(); parentSplitParentID != nil { + isSet = true + objKey = objectKey(*parentSplitParentID, make([]byte, objectKeySize)) + } + return + } + isSet = true + objKey = objectKey(ech.Parent(), make([]byte, objectKeySize)) + return +} + type updateIndexItemFunc = func(tx *bbolt.Tx, item namedBucketItem) error func updateListIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFunc) error { From 5aacb8fc86210ee6a7da02ba15a9edc4a250f269 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 27 May 2024 15:52:08 +0300 Subject: [PATCH 1130/1943] [#1144] metabase: Save parent attributes for ec-chunks Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 ++-- pkg/local_object_storage/metabase/put.go | 10 ++++++++-- pkg/services/policer/ec_test.go | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 6edd305bb..093560953 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531121442-717a7d00ef21 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index dcae21bc3..6f5034f00 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531121442-717a7d00ef21 h1:ugo9k9s0+51BCMhD4mqHD+qD5P4BOTfDxcEMqTsM9+A= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531121442-717a7d00ef21/go.mod h1:dwBHqBoseOpU4EiIPSGxhNeQx2QOEao/1r8h26syswI= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f h1:vBLC1OSGMSn7lRJv/p1of0veifuBdZdztVrF9Vn+UFk= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d h1:hHk8FWbWgEnwm2I045CaBIrZBjy/o81CehIVOySA/pQ= diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 8a08adc36..966db9401 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -361,8 +361,6 @@ func updateFKBTIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun cnr, _ := obj.ContainerID() objKey := objectKey(id, make([]byte, objectKeySize)) - attrs := obj.Attributes() - key := make([]byte, bucketKeySize) err := f(tx, namedBucketItem{ name: ownerBucketName(cnr, key), @@ -373,6 +371,14 @@ func updateFKBTIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun return err } + var attrs []objectSDK.Attribute + if obj.ECHeader() != nil { + attrs = obj.ECHeader().ParentAttributes() + objKey = objectKey(obj.ECHeader().Parent(), make([]byte, objectKeySize)) + } else { + attrs = obj.Attributes() + } + // user specified attributes for i := range attrs { key = attributeBucketName(cnr, attrs[i].Key(), key) diff --git a/pkg/services/policer/ec_test.go b/pkg/services/policer/ec_test.go index 11e7a8cdf..9492787dc 100644 --- a/pkg/services/policer/ec_test.go +++ b/pkg/services/policer/ec_test.go @@ -114,7 +114,7 @@ func TestECChunkHasInvalidPlacement(t *testing.T) { chunkObject.SetID(chunkAddress.Object()) chunkObject.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) chunkObject.SetPayloadSize(uint64(10)) - chunkObject.SetECHeader(objectSDK.NewECHeader(parentID, nil, nil, 1, 3, []byte{}, 0)) + chunkObject.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: parentID}, 0, 3, []byte{}, 0)) var policy netmapSDK.PlacementPolicy require.NoError(t, policy.DecodeString("EC 2.1")) From f2d2908745a5fc55593e853faabca989b9896495 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 31 May 2024 21:58:09 +0300 Subject: [PATCH 1131/1943] Release v0.40.0 Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 14 ++++++++++++++ VERSION | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53e2c0bc7..0382d0316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,20 @@ Changelog for FrostFS Node ### Removed ### Updated +## [v0.40.0] + +### Added +- Support EC chunk reconstruction in policer (#1129) +- Support LOCK, DELETE and SEARCH methods on EC objects (#1147, 1144) +- apemanager service to manage APE chains (#1105) + +### Fixed +- Properly verify GetRangeHash response (#1083) +- Send `MONOTONIC_USEC` in sdnotify on reload (#1135) + +### Updated +- neo-go to `v0.106.0` + ## [v0.39.0] ### Added diff --git a/VERSION b/VERSION index 2302e306c..707d5fac2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.39.0 +v0.40.0 From 643480d6fa2585035457e647dcf840f2f34167e5 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 3 Jun 2024 13:54:59 +0300 Subject: [PATCH 1132/1943] [#1146] adm: Make --group-name flag required Signed-off-by: Alexander Chuprov --- cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index d2e45696e..541282967 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -194,6 +194,7 @@ func initFrostfsIDCreateGroupCmd() { frostfsidCreateGroupCmd.Flags().String(namespaceFlag, "", "Namespace where create group") frostfsidCreateGroupCmd.Flags().String(groupNameFlag, "", "Group name, must be unique in namespace") frostfsidCreateGroupCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + _ = frostfsidCreateGroupCmd.MarkFlagRequired(groupNameFlag) } func initFrostfsIDDeleteGroupCmd() { From cc2449beafb6407e48524a2f2cbece90f331d9c4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Jun 2024 13:30:30 +0300 Subject: [PATCH 1133/1943] [#1158] metabase: Fix EC storage schema Do not store EC info twice. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/delete.go | 40 --------------------- pkg/local_object_storage/metabase/get.go | 9 ++--- pkg/local_object_storage/metabase/put.go | 37 +------------------ pkg/local_object_storage/metabase/select.go | 2 +- pkg/local_object_storage/metabase/util.go | 11 ------ 5 files changed, 7 insertions(+), 92 deletions(-) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 4addeae77..21b98fca1 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -473,46 +473,6 @@ func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error name: rootBucketName(cnr, bucketName), key: objKey, }) - if obj.ECHeader() != nil { - err := delECInfo(tx, cnr, objKey, obj.ECHeader()) - if err != nil { - return err - } - } return nil } - -func delECInfo(tx *bbolt.Tx, cnr cid.ID, objKey []byte, ecHead *objectSDK.ECHeader) error { - parentID := objectKey(ecHead.Parent(), make([]byte, objectKeySize)) - bucketName := make([]byte, bucketKeySize) - - val := getFromBucket(tx, ecInfoBucketName(cnr, bucketName), parentID) - if len(val) > 0 { - if bytes.Equal(val, objKey) { - delUniqueIndexItem(tx, namedBucketItem{ - name: ecInfoBucketName(cnr, bucketName), - key: parentID, - }) - } else { - val = bytes.Clone(val) - offset := 0 - for offset < len(val) { - if bytes.Equal(objKey, val[offset:offset+objectKeySize]) { - val = append(val[:offset], val[offset+objectKeySize:]...) - break - } - offset += objectKeySize - } - err := putUniqueIndexItem(tx, namedBucketItem{ - name: ecInfoBucketName(cnr, make([]byte, bucketKeySize)), - key: parentID, - val: val, - }) - if err != nil { - return err - } - } - } - return nil -} diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index 355f62933..d9acd4ce2 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -192,10 +192,12 @@ func getSplitInfoError(tx *bbolt.Tx, cnr cid.ID, key []byte) error { } func getECInfoError(tx *bbolt.Tx, cnr cid.ID, data []byte) error { - offset := 0 + keys, err := decodeList(data) + if err != nil { + return err + } ecInfo := objectSDK.NewECInfo() - for offset < len(data) { - key := data[offset : offset+objectKeySize] + for _, key := range keys { // check in primary index ojbData := getFromBucket(tx, primaryBucketName(cnr, make([]byte, bucketKeySize)), key) if len(ojbData) != 0 { @@ -210,7 +212,6 @@ func getECInfoError(tx *bbolt.Tx, cnr cid.ID, data []byte) error { chunk.Total = obj.ECHeader().Total() ecInfo.AddChunk(chunk) } - offset += objectKeySize } return logicerr.Wrap(objectSDK.NewECInfoError(ecInfo)) } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 966db9401..4b6359ebf 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -1,7 +1,6 @@ package meta import ( - "bytes" "context" "encoding/binary" "errors" @@ -259,9 +258,6 @@ func putUniqueIndexes( isObjKeySet := true if ecHead := obj.ECHeader(); ecHead != nil { - if err = putECInfo(tx, cnr, objKey, ecHead); err != nil { - return err - } objKey, isObjKeySet = objectKeyByECHeader(ecHead) } if isObjKeySet { @@ -344,7 +340,7 @@ func updateListIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun if ech := obj.ECHeader(); ech != nil { err := f(tx, namedBucketItem{ - name: ecParentToChunksBucketName(cnr, bucketName), + name: ecInfoBucketName(cnr, bucketName), key: objectKey(ech.Parent(), make([]byte, objectKeySize)), val: objKey, }) @@ -615,34 +611,3 @@ func isLinkObject(obj *objectSDK.Object) bool { func isLastObject(obj *objectSDK.Object) bool { return len(obj.Children()) == 0 && obj.Parent() != nil } - -func putECInfo(tx *bbolt.Tx, - cnr cid.ID, objKey []byte, - ecHead *objectSDK.ECHeader, -) error { - parentID := objectKey(ecHead.Parent(), make([]byte, objectKeySize)) - bucketName := make([]byte, bucketKeySize) - - val := getFromBucket(tx, ecInfoBucketName(cnr, bucketName), parentID) - if len(val) == 0 { - val = objKey - } else { - offset := 0 - found := false - for offset < len(val) { - if bytes.Equal(objKey, val[offset:offset+objectKeySize]) { - found = true - break - } - offset += objectKeySize - } - if !found { - val = append(val, objKey...) - } - } - return putUniqueIndexItem(tx, namedBucketItem{ - name: ecInfoBucketName(cnr, make([]byte, bucketKeySize)), - key: parentID, - val: val, - }) -} diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index f544f95e2..3a4d7a227 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -212,7 +212,7 @@ func (db *DB) selectFastFilter( bucketName := splitBucketName(cnr, bucketName) db.selectFromList(tx, bucketName, f, to, fNum) case v2object.FilterHeaderECParent: - bucketName := ecParentToChunksBucketName(cnr, bucketName) + bucketName := ecInfoBucketName(cnr, bucketName) db.selectFromList(tx, bucketName, f, to, fNum) case v2object.FilterPropertyRoot: selectAllFromBucket(tx, rootBucketName(cnr, bucketName), to, fNum) diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index cfccff394..9249ae49b 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -124,12 +124,6 @@ const ( // Key: container ID + type // Value: Object id ecInfoPrefix - - // ecParentToChunksPrefix is used to store a relation between EC parent ID and chunks, - // but unlike for ecInfoPrefix the list of chunk IDs is encoded with encodeList. - // Key: EC parent ID - // Value: list of EC chunk IDs - ecParentToChunksPrefix ) const ( @@ -206,11 +200,6 @@ func ecInfoBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, ecInfoPrefix, key) } -// ecParentToChunksBucketName returns _ecParentToChunks. -func ecParentToChunksBucketName(cnr cid.ID, key []byte) []byte { - return bucketName(cnr, ecParentToChunksPrefix, key) -} - // addressKey returns key for K-V tables when key is a whole address. func addressKey(addr oid.Address, key []byte) []byte { addr.Container().Encode(key) From 6f2187a420e9c9c7bd2e1fa978a24720224efbd2 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 14 May 2024 14:44:55 +0300 Subject: [PATCH 1134/1943] [#1121] node: Refactor mods of shard Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/shard/mode/mode.go | 28 ++++++++++----------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go index 49c888d63..461611465 100644 --- a/pkg/local_object_storage/shard/mode/mode.go +++ b/pkg/local_object_storage/shard/mode/mode.go @@ -8,27 +8,25 @@ type Mode uint32 const ( // ReadWrite is a Mode value for shard that is available // for read and write operations. Default shard mode. - ReadWrite Mode = 0 + ReadWrite Mode = 0b000 - // DegradedReadOnly is a Mode value for shard that is set automatically - // after a certain number of errors is encountered. It is the same as - // `mode.Degraded` but also is read-only. - DegradedReadOnly = Degraded | ReadOnly + // ReadOnly is a Mode value for shard that does not + // accept write operation but is readable. + ReadOnly Mode = 0b001 + + // Degraded is a Mode value for shard when the metabase is unavailable. + // It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator. + Degraded Mode = 0b010 // Disabled mode is a mode where a shard is disabled. // An existing shard can't have this mode, but it can be used in // the configuration or control service commands. - Disabled = math.MaxUint32 -) + Disabled Mode = math.MaxUint32 -const ( - // ReadOnly is a Mode value for shard that does not - // accept write operation but is readable. - ReadOnly Mode = 1 << iota - - // Degraded is a Mode value for shard when the metabase is unavailable. - // It is hard to perform some modifying operations in this mode, thus it can only be set by an administrator. - Degraded + // DegradedReadOnly is a Mode value for shard that is set automatically + // after a certain number of errors is encountered. It is the same as + // `mode.Degraded` but also is read-only. + DegradedReadOnly Mode = Degraded | ReadOnly ) func (m Mode) String() string { From 806236da781d2c1cfd74faee2ff0da62ffd43485 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 4 Jun 2024 16:28:47 +0300 Subject: [PATCH 1135/1943] [#1121] node: Change mode of shard components Signed-off-by: Alexander Chuprov --- .../blobovniczatree/concurrency_test.go | 3 +- .../blobstor/blobovniczatree/control.go | 7 ++- .../blobstor/blobovniczatree/control_test.go | 9 +-- .../blobstor/blobovniczatree/exists_test.go | 3 +- .../blobstor/blobovniczatree/iterate_test.go | 3 +- .../blobstor/blobovniczatree/metrics.go | 5 +- .../blobovniczatree/rebuild_failover_test.go | 3 +- .../blobstor/blobovniczatree/rebuild_test.go | 9 +-- .../blobstor/common/storage.go | 3 +- pkg/local_object_storage/blobstor/control.go | 5 +- .../blobstor/fstree/control.go | 7 ++- .../blobstor/fstree/fstree_test.go | 3 +- .../blobstor/fstree/metrics.go | 10 ++- .../blobstor/internal/blobstortest/control.go | 5 +- .../blobstor/internal/blobstortest/delete.go | 3 +- .../blobstor/internal/blobstortest/exists.go | 3 +- .../blobstor/internal/blobstortest/get.go | 3 +- .../internal/blobstortest/get_range.go | 3 +- .../blobstor/internal/blobstortest/iterate.go | 3 +- .../blobstor/memstore/control.go | 9 ++- .../blobstor/memstore/memstore_test.go | 3 +- .../blobstor/perf_test.go | 3 +- .../blobstor/teststore/option.go | 9 +-- .../blobstor/teststore/teststore.go | 9 +-- .../engine/control_test.go | 2 +- pkg/local_object_storage/engine/writecache.go | 4 +- pkg/local_object_storage/metabase/control.go | 16 ++--- pkg/local_object_storage/metabase/metrics.go | 4 +- pkg/local_object_storage/metabase/mode.go | 2 +- pkg/local_object_storage/metabase/shard_id.go | 2 +- .../metrics/blobovnicza.go | 5 +- pkg/local_object_storage/metrics/fstree.go | 5 +- pkg/local_object_storage/metrics/metabase.go | 2 +- pkg/local_object_storage/metrics/pilorama.go | 4 +- pkg/local_object_storage/pilorama/boltdb.go | 8 +-- pkg/local_object_storage/pilorama/metrics.go | 4 +- pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/shard/mode/mode.go | 62 +++++++++++++++++++ .../writecache/cachebbolt.go | 10 +-- pkg/local_object_storage/writecache/flush.go | 2 +- .../writecache/metrics.go | 4 +- pkg/local_object_storage/writecache/mode.go | 4 +- pkg/local_object_storage/writecache/seal.go | 2 +- .../writecache/storage.go | 9 +-- pkg/metrics/blobovnicza.go | 7 ++- pkg/metrics/fstree.go | 7 ++- pkg/metrics/pilorama.go | 4 +- 47 files changed, 194 insertions(+), 100 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go index 97606df02..5bed86142 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "github.com/stretchr/testify/require" ) @@ -23,7 +24,7 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) { WithBlobovniczaShallowWidth(10), WithBlobovniczaShallowDepth(1), WithRootPath(t.TempDir())) - require.NoError(t, st.Open(false)) + require.NoError(t, st.Open(mode.ComponentReadWrite)) require.NoError(t, st.Init()) defer func() { require.NoError(t, st.Close()) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index bd98967cb..67e8c8f18 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -8,6 +8,7 @@ import ( "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -16,9 +17,9 @@ import ( var errFailedToChangeExtensionReadOnly = errors.New("failed to change blobovnicza extension: read only mode") // Open opens blobovnicza tree. -func (b *Blobovniczas) Open(readOnly bool) error { - b.readOnly = readOnly - b.metrics.SetMode(readOnly) +func (b *Blobovniczas) Open(mode mode.ComponentMode) error { + b.readOnly = mode.ReadOnly() + b.metrics.SetMode(mode) b.metrics.SetRebuildStatus(rebuildStatusNotStarted) b.openManagers() return nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go index 0356fbaae..7218cd6bb 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" ) @@ -54,7 +55,7 @@ func openAndCloseTestTree(t *testing.T, depth, width uint64, path string) { WithBlobovniczaShallowWidth(width), WithRootPath(path), ) - require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Open(mode.ComponentReadWrite)) require.NoError(t, blz.Init()) require.NoError(t, blz.Close()) } @@ -85,7 +86,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { WithRootPath(rootDir), ) - require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Open(mode.ComponentReadWrite)) require.NoError(t, blz.Init()) obj35 := blobstortest.NewObject(10 * 1024) @@ -123,7 +124,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { WithRootPath(rootDir), ) - require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Open(mode.ComponentReadWrite)) require.NoError(t, blz.Init()) gRes, err = blz.Get(context.Background(), common.GetPrm{ @@ -160,7 +161,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { WithBlobovniczaShallowWidth(5), WithRootPath(rootDir), ) - require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Open(mode.ComponentReadWrite)) require.NoError(t, blz.Init()) gRes, err = blz.Get(context.Background(), common.GetPrm{ diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 625e620b8..d6ffd8bce 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "github.com/stretchr/testify/require" ) @@ -24,7 +25,7 @@ func TestExistsInvalidStorageID(t *testing.T) { WithBlobovniczaShallowDepth(2), WithRootPath(dir), WithBlobovniczaSize(1<<20)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) defer func() { require.NoError(t, b.Close()) }() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go index 506dd9a4b..b067a164b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) @@ -18,7 +19,7 @@ func TestIterateSortedLeavesAndDBPathsAreSame(t *testing.T) { WithRootPath(t.TempDir()), ) blz.createDBInAdvance = true - require.NoError(t, blz.Open(false)) + require.NoError(t, blz.Open(mode.ComponentReadWrite)) require.NoError(t, blz.Init()) defer func() { require.NoError(t, blz.Close()) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go index 28289c19e..68dc7ff38 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/metrics.go @@ -4,6 +4,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) const ( @@ -18,7 +19,7 @@ type Metrics interface { SetParentID(parentID string) - SetMode(readOnly bool) + SetMode(mode.ComponentMode) Close() SetRebuildStatus(status string) @@ -37,7 +38,7 @@ type Metrics interface { type noopMetrics struct{} func (m *noopMetrics) SetParentID(string) {} -func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) SetMode(mode.ComponentMode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) SetRebuildStatus(string) {} func (m *noopMetrics) SetRebuildPercent(uint32) {} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index 31dadb533..a6afed60c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -138,7 +139,7 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object WithBlobovniczaSize(100*1024*1024), WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) var dPrm common.DeletePrm diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 6b8ae4b5c..7a1de4c13 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" @@ -53,7 +54,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000), WithMoveBatchSize(3)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) obj := blobstortest.NewObject(64 * 1024) // 64KB object @@ -81,7 +82,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000), WithMoveBatchSize(3)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) metaStub := &storageIDUpdateStub{ @@ -120,7 +121,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000), WithMoveBatchSize(3)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) eg, egCtx := errgroup.WithContext(context.Background()) @@ -161,7 +162,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000), WithMoveBatchSize(50)) - require.NoError(t, b.Open(false)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) for addr, storageID := range storageIDs { diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 8f629b1de..4f3a20993 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -4,12 +4,13 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) // Storage represents key-value object storage. // It is used as a building block for a blobstor of a shard. type Storage interface { - Open(readOnly bool) error + Open(mode mode.ComponentMode) error Init() error Close() error diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 45b0c3f10..9b414a9be 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -27,15 +27,14 @@ func (b *BlobStor) Open(ctx context.Context, mode mode.Mode) error { return nil } -func (b *BlobStor) openBlobStor(ctx context.Context, mode mode.Mode) error { - readOnly := mode.ReadOnly() +func (b *BlobStor) openBlobStor(ctx context.Context, mod mode.Mode) error { for i := range b.storage { select { case <-ctx.Done(): return ctx.Err() default: } - err := b.storage[i].Storage.Open(readOnly) + err := b.storage[i].Storage.Open(mode.ConvertToComponentMode(mod)) if err != nil { return err } diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index cec531f2e..c21d79f09 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -1,13 +1,14 @@ package fstree import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" ) // Open implements common.Storage. -func (t *FSTree) Open(ro bool) error { - t.readOnly = ro - t.metrics.SetMode(ro) +func (t *FSTree) Open(mode mode.ComponentMode) error { + t.readOnly = mode.ReadOnly() + t.metrics.SetMode(mode) return nil } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_test.go b/pkg/local_object_storage/blobstor/fstree/fstree_test.go index b81ce43f1..d633cbac3 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_test.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_test.go @@ -6,6 +6,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -43,7 +44,7 @@ func TestObjectCounter(t *testing.T) { WithDepth(2), WithDirNameLen(2), WithFileCounter(counter)) - require.NoError(t, fst.Open(false)) + require.NoError(t, fst.Open(mode.ComponentReadWrite)) require.NoError(t, fst.Init()) counterValue := counter.Value() diff --git a/pkg/local_object_storage/blobstor/fstree/metrics.go b/pkg/local_object_storage/blobstor/fstree/metrics.go index eeb686672..10de935eb 100644 --- a/pkg/local_object_storage/blobstor/fstree/metrics.go +++ b/pkg/local_object_storage/blobstor/fstree/metrics.go @@ -1,11 +1,15 @@ package fstree -import "time" +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) type Metrics interface { SetParentID(parentID string) - SetMode(readOnly bool) + SetMode(mode mode.ComponentMode) Close() Iterate(d time.Duration, success bool) @@ -20,7 +24,7 @@ type Metrics interface { type noopMetrics struct{} func (m *noopMetrics) SetParentID(string) {} -func (m *noopMetrics) SetMode(bool) {} +func (m *noopMetrics) SetMode(mode.ComponentMode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) Iterate(time.Duration, bool) {} func (m *noopMetrics) Delete(time.Duration, bool) {} diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go index 96d54dec3..a3bbc021d 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go @@ -7,6 +7,7 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" ) @@ -14,13 +15,13 @@ import ( // cons must return a storage which is NOT opened. func TestControl(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) objects := prepare(t, 10, s, min, max) require.NoError(t, s.Close()) - require.NoError(t, s.Open(true)) + require.NoError(t, s.Open(mode.ComponentReadOnly)) for i := range objects { var prm common.GetPrm prm.Address = objects[i].addr diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go index f551cbf5f..750619a30 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go @@ -5,6 +5,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -12,7 +13,7 @@ import ( func TestDelete(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go index 29d8bafee..33b50b12f 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go @@ -5,13 +5,14 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) func TestExists(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go index 65c0c3133..12f73c3e9 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go @@ -5,6 +5,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -12,7 +13,7 @@ import ( func TestGet(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go index 9b99c12d5..93de683c2 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go @@ -6,6 +6,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -14,7 +15,7 @@ import ( func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index 45c68cebc..e66fe87b6 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -6,12 +6,13 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" ) func TestIterate(t *testing.T, cons Constructor, min, max uint64) { s := cons(t) - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() diff --git a/pkg/local_object_storage/blobstor/memstore/control.go b/pkg/local_object_storage/blobstor/memstore/control.go index 7f6e85a16..449d4352a 100644 --- a/pkg/local_object_storage/blobstor/memstore/control.go +++ b/pkg/local_object_storage/blobstor/memstore/control.go @@ -1,9 +1,12 @@ package memstore -import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" +) -func (s *memstoreImpl) Open(readOnly bool) error { - s.readOnly = readOnly +func (s *memstoreImpl) Open(mod mode.ComponentMode) error { + s.readOnly = mod.ReadOnly() return nil } diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index bc9436350..8d1480dff 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" @@ -18,7 +19,7 @@ func TestSimpleLifecycle(t *testing.T) { WithLogger(test.NewLogger(t)), ) defer func() { require.NoError(t, s.Close()) }() - require.NoError(t, s.Open(false)) + require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) obj := blobstortest.NewObject(1024) diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index fb5afb88b..501c95a1d 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/memstore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) @@ -22,7 +23,7 @@ type storage struct { func (s storage) open(b *testing.B) common.Storage { st := s.create(b.TempDir()) - require.NoError(b, st.Open(false)) + require.NoError(b, st.Open(mode.ComponentReadWrite)) require.NoError(b, st.Init()) return st diff --git a/pkg/local_object_storage/blobstor/teststore/option.go b/pkg/local_object_storage/blobstor/teststore/option.go index b897832cf..bc0bed49d 100644 --- a/pkg/local_object_storage/blobstor/teststore/option.go +++ b/pkg/local_object_storage/blobstor/teststore/option.go @@ -3,12 +3,13 @@ package teststore import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) type cfg struct { st common.Storage overrides struct { - Open func(readOnly bool) error + Open func(mode mode.ComponentMode) error Init func() error Close func() error @@ -35,9 +36,9 @@ func WithSubstorage(st common.Storage) Option { } } -func WithOpen(f func(bool) error) Option { return func(c *cfg) { c.overrides.Open = f } } -func WithInit(f func() error) Option { return func(c *cfg) { c.overrides.Init = f } } -func WithClose(f func() error) Option { return func(c *cfg) { c.overrides.Close = f } } +func WithOpen(f func(mode.ComponentMode) error) Option { return func(c *cfg) { c.overrides.Open = f } } +func WithInit(f func() error) Option { return func(c *cfg) { c.overrides.Init = f } } +func WithClose(f func() error) Option { return func(c *cfg) { c.overrides.Close = f } } func WithType(f func() string) Option { return func(c *cfg) { c.overrides.Type = f } } func WithPath(f func() string) Option { return func(c *cfg) { c.overrides.Path = f } } diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 016fd520f..fea4a2d49 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -20,6 +20,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) // TestStore is a common.Storage implementation for testing/mocking purposes. @@ -50,16 +51,16 @@ func (s *TestStore) SetOption(opt Option) { opt(s.cfg) } -func (s *TestStore) Open(readOnly bool) error { +func (s *TestStore) Open(mod mode.ComponentMode) error { s.mu.RLock() defer s.mu.RUnlock() switch { case s.overrides.Open != nil: - return s.overrides.Open(readOnly) + return s.overrides.Open(mod) case s.st != nil: - return s.st.Open(readOnly) + return s.st.Open(mod) default: - panic(fmt.Sprintf("unexpected storage call: Open(%v)", readOnly)) + panic(fmt.Sprintf("unexpected storage call: Open(%v)", mod.String())) } } diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 6bf6beac6..f0809883c 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -74,7 +74,7 @@ func TestInitializationFailure(t *testing.T) { openFileMetabase: os.OpenFile, openFilePilorama: os.OpenFile, }) - largeFileStorage.SetOption(teststore.WithOpen(func(ro bool) error { + largeFileStorage.SetOption(teststore.WithOpen(func(primitiveMode mode.ComponentMode) error { return teststore.ErrDiskExploded })) beforeReload := func() { diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 670cfcbf5..023576716 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -171,8 +171,8 @@ func (m *writeCacheMetrics) SetEstimateSize(db, fstree uint64) { m.metrics.SetEstimateSize(m.shardID, m.path, writecache.StorageTypeFSTree.String(), fstree) } -func (m *writeCacheMetrics) SetMode(mode mode.Mode) { - m.metrics.SetMode(m.shardID, mode.String()) +func (m *writeCacheMetrics) SetMode(mod mode.ComponentMode) { + m.metrics.SetMode(m.shardID, mod.String()) } func (m *writeCacheMetrics) SetActualCounters(db, fstree uint64) { diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index cd53f0cd2..891a1e9b2 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -38,16 +38,16 @@ var ( ) // Open boltDB instance for metabase. -func (db *DB) Open(_ context.Context, mode mode.Mode) error { +func (db *DB) Open(_ context.Context, m mode.Mode) error { db.modeMtx.Lock() defer db.modeMtx.Unlock() - db.mode = mode - db.metrics.SetMode(mode) + db.mode = m + db.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) - if mode.NoMetabase() { + if m.NoMetabase() { return nil } - return db.openDB(mode) + return db.openDB(m) } func (db *DB) openDB(mode mode.Mode) error { @@ -239,15 +239,15 @@ func (db *DB) Reload(opts ...Option) (bool, error) { return false, err } - db.mode = mode.Degraded - db.metrics.SetMode(mode.Degraded) + db.mode = mode.Disabled + db.metrics.SetMode(mode.ComponentDisabled) db.info.Path = c.info.Path if err := db.openBolt(); err != nil { return false, metaerr.Wrap(fmt.Errorf("%w: %v", ErrDegradedMode, err)) } db.mode = mode.ReadWrite - db.metrics.SetMode(mode.ReadWrite) + db.metrics.SetMode(mode.ComponentReadWrite) return true, nil } diff --git a/pkg/local_object_storage/metabase/metrics.go b/pkg/local_object_storage/metabase/metrics.go index fc971bd81..d673560c7 100644 --- a/pkg/local_object_storage/metabase/metrics.go +++ b/pkg/local_object_storage/metabase/metrics.go @@ -9,7 +9,7 @@ import ( type Metrics interface { SetParentID(parentID string) - SetMode(m mode.Mode) + SetMode(m mode.ComponentMode) Close() AddMethodDuration(method string, d time.Duration, success bool) @@ -18,6 +18,6 @@ type Metrics interface { type noopMetrics struct{} func (m *noopMetrics) SetParentID(string) {} -func (m *noopMetrics) SetMode(mode.Mode) {} +func (m *noopMetrics) SetMode(mode.ComponentMode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index b382e99c2..2032ed6b2 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -35,6 +35,6 @@ func (db *DB) SetMode(m mode.Mode) error { } db.mode = m - db.metrics.SetMode(m) + db.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) return nil } diff --git a/pkg/local_object_storage/metabase/shard_id.go b/pkg/local_object_storage/metabase/shard_id.go index 7ae336a6c..9b24e98d7 100644 --- a/pkg/local_object_storage/metabase/shard_id.go +++ b/pkg/local_object_storage/metabase/shard_id.go @@ -69,7 +69,7 @@ func (db *DB) SetShardID(id []byte, mode metamode.Mode) error { err := db.writeShardID(id) if err == nil { - db.metrics.SetMode(mode) + db.metrics.SetMode(metamode.ConvertToComponentModeDegraded(mode)) } if cErr := db.close(); cErr != nil { diff --git a/pkg/local_object_storage/metrics/blobovnicza.go b/pkg/local_object_storage/metrics/blobovnicza.go index 1e294efa5..6886bbc1d 100644 --- a/pkg/local_object_storage/metrics/blobovnicza.go +++ b/pkg/local_object_storage/metrics/blobovnicza.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) @@ -34,8 +35,8 @@ func (m *blobovniczaTreeMetrics) SetParentID(parentID string) { m.shardID = parentID } -func (m *blobovniczaTreeMetrics) SetMode(readOnly bool) { - m.m.SetBlobobvnizcaTreeMode(m.shardID, m.path, readOnly) +func (m *blobovniczaTreeMetrics) SetMode(mod mode.ComponentMode) { + m.m.SetBlobobvnizcaTreeMode(m.shardID, m.path, mod) } func (m *blobovniczaTreeMetrics) Close() { diff --git a/pkg/local_object_storage/metrics/fstree.go b/pkg/local_object_storage/metrics/fstree.go index e035b3a46..e6b3175be 100644 --- a/pkg/local_object_storage/metrics/fstree.go +++ b/pkg/local_object_storage/metrics/fstree.go @@ -4,6 +4,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) @@ -25,8 +26,8 @@ func (m *fstreeMetrics) SetParentID(parentID string) { m.shardID = parentID } -func (m *fstreeMetrics) SetMode(readOnly bool) { - m.m.SetMode(m.shardID, m.path, readOnly) +func (m *fstreeMetrics) SetMode(mod mode.ComponentMode) { + m.m.SetMode(m.shardID, m.path, mod) } func (m *fstreeMetrics) Close() { diff --git a/pkg/local_object_storage/metrics/metabase.go b/pkg/local_object_storage/metrics/metabase.go index d0fb31936..1a02f2705 100644 --- a/pkg/local_object_storage/metrics/metabase.go +++ b/pkg/local_object_storage/metrics/metabase.go @@ -26,7 +26,7 @@ func (m *metabaseMetrics) SetParentID(parentID string) { m.shardID = parentID } -func (m *metabaseMetrics) SetMode(mode mode.Mode) { +func (m *metabaseMetrics) SetMode(mode mode.ComponentMode) { m.m.SetMode(m.shardID, m.path, mode.String()) } diff --git a/pkg/local_object_storage/metrics/pilorama.go b/pkg/local_object_storage/metrics/pilorama.go index 21f027a6e..1b715c2bd 100644 --- a/pkg/local_object_storage/metrics/pilorama.go +++ b/pkg/local_object_storage/metrics/pilorama.go @@ -24,8 +24,8 @@ func (m *piloramaMetrics) SetParentID(id string) { m.shardID = id } -func (m *piloramaMetrics) SetMode(mode mode.Mode) { - m.m.SetMode(m.shardID, mode) +func (m *piloramaMetrics) SetMode(mod mode.ComponentMode) { + m.m.SetMode(m.shardID, mod) } func (m *piloramaMetrics) Close() { diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 29a9306b6..fe6bf4fed 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -109,7 +109,7 @@ func (t *boltForest) SetMode(m mode.Mode) error { } t.mode = m - t.metrics.SetMode(m) + t.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) return nil } @@ -123,8 +123,8 @@ func (t *boltForest) Open(_ context.Context, mode mode.Mode) error { return t.openBolt(mode) } -func (t *boltForest) openBolt(mode mode.Mode) error { - readOnly := mode.ReadOnly() +func (t *boltForest) openBolt(m mode.Mode) error { + readOnly := m.ReadOnly() err := util.MkdirAllX(filepath.Dir(t.path), t.perm) if err != nil { return metaerr.Wrap(fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)) @@ -143,7 +143,7 @@ func (t *boltForest) openBolt(mode mode.Mode) error { t.db.MaxBatchSize = t.maxBatchSize t.db.MaxBatchDelay = t.maxBatchDelay - t.metrics.SetMode(mode) + t.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) return nil } diff --git a/pkg/local_object_storage/pilorama/metrics.go b/pkg/local_object_storage/pilorama/metrics.go index 543ad3e31..6ffc479e4 100644 --- a/pkg/local_object_storage/pilorama/metrics.go +++ b/pkg/local_object_storage/pilorama/metrics.go @@ -9,7 +9,7 @@ import ( type Metrics interface { SetParentID(id string) - SetMode(m mode.Mode) + SetMode(m mode.ComponentMode) Close() AddMethodDuration(method string, d time.Duration, success bool) @@ -18,6 +18,6 @@ type Metrics interface { type noopMetrics struct{} func (m *noopMetrics) SetParentID(string) {} -func (m *noopMetrics) SetMode(mode.Mode) {} +func (m *noopMetrics) SetMode(mode.ComponentMode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) AddMethodDuration(string, time.Duration, bool) {} diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 6712822a0..38a29cb12 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -37,7 +37,7 @@ func (s *Shard) handleMetabaseFailure(stage string, err error) error { err = s.SetMode(mode.DegradedReadOnly) if err != nil { - return fmt.Errorf("could not switch to mode %s", mode.DegradedReadOnly) + return fmt.Errorf("could not switch to mode %s", mode.Mode(mode.DegradedReadOnly)) } return nil } diff --git a/pkg/local_object_storage/shard/mode/mode.go b/pkg/local_object_storage/shard/mode/mode.go index 461611465..dc4d52b0e 100644 --- a/pkg/local_object_storage/shard/mode/mode.go +++ b/pkg/local_object_storage/shard/mode/mode.go @@ -29,6 +29,22 @@ const ( DegradedReadOnly Mode = Degraded | ReadOnly ) +// ComponentMode represents basic operation modes for shared components, including READ, READ_WRITE, and DISABLED. +type ComponentMode uint32 + +const ( + // ComponentReadWrite is a Mode value for component that is available + // for read and write operations. Default component mode. + ComponentReadWrite ComponentMode = 0 + + // ComponentReadOnly is a Mode value for component that does not + // accept write operation but is readable. + ComponentReadOnly ComponentMode = 0b001 + + // ComponentDisabled mode is a mode where a component is disabled. + ComponentDisabled ComponentMode = math.MaxUint32 +) + func (m Mode) String() string { switch m { default: @@ -46,6 +62,19 @@ func (m Mode) String() string { } } +func (m ComponentMode) String() string { + switch m { + default: + return "UNDEFINED" + case ComponentReadWrite: + return "READ_WRITE" + case ComponentReadOnly: + return "READ_ONLY" + case ComponentDisabled: + return "CLOSED" + } +} + // NoMetabase returns true iff m is operating without the metabase. func (m Mode) NoMetabase() bool { return m&Degraded != 0 @@ -56,6 +85,39 @@ func (m Mode) ReadOnly() bool { return m&ReadOnly != 0 } +// ReadOnly returns true iff m prohibits modifying operations with shard. +func (m ComponentMode) ReadOnly() bool { + return m&ComponentReadOnly != 0 +} + func (m Mode) Disabled() bool { return m == Disabled } + +func (m ComponentMode) Disabled() bool { + return m == ComponentDisabled +} + +// ConvertToComponentModeDegraded converts a ShardMode to a corresponding ComponentMode. +// Disables the component if the node is in degraded mode. Used in Metabase, Writecache, Pilorama. +func ConvertToComponentModeDegraded(m Mode) ComponentMode { + if m.NoMetabase() || m.Disabled() { + return ComponentDisabled + } + if m.ReadOnly() { + return ComponentReadOnly + } + return ComponentReadWrite +} + +// ConvertToComponentMode converts a ShardMode to a corresponding ComponentMode. +// Ignores the degraded mode of the node. Used in Blobstore. +func ConvertToComponentMode(m Mode) ComponentMode { + if m.Disabled() { + return ComponentDisabled + } + if m.ReadOnly() { + return ComponentReadOnly + } + return ComponentReadWrite +} diff --git a/pkg/local_object_storage/writecache/cachebbolt.go b/pkg/local_object_storage/writecache/cachebbolt.go index fdba8d409..f2b48d746 100644 --- a/pkg/local_object_storage/writecache/cachebbolt.go +++ b/pkg/local_object_storage/writecache/cachebbolt.go @@ -95,14 +95,14 @@ func (c *cache) DumpInfo() Info { } // Open opens and initializes database. Reads object counters from the ObjectCounters instance. -func (c *cache) Open(_ context.Context, mode mode.Mode) error { +func (c *cache) Open(_ context.Context, mod mode.Mode) error { c.modeMtx.Lock() defer c.modeMtx.Unlock() - c.mode = mode - if mode.NoMetabase() { + c.mode = mod + if mod.NoMetabase() { return nil } - err := c.openStore(mode.ReadOnly()) + err := c.openStore(mode.ConvertToComponentModeDegraded(mod)) if err != nil { return metaerr.Wrap(err) } @@ -112,7 +112,7 @@ func (c *cache) Open(_ context.Context, mode mode.Mode) error { // Init runs necessary services. func (c *cache) Init() error { - c.metrics.SetMode(c.mode) + c.metrics.SetMode(mode.ConvertToComponentModeDegraded(c.mode)) ctx, cancel := context.WithCancel(context.Background()) c.cancel = cancel c.runFlushLoop(ctx) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index c3e19c4d2..da7feda9a 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -294,7 +294,7 @@ func (c *cache) Flush(ctx context.Context, ignoreErrors, seal bool) error { if err := c.setMode(ctx, m, ignoreErrors); err != nil { return err } - c.metrics.SetMode(m) + c.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) } return nil } diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go index 962e22eea..e68b6d8be 100644 --- a/pkg/local_object_storage/writecache/metrics.go +++ b/pkg/local_object_storage/writecache/metrics.go @@ -27,7 +27,7 @@ type Metrics interface { Evict(st StorageType) SetEstimateSize(db, fstree uint64) - SetMode(m mode.Mode) + SetMode(m mode.ComponentMode) SetActualCounters(db, fstree uint64) SetPath(path string) Close() @@ -49,7 +49,7 @@ func (metricsStub) Put(time.Duration, bool, StorageType) {} func (metricsStub) SetEstimateSize(uint64, uint64) {} -func (metricsStub) SetMode(mode.Mode) {} +func (metricsStub) SetMode(mode.ComponentMode) {} func (metricsStub) SetActualCounters(uint64, uint64) {} diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index a10e593aa..4172cfbc8 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -27,7 +27,7 @@ func (c *cache) SetMode(m mode.Mode) error { err := c.setMode(ctx, m, true) if err == nil { - c.metrics.SetMode(m) + c.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) } return err } @@ -63,7 +63,7 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode, ignoreErrors bool) err return nil } - if err = c.openStore(m.ReadOnly()); err != nil { + if err = c.openStore(mode.ConvertToComponentModeDegraded(m)); err != nil { return err } diff --git a/pkg/local_object_storage/writecache/seal.go b/pkg/local_object_storage/writecache/seal.go index be4fec367..48107a75f 100644 --- a/pkg/local_object_storage/writecache/seal.go +++ b/pkg/local_object_storage/writecache/seal.go @@ -22,7 +22,7 @@ func (c *cache) Seal(ctx context.Context, ignoreErrors bool) error { // flush will be done by setMode err := c.setMode(ctx, mode.DegradedReadOnly, ignoreErrors) if err == nil { - c.metrics.SetMode(mode.DegradedReadOnly) + c.metrics.SetMode(mode.ComponentDisabled) } return err } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index 6cc3b06d0..caf997af8 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -25,13 +26,13 @@ type store struct { const dbName = "small.bolt" -func (c *cache) openStore(readOnly bool) error { +func (c *cache) openStore(mod mode.ComponentMode) error { err := util.MkdirAllX(c.path, os.ModePerm) if err != nil { return err } - c.db, err = OpenDB(c.path, readOnly, c.openFile) + c.db, err = OpenDB(c.path, mod.ReadOnly(), c.openFile) if err != nil { return fmt.Errorf("could not open database: %w", err) } @@ -39,7 +40,7 @@ func (c *cache) openStore(readOnly bool) error { c.db.MaxBatchSize = c.maxBatchSize c.db.MaxBatchDelay = c.maxBatchDelay - if !readOnly { + if !mod.ReadOnly() { err = c.db.Update(func(tx *bbolt.Tx) error { _, err := tx.CreateBucketIfNotExists(defaultBucket) return err @@ -57,7 +58,7 @@ func (c *cache) openStore(readOnly bool) error { fstree.WithNoSync(c.noSync), fstree.WithFileCounter(&c.objCounters), ) - if err := c.fsTree.Open(readOnly); err != nil { + if err := c.fsTree.Open(mod); err != nil { return fmt.Errorf("could not open FSTree: %w", err) } if err := c.fsTree.Init(); err != nil { diff --git a/pkg/metrics/blobovnicza.go b/pkg/metrics/blobovnicza.go index f3a14a96e..948272c88 100644 --- a/pkg/metrics/blobovnicza.go +++ b/pkg/metrics/blobovnicza.go @@ -4,12 +4,13 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) type BlobobvnizcaMetrics interface { - SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) + SetBlobobvnizcaTreeMode(shardID, path string, mode mode.ComponentMode) CloseBlobobvnizcaTree(shardID, path string) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) AddBlobobvnizcaTreePut(shardID, path string, size int) @@ -98,8 +99,8 @@ func newBlobovnicza() *blobovnicza { } } -func (b *blobovnicza) SetBlobobvnizcaTreeMode(shardID, path string, readOnly bool) { - b.treeMode.SetMode(shardID, path, modeFromBool(readOnly)) +func (b *blobovnicza) SetBlobobvnizcaTreeMode(shardID, path string, mod mode.ComponentMode) { + b.treeMode.SetMode(shardID, path, mod.String()) } func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) { diff --git a/pkg/metrics/fstree.go b/pkg/metrics/fstree.go index 4d4f0693b..ecd4352bb 100644 --- a/pkg/metrics/fstree.go +++ b/pkg/metrics/fstree.go @@ -4,12 +4,13 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) type FSTreeMetrics interface { - SetMode(shardID, path string, readOnly bool) + SetMode(shardID, path string, mode mode.ComponentMode) Close(shardID, path string) MethodDuration(shardID, path string, method string, d time.Duration, success bool) @@ -48,8 +49,8 @@ func newFSTreeMetrics() *fstreeMetrics { } } -func (m *fstreeMetrics) SetMode(shardID, path string, readOnly bool) { - m.mode.SetMode(shardID, path, modeFromBool(readOnly)) +func (m *fstreeMetrics) SetMode(shardID, path string, mod mode.ComponentMode) { + m.mode.SetMode(shardID, path, mod.String()) } func (m *fstreeMetrics) Close(shardID, path string) { diff --git a/pkg/metrics/pilorama.go b/pkg/metrics/pilorama.go index 41672a4b5..c669275fe 100644 --- a/pkg/metrics/pilorama.go +++ b/pkg/metrics/pilorama.go @@ -10,7 +10,7 @@ import ( ) type PiloramaMetrics interface { - SetMode(shardID string, m mode.Mode) + SetMode(shardID string, m mode.ComponentMode) Close(shardID string) AddMethodDuration(shardID string, method string, d time.Duration, success bool) @@ -33,7 +33,7 @@ type piloramaMetrics struct { reqDuration *prometheus.HistogramVec } -func (m *piloramaMetrics) SetMode(shardID string, mode mode.Mode) { +func (m *piloramaMetrics) SetMode(shardID string, mode mode.ComponentMode) { m.mode.SetMode(shardID, mode.String()) } From 8fcd0f8f8d40a0e882cf341384d5987c455e2e47 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 4 Jun 2024 16:29:27 +0300 Subject: [PATCH 1136/1943] [#1121] docs: Change mode of shard components Signed-off-by: Alexander Chuprov --- docs/shard-modes.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/shard-modes.md b/docs/shard-modes.md index 789a1c241..3b459335b 100644 --- a/docs/shard-modes.md +++ b/docs/shard-modes.md @@ -14,7 +14,16 @@ Each mode is characterized by two important properties: | `read-only` | Read-only mode, only read operations are allowed, metabase is available. | | `degraded` | Degraded mode in which metabase and write-cache is disabled. It shouldn't be used at all, because metabase can contain important indices, such as LOCK objects info and modifying operation in this mode can lead to unexpected behaviour. The purpose of this mode is to allow PUT/DELETE operations without the metabase if really necessary. | | `degraded-read-only` | Same as `degraded`, but with only read operations allowed. This mode is used during SSD replacement and/or when the metabase error counter exceeds threshold. | -| `disabled` | Currently used only in config file to temporarily disable a shard. | +| `disabled` | Currently used only in config file to temporarily disable a shard. + +## Shard and Component Status + +| Shard Mode | Metabase Mode | Blobstore Mode | Writecache Mode | Pilorama Mode | Blobovnicza Tree Mode | FSTree Mode | +|-----------------------|---------------|----------------|-----------------|---------------|-----------------------|-------------| +| `Read-Write` | READ_WRITE | READ_WRITE | READ_WRITE | READ_WRITE | READ_WRITE | READ_WRITE | +| `Read-Only` | READ_ONLY | READ_ONLY | READ_ONLY | READ_ONLY | READ_ONLY | READ_ONLY | +| `Degraded-Read-Write` | CLOSED | READ_WRITE | CLOSED | CLOSED | READ_WRITE | READ_WRITE | +| `Degraded-Read-Only` | CLOSED | READ_ONLY | CLOSED | CLOSED | READ_ONLY | READ_ONLY | ## Transition order From 2e074d3846cc41c234b0165c21221b6aadf98e5b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 5 Jun 2024 11:14:06 +0300 Subject: [PATCH 1137/1943] [#1163] metabase: Properly save EC parent split ID Search by SplitID should return all parts of a complex object. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/put.go | 12 ++ .../metabase/select_test.go | 103 ++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 4b6359ebf..05922d1bd 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -347,6 +347,18 @@ func updateListIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun if err != nil { return err } + + if ech.ParentSplitID() != nil { + objKey := objectKey(ech.Parent(), make([]byte, objectKeySize)) + err := f(tx, namedBucketItem{ + name: splitBucketName(cnr, bucketName), + key: ech.ParentSplitID().ToV2(), + val: objKey, + }) + if err != nil { + return err + } + } } return nil diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 9ff2fd65e..8951a2dbe 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -3,6 +3,7 @@ package meta_test import ( "context" "encoding/hex" + "math/rand" "strconv" "testing" @@ -13,9 +14,13 @@ import ( cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" ) @@ -628,6 +633,104 @@ func TestDB_SelectObjectID(t *testing.T) { }) } +type testTarget struct { + objects []*objectSDK.Object +} + +func (tt *testTarget) WriteObject(_ context.Context, obj *objectSDK.Object) error { + tt.objects = append(tt.objects, obj) + return nil +} + +func cutObject(t *testing.T, p transformer.ChunkedObjectWriter, hdr *objectSDK.Object, size int) { + ctx := context.Background() + require.NoError(t, p.WriteHeader(ctx, hdr)) + + payload := make([]byte, size) + rand.New(rand.NewSource(0)).Read(payload) + + _, err := p.Write(ctx, payload) + require.NoError(t, err) + + _, err = p.Close(ctx) + require.NoError(t, err) +} + +func TestDB_SelectSplitID_EC(t *testing.T) { + t.Parallel() + + const ( + partSize = 10 + partCount = 2 + dataCount = 2 + parityCount = 1 + ) + + db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() + + cnr := cidtest.ID() + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + tt := new(testTarget) + p := transformer.NewPayloadSizeLimiter(transformer.Params{ + Key: &pk.PrivateKey, + NextTargetInit: func() transformer.ObjectWriter { return tt }, + NetworkState: epochState{e: 1}, + MaxSize: partSize, + }) + + hdr := objectSDK.New() + hdr.SetContainerID(cnr) + hdr.SetOwnerID(usertest.ID()) + cutObject(t, p, hdr, partSize*partCount) + require.Equal(t, len(tt.objects), partCount+1) + + split := tt.objects[0].SplitID() + require.NotNil(t, split) + + ec, err := erasurecode.NewConstructor(dataCount, parityCount) + require.NoError(t, err) + + for i := 0; i < partCount; i++ { + cs, err := ec.Split(tt.objects[i], &pk.PrivateKey) + require.NoError(t, err) + + require.NoError(t, putBig(db, cs[0])) + } + + t.Run("not present", func(t *testing.T) { + fs := objectSDK.SearchFilters{} + fs.AddFilter(v2object.FilterHeaderSplitID, "", objectSDK.MatchNotPresent) + testSelect(t, db, cnr, fs) + }) + + t.Run("split id", func(t *testing.T) { + fs := objectSDK.SearchFilters{} + fs.AddFilter(v2object.FilterHeaderSplitID, split.String(), objectSDK.MatchStringEqual) + testSelect(t, db, cnr, fs, + object.AddressOf(tt.objects[0]), + object.AddressOf(tt.objects[1]), + ) + }) + + t.Run("empty split", func(t *testing.T) { + fs := objectSDK.SearchFilters{} + fs.AddFilter(v2object.FilterHeaderSplitID, "", objectSDK.MatchStringEqual) + testSelect(t, db, cnr, fs) + }) + + t.Run("unknown split id", func(t *testing.T) { + fs := objectSDK.SearchFilters{} + fs.AddFilter(v2object.FilterHeaderSplitID, + objectSDK.NewSplitID().String(), + objectSDK.MatchStringEqual) + testSelect(t, db, cnr, fs) + }) +} + func TestDB_SelectSplitID(t *testing.T) { t.Parallel() From 3f1961157ec794af9a36df858f2cb32f2a42907f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 6 Jun 2024 10:52:17 +0300 Subject: [PATCH 1138/1943] [#1163] metabase: Handle multiple splitInfos for EC For REP updating split info is handled explicitly by a high-level PUT logic. For EC it is trickier, because the address of an object we put is only distantly related to a split info. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/put.go | 115 +++++++--------- .../metabase/select_test.go | 124 +++++++++++++++++- 2 files changed, 168 insertions(+), 71 deletions(-) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 05922d1bd..ceb79758f 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -244,52 +244,49 @@ func putUniqueIndexes( // index root object if obj.Type() == objectSDK.TypeRegular && !obj.HasParent() { - var ( - err error - splitInfo []byte - ) - - if isParent { - splitInfo, err = si.Marshal() - if err != nil { - return fmt.Errorf("can't marshal split info: %w", err) - } - } - - isObjKeySet := true if ecHead := obj.ECHeader(); ecHead != nil { - objKey, isObjKeySet = objectKeyByECHeader(ecHead) - } - if isObjKeySet { - err = putUniqueIndexItem(tx, namedBucketItem{ - name: rootBucketName(cnr, bucketName), - key: objKey, - val: splitInfo, - }) - if err != nil { - return err + parentID := ecHead.Parent() + if ecHead.ParentSplitID() != nil { + parentSplitParentID := ecHead.ParentSplitParentID() + if parentSplitParentID == nil { + return nil + } + + si = objectSDK.NewSplitInfo() + si.SetSplitID(ecHead.ParentSplitID()) + si.SetLastPart(ecHead.Parent()) + + parentID = *parentSplitParentID } + objKey = objectKey(parentID, objKey) } + return updateSplitInfoIndex(tx, objKey, cnr, bucketName, si) } return nil } -// objectKeyByECHeader returns objectKey for an object that has EC Header. -// If object's parent is in Split, then parent's non-nil Split parent ID is set to object key. -// If object's parent is not in Split, then its ID is set to object key. -// Otherwise, such object keys should be ignored -- they are not put to the root bucket. -func objectKeyByECHeader(ech *objectSDK.ECHeader) (objKey []byte, isSet bool) { - if ech.ParentSplitID() != nil { - if parentSplitParentID := ech.ParentSplitParentID(); parentSplitParentID != nil { - isSet = true - objKey = objectKey(*parentSplitParentID, make([]byte, objectKeySize)) +func updateSplitInfoIndex(tx *bbolt.Tx, objKey []byte, cnr cid.ID, bucketName []byte, si *objectSDK.SplitInfo) error { + return updateUniqueIndexItem(tx, namedBucketItem{ + name: rootBucketName(cnr, bucketName), + key: objKey, + }, func(old, _ []byte) ([]byte, error) { + switch { + case si == nil && old == nil: + return []byte{}, nil + case si == nil: + return old, nil + case old == nil: + return si.Marshal() + default: + oldSI := objectSDK.NewSplitInfo() + if err := oldSI.Unmarshal(old); err != nil { + return nil, err + } + si = util.MergeSplitInfo(si, oldSI) + return si.Marshal() } - return - } - isSet = true - objKey = objectKey(ech.Parent(), make([]byte, objectKeySize)) - return + }) } type updateIndexItemFunc = func(tx *bbolt.Tx, item namedBucketItem) error @@ -416,13 +413,21 @@ func createBucketLikelyExists[T bucketContainer](tx T, name []byte) (*bbolt.Buck return tx.CreateBucket(name) } -func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error { +func updateUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem, update func(oldData, newData []byte) ([]byte, error)) error { bkt, err := createBucketLikelyExists(tx, item.name) if err != nil { return fmt.Errorf("can't create index %v: %w", item.name, err) } - return bkt.Put(item.key, item.val) + data, err := update(bkt.Get(item.key), item.val) + if err != nil { + return err + } + return bkt.Put(item.key, data) +} + +func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error { + return updateUniqueIndexItem(tx, item, func(_, val []byte) ([]byte, error) { return val, nil }) } func putFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { @@ -548,36 +553,8 @@ func setStorageID(tx *bbolt.Tx, addr oid.Address, id []byte, override bool) erro // updateSpliInfo for existing objects if storage filled with extra information // about last object in split hierarchy or linking object. func updateSplitInfo(tx *bbolt.Tx, addr oid.Address, from *objectSDK.SplitInfo) error { - key := make([]byte, bucketKeySize) - bkt := tx.Bucket(rootBucketName(addr.Container(), key)) - if bkt == nil { - // if object doesn't exists and we want to update split info on it - // then ignore, this should never happen - return ErrIncorrectSplitInfoUpdate - } - - objectKey := objectKey(addr.Object(), key) - - rawSplitInfo := bkt.Get(objectKey) - if len(rawSplitInfo) == 0 { - return ErrIncorrectSplitInfoUpdate - } - - to := objectSDK.NewSplitInfo() - - err := to.Unmarshal(rawSplitInfo) - if err != nil { - return fmt.Errorf("can't unmarshal split info from root index: %w", err) - } - - result := util.MergeSplitInfo(from, to) - - rawSplitInfo, err = result.Marshal() - if err != nil { - return fmt.Errorf("can't marhsal merged split info: %w", err) - } - - return bkt.Put(objectKey, rawSplitInfo) + objKey := objectKey(addr.Object(), make([]byte, bucketKeySize)) + return updateSplitInfoIndex(tx, objKey, addr.Container(), make([]byte, bucketKeySize), from) } // splitInfoFromObject returns split info based on last or linkin object. diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 8951a2dbe..8f9294d07 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -642,7 +642,7 @@ func (tt *testTarget) WriteObject(_ context.Context, obj *objectSDK.Object) erro return nil } -func cutObject(t *testing.T, p transformer.ChunkedObjectWriter, hdr *objectSDK.Object, size int) { +func cutObject(t *testing.T, p transformer.ChunkedObjectWriter, hdr *objectSDK.Object, size int) *transformer.AccessIdentifiers { ctx := context.Background() require.NoError(t, p.WriteHeader(ctx, hdr)) @@ -652,8 +652,128 @@ func cutObject(t *testing.T, p transformer.ChunkedObjectWriter, hdr *objectSDK.O _, err := p.Write(ctx, payload) require.NoError(t, err) - _, err = p.Close(ctx) + ids, err := p.Close(ctx) require.NoError(t, err) + return ids +} + +func TestDB_RawHead_SplitInfo(t *testing.T) { + t.Parallel() + + const ( + partSize = 10 + partCount = 2 + dataCount = 2 + parityCount = 1 + ) + + db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() + + cnr := cidtest.ID() + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + tt := new(testTarget) + p := transformer.NewPayloadSizeLimiter(transformer.Params{ + Key: &pk.PrivateKey, + NextTargetInit: func() transformer.ObjectWriter { return tt }, + NetworkState: epochState{e: 1}, + MaxSize: partSize, + }) + + hdr := objectSDK.New() + hdr.SetContainerID(cnr) + hdr.SetOwnerID(usertest.ID()) + ids := cutObject(t, p, hdr, partSize*partCount) + require.Equal(t, len(tt.objects), partCount+1) + + t.Run("rep", func(t *testing.T) { + testGetRawSplitInfo(t, cnr, ids, tt.objects[partCount], tt.objects[partCount-1]) + }) + t.Run("with ec", func(t *testing.T) { + ec, err := erasurecode.NewConstructor(dataCount, parityCount) + require.NoError(t, err) + + cs, err := ec.Split(tt.objects[partCount-1], &pk.PrivateKey) + require.NoError(t, err) + + testGetRawSplitInfo(t, cnr, ids, tt.objects[partCount], cs[0]) + }) +} + +func testGetRawSplitInfo(t *testing.T, cnr cidSDK.ID, ids *transformer.AccessIdentifiers, linking, lastPart *objectSDK.Object) { + expectedLinkID, ok := linking.ID() + require.True(t, ok) + + t.Run("first last, then linking", func(t *testing.T) { + db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() + + require.NoError(t, metaPut(db, lastPart, nil)) + require.NoError(t, metaPut(db, linking, nil)) + + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(*ids.ParentID) + + _, err := metaGet(db, addr, true) + + var siErr *objectSDK.SplitInfoError + require.ErrorAs(t, err, &siErr) + + lastID, ok := siErr.SplitInfo().LastPart() + require.True(t, ok) + require.Equal(t, ids.SelfID, lastID) + + linkID, ok := siErr.SplitInfo().Link() + require.True(t, ok) + require.Equal(t, expectedLinkID, linkID) + }) + t.Run("first linking, then last", func(t *testing.T) { + db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() + + require.NoError(t, metaPut(db, linking, nil)) + require.NoError(t, metaPut(db, lastPart, nil)) + + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(*ids.ParentID) + + _, err := metaGet(db, addr, true) + + var siErr *objectSDK.SplitInfoError + require.ErrorAs(t, err, &siErr) + + lastID, ok := siErr.SplitInfo().LastPart() + require.True(t, ok) + require.Equal(t, ids.SelfID, lastID) + + linkID, ok := siErr.SplitInfo().Link() + require.True(t, ok) + require.Equal(t, expectedLinkID, linkID) + }) + t.Run("only last part", func(t *testing.T) { + db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() + + require.NoError(t, metaPut(db, lastPart, nil)) + + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(*ids.ParentID) + + _, err := metaGet(db, addr, true) + + var siErr *objectSDK.SplitInfoError + require.ErrorAs(t, err, &siErr) + + lastPart, ok := siErr.SplitInfo().LastPart() + require.True(t, ok) + require.Equal(t, ids.SelfID, lastPart) + }) } func TestDB_SelectSplitID_EC(t *testing.T) { From 67b3002743f056ab20abb0b5b59d520e6fe2a286 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 6 Jun 2024 15:25:51 +0300 Subject: [PATCH 1139/1943] [#951] adm: Check for error when reading contracts from archive Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/internal/modules/morph/helper/util.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/util.go b/cmd/frostfs-adm/internal/modules/morph/helper/util.go index f0399ab37..ba557a033 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/util.go @@ -122,11 +122,11 @@ func readContractsFromArchive(file io.Reader, names []string) (map[string]*Contr } r := tar.NewReader(gr) - for h, err := r.Next(); ; h, err = r.Next() { - if err != nil { - break + var h *tar.Header + for h, err = r.Next(); err == nil && h != nil; h, err = r.Next() { + if h.Typeflag != tar.TypeReg { + continue } - dir, _ := filepath.Split(h.Name) ctrName := filepath.Base(dir) @@ -149,6 +149,9 @@ func readContractsFromArchive(file io.Reader, names []string) (map[string]*Contr } m[ctrName] = cs } + if err != nil && err != io.EOF { + return nil, fmt.Errorf("can't read contracts from archive: %w", err) + } for ctrName, cs := range m { if cs.RawNEF == nil { From a849236b687e4cd9fe86960102da5e04de147b0f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 4 Jun 2024 15:31:47 +0300 Subject: [PATCH 1140/1943] [#1161] node: Remove notification functionality It is unused and will be reworked in future. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config.go | 7 - cmd/frostfs-node/config/node/config.go | 80 --------- cmd/frostfs-node/config/node/config_test.go | 29 ---- cmd/frostfs-node/main.go | 2 - cmd/frostfs-node/notificator.go | 172 -------------------- cmd/frostfs-node/object.go | 51 ------ config/example/node.env | 7 - config/example/node.json | 9 - config/example/node.yaml | 8 - docs/storage-node-configuration.md | 23 --- go.mod | 3 - go.sum | 6 - internal/logs/logs.go | 4 - pkg/services/notificator/deps.go | 22 --- pkg/services/notificator/nats/options.go | 38 ----- pkg/services/notificator/nats/service.go | 129 --------------- pkg/services/notificator/service.go | 88 ---------- 17 files changed, 678 deletions(-) delete mode 100644 cmd/frostfs-node/notificator.go delete mode 100644 pkg/services/notificator/deps.go delete mode 100644 pkg/services/notificator/nats/options.go delete mode 100644 pkg/services/notificator/nats/service.go delete mode 100644 pkg/services/notificator/service.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d713cf2fb..cdfa2118c 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -474,7 +474,6 @@ type cfg struct { cfgNetmap cfgNetmap cfgControlService cfgControlService cfgObject cfgObject - cfgNotifications cfgNotifications } // ReadCurrentNetMap reads network map which has been cached at the @@ -633,12 +632,6 @@ type cfgObject struct { skipSessionTokenIssuerVerification bool } -type cfgNotifications struct { - enabled bool - nw notificationWriter - defaultTopic string -} - type cfgLocalStorage struct { localStorage *engine.StorageEngine } diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index 90338556e..97aca274a 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -5,7 +5,6 @@ import ( "io/fs" "os" "strconv" - "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" @@ -25,12 +24,6 @@ type PersistentStateConfig struct { cfg *config.Config } -// NotificationConfig is a wrapper over "notification" config section -// which provides access to object notification configuration of node. -type NotificationConfig struct { - cfg *config.Config -} - // PersistentPolicyRulesConfig is a wrapper over "persistent_policy_rules" config section // which provides access to persistent policy rules storage configuration of node. type PersistentPolicyRulesConfig struct { @@ -41,16 +34,12 @@ const ( subsection = "node" persistentSessionsSubsection = "persistent_sessions" persistentStateSubsection = "persistent_state" - notificationSubsection = "notification" persistentPolicyRulesSubsection = "persistent_policy_rules" attributePrefix = "attribute" // PersistentStatePathDefault is a default path for persistent state file. PersistentStatePathDefault = ".frostfs-storage-state" - - // NotificationTimeoutDefault is a default timeout for object notification operation. - NotificationTimeoutDefault = 5 * time.Second ) // Key returns the value of "key" config parameter @@ -185,75 +174,6 @@ func (p PersistentStateConfig) Path() string { return PersistentStatePathDefault } -// Notification returns structure that provides access to "notification" -// subsection of "node" section. -func Notification(c *config.Config) NotificationConfig { - return NotificationConfig{ - c.Sub(subsection).Sub(notificationSubsection), - } -} - -// Enabled returns the value of "enabled" config parameter from "notification" -// subsection of "node" section. -// -// Returns false if the value is not presented. -func (n NotificationConfig) Enabled() bool { - return config.BoolSafe(n.cfg, "enabled") -} - -// DefaultTopic returns the value of "default_topic" config parameter from -// "notification" subsection of "node" section. -// -// Returns empty string if the value is not presented. -func (n NotificationConfig) DefaultTopic() string { - return config.StringSafe(n.cfg, "default_topic") -} - -// Endpoint returns the value of "endpoint" config parameter from "notification" -// subsection of "node" section. -// -// Returns empty string if the value is not presented. -func (n NotificationConfig) Endpoint() string { - return config.StringSafe(n.cfg, "endpoint") -} - -// Timeout returns the value of "timeout" config parameter from "notification" -// subsection of "node" section. -// -// Returns NotificationTimeoutDefault if the value is not positive. -func (n NotificationConfig) Timeout() time.Duration { - v := config.DurationSafe(n.cfg, "timeout") - if v > 0 { - return v - } - - return NotificationTimeoutDefault -} - -// CertPath returns the value of "certificate_path" config parameter from "notification" -// subsection of "node" section. -// -// Returns empty string if the value is not presented. -func (n NotificationConfig) CertPath() string { - return config.StringSafe(n.cfg, "certificate") -} - -// KeyPath returns the value of "key_path" config parameter from -// "notification" subsection of "node" section. -// -// Returns empty string if the value is not presented. -func (n NotificationConfig) KeyPath() string { - return config.StringSafe(n.cfg, "key") -} - -// CAPath returns the value of "ca_path" config parameter from -// "notification" subsection of "node" section. -// -// Returns empty string if the value is not presented. -func (n NotificationConfig) CAPath() string { - return config.StringSafe(n.cfg, "ca") -} - const ( // PermDefault is a default permission bits for local override storage file. PermDefault = 0o644 diff --git a/cmd/frostfs-node/config/node/config_test.go b/cmd/frostfs-node/config/node/config_test.go index b0041c870..7b9adecf4 100644 --- a/cmd/frostfs-node/config/node/config_test.go +++ b/cmd/frostfs-node/config/node/config_test.go @@ -2,7 +2,6 @@ package nodeconfig import ( "testing" - "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" @@ -33,25 +32,11 @@ func TestNodeSection(t *testing.T) { relay := Relay(empty) persisessionsPath := PersistentSessions(empty).Path() persistatePath := PersistentState(empty).Path() - notificationDefaultEnabled := Notification(empty).Enabled() - notificationDefaultEndpoint := Notification(empty).Endpoint() - notificationDefaultTimeout := Notification(empty).Timeout() - notificationDefaultTopic := Notification(empty).DefaultTopic() - notificationDefaultCertPath := Notification(empty).CertPath() - notificationDefaultKeyPath := Notification(empty).KeyPath() - notificationDefaultCAPath := Notification(empty).CAPath() require.Empty(t, attribute) require.Equal(t, false, relay) require.Equal(t, "", persisessionsPath) require.Equal(t, PersistentStatePathDefault, persistatePath) - require.Equal(t, false, notificationDefaultEnabled) - require.Equal(t, "", notificationDefaultEndpoint) - require.Equal(t, NotificationTimeoutDefault, notificationDefaultTimeout) - require.Equal(t, "", notificationDefaultTopic) - require.Equal(t, "", notificationDefaultCertPath) - require.Equal(t, "", notificationDefaultKeyPath) - require.Equal(t, "", notificationDefaultCAPath) }) const path = "../../../../config/example/node" @@ -64,13 +49,6 @@ func TestNodeSection(t *testing.T) { wKey := Wallet(c) persisessionsPath := PersistentSessions(c).Path() persistatePath := PersistentState(c).Path() - notificationEnabled := Notification(c).Enabled() - notificationEndpoint := Notification(c).Endpoint() - notificationTimeout := Notification(c).Timeout() - notificationDefaultTopic := Notification(c).DefaultTopic() - notificationCertPath := Notification(c).CertPath() - notificationKeyPath := Notification(c).KeyPath() - notificationCAPath := Notification(c).CAPath() expectedAddr := []struct { str string @@ -122,13 +100,6 @@ func TestNodeSection(t *testing.T) { require.Equal(t, "/sessions", persisessionsPath) require.Equal(t, "/state", persistatePath) - require.Equal(t, true, notificationEnabled) - require.Equal(t, "tls://localhost:4222", notificationEndpoint) - require.Equal(t, 6*time.Second, notificationTimeout) - require.Equal(t, "topic", notificationDefaultTopic) - require.Equal(t, "/cert/path", notificationCertPath) - require.Equal(t, "/key/path", notificationKeyPath) - require.Equal(t, "/ca/path", notificationCAPath) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index dbef1e494..e4f0a434c 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -111,7 +111,6 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(c, "accounting", func(c *cfg) { initAccountingService(ctx, c) }) initAndLog(c, "container", func(c *cfg) { initContainerService(ctx, c) }) initAndLog(c, "session", initSessionService) - initAndLog(c, "notification", func(c *cfg) { initNotifications(ctx, c) }) initAndLog(c, "object", initObjectService) initAndLog(c, "tree", initTreeService) initAndLog(c, "apemanager", initAPEManagerService) @@ -143,7 +142,6 @@ func stopAndLog(c *cfg, name string, stopper func() error) { } func bootUp(ctx context.Context, c *cfg) { - runAndLog(ctx, c, "NATS", true, connectNats) runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) }) runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit) diff --git a/cmd/frostfs-node/notificator.go b/cmd/frostfs-node/notificator.go deleted file mode 100644 index f9bb31fed..000000000 --- a/cmd/frostfs-node/notificator.go +++ /dev/null @@ -1,172 +0,0 @@ -package main - -import ( - "context" - "encoding/hex" - "fmt" - - nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/notificator" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/notificator/nats" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.uber.org/zap" -) - -type notificationSource struct { - e *engine.StorageEngine - l *logger.Logger - defaultTopic string -} - -func (n *notificationSource) Iterate(ctx context.Context, epoch uint64, handler func(topic string, addr oid.Address)) { - log := n.l.With(zap.Uint64("epoch", epoch)) - - listRes, err := n.e.ListContainers(ctx, engine.ListContainersPrm{}) - if err != nil { - log.Error(logs.FrostFSNodeNotificatorCouldNotListContainers, zap.Error(err)) - return - } - - filters := objectSDK.NewSearchFilters() - filters.AddNotificationEpochFilter(epoch) - - var selectPrm engine.SelectPrm - selectPrm.WithFilters(filters) - - for _, c := range listRes.Containers() { - selectPrm.WithContainerID(c) - - selectRes, err := n.e.Select(ctx, selectPrm) - if err != nil { - log.Error(logs.FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer, - zap.Stringer("cid", c), - zap.Error(err), - ) - continue - } - - for _, a := range selectRes.AddressList() { - err = n.processAddress(ctx, a, handler) - if err != nil { - log.Error(logs.FrostFSNodeNotificatorCouldNotProcessObject, - zap.Stringer("address", a), - zap.Error(err), - ) - continue - } - } - } - - log.Debug(logs.FrostFSNodeNotificatorFinishedProcessingObjectNotifications) -} - -func (n *notificationSource) processAddress( - ctx context.Context, - a oid.Address, - h func(topic string, addr oid.Address), -) error { - var prm engine.HeadPrm - prm.WithAddress(a) - - res, err := n.e.Head(ctx, prm) - if err != nil { - return err - } - - ni, err := res.Header().NotificationInfo() - if err != nil { - return fmt.Errorf("could not retrieve notification topic from object: %w", err) - } - - topic := ni.Topic() - - if topic == "" { - topic = n.defaultTopic - } - - h(topic, a) - - return nil -} - -type notificationWriter struct { - l *logger.Logger - w *nats.Writer -} - -func (n notificationWriter) Notify(topic string, address oid.Address) { - if err := n.w.Notify(topic, address); err != nil { - n.l.Warn(logs.FrostFSNodeCouldNotWriteObjectNotification, - zap.Stringer("address", address), - zap.String("topic", topic), - zap.Error(err), - ) - } -} - -func initNotifications(ctx context.Context, c *cfg) { - if nodeconfig.Notification(c.appCfg).Enabled() { - topic := nodeconfig.Notification(c.appCfg).DefaultTopic() - pubKey := hex.EncodeToString(c.cfgNodeInfo.localInfo.PublicKey()) - - if topic == "" { - topic = pubKey - } - - natsSvc := nats.New( - nats.WithConnectionName("FrostFS Storage Node: "+pubKey), // connection name is used in the server side logs - nats.WithTimeout(nodeconfig.Notification(c.appCfg).Timeout()), - nats.WithClientCert( - nodeconfig.Notification(c.appCfg).CertPath(), - nodeconfig.Notification(c.appCfg).KeyPath(), - ), - nats.WithRootCA(nodeconfig.Notification(c.appCfg).CAPath()), - nats.WithLogger(c.log), - ) - - c.cfgNotifications = cfgNotifications{ - enabled: true, - nw: notificationWriter{ - l: c.log, - w: natsSvc, - }, - defaultTopic: topic, - } - - n := notificator.New(new(notificator.Prm). - SetLogger(c.log). - SetNotificationSource( - ¬ificationSource{ - e: c.cfgObject.cfgLocalStorage.localStorage, - l: c.log, - defaultTopic: topic, - }). - SetWriter(c.cfgNotifications.nw), - ) - - addNewEpochAsyncNotificationHandler(c, func(e event.Event) { - ev := e.(netmap.NewEpoch) - - n.ProcessEpoch(ctx, ev.EpochNumber()) - }) - } -} - -func connectNats(ctx context.Context, c *cfg) { - if !c.cfgNotifications.enabled { - return - } - - endpoint := nodeconfig.Notification(c.appCfg).Endpoint() - err := c.cfgNotifications.nw.w.Connect(ctx, endpoint) - if err != nil { - panic(fmt.Sprintf("could not connect to a nats endpoint %s: %v", endpoint, err)) - } - c.log.Info(logs.NatsConnectedToEndpoint, zap.String("endpoint", endpoint)) -} diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 9f34896bc..9d1e2218f 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -14,7 +14,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" @@ -333,15 +332,6 @@ func createPutSvc(c *cfg, keyStorage *util.KeyStorage, irFetcher *cachedIRFetche engine: ls, } - if c.cfgNotifications.enabled { - os = engineWithNotifications{ - base: os, - nw: c.cfgNotifications.nw, - ns: c.cfgNetmap.state, - defaultTopic: c.cfgNotifications.defaultTopic, - } - } - return putsvc.NewService( keyStorage, c.putClientCache, @@ -503,47 +493,6 @@ func (s *morphEACLFetcher) GetEACL(cnr cid.ID) (*containercore.EACL, error) { return eaclInfo, nil } -type engineWithNotifications struct { - base putsvc.ObjectStorage - nw notificationWriter - ns netmap.State - - defaultTopic string -} - -func (e engineWithNotifications) IsLocked(ctx context.Context, address oid.Address) (bool, error) { - return e.base.IsLocked(ctx, address) -} - -func (e engineWithNotifications) Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error { - return e.base.Delete(ctx, tombstone, toDelete) -} - -func (e engineWithNotifications) Lock(ctx context.Context, locker oid.Address, toLock []oid.ID) error { - return e.base.Lock(ctx, locker, toLock) -} - -func (e engineWithNotifications) Put(ctx context.Context, o *objectSDK.Object) error { - if err := e.base.Put(ctx, o); err != nil { - return err - } - - ni, err := o.NotificationInfo() - if err == nil { - if epoch := ni.Epoch(); epoch == 0 || epoch == e.ns.CurrentEpoch() { - topic := ni.Topic() - - if topic == "" { - topic = e.defaultTopic - } - - e.nw.Notify(topic, objectCore.AddressOf(o)) - } - } - - return nil -} - type engineWithoutNotifications struct { engine *engine.StorageEngine } diff --git a/config/example/node.env b/config/example/node.env index db31ae35a..9f15c4042 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -22,13 +22,6 @@ FROSTFS_NODE_ATTRIBUTE_1="UN-LOCODE:RU MSK" FROSTFS_NODE_RELAY=true FROSTFS_NODE_PERSISTENT_SESSIONS_PATH=/sessions FROSTFS_NODE_PERSISTENT_STATE_PATH=/state -FROSTFS_NODE_NOTIFICATION_ENABLED=true -FROSTFS_NODE_NOTIFICATION_ENDPOINT=tls://localhost:4222 -FROSTFS_NODE_NOTIFICATION_TIMEOUT=6s -FROSTFS_NODE_NOTIFICATION_DEFAULT_TOPIC=topic -FROSTFS_NODE_NOTIFICATION_CERTIFICATE=/cert/path -FROSTFS_NODE_NOTIFICATION_KEY=/key/path -FROSTFS_NODE_NOTIFICATION_CA=/ca/path # Tree service section FROSTFS_TREE_ENABLED=true diff --git a/config/example/node.json b/config/example/node.json index 80140e5a2..79e6fe89a 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -36,15 +36,6 @@ }, "persistent_state": { "path": "/state" - }, - "notification": { - "enabled": true, - "endpoint": "tls://localhost:4222", - "timeout": "6s", - "default_topic": "topic", - "certificate": "/cert/path", - "key": "/key/path", - "ca": "/ca/path" } }, "grpc": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 13c1823d8..34e796ac4 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -35,14 +35,6 @@ node: path: /sessions # path to persistent session tokens file of Storage node (default: in-memory sessions) persistent_state: path: /state # path to persistent state file of Storage node - notification: - enabled: true # turn on object notification service - endpoint: "tls://localhost:4222" # notification server endpoint - timeout: "6s" # timeout for object notification client connection - default_topic: "topic" # default topic for object notifications if not found in object's meta - certificate: "/cert/path" # path to TLS certificate - key: "/key/path" # path to TLS key - ca: "/ca/path" # path to optional CA certificate grpc: - endpoint: s01.frostfs.devenv:8080 # endpoint for gRPC server diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index ce9065ac2..3a6969abd 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -322,14 +322,6 @@ node: path: /sessions persistent_state: path: /state - notification: - enabled: true - endpoint: tls://localhost:4222 - timeout: 6s - default_topic: topic - certificate: /path/to/cert.pem - key: /path/to/key.pem - ca: /path/to/ca.pem ``` | Parameter | Type | Default value | Description | @@ -341,8 +333,6 @@ node: | `relay` | `bool` | | Enable relay mode. | | `persistent_sessions` | [Persistent sessions config](#persistent_sessions-subsection) | | Persistent session token store configuration. | | `persistent_state` | [Persistent state config](#persistent_state-subsection) | | Persistent state configuration. | -| `notification` | [Notification config](#notification-subsection) | | NATS configuration. | - ## `wallet` subsection N3 wallet configuration. @@ -369,19 +359,6 @@ It is used to correctly handle node restarts or crashes. |-----------|----------|------------------------|------------------------| | `path` | `string` | `.frostfs-storage-state` | Path to the database. | -## `notification` subsection -This is an advanced section, use with caution. - -| Parameter | Type | Default value | Description | -|-----------------|------------|-------------------|-------------------------------------------------------------------| -| `enabled` | `bool` | `false` | Flag to enable the service. | -| `endpoint` | `string` | | NATS endpoint to connect to. | -| `timeout` | `duration` | `5s` | Timeout for the object notification operation. | -| `default_topic` | `string` | node's public key | Default topic to use if an object has no corresponding attribute. | -| `certificate` | `string` | | Path to the client certificate. | -| `key` | `string` | | Path to the client key. | -| `ca` | `string` | | Override root CA used to verify server certificates. | - # `apiclient` section Configuration for the FrostFS API client used for communication with other FrostFS nodes. diff --git a/go.mod b/go.mod index 093560953..42896ed8c 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.12.1 - github.com/nats-io/nats.go v1.32.0 github.com/nspcc-dev/neo-go v0.106.0 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 @@ -96,8 +95,6 @@ require ( github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/nats-io/nkeys v0.4.7 // indirect - github.com/nats-io/nuid v1.0.1 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect diff --git a/go.sum b/go.sum index 6f5034f00..07632a032 100644 --- a/go.sum +++ b/go.sum @@ -184,12 +184,6 @@ github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7B github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/nats-io/nats.go v1.32.0 h1:Bx9BZS+aXYlxW08k8Gd3yR2s73pV5XSoAQUyp1Kwvp0= -github.com/nats-io/nats.go v1.32.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= -github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= -github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc= github.com/nspcc-dev/neo-go v0.106.0 h1:YiOdW/GcLmbVSvxMRfD5aytO6n3TDHrEA97VHMawy6g= diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 7dc63341d..8dae61c3d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -132,10 +132,6 @@ const ( UtilCouldNotPushTaskToWorkerPool = "could not push task to worker pool" V2CantCheckIfRequestFromInnerRing = "can't check if request from inner ring" V2CantCheckIfRequestFromContainerNode = "can't check if request from container node" - NatsNatsConnectionWasLost = "nats: connection was lost" - NatsNatsReconnectedToTheServer = "nats: reconnected to the server" - NatsNatsClosingConnectionAsTheContextIsDone = "nats: closing connection as the context is done" - NatsConnectedToEndpoint = "nats: successfully connected to endpoint" ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch = "could not restore block subscription after RPC switch" ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch = "could not restore notification subscription after RPC switch" ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch = "could not restore notary notification subscription after RPC switch" diff --git a/pkg/services/notificator/deps.go b/pkg/services/notificator/deps.go deleted file mode 100644 index d6330f788..000000000 --- a/pkg/services/notificator/deps.go +++ /dev/null @@ -1,22 +0,0 @@ -package notificator - -import ( - "context" - - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -// NotificationSource is a source of object notifications. -type NotificationSource interface { - // Iterate must iterate over all notifications for the - // provided epoch and call handler for all of them. - Iterate(ctx context.Context, epoch uint64, handler func(topic string, addr oid.Address)) -} - -// NotificationWriter notifies all the subscribers -// about new object notifications. -type NotificationWriter interface { - // Notify must notify about an event generated - // from an object with a specific topic. - Notify(topic string, address oid.Address) -} diff --git a/pkg/services/notificator/nats/options.go b/pkg/services/notificator/nats/options.go deleted file mode 100644 index c9ba2ed26..000000000 --- a/pkg/services/notificator/nats/options.go +++ /dev/null @@ -1,38 +0,0 @@ -package nats - -import ( - "time" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/nats-io/nats.go" -) - -func WithClientCert(certPath, keyPath string) Option { - return func(o *opts) { - o.nOpts = append(o.nOpts, nats.ClientCert(certPath, keyPath)) - } -} - -func WithRootCA(paths ...string) Option { - return func(o *opts) { - o.nOpts = append(o.nOpts, nats.RootCAs(paths...)) - } -} - -func WithTimeout(timeout time.Duration) Option { - return func(o *opts) { - o.nOpts = append(o.nOpts, nats.Timeout(timeout)) - } -} - -func WithConnectionName(name string) Option { - return func(o *opts) { - o.nOpts = append(o.nOpts, nats.Name(name)) - } -} - -func WithLogger(logger *logger.Logger) Option { - return func(o *opts) { - o.log = logger - } -} diff --git a/pkg/services/notificator/nats/service.go b/pkg/services/notificator/nats/service.go deleted file mode 100644 index 9de538f2b..000000000 --- a/pkg/services/notificator/nats/service.go +++ /dev/null @@ -1,129 +0,0 @@ -package nats - -import ( - "context" - "errors" - "fmt" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/nats-io/nats.go" - "go.uber.org/zap" -) - -// Writer is a NATS object notification writer. -// It handles NATS JetStream connections and allows -// sending string representation of the address to -// the NATS server. -// -// For correct operation must be created via New function. -// new(Writer) or Writer{} construction leads to undefined -// behaviour and is not safe. -type Writer struct { - js nats.JetStreamContext - nc *nats.Conn - - m sync.RWMutex - createdStreams map[string]struct{} - opts -} - -type opts struct { - log *logger.Logger - nOpts []nats.Option -} - -type Option func(*opts) - -var errConnIsClosed = errors.New("connection to the server is closed") - -// Notify sends object address's string representation to the provided topic. -// Uses first 4 bytes of object ID as a message ID to support 'exactly once' -// message delivery. -// -// Returns error only if: -// 1. underlying connection was closed and has not been established again; -// 2. NATS server could not respond that it has saved the message. -func (n *Writer) Notify(topic string, address oid.Address) error { - if !n.nc.IsConnected() { - return errConnIsClosed - } - - // use first 4 byte of the encoded string as - // message ID for the 'exactly once' delivery - messageID := address.Object().EncodeToString()[:4] - - // check if the stream was previously created - n.m.RLock() - _, created := n.createdStreams[topic] - n.m.RUnlock() - - if !created { - _, err := n.js.AddStream(&nats.StreamConfig{ - Name: topic, - }) - if err != nil { - return fmt.Errorf("could not add stream: %w", err) - } - - n.m.Lock() - n.createdStreams[topic] = struct{}{} - n.m.Unlock() - } - - _, err := n.js.Publish(topic, []byte(address.EncodeToString()), nats.MsgId(messageID)) - return err -} - -// New creates new Writer. -func New(oo ...Option) *Writer { - w := &Writer{ - createdStreams: make(map[string]struct{}), - opts: opts{ - log: &logger.Logger{Logger: zap.L()}, - nOpts: make([]nats.Option, 0, len(oo)+3), - }, - } - - for _, o := range oo { - o(&w.opts) - } - - w.opts.nOpts = append(w.opts.nOpts, - nats.NoCallbacksAfterClientClose(), // do not call callbacks when it was planned writer stop - nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { - w.log.Error(logs.NatsNatsConnectionWasLost, zap.Error(err)) - }), - nats.ReconnectHandler(func(_ *nats.Conn) { - w.log.Warn(logs.NatsNatsReconnectedToTheServer) - }), - ) - - return w -} - -// Connect tries to connect to a specified NATS endpoint. -// -// Connection is closed when passed context is done. -func (n *Writer) Connect(ctx context.Context, endpoint string) error { - nc, err := nats.Connect(endpoint, n.opts.nOpts...) - if err != nil { - return fmt.Errorf("could not connect to server: %w", err) - } - - n.nc = nc - - // usage w/o options is error-free - n.js, _ = nc.JetStream() - - go func() { - <-ctx.Done() - n.opts.log.Info(logs.NatsNatsClosingConnectionAsTheContextIsDone) - - nc.Close() - }() - - return nil -} diff --git a/pkg/services/notificator/service.go b/pkg/services/notificator/service.go deleted file mode 100644 index bbf4e4823..000000000 --- a/pkg/services/notificator/service.go +++ /dev/null @@ -1,88 +0,0 @@ -package notificator - -import ( - "context" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.uber.org/zap" -) - -// Prm groups Notificator constructor's -// parameters. All are required. -type Prm struct { - writer NotificationWriter - notificationSource NotificationSource - logger *logger.Logger -} - -// SetLogger sets a logger. -func (prm *Prm) SetLogger(v *logger.Logger) *Prm { - prm.logger = v - return prm -} - -// SetWriter sets notification writer. -func (prm *Prm) SetWriter(v NotificationWriter) *Prm { - prm.writer = v - return prm -} - -// SetNotificationSource sets notification source. -func (prm *Prm) SetNotificationSource(v NotificationSource) *Prm { - prm.notificationSource = v - return prm -} - -// Notificator is a notification producer that handles -// objects with defined notification epoch. -// -// Working client must be created via constructor New. -// Using the Client that has been created with new(Client) -// expression (or just declaring a Client variable) is unsafe -// and can lead to panic. -type Notificator struct { - w NotificationWriter - ns NotificationSource - l *logger.Logger -} - -// New creates, initializes and returns the Notificator instance. -// -// Panics if any field of the passed Prm structure is not set/set -// to nil. -func New(prm *Prm) *Notificator { - panicOnNil := func(v any, name string) { - if v == nil { - panic(fmt.Sprintf("Notificator constructor: %s is nil\n", name)) - } - } - - panicOnNil(prm.writer, "NotificationWriter") - panicOnNil(prm.notificationSource, "NotificationSource") - panicOnNil(prm.logger, "Logger") - - return &Notificator{ - w: prm.writer, - ns: prm.notificationSource, - l: prm.logger, - } -} - -// ProcessEpoch looks for all objects with defined epoch in the storage -// and passes their addresses to the NotificationWriter. -func (n *Notificator) ProcessEpoch(ctx context.Context, epoch uint64) { - logger := n.l.With(zap.Uint64("epoch", epoch)) - logger.Debug(logs.NotificatorNotificatorStartProcessingObjectNotifications) - - n.ns.Iterate(ctx, epoch, func(topic string, addr oid.Address) { - n.l.Debug(logs.NotificatorNotificatorProcessingObjectNotification, - zap.String("topic", topic), - zap.Stringer("address", addr), - ) - - n.w.Notify(topic, addr) - }) -} From a90310335d0b802700627fa91399ba1a0a4dc0f9 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 4 Jun 2024 18:27:37 +0300 Subject: [PATCH 1141/1943] [#1156] ape: Return `not found` when removing local overrides Signed-off-by: Anton Nikiforov --- pkg/services/control/server/policy_engine.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index e2e1fd731..98daac8a6 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -174,10 +174,8 @@ func (s *Server) RemoveChainLocalOverride(_ context.Context, req *control.Remove return nil, err } - if err = s.localOverrideStorage.LocalStorage().RemoveOverride(apechain.Ingress, target, apechain.ID(req.GetBody().GetChainId())); err != nil { - if code := getCodeByLocalStorageErr(err); code != codes.NotFound { - return nil, status.Error(code, err.Error()) - } + if err = s.localOverrideStorage.LocalStorage().RemoveOverride(apechain.Ingress, target, req.GetBody().GetChainId()); err != nil { + return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } resp := &control.RemoveChainLocalOverrideResponse{ Body: &control.RemoveChainLocalOverrideResponse_Body{}, @@ -200,9 +198,7 @@ func (s *Server) RemoveChainLocalOverridesByTarget(_ context.Context, req *contr } if err = s.localOverrideStorage.LocalStorage().RemoveOverridesByTarget(apechain.Ingress, target); err != nil { - if code := getCodeByLocalStorageErr(err); code != codes.NotFound { - return nil, status.Error(code, err.Error()) - } + return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } resp := &control.RemoveChainLocalOverridesByTargetResponse{ Body: &control.RemoveChainLocalOverridesByTargetResponse_Body{}, @@ -246,7 +242,8 @@ func (s *Server) ListTargetsLocalOverrides(_ context.Context, req *control.ListT } func getCodeByLocalStorageErr(err error) codes.Code { - if errors.Is(err, engine.ErrChainNotFound) || errors.Is(err, engine.ErrChainNameNotFound) { + if errors.Is(err, engine.ErrChainNotFound) || errors.Is(err, engine.ErrChainNameNotFound) || + errors.Is(err, engine.ErrResourceNotFound) { return codes.NotFound } return codes.Internal From 4edff5eea60d8106f24d138582ea3e9c1a6fbffa Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 3 Jun 2024 15:32:54 +0300 Subject: [PATCH 1142/1943] [#1157] ape: Introduce single-run chain router Signed-off-by: Airat Arifullin --- pkg/ape/router/single_pass.go | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 pkg/ape/router/single_pass.go diff --git a/pkg/ape/router/single_pass.go b/pkg/ape/router/single_pass.go new file mode 100644 index 000000000..c0c78db76 --- /dev/null +++ b/pkg/ape/router/single_pass.go @@ -0,0 +1,55 @@ +package router + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" +) + +func newTarget(ct ape.ChainTarget) (engine.Target, error) { + var target engine.Target + switch ct.TargetType { + case ape.TargetTypeContainer: + var cid cidSDK.ID + err := cid.DecodeString(ct.Name) + if err != nil { + return target, fmt.Errorf("invalid cid format: %s", target.Name) + } + target.Type = engine.Container + case ape.TargetTypeGroup: + target.Type = engine.Group + case ape.TargetTypeNamespace: + target.Type = engine.Namespace + case ape.TargetTypeUser: + target.Type = engine.User + default: + return target, fmt.Errorf("unsupported target type: %v", ct.TargetType) + } + target.Name = ct.Name + return target, nil +} + +// SingleUseRouterWithBearerTokenChains creates chain router with inmemory storage implementation and +// fed with APE chains defined in Bearer token. +func SingleUseRouterWithBearerTokenChains(overrides []bearer.APEOverride) (engine.ChainRouter, error) { + storage := inmemory.NewInmemoryMorphRuleChainStorage() + for _, override := range overrides { + target, err := newTarget(override.Target) + if err != nil { + return nil, err + } + for i := range override.Chains { + chain := new(apechain.Chain) + if err := chain.DecodeBytes(override.Chains[i].Raw); err != nil { + return nil, fmt.Errorf("invalid ape chain: %w", err) + } + _, _, _ = storage.AddMorphRuleChain(apechain.Ingress, target, chain) + } + } + return engine.NewDefaultChainRouter(storage), nil +} From 04a3f891fda3f01e7f8eb4a55b5ec8f3b5c9f4eb Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 29 May 2024 11:11:45 +0300 Subject: [PATCH 1143/1943] [#1157] object: Make APE checker use Bearer-token's APE overrides Signed-off-by: Airat Arifullin --- cmd/frostfs-node/object.go | 1 + pkg/services/object/acl/v2/service.go | 4 + pkg/services/object/ape/checker.go | 90 +++++- pkg/services/object/ape/checker_test.go | 414 ++++++++++++++---------- pkg/services/object/ape/service.go | 14 + pkg/services/object/request_context.go | 3 + 6 files changed, 358 insertions(+), 168 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 9d1e2218f..907e517a5 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -463,6 +463,7 @@ func createAPEService(c *cfg, splitSvc *objectService.TransportSplitter) *object objectAPE.NewStorageEngineHeaderProvider(c.cfgObject.cfgLocalStorage.localStorage, c.cfgObject.getSvc), c.shared.frostfsidClient, c.netMapSource, + c.cfgNetmap.state, c.cfgObject.cnrSource, c.binPublicKey, ), diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 1f1c275cf..3e128836f 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -118,6 +118,7 @@ func (w *wrappedGetObjectStream) Context() context.Context { SenderKey: w.requestInfo.SenderKey(), Role: w.requestInfo.RequestRole(), SoftAPECheck: w.requestInfo.IsSoftAPECheck(), + BearerToken: w.requestInfo.Bearer(), }) } @@ -143,6 +144,7 @@ func (w *wrappedRangeStream) Context() context.Context { SenderKey: w.requestInfo.SenderKey(), Role: w.requestInfo.RequestRole(), SoftAPECheck: w.requestInfo.IsSoftAPECheck(), + BearerToken: w.requestInfo.Bearer(), }) } @@ -168,6 +170,7 @@ func (w *wrappedSearchStream) Context() context.Context { SenderKey: w.requestInfo.SenderKey(), Role: w.requestInfo.RequestRole(), SoftAPECheck: w.requestInfo.IsSoftAPECheck(), + BearerToken: w.requestInfo.Bearer(), }) } @@ -479,6 +482,7 @@ func requestContext(ctx context.Context, reqInfo RequestInfo) context.Context { SenderKey: reqInfo.SenderKey(), Role: reqInfo.RequestRole(), SoftAPECheck: reqInfo.IsSoftAPECheck(), + BearerToken: reqInfo.Bearer(), }) } diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index c39f922f7..ee71e6e1d 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -2,13 +2,17 @@ package ape import ( "context" + "crypto/ecdsa" "errors" "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/router" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -23,16 +27,18 @@ type checkerImpl struct { headerProvider HeaderProvider frostFSIDClient frostfsidcore.SubjectProvider nm netmap.Source + st netmap.State cnrSource container.Source nodePK []byte } -func NewChecker(chainRouter policyengine.ChainRouter, headerProvider HeaderProvider, frostFSIDClient frostfsidcore.SubjectProvider, nm netmap.Source, cnrSource container.Source, nodePK []byte) Checker { +func NewChecker(chainRouter policyengine.ChainRouter, headerProvider HeaderProvider, frostFSIDClient frostfsidcore.SubjectProvider, nm netmap.Source, st netmap.State, cnrSource container.Source, nodePK []byte) Checker { return &checkerImpl{ chainRouter: chainRouter, headerProvider: headerProvider, frostFSIDClient: frostFSIDClient, nm: nm, + st: st, cnrSource: cnrSource, nodePK: nodePK, } @@ -67,9 +73,69 @@ type Prm struct { // If true, object headers will not retrieved from storage engine. WithoutHeaderRequest bool + + // The request's bearer token. It is used in order to check APE overrides with the token. + BearerToken *bearer.Token } -var errMissingOID = errors.New("object ID is not set") +var ( + errMissingOID = errors.New("object ID is not set") + errInvalidTargetType = errors.New("bearer token defines non-container target override") + errBearerExpired = errors.New("bearer token has expired") + errBearerInvalidSignature = errors.New("bearer token has invalid signature") + errBearerInvalidContainerID = errors.New("bearer token was created for another container") + errBearerNotSignedByOwner = errors.New("bearer token is not signed by the container owner") + errBearerInvalidOwner = errors.New("bearer token owner differs from the request sender") +) + +// isValidBearer checks whether bearer token was correctly signed by authorized +// entity. This method might be defined on whole ACL service because it will +// require fetching current epoch to check lifetime. +func isValidBearer(token *bearer.Token, ownerCnr user.ID, containerID cid.ID, publicKey *keys.PublicKey, st netmap.State) error { + if token == nil { + return nil + } + + // 1. First check token lifetime. Simplest verification. + if token.InvalidAt(st.CurrentEpoch()) { + return errBearerExpired + } + + // 2. Then check if bearer token is signed correctly. + if !token.VerifySignature() { + return errBearerInvalidSignature + } + + // 3. Then check if container is either empty or equal to the container in the request. + apeOverride := token.APEOverride() + if apeOverride.Target.TargetType != ape.TargetTypeContainer { + return errInvalidTargetType + } + + var targetCnr cid.ID + err := targetCnr.DecodeString(apeOverride.Target.Name) + if err != nil { + return fmt.Errorf("invalid cid format: %s", apeOverride.Target.Name) + } + if !containerID.Equals(targetCnr) { + return errBearerInvalidContainerID + } + + // 4. Then check if container owner signed this token. + if !bearer.ResolveIssuer(*token).Equals(ownerCnr) { + return errBearerNotSignedByOwner + } + + // 5. Then check if request sender has rights to use this token. + var usrSender user.ID + user.IDFromKey(&usrSender, (ecdsa.PublicKey)(*publicKey)) + + if !token.AssertUser(usrSender) { + return errBearerInvalidOwner + } + + return nil +} // CheckAPE checks if a request or a response is permitted creating an ape request and passing // it to chain router. @@ -99,6 +165,26 @@ func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { return err } + if prm.BearerToken != nil && !prm.BearerToken.Impersonate() { + if err := isValidBearer(prm.BearerToken, prm.ContainerOwner, prm.Container, pub, c.st); err != nil { + return fmt.Errorf("bearer token validation error: %w", err) + } + btRouter, err := router.SingleUseRouterWithBearerTokenChains([]bearer.APEOverride{prm.BearerToken.APEOverride()}) + if err != nil { + return err + } + status, found, err := btRouter.IsAllowed(apechain.Ingress, policyengine.NewRequestTargetWithContainer(prm.Container.EncodeToString()), r) + if err != nil { + return err + } + if found && status == apechain.Allow { + return nil + } + if status != apechain.NoRuleFound { + return fmt.Errorf("bearer token: method %s: %s", prm.Method, status) + } + } + rt := policyengine.NewRequestTargetExtended(prm.Namespace, prm.Container.EncodeToString(), fmt.Sprintf("%s:%s", prm.Namespace, pub.Address()), nil) status, ruleFound, err := c.chainRouter.IsAllowed(apechain.Ingress, rt, r) if err != nil { diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index ddb336127..09954602c 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -2,6 +2,7 @@ package ape import ( "context" + "crypto/ecdsa" "encoding/hex" "errors" "fmt" @@ -12,6 +13,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" + apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -228,170 +231,249 @@ func (f *frostfsIDProviderMock) GetSubjectExtended(key util.Uint160) (*client.Su return v, nil } +var apeCheckTestCases = []struct { + name string + container string + object *string + methods []string + header testHeader + containerRules []chain.Rule + expectAPEErr bool +}{ + { + name: "oid required requests are allowed", + container: containerID, + object: stringPtr(objectID), + methods: methodsRequiredOID, + containerRules: []chain.Rule{ + { + Status: chain.Allow, + Actions: chain.Actions{Names: methodsRequiredOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, + }, + }, + }, + }, + { + name: "oid optional requests are allowed", + container: containerID, + methods: methodsOptionalOID, + containerRules: []chain.Rule{ + { + Status: chain.Allow, + Actions: chain.Actions{Names: methodsOptionalOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + }, + }, + }, + { + name: "oid required requests are denied", + container: containerID, + object: stringPtr(objectID), + methods: methodsRequiredOID, + containerRules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{Names: methodsRequiredOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, + }, + }, + }, + expectAPEErr: true, + }, + { + name: "oid required requests are denied by an attribute", + container: containerID, + object: stringPtr(objectID), + methods: methodsRequiredOID, + header: testHeader{ + headerObjSDK: &headerObjectSDKParams{ + attributes: []struct { + key string + val string + }{ + { + key: "attr1", + val: "attribute_value", + }, + }, + }, + fromHeaderProvider: true, + }, + containerRules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{Names: methodsRequiredOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, + }, + Any: true, + Condition: []chain.Condition{ + { + Op: chain.CondStringLike, + Kind: chain.KindResource, + Key: "attr1", + Value: "attribute*", + }, + }, + }, + }, + expectAPEErr: true, + }, + { + name: "oid required requests are denied by sender", + container: containerID, + object: stringPtr(objectID), + methods: methodsRequiredOID, + header: testHeader{ + headerObjSDK: &headerObjectSDKParams{ + attributes: []struct { + key string + val string + }{ + { + key: "attr1", + val: "attribute_value", + }, + }, + }, + fromHeaderProvider: true, + }, + containerRules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{Names: methodsRequiredOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, + }, + Any: true, + Condition: []chain.Condition{ + { + Op: chain.CondStringLike, + Kind: chain.KindRequest, + Key: nativeschema.PropertyKeyActorPublicKey, + Value: senderKey, + }, + }, + }, + }, + expectAPEErr: true, + }, + { + name: "optional oid requests reached quota limit by an attribute", + container: containerID, + methods: methodsOptionalOID, + header: testHeader{ + headerObjSDK: &headerObjectSDKParams{ + payloadSize: 1000, + }, + fromRequestResponseHeader: true, + }, + containerRules: []chain.Rule{ + { + Status: chain.QuotaLimitReached, + Actions: chain.Actions{Names: methodsOptionalOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + Any: true, + Condition: []chain.Condition{ + { + Op: chain.CondStringEquals, + Kind: chain.KindResource, + Key: nativeschema.PropertyKeyObjectPayloadLength, + Value: "1000", + }, + }, + }, + }, + expectAPEErr: true, + }, +} + +type stMock struct{} + +func (m *stMock) CurrentEpoch() uint64 { + return 8 +} + +func TestAPECheck_BearerTokenOverrides(t *testing.T) { + for _, test := range apeCheckTestCases { + t.Run(test.name, func(t *testing.T) { + chain := chain.Chain{ + Rules: test.containerRules, + MatchType: chain.MatchTypeFirstMatch, + } + chainSDK := apeSDK.Chain{ + Raw: chain.Bytes(), + } + bt := new(bearer.Token) + bt.SetIat(1) + bt.SetExp(10) + bt.SetAPEOverride(bearer.APEOverride{ + Target: apeSDK.ChainTarget{ + TargetType: apeSDK.TargetTypeContainer, + Name: test.container, + }, + Chains: []apeSDK.Chain{chainSDK}, + }) + bt.Sign(senderPrivateKey.PrivateKey) + var cnrOwner user.ID + user.IDFromKey(&cnrOwner, (ecdsa.PublicKey)(*senderPrivateKey.PublicKey())) + + for _, method := range test.methods { + t.Run(method, func(t *testing.T) { + headerProvider := newHeaderProviderMock() + frostfsidProvider := newFrostfsIDProviderMock(t) + + cnr := newContainerIDSDK(t, test.container) + obj := newObjectIDSDK(t, test.object) + + ls := inmemory.NewInmemoryLocalStorage() + ms := inmemory.NewInmemoryMorphRuleChainStorage() + r := policyengine.NewDefaultChainRouterWithLocalOverrides(ms, ls) + + checker := NewChecker(r, headerProvider, frostfsidProvider, nil, &stMock{}, nil, nil) + + prm := Prm{ + Method: method, + Container: cnr, + Object: obj, + Role: role, + ContainerOwner: cnrOwner, + SenderKey: senderKey, + BearerToken: bt, + } + + var headerObjSDK *objectSDK.Object + if test.header.headerObjSDK != nil { + headerObjSDK = newHeaderObjectSDK(cnr, obj, test.header.headerObjSDK) + if test.header.fromHeaderProvider { + require.NotNil(t, obj, "oid is required if a header is expected to be found in header provider") + headerProvider.addHeader(cnr, *obj, headerObjSDK) + } else if test.header.fromRequestResponseHeader { + prm.Header = headerObjSDK.ToV2().GetHeader() + } + } + + err := checker.CheckAPE(context.Background(), prm) + if test.expectAPEErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } + }) + } +} + func TestAPECheck(t *testing.T) { - for _, test := range []struct { - name string - container string - object *string - methods []string - header testHeader - containerRules []chain.Rule - expectAPEErr bool - }{ - { - name: "oid required requests are allowed", - container: containerID, - object: stringPtr(objectID), - methods: methodsRequiredOID, - containerRules: []chain.Rule{ - { - Status: chain.Allow, - Actions: chain.Actions{Names: methodsRequiredOID}, - Resources: chain.Resources{ - Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, - }, - }, - }, - }, - { - name: "oid optional requests are allowed", - container: containerID, - methods: methodsOptionalOID, - containerRules: []chain.Rule{ - { - Status: chain.Allow, - Actions: chain.Actions{Names: methodsOptionalOID}, - Resources: chain.Resources{ - Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, - }, - }, - }, - }, - { - name: "oid required requests are denied", - container: containerID, - object: stringPtr(objectID), - methods: methodsRequiredOID, - containerRules: []chain.Rule{ - { - Status: chain.AccessDenied, - Actions: chain.Actions{Names: methodsRequiredOID}, - Resources: chain.Resources{ - Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, - }, - }, - }, - expectAPEErr: true, - }, - { - name: "oid required requests are denied by an attribute", - container: containerID, - object: stringPtr(objectID), - methods: methodsRequiredOID, - header: testHeader{ - headerObjSDK: &headerObjectSDKParams{ - attributes: []struct { - key string - val string - }{ - { - key: "attr1", - val: "attribute_value", - }, - }, - }, - fromHeaderProvider: true, - }, - containerRules: []chain.Rule{ - { - Status: chain.AccessDenied, - Actions: chain.Actions{Names: methodsRequiredOID}, - Resources: chain.Resources{ - Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, - }, - Any: true, - Condition: []chain.Condition{ - { - Op: chain.CondStringLike, - Kind: chain.KindResource, - Key: "attr1", - Value: "attribute*", - }, - }, - }, - }, - expectAPEErr: true, - }, - { - name: "oid required requests are denied by sender", - container: containerID, - object: stringPtr(objectID), - methods: methodsRequiredOID, - header: testHeader{ - headerObjSDK: &headerObjectSDKParams{ - attributes: []struct { - key string - val string - }{ - { - key: "attr1", - val: "attribute_value", - }, - }, - }, - fromHeaderProvider: true, - }, - containerRules: []chain.Rule{ - { - Status: chain.AccessDenied, - Actions: chain.Actions{Names: methodsRequiredOID}, - Resources: chain.Resources{ - Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, - }, - Any: true, - Condition: []chain.Condition{ - { - Op: chain.CondStringLike, - Kind: chain.KindRequest, - Key: nativeschema.PropertyKeyActorPublicKey, - Value: senderKey, - }, - }, - }, - }, - expectAPEErr: true, - }, - { - name: "optional oid requests reached quota limit by an attribute", - container: containerID, - methods: methodsOptionalOID, - header: testHeader{ - headerObjSDK: &headerObjectSDKParams{ - payloadSize: 1000, - }, - fromRequestResponseHeader: true, - }, - containerRules: []chain.Rule{ - { - Status: chain.QuotaLimitReached, - Actions: chain.Actions{Names: methodsOptionalOID}, - Resources: chain.Resources{ - Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, - }, - Any: true, - Condition: []chain.Condition{ - { - Op: chain.CondStringEquals, - Kind: chain.KindResource, - Key: nativeschema.PropertyKeyObjectPayloadLength, - Value: "1000", - }, - }, - }, - }, - expectAPEErr: true, - }, - } { + for _, test := range apeCheckTestCases { t.Run(test.name, func(t *testing.T) { for _, method := range test.methods { t.Run(method, func(t *testing.T) { @@ -411,7 +493,7 @@ func TestAPECheck(t *testing.T) { router := policyengine.NewDefaultChainRouterWithLocalOverrides(ms, ls) - checker := NewChecker(router, headerProvider, frostfsidProvider, nil, nil, nil) + checker := NewChecker(router, headerProvider, frostfsidProvider, nil, &stMock{}, nil, nil) prm := Prm{ Method: method, @@ -544,7 +626,7 @@ func TestPutECChunk(t *testing.T) { }, } - checker := NewChecker(router, headerProvider, frostfsidProvider, nm, cs, node1Key.PublicKey().Bytes()) + checker := NewChecker(router, headerProvider, frostfsidProvider, nm, &stMock{}, cs, node1Key.PublicKey().Bytes()) ecParentID := oidtest.ID() chunkHeader := newHeaderObjectSDK(cnr, obj, nil).ToV2().GetHeader() @@ -586,7 +668,7 @@ func TestPutECChunk(t *testing.T) { t.Run("access allowed for non container node", func(t *testing.T) { otherKey, err := keys.NewPrivateKey() require.NoError(t, err) - checker = NewChecker(router, headerProvider, frostfsidProvider, nm, cs, otherKey.PublicKey().Bytes()) + checker = NewChecker(router, headerProvider, frostfsidProvider, nm, &stMock{}, cs, otherKey.PublicKey().Bytes()) prm := Prm{ Method: nativeschema.MethodPutObject, Container: cnr, diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 63c2a9909..609b117d0 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -13,6 +13,7 @@ import ( getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -88,6 +89,8 @@ type getStreamBasicChecker struct { role string softAPECheck bool + + bearerToken *bearer.Token } func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { @@ -107,6 +110,7 @@ func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { ContainerOwner: g.containerOwner, Role: g.role, SoftAPECheck: g.softAPECheck, + BearerToken: g.bearerToken, } if err := g.apeChecker.CheckAPE(g.Context(), prm); err != nil { @@ -149,6 +153,7 @@ func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectSt ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, WithoutHeaderRequest: true, + BearerToken: reqCtx.BearerToken, }) if err != nil { return toStatusErr(err) @@ -161,6 +166,7 @@ func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectSt senderKey: reqCtx.SenderKey, role: nativeSchemaRole(reqCtx.Role), softAPECheck: reqCtx.SoftAPECheck, + bearerToken: reqCtx.BearerToken, }) } @@ -192,6 +198,7 @@ func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutR ContainerOwner: reqCtx.ContainerOwner, Role: nativeSchemaRole(reqCtx.Role), SoftAPECheck: reqCtx.SoftAPECheck, + BearerToken: reqCtx.BearerToken, } if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -236,6 +243,7 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, WithoutHeaderRequest: true, + BearerToken: reqCtx.BearerToken, }) if err != nil { return nil, toStatusErr(err) @@ -275,6 +283,7 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, + BearerToken: reqCtx.BearerToken, }) if err != nil { return nil, toStatusErr(err) @@ -303,6 +312,7 @@ func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.Searc SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, + BearerToken: reqCtx.BearerToken, }) if err != nil { return toStatusErr(err) @@ -331,6 +341,7 @@ func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) ( SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, + BearerToken: reqCtx.BearerToken, }) if err != nil { return nil, toStatusErr(err) @@ -364,6 +375,7 @@ func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.G SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, + BearerToken: reqCtx.BearerToken, }) if err != nil { return toStatusErr(err) @@ -392,6 +404,7 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, + BearerToken: reqCtx.BearerToken, } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -430,6 +443,7 @@ func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequ SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, + BearerToken: reqCtx.BearerToken, } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { diff --git a/pkg/services/object/request_context.go b/pkg/services/object/request_context.go index 6a0965b40..95d4c9d93 100644 --- a/pkg/services/object/request_context.go +++ b/pkg/services/object/request_context.go @@ -1,6 +1,7 @@ package object import ( + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) @@ -20,4 +21,6 @@ type RequestContext struct { Role acl.Role SoftAPECheck bool + + BearerToken *bearer.Token } From 239323eeef0bf518630d4acb3e8b92d7b2629f8b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 29 May 2024 11:18:31 +0300 Subject: [PATCH 1144/1943] [#1157] tree: Make tree service use Bearer token's APE overrides Signed-off-by: Airat Arifullin --- cmd/frostfs-node/tree.go | 1 + pkg/services/tree/ape.go | 121 +++++++++++++++++++++++++++++---- pkg/services/tree/options.go | 7 ++ pkg/services/tree/signature.go | 2 +- 4 files changed, 118 insertions(+), 13 deletions(-) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index 49ecb6fdc..daaaa64a2 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -66,6 +66,7 @@ func initTreeService(c *cfg) { tree.WithAuthorizedKeys(treeConfig.AuthorizedKeys()), tree.WithMetrics(c.metricsCollector.TreeService()), tree.WithAPERouter(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine), + tree.WithNetmapState(c.cfgNetmap.state), ) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index eabc02bd7..475567c5f 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -2,18 +2,25 @@ package tree import ( "context" + "crypto/ecdsa" "encoding/hex" + "errors" "fmt" "net" "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/converter" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/router" core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" @@ -22,20 +29,25 @@ import ( "google.golang.org/grpc/peer" ) -func (s *Service) checkAPE(ctx context.Context, container *core.Container, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey) error { - namespace := "" - cntNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(container.Value).Zone(), ".ns") - if hasNamespace { - namespace = cntNamespace - } +var ( + errInvalidTargetType = errors.New("bearer token defines non-container target override") + errBearerExpired = errors.New("bearer token has expired") + errBearerInvalidSignature = errors.New("bearer token has invalid signature") + errBearerInvalidContainerID = errors.New("bearer token was created for another container") + errBearerNotSignedByOwner = errors.New("bearer token is not signed by the container owner") + errBearerInvalidOwner = errors.New("bearer token owner differs from the request sender") +) +func (s *Service) newAPERequest(ctx context.Context, namespace string, + cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, +) (aperequest.Request, error) { schemaMethod, err := converter.SchemaMethodFromACLOperation(operation) if err != nil { - return apeErr(err) + return aperequest.Request{}, err } schemaRole, err := converter.SchemaRoleFromACLRole(role) if err != nil { - return apeErr(err) + return aperequest.Request{}, err } reqProps := map[string]string{ nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(publicKey.Bytes()), @@ -43,7 +55,7 @@ func (s *Service) checkAPE(ctx context.Context, container *core.Container, cid c } reqProps, err = s.fillWithUserClaimTags(reqProps, publicKey) if err != nil { - return apeErr(err) + return aperequest.Request{}, err } if p, ok := peer.FromContext(ctx); ok { if tcpAddr, ok := p.Addr.(*net.TCPAddr); ok { @@ -58,11 +70,96 @@ func (s *Service) checkAPE(ctx context.Context, container *core.Container, cid c resourceName = fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainerObjects, namespace, cid.EncodeToString()) } - request := aperequest.NewRequest( + return aperequest.NewRequest( schemaMethod, aperequest.NewResource(resourceName, make(map[string]string)), reqProps, - ) + ), nil +} + +// isValidBearer checks whether bearer token was correctly signed by authorized +// entity. This method might be defined on whole ACL service because it will +// require fetching current epoch to check lifetime. +func isValidBearer(token *bearer.Token, ownerCnr user.ID, cntID cid.ID, publicKey *keys.PublicKey, st netmap.State) error { + if token == nil { + return nil + } + + // 1. First check token lifetime. Simplest verification. + if token.InvalidAt(st.CurrentEpoch()) { + return errBearerExpired + } + + // 2. Then check if bearer token is signed correctly. + if !token.VerifySignature() { + return errBearerInvalidSignature + } + + // 3. Then check if container is either empty or equal to the container in the request. + apeOverride := token.APEOverride() + if apeOverride.Target.TargetType != ape.TargetTypeContainer { + return errInvalidTargetType + } + + var targetCnr cid.ID + err := targetCnr.DecodeString(apeOverride.Target.Name) + if err != nil { + return fmt.Errorf("invalid cid format: %s", apeOverride.Target.Name) + } + if !cntID.Equals(targetCnr) { + return errBearerInvalidContainerID + } + + // 4. Then check if container owner signed this token. + if !bearer.ResolveIssuer(*token).Equals(ownerCnr) { + return errBearerNotSignedByOwner + } + + // 5. Then check if request sender has rights to use this token. + var usrSender user.ID + user.IDFromKey(&usrSender, (ecdsa.PublicKey)(*publicKey)) + + if !token.AssertUser(usrSender) { + return errBearerInvalidOwner + } + + return nil +} + +func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, + container *core.Container, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, +) error { + namespace := "" + cntNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(container.Value).Zone(), ".ns") + if hasNamespace { + namespace = cntNamespace + } + + request, err := s.newAPERequest(ctx, namespace, cid, operation, role, publicKey) + if err != nil { + return apeErr(err) + } + + if bt != nil && !bt.Impersonate() { + if err := isValidBearer(bt, container.Value.Owner(), cid, publicKey, s.state); err != nil { + return fmt.Errorf("bearer validation error: %w", err) + } + btRouter, err := router.SingleUseRouterWithBearerTokenChains([]bearer.APEOverride{bt.APEOverride()}) + if err != nil { + return apeErr(err) + } + status, found, err := btRouter.IsAllowed(apechain.Ingress, engine.NewRequestTargetWithContainer(cid.EncodeToString()), request) + if err != nil { + return apeErr(err) + } + if found && status == apechain.Allow { + return nil + } + if status != apechain.NoRuleFound { + err = fmt.Errorf("access to operation %s is denied by access policy engine (bearer token): %s", request.Operation(), status.String()) + return apeErr(err) + } + } rt := engine.NewRequestTargetExtended(namespace, cid.EncodeToString(), fmt.Sprintf("%s:%s", namespace, publicKey.Address()), nil) status, found, err := s.router.IsAllowed(apechain.Ingress, rt, request) @@ -72,7 +169,7 @@ func (s *Service) checkAPE(ctx context.Context, container *core.Container, cid c if found && status == apechain.Allow { return nil } - err = fmt.Errorf("access to operation %s is denied by access policy engine: %s", schemaMethod, status.String()) + err = fmt.Errorf("access to operation %s is denied by access policy engine: %s", request.Operation(), status.String()) return apeErr(err) } diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index a99ba6046..ea5539938 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -29,6 +29,7 @@ type cfg struct { log *logger.Logger key *ecdsa.PrivateKey rawPub []byte + state netmap.State nmSource netmap.Source cnrSource ContainerSource frostfsidSubjectProvider frostfsidcore.SubjectProvider @@ -156,3 +157,9 @@ func WithAPERouter(router policyengine.ChainRouter) Option { c.router = router } } + +func WithNetmapState(state netmap.State) Option { + return func(c *cfg) { + c.state = state + } +} diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 0445ccfab..58cab659f 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -84,7 +84,7 @@ func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, // FIXME(@aarifullin): tree service temporiraly performs APE checks on // object verbs, because tree verbs have not been introduced yet. if basicACL == 0x0 { - return s.checkAPE(ctx, cnr, cid, op, role, pubKey) + return s.checkAPE(ctx, bt, cnr, cid, op, role, pubKey) } if !basicACL.IsOpAllowed(op, role) { From a0c588263bd550bb131a8ed2f1a5ad318811018c Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 3 Jun 2024 16:13:29 +0300 Subject: [PATCH 1145/1943] [#1157] cli: Support adding APE overrides to Bearer token Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/bearer/create.go | 24 +++- .../modules/bearer/generate_override.go | 115 ++++++++++++++++++ cmd/frostfs-cli/modules/bearer/root.go | 1 + 3 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 cmd/frostfs-cli/modules/bearer/generate_override.go diff --git a/cmd/frostfs-cli/modules/bearer/create.go b/cmd/frostfs-cli/modules/bearer/create.go index 8a7500db3..d94b39207 100644 --- a/cmd/frostfs-cli/modules/bearer/create.go +++ b/cmd/frostfs-cli/modules/bearer/create.go @@ -15,10 +15,12 @@ import ( eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/spf13/cobra" + "github.com/spf13/viper" ) const ( eaclFlag = "eacl" + apeFlag = "ape" issuedAtFlag = "issued-at" notValidBeforeFlag = "not-valid-before" ownerFlag = "owner" @@ -37,10 +39,17 @@ In this case --` + commonflags.RPC + ` flag should be specified and the epoch in is set to current epoch + n. `, Run: createToken, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + ff := cmd.Flags() + + _ = viper.BindPFlag(commonflags.WalletPath, ff.Lookup(commonflags.WalletPath)) + _ = viper.BindPFlag(commonflags.Account, ff.Lookup(commonflags.Account)) + }, } func init() { - createCmd.Flags().StringP(eaclFlag, "e", "", "Path to the extended ACL table (mutually exclusive with --impersonate flag)") + createCmd.Flags().StringP(eaclFlag, "e", "", "Path to the extended ACL table (mutually exclusive with --impersonate and --ape flag)") + createCmd.Flags().StringP(apeFlag, "a", "", "Path to the JSON-encoded APE override (mutually exclusive with --impersonate and --eacl flag)") createCmd.Flags().StringP(issuedAtFlag, "i", "+0", "Epoch to issue token at") createCmd.Flags().StringP(notValidBeforeFlag, "n", "+0", "Not valid before epoch") createCmd.Flags().StringP(commonflags.ExpireAt, "x", "", "The last active epoch for the token") @@ -49,10 +58,13 @@ func init() { createCmd.Flags().Bool(jsonFlag, false, "Output token in JSON") createCmd.Flags().Bool(impersonateFlag, false, "Mark token as impersonate to consider the token signer as the request owner (mutually exclusive with --eacl flag)") createCmd.Flags().StringP(commonflags.RPC, commonflags.RPCShorthand, commonflags.RPCDefault, commonflags.RPCUsage) + createCmd.Flags().StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, commonflags.WalletPathDefault, commonflags.WalletPathUsage) + createCmd.Flags().StringP(commonflags.Account, commonflags.AccountShorthand, commonflags.AccountDefault, commonflags.AccountUsage) - createCmd.MarkFlagsMutuallyExclusive(eaclFlag, impersonateFlag) + createCmd.MarkFlagsMutuallyExclusive(eaclFlag, apeFlag, impersonateFlag) _ = cobra.MarkFlagFilename(createCmd.Flags(), eaclFlag) + _ = cobra.MarkFlagFilename(createCmd.Flags(), apeFlag) _ = cobra.MarkFlagRequired(createCmd.Flags(), commonflags.ExpireAt) _ = cobra.MarkFlagRequired(createCmd.Flags(), ownerFlag) @@ -119,6 +131,14 @@ func createToken(cmd *cobra.Command, _ []string) { b.SetEACLTable(*table) } + apePath, _ := cmd.Flags().GetString(apeFlag) + if apePath != "" { + var apeOverride bearer.APEOverride + raw, err := os.ReadFile(apePath) + commonCmd.ExitOnErr(cmd, "can't read APE rules: %w", err) + commonCmd.ExitOnErr(cmd, "can't parse APE rules: %w", json.Unmarshal(raw, &apeOverride)) + b.SetAPEOverride(apeOverride) + } var data []byte toJSON, _ := cmd.Flags().GetBool(jsonFlag) diff --git a/cmd/frostfs-cli/modules/bearer/generate_override.go b/cmd/frostfs-cli/modules/bearer/generate_override.go new file mode 100644 index 000000000..482c0027e --- /dev/null +++ b/cmd/frostfs-cli/modules/bearer/generate_override.go @@ -0,0 +1,115 @@ +package bearer + +import ( + "errors" + "fmt" + "os" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "github.com/spf13/cobra" +) + +var ( + errChainIDCannotBeEmpty = errors.New("chain id cannot be empty") + errRuleIsNotParsed = errors.New("rule is not passed") +) + +const ( + chainIDFlag = "chain-id" + chainIDHexFlag = "chain-id-hex" + ruleFlag = "rule" + pathFlag = "path" + outputFlag = "output" +) + +var generateAPEOverrideCmd = &cobra.Command{ + Use: "generate-ape-override", + Short: "Generate APE override.", + Long: `Generate APE override by target and APE chains. Util command. + +Generated APE override can be dumped to a file in JSON format that is passed to +"create" command. +`, + Run: genereateAPEOverride, +} + +func genereateAPEOverride(cmd *cobra.Command, _ []string) { + c := parseChain(cmd) + + targetCID, _ := cmd.Flags().GetString(commonflags.CIDFlag) + var cid cidSDK.ID + commonCmd.ExitOnErr(cmd, "invalid cid format: %w", cid.DecodeString(targetCID)) + + override := &bearer.APEOverride{ + Target: apeSDK.ChainTarget{ + TargetType: apeSDK.TargetTypeContainer, + Name: targetCID, + }, + Chains: []apeSDK.Chain{ + { + Raw: c.Bytes(), + }, + }, + } + + overrideMarshalled, err := override.MarshalJSON() + commonCmd.ExitOnErr(cmd, "failed to marshal APE override: %w", err) + + outputPath, _ := cmd.Flags().GetString(outputFlag) + if outputPath != "" { + err := os.WriteFile(outputPath, []byte(overrideMarshalled), 0o644) + commonCmd.ExitOnErr(cmd, "dump error: %w", err) + } else { + fmt.Print("\n") + fmt.Println(string(overrideMarshalled)) + } +} + +func init() { + ff := generateAPEOverrideCmd.Flags() + + ff.StringP(commonflags.CIDFlag, "", "", "Target container ID.") + _ = cobra.MarkFlagRequired(createCmd.Flags(), commonflags.CIDFlag) + + ff.StringArray(ruleFlag, []string{}, "Rule statement") + ff.String(pathFlag, "", "Path to encoded chain in JSON or binary format") + ff.String(chainIDFlag, "", "Assign ID to the parsed chain") + ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") + + ff.String(outputFlag, "", "Output path to dump result JSON-encoded APE override") + _ = cobra.MarkFlagFilename(createCmd.Flags(), outputFlag) +} + +func parseChainID(cmd *cobra.Command) apechain.ID { + chainID, _ := cmd.Flags().GetString(chainIDFlag) + if chainID == "" { + commonCmd.ExitOnErr(cmd, "read chain id error: %w", + errChainIDCannotBeEmpty) + } + return apechain.ID(chainID) +} + +func parseChain(cmd *cobra.Command) *apechain.Chain { + chain := new(apechain.Chain) + + if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 { + commonCmd.ExitOnErr(cmd, "parser error: %w", parseutil.ParseAPEChain(chain, rules)) + } else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" { + commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", parseutil.ParseAPEChainBinaryOrJSON(chain, encPath)) + } else { + commonCmd.ExitOnErr(cmd, "parser error: %w", errRuleIsNotParsed) + } + + chain.ID = parseChainID(cmd) + + cmd.Println("Parsed chain:") + parseutil.PrintHumanReadableAPEChain(cmd, chain) + + return chain +} diff --git a/cmd/frostfs-cli/modules/bearer/root.go b/cmd/frostfs-cli/modules/bearer/root.go index 200d096ac..fa6aef6fb 100644 --- a/cmd/frostfs-cli/modules/bearer/root.go +++ b/cmd/frostfs-cli/modules/bearer/root.go @@ -11,4 +11,5 @@ var Cmd = &cobra.Command{ func init() { Cmd.AddCommand(createCmd) + Cmd.AddCommand(generateAPEOverrideCmd) } From e7d479f4c2bb0af1b1204a6acf13ce72a8b4f1ea Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jun 2024 19:00:07 +0300 Subject: [PATCH 1146/1943] [#1166] blobovnicza: Return NoSpaceLeft error instead of syscall.ENOSPC Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobovnicza/delete.go | 6 ++++++ pkg/local_object_storage/blobovnicza/errors.go | 6 ++++++ pkg/local_object_storage/blobovnicza/move.go | 15 +++++++++++++-- pkg/local_object_storage/blobovnicza/put.go | 4 ++++ 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 pkg/local_object_storage/blobovnicza/errors.go diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index ab8575ad7..5d6787897 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -2,6 +2,8 @@ package blobovnicza import ( "context" + "errors" + "syscall" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" @@ -97,5 +99,9 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err b.itemDeleted(recordSize) } + if errors.Is(err, syscall.ENOSPC) { + err = ErrNoSpace + } + return DeleteRes{}, err } diff --git a/pkg/local_object_storage/blobovnicza/errors.go b/pkg/local_object_storage/blobovnicza/errors.go new file mode 100644 index 000000000..cff8c1776 --- /dev/null +++ b/pkg/local_object_storage/blobovnicza/errors.go @@ -0,0 +1,6 @@ +package blobovnicza + +import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + +// ErrNoSpace returned if blobovnicza failed to perform an operation because of syscall.ENOSPC. +var ErrNoSpace = logicerr.New("no space left on device with blobovnicza") diff --git a/pkg/local_object_storage/blobovnicza/move.go b/pkg/local_object_storage/blobovnicza/move.go index 255198f66..420e22a48 100644 --- a/pkg/local_object_storage/blobovnicza/move.go +++ b/pkg/local_object_storage/blobovnicza/move.go @@ -2,7 +2,9 @@ package blobovnicza import ( "context" + "errors" "fmt" + "syscall" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -29,7 +31,7 @@ func (b *Blobovnicza) PutMoveInfo(ctx context.Context, prm MoveInfo) error { key := addressKey(prm.Address) - return b.boltDB.Update(func(tx *bbolt.Tx) error { + err := b.boltDB.Update(func(tx *bbolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists(incompletedMoveBucketName) if err != nil { return err @@ -41,6 +43,11 @@ func (b *Blobovnicza) PutMoveInfo(ctx context.Context, prm MoveInfo) error { return nil }) + + if errors.Is(err, syscall.ENOSPC) { + err = ErrNoSpace + } + return err } func (b *Blobovnicza) DropMoveInfo(ctx context.Context, address oid.Address) error { @@ -53,7 +60,7 @@ func (b *Blobovnicza) DropMoveInfo(ctx context.Context, address oid.Address) err key := addressKey(address) - return b.boltDB.Update(func(tx *bbolt.Tx) error { + err := b.boltDB.Update(func(tx *bbolt.Tx) error { bucket := tx.Bucket(incompletedMoveBucketName) if bucket == nil { return nil @@ -72,6 +79,10 @@ func (b *Blobovnicza) DropMoveInfo(ctx context.Context, address oid.Address) err return nil }) + if errors.Is(err, syscall.ENOSPC) { + err = ErrNoSpace + } + return err } func (b *Blobovnicza) ListMoveInfo(ctx context.Context) ([]MoveInfo, error) { diff --git a/pkg/local_object_storage/blobovnicza/put.go b/pkg/local_object_storage/blobovnicza/put.go index 9af8a93fe..ff223ba36 100644 --- a/pkg/local_object_storage/blobovnicza/put.go +++ b/pkg/local_object_storage/blobovnicza/put.go @@ -2,7 +2,9 @@ package blobovnicza import ( "context" + "errors" "fmt" + "syscall" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -95,6 +97,8 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) { }) if err == nil { b.itemAdded(recordSize) + } else if errors.Is(err, syscall.ENOSPC) { + err = ErrNoSpace } return PutRes{}, err From 6cf512e574694cdbdcbc88f114b8a45a4843c526 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Jun 2024 19:01:25 +0300 Subject: [PATCH 1147/1943] [#1166] blobovniczatree: Handle blobovnicza's NoSpaceLeft error Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobstor/blobovniczatree/put.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 96c4db67d..76c4953e4 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -2,6 +2,7 @@ package blobovniczatree import ( "context" + "errors" "path/filepath" "time" @@ -108,7 +109,9 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } - + if errors.Is(err, blobovnicza.ErrNoSpace) { + i.AllFull = true + } return false, nil } From 069c1559ccd083cac87fb36d7b0cfc1168a5e393 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 7 Jun 2024 16:18:31 +0300 Subject: [PATCH 1148/1943] [#1164] cli: Improve `object nodes` performance Do complex EC object parts flatten concurrently. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 79 ++++++++++++++++++------- 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index b5f839eb8..cdce7cd7b 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -33,7 +33,10 @@ const ( verifyPresenceAllFlag = "verify-presence-all" ) -var errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint") +var ( + errNoAvailableEndpoint = errors.New("failed to create client: no available endpoint") + errMalformedComplexObject = errors.New("object consists of EC and non EC parts") +) type phyObject struct { containerID cid.ID @@ -180,30 +183,66 @@ func getCompexObjectMembers(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, members []oid.ID, prmHead internalclient.HeadObjectPrm) []phyObject { result := make([]phyObject, 0, len(members)) + var resultGuard sync.Mutex + + if len(members) == 0 { + return result + } + + prmHead.SetRawFlag(true) // to get an error instead of whole object + + first := members[0] var addrObj oid.Address addrObj.SetContainer(cnrID) - prmHead.SetRawFlag(true) // to get an error instead of whole object - for _, partObjID := range members { - addrObj.SetObject(partObjID) - prmHead.SetAddress(addrObj) + addrObj.SetObject(first) + prmHead.SetAddress(addrObj) - _, err := internalclient.HeadObject(cmd.Context(), prmHead) - var ecInfoError *objectSDK.ECInfoError - if errors.As(err, &ecInfoError) { - chunks := getECObjectChunks(cmd, cnrID, partObjID, ecInfoError) - result = append(result, chunks...) - continue - } else if err == nil { // not EC object, so all members must be phy objects - for _, member := range members { - result = append(result, phyObject{ - containerID: cnrID, - objectID: member, - }) - } - break + _, err := internalclient.HeadObject(cmd.Context(), prmHead) + var ecInfoError *objectSDK.ECInfoError + if errors.As(err, &ecInfoError) { + chunks := getECObjectChunks(cmd, cnrID, first, ecInfoError) + result = append(result, chunks...) + } else if err == nil { // not EC object, so all members must be phy objects + for _, member := range members { + result = append(result, phyObject{ + containerID: cnrID, + objectID: member, + }) } - commonCmd.ExitOnErr(cmd, "failed to read EC chunk of complex object: %w", err) + return result + } else { + commonCmd.ExitOnErr(cmd, "failed to flatten parts of complex object: %w", err) } + + eg, egCtx := errgroup.WithContext(cmd.Context()) + for idx := 1; idx < len(members); idx++ { + partObjID := members[idx] + + eg.Go(func() error { + partHeadPrm := prmHead + var partAddr oid.Address + partAddr.SetContainer(cnrID) + partAddr.SetObject(partObjID) + partHeadPrm.SetAddress(partAddr) + + _, err := internalclient.HeadObject(egCtx, partHeadPrm) + var ecInfoError *objectSDK.ECInfoError + if errors.As(err, &ecInfoError) { + chunks := getECObjectChunks(cmd, cnrID, partObjID, ecInfoError) + + resultGuard.Lock() + defer resultGuard.Unlock() + result = append(result, chunks...) + + return nil + } else if err == nil { + return errMalformedComplexObject + } + return err + }) + } + + commonCmd.ExitOnErr(cmd, "failed to flatten parts of complex object: %w", eg.Wait()) return result } From b9fcaad21f03da4bf1562700c76b6e61e5cdc47e Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 10 Jun 2024 17:48:05 +0300 Subject: [PATCH 1149/1943] [#1168] shard: Set `Disabled` as default `mode` for components Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/blobstor/blobstor.go | 1 + pkg/local_object_storage/metabase/db.go | 1 + pkg/local_object_storage/pilorama/boltdb.go | 1 + pkg/local_object_storage/writecache/cachebbolt.go | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index bc9ab2b99..6f579a8ca 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -54,6 +54,7 @@ func initConfig(c *cfg) { // New creates, initializes and returns new BlobStor instance. func New(opts ...Option) *BlobStor { bs := new(BlobStor) + bs.mode = mode.Disabled initConfig(&bs.cfg) for i := range opts { diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index f2203f4d9..1f444a3ef 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -107,6 +107,7 @@ func New(opts ...Option) *DB { matchBucket: stringCommonPrefixMatcherBucket, }, }, + mode: mode.Disabled, } } diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index fe6bf4fed..d4ae35ff4 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -81,6 +81,7 @@ func NewBoltForest(opts ...Option) ForestStorage { openFile: os.OpenFile, metrics: &noopMetrics{}, }, + mode: mode.Disabled, } for i := range opts { diff --git a/pkg/local_object_storage/writecache/cachebbolt.go b/pkg/local_object_storage/writecache/cachebbolt.go index f2b48d746..254d8754f 100644 --- a/pkg/local_object_storage/writecache/cachebbolt.go +++ b/pkg/local_object_storage/writecache/cachebbolt.go @@ -60,7 +60,7 @@ var defaultBucket = []byte{0} func New(opts ...Option) Cache { c := &cache{ flushCh: make(chan objectInfo), - mode: mode.ReadWrite, + mode: mode.Disabled, compressFlags: make(map[string]struct{}), options: options{ From d4f425f86ab690935788db6f4268c297624abc76 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 7 Jun 2024 16:22:46 +0300 Subject: [PATCH 1150/1943] [#1167] adm/morph: Fix set-config parameter validation Signed-off-by: Alexander Chuprov --- .../internal/modules/morph/config/config.go | 27 +++++++++++---- .../modules/morph/config/config_test.go | 34 +++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/config/config_test.go diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index f35e7aa03..bfcd4ac00 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -147,13 +147,28 @@ func validateConfig(args map[string]any, forceFlag bool) error { } for k, v := range args { - value, ok := v.(int64) - if !ok || value < 0 { - return fmt.Errorf("%s must be >= 0, got %v", k, v) - } + switch k { + case netmap.ContainerFeeConfig, netmap.ContainerAliasFeeConfig, + netmap.EpochDurationConfig, netmap.IrCandidateFeeConfig, + netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig, + netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig: + value, ok := v.(int64) + if !ok { + return fmt.Errorf("%s has an invalid type. Expected type: int", k) + } - if k == netmap.MaxECDataCountConfig || k == netmap.MaxECParityCountConfig { - sumEC += value + if value < 0 { + return fmt.Errorf("%s must be >= 0, got %v", k, v) + } + + if k == netmap.MaxECDataCountConfig || k == netmap.MaxECParityCountConfig { + sumEC += value + } + case netmap.HomomorphicHashingDisabledKey, netmap.MaintenanceModeAllowedConfig: + _, ok := v.(bool) + if !ok { + return fmt.Errorf("%s has an invalid type. Expected type: bool", k) + } } } diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config_test.go b/cmd/frostfs-adm/internal/modules/morph/config/config_test.go new file mode 100644 index 000000000..c6d5b2827 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/config/config_test.go @@ -0,0 +1,34 @@ +package config + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + "github.com/stretchr/testify/require" +) + +func Test_ValidateConfig(t *testing.T) { + testArgs := make(map[string]any) + + testArgs[netmap.MaxECDataCountConfig] = int64(11) + require.Error(t, validateConfig(testArgs, false)) + + testArgs[netmap.MaxECParityCountConfig] = int64(256) + require.Error(t, validateConfig(testArgs, false)) + require.NoError(t, validateConfig(testArgs, true)) + + testArgs[netmap.MaxECParityCountConfig] = int64(-1) + require.Error(t, validateConfig(testArgs, false)) + + testArgs[netmap.MaxECParityCountConfig] = int64(55) + require.NoError(t, validateConfig(testArgs, false)) + + testArgs[netmap.HomomorphicHashingDisabledKey] = "1" + require.Error(t, validateConfig(testArgs, false)) + + testArgs[netmap.HomomorphicHashingDisabledKey] = true + require.NoError(t, validateConfig(testArgs, false)) + + testArgs["not-well-known-configuration-key"] = "key" + require.NoError(t, validateConfig(testArgs, false)) +} From 41e670c9ba6ec026c4c00b8267e6950eb0cb494f Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 7 Jun 2024 16:33:03 +0300 Subject: [PATCH 1151/1943] [#1167] adm/morph: Move literal to const Signed-off-by: Alexander Chuprov --- cmd/frostfs-adm/internal/modules/morph/config/config.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index bfcd4ac00..ba6e515c1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -137,6 +137,8 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { return wCtx.AwaitTx() } +const maxECSum = 256 + func validateConfig(args map[string]any, forceFlag bool) error { var sumEC int64 _, okData := args[netmap.MaxECDataCountConfig] @@ -172,9 +174,9 @@ func validateConfig(args map[string]any, forceFlag bool) error { } } - if sumEC > 256 && !forceFlag { - return fmt.Errorf("the sum of %s and %s must be <= 256, got %d", - netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig, sumEC) + if sumEC > maxECSum && !forceFlag { + return fmt.Errorf("the sum of %s and %s must be <= %d, got %d", + netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig, maxECSum, sumEC) } return nil } From 9d73f9c2c6571c0be0f64d50b235074b3e7b45ab Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Jun 2024 17:05:21 +0300 Subject: [PATCH 1152/1943] Reapply "[#446] engine: Move to read-only on blobstor errors" Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/engine.go | 33 +++++++++++-------- pkg/local_object_storage/engine/error_test.go | 2 +- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index b8ac6cf97..b87d77e6c 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -49,6 +50,7 @@ type shardWrapper struct { type setModeRequest struct { sh *shard.Shard + isMeta bool errorCount uint32 } @@ -74,7 +76,7 @@ func (e *StorageEngine) setModeLoop() { if !ok { inProgress[sid] = struct{}{} go func() { - e.moveToDegraded(r.sh, r.errorCount) + e.moveToDegraded(r.sh, r.errorCount, r.isMeta) mtx.Lock() delete(inProgress, sid) @@ -86,7 +88,7 @@ func (e *StorageEngine) setModeLoop() { } } -func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { +func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) { sid := sh.ID() log := e.log.With( zap.Stringer("shard_id", sid), @@ -95,21 +97,23 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) { e.mtx.RLock() defer e.mtx.RUnlock() - err := sh.SetMode(mode.DegradedReadOnly) - if err != nil { + if isMeta { + err := sh.SetMode(mode.DegradedReadOnly) + if err == nil { + log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) + return + } log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, zap.Error(err)) - - err = sh.SetMode(mode.ReadOnly) - if err != nil { - log.Error(logs.EngineFailedToMoveShardInReadonlyMode, - zap.Error(err)) - } else { - log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) - } - } else { - log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) } + + err := sh.SetMode(mode.ReadOnly) + if err != nil { + log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) + return + } + + log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } // reportShardErrorBackground increases shard error counter and logs an error. @@ -177,6 +181,7 @@ func (e *StorageEngine) reportShardErrorWithFlags( req := setModeRequest{ errorCount: errCount, sh: sh, + isMeta: errors.As(err, new(metaerr.Error)), } select { diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 6c44966f2..535435ceb 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -154,7 +154,7 @@ func TestErrorReporting(t *testing.T) { for i := uint32(0); i < 2; i++ { _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) - checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly) + checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly) checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } From 6a39c3d15e9be1f78ea16f622de0d41a94dc5a81 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 17 Apr 2024 14:15:08 +0300 Subject: [PATCH 1153/1943] [#1086] engine: Do not use metabase if shard looks bad Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/head.go | 1 + pkg/local_object_storage/shard/head.go | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index 1c61ca455..97fb5be3a 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -84,6 +84,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) shPrm.SetRaw(prm.raw) e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { + shPrm.ShardLooksBad = sh.errorCount.Load() >= e.errorsThreshold res, err := sh.Head(ctx, shPrm) if err != nil { switch { diff --git a/pkg/local_object_storage/shard/head.go b/pkg/local_object_storage/shard/head.go index a0ec231af..9d5d31260 100644 --- a/pkg/local_object_storage/shard/head.go +++ b/pkg/local_object_storage/shard/head.go @@ -13,8 +13,9 @@ import ( // HeadPrm groups the parameters of Head operation. type HeadPrm struct { - addr oid.Address - raw bool + addr oid.Address + raw bool + ShardLooksBad bool } // HeadRes groups the resulting values of Head operation. @@ -59,7 +60,8 @@ func (s *Shard) Head(ctx context.Context, prm HeadPrm) (HeadRes, error) { var obj *objectSDK.Object var err error - if s.GetMode().NoMetabase() { + mode := s.GetMode() + if mode.NoMetabase() || (mode.ReadOnly() && prm.ShardLooksBad) { var getPrm GetPrm getPrm.SetAddress(prm.addr) getPrm.SetIgnoreMeta(true) From 68ac490729e4a436926c1f9aa81219452eea2950 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 13 Jun 2024 10:10:55 +0300 Subject: [PATCH 1154/1943] [#1174] shard: Update metric `mode_info` on `Init` Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/shard/shard.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index cf59c0bfc..94f22feb5 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -484,6 +484,7 @@ func (s *Shard) updateMetrics(ctx context.Context) { s.setContainerObjectsCount(contID.EncodeToString(), logical, count.Logic) s.setContainerObjectsCount(contID.EncodeToString(), user, count.User) } + s.cfg.metricsWriter.SetMode(s.info.Mode) } // incObjectCounter increment both physical and logical object From 42ecc2f2b9c7750a864b6388c1d29c9802ca4fb1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 10 Jun 2024 11:48:14 +0300 Subject: [PATCH 1155/1943] [#1170] morph: Support mTLS Signed-off-by: Evgenii Stratonikov --- go.mod | 2 ++ go.sum | 4 ++-- pkg/morph/client/constructor.go | 13 +++++++++---- pkg/morph/client/mtls.go | 22 ++++++++++++++++++++++ pkg/morph/client/multi.go | 13 +++++++------ 5 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 pkg/morph/client/mtls.go diff --git a/go.mod b/go.mod index 42896ed8c..4ce8a0d27 100644 --- a/go.mod +++ b/go.mod @@ -127,3 +127,5 @@ require ( lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) + +replace github.com/nspcc-dev/neo-go => git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 diff --git a/go.sum b/go.sum index 07632a032..cb97af3d4 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= +git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 h1:JRjwcHaQajTbSCBCK3yZnqvyHvgWBaoThDGuT4kvIIc= +git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d h1:hHk8FWbWgEnwm2I045CaBIrZBjy/o81CehIVOySA/pQ= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= @@ -186,8 +188,6 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc= -github.com/nspcc-dev/neo-go v0.106.0 h1:YiOdW/GcLmbVSvxMRfD5aytO6n3TDHrEA97VHMawy6g= -github.com/nspcc-dev/neo-go v0.106.0/go.mod h1:9k7vBqqQeePuj4c2CqMN9uPFsfxvwSk9vnQFrkE5eg8= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d h1:Vcb7YkZuUSSIC+WF/xV3UDfHbAxZgyT2zGleJP3Ig5k= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d/go.mod h1:/vrbWSHc7YS1KSYhVOyyeucXW/e+1DkVBOgnBEXUCeY= github.com/nspcc-dev/rfc6979 v0.2.1 h1:8wWxkamHWFmO790GsewSoKUSJjVnL1fmdRpokU/RgRM= diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 50a9572d4..c499aa892 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -141,7 +141,7 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } else { var endpoint Endpoint for cli.endpoints.curr, endpoint = range cli.endpoints.list { - cli.client, act, err = cli.newCli(ctx, endpoint.Address) + cli.client, act, err = cli.newCli(ctx, endpoint) if err != nil { cli.logger.Warn(logs.FrostFSIRCouldntCreateRPCClientForEndpoint, zap.Error(err), zap.String("endpoint", endpoint.Address)) @@ -162,10 +162,15 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er return cli, nil } -func (c *Client) newCli(ctx context.Context, endpoint string) (*rpcclient.WSClient, *actor.Actor, error) { - cli, err := rpcclient.NewWS(ctx, endpoint, rpcclient.WSOptions{ +func (c *Client) newCli(ctx context.Context, endpoint Endpoint) (*rpcclient.WSClient, *actor.Actor, error) { + cfg, err := endpoint.MTLSConfig.parse() + if err != nil { + return nil, nil, fmt.Errorf("read mtls certificates: %w", err) + } + cli, err := rpcclient.NewWS(ctx, endpoint.Address, rpcclient.WSOptions{ Options: rpcclient.Options{ - DialTimeout: c.cfg.dialTimeout, + DialTimeout: c.cfg.dialTimeout, + TLSClientConfig: cfg, }, }) if err != nil { diff --git a/pkg/morph/client/mtls.go b/pkg/morph/client/mtls.go new file mode 100644 index 000000000..3de51afe7 --- /dev/null +++ b/pkg/morph/client/mtls.go @@ -0,0 +1,22 @@ +package client + +import ( + "crypto/tls" + + "github.com/nspcc-dev/neo-go/pkg/rpcclient" +) + +// MTLSConfig represents endpoint mTLS configuration. +type MTLSConfig struct { + TrustedCAList []string + KeyFile string + CertFile string +} + +func (m *MTLSConfig) parse() (*tls.Config, error) { + if m == nil { + return nil, nil + } + + return rpcclient.TLSClientConfig(m.TrustedCAList, m.CertFile, m.KeyFile) +} diff --git a/pkg/morph/client/multi.go b/pkg/morph/client/multi.go index 25ec626d1..10ed21582 100644 --- a/pkg/morph/client/multi.go +++ b/pkg/morph/client/multi.go @@ -11,8 +11,9 @@ import ( // Endpoint represents morph endpoint together with its priority. type Endpoint struct { - Address string - Priority int + Address string + Priority int + MTLSConfig *MTLSConfig } type endpoints struct { @@ -38,11 +39,11 @@ func (c *Client) SwitchRPC(ctx context.Context) bool { // Iterate endpoints in the order of decreasing priority. for c.endpoints.curr = range c.endpoints.list { - newEndpoint := c.endpoints.list[c.endpoints.curr].Address + newEndpoint := c.endpoints.list[c.endpoints.curr] cli, act, err := c.newCli(ctx, newEndpoint) if err != nil { c.logger.Warn(logs.ClientCouldNotEstablishConnectionToTheSwitchedRPCNode, - zap.String("endpoint", newEndpoint), + zap.String("endpoint", newEndpoint.Address), zap.Error(err), ) @@ -52,7 +53,7 @@ func (c *Client) SwitchRPC(ctx context.Context) bool { c.cache.invalidate() c.logger.Info(logs.ClientConnectionToTheNewRPCNodeHasBeenEstablished, - zap.String("endpoint", newEndpoint)) + zap.String("endpoint", newEndpoint.Address)) c.client = cli c.setActor(act) @@ -119,7 +120,7 @@ mainLoop: tryE := e.Address - cli, act, err := c.newCli(ctx, tryE) + cli, act, err := c.newCli(ctx, e) if err != nil { c.logger.Warn(logs.ClientCouldNotCreateClientToTheHigherPriorityNode, zap.String("endpoint", tryE), From a0e49fa5a5b33359ef2def3777aac250022dc225 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Jun 2024 15:46:10 +0300 Subject: [PATCH 1156/1943] [#1170] adm: Support morph mTLS Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/helper/n3client.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go index 55449b435..e62a21b3f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go @@ -2,6 +2,7 @@ package helper import ( "context" + "crypto/tls" "errors" "fmt" "time" @@ -60,9 +61,23 @@ func GetN3Client(v *viper.Viper) (Client, error) { if endpoint == "" { return nil, errors.New("missing endpoint") } + + var cfg *tls.Config + if rootCAs := v.GetStringSlice("tls.trusted_ca_list"); len(rootCAs) != 0 { + certFile := v.GetString("tls.certificate") + keyFile := v.GetString("tls.key") + + tlsConfig, err := rpcclient.TLSClientConfig(rootCAs, certFile, keyFile) + if err != nil { + return nil, err + } + + cfg = tlsConfig + } c, err := rpcclient.New(ctx, endpoint, rpcclient.Options{ MaxConnsPerHost: maxConnsPerHost, RequestTimeout: requestTimeout, + TLSClientConfig: cfg, }) if err != nil { return nil, err From 96fe271babcd538015ff04968f1cc1b7ff913c56 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Jun 2024 15:47:37 +0300 Subject: [PATCH 1157/1943] [#1170] innerring: Support morph mTLS Signed-off-by: Evgenii Stratonikov --- pkg/innerring/innerring.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 5dea18f9d..86424abef 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -463,9 +463,20 @@ func createClient(ctx context.Context, p *chainParams, errChan chan<- error) (*c priority = defaultPriority } + var mtlsConfig *client.MTLSConfig + rootCAs := p.cfg.GetStringSlice(fmt.Sprintf("%s.%d.trusted_ca_list", section, i)) + if len(rootCAs) != 0 { + mtlsConfig = &client.MTLSConfig{ + TrustedCAList: rootCAs, + KeyFile: p.cfg.GetString(fmt.Sprintf("%s.%d.key", section, i)), + CertFile: p.cfg.GetString(fmt.Sprintf("%s.%d.certificate", section, i)), + } + } + endpoints = append(endpoints, client.Endpoint{ - Address: addr, - Priority: priority, + Address: addr, + Priority: priority, + MTLSConfig: mtlsConfig, }) } From dbd3b238f75eab70d50cea01569269d505cb2b13 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 11 Jun 2024 15:48:06 +0300 Subject: [PATCH 1158/1943] [#1170] node: Support morph mTLS Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config/morph/config.go | 15 +++++++++++++-- cmd/frostfs-node/config/morph/config_test.go | 7 +++++++ config/example/node.env | 3 +++ config/example/node.json | 7 ++++++- config/example/node.yaml | 4 ++++ 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-node/config/morph/config.go b/cmd/frostfs-node/config/morph/config.go index b0c6527db..1c536a0e2 100644 --- a/cmd/frostfs-node/config/morph/config.go +++ b/cmd/frostfs-node/config/morph/config.go @@ -54,9 +54,20 @@ func RPCEndpoint(c *config.Config) []client.Endpoint { priority = PriorityDefault } + var mtlsConfig *client.MTLSConfig + rootCAs := config.StringSliceSafe(s, "trusted_ca_list") + if len(rootCAs) != 0 { + mtlsConfig = &client.MTLSConfig{ + TrustedCAList: rootCAs, + KeyFile: config.StringSafe(s, "key"), + CertFile: config.StringSafe(s, "certificate"), + } + } + es = append(es, client.Endpoint{ - Address: addr, - Priority: priority, + Address: addr, + Priority: priority, + MTLSConfig: mtlsConfig, }) } diff --git a/cmd/frostfs-node/config/morph/config_test.go b/cmd/frostfs-node/config/morph/config_test.go index 13da3d70d..5a021abc3 100644 --- a/cmd/frostfs-node/config/morph/config_test.go +++ b/cmd/frostfs-node/config/morph/config_test.go @@ -28,6 +28,13 @@ func TestMorphSection(t *testing.T) { { Address: "wss://rpc1.morph.frostfs.info:40341/ws", Priority: 1, + MTLSConfig: &client.MTLSConfig{ + TrustedCAList: []string{ + "/path/to/ca.pem", + }, + KeyFile: "/path/to/key", + CertFile: "/path/to/cert", + }, }, { Address: "wss://rpc2.morph.frostfs.info:40341/ws", diff --git a/config/example/node.env b/config/example/node.env index 9f15c4042..eedbe501d 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -61,6 +61,9 @@ FROSTFS_MORPH_CACHE_TTL=15s FROSTFS_MORPH_SWITCH_INTERVAL=3m FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS="wss://rpc1.morph.frostfs.info:40341/ws" FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY=0 +FROSTFS_MORPH_RPC_ENDPOINT_0_TRUSTED_CA_LIST="/path/to/ca.pem" +FROSTFS_MORPH_RPC_ENDPOINT_0_CERTIFICATE="/path/to/cert" +FROSTFS_MORPH_RPC_ENDPOINT_0_KEY="/path/to/key" FROSTFS_MORPH_RPC_ENDPOINT_1_ADDRESS="wss://rpc2.morph.frostfs.info:40341/ws" FROSTFS_MORPH_RPC_ENDPOINT_1_PRIORITY=2 FROSTFS_MORPH_APE_CHAIN_CACHE_SIZE=100000 diff --git a/config/example/node.json b/config/example/node.json index 79e6fe89a..2589f2c34 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -95,7 +95,12 @@ "rpc_endpoint": [ { "address": "wss://rpc1.morph.frostfs.info:40341/ws", - "priority": 0 + "priority": 0, + "trusted_ca_list": [ + "/path/to/ca.pem" + ], + "certificate": "/path/to/cert", + "key": "/path/to/key" }, { "address": "wss://rpc2.morph.frostfs.info:40341/ws", diff --git a/config/example/node.yaml b/config/example/node.yaml index 34e796ac4..1a9516ef3 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -84,6 +84,10 @@ morph: rpc_endpoint: # side chain NEO RPC endpoints; are shuffled and used one by one until the first success - address: wss://rpc1.morph.frostfs.info:40341/ws priority: 0 + trusted_ca_list: + - "/path/to/ca.pem" + certificate: "/path/to/cert" + key: "/path/to/key" - address: wss://rpc2.morph.frostfs.info:40341/ws priority: 2 ape_chain_cache_size: 100000 From 3fc8e0e08c15927620c98b471c8ef05811a3127d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 14 Jun 2024 09:57:47 +0300 Subject: [PATCH 1159/1943] Release v0.41.0 Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 11 +++++++++++ VERSION | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0382d0316..2abba1d09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ Changelog for FrostFS Node ### Removed ### Updated +## [v0.41.0] + +### Added +- Support mTLS for morph client (#1170) + +### Fixed +- Update shard state metric during shard init (#1174) +- Handle ENOSPC in blobovnicza (#1166) +- Handle multiple split-infos for EC objects (#1163) +- Set `Disabled` mode as the default for components (#1168) + ## [v0.40.0] ### Added diff --git a/VERSION b/VERSION index 707d5fac2..9dedf1e9b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.40.0 +v0.41.0 From b9d6c9d10c793f021eea1d83ba019cec7fbfbdd2 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 14 Jun 2024 13:06:11 +0300 Subject: [PATCH 1160/1943] [#1177] cli: Fix resource name parsing * If `root` name is given explicitly, then it should be translated to `//` but not `/root/`. Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/util/ape.go | 2 +- cmd/frostfs-cli/modules/util/ape_test.go | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index 50253b366..680e24d82 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -279,7 +279,7 @@ func parseResource(lexeme string, isObj bool) (string, error) { if isObj { if lexeme == "*" { return nativeschema.ResourceFormatAllObjects, nil - } else if lexeme == "/*" { + } else if lexeme == "/*" || lexeme == "root/*" { return nativeschema.ResourceFormatRootObjects, nil } else if strings.HasPrefix(lexeme, "/") { lexeme = lexeme[1:] diff --git a/cmd/frostfs-cli/modules/util/ape_test.go b/cmd/frostfs-cli/modules/util/ape_test.go index d93210f41..b275803df 100644 --- a/cmd/frostfs-cli/modules/util/ape_test.go +++ b/cmd/frostfs-cli/modules/util/ape_test.go @@ -26,7 +26,7 @@ func TestParseAPERule(t *testing.T) { }, }, { - name: "Valid rule for all objects in root namespace", + name: "Valid rule for all objects in implicit root namespace", rule: "allow Object.Put /*", expectRule: policyengine.Rule{ Status: policyengine.Allow, @@ -34,6 +34,15 @@ func TestParseAPERule(t *testing.T) { Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, }, }, + { + name: "Valid rule for all objects in explicit root namespace", + rule: "allow Object.Put root/*", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, + }, + }, { name: "Valid rule for all objects in root namespace and container", rule: "allow Object.Put /cid/*", From 76cf7a051b9bd33997dfb21df46e727e3a6cda38 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 17 Jun 2024 09:47:53 +0300 Subject: [PATCH 1161/1943] [#1178] shard: Check metabase existence before read shard ID Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/shard_id.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/local_object_storage/metabase/shard_id.go b/pkg/local_object_storage/metabase/shard_id.go index 9b24e98d7..88446494e 100644 --- a/pkg/local_object_storage/metabase/shard_id.go +++ b/pkg/local_object_storage/metabase/shard_id.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "os" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" metamode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -25,6 +26,10 @@ func (db *DB) GetShardID(mode metamode.Mode) ([]byte, error) { defer db.modeMtx.Unlock() db.mode = mode + if _, err := os.Stat(db.info.Path); errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err := db.openDB(mode); err != nil { return nil, fmt.Errorf("failed to open metabase: %w", err) } From 5b100699d7c9b2446539cbf19518bca963c5fc35 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 18 Jun 2024 10:20:45 +0300 Subject: [PATCH 1162/1943] [#566] policer: Move isClientErrMaintenance to frostfs-sdk-go Signed-off-by: Ekaterina Lebedeva --- go.mod | 2 +- go.sum | 4 ++-- pkg/services/policer/check.go | 27 +-------------------------- pkg/services/policer/ec.go | 2 +- 4 files changed, 5 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 4ce8a0d27..a04b94220 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index cb97af3d4..0849b74a1 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f h1:vBLC1OSGMSn7lRJv/p1of0veifuBdZdztVrF9Vn+UFk= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240531132048-ebd8fcd1685f/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de h1:OjsWY0jpGJV1t87XgwL/3PsDx7fJ6lfNMXtY8UhoUbM= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 h1:JRjwcHaQajTbSCBCK3yZnqvyHvgWBaoThDGuT4kvIIc= diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 7120ba0b8..bf67ec4d4 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -2,7 +2,6 @@ package policer import ( "context" - "errors" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -11,7 +10,6 @@ import ( policycore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -139,7 +137,7 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe if client.IsErrObjectNotFound(err) { checkedNodes.submitReplicaCandidate(nodes[i]) continue - } else if isClientErrMaintenance(err) { + } else if client.IsErrNodeUnderMaintenance(err) { shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) } else { p.log.Error(logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, @@ -204,26 +202,3 @@ func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address requirements.removeLocalCopy = true } } - -// isClientErrMaintenance checks if err corresponds to FrostFS status return -// which tells that node is currently under maintenance. Supports wrapped -// errors. -// -// Similar to client.IsErr___ errors, consider replacing to FrostFS SDK. -func isClientErrMaintenance(err error) bool { - switch unwrapErr(err).(type) { - default: - return false - case *apistatus.NodeUnderMaintenance: - return true - } -} - -// unwrapErr unwraps error using errors.Unwrap. -func unwrapErr(err error) error { - for e := errors.Unwrap(err); e != nil; e = errors.Unwrap(err) { - err = e - } - - return err -} diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index b15efdd15..5b3793f7b 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -126,7 +126,7 @@ func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, n Nodes: []netmap.NodeInfo{requiredNode}, } p.replicator.HandleReplicationTask(ctx, task, newNodeCache()) - } else if isClientErrMaintenance(err) { + } else if client.IsErrNodeUnderMaintenance(err) { // consider maintenance mode has object, but do not drop local copy p.log.Debug(logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(requiredNode))) } else { From 75eedf71f31c8d0d68500779954e708fa884c3ee Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 18 Jun 2024 15:06:09 +0300 Subject: [PATCH 1163/1943] [#1187] pilorama/test: Remove debug print Introduced in e12fcc041d. Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- pkg/local_object_storage/pilorama/forest_test.go | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a04b94220..ddc193897 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/go-pkgz/expirable-cache/v3 v3.0.0 github.com/google/uuid v1.6.0 @@ -65,6 +64,7 @@ require ( github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 6c1bc343f..9c343f632 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -16,7 +16,6 @@ import ( cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - "github.com/davecgh/go-spew/spew" "github.com/google/uuid" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -271,7 +270,6 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, RootID, last, count) require.NoError(t, err) result = append(result, res...) - spew.Dump(last, res) return cursor } From 7b8937ec3555b950d4fa36428391f3e973d4f2c2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 18 Jun 2024 12:34:42 +0300 Subject: [PATCH 1164/1943] [#1184] config: Add audit.enabled parameter for node Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 10 +++++++ cmd/frostfs-node/config/audit/config.go | 12 +++++++++ cmd/frostfs-node/config/audit/config_test.go | 28 ++++++++++++++++++++ config/example/node.env | 3 +++ config/example/node.json | 3 +++ config/example/node.yaml | 3 +++ docs/storage-node-configuration.md | 13 +++++++++ 7 files changed, 72 insertions(+) create mode 100644 cmd/frostfs-node/config/audit/config.go create mode 100644 cmd/frostfs-node/config/audit/config_test.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index cdfa2118c..2b185cfc8 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -18,6 +18,7 @@ import ( netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/audit" contractsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/contracts" engineconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine" shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard" @@ -375,6 +376,7 @@ type internals struct { healthStatus *atomic.Int32 // is node under maintenance isMaintenance atomic.Bool + audit *atomic.Bool sdNotify bool } @@ -722,6 +724,9 @@ func initInternals(appCfg *config.Config, log *logger.Logger) internals { var healthStatus atomic.Int32 healthStatus.Store(int32(control.HealthStatus_HEALTH_STATUS_UNDEFINED)) + var auditRequests atomic.Bool + auditRequests.Store(audit.Enabled(appCfg)) + return internals{ done: make(chan struct{}), appCfg: appCfg, @@ -730,6 +735,7 @@ func initInternals(appCfg *config.Config, log *logger.Logger) internals { apiVersion: version.Current(), healthStatus: &healthStatus, sdNotify: initSdNotify(appCfg), + audit: &auditRequests, } } @@ -1278,6 +1284,10 @@ func (c *cfg) reloadConfig(ctx context.Context) { setRuntimeParameters(c) return nil }}) + components = append(components, dCmp{"audit", func() error { + c.audit.Store(audit.Enabled(c.appCfg)) + return nil + }}) components = append(components, dCmp{"pools", c.reloadPools}) components = append(components, dCmp{"tracing", func() error { updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg)) diff --git a/cmd/frostfs-node/config/audit/config.go b/cmd/frostfs-node/config/audit/config.go new file mode 100644 index 000000000..8f728c850 --- /dev/null +++ b/cmd/frostfs-node/config/audit/config.go @@ -0,0 +1,12 @@ +package audit + +import "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + +const ( + subsection = "audit" +) + +// Enabled returns the value of "enabled" config parameter from "audit" section. +func Enabled(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "enabled") +} diff --git a/cmd/frostfs-node/config/audit/config_test.go b/cmd/frostfs-node/config/audit/config_test.go new file mode 100644 index 000000000..7731cc8e6 --- /dev/null +++ b/cmd/frostfs-node/config/audit/config_test.go @@ -0,0 +1,28 @@ +package audit + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "github.com/stretchr/testify/require" +) + +func TestAuditSection(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + empty := configtest.EmptyConfig() + require.Equal(t, false, Enabled(empty)) + }) + + const path = "../../../../config/example/node" + + fileConfigTest := func(c *config.Config) { + require.Equal(t, true, Enabled(c)) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) +} diff --git a/config/example/node.env b/config/example/node.env index eedbe501d..72f56e96c 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -202,3 +202,6 @@ FROSTFS_TRACING_ENDPOINT="localhost" FROSTFS_TRACING_EXPORTER="otlp_grpc" FROSTFS_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 + +# AUDIT section +FROSTFS_AUDIT_ENABLED=true diff --git a/config/example/node.json b/config/example/node.json index 2589f2c34..b9dc6014c 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -260,5 +260,8 @@ }, "runtime": { "soft_memory_limit": 1073741824 + }, + "audit": { + "enabled": true } } diff --git a/config/example/node.yaml b/config/example/node.yaml index 1a9516ef3..bad67816a 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -234,3 +234,6 @@ tracing: runtime: soft_memory_limit: 1gb + +audit: + enabled: true diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 3a6969abd..5389bfbb5 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -25,6 +25,7 @@ There are some custom types used for brevity: | `replicator` | [Replicator service configuration](#replicator-section) | | `storage` | [Storage engine configuration](#storage-section) | | `runtime` | [Runtime configuration](#runtime-section) | +| `audit` | [Audit configuration](#audit-section) | # `control` section @@ -428,3 +429,15 @@ runtime: | Parameter | Type | Default value | Description | |---------------------|--------|---------------|--------------------------------------------------------------------------| | `soft_memory_limit` | `size` | 0 | Soft memory limit for the runtime. Zero or no value stands for no limit. If `GOMEMLIMIT` environment variable is set, the value from the configuration file will be ignored. | + +# `audit` section +Contains audit parameters. + +```yaml +audit: + enabled: true +``` + +| Parameter | Type | Default value | Description | +|---------------------|--------|---------------|---------------------------------------------------| +| `soft_memory_limit` | `bool` | false | If `true` then audit event logs will be recorded. | From ac1f0147476395617a35152c6c5b83f1ebf67bfb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 18 Jun 2024 12:38:39 +0300 Subject: [PATCH 1165/1943] [#1184] node: Add audit package Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- internal/audit/consts.go | 7 ++ internal/audit/request.go | 41 ++++++++++ internal/audit/target.go | 158 ++++++++++++++++++++++++++++++++++++++ internal/logs/logs.go | 1 + 5 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 internal/audit/consts.go create mode 100644 internal/audit/request.go create mode 100644 internal/audit/target.go diff --git a/go.mod b/go.mod index ddc193897..38e66a20f 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( code.gitea.io/sdk/gitea v0.17.1 git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 + git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de git.frostfs.info/TrueCloudLab/hrw v1.2.1 @@ -54,7 +55,6 @@ require ( ) require ( - git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/internal/audit/consts.go b/internal/audit/consts.go new file mode 100644 index 000000000..f4fa19ab9 --- /dev/null +++ b/internal/audit/consts.go @@ -0,0 +1,7 @@ +package audit + +const ( + InvalidValue = "invalid_value" + NotDefined = "not_defined" + Empty = "empty" +) diff --git a/internal/audit/request.go b/internal/audit/request.go new file mode 100644 index 000000000..cd1771013 --- /dev/null +++ b/internal/audit/request.go @@ -0,0 +1,41 @@ +package audit + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + crypto "git.frostfs.info/TrueCloudLab/frostfs-crypto" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "go.uber.org/zap" +) + +type Request interface { + GetVerificationHeader() *session.RequestVerificationHeader +} + +type Target interface { + String() string +} + +func LogRequest(log *logger.Logger, operation string, req Request, target Target, status bool) { + LogRequestWithKey(log, operation, req.GetVerificationHeader().GetBodySignature().GetKey(), target, status) +} + +func LogRequestWithKey(log *logger.Logger, operation string, key []byte, target Target, status bool) { + object, subject := NotDefined, NotDefined + + publicKey := crypto.UnmarshalPublicKey(key) + if publicKey != nil { + subject = ((*keys.PublicKey)(publicKey)).StringCompressed() + } + + if target != nil { + object = target.String() + } + + log.Info(logs.AuditEventLogRecord, + zap.String("operation", operation), + zap.String("object", object), + zap.String("subject", subject), + zap.Bool("success", status)) +} diff --git a/internal/audit/target.go b/internal/audit/target.go new file mode 100644 index 000000000..1d760eca6 --- /dev/null +++ b/internal/audit/target.go @@ -0,0 +1,158 @@ +package audit + +import ( + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/mr-tron/base58" +) + +type ModelType[T any] interface { + ReadFromV2(m T) error + String() string +} + +func TargetFromRef[T any](ref *T, model ModelType[T]) Target { + if ref == nil { + return stringTarget{s: NotDefined} + } + if err := model.ReadFromV2(*ref); err != nil { + return stringTarget{s: InvalidValue} + } + return stringTarget{s: model.String()} +} + +func TargetFromRefs[T any](refs []*T, model ModelType[T]) Target { + if len(refs) == 0 { + return stringTarget{s: NotDefined} + } + sb := &strings.Builder{} + for idx, ref := range refs { + if idx > 0 { + sb.WriteString(";") + } + if ref == nil { + sb.WriteString(Empty) + continue + } + if err := model.ReadFromV2(*ref); err != nil { + sb.WriteString(InvalidValue) + } else { + sb.WriteString(model.String()) + } + } + return sb +} + +type stringTarget struct { + s string +} + +func (t stringTarget) String() string { + return t.s +} + +func TargetFromString(s string) Target { + if len(s) == 0 { + s = Empty + } + return stringTarget{s: s} +} + +func TargetFromStringSlice(s []string) Target { + if len(s) == 0 { + return stringTarget{s: NotDefined} + } + sb := &strings.Builder{} + for i, v := range s { + if i > 0 { + sb.WriteString(";") + } + if len(v) == 0 { + sb.WriteString(Empty) + } else { + sb.WriteString(v) + } + } + return sb +} + +func TargetFromChainID(chainTargetType, chainTargetName string, chainID []byte) Target { + if len(chainTargetType) == 0 && len(chainTargetName) == 0 && len(chainID) == 0 { + return stringTarget{s: NotDefined} + } + t, n, c := Empty, Empty, Empty + if len(chainTargetType) > 0 { + t = chainTargetType + } + if len(chainTargetName) > 0 { + n = chainTargetName + } + if len(chainID) > 0 { + c = string(chainID) + } + return stringTarget{s: t + ":" + n + ":" + c} +} + +func TargetFromShardIDs(v [][]byte) Target { + if len(v) == 0 { + return stringTarget{s: NotDefined} + } + sb := &strings.Builder{} + for i, s := range v { + if i > 0 { + sb.WriteString(";") + } + if len(s) == 0 { + sb.WriteString(Empty) + } else { + sb.WriteString(base58.Encode(s)) + } + } + return sb +} + +func TargetFromTreeID(containerID []byte, treeID string) Target { + if len(containerID) == 0 && len(treeID) == 0 { + return stringTarget{s: NotDefined} + } + c, t := Empty, Empty + if len(containerID) > 0 { + var cnr cid.ID + if err := cnr.Decode(containerID); err != nil { + c = InvalidValue + } else { + c = cnr.EncodeToString() + } + } + if len(treeID) > 0 { + t = treeID + } + return stringTarget{s: c + ":" + t} +} + +func TargetFromContainerIDObjectID(containerID *refs.ContainerID, objectID *refs.ObjectID) Target { + if containerID == nil && objectID == nil { + return stringTarget{s: NotDefined} + } + c, o := Empty, Empty + if containerID != nil { + var cnr cid.ID + if err := cnr.ReadFromV2(*containerID); err != nil { + c = InvalidValue + } else { + c = cnr.EncodeToString() + } + } + if objectID != nil { + var obj oid.ID + if err := obj.ReadFromV2(*objectID); err != nil { + o = InvalidValue + } else { + o = obj.EncodeToString() + } + } + return stringTarget{s: c + "/" + o} +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 8dae61c3d..eeb2fbf3f 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -539,4 +539,5 @@ const ( PolicerFailedToRestoreObject = "failed to restore EC object" PolicerCouldNotGetChunk = "could not get EC chunk" PolicerCouldNotGetChunks = "could not get EC chunks" + AuditEventLogRecord = "audit event log record" ) From ecd1ed7a5e461c2b48e67949b4b2ef2a353e5307 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 18 Jun 2024 12:40:03 +0300 Subject: [PATCH 1166/1943] [#1184] node: Add audit middleware for grpc services Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/accounting.go | 15 +- cmd/frostfs-node/apemanager.go | 3 +- cmd/frostfs-node/container.go | 14 +- cmd/frostfs-node/control.go | 5 +- cmd/frostfs-node/netmap.go | 28 +-- cmd/frostfs-node/object.go | 5 +- cmd/frostfs-node/session.go | 5 +- cmd/frostfs-node/tree.go | 2 +- pkg/services/accounting/audit.go | 42 ++++ pkg/services/apemanager/audit.go | 75 +++++++ pkg/services/container/audit.go | 116 +++++++++++ pkg/services/control/server/audit.go | 287 +++++++++++++++++++++++++++ pkg/services/netmap/audit.go | 60 ++++++ pkg/services/object/audit.go | 172 ++++++++++++++++ pkg/services/session/audit.go | 39 ++++ pkg/services/tree/audit.go | 135 +++++++++++++ 16 files changed, 967 insertions(+), 36 deletions(-) create mode 100644 pkg/services/accounting/audit.go create mode 100644 pkg/services/apemanager/audit.go create mode 100644 pkg/services/container/audit.go create mode 100644 pkg/services/control/server/audit.go create mode 100644 pkg/services/netmap/audit.go create mode 100644 pkg/services/object/audit.go create mode 100644 pkg/services/session/audit.go create mode 100644 pkg/services/tree/audit.go diff --git a/cmd/frostfs-node/accounting.go b/cmd/frostfs-node/accounting.go index ec737f8a0..f0e2abf38 100644 --- a/cmd/frostfs-node/accounting.go +++ b/cmd/frostfs-node/accounting.go @@ -20,15 +20,16 @@ func initAccountingService(ctx context.Context, c *cfg) { balanceMorphWrapper, err := balance.NewFromMorph(c.cfgMorph.client, c.cfgAccounting.scriptHash, 0) fatalOnErr(err) - server := accountingTransportGRPC.New( - accountingService.NewSignService( - &c.key.PrivateKey, - accountingService.NewExecutionService( - accounting.NewExecutor(balanceMorphWrapper), - c.respSvc, - ), + service := accountingService.NewSignService( + &c.key.PrivateKey, + accountingService.NewExecutionService( + accounting.NewExecutor(balanceMorphWrapper), + c.respSvc, ), ) + service = accountingService.NewAuditService(service, c.log, c.audit) + + server := accountingTransportGRPC.New(service) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { accountingGRPC.RegisterAccountingServiceServer(s, server) diff --git a/cmd/frostfs-node/apemanager.go b/cmd/frostfs-node/apemanager.go index b9928faaf..79c45c254 100644 --- a/cmd/frostfs-node/apemanager.go +++ b/cmd/frostfs-node/apemanager.go @@ -21,7 +21,8 @@ func initAPEManagerService(c *cfg) { execsvc := apemanager.New(c.cfgObject.cnrSource, contractStorage, apemanager.WithLogger(c.log)) sigsvc := apemanager.NewSignService(&c.key.PrivateKey, execsvc) - server := apemanager_transport.New(sigsvc) + auditSvc := apemanager.NewAuditService(sigsvc, c.log, c.audit) + server := apemanager_transport.New(auditSvc) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { apemanager_grpc.RegisterAPEManagerServiceServer(s, server) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 61600376b..b14e19161 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -46,15 +46,15 @@ func initContainerService(_ context.Context, c *cfg) { c.shared.frostfsidClient = frostfsIDSubjectProvider - server := containerTransportGRPC.New( - containerService.NewSignService( - &c.key.PrivateKey, - containerService.NewAPEServer(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine, cnrRdr, - newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, c.shared.frostfsidClient, - containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), - ), + service := containerService.NewSignService( + &c.key.PrivateKey, + containerService.NewAPEServer(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine, cnrRdr, + newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, c.shared.frostfsidClient, + containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), ), ) + service = containerService.NewAuditService(service, c.log, c.audit) + server := containerTransportGRPC.New(service) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { containerGRPC.RegisterContainerServiceServer(s, server) diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index e1e6e3ac9..8ee1ab69f 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -30,8 +30,8 @@ func initControlService(c *cfg) { for i := range pubs { rawPubs = append(rawPubs, pubs[i].Bytes()) } - - ctlSvc := controlSvc.New( + var ctlSvc control.ControlServiceServer + ctlSvc = controlSvc.New( controlSvc.WithKey(&c.key.PrivateKey), controlSvc.WithAuthorizedKeys(rawPubs), controlSvc.WithHealthChecker(c), @@ -43,6 +43,7 @@ func initControlService(c *cfg) { controlSvc.WithTreeService(c.treeService), controlSvc.WithLocalOverrideStorage(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine), ) + ctlSvc = controlSvc.NewAuditService(ctlSvc, c.log, c.audit) lis, err := net.Listen("tcp", endpoint) if err != nil { diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 56f2ca98f..6aff8ddf2 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -147,22 +147,22 @@ func initNetmapService(ctx context.Context, c *cfg) { initNetmapState(c) - server := netmapTransportGRPC.New( - netmapService.NewSignService( - &c.key.PrivateKey, - netmapService.NewExecutionService( - c, - c.apiVersion, - &netInfo{ - netState: c.cfgNetmap.state, - magic: c.cfgMorph.client, - morphClientNetMap: c.cfgNetmap.wrapper, - msPerBlockRdr: c.cfgMorph.client.MsPerBlock, - }, - c.respSvc, - ), + svc := netmapService.NewSignService( + &c.key.PrivateKey, + netmapService.NewExecutionService( + c, + c.apiVersion, + &netInfo{ + netState: c.cfgNetmap.state, + magic: c.cfgMorph.client, + morphClientNetMap: c.cfgNetmap.wrapper, + msPerBlockRdr: c.cfgMorph.client.MsPerBlock, + }, + c.respSvc, ), ) + svc = netmapService.NewAuditService(svc, c.log, c.audit) + server := netmapTransportGRPC.New(svc) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { netmapGRPC.RegisterNetmapServiceServer(s, server) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 907e517a5..62183d314 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -182,7 +182,7 @@ func initObjectService(c *cfg) { sDeleteV2 := createDeleteServiceV2(sDelete) // build service pipeline - // grpc | | signature | response | acl | ape | split + // grpc | audit | | signature | response | acl | ape | split splitSvc := createSplitService(c, sPutV2, sGetV2, sSearchV2, sDeleteV2) @@ -205,7 +205,8 @@ func initObjectService(c *cfg) { c.shared.metricsSvc = objectService.NewMetricCollector( signSvc, c.metricsCollector.ObjectService(), metricsconfig.Enabled(c.appCfg)) - server := objectTransportGRPC.New(c.shared.metricsSvc) + auditSvc := objectService.NewAuditService(c.shared.metricsSvc, c.log, c.audit) + server := objectTransportGRPC.New(auditSvc) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { objectGRPC.RegisterObjectServiceServer(s, server) diff --git a/cmd/frostfs-node/session.go b/cmd/frostfs-node/session.go index ee21ec230..d286fc631 100644 --- a/cmd/frostfs-node/session.go +++ b/cmd/frostfs-node/session.go @@ -52,12 +52,13 @@ func initSessionService(c *cfg) { c.privateTokenStore.RemoveOld(ev.(netmap.NewEpoch).EpochNumber()) }) - server := sessionTransportGRPC.New( + svc := sessionSvc.NewAuditService( sessionSvc.NewSignService( &c.key.PrivateKey, sessionSvc.NewExecutionService(c.privateTokenStore, c.respSvc, c.log), ), - ) + c.log, c.audit) + server := sessionTransportGRPC.New(svc) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { sessionGRPC.RegisterSessionServiceServer(s, server) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index daaaa64a2..9f5c89ef0 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -70,7 +70,7 @@ func initTreeService(c *cfg) { ) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { - tree.RegisterTreeServiceServer(s, c.treeService) + tree.RegisterTreeServiceServer(s, tree.NewAuditService(c.treeService, c.log, c.audit)) }) c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { diff --git a/pkg/services/accounting/audit.go b/pkg/services/accounting/audit.go new file mode 100644 index 000000000..1d8f8836f --- /dev/null +++ b/pkg/services/accounting/audit.go @@ -0,0 +1,42 @@ +package accounting + +import ( + "context" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" + acc_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" +) + +var _ Server = (*auditService)(nil) + +type auditService struct { + next Server + log *logger.Logger + enabled *atomic.Bool +} + +func NewAuditService(next Server, log *logger.Logger, enabled *atomic.Bool) Server { + return &auditService{ + next: next, + log: log, + enabled: enabled, + } +} + +// Balance implements Server. +func (l *auditService) Balance(ctx context.Context, req *accounting.BalanceRequest) (*accounting.BalanceResponse, error) { + res, err := l.next.Balance(ctx, req) + + if !l.enabled.Load() { + return res, err + } + + audit.LogRequest(l.log, acc_grpc.AccountingService_Balance_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetOwnerID(), &user.ID{}), err == nil) + + return res, err +} diff --git a/pkg/services/apemanager/audit.go b/pkg/services/apemanager/audit.go new file mode 100644 index 000000000..d132ae7db --- /dev/null +++ b/pkg/services/apemanager/audit.go @@ -0,0 +1,75 @@ +package apemanager + +import ( + "context" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" + ape_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" +) + +var _ Server = (*auditService)(nil) + +type auditService struct { + next Server + log *logger.Logger + enabled *atomic.Bool +} + +func NewAuditService(next Server, log *logger.Logger, enabled *atomic.Bool) Server { + return &auditService{ + next: next, + log: log, + enabled: enabled, + } +} + +// AddChain implements Server. +func (a *auditService) AddChain(ctx context.Context, req *apemanager.AddChainRequest) (*apemanager.AddChainResponse, error) { + res, err := a.next.AddChain(ctx, req) + if !a.enabled.Load() { + return res, err + } + + audit.LogRequest(a.log, ape_grpc.APEManagerService_AddChain_FullMethodName, req, + audit.TargetFromChainID(req.GetBody().GetTarget().GetTargetType().String(), + req.GetBody().GetTarget().GetName(), + res.GetBody().GetChainID()), + err == nil) + + return res, err +} + +// ListChains implements Server. +func (a *auditService) ListChains(ctx context.Context, req *apemanager.ListChainsRequest) (*apemanager.ListChainsResponse, error) { + res, err := a.next.ListChains(ctx, req) + if !a.enabled.Load() { + return res, err + } + + audit.LogRequest(a.log, ape_grpc.APEManagerService_ListChains_FullMethodName, req, + audit.TargetFromChainID(req.GetBody().GetTarget().GetTargetType().String(), + req.GetBody().GetTarget().GetName(), + nil), + err == nil) + + return res, err +} + +// RemoveChain implements Server. +func (a *auditService) RemoveChain(ctx context.Context, req *apemanager.RemoveChainRequest) (*apemanager.RemoveChainResponse, error) { + res, err := a.next.RemoveChain(ctx, req) + if !a.enabled.Load() { + return res, err + } + + audit.LogRequest(a.log, ape_grpc.APEManagerService_RemoveChain_FullMethodName, req, + audit.TargetFromChainID(req.GetBody().GetTarget().GetTargetType().String(), + req.GetBody().GetTarget().GetName(), + req.GetBody().GetChainID()), + err == nil) + + return res, err +} diff --git a/pkg/services/container/audit.go b/pkg/services/container/audit.go new file mode 100644 index 000000000..7ef432bb1 --- /dev/null +++ b/pkg/services/container/audit.go @@ -0,0 +1,116 @@ +package container + +import ( + "context" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" + container_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" +) + +var _ Server = (*auditService)(nil) + +type auditService struct { + next Server + log *logger.Logger + enabled *atomic.Bool +} + +func NewAuditService(next Server, log *logger.Logger, enabled *atomic.Bool) Server { + return &auditService{ + next: next, + log: log, + enabled: enabled, + } +} + +// AnnounceUsedSpace implements Server. +func (a *auditService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { + res, err := a.next.AnnounceUsedSpace(ctx, req) + if !a.enabled.Load() { + return res, err + } + + var ids []*refs.ContainerID + for _, v := range req.GetBody().GetAnnouncements() { + ids = append(ids, v.GetContainerID()) + } + + audit.LogRequest(a.log, container_grpc.ContainerService_AnnounceUsedSpace_FullMethodName, req, + audit.TargetFromRefs(ids, &cid.ID{}), err == nil) + + return res, err +} + +// Delete implements Server. +func (a *auditService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { + res, err := a.next.Delete(ctx, req) + if !a.enabled.Load() { + return res, err + } + + audit.LogRequest(a.log, container_grpc.ContainerService_Delete_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetContainerID(), &cid.ID{}), err == nil) + + return res, err +} + +// Get implements Server. +func (a *auditService) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { + res, err := a.next.Get(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequest(a.log, container_grpc.ContainerService_Get_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetContainerID(), &cid.ID{}), err == nil) + return res, err +} + +// GetExtendedACL implements Server. +func (a *auditService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { + res, err := a.next.GetExtendedACL(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequest(a.log, container_grpc.ContainerService_GetExtendedACL_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetContainerID(), &cid.ID{}), err == nil) + return res, err +} + +// List implements Server. +func (a *auditService) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { + res, err := a.next.List(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequest(a.log, container_grpc.ContainerService_List_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetOwnerID(), &user.ID{}), err == nil) + return res, err +} + +// Put implements Server. +func (a *auditService) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { + res, err := a.next.Put(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequest(a.log, container_grpc.ContainerService_Put_FullMethodName, req, + audit.TargetFromRef(res.GetBody().GetContainerID(), &cid.ID{}), err == nil) + return res, err +} + +// SetExtendedACL implements Server. +func (a *auditService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { + res, err := a.next.SetExtendedACL(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequest(a.log, container_grpc.ContainerService_SetExtendedACL_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetEACL().GetContainerID(), &cid.ID{}), err == nil) + return res, err +} diff --git a/pkg/services/control/server/audit.go b/pkg/services/control/server/audit.go new file mode 100644 index 000000000..16c04a8c9 --- /dev/null +++ b/pkg/services/control/server/audit.go @@ -0,0 +1,287 @@ +package control + +import ( + "context" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" + ctl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +var _ ctl.ControlServiceServer = (*auditService)(nil) + +type auditService struct { + next ctl.ControlServiceServer + log *logger.Logger + enabled *atomic.Bool +} + +func NewAuditService(next ctl.ControlServiceServer, log *logger.Logger, enabled *atomic.Bool) ctl.ControlServiceServer { + return &auditService{ + next: next, + log: log, + enabled: enabled, + } +} + +// AddChainLocalOverride implements control.ControlServiceServer. +func (a *auditService) AddChainLocalOverride(ctx context.Context, req *ctl.AddChainLocalOverrideRequest) (*ctl.AddChainLocalOverrideResponse, error) { + res, err := a.next.AddChainLocalOverride(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_AddChainLocalOverride_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromChainID(req.GetBody().GetTarget().GetType().String(), + req.GetBody().GetTarget().GetName(), + res.GetBody().GetChainId()), + err == nil) + return res, err +} + +// DetachShards implements control.ControlServiceServer. +func (a *auditService) DetachShards(ctx context.Context, req *ctl.DetachShardsRequest) (*ctl.DetachShardsResponse, error) { + res, err := a.next.DetachShards(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_DetachShards_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) + return res, err +} + +// Doctor implements control.ControlServiceServer. +func (a *auditService) Doctor(ctx context.Context, req *ctl.DoctorRequest) (*ctl.DoctorResponse, error) { + res, err := a.next.Doctor(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_Doctor_FullMethodName, req.GetSignature().GetKey(), nil, err == nil) + return res, err +} + +// DropObjects implements control.ControlServiceServer. +func (a *auditService) DropObjects(ctx context.Context, req *ctl.DropObjectsRequest) (*ctl.DropObjectsResponse, error) { + res, err := a.next.DropObjects(ctx, req) + if !a.enabled.Load() { + return res, err + } + var list []string + for _, v := range req.GetBody().GetAddressList() { + if len(v) == 0 { + list = append(list, audit.Empty) + continue + } + var a oid.Address + if e := a.DecodeString(string(v)); e != nil { + list = append(list, audit.InvalidValue) + } else { + list = append(list, a.EncodeToString()) + } + } + + audit.LogRequestWithKey(a.log, ctl.ControlService_DropObjects_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromStringSlice(list), err == nil) + return res, err +} + +// EvacuateShard implements control.ControlServiceServer. +func (a *auditService) EvacuateShard(ctx context.Context, req *ctl.EvacuateShardRequest) (*ctl.EvacuateShardResponse, error) { + res, err := a.next.EvacuateShard(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_EvacuateShard_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) + return res, err +} + +// FlushCache implements control.ControlServiceServer. +func (a *auditService) FlushCache(ctx context.Context, req *ctl.FlushCacheRequest) (*ctl.FlushCacheResponse, error) { + res, err := a.next.FlushCache(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_FlushCache_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) + return res, err +} + +// GetChainLocalOverride implements control.ControlServiceServer. +func (a *auditService) GetChainLocalOverride(ctx context.Context, req *ctl.GetChainLocalOverrideRequest) (*ctl.GetChainLocalOverrideResponse, error) { + res, err := a.next.GetChainLocalOverride(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_GetChainLocalOverride_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromChainID( + req.GetBody().GetTarget().GetType().String(), + req.GetBody().GetTarget().GetName(), + req.GetBody().GetChainId()), + err == nil) + return res, err +} + +// GetShardEvacuationStatus implements control.ControlServiceServer. +func (a *auditService) GetShardEvacuationStatus(ctx context.Context, req *ctl.GetShardEvacuationStatusRequest) (*ctl.GetShardEvacuationStatusResponse, error) { + res, err := a.next.GetShardEvacuationStatus(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_GetShardEvacuationStatus_FullMethodName, req.GetSignature().GetKey(), + nil, err == nil) + return res, err +} + +// HealthCheck implements control.ControlServiceServer. +func (a *auditService) HealthCheck(ctx context.Context, req *ctl.HealthCheckRequest) (*ctl.HealthCheckResponse, error) { + res, err := a.next.HealthCheck(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_HealthCheck_FullMethodName, req.GetSignature().GetKey(), + nil, err == nil) + return res, err +} + +// ListChainLocalOverrides implements control.ControlServiceServer. +func (a *auditService) ListChainLocalOverrides(ctx context.Context, req *ctl.ListChainLocalOverridesRequest) (*ctl.ListChainLocalOverridesResponse, error) { + res, err := a.next.ListChainLocalOverrides(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_ListChainLocalOverrides_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromChainID(req.GetBody().GetTarget().GetType().String(), + req.GetBody().GetTarget().GetName(), + nil), + err == nil) + return res, err +} + +// ListShards implements control.ControlServiceServer. +func (a *auditService) ListShards(ctx context.Context, req *ctl.ListShardsRequest) (*ctl.ListShardsResponse, error) { + res, err := a.next.ListShards(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_ListShards_FullMethodName, req.GetSignature().GetKey(), + nil, err == nil) + return res, err +} + +// ListTargetsLocalOverrides implements control.ControlServiceServer. +func (a *auditService) ListTargetsLocalOverrides(ctx context.Context, req *ctl.ListTargetsLocalOverridesRequest) (*ctl.ListTargetsLocalOverridesResponse, error) { + res, err := a.next.ListTargetsLocalOverrides(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_ListTargetsLocalOverrides_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromString(req.GetBody().GetChainName()), err == nil) + return res, err +} + +// RemoveChainLocalOverride implements control.ControlServiceServer. +func (a *auditService) RemoveChainLocalOverride(ctx context.Context, req *ctl.RemoveChainLocalOverrideRequest) (*ctl.RemoveChainLocalOverrideResponse, error) { + res, err := a.next.RemoveChainLocalOverride(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_RemoveChainLocalOverride_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromChainID(req.GetBody().GetTarget().GetType().String(), + req.GetBody().GetTarget().GetName(), + req.GetBody().GetChainId()), + err == nil) + return res, err +} + +// RemoveChainLocalOverridesByTarget implements control.ControlServiceServer. +func (a *auditService) RemoveChainLocalOverridesByTarget(ctx context.Context, req *ctl.RemoveChainLocalOverridesByTargetRequest) (*ctl.RemoveChainLocalOverridesByTargetResponse, error) { + res, err := a.next.RemoveChainLocalOverridesByTarget(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_RemoveChainLocalOverridesByTarget_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromChainID(req.GetBody().GetTarget().GetType().String(), + req.GetBody().GetTarget().GetName(), + nil), + err == nil) + return res, err +} + +// ResetShardEvacuationStatus implements control.ControlServiceServer. +func (a *auditService) ResetShardEvacuationStatus(ctx context.Context, req *ctl.ResetShardEvacuationStatusRequest) (*ctl.ResetShardEvacuationStatusResponse, error) { + res, err := a.next.ResetShardEvacuationStatus(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_ResetShardEvacuationStatus_FullMethodName, req.GetSignature().GetKey(), + nil, err == nil) + return res, err +} + +// SealWriteCache implements control.ControlServiceServer. +func (a *auditService) SealWriteCache(ctx context.Context, req *ctl.SealWriteCacheRequest) (*ctl.SealWriteCacheResponse, error) { + res, err := a.next.SealWriteCache(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_SealWriteCache_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) + return res, err +} + +// SetNetmapStatus implements control.ControlServiceServer. +func (a *auditService) SetNetmapStatus(ctx context.Context, req *ctl.SetNetmapStatusRequest) (*ctl.SetNetmapStatusResponse, error) { + res, err := a.next.SetNetmapStatus(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_SetNetmapStatus_FullMethodName, req.GetSignature().GetKey(), + nil, err == nil) + return res, err +} + +// SetShardMode implements control.ControlServiceServer. +func (a *auditService) SetShardMode(ctx context.Context, req *ctl.SetShardModeRequest) (*ctl.SetShardModeResponse, error) { + res, err := a.next.SetShardMode(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_SetShardMode_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) + return res, err +} + +// StartShardEvacuation implements control.ControlServiceServer. +func (a *auditService) StartShardEvacuation(ctx context.Context, req *ctl.StartShardEvacuationRequest) (*ctl.StartShardEvacuationResponse, error) { + res, err := a.next.StartShardEvacuation(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_StartShardEvacuation_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) + return res, err +} + +// StopShardEvacuation implements control.ControlServiceServer. +func (a *auditService) StopShardEvacuation(ctx context.Context, req *ctl.StopShardEvacuationRequest) (*ctl.StopShardEvacuationResponse, error) { + res, err := a.next.StopShardEvacuation(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_StopShardEvacuation_FullMethodName, req.GetSignature().GetKey(), + nil, err == nil) + return res, err +} + +// SynchronizeTree implements control.ControlServiceServer. +func (a *auditService) SynchronizeTree(ctx context.Context, req *ctl.SynchronizeTreeRequest) (*ctl.SynchronizeTreeResponse, error) { + res, err := a.next.SynchronizeTree(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_SynchronizeTree_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) + return res, err +} diff --git a/pkg/services/netmap/audit.go b/pkg/services/netmap/audit.go new file mode 100644 index 000000000..906fd398e --- /dev/null +++ b/pkg/services/netmap/audit.go @@ -0,0 +1,60 @@ +package netmap + +import ( + "context" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" + netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" +) + +var _ Server = (*auditService)(nil) + +type auditService struct { + next Server + log *logger.Logger + enabled *atomic.Bool +} + +func NewAuditService(next Server, log *logger.Logger, enabled *atomic.Bool) Server { + return &auditService{ + next: next, + log: log, + enabled: enabled, + } +} + +// LocalNodeInfo implements Server. +func (a *auditService) LocalNodeInfo(ctx context.Context, req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { + res, err := a.next.LocalNodeInfo(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequest(a.log, netmapGRPC.NetmapService_LocalNodeInfo_FullMethodName, req, + nil, err == nil) + return res, err +} + +// NetworkInfo implements Server. +func (a *auditService) NetworkInfo(ctx context.Context, req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { + res, err := a.next.NetworkInfo(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequest(a.log, netmapGRPC.NetmapService_NetworkInfo_FullMethodName, req, + nil, err == nil) + return res, err +} + +// Snapshot implements Server. +func (a *auditService) Snapshot(ctx context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { + res, err := a.next.Snapshot(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequest(a.log, netmapGRPC.NetmapService_NetmapSnapshot_FullMethodName, req, + nil, err == nil) + return res, err +} diff --git a/pkg/services/object/audit.go b/pkg/services/object/audit.go new file mode 100644 index 000000000..1305fa008 --- /dev/null +++ b/pkg/services/object/audit.go @@ -0,0 +1,172 @@ +package object + +import ( + "context" + "errors" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +var _ ServiceServer = (*auditService)(nil) + +type auditService struct { + next ServiceServer + log *logger.Logger + enabled *atomic.Bool +} + +func NewAuditService(next ServiceServer, log *logger.Logger, enabled *atomic.Bool) ServiceServer { + return &auditService{ + next: next, + log: log, + enabled: enabled, + } +} + +// Delete implements ServiceServer. +func (a *auditService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { + res, err := a.next.Delete(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequest(a.log, objectGRPC.ObjectService_Delete_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetAddress(), &oid.Address{}), err == nil) + return res, err +} + +// Get implements ServiceServer. +func (a *auditService) Get(req *object.GetRequest, stream GetObjectStream) error { + err := a.next.Get(req, stream) + if !a.enabled.Load() { + return err + } + audit.LogRequest(a.log, objectGRPC.ObjectService_Get_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetAddress(), &oid.Address{}), err == nil) + return err +} + +// GetRange implements ServiceServer. +func (a *auditService) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { + err := a.next.GetRange(req, stream) + if !a.enabled.Load() { + return err + } + audit.LogRequest(a.log, objectGRPC.ObjectService_GetRange_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetAddress(), &oid.Address{}), err == nil) + return err +} + +// GetRangeHash implements ServiceServer. +func (a *auditService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { + resp, err := a.next.GetRangeHash(ctx, req) + if !a.enabled.Load() { + return resp, err + } + audit.LogRequest(a.log, objectGRPC.ObjectService_GetRangeHash_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetAddress(), &oid.Address{}), err == nil) + return resp, err +} + +// Head implements ServiceServer. +func (a *auditService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { + resp, err := a.next.Head(ctx, req) + if !a.enabled.Load() { + return resp, err + } + audit.LogRequest(a.log, objectGRPC.ObjectService_Head_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetAddress(), &oid.Address{}), err == nil) + return resp, err +} + +// Put implements ServiceServer. +func (a *auditService) Put() (PutObjectStream, error) { + res, err := a.next.Put() + if !a.enabled.Load() { + return res, err + } + if err != nil { + audit.LogRequest(a.log, objectGRPC.ObjectService_Put_FullMethodName, nil, nil, false) + return res, err + } + return &auditPutStream{ + stream: res, + log: a.log, + }, nil +} + +// PutSingle implements ServiceServer. +func (a *auditService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + resp, err := a.next.PutSingle(ctx, req) + if !a.enabled.Load() { + return resp, err + } + audit.LogRequest(a.log, objectGRPC.ObjectService_PutSingle_FullMethodName, req, + audit.TargetFromContainerIDObjectID(req.GetBody().GetObject().GetHeader().GetContainerID(), + req.GetBody().GetObject().GetObjectID()), + err == nil) + return resp, err +} + +// Search implements ServiceServer. +func (a *auditService) Search(req *object.SearchRequest, stream SearchStream) error { + err := a.next.Search(req, stream) + if !a.enabled.Load() { + return err + } + audit.LogRequest(a.log, objectGRPC.ObjectService_Search_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetContainerID(), &cid.ID{}), err == nil) + return err +} + +var _ PutObjectStream = (*auditPutStream)(nil) + +type auditPutStream struct { + stream PutObjectStream + log *logger.Logger + + failed bool + key []byte + containerID *refs.ContainerID + objectID *refs.ObjectID +} + +// CloseAndRecv implements PutObjectStream. +func (a *auditPutStream) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { + resp, err := a.stream.CloseAndRecv(ctx) + if err != nil { + a.failed = true + } + a.objectID = resp.GetBody().GetObjectID() + audit.LogRequestWithKey(a.log, objectGRPC.ObjectService_Put_FullMethodName, a.key, + audit.TargetFromContainerIDObjectID(a.containerID, a.objectID), + !a.failed) + return resp, err +} + +// Send implements PutObjectStream. +func (a *auditPutStream) Send(ctx context.Context, req *object.PutRequest) error { + if partInit, ok := req.GetBody().GetObjectPart().(*object.PutObjectPartInit); ok { + a.containerID = partInit.GetHeader().GetContainerID() + a.objectID = partInit.GetObjectID() + a.key = req.GetVerificationHeader().GetBodySignature().GetKey() + } + + err := a.stream.Send(ctx, req) + if err != nil { + a.failed = true + } + if !errors.Is(err, util.ErrAbortStream) { // CloseAndRecv will not be called, so log here + audit.LogRequestWithKey(a.log, objectGRPC.ObjectService_Put_FullMethodName, a.key, + audit.TargetFromContainerIDObjectID(a.containerID, a.objectID), + !a.failed) + } + return err +} diff --git a/pkg/services/session/audit.go b/pkg/services/session/audit.go new file mode 100644 index 000000000..19d3383d9 --- /dev/null +++ b/pkg/services/session/audit.go @@ -0,0 +1,39 @@ +package session + +import ( + "context" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + sessionGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" +) + +var _ Server = (*auditService)(nil) + +type auditService struct { + next Server + log *logger.Logger + enabled *atomic.Bool +} + +func NewAuditService(next Server, log *logger.Logger, enabled *atomic.Bool) Server { + return &auditService{ + next: next, + log: log, + enabled: enabled, + } +} + +// Create implements Server. +func (a *auditService) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { + res, err := a.next.Create(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequest(a.log, sessionGRPC.SessionService_Create_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetOwnerID(), &user.ID{}), err == nil) + return res, err +} diff --git a/pkg/services/tree/audit.go b/pkg/services/tree/audit.go new file mode 100644 index 000000000..bec71f5d4 --- /dev/null +++ b/pkg/services/tree/audit.go @@ -0,0 +1,135 @@ +package tree + +import ( + "context" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" +) + +var _ TreeServiceServer = (*auditService)(nil) + +type auditService struct { + next TreeServiceServer + log *logger.Logger + enabled *atomic.Bool +} + +func NewAuditService(next TreeServiceServer, log *logger.Logger, enabled *atomic.Bool) TreeServiceServer { + return &auditService{ + next: next, + log: log, + enabled: enabled, + } +} + +// Add implements TreeServiceServer. +func (a *auditService) Add(ctx context.Context, req *AddRequest) (*AddResponse, error) { + res, err := a.next.Add(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, TreeService_Add_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) + return res, err +} + +// AddByPath implements TreeServiceServer. +func (a *auditService) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByPathResponse, error) { + res, err := a.next.AddByPath(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, TreeService_AddByPath_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) + return res, err +} + +// Apply implements TreeServiceServer. +func (a *auditService) Apply(ctx context.Context, req *ApplyRequest) (*ApplyResponse, error) { + res, err := a.next.Apply(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, TreeService_Apply_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) + return res, err +} + +// GetNodeByPath implements TreeServiceServer. +func (a *auditService) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) (*GetNodeByPathResponse, error) { + res, err := a.next.GetNodeByPath(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, TreeService_GetNodeByPath_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) + return res, err +} + +// GetOpLog implements TreeServiceServer. +func (a *auditService) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) error { + err := a.next.GetOpLog(req, srv) + if !a.enabled.Load() { + return err + } + audit.LogRequestWithKey(a.log, TreeService_GetOpLog_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) + return err +} + +// GetSubTree implements TreeServiceServer. +func (a *auditService) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeServer) error { + err := a.next.GetSubTree(req, srv) + if !a.enabled.Load() { + return err + } + audit.LogRequestWithKey(a.log, TreeService_GetSubTree_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) + return err +} + +// Healthcheck implements TreeServiceServer. +func (a *auditService) Healthcheck(ctx context.Context, req *HealthcheckRequest) (*HealthcheckResponse, error) { + res, err := a.next.Healthcheck(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, TreeService_Healthcheck_FullMethodName, req.GetSignature().GetKey(), + nil, err == nil) + return res, err +} + +// Move implements TreeServiceServer. +func (a *auditService) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, error) { + res, err := a.next.Move(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, TreeService_Move_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) + return res, err +} + +// Remove implements TreeServiceServer. +func (a *auditService) Remove(ctx context.Context, req *RemoveRequest) (*RemoveResponse, error) { + res, err := a.next.Remove(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, TreeService_Remove_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) + return res, err +} + +// TreeList implements TreeServiceServer. +func (a *auditService) TreeList(ctx context.Context, req *TreeListRequest) (*TreeListResponse, error) { + res, err := a.next.TreeList(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, TreeService_TreeList_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromTreeID(req.GetBody().GetContainerId(), ""), err == nil) + return res, err +} From fd28461def3bc9af15c699b071f0b179b1cf5e00 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 18 Jun 2024 12:40:38 +0300 Subject: [PATCH 1167/1943] [#1184] ir: Add grpc middleware for control service Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/config.go | 1 + cmd/frostfs-ir/defaults.go | 2 + cmd/frostfs-ir/main.go | 4 +- pkg/innerring/initialization.go | 8 +- pkg/innerring/innerring.go | 4 +- pkg/services/control/ir/server/audit.go | 108 ++++++++++++++++++++++++ 6 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 pkg/services/control/ir/server/audit.go diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index c73b68eb5..c3227a182 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -35,6 +35,7 @@ func reloadConfig() error { return err } cmode.Store(cfg.GetBool("node.kludge_compatibility_mode")) + audit.Store(cfg.GetBool("audit.enabled")) err = logPrm.SetLevelString(cfg.GetString("logger.level")) if err != nil { return err diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index 23a475591..e703301ae 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -45,6 +45,8 @@ func defaultConfiguration(cfg *viper.Viper) { cfg.SetDefault("governance.disable", false) cfg.SetDefault("node.kludge_compatibility_mode", false) + + cfg.SetDefault("audit.enabled", false) } func setControlDefaults(cfg *viper.Viper) { diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 31390dd74..206059b5d 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -39,6 +39,7 @@ var ( configFile *string configDir *string cmode = &atomic.Bool{} + audit = &atomic.Bool{} ) func exitErr(err error) { @@ -87,8 +88,9 @@ func main() { metricsCmp = newMetricsComponent() metricsCmp.init() + audit.Store(cfg.GetBool("audit.enabled")) - innerRing, err = innerring.New(ctx, log, cfg, intErr, metrics, cmode) + innerRing, err = innerring.New(ctx, log, cfg, intErr, metrics, cmode, audit) exitErr(err) pprofCmp.start() diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 1a4174289..7da0a9794 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "net" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" @@ -26,6 +27,7 @@ import ( control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" controlsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" utilConfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/spf13/viper" @@ -310,7 +312,7 @@ func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper) error { return bindMainnetProcessor(frostfsProcessor, s) } -func (s *Server) initGRPCServer(cfg *viper.Viper) error { +func (s *Server) initGRPCServer(cfg *viper.Viper, log *logger.Logger, audit *atomic.Bool) error { controlSvcEndpoint := cfg.GetString("control.grpc.endpoint") if controlSvcEndpoint == "" { s.log.Info(logs.InnerringNoControlServerEndpointSpecified) @@ -337,9 +339,9 @@ func (s *Server) initGRPCServer(cfg *viper.Viper) error { p.SetPrivateKey(*s.key) p.SetHealthChecker(s) - controlSvc := controlsrv.New(p, s.netmapClient, s.containerClient, + controlSvc := controlsrv.NewAuditService(controlsrv.New(p, s.netmapClient, s.containerClient, controlsrv.WithAllowedKeys(authKeys), - ) + ), log, audit) grpcControlSrv := grpc.NewServer() control.RegisterControlServiceServer(grpcControlSrv, controlSvc) diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 86424abef..944aa8b76 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -331,7 +331,7 @@ func (s *Server) registerStarter(f func() error) { // New creates instance of inner ring sever structure. func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error, - metrics *metrics.InnerRingServiceMetrics, cmode *atomic.Bool, + metrics *metrics.InnerRingServiceMetrics, cmode *atomic.Bool, audit *atomic.Bool, ) (*Server, error) { var err error server := &Server{ @@ -403,7 +403,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan server.initTimers(cfg, morphClients) - err = server.initGRPCServer(cfg) + err = server.initGRPCServer(cfg, log, audit) if err != nil { return nil, err } diff --git a/pkg/services/control/ir/server/audit.go b/pkg/services/control/ir/server/audit.go new file mode 100644 index 000000000..9f7a8b879 --- /dev/null +++ b/pkg/services/control/ir/server/audit.go @@ -0,0 +1,108 @@ +package control + +import ( + "context" + "encoding/hex" + "strings" + "sync/atomic" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" + control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" +) + +var _ control.ControlServiceServer = (*auditService)(nil) + +type auditService struct { + next *Server + log *logger.Logger + enabled *atomic.Bool +} + +func NewAuditService(next *Server, log *logger.Logger, enabled *atomic.Bool) control.ControlServiceServer { + return &auditService{ + next: next, + log: log, + enabled: enabled, + } +} + +// HealthCheck implements control.ControlServiceServer. +func (a *auditService) HealthCheck(ctx context.Context, req *control.HealthCheckRequest) (*control.HealthCheckResponse, error) { + res, err := a.next.HealthCheck(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, control.ControlService_HealthCheck_FullMethodName, req.GetSignature().GetKey(), nil, err == nil) + return res, err +} + +// RemoveContainer implements control.ControlServiceServer. +func (a *auditService) RemoveContainer(ctx context.Context, req *control.RemoveContainerRequest) (*control.RemoveContainerResponse, error) { + res, err := a.next.RemoveContainer(ctx, req) + if !a.enabled.Load() { + return res, err + } + + sb := &strings.Builder{} + var withConatiner bool + if len(req.GetBody().GetContainerId()) > 0 { + withConatiner = true + sb.WriteString("containerID:") + var containerID cid.ID + if err := containerID.Decode(req.GetBody().GetContainerId()); err != nil { + sb.WriteString(audit.InvalidValue) + } else { + sb.WriteString(containerID.EncodeToString()) + } + } + + if len(req.GetBody().GetOwner()) > 0 { + if withConatiner { + sb.WriteString(";") + } + sb.WriteString("owner:") + + var ownerID refs.OwnerID + if err := ownerID.Unmarshal(req.GetBody().GetOwner()); err != nil { + sb.WriteString(audit.InvalidValue) + } else { + var owner user.ID + if err := owner.ReadFromV2(ownerID); err != nil { + sb.WriteString(audit.InvalidValue) + } else { + sb.WriteString(owner.EncodeToString()) + } + } + } + + audit.LogRequestWithKey(a.log, control.ControlService_RemoveContainer_FullMethodName, req.GetSignature().GetKey(), sb, err == nil) + return res, err +} + +// RemoveNode implements control.ControlServiceServer. +func (a *auditService) RemoveNode(ctx context.Context, req *control.RemoveNodeRequest) (*control.RemoveNodeResponse, error) { + res, err := a.next.RemoveNode(ctx, req) + if !a.enabled.Load() { + return res, err + } + + audit.LogRequestWithKey(a.log, control.ControlService_RemoveNode_FullMethodName, req.GetSignature().GetKey(), + audit.TargetFromString(hex.EncodeToString(req.GetBody().GetKey())), err == nil) + return res, err +} + +// TickEpoch implements control.ControlServiceServer. +func (a *auditService) TickEpoch(ctx context.Context, req *control.TickEpochRequest) (*control.TickEpochResponse, error) { + res, err := a.next.TickEpoch(ctx, req) + if !a.enabled.Load() { + return res, err + } + + audit.LogRequestWithKey(a.log, control.ControlService_TickEpoch_FullMethodName, req.GetSignature().GetKey(), + nil, err == nil) + return res, err +} From 40b68bcb6c16331fd8036a0b730ee7442025ccdb Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 18 Jun 2024 15:06:54 +0300 Subject: [PATCH 1168/1943] [#1109] object: Validate attribute `EXPIRATION_EPOCH` on `put` Signed-off-by: Anton Nikiforov --- pkg/core/object/fmt.go | 56 +++++++++++++++++-------------------- pkg/core/object/fmt_test.go | 2 ++ 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index fa1e40dd0..b997aa182 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -124,15 +124,22 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *objectSDK.Object, u return fmt.Errorf("invalid attributes: %w", err) } + exp, err := expirationEpochAttribute(obj) + if err != nil { + if !errors.Is(err, errNoExpirationEpoch) { + return fmt.Errorf("object did not pass expiration check: %w", err) + } + } else if !unprepared && exp < v.netState.CurrentEpoch() { + if err := v.checkIfExpired(ctx, obj); err != nil { + return fmt.Errorf("object did not pass expiration check: %w", err) + } + } + if !unprepared { if err := v.validateSignatureKey(obj); err != nil { return fmt.Errorf("(%T) could not validate signature key: %w", v, err) } - if err := v.checkExpiration(ctx, obj); err != nil { - return fmt.Errorf("object did not pass expiration check: %w", err) - } - if err := objectSDK.CheckHeaderVerificationFields(obj); err != nil { return fmt.Errorf("(%T) could not validate header fields: %w", v, err) } @@ -348,35 +355,24 @@ func (v *FormatValidator) fillAndValidateTombstoneMeta(o *objectSDK.Object, meta var errExpired = errors.New("object has expired") -func (v *FormatValidator) checkExpiration(ctx context.Context, obj *objectSDK.Object) error { - exp, err := expirationEpochAttribute(obj) - if err != nil { - if errors.Is(err, errNoExpirationEpoch) { - return nil // objects without expiration attribute are valid - } +func (v *FormatValidator) checkIfExpired(ctx context.Context, obj *objectSDK.Object) error { + // an object could be expired but locked; + // put such an object is a correct operation - return err + cID, _ := obj.ContainerID() + oID, _ := obj.ID() + + var addr oid.Address + addr.SetContainer(cID) + addr.SetObject(oID) + + locked, err := v.e.IsLocked(ctx, addr) + if err != nil { + return fmt.Errorf("locking status check for an expired object: %w", err) } - if exp < v.netState.CurrentEpoch() { - // an object could be expired but locked; - // put such an object is a correct operation - - cID, _ := obj.ContainerID() - oID, _ := obj.ID() - - var addr oid.Address - addr.SetContainer(cID) - addr.SetObject(oID) - - locked, err := v.e.IsLocked(ctx, addr) - if err != nil { - return fmt.Errorf("locking status check for an expired object: %w", err) - } - - if !locked { - return errExpired - } + if !locked { + return errExpired } return nil diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index a8901ad6d..77afbfc45 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -195,6 +195,8 @@ func TestFormatValidator_Validate(t *testing.T) { val := "text" err := v.Validate(context.Background(), fn(val), false) require.Error(t, err) + err = v.Validate(context.Background(), fn(val), true) + require.Error(t, err) }) t.Run("expired object", func(t *testing.T) { From 11e880de7f87498385006b9c96f85e972ad65950 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 18 Jun 2024 14:33:45 +0300 Subject: [PATCH 1169/1943] [#1186] cli: Make owner field optional for bearer token Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/bearer/create.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-cli/modules/bearer/create.go b/cmd/frostfs-cli/modules/bearer/create.go index d94b39207..a86506c37 100644 --- a/cmd/frostfs-cli/modules/bearer/create.go +++ b/cmd/frostfs-cli/modules/bearer/create.go @@ -67,7 +67,6 @@ func init() { _ = cobra.MarkFlagFilename(createCmd.Flags(), apeFlag) _ = cobra.MarkFlagRequired(createCmd.Flags(), commonflags.ExpireAt) - _ = cobra.MarkFlagRequired(createCmd.Flags(), ownerFlag) _ = cobra.MarkFlagRequired(createCmd.Flags(), outFlag) } @@ -108,16 +107,16 @@ func createToken(cmd *cobra.Command, _ []string) { fmt.Errorf("expiration epoch is less than not-valid-before epoch: %d < %d", exp, nvb)) } - ownerStr, _ := cmd.Flags().GetString(ownerFlag) - - var ownerID user.ID - commonCmd.ExitOnErr(cmd, "can't parse recipient: %w", ownerID.DecodeString(ownerStr)) - var b bearer.Token b.SetExp(exp) b.SetNbf(nvb) b.SetIat(iat) - b.ForUser(ownerID) + + if ownerStr, _ := cmd.Flags().GetString(ownerFlag); ownerStr != "" { + var ownerID user.ID + commonCmd.ExitOnErr(cmd, "can't parse recipient: %w", ownerID.DecodeString(ownerStr)) + b.ForUser(ownerID) + } impersonate, _ := cmd.Flags().GetBool(impersonateFlag) b.SetImpersonate(impersonate) From 68eb68f59a10284025ae345407f486f63a7b620a Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 18 Jun 2024 18:29:56 +0300 Subject: [PATCH 1170/1943] [#1189] cli: Make util subcommand convert eACL to APE chains Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/util/convert_eacl.go | 33 ++++++++--- internal/ape/converter.go | 60 +++++++++++++++----- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/cmd/frostfs-cli/modules/util/convert_eacl.go b/cmd/frostfs-cli/modules/util/convert_eacl.go index 774d500bc..d588ba35d 100644 --- a/cmd/frostfs-cli/modules/util/convert_eacl.go +++ b/cmd/frostfs-cli/modules/util/convert_eacl.go @@ -6,9 +6,17 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeutil "git.frostfs.info/TrueCloudLab/frostfs-node/internal/ape" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" ) +const ( + fromFlagStr = "from" + toFlagStr = "to" + apeFlagStr = "ape" +) + var convertEACLCmd = &cobra.Command{ Use: "eacl", Short: "Convert representation of extended ACL table", @@ -18,24 +26,35 @@ var convertEACLCmd = &cobra.Command{ func initConvertEACLCmd() { flags := convertEACLCmd.Flags() - flags.String("from", "", "File with JSON or binary encoded extended ACL table") - _ = convertEACLCmd.MarkFlagFilename("from") - _ = convertEACLCmd.MarkFlagRequired("from") + flags.String(fromFlagStr, "", "File with JSON or binary encoded extended ACL table") + _ = convertEACLCmd.MarkFlagFilename(fromFlagStr) + _ = convertEACLCmd.MarkFlagRequired(fromFlagStr) - flags.String("to", "", "File to dump extended ACL table (default: binary encoded)") + flags.String(toFlagStr, "", "File to dump extended ACL table (default: binary encoded)") flags.Bool(commonflags.JSON, false, "Dump extended ACL table in JSON encoding") + + flags.Bool(apeFlagStr, false, "Dump converted eACL table to APE chain format") + + convertEACLCmd.MarkFlagsMutuallyExclusive(apeFlagStr, commonflags.JSON) } func convertEACLTable(cmd *cobra.Command, _ []string) { - pathFrom := cmd.Flag("from").Value.String() - to := cmd.Flag("to").Value.String() + pathFrom := cmd.Flag(fromFlagStr).Value.String() + to := cmd.Flag(toFlagStr).Value.String() jsonFlag, _ := cmd.Flags().GetBool(commonflags.JSON) + apeFlag, _ := cmd.Flags().GetBool(apeFlagStr) table := common.ReadEACL(cmd, pathFrom) var data []byte var err error - if jsonFlag || len(to) == 0 { + + if apeFlag { + var ch *chain.Chain + ch, err = apeutil.ConvertEACLToAPE(table) + commonCmd.ExitOnErr(cmd, "convert eACL table to APE chain error: %w", err) + data = ch.Bytes() + } else if jsonFlag || len(to) == 0 { data, err = table.MarshalJSON() commonCmd.ExitOnErr(cmd, "can't JSON encode extended ACL table: %w", err) } else { diff --git a/internal/ape/converter.go b/internal/ape/converter.go index a0cc138d1..eb80e7ded 100644 --- a/internal/ape/converter.go +++ b/internal/ape/converter.go @@ -69,6 +69,50 @@ func ConvertEACLToAPE(eaclTable *eacl.Table) (*apechain.Chain, error) { return res, nil } +func apeRoleConds(role eacl.Role) (res []apechain.Condition) { + switch role { + case eacl.RoleSystem: + res = append(res, + apechain.Condition{ + Op: apechain.CondStringEquals, + Kind: apechain.KindRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleContainer, + }, + ) + res = append(res, + apechain.Condition{ + Op: apechain.CondStringEquals, + Kind: apechain.KindRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleIR, + }, + ) + case eacl.RoleOthers: + res = append(res, + apechain.Condition{ + Op: apechain.CondStringEquals, + Kind: apechain.KindRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOthers, + }, + ) + case eacl.RoleUser: + res = append(res, + apechain.Condition{ + Op: apechain.CondStringEquals, + Kind: apechain.KindRequest, + Key: nativeschema.PropertyKeyActorRole, + Value: nativeschema.PropertyValueContainerRoleOwner, + }, + ) + case eacl.RoleUnknown: + // such condition has no effect + default: + } + return +} + func appendTargetsOnly(source []apechain.Rule, st apechain.Status, act apechain.Actions, res apechain.Resources, targets []eacl.Target) []apechain.Rule { // see https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/src/commit/ab75edd70939564421936d207ef80d6c1398b51b/eacl/validator.go#L101 // role OR public key must be equal @@ -79,13 +123,7 @@ func appendTargetsOnly(source []apechain.Rule, st apechain.Status, act apechain. Any: true, } for _, target := range targets { - var roleCondition apechain.Condition - roleCondition.Kind = apechain.KindRequest - roleCondition.Key = nativeschema.PropertyKeyActorRole - roleCondition.Value = target.Role().String() - roleCondition.Op = apechain.CondStringEquals - rule.Condition = append(rule.Condition, roleCondition) - + rule.Condition = append(rule.Condition, apeRoleConds(target.Role())...) for _, binKey := range target.BinaryKeys() { var pubKeyCondition apechain.Condition pubKeyCondition.Kind = apechain.KindRequest @@ -111,13 +149,7 @@ func appendTargetsAndFilters(source []apechain.Rule, st apechain.Status, act ape Actions: act, Resources: res, } - var roleCondition apechain.Condition - roleCondition.Kind = apechain.KindRequest - roleCondition.Key = nativeschema.PropertyKeyActorRole - roleCondition.Value = target.Role().String() - roleCondition.Op = apechain.CondStringEquals - - rule.Condition = append(rule.Condition, roleCondition) + rule.Condition = append(rule.Condition, apeRoleConds(target.Role())...) rule.Condition, err = appendFilters(rule.Condition, filters) if err != nil { return nil, err From ec76689ab796b1ba3ecca1c21893e337ec853695 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 20 Jun 2024 11:45:25 +0300 Subject: [PATCH 1171/1943] [#1189] cli: Fix unit-test for eACL converter Signed-off-by: Airat Arifullin --- internal/ape/converter_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/ape/converter_test.go b/internal/ape/converter_test.go index b5f8939c4..28125606c 100644 --- a/internal/ape/converter_test.go +++ b/internal/ape/converter_test.go @@ -105,7 +105,7 @@ func TestNoFilters(t *testing.T) { req := &testRequest{ props: map[string]string{ - nativeschema.PropertyKeyActorRole: eacl.RoleOthers.String(), + nativeschema.PropertyKeyActorRole: nativeschema.PropertyValueContainerRoleOthers, }, res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, } @@ -150,7 +150,7 @@ func TestNoFilters(t *testing.T) { req := &testRequest{ props: map[string]string{ - nativeschema.PropertyKeyActorRole: eacl.RoleOthers.String(), + nativeschema.PropertyKeyActorRole: nativeschema.PropertyValueContainerRoleOthers, nativeschema.PropertyKeyActorPublicKey: string(p2.PublicKey().Bytes()), }, res: &testResource{name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString())}, @@ -283,7 +283,7 @@ func TestWithFilters(t *testing.T) { req := &testRequest{ props: map[string]string{ - nativeschema.PropertyKeyActorRole: eacl.RoleOthers.String(), + nativeschema.PropertyKeyActorRole: nativeschema.PropertyValueContainerRoleOthers, }, res: &testResource{ name: fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, cnrID.EncodeToString()), @@ -335,7 +335,7 @@ func TestWithFilters(t *testing.T) { req := &testRequest{ props: map[string]string{ - nativeschema.PropertyKeyActorRole: eacl.RoleOthers.String(), + nativeschema.PropertyKeyActorRole: nativeschema.PropertyValueContainerRoleOthers, attrKey: attrValue, }, res: &testResource{ From 9ac74efc41882d1b5e88756197187ae4e78871ab Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 18 Jun 2024 11:11:56 +0300 Subject: [PATCH 1172/1943] [#1173] shard: Use `mode` from config on `reload` Signed-off-by: Anton Nikiforov --- internal/logs/logs.go | 1 - pkg/local_object_storage/shard/control.go | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index eeb2fbf3f..78f00c4ee 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -247,7 +247,6 @@ const ( ShardCouldNotCloseShardComponent = "could not close shard component" ShardCantOpenMetabaseMoveToADegradedMode = "can't open metabase, move to a degraded mode" ShardCantInitializeMetabaseMoveToADegradedreadonlyMode = "can't initialize metabase, move to a degraded-read-only mode" - ShardTryingToRestoreReadwriteMode = "trying to restore read-write mode" ShardStopEventListenerByClosedEventChannel = "stop event listener by closed `event` channel" ShardStopEventListenerByClosedStopChannel = "stop event listener by closed `stop` channel" ShardStopEventListenerByContext = "stop event listener by context" diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 38a29cb12..90d7afdd4 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -418,9 +418,7 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { return err } } - - s.log.Info(logs.ShardTryingToRestoreReadwriteMode) - return s.setMode(mode.ReadWrite) + return s.setMode(c.info.Mode) } func (s *Shard) lockExclusive() func() { From a83eeddb1d041611666c8481b275c5d2ce249293 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 20 Jun 2024 16:03:58 +0300 Subject: [PATCH 1173/1943] [#60] control: Add GetNetmapStatus method Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/netmap.go | 9 + pkg/services/control/rpc.go | 21 + pkg/services/control/server/audit.go | 11 + .../control/server/get_netmap_status.go | 35 + pkg/services/control/server/server.go | 2 + pkg/services/control/service.pb.go | 2617 +++++++++-------- pkg/services/control/service.proto | 31 + pkg/services/control/service_frostfs.pb.go | 165 ++ pkg/services/control/service_grpc.pb.go | 39 + 9 files changed, 1770 insertions(+), 1160 deletions(-) create mode 100644 pkg/services/control/server/get_netmap_status.go diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 6aff8ddf2..2d424eec6 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -374,6 +374,15 @@ func (c *cfg) SetNetmapStatus(st control.NetmapStatus) error { return c.updateNetMapState(func(*nmClient.UpdatePeerPrm) {}) } +func (c *cfg) GetNetmapStatus() (control.NetmapStatus, uint64, error) { + epoch, err := c.netMapSource.Epoch() + if err != nil { + return control.NetmapStatus_STATUS_UNDEFINED, 0, fmt.Errorf("failed to get current epoch: %w", err) + } + st := c.NetmapStatus() + return st, epoch, nil +} + func (c *cfg) ForceMaintenance() error { return c.setMaintenanceStatus(true) } diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index f9397c128..a90e58a65 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -10,6 +10,7 @@ const serviceName = "control.ControlService" const ( rpcHealthCheck = "HealthCheck" rpcSetNetmapStatus = "SetNetmapStatus" + rpcGetNetmapStatus = "GetNetmapStatus" rpcDropObjects = "DropObjects" rpcListShards = "ListShards" rpcSetShardMode = "SetShardMode" @@ -70,6 +71,26 @@ func SetNetmapStatus( return wResp.message, nil } +// GetNetmapStatus executes ControlService.GetNetmapStatus RPC. +func GetNetmapStatus( + cli *client.Client, + req *GetNetmapStatusRequest, + opts ...client.CallOption, +) (*GetNetmapStatusResponse, error) { + wResp := newResponseWrapper[GetNetmapStatusResponse]() + + wReq := &requestWrapper{ + m: req, + } + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcGetNetmapStatus), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} + // DropObjects executes ControlService.DropObjects RPC. func DropObjects( cli *client.Client, diff --git a/pkg/services/control/server/audit.go b/pkg/services/control/server/audit.go index 16c04a8c9..6443ea378 100644 --- a/pkg/services/control/server/audit.go +++ b/pkg/services/control/server/audit.go @@ -242,6 +242,17 @@ func (a *auditService) SetNetmapStatus(ctx context.Context, req *ctl.SetNetmapSt return res, err } +// GetNetmapStatus implements control.ControlServiceServer. +func (a *auditService) GetNetmapStatus(ctx context.Context, req *ctl.GetNetmapStatusRequest) (*ctl.GetNetmapStatusResponse, error) { + res, err := a.next.GetNetmapStatus(ctx, req) + if !a.enabled.Load() { + return res, err + } + audit.LogRequestWithKey(a.log, ctl.ControlService_GetNetmapStatus_FullMethodName, req.GetSignature().GetKey(), + nil, err == nil) + return res, err +} + // SetShardMode implements control.ControlServiceServer. func (a *auditService) SetShardMode(ctx context.Context, req *ctl.SetShardModeRequest) (*ctl.SetShardModeResponse, error) { res, err := a.next.SetShardMode(ctx, req) diff --git a/pkg/services/control/server/get_netmap_status.go b/pkg/services/control/server/get_netmap_status.go new file mode 100644 index 000000000..1c038253a --- /dev/null +++ b/pkg/services/control/server/get_netmap_status.go @@ -0,0 +1,35 @@ +package control + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// GetNetmapStatus gets node status in FrostFS network. +func (s *Server) GetNetmapStatus(_ context.Context, req *control.GetNetmapStatusRequest) (*control.GetNetmapStatusResponse, error) { + if err := s.isValidRequest(req); err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + st, epoch, err := s.nodeState.GetNetmapStatus() + if err != nil { + return nil, err + } + + resp := &control.GetNetmapStatusResponse{ + Body: &control.GetNetmapStatusResponse_Body{ + Status: st, + Epoch: epoch, + }, + } + + if err := ctrlmessage.Sign(s.key, resp); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return resp, nil +} diff --git a/pkg/services/control/server/server.go b/pkg/services/control/server/server.go index 7cfa93f05..f3fe56a46 100644 --- a/pkg/services/control/server/server.go +++ b/pkg/services/control/server/server.go @@ -50,6 +50,8 @@ type NodeState interface { // ForceMaintenance works like SetNetmapStatus(control.NetmapStatus_MAINTENANCE) // but starts local maintenance regardless of the network settings. ForceMaintenance() error + + GetNetmapStatus() (control.NetmapStatus, uint64, error) } // LocalOverrideStorageDecorator interface provides methods to decorate LocalOverrideEngine diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 9c597beec..727dd1218 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -66,7 +66,7 @@ func (x StartShardEvacuationRequest_Body_Scope) Number() protoreflect.EnumNumber // Deprecated: Use StartShardEvacuationRequest_Body_Scope.Descriptor instead. func (StartShardEvacuationRequest_Body_Scope) EnumDescriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18, 0, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20, 0, 0} } // Evacuate status enum. @@ -116,7 +116,7 @@ func (x GetShardEvacuationStatusResponse_Body_Status) Number() protoreflect.Enum // Deprecated: Use GetShardEvacuationStatusResponse_Body_Status.Descriptor instead. func (GetShardEvacuationStatusResponse_Body_Status) EnumDescriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0, 0} } // Health check request. @@ -351,6 +351,122 @@ func (x *SetNetmapStatusResponse) GetSignature() *Signature { return nil } +// Get netmap status request. +type GetNetmapStatusRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Body of set netmap status request message. + Body *GetNetmapStatusRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + // Body signature. + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *GetNetmapStatusRequest) Reset() { + *x = GetNetmapStatusRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetNetmapStatusRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetNetmapStatusRequest) ProtoMessage() {} + +func (x *GetNetmapStatusRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetNetmapStatusRequest.ProtoReflect.Descriptor instead. +func (*GetNetmapStatusRequest) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{4} +} + +func (x *GetNetmapStatusRequest) GetBody() *GetNetmapStatusRequest_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *GetNetmapStatusRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + +// Get netmap status response. +type GetNetmapStatusResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Body of get netmap status response message. + Body *GetNetmapStatusResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` + // Body signature. + Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` +} + +func (x *GetNetmapStatusResponse) Reset() { + *x = GetNetmapStatusResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetNetmapStatusResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetNetmapStatusResponse) ProtoMessage() {} + +func (x *GetNetmapStatusResponse) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetNetmapStatusResponse.ProtoReflect.Descriptor instead. +func (*GetNetmapStatusResponse) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{5} +} + +func (x *GetNetmapStatusResponse) GetBody() *GetNetmapStatusResponse_Body { + if x != nil { + return x.Body + } + return nil +} + +func (x *GetNetmapStatusResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} + // Request to drop the objects. type DropObjectsRequest struct { state protoimpl.MessageState @@ -366,7 +482,7 @@ type DropObjectsRequest struct { func (x *DropObjectsRequest) Reset() { *x = DropObjectsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[4] + mi := &file_pkg_services_control_service_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -379,7 +495,7 @@ func (x *DropObjectsRequest) String() string { func (*DropObjectsRequest) ProtoMessage() {} func (x *DropObjectsRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[4] + mi := &file_pkg_services_control_service_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -392,7 +508,7 @@ func (x *DropObjectsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DropObjectsRequest.ProtoReflect.Descriptor instead. func (*DropObjectsRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{4} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{6} } func (x *DropObjectsRequest) GetBody() *DropObjectsRequest_Body { @@ -424,7 +540,7 @@ type DropObjectsResponse struct { func (x *DropObjectsResponse) Reset() { *x = DropObjectsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[5] + mi := &file_pkg_services_control_service_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -437,7 +553,7 @@ func (x *DropObjectsResponse) String() string { func (*DropObjectsResponse) ProtoMessage() {} func (x *DropObjectsResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[5] + mi := &file_pkg_services_control_service_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -450,7 +566,7 @@ func (x *DropObjectsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DropObjectsResponse.ProtoReflect.Descriptor instead. func (*DropObjectsResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{5} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{7} } func (x *DropObjectsResponse) GetBody() *DropObjectsResponse_Body { @@ -482,7 +598,7 @@ type ListShardsRequest struct { func (x *ListShardsRequest) Reset() { *x = ListShardsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[6] + mi := &file_pkg_services_control_service_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -495,7 +611,7 @@ func (x *ListShardsRequest) String() string { func (*ListShardsRequest) ProtoMessage() {} func (x *ListShardsRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[6] + mi := &file_pkg_services_control_service_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -508,7 +624,7 @@ func (x *ListShardsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListShardsRequest.ProtoReflect.Descriptor instead. func (*ListShardsRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{6} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{8} } func (x *ListShardsRequest) GetBody() *ListShardsRequest_Body { @@ -540,7 +656,7 @@ type ListShardsResponse struct { func (x *ListShardsResponse) Reset() { *x = ListShardsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[7] + mi := &file_pkg_services_control_service_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -553,7 +669,7 @@ func (x *ListShardsResponse) String() string { func (*ListShardsResponse) ProtoMessage() {} func (x *ListShardsResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[7] + mi := &file_pkg_services_control_service_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -566,7 +682,7 @@ func (x *ListShardsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListShardsResponse.ProtoReflect.Descriptor instead. func (*ListShardsResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{7} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{9} } func (x *ListShardsResponse) GetBody() *ListShardsResponse_Body { @@ -598,7 +714,7 @@ type SetShardModeRequest struct { func (x *SetShardModeRequest) Reset() { *x = SetShardModeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[8] + mi := &file_pkg_services_control_service_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -611,7 +727,7 @@ func (x *SetShardModeRequest) String() string { func (*SetShardModeRequest) ProtoMessage() {} func (x *SetShardModeRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[8] + mi := &file_pkg_services_control_service_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -624,7 +740,7 @@ func (x *SetShardModeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SetShardModeRequest.ProtoReflect.Descriptor instead. func (*SetShardModeRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{8} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{10} } func (x *SetShardModeRequest) GetBody() *SetShardModeRequest_Body { @@ -656,7 +772,7 @@ type SetShardModeResponse struct { func (x *SetShardModeResponse) Reset() { *x = SetShardModeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[9] + mi := &file_pkg_services_control_service_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -669,7 +785,7 @@ func (x *SetShardModeResponse) String() string { func (*SetShardModeResponse) ProtoMessage() {} func (x *SetShardModeResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[9] + mi := &file_pkg_services_control_service_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -682,7 +798,7 @@ func (x *SetShardModeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SetShardModeResponse.ProtoReflect.Descriptor instead. func (*SetShardModeResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{9} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{11} } func (x *SetShardModeResponse) GetBody() *SetShardModeResponse_Body { @@ -714,7 +830,7 @@ type SynchronizeTreeRequest struct { func (x *SynchronizeTreeRequest) Reset() { *x = SynchronizeTreeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[10] + mi := &file_pkg_services_control_service_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -727,7 +843,7 @@ func (x *SynchronizeTreeRequest) String() string { func (*SynchronizeTreeRequest) ProtoMessage() {} func (x *SynchronizeTreeRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[10] + mi := &file_pkg_services_control_service_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -740,7 +856,7 @@ func (x *SynchronizeTreeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SynchronizeTreeRequest.ProtoReflect.Descriptor instead. func (*SynchronizeTreeRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{10} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{12} } func (x *SynchronizeTreeRequest) GetBody() *SynchronizeTreeRequest_Body { @@ -772,7 +888,7 @@ type SynchronizeTreeResponse struct { func (x *SynchronizeTreeResponse) Reset() { *x = SynchronizeTreeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[11] + mi := &file_pkg_services_control_service_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -785,7 +901,7 @@ func (x *SynchronizeTreeResponse) String() string { func (*SynchronizeTreeResponse) ProtoMessage() {} func (x *SynchronizeTreeResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[11] + mi := &file_pkg_services_control_service_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -798,7 +914,7 @@ func (x *SynchronizeTreeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SynchronizeTreeResponse.ProtoReflect.Descriptor instead. func (*SynchronizeTreeResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{11} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{13} } func (x *SynchronizeTreeResponse) GetBody() *SynchronizeTreeResponse_Body { @@ -828,7 +944,7 @@ type EvacuateShardRequest struct { func (x *EvacuateShardRequest) Reset() { *x = EvacuateShardRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[12] + mi := &file_pkg_services_control_service_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -841,7 +957,7 @@ func (x *EvacuateShardRequest) String() string { func (*EvacuateShardRequest) ProtoMessage() {} func (x *EvacuateShardRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[12] + mi := &file_pkg_services_control_service_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -854,7 +970,7 @@ func (x *EvacuateShardRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use EvacuateShardRequest.ProtoReflect.Descriptor instead. func (*EvacuateShardRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{12} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{14} } func (x *EvacuateShardRequest) GetBody() *EvacuateShardRequest_Body { @@ -884,7 +1000,7 @@ type EvacuateShardResponse struct { func (x *EvacuateShardResponse) Reset() { *x = EvacuateShardResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[13] + mi := &file_pkg_services_control_service_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -897,7 +1013,7 @@ func (x *EvacuateShardResponse) String() string { func (*EvacuateShardResponse) ProtoMessage() {} func (x *EvacuateShardResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[13] + mi := &file_pkg_services_control_service_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -910,7 +1026,7 @@ func (x *EvacuateShardResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use EvacuateShardResponse.ProtoReflect.Descriptor instead. func (*EvacuateShardResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{13} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{15} } func (x *EvacuateShardResponse) GetBody() *EvacuateShardResponse_Body { @@ -940,7 +1056,7 @@ type FlushCacheRequest struct { func (x *FlushCacheRequest) Reset() { *x = FlushCacheRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[14] + mi := &file_pkg_services_control_service_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -953,7 +1069,7 @@ func (x *FlushCacheRequest) String() string { func (*FlushCacheRequest) ProtoMessage() {} func (x *FlushCacheRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[14] + mi := &file_pkg_services_control_service_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -966,7 +1082,7 @@ func (x *FlushCacheRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FlushCacheRequest.ProtoReflect.Descriptor instead. func (*FlushCacheRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{14} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{16} } func (x *FlushCacheRequest) GetBody() *FlushCacheRequest_Body { @@ -996,7 +1112,7 @@ type FlushCacheResponse struct { func (x *FlushCacheResponse) Reset() { *x = FlushCacheResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[15] + mi := &file_pkg_services_control_service_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1009,7 +1125,7 @@ func (x *FlushCacheResponse) String() string { func (*FlushCacheResponse) ProtoMessage() {} func (x *FlushCacheResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[15] + mi := &file_pkg_services_control_service_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1022,7 +1138,7 @@ func (x *FlushCacheResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FlushCacheResponse.ProtoReflect.Descriptor instead. func (*FlushCacheResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{15} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{17} } func (x *FlushCacheResponse) GetBody() *FlushCacheResponse_Body { @@ -1052,7 +1168,7 @@ type DoctorRequest struct { func (x *DoctorRequest) Reset() { *x = DoctorRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[16] + mi := &file_pkg_services_control_service_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1065,7 +1181,7 @@ func (x *DoctorRequest) String() string { func (*DoctorRequest) ProtoMessage() {} func (x *DoctorRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[16] + mi := &file_pkg_services_control_service_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1078,7 +1194,7 @@ func (x *DoctorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DoctorRequest.ProtoReflect.Descriptor instead. func (*DoctorRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{16} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18} } func (x *DoctorRequest) GetBody() *DoctorRequest_Body { @@ -1108,7 +1224,7 @@ type DoctorResponse struct { func (x *DoctorResponse) Reset() { *x = DoctorResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[17] + mi := &file_pkg_services_control_service_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1121,7 +1237,7 @@ func (x *DoctorResponse) String() string { func (*DoctorResponse) ProtoMessage() {} func (x *DoctorResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[17] + mi := &file_pkg_services_control_service_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1134,7 +1250,7 @@ func (x *DoctorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DoctorResponse.ProtoReflect.Descriptor instead. func (*DoctorResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{17} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{19} } func (x *DoctorResponse) GetBody() *DoctorResponse_Body { @@ -1164,7 +1280,7 @@ type StartShardEvacuationRequest struct { func (x *StartShardEvacuationRequest) Reset() { *x = StartShardEvacuationRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[18] + mi := &file_pkg_services_control_service_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1177,7 +1293,7 @@ func (x *StartShardEvacuationRequest) String() string { func (*StartShardEvacuationRequest) ProtoMessage() {} func (x *StartShardEvacuationRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[18] + mi := &file_pkg_services_control_service_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1190,7 +1306,7 @@ func (x *StartShardEvacuationRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShardEvacuationRequest.ProtoReflect.Descriptor instead. func (*StartShardEvacuationRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20} } func (x *StartShardEvacuationRequest) GetBody() *StartShardEvacuationRequest_Body { @@ -1220,7 +1336,7 @@ type StartShardEvacuationResponse struct { func (x *StartShardEvacuationResponse) Reset() { *x = StartShardEvacuationResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[19] + mi := &file_pkg_services_control_service_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1233,7 +1349,7 @@ func (x *StartShardEvacuationResponse) String() string { func (*StartShardEvacuationResponse) ProtoMessage() {} func (x *StartShardEvacuationResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[19] + mi := &file_pkg_services_control_service_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1246,7 +1362,7 @@ func (x *StartShardEvacuationResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShardEvacuationResponse.ProtoReflect.Descriptor instead. func (*StartShardEvacuationResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{19} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21} } func (x *StartShardEvacuationResponse) GetBody() *StartShardEvacuationResponse_Body { @@ -1276,7 +1392,7 @@ type GetShardEvacuationStatusRequest struct { func (x *GetShardEvacuationStatusRequest) Reset() { *x = GetShardEvacuationStatusRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[20] + mi := &file_pkg_services_control_service_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1289,7 +1405,7 @@ func (x *GetShardEvacuationStatusRequest) String() string { func (*GetShardEvacuationStatusRequest) ProtoMessage() {} func (x *GetShardEvacuationStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[20] + mi := &file_pkg_services_control_service_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1302,7 +1418,7 @@ func (x *GetShardEvacuationStatusRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetShardEvacuationStatusRequest.ProtoReflect.Descriptor instead. func (*GetShardEvacuationStatusRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22} } func (x *GetShardEvacuationStatusRequest) GetBody() *GetShardEvacuationStatusRequest_Body { @@ -1332,7 +1448,7 @@ type GetShardEvacuationStatusResponse struct { func (x *GetShardEvacuationStatusResponse) Reset() { *x = GetShardEvacuationStatusResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[21] + mi := &file_pkg_services_control_service_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1345,7 +1461,7 @@ func (x *GetShardEvacuationStatusResponse) String() string { func (*GetShardEvacuationStatusResponse) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[21] + mi := &file_pkg_services_control_service_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1358,7 +1474,7 @@ func (x *GetShardEvacuationStatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetShardEvacuationStatusResponse.ProtoReflect.Descriptor instead. func (*GetShardEvacuationStatusResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23} } func (x *GetShardEvacuationStatusResponse) GetBody() *GetShardEvacuationStatusResponse_Body { @@ -1388,7 +1504,7 @@ type ResetShardEvacuationStatusRequest struct { func (x *ResetShardEvacuationStatusRequest) Reset() { *x = ResetShardEvacuationStatusRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[22] + mi := &file_pkg_services_control_service_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1401,7 +1517,7 @@ func (x *ResetShardEvacuationStatusRequest) String() string { func (*ResetShardEvacuationStatusRequest) ProtoMessage() {} func (x *ResetShardEvacuationStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[22] + mi := &file_pkg_services_control_service_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1414,7 +1530,7 @@ func (x *ResetShardEvacuationStatusRequest) ProtoReflect() protoreflect.Message // Deprecated: Use ResetShardEvacuationStatusRequest.ProtoReflect.Descriptor instead. func (*ResetShardEvacuationStatusRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24} } func (x *ResetShardEvacuationStatusRequest) GetBody() *ResetShardEvacuationStatusRequest_Body { @@ -1444,7 +1560,7 @@ type ResetShardEvacuationStatusResponse struct { func (x *ResetShardEvacuationStatusResponse) Reset() { *x = ResetShardEvacuationStatusResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[23] + mi := &file_pkg_services_control_service_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1457,7 +1573,7 @@ func (x *ResetShardEvacuationStatusResponse) String() string { func (*ResetShardEvacuationStatusResponse) ProtoMessage() {} func (x *ResetShardEvacuationStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[23] + mi := &file_pkg_services_control_service_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1470,7 +1586,7 @@ func (x *ResetShardEvacuationStatusResponse) ProtoReflect() protoreflect.Message // Deprecated: Use ResetShardEvacuationStatusResponse.ProtoReflect.Descriptor instead. func (*ResetShardEvacuationStatusResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25} } func (x *ResetShardEvacuationStatusResponse) GetBody() *ResetShardEvacuationStatusResponse_Body { @@ -1500,7 +1616,7 @@ type StopShardEvacuationRequest struct { func (x *StopShardEvacuationRequest) Reset() { *x = StopShardEvacuationRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1513,7 +1629,7 @@ func (x *StopShardEvacuationRequest) String() string { func (*StopShardEvacuationRequest) ProtoMessage() {} func (x *StopShardEvacuationRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[24] + mi := &file_pkg_services_control_service_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1526,7 +1642,7 @@ func (x *StopShardEvacuationRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShardEvacuationRequest.ProtoReflect.Descriptor instead. func (*StopShardEvacuationRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26} } func (x *StopShardEvacuationRequest) GetBody() *StopShardEvacuationRequest_Body { @@ -1556,7 +1672,7 @@ type StopShardEvacuationResponse struct { func (x *StopShardEvacuationResponse) Reset() { *x = StopShardEvacuationResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1569,7 +1685,7 @@ func (x *StopShardEvacuationResponse) String() string { func (*StopShardEvacuationResponse) ProtoMessage() {} func (x *StopShardEvacuationResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[25] + mi := &file_pkg_services_control_service_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1582,7 +1698,7 @@ func (x *StopShardEvacuationResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShardEvacuationResponse.ProtoReflect.Descriptor instead. func (*StopShardEvacuationResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27} } func (x *StopShardEvacuationResponse) GetBody() *StopShardEvacuationResponse_Body { @@ -1612,7 +1728,7 @@ type AddChainLocalOverrideRequest struct { func (x *AddChainLocalOverrideRequest) Reset() { *x = AddChainLocalOverrideRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1625,7 +1741,7 @@ func (x *AddChainLocalOverrideRequest) String() string { func (*AddChainLocalOverrideRequest) ProtoMessage() {} func (x *AddChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[26] + mi := &file_pkg_services_control_service_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1638,7 +1754,7 @@ func (x *AddChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AddChainLocalOverrideRequest.ProtoReflect.Descriptor instead. func (*AddChainLocalOverrideRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28} } func (x *AddChainLocalOverrideRequest) GetBody() *AddChainLocalOverrideRequest_Body { @@ -1668,7 +1784,7 @@ type AddChainLocalOverrideResponse struct { func (x *AddChainLocalOverrideResponse) Reset() { *x = AddChainLocalOverrideResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1681,7 +1797,7 @@ func (x *AddChainLocalOverrideResponse) String() string { func (*AddChainLocalOverrideResponse) ProtoMessage() {} func (x *AddChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[27] + mi := &file_pkg_services_control_service_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1694,7 +1810,7 @@ func (x *AddChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AddChainLocalOverrideResponse.ProtoReflect.Descriptor instead. func (*AddChainLocalOverrideResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29} } func (x *AddChainLocalOverrideResponse) GetBody() *AddChainLocalOverrideResponse_Body { @@ -1724,7 +1840,7 @@ type GetChainLocalOverrideRequest struct { func (x *GetChainLocalOverrideRequest) Reset() { *x = GetChainLocalOverrideRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1737,7 +1853,7 @@ func (x *GetChainLocalOverrideRequest) String() string { func (*GetChainLocalOverrideRequest) ProtoMessage() {} func (x *GetChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[28] + mi := &file_pkg_services_control_service_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1750,7 +1866,7 @@ func (x *GetChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetChainLocalOverrideRequest.ProtoReflect.Descriptor instead. func (*GetChainLocalOverrideRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30} } func (x *GetChainLocalOverrideRequest) GetBody() *GetChainLocalOverrideRequest_Body { @@ -1780,7 +1896,7 @@ type GetChainLocalOverrideResponse struct { func (x *GetChainLocalOverrideResponse) Reset() { *x = GetChainLocalOverrideResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1793,7 +1909,7 @@ func (x *GetChainLocalOverrideResponse) String() string { func (*GetChainLocalOverrideResponse) ProtoMessage() {} func (x *GetChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[29] + mi := &file_pkg_services_control_service_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1806,7 +1922,7 @@ func (x *GetChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetChainLocalOverrideResponse.ProtoReflect.Descriptor instead. func (*GetChainLocalOverrideResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31} } func (x *GetChainLocalOverrideResponse) GetBody() *GetChainLocalOverrideResponse_Body { @@ -1836,7 +1952,7 @@ type ListChainLocalOverridesRequest struct { func (x *ListChainLocalOverridesRequest) Reset() { *x = ListChainLocalOverridesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1849,7 +1965,7 @@ func (x *ListChainLocalOverridesRequest) String() string { func (*ListChainLocalOverridesRequest) ProtoMessage() {} func (x *ListChainLocalOverridesRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[30] + mi := &file_pkg_services_control_service_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1862,7 +1978,7 @@ func (x *ListChainLocalOverridesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListChainLocalOverridesRequest.ProtoReflect.Descriptor instead. func (*ListChainLocalOverridesRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32} } func (x *ListChainLocalOverridesRequest) GetBody() *ListChainLocalOverridesRequest_Body { @@ -1892,7 +2008,7 @@ type ListChainLocalOverridesResponse struct { func (x *ListChainLocalOverridesResponse) Reset() { *x = ListChainLocalOverridesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1905,7 +2021,7 @@ func (x *ListChainLocalOverridesResponse) String() string { func (*ListChainLocalOverridesResponse) ProtoMessage() {} func (x *ListChainLocalOverridesResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[31] + mi := &file_pkg_services_control_service_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1918,7 +2034,7 @@ func (x *ListChainLocalOverridesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListChainLocalOverridesResponse.ProtoReflect.Descriptor instead. func (*ListChainLocalOverridesResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33} } func (x *ListChainLocalOverridesResponse) GetBody() *ListChainLocalOverridesResponse_Body { @@ -1948,7 +2064,7 @@ type ListTargetsLocalOverridesRequest struct { func (x *ListTargetsLocalOverridesRequest) Reset() { *x = ListTargetsLocalOverridesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1961,7 +2077,7 @@ func (x *ListTargetsLocalOverridesRequest) String() string { func (*ListTargetsLocalOverridesRequest) ProtoMessage() {} func (x *ListTargetsLocalOverridesRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[32] + mi := &file_pkg_services_control_service_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1974,7 +2090,7 @@ func (x *ListTargetsLocalOverridesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListTargetsLocalOverridesRequest.ProtoReflect.Descriptor instead. func (*ListTargetsLocalOverridesRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34} } func (x *ListTargetsLocalOverridesRequest) GetBody() *ListTargetsLocalOverridesRequest_Body { @@ -2004,7 +2120,7 @@ type ListTargetsLocalOverridesResponse struct { func (x *ListTargetsLocalOverridesResponse) Reset() { *x = ListTargetsLocalOverridesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2017,7 +2133,7 @@ func (x *ListTargetsLocalOverridesResponse) String() string { func (*ListTargetsLocalOverridesResponse) ProtoMessage() {} func (x *ListTargetsLocalOverridesResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[33] + mi := &file_pkg_services_control_service_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2030,7 +2146,7 @@ func (x *ListTargetsLocalOverridesResponse) ProtoReflect() protoreflect.Message // Deprecated: Use ListTargetsLocalOverridesResponse.ProtoReflect.Descriptor instead. func (*ListTargetsLocalOverridesResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35} } func (x *ListTargetsLocalOverridesResponse) GetBody() *ListTargetsLocalOverridesResponse_Body { @@ -2059,7 +2175,7 @@ type RemoveChainLocalOverrideRequest struct { func (x *RemoveChainLocalOverrideRequest) Reset() { *x = RemoveChainLocalOverrideRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2072,7 +2188,7 @@ func (x *RemoveChainLocalOverrideRequest) String() string { func (*RemoveChainLocalOverrideRequest) ProtoMessage() {} func (x *RemoveChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[34] + mi := &file_pkg_services_control_service_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2085,7 +2201,7 @@ func (x *RemoveChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveChainLocalOverrideRequest.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36} } func (x *RemoveChainLocalOverrideRequest) GetBody() *RemoveChainLocalOverrideRequest_Body { @@ -2114,7 +2230,7 @@ type RemoveChainLocalOverrideResponse struct { func (x *RemoveChainLocalOverrideResponse) Reset() { *x = RemoveChainLocalOverrideResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2127,7 +2243,7 @@ func (x *RemoveChainLocalOverrideResponse) String() string { func (*RemoveChainLocalOverrideResponse) ProtoMessage() {} func (x *RemoveChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[35] + mi := &file_pkg_services_control_service_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2140,7 +2256,7 @@ func (x *RemoveChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveChainLocalOverrideResponse.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37} } func (x *RemoveChainLocalOverrideResponse) GetBody() *RemoveChainLocalOverrideResponse_Body { @@ -2169,7 +2285,7 @@ type RemoveChainLocalOverridesByTargetRequest struct { func (x *RemoveChainLocalOverridesByTargetRequest) Reset() { *x = RemoveChainLocalOverridesByTargetRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2182,7 +2298,7 @@ func (x *RemoveChainLocalOverridesByTargetRequest) String() string { func (*RemoveChainLocalOverridesByTargetRequest) ProtoMessage() {} func (x *RemoveChainLocalOverridesByTargetRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[36] + mi := &file_pkg_services_control_service_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2195,7 +2311,7 @@ func (x *RemoveChainLocalOverridesByTargetRequest) ProtoReflect() protoreflect.M // Deprecated: Use RemoveChainLocalOverridesByTargetRequest.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverridesByTargetRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38} } func (x *RemoveChainLocalOverridesByTargetRequest) GetBody() *RemoveChainLocalOverridesByTargetRequest_Body { @@ -2224,7 +2340,7 @@ type RemoveChainLocalOverridesByTargetResponse struct { func (x *RemoveChainLocalOverridesByTargetResponse) Reset() { *x = RemoveChainLocalOverridesByTargetResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2237,7 +2353,7 @@ func (x *RemoveChainLocalOverridesByTargetResponse) String() string { func (*RemoveChainLocalOverridesByTargetResponse) ProtoMessage() {} func (x *RemoveChainLocalOverridesByTargetResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[37] + mi := &file_pkg_services_control_service_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2250,7 +2366,7 @@ func (x *RemoveChainLocalOverridesByTargetResponse) ProtoReflect() protoreflect. // Deprecated: Use RemoveChainLocalOverridesByTargetResponse.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverridesByTargetResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39} } func (x *RemoveChainLocalOverridesByTargetResponse) GetBody() *RemoveChainLocalOverridesByTargetResponse_Body { @@ -2279,7 +2395,7 @@ type SealWriteCacheRequest struct { func (x *SealWriteCacheRequest) Reset() { *x = SealWriteCacheRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2292,7 +2408,7 @@ func (x *SealWriteCacheRequest) String() string { func (*SealWriteCacheRequest) ProtoMessage() {} func (x *SealWriteCacheRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[38] + mi := &file_pkg_services_control_service_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2305,7 +2421,7 @@ func (x *SealWriteCacheRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheRequest.ProtoReflect.Descriptor instead. func (*SealWriteCacheRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{40} } func (x *SealWriteCacheRequest) GetBody() *SealWriteCacheRequest_Body { @@ -2334,7 +2450,7 @@ type SealWriteCacheResponse struct { func (x *SealWriteCacheResponse) Reset() { *x = SealWriteCacheResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2347,7 +2463,7 @@ func (x *SealWriteCacheResponse) String() string { func (*SealWriteCacheResponse) ProtoMessage() {} func (x *SealWriteCacheResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[39] + mi := &file_pkg_services_control_service_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2360,7 +2476,7 @@ func (x *SealWriteCacheResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheResponse.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{41} } func (x *SealWriteCacheResponse) GetBody() *SealWriteCacheResponse_Body { @@ -2389,7 +2505,7 @@ type DetachShardsRequest struct { func (x *DetachShardsRequest) Reset() { *x = DetachShardsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2402,7 +2518,7 @@ func (x *DetachShardsRequest) String() string { func (*DetachShardsRequest) ProtoMessage() {} func (x *DetachShardsRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[40] + mi := &file_pkg_services_control_service_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2415,7 +2531,7 @@ func (x *DetachShardsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsRequest.ProtoReflect.Descriptor instead. func (*DetachShardsRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{40} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{42} } func (x *DetachShardsRequest) GetBody() *DetachShardsRequest_Body { @@ -2444,7 +2560,7 @@ type DetachShardsResponse struct { func (x *DetachShardsResponse) Reset() { *x = DetachShardsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2457,7 +2573,7 @@ func (x *DetachShardsResponse) String() string { func (*DetachShardsResponse) ProtoMessage() {} func (x *DetachShardsResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[41] + mi := &file_pkg_services_control_service_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2470,7 +2586,7 @@ func (x *DetachShardsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsResponse.ProtoReflect.Descriptor instead. func (*DetachShardsResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{41} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{43} } func (x *DetachShardsResponse) GetBody() *DetachShardsResponse_Body { @@ -2497,7 +2613,7 @@ type HealthCheckRequest_Body struct { func (x *HealthCheckRequest_Body) Reset() { *x = HealthCheckRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2510,7 +2626,7 @@ func (x *HealthCheckRequest_Body) String() string { func (*HealthCheckRequest_Body) ProtoMessage() {} func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[42] + mi := &file_pkg_services_control_service_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2541,7 +2657,7 @@ type HealthCheckResponse_Body struct { func (x *HealthCheckResponse_Body) Reset() { *x = HealthCheckResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2554,7 +2670,7 @@ func (x *HealthCheckResponse_Body) String() string { func (*HealthCheckResponse_Body) ProtoMessage() {} func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[43] + mi := &file_pkg_services_control_service_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2605,7 +2721,7 @@ type SetNetmapStatusRequest_Body struct { func (x *SetNetmapStatusRequest_Body) Reset() { *x = SetNetmapStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2618,7 +2734,7 @@ func (x *SetNetmapStatusRequest_Body) String() string { func (*SetNetmapStatusRequest_Body) ProtoMessage() {} func (x *SetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[44] + mi := &file_pkg_services_control_service_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2658,7 +2774,7 @@ type SetNetmapStatusResponse_Body struct { func (x *SetNetmapStatusResponse_Body) Reset() { *x = SetNetmapStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2671,7 +2787,7 @@ func (x *SetNetmapStatusResponse_Body) String() string { func (*SetNetmapStatusResponse_Body) ProtoMessage() {} func (x *SetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[45] + mi := &file_pkg_services_control_service_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2687,6 +2803,101 @@ func (*SetNetmapStatusResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_control_service_proto_rawDescGZIP(), []int{3, 0} } +type GetNetmapStatusRequest_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *GetNetmapStatusRequest_Body) Reset() { + *x = GetNetmapStatusRequest_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[48] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetNetmapStatusRequest_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetNetmapStatusRequest_Body) ProtoMessage() {} + +func (x *GetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[48] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetNetmapStatusRequest_Body.ProtoReflect.Descriptor instead. +func (*GetNetmapStatusRequest_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{4, 0} +} + +type GetNetmapStatusResponse_Body struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Storage node status in FrostFS network map. + Status NetmapStatus `protobuf:"varint,1,opt,name=status,proto3,enum=control.NetmapStatus" json:"status,omitempty"` + // Network map epoch. + Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` +} + +func (x *GetNetmapStatusResponse_Body) Reset() { + *x = GetNetmapStatusResponse_Body{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_services_control_service_proto_msgTypes[49] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetNetmapStatusResponse_Body) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetNetmapStatusResponse_Body) ProtoMessage() {} + +func (x *GetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { + mi := &file_pkg_services_control_service_proto_msgTypes[49] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetNetmapStatusResponse_Body.ProtoReflect.Descriptor instead. +func (*GetNetmapStatusResponse_Body) Descriptor() ([]byte, []int) { + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{5, 0} +} + +func (x *GetNetmapStatusResponse_Body) GetStatus() NetmapStatus { + if x != nil { + return x.Status + } + return NetmapStatus_STATUS_UNDEFINED +} + +func (x *GetNetmapStatusResponse_Body) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + // Request body structure. type DropObjectsRequest_Body struct { state protoimpl.MessageState @@ -2701,7 +2912,7 @@ type DropObjectsRequest_Body struct { func (x *DropObjectsRequest_Body) Reset() { *x = DropObjectsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2714,7 +2925,7 @@ func (x *DropObjectsRequest_Body) String() string { func (*DropObjectsRequest_Body) ProtoMessage() {} func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[46] + mi := &file_pkg_services_control_service_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2727,7 +2938,7 @@ func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DropObjectsRequest_Body.ProtoReflect.Descriptor instead. func (*DropObjectsRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{4, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{6, 0} } func (x *DropObjectsRequest_Body) GetAddressList() [][]byte { @@ -2747,7 +2958,7 @@ type DropObjectsResponse_Body struct { func (x *DropObjectsResponse_Body) Reset() { *x = DropObjectsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2760,7 +2971,7 @@ func (x *DropObjectsResponse_Body) String() string { func (*DropObjectsResponse_Body) ProtoMessage() {} func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[47] + mi := &file_pkg_services_control_service_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2773,7 +2984,7 @@ func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DropObjectsResponse_Body.ProtoReflect.Descriptor instead. func (*DropObjectsResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{5, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{7, 0} } // Request body structure. @@ -2786,7 +2997,7 @@ type ListShardsRequest_Body struct { func (x *ListShardsRequest_Body) Reset() { *x = ListShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2799,7 +3010,7 @@ func (x *ListShardsRequest_Body) String() string { func (*ListShardsRequest_Body) ProtoMessage() {} func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[48] + mi := &file_pkg_services_control_service_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2812,7 +3023,7 @@ func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use ListShardsRequest_Body.ProtoReflect.Descriptor instead. func (*ListShardsRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{6, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{8, 0} } // Response body structure. @@ -2828,7 +3039,7 @@ type ListShardsResponse_Body struct { func (x *ListShardsResponse_Body) Reset() { *x = ListShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2841,7 +3052,7 @@ func (x *ListShardsResponse_Body) String() string { func (*ListShardsResponse_Body) ProtoMessage() {} func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[49] + mi := &file_pkg_services_control_service_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2854,7 +3065,7 @@ func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use ListShardsResponse_Body.ProtoReflect.Descriptor instead. func (*ListShardsResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{7, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{9, 0} } func (x *ListShardsResponse_Body) GetShards() []*ShardInfo { @@ -2881,7 +3092,7 @@ type SetShardModeRequest_Body struct { func (x *SetShardModeRequest_Body) Reset() { *x = SetShardModeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2894,7 +3105,7 @@ func (x *SetShardModeRequest_Body) String() string { func (*SetShardModeRequest_Body) ProtoMessage() {} func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[50] + mi := &file_pkg_services_control_service_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2907,7 +3118,7 @@ func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SetShardModeRequest_Body.ProtoReflect.Descriptor instead. func (*SetShardModeRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{8, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{10, 0} } func (x *SetShardModeRequest_Body) GetShard_ID() [][]byte { @@ -2941,7 +3152,7 @@ type SetShardModeResponse_Body struct { func (x *SetShardModeResponse_Body) Reset() { *x = SetShardModeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2954,7 +3165,7 @@ func (x *SetShardModeResponse_Body) String() string { func (*SetShardModeResponse_Body) ProtoMessage() {} func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[51] + mi := &file_pkg_services_control_service_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2967,7 +3178,7 @@ func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SetShardModeResponse_Body.ProtoReflect.Descriptor instead. func (*SetShardModeResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{9, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{11, 0} } // Request body structure. @@ -2985,7 +3196,7 @@ type SynchronizeTreeRequest_Body struct { func (x *SynchronizeTreeRequest_Body) Reset() { *x = SynchronizeTreeRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2998,7 +3209,7 @@ func (x *SynchronizeTreeRequest_Body) String() string { func (*SynchronizeTreeRequest_Body) ProtoMessage() {} func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[52] + mi := &file_pkg_services_control_service_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3011,7 +3222,7 @@ func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SynchronizeTreeRequest_Body.ProtoReflect.Descriptor instead. func (*SynchronizeTreeRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{10, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{12, 0} } func (x *SynchronizeTreeRequest_Body) GetContainerId() []byte { @@ -3045,7 +3256,7 @@ type SynchronizeTreeResponse_Body struct { func (x *SynchronizeTreeResponse_Body) Reset() { *x = SynchronizeTreeResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3058,7 +3269,7 @@ func (x *SynchronizeTreeResponse_Body) String() string { func (*SynchronizeTreeResponse_Body) ProtoMessage() {} func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[53] + mi := &file_pkg_services_control_service_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3071,7 +3282,7 @@ func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SynchronizeTreeResponse_Body.ProtoReflect.Descriptor instead. func (*SynchronizeTreeResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{11, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{13, 0} } // Request body structure. @@ -3089,7 +3300,7 @@ type EvacuateShardRequest_Body struct { func (x *EvacuateShardRequest_Body) Reset() { *x = EvacuateShardRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3102,7 +3313,7 @@ func (x *EvacuateShardRequest_Body) String() string { func (*EvacuateShardRequest_Body) ProtoMessage() {} func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[54] + mi := &file_pkg_services_control_service_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3115,7 +3326,7 @@ func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use EvacuateShardRequest_Body.ProtoReflect.Descriptor instead. func (*EvacuateShardRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{12, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{14, 0} } func (x *EvacuateShardRequest_Body) GetShard_ID() [][]byte { @@ -3144,7 +3355,7 @@ type EvacuateShardResponse_Body struct { func (x *EvacuateShardResponse_Body) Reset() { *x = EvacuateShardResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3157,7 +3368,7 @@ func (x *EvacuateShardResponse_Body) String() string { func (*EvacuateShardResponse_Body) ProtoMessage() {} func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[55] + mi := &file_pkg_services_control_service_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3170,7 +3381,7 @@ func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use EvacuateShardResponse_Body.ProtoReflect.Descriptor instead. func (*EvacuateShardResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{13, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{15, 0} } func (x *EvacuateShardResponse_Body) GetCount() uint32 { @@ -3196,7 +3407,7 @@ type FlushCacheRequest_Body struct { func (x *FlushCacheRequest_Body) Reset() { *x = FlushCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3209,7 +3420,7 @@ func (x *FlushCacheRequest_Body) String() string { func (*FlushCacheRequest_Body) ProtoMessage() {} func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[56] + mi := &file_pkg_services_control_service_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3222,7 +3433,7 @@ func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use FlushCacheRequest_Body.ProtoReflect.Descriptor instead. func (*FlushCacheRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{14, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{16, 0} } func (x *FlushCacheRequest_Body) GetShard_ID() [][]byte { @@ -3249,7 +3460,7 @@ type FlushCacheResponse_Body struct { func (x *FlushCacheResponse_Body) Reset() { *x = FlushCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3262,7 +3473,7 @@ func (x *FlushCacheResponse_Body) String() string { func (*FlushCacheResponse_Body) ProtoMessage() {} func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[57] + mi := &file_pkg_services_control_service_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3275,7 +3486,7 @@ func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use FlushCacheResponse_Body.ProtoReflect.Descriptor instead. func (*FlushCacheResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{15, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{17, 0} } // Request body structure. @@ -3293,7 +3504,7 @@ type DoctorRequest_Body struct { func (x *DoctorRequest_Body) Reset() { *x = DoctorRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3306,7 +3517,7 @@ func (x *DoctorRequest_Body) String() string { func (*DoctorRequest_Body) ProtoMessage() {} func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[58] + mi := &file_pkg_services_control_service_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3319,7 +3530,7 @@ func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DoctorRequest_Body.ProtoReflect.Descriptor instead. func (*DoctorRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{16, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18, 0} } func (x *DoctorRequest_Body) GetConcurrency() uint32 { @@ -3346,7 +3557,7 @@ type DoctorResponse_Body struct { func (x *DoctorResponse_Body) Reset() { *x = DoctorResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3359,7 +3570,7 @@ func (x *DoctorResponse_Body) String() string { func (*DoctorResponse_Body) ProtoMessage() {} func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[59] + mi := &file_pkg_services_control_service_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3372,7 +3583,7 @@ func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DoctorResponse_Body.ProtoReflect.Descriptor instead. func (*DoctorResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{17, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{19, 0} } // Request body structure. @@ -3392,7 +3603,7 @@ type StartShardEvacuationRequest_Body struct { func (x *StartShardEvacuationRequest_Body) Reset() { *x = StartShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3405,7 +3616,7 @@ func (x *StartShardEvacuationRequest_Body) String() string { func (*StartShardEvacuationRequest_Body) ProtoMessage() {} func (x *StartShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[60] + mi := &file_pkg_services_control_service_proto_msgTypes[64] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3418,7 +3629,7 @@ func (x *StartShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShardEvacuationRequest_Body.ProtoReflect.Descriptor instead. func (*StartShardEvacuationRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20, 0} } func (x *StartShardEvacuationRequest_Body) GetShard_ID() [][]byte { @@ -3452,7 +3663,7 @@ type StartShardEvacuationResponse_Body struct { func (x *StartShardEvacuationResponse_Body) Reset() { *x = StartShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3465,7 +3676,7 @@ func (x *StartShardEvacuationResponse_Body) String() string { func (*StartShardEvacuationResponse_Body) ProtoMessage() {} func (x *StartShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[61] + mi := &file_pkg_services_control_service_proto_msgTypes[65] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3478,7 +3689,7 @@ func (x *StartShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message // Deprecated: Use StartShardEvacuationResponse_Body.ProtoReflect.Descriptor instead. func (*StartShardEvacuationResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{19, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0} } // Request body structure. @@ -3491,7 +3702,7 @@ type GetShardEvacuationStatusRequest_Body struct { func (x *GetShardEvacuationStatusRequest_Body) Reset() { *x = GetShardEvacuationStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3504,7 +3715,7 @@ func (x *GetShardEvacuationStatusRequest_Body) String() string { func (*GetShardEvacuationStatusRequest_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[62] + mi := &file_pkg_services_control_service_proto_msgTypes[66] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3517,7 +3728,7 @@ func (x *GetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Messa // Deprecated: Use GetShardEvacuationStatusRequest_Body.ProtoReflect.Descriptor instead. func (*GetShardEvacuationStatusRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22, 0} } // Response body structure. @@ -3556,7 +3767,7 @@ type GetShardEvacuationStatusResponse_Body struct { func (x *GetShardEvacuationStatusResponse_Body) Reset() { *x = GetShardEvacuationStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3569,7 +3780,7 @@ func (x *GetShardEvacuationStatusResponse_Body) String() string { func (*GetShardEvacuationStatusResponse_Body) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[63] + mi := &file_pkg_services_control_service_proto_msgTypes[67] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3582,7 +3793,7 @@ func (x *GetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Mess // Deprecated: Use GetShardEvacuationStatusResponse_Body.ProtoReflect.Descriptor instead. func (*GetShardEvacuationStatusResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0} } func (x *GetShardEvacuationStatusResponse_Body) GetTotalObjects() uint64 { @@ -3681,7 +3892,7 @@ type GetShardEvacuationStatusResponse_Body_UnixTimestamp struct { func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) Reset() { *x = GetShardEvacuationStatusResponse_Body_UnixTimestamp{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3694,7 +3905,7 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) String() string { func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[64] + mi := &file_pkg_services_control_service_proto_msgTypes[68] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3707,7 +3918,7 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoReflect() pro // Deprecated: Use GetShardEvacuationStatusResponse_Body_UnixTimestamp.ProtoReflect.Descriptor instead. func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0, 0} } func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) GetValue() int64 { @@ -3729,7 +3940,7 @@ type GetShardEvacuationStatusResponse_Body_Duration struct { func (x *GetShardEvacuationStatusResponse_Body_Duration) Reset() { *x = GetShardEvacuationStatusResponse_Body_Duration{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3742,7 +3953,7 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) String() string { func (*GetShardEvacuationStatusResponse_Body_Duration) ProtoMessage() {} func (x *GetShardEvacuationStatusResponse_Body_Duration) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[65] + mi := &file_pkg_services_control_service_proto_msgTypes[69] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3755,7 +3966,7 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) ProtoReflect() protoref // Deprecated: Use GetShardEvacuationStatusResponse_Body_Duration.ProtoReflect.Descriptor instead. func (*GetShardEvacuationStatusResponse_Body_Duration) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0, 1} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0, 1} } func (x *GetShardEvacuationStatusResponse_Body_Duration) GetSeconds() int64 { @@ -3774,7 +3985,7 @@ type ResetShardEvacuationStatusRequest_Body struct { func (x *ResetShardEvacuationStatusRequest_Body) Reset() { *x = ResetShardEvacuationStatusRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[66] + mi := &file_pkg_services_control_service_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3787,7 +3998,7 @@ func (x *ResetShardEvacuationStatusRequest_Body) String() string { func (*ResetShardEvacuationStatusRequest_Body) ProtoMessage() {} func (x *ResetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[66] + mi := &file_pkg_services_control_service_proto_msgTypes[70] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3800,7 +4011,7 @@ func (x *ResetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Mes // Deprecated: Use ResetShardEvacuationStatusRequest_Body.ProtoReflect.Descriptor instead. func (*ResetShardEvacuationStatusRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24, 0} } type ResetShardEvacuationStatusResponse_Body struct { @@ -3812,7 +4023,7 @@ type ResetShardEvacuationStatusResponse_Body struct { func (x *ResetShardEvacuationStatusResponse_Body) Reset() { *x = ResetShardEvacuationStatusResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[67] + mi := &file_pkg_services_control_service_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3825,7 +4036,7 @@ func (x *ResetShardEvacuationStatusResponse_Body) String() string { func (*ResetShardEvacuationStatusResponse_Body) ProtoMessage() {} func (x *ResetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[67] + mi := &file_pkg_services_control_service_proto_msgTypes[71] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3838,7 +4049,7 @@ func (x *ResetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Me // Deprecated: Use ResetShardEvacuationStatusResponse_Body.ProtoReflect.Descriptor instead. func (*ResetShardEvacuationStatusResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25, 0} } // Request body structure. @@ -3851,7 +4062,7 @@ type StopShardEvacuationRequest_Body struct { func (x *StopShardEvacuationRequest_Body) Reset() { *x = StopShardEvacuationRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[68] + mi := &file_pkg_services_control_service_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3864,7 +4075,7 @@ func (x *StopShardEvacuationRequest_Body) String() string { func (*StopShardEvacuationRequest_Body) ProtoMessage() {} func (x *StopShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[68] + mi := &file_pkg_services_control_service_proto_msgTypes[72] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3877,7 +4088,7 @@ func (x *StopShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShardEvacuationRequest_Body.ProtoReflect.Descriptor instead. func (*StopShardEvacuationRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26, 0} } // Response body structure. @@ -3890,7 +4101,7 @@ type StopShardEvacuationResponse_Body struct { func (x *StopShardEvacuationResponse_Body) Reset() { *x = StopShardEvacuationResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[69] + mi := &file_pkg_services_control_service_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3903,7 +4114,7 @@ func (x *StopShardEvacuationResponse_Body) String() string { func (*StopShardEvacuationResponse_Body) ProtoMessage() {} func (x *StopShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[69] + mi := &file_pkg_services_control_service_proto_msgTypes[73] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3916,7 +4127,7 @@ func (x *StopShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShardEvacuationResponse_Body.ProtoReflect.Descriptor instead. func (*StopShardEvacuationResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27, 0} } type AddChainLocalOverrideRequest_Body struct { @@ -3935,7 +4146,7 @@ type AddChainLocalOverrideRequest_Body struct { func (x *AddChainLocalOverrideRequest_Body) Reset() { *x = AddChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[70] + mi := &file_pkg_services_control_service_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3948,7 +4159,7 @@ func (x *AddChainLocalOverrideRequest_Body) String() string { func (*AddChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *AddChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[70] + mi := &file_pkg_services_control_service_proto_msgTypes[74] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3961,7 +4172,7 @@ func (x *AddChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message // Deprecated: Use AddChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. func (*AddChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28, 0} } func (x *AddChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { @@ -3992,7 +4203,7 @@ type AddChainLocalOverrideResponse_Body struct { func (x *AddChainLocalOverrideResponse_Body) Reset() { *x = AddChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[71] + mi := &file_pkg_services_control_service_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4005,7 +4216,7 @@ func (x *AddChainLocalOverrideResponse_Body) String() string { func (*AddChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *AddChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[71] + mi := &file_pkg_services_control_service_proto_msgTypes[75] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4018,7 +4229,7 @@ func (x *AddChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message // Deprecated: Use AddChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. func (*AddChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29, 0} } func (x *AddChainLocalOverrideResponse_Body) GetChainId() []byte { @@ -4042,7 +4253,7 @@ type GetChainLocalOverrideRequest_Body struct { func (x *GetChainLocalOverrideRequest_Body) Reset() { *x = GetChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[72] + mi := &file_pkg_services_control_service_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4055,7 +4266,7 @@ func (x *GetChainLocalOverrideRequest_Body) String() string { func (*GetChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *GetChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[72] + mi := &file_pkg_services_control_service_proto_msgTypes[76] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4068,7 +4279,7 @@ func (x *GetChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message // Deprecated: Use GetChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. func (*GetChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30, 0} } func (x *GetChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { @@ -4097,7 +4308,7 @@ type GetChainLocalOverrideResponse_Body struct { func (x *GetChainLocalOverrideResponse_Body) Reset() { *x = GetChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[73] + mi := &file_pkg_services_control_service_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4110,7 +4321,7 @@ func (x *GetChainLocalOverrideResponse_Body) String() string { func (*GetChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *GetChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[73] + mi := &file_pkg_services_control_service_proto_msgTypes[77] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4123,7 +4334,7 @@ func (x *GetChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message // Deprecated: Use GetChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. func (*GetChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31, 0} } func (x *GetChainLocalOverrideResponse_Body) GetChain() []byte { @@ -4145,7 +4356,7 @@ type ListChainLocalOverridesRequest_Body struct { func (x *ListChainLocalOverridesRequest_Body) Reset() { *x = ListChainLocalOverridesRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[74] + mi := &file_pkg_services_control_service_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4158,7 +4369,7 @@ func (x *ListChainLocalOverridesRequest_Body) String() string { func (*ListChainLocalOverridesRequest_Body) ProtoMessage() {} func (x *ListChainLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[74] + mi := &file_pkg_services_control_service_proto_msgTypes[78] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4171,7 +4382,7 @@ func (x *ListChainLocalOverridesRequest_Body) ProtoReflect() protoreflect.Messag // Deprecated: Use ListChainLocalOverridesRequest_Body.ProtoReflect.Descriptor instead. func (*ListChainLocalOverridesRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32, 0} } func (x *ListChainLocalOverridesRequest_Body) GetTarget() *ChainTarget { @@ -4193,7 +4404,7 @@ type ListChainLocalOverridesResponse_Body struct { func (x *ListChainLocalOverridesResponse_Body) Reset() { *x = ListChainLocalOverridesResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[75] + mi := &file_pkg_services_control_service_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4206,7 +4417,7 @@ func (x *ListChainLocalOverridesResponse_Body) String() string { func (*ListChainLocalOverridesResponse_Body) ProtoMessage() {} func (x *ListChainLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[75] + mi := &file_pkg_services_control_service_proto_msgTypes[79] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4219,7 +4430,7 @@ func (x *ListChainLocalOverridesResponse_Body) ProtoReflect() protoreflect.Messa // Deprecated: Use ListChainLocalOverridesResponse_Body.ProtoReflect.Descriptor instead. func (*ListChainLocalOverridesResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33, 0} } func (x *ListChainLocalOverridesResponse_Body) GetChains() [][]byte { @@ -4241,7 +4452,7 @@ type ListTargetsLocalOverridesRequest_Body struct { func (x *ListTargetsLocalOverridesRequest_Body) Reset() { *x = ListTargetsLocalOverridesRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[76] + mi := &file_pkg_services_control_service_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4254,7 +4465,7 @@ func (x *ListTargetsLocalOverridesRequest_Body) String() string { func (*ListTargetsLocalOverridesRequest_Body) ProtoMessage() {} func (x *ListTargetsLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[76] + mi := &file_pkg_services_control_service_proto_msgTypes[80] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4267,7 +4478,7 @@ func (x *ListTargetsLocalOverridesRequest_Body) ProtoReflect() protoreflect.Mess // Deprecated: Use ListTargetsLocalOverridesRequest_Body.ProtoReflect.Descriptor instead. func (*ListTargetsLocalOverridesRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34, 0} } func (x *ListTargetsLocalOverridesRequest_Body) GetChainName() string { @@ -4289,7 +4500,7 @@ type ListTargetsLocalOverridesResponse_Body struct { func (x *ListTargetsLocalOverridesResponse_Body) Reset() { *x = ListTargetsLocalOverridesResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[77] + mi := &file_pkg_services_control_service_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4302,7 +4513,7 @@ func (x *ListTargetsLocalOverridesResponse_Body) String() string { func (*ListTargetsLocalOverridesResponse_Body) ProtoMessage() {} func (x *ListTargetsLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[77] + mi := &file_pkg_services_control_service_proto_msgTypes[81] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4315,7 +4526,7 @@ func (x *ListTargetsLocalOverridesResponse_Body) ProtoReflect() protoreflect.Mes // Deprecated: Use ListTargetsLocalOverridesResponse_Body.ProtoReflect.Descriptor instead. func (*ListTargetsLocalOverridesResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35, 0} } func (x *ListTargetsLocalOverridesResponse_Body) GetTargets() []*ChainTarget { @@ -4339,7 +4550,7 @@ type RemoveChainLocalOverrideRequest_Body struct { func (x *RemoveChainLocalOverrideRequest_Body) Reset() { *x = RemoveChainLocalOverrideRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[78] + mi := &file_pkg_services_control_service_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4352,7 +4563,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) String() string { func (*RemoveChainLocalOverrideRequest_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[78] + mi := &file_pkg_services_control_service_proto_msgTypes[82] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4365,7 +4576,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Messa // Deprecated: Use RemoveChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36, 0} } func (x *RemoveChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { @@ -4391,7 +4602,7 @@ type RemoveChainLocalOverrideResponse_Body struct { func (x *RemoveChainLocalOverrideResponse_Body) Reset() { *x = RemoveChainLocalOverrideResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[79] + mi := &file_pkg_services_control_service_proto_msgTypes[83] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4404,7 +4615,7 @@ func (x *RemoveChainLocalOverrideResponse_Body) String() string { func (*RemoveChainLocalOverrideResponse_Body) ProtoMessage() {} func (x *RemoveChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[79] + mi := &file_pkg_services_control_service_proto_msgTypes[83] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4417,7 +4628,7 @@ func (x *RemoveChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Mess // Deprecated: Use RemoveChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37, 0} } type RemoveChainLocalOverridesByTargetRequest_Body struct { @@ -4432,7 +4643,7 @@ type RemoveChainLocalOverridesByTargetRequest_Body struct { func (x *RemoveChainLocalOverridesByTargetRequest_Body) Reset() { *x = RemoveChainLocalOverridesByTargetRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[80] + mi := &file_pkg_services_control_service_proto_msgTypes[84] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4445,7 +4656,7 @@ func (x *RemoveChainLocalOverridesByTargetRequest_Body) String() string { func (*RemoveChainLocalOverridesByTargetRequest_Body) ProtoMessage() {} func (x *RemoveChainLocalOverridesByTargetRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[80] + mi := &file_pkg_services_control_service_proto_msgTypes[84] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4458,7 +4669,7 @@ func (x *RemoveChainLocalOverridesByTargetRequest_Body) ProtoReflect() protorefl // Deprecated: Use RemoveChainLocalOverridesByTargetRequest_Body.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverridesByTargetRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38, 0} } func (x *RemoveChainLocalOverridesByTargetRequest_Body) GetTarget() *ChainTarget { @@ -4477,7 +4688,7 @@ type RemoveChainLocalOverridesByTargetResponse_Body struct { func (x *RemoveChainLocalOverridesByTargetResponse_Body) Reset() { *x = RemoveChainLocalOverridesByTargetResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[81] + mi := &file_pkg_services_control_service_proto_msgTypes[85] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4490,7 +4701,7 @@ func (x *RemoveChainLocalOverridesByTargetResponse_Body) String() string { func (*RemoveChainLocalOverridesByTargetResponse_Body) ProtoMessage() {} func (x *RemoveChainLocalOverridesByTargetResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[81] + mi := &file_pkg_services_control_service_proto_msgTypes[85] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4503,7 +4714,7 @@ func (x *RemoveChainLocalOverridesByTargetResponse_Body) ProtoReflect() protoref // Deprecated: Use RemoveChainLocalOverridesByTargetResponse_Body.ProtoReflect.Descriptor instead. func (*RemoveChainLocalOverridesByTargetResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39, 0} } // Request body structure. @@ -4521,7 +4732,7 @@ type SealWriteCacheRequest_Body struct { func (x *SealWriteCacheRequest_Body) Reset() { *x = SealWriteCacheRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[82] + mi := &file_pkg_services_control_service_proto_msgTypes[86] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4534,7 +4745,7 @@ func (x *SealWriteCacheRequest_Body) String() string { func (*SealWriteCacheRequest_Body) ProtoMessage() {} func (x *SealWriteCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[82] + mi := &file_pkg_services_control_service_proto_msgTypes[86] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4547,7 +4758,7 @@ func (x *SealWriteCacheRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheRequest_Body.ProtoReflect.Descriptor instead. func (*SealWriteCacheRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{40, 0} } func (x *SealWriteCacheRequest_Body) GetShard_ID() [][]byte { @@ -4575,7 +4786,7 @@ type SealWriteCacheResponse_Body struct { func (x *SealWriteCacheResponse_Body) Reset() { *x = SealWriteCacheResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[83] + mi := &file_pkg_services_control_service_proto_msgTypes[87] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4588,7 +4799,7 @@ func (x *SealWriteCacheResponse_Body) String() string { func (*SealWriteCacheResponse_Body) ProtoMessage() {} func (x *SealWriteCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[83] + mi := &file_pkg_services_control_service_proto_msgTypes[87] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4601,7 +4812,7 @@ func (x *SealWriteCacheResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use SealWriteCacheResponse_Body.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{41, 0} } func (x *SealWriteCacheResponse_Body) GetResults() []*SealWriteCacheResponse_Body_Status { @@ -4624,7 +4835,7 @@ type SealWriteCacheResponse_Body_Status struct { func (x *SealWriteCacheResponse_Body_Status) Reset() { *x = SealWriteCacheResponse_Body_Status{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[84] + mi := &file_pkg_services_control_service_proto_msgTypes[88] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4637,7 +4848,7 @@ func (x *SealWriteCacheResponse_Body_Status) String() string { func (*SealWriteCacheResponse_Body_Status) ProtoMessage() {} func (x *SealWriteCacheResponse_Body_Status) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[84] + mi := &file_pkg_services_control_service_proto_msgTypes[88] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4650,7 +4861,7 @@ func (x *SealWriteCacheResponse_Body_Status) ProtoReflect() protoreflect.Message // Deprecated: Use SealWriteCacheResponse_Body_Status.ProtoReflect.Descriptor instead. func (*SealWriteCacheResponse_Body_Status) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39, 0, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{41, 0, 0} } func (x *SealWriteCacheResponse_Body_Status) GetShard_ID() []byte { @@ -4685,7 +4896,7 @@ type DetachShardsRequest_Body struct { func (x *DetachShardsRequest_Body) Reset() { *x = DetachShardsRequest_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[85] + mi := &file_pkg_services_control_service_proto_msgTypes[89] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4698,7 +4909,7 @@ func (x *DetachShardsRequest_Body) String() string { func (*DetachShardsRequest_Body) ProtoMessage() {} func (x *DetachShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[85] + mi := &file_pkg_services_control_service_proto_msgTypes[89] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4711,7 +4922,7 @@ func (x *DetachShardsRequest_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsRequest_Body.ProtoReflect.Descriptor instead. func (*DetachShardsRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{40, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{42, 0} } func (x *DetachShardsRequest_Body) GetShard_ID() [][]byte { @@ -4730,7 +4941,7 @@ type DetachShardsResponse_Body struct { func (x *DetachShardsResponse_Body) Reset() { *x = DetachShardsResponse_Body{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[86] + mi := &file_pkg_services_control_service_proto_msgTypes[90] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4743,7 +4954,7 @@ func (x *DetachShardsResponse_Body) String() string { func (*DetachShardsResponse_Body) ProtoMessage() {} func (x *DetachShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[86] + mi := &file_pkg_services_control_service_proto_msgTypes[90] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4756,7 +4967,7 @@ func (x *DetachShardsResponse_Body) ProtoReflect() protoreflect.Message { // Deprecated: Use DetachShardsResponse_Body.ProtoReflect.Descriptor instead. func (*DetachShardsResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{41, 0} + return file_pkg_services_control_service_proto_rawDescGZIP(), []int{43, 0} } var File_pkg_services_control_service_proto protoreflect.FileDescriptor @@ -4815,620 +5026,647 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x22, 0xa7, 0x01, 0x0a, 0x12, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x29, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x64, - 0x64, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, - 0x52, 0x0b, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x22, 0x86, 0x01, - 0x0a, 0x13, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, - 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x82, 0x01, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x33, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xb0, 0x01, 0x0a, 0x12, - 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x32, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x12, 0x2a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x22, 0xf7, - 0x01, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x77, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, - 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, - 0x49, 0x44, 0x12, 0x26, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x2c, 0x0a, 0x11, 0x72, 0x65, - 0x73, 0x65, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x72, 0x65, 0x73, 0x65, 0x74, 0x45, 0x72, 0x72, 0x6f, - 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x53, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x22, 0xe0, 0x01, 0x0a, 0x16, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, - 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, + 0x6f, 0x64, 0x79, 0x22, 0x8c, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, + 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, - 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x5a, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x16, - 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, - 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x8e, 0x01, 0x0a, 0x17, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x39, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xc8, 0x01, 0x0a, 0x14, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, - 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, - 0x72, 0x73, 0x22, 0xa0, 0x01, 0x0a, 0x15, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xb1, 0x01, 0x0a, 0x11, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x33, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x35, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, - 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, - 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x65, 0x61, 0x6c, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x04, 0x73, 0x65, 0x61, 0x6c, 0x22, 0x84, 0x01, 0x0a, 0x12, 0x46, 0x6c, - 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x22, 0xc9, 0x01, 0x0a, 0x0d, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, - 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x55, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x20, 0x0a, - 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x12, - 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x64, 0x75, 0x70, 0x6c, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x44, 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x22, 0x7c, 0x0a, 0x0e, - 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x98, 0x02, 0x0a, 0x1b, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x87, 0x01, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, - 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, - 0x72, 0x6f, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x22, 0x29, 0x0a, 0x05, 0x53, 0x63, - 0x6f, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x0b, 0x0a, - 0x07, 0x4f, 0x42, 0x4a, 0x45, 0x43, 0x54, 0x53, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, - 0x45, 0x45, 0x53, 0x10, 0x02, 0x22, 0x98, 0x01, 0x0a, 0x1c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x22, 0x9e, 0x01, 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, - 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, - 0x79, 0x22, 0x89, 0x07, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xee, 0x05, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x74, 0x6f, - 0x74, 0x61, 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x65, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, - 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x65, - 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x0d, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x19, - 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, - 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x4d, 0x0a, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x35, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x5b, 0x0a, - 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x3c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, - 0x79, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, - 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, - 0x27, 0x0a, 0x0f, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, - 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, - 0x6c, 0x5f, 0x74, 0x72, 0x65, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x74, - 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x72, 0x65, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x72, 0x65, 0x65, 0x73, 0x18, 0x0b, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0e, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x54, 0x72, 0x65, - 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x74, 0x72, 0x65, - 0x65, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, - 0x54, 0x72, 0x65, 0x65, 0x73, 0x1a, 0x25, 0x0a, 0x0d, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x24, 0x0a, 0x08, - 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x6f, - 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, - 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x23, 0x0a, 0x1f, - 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, - 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, - 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, 0x22, 0xa2, 0x01, - 0x0a, 0x21, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x22, 0xa4, 0x01, 0x0a, 0x22, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x64, 0x79, 0x22, 0xd3, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4b, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4e, 0x65, + 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x22, 0xa7, 0x01, 0x0a, 0x12, 0x44, 0x72, 0x6f, + 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x29, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x21, 0x0a, 0x0c, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0b, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x4c, 0x69, + 0x73, 0x74, 0x22, 0x86, 0x01, 0x0a, 0x13, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x82, 0x01, 0x0a, 0x11, + 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x33, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x22, 0xb0, 0x01, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, + 0x32, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x64, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x22, 0xf7, 0x01, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x1a, 0x77, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, + 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, + 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x26, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, + 0x2c, 0x0a, 0x11, 0x72, 0x65, 0x73, 0x65, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x75, + 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x72, 0x65, 0x73, 0x65, + 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x88, 0x01, + 0x0a, 0x14, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, + 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xe0, 0x01, 0x0a, 0x16, 0x53, 0x79, 0x6e, + 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, + 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, + 0x5a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, + 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, + 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x8e, 0x01, 0x0a, 0x17, + 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xc8, 0x01, 0x0a, + 0x14, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, + 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, + 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, + 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, + 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0xa0, 0x01, 0x0a, 0x15, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xb1, 0x01, 0x0a, 0x11, 0x46, + 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x33, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x35, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x65, + 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x73, 0x65, 0x61, 0x6c, 0x22, 0x84, + 0x01, 0x0a, 0x12, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, + 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xc9, 0x01, 0x0a, 0x0d, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x55, 0x0a, 0x04, 0x42, 0x6f, + 0x64, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, + 0x65, 0x6e, 0x63, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x64, + 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x10, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x44, 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, + 0x73, 0x22, 0x7c, 0x0a, 0x0e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, + 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, + 0x98, 0x02, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, + 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x1a, 0x87, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, + 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, + 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x22, + 0x29, 0x0a, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f, 0x4e, 0x45, + 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x42, 0x4a, 0x45, 0x43, 0x54, 0x53, 0x10, 0x01, 0x12, + 0x09, 0x0a, 0x05, 0x54, 0x52, 0x45, 0x45, 0x53, 0x10, 0x02, 0x22, 0x98, 0x01, 0x0a, 0x1c, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x9e, 0x01, 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x89, 0x07, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x94, 0x01, 0x0a, 0x1a, 0x53, 0x74, - 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x22, 0x96, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, - 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xdc, 0x01, 0x0a, 0x1c, 0x41, 0x64, - 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4a, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xb5, 0x01, 0x0a, 0x1d, 0x41, 0x64, 0x64, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, - 0x22, 0xe1, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, - 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, - 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, - 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x64, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xca, 0x01, 0x0a, 0x1e, 0x4c, 0x69, 0x73, 0x74, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x34, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x22, 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1e, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x73, 0x22, 0xbe, 0x01, - 0x0a, 0x20, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x24, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xd2, - 0x01, 0x0a, 0x21, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x36, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x2e, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x73, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xa0, 0x01, - 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x22, 0xde, 0x01, 0x0a, 0x28, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4a, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, - 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x22, 0xb2, 0x01, 0x0a, 0x29, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, - 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4b, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xca, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, - 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, - 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x46, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, - 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, - 0x6f, 0x72, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, - 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, - 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, - 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, - 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x65, 0x1a, 0xee, 0x05, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, + 0x74, 0x61, 0x6c, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, + 0x2b, 0x0a, 0x11, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x65, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x25, 0x0a, 0x0e, + 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, + 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x4d, + 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x35, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, + 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, + 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x5b, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, + 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x5f, + 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x73, + 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1f, 0x0a, + 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x74, 0x72, 0x65, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x72, 0x65, 0x65, 0x73, 0x12, 0x27, + 0x0a, 0x0f, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x72, 0x65, 0x65, + 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x65, 0x64, 0x54, 0x72, 0x65, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x61, 0x69, 0x6c, 0x65, + 0x64, 0x5f, 0x74, 0x72, 0x65, 0x65, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x66, + 0x61, 0x69, 0x6c, 0x65, 0x64, 0x54, 0x72, 0x65, 0x65, 0x73, 0x1a, 0x25, 0x0a, 0x0d, 0x55, 0x6e, + 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x1a, 0x24, 0x0a, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, + 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, + 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x23, 0x0a, 0x1f, 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x48, + 0x41, 0x52, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, + 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, + 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, + 0x10, 0x02, 0x22, 0xa2, 0x01, 0x0a, 0x21, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, + 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xa4, 0x01, 0x0a, 0x22, 0x52, 0x65, 0x73, 0x65, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0xb5, - 0x0f, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, - 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, - 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, - 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, - 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, - 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, - 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, - 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, - 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, - 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, - 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, - 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, - 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, - 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, - 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, - 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, - 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, - 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x94, + 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, + 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xdc, + 0x01, 0x0a, 0x1c, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x4a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, + 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xb5, 0x01, + 0x0a, 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8a, 0x01, 0x0a, 0x21, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, + 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, + 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xe1, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, + 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, + 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xca, 0x01, 0x0a, + 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x40, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, + 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, + 0x61, 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, + 0x6e, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x20, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x24, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, + 0x61, 0x6d, 0x65, 0x22, 0xd2, 0x01, 0x0a, 0x21, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, + 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x1a, 0x36, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2e, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, + 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, + 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, + 0x49, 0x64, 0x22, 0xa0, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xde, 0x01, 0x0a, 0x28, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x31, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, - 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, + 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x4a, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, + 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb2, 0x01, 0x0a, 0x29, 0x52, 0x65, 0x6d, 0x6f, 0x76, + 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xca, 0x01, 0x0a, 0x15, + 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, + 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, + 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, + 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, + 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, + 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, + 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, + 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, + 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, + 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, + 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, + 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, + 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, + 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, + 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, + 0x6f, 0x64, 0x79, 0x32, 0x8b, 0x10, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, + 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, + 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, + 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, + 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, + 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, + 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, + 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, + 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, + 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, + 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, + 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, + 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, - 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x8a, 0x01, 0x0a, 0x21, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, + 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, - 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, - 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, - 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, - 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, - 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, + 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, + 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, + 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, + 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, + 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, + 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -5444,7 +5682,7 @@ func file_pkg_services_control_service_proto_rawDescGZIP() []byte { } var file_pkg_services_control_service_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 87) +var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 91) var file_pkg_services_control_service_proto_goTypes = []interface{}{ (StartShardEvacuationRequest_Body_Scope)(0), // 0: control.StartShardEvacuationRequest.Body.Scope (GetShardEvacuationStatusResponse_Body_Status)(0), // 1: control.GetShardEvacuationStatusResponse.Body.Status @@ -5452,243 +5690,254 @@ var file_pkg_services_control_service_proto_goTypes = []interface{}{ (*HealthCheckResponse)(nil), // 3: control.HealthCheckResponse (*SetNetmapStatusRequest)(nil), // 4: control.SetNetmapStatusRequest (*SetNetmapStatusResponse)(nil), // 5: control.SetNetmapStatusResponse - (*DropObjectsRequest)(nil), // 6: control.DropObjectsRequest - (*DropObjectsResponse)(nil), // 7: control.DropObjectsResponse - (*ListShardsRequest)(nil), // 8: control.ListShardsRequest - (*ListShardsResponse)(nil), // 9: control.ListShardsResponse - (*SetShardModeRequest)(nil), // 10: control.SetShardModeRequest - (*SetShardModeResponse)(nil), // 11: control.SetShardModeResponse - (*SynchronizeTreeRequest)(nil), // 12: control.SynchronizeTreeRequest - (*SynchronizeTreeResponse)(nil), // 13: control.SynchronizeTreeResponse - (*EvacuateShardRequest)(nil), // 14: control.EvacuateShardRequest - (*EvacuateShardResponse)(nil), // 15: control.EvacuateShardResponse - (*FlushCacheRequest)(nil), // 16: control.FlushCacheRequest - (*FlushCacheResponse)(nil), // 17: control.FlushCacheResponse - (*DoctorRequest)(nil), // 18: control.DoctorRequest - (*DoctorResponse)(nil), // 19: control.DoctorResponse - (*StartShardEvacuationRequest)(nil), // 20: control.StartShardEvacuationRequest - (*StartShardEvacuationResponse)(nil), // 21: control.StartShardEvacuationResponse - (*GetShardEvacuationStatusRequest)(nil), // 22: control.GetShardEvacuationStatusRequest - (*GetShardEvacuationStatusResponse)(nil), // 23: control.GetShardEvacuationStatusResponse - (*ResetShardEvacuationStatusRequest)(nil), // 24: control.ResetShardEvacuationStatusRequest - (*ResetShardEvacuationStatusResponse)(nil), // 25: control.ResetShardEvacuationStatusResponse - (*StopShardEvacuationRequest)(nil), // 26: control.StopShardEvacuationRequest - (*StopShardEvacuationResponse)(nil), // 27: control.StopShardEvacuationResponse - (*AddChainLocalOverrideRequest)(nil), // 28: control.AddChainLocalOverrideRequest - (*AddChainLocalOverrideResponse)(nil), // 29: control.AddChainLocalOverrideResponse - (*GetChainLocalOverrideRequest)(nil), // 30: control.GetChainLocalOverrideRequest - (*GetChainLocalOverrideResponse)(nil), // 31: control.GetChainLocalOverrideResponse - (*ListChainLocalOverridesRequest)(nil), // 32: control.ListChainLocalOverridesRequest - (*ListChainLocalOverridesResponse)(nil), // 33: control.ListChainLocalOverridesResponse - (*ListTargetsLocalOverridesRequest)(nil), // 34: control.ListTargetsLocalOverridesRequest - (*ListTargetsLocalOverridesResponse)(nil), // 35: control.ListTargetsLocalOverridesResponse - (*RemoveChainLocalOverrideRequest)(nil), // 36: control.RemoveChainLocalOverrideRequest - (*RemoveChainLocalOverrideResponse)(nil), // 37: control.RemoveChainLocalOverrideResponse - (*RemoveChainLocalOverridesByTargetRequest)(nil), // 38: control.RemoveChainLocalOverridesByTargetRequest - (*RemoveChainLocalOverridesByTargetResponse)(nil), // 39: control.RemoveChainLocalOverridesByTargetResponse - (*SealWriteCacheRequest)(nil), // 40: control.SealWriteCacheRequest - (*SealWriteCacheResponse)(nil), // 41: control.SealWriteCacheResponse - (*DetachShardsRequest)(nil), // 42: control.DetachShardsRequest - (*DetachShardsResponse)(nil), // 43: control.DetachShardsResponse - (*HealthCheckRequest_Body)(nil), // 44: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 45: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 46: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 47: control.SetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 48: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 49: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 50: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 51: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 52: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 53: control.SetShardModeResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 54: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 55: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 56: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 57: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 58: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 59: control.FlushCacheResponse.Body - (*DoctorRequest_Body)(nil), // 60: control.DoctorRequest.Body - (*DoctorResponse_Body)(nil), // 61: control.DoctorResponse.Body - (*StartShardEvacuationRequest_Body)(nil), // 62: control.StartShardEvacuationRequest.Body - (*StartShardEvacuationResponse_Body)(nil), // 63: control.StartShardEvacuationResponse.Body - (*GetShardEvacuationStatusRequest_Body)(nil), // 64: control.GetShardEvacuationStatusRequest.Body - (*GetShardEvacuationStatusResponse_Body)(nil), // 65: control.GetShardEvacuationStatusResponse.Body - (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 66: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 67: control.GetShardEvacuationStatusResponse.Body.Duration - (*ResetShardEvacuationStatusRequest_Body)(nil), // 68: control.ResetShardEvacuationStatusRequest.Body - (*ResetShardEvacuationStatusResponse_Body)(nil), // 69: control.ResetShardEvacuationStatusResponse.Body - (*StopShardEvacuationRequest_Body)(nil), // 70: control.StopShardEvacuationRequest.Body - (*StopShardEvacuationResponse_Body)(nil), // 71: control.StopShardEvacuationResponse.Body - (*AddChainLocalOverrideRequest_Body)(nil), // 72: control.AddChainLocalOverrideRequest.Body - (*AddChainLocalOverrideResponse_Body)(nil), // 73: control.AddChainLocalOverrideResponse.Body - (*GetChainLocalOverrideRequest_Body)(nil), // 74: control.GetChainLocalOverrideRequest.Body - (*GetChainLocalOverrideResponse_Body)(nil), // 75: control.GetChainLocalOverrideResponse.Body - (*ListChainLocalOverridesRequest_Body)(nil), // 76: control.ListChainLocalOverridesRequest.Body - (*ListChainLocalOverridesResponse_Body)(nil), // 77: control.ListChainLocalOverridesResponse.Body - (*ListTargetsLocalOverridesRequest_Body)(nil), // 78: control.ListTargetsLocalOverridesRequest.Body - (*ListTargetsLocalOverridesResponse_Body)(nil), // 79: control.ListTargetsLocalOverridesResponse.Body - (*RemoveChainLocalOverrideRequest_Body)(nil), // 80: control.RemoveChainLocalOverrideRequest.Body - (*RemoveChainLocalOverrideResponse_Body)(nil), // 81: control.RemoveChainLocalOverrideResponse.Body - (*RemoveChainLocalOverridesByTargetRequest_Body)(nil), // 82: control.RemoveChainLocalOverridesByTargetRequest.Body - (*RemoveChainLocalOverridesByTargetResponse_Body)(nil), // 83: control.RemoveChainLocalOverridesByTargetResponse.Body - (*SealWriteCacheRequest_Body)(nil), // 84: control.SealWriteCacheRequest.Body - (*SealWriteCacheResponse_Body)(nil), // 85: control.SealWriteCacheResponse.Body - (*SealWriteCacheResponse_Body_Status)(nil), // 86: control.SealWriteCacheResponse.Body.Status - (*DetachShardsRequest_Body)(nil), // 87: control.DetachShardsRequest.Body - (*DetachShardsResponse_Body)(nil), // 88: control.DetachShardsResponse.Body - (*Signature)(nil), // 89: control.Signature - (NetmapStatus)(0), // 90: control.NetmapStatus - (HealthStatus)(0), // 91: control.HealthStatus - (*ShardInfo)(nil), // 92: control.ShardInfo - (ShardMode)(0), // 93: control.ShardMode - (*ChainTarget)(nil), // 94: control.ChainTarget + (*GetNetmapStatusRequest)(nil), // 6: control.GetNetmapStatusRequest + (*GetNetmapStatusResponse)(nil), // 7: control.GetNetmapStatusResponse + (*DropObjectsRequest)(nil), // 8: control.DropObjectsRequest + (*DropObjectsResponse)(nil), // 9: control.DropObjectsResponse + (*ListShardsRequest)(nil), // 10: control.ListShardsRequest + (*ListShardsResponse)(nil), // 11: control.ListShardsResponse + (*SetShardModeRequest)(nil), // 12: control.SetShardModeRequest + (*SetShardModeResponse)(nil), // 13: control.SetShardModeResponse + (*SynchronizeTreeRequest)(nil), // 14: control.SynchronizeTreeRequest + (*SynchronizeTreeResponse)(nil), // 15: control.SynchronizeTreeResponse + (*EvacuateShardRequest)(nil), // 16: control.EvacuateShardRequest + (*EvacuateShardResponse)(nil), // 17: control.EvacuateShardResponse + (*FlushCacheRequest)(nil), // 18: control.FlushCacheRequest + (*FlushCacheResponse)(nil), // 19: control.FlushCacheResponse + (*DoctorRequest)(nil), // 20: control.DoctorRequest + (*DoctorResponse)(nil), // 21: control.DoctorResponse + (*StartShardEvacuationRequest)(nil), // 22: control.StartShardEvacuationRequest + (*StartShardEvacuationResponse)(nil), // 23: control.StartShardEvacuationResponse + (*GetShardEvacuationStatusRequest)(nil), // 24: control.GetShardEvacuationStatusRequest + (*GetShardEvacuationStatusResponse)(nil), // 25: control.GetShardEvacuationStatusResponse + (*ResetShardEvacuationStatusRequest)(nil), // 26: control.ResetShardEvacuationStatusRequest + (*ResetShardEvacuationStatusResponse)(nil), // 27: control.ResetShardEvacuationStatusResponse + (*StopShardEvacuationRequest)(nil), // 28: control.StopShardEvacuationRequest + (*StopShardEvacuationResponse)(nil), // 29: control.StopShardEvacuationResponse + (*AddChainLocalOverrideRequest)(nil), // 30: control.AddChainLocalOverrideRequest + (*AddChainLocalOverrideResponse)(nil), // 31: control.AddChainLocalOverrideResponse + (*GetChainLocalOverrideRequest)(nil), // 32: control.GetChainLocalOverrideRequest + (*GetChainLocalOverrideResponse)(nil), // 33: control.GetChainLocalOverrideResponse + (*ListChainLocalOverridesRequest)(nil), // 34: control.ListChainLocalOverridesRequest + (*ListChainLocalOverridesResponse)(nil), // 35: control.ListChainLocalOverridesResponse + (*ListTargetsLocalOverridesRequest)(nil), // 36: control.ListTargetsLocalOverridesRequest + (*ListTargetsLocalOverridesResponse)(nil), // 37: control.ListTargetsLocalOverridesResponse + (*RemoveChainLocalOverrideRequest)(nil), // 38: control.RemoveChainLocalOverrideRequest + (*RemoveChainLocalOverrideResponse)(nil), // 39: control.RemoveChainLocalOverrideResponse + (*RemoveChainLocalOverridesByTargetRequest)(nil), // 40: control.RemoveChainLocalOverridesByTargetRequest + (*RemoveChainLocalOverridesByTargetResponse)(nil), // 41: control.RemoveChainLocalOverridesByTargetResponse + (*SealWriteCacheRequest)(nil), // 42: control.SealWriteCacheRequest + (*SealWriteCacheResponse)(nil), // 43: control.SealWriteCacheResponse + (*DetachShardsRequest)(nil), // 44: control.DetachShardsRequest + (*DetachShardsResponse)(nil), // 45: control.DetachShardsResponse + (*HealthCheckRequest_Body)(nil), // 46: control.HealthCheckRequest.Body + (*HealthCheckResponse_Body)(nil), // 47: control.HealthCheckResponse.Body + (*SetNetmapStatusRequest_Body)(nil), // 48: control.SetNetmapStatusRequest.Body + (*SetNetmapStatusResponse_Body)(nil), // 49: control.SetNetmapStatusResponse.Body + (*GetNetmapStatusRequest_Body)(nil), // 50: control.GetNetmapStatusRequest.Body + (*GetNetmapStatusResponse_Body)(nil), // 51: control.GetNetmapStatusResponse.Body + (*DropObjectsRequest_Body)(nil), // 52: control.DropObjectsRequest.Body + (*DropObjectsResponse_Body)(nil), // 53: control.DropObjectsResponse.Body + (*ListShardsRequest_Body)(nil), // 54: control.ListShardsRequest.Body + (*ListShardsResponse_Body)(nil), // 55: control.ListShardsResponse.Body + (*SetShardModeRequest_Body)(nil), // 56: control.SetShardModeRequest.Body + (*SetShardModeResponse_Body)(nil), // 57: control.SetShardModeResponse.Body + (*SynchronizeTreeRequest_Body)(nil), // 58: control.SynchronizeTreeRequest.Body + (*SynchronizeTreeResponse_Body)(nil), // 59: control.SynchronizeTreeResponse.Body + (*EvacuateShardRequest_Body)(nil), // 60: control.EvacuateShardRequest.Body + (*EvacuateShardResponse_Body)(nil), // 61: control.EvacuateShardResponse.Body + (*FlushCacheRequest_Body)(nil), // 62: control.FlushCacheRequest.Body + (*FlushCacheResponse_Body)(nil), // 63: control.FlushCacheResponse.Body + (*DoctorRequest_Body)(nil), // 64: control.DoctorRequest.Body + (*DoctorResponse_Body)(nil), // 65: control.DoctorResponse.Body + (*StartShardEvacuationRequest_Body)(nil), // 66: control.StartShardEvacuationRequest.Body + (*StartShardEvacuationResponse_Body)(nil), // 67: control.StartShardEvacuationResponse.Body + (*GetShardEvacuationStatusRequest_Body)(nil), // 68: control.GetShardEvacuationStatusRequest.Body + (*GetShardEvacuationStatusResponse_Body)(nil), // 69: control.GetShardEvacuationStatusResponse.Body + (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 70: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 71: control.GetShardEvacuationStatusResponse.Body.Duration + (*ResetShardEvacuationStatusRequest_Body)(nil), // 72: control.ResetShardEvacuationStatusRequest.Body + (*ResetShardEvacuationStatusResponse_Body)(nil), // 73: control.ResetShardEvacuationStatusResponse.Body + (*StopShardEvacuationRequest_Body)(nil), // 74: control.StopShardEvacuationRequest.Body + (*StopShardEvacuationResponse_Body)(nil), // 75: control.StopShardEvacuationResponse.Body + (*AddChainLocalOverrideRequest_Body)(nil), // 76: control.AddChainLocalOverrideRequest.Body + (*AddChainLocalOverrideResponse_Body)(nil), // 77: control.AddChainLocalOverrideResponse.Body + (*GetChainLocalOverrideRequest_Body)(nil), // 78: control.GetChainLocalOverrideRequest.Body + (*GetChainLocalOverrideResponse_Body)(nil), // 79: control.GetChainLocalOverrideResponse.Body + (*ListChainLocalOverridesRequest_Body)(nil), // 80: control.ListChainLocalOverridesRequest.Body + (*ListChainLocalOverridesResponse_Body)(nil), // 81: control.ListChainLocalOverridesResponse.Body + (*ListTargetsLocalOverridesRequest_Body)(nil), // 82: control.ListTargetsLocalOverridesRequest.Body + (*ListTargetsLocalOverridesResponse_Body)(nil), // 83: control.ListTargetsLocalOverridesResponse.Body + (*RemoveChainLocalOverrideRequest_Body)(nil), // 84: control.RemoveChainLocalOverrideRequest.Body + (*RemoveChainLocalOverrideResponse_Body)(nil), // 85: control.RemoveChainLocalOverrideResponse.Body + (*RemoveChainLocalOverridesByTargetRequest_Body)(nil), // 86: control.RemoveChainLocalOverridesByTargetRequest.Body + (*RemoveChainLocalOverridesByTargetResponse_Body)(nil), // 87: control.RemoveChainLocalOverridesByTargetResponse.Body + (*SealWriteCacheRequest_Body)(nil), // 88: control.SealWriteCacheRequest.Body + (*SealWriteCacheResponse_Body)(nil), // 89: control.SealWriteCacheResponse.Body + (*SealWriteCacheResponse_Body_Status)(nil), // 90: control.SealWriteCacheResponse.Body.Status + (*DetachShardsRequest_Body)(nil), // 91: control.DetachShardsRequest.Body + (*DetachShardsResponse_Body)(nil), // 92: control.DetachShardsResponse.Body + (*Signature)(nil), // 93: control.Signature + (NetmapStatus)(0), // 94: control.NetmapStatus + (HealthStatus)(0), // 95: control.HealthStatus + (*ShardInfo)(nil), // 96: control.ShardInfo + (ShardMode)(0), // 97: control.ShardMode + (*ChainTarget)(nil), // 98: control.ChainTarget } var file_pkg_services_control_service_proto_depIdxs = []int32{ - 44, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 89, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 45, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 89, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 46, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 89, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 47, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 89, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 48, // 8: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 89, // 9: control.DropObjectsRequest.signature:type_name -> control.Signature - 49, // 10: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 89, // 11: control.DropObjectsResponse.signature:type_name -> control.Signature - 50, // 12: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 89, // 13: control.ListShardsRequest.signature:type_name -> control.Signature - 51, // 14: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 89, // 15: control.ListShardsResponse.signature:type_name -> control.Signature - 52, // 16: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 89, // 17: control.SetShardModeRequest.signature:type_name -> control.Signature - 53, // 18: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 89, // 19: control.SetShardModeResponse.signature:type_name -> control.Signature - 54, // 20: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 89, // 21: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 55, // 22: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 89, // 23: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 56, // 24: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 89, // 25: control.EvacuateShardRequest.signature:type_name -> control.Signature - 57, // 26: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 89, // 27: control.EvacuateShardResponse.signature:type_name -> control.Signature - 58, // 28: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 89, // 29: control.FlushCacheRequest.signature:type_name -> control.Signature - 59, // 30: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 89, // 31: control.FlushCacheResponse.signature:type_name -> control.Signature - 60, // 32: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body - 89, // 33: control.DoctorRequest.signature:type_name -> control.Signature - 61, // 34: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body - 89, // 35: control.DoctorResponse.signature:type_name -> control.Signature - 62, // 36: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body - 89, // 37: control.StartShardEvacuationRequest.signature:type_name -> control.Signature - 63, // 38: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body - 89, // 39: control.StartShardEvacuationResponse.signature:type_name -> control.Signature - 64, // 40: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body - 89, // 41: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 65, // 42: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body - 89, // 43: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 68, // 44: control.ResetShardEvacuationStatusRequest.body:type_name -> control.ResetShardEvacuationStatusRequest.Body - 89, // 45: control.ResetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 69, // 46: control.ResetShardEvacuationStatusResponse.body:type_name -> control.ResetShardEvacuationStatusResponse.Body - 89, // 47: control.ResetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 70, // 48: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body - 89, // 49: control.StopShardEvacuationRequest.signature:type_name -> control.Signature - 71, // 50: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body - 89, // 51: control.StopShardEvacuationResponse.signature:type_name -> control.Signature - 72, // 52: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body - 89, // 53: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature - 73, // 54: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body - 89, // 55: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature - 74, // 56: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body - 89, // 57: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature - 75, // 58: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body - 89, // 59: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature - 76, // 60: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body - 89, // 61: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature - 77, // 62: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body - 89, // 63: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature - 78, // 64: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body - 89, // 65: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature - 79, // 66: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body - 89, // 67: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature - 80, // 68: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body - 89, // 69: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature - 81, // 70: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body - 89, // 71: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature - 82, // 72: control.RemoveChainLocalOverridesByTargetRequest.body:type_name -> control.RemoveChainLocalOverridesByTargetRequest.Body - 89, // 73: control.RemoveChainLocalOverridesByTargetRequest.signature:type_name -> control.Signature - 83, // 74: control.RemoveChainLocalOverridesByTargetResponse.body:type_name -> control.RemoveChainLocalOverridesByTargetResponse.Body - 89, // 75: control.RemoveChainLocalOverridesByTargetResponse.signature:type_name -> control.Signature - 84, // 76: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body - 89, // 77: control.SealWriteCacheRequest.signature:type_name -> control.Signature - 85, // 78: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body - 89, // 79: control.SealWriteCacheResponse.signature:type_name -> control.Signature - 87, // 80: control.DetachShardsRequest.body:type_name -> control.DetachShardsRequest.Body - 89, // 81: control.DetachShardsRequest.signature:type_name -> control.Signature - 88, // 82: control.DetachShardsResponse.body:type_name -> control.DetachShardsResponse.Body - 89, // 83: control.DetachShardsResponse.signature:type_name -> control.Signature - 90, // 84: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 91, // 85: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 90, // 86: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 92, // 87: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 93, // 88: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 1, // 89: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status - 67, // 90: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration - 66, // 91: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - 94, // 92: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 94, // 93: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 94, // 94: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget - 94, // 95: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget - 94, // 96: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 94, // 97: control.RemoveChainLocalOverridesByTargetRequest.Body.target:type_name -> control.ChainTarget - 86, // 98: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status - 2, // 99: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 4, // 100: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 6, // 101: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 8, // 102: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 10, // 103: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 12, // 104: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 14, // 105: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 20, // 106: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest - 22, // 107: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest - 24, // 108: control.ControlService.ResetShardEvacuationStatus:input_type -> control.ResetShardEvacuationStatusRequest - 26, // 109: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest - 16, // 110: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 18, // 111: control.ControlService.Doctor:input_type -> control.DoctorRequest - 28, // 112: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest - 30, // 113: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest - 32, // 114: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest - 36, // 115: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest - 38, // 116: control.ControlService.RemoveChainLocalOverridesByTarget:input_type -> control.RemoveChainLocalOverridesByTargetRequest - 34, // 117: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest - 40, // 118: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest - 42, // 119: control.ControlService.DetachShards:input_type -> control.DetachShardsRequest - 3, // 120: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 5, // 121: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 7, // 122: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 9, // 123: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 11, // 124: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 13, // 125: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 15, // 126: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 21, // 127: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse - 23, // 128: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse - 25, // 129: control.ControlService.ResetShardEvacuationStatus:output_type -> control.ResetShardEvacuationStatusResponse - 27, // 130: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse - 17, // 131: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 19, // 132: control.ControlService.Doctor:output_type -> control.DoctorResponse - 29, // 133: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse - 31, // 134: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse - 33, // 135: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse - 37, // 136: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse - 39, // 137: control.ControlService.RemoveChainLocalOverridesByTarget:output_type -> control.RemoveChainLocalOverridesByTargetResponse - 35, // 138: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse - 41, // 139: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse - 43, // 140: control.ControlService.DetachShards:output_type -> control.DetachShardsResponse - 120, // [120:141] is the sub-list for method output_type - 99, // [99:120] is the sub-list for method input_type - 99, // [99:99] is the sub-list for extension type_name - 99, // [99:99] is the sub-list for extension extendee - 0, // [0:99] is the sub-list for field type_name + 46, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body + 93, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature + 47, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body + 93, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature + 48, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body + 93, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature + 49, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body + 93, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature + 50, // 8: control.GetNetmapStatusRequest.body:type_name -> control.GetNetmapStatusRequest.Body + 93, // 9: control.GetNetmapStatusRequest.signature:type_name -> control.Signature + 51, // 10: control.GetNetmapStatusResponse.body:type_name -> control.GetNetmapStatusResponse.Body + 93, // 11: control.GetNetmapStatusResponse.signature:type_name -> control.Signature + 52, // 12: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body + 93, // 13: control.DropObjectsRequest.signature:type_name -> control.Signature + 53, // 14: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body + 93, // 15: control.DropObjectsResponse.signature:type_name -> control.Signature + 54, // 16: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body + 93, // 17: control.ListShardsRequest.signature:type_name -> control.Signature + 55, // 18: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body + 93, // 19: control.ListShardsResponse.signature:type_name -> control.Signature + 56, // 20: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body + 93, // 21: control.SetShardModeRequest.signature:type_name -> control.Signature + 57, // 22: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body + 93, // 23: control.SetShardModeResponse.signature:type_name -> control.Signature + 58, // 24: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body + 93, // 25: control.SynchronizeTreeRequest.signature:type_name -> control.Signature + 59, // 26: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body + 93, // 27: control.SynchronizeTreeResponse.signature:type_name -> control.Signature + 60, // 28: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body + 93, // 29: control.EvacuateShardRequest.signature:type_name -> control.Signature + 61, // 30: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body + 93, // 31: control.EvacuateShardResponse.signature:type_name -> control.Signature + 62, // 32: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body + 93, // 33: control.FlushCacheRequest.signature:type_name -> control.Signature + 63, // 34: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body + 93, // 35: control.FlushCacheResponse.signature:type_name -> control.Signature + 64, // 36: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body + 93, // 37: control.DoctorRequest.signature:type_name -> control.Signature + 65, // 38: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body + 93, // 39: control.DoctorResponse.signature:type_name -> control.Signature + 66, // 40: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body + 93, // 41: control.StartShardEvacuationRequest.signature:type_name -> control.Signature + 67, // 42: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body + 93, // 43: control.StartShardEvacuationResponse.signature:type_name -> control.Signature + 68, // 44: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body + 93, // 45: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 69, // 46: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body + 93, // 47: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 72, // 48: control.ResetShardEvacuationStatusRequest.body:type_name -> control.ResetShardEvacuationStatusRequest.Body + 93, // 49: control.ResetShardEvacuationStatusRequest.signature:type_name -> control.Signature + 73, // 50: control.ResetShardEvacuationStatusResponse.body:type_name -> control.ResetShardEvacuationStatusResponse.Body + 93, // 51: control.ResetShardEvacuationStatusResponse.signature:type_name -> control.Signature + 74, // 52: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body + 93, // 53: control.StopShardEvacuationRequest.signature:type_name -> control.Signature + 75, // 54: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body + 93, // 55: control.StopShardEvacuationResponse.signature:type_name -> control.Signature + 76, // 56: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body + 93, // 57: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature + 77, // 58: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body + 93, // 59: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature + 78, // 60: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body + 93, // 61: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature + 79, // 62: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body + 93, // 63: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature + 80, // 64: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body + 93, // 65: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature + 81, // 66: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body + 93, // 67: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature + 82, // 68: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body + 93, // 69: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature + 83, // 70: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body + 93, // 71: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature + 84, // 72: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body + 93, // 73: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature + 85, // 74: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body + 93, // 75: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature + 86, // 76: control.RemoveChainLocalOverridesByTargetRequest.body:type_name -> control.RemoveChainLocalOverridesByTargetRequest.Body + 93, // 77: control.RemoveChainLocalOverridesByTargetRequest.signature:type_name -> control.Signature + 87, // 78: control.RemoveChainLocalOverridesByTargetResponse.body:type_name -> control.RemoveChainLocalOverridesByTargetResponse.Body + 93, // 79: control.RemoveChainLocalOverridesByTargetResponse.signature:type_name -> control.Signature + 88, // 80: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body + 93, // 81: control.SealWriteCacheRequest.signature:type_name -> control.Signature + 89, // 82: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body + 93, // 83: control.SealWriteCacheResponse.signature:type_name -> control.Signature + 91, // 84: control.DetachShardsRequest.body:type_name -> control.DetachShardsRequest.Body + 93, // 85: control.DetachShardsRequest.signature:type_name -> control.Signature + 92, // 86: control.DetachShardsResponse.body:type_name -> control.DetachShardsResponse.Body + 93, // 87: control.DetachShardsResponse.signature:type_name -> control.Signature + 94, // 88: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus + 95, // 89: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus + 94, // 90: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus + 94, // 91: control.GetNetmapStatusResponse.Body.status:type_name -> control.NetmapStatus + 96, // 92: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo + 97, // 93: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode + 1, // 94: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status + 71, // 95: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration + 70, // 96: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp + 98, // 97: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 98, // 98: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 98, // 99: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget + 98, // 100: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget + 98, // 101: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget + 98, // 102: control.RemoveChainLocalOverridesByTargetRequest.Body.target:type_name -> control.ChainTarget + 90, // 103: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status + 2, // 104: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest + 4, // 105: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest + 6, // 106: control.ControlService.GetNetmapStatus:input_type -> control.GetNetmapStatusRequest + 8, // 107: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest + 10, // 108: control.ControlService.ListShards:input_type -> control.ListShardsRequest + 12, // 109: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest + 14, // 110: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest + 16, // 111: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest + 22, // 112: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest + 24, // 113: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest + 26, // 114: control.ControlService.ResetShardEvacuationStatus:input_type -> control.ResetShardEvacuationStatusRequest + 28, // 115: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest + 18, // 116: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest + 20, // 117: control.ControlService.Doctor:input_type -> control.DoctorRequest + 30, // 118: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest + 32, // 119: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest + 34, // 120: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest + 38, // 121: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest + 40, // 122: control.ControlService.RemoveChainLocalOverridesByTarget:input_type -> control.RemoveChainLocalOverridesByTargetRequest + 36, // 123: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest + 42, // 124: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest + 44, // 125: control.ControlService.DetachShards:input_type -> control.DetachShardsRequest + 3, // 126: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse + 5, // 127: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse + 7, // 128: control.ControlService.GetNetmapStatus:output_type -> control.GetNetmapStatusResponse + 9, // 129: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse + 11, // 130: control.ControlService.ListShards:output_type -> control.ListShardsResponse + 13, // 131: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse + 15, // 132: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse + 17, // 133: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse + 23, // 134: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse + 25, // 135: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse + 27, // 136: control.ControlService.ResetShardEvacuationStatus:output_type -> control.ResetShardEvacuationStatusResponse + 29, // 137: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse + 19, // 138: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse + 21, // 139: control.ControlService.Doctor:output_type -> control.DoctorResponse + 31, // 140: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse + 33, // 141: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse + 35, // 142: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse + 39, // 143: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse + 41, // 144: control.ControlService.RemoveChainLocalOverridesByTarget:output_type -> control.RemoveChainLocalOverridesByTargetResponse + 37, // 145: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse + 43, // 146: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse + 45, // 147: control.ControlService.DetachShards:output_type -> control.DetachShardsResponse + 126, // [126:148] is the sub-list for method output_type + 104, // [104:126] is the sub-list for method input_type + 104, // [104:104] is the sub-list for extension type_name + 104, // [104:104] is the sub-list for extension extendee + 0, // [0:104] is the sub-list for field type_name } func init() { file_pkg_services_control_service_proto_init() } @@ -5747,7 +5996,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest); i { + switch v := v.(*GetNetmapStatusRequest); i { case 0: return &v.state case 1: @@ -5759,7 +6008,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse); i { + switch v := v.(*GetNetmapStatusResponse); i { case 0: return &v.state case 1: @@ -5771,7 +6020,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest); i { + switch v := v.(*DropObjectsRequest); i { case 0: return &v.state case 1: @@ -5783,7 +6032,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse); i { + switch v := v.(*DropObjectsResponse); i { case 0: return &v.state case 1: @@ -5795,7 +6044,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest); i { + switch v := v.(*ListShardsRequest); i { case 0: return &v.state case 1: @@ -5807,7 +6056,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse); i { + switch v := v.(*ListShardsResponse); i { case 0: return &v.state case 1: @@ -5819,7 +6068,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest); i { + switch v := v.(*SetShardModeRequest); i { case 0: return &v.state case 1: @@ -5831,7 +6080,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse); i { + switch v := v.(*SetShardModeResponse); i { case 0: return &v.state case 1: @@ -5843,7 +6092,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest); i { + switch v := v.(*SynchronizeTreeRequest); i { case 0: return &v.state case 1: @@ -5855,7 +6104,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse); i { + switch v := v.(*SynchronizeTreeResponse); i { case 0: return &v.state case 1: @@ -5867,7 +6116,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest); i { + switch v := v.(*EvacuateShardRequest); i { case 0: return &v.state case 1: @@ -5879,7 +6128,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheResponse); i { + switch v := v.(*EvacuateShardResponse); i { case 0: return &v.state case 1: @@ -5891,7 +6140,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorRequest); i { + switch v := v.(*FlushCacheRequest); i { case 0: return &v.state case 1: @@ -5903,7 +6152,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorResponse); i { + switch v := v.(*FlushCacheResponse); i { case 0: return &v.state case 1: @@ -5915,7 +6164,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationRequest); i { + switch v := v.(*DoctorRequest); i { case 0: return &v.state case 1: @@ -5927,7 +6176,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationResponse); i { + switch v := v.(*DoctorResponse); i { case 0: return &v.state case 1: @@ -5939,7 +6188,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusRequest); i { + switch v := v.(*StartShardEvacuationRequest); i { case 0: return &v.state case 1: @@ -5951,7 +6200,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse); i { + switch v := v.(*StartShardEvacuationResponse); i { case 0: return &v.state case 1: @@ -5963,7 +6212,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResetShardEvacuationStatusRequest); i { + switch v := v.(*GetShardEvacuationStatusRequest); i { case 0: return &v.state case 1: @@ -5975,7 +6224,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResetShardEvacuationStatusResponse); i { + switch v := v.(*GetShardEvacuationStatusResponse); i { case 0: return &v.state case 1: @@ -5987,7 +6236,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationRequest); i { + switch v := v.(*ResetShardEvacuationStatusRequest); i { case 0: return &v.state case 1: @@ -5999,7 +6248,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationResponse); i { + switch v := v.(*ResetShardEvacuationStatusResponse); i { case 0: return &v.state case 1: @@ -6011,7 +6260,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideRequest); i { + switch v := v.(*StopShardEvacuationRequest); i { case 0: return &v.state case 1: @@ -6023,7 +6272,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideResponse); i { + switch v := v.(*StopShardEvacuationResponse); i { case 0: return &v.state case 1: @@ -6035,7 +6284,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideRequest); i { + switch v := v.(*AddChainLocalOverrideRequest); i { case 0: return &v.state case 1: @@ -6047,7 +6296,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideResponse); i { + switch v := v.(*AddChainLocalOverrideResponse); i { case 0: return &v.state case 1: @@ -6059,7 +6308,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesRequest); i { + switch v := v.(*GetChainLocalOverrideRequest); i { case 0: return &v.state case 1: @@ -6071,7 +6320,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesResponse); i { + switch v := v.(*GetChainLocalOverrideResponse); i { case 0: return &v.state case 1: @@ -6083,7 +6332,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesRequest); i { + switch v := v.(*ListChainLocalOverridesRequest); i { case 0: return &v.state case 1: @@ -6095,7 +6344,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesResponse); i { + switch v := v.(*ListChainLocalOverridesResponse); i { case 0: return &v.state case 1: @@ -6107,7 +6356,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideRequest); i { + switch v := v.(*ListTargetsLocalOverridesRequest); i { case 0: return &v.state case 1: @@ -6119,7 +6368,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideResponse); i { + switch v := v.(*ListTargetsLocalOverridesResponse); i { case 0: return &v.state case 1: @@ -6131,7 +6380,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverridesByTargetRequest); i { + switch v := v.(*RemoveChainLocalOverrideRequest); i { case 0: return &v.state case 1: @@ -6143,7 +6392,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverridesByTargetResponse); i { + switch v := v.(*RemoveChainLocalOverrideResponse); i { case 0: return &v.state case 1: @@ -6155,7 +6404,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheRequest); i { + switch v := v.(*RemoveChainLocalOverridesByTargetRequest); i { case 0: return &v.state case 1: @@ -6167,7 +6416,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse); i { + switch v := v.(*RemoveChainLocalOverridesByTargetResponse); i { case 0: return &v.state case 1: @@ -6179,7 +6428,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsRequest); i { + switch v := v.(*SealWriteCacheRequest); i { case 0: return &v.state case 1: @@ -6191,7 +6440,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsResponse); i { + switch v := v.(*SealWriteCacheResponse); i { case 0: return &v.state case 1: @@ -6203,7 +6452,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { + switch v := v.(*DetachShardsRequest); i { case 0: return &v.state case 1: @@ -6215,7 +6464,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { + switch v := v.(*DetachShardsResponse); i { case 0: return &v.state case 1: @@ -6227,7 +6476,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusRequest_Body); i { + switch v := v.(*HealthCheckRequest_Body); i { case 0: return &v.state case 1: @@ -6239,7 +6488,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusResponse_Body); i { + switch v := v.(*HealthCheckResponse_Body); i { case 0: return &v.state case 1: @@ -6251,7 +6500,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest_Body); i { + switch v := v.(*SetNetmapStatusRequest_Body); i { case 0: return &v.state case 1: @@ -6263,7 +6512,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse_Body); i { + switch v := v.(*SetNetmapStatusResponse_Body); i { case 0: return &v.state case 1: @@ -6275,7 +6524,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest_Body); i { + switch v := v.(*GetNetmapStatusRequest_Body); i { case 0: return &v.state case 1: @@ -6287,7 +6536,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse_Body); i { + switch v := v.(*GetNetmapStatusResponse_Body); i { case 0: return &v.state case 1: @@ -6299,7 +6548,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest_Body); i { + switch v := v.(*DropObjectsRequest_Body); i { case 0: return &v.state case 1: @@ -6311,7 +6560,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse_Body); i { + switch v := v.(*DropObjectsResponse_Body); i { case 0: return &v.state case 1: @@ -6323,7 +6572,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest_Body); i { + switch v := v.(*ListShardsRequest_Body); i { case 0: return &v.state case 1: @@ -6335,7 +6584,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse_Body); i { + switch v := v.(*ListShardsResponse_Body); i { case 0: return &v.state case 1: @@ -6347,7 +6596,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest_Body); i { + switch v := v.(*SetShardModeRequest_Body); i { case 0: return &v.state case 1: @@ -6359,7 +6608,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse_Body); i { + switch v := v.(*SetShardModeResponse_Body); i { case 0: return &v.state case 1: @@ -6371,7 +6620,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest_Body); i { + switch v := v.(*SynchronizeTreeRequest_Body); i { case 0: return &v.state case 1: @@ -6383,7 +6632,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheResponse_Body); i { + switch v := v.(*SynchronizeTreeResponse_Body); i { case 0: return &v.state case 1: @@ -6395,7 +6644,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorRequest_Body); i { + switch v := v.(*EvacuateShardRequest_Body); i { case 0: return &v.state case 1: @@ -6407,7 +6656,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorResponse_Body); i { + switch v := v.(*EvacuateShardResponse_Body); i { case 0: return &v.state case 1: @@ -6419,7 +6668,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationRequest_Body); i { + switch v := v.(*FlushCacheRequest_Body); i { case 0: return &v.state case 1: @@ -6431,7 +6680,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationResponse_Body); i { + switch v := v.(*FlushCacheResponse_Body); i { case 0: return &v.state case 1: @@ -6443,7 +6692,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusRequest_Body); i { + switch v := v.(*DoctorRequest_Body); i { case 0: return &v.state case 1: @@ -6455,7 +6704,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body); i { + switch v := v.(*DoctorResponse_Body); i { case 0: return &v.state case 1: @@ -6467,7 +6716,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { + switch v := v.(*StartShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -6479,7 +6728,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { + switch v := v.(*StartShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -6491,7 +6740,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResetShardEvacuationStatusRequest_Body); i { + switch v := v.(*GetShardEvacuationStatusRequest_Body); i { case 0: return &v.state case 1: @@ -6503,7 +6752,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResetShardEvacuationStatusResponse_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body); i { case 0: return &v.state case 1: @@ -6515,7 +6764,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationRequest_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { case 0: return &v.state case 1: @@ -6527,7 +6776,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationResponse_Body); i { + switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { case 0: return &v.state case 1: @@ -6539,7 +6788,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideRequest_Body); i { + switch v := v.(*ResetShardEvacuationStatusRequest_Body); i { case 0: return &v.state case 1: @@ -6551,7 +6800,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideResponse_Body); i { + switch v := v.(*ResetShardEvacuationStatusResponse_Body); i { case 0: return &v.state case 1: @@ -6563,7 +6812,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideRequest_Body); i { + switch v := v.(*StopShardEvacuationRequest_Body); i { case 0: return &v.state case 1: @@ -6575,7 +6824,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideResponse_Body); i { + switch v := v.(*StopShardEvacuationResponse_Body); i { case 0: return &v.state case 1: @@ -6587,7 +6836,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesRequest_Body); i { + switch v := v.(*AddChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -6599,7 +6848,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesResponse_Body); i { + switch v := v.(*AddChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -6611,7 +6860,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesRequest_Body); i { + switch v := v.(*GetChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -6623,7 +6872,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[77].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesResponse_Body); i { + switch v := v.(*GetChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -6635,7 +6884,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[78].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { + switch v := v.(*ListChainLocalOverridesRequest_Body); i { case 0: return &v.state case 1: @@ -6647,7 +6896,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[79].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { + switch v := v.(*ListChainLocalOverridesResponse_Body); i { case 0: return &v.state case 1: @@ -6659,7 +6908,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[80].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverridesByTargetRequest_Body); i { + switch v := v.(*ListTargetsLocalOverridesRequest_Body); i { case 0: return &v.state case 1: @@ -6671,7 +6920,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[81].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverridesByTargetResponse_Body); i { + switch v := v.(*ListTargetsLocalOverridesResponse_Body); i { case 0: return &v.state case 1: @@ -6683,7 +6932,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[82].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheRequest_Body); i { + switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { case 0: return &v.state case 1: @@ -6695,7 +6944,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[83].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse_Body); i { + switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { case 0: return &v.state case 1: @@ -6707,7 +6956,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[84].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse_Body_Status); i { + switch v := v.(*RemoveChainLocalOverridesByTargetRequest_Body); i { case 0: return &v.state case 1: @@ -6719,7 +6968,7 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[85].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsRequest_Body); i { + switch v := v.(*RemoveChainLocalOverridesByTargetResponse_Body); i { case 0: return &v.state case 1: @@ -6731,6 +6980,54 @@ func file_pkg_services_control_service_proto_init() { } } file_pkg_services_control_service_proto_msgTypes[86].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[87].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheResponse_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[88].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SealWriteCacheResponse_Body_Status); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[89].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DetachShardsRequest_Body); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_services_control_service_proto_msgTypes[90].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DetachShardsResponse_Body); i { case 0: return &v.state @@ -6749,7 +7046,7 @@ func file_pkg_services_control_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_services_control_service_proto_rawDesc, NumEnums: 2, - NumMessages: 87, + NumMessages: 91, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 94032b346..2cd8434fc 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -15,6 +15,9 @@ service ControlService { // Sets status of the storage node in FrostFS network map. rpc SetNetmapStatus(SetNetmapStatusRequest) returns (SetNetmapStatusResponse); + // Gets status of the storage node in FrostFS network map. + rpc GetNetmapStatus(GetNetmapStatusRequest) returns (GetNetmapStatusResponse); + // Mark objects to be removed from node's local object storage. rpc DropObjects(DropObjectsRequest) returns (DropObjectsResponse); @@ -156,6 +159,34 @@ message SetNetmapStatusResponse { Signature signature = 2; } +// Get netmap status request. +message GetNetmapStatusRequest { + message Body {} + + // Body of set netmap status request message. + Body body = 1; + + // Body signature. + Signature signature = 2; +} + +// Get netmap status response. +message GetNetmapStatusResponse { + message Body { + // Storage node status in FrostFS network map. + NetmapStatus status = 1; + + // Network map epoch. + uint64 epoch = 2; + } + + // Body of get netmap status response message. + Body body = 1; + + // Body signature. + Signature signature = 2; +} + // Request to drop the objects. message DropObjectsRequest { // Request body structure. diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 9d8d6376c..a287606fa 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -334,6 +334,171 @@ func (x *SetNetmapStatusResponse) SetSignature(sig *Signature) { x.Signature = sig } +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetNetmapStatusRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetNetmapStatusRequest_Body) StableMarshal(buf []byte) []byte { + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetNetmapStatusRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetNetmapStatusRequest) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *GetNetmapStatusRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *GetNetmapStatusRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *GetNetmapStatusRequest) SetSignature(sig *Signature) { + x.Signature = sig +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetNetmapStatusResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.EnumSize(1, int32(x.Status)) + size += proto.UInt64Size(2, x.Epoch) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetNetmapStatusResponse_Body) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.EnumMarshal(1, buf[offset:], int32(x.Status)) + offset += proto.UInt64Marshal(2, buf[offset:], x.Epoch) + return buf +} + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *GetNetmapStatusResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// StableMarshal marshals x in protobuf binary format with stable field order. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *GetNetmapStatusResponse) StableMarshal(buf []byte) []byte { + if x == nil { + return []byte{} + } + if buf == nil { + buf = make([]byte, x.StableSize()) + } + var offset int + offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) + offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) + return buf +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *GetNetmapStatusResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *GetNetmapStatusResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf), nil +} + +func (x *GetNetmapStatusResponse) SetSignature(sig *Signature) { + x.Signature = sig +} + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index feeee0006..fa9de974a 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -21,6 +21,7 @@ const _ = grpc.SupportPackageIsVersion7 const ( ControlService_HealthCheck_FullMethodName = "/control.ControlService/HealthCheck" ControlService_SetNetmapStatus_FullMethodName = "/control.ControlService/SetNetmapStatus" + ControlService_GetNetmapStatus_FullMethodName = "/control.ControlService/GetNetmapStatus" ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects" ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" @@ -50,6 +51,8 @@ type ControlServiceClient interface { HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) // Sets status of the storage node in FrostFS network map. SetNetmapStatus(ctx context.Context, in *SetNetmapStatusRequest, opts ...grpc.CallOption) (*SetNetmapStatusResponse, error) + // Gets status of the storage node in FrostFS network map. + GetNetmapStatus(ctx context.Context, in *GetNetmapStatusRequest, opts ...grpc.CallOption) (*GetNetmapStatusResponse, error) // Mark objects to be removed from node's local object storage. DropObjects(ctx context.Context, in *DropObjectsRequest, opts ...grpc.CallOption) (*DropObjectsResponse, error) // Returns list that contains information about all shards of a node. @@ -122,6 +125,15 @@ func (c *controlServiceClient) SetNetmapStatus(ctx context.Context, in *SetNetma return out, nil } +func (c *controlServiceClient) GetNetmapStatus(ctx context.Context, in *GetNetmapStatusRequest, opts ...grpc.CallOption) (*GetNetmapStatusResponse, error) { + out := new(GetNetmapStatusResponse) + err := c.cc.Invoke(ctx, ControlService_GetNetmapStatus_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *controlServiceClient) DropObjects(ctx context.Context, in *DropObjectsRequest, opts ...grpc.CallOption) (*DropObjectsResponse, error) { out := new(DropObjectsResponse) err := c.cc.Invoke(ctx, ControlService_DropObjects_FullMethodName, in, out, opts...) @@ -301,6 +313,8 @@ type ControlServiceServer interface { HealthCheck(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) // Sets status of the storage node in FrostFS network map. SetNetmapStatus(context.Context, *SetNetmapStatusRequest) (*SetNetmapStatusResponse, error) + // Gets status of the storage node in FrostFS network map. + GetNetmapStatus(context.Context, *GetNetmapStatusRequest) (*GetNetmapStatusResponse, error) // Mark objects to be removed from node's local object storage. DropObjects(context.Context, *DropObjectsRequest) (*DropObjectsResponse, error) // Returns list that contains information about all shards of a node. @@ -357,6 +371,9 @@ func (UnimplementedControlServiceServer) HealthCheck(context.Context, *HealthChe func (UnimplementedControlServiceServer) SetNetmapStatus(context.Context, *SetNetmapStatusRequest) (*SetNetmapStatusResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method SetNetmapStatus not implemented") } +func (UnimplementedControlServiceServer) GetNetmapStatus(context.Context, *GetNetmapStatusRequest) (*GetNetmapStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetNetmapStatus not implemented") +} func (UnimplementedControlServiceServer) DropObjects(context.Context, *DropObjectsRequest) (*DropObjectsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method DropObjects not implemented") } @@ -462,6 +479,24 @@ func _ControlService_SetNetmapStatus_Handler(srv interface{}, ctx context.Contex return interceptor(ctx, in, info, handler) } +func _ControlService_GetNetmapStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetNetmapStatusRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).GetNetmapStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_GetNetmapStatus_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).GetNetmapStatus(ctx, req.(*GetNetmapStatusRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _ControlService_DropObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DropObjectsRequest) if err := dec(in); err != nil { @@ -819,6 +854,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "SetNetmapStatus", Handler: _ControlService_SetNetmapStatus_Handler, }, + { + MethodName: "GetNetmapStatus", + Handler: _ControlService_GetNetmapStatus_Handler, + }, { MethodName: "DropObjects", Handler: _ControlService_DropObjects_Handler, From 46732b61d7399f9b067c1490d719bdb959d5de40 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 20 Jun 2024 16:04:55 +0300 Subject: [PATCH 1174/1943] [#60] cli: Add `await` flag to `control set-status` Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/internal/commonflags/flags.go | 3 + .../modules/control/set_netmap_status.go | 64 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/cmd/frostfs-cli/internal/commonflags/flags.go b/cmd/frostfs-cli/internal/commonflags/flags.go index 6915ef42b..23dbc8c2b 100644 --- a/cmd/frostfs-cli/internal/commonflags/flags.go +++ b/cmd/frostfs-cli/internal/commonflags/flags.go @@ -50,6 +50,9 @@ const ( TracingFlag = "trace" TracingFlagUsage = "Generate trace ID and print it." + + AwaitFlag = "await" + AwaitFlagUsage = "Wait for the operation to complete" ) // Init adds common flags to the command: diff --git a/cmd/frostfs-cli/modules/control/set_netmap_status.go b/cmd/frostfs-cli/modules/control/set_netmap_status.go index 3aa741689..31ade1eb9 100644 --- a/cmd/frostfs-cli/modules/control/set_netmap_status.go +++ b/cmd/frostfs-cli/modules/control/set_netmap_status.go @@ -1,7 +1,10 @@ package control import ( + "crypto/ecdsa" + "errors" "fmt" + "time" rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" @@ -9,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" ) @@ -18,8 +22,13 @@ const ( netmapStatusOnline = "online" netmapStatusOffline = "offline" netmapStatusMaintenance = "maintenance" + + maxSetStatusMaxWaitTime = 30 * time.Minute + setStatusWaitTimeout = 30 * time.Second ) +var errNetmapStatusAwaitFailed = errors.New("netmap status hasn't changed for 30 minutes") + var setNetmapStatusCmd = &cobra.Command{ Use: "set-status", Short: "Set status of the storage node in FrostFS network map", @@ -43,6 +52,8 @@ func initControlSetNetmapStatusCmd() { flags.BoolP(commonflags.ForceFlag, commonflags.ForceFlagShorthand, false, "Force turning to local maintenance") + + flags.Bool(commonflags.AwaitFlag, false, commonflags.AwaitFlagUsage) } func setNetmapStatus(cmd *cobra.Command, _ []string) { @@ -56,15 +67,19 @@ func setNetmapStatus(cmd *cobra.Command, _ []string) { } } + await, _ := cmd.Flags().GetBool(commonflags.AwaitFlag) + var targetStatus control.NetmapStatus switch st, _ := cmd.Flags().GetString(netmapStatusFlag); st { default: commonCmd.ExitOnErr(cmd, "", fmt.Errorf("unsupported status %s", st)) case netmapStatusOnline: body.SetStatus(control.NetmapStatus_ONLINE) printIgnoreForce(control.NetmapStatus_ONLINE) + targetStatus = control.NetmapStatus_ONLINE case netmapStatusOffline: body.SetStatus(control.NetmapStatus_OFFLINE) printIgnoreForce(control.NetmapStatus_OFFLINE) + targetStatus = control.NetmapStatus_OFFLINE case netmapStatusMaintenance: body.SetStatus(control.NetmapStatus_MAINTENANCE) @@ -72,6 +87,7 @@ func setNetmapStatus(cmd *cobra.Command, _ []string) { body.SetForceMaintenance() common.PrintVerbose(cmd, "Local maintenance will be forced.") } + targetStatus = control.NetmapStatus_MAINTENANCE } req := new(control.SetNetmapStatusRequest) @@ -92,4 +108,52 @@ func setNetmapStatus(cmd *cobra.Command, _ []string) { verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) cmd.Println("Network status update request successfully sent.") + + if await { + awaitSetNetmapStatus(cmd, pk, cli, targetStatus) + } +} + +func awaitSetNetmapStatus(cmd *cobra.Command, pk *ecdsa.PrivateKey, cli *client.Client, targetStatus control.NetmapStatus) { + req := &control.GetNetmapStatusRequest{ + Body: &control.GetNetmapStatusRequest_Body{}, + } + signRequest(cmd, pk, req) + var epoch uint64 + var status control.NetmapStatus + startTime := time.Now() + cmd.Println("Wait until epoch and netmap status change...") + for { + var resp *control.GetNetmapStatusResponse + var err error + err = cli.ExecRaw(func(client *rawclient.Client) error { + resp, err = control.GetNetmapStatus(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "failed to get current netmap status: %w", err) + + if epoch == 0 { + epoch = resp.GetBody().GetEpoch() + } + + status = resp.GetBody().GetStatus() + if resp.GetBody().GetEpoch() > epoch { + epoch = resp.GetBody().GetEpoch() + cmd.Printf("Epoch changed to %d\n", resp.GetBody().GetEpoch()) + } + + if status == targetStatus { + break + } + + if time.Since(startTime) > maxSetStatusMaxWaitTime { + commonCmd.ExitOnErr(cmd, "failed to wait netmap status: %w", errNetmapStatusAwaitFailed) + return + } + + time.Sleep(setStatusWaitTimeout) + + cmd.Printf("Current netmap status '%s', target status '%s'\n", status.String(), targetStatus.String()) + } + cmd.Printf("Netmap status changed to '%s' successfully.\n", status.String()) } From a1f7615b7e97c47a194715ede85189b61a34c84c Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 20 Jun 2024 14:50:17 +0300 Subject: [PATCH 1175/1943] [#1190] ape: Introduce `Groups` util function to retrieve actor's groupIDs Signed-off-by: Airat Arifullin --- pkg/ape/request/frostfsid.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/ape/request/frostfsid.go b/pkg/ape/request/frostfsid.go index fa620e58c..c0413678d 100644 --- a/pkg/ape/request/frostfsid.go +++ b/pkg/ape/request/frostfsid.go @@ -34,3 +34,19 @@ func FormFrostfsIDRequestProperties(frostFSIDClient frostfsidcore.SubjectProvide return reqProps, nil } + +// Groups return the actor's group ids from frostfsid contract. +func Groups(frostFSIDClient frostfsidcore.SubjectProvider, pk *keys.PublicKey) ([]string, error) { + subj, err := frostFSIDClient.GetSubjectExtended(pk.GetScriptHash()) + if err != nil { + if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { + return nil, fmt.Errorf("get subject error: %w", err) + } + return []string{}, nil + } + groups := make([]string, len(subj.Groups)) + for i, group := range subj.Groups { + groups[i] = strconv.FormatInt(group.ID, 10) + } + return groups, nil +} From 621dbf58abe4401971d497f79fd52987e6b6d3c5 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 20 Jun 2024 15:20:48 +0300 Subject: [PATCH 1176/1943] [#1190] container: GroupIDs must also be target of APE checks * Also add new test case for ape middleware in container service. Signed-off-by: Airat Arifullin --- pkg/services/container/ape.go | 42 ++++++++++++- pkg/services/container/ape_test.go | 99 ++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index fb1cc8121..3ea591c6a 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -164,11 +164,26 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co reqProps, ) + groups, err := aperequest.Groups(ac.frostFSIDClient, pk) + if err != nil { + return nil, fmt.Errorf("failed to get group ids: %w", err) + } + + // Policy contract keeps group related chains as namespace-group pair. + for i := range groups { + groups[i] = fmt.Sprintf("%s:%s", namespace, groups[i]) + } + rt := policyengine.NewRequestTargetWithNamespace(namespace) rt.User = &policyengine.Target{ Type: policyengine.User, Name: fmt.Sprintf("%s:%s", namespace, pk.Address()), } + rt.Groups = make([]policyengine.Target, len(groups)) + for i := range groups { + rt.Groups[i] = policyengine.GroupTarget(groups[i]) + } + s, found, err := ac.router.IsAllowed(apechain.Ingress, rt, request) if err != nil { return nil, err @@ -222,11 +237,26 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont reqProps, ) + groups, err := aperequest.Groups(ac.frostFSIDClient, pk) + if err != nil { + return nil, fmt.Errorf("failed to get group ids: %w", err) + } + + // Policy contract keeps group related chains as namespace-group pair. + for i := range groups { + groups[i] = fmt.Sprintf("%s:%s", namespace, groups[i]) + } + rt := policyengine.NewRequestTargetWithNamespace(namespace) rt.User = &policyengine.Target{ Type: policyengine.User, Name: fmt.Sprintf("%s:%s", namespace, pk.Address()), } + rt.Groups = make([]policyengine.Target, len(groups)) + for i := range groups { + rt.Groups[i] = policyengine.GroupTarget(groups[i]) + } + s, found, err := ac.router.IsAllowed(apechain.Ingress, rt, request) if err != nil { return nil, err @@ -311,6 +341,16 @@ func (ac *apeChecker) validateContainerBoundedOperation(ctx context.Context, con namespace = cntNamespace } + groups, err := aperequest.Groups(ac.frostFSIDClient, pk) + if err != nil { + return fmt.Errorf("failed to get group ids: %w", err) + } + + // Policy contract keeps group related chains as namespace-group pair. + for i := range groups { + groups[i] = fmt.Sprintf("%s:%s", namespace, groups[i]) + } + request := aperequest.NewRequest( op, aperequest.NewResource( @@ -321,7 +361,7 @@ func (ac *apeChecker) validateContainerBoundedOperation(ctx context.Context, con ) s, found, err := ac.router.IsAllowed(apechain.Ingress, - policyengine.NewRequestTargetExtended(namespace, id.EncodeToString(), fmt.Sprintf("%s:%s", namespace, pk.Address()), nil), + policyengine.NewRequestTargetExtended(namespace, id.EncodeToString(), fmt.Sprintf("%s:%s", namespace, pk.Address()), groups), request) if err != nil { return err diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index 90c0225dd..a6f0fb222 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -44,6 +44,7 @@ const ( func TestAPE(t *testing.T) { t.Parallel() t.Run("allow then deny get container", testAllowThenDenyGetContainerRuleDefined) + t.Run("allow by group id", TestAllowByGroupIDs) t.Run("deny get container no rule found", testDenyGetContainerNoRuleFound) t.Run("deny get container for others", testDenyGetContainerForOthers) t.Run("deny get container by user claim tag", testDenyGetContainerByUserClaimTag) @@ -145,6 +146,104 @@ func testAllowThenDenyGetContainerRuleDefined(t *testing.T) { require.Contains(t, errAccessDenied.Reason(), chain.AccessDenied.String()) } +func TestAllowByGroupIDs(t *testing.T) { + t.Parallel() + srv := &srvStub{ + calls: map[string]int{}, + } + router := inmemory.NewInMemory() + contRdr := &containerStub{ + c: map[cid.ID]*containercore.Container{}, + } + ir := &irStub{ + keys: [][]byte{}, + } + nm := &netmapStub{} + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{ + pk.PublicKey().GetScriptHash(): { + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + }, + }, + subjectsExt: map[util.Uint160]*client.SubjectExtended{ + pk.PublicKey().GetScriptHash(): { + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + Groups: []*client.Group{ + { + ID: 1, + Name: "Group#1", + }, + }, + }, + }, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) + + contID := cidtest.ID() + testContainer := containertest.Container() + pp := netmap.PlacementPolicy{} + require.NoError(t, pp.DecodeString("REP 1")) + testContainer.SetPlacementPolicy(pp) + contRdr.c[contID] = &containercore.Container{Value: testContainer} + + nm.currentEpoch = 100 + nm.netmaps = map[uint64]*netmap.NetMap{} + var testNetmap netmap.NetMap + testNetmap.SetEpoch(nm.currentEpoch) + testNetmap.SetNodes([]netmap.NodeInfo{{}}) + nm.netmaps[nm.currentEpoch] = &testNetmap + nm.netmaps[nm.currentEpoch-1] = &testNetmap + + req := &container.GetRequest{} + req.SetBody(&container.GetRequestBody{}) + var refContID refs.ContainerID + contID.WriteToV2(&refContID) + req.GetBody().SetContainerID(&refContID) + + require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) + + _, _, err = router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.GroupTarget(":1"), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.Allow, + Actions: chain.Actions{ + Names: []string{ + nativeschema.MethodGetContainer, + }, + }, + Resources: chain.Resources{ + Names: []string{ + fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), + }, + }, + Condition: []chain.Condition{ + { + Kind: chain.KindRequest, + Key: commonschema.PropertyKeyFrostFSIDGroupID, + Value: "1", + Op: chain.CondStringEquals, + }, + }, + }, + }, + }) + require.NoError(t, err) + + resp, err := apeSrv.Get(context.Background(), req) + require.NotNil(t, resp) + require.NoError(t, err) +} + func testDenyGetContainerNoRuleFound(t *testing.T) { t.Parallel() srv := &srvStub{ From 0b87388c183618882b8a1bf8d446c69ceb4df8a6 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 20 Jun 2024 15:39:39 +0300 Subject: [PATCH 1177/1943] [#1190] object: GroupIDs must also be target of APE checks * Also add new test case for ape middleware in container service. Signed-off-by: Airat Arifullin --- pkg/services/object/ape/checker.go | 13 ++++++- pkg/services/object/ape/checker_test.go | 51 +++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index ee71e6e1d..ee12d7b97 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -7,6 +7,7 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/router" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" @@ -159,11 +160,19 @@ func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { if err != nil { return fmt.Errorf("failed to create ape request: %w", err) } - pub, err := keys.NewPublicKeyFromString(prm.SenderKey) if err != nil { return err } + groups, err := aperequest.Groups(c.frostFSIDClient, pub) + if err != nil { + return fmt.Errorf("failed to get group ids: %w", err) + } + + // Policy contract keeps group related chains as namespace-group pair. + for i := range groups { + groups[i] = fmt.Sprintf("%s:%s", prm.Namespace, groups[i]) + } if prm.BearerToken != nil && !prm.BearerToken.Impersonate() { if err := isValidBearer(prm.BearerToken, prm.ContainerOwner, prm.Container, pub, c.st); err != nil { @@ -185,7 +194,7 @@ func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { } } - rt := policyengine.NewRequestTargetExtended(prm.Namespace, prm.Container.EncodeToString(), fmt.Sprintf("%s:%s", prm.Namespace, pub.Address()), nil) + rt := policyengine.NewRequestTargetExtended(prm.Namespace, prm.Container.EncodeToString(), fmt.Sprintf("%s:%s", prm.Namespace, pub.Address()), groups) status, ruleFound, err := c.chainRouter.IsAllowed(apechain.Ingress, rt, r) if err != nil { return err diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index 09954602c..5efd3669b 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -27,6 +27,7 @@ import ( "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" + commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" @@ -159,6 +160,8 @@ var ( objectID = "BzQw5HH3feoxFDD5tCT87Y1726qzgLfxEE7wgtoRzB3R" + groupID = "1" + role = "Container" senderPrivateKey, _ = keys.NewPrivateKey() @@ -238,6 +241,7 @@ var apeCheckTestCases = []struct { methods []string header testHeader containerRules []chain.Rule + groupidRules []chain.Rule expectAPEErr bool }{ { @@ -393,6 +397,36 @@ var apeCheckTestCases = []struct { }, expectAPEErr: true, }, + { + name: "optional oid requests reached quota limit by group-id", + container: containerID, + methods: methodsOptionalOID, + header: testHeader{ + headerObjSDK: &headerObjectSDKParams{ + payloadSize: 1000, + }, + fromRequestResponseHeader: true, + }, + groupidRules: []chain.Rule{ + { + Status: chain.QuotaLimitReached, + Actions: chain.Actions{Names: methodsOptionalOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + Any: true, + Condition: []chain.Condition{ + { + Op: chain.CondStringEquals, + Kind: chain.KindRequest, + Key: commonschema.PropertyKeyFrostFSIDGroupID, + Value: groupID, + }, + }, + }, + }, + expectAPEErr: true, + }, } type stMock struct{} @@ -486,10 +520,19 @@ func TestAPECheck(t *testing.T) { ls := inmemory.NewInmemoryLocalStorage() ms := inmemory.NewInmemoryMorphRuleChainStorage() - ls.AddOverride(chain.Ingress, policyengine.ContainerTarget(test.container), &chain.Chain{ - Rules: test.containerRules, - MatchType: chain.MatchTypeFirstMatch, - }) + if len(test.containerRules) > 0 { + ls.AddOverride(chain.Ingress, policyengine.ContainerTarget(test.container), &chain.Chain{ + Rules: test.containerRules, + MatchType: chain.MatchTypeFirstMatch, + }) + } + + if len(test.groupidRules) > 0 { + ls.AddOverride(chain.Ingress, policyengine.GroupTarget(":"+groupID), &chain.Chain{ + Rules: test.groupidRules, + MatchType: chain.MatchTypeFirstMatch, + }) + } router := policyengine.NewDefaultChainRouterWithLocalOverrides(ms, ls) From 11a38a0a8426de556ac17d4518cded38fa98e902 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 20 Jun 2024 15:49:22 +0300 Subject: [PATCH 1178/1943] [#1190] tree: GroupIDs must also be target of APE checks Signed-off-by: Airat Arifullin --- pkg/services/tree/ape.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 475567c5f..116adf5db 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -161,7 +161,17 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, } } - rt := engine.NewRequestTargetExtended(namespace, cid.EncodeToString(), fmt.Sprintf("%s:%s", namespace, publicKey.Address()), nil) + groups, err := aperequest.Groups(s.frostfsidSubjectProvider, publicKey) + if err != nil { + return fmt.Errorf("failed to get group ids: %w", err) + } + + // Policy contract keeps group related chains as namespace-group pair. + for i := range groups { + groups[i] = fmt.Sprintf("%s:%s", namespace, groups[i]) + } + + rt := engine.NewRequestTargetExtended(namespace, cid.EncodeToString(), fmt.Sprintf("%s:%s", namespace, publicKey.Address()), groups) status, found, err := s.router.IsAllowed(apechain.Ingress, rt, request) if err != nil { return apeErr(err) From 81ea91de52956ff0db8fe31728a172ab7f2b7711 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Jun 2024 17:12:26 +0300 Subject: [PATCH 1179/1943] [#451] metrics: Move to internal `metrics` don't look like something others want to import. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/main.go | 2 +- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/netmap.go | 2 +- {pkg => internal}/metrics/blobovnicza.go | 0 {pkg => internal}/metrics/blobstore.go | 0 {pkg => internal}/metrics/consts.go | 0 {pkg => internal}/metrics/engine.go | 0 {pkg => internal}/metrics/fstree.go | 0 {pkg => internal}/metrics/gc.go | 0 {pkg => internal}/metrics/grpc.go | 0 {pkg => internal}/metrics/innerring.go | 0 {pkg => internal}/metrics/metabase.go | 0 {pkg => internal}/metrics/mode.go | 0 {pkg => internal}/metrics/morph.go | 0 {pkg => internal}/metrics/morphcache.go | 0 {pkg => internal}/metrics/node.go | 0 {pkg => internal}/metrics/object.go | 0 {pkg => internal}/metrics/pilorama.go | 0 {pkg => internal}/metrics/policer.go | 0 {pkg => internal}/metrics/replicator.go | 0 {pkg => internal}/metrics/state.go | 0 {pkg => internal}/metrics/treeservice.go | 0 {pkg => internal}/metrics/types.go | 0 {pkg => internal}/metrics/writecache.go | 0 pkg/innerring/innerring.go | 2 +- pkg/local_object_storage/engine/metrics.go | 2 +- pkg/local_object_storage/engine/writecache.go | 2 +- pkg/local_object_storage/metrics/blobovnicza.go | 2 +- pkg/local_object_storage/metrics/blobstore.go | 2 +- pkg/local_object_storage/metrics/fstree.go | 2 +- pkg/local_object_storage/metrics/metabase.go | 2 +- pkg/local_object_storage/metrics/pilorama.go | 2 +- pkg/morph/client/client.go | 2 +- pkg/morph/client/constructor.go | 2 +- scripts/export-metrics/main.go | 2 +- 35 files changed, 14 insertions(+), 14 deletions(-) rename {pkg => internal}/metrics/blobovnicza.go (100%) rename {pkg => internal}/metrics/blobstore.go (100%) rename {pkg => internal}/metrics/consts.go (100%) rename {pkg => internal}/metrics/engine.go (100%) rename {pkg => internal}/metrics/fstree.go (100%) rename {pkg => internal}/metrics/gc.go (100%) rename {pkg => internal}/metrics/grpc.go (100%) rename {pkg => internal}/metrics/innerring.go (100%) rename {pkg => internal}/metrics/metabase.go (100%) rename {pkg => internal}/metrics/mode.go (100%) rename {pkg => internal}/metrics/morph.go (100%) rename {pkg => internal}/metrics/morphcache.go (100%) rename {pkg => internal}/metrics/node.go (100%) rename {pkg => internal}/metrics/object.go (100%) rename {pkg => internal}/metrics/pilorama.go (100%) rename {pkg => internal}/metrics/policer.go (100%) rename {pkg => internal}/metrics/replicator.go (100%) rename {pkg => internal}/metrics/state.go (100%) rename {pkg => internal}/metrics/treeservice.go (100%) rename {pkg => internal}/metrics/types.go (100%) rename {pkg => internal}/metrics/writecache.go (100%) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 206059b5d..5b852c37c 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -9,9 +9,9 @@ import ( "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + irMetrics "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" - irMetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/spf13/viper" "go.uber.org/zap" diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 2b185cfc8..ef2c0424b 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -31,6 +31,7 @@ import ( replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" @@ -45,7 +46,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" shardmode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" containerClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 2d424eec6..d2c0af52d 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -10,8 +10,8 @@ import ( netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" diff --git a/pkg/metrics/blobovnicza.go b/internal/metrics/blobovnicza.go similarity index 100% rename from pkg/metrics/blobovnicza.go rename to internal/metrics/blobovnicza.go diff --git a/pkg/metrics/blobstore.go b/internal/metrics/blobstore.go similarity index 100% rename from pkg/metrics/blobstore.go rename to internal/metrics/blobstore.go diff --git a/pkg/metrics/consts.go b/internal/metrics/consts.go similarity index 100% rename from pkg/metrics/consts.go rename to internal/metrics/consts.go diff --git a/pkg/metrics/engine.go b/internal/metrics/engine.go similarity index 100% rename from pkg/metrics/engine.go rename to internal/metrics/engine.go diff --git a/pkg/metrics/fstree.go b/internal/metrics/fstree.go similarity index 100% rename from pkg/metrics/fstree.go rename to internal/metrics/fstree.go diff --git a/pkg/metrics/gc.go b/internal/metrics/gc.go similarity index 100% rename from pkg/metrics/gc.go rename to internal/metrics/gc.go diff --git a/pkg/metrics/grpc.go b/internal/metrics/grpc.go similarity index 100% rename from pkg/metrics/grpc.go rename to internal/metrics/grpc.go diff --git a/pkg/metrics/innerring.go b/internal/metrics/innerring.go similarity index 100% rename from pkg/metrics/innerring.go rename to internal/metrics/innerring.go diff --git a/pkg/metrics/metabase.go b/internal/metrics/metabase.go similarity index 100% rename from pkg/metrics/metabase.go rename to internal/metrics/metabase.go diff --git a/pkg/metrics/mode.go b/internal/metrics/mode.go similarity index 100% rename from pkg/metrics/mode.go rename to internal/metrics/mode.go diff --git a/pkg/metrics/morph.go b/internal/metrics/morph.go similarity index 100% rename from pkg/metrics/morph.go rename to internal/metrics/morph.go diff --git a/pkg/metrics/morphcache.go b/internal/metrics/morphcache.go similarity index 100% rename from pkg/metrics/morphcache.go rename to internal/metrics/morphcache.go diff --git a/pkg/metrics/node.go b/internal/metrics/node.go similarity index 100% rename from pkg/metrics/node.go rename to internal/metrics/node.go diff --git a/pkg/metrics/object.go b/internal/metrics/object.go similarity index 100% rename from pkg/metrics/object.go rename to internal/metrics/object.go diff --git a/pkg/metrics/pilorama.go b/internal/metrics/pilorama.go similarity index 100% rename from pkg/metrics/pilorama.go rename to internal/metrics/pilorama.go diff --git a/pkg/metrics/policer.go b/internal/metrics/policer.go similarity index 100% rename from pkg/metrics/policer.go rename to internal/metrics/policer.go diff --git a/pkg/metrics/replicator.go b/internal/metrics/replicator.go similarity index 100% rename from pkg/metrics/replicator.go rename to internal/metrics/replicator.go diff --git a/pkg/metrics/state.go b/internal/metrics/state.go similarity index 100% rename from pkg/metrics/state.go rename to internal/metrics/state.go diff --git a/pkg/metrics/treeservice.go b/internal/metrics/treeservice.go similarity index 100% rename from pkg/metrics/treeservice.go rename to internal/metrics/treeservice.go diff --git a/pkg/metrics/types.go b/internal/metrics/types.go similarity index 100% rename from pkg/metrics/types.go rename to internal/metrics/types.go diff --git a/pkg/metrics/writecache.go b/internal/metrics/writecache.go similarity index 100% rename from pkg/metrics/writecache.go rename to internal/metrics/writecache.go diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 944aa8b76..50a37845b 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -8,12 +8,12 @@ import ( "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 6b99dff7e..72b5ae252 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -3,8 +3,8 @@ package engine import ( "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) type MetricRegister interface { diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 023576716..da488260a 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -5,10 +5,10 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/local_object_storage/metrics/blobovnicza.go b/pkg/local_object_storage/metrics/blobovnicza.go index 6886bbc1d..460d6e638 100644 --- a/pkg/local_object_storage/metrics/blobovnicza.go +++ b/pkg/local_object_storage/metrics/blobovnicza.go @@ -3,10 +3,10 @@ package metrics import ( "time" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) func NewBlobovniczaTreeMetrics(path string, m metrics_impl.BlobobvnizcaMetrics) blobovniczatree.Metrics { diff --git a/pkg/local_object_storage/metrics/blobstore.go b/pkg/local_object_storage/metrics/blobstore.go index b3871dfca..9a41f01d0 100644 --- a/pkg/local_object_storage/metrics/blobstore.go +++ b/pkg/local_object_storage/metrics/blobstore.go @@ -3,8 +3,8 @@ package metrics import ( "time" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) type blobstoreMetrics struct { diff --git a/pkg/local_object_storage/metrics/fstree.go b/pkg/local_object_storage/metrics/fstree.go index e6b3175be..76822ac2c 100644 --- a/pkg/local_object_storage/metrics/fstree.go +++ b/pkg/local_object_storage/metrics/fstree.go @@ -3,9 +3,9 @@ package metrics import ( "time" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) func NewFSTreeMetricsWithoutShardID(path string, m metrics_impl.FSTreeMetrics) fstree.Metrics { diff --git a/pkg/local_object_storage/metrics/metabase.go b/pkg/local_object_storage/metrics/metabase.go index 1a02f2705..e962e37cb 100644 --- a/pkg/local_object_storage/metrics/metabase.go +++ b/pkg/local_object_storage/metrics/metabase.go @@ -3,9 +3,9 @@ package metrics import ( "time" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) func NewMetabaseMetrics(path string, m metrics_impl.MetabaseMetrics) meta.Metrics { diff --git a/pkg/local_object_storage/metrics/pilorama.go b/pkg/local_object_storage/metrics/pilorama.go index 1b715c2bd..050b769a0 100644 --- a/pkg/local_object_storage/metrics/pilorama.go +++ b/pkg/local_object_storage/metrics/pilorama.go @@ -3,9 +3,9 @@ package metrics import ( "time" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" ) func NewPiloramaMetrics(m metrics_impl.PiloramaMetrics) pilorama.Metrics { diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index c9f819f04..df521f56b 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -10,7 +10,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/google/uuid" diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index c499aa892..648c7d3c0 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -7,7 +7,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go index 63571e96d..51705ee49 100644 --- a/scripts/export-metrics/main.go +++ b/scripts/export-metrics/main.go @@ -6,7 +6,7 @@ import ( "fmt" "os" - local_metrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + local_metrics "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) From df894fbac7689082da2f44c73b3a8f5d3dce6156 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Jun 2024 18:04:35 +0300 Subject: [PATCH 1180/1943] [#451] frostfs-node: Add cache metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/cache.go | 67 ++++++++++++++++++++++++++++------ cmd/frostfs-node/cache_test.go | 6 ++- internal/metrics/cache.go | 35 ++++++++++++++++++ internal/metrics/consts.go | 3 ++ 4 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 internal/metrics/cache.go diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index bf68d245f..81d552729 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -4,6 +4,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" @@ -28,15 +29,17 @@ type ttlNetCache[K comparable, V any] struct { cache *expirable.LRU[K, *valueWithError[V]] netRdr netValueReader[K, V] keyLocker *utilSync.KeyLocker[K] + metrics cacheMetrics } // complicates netValueReader with TTL caching mechanism. -func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr netValueReader[K, V]) *ttlNetCache[K, V] { +func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr netValueReader[K, V], metrics cacheMetrics) *ttlNetCache[K, V] { cache := expirable.NewLRU[K, *valueWithError[V]](sz, nil, ttl) return &ttlNetCache[K, V]{ cache: cache, netRdr: netRdr, + metrics: metrics, keyLocker: utilSync.NewKeyLocker[K](), } } @@ -47,8 +50,15 @@ func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr n // // returned value should not be modified. func (c *ttlNetCache[K, V]) get(key K) (V, error) { + hit := false + startedAt := time.Now() + defer func() { + c.metrics.AddMethodDuration("Get", time.Since(startedAt), hit) + }() + val, ok := c.cache.Peek(key) if ok { + hit = true return val.v, val.e } @@ -57,6 +67,7 @@ func (c *ttlNetCache[K, V]) get(key K) (V, error) { val, ok = c.cache.Peek(key) if ok { + hit = true return val.v, val.e } @@ -71,6 +82,11 @@ func (c *ttlNetCache[K, V]) get(key K) (V, error) { } func (c *ttlNetCache[K, V]) set(k K, v V, e error) { + startedAt := time.Now() + defer func() { + c.metrics.AddMethodDuration("Set", time.Since(startedAt), false) + }() + c.keyLocker.Lock(k) defer c.keyLocker.Unlock(k) @@ -81,10 +97,16 @@ func (c *ttlNetCache[K, V]) set(k K, v V, e error) { } func (c *ttlNetCache[K, V]) remove(key K) { + hit := false + startedAt := time.Now() + defer func() { + c.metrics.AddMethodDuration("Remove", time.Since(startedAt), hit) + }() + c.keyLocker.Lock(key) defer c.keyLocker.Unlock(key) - c.cache.Remove(key) + hit = c.cache.Remove(key) } // entity that provides LRU cache interface. @@ -92,16 +114,19 @@ type lruNetCache struct { cache *lru.Cache[uint64, *netmapSDK.NetMap] netRdr netValueReader[uint64, *netmapSDK.NetMap] + + metrics cacheMetrics } // newNetworkLRUCache returns wrapper over netValueReader with LRU cache. -func newNetworkLRUCache(sz int, netRdr netValueReader[uint64, *netmapSDK.NetMap]) *lruNetCache { +func newNetworkLRUCache(sz int, netRdr netValueReader[uint64, *netmapSDK.NetMap], metrics cacheMetrics) *lruNetCache { cache, err := lru.New[uint64, *netmapSDK.NetMap](sz) fatalOnErr(err) return &lruNetCache{ - cache: cache, - netRdr: netRdr, + cache: cache, + netRdr: netRdr, + metrics: metrics, } } @@ -111,8 +136,15 @@ func newNetworkLRUCache(sz int, netRdr netValueReader[uint64, *netmapSDK.NetMap] // // returned value should not be modified. func (c *lruNetCache) get(key uint64) (*netmapSDK.NetMap, error) { + hit := false + startedAt := time.Now() + defer func() { + c.metrics.AddMethodDuration("Get", time.Since(startedAt), hit) + }() + val, ok := c.cache.Get(key) if ok { + hit = true return val, nil } @@ -138,10 +170,10 @@ func newCachedContainerStorage(v container.Source, ttl time.Duration) ttlContain lruCnrCache := newNetworkTTLCache(containerCacheSize, ttl, func(id cid.ID) (*container.Container, error) { return v.Get(id) - }) + }, metrics.NewCacheMetrics("container")) lruDelInfoCache := newNetworkTTLCache(containerCacheSize, ttl, func(id cid.ID) (*container.DelInfo, error) { return v.DeletionInfo(id) - }) + }, metrics.NewCacheMetrics("container_deletion_info")) return ttlContainerStorage{ containerCache: lruCnrCache, @@ -175,7 +207,7 @@ func newCachedEACLStorage(v container.EACLSource, ttl time.Duration) ttlEACLStor lruCnrCache := newNetworkTTLCache(eaclCacheSize, ttl, func(id cid.ID) (*container.EACL, error) { return v.GetEACL(id) - }) + }, metrics.NewCacheMetrics("eacl")) return ttlEACLStorage{lruCnrCache} } @@ -202,7 +234,7 @@ func newCachedNetmapStorage(s netmap.State, v netmap.Source) netmap.Source { lruNetmapCache := newNetworkLRUCache(netmapCacheSize, func(key uint64) (*netmapSDK.NetMap, error) { return v.GetNetMapByEpoch(key) - }) + }, metrics.NewCacheMetrics("netmap")) return &lruNetmapSource{ netState: s, @@ -251,7 +283,7 @@ func newCachedIRFetcher(f interface{ InnerRingKeys() ([][]byte, error) }) cached irFetcherCache := newNetworkTTLCache(irFetcherCacheSize, irFetcherCacheTTL, func(_ struct{}) ([][]byte, error) { return f.InnerRingKeys() - }, + }, metrics.NewCacheMetrics("ir_keys"), ) return cachedIRFetcher{irFetcherCache} @@ -274,23 +306,32 @@ type ttlMaxObjectSizeCache struct { lastUpdated time.Time lastSize uint64 src putsvc.MaxSizeSource + metrics cacheMetrics } func newCachedMaxObjectSizeSource(src putsvc.MaxSizeSource) putsvc.MaxSizeSource { return &ttlMaxObjectSizeCache{ - src: src, + src: src, + metrics: metrics.NewCacheMetrics("max_object_size"), } } func (c *ttlMaxObjectSizeCache) MaxObjectSize() uint64 { const ttl = time.Second * 30 + hit := false + startedAt := time.Now() + defer func() { + c.metrics.AddMethodDuration("Get", time.Since(startedAt), hit) + }() + c.mtx.RLock() prevUpdated := c.lastUpdated size := c.lastSize c.mtx.RUnlock() if time.Since(prevUpdated) < ttl { + hit = true return size } @@ -305,3 +346,7 @@ func (c *ttlMaxObjectSizeCache) MaxObjectSize() uint64 { return size } + +type cacheMetrics interface { + AddMethodDuration(method string, d time.Duration, hit bool) +} diff --git a/cmd/frostfs-node/cache_test.go b/cmd/frostfs-node/cache_test.go index 6981ac721..f8c324a2f 100644 --- a/cmd/frostfs-node/cache_test.go +++ b/cmd/frostfs-node/cache_test.go @@ -10,7 +10,7 @@ import ( func TestTTLNetCache(t *testing.T) { ttlDuration := time.Millisecond * 50 - cache := newNetworkTTLCache(10, ttlDuration, testNetValueReader) + cache := newNetworkTTLCache(10, ttlDuration, testNetValueReader, &noopCacheMetricts{}) key := "key" @@ -54,3 +54,7 @@ func testNetValueReader(key string) (time.Time, error) { } return time.Now(), nil } + +type noopCacheMetricts struct{} + +func (m *noopCacheMetricts) AddMethodDuration(method string, d time.Duration, hit bool) {} diff --git a/internal/metrics/cache.go b/internal/metrics/cache.go new file mode 100644 index 000000000..8181586e2 --- /dev/null +++ b/internal/metrics/cache.go @@ -0,0 +1,35 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +var cacheRequests = metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: commonCacheSubsystem, + Name: "request_duration_seconds", + Help: "Accumulated common cache request process duration", +}, []string{hitLabel, methodLabel, cacheLabel}) + +type CacheMetrics struct { + cache string +} + +// NewCacheMetrics returns new CacheMetrics instance for cache specified. +func NewCacheMetrics(cache string) *CacheMetrics { + return &CacheMetrics{ + cache: cache, + } +} + +func (m *CacheMetrics) AddMethodDuration(method string, d time.Duration, hit bool) { + cacheRequests.With(prometheus.Labels{ + hitLabel: strconv.FormatBool(hit), + methodLabel: method, + cacheLabel: m.cache, + }).Observe(d.Seconds()) +} diff --git a/internal/metrics/consts.go b/internal/metrics/consts.go index f3b83a2e5..3aa51c0f0 100644 --- a/internal/metrics/consts.go +++ b/internal/metrics/consts.go @@ -21,6 +21,7 @@ const ( writeCacheSubsystem = "writecache" grpcServerSubsystem = "grpc_server" policerSubsystem = "policer" + commonCacheSubsystem = "common_cache" successLabel = "success" shardIDLabel = "shard_id" @@ -38,6 +39,8 @@ const ( storageLabel = "storage" operationLabel = "operation" endpointLabel = "endpoint" + hitLabel = "hit" + cacheLabel = "cache" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" From a0e5fc733e412beb674f38d4b5994e443e63c937 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 24 Jun 2024 15:05:25 +0300 Subject: [PATCH 1181/1943] [#1145] node/ir: Handle double SIGHUP correctly Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-ir/config.go | 39 ++++++++++++++++++++++++---------- cmd/frostfs-node/config.go | 43 ++++++++++++++++++++++++++++---------- 2 files changed, 60 insertions(+), 22 deletions(-) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index c3227a182..955195477 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -45,18 +45,41 @@ func reloadConfig() error { func watchForSignal(cancel func()) { ch := make(chan os.Signal, 1) - signal.Notify(ch, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) + + sighupCh := make(chan os.Signal, 1) + signal.Notify(sighupCh, syscall.SIGHUP) for { select { - case err := <-intErr: + // signals causing application to shut down should have priority over + // reconfiguration signal + case <-ch: + log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + cancel() + shutdown() + log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) + return + case err := <-intErr: // internal application error log.Info(logs.FrostFSIRInternalError, zap.String("msg", err.Error())) cancel() shutdown() return - case sig := <-ch: - switch sig { - case syscall.SIGHUP: + default: + // block until any signal is receieved + select { + case <-ch: + log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + cancel() + shutdown() + log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) + return + case err := <-intErr: // internal application error + log.Info(logs.FrostFSIRInternalError, zap.String("msg", err.Error())) + cancel() + shutdown() + return + case <-sighupCh: log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) err := reloadConfig() if err != nil { @@ -70,12 +93,6 @@ func watchForSignal(cancel func()) { log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) } log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) - case syscall.SIGTERM, syscall.SIGINT: - log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) - cancel() - shutdown() - log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) - return } } } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index ef2c0424b..3aded7020 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1224,22 +1224,22 @@ type dCmp struct { func (c *cfg) signalWatcher(ctx context.Context) { ch := make(chan os.Signal, 1) - signal.Notify(ch, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) + + sighupCh := make(chan os.Signal, 1) + signal.Notify(sighupCh, syscall.SIGHUP) for { select { - case sig := <-ch: - switch sig { - case syscall.SIGHUP: - c.reloadConfig(ctx) - case syscall.SIGTERM, syscall.SIGINT: - c.log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + // signals causing application to shut down should have priority over + // reconfiguration signal + case <-ch: + c.log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) - c.shutdown() + c.shutdown() - c.log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) - return - } + c.log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) + return case err := <-c.internalErr: // internal application error c.log.Warn(logs.FrostFSNodeInternalApplicationError, zap.String("message", err.Error())) @@ -1248,6 +1248,27 @@ func (c *cfg) signalWatcher(ctx context.Context) { c.log.Info(logs.FrostFSNodeInternalErrorProcessingIsComplete) return + default: + // block until any signal is receieved + select { + case <-sighupCh: + c.reloadConfig(ctx) + case <-ch: + c.log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + + c.shutdown() + + c.log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) + return + case err := <-c.internalErr: // internal application error + c.log.Warn(logs.FrostFSNodeInternalApplicationError, + zap.String("message", err.Error())) + + c.shutdown() + + c.log.Info(logs.FrostFSNodeInternalErrorProcessingIsComplete) + return + } } } } From 4951babd5f2e87823e71f4459caba8eaaeb8a44f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 27 Jun 2024 11:55:00 +0300 Subject: [PATCH 1182/1943] [#1208] blobstor: Fix delete without storage id Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobstor/delete.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index 8930980a6..c91508e6d 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -6,8 +6,10 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -42,6 +44,7 @@ func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.Del return res, err } } + return common.DeleteRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } var st common.Storage From 4f7d76c9eff284cb8f3aa432999a808e515f8011 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 26 Jun 2024 12:37:20 +0300 Subject: [PATCH 1183/1943] [#1206] audit: Drop not required events Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/accounting.go | 15 +- cmd/frostfs-node/control.go | 5 +- cmd/frostfs-node/netmap.go | 28 +-- cmd/frostfs-node/session.go | 5 +- cmd/frostfs-node/tree.go | 2 +- internal/audit/target.go | 56 ----- pkg/services/accounting/audit.go | 42 ---- pkg/services/control/server/audit.go | 298 --------------------------- pkg/services/netmap/audit.go | 60 ------ pkg/services/session/audit.go | 39 ---- pkg/services/tree/audit.go | 135 ------------ 11 files changed, 26 insertions(+), 659 deletions(-) delete mode 100644 pkg/services/accounting/audit.go delete mode 100644 pkg/services/control/server/audit.go delete mode 100644 pkg/services/netmap/audit.go delete mode 100644 pkg/services/session/audit.go delete mode 100644 pkg/services/tree/audit.go diff --git a/cmd/frostfs-node/accounting.go b/cmd/frostfs-node/accounting.go index f0e2abf38..ec737f8a0 100644 --- a/cmd/frostfs-node/accounting.go +++ b/cmd/frostfs-node/accounting.go @@ -20,16 +20,15 @@ func initAccountingService(ctx context.Context, c *cfg) { balanceMorphWrapper, err := balance.NewFromMorph(c.cfgMorph.client, c.cfgAccounting.scriptHash, 0) fatalOnErr(err) - service := accountingService.NewSignService( - &c.key.PrivateKey, - accountingService.NewExecutionService( - accounting.NewExecutor(balanceMorphWrapper), - c.respSvc, + server := accountingTransportGRPC.New( + accountingService.NewSignService( + &c.key.PrivateKey, + accountingService.NewExecutionService( + accounting.NewExecutor(balanceMorphWrapper), + c.respSvc, + ), ), ) - service = accountingService.NewAuditService(service, c.log, c.audit) - - server := accountingTransportGRPC.New(service) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { accountingGRPC.RegisterAccountingServiceServer(s, server) diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index 8ee1ab69f..e1e6e3ac9 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -30,8 +30,8 @@ func initControlService(c *cfg) { for i := range pubs { rawPubs = append(rawPubs, pubs[i].Bytes()) } - var ctlSvc control.ControlServiceServer - ctlSvc = controlSvc.New( + + ctlSvc := controlSvc.New( controlSvc.WithKey(&c.key.PrivateKey), controlSvc.WithAuthorizedKeys(rawPubs), controlSvc.WithHealthChecker(c), @@ -43,7 +43,6 @@ func initControlService(c *cfg) { controlSvc.WithTreeService(c.treeService), controlSvc.WithLocalOverrideStorage(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine), ) - ctlSvc = controlSvc.NewAuditService(ctlSvc, c.log, c.audit) lis, err := net.Listen("tcp", endpoint) if err != nil { diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index d2c0af52d..128cc3005 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -147,22 +147,22 @@ func initNetmapService(ctx context.Context, c *cfg) { initNetmapState(c) - svc := netmapService.NewSignService( - &c.key.PrivateKey, - netmapService.NewExecutionService( - c, - c.apiVersion, - &netInfo{ - netState: c.cfgNetmap.state, - magic: c.cfgMorph.client, - morphClientNetMap: c.cfgNetmap.wrapper, - msPerBlockRdr: c.cfgMorph.client.MsPerBlock, - }, - c.respSvc, + server := netmapTransportGRPC.New( + netmapService.NewSignService( + &c.key.PrivateKey, + netmapService.NewExecutionService( + c, + c.apiVersion, + &netInfo{ + netState: c.cfgNetmap.state, + magic: c.cfgMorph.client, + morphClientNetMap: c.cfgNetmap.wrapper, + msPerBlockRdr: c.cfgMorph.client.MsPerBlock, + }, + c.respSvc, + ), ), ) - svc = netmapService.NewAuditService(svc, c.log, c.audit) - server := netmapTransportGRPC.New(svc) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { netmapGRPC.RegisterNetmapServiceServer(s, server) diff --git a/cmd/frostfs-node/session.go b/cmd/frostfs-node/session.go index d286fc631..ee21ec230 100644 --- a/cmd/frostfs-node/session.go +++ b/cmd/frostfs-node/session.go @@ -52,13 +52,12 @@ func initSessionService(c *cfg) { c.privateTokenStore.RemoveOld(ev.(netmap.NewEpoch).EpochNumber()) }) - svc := sessionSvc.NewAuditService( + server := sessionTransportGRPC.New( sessionSvc.NewSignService( &c.key.PrivateKey, sessionSvc.NewExecutionService(c.privateTokenStore, c.respSvc, c.log), ), - c.log, c.audit) - server := sessionTransportGRPC.New(svc) + ) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { sessionGRPC.RegisterSessionServiceServer(s, server) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index 9f5c89ef0..daaaa64a2 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -70,7 +70,7 @@ func initTreeService(c *cfg) { ) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { - tree.RegisterTreeServiceServer(s, tree.NewAuditService(c.treeService, c.log, c.audit)) + tree.RegisterTreeServiceServer(s, c.treeService) }) c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { diff --git a/internal/audit/target.go b/internal/audit/target.go index 1d760eca6..8bc87ee8e 100644 --- a/internal/audit/target.go +++ b/internal/audit/target.go @@ -6,7 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/mr-tron/base58" ) type ModelType[T any] interface { @@ -61,24 +60,6 @@ func TargetFromString(s string) Target { return stringTarget{s: s} } -func TargetFromStringSlice(s []string) Target { - if len(s) == 0 { - return stringTarget{s: NotDefined} - } - sb := &strings.Builder{} - for i, v := range s { - if i > 0 { - sb.WriteString(";") - } - if len(v) == 0 { - sb.WriteString(Empty) - } else { - sb.WriteString(v) - } - } - return sb -} - func TargetFromChainID(chainTargetType, chainTargetName string, chainID []byte) Target { if len(chainTargetType) == 0 && len(chainTargetName) == 0 && len(chainID) == 0 { return stringTarget{s: NotDefined} @@ -96,43 +77,6 @@ func TargetFromChainID(chainTargetType, chainTargetName string, chainID []byte) return stringTarget{s: t + ":" + n + ":" + c} } -func TargetFromShardIDs(v [][]byte) Target { - if len(v) == 0 { - return stringTarget{s: NotDefined} - } - sb := &strings.Builder{} - for i, s := range v { - if i > 0 { - sb.WriteString(";") - } - if len(s) == 0 { - sb.WriteString(Empty) - } else { - sb.WriteString(base58.Encode(s)) - } - } - return sb -} - -func TargetFromTreeID(containerID []byte, treeID string) Target { - if len(containerID) == 0 && len(treeID) == 0 { - return stringTarget{s: NotDefined} - } - c, t := Empty, Empty - if len(containerID) > 0 { - var cnr cid.ID - if err := cnr.Decode(containerID); err != nil { - c = InvalidValue - } else { - c = cnr.EncodeToString() - } - } - if len(treeID) > 0 { - t = treeID - } - return stringTarget{s: c + ":" + t} -} - func TargetFromContainerIDObjectID(containerID *refs.ContainerID, objectID *refs.ObjectID) Target { if containerID == nil && objectID == nil { return stringTarget{s: NotDefined} diff --git a/pkg/services/accounting/audit.go b/pkg/services/accounting/audit.go deleted file mode 100644 index 1d8f8836f..000000000 --- a/pkg/services/accounting/audit.go +++ /dev/null @@ -1,42 +0,0 @@ -package accounting - -import ( - "context" - "sync/atomic" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" - acc_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" -) - -var _ Server = (*auditService)(nil) - -type auditService struct { - next Server - log *logger.Logger - enabled *atomic.Bool -} - -func NewAuditService(next Server, log *logger.Logger, enabled *atomic.Bool) Server { - return &auditService{ - next: next, - log: log, - enabled: enabled, - } -} - -// Balance implements Server. -func (l *auditService) Balance(ctx context.Context, req *accounting.BalanceRequest) (*accounting.BalanceResponse, error) { - res, err := l.next.Balance(ctx, req) - - if !l.enabled.Load() { - return res, err - } - - audit.LogRequest(l.log, acc_grpc.AccountingService_Balance_FullMethodName, req, - audit.TargetFromRef(req.GetBody().GetOwnerID(), &user.ID{}), err == nil) - - return res, err -} diff --git a/pkg/services/control/server/audit.go b/pkg/services/control/server/audit.go deleted file mode 100644 index 6443ea378..000000000 --- a/pkg/services/control/server/audit.go +++ /dev/null @@ -1,298 +0,0 @@ -package control - -import ( - "context" - "sync/atomic" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" - ctl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -var _ ctl.ControlServiceServer = (*auditService)(nil) - -type auditService struct { - next ctl.ControlServiceServer - log *logger.Logger - enabled *atomic.Bool -} - -func NewAuditService(next ctl.ControlServiceServer, log *logger.Logger, enabled *atomic.Bool) ctl.ControlServiceServer { - return &auditService{ - next: next, - log: log, - enabled: enabled, - } -} - -// AddChainLocalOverride implements control.ControlServiceServer. -func (a *auditService) AddChainLocalOverride(ctx context.Context, req *ctl.AddChainLocalOverrideRequest) (*ctl.AddChainLocalOverrideResponse, error) { - res, err := a.next.AddChainLocalOverride(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_AddChainLocalOverride_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromChainID(req.GetBody().GetTarget().GetType().String(), - req.GetBody().GetTarget().GetName(), - res.GetBody().GetChainId()), - err == nil) - return res, err -} - -// DetachShards implements control.ControlServiceServer. -func (a *auditService) DetachShards(ctx context.Context, req *ctl.DetachShardsRequest) (*ctl.DetachShardsResponse, error) { - res, err := a.next.DetachShards(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_DetachShards_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) - return res, err -} - -// Doctor implements control.ControlServiceServer. -func (a *auditService) Doctor(ctx context.Context, req *ctl.DoctorRequest) (*ctl.DoctorResponse, error) { - res, err := a.next.Doctor(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_Doctor_FullMethodName, req.GetSignature().GetKey(), nil, err == nil) - return res, err -} - -// DropObjects implements control.ControlServiceServer. -func (a *auditService) DropObjects(ctx context.Context, req *ctl.DropObjectsRequest) (*ctl.DropObjectsResponse, error) { - res, err := a.next.DropObjects(ctx, req) - if !a.enabled.Load() { - return res, err - } - var list []string - for _, v := range req.GetBody().GetAddressList() { - if len(v) == 0 { - list = append(list, audit.Empty) - continue - } - var a oid.Address - if e := a.DecodeString(string(v)); e != nil { - list = append(list, audit.InvalidValue) - } else { - list = append(list, a.EncodeToString()) - } - } - - audit.LogRequestWithKey(a.log, ctl.ControlService_DropObjects_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromStringSlice(list), err == nil) - return res, err -} - -// EvacuateShard implements control.ControlServiceServer. -func (a *auditService) EvacuateShard(ctx context.Context, req *ctl.EvacuateShardRequest) (*ctl.EvacuateShardResponse, error) { - res, err := a.next.EvacuateShard(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_EvacuateShard_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) - return res, err -} - -// FlushCache implements control.ControlServiceServer. -func (a *auditService) FlushCache(ctx context.Context, req *ctl.FlushCacheRequest) (*ctl.FlushCacheResponse, error) { - res, err := a.next.FlushCache(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_FlushCache_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) - return res, err -} - -// GetChainLocalOverride implements control.ControlServiceServer. -func (a *auditService) GetChainLocalOverride(ctx context.Context, req *ctl.GetChainLocalOverrideRequest) (*ctl.GetChainLocalOverrideResponse, error) { - res, err := a.next.GetChainLocalOverride(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_GetChainLocalOverride_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromChainID( - req.GetBody().GetTarget().GetType().String(), - req.GetBody().GetTarget().GetName(), - req.GetBody().GetChainId()), - err == nil) - return res, err -} - -// GetShardEvacuationStatus implements control.ControlServiceServer. -func (a *auditService) GetShardEvacuationStatus(ctx context.Context, req *ctl.GetShardEvacuationStatusRequest) (*ctl.GetShardEvacuationStatusResponse, error) { - res, err := a.next.GetShardEvacuationStatus(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_GetShardEvacuationStatus_FullMethodName, req.GetSignature().GetKey(), - nil, err == nil) - return res, err -} - -// HealthCheck implements control.ControlServiceServer. -func (a *auditService) HealthCheck(ctx context.Context, req *ctl.HealthCheckRequest) (*ctl.HealthCheckResponse, error) { - res, err := a.next.HealthCheck(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_HealthCheck_FullMethodName, req.GetSignature().GetKey(), - nil, err == nil) - return res, err -} - -// ListChainLocalOverrides implements control.ControlServiceServer. -func (a *auditService) ListChainLocalOverrides(ctx context.Context, req *ctl.ListChainLocalOverridesRequest) (*ctl.ListChainLocalOverridesResponse, error) { - res, err := a.next.ListChainLocalOverrides(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_ListChainLocalOverrides_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromChainID(req.GetBody().GetTarget().GetType().String(), - req.GetBody().GetTarget().GetName(), - nil), - err == nil) - return res, err -} - -// ListShards implements control.ControlServiceServer. -func (a *auditService) ListShards(ctx context.Context, req *ctl.ListShardsRequest) (*ctl.ListShardsResponse, error) { - res, err := a.next.ListShards(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_ListShards_FullMethodName, req.GetSignature().GetKey(), - nil, err == nil) - return res, err -} - -// ListTargetsLocalOverrides implements control.ControlServiceServer. -func (a *auditService) ListTargetsLocalOverrides(ctx context.Context, req *ctl.ListTargetsLocalOverridesRequest) (*ctl.ListTargetsLocalOverridesResponse, error) { - res, err := a.next.ListTargetsLocalOverrides(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_ListTargetsLocalOverrides_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromString(req.GetBody().GetChainName()), err == nil) - return res, err -} - -// RemoveChainLocalOverride implements control.ControlServiceServer. -func (a *auditService) RemoveChainLocalOverride(ctx context.Context, req *ctl.RemoveChainLocalOverrideRequest) (*ctl.RemoveChainLocalOverrideResponse, error) { - res, err := a.next.RemoveChainLocalOverride(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_RemoveChainLocalOverride_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromChainID(req.GetBody().GetTarget().GetType().String(), - req.GetBody().GetTarget().GetName(), - req.GetBody().GetChainId()), - err == nil) - return res, err -} - -// RemoveChainLocalOverridesByTarget implements control.ControlServiceServer. -func (a *auditService) RemoveChainLocalOverridesByTarget(ctx context.Context, req *ctl.RemoveChainLocalOverridesByTargetRequest) (*ctl.RemoveChainLocalOverridesByTargetResponse, error) { - res, err := a.next.RemoveChainLocalOverridesByTarget(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_RemoveChainLocalOverridesByTarget_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromChainID(req.GetBody().GetTarget().GetType().String(), - req.GetBody().GetTarget().GetName(), - nil), - err == nil) - return res, err -} - -// ResetShardEvacuationStatus implements control.ControlServiceServer. -func (a *auditService) ResetShardEvacuationStatus(ctx context.Context, req *ctl.ResetShardEvacuationStatusRequest) (*ctl.ResetShardEvacuationStatusResponse, error) { - res, err := a.next.ResetShardEvacuationStatus(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_ResetShardEvacuationStatus_FullMethodName, req.GetSignature().GetKey(), - nil, err == nil) - return res, err -} - -// SealWriteCache implements control.ControlServiceServer. -func (a *auditService) SealWriteCache(ctx context.Context, req *ctl.SealWriteCacheRequest) (*ctl.SealWriteCacheResponse, error) { - res, err := a.next.SealWriteCache(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_SealWriteCache_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) - return res, err -} - -// SetNetmapStatus implements control.ControlServiceServer. -func (a *auditService) SetNetmapStatus(ctx context.Context, req *ctl.SetNetmapStatusRequest) (*ctl.SetNetmapStatusResponse, error) { - res, err := a.next.SetNetmapStatus(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_SetNetmapStatus_FullMethodName, req.GetSignature().GetKey(), - nil, err == nil) - return res, err -} - -// GetNetmapStatus implements control.ControlServiceServer. -func (a *auditService) GetNetmapStatus(ctx context.Context, req *ctl.GetNetmapStatusRequest) (*ctl.GetNetmapStatusResponse, error) { - res, err := a.next.GetNetmapStatus(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_GetNetmapStatus_FullMethodName, req.GetSignature().GetKey(), - nil, err == nil) - return res, err -} - -// SetShardMode implements control.ControlServiceServer. -func (a *auditService) SetShardMode(ctx context.Context, req *ctl.SetShardModeRequest) (*ctl.SetShardModeResponse, error) { - res, err := a.next.SetShardMode(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_SetShardMode_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) - return res, err -} - -// StartShardEvacuation implements control.ControlServiceServer. -func (a *auditService) StartShardEvacuation(ctx context.Context, req *ctl.StartShardEvacuationRequest) (*ctl.StartShardEvacuationResponse, error) { - res, err := a.next.StartShardEvacuation(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_StartShardEvacuation_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromShardIDs(req.GetBody().GetShard_ID()), err == nil) - return res, err -} - -// StopShardEvacuation implements control.ControlServiceServer. -func (a *auditService) StopShardEvacuation(ctx context.Context, req *ctl.StopShardEvacuationRequest) (*ctl.StopShardEvacuationResponse, error) { - res, err := a.next.StopShardEvacuation(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_StopShardEvacuation_FullMethodName, req.GetSignature().GetKey(), - nil, err == nil) - return res, err -} - -// SynchronizeTree implements control.ControlServiceServer. -func (a *auditService) SynchronizeTree(ctx context.Context, req *ctl.SynchronizeTreeRequest) (*ctl.SynchronizeTreeResponse, error) { - res, err := a.next.SynchronizeTree(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, ctl.ControlService_SynchronizeTree_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) - return res, err -} diff --git a/pkg/services/netmap/audit.go b/pkg/services/netmap/audit.go deleted file mode 100644 index 906fd398e..000000000 --- a/pkg/services/netmap/audit.go +++ /dev/null @@ -1,60 +0,0 @@ -package netmap - -import ( - "context" - "sync/atomic" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" - netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" -) - -var _ Server = (*auditService)(nil) - -type auditService struct { - next Server - log *logger.Logger - enabled *atomic.Bool -} - -func NewAuditService(next Server, log *logger.Logger, enabled *atomic.Bool) Server { - return &auditService{ - next: next, - log: log, - enabled: enabled, - } -} - -// LocalNodeInfo implements Server. -func (a *auditService) LocalNodeInfo(ctx context.Context, req *netmap.LocalNodeInfoRequest) (*netmap.LocalNodeInfoResponse, error) { - res, err := a.next.LocalNodeInfo(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequest(a.log, netmapGRPC.NetmapService_LocalNodeInfo_FullMethodName, req, - nil, err == nil) - return res, err -} - -// NetworkInfo implements Server. -func (a *auditService) NetworkInfo(ctx context.Context, req *netmap.NetworkInfoRequest) (*netmap.NetworkInfoResponse, error) { - res, err := a.next.NetworkInfo(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequest(a.log, netmapGRPC.NetmapService_NetworkInfo_FullMethodName, req, - nil, err == nil) - return res, err -} - -// Snapshot implements Server. -func (a *auditService) Snapshot(ctx context.Context, req *netmap.SnapshotRequest) (*netmap.SnapshotResponse, error) { - res, err := a.next.Snapshot(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequest(a.log, netmapGRPC.NetmapService_NetmapSnapshot_FullMethodName, req, - nil, err == nil) - return res, err -} diff --git a/pkg/services/session/audit.go b/pkg/services/session/audit.go deleted file mode 100644 index 19d3383d9..000000000 --- a/pkg/services/session/audit.go +++ /dev/null @@ -1,39 +0,0 @@ -package session - -import ( - "context" - "sync/atomic" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - sessionGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" -) - -var _ Server = (*auditService)(nil) - -type auditService struct { - next Server - log *logger.Logger - enabled *atomic.Bool -} - -func NewAuditService(next Server, log *logger.Logger, enabled *atomic.Bool) Server { - return &auditService{ - next: next, - log: log, - enabled: enabled, - } -} - -// Create implements Server. -func (a *auditService) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - res, err := a.next.Create(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequest(a.log, sessionGRPC.SessionService_Create_FullMethodName, req, - audit.TargetFromRef(req.GetBody().GetOwnerID(), &user.ID{}), err == nil) - return res, err -} diff --git a/pkg/services/tree/audit.go b/pkg/services/tree/audit.go deleted file mode 100644 index bec71f5d4..000000000 --- a/pkg/services/tree/audit.go +++ /dev/null @@ -1,135 +0,0 @@ -package tree - -import ( - "context" - "sync/atomic" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" -) - -var _ TreeServiceServer = (*auditService)(nil) - -type auditService struct { - next TreeServiceServer - log *logger.Logger - enabled *atomic.Bool -} - -func NewAuditService(next TreeServiceServer, log *logger.Logger, enabled *atomic.Bool) TreeServiceServer { - return &auditService{ - next: next, - log: log, - enabled: enabled, - } -} - -// Add implements TreeServiceServer. -func (a *auditService) Add(ctx context.Context, req *AddRequest) (*AddResponse, error) { - res, err := a.next.Add(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, TreeService_Add_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) - return res, err -} - -// AddByPath implements TreeServiceServer. -func (a *auditService) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByPathResponse, error) { - res, err := a.next.AddByPath(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, TreeService_AddByPath_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) - return res, err -} - -// Apply implements TreeServiceServer. -func (a *auditService) Apply(ctx context.Context, req *ApplyRequest) (*ApplyResponse, error) { - res, err := a.next.Apply(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, TreeService_Apply_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) - return res, err -} - -// GetNodeByPath implements TreeServiceServer. -func (a *auditService) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) (*GetNodeByPathResponse, error) { - res, err := a.next.GetNodeByPath(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, TreeService_GetNodeByPath_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) - return res, err -} - -// GetOpLog implements TreeServiceServer. -func (a *auditService) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) error { - err := a.next.GetOpLog(req, srv) - if !a.enabled.Load() { - return err - } - audit.LogRequestWithKey(a.log, TreeService_GetOpLog_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) - return err -} - -// GetSubTree implements TreeServiceServer. -func (a *auditService) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeServer) error { - err := a.next.GetSubTree(req, srv) - if !a.enabled.Load() { - return err - } - audit.LogRequestWithKey(a.log, TreeService_GetSubTree_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) - return err -} - -// Healthcheck implements TreeServiceServer. -func (a *auditService) Healthcheck(ctx context.Context, req *HealthcheckRequest) (*HealthcheckResponse, error) { - res, err := a.next.Healthcheck(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, TreeService_Healthcheck_FullMethodName, req.GetSignature().GetKey(), - nil, err == nil) - return res, err -} - -// Move implements TreeServiceServer. -func (a *auditService) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, error) { - res, err := a.next.Move(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, TreeService_Move_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) - return res, err -} - -// Remove implements TreeServiceServer. -func (a *auditService) Remove(ctx context.Context, req *RemoveRequest) (*RemoveResponse, error) { - res, err := a.next.Remove(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, TreeService_Remove_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromTreeID(req.GetBody().GetContainerId(), req.GetBody().GetTreeId()), err == nil) - return res, err -} - -// TreeList implements TreeServiceServer. -func (a *auditService) TreeList(ctx context.Context, req *TreeListRequest) (*TreeListResponse, error) { - res, err := a.next.TreeList(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequestWithKey(a.log, TreeService_TreeList_FullMethodName, req.GetSignature().GetKey(), - audit.TargetFromTreeID(req.GetBody().GetContainerId(), ""), err == nil) - return res, err -} From 87a4a6e8d05b766b29db2b593350ba79186b72c9 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 26 Jun 2024 14:46:32 +0300 Subject: [PATCH 1184/1943] [#1154] node: Add info metric Signed-off-by: Alexander Chuprov --- internal/metrics/application.go | 21 +++++++++++++++++++++ internal/metrics/node.go | 3 +++ 2 files changed, 24 insertions(+) create mode 100644 internal/metrics/application.go diff --git a/internal/metrics/application.go b/internal/metrics/application.go new file mode 100644 index 000000000..8bc408ab6 --- /dev/null +++ b/internal/metrics/application.go @@ -0,0 +1,21 @@ +package metrics + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type ApplicationInfo struct { + versionValue *prometheus.GaugeVec +} + +func NewApplicationInfo(version string) *ApplicationInfo { + appInfo := &ApplicationInfo{ + versionValue: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Name: "app_info", + Help: "General information about the application.", + }, []string{"version"}), + } + appInfo.versionValue.With(prometheus.Labels{"version": version}) + return appInfo +} diff --git a/internal/metrics/node.go b/internal/metrics/node.go index 0dd86d90e..d9e401446 100644 --- a/internal/metrics/node.go +++ b/internal/metrics/node.go @@ -1,6 +1,7 @@ package metrics import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/misc" morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" @@ -24,6 +25,7 @@ type NodeMetrics struct { morphClient *morphClientMetrics morphCache *morphCacheMetrics log logger.LogMetrics + appInfo *ApplicationInfo } func NewNodeMetrics() *NodeMetrics { @@ -49,6 +51,7 @@ func NewNodeMetrics() *NodeMetrics { morphClient: newMorphClientMetrics(), morphCache: newMorphCacheMetrics(namespace), log: logger.NewLogMetrics(namespace), + appInfo: NewApplicationInfo(misc.Version), } } From 452faa3c893ec31eb09c43008eb503e0f1616857 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 26 Jun 2024 14:47:16 +0300 Subject: [PATCH 1185/1943] [#1154] ir: Add info metric Signed-off-by: Alexander Chuprov --- internal/metrics/innerring.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/metrics/innerring.go b/internal/metrics/innerring.go index 3e438a01b..f6b14a632 100644 --- a/internal/metrics/innerring.go +++ b/internal/metrics/innerring.go @@ -4,6 +4,7 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" @@ -16,6 +17,7 @@ type InnerRingServiceMetrics struct { eventDuration *prometheus.HistogramVec morphCacheMetrics *morphCacheMetrics logMetrics logger.LogMetrics + appInfo *ApplicationInfo } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. @@ -46,6 +48,7 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { health: health, eventDuration: eventDuration, morphCacheMetrics: newMorphCacheMetrics(innerRingNamespace), + appInfo: NewApplicationInfo(misc.Version), logMetrics: logger.NewLogMetrics(innerRingNamespace), } } From 81070ada01cc071dc1f211d852ff0d0667605454 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 26 Jun 2024 14:44:41 +0300 Subject: [PATCH 1186/1943] [#1091] cli: Check `NotFound` properly for `control list-targets` Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/control/list_targets.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/frostfs-cli/modules/control/list_targets.go b/cmd/frostfs-cli/modules/control/list_targets.go index ddd80b31b..6a988c355 100644 --- a/cmd/frostfs-cli/modules/control/list_targets.go +++ b/cmd/frostfs-cli/modules/control/list_targets.go @@ -13,6 +13,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/spf13/cobra" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -52,6 +54,10 @@ func listTargets(cmd *cobra.Command, _ []string) { resp, err = control.ListTargetsLocalOverrides(client, req) return err }) + if err != nil && status.Code(err) == codes.NotFound { + cmd.Println("Local overrides are not defined for any target.") + return + } commonCmd.ExitOnErr(cmd, "rpc error: %w", err) verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) From 4c7ff159ec90579b962c1d81454047c0752c17ad Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 26 Jun 2024 11:19:33 +0300 Subject: [PATCH 1187/1943] [#1201] writecache: Cancel background flush without lock Signed-off-by: Dmitrii Stepanov --- .../writecache/cachebbolt.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/writecache/cachebbolt.go b/pkg/local_object_storage/writecache/cachebbolt.go index 254d8754f..cdd4ed442 100644 --- a/pkg/local_object_storage/writecache/cachebbolt.go +++ b/pkg/local_object_storage/writecache/cachebbolt.go @@ -4,6 +4,7 @@ import ( "context" "os" "sync" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" @@ -30,7 +31,7 @@ type cache struct { // flushCh is a channel with objects to flush. flushCh chan objectInfo // cancel is cancel function, protected by modeMtx in Close. - cancel func() + cancel atomic.Value // wg is a wait group for flush workers. wg sync.WaitGroup // store contains underlying database. @@ -54,7 +55,10 @@ const ( defaultMaxCacheSize = 1 << 30 // 1 GiB ) -var defaultBucket = []byte{0} +var ( + defaultBucket = []byte{0} + dummyCanceler context.CancelFunc = func() {} +) // New creates new writecache instance. func New(opts ...Option) Cache { @@ -114,20 +118,19 @@ func (c *cache) Open(_ context.Context, mod mode.Mode) error { func (c *cache) Init() error { c.metrics.SetMode(mode.ConvertToComponentModeDegraded(c.mode)) ctx, cancel := context.WithCancel(context.Background()) - c.cancel = cancel + c.cancel.Store(cancel) c.runFlushLoop(ctx) return nil } // Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. func (c *cache) Close() error { + if cancelValue := c.cancel.Swap(dummyCanceler); cancelValue != nil { + cancelValue.(context.CancelFunc)() + } // We cannot lock mutex for the whole operation duration // because it is taken by some background workers, so `wg.Wait()` is done without modeMtx. c.modeMtx.Lock() - if c.cancel != nil { - c.cancel() - c.cancel = nil - } c.mode = mode.DegradedReadOnly // prevent new operations from being processed c.modeMtx.Unlock() From 7085723c6bde2e2cf90dc5029bd6a170db621753 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 4 Apr 2024 10:40:21 +0300 Subject: [PATCH 1188/1943] [#1074] pilorama: Allow empty filenames in SortedByFilename() Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/tree.go | 4 +-- pkg/local_object_storage/pilorama/boltdb.go | 17 ++++++++----- pkg/local_object_storage/pilorama/forest.go | 23 +++++++++++------ .../pilorama/forest_test.go | 25 +++++++++++-------- pkg/local_object_storage/pilorama/heap.go | 6 ++--- .../pilorama/interface.go | 2 +- pkg/local_object_storage/shard/tree.go | 6 ++--- pkg/services/tree/service.go | 4 +-- 8 files changed, 53 insertions(+), 34 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 52ea399fe..8bacdba76 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -210,7 +210,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree } // TreeSortedByFilename implements the pilorama.Forest interface. -func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node, last string, count int) ([]pilorama.NodeInfo, string, error) { +func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node, last *string, count int) ([]pilorama.NodeInfo, *string, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeSortedByFilename", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -222,7 +222,7 @@ func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, var err error var nodes []pilorama.NodeInfo - var cursor string + var cursor *string for _, sh := range e.sortShards(cid) { nodes, cursor, err = sh.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count) if err != nil { diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index d4ae35ff4..9fc95dbbb 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1004,7 +1004,7 @@ func (t *boltForest) hasFewChildren(b *bbolt.Bucket, nodeID Node, threshold int) } // TreeSortedByFilename implements the Forest interface. -func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node, last string, count int) ([]NodeInfo, string, error) { +func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node, last *string, count int) ([]NodeInfo, *string, error) { var ( startedAt = time.Now() success = false @@ -1026,7 +1026,7 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr defer t.modeMtx.RUnlock() if t.mode.NoMetabase() { - return nil, "", ErrDegradedMode + return nil, last, ErrDegradedMode } h := newHeap(last, count) @@ -1070,20 +1070,25 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr } if fewChildren { - result = sortAndCut(result, []byte(last)) + result = sortAndCut(result, last) } if len(result) != 0 { - last = string(result[len(result)-1].Meta.GetAttr(AttributeFilename)) + s := string(result[len(result)-1].Meta.GetAttr(AttributeFilename)) + last = &s } return result, last, metaerr.Wrap(err) } -func sortAndCut(result []NodeInfo, last []byte) []NodeInfo { +func sortAndCut(result []NodeInfo, last *string) []NodeInfo { + var lastBytes []byte + if last != nil { + lastBytes = []byte(*last) + } sort.Slice(result, func(i, j int) bool { return bytes.Compare(result[i].Meta.GetAttr(AttributeFilename), result[j].Meta.GetAttr(AttributeFilename)) == -1 }) for i := range result { - if bytes.Compare(last, result[i].Meta.GetAttr(AttributeFilename)) == -1 { + if lastBytes == nil || bytes.Compare(lastBytes, result[i].Meta.GetAttr(AttributeFilename)) == -1 { return result[i:] } } diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index fedd3f90d..b45a77b99 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -156,11 +156,11 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string, } // TreeSortedByFilename implements the Forest interface. -func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeID Node, start string, count int) ([]NodeInfo, string, error) { +func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeID Node, start *string, count int) ([]NodeInfo, *string, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { - return nil, "", ErrTreeNotFound + return nil, start, ErrTreeNotFound } if count == 0 { return nil, start, nil @@ -169,7 +169,14 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI children := s.tree.getChildren(nodeID) res := make([]NodeInfo, 0, len(children)) for _, childID := range children { - if len(s.infoMap[childID].Meta.GetAttr(AttributeFilename)) == 0 { + var found bool + for _, kv := range s.infoMap[childID].Meta.Items { + if kv.Key == AttributeFilename { + found = true + break + } + } + if !found { continue } res = append(res, NodeInfo{ @@ -179,22 +186,24 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI }) } if len(res) == 0 { - return res, "", nil + return res, start, nil } sort.Slice(res, func(i, j int) bool { return bytes.Compare(res[i].Meta.GetAttr(AttributeFilename), res[j].Meta.GetAttr(AttributeFilename)) == -1 }) for i := range res { - if string(res[i].Meta.GetAttr(AttributeFilename)) > start { + if start == nil || string(res[i].Meta.GetAttr(AttributeFilename)) > *start { finish := i + count if len(res) < finish { finish = len(res) } - return res[i:finish], string(res[finish-1].Meta.GetAttr(AttributeFilename)), nil + last := string(res[finish-1].Meta.GetAttr(AttributeFilename)) + return res[i:finish], &last, nil } } - return nil, string(res[len(res)-1].Meta.GetAttr(AttributeFilename)), nil + last := string(res[len(res)-1].Meta.GetAttr(AttributeFilename)) + return nil, &last, nil } // TreeGetChildren implements the Forest interface. diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 9c343f632..9da0177cc 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -215,7 +215,7 @@ func BenchmarkForestSortedIteration(b *testing.B) { b.Run(providers[i].name+",root", func(b *testing.B) { for i := 0; i < b.N; i++ { - res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, RootID, "", 100) + res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, RootID, nil, 100) if err != nil || len(res) != 100 { b.Fatalf("err %v, count %d", err, len(res)) } @@ -223,7 +223,7 @@ func BenchmarkForestSortedIteration(b *testing.B) { }) b.Run(providers[i].name+",leaf", func(b *testing.B) { for i := 0; i < b.N; i++ { - res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, 1, "", 100) + res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, 1, nil, 100) if err != nil || len(res) != 0 { b.FailNow() } @@ -246,14 +246,14 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { cid := cidtest.ID() d := CIDDescriptor{cid, 0, 1} treeID := "version" - treeAdd := func(t *testing.T, ts int) { + treeAdd := func(t *testing.T, ts int, filename string) { _, err := s.TreeMove(context.Background(), d, treeID, &Move{ Child: RootID + uint64(ts), Parent: RootID, Meta: Meta{ Time: Timestamp(ts), Items: []KeyValue{ - {Key: AttributeFilename, Value: []byte(strconv.Itoa(ts))}, + {Key: AttributeFilename, Value: []byte(filename)}, }, }, }) @@ -261,19 +261,20 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { } const count = 9 - for i := 0; i < count; i++ { - treeAdd(t, i+1) + treeAdd(t, 1, "") + for i := 1; i < count; i++ { + treeAdd(t, i+1, strconv.Itoa(i+1)) } var result []NodeInfo - treeAppend := func(t *testing.T, last string, count int) string { + treeAppend := func(t *testing.T, last *string, count int) *string { res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, RootID, last, count) require.NoError(t, err) result = append(result, res...) return cursor } - last := treeAppend(t, "", 2) + last := treeAppend(t, nil, 2) last = treeAppend(t, last, 3) last = treeAppend(t, last, 0) last = treeAppend(t, last, 1) @@ -282,7 +283,11 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { require.Len(t, result, count) for i := range result { require.Equal(t, RootID+uint64(i+1), result[i].ID) - require.Equal(t, strconv.Itoa(RootID+i+1), string(result[i].Meta.GetAttr(AttributeFilename))) + if i == 0 { + require.Equal(t, "", string(result[i].Meta.GetAttr(AttributeFilename))) + } else { + require.Equal(t, strconv.Itoa(RootID+i+1), string(result[i].Meta.GetAttr(AttributeFilename))) + } } } @@ -341,7 +346,7 @@ func testForestTreeSortedByFilename(t *testing.T, s ForestStorage) { } getChildren := func(t *testing.T, id Node) []NodeInfo { - res, _, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, id, "", len(items)) + res, _, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, id, nil, len(items)) require.NoError(t, err) return res } diff --git a/pkg/local_object_storage/pilorama/heap.go b/pkg/local_object_storage/pilorama/heap.go index d81ec735d..e8a21e221 100644 --- a/pkg/local_object_storage/pilorama/heap.go +++ b/pkg/local_object_storage/pilorama/heap.go @@ -28,12 +28,12 @@ func (h *filenameHeap) Pop() any { // fixedHeap maintains a fixed number of smallest elements started at some point. type fixedHeap struct { - start string + start *string count int h *filenameHeap } -func newHeap(start string, count int) *fixedHeap { +func newHeap(start *string, count int) *fixedHeap { h := new(filenameHeap) heap.Init(h) @@ -45,7 +45,7 @@ func newHeap(start string, count int) *fixedHeap { } func (h *fixedHeap) push(id Node, filename string) bool { - if filename == "" || filename <= h.start { + if h.start != nil && filename <= *h.start { return false } heap.Push(h.h, heapInfo{id: id, filename: filename}) diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 79449e080..f7f5a85b1 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -35,7 +35,7 @@ type Forest interface { TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) // TreeSortedByFilename returns children of the node with the specified ID. The nodes are sorted by the filename attribute.. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. - TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node, last string, count int) ([]NodeInfo, string, error) + TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node, last *string, count int) ([]NodeInfo, *string, error) // TreeGetOpLog returns first log operation stored at or above the height. // In case no such operation is found, empty Move and nil error should be returned. TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 9a78f99ea..e92a61e5b 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -184,7 +184,7 @@ func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID strin } // TreeSortedByFilename implements the pilorama.Forest interface. -func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node, last string, count int) ([]pilorama.NodeInfo, string, error) { +func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node, last *string, count int) ([]pilorama.NodeInfo, *string, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeSortedByFilename", trace.WithAttributes( attribute.String("shard_id", s.ID().String()), @@ -196,14 +196,14 @@ func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID defer span.End() if s.pilorama == nil { - return nil, "", ErrPiloramaDisabled + return nil, last, ErrPiloramaDisabled } s.m.RLock() defer s.m.RUnlock() if s.info.Mode.NoMetabase() { - return nil, "", ErrDegradedMode + return nil, last, ErrDegradedMode } return s.pilorama.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count) } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index cce41e94d..85f531466 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -446,7 +446,7 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid type stackItem struct { values []pilorama.NodeInfo parent pilorama.Node - last string + last *string } // Traverse the tree in a DFS manner. Because we need to support arbitrary depth, @@ -502,7 +502,7 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid } if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() { - children, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), node.ID, "", batchSize) + children, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), node.ID, nil, batchSize) if err != nil { return err } From dc6778f38516eaa3f75e7d0b868a725efe1a5ec0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Jun 2024 16:22:12 +0300 Subject: [PATCH 1189/1943] [#1213] fmt: Drop unused interfaces Signed-off-by: Dmitrii Stepanov --- pkg/core/object/fmt.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index b997aa182..96f721806 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -14,7 +14,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -41,30 +40,12 @@ type cfg struct { verifyTokenIssuer bool } -// DeleteHandler is an interface of delete queue processor. -type DeleteHandler interface { - // DeleteObjects places objects to a removal queue. - // - // Returns apistatus.LockNonRegularObject if at least one object - // is locked. - DeleteObjects(oid.Address, ...oid.Address) error -} - // LockSource is a source of lock relations between the objects. type LockSource interface { // IsLocked must clarify object's lock status. IsLocked(ctx context.Context, address oid.Address) (bool, error) } -// Locker is an object lock storage interface. -type Locker interface { - // Lock list of objects as locked by locker in the specified container. - // - // Returns apistatus.LockNonRegularObject if at least object in locked - // list is irregular (not type of REGULAR). - Lock(idCnr cid.ID, locker oid.ID, locked []oid.ID) error -} - var errNilObject = errors.New("object is nil") var errNilID = errors.New("missing identifier") From dc2867682fb89c10618a463ce7d422f39c12ca4f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Jun 2024 16:22:37 +0300 Subject: [PATCH 1190/1943] [#1213] deleteSvc: Do not allow to delete EC chunks Signed-off-by: Dmitrii Stepanov --- pkg/services/object/delete/exec.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index 22928dcd5..c2f92950f 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -17,6 +17,8 @@ import ( "go.uber.org/zap" ) +var errDeleteECChunk = errors.New("invalid operation: delete EC object chunk") + type execCtx struct { svc *Service @@ -66,13 +68,16 @@ func (exec *execCtx) newAddress(id oid.ID) oid.Address { } func (exec *execCtx) formExtendedInfo(ctx context.Context) error { - _, err := exec.svc.header.head(ctx, exec) + obj, err := exec.svc.header.head(ctx, exec) var errSplitInfo *objectSDK.SplitInfoError var errECInfo *objectSDK.ECInfoError switch { case err == nil: + if ech := obj.ECHeader(); ech != nil { + return errDeleteECChunk + } return nil case errors.As(err, &errSplitInfo): exec.splitInfo = errSplitInfo.SplitInfo() From 10497e913697741840278eb134efb6588a8b4f0a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Jun 2024 17:45:00 +0300 Subject: [PATCH 1191/1943] [#1213] cli: Do not allow to lock EC chunks Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/util.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index ff6b3219d..381c790e9 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -354,7 +354,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, Prepare(cmd, &prmHead) - _, err := internal.HeadObject(cmd.Context(), prmHead) + o, err := internal.HeadObject(cmd.Context(), prmHead) var errSplit *objectSDK.SplitInfoError var errEC *objectSDK.ECInfoError @@ -364,6 +364,9 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, commonCmd.ExitOnErr(cmd, "failed to get raw object header: %w", err) case err == nil: common.PrintVerbose(cmd, "Raw header received - object is singular.") + if ech := o.Header().ECHeader(); ech != nil { + commonCmd.ExitOnErr(cmd, "Lock EC chunk failed: %w", errors.ErrUnsupported) + } return nil case errors.As(err, &errSplit): common.PrintVerbose(cmd, "Split information received - object is virtual.") From 7a8ac4907a39a12c907c7e1167909ef4447b570c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Jun 2024 22:36:29 +0300 Subject: [PATCH 1192/1943] [#1213] engine: Drop unused Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/head.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index 97fb5be3a..92d1b20fc 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -151,18 +151,3 @@ func Head(ctx context.Context, storage *StorageEngine, addr oid.Address) (*objec return res.Header(), nil } - -// HeadRaw reads object header from local storage by provided address and raw -// flag. -func HeadRaw(ctx context.Context, storage *StorageEngine, addr oid.Address, raw bool) (*objectSDK.Object, error) { - var headPrm HeadPrm - headPrm.WithAddress(addr) - headPrm.WithRaw(raw) - - res, err := storage.Head(ctx, headPrm) - if err != nil { - return nil, err - } - - return res.Header(), nil -} From 72ab373b719887dd04859adbd945d6ac29a752ba Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 26 Jun 2024 17:10:27 +0300 Subject: [PATCH 1193/1943] [#218] adm: Update doc for `morph generate-alphabet` Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/README.md | 3 ++- cmd/frostfs-adm/docs/deploy.md | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/README.md b/cmd/frostfs-adm/README.md index f3dbe6950..3dfcc8781 100644 --- a/cmd/frostfs-adm/README.md +++ b/cmd/frostfs-adm/README.md @@ -56,7 +56,8 @@ credentials: # passwords for consensus node / alphabet wallets #### Network deployment - `generate-alphabet` generates a set of wallets for consensus and - Alphabet nodes. + Alphabet nodes. The list of the name for alphabet wallets(no gaps between names allowed, order is important): + - az, buky, vedi, glagoli, dobro, yest, zhivete, dzelo, zemlja, izhe, izhei, gerv, kako, ljudi, mislete, nash, on, pokoj, rtsi, slovo, tverdo, uk - `init` initializes the sidechain by deploying smart contracts and setting provided FrostFS network configuration. diff --git a/cmd/frostfs-adm/docs/deploy.md b/cmd/frostfs-adm/docs/deploy.md index e510f478d..87d2e47c1 100644 --- a/cmd/frostfs-adm/docs/deploy.md +++ b/cmd/frostfs-adm/docs/deploy.md @@ -64,6 +64,11 @@ alphabet-wallets: /home/user/deploy/alphabet-wallets wallet[0]: hunter2 ``` +This command generates wallets with the following names: + - az, buky, vedi, glagoli, dobro, yest, zhivete, dzelo, zemlja, izhe, izhei, gerv, kako, ljudi, mislete, nash, on, pokoj, rtsi, slovo, tverdo, uk + +No gaps between names allowed, order is important. + Do not lose wallet files and network config. Store it in an encrypted backed up storage. From 36eab4059c43af7f1d7f27b91cde85450a166dcb Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 26 Jun 2024 14:30:31 +0300 Subject: [PATCH 1194/1943] [#218] adm: Refactor `helper` in part of reading alphabet wallets Signed-off-by: Anton Nikiforov --- .../internal/modules/morph/helper/util.go | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/util.go b/cmd/frostfs-adm/internal/modules/morph/helper/util.go index ba557a033..ea8266b38 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/util.go @@ -40,45 +40,48 @@ func openAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, er return nil, fmt.Errorf("can't read alphabet wallets dir: %w", err) } - var size int -loop: - for i := 0; i < len(walletFiles); i++ { - name := innerring.GlagoliticLetter(i).String() + ".json" - for j := range walletFiles { - if walletFiles[j].Name() == name { - size++ - continue loop - } - } - break - } - if size == 0 { - return nil, errors.New("alphabet wallets dir is empty (run `generate-alphabet` command first)") - } - - wallets := make([]*wallet.Wallet, size) - for i := 0; i < size; i++ { - letter := innerring.GlagoliticLetter(i).String() + var wallets []*wallet.Wallet + var letter string + for i := 0; i < constants.MaxAlphabetNodes; i++ { + letter = innerring.GlagoliticLetter(i).String() p := filepath.Join(walletDir, letter+".json") - w, err := wallet.NewWalletFromFile(p) + var w *wallet.Wallet + w, err = wallet.NewWalletFromFile(p) if err != nil { - return nil, fmt.Errorf("can't open wallet: %w", err) + if errors.Is(err, os.ErrNotExist) { + err = nil + } else { + err = fmt.Errorf("can't open wallet: %w", err) + } + break } - password, err := config.GetPassword(v, letter) + var password string + password, err = config.GetPassword(v, letter) if err != nil { - return nil, fmt.Errorf("can't fetch password: %w", err) + err = fmt.Errorf("can't fetch password: %w", err) + break } for i := range w.Accounts { - if err := w.Accounts[i].Decrypt(password, keys.NEP2ScryptParams()); err != nil { - return nil, fmt.Errorf("can't unlock wallet: %w", err) + if err = w.Accounts[i].Decrypt(password, keys.NEP2ScryptParams()); err != nil { + err = fmt.Errorf("can't unlock wallet: %w", err) + break } } - wallets[i] = w + wallets = append(wallets, w) + } + if err != nil { + return nil, fmt.Errorf("can't read wallet for letter '%s': %w", letter, err) + } + if len(wallets) == 0 { + err = errors.New("there are no alphabet wallets in dir (run `generate-alphabet` command first)") + if len(walletFiles) > 0 { + err = fmt.Errorf("use glagolitic names for wallets(run `print-alphabet`): %w", err) + } + return nil, err } - return wallets, nil } From 56eeb630b6fbe67a26e043e6de8d523839327723 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 1 Jul 2024 18:58:37 +0300 Subject: [PATCH 1195/1943] [#1217] Fix grammar mistakes and misspelling Signed-off-by: Ekaterina Lebedeva --- .forgejo/workflows/pre-commit.yml | 2 +- .../internal/modules/morph/frostfsid/frostfsid.go | 2 +- cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go | 2 +- cmd/frostfs-adm/internal/modules/morph/policy/policy.go | 2 +- cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go | 2 +- cmd/frostfs-cli/modules/control/shards_list.go | 2 +- cmd/frostfs-cli/modules/tree/add.go | 2 +- cmd/frostfs-node/config/cast.go | 8 ++++---- internal/logs/logs.go | 6 +++--- .../blobstor/blobovniczatree/cache.go | 2 +- pkg/local_object_storage/metabase/graveyard.go | 2 +- pkg/local_object_storage/metabase/list.go | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.forgejo/workflows/pre-commit.yml b/.forgejo/workflows/pre-commit.yml index 6453b5029..117cda93b 100644 --- a/.forgejo/workflows/pre-commit.yml +++ b/.forgejo/workflows/pre-commit.yml @@ -9,7 +9,7 @@ jobs: SKIP: make-lint,go-staticcheck-repo-mod,go-unit-tests,gofumpt runs-on: ubuntu-22.04 # If we use actions/setup-python from either Github or Gitea, - # the line above fails with a cryptic error about not being able to found python. + # the line above fails with a cryptic error about not being able to find python. # So install everything manually. steps: - uses: actions/checkout@v3 diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 541282967..6b274ddfb 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -426,7 +426,7 @@ type frostfsidClient struct { func newFrostfsIDClient(cmd *cobra.Command) (*frostfsidClient, error) { wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { - return nil, fmt.Errorf("can't to initialize context: %w", err) + return nil, fmt.Errorf("can't initialize context: %w", err) } r := management.NewReader(wCtx.ReadOnlyInvoker) diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go index 4fd671f2a..e60dc5673 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go @@ -15,7 +15,7 @@ import ( func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { - return fmt.Errorf("can't to initialize context: %w", err) + return fmt.Errorf("can't initialize context: %w", err) } r := management.NewReader(wCtx.ReadOnlyInvoker) diff --git a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go index 5ab05e429..36547e22c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go @@ -30,7 +30,7 @@ var errInvalidParameterFormat = errors.New("invalid parameter format, must be Pa func SetPolicyCmd(cmd *cobra.Command, args []string) error { wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { - return fmt.Errorf("can't to initialize context: %w", err) + return fmt.Errorf("can't initialize context: %w", err) } bw := io.NewBufBinWriter() diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go index 7e173ac49..ba48465f7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go @@ -39,7 +39,7 @@ func removeProxyAccount(cmd *cobra.Command, _ []string) { func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error { wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { - return fmt.Errorf("can't to initialize context: %w", err) + return fmt.Errorf("can't initialize context: %w", err) } r := management.NewReader(wCtx.ReadOnlyInvoker) diff --git a/cmd/frostfs-cli/modules/control/shards_list.go b/cmd/frostfs-cli/modules/control/shards_list.go index 3eb74ec82..07c5bcd9a 100644 --- a/cmd/frostfs-cli/modules/control/shards_list.go +++ b/cmd/frostfs-cli/modules/control/shards_list.go @@ -80,7 +80,7 @@ func prettyPrintShardsJSON(cmd *cobra.Command, ii []*control.ShardInfo) { enc.SetIndent("", " ") commonCmd.ExitOnErr(cmd, "cannot shard info to JSON: %w", enc.Encode(out)) - cmd.Print(buf.String()) // pretty printer emits newline, to no need for Println + cmd.Print(buf.String()) // pretty printer emits newline, so no need for Println } func prettyPrintShards(cmd *cobra.Command, ii []*control.ShardInfo) { diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index 8ceea4173..068b1d185 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -72,7 +72,7 @@ func add(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "signing message: %w", tree.SignMessage(req, pk)) resp, err := cli.Add(ctx, req) - commonCmd.ExitOnErr(cmd, "failed to cal add: %w", err) + commonCmd.ExitOnErr(cmd, "failed to call add: %w", err) cmd.Println("Node ID: ", resp.GetBody().GetNodeId()) } diff --git a/cmd/frostfs-node/config/cast.go b/cmd/frostfs-node/config/cast.go index c99d33569..beec09a9e 100644 --- a/cmd/frostfs-node/config/cast.go +++ b/cmd/frostfs-node/config/cast.go @@ -16,7 +16,7 @@ func panicOnErr(err error) { } // StringSlice reads a configuration value -// from c by name and casts it to a []string. +// from c by name and casts it to []string. // // Panics if the value can not be casted. func StringSlice(c *Config, name string) []string { @@ -27,7 +27,7 @@ func StringSlice(c *Config, name string) []string { } // StringSliceSafe reads a configuration value -// from c by name and casts it to a []string. +// from c by name and casts it to []string. // // Returns nil if the value can not be casted. func StringSliceSafe(c *Config, name string) []string { @@ -35,7 +35,7 @@ func StringSliceSafe(c *Config, name string) []string { } // String reads a configuration value -// from c by name and casts it to a string. +// from c by name and casts it to string. // // Panics if the value can not be casted. func String(c *Config, name string) string { @@ -46,7 +46,7 @@ func String(c *Config, name string) string { } // StringSafe reads a configuration value -// from c by name and casts it to a string. +// from c by name and casts it to string. // // Returns "" if the value can not be casted. func StringSafe(c *Config, name string) string { diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 78f00c4ee..c5e4a8e28 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -402,9 +402,9 @@ const ( FrostFSNodeInternalApplicationError = "internal application error" FrostFSNodeInternalErrorProcessingIsComplete = "internal error processing is complete" FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration = "SIGHUP has been received, rereading configuration..." - FrostFSNodeSIGHUPSkip = "node not ready for reconfiguration, skipped SIGHUP" - FrostFSNodeShutdownSkip = "node already is going to shutting down, skipped shutdown" - FrostFSNodeShutdownWhenNotReady = "node is going to shutting down when subsystems still initializing" + FrostFSNodeSIGHUPSkip = "node is not ready for reconfiguration, skipped SIGHUP" + FrostFSNodeShutdownSkip = "node is already shutting down, skipped shutdown" + FrostFSNodeShutdownWhenNotReady = "node is going to shut down when subsystems are still initializing" FrostFSNodeConfigurationReading = "configuration reading" FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" FrostFSNodeTracingConfigationUpdated = "tracing configation updated" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go index 4fce3e81a..5c103c1bb 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -12,7 +12,7 @@ import ( // dbCache caches sharedDB instances that are NOT open for Put. // // Uses dbManager for opening/closing sharedDB instances. -// Stores a reference to an cached sharedDB, so dbManager does not close it. +// Stores a reference to a cached sharedDB, so dbManager does not close it. type dbCache struct { cacheGuard *sync.Mutex cache cache.Cache[string, *sharedDB] diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index df9a3d302..80d40fb78 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -128,7 +128,7 @@ func (g *GraveyardIterationPrm) SetHandler(h TombstonedHandler) { // Note: if offset is not found in db, iteration starts // from the element that WOULD BE the following after the // offset if offset was presented. That means that it is -// safe to delete offset element and pass if to the +// safe to delete offset element and pass it to the // iteration once again: iteration would start from the // next element. // diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 544b2e666..b4326a92c 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -66,7 +66,7 @@ func (l ListRes) Cursor() *Cursor { // Use cursor value from response for consecutive requests. // // Returns ErrEndOfListing if there are no more objects to return or count -// parameter set to zero. +// parameter is set to zero. func (db *DB) ListWithCursor(ctx context.Context, prm ListPrm) (res ListRes, err error) { var ( startedAt = time.Now() From 259007540fc8ca0899a3cd4598f9e2e33a442e01 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 2 Jul 2024 15:11:35 +0300 Subject: [PATCH 1196/1943] [#1196] morph/client: Remove duplicate utility functions We used several utility functions to parse frostfsid client subject and extended subject. However, following the changes in TrueCloudLab/frostfs-contract#97, these utility functions have become public. So there is no more need to have them here. Signed-off-by: Ekaterina Lebedeva --- go.mod | 2 +- go.sum | 4 +- pkg/morph/client/frostfsid/subject.go | 202 +------------------------- 3 files changed, 5 insertions(+), 203 deletions(-) diff --git a/go.mod b/go.mod index 38e66a20f..d4eb52f4d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 - git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 + git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de diff --git a/go.sum b/go.sum index 0849b74a1..84fdbd47a 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 h1:H5GvrVlowIMWfzqQkhY0p0myooJxQ1sMRVSFfXawwWg= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 h1:Tp4I+XOLp3VCJORfxSamQtj3RZNISbaLM4WD5iIzXxg= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= diff --git a/pkg/morph/client/frostfsid/subject.go b/pkg/morph/client/frostfsid/subject.go index 169be6bc5..50733d89a 100644 --- a/pkg/morph/client/frostfsid/subject.go +++ b/pkg/morph/client/frostfsid/subject.go @@ -1,16 +1,11 @@ package frostfsid import ( - "errors" "fmt" frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" ) const ( @@ -28,7 +23,7 @@ func (c *Client) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) return nil, fmt.Errorf("could not perform test invocation (%s): %w", methodGetSubject, err) } - subj, err := parseSubject(res) + subj, err := frostfsidclient.ParseSubject(res) if err != nil { return nil, fmt.Errorf("could not parse test invocation result (%s): %w", methodGetSubject, err) } @@ -46,203 +41,10 @@ func (c *Client) GetSubjectExtended(addr util.Uint160) (*frostfsidclient.Subject return nil, fmt.Errorf("could not perform test invocation (%s): %w", methodGetSubject, err) } - subj, err := parseSubjectExtended(res) + subj, err := frostfsidclient.ParseSubjectExtended(res) if err != nil { return nil, fmt.Errorf("could not parse test invocation result (%s): %w", methodGetSubject, err) } return subj, nil } - -func parseSubject(res []stackitem.Item) (*frostfsidclient.Subject, error) { - if ln := len(res); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", methodGetSubject, ln) - } - - structArr, err := client.ArrayFromStackItem(res[0]) - if err != nil { - return nil, fmt.Errorf("could not get item array of container (%s): %w", methodGetSubject, err) - } - - var subj frostfsidclient.Subject - - subj.PrimaryKey, err = unwrap.PublicKey(makeValidRes(structArr[0])) - if err != nil { - return nil, err - } - - if !structArr[1].Equals(stackitem.Null{}) { - subj.AdditionalKeys, err = unwrap.ArrayOfPublicKeys(makeValidRes(structArr[1])) - if err != nil { - return nil, err - } - } - - if !structArr[2].Equals(stackitem.Null{}) { - subj.Namespace, err = unwrap.UTF8String(makeValidRes(structArr[2])) - if err != nil { - return nil, err - } - } - - if !structArr[3].Equals(stackitem.Null{}) { - subj.Name, err = unwrap.UTF8String(makeValidRes(structArr[3])) - if err != nil { - return nil, err - } - } - - subj.KV, err = parseMap(structArr[4]) - if err != nil { - return nil, err - } - - return &subj, nil -} - -func parseSubjectExtended(res []stackitem.Item) (*frostfsidclient.SubjectExtended, error) { - if ln := len(res); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", methodGetSubject, ln) - } - - structArr, err := client.ArrayFromStackItem(res[0]) - if err != nil { - return nil, fmt.Errorf("could not get item array of container (%s): %w", methodGetSubject, err) - } - - var subj frostfsidclient.SubjectExtended - - subj.PrimaryKey, err = unwrap.PublicKey(makeValidRes(structArr[0])) - if err != nil { - return nil, err - } - - if !structArr[1].Equals(stackitem.Null{}) { - subj.AdditionalKeys, err = unwrap.ArrayOfPublicKeys(makeValidRes(structArr[1])) - if err != nil { - return nil, err - } - } - - if !structArr[2].Equals(stackitem.Null{}) { - subj.Namespace, err = unwrap.UTF8String(makeValidRes(structArr[2])) - if err != nil { - return nil, err - } - } - - if !structArr[3].Equals(stackitem.Null{}) { - subj.Name, err = unwrap.UTF8String(makeValidRes(structArr[3])) - if err != nil { - return nil, err - } - } - - subj.KV, err = parseMap(structArr[4]) - if err != nil { - return nil, err - } - - if !structArr[5].Equals(stackitem.Null{}) { - groupItems, ok := structArr[5].Value().([]stackitem.Item) - if !ok { - return nil, errors.New("invalid groups field") - } - - subj.Groups, err = parseGroups(groupItems) - if err != nil { - return nil, err - } - } - - return &subj, nil -} - -func makeValidRes(item stackitem.Item) (*result.Invoke, error) { - return &result.Invoke{ - Stack: []stackitem.Item{item}, - State: vmstate.Halt.String(), - }, nil -} - -func parseMap(item stackitem.Item) (map[string]string, error) { - if item.Equals(stackitem.Null{}) { - return nil, nil - } - - metaMap, err := unwrap.Map(makeValidRes(item)) - if err != nil { - return nil, err - } - - meta, ok := metaMap.Value().([]stackitem.MapElement) - if !ok { - return nil, errors.New("invalid map type") - } - - res := make(map[string]string, len(meta)) - for _, element := range meta { - key, err := element.Key.TryBytes() - if err != nil { - return nil, err - } - val, err := element.Value.TryBytes() - if err != nil { - return nil, err - } - res[string(key)] = string(val) - } - - return res, nil -} - -func parseGroups(items []stackitem.Item) ([]*frostfsidclient.Group, error) { - var err error - res := make([]*frostfsidclient.Group, len(items)) - - for i := 0; i < len(items); i++ { - arr, ok := items[i].Value().([]stackitem.Item) - if !ok { - return nil, errors.New("invalid group type") - } - res[i], err = parseGroup(arr) - if err != nil { - return nil, err - } - } - - return res, nil -} - -func parseGroup(structArr []stackitem.Item) (*frostfsidclient.Group, error) { - if len(structArr) < 4 { - return nil, errors.New("invalid response group struct") - } - - groupID, err := structArr[0].TryInteger() - if err != nil { - return nil, err - } - - name, err := structArr[1].TryBytes() - if err != nil { - return nil, err - } - - namespace, err := structArr[2].TryBytes() - if err != nil { - return nil, err - } - - kvs, err := parseMap(structArr[3]) - if err != nil { - return nil, err - } - - return &frostfsidclient.Group{ - ID: groupID.Int64(), - Name: string(name), - Namespace: string(namespace), - KV: kvs, - }, nil -} From 74842e7f4337154a0d047461b8b011c1ff7e4127 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 28 Jun 2024 16:35:18 +0300 Subject: [PATCH 1197/1943] [#1210] adm: Fix error handling when contract not found Signed-off-by: Alexander Chuprov --- .../internal/modules/morph/ape/ape_util.go | 4 ++-- .../internal/modules/morph/balance/balance.go | 2 +- .../internal/modules/morph/config/config.go | 4 ++-- .../modules/morph/container/container.go | 4 ++-- .../internal/modules/morph/contract/deploy.go | 2 +- .../modules/morph/contract/dump_hashes.go | 2 +- .../modules/morph/frostfsid/frostfsid.go | 2 +- .../internal/modules/morph/helper/contract.go | 4 ++-- .../internal/modules/morph/helper/netmap.go | 2 +- .../internal/modules/morph/helper/util.go | 17 +++++++++++++++++ .../modules/morph/initialize/initialize_nns.go | 2 +- .../internal/modules/morph/netmap/epoch.go | 2 +- .../modules/morph/netmap/netmap_candidates.go | 2 +- .../internal/modules/morph/nns/helper.go | 2 +- .../internal/modules/morph/node/remove.go | 2 +- .../internal/modules/morph/proxy/proxy.go | 2 +- 16 files changed, 36 insertions(+), 19 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index dfa7c6392..5e17f4014 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -115,7 +115,7 @@ func newPolicyContractReaderInterface(cmd *cobra.Command) (*morph.ContractStorag inv := invoker.New(c, nil) var ch util.Uint160 r := management.NewReader(inv) - nnsCs, err := r.GetContractByID(1) + nnsCs, err := helper.GetContractByID(r, 1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) ch, err = helper.NNSResolveHash(inv, nnsCs.Hash, helper.DomainOf(constants.PolicyContract)) @@ -138,7 +138,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *he var ch util.Uint160 r := management.NewReader(ac.Invoker) - nnsCs, err := r.GetContractByID(1) + nnsCs, err := helper.GetContractByID(r, 1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) ch, err = helper.NNSResolveHash(ac.Invoker, nnsCs.Hash, helper.DomainOf(constants.PolicyContract)) diff --git a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go index 428fb98b9..5519705d4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go @@ -60,7 +60,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { if dumpStorage || dumpAlphabet || dumpProxy { r := management.NewReader(inv) - nnsCs, err = r.GetContractByID(1) + nnsCs, err = helper.GetContractByID(r, 1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index ba6e515c1..3a7f84acb 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -34,7 +34,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { inv := invoker.New(c, nil) r := management.NewReader(inv) - cs, err := r.GetContractByID(1) + cs, err := helper.GetContractByID(r, 1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) } @@ -93,7 +93,7 @@ func SetConfigCmd(cmd *cobra.Command, args []string) error { } r := management.NewReader(wCtx.ReadOnlyInvoker) - cs, err := r.GetContractByID(1) + cs, err := helper.GetContractByID(r, 1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/container/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go index eda388d37..a66438975 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -34,7 +34,7 @@ func getContainerContractHash(cmd *cobra.Command, inv *invoker.Invoker) (util.Ui } if err != nil { r := management.NewReader(inv) - nnsCs, err := r.GetContractByID(1) + nnsCs, err := helper.GetContractByID(r, 1) if err != nil { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } @@ -304,7 +304,7 @@ func parseContainers(filename string) ([]Container, error) { func fetchContainerContractHash(wCtx *helper.InitializeContext) (util.Uint160, error) { r := management.NewReader(wCtx.ReadOnlyInvoker) - nnsCs, err := r.GetContractByID(1) + nnsCs, err := helper.GetContractByID(r, 1) if err != nil { return util.Uint160{}, fmt.Errorf("can't get NNS contract state: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go index 7a8f34a5d..5adb480da 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go @@ -79,7 +79,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { } r := management.NewReader(c.ReadOnlyInvoker) - nnsCs, err := r.GetContractByID(1) + nnsCs, err := helper.GetContractByID(r, 1) if err != nil { return fmt.Errorf("can't fetch NNS contract state: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index 11cc3bd04..5a0d29550 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -42,7 +42,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { } r := management.NewReader(invoker.New(c, nil)) - cs, err := r.GetContractByID(1) + cs, err := helper.GetContractByID(r, 1) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 6b274ddfb..442920912 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -430,7 +430,7 @@ func newFrostfsIDClient(cmd *cobra.Command) (*frostfsidClient, error) { } r := management.NewReader(wCtx.ReadOnlyInvoker) - cs, err := r.GetContractByID(1) + cs, err := helper.GetContractByID(r, 1) if err != nil { return nil, fmt.Errorf("can't get NNS contract info: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/contract.go b/cmd/frostfs-adm/internal/modules/morph/helper/contract.go index 7326ec1f5..2011301d1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/contract.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/contract.go @@ -15,7 +15,7 @@ import ( func getFrostfsIDAdminFromContract(roInvoker *invoker.Invoker) (util.Uint160, bool, error) { r := management.NewReader(roInvoker) - cs, err := r.GetContractByID(1) + cs, err := GetContractByID(r, 1) if err != nil { return util.Uint160{}, false, fmt.Errorf("get nns contract: %w", err) } @@ -62,7 +62,7 @@ func GetContractDeployData(c *InitializeContext, ctrName string, keysParam []any // In case if NNS is updated multiple times, we can't calculate // it's actual hash based on local data, thus query chain. r := management.NewReader(c.ReadOnlyInvoker) - nnsCs, err := r.GetContractByID(1) + nnsCs, err := GetContractByID(r, 1) if err != nil { return nil, fmt.Errorf("get nns contract: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go index 9ea27d3a3..7a778f8c3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go @@ -89,7 +89,7 @@ func EmitNewEpochCall(bw *io.BufBinWriter, wCtx *InitializeContext, nmHash util. func GetNetConfigFromNetmapContract(roInvoker *invoker.Invoker) ([]stackitem.Item, error) { r := management.NewReader(roInvoker) - cs, err := r.GetContractByID(1) + cs, err := GetContractByID(r, 1) if err != nil { return nil, fmt.Errorf("get nns contract: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/util.go b/cmd/frostfs-adm/internal/modules/morph/helper/util.go index ea8266b38..2d9281c24 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/util.go @@ -14,10 +14,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" + "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/viper" ) @@ -181,3 +183,18 @@ func ParseGASAmount(s string) (fixedn.Fixed8, error) { } return gasAmount, nil } + +// GetContractByID retrieves a contract by its ID using the standard GetContractByID method. +// However, if the returned state.Contract is nil, it returns an error indicating that the contract was not found. +// See https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/1210 +func GetContractByID(r *management.ContractReader, id int32) (*state.Contract, error) { + cs, err := r.GetContractByID(id) + if err != nil { + return nil, err + } + + if cs == nil { + return nil, errors.New("contract not found") + } + return cs, nil +} diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go index 14f12cf8e..e127ca545 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go @@ -21,7 +21,7 @@ import ( func setNNS(c *helper.InitializeContext) error { r := management.NewReader(c.ReadOnlyInvoker) - nnsCs, err := r.GetContractByID(1) + nnsCs, err := helper.GetContractByID(r, 1) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go index e60dc5673..df9a03fd1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go @@ -19,7 +19,7 @@ func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { } r := management.NewReader(wCtx.ReadOnlyInvoker) - cs, err := r.GetContractByID(1) + cs, err := helper.GetContractByID(r, 1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go b/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go index e9404e910..d8471bb9a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go @@ -19,7 +19,7 @@ func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) { inv := invoker.New(c, nil) r := management.NewReader(inv) - cs, err := r.GetContractByID(1) + cs, err := helper.GetContractByID(r, 1) commonCmd.ExitOnErr(cmd, "can't get NNS contract info: %w", err) nmHash, err := helper.NNSResolveHash(inv, cs.Hash, helper.DomainOf(constants.NetmapContract)) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go index 36c3dd2fd..0eaa5ac58 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go @@ -19,7 +19,7 @@ func getRPCClient(cmd *cobra.Command) (*client.Contract, *helper.LocalActor, uti commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) r := management.NewReader(ac.Invoker) - nnsCs, err := r.GetContractByID(1) + nnsCs, err := helper.GetContractByID(r, 1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) return client.New(ac, nnsCs.Hash), ac, nnsCs.Hash } diff --git a/cmd/frostfs-adm/internal/modules/morph/node/remove.go b/cmd/frostfs-adm/internal/modules/morph/node/remove.go index 0d14bc646..0a19102ba 100644 --- a/cmd/frostfs-adm/internal/modules/morph/node/remove.go +++ b/cmd/frostfs-adm/internal/modules/morph/node/remove.go @@ -37,7 +37,7 @@ func RemoveNodesCmd(cmd *cobra.Command, args []string) error { defer wCtx.Close() r := management.NewReader(wCtx.ReadOnlyInvoker) - cs, err := r.GetContractByID(1) + cs, err := helper.GetContractByID(r, 1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go index ba48465f7..cb575b657 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go @@ -43,7 +43,7 @@ func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error } r := management.NewReader(wCtx.ReadOnlyInvoker) - cs, err := r.GetContractByID(1) + cs, err := helper.GetContractByID(r, 1) if err != nil { return fmt.Errorf("can't get NNS contract info: %w", err) } From 91bed3b0ba596a70f7365385ddc1dabf0a7bb9b4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 2 Jul 2024 14:09:03 +0300 Subject: [PATCH 1198/1943] [#1219] Remove Container.SetEACL method Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/container.go | 14 --- internal/logs/logs.go | 2 + .../processors/container/handlers.go | 19 +--- .../processors/container/handlers_test.go | 87 ----------------- .../processors/container/process_eacl.go | 82 ---------------- pkg/morph/client/container/eacl_set.go | 93 ------------------- pkg/services/container/morph/executor.go | 38 +------- pkg/services/container/morph/executor_test.go | 11 --- 8 files changed, 8 insertions(+), 338 deletions(-) delete mode 100644 pkg/innerring/processors/container/process_eacl.go delete mode 100644 pkg/morph/client/container/eacl_set.go diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index b14e19161..99831fe98 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -254,17 +254,3 @@ func (m morphContainerWriter) Put(cnr containerCore.Container) (*cid.ID, error) func (m morphContainerWriter) Delete(witness containerCore.RemovalWitness) error { return cntClient.Delete(m.neoClient, witness) } - -func (m morphContainerWriter) PutEACL(eaclInfo containerCore.EACL) error { - err := cntClient.PutEACL(m.neoClient, eaclInfo) - if err != nil { - return err - } - - if m.cacheEnabled { - id, _ := eaclInfo.Value.CID() - m.eacls.InvalidateEACL(id) - } - - return nil -} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index c5e4a8e28..2addc4482 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -14,6 +14,8 @@ const ( InterruptPlacementIterationByContext = "interrupt placement iteration by context" Notification = "notification" + + SkipDeprecatedNotification = "skip deprecated notification" ) const ( diff --git a/pkg/innerring/processors/container/handlers.go b/pkg/innerring/processors/container/handlers.go index 3ec10b889..8bb89abe2 100644 --- a/pkg/innerring/processors/container/handlers.go +++ b/pkg/innerring/processors/container/handlers.go @@ -49,21 +49,6 @@ func (cp *Processor) handleDelete(ev event.Event) { } } -func (cp *Processor) handleSetEACL(ev event.Event) { - e := ev.(containerEvent.SetEACL) - - cp.log.Info(logs.Notification, - zap.String("type", "set EACL"), - ) - - // send an event to the worker pool - - err := processors.SubmitEvent(cp.pool, cp.metrics, "container_set_eacl", func() bool { - return cp.processSetEACL(e) - }) - if err != nil { - // there system can be moved into controlled degradation stage - cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, - zap.Int("capacity", cp.pool.Cap())) - } +func (cp *Processor) handleSetEACL(_ event.Event) { + cp.log.Warn(logs.SkipDeprecatedNotification, zap.String("type", "set EACL")) } diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index a69d0e4c4..1aac31ae3 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -18,10 +18,8 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/network/payload" @@ -159,91 +157,6 @@ func TestDeleteEvent(t *testing.T) { require.EqualValues(t, []*transaction.Transaction{nr.MainTransaction}, mc.transactions, "invalid notary requests") } -func TestSetEACLEvent(t *testing.T) { - t.Parallel() - nst := &testNetworkState{ - homHashDisabled: true, - epoch: 100, - } - cc := &testContainerClient{ - get: make(map[string]*containercore.Container), - } - mc := &testMorphClient{} - - proc, err := New(&Params{ - Log: test.NewLogger(t), - PoolSize: 2, - AlphabetState: &testAlphabetState{isAlphabet: true}, - NetworkState: nst, - ContainerClient: cc, - MorphClient: mc, - FrostFSIDClient: &testFrostFSIDClient{}, - }) - require.NoError(t, err, "failed to create processor") - - p, err := keys.NewPrivateKey() - require.NoError(t, err) - - var usr user.ID - user.IDFromKey(&usr, (ecdsa.PublicKey)(*p.PublicKey())) - - var pp netmap.PlacementPolicy - pp.AddReplicas(netmap.ReplicaDescriptor{}) - - var cnr containerSDK.Container - cnr.Init() - cnr.SetOwner(usr) - cnr.SetPlacementPolicy(pp) - cnr.SetBasicACL(acl.PrivateExtended) - containerSDK.DisableHomomorphicHashing(&cnr) - - var cid cid.ID - containerSDK.CalculateID(&cid, cnr) - cidBytes := make([]byte, 32) - cid.Encode(cidBytes) - - var signature frostfscrypto.Signature - signer := frostfsecdsa.Signer(p.PrivateKey) - require.NoError(t, signature.Calculate(signer, cidBytes), "failed to calculate signature") - - cc.get[hex.EncodeToString(cidBytes)] = &containercore.Container{ - Value: cnr, - Signature: signature, - } - - table := eacl.NewTable() - table.SetCID(cid) - table.SetVersion(version.Current()) - - r := &eacl.Record{} - r.AddObjectContainerIDFilter(eacl.MatchStringEqual, cid) - - table.AddRecord(r) - - nr := &payload.P2PNotaryRequest{ - MainTransaction: &transaction.Transaction{}, - } - event := containerEvent.SetEACL{ - TableValue: table.ToV2().StableMarshal(nil), - PublicKeyValue: p.PublicKey().Bytes(), - SignatureValue: p.Sign(table.ToV2().StableMarshal(nil)), - NotaryRequestValue: nr, - } - - proc.handleSetEACL(event) - - for proc.pool.Running() > 0 { - time.Sleep(10 * time.Millisecond) - } - - var expectedPutEACL cntClient.PutEACLPrm - expectedPutEACL.SetTable(table.ToV2().StableMarshal(nil)) - expectedPutEACL.SetKey(p.PublicKey().Bytes()) - expectedPutEACL.SetSignature(p.Sign(table.ToV2().StableMarshal(nil))) - - require.EqualValues(t, []*transaction.Transaction{nr.MainTransaction}, mc.transactions, "invalid notary requests") -} - type testAlphabetState struct { isAlphabet bool } diff --git a/pkg/innerring/processors/container/process_eacl.go b/pkg/innerring/processors/container/process_eacl.go deleted file mode 100644 index 8ab0d5c39..000000000 --- a/pkg/innerring/processors/container/process_eacl.go +++ /dev/null @@ -1,82 +0,0 @@ -package container - -import ( - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" - "go.uber.org/zap" -) - -func (cp *Processor) processSetEACL(e containerEvent.SetEACL) bool { - if !cp.alphabetState.IsAlphabet() { - cp.log.Info(logs.ContainerNonAlphabetModeIgnoreSetEACL) - return true - } - - err := cp.checkSetEACL(e) - if err != nil { - cp.log.Error(logs.ContainerSetEACLCheckFailed, - zap.String("error", err.Error()), - ) - - return false - } - - if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { - cp.log.Error(logs.ContainerCouldNotApproveSetEACL, - zap.String("error", err.Error()), - ) - return false - } - - return true -} - -func (cp *Processor) checkSetEACL(e containerEvent.SetEACL) error { - binTable := e.Table() - - // unmarshal table - table := eacl.NewTable() - - err := table.Unmarshal(binTable) - if err != nil { - return fmt.Errorf("invalid binary table: %w", err) - } - - idCnr, ok := table.CID() - if !ok { - return errors.New("missing container ID in eACL table") - } - - // receive owner of the related container - cnr, err := cntClient.Get(cp.cnrClient, idCnr) - if err != nil { - return fmt.Errorf("could not receive the container: %w", err) - } - - // ACL extensions can be disabled by basic ACL, check it - if !cnr.Value.BasicACL().Extendable() { - return errors.New("ACL extension disabled by container basic ACL") - } - - err = cp.verifySignature(signatureVerificationData{ - ownerContainer: cnr.Value.Owner(), - verb: session.VerbContainerSetEACL, - idContainerSet: true, - idContainer: idCnr, - binTokenSession: e.SessionToken(), - binPublicKey: e.PublicKey(), - signature: e.Signature(), - signedData: binTable, - }) - if err != nil { - return fmt.Errorf("auth eACL table setting: %w", err) - } - - return nil -} diff --git a/pkg/morph/client/container/eacl_set.go b/pkg/morph/client/container/eacl_set.go deleted file mode 100644 index c68104593..000000000 --- a/pkg/morph/client/container/eacl_set.go +++ /dev/null @@ -1,93 +0,0 @@ -package container - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" -) - -// PutEACL marshals table, and passes it to Wrapper's PutEACLBinary method -// along with sig.Key() and sig.Sign(). -// -// Returns error if table is nil. -// -// If TryNotary is provided, calls notary contract. -func PutEACL(c *Client, eaclInfo containercore.EACL) error { - if eaclInfo.Value == nil { - return errNilArgument - } - - data, err := eaclInfo.Value.Marshal() - if err != nil { - return fmt.Errorf("can't marshal eacl table: %w", err) - } - - var prm PutEACLPrm - prm.SetTable(data) - - if eaclInfo.Session != nil { - prm.SetToken(eaclInfo.Session.Marshal()) - } - - // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion - var sigV2 refs.Signature - eaclInfo.Signature.WriteToV2(&sigV2) - - prm.SetKey(sigV2.GetKey()) - prm.SetSignature(sigV2.GetSign()) - - return c.PutEACL(prm) -} - -// PutEACLPrm groups parameters of PutEACL operation. -type PutEACLPrm struct { - table []byte - key []byte - sig []byte - token []byte - - client.InvokePrmOptional -} - -// SetTable sets table. -func (p *PutEACLPrm) SetTable(table []byte) { - p.table = table -} - -// SetKey sets key. -func (p *PutEACLPrm) SetKey(key []byte) { - p.key = key -} - -// SetSignature sets signature. -func (p *PutEACLPrm) SetSignature(sig []byte) { - p.sig = sig -} - -// SetToken sets session token. -func (p *PutEACLPrm) SetToken(token []byte) { - p.token = token -} - -// PutEACL saves binary eACL table with its session token, key and signature -// in FrostFS system through Container contract call. -// -// Returns any error encountered that caused the saving to interrupt. -func (c *Client) PutEACL(p PutEACLPrm) error { - if len(p.sig) == 0 || len(p.key) == 0 { - return errNilArgument - } - - prm := client.InvokePrm{} - prm.SetMethod(setEACLMethod) - prm.SetArgs(p.table, p.sig, p.key, p.token) - prm.InvokePrmOptional = p.InvokePrmOptional - - _, err := c.client.Invoke(prm) - if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", setEACLMethod, err) - } - return nil -} diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index a653fa066..e2e79f3d2 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -11,9 +11,10 @@ import ( containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" containerSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var errMissingUserID = errors.New("missing user ID") @@ -40,8 +41,6 @@ type Writer interface { Put(containercore.Container) (*cid.ID, error) // Delete removes specified container from the side chain. Delete(containercore.RemovalWitness) error - // PutEACL updates extended ACL table of specified container in the side chain. - PutEACL(containercore.EACL) error } func NewExecutor(rdr Reader, wrt Writer) containerSvc.ServiceExecutor { @@ -205,37 +204,8 @@ func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) return res, nil } -func (s *morphExecutor) SetExtendedACL(_ context.Context, tokV2 *sessionV2.Token, body *container.SetExtendedACLRequestBody) (*container.SetExtendedACLResponseBody, error) { - sigV2 := body.GetSignature() - if sigV2 == nil { - // TODO(@cthulhu-rider): #468 use "const" error - return nil, errors.New("missing signature") - } - - eaclInfo := containercore.EACL{ - Value: eaclSDK.NewTableFromV2(body.GetEACL()), - } - - err := eaclInfo.Signature.ReadFromV2(*sigV2) - if err != nil { - return nil, fmt.Errorf("can't read signature: %w", err) - } - - if tokV2 != nil { - eaclInfo.Session = new(session.Container) - - err := eaclInfo.Session.ReadFromV2(*tokV2) - if err != nil { - return nil, fmt.Errorf("invalid session token: %w", err) - } - } - - err = s.wrt.PutEACL(eaclInfo) - if err != nil { - return nil, err - } - - return new(container.SetExtendedACLResponseBody), nil +func (s *morphExecutor) SetExtendedACL(_ context.Context, _ *sessionV2.Token, _ *container.SetExtendedACLRequestBody) (*container.SetExtendedACLResponseBody, error) { + return nil, status.Errorf(codes.Unimplemented, "method SetExtendedACL not implemented") } func (s *morphExecutor) GetExtendedACL(_ context.Context, body *container.GetExtendedACLRequestBody) (*container.GetExtendedACLResponseBody, error) { diff --git a/pkg/services/container/morph/executor_test.go b/pkg/services/container/morph/executor_test.go index 42dd0b77e..560c69232 100644 --- a/pkg/services/container/morph/executor_test.go +++ b/pkg/services/container/morph/executor_test.go @@ -96,17 +96,6 @@ func TestInvalidToken(t *testing.T) { return }, }, - { - name: "setEACL", - op: func(e containerSvc.ServiceExecutor, tokV2 *session.Token) (err error) { - var reqBody container.SetExtendedACLRequestBody - reqBody.SetSignature(new(refs.Signature)) - sign(&reqBody) - - _, err = e.SetExtendedACL(context.TODO(), tokV2, &reqBody) - return - }, - }, } for _, test := range tests { From a378ff9cf6c5ded5edc32891d586729a9fdbf05b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 2 Jul 2024 13:02:26 +0300 Subject: [PATCH 1199/1943] [#1218] object: Pass container owner for backward get method check * `getStreamBasicChecker` must define `containerOwner` for backward checks, otherwise bearer token cannot be validated for the token issuer. Signed-off-by: Airat Arifullin --- pkg/services/object/ape/service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 609b117d0..853c3b80d 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -164,6 +164,7 @@ func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectSt apeChecker: c.apeChecker, namespace: reqCtx.Namespace, senderKey: reqCtx.SenderKey, + containerOwner: reqCtx.ContainerOwner, role: nativeSchemaRole(reqCtx.Role), softAPECheck: reqCtx.SoftAPECheck, bearerToken: reqCtx.BearerToken, From f3a861806e135eff2b317cffa7105a6769f55b3e Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 2 Jul 2024 13:08:59 +0300 Subject: [PATCH 1200/1943] [#1218] object: Fix bearer token validation Signed-off-by: Airat Arifullin --- pkg/services/object/ape/checker.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index ee12d7b97..7541ad116 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -97,22 +97,23 @@ func isValidBearer(token *bearer.Token, ownerCnr user.ID, containerID cid.ID, pu return nil } - // 1. First check token lifetime. Simplest verification. + // First check token lifetime. Simplest verification. if token.InvalidAt(st.CurrentEpoch()) { return errBearerExpired } - // 2. Then check if bearer token is signed correctly. + // Then check if bearer token is signed correctly. if !token.VerifySignature() { return errBearerInvalidSignature } - // 3. Then check if container is either empty or equal to the container in the request. + // Check for ape overrides defined in the bearer token. apeOverride := token.APEOverride() - if apeOverride.Target.TargetType != ape.TargetTypeContainer { - return errInvalidTargetType + if len(apeOverride.Chains) > 0 && apeOverride.Target.TargetType != ape.TargetTypeContainer { + return fmt.Errorf("%w: %s", errInvalidTargetType, apeOverride.Target.TargetType.ToV2().String()) } + // Then check if container is either empty or equal to the container in the request. var targetCnr cid.ID err := targetCnr.DecodeString(apeOverride.Target.Name) if err != nil { @@ -122,12 +123,12 @@ func isValidBearer(token *bearer.Token, ownerCnr user.ID, containerID cid.ID, pu return errBearerInvalidContainerID } - // 4. Then check if container owner signed this token. + // Then check if container owner signed this token. if !bearer.ResolveIssuer(*token).Equals(ownerCnr) { return errBearerNotSignedByOwner } - // 5. Then check if request sender has rights to use this token. + // Then check if request sender has rights to use this token. var usrSender user.ID user.IDFromKey(&usrSender, (ecdsa.PublicKey)(*publicKey)) From 4bd466779112cb7bf215d1ae5bfbe2daf9502eef Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 2 Jul 2024 13:10:32 +0300 Subject: [PATCH 1201/1943] [#1218] tree: Fix bearer token validation Signed-off-by: Airat Arifullin --- pkg/services/tree/ape.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 116adf5db..a6202d1a3 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -85,22 +85,23 @@ func isValidBearer(token *bearer.Token, ownerCnr user.ID, cntID cid.ID, publicKe return nil } - // 1. First check token lifetime. Simplest verification. + // First check token lifetime. Simplest verification. if token.InvalidAt(st.CurrentEpoch()) { return errBearerExpired } - // 2. Then check if bearer token is signed correctly. + // Then check if bearer token is signed correctly. if !token.VerifySignature() { return errBearerInvalidSignature } - // 3. Then check if container is either empty or equal to the container in the request. + // Check for ape overrides defined in the bearer token. apeOverride := token.APEOverride() - if apeOverride.Target.TargetType != ape.TargetTypeContainer { - return errInvalidTargetType + if len(apeOverride.Chains) > 0 && apeOverride.Target.TargetType != ape.TargetTypeContainer { + return fmt.Errorf("%w: %s", errInvalidTargetType, apeOverride.Target.TargetType.ToV2().String()) } + // Then check if container is either empty or equal to the container in the request. var targetCnr cid.ID err := targetCnr.DecodeString(apeOverride.Target.Name) if err != nil { @@ -110,12 +111,12 @@ func isValidBearer(token *bearer.Token, ownerCnr user.ID, cntID cid.ID, publicKe return errBearerInvalidContainerID } - // 4. Then check if container owner signed this token. + // Then check if container owner signed this token. if !bearer.ResolveIssuer(*token).Equals(ownerCnr) { return errBearerNotSignedByOwner } - // 5. Then check if request sender has rights to use this token. + // Then check if request sender has rights to use this token. var usrSender user.ID user.IDFromKey(&usrSender, (ecdsa.PublicKey)(*publicKey)) From 80d7459560e85a256081ba09b44aa690344482e3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 3 Jul 2024 15:31:39 +0300 Subject: [PATCH 1202/1943] Revert "[#1196] morph/client: Remove duplicate utility functions" This reverts commit 259007540fc8ca0899a3cd4598f9e2e33a442e01. Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 +- pkg/morph/client/frostfsid/subject.go | 202 +++++++++++++++++++++++++- 3 files changed, 203 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index d4eb52f4d..38e66a20f 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 - git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e + git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de diff --git a/go.sum b/go.sum index 84fdbd47a..0849b74a1 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 h1:H5GvrVlowIMWfzqQkhY0p0myooJxQ1sMRVSFfXawwWg= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 h1:Tp4I+XOLp3VCJORfxSamQtj3RZNISbaLM4WD5iIzXxg= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= diff --git a/pkg/morph/client/frostfsid/subject.go b/pkg/morph/client/frostfsid/subject.go index 50733d89a..169be6bc5 100644 --- a/pkg/morph/client/frostfsid/subject.go +++ b/pkg/morph/client/frostfsid/subject.go @@ -1,11 +1,16 @@ package frostfsid import ( + "errors" "fmt" frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" + "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" ) const ( @@ -23,7 +28,7 @@ func (c *Client) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) return nil, fmt.Errorf("could not perform test invocation (%s): %w", methodGetSubject, err) } - subj, err := frostfsidclient.ParseSubject(res) + subj, err := parseSubject(res) if err != nil { return nil, fmt.Errorf("could not parse test invocation result (%s): %w", methodGetSubject, err) } @@ -41,10 +46,203 @@ func (c *Client) GetSubjectExtended(addr util.Uint160) (*frostfsidclient.Subject return nil, fmt.Errorf("could not perform test invocation (%s): %w", methodGetSubject, err) } - subj, err := frostfsidclient.ParseSubjectExtended(res) + subj, err := parseSubjectExtended(res) if err != nil { return nil, fmt.Errorf("could not parse test invocation result (%s): %w", methodGetSubject, err) } return subj, nil } + +func parseSubject(res []stackitem.Item) (*frostfsidclient.Subject, error) { + if ln := len(res); ln != 1 { + return nil, fmt.Errorf("unexpected stack item count (%s): %d", methodGetSubject, ln) + } + + structArr, err := client.ArrayFromStackItem(res[0]) + if err != nil { + return nil, fmt.Errorf("could not get item array of container (%s): %w", methodGetSubject, err) + } + + var subj frostfsidclient.Subject + + subj.PrimaryKey, err = unwrap.PublicKey(makeValidRes(structArr[0])) + if err != nil { + return nil, err + } + + if !structArr[1].Equals(stackitem.Null{}) { + subj.AdditionalKeys, err = unwrap.ArrayOfPublicKeys(makeValidRes(structArr[1])) + if err != nil { + return nil, err + } + } + + if !structArr[2].Equals(stackitem.Null{}) { + subj.Namespace, err = unwrap.UTF8String(makeValidRes(structArr[2])) + if err != nil { + return nil, err + } + } + + if !structArr[3].Equals(stackitem.Null{}) { + subj.Name, err = unwrap.UTF8String(makeValidRes(structArr[3])) + if err != nil { + return nil, err + } + } + + subj.KV, err = parseMap(structArr[4]) + if err != nil { + return nil, err + } + + return &subj, nil +} + +func parseSubjectExtended(res []stackitem.Item) (*frostfsidclient.SubjectExtended, error) { + if ln := len(res); ln != 1 { + return nil, fmt.Errorf("unexpected stack item count (%s): %d", methodGetSubject, ln) + } + + structArr, err := client.ArrayFromStackItem(res[0]) + if err != nil { + return nil, fmt.Errorf("could not get item array of container (%s): %w", methodGetSubject, err) + } + + var subj frostfsidclient.SubjectExtended + + subj.PrimaryKey, err = unwrap.PublicKey(makeValidRes(structArr[0])) + if err != nil { + return nil, err + } + + if !structArr[1].Equals(stackitem.Null{}) { + subj.AdditionalKeys, err = unwrap.ArrayOfPublicKeys(makeValidRes(structArr[1])) + if err != nil { + return nil, err + } + } + + if !structArr[2].Equals(stackitem.Null{}) { + subj.Namespace, err = unwrap.UTF8String(makeValidRes(structArr[2])) + if err != nil { + return nil, err + } + } + + if !structArr[3].Equals(stackitem.Null{}) { + subj.Name, err = unwrap.UTF8String(makeValidRes(structArr[3])) + if err != nil { + return nil, err + } + } + + subj.KV, err = parseMap(structArr[4]) + if err != nil { + return nil, err + } + + if !structArr[5].Equals(stackitem.Null{}) { + groupItems, ok := structArr[5].Value().([]stackitem.Item) + if !ok { + return nil, errors.New("invalid groups field") + } + + subj.Groups, err = parseGroups(groupItems) + if err != nil { + return nil, err + } + } + + return &subj, nil +} + +func makeValidRes(item stackitem.Item) (*result.Invoke, error) { + return &result.Invoke{ + Stack: []stackitem.Item{item}, + State: vmstate.Halt.String(), + }, nil +} + +func parseMap(item stackitem.Item) (map[string]string, error) { + if item.Equals(stackitem.Null{}) { + return nil, nil + } + + metaMap, err := unwrap.Map(makeValidRes(item)) + if err != nil { + return nil, err + } + + meta, ok := metaMap.Value().([]stackitem.MapElement) + if !ok { + return nil, errors.New("invalid map type") + } + + res := make(map[string]string, len(meta)) + for _, element := range meta { + key, err := element.Key.TryBytes() + if err != nil { + return nil, err + } + val, err := element.Value.TryBytes() + if err != nil { + return nil, err + } + res[string(key)] = string(val) + } + + return res, nil +} + +func parseGroups(items []stackitem.Item) ([]*frostfsidclient.Group, error) { + var err error + res := make([]*frostfsidclient.Group, len(items)) + + for i := 0; i < len(items); i++ { + arr, ok := items[i].Value().([]stackitem.Item) + if !ok { + return nil, errors.New("invalid group type") + } + res[i], err = parseGroup(arr) + if err != nil { + return nil, err + } + } + + return res, nil +} + +func parseGroup(structArr []stackitem.Item) (*frostfsidclient.Group, error) { + if len(structArr) < 4 { + return nil, errors.New("invalid response group struct") + } + + groupID, err := structArr[0].TryInteger() + if err != nil { + return nil, err + } + + name, err := structArr[1].TryBytes() + if err != nil { + return nil, err + } + + namespace, err := structArr[2].TryBytes() + if err != nil { + return nil, err + } + + kvs, err := parseMap(structArr[3]) + if err != nil { + return nil, err + } + + return &frostfsidclient.Group{ + ID: groupID.Int64(), + Name: string(name), + Namespace: string(namespace), + KV: kvs, + }, nil +} From bbe95dac8b37fd11eb474acfc60514c2f5b350aa Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 3 Jul 2024 17:14:47 +0300 Subject: [PATCH 1203/1943] [#1225] engine: Log the error when check object existence Signed-off-by: Anton Nikiforov --- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/put.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 2addc4482..4f0b29124 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -205,6 +205,7 @@ const ( EngineCouldNotReloadAShard = "could not reload a shard" EngineAddedNewShard = "added new shard" EngineCouldNotPutObjectToShard = "could not put object to shard" + EngineCouldNotCheckObjectExistence = "could not check object existence when put object to shard" EngineErrorDuringSearchingForObjectChildren = "error during searching for object children" EngineCouldNotInhumeObjectInShard = "could not inhume object in shard" EngineStartingRemovalOfLocallyredundantCopies = "starting removal of locally-redundant copies" diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 2a78febed..54385910b 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -149,6 +149,11 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti // object is already found but // expired => do nothing with it res.status = putToShardExists + } else { + e.log.Warn(logs.EngineCouldNotCheckObjectExistence, + zap.Stringer("shard_id", sh.ID()), + zap.String("error", err.Error()), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return // this is not ErrAlreadyRemoved error so we can go to the next shard From 2bac82cd6fd378b1958e1e9633b5ef98d2ff6553 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 3 Jul 2024 09:55:04 +0300 Subject: [PATCH 1204/1943] [#1222] engine: Fix object evacuation Do not fail evacuation if it unable to evacuate object to other node. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 14 +++++++++--- .../engine/evacuate_test.go | 22 +++++++++---------- pkg/services/control/server/evacuate.go | 14 +++++++----- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 87542eec1..0ce59a0be 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -74,7 +74,7 @@ func (s EvacuateScope) TreesOnly() bool { // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { ShardID []*shard.ID - ObjectsHandler func(context.Context, oid.Address, *objectSDK.Object) error + ObjectsHandler func(context.Context, oid.Address, *objectSDK.Object) (bool, error) TreeHandler func(context.Context, cid.ID, string, pilorama.Forest) (string, error) IgnoreErrors bool Async bool @@ -668,13 +668,21 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to return fmt.Errorf("%w: %s", errPutShard, toEvacuate[i]) } - err = prm.ObjectsHandler(ctx, addr, getRes.Object()) + moved, err := prm.ObjectsHandler(ctx, addr, getRes.Object()) if err != nil { e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } - res.objEvacuated.Add(1) + if moved { + res.objEvacuated.Add(1) + } else if prm.IgnoreErrors { + res.objFailed.Add(1) + e.log.Warn(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } else { + return fmt.Errorf("object %s was not replicated", addr) + } } return nil } diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index e8d9a449f..f097665bf 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -145,18 +145,18 @@ func TestEvacuateObjectsNetwork(t *testing.T) { errReplication := errors.New("handler error") - acceptOneOf := func(objects []*objectSDK.Object, max uint64) func(context.Context, oid.Address, *objectSDK.Object) error { + acceptOneOf := func(objects []*objectSDK.Object, max uint64) func(context.Context, oid.Address, *objectSDK.Object) (bool, error) { var n uint64 - return func(_ context.Context, addr oid.Address, obj *objectSDK.Object) error { + return func(_ context.Context, addr oid.Address, obj *objectSDK.Object) (bool, error) { if n == max { - return errReplication + return false, errReplication } n++ for i := range objects { if addr == objectCore.AddressOf(objects[i]) { require.Equal(t, objects[i], obj) - return nil + return true, nil } } require.FailNow(t, "handler was called with an unexpected object: %s", addr) @@ -268,13 +268,13 @@ func TestEvacuateCancellation(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] - prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) (bool, error) { select { case <-ctx.Done(): - return ctx.Err() + return false, ctx.Err() default: } - return nil + return true, nil } prm.Scope = EvacuateScopeObjects @@ -301,14 +301,14 @@ func TestEvacuateSingleProcess(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] prm.Scope = EvacuateScopeObjects - prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) (bool, error) { select { case <-running: default: close(running) } <-blocker - return nil + return true, nil } eg, egCtx := errgroup.WithContext(context.Background()) @@ -344,14 +344,14 @@ func TestEvacuateObjectsAsync(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] prm.Scope = EvacuateScopeObjects - prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) (bool, error) { select { case <-running: default: close(running) } <-blocker - return nil + return true, nil } st, err := e.GetEvacuationState(context.Background()) diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index 794bc199a..dd609caec 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -57,17 +57,21 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe return resp, nil } -func (s *Server) replicateObject(ctx context.Context, addr oid.Address, obj *objectSDK.Object) error { +func (s *Server) replicateObject(ctx context.Context, addr oid.Address, obj *objectSDK.Object) (bool, error) { cid, ok := obj.ContainerID() if !ok { // Return nil to prevent situations where a shard can't be evacuated // because of a single bad/corrupted object. - return nil + return false, nil } nodes, err := s.getContainerNodes(cid) if err != nil { - return err + return false, err + } + + if len(nodes) == 0 { + return false, nil } var res replicatorResult @@ -80,9 +84,9 @@ func (s *Server) replicateObject(ctx context.Context, addr oid.Address, obj *obj s.replicator.HandleReplicationTask(ctx, task, &res) if res.count == 0 { - return errors.New("object was not replicated") + return false, errors.New("object was not replicated") } - return nil + return true, nil } func (s *Server) replicateTree(ctx context.Context, contID cid.ID, treeID string, forest pilorama.Forest) (string, error) { From 3a797e46826e82575bb572192a35fe2afc439813 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 3 Jul 2024 11:47:50 +0300 Subject: [PATCH 1205/1943] [#1222] engine: Fix tree evacuation Do not fail if it is unable to evacuate tree to other node. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 32 +++++++++++++------ .../engine/evacuate_test.go | 4 +-- pkg/services/control/server/evacuate.go | 10 +++--- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 0ce59a0be..04e427e49 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -75,7 +75,7 @@ func (s EvacuateScope) TreesOnly() bool { type EvacuateShardPrm struct { ShardID []*shard.ID ObjectsHandler func(context.Context, oid.Address, *objectSDK.Object) (bool, error) - TreeHandler func(context.Context, cid.ID, string, pilorama.Forest) (string, error) + TreeHandler func(context.Context, cid.ID, string, pilorama.Forest) (bool, string, error) IgnoreErrors bool Async bool Scope EvacuateScope @@ -450,7 +450,7 @@ func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, tree continue } - nodePK, err := e.evacuateTreeToOtherNode(ctx, sh, contTree, prm) + moved, nodePK, err := e.evacuateTreeToOtherNode(ctx, sh, contTree, prm) if err != nil { e.log.Error(logs.EngineShardsEvacuationFailedToMoveTree, zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), @@ -458,18 +458,32 @@ func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, tree zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } - e.log.Debug(logs.EngineShardsEvacuationTreeEvacuatedRemote, - zap.String("cid", contTree.CID.EncodeToString()), zap.String("treeID", contTree.TreeID), - zap.String("from_shardID", sh.ID().String()), zap.String("to_node", nodePK), - evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) - res.trEvacuated.Add(1) + if moved { + e.log.Debug(logs.EngineShardsEvacuationTreeEvacuatedRemote, + zap.String("cid", contTree.CID.EncodeToString()), zap.String("treeID", contTree.TreeID), + zap.String("from_shardID", sh.ID().String()), zap.String("to_node", nodePK), + evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + res.trEvacuated.Add(1) + } else if prm.IgnoreErrors { + res.trFailed.Add(1) + e.log.Warn(logs.EngineShardsEvacuationFailedToMoveTree, + zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), + zap.String("from_shard_id", sh.ID().String()), evacuationOperationLogField, + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } else { + e.log.Error(logs.EngineShardsEvacuationFailedToMoveTree, + zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), + zap.String("from_shard_id", sh.ID().String()), evacuationOperationLogField, + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + return fmt.Errorf("no remote nodes available to replicate tree '%s' of container %s", contTree.TreeID, contTree.CID) + } } return nil } -func (e *StorageEngine) evacuateTreeToOtherNode(ctx context.Context, sh *shard.Shard, tree pilorama.ContainerIDTreeID, prm EvacuateShardPrm) (string, error) { +func (e *StorageEngine) evacuateTreeToOtherNode(ctx context.Context, sh *shard.Shard, tree pilorama.ContainerIDTreeID, prm EvacuateShardPrm) (bool, string, error) { if prm.TreeHandler == nil { - return "", fmt.Errorf("failed to evacuate tree '%s' for container %s from shard %s: local evacuation failed, but no remote evacuation available", tree.TreeID, tree.CID, sh.ID()) + return false, "", fmt.Errorf("failed to evacuate tree '%s' for container %s from shard %s: local evacuation failed, but no remote evacuation available", tree.TreeID, tree.CID, sh.ID()) } return prm.TreeHandler(ctx, tree.CID, tree.TreeID, sh) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index f097665bf..55268b549 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -507,7 +507,7 @@ func TestEvacuateTreesRemote(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids prm.Scope = EvacuateScopeTrees - prm.TreeHandler = func(ctx context.Context, contID cid.ID, treeID string, f pilorama.Forest) (string, error) { + prm.TreeHandler = func(ctx context.Context, contID cid.ID, treeID string, f pilorama.Forest) (bool, string, error) { key := contID.String() + treeID var height uint64 for { @@ -515,7 +515,7 @@ func TestEvacuateTreesRemote(t *testing.T) { require.NoError(t, err) if op.Time == 0 { - return "", nil + return true, "", nil } evacuatedTreeOps[key] = append(evacuatedTreeOps[key], &op) height = op.Time + 1 diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index dd609caec..0ba8be765 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -89,22 +89,22 @@ func (s *Server) replicateObject(ctx context.Context, addr oid.Address, obj *obj return true, nil } -func (s *Server) replicateTree(ctx context.Context, contID cid.ID, treeID string, forest pilorama.Forest) (string, error) { +func (s *Server) replicateTree(ctx context.Context, contID cid.ID, treeID string, forest pilorama.Forest) (bool, string, error) { nodes, err := s.getContainerNodes(contID) if err != nil { - return "", err + return false, "", err } if len(nodes) == 0 { - return "", fmt.Errorf("no remote nodes available to replicate tree '%s' of container %s", treeID, contID) + return false, "", nil } for _, node := range nodes { err = s.replicateTreeToNode(ctx, forest, contID, treeID, node) if err == nil { - return hex.EncodeToString(node.PublicKey()), nil + return true, hex.EncodeToString(node.PublicKey()), nil } } - return "", err + return false, "", err } func (s *Server) replicateTreeToNode(ctx context.Context, forest pilorama.Forest, contID cid.ID, treeID string, node netmap.NodeInfo) error { From 40c9ddb6bac36e31189ced1e97f178192180310b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 3 Jul 2024 16:45:30 +0300 Subject: [PATCH 1206/1943] [#1226] blobovniczatree: Drop init in advance option To make blobovniczatree unlimited. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 -- cmd/frostfs-node/config/engine/config_test.go | 1 - .../shard/blobstor/blobovnicza/config.go | 10 ----- config/example/node.env | 1 - config/example/node.json | 1 - config/example/node.yaml | 1 - docs/storage-node-configuration.md | 1 - .../blobstor/blobovniczatree/control.go | 25 ----------- .../blobstor/blobovniczatree/iterate_test.go | 44 ------------------- .../blobstor/blobovniczatree/option.go | 8 ---- 10 files changed, 95 deletions(-) delete mode 100644 pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 3aded7020..b51073d4e 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -191,7 +191,6 @@ type subStorageCfg struct { leafWidth uint64 openedCacheSize int initWorkerCount int - initInAdvance bool rebuildDropTimeout time.Duration openedCacheTTL time.Duration openedCacheExpInterval time.Duration @@ -319,7 +318,6 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.openedCacheTTL = sub.OpenedCacheTTL() sCfg.openedCacheExpInterval = sub.OpenedCacheExpInterval() sCfg.initWorkerCount = sub.InitWorkerCount() - sCfg.initInAdvance = sub.InitInAdvance() sCfg.rebuildDropTimeout = sub.RebuildDropTimeout() case fstree.Type: sub := fstreeconfig.From((*config.Config)(storagesCfg[i])) @@ -911,7 +909,6 @@ func (c *cfg) getSubstorageOpts(ctx context.Context, shCfg shardCfg) []blobstor. blobovniczatree.WithOpenedCacheTTL(sRead.openedCacheTTL), blobovniczatree.WithOpenedCacheExpInterval(sRead.openedCacheExpInterval), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), - blobovniczatree.WithInitInAdvance(sRead.initInAdvance), blobovniczatree.WithWaitBeforeDropDB(sRead.rebuildDropTimeout), blobovniczatree.WithLogger(c.log), blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 3f9c7ec71..8e95b97b0 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -102,7 +102,6 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 30*time.Second, blz.OpenedCacheExpInterval()) require.EqualValues(t, 10, blz.LeafWidth()) require.EqualValues(t, 10, blz.InitWorkerCount()) - require.EqualValues(t, true, blz.InitInAdvance()) require.EqualValues(t, 30*time.Second, blz.RebuildDropTimeout()) require.Equal(t, "tmp/0/blob", ss[1].Path()) diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index c8ea4c2cb..9feee4f64 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -175,16 +175,6 @@ func (x *Config) InitWorkerCount() int { return InitWorkerCountDefault } -// InitInAdvance returns the value of "init_in_advance" config parameter. -// -// Returns False if the value is not defined or invalid. -func (x *Config) InitInAdvance() bool { - return config.BoolSafe( - (*config.Config)(x), - "init_in_advance", - ) -} - // RebuildDropTimeout returns the value of "rebuild_drop_timeout" config parameter. // // Returns RebuildDropTimeoutDefault if the value is not defined or invalid. diff --git a/config/example/node.env b/config/example/node.env index 72f56e96c..ca4351363 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -128,7 +128,6 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=1m FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_EXP_INTERVAL=30s FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 -FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_REBUILD_DROP_TIMEOUT=30s ### FSTree config FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree diff --git a/config/example/node.json b/config/example/node.json index b9dc6014c..7b502b6bd 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -176,7 +176,6 @@ "opened_cache_exp_interval": "30s", "leaf_width": 10, "init_worker_count": 10, - "init_in_advance": true, "rebuild_drop_timeout": "30s" }, { diff --git a/config/example/node.yaml b/config/example/node.yaml index bad67816a..c02f9d37b 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -189,7 +189,6 @@ storage: - type: blobovnicza path: tmp/0/blob/blobovnicza init_worker_count: 10 #count of workers to initialize blobovniczas - init_in_advance: true rebuild_drop_timeout: 30s # timeout before drop single blobovnicza opened_cache_ttl: 1m opened_cache_exp_interval: 30s diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 5389bfbb5..4a6e5ba6d 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -242,7 +242,6 @@ blobstor: | `opened_cache_ttl` | `duration` | `0` | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. | | `opened_cache_exp_interval` | `duration` | `15s` | Cache cleanup interval for expired blobovnicza's. | | `init_worker_count` | `int` | `5` | Maximum number of concurrent initialization workers. | -| `init_in_advance` | `bool` | `false` | If `true`, than all the blobovnicza files will be created on startup. | | `rebuild_drop_timeout` | `duration` | `10s` | Timeout before drop empty blobovnicza file during rebuild. | ### `gc` subsection diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 67e8c8f18..eeade9425 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -83,31 +83,6 @@ func (b *Blobovniczas) initializeDBs(ctx context.Context) error { return err } - if b.createDBInAdvance { - err = b.iterateSortedLeaves(egCtx, nil, func(p string) (bool, error) { - if _, found := visited[p]; found { - return false, nil - } - eg.Go(func() error { - shBlz := b.getBlobovniczaWithoutCaching(p) - _, err := shBlz.Open() - if err != nil { - return err - } - defer shBlz.Close() - - b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) - return nil - }) - return false, nil - }) - - if err != nil { - _ = eg.Wait() - return err - } - } - return eg.Wait() } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go deleted file mode 100644 index b067a164b..000000000 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package blobovniczatree - -import ( - "context" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - "github.com/stretchr/testify/require" -) - -func TestIterateSortedLeavesAndDBPathsAreSame(t *testing.T) { - t.Parallel() - - blz := NewBlobovniczaTree( - context.Background(), - WithBlobovniczaShallowDepth(3), - WithBlobovniczaShallowWidth(5), - WithRootPath(t.TempDir()), - ) - blz.createDBInAdvance = true - require.NoError(t, blz.Open(mode.ComponentReadWrite)) - require.NoError(t, blz.Init()) - defer func() { - require.NoError(t, blz.Close()) - }() - - addr := oidtest.Address() - - var leaves []string - var dbPaths []string - - blz.iterateSortedLeaves(context.Background(), &addr, func(s string) (bool, error) { - leaves = append(leaves, s) - return false, nil - }) - - blz.iterateSortedDBPaths(context.Background(), addr, func(s string) (bool, error) { - dbPaths = append(dbPaths, s) - return false, nil - }) - - require.Equal(t, leaves, dbPaths) -} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 634499275..1bf7b2207 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -26,7 +26,6 @@ type cfg struct { waitBeforeDropDB time.Duration blzInitWorkerCount int blzMoveBatchSize int - createDBInAdvance bool // TTL for blobovnicza's cache openedCacheTTL time.Duration // Interval for deletion expired blobovnicza's @@ -160,10 +159,3 @@ func WithInitWorkerCount(v int) Option { c.blzInitWorkerCount = v } } - -// WithInitInAdvance returns an option to create blobovnicza tree DB's in advance. -func WithInitInAdvance(v bool) Option { - return func(c *cfg) { - c.createDBInAdvance = v - } -} From 78b1d9b18db2b15a53955d331bfebade29498846 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 4 Jul 2024 09:18:17 +0300 Subject: [PATCH 1207/1943] [#1226] blobovniczatree: Drop leaf width limitation Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 -- cmd/frostfs-node/config/engine/config_test.go | 2 - .../shard/blobstor/blobovnicza/config.go | 10 ----- config/example/node.env | 2 - config/example/node.json | 4 +- config/example/node.yaml | 1 - .../blobstor/blobovniczatree/active.go | 37 ++++++++----------- .../blobstor/blobovniczatree/blobovnicza.go | 30 ++++++++++++--- .../blobstor/blobovniczatree/iterate.go | 9 ++++- .../blobstor/blobovniczatree/option.go | 7 ---- 10 files changed, 50 insertions(+), 55 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index b51073d4e..5b91e7819 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -188,7 +188,6 @@ type subStorageCfg struct { // blobovnicza-specific size uint64 width uint64 - leafWidth uint64 openedCacheSize int initWorkerCount int rebuildDropTimeout time.Duration @@ -313,7 +312,6 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.size = sub.Size() sCfg.depth = sub.ShallowDepth() sCfg.width = sub.ShallowWidth() - sCfg.leafWidth = sub.LeafWidth() sCfg.openedCacheSize = sub.OpenedCacheSize() sCfg.openedCacheTTL = sub.OpenedCacheTTL() sCfg.openedCacheExpInterval = sub.OpenedCacheExpInterval() @@ -904,7 +902,6 @@ func (c *cfg) getSubstorageOpts(ctx context.Context, shCfg shardCfg) []blobstor. blobovniczatree.WithBlobovniczaSize(sRead.size), blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), - blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), blobovniczatree.WithOpenedCacheTTL(sRead.openedCacheTTL), blobovniczatree.WithOpenedCacheExpInterval(sRead.openedCacheExpInterval), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 8e95b97b0..7473afefb 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -100,7 +100,6 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 50, blz.OpenedCacheSize()) require.EqualValues(t, time.Minute, blz.OpenedCacheTTL()) require.EqualValues(t, 30*time.Second, blz.OpenedCacheExpInterval()) - require.EqualValues(t, 10, blz.LeafWidth()) require.EqualValues(t, 10, blz.InitWorkerCount()) require.EqualValues(t, 30*time.Second, blz.RebuildDropTimeout()) @@ -154,7 +153,6 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 50, blz.OpenedCacheSize()) require.EqualValues(t, 5*time.Minute, blz.OpenedCacheTTL()) require.EqualValues(t, 15*time.Second, blz.OpenedCacheExpInterval()) - require.EqualValues(t, 10, blz.LeafWidth()) require.EqualValues(t, blobovniczaconfig.InitWorkerCountDefault, blz.InitWorkerCount()) require.EqualValues(t, blobovniczaconfig.RebuildDropTimeoutDefault, blz.RebuildDropTimeout()) diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index 9feee4f64..ac69c4c4f 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -149,16 +149,6 @@ func (x *Config) BoltDB() *boltdbconfig.Config { return (*boltdbconfig.Config)(x) } -// LeafWidth returns the value of "leaf_width" config parameter. -// -// Returns 0 if the value is not a positive number. -func (x *Config) LeafWidth() uint64 { - return config.UintSafe( - (*config.Config)(x), - "leaf_width", - ) -} - // InitWorkerCount returns the value of "init_worker_count" config parameter. // // Returns InitWorkerCountDefault if the value is not a positive number. diff --git a/config/example/node.env b/config/example/node.env index ca4351363..00190eb39 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -126,7 +126,6 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=1m FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_EXP_INTERVAL=30s -FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_REBUILD_DROP_TIMEOUT=30s ### FSTree config @@ -177,7 +176,6 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_TTL=5m FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_EXP_INTERVAL=15s -FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10 ### FSTree config FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH=tmp/1/blob diff --git a/config/example/node.json b/config/example/node.json index 7b502b6bd..9051d2bb7 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -174,7 +174,6 @@ "opened_cache_capacity": 50, "opened_cache_ttl": "1m", "opened_cache_exp_interval": "30s", - "leaf_width": 10, "init_worker_count": 10, "rebuild_drop_timeout": "30s" }, @@ -227,8 +226,7 @@ "width": 4, "opened_cache_capacity": 50, "opened_cache_ttl": "5m", - "opened_cache_exp_interval": "15s", - "leaf_width": 10 + "opened_cache_exp_interval": "15s" }, { "type": "fstree", diff --git a/config/example/node.yaml b/config/example/node.yaml index c02f9d37b..bcc8552b3 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -149,7 +149,6 @@ storage: opened_cache_capacity: 50 # maximum number of opened database files opened_cache_ttl: 5m # ttl for opened database file opened_cache_exp_interval: 15s # cache cleanup interval for expired blobovnicza's - leaf_width: 10 # max count of key-value DB on leafs of object tree storage - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) depth: 5 # max depth of object tree storage in FS diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/active.go b/pkg/local_object_storage/blobstor/blobovniczatree/active.go index 0e3497383..603c6abe3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/active.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/active.go @@ -37,17 +37,17 @@ type activeDBManager struct { closed bool dbManager *dbManager - leafWidth uint64 + rootPath string } -func newActiveDBManager(dbManager *dbManager, leafWidth uint64) *activeDBManager { +func newActiveDBManager(dbManager *dbManager, rootPath string) *activeDBManager { return &activeDBManager{ levelToActiveDBGuard: &sync.RWMutex{}, levelToActiveDB: make(map[string]*sharedDB), levelLock: utilSync.NewKeyLocker[string](), dbManager: dbManager, - leafWidth: leafWidth, + rootPath: rootPath, } } @@ -144,30 +144,25 @@ func (m *activeDBManager) updateAndGetActive(lvlPath string) (*activeDB, error) } func (m *activeDBManager) getNextSharedDB(lvlPath string) (*sharedDB, error) { - var idx uint64 - var iterCount uint64 + var nextActiveDBIdx uint64 hasActive, currentIdx := m.hasActiveDB(lvlPath) if hasActive { - idx = (currentIdx + 1) % m.leafWidth - } - - var next *sharedDB - - for iterCount < m.leafWidth { - path := filepath.Join(lvlPath, u64ToHexStringExt(idx)) - shDB := m.dbManager.GetByPath(path) - db, err := shDB.Open() // open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close() + nextActiveDBIdx = currentIdx + 1 + } else { + hasDBs, maxIdx, err := getBlobovniczaMaxIndex(filepath.Join(m.rootPath, lvlPath)) if err != nil { return nil, err } - if db.IsFull() { - shDB.Close() - } else { - next = shDB - break + if hasDBs { + nextActiveDBIdx = maxIdx } - idx = (idx + 1) % m.leafWidth - iterCount++ + } + + path := filepath.Join(lvlPath, u64ToHexStringExt(nextActiveDBIdx)) + next := m.dbManager.GetByPath(path) + _, err := next.Open() // open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close() + if err != nil { + return nil, err } previous, updated := m.replace(lvlPath, next) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 58466f759..952203367 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -3,6 +3,7 @@ package blobovniczatree import ( "context" "errors" + "os" "strconv" "strings" "sync" @@ -81,12 +82,8 @@ func NewBlobovniczaTree(ctx context.Context, opts ...Option) (blz *Blobovniczas) opts[i](&blz.cfg) } - if blz.blzLeafWidth == 0 { - blz.blzLeafWidth = blz.blzShallowWidth - } - blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) - blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) + blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.rootPath) blz.dbCache = newDBCache(ctx, blz.openedCacheSize, blz.openedCacheTTL, blz.openedCacheExpInterval, blz.commondbManager) blz.deleteProtectedObjects = newAddressMap() @@ -124,6 +121,29 @@ func u64FromHexString(str string) uint64 { return v } +func getBlobovniczaMaxIndex(directory string) (bool, uint64, error) { + entries, err := os.ReadDir(directory) + if os.IsNotExist(err) { // non initialized tree + return false, 0, nil + } + if err != nil { + return false, 0, err + } + if len(entries) == 0 { + return false, 0, nil + } + var hasDBs bool + var maxIdx uint64 + for _, e := range entries { + if e.IsDir() { + continue + } + hasDBs = true + maxIdx = max(u64FromHexString(e.Name()), maxIdx) + } + return hasDBs, maxIdx, nil +} + // Type is blobovniczatree storage type used in logs and configuration. const Type = "blobovnicza" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 92014fd55..af3d9e720 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -130,7 +130,14 @@ func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, cur isLeafLevel := uint64(len(curPath)) == b.blzShallowDepth levelWidth := b.blzShallowWidth if isLeafLevel { - levelWidth = b.blzLeafWidth + hasDBs, maxIdx, err := getBlobovniczaMaxIndex(filepath.Join(append([]string{b.rootPath}, curPath...)...)) + if err != nil { + return false, err + } + levelWidth = 0 + if hasDBs { + levelWidth = maxIdx + 1 + } } indices := indexSlice(levelWidth) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 1bf7b2207..008be9543 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -18,7 +18,6 @@ type cfg struct { openedCacheSize int blzShallowDepth uint64 blzShallowWidth uint64 - blzLeafWidth uint64 compression *compression.Config blzOpts []blobovnicza.Option reportError func(string, error) // reportError is the function called when encountering disk errors. @@ -82,12 +81,6 @@ func WithBlobovniczaShallowWidth(width uint64) Option { } } -func WithBlobovniczaLeafWidth(w uint64) Option { - return func(c *cfg) { - c.blzLeafWidth = w - } -} - func WithBlobovniczaShallowDepth(depth uint64) Option { return func(c *cfg) { c.blzShallowDepth = depth From 62cbb72a5edd045d7d037402fe68b98a615d7abe Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 4 Jul 2024 10:11:30 +0300 Subject: [PATCH 1208/1943] [#1226] blobovniczatree: Delete fix db extensions in Init() Since several releases have been released, this code is not relevant. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 6 -- .../blobstor/blobovniczatree/control.go | 47 -------------- .../blobstor/blobovniczatree/control_test.go | 63 ------------------- 3 files changed, 116 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 4f0b29124..ae8834517 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -480,12 +480,6 @@ const ( BlobstoreRebuildStarted = "blobstore rebuild started" BlobstoreRebuildCompletedSuccessfully = "blobstore rebuild completed successfully" BlobstoreRebuildStopped = "blobstore rebuild stopped" - BlobovniczaTreeFixingFileExtensions = "fixing blobovnicza tree file extensions..." - BlobovniczaTreeFixingFileExtensionsCompletedSuccessfully = "fixing blobovnicza tree file extensions completed successfully" - BlobovniczaTreeFixingFileExtensionsFailed = "failed to fix blobovnicza tree file extensions" - BlobovniczaTreeFixingFileExtensionForFile = "fixing blobovnicza file extension..." - BlobovniczaTreeFixingFileExtensionCompletedSuccessfully = "fixing blobovnicza file extension completed successfully" - BlobovniczaTreeFixingFileExtensionFailed = "failed to fix blobovnicza file extension" BlobstorRebuildFailedToRebuildStorages = "failed to rebuild storages" BlobstorRebuildRebuildStoragesCompleted = "storages rebuild completed" BlobovniczaTreeCollectingDBToRebuild = "collecting blobovniczas to rebuild..." diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index eeade9425..a31e9d6cb 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -2,10 +2,6 @@ package blobovniczatree import ( "context" - "errors" - "os" - "path/filepath" - "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -14,8 +10,6 @@ import ( "golang.org/x/sync/errgroup" ) -var errFailedToChangeExtensionReadOnly = errors.New("failed to change blobovnicza extension: read only mode") - // Open opens blobovnicza tree. func (b *Blobovniczas) Open(mode mode.ComponentMode) error { b.readOnly = mode.ReadOnly() @@ -31,13 +25,6 @@ func (b *Blobovniczas) Open(mode mode.ComponentMode) error { func (b *Blobovniczas) Init() error { b.log.Debug(logs.BlobovniczatreeInitializingBlobovniczas) - b.log.Debug(logs.BlobovniczaTreeFixingFileExtensions) - if err := b.addDBExtensionToDBs(b.rootPath, 0); err != nil { - b.log.Error(logs.BlobovniczaTreeFixingFileExtensionsFailed, zap.Error(err)) - return err - } - b.log.Debug(logs.BlobovniczaTreeFixingFileExtensionsCompletedSuccessfully) - if b.readOnly { b.log.Debug(logs.BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization) return nil @@ -111,37 +98,3 @@ func (b *Blobovniczas) getBlobovnicza(p string) *sharedDB { func (b *Blobovniczas) getBlobovniczaWithoutCaching(p string) *sharedDB { return b.commondbManager.GetByPath(p) } - -func (b *Blobovniczas) addDBExtensionToDBs(path string, depth uint64) error { - entries, err := os.ReadDir(path) - if os.IsNotExist(err) && depth == 0 { - return nil - } - - for _, entry := range entries { - if entry.IsDir() { - if err := b.addDBExtensionToDBs(filepath.Join(path, entry.Name()), depth+1); err != nil { - return err - } - continue - } - - if strings.HasSuffix(entry.Name(), dbExtension) { - continue - } - if b.readOnly { - return errFailedToChangeExtensionReadOnly - } - - sourcePath := filepath.Join(path, entry.Name()) - targetPath := filepath.Join(path, entry.Name()+dbExtension) - b.log.Debug(logs.BlobovniczaTreeFixingFileExtensionForFile, zap.String("source", sourcePath), zap.String("target", targetPath)) - if err := os.Rename(sourcePath, targetPath); err != nil { - b.log.Error(logs.BlobovniczaTreeFixingFileExtensionFailed, zap.String("source", sourcePath), zap.String("target", targetPath), zap.Error(err)) - return err - } - b.log.Debug(logs.BlobovniczaTreeFixingFileExtensionCompletedSuccessfully, zap.String("source", sourcePath), zap.String("target", targetPath)) - } - - return nil -} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go index 7218cd6bb..f0a32ded1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go @@ -2,78 +2,15 @@ package blobovniczatree import ( "context" - "os" - "path/filepath" - "strings" "testing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" ) -func TestDBExtensionFix(t *testing.T) { - root := t.TempDir() - createTestTree(t, 0, 2, 3, root) - t.Run("adds suffix if not exists", func(t *testing.T) { - openAndCloseTestTree(t, 2, 3, root) - validateTestTree(t, root) - }) - - t.Run("not adds second suffix if exists", func(t *testing.T) { - openAndCloseTestTree(t, 2, 3, root) - validateTestTree(t, root) - }) -} - -func createTestTree(t *testing.T, currentDepth, depth, width uint64, path string) { - if currentDepth == depth { - var w uint64 - for ; w < width; w++ { - dbPath := filepath.Join(path, u64ToHexString(w)) - b := blobovnicza.New(blobovnicza.WithPath(dbPath)) - require.NoError(t, b.Open()) - require.NoError(t, b.Init()) - require.NoError(t, b.Close()) - } - return - } - - var w uint64 - for ; w < width; w++ { - createTestTree(t, currentDepth+1, depth, width, filepath.Join(path, u64ToHexString(w))) - } -} - -func openAndCloseTestTree(t *testing.T, depth, width uint64, path string) { - blz := NewBlobovniczaTree( - context.Background(), - WithBlobovniczaShallowDepth(depth), - WithBlobovniczaShallowWidth(width), - WithRootPath(path), - ) - require.NoError(t, blz.Open(mode.ComponentReadWrite)) - require.NoError(t, blz.Init()) - require.NoError(t, blz.Close()) -} - -func validateTestTree(t *testing.T, path string) { - entries, err := os.ReadDir(path) - require.NoError(t, err) - - for _, entry := range entries { - if entry.IsDir() { - validateTestTree(t, filepath.Join(path, entry.Name())) - } else { - require.True(t, strings.HasSuffix(entry.Name(), dbExtension)) - require.False(t, strings.HasSuffix(strings.TrimSuffix(entry.Name(), dbExtension), dbExtension)) - } - } -} - func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { t.Parallel() From cfd5e3d40371cb3e26c0637e3773135d7bfd0058 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Thu, 4 Jul 2024 12:09:18 +0300 Subject: [PATCH 1209/1943] [#1227] morph/event: Release ants.Pool on listener stopping Signed-off-by: Denis Kirillov --- pkg/morph/event/listener.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 4fb92da50..dd3c7d216 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -621,6 +621,7 @@ func (l *listener) RegisterNotaryHandler(hi NotaryHandlerInfo) { func (l *listener) Stop() { l.stopOnce.Do(func() { l.subscriber.Close() + l.pool.Release() }) l.wg.Wait() } From 8eb591d668817ee0e438522845bb3e9ff1b574c8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 5 Jul 2024 12:06:28 +0300 Subject: [PATCH 1210/1943] [#1231] policer: Add EC node-off unit test Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/ec_test.go | 145 ++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/pkg/services/policer/ec_test.go b/pkg/services/policer/ec_test.go index 9492787dc..e230153f9 100644 --- a/pkg/services/policer/ec_test.go +++ b/pkg/services/policer/ec_test.go @@ -563,3 +563,148 @@ func TestECChunkRestore(t *testing.T) { require.NoError(t, err) require.EqualValues(t, string(expectedData), string(actualData), "invalid restored objects") } + +func TestECChunkRestoreNodeOff(t *testing.T) { + // node0 has chunk0, node1 has chunk1, node2 has chunk2, node3 is out of netmap + t.Parallel() + + payload := make([]byte, 64) + rand.Read(payload) + parentAddress := oidtest.Address() + parentObject := objectSDK.New() + parentObject.SetContainerID(parentAddress.Container()) + parentObject.SetPayload(payload) + parentObject.SetPayloadSize(64) + objectSDK.CalculateAndSetPayloadChecksum(parentObject) + err := objectSDK.CalculateAndSetID(parentObject) + require.NoError(t, err) + id, _ := parentObject.ID() + parentAddress.SetObject(id) + + chunkIDs := make([]oid.ID, 4) + c, err := erasurecode.NewConstructor(3, 1) + require.NoError(t, err) + key, err := keys.NewPrivateKey() + require.NoError(t, err) + chunks, err := c.Split(parentObject, &key.PrivateKey) + require.NoError(t, err) + for i, ch := range chunks { + chunkIDs[i], _ = ch.ID() + } + + var policy netmapSDK.PlacementPolicy + require.NoError(t, policy.DecodeString("EC 3.1")) + + cnr := &container.Container{} + cnr.Value.Init() + cnr.Value.SetPlacementPolicy(policy) + containerSrc := containerSrc{ + get: func(id cid.ID) (*container.Container, error) { + if id.Equals(parentAddress.Container()) { + return cnr, nil + } + return nil, new(apistatus.ContainerNotFound) + }, + } + + nodes := make([]netmapSDK.NodeInfo, 3) + for i := range nodes { + nodes[i].SetPublicKey([]byte{byte(i)}) + } + + placementBuilder := func(cnr cid.ID, obj *oid.ID, p netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) { + if cnr.Equals(parentAddress.Container()) && obj.Equals(parentAddress.Object()) { + return [][]netmapSDK.NodeInfo{nodes}, nil + } + return nil, errors.New("unexpected placement build") + } + remoteHeadFn := func(_ context.Context, ni netmapSDK.NodeInfo, a oid.Address, raw bool) (*objectSDK.Object, error) { + require.True(t, raw, "remote header for parent object must be called with raw flag") + index := int(ni.PublicKey()[0]) + require.True(t, index == 1 || index == 2, "invalid node to get parent header") + require.True(t, a == parentAddress, "invalid address to get remote header") + if index == 1 { + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(chunkIDs[1]) + ch.Index = uint32(1) + ch.Total = 4 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + if index == 2 { + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(chunkIDs[2]) + ch.Index = uint32(2) + ch.Total = 4 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + + return nil, new(apistatus.ObjectNotFound) + } + + localHeadFn := func(_ context.Context, a oid.Address) (*objectSDK.Object, error) { + require.True(t, a == parentAddress, "invalid address to get remote header") + ei := objectSDK.NewECInfo() + var ch objectSDK.ECChunk + ch.SetID(chunkIDs[0]) + ch.Index = uint32(0) + ch.Total = 4 + ei.AddChunk(ch) + return nil, objectSDK.NewECInfoError(ei) + } + + var replicatedObj []*objectSDK.Object + p := New( + WithContainerSource(containerSrc), + WithPlacementBuilder(placementBuilderFunc(placementBuilder)), + WithNetmapKeys(announcedKeysFunc(func(k []byte) bool { + return bytes.Equal(k, nodes[0].PublicKey()) + })), + WithRemoteObjectHeaderFunc(remoteHeadFn), + WithLocalObjectHeaderFunc(localHeadFn), + WithReplicator(&testReplicator{ + handleLocalPutTask: func(ctx context.Context, task replicator.Task) { + if task.Obj != nil { + replicatedObj = append(replicatedObj, task.Obj) + } + }, + }), + WithLocalObjectGetFunc(func(ctx context.Context, a oid.Address) (*objectSDK.Object, error) { + require.True(t, a.Container() == parentAddress.Container() && a.Object() == chunkIDs[0], "invalid local object request") + return chunks[0], nil + }), + WithRemoteObjectGetFunc(func(ctx context.Context, ni netmapSDK.NodeInfo, a oid.Address) (*objectSDK.Object, error) { + index := ni.PublicKey()[0] + return chunks[index], nil + }), + WithPool(testPool(t)), + WithKeyStorage(util.NewKeyStorage(&key.PrivateKey, nil, nil)), + ) + + var chunkAddress oid.Address + chunkAddress.SetContainer(parentAddress.Container()) + chunkAddress.SetObject(chunkIDs[0]) + objInfo := objectcore.Info{ + Address: chunkAddress, + Type: objectSDK.TypeRegular, + ECInfo: &objectcore.ECInfo{ + ParentID: parentAddress.Object(), + Index: 0, + Total: 4, + }, + } + err = p.processObject(context.Background(), objInfo) + require.NoError(t, err) + + require.Equal(t, 1, len(replicatedObj), "invalid replicated objects count") + chunks[3].SetSignature(nil) + expectedData, err := chunks[3].MarshalJSON() + require.NoError(t, err) + replicatedObj[0].SetSignature(nil) + actualData, err := replicatedObj[0].MarshalJSON() + require.NoError(t, err) + require.EqualValues(t, string(expectedData), string(actualData), "invalid restored objects") +} From 47bcd346d3bdc9fd50e9fc98c8ee485da5b7f2c6 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 1 Jul 2024 16:37:56 +0300 Subject: [PATCH 1211/1943] [#1216] ape: Introduce `BearerChainFeedRouter` * Unlike default chain router, `BearerChainFedRouter` performs checks for overrides defined in the bearer token; * Add unit-test for the introduced router. Signed-off-by: Airat Arifullin --- pkg/ape/router/bearer_overrides.go | 94 ++++++++++++++ pkg/ape/router/bearer_overrides_test.go | 165 ++++++++++++++++++++++++ pkg/ape/router/single_pass.go | 25 ---- 3 files changed, 259 insertions(+), 25 deletions(-) create mode 100644 pkg/ape/router/bearer_overrides.go create mode 100644 pkg/ape/router/bearer_overrides_test.go diff --git a/pkg/ape/router/bearer_overrides.go b/pkg/ape/router/bearer_overrides.go new file mode 100644 index 000000000..2bc8ad614 --- /dev/null +++ b/pkg/ape/router/bearer_overrides.go @@ -0,0 +1,94 @@ +package router + +import ( + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" +) + +func newTarget(ct ape.ChainTarget) (policyengine.Target, error) { + var target policyengine.Target + switch ct.TargetType { + case ape.TargetTypeContainer: + var cid cidSDK.ID + err := cid.DecodeString(ct.Name) + if err != nil { + return target, fmt.Errorf("invalid cid format: %s", target.Name) + } + target.Type = policyengine.Container + case ape.TargetTypeGroup: + target.Type = policyengine.Group + case ape.TargetTypeNamespace: + target.Type = policyengine.Namespace + case ape.TargetTypeUser: + target.Type = policyengine.User + default: + return target, fmt.Errorf("unsupported target type: %v", ct.TargetType) + } + target.Name = ct.Name + return target, nil +} + +type morphReaderDecorator struct { + policyengine.MorphRuleChainStorageReader + + bearerTokenTarget policyengine.Target + + bearerTokenChains []*chain.Chain +} + +func newMorphReaderDecorator(r policyengine.MorphRuleChainStorageReader, override bearer.APEOverride) (*morphReaderDecorator, error) { + if r == nil { + return nil, errors.New("empty morph chain rule reader") + } + t, err := newTarget(override.Target) + if err != nil { + return nil, err + } + + bearerTokenChains := make([]*chain.Chain, len(override.Chains)) + for i := range override.Chains { + chain := new(chain.Chain) + if err := chain.DecodeBytes(override.Chains[i].Raw); err != nil { + return nil, fmt.Errorf("invalid ape chain: %w", err) + } + bearerTokenChains[i] = chain + } + + return &morphReaderDecorator{ + MorphRuleChainStorageReader: r, + bearerTokenTarget: t, + bearerTokenChains: bearerTokenChains, + }, nil +} + +func (m *morphReaderDecorator) ListMorphRuleChains(name chain.Name, target policyengine.Target) ([]*chain.Chain, error) { + if len(m.bearerTokenChains) > 0 && m.bearerTokenTarget.Type == target.Type { + if m.bearerTokenTarget.Name != target.Name { + return nil, fmt.Errorf("unexpected bearer token target: %s", m.bearerTokenTarget.Name) + } + return m.bearerTokenChains, nil + } + return m.MorphRuleChainStorageReader.ListMorphRuleChains(name, target) +} + +// BearerChainFeedRouter creates a chain router emplacing bearer token rule chains. +// Bearer token chains override only container target chains within Policy contract. This means the order of checking +// is as follows: +// +// 1. Local overrides; +// 2. Policy contract chains for a namespace target (as namespace chains have higher priority); +// 3. Bearer token chains for a container target - if they're not defined, then it checks Policy contract chains; +// 4. Checks for the remaining targets. +func BearerChainFeedRouter(localOverrideStorage policyengine.LocalOverrideStorage, morphChainStorage policyengine.MorphRuleChainStorageReader, override bearer.APEOverride) (policyengine.ChainRouter, error) { + mr, err := newMorphReaderDecorator(morphChainStorage, override) + if err != nil { + return nil, fmt.Errorf("create morph reader with bearer override error: %w", err) + } + return policyengine.NewDefaultChainRouterWithLocalOverrides(mr, localOverrideStorage), nil +} diff --git a/pkg/ape/router/bearer_overrides_test.go b/pkg/ape/router/bearer_overrides_test.go new file mode 100644 index 000000000..3c12ee6fa --- /dev/null +++ b/pkg/ape/router/bearer_overrides_test.go @@ -0,0 +1,165 @@ +package router_test + +import ( + "fmt" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/router" + apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + bearerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" + resourcetest "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource/testutil" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/stretchr/testify/require" +) + +const ( + container = "67ETTZzbzJC6WxdQhHHHsJNCttVMBqYrSoFaUFVDNfiX" + rootNs = "" +) + +var ( + allowBySourceIP = &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.Allow, + Actions: chain.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: chain.Resources{Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainer, container)}}, + Condition: []chain.Condition{ + { + Op: chain.CondStringEquals, + Kind: chain.KindRequest, + Key: "SourceIP", + Value: "10.122.1.20", + }, + }, + }, + }, + } + + denyBySourceIP = &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: chain.Resources{Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainer, container)}}, + Condition: []chain.Condition{ + { + Op: chain.CondStringEquals, + Kind: chain.KindRequest, + Key: "SourceIP", + Value: "10.122.1.20", + }, + }, + }, + }, + } +) + +func TestBearerChainFedRouter(t *testing.T) { + t.Run("no bearer token overrides", func(t *testing.T) { + inmem := inmemory.NewInMemoryLocalOverrides() + + inmem.LocalStorage().AddOverride(chain.Ingress, engine.ContainerTarget(container), denyBySourceIP) + inmem.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(container), allowBySourceIP) + + _, err := router.BearerChainFeedRouter(inmem.LocalStorage(), inmem.MorphRuleChainStorage(), bearerSDK.APEOverride{}) + require.Error(t, err) + }) + t.Run("allow by container with deny by bearer overrides", func(t *testing.T) { + inmem := inmemory.NewInMemoryLocalOverrides() + + inmem.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(container), allowBySourceIP) + + bt := bearerSDK.APEOverride{ + Target: apeSDK.ChainTarget{ + TargetType: apeSDK.TargetTypeContainer, + Name: container, + }, + Chains: []apeSDK.Chain{{ + Raw: denyBySourceIP.Bytes(), + }}, + } + + r, err := router.BearerChainFeedRouter(inmem.LocalStorage(), inmem.MorphRuleChainStorage(), bt) + require.NoError(t, err) + + req := resourcetest.NewRequest(nativeschema.MethodPutObject, + resourcetest.NewResource(fmt.Sprintf(nativeschema.ResourceFormatRootContainer, container), map[string]string{}), + map[string]string{ + "SourceIP": "10.122.1.20", + "Actor": "someOwner", + }, + ) + + st, found, err := r.IsAllowed(chain.Ingress, engine.NewRequestTarget(rootNs, container), req) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, st, chain.AccessDenied) + }) + t.Run("allow by namespace with deny by bearer overrides", func(t *testing.T) { + inmem := inmemory.NewInMemoryLocalOverrides() + + inmem.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(container), allowBySourceIP) + inmem.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(rootNs), allowBySourceIP) + + bt := bearerSDK.APEOverride{ + Target: apeSDK.ChainTarget{ + TargetType: apeSDK.TargetTypeContainer, + Name: container, + }, + Chains: []apeSDK.Chain{{ + Raw: denyBySourceIP.Bytes(), + }}, + } + + r, err := router.BearerChainFeedRouter(inmem.LocalStorage(), inmem.MorphRuleChainStorage(), bt) + require.NoError(t, err) + + req := resourcetest.NewRequest(nativeschema.MethodPutObject, + resourcetest.NewResource(fmt.Sprintf(nativeschema.ResourceFormatRootContainer, container), map[string]string{}), + map[string]string{ + "SourceIP": "10.122.1.20", + "Actor": "someOwner", + }, + ) + + st, found, err := r.IsAllowed(chain.Ingress, engine.NewRequestTarget(rootNs, container), req) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, st, chain.AccessDenied) + }) + t.Run("deny by namespace with allow by bearer overrides", func(t *testing.T) { + inmem := inmemory.NewInMemoryLocalOverrides() + + inmem.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(rootNs), denyBySourceIP) + + bt := bearerSDK.APEOverride{ + Target: apeSDK.ChainTarget{ + TargetType: apeSDK.TargetTypeContainer, + Name: container, + }, + Chains: []apeSDK.Chain{{ + Raw: allowBySourceIP.Bytes(), + }}, + } + + r, err := router.BearerChainFeedRouter(inmem.LocalStorage(), inmem.MorphRuleChainStorage(), bt) + require.NoError(t, err) + + req := resourcetest.NewRequest(nativeschema.MethodPutObject, + resourcetest.NewResource(fmt.Sprintf(nativeschema.ResourceFormatRootContainer, container), map[string]string{}), + map[string]string{ + "SourceIP": "10.122.1.20", + "Actor": "someOwner", + }, + ) + + st, found, err := r.IsAllowed(chain.Ingress, engine.NewRequestTarget(rootNs, container), req) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, st, chain.AccessDenied) + }) +} diff --git a/pkg/ape/router/single_pass.go b/pkg/ape/router/single_pass.go index c0c78db76..ec9244bae 100644 --- a/pkg/ape/router/single_pass.go +++ b/pkg/ape/router/single_pass.go @@ -3,37 +3,12 @@ package router import ( "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" ) -func newTarget(ct ape.ChainTarget) (engine.Target, error) { - var target engine.Target - switch ct.TargetType { - case ape.TargetTypeContainer: - var cid cidSDK.ID - err := cid.DecodeString(ct.Name) - if err != nil { - return target, fmt.Errorf("invalid cid format: %s", target.Name) - } - target.Type = engine.Container - case ape.TargetTypeGroup: - target.Type = engine.Group - case ape.TargetTypeNamespace: - target.Type = engine.Namespace - case ape.TargetTypeUser: - target.Type = engine.User - default: - return target, fmt.Errorf("unsupported target type: %v", ct.TargetType) - } - target.Name = ct.Name - return target, nil -} - // SingleUseRouterWithBearerTokenChains creates chain router with inmemory storage implementation and // fed with APE chains defined in Bearer token. func SingleUseRouterWithBearerTokenChains(overrides []bearer.APEOverride) (engine.ChainRouter, error) { From 0c2b6f3dac70c1f0e72918d3acfb9f64ec7c6f34 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 1 Jul 2024 18:42:15 +0300 Subject: [PATCH 1212/1943] [#1216] ape: Make services use bearer chains fed router * Refactor object and tree service - they should instantiate chain router cheking the bearer token. If there are no bearer token rules, then defaul chain router is used. * Fix unit-tests. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/container.go | 7 +++- cmd/frostfs-node/object.go | 3 +- cmd/frostfs-node/policy_engine.go | 30 ++++----------- cmd/frostfs-node/tree.go | 3 +- pkg/services/object/ape/checker.go | 51 +++++++++++-------------- pkg/services/object/ape/checker_test.go | 13 ++----- pkg/services/tree/ape.go | 20 +++------- pkg/services/tree/options.go | 13 +++++-- 8 files changed, 60 insertions(+), 80 deletions(-) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 99831fe98..5a29aac76 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -19,6 +19,7 @@ import ( containerMorph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/morph" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -46,9 +47,13 @@ func initContainerService(_ context.Context, c *cfg) { c.shared.frostfsidClient = frostfsIDSubjectProvider + defaultChainRouter := engine.NewDefaultChainRouterWithLocalOverrides( + c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.MorphRuleChainStorage(), + c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalStorage(), + ) service := containerService.NewSignService( &c.key.PrivateKey, - containerService.NewAPEServer(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine, cnrRdr, + containerService.NewAPEServer(defaultChainRouter, cnrRdr, newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, c.shared.frostfsidClient, containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), ), diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 62183d314..0124bf772 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -460,7 +460,8 @@ func createAPEService(c *cfg, splitSvc *objectService.TransportSplitter) *object return objectAPE.NewService( c.log, objectAPE.NewChecker( - c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.chainRouter, + c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalStorage(), + c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.MorphRuleChainStorage(), objectAPE.NewStorageEngineHeaderProvider(c.cfgObject.cfgLocalStorage.localStorage, c.cfgObject.getSvc), c.shared.frostfsidClient, c.netMapSource, diff --git a/cmd/frostfs-node/policy_engine.go b/cmd/frostfs-node/policy_engine.go index 13e30a17e..22fda2b4c 100644 --- a/cmd/frostfs-node/policy_engine.go +++ b/cmd/frostfs-node/policy_engine.go @@ -1,13 +1,11 @@ package main import ( - "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" - "git.frostfs.info/TrueCloudLab/policy-engine/pkg/resource" "github.com/google/uuid" "github.com/hashicorp/golang-lru/v2/expirable" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" @@ -15,11 +13,9 @@ import ( ) type accessPolicyEngine struct { - mtx sync.RWMutex - - chainRouter engine.ChainRouter - localOverrideDatabase chainbase.LocalOverrideDatabase + + morphChainStorage engine.MorphRuleChainStorageReader } var _ engine.MorphRuleChainStorageReader = (*morphAPEChainCache)(nil) @@ -70,32 +66,20 @@ func newAccessPolicyEngine( localOverrideDatabase chainbase.LocalOverrideDatabase, ) *accessPolicyEngine { return &accessPolicyEngine{ - chainRouter: engine.NewDefaultChainRouterWithLocalOverrides( - morphChainStorage, - localOverrideDatabase, - ), + morphChainStorage: morphChainStorage, localOverrideDatabase: localOverrideDatabase, } } -func (a *accessPolicyEngine) IsAllowed(name chain.Name, target engine.RequestTarget, r resource.Request) (status chain.Status, found bool, err error) { - a.mtx.RLock() - defer a.mtx.RUnlock() - - return a.chainRouter.IsAllowed(name, target, r) +func (a *accessPolicyEngine) LocalStorage() engine.LocalOverrideStorage { + return a.localOverrideDatabase } -func (a *accessPolicyEngine) LocalStorage() engine.LocalOverrideStorage { - a.mtx.Lock() - defer a.mtx.Unlock() - - return a.localOverrideDatabase +func (a *accessPolicyEngine) MorphRuleChainStorage() engine.MorphRuleChainStorageReader { + return a.morphChainStorage } func (a *accessPolicyEngine) LocalOverrideDatabaseCore() chainbase.DatabaseCore { - a.mtx.Lock() - defer a.mtx.Unlock() - return a.localOverrideDatabase } diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index daaaa64a2..d22e510de 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -65,7 +65,8 @@ func initTreeService(c *cfg) { tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount()), tree.WithAuthorizedKeys(treeConfig.AuthorizedKeys()), tree.WithMetrics(c.metricsCollector.TreeService()), - tree.WithAPERouter(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine), + tree.WithAPELocalOverrideStorage(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalStorage()), + tree.WithAPEMorphRuleStorage(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.MorphRuleChainStorage()), tree.WithNetmapState(c.cfgNetmap.state), ) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 7541ad116..c6f9b711d 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -24,24 +24,26 @@ import ( ) type checkerImpl struct { - chainRouter policyengine.ChainRouter - headerProvider HeaderProvider - frostFSIDClient frostfsidcore.SubjectProvider - nm netmap.Source - st netmap.State - cnrSource container.Source - nodePK []byte + localOverrideStorage policyengine.LocalOverrideStorage + morphChainStorage policyengine.MorphRuleChainStorageReader + headerProvider HeaderProvider + frostFSIDClient frostfsidcore.SubjectProvider + nm netmap.Source + st netmap.State + cnrSource container.Source + nodePK []byte } -func NewChecker(chainRouter policyengine.ChainRouter, headerProvider HeaderProvider, frostFSIDClient frostfsidcore.SubjectProvider, nm netmap.Source, st netmap.State, cnrSource container.Source, nodePK []byte) Checker { +func NewChecker(localOverrideStorage policyengine.LocalOverrideStorage, morphChainStorage policyengine.MorphRuleChainStorageReader, headerProvider HeaderProvider, frostFSIDClient frostfsidcore.SubjectProvider, nm netmap.Source, st netmap.State, cnrSource container.Source, nodePK []byte) Checker { return &checkerImpl{ - chainRouter: chainRouter, - headerProvider: headerProvider, - frostFSIDClient: frostFSIDClient, - nm: nm, - st: st, - cnrSource: cnrSource, - nodePK: nodePK, + localOverrideStorage: localOverrideStorage, + morphChainStorage: morphChainStorage, + headerProvider: headerProvider, + frostFSIDClient: frostFSIDClient, + nm: nm, + st: st, + cnrSource: cnrSource, + nodePK: nodePK, } } @@ -175,28 +177,21 @@ func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { groups[i] = fmt.Sprintf("%s:%s", prm.Namespace, groups[i]) } + var cr policyengine.ChainRouter if prm.BearerToken != nil && !prm.BearerToken.Impersonate() { if err := isValidBearer(prm.BearerToken, prm.ContainerOwner, prm.Container, pub, c.st); err != nil { return fmt.Errorf("bearer token validation error: %w", err) } - btRouter, err := router.SingleUseRouterWithBearerTokenChains([]bearer.APEOverride{prm.BearerToken.APEOverride()}) + cr, err = router.BearerChainFeedRouter(c.localOverrideStorage, c.morphChainStorage, prm.BearerToken.APEOverride()) if err != nil { - return err - } - status, found, err := btRouter.IsAllowed(apechain.Ingress, policyengine.NewRequestTargetWithContainer(prm.Container.EncodeToString()), r) - if err != nil { - return err - } - if found && status == apechain.Allow { - return nil - } - if status != apechain.NoRuleFound { - return fmt.Errorf("bearer token: method %s: %s", prm.Method, status) + return fmt.Errorf("create chain router error: %w", err) } + } else { + cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.morphChainStorage, c.localOverrideStorage) } rt := policyengine.NewRequestTargetExtended(prm.Namespace, prm.Container.EncodeToString(), fmt.Sprintf("%s:%s", prm.Namespace, pub.Address()), groups) - status, ruleFound, err := c.chainRouter.IsAllowed(apechain.Ingress, rt, r) + status, ruleFound, err := cr.IsAllowed(apechain.Ingress, rt, r) if err != nil { return err } diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index 5efd3669b..27c314dfb 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -469,9 +469,8 @@ func TestAPECheck_BearerTokenOverrides(t *testing.T) { ls := inmemory.NewInmemoryLocalStorage() ms := inmemory.NewInmemoryMorphRuleChainStorage() - r := policyengine.NewDefaultChainRouterWithLocalOverrides(ms, ls) - checker := NewChecker(r, headerProvider, frostfsidProvider, nil, &stMock{}, nil, nil) + checker := NewChecker(ls, ms, headerProvider, frostfsidProvider, nil, &stMock{}, nil, nil) prm := Prm{ Method: method, @@ -534,9 +533,7 @@ func TestAPECheck(t *testing.T) { }) } - router := policyengine.NewDefaultChainRouterWithLocalOverrides(ms, ls) - - checker := NewChecker(router, headerProvider, frostfsidProvider, nil, &stMock{}, nil, nil) + checker := NewChecker(ls, ms, headerProvider, frostfsidProvider, nil, &stMock{}, nil, nil) prm := Prm{ Method: method, @@ -639,8 +636,6 @@ func TestPutECChunk(t *testing.T) { MatchType: chain.MatchTypeFirstMatch, }) - router := policyengine.NewDefaultChainRouterWithLocalOverrides(ms, ls) - node1Key, err := keys.NewPrivateKey() require.NoError(t, err) node1 := netmapSDK.NodeInfo{} @@ -669,7 +664,7 @@ func TestPutECChunk(t *testing.T) { }, } - checker := NewChecker(router, headerProvider, frostfsidProvider, nm, &stMock{}, cs, node1Key.PublicKey().Bytes()) + checker := NewChecker(ls, ms, headerProvider, frostfsidProvider, nm, &stMock{}, cs, node1Key.PublicKey().Bytes()) ecParentID := oidtest.ID() chunkHeader := newHeaderObjectSDK(cnr, obj, nil).ToV2().GetHeader() @@ -711,7 +706,7 @@ func TestPutECChunk(t *testing.T) { t.Run("access allowed for non container node", func(t *testing.T) { otherKey, err := keys.NewPrivateKey() require.NoError(t, err) - checker = NewChecker(router, headerProvider, frostfsidProvider, nm, &stMock{}, cs, otherKey.PublicKey().Bytes()) + checker = NewChecker(ls, ms, headerProvider, frostfsidProvider, nm, &stMock{}, cs, otherKey.PublicKey().Bytes()) prm := Prm{ Method: nativeschema.MethodPutObject, Container: cnr, diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index a6202d1a3..6e78bf4ec 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -141,25 +141,17 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, return apeErr(err) } + var cr engine.ChainRouter if bt != nil && !bt.Impersonate() { if err := isValidBearer(bt, container.Value.Owner(), cid, publicKey, s.state); err != nil { return fmt.Errorf("bearer validation error: %w", err) } - btRouter, err := router.SingleUseRouterWithBearerTokenChains([]bearer.APEOverride{bt.APEOverride()}) + cr, err = router.BearerChainFeedRouter(s.localOverrideStorage, s.morphChainStorage, bt.APEOverride()) if err != nil { - return apeErr(err) - } - status, found, err := btRouter.IsAllowed(apechain.Ingress, engine.NewRequestTargetWithContainer(cid.EncodeToString()), request) - if err != nil { - return apeErr(err) - } - if found && status == apechain.Allow { - return nil - } - if status != apechain.NoRuleFound { - err = fmt.Errorf("access to operation %s is denied by access policy engine (bearer token): %s", request.Operation(), status.String()) - return apeErr(err) + return fmt.Errorf("create chain router error: %w", err) } + } else { + cr = engine.NewDefaultChainRouterWithLocalOverrides(s.morphChainStorage, s.localOverrideStorage) } groups, err := aperequest.Groups(s.frostfsidSubjectProvider, publicKey) @@ -173,7 +165,7 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, } rt := engine.NewRequestTargetExtended(namespace, cid.EncodeToString(), fmt.Sprintf("%s:%s", namespace, publicKey.Address()), groups) - status, found, err := s.router.IsAllowed(apechain.Ingress, rt, request) + status, found, err := cr.IsAllowed(apechain.Ingress, rt, request) if err != nil { return apeErr(err) } diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index ea5539938..6a20fe5cc 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -42,7 +42,8 @@ type cfg struct { containerCacheSize int authorizedKeys [][]byte - router policyengine.ChainRouter + localOverrideStorage policyengine.LocalOverrideStorage + morphChainStorage policyengine.MorphRuleChainStorageReader metrics MetricsRegister } @@ -152,9 +153,15 @@ func WithAuthorizedKeys(keys keys.PublicKeys) Option { } } -func WithAPERouter(router policyengine.ChainRouter) Option { +func WithAPELocalOverrideStorage(localOverrideStorage policyengine.LocalOverrideStorage) Option { return func(c *cfg) { - c.router = router + c.localOverrideStorage = localOverrideStorage + } +} + +func WithAPEMorphRuleStorage(morphRuleStorage policyengine.MorphRuleChainStorageReader) Option { + return func(c *cfg) { + c.morphChainStorage = morphRuleStorage } } From d90aab54541f459b673ca9908e6c020e5373e7a5 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 5 Jul 2024 11:50:11 +0300 Subject: [PATCH 1213/1943] [#1229] util: Fix session token expiration check * Make session token expired at `current_epoch + 1` but not at `current_epoch` when it's still valid. Signed-off-by: Airat Arifullin --- pkg/services/object/util/key.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/util/key.go b/pkg/services/object/util/key.go index 8304bf13a..23d6c1c68 100644 --- a/pkg/services/object/util/key.go +++ b/pkg/services/object/util/key.go @@ -67,7 +67,7 @@ func (s *KeyStorage) GetKey(info *SessionInfo) (*ecdsa.PrivateKey, error) { pToken := s.tokenStore.Get(info.Owner, binID) if pToken != nil { - if pToken.ExpiredAt() <= s.networkState.CurrentEpoch() { + if pToken.ExpiredAt() < s.networkState.CurrentEpoch() { return nil, new(apistatus.SessionTokenExpired) } return pToken.SessionKey(), nil From 6ef38c07bdb80be33f51096d1a29fe7641a32ab3 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 8 Jul 2024 15:29:29 +0300 Subject: [PATCH 1214/1943] [#1235] cli: Fix parse action * Parsed `object.*` lexeme should also include `MethodRangeObject` constant. Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/util/ape.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index 680e24d82..532dc0a50 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -246,6 +246,7 @@ func parseAction(lexeme string) ([]string, bool, error) { nativeschema.MethodHeadObject, nativeschema.MethodDeleteObject, nativeschema.MethodSearchObject, + nativeschema.MethodRangeObject, nativeschema.MethodHashObject, }, true, nil case "container.put": From 84ecd61dfd8154a8155b66e8bfa802e00ff96669 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 8 Jul 2024 11:33:40 +0300 Subject: [PATCH 1215/1943] [#1233] putSvc: Try to put EC chunk to any node Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/ec.go | 91 ++++++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 11 deletions(-) diff --git a/pkg/services/object/put/ec.go b/pkg/services/object/put/ec.go index 6237872fb..6da50195e 100644 --- a/pkg/services/object/put/ec.go +++ b/pkg/services/object/put/ec.go @@ -3,8 +3,10 @@ package putsvc import ( "context" "crypto/ecdsa" + "encoding/hex" "errors" "fmt" + "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" @@ -160,7 +162,7 @@ func (e *ecWriter) writeECPart(ctx context.Context, obj *objectSDK.Object) error } eg.Go(func() error { - return e.writePart(egCtx, obj, int(obj.ECHeader().Index()), nodes) + return e.writePart(egCtx, obj, int(obj.ECHeader().Index()), nodes, make([]atomic.Bool, len(nodes))) }) t.SubmitSuccess() } @@ -195,10 +197,15 @@ func (e *ecWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) er break } + visited := make([]atomic.Bool, len(nodes)) + for idx := range parts { + visited[idx%len(nodes)].Store(true) + } + for idx := range parts { idx := idx eg.Go(func() error { - return e.writePart(egCtx, parts[idx], idx, nodes) + return e.writePart(egCtx, parts[idx], idx, nodes, visited) }) t.SubmitSuccess() } @@ -211,19 +218,81 @@ func (e *ecWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) er return nil } -func (e *ecWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx int, nodes []placement.Node) error { - var err error - node := nodes[partIdx%len(nodes)] - if e.cfg.netmapKeys.IsLocalKey(node.PublicKey()) { - err = e.writePartLocal(ctx, obj) - } else { - err = e.writePartRemote(ctx, obj, node) +func (e *ecWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx int, nodes []placement.Node, visited []atomic.Bool) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: } + + // try to save to node for current part index + node := nodes[partIdx%len(nodes)] + err := e.putECPartToNode(ctx, obj, node) if err == nil { return nil } - e.cfg.log.Warn(logs.ECFailedToSaveECPart, zap.Stringer("parent_object", object.AddressOf(obj)), zap.Error(err)) - return err + e.cfg.log.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), + zap.Stringer("parent_address", obj.ECHeader().Parent()), zap.Int("part_index", partIdx), + zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Error(err)) + + partVisited := make([]bool, len(nodes)) + partVisited[partIdx%len(nodes)] = true + + // try to save to any node not visited by any of other parts + for i := 1; i < len(nodes); i++ { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + idx := (partIdx + i) % len(nodes) + if !visited[idx].CompareAndSwap(false, true) { + continue + } + node = nodes[idx] + err := e.putECPartToNode(ctx, obj, node) + if err == nil { + return nil + } + e.cfg.log.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), + zap.Stringer("parent_address", obj.ECHeader().Parent()), zap.Int("part_index", partIdx), + zap.String("node", hex.EncodeToString(node.PublicKey())), + zap.Error(err)) + + partVisited[idx] = true + } + + // try to save to any node not visited by current part + for i := 0; i < len(nodes); i++ { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if partVisited[i] { + continue + } + node = nodes[i] + err := e.putECPartToNode(ctx, obj, node) + if err == nil { + return nil + } + e.cfg.log.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), + zap.Stringer("parent_address", obj.ECHeader().Parent()), zap.Int("part_index", partIdx), + zap.String("node", hex.EncodeToString(node.PublicKey())), + zap.Error(err)) + } + + return fmt.Errorf("failed to save EC chunk %s to any node", object.AddressOf(obj)) +} + +func (e *ecWriter) putECPartToNode(ctx context.Context, obj *objectSDK.Object, node placement.Node) error { + if e.cfg.netmapKeys.IsLocalKey(node.PublicKey()) { + return e.writePartLocal(ctx, obj) + } + return e.writePartRemote(ctx, obj, node) } func (e *ecWriter) writePartLocal(ctx context.Context, obj *objectSDK.Object) error { From ca974b8b4c1c3be1a08811e0522552e93027f481 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 8 Jul 2024 11:36:58 +0300 Subject: [PATCH 1216/1943] [#1233] cli: Drop debugee from `object nodes` Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index cdce7cd7b..d04cf6f04 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -497,7 +497,6 @@ func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, if errors.As(err, ¬Found) || errors.As(err, &removed) { return false, nil } - cmd.Printf("failed to get object %s from client\n", objID.EncodeToString()) return false, err } From 784e8ef85775f4f388d03453eba670a9cd0ca1f9 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 8 Jul 2024 18:04:30 +0300 Subject: [PATCH 1217/1943] [#1209] cli: Add --quiet flag to healthcheck command Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/internal/commonflags/flags.go | 4 ++++ cmd/frostfs-cli/modules/control/healthcheck.go | 10 ++++++++++ cmd/frostfs-cli/modules/control/ir_healthcheck.go | 11 +++++++++++ 3 files changed, 25 insertions(+) diff --git a/cmd/frostfs-cli/internal/commonflags/flags.go b/cmd/frostfs-cli/internal/commonflags/flags.go index 23dbc8c2b..cd46d63eb 100644 --- a/cmd/frostfs-cli/internal/commonflags/flags.go +++ b/cmd/frostfs-cli/internal/commonflags/flags.go @@ -53,6 +53,10 @@ const ( AwaitFlag = "await" AwaitFlagUsage = "Wait for the operation to complete" + + QuietFlag = "quiet" + QuietFlagShorthand = "q" + QuietFlagUsage = "Print nothing and exit with non-zero code on failure" ) // Init adds common flags to the command: diff --git a/cmd/frostfs-cli/modules/control/healthcheck.go b/cmd/frostfs-cli/modules/control/healthcheck.go index 097fba540..2241a403f 100644 --- a/cmd/frostfs-cli/modules/control/healthcheck.go +++ b/cmd/frostfs-cli/modules/control/healthcheck.go @@ -1,7 +1,10 @@ package control import ( + "os" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" @@ -24,6 +27,7 @@ func initControlHealthCheckCmd() { flags := healthCheckCmd.Flags() flags.Bool(healthcheckIRFlag, false, "Communicate with IR node") + flags.BoolP(commonflags.QuietFlag, commonflags.QuietFlagShorthand, false, commonflags.QuietFlagUsage) _ = flags.MarkDeprecated(healthcheckIRFlag, "for health check of inner ring nodes, use the 'control ir healthcheck' command instead.") } @@ -50,6 +54,12 @@ func healthCheck(cmd *cobra.Command, args []string) { commonCmd.ExitOnErr(cmd, "rpc error: %w", err) verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + if quietFlag, _ := cmd.Flags().GetBool(commonflags.QuietFlag); quietFlag { + if resp.GetBody().GetHealthStatus() == control.HealthStatus_READY { + return + } + os.Exit(1) + } cmd.Printf("Network status: %s\n", resp.GetBody().GetNetmapStatus()) cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus()) diff --git a/cmd/frostfs-cli/modules/control/ir_healthcheck.go b/cmd/frostfs-cli/modules/control/ir_healthcheck.go index e70538ce2..4f272c1b4 100644 --- a/cmd/frostfs-cli/modules/control/ir_healthcheck.go +++ b/cmd/frostfs-cli/modules/control/ir_healthcheck.go @@ -1,7 +1,10 @@ package control import ( + "os" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" @@ -18,6 +21,8 @@ var irHealthCheckCmd = &cobra.Command{ func initControlIRHealthCheckCmd() { initControlFlags(irHealthCheckCmd) + flags := irHealthCheckCmd.Flags() + flags.BoolP(commonflags.QuietFlag, commonflags.QuietFlagShorthand, false, commonflags.QuietFlagUsage) } func irHealthCheck(cmd *cobra.Command, _ []string) { @@ -39,6 +44,12 @@ func irHealthCheck(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "rpc error: %w", err) verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + if quietFlag, _ := cmd.Flags().GetBool(commonflags.QuietFlag); quietFlag { + if resp.GetBody().GetHealthStatus() == ircontrol.HealthStatus_READY { + return + } + os.Exit(1) + } cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus()) } From 6ace2f597e8ee86b324abe029859b5d75de11549 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 8 Jul 2024 13:48:39 +0300 Subject: [PATCH 1218/1943] [#1234] pilorama: Add test for duplicate directory behaviour When AddByPath() is called concurrently on 2 different nodes, internal path components may be created twice. This violates some of our assumptions in GetByPath() and, indirectly, in S3 handling of GetSubTree() results. Add a test for the correct behaviour, fixes will follow. Signed-off-by: Evgenii Stratonikov --- .../pilorama/split_test.go | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 pkg/local_object_storage/pilorama/split_test.go diff --git a/pkg/local_object_storage/pilorama/split_test.go b/pkg/local_object_storage/pilorama/split_test.go new file mode 100644 index 000000000..85c6c3097 --- /dev/null +++ b/pkg/local_object_storage/pilorama/split_test.go @@ -0,0 +1,99 @@ +package pilorama + +import ( + "context" + "strings" + "testing" + + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "github.com/stretchr/testify/require" +) + +func TestDuplicateDirectory(t *testing.T) { + t.Skip() + for i := range providers { + if providers[i].name == "inmemory" { + continue + } + t.Run(providers[i].name, func(t *testing.T) { + testDuplicateDirectory(t, providers[i].construct(t)) + }) + } +} + +func testDuplicateDirectory(t *testing.T, f Forest) { + ctx := context.Background() + d := CIDDescriptor{CID: cidtest.ID(), Size: 1} + treeID := "sometree" + + treeApply := func(t *testing.T, parent, child uint64, filename string, internal bool) { + // Nothing magic here, we add items in order and children are unique. + // This simplifies function interface a bit. + ts := child + + kv := []KeyValue{{Key: AttributeFilename, Value: []byte(filename)}} + if !internal { + kv = append(kv, KeyValue{Key: "uniqueAttr", Value: []byte{byte(child)}}) + } + + err := f.TreeApply(ctx, d.CID, treeID, &Move{ + Parent: parent, + Child: child, + Meta: Meta{ + Time: ts, + Items: kv, + }, + }, true) + require.NoError(t, err) + } + + // The following tree is constructed: + // 0 + // [1] |-- dir1 (internal) + // [2] |-- value1 + // [3] |-- dir3 (internal) + // [4] |-- value3 + // [5] |-- dir1 (internal) + // [6] |-- value2 + // [7] |-- dir3 (internal) + // [8] |-- value4 + // [9] |-- dir2 (internal) + // [10] |-- value0 + treeApply(t, RootID, 1, "dir1", true) + treeApply(t, 1, 2, "value1", false) + treeApply(t, 1, 3, "dir3", true) + treeApply(t, 3, 4, "value3", false) + treeApply(t, RootID, 5, "dir1", true) + treeApply(t, 5, 6, "value2", false) + treeApply(t, 5, 7, "dir3", true) + treeApply(t, 7, 8, "value4", false) + treeApply(t, RootID, 9, "dir2", true) + treeApply(t, RootID, 10, "value0", false) + + // The compacted view: + // 0 + // [1,5] |-- dir1 (internal) + // [2] |-- value1 + // [3,7] |-- dir3 (internal) + // [4] |-- value3 + // [8] |-- value4 + // [6] |-- value2 + // [9] |-- dir2 (internal) + // [10] |-- value0 + testGetByPath := func(t *testing.T, p string) []byte { + pp := strings.Split(p, "/") + nodes, err := f.TreeGetByPath(context.Background(), d.CID, treeID, AttributeFilename, pp, false) + require.NoError(t, err) + require.Equal(t, 1, len(nodes)) + + meta, _, err := f.TreeGetMeta(ctx, d.CID, treeID, nodes[0]) + require.NoError(t, err) + require.Equal(t, []byte(pp[len(pp)-1]), meta.GetAttr(AttributeFilename)) + return meta.GetAttr("uniqueAttr") + } + + require.Equal(t, []byte{2}, testGetByPath(t, "dir1/value1")) + require.Equal(t, []byte{4}, testGetByPath(t, "dir1/dir3/value3")) + require.Equal(t, []byte{8}, testGetByPath(t, "dir1/dir3/value4")) + require.Equal(t, []byte{10}, testGetByPath(t, "value0")) +} From b027a7f91e550ff2e24779c648c3511f39339bba Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 8 Jul 2024 13:52:59 +0300 Subject: [PATCH 1219/1943] [#1234] pilorama: Fix GetByPath() on duplicate directories Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/boltdb.go | 60 ++++++++++++++----- .../pilorama/split_test.go | 1 - 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 9fc95dbbb..ee28df426 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -906,7 +906,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st b := treeRoot.Bucket(dataBucket) - i, curNode, err := t.getPathPrefix(b, attr, path[:len(path)-1]) + i, curNodes, err := t.getPathPrefixMultiTraversal(b, attr, path[:len(path)-1]) if err != nil { return err } @@ -918,21 +918,23 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st c := b.Cursor() - attrKey := internalKey(nil, attr, path[len(path)-1], curNode, 0) - attrKey = attrKey[:len(attrKey)-8] - childKey, _ := c.Seek(attrKey) - for len(childKey) == len(attrKey)+8 && bytes.Equal(attrKey, childKey[:len(childKey)-8]) { - child := binary.LittleEndian.Uint64(childKey[len(childKey)-8:]) - if latest { - _, ts, _, _ := t.getState(b, stateKey(make([]byte, 9), child)) - if ts >= maxTimestamp { - nodes = append(nodes[:0], child) - maxTimestamp = ts + for i := range curNodes { + attrKey := internalKey(nil, attr, path[len(path)-1], curNodes[i], 0) + attrKey = attrKey[:len(attrKey)-8] + childKey, _ := c.Seek(attrKey) + for len(childKey) == len(attrKey)+8 && bytes.Equal(attrKey, childKey[:len(childKey)-8]) { + child := binary.LittleEndian.Uint64(childKey[len(childKey)-8:]) + if latest { + _, ts, _, _ := t.getState(b, stateKey(make([]byte, 9), child)) + if ts >= maxTimestamp { + nodes = append(nodes[:0], child) + maxTimestamp = ts + } + } else { + nodes = append(nodes, child) } - } else { - nodes = append(nodes, child) + childKey, _ = c.Next() } - childKey, _ = c.Next() } return nil })) @@ -1412,6 +1414,36 @@ func (t *boltForest) TreeListTrees(ctx context.Context, prm TreeListTreesPrm) (* return &res, nil } +func (t *boltForest) getPathPrefixMultiTraversal(bTree *bbolt.Bucket, attr string, path []string) (int, []Node, error) { + c := bTree.Cursor() + + var curNodes []Node + nextNodes := []Node{RootID} + var attrKey []byte + + for i := range path { + curNodes, nextNodes = nextNodes, curNodes[:0] + for j := range curNodes { + attrKey = internalKey(attrKey, attr, path[i], curNodes[j], 0) + attrKey = attrKey[:len(attrKey)-8] + + childKey, value := c.Seek(attrKey) + for len(childKey) == len(attrKey)+8 && bytes.Equal(attrKey, childKey[:len(childKey)-8]) { + if len(value) == 1 && value[0] == 1 { + nextNodes = append(nextNodes, binary.LittleEndian.Uint64(childKey[len(childKey)-8:])) + } + childKey, value = c.Next() + } + } + + if len(nextNodes) == 0 { + return i, curNodes, nil + } + } + + return len(path), nextNodes, nil +} + func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node, error) { c := bTree.Cursor() diff --git a/pkg/local_object_storage/pilorama/split_test.go b/pkg/local_object_storage/pilorama/split_test.go index 85c6c3097..3bd581ce6 100644 --- a/pkg/local_object_storage/pilorama/split_test.go +++ b/pkg/local_object_storage/pilorama/split_test.go @@ -10,7 +10,6 @@ import ( ) func TestDuplicateDirectory(t *testing.T) { - t.Skip() for i := range providers { if providers[i].name == "inmemory" { continue From 3bf6e6dde60f15f4f90caf44eda429e8f6269cc1 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 10 Jul 2024 11:27:40 +0300 Subject: [PATCH 1220/1943] [#1238] engine: Add non-blocking send in the shard's notification channel Signed-off-by: Anton Nikiforov --- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/shards.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index ae8834517..5a195f688 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -252,6 +252,7 @@ const ( ShardCantInitializeMetabaseMoveToADegradedreadonlyMode = "can't initialize metabase, move to a degraded-read-only mode" ShardStopEventListenerByClosedEventChannel = "stop event listener by closed `event` channel" ShardStopEventListenerByClosedStopChannel = "stop event listener by closed `stop` channel" + ShardEventProcessingInProgress = "event processing is in progress, skip the received" ShardStopEventListenerByContext = "stop event listener by context" ShardCouldNotSubmitGCJobToWorkerPool = "could not submit GC job to worker pool" ShardGCIsStopped = "GC is stopped" diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 54d3e7d65..56d4fcd4a 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -340,6 +340,9 @@ func (e *StorageEngine) HandleNewEpoch(ctx context.Context, epoch uint64) { case <-ctx.Done(): return case sh.NotificationChannel() <- ev: + default: + e.log.Debug(logs.ShardEventProcessingInProgress, + zap.Uint64("epoch", epoch), zap.Stringer("shard", sh.ID())) } } } From 36956db12305fd7d58e5b9cabe6bfd9334b329e4 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 3 Jul 2024 12:03:53 +0300 Subject: [PATCH 1221/1943] [#1159] adm/frostfsid: Remove wallet requirement for list-* Signed-off-by: Alexander Chuprov --- .../modules/morph/frostfsid/frostfsid.go | 110 ++++++++++++++---- 1 file changed, 86 insertions(+), 24 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 442920912..091d6634a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -3,24 +3,32 @@ package frostfsid import ( "errors" "fmt" + "math/big" "sort" frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + frostfsidrpclient "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/emit" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/spf13/cobra" "github.com/spf13/viper" ) +const iteratorBatchSize = 1 + const ( namespaceFlag = "namespace" subjectNameFlag = "subject-name" @@ -250,12 +258,15 @@ func frostfsidCreateNamespace(cmd *cobra.Command, _ []string) { } func frostfsidListNamespaces(cmd *cobra.Command, _ []string) { - ffsid, err := newFrostfsIDClient(cmd) - commonCmd.ExitOnErr(cmd, "init contract invoker: %w", err) - - namespaces, err := ffsid.roCli.ListNamespaces() - commonCmd.ExitOnErr(cmd, "list namespaces: %w", err) + inv, _, hash := initInvoker(cmd) + reader := frostfsidrpclient.NewReader(inv, hash) + sessionID, it, err := reader.ListNamespaces() + commonCmd.ExitOnErr(cmd, "can't get namespace: %w", err) + items, err := readIterator(inv, &it, iteratorBatchSize, sessionID) + commonCmd.ExitOnErr(cmd, "can't read iterator: %w", err) + namespaces, err := frostfsidclient.ParseNamespaces(items) + commonCmd.ExitOnErr(cmd, "can't parse namespace: %w", err) sort.Slice(namespaces, func(i, j int) bool { return namespaces[i].Name < namespaces[j].Name }) for _, namespace := range namespaces { @@ -296,14 +307,15 @@ func frostfsidDeleteSubject(cmd *cobra.Command, _ []string) { } func frostfsidListSubjects(cmd *cobra.Command, _ []string) { - ns := getFrostfsIDNamespace(cmd) includeNames, _ := cmd.Flags().GetBool(includeNamesFlag) + ns := getFrostfsIDNamespace(cmd) + inv, _, hash := initInvoker(cmd) + reader := frostfsidrpclient.NewReader(inv, hash) + sessionID, it, err := reader.ListNamespaceSubjects(ns) + commonCmd.ExitOnErr(cmd, "can't get namespace: %w", err) - ffsid, err := newFrostfsIDClient(cmd) - commonCmd.ExitOnErr(cmd, "init contract invoker: %w", err) - - subAddresses, err := ffsid.roCli.ListNamespaceSubjects(ns) - commonCmd.ExitOnErr(cmd, "list subjects: %w", err) + subAddresses, err := frostfsidclient.UnwrapArrayOfUint160(readIterator(inv, &it, iteratorBatchSize, sessionID)) + commonCmd.ExitOnErr(cmd, "can't unwrap: %w", err) sort.Slice(subAddresses, func(i, j int) bool { return subAddresses[i].Less(subAddresses[j]) }) @@ -313,8 +325,14 @@ func frostfsidListSubjects(cmd *cobra.Command, _ []string) { continue } - subj, err := ffsid.roCli.GetSubject(addr) - commonCmd.ExitOnErr(cmd, "get subject: %w", err) + sessionID, it, err := reader.ListSubjects() + commonCmd.ExitOnErr(cmd, "can't get subject: %w", err) + + items, err := readIterator(inv, &it, iteratorBatchSize, sessionID) + commonCmd.ExitOnErr(cmd, "can't read iterator: %w", err) + + subj, err := frostfsidclient.ParseSubject(items) + commonCmd.ExitOnErr(cmd, "can't parse subject: %w", err) cmd.Printf("%s (%s)\n", address.Uint160ToString(addr), subj.Name) } @@ -349,13 +367,17 @@ func frostfsidDeleteGroup(cmd *cobra.Command, _ []string) { } func frostfsidListGroups(cmd *cobra.Command, _ []string) { + inv, _, hash := initInvoker(cmd) ns := getFrostfsIDNamespace(cmd) - ffsid, err := newFrostfsIDClient(cmd) - commonCmd.ExitOnErr(cmd, "init contract invoker: %w", err) + reader := frostfsidrpclient.NewReader(inv, hash) + sessionID, it, err := reader.ListGroups(ns) + commonCmd.ExitOnErr(cmd, "can't get namespace: %w", err) - groups, err := ffsid.roCli.ListGroups(ns) - commonCmd.ExitOnErr(cmd, "list groups: %w", err) + items, err := readIterator(inv, &it, iteratorBatchSize, sessionID) + commonCmd.ExitOnErr(cmd, "can't list groups: %w", err) + groups, err := frostfsidclient.ParseGroups(items) + commonCmd.ExitOnErr(cmd, "can't parse groups: %w", err) sort.Slice(groups, func(i, j int) bool { return groups[i].Name < groups[j].Name }) @@ -394,12 +416,19 @@ func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { ns := getFrostfsIDNamespace(cmd) groupID := getFrostfsIDGroupID(cmd) includeNames, _ := cmd.Flags().GetBool(includeNamesFlag) + inv, cs, hash := initInvoker(cmd) + _, err := helper.NNSResolveHash(inv, cs.Hash, helper.DomainOf(constants.FrostfsIDContract)) + commonCmd.ExitOnErr(cmd, "can't get netmap contract hash: %w", err) - ffsid, err := newFrostfsIDClient(cmd) - commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + reader := frostfsidrpclient.NewReader(inv, hash) + sessionID, it, err := reader.ListGroupSubjects(ns, big.NewInt(groupID)) + commonCmd.ExitOnErr(cmd, "can't list groups: %w", err) - subjects, err := ffsid.roCli.ListGroupSubjects(ns, groupID) - commonCmd.ExitOnErr(cmd, "list group subjects: %w", err) + items, err := readIterator(inv, &it, iteratorBatchSize, sessionID) + commonCmd.ExitOnErr(cmd, "can't read iterator: %w", err) + + subjects, err := frostfsidclient.UnwrapArrayOfUint160(items, err) + commonCmd.ExitOnErr(cmd, "can't unwrap: %w", err) sort.Slice(subjects, func(i, j int) bool { return subjects[i].Less(subjects[j]) }) @@ -409,9 +438,10 @@ func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { continue } - subj, err := ffsid.roCli.GetSubject(subjAddr) - commonCmd.ExitOnErr(cmd, "get subject: %w", err) - + items, err := reader.GetSubject(subjAddr) + commonCmd.ExitOnErr(cmd, "can't get subject: %w", err) + subj, err := frostfsidclient.ParseSubject(items) + commonCmd.ExitOnErr(cmd, "can't parse subject: %w", err) cmd.Printf("%s (%s)\n", address.Uint160ToString(subjAddr), subj.Name) } } @@ -474,3 +504,35 @@ func (f *frostfsidClient) sendWaitRes() (*state.AppExecResult, error) { f.wCtx.Command.Println("Waiting for transactions to persist...") return f.roCli.Wait(f.wCtx.SentTxs[0].Hash, f.wCtx.SentTxs[0].Vub, nil) } + +func readIterator(inv *invoker.Invoker, iter *result.Iterator, batchSize int, sessionID uuid.UUID) ([]stackitem.Item, error) { + var shouldStop bool + res := make([]stackitem.Item, 0) + for !shouldStop { + items, err := inv.TraverseIterator(sessionID, iter, batchSize) + if err != nil { + return nil, err + } + + res = append(res, items...) + shouldStop = len(items) < batchSize + } + + return res, nil +} + +func initInvoker(cmd *cobra.Command) (*invoker.Invoker, *state.Contract, util.Uint160) { + c, err := helper.GetN3Client(viper.GetViper()) + commonCmd.ExitOnErr(cmd, "can't create N3 client: %w", err) + + inv := invoker.New(c, nil) + r := management.NewReader(inv) + + cs, err := r.GetContractByID(1) + commonCmd.ExitOnErr(cmd, "can't get NNS contract info: %w", err) + + nmHash, err := helper.NNSResolveHash(inv, cs.Hash, helper.DomainOf(constants.FrostfsIDContract)) + commonCmd.ExitOnErr(cmd, "can't get netmap contract hash: %w", err) + + return inv, cs, nmHash +} From 0308835d48c8b8d62f8e062e0d7d90b4197e14fd Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 10 Jul 2024 16:25:42 +0300 Subject: [PATCH 1222/1943] [#1159] adm/frostfsid: Update frostfs-contract version Signed-off-by: Alexander Chuprov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 38e66a20f..d4eb52f4d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 - git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 + git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de diff --git a/go.sum b/go.sum index 0849b74a1..84fdbd47a 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 h1:H5GvrVlowIMWfzqQkhY0p0myooJxQ1sMRVSFfXawwWg= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45 h1:Tp4I+XOLp3VCJORfxSamQtj3RZNISbaLM4WD5iIzXxg= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409111539-e7a05a49ff45/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= From 08953a2f94ecb50ff8b006d25ac34b4b2c623418 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 11 Jul 2024 18:28:14 +0300 Subject: [PATCH 1223/1943] [#1239] adm/morph: Fix transfer GAS to alphabet nodes Signed-off-by: Alexander Chuprov --- .../modules/morph/initialize/initialize_transfer.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go index 8bcd7eb58..b7102fa13 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go @@ -26,6 +26,10 @@ const ( initialAlphabetGASAmount = 10_000 * native.GASFactor // initialProxyGASAmount represents the amount of GAS given to a proxy contract. initialProxyGASAmount = 50_000 * native.GASFactor + // alphabetGasRatio is a coefficient that defines the threshold below which + // the balance of the alphabet node is considered not replenished. The value + // of this coefficient is determined empirically. + alphabetGasRatio = 5 ) func transferFunds(c *helper.InitializeContext) error { @@ -80,7 +84,7 @@ func transferFundsFinished(c *helper.InitializeContext) (bool, error) { r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) res, err := r.BalanceOf(acc.Contract.ScriptHash()) - return res.Cmp(big.NewInt(initialAlphabetGASAmount/2)) == 1, err + return res.Cmp(big.NewInt(alphabetGasRatio*native.GASFactor)) == 1, err } func transferGASToProxy(c *helper.InitializeContext) error { From d02a7c2d383e832c1cef9fcbb77bd1a6653218db Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 12 Jul 2024 17:56:04 +0300 Subject: [PATCH 1224/1943] [#1196] morph/client: Remove duplicate utility functions * We used several utility functions to parse frostfsid client subject and extended subject. However, following the changes in TrueCloudLab/frostfs-contract#97, these utility functions have become public. So there is no more need to have them here. * There was a mismatch of slice parameter required length between frostfs-node's and frostfs-contract's utility functions, `checkStackItem()` solves this problem. Signed-off-by: Ekaterina Lebedeva --- pkg/morph/client/frostfsid/subject.go | 207 ++------------------------ 1 file changed, 16 insertions(+), 191 deletions(-) diff --git a/pkg/morph/client/frostfsid/subject.go b/pkg/morph/client/frostfsid/subject.go index 169be6bc5..0852f536c 100644 --- a/pkg/morph/client/frostfsid/subject.go +++ b/pkg/morph/client/frostfsid/subject.go @@ -1,16 +1,12 @@ package frostfsid import ( - "errors" "fmt" frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" ) const ( @@ -28,7 +24,12 @@ func (c *Client) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) return nil, fmt.Errorf("could not perform test invocation (%s): %w", methodGetSubject, err) } - subj, err := parseSubject(res) + structArr, err := checkStackItem(res) + if err != nil { + return nil, fmt.Errorf("invalid test invocation result (%s): %w", methodGetSubjectExtended, err) + } + + subj, err := frostfsidclient.ParseSubject(structArr) if err != nil { return nil, fmt.Errorf("could not parse test invocation result (%s): %w", methodGetSubject, err) } @@ -43,10 +44,15 @@ func (c *Client) GetSubjectExtended(addr util.Uint160) (*frostfsidclient.Subject res, err := c.client.TestInvoke(prm) if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", methodGetSubject, err) + return nil, fmt.Errorf("could not perform test invocation (%s): %w", methodGetSubjectExtended, err) } - subj, err := parseSubjectExtended(res) + structArr, err := checkStackItem(res) + if err != nil { + return nil, fmt.Errorf("invalid test invocation result (%s): %w", methodGetSubjectExtended, err) + } + + subj, err := frostfsidclient.ParseSubjectExtended(structArr) if err != nil { return nil, fmt.Errorf("could not parse test invocation result (%s): %w", methodGetSubject, err) } @@ -54,195 +60,14 @@ func (c *Client) GetSubjectExtended(addr util.Uint160) (*frostfsidclient.Subject return subj, nil } -func parseSubject(res []stackitem.Item) (*frostfsidclient.Subject, error) { +func checkStackItem(res []stackitem.Item) (structArr []stackitem.Item, err error) { if ln := len(res); ln != 1 { return nil, fmt.Errorf("unexpected stack item count (%s): %d", methodGetSubject, ln) } - structArr, err := client.ArrayFromStackItem(res[0]) + structArr, err = client.ArrayFromStackItem(res[0]) if err != nil { return nil, fmt.Errorf("could not get item array of container (%s): %w", methodGetSubject, err) } - - var subj frostfsidclient.Subject - - subj.PrimaryKey, err = unwrap.PublicKey(makeValidRes(structArr[0])) - if err != nil { - return nil, err - } - - if !structArr[1].Equals(stackitem.Null{}) { - subj.AdditionalKeys, err = unwrap.ArrayOfPublicKeys(makeValidRes(structArr[1])) - if err != nil { - return nil, err - } - } - - if !structArr[2].Equals(stackitem.Null{}) { - subj.Namespace, err = unwrap.UTF8String(makeValidRes(structArr[2])) - if err != nil { - return nil, err - } - } - - if !structArr[3].Equals(stackitem.Null{}) { - subj.Name, err = unwrap.UTF8String(makeValidRes(structArr[3])) - if err != nil { - return nil, err - } - } - - subj.KV, err = parseMap(structArr[4]) - if err != nil { - return nil, err - } - - return &subj, nil -} - -func parseSubjectExtended(res []stackitem.Item) (*frostfsidclient.SubjectExtended, error) { - if ln := len(res); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", methodGetSubject, ln) - } - - structArr, err := client.ArrayFromStackItem(res[0]) - if err != nil { - return nil, fmt.Errorf("could not get item array of container (%s): %w", methodGetSubject, err) - } - - var subj frostfsidclient.SubjectExtended - - subj.PrimaryKey, err = unwrap.PublicKey(makeValidRes(structArr[0])) - if err != nil { - return nil, err - } - - if !structArr[1].Equals(stackitem.Null{}) { - subj.AdditionalKeys, err = unwrap.ArrayOfPublicKeys(makeValidRes(structArr[1])) - if err != nil { - return nil, err - } - } - - if !structArr[2].Equals(stackitem.Null{}) { - subj.Namespace, err = unwrap.UTF8String(makeValidRes(structArr[2])) - if err != nil { - return nil, err - } - } - - if !structArr[3].Equals(stackitem.Null{}) { - subj.Name, err = unwrap.UTF8String(makeValidRes(structArr[3])) - if err != nil { - return nil, err - } - } - - subj.KV, err = parseMap(structArr[4]) - if err != nil { - return nil, err - } - - if !structArr[5].Equals(stackitem.Null{}) { - groupItems, ok := structArr[5].Value().([]stackitem.Item) - if !ok { - return nil, errors.New("invalid groups field") - } - - subj.Groups, err = parseGroups(groupItems) - if err != nil { - return nil, err - } - } - - return &subj, nil -} - -func makeValidRes(item stackitem.Item) (*result.Invoke, error) { - return &result.Invoke{ - Stack: []stackitem.Item{item}, - State: vmstate.Halt.String(), - }, nil -} - -func parseMap(item stackitem.Item) (map[string]string, error) { - if item.Equals(stackitem.Null{}) { - return nil, nil - } - - metaMap, err := unwrap.Map(makeValidRes(item)) - if err != nil { - return nil, err - } - - meta, ok := metaMap.Value().([]stackitem.MapElement) - if !ok { - return nil, errors.New("invalid map type") - } - - res := make(map[string]string, len(meta)) - for _, element := range meta { - key, err := element.Key.TryBytes() - if err != nil { - return nil, err - } - val, err := element.Value.TryBytes() - if err != nil { - return nil, err - } - res[string(key)] = string(val) - } - - return res, nil -} - -func parseGroups(items []stackitem.Item) ([]*frostfsidclient.Group, error) { - var err error - res := make([]*frostfsidclient.Group, len(items)) - - for i := 0; i < len(items); i++ { - arr, ok := items[i].Value().([]stackitem.Item) - if !ok { - return nil, errors.New("invalid group type") - } - res[i], err = parseGroup(arr) - if err != nil { - return nil, err - } - } - - return res, nil -} - -func parseGroup(structArr []stackitem.Item) (*frostfsidclient.Group, error) { - if len(structArr) < 4 { - return nil, errors.New("invalid response group struct") - } - - groupID, err := structArr[0].TryInteger() - if err != nil { - return nil, err - } - - name, err := structArr[1].TryBytes() - if err != nil { - return nil, err - } - - namespace, err := structArr[2].TryBytes() - if err != nil { - return nil, err - } - - kvs, err := parseMap(structArr[3]) - if err != nil { - return nil, err - } - - return &frostfsidclient.Group{ - ID: groupID.Int64(), - Name: string(name), - Namespace: string(namespace), - KV: kvs, - }, nil + return } From 39866a957c2f67801d30a70b8c1430cd3a1d8b4b Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 12 Jul 2024 17:57:11 +0300 Subject: [PATCH 1225/1943] [#1196] morph/client: Fix `ArrayFromStackItem()` description Signed-off-by: Ekaterina Lebedeva --- pkg/morph/client/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/morph/client/util.go b/pkg/morph/client/util.go index ff8e507fe..cd55d6bd2 100644 --- a/pkg/morph/client/util.go +++ b/pkg/morph/client/util.go @@ -69,7 +69,7 @@ func BytesFromStackItem(param stackitem.Item) ([]byte, error) { // ArrayFromStackItem returns the slice contract parameters from passed parameter. // -// If passed parameter carries boolean false value, (nil, nil) returns. +// If passed parameter carries boolean false value, returns (nil, nil). func ArrayFromStackItem(param stackitem.Item) ([]stackitem.Item, error) { switch param.Type() { case stackitem.AnyT: From d5dc14c639a99b7c0338dea665851e88a5a59202 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 12 Jul 2024 12:02:20 +0300 Subject: [PATCH 1226/1943] [#1243] object: Make APE checker set x-headers to request properties * Update go.mod, go.sum; * Add x-headers to request properties; * Add a unit-test. Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 +-- pkg/services/object/ape/checker.go | 4 +++ pkg/services/object/ape/checker_test.go | 48 +++++++++++++++++++++++++ pkg/services/object/ape/request.go | 6 ++++ pkg/services/object/ape/service.go | 10 ++++++ 6 files changed, 71 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d4eb52f4d..ee8b1bb16 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index 84fdbd47a..c7c3b87eb 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8l git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 h1:JRjwcHaQajTbSCBCK3yZnqvyHvgWBaoThDGuT4kvIIc= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d h1:hHk8FWbWgEnwm2I045CaBIrZBjy/o81CehIVOySA/pQ= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240513163744-1f6f4163d40d/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984 h1:O3F2Grz07RWZ68mRz1xsYsNPNvZLwY00BM+xoYb1kNk= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index c6f9b711d..a1972292e 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -7,6 +7,7 @@ import ( "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/router" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" @@ -79,6 +80,9 @@ type Prm struct { // The request's bearer token. It is used in order to check APE overrides with the token. BearerToken *bearer.Token + + // XHeaders from the request. + XHeaders []session.XHeader } var ( diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index 27c314dfb..090f6a83c 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" @@ -240,6 +241,7 @@ var apeCheckTestCases = []struct { object *string methods []string header testHeader + xHeaders []session.XHeader containerRules []chain.Rule groupidRules []chain.Rule expectAPEErr bool @@ -367,6 +369,52 @@ var apeCheckTestCases = []struct { }, expectAPEErr: true, }, + { + name: "oid required requests are denied by xheader", + container: containerID, + object: stringPtr(objectID), + methods: methodsRequiredOID, + header: testHeader{ + headerObjSDK: &headerObjectSDKParams{ + attributes: []struct { + key string + val string + }{ + { + key: "attr1", + val: "attribute_value", + }, + }, + }, + fromHeaderProvider: true, + }, + xHeaders: []session.XHeader{ + func() (xhead session.XHeader) { + xhead.SetKey("X-Test-ID") + xhead.SetValue("aezakmi") + return + }(), + }, + containerRules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{Names: methodsRequiredOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObject, containerID, objectID)}, + }, + Any: true, + Condition: []chain.Condition{ + { + Op: chain.CondStringLike, + Kind: chain.KindRequest, + Key: fmt.Sprintf(commonschema.PropertyKeyFrostFSXHeader, "X-Test-ID"), + Value: "aezakmi", + }, + }, + }, + }, + expectAPEErr: true, + }, { name: "optional oid requests reached quota limit by an attribute", container: containerID, diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index 02ead8c60..1c129f65f 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -126,6 +126,12 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, nativeschema.PropertyKeyActorRole: prm.Role, } + + for _, xhead := range prm.XHeaders { + xheadKey := fmt.Sprintf(commonschema.PropertyKeyFrostFSXHeader, xhead.GetKey()) + reqProps[xheadKey] = xhead.GetValue() + } + var err error reqProps, err = c.fillWithUserClaimTags(reqProps, prm) if err != nil { diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 853c3b80d..56c66002d 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -111,6 +111,7 @@ func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { Role: g.role, SoftAPECheck: g.softAPECheck, BearerToken: g.bearerToken, + XHeaders: resp.GetMetaHeader().GetXHeaders(), } if err := g.apeChecker.CheckAPE(g.Context(), prm); err != nil { @@ -154,6 +155,7 @@ func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectSt SoftAPECheck: reqCtx.SoftAPECheck, WithoutHeaderRequest: true, BearerToken: reqCtx.BearerToken, + XHeaders: request.GetMetaHeader().GetXHeaders(), }) if err != nil { return toStatusErr(err) @@ -200,6 +202,7 @@ func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutR Role: nativeSchemaRole(reqCtx.Role), SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, + XHeaders: request.GetMetaHeader().GetXHeaders(), } if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -245,6 +248,7 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj SoftAPECheck: reqCtx.SoftAPECheck, WithoutHeaderRequest: true, BearerToken: reqCtx.BearerToken, + XHeaders: request.GetMetaHeader().GetXHeaders(), }) if err != nil { return nil, toStatusErr(err) @@ -285,6 +289,7 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, + XHeaders: request.GetMetaHeader().GetXHeaders(), }) if err != nil { return nil, toStatusErr(err) @@ -314,6 +319,7 @@ func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.Searc ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, + XHeaders: request.GetMetaHeader().GetXHeaders(), }) if err != nil { return toStatusErr(err) @@ -343,6 +349,7 @@ func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) ( ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, + XHeaders: request.GetMetaHeader().GetXHeaders(), }) if err != nil { return nil, toStatusErr(err) @@ -377,6 +384,7 @@ func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.G ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, + XHeaders: request.GetMetaHeader().GetXHeaders(), }) if err != nil { return toStatusErr(err) @@ -406,6 +414,7 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, + XHeaders: request.GetMetaHeader().GetXHeaders(), } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -445,6 +454,7 @@ func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequ ContainerOwner: reqCtx.ContainerOwner, SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, + XHeaders: request.GetMetaHeader().GetXHeaders(), } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { From 21431f22c0c5c90efb24a5c0937ea525f10a6849 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 Jul 2024 17:04:07 +0300 Subject: [PATCH 1227/1943] [#1248] placement: Use cid.ID as key in the cache Signed-off-by: Evgenii Stratonikov --- pkg/services/object_manager/placement/netmap.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/services/object_manager/placement/netmap.go b/pkg/services/object_manager/placement/netmap.go index 787c04421..41267dbae 100644 --- a/pkg/services/object_manager/placement/netmap.go +++ b/pkg/services/object_manager/placement/netmap.go @@ -19,7 +19,7 @@ type netMapBuilder struct { lastNm *netmapSDK.NetMap // containerCache caches container nodes by ID. It is used to skip `GetContainerNodes` invocation if // neither netmap nor container has changed. - containerCache simplelru.LRUCache[string, [][]netmapSDK.NodeInfo] + containerCache simplelru.LRUCache[cid.ID, [][]netmapSDK.NodeInfo] } type netMapSrc struct { @@ -32,7 +32,7 @@ type netMapSrc struct { const defaultContainerCacheSize = 10 func NewNetworkMapBuilder(nm *netmapSDK.NetMap) Builder { - cache, _ := simplelru.NewLRU[string, [][]netmapSDK.NodeInfo](defaultContainerCacheSize, nil) // no error + cache, _ := simplelru.NewLRU[cid.ID, [][]netmapSDK.NodeInfo](defaultContainerCacheSize, nil) // no error return &netMapBuilder{ nmSrc: &netMapSrc{nm: nm}, containerCache: cache, @@ -40,7 +40,7 @@ func NewNetworkMapBuilder(nm *netmapSDK.NetMap) Builder { } func NewNetworkMapSourceBuilder(nmSrc netmap.Source) Builder { - cache, _ := simplelru.NewLRU[string, [][]netmapSDK.NodeInfo](defaultContainerCacheSize, nil) // no error + cache, _ := simplelru.NewLRU[cid.ID, [][]netmapSDK.NodeInfo](defaultContainerCacheSize, nil) // no error return &netMapBuilder{ nmSrc: nmSrc, containerCache: cache, @@ -62,7 +62,7 @@ func (b *netMapBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, p netmapSDK.Plac b.mtx.Lock() if nm == b.lastNm { - raw, ok := b.containerCache.Get(string(binCnr)) + raw, ok := b.containerCache.Get(cnr) b.mtx.Unlock() if ok { return BuildObjectPlacement(nm, raw, obj) @@ -78,7 +78,7 @@ func (b *netMapBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, p netmapSDK.Plac } b.mtx.Lock() - b.containerCache.Add(string(binCnr), cn) + b.containerCache.Add(cnr, cn) b.mtx.Unlock() return BuildObjectPlacement(nm, cn, obj) From 3a48b282b64429149ca7f9770eacb8a5bdc7173a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 Jul 2024 17:16:06 +0300 Subject: [PATCH 1228/1943] [#1248] placement: Use epoch to track netmap versions Previously we used pointer, this could have worked, because most of the time, the netmap is cached. This didn't work, however, because `lastNm` field was always nil. Rework the mechanism completely: 1. Use epoch to track netmap versions, as it it simpler and is unrelated to the TTL of an underlying cache. 2. Fix a bug where the epoch could change while mutex was unlocked. Signed-off-by: Evgenii Stratonikov --- pkg/services/object_manager/placement/netmap.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/services/object_manager/placement/netmap.go b/pkg/services/object_manager/placement/netmap.go index 41267dbae..8163529ed 100644 --- a/pkg/services/object_manager/placement/netmap.go +++ b/pkg/services/object_manager/placement/netmap.go @@ -14,9 +14,10 @@ import ( type netMapBuilder struct { nmSrc netmap.Source - // mtx protects lastNm and containerCache fields. - mtx sync.Mutex - lastNm *netmapSDK.NetMap + // mtx protects lastEpoch and containerCache fields. + mtx sync.Mutex + // lastEpoch contains contains network map epoch for all values in the container cache. + lastEpoch uint64 // containerCache caches container nodes by ID. It is used to skip `GetContainerNodes` invocation if // neither netmap nor container has changed. containerCache simplelru.LRUCache[cid.ID, [][]netmapSDK.NodeInfo] @@ -61,13 +62,14 @@ func (b *netMapBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, p netmapSDK.Plac cnr.Encode(binCnr) b.mtx.Lock() - if nm == b.lastNm { + if nm.Epoch() == b.lastEpoch { raw, ok := b.containerCache.Get(cnr) b.mtx.Unlock() if ok { return BuildObjectPlacement(nm, raw, obj) } } else { + b.lastEpoch = nm.Epoch() b.containerCache.Purge() b.mtx.Unlock() } @@ -78,7 +80,9 @@ func (b *netMapBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, p netmapSDK.Plac } b.mtx.Lock() - b.containerCache.Add(cnr, cn) + if b.lastEpoch == nm.Epoch() { + b.containerCache.Add(cnr, cn) + } b.mtx.Unlock() return BuildObjectPlacement(nm, cn, obj) From 286df198c9401706f5152b31dbecab3ff2df6271 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 12 Jul 2024 17:27:59 +0300 Subject: [PATCH 1229/1943] [#1248] placement: Decouple ContainerNodes() cache from the placement builder. Also, write tests. Signed-off-by: Evgenii Stratonikov --- .../object_manager/placement/cache.go | 69 ++++++++++++++ .../object_manager/placement/cache_test.go | 95 +++++++++++++++++++ .../object_manager/placement/netmap.go | 47 ++------- 3 files changed, 170 insertions(+), 41 deletions(-) create mode 100644 pkg/services/object_manager/placement/cache.go create mode 100644 pkg/services/object_manager/placement/cache_test.go diff --git a/pkg/services/object_manager/placement/cache.go b/pkg/services/object_manager/placement/cache.go new file mode 100644 index 000000000..217261877 --- /dev/null +++ b/pkg/services/object_manager/placement/cache.go @@ -0,0 +1,69 @@ +package placement + +import ( + "crypto/sha256" + "fmt" + "sync" + + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/hashicorp/golang-lru/v2/simplelru" +) + +// ContainerNodesCache caches results of ContainerNodes() invocation between epochs. +type ContainerNodesCache struct { + // mtx protects lastEpoch and containerCache fields. + mtx sync.Mutex + // lastEpoch contains network map epoch for all values in the container cache. + lastEpoch uint64 + // containerCache caches container nodes by ID. It is used to skip `GetContainerNodes` invocation if + // neither netmap nor container has changed. + containerCache simplelru.LRUCache[cid.ID, [][]netmapSDK.NodeInfo] +} + +// defaultContainerCacheSize is the default size for the container cache. +const defaultContainerCacheSize = 10 + +// NewContainerNodesCache creates new cache which saves the result of the ContainerNodes() invocations. +// If size is <= 0, defaultContainerCacheSize (10) is used. +func NewContainerNodesCache(size int) *ContainerNodesCache { + if size <= 0 { + size = defaultContainerCacheSize + } + + cache, _ := simplelru.NewLRU[cid.ID, [][]netmapSDK.NodeInfo](size, nil) // no error + return &ContainerNodesCache{ + containerCache: cache, + } +} + +// ContainerNodes returns the result of nm.ContainerNodes(), possibly from the cache. +func (c *ContainerNodesCache) ContainerNodes(nm *netmapSDK.NetMap, cnr cid.ID, p netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) { + c.mtx.Lock() + if nm.Epoch() == c.lastEpoch { + raw, ok := c.containerCache.Get(cnr) + c.mtx.Unlock() + if ok { + return raw, nil + } + } else { + c.lastEpoch = nm.Epoch() + c.containerCache.Purge() + c.mtx.Unlock() + } + + binCnr := make([]byte, sha256.Size) + cnr.Encode(binCnr) + + cn, err := nm.ContainerNodes(p, binCnr) + if err != nil { + return nil, fmt.Errorf("could not get container nodes: %w", err) + } + + c.mtx.Lock() + if c.lastEpoch == nm.Epoch() { + c.containerCache.Add(cnr, cn) + } + c.mtx.Unlock() + return cn, nil +} diff --git a/pkg/services/object_manager/placement/cache_test.go b/pkg/services/object_manager/placement/cache_test.go new file mode 100644 index 000000000..5816d59fe --- /dev/null +++ b/pkg/services/object_manager/placement/cache_test.go @@ -0,0 +1,95 @@ +package placement_test + +import ( + "strconv" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/stretchr/testify/require" +) + +func TestContainerNodesCache(t *testing.T) { + const size = 3 + + nodes := [6]netmapSDK.NodeInfo{} + for i := range nodes { + nodes[i].SetAttribute("ATTR", strconv.Itoa(i)) + } + + nm := func(epoch uint64, nodes []netmapSDK.NodeInfo) *netmapSDK.NetMap { + var nm netmapSDK.NetMap + nm.SetEpoch(epoch) + nm.SetNodes(nodes) + return &nm + } + + var pp netmapSDK.PlacementPolicy + require.NoError(t, pp.DecodeString("REP 1")) + + t.Run("update netmap on the new epoch", func(t *testing.T) { + c := placement.NewContainerNodesCache(size) + + cnr := cidtest.ID() + res, err := c.ContainerNodes(nm(1, nodes[0:1]), cnr, pp) + require.NoError(t, err) + + // Use other nodes in the argument to ensure the result is taken from cache. + resCached, err := c.ContainerNodes(nm(1, nodes[1:2]), cnr, pp) + require.NoError(t, err) + require.Equal(t, res, resCached) + + // Update epoch, netmap should be purged. + resCached, err = c.ContainerNodes(nm(2, nodes[2:3]), cnr, pp) + require.NoError(t, err) + require.NotEqual(t, res, resCached) + }) + t.Run("cache uses container as a key", func(t *testing.T) { + c := placement.NewContainerNodesCache(size) + + res1, err := c.ContainerNodes(nm(1, nodes[0:1]), cidtest.ID(), pp) + require.NoError(t, err) + + res2, err := c.ContainerNodes(nm(1, nodes[1:2]), cidtest.ID(), pp) + require.NoError(t, err) + + require.NotEqual(t, res1, res2) + }) + t.Run("cache respects size parameter", func(t *testing.T) { + c := placement.NewContainerNodesCache(size) + + nm1 := nm(1, nodes[0:1]) + nm2 := nm(1, nodes[1:2]) + cnr := [size * 2]cid.ID{} + res := [size * 2][][]netmapSDK.NodeInfo{} + for i := 0; i < size*2; i++ { + cnr[i] = cidtest.ID() + + var err error + res[i], err = c.ContainerNodes(nm1, cnr[i], pp) + require.NoError(t, err) + } + + for i := size; i < size*2; i++ { + r, err := c.ContainerNodes(nm2, cnr[i], pp) + require.NoError(t, err) + require.Equal(t, res[i], r) + } + for i := 0; i < size; i++ { + r, err := c.ContainerNodes(nm2, cnr[i], pp) + require.NoError(t, err) + require.NotEqual(t, res[i], r) + } + }) + t.Run("the error is propagated", func(t *testing.T) { + var pp netmapSDK.PlacementPolicy + require.NoError(t, pp.DecodeString("REP 1 SELECT 1 FROM X FILTER ATTR EQ 42 AS X")) + + c := placement.NewContainerNodesCache(size) + _, err := c.ContainerNodes(nm(1, nodes[0:1]), cidtest.ID(), pp) + require.Error(t, err) + }) + +} diff --git a/pkg/services/object_manager/placement/netmap.go b/pkg/services/object_manager/placement/netmap.go index 8163529ed..1782e27ea 100644 --- a/pkg/services/object_manager/placement/netmap.go +++ b/pkg/services/object_manager/placement/netmap.go @@ -3,24 +3,16 @@ package placement import ( "crypto/sha256" "fmt" - "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/hashicorp/golang-lru/v2/simplelru" ) type netMapBuilder struct { - nmSrc netmap.Source - // mtx protects lastEpoch and containerCache fields. - mtx sync.Mutex - // lastEpoch contains contains network map epoch for all values in the container cache. - lastEpoch uint64 - // containerCache caches container nodes by ID. It is used to skip `GetContainerNodes` invocation if - // neither netmap nor container has changed. - containerCache simplelru.LRUCache[cid.ID, [][]netmapSDK.NodeInfo] + nmSrc netmap.Source + containerCache *ContainerNodesCache } type netMapSrc struct { @@ -29,22 +21,17 @@ type netMapSrc struct { nm *netmapSDK.NetMap } -// defaultContainerCacheSize is the default size for the container cache. -const defaultContainerCacheSize = 10 - func NewNetworkMapBuilder(nm *netmapSDK.NetMap) Builder { - cache, _ := simplelru.NewLRU[cid.ID, [][]netmapSDK.NodeInfo](defaultContainerCacheSize, nil) // no error return &netMapBuilder{ nmSrc: &netMapSrc{nm: nm}, - containerCache: cache, + containerCache: NewContainerNodesCache(0), } } func NewNetworkMapSourceBuilder(nmSrc netmap.Source) Builder { - cache, _ := simplelru.NewLRU[cid.ID, [][]netmapSDK.NodeInfo](defaultContainerCacheSize, nil) // no error return &netMapBuilder{ nmSrc: nmSrc, - containerCache: cache, + containerCache: NewContainerNodesCache(0), } } @@ -58,33 +45,11 @@ func (b *netMapBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, p netmapSDK.Plac return nil, fmt.Errorf("could not get network map: %w", err) } - binCnr := make([]byte, sha256.Size) - cnr.Encode(binCnr) - - b.mtx.Lock() - if nm.Epoch() == b.lastEpoch { - raw, ok := b.containerCache.Get(cnr) - b.mtx.Unlock() - if ok { - return BuildObjectPlacement(nm, raw, obj) - } - } else { - b.lastEpoch = nm.Epoch() - b.containerCache.Purge() - b.mtx.Unlock() - } - - cn, err := nm.ContainerNodes(p, binCnr) + cn, err := b.containerCache.ContainerNodes(nm, cnr, p) if err != nil { - return nil, fmt.Errorf("could not get container nodes: %w", err) + return nil, err } - b.mtx.Lock() - if b.lastEpoch == nm.Epoch() { - b.containerCache.Add(cnr, cn) - } - b.mtx.Unlock() - return BuildObjectPlacement(nm, cn, obj) } From e5767c90021eb0383db2b6fcde4fb8fed8ce68d0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 16 Jul 2024 15:20:07 +0300 Subject: [PATCH 1230/1943] [#1250] *: Reformat proto filets with clang-format Signed-off-by: Evgenii Stratonikov --- pkg/services/control/ir/service.proto | 132 +++++++++++++------------- pkg/services/control/ir/types.proto | 24 ++--- pkg/services/tree/service.proto | 38 +++----- pkg/services/tree/types.proto | 14 +-- 4 files changed, 99 insertions(+), 109 deletions(-) diff --git a/pkg/services/control/ir/service.proto b/pkg/services/control/ir/service.proto index 967ff979e..fa58db568 100644 --- a/pkg/services/control/ir/service.proto +++ b/pkg/services/control/ir/service.proto @@ -6,106 +6,108 @@ import "pkg/services/control/ir/types.proto"; option go_package = "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/ir/control"; -// `ControlService` provides an interface for internal work with the Inner Ring node. +// `ControlService` provides an interface for internal work with the Inner Ring +// node. service ControlService { - // Performs health check of the IR node. - rpc HealthCheck (HealthCheckRequest) returns (HealthCheckResponse); - // Forces a new epoch to be signaled by the IR node with high probability. - rpc TickEpoch (TickEpochRequest) returns (TickEpochResponse); - // Forces a node removal to be signaled by the IR node with high probability. - rpc RemoveNode (RemoveNodeRequest) returns (RemoveNodeResponse); - // Forces a container removal to be signaled by the IR node with high probability. - rpc RemoveContainer(RemoveContainerRequest) returns (RemoveContainerResponse); + // Performs health check of the IR node. + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); + // Forces a new epoch to be signaled by the IR node with high probability. + rpc TickEpoch(TickEpochRequest) returns (TickEpochResponse); + // Forces a node removal to be signaled by the IR node with high probability. + rpc RemoveNode(RemoveNodeRequest) returns (RemoveNodeResponse); + // Forces a container removal to be signaled by the IR node with high + // probability. + rpc RemoveContainer(RemoveContainerRequest) returns (RemoveContainerResponse); } // Health check request. message HealthCheckRequest { - // Health check request body. - message Body {} + // Health check request body. + message Body {} - // Body of health check request message. - Body body = 1; + // Body of health check request message. + Body body = 1; - // Body signature. - // Should be signed by node key or one of - // the keys configured by the node. - Signature signature = 2; + // Body signature. + // Should be signed by node key or one of + // the keys configured by the node. + Signature signature = 2; } // Health check response. message HealthCheckResponse { - // Health check response body - message Body { - // Health status of IR node application. - HealthStatus health_status = 1; - } + // Health check response body + message Body { + // Health status of IR node application. + HealthStatus health_status = 1; + } - // Body of health check response message. - Body body = 1; + // Body of health check response message. + Body body = 1; - // Body signature. - Signature signature = 2; + // Body signature. + Signature signature = 2; } message TickEpochRequest { - message Body{ - // Valid until block value override. - uint32 vub = 1; - } + message Body { + // Valid until block value override. + uint32 vub = 1; + } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } message TickEpochResponse { - message Body{ - // Valid until block value for transaction. - uint32 vub = 1; - } + message Body { + // Valid until block value for transaction. + uint32 vub = 1; + } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } message RemoveNodeRequest { - message Body{ - bytes key = 1; - // Valid until block value override. - uint32 vub = 2; - } + message Body { + bytes key = 1; + // Valid until block value override. + uint32 vub = 2; + } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } message RemoveNodeResponse { - message Body{ - // Valid until block value for transaction. - uint32 vub = 1; - } + message Body { + // Valid until block value for transaction. + uint32 vub = 1; + } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } message RemoveContainerRequest { - message Body{ - bytes container_id = 1; - bytes owner = 2; - // Valid until block value override. - uint32 vub = 3; - } + message Body { + bytes container_id = 1; + bytes owner = 2; + // Valid until block value override. + uint32 vub = 3; + } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } message RemoveContainerResponse { - message Body{ - // Valid until block value for transaction. - uint32 vub = 1; - } + message Body { + // Valid until block value for transaction. + uint32 vub = 1; + } - Body body = 1; - Signature signature = 2; + Body body = 1; + Signature signature = 2; } diff --git a/pkg/services/control/ir/types.proto b/pkg/services/control/ir/types.proto index a6897fad1..9b6731cf8 100644 --- a/pkg/services/control/ir/types.proto +++ b/pkg/services/control/ir/types.proto @@ -6,24 +6,24 @@ option go_package = "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/ir/ // Signature of some message. message Signature { - // Public key used for signing. - bytes key = 1 [json_name = "key"]; + // Public key used for signing. + bytes key = 1 [ json_name = "key" ]; - // Binary signature. - bytes sign = 2 [json_name = "signature"]; + // Binary signature. + bytes sign = 2 [ json_name = "signature" ]; } // Health status of the IR application. enum HealthStatus { - // Undefined status, default value. - HEALTH_STATUS_UNDEFINED = 0; + // Undefined status, default value. + HEALTH_STATUS_UNDEFINED = 0; - // IR application is starting. - STARTING = 1; + // IR application is starting. + STARTING = 1; - // IR application is started and serves all services. - READY = 2; + // IR application is started and serves all services. + READY = 2; - // IR application is shutting down. - SHUTTING_DOWN = 3; + // IR application is shutting down. + SHUTTING_DOWN = 3; } diff --git a/pkg/services/tree/service.proto b/pkg/services/tree/service.proto index ec63d88ec..582e74847 100644 --- a/pkg/services/tree/service.proto +++ b/pkg/services/tree/service.proto @@ -28,25 +28,25 @@ service TreeService { // Otherwise, a request is denied. // Add adds new node to the tree. Invoked by a client. - rpc Add (AddRequest) returns (AddResponse); + rpc Add(AddRequest) returns (AddResponse); // AddByPath adds new node to the tree by path. Invoked by a client. - rpc AddByPath (AddByPathRequest) returns (AddByPathResponse); + rpc AddByPath(AddByPathRequest) returns (AddByPathResponse); // Remove removes node from the tree. Invoked by a client. - rpc Remove (RemoveRequest) returns (RemoveResponse); + rpc Remove(RemoveRequest) returns (RemoveResponse); // Move moves node from one parent to another. Invoked by a client. - rpc Move (MoveRequest) returns (MoveResponse); + rpc Move(MoveRequest) returns (MoveResponse); // GetNodeByPath returns list of IDs corresponding to a specific filepath. - rpc GetNodeByPath (GetNodeByPathRequest) returns (GetNodeByPathResponse); + rpc GetNodeByPath(GetNodeByPathRequest) returns (GetNodeByPathResponse); // GetSubTree returns tree corresponding to a specific node. - rpc GetSubTree (GetSubTreeRequest) returns (stream GetSubTreeResponse); + rpc GetSubTree(GetSubTreeRequest) returns (stream GetSubTreeResponse); // TreeList return list of the existing trees in the container. - rpc TreeList (TreeListRequest) returns (TreeListResponse); + rpc TreeList(TreeListRequest) returns (TreeListResponse); /* Synchronization API */ // Apply pushes log operation from another node to the current. // The request must be signed by a container node. - rpc Apply (ApplyRequest) returns (ApplyResponse); + rpc Apply(ApplyRequest) returns (ApplyResponse); // GetOpLog returns a stream of logged operations starting from some height. rpc GetOpLog(GetOpLogRequest) returns (stream GetOpLogResponse); // Healthcheck is a dummy rpc to check service availability @@ -85,7 +85,6 @@ message AddResponse { Signature signature = 2; }; - message AddByPathRequest { message Body { // Container ID in V2 format. @@ -122,7 +121,6 @@ message AddByPathResponse { Signature signature = 2; }; - message RemoveRequest { message Body { // Container ID in V2 format. @@ -142,8 +140,7 @@ message RemoveRequest { } message RemoveResponse { - message Body { - } + message Body {} // Response body. Body body = 1; @@ -151,7 +148,6 @@ message RemoveResponse { Signature signature = 2; }; - message MoveRequest { message Body { // TODO import neo.fs.v2.refs.ContainerID directly. @@ -176,8 +172,7 @@ message MoveRequest { } message MoveResponse { - message Body { - } + message Body {} // Response body. Body body = 1; @@ -185,7 +180,6 @@ message MoveResponse { Signature signature = 2; }; - message GetNodeByPathRequest { message Body { // Container ID in V2 format. @@ -235,7 +229,6 @@ message GetNodeByPathResponse { Signature signature = 2; }; - message GetSubTreeRequest { message Body { message Order { @@ -307,7 +300,6 @@ message TreeListResponse { Signature signature = 2; } - message ApplyRequest { message Body { // Container ID in V2 format. @@ -325,8 +317,7 @@ message ApplyRequest { } message ApplyResponse { - message Body { - } + message Body {} // Response body. Body body = 1; @@ -334,7 +325,6 @@ message ApplyResponse { Signature signature = 2; }; - message GetOpLogRequest { message Body { // Container ID in V2 format. @@ -366,8 +356,7 @@ message GetOpLogResponse { }; message HealthcheckResponse { - message Body { - } + message Body {} // Response body. Body body = 1; @@ -376,8 +365,7 @@ message HealthcheckResponse { }; message HealthcheckRequest { - message Body { - } + message Body {} // Request body. Body body = 1; diff --git a/pkg/services/tree/types.proto b/pkg/services/tree/types.proto index 23d73b9ad..f122c7cf4 100644 --- a/pkg/services/tree/types.proto +++ b/pkg/services/tree/types.proto @@ -10,25 +10,25 @@ option go_package = "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tre // KeyValue represents key-value pair attached to an object. message KeyValue { // Attribute name. - string key = 1 [json_name = "key"]; + string key = 1 [ json_name = "key" ]; // Attribute value. - bytes value = 2 [json_name = "value"]; + bytes value = 2 [ json_name = "value" ]; } // LogMove represents log-entry for a single move operation. message LogMove { // ID of the parent node. - uint64 parent_id = 1 [json_name = "parentID"]; + uint64 parent_id = 1 [ json_name = "parentID" ]; // Node meta information, including operation timestamp. - bytes meta = 2 [json_name = "meta"]; + bytes meta = 2 [ json_name = "meta" ]; // ID of the node to move. - uint64 child_id = 3 [json_name = "childID"]; + uint64 child_id = 3 [ json_name = "childID" ]; } // Signature of a message. message Signature { // Serialized public key as defined in FrostFS API. - bytes key = 1 [json_name = "key"]; + bytes key = 1 [ json_name = "key" ]; // Signature of a message body. - bytes sign = 2 [json_name = "signature"]; + bytes sign = 2 [ json_name = "signature" ]; } From 3940bc17c10473440cb78ea81553d8fe42d187a1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 10 Jul 2024 09:30:01 +0300 Subject: [PATCH 1231/1943] [#1251] pilorama: Allow traversing multiple branches in parallel Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/tree/move.go | 2 +- cmd/frostfs-cli/modules/tree/subtree.go | 15 +- pkg/local_object_storage/engine/tree.go | 5 +- pkg/local_object_storage/pilorama/boltdb.go | 113 +++--- pkg/local_object_storage/pilorama/forest.go | 49 +-- .../pilorama/forest_test.go | 34 +- pkg/local_object_storage/pilorama/heap.go | 4 +- .../pilorama/interface.go | 2 +- pkg/local_object_storage/pilorama/meta.go | 6 +- .../pilorama/multinode.go | 49 +++ .../pilorama/split_test.go | 57 +++ pkg/local_object_storage/shard/tree.go | 3 +- pkg/services/tree/getsubtree_test.go | 35 +- pkg/services/tree/service.go | 92 +++-- pkg/services/tree/service.pb.go | 326 +++++++++--------- pkg/services/tree/service.proto | 10 +- pkg/services/tree/service_frostfs.pb.go | 53 ++- pkg/services/tree/service_grpc.pb.go | 44 ++- 18 files changed, 571 insertions(+), 328 deletions(-) create mode 100644 pkg/local_object_storage/pilorama/multinode.go diff --git a/cmd/frostfs-cli/modules/tree/move.go b/cmd/frostfs-cli/modules/tree/move.go index 84b2fb80e..95516940c 100644 --- a/cmd/frostfs-cli/modules/tree/move.go +++ b/cmd/frostfs-cli/modules/tree/move.go @@ -66,7 +66,7 @@ func move(cmd *cobra.Command, _ []string) { Body: &tree.GetSubTreeRequest_Body{ ContainerId: rawCID, TreeId: tid, - RootId: nid, + RootId: []uint64{nid}, Depth: 1, BearerToken: bt, }, diff --git a/cmd/frostfs-cli/modules/tree/subtree.go b/cmd/frostfs-cli/modules/tree/subtree.go index 64cb351ec..e58a13fd6 100644 --- a/cmd/frostfs-cli/modules/tree/subtree.go +++ b/cmd/frostfs-cli/modules/tree/subtree.go @@ -68,7 +68,7 @@ func getSubTree(cmd *cobra.Command, _ []string) { Body: &tree.GetSubTreeRequest_Body{ ContainerId: rawCID, TreeId: tid, - RootId: rid, + RootId: []uint64{rid}, Depth: depth, BearerToken: bt, }, @@ -83,10 +83,15 @@ func getSubTree(cmd *cobra.Command, _ []string) { for ; err == nil; subtreeResp, err = resp.Recv() { b := subtreeResp.GetBody() - cmd.Printf("Node ID: %d\n", b.GetNodeId()) - - cmd.Println("\tParent ID: ", b.GetParentId()) - cmd.Println("\tTimestamp: ", b.GetTimestamp()) + if len(b.GetNodeId()) == 1 { + cmd.Printf("Node ID: %d\n", b.GetNodeId()) + cmd.Println("\tParent ID: ", b.GetParentId()) + cmd.Println("\tTimestamp: ", b.GetTimestamp()) + } else { + cmd.Printf("Node IDs: %v\n", b.GetNodeId()) + cmd.Println("\tParent IDs: ", b.GetParentId()) + cmd.Println("\tTimestamps: ", b.GetTimestamp()) + } if meta := b.GetMeta(); len(meta) > 0 { cmd.Println("\tMeta pairs: ") diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 8bacdba76..39122628f 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -210,18 +210,17 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree } // TreeSortedByFilename implements the pilorama.Forest interface. -func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node, last *string, count int) ([]pilorama.NodeInfo, *string, error) { +func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeSortedByFilename", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() var err error - var nodes []pilorama.NodeInfo + var nodes []pilorama.MultiNodeInfo var cursor *string for _, sh := range e.sortShards(cid) { nodes, cursor, err = sh.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index ee28df426..48363ceac 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -9,6 +9,7 @@ import ( "math/rand" "os" "path/filepath" + "slices" "sort" "strconv" "sync" @@ -990,23 +991,26 @@ func (t *boltForest) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID stri return m, parentID, metaerr.Wrap(err) } -func (t *boltForest) hasFewChildren(b *bbolt.Bucket, nodeID Node, threshold int) bool { +func (t *boltForest) hasFewChildren(b *bbolt.Bucket, nodeIDs MultiNode, threshold int) bool { key := make([]byte, 9) key[0] = 'c' - binary.LittleEndian.PutUint64(key[1:], nodeID) count := 0 - c := b.Cursor() - for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { - if count++; count > threshold { - return false + for _, nodeID := range nodeIDs { + binary.LittleEndian.PutUint64(key[1:], nodeID) + + c := b.Cursor() + for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { + if count++; count > threshold { + return false + } } } return true } // TreeSortedByFilename implements the Forest interface. -func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node, last *string, count int) ([]NodeInfo, *string, error) { +func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeIDs MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error) { var ( startedAt = time.Now() success = false @@ -1019,7 +1023,6 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() @@ -1030,6 +1033,9 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr if t.mode.NoMetabase() { return nil, last, ErrDegradedMode } + if len(nodeIDs) == 0 { + return nil, last, errors.New("empty node list") + } h := newHeap(last, count) key := make([]byte, 9) @@ -1048,20 +1054,22 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr // If the node is a leaf, we could scan all filenames in the tree. // To prevent this we first count the number of children: if it is less than // the number of nodes we need to return, fallback to TreeGetChildren() implementation. - if fewChildren = t.hasFewChildren(b, nodeID, count); fewChildren { + if fewChildren = t.hasFewChildren(b, nodeIDs, count); fewChildren { var err error - result, err = t.getChildren(b, nodeID) + result, err = t.getChildren(b, nodeIDs) return err } - t.fillSortedChildren(b, nodeID, h) + t.fillSortedChildren(b, nodeIDs, h) for info, ok := h.pop(); ok; info, ok = h.pop() { - childInfo, err := t.getChildInfo(b, key, info.id) - if err != nil { - return err + for _, id := range info.id { + childInfo, err := t.getChildInfo(b, key, id) + if err != nil { + return err + } + result = append(result, childInfo) } - result = append(result, childInfo) } return nil }) @@ -1074,11 +1082,15 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr if fewChildren { result = sortAndCut(result, last) } - if len(result) != 0 { - s := string(result[len(result)-1].Meta.GetAttr(AttributeFilename)) + res := mergeNodeInfos(result) + if len(res) > count { + res = res[:count] + } + if len(res) != 0 { + s := string(findAttr(res[len(res)-1].Meta, AttributeFilename)) last = &s } - return result, last, metaerr.Wrap(err) + return res, last, metaerr.Wrap(err) } func sortAndCut(result []NodeInfo, last *string) []NodeInfo { @@ -1109,37 +1121,56 @@ func (t *boltForest) getChildInfo(b *bbolt.Bucket, key []byte, childID Node) (No return childInfo, nil } -func (t *boltForest) fillSortedChildren(b *bbolt.Bucket, nodeID Node, h *fixedHeap) { +func (t *boltForest) fillSortedChildren(b *bbolt.Bucket, nodeIDs MultiNode, h *fixedHeap) { c := b.Cursor() prefix := internalKeyPrefix(nil, AttributeFilename) length := uint16(0) count := 0 + + var nodes []uint64 + var lastFilename *string for k, _ := c.Seek(prefix); len(k) > 0 && k[0] == 'i'; k, _ = c.Next() { if len(k) < len(prefix)+2+16 { continue } parentID := binary.LittleEndian.Uint64(k[len(k)-16:]) - if parentID != nodeID { + if !slices.Contains(nodeIDs, parentID) { continue } actualLength := binary.LittleEndian.Uint16(k[len(prefix):]) childID := binary.LittleEndian.Uint64(k[len(k)-8:]) filename := string(k[len(prefix)+2 : len(k)-16]) - processed := h.push(childID, filename) - if actualLength != length { - length = actualLength - count = 1 - } else if processed { - if count++; count > h.count { - length = actualLength + 1 - c.Seek(append(prefix, byte(length), byte(length>>8))) - c.Prev() // c.Next() will be performed by for loop + + if lastFilename == nil { + lastFilename = &filename + nodes = append(nodes, childID) + } else if *lastFilename == filename { + nodes = append(nodes, childID) + } else { + processed := h.push(nodes, *lastFilename) + nodes = MultiNode{childID} + lastFilename = &filename + if actualLength != length { + length = actualLength + count = 1 + } else if processed { + if count++; count > h.count { + lastFilename = nil + nodes = nil + length = actualLength + 1 + c.Seek(append(prefix, byte(length), byte(length>>8))) + c.Prev() // c.Next() will be performed by for loop + } } } } + + if len(nodes) != 0 && lastFilename != nil { + h.push(nodes, *lastFilename) + } } // TreeGetChildren implements the Forest interface. @@ -1179,28 +1210,30 @@ func (t *boltForest) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID b := treeRoot.Bucket(dataBucket) var err error - result, err = t.getChildren(b, nodeID) + result, err = t.getChildren(b, []Node{nodeID}) return err }) success = err == nil return result, metaerr.Wrap(err) } -func (t *boltForest) getChildren(b *bbolt.Bucket, nodeID Node) ([]NodeInfo, error) { +func (t *boltForest) getChildren(b *bbolt.Bucket, nodeIDs MultiNode) ([]NodeInfo, error) { var result []NodeInfo key := make([]byte, 9) - key[0] = 'c' - binary.LittleEndian.PutUint64(key[1:], nodeID) + for _, nodeID := range nodeIDs { + key[0] = 'c' + binary.LittleEndian.PutUint64(key[1:], nodeID) - c := b.Cursor() - for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { - childID := binary.LittleEndian.Uint64(k[9:]) - childInfo, err := t.getChildInfo(b, key, childID) - if err != nil { - return nil, err + c := b.Cursor() + for k, _ := c.Seek(key); len(k) == childrenKeySize && binary.LittleEndian.Uint64(k[1:]) == nodeID; k, _ = c.Next() { + childID := binary.LittleEndian.Uint64(k[9:]) + childInfo, err := t.getChildInfo(b, key, childID) + if err != nil { + return nil, err + } + result = append(result, childInfo) } - result = append(result, childInfo) } return result, nil } diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index b45a77b99..78503bada 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -156,7 +156,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string, } // TreeSortedByFilename implements the Forest interface. -func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeID Node, start *string, count int) ([]NodeInfo, *string, error) { +func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeIDs MultiNode, start *string, count int) ([]MultiNodeInfo, *string, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -166,40 +166,45 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI return nil, start, nil } - children := s.tree.getChildren(nodeID) - res := make([]NodeInfo, 0, len(children)) - for _, childID := range children { - var found bool - for _, kv := range s.infoMap[childID].Meta.Items { - if kv.Key == AttributeFilename { - found = true - break + var res []NodeInfo + + for _, nodeID := range nodeIDs { + children := s.tree.getChildren(nodeID) + for _, childID := range children { + var found bool + for _, kv := range s.infoMap[childID].Meta.Items { + if kv.Key == AttributeFilename { + found = true + break + } } + if !found { + continue + } + res = append(res, NodeInfo{ + ID: childID, + Meta: s.infoMap[childID].Meta, + ParentID: s.infoMap[childID].Parent, + }) } - if !found { - continue - } - res = append(res, NodeInfo{ - ID: childID, - Meta: s.infoMap[childID].Meta, - ParentID: s.infoMap[childID].Parent, - }) } if len(res) == 0 { - return res, start, nil + return nil, start, nil } sort.Slice(res, func(i, j int) bool { return bytes.Compare(res[i].Meta.GetAttr(AttributeFilename), res[j].Meta.GetAttr(AttributeFilename)) == -1 }) - for i := range res { - if start == nil || string(res[i].Meta.GetAttr(AttributeFilename)) > *start { + + r := mergeNodeInfos(res) + for i := range r { + if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > *start { finish := i + count if len(res) < finish { finish = len(res) } - last := string(res[finish-1].Meta.GetAttr(AttributeFilename)) - return res[i:finish], &last, nil + last := string(findAttr(r[finish-1].Meta, AttributeFilename)) + return r[i:finish], &last, nil } } last := string(res[len(res)-1].Meta.GetAttr(AttributeFilename)) diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 9da0177cc..001d095c8 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -215,7 +215,7 @@ func BenchmarkForestSortedIteration(b *testing.B) { b.Run(providers[i].name+",root", func(b *testing.B) { for i := 0; i < b.N; i++ { - res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, RootID, nil, 100) + res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, MultiNode{RootID}, nil, 100) if err != nil || len(res) != 100 { b.Fatalf("err %v, count %d", err, len(res)) } @@ -223,7 +223,7 @@ func BenchmarkForestSortedIteration(b *testing.B) { }) b.Run(providers[i].name+",leaf", func(b *testing.B) { for i := 0; i < b.N; i++ { - res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, 1, nil, 100) + res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, MultiNode{1}, nil, 100) if err != nil || len(res) != 0 { b.FailNow() } @@ -266,9 +266,9 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { treeAdd(t, i+1, strconv.Itoa(i+1)) } - var result []NodeInfo + var result []MultiNodeInfo treeAppend := func(t *testing.T, last *string, count int) *string { - res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, RootID, last, count) + res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, last, count) require.NoError(t, err) result = append(result, res...) return cursor @@ -282,11 +282,11 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { require.Len(t, result, count) for i := range result { - require.Equal(t, RootID+uint64(i+1), result[i].ID) + require.Equal(t, MultiNode{RootID + uint64(i+1)}, result[i].Children) if i == 0 { - require.Equal(t, "", string(result[i].Meta.GetAttr(AttributeFilename))) + require.Equal(t, "", string(findAttr(result[i].Meta, AttributeFilename))) } else { - require.Equal(t, strconv.Itoa(RootID+i+1), string(result[i].Meta.GetAttr(AttributeFilename))) + require.Equal(t, strconv.Itoa(RootID+i+1), string(findAttr(result[i].Meta, AttributeFilename))) } } } @@ -318,12 +318,12 @@ func testForestTreeSortedByFilename(t *testing.T, s ForestStorage) { require.NoError(t, err) } - expectAttributes := func(t *testing.T, attr string, expected []string, res []NodeInfo) { + expectAttributes := func(t *testing.T, attr string, expected []string, res []MultiNodeInfo) { require.Equal(t, len(expected), len(res)) actual := make([]string, len(res)) for i := range actual { - actual[i] = string(res[i].Meta.GetAttr(attr)) + actual[i] = string(findAttr(res[i].Meta, attr)) } require.Equal(t, expected, actual) } @@ -345,40 +345,40 @@ func testForestTreeSortedByFilename(t *testing.T, s ForestStorage) { treeAddByPath(t, items[i]) } - getChildren := func(t *testing.T, id Node) []NodeInfo { + getChildren := func(t *testing.T, id MultiNode) []MultiNodeInfo { res, _, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, id, nil, len(items)) require.NoError(t, err) return res } - res := getChildren(t, RootID) + res := getChildren(t, MultiNode{RootID}) expectAttributes(t, AttributeFilename, []string{"a", "b", "c"}, res) expectAttributes(t, controlAttr, []string{"", "", "c"}, res) { - ra := getChildren(t, res[0].ID) + ra := getChildren(t, res[0].Children) expectAttributes(t, AttributeFilename, []string{"bbb"}, ra) expectAttributes(t, controlAttr, []string{""}, ra) - rabbb := getChildren(t, ra[0].ID) + rabbb := getChildren(t, ra[0].Children) expectAttributes(t, AttributeFilename, []string{"ccc", "xxx", "z"}, rabbb) expectAttributes(t, controlAttr, []string{"a/bbb/ccc", "a/bbb/xxx", "a/bbb/z"}, rabbb) } { - rb := getChildren(t, res[1].ID) + rb := getChildren(t, res[1].Children) expectAttributes(t, AttributeFilename, []string{"bbb", "xxx"}, rb) expectAttributes(t, controlAttr, []string{"", ""}, rb) - rbbbb := getChildren(t, rb[0].ID) + rbbbb := getChildren(t, rb[0].Children) expectAttributes(t, AttributeFilename, []string{"ccc"}, rbbbb) expectAttributes(t, controlAttr, []string{"b/bbb/ccc"}, rbbbb) - rbxxx := getChildren(t, rb[1].ID) + rbxxx := getChildren(t, rb[1].Children) expectAttributes(t, AttributeFilename, []string{"z"}, rbxxx) expectAttributes(t, controlAttr, []string{"b/xxx/z"}, rbxxx) } { - rc := getChildren(t, res[2].ID) + rc := getChildren(t, res[2].Children) require.Len(t, rc, 0) } } diff --git a/pkg/local_object_storage/pilorama/heap.go b/pkg/local_object_storage/pilorama/heap.go index e8a21e221..ec57b9e1f 100644 --- a/pkg/local_object_storage/pilorama/heap.go +++ b/pkg/local_object_storage/pilorama/heap.go @@ -5,7 +5,7 @@ import ( ) type heapInfo struct { - id Node + id MultiNode filename string } @@ -44,7 +44,7 @@ func newHeap(start *string, count int) *fixedHeap { } } -func (h *fixedHeap) push(id Node, filename string) bool { +func (h *fixedHeap) push(id MultiNode, filename string) bool { if h.start != nil && filename <= *h.start { return false } diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index f7f5a85b1..61a3849bf 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -35,7 +35,7 @@ type Forest interface { TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) // TreeSortedByFilename returns children of the node with the specified ID. The nodes are sorted by the filename attribute.. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. - TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node, last *string, count int) ([]NodeInfo, *string, error) + TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error) // TreeGetOpLog returns first log operation stored at or above the height. // In case no such operation is found, empty Move and nil error should be returned. TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) diff --git a/pkg/local_object_storage/pilorama/meta.go b/pkg/local_object_storage/pilorama/meta.go index 49b7f6477..45e9c2f79 100644 --- a/pkg/local_object_storage/pilorama/meta.go +++ b/pkg/local_object_storage/pilorama/meta.go @@ -21,7 +21,11 @@ func (x Meta) Bytes() []byte { } func (x Meta) GetAttr(name string) []byte { - for _, kv := range x.Items { + return findAttr(x.Items, name) +} + +func findAttr(ms []KeyValue, name string) []byte { + for _, kv := range ms { if kv.Key == name { return kv.Value } diff --git a/pkg/local_object_storage/pilorama/multinode.go b/pkg/local_object_storage/pilorama/multinode.go new file mode 100644 index 000000000..106ba6ae9 --- /dev/null +++ b/pkg/local_object_storage/pilorama/multinode.go @@ -0,0 +1,49 @@ +package pilorama + +import "bytes" + +// MultiNode represents a group of internal nodes accessible by the same path, but having different id. +type MultiNode []Node + +// MultiNodeInfo represents a group of internal nodes accessible by the same path, but having different id. +type MultiNodeInfo struct { + Children MultiNode + Parents MultiNode + Timestamps []uint64 + Meta []KeyValue +} + +func (r *MultiNodeInfo) Add(info NodeInfo) bool { + if !isInternal(info.Meta.Items) || !isInternal(r.Meta) || + !bytes.Equal(r.Meta[0].Value, info.Meta.Items[0].Value) { + return false + } + + r.Children = append(r.Children, info.ID) + r.Parents = append(r.Parents, info.ParentID) + r.Timestamps = append(r.Timestamps, info.Meta.Time) + return true +} + +func (n NodeInfo) ToMultiNode() MultiNodeInfo { + return MultiNodeInfo{ + Children: MultiNode{n.ID}, + Parents: MultiNode{n.ParentID}, + Timestamps: []uint64{n.Meta.Time}, + Meta: n.Meta.Items, + } +} + +func isInternal(m []KeyValue) bool { + return len(m) == 1 && m[0].Key == AttributeFilename +} + +func mergeNodeInfos(ns []NodeInfo) []MultiNodeInfo { + var r []MultiNodeInfo + for _, info := range ns { + if len(r) == 0 || !r[len(r)-1].Add(info) { + r = append(r, info.ToMultiNode()) + } + } + return r +} diff --git a/pkg/local_object_storage/pilorama/split_test.go b/pkg/local_object_storage/pilorama/split_test.go index 3bd581ce6..54c2b90a6 100644 --- a/pkg/local_object_storage/pilorama/split_test.go +++ b/pkg/local_object_storage/pilorama/split_test.go @@ -95,4 +95,61 @@ func testDuplicateDirectory(t *testing.T, f Forest) { require.Equal(t, []byte{4}, testGetByPath(t, "dir1/dir3/value3")) require.Equal(t, []byte{8}, testGetByPath(t, "dir1/dir3/value4")) require.Equal(t, []byte{10}, testGetByPath(t, "value0")) + + testSortedByFilename := func(t *testing.T, root MultiNode, last *string, batchSize int) ([]MultiNodeInfo, *string) { + res, last, err := f.TreeSortedByFilename(context.Background(), d.CID, treeID, root, last, batchSize) + require.NoError(t, err) + return res, last + } + + t.Run("test sorted listing, full children branch", func(t *testing.T) { + t.Run("big batch size", func(t *testing.T) { + res, _ := testSortedByFilename(t, MultiNode{RootID}, nil, 10) + require.Equal(t, 3, len(res)) + require.Equal(t, MultiNode{1, 5}, res[0].Children) + require.Equal(t, MultiNode{9}, res[1].Children) + require.Equal(t, MultiNode{10}, res[2].Children) + + t.Run("multi-root", func(t *testing.T) { + res, _ := testSortedByFilename(t, MultiNode{1, 5}, nil, 10) + require.Equal(t, 3, len(res)) + require.Equal(t, MultiNode{3, 7}, res[0].Children) + require.Equal(t, MultiNode{2}, res[1].Children) + require.Equal(t, MultiNode{6}, res[2].Children) + }) + }) + t.Run("small batch size", func(t *testing.T) { + res, last := testSortedByFilename(t, MultiNode{RootID}, nil, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{1, 5}, res[0].Children) + + res, last = testSortedByFilename(t, MultiNode{RootID}, last, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{9}, res[0].Children) + + res, last = testSortedByFilename(t, MultiNode{RootID}, last, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{10}, res[0].Children) + + res, _ = testSortedByFilename(t, MultiNode{RootID}, last, 1) + require.Equal(t, 0, len(res)) + + t.Run("multi-root", func(t *testing.T) { + res, last := testSortedByFilename(t, MultiNode{1, 5}, nil, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{3, 7}, res[0].Children) + + res, last = testSortedByFilename(t, MultiNode{1, 5}, last, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{2}, res[0].Children) + + res, last = testSortedByFilename(t, MultiNode{1, 5}, last, 1) + require.Equal(t, 1, len(res)) + require.Equal(t, MultiNode{6}, res[0].Children) + + res, _ = testSortedByFilename(t, MultiNode{RootID}, last, 1) + require.Equal(t, 0, len(res)) + }) + }) + }) } diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index e92a61e5b..26dc8ec1e 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -184,13 +184,12 @@ func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID strin } // TreeSortedByFilename implements the pilorama.Forest interface. -func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.Node, last *string, count int) ([]pilorama.NodeInfo, *string, error) { +func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeSortedByFilename", trace.WithAttributes( attribute.String("shard_id", s.ID().String()), attribute.String("container_id", cid.EncodeToString()), attribute.String("tree_id", treeID), - attribute.String("node_id", strconv.FormatUint(nodeID, 10)), ), ) defer span.End() diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index 305c2bac9..677431889 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -48,7 +48,7 @@ func TestGetSubTree(t *testing.T) { acc := subTreeAcc{errIndex: errIndex} err := getSubTree(context.Background(), &acc, d.CID, &GetSubTreeRequest_Body{ TreeId: treeID, - RootId: rootID, + RootId: []uint64{rootID}, Depth: depth, }, p) if errIndex == -1 { @@ -58,12 +58,12 @@ func TestGetSubTree(t *testing.T) { } // GetSubTree must return child only after is has returned the parent. - require.Equal(t, rootID, acc.seen[0].Body.NodeId) + require.Equal(t, rootID, acc.seen[0].Body.NodeId[0]) loop: for i := 1; i < len(acc.seen); i++ { parent := acc.seen[i].Body.ParentId for j := 0; j < i; j++ { - if acc.seen[j].Body.NodeId == parent { + if acc.seen[j].Body.NodeId[0] == parent[0] { continue loop } } @@ -73,16 +73,16 @@ func TestGetSubTree(t *testing.T) { // GetSubTree must return valid meta. for i := range acc.seen { b := acc.seen[i].Body - meta, node, err := p.TreeGetMeta(context.Background(), d.CID, treeID, b.NodeId) + meta, node, err := p.TreeGetMeta(context.Background(), d.CID, treeID, b.NodeId[0]) require.NoError(t, err) - require.Equal(t, node, b.ParentId) - require.Equal(t, meta.Time, b.Timestamp) + require.Equal(t, node, b.ParentId[0]) + require.Equal(t, meta.Time, b.Timestamp[0]) require.Equal(t, metaToProto(meta.Items), b.Meta) } ordered := make([]uint64, len(acc.seen)) for i := range acc.seen { - ordered[i] = acc.seen[i].Body.NodeId + ordered[i] = acc.seen[i].Body.NodeId[0] } return ordered } @@ -184,7 +184,7 @@ func testGetSubTreeOrderAsc(t *testing.T, p pilorama.ForestStorage) { } found := false for j := range tree { - if acc.seen[i].Body.NodeId == tree[j].id { + if acc.seen[i].Body.NodeId[0] == tree[j].id { found = true paths = append(paths, path.Join(tree[j].path...)) } @@ -205,7 +205,7 @@ func testGetSubTreeOrderAsc(t *testing.T, p pilorama.ForestStorage) { }, p) require.NoError(t, err) require.Len(t, acc.seen, 1) - require.Equal(t, uint64(0), acc.seen[0].Body.NodeId) + require.Equal(t, uint64(0), acc.seen[0].Body.NodeId[0]) }) t.Run("depth=2", func(t *testing.T) { acc := subTreeAcc{errIndex: -1} @@ -218,15 +218,16 @@ func testGetSubTreeOrderAsc(t *testing.T, p pilorama.ForestStorage) { }, p) require.NoError(t, err) require.Len(t, acc.seen, 3) - require.Equal(t, uint64(0), acc.seen[0].Body.NodeId) - require.Equal(t, uint64(0), acc.seen[1].GetBody().GetParentId()) - require.Equal(t, uint64(0), acc.seen[2].GetBody().GetParentId()) + require.Equal(t, uint64(0), acc.seen[0].Body.NodeId[0]) + require.Equal(t, uint64(0), acc.seen[1].GetBody().GetParentId()[0]) + require.Equal(t, uint64(0), acc.seen[2].GetBody().GetParentId()[0]) }) } var ( errSubTreeSend = errors.New("send finished with error") errSubTreeSendAfterError = errors.New("send was invoked after an error occurred") + errInvalidResponse = errors.New("send got invalid response") ) type subTreeAcc struct { @@ -239,6 +240,16 @@ type subTreeAcc struct { var _ TreeService_GetSubTreeServer = &subTreeAcc{} func (s *subTreeAcc) Send(r *GetSubTreeResponse) error { + b := r.GetBody() + if len(b.GetNodeId()) > 1 { + return errInvalidResponse + } + if len(b.GetParentId()) > 1 { + return errInvalidResponse + } + if len(b.GetTimestamp()) > 1 { + return errInvalidResponse + } s.seen = append(s.seen, r) if s.errIndex >= 0 { if len(s.seen) == s.errIndex+1 { diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 85f531466..2012f53d2 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -16,6 +16,8 @@ import ( netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // Service represents tree-service capable of working with multiple @@ -440,29 +442,50 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS return getSubTree(srv.Context(), srv, cid, b, s.forest) } +type stackItem struct { + values []pilorama.MultiNodeInfo + parent pilorama.MultiNode + last *string +} + func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { const batchSize = 1000 - type stackItem struct { - values []pilorama.NodeInfo - parent pilorama.Node - last *string + // For backward compatibility. + rootIDs := b.GetRootId() + if len(rootIDs) == 0 { + rootIDs = []uint64{0} } // Traverse the tree in a DFS manner. Because we need to support arbitrary depth, // recursive implementation is not suitable here, so we maintain explicit stack. - m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId()) - if err != nil { - return err + var ms []pilorama.KeyValue + var ps []uint64 + var ts []uint64 + for _, rootID := range rootIDs { + m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), rootID) + if err != nil { + return err + } + if ms == nil { + ms = m.Items + } else { + if len(m.Items) != 1 { + return status.Error(codes.InvalidArgument, "multiple non-internal nodes provided") + } + } + ts = append(ts, m.Time) + ps = append(ps, p) } stack := []stackItem{{ - values: []pilorama.NodeInfo{{ - ID: b.GetRootId(), - Meta: m, - ParentID: p, + values: []pilorama.MultiNodeInfo{{ + Children: rootIDs, + Timestamps: ts, + Meta: ms, + Parents: ps, }}, - parent: p, + parent: ps, }} for { @@ -486,30 +509,20 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid } } - node := stack[len(stack)-1].values[0] - stack[len(stack)-1].values = stack[len(stack)-1].values[1:] - - err = srv.Send(&GetSubTreeResponse{ - Body: &GetSubTreeResponse_Body{ - NodeId: node.ID, - ParentId: node.ParentID, - Timestamp: node.Meta.Time, - Meta: metaToProto(node.Meta.Items), - }, - }) + node, err := stackPopAndSend(stack, srv) if err != nil { return err } if b.GetDepth() == 0 || uint32(len(stack)) < b.GetDepth() { - children, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), node.ID, nil, batchSize) + children, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), node.Children, nil, batchSize) if err != nil { return err } if len(children) != 0 { stack = append(stack, stackItem{ values: children, - parent: node.ID, + parent: node.Children, last: last, }) } @@ -518,19 +531,38 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid return nil } +func stackPopAndSend(stack []stackItem, srv TreeService_GetSubTreeServer) (pilorama.MultiNodeInfo, error) { + node := stack[len(stack)-1].values[0] + stack[len(stack)-1].values = stack[len(stack)-1].values[1:] + + return node, srv.Send(&GetSubTreeResponse{ + Body: &GetSubTreeResponse_Body{ + NodeId: node.Children, + ParentId: node.Parents, + Timestamp: node.Timestamps, + Meta: metaToProto(node.Meta), + }, + }) +} + func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { if b.GetOrderBy().GetDirection() == GetSubTreeRequest_Body_Order_Asc { return getSortedSubTree(ctx, srv, cid, b, forest) } + var rootID uint64 + if len(b.GetRootId()) > 0 { + rootID = b.GetRootId()[0] + } + // Traverse the tree in a DFS manner. Because we need to support arbitrary depth, // recursive implementation is not suitable here, so we maintain explicit stack. - m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), b.GetRootId()) + m, p, err := forest.TreeGetMeta(ctx, cid, b.GetTreeId(), rootID) if err != nil { return err } stack := [][]pilorama.NodeInfo{{{ - ID: b.GetRootId(), + ID: rootID, Meta: m, ParentID: p, }}} @@ -548,9 +580,9 @@ func getSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSD err = srv.Send(&GetSubTreeResponse{ Body: &GetSubTreeResponse_Body{ - NodeId: node.ID, - ParentId: node.ParentID, - Timestamp: node.Meta.Time, + NodeId: []uint64{node.ID}, + ParentId: []uint64{node.ParentID}, + Timestamp: []uint64{node.Meta.Time}, Meta: metaToProto(node.Meta.Items), }, }) diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go index 7930361e1..f439e3f28 100644 --- a/pkg/services/tree/service.pb.go +++ b/pkg/services/tree/service.pb.go @@ -1979,8 +1979,8 @@ type GetSubTreeRequest_Body struct { ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` // The name of the tree. TreeId string `protobuf:"bytes,2,opt,name=tree_id,json=treeId,proto3" json:"tree_id,omitempty"` - // ID of the root node of a subtree. - RootId uint64 `protobuf:"varint,3,opt,name=root_id,json=rootId,proto3" json:"root_id,omitempty"` + // IDs of the root nodes of a subtree forest. + RootId []uint64 `protobuf:"varint,3,rep,name=root_id,json=rootId,proto3" json:"root_id,omitempty"` // Optional depth of the traversal. Zero means return only root. // Maximum depth is 10. Depth uint32 `protobuf:"varint,4,opt,name=depth,proto3" json:"depth,omitempty"` @@ -2036,11 +2036,11 @@ func (x *GetSubTreeRequest_Body) GetTreeId() string { return "" } -func (x *GetSubTreeRequest_Body) GetRootId() uint64 { +func (x *GetSubTreeRequest_Body) GetRootId() []uint64 { if x != nil { return x.RootId } - return 0 + return nil } func (x *GetSubTreeRequest_Body) GetDepth() uint32 { @@ -2117,11 +2117,11 @@ type GetSubTreeResponse_Body struct { unknownFields protoimpl.UnknownFields // ID of the node. - NodeId uint64 `protobuf:"varint,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + NodeId []uint64 `protobuf:"varint,1,rep,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` // ID of the parent. - ParentId uint64 `protobuf:"varint,2,opt,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"` + ParentId []uint64 `protobuf:"varint,2,rep,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"` // Time node was first added to a tree. - Timestamp uint64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + Timestamp []uint64 `protobuf:"varint,3,rep,name=timestamp,proto3" json:"timestamp,omitempty"` // Node meta-information. Meta []*KeyValue `protobuf:"bytes,4,rep,name=meta,proto3" json:"meta,omitempty"` } @@ -2158,25 +2158,25 @@ func (*GetSubTreeResponse_Body) Descriptor() ([]byte, []int) { return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{11, 0} } -func (x *GetSubTreeResponse_Body) GetNodeId() uint64 { +func (x *GetSubTreeResponse_Body) GetNodeId() []uint64 { if x != nil { return x.NodeId } - return 0 + return nil } -func (x *GetSubTreeResponse_Body) GetParentId() uint64 { +func (x *GetSubTreeResponse_Body) GetParentId() []uint64 { if x != nil { return x.ParentId } - return 0 + return nil } -func (x *GetSubTreeResponse_Body) GetTimestamp() uint64 { +func (x *GetSubTreeResponse_Body) GetTimestamp() []uint64 { if x != nil { return x.Timestamp } - return 0 + return nil } func (x *GetSubTreeResponse_Body) GetMeta() []*KeyValue { @@ -2742,173 +2742,175 @@ var file_pkg_services_tree_service_proto_rawDesc = []byte{ 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, - 0x73, 0x22, 0xbf, 0x03, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, + 0x73, 0x22, 0xc3, 0x03, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x30, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xc8, 0x02, 0x0a, 0x04, 0x42, 0x6f, 0x64, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xcc, 0x02, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x17, 0x0a, - 0x07, 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, - 0x72, 0x6f, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, 0x12, 0x21, 0x0a, 0x0c, - 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, - 0x3d, 0x0a, 0x08, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x5f, 0x62, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, - 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, - 0x4f, 0x72, 0x64, 0x65, 0x72, 0x52, 0x07, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x42, 0x79, 0x1a, 0x73, - 0x0a, 0x05, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x4a, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x74, 0x72, 0x65, - 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x2e, 0x44, - 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x22, 0x1e, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x73, - 0x63, 0x10, 0x01, 0x22, 0xf6, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, - 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, - 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x7e, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, - 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x22, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4b, 0x65, - 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x22, 0x9b, 0x01, 0x0a, - 0x0f, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x2e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x29, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0x8c, 0x01, 0x0a, 0x10, 0x54, - 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, - 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x18, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x64, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x69, 0x64, 0x73, 0x22, 0xdb, 0x01, 0x0a, 0x0c, 0x41, 0x70, - 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, - 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, - 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, - 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x6f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, - 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, - 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x09, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, - 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4c, 0x6f, 0x67, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x09, 0x6f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x74, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x6c, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, - 0x70, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, - 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xe2, 0x01, - 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, + 0x07, 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x04, 0x42, 0x02, + 0x10, 0x00, 0x52, 0x06, 0x72, 0x6f, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, + 0x70, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, + 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x12, 0x3d, 0x0a, 0x08, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x5f, 0x62, 0x79, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, + 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x2e, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x52, 0x07, 0x6f, 0x72, 0x64, 0x65, 0x72, + 0x42, 0x79, 0x1a, 0x73, 0x0a, 0x05, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x4a, 0x0a, 0x09, 0x64, + 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, + 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x4f, 0x72, 0x64, + 0x65, 0x72, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x64, 0x69, + 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x1e, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x07, + 0x0a, 0x03, 0x41, 0x73, 0x63, 0x10, 0x01, 0x22, 0x83, 0x02, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x53, + 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x74, + 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x1a, 0x8a, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x1b, 0x0a, 0x07, 0x6e, 0x6f, 0x64, + 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x04, 0x42, 0x02, 0x10, 0x00, 0x52, 0x06, + 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, + 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x04, 0x42, 0x02, 0x10, 0x00, 0x52, 0x08, 0x70, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x20, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x03, 0x28, 0x04, 0x42, 0x02, 0x10, 0x00, 0x52, 0x09, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x22, 0x0a, 0x04, 0x6d, 0x65, 0x74, + 0x61, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4b, + 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x22, 0x9b, 0x01, + 0x0a, 0x0f, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, + 0x1a, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x70, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, + 0x1a, 0x29, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, - 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, - 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, - 0x65, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x14, 0x0a, 0x05, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, - 0x6e, 0x74, 0x22, 0xa7, 0x01, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, - 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0x8c, 0x01, 0x0a, 0x10, + 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, + 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x1a, 0x18, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x64, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x69, 0x64, 0x73, 0x22, 0xdb, 0x01, 0x0a, 0x0c, 0x41, + 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, + 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x33, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x2b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4c, 0x6f, 0x67, 0x4d, 0x6f, 0x76, - 0x65, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x80, 0x01, 0x0a, - 0x13, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, - 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, - 0x7e, 0x0a, 0x12, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, - 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, - 0xd6, 0x04, 0x0a, 0x0b, 0x54, 0x72, 0x65, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, - 0x2a, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x12, 0x10, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, - 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, - 0x41, 0x64, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x09, 0x41, - 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x12, 0x16, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, - 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, - 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x06, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x12, 0x13, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, - 0x0a, 0x04, 0x4d, 0x6f, 0x76, 0x65, 0x12, 0x11, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4d, 0x6f, - 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x74, 0x72, 0x65, 0x65, - 0x2e, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, - 0x0d, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1a, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, - 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x74, 0x72, 0x65, - 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x53, 0x75, - 0x62, 0x54, 0x72, 0x65, 0x65, 0x12, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, - 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x39, 0x0a, 0x08, 0x54, 0x72, - 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x15, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, - 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, - 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x12, 0x12, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4f, 0x70, - 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, - 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x74, 0x72, 0x65, - 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, - 0x65, 0x63, 0x6b, 0x12, 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, - 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x2e, - 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, - 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, - 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x65, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x6f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x09, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, + 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4c, 0x6f, 0x67, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x09, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x74, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x6c, + 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x62, 0x6f, 0x64, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, + 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, + 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, + 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xe2, + 0x01, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x2e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, + 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x70, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, + 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, + 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x14, 0x0a, + 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x22, 0xa7, 0x01, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, + 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, + 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x33, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x2b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4c, 0x6f, 0x67, 0x4d, 0x6f, + 0x76, 0x65, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x80, 0x01, + 0x0a, 0x13, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, + 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x22, 0x7e, 0x0a, 0x12, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, + 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x32, 0xd6, 0x04, 0x0a, 0x0b, 0x54, 0x72, 0x65, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x2a, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x12, 0x10, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, + 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x74, 0x72, 0x65, 0x65, + 0x2e, 0x41, 0x64, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x09, + 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x12, 0x16, 0x2e, 0x74, 0x72, 0x65, 0x65, + 0x2e, 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, + 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x06, 0x52, 0x65, + 0x6d, 0x6f, 0x76, 0x65, 0x12, 0x13, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x74, 0x72, 0x65, 0x65, + 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x2d, 0x0a, 0x04, 0x4d, 0x6f, 0x76, 0x65, 0x12, 0x11, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4d, + 0x6f, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x74, 0x72, 0x65, + 0x65, 0x2e, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, + 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x12, + 0x1a, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, + 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x74, 0x72, + 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x53, + 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x12, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, + 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x39, 0x0a, 0x08, 0x54, + 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x15, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, + 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x12, + 0x12, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4f, + 0x70, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, + 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x74, 0x72, + 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, + 0x68, 0x65, 0x63, 0x6b, 0x12, 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, + 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, + 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, + 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, + 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, + 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x65, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/pkg/services/tree/service.proto b/pkg/services/tree/service.proto index 582e74847..88bf0bca4 100644 --- a/pkg/services/tree/service.proto +++ b/pkg/services/tree/service.proto @@ -242,8 +242,8 @@ message GetSubTreeRequest { bytes container_id = 1; // The name of the tree. string tree_id = 2; - // ID of the root node of a subtree. - uint64 root_id = 3; + // IDs of the root nodes of a subtree forest. + repeated uint64 root_id = 3 [ packed = false ]; // Optional depth of the traversal. Zero means return only root. // Maximum depth is 10. uint32 depth = 4; @@ -262,11 +262,11 @@ message GetSubTreeRequest { message GetSubTreeResponse { message Body { // ID of the node. - uint64 node_id = 1; + repeated uint64 node_id = 1 [ packed = false ]; // ID of the parent. - uint64 parent_id = 2; + repeated uint64 parent_id = 2 [ packed = false ]; // Time node was first added to a tree. - uint64 timestamp = 3; + repeated uint64 timestamp = 3 [ packed = false ]; // Node meta-information. repeated KeyValue meta = 4; } diff --git a/pkg/services/tree/service_frostfs.pb.go b/pkg/services/tree/service_frostfs.pb.go index 4b50af2aa..1a49c5c0c 100644 --- a/pkg/services/tree/service_frostfs.pb.go +++ b/pkg/services/tree/service_frostfs.pb.go @@ -2,6 +2,11 @@ package tree +import ( + binary "encoding/binary" + protowire "google.golang.org/protobuf/encoding/protowire" +) + import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" // StableSize returns the size of x in protobuf format. @@ -995,7 +1000,9 @@ func (x *GetSubTreeRequest_Body) StableSize() (size int) { } size += proto.BytesSize(1, x.ContainerId) size += proto.StringSize(2, x.TreeId) - size += proto.UInt64Size(3, x.RootId) + for i := range x.RootId { + size += protowire.SizeGroup(protowire.Number(3), protowire.SizeVarint(x.RootId[i])) + } size += proto.UInt32Size(4, x.Depth) size += proto.BytesSize(5, x.BearerToken) size += proto.NestedStructureSize(6, x.OrderBy) @@ -1020,7 +1027,13 @@ func (x *GetSubTreeRequest_Body) StableMarshal(buf []byte) []byte { var offset int offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) offset += proto.StringMarshal(2, buf[offset:], x.TreeId) - offset += proto.UInt64Marshal(3, buf[offset:], x.RootId) + for i := range x.RootId { + { + prefix := protowire.EncodeTag(protowire.Number(3), protowire.VarintType) + offset += binary.PutUvarint(buf[offset:], uint64(prefix)) + offset += binary.PutUvarint(buf[offset:], x.RootId[i]) + } + } offset += proto.UInt32Marshal(4, buf[offset:], x.Depth) offset += proto.BytesMarshal(5, buf[offset:], x.BearerToken) offset += proto.NestedStructureMarshal(6, buf[offset:], x.OrderBy) @@ -1089,9 +1102,15 @@ func (x *GetSubTreeResponse_Body) StableSize() (size int) { if x == nil { return 0 } - size += proto.UInt64Size(1, x.NodeId) - size += proto.UInt64Size(2, x.ParentId) - size += proto.UInt64Size(3, x.Timestamp) + for i := range x.NodeId { + size += protowire.SizeGroup(protowire.Number(1), protowire.SizeVarint(x.NodeId[i])) + } + for i := range x.ParentId { + size += protowire.SizeGroup(protowire.Number(2), protowire.SizeVarint(x.ParentId[i])) + } + for i := range x.Timestamp { + size += protowire.SizeGroup(protowire.Number(3), protowire.SizeVarint(x.Timestamp[i])) + } for i := range x.Meta { size += proto.NestedStructureSize(4, x.Meta[i]) } @@ -1114,9 +1133,27 @@ func (x *GetSubTreeResponse_Body) StableMarshal(buf []byte) []byte { buf = make([]byte, x.StableSize()) } var offset int - offset += proto.UInt64Marshal(1, buf[offset:], x.NodeId) - offset += proto.UInt64Marshal(2, buf[offset:], x.ParentId) - offset += proto.UInt64Marshal(3, buf[offset:], x.Timestamp) + for i := range x.NodeId { + { + prefix := protowire.EncodeTag(protowire.Number(1), protowire.VarintType) + offset += binary.PutUvarint(buf[offset:], uint64(prefix)) + offset += binary.PutUvarint(buf[offset:], x.NodeId[i]) + } + } + for i := range x.ParentId { + { + prefix := protowire.EncodeTag(protowire.Number(2), protowire.VarintType) + offset += binary.PutUvarint(buf[offset:], uint64(prefix)) + offset += binary.PutUvarint(buf[offset:], x.ParentId[i]) + } + } + for i := range x.Timestamp { + { + prefix := protowire.EncodeTag(protowire.Number(3), protowire.VarintType) + offset += binary.PutUvarint(buf[offset:], uint64(prefix)) + offset += binary.PutUvarint(buf[offset:], x.Timestamp[i]) + } + } for i := range x.Meta { offset += proto.NestedStructureMarshal(4, buf[offset:], x.Meta[i]) } diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index 63f96e11a..4c293a4c0 100644 --- a/pkg/services/tree/service_grpc.pb.go +++ b/pkg/services/tree/service_grpc.pb.go @@ -3,7 +3,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.3.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v4.25.0 // source: pkg/services/tree/service.proto @@ -18,8 +18,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 const ( TreeService_Add_FullMethodName = "/tree.TreeService/Add" @@ -70,8 +70,9 @@ func NewTreeServiceClient(cc grpc.ClientConnInterface) TreeServiceClient { } func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*AddResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AddResponse) - err := c.cc.Invoke(ctx, TreeService_Add_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Add_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -79,8 +80,9 @@ func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grp } func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, opts ...grpc.CallOption) (*AddByPathResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AddByPathResponse) - err := c.cc.Invoke(ctx, TreeService_AddByPath_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_AddByPath_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -88,8 +90,9 @@ func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, } func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts ...grpc.CallOption) (*RemoveResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RemoveResponse) - err := c.cc.Invoke(ctx, TreeService_Remove_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Remove_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -97,8 +100,9 @@ func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts } func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...grpc.CallOption) (*MoveResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MoveResponse) - err := c.cc.Invoke(ctx, TreeService_Move_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Move_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -106,8 +110,9 @@ func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...g } func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPathRequest, opts ...grpc.CallOption) (*GetNodeByPathResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetNodeByPathResponse) - err := c.cc.Invoke(ctx, TreeService_GetNodeByPath_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_GetNodeByPath_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -115,11 +120,12 @@ func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPath } func (c *treeServiceClient) GetSubTree(ctx context.Context, in *GetSubTreeRequest, opts ...grpc.CallOption) (TreeService_GetSubTreeClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], TreeService_GetSubTree_FullMethodName, opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], TreeService_GetSubTree_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &treeServiceGetSubTreeClient{stream} + x := &treeServiceGetSubTreeClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -147,8 +153,9 @@ func (x *treeServiceGetSubTreeClient) Recv() (*GetSubTreeResponse, error) { } func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, opts ...grpc.CallOption) (*TreeListResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(TreeListResponse) - err := c.cc.Invoke(ctx, TreeService_TreeList_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_TreeList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -156,8 +163,9 @@ func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, o } func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts ...grpc.CallOption) (*ApplyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ApplyResponse) - err := c.cc.Invoke(ctx, TreeService_Apply_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Apply_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -165,11 +173,12 @@ func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts .. } func (c *treeServiceClient) GetOpLog(ctx context.Context, in *GetOpLogRequest, opts ...grpc.CallOption) (TreeService_GetOpLogClient, error) { - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], TreeService_GetOpLog_FullMethodName, opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], TreeService_GetOpLog_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &treeServiceGetOpLogClient{stream} + x := &treeServiceGetOpLogClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -197,8 +206,9 @@ func (x *treeServiceGetOpLogClient) Recv() (*GetOpLogResponse, error) { } func (c *treeServiceClient) Healthcheck(ctx context.Context, in *HealthcheckRequest, opts ...grpc.CallOption) (*HealthcheckResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(HealthcheckResponse) - err := c.cc.Invoke(ctx, TreeService_Healthcheck_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TreeService_Healthcheck_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -373,7 +383,7 @@ func _TreeService_GetSubTree_Handler(srv interface{}, stream grpc.ServerStream) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(TreeServiceServer).GetSubTree(m, &treeServiceGetSubTreeServer{stream}) + return srv.(TreeServiceServer).GetSubTree(m, &treeServiceGetSubTreeServer{ServerStream: stream}) } type TreeService_GetSubTreeServer interface { @@ -430,7 +440,7 @@ func _TreeService_GetOpLog_Handler(srv interface{}, stream grpc.ServerStream) er if err := stream.RecvMsg(m); err != nil { return err } - return srv.(TreeServiceServer).GetOpLog(m, &treeServiceGetOpLogServer{stream}) + return srv.(TreeServiceServer).GetOpLog(m, &treeServiceGetOpLogServer{ServerStream: stream}) } type TreeService_GetOpLogServer interface { From 00a88b99368d7141380f65d9cb446f1e4771ba01 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 17 Jul 2024 09:54:03 +0300 Subject: [PATCH 1232/1943] [#1251] *: Run gofumpt Signed-off-by: Evgenii Stratonikov --- pkg/services/object_manager/placement/cache_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/services/object_manager/placement/cache_test.go b/pkg/services/object_manager/placement/cache_test.go index 5816d59fe..07e9340dc 100644 --- a/pkg/services/object_manager/placement/cache_test.go +++ b/pkg/services/object_manager/placement/cache_test.go @@ -91,5 +91,4 @@ func TestContainerNodesCache(t *testing.T) { _, err := c.ContainerNodes(nm(1, nodes[0:1]), cidtest.ID(), pp) require.Error(t, err) }) - } From fc383ea6ae3bfa98ade02a53bb591f5ad0cb4780 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 15 Jul 2024 18:03:43 +0300 Subject: [PATCH 1233/1943] [#1253] getSvc: Fix EC objects get Now EC objects assembling is performed concurrently. Also fixed issue with an error in case of getting EC object via non-container node. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 2 + pkg/services/object/get/assemble.go | 2 +- pkg/services/object/get/assembleec.go | 19 +- pkg/services/object/get/assemblerec.go | 358 +++++++++++++++++----- pkg/services/object/get/get.go | 13 +- pkg/services/object/get/get_test.go | 9 +- pkg/services/object/get/local.go | 4 +- pkg/services/object/get/remote.go | 68 ++-- pkg/services/object/get/request.go | 4 +- pkg/services/object/get/types.go | 51 ++- pkg/services/object/search/container.go | 2 +- pkg/services/object/search/search_test.go | 6 +- pkg/services/object/search/service.go | 3 +- pkg/services/object/util/placement.go | 12 +- 14 files changed, 429 insertions(+), 124 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 5a195f688..67f173f29 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -108,6 +108,7 @@ const ( GetUnableToGetPartECObject = "unable to get part of the erasure-encoded object" GetUnableToHeadPartECObject = "unable to head part of the erasure-encoded object" GetUnableToGetECObjectContainer = "unable to get container for erasure-coded object" + GetUnableToHeadPartsECObject = "unable to head parts of the erasure-encoded object" GetAssemblingSplittedObjectCompleted = "assembling splitted object completed" GetAssemblingECObjectCompleted = "assembling erasure-coded object completed" GetFailedToAssembleSplittedObject = "failed to assemble splitted object" @@ -123,6 +124,7 @@ const ( GetRequestedObjectIsVirtual = "requested object is virtual" GetRequestedObjectIsEC = "requested object is erasure-coded" GetRequestedRangeIsOutOfObjectBounds = "requested range is out of object bounds" + GetUnexpectedECObject = "failed to get EC object from node: expected EC info, but got full object" PutAdditionalContainerBroadcastFailure = "additional container broadcast failure" SearchReturnResultDirectly = "return result directly" SearchCouldNotConstructRemoteNodeClient = "could not construct remote node client" diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index 66c4580b0..ba6fddec5 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -140,7 +140,7 @@ func (r *request) getObjectWithIndependentRequest(ctx context.Context, prm Reque prm: prm, infoSplit: objectSDK.NewSplitInfo(), - infoEC: objectSDK.NewECInfo(), + infoEC: newECInfo(), log: r.log, } diff --git a/pkg/services/object/get/assembleec.go b/pkg/services/object/get/assembleec.go index 5c999929a..7bbd9ca1e 100644 --- a/pkg/services/object/get/assembleec.go +++ b/pkg/services/object/get/assembleec.go @@ -6,11 +6,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" ) func (r *request) assembleEC(ctx context.Context) { - if r.isRaw() { + if r.isRaw() && r.isLocal() { r.log.Debug(logs.GetCanNotAssembleTheObject) return } @@ -35,8 +36,14 @@ func (r *request) assembleEC(ctx context.Context) { r.log.Debug(logs.GetTryingToAssembleTheECObject) + // initialize epoch number + ok := r.initEpoch() + if !ok { + return + } + r.prm.common = r.prm.common.WithLocalOnly(false) - assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.containerSource, r.log) + assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.localStorage, r.containerSource, r.log, r.headOnly(), r.isRaw(), r.traverserGenerator, r.curProcEpoch) r.log.Debug(logs.GetAssemblingECObject, zap.Uint64("range_offset", r.ctxRange().GetOffset()), @@ -47,8 +54,8 @@ func (r *request) assembleEC(ctx context.Context) { zap.Uint64("range_length", r.ctxRange().GetLength()), ) - obj, err := assembler.Assemble(ctx, r.prm.objWriter, r.headOnly()) - if err != nil { + obj, err := assembler.Assemble(ctx, r.prm.objWriter) + if err != nil && !errors.As(err, new(*objectSDK.ECInfoError)) { r.log.Warn(logs.GetFailedToAssembleECObject, zap.Error(err), zap.Uint64("range_offset", r.ctxRange().GetOffset()), @@ -58,6 +65,7 @@ func (r *request) assembleEC(ctx context.Context) { var errRemoved *apistatus.ObjectAlreadyRemoved var errOutOfRange *apistatus.ObjectOutOfRange + var errECInfo *objectSDK.ECInfoError switch { default: @@ -73,5 +81,8 @@ func (r *request) assembleEC(ctx context.Context) { case errors.As(err, &errOutOfRange): r.status = statusOutOfRange r.err = errOutOfRange + case errors.As(err, &errECInfo): + r.status = statusEC + r.err = err } } diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index 4a624e467..a4021ee5e 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -2,11 +2,16 @@ package getsvc import ( "context" + "encoding/hex" + "errors" "fmt" + "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -16,61 +21,79 @@ import ( "golang.org/x/sync/errgroup" ) +var errECPartsRetrieveCompleted = errors.New("EC parts receive completed") + +type ecRemoteStorage interface { + getObjectFromNode(ctx context.Context, addr oid.Address, info client.NodeInfo) (*objectSDK.Object, error) + headObjectFromNode(ctx context.Context, addr oid.Address, info client.NodeInfo, raw bool) (*objectSDK.Object, error) +} + type assemblerec struct { - addr oid.Address - ecInfo *objectSDK.ECInfo - rng *objectSDK.Range - objGetter objectGetter - cs container.Source - log *logger.Logger + addr oid.Address + ecInfo *ecInfo + rng *objectSDK.Range + remoteStorage ecRemoteStorage + localStorage localStorage + cs container.Source + log *logger.Logger + head bool + raw bool + traverserGenerator traverserGenerator + epoch uint64 } func newAssemblerEC( addr oid.Address, - ecInfo *objectSDK.ECInfo, + ecInfo *ecInfo, rng *objectSDK.Range, - objGetter objectGetter, + remoteStorage ecRemoteStorage, + localStorage localStorage, cs container.Source, log *logger.Logger, + head bool, + raw bool, + tg traverserGenerator, + epoch uint64, ) *assemblerec { return &assemblerec{ - addr: addr, - rng: rng, - ecInfo: ecInfo, - objGetter: objGetter, - cs: cs, - log: log, + addr: addr, + rng: rng, + ecInfo: ecInfo, + remoteStorage: remoteStorage, + localStorage: localStorage, + cs: cs, + log: log, + head: head, + raw: raw, + traverserGenerator: tg, + epoch: epoch, } } // Assemble assembles erasure-coded object and writes it's content to ObjectWriter. // It returns parent object. -func (a *assemblerec) Assemble(ctx context.Context, writer ObjectWriter, headOnly bool) (*objectSDK.Object, error) { - if headOnly { +func (a *assemblerec) Assemble(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { + switch { + case a.raw: + err := a.reconstructRawError(ctx) + return nil, err + case a.head: return a.reconstructHeader(ctx, writer) - } else if a.rng != nil { + case a.rng != nil: return a.reconstructRange(ctx, writer) + default: + return a.reconstructObject(ctx, writer) } - return a.reconstructObject(ctx, writer) } -func (a *assemblerec) getConstructor() (*erasurecode.Constructor, error) { - cnt, err := a.cs.Get(a.addr.Container()) - if err != nil { - return nil, err - } - dataCount := policy.ECDataCount(cnt.Value.PlacementPolicy()) - parityCount := policy.ECParityCount(cnt.Value.PlacementPolicy()) +func (a *assemblerec) getConstructor(cnr *container.Container) (*erasurecode.Constructor, error) { + dataCount := policy.ECDataCount(cnr.Value.PlacementPolicy()) + parityCount := policy.ECParityCount(cnr.Value.PlacementPolicy()) return erasurecode.NewConstructor(dataCount, parityCount) } func (a *assemblerec) reconstructHeader(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { - parts := a.retrieveParts(ctx, true) - c, err := a.getConstructor() - if err != nil { - return nil, err - } - obj, err := c.ReconstructHeader(parts) + obj, err := a.reconstructObjectFromParts(ctx, true) if err == nil { return obj, writer.WriteHeader(ctx, obj) } @@ -78,12 +101,7 @@ func (a *assemblerec) reconstructHeader(ctx context.Context, writer ObjectWriter } func (a *assemblerec) reconstructRange(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { - parts := a.retrieveParts(ctx, false) - c, err := a.getConstructor() - if err != nil { - return nil, err - } - obj, err := c.Reconstruct(parts) + obj, err := a.reconstructObjectFromParts(ctx, false) if err != nil { return nil, err } @@ -101,12 +119,7 @@ func (a *assemblerec) reconstructRange(ctx context.Context, writer ObjectWriter) } func (a *assemblerec) reconstructObject(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { - parts := a.retrieveParts(ctx, false) - c, err := a.getConstructor() - if err != nil { - return nil, err - } - obj, err := c.Reconstruct(parts) + obj, err := a.reconstructObjectFromParts(ctx, false) if err == nil { err = writer.WriteHeader(ctx, obj.CutPayload()) if err == nil { @@ -119,41 +132,238 @@ func (a *assemblerec) reconstructObject(ctx context.Context, writer ObjectWriter return obj, err } -func (a *assemblerec) retrieveParts(mainCtx context.Context, headOnly bool) []*objectSDK.Object { - parts := make([]*objectSDK.Object, int(a.ecInfo.Chunks[0].Total)) - errGroup, ctx := errgroup.WithContext(mainCtx) +func (a *assemblerec) reconstructObjectFromParts(ctx context.Context, headers bool) (*objectSDK.Object, error) { + objID := a.addr.Object() + trav, cnr, err := a.traverserGenerator.GenerateTraverser(a.addr.Container(), &objID, a.epoch) + if err != nil { + return nil, err + } + c, err := a.getConstructor(cnr) + if err != nil { + return nil, err + } + parts := a.retrieveParts(ctx, trav, cnr) + if headers { + return c.ReconstructHeader(parts) + } + return c.Reconstruct(parts) +} - for i := range a.ecInfo.Chunks { - chunk := a.ecInfo.Chunks[i] - errGroup.Go(func() error { - objID := new(oid.ID) - err := objID.ReadFromV2(chunk.ID) - if err != nil { - return fmt.Errorf("invalid object ID: %w", err) - } - var obj *objectSDK.Object - if headOnly { - obj, err = a.objGetter.HeadObject(ctx, *objID) - if err != nil { - a.log.Debug(logs.GetUnableToHeadPartECObject, zap.Stringer("part_id", objID), zap.Error(err)) - return nil - } - } else { - sow := NewSimpleObjectWriter() - obj, err = a.objGetter.GetObjectAndWritePayload(ctx, *objID, nil, sow) - if err != nil { - a.log.Debug(logs.GetUnableToGetPartECObject, zap.Stringer("part_id", objID), zap.Error(err)) - return nil - } - obj.SetPayload(sow.pld) - } - parts[chunk.Index] = obj - return nil - }) +func (a *assemblerec) reconstructRawError(ctx context.Context) error { + chunks := make(map[string]objectSDK.ECChunk) + var chunksGuard sync.Mutex + for _, ch := range a.ecInfo.localChunks { + chunks[string(ch.ID.GetValue())] = ch } - if err := errGroup.Wait(); err != nil { + objID := a.addr.Object() + trav, _, err := a.traverserGenerator.GenerateTraverser(a.addr.Container(), &objID, a.epoch) + if err != nil { + return err + } + + eg, ctx := errgroup.WithContext(ctx) + for { + batch := trav.Next() + if len(batch) == 0 { + break + } + for _, node := range batch { + var info client.NodeInfo + client.NodeInfoFromNetmapElement(&info, node) + eg.Go(func() error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if _, found := a.ecInfo.remoteChunks[string(info.PublicKey())]; found { + return nil + } + + nodeChunks := a.tryGetChunkListFromNode(ctx, info) + + chunksGuard.Lock() + defer chunksGuard.Unlock() + for _, ch := range nodeChunks { + chunks[string(ch.ID.GetValue())] = ch + } + return nil + }) + } + } + if err = eg.Wait(); err != nil { + return err + } + return createECInfoErr(chunks) +} + +func (a *assemblerec) retrieveParts(ctx context.Context, trav *placement.Traverser, cnr *container.Container) []*objectSDK.Object { + dataCount := policy.ECDataCount(cnr.Value.PlacementPolicy()) + parityCount := policy.ECParityCount(cnr.Value.PlacementPolicy()) + + remoteNodes := make([]placement.Node, 0) + for { + batch := trav.Next() + if len(batch) == 0 { + break + } + remoteNodes = append(remoteNodes, batch...) + } + + parts, err := a.processECNodesRequests(ctx, remoteNodes, dataCount, parityCount) + if err != nil { a.log.Debug(logs.GetUnableToGetAllPartsECObject, zap.Error(err)) } return parts } + +func (a *assemblerec) processECNodesRequests(ctx context.Context, nodes []placement.Node, dataCount, parityCount int) ([]*objectSDK.Object, error) { + foundChunks := make(map[uint32]*objectSDK.Object) + var foundChunksGuard sync.Mutex + eg, ctx := errgroup.WithContext(ctx) + eg.SetLimit(dataCount) + + for _, ch := range a.ecInfo.localChunks { + ch := ch + eg.Go(func() error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + object := a.tryGetChunkFromLocalStorage(ctx, ch) + if object == nil { + return nil + } + foundChunksGuard.Lock() + foundChunks[ch.Index] = object + count := len(foundChunks) + foundChunksGuard.Unlock() + if count >= dataCount { + return errECPartsRetrieveCompleted + } + return nil + }) + } + + for _, node := range nodes { + var info client.NodeInfo + client.NodeInfoFromNetmapElement(&info, node) + eg.Go(func() error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + chunks := a.tryGetChunkListFromNode(ctx, info) + for _, ch := range chunks { + object := a.tryGetChunkFromRemoteStorage(ctx, info, ch) + if object == nil { + continue + } + foundChunksGuard.Lock() + foundChunks[ch.Index] = object + count := len(foundChunks) + foundChunksGuard.Unlock() + if count >= dataCount { + return errECPartsRetrieveCompleted + } + } + return nil + }) + } + err := eg.Wait() + if err == nil || errors.Is(err, errECPartsRetrieveCompleted) { + parts := make([]*objectSDK.Object, dataCount+parityCount) + for idx, chunk := range foundChunks { + parts[idx] = chunk + } + return parts, nil + } + return nil, err +} + +func (a *assemblerec) tryGetChunkFromLocalStorage(ctx context.Context, ch objectSDK.ECChunk) *objectSDK.Object { + var objID oid.ID + err := objID.ReadFromV2(ch.ID) + if err != nil { + a.log.Error(logs.GetUnableToHeadPartECObject, zap.String("node", "local"), zap.Uint32("part_index", ch.Index), zap.Error(fmt.Errorf("invalid object ID: %w", err))) + return nil + } + var addr oid.Address + addr.SetContainer(addr.Container()) + addr.SetObject(objID) + var object *objectSDK.Object + if a.head { + object, err = a.localStorage.Head(ctx, addr, false) + if err != nil { + a.log.Warn(logs.GetUnableToHeadPartECObject, zap.String("node", "local"), zap.Stringer("part_id", objID), zap.Error(err)) + return nil + } + } else { + object, err = a.localStorage.Get(ctx, addr) + if err != nil { + a.log.Warn(logs.GetUnableToGetPartECObject, zap.String("node", "local"), zap.Stringer("part_id", objID), zap.Error(err)) + return nil + } + } + return object +} + +func (a *assemblerec) tryGetChunkListFromNode(ctx context.Context, node client.NodeInfo) []objectSDK.ECChunk { + if chunks, found := a.ecInfo.remoteChunks[string(node.PublicKey())]; found { + return chunks + } + var errECInfo *objectSDK.ECInfoError + _, err := a.remoteStorage.headObjectFromNode(ctx, a.addr, node, true) + if err == nil { + a.log.Error(logs.GetUnexpectedECObject, zap.String("node", hex.EncodeToString(node.PublicKey()))) + return nil + } + if !errors.As(err, &errECInfo) { + a.log.Warn(logs.GetUnableToHeadPartsECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Error(err)) + return nil + } + result := make([]objectSDK.ECChunk, 0, len(errECInfo.ECInfo().Chunks)) + for _, ch := range errECInfo.ECInfo().Chunks { + result = append(result, objectSDK.ECChunk(ch)) + } + return result +} + +func (a *assemblerec) tryGetChunkFromRemoteStorage(ctx context.Context, node client.NodeInfo, ch objectSDK.ECChunk) *objectSDK.Object { + var objID oid.ID + err := objID.ReadFromV2(ch.ID) + if err != nil { + a.log.Error(logs.GetUnableToHeadPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Uint32("part_index", ch.Index), zap.Error(fmt.Errorf("invalid object ID: %w", err))) + return nil + } + var addr oid.Address + addr.SetContainer(a.addr.Container()) + addr.SetObject(objID) + var object *objectSDK.Object + if a.head { + object, err = a.remoteStorage.headObjectFromNode(ctx, addr, node, false) + if err != nil { + a.log.Warn(logs.GetUnableToHeadPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Stringer("part_id", objID), zap.Error(err)) + return nil + } + } else { + object, err = a.remoteStorage.getObjectFromNode(ctx, addr, node) + if err != nil { + a.log.Warn(logs.GetUnableToGetPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Stringer("part_id", objID), zap.Error(err)) + return nil + } + } + return object +} + +func createECInfoErr(chunks map[string]objectSDK.ECChunk) *objectSDK.ECInfoError { + info := objectSDK.NewECInfo() + for _, ch := range chunks { + info.AddChunk(ch) + } + return objectSDK.NewECInfoError(info) +} diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 9738935d2..5a57bc56e 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -77,7 +77,7 @@ func (s *Service) get(ctx context.Context, prm RequestParameters) error { prm: prm, infoSplit: objectSDK.NewSplitInfo(), - infoEC: objectSDK.NewECInfo(), + infoEC: newECInfo(), log: s.log, } @@ -110,15 +110,8 @@ func (exec *request) analyzeStatus(ctx context.Context, execCnr bool) { case statusOutOfRange: exec.log.Debug(logs.GetRequestedRangeIsOutOfObjectBounds) case statusEC: - if !exec.isLocal() { - if execCnr { - exec.executeOnContainer(ctx) - exec.analyzeStatus(ctx, false) - } else { - exec.log.Debug(logs.GetRequestedObjectIsEC) - exec.assembleEC(ctx) - } - } + exec.log.Debug(logs.GetRequestedObjectIsEC) + exec.assembleEC(ctx) default: exec.log.Debug(logs.OperationFinishedWithError, zap.Error(exec.err), diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 988cd6982..29a15ba78 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -11,6 +11,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" @@ -78,7 +79,7 @@ func newTestStorage() *testStorage { } } -func (g *testTraverserGenerator) GenerateTraverser(cnr cid.ID, obj *oid.ID, e uint64) (*placement.Traverser, error) { +func (g *testTraverserGenerator) GenerateTraverser(cnr cid.ID, obj *oid.ID, e uint64) (*placement.Traverser, *containerCore.Container, error) { opts := make([]placement.Option, 0, 4) opts = append(opts, placement.ForContainer(g.c), @@ -90,7 +91,10 @@ func (g *testTraverserGenerator) GenerateTraverser(cnr cid.ID, obj *oid.ID, e ui opts = append(opts, placement.ForObject(*obj)) } - return placement.NewTraverser(opts...) + t, err := placement.NewTraverser(opts...) + return t, &containerCore.Container{ + Value: g.c, + }, err } func (p *testPlacementBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, _ netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { @@ -474,6 +478,7 @@ func testNodeMatrix(t testing.TB, dim []int) ([][]netmap.NodeInfo, [][]string) { var ni netmap.NodeInfo ni.SetNetworkEndpoints(a) + ni.SetPublicKey([]byte(a)) var na network.AddressGroup diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index fcfc9befc..1cd5e549c 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -5,7 +5,6 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -46,8 +45,7 @@ func (r *request) executeLocal(ctx context.Context) { r.err = objectSDK.NewSplitInfoError(r.infoSplit) case errors.As(err, &errECInfo): r.status = statusEC - util.MergeECInfo(errECInfo.ECInfo(), r.infoEC) - r.err = objectSDK.NewECInfoError(r.infoEC) + r.err = r.infoEC.addLocal(errECInfo.ECInfo()) case errors.As(err, &errOutOfRange): r.status = statusOutOfRange r.err = errOutOfRange diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index 302a4a4bc..4dee15242 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -7,11 +7,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" ) @@ -36,11 +35,6 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { switch { default: r.log.Debug(logs.GetRemoteCallFailed, zap.Error(err)) - if r.status == statusEC { - // we need to continue getting another chunks from another nodes - // in case of network issue - return false - } r.status = statusUndefined r.err = new(apistatus.ObjectNotFound) case err == nil: @@ -66,18 +60,7 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { r.err = objectSDK.NewSplitInfoError(r.infoSplit) case errors.As(err, &errECInfo): r.status = statusEC - util.MergeECInfo(r.infoEC, errECInfo.ECInfo()) - r.infoEC = errECInfo.ECInfo() - r.err = objectSDK.NewECInfoError(r.infoEC) - if r.isRaw() { - return len(r.infoEC.Chunks) == int(r.infoEC.Chunks[0].Total) - } - cnt, err := r.containerSource.Get(r.address().Container()) - if err == nil { - return len(r.infoEC.Chunks) == policy.ECDataCount(cnt.Value.PlacementPolicy()) - } - r.log.Debug(logs.GetUnableToGetECObjectContainer, zap.Error(err)) - return len(r.infoEC.Chunks) == int(r.infoEC.Chunks[0].Total) + r.err = r.infoEC.addRemote(string(info.PublicKey()), errECInfo.ECInfo()) } return r.status != statusUndefined @@ -116,3 +99,50 @@ func (r *request) getRemote(ctx context.Context, rs remoteStorage, info client.N return rs.Get(ctx, r.address(), prm) } + +func (r *request) getObjectFromNode(ctx context.Context, addr oid.Address, info client.NodeInfo) (*objectSDK.Object, error) { + rs, err := r.remoteStorageConstructor.Get(info) + if err != nil { + return nil, err + } + + key, err := r.key() + if err != nil { + return nil, err + } + + prm := RemoteRequestParams{ + Epoch: r.curProcEpoch, + TTL: 1, + PrivateKey: key, + SessionToken: r.prm.common.SessionToken(), + BearerToken: r.prm.common.BearerToken(), + XHeaders: r.prm.common.XHeaders(), + } + + return rs.Get(ctx, addr, prm) +} + +func (r *request) headObjectFromNode(ctx context.Context, addr oid.Address, info client.NodeInfo, raw bool) (*objectSDK.Object, error) { + rs, err := r.remoteStorageConstructor.Get(info) + if err != nil { + return nil, err + } + + key, err := r.key() + if err != nil { + return nil, err + } + + prm := RemoteRequestParams{ + Epoch: r.curProcEpoch, + TTL: 1, + PrivateKey: key, + SessionToken: r.prm.common.SessionToken(), + BearerToken: r.prm.common.BearerToken(), + XHeaders: r.prm.common.XHeaders(), + IsRaw: raw, + } + + return rs.Head(ctx, addr, prm) +} diff --git a/pkg/services/object/get/request.go b/pkg/services/object/get/request.go index d0b79e30c..9ddfeddf2 100644 --- a/pkg/services/object/get/request.go +++ b/pkg/services/object/get/request.go @@ -23,7 +23,7 @@ type request struct { infoSplit *objectSDK.SplitInfo - infoEC *objectSDK.ECInfo + infoEC *ecInfo log *logger.Logger @@ -141,7 +141,7 @@ func (r *request) initEpoch() bool { func (r *request) generateTraverser(addr oid.Address) (*placement.Traverser, bool) { obj := addr.Object() - t, err := r.traverserGenerator.GenerateTraverser(addr.Container(), &obj, r.curProcEpoch) + t, _, err := r.traverserGenerator.GenerateTraverser(addr.Container(), &obj, r.curProcEpoch) switch { default: diff --git a/pkg/services/object/get/types.go b/pkg/services/object/get/types.go index a866132cc..9669afdba 100644 --- a/pkg/services/object/get/types.go +++ b/pkg/services/object/get/types.go @@ -6,6 +6,7 @@ import ( "errors" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -23,7 +24,7 @@ type epochSource interface { } type traverserGenerator interface { - GenerateTraverser(cid.ID, *oid.ID, uint64) (*placement.Traverser, error) + GenerateTraverser(cid.ID, *oid.ID, uint64) (*placement.Traverser, *container.Container, error) } type keyStorage interface { @@ -236,3 +237,51 @@ type RangeHashRes struct { func (r *RangeHashRes) Hashes() [][]byte { return r.hashes } + +type ecInfo struct { + localChunks []objectSDK.ECChunk + remoteChunks map[string][]objectSDK.ECChunk // node pk -> chunk slice +} + +func newECInfo() *ecInfo { + return &ecInfo{ + localChunks: make([]objectSDK.ECChunk, 0), + remoteChunks: make(map[string][]objectSDK.ECChunk), + } +} + +func (e *ecInfo) addLocal(ecInfo *objectSDK.ECInfo) *objectSDK.ECInfoError { + for _, ch := range ecInfo.Chunks { + e.localChunks = append(e.localChunks, objectSDK.ECChunk(ch)) + } + return e.createECInfoErr() +} + +func (e *ecInfo) addRemote(nodePK string, ecInfo *objectSDK.ECInfo) *objectSDK.ECInfoError { + for _, ch := range ecInfo.Chunks { + e.remoteChunks[nodePK] = append(e.remoteChunks[nodePK], objectSDK.ECChunk(ch)) + } + return e.createECInfoErr() +} + +func (e *ecInfo) createECInfoErr() *objectSDK.ECInfoError { + unique := make(map[string]struct{}) + result := objectSDK.NewECInfo() + for _, ch := range e.localChunks { + if _, found := unique[string(ch.ID.GetValue())]; found { + continue + } + result.AddChunk(ch) + unique[string(ch.ID.GetValue())] = struct{}{} + } + for _, chunks := range e.remoteChunks { + for _, ch := range chunks { + if _, found := unique[string(ch.ID.GetValue())]; found { + continue + } + result.AddChunk(ch) + unique[string(ch.ID.GetValue())] = struct{}{} + } + } + return objectSDK.NewECInfoError(result) +} diff --git a/pkg/services/object/search/container.go b/pkg/services/object/search/container.go index a8865f5f0..d70574156 100644 --- a/pkg/services/object/search/container.go +++ b/pkg/services/object/search/container.go @@ -47,7 +47,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { zap.Uint64("number", exec.curProcEpoch), ) - traverser, err := exec.svc.traverserGenerator.GenerateTraverser(exec.containerID(), nil, exec.curProcEpoch) + traverser, _, err := exec.svc.traverserGenerator.GenerateTraverser(exec.containerID(), nil, exec.curProcEpoch) if err != nil { return fmt.Errorf("%s: %w", logs.SearchCouldNotGenerateContainerTraverser, err) } diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index 802fa33ef..679380402 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -80,12 +81,13 @@ func newTestStorage() *testStorage { } } -func (g *testTraverserGenerator) GenerateTraverser(_ cid.ID, _ *oid.ID, epoch uint64) (*placement.Traverser, error) { - return placement.NewTraverser( +func (g *testTraverserGenerator) GenerateTraverser(_ cid.ID, _ *oid.ID, epoch uint64) (*placement.Traverser, *containerCore.Container, error) { + t, err := placement.NewTraverser( placement.ForContainer(g.c), placement.UseBuilder(g.b[epoch]), placement.WithoutSuccessTracking(), ) + return t, &containerCore.Container{Value: g.c}, err } func (p *testPlacementBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, _ netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index 0a54f54c5..cc388c1b2 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -4,6 +4,7 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -45,7 +46,7 @@ type cfg struct { } traverserGenerator interface { - GenerateTraverser(cid.ID, *oid.ID, uint64) (*placement.Traverser, error) + GenerateTraverser(cid.ID, *oid.ID, uint64) (*placement.Traverser, *container.Container, error) } currentEpochReceiver interface { diff --git a/pkg/services/object/util/placement.go b/pkg/services/object/util/placement.go index 6cd3856f4..1bd39f9ea 100644 --- a/pkg/services/object/util/placement.go +++ b/pkg/services/object/util/placement.go @@ -122,17 +122,17 @@ func (g *TraverserGenerator) WithTraverseOptions(opts ...placement.Option) *Trav // GenerateTraverser generates placement Traverser for provided object address // using epoch-th network map. -func (g *TraverserGenerator) GenerateTraverser(idCnr cid.ID, idObj *oid.ID, epoch uint64) (*placement.Traverser, error) { +func (g *TraverserGenerator) GenerateTraverser(idCnr cid.ID, idObj *oid.ID, epoch uint64) (*placement.Traverser, *container.Container, error) { // get network map by epoch nm, err := g.netMapSrc.GetNetMapByEpoch(epoch) if err != nil { - return nil, fmt.Errorf("could not get network map #%d: %w", epoch, err) + return nil, nil, fmt.Errorf("could not get network map #%d: %w", epoch, err) } // get container related container cnr, err := g.cnrSrc.Get(idCnr) if err != nil { - return nil, fmt.Errorf("could not get container: %w", err) + return nil, nil, fmt.Errorf("could not get container: %w", err) } // allocate placement traverser options @@ -160,5 +160,9 @@ func (g *TraverserGenerator) GenerateTraverser(idCnr cid.ID, idObj *oid.ID, epoc ) } - return placement.NewTraverser(traverseOpts...) + t, err := placement.NewTraverser(traverseOpts...) + if err != nil { + return nil, nil, err + } + return t, cnr, nil } From e83d39e33f901a2158f82434fb8b98bd0b0d9b2e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 15 Jul 2024 23:04:41 +0300 Subject: [PATCH 1234/1943] [#1253] deleteSvc: Use copy of common parameters getSvc may change the values of some fields, so Head will affect Delete or Put. In this case, the change is necessary so that the session token is stored in the tombstone object (EC assemble calls `ForgetTokens`). Signed-off-by: Dmitrii Stepanov --- pkg/services/object/delete/util.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/services/object/delete/util.go b/pkg/services/object/delete/util.go index bb2b5f00b..a78fd7747 100644 --- a/pkg/services/object/delete/util.go +++ b/pkg/services/object/delete/util.go @@ -30,7 +30,12 @@ func (w *headSvcWrapper) headAddress(ctx context.Context, exec *execCtx, addr oi wr := getsvc.NewSimpleObjectWriter() p := getsvc.HeadPrm{} - p.SetCommonParameters(exec.commonParameters()) + + if cp := exec.commonParameters(); cp != nil { + commonParameters := *cp + p.SetCommonParameters(&commonParameters) + } + p.SetHeaderWriter(wr) p.WithRawFlag(true) p.WithAddress(addr) From 27caa8a72f68924721496be17d549e2325b0d081 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 17 Jul 2024 16:48:10 +0300 Subject: [PATCH 1235/1943] [#1256] metabase: Put split parent object ID for EC chunks It is required to save split parent ID too, not only split ID. Otherwise inhume operation works incorrect: shard with last part may be skipped and parent object will be available. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/put.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index ceb79758f..f351cb485 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -356,6 +356,18 @@ func updateListIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun return err } } + + if parentSplitParentID := ech.ParentSplitParentID(); parentSplitParentID != nil { + objKey := objectKey(ech.Parent(), make([]byte, objectKeySize)) + err := f(tx, namedBucketItem{ + name: parentBucketName(cnr, bucketName), + key: objectKey(*parentSplitParentID, make([]byte, objectKeySize)), + val: objKey, + }) + if err != nil { + return err + } + } } return nil From 5e5ee545b6c0f42c08c492fa325e0d43de0e93c2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 17 Jul 2024 14:30:51 +0300 Subject: [PATCH 1236/1943] [#1254] policer: Fix svacer warning Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/ec.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index 5b3793f7b..0a118797d 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -25,7 +25,7 @@ type ecChunkProcessResult struct { removeLocal bool } -var errInvalidECPlacement = errors.New("invalid EC placement: EC placement must have one placement vector") +var errInvalidECPlacement = errors.New("invalid EC placement: EC placement must have one placement vector with at least one node") func (p *Policer) processECContainerObject(ctx context.Context, objInfo objectcore.Info, policy netmap.PlacementPolicy) error { if objInfo.ECInfo == nil { @@ -42,7 +42,7 @@ func (p *Policer) processECContainerRepObject(ctx context.Context, objInfo objec if err != nil { return fmt.Errorf("%s: %w", logs.PolicerCouldNotBuildPlacementVectorForObject, err) } - if len(nn) != 1 { + if len(nn) != 1 || len(nn[0]) == 0 { return errInvalidECPlacement } @@ -72,7 +72,7 @@ func (p *Policer) processECContainerECObject(ctx context.Context, objInfo object if err != nil { return fmt.Errorf("%s: %w", logs.PolicerCouldNotBuildPlacementVectorForObject, err) } - if len(nn) != 1 { + if len(nn) != 1 || len(nn[0]) == 0 { return errInvalidECPlacement } From eadcea8df057d6eb502161c0a2d1f202f0f843be Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 18 Jul 2024 13:23:17 +0300 Subject: [PATCH 1237/1943] [#1249] object: Remove all APE pre-checks in handlers * Methods `Head`, `Get`, `GetRangeHash` should no longer use APE pre-checks as that leads only to incorrect rule chain processing for requests: 1. Immediate return with `NoRuleFound` may be unexpected as some `Allow` rule is actually defined but can't be matched yet as it gets no object attributes; 2. Immdediate return with `Allow` may be incorrect as some `Deny` rule is actually defined but can't bet matched yet as it gets no object attirbutes; 3. Pre-check breaks compatibility for converted EACL-tables. Signed-off-by: Airat Arifullin --- pkg/services/object/ape/service.go | 43 ------------------------------ 1 file changed, 43 deletions(-) diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 56c66002d..2adb1b736 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -134,33 +134,11 @@ func requestContext(ctx context.Context) (*objectSvc.RequestContext, error) { } func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectStream) error { - cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) - if err != nil { - return toStatusErr(err) - } - reqCtx, err := requestContext(stream.Context()) if err != nil { return toStatusErr(err) } - err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodGetObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - SoftAPECheck: reqCtx.SoftAPECheck, - WithoutHeaderRequest: true, - BearerToken: reqCtx.BearerToken, - XHeaders: request.GetMetaHeader().GetXHeaders(), - }) - if err != nil { - return toStatusErr(err) - } - return c.next.Get(request, &getStreamBasicChecker{ GetObjectStream: stream, apeChecker: c.apeChecker, @@ -237,23 +215,6 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj return nil, err } - err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, - Method: nativeschema.MethodHeadObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - SoftAPECheck: reqCtx.SoftAPECheck, - WithoutHeaderRequest: true, - BearerToken: reqCtx.BearerToken, - XHeaders: request.GetMetaHeader().GetXHeaders(), - }) - if err != nil { - return nil, toStatusErr(err) - } - resp, err := c.next.Head(ctx, request) if err != nil { return nil, err @@ -417,10 +378,6 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa XHeaders: request.GetMetaHeader().GetXHeaders(), } - if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { - return nil, toStatusErr(err) - } - resp, err := c.next.GetRangeHash(ctx, request) if err != nil { return nil, err From e377a92d97f6732a1bbe9e815bd25e17fc02c6d3 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 19 Jul 2024 11:23:02 +0300 Subject: [PATCH 1238/1943] [#1258] audit: Fix panic in LogRequest method * Make `LogRequest` process `req=nil` to avoid panic. Signed-off-by: Airat Arifullin --- internal/audit/request.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/audit/request.go b/internal/audit/request.go index cd1771013..cf0797300 100644 --- a/internal/audit/request.go +++ b/internal/audit/request.go @@ -18,7 +18,11 @@ type Target interface { } func LogRequest(log *logger.Logger, operation string, req Request, target Target, status bool) { - LogRequestWithKey(log, operation, req.GetVerificationHeader().GetBodySignature().GetKey(), target, status) + var key []byte + if req != nil { + key = req.GetVerificationHeader().GetBodySignature().GetKey() + } + LogRequestWithKey(log, operation, key, target, status) } func LogRequestWithKey(log *logger.Logger, operation string, key []byte, target Target, status bool) { From 3119f2fbc3b3a38b9b4b8011cf0859235ab2963e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 18 Jul 2024 16:14:36 +0300 Subject: [PATCH 1239/1943] [#1257] metabase: Delete EC gc marks and split info EC parent and split gc marks should be deleted after last EC chunk delete. Also should delete split info for EC parent and split parent. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/delete.go | 75 +++ .../metabase/delete_ec_test.go | 459 ++++++++++++++++++ 2 files changed, 534 insertions(+) create mode 100644 pkg/local_object_storage/metabase/delete_ec_test.go diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 21b98fca1..ae10564a8 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -314,6 +314,10 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter return deleteSingleResult{}, fmt.Errorf("could not remove object: %w", err) } + if err := deleteECRelatedInfo(tx, garbageBKT, obj, addr.Container(), refCounter); err != nil { + return deleteSingleResult{}, err + } + return deleteSingleResult{ Phy: true, Logic: removeAvailableObject, @@ -476,3 +480,74 @@ func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error return nil } + +func deleteECRelatedInfo(tx *bbolt.Tx, garbageBKT *bbolt.Bucket, obj *objectSDK.Object, cnr cid.ID, refCounter referenceCounter) error { + ech := obj.ECHeader() + if ech == nil { + return nil + } + + hasAnyChunks := hasAnyECChunks(tx, ech, cnr) + // drop EC parent GC mark if current EC chunk is the last one + if !hasAnyChunks && garbageBKT != nil { + var ecParentAddress oid.Address + ecParentAddress.SetContainer(cnr) + ecParentAddress.SetObject(ech.Parent()) + addrKey := addressKey(ecParentAddress, make([]byte, addressKeySize)) + err := garbageBKT.Delete(addrKey) + if err != nil { + return fmt.Errorf("could not remove EC parent from garbage bucket: %w", err) + } + } + + // also drop EC parent root info if current EC chunk is the last one + if !hasAnyChunks { + delUniqueIndexItem(tx, namedBucketItem{ + name: rootBucketName(cnr, make([]byte, bucketKeySize)), + key: objectKey(ech.Parent(), make([]byte, objectKeySize)), + }) + } + + if ech.ParentSplitParentID() == nil { + return nil + } + + var splitParentAddress oid.Address + splitParentAddress.SetContainer(cnr) + splitParentAddress.SetObject(*ech.ParentSplitParentID()) + + if ref, ok := refCounter[string(addressKey(splitParentAddress, make([]byte, addressKeySize)))]; ok { + // linking object is already processing + // so just inform that one more reference was deleted + // split info and gc marks will be deleted after linking object delete + ref.cur++ + return nil + } + + if parentLength(tx, splitParentAddress) > 0 { + // linking object still exists, so leave split info and gc mark deletion for linking object processing + return nil + } + + // drop split parent gc mark + if garbageBKT != nil { + addrKey := addressKey(splitParentAddress, make([]byte, addressKeySize)) + err := garbageBKT.Delete(addrKey) + if err != nil { + return fmt.Errorf("could not remove EC parent from garbage bucket: %w", err) + } + } + + // drop split info + delUniqueIndexItem(tx, namedBucketItem{ + name: rootBucketName(cnr, make([]byte, bucketKeySize)), + key: objectKey(*ech.ParentSplitParentID(), make([]byte, objectKeySize)), + }) + return nil +} + +func hasAnyECChunks(tx *bbolt.Tx, ech *objectSDK.ECHeader, cnr cid.ID) bool { + data := getFromBucket(tx, ecInfoBucketName(cnr, make([]byte, bucketKeySize)), + objectKey(ech.Parent(), make([]byte, objectKeySize))) + return len(data) > 0 +} diff --git a/pkg/local_object_storage/metabase/delete_ec_test.go b/pkg/local_object_storage/metabase/delete_ec_test.go new file mode 100644 index 000000000..0e627f095 --- /dev/null +++ b/pkg/local_object_storage/metabase/delete_ec_test.go @@ -0,0 +1,459 @@ +package meta + +import ( + "bytes" + "context" + "fmt" + "path/filepath" + "slices" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" +) + +func TestDeleteECObject_WithoutSplit(t *testing.T) { + t.Parallel() + + db := New( + WithPath(filepath.Join(t.TempDir(), "metabase")), + WithPermissions(0o600), + WithEpochState(epochState{uint64(12)}), + ) + + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + require.NoError(t, db.Init()) + defer func() { require.NoError(t, db.Close()) }() + + cnr := cidtest.ID() + ecChunk := oidtest.ID() + ecParent := oidtest.ID() + tombstoneID := oidtest.ID() + + chunkObj := testutil.GenerateObjectWithCID(cnr) + chunkObj.SetContainerID(cnr) + chunkObj.SetID(ecChunk) + chunkObj.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + chunkObj.SetPayloadSize(uint64(10)) + chunkObj.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent}, 0, 3, []byte{}, 0)) + + // put object with EC + + var prm PutPrm + prm.SetObject(chunkObj) + prm.SetStorageID([]byte("0/0")) + _, err := db.Put(context.Background(), prm) + require.NoError(t, err) + + var ecChunkAddress oid.Address + ecChunkAddress.SetContainer(cnr) + ecChunkAddress.SetObject(ecChunk) + + var ecParentAddress oid.Address + ecParentAddress.SetContainer(cnr) + ecParentAddress.SetObject(ecParent) + + var getPrm GetPrm + + getPrm.SetAddress(ecChunkAddress) + _, err = db.Get(context.Background(), getPrm) + require.NoError(t, err) + + var ecInfoError *objectSDK.ECInfoError + getPrm.SetAddress(ecParentAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, &ecInfoError) + require.True(t, len(ecInfoError.ECInfo().Chunks) == 1 && + ecInfoError.ECInfo().Chunks[0].Index == 0 && + ecInfoError.ECInfo().Chunks[0].Total == 3) + + // inhume EC parent (like Delete does) + + var inhumePrm InhumePrm + var tombAddress oid.Address + tombAddress.SetContainer(cnr) + tombAddress.SetObject(tombstoneID) + inhumePrm.SetAddresses(ecParentAddress) + inhumePrm.SetTombstoneAddress(tombAddress) + _, err = db.Inhume(context.Background(), inhumePrm) + require.NoError(t, err) + + getPrm.SetAddress(ecParentAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) + + getPrm.SetAddress(ecChunkAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) + + // GC finds and deletes split, EC parent and EC chunk + + var garbageAddresses []oid.Address + var itPrm GarbageIterationPrm + itPrm.SetHandler(func(g GarbageObject) error { + garbageAddresses = append(garbageAddresses, g.Address()) + return nil + }) + require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm)) + require.Equal(t, 2, len(garbageAddresses)) + require.True(t, slices.Contains(garbageAddresses, ecParentAddress)) + require.True(t, slices.Contains(garbageAddresses, ecChunkAddress)) + + var deletePrm DeletePrm + deletePrm.SetAddresses(garbageAddresses...) + _, err = db.Delete(context.Background(), deletePrm) + require.NoError(t, err) + + garbageAddresses = nil + itPrm.SetHandler(func(g GarbageObject) error { + garbageAddresses = append(garbageAddresses, g.Address()) + return nil + }) + require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm)) + require.Equal(t, 0, len(garbageAddresses)) + + // after tombstone expired GC inhumes tombstone and drops graves + + var tombstonedObjects []TombstonedObject + var graveyardIterationPrm GraveyardIterationPrm + graveyardIterationPrm.SetHandler(func(object TombstonedObject) error { + tombstonedObjects = append(tombstonedObjects, object) + return nil + }) + require.NoError(t, db.IterateOverGraveyard(context.Background(), graveyardIterationPrm)) + require.Equal(t, 2, len(tombstonedObjects)) + + var tombstones []oid.Address + for _, tss := range tombstonedObjects { + tombstones = append(tombstones, tss.tomb) + } + inhumePrm.SetAddresses(tombstones...) + inhumePrm.SetGCMark() + _, err = db.Inhume(context.Background(), inhumePrm) + require.NoError(t, err) + + require.NoError(t, db.DropGraves(context.Background(), tombstonedObjects)) + + // GC finds tombstone as garbage and deletes it + + garbageAddresses = nil + itPrm.SetHandler(func(g GarbageObject) error { + garbageAddresses = append(garbageAddresses, g.Address()) + return nil + }) + require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm)) + require.Equal(t, 1, len(garbageAddresses)) + require.Equal(t, tombstoneID, garbageAddresses[0].Object()) + + deletePrm.SetAddresses(garbageAddresses...) + _, err = db.Delete(context.Background(), deletePrm) + require.NoError(t, err) + + // no more objects should left as garbage + + itPrm.SetHandler(func(g GarbageObject) error { + require.FailNow(t, "no garbage objects should left") + return nil + }) + require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm)) + + require.NoError(t, db.boltDB.View(testVerifyNoObjectDataLeft)) + + require.NoError(t, testCountersAreZero(db, cnr)) +} + +func TestDeleteECObject_WithSplit(t *testing.T) { + t.Parallel() + for _, c := range []int{1, 2, 3} { + for _, l := range []bool{true, false} { + test := fmt.Sprintf("%d EC chunks with split info without linking object", c) + if l { + test = fmt.Sprintf("%d EC chunks with split info with linking object", c) + } + t.Run(test, func(t *testing.T) { + testDeleteECObjectWithSplit(t, c, l) + }) + } + } +} + +func testDeleteECObjectWithSplit(t *testing.T, chunksCount int, withLinking bool) { + t.Parallel() + + db := New( + WithPath(filepath.Join(t.TempDir(), "metabase")), + WithPermissions(0o600), + WithEpochState(epochState{uint64(12)}), + ) + + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + require.NoError(t, db.Init()) + defer func() { require.NoError(t, db.Close()) }() + + cnr := cidtest.ID() + ecChunks := make([]oid.ID, chunksCount) + for idx := range ecChunks { + ecChunks[idx] = oidtest.ID() + } + ecParentID := oidtest.ID() + splitParentID := oidtest.ID() + tombstoneID := oidtest.ID() + splitID := objectSDK.NewSplitID() + linkingID := oidtest.ID() + + ecChunkObjects := make([]*objectSDK.Object, chunksCount) + for idx := range ecChunkObjects { + ecChunkObjects[idx] = testutil.GenerateObjectWithCID(cnr) + ecChunkObjects[idx].SetContainerID(cnr) + ecChunkObjects[idx].SetID(ecChunks[idx]) + ecChunkObjects[idx].SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + ecChunkObjects[idx].SetPayloadSize(uint64(10)) + ecChunkObjects[idx].SetECHeader(objectSDK.NewECHeader( + objectSDK.ECParentInfo{ + ID: ecParentID, + SplitParentID: &splitParentID, SplitID: splitID, + }, uint32(idx), uint32(chunksCount+1), []byte{}, 0)) + } + + splitParentObj := testutil.GenerateObjectWithCID(cnr) + splitParentObj.SetID(splitParentID) + + var linkingAddress oid.Address + linkingAddress.SetContainer(cnr) + linkingAddress.SetObject(linkingID) + + linkingObj := testutil.GenerateObjectWithCID(cnr) + linkingObj.SetID(linkingID) + linkingObj.SetParent(splitParentObj) + linkingObj.SetParentID(splitParentID) + linkingObj.SetChildren(ecParentID, oidtest.ID(), oidtest.ID()) + linkingObj.SetSplitID(splitID) + + // put object with EC and split info + + var prm PutPrm + prm.SetStorageID([]byte("0/0")) + for _, obj := range ecChunkObjects { + prm.SetObject(obj) + _, err := db.Put(context.Background(), prm) + require.NoError(t, err) + } + + if withLinking { + prm.SetObject(linkingObj) + _, err := db.Put(context.Background(), prm) + require.NoError(t, err) + } + + var ecParentAddress oid.Address + ecParentAddress.SetContainer(cnr) + ecParentAddress.SetObject(ecParentID) + + var getPrm GetPrm + var ecInfoError *objectSDK.ECInfoError + getPrm.SetAddress(ecParentAddress) + _, err := db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, &ecInfoError) + require.True(t, len(ecInfoError.ECInfo().Chunks) == chunksCount) + + var splitParentAddress oid.Address + splitParentAddress.SetContainer(cnr) + splitParentAddress.SetObject(splitParentID) + + var splitInfoError *objectSDK.SplitInfoError + getPrm.SetAddress(splitParentAddress) + getPrm.SetRaw(true) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, &splitInfoError) + require.True(t, splitInfoError.SplitInfo() != nil) + require.Equal(t, splitID, splitInfoError.SplitInfo().SplitID()) + lastPart, set := splitInfoError.SplitInfo().LastPart() + require.True(t, set) + require.Equal(t, lastPart, ecParentID) + if withLinking { + l, ok := splitInfoError.SplitInfo().Link() + require.True(t, ok) + require.Equal(t, linkingID, l) + } + getPrm.SetRaw(false) + + // inhume EC parent and split objects (like Delete does) + + inhumeAddresses := []oid.Address{splitParentAddress, ecParentAddress} + if withLinking { + inhumeAddresses = append(inhumeAddresses, linkingAddress) + } + + var inhumePrm InhumePrm + var tombAddress oid.Address + tombAddress.SetContainer(cnr) + tombAddress.SetObject(tombstoneID) + inhumePrm.SetAddresses(inhumeAddresses...) + inhumePrm.SetTombstoneAddress(tombAddress) + _, err = db.Inhume(context.Background(), inhumePrm) + require.NoError(t, err) + + getPrm.SetAddress(ecParentAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) + + getPrm.SetAddress(splitParentAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) + + if withLinking { + getPrm.SetAddress(linkingAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) + } + + for _, id := range ecChunks { + var ecChunkAddress oid.Address + ecChunkAddress.SetContainer(cnr) + ecChunkAddress.SetObject(id) + getPrm.SetAddress(ecChunkAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) + } + + // GC finds and deletes split, EC parent and EC chunks + + parentCount := 2 // split + ec + if withLinking { + parentCount = 3 + } + + var garbageAddresses []oid.Address + var itPrm GarbageIterationPrm + itPrm.SetHandler(func(g GarbageObject) error { + garbageAddresses = append(garbageAddresses, g.Address()) + return nil + }) + require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm)) + require.Equal(t, parentCount+chunksCount, len(garbageAddresses)) + require.True(t, slices.Contains(garbageAddresses, splitParentAddress)) + require.True(t, slices.Contains(garbageAddresses, ecParentAddress)) + if withLinking { + require.True(t, slices.Contains(garbageAddresses, linkingAddress)) + } + for _, id := range ecChunks { + var ecChunkAddress oid.Address + ecChunkAddress.SetContainer(cnr) + ecChunkAddress.SetObject(id) + require.True(t, slices.Contains(garbageAddresses, ecChunkAddress)) + } + + var deletePrm DeletePrm + deletePrm.SetAddresses(garbageAddresses...) + _, err = db.Delete(context.Background(), deletePrm) + require.NoError(t, err) + + var garbageStub []oid.Address + itPrm.SetHandler(func(g GarbageObject) error { + garbageStub = append(garbageStub, g.Address()) + return nil + }) + require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm)) + require.Equal(t, 0, len(garbageStub)) + + // after tombstone expired GC inhumes tombstone and drops graves + + var tombstonedObjects []TombstonedObject + var graveyardIterationPrm GraveyardIterationPrm + graveyardIterationPrm.SetHandler(func(object TombstonedObject) error { + tombstonedObjects = append(tombstonedObjects, object) + return nil + }) + require.NoError(t, db.IterateOverGraveyard(context.Background(), graveyardIterationPrm)) + require.True(t, len(tombstonedObjects) == parentCount+chunksCount) + + var tombstones []oid.Address + for _, tss := range tombstonedObjects { + tombstones = append(tombstones, tss.tomb) + } + inhumePrm.SetAddresses(tombstones...) + inhumePrm.SetGCMark() + _, err = db.Inhume(context.Background(), inhumePrm) + require.NoError(t, err) + + require.NoError(t, db.DropGraves(context.Background(), tombstonedObjects)) + + // GC finds tombstone as garbage and deletes it + + garbageAddresses = nil + itPrm.SetHandler(func(g GarbageObject) error { + garbageAddresses = append(garbageAddresses, g.Address()) + return nil + }) + require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm)) + require.Equal(t, 1, len(garbageAddresses)) + require.Equal(t, tombstoneID, garbageAddresses[0].Object()) + + deletePrm.SetAddresses(garbageAddresses...) + _, err = db.Delete(context.Background(), deletePrm) + require.NoError(t, err) + + // no more objects should left as garbage + + itPrm.SetHandler(func(g GarbageObject) error { + require.FailNow(t, "no garbage objects should left") + return nil + }) + require.NoError(t, db.IterateOverGarbage(context.Background(), itPrm)) + + require.NoError(t, db.boltDB.View(testVerifyNoObjectDataLeft)) + + require.NoError(t, testCountersAreZero(db, cnr)) +} + +func testVerifyNoObjectDataLeft(tx *bbolt.Tx) error { + return tx.ForEach(func(name []byte, b *bbolt.Bucket) error { + if bytes.Equal(name, shardInfoBucket) || + bytes.Equal(name, containerCounterBucketName) || + bytes.Equal(name, containerVolumeBucketName) { + return nil + } + return testBucketEmpty(name, b) + }) +} + +func testBucketEmpty(name []byte, b *bbolt.Bucket) error { + err := b.ForEach(func(k, v []byte) error { + if len(v) > 0 { + return fmt.Errorf("bucket %v is not empty", name) + } + return nil + }) + if err != nil { + return err + } + return b.ForEachBucket(func(k []byte) error { + return testBucketEmpty(k, b.Bucket(k)) + }) +} + +func testCountersAreZero(db *DB, cnr cid.ID) error { + c, err := db.ContainerCount(context.Background(), cnr) + if err != nil { + return err + } + if !c.IsZero() { + return fmt.Errorf("container %s has non zero counters", cnr.EncodeToString()) + } + s, err := db.ContainerSize(cnr) + if err != nil { + return err + } + if s != 0 { + return fmt.Errorf("container %s has non zero size", cnr.EncodeToString()) + } + return nil +} From 18182e578ec55fd035403e3915f5e4f66ad4c8c3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 22 Jul 2024 13:54:06 +0300 Subject: [PATCH 1240/1943] [#1261] shard: Fix delete objects from FSTree Replace nil storageID with empty like by shard.Get. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/delete.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 44f6c6b48..c898fdf41 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -117,6 +117,12 @@ func (s *Shard) deleteFromBlobstor(ctx context.Context, addr oid.Address) error return err } storageID := res.StorageID() + if storageID == nil { + // if storageID is nil it means: + // 1. there is no such object + // 2. object stored by writecache: should not happen, as `validateWritecacheDoesntContainObject` called before `deleteFromBlobstor` + return nil + } var delPrm common.DeletePrm delPrm.Address = addr From ca4d6df1ccfd5e44093d313f8531451a28ffa0f4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 19 Jul 2024 09:28:36 +0300 Subject: [PATCH 1241/1943] Release v0.42.0 Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 24 ++++++++++++++++++++++++ VERSION | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2abba1d09..e4ba6a5d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,30 @@ Changelog for FrostFS Node ### Removed ### Updated +## [v0.42.0] + +### Added +- Add audit logs for gRPC requests (#1184) +- Add CLI command to convert eACL to APE (#1189) +- Add `--await` flag to `control set-status` (#60) +- `app_info` metric for binary version (#1154) +- `--quiet` flag for healthcheck command (#1209) + +### Changed +- Deprecate Container.SetEACL RPC (#1219) + +### Fixed +- Take groups into account during APE processing (#1190) +- Handle double SIGHUP correctly (#1145) +- Handle empty filenames in tree listing (#1074) +- Handle duplicate tree nodes in the split-brain scenario (#1234, #1251) +- Remove APE pre-check in Object.GET/HEAD/RANGE RPC (#1249) +- Delete EC gc marks and split info (#1257) +- Do not search for non-existent objects on deletion (#1261) + +### Updated +- Make putting EC chunks more robust (#1233) + ## [v0.41.0] ### Added diff --git a/VERSION b/VERSION index 9dedf1e9b..01efe7f3a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.41.0 +v0.42.0 From 9ee1bd36697276bf2c91aad7e6f5d55ca5cbe376 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Tue, 23 Jul 2024 10:31:37 +0300 Subject: [PATCH 1242/1943] [#1265] *: Run gofumpt Signed-off-by: Aleksey Savchuk --- pkg/innerring/processors/netmap/process_epoch.go | 1 - pkg/innerring/processors/netmap/process_peers.go | 1 - 2 files changed, 2 deletions(-) diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index f3cb9837f..4dfa3997b 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -51,7 +51,6 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) bool { if epoch > 0 && np.alphabetState.IsAlphabet() { // estimates are invalid in genesis epoch err = np.containerWrp.StartEstimation(prm) - if err != nil { np.log.Warn(logs.NetmapCantStartContainerSizeEstimation, zap.Uint64("epoch", epoch), diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index 41e4bfb7e..9e6e8c283 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -83,7 +83,6 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { methodAddPeerNotary, nodeInfoBinary, ) - if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapAddPeer, zap.Error(err)) return false From 658e3cb92f2901c95a28e6eb742aecfb63835469 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 23 Jul 2024 09:35:41 +0300 Subject: [PATCH 1243/1943] [#1264] go.mod: Update bbolt to v1.3.10 Signed-off-by: Evgenii Stratonikov --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ee8b1bb16..a89520e34 100644 --- a/go.mod +++ b/go.mod @@ -34,13 +34,13 @@ require ( github.com/spf13/viper v1.18.2 github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.9.0 - go.etcd.io/bbolt v1.3.9 + go.etcd.io/bbolt v1.3.10 go.opentelemetry.io/otel v1.22.0 go.opentelemetry.io/otel/trace v1.22.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/sync v0.6.0 - golang.org/x/sys v0.18.0 + golang.org/x/sys v0.22.0 golang.org/x/term v0.18.0 google.golang.org/grpc v1.63.2 google.golang.org/protobuf v1.33.0 diff --git a/go.sum b/go.sum index c7c3b87eb..163c53b27 100644 --- a/go.sum +++ b/go.sum @@ -285,8 +285,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk= go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= @@ -379,8 +379,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= From 7ddba70030118f681396f3eae37b6e3de3b06162 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 23 Jul 2024 09:39:14 +0300 Subject: [PATCH 1244/1943] [#1264] go.mod: Update dependencies Signed-off-by: Evgenii Stratonikov --- go.mod | 26 ++++++++++++------------ go.sum | 62 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index a89520e34..0f608f74d 100644 --- a/go.mod +++ b/go.mod @@ -29,17 +29,17 @@ require ( github.com/paulmach/orb v0.11.0 github.com/prometheus/client_golang v1.19.0 github.com/spf13/cast v1.6.0 - github.com/spf13/cobra v1.8.0 + github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.18.2 + github.com/spf13/viper v1.19.0 github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.9.0 go.etcd.io/bbolt v1.3.10 - go.opentelemetry.io/otel v1.22.0 - go.opentelemetry.io/otel/trace v1.22.0 + go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240119083558-1b970713d09a - golang.org/x/sync v0.6.0 + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 + golang.org/x/sync v0.7.0 golang.org/x/sys v0.22.0 golang.org/x/term v0.18.0 google.golang.org/grpc v1.63.2 @@ -48,7 +48,7 @@ require ( ) require ( - github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect go.mongodb.org/mongo-driver v1.13.1 // indirect @@ -63,7 +63,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect @@ -98,7 +98,7 @@ require ( github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect - github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect @@ -114,15 +114,15 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0 // indirect - go.opentelemetry.io/otel/metric v1.22.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/sdk v1.22.0 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/net v0.23.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 163c53b27..355e176f7 100644 --- a/go.sum +++ b/go.sum @@ -49,8 +49,8 @@ github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 h1:tYj5Ydh5D7Xg2R1tJnoG36Yta7NVB8C0vx36oPA3Bbw= github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2/go.mod h1:wKqwsieaKPThcFkHe0d0zMsbHEUWFmZcG7KBCse210o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -213,8 +213,8 @@ github.com/panjf2000/ants/v2 v2.9.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnl github.com/paulmach/orb v0.11.0 h1:JfVXJUBeH9ifc/OrhBY0lL16QsmPgpCHMlqSSYhcgAA= github.com/paulmach/orb v0.11.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= -github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= -github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -234,8 +234,8 @@ github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDN github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= -github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= +github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -246,17 +246,18 @@ github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= -github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/ssgreg/journald v1.0.0 h1:0YmTDPJXxcWDPba12qNMdO6TxvfkFSYpFIJ31CwmLcU= github.com/ssgreg/journald v1.0.0/go.mod h1:RUckwmTM8ghGWPslq2+ZBZzbb9/2KgjzYZ4JEP+oRt0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -290,20 +291,20 @@ go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk= go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= -go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= -go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 h1:H2JFgRcGiyHg7H7bwcwaQJYrNFqCqrbTQ8K4p1OvDu8= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0/go.mod h1:WfCWp1bGoYK8MeULtI15MmQVczfR+bFkk0DF3h06QmQ= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0 h1:zr8ymM5OWWjjiWRzwTfZ67c905+2TMHYp2lMJ52QTyM= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0/go.mod h1:sQs7FT2iLVJ+67vYngGJkPe1qr39IzaBzaj9IDNNY8k= -go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= -go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= -go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= -go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -321,14 +322,14 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -353,8 +354,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -395,8 +396,9 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -404,19 +406,17 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c h1:lfpJ/2rWPa/kJgxyyXM8PrNnfCzcmxJ265mADgwmvLI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= From 94f07b4778693842cbc7ac8e6b9228a688e71da1 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 23 Jul 2024 12:15:20 +0300 Subject: [PATCH 1245/1943] [#1245] docker: Fix warnings Signed-off-by: Ekaterina Lebedeva --- .docker/Dockerfile.adm | 2 +- .docker/Dockerfile.cli | 2 +- .docker/Dockerfile.ir | 2 +- .docker/Dockerfile.storage | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.docker/Dockerfile.adm b/.docker/Dockerfile.adm index b3dad06d3..5d67a1d04 100644 --- a/.docker/Dockerfile.adm +++ b/.docker/Dockerfile.adm @@ -1,4 +1,4 @@ -FROM golang:1.22 as builder +FROM golang:1.22 AS builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.cli b/.docker/Dockerfile.cli index 5adedc140..16f643b61 100644 --- a/.docker/Dockerfile.cli +++ b/.docker/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM golang:1.22 as builder +FROM golang:1.22 AS builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ir b/.docker/Dockerfile.ir index 25025bb2f..f2cb764e5 100644 --- a/.docker/Dockerfile.ir +++ b/.docker/Dockerfile.ir @@ -1,4 +1,4 @@ -FROM golang:1.22 as builder +FROM golang:1.22 AS builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage b/.docker/Dockerfile.storage index a16005516..cf7f97748 100644 --- a/.docker/Dockerfile.storage +++ b/.docker/Dockerfile.storage @@ -1,4 +1,4 @@ -FROM golang:1.22 as builder +FROM golang:1.22 AS builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository From 1032075a2195c07a46d303b254c6f08cb032f93c Mon Sep 17 00:00:00 2001 From: George Bartolomey Date: Mon, 22 Jul 2024 11:08:09 +0300 Subject: [PATCH 1246/1943] [#1259] cli: Remove locode subcommand Removed `frostfs-cli util locode` subcommand. Alternative command could be found in `git.frostfs.info/TrueCloudLab/frostfs-locode-db`. Signed-off-by: George Bartolomey --- cmd/frostfs-cli/modules/util/locode.go | 18 ---- .../modules/util/locode_generate.go | 96 ------------------- cmd/frostfs-cli/modules/util/locode_info.go | 56 ----------- cmd/frostfs-cli/modules/util/root.go | 2 - 4 files changed, 172 deletions(-) delete mode 100644 cmd/frostfs-cli/modules/util/locode.go delete mode 100644 cmd/frostfs-cli/modules/util/locode_generate.go delete mode 100644 cmd/frostfs-cli/modules/util/locode_info.go diff --git a/cmd/frostfs-cli/modules/util/locode.go b/cmd/frostfs-cli/modules/util/locode.go deleted file mode 100644 index a1f0f4d3f..000000000 --- a/cmd/frostfs-cli/modules/util/locode.go +++ /dev/null @@ -1,18 +0,0 @@ -package util - -import ( - "github.com/spf13/cobra" -) - -// locode section. -var locodeCmd = &cobra.Command{ - Use: "locode", - Short: "Working with FrostFS UN/LOCODE database", -} - -func initLocodeCmd() { - locodeCmd.AddCommand(locodeGenerateCmd, locodeInfoCmd) - - initUtilLocodeInfoCmd() - initUtilLocodeGenerateCmd() -} diff --git a/cmd/frostfs-cli/modules/util/locode_generate.go b/cmd/frostfs-cli/modules/util/locode_generate.go deleted file mode 100644 index 319dee1c6..000000000 --- a/cmd/frostfs-cli/modules/util/locode_generate.go +++ /dev/null @@ -1,96 +0,0 @@ -package util - -import ( - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db" - airportsdb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db/airports" - locodebolt "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db/boltdb" - continentsdb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db/continents/geojson" - csvlocode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/table/csv" - "github.com/spf13/cobra" -) - -type namesDB struct { - *airportsdb.DB - *csvlocode.Table -} - -const ( - locodeGenerateInputFlag = "in" - locodeGenerateSubDivFlag = "subdiv" - locodeGenerateAirportsFlag = "airports" - locodeGenerateCountriesFlag = "countries" - locodeGenerateContinentsFlag = "continents" - locodeGenerateOutputFlag = "out" -) - -var ( - locodeGenerateInPaths []string - locodeGenerateSubDivPath string - locodeGenerateAirportsPath string - locodeGenerateCountriesPath string - locodeGenerateContinentsPath string - locodeGenerateOutPath string - - locodeGenerateCmd = &cobra.Command{ - Use: "generate", - Short: "Generate UN/LOCODE database for FrostFS", - Run: func(cmd *cobra.Command, _ []string) { - locodeDB := csvlocode.New( - csvlocode.Prm{ - Path: locodeGenerateInPaths[0], - SubDivPath: locodeGenerateSubDivPath, - }, - csvlocode.WithExtraPaths(locodeGenerateInPaths[1:]...), - ) - - airportDB := airportsdb.New(airportsdb.Prm{ - AirportsPath: locodeGenerateAirportsPath, - CountriesPath: locodeGenerateCountriesPath, - }) - - continentsDB := continentsdb.New(continentsdb.Prm{ - Path: locodeGenerateContinentsPath, - }) - - targetDB := locodebolt.New(locodebolt.Prm{ - Path: locodeGenerateOutPath, - }) - - err := targetDB.Open() - commonCmd.ExitOnErr(cmd, "", err) - - defer targetDB.Close() - - names := &namesDB{ - DB: airportDB, - Table: locodeDB, - } - - err = locodedb.FillDatabase(locodeDB, airportDB, continentsDB, names, targetDB) - commonCmd.ExitOnErr(cmd, "", err) - }, - } -) - -func initUtilLocodeGenerateCmd() { - flags := locodeGenerateCmd.Flags() - - flags.StringSliceVar(&locodeGenerateInPaths, locodeGenerateInputFlag, nil, "List of paths to UN/LOCODE tables (csv)") - _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateInputFlag) - - flags.StringVar(&locodeGenerateSubDivPath, locodeGenerateSubDivFlag, "", "Path to UN/LOCODE subdivision database (csv)") - _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateSubDivFlag) - - flags.StringVar(&locodeGenerateAirportsPath, locodeGenerateAirportsFlag, "", "Path to OpenFlights airport database (csv)") - _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateAirportsFlag) - - flags.StringVar(&locodeGenerateCountriesPath, locodeGenerateCountriesFlag, "", "Path to OpenFlights country database (csv)") - _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateCountriesFlag) - - flags.StringVar(&locodeGenerateContinentsPath, locodeGenerateContinentsFlag, "", "Path to continent polygons (GeoJSON)") - _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateContinentsFlag) - - flags.StringVar(&locodeGenerateOutPath, locodeGenerateOutputFlag, "", "Target path for generated database") - _ = locodeGenerateCmd.MarkFlagRequired(locodeGenerateOutputFlag) -} diff --git a/cmd/frostfs-cli/modules/util/locode_info.go b/cmd/frostfs-cli/modules/util/locode_info.go deleted file mode 100644 index e89252dea..000000000 --- a/cmd/frostfs-cli/modules/util/locode_info.go +++ /dev/null @@ -1,56 +0,0 @@ -package util - -import ( - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db" - locodebolt "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db/boltdb" - "github.com/spf13/cobra" -) - -const ( - locodeInfoDBFlag = "db" - locodeInfoCodeFlag = "locode" -) - -var ( - locodeInfoDBPath string - locodeInfoCode string - - locodeInfoCmd = &cobra.Command{ - Use: "info", - Short: "Print information about UN/LOCODE from FrostFS database", - Run: func(cmd *cobra.Command, _ []string) { - targetDB := locodebolt.New(locodebolt.Prm{ - Path: locodeInfoDBPath, - }, locodebolt.ReadOnly()) - - err := targetDB.Open() - commonCmd.ExitOnErr(cmd, "", err) - - defer targetDB.Close() - - record, err := locodedb.LocodeRecord(targetDB, locodeInfoCode) - commonCmd.ExitOnErr(cmd, "", err) - - cmd.Printf("Country: %s\n", record.CountryName()) - cmd.Printf("Location: %s\n", record.LocationName()) - cmd.Printf("Continent: %s\n", record.Continent()) - if subDivCode := record.SubDivCode(); subDivCode != "" { - cmd.Printf("Subdivision: [%s] %s\n", subDivCode, record.SubDivName()) - } - - geoPoint := record.GeoPoint() - cmd.Printf("Coordinates: %0.2f, %0.2f\n", geoPoint.Latitude(), geoPoint.Longitude()) - }, - } -) - -func initUtilLocodeInfoCmd() { - flags := locodeInfoCmd.Flags() - - flags.StringVar(&locodeInfoDBPath, locodeInfoDBFlag, "", "Path to FrostFS UN/LOCODE database") - _ = locodeInfoCmd.MarkFlagRequired(locodeInfoDBFlag) - - flags.StringVar(&locodeInfoCode, locodeInfoCodeFlag, "", "UN/LOCODE") - _ = locodeInfoCmd.MarkFlagRequired(locodeInfoCodeFlag) -} diff --git a/cmd/frostfs-cli/modules/util/root.go b/cmd/frostfs-cli/modules/util/root.go index 4a6b4403b..a909e6899 100644 --- a/cmd/frostfs-cli/modules/util/root.go +++ b/cmd/frostfs-cli/modules/util/root.go @@ -23,11 +23,9 @@ func init() { signCmd, convertCmd, keyerCmd, - locodeCmd, ) initSignCmd() initConvertCmd() initKeyerCmd() - initLocodeCmd() } From 9c2c76ca32be3aef1169480fdce203e3947805d3 Mon Sep 17 00:00:00 2001 From: George Bartolomey Date: Mon, 22 Jul 2024 11:12:41 +0300 Subject: [PATCH 1247/1943] [#1259] Move pkg/util/locode to frostfs-locode-db Removed pkg/util/locode package, added git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode dependency. Signed-off-by: George Bartolomey --- go.mod | 3 +- go.sum | 39 +--- pkg/innerring/locode.go | 6 +- .../netmap/nodevalidation/locode/calls.go | 2 +- .../nodevalidation/locode/calls_test.go | 4 +- .../netmap/nodevalidation/locode/deps.go | 4 +- pkg/util/locode/column/coordinates.go | 193 ----------------- pkg/util/locode/column/country.go | 38 ---- pkg/util/locode/column/location.go | 38 ---- pkg/util/locode/column/util.go | 9 - pkg/util/locode/db/airports/calls.go | 194 ------------------ pkg/util/locode/db/airports/db.go | 83 -------- pkg/util/locode/db/airports/opts.go | 19 -- pkg/util/locode/db/boltdb/calls.go | 166 --------------- pkg/util/locode/db/boltdb/db.go | 73 ------- pkg/util/locode/db/boltdb/opts.go | 37 ---- pkg/util/locode/db/continent.go | 81 -------- .../locode/db/continents/geojson/calls.go | 98 --------- pkg/util/locode/db/continents/geojson/db.go | 63 ------ pkg/util/locode/db/continents/geojson/opts.go | 10 - pkg/util/locode/db/country.go | 32 --- pkg/util/locode/db/db.go | 183 ----------------- pkg/util/locode/db/location.go | 32 --- pkg/util/locode/db/point.go | 93 --------- pkg/util/locode/db/point_test.go | 51 ----- pkg/util/locode/db/record.go | 140 ------------- pkg/util/locode/record.go | 83 -------- pkg/util/locode/table/csv/calls.go | 156 -------------- pkg/util/locode/table/csv/opts.go | 28 --- pkg/util/locode/table/csv/table.go | 75 ------- 30 files changed, 11 insertions(+), 2022 deletions(-) delete mode 100644 pkg/util/locode/column/coordinates.go delete mode 100644 pkg/util/locode/column/country.go delete mode 100644 pkg/util/locode/column/location.go delete mode 100644 pkg/util/locode/column/util.go delete mode 100644 pkg/util/locode/db/airports/calls.go delete mode 100644 pkg/util/locode/db/airports/db.go delete mode 100644 pkg/util/locode/db/airports/opts.go delete mode 100644 pkg/util/locode/db/boltdb/calls.go delete mode 100644 pkg/util/locode/db/boltdb/db.go delete mode 100644 pkg/util/locode/db/boltdb/opts.go delete mode 100644 pkg/util/locode/db/continent.go delete mode 100644 pkg/util/locode/db/continents/geojson/calls.go delete mode 100644 pkg/util/locode/db/continents/geojson/db.go delete mode 100644 pkg/util/locode/db/continents/geojson/opts.go delete mode 100644 pkg/util/locode/db/country.go delete mode 100644 pkg/util/locode/db/db.go delete mode 100644 pkg/util/locode/db/location.go delete mode 100644 pkg/util/locode/db/point.go delete mode 100644 pkg/util/locode/db/point_test.go delete mode 100644 pkg/util/locode/db/record.go delete mode 100644 pkg/util/locode/record.go delete mode 100644 pkg/util/locode/table/csv/calls.go delete mode 100644 pkg/util/locode/table/csv/opts.go delete mode 100644 pkg/util/locode/table/csv/table.go diff --git a/go.mod b/go.mod index 0f608f74d..6a97f7850 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 + git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de git.frostfs.info/TrueCloudLab/hrw v1.2.1 @@ -26,7 +27,6 @@ require ( github.com/nspcc-dev/neo-go v0.106.0 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 - github.com/paulmach/orb v0.11.0 github.com/prometheus/client_golang v1.19.0 github.com/spf13/cast v1.6.0 github.com/spf13/cobra v1.8.1 @@ -51,7 +51,6 @@ require ( github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - go.mongodb.org/mongo-driver v1.13.1 // indirect ) require ( diff --git a/go.sum b/go.sum index 355e176f7..d4cf863fa 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= +git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d h1:uJ/wvuMdepbkaV8XMS5uN9B0FQWMep0CttSuDZiDhq0= +git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de h1:OjsWY0jpGJV1t87XgwL/3PsDx7fJ6lfNMXtY8UhoUbM= @@ -81,7 +83,6 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-pkgz/expirable-cache/v3 v3.0.0 h1:u3/gcu3sabLYiTCevoRKv+WzjIn5oo7P8XtiXBeRDLw= github.com/go-pkgz/expirable-cache/v3 v3.0.0/go.mod h1:2OQiDyEGQalYecLWmXprm3maPXeVb5/6/X7yRPYTzec= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -91,13 +92,11 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -129,20 +128,14 @@ github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/reedsolomon v1.12.1 h1:NhWgum1efX1x58daOBGCFWcxtEhOhXKKl1HAPQUp03Q= github.com/klauspost/reedsolomon v1.12.1/go.mod h1:nEi5Kjb6QqtbofI6s+cbG/j1da11c96IBYBSnVGtuBs= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= @@ -171,7 +164,6 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= @@ -210,12 +202,8 @@ github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/panjf2000/ants/v2 v2.9.0 h1:SztCLkVxBRigbg+vt0S5QvF5vxAbxbKt09/YfAJ0tEo= github.com/panjf2000/ants/v2 v2.9.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= -github.com/paulmach/orb v0.11.0 h1:JfVXJUBeH9ifc/OrhBY0lL16QsmPgpCHMlqSSYhcgAA= -github.com/paulmach/orb v0.11.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= -github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -259,7 +247,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= @@ -272,25 +259,14 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= -github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk= -go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= @@ -318,13 +294,11 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -333,12 +307,10 @@ golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -348,9 +320,7 @@ golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= @@ -393,7 +363,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -401,9 +370,7 @@ golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= @@ -427,11 +394,9 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/pkg/innerring/locode.go b/pkg/innerring/locode.go index a9a9498b6..a0c3ea751 100644 --- a/pkg/innerring/locode.go +++ b/pkg/innerring/locode.go @@ -1,11 +1,11 @@ package innerring import ( + "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode" + locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db" + locodebolt "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db/boltdb" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap" irlocode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/locode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" - locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db" - locodebolt "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db/boltdb" "github.com/spf13/viper" ) diff --git a/pkg/innerring/processors/netmap/nodevalidation/locode/calls.go b/pkg/innerring/processors/netmap/nodevalidation/locode/calls.go index d071a7792..5e0558344 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/locode/calls.go +++ b/pkg/innerring/processors/netmap/nodevalidation/locode/calls.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" + "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) diff --git a/pkg/innerring/processors/netmap/nodevalidation/locode/calls_test.go b/pkg/innerring/processors/netmap/nodevalidation/locode/calls_test.go index 6697391e8..8ab174dfd 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/locode/calls_test.go +++ b/pkg/innerring/processors/netmap/nodevalidation/locode/calls_test.go @@ -5,9 +5,9 @@ import ( "fmt" "testing" + locodestd "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode" + locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/locode" - locodestd "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" - locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/stretchr/testify/require" ) diff --git a/pkg/innerring/processors/netmap/nodevalidation/locode/deps.go b/pkg/innerring/processors/netmap/nodevalidation/locode/deps.go index e6332261e..8f6667933 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/locode/deps.go +++ b/pkg/innerring/processors/netmap/nodevalidation/locode/deps.go @@ -1,8 +1,8 @@ package locode import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" - locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db" + "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode" + locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db" ) // Record is an interface of read-only diff --git a/pkg/util/locode/column/coordinates.go b/pkg/util/locode/column/coordinates.go deleted file mode 100644 index 5e32c016e..000000000 --- a/pkg/util/locode/column/coordinates.go +++ /dev/null @@ -1,193 +0,0 @@ -package locodecolumn - -import ( - "fmt" - "strings" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" -) - -const ( - minutesDigits = 2 - hemisphereSymbols = 1 -) - -const ( - latDegDigits = 2 - lngDegDigits = 3 -) - -type coordinateCode struct { - degDigits int - value []uint8 -} - -// LongitudeCode represents the value of the longitude -// of the location conforming to UN/LOCODE specification. -type LongitudeCode coordinateCode - -// LongitudeHemisphere represents the hemisphere of the earth -// // along the Greenwich meridian. -type LongitudeHemisphere [hemisphereSymbols]uint8 - -// LatitudeCode represents the value of the latitude -// of the location conforming to UN/LOCODE specification. -type LatitudeCode coordinateCode - -// LatitudeHemisphere represents the hemisphere of the earth -// along the equator. -type LatitudeHemisphere [hemisphereSymbols]uint8 - -func coordinateFromString(s string, degDigits int, hemisphereAlphabet []uint8) (*coordinateCode, error) { - if len(s) != degDigits+minutesDigits+hemisphereSymbols { - return nil, locode.ErrInvalidString - } - - for i := range s[:degDigits+minutesDigits] { - if !isDigit(s[i]) { - return nil, locode.ErrInvalidString - } - } - -loop: - for _, sym := range s[degDigits+minutesDigits:] { - for j := range hemisphereAlphabet { - if hemisphereAlphabet[j] == uint8(sym) { - continue loop - } - } - - return nil, locode.ErrInvalidString - } - - return &coordinateCode{ - degDigits: degDigits, - value: []uint8(s), - }, nil -} - -// LongitudeFromString parses a string and returns the location's longitude. -func LongitudeFromString(s string) (*LongitudeCode, error) { - cc, err := coordinateFromString(s, lngDegDigits, []uint8{'W', 'E'}) - if err != nil { - return nil, err - } - - return (*LongitudeCode)(cc), nil -} - -// LatitudeFromString parses a string and returns the location's latitude. -func LatitudeFromString(s string) (*LatitudeCode, error) { - cc, err := coordinateFromString(s, latDegDigits, []uint8{'N', 'S'}) - if err != nil { - return nil, err - } - - return (*LatitudeCode)(cc), nil -} - -func (cc *coordinateCode) degrees() []uint8 { - return cc.value[:cc.degDigits] -} - -// Degrees returns the longitude's degrees. -func (lc *LongitudeCode) Degrees() (l [lngDegDigits]uint8) { - copy(l[:], (*coordinateCode)(lc).degrees()) - return -} - -// Degrees returns the latitude's degrees. -func (lc *LatitudeCode) Degrees() (l [latDegDigits]uint8) { - copy(l[:], (*coordinateCode)(lc).degrees()) - return -} - -func (cc *coordinateCode) minutes() (mnt [minutesDigits]uint8) { - for i := 0; i < minutesDigits; i++ { - mnt[i] = cc.value[cc.degDigits+i] - } - - return -} - -// Minutes returns the longitude's minutes. -func (lc *LongitudeCode) Minutes() [minutesDigits]uint8 { - return (*coordinateCode)(lc).minutes() -} - -// Minutes returns the latitude's minutes. -func (lc *LatitudeCode) Minutes() [minutesDigits]uint8 { - return (*coordinateCode)(lc).minutes() -} - -// Hemisphere returns the longitude's hemisphere code. -func (lc *LongitudeCode) Hemisphere() LongitudeHemisphere { - return (*coordinateCode)(lc).hemisphere() -} - -// Hemisphere returns the latitude's hemisphere code. -func (lc *LatitudeCode) Hemisphere() LatitudeHemisphere { - return (*coordinateCode)(lc).hemisphere() -} - -func (cc *coordinateCode) hemisphere() (h [hemisphereSymbols]uint8) { - for i := 0; i < hemisphereSymbols; i++ { - h[i] = cc.value[cc.degDigits+minutesDigits+i] - } - - return h -} - -// North returns true for the northern hemisphere. -func (h LatitudeHemisphere) North() bool { - return h[0] == 'N' -} - -// East returns true for the eastern hemisphere. -func (h LongitudeHemisphere) East() bool { - return h[0] == 'E' -} - -// Coordinates represents the coordinates of the location from UN/LOCODE table. -type Coordinates struct { - lat *LatitudeCode - - lng *LongitudeCode -} - -// Latitude returns the location's latitude. -func (c *Coordinates) Latitude() *LatitudeCode { - return c.lat -} - -// Longitude returns the location's longitude. -func (c *Coordinates) Longitude() *LongitudeCode { - return c.lng -} - -// CoordinatesFromString parses a string and returns the location's coordinates. -func CoordinatesFromString(s string) (*Coordinates, error) { - if len(s) == 0 { - return nil, nil - } - - strs := strings.Split(s, " ") - if len(strs) != 2 { - return nil, locode.ErrInvalidString - } - - lat, err := LatitudeFromString(strs[0]) - if err != nil { - return nil, fmt.Errorf("could not parse latitude: %w", err) - } - - lng, err := LongitudeFromString(strs[1]) - if err != nil { - return nil, fmt.Errorf("could not parse longitude: %w", err) - } - - return &Coordinates{ - lat: lat, - lng: lng, - }, nil -} diff --git a/pkg/util/locode/column/country.go b/pkg/util/locode/column/country.go deleted file mode 100644 index 7b29a97c5..000000000 --- a/pkg/util/locode/column/country.go +++ /dev/null @@ -1,38 +0,0 @@ -package locodecolumn - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" -) - -const countryCodeLen = 2 - -// CountryCode represents ISO 3166 alpha-2 Country Code. -type CountryCode [countryCodeLen]uint8 - -// Symbols returns digits of the country code. -func (cc *CountryCode) Symbols() [countryCodeLen]uint8 { - return *cc -} - -// CountryCodeFromString parses a string and returns the country code. -func CountryCodeFromString(s string) (*CountryCode, error) { - if l := len(s); l != countryCodeLen { - return nil, fmt.Errorf("incorrect country code length: expect: %d, got: %d", - countryCodeLen, - l, - ) - } - - for i := range s { - if !isUpperAlpha(s[i]) { - return nil, locode.ErrInvalidString - } - } - - cc := CountryCode{} - copy(cc[:], s) - - return &cc, nil -} diff --git a/pkg/util/locode/column/location.go b/pkg/util/locode/column/location.go deleted file mode 100644 index 4303228fb..000000000 --- a/pkg/util/locode/column/location.go +++ /dev/null @@ -1,38 +0,0 @@ -package locodecolumn - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" -) - -const locationCodeLen = 3 - -// LocationCode represents 3-character code for the location. -type LocationCode [locationCodeLen]uint8 - -// Symbols returns characters of the location code. -func (lc *LocationCode) Symbols() [locationCodeLen]uint8 { - return *lc -} - -// LocationCodeFromString parses a string and returns the location code. -func LocationCodeFromString(s string) (*LocationCode, error) { - if l := len(s); l != locationCodeLen { - return nil, fmt.Errorf("incorrect location code length: expect: %d, got: %d", - locationCodeLen, - l, - ) - } - - for i := range s { - if !isUpperAlpha(s[i]) && !isDigit(s[i]) { - return nil, locode.ErrInvalidString - } - } - - lc := LocationCode{} - copy(lc[:], s) - - return &lc, nil -} diff --git a/pkg/util/locode/column/util.go b/pkg/util/locode/column/util.go deleted file mode 100644 index 8da1f9a25..000000000 --- a/pkg/util/locode/column/util.go +++ /dev/null @@ -1,9 +0,0 @@ -package locodecolumn - -func isDigit(sym uint8) bool { - return sym >= '0' && sym <= '9' -} - -func isUpperAlpha(sym uint8) bool { - return sym >= 'A' && sym <= 'Z' -} diff --git a/pkg/util/locode/db/airports/calls.go b/pkg/util/locode/db/airports/calls.go deleted file mode 100644 index dac8cce8b..000000000 --- a/pkg/util/locode/db/airports/calls.go +++ /dev/null @@ -1,194 +0,0 @@ -package airportsdb - -import ( - "encoding/csv" - "errors" - "fmt" - "io" - "os" - "strconv" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" - locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db" -) - -const ( - _ = iota - 1 - - _ // Airport ID - _ // Name - airportCity - airportCountry - airportIATA - _ // ICAO - airportLatitude - airportLongitude - _ // Altitude - _ // Timezone - _ // DST - _ // Tz database time zone - _ // Type - _ // Source - - airportFldNum -) - -type record struct { - city, - country, - iata, - lat, - lng string -} - -// Get scans the records of the OpenFlights Airport to an in-memory table (once), -// and returns an entry that matches the passed UN/LOCODE record. -// -// Records are matched if they have the same country code and either -// same IATA code or same city name (location name in UN/LOCODE). -// -// Returns locodedb.ErrAirportNotFound if no entry matches. -func (db *DB) Get(locodeRecord locode.Record) (*locodedb.AirportRecord, error) { - if err := db.initAirports(); err != nil { - return nil, err - } - - records := db.mAirports[locodeRecord.LOCODE.CountryCode()] - - for i := range records { - if locodeRecord.LOCODE.LocationCode() != records[i].iata && - locodeRecord.NameWoDiacritics != records[i].city { - continue - } - - lat, err := strconv.ParseFloat(records[i].lat, 64) - if err != nil { - return nil, err - } - - lng, err := strconv.ParseFloat(records[i].lng, 64) - if err != nil { - return nil, err - } - - return &locodedb.AirportRecord{ - CountryName: records[i].country, - Point: locodedb.NewPoint(lat, lng), - }, nil - } - - return nil, locodedb.ErrAirportNotFound -} - -const ( - _ = iota - 1 - - countryName - countryISOCode - _ // dafif_code - - countryFldNum -) - -// CountryName scans the records of the OpenFlights Country table to an in-memory table (once), -// and returns the name of the country by code. -// -// Returns locodedb.ErrCountryNotFound if no entry matches. -func (db *DB) CountryName(code *locodedb.CountryCode) (name string, err error) { - if err = db.initCountries(); err != nil { - return - } - - argCode := code.String() - - for cName, cCode := range db.mCountries { - if cCode == argCode { - name = cName - break - } - } - - if name == "" { - err = locodedb.ErrCountryNotFound - } - - return -} - -func (db *DB) initAirports() (err error) { - db.airportsOnce.Do(func() { - db.mAirports = make(map[string][]record) - - if err = db.initCountries(); err != nil { - return - } - - err = db.scanWords(db.airports, airportFldNum, func(words []string) error { - countryCode := db.mCountries[words[airportCountry]] - if countryCode != "" { - db.mAirports[countryCode] = append(db.mAirports[countryCode], record{ - city: words[airportCity], - country: words[airportCountry], - iata: words[airportIATA], - lat: words[airportLatitude], - lng: words[airportLongitude], - }) - } - - return nil - }) - }) - - return -} - -func (db *DB) initCountries() (err error) { - db.countriesOnce.Do(func() { - db.mCountries = make(map[string]string) - - err = db.scanWords(db.countries, countryFldNum, func(words []string) error { - db.mCountries[words[countryName]] = words[countryISOCode] - - return nil - }) - }) - - return -} - -var errScanInt = errors.New("interrupt scan") - -func (db *DB) scanWords(pm pathMode, num int, wordsHandler func([]string) error) error { - tableFile, err := os.OpenFile(pm.path, os.O_RDONLY, pm.mode) - if err != nil { - return err - } - - defer tableFile.Close() - - r := csv.NewReader(tableFile) - r.ReuseRecord = true - - for { - words, err := r.Read() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - - return err - } else if ln := len(words); ln != num { - return fmt.Errorf("unexpected number of words %d", ln) - } - - if err := wordsHandler(words); err != nil { - if errors.Is(err, errScanInt) { - break - } - - return err - } - } - - return nil -} diff --git a/pkg/util/locode/db/airports/db.go b/pkg/util/locode/db/airports/db.go deleted file mode 100644 index acfa3fd60..000000000 --- a/pkg/util/locode/db/airports/db.go +++ /dev/null @@ -1,83 +0,0 @@ -package airportsdb - -import ( - "fmt" - "io/fs" - "sync" -) - -// Prm groups the required parameters of the DB's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Path to OpenFlights Airport csv table. - // - // Must not be empty. - AirportsPath string - - // Path to OpenFlights Countries csv table. - // - // Must not be empty. - CountriesPath string -} - -// DB is a descriptor of the OpenFlights database in csv format. -// -// For correct operation, DB must be created -// using the constructor (New) based on the required parameters -// and optional components. After successful creation, -// The DB is immediately ready to work through API. -type DB struct { - airports, countries pathMode - - airportsOnce, countriesOnce sync.Once - - mCountries map[string]string - - mAirports map[string][]record -} - -type pathMode struct { - path string - mode fs.FileMode -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the DB. -// -// Panics if at least one value of the parameters is invalid. -// -// The created DB does not require additional -// initialization and is completely ready for work. -func New(prm Prm, opts ...Option) *DB { - switch { - case prm.AirportsPath == "": - panicOnPrmValue("AirportsPath", prm.AirportsPath) - case prm.CountriesPath == "": - panicOnPrmValue("CountriesPath", prm.CountriesPath) - } - - o := defaultOpts() - - for i := range opts { - opts[i](o) - } - - return &DB{ - airports: pathMode{ - path: prm.AirportsPath, - mode: o.airportMode, - }, - countries: pathMode{ - path: prm.CountriesPath, - mode: o.countryMode, - }, - } -} diff --git a/pkg/util/locode/db/airports/opts.go b/pkg/util/locode/db/airports/opts.go deleted file mode 100644 index 3799d9e27..000000000 --- a/pkg/util/locode/db/airports/opts.go +++ /dev/null @@ -1,19 +0,0 @@ -package airportsdb - -import ( - "io/fs" -) - -// Option sets an optional parameter of DB. -type Option func(*options) - -type options struct { - airportMode, countryMode fs.FileMode -} - -func defaultOpts() *options { - return &options{ - airportMode: fs.ModePerm, // 0777 - countryMode: fs.ModePerm, // 0777 - } -} diff --git a/pkg/util/locode/db/boltdb/calls.go b/pkg/util/locode/db/boltdb/calls.go deleted file mode 100644 index 6a80def3a..000000000 --- a/pkg/util/locode/db/boltdb/calls.go +++ /dev/null @@ -1,166 +0,0 @@ -package locodebolt - -import ( - "encoding/json" - "errors" - "fmt" - "path/filepath" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db" - "go.etcd.io/bbolt" -) - -// Open opens an underlying BoltDB instance. -// -// Timeout of BoltDB opening is 3s (only for Linux or Darwin). -// -// Opens BoltDB in read-only mode if DB is read-only. -func (db *DB) Open() error { - // copy-paste from metabase: - // consider universal Open/Close for BoltDB wrappers - - err := util.MkdirAllX(filepath.Dir(db.path), db.mode) - if err != nil { - return fmt.Errorf("could not create dir for BoltDB: %w", err) - } - - db.bolt, err = bbolt.Open(db.path, db.mode, db.boltOpts) - if err != nil { - return fmt.Errorf("could not open BoltDB: %w", err) - } - - return nil -} - -// Close closes an underlying BoltDB instance. -// -// Must not be called before successful Open call. -func (db *DB) Close() error { - return db.bolt.Close() -} - -func countryBucketKey(cc *locodedb.CountryCode) ([]byte, error) { - return []byte(cc.String()), nil -} - -func locationBucketKey(lc *locodedb.LocationCode) ([]byte, error) { - return []byte(lc.String()), nil -} - -type recordJSON struct { - CountryName string - LocationName string - SubDivName string - SubDivCode string - Latitude float64 - Longitude float64 - Continent string -} - -func recordValue(r locodedb.Record) ([]byte, error) { - p := r.GeoPoint() - - rj := &recordJSON{ - CountryName: r.CountryName(), - LocationName: r.LocationName(), - SubDivName: r.SubDivName(), - SubDivCode: r.SubDivCode(), - Latitude: p.Latitude(), - Longitude: p.Longitude(), - Continent: r.Continent().String(), - } - - return json.Marshal(rj) -} - -func recordFromValue(data []byte) (*locodedb.Record, error) { - rj := new(recordJSON) - - if err := json.Unmarshal(data, rj); err != nil { - return nil, err - } - - r := new(locodedb.Record) - r.SetCountryName(rj.CountryName) - r.SetLocationName(rj.LocationName) - r.SetSubDivName(rj.SubDivName) - r.SetSubDivCode(rj.SubDivCode) - r.SetGeoPoint(locodedb.NewPoint(rj.Latitude, rj.Longitude)) - - cont := locodedb.ContinentFromString(rj.Continent) - r.SetContinent(&cont) - - return r, nil -} - -// Put saves the record by key in an underlying BoltDB instance. -// -// Country code from the key is used for allocating the 1st level buckets. -// Records are stored in country buckets by the location code from the key. -// The records are stored in internal binary JSON format. -// -// Must not be called before successful Open call. -// Must not be called in read-only mode: behavior is undefined. -func (db *DB) Put(key locodedb.Key, rec locodedb.Record) error { - return db.bolt.Batch(func(tx *bbolt.Tx) error { - countryKey, err := countryBucketKey(key.CountryCode()) - if err != nil { - return err - } - - bktCountry, err := tx.CreateBucketIfNotExists(countryKey) - if err != nil { - return fmt.Errorf("could not create country bucket: %w", err) - } - - locationKey, err := locationBucketKey(key.LocationCode()) - if err != nil { - return err - } - - cont, err := recordValue(rec) - if err != nil { - return err - } - - return bktCountry.Put(locationKey, cont) - }) -} - -var errRecordNotFound = errors.New("record not found") - -// Get reads the record by key from underlying BoltDB instance. -// -// Returns an error if no record is presented by key in DB. -// -// Must not be called before successful Open call. -func (db *DB) Get(key locodedb.Key) (rec *locodedb.Record, err error) { - err = db.bolt.View(func(tx *bbolt.Tx) error { - countryKey, err := countryBucketKey(key.CountryCode()) - if err != nil { - return err - } - - bktCountry := tx.Bucket(countryKey) - if bktCountry == nil { - return errRecordNotFound - } - - locationKey, err := locationBucketKey(key.LocationCode()) - if err != nil { - return err - } - - data := bktCountry.Get(locationKey) - if data == nil { - return errRecordNotFound - } - - rec, err = recordFromValue(data) - - return err - }) - - return -} diff --git a/pkg/util/locode/db/boltdb/db.go b/pkg/util/locode/db/boltdb/db.go deleted file mode 100644 index 3d09a797d..000000000 --- a/pkg/util/locode/db/boltdb/db.go +++ /dev/null @@ -1,73 +0,0 @@ -package locodebolt - -import ( - "fmt" - "io/fs" - - "go.etcd.io/bbolt" -) - -// Prm groups the required parameters of the DB's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Path to BoltDB file with FrostFS location database. - // - // Must not be empty. - Path string -} - -// DB is a descriptor of the FrostFS BoltDB location database. -// -// For correct operation, DB must be created -// using the constructor (New) based on the required parameters -// and optional components. -// -// After successful creation, -// DB must be opened through Open call. After successful opening, -// DB is ready to work through API (until Close call). -// -// Upon completion of work with the DB, it must be closed -// by Close method. -type DB struct { - path string - - mode fs.FileMode - - boltOpts *bbolt.Options - - bolt *bbolt.DB -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the DB. -// -// Panics if at least one value of the parameters is invalid. -// -// The created DB requires calling the Open method in order -// to initialize required resources. -func New(prm Prm, opts ...Option) *DB { - switch { - case prm.Path == "": - panicOnPrmValue("Path", prm.Path) - } - - o := defaultOpts() - - for i := range opts { - opts[i](o) - } - - return &DB{ - path: prm.Path, - mode: o.mode, - boltOpts: o.boltOpts, - } -} diff --git a/pkg/util/locode/db/boltdb/opts.go b/pkg/util/locode/db/boltdb/opts.go deleted file mode 100644 index db0cccd3a..000000000 --- a/pkg/util/locode/db/boltdb/opts.go +++ /dev/null @@ -1,37 +0,0 @@ -package locodebolt - -import ( - "io/fs" - "os" - "time" - - "go.etcd.io/bbolt" -) - -// Option sets an optional parameter of DB. -type Option func(*options) - -type options struct { - mode fs.FileMode - - boltOpts *bbolt.Options -} - -func defaultOpts() *options { - return &options{ - mode: os.ModePerm, // 0777 - boltOpts: &bbolt.Options{ - Timeout: 3 * time.Second, - }, - } -} - -// ReadOnly enables read-only mode of the DB. -// -// Do not call DB.Put method on instances with -// this option: the behavior is undefined. -func ReadOnly() Option { - return func(o *options) { - o.boltOpts.ReadOnly = true - } -} diff --git a/pkg/util/locode/db/continent.go b/pkg/util/locode/db/continent.go deleted file mode 100644 index 863af7b57..000000000 --- a/pkg/util/locode/db/continent.go +++ /dev/null @@ -1,81 +0,0 @@ -package locodedb - -// Continent is an enumeration of Earth's continent. -type Continent uint8 - -const ( - // ContinentUnknown is an undefined Continent value. - ContinentUnknown = iota - - // ContinentEurope corresponds to Europe. - ContinentEurope - - // ContinentAfrica corresponds to Africa. - ContinentAfrica - - // ContinentNorthAmerica corresponds to North America. - ContinentNorthAmerica - - // ContinentSouthAmerica corresponds to South America. - ContinentSouthAmerica - - // ContinentAsia corresponds to Asia. - ContinentAsia - - // ContinentAntarctica corresponds to Antarctica. - ContinentAntarctica - - // ContinentOceania corresponds to Oceania. - ContinentOceania -) - -// Is checks if c is the same continent as c2. -func (c *Continent) Is(c2 Continent) bool { - return *c == c2 -} - -func (c Continent) String() string { - switch c { - case ContinentUnknown: - fallthrough - default: - return "Unknown" - case ContinentEurope: - return "Europe" - case ContinentAfrica: - return "Africa" - case ContinentNorthAmerica: - return "North America" - case ContinentSouthAmerica: - return "South America" - case ContinentAsia: - return "Asia" - case ContinentAntarctica: - return "Antarctica" - case ContinentOceania: - return "Oceania" - } -} - -// ContinentFromString returns Continent value -// corresponding to the passed string representation. -func ContinentFromString(str string) Continent { - switch str { - default: - return ContinentUnknown - case "Europe": - return ContinentEurope - case "Africa": - return ContinentAfrica - case "North America": - return ContinentNorthAmerica - case "South America": - return ContinentSouthAmerica - case "Asia": - return ContinentAsia - case "Antarctica": - return ContinentAntarctica - case "Oceania": - return ContinentOceania - } -} diff --git a/pkg/util/locode/db/continents/geojson/calls.go b/pkg/util/locode/db/continents/geojson/calls.go deleted file mode 100644 index 34467d5a2..000000000 --- a/pkg/util/locode/db/continents/geojson/calls.go +++ /dev/null @@ -1,98 +0,0 @@ -package continentsdb - -import ( - "fmt" - "os" - - locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db" - "github.com/paulmach/orb" - "github.com/paulmach/orb/geojson" - "github.com/paulmach/orb/planar" -) - -const continentProperty = "Continent" - -// PointContinent goes through all polygons and returns the continent -// in which the point is located. -// -// Returns locodedb.ContinentUnknown if no entry matches. -// -// All GeoJSON feature are parsed from file once and stored in memory. -func (db *DB) PointContinent(point *locodedb.Point) (*locodedb.Continent, error) { - var err error - - db.once.Do(func() { - err = db.init() - }) - - if err != nil { - return nil, err - } - - planarPoint := orb.Point{point.Longitude(), point.Latitude()} - - var ( - continent string - minDst float64 - ) - - for _, feature := range db.features { - if multiPolygon, ok := feature.Geometry.(orb.MultiPolygon); ok { - if planar.MultiPolygonContains(multiPolygon, planarPoint) { - continent = feature.Properties.MustString(continentProperty) - break - } - } else if polygon, ok := feature.Geometry.(orb.Polygon); ok { - if planar.PolygonContains(polygon, planarPoint) { - continent = feature.Properties.MustString(continentProperty) - break - } - } - distance := planar.DistanceFrom(feature.Geometry, planarPoint) - if minDst == 0 || minDst > distance { - minDst = distance - continent = feature.Properties.MustString(continentProperty) - } - } - - c := continentFromString(continent) - - return &c, nil -} - -func (db *DB) init() error { - data, err := os.ReadFile(db.path) - if err != nil { - return fmt.Errorf("could not read data file: %w", err) - } - - features, err := geojson.UnmarshalFeatureCollection(data) - if err != nil { - return fmt.Errorf("could not unmarshal GeoJSON feature collection: %w", err) - } - - db.features = features.Features - - return nil -} - -func continentFromString(c string) locodedb.Continent { - switch c { - default: - return locodedb.ContinentUnknown - case "Africa": - return locodedb.ContinentAfrica - case "Asia": - return locodedb.ContinentAsia - case "Europe": - return locodedb.ContinentEurope - case "North America": - return locodedb.ContinentNorthAmerica - case "South America": - return locodedb.ContinentSouthAmerica - case "Antarctica": - return locodedb.ContinentAntarctica - case "Australia", "Oceania": - return locodedb.ContinentOceania - } -} diff --git a/pkg/util/locode/db/continents/geojson/db.go b/pkg/util/locode/db/continents/geojson/db.go deleted file mode 100644 index ee43bd810..000000000 --- a/pkg/util/locode/db/continents/geojson/db.go +++ /dev/null @@ -1,63 +0,0 @@ -package continentsdb - -import ( - "fmt" - "sync" - - "github.com/paulmach/orb/geojson" -) - -// Prm groups the required parameters of the DB's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Path to polygons of Earth's continents in GeoJSON format. - // - // Must not be empty. - Path string -} - -// DB is a descriptor of the Earth's polygons in GeoJSON format. -// -// For correct operation, DB must be created -// using the constructor (New) based on the required parameters -// and optional components. After successful creation, -// The DB is immediately ready to work through API. -type DB struct { - path string - - once sync.Once - - features []*geojson.Feature -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the DB. -// -// Panics if at least one value of the parameters is invalid. -// -// The created DB does not require additional -// initialization and is completely ready for work. -func New(prm Prm, opts ...Option) *DB { - switch { - case prm.Path == "": - panicOnPrmValue("Path", prm.Path) - } - - o := defaultOpts() - - for i := range opts { - opts[i](o) - } - - return &DB{ - path: prm.Path, - } -} diff --git a/pkg/util/locode/db/continents/geojson/opts.go b/pkg/util/locode/db/continents/geojson/opts.go deleted file mode 100644 index 59831fcc5..000000000 --- a/pkg/util/locode/db/continents/geojson/opts.go +++ /dev/null @@ -1,10 +0,0 @@ -package continentsdb - -// Option sets an optional parameter of DB. -type Option func(*options) - -type options struct{} - -func defaultOpts() *options { - return &options{} -} diff --git a/pkg/util/locode/db/country.go b/pkg/util/locode/db/country.go deleted file mode 100644 index 2d13c6ef9..000000000 --- a/pkg/util/locode/db/country.go +++ /dev/null @@ -1,32 +0,0 @@ -package locodedb - -import ( - "fmt" - - locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/column" -) - -// CountryCode represents a country code for -// the storage in the FrostFS location database. -type CountryCode locodecolumn.CountryCode - -// CountryCodeFromString parses a string UN/LOCODE country code -// and returns a CountryCode. -func CountryCodeFromString(s string) (*CountryCode, error) { - cc, err := locodecolumn.CountryCodeFromString(s) - if err != nil { - return nil, fmt.Errorf("could not parse country code: %w", err) - } - - return CountryFromColumn(cc) -} - -// CountryFromColumn converts a UN/LOCODE country code to a CountryCode. -func CountryFromColumn(cc *locodecolumn.CountryCode) (*CountryCode, error) { - return (*CountryCode)(cc), nil -} - -func (c *CountryCode) String() string { - syms := (*locodecolumn.CountryCode)(c).Symbols() - return string(syms[:]) -} diff --git a/pkg/util/locode/db/db.go b/pkg/util/locode/db/db.go deleted file mode 100644 index 8c71ea794..000000000 --- a/pkg/util/locode/db/db.go +++ /dev/null @@ -1,183 +0,0 @@ -package locodedb - -import ( - "errors" - "fmt" - "runtime" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" - "golang.org/x/sync/errgroup" -) - -// SourceTable is an interface of the UN/LOCODE table. -type SourceTable interface { - // Must iterate over all entries of the table - // and pass next entry to the handler. - // - // Must return handler's errors directly. - IterateAll(func(locode.Record) error) error -} - -// DB is an interface of FrostFS location database. -type DB interface { - // Must save the record by key in the database. - Put(Key, Record) error - - // Must return the record by key from the database. - Get(Key) (*Record, error) -} - -// AirportRecord represents the entry in FrostFS airport database. -type AirportRecord struct { - // Name of the country where airport is located. - CountryName string - - // Geo point where airport is located. - Point *Point -} - -// ErrAirportNotFound is returned by AirportRecord readers -// when the required airport is not found. -var ErrAirportNotFound = errors.New("airport not found") - -// AirportDB is an interface of FrostFS airport database. -type AirportDB interface { - // Must return the record by UN/LOCODE table record. - // - // Must return ErrAirportNotFound if there is no - // related airport in the database. - Get(locode.Record) (*AirportRecord, error) -} - -// ContinentsDB is an interface of FrostFS continent database. -type ContinentsDB interface { - // Must return continent of the geo point. - PointContinent(*Point) (*Continent, error) -} - -var ErrSubDivNotFound = errors.New("subdivision not found") - -var ErrCountryNotFound = errors.New("country not found") - -// NamesDB is an interface of the FrostFS location namespace. -type NamesDB interface { - // Must resolve a country code to a country name. - // - // Must return ErrCountryNotFound if there is no - // country with the provided code. - CountryName(*CountryCode) (string, error) - - // Must resolve (country code, subdivision code) to - // a subdivision name. - // - // Must return ErrSubDivNotFound if either country or - // subdivision is not presented in database. - SubDivName(*CountryCode, string) (string, error) -} - -// FillDatabase generates the FrostFS location database based on the UN/LOCODE table. -func FillDatabase(table SourceTable, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error { - var errG errgroup.Group - - // Pick some sane default, after this the performance stopped increasing. - errG.SetLimit(runtime.NumCPU() * 4) - _ = table.IterateAll(func(tableRecord locode.Record) error { - errG.Go(func() error { - return processTableRecord(tableRecord, airports, continents, names, db) - }) - return nil - }) - return errG.Wait() -} - -func processTableRecord(tableRecord locode.Record, airports AirportDB, continents ContinentsDB, names NamesDB, db DB) error { - if tableRecord.LOCODE.LocationCode() == "" { - return nil - } - - dbKey, err := NewKey(tableRecord.LOCODE) - if err != nil { - return err - } - - dbRecord, err := NewRecord(tableRecord) - if err != nil { - if errors.Is(err, errParseCoordinates) { - return nil - } - - return err - } - - geoPoint := dbRecord.GeoPoint() - countryName := "" - - if geoPoint == nil { - airportRecord, err := airports.Get(tableRecord) - if err != nil { - if errors.Is(err, ErrAirportNotFound) { - return nil - } - - return err - } - - geoPoint = airportRecord.Point - countryName = airportRecord.CountryName - } - - dbRecord.SetGeoPoint(geoPoint) - - if countryName == "" { - countryName, err = names.CountryName(dbKey.CountryCode()) - if err != nil { - if errors.Is(err, ErrCountryNotFound) { - return nil - } - - return err - } - } - - dbRecord.SetCountryName(countryName) - - if subDivCode := dbRecord.SubDivCode(); subDivCode != "" { - subDivName, err := names.SubDivName(dbKey.CountryCode(), subDivCode) - if err != nil { - if errors.Is(err, ErrSubDivNotFound) { - return nil - } - - return err - } - - dbRecord.SetSubDivName(subDivName) - } - - continent, err := continents.PointContinent(geoPoint) - if err != nil { - return fmt.Errorf("could not calculate continent geo point: %w", err) - } else if continent.Is(ContinentUnknown) { - return nil - } - - dbRecord.SetContinent(continent) - - return db.Put(*dbKey, *dbRecord) -} - -// LocodeRecord returns the record from the FrostFS location database -// corresponding to the string representation of UN/LOCODE. -func LocodeRecord(db DB, sLocode string) (*Record, error) { - lc, err := locode.FromString(sLocode) - if err != nil { - return nil, fmt.Errorf("could not parse locode: %w", err) - } - - key, err := NewKey(*lc) - if err != nil { - return nil, err - } - - return db.Get(*key) -} diff --git a/pkg/util/locode/db/location.go b/pkg/util/locode/db/location.go deleted file mode 100644 index d22979170..000000000 --- a/pkg/util/locode/db/location.go +++ /dev/null @@ -1,32 +0,0 @@ -package locodedb - -import ( - "fmt" - - locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/column" -) - -// LocationCode represents a location code for -// the storage in the FrostFS location database. -type LocationCode locodecolumn.LocationCode - -// LocationCodeFromString parses a string UN/LOCODE location code -// and returns a LocationCode. -func LocationCodeFromString(s string) (*LocationCode, error) { - lc, err := locodecolumn.LocationCodeFromString(s) - if err != nil { - return nil, fmt.Errorf("could not parse location code: %w", err) - } - - return LocationFromColumn(lc) -} - -// LocationFromColumn converts a UN/LOCODE country code to a LocationCode. -func LocationFromColumn(cc *locodecolumn.LocationCode) (*LocationCode, error) { - return (*LocationCode)(cc), nil -} - -func (l *LocationCode) String() string { - syms := (*locodecolumn.LocationCode)(l).Symbols() - return string(syms[:]) -} diff --git a/pkg/util/locode/db/point.go b/pkg/util/locode/db/point.go deleted file mode 100644 index 72daebb2c..000000000 --- a/pkg/util/locode/db/point.go +++ /dev/null @@ -1,93 +0,0 @@ -package locodedb - -import ( - "fmt" - "strconv" - - locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/column" -) - -// Point represents a 2D geographic point. -type Point struct { - lat, lng float64 -} - -// NewPoint creates, initializes and returns a new Point. -func NewPoint(lat, lng float64) *Point { - return &Point{ - lat: lat, - lng: lng, - } -} - -// Latitude returns the Point's latitude. -func (p Point) Latitude() float64 { - return p.lat -} - -// Longitude returns the Point's longitude. -func (p Point) Longitude() float64 { - return p.lng -} - -// PointFromCoordinates converts a UN/LOCODE coordinates to a Point. -func PointFromCoordinates(crd *locodecolumn.Coordinates) (*Point, error) { - if crd == nil { - return nil, nil - } - - cLat := crd.Latitude() - cLatDeg := cLat.Degrees() - cLatMnt := cLat.Minutes() - - lat, err := toDecimal(cLatDeg[:], cLatMnt[:]) - if err != nil { - return nil, fmt.Errorf("could not parse latitude: %w", err) - } - - if !cLat.Hemisphere().North() { - lat = -lat - } - - cLng := crd.Longitude() - cLngDeg := cLng.Degrees() - cLngMnt := cLng.Minutes() - - lng, err := toDecimal(cLngDeg[:], cLngMnt[:]) - if err != nil { - return nil, fmt.Errorf("could not parse longitude: %w", err) - } - - if !cLng.Hemisphere().East() { - lng = -lng - } - - return &Point{ - lat: lat, - lng: lng, - }, nil -} - -func toDecimal(intRaw, minutesRaw []byte) (float64, error) { - integer, err := strconv.ParseFloat(string(intRaw), 64) - if err != nil { - return 0, fmt.Errorf("could not parse integer part: %w", err) - } - - decimal, err := minutesToDegrees(minutesRaw) - if err != nil { - return 0, fmt.Errorf("could not parse decimal part: %w", err) - } - - return integer + decimal, nil -} - -// minutesToDegrees converts minutes to decimal part of a degree. -func minutesToDegrees(raw []byte) (float64, error) { - minutes, err := strconv.ParseFloat(string(raw), 64) - if err != nil { - return 0, err - } - - return minutes / 60, nil -} diff --git a/pkg/util/locode/db/point_test.go b/pkg/util/locode/db/point_test.go deleted file mode 100644 index f91c0cf87..000000000 --- a/pkg/util/locode/db/point_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package locodedb - -import ( - "testing" - - locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/column" - "github.com/stretchr/testify/require" -) - -func TestPointFromCoordinates(t *testing.T) { - testCases := []struct { - latGot, longGot string - latWant, longWant float64 - }{ - { - latGot: "5915N", - longGot: "01806E", - latWant: 59.25, - longWant: 18.10, - }, - { - latGot: "1000N", - longGot: "02030E", - latWant: 10.00, - longWant: 20.50, - }, - { - latGot: "0145S", - longGot: "03512W", - latWant: -01.75, - longWant: -35.20, - }, - } - - var ( - crd *locodecolumn.Coordinates - point *Point - err error - ) - - for _, test := range testCases { - crd, err = locodecolumn.CoordinatesFromString(test.latGot + " " + test.longGot) - require.NoError(t, err) - - point, err = PointFromCoordinates(crd) - require.NoError(t, err) - - require.Equal(t, test.latWant, point.Latitude()) - require.Equal(t, test.longWant, point.Longitude()) - } -} diff --git a/pkg/util/locode/db/record.go b/pkg/util/locode/db/record.go deleted file mode 100644 index 4c414079f..000000000 --- a/pkg/util/locode/db/record.go +++ /dev/null @@ -1,140 +0,0 @@ -package locodedb - -import ( - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" - locodecolumn "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/column" -) - -// Key represents the key in FrostFS location database. -type Key struct { - cc *CountryCode - - lc *LocationCode -} - -// NewKey calculates Key from LOCODE. -func NewKey(lc locode.LOCODE) (*Key, error) { - country, err := CountryCodeFromString(lc.CountryCode()) - if err != nil { - return nil, fmt.Errorf("could not parse country: %w", err) - } - - location, err := LocationCodeFromString(lc.LocationCode()) - if err != nil { - return nil, fmt.Errorf("could not parse location: %w", err) - } - - return &Key{ - cc: country, - lc: location, - }, nil -} - -// CountryCode returns the location's country code. -func (k *Key) CountryCode() *CountryCode { - return k.cc -} - -// LocationCode returns the location code. -func (k *Key) LocationCode() *LocationCode { - return k.lc -} - -// Record represents the entry in FrostFS location database. -type Record struct { - countryName string - - locationName string - - subDivName string - - subDivCode string - - p *Point - - cont *Continent -} - -var errParseCoordinates = errors.New("invalid coordinates") - -// NewRecord calculates the Record from the UN/LOCODE table record. -func NewRecord(r locode.Record) (*Record, error) { - crd, err := locodecolumn.CoordinatesFromString(r.Coordinates) - if err != nil { - return nil, fmt.Errorf("%w: %v", errParseCoordinates, err) - } - - point, err := PointFromCoordinates(crd) - if err != nil { - return nil, fmt.Errorf("could not parse geo point: %w", err) - } - - return &Record{ - locationName: r.NameWoDiacritics, - subDivCode: r.SubDiv, - p: point, - }, nil -} - -// CountryName returns the country name. -func (r *Record) CountryName() string { - return r.countryName -} - -// SetCountryName sets the country name. -func (r *Record) SetCountryName(name string) { - r.countryName = name -} - -// LocationName returns the location name. -func (r *Record) LocationName() string { - return r.locationName -} - -// SetLocationName sets the location name. -func (r *Record) SetLocationName(name string) { - r.locationName = name -} - -// SubDivCode returns the subdivision code. -func (r *Record) SubDivCode() string { - return r.subDivCode -} - -// SetSubDivCode sets the subdivision code. -func (r *Record) SetSubDivCode(name string) { - r.subDivCode = name -} - -// SubDivName returns the subdivision name. -func (r *Record) SubDivName() string { - return r.subDivName -} - -// SetSubDivName sets the subdivision name. -func (r *Record) SetSubDivName(name string) { - r.subDivName = name -} - -// GeoPoint returns geo point of the location. -func (r *Record) GeoPoint() *Point { - return r.p -} - -// SetGeoPoint sets geo point of the location. -func (r *Record) SetGeoPoint(p *Point) { - r.p = p -} - -// Continent returns the location continent. -func (r *Record) Continent() *Continent { - return r.cont -} - -// SetContinent sets the location continent. -func (r *Record) SetContinent(c *Continent) { - r.cont = c -} diff --git a/pkg/util/locode/record.go b/pkg/util/locode/record.go deleted file mode 100644 index 7db746ff3..000000000 --- a/pkg/util/locode/record.go +++ /dev/null @@ -1,83 +0,0 @@ -package locode - -import ( - "errors" - "fmt" - "strings" -) - -// LOCODE represents code from UN/LOCODE coding scheme. -type LOCODE [2]string - -// Record represents a single record of the UN/LOCODE table. -type Record struct { - // Change Indicator. - Ch string - - // Combination of a 2-character country code and a 3-character location code. - LOCODE LOCODE - - // Name of the locations which has been allocated a UN/LOCODE. - Name string - - // Names of the locations which have been allocated a UN/LOCODE without diacritic signs. - NameWoDiacritics string - - // ISO 1-3 character alphabetic and/or numeric code for the administrative division of the country concerned. - SubDiv string - - // 8-digit function classifier code for the location. - Function string - - // Status of the entry by a 2-character code. - Status string - - // Last date when the location was updated/entered. - Date string - - // The IATA code for the location if different from location code in column LOCODE. - IATA string - - // Geographical coordinates (latitude/longitude) of the location, if there is any. - Coordinates string - - // Some general remarks regarding the UN/LOCODE in question. - Remarks string -} - -// ErrInvalidString is the error of incorrect string format of the LOCODE. -var ErrInvalidString = errors.New("invalid string format in UN/Locode") - -// FromString parses string and returns LOCODE. -// -// If string has incorrect format, ErrInvalidString returns. -func FromString(s string) (*LOCODE, error) { - const ( - locationSeparator = " " - locodePartsNumber = 2 - ) - - words := strings.Split(s, locationSeparator) - if ln := len(words); ln != locodePartsNumber { - return nil, fmt.Errorf( - "incorrect locode: it must consist of %d codes separated with a witespase, got: %d", - locodePartsNumber, - ln, - ) - } - - l := new(LOCODE) - copy(l[:], words) - - return l, nil -} - -// CountryCode returns a string representation of country code. -func (l *LOCODE) CountryCode() string { - return l[0] -} - -// LocationCode returns a string representation of location code. -func (l *LOCODE) LocationCode() string { - return l[1] -} diff --git a/pkg/util/locode/table/csv/calls.go b/pkg/util/locode/table/csv/calls.go deleted file mode 100644 index 5f40865be..000000000 --- a/pkg/util/locode/table/csv/calls.go +++ /dev/null @@ -1,156 +0,0 @@ -package csvlocode - -import ( - "encoding/csv" - "errors" - "io" - "os" - "strings" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode" - locodedb "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/locode/db" -) - -var errInvalidRecord = errors.New("invalid table record") - -// IterateAll scans a table record one-by-one, parses a UN/LOCODE record -// from it and passes it to f. -// -// Returns f's errors directly. -func (t *Table) IterateAll(f func(locode.Record) error) error { - const wordsPerRecord = 12 - - return t.scanWords(t.paths, wordsPerRecord, func(words []string) error { - lc, err := locode.FromString(strings.Join(words[1:3], " ")) - if err != nil { - return err - } - - record := locode.Record{ - Ch: words[0], - LOCODE: *lc, - Name: words[3], - NameWoDiacritics: words[4], - SubDiv: words[5], - Function: words[6], - Status: words[7], - Date: words[8], - IATA: words[9], - Coordinates: words[10], - Remarks: words[11], - } - - return f(record) - }) -} - -const ( - _ = iota - 1 - - subDivCountry - subDivSubdivision - subDivName - _ // subDivLevel - - subDivFldNum -) - -type subDivKey struct { - countryCode, - subDivCode string -} - -type subDivRecord struct { - name string -} - -// SubDivName scans a table record to an in-memory table (once), -// and returns the subdivision name of the country and the subdivision codes match. -// -// Returns locodedb.ErrSubDivNotFound if no entry matches. -func (t *Table) SubDivName(countryCode *locodedb.CountryCode, code string) (string, error) { - if err := t.initSubDiv(); err != nil { - return "", err - } - - rec, ok := t.mSubDiv[subDivKey{ - countryCode: countryCode.String(), - subDivCode: code, - }] - if !ok { - return "", locodedb.ErrSubDivNotFound - } - - return rec.name, nil -} - -func (t *Table) initSubDiv() (err error) { - t.subDivOnce.Do(func() { - t.mSubDiv = make(map[subDivKey]subDivRecord) - - err = t.scanWords([]string{t.subDivPath}, subDivFldNum, func(words []string) error { - t.mSubDiv[subDivKey{ - countryCode: words[subDivCountry], - subDivCode: words[subDivSubdivision], - }] = subDivRecord{ - name: words[subDivName], - } - - return nil - }) - }) - - return -} - -var errScanInt = errors.New("interrupt scan") - -func (t *Table) scanWords(paths []string, fpr int, wordsHandler func([]string) error) error { - var ( - rdrs = make([]io.Reader, 0, len(t.paths)) - closers = make([]io.Closer, 0, len(t.paths)) - ) - - for i := range paths { - file, err := os.OpenFile(paths[i], os.O_RDONLY, t.mode) - if err != nil { - return err - } - - rdrs = append(rdrs, file) - closers = append(closers, file) - } - - defer func() { - for i := range closers { - _ = closers[i].Close() - } - }() - - r := csv.NewReader(io.MultiReader(rdrs...)) - r.ReuseRecord = true - r.FieldsPerRecord = fpr - - for { - words, err := r.Read() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - - return err - } else if len(words) != fpr { - return errInvalidRecord - } - - if err := wordsHandler(words); err != nil { - if errors.Is(err, errScanInt) { - break - } - - return err - } - } - - return nil -} diff --git a/pkg/util/locode/table/csv/opts.go b/pkg/util/locode/table/csv/opts.go deleted file mode 100644 index 68e442899..000000000 --- a/pkg/util/locode/table/csv/opts.go +++ /dev/null @@ -1,28 +0,0 @@ -package csvlocode - -import ( - "io/fs" -) - -// Option sets an optional parameter of Table. -type Option func(*options) - -type options struct { - mode fs.FileMode - - extraPaths []string -} - -func defaultOpts() *options { - return &options{ - mode: 0o700, - } -} - -// WithExtraPaths returns an option to add extra paths -// to UN/LOCODE tables in csv format. -func WithExtraPaths(ps ...string) Option { - return func(o *options) { - o.extraPaths = append(o.extraPaths, ps...) - } -} diff --git a/pkg/util/locode/table/csv/table.go b/pkg/util/locode/table/csv/table.go deleted file mode 100644 index b84c2b705..000000000 --- a/pkg/util/locode/table/csv/table.go +++ /dev/null @@ -1,75 +0,0 @@ -package csvlocode - -import ( - "fmt" - "io/fs" - "sync" -) - -// Prm groups the required parameters of the Table's constructor. -// -// All values must comply with the requirements imposed on them. -// Passing incorrect parameter values will result in constructor -// failure (error or panic depending on the implementation). -type Prm struct { - // Path to UN/LOCODE csv table. - // - // Must not be empty. - Path string - - // Path to csv table of UN/LOCODE Subdivisions. - // - // Must not be empty. - SubDivPath string -} - -// Table is a descriptor of the UN/LOCODE table in csv format. -// -// For correct operation, Table must be created -// using the constructor (New) based on the required parameters -// and optional components. After successful creation, -// The Table is immediately ready to work through API. -type Table struct { - paths []string - - mode fs.FileMode - - subDivPath string - - subDivOnce sync.Once - - mSubDiv map[subDivKey]subDivRecord -} - -const invalidPrmValFmt = "invalid parameter %s (%T):%v" - -func panicOnPrmValue(n string, v any) { - panic(fmt.Sprintf(invalidPrmValFmt, n, v, v)) -} - -// New creates a new instance of the Table. -// -// Panics if at least one value of the parameters is invalid. -// -// The created Table does not require additional -// initialization and is completely ready for work. -func New(prm Prm, opts ...Option) *Table { - switch { - case prm.Path == "": - panicOnPrmValue("Path", prm.Path) - case prm.SubDivPath == "": - panicOnPrmValue("SubDivPath", prm.SubDivPath) - } - - o := defaultOpts() - - for i := range opts { - opts[i](o) - } - - return &Table{ - paths: append(o.extraPaths, prm.Path), - mode: o.mode, - subDivPath: prm.SubDivPath, - } -} From 8398a8b4b3d4eff5f3759a5d5dafaa4ba9046a4c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 23 Jul 2024 17:46:02 +0300 Subject: [PATCH 1248/1943] [#1271] getSvc: Fix `head --raw` assemble for EC Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assembleec.go | 4 +- pkg/services/object/get/assemblerec.go | 64 -------------------------- pkg/services/object/get/get.go | 4 ++ pkg/services/object/get/remote.go | 18 ++++++-- 4 files changed, 20 insertions(+), 70 deletions(-) diff --git a/pkg/services/object/get/assembleec.go b/pkg/services/object/get/assembleec.go index 7bbd9ca1e..a58602bf7 100644 --- a/pkg/services/object/get/assembleec.go +++ b/pkg/services/object/get/assembleec.go @@ -11,7 +11,7 @@ import ( ) func (r *request) assembleEC(ctx context.Context) { - if r.isRaw() && r.isLocal() { + if r.isRaw() { r.log.Debug(logs.GetCanNotAssembleTheObject) return } @@ -43,7 +43,7 @@ func (r *request) assembleEC(ctx context.Context) { } r.prm.common = r.prm.common.WithLocalOnly(false) - assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.localStorage, r.containerSource, r.log, r.headOnly(), r.isRaw(), r.traverserGenerator, r.curProcEpoch) + assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.localStorage, r.containerSource, r.log, r.headOnly(), r.traverserGenerator, r.curProcEpoch) r.log.Debug(logs.GetAssemblingECObject, zap.Uint64("range_offset", r.ctxRange().GetOffset()), diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index a4021ee5e..d64984d5c 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -37,7 +37,6 @@ type assemblerec struct { cs container.Source log *logger.Logger head bool - raw bool traverserGenerator traverserGenerator epoch uint64 } @@ -51,7 +50,6 @@ func newAssemblerEC( cs container.Source, log *logger.Logger, head bool, - raw bool, tg traverserGenerator, epoch uint64, ) *assemblerec { @@ -64,7 +62,6 @@ func newAssemblerEC( cs: cs, log: log, head: head, - raw: raw, traverserGenerator: tg, epoch: epoch, } @@ -74,9 +71,6 @@ func newAssemblerEC( // It returns parent object. func (a *assemblerec) Assemble(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { switch { - case a.raw: - err := a.reconstructRawError(ctx) - return nil, err case a.head: return a.reconstructHeader(ctx, writer) case a.rng != nil: @@ -149,56 +143,6 @@ func (a *assemblerec) reconstructObjectFromParts(ctx context.Context, headers bo return c.Reconstruct(parts) } -func (a *assemblerec) reconstructRawError(ctx context.Context) error { - chunks := make(map[string]objectSDK.ECChunk) - var chunksGuard sync.Mutex - for _, ch := range a.ecInfo.localChunks { - chunks[string(ch.ID.GetValue())] = ch - } - - objID := a.addr.Object() - trav, _, err := a.traverserGenerator.GenerateTraverser(a.addr.Container(), &objID, a.epoch) - if err != nil { - return err - } - - eg, ctx := errgroup.WithContext(ctx) - for { - batch := trav.Next() - if len(batch) == 0 { - break - } - for _, node := range batch { - var info client.NodeInfo - client.NodeInfoFromNetmapElement(&info, node) - eg.Go(func() error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - if _, found := a.ecInfo.remoteChunks[string(info.PublicKey())]; found { - return nil - } - - nodeChunks := a.tryGetChunkListFromNode(ctx, info) - - chunksGuard.Lock() - defer chunksGuard.Unlock() - for _, ch := range nodeChunks { - chunks[string(ch.ID.GetValue())] = ch - } - return nil - }) - } - } - if err = eg.Wait(); err != nil { - return err - } - return createECInfoErr(chunks) -} - func (a *assemblerec) retrieveParts(ctx context.Context, trav *placement.Traverser, cnr *container.Container) []*objectSDK.Object { dataCount := policy.ECDataCount(cnr.Value.PlacementPolicy()) parityCount := policy.ECParityCount(cnr.Value.PlacementPolicy()) @@ -359,11 +303,3 @@ func (a *assemblerec) tryGetChunkFromRemoteStorage(ctx context.Context, node cli } return object } - -func createECInfoErr(chunks map[string]objectSDK.ECChunk) *objectSDK.ECInfoError { - info := objectSDK.NewECInfo() - for _, ch := range chunks { - info.AddChunk(ch) - } - return objectSDK.NewECInfoError(info) -} diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 5a57bc56e..07a2f3a72 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -111,6 +111,10 @@ func (exec *request) analyzeStatus(ctx context.Context, execCnr bool) { exec.log.Debug(logs.GetRequestedRangeIsOutOfObjectBounds) case statusEC: exec.log.Debug(logs.GetRequestedObjectIsEC) + if exec.isRaw() && execCnr { + exec.executeOnContainer(ctx) + exec.analyzeStatus(ctx, false) + } exec.assembleEC(ctx) default: exec.log.Debug(logs.OperationFinishedWithError, diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index 4dee15242..ce9abfe1c 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -35,8 +35,12 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { switch { default: r.log.Debug(logs.GetRemoteCallFailed, zap.Error(err)) - r.status = statusUndefined - r.err = new(apistatus.ObjectNotFound) + if r.status != statusEC { + // for raw requests, continue to collect other parts + r.status = statusUndefined + r.err = new(apistatus.ObjectNotFound) + } + return false case err == nil: r.status = statusOK r.err = nil @@ -48,22 +52,28 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { r.collectedObject = obj r.writeCollectedObject(ctx) } + return true case errors.As(err, &errRemoved): r.status = statusINHUMED r.err = errRemoved + return true case errors.As(err, &errOutOfRange): r.status = statusOutOfRange r.err = errOutOfRange + return true case errors.As(err, &errSplitInfo): r.status = statusVIRTUAL mergeSplitInfo(r.splitInfo(), errSplitInfo.SplitInfo()) r.err = objectSDK.NewSplitInfoError(r.infoSplit) + return true case errors.As(err, &errECInfo): r.status = statusEC r.err = r.infoEC.addRemote(string(info.PublicKey()), errECInfo.ECInfo()) + if r.isRaw() { + return false // continue to collect all parts + } + return true } - - return r.status != statusUndefined } func (r *request) getRemote(ctx context.Context, rs remoteStorage, info client.NodeInfo) (*objectSDK.Object, error) { From 7fd7961dfa9ffef87f17382da758c4d74b54cd4f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 23 Jul 2024 22:15:13 +0300 Subject: [PATCH 1249/1943] [#1271] getSvc: Fix local EC chunk get Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assemblerec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index d64984d5c..6a02673c3 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -237,7 +237,7 @@ func (a *assemblerec) tryGetChunkFromLocalStorage(ctx context.Context, ch object return nil } var addr oid.Address - addr.SetContainer(addr.Container()) + addr.SetContainer(a.addr.Container()) addr.SetObject(objID) var object *objectSDK.Object if a.head { From dd459d399f11d8e49fa841f1372563453bc41a77 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 25 Jul 2024 22:52:48 +0300 Subject: [PATCH 1250/1943] [#1274] go.mod: Update neo-go version that fixes ws-client * Update go.mod; * This neo-go package version contains fix for the wsclient that allows to morph event listener refresh the invalidated websocket connection to neo-go. Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6a97f7850..c63eabe0d 100644 --- a/go.mod +++ b/go.mod @@ -127,4 +127,4 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/nspcc-dev/neo-go => git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 +replace github.com/nspcc-dev/neo-go => git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 diff --git a/go.sum b/go.sum index d4cf863fa..965bffd2d 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= -git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18 h1:JRjwcHaQajTbSCBCK3yZnqvyHvgWBaoThDGuT4kvIIc= -git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240611123832-594f716b3d18/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= +git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= +git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984 h1:O3F2Grz07RWZ68mRz1xsYsNPNvZLwY00BM+xoYb1kNk= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From 8377372a40a699a029f185be6e355748931e8234 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 26 Jul 2024 16:37:05 +0300 Subject: [PATCH 1251/1943] [#1276] go.mod: Update api-go Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/client/client.go | 23 ---- cmd/frostfs-cli/modules/container/root.go | 3 - cmd/frostfs-cli/modules/container/set_eacl.go | 108 ------------------ cmd/frostfs-cli/modules/container/util.go | 5 +- go.mod | 6 +- go.sum | 8 +- pkg/core/client/client.go | 1 - pkg/morph/client/container/load.go | 40 ------- pkg/network/cache/multi.go | 9 -- .../transport/container/grpc/service.go | 30 ----- pkg/services/container/ape.go | 21 ---- pkg/services/container/ape_test.go | 90 --------------- pkg/services/container/audit.go | 30 ----- pkg/services/container/executor.go | 19 --- pkg/services/container/morph/executor.go | 6 - pkg/services/container/server.go | 2 - pkg/services/container/sign.go | 18 --- 17 files changed, 9 insertions(+), 410 deletions(-) delete mode 100644 cmd/frostfs-cli/modules/container/set_eacl.go diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index a6d9968c5..215490dbe 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -214,29 +214,6 @@ func EACL(ctx context.Context, prm EACLPrm) (res EACLRes, err error) { return } -// SetEACLPrm groups parameters of SetEACL operation. -type SetEACLPrm struct { - Client *client.Client - ClientParams client.PrmContainerSetEACL -} - -// SetEACLRes groups the resulting values of SetEACL operation. -type SetEACLRes struct{} - -// SetEACL requests to save an eACL table in FrostFS. -// -// Operation is asynchronous and no guaranteed even in the absence of errors. -// The required time is also not predictable. -// -// Success can be verified by reading by container identifier. -// -// Returns any error which prevented the operation from completing correctly in error return. -func SetEACL(ctx context.Context, prm SetEACLPrm) (res SetEACLRes, err error) { - _, err = prm.Client.ContainerSetEACL(ctx, prm.ClientParams) - - return -} - // NetworkInfoPrm groups parameters of NetworkInfo operation. type NetworkInfoPrm struct { Client *client.Client diff --git a/cmd/frostfs-cli/modules/container/root.go b/cmd/frostfs-cli/modules/container/root.go index 99d1a4231..d5f0fd776 100644 --- a/cmd/frostfs-cli/modules/container/root.go +++ b/cmd/frostfs-cli/modules/container/root.go @@ -26,7 +26,6 @@ func init() { listContainerObjectsCmd, getContainerInfoCmd, getExtendedACLCmd, - setExtendedACLCmd, containerNodesCmd, policyPlaygroundCmd, } @@ -39,7 +38,6 @@ func init() { initContainerListObjectsCmd() initContainerInfoCmd() initContainerGetEACLCmd() - initContainerSetEACLCmd() initContainerNodesCmd() initContainerPolicyPlaygroundCmd() @@ -53,7 +51,6 @@ func init() { }{ {createContainerCmd, "PUT"}, {deleteContainerCmd, "DELETE"}, - {setExtendedACLCmd, "SETEACL"}, } { commonflags.InitSession(el.cmd, "container "+el.verb) } diff --git a/cmd/frostfs-cli/modules/container/set_eacl.go b/cmd/frostfs-cli/modules/container/set_eacl.go deleted file mode 100644 index 86aa50a57..000000000 --- a/cmd/frostfs-cli/modules/container/set_eacl.go +++ /dev/null @@ -1,108 +0,0 @@ -package container - -import ( - "bytes" - "errors" - "time" - - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - "github.com/spf13/cobra" -) - -var flagVarsSetEACL struct { - noPreCheck bool - - srcPath string -} - -var setExtendedACLCmd = &cobra.Command{ - Use: "set-eacl", - Short: "Set new extended ACL table for container", - Long: `Set new extended ACL table for container. -Container ID in EACL table will be substituted with ID from the CLI.`, - Run: func(cmd *cobra.Command, _ []string) { - id := parseContainerID(cmd) - eaclTable := common.ReadEACL(cmd, flagVarsSetEACL.srcPath) - - tok := getSession(cmd) - - eaclTable.SetCID(id) - - pk := key.GetOrGenerate(cmd) - cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - - if !flagVarsSetEACL.noPreCheck { - cmd.Println("Checking the ability to modify access rights in the container...") - - extendable, err := internalclient.IsACLExtendable(cmd.Context(), cli, id) - commonCmd.ExitOnErr(cmd, "Extensibility check failure: %w", err) - - if !extendable { - commonCmd.ExitOnErr(cmd, "", errors.New("container ACL is immutable")) - } - - cmd.Println("ACL extension is enabled in the container, continue processing.") - } - - setEACLPrm := internalclient.SetEACLPrm{ - Client: cli, - ClientParams: client.PrmContainerSetEACL{ - Table: eaclTable, - Session: tok, - }, - } - - _, err := internalclient.SetEACL(cmd.Context(), setEACLPrm) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - if containerAwait { - exp, err := eaclTable.Marshal() - commonCmd.ExitOnErr(cmd, "broken EACL table: %w", err) - - cmd.Println("awaiting...") - - getEACLPrm := internalclient.EACLPrm{ - Client: cli, - ClientParams: client.PrmContainerEACL{ - ContainerID: &id, - }, - } - - for i := 0; i < awaitTimeout; i++ { - time.Sleep(1 * time.Second) - - res, err := internalclient.EACL(cmd.Context(), getEACLPrm) - if err == nil { - // compare binary values because EACL could have been set already - table := res.EACL() - got, err := table.Marshal() - if err != nil { - continue - } - - if bytes.Equal(exp, got) { - cmd.Println("EACL has been persisted on sidechain") - return - } - } - } - - commonCmd.ExitOnErr(cmd, "", errSetEACLTimeout) - } - }, -} - -func initContainerSetEACLCmd() { - commonflags.Init(setExtendedACLCmd) - - flags := setExtendedACLCmd.Flags() - flags.StringVar(&containerID, commonflags.CIDFlag, "", commonflags.CIDFlagUsage) - flags.StringVar(&flagVarsSetEACL.srcPath, "table", "", "path to file with JSON or binary encoded EACL table") - flags.BoolVar(&containerAwait, "await", false, "block execution until EACL is persisted") - flags.BoolVar(&flagVarsSetEACL.noPreCheck, "no-precheck", false, "do not pre-check the extensibility of the container ACL") -} diff --git a/cmd/frostfs-cli/modules/container/util.go b/cmd/frostfs-cli/modules/container/util.go index 48265f785..4cb268ec5 100644 --- a/cmd/frostfs-cli/modules/container/util.go +++ b/cmd/frostfs-cli/modules/container/util.go @@ -18,9 +18,8 @@ const ( ) var ( - errCreateTimeout = errors.New("timeout: container has not been persisted on sidechain") - errDeleteTimeout = errors.New("timeout: container has not been removed from sidechain") - errSetEACLTimeout = errors.New("timeout: EACL has not been persisted on sidechain") + errCreateTimeout = errors.New("timeout: container has not been persisted on sidechain") + errDeleteTimeout = errors.New("timeout: container has not been removed from sidechain") ) func parseContainerID(cmd *cobra.Command) cid.ID { diff --git a/go.mod b/go.mod index c63eabe0d..09a098502 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240726111349-9da46f566fec git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 @@ -24,7 +24,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.12.1 - github.com/nspcc-dev/neo-go v0.106.0 + github.com/nspcc-dev/neo-go v0.106.2 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 github.com/prometheus/client_golang v1.19.0 diff --git a/go.sum b/go.sum index 965bffd2d..1034ff61f 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3 h1:H5GvrVlowIMWfzqQkhY0p0myooJxQ1sMRVSFfXawwWg= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240530152826-2f6d3209e1d3/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e h1:gEWT+70E/RvGkxtSv+PlyUN2vtJVymhQa1mypvrXukM= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de h1:OjsWY0jpGJV1t87XgwL/3PsDx7fJ6lfNMXtY8UhoUbM= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240617140730-1a5886e776de/go.mod h1:4AObM67VUqkXQJlODTFThFnuMGEuK8h9DrAXHDZqvCU= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240726111349-9da46f566fec h1:A09Swh7yogmmiABUf7Ht6MTQXJ07MyGx4+ziUQNelec= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240726111349-9da46f566fec/go.mod h1:DlJmgV4/qkFkx2ab+YWznlMijiF2yZHnrJswJOB7XGs= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= diff --git a/pkg/core/client/client.go b/pkg/core/client/client.go index 8c92901f2..854fbc49f 100644 --- a/pkg/core/client/client.go +++ b/pkg/core/client/client.go @@ -11,7 +11,6 @@ import ( // Client is an interface of FrostFS storage // node's client. type Client interface { - ContainerAnnounceUsedSpace(context.Context, client.PrmAnnounceSpace) (*client.ResAnnounceSpace, error) ObjectPutInit(context.Context, client.PrmObjectPutInit) (client.ObjectWriter, error) ObjectPutSingle(context.Context, client.PrmObjectPutSingle) (*client.ResObjectPutSingle, error) ObjectDelete(context.Context, client.PrmObjectDelete) (*client.ResObjectDelete, error) diff --git a/pkg/morph/client/container/load.go b/pkg/morph/client/container/load.go index b5263d7a6..5e2c3c2c3 100644 --- a/pkg/morph/client/container/load.go +++ b/pkg/morph/client/container/load.go @@ -1,53 +1,13 @@ package container import ( - "crypto/sha256" "fmt" v2refs "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) -// AnnounceLoadPrm groups parameters of AnnounceLoad operation. -type AnnounceLoadPrm struct { - a container.SizeEstimation - key []byte - - client.InvokePrmOptional -} - -// SetAnnouncement sets announcement. -func (a2 *AnnounceLoadPrm) SetAnnouncement(a container.SizeEstimation) { - a2.a = a -} - -// SetReporter sets public key of the reporter. -func (a2 *AnnounceLoadPrm) SetReporter(key []byte) { - a2.key = key -} - -// AnnounceLoad saves container size estimation calculated by storage node -// with key in FrostFS system through Container contract call. -// -// Returns any error encountered that caused the saving to interrupt. -func (c *Client) AnnounceLoad(p AnnounceLoadPrm) error { - binCnr := make([]byte, sha256.Size) - p.a.Container().Encode(binCnr) - - prm := client.InvokePrm{} - prm.SetMethod(putSizeMethod) - prm.SetArgs(p.a.Epoch(), binCnr, p.a.Value(), p.key) - prm.InvokePrmOptional = p.InvokePrmOptional - - _, err := c.client.Invoke(prm) - if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", putSizeMethod, err) - } - return nil -} - // EstimationID is an identity of container load estimation inside Container contract. type EstimationID []byte diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index f19510d76..9305c143b 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -239,15 +239,6 @@ func (x *multiClient) ObjectPutSingle(ctx context.Context, p client.PrmObjectPut return } -func (x *multiClient) ContainerAnnounceUsedSpace(ctx context.Context, prm client.PrmAnnounceSpace) (res *client.ResAnnounceSpace, err error) { - err = x.iterateClients(ctx, func(c clientcore.Client) error { - res, err = c.ContainerAnnounceUsedSpace(ctx, prm) - return err - }) - - return -} - func (x *multiClient) ObjectDelete(ctx context.Context, p client.PrmObjectDelete) (res *client.ResObjectDelete, err error) { err = x.iterateClients(ctx, func(c clientcore.Client) error { res, err = c.ObjectDelete(ctx, p) diff --git a/pkg/network/transport/container/grpc/service.go b/pkg/network/transport/container/grpc/service.go index ed514d6d4..f0206dd5c 100644 --- a/pkg/network/transport/container/grpc/service.go +++ b/pkg/network/transport/container/grpc/service.go @@ -81,21 +81,6 @@ func (s *Server) List(ctx context.Context, req *containerGRPC.ListRequest) (*con return resp.ToGRPCMessage().(*containerGRPC.ListResponse), nil } -// SetExtendedACL converts gRPC SetExtendedACLRequest message and passes it to internal Container service. -func (s *Server) SetExtendedACL(ctx context.Context, req *containerGRPC.SetExtendedACLRequest) (*containerGRPC.SetExtendedACLResponse, error) { - setEACLReq := new(container.SetExtendedACLRequest) - if err := setEACLReq.FromGRPCMessage(req); err != nil { - return nil, err - } - - resp, err := s.srv.SetExtendedACL(ctx, setEACLReq) - if err != nil { - return nil, err - } - - return resp.ToGRPCMessage().(*containerGRPC.SetExtendedACLResponse), nil -} - // GetExtendedACL converts gRPC GetExtendedACLRequest message and passes it to internal Container service. func (s *Server) GetExtendedACL(ctx context.Context, req *containerGRPC.GetExtendedACLRequest) (*containerGRPC.GetExtendedACLResponse, error) { getEACLReq := new(container.GetExtendedACLRequest) @@ -110,18 +95,3 @@ func (s *Server) GetExtendedACL(ctx context.Context, req *containerGRPC.GetExten return resp.ToGRPCMessage().(*containerGRPC.GetExtendedACLResponse), nil } - -// AnnounceUsedSpace converts gRPC AnnounceUsedSpaceRequest message and passes it to internal Container service. -func (s *Server) AnnounceUsedSpace(ctx context.Context, req *containerGRPC.AnnounceUsedSpaceRequest) (*containerGRPC.AnnounceUsedSpaceResponse, error) { - announceReq := new(container.AnnounceUsedSpaceRequest) - if err := announceReq.FromGRPCMessage(req); err != nil { - return nil, err - } - - resp, err := s.srv.AnnounceUsedSpace(ctx, announceReq) - if err != nil { - return nil, err - } - - return resp.ToGRPCMessage().(*containerGRPC.AnnounceUsedSpaceResponse), nil -} diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 3ea591c6a..8fe4dd2d9 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -78,15 +78,6 @@ func NewAPEServer(router policyengine.ChainRouter, reader containers, ir ir, nm } } -func (ac *apeChecker) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.AnnounceUsedSpace") - defer span.End() - - // this method is not used, so not checked - - return ac.next.AnnounceUsedSpace(ctx, req) -} - func (ac *apeChecker) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.Delete") defer span.End() @@ -303,18 +294,6 @@ func (ac *apeChecker) getRoleWithoutContainerID(oID *refs.OwnerID, mh *session.R return nativeschema.PropertyValueContainerRoleOthers, pk, nil } -func (ac *apeChecker) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.SetExtendedACL") - defer span.End() - - if err := ac.validateContainerBoundedOperation(ctx, req.GetBody().GetEACL().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), - nativeschema.MethodSetContainerEACL); err != nil { - return nil, err - } - - return ac.next.SetExtendedACL(ctx, req) -} - func (ac *apeChecker) validateContainerBoundedOperation(ctx context.Context, containerID *refs.ContainerID, mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader, op string) error { if vh == nil { return errMissingVerificationHeader diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index a6f0fb222..9eed469ca 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -9,7 +9,6 @@ import ( "net" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" @@ -50,7 +49,6 @@ func TestAPE(t *testing.T) { t.Run("deny get container by user claim tag", testDenyGetContainerByUserClaimTag) t.Run("deny get container by IP", testDenyGetContainerByIP) t.Run("deny get container by group id", testDenyGetContainerByGroupID) - t.Run("deny set container eACL for IR", testDenySetContainerEACLForIR) t.Run("deny get container eACL for IR with session token", testDenyGetContainerEACLForIRSessionToken) t.Run("deny put container for others with session token", testDenyPutContainerForOthersSessionToken) t.Run("deny put container, read namespace from frostfsID", testDenyPutContainerReadNamespaceFromFrostfsID) @@ -665,84 +663,6 @@ func testDenyGetContainerByGroupID(t *testing.T) { require.ErrorAs(t, err, &errAccessDenied) } -func testDenySetContainerEACLForIR(t *testing.T) { - t.Parallel() - srv := &srvStub{ - calls: map[string]int{}, - } - router := inmemory.NewInMemory() - contRdr := &containerStub{ - c: map[cid.ID]*containercore.Container{}, - } - ir := &irStub{ - keys: [][]byte{}, - } - nm := &netmapStub{} - frostfsIDSubjectReader := &frostfsidStub{ - subjects: map[util.Uint160]*client.Subject{}, - } - apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) - - contID := cidtest.ID() - testContainer := containertest.Container() - pp := netmap.PlacementPolicy{} - require.NoError(t, pp.DecodeString("REP 1")) - testContainer.SetPlacementPolicy(pp) - contRdr.c[contID] = &containercore.Container{Value: testContainer} - - nm.currentEpoch = 100 - nm.netmaps = map[uint64]*netmap.NetMap{} - var testNetmap netmap.NetMap - testNetmap.SetEpoch(nm.currentEpoch) - testNetmap.SetNodes([]netmap.NodeInfo{{}}) - nm.netmaps[nm.currentEpoch] = &testNetmap - nm.netmaps[nm.currentEpoch-1] = &testNetmap - - _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ - Rules: []chain.Rule{ - { - Status: chain.AccessDenied, - Actions: chain.Actions{ - Names: []string{ - nativeschema.MethodSetContainerEACL, - }, - }, - Resources: chain.Resources{ - Names: []string{ - fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), - }, - }, - Condition: []chain.Condition{ - { - Kind: chain.KindRequest, - Key: nativeschema.PropertyKeyActorRole, - Value: nativeschema.PropertyValueContainerRoleIR, - Op: chain.CondStringEquals, - }, - }, - }, - }, - }) - require.NoError(t, err) - - req := &container.SetExtendedACLRequest{} - req.SetBody(&container.SetExtendedACLRequestBody{}) - var refContID refs.ContainerID - contID.WriteToV2(&refContID) - req.GetBody().SetEACL(&acl.Table{}) - req.GetBody().GetEACL().SetContainerID(&refContID) - - pk, err := keys.NewPrivateKey() - require.NoError(t, err) - require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) - ir.keys = append(ir.keys, pk.PublicKey().Bytes()) - - resp, err := apeSrv.SetExtendedACL(context.Background(), req) - require.Nil(t, resp) - var errAccessDenied *apistatus.ObjectAccessDenied - require.ErrorAs(t, err, &errAccessDenied) -} - func testDenyGetContainerEACLForIRSessionToken(t *testing.T) { t.Parallel() srv := &srvStub{ @@ -1229,11 +1149,6 @@ type srvStub struct { calls map[string]int } -func (s *srvStub) AnnounceUsedSpace(context.Context, *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - s.calls["AnnounceUsedSpace"]++ - return &container.AnnounceUsedSpaceResponse{}, nil -} - func (s *srvStub) Delete(context.Context, *container.DeleteRequest) (*container.DeleteResponse, error) { s.calls["Delete"]++ return &container.DeleteResponse{}, nil @@ -1259,11 +1174,6 @@ func (s *srvStub) Put(context.Context, *container.PutRequest) (*container.PutRes return &container.PutResponse{}, nil } -func (s *srvStub) SetExtendedACL(context.Context, *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - s.calls["SetExtendedACL"]++ - return &container.SetExtendedACLResponse{}, nil -} - type irStub struct { keys [][]byte } diff --git a/pkg/services/container/audit.go b/pkg/services/container/audit.go index 7ef432bb1..34fd5923f 100644 --- a/pkg/services/container/audit.go +++ b/pkg/services/container/audit.go @@ -6,7 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" container_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -29,24 +28,6 @@ func NewAuditService(next Server, log *logger.Logger, enabled *atomic.Bool) Serv } } -// AnnounceUsedSpace implements Server. -func (a *auditService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - res, err := a.next.AnnounceUsedSpace(ctx, req) - if !a.enabled.Load() { - return res, err - } - - var ids []*refs.ContainerID - for _, v := range req.GetBody().GetAnnouncements() { - ids = append(ids, v.GetContainerID()) - } - - audit.LogRequest(a.log, container_grpc.ContainerService_AnnounceUsedSpace_FullMethodName, req, - audit.TargetFromRefs(ids, &cid.ID{}), err == nil) - - return res, err -} - // Delete implements Server. func (a *auditService) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { res, err := a.next.Delete(ctx, req) @@ -103,14 +84,3 @@ func (a *auditService) Put(ctx context.Context, req *container.PutRequest) (*con audit.TargetFromRef(res.GetBody().GetContainerID(), &cid.ID{}), err == nil) return res, err } - -// SetExtendedACL implements Server. -func (a *auditService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - res, err := a.next.SetExtendedACL(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequest(a.log, container_grpc.ContainerService_SetExtendedACL_FullMethodName, req, - audit.TargetFromRef(req.GetBody().GetEACL().GetContainerID(), &cid.ID{}), err == nil) - return res, err -} diff --git a/pkg/services/container/executor.go b/pkg/services/container/executor.go index d4ae11d62..b64963e25 100644 --- a/pkg/services/container/executor.go +++ b/pkg/services/container/executor.go @@ -14,7 +14,6 @@ type ServiceExecutor interface { Delete(context.Context, *session.Token, *container.DeleteRequestBody) (*container.DeleteResponseBody, error) Get(context.Context, *container.GetRequestBody) (*container.GetResponseBody, error) List(context.Context, *container.ListRequestBody) (*container.ListResponseBody, error) - SetExtendedACL(context.Context, *session.Token, *container.SetExtendedACLRequestBody) (*container.SetExtendedACLResponseBody, error) GetExtendedACL(context.Context, *container.GetExtendedACLRequestBody) (*container.GetExtendedACLResponseBody, error) } @@ -96,24 +95,6 @@ func (s *executorSvc) List(ctx context.Context, req *container.ListRequest) (*co return resp, nil } -func (s *executorSvc) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - meta := req.GetMetaHeader() - for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { - meta = origin - } - - respBody, err := s.exec.SetExtendedACL(ctx, meta.GetSessionToken(), req.GetBody()) - if err != nil { - return nil, fmt.Errorf("could not execute SetEACL request: %w", err) - } - - resp := new(container.SetExtendedACLResponse) - resp.SetBody(respBody) - - s.respSvc.SetMeta(resp) - return resp, nil -} - func (s *executorSvc) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { respBody, err := s.exec.GetExtendedACL(ctx, req.GetBody()) if err != nil { diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index e2e79f3d2..57dac32f0 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -13,8 +13,6 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) var errMissingUserID = errors.New("missing user ID") @@ -204,10 +202,6 @@ func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) return res, nil } -func (s *morphExecutor) SetExtendedACL(_ context.Context, _ *sessionV2.Token, _ *container.SetExtendedACLRequestBody) (*container.SetExtendedACLResponseBody, error) { - return nil, status.Errorf(codes.Unimplemented, "method SetExtendedACL not implemented") -} - func (s *morphExecutor) GetExtendedACL(_ context.Context, body *container.GetExtendedACLRequestBody) (*container.GetExtendedACLResponseBody, error) { idV2 := body.GetContainerID() if idV2 == nil { diff --git a/pkg/services/container/server.go b/pkg/services/container/server.go index 052a8c945..d714d7f02 100644 --- a/pkg/services/container/server.go +++ b/pkg/services/container/server.go @@ -12,7 +12,5 @@ type Server interface { Get(context.Context, *container.GetRequest) (*container.GetResponse, error) Delete(context.Context, *container.DeleteRequest) (*container.DeleteResponse, error) List(context.Context, *container.ListRequest) (*container.ListResponse, error) - SetExtendedACL(context.Context, *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) GetExtendedACL(context.Context, *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) - AnnounceUsedSpace(context.Context, *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) } diff --git a/pkg/services/container/sign.go b/pkg/services/container/sign.go index bba717f60..62aa3fe27 100644 --- a/pkg/services/container/sign.go +++ b/pkg/services/container/sign.go @@ -57,15 +57,6 @@ func (s *signService) List(ctx context.Context, req *container.ListRequest) (*co return resp, s.sigSvc.SignResponse(resp, err) } -func (s *signService) SetExtendedACL(ctx context.Context, req *container.SetExtendedACLRequest) (*container.SetExtendedACLResponse, error) { - if err := s.sigSvc.VerifyRequest(req); err != nil { - resp := new(container.SetExtendedACLResponse) - return resp, s.sigSvc.SignResponse(resp, err) - } - resp, err := util.EnsureNonNilResponse(s.svc.SetExtendedACL(ctx, req)) - return resp, s.sigSvc.SignResponse(resp, err) -} - func (s *signService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(container.GetExtendedACLResponse) @@ -74,12 +65,3 @@ func (s *signService) GetExtendedACL(ctx context.Context, req *container.GetExte resp, err := util.EnsureNonNilResponse(s.svc.GetExtendedACL(ctx, req)) return resp, s.sigSvc.SignResponse(resp, err) } - -func (s *signService) AnnounceUsedSpace(ctx context.Context, req *container.AnnounceUsedSpaceRequest) (*container.AnnounceUsedSpaceResponse, error) { - if err := s.sigSvc.VerifyRequest(req); err != nil { - resp := new(container.AnnounceUsedSpaceResponse) - return resp, s.sigSvc.SignResponse(resp, err) - } - resp, err := util.EnsureNonNilResponse(s.svc.AnnounceUsedSpace(ctx, req)) - return resp, s.sigSvc.SignResponse(resp, err) -} From 85a77b7c21accc667f4f5cb50ea1a0bf8632cedb Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 29 Jul 2024 15:33:36 +0300 Subject: [PATCH 1252/1943] [#1279] adm: Interpret "root" name as empty for namespace target type Signed-off-by: Airat Arifullin --- cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 5e17f4014..d4aedda2e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -38,6 +38,12 @@ var ( func parseTarget(cmd *cobra.Command) policyengine.Target { name, _ := cmd.Flags().GetString(targetNameFlag) typ, err := parseTargetType(cmd) + + // interpret "root" namespace as empty + if typ == policyengine.Namespace && name == "root" { + name = "" + } + commonCmd.ExitOnErr(cmd, "read target type error: %w", err) return policyengine.Target{ From a12c39667d2b95f65697205b90f5d2e61d09132e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 29 Jul 2024 11:31:05 +0300 Subject: [PATCH 1253/1943] [#1278] ir: Do not allow to create container without FrostFSID record Signed-off-by: Dmitrii Stepanov --- pkg/innerring/processors/container/handlers_test.go | 3 +-- .../processors/container/process_container.go | 10 +++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index 1aac31ae3..dc1e919bb 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -3,7 +3,6 @@ package container import ( "crypto/ecdsa" "encoding/hex" - "fmt" "testing" "time" @@ -238,5 +237,5 @@ func (c *testMorphClient) NotarySignAndInvokeTX(mainTx *transaction.Transaction) type testFrostFSIDClient struct{} func (c *testFrostFSIDClient) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) { - return nil, fmt.Errorf("subject not found") + return &frostfsidclient.Subject{}, nil } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index a950997fd..d89b63e82 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -180,11 +180,6 @@ func (cp *Processor) checkNNS(ctx *putContainerContext, cnr containerSDK.Contain } } - namespace, hasNamespace := strings.CutSuffix(ctx.d.Zone(), ".ns") - if !hasNamespace { - return nil - } - addr, err := util.Uint160DecodeBytesBE(cnr.Owner().WalletBytes()[1 : 1+util.Uint160Size]) if err != nil { return fmt.Errorf("could not get container owner address: %w", err) @@ -195,6 +190,11 @@ func (cp *Processor) checkNNS(ctx *putContainerContext, cnr containerSDK.Contain return fmt.Errorf("could not get subject from FrostfsID contract: %w", err) } + namespace, hasNamespace := strings.CutSuffix(ctx.d.Zone(), ".ns") + if !hasNamespace { + return nil + } + if subject.Namespace != namespace { return errContainerAndOwnerNamespaceDontMatch } From 7e04083c273c7c50eebd76cde5d46d91fcc494a2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 29 Jul 2024 13:03:55 +0300 Subject: [PATCH 1254/1943] [#1278] containerSvc: Validate FrostFSID subject exitence on Put Signed-off-by: Dmitrii Stepanov --- pkg/services/container/ape.go | 21 ++++++++++++++++++++- pkg/services/container/ape_test.go | 15 ++++++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 8fe4dd2d9..6f8a8e0e6 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -211,7 +211,7 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont } } - namespace, err := ac.namespaceByOwner(req.GetBody().GetContainer().GetOwnerID()) + namespace, err := ac.namespaceByKnownOwner(req.GetBody().GetContainer().GetOwnerID()) if err != nil { return nil, fmt.Errorf("get namespace error: %w", err) } @@ -608,6 +608,25 @@ func (ac *apeChecker) namespaceByOwner(owner *refs.OwnerID) (string, error) { return namespace, nil } +func (ac *apeChecker) namespaceByKnownOwner(owner *refs.OwnerID) (string, error) { + var ownerSDK user.ID + if owner == nil { + return "", errOwnerIDIsNotSet + } + if err := ownerSDK.ReadFromV2(*owner); err != nil { + return "", err + } + addr, err := ownerSDK.ScriptHash() + if err != nil { + return "", err + } + subject, err := ac.frostFSIDClient.GetSubject(addr) + if err != nil { + return "", fmt.Errorf("get subject error: %w", err) + } + return subject.Namespace, nil +} + // validateNamespace validates a namespace set in a container. // If frostfs-id contract stores a namespace N1 for an owner ID and a container within a request // is set with namespace N2 (via Zone() property), then N2 is invalid and the request is denied. diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index 9eed469ca..68c1158a6 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -765,17 +765,22 @@ func testDenyPutContainerForOthersSessionToken(t *testing.T) { keys: [][]byte{}, } nm := &netmapStub{} - frostfsIDSubjectReader := &frostfsidStub{ - subjects: map[util.Uint160]*client.Subject{}, - } - apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) testContainer := containertest.Container() + owner := testContainer.Owner() + ownerAddr, err := owner.ScriptHash() + require.NoError(t, err) + frostfsIDSubjectReader := &frostfsidStub{ + subjects: map[util.Uint160]*client.Subject{ + ownerAddr: {}, + }, + } + apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) nm.currentEpoch = 100 nm.netmaps = map[uint64]*netmap.NetMap{} - _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(""), &chain.Chain{ + _, _, err = router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(""), &chain.Chain{ Rules: []chain.Rule{ { Status: chain.AccessDenied, From c49982d22ac0f2f4314ba2d79ab835d93cc437c6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 30 Jul 2024 15:41:57 +0300 Subject: [PATCH 1255/1943] [#1282] cli: Allow to external addresses first for `object nodes` Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index d04cf6f04..42ae7324e 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -30,7 +30,8 @@ import ( ) const ( - verifyPresenceAllFlag = "verify-presence-all" + verifyPresenceAllFlag = "verify-presence-all" + preferInternalAddressesFlag = "prefer-internal-addresses" ) var ( @@ -97,6 +98,7 @@ func initObjectNodesCmd() { flags.Bool(verifyPresenceAllFlag, false, "Verify the actual presence of the object on all netmap nodes.") flags.Bool(commonflags.JSON, false, "Print information about the object placement as json.") + flags.Bool(preferInternalAddressesFlag, false, "Use internal addresses first to get object info.") } func objectNodes(cmd *cobra.Command, _ []string) { @@ -449,11 +451,20 @@ func getNodesToCheckObjectExistance(cmd *cobra.Command, netmap *netmapSDK.NetMap func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.NodeInfo, pk *ecdsa.PrivateKey) (*client.Client, error) { var cli *client.Client var addresses []string - candidate.IterateNetworkEndpoints(func(s string) bool { - addresses = append(addresses, s) - return false - }) - addresses = append(addresses, candidate.ExternalAddresses()...) + if preferInternal, _ := cmd.Flags().GetBool(preferInternalAddressesFlag); preferInternal { + candidate.IterateNetworkEndpoints(func(s string) bool { + addresses = append(addresses, s) + return false + }) + addresses = append(addresses, candidate.ExternalAddresses()...) + } else { + addresses = append(addresses, candidate.ExternalAddresses()...) + candidate.IterateNetworkEndpoints(func(s string) bool { + addresses = append(addresses, s) + return false + }) + } + var lastErr error for _, address := range addresses { var networkAddr network.Address From a55600893efeadfc48a83f3f2529d846f4d534c3 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 2 Aug 2024 12:31:59 +0300 Subject: [PATCH 1256/1943] [#1266] Makefile: Specify gofumpt version Add target to install gofumpt, fix target to run gofumpt. Signed-off-by: Ekaterina Lebedeva --- Makefile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c93d06aa8..11111d9a7 100755 --- a/Makefile +++ b/Makefile @@ -46,6 +46,10 @@ STATICCHECK_VERSION_DIR ?= $(STATICCHECK_DIR)/$(STATICCHECK_VERSION) SOURCES = $(shell find . -type f -name "*.go" -print) +GOFUMPT_VERSION ?= v0.6.0 +GOFUMPT_DIR ?= $(abspath $(BIN))/gofumpt +GOFUMPT_VERSION_DIR ?= $(GOFUMPT_DIR)/$(GOFUMPT_VERSION) + GOPLS_VERSION ?= v0.15.1 GOPLS_DIR ?= $(abspath $(BIN))/gopls GOPLS_VERSION_DIR ?= $(GOPLS_DIR)/$(GOPLS_VERSION) @@ -165,10 +169,19 @@ imports: @echo "⇒ Processing goimports check" @goimports -w cmd/ pkg/ misc/ +# Install gofumpt +fumpt-install: + @rm -rf $(GOFUMPT_DIR) + @mkdir $(GOFUMPT_DIR) + @GOBIN=$(GOFUMPT_VERSION_DIR) go install mvdan.cc/gofumpt@$(GOFUMPT_VERSION) + # Run gofumpt fumpt: + @if [ ! -d "$(GOFUMPT_VERSION_DIR)" ]; then \ + make fumpt-install; \ + fi @echo "⇒ Processing gofumpt check" - @gofumpt -l -w cmd/ pkg/ misc/ + $(GOFUMPT_VERSION_DIR)/gofumpt -l -w cmd/ pkg/ misc/ # Run Unit Test with go test test: GOFLAGS ?= "-count=1" From ef4cea6d19eb8a4d05e47647f00d874c892c17ea Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 2 Aug 2024 12:34:04 +0300 Subject: [PATCH 1257/1943] [#1266] .forgejo: Add gofumpt action `gofumpt` was skipped by pre-commit on CI, and now is used in a separate action. Signed-off-by: Ekaterina Lebedeva --- .forgejo/workflows/tests.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index b0c9adbf2..5d64d7bc4 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -89,3 +89,21 @@ jobs: - name: Run gopls run: make gopls-run + + fumpt: + name: Run gofumpt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.22' + cache: true + + - name: Install gofumpt + run: make fumpt-install + + - name: Run gofumpt + run: make fumpt From 8021bacc43c1ae065033bfefa163d802b9eaeb33 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 1 Aug 2024 16:08:50 +0300 Subject: [PATCH 1258/1943] [#1288] putSvc: Respect TTL for EC put Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/ec.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/pkg/services/object/put/ec.go b/pkg/services/object/put/ec.go index 6da50195e..fbb51912c 100644 --- a/pkg/services/object/put/ec.go +++ b/pkg/services/object/put/ec.go @@ -17,6 +17,7 @@ import ( containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -39,7 +40,7 @@ type ecWriter struct { } func (e *ecWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error { - relayed, err := e.relayIfNotContainerNode(ctx) + relayed, err := e.relayIfNotContainerNode(ctx, obj) if err != nil { return err } @@ -65,7 +66,7 @@ func (e *ecWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error return e.writeRawObject(ctx, obj) } -func (e *ecWriter) relayIfNotContainerNode(ctx context.Context) (bool, error) { +func (e *ecWriter) relayIfNotContainerNode(ctx context.Context, obj *objectSDK.Object) (bool, error) { if e.relay == nil { return false, nil } @@ -77,7 +78,13 @@ func (e *ecWriter) relayIfNotContainerNode(ctx context.Context) (bool, error) { // object can be splitted or saved local return false, nil } - if err := e.relayToContainerNode(ctx); err != nil { + objID := object.AddressOf(obj).Object() + var index uint32 + if obj.ECHeader() != nil { + objID = obj.ECHeader().Parent() + index = obj.ECHeader().Index() + } + if err := e.relayToContainerNode(ctx, objID, index); err != nil { return false, err } return true, nil @@ -102,18 +109,20 @@ func (e *ecWriter) currentNodeIsContainerNode() (bool, error) { return false, nil } -func (e *ecWriter) relayToContainerNode(ctx context.Context) error { - t, err := placement.NewTraverser(e.placementOpts...) +func (e *ecWriter) relayToContainerNode(ctx context.Context, objID oid.ID, index uint32) error { + t, err := placement.NewTraverser(append(e.placementOpts, placement.ForObject(objID))...) if err != nil { return err } var lastErr error + offset := int(index) for { nodes := t.Next() if len(nodes) == 0 { break } - for _, node := range nodes { + for idx := range nodes { + node := nodes[(idx+offset)%len(nodes)] var info client.NodeInfo client.NodeInfoFromNetmapElement(&info, node) @@ -149,6 +158,10 @@ func (e *ecWriter) relayToContainerNode(ctx context.Context) error { } func (e *ecWriter) writeECPart(ctx context.Context, obj *objectSDK.Object) error { + if e.commonPrm.LocalOnly() { + return e.writePartLocal(ctx, obj) + } + t, err := placement.NewTraverser(append(e.placementOpts, placement.ForObject(obj.ECHeader().Parent()))...) if err != nil { return err From dc3dcabadca1d1bfb20dab5947ec4c22f3d2362b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 2 Aug 2024 17:50:49 +0300 Subject: [PATCH 1259/1943] [#1291] morph: Reconnect to the highest priority endpoint Signed-off-by: Dmitrii Stepanov --- pkg/morph/client/constructor.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 648c7d3c0..78cb3e82f 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -148,6 +148,10 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } else { cli.logger.Info(logs.FrostFSIRCreatedRPCClientForEndpoint, zap.String("endpoint", endpoint.Address)) + if cli.endpoints.curr > 0 && cli.cfg.switchInterval != 0 { + cli.switchIsActive.Store(true) + go cli.switchToMostPrioritized(ctx) + } break } } From 327d364f34de730879f330ea51d8801f5c6bddc9 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 2 Aug 2024 16:20:51 +0300 Subject: [PATCH 1260/1943] [#1262] sdnotify: Get rid of go:linkname for nanotime Signed-off-by: Ekaterina Lebedeva --- pkg/util/sdnotify/clock.go | 10 ---------- pkg/util/sdnotify/clock.s | 2 -- pkg/util/sdnotify/sdnotify.go | 4 +++- 3 files changed, 3 insertions(+), 13 deletions(-) delete mode 100644 pkg/util/sdnotify/clock.go delete mode 100644 pkg/util/sdnotify/clock.s diff --git a/pkg/util/sdnotify/clock.go b/pkg/util/sdnotify/clock.go deleted file mode 100644 index f5419d027..000000000 --- a/pkg/util/sdnotify/clock.go +++ /dev/null @@ -1,10 +0,0 @@ -package sdnotify - -import ( - // For go:linkname to work. - _ "unsafe" -) - -//go:noescape -//go:linkname nanotime runtime.nanotime -func nanotime() int64 diff --git a/pkg/util/sdnotify/clock.s b/pkg/util/sdnotify/clock.s deleted file mode 100644 index ad033ff4f..000000000 --- a/pkg/util/sdnotify/clock.s +++ /dev/null @@ -1,2 +0,0 @@ -// The file is intentionally empty. -// It is a workaround for https://github.com/golang/go/issues/15006 diff --git a/pkg/util/sdnotify/sdnotify.go b/pkg/util/sdnotify/sdnotify.go index 16a3f11c1..a3af50b22 100644 --- a/pkg/util/sdnotify/sdnotify.go +++ b/pkg/util/sdnotify/sdnotify.go @@ -6,6 +6,7 @@ import ( "net" "os" "strings" + "time" ) const ( @@ -16,6 +17,7 @@ const ( var ( socket *net.UnixAddr + start = time.Now() errSocketVariableIsNotPresent = errors.New("\"NOTIFY_SOCKET\" environment variable is not present") errSocketIsNotInitialized = errors.New("socket is not initialized") @@ -51,7 +53,7 @@ func FlagAndStatus(status string) error { // must be sent, containing "READY=1". // // For MONOTONIC_USEC format refer to https://www.man7.org/linux/man-pages/man3/sd_notify.3.html - status += fmt.Sprintf("\nMONOTONIC_USEC=%d", uint64(nanotime())/1e3 /* microseconds in nanoseconds */) + status += fmt.Sprintf("\nMONOTONIC_USEC=%d", uint64(time.Since(start))/1e3 /* microseconds in nanoseconds */) } status += "\nSTATUS=" + strings.TrimSuffix(status, "=1") return Send(status) From eeca796d2e5f60b0331e05e29735d7651e9794ae Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Aug 2024 13:24:48 +0300 Subject: [PATCH 1261/1943] [#1295] engine: Log object address in case of error Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/delete.go | 6 +++--- pkg/local_object_storage/engine/exists.go | 3 ++- pkg/local_object_storage/engine/get.go | 2 +- pkg/local_object_storage/engine/head.go | 3 ++- pkg/local_object_storage/engine/inhume.go | 8 ++++---- pkg/local_object_storage/engine/lock.go | 13 +++++++++---- pkg/local_object_storage/engine/put.go | 2 +- pkg/local_object_storage/engine/range.go | 2 +- 8 files changed, 23 insertions(+), 16 deletions(-) diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 096528967..318f938fb 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -100,7 +100,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e return false } else { if !client.IsErrObjectNotFound(err) { - e.reportShardError(sh, "could not check object existence", err) + e.reportShardError(sh, "could not check object existence", err, zap.Stringer("address", prm.addr)) } return false } @@ -116,7 +116,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e _, err = sh.Inhume(ctx, shPrm) if err != nil { - e.reportShardError(sh, "could not inhume object in shard", err) + e.reportShardError(sh, "could not inhume object in shard", err, zap.Stringer("address", prm.addr)) var target *apistatus.ObjectLocked locked.is = errors.As(err, &target) @@ -191,7 +191,7 @@ func (e *StorageEngine) deleteChunks( var objID oid.ID err := objID.ReadFromV2(chunk.ID) if err != nil { - e.reportShardError(sh, "could not delete EC chunk", err) + e.reportShardError(sh, "could not delete EC chunk", err, zap.Stringer("address", prm.addr)) } addr.SetObject(objID) inhumePrm.MarkAsGarbage(addr) diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index c57f79691..d98101306 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "go.uber.org/zap" ) // exists return in the first value true if object exists. @@ -36,7 +37,7 @@ func (e *StorageEngine) exists(ctx context.Context, shPrm shard.ExistsPrm) (bool } if !client.IsErrObjectNotFound(err) { - e.reportShardError(sh, "could not check existence of object in shard", err) + e.reportShardError(sh, "could not check existence of object in shard", err, zap.Stringer("address", shPrm.Address)) } return false } diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 991af3d1a..253256c34 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -186,7 +186,7 @@ func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { i.ObjectExpired = true return true default: - i.Engine.reportShardError(sh, "could not get object from shard", err) + i.Engine.reportShardError(sh, "could not get object from shard", err, zap.Stringer("address", i.Address)) return false } }) diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index 92d1b20fc..dfe5e48a1 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -12,6 +12,7 @@ import ( apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" ) // HeadPrm groups the parameters of Head operation. @@ -118,7 +119,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) outError = new(apistatus.ObjectNotFound) return true default: - e.reportShardError(sh, "could not head object from shard", err) + e.reportShardError(sh, "could not head object from shard", err, zap.Stringer("address", prm.addr)) return false } } diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 991305af0..683713f94 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -154,7 +154,7 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh var siErr *objectSDK.SplitInfoError var ecErr *objectSDK.ECInfoError if !(errors.As(err, &siErr) || errors.As(err, &ecErr)) { - e.reportShardError(sh, "could not check for presents in shard", err) + e.reportShardError(sh, "could not check for presents in shard", err, zap.Stringer("address", addr)) return } @@ -179,7 +179,7 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh return true } - e.reportShardError(sh, "could not inhume object in shard", err) + e.reportShardError(sh, "could not inhume object in shard", err, zap.Stringer("address", addr)) return false } @@ -205,7 +205,7 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { locked, err = h.Shard.IsLocked(ctx, addr) if err != nil { - e.reportShardError(h, "can't check object's lockers", err, zap.Stringer("addr", addr), + e.reportShardError(h, "can't check object's lockers", err, zap.Stringer("address", addr), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) outErr = err return false @@ -235,7 +235,7 @@ func (e *StorageEngine) GetLocked(ctx context.Context, addr oid.Address) ([]oid. e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { ld, err := h.Shard.GetLocked(ctx, addr) if err != nil { - e.reportShardError(h, logs.EngineInterruptGettingLockers, err, zap.Stringer("addr", addr), + e.reportShardError(h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) outErr = err } diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 5354c205f..3a41a7848 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -13,6 +13,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" ) var errLockFailed = errors.New("lock operation failed") @@ -90,14 +91,16 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo var objID oid.ID err = objID.ReadFromV2(chunk.ID) if err != nil { - e.reportShardError(sh, "could not lock object in shard", err) + e.reportShardError(sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), + zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) return false } eclocked = append(eclocked, objID) } err = sh.Lock(ctx, idCnr, locker, eclocked) if err != nil { - e.reportShardError(sh, "could not lock object in shard", err) + e.reportShardError(sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), + zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) return false } root = true @@ -109,7 +112,8 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo return true } - e.reportShardError(sh, "could not check locked object for presence in shard", err) + e.reportShardError(sh, "could not check locked object for presence in shard", err, zap.Stringer("container_id", idCnr), + zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) return } @@ -121,7 +125,8 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo err := sh.Lock(ctx, idCnr, locker, []oid.ID{locked}) if err != nil { - e.reportShardError(sh, "could not lock object in shard", err) + e.reportShardError(sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), + zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) var errIrregular *apistatus.LockNonRegularObject if errors.As(err, &errIrregular) { diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 54385910b..f92d83745 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -187,7 +187,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti return } - e.reportShardError(sh, "could not put object to shard", err) + e.reportShardError(sh, "could not put object to shard", err, zap.Stringer("address", addr)) return } diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index f5b33a251..cbf26ff4e 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -208,7 +208,7 @@ func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) { return true // stop, return it back default: - i.Engine.reportShardError(sh, "could not get object from shard", err) + i.Engine.reportShardError(sh, "could not get object from shard", err, zap.Stringer("address", i.Address)) return false } }) From 10602b55b1974512b5cac603b87ae6f1b2fe84b5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Aug 2024 16:13:43 +0300 Subject: [PATCH 1262/1943] [#1295] engine: Resolve funlen linter Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/lock.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 3a41a7848..ac8fa9c6f 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -63,11 +63,9 @@ func (e *StorageEngine) lock(ctx context.Context, idCnr cid.ID, locker oid.ID, l func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, locked oid.ID, checkExists bool) (status uint8) { // code is pretty similar to inhumeAddr, maybe unify? root := false - var addrLocked oid.Address addrLocked.SetContainer(idCnr) addrLocked.SetObject(locked) - e.iterateOverSortedShards(addrLocked, func(_ int, sh hashedShard) (stop bool) { defer func() { // if object is root we continue since information about it @@ -80,7 +78,6 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo if checkExists { var existsPrm shard.ExistsPrm existsPrm.Address = addrLocked - exRes, err := sh.Exists(ctx, existsPrm) if err != nil { var siErr *objectSDK.SplitInfoError @@ -111,7 +108,6 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo // do not lock it return true } - e.reportShardError(sh, "could not check locked object for presence in shard", err, zap.Stringer("container_id", idCnr), zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) return @@ -133,14 +129,10 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo status = 1 return true } - return false } - status = 2 - return true }) - return } From 8e51d7849a41c1135695650d9a748d0ba29a9dfb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Aug 2024 09:32:40 +0300 Subject: [PATCH 1263/1943] [#1295] getSvc: Assemble complex EC object headers without linking object Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/head.go | 15 ++++----- pkg/local_object_storage/metabase/get.go | 22 +++++++++--- pkg/services/object/get/assemble.go | 4 +-- pkg/services/object/get/assembler.go | 43 ++++++++++++++++++++++++ pkg/services/object/get/get_test.go | 2 +- pkg/services/object/get/request.go | 4 +-- 6 files changed, 72 insertions(+), 18 deletions(-) diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index dfe5e48a1..6857a3631 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -127,17 +127,16 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) return true }) + if head != nil { + return HeadRes{head: head}, nil + } if outSI != nil { return HeadRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(outSI)) - } else if outEI != nil { - return HeadRes{}, logicerr.Wrap(objectSDK.NewECInfoError(outEI)) - } else if head == nil { - return HeadRes{}, outError } - - return HeadRes{ - head: head, - }, nil + if outEI != nil { + return HeadRes{}, logicerr.Wrap(objectSDK.NewECInfoError(outEI)) + } + return HeadRes{}, outError } // Head reads object header from local storage by provided address. diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index d9acd4ce2..b79f6cb14 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -160,11 +160,23 @@ func getVirtualObject(tx *bbolt.Tx, cnr cid.ID, key []byte, raw bool) (*objectSD return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) } - // pick last item, for now there is not difference which address to pick - // but later list might be sorted so first or last value can be more - // prioritized to choose - virtualOID := relativeLst[len(relativeLst)-1] - data := getFromBucket(tx, primaryBucketName(cnr, bucketName), virtualOID) + var data []byte + for i := 0; i < len(relativeLst) && len(data) == 0; i++ { + virtualOID := relativeLst[len(relativeLst)-i-1] + data = getFromBucket(tx, primaryBucketName(cnr, bucketName), virtualOID) + } + + if len(data) == 0 { + // check if any of the relatives is an EC object + for _, relative := range relativeLst { + data = getFromBucket(tx, ecInfoBucketName(cnr, bucketName), relative) + if len(data) > 0 { + // we can't return object headers, but can return error, + // so assembler can try to assemble complex object + return nil, getSplitInfoError(tx, cnr, key) + } + } + } child := objectSDK.New() diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index ba6fddec5..9f17f1e4c 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -12,7 +12,7 @@ import ( ) func (r *request) assemble(ctx context.Context) { - if !r.canAssemble() { + if !r.canAssembleComplexObject() { r.log.Debug(logs.GetCanNotAssembleTheObject) return } @@ -38,7 +38,7 @@ func (r *request) assemble(ctx context.Context) { r.log.Debug(logs.GetTryingToAssembleTheObject) r.prm.common = r.prm.common.WithLocalOnly(false) - assembler := newAssembler(r.address(), r.splitInfo(), r.ctxRange(), r) + assembler := newAssembler(r.address(), r.splitInfo(), r.ctxRange(), r, r.headOnly()) r.log.Debug(logs.GetAssemblingSplittedObject, zap.Uint64("range_offset", r.ctxRange().GetOffset()), diff --git a/pkg/services/object/get/assembler.go b/pkg/services/object/get/assembler.go index 025296ec7..ff3f90bf2 100644 --- a/pkg/services/object/get/assembler.go +++ b/pkg/services/object/get/assembler.go @@ -19,6 +19,7 @@ type assembler struct { splitInfo *objectSDK.SplitInfo rng *objectSDK.Range objGetter objectGetter + head bool currentOffset uint64 @@ -30,18 +31,23 @@ func newAssembler( splitInfo *objectSDK.SplitInfo, rng *objectSDK.Range, objGetter objectGetter, + head bool, ) *assembler { return &assembler{ addr: addr, rng: rng, splitInfo: splitInfo, objGetter: objGetter, + head: head, } } // Assemble assembles splitted large object and writes it's content to ObjectWriter. // It returns parent object. func (a *assembler) Assemble(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { + if a.head { + return a.assembleHeader(ctx, writer) + } sourceObjectID, ok := a.getLastPartOrLinkObjectID() if !ok { return nil, objectSDK.NewSplitInfoError(a.splitInfo) @@ -65,6 +71,43 @@ func (a *assembler) Assemble(ctx context.Context, writer ObjectWriter) (*objectS return a.parentObject, nil } +func (a *assembler) assembleHeader(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { + var sourceObjectIDs []oid.ID + sourceObjectID, ok := a.splitInfo.Link() + if ok { + sourceObjectIDs = append(sourceObjectIDs, sourceObjectID) + } + sourceObjectID, ok = a.splitInfo.LastPart() + if ok { + sourceObjectIDs = append(sourceObjectIDs, sourceObjectID) + } + if len(sourceObjectIDs) == 0 { + return nil, objectSDK.NewSplitInfoError(a.splitInfo) + } + for _, sourceObjectID = range sourceObjectIDs { + obj, err := a.getParent(ctx, sourceObjectID, writer) + if err == nil { + return obj, nil + } + } + return nil, objectSDK.NewSplitInfoError(a.splitInfo) +} + +func (a *assembler) getParent(ctx context.Context, sourceObjectID oid.ID, writer ObjectWriter) (*objectSDK.Object, error) { + obj, err := a.objGetter.HeadObject(ctx, sourceObjectID) + if err != nil { + return nil, err + } + parent := obj.Parent() + if parent == nil { + return nil, objectSDK.NewSplitInfoError(a.splitInfo) + } + if err := writer.WriteHeader(ctx, parent); err != nil { + return nil, err + } + return obj, nil +} + func (a *assembler) getLastPartOrLinkObjectID() (oid.ID, bool) { sourceObjectID, ok := a.splitInfo.Link() if ok { diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 29a15ba78..1fc6b7b20 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -730,7 +730,7 @@ func TestGetRemoteSmall(t *testing.T) { t.Run("VIRTUAL", func(t *testing.T) { testHeadVirtual := func(svc *Service, addr oid.Address, i *objectSDK.SplitInfo) { - headPrm := newHeadPrm(false, nil) + headPrm := newHeadPrm(true, nil) headPrm.WithAddress(addr) errSplit := objectSDK.NewSplitInfoError(objectSDK.NewSplitInfo()) diff --git a/pkg/services/object/get/request.go b/pkg/services/object/get/request.go index 9ddfeddf2..1a7a43a35 100644 --- a/pkg/services/object/get/request.go +++ b/pkg/services/object/get/request.go @@ -88,8 +88,8 @@ func (r *request) key() (*ecdsa.PrivateKey, error) { return r.keyStore.GetKey(sessionInfo) } -func (r *request) canAssemble() bool { - return !r.isRaw() && !r.headOnly() +func (r *request) canAssembleComplexObject() bool { + return !r.isRaw() } func (r *request) splitInfo() *objectSDK.SplitInfo { From 5c01bd5be89b40f2bd5edd4526b474644e053f11 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Aug 2024 13:20:33 +0300 Subject: [PATCH 1264/1943] [#1298] writecache: Add `restore-mode` flag for Seal command Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/writecache.go | 5 + pkg/local_object_storage/engine/writecache.go | 4 +- pkg/local_object_storage/shard/writecache.go | 5 +- pkg/local_object_storage/writecache/seal.go | 19 +- .../writecache/writecache.go | 7 +- pkg/services/control/ir/service_grpc.pb.go | 6 +- .../control/server/seal_writecache.go | 1 + pkg/services/control/service.pb.go | 347 +++++++++--------- pkg/services/control/service.proto | 3 + pkg/services/control/service_frostfs.pb.go | 2 + pkg/services/tree/service_grpc.pb.go | 44 +-- 11 files changed, 238 insertions(+), 205 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/writecache.go b/cmd/frostfs-cli/modules/control/writecache.go index abc4ed2e6..a665ccae8 100644 --- a/cmd/frostfs-cli/modules/control/writecache.go +++ b/cmd/frostfs-cli/modules/control/writecache.go @@ -9,6 +9,8 @@ import ( "github.com/spf13/cobra" ) +const restoreModeFlag = "restore-mode" + var writecacheShardCmd = &cobra.Command{ Use: "writecache", Short: "Operations with storage node's write-cache", @@ -26,10 +28,12 @@ func sealWritecache(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag) + restoreMode, _ := cmd.Flags().GetBool(restoreModeFlag) req := &control.SealWriteCacheRequest{Body: &control.SealWriteCacheRequest_Body{ Shard_ID: getShardIDList(cmd), IgnoreErrors: ignoreErrors, + RestoreMode: restoreMode, }} signRequest(cmd, pk, req) @@ -68,6 +72,7 @@ func initControlShardsWritecacheCmd() { ff.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") ff.Bool(shardAllFlag, false, "Process all shards") ff.Bool(ignoreErrorsFlag, true, "Skip invalid/unreadable objects") + ff.Bool(restoreModeFlag, false, "Restore writecache's mode after sealing") sealWritecacheShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) } diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index da488260a..8f37d7860 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -70,6 +70,7 @@ func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePr type SealWriteCachePrm struct { ShardIDs []*shard.ID IgnoreErrors bool + RestoreMode bool } type ShardSealResult struct { @@ -88,6 +89,7 @@ func (e *StorageEngine) SealWriteCache(ctx context.Context, prm SealWriteCachePr trace.WithAttributes( attribute.Int("shard_id_count", len(prm.ShardIDs)), attribute.Bool("ignore_errors", prm.IgnoreErrors), + attribute.Bool("restore_mode", prm.RestoreMode), )) defer span.End() @@ -114,7 +116,7 @@ func (e *StorageEngine) SealWriteCache(ctx context.Context, prm SealWriteCachePr return nil } - err := sh.SealWriteCache(egCtx, shard.SealWriteCachePrm{IgnoreErrors: prm.IgnoreErrors}) + err := sh.SealWriteCache(egCtx, shard.SealWriteCachePrm{IgnoreErrors: prm.IgnoreErrors, RestoreMode: prm.RestoreMode}) resGuard.Lock() defer resGuard.Unlock() diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index 05e014d29..9edad7170 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -59,6 +60,7 @@ func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error type SealWriteCachePrm struct { IgnoreErrors bool + RestoreMode bool } // SealWriteCache flushes all data from the write-cache and moves it to degraded read only mode. @@ -67,6 +69,7 @@ func (s *Shard) SealWriteCache(ctx context.Context, p SealWriteCachePrm) error { trace.WithAttributes( attribute.String("shard_id", s.ID().String()), attribute.Bool("ignore_errors", p.IgnoreErrors), + attribute.Bool("restore_mode", p.RestoreMode), )) defer span.End() @@ -84,5 +87,5 @@ func (s *Shard) SealWriteCache(ctx context.Context, p SealWriteCachePrm) error { return ErrDegradedMode } - return s.writeCache.Seal(ctx, p.IgnoreErrors) + return s.writeCache.Seal(ctx, writecache.SealPrm{IgnoreErrors: p.IgnoreErrors, RestoreMode: p.RestoreMode}) } diff --git a/pkg/local_object_storage/writecache/seal.go b/pkg/local_object_storage/writecache/seal.go index 48107a75f..22b4e0988 100644 --- a/pkg/local_object_storage/writecache/seal.go +++ b/pkg/local_object_storage/writecache/seal.go @@ -9,20 +9,29 @@ import ( "go.opentelemetry.io/otel/trace" ) -func (c *cache) Seal(ctx context.Context, ignoreErrors bool) error { +func (c *cache) Seal(ctx context.Context, prm SealPrm) error { ctx, span := tracing.StartSpanFromContext(ctx, "writecache.Seal", trace.WithAttributes( - attribute.Bool("ignore_errors", ignoreErrors), + attribute.Bool("ignore_errors", prm.IgnoreErrors), + attribute.Bool("restore_mode", prm.RestoreMode), )) defer span.End() c.modeMtx.Lock() defer c.modeMtx.Unlock() + sourceMode := c.mode // flush will be done by setMode - err := c.setMode(ctx, mode.DegradedReadOnly, ignoreErrors) - if err == nil { - c.metrics.SetMode(mode.ComponentDisabled) + err := c.setMode(ctx, mode.DegradedReadOnly, prm.IgnoreErrors) + if err != nil { + return err + } + c.metrics.SetMode(mode.ComponentDisabled) + if prm.RestoreMode { + err = c.setMode(ctx, sourceMode, prm.IgnoreErrors) + if err == nil { + c.metrics.SetMode(mode.ConvertToComponentMode(sourceMode)) + } } return err } diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 71dba61cf..7085a57b2 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -20,6 +20,11 @@ type Info struct { Path string } +type SealPrm struct { + IgnoreErrors bool + RestoreMode bool +} + // Cache represents write-cache for objects. type Cache interface { Get(ctx context.Context, address oid.Address) (*objectSDK.Object, error) @@ -36,7 +41,7 @@ type Cache interface { SetLogger(*logger.Logger) DumpInfo() Info Flush(context.Context, bool, bool) error - Seal(context.Context, bool) error + Seal(context.Context, SealPrm) error Init() error Open(ctx context.Context, mode mode.Mode) error diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index 724149c44..336bf5f70 100644 --- a/pkg/services/control/ir/service_grpc.pb.go +++ b/pkg/services/control/ir/service_grpc.pb.go @@ -35,7 +35,8 @@ type ControlServiceClient interface { TickEpoch(ctx context.Context, in *TickEpochRequest, opts ...grpc.CallOption) (*TickEpochResponse, error) // Forces a node removal to be signaled by the IR node with high probability. RemoveNode(ctx context.Context, in *RemoveNodeRequest, opts ...grpc.CallOption) (*RemoveNodeResponse, error) - // Forces a container removal to be signaled by the IR node with high probability. + // Forces a container removal to be signaled by the IR node with high + // probability. RemoveContainer(ctx context.Context, in *RemoveContainerRequest, opts ...grpc.CallOption) (*RemoveContainerResponse, error) } @@ -93,7 +94,8 @@ type ControlServiceServer interface { TickEpoch(context.Context, *TickEpochRequest) (*TickEpochResponse, error) // Forces a node removal to be signaled by the IR node with high probability. RemoveNode(context.Context, *RemoveNodeRequest) (*RemoveNodeResponse, error) - // Forces a container removal to be signaled by the IR node with high probability. + // Forces a container removal to be signaled by the IR node with high + // probability. RemoveContainer(context.Context, *RemoveContainerRequest) (*RemoveContainerResponse, error) } diff --git a/pkg/services/control/server/seal_writecache.go b/pkg/services/control/server/seal_writecache.go index e3f8b8caf..b663cfc81 100644 --- a/pkg/services/control/server/seal_writecache.go +++ b/pkg/services/control/server/seal_writecache.go @@ -19,6 +19,7 @@ func (s *Server) SealWriteCache(ctx context.Context, req *control.SealWriteCache prm := engine.SealWriteCachePrm{ ShardIDs: s.getShardIDList(req.GetBody().GetShard_ID()), IgnoreErrors: req.GetBody().GetIgnoreErrors(), + RestoreMode: req.GetBody().GetRestoreMode(), } res, err := s.s.SealWriteCache(ctx, prm) diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 727dd1218..895b74368 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -4727,6 +4727,8 @@ type SealWriteCacheRequest_Body struct { Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` // Flag indicating whether object read errors should be ignored. IgnoreErrors bool `protobuf:"varint,2,opt,name=ignore_errors,json=ignoreErrors,proto3" json:"ignore_errors,omitempty"` + // If true, then writecache will be sealed, but mode will be restored to the current one. + RestoreMode bool `protobuf:"varint,4,opt,name=restore_mode,json=restoreMode,proto3" json:"restore_mode,omitempty"` } func (x *SealWriteCacheRequest_Body) Reset() { @@ -4775,6 +4777,13 @@ func (x *SealWriteCacheRequest_Body) GetIgnoreErrors() bool { return false } +func (x *SealWriteCacheRequest_Body) GetRestoreMode() bool { + if x != nil { + return x.RestoreMode + } + return false +} + type SealWriteCacheResponse_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -5482,7 +5491,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xca, 0x01, 0x0a, 0x15, + 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xed, 0x01, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, @@ -5491,182 +5500,184 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, + 0x1a, 0x69, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, - 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, - 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, - 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, - 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, - 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, - 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, - 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, - 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, + 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, + 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0xa9, 0x02, 0x0a, 0x16, + 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, + 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, + 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, + 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, + 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, + 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, + 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, + 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x32, 0x8b, 0x10, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, - 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, - 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, - 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, - 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, + 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, + 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0x8b, 0x10, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, + 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, + 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, - 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, - 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, - 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, - 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, - 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, + 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, + 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, + 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, + 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, - 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, - 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, - 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, - 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, - 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, 0x65, 0x73, + 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, + 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, + 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, + 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, + 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, + 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, + 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x8a, 0x01, 0x0a, 0x21, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x8a, 0x01, 0x0a, 0x21, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, - 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, - 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, - 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, - 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, - 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, - 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, - 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, + 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, + 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, + 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, + 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, + 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, + 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 2cd8434fc..a10410025 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -655,6 +655,9 @@ message SealWriteCacheRequest { // Flag indicating whether object read errors should be ignored. bool ignore_errors = 2; + + // If true, then writecache will be sealed, but mode will be restored to the current one. + bool restore_mode = 4; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index a287606fa..b6b064973 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -3422,6 +3422,7 @@ func (x *SealWriteCacheRequest_Body) StableSize() (size int) { } size += proto.RepeatedBytesSize(1, x.Shard_ID) size += proto.BoolSize(2, x.IgnoreErrors) + size += proto.BoolSize(4, x.RestoreMode) return size } @@ -3443,6 +3444,7 @@ func (x *SealWriteCacheRequest_Body) StableMarshal(buf []byte) []byte { var offset int offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) offset += proto.BoolMarshal(2, buf[offset:], x.IgnoreErrors) + offset += proto.BoolMarshal(4, buf[offset:], x.RestoreMode) return buf } diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index 4c293a4c0..63f96e11a 100644 --- a/pkg/services/tree/service_grpc.pb.go +++ b/pkg/services/tree/service_grpc.pb.go @@ -3,7 +3,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.4.0 +// - protoc-gen-go-grpc v1.3.0 // - protoc v4.25.0 // source: pkg/services/tree/service.proto @@ -18,8 +18,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.62.0 or later. -const _ = grpc.SupportPackageIsVersion8 +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 const ( TreeService_Add_FullMethodName = "/tree.TreeService/Add" @@ -70,9 +70,8 @@ func NewTreeServiceClient(cc grpc.ClientConnInterface) TreeServiceClient { } func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grpc.CallOption) (*AddResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AddResponse) - err := c.cc.Invoke(ctx, TreeService_Add_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_Add_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -80,9 +79,8 @@ func (c *treeServiceClient) Add(ctx context.Context, in *AddRequest, opts ...grp } func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, opts ...grpc.CallOption) (*AddByPathResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AddByPathResponse) - err := c.cc.Invoke(ctx, TreeService_AddByPath_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_AddByPath_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -90,9 +88,8 @@ func (c *treeServiceClient) AddByPath(ctx context.Context, in *AddByPathRequest, } func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts ...grpc.CallOption) (*RemoveResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RemoveResponse) - err := c.cc.Invoke(ctx, TreeService_Remove_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_Remove_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -100,9 +97,8 @@ func (c *treeServiceClient) Remove(ctx context.Context, in *RemoveRequest, opts } func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...grpc.CallOption) (*MoveResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MoveResponse) - err := c.cc.Invoke(ctx, TreeService_Move_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_Move_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -110,9 +106,8 @@ func (c *treeServiceClient) Move(ctx context.Context, in *MoveRequest, opts ...g } func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPathRequest, opts ...grpc.CallOption) (*GetNodeByPathResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetNodeByPathResponse) - err := c.cc.Invoke(ctx, TreeService_GetNodeByPath_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_GetNodeByPath_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -120,12 +115,11 @@ func (c *treeServiceClient) GetNodeByPath(ctx context.Context, in *GetNodeByPath } func (c *treeServiceClient) GetSubTree(ctx context.Context, in *GetSubTreeRequest, opts ...grpc.CallOption) (TreeService_GetSubTreeClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], TreeService_GetSubTree_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[0], TreeService_GetSubTree_FullMethodName, opts...) if err != nil { return nil, err } - x := &treeServiceGetSubTreeClient{ClientStream: stream} + x := &treeServiceGetSubTreeClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -153,9 +147,8 @@ func (x *treeServiceGetSubTreeClient) Recv() (*GetSubTreeResponse, error) { } func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, opts ...grpc.CallOption) (*TreeListResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(TreeListResponse) - err := c.cc.Invoke(ctx, TreeService_TreeList_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_TreeList_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -163,9 +156,8 @@ func (c *treeServiceClient) TreeList(ctx context.Context, in *TreeListRequest, o } func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts ...grpc.CallOption) (*ApplyResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ApplyResponse) - err := c.cc.Invoke(ctx, TreeService_Apply_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_Apply_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -173,12 +165,11 @@ func (c *treeServiceClient) Apply(ctx context.Context, in *ApplyRequest, opts .. } func (c *treeServiceClient) GetOpLog(ctx context.Context, in *GetOpLogRequest, opts ...grpc.CallOption) (TreeService_GetOpLogClient, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], TreeService_GetOpLog_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &TreeService_ServiceDesc.Streams[1], TreeService_GetOpLog_FullMethodName, opts...) if err != nil { return nil, err } - x := &treeServiceGetOpLogClient{ClientStream: stream} + x := &treeServiceGetOpLogClient{stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -206,9 +197,8 @@ func (x *treeServiceGetOpLogClient) Recv() (*GetOpLogResponse, error) { } func (c *treeServiceClient) Healthcheck(ctx context.Context, in *HealthcheckRequest, opts ...grpc.CallOption) (*HealthcheckResponse, error) { - cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(HealthcheckResponse) - err := c.cc.Invoke(ctx, TreeService_Healthcheck_FullMethodName, in, out, cOpts...) + err := c.cc.Invoke(ctx, TreeService_Healthcheck_FullMethodName, in, out, opts...) if err != nil { return nil, err } @@ -383,7 +373,7 @@ func _TreeService_GetSubTree_Handler(srv interface{}, stream grpc.ServerStream) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(TreeServiceServer).GetSubTree(m, &treeServiceGetSubTreeServer{ServerStream: stream}) + return srv.(TreeServiceServer).GetSubTree(m, &treeServiceGetSubTreeServer{stream}) } type TreeService_GetSubTreeServer interface { @@ -440,7 +430,7 @@ func _TreeService_GetOpLog_Handler(srv interface{}, stream grpc.ServerStream) er if err := stream.RecvMsg(m); err != nil { return err } - return srv.(TreeServiceServer).GetOpLog(m, &treeServiceGetOpLogServer{ServerStream: stream}) + return srv.(TreeServiceServer).GetOpLog(m, &treeServiceGetOpLogServer{stream}) } type TreeService_GetOpLogServer interface { From 36efccd86251a8445e6efe1550d342db10b0230a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Aug 2024 15:45:53 +0300 Subject: [PATCH 1265/1943] [#1298] writecache: Add `shrink` flag for Seal command Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/writecache.go | 8 +- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/writecache.go | 3 +- pkg/local_object_storage/shard/writecache.go | 3 +- pkg/local_object_storage/writecache/flush.go | 2 +- pkg/local_object_storage/writecache/mode.go | 55 ++- pkg/local_object_storage/writecache/seal.go | 4 +- .../writecache/writecache.go | 1 + .../control/server/seal_writecache.go | 1 + pkg/services/control/service.pb.go | 359 +++++++++--------- pkg/services/control/service.proto | 3 + pkg/services/control/service_frostfs.pb.go | 2 + 12 files changed, 255 insertions(+), 187 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/writecache.go b/cmd/frostfs-cli/modules/control/writecache.go index a665ccae8..b725d8471 100644 --- a/cmd/frostfs-cli/modules/control/writecache.go +++ b/cmd/frostfs-cli/modules/control/writecache.go @@ -9,7 +9,10 @@ import ( "github.com/spf13/cobra" ) -const restoreModeFlag = "restore-mode" +const ( + restoreModeFlag = "restore-mode" + shrinkFlag = "shrink" +) var writecacheShardCmd = &cobra.Command{ Use: "writecache", @@ -29,11 +32,13 @@ func sealWritecache(cmd *cobra.Command, _ []string) { ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag) restoreMode, _ := cmd.Flags().GetBool(restoreModeFlag) + shrink, _ := cmd.Flags().GetBool(shrinkFlag) req := &control.SealWriteCacheRequest{Body: &control.SealWriteCacheRequest_Body{ Shard_ID: getShardIDList(cmd), IgnoreErrors: ignoreErrors, RestoreMode: restoreMode, + Shrink: shrink, }} signRequest(cmd, pk, req) @@ -73,6 +78,7 @@ func initControlShardsWritecacheCmd() { ff.Bool(shardAllFlag, false, "Process all shards") ff.Bool(ignoreErrorsFlag, true, "Skip invalid/unreadable objects") ff.Bool(restoreModeFlag, false, "Restore writecache's mode after sealing") + ff.Bool(shrinkFlag, false, "Shrink writecache's internal storage") sealWritecacheShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 67f173f29..ebb822e1c 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -539,4 +539,5 @@ const ( PolicerCouldNotGetChunk = "could not get EC chunk" PolicerCouldNotGetChunks = "could not get EC chunks" AuditEventLogRecord = "audit event log record" + WritecacheShrinkSkippedNotEmpty = "writecache shrink skipped: database is not empty" ) diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 8f37d7860..2c5e8cc3a 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -71,6 +71,7 @@ type SealWriteCachePrm struct { ShardIDs []*shard.ID IgnoreErrors bool RestoreMode bool + Shrink bool } type ShardSealResult struct { @@ -116,7 +117,7 @@ func (e *StorageEngine) SealWriteCache(ctx context.Context, prm SealWriteCachePr return nil } - err := sh.SealWriteCache(egCtx, shard.SealWriteCachePrm{IgnoreErrors: prm.IgnoreErrors, RestoreMode: prm.RestoreMode}) + err := sh.SealWriteCache(egCtx, shard.SealWriteCachePrm{IgnoreErrors: prm.IgnoreErrors, RestoreMode: prm.RestoreMode, Shrink: prm.Shrink}) resGuard.Lock() defer resGuard.Unlock() diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index 9edad7170..c29710930 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -61,6 +61,7 @@ func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error type SealWriteCachePrm struct { IgnoreErrors bool RestoreMode bool + Shrink bool } // SealWriteCache flushes all data from the write-cache and moves it to degraded read only mode. @@ -87,5 +88,5 @@ func (s *Shard) SealWriteCache(ctx context.Context, p SealWriteCachePrm) error { return ErrDegradedMode } - return s.writeCache.Seal(ctx, writecache.SealPrm{IgnoreErrors: p.IgnoreErrors, RestoreMode: p.RestoreMode}) + return s.writeCache.Seal(ctx, writecache.SealPrm{IgnoreErrors: p.IgnoreErrors, RestoreMode: p.RestoreMode, Shrink: p.Shrink}) } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index da7feda9a..e34f5a76b 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -291,7 +291,7 @@ func (c *cache) Flush(ctx context.Context, ignoreErrors, seal bool) error { if seal { m := c.mode | mode.ReadOnly - if err := c.setMode(ctx, m, ignoreErrors); err != nil { + if err := c.setMode(ctx, m, setModePrm{ignoreErrors: ignoreErrors}); err != nil { return err } c.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 4172cfbc8..44da9b36e 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -2,16 +2,25 @@ package writecache import ( "context" + "errors" "fmt" + "os" + "path/filepath" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) +type setModePrm struct { + ignoreErrors bool + shrink bool +} + // SetMode sets write-cache mode of operation. // When shard is put in read-only mode all objects in memory are flushed to disk // and all background jobs are suspended. @@ -25,7 +34,7 @@ func (c *cache) SetMode(m mode.Mode) error { c.modeMtx.Lock() defer c.modeMtx.Unlock() - err := c.setMode(ctx, m, true) + err := c.setMode(ctx, m, setModePrm{ignoreErrors: true}) if err == nil { c.metrics.SetMode(mode.ConvertToComponentModeDegraded(m)) } @@ -33,21 +42,19 @@ func (c *cache) SetMode(m mode.Mode) error { } // setMode applies new mode. Must be called with cache.modeMtx lock taken. -func (c *cache) setMode(ctx context.Context, m mode.Mode, ignoreErrors bool) error { +func (c *cache) setMode(ctx context.Context, m mode.Mode, prm setModePrm) error { var err error turnOffMeta := m.NoMetabase() if turnOffMeta && !c.mode.NoMetabase() { - err = c.flush(ctx, ignoreErrors) + err = c.flush(ctx, prm.ignoreErrors) if err != nil { return err } } - if c.db != nil { - if err = c.db.Close(); err != nil { - return fmt.Errorf("can't close write-cache database: %w", err) - } + if err := c.closeDB(prm.shrink); err != nil { + return err } // Suspend producers to ensure there are channel send operations in fly. @@ -71,6 +78,40 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode, ignoreErrors bool) err return nil } +func (c *cache) closeDB(shrink bool) error { + if c.db == nil { + return nil + } + if !shrink { + if err := c.db.Close(); err != nil { + return fmt.Errorf("can't close write-cache database: %w", err) + } + return nil + } + + var empty bool + err := c.db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(defaultBucket) + empty = b == nil || b.Stats().KeyN == 0 + return nil + }) + if err != nil && !errors.Is(err, bbolt.ErrDatabaseNotOpen) { + return fmt.Errorf("failed to check DB items: %w", err) + } + if err := c.db.Close(); err != nil { + return fmt.Errorf("can't close write-cache database: %w", err) + } + if empty { + err := os.Remove(filepath.Join(c.path, dbName)) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove DB file: %w", err) + } + } else { + c.log.Info(logs.WritecacheShrinkSkippedNotEmpty) + } + return nil +} + // readOnly returns true if current mode is read-only. // `c.modeMtx` must be taken. func (c *cache) readOnly() bool { diff --git a/pkg/local_object_storage/writecache/seal.go b/pkg/local_object_storage/writecache/seal.go index 22b4e0988..fa224f5e0 100644 --- a/pkg/local_object_storage/writecache/seal.go +++ b/pkg/local_object_storage/writecache/seal.go @@ -22,13 +22,13 @@ func (c *cache) Seal(ctx context.Context, prm SealPrm) error { sourceMode := c.mode // flush will be done by setMode - err := c.setMode(ctx, mode.DegradedReadOnly, prm.IgnoreErrors) + err := c.setMode(ctx, mode.DegradedReadOnly, setModePrm{ignoreErrors: prm.IgnoreErrors, shrink: prm.Shrink}) if err != nil { return err } c.metrics.SetMode(mode.ComponentDisabled) if prm.RestoreMode { - err = c.setMode(ctx, sourceMode, prm.IgnoreErrors) + err = c.setMode(ctx, sourceMode, setModePrm{ignoreErrors: prm.IgnoreErrors}) if err == nil { c.metrics.SetMode(mode.ConvertToComponentMode(sourceMode)) } diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 7085a57b2..a973df604 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -23,6 +23,7 @@ type Info struct { type SealPrm struct { IgnoreErrors bool RestoreMode bool + Shrink bool } // Cache represents write-cache for objects. diff --git a/pkg/services/control/server/seal_writecache.go b/pkg/services/control/server/seal_writecache.go index b663cfc81..697b91918 100644 --- a/pkg/services/control/server/seal_writecache.go +++ b/pkg/services/control/server/seal_writecache.go @@ -20,6 +20,7 @@ func (s *Server) SealWriteCache(ctx context.Context, req *control.SealWriteCache ShardIDs: s.getShardIDList(req.GetBody().GetShard_ID()), IgnoreErrors: req.GetBody().GetIgnoreErrors(), RestoreMode: req.GetBody().GetRestoreMode(), + Shrink: req.GetBody().GetShrink(), } res, err := s.s.SealWriteCache(ctx, prm) diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 895b74368..ac512f1a5 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -4729,6 +4729,8 @@ type SealWriteCacheRequest_Body struct { IgnoreErrors bool `protobuf:"varint,2,opt,name=ignore_errors,json=ignoreErrors,proto3" json:"ignore_errors,omitempty"` // If true, then writecache will be sealed, but mode will be restored to the current one. RestoreMode bool `protobuf:"varint,4,opt,name=restore_mode,json=restoreMode,proto3" json:"restore_mode,omitempty"` + // If true, then writecache will shrink internal storage. + Shrink bool `protobuf:"varint,5,opt,name=shrink,proto3" json:"shrink,omitempty"` } func (x *SealWriteCacheRequest_Body) Reset() { @@ -4784,6 +4786,13 @@ func (x *SealWriteCacheRequest_Body) GetRestoreMode() bool { return false } +func (x *SealWriteCacheRequest_Body) GetShrink() bool { + if x != nil { + return x.Shrink + } + return false +} + type SealWriteCacheResponse_Body struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -5491,7 +5500,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xed, 0x01, 0x0a, 0x15, + 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x86, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, @@ -5500,184 +5509,186 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x69, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, - 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, - 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0xa9, 0x02, 0x0a, 0x16, + 0x1a, 0x81, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, + 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0b, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, + 0x73, 0x68, 0x72, 0x69, 0x6e, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x68, + 0x72, 0x69, 0x6e, 0x6b, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, + 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, + 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, + 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, + 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, - 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, - 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, - 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, - 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, - 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0x8b, 0x10, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, - 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, - 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, - 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, - 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, - 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, - 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, + 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, + 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, + 0x8b, 0x10, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, + 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, + 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, + 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, + 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, + 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, + 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, + 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, + 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, + 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, + 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, + 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, 0x65, 0x73, - 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, - 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, - 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, - 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, + 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, + 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, + 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, + 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, - 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, - 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, + 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, - 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x8a, 0x01, 0x0a, 0x21, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, - 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, - 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, - 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, - 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, - 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, - 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, + 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, + 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, + 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8a, 0x01, 0x0a, + 0x21, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x12, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, + 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, + 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, + 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, + 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, + 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, + 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, + 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, + 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, + 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, + 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, + 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, + 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, + 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index a10410025..486f30a93 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -658,6 +658,9 @@ message SealWriteCacheRequest { // If true, then writecache will be sealed, but mode will be restored to the current one. bool restore_mode = 4; + + // If true, then writecache will shrink internal storage. + bool shrink = 5; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index b6b064973..417d25c05 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -3423,6 +3423,7 @@ func (x *SealWriteCacheRequest_Body) StableSize() (size int) { size += proto.RepeatedBytesSize(1, x.Shard_ID) size += proto.BoolSize(2, x.IgnoreErrors) size += proto.BoolSize(4, x.RestoreMode) + size += proto.BoolSize(5, x.Shrink) return size } @@ -3445,6 +3446,7 @@ func (x *SealWriteCacheRequest_Body) StableMarshal(buf []byte) []byte { offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) offset += proto.BoolMarshal(2, buf[offset:], x.IgnoreErrors) offset += proto.BoolMarshal(4, buf[offset:], x.RestoreMode) + offset += proto.BoolMarshal(5, buf[offset:], x.Shrink) return buf } From 08b1f18bca97c100831b8ded4d9e6633a5aae138 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Aug 2024 15:09:12 +0300 Subject: [PATCH 1266/1943] [#1296] writecache: Add count limit Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 +++ cmd/frostfs-node/config/engine/config_test.go | 3 +++ .../config/engine/shard/writecache/config.go | 19 ++++++++++++++++++ config/example/node.env | 1 + config/example/node.json | 3 ++- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 19 +++++++++--------- .../writecache/options.go | 10 ++++++++++ pkg/local_object_storage/writecache/put.go | 6 ++---- pkg/local_object_storage/writecache/state.go | 20 +++++++++++++------ 10 files changed, 65 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 5b91e7819..a3dd80684 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -152,6 +152,7 @@ type shardCfg struct { maxObjSize uint64 flushWorkerCount int sizeLimit uint64 + countLimit uint64 noSync bool } @@ -275,6 +276,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc.smallObjectSize = writeCacheCfg.SmallObjectSize() wc.flushWorkerCount = writeCacheCfg.WorkerCount() wc.sizeLimit = writeCacheCfg.SizeLimit() + wc.countLimit = writeCacheCfg.CountLimit() wc.noSync = writeCacheCfg.NoSync() } } @@ -867,6 +869,7 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { writecache.WithSmallObjectSize(wcRead.smallObjectSize), writecache.WithFlushWorkersCount(wcRead.flushWorkerCount), writecache.WithMaxCacheSize(wcRead.sizeLimit), + writecache.WithMaxCacheCount(wcRead.countLimit), writecache.WithNoSync(wcRead.noSync), writecache.WithLogger(c.log), ) diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 7473afefb..da4eeabfd 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -12,6 +12,7 @@ import ( fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" gcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/gc" piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama" + writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" @@ -78,6 +79,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 3221225472, wc.SizeLimit()) + require.EqualValues(t, 49, wc.CountLimit()) require.Equal(t, "tmp/0/meta", meta.Path()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) @@ -133,6 +135,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 4294967296, wc.SizeLimit()) + require.EqualValues(t, writecacheconfig.CountLimitDefault, wc.CountLimit()) require.Equal(t, "tmp/1/meta", meta.Path()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) diff --git a/cmd/frostfs-node/config/engine/shard/writecache/config.go b/cmd/frostfs-node/config/engine/shard/writecache/config.go index 5e31e04ad..bfe8144df 100644 --- a/cmd/frostfs-node/config/engine/shard/writecache/config.go +++ b/cmd/frostfs-node/config/engine/shard/writecache/config.go @@ -21,6 +21,9 @@ const ( // SizeLimitDefault is a default write-cache size limit. SizeLimitDefault = 1 << 30 + + // CountLimitDefault is a default write-cache count limit. + CountLimitDefault = 0 ) // From wraps config section into Config. @@ -115,6 +118,22 @@ func (x *Config) SizeLimit() uint64 { return SizeLimitDefault } +// CountLimit returns the value of "max_object_count" config parameter. +// +// Returns CountLimitDefault if the value is not a positive number. +func (x *Config) CountLimit() uint64 { + c := config.SizeInBytesSafe( + (*config.Config)(x), + "max_object_count", + ) + + if c > 0 { + return c + } + + return CountLimitDefault +} + // NoSync returns the value of "no_sync" config parameter. // // Returns false if the value is not a boolean. diff --git a/config/example/node.env b/config/example/node.env index 00190eb39..fc42cc3bf 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -105,6 +105,7 @@ FROSTFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384 FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728 FROSTFS_STORAGE_SHARD_0_WRITECACHE_FLUSH_WORKER_COUNT=30 FROSTFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472 +FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_COUNT=49 ### Metabase config FROSTFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644 diff --git a/config/example/node.json b/config/example/node.json index 9051d2bb7..9aa0dac53 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -148,7 +148,8 @@ "small_object_size": 16384, "max_object_size": 134217728, "flush_worker_count": 30, - "capacity": 3221225472 + "capacity": 3221225472, + "max_object_count": 49 }, "metabase": { "path": "tmp/0/meta", diff --git a/config/example/node.yaml b/config/example/node.yaml index bcc8552b3..2e6affae9 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -171,6 +171,7 @@ storage: no_sync: true path: tmp/0/cache # write-cache root directory capacity: 3221225472 # approximate write-cache total size, bytes + max_object_count: 49 metabase: path: tmp/0/meta # metabase path diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 4a6e5ba6d..424172ff5 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -292,15 +292,16 @@ writecache: flush_worker_count: 30 ``` -| Parameter | Type | Default value | Description | -|----------------------|------------|---------------|----------------------------------------------------------------------------------------------------------------------| -| `path` | `string` | | Path to the metabase file. | -| `capacity` | `size` | unrestricted | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | -| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | -| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. | -| `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | -| `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | -| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | +| Parameter | Type | Default value | Description | +|----------------------|------------|---------------|-------------------------------------------------------------------------------------------------------------------------------| +| `path` | `string` | | Path to the metabase file. | +| `capacity` | `size` | `1G` | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | +| `max_object_count` | `int` | unrestricted | Approximate maximum objects count in the writecache. If the writecache is full, objects are written to the blobstor directly. | +| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | +| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. | +| `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | +| `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | +| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | # `node` section diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index c8eb1bc45..66cd79bea 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -29,6 +29,9 @@ type options struct { // maxCacheSize is the maximum total size of all objects saved in cache (DB + FS). // 1 GiB by default. maxCacheSize uint64 + // maxCacheCount is the maximum total count of all object saved in cache. + // 0 (no limit) by default. + maxCacheCount uint64 // objCounters contains atomic counters for the number of objects stored in cache. objCounters counters // maxBatchSize is the maximum batch size for the small object database. @@ -108,6 +111,13 @@ func WithMaxCacheSize(sz uint64) Option { } } +// WithMaxCacheCount sets maximum write-cache objects count. +func WithMaxCacheCount(v uint64) Option { + return func(o *options) { + o.maxCacheCount = v + } +} + // WithMaxBatchSize sets max batch size for the small object database. func WithMaxBatchSize(sz int) Option { return func(o *options) { diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 0e419f95b..150399de8 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -76,8 +76,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro // putSmall persists small objects to the write-cache database and // pushes the to the flush workers queue. func (c *cache) putSmall(obj objectInfo) error { - cacheSize := c.estimateCacheSize() - if c.maxCacheSize < c.incSizeDB(cacheSize) { + if !c.hasEnoughSpaceDB() { return ErrOutOfSpace } @@ -107,8 +106,7 @@ func (c *cache) putSmall(obj objectInfo) error { // putBig writes object to FSTree and pushes it to the flush workers queue. func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) error { - cacheSz := c.estimateCacheSize() - if c.maxCacheSize < c.incSizeFS(cacheSz) { + if !c.hasEnoughSpaceFS() { return ErrOutOfSpace } diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index bc75aaf27..d03f4a63e 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -9,7 +9,7 @@ import ( "go.etcd.io/bbolt" ) -func (c *cache) estimateCacheSize() uint64 { +func (c *cache) estimateCacheSize() (uint64, uint64) { dbCount := c.objCounters.DB() fsCount := c.objCounters.FS() if fsCount > 0 { @@ -19,15 +19,23 @@ func (c *cache) estimateCacheSize() uint64 { fsSize := fsCount * c.maxObjectSize c.metrics.SetEstimateSize(dbSize, fsSize) c.metrics.SetActualCounters(dbCount, fsCount) - return dbSize + fsSize + return dbCount + fsCount, dbSize + fsSize } -func (c *cache) incSizeDB(sz uint64) uint64 { - return sz + c.smallObjectSize +func (c *cache) hasEnoughSpaceDB() bool { + return c.hasEnoughSpace(c.smallObjectSize) } -func (c *cache) incSizeFS(sz uint64) uint64 { - return sz + c.maxObjectSize +func (c *cache) hasEnoughSpaceFS() bool { + return c.hasEnoughSpace(c.maxObjectSize) +} + +func (c *cache) hasEnoughSpace(objectSize uint64) bool { + count, size := c.estimateCacheSize() + if c.maxCacheCount > 0 && count+1 > c.maxCacheCount { + return false + } + return c.maxCacheSize >= size+objectSize } var _ fstree.FileCounter = &counters{} From c985b1198f8ed2ec1ecb6e239503ff4ad3f59439 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 9 Aug 2024 10:38:45 +0300 Subject: [PATCH 1267/1943] [#1302] putSvc: Override SuccessAfter for non-regular objects in EC containers Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/single.go | 4 ++++ pkg/services/object/put/streamer.go | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 6d2f3dba8..9fa8ddb67 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -19,6 +19,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -229,6 +230,9 @@ func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumb if len(copiesNumber) > 0 && !result.isEC { result.placementOptions = append(result.placementOptions, placement.WithCopyNumbers(copiesNumber)) } + if container.IsECContainer(cnrInfo.Value) && !object.IsECSupported(obj) && !localOnly { + result.placementOptions = append(result.placementOptions, placement.SuccessAfter(uint32(policy.ECParityCount(cnrInfo.Value.PlacementPolicy())+1))) + } result.placementOptions = append(result.placementOptions, placement.ForContainer(cnrInfo.Value)) objID, ok := obj.ID() diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 14dae38d5..4e655ed54 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" @@ -212,10 +213,10 @@ func (p *Streamer) newObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { if container.IsECContainer(prm.cnr) && object.IsECSupported(prm.hdr) { return p.newECWriter(prm) } - return p.newDefaultObjectWriter(prm) + return p.newDefaultObjectWriter(prm, false) } -func (p *Streamer) newDefaultObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { +func (p *Streamer) newDefaultObjectWriter(prm *PutInitPrm, forECPlacement bool) transformer.ObjectWriter { var relay func(context.Context, nodeDesc) error if p.relay != nil { relay = func(ctx context.Context, node nodeDesc) error { @@ -232,9 +233,16 @@ func (p *Streamer) newDefaultObjectWriter(prm *PutInitPrm) transformer.ObjectWri } } + traverseOpts := prm.traverseOpts + if forECPlacement && !prm.common.LocalOnly() { + // save non-regular and linking object to EC container. + // EC 2.1 -> REP 2, EC 2.2 -> REP 3 etc. + traverseOpts = append(traverseOpts, placement.SuccessAfter(uint32(policy.ECParityCount(prm.cnr.PlacementPolicy())+1))) + } + return &distributedTarget{ cfg: p.cfg, - placementOpts: prm.traverseOpts, + placementOpts: traverseOpts, nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { return localTarget{ @@ -266,7 +274,7 @@ func (p *Streamer) newECWriter(prm *PutInitPrm) transformer.ObjectWriter { commonPrm: prm.common, relay: p.relay, }, - repWriter: p.newDefaultObjectWriter(prm), + repWriter: p.newDefaultObjectWriter(prm, true), } } From fa82854af4c0ec18a058eabede3da35715061125 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 8 Aug 2024 13:15:22 +0300 Subject: [PATCH 1268/1943] [#1302] writecache: Add put->flush->put benchmark Signed-off-by: Dmitrii Stepanov --- .../writecache/benchmark/writecache_test.go | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index c1c0e88b3..4f4398452 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -2,6 +2,7 @@ package benchmark import ( "context" + "fmt" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -27,6 +28,24 @@ func BenchmarkWritecachePar(b *testing.B) { }) } +func BenchmarkWriteAfterDelete(b *testing.B) { + const payloadSize = 32 << 10 + const parallel = 25 + + cache := newCache(b) + benchmarkPutPrepare(b, cache) + b.Run(fmt.Sprintf("%dB_before", payloadSize), func(b *testing.B) { + b.SetParallelism(parallel) + benchmarkRunPar(b, cache, payloadSize) + }) + require.NoError(b, cache.Flush(context.Background(), false, false)) + b.Run(fmt.Sprintf("%dB_after", payloadSize), func(b *testing.B) { + b.SetParallelism(parallel) + benchmarkRunPar(b, cache, payloadSize) + }) + require.NoError(b, cache.Close()) +} + func benchmarkPutSeq(b *testing.B, cache writecache.Cache, size uint64) { benchmarkPutPrepare(b, cache) defer func() { require.NoError(b, cache.Close()) }() @@ -54,6 +73,10 @@ func benchmarkPutPar(b *testing.B, cache writecache.Cache, size uint64) { benchmarkPutPrepare(b, cache) defer func() { require.NoError(b, cache.Close()) }() + benchmarkRunPar(b, cache, size) +} + +func benchmarkRunPar(b *testing.B, cache writecache.Cache, size uint64) { ctx := context.Background() b.ResetTimer() From 68029d756e59d036b11b95153d70e64588ee720e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 8 Aug 2024 13:32:18 +0300 Subject: [PATCH 1269/1943] [#1302] writecache: Allow to specify custom page size Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-lens/internal/writecache/inspect.go | 2 +- cmd/frostfs-lens/internal/writecache/list.go | 2 +- cmd/frostfs-node/config.go | 3 +++ cmd/frostfs-node/config/engine/config_test.go | 2 ++ cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go | 11 +++++++++++ config/example/node.env | 1 + config/example/node.json | 1 + config/example/node.yaml | 1 + docs/storage-node-configuration.md | 2 ++ pkg/local_object_storage/writecache/options.go | 9 +++++++++ pkg/local_object_storage/writecache/storage.go | 2 +- pkg/local_object_storage/writecache/util.go | 3 ++- 12 files changed, 35 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-lens/internal/writecache/inspect.go b/cmd/frostfs-lens/internal/writecache/inspect.go index afc986c8b..63c669a35 100644 --- a/cmd/frostfs-lens/internal/writecache/inspect.go +++ b/cmd/frostfs-lens/internal/writecache/inspect.go @@ -25,7 +25,7 @@ func init() { func inspectFunc(cmd *cobra.Command, _ []string) { var data []byte - db, err := writecache.OpenDB(vPath, true, os.OpenFile) + db, err := writecache.OpenDB(vPath, true, os.OpenFile, 0) common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) defer db.Close() diff --git a/cmd/frostfs-lens/internal/writecache/list.go b/cmd/frostfs-lens/internal/writecache/list.go index bcbae0ec9..9c8fa6138 100644 --- a/cmd/frostfs-lens/internal/writecache/list.go +++ b/cmd/frostfs-lens/internal/writecache/list.go @@ -31,7 +31,7 @@ func listFunc(cmd *cobra.Command, _ []string) { return err } - db, err := writecache.OpenDB(vPath, true, os.OpenFile) + db, err := writecache.OpenDB(vPath, true, os.OpenFile, 0) common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) defer db.Close() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index a3dd80684..1af27d733 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -154,6 +154,7 @@ type shardCfg struct { sizeLimit uint64 countLimit uint64 noSync bool + pageSize int } piloramaCfg struct { @@ -272,6 +273,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc.path = writeCacheCfg.Path() wc.maxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize() wc.maxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay() + wc.pageSize = writeCacheCfg.BoltDB().PageSize() wc.maxObjSize = writeCacheCfg.MaxObjectSize() wc.smallObjectSize = writeCacheCfg.SmallObjectSize() wc.flushWorkerCount = writeCacheCfg.WorkerCount() @@ -865,6 +867,7 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { writecache.WithPath(wcRead.path), writecache.WithMaxBatchSize(wcRead.maxBatchSize), writecache.WithMaxBatchDelay(wcRead.maxBatchDelay), + writecache.WithPageSize(wcRead.pageSize), writecache.WithMaxObjectSize(wcRead.maxObjSize), writecache.WithSmallObjectSize(wcRead.smallObjectSize), writecache.WithFlushWorkersCount(wcRead.flushWorkerCount), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index da4eeabfd..d53207ccc 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -79,6 +79,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 3221225472, wc.SizeLimit()) + require.EqualValues(t, 4096, wc.BoltDB().PageSize()) require.EqualValues(t, 49, wc.CountLimit()) require.Equal(t, "tmp/0/meta", meta.Path()) @@ -135,6 +136,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 4294967296, wc.SizeLimit()) + require.EqualValues(t, 0, wc.BoltDB().PageSize()) require.EqualValues(t, writecacheconfig.CountLimitDefault, wc.CountLimit()) require.Equal(t, "tmp/1/meta", meta.Path()) diff --git a/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go b/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go index 9e334cd8f..a51308b5b 100644 --- a/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go +++ b/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go @@ -60,3 +60,14 @@ func (x *Config) MaxBatchSize() int { func (x *Config) NoSync() bool { return config.BoolSafe((*config.Config)(x), "no_sync") } + +// PageSize returns the value of "page_size" config parameter. +// +// Returns 0 if the value is not a positive number. +func (x *Config) PageSize() int { + s := int(config.SizeInBytesSafe((*config.Config)(x), "page_size")) + if s < 0 { + s = 0 + } + return s +} diff --git a/config/example/node.env b/config/example/node.env index fc42cc3bf..b39423ffb 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -105,6 +105,7 @@ FROSTFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384 FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728 FROSTFS_STORAGE_SHARD_0_WRITECACHE_FLUSH_WORKER_COUNT=30 FROSTFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472 +FROSTFS_STORAGE_SHARD_0_WRITECACHE_PAGE_SIZE=4096 FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_COUNT=49 ### Metabase config FROSTFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta diff --git a/config/example/node.json b/config/example/node.json index 9aa0dac53..fe2de0e01 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -149,6 +149,7 @@ "max_object_size": 134217728, "flush_worker_count": 30, "capacity": 3221225472, + "page_size": 4096, "max_object_count": 49 }, "metabase": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 2e6affae9..cc339a427 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -172,6 +172,7 @@ storage: path: tmp/0/cache # write-cache root directory capacity: 3221225472 # approximate write-cache total size, bytes max_object_count: 49 + page_size: 4k metabase: path: tmp/0/meta # metabase path diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 424172ff5..5bf35cd65 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -290,6 +290,7 @@ writecache: small_object_size: 16384 max_object_size: 134217728 flush_worker_count: 30 + page_size: '4k' ``` | Parameter | Type | Default value | Description | @@ -302,6 +303,7 @@ writecache: | `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | | `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | | `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | +| `page_size` | `size` | `0` | Page size overrides the default OS page size for small objects storage. Does not affect the existing storage. | # `node` section diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index 66cd79bea..980cf9303 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -48,6 +48,8 @@ type options struct { metrics Metrics // disableBackgroundFlush is for testing purposes only. disableBackgroundFlush bool + // pageSize is bbolt's page size config value + pageSize int } // WithLogger sets logger. @@ -173,3 +175,10 @@ func WithDisableBackgroundFlush() Option { o.disableBackgroundFlush = true } } + +// WithPageSize sets bbolt's page size. +func WithPageSize(s int) Option { + return func(o *options) { + o.pageSize = s + } +} diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index caf997af8..57021cc17 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -32,7 +32,7 @@ func (c *cache) openStore(mod mode.ComponentMode) error { return err } - c.db, err = OpenDB(c.path, mod.ReadOnly(), c.openFile) + c.db, err = OpenDB(c.path, mod.ReadOnly(), c.openFile, c.pageSize) if err != nil { return fmt.Errorf("could not open database: %w", err) } diff --git a/pkg/local_object_storage/writecache/util.go b/pkg/local_object_storage/writecache/util.go index 0ed4a954e..ad3b443f3 100644 --- a/pkg/local_object_storage/writecache/util.go +++ b/pkg/local_object_storage/writecache/util.go @@ -10,11 +10,12 @@ import ( ) // OpenDB opens BoltDB instance for write-cache. Opens in read-only mode if ro is true. -func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error)) (*bbolt.DB, error) { +func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error), pageSize int) (*bbolt.DB, error) { return bbolt.Open(filepath.Join(p, dbName), os.ModePerm, &bbolt.Options{ NoFreelistSync: true, ReadOnly: ro, Timeout: 100 * time.Millisecond, OpenFile: openFile, + PageSize: pageSize, }) } From 93d63e1632f54869ca1c155cdd11d2c9966c3090 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 Jul 2024 16:30:07 +0300 Subject: [PATCH 1270/1943] [#1284] writecache: Allow to seal writecache async Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/writecache.go | 4 + internal/logs/logs.go | 3 + pkg/local_object_storage/engine/writecache.go | 3 +- pkg/local_object_storage/shard/control.go | 6 + pkg/local_object_storage/shard/shard.go | 6 +- pkg/local_object_storage/shard/writecache.go | 54 ++- .../control/server/seal_writecache.go | 1 + pkg/services/control/service.pb.go | 340 +++++++++--------- pkg/services/control/service.proto | 3 + pkg/services/control/service_frostfs.pb.go | 2 + 10 files changed, 252 insertions(+), 170 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/writecache.go b/cmd/frostfs-cli/modules/control/writecache.go index b725d8471..ffe9009ab 100644 --- a/cmd/frostfs-cli/modules/control/writecache.go +++ b/cmd/frostfs-cli/modules/control/writecache.go @@ -10,6 +10,7 @@ import ( ) const ( + asyncFlag = "async" restoreModeFlag = "restore-mode" shrinkFlag = "shrink" ) @@ -31,12 +32,14 @@ func sealWritecache(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag) + async, _ := cmd.Flags().GetBool(asyncFlag) restoreMode, _ := cmd.Flags().GetBool(restoreModeFlag) shrink, _ := cmd.Flags().GetBool(shrinkFlag) req := &control.SealWriteCacheRequest{Body: &control.SealWriteCacheRequest_Body{ Shard_ID: getShardIDList(cmd), IgnoreErrors: ignoreErrors, + Async: async, RestoreMode: restoreMode, Shrink: shrink, }} @@ -77,6 +80,7 @@ func initControlShardsWritecacheCmd() { ff.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") ff.Bool(shardAllFlag, false, "Process all shards") ff.Bool(ignoreErrorsFlag, true, "Skip invalid/unreadable objects") + ff.Bool(asyncFlag, false, "Run operation in background") ff.Bool(restoreModeFlag, false, "Restore writecache's mode after sealing") ff.Bool(shrinkFlag, false, "Shrink writecache's internal storage") diff --git a/internal/logs/logs.go b/internal/logs/logs.go index ebb822e1c..78bcd0c0e 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -539,5 +539,8 @@ const ( PolicerCouldNotGetChunk = "could not get EC chunk" PolicerCouldNotGetChunks = "could not get EC chunks" AuditEventLogRecord = "audit event log record" + StartedWritecacheSealAsync = "started writecache seal async" + WritecacheSealCompletedAsync = "writecache seal completed successfully" + FailedToSealWritecacheAsync = "failed to seal writecache async" WritecacheShrinkSkippedNotEmpty = "writecache shrink skipped: database is not empty" ) diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 2c5e8cc3a..3e8f387ef 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -70,6 +70,7 @@ func (e *StorageEngine) FlushWriteCache(ctx context.Context, p FlushWriteCachePr type SealWriteCachePrm struct { ShardIDs []*shard.ID IgnoreErrors bool + Async bool RestoreMode bool Shrink bool } @@ -117,7 +118,7 @@ func (e *StorageEngine) SealWriteCache(ctx context.Context, prm SealWriteCachePr return nil } - err := sh.SealWriteCache(egCtx, shard.SealWriteCachePrm{IgnoreErrors: prm.IgnoreErrors, RestoreMode: prm.RestoreMode, Shrink: prm.Shrink}) + err := sh.SealWriteCache(egCtx, shard.SealWriteCachePrm{IgnoreErrors: prm.IgnoreErrors, Async: prm.Async, RestoreMode: prm.RestoreMode, Shrink: prm.Shrink}) resGuard.Lock() defer resGuard.Unlock() diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 90d7afdd4..210744702 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -178,6 +178,7 @@ func (s *Shard) Init(ctx context.Context) error { if !m.NoMetabase() { s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) } + s.writecacheSealCancel.Store(dummyCancel) return nil } @@ -350,6 +351,8 @@ func (s *Shard) Close() error { } if s.hasWriteCache() { + prev := s.writecacheSealCancel.Swap(notInitializedCancel) + prev.cancel() // no need to wait: writecache.Seal and writecache.Close lock the same mutex components = append(components, s.writeCache) } @@ -428,6 +431,9 @@ func (s *Shard) lockExclusive() func() { cancelGC := val.(context.CancelFunc) cancelGC() } + if c := s.writecacheSealCancel.Load(); c != nil { + c.cancel() + } s.m.Lock() s.setModeRequested.Store(false) return s.m.Unlock diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 94f22feb5..93f5354a7 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -37,8 +37,9 @@ type Shard struct { rb *rebuilder - gcCancel atomic.Value - setModeRequested atomic.Bool + gcCancel atomic.Value + setModeRequested atomic.Bool + writecacheSealCancel atomic.Pointer[writecacheSealCanceler] } // Option represents Shard's constructor option. @@ -190,6 +191,7 @@ func New(opts ...Option) *Shard { } s.fillInfo() + s.writecacheSealCancel.Store(notInitializedCancel) return s } diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index c29710930..a6de07f03 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -4,12 +4,24 @@ import ( "context" "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" ) +var ( + dummyCancel = &writecacheSealCanceler{cancel: func() {}} + notInitializedCancel = &writecacheSealCanceler{cancel: func() {}} + errWriteCacheSealing = errors.New("writecache is already sealing or shard is not initialized") +) + +type writecacheSealCanceler struct { + cancel context.CancelFunc +} + // FlushWriteCachePrm represents parameters of a `FlushWriteCache` operation. type FlushWriteCachePrm struct { ignoreErrors bool @@ -60,6 +72,7 @@ func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error type SealWriteCachePrm struct { IgnoreErrors bool + Async bool RestoreMode bool Shrink bool } @@ -78,15 +91,52 @@ func (s *Shard) SealWriteCache(ctx context.Context, p SealWriteCachePrm) error { return errWriteCacheDisabled } + if p.Async { + ctx = context.WithoutCancel(ctx) + } + ctx, cancel := context.WithCancel(ctx) + canceler := &writecacheSealCanceler{cancel: cancel} + if !s.writecacheSealCancel.CompareAndSwap(dummyCancel, canceler) { + return errWriteCacheSealing + } s.m.RLock() - defer s.m.RUnlock() + cleanup := func() { + s.m.RUnlock() + s.writecacheSealCancel.Store(dummyCancel) + } if s.info.Mode.ReadOnly() { + cleanup() return ErrReadOnlyMode } if s.info.Mode.NoMetabase() { + cleanup() return ErrDegradedMode } - return s.writeCache.Seal(ctx, writecache.SealPrm{IgnoreErrors: p.IgnoreErrors, RestoreMode: p.RestoreMode, Shrink: p.Shrink}) + if !p.Async { + defer cleanup() + } + prm := writecache.SealPrm{IgnoreErrors: p.IgnoreErrors, RestoreMode: p.RestoreMode, Shrink: p.Shrink} + if p.Async { + started := make(chan struct{}) + go func() { + close(started) + defer cleanup() + + s.log.Info(logs.StartedWritecacheSealAsync) + if err := s.writeCache.Seal(ctx, prm); err != nil { + s.log.Warn(logs.FailedToSealWritecacheAsync, zap.Error(err)) + return + } + s.log.Info(logs.WritecacheSealCompletedAsync) + }() + select { + case <-ctx.Done(): + return ctx.Err() + case <-started: + return nil + } + } + return s.writeCache.Seal(ctx, prm) } diff --git a/pkg/services/control/server/seal_writecache.go b/pkg/services/control/server/seal_writecache.go index 697b91918..1737677b7 100644 --- a/pkg/services/control/server/seal_writecache.go +++ b/pkg/services/control/server/seal_writecache.go @@ -19,6 +19,7 @@ func (s *Server) SealWriteCache(ctx context.Context, req *control.SealWriteCache prm := engine.SealWriteCachePrm{ ShardIDs: s.getShardIDList(req.GetBody().GetShard_ID()), IgnoreErrors: req.GetBody().GetIgnoreErrors(), + Async: req.GetBody().GetAsync(), RestoreMode: req.GetBody().GetRestoreMode(), Shrink: req.GetBody().GetShrink(), } diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index ac512f1a5..e5a5ce24c 100644 --- a/pkg/services/control/service.pb.go +++ b/pkg/services/control/service.pb.go @@ -4727,6 +4727,8 @@ type SealWriteCacheRequest_Body struct { Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` // Flag indicating whether object read errors should be ignored. IgnoreErrors bool `protobuf:"varint,2,opt,name=ignore_errors,json=ignoreErrors,proto3" json:"ignore_errors,omitempty"` + // Flag indicating whether writecache will be sealed async. + Async bool `protobuf:"varint,3,opt,name=async,proto3" json:"async,omitempty"` // If true, then writecache will be sealed, but mode will be restored to the current one. RestoreMode bool `protobuf:"varint,4,opt,name=restore_mode,json=restoreMode,proto3" json:"restore_mode,omitempty"` // If true, then writecache will shrink internal storage. @@ -4779,6 +4781,13 @@ func (x *SealWriteCacheRequest_Body) GetIgnoreErrors() bool { return false } +func (x *SealWriteCacheRequest_Body) GetAsync() bool { + if x != nil { + return x.Async + } + return false +} + func (x *SealWriteCacheRequest_Body) GetRestoreMode() bool { if x != nil { return x.RestoreMode @@ -5500,7 +5509,7 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x86, 0x02, 0x0a, 0x15, + 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x9c, 0x02, 0x0a, 0x15, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, @@ -5509,186 +5518,187 @@ var file_pkg_services_control_service_proto_rawDesc = []byte{ 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x81, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, + 0x1a, 0x97, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, - 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x0b, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, - 0x73, 0x68, 0x72, 0x69, 0x6e, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x68, - 0x72, 0x69, 0x6e, 0x6b, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, - 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, - 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, - 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, - 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, + 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x73, 0x79, + 0x6e, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x12, + 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x4d, 0x6f, + 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, 0x72, 0x69, 0x6e, 0x6b, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x06, 0x73, 0x68, 0x72, 0x69, 0x6e, 0x6b, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, + 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, + 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, + 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, + 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, + 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, + 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, + 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, - 0x8b, 0x10, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, - 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, - 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, - 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, - 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, + 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, + 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, + 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, + 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0x8b, 0x10, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, - 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, - 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, - 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, - 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, - 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, - 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, - 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, + 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4e, + 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, + 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, + 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, + 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, + 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, + 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, + 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, + 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, + 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, + 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, 0x65, 0x73, 0x65, + 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, + 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, + 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, + 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, + 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, + 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, + 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, + 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, + 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, + 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, + 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, + 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, + 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, + 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, - 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, - 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, - 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, - 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, - 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, - 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8a, 0x01, 0x0a, - 0x21, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x12, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, - 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, - 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, - 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, - 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, - 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, - 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, - 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, - 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, - 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, - 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x8a, 0x01, 0x0a, 0x21, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, + 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, + 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x63, 0x6f, + 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, + 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, + 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, + 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, + 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, + 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, + 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, + 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, + 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, + 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 486f30a93..d6639cb48 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -656,6 +656,9 @@ message SealWriteCacheRequest { // Flag indicating whether object read errors should be ignored. bool ignore_errors = 2; + // Flag indicating whether writecache will be sealed async. + bool async = 3; + // If true, then writecache will be sealed, but mode will be restored to the current one. bool restore_mode = 4; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 417d25c05..822244e77 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -3422,6 +3422,7 @@ func (x *SealWriteCacheRequest_Body) StableSize() (size int) { } size += proto.RepeatedBytesSize(1, x.Shard_ID) size += proto.BoolSize(2, x.IgnoreErrors) + size += proto.BoolSize(3, x.Async) size += proto.BoolSize(4, x.RestoreMode) size += proto.BoolSize(5, x.Shrink) return size @@ -3445,6 +3446,7 @@ func (x *SealWriteCacheRequest_Body) StableMarshal(buf []byte) []byte { var offset int offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) offset += proto.BoolMarshal(2, buf[offset:], x.IgnoreErrors) + offset += proto.BoolMarshal(3, buf[offset:], x.Async) offset += proto.BoolMarshal(4, buf[offset:], x.RestoreMode) offset += proto.BoolMarshal(5, buf[offset:], x.Shrink) return buf From 80ce7c3a0095e707e44accf30be6bf7f61bfacc4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 31 Jul 2024 16:33:31 +0300 Subject: [PATCH 1271/1943] [#1284] shard: Resolve funlen linter Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/control.go | 71 +++++++++++++---------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 210744702..936a506c0 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -99,12 +99,50 @@ func (x *metabaseSynchronizer) Init() error { // Init initializes all Shard's components. func (s *Shard) Init(ctx context.Context) error { + m := s.GetMode() + if err := s.initializeComponents(m); err != nil { + return err + } + + s.updateMetrics(ctx) + + s.gc = &gc{ + gcCfg: &s.gcCfg, + remover: s.removeGarbage, + stopChannel: make(chan struct{}), + eventChan: make(chan Event), + mEventHandler: map[eventType]*eventHandlers{ + eventNewEpoch: { + cancelFunc: func() {}, + handlers: []eventHandler{ + s.collectExpiredLocks, + s.collectExpiredObjects, + s.collectExpiredTombstones, + s.collectExpiredMetrics, + }, + }, + }, + } + if s.gc.metrics != nil { + s.gc.metrics.SetShardID(s.info.ID.String()) + } + + s.gc.init(ctx) + + s.rb = newRebuilder(s.rebuildLimiter) + if !m.NoMetabase() { + s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) + } + s.writecacheSealCancel.Store(dummyCancel) + return nil +} + +func (s *Shard) initializeComponents(m mode.Mode) error { type initializer interface { Init() error } var components []initializer - m := s.GetMode() if !m.NoMetabase() { var initMetabase initializer @@ -148,37 +186,6 @@ func (s *Shard) Init(ctx context.Context) error { return fmt.Errorf("could not initialize %T: %w", component, err) } } - - s.updateMetrics(ctx) - - s.gc = &gc{ - gcCfg: &s.gcCfg, - remover: s.removeGarbage, - stopChannel: make(chan struct{}), - eventChan: make(chan Event), - mEventHandler: map[eventType]*eventHandlers{ - eventNewEpoch: { - cancelFunc: func() {}, - handlers: []eventHandler{ - s.collectExpiredLocks, - s.collectExpiredObjects, - s.collectExpiredTombstones, - s.collectExpiredMetrics, - }, - }, - }, - } - if s.gc.metrics != nil { - s.gc.metrics.SetShardID(s.info.ID.String()) - } - - s.gc.init(ctx) - - s.rb = newRebuilder(s.rebuildLimiter) - if !m.NoMetabase() { - s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) - } - s.writecacheSealCancel.Store(dummyCancel) return nil } From d055168e2a9651c357e4358c1dcc392bb4ae4718 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Wed, 14 Aug 2024 13:47:50 +0300 Subject: [PATCH 1272/1943] [#1135] ir: Add healthstatus RECONFIGURING Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-ir/config.go | 6 ++++++ go.mod | 6 +++--- go.sum | 12 ++++++------ pkg/innerring/state.go | 12 ++++++++++++ pkg/services/control/ir/types.pb.go | 18 ++++++++++++------ pkg/services/control/ir/types.proto | 3 +++ 6 files changed, 42 insertions(+), 15 deletions(-) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 955195477..4eaac845c 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -7,6 +7,7 @@ import ( configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -81,6 +82,10 @@ func watchForSignal(cancel func()) { return case <-sighupCh: log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) + if !innerRing.CompareAndSwapHealthStatus(control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) { + log.Info(logs.FrostFSNodeSIGHUPSkip) + break + } err := reloadConfig() if err != nil { log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) @@ -92,6 +97,7 @@ func watchForSignal(cancel func()) { if err != nil { log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) } + innerRing.CompareAndSwapHealthStatus(control.HealthStatus_RECONFIGURING, control.HealthStatus_READY) log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } } diff --git a/go.mod b/go.mod index 09a098502..196b4d463 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect @@ -120,8 +120,8 @@ require ( golang.org/x/crypto v0.21.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 1034ff61f..bd6d85882 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJR github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -380,10 +380,10 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c h1:lfpJ/2rWPa/kJgxyyXM8PrNnfCzcmxJ265mADgwmvLI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 1616dbb9f..d3071faad 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -161,6 +161,16 @@ func (s *Server) setHealthStatus(hs control.HealthStatus) { } } +func (s *Server) CompareAndSwapHealthStatus(oldSt, newSt control.HealthStatus) (swapped bool) { + if swapped = s.healthStatus.CompareAndSwap(int32(oldSt), int32(newSt)); swapped { + s.notifySystemd(newSt) + if s.irMetrics != nil { + s.irMetrics.SetHealth(int32(newSt)) + } + } + return +} + // HealthStatus returns the current health status of the IR application. func (s *Server) HealthStatus() control.HealthStatus { return control.HealthStatus(s.healthStatus.Load()) @@ -186,6 +196,8 @@ func (s *Server) notifySystemd(st control.HealthStatus) { err = sdnotify.FlagAndStatus(sdnotify.ReadyEnabled) case control.HealthStatus_SHUTTING_DOWN: err = sdnotify.FlagAndStatus(sdnotify.StoppingEnabled) + case control.HealthStatus_RECONFIGURING: + err = sdnotify.FlagAndStatus(sdnotify.ReloadingEnabled) default: err = sdnotify.Status(fmt.Sprintf("%v", st)) } diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go index 828814b25..840e0be67 100644 --- a/pkg/services/control/ir/types.pb.go +++ b/pkg/services/control/ir/types.pb.go @@ -32,6 +32,8 @@ const ( HealthStatus_READY HealthStatus = 2 // IR application is shutting down. HealthStatus_SHUTTING_DOWN HealthStatus = 3 + // IR application is reconfiguring. + HealthStatus_RECONFIGURING HealthStatus = 4 ) // Enum value maps for HealthStatus. @@ -41,12 +43,14 @@ var ( 1: "STARTING", 2: "READY", 3: "SHUTTING_DOWN", + 4: "RECONFIGURING", } HealthStatus_value = map[string]int32{ "HEALTH_STATUS_UNDEFINED": 0, "STARTING": 1, "READY": 2, "SHUTTING_DOWN": 3, + "RECONFIGURING": 4, } ) @@ -144,17 +148,19 @@ var file_pkg_services_control_ir_types_proto_rawDesc = []byte{ 0x22, 0x36, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x17, 0x0a, 0x04, 0x73, 0x69, 0x67, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x2a, 0x57, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x2a, 0x6a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x17, 0x48, 0x45, 0x41, 0x4c, 0x54, 0x48, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x48, 0x55, 0x54, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, - 0x03, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, - 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, - 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x69, 0x72, 0x2f, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x55, 0x52, 0x49, + 0x4e, 0x47, 0x10, 0x04, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, + 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, + 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, + 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, + 0x69, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/pkg/services/control/ir/types.proto b/pkg/services/control/ir/types.proto index 9b6731cf8..901a55918 100644 --- a/pkg/services/control/ir/types.proto +++ b/pkg/services/control/ir/types.proto @@ -26,4 +26,7 @@ enum HealthStatus { // IR application is shutting down. SHUTTING_DOWN = 3; + + // IR application is reconfiguring. + RECONFIGURING = 4; } From 5da41f1fe55f6bb906f0e05d2539ccf9a21cdfa6 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Wed, 14 Aug 2024 13:55:34 +0300 Subject: [PATCH 1273/1943] Revert "[#1262] sdnotify: Get rid of go:linkname for nanotime" This reverts commit 327d364f34de730879f330ea51d8801f5c6bddc9. Reverted due to the problem with reload signal sent by systemd. `frostfs-ir` service reconfigures correctly and service's statuses are being reported to systemd. However, since we replaced `go:linkname` & `nanotime()` with `time.Since()`, systemd refuses to accept reload signal response from `frostfs-ir`. To maintain correct behaviour it was decided to revevrt systemd-related changes until a better solution is found. Signed-off-by: Ekaterina Lebedeva --- pkg/util/sdnotify/clock.go | 10 ++++++++++ pkg/util/sdnotify/clock.s | 2 ++ pkg/util/sdnotify/sdnotify.go | 4 +--- 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 pkg/util/sdnotify/clock.go create mode 100644 pkg/util/sdnotify/clock.s diff --git a/pkg/util/sdnotify/clock.go b/pkg/util/sdnotify/clock.go new file mode 100644 index 000000000..f5419d027 --- /dev/null +++ b/pkg/util/sdnotify/clock.go @@ -0,0 +1,10 @@ +package sdnotify + +import ( + // For go:linkname to work. + _ "unsafe" +) + +//go:noescape +//go:linkname nanotime runtime.nanotime +func nanotime() int64 diff --git a/pkg/util/sdnotify/clock.s b/pkg/util/sdnotify/clock.s new file mode 100644 index 000000000..ad033ff4f --- /dev/null +++ b/pkg/util/sdnotify/clock.s @@ -0,0 +1,2 @@ +// The file is intentionally empty. +// It is a workaround for https://github.com/golang/go/issues/15006 diff --git a/pkg/util/sdnotify/sdnotify.go b/pkg/util/sdnotify/sdnotify.go index a3af50b22..16a3f11c1 100644 --- a/pkg/util/sdnotify/sdnotify.go +++ b/pkg/util/sdnotify/sdnotify.go @@ -6,7 +6,6 @@ import ( "net" "os" "strings" - "time" ) const ( @@ -17,7 +16,6 @@ const ( var ( socket *net.UnixAddr - start = time.Now() errSocketVariableIsNotPresent = errors.New("\"NOTIFY_SOCKET\" environment variable is not present") errSocketIsNotInitialized = errors.New("socket is not initialized") @@ -53,7 +51,7 @@ func FlagAndStatus(status string) error { // must be sent, containing "READY=1". // // For MONOTONIC_USEC format refer to https://www.man7.org/linux/man-pages/man3/sd_notify.3.html - status += fmt.Sprintf("\nMONOTONIC_USEC=%d", uint64(time.Since(start))/1e3 /* microseconds in nanoseconds */) + status += fmt.Sprintf("\nMONOTONIC_USEC=%d", uint64(nanotime())/1e3 /* microseconds in nanoseconds */) } status += "\nSTATUS=" + strings.TrimSuffix(status, "=1") return Send(status) From ec1509de4ebbccbade51115c434996cd1fd9be7e Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Thu, 15 Aug 2024 17:12:38 +0300 Subject: [PATCH 1274/1943] [#1262] sdnotify: Send system monotonic time on reload The synchronized service reload protocol added in systemd version 253 requires that the service provides a MONOTONIC_USEC field alongside the RELOADING=1 notification message for synchronization purposes. The value carried in this field must be the system CLOCK_MONOTONIC timestamp at the time the notification message was generated as systemd compares it to other CLOCK_MONOTONIC timestamps taken by pid1. Signed-off-by: Ekaterina Lebedeva --- pkg/util/sdnotify/clock.go | 10 ---------- pkg/util/sdnotify/clock.s | 2 -- pkg/util/sdnotify/sdnotify.go | 9 ++++++++- 3 files changed, 8 insertions(+), 13 deletions(-) delete mode 100644 pkg/util/sdnotify/clock.go delete mode 100644 pkg/util/sdnotify/clock.s diff --git a/pkg/util/sdnotify/clock.go b/pkg/util/sdnotify/clock.go deleted file mode 100644 index f5419d027..000000000 --- a/pkg/util/sdnotify/clock.go +++ /dev/null @@ -1,10 +0,0 @@ -package sdnotify - -import ( - // For go:linkname to work. - _ "unsafe" -) - -//go:noescape -//go:linkname nanotime runtime.nanotime -func nanotime() int64 diff --git a/pkg/util/sdnotify/clock.s b/pkg/util/sdnotify/clock.s deleted file mode 100644 index ad033ff4f..000000000 --- a/pkg/util/sdnotify/clock.s +++ /dev/null @@ -1,2 +0,0 @@ -// The file is intentionally empty. -// It is a workaround for https://github.com/golang/go/issues/15006 diff --git a/pkg/util/sdnotify/sdnotify.go b/pkg/util/sdnotify/sdnotify.go index 16a3f11c1..5235315cc 100644 --- a/pkg/util/sdnotify/sdnotify.go +++ b/pkg/util/sdnotify/sdnotify.go @@ -5,7 +5,10 @@ import ( "fmt" "net" "os" + "strconv" "strings" + + "golang.org/x/sys/unix" ) const ( @@ -51,7 +54,11 @@ func FlagAndStatus(status string) error { // must be sent, containing "READY=1". // // For MONOTONIC_USEC format refer to https://www.man7.org/linux/man-pages/man3/sd_notify.3.html - status += fmt.Sprintf("\nMONOTONIC_USEC=%d", uint64(nanotime())/1e3 /* microseconds in nanoseconds */) + var ts unix.Timespec + if err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &ts); err != nil { + return fmt.Errorf("clock_gettime: %w", err) + } + status += "\nMONOTONIC_USEC=" + strconv.FormatInt(ts.Nano()/1000, 10) } status += "\nSTATUS=" + strings.TrimSuffix(status, "=1") return Send(status) From a4a1c3f18ba75ba73de0fbc97bbccacf68625ee5 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 12 Aug 2024 13:01:57 +0300 Subject: [PATCH 1275/1943] [#1307] go.mod: Bump frostfs-sdk-go/frostfs-api-go/v2 versions * Also, resolve dependencies and conflicts for object service by creating stub for `Patch` method. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/object.go | 18 ++++++- go.mod | 6 +-- go.sum | 12 ++--- pkg/network/transport/object/grpc/service.go | 42 +++++++++++++++ pkg/services/object/acl/v2/service.go | 20 +++++++ pkg/services/object/ape/service.go | 20 +++++++ pkg/services/object/audit.go | 56 ++++++++++++++++++++ pkg/services/object/common.go | 8 +++ pkg/services/object/metrics.go | 37 +++++++++++++ pkg/services/object/patch/service.go | 22 ++++++++ pkg/services/object/patch/streamer.go | 28 ++++++++++ pkg/services/object/response.go | 34 ++++++++++++ pkg/services/object/server.go | 7 +++ pkg/services/object/sign.go | 42 +++++++++++++++ pkg/services/object/transport_splitter.go | 4 ++ 15 files changed, 345 insertions(+), 11 deletions(-) create mode 100644 pkg/services/object/patch/service.go create mode 100644 pkg/services/object/patch/streamer.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 0124bf772..eef142415 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -28,6 +28,7 @@ import ( deletesvcV2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/delete/v2" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" getsvcV2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get/v2" + patchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/patch" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" putsvcV2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put/v2" searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" @@ -54,6 +55,8 @@ type objectSvc struct { get *getsvcV2.Service delete *deletesvcV2.Service + + patch *patchsvc.Service } func (c *cfg) MaxObjectSize() uint64 { @@ -71,6 +74,10 @@ func (s *objectSvc) Put() (objectService.PutObjectStream, error) { return s.put.Put() } +func (s *objectSvc) Patch() (objectService.PatchObjectstream, error) { + return s.patch.Patch() +} + func (s *objectSvc) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { return s.put.PutSingle(ctx, req) } @@ -181,10 +188,12 @@ func initObjectService(c *cfg) { sDeleteV2 := createDeleteServiceV2(sDelete) + sPatch := createPatchSvc(sGet, sPut, keyStorage) + // build service pipeline // grpc | audit | | signature | response | acl | ape | split - splitSvc := createSplitService(c, sPutV2, sGetV2, sSearchV2, sDeleteV2) + splitSvc := createSplitService(c, sPutV2, sGetV2, sSearchV2, sDeleteV2, sPatch) apeSvc := createAPEService(c, splitSvc) @@ -353,6 +362,10 @@ func createPutSvcV2(sPut *putsvc.Service, keyStorage *util.KeyStorage) *putsvcV2 return putsvcV2.NewService(sPut, keyStorage) } +func createPatchSvc(sGet *getsvc.Service, sPut *putsvc.Service, keyStorage *util.KeyStorage) *patchsvc.Service { + return patchsvc.NewService(keyStorage, sGet, sPut) +} + func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache) *searchsvc.Service { ls := c.cfgObject.cfgLocalStorage.localStorage @@ -425,7 +438,7 @@ func createDeleteServiceV2(sDelete *deletesvc.Service) *deletesvcV2.Service { } func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Service, - sSearchV2 *searchsvcV2.Service, sDeleteV2 *deletesvcV2.Service, + sSearchV2 *searchsvcV2.Service, sDeleteV2 *deletesvcV2.Service, sPatch *patchsvc.Service, ) *objectService.TransportSplitter { return objectService.NewTransportSplitter( c.cfgGRPC.maxChunkSize, @@ -435,6 +448,7 @@ func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Servi search: sSearchV2, get: sGetV2, delete: sDeleteV2, + patch: sPatch, }, ) } diff --git a/go.mod b/go.mod index 196b4d463..93ed7d750 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,14 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240813155151-d112a28d382f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240726111349-9da46f566fec + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240813155821-98aabc45a720 git.frostfs.info/TrueCloudLab/hrw v1.2.1 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/cheggaaa/pb v1.0.29 diff --git a/go.sum b/go.sum index bd6d85882..803a065c3 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e h1:gEWT+70E/RvGkxtSv+PlyUN2vtJVymhQa1mypvrXukM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240813155151-d112a28d382f h1:xrJqsXOZeSkBFMSyN+PQ9DiCGxVULU3VIN/tuH/vtb8= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240813155151-d112a28d382f/go.mod h1:mc7j6Cc1GU1tJZNmDwEYiJJ339biNnU1Bz3wZGogMe0= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= @@ -10,14 +10,14 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240726111349-9da46f566fec h1:A09Swh7yogmmiABUf7Ht6MTQXJ07MyGx4+ziUQNelec= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240726111349-9da46f566fec/go.mod h1:DlJmgV4/qkFkx2ab+YWznlMijiF2yZHnrJswJOB7XGs= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240813155821-98aabc45a720 h1:15UXpW42bfshIv/X5kww92jG2o0drHgsdFd+UJ6zD7g= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240813155821-98aabc45a720/go.mod h1:XRX/bBQsDJKr040N/a0YnDhxJqaUv1XyMVj3qxnb5K0= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984 h1:O3F2Grz07RWZ68mRz1xsYsNPNvZLwY00BM+xoYb1kNk= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240712081403-2628f6184984/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 h1:vgbfkcnIexZUm3vREBBSa/Gv1Whjd1SFCUd0A+IaGPQ= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= diff --git a/pkg/network/transport/object/grpc/service.go b/pkg/network/transport/object/grpc/service.go index 7c6b395d5..d55e3d87f 100644 --- a/pkg/network/transport/object/grpc/service.go +++ b/pkg/network/transport/object/grpc/service.go @@ -24,6 +24,48 @@ func New(c objectSvc.ServiceServer) *Server { } } +// Patch opens internal Object patch stream and feeds it by the data read from gRPC stream. +func (s *Server) Patch(gStream objectGRPC.ObjectService_PatchServer) error { + stream, err := s.srv.Patch() + if err != nil { + return err + } + + for { + req, err := gStream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + resp, err := stream.CloseAndRecv(gStream.Context()) + if err != nil { + return err + } + + return gStream.SendAndClose(resp.ToGRPCMessage().(*objectGRPC.PatchResponse)) + } + + return err + } + + patchReq := new(object.PatchRequest) + if err := patchReq.FromGRPCMessage(req); err != nil { + return err + } + + if err := stream.Send(gStream.Context(), patchReq); err != nil { + if errors.Is(err, util.ErrAbortStream) { + resp, err := stream.CloseAndRecv(gStream.Context()) + if err != nil { + return err + } + + return gStream.SendAndClose(resp.ToGRPCMessage().(*objectGRPC.PatchResponse)) + } + + return err + } + } +} + // Put opens internal Object service Put stream and overtakes data from gRPC stream to it. func (s *Server) Put(gStream objectGRPC.ObjectService_PutServer) error { stream, err := s.srv.Put() diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 3e128836f..58557d611 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -249,6 +249,26 @@ func (b Service) Put() (object.PutObjectStream, error) { }, err } +type patchStreamBasicChecker struct { + next object.PatchObjectstream +} + +func (p patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.PatchRequest) error { + return p.next.Send(ctx, request) +} + +func (p patchStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PatchResponse, error) { + return p.next.CloseAndRecv(ctx) +} + +func (b Service) Patch() (object.PatchObjectstream, error) { + streamer, err := b.next.Patch() + + return &patchStreamBasicChecker{ + next: streamer, + }, err +} + func (b Service) Head( ctx context.Context, request *objectV2.HeadRequest, diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 2adb1b736..f005d0873 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -204,6 +204,26 @@ func (c *Service) Put() (objectSvc.PutObjectStream, error) { }, err } +type patchStreamBasicChecker struct { + next objectSvc.PatchObjectstream +} + +func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.PatchRequest) error { + return p.next.Send(ctx, request) +} + +func (p patchStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PatchResponse, error) { + return p.next.CloseAndRecv(ctx) +} + +func (c *Service) Patch() (objectSvc.PatchObjectstream, error) { + streamer, err := c.next.Patch() + + return &patchStreamBasicChecker{ + next: streamer, + }, err +} + func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { diff --git a/pkg/services/object/audit.go b/pkg/services/object/audit.go index 1305fa008..680a96c40 100644 --- a/pkg/services/object/audit.go +++ b/pkg/services/object/audit.go @@ -170,3 +170,59 @@ func (a *auditPutStream) Send(ctx context.Context, req *object.PutRequest) error } return err } + +type auditPatchStream struct { + stream PatchObjectstream + log *logger.Logger + + failed bool + key []byte + containerID *refs.ContainerID + objectID *refs.ObjectID +} + +func (a *auditService) Patch() (PatchObjectstream, error) { + res, err := a.next.Patch() + if !a.enabled.Load() { + return res, err + } + if err != nil { + audit.LogRequest(a.log, objectGRPC.ObjectService_Patch_FullMethodName, nil, nil, false) + return res, err + } + return &auditPatchStream{ + stream: res, + log: a.log, + }, nil +} + +// CloseAndRecv implements PutObjectStream. +func (a *auditPatchStream) CloseAndRecv(ctx context.Context) (*object.PatchResponse, error) { + resp, err := a.stream.CloseAndRecv(ctx) + if err != nil { + a.failed = true + } + a.objectID = resp.GetBody().ObjectID + audit.LogRequestWithKey(a.log, objectGRPC.ObjectService_Patch_FullMethodName, a.key, + audit.TargetFromContainerIDObjectID(a.containerID, a.objectID), + !a.failed) + return resp, err +} + +// Send implements PutObjectStream. +func (a *auditPatchStream) Send(ctx context.Context, req *object.PatchRequest) error { + a.containerID = req.GetBody().GetAddress().GetContainerID() + a.objectID = req.GetBody().GetAddress().GetObjectID() + a.key = req.GetVerificationHeader().GetBodySignature().GetKey() + + err := a.stream.Send(ctx, req) + if err != nil { + a.failed = true + } + if !errors.Is(err, util.ErrAbortStream) { // CloseAndRecv will not be called, so log here + audit.LogRequestWithKey(a.log, objectGRPC.ObjectService_Patch_FullMethodName, a.key, + audit.TargetFromContainerIDObjectID(a.containerID, a.objectID), + !a.failed) + } + return err +} diff --git a/pkg/services/object/common.go b/pkg/services/object/common.go index 73ee9f81b..841a3d021 100644 --- a/pkg/services/object/common.go +++ b/pkg/services/object/common.go @@ -48,6 +48,14 @@ func (x *Common) Put() (PutObjectStream, error) { return x.nextHandler.Put() } +func (x *Common) Patch() (PatchObjectstream, error) { + if x.state.IsMaintenance() { + return nil, new(apistatus.NodeUnderMaintenance) + } + + return x.nextHandler.Patch() +} + func (x *Common) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { if x.state.IsMaintenance() { return nil, new(apistatus.NodeUnderMaintenance) diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index f972f43ae..b64f879ac 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -27,6 +27,12 @@ type ( start time.Time } + patchStreamMetric struct { + stream PatchObjectstream + metrics MetricRegister + start time.Time + } + MetricRegister interface { AddRequestDuration(string, time.Duration, bool) AddPayloadSize(string, int) @@ -76,6 +82,24 @@ func (m MetricCollector) Put() (PutObjectStream, error) { return m.next.Put() } +func (m MetricCollector) Patch() (PatchObjectstream, error) { + if m.enabled { + t := time.Now() + + stream, err := m.next.Patch() + if err != nil { + return nil, err + } + + return &patchStreamMetric{ + stream: stream, + metrics: m.metrics, + start: t, + }, nil + } + return m.next.Patch() +} + func (m MetricCollector) PutSingle(ctx context.Context, request *object.PutSingleRequest) (*object.PutSingleResponse, error) { if m.enabled { t := time.Now() @@ -189,3 +213,16 @@ func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, return res, err } +func (s patchStreamMetric) Send(ctx context.Context, req *object.PatchRequest) error { + s.metrics.AddPayloadSize("Patch", len(req.GetBody().GetPatch().Chunk)) + + return s.stream.Send(ctx, req) +} + +func (s patchStreamMetric) CloseAndRecv(ctx context.Context) (*object.PatchResponse, error) { + res, err := s.stream.CloseAndRecv(ctx) + + s.metrics.AddRequestDuration("Patch", time.Since(s.start), err == nil) + + return res, err +} diff --git a/pkg/services/object/patch/service.go b/pkg/services/object/patch/service.go new file mode 100644 index 000000000..df6926e84 --- /dev/null +++ b/pkg/services/object/patch/service.go @@ -0,0 +1,22 @@ +package patchsvc + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" + getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" + putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" +) + +// Service implements Put operation of Object service v2. +type Service struct { +} + +// NewService constructs Service instance from provided options. +func NewService(_ *util.KeyStorage, _ *getsvc.Service, _ *putsvc.Service) *Service { + return &Service{} +} + +// Put calls internal service and returns v2 object streamer. +func (s *Service) Patch() (object.PatchObjectstream, error) { + return &Streamer{}, nil +} diff --git a/pkg/services/object/patch/streamer.go b/pkg/services/object/patch/streamer.go new file mode 100644 index 000000000..5d021b9c3 --- /dev/null +++ b/pkg/services/object/patch/streamer.go @@ -0,0 +1,28 @@ +package patchsvc + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" +) + +// Streamer for the patch handler is a pipeline that merges two incoming +// streams of patches and original object payload chunks. +// The merged result is fed to Put stream target. +type Streamer struct{} + +func (s *Streamer) Send(ctx context.Context, _ *object.PatchRequest) error { + _, span := tracing.StartSpanFromContext(ctx, "patch.streamer.Send") + defer span.End() + + return nil +} + +func (s *Streamer) CloseAndRecv(_ context.Context) (*object.PatchResponse, error) { + return &object.PatchResponse{ + Body: &object.PatchResponseBody{ + ObjectID: nil, + }, + }, nil +} diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index a10f26a34..c85259c1f 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -37,6 +37,11 @@ type putStreamResponser struct { respSvc *response.Service } +type patchStreamResponser struct { + stream PatchObjectstream + respSvc *response.Service +} + // NewResponseService returns object service instance that passes internal service // call to response service. func NewResponseService(objSvc ServiceServer, respSvc *response.Service) *ResponseService { @@ -87,6 +92,35 @@ func (s *ResponseService) Put() (PutObjectStream, error) { }, nil } +func (s *patchStreamResponser) Send(ctx context.Context, req *object.PatchRequest) error { + if err := s.stream.Send(ctx, req); err != nil { + return fmt.Errorf("could not send the request: %w", err) + } + return nil +} + +func (s *patchStreamResponser) CloseAndRecv(ctx context.Context) (*object.PatchResponse, error) { + r, err := s.stream.CloseAndRecv(ctx) + if err != nil { + return nil, fmt.Errorf("could not close stream and receive response: %w", err) + } + + s.respSvc.SetMeta(r) + return r, nil +} + +func (s *ResponseService) Patch() (PatchObjectstream, error) { + stream, err := s.svc.Patch() + if err != nil { + return nil, fmt.Errorf("could not create Put object streamer: %w", err) + } + + return &patchStreamResponser{ + stream: stream, + respSvc: s.respSvc, + }, nil +} + func (s *ResponseService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { resp, err := s.svc.PutSingle(ctx, req) if err != nil { diff --git a/pkg/services/object/server.go b/pkg/services/object/server.go index 73b88f233..c1b036ab3 100644 --- a/pkg/services/object/server.go +++ b/pkg/services/object/server.go @@ -31,11 +31,18 @@ type PutObjectStream interface { CloseAndRecv(context.Context) (*object.PutResponse, error) } +// PatchObjectstream is an interface of FrostFS API v2 compatible patch streamer. +type PatchObjectstream interface { + Send(context.Context, *object.PatchRequest) error + CloseAndRecv(context.Context) (*object.PatchResponse, error) +} + // ServiceServer is an interface of utility // serving v2 Object service. type ServiceServer interface { Get(*object.GetRequest, GetObjectStream) error Put() (PutObjectStream, error) + Patch() (PatchObjectstream, error) Head(context.Context, *object.HeadRequest) (*object.HeadResponse, error) Search(*object.SearchRequest, SearchStream) error Delete(context.Context, *object.DeleteRequest) (*object.DeleteResponse, error) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 4bf581b78..631c539af 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -35,6 +35,12 @@ type putStreamSigner struct { err error } +type patchStreamSigner struct { + sigSvc *util.SignService + stream PatchObjectstream + err error +} + type getRangeStreamSigner struct { GetObjectRangeStream sigSvc *util.SignService @@ -112,6 +118,42 @@ func (s *SignService) Put() (PutObjectStream, error) { }, nil } +func (s *patchStreamSigner) Send(ctx context.Context, req *object.PatchRequest) error { + if s.err = s.sigSvc.VerifyRequest(req); s.err != nil { + return util.ErrAbortStream + } + if s.err = s.stream.Send(ctx, req); s.err != nil { + return util.ErrAbortStream + } + return nil +} + +func (s *patchStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.PatchResponse, err error) { + if s.err != nil { + err = s.err + resp = new(object.PatchResponse) + } else { + resp, err = s.stream.CloseAndRecv(ctx) + if err != nil { + return nil, fmt.Errorf("could not close stream and receive response: %w", err) + } + } + + return resp, s.sigSvc.SignResponse(resp, err) +} + +func (s *SignService) Patch() (PatchObjectstream, error) { + stream, err := s.svc.Patch() + if err != nil { + return nil, fmt.Errorf("could not create Put object streamer: %w", err) + } + + return &patchStreamSigner{ + stream: stream, + sigSvc: s.sigSvc, + }, nil +} + func (s *SignService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { if err := s.sigSvc.VerifyRequest(req); err != nil { resp := new(object.HeadResponse) diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index 54e49cb12..5acfac06b 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -91,6 +91,10 @@ func (c TransportSplitter) Put() (PutObjectStream, error) { return c.next.Put() } +func (c TransportSplitter) Patch() (PatchObjectstream, error) { + return c.next.Patch() +} + func (c TransportSplitter) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) { return c.next.Head(ctx, request) } From e890f1b4b17d62cd95fe22d018212f91ff2be95d Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 12 Aug 2024 17:11:10 +0300 Subject: [PATCH 1276/1943] [#1307] object: Implement `Patch` method Signed-off-by: Airat Arifullin --- cmd/frostfs-node/object.go | 2 +- pkg/services/object/acl/v2/service.go | 20 +- pkg/services/object/acl/v2/util.go | 8 +- pkg/services/object/ape/checker_test.go | 19 +- pkg/services/object/ape/service.go | 20 +- pkg/services/object/audit.go | 19 +- pkg/services/object/common.go | 2 +- pkg/services/object/get/prm.go | 4 + pkg/services/object/metrics.go | 6 +- pkg/services/object/patch/range_provider.go | 63 ++++++ pkg/services/object/patch/service.go | 30 ++- pkg/services/object/patch/streamer.go | 215 +++++++++++++++++++- pkg/services/object/patch/util.go | 53 +++++ pkg/services/object/put/prm.go | 11 + pkg/services/object/put/streamer.go | 23 ++- pkg/services/object/response.go | 4 +- pkg/services/object/server.go | 6 +- pkg/services/object/sign.go | 4 +- pkg/services/object/transport_splitter.go | 2 +- 19 files changed, 430 insertions(+), 81 deletions(-) create mode 100644 pkg/services/object/patch/range_provider.go create mode 100644 pkg/services/object/patch/util.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index eef142415..467c5901b 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -74,7 +74,7 @@ func (s *objectSvc) Put() (objectService.PutObjectStream, error) { return s.put.Put() } -func (s *objectSvc) Patch() (objectService.PatchObjectstream, error) { +func (s *objectSvc) Patch() (objectService.PatchObjectStream, error) { return s.patch.Patch() } diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 58557d611..a9ddad7ca 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -249,24 +249,8 @@ func (b Service) Put() (object.PutObjectStream, error) { }, err } -type patchStreamBasicChecker struct { - next object.PatchObjectstream -} - -func (p patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.PatchRequest) error { - return p.next.Send(ctx, request) -} - -func (p patchStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PatchResponse, error) { - return p.next.CloseAndRecv(ctx) -} - -func (b Service) Patch() (object.PatchObjectstream, error) { - streamer, err := b.next.Patch() - - return &patchStreamBasicChecker{ - next: streamer, - }, err +func (b Service) Patch() (object.PatchObjectStream, error) { + return b.next.Patch() } func (b Service) Head( diff --git a/pkg/services/object/acl/v2/util.go b/pkg/services/object/acl/v2/util.go index feda6a3cf..76fd9651d 100644 --- a/pkg/services/object/acl/v2/util.go +++ b/pkg/services/object/acl/v2/util.go @@ -174,7 +174,7 @@ func isOwnerFromKey(id user.ID, key *keys.PublicKey) bool { func assertVerb(tok sessionSDK.Object, op acl.Op) bool { switch op { case acl.OpObjectPut: - return tok.AssertVerb(sessionSDK.VerbObjectPut, sessionSDK.VerbObjectDelete) + return tok.AssertVerb(sessionSDK.VerbObjectPut, sessionSDK.VerbObjectDelete, sessionSDK.VerbObjectPatch) case acl.OpObjectDelete: return tok.AssertVerb(sessionSDK.VerbObjectDelete) case acl.OpObjectGet: @@ -185,11 +185,13 @@ func assertVerb(tok sessionSDK.Object, op acl.Op) bool { sessionSDK.VerbObjectGet, sessionSDK.VerbObjectDelete, sessionSDK.VerbObjectRange, - sessionSDK.VerbObjectRangeHash) + sessionSDK.VerbObjectRangeHash, + sessionSDK.VerbObjectPatch, + ) case acl.OpObjectSearch: return tok.AssertVerb(sessionSDK.VerbObjectSearch, sessionSDK.VerbObjectDelete) case acl.OpObjectRange: - return tok.AssertVerb(sessionSDK.VerbObjectRange, sessionSDK.VerbObjectRangeHash) + return tok.AssertVerb(sessionSDK.VerbObjectRange, sessionSDK.VerbObjectRangeHash, sessionSDK.VerbObjectPatch) case acl.OpObjectHash: return tok.AssertVerb(sessionSDK.VerbObjectRangeHash) } diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index 090f6a83c..afe19fc51 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -518,7 +518,22 @@ func TestAPECheck_BearerTokenOverrides(t *testing.T) { ls := inmemory.NewInmemoryLocalStorage() ms := inmemory.NewInmemoryMorphRuleChainStorage() - checker := NewChecker(ls, ms, headerProvider, frostfsidProvider, nil, &stMock{}, nil, nil) + node1Key, err := keys.NewPrivateKey() + require.NoError(t, err) + node1 := netmapSDK.NodeInfo{} + node1.SetPublicKey(node1Key.PublicKey().Bytes()) + netmap := &netmapSDK.NetMap{} + netmap.SetEpoch(100) + netmap.SetNodes([]netmapSDK.NodeInfo{node1}) + + nm := &netmapStub{ + currentEpoch: 100, + netmaps: map[uint64]*netmapSDK.NetMap{ + 100: netmap, + }, + } + + checker := NewChecker(ls, ms, headerProvider, frostfsidProvider, nm, &stMock{}, nil, nil) prm := Prm{ Method: method, @@ -541,7 +556,7 @@ func TestAPECheck_BearerTokenOverrides(t *testing.T) { } } - err := checker.CheckAPE(context.Background(), prm) + err = checker.CheckAPE(context.Background(), prm) if test.expectAPEErr { require.Error(t, err) } else { diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index f005d0873..64dd19c24 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -204,24 +204,8 @@ func (c *Service) Put() (objectSvc.PutObjectStream, error) { }, err } -type patchStreamBasicChecker struct { - next objectSvc.PatchObjectstream -} - -func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.PatchRequest) error { - return p.next.Send(ctx, request) -} - -func (p patchStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PatchResponse, error) { - return p.next.CloseAndRecv(ctx) -} - -func (c *Service) Patch() (objectSvc.PatchObjectstream, error) { - streamer, err := c.next.Patch() - - return &patchStreamBasicChecker{ - next: streamer, - }, err +func (c *Service) Patch() (objectSvc.PatchObjectStream, error) { + return c.next.Patch() } func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { diff --git a/pkg/services/object/audit.go b/pkg/services/object/audit.go index 680a96c40..b924386d1 100644 --- a/pkg/services/object/audit.go +++ b/pkg/services/object/audit.go @@ -172,16 +172,18 @@ func (a *auditPutStream) Send(ctx context.Context, req *object.PutRequest) error } type auditPatchStream struct { - stream PatchObjectstream + stream PatchObjectStream log *logger.Logger failed bool key []byte containerID *refs.ContainerID objectID *refs.ObjectID + + nonFirstSend bool } -func (a *auditService) Patch() (PatchObjectstream, error) { +func (a *auditService) Patch() (PatchObjectStream, error) { res, err := a.next.Patch() if !a.enabled.Load() { return res, err @@ -196,7 +198,7 @@ func (a *auditService) Patch() (PatchObjectstream, error) { }, nil } -// CloseAndRecv implements PutObjectStream. +// CloseAndRecv implements PatchObjectStream. func (a *auditPatchStream) CloseAndRecv(ctx context.Context) (*object.PatchResponse, error) { resp, err := a.stream.CloseAndRecv(ctx) if err != nil { @@ -209,11 +211,14 @@ func (a *auditPatchStream) CloseAndRecv(ctx context.Context) (*object.PatchRespo return resp, err } -// Send implements PutObjectStream. +// Send implements PatchObjectStream. func (a *auditPatchStream) Send(ctx context.Context, req *object.PatchRequest) error { - a.containerID = req.GetBody().GetAddress().GetContainerID() - a.objectID = req.GetBody().GetAddress().GetObjectID() - a.key = req.GetVerificationHeader().GetBodySignature().GetKey() + if !a.nonFirstSend { + a.containerID = req.GetBody().GetAddress().GetContainerID() + a.objectID = req.GetBody().GetAddress().GetObjectID() + a.key = req.GetVerificationHeader().GetBodySignature().GetKey() + a.nonFirstSend = true + } err := a.stream.Send(ctx, req) if err != nil { diff --git a/pkg/services/object/common.go b/pkg/services/object/common.go index 841a3d021..f48cc5b3d 100644 --- a/pkg/services/object/common.go +++ b/pkg/services/object/common.go @@ -48,7 +48,7 @@ func (x *Common) Put() (PutObjectStream, error) { return x.nextHandler.Put() } -func (x *Common) Patch() (PatchObjectstream, error) { +func (x *Common) Patch() (PatchObjectStream, error) { if x.state.IsMaintenance() { return nil, new(apistatus.NodeUnderMaintenance) } diff --git a/pkg/services/object/get/prm.go b/pkg/services/object/get/prm.go index cbdb7a3e2..94c07381c 100644 --- a/pkg/services/object/get/prm.go +++ b/pkg/services/object/get/prm.go @@ -124,6 +124,10 @@ func (p *commonPrm) SetRequestForwarder(f RequestForwarder) { p.forwarder = f } +func (p *commonPrm) SetSignerKey(signerKey *ecdsa.PrivateKey) { + p.signerKey = signerKey +} + // WithAddress sets object address to be read. func (p *commonPrm) WithAddress(addr oid.Address) { p.addr = addr diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index b64f879ac..e53b7584f 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -28,7 +28,7 @@ type ( } patchStreamMetric struct { - stream PatchObjectstream + stream PatchObjectStream metrics MetricRegister start time.Time } @@ -82,7 +82,7 @@ func (m MetricCollector) Put() (PutObjectStream, error) { return m.next.Put() } -func (m MetricCollector) Patch() (PatchObjectstream, error) { +func (m MetricCollector) Patch() (PatchObjectStream, error) { if m.enabled { t := time.Now() @@ -214,7 +214,7 @@ func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, return res, err } func (s patchStreamMetric) Send(ctx context.Context, req *object.PatchRequest) error { - s.metrics.AddPayloadSize("Patch", len(req.GetBody().GetPatch().Chunk)) + s.metrics.AddPayloadSize("Patch", len(req.GetBody().GetPatch().GetChunk())) return s.stream.Send(ctx, req) } diff --git a/pkg/services/object/patch/range_provider.go b/pkg/services/object/patch/range_provider.go new file mode 100644 index 000000000..755c5bf60 --- /dev/null +++ b/pkg/services/object/patch/range_provider.go @@ -0,0 +1,63 @@ +package patchsvc + +import ( + "context" + "crypto/ecdsa" + "io" + + getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" + objectUtil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + patcherSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/patcher" +) + +func (p *pipeChunkWriter) WriteChunk(_ context.Context, chunk []byte) error { + _, err := p.wr.Write(chunk) + return err +} + +type rangeProvider struct { + getSvc *getsvc.Service + + addr oid.Address + + commonPrm *objectUtil.CommonPrm + + localNodeKey *ecdsa.PrivateKey +} + +var _ patcherSDK.RangeProvider = (*rangeProvider)(nil) + +func (r *rangeProvider) GetRange(ctx context.Context, rng *objectSDK.Range) io.Reader { + pipeReader, pipeWriter := io.Pipe() + + var rngPrm getsvc.RangePrm + rngPrm.SetSignerKey(r.localNodeKey) + rngPrm.SetCommonParameters(r.commonPrm) + + rngPrm.WithAddress(r.addr) + rngPrm.SetChunkWriter(&pipeChunkWriter{ + wr: pipeWriter, + }) + rngPrm.SetRange(rng) + + getRangeErr := make(chan error) + + go func() { + defer pipeWriter.Close() + + select { + case <-ctx.Done(): + pipeWriter.CloseWithError(ctx.Err()) + case err := <-getRangeErr: + pipeWriter.CloseWithError(err) + } + }() + + go func() { + getRangeErr <- r.getSvc.GetRange(ctx, rngPrm) + }() + + return pipeReader +} diff --git a/pkg/services/object/patch/service.go b/pkg/services/object/patch/service.go index df6926e84..c4ab15abf 100644 --- a/pkg/services/object/patch/service.go +++ b/pkg/services/object/patch/service.go @@ -9,14 +9,36 @@ import ( // Service implements Put operation of Object service v2. type Service struct { + keyStorage *util.KeyStorage + + getSvc *getsvc.Service + + putSvc *putsvc.Service } // NewService constructs Service instance from provided options. -func NewService(_ *util.KeyStorage, _ *getsvc.Service, _ *putsvc.Service) *Service { - return &Service{} +func NewService(ks *util.KeyStorage, getSvc *getsvc.Service, putSvc *putsvc.Service) *Service { + return &Service{ + keyStorage: ks, + + getSvc: getSvc, + + putSvc: putSvc, + } } // Put calls internal service and returns v2 object streamer. -func (s *Service) Patch() (object.PatchObjectstream, error) { - return &Streamer{}, nil +func (s *Service) Patch() (object.PatchObjectStream, error) { + nodeKey, err := s.keyStorage.GetKey(nil) + if err != nil { + return nil, err + } + + return &Streamer{ + getSvc: s.getSvc, + + putSvc: s.putSvc, + + localNodeKey: nodeKey, + }, nil } diff --git a/pkg/services/object/patch/streamer.go b/pkg/services/object/patch/streamer.go index 5d021b9c3..84363530e 100644 --- a/pkg/services/object/patch/streamer.go +++ b/pkg/services/object/patch/streamer.go @@ -2,27 +2,220 @@ package patchsvc import ( "context" + "crypto/ecdsa" + "errors" + "fmt" + "io" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + refsV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" + putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/patcher" ) -// Streamer for the patch handler is a pipeline that merges two incoming -// streams of patches and original object payload chunks. -// The merged result is fed to Put stream target. -type Streamer struct{} +// Streamer for the patch handler is a pipeline that merges two incoming streams of patches +// and original object payload chunks. The merged result is fed to Put stream target. +type Streamer struct { + // Patcher must be initialized at first Streamer.Send call. + patcher patcher.PatchApplier -func (s *Streamer) Send(ctx context.Context, _ *object.PatchRequest) error { - _, span := tracing.StartSpanFromContext(ctx, "patch.streamer.Send") + nonFirstSend bool + + getSvc *getsvc.Service + + putSvc *putsvc.Service + + localNodeKey *ecdsa.PrivateKey +} + +type pipeChunkWriter struct { + wr *io.PipeWriter +} + +type headResponseWriter struct { + body *objectV2.HeadResponseBody +} + +func (w *headResponseWriter) WriteHeader(_ context.Context, hdr *objectSDK.Object) error { + w.body.SetHeaderPart(toFullObjectHeader(hdr)) + return nil +} + +func toFullObjectHeader(hdr *objectSDK.Object) objectV2.GetHeaderPart { + obj := hdr.ToV2() + + hs := new(objectV2.HeaderWithSignature) + hs.SetHeader(obj.GetHeader()) + hs.SetSignature(obj.GetSignature()) + + return hs +} + +func (s *Streamer) init(ctx context.Context, req *objectV2.PatchRequest) error { + hdrWithSig, addr, err := s.readHeader(ctx, req) + if err != nil { + return err + } + + commonPrm, err := util.CommonPrmFromV2(req) + if err != nil { + return err + } + commonPrm.WithLocalOnly(false) + + rangeProvider := &rangeProvider{ + getSvc: s.getSvc, + + addr: addr, + + commonPrm: commonPrm, + + localNodeKey: s.localNodeKey, + } + + putstm, err := s.putSvc.Put() + if err != nil { + return err + } + + hdr := hdrWithSig.GetHeader() + oV2 := new(objectV2.Object) + hV2 := new(objectV2.Header) + oV2.SetHeader(hV2) + oV2.GetHeader().SetContainerID(hdr.GetContainerID()) + oV2.GetHeader().SetPayloadLength(hdr.GetPayloadLength()) + oV2.GetHeader().SetAttributes(hdr.GetAttributes()) + + ownerID, err := newOwnerID(req.GetVerificationHeader()) + if err != nil { + return err + } + oV2.GetHeader().SetOwnerID(ownerID) + + prm, err := s.putInitPrm(req, oV2) + if err != nil { + return err + } + + err = putstm.Init(ctx, prm) + if err != nil { + return err + } + + patcherPrm := patcher.Params{ + Header: objectSDK.NewFromV2(oV2), + + RangeProvider: rangeProvider, + + ObjectWriter: putstm.Target(), + } + + s.patcher = patcher.New(patcherPrm) + return nil +} + +func (s *Streamer) readHeader(ctx context.Context, req *objectV2.PatchRequest) (hdrWithSig *objectV2.HeaderWithSignature, addr oid.Address, err error) { + addrV2 := req.GetBody().GetAddress() + if addrV2 == nil { + err = errors.New("patch request has nil-address") + return + } + + if err = addr.ReadFromV2(*addrV2); err != nil { + err = fmt.Errorf("read address error: %w", err) + return + } + + commonPrm, err := util.CommonPrmFromV2(req) + if err != nil { + return + } + commonPrm.WithLocalOnly(false) + + var p getsvc.HeadPrm + p.SetSignerKey(s.localNodeKey) + p.SetCommonParameters(commonPrm) + + resp := new(objectV2.HeadResponse) + resp.SetBody(new(objectV2.HeadResponseBody)) + + p.WithAddress(addr) + p.SetHeaderWriter(&headResponseWriter{ + body: resp.GetBody(), + }) + + err = s.getSvc.Head(ctx, p) + if err != nil { + err = fmt.Errorf("get header error: %w", err) + return + } + + var ok bool + hdrPart := resp.GetBody().GetHeaderPart() + if hdrWithSig, ok = hdrPart.(*objectV2.HeaderWithSignature); !ok { + err = fmt.Errorf("unexpected header type: %T", hdrPart) + } + return +} + +func (s *Streamer) Send(ctx context.Context, req *objectV2.PatchRequest) error { + ctx, span := tracing.StartSpanFromContext(ctx, "patch.streamer.Send") defer span.End() + defer func() { + s.nonFirstSend = true + }() + + if !s.nonFirstSend { + if err := s.init(ctx, req); err != nil { + return fmt.Errorf("streamer init error: %w", err) + } + } + + patch := new(objectSDK.Patch) + patch.FromV2(req.GetBody()) + + if !s.nonFirstSend { + err := s.patcher.ApplyAttributesPatch(ctx, patch.NewAttributes, patch.ReplaceAttributes) + if err != nil { + return fmt.Errorf("patch attributes: %w", err) + } + } + + if patch.PayloadPatch != nil { + err := s.patcher.ApplyPayloadPatch(ctx, patch.PayloadPatch) + if err != nil { + return fmt.Errorf("patch payload: %w", err) + } + } else if s.nonFirstSend { + return errors.New("invalid non-first patch: empty payload") + } + return nil } -func (s *Streamer) CloseAndRecv(_ context.Context) (*object.PatchResponse, error) { - return &object.PatchResponse{ - Body: &object.PatchResponseBody{ - ObjectID: nil, +func (s *Streamer) CloseAndRecv(ctx context.Context) (*objectV2.PatchResponse, error) { + patcherResp, err := s.patcher.Close(ctx) + if err != nil { + return nil, err + } + + oidV2 := new(refsV2.ObjectID) + + if patcherResp.AccessIdentifiers.ParentID != nil { + patcherResp.AccessIdentifiers.ParentID.WriteToV2(oidV2) + } else { + patcherResp.AccessIdentifiers.SelfID.WriteToV2(oidV2) + } + + return &objectV2.PatchResponse{ + Body: &objectV2.PatchResponseBody{ + ObjectID: oidV2, }, }, nil } diff --git a/pkg/services/object/patch/util.go b/pkg/services/object/patch/util.go new file mode 100644 index 000000000..1218d6694 --- /dev/null +++ b/pkg/services/object/patch/util.go @@ -0,0 +1,53 @@ +package patchsvc + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "errors" + "fmt" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +// putInitPrm initializes put paramerer for Put stream. +func (s *Streamer) putInitPrm(req *objectV2.PatchRequest, obj *objectV2.Object) (*putsvc.PutInitPrm, error) { + commonPrm, err := util.CommonPrmFromV2(req) + if err != nil { + return nil, err + } + + prm := new(putsvc.PutInitPrm) + prm.WithObject(objectSDK.NewFromV2(obj)). + WithCommonPrm(commonPrm). + WithPrivateKey(s.localNodeKey) + + return prm, nil +} + +func newOwnerID(vh *session.RequestVerificationHeader) (*refs.OwnerID, error) { + for vh.GetOrigin() != nil { + vh = vh.GetOrigin() + } + sig := vh.GetBodySignature() + if sig == nil { + return nil, errors.New("empty body signature") + } + key, err := keys.NewPublicKeyFromBytes(sig.GetKey(), elliptic.P256()) + if err != nil { + return nil, fmt.Errorf("invalid signature key: %w", err) + } + + var userID user.ID + user.IDFromKey(&userID, (ecdsa.PublicKey)(*key)) + ownID := new(refs.OwnerID) + userID.WriteToV2(ownID) + + return ownID, nil +} diff --git a/pkg/services/object/put/prm.go b/pkg/services/object/put/prm.go index 52a7c102c..0c8f12b45 100644 --- a/pkg/services/object/put/prm.go +++ b/pkg/services/object/put/prm.go @@ -2,6 +2,7 @@ package putsvc import ( "context" + "crypto/ecdsa" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -20,6 +21,8 @@ type PutInitPrm struct { traverseOpts []placement.Option relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error + + privateKey *ecdsa.PrivateKey } type PutChunkPrm struct { @@ -65,3 +68,11 @@ func (p *PutChunkPrm) WithChunk(v []byte) *PutChunkPrm { return p } + +func (p *PutInitPrm) WithPrivateKey(v *ecdsa.PrivateKey) *PutInitPrm { + if p != nil { + p.privateKey = v + } + + return p +} diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 4e655ed54..969c8fa19 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -47,6 +47,11 @@ func (p *Streamer) Init(ctx context.Context, prm *PutInitPrm) error { return nil } +// Target accesses underlying target chunked object writer. +func (p *Streamer) Target() transformer.ChunkedObjectWriter { + return p.target +} + // MaxObjectSize returns maximum payload size for the streaming session. // // Must be called after the successful Init. @@ -79,11 +84,15 @@ func (p *Streamer) initTarget(prm *PutInitPrm) error { func (p *Streamer) initUntrustedTarget(prm *PutInitPrm) error { p.relay = prm.relay - nodeKey, err := p.cfg.keyStorage.GetKey(nil) - if err != nil { - return err + if prm.privateKey != nil { + p.privateKey = prm.privateKey + } else { + nodeKey, err := p.cfg.keyStorage.GetKey(nil) + if err != nil { + return err + } + p.privateKey = nodeKey } - p.privateKey = nodeKey // prepare untrusted-Put object target p.target = &validatingPreparedTarget{ @@ -136,7 +145,11 @@ func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { } } - p.privateKey = key + if prm.privateKey != nil { + p.privateKey = prm.privateKey + } else { + p.privateKey = key + } p.target = &validatingTarget{ fmt: p.fmtValidator, nextTarget: transformer.NewPayloadSizeLimiter(transformer.Params{ diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index c85259c1f..d7ba9f843 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -38,7 +38,7 @@ type putStreamResponser struct { } type patchStreamResponser struct { - stream PatchObjectstream + stream PatchObjectStream respSvc *response.Service } @@ -109,7 +109,7 @@ func (s *patchStreamResponser) CloseAndRecv(ctx context.Context) (*object.PatchR return r, nil } -func (s *ResponseService) Patch() (PatchObjectstream, error) { +func (s *ResponseService) Patch() (PatchObjectStream, error) { stream, err := s.svc.Patch() if err != nil { return nil, fmt.Errorf("could not create Put object streamer: %w", err) diff --git a/pkg/services/object/server.go b/pkg/services/object/server.go index c1b036ab3..da98ce245 100644 --- a/pkg/services/object/server.go +++ b/pkg/services/object/server.go @@ -31,8 +31,8 @@ type PutObjectStream interface { CloseAndRecv(context.Context) (*object.PutResponse, error) } -// PatchObjectstream is an interface of FrostFS API v2 compatible patch streamer. -type PatchObjectstream interface { +// PatchObjectStream is an interface of FrostFS API v2 compatible patch streamer. +type PatchObjectStream interface { Send(context.Context, *object.PatchRequest) error CloseAndRecv(context.Context) (*object.PatchResponse, error) } @@ -42,7 +42,7 @@ type PatchObjectstream interface { type ServiceServer interface { Get(*object.GetRequest, GetObjectStream) error Put() (PutObjectStream, error) - Patch() (PatchObjectstream, error) + Patch() (PatchObjectStream, error) Head(context.Context, *object.HeadRequest) (*object.HeadResponse, error) Search(*object.SearchRequest, SearchStream) error Delete(context.Context, *object.DeleteRequest) (*object.DeleteResponse, error) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 631c539af..35367aafe 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -37,7 +37,7 @@ type putStreamSigner struct { type patchStreamSigner struct { sigSvc *util.SignService - stream PatchObjectstream + stream PatchObjectStream err error } @@ -142,7 +142,7 @@ func (s *patchStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.Patc return resp, s.sigSvc.SignResponse(resp, err) } -func (s *SignService) Patch() (PatchObjectstream, error) { +func (s *SignService) Patch() (PatchObjectStream, error) { stream, err := s.svc.Patch() if err != nil { return nil, fmt.Errorf("could not create Put object streamer: %w", err) diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index 5acfac06b..e560d6d8c 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -91,7 +91,7 @@ func (c TransportSplitter) Put() (PutObjectStream, error) { return c.next.Put() } -func (c TransportSplitter) Patch() (PatchObjectstream, error) { +func (c TransportSplitter) Patch() (PatchObjectStream, error) { return c.next.Patch() } From 5ed317e24c8ddc505051a3d012eff73828238a4d Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 13 Aug 2024 15:01:01 +0300 Subject: [PATCH 1277/1943] [#1307] cli: Introduce `object patch` command Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/client.go | 65 ++++++++++ cmd/frostfs-cli/modules/object/patch.go | 151 ++++++++++++++++++++++ cmd/frostfs-cli/modules/object/root.go | 2 + cmd/frostfs-cli/modules/object/util.go | 2 + 4 files changed, 220 insertions(+) create mode 100644 cmd/frostfs-cli/modules/object/patch.go diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 215490dbe..a0fa22410 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -2,10 +2,13 @@ package internal import ( "bytes" + "cmp" "context" "errors" "fmt" "io" + "os" + "slices" "sort" "strings" @@ -869,3 +872,65 @@ func SyncContainerSettings(ctx context.Context, prm SyncContainerPrm) (*SyncCont return new(SyncContainerRes), nil } + +// PatchObjectPrm groups parameters of PatchObject operation. +type PatchObjectPrm struct { + commonObjectPrm + objectAddressPrm + + NewAttributes []objectSDK.Attribute + + ReplaceAttribute bool + + PayloadPatches []PayloadPatch +} + +type PayloadPatch struct { + Range objectSDK.Range + + PayloadPath string +} + +type PatchRes struct { + OID oid.ID +} + +func Patch(ctx context.Context, prm PatchObjectPrm) (*PatchRes, error) { + patchPrm := client.PrmObjectPatch{ + XHeaders: prm.xHeaders, + BearerToken: prm.bearerToken, + Session: prm.sessionToken, + Address: prm.objAddr, + } + + slices.SortFunc(prm.PayloadPatches, func(a, b PayloadPatch) int { + return cmp.Compare(a.Range.GetOffset(), b.Range.GetOffset()) + }) + + patcher, err := prm.cli.ObjectPatchInit(ctx, patchPrm) + if err != nil { + return nil, fmt.Errorf("init payload reading: %w", err) + } + + if patcher.PatchAttributes(ctx, prm.NewAttributes, prm.ReplaceAttribute) { + for _, pp := range prm.PayloadPatches { + payloadFile, err := os.OpenFile(pp.PayloadPath, os.O_RDONLY, os.ModePerm) + if err != nil { + return nil, err + } + applied := patcher.PatchPayload(ctx, &pp.Range, payloadFile) + _ = payloadFile.Close() + if !applied { + break + } + } + } + + res, err := patcher.Close(ctx) + if err != nil { + return nil, err + } + return &PatchRes{ + OID: res.ObjectID(), + }, nil +} diff --git a/cmd/frostfs-cli/modules/object/patch.go b/cmd/frostfs-cli/modules/object/patch.go new file mode 100644 index 000000000..8f03885ab --- /dev/null +++ b/cmd/frostfs-cli/modules/object/patch.go @@ -0,0 +1,151 @@ +package object + +import ( + "fmt" + "strconv" + "strings" + + internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" +) + +const ( + newAttrsFlagName = "new-attrs" + replaceAttrsFlagName = "replace-attrs" + rangeFlagName = "range" + payloadFlagName = "payload" +) + +var objectPatchCmd = &cobra.Command{ + Use: "patch", + Run: patch, + Short: "Patch FrostFS object", + Long: "Patch FrostFS object. Each range passed to the command requires to pass a corresponding patch payload.", + Example: ` +frostfs-cli -c config.yml -r 127.0.0.1:8080 object patch --cid --oid --new-attrs 'key1=val1,key2=val2' --replace-attrs +frostfs-cli -c config.yml -r 127.0.0.1:8080 object patch --cid --oid --range offX:lnX --payload /path/to/payloadX --range offY:lnY --payload /path/to/payloadY +frostfs-cli -c config.yml -r 127.0.0.1:8080 object patch --cid --oid --new-attrs 'key1=val1,key2=val2' --replace-attrs --range offX:lnX --payload /path/to/payload +`, +} + +func initObjectPatchCmd() { + commonflags.Init(objectPatchCmd) + initFlagSession(objectPatchCmd, "PATCH") + + flags := objectPatchCmd.Flags() + + flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + _ = objectRangeCmd.MarkFlagRequired(commonflags.CIDFlag) + + flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage) + _ = objectRangeCmd.MarkFlagRequired(commonflags.OIDFlag) + + flags.String(newAttrsFlagName, "", "New object attributes in form of Key1=Value1,Key2=Value2") + flags.Bool(replaceAttrsFlagName, false, "Replace object attributes by new ones.") + flags.StringSlice(rangeFlagName, []string{}, "Range to which patch payload is applied. Format: offset:length") + flags.StringSlice(payloadFlagName, []string{}, "Path to file with patch payload.") +} + +func patch(cmd *cobra.Command, _ []string) { + var cnr cid.ID + var obj oid.ID + + objAddr := readObjectAddress(cmd, &cnr, &obj) + + ranges, err := getRangeSlice(cmd) + commonCmd.ExitOnErr(cmd, "", err) + + payloads := patchPayloadPaths(cmd) + + if len(ranges) != len(payloads) { + commonCmd.ExitOnErr(cmd, "", fmt.Errorf("the number of ranges and payloads are not equal: ranges = %d, payloads = %d", len(ranges), len(payloads))) + } + + newAttrs, err := parseNewObjectAttrs(cmd) + commonCmd.ExitOnErr(cmd, "can't parse new object attributes: %w", err) + replaceAttrs, _ := cmd.Flags().GetBool(replaceAttrsFlagName) + + pk := key.GetOrGenerate(cmd) + + cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) + + var prm internalclient.PatchObjectPrm + prm.SetClient(cli) + Prepare(cmd, &prm) + ReadOrOpenSession(cmd, &prm, pk, cnr, nil) + + prm.SetAddress(objAddr) + prm.NewAttributes = newAttrs + prm.ReplaceAttribute = replaceAttrs + + for i := range ranges { + prm.PayloadPatches = append(prm.PayloadPatches, internalclient.PayloadPatch{ + Range: ranges[i], + PayloadPath: payloads[i], + }) + } + + res, err := internalclient.Patch(cmd.Context(), prm) + if err != nil { + commonCmd.ExitOnErr(cmd, "can't patch the object: %w", err) + } + cmd.Println("Patched object ID: ", res.OID.EncodeToString()) +} + +func parseNewObjectAttrs(cmd *cobra.Command) ([]objectSDK.Attribute, error) { + var rawAttrs []string + + raw := cmd.Flag(newAttrsFlagName).Value.String() + if len(raw) != 0 { + rawAttrs = strings.Split(raw, ",") + } + + attrs := make([]objectSDK.Attribute, len(rawAttrs), len(rawAttrs)+2) // name + timestamp attributes + for i := range rawAttrs { + k, v, found := strings.Cut(rawAttrs[i], "=") + if !found { + return nil, fmt.Errorf("invalid attribute format: %s", rawAttrs[i]) + } + attrs[i].SetKey(k) + attrs[i].SetValue(v) + } + return attrs, nil +} + +func getRangeSlice(cmd *cobra.Command) ([]objectSDK.Range, error) { + v, _ := cmd.Flags().GetStringSlice(rangeFlagName) + if len(v) == 0 { + return []objectSDK.Range{}, nil + } + rs := make([]objectSDK.Range, len(v)) + for i := range v { + before, after, found := strings.Cut(v[i], rangeSep) + if !found { + return nil, fmt.Errorf("invalid range specifier: %s", v[i]) + } + + offset, err := strconv.ParseUint(before, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid '%s' range offset specifier: %w", v[i], err) + } + length, err := strconv.ParseUint(after, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid '%s' range length specifier: %w", v[i], err) + } + + rs[i].SetOffset(offset) + rs[i].SetLength(length) + } + return rs, nil +} + +func patchPayloadPaths(cmd *cobra.Command) []string { + v, _ := cmd.Flags().GetStringSlice(payloadFlagName) + return v +} diff --git a/cmd/frostfs-cli/modules/object/root.go b/cmd/frostfs-cli/modules/object/root.go index 7d8008b10..b808a509e 100644 --- a/cmd/frostfs-cli/modules/object/root.go +++ b/cmd/frostfs-cli/modules/object/root.go @@ -29,6 +29,7 @@ func init() { objectRangeCmd, objectLockCmd, objectNodesCmd, + objectPatchCmd, } Cmd.AddCommand(objectChildCommands...) @@ -39,6 +40,7 @@ func init() { } initObjectPutCmd() + initObjectPatchCmd() initObjectDeleteCmd() initObjectGetCmd() initObjectSearchCmd() diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index 381c790e9..96b80fe1b 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -306,6 +306,8 @@ func finalizeSession(cmd *cobra.Command, dst SessionPrm, tok *session.Object, ke case *internal.PutObjectPrm: common.PrintVerbose(cmd, "Binding session to object PUT...") tok.ForVerb(session.VerbObjectPut) + case *internal.PatchObjectPrm: + tok.ForVerb(session.VerbObjectPatch) case *internal.DeleteObjectPrm: common.PrintVerbose(cmd, "Binding session to object DELETE...") tok.ForVerb(session.VerbObjectDelete) From eeab417dcf75846b69c0567b118dab5fb27a28a2 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 14 Aug 2024 14:38:01 +0300 Subject: [PATCH 1278/1943] [#1307] object: Add APE check for `Patch` handler Signed-off-by: Airat Arifullin --- pkg/services/object/acl/v2/service.go | 125 +++++++++++++++++++++++- pkg/services/object/acl/v2/util.go | 2 + pkg/services/object/ape/checker_test.go | 19 +--- pkg/services/object/ape/request.go | 3 +- pkg/services/object/ape/service.go | 54 +++++++++- 5 files changed, 183 insertions(+), 20 deletions(-) diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index a9ddad7ca..5a8e8b065 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -35,6 +35,12 @@ type putStreamBasicChecker struct { next object.PutObjectStream } +type patchStreamBasicChecker struct { + source *Service + next object.PatchObjectStream + nonFirstSend bool +} + type getStreamBasicChecker struct { checker ACLChecker @@ -250,7 +256,12 @@ func (b Service) Put() (object.PutObjectStream, error) { } func (b Service) Patch() (object.PatchObjectStream, error) { - return b.next.Patch() + streamer, err := b.next.Patch() + + return &patchStreamBasicChecker{ + source: &b, + next: streamer, + }, err } func (b Service) Head( @@ -738,6 +749,65 @@ func (g *searchStreamBasicChecker) Send(resp *objectV2.SearchResponse) error { return g.SearchStream.Send(resp) } +func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.PatchRequest) error { + body := request.GetBody() + if body == nil { + return errEmptyBody + } + + if !p.nonFirstSend { + p.nonFirstSend = true + + cnr, err := getContainerIDFromRequest(request) + if err != nil { + return err + } + + objV2 := request.GetBody().GetAddress().GetObjectID() + if objV2 == nil { + return errors.New("missing oid") + } + obj := new(oid.ID) + err = obj.ReadFromV2(*objV2) + if err != nil { + return err + } + + var sTok *sessionSDK.Object + sTok, err = readSessionToken(cnr, obj, request.GetMetaHeader().GetSessionToken()) + if err != nil { + return err + } + + bTok, err := originalBearerToken(request.GetMetaHeader()) + if err != nil { + return err + } + + req := MetaWithToken{ + vheader: request.GetVerificationHeader(), + token: sTok, + bearer: bTok, + src: request, + } + + reqInfo, err := p.source.findRequestInfoWithoutACLOperationAssert(req, cnr) + if err != nil { + return err + } + + reqInfo.obj = obj + + ctx = requestContext(ctx, reqInfo) + } + + return p.next.Send(ctx, request) +} + +func (p patchStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PatchResponse, error) { + return p.next.CloseAndRecv(ctx) +} + func (b Service) findRequestInfo(req MetaWithToken, idCnr cid.ID, op acl.Op) (info RequestInfo, err error) { cnr, err := b.containers.Get(idCnr) // fetch actual container if err != nil { @@ -794,3 +864,56 @@ func (b Service) findRequestInfo(req MetaWithToken, idCnr cid.ID, op acl.Op) (in return info, nil } + +// findRequestInfoWithoutACLOperationAssert is findRequestInfo without session token verb assert. +func (b Service) findRequestInfoWithoutACLOperationAssert(req MetaWithToken, idCnr cid.ID) (info RequestInfo, err error) { + cnr, err := b.containers.Get(idCnr) // fetch actual container + if err != nil { + return info, err + } + + if req.token != nil { + currentEpoch, err := b.nm.Epoch() + if err != nil { + return info, errors.New("can't fetch current epoch") + } + if req.token.ExpiredAt(currentEpoch) { + return info, new(apistatus.SessionTokenExpired) + } + if req.token.InvalidAt(currentEpoch) { + return info, fmt.Errorf("%s: token is invalid at %d epoch)", + invalidRequestMessage, currentEpoch) + } + } + + // find request role and key + ownerID, ownerKey, err := req.RequestOwner() + if err != nil { + return info, err + } + res, err := b.c.Classify(ownerID, ownerKey, idCnr, cnr.Value) + if err != nil { + return info, err + } + + info.basicACL = cnr.Value.BasicACL() + info.requestRole = res.Role + info.cnrOwner = cnr.Value.Owner() + info.idCnr = idCnr + + cnrNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(cnr.Value).Zone(), ".ns") + if hasNamespace { + info.cnrNamespace = cnrNamespace + } + + // it is assumed that at the moment the key will be valid, + // otherwise the request would not pass validation + info.senderKey = res.Key + + // add bearer token if it is present in request + info.bearer = req.bearer + + info.srcRequest = req.src + + return info, nil +} diff --git a/pkg/services/object/acl/v2/util.go b/pkg/services/object/acl/v2/util.go index 76fd9651d..c5225e8c4 100644 --- a/pkg/services/object/acl/v2/util.go +++ b/pkg/services/object/acl/v2/util.go @@ -46,6 +46,8 @@ func getContainerIDFromRequest(req any) (cid.ID, error) { idV2 = v.GetBody().GetAddress().GetContainerID() case *objectV2.PutSingleRequest: idV2 = v.GetBody().GetObject().GetHeader().GetContainerID() + case *objectV2.PatchRequest: + idV2 = v.GetBody().GetAddress().GetContainerID() default: return cid.ID{}, errors.New("unknown request type") } diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index afe19fc51..090f6a83c 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -518,22 +518,7 @@ func TestAPECheck_BearerTokenOverrides(t *testing.T) { ls := inmemory.NewInmemoryLocalStorage() ms := inmemory.NewInmemoryMorphRuleChainStorage() - node1Key, err := keys.NewPrivateKey() - require.NoError(t, err) - node1 := netmapSDK.NodeInfo{} - node1.SetPublicKey(node1Key.PublicKey().Bytes()) - netmap := &netmapSDK.NetMap{} - netmap.SetEpoch(100) - netmap.SetNodes([]netmapSDK.NodeInfo{node1}) - - nm := &netmapStub{ - currentEpoch: 100, - netmaps: map[uint64]*netmapSDK.NetMap{ - 100: netmap, - }, - } - - checker := NewChecker(ls, ms, headerProvider, frostfsidProvider, nm, &stMock{}, nil, nil) + checker := NewChecker(ls, ms, headerProvider, frostfsidProvider, nil, &stMock{}, nil, nil) prm := Prm{ Method: method, @@ -556,7 +541,7 @@ func TestAPECheck_BearerTokenOverrides(t *testing.T) { } } - err = checker.CheckAPE(context.Background(), prm) + err := checker.CheckAPE(context.Background(), prm) if test.expectAPEErr { require.Error(t, err) } else { diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index 1c129f65f..da5307ca7 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -103,7 +103,8 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re nativeschema.MethodHeadObject, nativeschema.MethodRangeObject, nativeschema.MethodHashObject, - nativeschema.MethodDeleteObject: + nativeschema.MethodDeleteObject, + nativeschema.MethodPatchObject: if prm.Object == nil { return defaultRequest, fmt.Errorf("method %s: %w", prm.Method, errMissingOID) } diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 64dd19c24..a1634e7c5 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -204,8 +204,60 @@ func (c *Service) Put() (objectSvc.PutObjectStream, error) { }, err } +type patchStreamBasicChecker struct { + apeChecker Checker + + next objectSvc.PatchObjectStream + + nonFirstSend bool +} + +func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.PatchRequest) error { + if !p.nonFirstSend { + p.nonFirstSend = true + + reqCtx, err := requestContext(ctx) + if err != nil { + return toStatusErr(err) + } + + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) + if err != nil { + return toStatusErr(err) + } + + prm := Prm{ + Namespace: reqCtx.Namespace, + Container: cnrID, + Object: objID, + Method: nativeschema.MethodPatchObject, + SenderKey: hex.EncodeToString(reqCtx.SenderKey), + ContainerOwner: reqCtx.ContainerOwner, + Role: nativeSchemaRole(reqCtx.Role), + SoftAPECheck: reqCtx.SoftAPECheck, + BearerToken: reqCtx.BearerToken, + XHeaders: request.GetMetaHeader().GetXHeaders(), + } + + if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { + return toStatusErr(err) + } + } + + return p.next.Send(ctx, request) +} + +func (p patchStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PatchResponse, error) { + return p.next.CloseAndRecv(ctx) +} + func (c *Service) Patch() (objectSvc.PatchObjectStream, error) { - return c.next.Patch() + streamer, err := c.next.Patch() + + return &patchStreamBasicChecker{ + apeChecker: c.apeChecker, + next: streamer, + }, err } func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { From 41104f2383226b9e92c13b1907403c3f9176bbae Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 14 Aug 2024 14:39:13 +0300 Subject: [PATCH 1279/1943] [#1307] cli: Make cli process object.patch Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/util/ape.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index 532dc0a50..9af57434a 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -239,6 +239,8 @@ func parseAction(lexeme string) ([]string, bool, error) { return []string{nativeschema.MethodRangeObject}, true, nil case "object.hash": return []string{nativeschema.MethodHashObject}, true, nil + case "object.patch": + return []string{nativeschema.MethodPatchObject}, true, nil case "object.*": return []string{ nativeschema.MethodPutObject, @@ -248,6 +250,7 @@ func parseAction(lexeme string) ([]string, bool, error) { nativeschema.MethodSearchObject, nativeschema.MethodRangeObject, nativeschema.MethodHashObject, + nativeschema.MethodPatchObject, }, true, nil case "container.put": return []string{nativeschema.MethodPutContainer}, false, nil From b7acb34fa4377a690f5f7fbe18c580c2d785b772 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 19 Aug 2024 15:50:35 +0300 Subject: [PATCH 1280/1943] [#1319] treeSvc: Do not wrap error from APE Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/ape.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 6e78bf4ec..ee4687911 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -138,7 +138,7 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, request, err := s.newAPERequest(ctx, namespace, cid, operation, role, publicKey) if err != nil { - return apeErr(err) + return fmt.Errorf("failed to create ape request: %w", err) } var cr engine.ChainRouter @@ -167,7 +167,7 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, rt := engine.NewRequestTargetExtended(namespace, cid.EncodeToString(), fmt.Sprintf("%s:%s", namespace, publicKey.Address()), groups) status, found, err := cr.IsAllowed(apechain.Ingress, rt, request) if err != nil { - return apeErr(err) + return err } if found && status == apechain.Allow { return nil From 8319b59238925960597650d3227310d8b6f187ec Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 19 Aug 2024 12:33:15 +0300 Subject: [PATCH 1281/1943] [#1318] Fix gofumpt issue Signed-off-by: Anton Nikiforov --- pkg/services/object/metrics.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index e53b7584f..61aed5003 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -213,6 +213,7 @@ func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, return res, err } + func (s patchStreamMetric) Send(ctx context.Context, req *object.PatchRequest) error { s.metrics.AddPayloadSize("Patch", len(req.GetBody().GetPatch().GetChunk())) From 6ff0b0996b31d0de3bbb12d41edb19bdd0c16dda Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 19 Aug 2024 12:34:05 +0300 Subject: [PATCH 1282/1943] [#1318] metrics: Fix `container_size_bytes` for EC When node put chunk into EC container, `policer` may remove it as redundant. This chunk marked as removed. When parent object removed and `gc` start iterating over chunk, node count removing chunk twice. Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/metabase/inhume.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index c265fb217..b62accc43 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -236,7 +236,7 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes return err } } else if errors.As(err, &ecErr) { - err = db.inhumeECInfo(tx, epoch, prm.tomb, res, garbageBKT, graveyardBKT, ecErr.ECInfo(), cnr, bkt, value, targetKey) + err = db.inhumeECInfo(tx, epoch, prm.tomb, res, garbageBKT, graveyardBKT, ecErr.ECInfo(), cnr, bkt, value) if err != nil { return err } @@ -280,7 +280,7 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes func (db *DB) inhumeECInfo(tx *bbolt.Tx, epoch uint64, tomb *oid.Address, res *InhumeRes, garbageBKT *bbolt.Bucket, graveyardBKT *bbolt.Bucket, - ecInfo *objectSDK.ECInfo, cnr cid.ID, targetBucket *bbolt.Bucket, value []byte, targetKey []byte, + ecInfo *objectSDK.ECInfo, cnr cid.ID, targetBucket *bbolt.Bucket, value []byte, ) error { for _, chunk := range ecInfo.Chunks { chunkBuf := make([]byte, addressKeySize) @@ -296,11 +296,11 @@ func (db *DB) inhumeECInfo(tx *bbolt.Tx, epoch uint64, tomb *oid.Address, res *I if err != nil { return err } - err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, targetKey, cnr, chunkObj, res) + chunkKey := addressKey(chunkAddr, chunkBuf) + err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, chunkKey, cnr, chunkObj, res) if err != nil { return err } - chunkKey := addressKey(chunkAddr, chunkBuf) if tomb != nil { _, err = db.markAsGC(graveyardBKT, garbageBKT, chunkKey) if err != nil { From cfda9003a7d4283946e9cbe742702114f93bbdce Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 20 Aug 2024 16:18:46 +0300 Subject: [PATCH 1283/1943] [#1318] meta: Add test `TestInhumeECObject` Signed-off-by: Anton Nikiforov --- .../metabase/inhume_ec_test.go | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 pkg/local_object_storage/metabase/inhume_ec_test.go diff --git a/pkg/local_object_storage/metabase/inhume_ec_test.go b/pkg/local_object_storage/metabase/inhume_ec_test.go new file mode 100644 index 000000000..c3b1e72da --- /dev/null +++ b/pkg/local_object_storage/metabase/inhume_ec_test.go @@ -0,0 +1,116 @@ +package meta + +import ( + "context" + "path/filepath" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +func TestInhumeECObject(t *testing.T) { + t.Parallel() + + db := New( + WithPath(filepath.Join(t.TempDir(), "metabase")), + WithPermissions(0o600), + WithEpochState(epochState{uint64(12)}), + ) + + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + require.NoError(t, db.Init()) + defer func() { require.NoError(t, db.Close()) }() + + cnr := cidtest.ID() + ecChunk := oidtest.ID() + ecChunk2 := oidtest.ID() + ecParent := oidtest.ID() + tombstoneID := oidtest.ID() + + chunkObj := testutil.GenerateObjectWithCID(cnr) + chunkObj.SetContainerID(cnr) + chunkObj.SetID(ecChunk) + chunkObj.SetPayload([]byte{0, 1, 2, 3, 4}) + chunkObj.SetPayloadSize(uint64(5)) + chunkObj.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent}, 0, 3, []byte{}, 0)) + + chunkObj2 := testutil.GenerateObjectWithCID(cnr) + chunkObj2.SetContainerID(cnr) + chunkObj2.SetID(ecChunk2) + chunkObj2.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + chunkObj2.SetPayloadSize(uint64(10)) + chunkObj2.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent}, 1, 3, []byte{}, 0)) + + // put object with EC + + var prm PutPrm + prm.SetObject(chunkObj) + prm.SetStorageID([]byte("0/0")) + _, err := db.Put(context.Background(), prm) + require.NoError(t, err) + + prm.SetObject(chunkObj2) + _, err = db.Put(context.Background(), prm) + require.NoError(t, err) + + var ecChunkAddress oid.Address + ecChunkAddress.SetContainer(cnr) + ecChunkAddress.SetObject(ecChunk) + + var ecParentAddress oid.Address + ecParentAddress.SetContainer(cnr) + ecParentAddress.SetObject(ecParent) + + var chunkObjectAddress oid.Address + chunkObjectAddress.SetContainer(cnr) + chunkObjectAddress.SetObject(ecChunk) + + var getPrm GetPrm + + getPrm.SetAddress(ecChunkAddress) + _, err = db.Get(context.Background(), getPrm) + require.NoError(t, err) + + var ecInfoError *objectSDK.ECInfoError + getPrm.SetAddress(ecParentAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, &ecInfoError) + require.True(t, len(ecInfoError.ECInfo().Chunks) == 2 && + ecInfoError.ECInfo().Chunks[0].Index == 0 && + ecInfoError.ECInfo().Chunks[0].Total == 3) + + // inhume Chunk + var inhumePrm InhumePrm + var tombAddress oid.Address + inhumePrm.SetAddresses(chunkObjectAddress) + res, err := db.Inhume(context.Background(), inhumePrm) + require.NoError(t, err) + require.True(t, len(res.deletionDetails) == 1) + require.True(t, res.deletionDetails[0].Size == 5) + + // inhume EC parent (like Delete does) + tombAddress.SetContainer(cnr) + tombAddress.SetObject(tombstoneID) + inhumePrm.SetAddresses(ecParentAddress) + inhumePrm.SetTombstoneAddress(tombAddress) + res, err = db.Inhume(context.Background(), inhumePrm) + require.NoError(t, err) + // Previously deleted chunk shouldn't be in the details, because it is marked as garbage + require.True(t, len(res.deletionDetails) == 1) + require.True(t, res.deletionDetails[0].Size == 10) + + getPrm.SetAddress(ecParentAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) + + getPrm.SetAddress(ecChunkAddress) + _, err = db.Get(context.Background(), getPrm) + require.ErrorAs(t, err, new(*apistatus.ObjectAlreadyRemoved)) +} From 8c1082b31a990273020bde006f63d160b51620df Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 20 Aug 2024 15:35:45 +0300 Subject: [PATCH 1284/1943] [#1316] go.mod: Bump go version to 1.22 Signed-off-by: Ekaterina Lebedeva --- .forgejo/workflows/build.yml | 2 +- .forgejo/workflows/pre-commit.yml | 2 +- .forgejo/workflows/tests.yml | 12 ++++++------ .forgejo/workflows/vulncheck.yml | 2 +- Makefile | 8 ++++---- README.md | 2 +- go.mod | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 86943fe88..ce2d64dd9 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.21', '1.22' ] + go_versions: [ '1.22', '1.23' ] steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/pre-commit.yml b/.forgejo/workflows/pre-commit.yml index 117cda93b..8b06a2fdf 100644 --- a/.forgejo/workflows/pre-commit.yml +++ b/.forgejo/workflows/pre-commit.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.22 + go-version: 1.23 - name: Set up Python run: | apt update diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 5d64d7bc4..a908c6278 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -11,7 +11,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.23' cache: true - name: Install linters @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.21', '1.22' ] + go_versions: [ '1.22', '1.23' ] fail-fast: false steps: - uses: actions/checkout@v3 @@ -48,7 +48,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' cache: true - name: Run tests @@ -63,7 +63,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.23' cache: true - name: Install staticcheck @@ -81,7 +81,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.21' + go-version: '1.22' cache: true - name: Install gopls @@ -99,7 +99,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.23' cache: true - name: Install gofumpt diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 3af564c4b..2951a8059 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.23' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/Makefile b/Makefile index 11111d9a7..8d1238b6f 100755 --- a/Makefile +++ b/Makefile @@ -8,8 +8,8 @@ HUB_IMAGE ?= truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.22 -LINT_VERSION ?= 1.56.1 -TRUECLOUDLAB_LINT_VERSION ?= 0.0.5 +LINT_VERSION ?= 1.60.1 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.6 PROTOC_VERSION ?= 25.0 PROTOC_GEN_GO_VERSION ?= $(shell go list -f '{{.Version}}' -m google.golang.org/protobuf) PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2) @@ -17,7 +17,7 @@ PROTOC_OS_VERSION=osx-x86_64 ifeq ($(shell uname), Linux) PROTOC_OS_VERSION=linux-x86_64 endif -STATICCHECK_VERSION ?= 2023.1.6 +STATICCHECK_VERSION ?= 2024.1.1 ARCH = amd64 BIN = bin @@ -46,7 +46,7 @@ STATICCHECK_VERSION_DIR ?= $(STATICCHECK_DIR)/$(STATICCHECK_VERSION) SOURCES = $(shell find . -type f -name "*.go" -print) -GOFUMPT_VERSION ?= v0.6.0 +GOFUMPT_VERSION ?= v0.7.0 GOFUMPT_DIR ?= $(abspath $(BIN))/gofumpt GOFUMPT_VERSION_DIR ?= $(GOFUMPT_DIR)/$(GOFUMPT_VERSION) diff --git a/README.md b/README.md index 413010372..8225f56c5 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The latest version of frostfs-node works with frostfs-contract # Building -To make all binaries you need Go 1.21+ and `make`: +To make all binaries you need Go 1.22+ and `make`: ``` make all ``` diff --git a/go.mod b/go.mod index 93ed7d750..79dcc9553 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-node -go 1.21 +go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 From a345c972bffd63c5df513a4e6770b5bc12811e3b Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 20 Aug 2024 18:37:04 +0300 Subject: [PATCH 1285/1943] [#1316] lint: Fix warnings Renamed parameters `min/max` to avoid conflicts with predeclared identifiers. Replaced background context with parent context without cancellation in closer functions in frostfs-node. Signed-off-by: Ekaterina Lebedeva --- .golangci.yml | 3 ++- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/tracing.go | 2 +- .../blobstor/internal/blobstortest/common.go | 16 ++++++++-------- .../blobstor/internal/blobstortest/control.go | 6 +++--- .../blobstor/internal/blobstortest/delete.go | 4 ++-- .../blobstor/internal/blobstortest/exists.go | 4 ++-- .../blobstor/internal/blobstortest/get.go | 4 ++-- .../blobstor/internal/blobstortest/get_range.go | 4 ++-- .../blobstor/internal/blobstortest/iterate.go | 4 ++-- pkg/morph/client/notary.go | 4 ++-- 11 files changed, 27 insertions(+), 26 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index d209693aa..2e9e78fc3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -12,7 +12,8 @@ run: # output configuration options output: # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" - format: tab + formats: + - format: tab # all available settings of specific linters linters-settings: diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 1af27d733..110281418 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1073,7 +1073,7 @@ func initLocalStorage(ctx context.Context, c *cfg) { c.onShutdown(func() { c.log.Info(logs.FrostFSNodeClosingComponentsOfTheStorageEngine) - err := ls.Close(context.Background()) + err := ls.Close(context.WithoutCancel(ctx)) if err != nil { c.log.Info(logs.FrostFSNodeStorageEngineClosingFailure, zap.String("error", err.Error()), diff --git a/cmd/frostfs-node/tracing.go b/cmd/frostfs-node/tracing.go index 312adfb8d..675c31374 100644 --- a/cmd/frostfs-node/tracing.go +++ b/cmd/frostfs-node/tracing.go @@ -21,7 +21,7 @@ func initTracing(ctx context.Context, c *cfg) { c.closers = append(c.closers, closer{ name: "tracing", fn: func() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Second*5) defer cancel() err := tracing.Shutdown(ctx) // cfg context cancels before close if err != nil { diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/common.go b/pkg/local_object_storage/blobstor/internal/blobstortest/common.go index c08e39bf1..5d14a9a3a 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/common.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/common.go @@ -27,21 +27,21 @@ type objectDesc struct { storageID []byte } -func TestAll(t *testing.T, cons Constructor, min, max uint64) { +func TestAll(t *testing.T, cons Constructor, minSize, maxSize uint64) { t.Run("get", func(t *testing.T) { - TestGet(t, cons, min, max) + TestGet(t, cons, minSize, maxSize) }) t.Run("get range", func(t *testing.T) { - TestGetRange(t, cons, min, max) + TestGetRange(t, cons, minSize, maxSize) }) t.Run("delete", func(t *testing.T) { - TestDelete(t, cons, min, max) + TestDelete(t, cons, minSize, maxSize) }) t.Run("exists", func(t *testing.T) { - TestExists(t, cons, min, max) + TestExists(t, cons, minSize, maxSize) }) t.Run("iterate", func(t *testing.T) { - TestIterate(t, cons, min, max) + TestIterate(t, cons, minSize, maxSize) }) } @@ -51,12 +51,12 @@ func TestInfo(t *testing.T, cons Constructor, expectedType string, expectedPath require.Equal(t, expectedPath, s.Path()) } -func prepare(t *testing.T, count int, s common.Storage, min, max uint64) []objectDesc { +func prepare(t *testing.T, count int, s common.Storage, minSize, maxSize uint64) []objectDesc { objects := make([]objectDesc, count) r := mrand.New(mrand.NewSource(0)) for i := range objects { - objects[i].obj = NewObject(min + uint64(r.Intn(int(max-min+1)))) // not too large + objects[i].obj = NewObject(minSize + uint64(r.Intn(int(maxSize-minSize+1)))) // not too large objects[i].addr = objectCore.AddressOf(objects[i].obj) raw, err := objects[i].obj.Marshal() diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go index a3bbc021d..21c80b089 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go @@ -13,12 +13,12 @@ import ( // TestControl checks correctness of a read-only mode. // cons must return a storage which is NOT opened. -func TestControl(t *testing.T, cons Constructor, min, max uint64) { +func TestControl(t *testing.T, cons Constructor, minSize, maxSize uint64) { s := cons(t) require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) - objects := prepare(t, 10, s, min, max) + objects := prepare(t, 10, s, minSize, maxSize) require.NoError(t, s.Close()) require.NoError(t, s.Open(mode.ComponentReadOnly)) @@ -34,7 +34,7 @@ func TestControl(t *testing.T, cons Constructor, min, max uint64) { t.Run("put fails", func(t *testing.T) { var prm common.PutPrm - prm.Object = NewObject(min + uint64(rand.Intn(int(max-min+1)))) + prm.Object = NewObject(minSize + uint64(rand.Intn(int(maxSize-minSize+1)))) prm.Address = objectCore.AddressOf(prm.Object) _, err := s.Put(context.Background(), prm) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go index 750619a30..cf4e76513 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go @@ -11,13 +11,13 @@ import ( "github.com/stretchr/testify/require" ) -func TestDelete(t *testing.T, cons Constructor, min, max uint64) { +func TestDelete(t *testing.T, cons Constructor, minSize, maxSize uint64) { s := cons(t) require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() - objects := prepare(t, 4, s, min, max) + objects := prepare(t, 4, s, minSize, maxSize) t.Run("delete non-existent", func(t *testing.T) { var prm common.DeletePrm diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go index 33b50b12f..08465ed5e 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go @@ -10,13 +10,13 @@ import ( "github.com/stretchr/testify/require" ) -func TestExists(t *testing.T, cons Constructor, min, max uint64) { +func TestExists(t *testing.T, cons Constructor, minSize, maxSize uint64) { s := cons(t) require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() - objects := prepare(t, 1, s, min, max) + objects := prepare(t, 1, s, minSize, maxSize) t.Run("missing object", func(t *testing.T) { prm := common.ExistsPrm{Address: oidtest.Address()} diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go index 12f73c3e9..d1f709b0c 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go @@ -11,13 +11,13 @@ import ( "github.com/stretchr/testify/require" ) -func TestGet(t *testing.T, cons Constructor, min, max uint64) { +func TestGet(t *testing.T, cons Constructor, minSize, maxSize uint64) { s := cons(t) require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() - objects := prepare(t, 2, s, min, max) + objects := prepare(t, 2, s, minSize, maxSize) t.Run("missing object", func(t *testing.T) { gPrm := common.GetPrm{Address: oidtest.Address()} diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go index 93de683c2..fcbeddac7 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go @@ -13,13 +13,13 @@ import ( "github.com/stretchr/testify/require" ) -func TestGetRange(t *testing.T, cons Constructor, min, max uint64) { +func TestGetRange(t *testing.T, cons Constructor, minSize, maxSize uint64) { s := cons(t) require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() - objects := prepare(t, 1, s, min, max) + objects := prepare(t, 1, s, minSize, maxSize) t.Run("missing object", func(t *testing.T) { gPrm := common.GetRangePrm{Address: oidtest.Address()} diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index e66fe87b6..3a6c8b699 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -10,13 +10,13 @@ import ( "github.com/stretchr/testify/require" ) -func TestIterate(t *testing.T, cons Constructor, min, max uint64) { +func TestIterate(t *testing.T, cons Constructor, minSize, maxSize uint64) { s := cons(t) require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) defer func() { require.NoError(t, s.Close()) }() - objects := prepare(t, 10, s, min, max) + objects := prepare(t, 10, s, minSize, maxSize) // Delete random object to ensure it is not iterated over. const delID = 2 diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 4865b43ef..616b3b5c3 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -641,8 +641,8 @@ func (c *Client) notaryTxValidationLimit() (uint32, error) { return 0, fmt.Errorf("can't get current blockchain height: %w", err) } - min := bc + c.notary.txValidTime - rounded := (min/c.notary.roundTime + 1) * c.notary.roundTime + minTime := bc + c.notary.txValidTime + rounded := (minTime/c.notary.roundTime + 1) * c.notary.roundTime return rounded, nil } From 7bca428db03716e37e1fec315ef974886a2e9fea Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 19 Aug 2024 18:28:53 +0300 Subject: [PATCH 1286/1943] [#1322] Use new protobuf marshaler Signed-off-by: Evgenii Stratonikov --- Makefile | 8 +- .../modules/control/set_netmap_status.go | 2 +- .../modules/control/shards_set_mode.go | 4 +- cmd/frostfs-cli/modules/control/util.go | 4 +- cmd/frostfs-node/netmap.go | 2 +- go.mod | 7 +- go.sum | 10 +- pkg/local_object_storage/metabase/get.go | 7 +- pkg/services/control/common_test.go | 33 - pkg/services/control/ir/service.go | 46 - pkg/services/control/ir/service.pb.go | 1277 -- pkg/services/control/ir/service_frostfs.pb.go | 2584 ++- pkg/services/control/ir/service_test.go | 44 - pkg/services/control/ir/types.go | 15 - pkg/services/control/ir/types.pb.go | 230 - pkg/services/control/ir/types_frostfs.pb.go | 218 +- pkg/services/control/server/list_shards.go | 4 +- pkg/services/control/service.go | 142 - pkg/services/control/service.pb.go | 7094 -------- pkg/services/control/service_frostfs.pb.go | 14829 ++++++++++++++-- pkg/services/control/service_test.go | 181 - pkg/services/control/types.go | 118 - pkg/services/control/types.pb.go | 1011 -- pkg/services/control/types_frostfs.pb.go | 1721 +- pkg/services/control/types_test.go | 151 - pkg/services/tree/service.pb.go | 3587 ---- pkg/services/tree/service_frostfs.pb.go | 7928 ++++++++- pkg/services/tree/types.pb.go | 320 - pkg/services/tree/types_frostfs.pb.go | 537 +- 29 files changed, 24581 insertions(+), 17533 deletions(-) delete mode 100644 pkg/services/control/common_test.go delete mode 100644 pkg/services/control/ir/service.go delete mode 100644 pkg/services/control/ir/service.pb.go delete mode 100644 pkg/services/control/ir/service_test.go delete mode 100644 pkg/services/control/ir/types.go delete mode 100644 pkg/services/control/ir/types.pb.go delete mode 100644 pkg/services/control/service.go delete mode 100644 pkg/services/control/service.pb.go delete mode 100644 pkg/services/control/service_test.go delete mode 100644 pkg/services/control/types.go delete mode 100644 pkg/services/control/types.pb.go delete mode 100644 pkg/services/control/types_test.go delete mode 100644 pkg/services/tree/service.pb.go delete mode 100644 pkg/services/tree/types.pb.go diff --git a/Makefile b/Makefile index 8d1238b6f..94a8a14c3 100755 --- a/Makefile +++ b/Makefile @@ -11,7 +11,6 @@ GO_VERSION ?= 1.22 LINT_VERSION ?= 1.60.1 TRUECLOUDLAB_LINT_VERSION ?= 0.0.6 PROTOC_VERSION ?= 25.0 -PROTOC_GEN_GO_VERSION ?= $(shell go list -f '{{.Version}}' -m google.golang.org/protobuf) PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2) PROTOC_OS_VERSION=osx-x86_64 ifeq ($(shell uname), Linux) @@ -39,7 +38,6 @@ LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT TMP_DIR := .cache PROTOBUF_DIR ?= $(abspath $(BIN))/protobuf PROTOC_DIR ?= $(PROTOBUF_DIR)/protoc-v$(PROTOC_VERSION) -PROTOC_GEN_GO_DIR ?= $(PROTOBUF_DIR)/protoc-gen-go-$(PROTOC_GEN_GO_VERSION) PROTOGEN_FROSTFS_DIR ?= $(PROTOBUF_DIR)/protogen-$(PROTOGEN_FROSTFS_VERSION) STATICCHECK_DIR ?= $(abspath $(BIN))/staticcheck STATICCHECK_VERSION_DIR ?= $(STATICCHECK_DIR)/$(STATICCHECK_VERSION) @@ -107,17 +105,15 @@ export-metrics: dep # Regenerate proto files: protoc: - @if [ ! -d "$(PROTOC_DIR)" ] || [ ! -d "$(PROTOC_GEN_GO_DIR)" ] || [ ! -d "$(PROTOGEN_FROSTFS_DIR)" ]; then \ + @if [ ! -d "$(PROTOC_DIR)" ] || [ ! -d "$(PROTOGEN_FROSTFS_DIR)" ]; then \ make protoc-install; \ fi @for f in `find . -type f -name '*.proto' -not -path './bin/*'`; do \ echo "⇒ Processing $$f "; \ $(PROTOC_DIR)/bin/protoc \ --proto_path=.:$(PROTOC_DIR)/include:/usr/local/include \ - --plugin=protoc-gen-go=$(PROTOC_GEN_GO_DIR)/protoc-gen-go \ --plugin=protoc-gen-go-frostfs=$(PROTOGEN_FROSTFS_DIR)/protogen \ --go-frostfs_out=. --go-frostfs_opt=paths=source_relative \ - --go_out=. --go_opt=paths=source_relative \ --go-grpc_opt=require_unimplemented_servers=false \ --go-grpc_out=. --go-grpc_opt=paths=source_relative $$f; \ done @@ -130,8 +126,6 @@ protoc-install: @wget -q -O $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip 'https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_OS_VERSION).zip' @unzip -q -o $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip -d $(PROTOC_DIR) @rm $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip - @echo "⇒ Installing protoc-gen-go..." - @GOBIN=$(PROTOC_GEN_GO_DIR) go install -v google.golang.org/protobuf/...@$(PROTOC_GEN_GO_VERSION) @echo "⇒ Instaling protogen FrostFS plugin..." @GOBIN=$(PROTOGEN_FROSTFS_DIR) go install -mod=mod -v git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/protogen@$(PROTOGEN_FROSTFS_VERSION) diff --git a/cmd/frostfs-cli/modules/control/set_netmap_status.go b/cmd/frostfs-cli/modules/control/set_netmap_status.go index 31ade1eb9..a107b2b53 100644 --- a/cmd/frostfs-cli/modules/control/set_netmap_status.go +++ b/cmd/frostfs-cli/modules/control/set_netmap_status.go @@ -84,7 +84,7 @@ func setNetmapStatus(cmd *cobra.Command, _ []string) { body.SetStatus(control.NetmapStatus_MAINTENANCE) if force { - body.SetForceMaintenance() + body.SetForceMaintenance(true) common.PrintVerbose(cmd, "Local maintenance will be forced.") } targetStatus = control.NetmapStatus_MAINTENANCE diff --git a/cmd/frostfs-cli/modules/control/shards_set_mode.go b/cmd/frostfs-cli/modules/control/shards_set_mode.go index e73f15178..dd0d77748 100644 --- a/cmd/frostfs-cli/modules/control/shards_set_mode.go +++ b/cmd/frostfs-cli/modules/control/shards_set_mode.go @@ -117,10 +117,10 @@ func setShardMode(cmd *cobra.Command, _ []string) { req.SetBody(body) body.SetMode(mode) - body.SetShardIDList(getShardIDList(cmd)) + body.SetShard_ID(getShardIDList(cmd)) reset, _ := cmd.Flags().GetBool(shardClearErrorsFlag) - body.ClearErrorCounter(reset) + body.SetResetErrorCounter(reset) signRequest(cmd, pk, req) diff --git a/cmd/frostfs-cli/modules/control/util.go b/cmd/frostfs-cli/modules/control/util.go index c0577ac0c..ef547681f 100644 --- a/cmd/frostfs-cli/modules/control/util.go +++ b/cmd/frostfs-cli/modules/control/util.go @@ -44,7 +44,7 @@ func verifyResponse(cmd *cobra.Command, GetSign() []byte }, body interface { - StableMarshal([]byte) []byte + MarshalProtobuf([]byte) []byte }, ) { if sigControl == nil { @@ -60,7 +60,7 @@ func verifyResponse(cmd *cobra.Command, var sig frostfscrypto.Signature commonCmd.ExitOnErr(cmd, "can't read signature: %w", sig.ReadFromV2(sigV2)) - if !sig.Verify(body.StableMarshal(nil)) { + if !sig.Verify(body.MarshalProtobuf(nil)) { commonCmd.ExitOnErr(cmd, "", errors.New("invalid response signature")) } } diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 128cc3005..8104b1dc1 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -84,7 +84,7 @@ func (s *networkState) setNodeInfo(ni *netmapSDK.NodeInfo) { } } - s.setControlNetmapStatus(ctrlNetSt) + s.setControlNetmapStatus(control.NetmapStatus(ctrlNetSt)) } // sets the current node state to the given value. Subsequent cfg.bootstrap diff --git a/go.mod b/go.mod index 79dcc9553..b665709cc 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,17 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240813155151-d112a28d382f + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240819074700-a43110e36326 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240813155821-98aabc45a720 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240820072028-6dd7be11d13b git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 + github.com/VictoriaMetrics/easyproto v0.1.4 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 @@ -21,6 +22,7 @@ require ( github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/klauspost/compress v1.17.4 + github.com/mailru/easyjson v0.7.7 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.12.1 @@ -84,7 +86,6 @@ require ( github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/klauspost/reedsolomon v1.12.1 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect diff --git a/go.sum b/go.sum index 803a065c3..54e0d0301 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240813155151-d112a28d382f h1:xrJqsXOZeSkBFMSyN+PQ9DiCGxVULU3VIN/tuH/vtb8= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240813155151-d112a28d382f/go.mod h1:mc7j6Cc1GU1tJZNmDwEYiJJ339biNnU1Bz3wZGogMe0= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240819074700-a43110e36326 h1:TkH+NSsY4C/Z8MocIJyMcqLm5vEhZcSowOldJyilKKA= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240819074700-a43110e36326/go.mod h1:zZnHiRv9m5+ESYLhBXY9Jds9A/YIDEUGiuyPUS09HwM= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240813155821-98aabc45a720 h1:15UXpW42bfshIv/X5kww92jG2o0drHgsdFd+UJ6zD7g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240813155821-98aabc45a720/go.mod h1:XRX/bBQsDJKr040N/a0YnDhxJqaUv1XyMVj3qxnb5K0= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240820072028-6dd7be11d13b h1:ZCJBVmQDcdv0twpX9xJU/AQwX+dXyvVfqr0Pq3x+3yk= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240820072028-6dd7be11d13b/go.mod h1:aaC2OR34tVrBwd0Z2gqoN5WLtV/idKqpqPDhb4XqmCo= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= @@ -25,6 +25,8 @@ git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 h1:HeY8n27VyPRQe49l/fzyVMkWEB2fsLJYKp64pwA7tz4= git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02/go.mod h1:rQFJJdEOV7KbbMtQYR2lNfiZk+ONRDJSbMCTWxKt8Fw= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/VictoriaMetrics/easyproto v0.1.4 h1:r8cNvo8o6sR4QShBXQd1bKw/VVLSQma/V2KhTBPf+Sc= +github.com/VictoriaMetrics/easyproto v0.1.4/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index b79f6cb14..d979b4f0f 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -1,6 +1,7 @@ package meta import ( + "bytes" "context" "fmt" "time" @@ -107,7 +108,7 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b // check in primary index data := getFromBucket(tx, primaryBucketName(cnr, bucketName), key) if len(data) != 0 { - return obj, obj.Unmarshal(data) + return obj, obj.Unmarshal(bytes.Clone(data)) } data = getFromBucket(tx, ecInfoBucketName(cnr, bucketName), key) @@ -118,13 +119,13 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b // if not found then check in tombstone index data = getFromBucket(tx, tombstoneBucketName(cnr, bucketName), key) if len(data) != 0 { - return obj, obj.Unmarshal(data) + return obj, obj.Unmarshal(bytes.Clone(data)) } // if not found then check in locker index data = getFromBucket(tx, bucketNameLockers(cnr, bucketName), key) if len(data) != 0 { - return obj, obj.Unmarshal(data) + return obj, obj.Unmarshal(bytes.Clone(data)) } // if not found then check if object is a virtual diff --git a/pkg/services/control/common_test.go b/pkg/services/control/common_test.go deleted file mode 100644 index bc512b4be..000000000 --- a/pkg/services/control/common_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package control_test - -import ( - "crypto/rand" - "testing" - - "github.com/mr-tron/base58" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" -) - -type protoMessage interface { - StableMarshal([]byte) []byte - proto.Message -} - -func testStableMarshal(t *testing.T, m1, m2 protoMessage, cmp func(m1, m2 protoMessage) bool) { - require.NoError(t, proto.Unmarshal(m1.StableMarshal(nil), m2)) - - require.True(t, cmp(m1, m2)) -} - -func testData(sz int) []byte { - d := make([]byte, sz) - - _, _ = rand.Read(d) - - return d -} - -func testString() string { - return base58.Encode(testData(10)) -} diff --git a/pkg/services/control/ir/service.go b/pkg/services/control/ir/service.go deleted file mode 100644 index b2db2b43a..000000000 --- a/pkg/services/control/ir/service.go +++ /dev/null @@ -1,46 +0,0 @@ -package control - -// SetBody sets health check request body. -func (x *HealthCheckRequest) SetBody(v *HealthCheckRequest_Body) { - if x != nil { - x.Body = v - } -} - -// SetHealthStatus sets health status of the IR application. -func (x *HealthCheckResponse_Body) SetHealthStatus(v HealthStatus) { - if x != nil { - x.HealthStatus = v - } -} - -// SetBody sets health check response body. -func (x *HealthCheckResponse) SetBody(v *HealthCheckResponse_Body) { - if x != nil { - x.Body = v - } -} - -func (x *TickEpochRequest) SetBody(v *TickEpochRequest_Body) { - if x != nil { - x.Body = v - } -} - -func (x *TickEpochResponse) SetBody(v *TickEpochResponse_Body) { - if x != nil { - x.Body = v - } -} - -func (x *RemoveNodeRequest) SetBody(v *RemoveNodeRequest_Body) { - if x != nil { - x.Body = v - } -} - -func (x *RemoveNodeResponse) SetBody(v *RemoveNodeResponse_Body) { - if x != nil { - x.Body = v - } -} diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go deleted file mode 100644 index d1e253bf5..000000000 --- a/pkg/services/control/ir/service.pb.go +++ /dev/null @@ -1,1277 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.33.0 -// protoc v4.25.0 -// source: pkg/services/control/ir/service.proto - -package control - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// Health check request. -type HealthCheckRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of health check request message. - Body *HealthCheckRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - // Should be signed by node key or one of - // the keys configured by the node. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *HealthCheckRequest) Reset() { - *x = HealthCheckRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthCheckRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthCheckRequest) ProtoMessage() {} - -func (x *HealthCheckRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthCheckRequest.ProtoReflect.Descriptor instead. -func (*HealthCheckRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{0} -} - -func (x *HealthCheckRequest) GetBody() *HealthCheckRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *HealthCheckRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Health check response. -type HealthCheckResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of health check response message. - Body *HealthCheckResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *HealthCheckResponse) Reset() { - *x = HealthCheckResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthCheckResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthCheckResponse) ProtoMessage() {} - -func (x *HealthCheckResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthCheckResponse.ProtoReflect.Descriptor instead. -func (*HealthCheckResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{1} -} - -func (x *HealthCheckResponse) GetBody() *HealthCheckResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *HealthCheckResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type TickEpochRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *TickEpochRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *TickEpochRequest) Reset() { - *x = TickEpochRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TickEpochRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TickEpochRequest) ProtoMessage() {} - -func (x *TickEpochRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TickEpochRequest.ProtoReflect.Descriptor instead. -func (*TickEpochRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{2} -} - -func (x *TickEpochRequest) GetBody() *TickEpochRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *TickEpochRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type TickEpochResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *TickEpochResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *TickEpochResponse) Reset() { - *x = TickEpochResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TickEpochResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TickEpochResponse) ProtoMessage() {} - -func (x *TickEpochResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TickEpochResponse.ProtoReflect.Descriptor instead. -func (*TickEpochResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{3} -} - -func (x *TickEpochResponse) GetBody() *TickEpochResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *TickEpochResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type RemoveNodeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *RemoveNodeRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RemoveNodeRequest) Reset() { - *x = RemoveNodeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveNodeRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveNodeRequest) ProtoMessage() {} - -func (x *RemoveNodeRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveNodeRequest.ProtoReflect.Descriptor instead. -func (*RemoveNodeRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{4} -} - -func (x *RemoveNodeRequest) GetBody() *RemoveNodeRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RemoveNodeRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type RemoveNodeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *RemoveNodeResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RemoveNodeResponse) Reset() { - *x = RemoveNodeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveNodeResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveNodeResponse) ProtoMessage() {} - -func (x *RemoveNodeResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveNodeResponse.ProtoReflect.Descriptor instead. -func (*RemoveNodeResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{5} -} - -func (x *RemoveNodeResponse) GetBody() *RemoveNodeResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RemoveNodeResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type RemoveContainerRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *RemoveContainerRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RemoveContainerRequest) Reset() { - *x = RemoveContainerRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveContainerRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveContainerRequest) ProtoMessage() {} - -func (x *RemoveContainerRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveContainerRequest.ProtoReflect.Descriptor instead. -func (*RemoveContainerRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{6} -} - -func (x *RemoveContainerRequest) GetBody() *RemoveContainerRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RemoveContainerRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type RemoveContainerResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *RemoveContainerResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RemoveContainerResponse) Reset() { - *x = RemoveContainerResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveContainerResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveContainerResponse) ProtoMessage() {} - -func (x *RemoveContainerResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveContainerResponse.ProtoReflect.Descriptor instead. -func (*RemoveContainerResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{7} -} - -func (x *RemoveContainerResponse) GetBody() *RemoveContainerResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RemoveContainerResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Health check request body. -type HealthCheckRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *HealthCheckRequest_Body) Reset() { - *x = HealthCheckRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthCheckRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthCheckRequest_Body) ProtoMessage() {} - -func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthCheckRequest_Body.ProtoReflect.Descriptor instead. -func (*HealthCheckRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{0, 0} -} - -// Health check response body -type HealthCheckResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Health status of IR node application. - HealthStatus HealthStatus `protobuf:"varint,1,opt,name=health_status,json=healthStatus,proto3,enum=ircontrol.HealthStatus" json:"health_status,omitempty"` -} - -func (x *HealthCheckResponse_Body) Reset() { - *x = HealthCheckResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthCheckResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthCheckResponse_Body) ProtoMessage() {} - -func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthCheckResponse_Body.ProtoReflect.Descriptor instead. -func (*HealthCheckResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{1, 0} -} - -func (x *HealthCheckResponse_Body) GetHealthStatus() HealthStatus { - if x != nil { - return x.HealthStatus - } - return HealthStatus_HEALTH_STATUS_UNDEFINED -} - -type TickEpochRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Valid until block value override. - Vub uint32 `protobuf:"varint,1,opt,name=vub,proto3" json:"vub,omitempty"` -} - -func (x *TickEpochRequest_Body) Reset() { - *x = TickEpochRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TickEpochRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TickEpochRequest_Body) ProtoMessage() {} - -func (x *TickEpochRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TickEpochRequest_Body.ProtoReflect.Descriptor instead. -func (*TickEpochRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{2, 0} -} - -func (x *TickEpochRequest_Body) GetVub() uint32 { - if x != nil { - return x.Vub - } - return 0 -} - -type TickEpochResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Valid until block value for transaction. - Vub uint32 `protobuf:"varint,1,opt,name=vub,proto3" json:"vub,omitempty"` -} - -func (x *TickEpochResponse_Body) Reset() { - *x = TickEpochResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TickEpochResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TickEpochResponse_Body) ProtoMessage() {} - -func (x *TickEpochResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TickEpochResponse_Body.ProtoReflect.Descriptor instead. -func (*TickEpochResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{3, 0} -} - -func (x *TickEpochResponse_Body) GetVub() uint32 { - if x != nil { - return x.Vub - } - return 0 -} - -type RemoveNodeRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - // Valid until block value override. - Vub uint32 `protobuf:"varint,2,opt,name=vub,proto3" json:"vub,omitempty"` -} - -func (x *RemoveNodeRequest_Body) Reset() { - *x = RemoveNodeRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveNodeRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveNodeRequest_Body) ProtoMessage() {} - -func (x *RemoveNodeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveNodeRequest_Body.ProtoReflect.Descriptor instead. -func (*RemoveNodeRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{4, 0} -} - -func (x *RemoveNodeRequest_Body) GetKey() []byte { - if x != nil { - return x.Key - } - return nil -} - -func (x *RemoveNodeRequest_Body) GetVub() uint32 { - if x != nil { - return x.Vub - } - return 0 -} - -type RemoveNodeResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Valid until block value for transaction. - Vub uint32 `protobuf:"varint,1,opt,name=vub,proto3" json:"vub,omitempty"` -} - -func (x *RemoveNodeResponse_Body) Reset() { - *x = RemoveNodeResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveNodeResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveNodeResponse_Body) ProtoMessage() {} - -func (x *RemoveNodeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveNodeResponse_Body.ProtoReflect.Descriptor instead. -func (*RemoveNodeResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{5, 0} -} - -func (x *RemoveNodeResponse_Body) GetVub() uint32 { - if x != nil { - return x.Vub - } - return 0 -} - -type RemoveContainerRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` - Owner []byte `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` - // Valid until block value override. - Vub uint32 `protobuf:"varint,3,opt,name=vub,proto3" json:"vub,omitempty"` -} - -func (x *RemoveContainerRequest_Body) Reset() { - *x = RemoveContainerRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[14] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveContainerRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveContainerRequest_Body) ProtoMessage() {} - -func (x *RemoveContainerRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveContainerRequest_Body.ProtoReflect.Descriptor instead. -func (*RemoveContainerRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{6, 0} -} - -func (x *RemoveContainerRequest_Body) GetContainerId() []byte { - if x != nil { - return x.ContainerId - } - return nil -} - -func (x *RemoveContainerRequest_Body) GetOwner() []byte { - if x != nil { - return x.Owner - } - return nil -} - -func (x *RemoveContainerRequest_Body) GetVub() uint32 { - if x != nil { - return x.Vub - } - return 0 -} - -type RemoveContainerResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Valid until block value for transaction. - Vub uint32 `protobuf:"varint,1,opt,name=vub,proto3" json:"vub,omitempty"` -} - -func (x *RemoveContainerResponse_Body) Reset() { - *x = RemoveContainerResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[15] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveContainerResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveContainerResponse_Body) ProtoMessage() {} - -func (x *RemoveContainerResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_service_proto_msgTypes[15] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveContainerResponse_Body.ProtoReflect.Descriptor instead. -func (*RemoveContainerResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_service_proto_rawDescGZIP(), []int{7, 0} -} - -func (x *RemoveContainerResponse_Body) GetVub() uint32 { - if x != nil { - return x.Vub - } - return 0 -} - -var File_pkg_services_control_ir_service_proto protoreflect.FileDescriptor - -var file_pkg_services_control_ir_service_proto_rawDesc = []byte{ - 0x0a, 0x25, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x69, 0x72, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x1a, 0x23, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x69, 0x72, 0x2f, 0x74, 0x79, 0x70, 0x65, - 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x88, 0x01, 0x0a, 0x12, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x36, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x69, - 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, - 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x22, 0xc8, 0x01, 0x0a, 0x13, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x44, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x3c, 0x0a, 0x0d, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x0c, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x96, 0x01, - 0x0a, 0x10, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x20, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, - 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x18, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x75, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x03, 0x76, 0x75, 0x62, 0x22, 0x98, 0x01, 0x0a, 0x11, 0x54, 0x69, 0x63, 0x6b, 0x45, - 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x69, 0x72, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x18, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x76, 0x75, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x76, 0x75, - 0x62, 0x22, 0xaa, 0x01, 0x0a, 0x11, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, - 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x2a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x76, 0x75, 0x62, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x76, 0x75, 0x62, 0x22, 0x9a, - 0x01, 0x0a, 0x12, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x32, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x18, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x75, 0x62, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x76, 0x75, 0x62, 0x22, 0xdb, 0x01, 0x0a, 0x16, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x51, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, - 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, - 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x75, 0x62, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x76, 0x75, 0x62, 0x22, 0xa4, 0x01, 0x0a, 0x17, 0x52, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x32, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x18, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x76, 0x75, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x76, 0x75, 0x62, - 0x32, 0xcb, 0x02, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x12, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1e, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, - 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x46, 0x0a, 0x09, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x1b, - 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, - 0x70, 0x6f, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x69, 0x72, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x45, 0x70, 0x6f, 0x63, - 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x0a, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x0f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x21, 0x2e, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x69, 0x72, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x44, - 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, - 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, - 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, - 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x69, 0x72, 0x2f, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_pkg_services_control_ir_service_proto_rawDescOnce sync.Once - file_pkg_services_control_ir_service_proto_rawDescData = file_pkg_services_control_ir_service_proto_rawDesc -) - -func file_pkg_services_control_ir_service_proto_rawDescGZIP() []byte { - file_pkg_services_control_ir_service_proto_rawDescOnce.Do(func() { - file_pkg_services_control_ir_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_services_control_ir_service_proto_rawDescData) - }) - return file_pkg_services_control_ir_service_proto_rawDescData -} - -var file_pkg_services_control_ir_service_proto_msgTypes = make([]protoimpl.MessageInfo, 16) -var file_pkg_services_control_ir_service_proto_goTypes = []interface{}{ - (*HealthCheckRequest)(nil), // 0: ircontrol.HealthCheckRequest - (*HealthCheckResponse)(nil), // 1: ircontrol.HealthCheckResponse - (*TickEpochRequest)(nil), // 2: ircontrol.TickEpochRequest - (*TickEpochResponse)(nil), // 3: ircontrol.TickEpochResponse - (*RemoveNodeRequest)(nil), // 4: ircontrol.RemoveNodeRequest - (*RemoveNodeResponse)(nil), // 5: ircontrol.RemoveNodeResponse - (*RemoveContainerRequest)(nil), // 6: ircontrol.RemoveContainerRequest - (*RemoveContainerResponse)(nil), // 7: ircontrol.RemoveContainerResponse - (*HealthCheckRequest_Body)(nil), // 8: ircontrol.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 9: ircontrol.HealthCheckResponse.Body - (*TickEpochRequest_Body)(nil), // 10: ircontrol.TickEpochRequest.Body - (*TickEpochResponse_Body)(nil), // 11: ircontrol.TickEpochResponse.Body - (*RemoveNodeRequest_Body)(nil), // 12: ircontrol.RemoveNodeRequest.Body - (*RemoveNodeResponse_Body)(nil), // 13: ircontrol.RemoveNodeResponse.Body - (*RemoveContainerRequest_Body)(nil), // 14: ircontrol.RemoveContainerRequest.Body - (*RemoveContainerResponse_Body)(nil), // 15: ircontrol.RemoveContainerResponse.Body - (*Signature)(nil), // 16: ircontrol.Signature - (HealthStatus)(0), // 17: ircontrol.HealthStatus -} -var file_pkg_services_control_ir_service_proto_depIdxs = []int32{ - 8, // 0: ircontrol.HealthCheckRequest.body:type_name -> ircontrol.HealthCheckRequest.Body - 16, // 1: ircontrol.HealthCheckRequest.signature:type_name -> ircontrol.Signature - 9, // 2: ircontrol.HealthCheckResponse.body:type_name -> ircontrol.HealthCheckResponse.Body - 16, // 3: ircontrol.HealthCheckResponse.signature:type_name -> ircontrol.Signature - 10, // 4: ircontrol.TickEpochRequest.body:type_name -> ircontrol.TickEpochRequest.Body - 16, // 5: ircontrol.TickEpochRequest.signature:type_name -> ircontrol.Signature - 11, // 6: ircontrol.TickEpochResponse.body:type_name -> ircontrol.TickEpochResponse.Body - 16, // 7: ircontrol.TickEpochResponse.signature:type_name -> ircontrol.Signature - 12, // 8: ircontrol.RemoveNodeRequest.body:type_name -> ircontrol.RemoveNodeRequest.Body - 16, // 9: ircontrol.RemoveNodeRequest.signature:type_name -> ircontrol.Signature - 13, // 10: ircontrol.RemoveNodeResponse.body:type_name -> ircontrol.RemoveNodeResponse.Body - 16, // 11: ircontrol.RemoveNodeResponse.signature:type_name -> ircontrol.Signature - 14, // 12: ircontrol.RemoveContainerRequest.body:type_name -> ircontrol.RemoveContainerRequest.Body - 16, // 13: ircontrol.RemoveContainerRequest.signature:type_name -> ircontrol.Signature - 15, // 14: ircontrol.RemoveContainerResponse.body:type_name -> ircontrol.RemoveContainerResponse.Body - 16, // 15: ircontrol.RemoveContainerResponse.signature:type_name -> ircontrol.Signature - 17, // 16: ircontrol.HealthCheckResponse.Body.health_status:type_name -> ircontrol.HealthStatus - 0, // 17: ircontrol.ControlService.HealthCheck:input_type -> ircontrol.HealthCheckRequest - 2, // 18: ircontrol.ControlService.TickEpoch:input_type -> ircontrol.TickEpochRequest - 4, // 19: ircontrol.ControlService.RemoveNode:input_type -> ircontrol.RemoveNodeRequest - 6, // 20: ircontrol.ControlService.RemoveContainer:input_type -> ircontrol.RemoveContainerRequest - 1, // 21: ircontrol.ControlService.HealthCheck:output_type -> ircontrol.HealthCheckResponse - 3, // 22: ircontrol.ControlService.TickEpoch:output_type -> ircontrol.TickEpochResponse - 5, // 23: ircontrol.ControlService.RemoveNode:output_type -> ircontrol.RemoveNodeResponse - 7, // 24: ircontrol.ControlService.RemoveContainer:output_type -> ircontrol.RemoveContainerResponse - 21, // [21:25] is the sub-list for method output_type - 17, // [17:21] is the sub-list for method input_type - 17, // [17:17] is the sub-list for extension type_name - 17, // [17:17] is the sub-list for extension extendee - 0, // [0:17] is the sub-list for field type_name -} - -func init() { file_pkg_services_control_ir_service_proto_init() } -func file_pkg_services_control_ir_service_proto_init() { - if File_pkg_services_control_ir_service_proto != nil { - return - } - file_pkg_services_control_ir_types_proto_init() - if !protoimpl.UnsafeEnabled { - file_pkg_services_control_ir_service_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TickEpochRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TickEpochResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveNodeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveNodeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveContainerRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveContainerResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TickEpochRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TickEpochResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveNodeRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveNodeResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveContainerRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_ir_service_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveContainerResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_pkg_services_control_ir_service_proto_rawDesc, - NumEnums: 0, - NumMessages: 16, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_pkg_services_control_ir_service_proto_goTypes, - DependencyIndexes: file_pkg_services_control_ir_service_proto_depIdxs, - MessageInfos: file_pkg_services_control_ir_service_proto_msgTypes, - }.Build() - File_pkg_services_control_ir_service_proto = out.File - file_pkg_services_control_ir_service_proto_rawDesc = nil - file_pkg_services_control_ir_service_proto_goTypes = nil - file_pkg_services_control_ir_service_proto_depIdxs = nil -} diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index e22d0013f..786095802 100644 --- a/pkg/services/control/ir/service_frostfs.pb.go +++ b/pkg/services/control/ir/service_frostfs.pb.go @@ -2,7 +2,27 @@ package control -import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" +import ( + json "encoding/json" + fmt "fmt" + pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + easyproto "github.com/VictoriaMetrics/easyproto" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" + strconv "strconv" +) + +type HealthCheckRequest_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*HealthCheckRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*HealthCheckRequest_Body)(nil) + _ json.Marshaler = (*HealthCheckRequest_Body)(nil) + _ json.Unmarshaler = (*HealthCheckRequest_Body)(nil) +) // StableSize returns the size of x in protobuf format. // @@ -14,18 +34,93 @@ func (x *HealthCheckRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthCheckRequest_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthCheckRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *HealthCheckRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthCheckRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthCheckRequest_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthCheckRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthCheckRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthCheckRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthCheckRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type HealthCheckRequest struct { + Body *HealthCheckRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*HealthCheckRequest)(nil) + _ encoding.ProtoUnmarshaler = (*HealthCheckRequest)(nil) + _ json.Marshaler = (*HealthCheckRequest)(nil) + _ json.Unmarshaler = (*HealthCheckRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -38,27 +133,6 @@ func (x *HealthCheckRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthCheckRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -74,13 +148,164 @@ func (x *HealthCheckRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *HealthCheckRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *HealthCheckRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthCheckRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *HealthCheckRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthCheckRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthCheckRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(HealthCheckRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *HealthCheckRequest) GetBody() *HealthCheckRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *HealthCheckRequest) SetBody(v *HealthCheckRequest_Body) { + x.Body = v +} +func (x *HealthCheckRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *HealthCheckRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthCheckRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthCheckRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthCheckRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthCheckRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *HealthCheckRequest_Body + f = new(HealthCheckRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type HealthCheckResponse_Body struct { + HealthStatus HealthStatus `json:"healthStatus"` +} + +var ( + _ encoding.ProtoMarshaler = (*HealthCheckResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*HealthCheckResponse_Body)(nil) + _ json.Marshaler = (*HealthCheckResponse_Body)(nil) + _ json.Unmarshaler = (*HealthCheckResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -92,26 +317,141 @@ func (x *HealthCheckResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthCheckResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.EnumMarshal(1, buf[offset:], int32(x.HealthStatus)) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthCheckResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *HealthCheckResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if int32(x.HealthStatus) != 0 { + mm.AppendInt32(1, int32(x.HealthStatus)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthCheckResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthCheckResponse_Body") + } + switch fc.FieldNum { + case 1: // HealthStatus + data, ok := fc.Int32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "HealthStatus") + } + x.HealthStatus = HealthStatus(data) + } + } + return nil +} +func (x *HealthCheckResponse_Body) GetHealthStatus() HealthStatus { + if x != nil { + return x.HealthStatus + } + return 0 +} +func (x *HealthCheckResponse_Body) SetHealthStatus(v HealthStatus) { + x.HealthStatus = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthCheckResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthCheckResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"healthStatus\":" + out.RawString(prefix[1:]) + out.Int32(int32(x.HealthStatus)) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthCheckResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthCheckResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "healthStatus": + { + var f HealthStatus + var parsedValue HealthStatus + switch v := in.Interface().(type) { + case string: + if vv, ok := HealthStatus_value[v]; ok { + parsedValue = HealthStatus(vv) + break + } + vv, err := strconv.ParseInt(v, 10, 32) + if err != nil { + in.AddError(err) + return + } + parsedValue = HealthStatus(vv) + case float64: + parsedValue = HealthStatus(v) + } + f = parsedValue + x.HealthStatus = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type HealthCheckResponse struct { + Body *HealthCheckResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*HealthCheckResponse)(nil) + _ encoding.ProtoUnmarshaler = (*HealthCheckResponse)(nil) + _ json.Marshaler = (*HealthCheckResponse)(nil) + _ json.Unmarshaler = (*HealthCheckResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -124,27 +464,6 @@ func (x *HealthCheckResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthCheckResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -160,13 +479,164 @@ func (x *HealthCheckResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *HealthCheckResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *HealthCheckResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthCheckResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *HealthCheckResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthCheckResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthCheckResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(HealthCheckResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *HealthCheckResponse) GetBody() *HealthCheckResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *HealthCheckResponse) SetBody(v *HealthCheckResponse_Body) { + x.Body = v +} +func (x *HealthCheckResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *HealthCheckResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthCheckResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthCheckResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthCheckResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthCheckResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *HealthCheckResponse_Body + f = new(HealthCheckResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type TickEpochRequest_Body struct { + Vub uint32 `json:"vub"` +} + +var ( + _ encoding.ProtoMarshaler = (*TickEpochRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*TickEpochRequest_Body)(nil) + _ json.Marshaler = (*TickEpochRequest_Body)(nil) + _ json.Unmarshaler = (*TickEpochRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -178,26 +648,125 @@ func (x *TickEpochRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *TickEpochRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.UInt32Marshal(1, buf[offset:], x.Vub) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *TickEpochRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *TickEpochRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Vub != 0 { + mm.AppendUint32(1, x.Vub) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *TickEpochRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "TickEpochRequest_Body") + } + switch fc.FieldNum { + case 1: // Vub + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Vub") + } + x.Vub = data + } + } + return nil +} +func (x *TickEpochRequest_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} +func (x *TickEpochRequest_Body) SetVub(v uint32) { + x.Vub = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *TickEpochRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *TickEpochRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"vub\":" + out.RawString(prefix[1:]) + out.Uint32(x.Vub) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *TickEpochRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *TickEpochRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "vub": + { + var f uint32 + f = in.Uint32() + x.Vub = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type TickEpochRequest struct { + Body *TickEpochRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*TickEpochRequest)(nil) + _ encoding.ProtoUnmarshaler = (*TickEpochRequest)(nil) + _ json.Marshaler = (*TickEpochRequest)(nil) + _ json.Unmarshaler = (*TickEpochRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -210,27 +779,6 @@ func (x *TickEpochRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *TickEpochRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -246,13 +794,164 @@ func (x *TickEpochRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *TickEpochRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *TickEpochRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *TickEpochRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *TickEpochRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *TickEpochRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "TickEpochRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(TickEpochRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *TickEpochRequest) GetBody() *TickEpochRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *TickEpochRequest) SetBody(v *TickEpochRequest_Body) { + x.Body = v +} +func (x *TickEpochRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *TickEpochRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *TickEpochRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *TickEpochRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *TickEpochRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *TickEpochRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *TickEpochRequest_Body + f = new(TickEpochRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type TickEpochResponse_Body struct { + Vub uint32 `json:"vub"` +} + +var ( + _ encoding.ProtoMarshaler = (*TickEpochResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*TickEpochResponse_Body)(nil) + _ json.Marshaler = (*TickEpochResponse_Body)(nil) + _ json.Unmarshaler = (*TickEpochResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -264,26 +963,125 @@ func (x *TickEpochResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *TickEpochResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.UInt32Marshal(1, buf[offset:], x.Vub) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *TickEpochResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *TickEpochResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Vub != 0 { + mm.AppendUint32(1, x.Vub) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *TickEpochResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "TickEpochResponse_Body") + } + switch fc.FieldNum { + case 1: // Vub + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Vub") + } + x.Vub = data + } + } + return nil +} +func (x *TickEpochResponse_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} +func (x *TickEpochResponse_Body) SetVub(v uint32) { + x.Vub = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *TickEpochResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *TickEpochResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"vub\":" + out.RawString(prefix[1:]) + out.Uint32(x.Vub) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *TickEpochResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *TickEpochResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "vub": + { + var f uint32 + f = in.Uint32() + x.Vub = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type TickEpochResponse struct { + Body *TickEpochResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*TickEpochResponse)(nil) + _ encoding.ProtoUnmarshaler = (*TickEpochResponse)(nil) + _ json.Marshaler = (*TickEpochResponse)(nil) + _ json.Unmarshaler = (*TickEpochResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -296,27 +1094,6 @@ func (x *TickEpochResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *TickEpochResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -332,13 +1109,165 @@ func (x *TickEpochResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *TickEpochResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *TickEpochResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *TickEpochResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *TickEpochResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *TickEpochResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "TickEpochResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(TickEpochResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *TickEpochResponse) GetBody() *TickEpochResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *TickEpochResponse) SetBody(v *TickEpochResponse_Body) { + x.Body = v +} +func (x *TickEpochResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *TickEpochResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *TickEpochResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *TickEpochResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *TickEpochResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *TickEpochResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *TickEpochResponse_Body + f = new(TickEpochResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveNodeRequest_Body struct { + Key []byte `json:"key"` + Vub uint32 `json:"vub"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveNodeRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveNodeRequest_Body)(nil) + _ json.Marshaler = (*RemoveNodeRequest_Body)(nil) + _ json.Unmarshaler = (*RemoveNodeRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -351,27 +1280,154 @@ func (x *RemoveNodeRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveNodeRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.Key) - offset += proto.UInt32Marshal(2, buf[offset:], x.Vub) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveNodeRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveNodeRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Key) != 0 { + mm.AppendBytes(1, x.Key) + } + if x.Vub != 0 { + mm.AppendUint32(2, x.Vub) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveNodeRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveNodeRequest_Body") + } + switch fc.FieldNum { + case 1: // Key + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Key") + } + x.Key = data + case 2: // Vub + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Vub") + } + x.Vub = data + } + } + return nil +} +func (x *RemoveNodeRequest_Body) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} +func (x *RemoveNodeRequest_Body) SetKey(v []byte) { + x.Key = v +} +func (x *RemoveNodeRequest_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} +func (x *RemoveNodeRequest_Body) SetVub(v uint32) { + x.Vub = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveNodeRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveNodeRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"key\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.Key) + } + { + const prefix string = ",\"vub\":" + out.RawString(prefix) + out.Uint32(x.Vub) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveNodeRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveNodeRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "key": + { + var f []byte + f = in.Bytes() + x.Key = f + } + case "vub": + { + var f uint32 + f = in.Uint32() + x.Vub = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveNodeRequest struct { + Body *RemoveNodeRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveNodeRequest)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveNodeRequest)(nil) + _ json.Marshaler = (*RemoveNodeRequest)(nil) + _ json.Unmarshaler = (*RemoveNodeRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -384,27 +1440,6 @@ func (x *RemoveNodeRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveNodeRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -420,13 +1455,164 @@ func (x *RemoveNodeRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *RemoveNodeRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *RemoveNodeRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveNodeRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveNodeRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveNodeRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveNodeRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(RemoveNodeRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *RemoveNodeRequest) GetBody() *RemoveNodeRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *RemoveNodeRequest) SetBody(v *RemoveNodeRequest_Body) { + x.Body = v +} +func (x *RemoveNodeRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *RemoveNodeRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveNodeRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveNodeRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveNodeRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveNodeRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *RemoveNodeRequest_Body + f = new(RemoveNodeRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveNodeResponse_Body struct { + Vub uint32 `json:"vub"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveNodeResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveNodeResponse_Body)(nil) + _ json.Marshaler = (*RemoveNodeResponse_Body)(nil) + _ json.Unmarshaler = (*RemoveNodeResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -438,26 +1624,125 @@ func (x *RemoveNodeResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveNodeResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.UInt32Marshal(1, buf[offset:], x.Vub) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveNodeResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveNodeResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Vub != 0 { + mm.AppendUint32(1, x.Vub) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveNodeResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveNodeResponse_Body") + } + switch fc.FieldNum { + case 1: // Vub + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Vub") + } + x.Vub = data + } + } + return nil +} +func (x *RemoveNodeResponse_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} +func (x *RemoveNodeResponse_Body) SetVub(v uint32) { + x.Vub = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveNodeResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveNodeResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"vub\":" + out.RawString(prefix[1:]) + out.Uint32(x.Vub) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveNodeResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveNodeResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "vub": + { + var f uint32 + f = in.Uint32() + x.Vub = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveNodeResponse struct { + Body *RemoveNodeResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveNodeResponse)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveNodeResponse)(nil) + _ json.Marshaler = (*RemoveNodeResponse)(nil) + _ json.Unmarshaler = (*RemoveNodeResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -470,27 +1755,6 @@ func (x *RemoveNodeResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveNodeResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -506,13 +1770,166 @@ func (x *RemoveNodeResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *RemoveNodeResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *RemoveNodeResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveNodeResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveNodeResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveNodeResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveNodeResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(RemoveNodeResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *RemoveNodeResponse) GetBody() *RemoveNodeResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *RemoveNodeResponse) SetBody(v *RemoveNodeResponse_Body) { + x.Body = v +} +func (x *RemoveNodeResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *RemoveNodeResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveNodeResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveNodeResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveNodeResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveNodeResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *RemoveNodeResponse_Body + f = new(RemoveNodeResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveContainerRequest_Body struct { + ContainerId []byte `json:"containerId"` + Owner []byte `json:"owner"` + Vub uint32 `json:"vub"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveContainerRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveContainerRequest_Body)(nil) + _ json.Marshaler = (*RemoveContainerRequest_Body)(nil) + _ json.Unmarshaler = (*RemoveContainerRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -526,28 +1943,183 @@ func (x *RemoveContainerRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveContainerRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) - offset += proto.BytesMarshal(2, buf[offset:], x.Owner) - offset += proto.UInt32Marshal(3, buf[offset:], x.Vub) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveContainerRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveContainerRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ContainerId) != 0 { + mm.AppendBytes(1, x.ContainerId) + } + if len(x.Owner) != 0 { + mm.AppendBytes(2, x.Owner) + } + if x.Vub != 0 { + mm.AppendUint32(3, x.Vub) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveContainerRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveContainerRequest_Body") + } + switch fc.FieldNum { + case 1: // ContainerId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + case 2: // Owner + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Owner") + } + x.Owner = data + case 3: // Vub + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Vub") + } + x.Vub = data + } + } + return nil +} +func (x *RemoveContainerRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} +func (x *RemoveContainerRequest_Body) SetContainerId(v []byte) { + x.ContainerId = v +} +func (x *RemoveContainerRequest_Body) GetOwner() []byte { + if x != nil { + return x.Owner + } + return nil +} +func (x *RemoveContainerRequest_Body) SetOwner(v []byte) { + x.Owner = v +} +func (x *RemoveContainerRequest_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} +func (x *RemoveContainerRequest_Body) SetVub(v uint32) { + x.Vub = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveContainerRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveContainerRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"containerId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ContainerId) + } + { + const prefix string = ",\"owner\":" + out.RawString(prefix) + out.Base64Bytes(x.Owner) + } + { + const prefix string = ",\"vub\":" + out.RawString(prefix) + out.Uint32(x.Vub) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveContainerRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveContainerRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "containerId": + { + var f []byte + f = in.Bytes() + x.ContainerId = f + } + case "owner": + { + var f []byte + f = in.Bytes() + x.Owner = f + } + case "vub": + { + var f uint32 + f = in.Uint32() + x.Vub = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveContainerRequest struct { + Body *RemoveContainerRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveContainerRequest)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveContainerRequest)(nil) + _ json.Marshaler = (*RemoveContainerRequest)(nil) + _ json.Unmarshaler = (*RemoveContainerRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -560,27 +2132,6 @@ func (x *RemoveContainerRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveContainerRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -596,13 +2147,164 @@ func (x *RemoveContainerRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *RemoveContainerRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *RemoveContainerRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveContainerRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveContainerRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveContainerRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveContainerRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(RemoveContainerRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *RemoveContainerRequest) GetBody() *RemoveContainerRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *RemoveContainerRequest) SetBody(v *RemoveContainerRequest_Body) { + x.Body = v +} +func (x *RemoveContainerRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *RemoveContainerRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveContainerRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveContainerRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveContainerRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveContainerRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *RemoveContainerRequest_Body + f = new(RemoveContainerRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveContainerResponse_Body struct { + Vub uint32 `json:"vub"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveContainerResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveContainerResponse_Body)(nil) + _ json.Marshaler = (*RemoveContainerResponse_Body)(nil) + _ json.Unmarshaler = (*RemoveContainerResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -614,26 +2316,125 @@ func (x *RemoveContainerResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveContainerResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.UInt32Marshal(1, buf[offset:], x.Vub) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveContainerResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveContainerResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Vub != 0 { + mm.AppendUint32(1, x.Vub) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveContainerResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveContainerResponse_Body") + } + switch fc.FieldNum { + case 1: // Vub + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Vub") + } + x.Vub = data + } + } + return nil +} +func (x *RemoveContainerResponse_Body) GetVub() uint32 { + if x != nil { + return x.Vub + } + return 0 +} +func (x *RemoveContainerResponse_Body) SetVub(v uint32) { + x.Vub = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveContainerResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveContainerResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"vub\":" + out.RawString(prefix[1:]) + out.Uint32(x.Vub) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveContainerResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveContainerResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "vub": + { + var f uint32 + f = in.Uint32() + x.Vub = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveContainerResponse struct { + Body *RemoveContainerResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveContainerResponse)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveContainerResponse)(nil) + _ json.Marshaler = (*RemoveContainerResponse)(nil) + _ json.Unmarshaler = (*RemoveContainerResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -646,27 +2447,6 @@ func (x *RemoveContainerResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveContainerResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -682,9 +2462,149 @@ func (x *RemoveContainerResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *RemoveContainerResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *RemoveContainerResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveContainerResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *RemoveContainerResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveContainerResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveContainerResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(RemoveContainerResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *RemoveContainerResponse) GetBody() *RemoveContainerResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *RemoveContainerResponse) SetBody(v *RemoveContainerResponse_Body) { + x.Body = v +} +func (x *RemoveContainerResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *RemoveContainerResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveContainerResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveContainerResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveContainerResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveContainerResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *RemoveContainerResponse_Body + f = new(RemoveContainerResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } } diff --git a/pkg/services/control/ir/service_test.go b/pkg/services/control/ir/service_test.go deleted file mode 100644 index 54eef5148..000000000 --- a/pkg/services/control/ir/service_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package control_test - -import ( - "testing" - - control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" -) - -type protoMessage interface { - StableMarshal([]byte) []byte - proto.Message -} - -func testStableMarshal(t *testing.T, m1, m2 protoMessage, cmp func(m1, m2 protoMessage) bool) { - require.NoError(t, proto.Unmarshal(m1.StableMarshal(nil), m2)) - - require.True(t, cmp(m1, m2)) -} - -func TestHealthCheckResponse_Body_StableMarshal(t *testing.T) { - testStableMarshal(t, - generateHealthCheckResponseBody(), - new(control.HealthCheckResponse_Body), - func(m1, m2 protoMessage) bool { - return equalHealthCheckResponseBodies( - m1.(*control.HealthCheckResponse_Body), - m2.(*control.HealthCheckResponse_Body), - ) - }, - ) -} - -func generateHealthCheckResponseBody() *control.HealthCheckResponse_Body { - body := new(control.HealthCheckResponse_Body) - body.SetHealthStatus(control.HealthStatus_SHUTTING_DOWN) - - return body -} - -func equalHealthCheckResponseBodies(b1, b2 *control.HealthCheckResponse_Body) bool { - return b1.GetHealthStatus() == b2.GetHealthStatus() -} diff --git a/pkg/services/control/ir/types.go b/pkg/services/control/ir/types.go deleted file mode 100644 index 97ffd3ce3..000000000 --- a/pkg/services/control/ir/types.go +++ /dev/null @@ -1,15 +0,0 @@ -package control - -// SetKey sets public key used for signing. -func (x *Signature) SetKey(v []byte) { - if x != nil { - x.Key = v - } -} - -// SetSign sets binary signature. -func (x *Signature) SetSign(v []byte) { - if x != nil { - x.Sign = v - } -} diff --git a/pkg/services/control/ir/types.pb.go b/pkg/services/control/ir/types.pb.go deleted file mode 100644 index 840e0be67..000000000 --- a/pkg/services/control/ir/types.pb.go +++ /dev/null @@ -1,230 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.33.0 -// protoc v4.25.0 -// source: pkg/services/control/ir/types.proto - -package control - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// Health status of the IR application. -type HealthStatus int32 - -const ( - // Undefined status, default value. - HealthStatus_HEALTH_STATUS_UNDEFINED HealthStatus = 0 - // IR application is starting. - HealthStatus_STARTING HealthStatus = 1 - // IR application is started and serves all services. - HealthStatus_READY HealthStatus = 2 - // IR application is shutting down. - HealthStatus_SHUTTING_DOWN HealthStatus = 3 - // IR application is reconfiguring. - HealthStatus_RECONFIGURING HealthStatus = 4 -) - -// Enum value maps for HealthStatus. -var ( - HealthStatus_name = map[int32]string{ - 0: "HEALTH_STATUS_UNDEFINED", - 1: "STARTING", - 2: "READY", - 3: "SHUTTING_DOWN", - 4: "RECONFIGURING", - } - HealthStatus_value = map[string]int32{ - "HEALTH_STATUS_UNDEFINED": 0, - "STARTING": 1, - "READY": 2, - "SHUTTING_DOWN": 3, - "RECONFIGURING": 4, - } -) - -func (x HealthStatus) Enum() *HealthStatus { - p := new(HealthStatus) - *p = x - return p -} - -func (x HealthStatus) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (HealthStatus) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_services_control_ir_types_proto_enumTypes[0].Descriptor() -} - -func (HealthStatus) Type() protoreflect.EnumType { - return &file_pkg_services_control_ir_types_proto_enumTypes[0] -} - -func (x HealthStatus) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use HealthStatus.Descriptor instead. -func (HealthStatus) EnumDescriptor() ([]byte, []int) { - return file_pkg_services_control_ir_types_proto_rawDescGZIP(), []int{0} -} - -// Signature of some message. -type Signature struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Public key used for signing. - Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - // Binary signature. - Sign []byte `protobuf:"bytes,2,opt,name=sign,json=signature,proto3" json:"sign,omitempty"` -} - -func (x *Signature) Reset() { - *x = Signature{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_ir_types_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Signature) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Signature) ProtoMessage() {} - -func (x *Signature) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_ir_types_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Signature.ProtoReflect.Descriptor instead. -func (*Signature) Descriptor() ([]byte, []int) { - return file_pkg_services_control_ir_types_proto_rawDescGZIP(), []int{0} -} - -func (x *Signature) GetKey() []byte { - if x != nil { - return x.Key - } - return nil -} - -func (x *Signature) GetSign() []byte { - if x != nil { - return x.Sign - } - return nil -} - -var File_pkg_services_control_ir_types_proto protoreflect.FileDescriptor - -var file_pkg_services_control_ir_types_proto_rawDesc = []byte{ - 0x0a, 0x23, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x69, 0x72, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x69, 0x72, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x22, 0x36, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x17, 0x0a, 0x04, 0x73, 0x69, 0x67, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x2a, 0x6a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x17, 0x48, 0x45, 0x41, 0x4c, - 0x54, 0x48, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, - 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, - 0x47, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11, - 0x0a, 0x0d, 0x53, 0x48, 0x55, 0x54, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, - 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x55, 0x52, 0x49, - 0x4e, 0x47, 0x10, 0x04, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, - 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, - 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, - 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, - 0x69, 0x72, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, -} - -var ( - file_pkg_services_control_ir_types_proto_rawDescOnce sync.Once - file_pkg_services_control_ir_types_proto_rawDescData = file_pkg_services_control_ir_types_proto_rawDesc -) - -func file_pkg_services_control_ir_types_proto_rawDescGZIP() []byte { - file_pkg_services_control_ir_types_proto_rawDescOnce.Do(func() { - file_pkg_services_control_ir_types_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_services_control_ir_types_proto_rawDescData) - }) - return file_pkg_services_control_ir_types_proto_rawDescData -} - -var file_pkg_services_control_ir_types_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_pkg_services_control_ir_types_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_pkg_services_control_ir_types_proto_goTypes = []interface{}{ - (HealthStatus)(0), // 0: ircontrol.HealthStatus - (*Signature)(nil), // 1: ircontrol.Signature -} -var file_pkg_services_control_ir_types_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_pkg_services_control_ir_types_proto_init() } -func file_pkg_services_control_ir_types_proto_init() { - if File_pkg_services_control_ir_types_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_pkg_services_control_ir_types_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Signature); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_pkg_services_control_ir_types_proto_rawDesc, - NumEnums: 1, - NumMessages: 1, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_pkg_services_control_ir_types_proto_goTypes, - DependencyIndexes: file_pkg_services_control_ir_types_proto_depIdxs, - EnumInfos: file_pkg_services_control_ir_types_proto_enumTypes, - MessageInfos: file_pkg_services_control_ir_types_proto_msgTypes, - }.Build() - File_pkg_services_control_ir_types_proto = out.File - file_pkg_services_control_ir_types_proto_rawDesc = nil - file_pkg_services_control_ir_types_proto_goTypes = nil - file_pkg_services_control_ir_types_proto_depIdxs = nil -} diff --git a/pkg/services/control/ir/types_frostfs.pb.go b/pkg/services/control/ir/types_frostfs.pb.go index ef2fc458e..b230726a9 100644 --- a/pkg/services/control/ir/types_frostfs.pb.go +++ b/pkg/services/control/ir/types_frostfs.pb.go @@ -2,7 +2,70 @@ package control -import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" +import ( + json "encoding/json" + fmt "fmt" + pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + easyproto "github.com/VictoriaMetrics/easyproto" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" + strconv "strconv" +) + +type HealthStatus int32 + +const ( + HealthStatus_HEALTH_STATUS_UNDEFINED HealthStatus = 0 + HealthStatus_STARTING HealthStatus = 1 + HealthStatus_READY HealthStatus = 2 + HealthStatus_SHUTTING_DOWN HealthStatus = 3 + HealthStatus_RECONFIGURING HealthStatus = 4 +) + +var ( + HealthStatus_name = map[int32]string{ + 0: "HEALTH_STATUS_UNDEFINED", + 1: "STARTING", + 2: "READY", + 3: "SHUTTING_DOWN", + 4: "RECONFIGURING", + } + HealthStatus_value = map[string]int32{ + "HEALTH_STATUS_UNDEFINED": 0, + "STARTING": 1, + "READY": 2, + "SHUTTING_DOWN": 3, + "RECONFIGURING": 4, + } +) + +func (x HealthStatus) String() string { + if v, ok := HealthStatus_name[int32(x)]; ok { + return v + } + return strconv.FormatInt(int64(x), 10) +} +func (x *HealthStatus) FromString(s string) bool { + if v, ok := HealthStatus_value[s]; ok { + *x = HealthStatus(v) + return true + } + return false +} + +type Signature struct { + Key []byte `json:"key"` + Sign []byte `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*Signature)(nil) + _ encoding.ProtoUnmarshaler = (*Signature)(nil) + _ json.Marshaler = (*Signature)(nil) + _ json.Unmarshaler = (*Signature)(nil) +) // StableSize returns the size of x in protobuf format. // @@ -16,23 +79,138 @@ func (x *Signature) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *Signature) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.Key) - offset += proto.BytesMarshal(2, buf[offset:], x.Sign) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *Signature) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *Signature) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Key) != 0 { + mm.AppendBytes(1, x.Key) + } + if len(x.Sign) != 0 { + mm.AppendBytes(2, x.Sign) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *Signature) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "Signature") + } + switch fc.FieldNum { + case 1: // Key + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Key") + } + x.Key = data + case 2: // Sign + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Sign") + } + x.Sign = data + } + } + return nil +} +func (x *Signature) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} +func (x *Signature) SetKey(v []byte) { + x.Key = v +} +func (x *Signature) GetSign() []byte { + if x != nil { + return x.Sign + } + return nil +} +func (x *Signature) SetSign(v []byte) { + x.Sign = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *Signature) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *Signature) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"key\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.Key) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + out.Base64Bytes(x.Sign) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *Signature) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *Signature) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "key": + { + var f []byte + f = in.Bytes() + x.Key = f + } + case "signature": + { + var f []byte + f = in.Bytes() + x.Sign = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } } diff --git a/pkg/services/control/server/list_shards.go b/pkg/services/control/server/list_shards.go index d6531b947..b639245c1 100644 --- a/pkg/services/control/server/list_shards.go +++ b/pkg/services/control/server/list_shards.go @@ -30,10 +30,10 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) ( for _, sh := range info.Shards { si := new(control.ShardInfo) - si.SetID(*sh.ID) + si.SetShard_ID(*sh.ID) si.SetMetabasePath(sh.MetaBaseInfo.Path) si.Blobstor = blobstorInfoToProto(sh.BlobStorInfo) - si.SetWriteCachePath(sh.WriteCacheInfo.Path) + si.SetWritecachePath(sh.WriteCacheInfo.Path) si.SetPiloramaPath(sh.PiloramaInfo.Path) var m control.ShardMode diff --git a/pkg/services/control/service.go b/pkg/services/control/service.go deleted file mode 100644 index ef0c0a8d2..000000000 --- a/pkg/services/control/service.go +++ /dev/null @@ -1,142 +0,0 @@ -package control - -// SetBody sets health check request body. -func (x *HealthCheckRequest) SetBody(v *HealthCheckRequest_Body) { - if x != nil { - x.Body = v - } -} - -// SetNetmapStatus sets status of the storage node in FrostFS network map. -func (x *HealthCheckResponse_Body) SetNetmapStatus(v NetmapStatus) { - if x != nil { - x.NetmapStatus = v - } -} - -// SetHealthStatus sets health status of the storage node application. -func (x *HealthCheckResponse_Body) SetHealthStatus(v HealthStatus) { - if x != nil { - x.HealthStatus = v - } -} - -// SetBody sets health check response body. -func (x *HealthCheckResponse) SetBody(v *HealthCheckResponse_Body) { - if x != nil { - x.Body = v - } -} - -// SetStatus sets new storage node status in FrostFS network map. -func (x *SetNetmapStatusRequest_Body) SetStatus(v NetmapStatus) { - if x != nil { - x.Status = v - } -} - -// SetForceMaintenance sets force_maintenance flag in the message. -func (x *SetNetmapStatusRequest_Body) SetForceMaintenance() { - x.ForceMaintenance = true -} - -// SetBody sets body of the set netmap status request . -func (x *SetNetmapStatusRequest) SetBody(v *SetNetmapStatusRequest_Body) { - if x != nil { - x.Body = v - } -} - -// SetBody sets set body of the netmap status response. -func (x *SetNetmapStatusResponse) SetBody(v *SetNetmapStatusResponse_Body) { - if x != nil { - x.Body = v - } -} - -// SetAddressList sets list of objects to be removed in FrostFS API binary format. -func (x *DropObjectsRequest_Body) SetAddressList(v [][]byte) { - if x != nil { - x.AddressList = v - } -} - -// SetBody sets body of the set "Drop objects" request. -func (x *DropObjectsRequest) SetBody(v *DropObjectsRequest_Body) { - if x != nil { - x.Body = v - } -} - -// SetBody sets set body of the "Drop objects" response. -func (x *DropObjectsResponse) SetBody(v *DropObjectsResponse_Body) { - if x != nil { - x.Body = v - } -} - -// SetBody sets list shards request body. -func (x *ListShardsRequest) SetBody(v *ListShardsRequest_Body) { - if x != nil { - x.Body = v - } -} - -// SetShards sets shards of the storage node. -func (x *ListShardsResponse_Body) SetShards(v []*ShardInfo) { - if x != nil { - x.Shards = v - } -} - -// SetBody sets list shards response body. -func (x *ListShardsResponse) SetBody(v *ListShardsResponse_Body) { - if x != nil { - x.Body = v - } -} - -// SetShardIDList sets shard ID whose mode is requested to be set. -func (x *SetShardModeRequest_Body) SetShardIDList(v [][]byte) { - if v != nil { - x.Shard_ID = v - } -} - -// SetMode sets mode of the shard. -func (x *SetShardModeRequest_Body) SetMode(v ShardMode) { - x.Mode = v -} - -// ClearErrorCounter sets flag signifying whether error counter for shard should be cleared. -func (x *SetShardModeRequest_Body) ClearErrorCounter(reset bool) { - x.ResetErrorCounter = reset -} - -// SetBody sets request body. -func (x *SetShardModeRequest) SetBody(v *SetShardModeRequest_Body) { - if x != nil { - x.Body = v - } -} - -// SetBody sets body of the set shard mode response. -func (x *SetShardModeResponse) SetBody(v *SetShardModeResponse_Body) { - if x != nil { - x.Body = v - } -} - -// SetBody sets list shards request body. -func (x *SynchronizeTreeRequest) SetBody(v *SynchronizeTreeRequest_Body) { - if x != nil { - x.Body = v - } -} - -// SetBody sets list shards response body. -func (x *SynchronizeTreeResponse) SetBody(v *SynchronizeTreeResponse_Body) { - if x != nil { - x.Body = v - } -} diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go deleted file mode 100644 index e5a5ce24c..000000000 --- a/pkg/services/control/service.pb.go +++ /dev/null @@ -1,7094 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.33.0 -// protoc v4.25.0 -// source: pkg/services/control/service.proto - -package control - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type StartShardEvacuationRequest_Body_Scope int32 - -const ( - StartShardEvacuationRequest_Body_NONE StartShardEvacuationRequest_Body_Scope = 0 - StartShardEvacuationRequest_Body_OBJECTS StartShardEvacuationRequest_Body_Scope = 1 - StartShardEvacuationRequest_Body_TREES StartShardEvacuationRequest_Body_Scope = 2 -) - -// Enum value maps for StartShardEvacuationRequest_Body_Scope. -var ( - StartShardEvacuationRequest_Body_Scope_name = map[int32]string{ - 0: "NONE", - 1: "OBJECTS", - 2: "TREES", - } - StartShardEvacuationRequest_Body_Scope_value = map[string]int32{ - "NONE": 0, - "OBJECTS": 1, - "TREES": 2, - } -) - -func (x StartShardEvacuationRequest_Body_Scope) Enum() *StartShardEvacuationRequest_Body_Scope { - p := new(StartShardEvacuationRequest_Body_Scope) - *p = x - return p -} - -func (x StartShardEvacuationRequest_Body_Scope) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (StartShardEvacuationRequest_Body_Scope) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_services_control_service_proto_enumTypes[0].Descriptor() -} - -func (StartShardEvacuationRequest_Body_Scope) Type() protoreflect.EnumType { - return &file_pkg_services_control_service_proto_enumTypes[0] -} - -func (x StartShardEvacuationRequest_Body_Scope) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use StartShardEvacuationRequest_Body_Scope.Descriptor instead. -func (StartShardEvacuationRequest_Body_Scope) EnumDescriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20, 0, 0} -} - -// Evacuate status enum. -type GetShardEvacuationStatusResponse_Body_Status int32 - -const ( - GetShardEvacuationStatusResponse_Body_EVACUATE_SHARD_STATUS_UNDEFINED GetShardEvacuationStatusResponse_Body_Status = 0 - GetShardEvacuationStatusResponse_Body_RUNNING GetShardEvacuationStatusResponse_Body_Status = 1 - GetShardEvacuationStatusResponse_Body_COMPLETED GetShardEvacuationStatusResponse_Body_Status = 2 -) - -// Enum value maps for GetShardEvacuationStatusResponse_Body_Status. -var ( - GetShardEvacuationStatusResponse_Body_Status_name = map[int32]string{ - 0: "EVACUATE_SHARD_STATUS_UNDEFINED", - 1: "RUNNING", - 2: "COMPLETED", - } - GetShardEvacuationStatusResponse_Body_Status_value = map[string]int32{ - "EVACUATE_SHARD_STATUS_UNDEFINED": 0, - "RUNNING": 1, - "COMPLETED": 2, - } -) - -func (x GetShardEvacuationStatusResponse_Body_Status) Enum() *GetShardEvacuationStatusResponse_Body_Status { - p := new(GetShardEvacuationStatusResponse_Body_Status) - *p = x - return p -} - -func (x GetShardEvacuationStatusResponse_Body_Status) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (GetShardEvacuationStatusResponse_Body_Status) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_services_control_service_proto_enumTypes[1].Descriptor() -} - -func (GetShardEvacuationStatusResponse_Body_Status) Type() protoreflect.EnumType { - return &file_pkg_services_control_service_proto_enumTypes[1] -} - -func (x GetShardEvacuationStatusResponse_Body_Status) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use GetShardEvacuationStatusResponse_Body_Status.Descriptor instead. -func (GetShardEvacuationStatusResponse_Body_Status) EnumDescriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0, 0} -} - -// Health check request. -type HealthCheckRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of health check request message. - Body *HealthCheckRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *HealthCheckRequest) Reset() { - *x = HealthCheckRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthCheckRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthCheckRequest) ProtoMessage() {} - -func (x *HealthCheckRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthCheckRequest.ProtoReflect.Descriptor instead. -func (*HealthCheckRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{0} -} - -func (x *HealthCheckRequest) GetBody() *HealthCheckRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *HealthCheckRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Health check request. -type HealthCheckResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of health check response message. - Body *HealthCheckResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *HealthCheckResponse) Reset() { - *x = HealthCheckResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthCheckResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthCheckResponse) ProtoMessage() {} - -func (x *HealthCheckResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthCheckResponse.ProtoReflect.Descriptor instead. -func (*HealthCheckResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{1} -} - -func (x *HealthCheckResponse) GetBody() *HealthCheckResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *HealthCheckResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Set netmap status request. -type SetNetmapStatusRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of set netmap status request message. - Body *SetNetmapStatusRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *SetNetmapStatusRequest) Reset() { - *x = SetNetmapStatusRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SetNetmapStatusRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SetNetmapStatusRequest) ProtoMessage() {} - -func (x *SetNetmapStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SetNetmapStatusRequest.ProtoReflect.Descriptor instead. -func (*SetNetmapStatusRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{2} -} - -func (x *SetNetmapStatusRequest) GetBody() *SetNetmapStatusRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *SetNetmapStatusRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Set netmap status response. -type SetNetmapStatusResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of set netmap status response message. - Body *SetNetmapStatusResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *SetNetmapStatusResponse) Reset() { - *x = SetNetmapStatusResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SetNetmapStatusResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SetNetmapStatusResponse) ProtoMessage() {} - -func (x *SetNetmapStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SetNetmapStatusResponse.ProtoReflect.Descriptor instead. -func (*SetNetmapStatusResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{3} -} - -func (x *SetNetmapStatusResponse) GetBody() *SetNetmapStatusResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *SetNetmapStatusResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Get netmap status request. -type GetNetmapStatusRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of set netmap status request message. - Body *GetNetmapStatusRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetNetmapStatusRequest) Reset() { - *x = GetNetmapStatusRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetNetmapStatusRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetNetmapStatusRequest) ProtoMessage() {} - -func (x *GetNetmapStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetNetmapStatusRequest.ProtoReflect.Descriptor instead. -func (*GetNetmapStatusRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{4} -} - -func (x *GetNetmapStatusRequest) GetBody() *GetNetmapStatusRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetNetmapStatusRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Get netmap status response. -type GetNetmapStatusResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of get netmap status response message. - Body *GetNetmapStatusResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetNetmapStatusResponse) Reset() { - *x = GetNetmapStatusResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetNetmapStatusResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetNetmapStatusResponse) ProtoMessage() {} - -func (x *GetNetmapStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetNetmapStatusResponse.ProtoReflect.Descriptor instead. -func (*GetNetmapStatusResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{5} -} - -func (x *GetNetmapStatusResponse) GetBody() *GetNetmapStatusResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetNetmapStatusResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Request to drop the objects. -type DropObjectsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of the request message. - Body *DropObjectsRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *DropObjectsRequest) Reset() { - *x = DropObjectsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DropObjectsRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DropObjectsRequest) ProtoMessage() {} - -func (x *DropObjectsRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DropObjectsRequest.ProtoReflect.Descriptor instead. -func (*DropObjectsRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{6} -} - -func (x *DropObjectsRequest) GetBody() *DropObjectsRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *DropObjectsRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Response to request to drop the objects. -type DropObjectsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of the response message. - Body *DropObjectsResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *DropObjectsResponse) Reset() { - *x = DropObjectsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DropObjectsResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DropObjectsResponse) ProtoMessage() {} - -func (x *DropObjectsResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DropObjectsResponse.ProtoReflect.Descriptor instead. -func (*DropObjectsResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{7} -} - -func (x *DropObjectsResponse) GetBody() *DropObjectsResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *DropObjectsResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Request to list all shards of the node. -type ListShardsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of the request message. - Body *ListShardsRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *ListShardsRequest) Reset() { - *x = ListShardsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListShardsRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListShardsRequest) ProtoMessage() {} - -func (x *ListShardsRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListShardsRequest.ProtoReflect.Descriptor instead. -func (*ListShardsRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{8} -} - -func (x *ListShardsRequest) GetBody() *ListShardsRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *ListShardsRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// ListShards response. -type ListShardsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of the response message. - Body *ListShardsResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *ListShardsResponse) Reset() { - *x = ListShardsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListShardsResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListShardsResponse) ProtoMessage() {} - -func (x *ListShardsResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListShardsResponse.ProtoReflect.Descriptor instead. -func (*ListShardsResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{9} -} - -func (x *ListShardsResponse) GetBody() *ListShardsResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *ListShardsResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Request to set mode of the shard. -type SetShardModeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of set shard mode request message. - Body *SetShardModeRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *SetShardModeRequest) Reset() { - *x = SetShardModeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SetShardModeRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SetShardModeRequest) ProtoMessage() {} - -func (x *SetShardModeRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SetShardModeRequest.ProtoReflect.Descriptor instead. -func (*SetShardModeRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{10} -} - -func (x *SetShardModeRequest) GetBody() *SetShardModeRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *SetShardModeRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// SetShardMode response. -type SetShardModeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of set shard mode response message. - Body *SetShardModeResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *SetShardModeResponse) Reset() { - *x = SetShardModeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SetShardModeResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SetShardModeResponse) ProtoMessage() {} - -func (x *SetShardModeResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SetShardModeResponse.ProtoReflect.Descriptor instead. -func (*SetShardModeResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{11} -} - -func (x *SetShardModeResponse) GetBody() *SetShardModeResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *SetShardModeResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// SynchronizeTree request. -type SynchronizeTreeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of restore shard request message. - Body *SynchronizeTreeRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *SynchronizeTreeRequest) Reset() { - *x = SynchronizeTreeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SynchronizeTreeRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SynchronizeTreeRequest) ProtoMessage() {} - -func (x *SynchronizeTreeRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SynchronizeTreeRequest.ProtoReflect.Descriptor instead. -func (*SynchronizeTreeRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{12} -} - -func (x *SynchronizeTreeRequest) GetBody() *SynchronizeTreeRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *SynchronizeTreeRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// SynchronizeTree response. -type SynchronizeTreeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Body of restore shard response message. - Body *SynchronizeTreeResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Body signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *SynchronizeTreeResponse) Reset() { - *x = SynchronizeTreeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SynchronizeTreeResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SynchronizeTreeResponse) ProtoMessage() {} - -func (x *SynchronizeTreeResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SynchronizeTreeResponse.ProtoReflect.Descriptor instead. -func (*SynchronizeTreeResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{13} -} - -func (x *SynchronizeTreeResponse) GetBody() *SynchronizeTreeResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *SynchronizeTreeResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// EvacuateShard request. -type EvacuateShardRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *EvacuateShardRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *EvacuateShardRequest) Reset() { - *x = EvacuateShardRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[14] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EvacuateShardRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EvacuateShardRequest) ProtoMessage() {} - -func (x *EvacuateShardRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EvacuateShardRequest.ProtoReflect.Descriptor instead. -func (*EvacuateShardRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{14} -} - -func (x *EvacuateShardRequest) GetBody() *EvacuateShardRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *EvacuateShardRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// EvacuateShard response. -type EvacuateShardResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *EvacuateShardResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *EvacuateShardResponse) Reset() { - *x = EvacuateShardResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[15] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EvacuateShardResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EvacuateShardResponse) ProtoMessage() {} - -func (x *EvacuateShardResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[15] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EvacuateShardResponse.ProtoReflect.Descriptor instead. -func (*EvacuateShardResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{15} -} - -func (x *EvacuateShardResponse) GetBody() *EvacuateShardResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *EvacuateShardResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// FlushCache request. -type FlushCacheRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *FlushCacheRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *FlushCacheRequest) Reset() { - *x = FlushCacheRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[16] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FlushCacheRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FlushCacheRequest) ProtoMessage() {} - -func (x *FlushCacheRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[16] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FlushCacheRequest.ProtoReflect.Descriptor instead. -func (*FlushCacheRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{16} -} - -func (x *FlushCacheRequest) GetBody() *FlushCacheRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *FlushCacheRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// FlushCache response. -type FlushCacheResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *FlushCacheResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *FlushCacheResponse) Reset() { - *x = FlushCacheResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[17] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FlushCacheResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FlushCacheResponse) ProtoMessage() {} - -func (x *FlushCacheResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[17] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FlushCacheResponse.ProtoReflect.Descriptor instead. -func (*FlushCacheResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{17} -} - -func (x *FlushCacheResponse) GetBody() *FlushCacheResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *FlushCacheResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Doctor request. -type DoctorRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *DoctorRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *DoctorRequest) Reset() { - *x = DoctorRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[18] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DoctorRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DoctorRequest) ProtoMessage() {} - -func (x *DoctorRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[18] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DoctorRequest.ProtoReflect.Descriptor instead. -func (*DoctorRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18} -} - -func (x *DoctorRequest) GetBody() *DoctorRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *DoctorRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Doctor response. -type DoctorResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *DoctorResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *DoctorResponse) Reset() { - *x = DoctorResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[19] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DoctorResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DoctorResponse) ProtoMessage() {} - -func (x *DoctorResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[19] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DoctorResponse.ProtoReflect.Descriptor instead. -func (*DoctorResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{19} -} - -func (x *DoctorResponse) GetBody() *DoctorResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *DoctorResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// StartShardEvacuation request. -type StartShardEvacuationRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *StartShardEvacuationRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *StartShardEvacuationRequest) Reset() { - *x = StartShardEvacuationRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[20] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StartShardEvacuationRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StartShardEvacuationRequest) ProtoMessage() {} - -func (x *StartShardEvacuationRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[20] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StartShardEvacuationRequest.ProtoReflect.Descriptor instead. -func (*StartShardEvacuationRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20} -} - -func (x *StartShardEvacuationRequest) GetBody() *StartShardEvacuationRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *StartShardEvacuationRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// StartShardEvacuation response. -type StartShardEvacuationResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *StartShardEvacuationResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *StartShardEvacuationResponse) Reset() { - *x = StartShardEvacuationResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[21] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StartShardEvacuationResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StartShardEvacuationResponse) ProtoMessage() {} - -func (x *StartShardEvacuationResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[21] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StartShardEvacuationResponse.ProtoReflect.Descriptor instead. -func (*StartShardEvacuationResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21} -} - -func (x *StartShardEvacuationResponse) GetBody() *StartShardEvacuationResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *StartShardEvacuationResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// GetShardEvacuationStatus request. -type GetShardEvacuationStatusRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *GetShardEvacuationStatusRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetShardEvacuationStatusRequest) Reset() { - *x = GetShardEvacuationStatusRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[22] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetShardEvacuationStatusRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetShardEvacuationStatusRequest) ProtoMessage() {} - -func (x *GetShardEvacuationStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[22] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetShardEvacuationStatusRequest.ProtoReflect.Descriptor instead. -func (*GetShardEvacuationStatusRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22} -} - -func (x *GetShardEvacuationStatusRequest) GetBody() *GetShardEvacuationStatusRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetShardEvacuationStatusRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// GetShardEvacuationStatus response. -type GetShardEvacuationStatusResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *GetShardEvacuationStatusResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetShardEvacuationStatusResponse) Reset() { - *x = GetShardEvacuationStatusResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[23] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetShardEvacuationStatusResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetShardEvacuationStatusResponse) ProtoMessage() {} - -func (x *GetShardEvacuationStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[23] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetShardEvacuationStatusResponse.ProtoReflect.Descriptor instead. -func (*GetShardEvacuationStatusResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23} -} - -func (x *GetShardEvacuationStatusResponse) GetBody() *GetShardEvacuationStatusResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetShardEvacuationStatusResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// ResetShardEvacuationStatus request. -type ResetShardEvacuationStatusRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *ResetShardEvacuationStatusRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *ResetShardEvacuationStatusRequest) Reset() { - *x = ResetShardEvacuationStatusRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[24] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ResetShardEvacuationStatusRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ResetShardEvacuationStatusRequest) ProtoMessage() {} - -func (x *ResetShardEvacuationStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[24] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ResetShardEvacuationStatusRequest.ProtoReflect.Descriptor instead. -func (*ResetShardEvacuationStatusRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24} -} - -func (x *ResetShardEvacuationStatusRequest) GetBody() *ResetShardEvacuationStatusRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *ResetShardEvacuationStatusRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// ResetShardEvacuationStatus response. -type ResetShardEvacuationStatusResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *ResetShardEvacuationStatusResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *ResetShardEvacuationStatusResponse) Reset() { - *x = ResetShardEvacuationStatusResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[25] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ResetShardEvacuationStatusResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ResetShardEvacuationStatusResponse) ProtoMessage() {} - -func (x *ResetShardEvacuationStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[25] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ResetShardEvacuationStatusResponse.ProtoReflect.Descriptor instead. -func (*ResetShardEvacuationStatusResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25} -} - -func (x *ResetShardEvacuationStatusResponse) GetBody() *ResetShardEvacuationStatusResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *ResetShardEvacuationStatusResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// StopShardEvacuation request. -type StopShardEvacuationRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *StopShardEvacuationRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *StopShardEvacuationRequest) Reset() { - *x = StopShardEvacuationRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[26] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StopShardEvacuationRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StopShardEvacuationRequest) ProtoMessage() {} - -func (x *StopShardEvacuationRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[26] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StopShardEvacuationRequest.ProtoReflect.Descriptor instead. -func (*StopShardEvacuationRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26} -} - -func (x *StopShardEvacuationRequest) GetBody() *StopShardEvacuationRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *StopShardEvacuationRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// StopShardEvacuation response. -type StopShardEvacuationResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *StopShardEvacuationResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *StopShardEvacuationResponse) Reset() { - *x = StopShardEvacuationResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[27] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StopShardEvacuationResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StopShardEvacuationResponse) ProtoMessage() {} - -func (x *StopShardEvacuationResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[27] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StopShardEvacuationResponse.ProtoReflect.Descriptor instead. -func (*StopShardEvacuationResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27} -} - -func (x *StopShardEvacuationResponse) GetBody() *StopShardEvacuationResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *StopShardEvacuationResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// AddChainLocalOverride request. -type AddChainLocalOverrideRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *AddChainLocalOverrideRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *AddChainLocalOverrideRequest) Reset() { - *x = AddChainLocalOverrideRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[28] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddChainLocalOverrideRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddChainLocalOverrideRequest) ProtoMessage() {} - -func (x *AddChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[28] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddChainLocalOverrideRequest.ProtoReflect.Descriptor instead. -func (*AddChainLocalOverrideRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28} -} - -func (x *AddChainLocalOverrideRequest) GetBody() *AddChainLocalOverrideRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *AddChainLocalOverrideRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// AddChainLocalOverride response. -type AddChainLocalOverrideResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *AddChainLocalOverrideResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *AddChainLocalOverrideResponse) Reset() { - *x = AddChainLocalOverrideResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[29] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddChainLocalOverrideResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddChainLocalOverrideResponse) ProtoMessage() {} - -func (x *AddChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[29] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddChainLocalOverrideResponse.ProtoReflect.Descriptor instead. -func (*AddChainLocalOverrideResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29} -} - -func (x *AddChainLocalOverrideResponse) GetBody() *AddChainLocalOverrideResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *AddChainLocalOverrideResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// GetChainLocalOverride request. -type GetChainLocalOverrideRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *GetChainLocalOverrideRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetChainLocalOverrideRequest) Reset() { - *x = GetChainLocalOverrideRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[30] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetChainLocalOverrideRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetChainLocalOverrideRequest) ProtoMessage() {} - -func (x *GetChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[30] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetChainLocalOverrideRequest.ProtoReflect.Descriptor instead. -func (*GetChainLocalOverrideRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30} -} - -func (x *GetChainLocalOverrideRequest) GetBody() *GetChainLocalOverrideRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetChainLocalOverrideRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// GetChainLocalOverride response. -type GetChainLocalOverrideResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *GetChainLocalOverrideResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetChainLocalOverrideResponse) Reset() { - *x = GetChainLocalOverrideResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[31] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetChainLocalOverrideResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetChainLocalOverrideResponse) ProtoMessage() {} - -func (x *GetChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[31] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetChainLocalOverrideResponse.ProtoReflect.Descriptor instead. -func (*GetChainLocalOverrideResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31} -} - -func (x *GetChainLocalOverrideResponse) GetBody() *GetChainLocalOverrideResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetChainLocalOverrideResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// ListChainLocalOverrides request. -type ListChainLocalOverridesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *ListChainLocalOverridesRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *ListChainLocalOverridesRequest) Reset() { - *x = ListChainLocalOverridesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[32] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListChainLocalOverridesRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListChainLocalOverridesRequest) ProtoMessage() {} - -func (x *ListChainLocalOverridesRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[32] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListChainLocalOverridesRequest.ProtoReflect.Descriptor instead. -func (*ListChainLocalOverridesRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32} -} - -func (x *ListChainLocalOverridesRequest) GetBody() *ListChainLocalOverridesRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *ListChainLocalOverridesRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// ListChainLocalOverrides response. -type ListChainLocalOverridesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *ListChainLocalOverridesResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *ListChainLocalOverridesResponse) Reset() { - *x = ListChainLocalOverridesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[33] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListChainLocalOverridesResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListChainLocalOverridesResponse) ProtoMessage() {} - -func (x *ListChainLocalOverridesResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[33] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListChainLocalOverridesResponse.ProtoReflect.Descriptor instead. -func (*ListChainLocalOverridesResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33} -} - -func (x *ListChainLocalOverridesResponse) GetBody() *ListChainLocalOverridesResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *ListChainLocalOverridesResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// ListTargetsLocalOverrides request. -type ListTargetsLocalOverridesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *ListTargetsLocalOverridesRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *ListTargetsLocalOverridesRequest) Reset() { - *x = ListTargetsLocalOverridesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[34] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListTargetsLocalOverridesRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListTargetsLocalOverridesRequest) ProtoMessage() {} - -func (x *ListTargetsLocalOverridesRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[34] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListTargetsLocalOverridesRequest.ProtoReflect.Descriptor instead. -func (*ListTargetsLocalOverridesRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34} -} - -func (x *ListTargetsLocalOverridesRequest) GetBody() *ListTargetsLocalOverridesRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *ListTargetsLocalOverridesRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// ListTargetsLocalOverrides response. -type ListTargetsLocalOverridesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *ListTargetsLocalOverridesResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *ListTargetsLocalOverridesResponse) Reset() { - *x = ListTargetsLocalOverridesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[35] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListTargetsLocalOverridesResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListTargetsLocalOverridesResponse) ProtoMessage() {} - -func (x *ListTargetsLocalOverridesResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[35] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListTargetsLocalOverridesResponse.ProtoReflect.Descriptor instead. -func (*ListTargetsLocalOverridesResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35} -} - -func (x *ListTargetsLocalOverridesResponse) GetBody() *ListTargetsLocalOverridesResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *ListTargetsLocalOverridesResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type RemoveChainLocalOverrideRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *RemoveChainLocalOverrideRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RemoveChainLocalOverrideRequest) Reset() { - *x = RemoveChainLocalOverrideRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[36] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveChainLocalOverrideRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveChainLocalOverrideRequest) ProtoMessage() {} - -func (x *RemoveChainLocalOverrideRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[36] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveChainLocalOverrideRequest.ProtoReflect.Descriptor instead. -func (*RemoveChainLocalOverrideRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36} -} - -func (x *RemoveChainLocalOverrideRequest) GetBody() *RemoveChainLocalOverrideRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RemoveChainLocalOverrideRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type RemoveChainLocalOverrideResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *RemoveChainLocalOverrideResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RemoveChainLocalOverrideResponse) Reset() { - *x = RemoveChainLocalOverrideResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[37] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveChainLocalOverrideResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveChainLocalOverrideResponse) ProtoMessage() {} - -func (x *RemoveChainLocalOverrideResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[37] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveChainLocalOverrideResponse.ProtoReflect.Descriptor instead. -func (*RemoveChainLocalOverrideResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37} -} - -func (x *RemoveChainLocalOverrideResponse) GetBody() *RemoveChainLocalOverrideResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RemoveChainLocalOverrideResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type RemoveChainLocalOverridesByTargetRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *RemoveChainLocalOverridesByTargetRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RemoveChainLocalOverridesByTargetRequest) Reset() { - *x = RemoveChainLocalOverridesByTargetRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[38] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveChainLocalOverridesByTargetRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveChainLocalOverridesByTargetRequest) ProtoMessage() {} - -func (x *RemoveChainLocalOverridesByTargetRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[38] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveChainLocalOverridesByTargetRequest.ProtoReflect.Descriptor instead. -func (*RemoveChainLocalOverridesByTargetRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38} -} - -func (x *RemoveChainLocalOverridesByTargetRequest) GetBody() *RemoveChainLocalOverridesByTargetRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RemoveChainLocalOverridesByTargetRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type RemoveChainLocalOverridesByTargetResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *RemoveChainLocalOverridesByTargetResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RemoveChainLocalOverridesByTargetResponse) Reset() { - *x = RemoveChainLocalOverridesByTargetResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[39] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveChainLocalOverridesByTargetResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveChainLocalOverridesByTargetResponse) ProtoMessage() {} - -func (x *RemoveChainLocalOverridesByTargetResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[39] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveChainLocalOverridesByTargetResponse.ProtoReflect.Descriptor instead. -func (*RemoveChainLocalOverridesByTargetResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39} -} - -func (x *RemoveChainLocalOverridesByTargetResponse) GetBody() *RemoveChainLocalOverridesByTargetResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RemoveChainLocalOverridesByTargetResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type SealWriteCacheRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *SealWriteCacheRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *SealWriteCacheRequest) Reset() { - *x = SealWriteCacheRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[40] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SealWriteCacheRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SealWriteCacheRequest) ProtoMessage() {} - -func (x *SealWriteCacheRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[40] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SealWriteCacheRequest.ProtoReflect.Descriptor instead. -func (*SealWriteCacheRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{40} -} - -func (x *SealWriteCacheRequest) GetBody() *SealWriteCacheRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *SealWriteCacheRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type SealWriteCacheResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *SealWriteCacheResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *SealWriteCacheResponse) Reset() { - *x = SealWriteCacheResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[41] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SealWriteCacheResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SealWriteCacheResponse) ProtoMessage() {} - -func (x *SealWriteCacheResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[41] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SealWriteCacheResponse.ProtoReflect.Descriptor instead. -func (*SealWriteCacheResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{41} -} - -func (x *SealWriteCacheResponse) GetBody() *SealWriteCacheResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *SealWriteCacheResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type DetachShardsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *DetachShardsRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *DetachShardsRequest) Reset() { - *x = DetachShardsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[42] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DetachShardsRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DetachShardsRequest) ProtoMessage() {} - -func (x *DetachShardsRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[42] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DetachShardsRequest.ProtoReflect.Descriptor instead. -func (*DetachShardsRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{42} -} - -func (x *DetachShardsRequest) GetBody() *DetachShardsRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *DetachShardsRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type DetachShardsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Body *DetachShardsResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *DetachShardsResponse) Reset() { - *x = DetachShardsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[43] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DetachShardsResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DetachShardsResponse) ProtoMessage() {} - -func (x *DetachShardsResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[43] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DetachShardsResponse.ProtoReflect.Descriptor instead. -func (*DetachShardsResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{43} -} - -func (x *DetachShardsResponse) GetBody() *DetachShardsResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *DetachShardsResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -// Health check request body. -type HealthCheckRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *HealthCheckRequest_Body) Reset() { - *x = HealthCheckRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[44] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthCheckRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthCheckRequest_Body) ProtoMessage() {} - -func (x *HealthCheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[44] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthCheckRequest_Body.ProtoReflect.Descriptor instead. -func (*HealthCheckRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{0, 0} -} - -// Health check response body -type HealthCheckResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Status of the storage node in FrostFS network map. - NetmapStatus NetmapStatus `protobuf:"varint,1,opt,name=netmap_status,json=netmapStatus,proto3,enum=control.NetmapStatus" json:"netmap_status,omitempty"` - // Health status of storage node application. - HealthStatus HealthStatus `protobuf:"varint,2,opt,name=health_status,json=healthStatus,proto3,enum=control.HealthStatus" json:"health_status,omitempty"` -} - -func (x *HealthCheckResponse_Body) Reset() { - *x = HealthCheckResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[45] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthCheckResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthCheckResponse_Body) ProtoMessage() {} - -func (x *HealthCheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[45] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthCheckResponse_Body.ProtoReflect.Descriptor instead. -func (*HealthCheckResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{1, 0} -} - -func (x *HealthCheckResponse_Body) GetNetmapStatus() NetmapStatus { - if x != nil { - return x.NetmapStatus - } - return NetmapStatus_STATUS_UNDEFINED -} - -func (x *HealthCheckResponse_Body) GetHealthStatus() HealthStatus { - if x != nil { - return x.HealthStatus - } - return HealthStatus_HEALTH_STATUS_UNDEFINED -} - -// Set netmap status request body. -type SetNetmapStatusRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // New storage node status in FrostFS network map. - // If status is MAINTENANCE, the node checks whether maintenance is - // allowed in the network settings. In case of prohibition, the request - // is denied. Otherwise, node switches to local maintenance state. To - // force local maintenance, use `force_maintenance` flag. - Status NetmapStatus `protobuf:"varint,1,opt,name=status,proto3,enum=control.NetmapStatus" json:"status,omitempty"` - // MAINTENANCE status validation skip flag. If set, node starts local - // maintenance regardless of network settings. The flag MUST NOT be - // set for any other status. - ForceMaintenance bool `protobuf:"varint,2,opt,name=force_maintenance,json=forceMaintenance,proto3" json:"force_maintenance,omitempty"` -} - -func (x *SetNetmapStatusRequest_Body) Reset() { - *x = SetNetmapStatusRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[46] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SetNetmapStatusRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SetNetmapStatusRequest_Body) ProtoMessage() {} - -func (x *SetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[46] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SetNetmapStatusRequest_Body.ProtoReflect.Descriptor instead. -func (*SetNetmapStatusRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{2, 0} -} - -func (x *SetNetmapStatusRequest_Body) GetStatus() NetmapStatus { - if x != nil { - return x.Status - } - return NetmapStatus_STATUS_UNDEFINED -} - -func (x *SetNetmapStatusRequest_Body) GetForceMaintenance() bool { - if x != nil { - return x.ForceMaintenance - } - return false -} - -// Set netmap status response body -type SetNetmapStatusResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *SetNetmapStatusResponse_Body) Reset() { - *x = SetNetmapStatusResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[47] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SetNetmapStatusResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SetNetmapStatusResponse_Body) ProtoMessage() {} - -func (x *SetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[47] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SetNetmapStatusResponse_Body.ProtoReflect.Descriptor instead. -func (*SetNetmapStatusResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{3, 0} -} - -type GetNetmapStatusRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *GetNetmapStatusRequest_Body) Reset() { - *x = GetNetmapStatusRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[48] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetNetmapStatusRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetNetmapStatusRequest_Body) ProtoMessage() {} - -func (x *GetNetmapStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[48] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetNetmapStatusRequest_Body.ProtoReflect.Descriptor instead. -func (*GetNetmapStatusRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{4, 0} -} - -type GetNetmapStatusResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Storage node status in FrostFS network map. - Status NetmapStatus `protobuf:"varint,1,opt,name=status,proto3,enum=control.NetmapStatus" json:"status,omitempty"` - // Network map epoch. - Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` -} - -func (x *GetNetmapStatusResponse_Body) Reset() { - *x = GetNetmapStatusResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[49] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetNetmapStatusResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetNetmapStatusResponse_Body) ProtoMessage() {} - -func (x *GetNetmapStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[49] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetNetmapStatusResponse_Body.ProtoReflect.Descriptor instead. -func (*GetNetmapStatusResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{5, 0} -} - -func (x *GetNetmapStatusResponse_Body) GetStatus() NetmapStatus { - if x != nil { - return x.Status - } - return NetmapStatus_STATUS_UNDEFINED -} - -func (x *GetNetmapStatusResponse_Body) GetEpoch() uint64 { - if x != nil { - return x.Epoch - } - return 0 -} - -// Request body structure. -type DropObjectsRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // List of object addresses to be removed. - // in FrostFS API binary format. - AddressList [][]byte `protobuf:"bytes,1,rep,name=address_list,json=addressList,proto3" json:"address_list,omitempty"` -} - -func (x *DropObjectsRequest_Body) Reset() { - *x = DropObjectsRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[50] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DropObjectsRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DropObjectsRequest_Body) ProtoMessage() {} - -func (x *DropObjectsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[50] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DropObjectsRequest_Body.ProtoReflect.Descriptor instead. -func (*DropObjectsRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{6, 0} -} - -func (x *DropObjectsRequest_Body) GetAddressList() [][]byte { - if x != nil { - return x.AddressList - } - return nil -} - -// Response body structure. -type DropObjectsResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *DropObjectsResponse_Body) Reset() { - *x = DropObjectsResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[51] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DropObjectsResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DropObjectsResponse_Body) ProtoMessage() {} - -func (x *DropObjectsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[51] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DropObjectsResponse_Body.ProtoReflect.Descriptor instead. -func (*DropObjectsResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{7, 0} -} - -// Request body structure. -type ListShardsRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *ListShardsRequest_Body) Reset() { - *x = ListShardsRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[52] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListShardsRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListShardsRequest_Body) ProtoMessage() {} - -func (x *ListShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[52] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListShardsRequest_Body.ProtoReflect.Descriptor instead. -func (*ListShardsRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{8, 0} -} - -// Response body structure. -type ListShardsResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // List of the node's shards. - Shards []*ShardInfo `protobuf:"bytes,1,rep,name=shards,proto3" json:"shards,omitempty"` -} - -func (x *ListShardsResponse_Body) Reset() { - *x = ListShardsResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[53] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListShardsResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListShardsResponse_Body) ProtoMessage() {} - -func (x *ListShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[53] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListShardsResponse_Body.ProtoReflect.Descriptor instead. -func (*ListShardsResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{9, 0} -} - -func (x *ListShardsResponse_Body) GetShards() []*ShardInfo { - if x != nil { - return x.Shards - } - return nil -} - -// Request body structure. -type SetShardModeRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // ID of the shard. - Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` - // Mode that requested to be set. - Mode ShardMode `protobuf:"varint,2,opt,name=mode,proto3,enum=control.ShardMode" json:"mode,omitempty"` - // Flag signifying whether error counter should be set to 0. - ResetErrorCounter bool `protobuf:"varint,3,opt,name=resetErrorCounter,proto3" json:"resetErrorCounter,omitempty"` -} - -func (x *SetShardModeRequest_Body) Reset() { - *x = SetShardModeRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[54] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SetShardModeRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SetShardModeRequest_Body) ProtoMessage() {} - -func (x *SetShardModeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[54] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SetShardModeRequest_Body.ProtoReflect.Descriptor instead. -func (*SetShardModeRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{10, 0} -} - -func (x *SetShardModeRequest_Body) GetShard_ID() [][]byte { - if x != nil { - return x.Shard_ID - } - return nil -} - -func (x *SetShardModeRequest_Body) GetMode() ShardMode { - if x != nil { - return x.Mode - } - return ShardMode_SHARD_MODE_UNDEFINED -} - -func (x *SetShardModeRequest_Body) GetResetErrorCounter() bool { - if x != nil { - return x.ResetErrorCounter - } - return false -} - -// Response body structure. -type SetShardModeResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *SetShardModeResponse_Body) Reset() { - *x = SetShardModeResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[55] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SetShardModeResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SetShardModeResponse_Body) ProtoMessage() {} - -func (x *SetShardModeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[55] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SetShardModeResponse_Body.ProtoReflect.Descriptor instead. -func (*SetShardModeResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{11, 0} -} - -// Request body structure. -type SynchronizeTreeRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` - TreeId string `protobuf:"bytes,2,opt,name=tree_id,json=treeId,proto3" json:"tree_id,omitempty"` - // Starting height for the synchronization. Can be omitted. - Height uint64 `protobuf:"varint,3,opt,name=height,proto3" json:"height,omitempty"` -} - -func (x *SynchronizeTreeRequest_Body) Reset() { - *x = SynchronizeTreeRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[56] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SynchronizeTreeRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SynchronizeTreeRequest_Body) ProtoMessage() {} - -func (x *SynchronizeTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[56] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SynchronizeTreeRequest_Body.ProtoReflect.Descriptor instead. -func (*SynchronizeTreeRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{12, 0} -} - -func (x *SynchronizeTreeRequest_Body) GetContainerId() []byte { - if x != nil { - return x.ContainerId - } - return nil -} - -func (x *SynchronizeTreeRequest_Body) GetTreeId() string { - if x != nil { - return x.TreeId - } - return "" -} - -func (x *SynchronizeTreeRequest_Body) GetHeight() uint64 { - if x != nil { - return x.Height - } - return 0 -} - -// Response body structure. -type SynchronizeTreeResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *SynchronizeTreeResponse_Body) Reset() { - *x = SynchronizeTreeResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[57] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SynchronizeTreeResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SynchronizeTreeResponse_Body) ProtoMessage() {} - -func (x *SynchronizeTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[57] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SynchronizeTreeResponse_Body.ProtoReflect.Descriptor instead. -func (*SynchronizeTreeResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{13, 0} -} - -// Request body structure. -type EvacuateShardRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // ID of the shard. - Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` - // Flag indicating whether object read errors should be ignored. - IgnoreErrors bool `protobuf:"varint,2,opt,name=ignore_errors,json=ignoreErrors,proto3" json:"ignore_errors,omitempty"` -} - -func (x *EvacuateShardRequest_Body) Reset() { - *x = EvacuateShardRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[58] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EvacuateShardRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EvacuateShardRequest_Body) ProtoMessage() {} - -func (x *EvacuateShardRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[58] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EvacuateShardRequest_Body.ProtoReflect.Descriptor instead. -func (*EvacuateShardRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{14, 0} -} - -func (x *EvacuateShardRequest_Body) GetShard_ID() [][]byte { - if x != nil { - return x.Shard_ID - } - return nil -} - -func (x *EvacuateShardRequest_Body) GetIgnoreErrors() bool { - if x != nil { - return x.IgnoreErrors - } - return false -} - -// Response body structure. -type EvacuateShardResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Count uint32 `protobuf:"varint,1,opt,name=count,proto3" json:"count,omitempty"` -} - -func (x *EvacuateShardResponse_Body) Reset() { - *x = EvacuateShardResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[59] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EvacuateShardResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EvacuateShardResponse_Body) ProtoMessage() {} - -func (x *EvacuateShardResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[59] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EvacuateShardResponse_Body.ProtoReflect.Descriptor instead. -func (*EvacuateShardResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{15, 0} -} - -func (x *EvacuateShardResponse_Body) GetCount() uint32 { - if x != nil { - return x.Count - } - return 0 -} - -// Request body structure. -type FlushCacheRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // ID of the shard. - Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` - // If true, then writecache will be left in read-only mode after flush - // completed. - Seal bool `protobuf:"varint,2,opt,name=seal,proto3" json:"seal,omitempty"` -} - -func (x *FlushCacheRequest_Body) Reset() { - *x = FlushCacheRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[60] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FlushCacheRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FlushCacheRequest_Body) ProtoMessage() {} - -func (x *FlushCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[60] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FlushCacheRequest_Body.ProtoReflect.Descriptor instead. -func (*FlushCacheRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{16, 0} -} - -func (x *FlushCacheRequest_Body) GetShard_ID() [][]byte { - if x != nil { - return x.Shard_ID - } - return nil -} - -func (x *FlushCacheRequest_Body) GetSeal() bool { - if x != nil { - return x.Seal - } - return false -} - -// Response body structure. -type FlushCacheResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *FlushCacheResponse_Body) Reset() { - *x = FlushCacheResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[61] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FlushCacheResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FlushCacheResponse_Body) ProtoMessage() {} - -func (x *FlushCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[61] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FlushCacheResponse_Body.ProtoReflect.Descriptor instead. -func (*FlushCacheResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{17, 0} -} - -// Request body structure. -type DoctorRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Number of threads to use for the operation. - Concurrency uint32 `protobuf:"varint,1,opt,name=concurrency,proto3" json:"concurrency,omitempty"` - // Flag to search engine for duplicate objects and leave only one copy. - RemoveDuplicates bool `protobuf:"varint,2,opt,name=remove_duplicates,json=removeDuplicates,proto3" json:"remove_duplicates,omitempty"` -} - -func (x *DoctorRequest_Body) Reset() { - *x = DoctorRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[62] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DoctorRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DoctorRequest_Body) ProtoMessage() {} - -func (x *DoctorRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[62] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DoctorRequest_Body.ProtoReflect.Descriptor instead. -func (*DoctorRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{18, 0} -} - -func (x *DoctorRequest_Body) GetConcurrency() uint32 { - if x != nil { - return x.Concurrency - } - return 0 -} - -func (x *DoctorRequest_Body) GetRemoveDuplicates() bool { - if x != nil { - return x.RemoveDuplicates - } - return false -} - -// Response body structure. -type DoctorResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *DoctorResponse_Body) Reset() { - *x = DoctorResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[63] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DoctorResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DoctorResponse_Body) ProtoMessage() {} - -func (x *DoctorResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[63] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DoctorResponse_Body.ProtoReflect.Descriptor instead. -func (*DoctorResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{19, 0} -} - -// Request body structure. -type StartShardEvacuationRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // IDs of the shards. - Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` - // Flag indicating whether object read errors should be ignored. - IgnoreErrors bool `protobuf:"varint,2,opt,name=ignore_errors,json=ignoreErrors,proto3" json:"ignore_errors,omitempty"` - // Evacuation scope. - Scope uint32 `protobuf:"varint,3,opt,name=scope,proto3" json:"scope,omitempty"` -} - -func (x *StartShardEvacuationRequest_Body) Reset() { - *x = StartShardEvacuationRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[64] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StartShardEvacuationRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StartShardEvacuationRequest_Body) ProtoMessage() {} - -func (x *StartShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[64] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StartShardEvacuationRequest_Body.ProtoReflect.Descriptor instead. -func (*StartShardEvacuationRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{20, 0} -} - -func (x *StartShardEvacuationRequest_Body) GetShard_ID() [][]byte { - if x != nil { - return x.Shard_ID - } - return nil -} - -func (x *StartShardEvacuationRequest_Body) GetIgnoreErrors() bool { - if x != nil { - return x.IgnoreErrors - } - return false -} - -func (x *StartShardEvacuationRequest_Body) GetScope() uint32 { - if x != nil { - return x.Scope - } - return 0 -} - -// Response body structure. -type StartShardEvacuationResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *StartShardEvacuationResponse_Body) Reset() { - *x = StartShardEvacuationResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[65] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StartShardEvacuationResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StartShardEvacuationResponse_Body) ProtoMessage() {} - -func (x *StartShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[65] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StartShardEvacuationResponse_Body.ProtoReflect.Descriptor instead. -func (*StartShardEvacuationResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{21, 0} -} - -// Request body structure. -type GetShardEvacuationStatusRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *GetShardEvacuationStatusRequest_Body) Reset() { - *x = GetShardEvacuationStatusRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[66] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetShardEvacuationStatusRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetShardEvacuationStatusRequest_Body) ProtoMessage() {} - -func (x *GetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[66] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetShardEvacuationStatusRequest_Body.ProtoReflect.Descriptor instead. -func (*GetShardEvacuationStatusRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{22, 0} -} - -// Response body structure. -type GetShardEvacuationStatusResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Total objects to evacuate count. The value is approximate, so evacuated + - // failed + skipped == total is not guaranteed after completion. - TotalObjects uint64 `protobuf:"varint,1,opt,name=total_objects,json=totalObjects,proto3" json:"total_objects,omitempty"` - // Evacuated objects count. - EvacuatedObjects uint64 `protobuf:"varint,2,opt,name=evacuated_objects,json=evacuatedObjects,proto3" json:"evacuated_objects,omitempty"` - // Failed objects count. - FailedObjects uint64 `protobuf:"varint,3,opt,name=failed_objects,json=failedObjects,proto3" json:"failed_objects,omitempty"` - // Shard IDs. - Shard_ID [][]byte `protobuf:"bytes,4,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` - // Evacuation process status. - Status GetShardEvacuationStatusResponse_Body_Status `protobuf:"varint,5,opt,name=status,proto3,enum=control.GetShardEvacuationStatusResponse_Body_Status" json:"status,omitempty"` - // Evacuation process duration. - Duration *GetShardEvacuationStatusResponse_Body_Duration `protobuf:"bytes,6,opt,name=duration,proto3" json:"duration,omitempty"` - // Evacuation process started at timestamp. - StartedAt *GetShardEvacuationStatusResponse_Body_UnixTimestamp `protobuf:"bytes,7,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` - // Error message if evacuation failed. - ErrorMessage string `protobuf:"bytes,8,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` - // Skipped objects count. - SkippedObjects uint64 `protobuf:"varint,9,opt,name=skipped_objects,json=skippedObjects,proto3" json:"skipped_objects,omitempty"` - // Total trees to evacuate count. - TotalTrees uint64 `protobuf:"varint,10,opt,name=total_trees,json=totalTrees,proto3" json:"total_trees,omitempty"` - // Evacuated trees count. - EvacuatedTrees uint64 `protobuf:"varint,11,opt,name=evacuated_trees,json=evacuatedTrees,proto3" json:"evacuated_trees,omitempty"` - // Failed trees count. - FailedTrees uint64 `protobuf:"varint,12,opt,name=failed_trees,json=failedTrees,proto3" json:"failed_trees,omitempty"` -} - -func (x *GetShardEvacuationStatusResponse_Body) Reset() { - *x = GetShardEvacuationStatusResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[67] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetShardEvacuationStatusResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetShardEvacuationStatusResponse_Body) ProtoMessage() {} - -func (x *GetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[67] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetShardEvacuationStatusResponse_Body.ProtoReflect.Descriptor instead. -func (*GetShardEvacuationStatusResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0} -} - -func (x *GetShardEvacuationStatusResponse_Body) GetTotalObjects() uint64 { - if x != nil { - return x.TotalObjects - } - return 0 -} - -func (x *GetShardEvacuationStatusResponse_Body) GetEvacuatedObjects() uint64 { - if x != nil { - return x.EvacuatedObjects - } - return 0 -} - -func (x *GetShardEvacuationStatusResponse_Body) GetFailedObjects() uint64 { - if x != nil { - return x.FailedObjects - } - return 0 -} - -func (x *GetShardEvacuationStatusResponse_Body) GetShard_ID() [][]byte { - if x != nil { - return x.Shard_ID - } - return nil -} - -func (x *GetShardEvacuationStatusResponse_Body) GetStatus() GetShardEvacuationStatusResponse_Body_Status { - if x != nil { - return x.Status - } - return GetShardEvacuationStatusResponse_Body_EVACUATE_SHARD_STATUS_UNDEFINED -} - -func (x *GetShardEvacuationStatusResponse_Body) GetDuration() *GetShardEvacuationStatusResponse_Body_Duration { - if x != nil { - return x.Duration - } - return nil -} - -func (x *GetShardEvacuationStatusResponse_Body) GetStartedAt() *GetShardEvacuationStatusResponse_Body_UnixTimestamp { - if x != nil { - return x.StartedAt - } - return nil -} - -func (x *GetShardEvacuationStatusResponse_Body) GetErrorMessage() string { - if x != nil { - return x.ErrorMessage - } - return "" -} - -func (x *GetShardEvacuationStatusResponse_Body) GetSkippedObjects() uint64 { - if x != nil { - return x.SkippedObjects - } - return 0 -} - -func (x *GetShardEvacuationStatusResponse_Body) GetTotalTrees() uint64 { - if x != nil { - return x.TotalTrees - } - return 0 -} - -func (x *GetShardEvacuationStatusResponse_Body) GetEvacuatedTrees() uint64 { - if x != nil { - return x.EvacuatedTrees - } - return 0 -} - -func (x *GetShardEvacuationStatusResponse_Body) GetFailedTrees() uint64 { - if x != nil { - return x.FailedTrees - } - return 0 -} - -// Unix timestamp value. -type GetShardEvacuationStatusResponse_Body_UnixTimestamp struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Value int64 `protobuf:"varint,1,opt,name=value,proto3" json:"value,omitempty"` -} - -func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) Reset() { - *x = GetShardEvacuationStatusResponse_Body_UnixTimestamp{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[68] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoMessage() {} - -func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[68] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetShardEvacuationStatusResponse_Body_UnixTimestamp.ProtoReflect.Descriptor instead. -func (*GetShardEvacuationStatusResponse_Body_UnixTimestamp) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0, 0} -} - -func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) GetValue() int64 { - if x != nil { - return x.Value - } - return 0 -} - -// Duration in seconds. -type GetShardEvacuationStatusResponse_Body_Duration struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Seconds int64 `protobuf:"varint,1,opt,name=seconds,proto3" json:"seconds,omitempty"` -} - -func (x *GetShardEvacuationStatusResponse_Body_Duration) Reset() { - *x = GetShardEvacuationStatusResponse_Body_Duration{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[69] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetShardEvacuationStatusResponse_Body_Duration) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetShardEvacuationStatusResponse_Body_Duration) ProtoMessage() {} - -func (x *GetShardEvacuationStatusResponse_Body_Duration) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[69] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetShardEvacuationStatusResponse_Body_Duration.ProtoReflect.Descriptor instead. -func (*GetShardEvacuationStatusResponse_Body_Duration) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{23, 0, 1} -} - -func (x *GetShardEvacuationStatusResponse_Body_Duration) GetSeconds() int64 { - if x != nil { - return x.Seconds - } - return 0 -} - -type ResetShardEvacuationStatusRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *ResetShardEvacuationStatusRequest_Body) Reset() { - *x = ResetShardEvacuationStatusRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[70] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ResetShardEvacuationStatusRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ResetShardEvacuationStatusRequest_Body) ProtoMessage() {} - -func (x *ResetShardEvacuationStatusRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[70] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ResetShardEvacuationStatusRequest_Body.ProtoReflect.Descriptor instead. -func (*ResetShardEvacuationStatusRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{24, 0} -} - -type ResetShardEvacuationStatusResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *ResetShardEvacuationStatusResponse_Body) Reset() { - *x = ResetShardEvacuationStatusResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[71] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ResetShardEvacuationStatusResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ResetShardEvacuationStatusResponse_Body) ProtoMessage() {} - -func (x *ResetShardEvacuationStatusResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[71] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ResetShardEvacuationStatusResponse_Body.ProtoReflect.Descriptor instead. -func (*ResetShardEvacuationStatusResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{25, 0} -} - -// Request body structure. -type StopShardEvacuationRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *StopShardEvacuationRequest_Body) Reset() { - *x = StopShardEvacuationRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[72] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StopShardEvacuationRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StopShardEvacuationRequest_Body) ProtoMessage() {} - -func (x *StopShardEvacuationRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[72] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StopShardEvacuationRequest_Body.ProtoReflect.Descriptor instead. -func (*StopShardEvacuationRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{26, 0} -} - -// Response body structure. -type StopShardEvacuationResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *StopShardEvacuationResponse_Body) Reset() { - *x = StopShardEvacuationResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[73] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StopShardEvacuationResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StopShardEvacuationResponse_Body) ProtoMessage() {} - -func (x *StopShardEvacuationResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[73] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StopShardEvacuationResponse_Body.ProtoReflect.Descriptor instead. -func (*StopShardEvacuationResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{27, 0} -} - -type AddChainLocalOverrideRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Target for which the overrides are applied. - Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` - // Serialized rule chain. If chain ID is left empty - // in the chain, then it will be generated and returned - // in the response. - Chain []byte `protobuf:"bytes,2,opt,name=chain,proto3" json:"chain,omitempty"` -} - -func (x *AddChainLocalOverrideRequest_Body) Reset() { - *x = AddChainLocalOverrideRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[74] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddChainLocalOverrideRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddChainLocalOverrideRequest_Body) ProtoMessage() {} - -func (x *AddChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[74] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. -func (*AddChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{28, 0} -} - -func (x *AddChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { - if x != nil { - return x.Target - } - return nil -} - -func (x *AddChainLocalOverrideRequest_Body) GetChain() []byte { - if x != nil { - return x.Chain - } - return nil -} - -type AddChainLocalOverrideResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Chain ID assigned for the added rule chain. - // If chain ID is left empty in the request, then - // it will be generated. - ChainId []byte `protobuf:"bytes,1,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` -} - -func (x *AddChainLocalOverrideResponse_Body) Reset() { - *x = AddChainLocalOverrideResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[75] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddChainLocalOverrideResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddChainLocalOverrideResponse_Body) ProtoMessage() {} - -func (x *AddChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[75] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. -func (*AddChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{29, 0} -} - -func (x *AddChainLocalOverrideResponse_Body) GetChainId() []byte { - if x != nil { - return x.ChainId - } - return nil -} - -type GetChainLocalOverrideRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Target for which the overrides are applied. - Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` - // Chain ID assigned for the added rule chain. - ChainId []byte `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` -} - -func (x *GetChainLocalOverrideRequest_Body) Reset() { - *x = GetChainLocalOverrideRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[76] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetChainLocalOverrideRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetChainLocalOverrideRequest_Body) ProtoMessage() {} - -func (x *GetChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[76] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. -func (*GetChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{30, 0} -} - -func (x *GetChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { - if x != nil { - return x.Target - } - return nil -} - -func (x *GetChainLocalOverrideRequest_Body) GetChainId() []byte { - if x != nil { - return x.ChainId - } - return nil -} - -type GetChainLocalOverrideResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Serialized rule chain. - Chain []byte `protobuf:"bytes,1,opt,name=chain,proto3" json:"chain,omitempty"` -} - -func (x *GetChainLocalOverrideResponse_Body) Reset() { - *x = GetChainLocalOverrideResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[77] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetChainLocalOverrideResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetChainLocalOverrideResponse_Body) ProtoMessage() {} - -func (x *GetChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[77] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. -func (*GetChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{31, 0} -} - -func (x *GetChainLocalOverrideResponse_Body) GetChain() []byte { - if x != nil { - return x.Chain - } - return nil -} - -type ListChainLocalOverridesRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Target for which the overrides are applied. - Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` -} - -func (x *ListChainLocalOverridesRequest_Body) Reset() { - *x = ListChainLocalOverridesRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[78] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListChainLocalOverridesRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListChainLocalOverridesRequest_Body) ProtoMessage() {} - -func (x *ListChainLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[78] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListChainLocalOverridesRequest_Body.ProtoReflect.Descriptor instead. -func (*ListChainLocalOverridesRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{32, 0} -} - -func (x *ListChainLocalOverridesRequest_Body) GetTarget() *ChainTarget { - if x != nil { - return x.Target - } - return nil -} - -type ListChainLocalOverridesResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // The list of serialized rule chain. - Chains [][]byte `protobuf:"bytes,1,rep,name=chains,proto3" json:"chains,omitempty"` -} - -func (x *ListChainLocalOverridesResponse_Body) Reset() { - *x = ListChainLocalOverridesResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[79] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListChainLocalOverridesResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListChainLocalOverridesResponse_Body) ProtoMessage() {} - -func (x *ListChainLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[79] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListChainLocalOverridesResponse_Body.ProtoReflect.Descriptor instead. -func (*ListChainLocalOverridesResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{33, 0} -} - -func (x *ListChainLocalOverridesResponse_Body) GetChains() [][]byte { - if x != nil { - return x.Chains - } - return nil -} - -type ListTargetsLocalOverridesRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Target for which the overrides are applied. - ChainName string `protobuf:"bytes,1,opt,name=chainName,proto3" json:"chainName,omitempty"` -} - -func (x *ListTargetsLocalOverridesRequest_Body) Reset() { - *x = ListTargetsLocalOverridesRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[80] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListTargetsLocalOverridesRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListTargetsLocalOverridesRequest_Body) ProtoMessage() {} - -func (x *ListTargetsLocalOverridesRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[80] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListTargetsLocalOverridesRequest_Body.ProtoReflect.Descriptor instead. -func (*ListTargetsLocalOverridesRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{34, 0} -} - -func (x *ListTargetsLocalOverridesRequest_Body) GetChainName() string { - if x != nil { - return x.ChainName - } - return "" -} - -type ListTargetsLocalOverridesResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // The list of chain targets. - Targets []*ChainTarget `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty"` -} - -func (x *ListTargetsLocalOverridesResponse_Body) Reset() { - *x = ListTargetsLocalOverridesResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[81] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListTargetsLocalOverridesResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListTargetsLocalOverridesResponse_Body) ProtoMessage() {} - -func (x *ListTargetsLocalOverridesResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[81] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListTargetsLocalOverridesResponse_Body.ProtoReflect.Descriptor instead. -func (*ListTargetsLocalOverridesResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{35, 0} -} - -func (x *ListTargetsLocalOverridesResponse_Body) GetTargets() []*ChainTarget { - if x != nil { - return x.Targets - } - return nil -} - -type RemoveChainLocalOverrideRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Target for which the overrides are applied. - Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` - // Chain ID assigned for the added rule chain. - ChainId []byte `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` -} - -func (x *RemoveChainLocalOverrideRequest_Body) Reset() { - *x = RemoveChainLocalOverrideRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[82] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveChainLocalOverrideRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveChainLocalOverrideRequest_Body) ProtoMessage() {} - -func (x *RemoveChainLocalOverrideRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[82] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveChainLocalOverrideRequest_Body.ProtoReflect.Descriptor instead. -func (*RemoveChainLocalOverrideRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{36, 0} -} - -func (x *RemoveChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { - if x != nil { - return x.Target - } - return nil -} - -func (x *RemoveChainLocalOverrideRequest_Body) GetChainId() []byte { - if x != nil { - return x.ChainId - } - return nil -} - -type RemoveChainLocalOverrideResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *RemoveChainLocalOverrideResponse_Body) Reset() { - *x = RemoveChainLocalOverrideResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[83] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveChainLocalOverrideResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveChainLocalOverrideResponse_Body) ProtoMessage() {} - -func (x *RemoveChainLocalOverrideResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[83] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveChainLocalOverrideResponse_Body.ProtoReflect.Descriptor instead. -func (*RemoveChainLocalOverrideResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{37, 0} -} - -type RemoveChainLocalOverridesByTargetRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Target for which the overrides are applied. - Target *ChainTarget `protobuf:"bytes,1,opt,name=target,proto3" json:"target,omitempty"` -} - -func (x *RemoveChainLocalOverridesByTargetRequest_Body) Reset() { - *x = RemoveChainLocalOverridesByTargetRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[84] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveChainLocalOverridesByTargetRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveChainLocalOverridesByTargetRequest_Body) ProtoMessage() {} - -func (x *RemoveChainLocalOverridesByTargetRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[84] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveChainLocalOverridesByTargetRequest_Body.ProtoReflect.Descriptor instead. -func (*RemoveChainLocalOverridesByTargetRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{38, 0} -} - -func (x *RemoveChainLocalOverridesByTargetRequest_Body) GetTarget() *ChainTarget { - if x != nil { - return x.Target - } - return nil -} - -type RemoveChainLocalOverridesByTargetResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *RemoveChainLocalOverridesByTargetResponse_Body) Reset() { - *x = RemoveChainLocalOverridesByTargetResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[85] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveChainLocalOverridesByTargetResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveChainLocalOverridesByTargetResponse_Body) ProtoMessage() {} - -func (x *RemoveChainLocalOverridesByTargetResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[85] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveChainLocalOverridesByTargetResponse_Body.ProtoReflect.Descriptor instead. -func (*RemoveChainLocalOverridesByTargetResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{39, 0} -} - -// Request body structure. -type SealWriteCacheRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // ID of the shard. - Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` - // Flag indicating whether object read errors should be ignored. - IgnoreErrors bool `protobuf:"varint,2,opt,name=ignore_errors,json=ignoreErrors,proto3" json:"ignore_errors,omitempty"` - // Flag indicating whether writecache will be sealed async. - Async bool `protobuf:"varint,3,opt,name=async,proto3" json:"async,omitempty"` - // If true, then writecache will be sealed, but mode will be restored to the current one. - RestoreMode bool `protobuf:"varint,4,opt,name=restore_mode,json=restoreMode,proto3" json:"restore_mode,omitempty"` - // If true, then writecache will shrink internal storage. - Shrink bool `protobuf:"varint,5,opt,name=shrink,proto3" json:"shrink,omitempty"` -} - -func (x *SealWriteCacheRequest_Body) Reset() { - *x = SealWriteCacheRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[86] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SealWriteCacheRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SealWriteCacheRequest_Body) ProtoMessage() {} - -func (x *SealWriteCacheRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[86] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SealWriteCacheRequest_Body.ProtoReflect.Descriptor instead. -func (*SealWriteCacheRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{40, 0} -} - -func (x *SealWriteCacheRequest_Body) GetShard_ID() [][]byte { - if x != nil { - return x.Shard_ID - } - return nil -} - -func (x *SealWriteCacheRequest_Body) GetIgnoreErrors() bool { - if x != nil { - return x.IgnoreErrors - } - return false -} - -func (x *SealWriteCacheRequest_Body) GetAsync() bool { - if x != nil { - return x.Async - } - return false -} - -func (x *SealWriteCacheRequest_Body) GetRestoreMode() bool { - if x != nil { - return x.RestoreMode - } - return false -} - -func (x *SealWriteCacheRequest_Body) GetShrink() bool { - if x != nil { - return x.Shrink - } - return false -} - -type SealWriteCacheResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Results []*SealWriteCacheResponse_Body_Status `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` -} - -func (x *SealWriteCacheResponse_Body) Reset() { - *x = SealWriteCacheResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[87] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SealWriteCacheResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SealWriteCacheResponse_Body) ProtoMessage() {} - -func (x *SealWriteCacheResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[87] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SealWriteCacheResponse_Body.ProtoReflect.Descriptor instead. -func (*SealWriteCacheResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{41, 0} -} - -func (x *SealWriteCacheResponse_Body) GetResults() []*SealWriteCacheResponse_Body_Status { - if x != nil { - return x.Results - } - return nil -} - -type SealWriteCacheResponse_Body_Status struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Shard_ID []byte `protobuf:"bytes,1,opt,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` - Success bool `protobuf:"varint,2,opt,name=success,proto3" json:"success,omitempty"` - Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` -} - -func (x *SealWriteCacheResponse_Body_Status) Reset() { - *x = SealWriteCacheResponse_Body_Status{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[88] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SealWriteCacheResponse_Body_Status) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SealWriteCacheResponse_Body_Status) ProtoMessage() {} - -func (x *SealWriteCacheResponse_Body_Status) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[88] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SealWriteCacheResponse_Body_Status.ProtoReflect.Descriptor instead. -func (*SealWriteCacheResponse_Body_Status) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{41, 0, 0} -} - -func (x *SealWriteCacheResponse_Body_Status) GetShard_ID() []byte { - if x != nil { - return x.Shard_ID - } - return nil -} - -func (x *SealWriteCacheResponse_Body_Status) GetSuccess() bool { - if x != nil { - return x.Success - } - return false -} - -func (x *SealWriteCacheResponse_Body_Status) GetError() string { - if x != nil { - return x.Error - } - return "" -} - -type DetachShardsRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Shard_ID [][]byte `protobuf:"bytes,1,rep,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` -} - -func (x *DetachShardsRequest_Body) Reset() { - *x = DetachShardsRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[89] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DetachShardsRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DetachShardsRequest_Body) ProtoMessage() {} - -func (x *DetachShardsRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[89] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DetachShardsRequest_Body.ProtoReflect.Descriptor instead. -func (*DetachShardsRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{42, 0} -} - -func (x *DetachShardsRequest_Body) GetShard_ID() [][]byte { - if x != nil { - return x.Shard_ID - } - return nil -} - -type DetachShardsResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *DetachShardsResponse_Body) Reset() { - *x = DetachShardsResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_service_proto_msgTypes[90] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DetachShardsResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DetachShardsResponse_Body) ProtoMessage() {} - -func (x *DetachShardsResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_service_proto_msgTypes[90] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DetachShardsResponse_Body.ProtoReflect.Descriptor instead. -func (*DetachShardsResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_control_service_proto_rawDescGZIP(), []int{43, 0} -} - -var File_pkg_services_control_service_proto protoreflect.FileDescriptor - -var file_pkg_services_control_service_proto_rawDesc = []byte{ - 0x0a, 0x22, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x1a, 0x20, 0x70, - 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, - 0x84, 0x01, 0x0a, 0x12, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xfe, 0x01, 0x0a, 0x13, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x7e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x3a, 0x0a, 0x0d, 0x6e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0c, 0x6e, - 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x3a, 0x0a, 0x0d, 0x68, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, 0x65, 0x61, - 0x6c, 0x74, 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0c, 0x68, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0xe8, 0x01, 0x0a, 0x16, 0x53, 0x65, 0x74, 0x4e, - 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, - 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x62, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x6d, - 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x10, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x4d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, - 0x63, 0x65, 0x22, 0x8e, 0x01, 0x0a, 0x17, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x22, 0x8c, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, - 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x22, 0xd3, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4b, 0x0a, 0x04, 0x42, - 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4e, 0x65, - 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x22, 0xa7, 0x01, 0x0a, 0x12, 0x44, 0x72, 0x6f, - 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x29, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x21, 0x0a, 0x0c, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0b, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x4c, 0x69, - 0x73, 0x74, 0x22, 0x86, 0x01, 0x0a, 0x13, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x82, 0x01, 0x0a, 0x11, - 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x33, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x22, 0xb0, 0x01, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x32, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2a, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x64, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x73, 0x22, 0xf7, 0x01, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x1a, 0x77, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, - 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, - 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x26, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, - 0x2c, 0x0a, 0x11, 0x72, 0x65, 0x73, 0x65, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x75, - 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x72, 0x65, 0x73, 0x65, - 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x22, 0x88, 0x01, - 0x0a, 0x14, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, - 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xe0, 0x01, 0x0a, 0x16, 0x53, 0x79, 0x6e, - 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, - 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x5a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, - 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, - 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0x8e, 0x01, 0x0a, 0x17, - 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xc8, 0x01, 0x0a, - 0x14, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x46, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, - 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, - 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, 0x6f, 0x72, - 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0xa0, 0x01, 0x0a, 0x15, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, 0x0a, 0x04, - 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xb1, 0x01, 0x0a, 0x11, 0x46, - 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x33, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, - 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x35, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x65, - 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x73, 0x65, 0x61, 0x6c, 0x22, 0x84, - 0x01, 0x0a, 0x12, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, - 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xc9, 0x01, 0x0a, 0x0d, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x55, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, - 0x65, 0x6e, 0x63, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x64, - 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x10, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x44, 0x75, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x73, 0x22, 0x7c, 0x0a, 0x0e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, - 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, - 0x98, 0x02, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, - 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x87, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, - 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, - 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x22, - 0x29, 0x0a, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f, 0x4e, 0x45, - 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x42, 0x4a, 0x45, 0x43, 0x54, 0x53, 0x10, 0x01, 0x12, - 0x09, 0x0a, 0x05, 0x54, 0x52, 0x45, 0x45, 0x53, 0x10, 0x02, 0x22, 0x98, 0x01, 0x0a, 0x1c, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x9e, 0x01, 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x89, 0x07, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, - 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0xee, 0x05, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, - 0x74, 0x61, 0x6c, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, - 0x2b, 0x0a, 0x11, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x65, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x25, 0x0a, 0x0e, - 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x4d, - 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x35, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x53, 0x0a, - 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, - 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x5b, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, - 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x5f, - 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x73, - 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1f, 0x0a, - 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x74, 0x72, 0x65, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x72, 0x65, 0x65, 0x73, 0x12, 0x27, - 0x0a, 0x0f, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x72, 0x65, 0x65, - 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x65, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, - 0x65, 0x64, 0x54, 0x72, 0x65, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x61, 0x69, 0x6c, 0x65, - 0x64, 0x5f, 0x74, 0x72, 0x65, 0x65, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x66, - 0x61, 0x69, 0x6c, 0x65, 0x64, 0x54, 0x72, 0x65, 0x65, 0x73, 0x1a, 0x25, 0x0a, 0x0d, 0x55, 0x6e, - 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x1a, 0x24, 0x0a, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, - 0x07, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, - 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x49, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x23, 0x0a, 0x1f, 0x45, 0x56, 0x41, 0x43, 0x55, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x48, - 0x41, 0x52, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, - 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, - 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, - 0x10, 0x02, 0x22, 0xa2, 0x01, 0x0a, 0x21, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, - 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xa4, 0x01, 0x0a, 0x22, 0x52, 0x65, 0x73, 0x65, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x94, - 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x96, 0x01, 0x0a, 0x1b, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, - 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xdc, - 0x01, 0x0a, 0x1c, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, - 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x4a, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, - 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xb5, 0x01, - 0x0a, 0x1d, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x3f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xe1, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, - 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, - 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xb0, 0x01, 0x0a, 0x1d, 0x47, 0x65, - 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, - 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x1c, - 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x22, 0xca, 0x01, 0x0a, - 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, - 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x40, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb6, 0x01, 0x0a, 0x1f, 0x4c, 0x69, - 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x1e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x68, - 0x61, 0x69, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x68, 0x61, 0x69, - 0x6e, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x20, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x24, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, 0x61, - 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x4e, - 0x61, 0x6d, 0x65, 0x22, 0xd2, 0x01, 0x0a, 0x21, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, - 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x36, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2e, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, - 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, - 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x4f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, - 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x49, 0x64, 0x22, 0xa0, 0x01, 0x0a, 0x20, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xde, 0x01, 0x0a, 0x28, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, - 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x4a, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x36, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, - 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x34, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb2, 0x01, 0x0a, 0x29, 0x52, 0x65, 0x6d, 0x6f, 0x76, - 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, - 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x9c, 0x02, 0x0a, 0x15, - 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, - 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, - 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x97, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x5f, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x67, 0x6e, - 0x6f, 0x72, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x73, 0x79, - 0x6e, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x12, - 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x4d, 0x6f, - 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, 0x72, 0x69, 0x6e, 0x6b, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x06, 0x73, 0x68, 0x72, 0x69, 0x6e, 0x6b, 0x22, 0xa9, 0x02, 0x0a, 0x16, 0x53, - 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, - 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, - 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0xa2, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x45, 0x0a, 0x07, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, - 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x73, 0x1a, 0x53, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, - 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa1, 0x01, 0x0a, 0x13, 0x44, 0x65, 0x74, 0x61, 0x63, - 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x35, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x21, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x19, 0x0a, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0c, 0x52, 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x22, 0x88, 0x01, 0x0a, 0x14, 0x44, - 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x22, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x65, 0x74, 0x61, - 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x30, 0x0a, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, - 0x04, 0x42, 0x6f, 0x64, 0x79, 0x32, 0x8b, 0x10, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x48, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4e, - 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, - 0x0a, 0x0b, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1b, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x72, 0x6f, 0x70, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x12, - 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, - 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, - 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x12, - 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, 0x72, - 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x20, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x68, - 0x72, 0x6f, 0x6e, 0x69, 0x7a, 0x65, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x12, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, - 0x61, 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x45, 0x76, 0x61, - 0x63, 0x75, 0x61, 0x74, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x63, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x75, 0x0a, 0x1a, 0x52, 0x65, 0x73, 0x65, - 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x73, - 0x65, 0x74, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x60, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, - 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, 0x76, 0x61, 0x63, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x68, 0x61, 0x72, 0x64, 0x45, - 0x76, 0x61, 0x63, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, - 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x6f, 0x63, 0x74, - 0x6f, 0x72, 0x12, 0x16, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, - 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x63, 0x6f, 0x6e, - 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, 0x6f, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x41, 0x64, - 0x64, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x15, 0x47, - 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x12, 0x25, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, - 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x27, - 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x6f, 0x0a, 0x18, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, - 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x28, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, - 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x8a, 0x01, 0x0a, 0x21, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, - 0x69, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, - 0x42, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x31, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, 0x79, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x63, 0x6f, - 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x68, 0x61, 0x69, - 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x42, - 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x72, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x29, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x73, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x4c, 0x6f, 0x63, - 0x61, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x53, 0x65, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, - 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x12, 0x1c, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2e, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x44, - 0x65, 0x74, 0x61, 0x63, 0x68, 0x53, 0x68, 0x61, 0x72, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, - 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, - 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, - 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_pkg_services_control_service_proto_rawDescOnce sync.Once - file_pkg_services_control_service_proto_rawDescData = file_pkg_services_control_service_proto_rawDesc -) - -func file_pkg_services_control_service_proto_rawDescGZIP() []byte { - file_pkg_services_control_service_proto_rawDescOnce.Do(func() { - file_pkg_services_control_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_services_control_service_proto_rawDescData) - }) - return file_pkg_services_control_service_proto_rawDescData -} - -var file_pkg_services_control_service_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_pkg_services_control_service_proto_msgTypes = make([]protoimpl.MessageInfo, 91) -var file_pkg_services_control_service_proto_goTypes = []interface{}{ - (StartShardEvacuationRequest_Body_Scope)(0), // 0: control.StartShardEvacuationRequest.Body.Scope - (GetShardEvacuationStatusResponse_Body_Status)(0), // 1: control.GetShardEvacuationStatusResponse.Body.Status - (*HealthCheckRequest)(nil), // 2: control.HealthCheckRequest - (*HealthCheckResponse)(nil), // 3: control.HealthCheckResponse - (*SetNetmapStatusRequest)(nil), // 4: control.SetNetmapStatusRequest - (*SetNetmapStatusResponse)(nil), // 5: control.SetNetmapStatusResponse - (*GetNetmapStatusRequest)(nil), // 6: control.GetNetmapStatusRequest - (*GetNetmapStatusResponse)(nil), // 7: control.GetNetmapStatusResponse - (*DropObjectsRequest)(nil), // 8: control.DropObjectsRequest - (*DropObjectsResponse)(nil), // 9: control.DropObjectsResponse - (*ListShardsRequest)(nil), // 10: control.ListShardsRequest - (*ListShardsResponse)(nil), // 11: control.ListShardsResponse - (*SetShardModeRequest)(nil), // 12: control.SetShardModeRequest - (*SetShardModeResponse)(nil), // 13: control.SetShardModeResponse - (*SynchronizeTreeRequest)(nil), // 14: control.SynchronizeTreeRequest - (*SynchronizeTreeResponse)(nil), // 15: control.SynchronizeTreeResponse - (*EvacuateShardRequest)(nil), // 16: control.EvacuateShardRequest - (*EvacuateShardResponse)(nil), // 17: control.EvacuateShardResponse - (*FlushCacheRequest)(nil), // 18: control.FlushCacheRequest - (*FlushCacheResponse)(nil), // 19: control.FlushCacheResponse - (*DoctorRequest)(nil), // 20: control.DoctorRequest - (*DoctorResponse)(nil), // 21: control.DoctorResponse - (*StartShardEvacuationRequest)(nil), // 22: control.StartShardEvacuationRequest - (*StartShardEvacuationResponse)(nil), // 23: control.StartShardEvacuationResponse - (*GetShardEvacuationStatusRequest)(nil), // 24: control.GetShardEvacuationStatusRequest - (*GetShardEvacuationStatusResponse)(nil), // 25: control.GetShardEvacuationStatusResponse - (*ResetShardEvacuationStatusRequest)(nil), // 26: control.ResetShardEvacuationStatusRequest - (*ResetShardEvacuationStatusResponse)(nil), // 27: control.ResetShardEvacuationStatusResponse - (*StopShardEvacuationRequest)(nil), // 28: control.StopShardEvacuationRequest - (*StopShardEvacuationResponse)(nil), // 29: control.StopShardEvacuationResponse - (*AddChainLocalOverrideRequest)(nil), // 30: control.AddChainLocalOverrideRequest - (*AddChainLocalOverrideResponse)(nil), // 31: control.AddChainLocalOverrideResponse - (*GetChainLocalOverrideRequest)(nil), // 32: control.GetChainLocalOverrideRequest - (*GetChainLocalOverrideResponse)(nil), // 33: control.GetChainLocalOverrideResponse - (*ListChainLocalOverridesRequest)(nil), // 34: control.ListChainLocalOverridesRequest - (*ListChainLocalOverridesResponse)(nil), // 35: control.ListChainLocalOverridesResponse - (*ListTargetsLocalOverridesRequest)(nil), // 36: control.ListTargetsLocalOverridesRequest - (*ListTargetsLocalOverridesResponse)(nil), // 37: control.ListTargetsLocalOverridesResponse - (*RemoveChainLocalOverrideRequest)(nil), // 38: control.RemoveChainLocalOverrideRequest - (*RemoveChainLocalOverrideResponse)(nil), // 39: control.RemoveChainLocalOverrideResponse - (*RemoveChainLocalOverridesByTargetRequest)(nil), // 40: control.RemoveChainLocalOverridesByTargetRequest - (*RemoveChainLocalOverridesByTargetResponse)(nil), // 41: control.RemoveChainLocalOverridesByTargetResponse - (*SealWriteCacheRequest)(nil), // 42: control.SealWriteCacheRequest - (*SealWriteCacheResponse)(nil), // 43: control.SealWriteCacheResponse - (*DetachShardsRequest)(nil), // 44: control.DetachShardsRequest - (*DetachShardsResponse)(nil), // 45: control.DetachShardsResponse - (*HealthCheckRequest_Body)(nil), // 46: control.HealthCheckRequest.Body - (*HealthCheckResponse_Body)(nil), // 47: control.HealthCheckResponse.Body - (*SetNetmapStatusRequest_Body)(nil), // 48: control.SetNetmapStatusRequest.Body - (*SetNetmapStatusResponse_Body)(nil), // 49: control.SetNetmapStatusResponse.Body - (*GetNetmapStatusRequest_Body)(nil), // 50: control.GetNetmapStatusRequest.Body - (*GetNetmapStatusResponse_Body)(nil), // 51: control.GetNetmapStatusResponse.Body - (*DropObjectsRequest_Body)(nil), // 52: control.DropObjectsRequest.Body - (*DropObjectsResponse_Body)(nil), // 53: control.DropObjectsResponse.Body - (*ListShardsRequest_Body)(nil), // 54: control.ListShardsRequest.Body - (*ListShardsResponse_Body)(nil), // 55: control.ListShardsResponse.Body - (*SetShardModeRequest_Body)(nil), // 56: control.SetShardModeRequest.Body - (*SetShardModeResponse_Body)(nil), // 57: control.SetShardModeResponse.Body - (*SynchronizeTreeRequest_Body)(nil), // 58: control.SynchronizeTreeRequest.Body - (*SynchronizeTreeResponse_Body)(nil), // 59: control.SynchronizeTreeResponse.Body - (*EvacuateShardRequest_Body)(nil), // 60: control.EvacuateShardRequest.Body - (*EvacuateShardResponse_Body)(nil), // 61: control.EvacuateShardResponse.Body - (*FlushCacheRequest_Body)(nil), // 62: control.FlushCacheRequest.Body - (*FlushCacheResponse_Body)(nil), // 63: control.FlushCacheResponse.Body - (*DoctorRequest_Body)(nil), // 64: control.DoctorRequest.Body - (*DoctorResponse_Body)(nil), // 65: control.DoctorResponse.Body - (*StartShardEvacuationRequest_Body)(nil), // 66: control.StartShardEvacuationRequest.Body - (*StartShardEvacuationResponse_Body)(nil), // 67: control.StartShardEvacuationResponse.Body - (*GetShardEvacuationStatusRequest_Body)(nil), // 68: control.GetShardEvacuationStatusRequest.Body - (*GetShardEvacuationStatusResponse_Body)(nil), // 69: control.GetShardEvacuationStatusResponse.Body - (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil), // 70: control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - (*GetShardEvacuationStatusResponse_Body_Duration)(nil), // 71: control.GetShardEvacuationStatusResponse.Body.Duration - (*ResetShardEvacuationStatusRequest_Body)(nil), // 72: control.ResetShardEvacuationStatusRequest.Body - (*ResetShardEvacuationStatusResponse_Body)(nil), // 73: control.ResetShardEvacuationStatusResponse.Body - (*StopShardEvacuationRequest_Body)(nil), // 74: control.StopShardEvacuationRequest.Body - (*StopShardEvacuationResponse_Body)(nil), // 75: control.StopShardEvacuationResponse.Body - (*AddChainLocalOverrideRequest_Body)(nil), // 76: control.AddChainLocalOverrideRequest.Body - (*AddChainLocalOverrideResponse_Body)(nil), // 77: control.AddChainLocalOverrideResponse.Body - (*GetChainLocalOverrideRequest_Body)(nil), // 78: control.GetChainLocalOverrideRequest.Body - (*GetChainLocalOverrideResponse_Body)(nil), // 79: control.GetChainLocalOverrideResponse.Body - (*ListChainLocalOverridesRequest_Body)(nil), // 80: control.ListChainLocalOverridesRequest.Body - (*ListChainLocalOverridesResponse_Body)(nil), // 81: control.ListChainLocalOverridesResponse.Body - (*ListTargetsLocalOverridesRequest_Body)(nil), // 82: control.ListTargetsLocalOverridesRequest.Body - (*ListTargetsLocalOverridesResponse_Body)(nil), // 83: control.ListTargetsLocalOverridesResponse.Body - (*RemoveChainLocalOverrideRequest_Body)(nil), // 84: control.RemoveChainLocalOverrideRequest.Body - (*RemoveChainLocalOverrideResponse_Body)(nil), // 85: control.RemoveChainLocalOverrideResponse.Body - (*RemoveChainLocalOverridesByTargetRequest_Body)(nil), // 86: control.RemoveChainLocalOverridesByTargetRequest.Body - (*RemoveChainLocalOverridesByTargetResponse_Body)(nil), // 87: control.RemoveChainLocalOverridesByTargetResponse.Body - (*SealWriteCacheRequest_Body)(nil), // 88: control.SealWriteCacheRequest.Body - (*SealWriteCacheResponse_Body)(nil), // 89: control.SealWriteCacheResponse.Body - (*SealWriteCacheResponse_Body_Status)(nil), // 90: control.SealWriteCacheResponse.Body.Status - (*DetachShardsRequest_Body)(nil), // 91: control.DetachShardsRequest.Body - (*DetachShardsResponse_Body)(nil), // 92: control.DetachShardsResponse.Body - (*Signature)(nil), // 93: control.Signature - (NetmapStatus)(0), // 94: control.NetmapStatus - (HealthStatus)(0), // 95: control.HealthStatus - (*ShardInfo)(nil), // 96: control.ShardInfo - (ShardMode)(0), // 97: control.ShardMode - (*ChainTarget)(nil), // 98: control.ChainTarget -} -var file_pkg_services_control_service_proto_depIdxs = []int32{ - 46, // 0: control.HealthCheckRequest.body:type_name -> control.HealthCheckRequest.Body - 93, // 1: control.HealthCheckRequest.signature:type_name -> control.Signature - 47, // 2: control.HealthCheckResponse.body:type_name -> control.HealthCheckResponse.Body - 93, // 3: control.HealthCheckResponse.signature:type_name -> control.Signature - 48, // 4: control.SetNetmapStatusRequest.body:type_name -> control.SetNetmapStatusRequest.Body - 93, // 5: control.SetNetmapStatusRequest.signature:type_name -> control.Signature - 49, // 6: control.SetNetmapStatusResponse.body:type_name -> control.SetNetmapStatusResponse.Body - 93, // 7: control.SetNetmapStatusResponse.signature:type_name -> control.Signature - 50, // 8: control.GetNetmapStatusRequest.body:type_name -> control.GetNetmapStatusRequest.Body - 93, // 9: control.GetNetmapStatusRequest.signature:type_name -> control.Signature - 51, // 10: control.GetNetmapStatusResponse.body:type_name -> control.GetNetmapStatusResponse.Body - 93, // 11: control.GetNetmapStatusResponse.signature:type_name -> control.Signature - 52, // 12: control.DropObjectsRequest.body:type_name -> control.DropObjectsRequest.Body - 93, // 13: control.DropObjectsRequest.signature:type_name -> control.Signature - 53, // 14: control.DropObjectsResponse.body:type_name -> control.DropObjectsResponse.Body - 93, // 15: control.DropObjectsResponse.signature:type_name -> control.Signature - 54, // 16: control.ListShardsRequest.body:type_name -> control.ListShardsRequest.Body - 93, // 17: control.ListShardsRequest.signature:type_name -> control.Signature - 55, // 18: control.ListShardsResponse.body:type_name -> control.ListShardsResponse.Body - 93, // 19: control.ListShardsResponse.signature:type_name -> control.Signature - 56, // 20: control.SetShardModeRequest.body:type_name -> control.SetShardModeRequest.Body - 93, // 21: control.SetShardModeRequest.signature:type_name -> control.Signature - 57, // 22: control.SetShardModeResponse.body:type_name -> control.SetShardModeResponse.Body - 93, // 23: control.SetShardModeResponse.signature:type_name -> control.Signature - 58, // 24: control.SynchronizeTreeRequest.body:type_name -> control.SynchronizeTreeRequest.Body - 93, // 25: control.SynchronizeTreeRequest.signature:type_name -> control.Signature - 59, // 26: control.SynchronizeTreeResponse.body:type_name -> control.SynchronizeTreeResponse.Body - 93, // 27: control.SynchronizeTreeResponse.signature:type_name -> control.Signature - 60, // 28: control.EvacuateShardRequest.body:type_name -> control.EvacuateShardRequest.Body - 93, // 29: control.EvacuateShardRequest.signature:type_name -> control.Signature - 61, // 30: control.EvacuateShardResponse.body:type_name -> control.EvacuateShardResponse.Body - 93, // 31: control.EvacuateShardResponse.signature:type_name -> control.Signature - 62, // 32: control.FlushCacheRequest.body:type_name -> control.FlushCacheRequest.Body - 93, // 33: control.FlushCacheRequest.signature:type_name -> control.Signature - 63, // 34: control.FlushCacheResponse.body:type_name -> control.FlushCacheResponse.Body - 93, // 35: control.FlushCacheResponse.signature:type_name -> control.Signature - 64, // 36: control.DoctorRequest.body:type_name -> control.DoctorRequest.Body - 93, // 37: control.DoctorRequest.signature:type_name -> control.Signature - 65, // 38: control.DoctorResponse.body:type_name -> control.DoctorResponse.Body - 93, // 39: control.DoctorResponse.signature:type_name -> control.Signature - 66, // 40: control.StartShardEvacuationRequest.body:type_name -> control.StartShardEvacuationRequest.Body - 93, // 41: control.StartShardEvacuationRequest.signature:type_name -> control.Signature - 67, // 42: control.StartShardEvacuationResponse.body:type_name -> control.StartShardEvacuationResponse.Body - 93, // 43: control.StartShardEvacuationResponse.signature:type_name -> control.Signature - 68, // 44: control.GetShardEvacuationStatusRequest.body:type_name -> control.GetShardEvacuationStatusRequest.Body - 93, // 45: control.GetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 69, // 46: control.GetShardEvacuationStatusResponse.body:type_name -> control.GetShardEvacuationStatusResponse.Body - 93, // 47: control.GetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 72, // 48: control.ResetShardEvacuationStatusRequest.body:type_name -> control.ResetShardEvacuationStatusRequest.Body - 93, // 49: control.ResetShardEvacuationStatusRequest.signature:type_name -> control.Signature - 73, // 50: control.ResetShardEvacuationStatusResponse.body:type_name -> control.ResetShardEvacuationStatusResponse.Body - 93, // 51: control.ResetShardEvacuationStatusResponse.signature:type_name -> control.Signature - 74, // 52: control.StopShardEvacuationRequest.body:type_name -> control.StopShardEvacuationRequest.Body - 93, // 53: control.StopShardEvacuationRequest.signature:type_name -> control.Signature - 75, // 54: control.StopShardEvacuationResponse.body:type_name -> control.StopShardEvacuationResponse.Body - 93, // 55: control.StopShardEvacuationResponse.signature:type_name -> control.Signature - 76, // 56: control.AddChainLocalOverrideRequest.body:type_name -> control.AddChainLocalOverrideRequest.Body - 93, // 57: control.AddChainLocalOverrideRequest.signature:type_name -> control.Signature - 77, // 58: control.AddChainLocalOverrideResponse.body:type_name -> control.AddChainLocalOverrideResponse.Body - 93, // 59: control.AddChainLocalOverrideResponse.signature:type_name -> control.Signature - 78, // 60: control.GetChainLocalOverrideRequest.body:type_name -> control.GetChainLocalOverrideRequest.Body - 93, // 61: control.GetChainLocalOverrideRequest.signature:type_name -> control.Signature - 79, // 62: control.GetChainLocalOverrideResponse.body:type_name -> control.GetChainLocalOverrideResponse.Body - 93, // 63: control.GetChainLocalOverrideResponse.signature:type_name -> control.Signature - 80, // 64: control.ListChainLocalOverridesRequest.body:type_name -> control.ListChainLocalOverridesRequest.Body - 93, // 65: control.ListChainLocalOverridesRequest.signature:type_name -> control.Signature - 81, // 66: control.ListChainLocalOverridesResponse.body:type_name -> control.ListChainLocalOverridesResponse.Body - 93, // 67: control.ListChainLocalOverridesResponse.signature:type_name -> control.Signature - 82, // 68: control.ListTargetsLocalOverridesRequest.body:type_name -> control.ListTargetsLocalOverridesRequest.Body - 93, // 69: control.ListTargetsLocalOverridesRequest.signature:type_name -> control.Signature - 83, // 70: control.ListTargetsLocalOverridesResponse.body:type_name -> control.ListTargetsLocalOverridesResponse.Body - 93, // 71: control.ListTargetsLocalOverridesResponse.signature:type_name -> control.Signature - 84, // 72: control.RemoveChainLocalOverrideRequest.body:type_name -> control.RemoveChainLocalOverrideRequest.Body - 93, // 73: control.RemoveChainLocalOverrideRequest.signature:type_name -> control.Signature - 85, // 74: control.RemoveChainLocalOverrideResponse.body:type_name -> control.RemoveChainLocalOverrideResponse.Body - 93, // 75: control.RemoveChainLocalOverrideResponse.signature:type_name -> control.Signature - 86, // 76: control.RemoveChainLocalOverridesByTargetRequest.body:type_name -> control.RemoveChainLocalOverridesByTargetRequest.Body - 93, // 77: control.RemoveChainLocalOverridesByTargetRequest.signature:type_name -> control.Signature - 87, // 78: control.RemoveChainLocalOverridesByTargetResponse.body:type_name -> control.RemoveChainLocalOverridesByTargetResponse.Body - 93, // 79: control.RemoveChainLocalOverridesByTargetResponse.signature:type_name -> control.Signature - 88, // 80: control.SealWriteCacheRequest.body:type_name -> control.SealWriteCacheRequest.Body - 93, // 81: control.SealWriteCacheRequest.signature:type_name -> control.Signature - 89, // 82: control.SealWriteCacheResponse.body:type_name -> control.SealWriteCacheResponse.Body - 93, // 83: control.SealWriteCacheResponse.signature:type_name -> control.Signature - 91, // 84: control.DetachShardsRequest.body:type_name -> control.DetachShardsRequest.Body - 93, // 85: control.DetachShardsRequest.signature:type_name -> control.Signature - 92, // 86: control.DetachShardsResponse.body:type_name -> control.DetachShardsResponse.Body - 93, // 87: control.DetachShardsResponse.signature:type_name -> control.Signature - 94, // 88: control.HealthCheckResponse.Body.netmap_status:type_name -> control.NetmapStatus - 95, // 89: control.HealthCheckResponse.Body.health_status:type_name -> control.HealthStatus - 94, // 90: control.SetNetmapStatusRequest.Body.status:type_name -> control.NetmapStatus - 94, // 91: control.GetNetmapStatusResponse.Body.status:type_name -> control.NetmapStatus - 96, // 92: control.ListShardsResponse.Body.shards:type_name -> control.ShardInfo - 97, // 93: control.SetShardModeRequest.Body.mode:type_name -> control.ShardMode - 1, // 94: control.GetShardEvacuationStatusResponse.Body.status:type_name -> control.GetShardEvacuationStatusResponse.Body.Status - 71, // 95: control.GetShardEvacuationStatusResponse.Body.duration:type_name -> control.GetShardEvacuationStatusResponse.Body.Duration - 70, // 96: control.GetShardEvacuationStatusResponse.Body.started_at:type_name -> control.GetShardEvacuationStatusResponse.Body.UnixTimestamp - 98, // 97: control.AddChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 98, // 98: control.GetChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 98, // 99: control.ListChainLocalOverridesRequest.Body.target:type_name -> control.ChainTarget - 98, // 100: control.ListTargetsLocalOverridesResponse.Body.targets:type_name -> control.ChainTarget - 98, // 101: control.RemoveChainLocalOverrideRequest.Body.target:type_name -> control.ChainTarget - 98, // 102: control.RemoveChainLocalOverridesByTargetRequest.Body.target:type_name -> control.ChainTarget - 90, // 103: control.SealWriteCacheResponse.Body.results:type_name -> control.SealWriteCacheResponse.Body.Status - 2, // 104: control.ControlService.HealthCheck:input_type -> control.HealthCheckRequest - 4, // 105: control.ControlService.SetNetmapStatus:input_type -> control.SetNetmapStatusRequest - 6, // 106: control.ControlService.GetNetmapStatus:input_type -> control.GetNetmapStatusRequest - 8, // 107: control.ControlService.DropObjects:input_type -> control.DropObjectsRequest - 10, // 108: control.ControlService.ListShards:input_type -> control.ListShardsRequest - 12, // 109: control.ControlService.SetShardMode:input_type -> control.SetShardModeRequest - 14, // 110: control.ControlService.SynchronizeTree:input_type -> control.SynchronizeTreeRequest - 16, // 111: control.ControlService.EvacuateShard:input_type -> control.EvacuateShardRequest - 22, // 112: control.ControlService.StartShardEvacuation:input_type -> control.StartShardEvacuationRequest - 24, // 113: control.ControlService.GetShardEvacuationStatus:input_type -> control.GetShardEvacuationStatusRequest - 26, // 114: control.ControlService.ResetShardEvacuationStatus:input_type -> control.ResetShardEvacuationStatusRequest - 28, // 115: control.ControlService.StopShardEvacuation:input_type -> control.StopShardEvacuationRequest - 18, // 116: control.ControlService.FlushCache:input_type -> control.FlushCacheRequest - 20, // 117: control.ControlService.Doctor:input_type -> control.DoctorRequest - 30, // 118: control.ControlService.AddChainLocalOverride:input_type -> control.AddChainLocalOverrideRequest - 32, // 119: control.ControlService.GetChainLocalOverride:input_type -> control.GetChainLocalOverrideRequest - 34, // 120: control.ControlService.ListChainLocalOverrides:input_type -> control.ListChainLocalOverridesRequest - 38, // 121: control.ControlService.RemoveChainLocalOverride:input_type -> control.RemoveChainLocalOverrideRequest - 40, // 122: control.ControlService.RemoveChainLocalOverridesByTarget:input_type -> control.RemoveChainLocalOverridesByTargetRequest - 36, // 123: control.ControlService.ListTargetsLocalOverrides:input_type -> control.ListTargetsLocalOverridesRequest - 42, // 124: control.ControlService.SealWriteCache:input_type -> control.SealWriteCacheRequest - 44, // 125: control.ControlService.DetachShards:input_type -> control.DetachShardsRequest - 3, // 126: control.ControlService.HealthCheck:output_type -> control.HealthCheckResponse - 5, // 127: control.ControlService.SetNetmapStatus:output_type -> control.SetNetmapStatusResponse - 7, // 128: control.ControlService.GetNetmapStatus:output_type -> control.GetNetmapStatusResponse - 9, // 129: control.ControlService.DropObjects:output_type -> control.DropObjectsResponse - 11, // 130: control.ControlService.ListShards:output_type -> control.ListShardsResponse - 13, // 131: control.ControlService.SetShardMode:output_type -> control.SetShardModeResponse - 15, // 132: control.ControlService.SynchronizeTree:output_type -> control.SynchronizeTreeResponse - 17, // 133: control.ControlService.EvacuateShard:output_type -> control.EvacuateShardResponse - 23, // 134: control.ControlService.StartShardEvacuation:output_type -> control.StartShardEvacuationResponse - 25, // 135: control.ControlService.GetShardEvacuationStatus:output_type -> control.GetShardEvacuationStatusResponse - 27, // 136: control.ControlService.ResetShardEvacuationStatus:output_type -> control.ResetShardEvacuationStatusResponse - 29, // 137: control.ControlService.StopShardEvacuation:output_type -> control.StopShardEvacuationResponse - 19, // 138: control.ControlService.FlushCache:output_type -> control.FlushCacheResponse - 21, // 139: control.ControlService.Doctor:output_type -> control.DoctorResponse - 31, // 140: control.ControlService.AddChainLocalOverride:output_type -> control.AddChainLocalOverrideResponse - 33, // 141: control.ControlService.GetChainLocalOverride:output_type -> control.GetChainLocalOverrideResponse - 35, // 142: control.ControlService.ListChainLocalOverrides:output_type -> control.ListChainLocalOverridesResponse - 39, // 143: control.ControlService.RemoveChainLocalOverride:output_type -> control.RemoveChainLocalOverrideResponse - 41, // 144: control.ControlService.RemoveChainLocalOverridesByTarget:output_type -> control.RemoveChainLocalOverridesByTargetResponse - 37, // 145: control.ControlService.ListTargetsLocalOverrides:output_type -> control.ListTargetsLocalOverridesResponse - 43, // 146: control.ControlService.SealWriteCache:output_type -> control.SealWriteCacheResponse - 45, // 147: control.ControlService.DetachShards:output_type -> control.DetachShardsResponse - 126, // [126:148] is the sub-list for method output_type - 104, // [104:126] is the sub-list for method input_type - 104, // [104:104] is the sub-list for extension type_name - 104, // [104:104] is the sub-list for extension extendee - 0, // [0:104] is the sub-list for field type_name -} - -func init() { file_pkg_services_control_service_proto_init() } -func file_pkg_services_control_service_proto_init() { - if File_pkg_services_control_service_proto != nil { - return - } - file_pkg_services_control_types_proto_init() - if !protoimpl.UnsafeEnabled { - file_pkg_services_control_service_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetNetmapStatusRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetNetmapStatusResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResetShardEvacuationStatusRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResetShardEvacuationStatusResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverridesByTargetRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverridesByTargetResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthCheckResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetNetmapStatusResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetNetmapStatusRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetNetmapStatusResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DropObjectsResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListShardsResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetShardModeResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SynchronizeTreeResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EvacuateShardResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FlushCacheResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DoctorResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartShardEvacuationResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_UnixTimestamp); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetShardEvacuationStatusResponse_Body_Duration); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResetShardEvacuationStatusRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResetShardEvacuationStatusResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopShardEvacuationResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddChainLocalOverrideResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[77].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChainLocalOverrideResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[78].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[79].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListChainLocalOverridesResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[80].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[81].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListTargetsLocalOverridesResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[82].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[83].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverrideResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[84].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverridesByTargetRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[85].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveChainLocalOverridesByTargetResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[86].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[87].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[88].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SealWriteCacheResponse_Body_Status); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[89].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_service_proto_msgTypes[90].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DetachShardsResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_pkg_services_control_service_proto_rawDesc, - NumEnums: 2, - NumMessages: 91, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_pkg_services_control_service_proto_goTypes, - DependencyIndexes: file_pkg_services_control_service_proto_depIdxs, - EnumInfos: file_pkg_services_control_service_proto_enumTypes, - MessageInfos: file_pkg_services_control_service_proto_msgTypes, - }.Build() - File_pkg_services_control_service_proto = out.File - file_pkg_services_control_service_proto_rawDesc = nil - file_pkg_services_control_service_proto_goTypes = nil - file_pkg_services_control_service_proto_depIdxs = nil -} diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 822244e77..afd1c3c41 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -2,7 +2,27 @@ package control -import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" +import ( + json "encoding/json" + fmt "fmt" + pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + easyproto "github.com/VictoriaMetrics/easyproto" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" + strconv "strconv" +) + +type HealthCheckRequest_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*HealthCheckRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*HealthCheckRequest_Body)(nil) + _ json.Marshaler = (*HealthCheckRequest_Body)(nil) + _ json.Unmarshaler = (*HealthCheckRequest_Body)(nil) +) // StableSize returns the size of x in protobuf format. // @@ -14,18 +34,93 @@ func (x *HealthCheckRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthCheckRequest_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthCheckRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *HealthCheckRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthCheckRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthCheckRequest_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthCheckRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthCheckRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthCheckRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthCheckRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type HealthCheckRequest struct { + Body *HealthCheckRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*HealthCheckRequest)(nil) + _ encoding.ProtoUnmarshaler = (*HealthCheckRequest)(nil) + _ json.Marshaler = (*HealthCheckRequest)(nil) + _ json.Unmarshaler = (*HealthCheckRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -38,27 +133,6 @@ func (x *HealthCheckRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthCheckRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -74,13 +148,165 @@ func (x *HealthCheckRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *HealthCheckRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *HealthCheckRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthCheckRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *HealthCheckRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthCheckRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthCheckRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(HealthCheckRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *HealthCheckRequest) GetBody() *HealthCheckRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *HealthCheckRequest) SetBody(v *HealthCheckRequest_Body) { + x.Body = v +} +func (x *HealthCheckRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *HealthCheckRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthCheckRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthCheckRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthCheckRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthCheckRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *HealthCheckRequest_Body + f = new(HealthCheckRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type HealthCheckResponse_Body struct { + NetmapStatus NetmapStatus `json:"netmapStatus"` + HealthStatus HealthStatus `json:"healthStatus"` +} + +var ( + _ encoding.ProtoMarshaler = (*HealthCheckResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*HealthCheckResponse_Body)(nil) + _ json.Marshaler = (*HealthCheckResponse_Body)(nil) + _ json.Unmarshaler = (*HealthCheckResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -93,27 +319,186 @@ func (x *HealthCheckResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthCheckResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.EnumMarshal(1, buf[offset:], int32(x.NetmapStatus)) - offset += proto.EnumMarshal(2, buf[offset:], int32(x.HealthStatus)) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthCheckResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *HealthCheckResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if int32(x.NetmapStatus) != 0 { + mm.AppendInt32(1, int32(x.NetmapStatus)) + } + if int32(x.HealthStatus) != 0 { + mm.AppendInt32(2, int32(x.HealthStatus)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthCheckResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthCheckResponse_Body") + } + switch fc.FieldNum { + case 1: // NetmapStatus + data, ok := fc.Int32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "NetmapStatus") + } + x.NetmapStatus = NetmapStatus(data) + case 2: // HealthStatus + data, ok := fc.Int32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "HealthStatus") + } + x.HealthStatus = HealthStatus(data) + } + } + return nil +} +func (x *HealthCheckResponse_Body) GetNetmapStatus() NetmapStatus { + if x != nil { + return x.NetmapStatus + } + return 0 +} +func (x *HealthCheckResponse_Body) SetNetmapStatus(v NetmapStatus) { + x.NetmapStatus = v +} +func (x *HealthCheckResponse_Body) GetHealthStatus() HealthStatus { + if x != nil { + return x.HealthStatus + } + return 0 +} +func (x *HealthCheckResponse_Body) SetHealthStatus(v HealthStatus) { + x.HealthStatus = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthCheckResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthCheckResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"netmapStatus\":" + out.RawString(prefix[1:]) + out.Int32(int32(x.NetmapStatus)) + } + { + const prefix string = ",\"healthStatus\":" + out.RawString(prefix) + out.Int32(int32(x.HealthStatus)) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthCheckResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthCheckResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "netmapStatus": + { + var f NetmapStatus + var parsedValue NetmapStatus + switch v := in.Interface().(type) { + case string: + if vv, ok := NetmapStatus_value[v]; ok { + parsedValue = NetmapStatus(vv) + break + } + vv, err := strconv.ParseInt(v, 10, 32) + if err != nil { + in.AddError(err) + return + } + parsedValue = NetmapStatus(vv) + case float64: + parsedValue = NetmapStatus(v) + } + f = parsedValue + x.NetmapStatus = f + } + case "healthStatus": + { + var f HealthStatus + var parsedValue HealthStatus + switch v := in.Interface().(type) { + case string: + if vv, ok := HealthStatus_value[v]; ok { + parsedValue = HealthStatus(vv) + break + } + vv, err := strconv.ParseInt(v, 10, 32) + if err != nil { + in.AddError(err) + return + } + parsedValue = HealthStatus(vv) + case float64: + parsedValue = HealthStatus(v) + } + f = parsedValue + x.HealthStatus = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type HealthCheckResponse struct { + Body *HealthCheckResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*HealthCheckResponse)(nil) + _ encoding.ProtoUnmarshaler = (*HealthCheckResponse)(nil) + _ json.Marshaler = (*HealthCheckResponse)(nil) + _ json.Unmarshaler = (*HealthCheckResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -126,27 +511,6 @@ func (x *HealthCheckResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthCheckResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -162,13 +526,165 @@ func (x *HealthCheckResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *HealthCheckResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *HealthCheckResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthCheckResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *HealthCheckResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthCheckResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthCheckResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(HealthCheckResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *HealthCheckResponse) GetBody() *HealthCheckResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *HealthCheckResponse) SetBody(v *HealthCheckResponse_Body) { + x.Body = v +} +func (x *HealthCheckResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *HealthCheckResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthCheckResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthCheckResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthCheckResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthCheckResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *HealthCheckResponse_Body + f = new(HealthCheckResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SetNetmapStatusRequest_Body struct { + Status NetmapStatus `json:"status"` + ForceMaintenance bool `json:"forceMaintenance"` +} + +var ( + _ encoding.ProtoMarshaler = (*SetNetmapStatusRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*SetNetmapStatusRequest_Body)(nil) + _ json.Marshaler = (*SetNetmapStatusRequest_Body)(nil) + _ json.Unmarshaler = (*SetNetmapStatusRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -181,27 +697,170 @@ func (x *SetNetmapStatusRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SetNetmapStatusRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.EnumMarshal(1, buf[offset:], int32(x.Status)) - offset += proto.BoolMarshal(2, buf[offset:], x.ForceMaintenance) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SetNetmapStatusRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SetNetmapStatusRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if int32(x.Status) != 0 { + mm.AppendInt32(1, int32(x.Status)) + } + if x.ForceMaintenance { + mm.AppendBool(2, x.ForceMaintenance) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SetNetmapStatusRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SetNetmapStatusRequest_Body") + } + switch fc.FieldNum { + case 1: // Status + data, ok := fc.Int32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Status") + } + x.Status = NetmapStatus(data) + case 2: // ForceMaintenance + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ForceMaintenance") + } + x.ForceMaintenance = data + } + } + return nil +} +func (x *SetNetmapStatusRequest_Body) GetStatus() NetmapStatus { + if x != nil { + return x.Status + } + return 0 +} +func (x *SetNetmapStatusRequest_Body) SetStatus(v NetmapStatus) { + x.Status = v +} +func (x *SetNetmapStatusRequest_Body) GetForceMaintenance() bool { + if x != nil { + return x.ForceMaintenance + } + return false +} +func (x *SetNetmapStatusRequest_Body) SetForceMaintenance(v bool) { + x.ForceMaintenance = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SetNetmapStatusRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SetNetmapStatusRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"status\":" + out.RawString(prefix[1:]) + out.Int32(int32(x.Status)) + } + { + const prefix string = ",\"forceMaintenance\":" + out.RawString(prefix) + out.Bool(x.ForceMaintenance) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SetNetmapStatusRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SetNetmapStatusRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "status": + { + var f NetmapStatus + var parsedValue NetmapStatus + switch v := in.Interface().(type) { + case string: + if vv, ok := NetmapStatus_value[v]; ok { + parsedValue = NetmapStatus(vv) + break + } + vv, err := strconv.ParseInt(v, 10, 32) + if err != nil { + in.AddError(err) + return + } + parsedValue = NetmapStatus(vv) + case float64: + parsedValue = NetmapStatus(v) + } + f = parsedValue + x.Status = f + } + case "forceMaintenance": + { + var f bool + f = in.Bool() + x.ForceMaintenance = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SetNetmapStatusRequest struct { + Body *SetNetmapStatusRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*SetNetmapStatusRequest)(nil) + _ encoding.ProtoUnmarshaler = (*SetNetmapStatusRequest)(nil) + _ json.Marshaler = (*SetNetmapStatusRequest)(nil) + _ json.Unmarshaler = (*SetNetmapStatusRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -214,27 +873,6 @@ func (x *SetNetmapStatusRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SetNetmapStatusRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -250,13 +888,163 @@ func (x *SetNetmapStatusRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *SetNetmapStatusRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *SetNetmapStatusRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SetNetmapStatusRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SetNetmapStatusRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SetNetmapStatusRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SetNetmapStatusRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(SetNetmapStatusRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *SetNetmapStatusRequest) GetBody() *SetNetmapStatusRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *SetNetmapStatusRequest) SetBody(v *SetNetmapStatusRequest_Body) { + x.Body = v +} +func (x *SetNetmapStatusRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *SetNetmapStatusRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SetNetmapStatusRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SetNetmapStatusRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SetNetmapStatusRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SetNetmapStatusRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *SetNetmapStatusRequest_Body + f = new(SetNetmapStatusRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SetNetmapStatusResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*SetNetmapStatusResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*SetNetmapStatusResponse_Body)(nil) + _ json.Marshaler = (*SetNetmapStatusResponse_Body)(nil) + _ json.Unmarshaler = (*SetNetmapStatusResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -267,18 +1055,93 @@ func (x *SetNetmapStatusResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SetNetmapStatusResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SetNetmapStatusResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SetNetmapStatusResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SetNetmapStatusResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SetNetmapStatusResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SetNetmapStatusResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SetNetmapStatusResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SetNetmapStatusResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SetNetmapStatusResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SetNetmapStatusResponse struct { + Body *SetNetmapStatusResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*SetNetmapStatusResponse)(nil) + _ encoding.ProtoUnmarshaler = (*SetNetmapStatusResponse)(nil) + _ json.Marshaler = (*SetNetmapStatusResponse)(nil) + _ json.Unmarshaler = (*SetNetmapStatusResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -291,27 +1154,6 @@ func (x *SetNetmapStatusResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SetNetmapStatusResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -327,13 +1169,163 @@ func (x *SetNetmapStatusResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *SetNetmapStatusResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *SetNetmapStatusResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SetNetmapStatusResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SetNetmapStatusResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SetNetmapStatusResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SetNetmapStatusResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(SetNetmapStatusResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *SetNetmapStatusResponse) GetBody() *SetNetmapStatusResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *SetNetmapStatusResponse) SetBody(v *SetNetmapStatusResponse_Body) { + x.Body = v +} +func (x *SetNetmapStatusResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *SetNetmapStatusResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SetNetmapStatusResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SetNetmapStatusResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SetNetmapStatusResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SetNetmapStatusResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *SetNetmapStatusResponse_Body + f = new(SetNetmapStatusResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetNetmapStatusRequest_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*GetNetmapStatusRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetNetmapStatusRequest_Body)(nil) + _ json.Marshaler = (*GetNetmapStatusRequest_Body)(nil) + _ json.Unmarshaler = (*GetNetmapStatusRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -344,18 +1336,93 @@ func (x *GetNetmapStatusRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetNetmapStatusRequest_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetNetmapStatusRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetNetmapStatusRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetNetmapStatusRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetNetmapStatusRequest_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetNetmapStatusRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetNetmapStatusRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetNetmapStatusRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetNetmapStatusRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetNetmapStatusRequest struct { + Body *GetNetmapStatusRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetNetmapStatusRequest)(nil) + _ encoding.ProtoUnmarshaler = (*GetNetmapStatusRequest)(nil) + _ json.Marshaler = (*GetNetmapStatusRequest)(nil) + _ json.Unmarshaler = (*GetNetmapStatusRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -368,27 +1435,6 @@ func (x *GetNetmapStatusRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetNetmapStatusRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -404,13 +1450,165 @@ func (x *GetNetmapStatusRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetNetmapStatusRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetNetmapStatusRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetNetmapStatusRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetNetmapStatusRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetNetmapStatusRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetNetmapStatusRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetNetmapStatusRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetNetmapStatusRequest) GetBody() *GetNetmapStatusRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetNetmapStatusRequest) SetBody(v *GetNetmapStatusRequest_Body) { + x.Body = v +} +func (x *GetNetmapStatusRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetNetmapStatusRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetNetmapStatusRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetNetmapStatusRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetNetmapStatusRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetNetmapStatusRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetNetmapStatusRequest_Body + f = new(GetNetmapStatusRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetNetmapStatusResponse_Body struct { + Status NetmapStatus `json:"status"` + Epoch uint64 `json:"epoch"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetNetmapStatusResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetNetmapStatusResponse_Body)(nil) + _ json.Marshaler = (*GetNetmapStatusResponse_Body)(nil) + _ json.Unmarshaler = (*GetNetmapStatusResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -423,27 +1621,170 @@ func (x *GetNetmapStatusResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetNetmapStatusResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.EnumMarshal(1, buf[offset:], int32(x.Status)) - offset += proto.UInt64Marshal(2, buf[offset:], x.Epoch) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetNetmapStatusResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetNetmapStatusResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if int32(x.Status) != 0 { + mm.AppendInt32(1, int32(x.Status)) + } + if x.Epoch != 0 { + mm.AppendUint64(2, x.Epoch) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetNetmapStatusResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetNetmapStatusResponse_Body") + } + switch fc.FieldNum { + case 1: // Status + data, ok := fc.Int32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Status") + } + x.Status = NetmapStatus(data) + case 2: // Epoch + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Epoch") + } + x.Epoch = data + } + } + return nil +} +func (x *GetNetmapStatusResponse_Body) GetStatus() NetmapStatus { + if x != nil { + return x.Status + } + return 0 +} +func (x *GetNetmapStatusResponse_Body) SetStatus(v NetmapStatus) { + x.Status = v +} +func (x *GetNetmapStatusResponse_Body) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} +func (x *GetNetmapStatusResponse_Body) SetEpoch(v uint64) { + x.Epoch = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetNetmapStatusResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetNetmapStatusResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"status\":" + out.RawString(prefix[1:]) + out.Int32(int32(x.Status)) + } + { + const prefix string = ",\"epoch\":" + out.RawString(prefix) + out.Uint64(x.Epoch) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetNetmapStatusResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetNetmapStatusResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "status": + { + var f NetmapStatus + var parsedValue NetmapStatus + switch v := in.Interface().(type) { + case string: + if vv, ok := NetmapStatus_value[v]; ok { + parsedValue = NetmapStatus(vv) + break + } + vv, err := strconv.ParseInt(v, 10, 32) + if err != nil { + in.AddError(err) + return + } + parsedValue = NetmapStatus(vv) + case float64: + parsedValue = NetmapStatus(v) + } + f = parsedValue + x.Status = f + } + case "epoch": + { + var f uint64 + f = in.Uint64() + x.Epoch = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetNetmapStatusResponse struct { + Body *GetNetmapStatusResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetNetmapStatusResponse)(nil) + _ encoding.ProtoUnmarshaler = (*GetNetmapStatusResponse)(nil) + _ json.Marshaler = (*GetNetmapStatusResponse)(nil) + _ json.Unmarshaler = (*GetNetmapStatusResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -456,27 +1797,6 @@ func (x *GetNetmapStatusResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetNetmapStatusResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -492,13 +1812,164 @@ func (x *GetNetmapStatusResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetNetmapStatusResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetNetmapStatusResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetNetmapStatusResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetNetmapStatusResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetNetmapStatusResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetNetmapStatusResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetNetmapStatusResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetNetmapStatusResponse) GetBody() *GetNetmapStatusResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetNetmapStatusResponse) SetBody(v *GetNetmapStatusResponse_Body) { + x.Body = v +} +func (x *GetNetmapStatusResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetNetmapStatusResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetNetmapStatusResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetNetmapStatusResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetNetmapStatusResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetNetmapStatusResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetNetmapStatusResponse_Body + f = new(GetNetmapStatusResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DropObjectsRequest_Body struct { + AddressList [][]byte `json:"addressList"` +} + +var ( + _ encoding.ProtoMarshaler = (*DropObjectsRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*DropObjectsRequest_Body)(nil) + _ json.Marshaler = (*DropObjectsRequest_Body)(nil) + _ json.Unmarshaler = (*DropObjectsRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -510,26 +1981,139 @@ func (x *DropObjectsRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DropObjectsRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.AddressList) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DropObjectsRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *DropObjectsRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for j := range x.AddressList { + mm.AppendBytes(1, x.AddressList[j]) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DropObjectsRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DropObjectsRequest_Body") + } + switch fc.FieldNum { + case 1: // AddressList + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "AddressList") + } + x.AddressList = append(x.AddressList, data) + } + } + return nil +} +func (x *DropObjectsRequest_Body) GetAddressList() [][]byte { + if x != nil { + return x.AddressList + } + return nil +} +func (x *DropObjectsRequest_Body) SetAddressList(v [][]byte) { + x.AddressList = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DropObjectsRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DropObjectsRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"addressList\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.AddressList { + if i != 0 { + out.RawByte(',') + } + out.Base64Bytes(x.AddressList[i]) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DropObjectsRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DropObjectsRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "addressList": + { + var f []byte + var list [][]byte + in.Delim('[') + for !in.IsDelim(']') { + f = in.Bytes() + list = append(list, f) + in.WantComma() + } + x.AddressList = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DropObjectsRequest struct { + Body *DropObjectsRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*DropObjectsRequest)(nil) + _ encoding.ProtoUnmarshaler = (*DropObjectsRequest)(nil) + _ json.Marshaler = (*DropObjectsRequest)(nil) + _ json.Unmarshaler = (*DropObjectsRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -542,27 +2126,6 @@ func (x *DropObjectsRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DropObjectsRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -578,13 +2141,163 @@ func (x *DropObjectsRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *DropObjectsRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *DropObjectsRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DropObjectsRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *DropObjectsRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DropObjectsRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DropObjectsRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(DropObjectsRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *DropObjectsRequest) GetBody() *DropObjectsRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *DropObjectsRequest) SetBody(v *DropObjectsRequest_Body) { + x.Body = v +} +func (x *DropObjectsRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *DropObjectsRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DropObjectsRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DropObjectsRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DropObjectsRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DropObjectsRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *DropObjectsRequest_Body + f = new(DropObjectsRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DropObjectsResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*DropObjectsResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*DropObjectsResponse_Body)(nil) + _ json.Marshaler = (*DropObjectsResponse_Body)(nil) + _ json.Unmarshaler = (*DropObjectsResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -595,18 +2308,93 @@ func (x *DropObjectsResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DropObjectsResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DropObjectsResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *DropObjectsResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DropObjectsResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DropObjectsResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DropObjectsResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DropObjectsResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DropObjectsResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DropObjectsResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DropObjectsResponse struct { + Body *DropObjectsResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*DropObjectsResponse)(nil) + _ encoding.ProtoUnmarshaler = (*DropObjectsResponse)(nil) + _ json.Marshaler = (*DropObjectsResponse)(nil) + _ json.Unmarshaler = (*DropObjectsResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -619,27 +2407,6 @@ func (x *DropObjectsResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DropObjectsResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -655,13 +2422,163 @@ func (x *DropObjectsResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *DropObjectsResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *DropObjectsResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DropObjectsResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *DropObjectsResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DropObjectsResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DropObjectsResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(DropObjectsResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *DropObjectsResponse) GetBody() *DropObjectsResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *DropObjectsResponse) SetBody(v *DropObjectsResponse_Body) { + x.Body = v +} +func (x *DropObjectsResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *DropObjectsResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DropObjectsResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DropObjectsResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DropObjectsResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DropObjectsResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *DropObjectsResponse_Body + f = new(DropObjectsResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListShardsRequest_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*ListShardsRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ListShardsRequest_Body)(nil) + _ json.Marshaler = (*ListShardsRequest_Body)(nil) + _ json.Unmarshaler = (*ListShardsRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -672,18 +2589,93 @@ func (x *ListShardsRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListShardsRequest_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListShardsRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListShardsRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListShardsRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListShardsRequest_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListShardsRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListShardsRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListShardsRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListShardsRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListShardsRequest struct { + Body *ListShardsRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListShardsRequest)(nil) + _ encoding.ProtoUnmarshaler = (*ListShardsRequest)(nil) + _ json.Marshaler = (*ListShardsRequest)(nil) + _ json.Unmarshaler = (*ListShardsRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -696,27 +2688,6 @@ func (x *ListShardsRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListShardsRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -732,13 +2703,164 @@ func (x *ListShardsRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *ListShardsRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *ListShardsRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListShardsRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListShardsRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListShardsRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListShardsRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ListShardsRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ListShardsRequest) GetBody() *ListShardsRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ListShardsRequest) SetBody(v *ListShardsRequest_Body) { + x.Body = v +} +func (x *ListShardsRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ListShardsRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListShardsRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListShardsRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListShardsRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListShardsRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ListShardsRequest_Body + f = new(ListShardsRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListShardsResponse_Body struct { + Shards []*ShardInfo `json:"shards"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListShardsResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ListShardsResponse_Body)(nil) + _ json.Marshaler = (*ListShardsResponse_Body)(nil) + _ json.Unmarshaler = (*ListShardsResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -752,28 +2874,146 @@ func (x *ListShardsResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListShardsResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - for i := range x.Shards { - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Shards[i]) - } - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListShardsResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListShardsResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for i := range x.Shards { + if x.Shards[i] != nil && x.Shards[i].StableSize() != 0 { + x.Shards[i].EmitProtobuf(mm.AppendMessage(1)) + } + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListShardsResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListShardsResponse_Body") + } + switch fc.FieldNum { + case 1: // Shards + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shards") + } + x.Shards = append(x.Shards, new(ShardInfo)) + ff := x.Shards[len(x.Shards)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ListShardsResponse_Body) GetShards() []*ShardInfo { + if x != nil { + return x.Shards + } + return nil +} +func (x *ListShardsResponse_Body) SetShards(v []*ShardInfo) { + x.Shards = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListShardsResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListShardsResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"shards\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Shards { + if i != 0 { + out.RawByte(',') + } + x.Shards[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListShardsResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListShardsResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shards": + { + var f *ShardInfo + var list []*ShardInfo + in.Delim('[') + for !in.IsDelim(']') { + f = new(ShardInfo) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Shards = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListShardsResponse struct { + Body *ListShardsResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListShardsResponse)(nil) + _ encoding.ProtoUnmarshaler = (*ListShardsResponse)(nil) + _ json.Marshaler = (*ListShardsResponse)(nil) + _ json.Unmarshaler = (*ListShardsResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -786,27 +3026,6 @@ func (x *ListShardsResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListShardsResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -822,13 +3041,166 @@ func (x *ListShardsResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *ListShardsResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *ListShardsResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListShardsResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListShardsResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListShardsResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListShardsResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ListShardsResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ListShardsResponse) GetBody() *ListShardsResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ListShardsResponse) SetBody(v *ListShardsResponse_Body) { + x.Body = v +} +func (x *ListShardsResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ListShardsResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListShardsResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListShardsResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListShardsResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListShardsResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ListShardsResponse_Body + f = new(ListShardsResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SetShardModeRequest_Body struct { + Shard_ID [][]byte `json:"shardID"` + Mode ShardMode `json:"mode"` + ResetErrorCounter bool `json:"resetErrorCounter"` +} + +var ( + _ encoding.ProtoMarshaler = (*SetShardModeRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*SetShardModeRequest_Body)(nil) + _ json.Marshaler = (*SetShardModeRequest_Body)(nil) + _ json.Unmarshaler = (*SetShardModeRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -842,28 +3214,213 @@ func (x *SetShardModeRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SetShardModeRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) - offset += proto.EnumMarshal(2, buf[offset:], int32(x.Mode)) - offset += proto.BoolMarshal(3, buf[offset:], x.ResetErrorCounter) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SetShardModeRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SetShardModeRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for j := range x.Shard_ID { + mm.AppendBytes(1, x.Shard_ID[j]) + } + if int32(x.Mode) != 0 { + mm.AppendInt32(2, int32(x.Mode)) + } + if x.ResetErrorCounter { + mm.AppendBool(3, x.ResetErrorCounter) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SetShardModeRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SetShardModeRequest_Body") + } + switch fc.FieldNum { + case 1: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = append(x.Shard_ID, data) + case 2: // Mode + data, ok := fc.Int32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Mode") + } + x.Mode = ShardMode(data) + case 3: // ResetErrorCounter + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ResetErrorCounter") + } + x.ResetErrorCounter = data + } + } + return nil +} +func (x *SetShardModeRequest_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *SetShardModeRequest_Body) SetShard_ID(v [][]byte) { + x.Shard_ID = v +} +func (x *SetShardModeRequest_Body) GetMode() ShardMode { + if x != nil { + return x.Mode + } + return 0 +} +func (x *SetShardModeRequest_Body) SetMode(v ShardMode) { + x.Mode = v +} +func (x *SetShardModeRequest_Body) GetResetErrorCounter() bool { + if x != nil { + return x.ResetErrorCounter + } + return false +} +func (x *SetShardModeRequest_Body) SetResetErrorCounter(v bool) { + x.ResetErrorCounter = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SetShardModeRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SetShardModeRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"shardID\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Shard_ID { + if i != 0 { + out.RawByte(',') + } + out.Base64Bytes(x.Shard_ID[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"mode\":" + out.RawString(prefix) + out.Int32(int32(x.Mode)) + } + { + const prefix string = ",\"resetErrorCounter\":" + out.RawString(prefix) + out.Bool(x.ResetErrorCounter) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SetShardModeRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SetShardModeRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shardID": + { + var f []byte + var list [][]byte + in.Delim('[') + for !in.IsDelim(']') { + f = in.Bytes() + list = append(list, f) + in.WantComma() + } + x.Shard_ID = list + in.Delim(']') + } + case "mode": + { + var f ShardMode + var parsedValue ShardMode + switch v := in.Interface().(type) { + case string: + if vv, ok := ShardMode_value[v]; ok { + parsedValue = ShardMode(vv) + break + } + vv, err := strconv.ParseInt(v, 10, 32) + if err != nil { + in.AddError(err) + return + } + parsedValue = ShardMode(vv) + case float64: + parsedValue = ShardMode(v) + } + f = parsedValue + x.Mode = f + } + case "resetErrorCounter": + { + var f bool + f = in.Bool() + x.ResetErrorCounter = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SetShardModeRequest struct { + Body *SetShardModeRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*SetShardModeRequest)(nil) + _ encoding.ProtoUnmarshaler = (*SetShardModeRequest)(nil) + _ json.Marshaler = (*SetShardModeRequest)(nil) + _ json.Unmarshaler = (*SetShardModeRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -876,27 +3433,6 @@ func (x *SetShardModeRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SetShardModeRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -912,13 +3448,163 @@ func (x *SetShardModeRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *SetShardModeRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *SetShardModeRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SetShardModeRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SetShardModeRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SetShardModeRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SetShardModeRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(SetShardModeRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *SetShardModeRequest) GetBody() *SetShardModeRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *SetShardModeRequest) SetBody(v *SetShardModeRequest_Body) { + x.Body = v +} +func (x *SetShardModeRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *SetShardModeRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SetShardModeRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SetShardModeRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SetShardModeRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SetShardModeRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *SetShardModeRequest_Body + f = new(SetShardModeRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SetShardModeResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*SetShardModeResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*SetShardModeResponse_Body)(nil) + _ json.Marshaler = (*SetShardModeResponse_Body)(nil) + _ json.Unmarshaler = (*SetShardModeResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -929,18 +3615,93 @@ func (x *SetShardModeResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SetShardModeResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SetShardModeResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SetShardModeResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SetShardModeResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SetShardModeResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SetShardModeResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SetShardModeResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SetShardModeResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SetShardModeResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SetShardModeResponse struct { + Body *SetShardModeResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*SetShardModeResponse)(nil) + _ encoding.ProtoUnmarshaler = (*SetShardModeResponse)(nil) + _ json.Marshaler = (*SetShardModeResponse)(nil) + _ json.Unmarshaler = (*SetShardModeResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -953,27 +3714,6 @@ func (x *SetShardModeResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SetShardModeResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -989,13 +3729,166 @@ func (x *SetShardModeResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *SetShardModeResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *SetShardModeResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SetShardModeResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SetShardModeResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SetShardModeResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SetShardModeResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(SetShardModeResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *SetShardModeResponse) GetBody() *SetShardModeResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *SetShardModeResponse) SetBody(v *SetShardModeResponse_Body) { + x.Body = v +} +func (x *SetShardModeResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *SetShardModeResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SetShardModeResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SetShardModeResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SetShardModeResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SetShardModeResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *SetShardModeResponse_Body + f = new(SetShardModeResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SynchronizeTreeRequest_Body struct { + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + Height uint64 `json:"height"` +} + +var ( + _ encoding.ProtoMarshaler = (*SynchronizeTreeRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*SynchronizeTreeRequest_Body)(nil) + _ json.Marshaler = (*SynchronizeTreeRequest_Body)(nil) + _ json.Unmarshaler = (*SynchronizeTreeRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1009,28 +3902,183 @@ func (x *SynchronizeTreeRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SynchronizeTreeRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) - offset += proto.StringMarshal(2, buf[offset:], x.TreeId) - offset += proto.UInt64Marshal(3, buf[offset:], x.Height) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SynchronizeTreeRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SynchronizeTreeRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ContainerId) != 0 { + mm.AppendBytes(1, x.ContainerId) + } + if len(x.TreeId) != 0 { + mm.AppendString(2, x.TreeId) + } + if x.Height != 0 { + mm.AppendUint64(3, x.Height) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SynchronizeTreeRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SynchronizeTreeRequest_Body") + } + switch fc.FieldNum { + case 1: // ContainerId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + case 2: // TreeId + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TreeId") + } + x.TreeId = data + case 3: // Height + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Height") + } + x.Height = data + } + } + return nil +} +func (x *SynchronizeTreeRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} +func (x *SynchronizeTreeRequest_Body) SetContainerId(v []byte) { + x.ContainerId = v +} +func (x *SynchronizeTreeRequest_Body) GetTreeId() string { + if x != nil { + return x.TreeId + } + return "" +} +func (x *SynchronizeTreeRequest_Body) SetTreeId(v string) { + x.TreeId = v +} +func (x *SynchronizeTreeRequest_Body) GetHeight() uint64 { + if x != nil { + return x.Height + } + return 0 +} +func (x *SynchronizeTreeRequest_Body) SetHeight(v uint64) { + x.Height = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SynchronizeTreeRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SynchronizeTreeRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"containerId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ContainerId) + } + { + const prefix string = ",\"treeId\":" + out.RawString(prefix) + out.String(x.TreeId) + } + { + const prefix string = ",\"height\":" + out.RawString(prefix) + out.Uint64(x.Height) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SynchronizeTreeRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SynchronizeTreeRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "containerId": + { + var f []byte + f = in.Bytes() + x.ContainerId = f + } + case "treeId": + { + var f string + f = in.String() + x.TreeId = f + } + case "height": + { + var f uint64 + f = in.Uint64() + x.Height = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SynchronizeTreeRequest struct { + Body *SynchronizeTreeRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*SynchronizeTreeRequest)(nil) + _ encoding.ProtoUnmarshaler = (*SynchronizeTreeRequest)(nil) + _ json.Marshaler = (*SynchronizeTreeRequest)(nil) + _ json.Unmarshaler = (*SynchronizeTreeRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1043,27 +4091,6 @@ func (x *SynchronizeTreeRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SynchronizeTreeRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1079,13 +4106,163 @@ func (x *SynchronizeTreeRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *SynchronizeTreeRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *SynchronizeTreeRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SynchronizeTreeRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SynchronizeTreeRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SynchronizeTreeRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SynchronizeTreeRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(SynchronizeTreeRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *SynchronizeTreeRequest) GetBody() *SynchronizeTreeRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *SynchronizeTreeRequest) SetBody(v *SynchronizeTreeRequest_Body) { + x.Body = v +} +func (x *SynchronizeTreeRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *SynchronizeTreeRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SynchronizeTreeRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SynchronizeTreeRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SynchronizeTreeRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SynchronizeTreeRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *SynchronizeTreeRequest_Body + f = new(SynchronizeTreeRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SynchronizeTreeResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*SynchronizeTreeResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*SynchronizeTreeResponse_Body)(nil) + _ json.Marshaler = (*SynchronizeTreeResponse_Body)(nil) + _ json.Unmarshaler = (*SynchronizeTreeResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1096,18 +4273,93 @@ func (x *SynchronizeTreeResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SynchronizeTreeResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SynchronizeTreeResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SynchronizeTreeResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SynchronizeTreeResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SynchronizeTreeResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SynchronizeTreeResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SynchronizeTreeResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SynchronizeTreeResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SynchronizeTreeResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SynchronizeTreeResponse struct { + Body *SynchronizeTreeResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*SynchronizeTreeResponse)(nil) + _ encoding.ProtoUnmarshaler = (*SynchronizeTreeResponse)(nil) + _ json.Marshaler = (*SynchronizeTreeResponse)(nil) + _ json.Unmarshaler = (*SynchronizeTreeResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1120,27 +4372,6 @@ func (x *SynchronizeTreeResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SynchronizeTreeResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1156,13 +4387,165 @@ func (x *SynchronizeTreeResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *SynchronizeTreeResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *SynchronizeTreeResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SynchronizeTreeResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SynchronizeTreeResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SynchronizeTreeResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SynchronizeTreeResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(SynchronizeTreeResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *SynchronizeTreeResponse) GetBody() *SynchronizeTreeResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *SynchronizeTreeResponse) SetBody(v *SynchronizeTreeResponse_Body) { + x.Body = v +} +func (x *SynchronizeTreeResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *SynchronizeTreeResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SynchronizeTreeResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SynchronizeTreeResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SynchronizeTreeResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SynchronizeTreeResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *SynchronizeTreeResponse_Body + f = new(SynchronizeTreeResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type EvacuateShardRequest_Body struct { + Shard_ID [][]byte `json:"shardID"` + IgnoreErrors bool `json:"ignoreErrors"` +} + +var ( + _ encoding.ProtoMarshaler = (*EvacuateShardRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*EvacuateShardRequest_Body)(nil) + _ json.Marshaler = (*EvacuateShardRequest_Body)(nil) + _ json.Unmarshaler = (*EvacuateShardRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1175,27 +4558,168 @@ func (x *EvacuateShardRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *EvacuateShardRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) - offset += proto.BoolMarshal(2, buf[offset:], x.IgnoreErrors) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *EvacuateShardRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *EvacuateShardRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for j := range x.Shard_ID { + mm.AppendBytes(1, x.Shard_ID[j]) + } + if x.IgnoreErrors { + mm.AppendBool(2, x.IgnoreErrors) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *EvacuateShardRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "EvacuateShardRequest_Body") + } + switch fc.FieldNum { + case 1: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = append(x.Shard_ID, data) + case 2: // IgnoreErrors + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "IgnoreErrors") + } + x.IgnoreErrors = data + } + } + return nil +} +func (x *EvacuateShardRequest_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *EvacuateShardRequest_Body) SetShard_ID(v [][]byte) { + x.Shard_ID = v +} +func (x *EvacuateShardRequest_Body) GetIgnoreErrors() bool { + if x != nil { + return x.IgnoreErrors + } + return false +} +func (x *EvacuateShardRequest_Body) SetIgnoreErrors(v bool) { + x.IgnoreErrors = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *EvacuateShardRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *EvacuateShardRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"shardID\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Shard_ID { + if i != 0 { + out.RawByte(',') + } + out.Base64Bytes(x.Shard_ID[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"ignoreErrors\":" + out.RawString(prefix) + out.Bool(x.IgnoreErrors) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *EvacuateShardRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *EvacuateShardRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shardID": + { + var f []byte + var list [][]byte + in.Delim('[') + for !in.IsDelim(']') { + f = in.Bytes() + list = append(list, f) + in.WantComma() + } + x.Shard_ID = list + in.Delim(']') + } + case "ignoreErrors": + { + var f bool + f = in.Bool() + x.IgnoreErrors = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type EvacuateShardRequest struct { + Body *EvacuateShardRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*EvacuateShardRequest)(nil) + _ encoding.ProtoUnmarshaler = (*EvacuateShardRequest)(nil) + _ json.Marshaler = (*EvacuateShardRequest)(nil) + _ json.Unmarshaler = (*EvacuateShardRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1208,27 +4732,6 @@ func (x *EvacuateShardRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *EvacuateShardRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1244,13 +4747,164 @@ func (x *EvacuateShardRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *EvacuateShardRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *EvacuateShardRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *EvacuateShardRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *EvacuateShardRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *EvacuateShardRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "EvacuateShardRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(EvacuateShardRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *EvacuateShardRequest) GetBody() *EvacuateShardRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *EvacuateShardRequest) SetBody(v *EvacuateShardRequest_Body) { + x.Body = v +} +func (x *EvacuateShardRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *EvacuateShardRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *EvacuateShardRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *EvacuateShardRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *EvacuateShardRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *EvacuateShardRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *EvacuateShardRequest_Body + f = new(EvacuateShardRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type EvacuateShardResponse_Body struct { + Count uint32 `json:"count"` +} + +var ( + _ encoding.ProtoMarshaler = (*EvacuateShardResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*EvacuateShardResponse_Body)(nil) + _ json.Marshaler = (*EvacuateShardResponse_Body)(nil) + _ json.Unmarshaler = (*EvacuateShardResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1262,26 +4916,125 @@ func (x *EvacuateShardResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *EvacuateShardResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.UInt32Marshal(1, buf[offset:], x.Count) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *EvacuateShardResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *EvacuateShardResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Count != 0 { + mm.AppendUint32(1, x.Count) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *EvacuateShardResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "EvacuateShardResponse_Body") + } + switch fc.FieldNum { + case 1: // Count + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Count") + } + x.Count = data + } + } + return nil +} +func (x *EvacuateShardResponse_Body) GetCount() uint32 { + if x != nil { + return x.Count + } + return 0 +} +func (x *EvacuateShardResponse_Body) SetCount(v uint32) { + x.Count = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *EvacuateShardResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *EvacuateShardResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"count\":" + out.RawString(prefix[1:]) + out.Uint32(x.Count) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *EvacuateShardResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *EvacuateShardResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "count": + { + var f uint32 + f = in.Uint32() + x.Count = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type EvacuateShardResponse struct { + Body *EvacuateShardResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*EvacuateShardResponse)(nil) + _ encoding.ProtoUnmarshaler = (*EvacuateShardResponse)(nil) + _ json.Marshaler = (*EvacuateShardResponse)(nil) + _ json.Unmarshaler = (*EvacuateShardResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1294,27 +5047,6 @@ func (x *EvacuateShardResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *EvacuateShardResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1330,13 +5062,165 @@ func (x *EvacuateShardResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *EvacuateShardResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *EvacuateShardResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *EvacuateShardResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *EvacuateShardResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *EvacuateShardResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "EvacuateShardResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(EvacuateShardResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *EvacuateShardResponse) GetBody() *EvacuateShardResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *EvacuateShardResponse) SetBody(v *EvacuateShardResponse_Body) { + x.Body = v +} +func (x *EvacuateShardResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *EvacuateShardResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *EvacuateShardResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *EvacuateShardResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *EvacuateShardResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *EvacuateShardResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *EvacuateShardResponse_Body + f = new(EvacuateShardResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type FlushCacheRequest_Body struct { + Shard_ID [][]byte `json:"shardID"` + Seal bool `json:"seal"` +} + +var ( + _ encoding.ProtoMarshaler = (*FlushCacheRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*FlushCacheRequest_Body)(nil) + _ json.Marshaler = (*FlushCacheRequest_Body)(nil) + _ json.Unmarshaler = (*FlushCacheRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1349,27 +5233,168 @@ func (x *FlushCacheRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *FlushCacheRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) - offset += proto.BoolMarshal(2, buf[offset:], x.Seal) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *FlushCacheRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *FlushCacheRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for j := range x.Shard_ID { + mm.AppendBytes(1, x.Shard_ID[j]) + } + if x.Seal { + mm.AppendBool(2, x.Seal) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *FlushCacheRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "FlushCacheRequest_Body") + } + switch fc.FieldNum { + case 1: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = append(x.Shard_ID, data) + case 2: // Seal + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Seal") + } + x.Seal = data + } + } + return nil +} +func (x *FlushCacheRequest_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *FlushCacheRequest_Body) SetShard_ID(v [][]byte) { + x.Shard_ID = v +} +func (x *FlushCacheRequest_Body) GetSeal() bool { + if x != nil { + return x.Seal + } + return false +} +func (x *FlushCacheRequest_Body) SetSeal(v bool) { + x.Seal = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *FlushCacheRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *FlushCacheRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"shardID\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Shard_ID { + if i != 0 { + out.RawByte(',') + } + out.Base64Bytes(x.Shard_ID[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"seal\":" + out.RawString(prefix) + out.Bool(x.Seal) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *FlushCacheRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *FlushCacheRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shardID": + { + var f []byte + var list [][]byte + in.Delim('[') + for !in.IsDelim(']') { + f = in.Bytes() + list = append(list, f) + in.WantComma() + } + x.Shard_ID = list + in.Delim(']') + } + case "seal": + { + var f bool + f = in.Bool() + x.Seal = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type FlushCacheRequest struct { + Body *FlushCacheRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*FlushCacheRequest)(nil) + _ encoding.ProtoUnmarshaler = (*FlushCacheRequest)(nil) + _ json.Marshaler = (*FlushCacheRequest)(nil) + _ json.Unmarshaler = (*FlushCacheRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1382,27 +5407,6 @@ func (x *FlushCacheRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *FlushCacheRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1418,13 +5422,163 @@ func (x *FlushCacheRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *FlushCacheRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *FlushCacheRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *FlushCacheRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *FlushCacheRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *FlushCacheRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "FlushCacheRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(FlushCacheRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *FlushCacheRequest) GetBody() *FlushCacheRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *FlushCacheRequest) SetBody(v *FlushCacheRequest_Body) { + x.Body = v +} +func (x *FlushCacheRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *FlushCacheRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *FlushCacheRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *FlushCacheRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *FlushCacheRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *FlushCacheRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *FlushCacheRequest_Body + f = new(FlushCacheRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type FlushCacheResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*FlushCacheResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*FlushCacheResponse_Body)(nil) + _ json.Marshaler = (*FlushCacheResponse_Body)(nil) + _ json.Unmarshaler = (*FlushCacheResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1435,18 +5589,93 @@ func (x *FlushCacheResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *FlushCacheResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *FlushCacheResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *FlushCacheResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *FlushCacheResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "FlushCacheResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *FlushCacheResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *FlushCacheResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *FlushCacheResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *FlushCacheResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type FlushCacheResponse struct { + Body *FlushCacheResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*FlushCacheResponse)(nil) + _ encoding.ProtoUnmarshaler = (*FlushCacheResponse)(nil) + _ json.Marshaler = (*FlushCacheResponse)(nil) + _ json.Unmarshaler = (*FlushCacheResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1459,27 +5688,6 @@ func (x *FlushCacheResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *FlushCacheResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1495,13 +5703,165 @@ func (x *FlushCacheResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *FlushCacheResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *FlushCacheResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *FlushCacheResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *FlushCacheResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *FlushCacheResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "FlushCacheResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(FlushCacheResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *FlushCacheResponse) GetBody() *FlushCacheResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *FlushCacheResponse) SetBody(v *FlushCacheResponse_Body) { + x.Body = v +} +func (x *FlushCacheResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *FlushCacheResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *FlushCacheResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *FlushCacheResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *FlushCacheResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *FlushCacheResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *FlushCacheResponse_Body + f = new(FlushCacheResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DoctorRequest_Body struct { + Concurrency uint32 `json:"concurrency"` + RemoveDuplicates bool `json:"removeDuplicates"` +} + +var ( + _ encoding.ProtoMarshaler = (*DoctorRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*DoctorRequest_Body)(nil) + _ json.Marshaler = (*DoctorRequest_Body)(nil) + _ json.Unmarshaler = (*DoctorRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1514,27 +5874,154 @@ func (x *DoctorRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DoctorRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.UInt32Marshal(1, buf[offset:], x.Concurrency) - offset += proto.BoolMarshal(2, buf[offset:], x.RemoveDuplicates) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DoctorRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *DoctorRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Concurrency != 0 { + mm.AppendUint32(1, x.Concurrency) + } + if x.RemoveDuplicates { + mm.AppendBool(2, x.RemoveDuplicates) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DoctorRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DoctorRequest_Body") + } + switch fc.FieldNum { + case 1: // Concurrency + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Concurrency") + } + x.Concurrency = data + case 2: // RemoveDuplicates + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "RemoveDuplicates") + } + x.RemoveDuplicates = data + } + } + return nil +} +func (x *DoctorRequest_Body) GetConcurrency() uint32 { + if x != nil { + return x.Concurrency + } + return 0 +} +func (x *DoctorRequest_Body) SetConcurrency(v uint32) { + x.Concurrency = v +} +func (x *DoctorRequest_Body) GetRemoveDuplicates() bool { + if x != nil { + return x.RemoveDuplicates + } + return false +} +func (x *DoctorRequest_Body) SetRemoveDuplicates(v bool) { + x.RemoveDuplicates = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DoctorRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DoctorRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"concurrency\":" + out.RawString(prefix[1:]) + out.Uint32(x.Concurrency) + } + { + const prefix string = ",\"removeDuplicates\":" + out.RawString(prefix) + out.Bool(x.RemoveDuplicates) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DoctorRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DoctorRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "concurrency": + { + var f uint32 + f = in.Uint32() + x.Concurrency = f + } + case "removeDuplicates": + { + var f bool + f = in.Bool() + x.RemoveDuplicates = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DoctorRequest struct { + Body *DoctorRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*DoctorRequest)(nil) + _ encoding.ProtoUnmarshaler = (*DoctorRequest)(nil) + _ json.Marshaler = (*DoctorRequest)(nil) + _ json.Unmarshaler = (*DoctorRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1547,27 +6034,6 @@ func (x *DoctorRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DoctorRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1583,13 +6049,163 @@ func (x *DoctorRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *DoctorRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *DoctorRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DoctorRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *DoctorRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DoctorRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DoctorRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(DoctorRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *DoctorRequest) GetBody() *DoctorRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *DoctorRequest) SetBody(v *DoctorRequest_Body) { + x.Body = v +} +func (x *DoctorRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *DoctorRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DoctorRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DoctorRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DoctorRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DoctorRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *DoctorRequest_Body + f = new(DoctorRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DoctorResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*DoctorResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*DoctorResponse_Body)(nil) + _ json.Marshaler = (*DoctorResponse_Body)(nil) + _ json.Unmarshaler = (*DoctorResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1600,18 +6216,93 @@ func (x *DoctorResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DoctorResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DoctorResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *DoctorResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DoctorResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DoctorResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DoctorResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DoctorResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DoctorResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DoctorResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DoctorResponse struct { + Body *DoctorResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*DoctorResponse)(nil) + _ encoding.ProtoUnmarshaler = (*DoctorResponse)(nil) + _ json.Marshaler = (*DoctorResponse)(nil) + _ json.Unmarshaler = (*DoctorResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1624,27 +6315,6 @@ func (x *DoctorResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DoctorResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1660,13 +6330,201 @@ func (x *DoctorResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *DoctorResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *DoctorResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DoctorResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *DoctorResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DoctorResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DoctorResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(DoctorResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *DoctorResponse) GetBody() *DoctorResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *DoctorResponse) SetBody(v *DoctorResponse_Body) { + x.Body = v +} +func (x *DoctorResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *DoctorResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DoctorResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DoctorResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DoctorResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DoctorResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *DoctorResponse_Body + f = new(DoctorResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StartShardEvacuationRequest_Body_Scope int32 + +const ( + StartShardEvacuationRequest_Body_NONE StartShardEvacuationRequest_Body_Scope = 0 + StartShardEvacuationRequest_Body_OBJECTS StartShardEvacuationRequest_Body_Scope = 1 + StartShardEvacuationRequest_Body_TREES StartShardEvacuationRequest_Body_Scope = 2 +) + +var ( + StartShardEvacuationRequest_Body_Scope_name = map[int32]string{ + 0: "NONE", + 1: "OBJECTS", + 2: "TREES", + } + StartShardEvacuationRequest_Body_Scope_value = map[string]int32{ + "NONE": 0, + "OBJECTS": 1, + "TREES": 2, + } +) + +func (x StartShardEvacuationRequest_Body_Scope) String() string { + if v, ok := StartShardEvacuationRequest_Body_Scope_name[int32(x)]; ok { + return v + } + return strconv.FormatInt(int64(x), 10) +} +func (x *StartShardEvacuationRequest_Body_Scope) FromString(s string) bool { + if v, ok := StartShardEvacuationRequest_Body_Scope_value[s]; ok { + *x = StartShardEvacuationRequest_Body_Scope(v) + return true + } + return false +} + +type StartShardEvacuationRequest_Body struct { + Shard_ID [][]byte `json:"shardID"` + IgnoreErrors bool `json:"ignoreErrors"` + Scope uint32 `json:"scope"` +} + +var ( + _ encoding.ProtoMarshaler = (*StartShardEvacuationRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*StartShardEvacuationRequest_Body)(nil) + _ json.Marshaler = (*StartShardEvacuationRequest_Body)(nil) + _ json.Unmarshaler = (*StartShardEvacuationRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1680,28 +6538,197 @@ func (x *StartShardEvacuationRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *StartShardEvacuationRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) - offset += proto.BoolMarshal(2, buf[offset:], x.IgnoreErrors) - offset += proto.UInt32Marshal(3, buf[offset:], x.Scope) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StartShardEvacuationRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *StartShardEvacuationRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for j := range x.Shard_ID { + mm.AppendBytes(1, x.Shard_ID[j]) + } + if x.IgnoreErrors { + mm.AppendBool(2, x.IgnoreErrors) + } + if x.Scope != 0 { + mm.AppendUint32(3, x.Scope) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StartShardEvacuationRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StartShardEvacuationRequest_Body") + } + switch fc.FieldNum { + case 1: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = append(x.Shard_ID, data) + case 2: // IgnoreErrors + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "IgnoreErrors") + } + x.IgnoreErrors = data + case 3: // Scope + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Scope") + } + x.Scope = data + } + } + return nil +} +func (x *StartShardEvacuationRequest_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *StartShardEvacuationRequest_Body) SetShard_ID(v [][]byte) { + x.Shard_ID = v +} +func (x *StartShardEvacuationRequest_Body) GetIgnoreErrors() bool { + if x != nil { + return x.IgnoreErrors + } + return false +} +func (x *StartShardEvacuationRequest_Body) SetIgnoreErrors(v bool) { + x.IgnoreErrors = v +} +func (x *StartShardEvacuationRequest_Body) GetScope() uint32 { + if x != nil { + return x.Scope + } + return 0 +} +func (x *StartShardEvacuationRequest_Body) SetScope(v uint32) { + x.Scope = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StartShardEvacuationRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StartShardEvacuationRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"shardID\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Shard_ID { + if i != 0 { + out.RawByte(',') + } + out.Base64Bytes(x.Shard_ID[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"ignoreErrors\":" + out.RawString(prefix) + out.Bool(x.IgnoreErrors) + } + { + const prefix string = ",\"scope\":" + out.RawString(prefix) + out.Uint32(x.Scope) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StartShardEvacuationRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StartShardEvacuationRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shardID": + { + var f []byte + var list [][]byte + in.Delim('[') + for !in.IsDelim(']') { + f = in.Bytes() + list = append(list, f) + in.WantComma() + } + x.Shard_ID = list + in.Delim(']') + } + case "ignoreErrors": + { + var f bool + f = in.Bool() + x.IgnoreErrors = f + } + case "scope": + { + var f uint32 + f = in.Uint32() + x.Scope = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StartShardEvacuationRequest struct { + Body *StartShardEvacuationRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*StartShardEvacuationRequest)(nil) + _ encoding.ProtoUnmarshaler = (*StartShardEvacuationRequest)(nil) + _ json.Marshaler = (*StartShardEvacuationRequest)(nil) + _ json.Unmarshaler = (*StartShardEvacuationRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1714,27 +6741,6 @@ func (x *StartShardEvacuationRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *StartShardEvacuationRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1750,13 +6756,163 @@ func (x *StartShardEvacuationRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *StartShardEvacuationRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *StartShardEvacuationRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StartShardEvacuationRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *StartShardEvacuationRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StartShardEvacuationRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StartShardEvacuationRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(StartShardEvacuationRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *StartShardEvacuationRequest) GetBody() *StartShardEvacuationRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *StartShardEvacuationRequest) SetBody(v *StartShardEvacuationRequest_Body) { + x.Body = v +} +func (x *StartShardEvacuationRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *StartShardEvacuationRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StartShardEvacuationRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StartShardEvacuationRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StartShardEvacuationRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StartShardEvacuationRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *StartShardEvacuationRequest_Body + f = new(StartShardEvacuationRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StartShardEvacuationResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*StartShardEvacuationResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*StartShardEvacuationResponse_Body)(nil) + _ json.Marshaler = (*StartShardEvacuationResponse_Body)(nil) + _ json.Unmarshaler = (*StartShardEvacuationResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1767,18 +6923,93 @@ func (x *StartShardEvacuationResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *StartShardEvacuationResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StartShardEvacuationResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *StartShardEvacuationResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StartShardEvacuationResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StartShardEvacuationResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StartShardEvacuationResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StartShardEvacuationResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StartShardEvacuationResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StartShardEvacuationResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StartShardEvacuationResponse struct { + Body *StartShardEvacuationResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*StartShardEvacuationResponse)(nil) + _ encoding.ProtoUnmarshaler = (*StartShardEvacuationResponse)(nil) + _ json.Marshaler = (*StartShardEvacuationResponse)(nil) + _ json.Unmarshaler = (*StartShardEvacuationResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1791,27 +7022,6 @@ func (x *StartShardEvacuationResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *StartShardEvacuationResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1827,13 +7037,163 @@ func (x *StartShardEvacuationResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *StartShardEvacuationResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *StartShardEvacuationResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StartShardEvacuationResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *StartShardEvacuationResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StartShardEvacuationResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StartShardEvacuationResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(StartShardEvacuationResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *StartShardEvacuationResponse) GetBody() *StartShardEvacuationResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *StartShardEvacuationResponse) SetBody(v *StartShardEvacuationResponse_Body) { + x.Body = v +} +func (x *StartShardEvacuationResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *StartShardEvacuationResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StartShardEvacuationResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StartShardEvacuationResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StartShardEvacuationResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StartShardEvacuationResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *StartShardEvacuationResponse_Body + f = new(StartShardEvacuationResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetShardEvacuationStatusRequest_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*GetShardEvacuationStatusRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetShardEvacuationStatusRequest_Body)(nil) + _ json.Marshaler = (*GetShardEvacuationStatusRequest_Body)(nil) + _ json.Unmarshaler = (*GetShardEvacuationStatusRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1844,18 +7204,93 @@ func (x *GetShardEvacuationStatusRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetShardEvacuationStatusRequest_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetShardEvacuationStatusRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetShardEvacuationStatusRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetShardEvacuationStatusRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetShardEvacuationStatusRequest_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetShardEvacuationStatusRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetShardEvacuationStatusRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetShardEvacuationStatusRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetShardEvacuationStatusRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetShardEvacuationStatusRequest struct { + Body *GetShardEvacuationStatusRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetShardEvacuationStatusRequest)(nil) + _ encoding.ProtoUnmarshaler = (*GetShardEvacuationStatusRequest)(nil) + _ json.Marshaler = (*GetShardEvacuationStatusRequest)(nil) + _ json.Unmarshaler = (*GetShardEvacuationStatusRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1868,27 +7303,6 @@ func (x *GetShardEvacuationStatusRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetShardEvacuationStatusRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1904,13 +7318,199 @@ func (x *GetShardEvacuationStatusRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetShardEvacuationStatusRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetShardEvacuationStatusRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetShardEvacuationStatusRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetShardEvacuationStatusRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetShardEvacuationStatusRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetShardEvacuationStatusRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetShardEvacuationStatusRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetShardEvacuationStatusRequest) GetBody() *GetShardEvacuationStatusRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetShardEvacuationStatusRequest) SetBody(v *GetShardEvacuationStatusRequest_Body) { + x.Body = v +} +func (x *GetShardEvacuationStatusRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetShardEvacuationStatusRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetShardEvacuationStatusRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetShardEvacuationStatusRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetShardEvacuationStatusRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetShardEvacuationStatusRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetShardEvacuationStatusRequest_Body + f = new(GetShardEvacuationStatusRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetShardEvacuationStatusResponse_Body_Status int32 + +const ( + GetShardEvacuationStatusResponse_Body_EVACUATE_SHARD_STATUS_UNDEFINED GetShardEvacuationStatusResponse_Body_Status = 0 + GetShardEvacuationStatusResponse_Body_RUNNING GetShardEvacuationStatusResponse_Body_Status = 1 + GetShardEvacuationStatusResponse_Body_COMPLETED GetShardEvacuationStatusResponse_Body_Status = 2 +) + +var ( + GetShardEvacuationStatusResponse_Body_Status_name = map[int32]string{ + 0: "EVACUATE_SHARD_STATUS_UNDEFINED", + 1: "RUNNING", + 2: "COMPLETED", + } + GetShardEvacuationStatusResponse_Body_Status_value = map[string]int32{ + "EVACUATE_SHARD_STATUS_UNDEFINED": 0, + "RUNNING": 1, + "COMPLETED": 2, + } +) + +func (x GetShardEvacuationStatusResponse_Body_Status) String() string { + if v, ok := GetShardEvacuationStatusResponse_Body_Status_name[int32(x)]; ok { + return v + } + return strconv.FormatInt(int64(x), 10) +} +func (x *GetShardEvacuationStatusResponse_Body_Status) FromString(s string) bool { + if v, ok := GetShardEvacuationStatusResponse_Body_Status_value[s]; ok { + *x = GetShardEvacuationStatusResponse_Body_Status(v) + return true + } + return false +} + +type GetShardEvacuationStatusResponse_Body_UnixTimestamp struct { + Value int64 `json:"value"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil) + _ encoding.ProtoUnmarshaler = (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil) + _ json.Marshaler = (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil) + _ json.Unmarshaler = (*GetShardEvacuationStatusResponse_Body_UnixTimestamp)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1922,26 +7522,124 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) StableSize() (size return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.Int64Marshal(1, buf[offset:], x.Value) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Value != 0 { + mm.AppendInt64(1, x.Value) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetShardEvacuationStatusResponse_Body_UnixTimestamp") + } + switch fc.FieldNum { + case 1: // Value + data, ok := fc.Int64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Value") + } + x.Value = data + } + } + return nil +} +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) GetValue() int64 { + if x != nil { + return x.Value + } + return 0 +} +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) SetValue(v int64) { + x.Value = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"value\":" + out.RawString(prefix[1:]) + out.Int64(x.Value) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "value": + { + var f int64 + f = in.Int64() + x.Value = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetShardEvacuationStatusResponse_Body_Duration struct { + Seconds int64 `json:"seconds"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetShardEvacuationStatusResponse_Body_Duration)(nil) + _ encoding.ProtoUnmarshaler = (*GetShardEvacuationStatusResponse_Body_Duration)(nil) + _ json.Marshaler = (*GetShardEvacuationStatusResponse_Body_Duration)(nil) + _ json.Unmarshaler = (*GetShardEvacuationStatusResponse_Body_Duration)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1953,26 +7651,135 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) StableSize() (size int) return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetShardEvacuationStatusResponse_Body_Duration) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.Int64Marshal(1, buf[offset:], x.Seconds) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetShardEvacuationStatusResponse_Body_Duration) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetShardEvacuationStatusResponse_Body_Duration) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Seconds != 0 { + mm.AppendInt64(1, x.Seconds) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetShardEvacuationStatusResponse_Body_Duration) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetShardEvacuationStatusResponse_Body_Duration") + } + switch fc.FieldNum { + case 1: // Seconds + data, ok := fc.Int64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Seconds") + } + x.Seconds = data + } + } + return nil +} +func (x *GetShardEvacuationStatusResponse_Body_Duration) GetSeconds() int64 { + if x != nil { + return x.Seconds + } + return 0 +} +func (x *GetShardEvacuationStatusResponse_Body_Duration) SetSeconds(v int64) { + x.Seconds = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetShardEvacuationStatusResponse_Body_Duration) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetShardEvacuationStatusResponse_Body_Duration) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"seconds\":" + out.RawString(prefix[1:]) + out.Int64(x.Seconds) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetShardEvacuationStatusResponse_Body_Duration) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetShardEvacuationStatusResponse_Body_Duration) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "seconds": + { + var f int64 + f = in.Int64() + x.Seconds = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetShardEvacuationStatusResponse_Body struct { + TotalObjects uint64 `json:"totalObjects"` + EvacuatedObjects uint64 `json:"evacuatedObjects"` + FailedObjects uint64 `json:"failedObjects"` + Shard_ID [][]byte `json:"shardID"` + Status GetShardEvacuationStatusResponse_Body_Status `json:"status"` + Duration *GetShardEvacuationStatusResponse_Body_Duration `json:"duration"` + StartedAt *GetShardEvacuationStatusResponse_Body_UnixTimestamp `json:"startedAt"` + ErrorMessage string `json:"errorMessage"` + SkippedObjects uint64 `json:"skippedObjects"` + TotalTrees uint64 `json:"totalTrees"` + EvacuatedTrees uint64 `json:"evacuatedTrees"` + FailedTrees uint64 `json:"failedTrees"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetShardEvacuationStatusResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetShardEvacuationStatusResponse_Body)(nil) + _ json.Marshaler = (*GetShardEvacuationStatusResponse_Body)(nil) + _ json.Unmarshaler = (*GetShardEvacuationStatusResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1995,37 +7802,482 @@ func (x *GetShardEvacuationStatusResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetShardEvacuationStatusResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.UInt64Marshal(1, buf[offset:], x.TotalObjects) - offset += proto.UInt64Marshal(2, buf[offset:], x.EvacuatedObjects) - offset += proto.UInt64Marshal(3, buf[offset:], x.FailedObjects) - offset += proto.RepeatedBytesMarshal(4, buf[offset:], x.Shard_ID) - offset += proto.EnumMarshal(5, buf[offset:], int32(x.Status)) - offset += proto.NestedStructureMarshal(6, buf[offset:], x.Duration) - offset += proto.NestedStructureMarshal(7, buf[offset:], x.StartedAt) - offset += proto.StringMarshal(8, buf[offset:], x.ErrorMessage) - offset += proto.UInt64Marshal(9, buf[offset:], x.SkippedObjects) - offset += proto.UInt64Marshal(10, buf[offset:], x.TotalTrees) - offset += proto.UInt64Marshal(11, buf[offset:], x.EvacuatedTrees) - offset += proto.UInt64Marshal(12, buf[offset:], x.FailedTrees) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetShardEvacuationStatusResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetShardEvacuationStatusResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.TotalObjects != 0 { + mm.AppendUint64(1, x.TotalObjects) + } + if x.EvacuatedObjects != 0 { + mm.AppendUint64(2, x.EvacuatedObjects) + } + if x.FailedObjects != 0 { + mm.AppendUint64(3, x.FailedObjects) + } + for j := range x.Shard_ID { + mm.AppendBytes(4, x.Shard_ID[j]) + } + if int32(x.Status) != 0 { + mm.AppendInt32(5, int32(x.Status)) + } + if x.Duration != nil && x.Duration.StableSize() != 0 { + x.Duration.EmitProtobuf(mm.AppendMessage(6)) + } + if x.StartedAt != nil && x.StartedAt.StableSize() != 0 { + x.StartedAt.EmitProtobuf(mm.AppendMessage(7)) + } + if len(x.ErrorMessage) != 0 { + mm.AppendString(8, x.ErrorMessage) + } + if x.SkippedObjects != 0 { + mm.AppendUint64(9, x.SkippedObjects) + } + if x.TotalTrees != 0 { + mm.AppendUint64(10, x.TotalTrees) + } + if x.EvacuatedTrees != 0 { + mm.AppendUint64(11, x.EvacuatedTrees) + } + if x.FailedTrees != 0 { + mm.AppendUint64(12, x.FailedTrees) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetShardEvacuationStatusResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetShardEvacuationStatusResponse_Body") + } + switch fc.FieldNum { + case 1: // TotalObjects + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TotalObjects") + } + x.TotalObjects = data + case 2: // EvacuatedObjects + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "EvacuatedObjects") + } + x.EvacuatedObjects = data + case 3: // FailedObjects + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "FailedObjects") + } + x.FailedObjects = data + case 4: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = append(x.Shard_ID, data) + case 5: // Status + data, ok := fc.Int32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Status") + } + x.Status = GetShardEvacuationStatusResponse_Body_Status(data) + case 6: // Duration + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Duration") + } + x.Duration = new(GetShardEvacuationStatusResponse_Body_Duration) + if err := x.Duration.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 7: // StartedAt + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "StartedAt") + } + x.StartedAt = new(GetShardEvacuationStatusResponse_Body_UnixTimestamp) + if err := x.StartedAt.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 8: // ErrorMessage + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ErrorMessage") + } + x.ErrorMessage = data + case 9: // SkippedObjects + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "SkippedObjects") + } + x.SkippedObjects = data + case 10: // TotalTrees + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TotalTrees") + } + x.TotalTrees = data + case 11: // EvacuatedTrees + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "EvacuatedTrees") + } + x.EvacuatedTrees = data + case 12: // FailedTrees + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "FailedTrees") + } + x.FailedTrees = data + } + } + return nil +} +func (x *GetShardEvacuationStatusResponse_Body) GetTotalObjects() uint64 { + if x != nil { + return x.TotalObjects + } + return 0 +} +func (x *GetShardEvacuationStatusResponse_Body) SetTotalObjects(v uint64) { + x.TotalObjects = v +} +func (x *GetShardEvacuationStatusResponse_Body) GetEvacuatedObjects() uint64 { + if x != nil { + return x.EvacuatedObjects + } + return 0 +} +func (x *GetShardEvacuationStatusResponse_Body) SetEvacuatedObjects(v uint64) { + x.EvacuatedObjects = v +} +func (x *GetShardEvacuationStatusResponse_Body) GetFailedObjects() uint64 { + if x != nil { + return x.FailedObjects + } + return 0 +} +func (x *GetShardEvacuationStatusResponse_Body) SetFailedObjects(v uint64) { + x.FailedObjects = v +} +func (x *GetShardEvacuationStatusResponse_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *GetShardEvacuationStatusResponse_Body) SetShard_ID(v [][]byte) { + x.Shard_ID = v +} +func (x *GetShardEvacuationStatusResponse_Body) GetStatus() GetShardEvacuationStatusResponse_Body_Status { + if x != nil { + return x.Status + } + return 0 +} +func (x *GetShardEvacuationStatusResponse_Body) SetStatus(v GetShardEvacuationStatusResponse_Body_Status) { + x.Status = v +} +func (x *GetShardEvacuationStatusResponse_Body) GetDuration() *GetShardEvacuationStatusResponse_Body_Duration { + if x != nil { + return x.Duration + } + return nil +} +func (x *GetShardEvacuationStatusResponse_Body) SetDuration(v *GetShardEvacuationStatusResponse_Body_Duration) { + x.Duration = v +} +func (x *GetShardEvacuationStatusResponse_Body) GetStartedAt() *GetShardEvacuationStatusResponse_Body_UnixTimestamp { + if x != nil { + return x.StartedAt + } + return nil +} +func (x *GetShardEvacuationStatusResponse_Body) SetStartedAt(v *GetShardEvacuationStatusResponse_Body_UnixTimestamp) { + x.StartedAt = v +} +func (x *GetShardEvacuationStatusResponse_Body) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" +} +func (x *GetShardEvacuationStatusResponse_Body) SetErrorMessage(v string) { + x.ErrorMessage = v +} +func (x *GetShardEvacuationStatusResponse_Body) GetSkippedObjects() uint64 { + if x != nil { + return x.SkippedObjects + } + return 0 +} +func (x *GetShardEvacuationStatusResponse_Body) SetSkippedObjects(v uint64) { + x.SkippedObjects = v +} +func (x *GetShardEvacuationStatusResponse_Body) GetTotalTrees() uint64 { + if x != nil { + return x.TotalTrees + } + return 0 +} +func (x *GetShardEvacuationStatusResponse_Body) SetTotalTrees(v uint64) { + x.TotalTrees = v +} +func (x *GetShardEvacuationStatusResponse_Body) GetEvacuatedTrees() uint64 { + if x != nil { + return x.EvacuatedTrees + } + return 0 +} +func (x *GetShardEvacuationStatusResponse_Body) SetEvacuatedTrees(v uint64) { + x.EvacuatedTrees = v +} +func (x *GetShardEvacuationStatusResponse_Body) GetFailedTrees() uint64 { + if x != nil { + return x.FailedTrees + } + return 0 +} +func (x *GetShardEvacuationStatusResponse_Body) SetFailedTrees(v uint64) { + x.FailedTrees = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetShardEvacuationStatusResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetShardEvacuationStatusResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"totalObjects\":" + out.RawString(prefix[1:]) + out.Uint64(x.TotalObjects) + } + { + const prefix string = ",\"evacuatedObjects\":" + out.RawString(prefix) + out.Uint64(x.EvacuatedObjects) + } + { + const prefix string = ",\"failedObjects\":" + out.RawString(prefix) + out.Uint64(x.FailedObjects) + } + { + const prefix string = ",\"shardID\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Shard_ID { + if i != 0 { + out.RawByte(',') + } + out.Base64Bytes(x.Shard_ID[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"status\":" + out.RawString(prefix) + out.Int32(int32(x.Status)) + } + { + const prefix string = ",\"duration\":" + out.RawString(prefix) + x.Duration.MarshalEasyJSON(out) + } + { + const prefix string = ",\"startedAt\":" + out.RawString(prefix) + x.StartedAt.MarshalEasyJSON(out) + } + { + const prefix string = ",\"errorMessage\":" + out.RawString(prefix) + out.String(x.ErrorMessage) + } + { + const prefix string = ",\"skippedObjects\":" + out.RawString(prefix) + out.Uint64(x.SkippedObjects) + } + { + const prefix string = ",\"totalTrees\":" + out.RawString(prefix) + out.Uint64(x.TotalTrees) + } + { + const prefix string = ",\"evacuatedTrees\":" + out.RawString(prefix) + out.Uint64(x.EvacuatedTrees) + } + { + const prefix string = ",\"failedTrees\":" + out.RawString(prefix) + out.Uint64(x.FailedTrees) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetShardEvacuationStatusResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetShardEvacuationStatusResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "totalObjects": + { + var f uint64 + f = in.Uint64() + x.TotalObjects = f + } + case "evacuatedObjects": + { + var f uint64 + f = in.Uint64() + x.EvacuatedObjects = f + } + case "failedObjects": + { + var f uint64 + f = in.Uint64() + x.FailedObjects = f + } + case "shardID": + { + var f []byte + var list [][]byte + in.Delim('[') + for !in.IsDelim(']') { + f = in.Bytes() + list = append(list, f) + in.WantComma() + } + x.Shard_ID = list + in.Delim(']') + } + case "status": + { + var f GetShardEvacuationStatusResponse_Body_Status + var parsedValue GetShardEvacuationStatusResponse_Body_Status + switch v := in.Interface().(type) { + case string: + if vv, ok := GetShardEvacuationStatusResponse_Body_Status_value[v]; ok { + parsedValue = GetShardEvacuationStatusResponse_Body_Status(vv) + break + } + vv, err := strconv.ParseInt(v, 10, 32) + if err != nil { + in.AddError(err) + return + } + parsedValue = GetShardEvacuationStatusResponse_Body_Status(vv) + case float64: + parsedValue = GetShardEvacuationStatusResponse_Body_Status(v) + } + f = parsedValue + x.Status = f + } + case "duration": + { + var f *GetShardEvacuationStatusResponse_Body_Duration + f = new(GetShardEvacuationStatusResponse_Body_Duration) + f.UnmarshalEasyJSON(in) + x.Duration = f + } + case "startedAt": + { + var f *GetShardEvacuationStatusResponse_Body_UnixTimestamp + f = new(GetShardEvacuationStatusResponse_Body_UnixTimestamp) + f.UnmarshalEasyJSON(in) + x.StartedAt = f + } + case "errorMessage": + { + var f string + f = in.String() + x.ErrorMessage = f + } + case "skippedObjects": + { + var f uint64 + f = in.Uint64() + x.SkippedObjects = f + } + case "totalTrees": + { + var f uint64 + f = in.Uint64() + x.TotalTrees = f + } + case "evacuatedTrees": + { + var f uint64 + f = in.Uint64() + x.EvacuatedTrees = f + } + case "failedTrees": + { + var f uint64 + f = in.Uint64() + x.FailedTrees = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetShardEvacuationStatusResponse struct { + Body *GetShardEvacuationStatusResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetShardEvacuationStatusResponse)(nil) + _ encoding.ProtoUnmarshaler = (*GetShardEvacuationStatusResponse)(nil) + _ json.Marshaler = (*GetShardEvacuationStatusResponse)(nil) + _ json.Unmarshaler = (*GetShardEvacuationStatusResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2038,27 +8290,6 @@ func (x *GetShardEvacuationStatusResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetShardEvacuationStatusResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2074,13 +8305,163 @@ func (x *GetShardEvacuationStatusResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetShardEvacuationStatusResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetShardEvacuationStatusResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetShardEvacuationStatusResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetShardEvacuationStatusResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetShardEvacuationStatusResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetShardEvacuationStatusResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetShardEvacuationStatusResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetShardEvacuationStatusResponse) GetBody() *GetShardEvacuationStatusResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetShardEvacuationStatusResponse) SetBody(v *GetShardEvacuationStatusResponse_Body) { + x.Body = v +} +func (x *GetShardEvacuationStatusResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetShardEvacuationStatusResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetShardEvacuationStatusResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetShardEvacuationStatusResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetShardEvacuationStatusResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetShardEvacuationStatusResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetShardEvacuationStatusResponse_Body + f = new(GetShardEvacuationStatusResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ResetShardEvacuationStatusRequest_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*ResetShardEvacuationStatusRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ResetShardEvacuationStatusRequest_Body)(nil) + _ json.Marshaler = (*ResetShardEvacuationStatusRequest_Body)(nil) + _ json.Unmarshaler = (*ResetShardEvacuationStatusRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2091,18 +8472,93 @@ func (x *ResetShardEvacuationStatusRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ResetShardEvacuationStatusRequest_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ResetShardEvacuationStatusRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ResetShardEvacuationStatusRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ResetShardEvacuationStatusRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ResetShardEvacuationStatusRequest_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ResetShardEvacuationStatusRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ResetShardEvacuationStatusRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ResetShardEvacuationStatusRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ResetShardEvacuationStatusRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ResetShardEvacuationStatusRequest struct { + Body *ResetShardEvacuationStatusRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ResetShardEvacuationStatusRequest)(nil) + _ encoding.ProtoUnmarshaler = (*ResetShardEvacuationStatusRequest)(nil) + _ json.Marshaler = (*ResetShardEvacuationStatusRequest)(nil) + _ json.Unmarshaler = (*ResetShardEvacuationStatusRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2115,27 +8571,6 @@ func (x *ResetShardEvacuationStatusRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ResetShardEvacuationStatusRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2151,13 +8586,163 @@ func (x *ResetShardEvacuationStatusRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *ResetShardEvacuationStatusRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *ResetShardEvacuationStatusRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ResetShardEvacuationStatusRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ResetShardEvacuationStatusRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ResetShardEvacuationStatusRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ResetShardEvacuationStatusRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ResetShardEvacuationStatusRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ResetShardEvacuationStatusRequest) GetBody() *ResetShardEvacuationStatusRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ResetShardEvacuationStatusRequest) SetBody(v *ResetShardEvacuationStatusRequest_Body) { + x.Body = v +} +func (x *ResetShardEvacuationStatusRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ResetShardEvacuationStatusRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ResetShardEvacuationStatusRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ResetShardEvacuationStatusRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ResetShardEvacuationStatusRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ResetShardEvacuationStatusRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ResetShardEvacuationStatusRequest_Body + f = new(ResetShardEvacuationStatusRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ResetShardEvacuationStatusResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*ResetShardEvacuationStatusResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ResetShardEvacuationStatusResponse_Body)(nil) + _ json.Marshaler = (*ResetShardEvacuationStatusResponse_Body)(nil) + _ json.Unmarshaler = (*ResetShardEvacuationStatusResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2168,18 +8753,93 @@ func (x *ResetShardEvacuationStatusResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ResetShardEvacuationStatusResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ResetShardEvacuationStatusResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ResetShardEvacuationStatusResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ResetShardEvacuationStatusResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ResetShardEvacuationStatusResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ResetShardEvacuationStatusResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ResetShardEvacuationStatusResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ResetShardEvacuationStatusResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ResetShardEvacuationStatusResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ResetShardEvacuationStatusResponse struct { + Body *ResetShardEvacuationStatusResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ResetShardEvacuationStatusResponse)(nil) + _ encoding.ProtoUnmarshaler = (*ResetShardEvacuationStatusResponse)(nil) + _ json.Marshaler = (*ResetShardEvacuationStatusResponse)(nil) + _ json.Unmarshaler = (*ResetShardEvacuationStatusResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2192,27 +8852,6 @@ func (x *ResetShardEvacuationStatusResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ResetShardEvacuationStatusResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2228,13 +8867,163 @@ func (x *ResetShardEvacuationStatusResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *ResetShardEvacuationStatusResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *ResetShardEvacuationStatusResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ResetShardEvacuationStatusResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ResetShardEvacuationStatusResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ResetShardEvacuationStatusResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ResetShardEvacuationStatusResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ResetShardEvacuationStatusResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ResetShardEvacuationStatusResponse) GetBody() *ResetShardEvacuationStatusResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ResetShardEvacuationStatusResponse) SetBody(v *ResetShardEvacuationStatusResponse_Body) { + x.Body = v +} +func (x *ResetShardEvacuationStatusResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ResetShardEvacuationStatusResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ResetShardEvacuationStatusResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ResetShardEvacuationStatusResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ResetShardEvacuationStatusResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ResetShardEvacuationStatusResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ResetShardEvacuationStatusResponse_Body + f = new(ResetShardEvacuationStatusResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StopShardEvacuationRequest_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*StopShardEvacuationRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*StopShardEvacuationRequest_Body)(nil) + _ json.Marshaler = (*StopShardEvacuationRequest_Body)(nil) + _ json.Unmarshaler = (*StopShardEvacuationRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2245,18 +9034,93 @@ func (x *StopShardEvacuationRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *StopShardEvacuationRequest_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StopShardEvacuationRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *StopShardEvacuationRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StopShardEvacuationRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StopShardEvacuationRequest_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StopShardEvacuationRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StopShardEvacuationRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StopShardEvacuationRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StopShardEvacuationRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StopShardEvacuationRequest struct { + Body *StopShardEvacuationRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*StopShardEvacuationRequest)(nil) + _ encoding.ProtoUnmarshaler = (*StopShardEvacuationRequest)(nil) + _ json.Marshaler = (*StopShardEvacuationRequest)(nil) + _ json.Unmarshaler = (*StopShardEvacuationRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2269,27 +9133,6 @@ func (x *StopShardEvacuationRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *StopShardEvacuationRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2305,13 +9148,163 @@ func (x *StopShardEvacuationRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *StopShardEvacuationRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *StopShardEvacuationRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StopShardEvacuationRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *StopShardEvacuationRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StopShardEvacuationRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StopShardEvacuationRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(StopShardEvacuationRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *StopShardEvacuationRequest) GetBody() *StopShardEvacuationRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *StopShardEvacuationRequest) SetBody(v *StopShardEvacuationRequest_Body) { + x.Body = v +} +func (x *StopShardEvacuationRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *StopShardEvacuationRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StopShardEvacuationRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StopShardEvacuationRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StopShardEvacuationRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StopShardEvacuationRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *StopShardEvacuationRequest_Body + f = new(StopShardEvacuationRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StopShardEvacuationResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*StopShardEvacuationResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*StopShardEvacuationResponse_Body)(nil) + _ json.Marshaler = (*StopShardEvacuationResponse_Body)(nil) + _ json.Unmarshaler = (*StopShardEvacuationResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2322,18 +9315,93 @@ func (x *StopShardEvacuationResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *StopShardEvacuationResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StopShardEvacuationResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *StopShardEvacuationResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StopShardEvacuationResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StopShardEvacuationResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StopShardEvacuationResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StopShardEvacuationResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StopShardEvacuationResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StopShardEvacuationResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StopShardEvacuationResponse struct { + Body *StopShardEvacuationResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*StopShardEvacuationResponse)(nil) + _ encoding.ProtoUnmarshaler = (*StopShardEvacuationResponse)(nil) + _ json.Marshaler = (*StopShardEvacuationResponse)(nil) + _ json.Unmarshaler = (*StopShardEvacuationResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2346,27 +9414,6 @@ func (x *StopShardEvacuationResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *StopShardEvacuationResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2382,13 +9429,165 @@ func (x *StopShardEvacuationResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *StopShardEvacuationResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *StopShardEvacuationResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StopShardEvacuationResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *StopShardEvacuationResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StopShardEvacuationResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StopShardEvacuationResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(StopShardEvacuationResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *StopShardEvacuationResponse) GetBody() *StopShardEvacuationResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *StopShardEvacuationResponse) SetBody(v *StopShardEvacuationResponse_Body) { + x.Body = v +} +func (x *StopShardEvacuationResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *StopShardEvacuationResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StopShardEvacuationResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StopShardEvacuationResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StopShardEvacuationResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StopShardEvacuationResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *StopShardEvacuationResponse_Body + f = new(StopShardEvacuationResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type AddChainLocalOverrideRequest_Body struct { + Target *ChainTarget `json:"target"` + Chain []byte `json:"chain"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddChainLocalOverrideRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*AddChainLocalOverrideRequest_Body)(nil) + _ json.Marshaler = (*AddChainLocalOverrideRequest_Body)(nil) + _ json.Unmarshaler = (*AddChainLocalOverrideRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2401,27 +9600,158 @@ func (x *AddChainLocalOverrideRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddChainLocalOverrideRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) - offset += proto.BytesMarshal(2, buf[offset:], x.Chain) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddChainLocalOverrideRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddChainLocalOverrideRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Target != nil && x.Target.StableSize() != 0 { + x.Target.EmitProtobuf(mm.AppendMessage(1)) + } + if len(x.Chain) != 0 { + mm.AppendBytes(2, x.Chain) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddChainLocalOverrideRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddChainLocalOverrideRequest_Body") + } + switch fc.FieldNum { + case 1: // Target + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Target") + } + x.Target = new(ChainTarget) + if err := x.Target.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Chain + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Chain") + } + x.Chain = data + } + } + return nil +} +func (x *AddChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { + if x != nil { + return x.Target + } + return nil +} +func (x *AddChainLocalOverrideRequest_Body) SetTarget(v *ChainTarget) { + x.Target = v +} +func (x *AddChainLocalOverrideRequest_Body) GetChain() []byte { + if x != nil { + return x.Chain + } + return nil +} +func (x *AddChainLocalOverrideRequest_Body) SetChain(v []byte) { + x.Chain = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddChainLocalOverrideRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddChainLocalOverrideRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"target\":" + out.RawString(prefix[1:]) + x.Target.MarshalEasyJSON(out) + } + { + const prefix string = ",\"chain\":" + out.RawString(prefix) + out.Base64Bytes(x.Chain) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddChainLocalOverrideRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddChainLocalOverrideRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "target": + { + var f *ChainTarget + f = new(ChainTarget) + f.UnmarshalEasyJSON(in) + x.Target = f + } + case "chain": + { + var f []byte + f = in.Bytes() + x.Chain = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type AddChainLocalOverrideRequest struct { + Body *AddChainLocalOverrideRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddChainLocalOverrideRequest)(nil) + _ encoding.ProtoUnmarshaler = (*AddChainLocalOverrideRequest)(nil) + _ json.Marshaler = (*AddChainLocalOverrideRequest)(nil) + _ json.Unmarshaler = (*AddChainLocalOverrideRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2434,27 +9764,6 @@ func (x *AddChainLocalOverrideRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddChainLocalOverrideRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2470,13 +9779,164 @@ func (x *AddChainLocalOverrideRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *AddChainLocalOverrideRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *AddChainLocalOverrideRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddChainLocalOverrideRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddChainLocalOverrideRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddChainLocalOverrideRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddChainLocalOverrideRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(AddChainLocalOverrideRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *AddChainLocalOverrideRequest) GetBody() *AddChainLocalOverrideRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *AddChainLocalOverrideRequest) SetBody(v *AddChainLocalOverrideRequest_Body) { + x.Body = v +} +func (x *AddChainLocalOverrideRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *AddChainLocalOverrideRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddChainLocalOverrideRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddChainLocalOverrideRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddChainLocalOverrideRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddChainLocalOverrideRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *AddChainLocalOverrideRequest_Body + f = new(AddChainLocalOverrideRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type AddChainLocalOverrideResponse_Body struct { + ChainId []byte `json:"chainId"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddChainLocalOverrideResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*AddChainLocalOverrideResponse_Body)(nil) + _ json.Marshaler = (*AddChainLocalOverrideResponse_Body)(nil) + _ json.Unmarshaler = (*AddChainLocalOverrideResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2488,26 +9948,125 @@ func (x *AddChainLocalOverrideResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddChainLocalOverrideResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ChainId) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddChainLocalOverrideResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddChainLocalOverrideResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ChainId) != 0 { + mm.AppendBytes(1, x.ChainId) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddChainLocalOverrideResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddChainLocalOverrideResponse_Body") + } + switch fc.FieldNum { + case 1: // ChainId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ChainId") + } + x.ChainId = data + } + } + return nil +} +func (x *AddChainLocalOverrideResponse_Body) GetChainId() []byte { + if x != nil { + return x.ChainId + } + return nil +} +func (x *AddChainLocalOverrideResponse_Body) SetChainId(v []byte) { + x.ChainId = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddChainLocalOverrideResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddChainLocalOverrideResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"chainId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ChainId) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddChainLocalOverrideResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddChainLocalOverrideResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "chainId": + { + var f []byte + f = in.Bytes() + x.ChainId = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type AddChainLocalOverrideResponse struct { + Body *AddChainLocalOverrideResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddChainLocalOverrideResponse)(nil) + _ encoding.ProtoUnmarshaler = (*AddChainLocalOverrideResponse)(nil) + _ json.Marshaler = (*AddChainLocalOverrideResponse)(nil) + _ json.Unmarshaler = (*AddChainLocalOverrideResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2520,27 +10079,6 @@ func (x *AddChainLocalOverrideResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddChainLocalOverrideResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2556,13 +10094,165 @@ func (x *AddChainLocalOverrideResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *AddChainLocalOverrideResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *AddChainLocalOverrideResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddChainLocalOverrideResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddChainLocalOverrideResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddChainLocalOverrideResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddChainLocalOverrideResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(AddChainLocalOverrideResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *AddChainLocalOverrideResponse) GetBody() *AddChainLocalOverrideResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *AddChainLocalOverrideResponse) SetBody(v *AddChainLocalOverrideResponse_Body) { + x.Body = v +} +func (x *AddChainLocalOverrideResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *AddChainLocalOverrideResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddChainLocalOverrideResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddChainLocalOverrideResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddChainLocalOverrideResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddChainLocalOverrideResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *AddChainLocalOverrideResponse_Body + f = new(AddChainLocalOverrideResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetChainLocalOverrideRequest_Body struct { + Target *ChainTarget `json:"target"` + ChainId []byte `json:"chainId"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetChainLocalOverrideRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetChainLocalOverrideRequest_Body)(nil) + _ json.Marshaler = (*GetChainLocalOverrideRequest_Body)(nil) + _ json.Unmarshaler = (*GetChainLocalOverrideRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2575,27 +10265,158 @@ func (x *GetChainLocalOverrideRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetChainLocalOverrideRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) - offset += proto.BytesMarshal(2, buf[offset:], x.ChainId) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetChainLocalOverrideRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetChainLocalOverrideRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Target != nil && x.Target.StableSize() != 0 { + x.Target.EmitProtobuf(mm.AppendMessage(1)) + } + if len(x.ChainId) != 0 { + mm.AppendBytes(2, x.ChainId) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetChainLocalOverrideRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetChainLocalOverrideRequest_Body") + } + switch fc.FieldNum { + case 1: // Target + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Target") + } + x.Target = new(ChainTarget) + if err := x.Target.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // ChainId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ChainId") + } + x.ChainId = data + } + } + return nil +} +func (x *GetChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { + if x != nil { + return x.Target + } + return nil +} +func (x *GetChainLocalOverrideRequest_Body) SetTarget(v *ChainTarget) { + x.Target = v +} +func (x *GetChainLocalOverrideRequest_Body) GetChainId() []byte { + if x != nil { + return x.ChainId + } + return nil +} +func (x *GetChainLocalOverrideRequest_Body) SetChainId(v []byte) { + x.ChainId = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetChainLocalOverrideRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetChainLocalOverrideRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"target\":" + out.RawString(prefix[1:]) + x.Target.MarshalEasyJSON(out) + } + { + const prefix string = ",\"chainId\":" + out.RawString(prefix) + out.Base64Bytes(x.ChainId) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetChainLocalOverrideRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetChainLocalOverrideRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "target": + { + var f *ChainTarget + f = new(ChainTarget) + f.UnmarshalEasyJSON(in) + x.Target = f + } + case "chainId": + { + var f []byte + f = in.Bytes() + x.ChainId = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetChainLocalOverrideRequest struct { + Body *GetChainLocalOverrideRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetChainLocalOverrideRequest)(nil) + _ encoding.ProtoUnmarshaler = (*GetChainLocalOverrideRequest)(nil) + _ json.Marshaler = (*GetChainLocalOverrideRequest)(nil) + _ json.Unmarshaler = (*GetChainLocalOverrideRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2608,27 +10429,6 @@ func (x *GetChainLocalOverrideRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetChainLocalOverrideRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2644,13 +10444,164 @@ func (x *GetChainLocalOverrideRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetChainLocalOverrideRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetChainLocalOverrideRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetChainLocalOverrideRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetChainLocalOverrideRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetChainLocalOverrideRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetChainLocalOverrideRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetChainLocalOverrideRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetChainLocalOverrideRequest) GetBody() *GetChainLocalOverrideRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetChainLocalOverrideRequest) SetBody(v *GetChainLocalOverrideRequest_Body) { + x.Body = v +} +func (x *GetChainLocalOverrideRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetChainLocalOverrideRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetChainLocalOverrideRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetChainLocalOverrideRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetChainLocalOverrideRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetChainLocalOverrideRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetChainLocalOverrideRequest_Body + f = new(GetChainLocalOverrideRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetChainLocalOverrideResponse_Body struct { + Chain []byte `json:"chain"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetChainLocalOverrideResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetChainLocalOverrideResponse_Body)(nil) + _ json.Marshaler = (*GetChainLocalOverrideResponse_Body)(nil) + _ json.Unmarshaler = (*GetChainLocalOverrideResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2662,26 +10613,125 @@ func (x *GetChainLocalOverrideResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetChainLocalOverrideResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.Chain) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetChainLocalOverrideResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetChainLocalOverrideResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Chain) != 0 { + mm.AppendBytes(1, x.Chain) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetChainLocalOverrideResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetChainLocalOverrideResponse_Body") + } + switch fc.FieldNum { + case 1: // Chain + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Chain") + } + x.Chain = data + } + } + return nil +} +func (x *GetChainLocalOverrideResponse_Body) GetChain() []byte { + if x != nil { + return x.Chain + } + return nil +} +func (x *GetChainLocalOverrideResponse_Body) SetChain(v []byte) { + x.Chain = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetChainLocalOverrideResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetChainLocalOverrideResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"chain\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.Chain) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetChainLocalOverrideResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetChainLocalOverrideResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "chain": + { + var f []byte + f = in.Bytes() + x.Chain = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetChainLocalOverrideResponse struct { + Body *GetChainLocalOverrideResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetChainLocalOverrideResponse)(nil) + _ encoding.ProtoUnmarshaler = (*GetChainLocalOverrideResponse)(nil) + _ json.Marshaler = (*GetChainLocalOverrideResponse)(nil) + _ json.Unmarshaler = (*GetChainLocalOverrideResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2694,27 +10744,6 @@ func (x *GetChainLocalOverrideResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetChainLocalOverrideResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2730,13 +10759,164 @@ func (x *GetChainLocalOverrideResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetChainLocalOverrideResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetChainLocalOverrideResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetChainLocalOverrideResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetChainLocalOverrideResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetChainLocalOverrideResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetChainLocalOverrideResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetChainLocalOverrideResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetChainLocalOverrideResponse) GetBody() *GetChainLocalOverrideResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetChainLocalOverrideResponse) SetBody(v *GetChainLocalOverrideResponse_Body) { + x.Body = v +} +func (x *GetChainLocalOverrideResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetChainLocalOverrideResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetChainLocalOverrideResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetChainLocalOverrideResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetChainLocalOverrideResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetChainLocalOverrideResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetChainLocalOverrideResponse_Body + f = new(GetChainLocalOverrideResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListChainLocalOverridesRequest_Body struct { + Target *ChainTarget `json:"target"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListChainLocalOverridesRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ListChainLocalOverridesRequest_Body)(nil) + _ json.Marshaler = (*ListChainLocalOverridesRequest_Body)(nil) + _ json.Unmarshaler = (*ListChainLocalOverridesRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2748,26 +10928,129 @@ func (x *ListChainLocalOverridesRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListChainLocalOverridesRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListChainLocalOverridesRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListChainLocalOverridesRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Target != nil && x.Target.StableSize() != 0 { + x.Target.EmitProtobuf(mm.AppendMessage(1)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListChainLocalOverridesRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListChainLocalOverridesRequest_Body") + } + switch fc.FieldNum { + case 1: // Target + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Target") + } + x.Target = new(ChainTarget) + if err := x.Target.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ListChainLocalOverridesRequest_Body) GetTarget() *ChainTarget { + if x != nil { + return x.Target + } + return nil +} +func (x *ListChainLocalOverridesRequest_Body) SetTarget(v *ChainTarget) { + x.Target = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListChainLocalOverridesRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListChainLocalOverridesRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"target\":" + out.RawString(prefix[1:]) + x.Target.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListChainLocalOverridesRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListChainLocalOverridesRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "target": + { + var f *ChainTarget + f = new(ChainTarget) + f.UnmarshalEasyJSON(in) + x.Target = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListChainLocalOverridesRequest struct { + Body *ListChainLocalOverridesRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListChainLocalOverridesRequest)(nil) + _ encoding.ProtoUnmarshaler = (*ListChainLocalOverridesRequest)(nil) + _ json.Marshaler = (*ListChainLocalOverridesRequest)(nil) + _ json.Unmarshaler = (*ListChainLocalOverridesRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2780,27 +11063,6 @@ func (x *ListChainLocalOverridesRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListChainLocalOverridesRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2816,13 +11078,164 @@ func (x *ListChainLocalOverridesRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *ListChainLocalOverridesRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *ListChainLocalOverridesRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListChainLocalOverridesRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListChainLocalOverridesRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListChainLocalOverridesRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListChainLocalOverridesRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ListChainLocalOverridesRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ListChainLocalOverridesRequest) GetBody() *ListChainLocalOverridesRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ListChainLocalOverridesRequest) SetBody(v *ListChainLocalOverridesRequest_Body) { + x.Body = v +} +func (x *ListChainLocalOverridesRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ListChainLocalOverridesRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListChainLocalOverridesRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListChainLocalOverridesRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListChainLocalOverridesRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListChainLocalOverridesRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ListChainLocalOverridesRequest_Body + f = new(ListChainLocalOverridesRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListChainLocalOverridesResponse_Body struct { + Chains [][]byte `json:"chains"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListChainLocalOverridesResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ListChainLocalOverridesResponse_Body)(nil) + _ json.Marshaler = (*ListChainLocalOverridesResponse_Body)(nil) + _ json.Unmarshaler = (*ListChainLocalOverridesResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2834,26 +11247,139 @@ func (x *ListChainLocalOverridesResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListChainLocalOverridesResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Chains) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListChainLocalOverridesResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListChainLocalOverridesResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for j := range x.Chains { + mm.AppendBytes(1, x.Chains[j]) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListChainLocalOverridesResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListChainLocalOverridesResponse_Body") + } + switch fc.FieldNum { + case 1: // Chains + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Chains") + } + x.Chains = append(x.Chains, data) + } + } + return nil +} +func (x *ListChainLocalOverridesResponse_Body) GetChains() [][]byte { + if x != nil { + return x.Chains + } + return nil +} +func (x *ListChainLocalOverridesResponse_Body) SetChains(v [][]byte) { + x.Chains = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListChainLocalOverridesResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListChainLocalOverridesResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"chains\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Chains { + if i != 0 { + out.RawByte(',') + } + out.Base64Bytes(x.Chains[i]) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListChainLocalOverridesResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListChainLocalOverridesResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "chains": + { + var f []byte + var list [][]byte + in.Delim('[') + for !in.IsDelim(']') { + f = in.Bytes() + list = append(list, f) + in.WantComma() + } + x.Chains = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListChainLocalOverridesResponse struct { + Body *ListChainLocalOverridesResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListChainLocalOverridesResponse)(nil) + _ encoding.ProtoUnmarshaler = (*ListChainLocalOverridesResponse)(nil) + _ json.Marshaler = (*ListChainLocalOverridesResponse)(nil) + _ json.Unmarshaler = (*ListChainLocalOverridesResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2866,27 +11392,6 @@ func (x *ListChainLocalOverridesResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListChainLocalOverridesResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2902,13 +11407,164 @@ func (x *ListChainLocalOverridesResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *ListChainLocalOverridesResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *ListChainLocalOverridesResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListChainLocalOverridesResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListChainLocalOverridesResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListChainLocalOverridesResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListChainLocalOverridesResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ListChainLocalOverridesResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ListChainLocalOverridesResponse) GetBody() *ListChainLocalOverridesResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ListChainLocalOverridesResponse) SetBody(v *ListChainLocalOverridesResponse_Body) { + x.Body = v +} +func (x *ListChainLocalOverridesResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ListChainLocalOverridesResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListChainLocalOverridesResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListChainLocalOverridesResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListChainLocalOverridesResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListChainLocalOverridesResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ListChainLocalOverridesResponse_Body + f = new(ListChainLocalOverridesResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListTargetsLocalOverridesRequest_Body struct { + ChainName string `json:"chainName"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListTargetsLocalOverridesRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ListTargetsLocalOverridesRequest_Body)(nil) + _ json.Marshaler = (*ListTargetsLocalOverridesRequest_Body)(nil) + _ json.Unmarshaler = (*ListTargetsLocalOverridesRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2920,26 +11576,125 @@ func (x *ListTargetsLocalOverridesRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListTargetsLocalOverridesRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.StringMarshal(1, buf[offset:], x.ChainName) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListTargetsLocalOverridesRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListTargetsLocalOverridesRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ChainName) != 0 { + mm.AppendString(1, x.ChainName) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListTargetsLocalOverridesRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListTargetsLocalOverridesRequest_Body") + } + switch fc.FieldNum { + case 1: // ChainName + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ChainName") + } + x.ChainName = data + } + } + return nil +} +func (x *ListTargetsLocalOverridesRequest_Body) GetChainName() string { + if x != nil { + return x.ChainName + } + return "" +} +func (x *ListTargetsLocalOverridesRequest_Body) SetChainName(v string) { + x.ChainName = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListTargetsLocalOverridesRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListTargetsLocalOverridesRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"chainName\":" + out.RawString(prefix[1:]) + out.String(x.ChainName) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListTargetsLocalOverridesRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListTargetsLocalOverridesRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "chainName": + { + var f string + f = in.String() + x.ChainName = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListTargetsLocalOverridesRequest struct { + Body *ListTargetsLocalOverridesRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListTargetsLocalOverridesRequest)(nil) + _ encoding.ProtoUnmarshaler = (*ListTargetsLocalOverridesRequest)(nil) + _ json.Marshaler = (*ListTargetsLocalOverridesRequest)(nil) + _ json.Unmarshaler = (*ListTargetsLocalOverridesRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -2952,27 +11707,6 @@ func (x *ListTargetsLocalOverridesRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListTargetsLocalOverridesRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -2988,13 +11722,164 @@ func (x *ListTargetsLocalOverridesRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *ListTargetsLocalOverridesRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *ListTargetsLocalOverridesRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListTargetsLocalOverridesRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListTargetsLocalOverridesRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListTargetsLocalOverridesRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListTargetsLocalOverridesRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ListTargetsLocalOverridesRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ListTargetsLocalOverridesRequest) GetBody() *ListTargetsLocalOverridesRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ListTargetsLocalOverridesRequest) SetBody(v *ListTargetsLocalOverridesRequest_Body) { + x.Body = v +} +func (x *ListTargetsLocalOverridesRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ListTargetsLocalOverridesRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListTargetsLocalOverridesRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListTargetsLocalOverridesRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListTargetsLocalOverridesRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListTargetsLocalOverridesRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ListTargetsLocalOverridesRequest_Body + f = new(ListTargetsLocalOverridesRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListTargetsLocalOverridesResponse_Body struct { + Targets []*ChainTarget `json:"targets"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListTargetsLocalOverridesResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ListTargetsLocalOverridesResponse_Body)(nil) + _ json.Marshaler = (*ListTargetsLocalOverridesResponse_Body)(nil) + _ json.Unmarshaler = (*ListTargetsLocalOverridesResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3008,28 +11893,146 @@ func (x *ListTargetsLocalOverridesResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListTargetsLocalOverridesResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - for i := range x.Targets { - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Targets[i]) - } - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListTargetsLocalOverridesResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListTargetsLocalOverridesResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for i := range x.Targets { + if x.Targets[i] != nil && x.Targets[i].StableSize() != 0 { + x.Targets[i].EmitProtobuf(mm.AppendMessage(1)) + } + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListTargetsLocalOverridesResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListTargetsLocalOverridesResponse_Body") + } + switch fc.FieldNum { + case 1: // Targets + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Targets") + } + x.Targets = append(x.Targets, new(ChainTarget)) + ff := x.Targets[len(x.Targets)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ListTargetsLocalOverridesResponse_Body) GetTargets() []*ChainTarget { + if x != nil { + return x.Targets + } + return nil +} +func (x *ListTargetsLocalOverridesResponse_Body) SetTargets(v []*ChainTarget) { + x.Targets = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListTargetsLocalOverridesResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListTargetsLocalOverridesResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"targets\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Targets { + if i != 0 { + out.RawByte(',') + } + x.Targets[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListTargetsLocalOverridesResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListTargetsLocalOverridesResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "targets": + { + var f *ChainTarget + var list []*ChainTarget + in.Delim('[') + for !in.IsDelim(']') { + f = new(ChainTarget) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Targets = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListTargetsLocalOverridesResponse struct { + Body *ListTargetsLocalOverridesResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListTargetsLocalOverridesResponse)(nil) + _ encoding.ProtoUnmarshaler = (*ListTargetsLocalOverridesResponse)(nil) + _ json.Marshaler = (*ListTargetsLocalOverridesResponse)(nil) + _ json.Unmarshaler = (*ListTargetsLocalOverridesResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3042,27 +12045,6 @@ func (x *ListTargetsLocalOverridesResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ListTargetsLocalOverridesResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -3078,13 +12060,165 @@ func (x *ListTargetsLocalOverridesResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *ListTargetsLocalOverridesResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *ListTargetsLocalOverridesResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListTargetsLocalOverridesResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ListTargetsLocalOverridesResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListTargetsLocalOverridesResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListTargetsLocalOverridesResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ListTargetsLocalOverridesResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ListTargetsLocalOverridesResponse) GetBody() *ListTargetsLocalOverridesResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ListTargetsLocalOverridesResponse) SetBody(v *ListTargetsLocalOverridesResponse_Body) { + x.Body = v +} +func (x *ListTargetsLocalOverridesResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ListTargetsLocalOverridesResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListTargetsLocalOverridesResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListTargetsLocalOverridesResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListTargetsLocalOverridesResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListTargetsLocalOverridesResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ListTargetsLocalOverridesResponse_Body + f = new(ListTargetsLocalOverridesResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveChainLocalOverrideRequest_Body struct { + Target *ChainTarget `json:"target"` + ChainId []byte `json:"chainId"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveChainLocalOverrideRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveChainLocalOverrideRequest_Body)(nil) + _ json.Marshaler = (*RemoveChainLocalOverrideRequest_Body)(nil) + _ json.Unmarshaler = (*RemoveChainLocalOverrideRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3097,27 +12231,158 @@ func (x *RemoveChainLocalOverrideRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveChainLocalOverrideRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) - offset += proto.BytesMarshal(2, buf[offset:], x.ChainId) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveChainLocalOverrideRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveChainLocalOverrideRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Target != nil && x.Target.StableSize() != 0 { + x.Target.EmitProtobuf(mm.AppendMessage(1)) + } + if len(x.ChainId) != 0 { + mm.AppendBytes(2, x.ChainId) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveChainLocalOverrideRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveChainLocalOverrideRequest_Body") + } + switch fc.FieldNum { + case 1: // Target + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Target") + } + x.Target = new(ChainTarget) + if err := x.Target.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // ChainId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ChainId") + } + x.ChainId = data + } + } + return nil +} +func (x *RemoveChainLocalOverrideRequest_Body) GetTarget() *ChainTarget { + if x != nil { + return x.Target + } + return nil +} +func (x *RemoveChainLocalOverrideRequest_Body) SetTarget(v *ChainTarget) { + x.Target = v +} +func (x *RemoveChainLocalOverrideRequest_Body) GetChainId() []byte { + if x != nil { + return x.ChainId + } + return nil +} +func (x *RemoveChainLocalOverrideRequest_Body) SetChainId(v []byte) { + x.ChainId = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveChainLocalOverrideRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveChainLocalOverrideRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"target\":" + out.RawString(prefix[1:]) + x.Target.MarshalEasyJSON(out) + } + { + const prefix string = ",\"chainId\":" + out.RawString(prefix) + out.Base64Bytes(x.ChainId) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveChainLocalOverrideRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveChainLocalOverrideRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "target": + { + var f *ChainTarget + f = new(ChainTarget) + f.UnmarshalEasyJSON(in) + x.Target = f + } + case "chainId": + { + var f []byte + f = in.Bytes() + x.ChainId = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveChainLocalOverrideRequest struct { + Body *RemoveChainLocalOverrideRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveChainLocalOverrideRequest)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveChainLocalOverrideRequest)(nil) + _ json.Marshaler = (*RemoveChainLocalOverrideRequest)(nil) + _ json.Unmarshaler = (*RemoveChainLocalOverrideRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3130,27 +12395,6 @@ func (x *RemoveChainLocalOverrideRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveChainLocalOverrideRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -3166,13 +12410,163 @@ func (x *RemoveChainLocalOverrideRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *RemoveChainLocalOverrideRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *RemoveChainLocalOverrideRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveChainLocalOverrideRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveChainLocalOverrideRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveChainLocalOverrideRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveChainLocalOverrideRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(RemoveChainLocalOverrideRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *RemoveChainLocalOverrideRequest) GetBody() *RemoveChainLocalOverrideRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *RemoveChainLocalOverrideRequest) SetBody(v *RemoveChainLocalOverrideRequest_Body) { + x.Body = v +} +func (x *RemoveChainLocalOverrideRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *RemoveChainLocalOverrideRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveChainLocalOverrideRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveChainLocalOverrideRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveChainLocalOverrideRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveChainLocalOverrideRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *RemoveChainLocalOverrideRequest_Body + f = new(RemoveChainLocalOverrideRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveChainLocalOverrideResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveChainLocalOverrideResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveChainLocalOverrideResponse_Body)(nil) + _ json.Marshaler = (*RemoveChainLocalOverrideResponse_Body)(nil) + _ json.Unmarshaler = (*RemoveChainLocalOverrideResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3183,18 +12577,93 @@ func (x *RemoveChainLocalOverrideResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveChainLocalOverrideResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveChainLocalOverrideResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveChainLocalOverrideResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveChainLocalOverrideResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveChainLocalOverrideResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveChainLocalOverrideResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveChainLocalOverrideResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveChainLocalOverrideResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveChainLocalOverrideResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveChainLocalOverrideResponse struct { + Body *RemoveChainLocalOverrideResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveChainLocalOverrideResponse)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveChainLocalOverrideResponse)(nil) + _ json.Marshaler = (*RemoveChainLocalOverrideResponse)(nil) + _ json.Unmarshaler = (*RemoveChainLocalOverrideResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3207,27 +12676,6 @@ func (x *RemoveChainLocalOverrideResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveChainLocalOverrideResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -3243,13 +12691,164 @@ func (x *RemoveChainLocalOverrideResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *RemoveChainLocalOverrideResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *RemoveChainLocalOverrideResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveChainLocalOverrideResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveChainLocalOverrideResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveChainLocalOverrideResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveChainLocalOverrideResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(RemoveChainLocalOverrideResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *RemoveChainLocalOverrideResponse) GetBody() *RemoveChainLocalOverrideResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *RemoveChainLocalOverrideResponse) SetBody(v *RemoveChainLocalOverrideResponse_Body) { + x.Body = v +} +func (x *RemoveChainLocalOverrideResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *RemoveChainLocalOverrideResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveChainLocalOverrideResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveChainLocalOverrideResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveChainLocalOverrideResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveChainLocalOverrideResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *RemoveChainLocalOverrideResponse_Body + f = new(RemoveChainLocalOverrideResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveChainLocalOverridesByTargetRequest_Body struct { + Target *ChainTarget `json:"target"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveChainLocalOverridesByTargetRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveChainLocalOverridesByTargetRequest_Body)(nil) + _ json.Marshaler = (*RemoveChainLocalOverridesByTargetRequest_Body)(nil) + _ json.Unmarshaler = (*RemoveChainLocalOverridesByTargetRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3261,26 +12860,129 @@ func (x *RemoveChainLocalOverridesByTargetRequest_Body) StableSize() (size int) return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveChainLocalOverridesByTargetRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Target) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveChainLocalOverridesByTargetRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveChainLocalOverridesByTargetRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Target != nil && x.Target.StableSize() != 0 { + x.Target.EmitProtobuf(mm.AppendMessage(1)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveChainLocalOverridesByTargetRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveChainLocalOverridesByTargetRequest_Body") + } + switch fc.FieldNum { + case 1: // Target + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Target") + } + x.Target = new(ChainTarget) + if err := x.Target.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *RemoveChainLocalOverridesByTargetRequest_Body) GetTarget() *ChainTarget { + if x != nil { + return x.Target + } + return nil +} +func (x *RemoveChainLocalOverridesByTargetRequest_Body) SetTarget(v *ChainTarget) { + x.Target = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveChainLocalOverridesByTargetRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveChainLocalOverridesByTargetRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"target\":" + out.RawString(prefix[1:]) + x.Target.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveChainLocalOverridesByTargetRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveChainLocalOverridesByTargetRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "target": + { + var f *ChainTarget + f = new(ChainTarget) + f.UnmarshalEasyJSON(in) + x.Target = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveChainLocalOverridesByTargetRequest struct { + Body *RemoveChainLocalOverridesByTargetRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveChainLocalOverridesByTargetRequest)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveChainLocalOverridesByTargetRequest)(nil) + _ json.Marshaler = (*RemoveChainLocalOverridesByTargetRequest)(nil) + _ json.Unmarshaler = (*RemoveChainLocalOverridesByTargetRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3293,27 +12995,6 @@ func (x *RemoveChainLocalOverridesByTargetRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveChainLocalOverridesByTargetRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -3329,13 +13010,163 @@ func (x *RemoveChainLocalOverridesByTargetRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *RemoveChainLocalOverridesByTargetRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *RemoveChainLocalOverridesByTargetRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveChainLocalOverridesByTargetRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveChainLocalOverridesByTargetRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveChainLocalOverridesByTargetRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveChainLocalOverridesByTargetRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(RemoveChainLocalOverridesByTargetRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *RemoveChainLocalOverridesByTargetRequest) GetBody() *RemoveChainLocalOverridesByTargetRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *RemoveChainLocalOverridesByTargetRequest) SetBody(v *RemoveChainLocalOverridesByTargetRequest_Body) { + x.Body = v +} +func (x *RemoveChainLocalOverridesByTargetRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *RemoveChainLocalOverridesByTargetRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveChainLocalOverridesByTargetRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveChainLocalOverridesByTargetRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveChainLocalOverridesByTargetRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveChainLocalOverridesByTargetRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *RemoveChainLocalOverridesByTargetRequest_Body + f = new(RemoveChainLocalOverridesByTargetRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveChainLocalOverridesByTargetResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveChainLocalOverridesByTargetResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveChainLocalOverridesByTargetResponse_Body)(nil) + _ json.Marshaler = (*RemoveChainLocalOverridesByTargetResponse_Body)(nil) + _ json.Unmarshaler = (*RemoveChainLocalOverridesByTargetResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3346,18 +13177,93 @@ func (x *RemoveChainLocalOverridesByTargetResponse_Body) StableSize() (size int) return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveChainLocalOverridesByTargetResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveChainLocalOverridesByTargetResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveChainLocalOverridesByTargetResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveChainLocalOverridesByTargetResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveChainLocalOverridesByTargetResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveChainLocalOverridesByTargetResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveChainLocalOverridesByTargetResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveChainLocalOverridesByTargetResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveChainLocalOverridesByTargetResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveChainLocalOverridesByTargetResponse struct { + Body *RemoveChainLocalOverridesByTargetResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveChainLocalOverridesByTargetResponse)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveChainLocalOverridesByTargetResponse)(nil) + _ json.Marshaler = (*RemoveChainLocalOverridesByTargetResponse)(nil) + _ json.Unmarshaler = (*RemoveChainLocalOverridesByTargetResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3370,27 +13276,6 @@ func (x *RemoveChainLocalOverridesByTargetResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveChainLocalOverridesByTargetResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -3406,13 +13291,168 @@ func (x *RemoveChainLocalOverridesByTargetResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *RemoveChainLocalOverridesByTargetResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *RemoveChainLocalOverridesByTargetResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveChainLocalOverridesByTargetResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveChainLocalOverridesByTargetResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveChainLocalOverridesByTargetResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveChainLocalOverridesByTargetResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(RemoveChainLocalOverridesByTargetResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *RemoveChainLocalOverridesByTargetResponse) GetBody() *RemoveChainLocalOverridesByTargetResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *RemoveChainLocalOverridesByTargetResponse) SetBody(v *RemoveChainLocalOverridesByTargetResponse_Body) { + x.Body = v +} +func (x *RemoveChainLocalOverridesByTargetResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *RemoveChainLocalOverridesByTargetResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveChainLocalOverridesByTargetResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveChainLocalOverridesByTargetResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveChainLocalOverridesByTargetResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveChainLocalOverridesByTargetResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *RemoveChainLocalOverridesByTargetResponse_Body + f = new(RemoveChainLocalOverridesByTargetResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SealWriteCacheRequest_Body struct { + Shard_ID [][]byte `json:"shardID"` + IgnoreErrors bool `json:"ignoreErrors"` + Async bool `json:"async"` + RestoreMode bool `json:"restoreMode"` + Shrink bool `json:"shrink"` +} + +var ( + _ encoding.ProtoMarshaler = (*SealWriteCacheRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*SealWriteCacheRequest_Body)(nil) + _ json.Marshaler = (*SealWriteCacheRequest_Body)(nil) + _ json.Unmarshaler = (*SealWriteCacheRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3428,30 +13468,255 @@ func (x *SealWriteCacheRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SealWriteCacheRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) - offset += proto.BoolMarshal(2, buf[offset:], x.IgnoreErrors) - offset += proto.BoolMarshal(3, buf[offset:], x.Async) - offset += proto.BoolMarshal(4, buf[offset:], x.RestoreMode) - offset += proto.BoolMarshal(5, buf[offset:], x.Shrink) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SealWriteCacheRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SealWriteCacheRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for j := range x.Shard_ID { + mm.AppendBytes(1, x.Shard_ID[j]) + } + if x.IgnoreErrors { + mm.AppendBool(2, x.IgnoreErrors) + } + if x.Async { + mm.AppendBool(3, x.Async) + } + if x.RestoreMode { + mm.AppendBool(4, x.RestoreMode) + } + if x.Shrink { + mm.AppendBool(5, x.Shrink) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SealWriteCacheRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SealWriteCacheRequest_Body") + } + switch fc.FieldNum { + case 1: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = append(x.Shard_ID, data) + case 2: // IgnoreErrors + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "IgnoreErrors") + } + x.IgnoreErrors = data + case 3: // Async + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Async") + } + x.Async = data + case 4: // RestoreMode + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "RestoreMode") + } + x.RestoreMode = data + case 5: // Shrink + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shrink") + } + x.Shrink = data + } + } + return nil +} +func (x *SealWriteCacheRequest_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *SealWriteCacheRequest_Body) SetShard_ID(v [][]byte) { + x.Shard_ID = v +} +func (x *SealWriteCacheRequest_Body) GetIgnoreErrors() bool { + if x != nil { + return x.IgnoreErrors + } + return false +} +func (x *SealWriteCacheRequest_Body) SetIgnoreErrors(v bool) { + x.IgnoreErrors = v +} +func (x *SealWriteCacheRequest_Body) GetAsync() bool { + if x != nil { + return x.Async + } + return false +} +func (x *SealWriteCacheRequest_Body) SetAsync(v bool) { + x.Async = v +} +func (x *SealWriteCacheRequest_Body) GetRestoreMode() bool { + if x != nil { + return x.RestoreMode + } + return false +} +func (x *SealWriteCacheRequest_Body) SetRestoreMode(v bool) { + x.RestoreMode = v +} +func (x *SealWriteCacheRequest_Body) GetShrink() bool { + if x != nil { + return x.Shrink + } + return false +} +func (x *SealWriteCacheRequest_Body) SetShrink(v bool) { + x.Shrink = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SealWriteCacheRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SealWriteCacheRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"shardID\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Shard_ID { + if i != 0 { + out.RawByte(',') + } + out.Base64Bytes(x.Shard_ID[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"ignoreErrors\":" + out.RawString(prefix) + out.Bool(x.IgnoreErrors) + } + { + const prefix string = ",\"async\":" + out.RawString(prefix) + out.Bool(x.Async) + } + { + const prefix string = ",\"restoreMode\":" + out.RawString(prefix) + out.Bool(x.RestoreMode) + } + { + const prefix string = ",\"shrink\":" + out.RawString(prefix) + out.Bool(x.Shrink) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SealWriteCacheRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SealWriteCacheRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shardID": + { + var f []byte + var list [][]byte + in.Delim('[') + for !in.IsDelim(']') { + f = in.Bytes() + list = append(list, f) + in.WantComma() + } + x.Shard_ID = list + in.Delim(']') + } + case "ignoreErrors": + { + var f bool + f = in.Bool() + x.IgnoreErrors = f + } + case "async": + { + var f bool + f = in.Bool() + x.Async = f + } + case "restoreMode": + { + var f bool + f = in.Bool() + x.RestoreMode = f + } + case "shrink": + { + var f bool + f = in.Bool() + x.Shrink = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SealWriteCacheRequest struct { + Body *SealWriteCacheRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*SealWriteCacheRequest)(nil) + _ encoding.ProtoUnmarshaler = (*SealWriteCacheRequest)(nil) + _ json.Marshaler = (*SealWriteCacheRequest)(nil) + _ json.Unmarshaler = (*SealWriteCacheRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3464,27 +13729,6 @@ func (x *SealWriteCacheRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SealWriteCacheRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -3500,13 +13744,166 @@ func (x *SealWriteCacheRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *SealWriteCacheRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *SealWriteCacheRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SealWriteCacheRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SealWriteCacheRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SealWriteCacheRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SealWriteCacheRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(SealWriteCacheRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *SealWriteCacheRequest) GetBody() *SealWriteCacheRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *SealWriteCacheRequest) SetBody(v *SealWriteCacheRequest_Body) { + x.Body = v +} +func (x *SealWriteCacheRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *SealWriteCacheRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SealWriteCacheRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SealWriteCacheRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SealWriteCacheRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SealWriteCacheRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *SealWriteCacheRequest_Body + f = new(SealWriteCacheRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SealWriteCacheResponse_Body_Status struct { + Shard_ID []byte `json:"shardID"` + Success bool `json:"success"` + Error string `json:"error"` +} + +var ( + _ encoding.ProtoMarshaler = (*SealWriteCacheResponse_Body_Status)(nil) + _ encoding.ProtoUnmarshaler = (*SealWriteCacheResponse_Body_Status)(nil) + _ json.Marshaler = (*SealWriteCacheResponse_Body_Status)(nil) + _ json.Unmarshaler = (*SealWriteCacheResponse_Body_Status)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3520,28 +13917,182 @@ func (x *SealWriteCacheResponse_Body_Status) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SealWriteCacheResponse_Body_Status) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.Shard_ID) - offset += proto.BoolMarshal(2, buf[offset:], x.Success) - offset += proto.StringMarshal(3, buf[offset:], x.Error) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SealWriteCacheResponse_Body_Status) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SealWriteCacheResponse_Body_Status) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Shard_ID) != 0 { + mm.AppendBytes(1, x.Shard_ID) + } + if x.Success { + mm.AppendBool(2, x.Success) + } + if len(x.Error) != 0 { + mm.AppendString(3, x.Error) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SealWriteCacheResponse_Body_Status) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SealWriteCacheResponse_Body_Status") + } + switch fc.FieldNum { + case 1: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = data + case 2: // Success + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Success") + } + x.Success = data + case 3: // Error + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Error") + } + x.Error = data + } + } + return nil +} +func (x *SealWriteCacheResponse_Body_Status) GetShard_ID() []byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *SealWriteCacheResponse_Body_Status) SetShard_ID(v []byte) { + x.Shard_ID = v +} +func (x *SealWriteCacheResponse_Body_Status) GetSuccess() bool { + if x != nil { + return x.Success + } + return false +} +func (x *SealWriteCacheResponse_Body_Status) SetSuccess(v bool) { + x.Success = v +} +func (x *SealWriteCacheResponse_Body_Status) GetError() string { + if x != nil { + return x.Error + } + return "" +} +func (x *SealWriteCacheResponse_Body_Status) SetError(v string) { + x.Error = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SealWriteCacheResponse_Body_Status) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SealWriteCacheResponse_Body_Status) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"shardID\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.Shard_ID) + } + { + const prefix string = ",\"success\":" + out.RawString(prefix) + out.Bool(x.Success) + } + { + const prefix string = ",\"error\":" + out.RawString(prefix) + out.String(x.Error) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SealWriteCacheResponse_Body_Status) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SealWriteCacheResponse_Body_Status) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shardID": + { + var f []byte + f = in.Bytes() + x.Shard_ID = f + } + case "success": + { + var f bool + f = in.Bool() + x.Success = f + } + case "error": + { + var f string + f = in.String() + x.Error = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SealWriteCacheResponse_Body struct { + Results []*SealWriteCacheResponse_Body_Status `json:"results"` +} + +var ( + _ encoding.ProtoMarshaler = (*SealWriteCacheResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*SealWriteCacheResponse_Body)(nil) + _ json.Marshaler = (*SealWriteCacheResponse_Body)(nil) + _ json.Unmarshaler = (*SealWriteCacheResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3555,28 +14106,146 @@ func (x *SealWriteCacheResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SealWriteCacheResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - for i := range x.Results { - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Results[i]) - } - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SealWriteCacheResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SealWriteCacheResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for i := range x.Results { + if x.Results[i] != nil && x.Results[i].StableSize() != 0 { + x.Results[i].EmitProtobuf(mm.AppendMessage(1)) + } + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SealWriteCacheResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SealWriteCacheResponse_Body") + } + switch fc.FieldNum { + case 1: // Results + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Results") + } + x.Results = append(x.Results, new(SealWriteCacheResponse_Body_Status)) + ff := x.Results[len(x.Results)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *SealWriteCacheResponse_Body) GetResults() []*SealWriteCacheResponse_Body_Status { + if x != nil { + return x.Results + } + return nil +} +func (x *SealWriteCacheResponse_Body) SetResults(v []*SealWriteCacheResponse_Body_Status) { + x.Results = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SealWriteCacheResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SealWriteCacheResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"results\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Results { + if i != 0 { + out.RawByte(',') + } + x.Results[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SealWriteCacheResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SealWriteCacheResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "results": + { + var f *SealWriteCacheResponse_Body_Status + var list []*SealWriteCacheResponse_Body_Status + in.Delim('[') + for !in.IsDelim(']') { + f = new(SealWriteCacheResponse_Body_Status) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Results = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type SealWriteCacheResponse struct { + Body *SealWriteCacheResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*SealWriteCacheResponse)(nil) + _ encoding.ProtoUnmarshaler = (*SealWriteCacheResponse)(nil) + _ json.Marshaler = (*SealWriteCacheResponse)(nil) + _ json.Unmarshaler = (*SealWriteCacheResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3589,27 +14258,6 @@ func (x *SealWriteCacheResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *SealWriteCacheResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -3625,13 +14273,164 @@ func (x *SealWriteCacheResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *SealWriteCacheResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *SealWriteCacheResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *SealWriteCacheResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *SealWriteCacheResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *SealWriteCacheResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "SealWriteCacheResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(SealWriteCacheResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *SealWriteCacheResponse) GetBody() *SealWriteCacheResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *SealWriteCacheResponse) SetBody(v *SealWriteCacheResponse_Body) { + x.Body = v +} +func (x *SealWriteCacheResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *SealWriteCacheResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *SealWriteCacheResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *SealWriteCacheResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *SealWriteCacheResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *SealWriteCacheResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *SealWriteCacheResponse_Body + f = new(SealWriteCacheResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DetachShardsRequest_Body struct { + Shard_ID [][]byte `json:"shardID"` +} + +var ( + _ encoding.ProtoMarshaler = (*DetachShardsRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*DetachShardsRequest_Body)(nil) + _ json.Marshaler = (*DetachShardsRequest_Body)(nil) + _ json.Unmarshaler = (*DetachShardsRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3643,26 +14442,139 @@ func (x *DetachShardsRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DetachShardsRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.RepeatedBytesMarshal(1, buf[offset:], x.Shard_ID) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DetachShardsRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *DetachShardsRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for j := range x.Shard_ID { + mm.AppendBytes(1, x.Shard_ID[j]) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DetachShardsRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DetachShardsRequest_Body") + } + switch fc.FieldNum { + case 1: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = append(x.Shard_ID, data) + } + } + return nil +} +func (x *DetachShardsRequest_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *DetachShardsRequest_Body) SetShard_ID(v [][]byte) { + x.Shard_ID = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DetachShardsRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DetachShardsRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"shardID\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Shard_ID { + if i != 0 { + out.RawByte(',') + } + out.Base64Bytes(x.Shard_ID[i]) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DetachShardsRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DetachShardsRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shardID": + { + var f []byte + var list [][]byte + in.Delim('[') + for !in.IsDelim(']') { + f = in.Bytes() + list = append(list, f) + in.WantComma() + } + x.Shard_ID = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DetachShardsRequest struct { + Body *DetachShardsRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*DetachShardsRequest)(nil) + _ encoding.ProtoUnmarshaler = (*DetachShardsRequest)(nil) + _ json.Marshaler = (*DetachShardsRequest)(nil) + _ json.Unmarshaler = (*DetachShardsRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3675,27 +14587,6 @@ func (x *DetachShardsRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DetachShardsRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -3711,13 +14602,163 @@ func (x *DetachShardsRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *DetachShardsRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *DetachShardsRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DetachShardsRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *DetachShardsRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DetachShardsRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DetachShardsRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(DetachShardsRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *DetachShardsRequest) GetBody() *DetachShardsRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *DetachShardsRequest) SetBody(v *DetachShardsRequest_Body) { + x.Body = v +} +func (x *DetachShardsRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *DetachShardsRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DetachShardsRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DetachShardsRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DetachShardsRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DetachShardsRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *DetachShardsRequest_Body + f = new(DetachShardsRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DetachShardsResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*DetachShardsResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*DetachShardsResponse_Body)(nil) + _ json.Marshaler = (*DetachShardsResponse_Body)(nil) + _ json.Unmarshaler = (*DetachShardsResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3728,18 +14769,93 @@ func (x *DetachShardsResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DetachShardsResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DetachShardsResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *DetachShardsResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DetachShardsResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DetachShardsResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DetachShardsResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DetachShardsResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DetachShardsResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DetachShardsResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type DetachShardsResponse struct { + Body *DetachShardsResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*DetachShardsResponse)(nil) + _ encoding.ProtoUnmarshaler = (*DetachShardsResponse)(nil) + _ json.Marshaler = (*DetachShardsResponse)(nil) + _ json.Unmarshaler = (*DetachShardsResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -3752,27 +14868,6 @@ func (x *DetachShardsResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *DetachShardsResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -3788,9 +14883,149 @@ func (x *DetachShardsResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *DetachShardsResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *DetachShardsResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *DetachShardsResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *DetachShardsResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *DetachShardsResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "DetachShardsResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(DetachShardsResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *DetachShardsResponse) GetBody() *DetachShardsResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *DetachShardsResponse) SetBody(v *DetachShardsResponse_Body) { + x.Body = v +} +func (x *DetachShardsResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *DetachShardsResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *DetachShardsResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *DetachShardsResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *DetachShardsResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *DetachShardsResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *DetachShardsResponse_Body + f = new(DetachShardsResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } } diff --git a/pkg/services/control/service_test.go b/pkg/services/control/service_test.go deleted file mode 100644 index 1d98cc6f1..000000000 --- a/pkg/services/control/service_test.go +++ /dev/null @@ -1,181 +0,0 @@ -package control_test - -import ( - "bytes" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" -) - -func TestHealthCheckResponse_Body_StableMarshal(t *testing.T) { - testStableMarshal(t, - generateHealthCheckResponseBody(), - new(control.HealthCheckResponse_Body), - func(m1, m2 protoMessage) bool { - return equalHealthCheckResponseBodies( - m1.(*control.HealthCheckResponse_Body), - m2.(*control.HealthCheckResponse_Body), - ) - }, - ) -} - -func generateHealthCheckResponseBody() *control.HealthCheckResponse_Body { - body := new(control.HealthCheckResponse_Body) - body.SetNetmapStatus(control.NetmapStatus_ONLINE) - body.SetHealthStatus(control.HealthStatus_SHUTTING_DOWN) - - return body -} - -func equalHealthCheckResponseBodies(b1, b2 *control.HealthCheckResponse_Body) bool { - return b1.GetNetmapStatus() == b2.GetNetmapStatus() && - b1.GetHealthStatus() == b2.GetHealthStatus() -} - -func TestSetNetmapStatusRequest_Body_StableMarshal(t *testing.T) { - testStableMarshal(t, - generateSetNetmapStatusRequestBody(), - new(control.SetNetmapStatusRequest_Body), - func(m1, m2 protoMessage) bool { - return equalSetnetmapStatusRequestBodies( - m1.(*control.SetNetmapStatusRequest_Body), - m2.(*control.SetNetmapStatusRequest_Body), - ) - }, - ) -} - -func generateSetNetmapStatusRequestBody() *control.SetNetmapStatusRequest_Body { - body := new(control.SetNetmapStatusRequest_Body) - body.SetStatus(control.NetmapStatus_ONLINE) - - return body -} - -func equalSetnetmapStatusRequestBodies(b1, b2 *control.SetNetmapStatusRequest_Body) bool { - return b1.GetStatus() == b2.GetStatus() -} - -func TestListShardsResponse_Body_StableMarshal(t *testing.T) { - testStableMarshal(t, - generateListShardsResponseBody(), - new(control.ListShardsResponse_Body), - func(m1, m2 protoMessage) bool { - return equalListShardResponseBodies( - m1.(*control.ListShardsResponse_Body), - m2.(*control.ListShardsResponse_Body), - ) - }, - ) -} - -func equalListShardResponseBodies(b1, b2 *control.ListShardsResponse_Body) bool { - if len(b1.Shards) != len(b2.Shards) { - return false - } - - for i := range b1.Shards { - if b1.Shards[i].GetMetabasePath() != b2.Shards[i].GetMetabasePath() || - b1.Shards[i].GetWritecachePath() != b2.Shards[i].GetWritecachePath() || - b1.Shards[i].GetPiloramaPath() != b2.Shards[i].GetPiloramaPath() || - !bytes.Equal(b1.Shards[i].GetShard_ID(), b2.Shards[i].GetShard_ID()) { - return false - } - - info1 := b1.Shards[i].GetBlobstor() - info2 := b2.Shards[i].GetBlobstor() - if !compareBlobstorInfo(info1, info2) { - return false - } - } - - for i := range b1.Shards { - for j := i + 1; j < len(b1.Shards); j++ { - if b1.Shards[i].GetMetabasePath() == b2.Shards[j].GetMetabasePath() || - !compareBlobstorInfo(b1.Shards[i].Blobstor, b2.Shards[i].Blobstor) || - b1.Shards[i].GetWritecachePath() == b2.Shards[j].GetWritecachePath() || - bytes.Equal(b1.Shards[i].GetShard_ID(), b2.Shards[j].GetShard_ID()) { - return false - } - } - } - - return true -} - -func compareBlobstorInfo(a, b []*control.BlobstorInfo) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i].Type != b[i].Type || - a[i].Path != b[i].Path { - return false - } - } - return true -} - -func generateListShardsResponseBody() *control.ListShardsResponse_Body { - body := new(control.ListShardsResponse_Body) - body.SetShards([]*control.ShardInfo{ - generateShardInfo(0), - generateShardInfo(1), - }) - - return body -} - -func TestSetShardModeRequest_Body_StableMarshal(t *testing.T) { - testStableMarshal(t, - generateSetShardModeRequestBody(), - new(control.SetShardModeRequest_Body), - func(m1, m2 protoMessage) bool { - return equalSetShardModeRequestBodies( - m1.(*control.SetShardModeRequest_Body), - m2.(*control.SetShardModeRequest_Body), - ) - }, - ) -} - -func generateSetShardModeRequestBody() *control.SetShardModeRequest_Body { - body := new(control.SetShardModeRequest_Body) - body.SetShardIDList([][]byte{{0, 1, 2, 3, 4}}) - body.SetMode(control.ShardMode_READ_WRITE) - - return body -} - -func equalSetShardModeRequestBodies(b1, b2 *control.SetShardModeRequest_Body) bool { - if b1.GetMode() != b2.GetMode() || len(b1.Shard_ID) != len(b2.Shard_ID) { - return false - } - - for i := range b1.Shard_ID { - if !bytes.Equal(b1.Shard_ID[i], b2.Shard_ID[i]) { - return false - } - } - - return true -} - -func TestSynchronizeTreeRequest_Body_StableMarshal(t *testing.T) { - testStableMarshal(t, - &control.SynchronizeTreeRequest_Body{ - ContainerId: []byte{1, 2, 3, 4, 5, 6, 7}, - TreeId: "someID", - Height: 42, - }, - new(control.SynchronizeTreeRequest_Body), - func(m1, m2 protoMessage) bool { - b1 := m1.(*control.SynchronizeTreeRequest_Body) - b2 := m2.(*control.SynchronizeTreeRequest_Body) - return bytes.Equal(b1.GetContainerId(), b2.GetContainerId()) && - b1.GetTreeId() == b2.GetTreeId() && - b1.GetHeight() == b2.GetHeight() - }, - ) -} diff --git a/pkg/services/control/types.go b/pkg/services/control/types.go deleted file mode 100644 index 94f681c55..000000000 --- a/pkg/services/control/types.go +++ /dev/null @@ -1,118 +0,0 @@ -package control - -import ( - "google.golang.org/protobuf/encoding/protojson" -) - -// SetKey sets public key used for signing. -func (x *Signature) SetKey(v []byte) { - if x != nil { - x.Key = v - } -} - -// SetSign sets binary signature. -func (x *Signature) SetSign(v []byte) { - if x != nil { - x.Sign = v - } -} - -// SetKey sets key of the node attribute. -func (x *NodeInfo_Attribute) SetKey(v string) { - if x != nil { - x.Key = v - } -} - -// SetValue sets value of the node attribute. -func (x *NodeInfo_Attribute) SetValue(v string) { - if x != nil { - x.Value = v - } -} - -// SetParents sets parent keys. -func (x *NodeInfo_Attribute) SetParents(v []string) { - if x != nil { - x.Parents = v - } -} - -// SetPublicKey sets public key of the FrostFS node in a binary format. -func (x *NodeInfo) SetPublicKey(v []byte) { - if x != nil { - x.PublicKey = v - } -} - -// SetAddresses sets ways to connect to a node. -func (x *NodeInfo) SetAddresses(v []string) { - if x != nil { - x.Addresses = v - } -} - -// SetAttributes sets attributes of the FrostFS Storage Node. -func (x *NodeInfo) SetAttributes(v []*NodeInfo_Attribute) { - if x != nil { - x.Attributes = v - } -} - -// SetState sets state of the FrostFS node. -func (x *NodeInfo) SetState(v NetmapStatus) { - if x != nil { - x.State = v - } -} - -// SetEpoch sets revision number of the network map. -func (x *Netmap) SetEpoch(v uint64) { - if x != nil { - x.Epoch = v - } -} - -// SetNodes sets nodes presented in network. -func (x *Netmap) SetNodes(v []*NodeInfo) { - if x != nil { - x.Nodes = v - } -} - -func (x *Netmap) MarshalJSON() ([]byte, error) { - return protojson.MarshalOptions{ - EmitUnpopulated: true, - }.Marshal(x) -} - -// SetID sets identificator of the shard. -func (x *ShardInfo) SetID(v []byte) { - x.Shard_ID = v -} - -// SetMetabasePath sets path to shard's metabase. -func (x *ShardInfo) SetMetabasePath(v string) { - x.MetabasePath = v -} - -// SetWriteCachePath sets path to shard's write-cache. -func (x *ShardInfo) SetWriteCachePath(v string) { - x.WritecachePath = v -} - -// SetPiloramaPath sets path to shard's pilorama. -func (x *ShardInfo) SetPiloramaPath(v string) { - x.PiloramaPath = v -} - -// SetMode sets path to shard's work mode. -func (x *ShardInfo) SetMode(v ShardMode) { - x.Mode = v -} - -// SetErrorCount sets shard's error counter. -func (x *ShardInfo) SetErrorCount(count uint32) { - x.ErrorCount = count -} diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go deleted file mode 100644 index 858755694..000000000 --- a/pkg/services/control/types.pb.go +++ /dev/null @@ -1,1011 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.33.0 -// protoc v4.25.0 -// source: pkg/services/control/types.proto - -package control - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// Status of the storage node in the FrostFS network map. -type NetmapStatus int32 - -const ( - // Undefined status, default value. - NetmapStatus_STATUS_UNDEFINED NetmapStatus = 0 - // Node is online. - NetmapStatus_ONLINE NetmapStatus = 1 - // Node is offline. - NetmapStatus_OFFLINE NetmapStatus = 2 - // Node is maintained by the owner. - NetmapStatus_MAINTENANCE NetmapStatus = 3 -) - -// Enum value maps for NetmapStatus. -var ( - NetmapStatus_name = map[int32]string{ - 0: "STATUS_UNDEFINED", - 1: "ONLINE", - 2: "OFFLINE", - 3: "MAINTENANCE", - } - NetmapStatus_value = map[string]int32{ - "STATUS_UNDEFINED": 0, - "ONLINE": 1, - "OFFLINE": 2, - "MAINTENANCE": 3, - } -) - -func (x NetmapStatus) Enum() *NetmapStatus { - p := new(NetmapStatus) - *p = x - return p -} - -func (x NetmapStatus) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (NetmapStatus) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_services_control_types_proto_enumTypes[0].Descriptor() -} - -func (NetmapStatus) Type() protoreflect.EnumType { - return &file_pkg_services_control_types_proto_enumTypes[0] -} - -func (x NetmapStatus) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use NetmapStatus.Descriptor instead. -func (NetmapStatus) EnumDescriptor() ([]byte, []int) { - return file_pkg_services_control_types_proto_rawDescGZIP(), []int{0} -} - -// Health status of the storage node application. -type HealthStatus int32 - -const ( - // Undefined status, default value. - HealthStatus_HEALTH_STATUS_UNDEFINED HealthStatus = 0 - // Storage node application is starting. - HealthStatus_STARTING HealthStatus = 1 - // Storage node application is started and serves all services. - HealthStatus_READY HealthStatus = 2 - // Storage node application is shutting down. - HealthStatus_SHUTTING_DOWN HealthStatus = 3 - // Storage node application is reconfiguring. - HealthStatus_RECONFIGURING HealthStatus = 4 -) - -// Enum value maps for HealthStatus. -var ( - HealthStatus_name = map[int32]string{ - 0: "HEALTH_STATUS_UNDEFINED", - 1: "STARTING", - 2: "READY", - 3: "SHUTTING_DOWN", - 4: "RECONFIGURING", - } - HealthStatus_value = map[string]int32{ - "HEALTH_STATUS_UNDEFINED": 0, - "STARTING": 1, - "READY": 2, - "SHUTTING_DOWN": 3, - "RECONFIGURING": 4, - } -) - -func (x HealthStatus) Enum() *HealthStatus { - p := new(HealthStatus) - *p = x - return p -} - -func (x HealthStatus) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (HealthStatus) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_services_control_types_proto_enumTypes[1].Descriptor() -} - -func (HealthStatus) Type() protoreflect.EnumType { - return &file_pkg_services_control_types_proto_enumTypes[1] -} - -func (x HealthStatus) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use HealthStatus.Descriptor instead. -func (HealthStatus) EnumDescriptor() ([]byte, []int) { - return file_pkg_services_control_types_proto_rawDescGZIP(), []int{1} -} - -// Work mode of the shard. -type ShardMode int32 - -const ( - // Undefined mode, default value. - ShardMode_SHARD_MODE_UNDEFINED ShardMode = 0 - // Read-write. - ShardMode_READ_WRITE ShardMode = 1 - // Read-only. - ShardMode_READ_ONLY ShardMode = 2 - // Degraded. - ShardMode_DEGRADED ShardMode = 3 - // DegradedReadOnly. - ShardMode_DEGRADED_READ_ONLY ShardMode = 4 -) - -// Enum value maps for ShardMode. -var ( - ShardMode_name = map[int32]string{ - 0: "SHARD_MODE_UNDEFINED", - 1: "READ_WRITE", - 2: "READ_ONLY", - 3: "DEGRADED", - 4: "DEGRADED_READ_ONLY", - } - ShardMode_value = map[string]int32{ - "SHARD_MODE_UNDEFINED": 0, - "READ_WRITE": 1, - "READ_ONLY": 2, - "DEGRADED": 3, - "DEGRADED_READ_ONLY": 4, - } -) - -func (x ShardMode) Enum() *ShardMode { - p := new(ShardMode) - *p = x - return p -} - -func (x ShardMode) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (ShardMode) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_services_control_types_proto_enumTypes[2].Descriptor() -} - -func (ShardMode) Type() protoreflect.EnumType { - return &file_pkg_services_control_types_proto_enumTypes[2] -} - -func (x ShardMode) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use ShardMode.Descriptor instead. -func (ShardMode) EnumDescriptor() ([]byte, []int) { - return file_pkg_services_control_types_proto_rawDescGZIP(), []int{2} -} - -type ChainTarget_TargetType int32 - -const ( - ChainTarget_UNDEFINED ChainTarget_TargetType = 0 - ChainTarget_NAMESPACE ChainTarget_TargetType = 1 - ChainTarget_CONTAINER ChainTarget_TargetType = 2 - ChainTarget_USER ChainTarget_TargetType = 3 - ChainTarget_GROUP ChainTarget_TargetType = 4 -) - -// Enum value maps for ChainTarget_TargetType. -var ( - ChainTarget_TargetType_name = map[int32]string{ - 0: "UNDEFINED", - 1: "NAMESPACE", - 2: "CONTAINER", - 3: "USER", - 4: "GROUP", - } - ChainTarget_TargetType_value = map[string]int32{ - "UNDEFINED": 0, - "NAMESPACE": 1, - "CONTAINER": 2, - "USER": 3, - "GROUP": 4, - } -) - -func (x ChainTarget_TargetType) Enum() *ChainTarget_TargetType { - p := new(ChainTarget_TargetType) - *p = x - return p -} - -func (x ChainTarget_TargetType) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (ChainTarget_TargetType) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_services_control_types_proto_enumTypes[3].Descriptor() -} - -func (ChainTarget_TargetType) Type() protoreflect.EnumType { - return &file_pkg_services_control_types_proto_enumTypes[3] -} - -func (x ChainTarget_TargetType) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use ChainTarget_TargetType.Descriptor instead. -func (ChainTarget_TargetType) EnumDescriptor() ([]byte, []int) { - return file_pkg_services_control_types_proto_rawDescGZIP(), []int{5, 0} -} - -// Signature of some message. -type Signature struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Public key used for signing. - Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - // Binary signature. - Sign []byte `protobuf:"bytes,2,opt,name=sign,json=signature,proto3" json:"sign,omitempty"` -} - -func (x *Signature) Reset() { - *x = Signature{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_types_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Signature) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Signature) ProtoMessage() {} - -func (x *Signature) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_types_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Signature.ProtoReflect.Descriptor instead. -func (*Signature) Descriptor() ([]byte, []int) { - return file_pkg_services_control_types_proto_rawDescGZIP(), []int{0} -} - -func (x *Signature) GetKey() []byte { - if x != nil { - return x.Key - } - return nil -} - -func (x *Signature) GetSign() []byte { - if x != nil { - return x.Sign - } - return nil -} - -// FrostFS node description. -type NodeInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Public key of the FrostFS node in a binary format. - PublicKey []byte `protobuf:"bytes,1,opt,name=public_key,json=publicKey,proto3" json:"public_key,omitempty"` - // Ways to connect to a node. - Addresses []string `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses,omitempty"` - // Carries list of the FrostFS node attributes in a key-value form. Key name - // must be a node-unique valid UTF-8 string. Value can't be empty. NodeInfo - // structures with duplicated attribute names or attributes with empty values - // will be considered invalid. - Attributes []*NodeInfo_Attribute `protobuf:"bytes,3,rep,name=attributes,proto3" json:"attributes,omitempty"` - // Carries state of the FrostFS node. - State NetmapStatus `protobuf:"varint,4,opt,name=state,proto3,enum=control.NetmapStatus" json:"state,omitempty"` -} - -func (x *NodeInfo) Reset() { - *x = NodeInfo{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_types_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NodeInfo) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NodeInfo) ProtoMessage() {} - -func (x *NodeInfo) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_types_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NodeInfo.ProtoReflect.Descriptor instead. -func (*NodeInfo) Descriptor() ([]byte, []int) { - return file_pkg_services_control_types_proto_rawDescGZIP(), []int{1} -} - -func (x *NodeInfo) GetPublicKey() []byte { - if x != nil { - return x.PublicKey - } - return nil -} - -func (x *NodeInfo) GetAddresses() []string { - if x != nil { - return x.Addresses - } - return nil -} - -func (x *NodeInfo) GetAttributes() []*NodeInfo_Attribute { - if x != nil { - return x.Attributes - } - return nil -} - -func (x *NodeInfo) GetState() NetmapStatus { - if x != nil { - return x.State - } - return NetmapStatus_STATUS_UNDEFINED -} - -// Network map structure. -type Netmap struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Network map revision number. - Epoch uint64 `protobuf:"varint,1,opt,name=epoch,proto3" json:"epoch,omitempty"` - // Nodes presented in network. - Nodes []*NodeInfo `protobuf:"bytes,2,rep,name=nodes,proto3" json:"nodes,omitempty"` -} - -func (x *Netmap) Reset() { - *x = Netmap{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_types_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Netmap) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Netmap) ProtoMessage() {} - -func (x *Netmap) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_types_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Netmap.ProtoReflect.Descriptor instead. -func (*Netmap) Descriptor() ([]byte, []int) { - return file_pkg_services_control_types_proto_rawDescGZIP(), []int{2} -} - -func (x *Netmap) GetEpoch() uint64 { - if x != nil { - return x.Epoch - } - return 0 -} - -func (x *Netmap) GetNodes() []*NodeInfo { - if x != nil { - return x.Nodes - } - return nil -} - -// Shard description. -type ShardInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // ID of the shard. - Shard_ID []byte `protobuf:"bytes,1,opt,name=shard_ID,json=shardID,proto3" json:"shard_ID,omitempty"` - // Path to shard's metabase. - MetabasePath string `protobuf:"bytes,2,opt,name=metabase_path,json=metabasePath,proto3" json:"metabase_path,omitempty"` - // Shard's blobstor info. - Blobstor []*BlobstorInfo `protobuf:"bytes,3,rep,name=blobstor,proto3" json:"blobstor,omitempty"` - // Path to shard's write-cache, empty if disabled. - WritecachePath string `protobuf:"bytes,4,opt,name=writecache_path,json=writecachePath,proto3" json:"writecache_path,omitempty"` - // Work mode of the shard. - Mode ShardMode `protobuf:"varint,5,opt,name=mode,proto3,enum=control.ShardMode" json:"mode,omitempty"` - // Amount of errors occured. - ErrorCount uint32 `protobuf:"varint,6,opt,name=errorCount,proto3" json:"errorCount,omitempty"` - // Path to shard's pilorama storage. - PiloramaPath string `protobuf:"bytes,7,opt,name=pilorama_path,json=piloramaPath,proto3" json:"pilorama_path,omitempty"` -} - -func (x *ShardInfo) Reset() { - *x = ShardInfo{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_types_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ShardInfo) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ShardInfo) ProtoMessage() {} - -func (x *ShardInfo) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_types_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ShardInfo.ProtoReflect.Descriptor instead. -func (*ShardInfo) Descriptor() ([]byte, []int) { - return file_pkg_services_control_types_proto_rawDescGZIP(), []int{3} -} - -func (x *ShardInfo) GetShard_ID() []byte { - if x != nil { - return x.Shard_ID - } - return nil -} - -func (x *ShardInfo) GetMetabasePath() string { - if x != nil { - return x.MetabasePath - } - return "" -} - -func (x *ShardInfo) GetBlobstor() []*BlobstorInfo { - if x != nil { - return x.Blobstor - } - return nil -} - -func (x *ShardInfo) GetWritecachePath() string { - if x != nil { - return x.WritecachePath - } - return "" -} - -func (x *ShardInfo) GetMode() ShardMode { - if x != nil { - return x.Mode - } - return ShardMode_SHARD_MODE_UNDEFINED -} - -func (x *ShardInfo) GetErrorCount() uint32 { - if x != nil { - return x.ErrorCount - } - return 0 -} - -func (x *ShardInfo) GetPiloramaPath() string { - if x != nil { - return x.PiloramaPath - } - return "" -} - -// Blobstor component description. -type BlobstorInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Path to the root. - Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` - // Component type. - Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"` -} - -func (x *BlobstorInfo) Reset() { - *x = BlobstorInfo{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_types_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BlobstorInfo) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BlobstorInfo) ProtoMessage() {} - -func (x *BlobstorInfo) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_types_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BlobstorInfo.ProtoReflect.Descriptor instead. -func (*BlobstorInfo) Descriptor() ([]byte, []int) { - return file_pkg_services_control_types_proto_rawDescGZIP(), []int{4} -} - -func (x *BlobstorInfo) GetPath() string { - if x != nil { - return x.Path - } - return "" -} - -func (x *BlobstorInfo) GetType() string { - if x != nil { - return x.Type - } - return "" -} - -// ChainTarget is an object to which local overrides -// are applied. -type ChainTarget struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Type ChainTarget_TargetType `protobuf:"varint,1,opt,name=type,proto3,enum=control.ChainTarget_TargetType" json:"type,omitempty"` - Name string `protobuf:"bytes,2,opt,name=Name,proto3" json:"Name,omitempty"` -} - -func (x *ChainTarget) Reset() { - *x = ChainTarget{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_types_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ChainTarget) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ChainTarget) ProtoMessage() {} - -func (x *ChainTarget) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_types_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ChainTarget.ProtoReflect.Descriptor instead. -func (*ChainTarget) Descriptor() ([]byte, []int) { - return file_pkg_services_control_types_proto_rawDescGZIP(), []int{5} -} - -func (x *ChainTarget) GetType() ChainTarget_TargetType { - if x != nil { - return x.Type - } - return ChainTarget_UNDEFINED -} - -func (x *ChainTarget) GetName() string { - if x != nil { - return x.Name - } - return "" -} - -// Administrator-defined Attributes of the FrostFS Storage Node. -// -// `Attribute` is a Key-Value metadata pair. Key name must be a valid UTF-8 -// string. Value can't be empty. -// -// Node's attributes are mostly used during Storage Policy evaluation to -// calculate object's placement and find a set of nodes satisfying policy -// requirements. There are some "well-known" node attributes common to all the -// Storage Nodes in the network and used implicitly with default values if not -// explicitly set: -// -// - Capacity \ -// Total available disk space in Gigabytes. -// - Price \ -// Price in GAS tokens for storing one GB of data during one Epoch. In node -// attributes it's a string presenting floating point number with comma or -// point delimiter for decimal part. In the Network Map it will be saved as -// 64-bit unsigned integer representing number of minimal token fractions. -// - Locode \ -// Node's geographic location in -// [UN/LOCODE](https://www.unece.org/cefact/codesfortrade/codes_index.html) -// format approximated to the nearest point defined in standard. -// - Country \ -// Country code in -// [ISO 3166-1_alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) -// format. Calculated automatically from `Locode` attribute -// - Region \ -// Country's administative subdivision where node is located. Calculated -// automatically from `Locode` attribute based on `SubDiv` field. Presented -// in [ISO 3166-2](https://en.wikipedia.org/wiki/ISO_3166-2) format. -// - City \ -// City, town, village or rural area name where node is located written -// without diacritics . Calculated automatically from `Locode` attribute. -// -// For detailed description of each well-known attribute please see the -// corresponding section in FrostFS Technical specification. -type NodeInfo_Attribute struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Key of the node attribute. - Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - // Value of the node attribute. - Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - // Parent keys, if any. For example for `City` it could be `Region` and - // `Country`. - Parents []string `protobuf:"bytes,3,rep,name=parents,proto3" json:"parents,omitempty"` -} - -func (x *NodeInfo_Attribute) Reset() { - *x = NodeInfo_Attribute{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_control_types_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NodeInfo_Attribute) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NodeInfo_Attribute) ProtoMessage() {} - -func (x *NodeInfo_Attribute) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_control_types_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NodeInfo_Attribute.ProtoReflect.Descriptor instead. -func (*NodeInfo_Attribute) Descriptor() ([]byte, []int) { - return file_pkg_services_control_types_proto_rawDescGZIP(), []int{1, 0} -} - -func (x *NodeInfo_Attribute) GetKey() string { - if x != nil { - return x.Key - } - return "" -} - -func (x *NodeInfo_Attribute) GetValue() string { - if x != nil { - return x.Value - } - return "" -} - -func (x *NodeInfo_Attribute) GetParents() []string { - if x != nil { - return x.Parents - } - return nil -} - -var File_pkg_services_control_types_proto protoreflect.FileDescriptor - -var file_pkg_services_control_types_proto_rawDesc = []byte{ - 0x0a, 0x20, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x12, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x22, 0x36, 0x0a, 0x09, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x17, 0x0a, 0x04, 0x73, 0x69, - 0x67, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x22, 0x80, 0x02, 0x0a, 0x08, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, - 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, - 0x1c, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x12, 0x3b, 0x0a, - 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1b, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x4e, 0x6f, 0x64, 0x65, - 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x52, 0x0a, - 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x2b, 0x0a, 0x05, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x72, 0x6f, 0x6c, 0x2e, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x1a, 0x4d, 0x0a, 0x09, 0x41, 0x74, 0x74, 0x72, 0x69, - 0x62, 0x75, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x18, 0x0a, 0x07, - 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x70, - 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x47, 0x0a, 0x06, 0x4e, 0x65, 0x74, 0x6d, 0x61, 0x70, - 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x27, 0x0a, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, - 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x22, - 0x94, 0x02, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, - 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x07, 0x73, 0x68, 0x61, 0x72, 0x64, 0x49, 0x44, 0x12, 0x23, 0x0a, 0x0d, 0x6d, 0x65, 0x74, 0x61, - 0x62, 0x61, 0x73, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0c, 0x6d, 0x65, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x31, 0x0a, - 0x08, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x15, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x74, - 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 0x72, - 0x12, 0x27, 0x0a, 0x0f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x70, - 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x77, 0x72, 0x69, 0x74, 0x65, - 0x63, 0x61, 0x63, 0x68, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x26, 0x0a, 0x04, 0x6d, 0x6f, 0x64, - 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6d, 0x6f, 0x64, - 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x75, 0x6e, - 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x69, 0x6c, 0x6f, 0x72, 0x61, 0x6d, 0x61, 0x5f, 0x70, 0x61, - 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x69, 0x6c, 0x6f, 0x72, 0x61, - 0x6d, 0x61, 0x50, 0x61, 0x74, 0x68, 0x22, 0x36, 0x0a, 0x0c, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x74, - 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xa6, - 0x01, 0x0a, 0x0b, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x33, - 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, - 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x4e, 0x0a, 0x0a, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, - 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, - 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x55, 0x53, 0x45, 0x52, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, - 0x47, 0x52, 0x4f, 0x55, 0x50, 0x10, 0x04, 0x2a, 0x4e, 0x0a, 0x0c, 0x4e, 0x65, 0x74, 0x6d, 0x61, - 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x54, 0x41, 0x54, 0x55, - 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, - 0x06, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x46, 0x46, - 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, - 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x2a, 0x6a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x17, 0x48, 0x45, 0x41, 0x4c, 0x54, - 0x48, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, - 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x11, 0x0a, - 0x0d, 0x53, 0x48, 0x55, 0x54, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x03, - 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x55, 0x52, 0x49, 0x4e, - 0x47, 0x10, 0x04, 0x2a, 0x6a, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x6f, 0x64, 0x65, - 0x12, 0x18, 0x0a, 0x14, 0x53, 0x48, 0x41, 0x52, 0x44, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, - 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, - 0x41, 0x44, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, - 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x47, - 0x52, 0x41, 0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x47, 0x52, 0x41, - 0x44, 0x45, 0x44, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x04, 0x42, - 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, - 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, - 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_pkg_services_control_types_proto_rawDescOnce sync.Once - file_pkg_services_control_types_proto_rawDescData = file_pkg_services_control_types_proto_rawDesc -) - -func file_pkg_services_control_types_proto_rawDescGZIP() []byte { - file_pkg_services_control_types_proto_rawDescOnce.Do(func() { - file_pkg_services_control_types_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_services_control_types_proto_rawDescData) - }) - return file_pkg_services_control_types_proto_rawDescData -} - -var file_pkg_services_control_types_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_pkg_services_control_types_proto_msgTypes = make([]protoimpl.MessageInfo, 7) -var file_pkg_services_control_types_proto_goTypes = []interface{}{ - (NetmapStatus)(0), // 0: control.NetmapStatus - (HealthStatus)(0), // 1: control.HealthStatus - (ShardMode)(0), // 2: control.ShardMode - (ChainTarget_TargetType)(0), // 3: control.ChainTarget.TargetType - (*Signature)(nil), // 4: control.Signature - (*NodeInfo)(nil), // 5: control.NodeInfo - (*Netmap)(nil), // 6: control.Netmap - (*ShardInfo)(nil), // 7: control.ShardInfo - (*BlobstorInfo)(nil), // 8: control.BlobstorInfo - (*ChainTarget)(nil), // 9: control.ChainTarget - (*NodeInfo_Attribute)(nil), // 10: control.NodeInfo.Attribute -} -var file_pkg_services_control_types_proto_depIdxs = []int32{ - 10, // 0: control.NodeInfo.attributes:type_name -> control.NodeInfo.Attribute - 0, // 1: control.NodeInfo.state:type_name -> control.NetmapStatus - 5, // 2: control.Netmap.nodes:type_name -> control.NodeInfo - 8, // 3: control.ShardInfo.blobstor:type_name -> control.BlobstorInfo - 2, // 4: control.ShardInfo.mode:type_name -> control.ShardMode - 3, // 5: control.ChainTarget.type:type_name -> control.ChainTarget.TargetType - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name -} - -func init() { file_pkg_services_control_types_proto_init() } -func file_pkg_services_control_types_proto_init() { - if File_pkg_services_control_types_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_pkg_services_control_types_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Signature); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_types_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeInfo); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_types_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Netmap); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_types_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ShardInfo); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_types_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BlobstorInfo); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_types_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ChainTarget); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_control_types_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeInfo_Attribute); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_pkg_services_control_types_proto_rawDesc, - NumEnums: 4, - NumMessages: 7, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_pkg_services_control_types_proto_goTypes, - DependencyIndexes: file_pkg_services_control_types_proto_depIdxs, - EnumInfos: file_pkg_services_control_types_proto_enumTypes, - MessageInfos: file_pkg_services_control_types_proto_msgTypes, - }.Build() - File_pkg_services_control_types_proto = out.File - file_pkg_services_control_types_proto_rawDesc = nil - file_pkg_services_control_types_proto_goTypes = nil - file_pkg_services_control_types_proto_depIdxs = nil -} diff --git a/pkg/services/control/types_frostfs.pb.go b/pkg/services/control/types_frostfs.pb.go index 858d85a1c..9aff26a98 100644 --- a/pkg/services/control/types_frostfs.pb.go +++ b/pkg/services/control/types_frostfs.pb.go @@ -2,7 +2,149 @@ package control -import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" +import ( + json "encoding/json" + fmt "fmt" + pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + easyproto "github.com/VictoriaMetrics/easyproto" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" + strconv "strconv" +) + +type NetmapStatus int32 + +const ( + NetmapStatus_STATUS_UNDEFINED NetmapStatus = 0 + NetmapStatus_ONLINE NetmapStatus = 1 + NetmapStatus_OFFLINE NetmapStatus = 2 + NetmapStatus_MAINTENANCE NetmapStatus = 3 +) + +var ( + NetmapStatus_name = map[int32]string{ + 0: "STATUS_UNDEFINED", + 1: "ONLINE", + 2: "OFFLINE", + 3: "MAINTENANCE", + } + NetmapStatus_value = map[string]int32{ + "STATUS_UNDEFINED": 0, + "ONLINE": 1, + "OFFLINE": 2, + "MAINTENANCE": 3, + } +) + +func (x NetmapStatus) String() string { + if v, ok := NetmapStatus_name[int32(x)]; ok { + return v + } + return strconv.FormatInt(int64(x), 10) +} +func (x *NetmapStatus) FromString(s string) bool { + if v, ok := NetmapStatus_value[s]; ok { + *x = NetmapStatus(v) + return true + } + return false +} + +type HealthStatus int32 + +const ( + HealthStatus_HEALTH_STATUS_UNDEFINED HealthStatus = 0 + HealthStatus_STARTING HealthStatus = 1 + HealthStatus_READY HealthStatus = 2 + HealthStatus_SHUTTING_DOWN HealthStatus = 3 + HealthStatus_RECONFIGURING HealthStatus = 4 +) + +var ( + HealthStatus_name = map[int32]string{ + 0: "HEALTH_STATUS_UNDEFINED", + 1: "STARTING", + 2: "READY", + 3: "SHUTTING_DOWN", + 4: "RECONFIGURING", + } + HealthStatus_value = map[string]int32{ + "HEALTH_STATUS_UNDEFINED": 0, + "STARTING": 1, + "READY": 2, + "SHUTTING_DOWN": 3, + "RECONFIGURING": 4, + } +) + +func (x HealthStatus) String() string { + if v, ok := HealthStatus_name[int32(x)]; ok { + return v + } + return strconv.FormatInt(int64(x), 10) +} +func (x *HealthStatus) FromString(s string) bool { + if v, ok := HealthStatus_value[s]; ok { + *x = HealthStatus(v) + return true + } + return false +} + +type ShardMode int32 + +const ( + ShardMode_SHARD_MODE_UNDEFINED ShardMode = 0 + ShardMode_READ_WRITE ShardMode = 1 + ShardMode_READ_ONLY ShardMode = 2 + ShardMode_DEGRADED ShardMode = 3 + ShardMode_DEGRADED_READ_ONLY ShardMode = 4 +) + +var ( + ShardMode_name = map[int32]string{ + 0: "SHARD_MODE_UNDEFINED", + 1: "READ_WRITE", + 2: "READ_ONLY", + 3: "DEGRADED", + 4: "DEGRADED_READ_ONLY", + } + ShardMode_value = map[string]int32{ + "SHARD_MODE_UNDEFINED": 0, + "READ_WRITE": 1, + "READ_ONLY": 2, + "DEGRADED": 3, + "DEGRADED_READ_ONLY": 4, + } +) + +func (x ShardMode) String() string { + if v, ok := ShardMode_name[int32(x)]; ok { + return v + } + return strconv.FormatInt(int64(x), 10) +} +func (x *ShardMode) FromString(s string) bool { + if v, ok := ShardMode_value[s]; ok { + *x = ShardMode(v) + return true + } + return false +} + +type Signature struct { + Key []byte `json:"key"` + Sign []byte `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*Signature)(nil) + _ encoding.ProtoUnmarshaler = (*Signature)(nil) + _ json.Marshaler = (*Signature)(nil) + _ json.Unmarshaler = (*Signature)(nil) +) // StableSize returns the size of x in protobuf format. // @@ -16,27 +158,155 @@ func (x *Signature) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *Signature) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.Key) - offset += proto.BytesMarshal(2, buf[offset:], x.Sign) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *Signature) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *Signature) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Key) != 0 { + mm.AppendBytes(1, x.Key) + } + if len(x.Sign) != 0 { + mm.AppendBytes(2, x.Sign) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *Signature) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "Signature") + } + switch fc.FieldNum { + case 1: // Key + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Key") + } + x.Key = data + case 2: // Sign + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Sign") + } + x.Sign = data + } + } + return nil +} +func (x *Signature) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} +func (x *Signature) SetKey(v []byte) { + x.Key = v +} +func (x *Signature) GetSign() []byte { + if x != nil { + return x.Sign + } + return nil +} +func (x *Signature) SetSign(v []byte) { + x.Sign = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *Signature) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *Signature) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"key\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.Key) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + out.Base64Bytes(x.Sign) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *Signature) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *Signature) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "key": + { + var f []byte + f = in.Bytes() + x.Key = f + } + case "signature": + { + var f []byte + f = in.Bytes() + x.Sign = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type NodeInfo_Attribute struct { + Key string `json:"key"` + Value string `json:"value"` + Parents []string `json:"parents"` +} + +var ( + _ encoding.ProtoMarshaler = (*NodeInfo_Attribute)(nil) + _ encoding.ProtoUnmarshaler = (*NodeInfo_Attribute)(nil) + _ json.Marshaler = (*NodeInfo_Attribute)(nil) + _ json.Unmarshaler = (*NodeInfo_Attribute)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -50,28 +320,199 @@ func (x *NodeInfo_Attribute) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *NodeInfo_Attribute) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.StringMarshal(1, buf[offset:], x.Key) - offset += proto.StringMarshal(2, buf[offset:], x.Value) - offset += proto.RepeatedStringMarshal(3, buf[offset:], x.Parents) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *NodeInfo_Attribute) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *NodeInfo_Attribute) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Key) != 0 { + mm.AppendString(1, x.Key) + } + if len(x.Value) != 0 { + mm.AppendString(2, x.Value) + } + for j := range x.Parents { + mm.AppendString(3, x.Parents[j]) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *NodeInfo_Attribute) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "NodeInfo_Attribute") + } + switch fc.FieldNum { + case 1: // Key + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Key") + } + x.Key = data + case 2: // Value + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Value") + } + x.Value = data + case 3: // Parents + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Parents") + } + x.Parents = append(x.Parents, data) + } + } + return nil +} +func (x *NodeInfo_Attribute) GetKey() string { + if x != nil { + return x.Key + } + return "" +} +func (x *NodeInfo_Attribute) SetKey(v string) { + x.Key = v +} +func (x *NodeInfo_Attribute) GetValue() string { + if x != nil { + return x.Value + } + return "" +} +func (x *NodeInfo_Attribute) SetValue(v string) { + x.Value = v +} +func (x *NodeInfo_Attribute) GetParents() []string { + if x != nil { + return x.Parents + } + return nil +} +func (x *NodeInfo_Attribute) SetParents(v []string) { + x.Parents = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *NodeInfo_Attribute) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *NodeInfo_Attribute) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"key\":" + out.RawString(prefix[1:]) + out.String(x.Key) + } + { + const prefix string = ",\"value\":" + out.RawString(prefix) + out.String(x.Value) + } + { + const prefix string = ",\"parents\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Parents { + if i != 0 { + out.RawByte(',') + } + out.String(x.Parents[i]) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *NodeInfo_Attribute) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *NodeInfo_Attribute) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "key": + { + var f string + f = in.String() + x.Key = f + } + case "value": + { + var f string + f = in.String() + x.Value = f + } + case "parents": + { + var f string + var list []string + in.Delim('[') + for !in.IsDelim(']') { + f = in.String() + list = append(list, f) + in.WantComma() + } + x.Parents = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type NodeInfo struct { + PublicKey []byte `json:"publicKey"` + Addresses []string `json:"addresses"` + Attributes []*NodeInfo_Attribute `json:"attributes"` + State NetmapStatus `json:"state"` +} + +var ( + _ encoding.ProtoMarshaler = (*NodeInfo)(nil) + _ encoding.ProtoUnmarshaler = (*NodeInfo)(nil) + _ json.Marshaler = (*NodeInfo)(nil) + _ json.Unmarshaler = (*NodeInfo)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -88,31 +529,263 @@ func (x *NodeInfo) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *NodeInfo) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.PublicKey) - offset += proto.RepeatedStringMarshal(2, buf[offset:], x.Addresses) - for i := range x.Attributes { - offset += proto.NestedStructureMarshal(3, buf[offset:], x.Attributes[i]) - } - offset += proto.EnumMarshal(4, buf[offset:], int32(x.State)) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *NodeInfo) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *NodeInfo) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.PublicKey) != 0 { + mm.AppendBytes(1, x.PublicKey) + } + for j := range x.Addresses { + mm.AppendString(2, x.Addresses[j]) + } + for i := range x.Attributes { + if x.Attributes[i] != nil && x.Attributes[i].StableSize() != 0 { + x.Attributes[i].EmitProtobuf(mm.AppendMessage(3)) + } + } + if int32(x.State) != 0 { + mm.AppendInt32(4, int32(x.State)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *NodeInfo) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "NodeInfo") + } + switch fc.FieldNum { + case 1: // PublicKey + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "PublicKey") + } + x.PublicKey = data + case 2: // Addresses + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Addresses") + } + x.Addresses = append(x.Addresses, data) + case 3: // Attributes + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Attributes") + } + x.Attributes = append(x.Attributes, new(NodeInfo_Attribute)) + ff := x.Attributes[len(x.Attributes)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 4: // State + data, ok := fc.Int32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "State") + } + x.State = NetmapStatus(data) + } + } + return nil +} +func (x *NodeInfo) GetPublicKey() []byte { + if x != nil { + return x.PublicKey + } + return nil +} +func (x *NodeInfo) SetPublicKey(v []byte) { + x.PublicKey = v +} +func (x *NodeInfo) GetAddresses() []string { + if x != nil { + return x.Addresses + } + return nil +} +func (x *NodeInfo) SetAddresses(v []string) { + x.Addresses = v +} +func (x *NodeInfo) GetAttributes() []*NodeInfo_Attribute { + if x != nil { + return x.Attributes + } + return nil +} +func (x *NodeInfo) SetAttributes(v []*NodeInfo_Attribute) { + x.Attributes = v +} +func (x *NodeInfo) GetState() NetmapStatus { + if x != nil { + return x.State + } + return 0 +} +func (x *NodeInfo) SetState(v NetmapStatus) { + x.State = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *NodeInfo) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *NodeInfo) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"publicKey\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.PublicKey) + } + { + const prefix string = ",\"addresses\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Addresses { + if i != 0 { + out.RawByte(',') + } + out.String(x.Addresses[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"attributes\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Attributes { + if i != 0 { + out.RawByte(',') + } + x.Attributes[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + { + const prefix string = ",\"state\":" + out.RawString(prefix) + out.Int32(int32(x.State)) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *NodeInfo) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *NodeInfo) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "publicKey": + { + var f []byte + f = in.Bytes() + x.PublicKey = f + } + case "addresses": + { + var f string + var list []string + in.Delim('[') + for !in.IsDelim(']') { + f = in.String() + list = append(list, f) + in.WantComma() + } + x.Addresses = list + in.Delim(']') + } + case "attributes": + { + var f *NodeInfo_Attribute + var list []*NodeInfo_Attribute + in.Delim('[') + for !in.IsDelim(']') { + f = new(NodeInfo_Attribute) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Attributes = list + in.Delim(']') + } + case "state": + { + var f NetmapStatus + var parsedValue NetmapStatus + switch v := in.Interface().(type) { + case string: + if vv, ok := NetmapStatus_value[v]; ok { + parsedValue = NetmapStatus(vv) + break + } + vv, err := strconv.ParseInt(v, 10, 32) + if err != nil { + in.AddError(err) + return + } + parsedValue = NetmapStatus(vv) + case float64: + parsedValue = NetmapStatus(v) + } + f = parsedValue + x.State = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type Netmap struct { + Epoch uint64 `json:"epoch"` + Nodes []*NodeInfo `json:"nodes"` +} + +var ( + _ encoding.ProtoMarshaler = (*Netmap)(nil) + _ encoding.ProtoUnmarshaler = (*Netmap)(nil) + _ json.Marshaler = (*Netmap)(nil) + _ json.Unmarshaler = (*Netmap)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -127,29 +800,180 @@ func (x *Netmap) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *Netmap) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.UInt64Marshal(1, buf[offset:], x.Epoch) - for i := range x.Nodes { - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Nodes[i]) - } - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *Netmap) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *Netmap) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Epoch != 0 { + mm.AppendUint64(1, x.Epoch) + } + for i := range x.Nodes { + if x.Nodes[i] != nil && x.Nodes[i].StableSize() != 0 { + x.Nodes[i].EmitProtobuf(mm.AppendMessage(2)) + } + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *Netmap) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "Netmap") + } + switch fc.FieldNum { + case 1: // Epoch + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Epoch") + } + x.Epoch = data + case 2: // Nodes + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Nodes") + } + x.Nodes = append(x.Nodes, new(NodeInfo)) + ff := x.Nodes[len(x.Nodes)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *Netmap) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} +func (x *Netmap) SetEpoch(v uint64) { + x.Epoch = v +} +func (x *Netmap) GetNodes() []*NodeInfo { + if x != nil { + return x.Nodes + } + return nil +} +func (x *Netmap) SetNodes(v []*NodeInfo) { + x.Nodes = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *Netmap) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *Netmap) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"epoch\":" + out.RawString(prefix[1:]) + out.Uint64(x.Epoch) + } + { + const prefix string = ",\"nodes\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Nodes { + if i != 0 { + out.RawByte(',') + } + x.Nodes[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *Netmap) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *Netmap) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "epoch": + { + var f uint64 + f = in.Uint64() + x.Epoch = f + } + case "nodes": + { + var f *NodeInfo + var list []*NodeInfo + in.Delim('[') + for !in.IsDelim(']') { + f = new(NodeInfo) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Nodes = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ShardInfo struct { + Shard_ID []byte `json:"shardID"` + MetabasePath string `json:"metabasePath"` + Blobstor []*BlobstorInfo `json:"blobstor"` + WritecachePath string `json:"writecachePath"` + Mode ShardMode `json:"mode"` + ErrorCount uint32 `json:"errorCount"` + PiloramaPath string `json:"piloramaPath"` +} + +var ( + _ encoding.ProtoMarshaler = (*ShardInfo)(nil) + _ encoding.ProtoUnmarshaler = (*ShardInfo)(nil) + _ json.Marshaler = (*ShardInfo)(nil) + _ json.Unmarshaler = (*ShardInfo)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -169,34 +993,336 @@ func (x *ShardInfo) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ShardInfo) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.Shard_ID) - offset += proto.StringMarshal(2, buf[offset:], x.MetabasePath) - for i := range x.Blobstor { - offset += proto.NestedStructureMarshal(3, buf[offset:], x.Blobstor[i]) - } - offset += proto.StringMarshal(4, buf[offset:], x.WritecachePath) - offset += proto.EnumMarshal(5, buf[offset:], int32(x.Mode)) - offset += proto.UInt32Marshal(6, buf[offset:], x.ErrorCount) - offset += proto.StringMarshal(7, buf[offset:], x.PiloramaPath) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ShardInfo) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ShardInfo) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Shard_ID) != 0 { + mm.AppendBytes(1, x.Shard_ID) + } + if len(x.MetabasePath) != 0 { + mm.AppendString(2, x.MetabasePath) + } + for i := range x.Blobstor { + if x.Blobstor[i] != nil && x.Blobstor[i].StableSize() != 0 { + x.Blobstor[i].EmitProtobuf(mm.AppendMessage(3)) + } + } + if len(x.WritecachePath) != 0 { + mm.AppendString(4, x.WritecachePath) + } + if int32(x.Mode) != 0 { + mm.AppendInt32(5, int32(x.Mode)) + } + if x.ErrorCount != 0 { + mm.AppendUint32(6, x.ErrorCount) + } + if len(x.PiloramaPath) != 0 { + mm.AppendString(7, x.PiloramaPath) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ShardInfo) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ShardInfo") + } + switch fc.FieldNum { + case 1: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = data + case 2: // MetabasePath + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "MetabasePath") + } + x.MetabasePath = data + case 3: // Blobstor + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Blobstor") + } + x.Blobstor = append(x.Blobstor, new(BlobstorInfo)) + ff := x.Blobstor[len(x.Blobstor)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 4: // WritecachePath + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "WritecachePath") + } + x.WritecachePath = data + case 5: // Mode + data, ok := fc.Int32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Mode") + } + x.Mode = ShardMode(data) + case 6: // ErrorCount + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ErrorCount") + } + x.ErrorCount = data + case 7: // PiloramaPath + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "PiloramaPath") + } + x.PiloramaPath = data + } + } + return nil +} +func (x *ShardInfo) GetShard_ID() []byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *ShardInfo) SetShard_ID(v []byte) { + x.Shard_ID = v +} +func (x *ShardInfo) GetMetabasePath() string { + if x != nil { + return x.MetabasePath + } + return "" +} +func (x *ShardInfo) SetMetabasePath(v string) { + x.MetabasePath = v +} +func (x *ShardInfo) GetBlobstor() []*BlobstorInfo { + if x != nil { + return x.Blobstor + } + return nil +} +func (x *ShardInfo) SetBlobstor(v []*BlobstorInfo) { + x.Blobstor = v +} +func (x *ShardInfo) GetWritecachePath() string { + if x != nil { + return x.WritecachePath + } + return "" +} +func (x *ShardInfo) SetWritecachePath(v string) { + x.WritecachePath = v +} +func (x *ShardInfo) GetMode() ShardMode { + if x != nil { + return x.Mode + } + return 0 +} +func (x *ShardInfo) SetMode(v ShardMode) { + x.Mode = v +} +func (x *ShardInfo) GetErrorCount() uint32 { + if x != nil { + return x.ErrorCount + } + return 0 +} +func (x *ShardInfo) SetErrorCount(v uint32) { + x.ErrorCount = v +} +func (x *ShardInfo) GetPiloramaPath() string { + if x != nil { + return x.PiloramaPath + } + return "" +} +func (x *ShardInfo) SetPiloramaPath(v string) { + x.PiloramaPath = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ShardInfo) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ShardInfo) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"shardID\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.Shard_ID) + } + { + const prefix string = ",\"metabasePath\":" + out.RawString(prefix) + out.String(x.MetabasePath) + } + { + const prefix string = ",\"blobstor\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Blobstor { + if i != 0 { + out.RawByte(',') + } + x.Blobstor[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + { + const prefix string = ",\"writecachePath\":" + out.RawString(prefix) + out.String(x.WritecachePath) + } + { + const prefix string = ",\"mode\":" + out.RawString(prefix) + out.Int32(int32(x.Mode)) + } + { + const prefix string = ",\"errorCount\":" + out.RawString(prefix) + out.Uint32(x.ErrorCount) + } + { + const prefix string = ",\"piloramaPath\":" + out.RawString(prefix) + out.String(x.PiloramaPath) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ShardInfo) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ShardInfo) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shardID": + { + var f []byte + f = in.Bytes() + x.Shard_ID = f + } + case "metabasePath": + { + var f string + f = in.String() + x.MetabasePath = f + } + case "blobstor": + { + var f *BlobstorInfo + var list []*BlobstorInfo + in.Delim('[') + for !in.IsDelim(']') { + f = new(BlobstorInfo) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Blobstor = list + in.Delim(']') + } + case "writecachePath": + { + var f string + f = in.String() + x.WritecachePath = f + } + case "mode": + { + var f ShardMode + var parsedValue ShardMode + switch v := in.Interface().(type) { + case string: + if vv, ok := ShardMode_value[v]; ok { + parsedValue = ShardMode(vv) + break + } + vv, err := strconv.ParseInt(v, 10, 32) + if err != nil { + in.AddError(err) + return + } + parsedValue = ShardMode(vv) + case float64: + parsedValue = ShardMode(v) + } + f = parsedValue + x.Mode = f + } + case "errorCount": + { + var f uint32 + f = in.Uint32() + x.ErrorCount = f + } + case "piloramaPath": + { + var f string + f = in.String() + x.PiloramaPath = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type BlobstorInfo struct { + Path string `json:"path"` + Type string `json:"type"` +} + +var ( + _ encoding.ProtoMarshaler = (*BlobstorInfo)(nil) + _ encoding.ProtoUnmarshaler = (*BlobstorInfo)(nil) + _ json.Marshaler = (*BlobstorInfo)(nil) + _ json.Unmarshaler = (*BlobstorInfo)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -209,27 +1335,195 @@ func (x *BlobstorInfo) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *BlobstorInfo) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.StringMarshal(1, buf[offset:], x.Path) - offset += proto.StringMarshal(2, buf[offset:], x.Type) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *BlobstorInfo) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *BlobstorInfo) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Path) != 0 { + mm.AppendString(1, x.Path) + } + if len(x.Type) != 0 { + mm.AppendString(2, x.Type) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *BlobstorInfo) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "BlobstorInfo") + } + switch fc.FieldNum { + case 1: // Path + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Path") + } + x.Path = data + case 2: // Type + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Type") + } + x.Type = data + } + } + return nil +} +func (x *BlobstorInfo) GetPath() string { + if x != nil { + return x.Path + } + return "" +} +func (x *BlobstorInfo) SetPath(v string) { + x.Path = v +} +func (x *BlobstorInfo) GetType() string { + if x != nil { + return x.Type + } + return "" +} +func (x *BlobstorInfo) SetType(v string) { + x.Type = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *BlobstorInfo) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *BlobstorInfo) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"path\":" + out.RawString(prefix[1:]) + out.String(x.Path) + } + { + const prefix string = ",\"type\":" + out.RawString(prefix) + out.String(x.Type) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *BlobstorInfo) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *BlobstorInfo) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "path": + { + var f string + f = in.String() + x.Path = f + } + case "type": + { + var f string + f = in.String() + x.Type = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ChainTarget_TargetType int32 + +const ( + ChainTarget_UNDEFINED ChainTarget_TargetType = 0 + ChainTarget_NAMESPACE ChainTarget_TargetType = 1 + ChainTarget_CONTAINER ChainTarget_TargetType = 2 + ChainTarget_USER ChainTarget_TargetType = 3 + ChainTarget_GROUP ChainTarget_TargetType = 4 +) + +var ( + ChainTarget_TargetType_name = map[int32]string{ + 0: "UNDEFINED", + 1: "NAMESPACE", + 2: "CONTAINER", + 3: "USER", + 4: "GROUP", + } + ChainTarget_TargetType_value = map[string]int32{ + "UNDEFINED": 0, + "NAMESPACE": 1, + "CONTAINER": 2, + "USER": 3, + "GROUP": 4, + } +) + +func (x ChainTarget_TargetType) String() string { + if v, ok := ChainTarget_TargetType_name[int32(x)]; ok { + return v + } + return strconv.FormatInt(int64(x), 10) +} +func (x *ChainTarget_TargetType) FromString(s string) bool { + if v, ok := ChainTarget_TargetType_value[s]; ok { + *x = ChainTarget_TargetType(v) + return true + } + return false +} + +type ChainTarget struct { + Type ChainTarget_TargetType `json:"type"` + Name string `json:"Name"` +} + +var ( + _ encoding.ProtoMarshaler = (*ChainTarget)(nil) + _ encoding.ProtoUnmarshaler = (*ChainTarget)(nil) + _ json.Marshaler = (*ChainTarget)(nil) + _ json.Unmarshaler = (*ChainTarget)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -242,23 +1536,154 @@ func (x *ChainTarget) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ChainTarget) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.EnumMarshal(1, buf[offset:], int32(x.Type)) - offset += proto.StringMarshal(2, buf[offset:], x.Name) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ChainTarget) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *ChainTarget) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if int32(x.Type) != 0 { + mm.AppendInt32(1, int32(x.Type)) + } + if len(x.Name) != 0 { + mm.AppendString(2, x.Name) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ChainTarget) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ChainTarget") + } + switch fc.FieldNum { + case 1: // Type + data, ok := fc.Int32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Type") + } + x.Type = ChainTarget_TargetType(data) + case 2: // Name + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Name") + } + x.Name = data + } + } + return nil +} +func (x *ChainTarget) GetType() ChainTarget_TargetType { + if x != nil { + return x.Type + } + return 0 +} +func (x *ChainTarget) SetType(v ChainTarget_TargetType) { + x.Type = v +} +func (x *ChainTarget) GetName() string { + if x != nil { + return x.Name + } + return "" +} +func (x *ChainTarget) SetName(v string) { + x.Name = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ChainTarget) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ChainTarget) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"type\":" + out.RawString(prefix[1:]) + out.Int32(int32(x.Type)) + } + { + const prefix string = ",\"Name\":" + out.RawString(prefix) + out.String(x.Name) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ChainTarget) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ChainTarget) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "type": + { + var f ChainTarget_TargetType + var parsedValue ChainTarget_TargetType + switch v := in.Interface().(type) { + case string: + if vv, ok := ChainTarget_TargetType_value[v]; ok { + parsedValue = ChainTarget_TargetType(vv) + break + } + vv, err := strconv.ParseInt(v, 10, 32) + if err != nil { + in.AddError(err) + return + } + parsedValue = ChainTarget_TargetType(vv) + case float64: + parsedValue = ChainTarget_TargetType(v) + } + f = parsedValue + x.Type = f + } + case "Name": + { + var f string + f = in.String() + x.Name = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } } diff --git a/pkg/services/control/types_test.go b/pkg/services/control/types_test.go deleted file mode 100644 index df0cdf141..000000000 --- a/pkg/services/control/types_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package control_test - -import ( - "bytes" - "path/filepath" - "strconv" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "github.com/google/uuid" -) - -func TestNetmap_StableMarshal(t *testing.T) { - testStableMarshal(t, generateNetmap(), new(control.Netmap), func(m1, m2 protoMessage) bool { - return equalNetmaps(m1.(*control.Netmap), m2.(*control.Netmap)) - }) -} - -func generateNetmap() *control.Netmap { - nm := new(control.Netmap) - nm.SetEpoch(13) - - const nodeCount = 2 - - nodes := make([]*control.NodeInfo, 0, nodeCount) - - for i := 0; i < nodeCount; i++ { - n := new(control.NodeInfo) - n.SetPublicKey(testData(33)) - n.SetAddresses([]string{testString(), testString()}) - n.SetState(control.NetmapStatus_ONLINE) - - const attrCount = 2 - - attrs := make([]*control.NodeInfo_Attribute, 0, attrCount) - - for j := 0; j < attrCount; j++ { - a := new(control.NodeInfo_Attribute) - a.SetKey(testString()) - a.SetValue(testString()) - - const parentsCount = 2 - - parents := make([]string, 0, parentsCount) - - for k := 0; k < parentsCount; k++ { - parents = append(parents, testString()) - } - - a.SetParents(parents) - - attrs = append(attrs, a) - } - - n.SetAttributes(attrs) - - nodes = append(nodes, n) - } - - nm.SetNodes(nodes) - - return nm -} - -func equalNetmaps(nm1, nm2 *control.Netmap) bool { - if nm1.GetEpoch() != nm2.GetEpoch() { - return false - } - - n1, n2 := nm1.GetNodes(), nm2.GetNodes() - - if len(n1) != len(n2) { - return false - } - - for i := range n1 { - if !equalNodeInfos(n1[i], n2[i]) { - return false - } - } - - return true -} - -func equalNodeInfos(n1, n2 *control.NodeInfo) bool { - if !bytes.Equal(n1.GetPublicKey(), n2.GetPublicKey()) || - n1.GetState() != n2.GetState() { - return false - } - - na1, na2 := n1.GetAddresses(), n2.GetAddresses() - - if len(na1) != len(na2) { - return false - } - - for i := range na1 { - if na1[i] != na2[i] { - return false - } - } - - a1, a2 := n1.GetAttributes(), n2.GetAttributes() - - if len(a1) != len(a2) { - return false - } - - for i := range a1 { - if a1[i].GetKey() != a2[i].GetKey() || a1[i].GetValue() != a2[i].GetValue() { - return false - } - - p1, p2 := a1[i].GetParents(), a2[i].GetParents() - - if len(p1) != len(p2) { - return false - } - - for j := range p1 { - if p1[j] != p2[j] { - return false - } - } - } - - return true -} - -func generateShardInfo(id int) *control.ShardInfo { - si := new(control.ShardInfo) - - path := "/nice/dir/awesome/files/" + strconv.Itoa(id) - - uid, _ := uuid.NewRandom() - bin, _ := uid.MarshalBinary() - - si.SetID(bin) - si.SetMode(control.ShardMode_READ_WRITE) - si.SetMetabasePath(filepath.Join(path, "meta")) - si.Blobstor = []*control.BlobstorInfo{ - {Type: fstree.Type, Path: filepath.Join(path, "fstree")}, - {Type: blobovniczatree.Type, Path: filepath.Join(path, "blobtree")}, - } - si.SetWriteCachePath(filepath.Join(path, "writecache")) - si.SetPiloramaPath(filepath.Join(path, "pilorama")) - - return si -} diff --git a/pkg/services/tree/service.pb.go b/pkg/services/tree/service.pb.go deleted file mode 100644 index f439e3f28..000000000 --- a/pkg/services/tree/service.pb.go +++ /dev/null @@ -1,3587 +0,0 @@ -//* -// Service for working with CRDT tree. - -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.33.0 -// protoc v4.25.0 -// source: pkg/services/tree/service.proto - -package tree - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type GetSubTreeRequest_Body_Order_Direction int32 - -const ( - GetSubTreeRequest_Body_Order_None GetSubTreeRequest_Body_Order_Direction = 0 - GetSubTreeRequest_Body_Order_Asc GetSubTreeRequest_Body_Order_Direction = 1 -) - -// Enum value maps for GetSubTreeRequest_Body_Order_Direction. -var ( - GetSubTreeRequest_Body_Order_Direction_name = map[int32]string{ - 0: "None", - 1: "Asc", - } - GetSubTreeRequest_Body_Order_Direction_value = map[string]int32{ - "None": 0, - "Asc": 1, - } -) - -func (x GetSubTreeRequest_Body_Order_Direction) Enum() *GetSubTreeRequest_Body_Order_Direction { - p := new(GetSubTreeRequest_Body_Order_Direction) - *p = x - return p -} - -func (x GetSubTreeRequest_Body_Order_Direction) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (GetSubTreeRequest_Body_Order_Direction) Descriptor() protoreflect.EnumDescriptor { - return file_pkg_services_tree_service_proto_enumTypes[0].Descriptor() -} - -func (GetSubTreeRequest_Body_Order_Direction) Type() protoreflect.EnumType { - return &file_pkg_services_tree_service_proto_enumTypes[0] -} - -func (x GetSubTreeRequest_Body_Order_Direction) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use GetSubTreeRequest_Body_Order_Direction.Descriptor instead. -func (GetSubTreeRequest_Body_Order_Direction) EnumDescriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{10, 0, 0, 0} -} - -type AddRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Request body. - Body *AddRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Request signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *AddRequest) Reset() { - *x = AddRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddRequest) ProtoMessage() {} - -func (x *AddRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddRequest.ProtoReflect.Descriptor instead. -func (*AddRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{0} -} - -func (x *AddRequest) GetBody() *AddRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *AddRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type AddResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Response body. - Body *AddResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Response signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *AddResponse) Reset() { - *x = AddResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddResponse) ProtoMessage() {} - -func (x *AddResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddResponse.ProtoReflect.Descriptor instead. -func (*AddResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{1} -} - -func (x *AddResponse) GetBody() *AddResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *AddResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type AddByPathRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Request body. - Body *AddByPathRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Request signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *AddByPathRequest) Reset() { - *x = AddByPathRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddByPathRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddByPathRequest) ProtoMessage() {} - -func (x *AddByPathRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddByPathRequest.ProtoReflect.Descriptor instead. -func (*AddByPathRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{2} -} - -func (x *AddByPathRequest) GetBody() *AddByPathRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *AddByPathRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type AddByPathResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Response body. - Body *AddByPathResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Response signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *AddByPathResponse) Reset() { - *x = AddByPathResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddByPathResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddByPathResponse) ProtoMessage() {} - -func (x *AddByPathResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddByPathResponse.ProtoReflect.Descriptor instead. -func (*AddByPathResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{3} -} - -func (x *AddByPathResponse) GetBody() *AddByPathResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *AddByPathResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type RemoveRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Request body. - Body *RemoveRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Request signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RemoveRequest) Reset() { - *x = RemoveRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveRequest) ProtoMessage() {} - -func (x *RemoveRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveRequest.ProtoReflect.Descriptor instead. -func (*RemoveRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{4} -} - -func (x *RemoveRequest) GetBody() *RemoveRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RemoveRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type RemoveResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Response body. - Body *RemoveResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Response signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *RemoveResponse) Reset() { - *x = RemoveResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveResponse) ProtoMessage() {} - -func (x *RemoveResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveResponse.ProtoReflect.Descriptor instead. -func (*RemoveResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{5} -} - -func (x *RemoveResponse) GetBody() *RemoveResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *RemoveResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type MoveRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Request body. - Body *MoveRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Request signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *MoveRequest) Reset() { - *x = MoveRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MoveRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MoveRequest) ProtoMessage() {} - -func (x *MoveRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MoveRequest.ProtoReflect.Descriptor instead. -func (*MoveRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{6} -} - -func (x *MoveRequest) GetBody() *MoveRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *MoveRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type MoveResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Response body. - Body *MoveResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Response signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *MoveResponse) Reset() { - *x = MoveResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MoveResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MoveResponse) ProtoMessage() {} - -func (x *MoveResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MoveResponse.ProtoReflect.Descriptor instead. -func (*MoveResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{7} -} - -func (x *MoveResponse) GetBody() *MoveResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *MoveResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type GetNodeByPathRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Request body. - Body *GetNodeByPathRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Request signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetNodeByPathRequest) Reset() { - *x = GetNodeByPathRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetNodeByPathRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetNodeByPathRequest) ProtoMessage() {} - -func (x *GetNodeByPathRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetNodeByPathRequest.ProtoReflect.Descriptor instead. -func (*GetNodeByPathRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{8} -} - -func (x *GetNodeByPathRequest) GetBody() *GetNodeByPathRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetNodeByPathRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type GetNodeByPathResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Response body. - Body *GetNodeByPathResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Response signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetNodeByPathResponse) Reset() { - *x = GetNodeByPathResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetNodeByPathResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetNodeByPathResponse) ProtoMessage() {} - -func (x *GetNodeByPathResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetNodeByPathResponse.ProtoReflect.Descriptor instead. -func (*GetNodeByPathResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{9} -} - -func (x *GetNodeByPathResponse) GetBody() *GetNodeByPathResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetNodeByPathResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type GetSubTreeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Request body. - Body *GetSubTreeRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Request signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetSubTreeRequest) Reset() { - *x = GetSubTreeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetSubTreeRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetSubTreeRequest) ProtoMessage() {} - -func (x *GetSubTreeRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetSubTreeRequest.ProtoReflect.Descriptor instead. -func (*GetSubTreeRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{10} -} - -func (x *GetSubTreeRequest) GetBody() *GetSubTreeRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetSubTreeRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type GetSubTreeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Response body. - Body *GetSubTreeResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Response signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetSubTreeResponse) Reset() { - *x = GetSubTreeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetSubTreeResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetSubTreeResponse) ProtoMessage() {} - -func (x *GetSubTreeResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetSubTreeResponse.ProtoReflect.Descriptor instead. -func (*GetSubTreeResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{11} -} - -func (x *GetSubTreeResponse) GetBody() *GetSubTreeResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetSubTreeResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type TreeListRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Request body. - Body *TreeListRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Request signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *TreeListRequest) Reset() { - *x = TreeListRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TreeListRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TreeListRequest) ProtoMessage() {} - -func (x *TreeListRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TreeListRequest.ProtoReflect.Descriptor instead. -func (*TreeListRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{12} -} - -func (x *TreeListRequest) GetBody() *TreeListRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *TreeListRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type TreeListResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Response body. - Body *TreeListResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *TreeListResponse) Reset() { - *x = TreeListResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TreeListResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TreeListResponse) ProtoMessage() {} - -func (x *TreeListResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TreeListResponse.ProtoReflect.Descriptor instead. -func (*TreeListResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{13} -} - -func (x *TreeListResponse) GetBody() *TreeListResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *TreeListResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type ApplyRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Request body. - Body *ApplyRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Request signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *ApplyRequest) Reset() { - *x = ApplyRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[14] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ApplyRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ApplyRequest) ProtoMessage() {} - -func (x *ApplyRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ApplyRequest.ProtoReflect.Descriptor instead. -func (*ApplyRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{14} -} - -func (x *ApplyRequest) GetBody() *ApplyRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *ApplyRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type ApplyResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Response body. - Body *ApplyResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Response signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *ApplyResponse) Reset() { - *x = ApplyResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[15] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ApplyResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ApplyResponse) ProtoMessage() {} - -func (x *ApplyResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[15] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ApplyResponse.ProtoReflect.Descriptor instead. -func (*ApplyResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{15} -} - -func (x *ApplyResponse) GetBody() *ApplyResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *ApplyResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type GetOpLogRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Request body. - Body *GetOpLogRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Request signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetOpLogRequest) Reset() { - *x = GetOpLogRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[16] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetOpLogRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetOpLogRequest) ProtoMessage() {} - -func (x *GetOpLogRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[16] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetOpLogRequest.ProtoReflect.Descriptor instead. -func (*GetOpLogRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{16} -} - -func (x *GetOpLogRequest) GetBody() *GetOpLogRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetOpLogRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type GetOpLogResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Response body. - Body *GetOpLogResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Response signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *GetOpLogResponse) Reset() { - *x = GetOpLogResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[17] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetOpLogResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetOpLogResponse) ProtoMessage() {} - -func (x *GetOpLogResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[17] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetOpLogResponse.ProtoReflect.Descriptor instead. -func (*GetOpLogResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{17} -} - -func (x *GetOpLogResponse) GetBody() *GetOpLogResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *GetOpLogResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type HealthcheckResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Response body. - Body *HealthcheckResponse_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Response signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *HealthcheckResponse) Reset() { - *x = HealthcheckResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[18] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthcheckResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthcheckResponse) ProtoMessage() {} - -func (x *HealthcheckResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[18] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthcheckResponse.ProtoReflect.Descriptor instead. -func (*HealthcheckResponse) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{18} -} - -func (x *HealthcheckResponse) GetBody() *HealthcheckResponse_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *HealthcheckResponse) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type HealthcheckRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Request body. - Body *HealthcheckRequest_Body `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - // Request signature. - Signature *Signature `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` -} - -func (x *HealthcheckRequest) Reset() { - *x = HealthcheckRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[19] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthcheckRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthcheckRequest) ProtoMessage() {} - -func (x *HealthcheckRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[19] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthcheckRequest.ProtoReflect.Descriptor instead. -func (*HealthcheckRequest) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{19} -} - -func (x *HealthcheckRequest) GetBody() *HealthcheckRequest_Body { - if x != nil { - return x.Body - } - return nil -} - -func (x *HealthcheckRequest) GetSignature() *Signature { - if x != nil { - return x.Signature - } - return nil -} - -type AddRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Container ID in V2 format. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` - // The name of the tree. - TreeId string `protobuf:"bytes,2,opt,name=tree_id,json=treeId,proto3" json:"tree_id,omitempty"` - // ID of the parent to attach node to. - ParentId uint64 `protobuf:"varint,3,opt,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"` - // Key-Value pairs with meta information. - Meta []*KeyValue `protobuf:"bytes,4,rep,name=meta,proto3" json:"meta,omitempty"` - // Bearer token in V2 format. - BearerToken []byte `protobuf:"bytes,5,opt,name=bearer_token,json=bearerToken,proto3" json:"bearer_token,omitempty"` -} - -func (x *AddRequest_Body) Reset() { - *x = AddRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[20] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddRequest_Body) ProtoMessage() {} - -func (x *AddRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[20] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddRequest_Body.ProtoReflect.Descriptor instead. -func (*AddRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{0, 0} -} - -func (x *AddRequest_Body) GetContainerId() []byte { - if x != nil { - return x.ContainerId - } - return nil -} - -func (x *AddRequest_Body) GetTreeId() string { - if x != nil { - return x.TreeId - } - return "" -} - -func (x *AddRequest_Body) GetParentId() uint64 { - if x != nil { - return x.ParentId - } - return 0 -} - -func (x *AddRequest_Body) GetMeta() []*KeyValue { - if x != nil { - return x.Meta - } - return nil -} - -func (x *AddRequest_Body) GetBearerToken() []byte { - if x != nil { - return x.BearerToken - } - return nil -} - -type AddResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // ID of the created node. - NodeId uint64 `protobuf:"varint,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` -} - -func (x *AddResponse_Body) Reset() { - *x = AddResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[21] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddResponse_Body) ProtoMessage() {} - -func (x *AddResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[21] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddResponse_Body.ProtoReflect.Descriptor instead. -func (*AddResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{1, 0} -} - -func (x *AddResponse_Body) GetNodeId() uint64 { - if x != nil { - return x.NodeId - } - return 0 -} - -type AddByPathRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Container ID in V2 format. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` - // The name of the tree. - TreeId string `protobuf:"bytes,2,opt,name=tree_id,json=treeId,proto3" json:"tree_id,omitempty"` - // Attribute to build path with. Default: "FileName". - PathAttribute string `protobuf:"bytes,3,opt,name=path_attribute,json=pathAttribute,proto3" json:"path_attribute,omitempty"` - // List of path components. - Path []string `protobuf:"bytes,4,rep,name=path,proto3" json:"path,omitempty"` - // Node meta-information. - Meta []*KeyValue `protobuf:"bytes,5,rep,name=meta,proto3" json:"meta,omitempty"` - // Bearer token in V2 format. - BearerToken []byte `protobuf:"bytes,6,opt,name=bearer_token,json=bearerToken,proto3" json:"bearer_token,omitempty"` -} - -func (x *AddByPathRequest_Body) Reset() { - *x = AddByPathRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[22] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddByPathRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddByPathRequest_Body) ProtoMessage() {} - -func (x *AddByPathRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[22] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddByPathRequest_Body.ProtoReflect.Descriptor instead. -func (*AddByPathRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{2, 0} -} - -func (x *AddByPathRequest_Body) GetContainerId() []byte { - if x != nil { - return x.ContainerId - } - return nil -} - -func (x *AddByPathRequest_Body) GetTreeId() string { - if x != nil { - return x.TreeId - } - return "" -} - -func (x *AddByPathRequest_Body) GetPathAttribute() string { - if x != nil { - return x.PathAttribute - } - return "" -} - -func (x *AddByPathRequest_Body) GetPath() []string { - if x != nil { - return x.Path - } - return nil -} - -func (x *AddByPathRequest_Body) GetMeta() []*KeyValue { - if x != nil { - return x.Meta - } - return nil -} - -func (x *AddByPathRequest_Body) GetBearerToken() []byte { - if x != nil { - return x.BearerToken - } - return nil -} - -type AddByPathResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // List of all created nodes. The first one is the leaf. - Nodes []uint64 `protobuf:"varint,1,rep,packed,name=nodes,proto3" json:"nodes,omitempty"` - // ID of the parent node where new nodes were attached. - ParentId uint64 `protobuf:"varint,2,opt,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"` -} - -func (x *AddByPathResponse_Body) Reset() { - *x = AddByPathResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[23] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *AddByPathResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*AddByPathResponse_Body) ProtoMessage() {} - -func (x *AddByPathResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[23] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use AddByPathResponse_Body.ProtoReflect.Descriptor instead. -func (*AddByPathResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{3, 0} -} - -func (x *AddByPathResponse_Body) GetNodes() []uint64 { - if x != nil { - return x.Nodes - } - return nil -} - -func (x *AddByPathResponse_Body) GetParentId() uint64 { - if x != nil { - return x.ParentId - } - return 0 -} - -type RemoveRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Container ID in V2 format. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` - // The name of the tree. - TreeId string `protobuf:"bytes,2,opt,name=tree_id,json=treeId,proto3" json:"tree_id,omitempty"` - // ID of the node to remove. - NodeId uint64 `protobuf:"varint,3,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` - // Bearer token in V2 format. - BearerToken []byte `protobuf:"bytes,4,opt,name=bearer_token,json=bearerToken,proto3" json:"bearer_token,omitempty"` -} - -func (x *RemoveRequest_Body) Reset() { - *x = RemoveRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[24] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveRequest_Body) ProtoMessage() {} - -func (x *RemoveRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[24] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveRequest_Body.ProtoReflect.Descriptor instead. -func (*RemoveRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{4, 0} -} - -func (x *RemoveRequest_Body) GetContainerId() []byte { - if x != nil { - return x.ContainerId - } - return nil -} - -func (x *RemoveRequest_Body) GetTreeId() string { - if x != nil { - return x.TreeId - } - return "" -} - -func (x *RemoveRequest_Body) GetNodeId() uint64 { - if x != nil { - return x.NodeId - } - return 0 -} - -func (x *RemoveRequest_Body) GetBearerToken() []byte { - if x != nil { - return x.BearerToken - } - return nil -} - -type RemoveResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *RemoveResponse_Body) Reset() { - *x = RemoveResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[25] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemoveResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemoveResponse_Body) ProtoMessage() {} - -func (x *RemoveResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[25] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemoveResponse_Body.ProtoReflect.Descriptor instead. -func (*RemoveResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{5, 0} -} - -type MoveRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // TODO import neo.fs.v2.refs.ContainerID directly. - // Container ID in V2 format. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` - // The name of the tree. - TreeId string `protobuf:"bytes,2,opt,name=tree_id,json=treeId,proto3" json:"tree_id,omitempty"` - // ID of the new parent. - ParentId uint64 `protobuf:"varint,3,opt,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"` - // ID of the node to move. - NodeId uint64 `protobuf:"varint,4,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` - // Node meta-information. - Meta []*KeyValue `protobuf:"bytes,5,rep,name=meta,proto3" json:"meta,omitempty"` - // Bearer token in V2 format. - BearerToken []byte `protobuf:"bytes,6,opt,name=bearer_token,json=bearerToken,proto3" json:"bearer_token,omitempty"` -} - -func (x *MoveRequest_Body) Reset() { - *x = MoveRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[26] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MoveRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MoveRequest_Body) ProtoMessage() {} - -func (x *MoveRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[26] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MoveRequest_Body.ProtoReflect.Descriptor instead. -func (*MoveRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{6, 0} -} - -func (x *MoveRequest_Body) GetContainerId() []byte { - if x != nil { - return x.ContainerId - } - return nil -} - -func (x *MoveRequest_Body) GetTreeId() string { - if x != nil { - return x.TreeId - } - return "" -} - -func (x *MoveRequest_Body) GetParentId() uint64 { - if x != nil { - return x.ParentId - } - return 0 -} - -func (x *MoveRequest_Body) GetNodeId() uint64 { - if x != nil { - return x.NodeId - } - return 0 -} - -func (x *MoveRequest_Body) GetMeta() []*KeyValue { - if x != nil { - return x.Meta - } - return nil -} - -func (x *MoveRequest_Body) GetBearerToken() []byte { - if x != nil { - return x.BearerToken - } - return nil -} - -type MoveResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *MoveResponse_Body) Reset() { - *x = MoveResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[27] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MoveResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MoveResponse_Body) ProtoMessage() {} - -func (x *MoveResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[27] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MoveResponse_Body.ProtoReflect.Descriptor instead. -func (*MoveResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{7, 0} -} - -type GetNodeByPathRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Container ID in V2 format. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` - // The name of the tree. - TreeId string `protobuf:"bytes,2,opt,name=tree_id,json=treeId,proto3" json:"tree_id,omitempty"` - // Attribute to build path with. Default: "FileName". - PathAttribute string `protobuf:"bytes,3,opt,name=path_attribute,json=pathAttribute,proto3" json:"path_attribute,omitempty"` - // List of path components. - Path []string `protobuf:"bytes,4,rep,name=path,proto3" json:"path,omitempty"` - // List of attributes to include in response. - Attributes []string `protobuf:"bytes,5,rep,name=attributes,proto3" json:"attributes,omitempty"` - // Flag to return only the latest version of node. - LatestOnly bool `protobuf:"varint,6,opt,name=latest_only,json=latestOnly,proto3" json:"latest_only,omitempty"` - // Flag to return all stored attributes. - AllAttributes bool `protobuf:"varint,7,opt,name=all_attributes,json=allAttributes,proto3" json:"all_attributes,omitempty"` - // Bearer token in V2 format. - BearerToken []byte `protobuf:"bytes,8,opt,name=bearer_token,json=bearerToken,proto3" json:"bearer_token,omitempty"` -} - -func (x *GetNodeByPathRequest_Body) Reset() { - *x = GetNodeByPathRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[28] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetNodeByPathRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetNodeByPathRequest_Body) ProtoMessage() {} - -func (x *GetNodeByPathRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[28] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetNodeByPathRequest_Body.ProtoReflect.Descriptor instead. -func (*GetNodeByPathRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{8, 0} -} - -func (x *GetNodeByPathRequest_Body) GetContainerId() []byte { - if x != nil { - return x.ContainerId - } - return nil -} - -func (x *GetNodeByPathRequest_Body) GetTreeId() string { - if x != nil { - return x.TreeId - } - return "" -} - -func (x *GetNodeByPathRequest_Body) GetPathAttribute() string { - if x != nil { - return x.PathAttribute - } - return "" -} - -func (x *GetNodeByPathRequest_Body) GetPath() []string { - if x != nil { - return x.Path - } - return nil -} - -func (x *GetNodeByPathRequest_Body) GetAttributes() []string { - if x != nil { - return x.Attributes - } - return nil -} - -func (x *GetNodeByPathRequest_Body) GetLatestOnly() bool { - if x != nil { - return x.LatestOnly - } - return false -} - -func (x *GetNodeByPathRequest_Body) GetAllAttributes() bool { - if x != nil { - return x.AllAttributes - } - return false -} - -func (x *GetNodeByPathRequest_Body) GetBearerToken() []byte { - if x != nil { - return x.BearerToken - } - return nil -} - -// Information about a single tree node. -type GetNodeByPathResponse_Info struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Node ID. - NodeId uint64 `protobuf:"varint,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` - // Timestamp of the last operation with the node. - Timestamp uint64 `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - // Node meta-information. - Meta []*KeyValue `protobuf:"bytes,3,rep,name=meta,proto3" json:"meta,omitempty"` - // Parent ID. - ParentId uint64 `protobuf:"varint,4,opt,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"` -} - -func (x *GetNodeByPathResponse_Info) Reset() { - *x = GetNodeByPathResponse_Info{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[29] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetNodeByPathResponse_Info) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetNodeByPathResponse_Info) ProtoMessage() {} - -func (x *GetNodeByPathResponse_Info) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[29] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetNodeByPathResponse_Info.ProtoReflect.Descriptor instead. -func (*GetNodeByPathResponse_Info) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{9, 0} -} - -func (x *GetNodeByPathResponse_Info) GetNodeId() uint64 { - if x != nil { - return x.NodeId - } - return 0 -} - -func (x *GetNodeByPathResponse_Info) GetTimestamp() uint64 { - if x != nil { - return x.Timestamp - } - return 0 -} - -func (x *GetNodeByPathResponse_Info) GetMeta() []*KeyValue { - if x != nil { - return x.Meta - } - return nil -} - -func (x *GetNodeByPathResponse_Info) GetParentId() uint64 { - if x != nil { - return x.ParentId - } - return 0 -} - -type GetNodeByPathResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // List of nodes stored by path. - Nodes []*GetNodeByPathResponse_Info `protobuf:"bytes,1,rep,name=nodes,proto3" json:"nodes,omitempty"` -} - -func (x *GetNodeByPathResponse_Body) Reset() { - *x = GetNodeByPathResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[30] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetNodeByPathResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetNodeByPathResponse_Body) ProtoMessage() {} - -func (x *GetNodeByPathResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[30] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetNodeByPathResponse_Body.ProtoReflect.Descriptor instead. -func (*GetNodeByPathResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{9, 1} -} - -func (x *GetNodeByPathResponse_Body) GetNodes() []*GetNodeByPathResponse_Info { - if x != nil { - return x.Nodes - } - return nil -} - -type GetSubTreeRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Container ID in V2 format. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` - // The name of the tree. - TreeId string `protobuf:"bytes,2,opt,name=tree_id,json=treeId,proto3" json:"tree_id,omitempty"` - // IDs of the root nodes of a subtree forest. - RootId []uint64 `protobuf:"varint,3,rep,name=root_id,json=rootId,proto3" json:"root_id,omitempty"` - // Optional depth of the traversal. Zero means return only root. - // Maximum depth is 10. - Depth uint32 `protobuf:"varint,4,opt,name=depth,proto3" json:"depth,omitempty"` - // Bearer token in V2 format. - BearerToken []byte `protobuf:"bytes,5,opt,name=bearer_token,json=bearerToken,proto3" json:"bearer_token,omitempty"` - // Result ordering. - OrderBy *GetSubTreeRequest_Body_Order `protobuf:"bytes,6,opt,name=order_by,json=orderBy,proto3" json:"order_by,omitempty"` -} - -func (x *GetSubTreeRequest_Body) Reset() { - *x = GetSubTreeRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[31] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetSubTreeRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetSubTreeRequest_Body) ProtoMessage() {} - -func (x *GetSubTreeRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[31] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetSubTreeRequest_Body.ProtoReflect.Descriptor instead. -func (*GetSubTreeRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{10, 0} -} - -func (x *GetSubTreeRequest_Body) GetContainerId() []byte { - if x != nil { - return x.ContainerId - } - return nil -} - -func (x *GetSubTreeRequest_Body) GetTreeId() string { - if x != nil { - return x.TreeId - } - return "" -} - -func (x *GetSubTreeRequest_Body) GetRootId() []uint64 { - if x != nil { - return x.RootId - } - return nil -} - -func (x *GetSubTreeRequest_Body) GetDepth() uint32 { - if x != nil { - return x.Depth - } - return 0 -} - -func (x *GetSubTreeRequest_Body) GetBearerToken() []byte { - if x != nil { - return x.BearerToken - } - return nil -} - -func (x *GetSubTreeRequest_Body) GetOrderBy() *GetSubTreeRequest_Body_Order { - if x != nil { - return x.OrderBy - } - return nil -} - -type GetSubTreeRequest_Body_Order struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Direction GetSubTreeRequest_Body_Order_Direction `protobuf:"varint,1,opt,name=direction,proto3,enum=tree.GetSubTreeRequest_Body_Order_Direction" json:"direction,omitempty"` -} - -func (x *GetSubTreeRequest_Body_Order) Reset() { - *x = GetSubTreeRequest_Body_Order{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[32] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetSubTreeRequest_Body_Order) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetSubTreeRequest_Body_Order) ProtoMessage() {} - -func (x *GetSubTreeRequest_Body_Order) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[32] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetSubTreeRequest_Body_Order.ProtoReflect.Descriptor instead. -func (*GetSubTreeRequest_Body_Order) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{10, 0, 0} -} - -func (x *GetSubTreeRequest_Body_Order) GetDirection() GetSubTreeRequest_Body_Order_Direction { - if x != nil { - return x.Direction - } - return GetSubTreeRequest_Body_Order_None -} - -type GetSubTreeResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // ID of the node. - NodeId []uint64 `protobuf:"varint,1,rep,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` - // ID of the parent. - ParentId []uint64 `protobuf:"varint,2,rep,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"` - // Time node was first added to a tree. - Timestamp []uint64 `protobuf:"varint,3,rep,name=timestamp,proto3" json:"timestamp,omitempty"` - // Node meta-information. - Meta []*KeyValue `protobuf:"bytes,4,rep,name=meta,proto3" json:"meta,omitempty"` -} - -func (x *GetSubTreeResponse_Body) Reset() { - *x = GetSubTreeResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[33] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetSubTreeResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetSubTreeResponse_Body) ProtoMessage() {} - -func (x *GetSubTreeResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[33] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetSubTreeResponse_Body.ProtoReflect.Descriptor instead. -func (*GetSubTreeResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{11, 0} -} - -func (x *GetSubTreeResponse_Body) GetNodeId() []uint64 { - if x != nil { - return x.NodeId - } - return nil -} - -func (x *GetSubTreeResponse_Body) GetParentId() []uint64 { - if x != nil { - return x.ParentId - } - return nil -} - -func (x *GetSubTreeResponse_Body) GetTimestamp() []uint64 { - if x != nil { - return x.Timestamp - } - return nil -} - -func (x *GetSubTreeResponse_Body) GetMeta() []*KeyValue { - if x != nil { - return x.Meta - } - return nil -} - -type TreeListRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Container ID in V2 format. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` -} - -func (x *TreeListRequest_Body) Reset() { - *x = TreeListRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[34] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TreeListRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TreeListRequest_Body) ProtoMessage() {} - -func (x *TreeListRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[34] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TreeListRequest_Body.ProtoReflect.Descriptor instead. -func (*TreeListRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{12, 0} -} - -func (x *TreeListRequest_Body) GetContainerId() []byte { - if x != nil { - return x.ContainerId - } - return nil -} - -type TreeListResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Tree IDs. - Ids []string `protobuf:"bytes,1,rep,name=ids,proto3" json:"ids,omitempty"` -} - -func (x *TreeListResponse_Body) Reset() { - *x = TreeListResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[35] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TreeListResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TreeListResponse_Body) ProtoMessage() {} - -func (x *TreeListResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[35] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TreeListResponse_Body.ProtoReflect.Descriptor instead. -func (*TreeListResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{13, 0} -} - -func (x *TreeListResponse_Body) GetIds() []string { - if x != nil { - return x.Ids - } - return nil -} - -type ApplyRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Container ID in V2 format. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` - // The name of the tree. - TreeId string `protobuf:"bytes,2,opt,name=tree_id,json=treeId,proto3" json:"tree_id,omitempty"` - // Operation to be applied. - Operation *LogMove `protobuf:"bytes,3,opt,name=operation,proto3" json:"operation,omitempty"` -} - -func (x *ApplyRequest_Body) Reset() { - *x = ApplyRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[36] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ApplyRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ApplyRequest_Body) ProtoMessage() {} - -func (x *ApplyRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[36] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ApplyRequest_Body.ProtoReflect.Descriptor instead. -func (*ApplyRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{14, 0} -} - -func (x *ApplyRequest_Body) GetContainerId() []byte { - if x != nil { - return x.ContainerId - } - return nil -} - -func (x *ApplyRequest_Body) GetTreeId() string { - if x != nil { - return x.TreeId - } - return "" -} - -func (x *ApplyRequest_Body) GetOperation() *LogMove { - if x != nil { - return x.Operation - } - return nil -} - -type ApplyResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *ApplyResponse_Body) Reset() { - *x = ApplyResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[37] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ApplyResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ApplyResponse_Body) ProtoMessage() {} - -func (x *ApplyResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[37] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ApplyResponse_Body.ProtoReflect.Descriptor instead. -func (*ApplyResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{15, 0} -} - -type GetOpLogRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Container ID in V2 format. - ContainerId []byte `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` - // The name of the tree. - TreeId string `protobuf:"bytes,2,opt,name=tree_id,json=treeId,proto3" json:"tree_id,omitempty"` - // Starting height to return logs from. - Height uint64 `protobuf:"varint,3,opt,name=height,proto3" json:"height,omitempty"` - // Amount of operations to return. - Count uint64 `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"` -} - -func (x *GetOpLogRequest_Body) Reset() { - *x = GetOpLogRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[38] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetOpLogRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetOpLogRequest_Body) ProtoMessage() {} - -func (x *GetOpLogRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[38] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetOpLogRequest_Body.ProtoReflect.Descriptor instead. -func (*GetOpLogRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{16, 0} -} - -func (x *GetOpLogRequest_Body) GetContainerId() []byte { - if x != nil { - return x.ContainerId - } - return nil -} - -func (x *GetOpLogRequest_Body) GetTreeId() string { - if x != nil { - return x.TreeId - } - return "" -} - -func (x *GetOpLogRequest_Body) GetHeight() uint64 { - if x != nil { - return x.Height - } - return 0 -} - -func (x *GetOpLogRequest_Body) GetCount() uint64 { - if x != nil { - return x.Count - } - return 0 -} - -type GetOpLogResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Operation on a tree. - Operation *LogMove `protobuf:"bytes,1,opt,name=operation,proto3" json:"operation,omitempty"` -} - -func (x *GetOpLogResponse_Body) Reset() { - *x = GetOpLogResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[39] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetOpLogResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetOpLogResponse_Body) ProtoMessage() {} - -func (x *GetOpLogResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[39] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetOpLogResponse_Body.ProtoReflect.Descriptor instead. -func (*GetOpLogResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{17, 0} -} - -func (x *GetOpLogResponse_Body) GetOperation() *LogMove { - if x != nil { - return x.Operation - } - return nil -} - -type HealthcheckResponse_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *HealthcheckResponse_Body) Reset() { - *x = HealthcheckResponse_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[40] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthcheckResponse_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthcheckResponse_Body) ProtoMessage() {} - -func (x *HealthcheckResponse_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[40] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthcheckResponse_Body.ProtoReflect.Descriptor instead. -func (*HealthcheckResponse_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{18, 0} -} - -type HealthcheckRequest_Body struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *HealthcheckRequest_Body) Reset() { - *x = HealthcheckRequest_Body{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_service_proto_msgTypes[41] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HealthcheckRequest_Body) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HealthcheckRequest_Body) ProtoMessage() {} - -func (x *HealthcheckRequest_Body) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_service_proto_msgTypes[41] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HealthcheckRequest_Body.ProtoReflect.Descriptor instead. -func (*HealthcheckRequest_Body) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_service_proto_rawDescGZIP(), []int{19, 0} -} - -var File_pkg_services_tree_service_proto protoreflect.FileDescriptor - -var file_pkg_services_tree_service_proto_rawDesc = []byte{ - 0x0a, 0x1f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x74, - 0x72, 0x65, 0x65, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x12, 0x04, 0x74, 0x72, 0x65, 0x65, 0x1a, 0x1d, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x65, 0x65, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8f, 0x02, 0x0a, 0x0a, 0x41, 0x64, 0x64, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, 0x64, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, - 0xa6, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, - 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, - 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, - 0x65, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, - 0x64, 0x12, 0x22, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x0e, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, - 0x04, 0x6d, 0x65, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x5f, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x62, 0x65, 0x61, - 0x72, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x89, 0x01, 0x0a, 0x0b, 0x41, 0x64, 0x64, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, - 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x1a, 0x1f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x17, 0x0a, 0x07, 0x6e, - 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6e, 0x6f, - 0x64, 0x65, 0x49, 0x64, 0x22, 0xb9, 0x02, 0x0a, 0x10, 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, - 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, - 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, - 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xc4, 0x01, 0x0a, 0x04, 0x42, 0x6f, - 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x25, - 0x0a, 0x0e, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x70, 0x61, 0x74, 0x68, 0x41, 0x74, 0x74, 0x72, - 0x69, 0x62, 0x75, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x04, 0x6d, 0x65, 0x74, - 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4b, - 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x12, 0x21, 0x0a, - 0x0c, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, - 0x22, 0xaf, 0x01, 0x0a, 0x11, 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, 0x64, 0x42, - 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, - 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x39, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x04, 0x52, 0x05, - 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x49, 0x64, 0x22, 0xec, 0x01, 0x0a, 0x0d, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x7e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, - 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, - 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x21, - 0x0a, 0x0c, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x22, 0x76, 0x0a, 0x0e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x19, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xaa, 0x02, 0x0a, 0x0b, 0x4d, 0x6f, - 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4d, - 0x6f, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, - 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x1a, 0xbf, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, - 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, - 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x72, - 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x70, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, - 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, - 0x22, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, - 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x6d, - 0x65, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x5f, 0x74, 0x6f, - 0x6b, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, - 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x72, 0x0a, 0x0c, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4d, 0x6f, 0x76, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0x85, 0x03, 0x0a, 0x14, 0x47, - 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x33, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, - 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, - 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x88, 0x02, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, - 0x70, 0x61, 0x74, 0x68, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x70, 0x61, 0x74, 0x68, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, - 0x75, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, - 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x74, 0x74, - 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x61, 0x74, 0x65, 0x73, - 0x74, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x6c, 0x61, - 0x74, 0x65, 0x73, 0x74, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x61, 0x6c, 0x6c, 0x5f, - 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0d, 0x61, 0x6c, 0x6c, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x12, - 0x21, 0x0a, 0x0c, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, 0x6b, - 0x65, 0x6e, 0x22, 0xbc, 0x02, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, - 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x04, - 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x72, 0x65, - 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x7e, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, - 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, - 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x12, 0x22, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, - 0x6d, 0x65, 0x74, 0x61, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, - 0x64, 0x1a, 0x3e, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x36, 0x0a, 0x05, 0x6e, 0x6f, 0x64, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, - 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, - 0x73, 0x22, 0xc3, 0x03, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x30, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, - 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, - 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0xcc, 0x02, 0x0a, 0x04, 0x42, 0x6f, 0x64, - 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, - 0x07, 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x04, 0x42, 0x02, - 0x10, 0x00, 0x52, 0x06, 0x72, 0x6f, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, - 0x70, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x64, 0x65, 0x70, 0x74, 0x68, - 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x12, 0x3d, 0x0a, 0x08, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x5f, 0x62, 0x79, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, - 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x2e, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x52, 0x07, 0x6f, 0x72, 0x64, 0x65, 0x72, - 0x42, 0x79, 0x1a, 0x73, 0x0a, 0x05, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x4a, 0x0a, 0x09, 0x64, - 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x4f, 0x72, 0x64, - 0x65, 0x72, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x64, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x1e, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x07, - 0x0a, 0x03, 0x41, 0x73, 0x63, 0x10, 0x01, 0x22, 0x83, 0x02, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x53, - 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, - 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x74, - 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x8a, 0x01, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x1b, 0x0a, 0x07, 0x6e, 0x6f, 0x64, - 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x04, 0x42, 0x02, 0x10, 0x00, 0x52, 0x06, - 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x04, 0x42, 0x02, 0x10, 0x00, 0x52, 0x08, 0x70, - 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x20, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x03, 0x28, 0x04, 0x42, 0x02, 0x10, 0x00, 0x52, 0x09, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x22, 0x0a, 0x04, 0x6d, 0x65, 0x74, - 0x61, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4b, - 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x22, 0x9b, 0x01, - 0x0a, 0x0f, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x2e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x29, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, - 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0x8c, 0x01, 0x0a, 0x10, - 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x1a, 0x18, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x64, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x69, 0x64, 0x73, 0x22, 0xdb, 0x01, 0x0a, 0x0c, 0x41, - 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, - 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, - 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, - 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x6f, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, - 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x09, 0x6f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4c, 0x6f, 0x67, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x09, 0x6f, - 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x74, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x6c, - 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x62, 0x6f, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, - 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, 0x6f, 0x64, - 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, - 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x22, 0xe2, - 0x01, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x2e, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, - 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x1a, 0x70, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, - 0x74, 0x72, 0x65, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, - 0x72, 0x65, 0x65, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x14, 0x0a, - 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x22, 0xa7, 0x01, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, - 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, - 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x33, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x12, 0x2b, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4c, 0x6f, 0x67, 0x4d, 0x6f, - 0x76, 0x65, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x80, 0x01, - 0x0a, 0x13, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, - 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x22, 0x7e, 0x0a, 0x12, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x31, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, - 0x6f, 0x64, 0x79, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, - 0x72, 0x65, 0x65, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x1a, 0x06, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, - 0x32, 0xd6, 0x04, 0x0a, 0x0b, 0x54, 0x72, 0x65, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x12, 0x2a, 0x0a, 0x03, 0x41, 0x64, 0x64, 0x12, 0x10, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, - 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x74, 0x72, 0x65, 0x65, - 0x2e, 0x41, 0x64, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x09, - 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x12, 0x16, 0x2e, 0x74, 0x72, 0x65, 0x65, - 0x2e, 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x64, 0x64, 0x42, 0x79, 0x50, 0x61, - 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x06, 0x52, 0x65, - 0x6d, 0x6f, 0x76, 0x65, 0x12, 0x13, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x52, 0x65, 0x6d, 0x6f, - 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x74, 0x72, 0x65, 0x65, - 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x2d, 0x0a, 0x04, 0x4d, 0x6f, 0x76, 0x65, 0x12, 0x11, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x4d, - 0x6f, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x74, 0x72, 0x65, - 0x65, 0x2e, 0x4d, 0x6f, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, - 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, 0x12, - 0x1a, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, - 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x74, 0x72, - 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x79, 0x50, 0x61, 0x74, 0x68, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x53, - 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x12, 0x17, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, - 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x54, 0x72, 0x65, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x39, 0x0a, 0x08, 0x54, - 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x15, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, - 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x54, 0x72, 0x65, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x12, - 0x12, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x4f, - 0x70, 0x4c, 0x6f, 0x67, 0x12, 0x15, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, - 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x74, 0x72, - 0x65, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x70, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, - 0x68, 0x65, 0x63, 0x6b, 0x12, 0x18, 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, - 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, - 0x2e, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x63, 0x68, 0x65, 0x63, - 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, - 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, - 0x75, 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, - 0x66, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x65, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, -} - -var ( - file_pkg_services_tree_service_proto_rawDescOnce sync.Once - file_pkg_services_tree_service_proto_rawDescData = file_pkg_services_tree_service_proto_rawDesc -) - -func file_pkg_services_tree_service_proto_rawDescGZIP() []byte { - file_pkg_services_tree_service_proto_rawDescOnce.Do(func() { - file_pkg_services_tree_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_services_tree_service_proto_rawDescData) - }) - return file_pkg_services_tree_service_proto_rawDescData -} - -var file_pkg_services_tree_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_pkg_services_tree_service_proto_msgTypes = make([]protoimpl.MessageInfo, 42) -var file_pkg_services_tree_service_proto_goTypes = []interface{}{ - (GetSubTreeRequest_Body_Order_Direction)(0), // 0: tree.GetSubTreeRequest.Body.Order.Direction - (*AddRequest)(nil), // 1: tree.AddRequest - (*AddResponse)(nil), // 2: tree.AddResponse - (*AddByPathRequest)(nil), // 3: tree.AddByPathRequest - (*AddByPathResponse)(nil), // 4: tree.AddByPathResponse - (*RemoveRequest)(nil), // 5: tree.RemoveRequest - (*RemoveResponse)(nil), // 6: tree.RemoveResponse - (*MoveRequest)(nil), // 7: tree.MoveRequest - (*MoveResponse)(nil), // 8: tree.MoveResponse - (*GetNodeByPathRequest)(nil), // 9: tree.GetNodeByPathRequest - (*GetNodeByPathResponse)(nil), // 10: tree.GetNodeByPathResponse - (*GetSubTreeRequest)(nil), // 11: tree.GetSubTreeRequest - (*GetSubTreeResponse)(nil), // 12: tree.GetSubTreeResponse - (*TreeListRequest)(nil), // 13: tree.TreeListRequest - (*TreeListResponse)(nil), // 14: tree.TreeListResponse - (*ApplyRequest)(nil), // 15: tree.ApplyRequest - (*ApplyResponse)(nil), // 16: tree.ApplyResponse - (*GetOpLogRequest)(nil), // 17: tree.GetOpLogRequest - (*GetOpLogResponse)(nil), // 18: tree.GetOpLogResponse - (*HealthcheckResponse)(nil), // 19: tree.HealthcheckResponse - (*HealthcheckRequest)(nil), // 20: tree.HealthcheckRequest - (*AddRequest_Body)(nil), // 21: tree.AddRequest.Body - (*AddResponse_Body)(nil), // 22: tree.AddResponse.Body - (*AddByPathRequest_Body)(nil), // 23: tree.AddByPathRequest.Body - (*AddByPathResponse_Body)(nil), // 24: tree.AddByPathResponse.Body - (*RemoveRequest_Body)(nil), // 25: tree.RemoveRequest.Body - (*RemoveResponse_Body)(nil), // 26: tree.RemoveResponse.Body - (*MoveRequest_Body)(nil), // 27: tree.MoveRequest.Body - (*MoveResponse_Body)(nil), // 28: tree.MoveResponse.Body - (*GetNodeByPathRequest_Body)(nil), // 29: tree.GetNodeByPathRequest.Body - (*GetNodeByPathResponse_Info)(nil), // 30: tree.GetNodeByPathResponse.Info - (*GetNodeByPathResponse_Body)(nil), // 31: tree.GetNodeByPathResponse.Body - (*GetSubTreeRequest_Body)(nil), // 32: tree.GetSubTreeRequest.Body - (*GetSubTreeRequest_Body_Order)(nil), // 33: tree.GetSubTreeRequest.Body.Order - (*GetSubTreeResponse_Body)(nil), // 34: tree.GetSubTreeResponse.Body - (*TreeListRequest_Body)(nil), // 35: tree.TreeListRequest.Body - (*TreeListResponse_Body)(nil), // 36: tree.TreeListResponse.Body - (*ApplyRequest_Body)(nil), // 37: tree.ApplyRequest.Body - (*ApplyResponse_Body)(nil), // 38: tree.ApplyResponse.Body - (*GetOpLogRequest_Body)(nil), // 39: tree.GetOpLogRequest.Body - (*GetOpLogResponse_Body)(nil), // 40: tree.GetOpLogResponse.Body - (*HealthcheckResponse_Body)(nil), // 41: tree.HealthcheckResponse.Body - (*HealthcheckRequest_Body)(nil), // 42: tree.HealthcheckRequest.Body - (*Signature)(nil), // 43: tree.Signature - (*KeyValue)(nil), // 44: tree.KeyValue - (*LogMove)(nil), // 45: tree.LogMove -} -var file_pkg_services_tree_service_proto_depIdxs = []int32{ - 21, // 0: tree.AddRequest.body:type_name -> tree.AddRequest.Body - 43, // 1: tree.AddRequest.signature:type_name -> tree.Signature - 22, // 2: tree.AddResponse.body:type_name -> tree.AddResponse.Body - 43, // 3: tree.AddResponse.signature:type_name -> tree.Signature - 23, // 4: tree.AddByPathRequest.body:type_name -> tree.AddByPathRequest.Body - 43, // 5: tree.AddByPathRequest.signature:type_name -> tree.Signature - 24, // 6: tree.AddByPathResponse.body:type_name -> tree.AddByPathResponse.Body - 43, // 7: tree.AddByPathResponse.signature:type_name -> tree.Signature - 25, // 8: tree.RemoveRequest.body:type_name -> tree.RemoveRequest.Body - 43, // 9: tree.RemoveRequest.signature:type_name -> tree.Signature - 26, // 10: tree.RemoveResponse.body:type_name -> tree.RemoveResponse.Body - 43, // 11: tree.RemoveResponse.signature:type_name -> tree.Signature - 27, // 12: tree.MoveRequest.body:type_name -> tree.MoveRequest.Body - 43, // 13: tree.MoveRequest.signature:type_name -> tree.Signature - 28, // 14: tree.MoveResponse.body:type_name -> tree.MoveResponse.Body - 43, // 15: tree.MoveResponse.signature:type_name -> tree.Signature - 29, // 16: tree.GetNodeByPathRequest.body:type_name -> tree.GetNodeByPathRequest.Body - 43, // 17: tree.GetNodeByPathRequest.signature:type_name -> tree.Signature - 31, // 18: tree.GetNodeByPathResponse.body:type_name -> tree.GetNodeByPathResponse.Body - 43, // 19: tree.GetNodeByPathResponse.signature:type_name -> tree.Signature - 32, // 20: tree.GetSubTreeRequest.body:type_name -> tree.GetSubTreeRequest.Body - 43, // 21: tree.GetSubTreeRequest.signature:type_name -> tree.Signature - 34, // 22: tree.GetSubTreeResponse.body:type_name -> tree.GetSubTreeResponse.Body - 43, // 23: tree.GetSubTreeResponse.signature:type_name -> tree.Signature - 35, // 24: tree.TreeListRequest.body:type_name -> tree.TreeListRequest.Body - 43, // 25: tree.TreeListRequest.signature:type_name -> tree.Signature - 36, // 26: tree.TreeListResponse.body:type_name -> tree.TreeListResponse.Body - 43, // 27: tree.TreeListResponse.signature:type_name -> tree.Signature - 37, // 28: tree.ApplyRequest.body:type_name -> tree.ApplyRequest.Body - 43, // 29: tree.ApplyRequest.signature:type_name -> tree.Signature - 38, // 30: tree.ApplyResponse.body:type_name -> tree.ApplyResponse.Body - 43, // 31: tree.ApplyResponse.signature:type_name -> tree.Signature - 39, // 32: tree.GetOpLogRequest.body:type_name -> tree.GetOpLogRequest.Body - 43, // 33: tree.GetOpLogRequest.signature:type_name -> tree.Signature - 40, // 34: tree.GetOpLogResponse.body:type_name -> tree.GetOpLogResponse.Body - 43, // 35: tree.GetOpLogResponse.signature:type_name -> tree.Signature - 41, // 36: tree.HealthcheckResponse.body:type_name -> tree.HealthcheckResponse.Body - 43, // 37: tree.HealthcheckResponse.signature:type_name -> tree.Signature - 42, // 38: tree.HealthcheckRequest.body:type_name -> tree.HealthcheckRequest.Body - 43, // 39: tree.HealthcheckRequest.signature:type_name -> tree.Signature - 44, // 40: tree.AddRequest.Body.meta:type_name -> tree.KeyValue - 44, // 41: tree.AddByPathRequest.Body.meta:type_name -> tree.KeyValue - 44, // 42: tree.MoveRequest.Body.meta:type_name -> tree.KeyValue - 44, // 43: tree.GetNodeByPathResponse.Info.meta:type_name -> tree.KeyValue - 30, // 44: tree.GetNodeByPathResponse.Body.nodes:type_name -> tree.GetNodeByPathResponse.Info - 33, // 45: tree.GetSubTreeRequest.Body.order_by:type_name -> tree.GetSubTreeRequest.Body.Order - 0, // 46: tree.GetSubTreeRequest.Body.Order.direction:type_name -> tree.GetSubTreeRequest.Body.Order.Direction - 44, // 47: tree.GetSubTreeResponse.Body.meta:type_name -> tree.KeyValue - 45, // 48: tree.ApplyRequest.Body.operation:type_name -> tree.LogMove - 45, // 49: tree.GetOpLogResponse.Body.operation:type_name -> tree.LogMove - 1, // 50: tree.TreeService.Add:input_type -> tree.AddRequest - 3, // 51: tree.TreeService.AddByPath:input_type -> tree.AddByPathRequest - 5, // 52: tree.TreeService.Remove:input_type -> tree.RemoveRequest - 7, // 53: tree.TreeService.Move:input_type -> tree.MoveRequest - 9, // 54: tree.TreeService.GetNodeByPath:input_type -> tree.GetNodeByPathRequest - 11, // 55: tree.TreeService.GetSubTree:input_type -> tree.GetSubTreeRequest - 13, // 56: tree.TreeService.TreeList:input_type -> tree.TreeListRequest - 15, // 57: tree.TreeService.Apply:input_type -> tree.ApplyRequest - 17, // 58: tree.TreeService.GetOpLog:input_type -> tree.GetOpLogRequest - 20, // 59: tree.TreeService.Healthcheck:input_type -> tree.HealthcheckRequest - 2, // 60: tree.TreeService.Add:output_type -> tree.AddResponse - 4, // 61: tree.TreeService.AddByPath:output_type -> tree.AddByPathResponse - 6, // 62: tree.TreeService.Remove:output_type -> tree.RemoveResponse - 8, // 63: tree.TreeService.Move:output_type -> tree.MoveResponse - 10, // 64: tree.TreeService.GetNodeByPath:output_type -> tree.GetNodeByPathResponse - 12, // 65: tree.TreeService.GetSubTree:output_type -> tree.GetSubTreeResponse - 14, // 66: tree.TreeService.TreeList:output_type -> tree.TreeListResponse - 16, // 67: tree.TreeService.Apply:output_type -> tree.ApplyResponse - 18, // 68: tree.TreeService.GetOpLog:output_type -> tree.GetOpLogResponse - 19, // 69: tree.TreeService.Healthcheck:output_type -> tree.HealthcheckResponse - 60, // [60:70] is the sub-list for method output_type - 50, // [50:60] is the sub-list for method input_type - 50, // [50:50] is the sub-list for extension type_name - 50, // [50:50] is the sub-list for extension extendee - 0, // [0:50] is the sub-list for field type_name -} - -func init() { file_pkg_services_tree_service_proto_init() } -func file_pkg_services_tree_service_proto_init() { - if File_pkg_services_tree_service_proto != nil { - return - } - file_pkg_services_tree_types_proto_init() - if !protoimpl.UnsafeEnabled { - file_pkg_services_tree_service_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddByPathRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddByPathResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MoveRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MoveResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetNodeByPathRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetNodeByPathResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetSubTreeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetSubTreeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TreeListRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TreeListResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ApplyRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ApplyResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOpLogRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOpLogResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthcheckResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthcheckRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddByPathRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AddByPathResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemoveResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MoveRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MoveResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetNodeByPathRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetNodeByPathResponse_Info); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetNodeByPathResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetSubTreeRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetSubTreeRequest_Body_Order); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetSubTreeResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TreeListRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TreeListResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ApplyRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ApplyResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOpLogRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOpLogResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthcheckResponse_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HealthcheckRequest_Body); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_pkg_services_tree_service_proto_rawDesc, - NumEnums: 1, - NumMessages: 42, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_pkg_services_tree_service_proto_goTypes, - DependencyIndexes: file_pkg_services_tree_service_proto_depIdxs, - EnumInfos: file_pkg_services_tree_service_proto_enumTypes, - MessageInfos: file_pkg_services_tree_service_proto_msgTypes, - }.Build() - File_pkg_services_tree_service_proto = out.File - file_pkg_services_tree_service_proto_rawDesc = nil - file_pkg_services_tree_service_proto_goTypes = nil - file_pkg_services_tree_service_proto_depIdxs = nil -} diff --git a/pkg/services/tree/service_frostfs.pb.go b/pkg/services/tree/service_frostfs.pb.go index 1a49c5c0c..c4d44253d 100644 --- a/pkg/services/tree/service_frostfs.pb.go +++ b/pkg/services/tree/service_frostfs.pb.go @@ -3,11 +3,32 @@ package tree import ( - binary "encoding/binary" + json "encoding/json" + fmt "fmt" + pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + easyproto "github.com/VictoriaMetrics/easyproto" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" protowire "google.golang.org/protobuf/encoding/protowire" + strconv "strconv" ) -import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" +type AddRequest_Body struct { + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + ParentId uint64 `json:"parentId"` + Meta []*KeyValue `json:"meta"` + BearerToken []byte `json:"bearerToken"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*AddRequest_Body)(nil) + _ json.Marshaler = (*AddRequest_Body)(nil) + _ json.Unmarshaler = (*AddRequest_Body)(nil) +) // StableSize returns the size of x in protobuf format. // @@ -26,32 +47,262 @@ func (x *AddRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) - offset += proto.StringMarshal(2, buf[offset:], x.TreeId) - offset += proto.UInt64Marshal(3, buf[offset:], x.ParentId) - for i := range x.Meta { - offset += proto.NestedStructureMarshal(4, buf[offset:], x.Meta[i]) - } - offset += proto.BytesMarshal(5, buf[offset:], x.BearerToken) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ContainerId) != 0 { + mm.AppendBytes(1, x.ContainerId) + } + if len(x.TreeId) != 0 { + mm.AppendString(2, x.TreeId) + } + if x.ParentId != 0 { + mm.AppendUint64(3, x.ParentId) + } + for i := range x.Meta { + if x.Meta[i] != nil && x.Meta[i].StableSize() != 0 { + x.Meta[i].EmitProtobuf(mm.AppendMessage(4)) + } + } + if len(x.BearerToken) != 0 { + mm.AppendBytes(5, x.BearerToken) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddRequest_Body") + } + switch fc.FieldNum { + case 1: // ContainerId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + case 2: // TreeId + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TreeId") + } + x.TreeId = data + case 3: // ParentId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ParentId") + } + x.ParentId = data + case 4: // Meta + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Meta") + } + x.Meta = append(x.Meta, new(KeyValue)) + ff := x.Meta[len(x.Meta)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 5: // BearerToken + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "BearerToken") + } + x.BearerToken = data + } + } + return nil +} +func (x *AddRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} +func (x *AddRequest_Body) SetContainerId(v []byte) { + x.ContainerId = v +} +func (x *AddRequest_Body) GetTreeId() string { + if x != nil { + return x.TreeId + } + return "" +} +func (x *AddRequest_Body) SetTreeId(v string) { + x.TreeId = v +} +func (x *AddRequest_Body) GetParentId() uint64 { + if x != nil { + return x.ParentId + } + return 0 +} +func (x *AddRequest_Body) SetParentId(v uint64) { + x.ParentId = v +} +func (x *AddRequest_Body) GetMeta() []*KeyValue { + if x != nil { + return x.Meta + } + return nil +} +func (x *AddRequest_Body) SetMeta(v []*KeyValue) { + x.Meta = v +} +func (x *AddRequest_Body) GetBearerToken() []byte { + if x != nil { + return x.BearerToken + } + return nil +} +func (x *AddRequest_Body) SetBearerToken(v []byte) { + x.BearerToken = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"containerId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ContainerId) + } + { + const prefix string = ",\"treeId\":" + out.RawString(prefix) + out.String(x.TreeId) + } + { + const prefix string = ",\"parentId\":" + out.RawString(prefix) + out.Uint64(x.ParentId) + } + { + const prefix string = ",\"meta\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Meta { + if i != 0 { + out.RawByte(',') + } + x.Meta[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + { + const prefix string = ",\"bearerToken\":" + out.RawString(prefix) + out.Base64Bytes(x.BearerToken) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "containerId": + { + var f []byte + f = in.Bytes() + x.ContainerId = f + } + case "treeId": + { + var f string + f = in.String() + x.TreeId = f + } + case "parentId": + { + var f uint64 + f = in.Uint64() + x.ParentId = f + } + case "meta": + { + var f *KeyValue + var list []*KeyValue + in.Delim('[') + for !in.IsDelim(']') { + f = new(KeyValue) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Meta = list + in.Delim(']') + } + case "bearerToken": + { + var f []byte + f = in.Bytes() + x.BearerToken = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type AddRequest struct { + Body *AddRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddRequest)(nil) + _ encoding.ProtoUnmarshaler = (*AddRequest)(nil) + _ json.Marshaler = (*AddRequest)(nil) + _ json.Unmarshaler = (*AddRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -64,27 +315,6 @@ func (x *AddRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -100,13 +330,164 @@ func (x *AddRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *AddRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *AddRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(AddRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *AddRequest) GetBody() *AddRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *AddRequest) SetBody(v *AddRequest_Body) { + x.Body = v +} +func (x *AddRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *AddRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *AddRequest_Body + f = new(AddRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type AddResponse_Body struct { + NodeId uint64 `json:"nodeId"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*AddResponse_Body)(nil) + _ json.Marshaler = (*AddResponse_Body)(nil) + _ json.Unmarshaler = (*AddResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -118,26 +499,125 @@ func (x *AddResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.UInt64Marshal(1, buf[offset:], x.NodeId) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.NodeId != 0 { + mm.AppendUint64(1, x.NodeId) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddResponse_Body") + } + switch fc.FieldNum { + case 1: // NodeId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "NodeId") + } + x.NodeId = data + } + } + return nil +} +func (x *AddResponse_Body) GetNodeId() uint64 { + if x != nil { + return x.NodeId + } + return 0 +} +func (x *AddResponse_Body) SetNodeId(v uint64) { + x.NodeId = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"nodeId\":" + out.RawString(prefix[1:]) + out.Uint64(x.NodeId) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "nodeId": + { + var f uint64 + f = in.Uint64() + x.NodeId = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type AddResponse struct { + Body *AddResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddResponse)(nil) + _ encoding.ProtoUnmarshaler = (*AddResponse)(nil) + _ json.Marshaler = (*AddResponse)(nil) + _ json.Unmarshaler = (*AddResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -150,27 +630,6 @@ func (x *AddResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -186,13 +645,169 @@ func (x *AddResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *AddResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *AddResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(AddResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *AddResponse) GetBody() *AddResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *AddResponse) SetBody(v *AddResponse_Body) { + x.Body = v +} +func (x *AddResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *AddResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *AddResponse_Body + f = new(AddResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type AddByPathRequest_Body struct { + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + PathAttribute string `json:"pathAttribute"` + Path []string `json:"path"` + Meta []*KeyValue `json:"meta"` + BearerToken []byte `json:"bearerToken"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddByPathRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*AddByPathRequest_Body)(nil) + _ json.Marshaler = (*AddByPathRequest_Body)(nil) + _ json.Unmarshaler = (*AddByPathRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -211,33 +826,305 @@ func (x *AddByPathRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddByPathRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) - offset += proto.StringMarshal(2, buf[offset:], x.TreeId) - offset += proto.StringMarshal(3, buf[offset:], x.PathAttribute) - offset += proto.RepeatedStringMarshal(4, buf[offset:], x.Path) - for i := range x.Meta { - offset += proto.NestedStructureMarshal(5, buf[offset:], x.Meta[i]) - } - offset += proto.BytesMarshal(6, buf[offset:], x.BearerToken) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddByPathRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddByPathRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ContainerId) != 0 { + mm.AppendBytes(1, x.ContainerId) + } + if len(x.TreeId) != 0 { + mm.AppendString(2, x.TreeId) + } + if len(x.PathAttribute) != 0 { + mm.AppendString(3, x.PathAttribute) + } + for j := range x.Path { + mm.AppendString(4, x.Path[j]) + } + for i := range x.Meta { + if x.Meta[i] != nil && x.Meta[i].StableSize() != 0 { + x.Meta[i].EmitProtobuf(mm.AppendMessage(5)) + } + } + if len(x.BearerToken) != 0 { + mm.AppendBytes(6, x.BearerToken) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddByPathRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddByPathRequest_Body") + } + switch fc.FieldNum { + case 1: // ContainerId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + case 2: // TreeId + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TreeId") + } + x.TreeId = data + case 3: // PathAttribute + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "PathAttribute") + } + x.PathAttribute = data + case 4: // Path + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Path") + } + x.Path = append(x.Path, data) + case 5: // Meta + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Meta") + } + x.Meta = append(x.Meta, new(KeyValue)) + ff := x.Meta[len(x.Meta)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 6: // BearerToken + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "BearerToken") + } + x.BearerToken = data + } + } + return nil +} +func (x *AddByPathRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} +func (x *AddByPathRequest_Body) SetContainerId(v []byte) { + x.ContainerId = v +} +func (x *AddByPathRequest_Body) GetTreeId() string { + if x != nil { + return x.TreeId + } + return "" +} +func (x *AddByPathRequest_Body) SetTreeId(v string) { + x.TreeId = v +} +func (x *AddByPathRequest_Body) GetPathAttribute() string { + if x != nil { + return x.PathAttribute + } + return "" +} +func (x *AddByPathRequest_Body) SetPathAttribute(v string) { + x.PathAttribute = v +} +func (x *AddByPathRequest_Body) GetPath() []string { + if x != nil { + return x.Path + } + return nil +} +func (x *AddByPathRequest_Body) SetPath(v []string) { + x.Path = v +} +func (x *AddByPathRequest_Body) GetMeta() []*KeyValue { + if x != nil { + return x.Meta + } + return nil +} +func (x *AddByPathRequest_Body) SetMeta(v []*KeyValue) { + x.Meta = v +} +func (x *AddByPathRequest_Body) GetBearerToken() []byte { + if x != nil { + return x.BearerToken + } + return nil +} +func (x *AddByPathRequest_Body) SetBearerToken(v []byte) { + x.BearerToken = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddByPathRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddByPathRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"containerId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ContainerId) + } + { + const prefix string = ",\"treeId\":" + out.RawString(prefix) + out.String(x.TreeId) + } + { + const prefix string = ",\"pathAttribute\":" + out.RawString(prefix) + out.String(x.PathAttribute) + } + { + const prefix string = ",\"path\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Path { + if i != 0 { + out.RawByte(',') + } + out.String(x.Path[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"meta\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Meta { + if i != 0 { + out.RawByte(',') + } + x.Meta[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + { + const prefix string = ",\"bearerToken\":" + out.RawString(prefix) + out.Base64Bytes(x.BearerToken) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddByPathRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddByPathRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "containerId": + { + var f []byte + f = in.Bytes() + x.ContainerId = f + } + case "treeId": + { + var f string + f = in.String() + x.TreeId = f + } + case "pathAttribute": + { + var f string + f = in.String() + x.PathAttribute = f + } + case "path": + { + var f string + var list []string + in.Delim('[') + for !in.IsDelim(']') { + f = in.String() + list = append(list, f) + in.WantComma() + } + x.Path = list + in.Delim(']') + } + case "meta": + { + var f *KeyValue + var list []*KeyValue + in.Delim('[') + for !in.IsDelim(']') { + f = new(KeyValue) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Meta = list + in.Delim(']') + } + case "bearerToken": + { + var f []byte + f = in.Bytes() + x.BearerToken = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type AddByPathRequest struct { + Body *AddByPathRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddByPathRequest)(nil) + _ encoding.ProtoUnmarshaler = (*AddByPathRequest)(nil) + _ json.Marshaler = (*AddByPathRequest)(nil) + _ json.Unmarshaler = (*AddByPathRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -250,27 +1137,6 @@ func (x *AddByPathRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddByPathRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -286,13 +1152,165 @@ func (x *AddByPathRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *AddByPathRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *AddByPathRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddByPathRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddByPathRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddByPathRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddByPathRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(AddByPathRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *AddByPathRequest) GetBody() *AddByPathRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *AddByPathRequest) SetBody(v *AddByPathRequest_Body) { + x.Body = v +} +func (x *AddByPathRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *AddByPathRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddByPathRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddByPathRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddByPathRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddByPathRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *AddByPathRequest_Body + f = new(AddByPathRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type AddByPathResponse_Body struct { + Nodes []uint64 `json:"nodes"` + ParentId uint64 `json:"parentId"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddByPathResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*AddByPathResponse_Body)(nil) + _ json.Marshaler = (*AddByPathResponse_Body)(nil) + _ json.Unmarshaler = (*AddByPathResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -307,27 +1325,168 @@ func (x *AddByPathResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddByPathResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.RepeatedUInt64Marshal(1, buf[offset:], x.Nodes) - offset += proto.UInt64Marshal(2, buf[offset:], x.ParentId) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddByPathResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddByPathResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Nodes) != 0 { + mm.AppendUint64s(1, x.Nodes) + } + if x.ParentId != 0 { + mm.AppendUint64(2, x.ParentId) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddByPathResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddByPathResponse_Body") + } + switch fc.FieldNum { + case 1: // Nodes + data, ok := fc.UnpackUint64s(nil) + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Nodes") + } + x.Nodes = data + case 2: // ParentId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ParentId") + } + x.ParentId = data + } + } + return nil +} +func (x *AddByPathResponse_Body) GetNodes() []uint64 { + if x != nil { + return x.Nodes + } + return nil +} +func (x *AddByPathResponse_Body) SetNodes(v []uint64) { + x.Nodes = v +} +func (x *AddByPathResponse_Body) GetParentId() uint64 { + if x != nil { + return x.ParentId + } + return 0 +} +func (x *AddByPathResponse_Body) SetParentId(v uint64) { + x.ParentId = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddByPathResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddByPathResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"nodes\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Nodes { + if i != 0 { + out.RawByte(',') + } + out.Uint64(x.Nodes[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"parentId\":" + out.RawString(prefix) + out.Uint64(x.ParentId) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddByPathResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddByPathResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "nodes": + { + var f uint64 + var list []uint64 + in.Delim('[') + for !in.IsDelim(']') { + f = in.Uint64() + list = append(list, f) + in.WantComma() + } + x.Nodes = list + in.Delim(']') + } + case "parentId": + { + var f uint64 + f = in.Uint64() + x.ParentId = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type AddByPathResponse struct { + Body *AddByPathResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*AddByPathResponse)(nil) + _ encoding.ProtoUnmarshaler = (*AddByPathResponse)(nil) + _ json.Marshaler = (*AddByPathResponse)(nil) + _ json.Unmarshaler = (*AddByPathResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -340,27 +1499,6 @@ func (x *AddByPathResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *AddByPathResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -376,13 +1514,167 @@ func (x *AddByPathResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *AddByPathResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *AddByPathResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *AddByPathResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *AddByPathResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *AddByPathResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "AddByPathResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(AddByPathResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *AddByPathResponse) GetBody() *AddByPathResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *AddByPathResponse) SetBody(v *AddByPathResponse_Body) { + x.Body = v +} +func (x *AddByPathResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *AddByPathResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *AddByPathResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *AddByPathResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *AddByPathResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *AddByPathResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *AddByPathResponse_Body + f = new(AddByPathResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveRequest_Body struct { + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + NodeId uint64 `json:"nodeId"` + BearerToken []byte `json:"bearerToken"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveRequest_Body)(nil) + _ json.Marshaler = (*RemoveRequest_Body)(nil) + _ json.Unmarshaler = (*RemoveRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -397,29 +1689,212 @@ func (x *RemoveRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) - offset += proto.StringMarshal(2, buf[offset:], x.TreeId) - offset += proto.UInt64Marshal(3, buf[offset:], x.NodeId) - offset += proto.BytesMarshal(4, buf[offset:], x.BearerToken) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ContainerId) != 0 { + mm.AppendBytes(1, x.ContainerId) + } + if len(x.TreeId) != 0 { + mm.AppendString(2, x.TreeId) + } + if x.NodeId != 0 { + mm.AppendUint64(3, x.NodeId) + } + if len(x.BearerToken) != 0 { + mm.AppendBytes(4, x.BearerToken) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveRequest_Body") + } + switch fc.FieldNum { + case 1: // ContainerId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + case 2: // TreeId + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TreeId") + } + x.TreeId = data + case 3: // NodeId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "NodeId") + } + x.NodeId = data + case 4: // BearerToken + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "BearerToken") + } + x.BearerToken = data + } + } + return nil +} +func (x *RemoveRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} +func (x *RemoveRequest_Body) SetContainerId(v []byte) { + x.ContainerId = v +} +func (x *RemoveRequest_Body) GetTreeId() string { + if x != nil { + return x.TreeId + } + return "" +} +func (x *RemoveRequest_Body) SetTreeId(v string) { + x.TreeId = v +} +func (x *RemoveRequest_Body) GetNodeId() uint64 { + if x != nil { + return x.NodeId + } + return 0 +} +func (x *RemoveRequest_Body) SetNodeId(v uint64) { + x.NodeId = v +} +func (x *RemoveRequest_Body) GetBearerToken() []byte { + if x != nil { + return x.BearerToken + } + return nil +} +func (x *RemoveRequest_Body) SetBearerToken(v []byte) { + x.BearerToken = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"containerId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ContainerId) + } + { + const prefix string = ",\"treeId\":" + out.RawString(prefix) + out.String(x.TreeId) + } + { + const prefix string = ",\"nodeId\":" + out.RawString(prefix) + out.Uint64(x.NodeId) + } + { + const prefix string = ",\"bearerToken\":" + out.RawString(prefix) + out.Base64Bytes(x.BearerToken) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "containerId": + { + var f []byte + f = in.Bytes() + x.ContainerId = f + } + case "treeId": + { + var f string + f = in.String() + x.TreeId = f + } + case "nodeId": + { + var f uint64 + f = in.Uint64() + x.NodeId = f + } + case "bearerToken": + { + var f []byte + f = in.Bytes() + x.BearerToken = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveRequest struct { + Body *RemoveRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveRequest)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveRequest)(nil) + _ json.Marshaler = (*RemoveRequest)(nil) + _ json.Unmarshaler = (*RemoveRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -432,27 +1907,6 @@ func (x *RemoveRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -468,13 +1922,163 @@ func (x *RemoveRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *RemoveRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *RemoveRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(RemoveRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *RemoveRequest) GetBody() *RemoveRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *RemoveRequest) SetBody(v *RemoveRequest_Body) { + x.Body = v +} +func (x *RemoveRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *RemoveRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *RemoveRequest_Body + f = new(RemoveRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveResponse_Body)(nil) + _ json.Marshaler = (*RemoveResponse_Body)(nil) + _ json.Unmarshaler = (*RemoveResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -485,18 +2089,93 @@ func (x *RemoveResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type RemoveResponse struct { + Body *RemoveResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*RemoveResponse)(nil) + _ encoding.ProtoUnmarshaler = (*RemoveResponse)(nil) + _ json.Marshaler = (*RemoveResponse)(nil) + _ json.Unmarshaler = (*RemoveResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -509,27 +2188,6 @@ func (x *RemoveResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *RemoveResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -545,13 +2203,169 @@ func (x *RemoveResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *RemoveResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *RemoveResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *RemoveResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *RemoveResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *RemoveResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "RemoveResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(RemoveResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *RemoveResponse) GetBody() *RemoveResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *RemoveResponse) SetBody(v *RemoveResponse_Body) { + x.Body = v +} +func (x *RemoveResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *RemoveResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *RemoveResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *RemoveResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *RemoveResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *RemoveResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *RemoveResponse_Body + f = new(RemoveResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type MoveRequest_Body struct { + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + ParentId uint64 `json:"parentId"` + NodeId uint64 `json:"nodeId"` + Meta []*KeyValue `json:"meta"` + BearerToken []byte `json:"bearerToken"` +} + +var ( + _ encoding.ProtoMarshaler = (*MoveRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*MoveRequest_Body)(nil) + _ json.Marshaler = (*MoveRequest_Body)(nil) + _ json.Unmarshaler = (*MoveRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -570,33 +2384,291 @@ func (x *MoveRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *MoveRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) - offset += proto.StringMarshal(2, buf[offset:], x.TreeId) - offset += proto.UInt64Marshal(3, buf[offset:], x.ParentId) - offset += proto.UInt64Marshal(4, buf[offset:], x.NodeId) - for i := range x.Meta { - offset += proto.NestedStructureMarshal(5, buf[offset:], x.Meta[i]) - } - offset += proto.BytesMarshal(6, buf[offset:], x.BearerToken) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *MoveRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *MoveRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ContainerId) != 0 { + mm.AppendBytes(1, x.ContainerId) + } + if len(x.TreeId) != 0 { + mm.AppendString(2, x.TreeId) + } + if x.ParentId != 0 { + mm.AppendUint64(3, x.ParentId) + } + if x.NodeId != 0 { + mm.AppendUint64(4, x.NodeId) + } + for i := range x.Meta { + if x.Meta[i] != nil && x.Meta[i].StableSize() != 0 { + x.Meta[i].EmitProtobuf(mm.AppendMessage(5)) + } + } + if len(x.BearerToken) != 0 { + mm.AppendBytes(6, x.BearerToken) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *MoveRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "MoveRequest_Body") + } + switch fc.FieldNum { + case 1: // ContainerId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + case 2: // TreeId + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TreeId") + } + x.TreeId = data + case 3: // ParentId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ParentId") + } + x.ParentId = data + case 4: // NodeId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "NodeId") + } + x.NodeId = data + case 5: // Meta + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Meta") + } + x.Meta = append(x.Meta, new(KeyValue)) + ff := x.Meta[len(x.Meta)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 6: // BearerToken + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "BearerToken") + } + x.BearerToken = data + } + } + return nil +} +func (x *MoveRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} +func (x *MoveRequest_Body) SetContainerId(v []byte) { + x.ContainerId = v +} +func (x *MoveRequest_Body) GetTreeId() string { + if x != nil { + return x.TreeId + } + return "" +} +func (x *MoveRequest_Body) SetTreeId(v string) { + x.TreeId = v +} +func (x *MoveRequest_Body) GetParentId() uint64 { + if x != nil { + return x.ParentId + } + return 0 +} +func (x *MoveRequest_Body) SetParentId(v uint64) { + x.ParentId = v +} +func (x *MoveRequest_Body) GetNodeId() uint64 { + if x != nil { + return x.NodeId + } + return 0 +} +func (x *MoveRequest_Body) SetNodeId(v uint64) { + x.NodeId = v +} +func (x *MoveRequest_Body) GetMeta() []*KeyValue { + if x != nil { + return x.Meta + } + return nil +} +func (x *MoveRequest_Body) SetMeta(v []*KeyValue) { + x.Meta = v +} +func (x *MoveRequest_Body) GetBearerToken() []byte { + if x != nil { + return x.BearerToken + } + return nil +} +func (x *MoveRequest_Body) SetBearerToken(v []byte) { + x.BearerToken = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *MoveRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *MoveRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"containerId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ContainerId) + } + { + const prefix string = ",\"treeId\":" + out.RawString(prefix) + out.String(x.TreeId) + } + { + const prefix string = ",\"parentId\":" + out.RawString(prefix) + out.Uint64(x.ParentId) + } + { + const prefix string = ",\"nodeId\":" + out.RawString(prefix) + out.Uint64(x.NodeId) + } + { + const prefix string = ",\"meta\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Meta { + if i != 0 { + out.RawByte(',') + } + x.Meta[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + { + const prefix string = ",\"bearerToken\":" + out.RawString(prefix) + out.Base64Bytes(x.BearerToken) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *MoveRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *MoveRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "containerId": + { + var f []byte + f = in.Bytes() + x.ContainerId = f + } + case "treeId": + { + var f string + f = in.String() + x.TreeId = f + } + case "parentId": + { + var f uint64 + f = in.Uint64() + x.ParentId = f + } + case "nodeId": + { + var f uint64 + f = in.Uint64() + x.NodeId = f + } + case "meta": + { + var f *KeyValue + var list []*KeyValue + in.Delim('[') + for !in.IsDelim(']') { + f = new(KeyValue) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Meta = list + in.Delim(']') + } + case "bearerToken": + { + var f []byte + f = in.Bytes() + x.BearerToken = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type MoveRequest struct { + Body *MoveRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*MoveRequest)(nil) + _ encoding.ProtoUnmarshaler = (*MoveRequest)(nil) + _ json.Marshaler = (*MoveRequest)(nil) + _ json.Unmarshaler = (*MoveRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -609,27 +2681,6 @@ func (x *MoveRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *MoveRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -645,13 +2696,163 @@ func (x *MoveRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *MoveRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *MoveRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *MoveRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *MoveRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *MoveRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "MoveRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(MoveRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *MoveRequest) GetBody() *MoveRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *MoveRequest) SetBody(v *MoveRequest_Body) { + x.Body = v +} +func (x *MoveRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *MoveRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *MoveRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *MoveRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *MoveRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *MoveRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *MoveRequest_Body + f = new(MoveRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type MoveResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*MoveResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*MoveResponse_Body)(nil) + _ json.Marshaler = (*MoveResponse_Body)(nil) + _ json.Unmarshaler = (*MoveResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -662,18 +2863,93 @@ func (x *MoveResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *MoveResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *MoveResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *MoveResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *MoveResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "MoveResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *MoveResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *MoveResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *MoveResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *MoveResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type MoveResponse struct { + Body *MoveResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*MoveResponse)(nil) + _ encoding.ProtoUnmarshaler = (*MoveResponse)(nil) + _ json.Marshaler = (*MoveResponse)(nil) + _ json.Unmarshaler = (*MoveResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -686,27 +2962,6 @@ func (x *MoveResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *MoveResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -722,13 +2977,171 @@ func (x *MoveResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *MoveResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *MoveResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *MoveResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *MoveResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *MoveResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "MoveResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(MoveResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *MoveResponse) GetBody() *MoveResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *MoveResponse) SetBody(v *MoveResponse_Body) { + x.Body = v +} +func (x *MoveResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *MoveResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *MoveResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *MoveResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *MoveResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *MoveResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *MoveResponse_Body + f = new(MoveResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetNodeByPathRequest_Body struct { + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + PathAttribute string `json:"pathAttribute"` + Path []string `json:"path"` + Attributes []string `json:"attributes"` + LatestOnly bool `json:"latestOnly"` + AllAttributes bool `json:"allAttributes"` + BearerToken []byte `json:"bearerToken"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetNodeByPathRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetNodeByPathRequest_Body)(nil) + _ json.Marshaler = (*GetNodeByPathRequest_Body)(nil) + _ json.Unmarshaler = (*GetNodeByPathRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -747,33 +3160,356 @@ func (x *GetNodeByPathRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetNodeByPathRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) - offset += proto.StringMarshal(2, buf[offset:], x.TreeId) - offset += proto.StringMarshal(3, buf[offset:], x.PathAttribute) - offset += proto.RepeatedStringMarshal(4, buf[offset:], x.Path) - offset += proto.RepeatedStringMarshal(5, buf[offset:], x.Attributes) - offset += proto.BoolMarshal(6, buf[offset:], x.LatestOnly) - offset += proto.BoolMarshal(7, buf[offset:], x.AllAttributes) - offset += proto.BytesMarshal(8, buf[offset:], x.BearerToken) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetNodeByPathRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetNodeByPathRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ContainerId) != 0 { + mm.AppendBytes(1, x.ContainerId) + } + if len(x.TreeId) != 0 { + mm.AppendString(2, x.TreeId) + } + if len(x.PathAttribute) != 0 { + mm.AppendString(3, x.PathAttribute) + } + for j := range x.Path { + mm.AppendString(4, x.Path[j]) + } + for j := range x.Attributes { + mm.AppendString(5, x.Attributes[j]) + } + if x.LatestOnly { + mm.AppendBool(6, x.LatestOnly) + } + if x.AllAttributes { + mm.AppendBool(7, x.AllAttributes) + } + if len(x.BearerToken) != 0 { + mm.AppendBytes(8, x.BearerToken) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetNodeByPathRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetNodeByPathRequest_Body") + } + switch fc.FieldNum { + case 1: // ContainerId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + case 2: // TreeId + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TreeId") + } + x.TreeId = data + case 3: // PathAttribute + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "PathAttribute") + } + x.PathAttribute = data + case 4: // Path + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Path") + } + x.Path = append(x.Path, data) + case 5: // Attributes + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Attributes") + } + x.Attributes = append(x.Attributes, data) + case 6: // LatestOnly + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "LatestOnly") + } + x.LatestOnly = data + case 7: // AllAttributes + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "AllAttributes") + } + x.AllAttributes = data + case 8: // BearerToken + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "BearerToken") + } + x.BearerToken = data + } + } + return nil +} +func (x *GetNodeByPathRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} +func (x *GetNodeByPathRequest_Body) SetContainerId(v []byte) { + x.ContainerId = v +} +func (x *GetNodeByPathRequest_Body) GetTreeId() string { + if x != nil { + return x.TreeId + } + return "" +} +func (x *GetNodeByPathRequest_Body) SetTreeId(v string) { + x.TreeId = v +} +func (x *GetNodeByPathRequest_Body) GetPathAttribute() string { + if x != nil { + return x.PathAttribute + } + return "" +} +func (x *GetNodeByPathRequest_Body) SetPathAttribute(v string) { + x.PathAttribute = v +} +func (x *GetNodeByPathRequest_Body) GetPath() []string { + if x != nil { + return x.Path + } + return nil +} +func (x *GetNodeByPathRequest_Body) SetPath(v []string) { + x.Path = v +} +func (x *GetNodeByPathRequest_Body) GetAttributes() []string { + if x != nil { + return x.Attributes + } + return nil +} +func (x *GetNodeByPathRequest_Body) SetAttributes(v []string) { + x.Attributes = v +} +func (x *GetNodeByPathRequest_Body) GetLatestOnly() bool { + if x != nil { + return x.LatestOnly + } + return false +} +func (x *GetNodeByPathRequest_Body) SetLatestOnly(v bool) { + x.LatestOnly = v +} +func (x *GetNodeByPathRequest_Body) GetAllAttributes() bool { + if x != nil { + return x.AllAttributes + } + return false +} +func (x *GetNodeByPathRequest_Body) SetAllAttributes(v bool) { + x.AllAttributes = v +} +func (x *GetNodeByPathRequest_Body) GetBearerToken() []byte { + if x != nil { + return x.BearerToken + } + return nil +} +func (x *GetNodeByPathRequest_Body) SetBearerToken(v []byte) { + x.BearerToken = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetNodeByPathRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetNodeByPathRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"containerId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ContainerId) + } + { + const prefix string = ",\"treeId\":" + out.RawString(prefix) + out.String(x.TreeId) + } + { + const prefix string = ",\"pathAttribute\":" + out.RawString(prefix) + out.String(x.PathAttribute) + } + { + const prefix string = ",\"path\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Path { + if i != 0 { + out.RawByte(',') + } + out.String(x.Path[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"attributes\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Attributes { + if i != 0 { + out.RawByte(',') + } + out.String(x.Attributes[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"latestOnly\":" + out.RawString(prefix) + out.Bool(x.LatestOnly) + } + { + const prefix string = ",\"allAttributes\":" + out.RawString(prefix) + out.Bool(x.AllAttributes) + } + { + const prefix string = ",\"bearerToken\":" + out.RawString(prefix) + out.Base64Bytes(x.BearerToken) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetNodeByPathRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetNodeByPathRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "containerId": + { + var f []byte + f = in.Bytes() + x.ContainerId = f + } + case "treeId": + { + var f string + f = in.String() + x.TreeId = f + } + case "pathAttribute": + { + var f string + f = in.String() + x.PathAttribute = f + } + case "path": + { + var f string + var list []string + in.Delim('[') + for !in.IsDelim(']') { + f = in.String() + list = append(list, f) + in.WantComma() + } + x.Path = list + in.Delim(']') + } + case "attributes": + { + var f string + var list []string + in.Delim('[') + for !in.IsDelim(']') { + f = in.String() + list = append(list, f) + in.WantComma() + } + x.Attributes = list + in.Delim(']') + } + case "latestOnly": + { + var f bool + f = in.Bool() + x.LatestOnly = f + } + case "allAttributes": + { + var f bool + f = in.Bool() + x.AllAttributes = f + } + case "bearerToken": + { + var f []byte + f = in.Bytes() + x.BearerToken = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetNodeByPathRequest struct { + Body *GetNodeByPathRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetNodeByPathRequest)(nil) + _ encoding.ProtoUnmarshaler = (*GetNodeByPathRequest)(nil) + _ json.Marshaler = (*GetNodeByPathRequest)(nil) + _ json.Unmarshaler = (*GetNodeByPathRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -786,27 +3522,6 @@ func (x *GetNodeByPathRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetNodeByPathRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -822,13 +3537,167 @@ func (x *GetNodeByPathRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetNodeByPathRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetNodeByPathRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetNodeByPathRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetNodeByPathRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetNodeByPathRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetNodeByPathRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetNodeByPathRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetNodeByPathRequest) GetBody() *GetNodeByPathRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetNodeByPathRequest) SetBody(v *GetNodeByPathRequest_Body) { + x.Body = v +} +func (x *GetNodeByPathRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetNodeByPathRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetNodeByPathRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetNodeByPathRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetNodeByPathRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetNodeByPathRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetNodeByPathRequest_Body + f = new(GetNodeByPathRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetNodeByPathResponse_Info struct { + NodeId uint64 `json:"nodeId"` + Timestamp uint64 `json:"timestamp"` + Meta []*KeyValue `json:"meta"` + ParentId uint64 `json:"parentId"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetNodeByPathResponse_Info)(nil) + _ encoding.ProtoUnmarshaler = (*GetNodeByPathResponse_Info)(nil) + _ json.Marshaler = (*GetNodeByPathResponse_Info)(nil) + _ json.Unmarshaler = (*GetNodeByPathResponse_Info)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -845,31 +3714,232 @@ func (x *GetNodeByPathResponse_Info) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetNodeByPathResponse_Info) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.UInt64Marshal(1, buf[offset:], x.NodeId) - offset += proto.UInt64Marshal(2, buf[offset:], x.Timestamp) - for i := range x.Meta { - offset += proto.NestedStructureMarshal(3, buf[offset:], x.Meta[i]) - } - offset += proto.UInt64Marshal(4, buf[offset:], x.ParentId) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetNodeByPathResponse_Info) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetNodeByPathResponse_Info) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.NodeId != 0 { + mm.AppendUint64(1, x.NodeId) + } + if x.Timestamp != 0 { + mm.AppendUint64(2, x.Timestamp) + } + for i := range x.Meta { + if x.Meta[i] != nil && x.Meta[i].StableSize() != 0 { + x.Meta[i].EmitProtobuf(mm.AppendMessage(3)) + } + } + if x.ParentId != 0 { + mm.AppendUint64(4, x.ParentId) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetNodeByPathResponse_Info) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetNodeByPathResponse_Info") + } + switch fc.FieldNum { + case 1: // NodeId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "NodeId") + } + x.NodeId = data + case 2: // Timestamp + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Timestamp") + } + x.Timestamp = data + case 3: // Meta + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Meta") + } + x.Meta = append(x.Meta, new(KeyValue)) + ff := x.Meta[len(x.Meta)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 4: // ParentId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ParentId") + } + x.ParentId = data + } + } + return nil +} +func (x *GetNodeByPathResponse_Info) GetNodeId() uint64 { + if x != nil { + return x.NodeId + } + return 0 +} +func (x *GetNodeByPathResponse_Info) SetNodeId(v uint64) { + x.NodeId = v +} +func (x *GetNodeByPathResponse_Info) GetTimestamp() uint64 { + if x != nil { + return x.Timestamp + } + return 0 +} +func (x *GetNodeByPathResponse_Info) SetTimestamp(v uint64) { + x.Timestamp = v +} +func (x *GetNodeByPathResponse_Info) GetMeta() []*KeyValue { + if x != nil { + return x.Meta + } + return nil +} +func (x *GetNodeByPathResponse_Info) SetMeta(v []*KeyValue) { + x.Meta = v +} +func (x *GetNodeByPathResponse_Info) GetParentId() uint64 { + if x != nil { + return x.ParentId + } + return 0 +} +func (x *GetNodeByPathResponse_Info) SetParentId(v uint64) { + x.ParentId = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetNodeByPathResponse_Info) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetNodeByPathResponse_Info) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"nodeId\":" + out.RawString(prefix[1:]) + out.Uint64(x.NodeId) + } + { + const prefix string = ",\"timestamp\":" + out.RawString(prefix) + out.Uint64(x.Timestamp) + } + { + const prefix string = ",\"meta\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Meta { + if i != 0 { + out.RawByte(',') + } + x.Meta[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + { + const prefix string = ",\"parentId\":" + out.RawString(prefix) + out.Uint64(x.ParentId) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetNodeByPathResponse_Info) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetNodeByPathResponse_Info) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "nodeId": + { + var f uint64 + f = in.Uint64() + x.NodeId = f + } + case "timestamp": + { + var f uint64 + f = in.Uint64() + x.Timestamp = f + } + case "meta": + { + var f *KeyValue + var list []*KeyValue + in.Delim('[') + for !in.IsDelim(']') { + f = new(KeyValue) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Meta = list + in.Delim(']') + } + case "parentId": + { + var f uint64 + f = in.Uint64() + x.ParentId = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetNodeByPathResponse_Body struct { + Nodes []*GetNodeByPathResponse_Info `json:"nodes"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetNodeByPathResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetNodeByPathResponse_Body)(nil) + _ json.Marshaler = (*GetNodeByPathResponse_Body)(nil) + _ json.Unmarshaler = (*GetNodeByPathResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -883,28 +3953,146 @@ func (x *GetNodeByPathResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetNodeByPathResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - for i := range x.Nodes { - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Nodes[i]) - } - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetNodeByPathResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetNodeByPathResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for i := range x.Nodes { + if x.Nodes[i] != nil && x.Nodes[i].StableSize() != 0 { + x.Nodes[i].EmitProtobuf(mm.AppendMessage(1)) + } + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetNodeByPathResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetNodeByPathResponse_Body") + } + switch fc.FieldNum { + case 1: // Nodes + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Nodes") + } + x.Nodes = append(x.Nodes, new(GetNodeByPathResponse_Info)) + ff := x.Nodes[len(x.Nodes)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetNodeByPathResponse_Body) GetNodes() []*GetNodeByPathResponse_Info { + if x != nil { + return x.Nodes + } + return nil +} +func (x *GetNodeByPathResponse_Body) SetNodes(v []*GetNodeByPathResponse_Info) { + x.Nodes = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetNodeByPathResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetNodeByPathResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"nodes\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Nodes { + if i != 0 { + out.RawByte(',') + } + x.Nodes[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetNodeByPathResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetNodeByPathResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "nodes": + { + var f *GetNodeByPathResponse_Info + var list []*GetNodeByPathResponse_Info + in.Delim('[') + for !in.IsDelim(']') { + f = new(GetNodeByPathResponse_Info) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Nodes = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetNodeByPathResponse struct { + Body *GetNodeByPathResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetNodeByPathResponse)(nil) + _ encoding.ProtoUnmarshaler = (*GetNodeByPathResponse)(nil) + _ json.Marshaler = (*GetNodeByPathResponse)(nil) + _ json.Unmarshaler = (*GetNodeByPathResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -917,27 +4105,6 @@ func (x *GetNodeByPathResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetNodeByPathResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -953,13 +4120,196 @@ func (x *GetNodeByPathResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetNodeByPathResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetNodeByPathResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetNodeByPathResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetNodeByPathResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetNodeByPathResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetNodeByPathResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetNodeByPathResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetNodeByPathResponse) GetBody() *GetNodeByPathResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetNodeByPathResponse) SetBody(v *GetNodeByPathResponse_Body) { + x.Body = v +} +func (x *GetNodeByPathResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetNodeByPathResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetNodeByPathResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetNodeByPathResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetNodeByPathResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetNodeByPathResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetNodeByPathResponse_Body + f = new(GetNodeByPathResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetSubTreeRequest_Body_Order_Direction int32 + +const ( + GetSubTreeRequest_Body_Order_None GetSubTreeRequest_Body_Order_Direction = 0 + GetSubTreeRequest_Body_Order_Asc GetSubTreeRequest_Body_Order_Direction = 1 +) + +var ( + GetSubTreeRequest_Body_Order_Direction_name = map[int32]string{ + 0: "None", + 1: "Asc", + } + GetSubTreeRequest_Body_Order_Direction_value = map[string]int32{ + "None": 0, + "Asc": 1, + } +) + +func (x GetSubTreeRequest_Body_Order_Direction) String() string { + if v, ok := GetSubTreeRequest_Body_Order_Direction_name[int32(x)]; ok { + return v + } + return strconv.FormatInt(int64(x), 10) +} +func (x *GetSubTreeRequest_Body_Order_Direction) FromString(s string) bool { + if v, ok := GetSubTreeRequest_Body_Order_Direction_value[s]; ok { + *x = GetSubTreeRequest_Body_Order_Direction(v) + return true + } + return false +} + +type GetSubTreeRequest_Body_Order struct { + Direction GetSubTreeRequest_Body_Order_Direction `json:"direction"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetSubTreeRequest_Body_Order)(nil) + _ encoding.ProtoUnmarshaler = (*GetSubTreeRequest_Body_Order)(nil) + _ json.Marshaler = (*GetSubTreeRequest_Body_Order)(nil) + _ json.Unmarshaler = (*GetSubTreeRequest_Body_Order)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -971,26 +4321,145 @@ func (x *GetSubTreeRequest_Body_Order) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetSubTreeRequest_Body_Order) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.EnumMarshal(1, buf[offset:], int32(x.Direction)) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetSubTreeRequest_Body_Order) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetSubTreeRequest_Body_Order) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if int32(x.Direction) != 0 { + mm.AppendInt32(1, int32(x.Direction)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetSubTreeRequest_Body_Order) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetSubTreeRequest_Body_Order") + } + switch fc.FieldNum { + case 1: // Direction + data, ok := fc.Int32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Direction") + } + x.Direction = GetSubTreeRequest_Body_Order_Direction(data) + } + } + return nil +} +func (x *GetSubTreeRequest_Body_Order) GetDirection() GetSubTreeRequest_Body_Order_Direction { + if x != nil { + return x.Direction + } + return 0 +} +func (x *GetSubTreeRequest_Body_Order) SetDirection(v GetSubTreeRequest_Body_Order_Direction) { + x.Direction = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetSubTreeRequest_Body_Order) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetSubTreeRequest_Body_Order) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"direction\":" + out.RawString(prefix[1:]) + out.Int32(int32(x.Direction)) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetSubTreeRequest_Body_Order) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetSubTreeRequest_Body_Order) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "direction": + { + var f GetSubTreeRequest_Body_Order_Direction + var parsedValue GetSubTreeRequest_Body_Order_Direction + switch v := in.Interface().(type) { + case string: + if vv, ok := GetSubTreeRequest_Body_Order_Direction_value[v]; ok { + parsedValue = GetSubTreeRequest_Body_Order_Direction(vv) + break + } + vv, err := strconv.ParseInt(v, 10, 32) + if err != nil { + in.AddError(err) + return + } + parsedValue = GetSubTreeRequest_Body_Order_Direction(vv) + case float64: + parsedValue = GetSubTreeRequest_Body_Order_Direction(v) + } + f = parsedValue + x.Direction = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetSubTreeRequest_Body struct { + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + RootId []uint64 `json:"rootId"` + Depth uint32 `json:"depth"` + BearerToken []byte `json:"bearerToken"` + OrderBy *GetSubTreeRequest_Body_Order `json:"orderBy"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetSubTreeRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetSubTreeRequest_Body)(nil) + _ json.Marshaler = (*GetSubTreeRequest_Body)(nil) + _ json.Unmarshaler = (*GetSubTreeRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1009,36 +4478,287 @@ func (x *GetSubTreeRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetSubTreeRequest_Body) StableMarshal(buf []byte) []byte { +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetSubTreeRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *GetSubTreeRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { - return []byte{} + return } - if buf == nil { - buf = make([]byte, x.StableSize()) + if len(x.ContainerId) != 0 { + mm.AppendBytes(1, x.ContainerId) } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) - offset += proto.StringMarshal(2, buf[offset:], x.TreeId) - for i := range x.RootId { - { - prefix := protowire.EncodeTag(protowire.Number(3), protowire.VarintType) - offset += binary.PutUvarint(buf[offset:], uint64(prefix)) - offset += binary.PutUvarint(buf[offset:], x.RootId[i]) + if len(x.TreeId) != 0 { + mm.AppendString(2, x.TreeId) + } + for j := range x.RootId { + mm.AppendUint64(3, x.RootId[j]) + } + if x.Depth != 0 { + mm.AppendUint32(4, x.Depth) + } + if len(x.BearerToken) != 0 { + mm.AppendBytes(5, x.BearerToken) + } + if x.OrderBy != nil && x.OrderBy.StableSize() != 0 { + x.OrderBy.EmitProtobuf(mm.AppendMessage(6)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetSubTreeRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetSubTreeRequest_Body") + } + switch fc.FieldNum { + case 1: // ContainerId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + case 2: // TreeId + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TreeId") + } + x.TreeId = data + case 3: // RootId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "RootId") + } + x.RootId = append(x.RootId, data) + case 4: // Depth + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Depth") + } + x.Depth = data + case 5: // BearerToken + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "BearerToken") + } + x.BearerToken = data + case 6: // OrderBy + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "OrderBy") + } + x.OrderBy = new(GetSubTreeRequest_Body_Order) + if err := x.OrderBy.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } } } - offset += proto.UInt32Marshal(4, buf[offset:], x.Depth) - offset += proto.BytesMarshal(5, buf[offset:], x.BearerToken) - offset += proto.NestedStructureMarshal(6, buf[offset:], x.OrderBy) - return buf + return nil } +func (x *GetSubTreeRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} +func (x *GetSubTreeRequest_Body) SetContainerId(v []byte) { + x.ContainerId = v +} +func (x *GetSubTreeRequest_Body) GetTreeId() string { + if x != nil { + return x.TreeId + } + return "" +} +func (x *GetSubTreeRequest_Body) SetTreeId(v string) { + x.TreeId = v +} +func (x *GetSubTreeRequest_Body) GetRootId() []uint64 { + if x != nil { + return x.RootId + } + return nil +} +func (x *GetSubTreeRequest_Body) SetRootId(v []uint64) { + x.RootId = v +} +func (x *GetSubTreeRequest_Body) GetDepth() uint32 { + if x != nil { + return x.Depth + } + return 0 +} +func (x *GetSubTreeRequest_Body) SetDepth(v uint32) { + x.Depth = v +} +func (x *GetSubTreeRequest_Body) GetBearerToken() []byte { + if x != nil { + return x.BearerToken + } + return nil +} +func (x *GetSubTreeRequest_Body) SetBearerToken(v []byte) { + x.BearerToken = v +} +func (x *GetSubTreeRequest_Body) GetOrderBy() *GetSubTreeRequest_Body_Order { + if x != nil { + return x.OrderBy + } + return nil +} +func (x *GetSubTreeRequest_Body) SetOrderBy(v *GetSubTreeRequest_Body_Order) { + x.OrderBy = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetSubTreeRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetSubTreeRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"containerId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ContainerId) + } + { + const prefix string = ",\"treeId\":" + out.RawString(prefix) + out.String(x.TreeId) + } + { + const prefix string = ",\"rootId\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.RootId { + if i != 0 { + out.RawByte(',') + } + out.Uint64(x.RootId[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"depth\":" + out.RawString(prefix) + out.Uint32(x.Depth) + } + { + const prefix string = ",\"bearerToken\":" + out.RawString(prefix) + out.Base64Bytes(x.BearerToken) + } + { + const prefix string = ",\"orderBy\":" + out.RawString(prefix) + x.OrderBy.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetSubTreeRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetSubTreeRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "containerId": + { + var f []byte + f = in.Bytes() + x.ContainerId = f + } + case "treeId": + { + var f string + f = in.String() + x.TreeId = f + } + case "rootId": + { + var f uint64 + var list []uint64 + in.Delim('[') + for !in.IsDelim(']') { + f = in.Uint64() + list = append(list, f) + in.WantComma() + } + x.RootId = list + in.Delim(']') + } + case "depth": + { + var f uint32 + f = in.Uint32() + x.Depth = f + } + case "bearerToken": + { + var f []byte + f = in.Bytes() + x.BearerToken = f + } + case "orderBy": + { + var f *GetSubTreeRequest_Body_Order + f = new(GetSubTreeRequest_Body_Order) + f.UnmarshalEasyJSON(in) + x.OrderBy = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetSubTreeRequest struct { + Body *GetSubTreeRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetSubTreeRequest)(nil) + _ encoding.ProtoUnmarshaler = (*GetSubTreeRequest)(nil) + _ json.Marshaler = (*GetSubTreeRequest)(nil) + _ json.Unmarshaler = (*GetSubTreeRequest)(nil) +) // StableSize returns the size of x in protobuf format. // @@ -1052,27 +4772,6 @@ func (x *GetSubTreeRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetSubTreeRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1088,13 +4787,167 @@ func (x *GetSubTreeRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetSubTreeRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetSubTreeRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetSubTreeRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetSubTreeRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetSubTreeRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetSubTreeRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetSubTreeRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetSubTreeRequest) GetBody() *GetSubTreeRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetSubTreeRequest) SetBody(v *GetSubTreeRequest_Body) { + x.Body = v +} +func (x *GetSubTreeRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetSubTreeRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetSubTreeRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetSubTreeRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetSubTreeRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetSubTreeRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetSubTreeRequest_Body + f = new(GetSubTreeRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetSubTreeResponse_Body struct { + NodeId []uint64 `json:"nodeId"` + ParentId []uint64 `json:"parentId"` + Timestamp []uint64 `json:"timestamp"` + Meta []*KeyValue `json:"meta"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetSubTreeResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetSubTreeResponse_Body)(nil) + _ json.Marshaler = (*GetSubTreeResponse_Body)(nil) + _ json.Unmarshaler = (*GetSubTreeResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1117,49 +4970,275 @@ func (x *GetSubTreeResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetSubTreeResponse_Body) StableMarshal(buf []byte) []byte { +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetSubTreeResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *GetSubTreeResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { - return []byte{} + return } - if buf == nil { - buf = make([]byte, x.StableSize()) + for j := range x.NodeId { + mm.AppendUint64(1, x.NodeId[j]) } - var offset int - for i := range x.NodeId { - { - prefix := protowire.EncodeTag(protowire.Number(1), protowire.VarintType) - offset += binary.PutUvarint(buf[offset:], uint64(prefix)) - offset += binary.PutUvarint(buf[offset:], x.NodeId[i]) - } + for j := range x.ParentId { + mm.AppendUint64(2, x.ParentId[j]) } - for i := range x.ParentId { - { - prefix := protowire.EncodeTag(protowire.Number(2), protowire.VarintType) - offset += binary.PutUvarint(buf[offset:], uint64(prefix)) - offset += binary.PutUvarint(buf[offset:], x.ParentId[i]) - } - } - for i := range x.Timestamp { - { - prefix := protowire.EncodeTag(protowire.Number(3), protowire.VarintType) - offset += binary.PutUvarint(buf[offset:], uint64(prefix)) - offset += binary.PutUvarint(buf[offset:], x.Timestamp[i]) - } + for j := range x.Timestamp { + mm.AppendUint64(3, x.Timestamp[j]) } for i := range x.Meta { - offset += proto.NestedStructureMarshal(4, buf[offset:], x.Meta[i]) + if x.Meta[i] != nil && x.Meta[i].StableSize() != 0 { + x.Meta[i].EmitProtobuf(mm.AppendMessage(4)) + } } - return buf } +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetSubTreeResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetSubTreeResponse_Body") + } + switch fc.FieldNum { + case 1: // NodeId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "NodeId") + } + x.NodeId = append(x.NodeId, data) + case 2: // ParentId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ParentId") + } + x.ParentId = append(x.ParentId, data) + case 3: // Timestamp + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Timestamp") + } + x.Timestamp = append(x.Timestamp, data) + case 4: // Meta + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Meta") + } + x.Meta = append(x.Meta, new(KeyValue)) + ff := x.Meta[len(x.Meta)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetSubTreeResponse_Body) GetNodeId() []uint64 { + if x != nil { + return x.NodeId + } + return nil +} +func (x *GetSubTreeResponse_Body) SetNodeId(v []uint64) { + x.NodeId = v +} +func (x *GetSubTreeResponse_Body) GetParentId() []uint64 { + if x != nil { + return x.ParentId + } + return nil +} +func (x *GetSubTreeResponse_Body) SetParentId(v []uint64) { + x.ParentId = v +} +func (x *GetSubTreeResponse_Body) GetTimestamp() []uint64 { + if x != nil { + return x.Timestamp + } + return nil +} +func (x *GetSubTreeResponse_Body) SetTimestamp(v []uint64) { + x.Timestamp = v +} +func (x *GetSubTreeResponse_Body) GetMeta() []*KeyValue { + if x != nil { + return x.Meta + } + return nil +} +func (x *GetSubTreeResponse_Body) SetMeta(v []*KeyValue) { + x.Meta = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetSubTreeResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetSubTreeResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"nodeId\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.NodeId { + if i != 0 { + out.RawByte(',') + } + out.Uint64(x.NodeId[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"parentId\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.ParentId { + if i != 0 { + out.RawByte(',') + } + out.Uint64(x.ParentId[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"timestamp\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Timestamp { + if i != 0 { + out.RawByte(',') + } + out.Uint64(x.Timestamp[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"meta\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Meta { + if i != 0 { + out.RawByte(',') + } + x.Meta[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetSubTreeResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetSubTreeResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "nodeId": + { + var f uint64 + var list []uint64 + in.Delim('[') + for !in.IsDelim(']') { + f = in.Uint64() + list = append(list, f) + in.WantComma() + } + x.NodeId = list + in.Delim(']') + } + case "parentId": + { + var f uint64 + var list []uint64 + in.Delim('[') + for !in.IsDelim(']') { + f = in.Uint64() + list = append(list, f) + in.WantComma() + } + x.ParentId = list + in.Delim(']') + } + case "timestamp": + { + var f uint64 + var list []uint64 + in.Delim('[') + for !in.IsDelim(']') { + f = in.Uint64() + list = append(list, f) + in.WantComma() + } + x.Timestamp = list + in.Delim(']') + } + case "meta": + { + var f *KeyValue + var list []*KeyValue + in.Delim('[') + for !in.IsDelim(']') { + f = new(KeyValue) + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Meta = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetSubTreeResponse struct { + Body *GetSubTreeResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetSubTreeResponse)(nil) + _ encoding.ProtoUnmarshaler = (*GetSubTreeResponse)(nil) + _ json.Marshaler = (*GetSubTreeResponse)(nil) + _ json.Unmarshaler = (*GetSubTreeResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1172,27 +5251,6 @@ func (x *GetSubTreeResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetSubTreeResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1208,13 +5266,164 @@ func (x *GetSubTreeResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetSubTreeResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetSubTreeResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetSubTreeResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetSubTreeResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetSubTreeResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetSubTreeResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetSubTreeResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetSubTreeResponse) GetBody() *GetSubTreeResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetSubTreeResponse) SetBody(v *GetSubTreeResponse_Body) { + x.Body = v +} +func (x *GetSubTreeResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetSubTreeResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetSubTreeResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetSubTreeResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetSubTreeResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetSubTreeResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetSubTreeResponse_Body + f = new(GetSubTreeResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type TreeListRequest_Body struct { + ContainerId []byte `json:"containerId"` +} + +var ( + _ encoding.ProtoMarshaler = (*TreeListRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*TreeListRequest_Body)(nil) + _ json.Marshaler = (*TreeListRequest_Body)(nil) + _ json.Unmarshaler = (*TreeListRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1226,26 +5435,125 @@ func (x *TreeListRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *TreeListRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *TreeListRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *TreeListRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ContainerId) != 0 { + mm.AppendBytes(1, x.ContainerId) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *TreeListRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "TreeListRequest_Body") + } + switch fc.FieldNum { + case 1: // ContainerId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + } + } + return nil +} +func (x *TreeListRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} +func (x *TreeListRequest_Body) SetContainerId(v []byte) { + x.ContainerId = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *TreeListRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *TreeListRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"containerId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ContainerId) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *TreeListRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *TreeListRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "containerId": + { + var f []byte + f = in.Bytes() + x.ContainerId = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type TreeListRequest struct { + Body *TreeListRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*TreeListRequest)(nil) + _ encoding.ProtoUnmarshaler = (*TreeListRequest)(nil) + _ json.Marshaler = (*TreeListRequest)(nil) + _ json.Unmarshaler = (*TreeListRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1258,27 +5566,6 @@ func (x *TreeListRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *TreeListRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1294,13 +5581,164 @@ func (x *TreeListRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *TreeListRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *TreeListRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *TreeListRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *TreeListRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *TreeListRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "TreeListRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(TreeListRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *TreeListRequest) GetBody() *TreeListRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *TreeListRequest) SetBody(v *TreeListRequest_Body) { + x.Body = v +} +func (x *TreeListRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *TreeListRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *TreeListRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *TreeListRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *TreeListRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *TreeListRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *TreeListRequest_Body + f = new(TreeListRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type TreeListResponse_Body struct { + Ids []string `json:"ids"` +} + +var ( + _ encoding.ProtoMarshaler = (*TreeListResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*TreeListResponse_Body)(nil) + _ json.Marshaler = (*TreeListResponse_Body)(nil) + _ json.Unmarshaler = (*TreeListResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1312,26 +5750,139 @@ func (x *TreeListResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *TreeListResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.RepeatedStringMarshal(1, buf[offset:], x.Ids) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *TreeListResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *TreeListResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for j := range x.Ids { + mm.AppendString(1, x.Ids[j]) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *TreeListResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "TreeListResponse_Body") + } + switch fc.FieldNum { + case 1: // Ids + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Ids") + } + x.Ids = append(x.Ids, data) + } + } + return nil +} +func (x *TreeListResponse_Body) GetIds() []string { + if x != nil { + return x.Ids + } + return nil +} +func (x *TreeListResponse_Body) SetIds(v []string) { + x.Ids = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *TreeListResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *TreeListResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"ids\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Ids { + if i != 0 { + out.RawByte(',') + } + out.String(x.Ids[i]) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *TreeListResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *TreeListResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "ids": + { + var f string + var list []string + in.Delim('[') + for !in.IsDelim(']') { + f = in.String() + list = append(list, f) + in.WantComma() + } + x.Ids = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type TreeListResponse struct { + Body *TreeListResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*TreeListResponse)(nil) + _ encoding.ProtoUnmarshaler = (*TreeListResponse)(nil) + _ json.Marshaler = (*TreeListResponse)(nil) + _ json.Unmarshaler = (*TreeListResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1344,27 +5895,6 @@ func (x *TreeListResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *TreeListResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1380,13 +5910,166 @@ func (x *TreeListResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *TreeListResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *TreeListResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *TreeListResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *TreeListResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *TreeListResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "TreeListResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(TreeListResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *TreeListResponse) GetBody() *TreeListResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *TreeListResponse) SetBody(v *TreeListResponse_Body) { + x.Body = v +} +func (x *TreeListResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *TreeListResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *TreeListResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *TreeListResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *TreeListResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *TreeListResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *TreeListResponse_Body + f = new(TreeListResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ApplyRequest_Body struct { + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + Operation *LogMove `json:"operation"` +} + +var ( + _ encoding.ProtoMarshaler = (*ApplyRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ApplyRequest_Body)(nil) + _ json.Marshaler = (*ApplyRequest_Body)(nil) + _ json.Unmarshaler = (*ApplyRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1400,28 +6083,187 @@ func (x *ApplyRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ApplyRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) - offset += proto.StringMarshal(2, buf[offset:], x.TreeId) - offset += proto.NestedStructureMarshal(3, buf[offset:], x.Operation) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ApplyRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ApplyRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ContainerId) != 0 { + mm.AppendBytes(1, x.ContainerId) + } + if len(x.TreeId) != 0 { + mm.AppendString(2, x.TreeId) + } + if x.Operation != nil && x.Operation.StableSize() != 0 { + x.Operation.EmitProtobuf(mm.AppendMessage(3)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ApplyRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ApplyRequest_Body") + } + switch fc.FieldNum { + case 1: // ContainerId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + case 2: // TreeId + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TreeId") + } + x.TreeId = data + case 3: // Operation + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Operation") + } + x.Operation = new(LogMove) + if err := x.Operation.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ApplyRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} +func (x *ApplyRequest_Body) SetContainerId(v []byte) { + x.ContainerId = v +} +func (x *ApplyRequest_Body) GetTreeId() string { + if x != nil { + return x.TreeId + } + return "" +} +func (x *ApplyRequest_Body) SetTreeId(v string) { + x.TreeId = v +} +func (x *ApplyRequest_Body) GetOperation() *LogMove { + if x != nil { + return x.Operation + } + return nil +} +func (x *ApplyRequest_Body) SetOperation(v *LogMove) { + x.Operation = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ApplyRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ApplyRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"containerId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ContainerId) + } + { + const prefix string = ",\"treeId\":" + out.RawString(prefix) + out.String(x.TreeId) + } + { + const prefix string = ",\"operation\":" + out.RawString(prefix) + x.Operation.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ApplyRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ApplyRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "containerId": + { + var f []byte + f = in.Bytes() + x.ContainerId = f + } + case "treeId": + { + var f string + f = in.String() + x.TreeId = f + } + case "operation": + { + var f *LogMove + f = new(LogMove) + f.UnmarshalEasyJSON(in) + x.Operation = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ApplyRequest struct { + Body *ApplyRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ApplyRequest)(nil) + _ encoding.ProtoUnmarshaler = (*ApplyRequest)(nil) + _ json.Marshaler = (*ApplyRequest)(nil) + _ json.Unmarshaler = (*ApplyRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1434,27 +6276,6 @@ func (x *ApplyRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ApplyRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1470,13 +6291,163 @@ func (x *ApplyRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *ApplyRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *ApplyRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ApplyRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ApplyRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ApplyRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ApplyRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ApplyRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ApplyRequest) GetBody() *ApplyRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ApplyRequest) SetBody(v *ApplyRequest_Body) { + x.Body = v +} +func (x *ApplyRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ApplyRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ApplyRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ApplyRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ApplyRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ApplyRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ApplyRequest_Body + f = new(ApplyRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ApplyResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*ApplyResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ApplyResponse_Body)(nil) + _ json.Marshaler = (*ApplyResponse_Body)(nil) + _ json.Unmarshaler = (*ApplyResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1487,18 +6458,93 @@ func (x *ApplyResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ApplyResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ApplyResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ApplyResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ApplyResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ApplyResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ApplyResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ApplyResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ApplyResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ApplyResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ApplyResponse struct { + Body *ApplyResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ApplyResponse)(nil) + _ encoding.ProtoUnmarshaler = (*ApplyResponse)(nil) + _ json.Marshaler = (*ApplyResponse)(nil) + _ json.Unmarshaler = (*ApplyResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1511,27 +6557,6 @@ func (x *ApplyResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *ApplyResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1547,13 +6572,167 @@ func (x *ApplyResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *ApplyResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *ApplyResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ApplyResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *ApplyResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ApplyResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ApplyResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ApplyResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ApplyResponse) GetBody() *ApplyResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ApplyResponse) SetBody(v *ApplyResponse_Body) { + x.Body = v +} +func (x *ApplyResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ApplyResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ApplyResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ApplyResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ApplyResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ApplyResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ApplyResponse_Body + f = new(ApplyResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetOpLogRequest_Body struct { + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + Height uint64 `json:"height"` + Count uint64 `json:"count"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetOpLogRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetOpLogRequest_Body)(nil) + _ json.Marshaler = (*GetOpLogRequest_Body)(nil) + _ json.Unmarshaler = (*GetOpLogRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1568,29 +6747,212 @@ func (x *GetOpLogRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetOpLogRequest_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.ContainerId) - offset += proto.StringMarshal(2, buf[offset:], x.TreeId) - offset += proto.UInt64Marshal(3, buf[offset:], x.Height) - offset += proto.UInt64Marshal(4, buf[offset:], x.Count) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetOpLogRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetOpLogRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ContainerId) != 0 { + mm.AppendBytes(1, x.ContainerId) + } + if len(x.TreeId) != 0 { + mm.AppendString(2, x.TreeId) + } + if x.Height != 0 { + mm.AppendUint64(3, x.Height) + } + if x.Count != 0 { + mm.AppendUint64(4, x.Count) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetOpLogRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetOpLogRequest_Body") + } + switch fc.FieldNum { + case 1: // ContainerId + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + case 2: // TreeId + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TreeId") + } + x.TreeId = data + case 3: // Height + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Height") + } + x.Height = data + case 4: // Count + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Count") + } + x.Count = data + } + } + return nil +} +func (x *GetOpLogRequest_Body) GetContainerId() []byte { + if x != nil { + return x.ContainerId + } + return nil +} +func (x *GetOpLogRequest_Body) SetContainerId(v []byte) { + x.ContainerId = v +} +func (x *GetOpLogRequest_Body) GetTreeId() string { + if x != nil { + return x.TreeId + } + return "" +} +func (x *GetOpLogRequest_Body) SetTreeId(v string) { + x.TreeId = v +} +func (x *GetOpLogRequest_Body) GetHeight() uint64 { + if x != nil { + return x.Height + } + return 0 +} +func (x *GetOpLogRequest_Body) SetHeight(v uint64) { + x.Height = v +} +func (x *GetOpLogRequest_Body) GetCount() uint64 { + if x != nil { + return x.Count + } + return 0 +} +func (x *GetOpLogRequest_Body) SetCount(v uint64) { + x.Count = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetOpLogRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetOpLogRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"containerId\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.ContainerId) + } + { + const prefix string = ",\"treeId\":" + out.RawString(prefix) + out.String(x.TreeId) + } + { + const prefix string = ",\"height\":" + out.RawString(prefix) + out.Uint64(x.Height) + } + { + const prefix string = ",\"count\":" + out.RawString(prefix) + out.Uint64(x.Count) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetOpLogRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetOpLogRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "containerId": + { + var f []byte + f = in.Bytes() + x.ContainerId = f + } + case "treeId": + { + var f string + f = in.String() + x.TreeId = f + } + case "height": + { + var f uint64 + f = in.Uint64() + x.Height = f + } + case "count": + { + var f uint64 + f = in.Uint64() + x.Count = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetOpLogRequest struct { + Body *GetOpLogRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetOpLogRequest)(nil) + _ encoding.ProtoUnmarshaler = (*GetOpLogRequest)(nil) + _ json.Marshaler = (*GetOpLogRequest)(nil) + _ json.Unmarshaler = (*GetOpLogRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1603,27 +6965,6 @@ func (x *GetOpLogRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetOpLogRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1639,13 +6980,164 @@ func (x *GetOpLogRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetOpLogRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetOpLogRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetOpLogRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetOpLogRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetOpLogRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetOpLogRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetOpLogRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetOpLogRequest) GetBody() *GetOpLogRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetOpLogRequest) SetBody(v *GetOpLogRequest_Body) { + x.Body = v +} +func (x *GetOpLogRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetOpLogRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetOpLogRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetOpLogRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetOpLogRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetOpLogRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetOpLogRequest_Body + f = new(GetOpLogRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetOpLogResponse_Body struct { + Operation *LogMove `json:"operation"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetOpLogResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*GetOpLogResponse_Body)(nil) + _ json.Marshaler = (*GetOpLogResponse_Body)(nil) + _ json.Unmarshaler = (*GetOpLogResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1657,26 +7149,129 @@ func (x *GetOpLogResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetOpLogResponse_Body) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Operation) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetOpLogResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetOpLogResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Operation != nil && x.Operation.StableSize() != 0 { + x.Operation.EmitProtobuf(mm.AppendMessage(1)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetOpLogResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetOpLogResponse_Body") + } + switch fc.FieldNum { + case 1: // Operation + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Operation") + } + x.Operation = new(LogMove) + if err := x.Operation.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetOpLogResponse_Body) GetOperation() *LogMove { + if x != nil { + return x.Operation + } + return nil +} +func (x *GetOpLogResponse_Body) SetOperation(v *LogMove) { + x.Operation = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetOpLogResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetOpLogResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"operation\":" + out.RawString(prefix[1:]) + x.Operation.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetOpLogResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetOpLogResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "operation": + { + var f *LogMove + f = new(LogMove) + f.UnmarshalEasyJSON(in) + x.Operation = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type GetOpLogResponse struct { + Body *GetOpLogResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*GetOpLogResponse)(nil) + _ encoding.ProtoUnmarshaler = (*GetOpLogResponse)(nil) + _ json.Marshaler = (*GetOpLogResponse)(nil) + _ json.Unmarshaler = (*GetOpLogResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1689,27 +7284,6 @@ func (x *GetOpLogResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *GetOpLogResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1725,13 +7299,163 @@ func (x *GetOpLogResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *GetOpLogResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *GetOpLogResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *GetOpLogResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *GetOpLogResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *GetOpLogResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "GetOpLogResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(GetOpLogResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *GetOpLogResponse) GetBody() *GetOpLogResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *GetOpLogResponse) SetBody(v *GetOpLogResponse_Body) { + x.Body = v +} +func (x *GetOpLogResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *GetOpLogResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *GetOpLogResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *GetOpLogResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *GetOpLogResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *GetOpLogResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *GetOpLogResponse_Body + f = new(GetOpLogResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type HealthcheckResponse_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*HealthcheckResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*HealthcheckResponse_Body)(nil) + _ json.Marshaler = (*HealthcheckResponse_Body)(nil) + _ json.Unmarshaler = (*HealthcheckResponse_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1742,18 +7466,93 @@ func (x *HealthcheckResponse_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthcheckResponse_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthcheckResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *HealthcheckResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthcheckResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthcheckResponse_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthcheckResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthcheckResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthcheckResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthcheckResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type HealthcheckResponse struct { + Body *HealthcheckResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*HealthcheckResponse)(nil) + _ encoding.ProtoUnmarshaler = (*HealthcheckResponse)(nil) + _ json.Marshaler = (*HealthcheckResponse)(nil) + _ json.Unmarshaler = (*HealthcheckResponse)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1766,27 +7565,6 @@ func (x *HealthcheckResponse) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthcheckResponse) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1802,13 +7580,163 @@ func (x *HealthcheckResponse) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *HealthcheckResponse) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *HealthcheckResponse) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthcheckResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *HealthcheckResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthcheckResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthcheckResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(HealthcheckResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *HealthcheckResponse) GetBody() *HealthcheckResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *HealthcheckResponse) SetBody(v *HealthcheckResponse_Body) { + x.Body = v +} +func (x *HealthcheckResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *HealthcheckResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthcheckResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthcheckResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthcheckResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthcheckResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *HealthcheckResponse_Body + f = new(HealthcheckResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type HealthcheckRequest_Body struct { +} + +var ( + _ encoding.ProtoMarshaler = (*HealthcheckRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*HealthcheckRequest_Body)(nil) + _ json.Marshaler = (*HealthcheckRequest_Body)(nil) + _ json.Unmarshaler = (*HealthcheckRequest_Body)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1819,18 +7747,93 @@ func (x *HealthcheckRequest_Body) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthcheckRequest_Body) StableMarshal(buf []byte) []byte { - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthcheckRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *HealthcheckRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthcheckRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthcheckRequest_Body") + } + switch fc.FieldNum { + } + } + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthcheckRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthcheckRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthcheckRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthcheckRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type HealthcheckRequest struct { + Body *HealthcheckRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*HealthcheckRequest)(nil) + _ encoding.ProtoUnmarshaler = (*HealthcheckRequest)(nil) + _ json.Marshaler = (*HealthcheckRequest)(nil) + _ json.Unmarshaler = (*HealthcheckRequest)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -1843,27 +7846,6 @@ func (x *HealthcheckRequest) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *HealthcheckRequest) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body) - offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature) - return buf -} - // ReadSignedData fills buf with signed data of x. // If buffer length is less than x.SignedDataSize(), new buffer is allocated. // @@ -1879,9 +7861,149 @@ func (x *HealthcheckRequest) SignedDataSize() int { // // Structures with the same field values have the same signed data size. func (x *HealthcheckRequest) ReadSignedData(buf []byte) ([]byte, error) { - return x.GetBody().StableMarshal(buf), nil + return x.GetBody().MarshalProtobuf(buf), nil } -func (x *HealthcheckRequest) SetSignature(sig *Signature) { - x.Signature = sig +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *HealthcheckRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *HealthcheckRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil && x.Body.StableSize() != 0 { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil && x.Signature.StableSize() != 0 { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *HealthcheckRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "HealthcheckRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(HealthcheckRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *HealthcheckRequest) GetBody() *HealthcheckRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *HealthcheckRequest) SetBody(v *HealthcheckRequest_Body) { + x.Body = v +} +func (x *HealthcheckRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *HealthcheckRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *HealthcheckRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *HealthcheckRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *HealthcheckRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *HealthcheckRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *HealthcheckRequest_Body + f = new(HealthcheckRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } } diff --git a/pkg/services/tree/types.pb.go b/pkg/services/tree/types.pb.go deleted file mode 100644 index 6464ccb77..000000000 --- a/pkg/services/tree/types.pb.go +++ /dev/null @@ -1,320 +0,0 @@ -//* -// Auxiliary structures to use with tree service. - -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.33.0 -// protoc v4.25.0 -// source: pkg/services/tree/types.proto - -package tree - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// KeyValue represents key-value pair attached to an object. -type KeyValue struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Attribute name. - Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - // Attribute value. - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` -} - -func (x *KeyValue) Reset() { - *x = KeyValue{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_types_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *KeyValue) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*KeyValue) ProtoMessage() {} - -func (x *KeyValue) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_types_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use KeyValue.ProtoReflect.Descriptor instead. -func (*KeyValue) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_types_proto_rawDescGZIP(), []int{0} -} - -func (x *KeyValue) GetKey() string { - if x != nil { - return x.Key - } - return "" -} - -func (x *KeyValue) GetValue() []byte { - if x != nil { - return x.Value - } - return nil -} - -// LogMove represents log-entry for a single move operation. -type LogMove struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // ID of the parent node. - ParentId uint64 `protobuf:"varint,1,opt,name=parent_id,json=parentID,proto3" json:"parent_id,omitempty"` - // Node meta information, including operation timestamp. - Meta []byte `protobuf:"bytes,2,opt,name=meta,proto3" json:"meta,omitempty"` - // ID of the node to move. - ChildId uint64 `protobuf:"varint,3,opt,name=child_id,json=childID,proto3" json:"child_id,omitempty"` -} - -func (x *LogMove) Reset() { - *x = LogMove{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_types_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *LogMove) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*LogMove) ProtoMessage() {} - -func (x *LogMove) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_types_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use LogMove.ProtoReflect.Descriptor instead. -func (*LogMove) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_types_proto_rawDescGZIP(), []int{1} -} - -func (x *LogMove) GetParentId() uint64 { - if x != nil { - return x.ParentId - } - return 0 -} - -func (x *LogMove) GetMeta() []byte { - if x != nil { - return x.Meta - } - return nil -} - -func (x *LogMove) GetChildId() uint64 { - if x != nil { - return x.ChildId - } - return 0 -} - -// Signature of a message. -type Signature struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Serialized public key as defined in FrostFS API. - Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - // Signature of a message body. - Sign []byte `protobuf:"bytes,2,opt,name=sign,json=signature,proto3" json:"sign,omitempty"` -} - -func (x *Signature) Reset() { - *x = Signature{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_services_tree_types_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Signature) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Signature) ProtoMessage() {} - -func (x *Signature) ProtoReflect() protoreflect.Message { - mi := &file_pkg_services_tree_types_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Signature.ProtoReflect.Descriptor instead. -func (*Signature) Descriptor() ([]byte, []int) { - return file_pkg_services_tree_types_proto_rawDescGZIP(), []int{2} -} - -func (x *Signature) GetKey() []byte { - if x != nil { - return x.Key - } - return nil -} - -func (x *Signature) GetSign() []byte { - if x != nil { - return x.Sign - } - return nil -} - -var File_pkg_services_tree_types_proto protoreflect.FileDescriptor - -var file_pkg_services_tree_types_proto_rawDesc = []byte{ - 0x0a, 0x1d, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x74, - 0x72, 0x65, 0x65, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, - 0x04, 0x74, 0x72, 0x65, 0x65, 0x22, 0x32, 0x0a, 0x08, 0x4b, 0x65, 0x79, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x55, 0x0a, 0x07, 0x4c, 0x6f, 0x67, - 0x4d, 0x6f, 0x76, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, - 0x44, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x04, 0x6d, 0x65, 0x74, 0x61, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x69, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x49, 0x44, - 0x22, 0x36, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x17, 0x0a, 0x04, 0x73, 0x69, 0x67, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x2e, - 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, 0x73, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2f, 0x54, 0x72, 0x75, - 0x65, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x4c, 0x61, 0x62, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x66, - 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x65, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_pkg_services_tree_types_proto_rawDescOnce sync.Once - file_pkg_services_tree_types_proto_rawDescData = file_pkg_services_tree_types_proto_rawDesc -) - -func file_pkg_services_tree_types_proto_rawDescGZIP() []byte { - file_pkg_services_tree_types_proto_rawDescOnce.Do(func() { - file_pkg_services_tree_types_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_services_tree_types_proto_rawDescData) - }) - return file_pkg_services_tree_types_proto_rawDescData -} - -var file_pkg_services_tree_types_proto_msgTypes = make([]protoimpl.MessageInfo, 3) -var file_pkg_services_tree_types_proto_goTypes = []interface{}{ - (*KeyValue)(nil), // 0: tree.KeyValue - (*LogMove)(nil), // 1: tree.LogMove - (*Signature)(nil), // 2: tree.Signature -} -var file_pkg_services_tree_types_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_pkg_services_tree_types_proto_init() } -func file_pkg_services_tree_types_proto_init() { - if File_pkg_services_tree_types_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_pkg_services_tree_types_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KeyValue); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_types_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LogMove); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_services_tree_types_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Signature); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_pkg_services_tree_types_proto_rawDesc, - NumEnums: 0, - NumMessages: 3, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_pkg_services_tree_types_proto_goTypes, - DependencyIndexes: file_pkg_services_tree_types_proto_depIdxs, - MessageInfos: file_pkg_services_tree_types_proto_msgTypes, - }.Build() - File_pkg_services_tree_types_proto = out.File - file_pkg_services_tree_types_proto_rawDesc = nil - file_pkg_services_tree_types_proto_goTypes = nil - file_pkg_services_tree_types_proto_depIdxs = nil -} diff --git a/pkg/services/tree/types_frostfs.pb.go b/pkg/services/tree/types_frostfs.pb.go index 707fcc3cc..4399f8a8b 100644 --- a/pkg/services/tree/types_frostfs.pb.go +++ b/pkg/services/tree/types_frostfs.pb.go @@ -2,7 +2,28 @@ package tree -import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" +import ( + json "encoding/json" + fmt "fmt" + pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + easyproto "github.com/VictoriaMetrics/easyproto" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" +) + +type KeyValue struct { + Key string `json:"key"` + Value []byte `json:"value"` +} + +var ( + _ encoding.ProtoMarshaler = (*KeyValue)(nil) + _ encoding.ProtoUnmarshaler = (*KeyValue)(nil) + _ json.Marshaler = (*KeyValue)(nil) + _ json.Unmarshaler = (*KeyValue)(nil) +) // StableSize returns the size of x in protobuf format. // @@ -16,27 +37,155 @@ func (x *KeyValue) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *KeyValue) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.StringMarshal(1, buf[offset:], x.Key) - offset += proto.BytesMarshal(2, buf[offset:], x.Value) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *KeyValue) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *KeyValue) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Key) != 0 { + mm.AppendString(1, x.Key) + } + if len(x.Value) != 0 { + mm.AppendBytes(2, x.Value) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *KeyValue) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "KeyValue") + } + switch fc.FieldNum { + case 1: // Key + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Key") + } + x.Key = data + case 2: // Value + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Value") + } + x.Value = data + } + } + return nil +} +func (x *KeyValue) GetKey() string { + if x != nil { + return x.Key + } + return "" +} +func (x *KeyValue) SetKey(v string) { + x.Key = v +} +func (x *KeyValue) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} +func (x *KeyValue) SetValue(v []byte) { + x.Value = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *KeyValue) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *KeyValue) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"key\":" + out.RawString(prefix[1:]) + out.String(x.Key) + } + { + const prefix string = ",\"value\":" + out.RawString(prefix) + out.Base64Bytes(x.Value) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *KeyValue) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *KeyValue) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "key": + { + var f string + f = in.String() + x.Key = f + } + case "value": + { + var f []byte + f = in.Bytes() + x.Value = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type LogMove struct { + ParentId uint64 `json:"parentID"` + Meta []byte `json:"meta"` + ChildId uint64 `json:"childID"` +} + +var ( + _ encoding.ProtoMarshaler = (*LogMove)(nil) + _ encoding.ProtoUnmarshaler = (*LogMove)(nil) + _ json.Marshaler = (*LogMove)(nil) + _ json.Unmarshaler = (*LogMove)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -50,28 +199,183 @@ func (x *LogMove) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *LogMove) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.UInt64Marshal(1, buf[offset:], x.ParentId) - offset += proto.BytesMarshal(2, buf[offset:], x.Meta) - offset += proto.UInt64Marshal(3, buf[offset:], x.ChildId) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *LogMove) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst } +func (x *LogMove) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.ParentId != 0 { + mm.AppendUint64(1, x.ParentId) + } + if len(x.Meta) != 0 { + mm.AppendBytes(2, x.Meta) + } + if x.ChildId != 0 { + mm.AppendUint64(3, x.ChildId) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *LogMove) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "LogMove") + } + switch fc.FieldNum { + case 1: // ParentId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ParentId") + } + x.ParentId = data + case 2: // Meta + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Meta") + } + x.Meta = data + case 3: // ChildId + data, ok := fc.Uint64() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ChildId") + } + x.ChildId = data + } + } + return nil +} +func (x *LogMove) GetParentId() uint64 { + if x != nil { + return x.ParentId + } + return 0 +} +func (x *LogMove) SetParentId(v uint64) { + x.ParentId = v +} +func (x *LogMove) GetMeta() []byte { + if x != nil { + return x.Meta + } + return nil +} +func (x *LogMove) SetMeta(v []byte) { + x.Meta = v +} +func (x *LogMove) GetChildId() uint64 { + if x != nil { + return x.ChildId + } + return 0 +} +func (x *LogMove) SetChildId(v uint64) { + x.ChildId = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *LogMove) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *LogMove) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"parentID\":" + out.RawString(prefix[1:]) + out.Uint64(x.ParentId) + } + { + const prefix string = ",\"meta\":" + out.RawString(prefix) + out.Base64Bytes(x.Meta) + } + { + const prefix string = ",\"childID\":" + out.RawString(prefix) + out.Uint64(x.ChildId) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *LogMove) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *LogMove) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "parentID": + { + var f uint64 + f = in.Uint64() + x.ParentId = f + } + case "meta": + { + var f []byte + f = in.Bytes() + x.Meta = f + } + case "childID": + { + var f uint64 + f = in.Uint64() + x.ChildId = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type Signature struct { + Key []byte `json:"key"` + Sign []byte `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*Signature)(nil) + _ encoding.ProtoUnmarshaler = (*Signature)(nil) + _ json.Marshaler = (*Signature)(nil) + _ json.Unmarshaler = (*Signature)(nil) +) + // StableSize returns the size of x in protobuf format. // // Structures with the same field values have the same binary size. @@ -84,23 +388,138 @@ func (x *Signature) StableSize() (size int) { return size } -// StableMarshal marshals x in protobuf binary format with stable field order. -// -// If buffer length is less than x.StableSize(), new buffer is allocated. -// -// Returns any error encountered which did not allow writing the data completely. -// Otherwise, returns the buffer in which the data is written. -// -// Structures with the same field values have the same binary format. -func (x *Signature) StableMarshal(buf []byte) []byte { - if x == nil { - return []byte{} - } - if buf == nil { - buf = make([]byte, x.StableSize()) - } - var offset int - offset += proto.BytesMarshal(1, buf[offset:], x.Key) - offset += proto.BytesMarshal(2, buf[offset:], x.Sign) - return buf +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *Signature) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *Signature) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Key) != 0 { + mm.AppendBytes(1, x.Key) + } + if len(x.Sign) != 0 { + mm.AppendBytes(2, x.Sign) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *Signature) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "Signature") + } + switch fc.FieldNum { + case 1: // Key + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Key") + } + x.Key = data + case 2: // Sign + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Sign") + } + x.Sign = data + } + } + return nil +} +func (x *Signature) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} +func (x *Signature) SetKey(v []byte) { + x.Key = v +} +func (x *Signature) GetSign() []byte { + if x != nil { + return x.Sign + } + return nil +} +func (x *Signature) SetSign(v []byte) { + x.Sign = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *Signature) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *Signature) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"key\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.Key) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + out.Base64Bytes(x.Sign) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *Signature) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *Signature) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "key": + { + var f []byte + f = in.Bytes() + x.Key = f + } + case "signature": + { + var f []byte + f = in.Bytes() + x.Sign = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } } From 15dae8685e48bf9c271039bd109a21a1612e4a3b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 19 Aug 2024 17:13:39 +0300 Subject: [PATCH 1287/1943] [#1323] metabase: Drop ownerID index Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/put.go | 9 -- pkg/local_object_storage/metabase/select.go | 8 +- .../metabase/select_test.go | 106 ++++++++++++++++++ pkg/local_object_storage/metabase/util.go | 9 +- 4 files changed, 113 insertions(+), 19 deletions(-) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index f351cb485..366cc1361 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -379,15 +379,6 @@ func updateFKBTIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun objKey := objectKey(id, make([]byte, objectKeySize)) key := make([]byte, bucketKeySize) - err := f(tx, namedBucketItem{ - name: ownerBucketName(cnr, key), - key: []byte(obj.OwnerID().EncodeToString()), - val: objKey, - }) - if err != nil { - return err - } - var attrs []objectSDK.Attribute if obj.ECHeader() != nil { attrs = obj.ECHeader().ParentAttributes() diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 3a4d7a227..cc40b81ba 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -196,8 +196,7 @@ func (db *DB) selectFastFilter( case v2object.FilterHeaderObjectID: db.selectObjectID(tx, f, cnr, to, fNum, currEpoch) case v2object.FilterHeaderOwnerID: - bucketName := ownerBucketName(cnr, bucketName) - db.selectFromFKBT(tx, bucketName, f, to, fNum) + return // moved to slow filters case v2object.FilterHeaderPayloadHash: bucketName := payloadHashBucketName(cnr, bucketName) db.selectFromList(tx, bucketName, f, to, fNum) @@ -510,6 +509,8 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.Searc case v2object.FilterHeaderPayloadLength: data = make([]byte, 8) binary.LittleEndian.PutUint64(data, obj.PayloadSize()) + case v2object.FilterHeaderOwnerID: + data = []byte(obj.OwnerID().EncodeToString()) default: continue // ignore unknown search attributes } @@ -544,7 +545,8 @@ func groupFilters(filters objectSDK.SearchFilters) (filterGroup, error) { v2object.FilterHeaderVersion, v2object.FilterHeaderCreationEpoch, v2object.FilterHeaderPayloadLength, - v2object.FilterHeaderHomomorphicHash: + v2object.FilterHeaderHomomorphicHash, + v2object.FilterHeaderOwnerID: res.slowFilters = append(res.slowFilters, filters[i]) default: // fast filters or user attributes if unknown res.fastFilters = append(res.fastFilters, filters[i]) diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 8f9294d07..4fbc5910e 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -633,6 +633,112 @@ func TestDB_SelectObjectID(t *testing.T) { }) } +func TestDB_SelectOwnerID(t *testing.T) { + t.Parallel() + + db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() + + cnr := cidtest.ID() + + // prepare + + parent := testutil.GenerateObjectWithCID(cnr) + + regular := testutil.GenerateObjectWithCID(cnr) + idParent, _ := parent.ID() + regular.SetParentID(idParent) + regular.SetParent(parent) + + err := putBig(db, regular) + require.NoError(t, err) + + ts := testutil.GenerateObjectWithCID(cnr) + ts.SetType(objectSDK.TypeTombstone) + err = putBig(db, ts) + require.NoError(t, err) + + lock := testutil.GenerateObjectWithCID(cnr) + lock.SetType(objectSDK.TypeLock) + err = putBig(db, lock) + require.NoError(t, err) + + t.Run("not found objects", func(t *testing.T) { + raw := testutil.GenerateObjectWithCID(cnr) + + fs := objectSDK.SearchFilters{} + fs.AddObjectOwnerIDFilter(objectSDK.MatchStringEqual, raw.OwnerID()) + + testSelect(t, db, cnr, fs) + + fs = objectSDK.SearchFilters{} + fs.AddObjectOwnerIDFilter(objectSDK.MatchStringNotEqual, raw.OwnerID()) + + testSelect(t, db, cnr, fs, + object.AddressOf(regular), + object.AddressOf(parent), + object.AddressOf(ts), + object.AddressOf(lock), + ) + }) + + t.Run("regular objects", func(t *testing.T) { + fs := objectSDK.SearchFilters{} + fs.AddObjectOwnerIDFilter(objectSDK.MatchStringEqual, regular.OwnerID()) + testSelect(t, db, cnr, fs, object.AddressOf(regular)) + + fs = objectSDK.SearchFilters{} + fs.AddObjectOwnerIDFilter(objectSDK.MatchStringNotEqual, regular.OwnerID()) + testSelect(t, db, cnr, fs, + object.AddressOf(parent), + object.AddressOf(ts), + object.AddressOf(lock), + ) + }) + + t.Run("tombstone objects", func(t *testing.T) { + fs := objectSDK.SearchFilters{} + fs.AddObjectOwnerIDFilter(objectSDK.MatchStringEqual, ts.OwnerID()) + testSelect(t, db, cnr, fs, object.AddressOf(ts)) + + fs = objectSDK.SearchFilters{} + fs.AddObjectOwnerIDFilter(objectSDK.MatchStringNotEqual, ts.OwnerID()) + testSelect(t, db, cnr, fs, + object.AddressOf(regular), + object.AddressOf(parent), + object.AddressOf(lock), + ) + }) + + t.Run("parent objects", func(t *testing.T) { + fs := objectSDK.SearchFilters{} + fs.AddObjectOwnerIDFilter(objectSDK.MatchStringEqual, parent.OwnerID()) + testSelect(t, db, cnr, fs, object.AddressOf(parent)) + + fs = objectSDK.SearchFilters{} + fs.AddObjectOwnerIDFilter(objectSDK.MatchStringNotEqual, parent.OwnerID()) + testSelect(t, db, cnr, fs, + object.AddressOf(regular), + object.AddressOf(ts), + object.AddressOf(lock), + ) + }) + + t.Run("lock objects", func(t *testing.T) { + fs := objectSDK.SearchFilters{} + fs.AddObjectOwnerIDFilter(objectSDK.MatchStringEqual, lock.OwnerID()) + testSelect(t, db, cnr, fs, object.AddressOf(lock)) + + fs = objectSDK.SearchFilters{} + fs.AddObjectOwnerIDFilter(objectSDK.MatchStringNotEqual, lock.OwnerID()) + testSelect(t, db, cnr, fs, + object.AddressOf(regular), + object.AddressOf(parent), + object.AddressOf(ts), + ) + }) +} + type testTarget struct { objects []*objectSDK.Object } diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index 9249ae49b..f9f9898e9 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -89,10 +89,10 @@ const ( // FKBT index buckets. // ==================== - // ownerPrefix is used for prefixing FKBT index buckets mapping owner to object IDs. + // ownerPrefix was used for prefixing FKBT index buckets mapping owner to object IDs. // Key: owner ID // Value: bucket containing object IDs as keys - ownerPrefix + _ // userAttributePrefix is used for prefixing FKBT index buckets containing objects. // Key: attribute value // Value: bucket containing object IDs as keys @@ -180,11 +180,6 @@ func rootBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, rootPrefix, key) } -// ownerBucketName returns _ownerid. -func ownerBucketName(cnr cid.ID, key []byte) []byte { - return bucketName(cnr, ownerPrefix, key) -} - // parentBucketName returns _parent. func parentBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, parentPrefix, key) From 2542d4f5dfda623a42ae144cc28ebfeb3d16ea89 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 19 Aug 2024 17:20:55 +0300 Subject: [PATCH 1288/1943] [#1323] metabase: Drop payload checksum index Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/put.go | 12 ------------ pkg/local_object_storage/metabase/select.go | 12 +++++++----- pkg/local_object_storage/metabase/util.go | 9 ++------- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 366cc1361..3fa79f1e2 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -297,18 +297,6 @@ func updateListIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun objKey := objectKey(idObj, make([]byte, objectKeySize)) bucketName := make([]byte, bucketKeySize) - cs, _ := obj.PayloadChecksum() - - // index payload hashes - err := f(tx, namedBucketItem{ - name: payloadHashBucketName(cnr, bucketName), - key: cs.Value(), - val: objKey, - }) - if err != nil { - return err - } - idParent, ok := obj.ParentID() // index parent ids diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index cc40b81ba..720b7b5b9 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -195,11 +195,9 @@ func (db *DB) selectFastFilter( switch f.Header() { case v2object.FilterHeaderObjectID: db.selectObjectID(tx, f, cnr, to, fNum, currEpoch) - case v2object.FilterHeaderOwnerID: + case v2object.FilterHeaderOwnerID, + v2object.FilterHeaderPayloadHash: return // moved to slow filters - case v2object.FilterHeaderPayloadHash: - bucketName := payloadHashBucketName(cnr, bucketName) - db.selectFromList(tx, bucketName, f, to, fNum) case v2object.FilterHeaderObjectType: for _, bucketName := range bucketNamesForType(cnr, f.Operation(), f.Value()) { selectAllFromBucket(tx, bucketName, to, fNum) @@ -511,6 +509,9 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.Searc binary.LittleEndian.PutUint64(data, obj.PayloadSize()) case v2object.FilterHeaderOwnerID: data = []byte(obj.OwnerID().EncodeToString()) + case v2object.FilterHeaderPayloadHash: + cs, _ := obj.PayloadChecksum() + data = cs.Value() default: continue // ignore unknown search attributes } @@ -546,7 +547,8 @@ func groupFilters(filters objectSDK.SearchFilters) (filterGroup, error) { v2object.FilterHeaderCreationEpoch, v2object.FilterHeaderPayloadLength, v2object.FilterHeaderHomomorphicHash, - v2object.FilterHeaderOwnerID: + v2object.FilterHeaderOwnerID, + v2object.FilterHeaderPayloadHash: res.slowFilters = append(res.slowFilters, filters[i]) default: // fast filters or user attributes if unknown res.fastFilters = append(res.fastFilters, filters[i]) diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index f9f9898e9..ebf1713d0 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -102,10 +102,10 @@ const ( // List index buckets. // ==================== - // payloadHashPrefix is used for prefixing List index buckets mapping payload hash to a list of object IDs. + // payloadHashPrefix was used for prefixing List index buckets mapping payload hash to a list of object IDs. // Key: payload hash // Value: list of object IDs - payloadHashPrefix + _ // parentPrefix is used for prefixing List index buckets mapping parent ID to a list of children IDs. // Key: parent ID // Value: list of object IDs @@ -170,11 +170,6 @@ func cidFromAttributeBucket(val []byte, attributeKey string) []byte { return val[1:bucketKeySize] } -// payloadHashBucketName returns _payloadhash. -func payloadHashBucketName(cnr cid.ID, key []byte) []byte { - return bucketName(cnr, payloadHashPrefix, key) -} - // rootBucketName returns _root. func rootBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, rootPrefix, key) From 7bf20c9f1f06975a53184c0803dbc7bc5ca4eaf5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Aug 2024 11:59:42 +0300 Subject: [PATCH 1289/1943] [#1323] metabase: Add expiration epoch buckets Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/control.go | 1 + pkg/local_object_storage/metabase/delete.go | 11 ++ .../metabase/delete_ec_test.go | 3 +- pkg/local_object_storage/metabase/exists.go | 26 ++-- pkg/local_object_storage/metabase/expired.go | 118 +++++------------- pkg/local_object_storage/metabase/get.go | 6 +- .../metabase/iterators.go | 81 ++++-------- pkg/local_object_storage/metabase/put.go | 41 ++++++ pkg/local_object_storage/metabase/select.go | 7 +- pkg/local_object_storage/metabase/util.go | 53 ++++++-- pkg/local_object_storage/shard/gc_test.go | 4 +- 11 files changed, 181 insertions(+), 170 deletions(-) diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 891a1e9b2..d6546d922 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -29,6 +29,7 @@ var ( string(garbageBucketName): {}, string(shardInfoBucket): {}, string(bucketNameLocked): {}, + string(expEpochToObjectBucketName): {}, } // deprecatedBuckets buckets that are not used anymore. diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index ae10564a8..683bd445f 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -478,6 +478,17 @@ func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error key: objKey, }) + if expEpoch, ok := hasExpirationEpoch(obj); ok { + delUniqueIndexItem(tx, namedBucketItem{ + name: expEpochToObjectBucketName, + key: expirationEpochKey(expEpoch, cnr, addr.Object()), + }) + delUniqueIndexItem(tx, namedBucketItem{ + name: objectToExpirationEpochBucketName(cnr, make([]byte, bucketKeySize)), + key: objKey, + }) + } + return nil } diff --git a/pkg/local_object_storage/metabase/delete_ec_test.go b/pkg/local_object_storage/metabase/delete_ec_test.go index 0e627f095..66c79ecd7 100644 --- a/pkg/local_object_storage/metabase/delete_ec_test.go +++ b/pkg/local_object_storage/metabase/delete_ec_test.go @@ -418,7 +418,8 @@ func testVerifyNoObjectDataLeft(tx *bbolt.Tx) error { return tx.ForEach(func(name []byte, b *bbolt.Bucket) error { if bytes.Equal(name, shardInfoBucket) || bytes.Equal(name, containerCounterBucketName) || - bytes.Equal(name, containerVolumeBucketName) { + bytes.Equal(name, containerVolumeBucketName) || + bytes.Equal(name, expEpochToObjectBucketName) { return nil } return testBucketEmpty(name, b) diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index 153d92110..2e1b1dce8 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -96,7 +95,11 @@ func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, parent oid.Address, currEpo locked = objectLocked(tx, parent.Container(), parent.Object()) } // check graveyard and object expiration first - switch objectStatus(tx, addr, currEpoch) { + st, err := objectStatus(tx, addr, currEpoch) + if err != nil { + return false, false, err + } + switch st { case 1: return false, locked, logicerr.Wrap(new(apistatus.ObjectNotFound)) case 2: @@ -138,30 +141,25 @@ func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, parent oid.Address, currEpo // - 1 if object with GC mark; // - 2 if object is covered with tombstone; // - 3 if object is expired. -func objectStatus(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) uint8 { +func objectStatus(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (uint8, error) { // locked object could not be removed/marked with GC/expired if objectLocked(tx, addr.Container(), addr.Object()) { - return 0 + return 0, nil } - // we check only if the object is expired in the current - // epoch since it is considered the only corner case: the - // GC is expected to collect all the objects that have - // expired previously for less than the one epoch duration - - expired := isExpiredWithAttribute(tx, objectV2.SysAttributeExpEpoch, addr, currEpoch) - if !expired { - expired = isExpiredWithAttribute(tx, objectV2.SysAttributeExpEpochNeoFS, addr, currEpoch) + expired, err := isExpired(tx, addr, currEpoch) + if err != nil { + return 0, err } if expired { - return 3 + return 3, nil } graveyardBkt := tx.Bucket(graveyardBucketName) garbageBkt := tx.Bucket(garbageBucketName) addrKey := addressKey(addr, make([]byte, addressKeySize)) - return inGraveyardWithKey(addrKey, graveyardBkt, garbageBkt) + return inGraveyardWithKey(addrKey, graveyardBkt, garbageBkt), nil } func inGraveyardWithKey(addrKey []byte, graveyard, garbageBCK *bbolt.Bucket) uint8 { diff --git a/pkg/local_object_storage/metabase/expired.go b/pkg/local_object_storage/metabase/expired.go index aa2cb6f20..68144d8b1 100644 --- a/pkg/local_object_storage/metabase/expired.go +++ b/pkg/local_object_storage/metabase/expired.go @@ -2,12 +2,11 @@ package meta import ( "context" + "encoding/binary" "errors" - "fmt" "strconv" "time" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -17,6 +16,8 @@ import ( "go.opentelemetry.io/otel/trace" ) +var errInvalidEpochValueLength = errors.New("could not parse expiration epoch: invalid data length") + // FilterExpired return expired items from addresses. // Address considered expired if metabase does contain information about expiration and // expiration epoch is less than epoch. @@ -57,29 +58,11 @@ func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.A default: } - expiredNeoFS, err := selectExpiredObjectIDs(tx, objectV2.SysAttributeExpEpochNeoFS, epoch, containerID, objectIDs) + expired, err := selectExpiredObjects(tx, epoch, containerID, objectIDs) if err != nil { return err } - - expiredSys, err := selectExpiredObjectIDs(tx, objectV2.SysAttributeExpEpoch, epoch, containerID, objectIDs) - if err != nil { - return err - } - - for _, o := range expiredNeoFS { - var a oid.Address - a.SetContainer(containerID) - a.SetObject(o) - result = append(result, a) - } - - for _, o := range expiredSys { - var a oid.Address - a.SetContainer(containerID) - a.SetObject(o) - result = append(result, a) - } + result = append(result, expired...) } return nil }) @@ -90,76 +73,39 @@ func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.A return result, nil } -func isExpiredWithAttribute(tx *bbolt.Tx, attr string, addr oid.Address, currEpoch uint64) bool { - // bucket with objects that have expiration attr - attrKey := make([]byte, bucketKeySize+len(attr)) - expirationBucket := tx.Bucket(attributeBucketName(addr.Container(), attr, attrKey)) - if expirationBucket != nil { - // bucket that contains objects that expire in the current epoch - prevEpochBkt := expirationBucket.Bucket([]byte(strconv.FormatUint(currEpoch-1, 10))) - if prevEpochBkt != nil { - rawOID := objectKey(addr.Object(), make([]byte, objectKeySize)) - if prevEpochBkt.Get(rawOID) != nil { - return true - } - } +func isExpired(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (bool, error) { + bucketName := make([]byte, bucketKeySize) + bucketName = objectToExpirationEpochBucketName(addr.Container(), bucketName) + b := tx.Bucket(bucketName) + if b == nil { + return false, nil } - - return false + key := make([]byte, objectKeySize) + addr.Object().Encode(key) + val := b.Get(key) + if len(val) == 0 { + return false, nil + } + if len(val) != epochSize { + return false, errInvalidEpochValueLength + } + expEpoch := binary.LittleEndian.Uint64(val) + return expEpoch < currEpoch, nil } -func selectExpiredObjectIDs(tx *bbolt.Tx, attr string, epoch uint64, containerID cid.ID, objectIDs []oid.ID) ([]oid.ID, error) { - result := make([]oid.ID, 0) - notResolved := make(map[oid.ID]struct{}) - for _, oid := range objectIDs { - notResolved[oid] = struct{}{} - } - - expiredBuffer := make([]oid.ID, 0) - objectKeyBuffer := make([]byte, objectKeySize) - - expirationBucketKey := make([]byte, bucketKeySize+len(attr)) - expirationBucket := tx.Bucket(attributeBucketName(containerID, attr, expirationBucketKey)) - if expirationBucket == nil { - return result, nil // all not expired - } - - err := expirationBucket.ForEach(func(epochExpBucketKey, _ []byte) error { - bucketExpiresAfter, err := strconv.ParseUint(string(epochExpBucketKey), 10, 64) +func selectExpiredObjects(tx *bbolt.Tx, epoch uint64, containerID cid.ID, objectIDs []oid.ID) ([]oid.Address, error) { + result := make([]oid.Address, 0) + var addr oid.Address + addr.SetContainer(containerID) + for _, objID := range objectIDs { + addr.SetObject(objID) + expired, err := isExpired(tx, addr, epoch) if err != nil { - return fmt.Errorf("could not parse expiration epoch: %w", err) - } else if bucketExpiresAfter >= epoch { - return nil + return nil, err } - - epochExpirationBucket := expirationBucket.Bucket(epochExpBucketKey) - if epochExpirationBucket == nil { - return nil + if expired { + result = append(result, addr) } - - expiredBuffer = expiredBuffer[:0] - for oid := range notResolved { - key := objectKey(oid, objectKeyBuffer) - if epochExpirationBucket.Get(key) != nil { - expiredBuffer = append(expiredBuffer, oid) - } - } - - for _, oid := range expiredBuffer { - delete(notResolved, oid) - result = append(result, oid) - } - - if len(notResolved) == 0 { - return errBreakBucketForEach - } - - return nil - }) - - if err != nil && !errors.Is(err, errBreakBucketForEach) { - return nil, err } - return result, nil } diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index d979b4f0f..776f5d27c 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -90,7 +90,11 @@ func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw bool, currEpoch uint64) (*objectSDK.Object, error) { if checkStatus { - switch objectStatus(tx, addr, currEpoch) { + st, err := objectStatus(tx, addr, currEpoch) + if err != nil { + return nil, err + } + switch st { case 1: return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) case 2: diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index 7b60b7d50..d44c51fb2 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -7,7 +7,6 @@ import ( "strconv" "time" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -79,63 +78,37 @@ func (db *DB) IterateExpired(ctx context.Context, epoch uint64, h ExpiredObjectH } func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) error { - err := tx.ForEach(func(name []byte, b *bbolt.Bucket) error { - cidBytes := cidFromAttributeBucket(name, objectV2.SysAttributeExpEpoch) - if cidBytes == nil { - cidBytes = cidFromAttributeBucket(name, objectV2.SysAttributeExpEpochNeoFS) - if cidBytes == nil { - return nil - } - } - - var cnrID cid.ID - err := cnrID.Decode(cidBytes) + b := tx.Bucket(expEpochToObjectBucketName) + c := b.Cursor() + for k, _ := c.First(); k != nil; k, _ = c.Next() { + expiresAfter, cnr, obj, err := parseExpirationEpochKey(k) if err != nil { - return fmt.Errorf("could not parse container ID of expired bucket: %w", err) + return err } - - return b.ForEachBucket(func(expKey []byte) error { - bktExpired := b.Bucket(expKey) - expiresAfter, err := strconv.ParseUint(string(expKey), 10, 64) - if err != nil { - return fmt.Errorf("could not parse expiration epoch: %w", err) - } else if expiresAfter >= epoch { - return nil - } - - return bktExpired.ForEach(func(idKey, _ []byte) error { - var id oid.ID - - err = id.Decode(idKey) - if err != nil { - return fmt.Errorf("could not parse ID of expired object: %w", err) - } - - // Ignore locked objects. - // - // To slightly optimize performance we can check only REGULAR objects - // (only they can be locked), but it's more reliable. - if objectLocked(tx, cnrID, id) { - return nil - } - - var addr oid.Address - addr.SetContainer(cnrID) - addr.SetObject(id) - - return h(&ExpiredObject{ - typ: firstIrregularObjectType(tx, cnrID, idKey), - addr: addr, - }) - }) + // bucket keys ordered by epoch, no need to continue lookup + if expiresAfter >= epoch { + return nil + } + if objectLocked(tx, cnr, obj) { + continue + } + var addr oid.Address + addr.SetContainer(cnr) + addr.SetObject(obj) + objKey := objectKey(addr.Object(), make([]byte, objectKeySize)) + err = h(&ExpiredObject{ + typ: firstIrregularObjectType(tx, cnr, objKey), + addr: addr, }) - }) - - if errors.Is(err, ErrInterruptIterator) { - err = nil + if err == nil { + continue + } + if errors.Is(err, ErrInterruptIterator) { + return nil + } + return err } - - return err + return nil } // IterateCoveredByTombstones iterates over all objects in DB which are covered diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 3fa79f1e2..d1706a7ab 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -6,8 +6,10 @@ import ( "errors" "fmt" gio "io" + "strconv" "time" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" @@ -242,6 +244,27 @@ func putUniqueIndexes( } } + if expEpoch, ok := hasExpirationEpoch(obj); ok { + err := putUniqueIndexItem(tx, namedBucketItem{ + name: expEpochToObjectBucketName, + key: expirationEpochKey(expEpoch, cnr, addr.Object()), + val: zeroValue, + }) + if err != nil { + return err + } + val := make([]byte, epochSize) + binary.LittleEndian.PutUint64(val, expEpoch) + err = putUniqueIndexItem(tx, namedBucketItem{ + name: objectToExpirationEpochBucketName(cnr, make([]byte, bucketKeySize)), + key: objKey, + val: val, + }) + if err != nil { + return err + } + } + // index root object if obj.Type() == objectSDK.TypeRegular && !obj.HasParent() { if ecHead := obj.ECHeader(); ecHead != nil { @@ -361,6 +384,24 @@ func updateListIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun return nil } +func hasExpirationEpoch(obj *objectSDK.Object) (uint64, bool) { + attributes := obj.Attributes() + if ech := obj.ECHeader(); ech != nil { + attributes = ech.ParentAttributes() + } + for _, attr := range attributes { + if attr.Key() == objectV2.SysAttributeExpEpochNeoFS { + expEpoch, err := strconv.ParseUint(attr.Value(), 10, 64) + return expEpoch, err == nil + } + if attr.Key() == objectV2.SysAttributeExpEpoch { + expEpoch, err := strconv.ParseUint(attr.Value(), 10, 64) + return expEpoch, err == nil + } + } + return 0, false +} + func updateFKBTIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFunc) error { id, _ := obj.ID() cnr, _ := obj.ContainerID() diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 720b7b5b9..42737a41a 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -142,8 +142,11 @@ func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters var addr oid.Address addr.SetContainer(cnr) addr.SetObject(id) - - if objectStatus(tx, addr, currEpoch) > 0 { + st, err := objectStatus(tx, addr, currEpoch) + if err != nil { + return nil, err + } + if st > 0 { continue // ignore removed objects } diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index ebf1713d0..012c0dcc8 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -1,9 +1,10 @@ package meta import ( - "bytes" "crypto/sha256" + "encoding/binary" "errors" + "fmt" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -23,6 +24,7 @@ var ( toMoveItBucketName = []byte{toMoveItPrefix} containerVolumeBucketName = []byte{containerVolumePrefix} containerCounterBucketName = []byte{containerCountersPrefix} + expEpochToObjectBucketName = []byte{expirationEpochToObjectPrefix} zeroValue = []byte{0xFF} @@ -124,6 +126,16 @@ const ( // Key: container ID + type // Value: Object id ecInfoPrefix + + // expirationEpochToObjectPrefix is used for storing relation between expiration epoch and object id. + // Key: expiration epoch + object address + // Value: zero + expirationEpochToObjectPrefix + + // objectToExpirationEpochPrefix is used for storing relation between expiration epoch and object id. + // Key: object address + // Value: expiration epoch + objectToExpirationEpochPrefix ) const ( @@ -131,6 +143,7 @@ const ( bucketKeySize = 1 + cidSize objectKeySize = sha256.Size addressKeySize = cidSize + objectKeySize + epochSize = 8 ) func bucketName(cnr cid.ID, prefix byte, key []byte) []byte { @@ -161,15 +174,6 @@ func attributeBucketName(cnr cid.ID, attributeKey string, key []byte) []byte { return append(key[:bucketKeySize], attributeKey...) } -// returns from attributeBucketName result, nil otherwise. -func cidFromAttributeBucket(val []byte, attributeKey string) []byte { - if len(val) < bucketKeySize || val[0] != userAttributePrefix || !bytes.Equal(val[bucketKeySize:], []byte(attributeKey)) { - return nil - } - - return val[1:bucketKeySize] -} - // rootBucketName returns _root. func rootBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, rootPrefix, key) @@ -190,6 +194,35 @@ func ecInfoBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, ecInfoPrefix, key) } +// objectToExpirationEpochBucketName returns objectToExpirationEpochPrefix_. +func objectToExpirationEpochBucketName(cnr cid.ID, key []byte) []byte { + return bucketName(cnr, objectToExpirationEpochPrefix, key) +} + +func expirationEpochKey(epoch uint64, cnr cid.ID, obj oid.ID) []byte { + result := make([]byte, epochSize+addressKeySize) + binary.BigEndian.PutUint64(result, epoch) + cnr.Encode(result[epochSize:]) + obj.Encode(result[epochSize+cidSize:]) + return result +} + +func parseExpirationEpochKey(key []byte) (uint64, cid.ID, oid.ID, error) { + if len(key) != epochSize+addressKeySize { + return 0, cid.ID{}, oid.ID{}, fmt.Errorf("unexpected expiration epoch to object key length: %d", len(key)) + } + epoch := binary.BigEndian.Uint64(key) + var cnr cid.ID + if err := cnr.Decode(key[epochSize : epochSize+cidSize]); err != nil { + return 0, cid.ID{}, oid.ID{}, fmt.Errorf("failed to decode expiration epoch to object key (container ID): %w", err) + } + var obj oid.ID + if err := obj.Decode(key[epochSize+cidSize:]); err != nil { + return 0, cid.ID{}, oid.ID{}, fmt.Errorf("failed to decode expiration epoch to object key (object ID): %w", err) + } + return epoch, cnr, obj, nil +} + // addressKey returns key for K-V tables when key is a whole address. func addressKey(addr oid.Address, key []byte) []byte { addr.Container().Encode(key) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index a438b5def..90958cd35 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -74,7 +74,7 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { var getPrm GetPrm getPrm.SetAddress(objectCore.AddressOf(obj)) _, err = sh.Get(context.Background(), getPrm) - require.True(t, client.IsErrObjectNotFound(err), "expired object must be deleted") + require.True(t, client.IsErrObjectNotFound(err) || IsErrObjectExpired(err), "expired object must be deleted") } func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { @@ -168,7 +168,7 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { sh.gc.handleEvent(context.Background(), EventNewEpoch(epoch.Value)) _, err = sh.Get(context.Background(), getPrm) - require.True(t, client.IsErrObjectNotFound(err), "expired complex object must be deleted on epoch after lock expires") + require.True(t, client.IsErrObjectNotFound(err) || IsErrObjectExpired(err), "expired complex object must be deleted on epoch after lock expires") } func TestGCDropsObjectInhumedFromWritecache(t *testing.T) { From 0f08a2efba6a39ec8a1c8f733729d510e3ea36bd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Aug 2024 12:21:39 +0300 Subject: [PATCH 1290/1943] [#1323] metabase: Resolve funlen linter Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/put.go | 74 ++++++++++++------------ 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index d1706a7ab..015f87f92 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -197,46 +197,17 @@ func (db *DB) insertObject(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *o return nil } -func putUniqueIndexes( - tx *bbolt.Tx, - obj *objectSDK.Object, - si *objectSDK.SplitInfo, - id []byte, -) error { +func putUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, si *objectSDK.SplitInfo, id []byte) error { isParent := si != nil addr := objectCore.AddressOf(obj) - cnr := addr.Container() objKey := objectKey(addr.Object(), make([]byte, objectKeySize)) bucketName := make([]byte, bucketKeySize) - // add value to primary unique bucket if !isParent { - switch obj.Type() { - case objectSDK.TypeRegular: - bucketName = primaryBucketName(cnr, bucketName) - case objectSDK.TypeTombstone: - bucketName = tombstoneBucketName(cnr, bucketName) - case objectSDK.TypeLock: - bucketName = bucketNameLockers(cnr, bucketName) - default: - return ErrUnknownObjectType - } - - rawObject, err := obj.CutPayload().Marshal() - if err != nil { - return fmt.Errorf("can't marshal object header: %w", err) - } - - err = putUniqueIndexItem(tx, namedBucketItem{ - name: bucketName, - key: objKey, - val: rawObject, - }) + err := putRawObjectData(tx, obj, bucketName, addr, objKey) if err != nil { return err } - - // index storageID if it is present if id != nil { if err = setStorageID(tx, objectCore.AddressOf(obj), id, false); err != nil { return err @@ -244,10 +215,40 @@ func putUniqueIndexes( } } + if err := putExpirationEpoch(tx, obj, addr, objKey); err != nil { + return err + } + + return putSplitInfo(tx, obj, bucketName, addr, si, objKey) +} + +func putRawObjectData(tx *bbolt.Tx, obj *objectSDK.Object, bucketName []byte, addr oid.Address, objKey []byte) error { + switch obj.Type() { + case objectSDK.TypeRegular: + bucketName = primaryBucketName(addr.Container(), bucketName) + case objectSDK.TypeTombstone: + bucketName = tombstoneBucketName(addr.Container(), bucketName) + case objectSDK.TypeLock: + bucketName = bucketNameLockers(addr.Container(), bucketName) + default: + return ErrUnknownObjectType + } + rawObject, err := obj.CutPayload().Marshal() + if err != nil { + return fmt.Errorf("can't marshal object header: %w", err) + } + return putUniqueIndexItem(tx, namedBucketItem{ + name: bucketName, + key: objKey, + val: rawObject, + }) +} + +func putExpirationEpoch(tx *bbolt.Tx, obj *objectSDK.Object, addr oid.Address, objKey []byte) error { if expEpoch, ok := hasExpirationEpoch(obj); ok { err := putUniqueIndexItem(tx, namedBucketItem{ name: expEpochToObjectBucketName, - key: expirationEpochKey(expEpoch, cnr, addr.Object()), + key: expirationEpochKey(expEpoch, addr.Container(), addr.Object()), val: zeroValue, }) if err != nil { @@ -256,7 +257,7 @@ func putUniqueIndexes( val := make([]byte, epochSize) binary.LittleEndian.PutUint64(val, expEpoch) err = putUniqueIndexItem(tx, namedBucketItem{ - name: objectToExpirationEpochBucketName(cnr, make([]byte, bucketKeySize)), + name: objectToExpirationEpochBucketName(addr.Container(), make([]byte, bucketKeySize)), key: objKey, val: val, }) @@ -264,8 +265,10 @@ func putUniqueIndexes( return err } } + return nil +} - // index root object +func putSplitInfo(tx *bbolt.Tx, obj *objectSDK.Object, bucketName []byte, addr oid.Address, si *objectSDK.SplitInfo, objKey []byte) error { if obj.Type() == objectSDK.TypeRegular && !obj.HasParent() { if ecHead := obj.ECHeader(); ecHead != nil { parentID := ecHead.Parent() @@ -283,9 +286,8 @@ func putUniqueIndexes( } objKey = objectKey(parentID, objKey) } - return updateSplitInfoIndex(tx, objKey, cnr, bucketName, si) + return updateSplitInfoIndex(tx, objKey, addr.Container(), bucketName, si) } - return nil } From 7d0d781db1b1d674b319a012d653f2bb510f82ca Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Aug 2024 12:59:59 +0300 Subject: [PATCH 1291/1943] [#1323] metabase: Drop user attribute index Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/delete.go | 20 --- pkg/local_object_storage/metabase/put.go | 49 ------- pkg/local_object_storage/metabase/select.go | 151 +++++--------------- pkg/local_object_storage/metabase/util.go | 11 +- 4 files changed, 38 insertions(+), 193 deletions(-) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 683bd445f..00c8d06e0 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -341,11 +341,6 @@ func (db *DB) deleteObject( return fmt.Errorf("can't remove list indexes: %w", err) } - err = updateFKBTIndexes(tx, obj, delFKBTIndexItem) - if err != nil { - return fmt.Errorf("can't remove fake bucket tree indexes: %w", err) - } - if isParent { // remove record from the garbage bucket, because regular object deletion does nothing for virtual object garbageBKT := tx.Bucket(garbageBucketName) @@ -386,21 +381,6 @@ func delUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) { } } -func delFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { - bkt := tx.Bucket(item.name) - if bkt == nil { - return nil - } - - fkbtRoot := bkt.Bucket(item.key) - if fkbtRoot == nil { - return nil - } - - _ = fkbtRoot.Delete(item.val) // ignore error, best effort there - return nil -} - func delListIndexItem(tx *bbolt.Tx, item namedBucketItem) error { bkt := tx.Bucket(item.name) if bkt == nil { diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 015f87f92..ff79a0387 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -175,11 +175,6 @@ func (db *DB) insertObject(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *o return fmt.Errorf("can't put list indexes: %w", err) } - err = updateFKBTIndexes(tx, obj, putFKBTIndexItem) - if err != nil { - return fmt.Errorf("can't put fake bucket tree indexes: %w", err) - } - // update container volume size estimation if obj.Type() == objectSDK.TypeRegular && !isParent { err = changeContainerSize(tx, cnr, obj.PayloadSize(), true) @@ -404,36 +399,6 @@ func hasExpirationEpoch(obj *objectSDK.Object) (uint64, bool) { return 0, false } -func updateFKBTIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFunc) error { - id, _ := obj.ID() - cnr, _ := obj.ContainerID() - objKey := objectKey(id, make([]byte, objectKeySize)) - - key := make([]byte, bucketKeySize) - var attrs []objectSDK.Attribute - if obj.ECHeader() != nil { - attrs = obj.ECHeader().ParentAttributes() - objKey = objectKey(obj.ECHeader().Parent(), make([]byte, objectKeySize)) - } else { - attrs = obj.Attributes() - } - - // user specified attributes - for i := range attrs { - key = attributeBucketName(cnr, attrs[i].Key(), key) - err := f(tx, namedBucketItem{ - name: key, - key: []byte(attrs[i].Value()), - val: objKey, - }) - if err != nil { - return err - } - } - - return nil -} - type bucketContainer interface { Bucket([]byte) *bbolt.Bucket CreateBucket([]byte) (*bbolt.Bucket, error) @@ -464,20 +429,6 @@ func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error { return updateUniqueIndexItem(tx, item, func(_, val []byte) ([]byte, error) { return val, nil }) } -func putFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { - bkt, err := createBucketLikelyExists(tx, item.name) - if err != nil { - return fmt.Errorf("can't create index %v: %w", item.name, err) - } - - fkbtRoot, err := createBucketLikelyExists(bkt, item.key) - if err != nil { - return fmt.Errorf("can't create fake bucket tree index %v: %w", item.key, err) - } - - return fkbtRoot.Put(item.val, zeroValue) -} - func putListIndexItem(tx *bbolt.Tx, item namedBucketItem) error { bkt, err := createBucketLikelyExists(tx, item.name) if err != nil { diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 42737a41a..1863fc25e 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -198,9 +198,6 @@ func (db *DB) selectFastFilter( switch f.Header() { case v2object.FilterHeaderObjectID: db.selectObjectID(tx, f, cnr, to, fNum, currEpoch) - case v2object.FilterHeaderOwnerID, - v2object.FilterHeaderPayloadHash: - return // moved to slow filters case v2object.FilterHeaderObjectType: for _, bucketName := range bucketNamesForType(cnr, f.Operation(), f.Value()) { selectAllFromBucket(tx, bucketName, to, fNum) @@ -220,14 +217,7 @@ func (db *DB) selectFastFilter( selectAllFromBucket(tx, primaryBucketName(cnr, bucketName), to, fNum) selectAllFromBucket(tx, tombstoneBucketName(cnr, bucketName), to, fNum) selectAllFromBucket(tx, bucketNameLockers(cnr, bucketName), to, fNum) - default: // user attribute - bucketName := attributeBucketName(cnr, f.Header(), bucketName) - - if f.Operation() == objectSDK.MatchNotPresent { - selectOutsideFKBT(tx, allBucketNames(cnr), bucketName, to, fNum) - } else { - db.selectFromFKBT(tx, bucketName, f, to, fNum) - } + default: } } @@ -237,16 +227,6 @@ var mBucketNaming = map[string][]func(cid.ID, []byte) []byte{ v2object.TypeLock.String(): {bucketNameLockers}, } -func allBucketNames(cnr cid.ID) (names [][]byte) { - for _, fns := range mBucketNaming { - for _, fn := range fns { - names = append(names, fn(cnr, make([]byte, bucketKeySize))) - } - } - - return -} - func bucketNamesForType(cnr cid.ID, mType objectSDK.SearchMatchType, typeVal string) (names [][]byte) { appendNames := func(key string) { fns, ok := mBucketNaming[key] @@ -278,83 +258,6 @@ func bucketNamesForType(cnr cid.ID, mType objectSDK.SearchMatchType, typeVal str return } -// selectFromList looks into index to find list of addresses to add in -// resulting cache. -func (db *DB) selectFromFKBT( - tx *bbolt.Tx, - name []byte, // fkbt root bucket name - f objectSDK.SearchFilter, // filter for operation and value - to map[string]int, // resulting cache - fNum int, // index of filter -) { // - matchFunc, ok := db.matchers[f.Operation()] - if !ok { - db.log.Debug(logs.MetabaseMissingMatcher, zap.Uint32("operation", uint32(f.Operation()))) - - return - } - - fkbtRoot := tx.Bucket(name) - if fkbtRoot == nil { - return - } - - err := matchFunc.matchBucket(fkbtRoot, f.Header(), f.Value(), func(k, _ []byte) error { - fkbtLeaf := fkbtRoot.Bucket(k) - if fkbtLeaf == nil { - return nil - } - - return fkbtLeaf.ForEach(func(k, _ []byte) error { - markAddressInCache(to, fNum, string(k)) - - return nil - }) - }) - if err != nil { - db.log.Debug(logs.MetabaseErrorInFKBTSelection, zap.String("error", err.Error())) - } -} - -// selectOutsideFKBT looks into all incl buckets to find list of addresses outside to add in -// resulting cache. -func selectOutsideFKBT( - tx *bbolt.Tx, - incl [][]byte, // buckets - name []byte, // fkbt root bucket name - to map[string]int, // resulting cache - fNum int, // index of filter -) { - mExcl := make(map[string]struct{}) - - bktExcl := tx.Bucket(name) - if bktExcl != nil { - _ = bktExcl.ForEachBucket(func(k []byte) error { - exclBktLeaf := bktExcl.Bucket(k) - return exclBktLeaf.ForEach(func(k, _ []byte) error { - mExcl[string(k)] = struct{}{} - - return nil - }) - }) - } - - for i := range incl { - bktIncl := tx.Bucket(incl[i]) - if bktIncl == nil { - continue - } - - _ = bktIncl.ForEach(func(k, _ []byte) error { - if _, ok := mExcl[string(k)]; !ok { - markAddressInCache(to, fNum, string(k)) - } - - return nil - }) - } -} - // selectFromList looks into index to find list of addresses to add in // resulting cache. func (db *DB) selectFromList( @@ -491,13 +394,7 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.Searc } for i := range f { - matchFunc, ok := db.matchers[f[i].Operation()] - if !ok { - return false - } - var data []byte - switch f[i].Header() { case v2object.FilterHeaderVersion: data = []byte(obj.Version().String()) @@ -515,8 +412,18 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.Searc case v2object.FilterHeaderPayloadHash: cs, _ := obj.PayloadChecksum() data = cs.Value() - default: - continue // ignore unknown search attributes + default: // user attribute + v, ok := attributeValue(obj, f[i].Header()) + if ok { + data = []byte(v) + } else { + return f[i].Operation() == objectSDK.MatchNotPresent + } + } + + matchFunc, ok := db.matchers[f[i].Operation()] + if !ok { + return false } if !matchFunc.matchSlow(f[i].Header(), data, f[i].Value()) { @@ -527,6 +434,19 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.Searc return true } +func attributeValue(obj *objectSDK.Object, attribute string) (string, bool) { + objectAttributes := obj.Attributes() + if ech := obj.ECHeader(); ech != nil { + objectAttributes = ech.ParentAttributes() + } + for _, attr := range objectAttributes { + if attr.Key() == attribute { + return attr.Value(), true + } + } + return "", false +} + // groupFilters divides filters in two groups: fast and slow. Fast filters // processed by indexes and slow filters processed after by unmarshaling // object headers. @@ -545,16 +465,17 @@ func groupFilters(filters objectSDK.SearchFilters) (filterGroup, error) { } res.withCnrFilter = true - case // slow filters - v2object.FilterHeaderVersion, - v2object.FilterHeaderCreationEpoch, - v2object.FilterHeaderPayloadLength, - v2object.FilterHeaderHomomorphicHash, - v2object.FilterHeaderOwnerID, - v2object.FilterHeaderPayloadHash: - res.slowFilters = append(res.slowFilters, filters[i]) - default: // fast filters or user attributes if unknown + case // fast filters + v2object.FilterHeaderObjectID, + v2object.FilterHeaderObjectType, + v2object.FilterHeaderParent, + v2object.FilterHeaderSplitID, + v2object.FilterHeaderECParent, + v2object.FilterPropertyRoot, + v2object.FilterPropertyPhy: res.fastFilters = append(res.fastFilters, filters[i]) + default: + res.slowFilters = append(res.slowFilters, filters[i]) } } diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index 012c0dcc8..9134616fe 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -95,10 +95,10 @@ const ( // Key: owner ID // Value: bucket containing object IDs as keys _ - // userAttributePrefix is used for prefixing FKBT index buckets containing objects. + // userAttributePrefix was used for prefixing FKBT index buckets containing objects. // Key: attribute value // Value: bucket containing object IDs as keys - userAttributePrefix + _ // ==================== // List index buckets. @@ -167,13 +167,6 @@ func smallBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, smallPrefix, key) } -// attributeBucketName returns _attr_. -func attributeBucketName(cnr cid.ID, attributeKey string, key []byte) []byte { - key[0] = userAttributePrefix - cnr.Encode(key[1:]) - return append(key[:bucketKeySize], attributeKey...) -} - // rootBucketName returns _root. func rootBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, rootPrefix, key) From 76f67ea34ea316a4b4f05bedb7a5aa2d9599912e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 20 Aug 2024 15:39:44 +0300 Subject: [PATCH 1292/1943] [#1323] metabase: Bump version Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/VERSION.md | 7 +++++++ pkg/local_object_storage/metabase/version.go | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/VERSION.md b/pkg/local_object_storage/metabase/VERSION.md index 97e514db1..9cfc95332 100644 --- a/pkg/local_object_storage/metabase/VERSION.md +++ b/pkg/local_object_storage/metabase/VERSION.md @@ -2,6 +2,8 @@ This file describes changes between the metabase versions. +Warning: database schema below is outdated and incomplete, see source code. + ## Current ### Primary buckets @@ -86,6 +88,11 @@ This file describes changes between the metabase versions. # History +## Version 3 + +- Payload hash, owner ID and FKBT buckets deleted +- Expiration epoch to object ID and object ID to expiration epoch added + ## Version 2 - Container ID is encoded as 32-byte slice diff --git a/pkg/local_object_storage/metabase/version.go b/pkg/local_object_storage/metabase/version.go index 5748b64ee..bb2b66d9b 100644 --- a/pkg/local_object_storage/metabase/version.go +++ b/pkg/local_object_storage/metabase/version.go @@ -9,7 +9,7 @@ import ( ) // version contains current metabase version. -const version = 2 +const version = 3 var versionKey = []byte("version") From dfe825b81b5eead451ae20b76a50b382618ae7ba Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 21 Aug 2024 17:08:44 +0300 Subject: [PATCH 1293/1943] [#1309] test: Clean up config after tests Signed-off-by: Aleksey Savchuk --- cmd/frostfs-cli/internal/key/key_test.go | 2 ++ cmd/frostfs-node/config/test/config.go | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/cmd/frostfs-cli/internal/key/key_test.go b/cmd/frostfs-cli/internal/key/key_test.go index e3127a3fe..37e4fd4ee 100644 --- a/cmd/frostfs-cli/internal/key/key_test.go +++ b/cmd/frostfs-cli/internal/key/key_test.go @@ -24,6 +24,8 @@ var testCmd = &cobra.Command{ } func Test_getOrGenerate(t *testing.T) { + t.Cleanup(viper.Reset) + dir := t.TempDir() wallPath := filepath.Join(dir, "wallet.json") diff --git a/cmd/frostfs-node/config/test/config.go b/cmd/frostfs-node/config/test/config.go index 28ec65291..a93d7e648 100644 --- a/cmd/frostfs-node/config/test/config.go +++ b/cmd/frostfs-node/config/test/config.go @@ -40,6 +40,15 @@ func ForEachFileType(pref string, f func(*config.Config)) { // ForEnvFileType creates config from `.env` file. func ForEnvFileType(t testing.TB, pref string, f func(*config.Config)) { + envs := os.Environ() + t.Cleanup(func() { + os.Clearenv() + for _, env := range envs { + keyValue := strings.Split(env, "=") + os.Setenv(keyValue[0], keyValue[1]) + } + }) + f(fromEnvFile(t, pref+".env")) } From bd24beecf8652ca3662b4339b2799bf0385507a8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 23 Aug 2024 12:28:09 +0300 Subject: [PATCH 1294/1943] [#1329] putSvc: Reset SuccessAfter for non-EC objects in EC container broadcasting Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/distributed.go | 9 +++++++++ pkg/services/object/put/single.go | 9 ++++++--- pkg/services/object/put/streamer.go | 7 +++++-- pkg/services/object_manager/placement/traverser.go | 7 +++++++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index c71427b67..5176f7a54 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -24,6 +24,8 @@ type distributedTarget struct { nodeTargetInitializer func(nodeDesc) preparedObjectTarget relay func(context.Context, nodeDesc) error + + resetSuccessAfterOnBroadcast bool } // parameters and state of container traversal. @@ -35,6 +37,8 @@ type traversal struct { // container nodes which was processed during the primary object placement mExclude map[string]*bool + + resetSuccessAfterOnBroadcast bool } // updates traversal parameters after the primary placement finish and @@ -44,6 +48,10 @@ func (x *traversal) submitPrimaryPlacementFinish() bool { // do not track success during container broadcast (best-effort) x.opts = append(x.opts, placement.WithoutSuccessTracking()) + if x.resetSuccessAfterOnBroadcast { + x.opts = append(x.opts, placement.ResetSuccessAfter()) + } + // avoid 2nd broadcast x.extraBroadcastEnabled = false @@ -118,5 +126,6 @@ func (t *distributedTarget) iteratePlacement(ctx context.Context) error { iter := t.cfg.newNodeIterator(append(t.placementOpts, placement.ForObject(id))) iter.extraBroadcastEnabled = needAdditionalBroadcast(t.obj, false /* Distributed target is for cluster-wide PUT */) + iter.resetSuccessAfterOnBroadcast = t.resetSuccessAfterOnBroadcast return iter.forEachNode(ctx, t.sendObject) } diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 9fa8ddb67..3cc8518f5 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -166,6 +166,7 @@ func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *o func (s *Service) saveToREPReplicas(ctx context.Context, placement putSinglePlacement, obj *objectSDK.Object, localOnly bool, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { iter := s.cfg.newNodeIterator(placement.placementOptions) iter.extraBroadcastEnabled = needAdditionalBroadcast(obj, localOnly) + iter.resetSuccessAfterOnBroadcast = placement.resetSuccessAfterOnBroadcast signer := &putSingleRequestSigner{ req: req, @@ -209,9 +210,10 @@ func (s *Service) saveToECReplicas(ctx context.Context, placement putSinglePlace } type putSinglePlacement struct { - placementOptions []placement.Option - isEC bool - container containerSDK.Container + placementOptions []placement.Option + isEC bool + container containerSDK.Container + resetSuccessAfterOnBroadcast bool } func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) (putSinglePlacement, error) { @@ -232,6 +234,7 @@ func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumb } if container.IsECContainer(cnrInfo.Value) && !object.IsECSupported(obj) && !localOnly { result.placementOptions = append(result.placementOptions, placement.SuccessAfter(uint32(policy.ECParityCount(cnrInfo.Value.PlacementPolicy())+1))) + result.resetSuccessAfterOnBroadcast = true } result.placementOptions = append(result.placementOptions, placement.ForContainer(cnrInfo.Value)) diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 969c8fa19..6b396ec96 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -246,16 +246,19 @@ func (p *Streamer) newDefaultObjectWriter(prm *PutInitPrm, forECPlacement bool) } } + var resetSuccessAfterOnBroadcast bool traverseOpts := prm.traverseOpts if forECPlacement && !prm.common.LocalOnly() { // save non-regular and linking object to EC container. // EC 2.1 -> REP 2, EC 2.2 -> REP 3 etc. traverseOpts = append(traverseOpts, placement.SuccessAfter(uint32(policy.ECParityCount(prm.cnr.PlacementPolicy())+1))) + resetSuccessAfterOnBroadcast = true } return &distributedTarget{ - cfg: p.cfg, - placementOpts: traverseOpts, + cfg: p.cfg, + placementOpts: traverseOpts, + resetSuccessAfterOnBroadcast: resetSuccessAfterOnBroadcast, nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { if node.local { return localTarget{ diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 306169571..9a5877c52 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -303,6 +303,13 @@ func SuccessAfter(v uint32) Option { } } +// ResetSuccessAfter resets flat success number setting option. +func ResetSuccessAfter() Option { + return func(c *cfg) { + c.flatSuccess = nil + } +} + // WithoutSuccessTracking disables success tracking in traversal. func WithoutSuccessTracking() Option { return func(c *cfg) { From a059a7dcf06ed41354f1defe5d1822b79d4c1deb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 23 Aug 2024 13:36:52 +0300 Subject: [PATCH 1295/1943] [#1329] cli: Skip linking objects in complex object processing Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/nodes.go | 74 ++++++++++++------------- cmd/frostfs-cli/modules/object/util.go | 9 +-- 2 files changed, 39 insertions(+), 44 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 42ae7324e..4efe04d16 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -172,7 +172,7 @@ func getComplexObjectParts(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli * func getCompexObjectMembers(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) []oid.ID { splitInfo := errSplitInfo.SplitInfo() - if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnrID, false); ok { + if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnrID); ok { return members } @@ -185,6 +185,7 @@ func getCompexObjectMembers(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, members []oid.ID, prmHead internalclient.HeadObjectPrm) []phyObject { result := make([]phyObject, 0, len(members)) + var hasNonEC, hasEC bool var resultGuard sync.Mutex if len(members) == 0 { @@ -193,31 +194,8 @@ func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, member prmHead.SetRawFlag(true) // to get an error instead of whole object - first := members[0] - var addrObj oid.Address - addrObj.SetContainer(cnrID) - addrObj.SetObject(first) - prmHead.SetAddress(addrObj) - - _, err := internalclient.HeadObject(cmd.Context(), prmHead) - var ecInfoError *objectSDK.ECInfoError - if errors.As(err, &ecInfoError) { - chunks := getECObjectChunks(cmd, cnrID, first, ecInfoError) - result = append(result, chunks...) - } else if err == nil { // not EC object, so all members must be phy objects - for _, member := range members { - result = append(result, phyObject{ - containerID: cnrID, - objectID: member, - }) - } - return result - } else { - commonCmd.ExitOnErr(cmd, "failed to flatten parts of complex object: %w", err) - } - eg, egCtx := errgroup.WithContext(cmd.Context()) - for idx := 1; idx < len(members); idx++ { + for idx := 0; idx < len(members); idx++ { partObjID := members[idx] eg.Go(func() error { @@ -227,24 +205,44 @@ func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, member partAddr.SetObject(partObjID) partHeadPrm.SetAddress(partAddr) - _, err := internalclient.HeadObject(egCtx, partHeadPrm) - var ecInfoError *objectSDK.ECInfoError - if errors.As(err, &ecInfoError) { - chunks := getECObjectChunks(cmd, cnrID, partObjID, ecInfoError) - - resultGuard.Lock() - defer resultGuard.Unlock() - result = append(result, chunks...) - - return nil - } else if err == nil { - return errMalformedComplexObject + obj, err := internalclient.HeadObject(egCtx, partHeadPrm) + if err != nil { + var ecInfoError *objectSDK.ECInfoError + if errors.As(err, &ecInfoError) { + resultGuard.Lock() + defer resultGuard.Unlock() + result = append(result, getECObjectChunks(cmd, cnrID, partObjID, ecInfoError)...) + hasEC = true + return nil + } + return err } - return err + + if obj.Header().Type() != objectSDK.TypeRegular { + commonCmd.ExitOnErr(cmd, "failed to flatten parts of complex object: %w", fmt.Errorf("object '%s' with type '%s' is not supported as part of complex object", partAddr, obj.Header().Type())) + } + + if len(obj.Header().Children()) > 0 { + // linking object is not data object, so skip it + return nil + } + + resultGuard.Lock() + defer resultGuard.Unlock() + result = append(result, phyObject{ + containerID: cnrID, + objectID: partObjID, + }) + hasNonEC = true + + return nil }) } commonCmd.ExitOnErr(cmd, "failed to flatten parts of complex object: %w", eg.Wait()) + if hasEC && hasNonEC { + commonCmd.ExitOnErr(cmd, "failed to flatten parts of complex object: %w", errMalformedComplexObject) + } return result } diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index 96b80fe1b..b090c9f8c 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -374,7 +374,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, common.PrintVerbose(cmd, "Split information received - object is virtual.") splitInfo := errSplit.SplitInfo() - if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnr, true); ok { + if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnr); ok { return members } @@ -390,7 +390,7 @@ func collectObjectRelatives(cmd *cobra.Command, cli *client.Client, cnr cid.ID, return nil } -func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, prmHead internal.HeadObjectPrm, cnr cid.ID, withLinking bool) ([]oid.ID, bool) { +func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *objectSDK.SplitInfo, prmHead internal.HeadObjectPrm, cnr cid.ID) ([]oid.ID, bool) { // collect split chain by the descending ease of operations (ease is evaluated heuristically). // If any approach fails, we don't try the next since we assume that it will fail too. @@ -411,10 +411,7 @@ func tryGetSplitMembersByLinkingObject(cmd *cobra.Command, splitInfo *objectSDK. common.PrintVerbose(cmd, "Received split members from the linking object: %v", children) - if withLinking { - return append(children, idLinking), true - } - return children, true + return append(children, idLinking), true } // linking object is not required for From 80099d9a2ff1685ecc08b6f71bcba777ca51ca1d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 23 Aug 2024 10:57:03 +0300 Subject: [PATCH 1296/1943] [#1328] pilorama: Add tricky test for SortedByFilename Signed-off-by: Evgenii Stratonikov --- .../pilorama/forest_test.go | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 001d095c8..c6c6e8c8b 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -1,11 +1,13 @@ package pilorama import ( + "bytes" "context" "crypto/rand" "fmt" mrand "math/rand" "path/filepath" + "slices" "strconv" "strings" "sync" @@ -232,6 +234,66 @@ func BenchmarkForestSortedIteration(b *testing.B) { } } +// The issue which we call "BugWithSkip" is easiest to understand when filenames are +// monotonically increasing numbers. We want the list of sorted filenames to have different length interleaved. +// The bug happens when we switch between length during listing. +// Thus this test contains numbers from 1 to 1000 and batch size of size 100. +func TestForest_TreeSortedIterationBugWithSkip(t *testing.T) { + t.Skip() + for i := range providers { + t.Run(providers[i].name, func(t *testing.T) { + testForestTreeSortedIterationBugWithSkip(t, providers[i].construct(t)) + }) + } +} + +func testForestTreeSortedIterationBugWithSkip(t *testing.T, s ForestStorage) { + defer func() { require.NoError(t, s.Close()) }() + + cid := cidtest.ID() + d := CIDDescriptor{cid, 0, 1} + treeID := "version" + treeAdd := func(t *testing.T, ts int, filename string) { + _, err := s.TreeMove(context.Background(), d, treeID, &Move{ + Child: RootID + uint64(ts), + Parent: RootID, + Meta: Meta{ + Time: Timestamp(ts), + Items: []KeyValue{ + {Key: AttributeFilename, Value: []byte(filename)}, + }, + }, + }) + require.NoError(t, err) + } + + const count = 2000 + treeAdd(t, 1, "") + for i := 1; i < count; i++ { + treeAdd(t, i+1, strconv.Itoa(i+1)) + } + + var result []MultiNodeInfo + treeAppend := func(t *testing.T, last *string, count int) *string { + res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, last, count) + require.NoError(t, err) + result = append(result, res...) + return cursor + } + + const batchSize = 10 + last := treeAppend(t, nil, batchSize) + for i := 1; i < count/batchSize; i++ { + last = treeAppend(t, last, batchSize) + } + require.Len(t, result, count) + require.True(t, slices.IsSortedFunc(result, func(a, b MultiNodeInfo) int { + filenameA := findAttr(a.Meta, AttributeFilename) + filenameB := findAttr(b.Meta, AttributeFilename) + return bytes.Compare(filenameA, filenameB) + })) +} + func TestForest_TreeSortedIteration(t *testing.T) { for i := range providers { t.Run(providers[i].name, func(t *testing.T) { From 806ea37101fc35cedca3e10681e43d7a91968d67 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 23 Aug 2024 10:58:33 +0300 Subject: [PATCH 1297/1943] [#1328] pilorama: Do not skip items in SortedByFilename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Benchmark results: ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ old │ new │ │ sec/op │ sec/op vs base │ ForestSortedIteration/bbolt,root-8 207.2µ ± 6% 173.6µ ± 6% -16.23% (p=0.000 n=10) ForestSortedIteration/bbolt,leaf-8 3.910µ ± 5% 3.928µ ± 7% ~ (p=0.529 n=10) geomean 28.46µ 26.11µ -8.27% ``` They are not representative, as the worst case is when we have multiple items of different lengths. However, `FileName` is usually less than 100 in practice, so the asymptotics is the same. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/boltdb.go | 1 + .../pilorama/forest_test.go | 3 +- pkg/local_object_storage/pilorama/heap.go | 38 +++++++++++++++---- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 48363ceac..29941be83 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1161,6 +1161,7 @@ func (t *boltForest) fillSortedChildren(b *bbolt.Bucket, nodeIDs MultiNode, h *f lastFilename = nil nodes = nil length = actualLength + 1 + count = 0 c.Seek(append(prefix, byte(length), byte(length>>8))) c.Prev() // c.Next() will be performed by for loop } diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index c6c6e8c8b..ecca9842f 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -237,9 +237,8 @@ func BenchmarkForestSortedIteration(b *testing.B) { // The issue which we call "BugWithSkip" is easiest to understand when filenames are // monotonically increasing numbers. We want the list of sorted filenames to have different length interleaved. // The bug happens when we switch between length during listing. -// Thus this test contains numbers from 1 to 1000 and batch size of size 100. +// Thus this test contains numbers from 1 to 2000 and batch size of size 10. func TestForest_TreeSortedIterationBugWithSkip(t *testing.T) { - t.Skip() for i := range providers { t.Run(providers[i].name, func(t *testing.T) { testForestTreeSortedIterationBugWithSkip(t, providers[i].construct(t)) diff --git a/pkg/local_object_storage/pilorama/heap.go b/pkg/local_object_storage/pilorama/heap.go index ec57b9e1f..5a00bcf7a 100644 --- a/pkg/local_object_storage/pilorama/heap.go +++ b/pkg/local_object_storage/pilorama/heap.go @@ -2,6 +2,8 @@ package pilorama import ( "container/heap" + "slices" + "strings" ) type heapInfo struct { @@ -28,9 +30,10 @@ func (h *filenameHeap) Pop() any { // fixedHeap maintains a fixed number of smallest elements started at some point. type fixedHeap struct { - start *string - count int - h *filenameHeap + start *string + sorted bool + count int + h *filenameHeap } func newHeap(start *string, count int) *fixedHeap { @@ -44,20 +47,39 @@ func newHeap(start *string, count int) *fixedHeap { } } +const amortizationMultiplier = 5 + func (h *fixedHeap) push(id MultiNode, filename string) bool { if h.start != nil && filename <= *h.start { return false } - heap.Push(h.h, heapInfo{id: id, filename: filename}) - if h.h.Len() > h.count { - heap.Remove(h.h, h.h.Len()-1) + + *h.h = append(*h.h, heapInfo{id: id, filename: filename}) + h.sorted = false + + if h.h.Len() > h.count*amortizationMultiplier { + slices.SortFunc(*h.h, func(a, b heapInfo) int { + return strings.Compare(a.filename, b.filename) + }) + *h.h = (*h.h)[:h.count] } return true } func (h *fixedHeap) pop() (heapInfo, bool) { - if h.h.Len() != 0 { - return heap.Pop(h.h).(heapInfo), true + if !h.sorted { + slices.SortFunc(*h.h, func(a, b heapInfo) int { + return strings.Compare(a.filename, b.filename) + }) + if len(*h.h) > h.count { + *h.h = (*h.h)[:h.count] + } + h.sorted = true + } + if len(*h.h) != 0 { + info := (*h.h)[0] + *h.h = (*h.h)[1:] + return info, true } return heapInfo{}, false } From fa7f9fbce240dde6e0a3c6ded13cb769bbe3b2bf Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 26 Aug 2024 15:36:39 +0300 Subject: [PATCH 1298/1943] [#1333] go.mod: Update api-go Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b665709cc..b0df97511 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240819074700-a43110e36326 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240826113619-9e82a5a31a88 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d diff --git a/go.sum b/go.sum index 54e0d0301..f3a17363d 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240819074700-a43110e36326 h1:TkH+NSsY4C/Z8MocIJyMcqLm5vEhZcSowOldJyilKKA= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240819074700-a43110e36326/go.mod h1:zZnHiRv9m5+ESYLhBXY9Jds9A/YIDEUGiuyPUS09HwM= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240826113619-9e82a5a31a88 h1:ckD87Z4pvPtu2hjpRcqPHlAtgOHPZfSW3x+zzwZztiY= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240826113619-9e82a5a31a88/go.mod h1:BDnEpkKMykCS8u1nLzR6SgNzCv6885RWlo5TnravQuI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= From 5e9a97fd3eabc58347260b06fc6e1da4f5cf2185 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 27 Aug 2024 14:46:56 +0300 Subject: [PATCH 1299/1943] [#1336] go.mod: Update api-go and sdk-go Signed-off-by: Evgenii Stratonikov --- go.mod | 4 +- go.sum | 8 +- pkg/services/control/ir/service_frostfs.pb.go | 32 +-- pkg/services/control/service_frostfs.pb.go | 196 +++++++++--------- pkg/services/control/types_frostfs.pb.go | 6 +- pkg/services/tree/service_frostfs.pb.go | 98 ++++----- 6 files changed, 172 insertions(+), 172 deletions(-) diff --git a/go.mod b/go.mod index b0df97511..b7f59c823 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240826113619-9e82a5a31a88 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240827104600-eba18f6e67ac git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240820072028-6dd7be11d13b + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240827083309-f0b9493ce3f7 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index f3a17363d..d2f926151 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240826113619-9e82a5a31a88 h1:ckD87Z4pvPtu2hjpRcqPHlAtgOHPZfSW3x+zzwZztiY= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240826113619-9e82a5a31a88/go.mod h1:BDnEpkKMykCS8u1nLzR6SgNzCv6885RWlo5TnravQuI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240827104600-eba18f6e67ac h1:Gu3oiPnsSZPgwsUYs2f3xTQwndM/OWM/zo3zbN4rOb8= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240827104600-eba18f6e67ac/go.mod h1:BDnEpkKMykCS8u1nLzR6SgNzCv6885RWlo5TnravQuI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240820072028-6dd7be11d13b h1:ZCJBVmQDcdv0twpX9xJU/AQwX+dXyvVfqr0Pq3x+3yk= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240820072028-6dd7be11d13b/go.mod h1:aaC2OR34tVrBwd0Z2gqoN5WLtV/idKqpqPDhb4XqmCo= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240827083309-f0b9493ce3f7 h1:9eZidZMT4tHOdc6GZRPlZR12IToKqHhUd5wzevdDUqo= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240827083309-f0b9493ce3f7/go.mod h1:VzVYcwo/eXjkdo5ktPdZeAE4fsnZX6zEun3g+5E2weo= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index 786095802..66d196617 100644 --- a/pkg/services/control/ir/service_frostfs.pb.go +++ b/pkg/services/control/ir/service_frostfs.pb.go @@ -164,10 +164,10 @@ func (x *HealthCheckRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -495,10 +495,10 @@ func (x *HealthCheckResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -810,10 +810,10 @@ func (x *TickEpochRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -1125,10 +1125,10 @@ func (x *TickEpochResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -1471,10 +1471,10 @@ func (x *RemoveNodeRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -1786,10 +1786,10 @@ func (x *RemoveNodeResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -2163,10 +2163,10 @@ func (x *RemoveContainerRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -2478,10 +2478,10 @@ func (x *RemoveContainerResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index afd1c3c41..a446c5e59 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -164,10 +164,10 @@ func (x *HealthCheckRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -542,10 +542,10 @@ func (x *HealthCheckResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -904,10 +904,10 @@ func (x *SetNetmapStatusRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -1185,10 +1185,10 @@ func (x *SetNetmapStatusResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -1466,10 +1466,10 @@ func (x *GetNetmapStatusRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -1828,10 +1828,10 @@ func (x *GetNetmapStatusResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -2157,10 +2157,10 @@ func (x *DropObjectsRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -2438,10 +2438,10 @@ func (x *DropObjectsResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -2719,10 +2719,10 @@ func (x *ListShardsRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -2888,7 +2888,7 @@ func (x *ListShardsResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { return } for i := range x.Shards { - if x.Shards[i] != nil && x.Shards[i].StableSize() != 0 { + if x.Shards[i] != nil { x.Shards[i].EmitProtobuf(mm.AppendMessage(1)) } } @@ -3057,10 +3057,10 @@ func (x *ListShardsResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -3464,10 +3464,10 @@ func (x *SetShardModeRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -3745,10 +3745,10 @@ func (x *SetShardModeResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -4122,10 +4122,10 @@ func (x *SynchronizeTreeRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -4403,10 +4403,10 @@ func (x *SynchronizeTreeResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -4763,10 +4763,10 @@ func (x *EvacuateShardRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -5078,10 +5078,10 @@ func (x *EvacuateShardResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -5438,10 +5438,10 @@ func (x *FlushCacheRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -5719,10 +5719,10 @@ func (x *FlushCacheResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -6065,10 +6065,10 @@ func (x *DoctorRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -6346,10 +6346,10 @@ func (x *DoctorResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -6772,10 +6772,10 @@ func (x *StartShardEvacuationRequest) EmitProtobuf(mm *easyproto.MessageMarshale if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -7053,10 +7053,10 @@ func (x *StartShardEvacuationResponse) EmitProtobuf(mm *easyproto.MessageMarshal if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -7334,10 +7334,10 @@ func (x *GetShardEvacuationStatusRequest) EmitProtobuf(mm *easyproto.MessageMars if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -7830,10 +7830,10 @@ func (x *GetShardEvacuationStatusResponse_Body) EmitProtobuf(mm *easyproto.Messa if int32(x.Status) != 0 { mm.AppendInt32(5, int32(x.Status)) } - if x.Duration != nil && x.Duration.StableSize() != 0 { + if x.Duration != nil { x.Duration.EmitProtobuf(mm.AppendMessage(6)) } - if x.StartedAt != nil && x.StartedAt.StableSize() != 0 { + if x.StartedAt != nil { x.StartedAt.EmitProtobuf(mm.AppendMessage(7)) } if len(x.ErrorMessage) != 0 { @@ -8321,10 +8321,10 @@ func (x *GetShardEvacuationStatusResponse) EmitProtobuf(mm *easyproto.MessageMar if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -8602,10 +8602,10 @@ func (x *ResetShardEvacuationStatusRequest) EmitProtobuf(mm *easyproto.MessageMa if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -8883,10 +8883,10 @@ func (x *ResetShardEvacuationStatusResponse) EmitProtobuf(mm *easyproto.MessageM if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -9164,10 +9164,10 @@ func (x *StopShardEvacuationRequest) EmitProtobuf(mm *easyproto.MessageMarshaler if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -9445,10 +9445,10 @@ func (x *StopShardEvacuationResponse) EmitProtobuf(mm *easyproto.MessageMarshale if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -9613,7 +9613,7 @@ func (x *AddChainLocalOverrideRequest_Body) EmitProtobuf(mm *easyproto.MessageMa if x == nil { return } - if x.Target != nil && x.Target.StableSize() != 0 { + if x.Target != nil { x.Target.EmitProtobuf(mm.AppendMessage(1)) } if len(x.Chain) != 0 { @@ -9795,10 +9795,10 @@ func (x *AddChainLocalOverrideRequest) EmitProtobuf(mm *easyproto.MessageMarshal if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -10110,10 +10110,10 @@ func (x *AddChainLocalOverrideResponse) EmitProtobuf(mm *easyproto.MessageMarsha if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -10278,7 +10278,7 @@ func (x *GetChainLocalOverrideRequest_Body) EmitProtobuf(mm *easyproto.MessageMa if x == nil { return } - if x.Target != nil && x.Target.StableSize() != 0 { + if x.Target != nil { x.Target.EmitProtobuf(mm.AppendMessage(1)) } if len(x.ChainId) != 0 { @@ -10460,10 +10460,10 @@ func (x *GetChainLocalOverrideRequest) EmitProtobuf(mm *easyproto.MessageMarshal if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -10775,10 +10775,10 @@ func (x *GetChainLocalOverrideResponse) EmitProtobuf(mm *easyproto.MessageMarsha if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -10941,7 +10941,7 @@ func (x *ListChainLocalOverridesRequest_Body) EmitProtobuf(mm *easyproto.Message if x == nil { return } - if x.Target != nil && x.Target.StableSize() != 0 { + if x.Target != nil { x.Target.EmitProtobuf(mm.AppendMessage(1)) } } @@ -11094,10 +11094,10 @@ func (x *ListChainLocalOverridesRequest) EmitProtobuf(mm *easyproto.MessageMarsh if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -11423,10 +11423,10 @@ func (x *ListChainLocalOverridesResponse) EmitProtobuf(mm *easyproto.MessageMars if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -11738,10 +11738,10 @@ func (x *ListTargetsLocalOverridesRequest) EmitProtobuf(mm *easyproto.MessageMar if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -11907,7 +11907,7 @@ func (x *ListTargetsLocalOverridesResponse_Body) EmitProtobuf(mm *easyproto.Mess return } for i := range x.Targets { - if x.Targets[i] != nil && x.Targets[i].StableSize() != 0 { + if x.Targets[i] != nil { x.Targets[i].EmitProtobuf(mm.AppendMessage(1)) } } @@ -12076,10 +12076,10 @@ func (x *ListTargetsLocalOverridesResponse) EmitProtobuf(mm *easyproto.MessageMa if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -12244,7 +12244,7 @@ func (x *RemoveChainLocalOverrideRequest_Body) EmitProtobuf(mm *easyproto.Messag if x == nil { return } - if x.Target != nil && x.Target.StableSize() != 0 { + if x.Target != nil { x.Target.EmitProtobuf(mm.AppendMessage(1)) } if len(x.ChainId) != 0 { @@ -12426,10 +12426,10 @@ func (x *RemoveChainLocalOverrideRequest) EmitProtobuf(mm *easyproto.MessageMars if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -12707,10 +12707,10 @@ func (x *RemoveChainLocalOverrideResponse) EmitProtobuf(mm *easyproto.MessageMar if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -12873,7 +12873,7 @@ func (x *RemoveChainLocalOverridesByTargetRequest_Body) EmitProtobuf(mm *easypro if x == nil { return } - if x.Target != nil && x.Target.StableSize() != 0 { + if x.Target != nil { x.Target.EmitProtobuf(mm.AppendMessage(1)) } } @@ -13026,10 +13026,10 @@ func (x *RemoveChainLocalOverridesByTargetRequest) EmitProtobuf(mm *easyproto.Me if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -13307,10 +13307,10 @@ func (x *RemoveChainLocalOverridesByTargetResponse) EmitProtobuf(mm *easyproto.M if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -13760,10 +13760,10 @@ func (x *SealWriteCacheRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -14120,7 +14120,7 @@ func (x *SealWriteCacheResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshale return } for i := range x.Results { - if x.Results[i] != nil && x.Results[i].StableSize() != 0 { + if x.Results[i] != nil { x.Results[i].EmitProtobuf(mm.AppendMessage(1)) } } @@ -14289,10 +14289,10 @@ func (x *SealWriteCacheResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -14618,10 +14618,10 @@ func (x *DetachShardsRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -14899,10 +14899,10 @@ func (x *DetachShardsResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } diff --git a/pkg/services/control/types_frostfs.pb.go b/pkg/services/control/types_frostfs.pb.go index 9aff26a98..3cc37245f 100644 --- a/pkg/services/control/types_frostfs.pb.go +++ b/pkg/services/control/types_frostfs.pb.go @@ -549,7 +549,7 @@ func (x *NodeInfo) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendString(2, x.Addresses[j]) } for i := range x.Attributes { - if x.Attributes[i] != nil && x.Attributes[i].StableSize() != 0 { + if x.Attributes[i] != nil { x.Attributes[i].EmitProtobuf(mm.AppendMessage(3)) } } @@ -817,7 +817,7 @@ func (x *Netmap) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendUint64(1, x.Epoch) } for i := range x.Nodes { - if x.Nodes[i] != nil && x.Nodes[i].StableSize() != 0 { + if x.Nodes[i] != nil { x.Nodes[i].EmitProtobuf(mm.AppendMessage(2)) } } @@ -1013,7 +1013,7 @@ func (x *ShardInfo) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendString(2, x.MetabasePath) } for i := range x.Blobstor { - if x.Blobstor[i] != nil && x.Blobstor[i].StableSize() != 0 { + if x.Blobstor[i] != nil { x.Blobstor[i].EmitProtobuf(mm.AppendMessage(3)) } } diff --git a/pkg/services/tree/service_frostfs.pb.go b/pkg/services/tree/service_frostfs.pb.go index c4d44253d..3c6ba21b7 100644 --- a/pkg/services/tree/service_frostfs.pb.go +++ b/pkg/services/tree/service_frostfs.pb.go @@ -70,7 +70,7 @@ func (x *AddRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendUint64(3, x.ParentId) } for i := range x.Meta { - if x.Meta[i] != nil && x.Meta[i].StableSize() != 0 { + if x.Meta[i] != nil { x.Meta[i].EmitProtobuf(mm.AppendMessage(4)) } } @@ -346,10 +346,10 @@ func (x *AddRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -661,10 +661,10 @@ func (x *AddResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -852,7 +852,7 @@ func (x *AddByPathRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendString(4, x.Path[j]) } for i := range x.Meta { - if x.Meta[i] != nil && x.Meta[i].StableSize() != 0 { + if x.Meta[i] != nil { x.Meta[i].EmitProtobuf(mm.AppendMessage(5)) } } @@ -1168,10 +1168,10 @@ func (x *AddByPathRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -1530,10 +1530,10 @@ func (x *AddByPathResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -1938,10 +1938,10 @@ func (x *RemoveRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -2219,10 +2219,10 @@ func (x *RemoveResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -2410,7 +2410,7 @@ func (x *MoveRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendUint64(4, x.NodeId) } for i := range x.Meta { - if x.Meta[i] != nil && x.Meta[i].StableSize() != 0 { + if x.Meta[i] != nil { x.Meta[i].EmitProtobuf(mm.AppendMessage(5)) } } @@ -2712,10 +2712,10 @@ func (x *MoveRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -2993,10 +2993,10 @@ func (x *MoveResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -3553,10 +3553,10 @@ func (x *GetNodeByPathRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -3734,7 +3734,7 @@ func (x *GetNodeByPathResponse_Info) EmitProtobuf(mm *easyproto.MessageMarshaler mm.AppendUint64(2, x.Timestamp) } for i := range x.Meta { - if x.Meta[i] != nil && x.Meta[i].StableSize() != 0 { + if x.Meta[i] != nil { x.Meta[i].EmitProtobuf(mm.AppendMessage(3)) } } @@ -3967,7 +3967,7 @@ func (x *GetNodeByPathResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler return } for i := range x.Nodes { - if x.Nodes[i] != nil && x.Nodes[i].StableSize() != 0 { + if x.Nodes[i] != nil { x.Nodes[i].EmitProtobuf(mm.AppendMessage(1)) } } @@ -4136,10 +4136,10 @@ func (x *GetNodeByPathResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -4506,7 +4506,7 @@ func (x *GetSubTreeRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { if len(x.BearerToken) != 0 { mm.AppendBytes(5, x.BearerToken) } - if x.OrderBy != nil && x.OrderBy.StableSize() != 0 { + if x.OrderBy != nil { x.OrderBy.EmitProtobuf(mm.AppendMessage(6)) } } @@ -4803,10 +4803,10 @@ func (x *GetSubTreeRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -4993,7 +4993,7 @@ func (x *GetSubTreeResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendUint64(3, x.Timestamp[j]) } for i := range x.Meta { - if x.Meta[i] != nil && x.Meta[i].StableSize() != 0 { + if x.Meta[i] != nil { x.Meta[i].EmitProtobuf(mm.AppendMessage(4)) } } @@ -5282,10 +5282,10 @@ func (x *GetSubTreeResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -5597,10 +5597,10 @@ func (x *TreeListRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -5926,10 +5926,10 @@ func (x *TreeListResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -6102,7 +6102,7 @@ func (x *ApplyRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { if len(x.TreeId) != 0 { mm.AppendString(2, x.TreeId) } - if x.Operation != nil && x.Operation.StableSize() != 0 { + if x.Operation != nil { x.Operation.EmitProtobuf(mm.AppendMessage(3)) } } @@ -6307,10 +6307,10 @@ func (x *ApplyRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -6588,10 +6588,10 @@ func (x *ApplyResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -6996,10 +6996,10 @@ func (x *GetOpLogRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -7162,7 +7162,7 @@ func (x *GetOpLogResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Operation != nil && x.Operation.StableSize() != 0 { + if x.Operation != nil { x.Operation.EmitProtobuf(mm.AppendMessage(1)) } } @@ -7315,10 +7315,10 @@ func (x *GetOpLogResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -7596,10 +7596,10 @@ func (x *HealthcheckResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } @@ -7877,10 +7877,10 @@ func (x *HealthcheckRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { if x == nil { return } - if x.Body != nil && x.Body.StableSize() != 0 { + if x.Body != nil { x.Body.EmitProtobuf(mm.AppendMessage(1)) } - if x.Signature != nil && x.Signature.StableSize() != 0 { + if x.Signature != nil { x.Signature.EmitProtobuf(mm.AppendMessage(2)) } } From d6b42972a81f18d4351d81b154f75815f39546ea Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 27 Aug 2024 16:09:13 +0300 Subject: [PATCH 1300/1943] [#1338] object: Fix audit patch stream Signed-off-by: Airat Arifullin --- pkg/services/object/audit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/audit.go b/pkg/services/object/audit.go index b924386d1..39e1f9f2d 100644 --- a/pkg/services/object/audit.go +++ b/pkg/services/object/audit.go @@ -204,7 +204,7 @@ func (a *auditPatchStream) CloseAndRecv(ctx context.Context) (*object.PatchRespo if err != nil { a.failed = true } - a.objectID = resp.GetBody().ObjectID + a.objectID = resp.GetBody().GetObjectID() audit.LogRequestWithKey(a.log, objectGRPC.ObjectService_Patch_FullMethodName, a.key, audit.TargetFromContainerIDObjectID(a.containerID, a.objectID), !a.failed) From 6488ddee882ddb2facd8554aa274986076de0bbc Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 27 Aug 2024 16:09:59 +0300 Subject: [PATCH 1301/1943] [#1338] object: Fix range provider in `Patch` handler Signed-off-by: Airat Arifullin --- pkg/services/object/patch/range_provider.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/services/object/patch/range_provider.go b/pkg/services/object/patch/range_provider.go index 755c5bf60..cb3f7c342 100644 --- a/pkg/services/object/patch/range_provider.go +++ b/pkg/services/object/patch/range_provider.go @@ -30,6 +30,12 @@ type rangeProvider struct { var _ patcherSDK.RangeProvider = (*rangeProvider)(nil) func (r *rangeProvider) GetRange(ctx context.Context, rng *objectSDK.Range) io.Reader { + // Remote GetRange request to a container node uses an SDK-client that fails range validation + // with zero-length. However, from the patcher's point of view, such request is still valid. + if rng.GetLength() == 0 { + return &nopReader{} + } + pipeReader, pipeWriter := io.Pipe() var rngPrm getsvc.RangePrm @@ -61,3 +67,9 @@ func (r *rangeProvider) GetRange(ctx context.Context, rng *objectSDK.Range) io.R return pipeReader } + +type nopReader struct{} + +func (nopReader) Read(_ []byte) (int, error) { + return 0, io.EOF +} From 7abbdca0641f1526c347da60012f437be57254e8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 28 Aug 2024 13:56:45 +0300 Subject: [PATCH 1302/1943] [#1340] getSvc: Fix access denied error handling Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/get.go | 8 +++++++ pkg/services/object/get/remote.go | 7 +++++- pkg/services/object/get/v2/get_forwarder.go | 26 ++++++++++++--------- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 07a2f3a72..03b7f8bf2 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -2,9 +2,11 @@ package getsvc import ( "context" + "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" ) @@ -120,6 +122,12 @@ func (exec *request) analyzeStatus(ctx context.Context, execCnr bool) { exec.log.Debug(logs.OperationFinishedWithError, zap.Error(exec.err), ) + var errAccessDenied *apistatus.ObjectAccessDenied + if execCnr && errors.As(exec.err, &errAccessDenied) { + // Local get can't return access denied error, so this error was returned by + // write to the output stream. So there is no need to try to find object on other nodes. + return + } if execCnr { exec.executeOnContainer(ctx) diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index ce9abfe1c..163767c43 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -31,6 +31,7 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { var errECInfo *objectSDK.ECInfoError var errRemoved *apistatus.ObjectAlreadyRemoved var errOutOfRange *apistatus.ObjectOutOfRange + var errAccessDenied *apistatus.ObjectAccessDenied switch { default: @@ -38,7 +39,11 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { if r.status != statusEC { // for raw requests, continue to collect other parts r.status = statusUndefined - r.err = new(apistatus.ObjectNotFound) + if errors.As(err, &errAccessDenied) { + r.err = err + } else { + r.err = new(apistatus.ObjectNotFound) + } } return false case err == nil: diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index 774f98643..18194c740 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -23,12 +23,14 @@ import ( ) type getRequestForwarder struct { - OnceResign sync.Once - OnceHeaderSending sync.Once - GlobalProgress int - Key *ecdsa.PrivateKey - Request *objectV2.GetRequest - Stream *streamObjectWriter + OnceResign sync.Once + GlobalProgress int + Key *ecdsa.PrivateKey + Request *objectV2.GetRequest + Stream *streamObjectWriter + + headerSent bool + headerSentGuard sync.Mutex } func (f *getRequestForwarder) forwardRequestToNode(ctx context.Context, addr network.Address, c client.MultiAddressClient, pubkey []byte) (*objectSDK.Object, error) { @@ -83,13 +85,15 @@ func (f *getRequestForwarder) writeHeader(ctx context.Context, v *objectV2.GetOb obj.SetSignature(v.GetSignature()) obj.SetHeader(v.GetHeader()) - var err error - f.OnceHeaderSending.Do(func() { - err = f.Stream.WriteHeader(ctx, objectSDK.NewFromV2(obj)) - }) - if err != nil { + f.headerSentGuard.Lock() + defer f.headerSentGuard.Unlock() + if f.headerSent { + return nil + } + if err := f.Stream.WriteHeader(ctx, objectSDK.NewFromV2(obj)); err != nil { return errCouldNotWriteObjHeader(err) } + f.headerSent = true return nil } From 01b6f1733cbc9d0ce133b5970a28817423686e2c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 28 Aug 2024 14:29:07 +0300 Subject: [PATCH 1303/1943] [#1341] Makefile: Build linter with -trimpath Fix error with go1.23: ``` Error: build linters: unable to load custom analyzer "truecloudlab-linters": ../linters/bin/external_linters.so, plugin.Open("/repo/frostfs/linters/bin/external_linters"): plugin was built with a different version of package cmp Failed executing command with error: build linters: unable to load custom analyzer "truecloudlab-linters": ../linters/bin/external_linters.so, plugin.Open("/repo/frostfs/linters/bin/external_linters"): plugin was built with a different version of package cmp ``` Signed-off-by: Evgenii Stratonikov --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 94a8a14c3..3a6dea7c3 100755 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.22 LINT_VERSION ?= 1.60.1 -TRUECLOUDLAB_LINT_VERSION ?= 0.0.6 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.7 PROTOC_VERSION ?= 25.0 PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2) PROTOC_OS_VERSION=osx-x86_64 @@ -197,7 +197,7 @@ lint-install: @@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR) @rm -rf $(TMP_DIR)/linters @rmdir $(TMP_DIR) 2>/dev/null || true - @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) + @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install -trimpath github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) # Run linters lint: From 7e97df4878b9bf95f88712394bb21201eb072012 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 28 Aug 2024 14:31:35 +0300 Subject: [PATCH 1304/1943] [#1341] Makefile: Update golangci-lint Signed-off-by: Evgenii Stratonikov --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3a6dea7c3..71492ef17 100755 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ HUB_IMAGE ?= truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.22 -LINT_VERSION ?= 1.60.1 +LINT_VERSION ?= 1.60.3 TRUECLOUDLAB_LINT_VERSION ?= 0.0.7 PROTOC_VERSION ?= 25.0 PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2) From 03976c6ed5df58b83f788a18416e27f9144342b1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 28 Aug 2024 14:45:57 +0300 Subject: [PATCH 1305/1943] [#1341] .golangci.yml: Replace exportloopref with copyloopvar exportloopref is deprecated. gopatch: ``` @@ var index, value identifier var slice expression @@ for index, value := range slice { ... -value := value ... } @@ var index, value identifier var slice expression @@ for index, value := range slice { ... -index := index ... } @@ var value identifier var channel expression @@ for value := range channel { ... -value := value ... } ``` Signed-off-by: Evgenii Stratonikov --- .golangci.yml | 2 +- cmd/frostfs-adm/internal/modules/morph/generate/generate.go | 2 -- cmd/frostfs-cli/modules/container/list.go | 1 - cmd/frostfs-cli/modules/object/nodes.go | 3 --- .../blobstor/blobovniczatree/rebuild.go | 3 --- pkg/local_object_storage/blobstor/info.go | 1 - pkg/local_object_storage/engine/control.go | 2 -- pkg/local_object_storage/engine/shards.go | 1 - pkg/local_object_storage/engine/writecache.go | 1 - pkg/local_object_storage/pilorama/forest_test.go | 1 - pkg/local_object_storage/writecache/flush_test.go | 1 - pkg/services/object/get/assemblerec.go | 1 - pkg/services/object/put/common.go | 1 - pkg/services/object/put/ec.go | 1 - pkg/services/policer/ec.go | 2 -- pkg/services/tree/sync.go | 6 +----- pkg/services/tree/sync_test.go | 2 -- 17 files changed, 2 insertions(+), 29 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 2e9e78fc3..971f0d0e7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -67,7 +67,7 @@ linters: - bidichk - durationcheck - exhaustive - - exportloopref + - copyloopvar - gofmt - goimports - misspell diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go index c7de599e5..7af776797 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go @@ -73,7 +73,6 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er return nil, fmt.Errorf("can't fetch password: %w", err) } - i := i errG.Go(func() error { p := filepath.Join(walletDir, innerring.GlagoliticLetter(i).String()+".json") f, err := os.OpenFile(p, os.O_CREATE, 0o644) @@ -107,7 +106,6 @@ func initializeWallets(v *viper.Viper, walletDir string, size int) ([]string, er // Create consensus account with 2*N/3+1 multi-signature. bftCount := smartcontract.GetDefaultHonestNodeCount(size) for i := range wallets { - i := i ps := pubs.Copy() errG.Go(func() error { if err := addMultisigAccount(wallets[i], majCount, constants.CommitteeAccountName, passwords[i], ps); err != nil { diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index a1410d7a0..6d0019ec4 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -70,7 +70,6 @@ var listContainersCmd = &cobra.Command{ continue } - cnrID := cnrID prmGet.ClientParams.ContainerID = &cnrID res, err := internalclient.GetContainer(cmd.Context(), prmGet) if err != nil { diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 4efe04d16..896f6f17f 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -393,8 +393,6 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa. eg, egCtx := errgroup.WithContext(cmd.Context()) for _, cand := range candidates { - cand := cand - eg.Go(func() error { cli, err := createClient(egCtx, cmd, cand, pk) if err != nil { @@ -405,7 +403,6 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa. } for _, object := range objects { - object := object eg.Go(func() error { stored, err := isObjectStoredOnNode(egCtx, cmd, object.containerID, object.objectID, cli, pk) resultMtx.Lock() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index 93ef8ba2e..cfc17cfae 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -160,9 +160,6 @@ func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovn eg, egCtx := errgroup.WithContext(ctx) for addr, data := range batch { - addr := addr - data := data - if err := limiter.AcquireWorkSlot(egCtx); err != nil { _ = eg.Wait() return result.Load(), err diff --git a/pkg/local_object_storage/blobstor/info.go b/pkg/local_object_storage/blobstor/info.go index 8a5bb870a..c1c47f3bb 100644 --- a/pkg/local_object_storage/blobstor/info.go +++ b/pkg/local_object_storage/blobstor/info.go @@ -43,7 +43,6 @@ func (b *BlobStor) ObjectsCount(ctx context.Context) (uint64, error) { eg, egCtx := errgroup.WithContext(ctx) for i := range b.storage { - i := i eg.Go(func() error { v, e := b.storage[i].Storage.ObjectsCount(egCtx) if e != nil { diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 03196400a..4778cf539 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -82,8 +82,6 @@ func (e *StorageEngine) Init(ctx context.Context) error { } for id, sh := range e.shards { - id := id - sh := sh eg.Go(func() error { if err := sh.Init(ctx); err != nil { errCh <- shardInitError{ diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 56d4fcd4a..980b38a63 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -367,7 +367,6 @@ func (e *StorageEngine) closeShards(deletedShards []hashedShard) error { var multiErrGuard sync.Mutex var eg errgroup.Group for _, sh := range deletedShards { - sh := sh eg.Go(func() error { err := sh.SetMode(mode.Disabled) if err != nil { diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 3e8f387ef..7710bc7f4 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -102,7 +102,6 @@ func (e *StorageEngine) SealWriteCache(ctx context.Context, prm SealWriteCachePr eg, egCtx := errgroup.WithContext(ctx) for _, shardID := range prm.ShardIDs { - shardID := shardID eg.Go(func() error { e.mtx.RLock() sh, ok := e.shards[shardID.String()] diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index ecca9842f..41d7a567c 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -1459,7 +1459,6 @@ func testTreeLastSyncHeight(t *testing.T, f ForestStorage) { func TestForest_ListTrees(t *testing.T) { for i := range providers { - i := i t.Run(providers[i].name, func(t *testing.T) { testTreeListTrees(t, providers[i].construct) }) diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 3c951bebe..a637da45d 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -170,7 +170,6 @@ func runFlushTest[Option any]( t.Run("ignore errors", func(t *testing.T) { for _, f := range failures { - f := f t.Run(f.Desc, func(t *testing.T) { errCountOpt, errCount := errCountOption() wc, bs, mb := newCache(t, createCacheFn, smallSize, errCountOpt) diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index 6a02673c3..dde0d7dad 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -170,7 +170,6 @@ func (a *assemblerec) processECNodesRequests(ctx context.Context, nodes []placem eg.SetLimit(dataCount) for _, ch := range a.ecInfo.localChunks { - ch := ch eg.Go(func() error { select { case <-ctx.Done(): diff --git a/pkg/services/object/put/common.go b/pkg/services/object/put/common.go index 6696a192b..cbb7f5f33 100644 --- a/pkg/services/object/put/common.go +++ b/pkg/services/object/put/common.go @@ -71,7 +71,6 @@ func (n *nodeIterator) forEachAddress(ctx context.Context, traverser *placement. var wg sync.WaitGroup for _, addr := range addrs { - addr := addr if ok := n.mExclude[string(addr.PublicKey())]; ok != nil { if *ok { traverser.SubmitSuccess() diff --git a/pkg/services/object/put/ec.go b/pkg/services/object/put/ec.go index fbb51912c..1fadf65fe 100644 --- a/pkg/services/object/put/ec.go +++ b/pkg/services/object/put/ec.go @@ -216,7 +216,6 @@ func (e *ecWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) er } for idx := range parts { - idx := idx eg.Go(func() error { return e.writePart(egCtx, parts[idx], idx, nodes, visited) }) diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index 0a118797d..61a65fc21 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -357,8 +357,6 @@ func (p *Policer) collectExistedChunks(ctx context.Context, objInfo objectcore.I parts := make([]*objectSDK.Object, objInfo.ECInfo.Total) errGroup, egCtx := errgroup.WithContext(ctx) for idx, nodes := range existedChunks { - idx := idx - nodes := nodes errGroup.Go(func() error { var objID oid.Address objID.SetContainer(parentAddress.Container()) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 0f85f50b1..be22074a5 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -190,8 +190,6 @@ func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID s var prev *pilorama.Move for m := range operationStream { - m := m - // skip already applied op if prev != nil && prev.Time == m.Time { continue @@ -287,8 +285,6 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, allNodesSynced.Store(true) for i, n := range nodes { - i := i - n := n errGroup.Go(func() error { var nodeSynced bool n.IterateNetworkEndpoints(func(addr string) bool { @@ -421,7 +417,7 @@ func (s *Service) syncContainers(ctx context.Context, cnrs []cid.ID) { var wg sync.WaitGroup for _, cnr := range cnrs { wg.Add(1) - cnr := cnr + err := s.syncPool.Submit(func() { defer wg.Done() s.log.Debug(logs.TreeSyncingContainerTrees, zap.Stringer("cid", cnr)) diff --git a/pkg/services/tree/sync_test.go b/pkg/services/tree/sync_test.go index 190b4ccbb..497d90554 100644 --- a/pkg/services/tree/sync_test.go +++ b/pkg/services/tree/sync_test.go @@ -51,8 +51,6 @@ func Test_mergeOperationStreams(t *testing.T) { // generate and put values to all chans for i, ch := range nodeOpChans { - i := i - ch := ch go func() { for _, tm := range tt.opTimes[i] { op := &pilorama.Move{} From 6c2146bbc13e118d1518010d144ebc4dfadd5d6d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 28 Aug 2024 18:32:30 +0300 Subject: [PATCH 1306/1943] [#1334] metabase: Add upgrade from v2 to v3 Signed-off-by: Dmitrii Stepanov --- .../internal/modules/metabase/root.go | 15 + .../internal/modules/metabase/upgrade.go | 86 +++++ cmd/frostfs-adm/internal/modules/root.go | 2 + pkg/local_object_storage/metabase/upgrade.go | 364 ++++++++++++++++++ .../metabase/upgrade_test.go | 215 +++++++++++ pkg/local_object_storage/metabase/util.go | 9 +- pkg/local_object_storage/metabase/version.go | 15 + 7 files changed, 703 insertions(+), 3 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/metabase/root.go create mode 100644 cmd/frostfs-adm/internal/modules/metabase/upgrade.go create mode 100644 pkg/local_object_storage/metabase/upgrade.go create mode 100644 pkg/local_object_storage/metabase/upgrade_test.go diff --git a/cmd/frostfs-adm/internal/modules/metabase/root.go b/cmd/frostfs-adm/internal/modules/metabase/root.go new file mode 100644 index 000000000..5b21ed273 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/metabase/root.go @@ -0,0 +1,15 @@ +package metabase + +import "github.com/spf13/cobra" + +// RootCmd is a root command of config section. +var RootCmd = &cobra.Command{ + Use: "metabase", + Short: "Section for metabase commands", +} + +func init() { + RootCmd.AddCommand(UpgradeCmd) + + initUpgradeCommand() +} diff --git a/cmd/frostfs-adm/internal/modules/metabase/upgrade.go b/cmd/frostfs-adm/internal/modules/metabase/upgrade.go new file mode 100644 index 000000000..83e085df4 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/metabase/upgrade.go @@ -0,0 +1,86 @@ +package metabase + +import ( + "errors" + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + engineconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine" + shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "github.com/spf13/cobra" +) + +const ( + pathFlag = "path" + noCompactFlag = "no-compact" +) + +var errNoPathsFound = errors.New("no metabase paths found") + +var path string + +var UpgradeCmd = &cobra.Command{ + Use: "upgrade", + Short: "Upgrade metabase to latest version", + RunE: upgrade, +} + +func upgrade(cmd *cobra.Command, _ []string) error { + configFile, err := cmd.Flags().GetString(commonflags.ConfigFlag) + if err != nil { + return err + } + configDir, err := cmd.Flags().GetString(commonflags.ConfigDirFlag) + if err != nil { + return err + } + noCompact, _ := cmd.Flags().GetBool(noCompactFlag) + var paths []string + if path != "" { + paths = append(paths, path) + } + appCfg := config.New(configFile, configDir, config.EnvPrefix) + if err := engineconfig.IterateShards(appCfg, false, func(sc *shardconfig.Config) error { + paths = append(paths, sc.Metabase().Path()) + return nil + }); err != nil { + return fmt.Errorf("failed to get metabase paths: %w", err) + } + if len(paths) == 0 { + return errNoPathsFound + } + cmd.Println("found", len(paths), "metabases:") + for i, path := range paths { + cmd.Println(i+1, ":", path) + } + result := make(map[string]bool) + for _, path := range paths { + cmd.Println("upgrading metabase", path, "...") + if err := meta.Upgrade(cmd.Context(), path, !noCompact, func(a ...any) { + cmd.Println(append([]any{time.Now().Format(time.RFC3339), ":", path, ":"}, a...)...) + }); err != nil { + result[path] = false + cmd.Println("error: failed to upgrade metabase", path, ":", err) + } else { + result[path] = true + cmd.Println("metabase", path, "upgraded successfully") + } + } + for mb, ok := range result { + if ok { + cmd.Println(mb, ": success") + } else { + cmd.Println(mb, ": failed") + } + } + return nil +} + +func initUpgradeCommand() { + flags := UpgradeCmd.Flags() + flags.StringVar(&path, pathFlag, "", "Path to metabase file") + flags.Bool(noCompactFlag, false, "Do not compact upgraded metabase file") +} diff --git a/cmd/frostfs-adm/internal/modules/root.go b/cmd/frostfs-adm/internal/modules/root.go index 8595483ab..defd898c8 100644 --- a/cmd/frostfs-adm/internal/modules/root.go +++ b/cmd/frostfs-adm/internal/modules/root.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/storagecfg" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" @@ -41,6 +42,7 @@ func init() { rootCmd.AddCommand(config.RootCmd) rootCmd.AddCommand(morph.RootCmd) rootCmd.AddCommand(storagecfg.RootCmd) + rootCmd.AddCommand(metabase.RootCmd) rootCmd.AddCommand(autocomplete.Command("frostfs-adm")) rootCmd.AddCommand(gendoc.Command(rootCmd, gendoc.Options{})) diff --git a/pkg/local_object_storage/metabase/upgrade.go b/pkg/local_object_storage/metabase/upgrade.go new file mode 100644 index 000000000..014e50286 --- /dev/null +++ b/pkg/local_object_storage/metabase/upgrade.go @@ -0,0 +1,364 @@ +package meta + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "os" + "strconv" + "sync/atomic" + "time" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.etcd.io/bbolt" + "golang.org/x/sync/errgroup" +) + +const ( + upgradeLogFrequency = 50_000 + upgradeWorkersCount = 1_000 + compactMaxTxSize = 256 << 20 + upgradeTimeout = 1 * time.Second +) + +var updates = map[uint64]func(ctx context.Context, db *bbolt.DB, log func(a ...any)) error{ + 2: upgradeFromV2ToV3, +} + +func Upgrade(ctx context.Context, path string, compact bool, log func(a ...any)) error { + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("check metabase existence: %w", err) + } + opts := bbolt.DefaultOptions + opts.Timeout = upgradeTimeout + db, err := bbolt.Open(path, os.ModePerm, opts) + if err != nil { + return fmt.Errorf("open metabase: %w", err) + } + var version uint64 + if err := db.View(func(tx *bbolt.Tx) error { + var e error + version, e = currentVersion(tx) + return e + }); err != nil { + return err + } + updater, found := updates[version] + if !found { + return fmt.Errorf("unsupported version %d: no update available", version) + } + if err := updater(ctx, db, log); err != nil { + return fmt.Errorf("update metabase schema: %w", err) + } + if compact { + log("compacting metabase...") + err := compactDB(db) + if err != nil { + return fmt.Errorf("compact metabase: %w", err) + } + log("metabase compacted") + } + return db.Close() +} + +func compactDB(db *bbolt.DB) error { + sourcePath := db.Path() + tmpFileName := sourcePath + "." + time.Now().Format(time.RFC3339) + f, err := os.Stat(sourcePath) + if err != nil { + return err + } + dst, err := bbolt.Open(tmpFileName, f.Mode(), &bbolt.Options{ + Timeout: 100 * time.Millisecond, + }) + if err != nil { + return fmt.Errorf("can't open new metabase to compact: %w", err) + } + if err := bbolt.Compact(dst, db, compactMaxTxSize); err != nil { + return fmt.Errorf("compact metabase: %w", errors.Join(err, dst.Close(), os.Remove(tmpFileName))) + } + if err := dst.Close(); err != nil { + return fmt.Errorf("close compacted metabase: %w", errors.Join(err, os.Remove(tmpFileName))) + } + if err := db.Close(); err != nil { + return fmt.Errorf("close source metabase: %w", errors.Join(err, os.Remove(tmpFileName))) + } + if err := os.Rename(tmpFileName, sourcePath); err != nil { + return fmt.Errorf("replace source metabase with compacted: %w", errors.Join(err, os.Remove(tmpFileName))) + } + return nil +} + +func upgradeFromV2ToV3(ctx context.Context, db *bbolt.DB, log func(a ...any)) error { + if err := createExpirationEpochBuckets(ctx, db, log); err != nil { + return err + } + if err := dropUserAttributes(ctx, db, log); err != nil { + return err + } + if err := dropOwnerIDIndex(ctx, db, log); err != nil { + return err + } + if err := dropPayloadChecksumIndex(ctx, db, log); err != nil { + return err + } + return db.Update(func(tx *bbolt.Tx) error { + return updateVersion(tx, version) + }) +} + +type objectIDToExpEpoch struct { + containerID cid.ID + objectID oid.ID + expirationEpoch uint64 +} + +func createExpirationEpochBuckets(ctx context.Context, db *bbolt.DB, log func(a ...any)) error { + log("filling expiration epoch buckets...") + if err := db.Update(func(tx *bbolt.Tx) error { + _, err := tx.CreateBucketIfNotExists(expEpochToObjectBucketName) + return err + }); err != nil { + return err + } + objects := make(chan objectIDToExpEpoch) + eg, ctx := errgroup.WithContext(ctx) + eg.Go(func() error { + return selectObjectsWithExpirationEpoch(ctx, db, objects) + }) + var count atomic.Uint64 + for i := 0; i < upgradeWorkersCount; i++ { + eg.Go(func() error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case obj, ok := <-objects: + if !ok { + return nil + } + if err := db.Batch(func(tx *bbolt.Tx) error { + if err := putUniqueIndexItem(tx, namedBucketItem{ + name: expEpochToObjectBucketName, + key: expirationEpochKey(obj.expirationEpoch, obj.containerID, obj.objectID), + val: zeroValue, + }); err != nil { + return err + } + val := make([]byte, epochSize) + binary.LittleEndian.PutUint64(val, obj.expirationEpoch) + return putUniqueIndexItem(tx, namedBucketItem{ + name: objectToExpirationEpochBucketName(obj.containerID, make([]byte, bucketKeySize)), + key: objectKey(obj.objectID, make([]byte, objectKeySize)), + val: val, + }) + }); err != nil { + return err + } + } + if c := count.Add(1); c%upgradeLogFrequency == 0 { + log("expiration epoch filled for", c, "objects...") + } + } + }) + } + err := eg.Wait() + if err != nil { + log("expiration epoch buckets completed completed with error:", err) + return err + } + log("filling expiration epoch buckets completed successfully, total", count.Load(), "objects") + return nil +} + +func selectObjectsWithExpirationEpoch(ctx context.Context, db *bbolt.DB, objects chan objectIDToExpEpoch) error { + defer close(objects) + + const batchSize = 1000 + it := &objectsWithExpirationEpochBatchIterator{ + lastAttributeKey: usrAttrPrefix, + } + for { + if err := getNextObjectsWithExpirationEpochBatch(ctx, db, it, batchSize); err != nil { + return err + } + for _, item := range it.items { + select { + case <-ctx.Done(): + return ctx.Err() + case objects <- item: + } + } + + if len(it.items) < batchSize { + return nil + } + it.items = nil + } +} + +var ( + usrAttrPrefix = []byte{userAttributePrefix} + errBatchSizeLimit = errors.New("batch size limit") +) + +type objectsWithExpirationEpochBatchIterator struct { + lastAttributeKey []byte + lastAttributeValue []byte + lastAttrKeyValueItem []byte + items []objectIDToExpEpoch +} + +// - {prefix}{containerID}{attributeKey} <- bucket +// -- {attributeValue} <- bucket, expirationEpoch +// --- {objectID}: zeroValue <- record + +func getNextObjectsWithExpirationEpochBatch(ctx context.Context, db *bbolt.DB, it *objectsWithExpirationEpochBatchIterator, batchSize int) error { + seekAttrValue := it.lastAttributeValue + seekAttrKVItem := it.lastAttrKeyValueItem + err := db.View(func(tx *bbolt.Tx) error { + attrKeyC := tx.Cursor() + for attrKey, _ := attrKeyC.Seek(it.lastAttributeKey); attrKey != nil && bytes.HasPrefix(attrKey, usrAttrPrefix); attrKey, _ = attrKeyC.Next() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if len(attrKey) <= 1+cidSize { + continue + } + attributeKey := string(attrKey[1+cidSize:]) + if attributeKey != objectV2.SysAttributeExpEpochNeoFS && attributeKey != objectV2.SysAttributeExpEpoch { + continue + } + var containerID cid.ID + if err := containerID.Decode(attrKey[1 : 1+cidSize]); err != nil { + return fmt.Errorf("decode container id from user attribute bucket: %w", err) + } + if err := iterateExpirationAttributeKeyBucket(ctx, tx.Bucket(attrKey), it, batchSize, containerID, attrKey, seekAttrValue, seekAttrKVItem); err != nil { + return err + } + seekAttrValue = nil + seekAttrKVItem = nil + } + return nil + }) + if err != nil && !errors.Is(err, errBatchSizeLimit) { + return err + } + return nil +} + +func iterateExpirationAttributeKeyBucket(ctx context.Context, b *bbolt.Bucket, it *objectsWithExpirationEpochBatchIterator, batchSize int, containerID cid.ID, attrKey, seekAttrValue, seekAttrKVItem []byte) error { + attrValueC := b.Cursor() + for attrValue, v := attrValueC.Seek(seekAttrValue); attrValue != nil; attrValue, v = attrValueC.Next() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if v != nil { + continue // need to iterate over buckets, not records + } + expirationEpoch, err := strconv.ParseUint(string(attrValue), 10, 64) + if err != nil { + return fmt.Errorf("could not parse expiration epoch: %w", err) + } + expirationEpochBucket := b.Bucket(attrValue) + attrKeyValueC := expirationEpochBucket.Cursor() + for attrKeyValueItem, v := attrKeyValueC.Seek(seekAttrKVItem); attrKeyValueItem != nil; attrKeyValueItem, v = attrKeyValueC.Next() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if v == nil { + continue // need to iterate over records, not buckets + } + if bytes.Equal(it.lastAttributeKey, attrKey) && bytes.Equal(it.lastAttributeValue, attrValue) && bytes.Equal(it.lastAttrKeyValueItem, attrKeyValueItem) { + continue + } + var objectID oid.ID + if err := objectID.Decode(attrKeyValueItem); err != nil { + return fmt.Errorf("decode object id from container '%s' expiration epoch %d: %w", containerID, expirationEpoch, err) + } + it.lastAttributeKey = bytes.Clone(attrKey) + it.lastAttributeValue = bytes.Clone(attrValue) + it.lastAttrKeyValueItem = bytes.Clone(attrKeyValueItem) + it.items = append(it.items, objectIDToExpEpoch{ + containerID: containerID, + objectID: objectID, + expirationEpoch: expirationEpoch, + }) + if len(it.items) == batchSize { + return errBatchSizeLimit + } + } + seekAttrKVItem = nil + } + return nil +} + +func dropUserAttributes(ctx context.Context, db *bbolt.DB, log func(a ...any)) error { + return dropBucketsByPrefix(ctx, db, []byte{userAttributePrefix}, func(a ...any) { + log(append([]any{"user attributes:"}, a...)...) + }) +} + +func dropOwnerIDIndex(ctx context.Context, db *bbolt.DB, log func(a ...any)) error { + return dropBucketsByPrefix(ctx, db, []byte{ownerPrefix}, func(a ...any) { + log(append([]any{"owner ID index:"}, a...)...) + }) +} + +func dropPayloadChecksumIndex(ctx context.Context, db *bbolt.DB, log func(a ...any)) error { + return dropBucketsByPrefix(ctx, db, []byte{payloadHashPrefix}, func(a ...any) { + log(append([]any{"payload checksum:"}, a...)...) + }) +} + +func dropBucketsByPrefix(ctx context.Context, db *bbolt.DB, prefix []byte, log func(a ...any)) error { + log("deleting buckets...") + const batch = 1000 + var count uint64 + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + var keys [][]byte + if err := db.View(func(tx *bbolt.Tx) error { + c := tx.Cursor() + for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix) && len(keys) < batch; k, _ = c.Next() { + keys = append(keys, bytes.Clone(k)) + } + return nil + }); err != nil { + log("deleting buckets completed with an error:", err) + return err + } + if len(keys) == 0 { + log("deleting buckets completed successfully, deleted", count, "buckets") + return nil + } + if err := db.Update(func(tx *bbolt.Tx) error { + for _, k := range keys { + if err := tx.DeleteBucket(k); err != nil { + return err + } + } + return nil + }); err != nil { + log("deleting buckets completed with an error:", err) + return err + } + if count += uint64(len(keys)); count%upgradeLogFrequency == 0 { + log("deleted", count, "buckets") + } + } +} diff --git a/pkg/local_object_storage/metabase/upgrade_test.go b/pkg/local_object_storage/metabase/upgrade_test.go new file mode 100644 index 000000000..dc3d7d07d --- /dev/null +++ b/pkg/local_object_storage/metabase/upgrade_test.go @@ -0,0 +1,215 @@ +//go:build integration + +package meta + +import ( + "context" + "fmt" + "io" + "os" + "strconv" + "testing" + "time" + + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +const upgradeFilePath = "/path/to/metabase.v2" + +func TestUpgradeV2ToV3(t *testing.T) { + path := createTempCopy(t, upgradeFilePath) + defer func() { + require.NoError(t, os.Remove(path)) + }() + db := New(WithPath(path), WithEpochState(epochState{e: 1000}), WithLogger(test.NewLogger(t))) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + require.ErrorIs(t, db.Init(), ErrOutdatedVersion) + require.NoError(t, db.Close()) + require.NoError(t, Upgrade(context.Background(), path, true, t.Log)) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + require.NoError(t, db.Init()) + require.NoError(t, db.Close()) + fmt.Println() +} + +func createTempCopy(t *testing.T, path string) string { + src, err := os.Open(path) + require.NoError(t, err) + + tmpPath := upgradeFilePath + time.Now().Format(time.RFC3339) + dest, err := os.Create(tmpPath) + require.NoError(t, err) + + _, err = io.Copy(dest, src) + require.NoError(t, err) + + require.NoError(t, src.Close()) + require.NoError(t, dest.Close()) + + return tmpPath +} + +func TestGenerateMetabaseFile(t *testing.T) { + t.Skip("for generating db") + const ( + containersCount = 10_000 + simpleObjectsCount = 500_000 + complexObjectsCount = 500_000 // x2 + deletedByGCMarksCount = 100_000 + deletedByTombstoneCount = 100_000 // x2 + lockedCount = 100_000 // x2 + + allocSize = 128 << 20 + generateWorkersCount = 1_000 + minEpoch = 1_000 + maxFilename = 1_000 + maxStorageID = 10_000 + ) + + db := New(WithPath(upgradeFilePath), WithEpochState(epochState{e: minEpoch}), WithLogger(test.NewLogger(t))) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + db.boltDB.AllocSize = allocSize + db.boltDB.NoSync = true + require.NoError(t, db.Init()) + containers := make([]cid.ID, containersCount) + for i := range containers { + containers[i] = cidtest.ID() + } + oc, err := db.ObjectCounters() + require.NoError(t, err) + require.True(t, oc.IsZero()) + eg, ctx := errgroup.WithContext(context.Background()) + eg.SetLimit(generateWorkersCount) + // simple objects + for i := 0; i < simpleObjectsCount; i++ { + i := i + eg.Go(func() error { + obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) + testutil.AddAttribute(obj, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) + testutil.AddAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.FormatUint(uint64(i%minEpoch+minEpoch), 10)) + _, err := db.Put(ctx, PutPrm{ + obj: obj, + id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), + }) + require.NoError(t, err) + return nil + }) + } + require.NoError(t, eg.Wait()) + db.log.Info("simple objects generated") + eg, ctx = errgroup.WithContext(context.Background()) + eg.SetLimit(generateWorkersCount) + // complex objects + for i := 0; i < complexObjectsCount; i++ { + i := i + eg.Go(func() error { + parent := testutil.GenerateObjectWithCID(containers[i%len(containers)]) + child := testutil.GenerateObjectWithCID(containers[i%len(containers)]) + child.SetParent(parent) + idParent, _ := parent.ID() + child.SetParentID(idParent) + testutil.AddAttribute(child, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) + testutil.AddAttribute(parent, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) + testutil.AddAttribute(child, objectV2.SysAttributeExpEpoch, strconv.FormatUint(uint64(i%minEpoch+minEpoch), 10)) + testutil.AddAttribute(parent, objectV2.SysAttributeExpEpoch, strconv.FormatUint(uint64(i%minEpoch+minEpoch), 10)) + _, err := db.Put(ctx, PutPrm{ + obj: child, + }) + require.NoError(t, err) + return nil + }) + } + require.NoError(t, eg.Wait()) + db.log.Info("complex objects generated") + eg, ctx = errgroup.WithContext(context.Background()) + eg.SetLimit(generateWorkersCount) + // simple objects deleted by gc marks + for i := 0; i < deletedByGCMarksCount; i++ { + i := i + eg.Go(func() error { + obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) + testutil.AddAttribute(obj, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) + _, err := db.Put(ctx, PutPrm{ + obj: obj, + id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), + }) + require.NoError(t, err) + _, err = db.Inhume(ctx, InhumePrm{ + target: []oid.Address{object.AddressOf(obj)}, + }) + require.NoError(t, err) + return nil + }) + } + require.NoError(t, eg.Wait()) + db.log.Info("simple objects deleted by gc marks generated") + eg, ctx = errgroup.WithContext(context.Background()) + eg.SetLimit(10000) + // simple objects deleted by tombstones + for i := 0; i < deletedByTombstoneCount; i++ { + i := i + eg.Go(func() error { + obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) + testutil.AddAttribute(obj, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) + _, err := db.Put(ctx, PutPrm{ + obj: obj, + id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), + }) + tomb := testutil.GenerateObjectWithCID(containers[i%len(containers)]) + tomb.SetType(objectSDK.TypeTombstone) + _, err = db.Put(ctx, PutPrm{ + obj: tomb, + id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), + }) + require.NoError(t, err) + tombAddr := object.AddressOf(tomb) + _, err = db.Inhume(ctx, InhumePrm{ + target: []oid.Address{object.AddressOf(obj)}, + tomb: &tombAddr, + }) + require.NoError(t, err) + return nil + }) + } + require.NoError(t, eg.Wait()) + db.log.Info("simple objects deleted by tombstones generated") + eg, ctx = errgroup.WithContext(context.Background()) + eg.SetLimit(generateWorkersCount) + // simple objects locked by locks + for i := 0; i < lockedCount; i++ { + i := i + eg.Go(func() error { + obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) + testutil.AddAttribute(obj, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) + _, err := db.Put(ctx, PutPrm{ + obj: obj, + id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), + }) + lock := testutil.GenerateObjectWithCID(containers[i%len(containers)]) + lock.SetType(objectSDK.TypeLock) + testutil.AddAttribute(lock, objectV2.SysAttributeExpEpoch, strconv.FormatUint(uint64(i%minEpoch+minEpoch), 10)) + _, err = db.Put(ctx, PutPrm{ + obj: lock, + id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), + }) + require.NoError(t, err) + err = db.Lock(ctx, containers[i%len(containers)], object.AddressOf(lock).Object(), []oid.ID{object.AddressOf(obj).Object()}) + require.NoError(t, err) + return nil + }) + } + require.NoError(t, eg.Wait()) + db.log.Info("simple objects locked by locks generated") + require.NoError(t, db.boltDB.Sync()) + require.NoError(t, db.Close()) +} diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index 9134616fe..eef7210dc 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -94,11 +94,13 @@ const ( // ownerPrefix was used for prefixing FKBT index buckets mapping owner to object IDs. // Key: owner ID // Value: bucket containing object IDs as keys - _ + // removed in version 3 + ownerPrefix // userAttributePrefix was used for prefixing FKBT index buckets containing objects. // Key: attribute value // Value: bucket containing object IDs as keys - _ + // removed in version 3 + userAttributePrefix // ==================== // List index buckets. @@ -107,7 +109,8 @@ const ( // payloadHashPrefix was used for prefixing List index buckets mapping payload hash to a list of object IDs. // Key: payload hash // Value: list of object IDs - _ + // removed in version 3 + payloadHashPrefix // parentPrefix is used for prefixing List index buckets mapping parent ID to a list of children IDs. // Key: parent ID // Value: list of object IDs diff --git a/pkg/local_object_storage/metabase/version.go b/pkg/local_object_storage/metabase/version.go index bb2b66d9b..9e15babbc 100644 --- a/pkg/local_object_storage/metabase/version.go +++ b/pkg/local_object_storage/metabase/version.go @@ -2,6 +2,7 @@ package meta import ( "encoding/binary" + "errors" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -18,6 +19,8 @@ var versionKey = []byte("version") // the current code version. var ErrOutdatedVersion = logicerr.New("invalid version, resynchronization is required") +var errVersionUndefinedNoInfoBucket = errors.New("version undefined: no info bucket") + func checkVersion(tx *bbolt.Tx, initialized bool) error { var knownVersion bool @@ -59,3 +62,15 @@ func updateVersion(tx *bbolt.Tx, version uint64) error { } return b.Put(versionKey, data) } + +func currentVersion(tx *bbolt.Tx) (uint64, error) { + b := tx.Bucket(shardInfoBucket) + if b == nil { + return 0, errVersionUndefinedNoInfoBucket + } + data := b.Get(versionKey) + if len(data) != 8 { + return 0, fmt.Errorf("version undefined: invalid version data length %d", len(data)) + } + return binary.LittleEndian.Uint64(data), nil +} From 882c068410628e4e8a7fe929189e97f4e796a5d1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 29 Aug 2024 16:33:30 +0300 Subject: [PATCH 1307/1943] [#1334] metabase: Store upgrade flag This allows to check if metabase upgrade was not completed. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/upgrade.go | 12 +++++++++++ pkg/local_object_storage/metabase/version.go | 11 +++++++++- .../metabase/version_test.go | 20 +++++++++++++++++++ pkg/local_object_storage/shard/control.go | 2 +- 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/metabase/upgrade.go b/pkg/local_object_storage/metabase/upgrade.go index 014e50286..a4c7707b4 100644 --- a/pkg/local_object_storage/metabase/upgrade.go +++ b/pkg/local_object_storage/metabase/upgrade.go @@ -51,9 +51,21 @@ func Upgrade(ctx context.Context, path string, compact bool, log func(a ...any)) if !found { return fmt.Errorf("unsupported version %d: no update available", version) } + if err := db.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket(shardInfoBucket) + return b.Put(upgradeKey, zeroValue) + }); err != nil { + return fmt.Errorf("set upgrade key %w", err) + } if err := updater(ctx, db, log); err != nil { return fmt.Errorf("update metabase schema: %w", err) } + if err := db.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket(shardInfoBucket) + return b.Delete(upgradeKey) + }); err != nil { + return fmt.Errorf("delete upgrade key %w", err) + } if compact { log("compacting metabase...") err := compactDB(db) diff --git a/pkg/local_object_storage/metabase/version.go b/pkg/local_object_storage/metabase/version.go index 9e15babbc..048bb9af6 100644 --- a/pkg/local_object_storage/metabase/version.go +++ b/pkg/local_object_storage/metabase/version.go @@ -12,13 +12,18 @@ import ( // version contains current metabase version. const version = 3 -var versionKey = []byte("version") +var ( + versionKey = []byte("version") + upgradeKey = []byte("upgrade") +) // ErrOutdatedVersion is returned on initializing // an existing metabase that is not compatible with // the current code version. var ErrOutdatedVersion = logicerr.New("invalid version, resynchronization is required") +var ErrIncompletedUpgrade = logicerr.New("metabase upgrade is not completed") + var errVersionUndefinedNoInfoBucket = errors.New("version undefined: no info bucket") func checkVersion(tx *bbolt.Tx, initialized bool) error { @@ -35,6 +40,10 @@ func checkVersion(tx *bbolt.Tx, initialized bool) error { return fmt.Errorf("%w: expected=%d, stored=%d", ErrOutdatedVersion, version, stored) } } + data = b.Get(upgradeKey) + if len(data) > 0 { + return ErrIncompletedUpgrade + } } if !initialized { diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index b2af428ff..75229a1b4 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -84,4 +84,24 @@ func TestVersion(t *testing.T) { require.NoError(t, db.Close()) }) }) + t.Run("incompleted upgrade", func(t *testing.T) { + db := newDB(t) + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + require.NoError(t, db.Init()) + require.NoError(t, db.Close()) + + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + require.NoError(t, db.boltDB.Update(func(tx *bbolt.Tx) error { + return tx.Bucket(shardInfoBucket).Put(upgradeKey, zeroValue) + })) + require.ErrorIs(t, db.Init(), ErrIncompletedUpgrade) + require.NoError(t, db.Close()) + + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + require.NoError(t, db.boltDB.Update(func(tx *bbolt.Tx) error { + return tx.Bucket(shardInfoBucket).Delete(upgradeKey) + })) + require.NoError(t, db.Init()) + require.NoError(t, db.Close()) + }) } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 936a506c0..1626d5804 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -171,7 +171,7 @@ func (s *Shard) initializeComponents(m mode.Mode) error { for _, component := range components { if err := component.Init(); err != nil { if component == s.metaBase { - if errors.Is(err, meta.ErrOutdatedVersion) { + if errors.Is(err, meta.ErrOutdatedVersion) || errors.Is(err, meta.ErrIncompletedUpgrade) { return fmt.Errorf("metabase initialization: %w", err) } From 98fe24cdb7347b9b4ddfb1c8fe5e2025465d7692 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 29 Aug 2024 15:45:33 +0300 Subject: [PATCH 1308/1943] [#1343] go.mod: Update api-go Signed-off-by: Evgenii Stratonikov --- .../modules/control/shards_list.go | 6 +- cmd/frostfs-cli/modules/tree/add.go | 6 +- cmd/frostfs-cli/modules/tree/move.go | 2 +- go.mod | 2 +- go.sum | 4 +- pkg/services/control/server/list_shards.go | 10 +- pkg/services/control/server/policy_engine.go | 4 +- .../control/server/seal_writecache.go | 4 +- pkg/services/control/service_frostfs.pb.go | 66 +++---- pkg/services/control/types_frostfs.pb.go | 86 ++++----- pkg/services/tree/service.go | 22 +-- pkg/services/tree/service_frostfs.pb.go | 172 ++++++++---------- pkg/services/tree/signature_test.go | 2 +- 13 files changed, 180 insertions(+), 206 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/shards_list.go b/cmd/frostfs-cli/modules/control/shards_list.go index 07c5bcd9a..e9e49bb29 100644 --- a/cmd/frostfs-cli/modules/control/shards_list.go +++ b/cmd/frostfs-cli/modules/control/shards_list.go @@ -61,7 +61,7 @@ func listShards(cmd *cobra.Command, _ []string) { } } -func prettyPrintShardsJSON(cmd *cobra.Command, ii []*control.ShardInfo) { +func prettyPrintShardsJSON(cmd *cobra.Command, ii []control.ShardInfo) { out := make([]map[string]any, 0, len(ii)) for _, i := range ii { out = append(out, map[string]any{ @@ -83,7 +83,7 @@ func prettyPrintShardsJSON(cmd *cobra.Command, ii []*control.ShardInfo) { cmd.Print(buf.String()) // pretty printer emits newline, so no need for Println } -func prettyPrintShards(cmd *cobra.Command, ii []*control.ShardInfo) { +func prettyPrintShards(cmd *cobra.Command, ii []control.ShardInfo) { for _, i := range ii { pathPrinter := func(name, path string) string { if path == "" { @@ -121,7 +121,7 @@ func shardModeToString(m control.ShardMode) string { return "unknown" } -func sortShardsByID(ii []*control.ShardInfo) { +func sortShardsByID(ii []control.ShardInfo) { sort.Slice(ii, func(i, j int) bool { return bytes.Compare(ii[i].GetShard_ID(), ii[j].GetShard_ID()) < 0 }) diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index 068b1d185..0b8dc292f 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -77,13 +77,13 @@ func add(cmd *cobra.Command, _ []string) { cmd.Println("Node ID: ", resp.GetBody().GetNodeId()) } -func parseMeta(cmd *cobra.Command) ([]*tree.KeyValue, error) { +func parseMeta(cmd *cobra.Command) ([]tree.KeyValue, error) { raws, _ := cmd.Flags().GetStringSlice(metaFlagKey) if len(raws) == 0 { return nil, nil } - pairs := make([]*tree.KeyValue, 0, len(raws)) + pairs := make([]tree.KeyValue, 0, len(raws)) for i := range raws { k, v, found := strings.Cut(raws[i], "=") if !found { @@ -94,7 +94,7 @@ func parseMeta(cmd *cobra.Command) ([]*tree.KeyValue, error) { pair.Key = k pair.Value = []byte(v) - pairs = append(pairs, &pair) + pairs = append(pairs, pair) } return pairs, nil diff --git a/cmd/frostfs-cli/modules/tree/move.go b/cmd/frostfs-cli/modules/tree/move.go index 95516940c..24abbd650 100644 --- a/cmd/frostfs-cli/modules/tree/move.go +++ b/cmd/frostfs-cli/modules/tree/move.go @@ -75,7 +75,7 @@ func move(cmd *cobra.Command, _ []string) { resp, err := cli.GetSubTree(ctx, subTreeReq) commonCmd.ExitOnErr(cmd, "rpc call: %w", err) - var meta []*tree.KeyValue + var meta []tree.KeyValue subtreeResp, err := resp.Recv() for ; err == nil; subtreeResp, err = resp.Recv() { meta = subtreeResp.GetBody().GetMeta() diff --git a/go.mod b/go.mod index b7f59c823..358370201 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240827104600-eba18f6e67ac + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240828085308-5e1c6a908f61 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d diff --git a/go.sum b/go.sum index d2f926151..be82bff70 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240827104600-eba18f6e67ac h1:Gu3oiPnsSZPgwsUYs2f3xTQwndM/OWM/zo3zbN4rOb8= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240827104600-eba18f6e67ac/go.mod h1:BDnEpkKMykCS8u1nLzR6SgNzCv6885RWlo5TnravQuI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240828085308-5e1c6a908f61 h1:bw9EVGWnfY9awFb5XYR52AGbzgg3o04gZF66yHob48c= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240828085308-5e1c6a908f61/go.mod h1:BDnEpkKMykCS8u1nLzR6SgNzCv6885RWlo5TnravQuI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= diff --git a/pkg/services/control/server/list_shards.go b/pkg/services/control/server/list_shards.go index b639245c1..56bd9fc1f 100644 --- a/pkg/services/control/server/list_shards.go +++ b/pkg/services/control/server/list_shards.go @@ -25,7 +25,7 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) ( info := s.s.DumpInfo() - shardInfos := make([]*control.ShardInfo, 0, len(info.Shards)) + shardInfos := make([]control.ShardInfo, 0, len(info.Shards)) for _, sh := range info.Shards { si := new(control.ShardInfo) @@ -54,7 +54,7 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) ( si.SetMode(m) si.SetErrorCount(sh.ErrorCount) - shardInfos = append(shardInfos, si) + shardInfos = append(shardInfos, *si) } body.SetShards(shardInfos) @@ -67,10 +67,10 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) ( return resp, nil } -func blobstorInfoToProto(info blobstor.Info) []*control.BlobstorInfo { - res := make([]*control.BlobstorInfo, len(info.SubStorages)) +func blobstorInfoToProto(info blobstor.Info) []control.BlobstorInfo { + res := make([]control.BlobstorInfo, len(info.SubStorages)) for i := range info.SubStorages { - res[i] = &control.BlobstorInfo{ + res[i] = control.BlobstorInfo{ Path: info.SubStorages[i].Path, Type: info.SubStorages[i].Type, } diff --git a/pkg/services/control/server/policy_engine.go b/pkg/services/control/server/policy_engine.go index 98daac8a6..ab8258e27 100644 --- a/pkg/services/control/server/policy_engine.go +++ b/pkg/services/control/server/policy_engine.go @@ -220,13 +220,13 @@ func (s *Server) ListTargetsLocalOverrides(_ context.Context, req *control.ListT if err != nil { return nil, status.Error(getCodeByLocalStorageErr(err), err.Error()) } - targets := make([]*control.ChainTarget, 0, len(apeTargets)) + targets := make([]control.ChainTarget, 0, len(apeTargets)) for i := range apeTargets { target, err := controlTarget(&apeTargets[i]) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } - targets = append(targets, &target) + targets = append(targets, target) } resp := &control.ListTargetsLocalOverridesResponse{ diff --git a/pkg/services/control/server/seal_writecache.go b/pkg/services/control/server/seal_writecache.go index 1737677b7..6799bdcac 100644 --- a/pkg/services/control/server/seal_writecache.go +++ b/pkg/services/control/server/seal_writecache.go @@ -32,12 +32,12 @@ func (s *Server) SealWriteCache(ctx context.Context, req *control.SealWriteCache resp := &control.SealWriteCacheResponse{Body: &control.SealWriteCacheResponse_Body{}} for _, r := range res.ShardResults { if r.Success { - resp.Body.Results = append(resp.GetBody().GetResults(), &control.SealWriteCacheResponse_Body_Status{ + resp.Body.Results = append(resp.GetBody().GetResults(), control.SealWriteCacheResponse_Body_Status{ Shard_ID: *r.ShardID, Success: true, }) } else { - resp.Body.Results = append(resp.GetBody().GetResults(), &control.SealWriteCacheResponse_Body_Status{ + resp.Body.Results = append(resp.GetBody().GetResults(), control.SealWriteCacheResponse_Body_Status{ Shard_ID: *r.ShardID, Error: r.ErrorMsg, }) diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index a446c5e59..eb0d95c64 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -2851,7 +2851,7 @@ func (x *ListShardsRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { } type ListShardsResponse_Body struct { - Shards []*ShardInfo `json:"shards"` + Shards []ShardInfo `json:"shards"` } var ( @@ -2869,7 +2869,7 @@ func (x *ListShardsResponse_Body) StableSize() (size int) { return 0 } for i := range x.Shards { - size += proto.NestedStructureSize(1, x.Shards[i]) + size += proto.NestedStructureSizeUnchecked(1, &x.Shards[i]) } return size } @@ -2888,9 +2888,7 @@ func (x *ListShardsResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { return } for i := range x.Shards { - if x.Shards[i] != nil { - x.Shards[i].EmitProtobuf(mm.AppendMessage(1)) - } + x.Shards[i].EmitProtobuf(mm.AppendMessage(1)) } } @@ -2908,8 +2906,8 @@ func (x *ListShardsResponse_Body) UnmarshalProtobuf(src []byte) (err error) { if !ok { return fmt.Errorf("cannot unmarshal field %s", "Shards") } - x.Shards = append(x.Shards, new(ShardInfo)) - ff := x.Shards[len(x.Shards)-1] + x.Shards = append(x.Shards, ShardInfo{}) + ff := &x.Shards[len(x.Shards)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -2917,13 +2915,13 @@ func (x *ListShardsResponse_Body) UnmarshalProtobuf(src []byte) (err error) { } return nil } -func (x *ListShardsResponse_Body) GetShards() []*ShardInfo { +func (x *ListShardsResponse_Body) GetShards() []ShardInfo { if x != nil { return x.Shards } return nil } -func (x *ListShardsResponse_Body) SetShards(v []*ShardInfo) { +func (x *ListShardsResponse_Body) SetShards(v []ShardInfo) { x.Shards = v } @@ -2981,11 +2979,11 @@ func (x *ListShardsResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { switch key { case "shards": { - var f *ShardInfo - var list []*ShardInfo + var f ShardInfo + var list []ShardInfo in.Delim('[') for !in.IsDelim(']') { - f = new(ShardInfo) + f = ShardInfo{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() @@ -11870,7 +11868,7 @@ func (x *ListTargetsLocalOverridesRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { } type ListTargetsLocalOverridesResponse_Body struct { - Targets []*ChainTarget `json:"targets"` + Targets []ChainTarget `json:"targets"` } var ( @@ -11888,7 +11886,7 @@ func (x *ListTargetsLocalOverridesResponse_Body) StableSize() (size int) { return 0 } for i := range x.Targets { - size += proto.NestedStructureSize(1, x.Targets[i]) + size += proto.NestedStructureSizeUnchecked(1, &x.Targets[i]) } return size } @@ -11907,9 +11905,7 @@ func (x *ListTargetsLocalOverridesResponse_Body) EmitProtobuf(mm *easyproto.Mess return } for i := range x.Targets { - if x.Targets[i] != nil { - x.Targets[i].EmitProtobuf(mm.AppendMessage(1)) - } + x.Targets[i].EmitProtobuf(mm.AppendMessage(1)) } } @@ -11927,8 +11923,8 @@ func (x *ListTargetsLocalOverridesResponse_Body) UnmarshalProtobuf(src []byte) ( if !ok { return fmt.Errorf("cannot unmarshal field %s", "Targets") } - x.Targets = append(x.Targets, new(ChainTarget)) - ff := x.Targets[len(x.Targets)-1] + x.Targets = append(x.Targets, ChainTarget{}) + ff := &x.Targets[len(x.Targets)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -11936,13 +11932,13 @@ func (x *ListTargetsLocalOverridesResponse_Body) UnmarshalProtobuf(src []byte) ( } return nil } -func (x *ListTargetsLocalOverridesResponse_Body) GetTargets() []*ChainTarget { +func (x *ListTargetsLocalOverridesResponse_Body) GetTargets() []ChainTarget { if x != nil { return x.Targets } return nil } -func (x *ListTargetsLocalOverridesResponse_Body) SetTargets(v []*ChainTarget) { +func (x *ListTargetsLocalOverridesResponse_Body) SetTargets(v []ChainTarget) { x.Targets = v } @@ -12000,11 +11996,11 @@ func (x *ListTargetsLocalOverridesResponse_Body) UnmarshalEasyJSON(in *jlexer.Le switch key { case "targets": { - var f *ChainTarget - var list []*ChainTarget + var f ChainTarget + var list []ChainTarget in.Delim('[') for !in.IsDelim(']') { - f = new(ChainTarget) + f = ChainTarget{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() @@ -14083,7 +14079,7 @@ func (x *SealWriteCacheResponse_Body_Status) UnmarshalEasyJSON(in *jlexer.Lexer) } type SealWriteCacheResponse_Body struct { - Results []*SealWriteCacheResponse_Body_Status `json:"results"` + Results []SealWriteCacheResponse_Body_Status `json:"results"` } var ( @@ -14101,7 +14097,7 @@ func (x *SealWriteCacheResponse_Body) StableSize() (size int) { return 0 } for i := range x.Results { - size += proto.NestedStructureSize(1, x.Results[i]) + size += proto.NestedStructureSizeUnchecked(1, &x.Results[i]) } return size } @@ -14120,9 +14116,7 @@ func (x *SealWriteCacheResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshale return } for i := range x.Results { - if x.Results[i] != nil { - x.Results[i].EmitProtobuf(mm.AppendMessage(1)) - } + x.Results[i].EmitProtobuf(mm.AppendMessage(1)) } } @@ -14140,8 +14134,8 @@ func (x *SealWriteCacheResponse_Body) UnmarshalProtobuf(src []byte) (err error) if !ok { return fmt.Errorf("cannot unmarshal field %s", "Results") } - x.Results = append(x.Results, new(SealWriteCacheResponse_Body_Status)) - ff := x.Results[len(x.Results)-1] + x.Results = append(x.Results, SealWriteCacheResponse_Body_Status{}) + ff := &x.Results[len(x.Results)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -14149,13 +14143,13 @@ func (x *SealWriteCacheResponse_Body) UnmarshalProtobuf(src []byte) (err error) } return nil } -func (x *SealWriteCacheResponse_Body) GetResults() []*SealWriteCacheResponse_Body_Status { +func (x *SealWriteCacheResponse_Body) GetResults() []SealWriteCacheResponse_Body_Status { if x != nil { return x.Results } return nil } -func (x *SealWriteCacheResponse_Body) SetResults(v []*SealWriteCacheResponse_Body_Status) { +func (x *SealWriteCacheResponse_Body) SetResults(v []SealWriteCacheResponse_Body_Status) { x.Results = v } @@ -14213,11 +14207,11 @@ func (x *SealWriteCacheResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { switch key { case "results": { - var f *SealWriteCacheResponse_Body_Status - var list []*SealWriteCacheResponse_Body_Status + var f SealWriteCacheResponse_Body_Status + var list []SealWriteCacheResponse_Body_Status in.Delim('[') for !in.IsDelim(']') { - f = new(SealWriteCacheResponse_Body_Status) + f = SealWriteCacheResponse_Body_Status{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() diff --git a/pkg/services/control/types_frostfs.pb.go b/pkg/services/control/types_frostfs.pb.go index 3cc37245f..42c1afa52 100644 --- a/pkg/services/control/types_frostfs.pb.go +++ b/pkg/services/control/types_frostfs.pb.go @@ -500,10 +500,10 @@ func (x *NodeInfo_Attribute) UnmarshalEasyJSON(in *jlexer.Lexer) { } type NodeInfo struct { - PublicKey []byte `json:"publicKey"` - Addresses []string `json:"addresses"` - Attributes []*NodeInfo_Attribute `json:"attributes"` - State NetmapStatus `json:"state"` + PublicKey []byte `json:"publicKey"` + Addresses []string `json:"addresses"` + Attributes []NodeInfo_Attribute `json:"attributes"` + State NetmapStatus `json:"state"` } var ( @@ -523,7 +523,7 @@ func (x *NodeInfo) StableSize() (size int) { size += proto.BytesSize(1, x.PublicKey) size += proto.RepeatedStringSize(2, x.Addresses) for i := range x.Attributes { - size += proto.NestedStructureSize(3, x.Attributes[i]) + size += proto.NestedStructureSizeUnchecked(3, &x.Attributes[i]) } size += proto.EnumSize(4, int32(x.State)) return size @@ -549,9 +549,7 @@ func (x *NodeInfo) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendString(2, x.Addresses[j]) } for i := range x.Attributes { - if x.Attributes[i] != nil { - x.Attributes[i].EmitProtobuf(mm.AppendMessage(3)) - } + x.Attributes[i].EmitProtobuf(mm.AppendMessage(3)) } if int32(x.State) != 0 { mm.AppendInt32(4, int32(x.State)) @@ -584,8 +582,8 @@ func (x *NodeInfo) UnmarshalProtobuf(src []byte) (err error) { if !ok { return fmt.Errorf("cannot unmarshal field %s", "Attributes") } - x.Attributes = append(x.Attributes, new(NodeInfo_Attribute)) - ff := x.Attributes[len(x.Attributes)-1] + x.Attributes = append(x.Attributes, NodeInfo_Attribute{}) + ff := &x.Attributes[len(x.Attributes)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -617,13 +615,13 @@ func (x *NodeInfo) GetAddresses() []string { func (x *NodeInfo) SetAddresses(v []string) { x.Addresses = v } -func (x *NodeInfo) GetAttributes() []*NodeInfo_Attribute { +func (x *NodeInfo) GetAttributes() []NodeInfo_Attribute { if x != nil { return x.Attributes } return nil } -func (x *NodeInfo) SetAttributes(v []*NodeInfo_Attribute) { +func (x *NodeInfo) SetAttributes(v []NodeInfo_Attribute) { x.Attributes = v } func (x *NodeInfo) GetState() NetmapStatus { @@ -731,11 +729,11 @@ func (x *NodeInfo) UnmarshalEasyJSON(in *jlexer.Lexer) { } case "attributes": { - var f *NodeInfo_Attribute - var list []*NodeInfo_Attribute + var f NodeInfo_Attribute + var list []NodeInfo_Attribute in.Delim('[') for !in.IsDelim(']') { - f = new(NodeInfo_Attribute) + f = NodeInfo_Attribute{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() @@ -775,8 +773,8 @@ func (x *NodeInfo) UnmarshalEasyJSON(in *jlexer.Lexer) { } type Netmap struct { - Epoch uint64 `json:"epoch"` - Nodes []*NodeInfo `json:"nodes"` + Epoch uint64 `json:"epoch"` + Nodes []NodeInfo `json:"nodes"` } var ( @@ -795,7 +793,7 @@ func (x *Netmap) StableSize() (size int) { } size += proto.UInt64Size(1, x.Epoch) for i := range x.Nodes { - size += proto.NestedStructureSize(2, x.Nodes[i]) + size += proto.NestedStructureSizeUnchecked(2, &x.Nodes[i]) } return size } @@ -817,9 +815,7 @@ func (x *Netmap) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendUint64(1, x.Epoch) } for i := range x.Nodes { - if x.Nodes[i] != nil { - x.Nodes[i].EmitProtobuf(mm.AppendMessage(2)) - } + x.Nodes[i].EmitProtobuf(mm.AppendMessage(2)) } } @@ -843,8 +839,8 @@ func (x *Netmap) UnmarshalProtobuf(src []byte) (err error) { if !ok { return fmt.Errorf("cannot unmarshal field %s", "Nodes") } - x.Nodes = append(x.Nodes, new(NodeInfo)) - ff := x.Nodes[len(x.Nodes)-1] + x.Nodes = append(x.Nodes, NodeInfo{}) + ff := &x.Nodes[len(x.Nodes)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -861,13 +857,13 @@ func (x *Netmap) GetEpoch() uint64 { func (x *Netmap) SetEpoch(v uint64) { x.Epoch = v } -func (x *Netmap) GetNodes() []*NodeInfo { +func (x *Netmap) GetNodes() []NodeInfo { if x != nil { return x.Nodes } return nil } -func (x *Netmap) SetNodes(v []*NodeInfo) { +func (x *Netmap) SetNodes(v []NodeInfo) { x.Nodes = v } @@ -936,11 +932,11 @@ func (x *Netmap) UnmarshalEasyJSON(in *jlexer.Lexer) { } case "nodes": { - var f *NodeInfo - var list []*NodeInfo + var f NodeInfo + var list []NodeInfo in.Delim('[') for !in.IsDelim(']') { - f = new(NodeInfo) + f = NodeInfo{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() @@ -958,13 +954,13 @@ func (x *Netmap) UnmarshalEasyJSON(in *jlexer.Lexer) { } type ShardInfo struct { - Shard_ID []byte `json:"shardID"` - MetabasePath string `json:"metabasePath"` - Blobstor []*BlobstorInfo `json:"blobstor"` - WritecachePath string `json:"writecachePath"` - Mode ShardMode `json:"mode"` - ErrorCount uint32 `json:"errorCount"` - PiloramaPath string `json:"piloramaPath"` + Shard_ID []byte `json:"shardID"` + MetabasePath string `json:"metabasePath"` + Blobstor []BlobstorInfo `json:"blobstor"` + WritecachePath string `json:"writecachePath"` + Mode ShardMode `json:"mode"` + ErrorCount uint32 `json:"errorCount"` + PiloramaPath string `json:"piloramaPath"` } var ( @@ -984,7 +980,7 @@ func (x *ShardInfo) StableSize() (size int) { size += proto.BytesSize(1, x.Shard_ID) size += proto.StringSize(2, x.MetabasePath) for i := range x.Blobstor { - size += proto.NestedStructureSize(3, x.Blobstor[i]) + size += proto.NestedStructureSizeUnchecked(3, &x.Blobstor[i]) } size += proto.StringSize(4, x.WritecachePath) size += proto.EnumSize(5, int32(x.Mode)) @@ -1013,9 +1009,7 @@ func (x *ShardInfo) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendString(2, x.MetabasePath) } for i := range x.Blobstor { - if x.Blobstor[i] != nil { - x.Blobstor[i].EmitProtobuf(mm.AppendMessage(3)) - } + x.Blobstor[i].EmitProtobuf(mm.AppendMessage(3)) } if len(x.WritecachePath) != 0 { mm.AppendString(4, x.WritecachePath) @@ -1057,8 +1051,8 @@ func (x *ShardInfo) UnmarshalProtobuf(src []byte) (err error) { if !ok { return fmt.Errorf("cannot unmarshal field %s", "Blobstor") } - x.Blobstor = append(x.Blobstor, new(BlobstorInfo)) - ff := x.Blobstor[len(x.Blobstor)-1] + x.Blobstor = append(x.Blobstor, BlobstorInfo{}) + ff := &x.Blobstor[len(x.Blobstor)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -1108,13 +1102,13 @@ func (x *ShardInfo) GetMetabasePath() string { func (x *ShardInfo) SetMetabasePath(v string) { x.MetabasePath = v } -func (x *ShardInfo) GetBlobstor() []*BlobstorInfo { +func (x *ShardInfo) GetBlobstor() []BlobstorInfo { if x != nil { return x.Blobstor } return nil } -func (x *ShardInfo) SetBlobstor(v []*BlobstorInfo) { +func (x *ShardInfo) SetBlobstor(v []BlobstorInfo) { x.Blobstor = v } func (x *ShardInfo) GetWritecachePath() string { @@ -1250,11 +1244,11 @@ func (x *ShardInfo) UnmarshalEasyJSON(in *jlexer.Lexer) { } case "blobstor": { - var f *BlobstorInfo - var list []*BlobstorInfo + var f BlobstorInfo + var list []BlobstorInfo in.Delim('[') for !in.IsDelim(']') { - f = new(BlobstorInfo) + f = BlobstorInfo{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 2012f53d2..4da61617f 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -360,7 +360,7 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) return nil, err } - info := make([]*GetNodeByPathResponse_Info, 0, len(nodes)) + info := make([]GetNodeByPathResponse_Info, 0, len(nodes)) for _, node := range nodes { m, parent, err := s.forest.TreeGetMeta(ctx, cid, b.GetTreeId(), node) if err != nil { @@ -374,11 +374,11 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) if b.GetAllAttributes() { x.Meta = metaToProto(m.Items) } else { - var metaValue []*KeyValue + var metaValue []KeyValue for _, kv := range m.Items { for _, attr := range b.GetAttributes() { if kv.Key == attr { - metaValue = append(metaValue, &KeyValue{ + metaValue = append(metaValue, KeyValue{ Key: kv.Key, Value: kv.Value, }) @@ -388,7 +388,7 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) } x.Meta = metaValue } - info = append(info, &x) + info = append(info, x) } return &GetNodeByPathResponse{ @@ -782,21 +782,19 @@ func (s *Service) TreeList(ctx context.Context, req *TreeListRequest) (*TreeList }, nil } -func protoToMeta(arr []*KeyValue) []pilorama.KeyValue { +func protoToMeta(arr []KeyValue) []pilorama.KeyValue { meta := make([]pilorama.KeyValue, len(arr)) for i, kv := range arr { - if kv != nil { - meta[i].Key = kv.GetKey() - meta[i].Value = kv.GetValue() - } + meta[i].Key = kv.GetKey() + meta[i].Value = kv.GetValue() } return meta } -func metaToProto(arr []pilorama.KeyValue) []*KeyValue { - meta := make([]*KeyValue, len(arr)) +func metaToProto(arr []pilorama.KeyValue) []KeyValue { + meta := make([]KeyValue, len(arr)) for i, kv := range arr { - meta[i] = &KeyValue{ + meta[i] = KeyValue{ Key: kv.Key, Value: kv.Value, } diff --git a/pkg/services/tree/service_frostfs.pb.go b/pkg/services/tree/service_frostfs.pb.go index 3c6ba21b7..7b6abb1dd 100644 --- a/pkg/services/tree/service_frostfs.pb.go +++ b/pkg/services/tree/service_frostfs.pb.go @@ -16,11 +16,11 @@ import ( ) type AddRequest_Body struct { - ContainerId []byte `json:"containerId"` - TreeId string `json:"treeId"` - ParentId uint64 `json:"parentId"` - Meta []*KeyValue `json:"meta"` - BearerToken []byte `json:"bearerToken"` + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + ParentId uint64 `json:"parentId"` + Meta []KeyValue `json:"meta"` + BearerToken []byte `json:"bearerToken"` } var ( @@ -41,7 +41,7 @@ func (x *AddRequest_Body) StableSize() (size int) { size += proto.StringSize(2, x.TreeId) size += proto.UInt64Size(3, x.ParentId) for i := range x.Meta { - size += proto.NestedStructureSize(4, x.Meta[i]) + size += proto.NestedStructureSizeUnchecked(4, &x.Meta[i]) } size += proto.BytesSize(5, x.BearerToken) return size @@ -70,9 +70,7 @@ func (x *AddRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendUint64(3, x.ParentId) } for i := range x.Meta { - if x.Meta[i] != nil { - x.Meta[i].EmitProtobuf(mm.AppendMessage(4)) - } + x.Meta[i].EmitProtobuf(mm.AppendMessage(4)) } if len(x.BearerToken) != 0 { mm.AppendBytes(5, x.BearerToken) @@ -111,8 +109,8 @@ func (x *AddRequest_Body) UnmarshalProtobuf(src []byte) (err error) { if !ok { return fmt.Errorf("cannot unmarshal field %s", "Meta") } - x.Meta = append(x.Meta, new(KeyValue)) - ff := x.Meta[len(x.Meta)-1] + x.Meta = append(x.Meta, KeyValue{}) + ff := &x.Meta[len(x.Meta)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -153,13 +151,13 @@ func (x *AddRequest_Body) GetParentId() uint64 { func (x *AddRequest_Body) SetParentId(v uint64) { x.ParentId = v } -func (x *AddRequest_Body) GetMeta() []*KeyValue { +func (x *AddRequest_Body) GetMeta() []KeyValue { if x != nil { return x.Meta } return nil } -func (x *AddRequest_Body) SetMeta(v []*KeyValue) { +func (x *AddRequest_Body) SetMeta(v []KeyValue) { x.Meta = v } func (x *AddRequest_Body) GetBearerToken() []byte { @@ -264,11 +262,11 @@ func (x *AddRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { } case "meta": { - var f *KeyValue - var list []*KeyValue + var f KeyValue + var list []KeyValue in.Delim('[') for !in.IsDelim(']') { - f = new(KeyValue) + f = KeyValue{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() @@ -793,12 +791,12 @@ func (x *AddResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { } type AddByPathRequest_Body struct { - ContainerId []byte `json:"containerId"` - TreeId string `json:"treeId"` - PathAttribute string `json:"pathAttribute"` - Path []string `json:"path"` - Meta []*KeyValue `json:"meta"` - BearerToken []byte `json:"bearerToken"` + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + PathAttribute string `json:"pathAttribute"` + Path []string `json:"path"` + Meta []KeyValue `json:"meta"` + BearerToken []byte `json:"bearerToken"` } var ( @@ -820,7 +818,7 @@ func (x *AddByPathRequest_Body) StableSize() (size int) { size += proto.StringSize(3, x.PathAttribute) size += proto.RepeatedStringSize(4, x.Path) for i := range x.Meta { - size += proto.NestedStructureSize(5, x.Meta[i]) + size += proto.NestedStructureSizeUnchecked(5, &x.Meta[i]) } size += proto.BytesSize(6, x.BearerToken) return size @@ -852,9 +850,7 @@ func (x *AddByPathRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendString(4, x.Path[j]) } for i := range x.Meta { - if x.Meta[i] != nil { - x.Meta[i].EmitProtobuf(mm.AppendMessage(5)) - } + x.Meta[i].EmitProtobuf(mm.AppendMessage(5)) } if len(x.BearerToken) != 0 { mm.AppendBytes(6, x.BearerToken) @@ -899,8 +895,8 @@ func (x *AddByPathRequest_Body) UnmarshalProtobuf(src []byte) (err error) { if !ok { return fmt.Errorf("cannot unmarshal field %s", "Meta") } - x.Meta = append(x.Meta, new(KeyValue)) - ff := x.Meta[len(x.Meta)-1] + x.Meta = append(x.Meta, KeyValue{}) + ff := &x.Meta[len(x.Meta)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -950,13 +946,13 @@ func (x *AddByPathRequest_Body) GetPath() []string { func (x *AddByPathRequest_Body) SetPath(v []string) { x.Path = v } -func (x *AddByPathRequest_Body) GetMeta() []*KeyValue { +func (x *AddByPathRequest_Body) GetMeta() []KeyValue { if x != nil { return x.Meta } return nil } -func (x *AddByPathRequest_Body) SetMeta(v []*KeyValue) { +func (x *AddByPathRequest_Body) SetMeta(v []KeyValue) { x.Meta = v } func (x *AddByPathRequest_Body) GetBearerToken() []byte { @@ -1086,11 +1082,11 @@ func (x *AddByPathRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { } case "meta": { - var f *KeyValue - var list []*KeyValue + var f KeyValue + var list []KeyValue in.Delim('[') for !in.IsDelim(']') { - f = new(KeyValue) + f = KeyValue{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() @@ -2351,12 +2347,12 @@ func (x *RemoveResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { } type MoveRequest_Body struct { - ContainerId []byte `json:"containerId"` - TreeId string `json:"treeId"` - ParentId uint64 `json:"parentId"` - NodeId uint64 `json:"nodeId"` - Meta []*KeyValue `json:"meta"` - BearerToken []byte `json:"bearerToken"` + ContainerId []byte `json:"containerId"` + TreeId string `json:"treeId"` + ParentId uint64 `json:"parentId"` + NodeId uint64 `json:"nodeId"` + Meta []KeyValue `json:"meta"` + BearerToken []byte `json:"bearerToken"` } var ( @@ -2378,7 +2374,7 @@ func (x *MoveRequest_Body) StableSize() (size int) { size += proto.UInt64Size(3, x.ParentId) size += proto.UInt64Size(4, x.NodeId) for i := range x.Meta { - size += proto.NestedStructureSize(5, x.Meta[i]) + size += proto.NestedStructureSizeUnchecked(5, &x.Meta[i]) } size += proto.BytesSize(6, x.BearerToken) return size @@ -2410,9 +2406,7 @@ func (x *MoveRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendUint64(4, x.NodeId) } for i := range x.Meta { - if x.Meta[i] != nil { - x.Meta[i].EmitProtobuf(mm.AppendMessage(5)) - } + x.Meta[i].EmitProtobuf(mm.AppendMessage(5)) } if len(x.BearerToken) != 0 { mm.AppendBytes(6, x.BearerToken) @@ -2457,8 +2451,8 @@ func (x *MoveRequest_Body) UnmarshalProtobuf(src []byte) (err error) { if !ok { return fmt.Errorf("cannot unmarshal field %s", "Meta") } - x.Meta = append(x.Meta, new(KeyValue)) - ff := x.Meta[len(x.Meta)-1] + x.Meta = append(x.Meta, KeyValue{}) + ff := &x.Meta[len(x.Meta)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -2508,13 +2502,13 @@ func (x *MoveRequest_Body) GetNodeId() uint64 { func (x *MoveRequest_Body) SetNodeId(v uint64) { x.NodeId = v } -func (x *MoveRequest_Body) GetMeta() []*KeyValue { +func (x *MoveRequest_Body) GetMeta() []KeyValue { if x != nil { return x.Meta } return nil } -func (x *MoveRequest_Body) SetMeta(v []*KeyValue) { +func (x *MoveRequest_Body) SetMeta(v []KeyValue) { x.Meta = v } func (x *MoveRequest_Body) GetBearerToken() []byte { @@ -2630,11 +2624,11 @@ func (x *MoveRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { } case "meta": { - var f *KeyValue - var list []*KeyValue + var f KeyValue + var list []KeyValue in.Delim('[') for !in.IsDelim(']') { - f = new(KeyValue) + f = KeyValue{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() @@ -3685,10 +3679,10 @@ func (x *GetNodeByPathRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { } type GetNodeByPathResponse_Info struct { - NodeId uint64 `json:"nodeId"` - Timestamp uint64 `json:"timestamp"` - Meta []*KeyValue `json:"meta"` - ParentId uint64 `json:"parentId"` + NodeId uint64 `json:"nodeId"` + Timestamp uint64 `json:"timestamp"` + Meta []KeyValue `json:"meta"` + ParentId uint64 `json:"parentId"` } var ( @@ -3708,7 +3702,7 @@ func (x *GetNodeByPathResponse_Info) StableSize() (size int) { size += proto.UInt64Size(1, x.NodeId) size += proto.UInt64Size(2, x.Timestamp) for i := range x.Meta { - size += proto.NestedStructureSize(3, x.Meta[i]) + size += proto.NestedStructureSizeUnchecked(3, &x.Meta[i]) } size += proto.UInt64Size(4, x.ParentId) return size @@ -3734,9 +3728,7 @@ func (x *GetNodeByPathResponse_Info) EmitProtobuf(mm *easyproto.MessageMarshaler mm.AppendUint64(2, x.Timestamp) } for i := range x.Meta { - if x.Meta[i] != nil { - x.Meta[i].EmitProtobuf(mm.AppendMessage(3)) - } + x.Meta[i].EmitProtobuf(mm.AppendMessage(3)) } if x.ParentId != 0 { mm.AppendUint64(4, x.ParentId) @@ -3769,8 +3761,8 @@ func (x *GetNodeByPathResponse_Info) UnmarshalProtobuf(src []byte) (err error) { if !ok { return fmt.Errorf("cannot unmarshal field %s", "Meta") } - x.Meta = append(x.Meta, new(KeyValue)) - ff := x.Meta[len(x.Meta)-1] + x.Meta = append(x.Meta, KeyValue{}) + ff := &x.Meta[len(x.Meta)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -3802,13 +3794,13 @@ func (x *GetNodeByPathResponse_Info) GetTimestamp() uint64 { func (x *GetNodeByPathResponse_Info) SetTimestamp(v uint64) { x.Timestamp = v } -func (x *GetNodeByPathResponse_Info) GetMeta() []*KeyValue { +func (x *GetNodeByPathResponse_Info) GetMeta() []KeyValue { if x != nil { return x.Meta } return nil } -func (x *GetNodeByPathResponse_Info) SetMeta(v []*KeyValue) { +func (x *GetNodeByPathResponse_Info) SetMeta(v []KeyValue) { x.Meta = v } func (x *GetNodeByPathResponse_Info) GetParentId() uint64 { @@ -3902,11 +3894,11 @@ func (x *GetNodeByPathResponse_Info) UnmarshalEasyJSON(in *jlexer.Lexer) { } case "meta": { - var f *KeyValue - var list []*KeyValue + var f KeyValue + var list []KeyValue in.Delim('[') for !in.IsDelim(']') { - f = new(KeyValue) + f = KeyValue{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() @@ -3930,7 +3922,7 @@ func (x *GetNodeByPathResponse_Info) UnmarshalEasyJSON(in *jlexer.Lexer) { } type GetNodeByPathResponse_Body struct { - Nodes []*GetNodeByPathResponse_Info `json:"nodes"` + Nodes []GetNodeByPathResponse_Info `json:"nodes"` } var ( @@ -3948,7 +3940,7 @@ func (x *GetNodeByPathResponse_Body) StableSize() (size int) { return 0 } for i := range x.Nodes { - size += proto.NestedStructureSize(1, x.Nodes[i]) + size += proto.NestedStructureSizeUnchecked(1, &x.Nodes[i]) } return size } @@ -3967,9 +3959,7 @@ func (x *GetNodeByPathResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler return } for i := range x.Nodes { - if x.Nodes[i] != nil { - x.Nodes[i].EmitProtobuf(mm.AppendMessage(1)) - } + x.Nodes[i].EmitProtobuf(mm.AppendMessage(1)) } } @@ -3987,8 +3977,8 @@ func (x *GetNodeByPathResponse_Body) UnmarshalProtobuf(src []byte) (err error) { if !ok { return fmt.Errorf("cannot unmarshal field %s", "Nodes") } - x.Nodes = append(x.Nodes, new(GetNodeByPathResponse_Info)) - ff := x.Nodes[len(x.Nodes)-1] + x.Nodes = append(x.Nodes, GetNodeByPathResponse_Info{}) + ff := &x.Nodes[len(x.Nodes)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -3996,13 +3986,13 @@ func (x *GetNodeByPathResponse_Body) UnmarshalProtobuf(src []byte) (err error) { } return nil } -func (x *GetNodeByPathResponse_Body) GetNodes() []*GetNodeByPathResponse_Info { +func (x *GetNodeByPathResponse_Body) GetNodes() []GetNodeByPathResponse_Info { if x != nil { return x.Nodes } return nil } -func (x *GetNodeByPathResponse_Body) SetNodes(v []*GetNodeByPathResponse_Info) { +func (x *GetNodeByPathResponse_Body) SetNodes(v []GetNodeByPathResponse_Info) { x.Nodes = v } @@ -4060,11 +4050,11 @@ func (x *GetNodeByPathResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { switch key { case "nodes": { - var f *GetNodeByPathResponse_Info - var list []*GetNodeByPathResponse_Info + var f GetNodeByPathResponse_Info + var list []GetNodeByPathResponse_Info in.Delim('[') for !in.IsDelim(']') { - f = new(GetNodeByPathResponse_Info) + f = GetNodeByPathResponse_Info{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() @@ -4935,10 +4925,10 @@ func (x *GetSubTreeRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { } type GetSubTreeResponse_Body struct { - NodeId []uint64 `json:"nodeId"` - ParentId []uint64 `json:"parentId"` - Timestamp []uint64 `json:"timestamp"` - Meta []*KeyValue `json:"meta"` + NodeId []uint64 `json:"nodeId"` + ParentId []uint64 `json:"parentId"` + Timestamp []uint64 `json:"timestamp"` + Meta []KeyValue `json:"meta"` } var ( @@ -4965,7 +4955,7 @@ func (x *GetSubTreeResponse_Body) StableSize() (size int) { size += protowire.SizeGroup(protowire.Number(3), protowire.SizeVarint(x.Timestamp[i])) } for i := range x.Meta { - size += proto.NestedStructureSize(4, x.Meta[i]) + size += proto.NestedStructureSizeUnchecked(4, &x.Meta[i]) } return size } @@ -4993,9 +4983,7 @@ func (x *GetSubTreeResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { mm.AppendUint64(3, x.Timestamp[j]) } for i := range x.Meta { - if x.Meta[i] != nil { - x.Meta[i].EmitProtobuf(mm.AppendMessage(4)) - } + x.Meta[i].EmitProtobuf(mm.AppendMessage(4)) } } @@ -5031,8 +5019,8 @@ func (x *GetSubTreeResponse_Body) UnmarshalProtobuf(src []byte) (err error) { if !ok { return fmt.Errorf("cannot unmarshal field %s", "Meta") } - x.Meta = append(x.Meta, new(KeyValue)) - ff := x.Meta[len(x.Meta)-1] + x.Meta = append(x.Meta, KeyValue{}) + ff := &x.Meta[len(x.Meta)-1] if err := ff.UnmarshalProtobuf(data); err != nil { return fmt.Errorf("unmarshal: %w", err) } @@ -5067,13 +5055,13 @@ func (x *GetSubTreeResponse_Body) GetTimestamp() []uint64 { func (x *GetSubTreeResponse_Body) SetTimestamp(v []uint64) { x.Timestamp = v } -func (x *GetSubTreeResponse_Body) GetMeta() []*KeyValue { +func (x *GetSubTreeResponse_Body) GetMeta() []KeyValue { if x != nil { return x.Meta } return nil } -func (x *GetSubTreeResponse_Body) SetMeta(v []*KeyValue) { +func (x *GetSubTreeResponse_Body) SetMeta(v []KeyValue) { x.Meta = v } @@ -5206,11 +5194,11 @@ func (x *GetSubTreeResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { } case "meta": { - var f *KeyValue - var list []*KeyValue + var f KeyValue + var list []KeyValue in.Delim('[') for !in.IsDelim(']') { - f = new(KeyValue) + f = KeyValue{} f.UnmarshalEasyJSON(in) list = append(list, f) in.WantComma() diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index ce5039f7c..3c3ebfe89 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -123,7 +123,7 @@ func TestMessageSign(t *testing.T) { ContainerId: rawCID1, ParentId: 1, NodeId: 2, - Meta: []*KeyValue{ + Meta: []KeyValue{ {Key: "kkk", Value: []byte("vvv")}, }, }, From 2b3fc50681f7e463a74c6075ad16788fc52cfa3f Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 30 Aug 2024 14:02:57 +0300 Subject: [PATCH 1309/1943] [#1320] shard: Fix TestGCDropsObjectInhumedFromWritecache flaky test The `TestGCDropsObjectInhumedFromWritecache` test was flaky because a running asynchronous rebuild operation prevented GC from deleting the object. A test-only shard option `WithDisabledRebuild` has been added to fix this. Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/shard/control.go | 4 ++-- pkg/local_object_storage/shard/gc_test.go | 2 +- pkg/local_object_storage/shard/shard.go | 10 ++++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 1626d5804..6efe4ec37 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -130,7 +130,7 @@ func (s *Shard) Init(ctx context.Context) error { s.gc.init(ctx) s.rb = newRebuilder(s.rebuildLimiter) - if !m.NoMetabase() { + if !m.NoMetabase() && !s.rebuildDisabled { s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) } s.writecacheSealCancel.Store(dummyCancel) @@ -398,7 +398,7 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { defer unlock() s.rb.Stop(s.log) - if !s.info.Mode.NoMetabase() { + if !s.info.Mode.NoMetabase() && !s.rebuildDisabled { defer func() { s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) }() diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 90958cd35..1c0ef1c2e 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -187,7 +187,7 @@ func TestGCDropsObjectInhumedFromWritecache(t *testing.T) { func testGCDropsObjectInhumedFromWritecache(t *testing.T, flushbeforeInhume bool) { sh := newCustomShard(t, true, shardOptions{ - additionalShardOptions: []Option{WithDisabledGC()}, + additionalShardOptions: []Option{WithDisabledGC(), WithDisabledRebuild()}, wcOpts: []writecache.Option{writecache.WithDisableBackgroundFlush()}, }) defer func() { require.NoError(t, sh.Close()) }() diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 93f5354a7..d11bcc36b 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -139,6 +139,8 @@ type cfg struct { reportErrorFunc func(selfID string, message string, err error) rebuildLimiter RebuildWorkerLimiter + + rebuildDisabled bool } func defaultCfg() *cfg { @@ -410,6 +412,14 @@ func WithZeroCountCallback(cb EmptyContainersCallback) Option { } } +// WithDisabledRebuild returns an option to disable a shard rebuild. +// For testing purposes only. +func WithDisabledRebuild() Option { + return func(c *cfg) { + c.rebuildDisabled = true + } +} + func (s *Shard) fillInfo() { s.cfg.info.MetaBaseInfo = s.metaBase.DumpInfo() s.cfg.info.BlobStorInfo = s.blobStor.DumpInfo() From a685fcdc963b0f58003059bb2dae2d21c925e25a Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 30 Aug 2024 19:20:55 +0300 Subject: [PATCH 1310/1943] [#1317] go.mod: Use range over int Since Go 1.22 a "for" statement with a "range" clause is able to iterate through integer values from zero to an upper limit. gopatch script: @@ var i, e expression @@ -for i := 0; i <= e - 1; i++ { +for i := range e { ... } @@ var i, e expression @@ -for i := 0; i <= e; i++ { +for i := range e + 1 { ... } @@ var i, e expression @@ -for i := 0; i < e; i++ { +for i := range e { ... } Signed-off-by: Ekaterina Lebedeva --- .../modules/morph/contract/dump_hashes.go | 4 ++-- .../modules/morph/helper/local_client.go | 2 +- .../internal/modules/morph/helper/util.go | 2 +- .../morph/initialize/initialize_test.go | 4 ++-- cmd/frostfs-cli/internal/client/client.go | 2 +- cmd/frostfs-cli/modules/container/create.go | 2 +- cmd/frostfs-cli/modules/container/delete.go | 2 +- cmd/frostfs-cli/modules/object/nodes.go | 2 +- cmd/frostfs-cli/modules/util/acl.go | 2 +- cmd/frostfs-node/config/node/config.go | 2 +- cmd/frostfs-node/morph.go | 2 +- pkg/innerring/indexer_test.go | 2 +- pkg/innerring/notary.go | 2 +- .../processors/alphabet/handlers_test.go | 6 ++--- .../processors/governance/handlers_test.go | 2 +- .../processors/governance/list_test.go | 4 ++-- .../blobovnicza/sizes_test.go | 2 +- .../blobovniczatree/concurrency_test.go | 2 +- .../blobstor/blobovniczatree/rebuild_test.go | 2 +- .../blobstor/blobstor_test.go | 12 +++++----- .../blobstor/compression/bench_test.go | 2 +- .../blobstor/fstree/fstree_test.go | 6 ++--- .../blobstor/perf_test.go | 4 ++-- .../engine/control_test.go | 4 ++-- .../engine/engine_test.go | 12 +++++----- .../engine/evacuate_test.go | 4 ++-- pkg/local_object_storage/engine/list_test.go | 2 +- .../engine/remove_copies.go | 2 +- .../engine/remove_copies_test.go | 4 ++-- .../engine/shards_test.go | 2 +- pkg/local_object_storage/engine/tree_test.go | 6 ++--- .../internal/testutil/generators_test.go | 6 ++--- .../metabase/containers_test.go | 8 +++---- .../metabase/counter_test.go | 8 +++---- .../metabase/delete_test.go | 2 +- pkg/local_object_storage/metabase/get_test.go | 4 ++-- .../metabase/list_test.go | 12 +++++----- .../metabase/lock_test.go | 4 ++-- pkg/local_object_storage/metabase/put_test.go | 2 +- .../metabase/reset_test.go | 2 +- .../metabase/select_test.go | 6 ++--- pkg/local_object_storage/metabase/upgrade.go | 2 +- .../metabase/upgrade_test.go | 10 ++++---- pkg/local_object_storage/pilorama/boltdb.go | 2 +- .../pilorama/forest_test.go | 24 +++++++++---------- pkg/local_object_storage/shard/list_test.go | 4 ++-- .../shard/metrics_test.go | 10 ++++---- pkg/local_object_storage/shard/refill_test.go | 4 ++-- .../writecache/benchmark/writecache_test.go | 2 +- pkg/local_object_storage/writecache/flush.go | 2 +- pkg/morph/event/notary_preparator_test.go | 2 +- pkg/morph/timer/block_test.go | 2 +- pkg/network/tls_test.go | 2 +- pkg/services/control/server/evacuate.go | 2 +- pkg/services/object/acl/v2/util_test.go | 2 +- pkg/services/object/get/get_test.go | 8 +++---- pkg/services/object/get/getrangeec_test.go | 2 +- pkg/services/object/put/ec.go | 2 +- pkg/services/object/search/search_test.go | 4 ++-- .../object_manager/placement/cache_test.go | 4 ++-- .../object_manager/placement/traverser.go | 4 ++-- .../placement/traverser_test.go | 6 ++--- .../storage/persistent/executor_test.go | 2 +- pkg/services/tree/getsubtree_test.go | 2 +- pkg/services/tree/replicator.go | 2 +- pkg/util/sync/key_locker_test.go | 2 +- 66 files changed, 135 insertions(+), 135 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index 5a0d29550..be2134b77 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -68,7 +68,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { if irSize != 0 { bw.Reset() - for i := 0; i < irSize; i++ { + for i := range irSize { emit.AppCall(bw.BinWriter, cs.Hash, "resolve", callflag.ReadOnly, helper.GetAlphabetNNSDomain(i), int64(nns.TXT)) @@ -79,7 +79,7 @@ func dumpContractHashes(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't fetch info from NNS: %w", err) } - for i := 0; i < irSize; i++ { + for i := range irSize { info := contractDumpInfo{name: fmt.Sprintf("alphabet %d", i)} if h, err := helper.ParseNNSResolveResult(alphaRes.Stack[i]); err == nil { info.hash = h diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go index 44d1b4ecf..375fa84d7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go @@ -224,7 +224,7 @@ func (l *LocalClient) CalculateNetworkFee(tx *transaction.Transaction) (int64, e paramz = []manifest.Parameter{{Type: smartcontract.SignatureType}} } else if nSigs, _, ok := vm.ParseMultiSigContract(w.VerificationScript); ok { paramz = make([]manifest.Parameter, nSigs) - for j := 0; j < nSigs; j++ { + for j := range nSigs { paramz[j] = manifest.Parameter{Type: smartcontract.SignatureType} } } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/util.go b/cmd/frostfs-adm/internal/modules/morph/helper/util.go index 2d9281c24..8c6b90539 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/util.go @@ -44,7 +44,7 @@ func openAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, er var wallets []*wallet.Wallet var letter string - for i := 0; i < constants.MaxAlphabetNodes; i++ { + for i := range constants.MaxAlphabetNodes { letter = innerring.GlagoliticLetter(i).String() p := filepath.Join(walletDir, letter+".json") var w *wallet.Wallet diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go index 6c52aa2ab..74f5d3e88 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go @@ -113,7 +113,7 @@ func generateTestData(dir string, size int) error { } var pubs []string - for i := 0; i < size; i++ { + for i := range size { p := filepath.Join(dir, innerring.GlagoliticLetter(i).String()+".json") w, err := wallet.NewWalletFromFile(p) if err != nil { @@ -148,7 +148,7 @@ func generateTestData(dir string, size int) error { } func setTestCredentials(v *viper.Viper, size int) { - for i := 0; i < size; i++ { + for i := range size { v.Set("credentials."+innerring.GlagoliticLetter(i).String(), strconv.FormatUint(uint64(i), 10)) } v.Set("credentials.contract", constants.TestContractPassword) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index a0fa22410..57bcf5620 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -692,7 +692,7 @@ func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes for { n, ok = rdr.Read(buf) - for i := 0; i < n; i++ { + for i := range n { list = append(list, buf[i]) } if !ok { diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index c6f576908..f37b169ce 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -139,7 +139,7 @@ It will be stored in sidechain when inner ring will accepts it.`, }, } - for i := 0; i < awaitTimeout; i++ { + for range awaitTimeout { time.Sleep(1 * time.Second) _, err := internalclient.GetContainer(cmd.Context(), getPrm) diff --git a/cmd/frostfs-cli/modules/container/delete.go b/cmd/frostfs-cli/modules/container/delete.go index e5425bf25..c20188884 100644 --- a/cmd/frostfs-cli/modules/container/delete.go +++ b/cmd/frostfs-cli/modules/container/delete.go @@ -110,7 +110,7 @@ Only owner of the container has a permission to remove container.`, }, } - for i := 0; i < awaitTimeout; i++ { + for range awaitTimeout { time.Sleep(1 * time.Second) _, err := internalclient.GetContainer(cmd.Context(), getPrm) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 896f6f17f..0eac4e6d2 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -195,7 +195,7 @@ func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, member prmHead.SetRawFlag(true) // to get an error instead of whole object eg, egCtx := errgroup.WithContext(cmd.Context()) - for idx := 0; idx < len(members); idx++ { + for idx := range len(members) { partObjID := members[idx] eg.Go(func() error { diff --git a/cmd/frostfs-cli/modules/util/acl.go b/cmd/frostfs-cli/modules/util/acl.go index 4c2e324b3..145dcc756 100644 --- a/cmd/frostfs-cli/modules/util/acl.go +++ b/cmd/frostfs-cli/modules/util/acl.go @@ -33,7 +33,7 @@ func PrettyPrintTableBACL(cmd *cobra.Command, bacl *acl.Basic) { fmt.Fprintln(w, strings.Join(bits, "\t")) // Footer footer := []string{"X F"} - for i := 0; i < 7; i++ { + for range 7 { footer = append(footer, "U S O B") } fmt.Fprintln(w, strings.Join(footer, "\t")) diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index 97aca274a..4d063245b 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -121,7 +121,7 @@ func BootstrapAddresses(c *config.Config) (addr network.AddressGroup) { func Attributes(c *config.Config) (attrs []string) { const maxAttributes = 100 - for i := 0; i < maxAttributes; i++ { + for i := range maxAttributes { attr := config.StringSafe(c.Sub(subsection), attributePrefix+"_"+strconv.Itoa(i)) if attr == "" { return diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 1b148095b..7178cd97d 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -157,7 +157,7 @@ var ( ) func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error { - for i := 0; i < notaryDepositRetriesAmount; i++ { + for range notaryDepositRetriesAmount { c.log.Debug(logs.ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted) select { case <-ctx.Done(): diff --git a/pkg/innerring/indexer_test.go b/pkg/innerring/indexer_test.go index 5bc2cc988..c8a819b5b 100644 --- a/pkg/innerring/indexer_test.go +++ b/pkg/innerring/indexer_test.go @@ -237,7 +237,7 @@ func BenchmarkKeyPosition(b *testing.B) { b.ResetTimer() b.ReportAllocs() - for i := 0; i < b.N; i++ { + for range b.N { if keyPosition(key, list) != 5 { b.FailNow() } diff --git a/pkg/innerring/notary.go b/pkg/innerring/notary.go index c601f5587..e6f2b1de4 100644 --- a/pkg/innerring/notary.go +++ b/pkg/innerring/notary.go @@ -90,7 +90,7 @@ func (s *Server) initNotary(ctx context.Context, deposit depositor, await awaite } func awaitNotaryDepositInClient(ctx context.Context, cli *client.Client, txHash util.Uint256) error { - for i := 0; i < notaryDepositTimeout; i++ { + for range notaryDepositTimeout { select { case <-ctx.Done(): return ctx.Err() diff --git a/pkg/innerring/processors/alphabet/handlers_test.go b/pkg/innerring/processors/alphabet/handlers_test.go index 346901949..dfda37472 100644 --- a/pkg/innerring/processors/alphabet/handlers_test.go +++ b/pkg/innerring/processors/alphabet/handlers_test.go @@ -21,7 +21,7 @@ func TestProcessorEmitsGasToNetmapAndAlphabet(t *testing.T) { var parsedWallets []util.Uint160 = []util.Uint160{{20}, {25}} alphabetContracts := innerring.NewAlphabetContracts() - for i := 0; i <= index; i++ { + for i := range index + 1 { alphabetContracts[innerring.GlagoliticLetter(i)] = util.Uint160{uint8(i)} } @@ -98,7 +98,7 @@ func TestProcessorEmitsGasToNetmapIfNoParsedWallets(t *testing.T) { var parsedWallets []util.Uint160 = []util.Uint160{} alphabetContracts := innerring.NewAlphabetContracts() - for i := 0; i <= index; i++ { + for i := range index + 1 { alphabetContracts[innerring.GlagoliticLetter(i)] = util.Uint160{uint8(i)} } @@ -170,7 +170,7 @@ func TestProcessorDoesntEmitGasIfNoNetmapOrParsedWallets(t *testing.T) { var parsedWallets []util.Uint160 = []util.Uint160{} alphabetContracts := innerring.NewAlphabetContracts() - for i := 0; i <= index; i++ { + for i := range index + 1 { alphabetContracts[innerring.GlagoliticLetter(i)] = util.Uint160{uint8(i)} } diff --git a/pkg/innerring/processors/governance/handlers_test.go b/pkg/innerring/processors/governance/handlers_test.go index 2a505f8d1..b73e24318 100644 --- a/pkg/innerring/processors/governance/handlers_test.go +++ b/pkg/innerring/processors/governance/handlers_test.go @@ -190,7 +190,7 @@ func generateTestKeys(t *testing.T) testKeys { for { var result testKeys - for i := 0; i < 4; i++ { + for range 4 { pk, err := keys.NewPrivateKey() require.NoError(t, err, "failed to create private key") result.sidechainKeys = append(result.sidechainKeys, pk.PublicKey()) diff --git a/pkg/innerring/processors/governance/list_test.go b/pkg/innerring/processors/governance/list_test.go index d099ec837..4ecebf05b 100644 --- a/pkg/innerring/processors/governance/list_test.go +++ b/pkg/innerring/processors/governance/list_test.go @@ -49,7 +49,7 @@ func TestNewAlphabetList(t *testing.T) { } ln := len(rounds) - for i := 0; i < ln; i++ { + for i := range ln { list, err = newAlphabetList(list, exp) require.NoError(t, err) require.True(t, equalPublicKeyLists(list, rounds[i])) @@ -131,7 +131,7 @@ func TestUpdateInnerRing(t *testing.T) { func generateKeys(n int) (keys.PublicKeys, error) { pubKeys := make(keys.PublicKeys, 0, n) - for i := 0; i < n; i++ { + for range n { privKey, err := keys.NewPrivateKey() if err != nil { return nil, err diff --git a/pkg/local_object_storage/blobovnicza/sizes_test.go b/pkg/local_object_storage/blobovnicza/sizes_test.go index 01093b8d7..d582fc5e4 100644 --- a/pkg/local_object_storage/blobovnicza/sizes_test.go +++ b/pkg/local_object_storage/blobovnicza/sizes_test.go @@ -42,7 +42,7 @@ func TestSizes(t *testing.T) { func BenchmarkUpperBound(b *testing.B) { for _, size := range []uint64{1, 1023, 65 * 1024, 40 * 1024 * 1024} { b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { _ = upperPowerOfTwo(size) } }) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go index 5bed86142..cc8a52d03 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go @@ -34,7 +34,7 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) { var cnt atomic.Int64 var wg sync.WaitGroup - for i := 0; i < 1000; i++ { + for range 1000 { wg.Add(1) go func() { defer wg.Done() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 7a1de4c13..4a51fd86a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -127,7 +127,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta eg, egCtx := errgroup.WithContext(context.Background()) storageIDs := make(map[oid.Address][]byte) storageIDsGuard := &sync.Mutex{} - for i := 0; i < 100; i++ { + for range 100 { eg.Go(func() error { obj := blobstortest.NewObject(1024) data, err := obj.Marshal() diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index c7d80dc84..bed5e0eb9 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -60,7 +60,7 @@ func TestCompression(t *testing.T) { bigObj := make([]*objectSDK.Object, objCount) smallObj := make([]*objectSDK.Object, objCount) - for i := 0; i < objCount; i++ { + for i := range objCount { bigObj[i] = testObject(smallSizeLimit * 2) smallObj[i] = testObject(smallSizeLimit / 2) } @@ -219,7 +219,7 @@ func TestConcurrentPut(t *testing.T) { bigObj := testObject(smallSizeLimit * 2) var wg sync.WaitGroup - for i := 0; i < concurrentPutCount; i++ { + for range concurrentPutCount { wg.Add(1) go func() { testPut(t, blobStor, bigObj) @@ -235,7 +235,7 @@ func TestConcurrentPut(t *testing.T) { bigObj := testObject(smallSizeLimit * 2) var wg sync.WaitGroup - for i := 0; i < concurrentPutCount+1; i++ { + for range concurrentPutCount + 1 { wg.Add(1) go func() { testPutFileExistsError(t, blobStor, bigObj) @@ -251,7 +251,7 @@ func TestConcurrentPut(t *testing.T) { smallObj := testObject(smallSizeLimit / 2) var wg sync.WaitGroup - for i := 0; i < concurrentPutCount; i++ { + for range concurrentPutCount { wg.Add(1) go func() { testPut(t, blobStor, smallObj) @@ -302,7 +302,7 @@ func TestConcurrentDelete(t *testing.T) { testPut(t, blobStor, bigObj) var wg sync.WaitGroup - for i := 0; i < 2; i++ { + for range 2 { wg.Add(1) go func() { testDelete(t, blobStor, bigObj) @@ -319,7 +319,7 @@ func TestConcurrentDelete(t *testing.T) { testPut(t, blobStor, smallObj) var wg sync.WaitGroup - for i := 0; i < 2; i++ { + for range 2 { wg.Add(1) go func() { testDelete(t, blobStor, smallObj) diff --git a/pkg/local_object_storage/blobstor/compression/bench_test.go b/pkg/local_object_storage/blobstor/compression/bench_test.go index 986912985..9f70f8ec2 100644 --- a/pkg/local_object_storage/blobstor/compression/bench_test.go +++ b/pkg/local_object_storage/blobstor/compression/bench_test.go @@ -36,7 +36,7 @@ func BenchmarkCompression(b *testing.B) { func benchWith(b *testing.B, c Config, data []byte) { b.ResetTimer() b.ReportAllocs() - for i := 0; i < b.N; i++ { + for range b.N { _ = c.Compress(data) } } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_test.go b/pkg/local_object_storage/blobstor/fstree/fstree_test.go index d633cbac3..5786dfd3b 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_test.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_test.go @@ -28,7 +28,7 @@ func Benchmark_addressFromString(b *testing.B) { b.ReportAllocs() b.ResetTimer() - for i := 0; i < b.N; i++ { + for range b.N { _, err := addressFromString(s) if err != nil { b.Fatalf("benchmark error: %v", err) @@ -73,7 +73,7 @@ func TestObjectCounter(t *testing.T) { eg, egCtx := errgroup.WithContext(context.Background()) eg.Go(func() error { - for j := 0; j < 1_000; j++ { + for range 1_000 { _, err := fst.Put(egCtx, putPrm) if err != nil { return err @@ -84,7 +84,7 @@ func TestObjectCounter(t *testing.T) { eg.Go(func() error { var le logicerr.Logical - for j := 0; j < 1_000; j++ { + for range 1_000 { _, err := fst.Delete(egCtx, delPrm) if err != nil && !errors.As(err, &le) { return err diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index 501c95a1d..1ac769e36 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -110,7 +110,7 @@ func BenchmarkSubstorageReadPerf(b *testing.B) { // Fill database var errG errgroup.Group - for i := 0; i < tt.size; i++ { + for range tt.size { obj := objGen.Next() addr := testutil.AddressFromObject(b, obj) errG.Go(func() error { @@ -203,7 +203,7 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) { defer func() { require.NoError(b, st.Close()) }() // Fill database - for i := 0; i < tt.size; i++ { + for range tt.size { obj := objGen.Next() addr := testutil.AddressFromObject(b, obj) raw, err := obj.Marshal() diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index f0809883c..2de92ae84 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -208,7 +208,7 @@ func TestPersistentShardID(t *testing.T) { require.NoError(t, te.ng.Close(context.Background())) newTe := newEngineWithErrorThreshold(t, dir, 1) - for i := 0; i < len(newTe.shards); i++ { + for i := range len(newTe.shards) { require.Equal(t, te.shards[i].id, newTe.shards[i].id) } require.NoError(t, newTe.ng.Close(context.Background())) @@ -269,7 +269,7 @@ func TestReload(t *testing.T) { e, currShards := engineWithShards(t, removePath, shardNum) var rcfg ReConfiguration - for i := 0; i < len(currShards)-1; i++ { // without one of the shards + for i := range len(currShards) - 1 { // without one of the shards rcfg.AddShard(currShards[i], nil) } diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 49976abbb..525e17f34 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -44,7 +44,7 @@ func BenchmarkExists(b *testing.B) { func benchmarkExists(b *testing.B, shardNum int) { shards := make([]*shard.Shard, shardNum) - for i := 0; i < shardNum; i++ { + for i := range shardNum { shards[i] = testNewShard(b) } @@ -52,7 +52,7 @@ func benchmarkExists(b *testing.B, shardNum int) { defer func() { require.NoError(b, e.Close(context.Background())) }() addr := oidtest.Address() - for i := 0; i < 100; i++ { + for range 100 { obj := testutil.GenerateObjectWithCID(cidtest.ID()) err := Put(context.Background(), e, obj) if err != nil { @@ -62,7 +62,7 @@ func benchmarkExists(b *testing.B, shardNum int) { b.ReportAllocs() b.ResetTimer() - for i := 0; i < b.N; i++ { + for range b.N { var shPrm shard.ExistsPrm shPrm.Address = addr shPrm.ParentAddress = oid.Address{} @@ -109,7 +109,7 @@ func (te *testEngineWrapper) setInitializedShards(t testing.TB, shards ...*shard func (te *testEngineWrapper) setShardsNum(t testing.TB, num int) *testEngineWrapper { shards := make([]*shard.Shard, 0, num) - for i := 0; i < num; i++ { + for range num { shards = append(shards, testNewShard(t)) } @@ -117,7 +117,7 @@ func (te *testEngineWrapper) setShardsNum(t testing.TB, num int) *testEngineWrap } func (te *testEngineWrapper) setShardsNumOpts(t testing.TB, num int, shardOpts func(id int) []shard.Option) *testEngineWrapper { - for i := 0; i < num; i++ { + for i := range num { opts := shardOpts(i) id, err := te.engine.AddShard(context.Background(), opts...) require.NoError(t, err) @@ -127,7 +127,7 @@ func (te *testEngineWrapper) setShardsNumOpts(t testing.TB, num int, shardOpts f } func (te *testEngineWrapper) setShardsNumAdditionalOpts(t testing.TB, num int, shardOpts func(id int) []shard.Option) *testEngineWrapper { - for i := 0; i < num; i++ { + for i := range num { defaultOpts := testDefaultShardOptions(t) opts := append(defaultOpts, shardOpts(i)...) id, err := te.engine.AddShard(context.Background(), opts...) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 55268b549..8d25dad4a 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -61,7 +61,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng } for _, sh := range ids { - for i := 0; i < objPerShard; i++ { + for range objPerShard { contID := cidtest.ID() obj := testutil.GenerateObjectWithCID(contID) objects = append(objects, obj) @@ -554,7 +554,7 @@ func TestEvacuateTreesRemote(t *testing.T) { require.Equal(t, "", st.ErrorMessage(), "invalid final error message") expectedTreeOps := make(map[string][]*pilorama.Move) - for i := 0; i < len(e.shards); i++ { + for i := range len(e.shards) { sourceTrees, err := pilorama.TreeListAll(context.Background(), e.shards[ids[i].String()]) require.NoError(t, err, "list source trees failed") require.Len(t, sourceTrees, 3) diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index dd8a2e8a0..11a6c7841 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -79,7 +79,7 @@ func TestListWithCursor(t *testing.T) { expected := make([]object.Info, 0, tt.objectNum) got := make([]object.Info, 0, tt.objectNum) - for i := 0; i < tt.objectNum; i++ { + for range tt.objectNum { containerID := cidtest.ID() obj := testutil.GenerateObjectWithCIDWithPayload(containerID, []byte{'a'}) diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index b99cf4f44..5e1ced56a 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -87,7 +87,7 @@ func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicat } }) - for i := 0; i < prm.Concurrency; i++ { + for range prm.Concurrency { errG.Go(func() error { return e.removeObjects(ctx, ch) }) diff --git a/pkg/local_object_storage/engine/remove_copies_test.go b/pkg/local_object_storage/engine/remove_copies_test.go index 99963576c..6d2291c74 100644 --- a/pkg/local_object_storage/engine/remove_copies_test.go +++ b/pkg/local_object_storage/engine/remove_copies_test.go @@ -96,7 +96,7 @@ loop: require.FailNow(t, "unexpected object was removed", removed[i].addr) } - for i := 0; i < copyCount; i++ { + for i := range copyCount { if i%3 == 0 { require.True(t, removedMask[i], "object %d was expected to be removed", i) } else { @@ -207,7 +207,7 @@ func TestRebalanceExitByContext(t *testing.T) { }() const removeCount = 3 - for i := 0; i < removeCount-1; i++ { + for range removeCount - 1 { <-deleteCh signal <- struct{}{} } diff --git a/pkg/local_object_storage/engine/shards_test.go b/pkg/local_object_storage/engine/shards_test.go index f4c7a4309..3347d58f1 100644 --- a/pkg/local_object_storage/engine/shards_test.go +++ b/pkg/local_object_storage/engine/shards_test.go @@ -72,7 +72,7 @@ func TestSortShardsByWeight(t *testing.T) { var shards1 []hashedShard var weights1 []float64 var shards2 []hashedShard - for i := 0; i < numOfShards; i++ { + for i := range numOfShards { shards1 = append(shards1, hashedShard{ hash: uint64(i), }) diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index 2739058e9..6f694f082 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -34,7 +34,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { d := pilorama.CIDDescriptor{CID: cid, Position: 0, Size: 1} treeID := "someTree" - for i := 0; i < objCount; i++ { + for i := range objCount { obj := testutil.GenerateObjectWithCID(cid) testutil.AddAttribute(obj, pilorama.AttributeFilename, strconv.Itoa(i)) err := Put(context.Background(), te.ng, obj) @@ -56,7 +56,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { fs.AddFilter(pilorama.AttributeFilename, strconv.Itoa(objCount/2), objectSDK.MatchStringEqual) prm.WithFilters(fs) - for i := 0; i < b.N; i++ { + for range b.N { res, err := te.ng.Select(context.Background(), prm) if err != nil { b.Fatal(err) @@ -67,7 +67,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { } }) b.Run("TreeGetByPath", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { nodes, err := te.ng.TreeGetByPath(context.Background(), cid, treeID, pilorama.AttributeFilename, []string{strconv.Itoa(objCount / 2)}, true) if err != nil { b.Fatal(err) diff --git a/pkg/local_object_storage/internal/testutil/generators_test.go b/pkg/local_object_storage/internal/testutil/generators_test.go index f7be6014d..cc6f726a4 100644 --- a/pkg/local_object_storage/internal/testutil/generators_test.go +++ b/pkg/local_object_storage/internal/testutil/generators_test.go @@ -13,7 +13,7 @@ func TestOverwriteObjGenerator(t *testing.T) { ObjSize: 10, MaxObjects: 4, } - for i := 0; i < 40; i++ { + for range 40 { obj := gen.Next() id, isSet := obj.ID() i := binary.LittleEndian.Uint64(id[:]) @@ -26,7 +26,7 @@ func TestOverwriteObjGenerator(t *testing.T) { func TestRandObjGenerator(t *testing.T) { gen := &RandObjGenerator{ObjSize: 10} - for i := 0; i < 10; i++ { + for range 10 { obj := gen.Next() require.Equal(t, gen.ObjSize, uint64(len(obj.Payload()))) @@ -50,7 +50,7 @@ func TestSeqObjGenerator(t *testing.T) { func TestRandAddrGenerator(t *testing.T) { gen := RandAddrGenerator(5) - for i := 0; i < 50; i++ { + for range 50 { addr := gen.Next() id := addr.Object() k := binary.LittleEndian.Uint64(id[:]) diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index 5d6788d7e..8b1874458 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -24,7 +24,7 @@ func TestDB_Containers(t *testing.T) { cids := make(map[string]int, N) - for i := 0; i < N; i++ { + for range N { obj := testutil.GenerateObject() cnr, _ := obj.ContainerID() @@ -95,7 +95,7 @@ func TestDB_ContainersCount(t *testing.T) { expected := make([]cid.ID, 0, R+T+SG+L) for _, upload := range uploadObjects { - for i := 0; i < upload.amount; i++ { + for range upload.amount { obj := testutil.GenerateObject() obj.SetType(upload.typ) @@ -126,11 +126,11 @@ func TestDB_ContainerSize(t *testing.T) { cids := make(map[cid.ID]int, C) objs := make(map[cid.ID][]*objectSDK.Object, C*N) - for i := 0; i < C; i++ { + for range C { cnr := cidtest.ID() cids[cnr] = 0 - for j := 0; j < N; j++ { + for range N { size := rand.Intn(1024) parent := testutil.GenerateObjectWithCID(cnr) diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 1797fc0aa..d1f808a63 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -39,14 +39,14 @@ func TestCounters(t *testing.T) { db := newDB(t) defer func() { require.NoError(t, db.Close()) }() oo := make([]*objectSDK.Object, 0, objCount) - for i := 0; i < objCount; i++ { + for range objCount { oo = append(oo, testutil.GenerateObject()) } var prm meta.PutPrm exp := make(map[cid.ID]meta.ObjectCounters) - for i := 0; i < objCount; i++ { + for i := range objCount { prm.SetObject(oo[i]) cnrID, _ := oo[i].ContainerID() c := meta.ObjectCounters{} @@ -187,7 +187,7 @@ func TestCounters(t *testing.T) { // put objects and check that parent info // does not affect the counter - for i := 0; i < objCount; i++ { + for i := range objCount { o := testutil.GenerateObject() if i < objCount/2 { // half of the objs will have the parent o.SetParent(parObj) @@ -535,7 +535,7 @@ func putObjs(t *testing.T, db *meta.DB, count int, withParent bool) []*objectSDK parent := testutil.GenerateObject() oo := make([]*objectSDK.Object, 0, count) - for i := 0; i < count; i++ { + for i := range count { o := testutil.GenerateObject() if withParent { o.SetParent(parent) diff --git a/pkg/local_object_storage/metabase/delete_test.go b/pkg/local_object_storage/metabase/delete_test.go index 2053874d0..cb85157e7 100644 --- a/pkg/local_object_storage/metabase/delete_test.go +++ b/pkg/local_object_storage/metabase/delete_test.go @@ -131,7 +131,7 @@ func TestDelete(t *testing.T) { defer func() { require.NoError(t, db.Close()) }() cnr := cidtest.ID() - for i := 0; i < 10; i++ { + for range 10 { obj := testutil.GenerateObjectWithCID(cnr) var prm meta.PutPrm diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index 247ddf9cd..7654d2cd8 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -223,7 +223,7 @@ func benchmarkGet(b *testing.B, numOfObj int) { defer func() { require.NoError(b, db.Close()) }() addrs := make([]oid.Address, 0, numOfObj) - for i := 0; i < numOfObj; i++ { + for range numOfObj { raw := testutil.GenerateObject() addrs = append(addrs, object.AddressOf(raw)) @@ -261,7 +261,7 @@ func benchmarkGet(b *testing.B, numOfObj int) { b.Run("serial", func(b *testing.B) { b.ReportAllocs() - for i := 0; i < b.N; i++ { + for i := range b.N { var getPrm meta.GetPrm getPrm.SetAddress(addrs[i%len(addrs)]) diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index a92e2eff4..6207497b1 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -35,7 +35,7 @@ func listWithCursorPrepareDB(b *testing.B) *meta.DB { defer func() { require.NoError(b, db.Close()) }() obj := testutil.GenerateObject() - for i := 0; i < 100_000; i++ { // should be a multiple of all batch sizes + for i := range 100_000 { // should be a multiple of all batch sizes obj.SetID(oidtest.ID()) if i%9 == 0 { // let's have 9 objects per container obj.SetContainerID(cidtest.ID()) @@ -51,7 +51,7 @@ func benchmarkListWithCursor(b *testing.B, db *meta.DB, batchSize int) { b.ResetTimer() b.ReportAllocs() - for i := 0; i < b.N; i++ { + for range b.N { res, err := db.ListWithCursor(context.Background(), prm) if err != nil { if err != meta.ErrEndOfListing { @@ -80,7 +80,7 @@ func TestLisObjectsWithCursor(t *testing.T) { expected := make([]object.Info, 0, total) // fill metabase with objects - for i := 0; i < containers; i++ { + for range containers { containerID := cidtest.ID() // add one regular object @@ -140,7 +140,7 @@ func TestLisObjectsWithCursor(t *testing.T) { expectedIterations-- } - for i := 0; i < expectedIterations; i++ { + for range expectedIterations { res, cursor, err = metaListWithCursor(db, uint32(countPerReq), cursor) require.NoError(t, err, "count:%d", countPerReq) got = append(got, res...) @@ -169,7 +169,7 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) { expected := make(map[string]int, total) // fill metabase with objects - for i := 0; i < total; i++ { + for range total { obj := testutil.GenerateObject() err := putBig(db, obj) require.NoError(t, err) @@ -186,7 +186,7 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) { } // add new objects - for i := 0; i < total; i++ { + for range total { obj := testutil.GenerateObject() err = putBig(db, obj) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 2d7bfc1cc..62a109b02 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -155,7 +155,7 @@ func TestDB_Lock(t *testing.T) { inhumePrm.SetGCMark() - for i := 0; i < objsNum; i++ { + for i := range objsNum { inhumePrm.SetAddresses(objectcore.AddressOf(objs[i])) res, err = db.Inhume(context.Background(), inhumePrm) @@ -255,7 +255,7 @@ func putAndLockObj(t *testing.T, db *meta.DB, numOfLockedObjs int) ([]*objectSDK lockedObjs := make([]*objectSDK.Object, 0, numOfLockedObjs) lockedObjIDs := make([]oid.ID, 0, numOfLockedObjs) - for i := 0; i < numOfLockedObjs; i++ { + for range numOfLockedObjs { obj := testutil.GenerateObjectWithCID(cnr) err := putBig(db, obj) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index 84e4029f2..914f5ef06 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -74,7 +74,7 @@ func BenchmarkPut(b *testing.B) { objs := prepareObjects(b.N) b.ResetTimer() b.ReportAllocs() - for i := 0; i < b.N; i++ { + for range b.N { if err := metaPut(db, objs[index.Add(1)], nil); err != nil { b.Fatal(err) } diff --git a/pkg/local_object_storage/metabase/reset_test.go b/pkg/local_object_storage/metabase/reset_test.go index 66f5eefc6..993079dce 100644 --- a/pkg/local_object_storage/metabase/reset_test.go +++ b/pkg/local_object_storage/metabase/reset_test.go @@ -34,7 +34,7 @@ func TestResetDropsContainerBuckets(t *testing.T) { defer func() { require.NoError(t, db.Close()) }() - for idx := 0; idx < 100; idx++ { + for idx := range 100 { var putPrm PutPrm putPrm.SetObject(testutil.GenerateObject()) putPrm.SetStorageID([]byte(fmt.Sprintf("0/%d", idx))) diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 4fbc5910e..0fab3a108 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -920,7 +920,7 @@ func TestDB_SelectSplitID_EC(t *testing.T) { ec, err := erasurecode.NewConstructor(dataCount, parityCount) require.NoError(t, err) - for i := 0; i < partCount; i++ { + for i := range partCount { cs, err := ec.Split(tt.objects[i], &pk.PrivateKey) require.NoError(t, err) @@ -1070,7 +1070,7 @@ func BenchmarkSelect(b *testing.B) { cid := cidtest.ID() - for i := 0; i < objCount; i++ { + for i := range objCount { var attr objectSDK.Attribute attr.SetKey("myHeader") attr.SetValue(strconv.Itoa(i)) @@ -1129,7 +1129,7 @@ func benchmarkSelect(b *testing.B, db *meta.DB, cid cidSDK.ID, fs objectSDK.Sear prm.SetContainerID(cid) prm.SetFilters(fs) - for i := 0; i < b.N; i++ { + for range b.N { res, err := db.Select(context.Background(), prm) if err != nil { b.Fatal(err) diff --git a/pkg/local_object_storage/metabase/upgrade.go b/pkg/local_object_storage/metabase/upgrade.go index a4c7707b4..e9abd746c 100644 --- a/pkg/local_object_storage/metabase/upgrade.go +++ b/pkg/local_object_storage/metabase/upgrade.go @@ -143,7 +143,7 @@ func createExpirationEpochBuckets(ctx context.Context, db *bbolt.DB, log func(a return selectObjectsWithExpirationEpoch(ctx, db, objects) }) var count atomic.Uint64 - for i := 0; i < upgradeWorkersCount; i++ { + for range upgradeWorkersCount { eg.Go(func() error { for { select { diff --git a/pkg/local_object_storage/metabase/upgrade_test.go b/pkg/local_object_storage/metabase/upgrade_test.go index dc3d7d07d..3797de0a4 100644 --- a/pkg/local_object_storage/metabase/upgrade_test.go +++ b/pkg/local_object_storage/metabase/upgrade_test.go @@ -91,7 +91,7 @@ func TestGenerateMetabaseFile(t *testing.T) { eg, ctx := errgroup.WithContext(context.Background()) eg.SetLimit(generateWorkersCount) // simple objects - for i := 0; i < simpleObjectsCount; i++ { + for i := range simpleObjectsCount { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) @@ -110,7 +110,7 @@ func TestGenerateMetabaseFile(t *testing.T) { eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(generateWorkersCount) // complex objects - for i := 0; i < complexObjectsCount; i++ { + for i := range complexObjectsCount { i := i eg.Go(func() error { parent := testutil.GenerateObjectWithCID(containers[i%len(containers)]) @@ -134,7 +134,7 @@ func TestGenerateMetabaseFile(t *testing.T) { eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(generateWorkersCount) // simple objects deleted by gc marks - for i := 0; i < deletedByGCMarksCount; i++ { + for i := range deletedByGCMarksCount { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) @@ -156,7 +156,7 @@ func TestGenerateMetabaseFile(t *testing.T) { eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(10000) // simple objects deleted by tombstones - for i := 0; i < deletedByTombstoneCount; i++ { + for i := range deletedByTombstoneCount { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) @@ -186,7 +186,7 @@ func TestGenerateMetabaseFile(t *testing.T) { eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(generateWorkersCount) // simple objects locked by locks - for i := 0; i < lockedCount; i++ { + for i := range lockedCount { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 29941be83..e2d69cafa 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -705,7 +705,7 @@ func (t *boltForest) applyOperation(logBucket, treeBucket *bbolt.Bucket, ms []*M key, value = c.Prev() } - for i := 0; i < len(ms); i++ { + for i := range len(ms) { // Loop invariant: key represents the next stored timestamp after ms[i].Time. // 2. Insert the operation. diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 41d7a567c..854fe0aad 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -194,7 +194,7 @@ func BenchmarkForestSortedIteration(b *testing.B) { const total = 100_000 d := CIDDescriptor{cnr, 0, 1} - for i := 0; i < total; i++ { + for i := range total { u, err := uuid.NewRandom() if err != nil { b.FailNow() @@ -216,7 +216,7 @@ func BenchmarkForestSortedIteration(b *testing.B) { } b.Run(providers[i].name+",root", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, MultiNode{RootID}, nil, 100) if err != nil || len(res) != 100 { b.Fatalf("err %v, count %d", err, len(res)) @@ -224,7 +224,7 @@ func BenchmarkForestSortedIteration(b *testing.B) { } }) b.Run(providers[i].name+",leaf", func(b *testing.B) { - for i := 0; i < b.N; i++ { + for range b.N { res, _, err := f.TreeSortedByFilename(context.Background(), cnr, treeID, MultiNode{1}, nil, 100) if err != nil || len(res) != 0 { b.FailNow() @@ -804,7 +804,7 @@ func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ defer func() { require.NoError(t, s.Close()) }() require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[0], false)) - for i := 0; i < batchSize; i++ { + for range batchSize { errG.Go(func() error { return s.TreeApply(ctx, cid, treeID, &logs[2], false) }) @@ -1043,7 +1043,7 @@ func TestForest_ParallelApply(t *testing.T) { // The operations are guaranteed to be applied and returned sorted by `Time`. func prepareRandomTree(nodeCount, opCount int) []Move { ops := make([]Move, nodeCount+opCount) - for i := 0; i < nodeCount; i++ { + for i := range nodeCount { ops[i] = Move{ Parent: 0, Meta: Meta{ @@ -1121,14 +1121,14 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ require.NoError(t, expected.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } - for i := 0; i < iterCount; i++ { + for range iterCount { // Shuffle random operations, leave initialization in place. r.Shuffle(len(ops), func(i, j int) { ops[i], ops[j] = ops[j], ops[i] }) actual := constructor(t, WithMaxBatchSize(batchSize), WithNoSync(true)) wg := new(sync.WaitGroup) ch := make(chan *Move) - for i := 0; i < batchSize; i++ { + for range batchSize { wg.Add(1) go func() { defer wg.Done() @@ -1170,7 +1170,7 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ .. } const iterCount = 200 - for i := 0; i < iterCount; i++ { + for range iterCount { // Shuffle random operations, leave initialization in place. r.Shuffle(len(ops), func(i, j int) { ops[i], ops[j] = ops[j], ops[i] }) @@ -1247,7 +1247,7 @@ func BenchmarkApplyReorderLast(b *testing.B) { Child: uint64(r.Intn(benchNodeCount)), } if i != 0 && i%blockSize == 0 { - for j := 0; j < blockSize/2; j++ { + for j := range blockSize / 2 { ops[i-j], ops[i+j-blockSize] = ops[i+j-blockSize], ops[i-j] } } @@ -1265,7 +1265,7 @@ func benchmarkApply(b *testing.B, s Forest, genFunc func(int) []Move) { cid := cidtest.ID() treeID := "version" ch := make(chan int, b.N) - for i := 0; i < b.N; i++ { + for i := range b.N { ch <- i } @@ -1311,7 +1311,7 @@ func testTreeGetByPath(t *testing.T, s ForestStorage) { if mf, ok := s.(*memoryForest); ok { single := mf.treeMap[cid.String()+"/"+treeID] t.Run("test meta", func(t *testing.T) { - for i := 0; i < 6; i++ { + for i := range 6 { require.Equal(t, uint64(i), single.infoMap[Node(i+1)].Meta.Time) } }) @@ -1492,7 +1492,7 @@ func testTreeListTreesCount(t *testing.T, constructor func(t testing.TB, _ ...Op var expected []ContainerIDTreeID treeIDs := []string{"version", "system", "s", "avada kedavra"} - for i := 0; i < count; i++ { + for i := range count { cid := cidtest.ID() treeID := treeIDs[i%len(treeIDs)] expected = append(expected, ContainerIDTreeID{ diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index 8a49a36fd..3414dc76a 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -39,11 +39,11 @@ func testShardList(t *testing.T, sh *Shard) { var errG errgroup.Group errG.SetLimit(C * N) - for i := 0; i < C; i++ { + for range C { errG.Go(func() error { cnr := cidtest.ID() - for j := 0; j < N; j++ { + for range N { errG.Go(func() error { obj := testutil.GenerateObjectWithCID(cnr) testutil.AddPayload(obj, 1<<2) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 38d465f31..1ef849c02 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -206,7 +206,7 @@ func TestCounters(t *testing.T) { const objNumber = 10 oo := make([]*objectSDK.Object, objNumber) - for i := 0; i < objNumber; i++ { + for i := range objNumber { oo[i] = testutil.GenerateObject() } @@ -248,7 +248,7 @@ func TestCounters(t *testing.T) { var prm PutPrm - for i := 0; i < objNumber; i++ { + for i := range objNumber { prm.SetObject(oo[i]) _, err := sh.Put(context.Background(), prm) @@ -269,7 +269,7 @@ func TestCounters(t *testing.T) { var prm InhumePrm inhumedNumber := objNumber / 4 - for i := 0; i < inhumedNumber; i++ { + for i := range inhumedNumber { prm.MarkAsGarbage(objectcore.AddressOf(oo[i])) _, err := sh.Inhume(context.Background(), prm) @@ -317,7 +317,7 @@ func TestCounters(t *testing.T) { _, err := sh.Inhume(context.Background(), prm) require.NoError(t, err) - for i := 0; i < inhumedNumber; i++ { + for i := range inhumedNumber { cid, ok := oo[i].ContainerID() require.True(t, ok) expectedLogicalSizes[cid.EncodeToString()] -= int64(oo[i].PayloadSize()) @@ -419,7 +419,7 @@ func shardWithMetrics(t *testing.T, path string) (*Shard, *metricsStore) { func addrFromObjs(oo []*objectSDK.Object) []oid.Address { aa := make([]oid.Address, len(oo)) - for i := 0; i < len(oo); i++ { + for i := range len(oo) { aa[i] = objectcore.AddressOf(oo[i]) } diff --git a/pkg/local_object_storage/shard/refill_test.go b/pkg/local_object_storage/shard/refill_test.go index 509ccaaa6..0025bb45a 100644 --- a/pkg/local_object_storage/shard/refill_test.go +++ b/pkg/local_object_storage/shard/refill_test.go @@ -38,7 +38,7 @@ func benchRefillMetabase(b *testing.B, objectsCount int) { var putPrm PutPrm - for i := 0; i < objectsCount/2; i++ { + for range objectsCount / 2 { obj := testutil.GenerateObject() testutil.AddAttribute(obj, "foo", "bar") testutil.AddPayload(obj, 1<<5) // blobvnicza tree obj @@ -49,7 +49,7 @@ func benchRefillMetabase(b *testing.B, objectsCount int) { require.NoError(b, err) } - for i := 0; i < objectsCount/2; i++ { + for range objectsCount / 2 { obj := testutil.GenerateObject() testutil.AddAttribute(obj, "foo", "bar") obj.SetID(oidtest.ID()) diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index 4f4398452..4da9a26d7 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -54,7 +54,7 @@ func benchmarkPutSeq(b *testing.B, cache writecache.Cache, size uint64) { objGen := testutil.RandObjGenerator{ObjSize: size} b.ResetTimer() - for n := 0; n < b.N; n++ { + for range b.N { obj := objGen.Next() rawData, err := obj.Marshal() require.NoError(b, err, "marshaling object") diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index e34f5a76b..930ac8431 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -41,7 +41,7 @@ func (c *cache) runFlushLoop(ctx context.Context) { if c.disableBackgroundFlush { return } - for i := 0; i < c.workersCount; i++ { + for range c.workersCount { c.wg.Add(1) go c.workerFlushSmall(ctx) } diff --git a/pkg/morph/event/notary_preparator_test.go b/pkg/morph/event/notary_preparator_test.go index 8da9d868a..4c269bcbd 100644 --- a/pkg/morph/event/notary_preparator_test.go +++ b/pkg/morph/event/notary_preparator_test.go @@ -439,7 +439,7 @@ func TestPrepare_CorrectNR(t *testing.T) { ) for _, test := range tests { - for i := 0; i < 1; i++ { // run tests against 3 and 4 witness NR + for i := range 1 { // run tests against 3 and 4 witness NR for _, dummyMultisig := range []bool{true, false} { // run tests against empty and dummy multisig/Notary witness additionalWitness := i == 0 nr := correctNR(script(test.hash, test.method, test.args...), dummyMultisig, additionalWitness) diff --git a/pkg/morph/timer/block_test.go b/pkg/morph/timer/block_test.go index 93bb04de5..7929754c1 100644 --- a/pkg/morph/timer/block_test.go +++ b/pkg/morph/timer/block_test.go @@ -208,7 +208,7 @@ func TestBlockTimer_TickSameHeight(t *testing.T) { require.NoError(t, bt.Reset()) check := func(t *testing.T, h uint32, base, delta int) { - for i := 0; i < 2*int(blockDur); i++ { + for range 2 * int(blockDur) { bt.Tick(h) require.Equal(t, base, baseCounter) require.Equal(t, delta, deltaCounter) diff --git a/pkg/network/tls_test.go b/pkg/network/tls_test.go index 6c352484b..14729f4c2 100644 --- a/pkg/network/tls_test.go +++ b/pkg/network/tls_test.go @@ -37,7 +37,7 @@ func BenchmarkAddressTLSEnabled(b *testing.B) { b.ReportAllocs() var enabled bool - for i := 0; i < b.N; i++ { + for range b.N { enabled = addr.IsTLSEnabled() } require.True(b, enabled) diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index 0ba8be765..300cb9dc9 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -169,7 +169,7 @@ func (s *Server) getContainerNodes(contID cid.ID) ([]netmap.NodeInfo, error) { nodes := placement.FlattenNodes(ns) bs := (*keys.PublicKey)(&s.key.PublicKey).Bytes() - for i := 0; i < len(nodes); i++ { + for i := range len(nodes) { if bytes.Equal(nodes[i].PublicKey(), bs) { copy(nodes[i:], nodes[i+1:]) nodes = nodes[:len(nodes)-1] diff --git a/pkg/services/object/acl/v2/util_test.go b/pkg/services/object/acl/v2/util_test.go index 394feef4e..435339683 100644 --- a/pkg/services/object/acl/v2/util_test.go +++ b/pkg/services/object/acl/v2/util_test.go @@ -33,7 +33,7 @@ func TestOriginalTokens(t *testing.T) { var sTokenV2 session.Token sToken.WriteToV2(&sTokenV2) - for i := 0; i < 10; i++ { + for i := range 10 { metaHeaders := testGenerateMetaHeader(uint32(i), &bTokenV2, &sTokenV2) res, err := originalSessionToken(metaHeaders) require.NoError(t, err) diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 1fc6b7b20..6827018dc 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -470,7 +470,7 @@ func testNodeMatrix(t testing.TB, dim []int) ([][]netmap.NodeInfo, [][]string) { ns := make([]netmap.NodeInfo, dim[i]) as := make([]string, dim[i]) - for j := 0; j < dim[i]; j++ { + for j := range dim[i] { a := fmt.Sprintf("/ip4/192.168.0.%s/tcp/%s", strconv.Itoa(i), strconv.Itoa(60000+j), @@ -508,7 +508,7 @@ func generateChain(ln int, cnr cid.ID) ([]*objectSDK.Object, []oid.ID, []byte) { ids := make([]oid.ID, 0, ln) payload := make([]byte, 0, ln*10) - for i := 0; i < ln; i++ { + for i := range ln { ids = append(ids, curID) addr.SetObject(curID) @@ -1750,7 +1750,7 @@ func TestGetRange(t *testing.T) { }, }) - for from := 0; from < totalSize-1; from++ { + for from := range totalSize - 1 { for to := from; to < totalSize; to++ { t.Run(fmt.Sprintf("from=%d,to=%d", from, to), func(t *testing.T) { testGetRange(t, svc, addr, uint64(from), uint64(to), payload) @@ -1811,7 +1811,7 @@ func TestGetRange(t *testing.T) { }, }) - for from := 0; from < totalSize-1; from++ { + for from := range totalSize - 1 { for to := from; to < totalSize; to++ { t.Run(fmt.Sprintf("from=%d,to=%d", from, to), func(t *testing.T) { testGetRange(t, svc, addr, uint64(from), uint64(to), payload) diff --git a/pkg/services/object/get/getrangeec_test.go b/pkg/services/object/get/getrangeec_test.go index b8497d7d1..a6882d4a8 100644 --- a/pkg/services/object/get/getrangeec_test.go +++ b/pkg/services/object/get/getrangeec_test.go @@ -131,7 +131,7 @@ func TestGetRangeEC(t *testing.T) { clients: clients, }) - for from := 0; from < totalSize-1; from++ { + for from := range totalSize - 1 { for to := from; to < totalSize; to++ { t.Run(fmt.Sprintf("from=%d,to=%d", from, to), func(t *testing.T) { testGetRange(t, svc, addr, uint64(from), uint64(to), obj.Payload()) diff --git a/pkg/services/object/put/ec.go b/pkg/services/object/put/ec.go index 1fadf65fe..9980f6d61 100644 --- a/pkg/services/object/put/ec.go +++ b/pkg/services/object/put/ec.go @@ -276,7 +276,7 @@ func (e *ecWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx } // try to save to any node not visited by current part - for i := 0; i < len(nodes); i++ { + for i := range len(nodes) { select { case <-ctx.Done(): return ctx.Err() diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index 679380402..44abcfe5b 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -151,7 +151,7 @@ func testSHA256() (cs [sha256.Size]byte) { func generateIDs(num int) []oid.ID { res := make([]oid.ID, num) - for i := 0; i < num; i++ { + for i := range num { res[i].SetSHA256(testSHA256()) } @@ -232,7 +232,7 @@ func testNodeMatrix(t testing.TB, dim []int) ([][]netmap.NodeInfo, [][]string) { ns := make([]netmap.NodeInfo, dim[i]) as := make([]string, dim[i]) - for j := 0; j < dim[i]; j++ { + for j := range dim[i] { a := fmt.Sprintf("/ip4/192.168.0.%s/tcp/%s", strconv.Itoa(i), strconv.Itoa(60000+j), diff --git a/pkg/services/object_manager/placement/cache_test.go b/pkg/services/object_manager/placement/cache_test.go index 07e9340dc..a890d5357 100644 --- a/pkg/services/object_manager/placement/cache_test.go +++ b/pkg/services/object_manager/placement/cache_test.go @@ -64,7 +64,7 @@ func TestContainerNodesCache(t *testing.T) { nm2 := nm(1, nodes[1:2]) cnr := [size * 2]cid.ID{} res := [size * 2][][]netmapSDK.NodeInfo{} - for i := 0; i < size*2; i++ { + for i := range size * 2 { cnr[i] = cidtest.ID() var err error @@ -77,7 +77,7 @@ func TestContainerNodesCache(t *testing.T) { require.NoError(t, err) require.Equal(t, res[i], r) } - for i := 0; i < size; i++ { + for i := range size { r, err := c.ContainerNodes(nm2, cnr[i], pp) require.NoError(t, err) require.NotEqual(t, res[i], r) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 9a5877c52..4e790628f 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -136,7 +136,7 @@ func defaultCopiesVector(policy netmap.PlacementPolicy) []int { replNum := policy.NumberOfReplicas() copyVector := make([]int, 0, replNum) - for i := 0; i < replNum; i++ { + for i := range replNum { copyVector = append(copyVector, int(policy.ReplicaDescriptor(i).NumberOfObjects()+policy.ReplicaDescriptor(i).GetECDataCount()+policy.ReplicaDescriptor(i).GetECParityCount())) } @@ -212,7 +212,7 @@ func (t *Traverser) Next() []Node { nodes := make([]Node, count) - for i := 0; i < count; i++ { + for i := range count { err := nodes[i].addresses.FromIterator(network.NodeEndpointsIterator(t.vectors[0][i])) if err != nil { return nil diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index f5731c81e..b3b57677d 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -48,7 +48,7 @@ func testPlacement(ss, rs []int) ([][]netmap.NodeInfo, container.Container) { for i := range ss { ns := make([]netmap.NodeInfo, 0, ss[i]) - for j := 0; j < ss[i]; j++ { + for range ss[i] { ns = append(ns, testNode(num)) num++ } @@ -125,7 +125,7 @@ func TestTraverserObjectScenarios(t *testing.T) { ) require.NoError(t, err) - for i := 0; i < len(nodes[0]); i++ { + for range len(nodes[0]) { require.NotNil(t, tr.Next()) } @@ -164,7 +164,7 @@ func TestTraverserObjectScenarios(t *testing.T) { require.Empty(t, tr.Next()) require.False(t, tr.Success()) - for i := 0; i < replicas[curVector]; i++ { + for range replicas[curVector] { tr.SubmitSuccess() } } diff --git a/pkg/services/session/storage/persistent/executor_test.go b/pkg/services/session/storage/persistent/executor_test.go index 39cb14359..124d36930 100644 --- a/pkg/services/session/storage/persistent/executor_test.go +++ b/pkg/services/session/storage/persistent/executor_test.go @@ -39,7 +39,7 @@ func TestTokenStore(t *testing.T) { tokens := make([]tok, 0, tokenNumber) - for i := 0; i < tokenNumber; i++ { + for i := range tokenNumber { req.SetExpiration(uint64(i)) res, err := ts.Create(context.Background(), req) diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index 677431889..95bdda34b 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -62,7 +62,7 @@ func TestGetSubTree(t *testing.T) { loop: for i := 1; i < len(acc.seen); i++ { parent := acc.seen[i].Body.ParentId - for j := 0; j < i; j++ { + for j := range i { if acc.seen[j].Body.NodeId[0] == parent[0] { continue loop } diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 346198b3c..95c8f8013 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -133,7 +133,7 @@ func (s *Service) ReplicateTreeOp(ctx context.Context, n netmapSDK.NodeInfo, req } func (s *Service) replicateLoop(ctx context.Context) { - for i := 0; i < s.replicatorWorkerCount; i++ { + for range s.replicatorWorkerCount { go s.replicationWorker(ctx) go s.localReplicationWorker(ctx) } diff --git a/pkg/util/sync/key_locker_test.go b/pkg/util/sync/key_locker_test.go index 3b3e6a694..259064ecf 100644 --- a/pkg/util/sync/key_locker_test.go +++ b/pkg/util/sync/key_locker_test.go @@ -13,7 +13,7 @@ func TestKeyLocker(t *testing.T) { taken := false eg, _ := errgroup.WithContext(context.Background()) keyLocker := NewKeyLocker[int]() - for i := 0; i < 100; i++ { + for range 100 { eg.Go(func() error { keyLocker.Lock(0) defer keyLocker.Unlock(0) From a4fb7f085b4cd2c5d7bb2ec91e6e626238dead54 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Tue, 3 Sep 2024 11:39:02 +0300 Subject: [PATCH 1311/1943] [#1348] go.mod: Update api-go and sdk-go Signed-off-by: Aleksey Savchuk --- cmd/frostfs-cli/internal/client/client.go | 26 ----- cmd/frostfs-cli/modules/container/get_eacl.go | 68 ------------- cmd/frostfs-cli/modules/container/root.go | 2 - cmd/frostfs-cli/modules/util/ape.go | 6 -- go.mod | 4 +- go.sum | 8 +- .../transport/container/grpc/service.go | 15 --- pkg/services/container/ape.go | 12 --- pkg/services/container/ape_test.go | 95 ------------------- pkg/services/container/audit.go | 11 --- pkg/services/container/executor.go | 14 --- pkg/services/container/morph/executor.go | 37 -------- pkg/services/container/server.go | 1 - pkg/services/container/sign.go | 9 -- 14 files changed, 6 insertions(+), 302 deletions(-) delete mode 100644 cmd/frostfs-cli/modules/container/get_eacl.go diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 57bcf5620..03a987a57 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -17,7 +17,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -192,31 +191,6 @@ func DeleteContainer(ctx context.Context, prm DeleteContainerPrm) (res DeleteCon return } -// EACLPrm groups parameters of EACL operation. -type EACLPrm struct { - Client *client.Client - ClientParams client.PrmContainerEACL -} - -// EACLRes groups the resulting values of EACL operation. -type EACLRes struct { - cliRes *client.ResContainerEACL -} - -// EACL returns requested eACL table. -func (x EACLRes) EACL() eacl.Table { - return x.cliRes.Table() -} - -// EACL reads eACL table from FrostFS by container ID. -// -// Returns any error which prevented the operation from completing correctly in error return. -func EACL(ctx context.Context, prm EACLPrm) (res EACLRes, err error) { - res.cliRes, err = prm.Client.ContainerEACL(ctx, prm.ClientParams) - - return -} - // NetworkInfoPrm groups parameters of NetworkInfo operation. type NetworkInfoPrm struct { Client *client.Client diff --git a/cmd/frostfs-cli/modules/container/get_eacl.go b/cmd/frostfs-cli/modules/container/get_eacl.go deleted file mode 100644 index 4ed1c82e1..000000000 --- a/cmd/frostfs-cli/modules/container/get_eacl.go +++ /dev/null @@ -1,68 +0,0 @@ -package container - -import ( - "os" - - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - "github.com/spf13/cobra" -) - -var getExtendedACLCmd = &cobra.Command{ - Use: "get-eacl", - Short: "Get extended ACL table of container", - Long: `Get extended ACL table of container`, - Run: func(cmd *cobra.Command, _ []string) { - id := parseContainerID(cmd) - pk := key.GetOrGenerate(cmd) - cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - - eaclPrm := internalclient.EACLPrm{ - Client: cli, - ClientParams: client.PrmContainerEACL{ - ContainerID: &id, - }, - } - - res, err := internalclient.EACL(cmd.Context(), eaclPrm) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - eaclTable := res.EACL() - - if containerPathTo == "" { - cmd.Println("eACL: ") - common.PrettyPrintJSON(cmd, &eaclTable, "eACL") - - return - } - - var data []byte - - if containerJSON { - data, err = eaclTable.MarshalJSON() - commonCmd.ExitOnErr(cmd, "can't encode to JSON: %w", err) - } else { - data, err = eaclTable.Marshal() - commonCmd.ExitOnErr(cmd, "can't encode to binary: %w", err) - } - - cmd.Println("dumping data to file:", containerPathTo) - - err = os.WriteFile(containerPathTo, data, 0o644) - commonCmd.ExitOnErr(cmd, "could not write eACL to file: %w", err) - }, -} - -func initContainerGetEACLCmd() { - commonflags.Init(getExtendedACLCmd) - - flags := getExtendedACLCmd.Flags() - - flags.StringVar(&containerID, commonflags.CIDFlag, "", commonflags.CIDFlagUsage) - flags.StringVar(&containerPathTo, "to", "", "Path to dump encoded container (default: binary encoded)") - flags.BoolVar(&containerJSON, commonflags.JSON, false, "Encode EACL table in json format") -} diff --git a/cmd/frostfs-cli/modules/container/root.go b/cmd/frostfs-cli/modules/container/root.go index d5f0fd776..2da21e767 100644 --- a/cmd/frostfs-cli/modules/container/root.go +++ b/cmd/frostfs-cli/modules/container/root.go @@ -25,7 +25,6 @@ func init() { deleteContainerCmd, listContainerObjectsCmd, getContainerInfoCmd, - getExtendedACLCmd, containerNodesCmd, policyPlaygroundCmd, } @@ -37,7 +36,6 @@ func init() { initContainerDeleteCmd() initContainerListObjectsCmd() initContainerInfoCmd() - initContainerGetEACLCmd() initContainerNodesCmd() initContainerPolicyPlaygroundCmd() diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index 9af57434a..73c368510 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -258,10 +258,6 @@ func parseAction(lexeme string) ([]string, bool, error) { return []string{nativeschema.MethodDeleteContainer}, false, nil case "container.get": return []string{nativeschema.MethodGetContainer}, false, nil - case "container.setcontainereacl": - return []string{nativeschema.MethodSetContainerEACL}, false, nil - case "container.getcontainereacl": - return []string{nativeschema.MethodGetContainerEACL}, false, nil case "container.list": return []string{nativeschema.MethodListContainers}, false, nil case "container.*": @@ -269,8 +265,6 @@ func parseAction(lexeme string) ([]string, bool, error) { nativeschema.MethodPutContainer, nativeschema.MethodDeleteContainer, nativeschema.MethodGetContainer, - nativeschema.MethodSetContainerEACL, - nativeschema.MethodGetContainerEACL, nativeschema.MethodListContainers, }, false, nil default: diff --git a/go.mod b/go.mod index 358370201..19bf7852f 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240828085308-5e1c6a908f61 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240902111049-c11f50efeccb git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240827083309-f0b9493ce3f7 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240903093628-8f751d9dd0ad git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index be82bff70..8ebd59157 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240828085308-5e1c6a908f61 h1:bw9EVGWnfY9awFb5XYR52AGbzgg3o04gZF66yHob48c= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240828085308-5e1c6a908f61/go.mod h1:BDnEpkKMykCS8u1nLzR6SgNzCv6885RWlo5TnravQuI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240902111049-c11f50efeccb h1:p9ByDsw+H6p6LyYSx8LKFtAG/oPKQpDVMNfjPqdevTw= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240902111049-c11f50efeccb/go.mod h1:BDnEpkKMykCS8u1nLzR6SgNzCv6885RWlo5TnravQuI= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240827083309-f0b9493ce3f7 h1:9eZidZMT4tHOdc6GZRPlZR12IToKqHhUd5wzevdDUqo= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240827083309-f0b9493ce3f7/go.mod h1:VzVYcwo/eXjkdo5ktPdZeAE4fsnZX6zEun3g+5E2weo= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240903093628-8f751d9dd0ad h1:cJGK/bXSF/0KMr6zkIy06qekQhQRU7eYzh+lWdQ0U8w= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240903093628-8f751d9dd0ad/go.mod h1:I4GzeEnftO++N2JHQn9QR88eV0rjQkGqlUCNMnCz1CY= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= diff --git a/pkg/network/transport/container/grpc/service.go b/pkg/network/transport/container/grpc/service.go index f0206dd5c..9fae22b45 100644 --- a/pkg/network/transport/container/grpc/service.go +++ b/pkg/network/transport/container/grpc/service.go @@ -80,18 +80,3 @@ func (s *Server) List(ctx context.Context, req *containerGRPC.ListRequest) (*con return resp.ToGRPCMessage().(*containerGRPC.ListResponse), nil } - -// GetExtendedACL converts gRPC GetExtendedACLRequest message and passes it to internal Container service. -func (s *Server) GetExtendedACL(ctx context.Context, req *containerGRPC.GetExtendedACLRequest) (*containerGRPC.GetExtendedACLResponse, error) { - getEACLReq := new(container.GetExtendedACLRequest) - if err := getEACLReq.FromGRPCMessage(req); err != nil { - return nil, err - } - - resp, err := s.srv.GetExtendedACL(ctx, getEACLReq) - if err != nil { - return nil, err - } - - return resp.ToGRPCMessage().(*containerGRPC.GetExtendedACLResponse), nil -} diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 6f8a8e0e6..d92ecf58b 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -102,18 +102,6 @@ func (ac *apeChecker) Get(ctx context.Context, req *container.GetRequest) (*cont return ac.next.Get(ctx, req) } -func (ac *apeChecker) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.GetExtendedACL") - defer span.End() - - if err := ac.validateContainerBoundedOperation(ctx, req.GetBody().GetContainerID(), req.GetMetaHeader(), req.GetVerificationHeader(), - nativeschema.MethodGetContainerEACL); err != nil { - return nil, err - } - - return ac.next.GetExtendedACL(ctx, req) -} - func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.List") defer span.End() diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index 68c1158a6..d6f9b75ef 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -49,7 +49,6 @@ func TestAPE(t *testing.T) { t.Run("deny get container by user claim tag", testDenyGetContainerByUserClaimTag) t.Run("deny get container by IP", testDenyGetContainerByIP) t.Run("deny get container by group id", testDenyGetContainerByGroupID) - t.Run("deny get container eACL for IR with session token", testDenyGetContainerEACLForIRSessionToken) t.Run("deny put container for others with session token", testDenyPutContainerForOthersSessionToken) t.Run("deny put container, read namespace from frostfsID", testDenyPutContainerReadNamespaceFromFrostfsID) t.Run("deny put container with invlaid namespace", testDenyPutContainerInvalidNamespace) @@ -663,95 +662,6 @@ func testDenyGetContainerByGroupID(t *testing.T) { require.ErrorAs(t, err, &errAccessDenied) } -func testDenyGetContainerEACLForIRSessionToken(t *testing.T) { - t.Parallel() - srv := &srvStub{ - calls: map[string]int{}, - } - router := inmemory.NewInMemory() - contRdr := &containerStub{ - c: map[cid.ID]*containercore.Container{}, - } - ir := &irStub{ - keys: [][]byte{}, - } - nm := &netmapStub{} - frostfsIDSubjectReader := &frostfsidStub{ - subjects: map[util.Uint160]*client.Subject{}, - } - apeSrv := NewAPEServer(router, contRdr, ir, nm, frostfsIDSubjectReader, srv) - - contID := cidtest.ID() - testContainer := containertest.Container() - pp := netmap.PlacementPolicy{} - require.NoError(t, pp.DecodeString("REP 1")) - testContainer.SetPlacementPolicy(pp) - contRdr.c[contID] = &containercore.Container{Value: testContainer} - - nm.currentEpoch = 100 - nm.netmaps = map[uint64]*netmap.NetMap{} - var testNetmap netmap.NetMap - testNetmap.SetEpoch(nm.currentEpoch) - testNetmap.SetNodes([]netmap.NodeInfo{{}}) - nm.netmaps[nm.currentEpoch] = &testNetmap - nm.netmaps[nm.currentEpoch-1] = &testNetmap - - _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(contID.EncodeToString()), &chain.Chain{ - Rules: []chain.Rule{ - { - Status: chain.AccessDenied, - Actions: chain.Actions{ - Names: []string{ - nativeschema.MethodGetContainerEACL, - }, - }, - Resources: chain.Resources{ - Names: []string{ - fmt.Sprintf(nativeschema.ResourceFormatRootContainer, contID.EncodeToString()), - }, - }, - Condition: []chain.Condition{ - { - Kind: chain.KindRequest, - Key: nativeschema.PropertyKeyActorRole, - Value: nativeschema.PropertyValueContainerRoleIR, - Op: chain.CondStringEquals, - }, - }, - }, - }, - }) - require.NoError(t, err) - - req := &container.GetExtendedACLRequest{} - req.SetBody(&container.GetExtendedACLRequestBody{}) - var refContID refs.ContainerID - contID.WriteToV2(&refContID) - req.GetBody().SetContainerID(&refContID) - - pk, err := keys.NewPrivateKey() - require.NoError(t, err) - require.NoError(t, signature.SignServiceMessage(&pk.PrivateKey, req)) - - sessionPK, err := keys.NewPrivateKey() - require.NoError(t, err) - sToken := sessiontest.ContainerSigned() - sToken.ApplyOnlyTo(contID) - require.NoError(t, sToken.Sign(sessionPK.PrivateKey)) - var sTokenV2 session.Token - sToken.WriteToV2(&sTokenV2) - metaHeader := new(session.RequestMetaHeader) - metaHeader.SetSessionToken(&sTokenV2) - req.SetMetaHeader(metaHeader) - - ir.keys = append(ir.keys, sessionPK.PublicKey().Bytes()) - - resp, err := apeSrv.GetExtendedACL(context.Background(), req) - require.Nil(t, resp) - var errAccessDenied *apistatus.ObjectAccessDenied - require.ErrorAs(t, err, &errAccessDenied) -} - func testDenyPutContainerForOthersSessionToken(t *testing.T) { t.Parallel() srv := &srvStub{ @@ -1164,11 +1074,6 @@ func (s *srvStub) Get(context.Context, *container.GetRequest) (*container.GetRes return &container.GetResponse{}, nil } -func (s *srvStub) GetExtendedACL(context.Context, *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { - s.calls["GetExtendedACL"]++ - return &container.GetExtendedACLResponse{}, nil -} - func (s *srvStub) List(context.Context, *container.ListRequest) (*container.ListResponse, error) { s.calls["List"]++ return &container.ListResponse{}, nil diff --git a/pkg/services/container/audit.go b/pkg/services/container/audit.go index 34fd5923f..b257272f5 100644 --- a/pkg/services/container/audit.go +++ b/pkg/services/container/audit.go @@ -52,17 +52,6 @@ func (a *auditService) Get(ctx context.Context, req *container.GetRequest) (*con return res, err } -// GetExtendedACL implements Server. -func (a *auditService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { - res, err := a.next.GetExtendedACL(ctx, req) - if !a.enabled.Load() { - return res, err - } - audit.LogRequest(a.log, container_grpc.ContainerService_GetExtendedACL_FullMethodName, req, - audit.TargetFromRef(req.GetBody().GetContainerID(), &cid.ID{}), err == nil) - return res, err -} - // List implements Server. func (a *auditService) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { res, err := a.next.List(ctx, req) diff --git a/pkg/services/container/executor.go b/pkg/services/container/executor.go index b64963e25..0917e3bd0 100644 --- a/pkg/services/container/executor.go +++ b/pkg/services/container/executor.go @@ -14,7 +14,6 @@ type ServiceExecutor interface { Delete(context.Context, *session.Token, *container.DeleteRequestBody) (*container.DeleteResponseBody, error) Get(context.Context, *container.GetRequestBody) (*container.GetResponseBody, error) List(context.Context, *container.ListRequestBody) (*container.ListResponseBody, error) - GetExtendedACL(context.Context, *container.GetExtendedACLRequestBody) (*container.GetExtendedACLResponseBody, error) } type executorSvc struct { @@ -94,16 +93,3 @@ func (s *executorSvc) List(ctx context.Context, req *container.ListRequest) (*co s.respSvc.SetMeta(resp) return resp, nil } - -func (s *executorSvc) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { - respBody, err := s.exec.GetExtendedACL(ctx, req.GetBody()) - if err != nil { - return nil, fmt.Errorf("could not execute GetEACL request: %w", err) - } - - resp := new(container.GetExtendedACLResponse) - resp.SetBody(respBody) - - s.respSvc.SetMeta(resp) - return resp, nil -} diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index 57dac32f0..05d8749cf 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -201,40 +201,3 @@ func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) return res, nil } - -func (s *morphExecutor) GetExtendedACL(_ context.Context, body *container.GetExtendedACLRequestBody) (*container.GetExtendedACLResponseBody, error) { - idV2 := body.GetContainerID() - if idV2 == nil { - return nil, errors.New("missing container ID") - } - - var id cid.ID - - err := id.ReadFromV2(*idV2) - if err != nil { - return nil, fmt.Errorf("invalid container ID: %w", err) - } - - eaclInfo, err := s.rdr.GetEACL(id) - if err != nil { - return nil, err - } - - var sigV2 refs.Signature - eaclInfo.Signature.WriteToV2(&sigV2) - - var tokV2 *sessionV2.Token - - if eaclInfo.Session != nil { - tokV2 = new(sessionV2.Token) - - eaclInfo.Session.WriteToV2(tokV2) - } - - res := new(container.GetExtendedACLResponseBody) - res.SetEACL(eaclInfo.Value.ToV2()) - res.SetSignature(&sigV2) - res.SetSessionToken(tokV2) - - return res, nil -} diff --git a/pkg/services/container/server.go b/pkg/services/container/server.go index d714d7f02..a19d83c56 100644 --- a/pkg/services/container/server.go +++ b/pkg/services/container/server.go @@ -12,5 +12,4 @@ type Server interface { Get(context.Context, *container.GetRequest) (*container.GetResponse, error) Delete(context.Context, *container.DeleteRequest) (*container.DeleteResponse, error) List(context.Context, *container.ListRequest) (*container.ListResponse, error) - GetExtendedACL(context.Context, *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) } diff --git a/pkg/services/container/sign.go b/pkg/services/container/sign.go index 62aa3fe27..f7f5d6486 100644 --- a/pkg/services/container/sign.go +++ b/pkg/services/container/sign.go @@ -56,12 +56,3 @@ func (s *signService) List(ctx context.Context, req *container.ListRequest) (*co resp, err := util.EnsureNonNilResponse(s.svc.List(ctx, req)) return resp, s.sigSvc.SignResponse(resp, err) } - -func (s *signService) GetExtendedACL(ctx context.Context, req *container.GetExtendedACLRequest) (*container.GetExtendedACLResponse, error) { - if err := s.sigSvc.VerifyRequest(req); err != nil { - resp := new(container.GetExtendedACLResponse) - return resp, s.sigSvc.SignResponse(resp, err) - } - resp, err := util.EnsureNonNilResponse(s.svc.GetExtendedACL(ctx, req)) - return resp, s.sigSvc.SignResponse(resp, err) -} From b9043433a098ae242c559a79afe6f1ced5068e68 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 15 Jul 2024 14:07:32 +0300 Subject: [PATCH 1312/1943] [#1223] scripts: Add script to populate metabase Signed-off-by: Aleksey Savchuk --- .../populate-metabase/internal/generate.go | 132 +++++++++ .../populate-metabase/internal/populate.go | 263 ++++++++++++++++++ scripts/populate-metabase/main.go | 159 +++++++++++ 3 files changed, 554 insertions(+) create mode 100644 scripts/populate-metabase/internal/generate.go create mode 100644 scripts/populate-metabase/internal/populate.go create mode 100644 scripts/populate-metabase/main.go diff --git a/scripts/populate-metabase/internal/generate.go b/scripts/populate-metabase/internal/generate.go new file mode 100644 index 000000000..d2004b673 --- /dev/null +++ b/scripts/populate-metabase/internal/generate.go @@ -0,0 +1,132 @@ +package internal + +import ( + "crypto/sha256" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "git.frostfs.info/TrueCloudLab/tzhash/tz" + "golang.org/x/exp/rand" +) + +func GeneratePayloadPool(count uint, size uint) [][]byte { + pool := [][]byte{} + for i := uint(0); i < count; i++ { + payload := make([]byte, size) + _, _ = rand.Read(payload) + + pool = append(pool, payload) + } + return pool +} + +func GenerateAttributePool(count uint) []objectSDK.Attribute { + pool := []objectSDK.Attribute{} + for i := uint(0); i < count; i++ { + for j := uint(0); j < count; j++ { + attr := *objectSDK.NewAttribute() + attr.SetKey(fmt.Sprintf("key%d", i)) + attr.SetValue(fmt.Sprintf("value%d", j)) + pool = append(pool, attr) + } + } + return pool +} + +func GenerateOwnerPool(count uint) []user.ID { + pool := []user.ID{} + for i := uint(0); i < count; i++ { + pool = append(pool, usertest.ID()) + } + return pool +} + +type ObjectOption func(obj *objectSDK.Object) + +func GenerateObject(options ...ObjectOption) *objectSDK.Object { + var ver version.Version + ver.SetMajor(2) + ver.SetMinor(1) + + payload := make([]byte, 0) + + var csum checksum.Checksum + csum.SetSHA256(sha256.Sum256(payload)) + + var csumTZ checksum.Checksum + csumTZ.SetTillichZemor(tz.Sum(csum.Value())) + + obj := objectSDK.New() + obj.SetID(oidtest.ID()) + obj.SetOwnerID(usertest.ID()) + obj.SetContainerID(cidtest.ID()) + + header := objecttest.Object().GetECHeader() + header.SetParent(oidtest.ID()) + obj.SetECHeader(header) + + obj.SetVersion(&ver) + obj.SetPayload(payload) + obj.SetPayloadSize(uint64(len(payload))) + obj.SetPayloadChecksum(csum) + obj.SetPayloadHomomorphicHash(csumTZ) + + for _, option := range options { + option(obj) + } + + return obj +} + +func WithContainerID(cid cid.ID) ObjectOption { + return func(obj *objectSDK.Object) { + obj.SetContainerID(cid) + } +} + +func WithType(typ objectSDK.Type) ObjectOption { + return func(obj *objectSDK.Object) { + obj.SetType(typ) + } +} + +func WithPayloadFromPool(pool [][]byte) ObjectOption { + payload := pool[rand.Intn(len(pool))] + + var csum checksum.Checksum + csum.SetSHA256(sha256.Sum256(payload)) + + var csumTZ checksum.Checksum + csumTZ.SetTillichZemor(tz.Sum(csum.Value())) + + return func(obj *objectSDK.Object) { + obj.SetPayload(payload) + obj.SetPayloadSize(uint64(len(payload))) + obj.SetPayloadChecksum(csum) + obj.SetPayloadHomomorphicHash(csumTZ) + } +} + +func WithAttributesFromPool(pool []objectSDK.Attribute, count uint) ObjectOption { + return func(obj *objectSDK.Object) { + attrs := []objectSDK.Attribute{} + for i := uint(0); i < count; i++ { + attrs = append(attrs, pool[rand.Intn(len(pool))]) + } + obj.SetAttributes(attrs...) + } +} + +func WithOwnerIDFromPool(pool []user.ID) ObjectOption { + return func(obj *objectSDK.Object) { + obj.SetOwnerID(pool[rand.Intn(len(pool))]) + } +} diff --git a/scripts/populate-metabase/internal/populate.go b/scripts/populate-metabase/internal/populate.go new file mode 100644 index 000000000..390c1cdc0 --- /dev/null +++ b/scripts/populate-metabase/internal/populate.go @@ -0,0 +1,263 @@ +package internal + +import ( + "context" + "fmt" + "math/rand" + "sync" + + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "golang.org/x/sync/errgroup" +) + +type EpochState struct{} + +func (s EpochState) CurrentEpoch() uint64 { + return 0 +} + +func PopulateWithObjects( + ctx context.Context, + db *meta.DB, + group *errgroup.Group, + count uint, + factory func() *objectSDK.Object, +) { + digits := "0123456789" + + for i := uint(0); i < count; i++ { + obj := factory() + + id := []byte(fmt.Sprintf( + "%c/%c/%c", + digits[rand.Int()%len(digits)], + digits[rand.Int()%len(digits)], + digits[rand.Int()%len(digits)], + )) + + prm := meta.PutPrm{} + prm.SetObject(obj) + prm.SetStorageID(id) + + group.Go(func() error { + if _, err := db.Put(ctx, prm); err != nil { + return fmt.Errorf("couldn't put an object: %w", err) + } + return nil + }) + } +} + +func PopulateWithBigObjects( + ctx context.Context, + db *meta.DB, + group *errgroup.Group, + count uint, + factory func() *objectSDK.Object, +) { + for i := uint(0); i < count; i++ { + group.Go(func() error { + if err := populateWithBigObject(ctx, db, factory); err != nil { + return fmt.Errorf("couldn't put a big object: %w", err) + } + return nil + }) + } +} + +func populateWithBigObject( + ctx context.Context, + db *meta.DB, + factory func() *objectSDK.Object, +) error { + t := &target{db: db} + + pk, _ := keys.NewPrivateKey() + p := transformer.NewPayloadSizeLimiter(transformer.Params{ + Key: &pk.PrivateKey, + NextTargetInit: func() transformer.ObjectWriter { return t }, + NetworkState: EpochState{}, + MaxSize: 10, + }) + + obj := factory() + payload := make([]byte, 30) + + err := p.WriteHeader(ctx, obj) + if err != nil { + return err + } + + _, err = p.Write(ctx, payload) + if err != nil { + return err + } + + _, err = p.Close(ctx) + if err != nil { + return err + } + + return nil +} + +type target struct { + db *meta.DB +} + +func (t *target) WriteObject(ctx context.Context, obj *objectSDK.Object) error { + prm := meta.PutPrm{} + prm.SetObject(obj) + + _, err := t.db.Put(ctx, prm) + return err +} + +func PopulateGraveyard( + ctx context.Context, + db *meta.DB, + group *errgroup.Group, + workBufferSize int, + count uint, + factory func() *objectSDK.Object, +) { + ts := factory() + ts.SetType(objectSDK.TypeTombstone) + + prm := meta.PutPrm{} + prm.SetObject(ts) + + group.Go(func() error { + if _, err := db.Put(ctx, prm); err != nil { + return fmt.Errorf("couldn't put a tombstone object: %w", err) + } + return nil + }) + + cID, _ := ts.ContainerID() + oID, _ := ts.ID() + + var tsAddr oid.Address + + tsAddr.SetContainer(cID) + tsAddr.SetObject(oID) + + addrs := make(chan oid.Address, workBufferSize) + + go func() { + defer close(addrs) + + wg := &sync.WaitGroup{} + wg.Add(int(count)) + + for i := uint(0); i < count; i++ { + obj := factory() + + prm := meta.PutPrm{} + prm.SetObject(obj) + + group.Go(func() error { + defer wg.Done() + + if _, err := db.Put(ctx, prm); err != nil { + return fmt.Errorf("couldn't put an object: %w", err) + } + + cID, _ := obj.ContainerID() + oID, _ := obj.ID() + + var addr oid.Address + addr.SetContainer(cID) + addr.SetObject(oID) + + addrs <- addr + return nil + }) + } + wg.Wait() + }() + + go func() { + for addr := range addrs { + prm := meta.InhumePrm{} + prm.SetAddresses(addr) + prm.SetTombstoneAddress(tsAddr) + + group.Go(func() error { + if _, err := db.Inhume(ctx, prm); err != nil { + return fmt.Errorf("couldn't inhume an object: %w", err) + } + return nil + }) + } + }() +} + +func PopulateLocked( + ctx context.Context, + db *meta.DB, + group *errgroup.Group, + workBufferSize int, + count uint, + factory func() *objectSDK.Object, +) { + locker := factory() + locker.SetType(objectSDK.TypeLock) + + prm := meta.PutPrm{} + prm.SetObject(locker) + + group.Go(func() error { + if _, err := db.Put(ctx, prm); err != nil { + return fmt.Errorf("couldn't put a locker object: %w", err) + } + return nil + }) + + ids := make(chan oid.ID, workBufferSize) + + go func() { + defer close(ids) + + wg := &sync.WaitGroup{} + wg.Add(int(count)) + + for i := uint(0); i < count; i++ { + defer wg.Done() + + obj := factory() + + prm := meta.PutPrm{} + prm.SetObject(obj) + + group.Go(func() error { + if _, err := db.Put(ctx, prm); err != nil { + return fmt.Errorf("couldn't put an object: %w", err) + } + + id, _ := obj.ID() + ids <- id + return nil + }) + } + wg.Wait() + }() + + go func() { + for id := range ids { + lockerCID, _ := locker.ContainerID() + lockerOID, _ := locker.ID() + + group.Go(func() error { + if err := db.Lock(ctx, lockerCID, lockerOID, []oid.ID{id}); err != nil { + return fmt.Errorf("couldn't lock an object: %w", err) + } + return nil + }) + } + }() +} diff --git a/scripts/populate-metabase/main.go b/scripts/populate-metabase/main.go new file mode 100644 index 000000000..2bc7a5553 --- /dev/null +++ b/scripts/populate-metabase/main.go @@ -0,0 +1,159 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/scripts/populate-metabase/internal" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "golang.org/x/sync/errgroup" +) + +var ( + path string + force bool + jobs uint + + numContainers, + numObjects, + numAttributesPerObj, + numOwners, + numPayloads, + numAttributes uint +) + +func main() { + flag.StringVar(&path, "path", "", "Path to metabase") + flag.BoolVar(&force, "force", false, "Rewrite existing database") + flag.UintVar(&jobs, "j", 10000, "Number of jobs to run") + + flag.UintVar(&numContainers, "containers", 0, "Number of containers to be created") + flag.UintVar(&numObjects, "objects", 0, "Number of objects per container") + flag.UintVar(&numAttributesPerObj, "attributes", 0, "Number of attributes per object") + + flag.UintVar(&numOwners, "distinct-owners", 10, "Number of distinct owners to be used") + flag.UintVar(&numPayloads, "distinct-payloads", 10, "Number of distinct payloads to be used") + flag.UintVar(&numAttributes, "distinct-attributes", 10, "Number of distinct attributes to be used") + + flag.Parse() + + exitIf(numPayloads == 0, "must have payloads\n") + exitIf(numAttributes == 0, "must have attributes\n") + exitIf(numOwners == 0, "must have owners\n") + exitIf(len(path) == 0, "path to metabase not specified\n") + exitIf( + numAttributesPerObj > numAttributes, + "object can't have more attributes than available\n", + ) + + info, err := os.Stat(path) + exitIf( + err != nil && !errors.Is(err, os.ErrNotExist), + "couldn't get path info: %s\n", err, + ) + + // Path exits. + if err == nil { + exitIf(info.IsDir(), "path is a directory\n") + exitIf(!force, "couldn't rewrite existing file, use '-force' flag\n") + + err = os.Remove(path) + exitIf(err != nil, "couldn't remove existing file: %s\n", err) + } + + err = populate() + exitIf(err != nil, "couldn't populate the metabase: %s\n", err) +} + +func getObjectFactory(opts ...internal.ObjectOption) func() *objectSDK.Object { + return func() *objectSDK.Object { + return internal.GenerateObject(opts...) + } +} + +func populate() (err error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + db := meta.New([]meta.Option{ + meta.WithPath(path), + meta.WithPermissions(0o600), + meta.WithEpochState(internal.EpochState{}), + }...) + + if err = db.Open(ctx, mode.ReadWrite); err != nil { + return fmt.Errorf("couldn't open the metabase: %w", err) + } + defer func() { + if errOnClose := db.Close(); errOnClose != nil { + err = errors.Join( + err, + fmt.Errorf("couldn't close the metabase: %w", db.Close()), + ) + } + }() + + if err = db.Init(); err != nil { + return fmt.Errorf("couldn't init the metabase: %w", err) + } + + payloads := internal.GeneratePayloadPool(numPayloads, 32) + attributes := internal.GenerateAttributePool(numAttributes) + owners := internal.GenerateOwnerPool(numOwners) + + types := []objectSDK.Type{ + objectSDK.TypeRegular, + objectSDK.TypeLock, + objectSDK.TypeTombstone, + } + + eg, ctx := errgroup.WithContext(ctx) + eg.SetLimit(int(jobs)) + + for i := uint(0); i < numContainers; i++ { + cid := cidtest.ID() + + for _, typ := range types { + internal.PopulateWithObjects(ctx, db, eg, numObjects, getObjectFactory( + internal.WithContainerID(cid), + internal.WithType(typ), + internal.WithPayloadFromPool(payloads), + internal.WithOwnerIDFromPool(owners), + internal.WithAttributesFromPool(attributes, numAttributesPerObj), + )) + } + internal.PopulateWithBigObjects(ctx, db, eg, numObjects, getObjectFactory( + internal.WithContainerID(cid), + internal.WithType(objectSDK.TypeRegular), + internal.WithAttributesFromPool(attributes, numAttributesPerObj), + internal.WithOwnerIDFromPool(owners), + )) + internal.PopulateGraveyard(ctx, db, eg, int(jobs), numObjects, getObjectFactory( + internal.WithContainerID(cid), + internal.WithType(objectSDK.TypeRegular), + internal.WithAttributesFromPool(attributes, numAttributesPerObj), + internal.WithOwnerIDFromPool(owners), + )) + internal.PopulateLocked(ctx, db, eg, int(jobs), numObjects, getObjectFactory( + internal.WithContainerID(cid), + internal.WithType(objectSDK.TypeRegular), + internal.WithAttributesFromPool(attributes, numAttributesPerObj), + internal.WithOwnerIDFromPool(owners), + )) + } + + return eg.Wait() +} + +func exitIf(cond bool, format string, args ...any) { + if cond { + fmt.Fprintf(os.Stderr, format, args...) + os.Exit(1) + } +} From 1ae86f35a8d95a3f2258eaefb772482d0af873f6 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 18 Jul 2024 18:26:11 +0300 Subject: [PATCH 1313/1943] [#1223] lens/tui: Add metabase schema Signed-off-by: Aleksey Savchuk --- .../internal/schema/common/format.go | 43 +++ .../internal/schema/common/raw.go | 29 ++ .../internal/schema/common/schema.go | 81 ++++++ .../schema/metabase/buckets/detailed.go | 29 ++ .../schema/metabase/buckets/filter.go | 81 ++++++ .../schema/metabase/buckets/parsers.go | 111 ++++++++ .../schema/metabase/buckets/prefix.go | 53 ++++ .../schema/metabase/buckets/string.go | 48 ++++ .../internal/schema/metabase/buckets/types.go | 166 ++++++++++++ .../internal/schema/metabase/parser.go | 29 ++ .../schema/metabase/records/detailed.go | 65 +++++ .../schema/metabase/records/filter.go | 145 ++++++++++ .../schema/metabase/records/parsers.go | 251 ++++++++++++++++++ .../schema/metabase/records/string.go | 135 ++++++++++ .../internal/schema/metabase/records/types.go | 82 ++++++ .../internal/schema/metabase/records/util.go | 20 ++ go.mod | 5 +- go.sum | 9 + 18 files changed, 1381 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-lens/internal/schema/common/format.go create mode 100644 cmd/frostfs-lens/internal/schema/common/raw.go create mode 100644 cmd/frostfs-lens/internal/schema/common/schema.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/detailed.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/filter.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/string.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/buckets/types.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/parser.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/detailed.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/filter.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/parsers.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/string.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/types.go create mode 100644 cmd/frostfs-lens/internal/schema/metabase/records/util.go diff --git a/cmd/frostfs-lens/internal/schema/common/format.go b/cmd/frostfs-lens/internal/schema/common/format.go new file mode 100644 index 000000000..4ed7e96f2 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/common/format.go @@ -0,0 +1,43 @@ +package common + +import ( + "fmt" + "strconv" + + "github.com/gdamore/tcell/v2" +) + +type FormatOptions struct { + Color tcell.Color + + Bold, + Italic, + Underline, + StrikeThrough bool +} + +func Format(s string, opts FormatOptions) string { + var boldTag, italicTag, underlineTag, strikeThroughTag string + + switch { + case opts.Bold: + boldTag = "b" + case opts.Italic: + italicTag = "i" + case opts.Underline: + underlineTag = "u" + case opts.StrikeThrough: + strikeThroughTag = "s" + } + + attrs := fmt.Sprintf( + "%s%s%s%s", boldTag, italicTag, underlineTag, strikeThroughTag, + ) + color := strconv.FormatInt(int64(opts.Color.Hex()), 16) + + return fmt.Sprintf("[#%06s::%s]%s[-::-]", color, attrs, s) +} + +func FormatSimple(s string, c tcell.Color) string { + return Format(s, FormatOptions{Color: c}) +} diff --git a/cmd/frostfs-lens/internal/schema/common/raw.go b/cmd/frostfs-lens/internal/schema/common/raw.go new file mode 100644 index 000000000..0990e24c3 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/common/raw.go @@ -0,0 +1,29 @@ +package common + +import ( + "github.com/davecgh/go-spew/spew" + "github.com/gdamore/tcell/v2" + "github.com/mr-tron/base58" +) + +type RawEntry struct { + key, value []byte +} + +var RawParser Parser = rawParser + +func rawParser(key, value []byte) (SchemaEntry, Parser, error) { + return &RawEntry{key: key, value: value}, rawParser, nil +} + +func (r *RawEntry) String() string { + return FormatSimple(base58.Encode(r.key), tcell.ColorRed) +} + +func (r *RawEntry) DetailedString() string { + return spew.Sdump(r) +} + +func (r *RawEntry) Filter(string, any) FilterResult { + return No +} diff --git a/cmd/frostfs-lens/internal/schema/common/schema.go b/cmd/frostfs-lens/internal/schema/common/schema.go new file mode 100644 index 000000000..9bad19032 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/common/schema.go @@ -0,0 +1,81 @@ +package common + +import ( + "errors" + "fmt" +) + +type FilterResult byte + +const ( + No FilterResult = iota + Maybe + Yes +) + +func IfThenElse(condition bool, onSuccess, onFailure FilterResult) FilterResult { + var res FilterResult + if condition { + res = onSuccess + } else { + res = onFailure + } + return res +} + +type SchemaEntry interface { + String() string + DetailedString() string + Filter(typ string, val any) FilterResult +} + +type ( + Parser func(key, value []byte) (SchemaEntry, Parser, error) + FallbackParser func(key, value []byte) (SchemaEntry, Parser) +) + +func Any(parsers ...Parser) Parser { + return func(key, value []byte) (SchemaEntry, Parser, error) { + var errs error + for _, parser := range parsers { + ret, next, err := parser(key, value) + if err == nil { + return ret, next, nil + } + errs = errors.Join(errs, err) + } + return nil, nil, fmt.Errorf("no parser succeeded: %w", errs) + } +} + +func WithFallback(parser Parser, fallback FallbackParser) Parser { + if parser == nil { + return fallback.ToParser() + } + return func(key, value []byte) (SchemaEntry, Parser, error) { + entry, next, err := parser(key, value) + if err == nil { + return entry, WithFallback(next, fallback), nil + } + return fallback.ToParser()(key, value) + } +} + +func (fp FallbackParser) ToParser() Parser { + return func(key, value []byte) (SchemaEntry, Parser, error) { + entry, next := fp(key, value) + return entry, next, nil + } +} + +func (p Parser) ToFallbackParser() FallbackParser { + return func(key, value []byte) (SchemaEntry, Parser) { + entry, next, err := p(key, value) + if err != nil { + panic(fmt.Errorf( + "couldn't use that parser as a fallback parser, it returned an error: %w", err, + )) + } + return entry, next + } +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/detailed.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/detailed.go new file mode 100644 index 000000000..6a08a723e --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/detailed.go @@ -0,0 +1,29 @@ +package buckets + +import ( + "github.com/davecgh/go-spew/spew" +) + +func (b *PrefixBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *PrefixContainerBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *UserBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *ContainerBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *UserAttributeKeyBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *UserAttributeValueBucket) DetailedString() string { + return spew.Sdump(*b) +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/filter.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/filter.go new file mode 100644 index 000000000..891c4004f --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/filter.go @@ -0,0 +1,81 @@ +package buckets + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" +) + +func (b *PrefixBucket) Filter(typ string, _ any) common.FilterResult { + switch typ { + case "cid": + return b.resolvers.cidResolver(false) + case "oid": + return b.resolvers.oidResolver(false) + default: + return common.No + } +} + +func (b *PrefixContainerBucket) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return b.resolvers.cidResolver(b.id.Equals(id)) + case "oid": + return b.resolvers.oidResolver(false) + default: + return common.No + } +} + +func (b *UserBucket) Filter(typ string, _ any) common.FilterResult { + switch typ { + case "cid": + return b.resolvers.cidResolver(false) + case "oid": + return b.resolvers.oidResolver(false) + default: + return common.No + } +} + +func (b *ContainerBucket) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return b.resolvers.cidResolver(b.id.Equals(id)) + case "oid": + return b.resolvers.oidResolver(false) + default: + return common.No + } +} + +func (b *UserAttributeKeyBucket) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(b.id.Equals(id), common.Yes, common.No) + case "oid": + return common.Maybe + case "key": + key := val.(string) + return common.IfThenElse(b.key == key, common.Yes, common.No) + case "value": + return common.Maybe + default: + return common.No + } +} + +func (b *UserAttributeValueBucket) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + return common.Maybe + case "value": + value := val.(string) + return common.IfThenElse(b.value == value, common.Yes, common.No) + default: + return common.No + } +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go new file mode 100644 index 000000000..24cc0e52d --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go @@ -0,0 +1,111 @@ +package buckets + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/metabase/records" +) + +var ( + GraveyardParser = NewPrefixBucketParser(Graveyard, records.GraveyardRecordParser, Resolvers{ + cidResolver: LenientResolver, + oidResolver: LenientResolver, + }) + + GarbageParser = NewPrefixBucketParser(Garbage, records.GarbageRecordParser, Resolvers{ + cidResolver: LenientResolver, + oidResolver: LenientResolver, + }) + + ContainerVolumeParser = NewPrefixBucketParser(ContainerVolume, records.ContainerVolumeRecordParser, Resolvers{ + cidResolver: LenientResolver, + oidResolver: StrictResolver, + }) + + LockedParser = NewPrefixBucketParser( + Locked, + NewContainerBucketParser( + records.LockedRecordParser, + Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }, + ), + Resolvers{ + cidResolver: LenientResolver, + oidResolver: LenientResolver, + }, + ) + + ShardInfoParser = NewPrefixBucketParser(ShardInfo, records.ShardInfoRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: StrictResolver, + }) + + PrimaryParser = NewPrefixContainerBucketParser(Primary, records.ObjectRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + LockersParser = NewPrefixContainerBucketParser(Lockers, records.ObjectRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + TombstoneParser = NewPrefixContainerBucketParser(Tombstone, records.ObjectRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + SmallParser = NewPrefixContainerBucketParser(Small, records.SmallRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + RootParser = NewPrefixContainerBucketParser(Root, records.RootRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + OwnerParser = NewPrefixContainerBucketParser( + Owner, + NewUserBucketParser( + records.OwnerRecordParser, + Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }, + ), + Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }, + ) + + UserAttributeParser = NewUserAttributeKeyBucketParser( + NewUserAttributeValueBucketParser(records.UserAttributeRecordParser), + ) + + PayloadHashParser = NewPrefixContainerBucketParser(PayloadHash, records.PayloadHashRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: StrictResolver, + }) + + ParentParser = NewPrefixContainerBucketParser(Parent, records.ParentRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) + + SplitParser = NewPrefixContainerBucketParser(Split, records.SplitRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: StrictResolver, + }) + + ContainerCountersParser = NewPrefixBucketParser(ContainerCounters, records.ContainerCountersRecordParser, Resolvers{ + cidResolver: LenientResolver, + oidResolver: StrictResolver, + }) + + ECInfoParser = NewPrefixContainerBucketParser(ECInfo, records.ECInfoRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) +) diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go new file mode 100644 index 000000000..2fb122940 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go @@ -0,0 +1,53 @@ +package buckets + +type Prefix byte + +const ( + Graveyard Prefix = iota + Garbage + ToMoveIt + ContainerVolume + Locked + ShardInfo + Primary + Lockers + _ + Tombstone + Small + Root + Owner + UserAttribute + PayloadHash + Parent + Split + ContainerCounters + ECInfo +) + +var x = map[Prefix]string{ + Graveyard: "Graveyard", + Garbage: "Garbage", + ToMoveIt: "To Move It", + ContainerVolume: "Container Volume", + Locked: "Locked", + ShardInfo: "Shard Info", + Primary: "Primary", + Lockers: "Lockers", + Tombstone: "Tombstone", + Small: "Small", + Root: "Root", + Owner: "Owner", + UserAttribute: "User Attribute", + PayloadHash: "Payload Hash", + Parent: "Parent", + Split: "Split", + ContainerCounters: "Container Counters", + ECInfo: "EC Info", +} + +func (p Prefix) String() string { + if s, ok := x[p]; ok { + return s + } + return "Unknown Prefix" +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/string.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/string.go new file mode 100644 index 000000000..db90bddbd --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/string.go @@ -0,0 +1,48 @@ +package buckets + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "github.com/gdamore/tcell/v2" +) + +func (b *PrefixBucket) String() string { + return common.FormatSimple( + fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime, + ) +} + +func (b *PrefixContainerBucket) String() string { + return fmt.Sprintf( + "%s CID %s", + common.FormatSimple( + fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime, + ), + common.FormatSimple(b.id.String(), tcell.ColorAqua), + ) +} + +func (b *UserBucket) String() string { + return "UID " + common.FormatSimple(b.id.String(), tcell.ColorAqua) +} + +func (b *ContainerBucket) String() string { + return "CID " + common.FormatSimple(b.id.String(), tcell.ColorAqua) +} + +func (b *UserAttributeKeyBucket) String() string { + return fmt.Sprintf("%s CID %s ATTR-KEY %s", + common.FormatSimple( + fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime, + ), + common.FormatSimple( + fmt.Sprintf("%-44s", b.id), tcell.ColorAqua, + ), + common.FormatSimple(b.key, tcell.ColorAqua), + ) +} + +func (b *UserAttributeValueBucket) String() string { + return "ATTR-VALUE " + common.FormatSimple(b.value, tcell.ColorAqua) +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/types.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/types.go new file mode 100644 index 000000000..82b47dd85 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/types.go @@ -0,0 +1,166 @@ +package buckets + +import ( + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/mr-tron/base58" +) + +type ( + PrefixBucket struct { + prefix Prefix + resolvers Resolvers + } + + PrefixContainerBucket struct { + prefix Prefix + id cid.ID + resolvers Resolvers + } + + ContainerBucket struct { + id cid.ID + resolvers Resolvers + } + + UserBucket struct { + id user.ID + resolvers Resolvers + } + + UserAttributeKeyBucket struct { + prefix Prefix + id cid.ID + key string + } + + UserAttributeValueBucket struct { + value string + } +) + +type ( + FilterResolver = func(result bool) common.FilterResult + + Resolvers struct { + cidResolver FilterResolver + oidResolver FilterResolver + } +) + +var ( + StrictResolver = func(x bool) common.FilterResult { return common.IfThenElse(x, common.Yes, common.No) } + LenientResolver = func(x bool) common.FilterResult { return common.IfThenElse(x, common.Yes, common.Maybe) } +) + +var ( + ErrNotBucket = errors.New("not a bucket") + ErrInvalidKeyLength = errors.New("invalid key length") + ErrInvalidValueLength = errors.New("invalid value length") + ErrInvalidPrefix = errors.New("invalid prefix") +) + +func NewPrefixBucketParser(prefix Prefix, next common.Parser, resolvers Resolvers) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + if len(key) != 1 { + return nil, nil, ErrInvalidKeyLength + } + var b PrefixBucket + if b.prefix = Prefix(key[0]); b.prefix != prefix { + return nil, nil, ErrInvalidPrefix + } + b.resolvers = resolvers + return &b, next, nil + } +} + +func NewPrefixContainerBucketParser(prefix Prefix, next common.Parser, resolvers Resolvers) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + if len(key) != 33 { + return nil, nil, ErrInvalidKeyLength + } + var b PrefixContainerBucket + if b.prefix = Prefix(key[0]); b.prefix != prefix { + return nil, nil, ErrInvalidPrefix + } + if err := b.id.Decode(key[1:]); err != nil { + return nil, nil, err + } + b.resolvers = resolvers + return &b, next, nil + } +} + +func NewUserBucketParser(next common.Parser, resolvers Resolvers) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + var b UserBucket + if err := b.id.DecodeString(base58.Encode(key)); err != nil { + return nil, nil, err + } + b.resolvers = resolvers + return &b, next, nil + } +} + +func NewContainerBucketParser(next common.Parser, resolvers Resolvers) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + if len(key) != 32 { + return nil, nil, ErrInvalidKeyLength + } + var b ContainerBucket + if err := b.id.Decode(key); err != nil { + return nil, nil, err + } + b.resolvers = resolvers + return &b, next, nil + } +} + +func NewUserAttributeKeyBucketParser(next common.Parser) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + if len(key) < 34 { + return nil, nil, ErrInvalidKeyLength + } + var b UserAttributeKeyBucket + if b.prefix = Prefix(key[0]); b.prefix != UserAttribute { + return nil, nil, ErrInvalidPrefix + } + if err := b.id.Decode(key[1:33]); err != nil { + return nil, nil, err + } + b.key = string(key[33:]) + return &b, next, nil + } +} + +func NewUserAttributeValueBucketParser(next common.Parser) common.Parser { + return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, ErrNotBucket + } + if len(key) == 0 { + return nil, nil, ErrInvalidKeyLength + } + var b UserAttributeValueBucket + b.value = string(key) + return &b, next, nil + } +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/parser.go b/cmd/frostfs-lens/internal/schema/metabase/parser.go new file mode 100644 index 000000000..ea095e207 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/parser.go @@ -0,0 +1,29 @@ +package metabase + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/metabase/buckets" +) + +var MetabaseParser = common.WithFallback( + common.Any( + buckets.GraveyardParser, + buckets.GarbageParser, + buckets.ContainerVolumeParser, + buckets.LockedParser, + buckets.ShardInfoParser, + buckets.PrimaryParser, + buckets.LockersParser, + buckets.TombstoneParser, + buckets.SmallParser, + buckets.RootParser, + buckets.OwnerParser, + buckets.UserAttributeParser, + buckets.PayloadHashParser, + buckets.ParentParser, + buckets.SplitParser, + buckets.ContainerCountersParser, + buckets.ECInfoParser, + ), + common.RawParser.ToFallbackParser(), +) diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/detailed.go b/cmd/frostfs-lens/internal/schema/metabase/records/detailed.go new file mode 100644 index 000000000..2dda15b4f --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/detailed.go @@ -0,0 +1,65 @@ +package records + +import ( + "github.com/davecgh/go-spew/spew" +) + +func (r *GraveyardRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *GarbageRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ContainerVolumeRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *LockedRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ShardInfoRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ObjectRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *SmallRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *RootRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *OwnerRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *UserAttributeRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *PayloadHashRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ParentRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *SplitRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ContainerCountersRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ECInfoRecord) DetailedString() string { + return spew.Sdump(*r) +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/filter.go b/cmd/frostfs-lens/internal/schema/metabase/records/filter.go new file mode 100644 index 000000000..880a7a8ff --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/filter.go @@ -0,0 +1,145 @@ +package records + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +func (r *GraveyardRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.object.Container().Equals(id), common.Yes, common.No) + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.object.Object().Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *GarbageRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.addr.Container().Equals(id), common.Yes, common.No) + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.addr.Object().Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *ContainerVolumeRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *ShardInfoRecord) Filter(string, any) common.FilterResult { + return common.No +} + +func (r *LockedRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *ObjectRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *SmallRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *RootRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *OwnerRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *UserAttributeRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *PayloadHashRecord) Filter(string, any) common.FilterResult { + return common.No +} + +func (r *ParentRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.parent.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *SplitRecord) Filter(string, any) common.FilterResult { + return common.No +} + +func (r *ContainerCountersRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *ECInfoRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.id.Equals(id), common.Yes, common.No) + default: + return common.No + } +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/parsers.go b/cmd/frostfs-lens/internal/schema/metabase/records/parsers.go new file mode 100644 index 000000000..1b070e2a0 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/parsers.go @@ -0,0 +1,251 @@ +package records + +import ( + "encoding/binary" + "errors" + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +var ( + ErrInvalidKeyLength = errors.New("invalid key length") + ErrInvalidValueLength = errors.New("invalid value length") + ErrInvalidPrefix = errors.New("invalid prefix") +) + +func GraveyardRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 64 { + return nil, nil, ErrInvalidKeyLength + } + if len(value) != 64 { + return nil, nil, ErrInvalidValueLength + } + var ( + cnr cid.ID + obj oid.ID + r GraveyardRecord + ) + + _ = cnr.Decode(key[:32]) + _ = obj.Decode(key[32:]) + + r.object.SetContainer(cnr) + r.object.SetObject(obj) + + _ = cnr.Decode(value[:32]) + _ = obj.Decode(value[32:]) + + r.tombstone.SetContainer(cnr) + r.tombstone.SetObject(obj) + + return &r, nil, nil +} + +func GarbageRecordParser(key, _ []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 64 { + return nil, nil, ErrInvalidKeyLength + } + var ( + cnr cid.ID + obj oid.ID + r GarbageRecord + ) + + _ = cnr.Decode(key[:32]) + _ = obj.Decode(key[32:]) + + r.addr.SetContainer(cnr) + r.addr.SetObject(obj) + + return &r, nil, nil +} + +func ContainerVolumeRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 32 { + return nil, nil, ErrInvalidKeyLength + } + if len(value) != 8 { + return nil, nil, ErrInvalidValueLength + } + var r ContainerVolumeRecord + + _ = r.id.Decode(key) + r.volume = binary.LittleEndian.Uint64(value) + + return &r, nil, nil +} + +func LockedRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var ( + r LockedRecord + err error + ) + + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + if r.ids, err = DecodeOIDs(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func ShardInfoRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) == 0 { + return nil, nil, ErrInvalidKeyLength + } + + var r ShardInfoRecord + if string(key) == "id" { + r.label = string(key) + r.value = shard.ID(value).String() + + return &r, nil, nil + } + + if len(value) != 8 { + return nil, nil, ErrInvalidValueLength + } + r.label = string(key) + r.value = strconv.FormatUint(binary.LittleEndian.Uint64(value), 10) + + return &r, nil, nil +} + +func ObjectRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 32 { + return nil, nil, ErrInvalidKeyLength + } + var r ObjectRecord + + _ = r.id.Decode(key) + if err := r.object.Unmarshal(value); err != nil { + return nil, nil, err + } + + return &r, nil, nil +} + +func SmallRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var r SmallRecord + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + if len(value) != 0 { + x := string(value) + r.storageID = &x + } + return &r, nil, nil +} + +func RootRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var r RootRecord + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + if len(value) == 0 { + return &r, nil, nil + } + r.info = &objectSDK.SplitInfo{} + if err := r.info.Unmarshal(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func OwnerRecordParser(key, _ []byte) (common.SchemaEntry, common.Parser, error) { + var r OwnerRecord + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func UserAttributeRecordParser(key, _ []byte) (common.SchemaEntry, common.Parser, error) { + var r UserAttributeRecord + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func PayloadHashRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 32 { + return nil, nil, ErrInvalidKeyLength + } + var ( + err error + r PayloadHashRecord + ) + + r.checksum.SetSHA256([32]byte(key)) + if r.ids, err = DecodeOIDs(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func ParentRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var ( + r ParentRecord + err error + ) + if err = r.parent.Decode(key); err != nil { + return nil, nil, err + } + if r.ids, err = DecodeOIDs(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func SplitRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var ( + err error + r SplitRecord + ) + if err = r.id.UnmarshalBinary(key); err != nil { + return nil, nil, err + } + if r.ids, err = DecodeOIDs(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} + +func ContainerCountersRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(value) != 24 { + return nil, nil, ErrInvalidValueLength + } + + var r ContainerCountersRecord + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + + r.logical = binary.LittleEndian.Uint64(value[:8]) + r.physical = binary.LittleEndian.Uint64(value[8:16]) + r.user = binary.LittleEndian.Uint64(value[16:24]) + + return &r, nil, nil +} + +func ECInfoRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var ( + r ECInfoRecord + err error + ) + + if err := r.id.Decode(key); err != nil { + return nil, nil, err + } + if r.ids, err = DecodeOIDs(value); err != nil { + return nil, nil, err + } + return &r, nil, nil +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/string.go b/cmd/frostfs-lens/internal/schema/metabase/records/string.go new file mode 100644 index 000000000..a6c70d537 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/string.go @@ -0,0 +1,135 @@ +package records + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +func (r *GraveyardRecord) String() string { + return fmt.Sprintf( + "Object CID %s OID %s %c Tombstone CID %s OID %s", + common.FormatSimple(fmt.Sprintf("%-44s", r.object.Container()), tcell.ColorAqua), + common.FormatSimple(fmt.Sprintf("%-44s", r.object.Object()), tcell.ColorAqua), + tview.Borders.Vertical, + common.FormatSimple(fmt.Sprintf("%-44s", r.tombstone.Container()), tcell.ColorAqua), + common.FormatSimple(fmt.Sprintf("%-44s", r.tombstone.Object()), tcell.ColorAqua), + ) +} + +func (r *GarbageRecord) String() string { + return fmt.Sprintf( + "CID %-44s OID %-44s", + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Container()), tcell.ColorAqua), + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Object()), tcell.ColorAqua), + ) +} + +func (r *ContainerVolumeRecord) String() string { + return fmt.Sprintf( + "CID %-44s %c %d", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + r.volume, + ) +} + +func (r *LockedRecord) String() string { + return fmt.Sprintf( + "Locker OID %s %c Locked [%d]OID {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + len(r.ids), + ) +} + +func (r *ShardInfoRecord) String() string { + return fmt.Sprintf("%-13s %c %s", r.label, tview.Borders.Vertical, r.value) +} + +func (r *ObjectRecord) String() string { + return fmt.Sprintf( + "OID %s %c Object {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + ) +} + +func (r *SmallRecord) String() string { + s := fmt.Sprintf( + "OID %s %c", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + ) + if r.storageID != nil { + s = fmt.Sprintf("%s %s", s, *r.storageID) + } + return s +} + +func (r *RootRecord) String() string { + s := fmt.Sprintf( + "Root OID %s %c", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + ) + if r.info != nil { + s += " Split info {...}" + } + return s +} + +func (r *OwnerRecord) String() string { + return "OID " + common.FormatSimple(r.id.String(), tcell.ColorAqua) +} + +func (r *UserAttributeRecord) String() string { + return "OID " + common.FormatSimple(r.id.String(), tcell.ColorAqua) +} + +func (r *PayloadHashRecord) String() string { + return fmt.Sprintf( + "Checksum %s %c [%d]OID {...}", + common.FormatSimple(r.checksum.String(), tcell.ColorAqua), + tview.Borders.Vertical, + len(r.ids), + ) +} + +func (r *ParentRecord) String() string { + return fmt.Sprintf( + "Parent OID %s %c [%d]OID {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.parent), tcell.ColorAqua), + tview.Borders.Vertical, + len(r.ids), + ) +} + +func (r *SplitRecord) String() string { + return fmt.Sprintf( + "Split ID %s %c [%d]OID {...}", + common.FormatSimple(r.id.String(), tcell.ColorAqua), + tview.Borders.Vertical, + len(r.ids), + ) +} + +func (r *ContainerCountersRecord) String() string { + return fmt.Sprintf( + "CID %s %c logical %d, physical %d, user %d", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + r.logical, r.physical, r.user, + ) +} + +func (r *ECInfoRecord) String() string { + return fmt.Sprintf( + "OID %s %c [%d]OID {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), + tview.Borders.Vertical, + len(r.ids), + ) +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/types.go b/cmd/frostfs-lens/internal/schema/metabase/records/types.go new file mode 100644 index 000000000..34c1c29fd --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/types.go @@ -0,0 +1,82 @@ +package records + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/google/uuid" +) + +type ( + GraveyardRecord struct { + object, tombstone oid.Address + } + + GarbageRecord struct { + addr oid.Address + } + + ContainerVolumeRecord struct { + id cid.ID + volume uint64 + } + + LockedRecord struct { + id oid.ID + ids []oid.ID + } + + ShardInfoRecord struct { + label string + value string + } + + ObjectRecord struct { + id oid.ID + object objectSDK.Object + } + + SmallRecord struct { + id oid.ID + storageID *string // optional + } + + RootRecord struct { + id oid.ID + info *objectSDK.SplitInfo // optional + } + + OwnerRecord struct { + id oid.ID + } + + UserAttributeRecord struct { + id oid.ID + } + + PayloadHashRecord struct { + checksum checksum.Checksum + ids []oid.ID + } + + ParentRecord struct { + parent oid.ID + ids []oid.ID + } + + SplitRecord struct { + id uuid.UUID + ids []oid.ID + } + + ContainerCountersRecord struct { + id cid.ID + logical, physical, user uint64 + } + + ECInfoRecord struct { + id oid.ID + ids []oid.ID + } +) diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/util.go b/cmd/frostfs-lens/internal/schema/metabase/records/util.go new file mode 100644 index 000000000..f50ebe951 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/metabase/records/util.go @@ -0,0 +1,20 @@ +package records + +import ( + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/nspcc-dev/neo-go/pkg/io" +) + +func DecodeOIDs(data []byte) ([]oid.ID, error) { + r := io.NewBinReaderFromBuf(data) + + size := r.ReadVarUint() + oids := make([]oid.ID, size) + + for i := uint64(0); i < size; i++ { + if err := oids[i].Decode(r.ReadVarBytes()); err != nil { + return nil, err + } + } + return oids, nil +} diff --git a/go.mod b/go.mod index 19bf7852f..be3c6e74d 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,9 @@ require ( github.com/VictoriaMetrics/easyproto v0.1.4 github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 + github.com/gdamore/tcell/v2 v2.7.4 github.com/go-pkgz/expirable-cache/v3 v3.0.0 github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 @@ -65,10 +67,10 @@ require ( github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/gdamore/encoding v1.0.0 // indirect github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -85,6 +87,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/klauspost/reedsolomon v1.12.1 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/minio/sha256-simd v1.0.1 // indirect diff --git a/go.sum b/go.sum index 8ebd59157..d0218a348 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,10 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= +github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= +github.com/gdamore/tcell/v2 v2.7.4 h1:sg6/UnTM9jGpZU+oFYAsDahfchWAFW8Xx2yFinNSAYU= +github.com/gdamore/tcell/v2 v2.7.4/go.mod h1:dSXtXTSK0VsW1biw65DZLZ2NKr7j0qP/0J7ONmsraWg= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -142,6 +146,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -218,6 +224,7 @@ github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5E github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= @@ -352,6 +359,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -359,6 +367,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 9cbd32bce8524e32711287f86196382517f37562 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 16 Aug 2024 14:33:03 +0300 Subject: [PATCH 1314/1943] [#1223] lens/tui: Add writecache schema Signed-off-by: Aleksey Savchuk --- .../internal/schema/writecache/parsers.go | 63 ++++++++++++++++++ .../internal/schema/writecache/types.go | 66 +++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 cmd/frostfs-lens/internal/schema/writecache/parsers.go create mode 100644 cmd/frostfs-lens/internal/schema/writecache/types.go diff --git a/cmd/frostfs-lens/internal/schema/writecache/parsers.go b/cmd/frostfs-lens/internal/schema/writecache/parsers.go new file mode 100644 index 000000000..7d70b27b2 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/writecache/parsers.go @@ -0,0 +1,63 @@ +package writecache + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/mr-tron/base58" +) + +var WritecacheParser = common.WithFallback( + DefaultBucketParser, + common.RawParser.ToFallbackParser(), +) + +func DefaultBucketParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, errors.New("not a bucket") + } + if !bytes.Equal(key, []byte{0}) { + return nil, nil, errors.New("invalid key") + } + return &DefaultBucket{}, DefaultRecordParser, nil +} + +func DefaultRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + parts := strings.Split(string(key), "/") + + if len(parts) != 2 { + return nil, nil, errors.New("invalid key, expected address string /") + } + + cnrRaw, err := base58.Decode(parts[0]) + if err != nil { + return nil, nil, errors.New("can't decode CID string") + } + objRaw, err := base58.Decode(parts[1]) + if err != nil { + return nil, nil, errors.New("can't decode OID string") + } + + cnr := cid.ID{} + if err := cnr.Decode(cnrRaw); err != nil { + return nil, nil, fmt.Errorf("can't decode CID: %w", err) + } + obj := oid.ID{} + if err := obj.Decode(objRaw); err != nil { + return nil, nil, fmt.Errorf("can't decode OID: %w", err) + } + + var r DefaultRecord + + r.addr.SetContainer(cnr) + r.addr.SetObject(obj) + + r.data = value[:] + + return &r, nil, nil +} diff --git a/cmd/frostfs-lens/internal/schema/writecache/types.go b/cmd/frostfs-lens/internal/schema/writecache/types.go new file mode 100644 index 000000000..3f71c5366 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/writecache/types.go @@ -0,0 +1,66 @@ +package writecache + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/davecgh/go-spew/spew" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type ( + DefaultBucket struct{} + + DefaultRecord struct { + addr oid.Address + data []byte + } +) + +func (b *DefaultBucket) String() string { + return common.FormatSimple("0 Default", tcell.ColorLime) +} + +func (r *DefaultRecord) String() string { + return fmt.Sprintf( + "CID %s OID %s %c Data {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Container()), tcell.ColorAqua), + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Object()), tcell.ColorAqua), + tview.Borders.Vertical, + ) +} + +func (b *DefaultBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (r *DefaultRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (b *DefaultBucket) Filter(typ string, _ any) common.FilterResult { + switch typ { + case "cid": + return common.Maybe + case "oid": + return common.Maybe + default: + return common.No + } +} + +func (r *DefaultRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.addr.Container().Equals(id), common.Yes, common.No) + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.addr.Object().Equals(id), common.Yes, common.No) + default: + return common.No + } +} From ed396448acbd3792da3ae0b756233e8c8fe67fee Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 11 Jul 2024 19:39:54 +0300 Subject: [PATCH 1315/1943] [#1223] lens/tui: Add TUI app to explore metabase Signed-off-by: Aleksey Savchuk --- cmd/frostfs-lens/internal/meta/root.go | 1 + cmd/frostfs-lens/internal/meta/tui.go | 82 ++++ cmd/frostfs-lens/internal/tui/buckets.go | 257 ++++++++++ cmd/frostfs-lens/internal/tui/db.go | 160 +++++++ cmd/frostfs-lens/internal/tui/detailed.go | 24 + cmd/frostfs-lens/internal/tui/filter.go | 44 ++ cmd/frostfs-lens/internal/tui/input.go | 77 +++ cmd/frostfs-lens/internal/tui/loading.go | 72 +++ cmd/frostfs-lens/internal/tui/records.go | 271 +++++++++++ cmd/frostfs-lens/internal/tui/types.go | 18 + cmd/frostfs-lens/internal/tui/ui.go | 548 ++++++++++++++++++++++ cmd/frostfs-lens/internal/tui/util.go | 97 ++++ go.mod | 3 +- go.sum | 6 +- 14 files changed, 1657 insertions(+), 3 deletions(-) create mode 100644 cmd/frostfs-lens/internal/meta/tui.go create mode 100644 cmd/frostfs-lens/internal/tui/buckets.go create mode 100644 cmd/frostfs-lens/internal/tui/db.go create mode 100644 cmd/frostfs-lens/internal/tui/detailed.go create mode 100644 cmd/frostfs-lens/internal/tui/filter.go create mode 100644 cmd/frostfs-lens/internal/tui/input.go create mode 100644 cmd/frostfs-lens/internal/tui/loading.go create mode 100644 cmd/frostfs-lens/internal/tui/records.go create mode 100644 cmd/frostfs-lens/internal/tui/types.go create mode 100644 cmd/frostfs-lens/internal/tui/ui.go create mode 100644 cmd/frostfs-lens/internal/tui/util.go diff --git a/cmd/frostfs-lens/internal/meta/root.go b/cmd/frostfs-lens/internal/meta/root.go index 6741abd0c..351d1ce80 100644 --- a/cmd/frostfs-lens/internal/meta/root.go +++ b/cmd/frostfs-lens/internal/meta/root.go @@ -32,6 +32,7 @@ func init() { inspectCMD, listGraveyardCMD, listGarbageCMD, + tuiCMD, ) } diff --git a/cmd/frostfs-lens/internal/meta/tui.go b/cmd/frostfs-lens/internal/meta/tui.go new file mode 100644 index 000000000..00e8bf117 --- /dev/null +++ b/cmd/frostfs-lens/internal/meta/tui.go @@ -0,0 +1,82 @@ +package meta + +import ( + "context" + "fmt" + + common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" + schema "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui" + "github.com/rivo/tview" + "github.com/spf13/cobra" + "go.etcd.io/bbolt" +) + +var tuiCMD = &cobra.Command{ + Use: "explore", + Short: "Metabase exploration with a terminal UI", + Long: `Launch a terminal UI to explore metabase and search for data. + +Available search filters: +- cid CID +- oid OID +- addr CID/OID +- attr key[/value] +`, + Run: tuiFunc, +} + +var initialPrompt string + +func init() { + common.AddComponentPathFlag(tuiCMD, &vPath) + + tuiCMD.Flags().StringVar( + &initialPrompt, + "filter", + "", + "Filter prompt to start with, format 'tag:value [+ tag:value]...'", + ) +} + +func tuiFunc(cmd *cobra.Command, _ []string) { + common.ExitOnErr(cmd, runTUI(cmd)) +} + +func runTUI(cmd *cobra.Command) error { + db, err := openDB(false) + if err != nil { + return fmt.Errorf("couldn't open database: %w", err) + } + defer db.Close() + + // Need if app was stopped with Ctrl-C. + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() + + app := tview.NewApplication() + ui := tui.NewUI(ctx, app, db, schema.MetabaseParser, nil) + + _ = ui.AddFilter("cid", tui.CIDParser, "CID") + _ = ui.AddFilter("oid", tui.OIDParser, "OID") + _ = ui.AddCompositeFilter("addr", tui.AddressParser, "CID/OID") + _ = ui.AddCompositeFilter("attr", tui.AttributeParser, "key[/value]") + + err = ui.WithPrompt(initialPrompt) + if err != nil { + return fmt.Errorf("invalid filter prompt: %w", err) + } + + app.SetRoot(ui, true).SetFocus(ui) + return app.Run() +} + +func openDB(writable bool) (*bbolt.DB, error) { + db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{ + ReadOnly: !writable, + }) + if err != nil { + return nil, err + } + return db, nil +} diff --git a/cmd/frostfs-lens/internal/tui/buckets.go b/cmd/frostfs-lens/internal/tui/buckets.go new file mode 100644 index 000000000..3f5088e7a --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/buckets.go @@ -0,0 +1,257 @@ +package tui + +import ( + "context" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type BucketsView struct { + *tview.Box + + mu sync.Mutex + + view *tview.TreeView + nodeToUpdate *tview.TreeNode + + ui *UI + filter *Filter +} + +type bucketNode struct { + bucket *Bucket + filter *Filter +} + +func NewBucketsView(ui *UI, filter *Filter) *BucketsView { + return &BucketsView{ + Box: tview.NewBox(), + view: tview.NewTreeView(), + ui: ui, + filter: filter, + } +} + +func (v *BucketsView) Mount(_ context.Context) error { + root := tview.NewTreeNode(".") + root.SetExpanded(false) + root.SetSelectable(false) + root.SetReference(&bucketNode{ + bucket: &Bucket{NextParser: v.ui.rootParser}, + filter: v.filter, + }) + + v.nodeToUpdate = root + + v.view.SetRoot(root) + v.view.SetCurrentNode(root) + + return nil +} + +func (v *BucketsView) Update(ctx context.Context) error { + if v.nodeToUpdate == nil { + return nil + } + defer func() { v.nodeToUpdate = nil }() + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + ready := make(chan struct{}) + errCh := make(chan error) + + tmp := tview.NewTreeNode(v.nodeToUpdate.GetText()) + tmp.SetReference(v.nodeToUpdate.GetReference()) + + node := v.nodeToUpdate.GetReference().(*bucketNode) + + go func() { + defer close(ready) + + hasBuckets, err := HasBuckets(ctx, v.ui.db, node.bucket.Path) + if err != nil { + errCh <- err + } + + // Show the selected bucket's records instead. + if !hasBuckets && node.bucket.NextParser != nil { + v.ui.moveNextPage(NewRecordsView(v.ui, node.bucket, node.filter)) + } + + if v.nodeToUpdate.IsExpanded() { + return + } + + err = v.loadNodeChildren(ctx, tmp, node.filter) + if err != nil { + errCh <- err + } + }() + + select { + case <-ctx.Done(): + case <-ready: + v.mu.Lock() + v.nodeToUpdate.SetChildren(tmp.GetChildren()) + v.nodeToUpdate.SetExpanded(!v.nodeToUpdate.IsExpanded()) + v.mu.Unlock() + case err := <-errCh: + return err + } + + return nil +} + +func (v *BucketsView) Unmount() { +} + +func (v *BucketsView) Draw(screen tcell.Screen) { + x, y, width, height := v.GetInnerRect() + v.view.SetRect(x, y, width, height) + + v.view.Draw(screen) +} + +func (v *BucketsView) loadNodeChildren( + ctx context.Context, node *tview.TreeNode, filter *Filter, +) error { + parentBucket := node.GetReference().(*bucketNode).bucket + + path := parentBucket.Path + parser := parentBucket.NextParser + + buffer, err := LoadBuckets(ctx, v.ui.db, path, v.ui.loadBufferSize) + if err != nil { + return err + } + + for item := range buffer { + if item.err != nil { + return item.err + } + bucket := item.val + + bucket.Entry, bucket.NextParser, err = parser(bucket.Name, nil) + if err != nil { + return err + } + + satisfies, err := v.bucketSatisfiesFilter(ctx, bucket, filter) + if err != nil { + return err + } + if !satisfies { + continue + } + + child := tview.NewTreeNode(bucket.Entry.String()). + SetSelectable(true). + SetExpanded(false). + SetReference(&bucketNode{ + bucket: bucket, + filter: filter.Apply(bucket.Entry), + }) + + node.AddChild(child) + } + + return nil +} + +func (v *BucketsView) bucketSatisfiesFilter( + ctx context.Context, bucket *Bucket, filter *Filter, +) (bool, error) { + // Does the current bucket satisfies the filter? + filter = filter.Apply(bucket.Entry) + + if filter.Result() == common.Yes { + return true, nil + } + + if filter.Result() == common.No { + return false, nil + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Check the current bucket's nested buckets if exist + bucketsBuffer, err := LoadBuckets(ctx, v.ui.db, bucket.Path, v.ui.loadBufferSize) + if err != nil { + return false, err + } + + for item := range bucketsBuffer { + if item.err != nil { + return false, item.err + } + b := item.val + + b.Entry, b.NextParser, err = bucket.NextParser(b.Name, nil) + if err != nil { + return false, err + } + + satisfies, err := v.bucketSatisfiesFilter(ctx, b, filter) + if err != nil { + return false, err + } + if satisfies { + return true, nil + } + } + + // Check the current bucket's nested records if exist + recordsBuffer, err := LoadRecords(ctx, v.ui.db, bucket.Path, v.ui.loadBufferSize) + if err != nil { + return false, err + } + + for item := range recordsBuffer { + if item.err != nil { + return false, item.err + } + r := item.val + + r.Entry, _, err = bucket.NextParser(r.Key, r.Value) + if err != nil { + return false, err + } + + if filter.Apply(r.Entry).Result() == common.Yes { + return true, nil + } + } + + return false, nil +} + +func (v *BucketsView) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) { + return v.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) { + currentNode := v.view.GetCurrentNode() + if currentNode == nil { + return + } + + switch event.Key() { + case tcell.KeyEnter: + // Expand or collapse the selected bucket's nested buckets, + // otherwise, navigate to that bucket's records. + v.nodeToUpdate = currentNode + case tcell.KeyCtrlR: + // Navigate to the selected bucket's records. + bucketNode := currentNode.GetReference().(*bucketNode) + v.ui.moveNextPage(NewRecordsView(v.ui, bucketNode.bucket, bucketNode.filter)) + case tcell.KeyCtrlD: + // Navigate to the selected bucket's detailed view. + bucketNode := currentNode.GetReference().(*bucketNode) + v.ui.moveNextPage(NewDetailedView(bucketNode.bucket.Entry.DetailedString())) + default: + v.view.InputHandler()(event, func(tview.Primitive) {}) + } + }) +} diff --git a/cmd/frostfs-lens/internal/tui/db.go b/cmd/frostfs-lens/internal/tui/db.go new file mode 100644 index 000000000..d0cf611d4 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/db.go @@ -0,0 +1,160 @@ +package tui + +import ( + "context" + "errors" + "fmt" + + "go.etcd.io/bbolt" +) + +type Item[T any] struct { + val T + err error +} + +func resolvePath(tx *bbolt.Tx, path [][]byte) (*bbolt.Bucket, error) { + if len(path) == 0 { + return nil, errors.New("can't find bucket without path") + } + + name := path[0] + bucket := tx.Bucket(name) + if bucket == nil { + return nil, fmt.Errorf("no bucket with name %s", name) + } + for _, name := range path[1:] { + bucket = bucket.Bucket(name) + if bucket == nil { + return nil, fmt.Errorf("no bucket with name %s", name) + } + } + return bucket, nil +} + +func load[T any]( + ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int, + filter func(key, value []byte) bool, transform func(key, value []byte) T, +) (<-chan Item[T], error) { + buffer := make(chan Item[T], bufferSize) + + go func() { + defer close(buffer) + + err := db.View(func(tx *bbolt.Tx) error { + var cursor *bbolt.Cursor + if len(path) == 0 { + cursor = tx.Cursor() + } else { + bucket, err := resolvePath(tx, path) + if err != nil { + buffer <- Item[T]{err: fmt.Errorf("can't find bucket: %w", err)} + return nil + } + cursor = bucket.Cursor() + } + + key, value := cursor.First() + for { + if key == nil { + return nil + } + if filter != nil && !filter(key, value) { + key, value = cursor.Next() + continue + } + + select { + case <-ctx.Done(): + return nil + case buffer <- Item[T]{val: transform(key, value)}: + key, value = cursor.Next() + } + } + }) + if err != nil { + buffer <- Item[T]{err: err} + } + }() + + return buffer, nil +} + +func LoadBuckets( + ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int, +) (<-chan Item[*Bucket], error) { + buffer, err := load( + ctx, db, path, bufferSize, + func(_, value []byte) bool { + return value == nil + }, + func(key, _ []byte) *Bucket { + base := make([][]byte, 0, len(path)) + base = append(base, path...) + + return &Bucket{ + Name: key, + Path: append(base, key), + } + }, + ) + if err != nil { + return nil, fmt.Errorf("can't start iterating bucket: %w", err) + } + + return buffer, nil +} + +func LoadRecords( + ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int, +) (<-chan Item[*Record], error) { + buffer, err := load( + ctx, db, path, bufferSize, + func(_, value []byte) bool { + return value != nil + }, + func(key, value []byte) *Record { + base := make([][]byte, 0, len(path)) + base = append(base, path...) + + return &Record{ + Key: key, + Value: value, + Path: append(base, key), + } + }, + ) + if err != nil { + return nil, fmt.Errorf("can't start iterating bucket: %w", err) + } + + return buffer, nil +} + +// HasBuckets checks if a bucket has nested buckets. It relies on assumption +// that a bucket can have either nested buckets or records but not both. +func HasBuckets(ctx context.Context, db *bbolt.DB, path [][]byte) (bool, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + buffer, err := load( + ctx, db, path, 1, + nil, + func(_, value []byte) []byte { return value }, + ) + if err != nil { + return false, err + } + + x, ok := <-buffer + if !ok { + return false, nil + } + if x.err != nil { + return false, err + } + if x.val != nil { + return false, err + } + return true, nil +} diff --git a/cmd/frostfs-lens/internal/tui/detailed.go b/cmd/frostfs-lens/internal/tui/detailed.go new file mode 100644 index 000000000..b2d897230 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/detailed.go @@ -0,0 +1,24 @@ +package tui + +import ( + "context" + + "github.com/rivo/tview" +) + +type DetailedView struct { + *tview.TextView +} + +func NewDetailedView(detailed string) *DetailedView { + v := &DetailedView{ + TextView: tview.NewTextView(), + } + v.SetDynamicColors(true) + v.SetText(detailed) + return v +} + +func (v *DetailedView) Mount(_ context.Context) error { return nil } +func (v *DetailedView) Update(_ context.Context) error { return nil } +func (v *DetailedView) Unmount() {} diff --git a/cmd/frostfs-lens/internal/tui/filter.go b/cmd/frostfs-lens/internal/tui/filter.go new file mode 100644 index 000000000..e7879eca7 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/filter.go @@ -0,0 +1,44 @@ +package tui + +import ( + "maps" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" +) + +type Filter struct { + values map[string]any + results map[string]common.FilterResult +} + +func NewFilter(values map[string]any) *Filter { + f := &Filter{ + values: maps.Clone(values), + results: make(map[string]common.FilterResult), + } + for tag := range values { + f.results[tag] = common.No + } + return f +} + +func (f *Filter) Apply(e common.SchemaEntry) *Filter { + filter := &Filter{ + values: f.values, + results: maps.Clone(f.results), + } + + for tag, value := range filter.values { + filter.results[tag] = max(filter.results[tag], e.Filter(tag, value)) + } + + return filter +} + +func (f *Filter) Result() common.FilterResult { + current := common.Yes + for _, r := range f.results { + current = min(r, current) + } + return current +} diff --git a/cmd/frostfs-lens/internal/tui/input.go b/cmd/frostfs-lens/internal/tui/input.go new file mode 100644 index 000000000..4fdf97119 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/input.go @@ -0,0 +1,77 @@ +package tui + +import ( + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type InputFieldWithHistory struct { + *tview.InputField + history []string + historyLimit int + historyPointer int + currentContent string +} + +func NewInputFieldWithHistory(historyLimit int) *InputFieldWithHistory { + return &InputFieldWithHistory{ + InputField: tview.NewInputField(), + historyLimit: historyLimit, + } +} + +func (f *InputFieldWithHistory) AddToHistory(s string) { + // Stop scrolling history on history change, need to start scrolling again. + defer func() { f.historyPointer = len(f.history) }() + + // Used history data for search prompt, so just make that data recent. + if f.historyPointer != len(f.history) && s == f.history[f.historyPointer] { + f.history = append(f.history[:f.historyPointer], f.history[f.historyPointer+1:]...) + f.history = append(f.history, s) + } + + if len(f.history) == f.historyLimit { + f.history = f.history[1:] + } + f.history = append(f.history, s) +} + +func (f *InputFieldWithHistory) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) { + return f.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) { + switch event.Key() { + case tcell.KeyDown: + if len(f.history) == 0 { + return + } + // Need to start iterating before. + if f.historyPointer == len(f.history) { + return + } + // Iterate to most recent prompts. + f.historyPointer++ + // Stop iterating over history. + if f.historyPointer == len(f.history) { + f.InputField.SetText(f.currentContent) + return + } + f.InputField.SetText(f.history[f.historyPointer]) + case tcell.KeyUp: + if len(f.history) == 0 { + return + } + // Start iterating over history. + if f.historyPointer == len(f.history) { + f.currentContent = f.InputField.GetText() + } + // End of history. + if f.historyPointer == 0 { + return + } + // Iterate to least recent prompts. + f.historyPointer-- + f.InputField.SetText(f.history[f.historyPointer]) + default: + f.InputField.InputHandler()(event, func(tview.Primitive) {}) + } + }) +} diff --git a/cmd/frostfs-lens/internal/tui/loading.go b/cmd/frostfs-lens/internal/tui/loading.go new file mode 100644 index 000000000..4b9384ad4 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/loading.go @@ -0,0 +1,72 @@ +package tui + +import ( + "context" + "fmt" + "sync/atomic" + "time" + + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type LoadingBar struct { + *tview.Box + view *tview.TextView + secondsElapsed atomic.Int64 + needDrawFunc func() + reset func() +} + +func NewLoadingBar(needDrawFunc func()) *LoadingBar { + b := &LoadingBar{ + Box: tview.NewBox(), + view: tview.NewTextView(), + needDrawFunc: needDrawFunc, + } + b.view.SetBackgroundColor(tview.Styles.PrimaryTextColor) + b.view.SetTextColor(b.GetBackgroundColor()) + + return b +} + +func (b *LoadingBar) Start(ctx context.Context) { + ctx, b.reset = context.WithCancel(ctx) + + go func() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + b.secondsElapsed.Store(0) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + b.secondsElapsed.Add(1) + b.needDrawFunc() + } + } + }() +} + +func (b *LoadingBar) Stop() { + b.reset() +} + +func (b *LoadingBar) Draw(screen tcell.Screen) { + seconds := b.secondsElapsed.Load() + + var time string + switch { + case seconds < 60: + time = fmt.Sprintf("%ds", seconds) + default: + time = fmt.Sprintf("%dm%ds", seconds/60, seconds%60) + } + b.view.SetText(fmt.Sprintf(" Loading... %s (press Escape to cancel) ", time)) + + x, y, width, _ := b.GetInnerRect() + b.view.SetRect(x, y, width, 1) + b.view.Draw(screen) +} diff --git a/cmd/frostfs-lens/internal/tui/records.go b/cmd/frostfs-lens/internal/tui/records.go new file mode 100644 index 000000000..5f53ed287 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/records.go @@ -0,0 +1,271 @@ +package tui + +import ( + "context" + "errors" + "fmt" + "math" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type updateType int + +const ( + other updateType = iota + moveToPrevPage + moveToNextPage + moveUp + moveDown + moveHome + moveEnd +) + +type RecordsView struct { + *tview.Box + + mu sync.RWMutex + + onUnmount func() + + bucket *Bucket + records []*Record + + buffer chan *Record + + firstRecordIndex int + lastRecordIndex int + selectedRecordIndex int + + updateType updateType + + ui *UI + filter *Filter +} + +func NewRecordsView(ui *UI, bucket *Bucket, filter *Filter) *RecordsView { + return &RecordsView{ + Box: tview.NewBox(), + bucket: bucket, + ui: ui, + filter: filter, + } +} + +func (v *RecordsView) Mount(ctx context.Context) error { + if v.onUnmount != nil { + return errors.New("try to mount already mounted component") + } + + ctx, v.onUnmount = context.WithCancel(ctx) + + tempBuffer, err := LoadRecords(ctx, v.ui.db, v.bucket.Path, v.ui.loadBufferSize) + if err != nil { + return err + } + + v.buffer = make(chan *Record, v.ui.loadBufferSize) + go func() { + defer close(v.buffer) + + for item := range tempBuffer { + if item.err != nil { + v.ui.stopOnError(err) + break + } + record := item.val + + record.Entry, _, err = v.bucket.NextParser(record.Key, record.Value) + if err != nil { + v.ui.stopOnError(err) + break + } + + if v.filter.Apply(record.Entry).Result() != common.Yes { + continue + } + + v.buffer <- record + } + }() + + return nil +} + +func (v *RecordsView) Unmount() { + if v.onUnmount == nil { + panic("try to unmount not mounted component") + } + v.onUnmount() + v.onUnmount = nil +} + +func (v *RecordsView) Update(ctx context.Context) error { + _, _, _, recordsPerPage := v.GetInnerRect() + firstRecordIndex, lastRecordIndex, selectedRecordIndex := v.getNewIndexes() + +loop: + for len(v.records) < lastRecordIndex { + select { + case <-ctx.Done(): + return nil + case record, ok := <-v.buffer: + if !ok { + break loop + } + v.records = append(v.records, record) + } + } + + // Set the update type to its default value after some specific key event + // has been handled. + v.updateType = other + + firstRecordIndex = max(0, min(firstRecordIndex, len(v.records)-recordsPerPage)) + lastRecordIndex = min(firstRecordIndex+recordsPerPage, len(v.records)) + selectedRecordIndex = min(selectedRecordIndex, lastRecordIndex-1) + + v.mu.Lock() + v.firstRecordIndex = firstRecordIndex + v.lastRecordIndex = lastRecordIndex + v.selectedRecordIndex = selectedRecordIndex + v.mu.Unlock() + + return nil +} + +func (v *RecordsView) getNewIndexes() (int, int, int) { + v.mu.RLock() + firstRecordIndex := v.firstRecordIndex + lastRecordIndex := v.lastRecordIndex + selectedRecordIndex := v.selectedRecordIndex + v.mu.RUnlock() + + _, _, _, recordsPerPage := v.GetInnerRect() + + switch v.updateType { + case moveUp: + if selectedRecordIndex != firstRecordIndex { + selectedRecordIndex-- + break + } + firstRecordIndex = max(0, firstRecordIndex-1) + lastRecordIndex = min(firstRecordIndex+recordsPerPage, len(v.records)) + selectedRecordIndex = firstRecordIndex + case moveToPrevPage: + if selectedRecordIndex != firstRecordIndex { + selectedRecordIndex = firstRecordIndex + break + } + firstRecordIndex = max(0, firstRecordIndex-recordsPerPage) + lastRecordIndex = firstRecordIndex + recordsPerPage + selectedRecordIndex = firstRecordIndex + case moveDown: + if selectedRecordIndex != lastRecordIndex-1 { + selectedRecordIndex++ + break + } + firstRecordIndex++ + lastRecordIndex++ + selectedRecordIndex++ + case moveToNextPage: + if selectedRecordIndex != lastRecordIndex-1 { + selectedRecordIndex = lastRecordIndex - 1 + break + } + firstRecordIndex += recordsPerPage + lastRecordIndex = firstRecordIndex + recordsPerPage + selectedRecordIndex = lastRecordIndex - 1 + case moveHome: + firstRecordIndex = 0 + lastRecordIndex = firstRecordIndex + recordsPerPage + selectedRecordIndex = 0 + case moveEnd: + lastRecordIndex = math.MaxInt32 + firstRecordIndex = lastRecordIndex - recordsPerPage + selectedRecordIndex = lastRecordIndex - 1 + default: + lastRecordIndex = firstRecordIndex + recordsPerPage + } + + return firstRecordIndex, lastRecordIndex, selectedRecordIndex +} + +func (v *RecordsView) GetInnerRect() (int, int, int, int) { + x, y, width, height := v.Box.GetInnerRect() + + // Left padding. + x = min(x+3, x+width-1) + width = max(width-3, 0) + + return x, y, width, height +} + +func (v *RecordsView) Draw(screen tcell.Screen) { + v.mu.RLock() + firstRecordIndex := v.firstRecordIndex + lastRecordIndex := v.lastRecordIndex + selectedRecordIndex := v.selectedRecordIndex + records := v.records + v.mu.RUnlock() + + v.DrawForSubclass(screen, v) + + x, y, width, height := v.GetInnerRect() + if height == 0 { + return + } + + // No records in that bucket. + if firstRecordIndex == lastRecordIndex { + tview.Print( + screen, "Empty Bucket", x, y, width, tview.AlignCenter, tview.Styles.PrimaryTextColor, + ) + return + } + + for index := firstRecordIndex; index < lastRecordIndex; index++ { + result := records[index].Entry + text := result.String() + + if index == selectedRecordIndex { + text = fmt.Sprintf("[:white]%s[:-]", text) + tview.Print(screen, text, x, y, width, tview.AlignLeft, tview.Styles.PrimitiveBackgroundColor) + } else { + tview.Print(screen, text, x, y, width, tview.AlignLeft, tview.Styles.PrimaryTextColor) + } + + y++ + } +} + +func (v *RecordsView) InputHandler() func(event *tcell.EventKey, _ func(p tview.Primitive)) { + return v.WrapInputHandler(func(event *tcell.EventKey, _ func(p tview.Primitive)) { + switch m, k := event.Modifiers(), event.Key(); { + case m == 0 && k == tcell.KeyPgUp: + v.updateType = moveToPrevPage + case m == 0 && k == tcell.KeyPgDn: + v.updateType = moveToNextPage + case m == 0 && k == tcell.KeyUp: + v.updateType = moveUp + case m == 0 && k == tcell.KeyDown: + v.updateType = moveDown + case m == 0 && k == tcell.KeyHome: + v.updateType = moveHome + case m == 0 && k == tcell.KeyEnd: + v.updateType = moveEnd + case k == tcell.KeyEnter: + v.mu.RLock() + selectedRecordIndex := v.selectedRecordIndex + records := v.records + v.mu.RUnlock() + if len(records) != 0 { + current := records[selectedRecordIndex] + v.ui.moveNextPage(NewDetailedView(current.Entry.DetailedString())) + } + } + }) +} diff --git a/cmd/frostfs-lens/internal/tui/types.go b/cmd/frostfs-lens/internal/tui/types.go new file mode 100644 index 000000000..4a227fe64 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/types.go @@ -0,0 +1,18 @@ +package tui + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" +) + +type Bucket struct { + Name []byte + Path [][]byte + Entry common.SchemaEntry + NextParser common.Parser +} + +type Record struct { + Key, Value []byte + Path [][]byte + Entry common.SchemaEntry +} diff --git a/cmd/frostfs-lens/internal/tui/ui.go b/cmd/frostfs-lens/internal/tui/ui.go new file mode 100644 index 000000000..701f2b331 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/ui.go @@ -0,0 +1,548 @@ +package tui + +import ( + "context" + "errors" + "fmt" + "strings" + "sync/atomic" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "github.com/davecgh/go-spew/spew" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" + "go.etcd.io/bbolt" +) + +type Config struct { + LoadBufferSize int + SearchHistorySize int + LoadingIndicatorLag time.Duration +} + +var DefaultConfig = Config{ + LoadBufferSize: 100, + SearchHistorySize: 100, + LoadingIndicatorLag: 500 * time.Millisecond, +} + +type Primitive interface { + tview.Primitive + + Mount(ctx context.Context) error + Update(ctx context.Context) error + Unmount() +} + +type UI struct { + *tview.Box + + // Need to use context while updating pages those read data from a database. + // Context should be shared among all mount and updates. Current TUI library + // doesn't use contexts at all, so I do that feature by myself. + //nolint:containedctx + ctx context.Context + onStop func() + + app *tview.Application + db *bbolt.DB + + pageHistory []Primitive + mountedPage Primitive + + pageToMount Primitive + + pageStub tview.Primitive + + infoBar *tview.TextView + searchBar *InputFieldWithHistory + loadingBar *LoadingBar + helpBar *tview.TextView + + searchErrorBar *tview.TextView + + isSearching bool + isLoading atomic.Bool + isShowingError bool + isShowingHelp bool + + loadBufferSize int + + rootParser common.Parser + + loadingIndicatorLag time.Duration + + cancelLoading func() + + filters map[string]func(string) (any, error) + compositeFilters map[string]func(string) (map[string]any, error) + filterHints map[string]string +} + +func NewUI( + ctx context.Context, + app *tview.Application, + db *bbolt.DB, + rootParser common.Parser, + cfg *Config, +) *UI { + spew.Config.DisableMethods = true + + if cfg == nil { + cfg = &DefaultConfig + } + + ui := &UI{ + Box: tview.NewBox(), + + app: app, + db: db, + rootParser: rootParser, + + filters: make(map[string]func(string) (any, error)), + compositeFilters: make(map[string]func(string) (map[string]any, error)), + filterHints: make(map[string]string), + + loadBufferSize: cfg.LoadBufferSize, + loadingIndicatorLag: cfg.LoadingIndicatorLag, + } + + ui.ctx, ui.onStop = context.WithCancel(ctx) + + backgroundColor := ui.GetBackgroundColor() + textColor := tview.Styles.PrimaryTextColor + + inverseBackgroundColor := textColor + inverseTextColor := backgroundColor + + alertTextColor := tcell.ColorRed + + ui.pageStub = tview.NewBox() + + ui.infoBar = tview.NewTextView() + ui.infoBar.SetBackgroundColor(inverseBackgroundColor) + ui.infoBar.SetTextColor(inverseTextColor) + ui.infoBar.SetText( + fmt.Sprintf(" %s (press h for help, q to quit) ", db.Path()), + ) + + ui.searchBar = NewInputFieldWithHistory(cfg.SearchHistorySize) + ui.searchBar.SetFieldBackgroundColor(backgroundColor) + ui.searchBar.SetFieldTextColor(textColor) + ui.searchBar.SetLabelColor(textColor) + ui.searchBar.Focus(nil) + ui.searchBar.SetLabel("/") + + ui.searchErrorBar = tview.NewTextView() + ui.searchErrorBar.SetBackgroundColor(backgroundColor) + ui.searchErrorBar.SetTextColor(alertTextColor) + + ui.helpBar = tview.NewTextView() + ui.helpBar.SetBackgroundColor(inverseBackgroundColor) + ui.helpBar.SetTextColor(inverseTextColor) + ui.helpBar.SetText(" Press Enter for next page or Escape to exit help ") + + ui.loadingBar = NewLoadingBar(ui.triggerDraw) + + ui.pageToMount = NewBucketsView(ui, NewFilter(nil)) + + return ui +} + +func (ui *UI) checkFilterExists(typ string) bool { + if _, ok := ui.filters[typ]; ok { + return true + } + if _, ok := ui.compositeFilters[typ]; ok { + return true + } + return false +} + +func (ui *UI) AddFilter( + typ string, + parser func(string) (any, error), + helpHint string, +) error { + if ui.checkFilterExists(typ) { + return fmt.Errorf("filter %s already exists", typ) + } + ui.filters[typ] = parser + ui.filterHints[typ] = helpHint + return nil +} + +func (ui *UI) AddCompositeFilter( + typ string, + parser func(string) (map[string]any, error), + helpHint string, +) error { + if ui.checkFilterExists(typ) { + return fmt.Errorf("filter %s already exists", typ) + } + ui.compositeFilters[typ] = parser + ui.filterHints[typ] = helpHint + return nil +} + +func (ui *UI) stopOnError(err error) { + if err != nil { + ui.onStop() + ui.app.QueueEvent(tcell.NewEventError(err)) + } +} + +func (ui *UI) stop() { + ui.onStop() + ui.app.Stop() +} + +func (ui *UI) movePrevPage() { + if len(ui.pageHistory) != 0 { + ui.mountedPage.Unmount() + ui.mountedPage = ui.pageHistory[len(ui.pageHistory)-1] + ui.pageHistory = ui.pageHistory[:len(ui.pageHistory)-1] + ui.triggerDraw() + } +} + +func (ui *UI) moveNextPage(page Primitive) { + ui.pageToMount = page + ui.triggerDraw() +} + +func (ui *UI) triggerDraw() { + go ui.app.QueueUpdateDraw(func() {}) +} + +func (ui *UI) Draw(screen tcell.Screen) { + if ui.isLoading.Load() { + ui.draw(screen) + return + } + + ui.isLoading.Store(true) + + ctx, cancel := context.WithCancel(ui.ctx) + + ready := make(chan struct{}) + go func() { + ui.load(ctx) + + cancel() + close(ready) + ui.isLoading.Store(false) + }() + + select { + case <-ready: + case <-time.After(ui.loadingIndicatorLag): + ui.loadingBar.Start(ui.ctx) + ui.cancelLoading = cancel + + go func() { + <-ready + ui.loadingBar.Stop() + ui.triggerDraw() + }() + } + + ui.draw(screen) +} + +func (ui *UI) load(ctx context.Context) { + if ui.mountedPage == nil && ui.pageToMount == nil { + ui.stop() + return + } + + if ui.pageToMount != nil { + ui.mountAndUpdate(ctx) + } else { + ui.update(ctx) + } +} + +func (ui *UI) draw(screen tcell.Screen) { + ui.DrawForSubclass(screen, ui) + x, y, width, height := ui.GetInnerRect() + + var ( + pageToDraw tview.Primitive + barToDraw tview.Primitive + ) + + switch { + case ui.isShowingHelp: + pageToDraw = ui.pageStub + case ui.mountedPage != nil: + pageToDraw = ui.mountedPage + default: + pageToDraw = ui.pageStub + } + + pageToDraw.SetRect(x, y, width, height-1) + pageToDraw.Draw(screen) + + // Search bar uses cursor and we need to hide it when another bar is drawn. + screen.HideCursor() + + switch { + case ui.isLoading.Load(): + barToDraw = ui.loadingBar + case ui.isSearching: + barToDraw = ui.searchBar + case ui.isShowingError: + barToDraw = ui.searchErrorBar + case ui.isShowingHelp: + barToDraw = ui.helpBar + default: + barToDraw = ui.infoBar + } + + barToDraw.SetRect(x, y+height-1, width, 1) + barToDraw.Draw(screen) +} + +func (ui *UI) mountAndUpdate(ctx context.Context) { + defer func() { + // Operation succeeded or was canceled, either way reset page to mount. + ui.pageToMount = nil + }() + + // Mount should use app global context. + //nolint:contextcheck + err := ui.pageToMount.Mount(ui.ctx) + if err != nil { + ui.stopOnError(err) + return + } + + x, y, width, height := ui.GetInnerRect() + ui.pageToMount.SetRect(x, y, width, height-1) + + s := loadOp(ctx, ui.pageToMount.Update) + if s.err != nil { + ui.pageToMount.Unmount() + ui.stopOnError(s.err) + return + } + // Update was canceled. + if !s.done { + ui.pageToMount.Unmount() + return + } + + if ui.mountedPage != nil { + ui.pageHistory = append(ui.pageHistory, ui.mountedPage) + } + ui.mountedPage = ui.pageToMount +} + +func (ui *UI) update(ctx context.Context) { + x, y, width, height := ui.GetInnerRect() + ui.mountedPage.SetRect(x, y, width, height-1) + + s := loadOp(ctx, ui.mountedPage.Update) + if s.err != nil { + ui.stopOnError(s.err) + return + } +} + +type status struct { + done bool + err error +} + +func loadOp(ctx context.Context, op func(ctx context.Context) error) status { + errCh := make(chan error) + go func() { + errCh <- op(ctx) + }() + + select { + case <-ctx.Done(): + return status{done: false, err: nil} + case err := <-errCh: + return status{done: true, err: err} + } +} + +func (ui *UI) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) { + return ui.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) { + switch { + case ui.isLoading.Load(): + ui.handleInputOnLoading(event) + case ui.isShowingHelp: + ui.handleInputOnShowingHelp(event) + case ui.isShowingError: + ui.handleInputOnShowingError() + case ui.isSearching: + ui.handleInputOnSearching(event) + default: + ui.handleInput(event) + } + }) +} + +func (ui *UI) handleInput(event *tcell.EventKey) { + m, k, r := event.Modifiers(), event.Key(), event.Rune() + + switch { + case k == tcell.KeyEsc: + ui.movePrevPage() + case m == 0 && k == tcell.KeyRune && r == 'h': + ui.isShowingHelp = true + case m == 0 && k == tcell.KeyRune && r == '/': + ui.isSearching = true + case m == 0 && k == tcell.KeyRune && r == 'q': + ui.stop() + default: + if ui.mountedPage != nil { + ui.mountedPage.InputHandler()(event, func(tview.Primitive) {}) + } + } +} + +func (ui *UI) handleInputOnLoading(event *tcell.EventKey) { + switch k, r := event.Key(), event.Rune(); { + case k == tcell.KeyEsc: + ui.cancelLoading() + case k == tcell.KeyRune && r == 'q': + ui.stop() + } +} + +func (ui *UI) handleInputOnShowingError() { + ui.isShowingError = false + ui.isSearching = true +} + +func (ui *UI) handleInputOnShowingHelp(event *tcell.EventKey) { + k, r := event.Key(), event.Rune() + + switch { + case k == tcell.KeyEsc: + ui.isShowingHelp = false + case k == tcell.KeyRune && r == 'q': + ui.stop() + default: + } +} + +func (ui *UI) handleInputOnSearching(event *tcell.EventKey) { + m, k := event.Modifiers(), event.Key() + + switch { + case k == tcell.KeyEnter: + prompt := ui.searchBar.GetText() + + res, err := ui.processPrompt(prompt) + if err != nil { + ui.isShowingError = true + ui.isSearching = false + ui.searchErrorBar.SetText(err.Error() + " (press any key to continue)") + return + } + + switch ui.mountedPage.(type) { + case *BucketsView: + ui.moveNextPage(NewBucketsView(ui, res)) + case *RecordsView: + bucket := ui.mountedPage.(*RecordsView).bucket + ui.moveNextPage(NewRecordsView(ui, bucket, res)) + } + + if ui.searchBar.GetText() != "" { + ui.searchBar.AddToHistory(ui.searchBar.GetText()) + } + + ui.searchBar.SetText("") + ui.isSearching = false + case k == tcell.KeyEsc: + ui.isSearching = false + case (k == tcell.KeyBackspace2 || m&tcell.ModCtrl != 0 && k == tcell.KeyETB) && len(ui.searchBar.GetText()) == 0: + ui.isSearching = false + default: + ui.searchBar.InputHandler()(event, func(tview.Primitive) {}) + } + + ui.Box.MouseHandler() +} + +func (ui *UI) WithPrompt(prompt string) error { + filter, err := ui.processPrompt(prompt) + if err != nil { + return err + } + + ui.pageToMount = NewBucketsView(ui, filter) + + if prompt != "" { + ui.searchBar.AddToHistory(prompt) + } + + return nil +} + +func (ui *UI) processPrompt(prompt string) (filter *Filter, err error) { + if prompt == "" { + return NewFilter(nil), nil + } + + filterMap := make(map[string]any) + + for _, filterString := range strings.Split(prompt, "+") { + parts := strings.Split(filterString, ":") + if len(parts) != 2 { + return nil, errors.New("expected 'tag:value [+ tag:value]...'") + } + + filterTag := strings.TrimSpace(parts[0]) + filterValueString := strings.TrimSpace(parts[1]) + + if _, exists := filterMap[filterTag]; exists { + return nil, fmt.Errorf("duplicate filter tag '%s'", filterTag) + } + + parser, ok := ui.filters[filterTag] + if ok { + filterValue, err := parser(filterValueString) + if err != nil { + return nil, fmt.Errorf("can't parse '%s' filter value: %w", filterTag, err) + } + + filterMap[filterTag] = filterValue + continue + } + + compositeParser, ok := ui.compositeFilters[filterTag] + if ok { + compositeFilterValue, err := compositeParser(filterValueString) + if err != nil { + return nil, fmt.Errorf( + "can't parse '%s' filter value '%s': %w", + filterTag, filterValueString, err, + ) + } + + for tag, value := range compositeFilterValue { + if _, exists := filterMap[tag]; exists { + return nil, fmt.Errorf( + "found duplicate filter tag '%s' while processing composite filter with tag '%s'", + tag, filterTag, + ) + } + + filterMap[tag] = value + } + continue + } + + return nil, fmt.Errorf("unknown filter tag '%s'", filterTag) + } + + return NewFilter(filterMap), nil +} diff --git a/cmd/frostfs-lens/internal/tui/util.go b/cmd/frostfs-lens/internal/tui/util.go new file mode 100644 index 000000000..d4e13b2a9 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/util.go @@ -0,0 +1,97 @@ +package tui + +import ( + "errors" + "strings" + + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/mr-tron/base58" +) + +func CIDParser(s string) (any, error) { + data, err := base58.Decode(s) + if err != nil { + return nil, err + } + var id cid.ID + if err = id.Decode(data); err != nil { + return nil, err + } + return id, nil +} + +func OIDParser(s string) (any, error) { + data, err := base58.Decode(s) + if err != nil { + return nil, err + } + var id oid.ID + if err = id.Decode(data); err != nil { + return nil, err + } + return id, nil +} + +func AddressParser(s string) (map[string]any, error) { + m := make(map[string]any) + + parts := strings.Split(s, "/") + if len(parts) != 2 { + return nil, errors.New("expected /") + } + cnr, err := CIDParser(parts[0]) + if err != nil { + return nil, err + } + obj, err := OIDParser(parts[1]) + if err != nil { + return nil, err + } + + m["cid"] = cnr + m["oid"] = obj + + return m, nil +} + +func keyParser(s string) (any, error) { + if s == "" { + return nil, errors.New("empty attribute key") + } + return s, nil +} + +func valueParser(s string) (any, error) { + if s == "" { + return nil, errors.New("empty attribute value") + } + return s, nil +} + +func AttributeParser(s string) (map[string]any, error) { + m := make(map[string]any) + + parts := strings.Split(s, "/") + if len(parts) != 1 && len(parts) != 2 { + return nil, errors.New("expected or /") + } + + key, err := keyParser(parts[0]) + if err != nil { + return nil, err + } + m["key"] = key + + if len(parts) == 1 { + return m, nil + } + + value, err := valueParser(parts[1]) + if err != nil { + return nil, err + } + m["value"] = value + + return m, nil +} diff --git a/go.mod b/go.mod index be3c6e74d..93eef5b8c 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 github.com/prometheus/client_golang v1.19.0 + github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130 github.com/spf13/cast v1.6.0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 @@ -106,7 +107,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.11.0 // indirect diff --git a/go.sum b/go.sum index d0218a348..102501484 100644 --- a/go.sum +++ b/go.sum @@ -223,10 +223,12 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130 h1:o1CYtoFOm6xJK3DvDAEG5wDJPLj+SoxUtUDFaQgt1iY= +github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130/go.mod h1:02iFIz7K/A9jGCvrizLPvoqr4cEIx7q54RH5Qudkrss= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= From e6553363905ef350b9faf12e7a42d52cf624815c Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 16 Aug 2024 17:27:35 +0300 Subject: [PATCH 1316/1943] [#1223] lens/tui: Add app help Signed-off-by: Aleksey Savchuk --- .../internal/tui/help-pages/hotkeys.txt | 38 +++++++ .../internal/tui/help-pages/searching.txt | 26 +++++ cmd/frostfs-lens/internal/tui/help.go | 101 ++++++++++++++++++ cmd/frostfs-lens/internal/tui/ui.go | 15 ++- 4 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-lens/internal/tui/help-pages/hotkeys.txt create mode 100644 cmd/frostfs-lens/internal/tui/help-pages/searching.txt create mode 100644 cmd/frostfs-lens/internal/tui/help.go diff --git a/cmd/frostfs-lens/internal/tui/help-pages/hotkeys.txt b/cmd/frostfs-lens/internal/tui/help-pages/hotkeys.txt new file mode 100644 index 000000000..c371b34e9 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/help-pages/hotkeys.txt @@ -0,0 +1,38 @@ +[green::b]HOTKEYS[-::-] + + [green::b]Navigation[-::-] + + [yellow::b]Down Arrow[-::-] / [yellow::b]j[-::-] + Scroll down. + + [yellow::b]Up Arrow[-::-] / [yellow::b]k[-::-] + Scroll up. + + [yellow::b]Page Down[-::-] / [yellow::b]Ctrl-f[-::-] + Scroll down by a full page. + + [yellow::b]Page Up[-::-] / [yellow::b]Ctrl-b[-::-] + Scroll up by a full page. + + [green::b]Actions[-::-] + + [yellow::b]Enter[-::-] + Perform actions based on the current context: + - In Buckets View: + - Expand/collapse the selected bucket to show/hide its nested buckets. + - If no nested buckets exist, navigate to the selected bucket's records. + - In Records View: Open the detailed view of the selected record. + + [yellow::b]Escape[-::-] + Return to the previous page, opposite of [yellow::b]Enter[-::-]. + + Refer to the [green::b]SEARCHING[-::-] section for more specific actions. + + + [green::b]Alternative Action Hotkeys[-::-] + + [yellow::b]Ctrl-r[-::-] + Directly navigate to the selected bucket's records. + + [yellow::b]Ctrl-d[-::-] + Access the detailed view of the selected bucket. diff --git a/cmd/frostfs-lens/internal/tui/help-pages/searching.txt b/cmd/frostfs-lens/internal/tui/help-pages/searching.txt new file mode 100644 index 000000000..bc2be512b --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/help-pages/searching.txt @@ -0,0 +1,26 @@ +[green::b]SEARCHING[-::-] + + [green::b]Hotkeys[-::-] + + [yellow::b]/[-::-] + Initiate the search prompt. + - The prompt follows this syntax: [yellow::b]tag:value [+ tag:value]...[-::-] + - Multiple filter can be combined with [yellow::b]+[-::-], the result is an intersection of those filters' result sets. + - Any leading and trailing whitespace will be ignored. + - An empty prompt will return all results with no filters applied. + - Refer to the [green::b]Available Search Filters[-::-] section below for a list of valid filter tags. + + [yellow::b]Enter[-::-] + Execute the search based on the entered prompt. + - If the prompt is invalid, an error message will be displayed. + + [yellow::b]Escape[-::-] + Exit the search prompt without performing a search. + + [yellow::b]Down Arrow[-::-], [yellow::b]Up Arrow[-::-] + Scroll through the search history. + + + [green::b]Available Search Filters[-::-] + +%s diff --git a/cmd/frostfs-lens/internal/tui/help.go b/cmd/frostfs-lens/internal/tui/help.go new file mode 100644 index 000000000..3ab8fede0 --- /dev/null +++ b/cmd/frostfs-lens/internal/tui/help.go @@ -0,0 +1,101 @@ +package tui + +import ( + _ "embed" + "fmt" + "strings" + + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +var ( + //go:embed help-pages/hotkeys.txt + hotkeysHelpText string + + //go:embed help-pages/searching.txt + searchingHelpText string +) + +type HelpPage struct { + *tview.Box + pages []*tview.TextView + currentPage int + + filters []string + filterHints map[string]string +} + +func NewHelpPage(filters []string, hints map[string]string) *HelpPage { + hp := &HelpPage{ + Box: tview.NewBox(), + filters: filters, + filterHints: hints, + } + + page := tview.NewTextView(). + SetDynamicColors(true). + SetText(hotkeysHelpText) + hp.addPage(page) + + page = tview.NewTextView(). + SetDynamicColors(true). + SetText(fmt.Sprintf(searchingHelpText, hp.getFiltersText())) + hp.addPage(page) + + return hp +} + +func (hp *HelpPage) addPage(page *tview.TextView) { + hp.pages = append(hp.pages, page) +} + +func (hp *HelpPage) getFiltersText() string { + if len(hp.filters) == 0 { + return "\t\tNo filters defined.\n" + } + + filtersText := strings.Builder{} + gapSize := 4 + + tagMaxWidth := 3 + for _, filter := range hp.filters { + tagMaxWidth = max(tagMaxWidth, len(filter)) + } + filtersText.WriteString("\t\t[yellow::b]Tag") + filtersText.WriteString(strings.Repeat(" ", gapSize)) + filtersText.WriteString("\tValue[-::-]\n\n") + + for _, filter := range hp.filters { + filtersText.WriteString("\t\t") + filtersText.WriteString(filter) + filtersText.WriteString(strings.Repeat(" ", tagMaxWidth-len(filter)+gapSize)) + filtersText.WriteString(hp.filterHints[filter]) + filtersText.WriteRune('\n') + } + + return filtersText.String() +} + +func (hp *HelpPage) Draw(screen tcell.Screen) { + x, y, width, height := hp.GetInnerRect() + hp.pages[hp.currentPage].SetRect(x+1, y+1, width-2, height-2) + hp.pages[hp.currentPage].Draw(screen) +} + +func (hp *HelpPage) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) { + return hp.WrapInputHandler(func(event *tcell.EventKey, _ func(tview.Primitive)) { + if event.Key() == tcell.KeyEnter { + hp.currentPage++ + hp.currentPage %= len(hp.pages) + return + } + hp.pages[hp.currentPage].InputHandler()(event, func(tview.Primitive) {}) + }) +} + +func (hp *HelpPage) MouseHandler() func(action tview.MouseAction, event *tcell.EventMouse, setFocus func(p tview.Primitive)) (consumed bool, capture tview.Primitive) { + return hp.WrapMouseHandler(func(action tview.MouseAction, event *tcell.EventMouse, _ func(tview.Primitive)) (consumed bool, capture tview.Primitive) { + return hp.pages[hp.currentPage].MouseHandler()(action, event, func(tview.Primitive) {}) + }) +} diff --git a/cmd/frostfs-lens/internal/tui/ui.go b/cmd/frostfs-lens/internal/tui/ui.go index 701f2b331..bcc082821 100644 --- a/cmd/frostfs-lens/internal/tui/ui.go +++ b/cmd/frostfs-lens/internal/tui/ui.go @@ -60,6 +60,8 @@ type UI struct { loadingBar *LoadingBar helpBar *tview.TextView + helpPage *HelpPage + searchErrorBar *tview.TextView isSearching bool @@ -275,7 +277,17 @@ func (ui *UI) draw(screen tcell.Screen) { switch { case ui.isShowingHelp: - pageToDraw = ui.pageStub + if ui.helpPage == nil { + var filters []string + for f := range ui.filters { + filters = append(filters, f) + } + for f := range ui.compositeFilters { + filters = append(filters, f) + } + ui.helpPage = NewHelpPage(filters, ui.filterHints) + } + pageToDraw = ui.helpPage case ui.mountedPage != nil: pageToDraw = ui.mountedPage default: @@ -429,6 +441,7 @@ func (ui *UI) handleInputOnShowingHelp(event *tcell.EventKey) { case k == tcell.KeyRune && r == 'q': ui.stop() default: + ui.helpPage.InputHandler()(event, func(tview.Primitive) {}) } } From 371d97f61adc7cc74da815764a75a5438e865eda Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 19 Aug 2024 18:02:11 +0300 Subject: [PATCH 1317/1943] [#1223] lens/tui: Add TUI app for write cache Signed-off-by: Aleksey Savchuk --- cmd/frostfs-lens/internal/writecache/root.go | 2 +- cmd/frostfs-lens/internal/writecache/tui.go | 79 ++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-lens/internal/writecache/tui.go diff --git a/cmd/frostfs-lens/internal/writecache/root.go b/cmd/frostfs-lens/internal/writecache/root.go index eb3b325b6..d7d6db240 100644 --- a/cmd/frostfs-lens/internal/writecache/root.go +++ b/cmd/frostfs-lens/internal/writecache/root.go @@ -17,5 +17,5 @@ var Root = &cobra.Command{ } func init() { - Root.AddCommand(listCMD, inspectCMD) + Root.AddCommand(listCMD, inspectCMD, tuiCMD) } diff --git a/cmd/frostfs-lens/internal/writecache/tui.go b/cmd/frostfs-lens/internal/writecache/tui.go new file mode 100644 index 000000000..6b7532b08 --- /dev/null +++ b/cmd/frostfs-lens/internal/writecache/tui.go @@ -0,0 +1,79 @@ +package writecache + +import ( + "context" + "fmt" + + common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" + schema "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui" + "github.com/rivo/tview" + "github.com/spf13/cobra" + "go.etcd.io/bbolt" +) + +var tuiCMD = &cobra.Command{ + Use: "explore", + Short: "Write cache exploration with a terminal UI", + Long: `Launch a terminal UI to explore write cache and search for data. + +Available search filters: +- cid CID +- oid OID +- addr CID/OID +`, + Run: tuiFunc, +} + +var initialPrompt string + +func init() { + common.AddComponentPathFlag(tuiCMD, &vPath) + + tuiCMD.Flags().StringVar( + &initialPrompt, + "filter", + "", + "Filter prompt to start with, format 'tag:value [+ tag:value]...'", + ) +} + +func tuiFunc(cmd *cobra.Command, _ []string) { + common.ExitOnErr(cmd, runTUI(cmd)) +} + +func runTUI(cmd *cobra.Command) error { + db, err := openDB(false) + if err != nil { + return fmt.Errorf("couldn't open database: %w", err) + } + defer db.Close() + + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() + + app := tview.NewApplication() + ui := tui.NewUI(ctx, app, db, schema.WritecacheParser, nil) + + _ = ui.AddFilter("cid", tui.CIDParser, "CID") + _ = ui.AddFilter("oid", tui.OIDParser, "OID") + _ = ui.AddCompositeFilter("addr", tui.AddressParser, "CID/OID") + + err = ui.WithPrompt(initialPrompt) + if err != nil { + return fmt.Errorf("invalid filter prompt: %w", err) + } + + app.SetRoot(ui, true).SetFocus(ui) + return app.Run() +} + +func openDB(writable bool) (*bbolt.DB, error) { + db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{ + ReadOnly: !writable, + }) + if err != nil { + return nil, err + } + return db, nil +} From 7768a482b595e578570f6b3c705b6f7754705fab Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 22 Aug 2024 15:07:51 +0300 Subject: [PATCH 1318/1943] [#1223] lens/tui: Add TUI app for blobovnicza Signed-off-by: Aleksey Savchuk --- cmd/frostfs-lens/internal/blobovnicza/root.go | 2 +- cmd/frostfs-lens/internal/blobovnicza/tui.go | 79 ++++++++++++++ .../internal/schema/blobovnicza/parsers.go | 96 +++++++++++++++++ .../internal/schema/blobovnicza/types.go | 101 ++++++++++++++++++ 4 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-lens/internal/blobovnicza/tui.go create mode 100644 cmd/frostfs-lens/internal/schema/blobovnicza/parsers.go create mode 100644 cmd/frostfs-lens/internal/schema/blobovnicza/types.go diff --git a/cmd/frostfs-lens/internal/blobovnicza/root.go b/cmd/frostfs-lens/internal/blobovnicza/root.go index 0a0cd955d..9d8ef3dad 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/root.go +++ b/cmd/frostfs-lens/internal/blobovnicza/root.go @@ -19,7 +19,7 @@ var Root = &cobra.Command{ } func init() { - Root.AddCommand(listCMD, inspectCMD) + Root.AddCommand(listCMD, inspectCMD, tuiCMD) } func openBlobovnicza(cmd *cobra.Command) *blobovnicza.Blobovnicza { diff --git a/cmd/frostfs-lens/internal/blobovnicza/tui.go b/cmd/frostfs-lens/internal/blobovnicza/tui.go new file mode 100644 index 000000000..eb4a5ff59 --- /dev/null +++ b/cmd/frostfs-lens/internal/blobovnicza/tui.go @@ -0,0 +1,79 @@ +package blobovnicza + +import ( + "context" + "fmt" + + common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" + schema "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/blobovnicza" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui" + "github.com/rivo/tview" + "github.com/spf13/cobra" + "go.etcd.io/bbolt" +) + +var tuiCMD = &cobra.Command{ + Use: "explore", + Short: "Blobovnicza exploration with a terminal UI", + Long: `Launch a terminal UI to explore blobovnicza and search for data. + +Available search filters: +- cid CID +- oid OID +- addr CID/OID +`, + Run: tuiFunc, +} + +var initialPrompt string + +func init() { + common.AddComponentPathFlag(tuiCMD, &vPath) + + tuiCMD.Flags().StringVar( + &initialPrompt, + "filter", + "", + "Filter prompt to start with, format 'tag:value [+ tag:value]...'", + ) +} + +func tuiFunc(cmd *cobra.Command, _ []string) { + common.ExitOnErr(cmd, runTUI(cmd)) +} + +func runTUI(cmd *cobra.Command) error { + db, err := openDB(false) + if err != nil { + return fmt.Errorf("couldn't open database: %w", err) + } + defer db.Close() + + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() + + app := tview.NewApplication() + ui := tui.NewUI(ctx, app, db, schema.BlobovniczaParser, nil) + + _ = ui.AddFilter("cid", tui.CIDParser, "CID") + _ = ui.AddFilter("oid", tui.OIDParser, "OID") + _ = ui.AddCompositeFilter("addr", tui.AddressParser, "CID/OID") + + err = ui.WithPrompt(initialPrompt) + if err != nil { + return fmt.Errorf("invalid filter prompt: %w", err) + } + + app.SetRoot(ui, true).SetFocus(ui) + return app.Run() +} + +func openDB(writable bool) (*bbolt.DB, error) { + db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{ + ReadOnly: !writable, + }) + if err != nil { + return nil, err + } + return db, nil +} diff --git a/cmd/frostfs-lens/internal/schema/blobovnicza/parsers.go b/cmd/frostfs-lens/internal/schema/blobovnicza/parsers.go new file mode 100644 index 000000000..02b6cf414 --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/blobovnicza/parsers.go @@ -0,0 +1,96 @@ +package blobovnicza + +import ( + "encoding/binary" + "errors" + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/mr-tron/base58" +) + +var BlobovniczaParser = common.WithFallback( + common.Any( + MetaBucketParser, + BucketParser, + ), + common.RawParser.ToFallbackParser(), +) + +func MetaBucketParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, errors.New("not a bucket") + } + + if string(key) != "META" { + return nil, nil, errors.New("invalid bucket name") + } + + return &MetaBucket{}, MetaRecordParser, nil +} + +func MetaRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + var r MetaRecord + + if len(key) == 0 { + return nil, nil, errors.New("invalid key") + } + + r.label = string(key) + r.count = binary.LittleEndian.Uint64(value) + + return &r, nil, nil +} + +func BucketParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if value != nil { + return nil, nil, errors.New("not a bucket") + } + + size, n := binary.Varint(key) + if n <= 0 { + return nil, nil, errors.New("invalid size") + } + + return &Bucket{size: size}, RecordParser, nil +} + +func RecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + parts := strings.Split(string(key), "/") + + if len(parts) != 2 { + return nil, nil, errors.New("invalid key, expected address string /") + } + + cnrRaw, err := base58.Decode(parts[0]) + if err != nil { + return nil, nil, errors.New("can't decode CID string") + } + objRaw, err := base58.Decode(parts[1]) + if err != nil { + return nil, nil, errors.New("can't decode OID string") + } + + cnr := cid.ID{} + if err := cnr.Decode(cnrRaw); err != nil { + return nil, nil, fmt.Errorf("can't decode CID: %w", err) + } + obj := oid.ID{} + if err := obj.Decode(objRaw); err != nil { + return nil, nil, fmt.Errorf("can't decode OID: %w", err) + } + + var r Record + + r.addr.SetContainer(cnr) + r.addr.SetObject(obj) + + if err := r.object.Unmarshal(value); err != nil { + return nil, nil, errors.New("can't unmarshal object") + } + + return &r, nil, nil +} diff --git a/cmd/frostfs-lens/internal/schema/blobovnicza/types.go b/cmd/frostfs-lens/internal/schema/blobovnicza/types.go new file mode 100644 index 000000000..c7ed08cdd --- /dev/null +++ b/cmd/frostfs-lens/internal/schema/blobovnicza/types.go @@ -0,0 +1,101 @@ +package blobovnicza + +import ( + "fmt" + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/davecgh/go-spew/spew" + "github.com/gdamore/tcell/v2" + "github.com/rivo/tview" +) + +type ( + MetaBucket struct{} + + MetaRecord struct { + label string + count uint64 + } + + Bucket struct { + size int64 + } + + Record struct { + addr oid.Address + object objectSDK.Object + } +) + +func (b *MetaBucket) String() string { + return common.FormatSimple("META", tcell.ColorLime) +} + +func (b *MetaBucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *MetaBucket) Filter(string, any) common.FilterResult { + return common.No +} + +func (r *MetaRecord) String() string { + return fmt.Sprintf("%-11s %c %d", r.label, tview.Borders.Vertical, r.count) +} + +func (r *MetaRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *MetaRecord) Filter(string, any) common.FilterResult { + return common.No +} + +func (b *Bucket) String() string { + return common.FormatSimple(strconv.FormatInt(b.size, 10), tcell.ColorLime) +} + +func (b *Bucket) DetailedString() string { + return spew.Sdump(*b) +} + +func (b *Bucket) Filter(typ string, _ any) common.FilterResult { + switch typ { + case "cid": + return common.Maybe + case "oid": + return common.Maybe + default: + return common.No + } +} + +func (r *Record) String() string { + return fmt.Sprintf( + "CID %s OID %s %c Object {...}", + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Container()), tcell.ColorAqua), + common.FormatSimple(fmt.Sprintf("%-44s", r.addr.Object()), tcell.ColorAqua), + tview.Borders.Vertical, + ) +} + +func (r *Record) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *Record) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.addr.Container().Equals(id), common.Yes, common.No) + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.addr.Object().Equals(id), common.Yes, common.No) + default: + return common.No + } +} From b3deb893ba26aa3ec9ce93213cef16243cc0f58d Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 30 Aug 2024 12:09:14 +0300 Subject: [PATCH 1319/1943] [#1310] object: Move target initialization to separate package * Split the logic of write target initialization to different packages; * Refactor patch and put services: since both service initialize the target themselves. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/cache.go | 6 +- cmd/frostfs-node/object.go | 17 +- .../object/{put => common/target}/builder.go | 2 +- .../object/{put => common/target}/pool.go | 2 +- pkg/services/object/common/target/target.go | 170 +++++++++++ .../{put => common/target}/validation.go | 2 +- .../object/{put => common/writer}/common.go | 42 +-- .../writer.go => common/writer/dispatcher.go} | 2 +- .../{put => common/writer}/distributed.go | 70 ++--- .../object/{put => common/writer}/ec.go | 104 +++---- .../object/{put => common/writer}/local.go | 14 +- .../object/{put => common/writer}/remote.go | 12 +- pkg/services/object/common/writer/writer.go | 183 +++++++++++ pkg/services/object/patch/service.go | 25 +- pkg/services/object/patch/streamer.go | 28 +- pkg/services/object/patch/util.go | 19 -- pkg/services/object/put/service.go | 114 ++----- pkg/services/object/put/single.go | 64 ++-- pkg/services/object/put/streamer.go | 289 ++---------------- pkg/services/object/put/v2/streamer.go | 9 +- pkg/services/replicator/process.go | 4 +- pkg/services/replicator/replicator.go | 6 +- 22 files changed, 599 insertions(+), 585 deletions(-) rename pkg/services/object/{put => common/target}/builder.go (98%) rename pkg/services/object/{put => common/target}/pool.go (96%) create mode 100644 pkg/services/object/common/target/target.go rename pkg/services/object/{put => common/target}/validation.go (99%) rename pkg/services/object/{put => common/writer}/common.go (65%) rename pkg/services/object/{put/writer.go => common/writer/dispatcher.go} (97%) rename pkg/services/object/{put => common/writer}/distributed.go (57%) rename pkg/services/object/{put => common/writer}/ec.go (69%) rename pkg/services/object/{put => common/writer}/local.go (81%) rename pkg/services/object/{put => common/writer}/remote.go (92%) create mode 100644 pkg/services/object/common/writer/writer.go diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index 81d552729..57f65d873 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -7,7 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" + objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -305,11 +305,11 @@ type ttlMaxObjectSizeCache struct { mtx sync.RWMutex lastUpdated time.Time lastSize uint64 - src putsvc.MaxSizeSource + src objectwriter.MaxSizeSource metrics cacheMetrics } -func newCachedMaxObjectSizeSource(src putsvc.MaxSizeSource) putsvc.MaxSizeSource { +func newCachedMaxObjectSizeSource(src objectwriter.MaxSizeSource) objectwriter.MaxSizeSource { return &ttlMaxObjectSizeCache{ src: src, metrics: metrics.NewCacheMetrics("max_object_size"), diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 467c5901b..610e2c363 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -24,6 +24,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl" v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" objectAPE "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/ape" + objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" deletesvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/delete" deletesvcV2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/delete/v2" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" @@ -188,7 +189,7 @@ func initObjectService(c *cfg) { sDeleteV2 := createDeleteServiceV2(sDelete) - sPatch := createPatchSvc(sGet, sPut, keyStorage) + sPatch := createPatchSvc(sGet, sPut) // build service pipeline // grpc | audit | | signature | response | acl | ape | split @@ -326,7 +327,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa ), replicator.WithLocalStorage(ls), replicator.WithRemoteSender( - putsvc.NewRemoteSender(keyStorage, cache), + objectwriter.NewRemoteSender(keyStorage, cache), ), replicator.WithRemoteGetter( getsvc.NewRemoteGetter(c.clientCache, c.netMapSource, keyStorage), @@ -338,7 +339,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa func createPutSvc(c *cfg, keyStorage *util.KeyStorage, irFetcher *cachedIRFetcher) *putsvc.Service { ls := c.cfgObject.cfgLocalStorage.localStorage - var os putsvc.ObjectStorage = engineWithoutNotifications{ + var os objectwriter.ObjectStorage = engineWithoutNotifications{ engine: ls, } @@ -352,9 +353,9 @@ func createPutSvc(c *cfg, keyStorage *util.KeyStorage, irFetcher *cachedIRFetche c, c.cfgNetmap.state, irFetcher, - putsvc.WithWorkerPools(c.cfgObject.pool.putRemote, c.cfgObject.pool.putLocal), - putsvc.WithLogger(c.log), - putsvc.WithVerifySessionTokenIssuer(!c.cfgObject.skipSessionTokenIssuerVerification), + objectwriter.WithWorkerPools(c.cfgObject.pool.putRemote, c.cfgObject.pool.putLocal), + objectwriter.WithLogger(c.log), + objectwriter.WithVerifySessionTokenIssuer(!c.cfgObject.skipSessionTokenIssuerVerification), ) } @@ -362,8 +363,8 @@ func createPutSvcV2(sPut *putsvc.Service, keyStorage *util.KeyStorage) *putsvcV2 return putsvcV2.NewService(sPut, keyStorage) } -func createPatchSvc(sGet *getsvc.Service, sPut *putsvc.Service, keyStorage *util.KeyStorage) *patchsvc.Service { - return patchsvc.NewService(keyStorage, sGet, sPut) +func createPatchSvc(sGet *getsvc.Service, sPut *putsvc.Service) *patchsvc.Service { + return patchsvc.NewService(sPut.Config, sGet) } func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache) *searchsvc.Service { diff --git a/pkg/services/object/put/builder.go b/pkg/services/object/common/target/builder.go similarity index 98% rename from pkg/services/object/put/builder.go rename to pkg/services/object/common/target/builder.go index 64baf4e05..ea68365a7 100644 --- a/pkg/services/object/put/builder.go +++ b/pkg/services/object/common/target/builder.go @@ -1,4 +1,4 @@ -package putsvc +package target import ( "context" diff --git a/pkg/services/object/put/pool.go b/pkg/services/object/common/target/pool.go similarity index 96% rename from pkg/services/object/put/pool.go rename to pkg/services/object/common/target/pool.go index ebe214caf..71da305ad 100644 --- a/pkg/services/object/put/pool.go +++ b/pkg/services/object/common/target/pool.go @@ -1,4 +1,4 @@ -package putsvc +package target import ( "sync" diff --git a/pkg/services/object/common/target/target.go b/pkg/services/object/common/target/target.go new file mode 100644 index 000000000..00080ace6 --- /dev/null +++ b/pkg/services/object/common/target/target.go @@ -0,0 +1,170 @@ +package target + +import ( + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" +) + +func New(prm *objectwriter.Params) (transformer.ChunkedObjectWriter, error) { + // prepare needed put parameters + if err := preparePrm(prm); err != nil { + return nil, fmt.Errorf("could not prepare put parameters: %w", err) + } + + if prm.Header.Signature() != nil { + return newUntrustedTarget(prm) + } + return newTrustedTarget(prm) +} + +func newUntrustedTarget(prm *objectwriter.Params) (transformer.ChunkedObjectWriter, error) { + maxPayloadSz := prm.Config.MaxSizeSrc.MaxObjectSize() + if maxPayloadSz == 0 { + return nil, errors.New("could not obtain max object size parameter") + } + + if prm.SignRequestPrivateKey == nil { + nodeKey, err := prm.Config.KeyStorage.GetKey(nil) + if err != nil { + return nil, err + } + prm.SignRequestPrivateKey = nodeKey + } + + // prepare untrusted-Put object target + return &validatingPreparedTarget{ + nextTarget: newInMemoryObjectBuilder(objectwriter.New(prm)), + fmt: prm.Config.FormatValidator, + + maxPayloadSz: maxPayloadSz, + }, nil +} + +func newTrustedTarget(prm *objectwriter.Params) (transformer.ChunkedObjectWriter, error) { + maxPayloadSz := prm.Config.MaxSizeSrc.MaxObjectSize() + if maxPayloadSz == 0 { + return nil, errors.New("could not obtain max object size parameter") + } + + sToken := prm.Common.SessionToken() + + // prepare trusted-Put object target + + // get private token from local storage + var sessionInfo *util.SessionInfo + + if sToken != nil { + sessionInfo = &util.SessionInfo{ + ID: sToken.ID(), + Owner: sToken.Issuer(), + } + } + + key, err := prm.Config.KeyStorage.GetKey(sessionInfo) + if err != nil { + return nil, fmt.Errorf("could not receive session key: %w", err) + } + + // In case session token is missing, the line above returns the default key. + // If it isn't owner key, replication attempts will fail, thus this check. + ownerObj := prm.Header.OwnerID() + if ownerObj.IsEmpty() { + return nil, errors.New("missing object owner") + } + + if sToken == nil { + var ownerSession user.ID + user.IDFromKey(&ownerSession, key.PublicKey) + + if !ownerObj.Equals(ownerSession) { + return nil, errors.New("session token is missing but object owner id is different from the default key") + } + } else { + if !ownerObj.Equals(sessionInfo.Owner) { + return nil, fmt.Errorf("different token issuer and object owner identifiers %s/%s", sessionInfo.Owner, ownerObj) + } + } + + if prm.SignRequestPrivateKey == nil { + prm.SignRequestPrivateKey = key + } + + return &validatingTarget{ + fmt: prm.Config.FormatValidator, + nextTarget: transformer.NewPayloadSizeLimiter(transformer.Params{ + Key: key, + NextTargetInit: func() transformer.ObjectWriter { return objectwriter.New(prm) }, + NetworkState: prm.Config.NetworkState, + MaxSize: maxPayloadSz, + WithoutHomomorphicHash: containerSDK.IsHomomorphicHashingDisabled(prm.Container), + SessionToken: sToken, + }), + }, nil +} + +func preparePrm(prm *objectwriter.Params) error { + var err error + + // get latest network map + nm, err := netmap.GetLatestNetworkMap(prm.Config.NetmapSource) + if err != nil { + //return fmt.Errorf("(%T) could not get latest network map: %w", p, err) + return fmt.Errorf("could not get latest network map: %w", err) + } + + idCnr, ok := prm.Header.ContainerID() + if !ok { + return errors.New("missing container ID") + } + + // get container to store the object + cnrInfo, err := prm.Config.ContainerSource.Get(idCnr) + if err != nil { + //return fmt.Errorf("(%T) could not get container by ID: %w", p, err) + return fmt.Errorf("could not get container by ID: %w", err) + } + + prm.Container = cnrInfo.Value + + // add common options + prm.TraverseOpts = append(prm.TraverseOpts, + // set processing container + placement.ForContainer(prm.Container), + ) + + if ech := prm.Header.ECHeader(); ech != nil { + prm.TraverseOpts = append(prm.TraverseOpts, + // set identifier of the processing object + placement.ForObject(ech.Parent()), + ) + } else if id, ok := prm.Header.ID(); ok { + prm.TraverseOpts = append(prm.TraverseOpts, + // set identifier of the processing object + placement.ForObject(id), + ) + } + + // create placement builder from network map + builder := placement.NewNetworkMapBuilder(nm) + + if prm.Common.LocalOnly() { + // restrict success count to 1 stored copy (to local storage) + prm.TraverseOpts = append(prm.TraverseOpts, placement.SuccessAfter(1)) + + // use local-only placement builder + builder = util.NewLocalPlacement(builder, prm.Config.NetmapKeys) + } + + // set placement builder + prm.TraverseOpts = append(prm.TraverseOpts, placement.UseBuilder(builder)) + + return nil +} diff --git a/pkg/services/object/put/validation.go b/pkg/services/object/common/target/validation.go similarity index 99% rename from pkg/services/object/put/validation.go rename to pkg/services/object/common/target/validation.go index c2b078ef5..b29721d01 100644 --- a/pkg/services/object/put/validation.go +++ b/pkg/services/object/common/target/validation.go @@ -1,4 +1,4 @@ -package putsvc +package target import ( "bytes" diff --git a/pkg/services/object/put/common.go b/pkg/services/object/common/writer/common.go similarity index 65% rename from pkg/services/object/put/common.go rename to pkg/services/object/common/writer/common.go index cbb7f5f33..6689557ee 100644 --- a/pkg/services/object/put/common.go +++ b/pkg/services/object/common/writer/common.go @@ -1,4 +1,4 @@ -package putsvc +package writer import ( "context" @@ -13,23 +13,23 @@ import ( "go.uber.org/zap" ) -type nodeIterator struct { - traversal - cfg *cfg +type NodeIterator struct { + Traversal + cfg *Config } -func (c *cfg) newNodeIterator(opts []placement.Option) *nodeIterator { - return &nodeIterator{ - traversal: traversal{ - opts: opts, - mExclude: make(map[string]*bool), +func (c *Config) NewNodeIterator(opts []placement.Option) *NodeIterator { + return &NodeIterator{ + Traversal: Traversal{ + Opts: opts, + Exclude: make(map[string]*bool), }, cfg: c, } } -func (n *nodeIterator) forEachNode(ctx context.Context, f func(context.Context, nodeDesc) error) error { - traverser, err := placement.NewTraverser(n.traversal.opts...) +func (n *NodeIterator) ForEachNode(ctx context.Context, f func(context.Context, NodeDescriptor) error) error { + traverser, err := placement.NewTraverser(n.Traversal.Opts...) if err != nil { return fmt.Errorf("could not create object placement traverser: %w", err) } @@ -56,10 +56,10 @@ func (n *nodeIterator) forEachNode(ctx context.Context, f func(context.Context, } // perform additional container broadcast if needed - if n.traversal.submitPrimaryPlacementFinish() { - err := n.forEachNode(ctx, f) + if n.Traversal.submitPrimaryPlacementFinish() { + err := n.ForEachNode(ctx, f) if err != nil { - n.cfg.log.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) + n.cfg.Logger.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) // we don't fail primary operation because of broadcast failure } } @@ -67,11 +67,11 @@ func (n *nodeIterator) forEachNode(ctx context.Context, f func(context.Context, return nil } -func (n *nodeIterator) forEachAddress(ctx context.Context, traverser *placement.Traverser, addrs []placement.Node, f func(context.Context, nodeDesc) error, resErr *atomic.Value) bool { +func (n *NodeIterator) forEachAddress(ctx context.Context, traverser *placement.Traverser, addrs []placement.Node, f func(context.Context, NodeDescriptor) error, resErr *atomic.Value) bool { var wg sync.WaitGroup for _, addr := range addrs { - if ok := n.mExclude[string(addr.PublicKey())]; ok != nil { + if ok := n.Exclude[string(addr.PublicKey())]; ok != nil { if *ok { traverser.SubmitSuccess() } @@ -86,10 +86,10 @@ func (n *nodeIterator) forEachAddress(ctx context.Context, traverser *placement. if err := workerPool.Submit(func() { defer wg.Done() - err := f(ctx, nodeDesc{local: isLocal, info: addr}) + err := f(ctx, NodeDescriptor{Local: isLocal, Info: addr}) if err != nil { resErr.Store(err) - svcutil.LogServiceError(n.cfg.log, "PUT", addr.Addresses(), err) + svcutil.LogServiceError(n.cfg.Logger, "PUT", addr.Addresses(), err) return } @@ -97,7 +97,7 @@ func (n *nodeIterator) forEachAddress(ctx context.Context, traverser *placement. *item = true }); err != nil { wg.Done() - svcutil.LogWorkerPoolError(n.cfg.log, "PUT", err) + svcutil.LogWorkerPoolError(n.cfg.Logger, "PUT", err) return true } @@ -105,7 +105,7 @@ func (n *nodeIterator) forEachAddress(ctx context.Context, traverser *placement. // in subsequent container broadcast. Note that we don't // process this node during broadcast if primary placement // on it failed. - n.traversal.submitProcessed(addr, item) + n.Traversal.submitProcessed(addr, item) } wg.Wait() @@ -113,6 +113,6 @@ func (n *nodeIterator) forEachAddress(ctx context.Context, traverser *placement. return false } -func needAdditionalBroadcast(obj *objectSDK.Object, localOnly bool) bool { +func NeedAdditionalBroadcast(obj *objectSDK.Object, localOnly bool) bool { return len(obj.Children()) > 0 || (!localOnly && (obj.Type() == objectSDK.TypeTombstone || obj.Type() == objectSDK.TypeLock)) } diff --git a/pkg/services/object/put/writer.go b/pkg/services/object/common/writer/dispatcher.go similarity index 97% rename from pkg/services/object/put/writer.go rename to pkg/services/object/common/writer/dispatcher.go index 53eee6006..bb9a54ce9 100644 --- a/pkg/services/object/put/writer.go +++ b/pkg/services/object/common/writer/dispatcher.go @@ -1,4 +1,4 @@ -package putsvc +package writer import ( "context" diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/common/writer/distributed.go similarity index 57% rename from pkg/services/object/put/distributed.go rename to pkg/services/object/common/writer/distributed.go index 5176f7a54..f62934bed 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/common/writer/distributed.go @@ -1,4 +1,4 @@ -package putsvc +package writer import ( "context" @@ -13,47 +13,47 @@ type preparedObjectTarget interface { WriteObject(context.Context, *objectSDK.Object, object.ContentMeta) error } -type distributedTarget struct { +type distributedWriter struct { + cfg *Config + placementOpts []placement.Option obj *objectSDK.Object objMeta object.ContentMeta - *cfg + nodeTargetInitializer func(NodeDescriptor) preparedObjectTarget - nodeTargetInitializer func(nodeDesc) preparedObjectTarget - - relay func(context.Context, nodeDesc) error + relay func(context.Context, NodeDescriptor) error resetSuccessAfterOnBroadcast bool } -// parameters and state of container traversal. -type traversal struct { - opts []placement.Option +// parameters and state of container Traversal. +type Traversal struct { + Opts []placement.Option // need of additional broadcast after the object is saved - extraBroadcastEnabled bool + ExtraBroadcastEnabled bool // container nodes which was processed during the primary object placement - mExclude map[string]*bool + Exclude map[string]*bool - resetSuccessAfterOnBroadcast bool + ResetSuccessAfterOnBroadcast bool } // updates traversal parameters after the primary placement finish and // returns true if additional container broadcast is needed. -func (x *traversal) submitPrimaryPlacementFinish() bool { - if x.extraBroadcastEnabled { +func (x *Traversal) submitPrimaryPlacementFinish() bool { + if x.ExtraBroadcastEnabled { // do not track success during container broadcast (best-effort) - x.opts = append(x.opts, placement.WithoutSuccessTracking()) + x.Opts = append(x.Opts, placement.WithoutSuccessTracking()) - if x.resetSuccessAfterOnBroadcast { - x.opts = append(x.opts, placement.ResetSuccessAfter()) + if x.ResetSuccessAfterOnBroadcast { + x.Opts = append(x.Opts, placement.ResetSuccessAfter()) } // avoid 2nd broadcast - x.extraBroadcastEnabled = false + x.ExtraBroadcastEnabled = false return true } @@ -62,22 +62,22 @@ func (x *traversal) submitPrimaryPlacementFinish() bool { } // marks the container node as processed during the primary object placement. -func (x *traversal) submitProcessed(n placement.Node, item *bool) { - if x.extraBroadcastEnabled { +func (x *Traversal) submitProcessed(n placement.Node, item *bool) { + if x.ExtraBroadcastEnabled { key := string(n.PublicKey()) - if x.mExclude == nil { - x.mExclude = make(map[string]*bool, 1) + if x.Exclude == nil { + x.Exclude = make(map[string]*bool, 1) } - x.mExclude[key] = item + x.Exclude[key] = item } } -type nodeDesc struct { - local bool +type NodeDescriptor struct { + Local bool - info placement.Node + Info placement.Node } // errIncompletePut is returned if processing on a container fails. @@ -96,19 +96,19 @@ func (x errIncompletePut) Error() string { } // WriteObject implements the transformer.ObjectWriter interface. -func (t *distributedTarget) WriteObject(ctx context.Context, obj *objectSDK.Object) error { +func (t *distributedWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error { t.obj = obj var err error - if t.objMeta, err = t.fmtValidator.ValidateContent(t.obj); err != nil { + if t.objMeta, err = t.cfg.FormatValidator.ValidateContent(t.obj); err != nil { return fmt.Errorf("(%T) could not validate payload content: %w", t, err) } return t.iteratePlacement(ctx) } -func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error { - if !node.local && t.relay != nil { +func (t *distributedWriter) sendObject(ctx context.Context, node NodeDescriptor) error { + if !node.Local && t.relay != nil { return t.relay(ctx, node) } @@ -121,11 +121,11 @@ func (t *distributedTarget) sendObject(ctx context.Context, node nodeDesc) error return nil } -func (t *distributedTarget) iteratePlacement(ctx context.Context) error { +func (t *distributedWriter) iteratePlacement(ctx context.Context) error { id, _ := t.obj.ID() - iter := t.cfg.newNodeIterator(append(t.placementOpts, placement.ForObject(id))) - iter.extraBroadcastEnabled = needAdditionalBroadcast(t.obj, false /* Distributed target is for cluster-wide PUT */) - iter.resetSuccessAfterOnBroadcast = t.resetSuccessAfterOnBroadcast - return iter.forEachNode(ctx, t.sendObject) + iter := t.cfg.NewNodeIterator(append(t.placementOpts, placement.ForObject(id))) + iter.ExtraBroadcastEnabled = NeedAdditionalBroadcast(t.obj, false /* Distributed target is for cluster-wide PUT */) + iter.ResetSuccessAfterOnBroadcast = t.resetSuccessAfterOnBroadcast + return iter.ForEachNode(ctx, t.sendObject) } diff --git a/pkg/services/object/put/ec.go b/pkg/services/object/common/writer/ec.go similarity index 69% rename from pkg/services/object/put/ec.go rename to pkg/services/object/common/writer/ec.go index 9980f6d61..fb0a8e4e5 100644 --- a/pkg/services/object/put/ec.go +++ b/pkg/services/object/common/writer/ec.go @@ -1,4 +1,4 @@ -package putsvc +package writer import ( "context" @@ -23,23 +23,23 @@ import ( "golang.org/x/sync/errgroup" ) -var _ transformer.ObjectWriter = (*ecWriter)(nil) +var _ transformer.ObjectWriter = (*ECWriter)(nil) var errUnsupportedECObject = errors.New("object is not supported for erasure coding") -type ecWriter struct { - cfg *cfg - placementOpts []placement.Option - container containerSDK.Container - key *ecdsa.PrivateKey - commonPrm *svcutil.CommonPrm - relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error +type ECWriter struct { + Config *Config + PlacementOpts []placement.Option + Container containerSDK.Container + Key *ecdsa.PrivateKey + CommonPrm *svcutil.CommonPrm + Relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error - objMeta object.ContentMeta - objMetaValid bool + ObjectMeta object.ContentMeta + ObjectMetaValid bool } -func (e *ecWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error { +func (e *ECWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error { relayed, err := e.relayIfNotContainerNode(ctx, obj) if err != nil { return err @@ -53,11 +53,11 @@ func (e *ecWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error return errUnsupportedECObject } - if !e.objMetaValid { - if e.objMeta, err = e.cfg.fmtValidator.ValidateContent(obj); err != nil { + if !e.ObjectMetaValid { + if e.ObjectMeta, err = e.Config.FormatValidator.ValidateContent(obj); err != nil { return fmt.Errorf("(%T) could not validate payload content: %w", e, err) } - e.objMetaValid = true + e.ObjectMetaValid = true } if obj.ECHeader() != nil { @@ -66,8 +66,8 @@ func (e *ecWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error return e.writeRawObject(ctx, obj) } -func (e *ecWriter) relayIfNotContainerNode(ctx context.Context, obj *objectSDK.Object) (bool, error) { - if e.relay == nil { +func (e *ECWriter) relayIfNotContainerNode(ctx context.Context, obj *objectSDK.Object) (bool, error) { + if e.Relay == nil { return false, nil } currentNodeIsContainerNode, err := e.currentNodeIsContainerNode() @@ -90,8 +90,8 @@ func (e *ecWriter) relayIfNotContainerNode(ctx context.Context, obj *objectSDK.O return true, nil } -func (e *ecWriter) currentNodeIsContainerNode() (bool, error) { - t, err := placement.NewTraverser(e.placementOpts...) +func (e *ECWriter) currentNodeIsContainerNode() (bool, error) { + t, err := placement.NewTraverser(e.PlacementOpts...) if err != nil { return false, err } @@ -101,7 +101,7 @@ func (e *ecWriter) currentNodeIsContainerNode() (bool, error) { break } for _, node := range nodes { - if e.cfg.netmapKeys.IsLocalKey(node.PublicKey()) { + if e.Config.NetmapKeys.IsLocalKey(node.PublicKey()) { return true, nil } } @@ -109,8 +109,8 @@ func (e *ecWriter) currentNodeIsContainerNode() (bool, error) { return false, nil } -func (e *ecWriter) relayToContainerNode(ctx context.Context, objID oid.ID, index uint32) error { - t, err := placement.NewTraverser(append(e.placementOpts, placement.ForObject(objID))...) +func (e *ECWriter) relayToContainerNode(ctx context.Context, objID oid.ID, index uint32) error { + t, err := placement.NewTraverser(append(e.PlacementOpts, placement.ForObject(objID))...) if err != nil { return err } @@ -126,18 +126,18 @@ func (e *ecWriter) relayToContainerNode(ctx context.Context, objID oid.ID, index var info client.NodeInfo client.NodeInfoFromNetmapElement(&info, node) - c, err := e.cfg.clientConstructor.Get(info) + c, err := e.Config.ClientConstructor.Get(info) if err != nil { return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) } completed := make(chan interface{}) - if poolErr := e.cfg.remotePool.Submit(func() { + if poolErr := e.Config.RemotePool.Submit(func() { defer close(completed) - err = e.relay(ctx, info, c) + err = e.Relay(ctx, info, c) }); poolErr != nil { close(completed) - svcutil.LogWorkerPoolError(e.cfg.log, "PUT", poolErr) + svcutil.LogWorkerPoolError(e.Config.Logger, "PUT", poolErr) return poolErr } <-completed @@ -145,7 +145,7 @@ func (e *ecWriter) relayToContainerNode(ctx context.Context, objID oid.ID, index if err == nil { return nil } - e.cfg.log.Logger.Warn(logs.ECFailedToSendToContainerNode, zap.Stringers("address_group", info.AddressGroup())) + e.Config.Logger.Logger.Warn(logs.ECFailedToSendToContainerNode, zap.Stringers("address_group", info.AddressGroup())) lastErr = err } } @@ -157,12 +157,12 @@ func (e *ecWriter) relayToContainerNode(ctx context.Context, objID oid.ID, index } } -func (e *ecWriter) writeECPart(ctx context.Context, obj *objectSDK.Object) error { - if e.commonPrm.LocalOnly() { +func (e *ECWriter) writeECPart(ctx context.Context, obj *objectSDK.Object) error { + if e.CommonPrm.LocalOnly() { return e.writePartLocal(ctx, obj) } - t, err := placement.NewTraverser(append(e.placementOpts, placement.ForObject(obj.ECHeader().Parent()))...) + t, err := placement.NewTraverser(append(e.PlacementOpts, placement.ForObject(obj.ECHeader().Parent()))...) if err != nil { return err } @@ -187,18 +187,18 @@ func (e *ecWriter) writeECPart(ctx context.Context, obj *objectSDK.Object) error return nil } -func (e *ecWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) error { +func (e *ECWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) error { // now only single EC policy is supported - c, err := erasurecode.NewConstructor(policy.ECDataCount(e.container.PlacementPolicy()), policy.ECParityCount(e.container.PlacementPolicy())) + c, err := erasurecode.NewConstructor(policy.ECDataCount(e.Container.PlacementPolicy()), policy.ECParityCount(e.Container.PlacementPolicy())) if err != nil { return err } - parts, err := c.Split(obj, e.key) + parts, err := c.Split(obj, e.Key) if err != nil { return err } objID, _ := obj.ID() - t, err := placement.NewTraverser(append(e.placementOpts, placement.ForObject(objID))...) + t, err := placement.NewTraverser(append(e.PlacementOpts, placement.ForObject(objID))...) if err != nil { return err } @@ -230,7 +230,7 @@ func (e *ecWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) er return nil } -func (e *ecWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx int, nodes []placement.Node, visited []atomic.Bool) error { +func (e *ECWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx int, nodes []placement.Node, visited []atomic.Bool) error { select { case <-ctx.Done(): return ctx.Err() @@ -243,7 +243,7 @@ func (e *ecWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx if err == nil { return nil } - e.cfg.log.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), + e.Config.Logger.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), zap.Stringer("parent_address", obj.ECHeader().Parent()), zap.Int("part_index", partIdx), zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Error(err)) @@ -267,7 +267,7 @@ func (e *ecWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx if err == nil { return nil } - e.cfg.log.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), + e.Config.Logger.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), zap.Stringer("parent_address", obj.ECHeader().Parent()), zap.Int("part_index", partIdx), zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Error(err)) @@ -291,7 +291,7 @@ func (e *ecWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx if err == nil { return nil } - e.cfg.log.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), + e.Config.Logger.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), zap.Stringer("parent_address", obj.ECHeader().Parent()), zap.Int("part_index", partIdx), zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Error(err)) @@ -300,22 +300,22 @@ func (e *ecWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx return fmt.Errorf("failed to save EC chunk %s to any node", object.AddressOf(obj)) } -func (e *ecWriter) putECPartToNode(ctx context.Context, obj *objectSDK.Object, node placement.Node) error { - if e.cfg.netmapKeys.IsLocalKey(node.PublicKey()) { +func (e *ECWriter) putECPartToNode(ctx context.Context, obj *objectSDK.Object, node placement.Node) error { + if e.Config.NetmapKeys.IsLocalKey(node.PublicKey()) { return e.writePartLocal(ctx, obj) } return e.writePartRemote(ctx, obj, node) } -func (e *ecWriter) writePartLocal(ctx context.Context, obj *objectSDK.Object) error { +func (e *ECWriter) writePartLocal(ctx context.Context, obj *objectSDK.Object) error { var err error - localTarget := localTarget{ - storage: e.cfg.localStore, + localTarget := LocalTarget{ + Storage: e.Config.LocalStore, } completed := make(chan interface{}) - if poolErr := e.cfg.localPool.Submit(func() { + if poolErr := e.Config.LocalPool.Submit(func() { defer close(completed) - err = localTarget.WriteObject(ctx, obj, e.objMeta) + err = localTarget.WriteObject(ctx, obj, e.ObjectMeta) }); poolErr != nil { close(completed) return poolErr @@ -324,22 +324,22 @@ func (e *ecWriter) writePartLocal(ctx context.Context, obj *objectSDK.Object) er return err } -func (e *ecWriter) writePartRemote(ctx context.Context, obj *objectSDK.Object, node placement.Node) error { +func (e *ECWriter) writePartRemote(ctx context.Context, obj *objectSDK.Object, node placement.Node) error { var clientNodeInfo client.NodeInfo client.NodeInfoFromNetmapElement(&clientNodeInfo, node) - remoteTaget := remoteTarget{ - privateKey: e.key, - clientConstructor: e.cfg.clientConstructor, - commonPrm: e.commonPrm, + remoteTaget := remoteWriter{ + privateKey: e.Key, + clientConstructor: e.Config.ClientConstructor, + commonPrm: e.CommonPrm, nodeInfo: clientNodeInfo, } var err error completed := make(chan interface{}) - if poolErr := e.cfg.remotePool.Submit(func() { + if poolErr := e.Config.RemotePool.Submit(func() { defer close(completed) - err = remoteTaget.WriteObject(ctx, obj, e.objMeta) + err = remoteTaget.WriteObject(ctx, obj, e.ObjectMeta) }); poolErr != nil { close(completed) return poolErr diff --git a/pkg/services/object/put/local.go b/pkg/services/object/common/writer/local.go similarity index 81% rename from pkg/services/object/put/local.go rename to pkg/services/object/common/writer/local.go index 54649adc7..02fd25b9e 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/common/writer/local.go @@ -1,4 +1,4 @@ -package putsvc +package writer import ( "context" @@ -24,19 +24,19 @@ type ObjectStorage interface { IsLocked(context.Context, oid.Address) (bool, error) } -type localTarget struct { - storage ObjectStorage +type LocalTarget struct { + Storage ObjectStorage } -func (t localTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, meta objectCore.ContentMeta) error { +func (t LocalTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, meta objectCore.ContentMeta) error { switch meta.Type() { case objectSDK.TypeTombstone: - err := t.storage.Delete(ctx, objectCore.AddressOf(obj), meta.Objects()) + err := t.Storage.Delete(ctx, objectCore.AddressOf(obj), meta.Objects()) if err != nil { return fmt.Errorf("could not delete objects from tombstone locally: %w", err) } case objectSDK.TypeLock: - err := t.storage.Lock(ctx, objectCore.AddressOf(obj), meta.Objects()) + err := t.Storage.Lock(ctx, objectCore.AddressOf(obj), meta.Objects()) if err != nil { return fmt.Errorf("could not lock object from lock objects locally: %w", err) } @@ -44,7 +44,7 @@ func (t localTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, met // objects that do not change meta storage } - if err := t.storage.Put(ctx, obj); err != nil { + if err := t.Storage.Put(ctx, obj); err != nil { return fmt.Errorf("(%T) could not put object to local storage: %w", t, err) } return nil diff --git a/pkg/services/object/put/remote.go b/pkg/services/object/common/writer/remote.go similarity index 92% rename from pkg/services/object/put/remote.go rename to pkg/services/object/common/writer/remote.go index ee8d64e7a..697613ff7 100644 --- a/pkg/services/object/put/remote.go +++ b/pkg/services/object/common/writer/remote.go @@ -1,4 +1,4 @@ -package putsvc +package writer import ( "context" @@ -16,7 +16,7 @@ import ( "google.golang.org/grpc/status" ) -type remoteTarget struct { +type remoteWriter struct { privateKey *ecdsa.PrivateKey commonPrm *util.CommonPrm @@ -41,7 +41,7 @@ type RemotePutPrm struct { obj *objectSDK.Object } -func (t *remoteTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, _ objectcore.ContentMeta) error { +func (t *remoteWriter) WriteObject(ctx context.Context, obj *objectSDK.Object, _ objectcore.ContentMeta) error { c, err := t.clientConstructor.Get(t.nodeInfo) if err != nil { return fmt.Errorf("(%T) could not create SDK client %s: %w", t, t.nodeInfo, err) @@ -64,7 +64,7 @@ func (t *remoteTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, _ return t.putStream(ctx, prm) } -func (t *remoteTarget) putStream(ctx context.Context, prm internalclient.PutObjectPrm) error { +func (t *remoteWriter) putStream(ctx context.Context, prm internalclient.PutObjectPrm) error { _, err := internalclient.PutObject(ctx, prm) if err != nil { return fmt.Errorf("(%T) could not put object to %s: %w", t, t.nodeInfo.AddressGroup(), err) @@ -72,7 +72,7 @@ func (t *remoteTarget) putStream(ctx context.Context, prm internalclient.PutObje return nil } -func (t *remoteTarget) putSingle(ctx context.Context, prm internalclient.PutObjectPrm) error { +func (t *remoteWriter) putSingle(ctx context.Context, prm internalclient.PutObjectPrm) error { _, err := internalclient.PutObjectSingle(ctx, prm) if err != nil { return fmt.Errorf("(%T) could not put single object to %s: %w", t, t.nodeInfo.AddressGroup(), err) @@ -113,7 +113,7 @@ func (s *RemoteSender) PutObject(ctx context.Context, p *RemotePutPrm) error { return err } - t := &remoteTarget{ + t := &remoteWriter{ privateKey: key, clientConstructor: s.clientConstructor, } diff --git a/pkg/services/object/common/writer/writer.go b/pkg/services/object/common/writer/writer.go new file mode 100644 index 000000000..3d50da988 --- /dev/null +++ b/pkg/services/object/common/writer/writer.go @@ -0,0 +1,183 @@ +package writer + +import ( + "context" + "crypto/ecdsa" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" + objutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" +) + +type MaxSizeSource interface { + // MaxObjectSize returns maximum payload size + // of physically stored object in system. + // + // Must return 0 if value can not be obtained. + MaxObjectSize() uint64 +} + +type ClientConstructor interface { + Get(client.NodeInfo) (client.MultiAddressClient, error) +} + +type InnerRing interface { + InnerRingKeys() ([][]byte, error) +} + +type FormatValidatorConfig interface { + VerifySessionTokenIssuer() bool +} + +// Config represents a set of static parameters that are established during +// the initialization phase of all services. +type Config struct { + KeyStorage *objutil.KeyStorage + + MaxSizeSrc MaxSizeSource + + LocalStore ObjectStorage + + ContainerSource container.Source + + NetmapSource netmap.Source + + RemotePool, LocalPool util.WorkerPool + + NetmapKeys netmap.AnnouncedKeys + + FormatValidator *object.FormatValidator + + NetworkState netmap.State + + ClientConstructor ClientConstructor + + Logger *logger.Logger + + VerifySessionTokenIssuer bool +} + +type Option func(*Config) + +func WithWorkerPools(remote, local util.WorkerPool) Option { + return func(c *Config) { + c.RemotePool, c.LocalPool = remote, local + } +} + +func WithLogger(l *logger.Logger) Option { + return func(c *Config) { + c.Logger = l + } +} + +func WithVerifySessionTokenIssuer(v bool) Option { + return func(c *Config) { + c.VerifySessionTokenIssuer = v + } +} + +func (c *Config) getWorkerPool(pub []byte) (util.WorkerPool, bool) { + if c.NetmapKeys.IsLocalKey(pub) { + return c.LocalPool, true + } + return c.RemotePool, false +} + +type Params struct { + Config *Config + + Common *objutil.CommonPrm + + Header *objectSDK.Object + + Container containerSDK.Container + + TraverseOpts []placement.Option + + Relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error + + SignRequestPrivateKey *ecdsa.PrivateKey +} + +func New(prm *Params) transformer.ObjectWriter { + if container.IsECContainer(prm.Container) && object.IsECSupported(prm.Header) { + return newECWriter(prm) + } + return newDefaultObjectWriter(prm, false) +} + +func newDefaultObjectWriter(prm *Params, forECPlacement bool) transformer.ObjectWriter { + var relay func(context.Context, NodeDescriptor) error + if prm.Relay != nil { + relay = func(ctx context.Context, node NodeDescriptor) error { + var info client.NodeInfo + + client.NodeInfoFromNetmapElement(&info, node.Info) + + c, err := prm.Config.ClientConstructor.Get(info) + if err != nil { + return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) + } + + return prm.Relay(ctx, info, c) + } + } + + var resetSuccessAfterOnBroadcast bool + traverseOpts := prm.TraverseOpts + if forECPlacement && !prm.Common.LocalOnly() { + // save non-regular and linking object to EC container. + // EC 2.1 -> REP 2, EC 2.2 -> REP 3 etc. + traverseOpts = append(traverseOpts, placement.SuccessAfter(uint32(policy.ECParityCount(prm.Container.PlacementPolicy())+1))) + resetSuccessAfterOnBroadcast = true + } + + return &distributedWriter{ + cfg: prm.Config, + placementOpts: traverseOpts, + resetSuccessAfterOnBroadcast: resetSuccessAfterOnBroadcast, + nodeTargetInitializer: func(node NodeDescriptor) preparedObjectTarget { + if node.Local { + return LocalTarget{ + Storage: prm.Config.LocalStore, + } + } + + rt := &remoteWriter{ + privateKey: prm.SignRequestPrivateKey, + commonPrm: prm.Common, + clientConstructor: prm.Config.ClientConstructor, + } + + client.NodeInfoFromNetmapElement(&rt.nodeInfo, node.Info) + + return rt + }, + relay: relay, + } +} + +func newECWriter(prm *Params) transformer.ObjectWriter { + return &objectWriterDispatcher{ + ecWriter: &ECWriter{ + Config: prm.Config, + PlacementOpts: append(prm.TraverseOpts, placement.WithCopyNumbers(nil)), // copies number ignored for EC + Container: prm.Container, + Key: prm.SignRequestPrivateKey, + CommonPrm: prm.Common, + Relay: prm.Relay, + }, + repWriter: newDefaultObjectWriter(prm, true), + } +} diff --git a/pkg/services/object/patch/service.go b/pkg/services/object/patch/service.go index c4ab15abf..f1082dfff 100644 --- a/pkg/services/object/patch/service.go +++ b/pkg/services/object/patch/service.go @@ -2,43 +2,40 @@ package patchsvc import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" + objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" - putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" ) // Service implements Put operation of Object service v2. type Service struct { - keyStorage *util.KeyStorage + *objectwriter.Config getSvc *getsvc.Service - - putSvc *putsvc.Service } // NewService constructs Service instance from provided options. -func NewService(ks *util.KeyStorage, getSvc *getsvc.Service, putSvc *putsvc.Service) *Service { +// +// Patch service can use the same objectwriter.Config initializied by Put service. +func NewService(cfg *objectwriter.Config, + getSvc *getsvc.Service, +) *Service { return &Service{ - keyStorage: ks, + Config: cfg, getSvc: getSvc, - - putSvc: putSvc, } } // Put calls internal service and returns v2 object streamer. func (s *Service) Patch() (object.PatchObjectStream, error) { - nodeKey, err := s.keyStorage.GetKey(nil) + nodeKey, err := s.Config.KeyStorage.GetKey(nil) if err != nil { return nil, err } return &Streamer{ - getSvc: s.getSvc, - - putSvc: s.putSvc, - + Config: s.Config, + getSvc: s.getSvc, localNodeKey: nodeKey, }, nil } diff --git a/pkg/services/object/patch/streamer.go b/pkg/services/object/patch/streamer.go index 84363530e..85c28cda0 100644 --- a/pkg/services/object/patch/streamer.go +++ b/pkg/services/object/patch/streamer.go @@ -9,8 +9,9 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" refsV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/target" + objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" - putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -21,6 +22,8 @@ import ( // Streamer for the patch handler is a pipeline that merges two incoming streams of patches // and original object payload chunks. The merged result is fed to Put stream target. type Streamer struct { + *objectwriter.Config + // Patcher must be initialized at first Streamer.Send call. patcher patcher.PatchApplier @@ -28,8 +31,6 @@ type Streamer struct { getSvc *getsvc.Service - putSvc *putsvc.Service - localNodeKey *ecdsa.PrivateKey } @@ -78,11 +79,6 @@ func (s *Streamer) init(ctx context.Context, req *objectV2.PatchRequest) error { localNodeKey: s.localNodeKey, } - putstm, err := s.putSvc.Put() - if err != nil { - return err - } - hdr := hdrWithSig.GetHeader() oV2 := new(objectV2.Object) hV2 := new(objectV2.Header) @@ -97,14 +93,14 @@ func (s *Streamer) init(ctx context.Context, req *objectV2.PatchRequest) error { } oV2.GetHeader().SetOwnerID(ownerID) - prm, err := s.putInitPrm(req, oV2) + target, err := target.New(&objectwriter.Params{ + Config: s.Config, + Common: commonPrm, + Header: objectSDK.NewFromV2(oV2), + SignRequestPrivateKey: s.localNodeKey, + }) if err != nil { - return err - } - - err = putstm.Init(ctx, prm) - if err != nil { - return err + return fmt.Errorf("target creation: %w", err) } patcherPrm := patcher.Params{ @@ -112,7 +108,7 @@ func (s *Streamer) init(ctx context.Context, req *objectV2.PatchRequest) error { RangeProvider: rangeProvider, - ObjectWriter: putstm.Target(), + ObjectWriter: target, } s.patcher = patcher.New(patcherPrm) diff --git a/pkg/services/object/patch/util.go b/pkg/services/object/patch/util.go index 1218d6694..4f3c3ef17 100644 --- a/pkg/services/object/patch/util.go +++ b/pkg/services/object/patch/util.go @@ -6,31 +6,12 @@ import ( "errors" "fmt" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) -// putInitPrm initializes put paramerer for Put stream. -func (s *Streamer) putInitPrm(req *objectV2.PatchRequest, obj *objectV2.Object) (*putsvc.PutInitPrm, error) { - commonPrm, err := util.CommonPrmFromV2(req) - if err != nil { - return nil, err - } - - prm := new(putsvc.PutInitPrm) - prm.WithObject(objectSDK.NewFromV2(obj)). - WithCommonPrm(commonPrm). - WithPrivateKey(s.localNodeKey) - - return prm, nil -} - func newOwnerID(vh *session.RequestVerificationHeader) (*refs.OwnerID, error) { for vh.GetOrigin() != nil { vh = vh.GetOrigin() diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index a93873738..8cf4f0d62 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -1,132 +1,66 @@ package putsvc import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" objutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) -type MaxSizeSource interface { - // MaxObjectSize returns maximum payload size - // of physically stored object in system. - // - // Must return 0 if value can not be obtained. - MaxObjectSize() uint64 -} - type Service struct { - *cfg -} - -type Option func(*cfg) - -type ClientConstructor interface { - Get(client.NodeInfo) (client.MultiAddressClient, error) -} - -type InnerRing interface { - InnerRingKeys() ([][]byte, error) -} - -type FormatValidatorConfig interface { - VerifySessionTokenIssuer() bool -} - -type cfg struct { - keyStorage *objutil.KeyStorage - - maxSizeSrc MaxSizeSource - - localStore ObjectStorage - - cnrSrc container.Source - - netMapSrc netmap.Source - - remotePool, localPool util.WorkerPool - - netmapKeys netmap.AnnouncedKeys - - fmtValidator *object.FormatValidator - - networkState netmap.State - - clientConstructor ClientConstructor - - log *logger.Logger - - verifySessionTokenIssuer bool + *objectwriter.Config } func NewService(ks *objutil.KeyStorage, - cc ClientConstructor, - ms MaxSizeSource, - os ObjectStorage, + cc objectwriter.ClientConstructor, + ms objectwriter.MaxSizeSource, + os objectwriter.ObjectStorage, cs container.Source, ns netmap.Source, nk netmap.AnnouncedKeys, nst netmap.State, - ir InnerRing, - opts ...Option, + ir objectwriter.InnerRing, + opts ...objectwriter.Option, ) *Service { - c := &cfg{ - remotePool: util.NewPseudoWorkerPool(), - localPool: util.NewPseudoWorkerPool(), - log: &logger.Logger{Logger: zap.L()}, - keyStorage: ks, - clientConstructor: cc, - maxSizeSrc: ms, - localStore: os, - cnrSrc: cs, - netMapSrc: ns, - netmapKeys: nk, - networkState: nst, + c := &objectwriter.Config{ + RemotePool: util.NewPseudoWorkerPool(), + LocalPool: util.NewPseudoWorkerPool(), + Logger: &logger.Logger{Logger: zap.L()}, + KeyStorage: ks, + ClientConstructor: cc, + MaxSizeSrc: ms, + LocalStore: os, + ContainerSource: cs, + NetmapSource: ns, + NetmapKeys: nk, + NetworkState: nst, } for i := range opts { opts[i](c) } - c.fmtValidator = object.NewFormatValidator( + c.FormatValidator = object.NewFormatValidator( object.WithLockSource(os), object.WithNetState(nst), object.WithInnerRing(ir), object.WithNetmapSource(ns), object.WithContainersSource(cs), - object.WithVerifySessionTokenIssuer(c.verifySessionTokenIssuer), - object.WithLogger(c.log), + object.WithVerifySessionTokenIssuer(c.VerifySessionTokenIssuer), + object.WithLogger(c.Logger), ) return &Service{ - cfg: c, + Config: c, } } func (p *Service) Put() (*Streamer, error) { return &Streamer{ - cfg: p.cfg, + Config: p.Config, }, nil } - -func WithWorkerPools(remote, local util.WorkerPool) Option { - return func(c *cfg) { - c.remotePool, c.localPool = remote, local - } -} - -func WithLogger(l *logger.Logger) Option { - return func(c *cfg) { - c.log = l - } -} - -func WithVerifySessionTokenIssuer(v bool) Option { - return func(c *cfg) { - c.verifySessionTokenIssuer = v - } -} diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 3cc8518f5..9b4163268 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -21,6 +21,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/target" + objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" @@ -97,12 +99,12 @@ func (s *Service) validatePutSingle(ctx context.Context, obj *objectSDK.Object) func (s *Service) validarePutSingleSize(obj *objectSDK.Object) error { if uint64(len(obj.Payload())) != obj.PayloadSize() { - return ErrWrongPayloadSize + return target.ErrWrongPayloadSize } - maxAllowedSize := s.maxSizeSrc.MaxObjectSize() + maxAllowedSize := s.Config.MaxSizeSrc.MaxObjectSize() if obj.PayloadSize() > maxAllowedSize { - return ErrExceedingMaxSize + return target.ErrExceedingMaxSize } return nil @@ -137,11 +139,11 @@ func (s *Service) validatePutSingleChecksum(obj *objectSDK.Object) error { } func (s *Service) validatePutSingleObject(ctx context.Context, obj *objectSDK.Object) (object.ContentMeta, error) { - if err := s.fmtValidator.Validate(ctx, obj, false); err != nil { + if err := s.FormatValidator.Validate(ctx, obj, false); err != nil { return object.ContentMeta{}, fmt.Errorf("coud not validate object format: %w", err) } - meta, err := s.fmtValidator.ValidateContent(obj) + meta, err := s.FormatValidator.ValidateContent(obj) if err != nil { return object.ContentMeta{}, fmt.Errorf("could not validate payload content: %w", err) } @@ -164,17 +166,17 @@ func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *o } func (s *Service) saveToREPReplicas(ctx context.Context, placement putSinglePlacement, obj *objectSDK.Object, localOnly bool, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { - iter := s.cfg.newNodeIterator(placement.placementOptions) - iter.extraBroadcastEnabled = needAdditionalBroadcast(obj, localOnly) - iter.resetSuccessAfterOnBroadcast = placement.resetSuccessAfterOnBroadcast + iter := s.Config.NewNodeIterator(placement.placementOptions) + iter.ExtraBroadcastEnabled = objectwriter.NeedAdditionalBroadcast(obj, localOnly) + iter.ResetSuccessAfterOnBroadcast = placement.resetSuccessAfterOnBroadcast signer := &putSingleRequestSigner{ req: req, - keyStorage: s.keyStorage, + keyStorage: s.Config.KeyStorage, signer: &sync.Once{}, } - return iter.forEachNode(ctx, func(ctx context.Context, nd nodeDesc) error { + return iter.ForEachNode(ctx, func(ctx context.Context, nd objectwriter.NodeDescriptor) error { return s.saveToPlacementNode(ctx, &nd, obj, signer, meta) }) } @@ -184,25 +186,25 @@ func (s *Service) saveToECReplicas(ctx context.Context, placement putSinglePlace if err != nil { return err } - key, err := s.cfg.keyStorage.GetKey(nil) + key, err := s.Config.KeyStorage.GetKey(nil) if err != nil { return err } signer := &putSingleRequestSigner{ req: req, - keyStorage: s.keyStorage, + keyStorage: s.Config.KeyStorage, signer: &sync.Once{}, } - w := ecWriter{ - cfg: s.cfg, - placementOpts: placement.placementOptions, - objMeta: meta, - objMetaValid: true, - commonPrm: commonPrm, - container: placement.container, - key: key, - relay: func(ctx context.Context, ni client.NodeInfo, mac client.MultiAddressClient) error { + w := objectwriter.ECWriter{ + Config: s.Config, + PlacementOpts: placement.placementOptions, + ObjectMeta: meta, + ObjectMetaValid: true, + CommonPrm: commonPrm, + Container: placement.container, + Key: key, + Relay: func(ctx context.Context, ni client.NodeInfo, mac client.MultiAddressClient) error { return s.redirectPutSingleRequest(ctx, signer, obj, ni, mac) }, } @@ -223,7 +225,7 @@ func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumb if !ok { return result, errors.New("missing container ID") } - cnrInfo, err := s.cnrSrc.Get(cnrID) + cnrInfo, err := s.Config.ContainerSource.Get(cnrID) if err != nil { return result, fmt.Errorf("could not get container by ID: %w", err) } @@ -247,31 +249,31 @@ func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumb } result.placementOptions = append(result.placementOptions, placement.ForObject(objID)) - latestNetmap, err := netmap.GetLatestNetworkMap(s.netMapSrc) + latestNetmap, err := netmap.GetLatestNetworkMap(s.Config.NetmapSource) if err != nil { return result, fmt.Errorf("could not get latest network map: %w", err) } builder := placement.NewNetworkMapBuilder(latestNetmap) if localOnly { result.placementOptions = append(result.placementOptions, placement.SuccessAfter(1)) - builder = svcutil.NewLocalPlacement(builder, s.netmapKeys) + builder = svcutil.NewLocalPlacement(builder, s.Config.NetmapKeys) } result.placementOptions = append(result.placementOptions, placement.UseBuilder(builder)) return result, nil } -func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *nodeDesc, obj *objectSDK.Object, +func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *objectwriter.NodeDescriptor, obj *objectSDK.Object, signer *putSingleRequestSigner, meta object.ContentMeta, ) error { - if nodeDesc.local { + if nodeDesc.Local { return s.saveLocal(ctx, obj, meta) } var info client.NodeInfo - client.NodeInfoFromNetmapElement(&info, nodeDesc.info) + client.NodeInfoFromNetmapElement(&info, nodeDesc.Info) - c, err := s.clientConstructor.Get(info) + c, err := s.Config.ClientConstructor.Get(info) if err != nil { return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) } @@ -280,8 +282,8 @@ func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *nodeDesc, o } func (s *Service) saveLocal(ctx context.Context, obj *objectSDK.Object, meta object.ContentMeta) error { - localTarget := &localTarget{ - storage: s.localStore, + localTarget := &objectwriter.LocalTarget{ + Storage: s.Config.LocalStore, } return localTarget.WriteObject(ctx, obj, meta) } @@ -314,7 +316,7 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, if err != nil { objID, _ := obj.ID() cnrID, _ := obj.ContainerID() - s.log.Warn(logs.PutSingleRedirectFailure, + s.Config.Logger.Warn(logs.PutSingleRedirectFailure, zap.Error(err), zap.Stringer("address", addr), zap.Stringer("object_id", objID), diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index 6b396ec96..f3803d433 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -2,33 +2,21 @@ package putsvc import ( "context" - "crypto/ecdsa" "errors" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - pkgutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" - containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/target" + objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) type Streamer struct { - *cfg - - privateKey *ecdsa.PrivateKey + *objectwriter.Config target transformer.ChunkedObjectWriter relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error - - maxPayloadSz uint64 // network config } var errNotInit = errors.New("stream not initialized") @@ -36,8 +24,23 @@ var errNotInit = errors.New("stream not initialized") var errInitRecall = errors.New("init recall") func (p *Streamer) Init(ctx context.Context, prm *PutInitPrm) error { + if p.target != nil { + return errInitRecall + } + // initialize destination target - if err := p.initTarget(prm); err != nil { + prmTarget := &objectwriter.Params{ + Config: p.Config, + Common: prm.common, + Header: prm.hdr, + Container: prm.cnr, + TraverseOpts: prm.traverseOpts, + Relay: p.relay, + } + + var err error + p.target, err = target.New(prmTarget) + if err != nil { return fmt.Errorf("(%T) could not initialize object target: %w", p, err) } @@ -47,253 +50,6 @@ func (p *Streamer) Init(ctx context.Context, prm *PutInitPrm) error { return nil } -// Target accesses underlying target chunked object writer. -func (p *Streamer) Target() transformer.ChunkedObjectWriter { - return p.target -} - -// MaxObjectSize returns maximum payload size for the streaming session. -// -// Must be called after the successful Init. -func (p *Streamer) MaxObjectSize() uint64 { - return p.maxPayloadSz -} - -func (p *Streamer) initTarget(prm *PutInitPrm) error { - // prevent re-calling - if p.target != nil { - return errInitRecall - } - - // prepare needed put parameters - if err := p.preparePrm(prm); err != nil { - return fmt.Errorf("(%T) could not prepare put parameters: %w", p, err) - } - - p.maxPayloadSz = p.maxSizeSrc.MaxObjectSize() - if p.maxPayloadSz == 0 { - return fmt.Errorf("(%T) could not obtain max object size parameter", p) - } - - if prm.hdr.Signature() != nil { - return p.initUntrustedTarget(prm) - } - return p.initTrustedTarget(prm) -} - -func (p *Streamer) initUntrustedTarget(prm *PutInitPrm) error { - p.relay = prm.relay - - if prm.privateKey != nil { - p.privateKey = prm.privateKey - } else { - nodeKey, err := p.cfg.keyStorage.GetKey(nil) - if err != nil { - return err - } - p.privateKey = nodeKey - } - - // prepare untrusted-Put object target - p.target = &validatingPreparedTarget{ - nextTarget: newInMemoryObjectBuilder(p.newObjectWriter(prm)), - fmt: p.fmtValidator, - - maxPayloadSz: p.maxPayloadSz, - } - - return nil -} - -func (p *Streamer) initTrustedTarget(prm *PutInitPrm) error { - sToken := prm.common.SessionToken() - - // prepare trusted-Put object target - - // get private token from local storage - var sessionInfo *util.SessionInfo - - if sToken != nil { - sessionInfo = &util.SessionInfo{ - ID: sToken.ID(), - Owner: sToken.Issuer(), - } - } - - key, err := p.keyStorage.GetKey(sessionInfo) - if err != nil { - return fmt.Errorf("(%T) could not receive session key: %w", p, err) - } - - // In case session token is missing, the line above returns the default key. - // If it isn't owner key, replication attempts will fail, thus this check. - ownerObj := prm.hdr.OwnerID() - if ownerObj.IsEmpty() { - return errors.New("missing object owner") - } - - if sToken == nil { - var ownerSession user.ID - user.IDFromKey(&ownerSession, key.PublicKey) - - if !ownerObj.Equals(ownerSession) { - return fmt.Errorf("(%T) session token is missing but object owner id is different from the default key", p) - } - } else { - if !ownerObj.Equals(sessionInfo.Owner) { - return fmt.Errorf("(%T) different token issuer and object owner identifiers %s/%s", p, sessionInfo.Owner, ownerObj) - } - } - - if prm.privateKey != nil { - p.privateKey = prm.privateKey - } else { - p.privateKey = key - } - p.target = &validatingTarget{ - fmt: p.fmtValidator, - nextTarget: transformer.NewPayloadSizeLimiter(transformer.Params{ - Key: key, - NextTargetInit: func() transformer.ObjectWriter { return p.newObjectWriter(prm) }, - NetworkState: p.networkState, - MaxSize: p.maxPayloadSz, - WithoutHomomorphicHash: containerSDK.IsHomomorphicHashingDisabled(prm.cnr), - SessionToken: sToken, - }), - } - - return nil -} - -func (p *Streamer) preparePrm(prm *PutInitPrm) error { - var err error - - // get latest network map - nm, err := netmap.GetLatestNetworkMap(p.netMapSrc) - if err != nil { - return fmt.Errorf("(%T) could not get latest network map: %w", p, err) - } - - idCnr, ok := prm.hdr.ContainerID() - if !ok { - return errors.New("missing container ID") - } - - // get container to store the object - cnrInfo, err := p.cnrSrc.Get(idCnr) - if err != nil { - return fmt.Errorf("(%T) could not get container by ID: %w", p, err) - } - - prm.cnr = cnrInfo.Value - - // add common options - prm.traverseOpts = append(prm.traverseOpts, - // set processing container - placement.ForContainer(prm.cnr), - ) - - if ech := prm.hdr.ECHeader(); ech != nil { - prm.traverseOpts = append(prm.traverseOpts, - // set identifier of the processing object - placement.ForObject(ech.Parent()), - ) - } else if id, ok := prm.hdr.ID(); ok { - prm.traverseOpts = append(prm.traverseOpts, - // set identifier of the processing object - placement.ForObject(id), - ) - } - - // create placement builder from network map - builder := placement.NewNetworkMapBuilder(nm) - - if prm.common.LocalOnly() { - // restrict success count to 1 stored copy (to local storage) - prm.traverseOpts = append(prm.traverseOpts, placement.SuccessAfter(1)) - - // use local-only placement builder - builder = util.NewLocalPlacement(builder, p.netmapKeys) - } - - // set placement builder - prm.traverseOpts = append(prm.traverseOpts, placement.UseBuilder(builder)) - - return nil -} - -func (p *Streamer) newObjectWriter(prm *PutInitPrm) transformer.ObjectWriter { - if container.IsECContainer(prm.cnr) && object.IsECSupported(prm.hdr) { - return p.newECWriter(prm) - } - return p.newDefaultObjectWriter(prm, false) -} - -func (p *Streamer) newDefaultObjectWriter(prm *PutInitPrm, forECPlacement bool) transformer.ObjectWriter { - var relay func(context.Context, nodeDesc) error - if p.relay != nil { - relay = func(ctx context.Context, node nodeDesc) error { - var info client.NodeInfo - - client.NodeInfoFromNetmapElement(&info, node.info) - - c, err := p.clientConstructor.Get(info) - if err != nil { - return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) - } - - return p.relay(ctx, info, c) - } - } - - var resetSuccessAfterOnBroadcast bool - traverseOpts := prm.traverseOpts - if forECPlacement && !prm.common.LocalOnly() { - // save non-regular and linking object to EC container. - // EC 2.1 -> REP 2, EC 2.2 -> REP 3 etc. - traverseOpts = append(traverseOpts, placement.SuccessAfter(uint32(policy.ECParityCount(prm.cnr.PlacementPolicy())+1))) - resetSuccessAfterOnBroadcast = true - } - - return &distributedTarget{ - cfg: p.cfg, - placementOpts: traverseOpts, - resetSuccessAfterOnBroadcast: resetSuccessAfterOnBroadcast, - nodeTargetInitializer: func(node nodeDesc) preparedObjectTarget { - if node.local { - return localTarget{ - storage: p.localStore, - } - } - - rt := &remoteTarget{ - privateKey: p.privateKey, - commonPrm: prm.common, - clientConstructor: p.clientConstructor, - } - - client.NodeInfoFromNetmapElement(&rt.nodeInfo, node.info) - - return rt - }, - relay: relay, - } -} - -func (p *Streamer) newECWriter(prm *PutInitPrm) transformer.ObjectWriter { - return &objectWriterDispatcher{ - ecWriter: &ecWriter{ - cfg: p.cfg, - placementOpts: append(prm.traverseOpts, placement.WithCopyNumbers(nil)), // copies number ignored for EC - container: prm.cnr, - key: p.privateKey, - commonPrm: prm.common, - relay: p.relay, - }, - repWriter: p.newDefaultObjectWriter(prm, true), - } -} - func (p *Streamer) SendChunk(ctx context.Context, prm *PutChunkPrm) error { if p.target == nil { return errNotInit @@ -327,10 +83,3 @@ func (p *Streamer) Close(ctx context.Context) (*PutResponse, error) { id: ids.SelfID, }, nil } - -func (c *cfg) getWorkerPool(pub []byte) (pkgutil.WorkerPool, bool) { - if c.netmapKeys.IsLocalKey(pub) { - return c.localPool, true - } - return c.remotePool, false -} diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 9c6de4ca8..5bf15b4cd 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/target" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" @@ -55,7 +56,7 @@ func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) s.saveChunks = v.GetSignature() != nil if s.saveChunks { - maxSz := s.stream.MaxObjectSize() + maxSz := s.stream.MaxSizeSrc.MaxObjectSize() s.sizes = &sizes{ payloadSz: uint64(v.GetHeader().GetPayloadLength()), @@ -63,7 +64,7 @@ func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) // check payload size limit overflow if s.payloadSz > maxSz { - return putsvc.ErrExceedingMaxSize + return target.ErrExceedingMaxSize } s.init = req @@ -74,7 +75,7 @@ func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) // check payload size overflow if s.writtenPayload > s.payloadSz { - return putsvc.ErrWrongPayloadSize + return target.ErrWrongPayloadSize } } @@ -117,7 +118,7 @@ func (s *streamer) CloseAndRecv(ctx context.Context) (*object.PutResponse, error if s.saveChunks { // check payload size correctness if s.writtenPayload != s.payloadSz { - return nil, putsvc.ErrWrongPayloadSize + return nil, target.ErrWrongPayloadSize } } diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 3d04b7084..7e5c6e093 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -5,7 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" + objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -52,7 +52,7 @@ func (p *Replicator) HandleReplicationTask(ctx context.Context, task Task, res T } } - prm := new(putsvc.RemotePutPrm). + prm := new(objectwriter.RemotePutPrm). WithObject(task.Obj) for i := 0; task.NumCopies > 0 && i < len(task.Nodes); i++ { diff --git a/pkg/services/replicator/replicator.go b/pkg/services/replicator/replicator.go index a67f2e766..f2f86daf0 100644 --- a/pkg/services/replicator/replicator.go +++ b/pkg/services/replicator/replicator.go @@ -4,8 +4,8 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" - putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -24,7 +24,7 @@ type cfg struct { log *logger.Logger - remoteSender *putsvc.RemoteSender + remoteSender *objectwriter.RemoteSender remoteGetter *getsvc.RemoteGetter @@ -67,7 +67,7 @@ func WithLogger(v *logger.Logger) Option { } // WithRemoteSender returns option to set remote object sender of Replicator. -func WithRemoteSender(v *putsvc.RemoteSender) Option { +func WithRemoteSender(v *objectwriter.RemoteSender) Option { return func(c *cfg) { c.remoteSender = v } From 108e4e07be5d75f852fee90f8c0d9e17952be3b5 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 3 Sep 2024 12:18:10 +0300 Subject: [PATCH 1320/1943] [#1349] node: Evacuate objects without setting mode to `MAINTENANCE` Signed-off-by: Anton Nikiforov --- .../modules/control/shards_list.go | 18 ++++---- docs/evacuation.md | 6 +++ internal/metrics/engine.go | 8 ++++ pkg/local_object_storage/engine/evacuate.go | 10 +++++ .../engine/evacuate_test.go | 28 ++++++++++++ pkg/local_object_storage/engine/metrics.go | 1 + pkg/local_object_storage/engine/shards.go | 4 ++ pkg/local_object_storage/shard/exists.go | 4 ++ pkg/local_object_storage/shard/get.go | 14 +++++- pkg/local_object_storage/shard/head.go | 7 +++ pkg/local_object_storage/shard/info.go | 3 ++ .../shard/metrics_test.go | 3 ++ pkg/local_object_storage/shard/range.go | 4 ++ pkg/local_object_storage/shard/shard.go | 11 +++++ pkg/services/control/server/evacuate_async.go | 3 ++ pkg/services/control/server/list_shards.go | 1 + pkg/services/control/types.proto | 3 ++ pkg/services/control/types_frostfs.pb.go | 45 ++++++++++++++++--- 18 files changed, 156 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/shards_list.go b/cmd/frostfs-cli/modules/control/shards_list.go index e9e49bb29..a81034a9e 100644 --- a/cmd/frostfs-cli/modules/control/shards_list.go +++ b/cmd/frostfs-cli/modules/control/shards_list.go @@ -65,13 +65,14 @@ func prettyPrintShardsJSON(cmd *cobra.Command, ii []control.ShardInfo) { out := make([]map[string]any, 0, len(ii)) for _, i := range ii { out = append(out, map[string]any{ - "shard_id": base58.Encode(i.GetShard_ID()), - "mode": shardModeToString(i.GetMode()), - "metabase": i.GetMetabasePath(), - "blobstor": i.GetBlobstor(), - "writecache": i.GetWritecachePath(), - "pilorama": i.GetPiloramaPath(), - "error_count": i.GetErrorCount(), + "shard_id": base58.Encode(i.GetShard_ID()), + "mode": shardModeToString(i.GetMode()), + "metabase": i.GetMetabasePath(), + "blobstor": i.GetBlobstor(), + "writecache": i.GetWritecachePath(), + "pilorama": i.GetPiloramaPath(), + "error_count": i.GetErrorCount(), + "evacuation_in_progress": i.GetEvacuationInProgress(), }) } @@ -105,7 +106,8 @@ func prettyPrintShards(cmd *cobra.Command, ii []control.ShardInfo) { sb.String()+ pathPrinter("Write-cache", i.GetWritecachePath())+ pathPrinter("Pilorama", i.GetPiloramaPath())+ - fmt.Sprintf("Error count: %d\n", i.GetErrorCount()), + fmt.Sprintf("Error count: %d\n", i.GetErrorCount())+ + fmt.Sprintf("Evacuation in progress: %t\n", i.GetEvacuationInProgress()), base58.Encode(i.GetShard_ID()), shardModeToString(i.GetMode()), ) diff --git a/docs/evacuation.md b/docs/evacuation.md index 9db514a9e..885ce169a 100644 --- a/docs/evacuation.md +++ b/docs/evacuation.md @@ -10,6 +10,12 @@ First of all, by the evacuation the data is transferred to other shards of the s Only one running evacuation process is allowed on the node at a time. +It is not necessary to turn maintenance mode on storage node. + +Once evacuation from shard started, it is impossible to read data from it via public API, except the case when evacuation stopped manually or node restarted. + +Because it is necessary to prevent removing by policer objects with policy `REP 1 ...` from remote node during evacuation. + `frostfs-cli` utility is used to manage evacuation. ## Commands diff --git a/internal/metrics/engine.go b/internal/metrics/engine.go index e37777e40..1d01c95ed 100644 --- a/internal/metrics/engine.go +++ b/internal/metrics/engine.go @@ -27,6 +27,7 @@ type EngineMetrics interface { IncRefillObjectsCount(shardID, path string, size int, success bool) SetRefillPercent(shardID, path string, percent uint32) SetRefillStatus(shardID, path, status string) + SetEvacuationInProgress(shardID string, value bool) WriteCache() WriteCacheMetrics GC() GCMetrics @@ -45,6 +46,7 @@ type engineMetrics struct { refillObjCounter *prometheus.GaugeVec refillPayloadCounter *prometheus.GaugeVec refillPercentCounter *prometheus.GaugeVec + evacuationInProgress *shardIDModeValue gc *gcMetrics writeCache *writeCacheMetrics @@ -72,6 +74,7 @@ func newEngineMetrics() *engineMetrics { refillObjCounter: newEngineGaugeVector("resync_metabase_objects_total", "Count of objects resynced from blobstore to metabase", []string{shardIDLabel, pathLabel, successLabel}), refillPayloadCounter: newEngineGaugeVector("resync_metabase_objects_size_bytes", "Size of objects resynced from blobstore to metabase", []string{shardIDLabel, pathLabel, successLabel}), refillPercentCounter: newEngineGaugeVector("resync_metabase_complete_percent", "Percent of resynced from blobstore to metabase completeness", []string{shardIDLabel, pathLabel}), + evacuationInProgress: newShardIDMode(engineSubsystem, "evacuation_in_progress", "Shard evacuation in progress"), } } @@ -124,6 +127,7 @@ func (m *engineMetrics) DeleteShardMetrics(shardID string) { m.refillPercentCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) m.mode.Delete(shardID) m.refillStatus.DeleteByShardID(shardID) + m.evacuationInProgress.Delete(shardID) } func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) { @@ -213,3 +217,7 @@ func (m *engineMetrics) SetRefillPercent(shardID, path string, percent uint32) { func (m *engineMetrics) SetRefillStatus(shardID, path, status string) { m.refillStatus.SetMode(shardID, path, status) } + +func (m *engineMetrics) SetEvacuationInProgress(shardID string, value bool) { + m.evacuationInProgress.SetMode(shardID, strconv.FormatBool(value)) +} diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 04e427e49..7bef6edfb 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -366,6 +366,7 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, shardID string listPrm.WithCount(defaultEvacuateBatchSize) sh := shardsToEvacuate[shardID] + sh.SetEvacuationInProgress(true) var c *meta.Cursor for { @@ -655,6 +656,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to var getPrm shard.GetPrm getPrm.SetAddress(addr) + getPrm.SkipEvacCheck(true) getRes, err := sh.Get(ctx, getPrm) if err != nil { @@ -765,3 +767,11 @@ func (e *StorageEngine) ResetEvacuationStatus(ctx context.Context) error { return e.evacuateLimiter.ResetEvacuationStatus() } + +func (e *StorageEngine) ResetEvacuationStatusForShards() { + e.mtx.RLock() + defer e.mtx.RUnlock() + for _, sh := range e.shards { + sh.SetEvacuationInProgress(false) + } +} diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 8d25dad4a..28529fab9 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -125,6 +125,34 @@ func TestEvacuateShardObjects(t *testing.T) { // Second case ensures that all objects are indeed moved and available. checkHasObjects(t) + // Objects on evacuated shards should be logically unavailable, but persisted on disk. + // This is necessary to prevent removing it by policer in case of `REP 1` policy. + for _, obj := range objects[len(objects)-objPerShard:] { + var prmGet shard.GetPrm + prmGet.SetAddress(objectCore.AddressOf(obj)) + _, err = e.shards[evacuateShardID].Get(context.Background(), prmGet) + require.Error(t, err) + + prmGet.SkipEvacCheck(true) + _, err = e.shards[evacuateShardID].Get(context.Background(), prmGet) + require.NoError(t, err) + + var prmHead shard.HeadPrm + prmHead.SetAddress(objectCore.AddressOf(obj)) + _, err = e.shards[evacuateShardID].Head(context.Background(), prmHead) + require.Error(t, err) + + var existsPrm shard.ExistsPrm + existsPrm.Address = objectCore.AddressOf(obj) + _, err = e.shards[evacuateShardID].Exists(context.Background(), existsPrm) + require.Error(t, err) + + var rngPrm shard.RngPrm + rngPrm.SetAddress(objectCore.AddressOf(obj)) + _, err = e.shards[evacuateShardID].GetRange(context.Background(), rngPrm) + require.Error(t, err) + } + // Calling it again is OK, but all objects are already moved, so no new PUTs should be done. res, err = e.Evacuate(context.Background(), prm) require.NoError(t, err) diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 72b5ae252..1c088c754 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -30,6 +30,7 @@ type MetricRegister interface { IncRefillObjectsCount(shardID, path string, size int, success bool) SetRefillPercent(shardID, path string, percent uint32) SetRefillStatus(shardID, path, status string) + SetEvacuationInProgress(shardID string, value bool) WriteCache() metrics.WriteCacheMetrics GC() metrics.GCMetrics diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 980b38a63..40584149e 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -97,6 +97,10 @@ func (m *metricsWithID) SetRefillStatus(path string, status string) { m.mw.SetRefillStatus(m.id, path, status) } +func (m *metricsWithID) SetEvacuationInProgress(value bool) { + m.mw.SetEvacuationInProgress(m.id, value) +} + // AddShard adds a new shard to the storage engine. // // Returns any error encountered that did not allow adding a shard. diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index b5a9604b4..784bf293a 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -5,7 +5,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -60,6 +62,8 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { if s.info.Mode.Disabled() { return ExistsRes{}, ErrShardDisabled + } else if s.info.EvacuationInProgress { + return ExistsRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) } else if s.info.Mode.NoMetabase() { var p common.ExistsPrm p.Address = prm.Address diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 2e7c84bcd..d1c393613 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -27,8 +27,9 @@ type storFetcher = func(stor *blobstor.BlobStor, id []byte) (*objectSDK.Object, // GetPrm groups the parameters of Get operation. type GetPrm struct { - addr oid.Address - skipMeta bool + addr oid.Address + skipMeta bool + skipEvacCheck bool } // GetRes groups the resulting values of Get operation. @@ -50,6 +51,11 @@ func (p *GetPrm) SetIgnoreMeta(ignore bool) { p.skipMeta = ignore } +// SkipEvacCheck is a Get option which instruct to skip check is evacuation in progress. +func (p *GetPrm) SkipEvacCheck(val bool) { + p.skipEvacCheck = val +} + // Object returns the requested object. func (r GetRes) Object() *objectSDK.Object { return r.obj @@ -85,6 +91,10 @@ func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { return GetRes{}, ErrShardDisabled } + if s.info.EvacuationInProgress && !prm.skipEvacCheck { + return GetRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) + } + cb := func(stor *blobstor.BlobStor, id []byte) (*objectSDK.Object, error) { var getPrm common.GetPrm getPrm.Address = prm.addr diff --git a/pkg/local_object_storage/shard/head.go b/pkg/local_object_storage/shard/head.go index 9d5d31260..ff57e3bf9 100644 --- a/pkg/local_object_storage/shard/head.go +++ b/pkg/local_object_storage/shard/head.go @@ -4,7 +4,9 @@ import ( "context" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" @@ -70,6 +72,11 @@ func (s *Shard) Head(ctx context.Context, prm HeadPrm) (HeadRes, error) { res, err = s.Get(ctx, getPrm) obj = res.Object() } else { + s.m.RLock() + defer s.m.RUnlock() + if s.info.EvacuationInProgress { + return HeadRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) + } var headParams meta.GetPrm headParams.SetAddress(prm.addr) headParams.SetRaw(prm.raw) diff --git a/pkg/local_object_storage/shard/info.go b/pkg/local_object_storage/shard/info.go index 1051ab3db..f01796ec7 100644 --- a/pkg/local_object_storage/shard/info.go +++ b/pkg/local_object_storage/shard/info.go @@ -16,6 +16,9 @@ type Info struct { // Shard mode. Mode mode.Mode + // True when evacuation is in progress. + EvacuationInProgress bool + // Information about the metabase. MetaBaseInfo meta.Info diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 1ef849c02..01a85da97 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -192,6 +192,9 @@ func (m *metricsStore) SetRefillStatus(_ string, status string) { m.refillStatus = status } +func (m *metricsStore) SetEvacuationInProgress(bool) { +} + func TestCounters(t *testing.T) { t.Parallel() diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 9491543c4..701268820 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -87,6 +87,10 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { s.m.RLock() defer s.m.RUnlock() + if s.info.EvacuationInProgress { + return RngRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) + } + if s.info.Mode.Disabled() { return RngRes{}, ErrShardDisabled } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index d11bcc36b..ac389b506 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -97,6 +97,8 @@ type MetricsWriter interface { SetRefillPercent(path string, percent uint32) // SetRefillStatus sets refill status. SetRefillStatus(path string, status string) + // SetEvacuationInProgress sets evacuation status + SetEvacuationInProgress(value bool) } type cfg struct { @@ -579,3 +581,12 @@ func (s *Shard) DeleteShardMetrics() { s.cfg.metricsWriter.DeleteShardMetrics() } } + +func (s *Shard) SetEvacuationInProgress(val bool) { + s.m.Lock() + defer s.m.Unlock() + s.info.EvacuationInProgress = val + if s.metricsWriter != nil { + s.metricsWriter.SetEvacuationInProgress(val) + } +} diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index b829573ec..aacebe9e3 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -101,6 +101,9 @@ func (s *Server) StopShardEvacuation(ctx context.Context, req *control.StopShard if err != nil { return nil, status.Error(codes.Internal, err.Error()) } + + s.s.ResetEvacuationStatusForShards() + return resp, nil } diff --git a/pkg/services/control/server/list_shards.go b/pkg/services/control/server/list_shards.go index 56bd9fc1f..efe2754ea 100644 --- a/pkg/services/control/server/list_shards.go +++ b/pkg/services/control/server/list_shards.go @@ -53,6 +53,7 @@ func (s *Server) ListShards(_ context.Context, req *control.ListShardsRequest) ( si.SetMode(m) si.SetErrorCount(sh.ErrorCount) + si.SetEvacuationInProgress(sh.EvacuationInProgress) shardInfos = append(shardInfos, *si) } diff --git a/pkg/services/control/types.proto b/pkg/services/control/types.proto index 55636d88a..d8135ed64 100644 --- a/pkg/services/control/types.proto +++ b/pkg/services/control/types.proto @@ -142,6 +142,9 @@ message ShardInfo { // Path to shard's pilorama storage. string pilorama_path = 7 [ json_name = "piloramaPath" ]; + + // Evacuation status. + bool evacuation_in_progress = 8 [ json_name = "evacuationInProgress" ]; } // Blobstor component description. diff --git a/pkg/services/control/types_frostfs.pb.go b/pkg/services/control/types_frostfs.pb.go index 42c1afa52..f92106589 100644 --- a/pkg/services/control/types_frostfs.pb.go +++ b/pkg/services/control/types_frostfs.pb.go @@ -954,13 +954,14 @@ func (x *Netmap) UnmarshalEasyJSON(in *jlexer.Lexer) { } type ShardInfo struct { - Shard_ID []byte `json:"shardID"` - MetabasePath string `json:"metabasePath"` - Blobstor []BlobstorInfo `json:"blobstor"` - WritecachePath string `json:"writecachePath"` - Mode ShardMode `json:"mode"` - ErrorCount uint32 `json:"errorCount"` - PiloramaPath string `json:"piloramaPath"` + Shard_ID []byte `json:"shardID"` + MetabasePath string `json:"metabasePath"` + Blobstor []BlobstorInfo `json:"blobstor"` + WritecachePath string `json:"writecachePath"` + Mode ShardMode `json:"mode"` + ErrorCount uint32 `json:"errorCount"` + PiloramaPath string `json:"piloramaPath"` + EvacuationInProgress bool `json:"evacuationInProgress"` } var ( @@ -986,6 +987,7 @@ func (x *ShardInfo) StableSize() (size int) { size += proto.EnumSize(5, int32(x.Mode)) size += proto.UInt32Size(6, x.ErrorCount) size += proto.StringSize(7, x.PiloramaPath) + size += proto.BoolSize(8, x.EvacuationInProgress) return size } @@ -1023,6 +1025,9 @@ func (x *ShardInfo) EmitProtobuf(mm *easyproto.MessageMarshaler) { if len(x.PiloramaPath) != 0 { mm.AppendString(7, x.PiloramaPath) } + if x.EvacuationInProgress { + mm.AppendBool(8, x.EvacuationInProgress) + } } // UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. @@ -1080,6 +1085,12 @@ func (x *ShardInfo) UnmarshalProtobuf(src []byte) (err error) { return fmt.Errorf("cannot unmarshal field %s", "PiloramaPath") } x.PiloramaPath = data + case 8: // EvacuationInProgress + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "EvacuationInProgress") + } + x.EvacuationInProgress = data } } return nil @@ -1147,6 +1158,15 @@ func (x *ShardInfo) GetPiloramaPath() string { func (x *ShardInfo) SetPiloramaPath(v string) { x.PiloramaPath = v } +func (x *ShardInfo) GetEvacuationInProgress() bool { + if x != nil { + return x.EvacuationInProgress + } + return false +} +func (x *ShardInfo) SetEvacuationInProgress(v bool) { + x.EvacuationInProgress = v +} // MarshalJSON implements the json.Marshaler interface. func (x *ShardInfo) MarshalJSON() ([]byte, error) { @@ -1202,6 +1222,11 @@ func (x *ShardInfo) MarshalEasyJSON(out *jwriter.Writer) { out.RawString(prefix) out.String(x.PiloramaPath) } + { + const prefix string = ",\"evacuationInProgress\":" + out.RawString(prefix) + out.Bool(x.EvacuationInProgress) + } out.RawByte('}') } @@ -1296,6 +1321,12 @@ func (x *ShardInfo) UnmarshalEasyJSON(in *jlexer.Lexer) { f = in.String() x.PiloramaPath = f } + case "evacuationInProgress": + { + var f bool + f = in.Bool() + x.EvacuationInProgress = f + } } in.WantComma() } From 273980cfb99a29eef0dbe4bd013c7a0041b6db79 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 5 Sep 2024 16:40:32 +0300 Subject: [PATCH 1321/1943] [#1310] object: Remove irrelevant comments Signed-off-by: Airat Arifullin --- pkg/services/object/common/target/target.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/services/object/common/target/target.go b/pkg/services/object/common/target/target.go index 00080ace6..980c4c6bd 100644 --- a/pkg/services/object/common/target/target.go +++ b/pkg/services/object/common/target/target.go @@ -116,7 +116,6 @@ func preparePrm(prm *objectwriter.Params) error { // get latest network map nm, err := netmap.GetLatestNetworkMap(prm.Config.NetmapSource) if err != nil { - //return fmt.Errorf("(%T) could not get latest network map: %w", p, err) return fmt.Errorf("could not get latest network map: %w", err) } @@ -128,7 +127,6 @@ func preparePrm(prm *objectwriter.Params) error { // get container to store the object cnrInfo, err := prm.Config.ContainerSource.Get(idCnr) if err != nil { - //return fmt.Errorf("(%T) could not get container by ID: %w", p, err) return fmt.Errorf("could not get container by ID: %w", err) } From f652518c241f405db22259c753be7f8685f39cc7 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 6 Sep 2024 13:09:58 +0300 Subject: [PATCH 1322/1943] [#1357] go: Fix panic caused by using range over slice len If slice is altered in `for` loop, we cannot use range over its length: it may cause panic if slice gets shorter. Signed-off-by: Ekaterina Lebedeva --- pkg/services/control/server/evacuate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index 300cb9dc9..ae3413373 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -169,7 +169,7 @@ func (s *Server) getContainerNodes(contID cid.ID) ([]netmap.NodeInfo, error) { nodes := placement.FlattenNodes(ns) bs := (*keys.PublicKey)(&s.key.PublicKey).Bytes() - for i := range len(nodes) { + for i := 0; i < len(nodes); i++ { // don't use range, slice mutates in body if bytes.Equal(nodes[i].PublicKey(), bs) { copy(nodes[i:], nodes[i+1:]) nodes = nodes[:len(nodes)-1] From 007827255ee65e9d7fd61c0f6bdd6bb59479bac1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 27 Aug 2024 15:51:55 +0300 Subject: [PATCH 1323/1943] [#1337] blobovniczatree: Add .rebuild temp files This allows to reduce open/close DBs to check incompleted rebuilds. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + .../blobstor/blobovniczatree/blobovnicza.go | 2 +- .../blobstor/blobovniczatree/control.go | 6 +-- .../blobstor/blobovniczatree/iterate.go | 21 ++++++++-- .../blobstor/blobovniczatree/rebuild.go | 38 ++++++++++++++++++- .../blobovniczatree/rebuild_failover_test.go | 12 ++++++ 6 files changed, 71 insertions(+), 9 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 78bcd0c0e..97b189529 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -543,4 +543,5 @@ const ( WritecacheSealCompletedAsync = "writecache seal completed successfully" FailedToSealWritecacheAsync = "failed to seal writecache async" WritecacheShrinkSkippedNotEmpty = "writecache shrink skipped: database is not empty" + BlobovniczatreeFailedToRemoveRebuildTempFile = "failed to remove rebuild temp file" ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index 952203367..c909113c7 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -135,7 +135,7 @@ func getBlobovniczaMaxIndex(directory string) (bool, uint64, error) { var hasDBs bool var maxIdx uint64 for _, e := range entries { - if e.IsDir() { + if e.IsDir() || strings.HasSuffix(e.Name(), rebuildSuffix) { continue } hasDBs = true diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index a31e9d6cb..681cf876c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -2,6 +2,7 @@ package blobovniczatree import ( "context" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -41,10 +42,9 @@ func (b *Blobovniczas) initializeDBs(ctx context.Context) error { eg, egCtx := errgroup.WithContext(ctx) eg.SetLimit(b.blzInitWorkerCount) - visited := make(map[string]struct{}) - err = b.iterateExistingDBPaths(egCtx, func(p string) (bool, error) { - visited[p] = struct{}{} + err = b.iterateIncompletedRebuildDBPaths(egCtx, func(p string) (bool, error) { eg.Go(func() error { + p = strings.TrimSuffix(p, rebuildSuffix) shBlz := b.getBlobovniczaWithoutCaching(p) blz, err := shBlz.Open() if err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index af3d9e720..f6acb46aa 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -188,11 +188,11 @@ func (b *Blobovniczas) iterateExistingDBPaths(ctx context.Context, f func(string b.dbFilesGuard.RLock() defer b.dbFilesGuard.RUnlock() - _, err := b.iterateExistingDBPathsDFS(ctx, "", f) + _, err := b.iterateExistingPathsDFS(ctx, "", f, func(path string) bool { return !strings.HasSuffix(path, rebuildSuffix) }) return err } -func (b *Blobovniczas) iterateExistingDBPathsDFS(ctx context.Context, path string, f func(string) (bool, error)) (bool, error) { +func (b *Blobovniczas) iterateExistingPathsDFS(ctx context.Context, path string, f func(string) (bool, error), fileFilter func(path string) bool) (bool, error) { sysPath := filepath.Join(b.rootPath, path) entries, err := os.ReadDir(sysPath) if os.IsNotExist(err) && b.readOnly && path == "" { // non initialized tree in read only mode @@ -208,7 +208,7 @@ func (b *Blobovniczas) iterateExistingDBPathsDFS(ctx context.Context, path strin default: } if entry.IsDir() { - stop, err := b.iterateExistingDBPathsDFS(ctx, filepath.Join(path, entry.Name()), f) + stop, err := b.iterateExistingPathsDFS(ctx, filepath.Join(path, entry.Name()), f, fileFilter) if err != nil { return false, err } @@ -216,6 +216,9 @@ func (b *Blobovniczas) iterateExistingDBPathsDFS(ctx context.Context, path strin return true, nil } } else { + if !fileFilter(entry.Name()) { + continue + } stop, err := f(filepath.Join(path, entry.Name())) if err != nil { return false, err @@ -228,6 +231,15 @@ func (b *Blobovniczas) iterateExistingDBPathsDFS(ctx context.Context, path strin return false, nil } +// iterateIncompletedRebuildDBPaths iterates over the paths of Blobovniczas with incompleted rebuild files without any order. +func (b *Blobovniczas) iterateIncompletedRebuildDBPaths(ctx context.Context, f func(string) (bool, error)) error { + b.dbFilesGuard.RLock() + defer b.dbFilesGuard.RUnlock() + + _, err := b.iterateExistingPathsDFS(ctx, "", f, func(path string) bool { return strings.HasSuffix(path, rebuildSuffix) }) + return err +} + func (b *Blobovniczas) iterateSortedDBPaths(ctx context.Context, addr oid.Address, f func(string) (bool, error)) error { b.dbFilesGuard.RLock() defer b.dbFilesGuard.RUnlock() @@ -249,6 +261,9 @@ func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path st var dirIdxs []uint64 for _, entry := range entries { + if strings.HasSuffix(entry.Name(), rebuildSuffix) { + continue + } idx := u64FromHexString(entry.Name()) if entry.IsDir() { dirIdxs = append(dirIdxs, idx) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index cfc17cfae..058fe1fb6 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -6,6 +6,7 @@ import ( "errors" "os" "path/filepath" + "strings" "sync" "sync/atomic" "time" @@ -19,6 +20,8 @@ import ( "golang.org/x/sync/errgroup" ) +const rebuildSuffix = ".rebuild" + var ( errRebuildInProgress = errors.New("rebuild is in progress, the operation cannot be performed") errBatchFull = errors.New("batch full") @@ -124,15 +127,36 @@ func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.M } shDB.Close() }() - + dropTempFile, err := b.addRebuildTempFile(path) + if err != nil { + return 0, err + } migratedObjects, err := b.moveObjects(ctx, blz, shDB.SystemPath(), meta, limiter) if err != nil { return migratedObjects, err } shDBClosed, err = b.dropDB(ctx, path, shDB) + if err == nil { + // drop only on success to continue rebuild on error + dropTempFile() + } return migratedObjects, err } +func (b *Blobovniczas) addRebuildTempFile(path string) (func(), error) { + sysPath := filepath.Join(b.rootPath, path) + sysPath = sysPath + rebuildSuffix + _, err := os.OpenFile(sysPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, b.perm) + if err != nil { + return nil, err + } + return func() { + if err := os.Remove(sysPath); err != nil { + b.log.Warn(logs.BlobovniczatreeFailedToRemoveRebuildTempFile, zap.Error(err)) + } + }, nil +} + func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovnicza, blzPath string, meta common.MetaStorage, limiter common.ConcurrentWorkersLimiter) (uint64, error) { var result atomic.Uint64 batch := make(map[oid.Address][]byte) @@ -256,7 +280,10 @@ func (b *Blobovniczas) dropDirectoryIfEmpty(path string) error { func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore common.MetaStorage) (uint64, error) { var count uint64 - return count, b.iterateExistingDBPaths(ctx, func(s string) (bool, error) { + var rebuildTempFilesToRemove []string + err := b.iterateIncompletedRebuildDBPaths(ctx, func(s string) (bool, error) { + rebuildTmpFilePath := s + s = strings.TrimSuffix(s, rebuildSuffix) shDB := b.getBlobovnicza(s) blz, err := shDB.Open() if err != nil { @@ -276,8 +303,15 @@ func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore co count++ } + rebuildTempFilesToRemove = append(rebuildTempFilesToRemove, rebuildTmpFilePath) return false, nil }) + for _, tmp := range rebuildTempFilesToRemove { + if err := os.Remove(filepath.Join(b.rootPath, tmp)); err != nil { + b.log.Warn(logs.BlobovniczatreeFailedToRemoveRebuildTempFile, zap.Error(err)) + } + } + return count, err } func (b *Blobovniczas) performMove(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index a6afed60c..9fec795ca 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -3,6 +3,7 @@ package blobovniczatree import ( "bytes" "context" + "os" "path/filepath" "sync" "testing" @@ -53,6 +54,8 @@ func testRebuildFailoverOnlyMoveInfoSaved(t *testing.T) { })) require.NoError(t, blz.Close()) + _, err = os.OpenFile(filepath.Join(dir, "0", "0", "1.db.rebuild"), os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, defaultPerm) + require.NoError(t, err) testRebuildFailoverValidate(t, dir, obj, true) } @@ -82,6 +85,9 @@ func testRebuildFailoverObjectSavedToTarget(t *testing.T) { require.NoError(t, blz.Close()) + _, err = os.OpenFile(filepath.Join(dir, "0", "0", "1.db.rebuild"), os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, defaultPerm) + require.NoError(t, err) + blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db"))) require.NoError(t, blz.Open()) require.NoError(t, blz.Init()) @@ -113,6 +119,9 @@ func testRebuildFailoverObjectDeletedFromSource(t *testing.T) { require.NoError(t, blz.Close()) + _, err = os.OpenFile(filepath.Join(dir, "0", "0", "1.db.rebuild"), os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, defaultPerm) + require.NoError(t, err) + blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db"))) require.NoError(t, blz.Open()) require.NoError(t, blz.Init()) @@ -194,4 +203,7 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object } require.NoError(t, blz.Close()) + + _, err = os.Stat(filepath.Join(dir, "0", "0", "1.db.rebuild")) + require.True(t, os.IsNotExist(err)) } From d508da8397026fac3fd2a5c77846f1a5ec61a665 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 29 Aug 2024 11:34:18 +0300 Subject: [PATCH 1324/1943] [#1337] blobovniczatree: Add rebuild by fill percent Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobovnicza/sizes.go | 4 + .../blobstor/blobovniczatree/rebuild.go | 93 ++++++++- .../blobstor/blobovniczatree/rebuild_test.go | 195 +++++++++++++++++- .../blobstor/common/rebuild.go | 8 + pkg/local_object_storage/blobstor/rebuild.go | 3 +- pkg/local_object_storage/shard/rebuild.go | 173 ++++++++++++++++ .../shard/rebuild_limiter.go | 13 -- pkg/local_object_storage/shard/rebuilder.go | 98 --------- pkg/local_object_storage/shard/shard.go | 2 +- 9 files changed, 470 insertions(+), 119 deletions(-) create mode 100644 pkg/local_object_storage/shard/rebuild.go delete mode 100644 pkg/local_object_storage/shard/rebuild_limiter.go delete mode 100644 pkg/local_object_storage/shard/rebuilder.go diff --git a/pkg/local_object_storage/blobovnicza/sizes.go b/pkg/local_object_storage/blobovnicza/sizes.go index 1dff75aed..9bbed0db5 100644 --- a/pkg/local_object_storage/blobovnicza/sizes.go +++ b/pkg/local_object_storage/blobovnicza/sizes.go @@ -57,3 +57,7 @@ func (b *Blobovnicza) itemDeleted(itemSize uint64) { func (b *Blobovnicza) IsFull() bool { return b.dataSize.Load() >= b.fullSizeLimit } + +func (b *Blobovnicza) FillPercent() int { + return int(100.0 * (float64(b.dataSize.Load()) / float64(b.fullSizeLimit))) +} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index 058fe1fb6..b7f20822e 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "os" "path/filepath" "strings" @@ -59,7 +60,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm b.log.Debug(logs.BlobovniczaTreeCompletedPreviousRebuildSuccess) b.log.Debug(logs.BlobovniczaTreeCollectingDBToRebuild) - dbsToMigrate, err := b.getDBsToRebuild(ctx) + dbsToMigrate, err := b.getDBsToRebuild(ctx, prm.Action) if err != nil { b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err)) success = false @@ -93,7 +94,33 @@ func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common. return res, nil } -func (b *Blobovniczas) getDBsToRebuild(ctx context.Context) ([]string, error) { +func (b *Blobovniczas) getDBsToRebuild(ctx context.Context, action common.RebuildAction) ([]string, error) { + schemaChange := make(map[string]struct{}) + fillPercent := make(map[string]struct{}) + var err error + if action.SchemaChange { + schemaChange, err = b.selectDBsDoNotMatchSchema(ctx) + if err != nil { + return nil, err + } + } + if action.FillPercent { + fillPercent, err = b.selectDBsDoNotMatchFillPercent(ctx, action.FillPercentValue) + if err != nil { + return nil, err + } + } + for k := range fillPercent { + schemaChange[k] = struct{}{} + } + result := make([]string, 0, len(schemaChange)) + for db := range schemaChange { + result = append(result, db) + } + return result, nil +} + +func (b *Blobovniczas) selectDBsDoNotMatchSchema(ctx context.Context) (map[string]struct{}, error) { dbsToMigrate := make(map[string]struct{}) if err := b.iterateExistingDBPaths(ctx, func(s string) (bool, error) { dbsToMigrate[s] = struct{}{} @@ -107,13 +134,69 @@ func (b *Blobovniczas) getDBsToRebuild(ctx context.Context) ([]string, error) { }); err != nil { return nil, err } - result := make([]string, 0, len(dbsToMigrate)) - for db := range dbsToMigrate { - result = append(result, db) + return dbsToMigrate, nil +} + +func (b *Blobovniczas) selectDBsDoNotMatchFillPercent(ctx context.Context, target int) (map[string]struct{}, error) { + if target <= 0 || target > 100 { + return nil, fmt.Errorf("invalid fill percent value %d: must be (0; 100]", target) + } + result := make(map[string]struct{}) + if err := b.iterateDeepest(ctx, oid.Address{}, func(lvlPath string) (bool, error) { + dir := filepath.Join(b.rootPath, lvlPath) + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { // non initialized tree + return false, nil + } + if err != nil { + return false, err + } + hasDBs := false + // db with maxIdx could be an active, so it should not be rebuilded + var maxIdx uint64 + for _, e := range entries { + if e.IsDir() || strings.HasSuffix(e.Name(), rebuildSuffix) { + continue + } + hasDBs = true + maxIdx = max(u64FromHexString(e.Name()), maxIdx) + } + if !hasDBs { + return false, nil + } + for _, e := range entries { + if e.IsDir() || strings.HasSuffix(e.Name(), rebuildSuffix) { + continue + } + if u64FromHexString(e.Name()) == maxIdx { + continue + } + path := filepath.Join(lvlPath, e.Name()) + resettlementRequired, err := b.fillPercentIsLow(path, target) + if err != nil { + return false, err + } + if resettlementRequired { + result[path] = struct{}{} + } + } + return false, nil + }); err != nil { + return nil, err } return result, nil } +func (b *Blobovniczas) fillPercentIsLow(path string, target int) (bool, error) { + shDB := b.getBlobovnicza(path) + blz, err := shDB.Open() + if err != nil { + return false, err + } + defer shDB.Close() + return blz.FillPercent() < target, nil +} + func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.MetaStorage, limiter common.ConcurrentWorkersLimiter) (uint64, error) { shDB := b.getBlobovnicza(path) blz, err := shDB.Open() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 4a51fd86a..62ae9ea90 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -15,7 +15,7 @@ import ( "golang.org/x/sync/errgroup" ) -func TestBlobovniczaTreeRebuild(t *testing.T) { +func TestBlobovniczaTreeSchemaRebuild(t *testing.T) { t.Parallel() t.Run("width increased", func(t *testing.T) { @@ -39,6 +39,197 @@ func TestBlobovniczaTreeRebuild(t *testing.T) { }) } +func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { + t.Parallel() + + t.Run("no rebuild by fill percent", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + b := NewBlobovniczaTree( + context.Background(), + WithLogger(test.NewLogger(t)), + WithObjectSizeLimit(64*1024), + WithBlobovniczaShallowWidth(1), // single directory + WithBlobovniczaShallowDepth(1), + WithRootPath(dir), + WithBlobovniczaSize(100*1024), // 100 KB limit for each blobovnicza + WithWaitBeforeDropDB(0), + WithOpenedCacheSize(1000), + WithMoveBatchSize(3)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) + require.NoError(t, b.Init()) + + storageIDs := make(map[oid.Address][]byte) + for i := 0; i < 100; i++ { + obj := blobstortest.NewObject(64 * 1024) // 64KB object + data, err := obj.Marshal() + require.NoError(t, err) + var prm common.PutPrm + prm.Address = object.AddressOf(obj) + prm.RawData = data + res, err := b.Put(context.Background(), prm) + require.NoError(t, err) + storageIDs[prm.Address] = res.StorageID + } + metaStub := &storageIDUpdateStub{ + storageIDs: storageIDs, + guard: &sync.Mutex{}, + } + rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ + MetaStorage: metaStub, + WorkerLimiter: &rebuildLimiterStub{}, + Action: common.RebuildAction{ + SchemaChange: false, + FillPercent: true, + FillPercentValue: 60, + }, + }) + require.NoError(t, err) + dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 + require.False(t, dataMigrated) + + for addr, storageID := range storageIDs { + var gPrm common.GetPrm + gPrm.Address = addr + gPrm.StorageID = storageID + _, err := b.Get(context.Background(), gPrm) + require.NoError(t, err) + } + + require.NoError(t, b.Close()) + }) + + t.Run("no rebuild single db", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + b := NewBlobovniczaTree( + context.Background(), + WithLogger(test.NewLogger(t)), + WithObjectSizeLimit(64*1024), + WithBlobovniczaShallowWidth(1), // single directory + WithBlobovniczaShallowDepth(1), + WithRootPath(dir), + WithBlobovniczaSize(100*1024), // 100 KB soft limit for each blobovnicza + WithWaitBeforeDropDB(0), + WithOpenedCacheSize(1000), + WithMoveBatchSize(3)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) + require.NoError(t, b.Init()) + + storageIDs := make(map[oid.Address][]byte) + obj := blobstortest.NewObject(64 * 1024) // 64KB object + data, err := obj.Marshal() + require.NoError(t, err) + var prm common.PutPrm + prm.Address = object.AddressOf(obj) + prm.RawData = data + res, err := b.Put(context.Background(), prm) + require.NoError(t, err) + storageIDs[prm.Address] = res.StorageID + metaStub := &storageIDUpdateStub{ + storageIDs: storageIDs, + guard: &sync.Mutex{}, + } + rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ + MetaStorage: metaStub, + WorkerLimiter: &rebuildLimiterStub{}, + Action: common.RebuildAction{ + SchemaChange: false, + FillPercent: true, + FillPercentValue: 90, // 64KB / 100KB = 64% + }, + }) + require.NoError(t, err) + dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 + require.False(t, dataMigrated) + + for addr, storageID := range storageIDs { + var gPrm common.GetPrm + gPrm.Address = addr + gPrm.StorageID = storageID + _, err := b.Get(context.Background(), gPrm) + require.NoError(t, err) + } + + require.NoError(t, b.Close()) + }) + + t.Run("rebuild by fill percent", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + b := NewBlobovniczaTree( + context.Background(), + WithLogger(test.NewLogger(t)), + WithObjectSizeLimit(64*1024), + WithBlobovniczaShallowWidth(1), // single directory + WithBlobovniczaShallowDepth(1), + WithRootPath(dir), + WithBlobovniczaSize(100*1024), // 100 KB limit for each blobovnicza + WithWaitBeforeDropDB(0), + WithOpenedCacheSize(1000), + WithMoveBatchSize(3)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) + require.NoError(t, b.Init()) + + storageIDs := make(map[oid.Address][]byte) + toDelete := make(map[oid.Address][]byte) + for i := 0; i < 100; i++ { // 2 objects for one blobovnicza, so 50 DBs total will be created + obj := blobstortest.NewObject(64 * 1024) + data, err := obj.Marshal() + require.NoError(t, err) + var prm common.PutPrm + prm.Address = object.AddressOf(obj) + prm.RawData = data + res, err := b.Put(context.Background(), prm) + require.NoError(t, err) + storageIDs[prm.Address] = res.StorageID + if i%2 == 1 { + toDelete[prm.Address] = res.StorageID + } + } + for addr, storageID := range toDelete { + var prm common.DeletePrm + prm.Address = addr + prm.StorageID = storageID + _, err := b.Delete(context.Background(), prm) + require.NoError(t, err) + } + metaStub := &storageIDUpdateStub{ + storageIDs: storageIDs, + guard: &sync.Mutex{}, + } + rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ + MetaStorage: metaStub, + WorkerLimiter: &rebuildLimiterStub{}, + Action: common.RebuildAction{ + SchemaChange: false, + FillPercent: true, + FillPercentValue: 80, + }, + }) + require.NoError(t, err) + require.Equal(t, uint64(49), rRes.FilesRemoved) + require.Equal(t, uint64(49), rRes.ObjectsMoved) // 49 DBs with 1 objects + require.Equal(t, uint64(49), metaStub.updatedCount) + + for addr, storageID := range storageIDs { + if _, found := toDelete[addr]; found { + continue + } + var gPrm common.GetPrm + gPrm.Address = addr + gPrm.StorageID = storageID + _, err := b.Get(context.Background(), gPrm) + require.NoError(t, err) + } + + require.NoError(t, b.Close()) + }) +} + func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { t.Parallel() @@ -92,6 +283,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { var rPrm common.RebuildPrm rPrm.MetaStorage = metaStub rPrm.WorkerLimiter = &rebuildLimiterStub{} + rPrm.Action = common.RebuildAction{SchemaChange: true} rRes, err := b.Rebuild(context.Background(), rPrm) require.NoError(t, err) dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 @@ -180,6 +372,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta var rPrm common.RebuildPrm rPrm.MetaStorage = metaStub rPrm.WorkerLimiter = &rebuildLimiterStub{} + rPrm.Action = common.RebuildAction{SchemaChange: true} rRes, err := b.Rebuild(context.Background(), rPrm) require.NoError(t, err) dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 diff --git a/pkg/local_object_storage/blobstor/common/rebuild.go b/pkg/local_object_storage/blobstor/common/rebuild.go index 9f629ef8c..020d9d022 100644 --- a/pkg/local_object_storage/blobstor/common/rebuild.go +++ b/pkg/local_object_storage/blobstor/common/rebuild.go @@ -11,9 +11,17 @@ type RebuildRes struct { FilesRemoved uint64 } +type RebuildAction struct { + SchemaChange bool + + FillPercent bool + FillPercentValue int +} + type RebuildPrm struct { MetaStorage MetaStorage WorkerLimiter ConcurrentWorkersLimiter + Action RebuildAction } type MetaStorage interface { diff --git a/pkg/local_object_storage/blobstor/rebuild.go b/pkg/local_object_storage/blobstor/rebuild.go index 101c60752..31bc2d167 100644 --- a/pkg/local_object_storage/blobstor/rebuild.go +++ b/pkg/local_object_storage/blobstor/rebuild.go @@ -18,13 +18,14 @@ type ConcurrentWorkersLimiter interface { ReleaseWorkSlot() } -func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate, limiter ConcurrentWorkersLimiter) error { +func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate, limiter ConcurrentWorkersLimiter, action common.RebuildAction) error { var summary common.RebuildRes var rErr error for _, storage := range b.storage { res, err := storage.Storage.Rebuild(ctx, common.RebuildPrm{ MetaStorage: upd, WorkerLimiter: limiter, + Action: action, }) summary.FilesRemoved += res.FilesRemoved summary.ObjectsMoved += res.ObjectsMoved diff --git a/pkg/local_object_storage/shard/rebuild.go b/pkg/local_object_storage/shard/rebuild.go new file mode 100644 index 000000000..998fcf08b --- /dev/null +++ b/pkg/local_object_storage/shard/rebuild.go @@ -0,0 +1,173 @@ +package shard + +import ( + "context" + "errors" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.uber.org/zap" +) + +type RebuildWorkerLimiter interface { + AcquireWorkSlot(ctx context.Context) error + ReleaseWorkSlot() +} + +type rebuildLimiter struct { + semaphore chan struct{} +} + +func newRebuildLimiter(workersCount uint32) *rebuildLimiter { + return &rebuildLimiter{ + semaphore: make(chan struct{}, workersCount), + } +} + +func (l *rebuildLimiter) AcquireWorkSlot(ctx context.Context) error { + select { + case l.semaphore <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (l *rebuildLimiter) ReleaseWorkSlot() { + <-l.semaphore +} + +type rebuildTask struct { + limiter RebuildWorkerLimiter + action common.RebuildAction +} + +type rebuilder struct { + mtx *sync.Mutex + wg *sync.WaitGroup + cancel func() + limiter RebuildWorkerLimiter + done chan struct{} + tasks chan rebuildTask +} + +func newRebuilder(l RebuildWorkerLimiter) *rebuilder { + return &rebuilder{ + mtx: &sync.Mutex{}, + wg: &sync.WaitGroup{}, + limiter: l, + tasks: make(chan rebuildTask, 10), + } +} + +func (r *rebuilder) Start(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *logger.Logger) { + r.mtx.Lock() + defer r.mtx.Unlock() + + if r.done != nil { + return // already started + } + ctx, cancel := context.WithCancel(ctx) + r.cancel = cancel + r.done = make(chan struct{}) + r.wg.Add(1) + go func() { + defer r.wg.Done() + for { + select { + case <-r.done: + return + case t, ok := <-r.tasks: + if !ok { + continue + } + runRebuild(ctx, bs, mb, log, t.action, t.limiter) + } + } + }() + select { + case <-ctx.Done(): + return + case r.tasks <- rebuildTask{ + limiter: r.limiter, + action: common.RebuildAction{ + SchemaChange: true, + }, + }: + return + } +} + +func runRebuild(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *logger.Logger, + action common.RebuildAction, limiter RebuildWorkerLimiter, +) { + select { + case <-ctx.Done(): + return + default: + } + log.Info(logs.BlobstoreRebuildStarted) + if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, limiter, action); err != nil { + log.Warn(logs.FailedToRebuildBlobstore, zap.Error(err)) + } else { + log.Info(logs.BlobstoreRebuildCompletedSuccessfully) + } +} + +func (r *rebuilder) ScheduleRebuild(ctx context.Context, limiter RebuildWorkerLimiter, action common.RebuildAction, +) error { + select { + case <-ctx.Done(): + return ctx.Err() + case r.tasks <- rebuildTask{ + limiter: limiter, + action: action, + }: + return nil + } +} + +func (r *rebuilder) Stop(log *logger.Logger) { + r.mtx.Lock() + defer r.mtx.Unlock() + + if r.done != nil { + close(r.done) + } + if r.cancel != nil { + r.cancel() + } + r.wg.Wait() + r.cancel = nil + r.done = nil + log.Info(logs.BlobstoreRebuildStopped) +} + +var errMBIsNotAvailable = errors.New("metabase is not available") + +type mbStorageIDUpdate struct { + mb *meta.DB +} + +func (u *mbStorageIDUpdate) UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if u.mb == nil { + return errMBIsNotAvailable + } + + var prm meta.UpdateStorageIDPrm + prm.SetAddress(addr) + prm.SetStorageID(storageID) + _, err := u.mb.UpdateStorageID(ctx, prm) + return err +} diff --git a/pkg/local_object_storage/shard/rebuild_limiter.go b/pkg/local_object_storage/shard/rebuild_limiter.go deleted file mode 100644 index efc21837c..000000000 --- a/pkg/local_object_storage/shard/rebuild_limiter.go +++ /dev/null @@ -1,13 +0,0 @@ -package shard - -import "context" - -type RebuildWorkerLimiter interface { - AcquireWorkSlot(ctx context.Context) error - ReleaseWorkSlot() -} - -type noopRebuildLimiter struct{} - -func (l *noopRebuildLimiter) AcquireWorkSlot(context.Context) error { return nil } -func (l *noopRebuildLimiter) ReleaseWorkSlot() {} diff --git a/pkg/local_object_storage/shard/rebuilder.go b/pkg/local_object_storage/shard/rebuilder.go deleted file mode 100644 index f18573c57..000000000 --- a/pkg/local_object_storage/shard/rebuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -package shard - -import ( - "context" - "errors" - "sync" - - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.uber.org/zap" -) - -type rebuilder struct { - mtx *sync.Mutex - wg *sync.WaitGroup - cancel func() - limiter RebuildWorkerLimiter -} - -func newRebuilder(l RebuildWorkerLimiter) *rebuilder { - return &rebuilder{ - mtx: &sync.Mutex{}, - wg: &sync.WaitGroup{}, - cancel: nil, - limiter: l, - } -} - -func (r *rebuilder) Start(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *logger.Logger) { - r.mtx.Lock() - defer r.mtx.Unlock() - - r.start(ctx, bs, mb, log) -} - -func (r *rebuilder) start(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *logger.Logger) { - if r.cancel != nil { - r.stop(log) - } - ctx, cancel := context.WithCancel(ctx) - r.cancel = cancel - r.wg.Add(1) - go func() { - defer r.wg.Done() - - log.Info(logs.BlobstoreRebuildStarted) - if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, r.limiter); err != nil { - log.Warn(logs.FailedToRebuildBlobstore, zap.Error(err)) - } else { - log.Info(logs.BlobstoreRebuildCompletedSuccessfully) - } - }() -} - -func (r *rebuilder) Stop(log *logger.Logger) { - r.mtx.Lock() - defer r.mtx.Unlock() - - r.stop(log) -} - -func (r *rebuilder) stop(log *logger.Logger) { - if r.cancel == nil { - return - } - - r.cancel() - r.wg.Wait() - r.cancel = nil - log.Info(logs.BlobstoreRebuildStopped) -} - -var errMBIsNotAvailable = errors.New("metabase is not available") - -type mbStorageIDUpdate struct { - mb *meta.DB -} - -func (u *mbStorageIDUpdate) UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - if u.mb == nil { - return errMBIsNotAvailable - } - - var prm meta.UpdateStorageIDPrm - prm.SetAddress(addr) - prm.SetStorageID(storageID) - _, err := u.mb.UpdateStorageID(ctx, prm) - return err -} diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index ac389b506..1eaee8815 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -151,7 +151,7 @@ func defaultCfg() *cfg { log: &logger.Logger{Logger: zap.L()}, gcCfg: defaultGCCfg(), reportErrorFunc: func(string, string, error) {}, - rebuildLimiter: &noopRebuildLimiter{}, + rebuildLimiter: newRebuildLimiter(1), zeroSizeContainersCallback: func(context.Context, []cid.ID) {}, zeroCountContainersCallback: func(context.Context, []cid.ID) {}, } From 6b6eabe41cd5750257adc3041f6a1c28df8197c7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 29 Aug 2024 12:11:19 +0300 Subject: [PATCH 1325/1943] [#1337] cli: Add `control shards rebuild` command Signed-off-by: Dmitrii Stepanov --- .../modules/control/rebuild_shards.go | 88 ++ cmd/frostfs-cli/modules/control/shards.go | 1 + pkg/local_object_storage/engine/rebuild.go | 90 ++ pkg/local_object_storage/shard/rebuild.go | 33 + pkg/services/control/rpc.go | 14 + pkg/services/control/server/rebuild.go | 59 ++ pkg/services/control/service.proto | 29 + pkg/services/control/service_frostfs.pb.go | 918 ++++++++++++++++++ pkg/services/control/service_grpc.pb.go | 39 + 9 files changed, 1271 insertions(+) create mode 100644 cmd/frostfs-cli/modules/control/rebuild_shards.go create mode 100644 pkg/local_object_storage/engine/rebuild.go create mode 100644 pkg/services/control/server/rebuild.go diff --git a/cmd/frostfs-cli/modules/control/rebuild_shards.go b/cmd/frostfs-cli/modules/control/rebuild_shards.go new file mode 100644 index 000000000..e2b408712 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/rebuild_shards.go @@ -0,0 +1,88 @@ +package control + +import ( + "fmt" + + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "github.com/mr-tron/base58" + "github.com/spf13/cobra" +) + +const ( + fillPercentFlag = "fill_percent" +) + +var shardsRebuildCmd = &cobra.Command{ + Use: "rebuild", + Short: "Rebuild shards", + Long: "Rebuild reclaims storage occupied by dead objects and adjusts the storage structure according to the configuration (for blobovnicza only now)", + Run: shardsRebuild, +} + +func shardsRebuild(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + + req := &control.StartShardRebuildRequest{ + Body: &control.StartShardRebuildRequest_Body{ + Shard_ID: getShardIDList(cmd), + TargetFillPercent: getFillPercentValue(cmd), + ConcurrencyLimit: getConcurrencyValue(cmd), + }, + } + + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var resp *control.StartShardRebuildResponse + var err error + err = cli.ExecRaw(func(client *rawclient.Client) error { + resp, err = control.StartShardRebuild(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + var success, failed uint + for _, res := range resp.GetBody().GetResults() { + if res.GetSuccess() { + success++ + cmd.Printf("Shard %s: OK\n", base58.Encode(res.GetShard_ID())) + } else { + failed++ + cmd.Printf("Shard %s: failed with error %q\n", base58.Encode(res.GetShard_ID()), res.GetError()) + } + } + cmd.Printf("Total: %d success, %d failed\n", success, failed) +} + +func getFillPercentValue(cmd *cobra.Command) uint32 { + v, _ := cmd.Flags().GetUint32(fillPercentFlag) + if v <= 0 || v > 100 { + commonCmd.ExitOnErr(cmd, "invalid fill_percent value", fmt.Errorf("fill_percent value must be (0, 100], current value: %d", v)) + } + return v +} + +func getConcurrencyValue(cmd *cobra.Command) uint32 { + v, _ := cmd.Flags().GetUint32(concurrencyFlag) + if v <= 0 || v > 10000 { + commonCmd.ExitOnErr(cmd, "invalid concurrency value", fmt.Errorf("concurrency value must be (0, 10 000], current value: %d", v)) + } + return v +} + +func initControlShardRebuildCmd() { + initControlFlags(shardsRebuildCmd) + + flags := shardsRebuildCmd.Flags() + flags.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") + flags.Bool(shardAllFlag, false, "Process all shards") + flags.Uint32(fillPercentFlag, 80, "Target fill percent to reclaim space") + flags.Uint32(concurrencyFlag, 20, "Maximum count of concurrently rebuilding files") + setShardModeCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) +} diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index d8198c426..d6c2a0b9b 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -28,4 +28,5 @@ func initControlShardsCmd() { initControlDoctorCmd() initControlShardsWritecacheCmd() initControlShardsDetachCmd() + initControlShardRebuildCmd() } diff --git a/pkg/local_object_storage/engine/rebuild.go b/pkg/local_object_storage/engine/rebuild.go new file mode 100644 index 000000000..3970aae89 --- /dev/null +++ b/pkg/local_object_storage/engine/rebuild.go @@ -0,0 +1,90 @@ +package engine + +import ( + "context" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/errgroup" +) + +type RebuildPrm struct { + ShardIDs []*shard.ID + ConcurrencyLimit uint32 + TargetFillPercent uint32 +} + +type ShardRebuildResult struct { + ShardID *shard.ID + Success bool + ErrorMsg string +} + +type RebuildRes struct { + ShardResults []ShardRebuildResult +} + +func (e *StorageEngine) Rebuild(ctx context.Context, prm RebuildPrm) (RebuildRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Rebuild", + trace.WithAttributes( + attribute.Int("shard_id_count", len(prm.ShardIDs)), + attribute.Int64("target_fill_percent", int64(prm.TargetFillPercent)), + attribute.Int64("concurrency_limit", int64(prm.ConcurrencyLimit)), + )) + defer span.End() + + res := RebuildRes{ + ShardResults: make([]ShardRebuildResult, 0, len(prm.ShardIDs)), + } + resGuard := &sync.Mutex{} + + limiter := newRebuildLimiter(prm.ConcurrencyLimit) + + eg, egCtx := errgroup.WithContext(ctx) + for _, shardID := range prm.ShardIDs { + eg.Go(func() error { + e.mtx.RLock() + sh, ok := e.shards[shardID.String()] + e.mtx.RUnlock() + + if !ok { + resGuard.Lock() + defer resGuard.Unlock() + res.ShardResults = append(res.ShardResults, ShardRebuildResult{ + ShardID: shardID, + ErrorMsg: errShardNotFound.Error(), + }) + return nil + } + + err := sh.ScheduleRebuild(egCtx, shard.RebuildPrm{ + ConcurrencyLimiter: limiter, + TargetFillPercent: prm.TargetFillPercent, + }) + + resGuard.Lock() + defer resGuard.Unlock() + + if err != nil { + res.ShardResults = append(res.ShardResults, ShardRebuildResult{ + ShardID: shardID, + ErrorMsg: err.Error(), + }) + } else { + res.ShardResults = append(res.ShardResults, ShardRebuildResult{ + ShardID: shardID, + Success: true, + }) + } + return nil + }) + } + + if err := eg.Wait(); err != nil { + return RebuildRes{}, err + } + return res, nil +} diff --git a/pkg/local_object_storage/shard/rebuild.go b/pkg/local_object_storage/shard/rebuild.go index 998fcf08b..f8051999e 100644 --- a/pkg/local_object_storage/shard/rebuild.go +++ b/pkg/local_object_storage/shard/rebuild.go @@ -10,7 +10,10 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -171,3 +174,33 @@ func (u *mbStorageIDUpdate) UpdateStorageID(ctx context.Context, addr oid.Addres _, err := u.mb.UpdateStorageID(ctx, prm) return err } + +type RebuildPrm struct { + ConcurrencyLimiter RebuildWorkerLimiter + TargetFillPercent uint32 +} + +func (s *Shard) ScheduleRebuild(ctx context.Context, p RebuildPrm) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.ScheduleRebuild", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.Int64("target_fill_percent", int64(p.TargetFillPercent)), + )) + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.ReadOnly() { + return ErrReadOnlyMode + } + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } + + return s.rb.ScheduleRebuild(ctx, p.ConcurrencyLimiter, common.RebuildAction{ + SchemaChange: true, + FillPercent: true, + FillPercentValue: int(p.TargetFillPercent), + }) +} diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index a90e58a65..80aece008 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -30,6 +30,7 @@ const ( rpcSealWriteCache = "SealWriteCache" rpcListTargetsLocalOverrides = "ListTargetsLocalOverrides" rpcDetachShards = "DetachShards" + rpcStartShardRebuild = "StartShardRebuild" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -361,3 +362,16 @@ func DetachShards( return wResp.message, nil } + +// StartShardRebuild executes ControlService.StartShardRebuild RPC. +func StartShardRebuild(cli *client.Client, req *StartShardRebuildRequest, opts ...client.CallOption) (*StartShardRebuildResponse, error) { + wResp := newResponseWrapper[StartShardRebuildResponse]() + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcStartShardRebuild), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} diff --git a/pkg/services/control/server/rebuild.go b/pkg/services/control/server/rebuild.go new file mode 100644 index 000000000..6ddfb8bf4 --- /dev/null +++ b/pkg/services/control/server/rebuild.go @@ -0,0 +1,59 @@ +package control + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *Server) StartShardRebuild(ctx context.Context, req *control.StartShardRebuildRequest) (*control.StartShardRebuildResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + if req.GetBody().GetConcurrencyLimit() == 0 || req.GetBody().GetConcurrencyLimit() > 10000 { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("concurrency limit must be in range (0; 10 000], current value %d", req.GetBody().GetConcurrencyLimit())) + } + + if req.GetBody().GetTargetFillPercent() == 0 || req.GetBody().GetTargetFillPercent() > 100 { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("fill percent must be in range (0; 100], current value %d", req.GetBody().GetTargetFillPercent())) + } + + prm := engine.RebuildPrm{ + ShardIDs: s.getShardIDList(req.GetBody().GetShard_ID()), + ConcurrencyLimit: req.GetBody().GetConcurrencyLimit(), + TargetFillPercent: req.GetBody().GetTargetFillPercent(), + } + + res, err := s.s.Rebuild(ctx, prm) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &control.StartShardRebuildResponse{Body: &control.StartShardRebuildResponse_Body{}} + for _, r := range res.ShardResults { + if r.Success { + resp.Body.Results = append(resp.GetBody().GetResults(), control.StartShardRebuildResponse_Body_Status{ + Shard_ID: *r.ShardID, + Success: true, + }) + } else { + resp.Body.Results = append(resp.GetBody().GetResults(), control.StartShardRebuildResponse_Body_Status{ + Shard_ID: *r.ShardID, + Error: r.ErrorMsg, + }) + } + } + + err = ctrlmessage.Sign(s.key, resp) + if err != nil { + return nil, err + } + return resp, nil +} diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index d6639cb48..04994328a 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -91,6 +91,9 @@ service ControlService { // DetachShards detaches and closes shards. rpc DetachShards(DetachShardsRequest) returns (DetachShardsResponse); + + // StartShardRebuild starts shard rebuild process. + rpc StartShardRebuild(StartShardRebuildRequest) returns (StartShardRebuildResponse); } // Health check request. @@ -699,3 +702,29 @@ message DetachShardsResponse { Signature signature = 2; } + +message StartShardRebuildRequest { + message Body { + repeated bytes shard_ID = 1; + uint32 target_fill_percent = 2; + uint32 concurrency_limit = 3; + } + + Body body = 1; + Signature signature = 2; +} + +message StartShardRebuildResponse { + message Body { + message Status { + bytes shard_ID = 1; + bool success = 2; + string error = 3; + } + repeated Status results = 1; + } + + Body body = 1; + + Signature signature = 2; +} diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index eb0d95c64..019cac290 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -15023,3 +15023,921 @@ func (x *DetachShardsResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { in.Consumed() } } + +type StartShardRebuildRequest_Body struct { + Shard_ID [][]byte `json:"shardID"` + TargetFillPercent uint32 `json:"targetFillPercent"` + ConcurrencyLimit uint32 `json:"concurrencyLimit"` +} + +var ( + _ encoding.ProtoMarshaler = (*StartShardRebuildRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*StartShardRebuildRequest_Body)(nil) + _ json.Marshaler = (*StartShardRebuildRequest_Body)(nil) + _ json.Unmarshaler = (*StartShardRebuildRequest_Body)(nil) +) + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StartShardRebuildRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.RepeatedBytesSize(1, x.Shard_ID) + size += proto.UInt32Size(2, x.TargetFillPercent) + size += proto.UInt32Size(3, x.ConcurrencyLimit) + return size +} + +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StartShardRebuildRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *StartShardRebuildRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for j := range x.Shard_ID { + mm.AppendBytes(1, x.Shard_ID[j]) + } + if x.TargetFillPercent != 0 { + mm.AppendUint32(2, x.TargetFillPercent) + } + if x.ConcurrencyLimit != 0 { + mm.AppendUint32(3, x.ConcurrencyLimit) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StartShardRebuildRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StartShardRebuildRequest_Body") + } + switch fc.FieldNum { + case 1: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = append(x.Shard_ID, data) + case 2: // TargetFillPercent + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "TargetFillPercent") + } + x.TargetFillPercent = data + case 3: // ConcurrencyLimit + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ConcurrencyLimit") + } + x.ConcurrencyLimit = data + } + } + return nil +} +func (x *StartShardRebuildRequest_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *StartShardRebuildRequest_Body) SetShard_ID(v [][]byte) { + x.Shard_ID = v +} +func (x *StartShardRebuildRequest_Body) GetTargetFillPercent() uint32 { + if x != nil { + return x.TargetFillPercent + } + return 0 +} +func (x *StartShardRebuildRequest_Body) SetTargetFillPercent(v uint32) { + x.TargetFillPercent = v +} +func (x *StartShardRebuildRequest_Body) GetConcurrencyLimit() uint32 { + if x != nil { + return x.ConcurrencyLimit + } + return 0 +} +func (x *StartShardRebuildRequest_Body) SetConcurrencyLimit(v uint32) { + x.ConcurrencyLimit = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StartShardRebuildRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StartShardRebuildRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"shardID\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Shard_ID { + if i != 0 { + out.RawByte(',') + } + out.Base64Bytes(x.Shard_ID[i]) + } + out.RawByte(']') + } + { + const prefix string = ",\"targetFillPercent\":" + out.RawString(prefix) + out.Uint32(x.TargetFillPercent) + } + { + const prefix string = ",\"concurrencyLimit\":" + out.RawString(prefix) + out.Uint32(x.ConcurrencyLimit) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StartShardRebuildRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StartShardRebuildRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shardID": + { + var f []byte + var list [][]byte + in.Delim('[') + for !in.IsDelim(']') { + f = in.Bytes() + list = append(list, f) + in.WantComma() + } + x.Shard_ID = list + in.Delim(']') + } + case "targetFillPercent": + { + var f uint32 + f = in.Uint32() + x.TargetFillPercent = f + } + case "concurrencyLimit": + { + var f uint32 + f = in.Uint32() + x.ConcurrencyLimit = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StartShardRebuildRequest struct { + Body *StartShardRebuildRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*StartShardRebuildRequest)(nil) + _ encoding.ProtoUnmarshaler = (*StartShardRebuildRequest)(nil) + _ json.Marshaler = (*StartShardRebuildRequest)(nil) + _ json.Unmarshaler = (*StartShardRebuildRequest)(nil) +) + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StartShardRebuildRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *StartShardRebuildRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *StartShardRebuildRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().MarshalProtobuf(buf), nil +} + +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StartShardRebuildRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *StartShardRebuildRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StartShardRebuildRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StartShardRebuildRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(StartShardRebuildRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *StartShardRebuildRequest) GetBody() *StartShardRebuildRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *StartShardRebuildRequest) SetBody(v *StartShardRebuildRequest_Body) { + x.Body = v +} +func (x *StartShardRebuildRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *StartShardRebuildRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StartShardRebuildRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StartShardRebuildRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StartShardRebuildRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StartShardRebuildRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *StartShardRebuildRequest_Body + f = new(StartShardRebuildRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StartShardRebuildResponse_Body_Status struct { + Shard_ID []byte `json:"shardID"` + Success bool `json:"success"` + Error string `json:"error"` +} + +var ( + _ encoding.ProtoMarshaler = (*StartShardRebuildResponse_Body_Status)(nil) + _ encoding.ProtoUnmarshaler = (*StartShardRebuildResponse_Body_Status)(nil) + _ json.Marshaler = (*StartShardRebuildResponse_Body_Status)(nil) + _ json.Unmarshaler = (*StartShardRebuildResponse_Body_Status)(nil) +) + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StartShardRebuildResponse_Body_Status) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.BytesSize(1, x.Shard_ID) + size += proto.BoolSize(2, x.Success) + size += proto.StringSize(3, x.Error) + return size +} + +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StartShardRebuildResponse_Body_Status) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *StartShardRebuildResponse_Body_Status) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.Shard_ID) != 0 { + mm.AppendBytes(1, x.Shard_ID) + } + if x.Success { + mm.AppendBool(2, x.Success) + } + if len(x.Error) != 0 { + mm.AppendString(3, x.Error) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StartShardRebuildResponse_Body_Status) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StartShardRebuildResponse_Body_Status") + } + switch fc.FieldNum { + case 1: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = data + case 2: // Success + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Success") + } + x.Success = data + case 3: // Error + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Error") + } + x.Error = data + } + } + return nil +} +func (x *StartShardRebuildResponse_Body_Status) GetShard_ID() []byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *StartShardRebuildResponse_Body_Status) SetShard_ID(v []byte) { + x.Shard_ID = v +} +func (x *StartShardRebuildResponse_Body_Status) GetSuccess() bool { + if x != nil { + return x.Success + } + return false +} +func (x *StartShardRebuildResponse_Body_Status) SetSuccess(v bool) { + x.Success = v +} +func (x *StartShardRebuildResponse_Body_Status) GetError() string { + if x != nil { + return x.Error + } + return "" +} +func (x *StartShardRebuildResponse_Body_Status) SetError(v string) { + x.Error = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StartShardRebuildResponse_Body_Status) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StartShardRebuildResponse_Body_Status) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"shardID\":" + out.RawString(prefix[1:]) + out.Base64Bytes(x.Shard_ID) + } + { + const prefix string = ",\"success\":" + out.RawString(prefix) + out.Bool(x.Success) + } + { + const prefix string = ",\"error\":" + out.RawString(prefix) + out.String(x.Error) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StartShardRebuildResponse_Body_Status) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StartShardRebuildResponse_Body_Status) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shardID": + { + var f []byte + f = in.Bytes() + x.Shard_ID = f + } + case "success": + { + var f bool + f = in.Bool() + x.Success = f + } + case "error": + { + var f string + f = in.String() + x.Error = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StartShardRebuildResponse_Body struct { + Results []StartShardRebuildResponse_Body_Status `json:"results"` +} + +var ( + _ encoding.ProtoMarshaler = (*StartShardRebuildResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*StartShardRebuildResponse_Body)(nil) + _ json.Marshaler = (*StartShardRebuildResponse_Body)(nil) + _ json.Unmarshaler = (*StartShardRebuildResponse_Body)(nil) +) + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StartShardRebuildResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + for i := range x.Results { + size += proto.NestedStructureSizeUnchecked(1, &x.Results[i]) + } + return size +} + +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StartShardRebuildResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *StartShardRebuildResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for i := range x.Results { + x.Results[i].EmitProtobuf(mm.AppendMessage(1)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StartShardRebuildResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StartShardRebuildResponse_Body") + } + switch fc.FieldNum { + case 1: // Results + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Results") + } + x.Results = append(x.Results, StartShardRebuildResponse_Body_Status{}) + ff := &x.Results[len(x.Results)-1] + if err := ff.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *StartShardRebuildResponse_Body) GetResults() []StartShardRebuildResponse_Body_Status { + if x != nil { + return x.Results + } + return nil +} +func (x *StartShardRebuildResponse_Body) SetResults(v []StartShardRebuildResponse_Body_Status) { + x.Results = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StartShardRebuildResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StartShardRebuildResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"results\":" + out.RawString(prefix[1:]) + out.RawByte('[') + for i := range x.Results { + if i != 0 { + out.RawByte(',') + } + x.Results[i].MarshalEasyJSON(out) + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StartShardRebuildResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StartShardRebuildResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "results": + { + var f StartShardRebuildResponse_Body_Status + var list []StartShardRebuildResponse_Body_Status + in.Delim('[') + for !in.IsDelim(']') { + f = StartShardRebuildResponse_Body_Status{} + f.UnmarshalEasyJSON(in) + list = append(list, f) + in.WantComma() + } + x.Results = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type StartShardRebuildResponse struct { + Body *StartShardRebuildResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*StartShardRebuildResponse)(nil) + _ encoding.ProtoUnmarshaler = (*StartShardRebuildResponse)(nil) + _ json.Marshaler = (*StartShardRebuildResponse)(nil) + _ json.Unmarshaler = (*StartShardRebuildResponse)(nil) +) + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *StartShardRebuildResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *StartShardRebuildResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *StartShardRebuildResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().MarshalProtobuf(buf), nil +} + +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *StartShardRebuildResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *StartShardRebuildResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *StartShardRebuildResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "StartShardRebuildResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(StartShardRebuildResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *StartShardRebuildResponse) GetBody() *StartShardRebuildResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *StartShardRebuildResponse) SetBody(v *StartShardRebuildResponse_Body) { + x.Body = v +} +func (x *StartShardRebuildResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *StartShardRebuildResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *StartShardRebuildResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *StartShardRebuildResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + out.RawByte('{') + { + const prefix string = ",\"body\":" + out.RawString(prefix[1:]) + x.Body.MarshalEasyJSON(out) + } + { + const prefix string = ",\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *StartShardRebuildResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *StartShardRebuildResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *StartShardRebuildResponse_Body + f = new(StartShardRebuildResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index fa9de974a..f5cfefa85 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -41,6 +41,7 @@ const ( ControlService_ListTargetsLocalOverrides_FullMethodName = "/control.ControlService/ListTargetsLocalOverrides" ControlService_SealWriteCache_FullMethodName = "/control.ControlService/SealWriteCache" ControlService_DetachShards_FullMethodName = "/control.ControlService/DetachShards" + ControlService_StartShardRebuild_FullMethodName = "/control.ControlService/StartShardRebuild" ) // ControlServiceClient is the client API for ControlService service. @@ -97,6 +98,8 @@ type ControlServiceClient interface { SealWriteCache(ctx context.Context, in *SealWriteCacheRequest, opts ...grpc.CallOption) (*SealWriteCacheResponse, error) // DetachShards detaches and closes shards. DetachShards(ctx context.Context, in *DetachShardsRequest, opts ...grpc.CallOption) (*DetachShardsResponse, error) + // StartShardRebuild starts shard rebuild process. + StartShardRebuild(ctx context.Context, in *StartShardRebuildRequest, opts ...grpc.CallOption) (*StartShardRebuildResponse, error) } type controlServiceClient struct { @@ -305,6 +308,15 @@ func (c *controlServiceClient) DetachShards(ctx context.Context, in *DetachShard return out, nil } +func (c *controlServiceClient) StartShardRebuild(ctx context.Context, in *StartShardRebuildRequest, opts ...grpc.CallOption) (*StartShardRebuildResponse, error) { + out := new(StartShardRebuildResponse) + err := c.cc.Invoke(ctx, ControlService_StartShardRebuild_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // ControlServiceServer is the server API for ControlService service. // All implementations should embed UnimplementedControlServiceServer // for forward compatibility @@ -359,6 +371,8 @@ type ControlServiceServer interface { SealWriteCache(context.Context, *SealWriteCacheRequest) (*SealWriteCacheResponse, error) // DetachShards detaches and closes shards. DetachShards(context.Context, *DetachShardsRequest) (*DetachShardsResponse, error) + // StartShardRebuild starts shard rebuild process. + StartShardRebuild(context.Context, *StartShardRebuildRequest) (*StartShardRebuildResponse, error) } // UnimplementedControlServiceServer should be embedded to have forward compatible implementations. @@ -431,6 +445,9 @@ func (UnimplementedControlServiceServer) SealWriteCache(context.Context, *SealWr func (UnimplementedControlServiceServer) DetachShards(context.Context, *DetachShardsRequest) (*DetachShardsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method DetachShards not implemented") } +func (UnimplementedControlServiceServer) StartShardRebuild(context.Context, *StartShardRebuildRequest) (*StartShardRebuildResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method StartShardRebuild not implemented") +} // UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ControlServiceServer will @@ -839,6 +856,24 @@ func _ControlService_DetachShards_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _ControlService_StartShardRebuild_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StartShardRebuildRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).StartShardRebuild(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_StartShardRebuild_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).StartShardRebuild(ctx, req.(*StartShardRebuildRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -934,6 +969,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "DetachShards", Handler: _ControlService_DetachShards_Handler, }, + { + MethodName: "StartShardRebuild", + Handler: _ControlService_StartShardRebuild_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "pkg/services/control/service.proto", From a61201a98753c4522a1169abe5b42f2631f639ad Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 29 Aug 2024 13:51:09 +0300 Subject: [PATCH 1326/1943] [#1337] config: Move `rebuild_worker_count` to shard section This makes it simple to limit performance degradation for every shard because of rebuild. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/control/shards.go | 1 + cmd/frostfs-node/config.go | 7 ++--- cmd/frostfs-node/config/engine/config.go | 11 -------- cmd/frostfs-node/config/engine/config_test.go | 4 +-- .../config/engine/shard/config.go | 15 +++++++++++ config/example/node.env | 2 +- config/example/node.json | 2 +- config/example/node.yaml | 2 +- docs/storage-node-configuration.md | 1 + pkg/local_object_storage/engine/engine.go | 16 ++---------- pkg/local_object_storage/engine/rebuild.go | 2 +- .../engine/rebuild_limiter.go | 26 ------------------- pkg/local_object_storage/engine/shards.go | 1 - pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/shard/rebuild.go | 2 +- pkg/local_object_storage/shard/shard.go | 10 +++---- 16 files changed, 36 insertions(+), 68 deletions(-) delete mode 100644 pkg/local_object_storage/engine/rebuild_limiter.go diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index d6c2a0b9b..329cb9100 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -19,6 +19,7 @@ func initControlShardsCmd() { shardsCmd.AddCommand(doctorCmd) shardsCmd.AddCommand(writecacheShardCmd) shardsCmd.AddCommand(shardsDetachCmd) + shardsCmd.AddCommand(shardsRebuildCmd) initControlShardsListCmd() initControlSetShardModeCmd() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 110281418..b59518d14 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -109,7 +109,6 @@ type applicationConfiguration struct { shardPoolSize uint32 shards []shardCfg lowMem bool - rebuildWorkers uint32 } // if need to run node in compatibility with other versions mode @@ -127,6 +126,8 @@ type shardCfg struct { refillMetabaseWorkersCount int mode shardmode.Mode + rebuildWorkersCount uint32 + metaCfg struct { path string perm fs.FileMode @@ -230,7 +231,6 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { a.EngineCfg.errorThreshold = engineconfig.ShardErrorThreshold(c) a.EngineCfg.shardPoolSize = engineconfig.ShardPoolSize(c) a.EngineCfg.lowMem = engineconfig.EngineLowMemoryConsumption(c) - a.EngineCfg.rebuildWorkers = engineconfig.EngineRebuildWorkersCount(c) return engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error { return a.updateShardConfig(c, sc) }) } @@ -240,6 +240,7 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, oldConfig newConfig.refillMetabase = oldConfig.RefillMetabase() newConfig.refillMetabaseWorkersCount = oldConfig.RefillMetabaseWorkersCount() + newConfig.rebuildWorkersCount = oldConfig.RebuildWorkerCount() newConfig.mode = oldConfig.Mode() newConfig.compress = oldConfig.Compress() newConfig.estimateCompressibility = oldConfig.EstimateCompressibility() @@ -835,7 +836,6 @@ func (c *cfg) engineOpts() []engine.Option { engine.WithErrorThreshold(c.EngineCfg.errorThreshold), engine.WithLogger(c.log), engine.WithLowMemoryConsumption(c.EngineCfg.lowMem), - engine.WithRebuildWorkersCount(c.EngineCfg.rebuildWorkers), ) if c.metricsCollector != nil { @@ -998,6 +998,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID shard.WithLogger(c.log), shard.WithRefillMetabase(shCfg.refillMetabase), shard.WithRefillMetabaseWorkersCount(shCfg.refillMetabaseWorkersCount), + shard.WithRebuildWorkersCount(shCfg.rebuildWorkersCount), shard.WithMode(shCfg.mode), shard.WithBlobStorOptions(blobstoreOpts...), shard.WithMetaBaseOptions(mbOptions...), diff --git a/cmd/frostfs-node/config/engine/config.go b/cmd/frostfs-node/config/engine/config.go index baa4e3c9d..c944d1c58 100644 --- a/cmd/frostfs-node/config/engine/config.go +++ b/cmd/frostfs-node/config/engine/config.go @@ -15,9 +15,6 @@ const ( // ShardPoolSizeDefault is a default value of routine pool size per-shard to // process object PUT operations in a storage engine. ShardPoolSizeDefault = 20 - // RebuildWorkersCountDefault is a default value of the workers count to - // process storage rebuild operations in a storage engine. - RebuildWorkersCountDefault = 100 ) // ErrNoShardConfigured is returned when at least 1 shard is required but none are found. @@ -91,11 +88,3 @@ func ShardErrorThreshold(c *config.Config) uint32 { func EngineLowMemoryConsumption(c *config.Config) bool { return config.BoolSafe(c.Sub(subsection), "low_mem") } - -// EngineRebuildWorkersCount returns value of "rebuild_workers_count" config parmeter from "storage" section. -func EngineRebuildWorkersCount(c *config.Config) uint32 { - if v := config.Uint32Safe(c.Sub(subsection), "rebuild_workers_count"); v > 0 { - return v - } - return RebuildWorkersCountDefault -} diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index d53207ccc..464d72556 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -39,7 +39,6 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 0, engineconfig.ShardErrorThreshold(empty)) require.EqualValues(t, engineconfig.ShardPoolSizeDefault, engineconfig.ShardPoolSize(empty)) require.EqualValues(t, mode.ReadWrite, shardconfig.From(empty).Mode()) - require.EqualValues(t, engineconfig.RebuildWorkersCountDefault, engineconfig.EngineRebuildWorkersCount(empty)) }) const path = "../../../../config/example/node" @@ -49,7 +48,6 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 100, engineconfig.ShardErrorThreshold(c)) require.EqualValues(t, 15, engineconfig.ShardPoolSize(c)) - require.EqualValues(t, uint32(1000), engineconfig.EngineRebuildWorkersCount(c)) err := engineconfig.IterateShards(c, true, func(sc *shardconfig.Config) error { defer func() { @@ -121,6 +119,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, false, sc.RefillMetabase()) require.Equal(t, mode.ReadOnly, sc.Mode()) require.Equal(t, 100, sc.RefillMetabaseWorkersCount()) + require.Equal(t, uint32(1000), sc.RebuildWorkerCount()) case 1: require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path()) require.Equal(t, fs.FileMode(0o644), pl.Perm()) @@ -176,6 +175,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, true, sc.RefillMetabase()) require.Equal(t, mode.ReadWrite, sc.Mode()) require.Equal(t, shardconfig.RefillMetabaseWorkersCountDefault, sc.RefillMetabaseWorkersCount()) + require.Equal(t, uint32(shardconfig.RebuildWorkersCountDefault), sc.RebuildWorkerCount()) } return nil }) diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go index 0620c9f63..ec9df0e89 100644 --- a/cmd/frostfs-node/config/engine/shard/config.go +++ b/cmd/frostfs-node/config/engine/shard/config.go @@ -19,6 +19,7 @@ const ( SmallSizeLimitDefault = 1 << 20 EstimateCompressibilityThresholdDefault = 0.1 RefillMetabaseWorkersCountDefault = 500 + RebuildWorkersCountDefault = 5 ) // From wraps config section into Config. @@ -149,6 +150,20 @@ func (x *Config) RefillMetabaseWorkersCount() int { return RefillMetabaseWorkersCountDefault } +// RebuildWorkersCount returns the value of "resync_metabase_worker_count" config parameter. +// +// Returns RebuildWorkersCountDefault if the value is not a positive number. +func (x *Config) RebuildWorkerCount() uint32 { + v := config.Uint32Safe( + (*config.Config)(x), + "rebuild_worker_count", + ) + if v > 0 { + return v + } + return RebuildWorkersCountDefault +} + // Mode return the value of "mode" config parameter. // // Panics if read the value is not one of predefined diff --git a/config/example/node.env b/config/example/node.env index b39423ffb..1eccd8a5d 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -90,11 +90,11 @@ FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10 # Storage engine section FROSTFS_STORAGE_SHARD_POOL_SIZE=15 FROSTFS_STORAGE_SHARD_RO_ERROR_THRESHOLD=100 -FROSTFS_STORAGE_REBUILD_WORKERS_COUNT=1000 ## 0 shard ### Flag to refill Metabase from BlobStor FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE=false FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE_WORKER_COUNT=100 +FROSTFS_STORAGE_SHARD_0_REBUILD_WORKER_COUNT=1000 ### Flag to set shard mode FROSTFS_STORAGE_SHARD_0_MODE=read-only ### Write cache config diff --git a/config/example/node.json b/config/example/node.json index fe2de0e01..be7ced77a 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -135,12 +135,12 @@ "storage": { "shard_pool_size": 15, "shard_ro_error_threshold": 100, - "rebuild_workers_count": 1000, "shard": { "0": { "mode": "read-only", "resync_metabase": false, "resync_metabase_worker_count": 100, + "rebuild_worker_count": 1000, "writecache": { "enabled": false, "no_sync": true, diff --git a/config/example/node.yaml b/config/example/node.yaml index cc339a427..4b9720655 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -117,7 +117,6 @@ storage: # note: shard configuration can be omitted for relay node (see `node.relay`) shard_pool_size: 15 # size of per-shard worker pools used for PUT operations shard_ro_error_threshold: 100 # amount of errors to occur before shard is made read-only (default: 0, ignore errors) - rebuild_workers_count: 1000 # count of rebuild storage concurrent workers shard: default: # section with the default shard parameters @@ -165,6 +164,7 @@ storage: # disabled (do not work with the shard, allows to not remove it from the config) resync_metabase: false # sync metabase with blobstor on start, expensive, leave false until complete understanding resync_metabase_worker_count: 100 + rebuild_worker_count: 1000 # count of rebuild storage concurrent workers writecache: enabled: false diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 5bf35cd65..f390d84a4 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -189,6 +189,7 @@ The following table describes configuration for each shard. | `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` | | `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. | | `resync_metabase_worker_count` | `int` | `1000` | Count of concurrent workers to resync metabase. | +| `rebuild_worker_count` | `int` | `5` | Count of concurrent workers to rebuild blobstore. | | `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. | | `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. | | `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index b87d77e6c..5e883a641 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -40,7 +40,6 @@ type StorageEngine struct { err error } evacuateLimiter *evacuationLimiter - rebuildLimiter *rebuildLimiter } type shardWrapper struct { @@ -213,16 +212,13 @@ type cfg struct { lowMem bool - rebuildWorkersCount uint32 - containerSource atomic.Pointer[containerSource] } func defaultCfg() *cfg { res := &cfg{ - log: &logger.Logger{Logger: zap.L()}, - shardPoolSize: 20, - rebuildWorkersCount: 100, + log: &logger.Logger{Logger: zap.L()}, + shardPoolSize: 20, } res.containerSource.Store(&containerSource{}) return res @@ -243,7 +239,6 @@ func New(opts ...Option) *StorageEngine { closeCh: make(chan struct{}), setModeCh: make(chan setModeRequest), evacuateLimiter: &evacuationLimiter{}, - rebuildLimiter: newRebuildLimiter(c.rebuildWorkersCount), } } @@ -282,13 +277,6 @@ func WithLowMemoryConsumption(lowMemCons bool) Option { } } -// WithRebuildWorkersCount returns an option to set the count of concurrent rebuild workers. -func WithRebuildWorkersCount(count uint32) Option { - return func(c *cfg) { - c.rebuildWorkersCount = count - } -} - // SetContainerSource sets container source. func (e *StorageEngine) SetContainerSource(cs container.Source) { e.containerSource.Store(&containerSource{cs: cs}) diff --git a/pkg/local_object_storage/engine/rebuild.go b/pkg/local_object_storage/engine/rebuild.go index 3970aae89..83c6a54ed 100644 --- a/pkg/local_object_storage/engine/rebuild.go +++ b/pkg/local_object_storage/engine/rebuild.go @@ -41,7 +41,7 @@ func (e *StorageEngine) Rebuild(ctx context.Context, prm RebuildPrm) (RebuildRes } resGuard := &sync.Mutex{} - limiter := newRebuildLimiter(prm.ConcurrencyLimit) + limiter := shard.NewRebuildLimiter(prm.ConcurrencyLimit) eg, egCtx := errgroup.WithContext(ctx) for _, shardID := range prm.ShardIDs { diff --git a/pkg/local_object_storage/engine/rebuild_limiter.go b/pkg/local_object_storage/engine/rebuild_limiter.go deleted file mode 100644 index 28b02b0a3..000000000 --- a/pkg/local_object_storage/engine/rebuild_limiter.go +++ /dev/null @@ -1,26 +0,0 @@ -package engine - -import "context" - -type rebuildLimiter struct { - semaphore chan struct{} -} - -func newRebuildLimiter(workersCount uint32) *rebuildLimiter { - return &rebuildLimiter{ - semaphore: make(chan struct{}, workersCount), - } -} - -func (l *rebuildLimiter) AcquireWorkSlot(ctx context.Context) error { - select { - case l.semaphore <- struct{}{}: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - -func (l *rebuildLimiter) ReleaseWorkSlot() { - <-l.semaphore -} diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 40584149e..2ad6859e4 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -137,7 +137,6 @@ func (e *StorageEngine) createShard(_ context.Context, opts []shard.Option) (*sh shard.WithExpiredLocksCallback(e.processExpiredLocks), shard.WithDeletedLockCallback(e.processDeletedLocks), shard.WithReportErrorFunc(e.reportShardErrorBackground), - shard.WithRebuildWorkerLimiter(e.rebuildLimiter), shard.WithZeroSizeCallback(e.processZeroSizeContainers), shard.WithZeroCountCallback(e.processZeroCountContainers), )...) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 6efe4ec37..5e9639a7b 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -129,7 +129,7 @@ func (s *Shard) Init(ctx context.Context) error { s.gc.init(ctx) - s.rb = newRebuilder(s.rebuildLimiter) + s.rb = newRebuilder(NewRebuildLimiter(s.rebuildWorkersCount)) if !m.NoMetabase() && !s.rebuildDisabled { s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) } diff --git a/pkg/local_object_storage/shard/rebuild.go b/pkg/local_object_storage/shard/rebuild.go index f8051999e..2eef456be 100644 --- a/pkg/local_object_storage/shard/rebuild.go +++ b/pkg/local_object_storage/shard/rebuild.go @@ -26,7 +26,7 @@ type rebuildLimiter struct { semaphore chan struct{} } -func newRebuildLimiter(workersCount uint32) *rebuildLimiter { +func NewRebuildLimiter(workersCount uint32) RebuildWorkerLimiter { return &rebuildLimiter{ semaphore: make(chan struct{}, workersCount), } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 1eaee8815..1e2bb7900 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -140,7 +140,7 @@ type cfg struct { reportErrorFunc func(selfID string, message string, err error) - rebuildLimiter RebuildWorkerLimiter + rebuildWorkersCount uint32 rebuildDisabled bool } @@ -151,7 +151,7 @@ func defaultCfg() *cfg { log: &logger.Logger{Logger: zap.L()}, gcCfg: defaultGCCfg(), reportErrorFunc: func(string, string, error) {}, - rebuildLimiter: newRebuildLimiter(1), + rebuildWorkersCount: 1, zeroSizeContainersCallback: func(context.Context, []cid.ID) {}, zeroCountContainersCallback: func(context.Context, []cid.ID) {}, } @@ -384,11 +384,11 @@ func WithExpiredCollectorWorkerCount(count int) Option { } } -// WithRebuildWorkerLimiter return option to set concurrent +// WithRebuildWorkersCount return option to set concurrent // workers count of storage rebuild operation. -func WithRebuildWorkerLimiter(l RebuildWorkerLimiter) Option { +func WithRebuildWorkersCount(count uint32) Option { return func(c *cfg) { - c.rebuildLimiter = l + c.rebuildWorkersCount = count } } From edb1747af7765fe685d9d4736626c409fbde7c79 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Sun, 1 Sep 2024 12:29:02 +0300 Subject: [PATCH 1327/1943] [#1337] blobovniczatree: Add rebuild by overflow Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/rebuild.go | 11 ++- .../blobstor/blobovniczatree/rebuild_test.go | 74 +++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index b7f20822e..b7b1dfd4b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -172,7 +172,7 @@ func (b *Blobovniczas) selectDBsDoNotMatchFillPercent(ctx context.Context, targe continue } path := filepath.Join(lvlPath, e.Name()) - resettlementRequired, err := b.fillPercentIsLow(path, target) + resettlementRequired, err := b.rebuildBySize(path, target) if err != nil { return false, err } @@ -187,14 +187,19 @@ func (b *Blobovniczas) selectDBsDoNotMatchFillPercent(ctx context.Context, targe return result, nil } -func (b *Blobovniczas) fillPercentIsLow(path string, target int) (bool, error) { +func (b *Blobovniczas) rebuildBySize(path string, targetFillPercent int) (bool, error) { shDB := b.getBlobovnicza(path) blz, err := shDB.Open() if err != nil { return false, err } defer shDB.Close() - return blz.FillPercent() < target, nil + fp := blz.FillPercent() + // accepted fill percent defines as + // |----|+++++++++++++++++|+++++++++++++++++|--------------- + // 0% target 100% 100+(100 - target) + // where `+` - accepted fill percent, `-` - not accepted fill percent + return fp < targetFillPercent || fp > 100+(100-targetFillPercent), nil } func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.MetaStorage, limiter common.ConcurrentWorkersLimiter) (uint64, error) { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 62ae9ea90..e6da1c553 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -228,6 +228,80 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { require.NoError(t, b.Close()) }) + + t.Run("rebuild by overflow", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + b := NewBlobovniczaTree( + context.Background(), + WithLogger(test.NewLogger(t)), + WithObjectSizeLimit(64*1024), + WithBlobovniczaShallowWidth(1), // single directory + WithBlobovniczaShallowDepth(1), + WithRootPath(dir), + WithBlobovniczaSize(100*1024), // 100 KB limit for each blobovnicza + WithWaitBeforeDropDB(0), + WithOpenedCacheSize(1000), + WithMoveBatchSize(3)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) + require.NoError(t, b.Init()) + + storageIDs := make(map[oid.Address][]byte) + for i := 0; i < 100; i++ { // 2 objects for one blobovnicza, so 50 DBs total will be created + obj := blobstortest.NewObject(64 * 1024) + data, err := obj.Marshal() + require.NoError(t, err) + var prm common.PutPrm + prm.Address = object.AddressOf(obj) + prm.RawData = data + res, err := b.Put(context.Background(), prm) + require.NoError(t, err) + storageIDs[prm.Address] = res.StorageID + } + metaStub := &storageIDUpdateStub{ + storageIDs: storageIDs, + guard: &sync.Mutex{}, + } + require.NoError(t, b.Close()) + b = NewBlobovniczaTree( + context.Background(), + WithLogger(test.NewLogger(t)), + WithObjectSizeLimit(64*1024), + WithBlobovniczaShallowWidth(1), + WithBlobovniczaShallowDepth(1), + WithRootPath(dir), + WithBlobovniczaSize(50*1024), // 50 KB limit for each blobovnicza + WithWaitBeforeDropDB(0), + WithOpenedCacheSize(1000), + WithMoveBatchSize(3)) + require.NoError(t, b.Open(mode.ComponentReadWrite)) + require.NoError(t, b.Init()) + + rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ + MetaStorage: metaStub, + WorkerLimiter: &rebuildLimiterStub{}, + Action: common.RebuildAction{ + SchemaChange: false, + FillPercent: true, + FillPercentValue: 80, + }, + }) + require.NoError(t, err) + require.Equal(t, uint64(49), rRes.FilesRemoved) + require.Equal(t, uint64(98), rRes.ObjectsMoved) // 49 DBs with 2 objects + require.Equal(t, uint64(98), metaStub.updatedCount) + + for addr, storageID := range storageIDs { + var gPrm common.GetPrm + gPrm.Address = addr + gPrm.StorageID = storageID + _, err := b.Get(context.Background(), gPrm) + require.NoError(t, err) + } + + require.NoError(t, b.Close()) + }) } func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { From d3b209c8e19edfdeb13035e1d8f4b4815cf77f08 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 4 Sep 2024 09:30:56 +0300 Subject: [PATCH 1328/1943] [#1337] shard: Disable background rebuild Since `frostfs-cli control shards rebuild` command was added, there is no need for background rebuild now. For failover tests used used value 1 to rebuild only schema change. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 4 -- cmd/frostfs-node/config/engine/config_test.go | 2 - .../config/engine/shard/config.go | 15 ----- config/example/node.env | 1 - config/example/node.json | 1 - config/example/node.yaml | 1 - docs/storage-node-configuration.md | 1 - .../blobstor/blobovniczatree/rebuild.go | 31 ++++------ .../blobovniczatree/rebuild_failover_test.go | 3 +- .../blobstor/blobovniczatree/rebuild_test.go | 28 ++------- .../blobstor/common/rebuild.go | 9 +-- pkg/local_object_storage/blobstor/rebuild.go | 4 +- pkg/local_object_storage/shard/control.go | 6 +- pkg/local_object_storage/shard/gc_test.go | 2 +- pkg/local_object_storage/shard/rebuild.go | 58 +++++++------------ pkg/local_object_storage/shard/shard.go | 21 ------- 16 files changed, 49 insertions(+), 138 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index b59518d14..16f49a082 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -126,8 +126,6 @@ type shardCfg struct { refillMetabaseWorkersCount int mode shardmode.Mode - rebuildWorkersCount uint32 - metaCfg struct { path string perm fs.FileMode @@ -240,7 +238,6 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, oldConfig newConfig.refillMetabase = oldConfig.RefillMetabase() newConfig.refillMetabaseWorkersCount = oldConfig.RefillMetabaseWorkersCount() - newConfig.rebuildWorkersCount = oldConfig.RebuildWorkerCount() newConfig.mode = oldConfig.Mode() newConfig.compress = oldConfig.Compress() newConfig.estimateCompressibility = oldConfig.EstimateCompressibility() @@ -998,7 +995,6 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID shard.WithLogger(c.log), shard.WithRefillMetabase(shCfg.refillMetabase), shard.WithRefillMetabaseWorkersCount(shCfg.refillMetabaseWorkersCount), - shard.WithRebuildWorkersCount(shCfg.rebuildWorkersCount), shard.WithMode(shCfg.mode), shard.WithBlobStorOptions(blobstoreOpts...), shard.WithMetaBaseOptions(mbOptions...), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 464d72556..ef6bf7f74 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -119,7 +119,6 @@ func TestEngineSection(t *testing.T) { require.Equal(t, false, sc.RefillMetabase()) require.Equal(t, mode.ReadOnly, sc.Mode()) require.Equal(t, 100, sc.RefillMetabaseWorkersCount()) - require.Equal(t, uint32(1000), sc.RebuildWorkerCount()) case 1: require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path()) require.Equal(t, fs.FileMode(0o644), pl.Perm()) @@ -175,7 +174,6 @@ func TestEngineSection(t *testing.T) { require.Equal(t, true, sc.RefillMetabase()) require.Equal(t, mode.ReadWrite, sc.Mode()) require.Equal(t, shardconfig.RefillMetabaseWorkersCountDefault, sc.RefillMetabaseWorkersCount()) - require.Equal(t, uint32(shardconfig.RebuildWorkersCountDefault), sc.RebuildWorkerCount()) } return nil }) diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go index ec9df0e89..0620c9f63 100644 --- a/cmd/frostfs-node/config/engine/shard/config.go +++ b/cmd/frostfs-node/config/engine/shard/config.go @@ -19,7 +19,6 @@ const ( SmallSizeLimitDefault = 1 << 20 EstimateCompressibilityThresholdDefault = 0.1 RefillMetabaseWorkersCountDefault = 500 - RebuildWorkersCountDefault = 5 ) // From wraps config section into Config. @@ -150,20 +149,6 @@ func (x *Config) RefillMetabaseWorkersCount() int { return RefillMetabaseWorkersCountDefault } -// RebuildWorkersCount returns the value of "resync_metabase_worker_count" config parameter. -// -// Returns RebuildWorkersCountDefault if the value is not a positive number. -func (x *Config) RebuildWorkerCount() uint32 { - v := config.Uint32Safe( - (*config.Config)(x), - "rebuild_worker_count", - ) - if v > 0 { - return v - } - return RebuildWorkersCountDefault -} - // Mode return the value of "mode" config parameter. // // Panics if read the value is not one of predefined diff --git a/config/example/node.env b/config/example/node.env index 1eccd8a5d..82553745e 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -94,7 +94,6 @@ FROSTFS_STORAGE_SHARD_RO_ERROR_THRESHOLD=100 ### Flag to refill Metabase from BlobStor FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE=false FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE_WORKER_COUNT=100 -FROSTFS_STORAGE_SHARD_0_REBUILD_WORKER_COUNT=1000 ### Flag to set shard mode FROSTFS_STORAGE_SHARD_0_MODE=read-only ### Write cache config diff --git a/config/example/node.json b/config/example/node.json index be7ced77a..da108c692 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -140,7 +140,6 @@ "mode": "read-only", "resync_metabase": false, "resync_metabase_worker_count": 100, - "rebuild_worker_count": 1000, "writecache": { "enabled": false, "no_sync": true, diff --git a/config/example/node.yaml b/config/example/node.yaml index 4b9720655..a79f48226 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -164,7 +164,6 @@ storage: # disabled (do not work with the shard, allows to not remove it from the config) resync_metabase: false # sync metabase with blobstor on start, expensive, leave false until complete understanding resync_metabase_worker_count: 100 - rebuild_worker_count: 1000 # count of rebuild storage concurrent workers writecache: enabled: false diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index f390d84a4..5bf35cd65 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -189,7 +189,6 @@ The following table describes configuration for each shard. | `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` | | `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. | | `resync_metabase_worker_count` | `int` | `1000` | Count of concurrent workers to resync metabase. | -| `rebuild_worker_count` | `int` | `5` | Count of concurrent workers to rebuild blobstore. | | `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. | | `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. | | `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index b7b1dfd4b..202d38cd7 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -60,7 +60,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm b.log.Debug(logs.BlobovniczaTreeCompletedPreviousRebuildSuccess) b.log.Debug(logs.BlobovniczaTreeCollectingDBToRebuild) - dbsToMigrate, err := b.getDBsToRebuild(ctx, prm.Action) + dbsToMigrate, err := b.getDBsToRebuild(ctx, prm.FillPercent) if err != nil { b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err)) success = false @@ -94,27 +94,20 @@ func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common. return res, nil } -func (b *Blobovniczas) getDBsToRebuild(ctx context.Context, action common.RebuildAction) ([]string, error) { - schemaChange := make(map[string]struct{}) - fillPercent := make(map[string]struct{}) - var err error - if action.SchemaChange { - schemaChange, err = b.selectDBsDoNotMatchSchema(ctx) - if err != nil { - return nil, err - } +func (b *Blobovniczas) getDBsToRebuild(ctx context.Context, fillPercent int) ([]string, error) { + withSchemaChange, err := b.selectDBsDoNotMatchSchema(ctx) + if err != nil { + return nil, err } - if action.FillPercent { - fillPercent, err = b.selectDBsDoNotMatchFillPercent(ctx, action.FillPercentValue) - if err != nil { - return nil, err - } + withFillPercent, err := b.selectDBsDoNotMatchFillPercent(ctx, fillPercent) + if err != nil { + return nil, err } - for k := range fillPercent { - schemaChange[k] = struct{}{} + for k := range withFillPercent { + withSchemaChange[k] = struct{}{} } - result := make([]string, 0, len(schemaChange)) - for db := range schemaChange { + result := make([]string, 0, len(withSchemaChange)) + for db := range withSchemaChange { result = append(result, db) } return result, nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index 9fec795ca..b177d20fc 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -145,7 +145,7 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), WithRootPath(dir), - WithBlobovniczaSize(100*1024*1024), + WithBlobovniczaSize(10*1024), WithWaitBeforeDropDB(0), WithOpenedCacheSize(1000)) require.NoError(t, b.Open(mode.ComponentReadWrite)) @@ -164,6 +164,7 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ MetaStorage: metaStub, WorkerLimiter: &rebuildLimiterStub{}, + FillPercent: 1, }) require.NoError(t, err) require.Equal(t, uint64(1), rRes.ObjectsMoved) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index e6da1c553..dfd928aaf 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -79,11 +79,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ MetaStorage: metaStub, WorkerLimiter: &rebuildLimiterStub{}, - Action: common.RebuildAction{ - SchemaChange: false, - FillPercent: true, - FillPercentValue: 60, - }, + FillPercent: 60, }) require.NoError(t, err) dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 @@ -135,11 +131,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ MetaStorage: metaStub, WorkerLimiter: &rebuildLimiterStub{}, - Action: common.RebuildAction{ - SchemaChange: false, - FillPercent: true, - FillPercentValue: 90, // 64KB / 100KB = 64% - }, + FillPercent: 90, // 64KB / 100KB = 64% }) require.NoError(t, err) dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 @@ -204,11 +196,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ MetaStorage: metaStub, WorkerLimiter: &rebuildLimiterStub{}, - Action: common.RebuildAction{ - SchemaChange: false, - FillPercent: true, - FillPercentValue: 80, - }, + FillPercent: 80, }) require.NoError(t, err) require.Equal(t, uint64(49), rRes.FilesRemoved) @@ -281,11 +269,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ MetaStorage: metaStub, WorkerLimiter: &rebuildLimiterStub{}, - Action: common.RebuildAction{ - SchemaChange: false, - FillPercent: true, - FillPercentValue: 80, - }, + FillPercent: 80, }) require.NoError(t, err) require.Equal(t, uint64(49), rRes.FilesRemoved) @@ -357,7 +341,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { var rPrm common.RebuildPrm rPrm.MetaStorage = metaStub rPrm.WorkerLimiter = &rebuildLimiterStub{} - rPrm.Action = common.RebuildAction{SchemaChange: true} + rPrm.FillPercent = 1 rRes, err := b.Rebuild(context.Background(), rPrm) require.NoError(t, err) dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 @@ -446,7 +430,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta var rPrm common.RebuildPrm rPrm.MetaStorage = metaStub rPrm.WorkerLimiter = &rebuildLimiterStub{} - rPrm.Action = common.RebuildAction{SchemaChange: true} + rPrm.FillPercent = 1 rRes, err := b.Rebuild(context.Background(), rPrm) require.NoError(t, err) dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 diff --git a/pkg/local_object_storage/blobstor/common/rebuild.go b/pkg/local_object_storage/blobstor/common/rebuild.go index 020d9d022..19e181ee7 100644 --- a/pkg/local_object_storage/blobstor/common/rebuild.go +++ b/pkg/local_object_storage/blobstor/common/rebuild.go @@ -11,17 +11,10 @@ type RebuildRes struct { FilesRemoved uint64 } -type RebuildAction struct { - SchemaChange bool - - FillPercent bool - FillPercentValue int -} - type RebuildPrm struct { MetaStorage MetaStorage WorkerLimiter ConcurrentWorkersLimiter - Action RebuildAction + FillPercent int } type MetaStorage interface { diff --git a/pkg/local_object_storage/blobstor/rebuild.go b/pkg/local_object_storage/blobstor/rebuild.go index 31bc2d167..7b2786ba2 100644 --- a/pkg/local_object_storage/blobstor/rebuild.go +++ b/pkg/local_object_storage/blobstor/rebuild.go @@ -18,14 +18,14 @@ type ConcurrentWorkersLimiter interface { ReleaseWorkSlot() } -func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate, limiter ConcurrentWorkersLimiter, action common.RebuildAction) error { +func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate, limiter ConcurrentWorkersLimiter, fillPercent int) error { var summary common.RebuildRes var rErr error for _, storage := range b.storage { res, err := storage.Storage.Rebuild(ctx, common.RebuildPrm{ MetaStorage: upd, WorkerLimiter: limiter, - Action: action, + FillPercent: fillPercent, }) summary.FilesRemoved += res.FilesRemoved summary.ObjectsMoved += res.ObjectsMoved diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 5e9639a7b..de881654a 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -129,8 +129,8 @@ func (s *Shard) Init(ctx context.Context) error { s.gc.init(ctx) - s.rb = newRebuilder(NewRebuildLimiter(s.rebuildWorkersCount)) - if !m.NoMetabase() && !s.rebuildDisabled { + s.rb = newRebuilder() + if !m.NoMetabase() { s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) } s.writecacheSealCancel.Store(dummyCancel) @@ -398,7 +398,7 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { defer unlock() s.rb.Stop(s.log) - if !s.info.Mode.NoMetabase() && !s.rebuildDisabled { + if !s.info.Mode.NoMetabase() { defer func() { s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) }() diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 1c0ef1c2e..90958cd35 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -187,7 +187,7 @@ func TestGCDropsObjectInhumedFromWritecache(t *testing.T) { func testGCDropsObjectInhumedFromWritecache(t *testing.T, flushbeforeInhume bool) { sh := newCustomShard(t, true, shardOptions{ - additionalShardOptions: []Option{WithDisabledGC(), WithDisabledRebuild()}, + additionalShardOptions: []Option{WithDisabledGC()}, wcOpts: []writecache.Option{writecache.WithDisableBackgroundFlush()}, }) defer func() { require.NoError(t, sh.Close()) }() diff --git a/pkg/local_object_storage/shard/rebuild.go b/pkg/local_object_storage/shard/rebuild.go index 2eef456be..0d83caa0c 100644 --- a/pkg/local_object_storage/shard/rebuild.go +++ b/pkg/local_object_storage/shard/rebuild.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -17,6 +16,8 @@ import ( "go.uber.org/zap" ) +var ErrRebuildInProgress = errors.New("shard rebuild in progress") + type RebuildWorkerLimiter interface { AcquireWorkSlot(ctx context.Context) error ReleaseWorkSlot() @@ -46,25 +47,23 @@ func (l *rebuildLimiter) ReleaseWorkSlot() { } type rebuildTask struct { - limiter RebuildWorkerLimiter - action common.RebuildAction + limiter RebuildWorkerLimiter + fillPercent int } type rebuilder struct { - mtx *sync.Mutex - wg *sync.WaitGroup - cancel func() - limiter RebuildWorkerLimiter - done chan struct{} - tasks chan rebuildTask + mtx *sync.Mutex + wg *sync.WaitGroup + cancel func() + done chan struct{} + tasks chan rebuildTask } -func newRebuilder(l RebuildWorkerLimiter) *rebuilder { +func newRebuilder() *rebuilder { return &rebuilder{ - mtx: &sync.Mutex{}, - wg: &sync.WaitGroup{}, - limiter: l, - tasks: make(chan rebuildTask, 10), + mtx: &sync.Mutex{}, + wg: &sync.WaitGroup{}, + tasks: make(chan rebuildTask), } } @@ -89,25 +88,14 @@ func (r *rebuilder) Start(ctx context.Context, bs *blobstor.BlobStor, mb *meta.D if !ok { continue } - runRebuild(ctx, bs, mb, log, t.action, t.limiter) + runRebuild(ctx, bs, mb, log, t.fillPercent, t.limiter) } } }() - select { - case <-ctx.Done(): - return - case r.tasks <- rebuildTask{ - limiter: r.limiter, - action: common.RebuildAction{ - SchemaChange: true, - }, - }: - return - } } func runRebuild(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *logger.Logger, - action common.RebuildAction, limiter RebuildWorkerLimiter, + fillPercent int, limiter RebuildWorkerLimiter, ) { select { case <-ctx.Done(): @@ -115,23 +103,25 @@ func runRebuild(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *lo default: } log.Info(logs.BlobstoreRebuildStarted) - if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, limiter, action); err != nil { + if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, limiter, fillPercent); err != nil { log.Warn(logs.FailedToRebuildBlobstore, zap.Error(err)) } else { log.Info(logs.BlobstoreRebuildCompletedSuccessfully) } } -func (r *rebuilder) ScheduleRebuild(ctx context.Context, limiter RebuildWorkerLimiter, action common.RebuildAction, +func (r *rebuilder) ScheduleRebuild(ctx context.Context, limiter RebuildWorkerLimiter, fillPercent int, ) error { select { case <-ctx.Done(): return ctx.Err() case r.tasks <- rebuildTask{ - limiter: limiter, - action: action, + limiter: limiter, + fillPercent: fillPercent, }: return nil + default: + return ErrRebuildInProgress } } @@ -198,9 +188,5 @@ func (s *Shard) ScheduleRebuild(ctx context.Context, p RebuildPrm) error { return ErrDegradedMode } - return s.rb.ScheduleRebuild(ctx, p.ConcurrencyLimiter, common.RebuildAction{ - SchemaChange: true, - FillPercent: true, - FillPercentValue: int(p.TargetFillPercent), - }) + return s.rb.ScheduleRebuild(ctx, p.ConcurrencyLimiter, int(p.TargetFillPercent)) } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 1e2bb7900..7496fc352 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -139,10 +139,6 @@ type cfg struct { metricsWriter MetricsWriter reportErrorFunc func(selfID string, message string, err error) - - rebuildWorkersCount uint32 - - rebuildDisabled bool } func defaultCfg() *cfg { @@ -151,7 +147,6 @@ func defaultCfg() *cfg { log: &logger.Logger{Logger: zap.L()}, gcCfg: defaultGCCfg(), reportErrorFunc: func(string, string, error) {}, - rebuildWorkersCount: 1, zeroSizeContainersCallback: func(context.Context, []cid.ID) {}, zeroCountContainersCallback: func(context.Context, []cid.ID) {}, } @@ -384,14 +379,6 @@ func WithExpiredCollectorWorkerCount(count int) Option { } } -// WithRebuildWorkersCount return option to set concurrent -// workers count of storage rebuild operation. -func WithRebuildWorkersCount(count uint32) Option { - return func(c *cfg) { - c.rebuildWorkersCount = count - } -} - // WithDisabledGC disables GC. // For testing purposes only. func WithDisabledGC() Option { @@ -414,14 +401,6 @@ func WithZeroCountCallback(cb EmptyContainersCallback) Option { } } -// WithDisabledRebuild returns an option to disable a shard rebuild. -// For testing purposes only. -func WithDisabledRebuild() Option { - return func(c *cfg) { - c.rebuildDisabled = true - } -} - func (s *Shard) fillInfo() { s.cfg.info.MetaBaseInfo = s.metaBase.DumpInfo() s.cfg.info.BlobStorInfo = s.blobStor.DumpInfo() From 654d970fadfe7eb47a5ce566be81ad15119df6a0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 6 Sep 2024 11:16:12 +0300 Subject: [PATCH 1329/1943] [#1355] adm: Run `metabase upgrade` concurrently Signed-off-by: Dmitrii Stepanov --- .../internal/modules/metabase/upgrade.go | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/metabase/upgrade.go b/cmd/frostfs-adm/internal/modules/metabase/upgrade.go index 83e085df4..96cb62f10 100644 --- a/cmd/frostfs-adm/internal/modules/metabase/upgrade.go +++ b/cmd/frostfs-adm/internal/modules/metabase/upgrade.go @@ -3,6 +3,7 @@ package metabase import ( "errors" "fmt" + "sync" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" @@ -11,6 +12,7 @@ import ( shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" ) const ( @@ -57,17 +59,28 @@ func upgrade(cmd *cobra.Command, _ []string) error { cmd.Println(i+1, ":", path) } result := make(map[string]bool) + var resultGuard sync.Mutex + eg, ctx := errgroup.WithContext(cmd.Context()) for _, path := range paths { - cmd.Println("upgrading metabase", path, "...") - if err := meta.Upgrade(cmd.Context(), path, !noCompact, func(a ...any) { - cmd.Println(append([]any{time.Now().Format(time.RFC3339), ":", path, ":"}, a...)...) - }); err != nil { - result[path] = false - cmd.Println("error: failed to upgrade metabase", path, ":", err) - } else { - result[path] = true - cmd.Println("metabase", path, "upgraded successfully") - } + eg.Go(func() error { + var success bool + cmd.Println("upgrading metabase", path, "...") + if err := meta.Upgrade(ctx, path, !noCompact, func(a ...any) { + cmd.Println(append([]any{time.Now().Format(time.RFC3339), ":", path, ":"}, a...)...) + }); err != nil { + cmd.Println("error: failed to upgrade metabase", path, ":", err) + } else { + success = true + cmd.Println("metabase", path, "upgraded successfully") + } + resultGuard.Lock() + result[path] = success + resultGuard.Unlock() + return nil + }) + } + if err := eg.Wait(); err != nil { + return err } for mb, ok := range result { if ok { From 4668efc0bfdfac3750307d0863effb991baa1a4b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 6 Sep 2024 11:17:02 +0300 Subject: [PATCH 1330/1943] [#1355] metabase: Upgrade improvements Do not fail on same latest version to run compact on upgraded metabase. Use NoSync on compact. Log every batch on bucket delete stage. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/upgrade.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/metabase/upgrade.go b/pkg/local_object_storage/metabase/upgrade.go index e9abd746c..f677dcf8e 100644 --- a/pkg/local_object_storage/metabase/upgrade.go +++ b/pkg/local_object_storage/metabase/upgrade.go @@ -27,6 +27,10 @@ const ( var updates = map[uint64]func(ctx context.Context, db *bbolt.DB, log func(a ...any)) error{ 2: upgradeFromV2ToV3, + 3: func(_ context.Context, _ *bbolt.DB, log func(a ...any)) error { + log("metabase already upgraded") + return nil + }, } func Upgrade(ctx context.Context, path string, compact bool, log func(a ...any)) error { @@ -86,6 +90,7 @@ func compactDB(db *bbolt.DB) error { } dst, err := bbolt.Open(tmpFileName, f.Mode(), &bbolt.Options{ Timeout: 100 * time.Millisecond, + NoSync: true, }) if err != nil { return fmt.Errorf("can't open new metabase to compact: %w", err) @@ -93,6 +98,9 @@ func compactDB(db *bbolt.DB) error { if err := bbolt.Compact(dst, db, compactMaxTxSize); err != nil { return fmt.Errorf("compact metabase: %w", errors.Join(err, dst.Close(), os.Remove(tmpFileName))) } + if err := dst.Sync(); err != nil { + return fmt.Errorf("sync compacted metabase: %w", errors.Join(err, os.Remove(tmpFileName))) + } if err := dst.Close(); err != nil { return fmt.Errorf("close compacted metabase: %w", errors.Join(err, os.Remove(tmpFileName))) } @@ -369,8 +377,7 @@ func dropBucketsByPrefix(ctx context.Context, db *bbolt.DB, prefix []byte, log f log("deleting buckets completed with an error:", err) return err } - if count += uint64(len(keys)); count%upgradeLogFrequency == 0 { - log("deleted", count, "buckets") - } + count += uint64(len(keys)) + log("deleted", count, "buckets") } } From 92fe5d90f50a4d0f3c3b5265a32e0127848559e4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 9 Sep 2024 18:39:22 +0300 Subject: [PATCH 1331/1943] [#1359] writecache: Drop unused Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/writecache/cachebbolt.go | 8 -------- pkg/local_object_storage/writecache/put.go | 5 ----- 2 files changed, 13 deletions(-) diff --git a/pkg/local_object_storage/writecache/cachebbolt.go b/pkg/local_object_storage/writecache/cachebbolt.go index cdd4ed442..f1e6a619a 100644 --- a/pkg/local_object_storage/writecache/cachebbolt.go +++ b/pkg/local_object_storage/writecache/cachebbolt.go @@ -18,16 +18,9 @@ import ( type cache struct { options - // mtx protects statistics, counters and compressFlags. - mtx sync.RWMutex - mode mode.Mode modeMtx sync.RWMutex - // compressFlags maps address of a big object to boolean value indicating - // whether object should be compressed. - compressFlags map[string]struct{} - // flushCh is a channel with objects to flush. flushCh chan objectInfo // cancel is cancel function, protected by modeMtx in Close. @@ -66,7 +59,6 @@ func New(opts ...Option) Cache { flushCh: make(chan objectInfo), mode: mode.Disabled, - compressFlags: make(map[string]struct{}), options: options{ log: &logger.Logger{Logger: zap.NewNop()}, maxObjectSize: defaultMaxObjectSize, diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 150399de8..ae0e8b77a 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -115,11 +115,6 @@ func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) erro return err } - if compressor := c.blobstor.Compressor(); compressor != nil && compressor.NeedsCompression(prm.Object) { - c.mtx.Lock() - c.compressFlags[addr] = struct{}{} - c.mtx.Unlock() - } storagelog.Write(c.log, storagelog.AddressField(addr), storagelog.StorageTypeField(wcStorageType), From a812932984531162648fdbfa985a6f496fdbd80e Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 10 Sep 2024 11:15:30 +0300 Subject: [PATCH 1332/1943] [#1362] ape: Move common APE check logic to separate package * Tree and object service have the same log for checking APE. So, this check should be moved to common package. Signed-off-by: Airat Arifullin --- pkg/services/common/ape/checker.go | 167 +++++++++++++++++++++++++++++ pkg/services/object/ape/checker.go | 139 +++++------------------- pkg/services/tree/ape.go | 116 ++------------------ pkg/services/tree/service.go | 5 + 4 files changed, 205 insertions(+), 222 deletions(-) create mode 100644 pkg/services/common/ape/checker.go diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go new file mode 100644 index 000000000..f24d22124 --- /dev/null +++ b/pkg/services/common/ape/checker.go @@ -0,0 +1,167 @@ +package ape + +import ( + "crypto/ecdsa" + "errors" + "fmt" + + aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/router" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +var ( + errInvalidTargetType = errors.New("bearer token defines non-container target override") + errBearerExpired = errors.New("bearer token has expired") + errBearerInvalidSignature = errors.New("bearer token has invalid signature") + errBearerInvalidContainerID = errors.New("bearer token was created for another container") + errBearerNotSignedByOwner = errors.New("bearer token is not signed by the container owner") + errBearerInvalidOwner = errors.New("bearer token owner differs from the request sender") +) + +type CheckPrm struct { + // Request is an APE-request that is checked by policy engine. + Request aperequest.Request + + Namespace string + + Container cid.ID + + // An encoded container's owner user ID. + ContainerOwner user.ID + + // PublicKey is public key of the request sender. + PublicKey *keys.PublicKey + + // The request's bearer token. It is used in order to check APE overrides with the token. + BearerToken *bearer.Token + + // If SoftAPECheck is set to true, then NoRuleFound is interpreted as allow. + SoftAPECheck bool +} + +// CheckCore provides methods to perform the common logic of APE check. +type CheckCore interface { + // CheckAPE performs the common policy-engine check logic on a prepared request. + CheckAPE(prm CheckPrm) error +} + +type checkerCoreImpl struct { + LocalOverrideStorage policyengine.LocalOverrideStorage + MorphChainStorage policyengine.MorphRuleChainStorageReader + FrostFSSubjectProvider frostfsidcore.SubjectProvider + State netmap.State +} + +func New(localOverrideStorage policyengine.LocalOverrideStorage, morphChainStorage policyengine.MorphRuleChainStorageReader, + frostFSSubjectProvider frostfsidcore.SubjectProvider, state netmap.State) CheckCore { + return &checkerCoreImpl{ + LocalOverrideStorage: localOverrideStorage, + MorphChainStorage: morphChainStorage, + FrostFSSubjectProvider: frostFSSubjectProvider, + State: state, + } +} + +// CheckAPE performs the common policy-engine check logic on a prepared request. +func (c *checkerCoreImpl) CheckAPE(prm CheckPrm) error { + var cr policyengine.ChainRouter + if prm.BearerToken != nil && !prm.BearerToken.Impersonate() { + var err error + if err = isValidBearer(prm.BearerToken, prm.ContainerOwner, prm.Container, prm.PublicKey, c.State); err != nil { + return fmt.Errorf("bearer validation error: %w", err) + } + cr, err = router.BearerChainFeedRouter(c.LocalOverrideStorage, c.MorphChainStorage, prm.BearerToken.APEOverride()) + if err != nil { + return fmt.Errorf("create chain router error: %w", err) + } + } else { + cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.MorphChainStorage, c.LocalOverrideStorage) + } + + groups, err := aperequest.Groups(c.FrostFSSubjectProvider, prm.PublicKey) + if err != nil { + return fmt.Errorf("failed to get group ids: %w", err) + } + + // Policy contract keeps group related chains as namespace-group pair. + for i := range groups { + groups[i] = fmt.Sprintf("%s:%s", prm.Namespace, groups[i]) + } + + rt := policyengine.NewRequestTargetExtended(prm.Namespace, prm.Container.EncodeToString(), fmt.Sprintf("%s:%s", prm.Namespace, prm.PublicKey.Address()), groups) + status, found, err := cr.IsAllowed(apechain.Ingress, rt, prm.Request) + if err != nil { + return err + } + if !found && prm.SoftAPECheck || status == apechain.Allow { + return nil + } + err = fmt.Errorf("access to operation %s is denied by access policy engine: %s", prm.Request.Operation(), status.String()) + return apeErr(err) +} + +func apeErr(err error) error { + errAccessDenied := &apistatus.ObjectAccessDenied{} + errAccessDenied.WriteReason(err.Error()) + return errAccessDenied +} + +// isValidBearer checks whether bearer token was correctly signed by authorized +// entity. This method might be defined on whole ACL service because it will +// require fetching current epoch to check lifetime. +func isValidBearer(token *bearer.Token, ownerCnr user.ID, cntID cid.ID, publicKey *keys.PublicKey, st netmap.State) error { + if token == nil { + return nil + } + + // First check token lifetime. Simplest verification. + if token.InvalidAt(st.CurrentEpoch()) { + return errBearerExpired + } + + // Then check if bearer token is signed correctly. + if !token.VerifySignature() { + return errBearerInvalidSignature + } + + // Check for ape overrides defined in the bearer token. + apeOverride := token.APEOverride() + if len(apeOverride.Chains) > 0 && apeOverride.Target.TargetType != ape.TargetTypeContainer { + return fmt.Errorf("%w: %s", errInvalidTargetType, apeOverride.Target.TargetType.ToV2().String()) + } + + // Then check if container is either empty or equal to the container in the request. + var targetCnr cid.ID + err := targetCnr.DecodeString(apeOverride.Target.Name) + if err != nil { + return fmt.Errorf("invalid cid format: %s", apeOverride.Target.Name) + } + if !cntID.Equals(targetCnr) { + return errBearerInvalidContainerID + } + + // Then check if container owner signed this token. + if !bearer.ResolveIssuer(*token).Equals(ownerCnr) { + return errBearerNotSignedByOwner + } + + // Then check if request sender has rights to use this token. + var usrSender user.ID + user.IDFromKey(&usrSender, (ecdsa.PublicKey)(*publicKey)) + + if !token.AssertUser(usrSender) { + return errBearerInvalidOwner + } + + return nil +} diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index a1972292e..3688638d0 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -2,49 +2,41 @@ package ape import ( "context" - "crypto/ecdsa" "errors" "fmt" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/router" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + checkercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) type checkerImpl struct { - localOverrideStorage policyengine.LocalOverrideStorage - morphChainStorage policyengine.MorphRuleChainStorageReader - headerProvider HeaderProvider - frostFSIDClient frostfsidcore.SubjectProvider - nm netmap.Source - st netmap.State - cnrSource container.Source - nodePK []byte + checkerCore checkercore.CheckCore + frostFSIDClient frostfsidcore.SubjectProvider + headerProvider HeaderProvider + nm netmap.Source + cnrSource container.Source + nodePK []byte } func NewChecker(localOverrideStorage policyengine.LocalOverrideStorage, morphChainStorage policyengine.MorphRuleChainStorageReader, headerProvider HeaderProvider, frostFSIDClient frostfsidcore.SubjectProvider, nm netmap.Source, st netmap.State, cnrSource container.Source, nodePK []byte) Checker { return &checkerImpl{ - localOverrideStorage: localOverrideStorage, - morphChainStorage: morphChainStorage, - headerProvider: headerProvider, - frostFSIDClient: frostFSIDClient, - nm: nm, - st: st, - cnrSource: cnrSource, - nodePK: nodePK, + checkerCore: checkercore.New(localOverrideStorage, morphChainStorage, frostFSIDClient, st), + frostFSIDClient: frostFSIDClient, + headerProvider: headerProvider, + nm: nm, + cnrSource: cnrSource, + nodePK: nodePK, } } @@ -85,68 +77,9 @@ type Prm struct { XHeaders []session.XHeader } -var ( - errMissingOID = errors.New("object ID is not set") - errInvalidTargetType = errors.New("bearer token defines non-container target override") - errBearerExpired = errors.New("bearer token has expired") - errBearerInvalidSignature = errors.New("bearer token has invalid signature") - errBearerInvalidContainerID = errors.New("bearer token was created for another container") - errBearerNotSignedByOwner = errors.New("bearer token is not signed by the container owner") - errBearerInvalidOwner = errors.New("bearer token owner differs from the request sender") -) +var errMissingOID = errors.New("object ID is not set") -// isValidBearer checks whether bearer token was correctly signed by authorized -// entity. This method might be defined on whole ACL service because it will -// require fetching current epoch to check lifetime. -func isValidBearer(token *bearer.Token, ownerCnr user.ID, containerID cid.ID, publicKey *keys.PublicKey, st netmap.State) error { - if token == nil { - return nil - } - - // First check token lifetime. Simplest verification. - if token.InvalidAt(st.CurrentEpoch()) { - return errBearerExpired - } - - // Then check if bearer token is signed correctly. - if !token.VerifySignature() { - return errBearerInvalidSignature - } - - // Check for ape overrides defined in the bearer token. - apeOverride := token.APEOverride() - if len(apeOverride.Chains) > 0 && apeOverride.Target.TargetType != ape.TargetTypeContainer { - return fmt.Errorf("%w: %s", errInvalidTargetType, apeOverride.Target.TargetType.ToV2().String()) - } - - // Then check if container is either empty or equal to the container in the request. - var targetCnr cid.ID - err := targetCnr.DecodeString(apeOverride.Target.Name) - if err != nil { - return fmt.Errorf("invalid cid format: %s", apeOverride.Target.Name) - } - if !containerID.Equals(targetCnr) { - return errBearerInvalidContainerID - } - - // Then check if container owner signed this token. - if !bearer.ResolveIssuer(*token).Equals(ownerCnr) { - return errBearerNotSignedByOwner - } - - // Then check if request sender has rights to use this token. - var usrSender user.ID - user.IDFromKey(&usrSender, (ecdsa.PublicKey)(*publicKey)) - - if !token.AssertUser(usrSender) { - return errBearerInvalidOwner - } - - return nil -} - -// CheckAPE checks if a request or a response is permitted creating an ape request and passing -// it to chain router. +// CheckAPE prepares an APE-request and checks if it is permitted by policies. func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { // APE check is ignored for some inter-node requests. if prm.Role == nativeschema.PropertyValueContainerRoleContainer { @@ -171,38 +104,14 @@ func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { if err != nil { return err } - groups, err := aperequest.Groups(c.frostFSIDClient, pub) - if err != nil { - return fmt.Errorf("failed to get group ids: %w", err) - } - // Policy contract keeps group related chains as namespace-group pair. - for i := range groups { - groups[i] = fmt.Sprintf("%s:%s", prm.Namespace, groups[i]) - } - - var cr policyengine.ChainRouter - if prm.BearerToken != nil && !prm.BearerToken.Impersonate() { - if err := isValidBearer(prm.BearerToken, prm.ContainerOwner, prm.Container, pub, c.st); err != nil { - return fmt.Errorf("bearer token validation error: %w", err) - } - cr, err = router.BearerChainFeedRouter(c.localOverrideStorage, c.morphChainStorage, prm.BearerToken.APEOverride()) - if err != nil { - return fmt.Errorf("create chain router error: %w", err) - } - } else { - cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.morphChainStorage, c.localOverrideStorage) - } - - rt := policyengine.NewRequestTargetExtended(prm.Namespace, prm.Container.EncodeToString(), fmt.Sprintf("%s:%s", prm.Namespace, pub.Address()), groups) - status, ruleFound, err := cr.IsAllowed(apechain.Ingress, rt, r) - if err != nil { - return err - } - - if !ruleFound && prm.SoftAPECheck || status == apechain.Allow { - return nil - } - - return fmt.Errorf("method %s: %s", prm.Method, status) + return c.checkerCore.CheckAPE(checkercore.CheckPrm{ + Request: r, + PublicKey: pub, + Namespace: prm.Method, + Container: prm.Container, + ContainerOwner: prm.ContainerOwner, + BearerToken: prm.BearerToken, + SoftAPECheck: prm.SoftAPECheck, + }) } diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index ee4687911..693b16e60 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -2,42 +2,25 @@ package tree import ( "context" - "crypto/ecdsa" "encoding/hex" - "errors" "fmt" "net" "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/converter" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/router" core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + checkercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" - "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" commonschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/common" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "google.golang.org/grpc/peer" ) -var ( - errInvalidTargetType = errors.New("bearer token defines non-container target override") - errBearerExpired = errors.New("bearer token has expired") - errBearerInvalidSignature = errors.New("bearer token has invalid signature") - errBearerInvalidContainerID = errors.New("bearer token was created for another container") - errBearerNotSignedByOwner = errors.New("bearer token is not signed by the container owner") - errBearerInvalidOwner = errors.New("bearer token owner differs from the request sender") -) - func (s *Service) newAPERequest(ctx context.Context, namespace string, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, ) (aperequest.Request, error) { @@ -77,56 +60,6 @@ func (s *Service) newAPERequest(ctx context.Context, namespace string, ), nil } -// isValidBearer checks whether bearer token was correctly signed by authorized -// entity. This method might be defined on whole ACL service because it will -// require fetching current epoch to check lifetime. -func isValidBearer(token *bearer.Token, ownerCnr user.ID, cntID cid.ID, publicKey *keys.PublicKey, st netmap.State) error { - if token == nil { - return nil - } - - // First check token lifetime. Simplest verification. - if token.InvalidAt(st.CurrentEpoch()) { - return errBearerExpired - } - - // Then check if bearer token is signed correctly. - if !token.VerifySignature() { - return errBearerInvalidSignature - } - - // Check for ape overrides defined in the bearer token. - apeOverride := token.APEOverride() - if len(apeOverride.Chains) > 0 && apeOverride.Target.TargetType != ape.TargetTypeContainer { - return fmt.Errorf("%w: %s", errInvalidTargetType, apeOverride.Target.TargetType.ToV2().String()) - } - - // Then check if container is either empty or equal to the container in the request. - var targetCnr cid.ID - err := targetCnr.DecodeString(apeOverride.Target.Name) - if err != nil { - return fmt.Errorf("invalid cid format: %s", apeOverride.Target.Name) - } - if !cntID.Equals(targetCnr) { - return errBearerInvalidContainerID - } - - // Then check if container owner signed this token. - if !bearer.ResolveIssuer(*token).Equals(ownerCnr) { - return errBearerNotSignedByOwner - } - - // Then check if request sender has rights to use this token. - var usrSender user.ID - user.IDFromKey(&usrSender, (ecdsa.PublicKey)(*publicKey)) - - if !token.AssertUser(usrSender) { - return errBearerInvalidOwner - } - - return nil -} - func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, container *core.Container, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, ) error { @@ -141,45 +74,14 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, return fmt.Errorf("failed to create ape request: %w", err) } - var cr engine.ChainRouter - if bt != nil && !bt.Impersonate() { - if err := isValidBearer(bt, container.Value.Owner(), cid, publicKey, s.state); err != nil { - return fmt.Errorf("bearer validation error: %w", err) - } - cr, err = router.BearerChainFeedRouter(s.localOverrideStorage, s.morphChainStorage, bt.APEOverride()) - if err != nil { - return fmt.Errorf("create chain router error: %w", err) - } - } else { - cr = engine.NewDefaultChainRouterWithLocalOverrides(s.morphChainStorage, s.localOverrideStorage) - } - - groups, err := aperequest.Groups(s.frostfsidSubjectProvider, publicKey) - if err != nil { - return fmt.Errorf("failed to get group ids: %w", err) - } - - // Policy contract keeps group related chains as namespace-group pair. - for i := range groups { - groups[i] = fmt.Sprintf("%s:%s", namespace, groups[i]) - } - - rt := engine.NewRequestTargetExtended(namespace, cid.EncodeToString(), fmt.Sprintf("%s:%s", namespace, publicKey.Address()), groups) - status, found, err := cr.IsAllowed(apechain.Ingress, rt, request) - if err != nil { - return err - } - if found && status == apechain.Allow { - return nil - } - err = fmt.Errorf("access to operation %s is denied by access policy engine: %s", request.Operation(), status.String()) - return apeErr(err) -} - -func apeErr(err error) error { - errAccessDenied := &apistatus.ObjectAccessDenied{} - errAccessDenied.WriteReason(err.Error()) - return errAccessDenied + return s.apeChecker.CheckAPE(checkercore.CheckPrm{ + Request: request, + Namespace: namespace, + Container: cid, + PublicKey: publicKey, + BearerToken: bt, + SoftAPECheck: false, + }) } // fillWithUserClaimTags fills ape request properties with user claim tags getting them from frostfsid contract by actor public key. diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 4da61617f..875e47ecb 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -10,6 +10,7 @@ import ( "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + checkercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -37,6 +38,8 @@ type Service struct { initialSyncDone atomic.Bool + apeChecker checkercore.CheckCore + // cnrMap contains existing (used) container IDs. cnrMap map[cidSDK.ID]struct{} // cnrMapMtx protects cnrMap @@ -72,6 +75,8 @@ func New(opts ...Option) *Service { s.syncChan = make(chan struct{}) s.syncPool, _ = ants.NewPool(defaultSyncWorkerCount) + s.apeChecker = checkercore.New(s.localOverrideStorage, s.morphChainStorage, s.frostfsidSubjectProvider, s.state) + return &s } From 2220f6a8091d9b861fd6a86b7afc90320591e9b1 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 10 Sep 2024 16:45:15 +0300 Subject: [PATCH 1333/1943] [#1365] Makefile: Fix HUB_IMAGE Signed-off-by: Alexander Chuprov --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 71492ef17..2f29ac19c 100755 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ SHELL = bash REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") -HUB_IMAGE ?= truecloudlab/frostfs +HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.22 From 5fac4058e8cbc9ef8484b2ddaaf583574684dded Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Tue, 10 Sep 2024 12:14:46 +0300 Subject: [PATCH 1334/1943] [#1364] cmd/common: Add tests for CreateViper and ReloadViper Add tests for `CreateViper` and `ReloadViper` to ensure that no extra files, except *.yaml, *.yml, *.json, are loaded from config directory. Signed-off-by: Aleksey Savchuk --- cmd/internal/common/config/viper_test.go | 107 +++++++++++++++++++++++ pkg/util/config/test/generate.go | 58 ++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 cmd/internal/common/config/viper_test.go create mode 100644 pkg/util/config/test/generate.go diff --git a/cmd/internal/common/config/viper_test.go b/cmd/internal/common/config/viper_test.go new file mode 100644 index 000000000..d533a15c2 --- /dev/null +++ b/cmd/internal/common/config/viper_test.go @@ -0,0 +1,107 @@ +package config_test + +import ( + "encoding/json" + "os" + "path" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config/test" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestCreateReloadViper(t *testing.T) { + type m = map[string]any + + dummyFileSize := 1 << 10 + + configPath := t.TempDir() + configFile := "000_a.yaml" + + configDirPath := path.Join(configPath, "conf.d") + require.NoError(t, os.Mkdir(configDirPath, 0o700)) + + configtest.PrepareConfigFiles(t, configPath, []configtest.ConfigFile{ + configtest.NewConfigFile(configFile, m{"a": "000"}, yaml.Marshal), + }) + + // Not valid configs, dummy files those appear lexicographically first. + configtest.PrepareDummyFiles(t, configDirPath, []configtest.DummyFile{ + configtest.NewDummyFile("000_file_1", dummyFileSize), + configtest.NewDummyFile("000_file_2", dummyFileSize), + configtest.NewDummyFile("000_file_3", dummyFileSize), + }) + + configtest.PrepareConfigFiles(t, configDirPath, []configtest.ConfigFile{ + // Valid configs with invalid extensions those appear lexicographically first. + configtest.NewConfigFile("001_a.yaml.un~", m{"a": "101"}, yaml.Marshal), + configtest.NewConfigFile("001_b.yml~", m{"b": m{"a": "102", "b": "103"}}, yaml.Marshal), + configtest.NewConfigFile("001_c.yaml.swp", m{"c": m{"a": "104", "b": "105"}}, yaml.Marshal), + configtest.NewConfigFile("001_d.json.swp", m{"d": m{"a": "106", "b": "107"}}, json.Marshal), + + // Valid configs with valid extensions those should be loaded. + configtest.NewConfigFile("010_a.yaml", m{"a": "1"}, yaml.Marshal), + configtest.NewConfigFile("020_b.yml", m{"b": m{"a": "2", "b": "3"}}, yaml.Marshal), + configtest.NewConfigFile("030_c.json", m{"c": m{"a": "4", "b": "5"}}, json.Marshal), + + // Valid configs with invalid extensions those appear lexicographically last. + configtest.NewConfigFile("099_a.yaml.un~", m{"a": "201"}, yaml.Marshal), + configtest.NewConfigFile("099_b.yml~", m{"b": m{"a": "202", "b": "203"}}, yaml.Marshal), + configtest.NewConfigFile("099_c.yaml.swp", m{"c": m{"a": "204", "b": "205"}}, yaml.Marshal), + configtest.NewConfigFile("099_c.json.swp", m{"d": m{"a": "206", "b": "207"}}, json.Marshal), + }) + + // Not valid configs, dummy files those appear lexicographically last. + configtest.PrepareDummyFiles(t, configDirPath, []configtest.DummyFile{ + configtest.NewDummyFile("999_file_1", dummyFileSize), + configtest.NewDummyFile("999_file_2", dummyFileSize), + configtest.NewDummyFile("999_file_3", dummyFileSize), + }) + + finalConfig := m{"a": "1", "b": m{"a": "2", "b": "3"}, "c": m{"a": "4", "b": "5"}} + + var ( + v *viper.Viper + err error + ) + + t.Run("create config with config dir only", func(t *testing.T) { + v, err = config.CreateViper( + config.WithConfigDir(configDirPath), + ) + require.NoError(t, err) + assert.Equal(t, finalConfig, v.AllSettings()) + }) + + t.Run("reload config with config dir only", func(t *testing.T) { + err = config.ReloadViper( + config.WithViper(v), + config.WithConfigDir(configDirPath), + ) + require.NoError(t, err) + assert.Equal(t, finalConfig, v.AllSettings()) + }) + + t.Run("create config with both config and config dir", func(t *testing.T) { + v, err = config.CreateViper( + config.WithConfigFile(path.Join(configPath, configFile)), + config.WithConfigDir(configDirPath), + ) + require.NoError(t, err) + assert.Equal(t, finalConfig, v.AllSettings()) + }) + + t.Run("reload config with both config and config dir", func(t *testing.T) { + err = config.ReloadViper( + config.WithViper(v), + config.WithConfigFile(path.Join(configPath, configFile)), + config.WithConfigDir(configDirPath), + ) + require.NoError(t, err) + assert.Equal(t, finalConfig, v.AllSettings()) + }) +} diff --git a/pkg/util/config/test/generate.go b/pkg/util/config/test/generate.go new file mode 100644 index 000000000..63e286615 --- /dev/null +++ b/pkg/util/config/test/generate.go @@ -0,0 +1,58 @@ +package configtest + +import ( + "crypto/rand" + "os" + "path" + "testing" + + "github.com/stretchr/testify/require" +) + +type MarshalFunc = func(any) ([]byte, error) + +type ConfigFile struct { + filename string + content map[string]any + marshal func(any) ([]byte, error) +} + +type DummyFile struct { + filename string + size int +} + +func NewConfigFile(filename string, content map[string]any, marshal MarshalFunc) ConfigFile { + return ConfigFile{ + filename: filename, + content: content, + marshal: marshal, + } +} + +func NewDummyFile(filename string, size int) DummyFile { + return DummyFile{ + filename: filename, + size: size, + } +} + +func PrepareConfigFiles(t *testing.T, dir string, files []ConfigFile) { + for _, file := range files { + data, err := file.marshal(file.content) + require.NoError(t, err) + + err = os.WriteFile(path.Join(dir, file.filename), data, 0o600) + require.NoError(t, err) + } +} + +func PrepareDummyFiles(t *testing.T, dir string, files []DummyFile) { + for _, file := range files { + data := make([]byte, file.size) + _, _ = rand.Read(data) + + err := os.WriteFile(path.Join(dir, file.filename), data, 0o600) + require.NoError(t, err) + } +} From dea6f031f97664259d407bff6a320b295dc4b3d0 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 9 Sep 2024 19:26:41 +0300 Subject: [PATCH 1335/1943] [#1331] cli/tree: Add order flag to `tree get-subtree` Added `--ordered` flag to sort output by ascending FileName. Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/modules/tree/root.go | 1 + cmd/frostfs-cli/modules/tree/subtree.go | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/cmd/frostfs-cli/modules/tree/root.go b/cmd/frostfs-cli/modules/tree/root.go index efd1c08b5..5a53c50d6 100644 --- a/cmd/frostfs-cli/modules/tree/root.go +++ b/cmd/frostfs-cli/modules/tree/root.go @@ -49,6 +49,7 @@ const ( heightFlagKey = "height" countFlagKey = "count" depthFlagKey = "depth" + orderFlagKey = "ordered" ) func initCTID(cmd *cobra.Command) { diff --git a/cmd/frostfs-cli/modules/tree/subtree.go b/cmd/frostfs-cli/modules/tree/subtree.go index e58a13fd6..e88ef79cb 100644 --- a/cmd/frostfs-cli/modules/tree/subtree.go +++ b/cmd/frostfs-cli/modules/tree/subtree.go @@ -30,6 +30,7 @@ func initGetSubtreeCmd() { ff := getSubtreeCmd.Flags() ff.Uint64(rootIDFlagKey, 0, "Root ID to traverse from.") ff.Uint32(depthFlagKey, 10, "Traversal depth.") + ff.Bool(orderFlagKey, false, "Sort output by ascending FileName.") _ = getSubtreeCmd.MarkFlagRequired(commonflags.CIDFlag) _ = getSubtreeCmd.MarkFlagRequired(treeIDFlagKey) @@ -59,6 +60,13 @@ func getSubTree(cmd *cobra.Command, _ []string) { depth, _ := cmd.Flags().GetUint32(depthFlagKey) + order, _ := cmd.Flags().GetBool(orderFlagKey) + + bodyOrder := tree.GetSubTreeRequest_Body_Order_None + if order { + bodyOrder = tree.GetSubTreeRequest_Body_Order_Asc + } + var bt []byte if t := common.ReadBearerToken(cmd, bearerFlagKey); t != nil { bt = t.Marshal() @@ -71,6 +79,9 @@ func getSubTree(cmd *cobra.Command, _ []string) { RootId: []uint64{rid}, Depth: depth, BearerToken: bt, + OrderBy: &tree.GetSubTreeRequest_Body_Order{ + Direction: bodyOrder, + }, }, } From ec8da4056704d81107f514bcb998aa6f3dd7b07f Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Wed, 11 Sep 2024 12:40:04 +0300 Subject: [PATCH 1336/1943] [#1369] Update obsolete URLs Signed-off-by: Vitaliy Potyarkin --- README.md | 15 +++++++-------- cmd/frostfs-adm/docs/deploy.md | 4 ++-- config/testnet/README.md | 2 +- docs/release-instruction.md | 6 +++--- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8225f56c5..47d812b18 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,8 @@

--- -[![Report](https://goreportcard.com/badge/github.com/TrueCloudLab/frostfs-node)](https://goreportcard.com/report/github.com/TrueCloudLab/frostfs-node) -![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/TrueCloudLab/frostfs-node?sort=semver) -![License](https://img.shields.io/github/license/TrueCloudLab/frostfs-node.svg?style=popout) +[![Report](https://goreportcard.com/badge/git.frostfs.info/TrueCloudLab/frostfs-node)](https://goreportcard.com/report/git.frostfs.info/TrueCloudLab/frostfs-node) +![Release (latest)](https://git.frostfs.info/TrueCloudLab/frostfs-node/badges/release.svg) # Overview @@ -33,8 +32,8 @@ manipulate large amounts of data without paying a prohibitive price. FrostFS has a native [gRPC API](https://git.frostfs.info/TrueCloudLab/frostfs-api) and has protocol gateways for popular protocols such as [AWS -S3](https://github.com/TrueCloudLab/frostfs-s3-gw), -[HTTP](https://github.com/TrueCloudLab/frostfs-http-gw), +S3](https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw), +[HTTP](https://git.frostfs.info/TrueCloudLab/frostfs-http-gw), [FUSE](https://wikipedia.org/wiki/Filesystem_in_Userspace) and [sFTP](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol) allowing developers to integrate applications without rewriting their code. @@ -45,7 +44,7 @@ Now, we only support GNU/Linux on amd64 CPUs with AVX/AVX2 instructions. More platforms will be officially supported after release `1.0`. The latest version of frostfs-node works with frostfs-contract -[v0.16.0](https://github.com/TrueCloudLab/frostfs-contract/releases/tag/v0.16.0). +[v0.19.2](https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/tag/v0.19.2). # Building @@ -71,7 +70,7 @@ make docker/bin/frostfs- # build a specific binary ## Docker images -To make docker images suitable for use in [frostfs-dev-env](https://github.com/TrueCloudLab/frostfs-dev-env/) use: +To make docker images suitable for use in [frostfs-dev-env](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env/) use: ``` make images ``` @@ -125,7 +124,7 @@ the feature/topic you are going to implement. # Credits -FrostFS is maintained by [True Cloud Lab](https://github.com/TrueCloudLab/) with the help and +FrostFS is maintained by [True Cloud Lab](https://git.frostfs.info/TrueCloudLab/) with the help and contributions from community members. Please see [CREDITS](CREDITS.md) for details. diff --git a/cmd/frostfs-adm/docs/deploy.md b/cmd/frostfs-adm/docs/deploy.md index 87d2e47c1..b4b1ed8e4 100644 --- a/cmd/frostfs-adm/docs/deploy.md +++ b/cmd/frostfs-adm/docs/deploy.md @@ -9,8 +9,8 @@ related configuration details. To follow this guide you need: - latest released version of [neo-go](https://github.com/nspcc-dev/neo-go/releases) (v0.97.2 at the moment), -- latest released version of [frostfs-adm](https://github.com/TrueCloudLab/frostfs-node/releases) utility (v0.25.1 at the moment), -- latest released version of compiled [frostfs-contract](https://github.com/TrueCloudLab/frostfs-contract/releases) (v0.11.0 at the moment). +- latest released version of [frostfs-adm](https://git.frostfs.info/TrueCloudLab/frostfs-node/releases) utility (v0.42.9 at the moment), +- latest released version of compiled [frostfs-contract](https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases) (v0.19.2 at the moment). ## Step 1: Prepare network configuration diff --git a/config/testnet/README.md b/config/testnet/README.md index b5faf2b27..e2cda33ec 100644 --- a/config/testnet/README.md +++ b/config/testnet/README.md @@ -67,7 +67,7 @@ NEOFS_NODE_ATTRIBUTE_2=UN-LOCODE:RU LED ``` You can validate UN/LOCODE attribute in -[NeoFS LOCODE database](https://github.com/TrueCloudLab/frostfs-locode-db/releases/tag/v0.1.0) +[NeoFS LOCODE database](https://git.frostfs.info/TrueCloudLab/frostfs-locode-db/releases/tag/v0.4.0) with frostfs-cli. ``` diff --git a/docs/release-instruction.md b/docs/release-instruction.md index ec7b8cdf3..3aebc8e66 100644 --- a/docs/release-instruction.md +++ b/docs/release-instruction.md @@ -9,7 +9,7 @@ These should run successfully: * `make lint` (should not change any files); * `make fmts` (should not change any files); * `go mod tidy` (should not change any files); -* integration tests in [frostfs-devenv](https://github.com/TrueCloudLab/frostfs-devenv). +* integration tests in [frostfs-devenv](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env). ## Make release commit @@ -123,12 +123,12 @@ the release. Publish the release. ### Update FrostFS Developer Environment -Prepare pull-request in [frostfs-devenv](https://github.com/TrueCloudLab/frostfs-devenv) +Prepare pull-request in [frostfs-devenv](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env) with new versions. ### Close GitHub milestone -Look up GitHub [milestones](https://github.com/TrueCloudLab/frostfs-node/milestones) and close the release one if exists. +Look up [milestones](https://git.frostfs.info/TrueCloudLab/frostfs-node/milestones) and close the release one if exists. ### Rebuild FrostFS LOCODE database From 99be4c83a7c8ab6717ef5242c80f9ccc51d470d7 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 12 Sep 2024 10:00:28 +0300 Subject: [PATCH 1337/1943] [#1368] *: Run gofumpt Signed-off-by: Aleksey Savchuk --- pkg/services/common/ape/checker.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index f24d22124..278f6da31 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -63,7 +63,8 @@ type checkerCoreImpl struct { } func New(localOverrideStorage policyengine.LocalOverrideStorage, morphChainStorage policyengine.MorphRuleChainStorageReader, - frostFSSubjectProvider frostfsidcore.SubjectProvider, state netmap.State) CheckCore { + frostFSSubjectProvider frostfsidcore.SubjectProvider, state netmap.State, +) CheckCore { return &checkerCoreImpl{ LocalOverrideStorage: localOverrideStorage, MorphChainStorage: morphChainStorage, From 66e17f4b8e968e1d58924ee9a38b720d0df7989c Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 12 Sep 2024 10:01:27 +0300 Subject: [PATCH 1338/1943] [#1368] cli/container: Use dedicated method to list user attributes Signed-off-by: Aleksey Savchuk --- cmd/frostfs-cli/modules/container/list.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index 6d0019ec4..f01e4db4d 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -1,9 +1,6 @@ package container import ( - "strings" - - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" @@ -84,12 +81,8 @@ var listContainersCmd = &cobra.Command{ cmd.Println(cnrID.String()) if flagVarListPrintAttr { - cnr.IterateAttributes(func(key, val string) { - if !strings.HasPrefix(key, container.SysAttributePrefix) && !strings.HasPrefix(key, container.SysAttributePrefixNeoFS) { - // FIXME(@cthulhu-rider): https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/97 - // Use dedicated method to skip system attributes. - cmd.Printf(" %s: %s\n", key, val) - } + cnr.IterateUserAttributes(func(key, val string) { + cmd.Printf(" %s: %s\n", key, val) }) } } From 5f6c7cbdb102c51e4e994198f805a33084f3b9de Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 9 Sep 2024 18:37:06 +0300 Subject: [PATCH 1339/1943] [#1367] writecache: Drop bbolt DB Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 2 +- .../writecache/{cachebbolt.go => cache.go} | 15 +- pkg/local_object_storage/writecache/delete.go | 39 +--- pkg/local_object_storage/writecache/flush.go | 192 ++++-------------- .../writecache/flush_test.go | 28 +-- pkg/local_object_storage/writecache/get.go | 14 +- pkg/local_object_storage/writecache/mode.go | 39 ++-- .../writecache/mode_test.go | 8 +- .../writecache/options.go | 11 - pkg/local_object_storage/writecache/put.go | 53 +---- pkg/local_object_storage/writecache/state.go | 35 +--- .../writecache/storage.go | 61 ------ 12 files changed, 82 insertions(+), 415 deletions(-) rename pkg/local_object_storage/writecache/{cachebbolt.go => cache.go} (94%) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 97b189529..87e4e0b43 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -542,6 +542,6 @@ const ( StartedWritecacheSealAsync = "started writecache seal async" WritecacheSealCompletedAsync = "writecache seal completed successfully" FailedToSealWritecacheAsync = "failed to seal writecache async" - WritecacheShrinkSkippedNotEmpty = "writecache shrink skipped: database is not empty" + WritecacheShrinkSkippedNotEmpty = "writecache shrink skipped: not empty" BlobovniczatreeFailedToRemoveRebuildTempFile = "failed to remove rebuild temp file" ) diff --git a/pkg/local_object_storage/writecache/cachebbolt.go b/pkg/local_object_storage/writecache/cache.go similarity index 94% rename from pkg/local_object_storage/writecache/cachebbolt.go rename to pkg/local_object_storage/writecache/cache.go index f1e6a619a..ff38de407 100644 --- a/pkg/local_object_storage/writecache/cachebbolt.go +++ b/pkg/local_object_storage/writecache/cache.go @@ -2,7 +2,7 @@ package writecache import ( "context" - "os" + "fmt" "sync" "sync/atomic" @@ -27,8 +27,6 @@ type cache struct { cancel atomic.Value // wg is a wait group for flush workers. wg sync.WaitGroup - // store contains underlying database. - store // fsTree contains big files stored directly on file-system. fsTree *fstree.FSTree } @@ -67,7 +65,6 @@ func New(opts ...Option) Cache { maxCacheSize: defaultMaxCacheSize, maxBatchSize: bbolt.DefaultMaxBatchSize, maxBatchDelay: bbolt.DefaultMaxBatchDelay, - openFile: os.OpenFile, metrics: DefaultMetrics(), }, } @@ -102,13 +99,15 @@ func (c *cache) Open(_ context.Context, mod mode.Mode) error { if err != nil { return metaerr.Wrap(err) } - return metaerr.Wrap(c.initCounters()) } // Init runs necessary services. func (c *cache) Init() error { c.metrics.SetMode(mode.ConvertToComponentModeDegraded(c.mode)) + if err := c.flushAndDropBBoltDB(context.Background()); err != nil { + return fmt.Errorf("flush previous version write-cache database: %w", err) + } ctx, cancel := context.WithCancel(context.Background()) c.cancel.Store(cancel) c.runFlushLoop(ctx) @@ -132,10 +131,10 @@ func (c *cache) Close() error { defer c.modeMtx.Unlock() var err error - if c.db != nil { - err = c.db.Close() + if c.fsTree != nil { + err = c.fsTree.Close() if err != nil { - c.db = nil + c.fsTree = nil } } c.metrics.Close() diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index b1a0511ee..dda284439 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -2,7 +2,6 @@ package writecache import ( "context" - "math" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -10,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -45,46 +43,11 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { return ErrDegraded } - saddr := addr.EncodeToString() - - var dataSize int - _ = c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - dataSize = len(b.Get([]byte(saddr))) - return nil - }) - - if dataSize > 0 { - storageType = StorageTypeDB - var recordDeleted bool - err := c.db.Update(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - key := []byte(saddr) - recordDeleted = b.Get(key) != nil - err := b.Delete(key) - return err - }) - if err != nil { - return err - } - storagelog.Write(c.log, - storagelog.AddressField(saddr), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("db DELETE"), - ) - if recordDeleted { - c.objCounters.cDB.Add(math.MaxUint64) - c.estimateCacheSize() - } - deleted = true - return nil - } - storageType = StorageTypeFSTree _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err == nil { storagelog.Write(c.log, - storagelog.AddressField(saddr), + storagelog.AddressField(addr.EncodeToString()), storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree DELETE"), ) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 930ac8431..074756e32 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -4,6 +4,9 @@ import ( "bytes" "context" "errors" + "fmt" + "os" + "path/filepath" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -16,7 +19,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/mr-tron/base58" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -24,10 +26,6 @@ import ( ) const ( - // flushBatchSize is amount of keys which will be read from cache to be flushed - // to the main storage. It is used to reduce contention between cache put - // and cache persist. - flushBatchSize = 512 // defaultFlushWorkersCount is number of workers for putting objects in main storage. defaultFlushWorkersCount = 20 // defaultFlushInterval is default time interval between successive flushes. @@ -41,112 +39,11 @@ func (c *cache) runFlushLoop(ctx context.Context) { if c.disableBackgroundFlush { return } - for range c.workersCount { - c.wg.Add(1) - go c.workerFlushSmall(ctx) - } - c.wg.Add(1) go func() { c.workerFlushBig(ctx) c.wg.Done() }() - - c.wg.Add(1) - go func() { - defer c.wg.Done() - - tt := time.NewTimer(defaultFlushInterval) - defer tt.Stop() - - for { - select { - case <-tt.C: - c.flushSmallObjects(ctx) - tt.Reset(defaultFlushInterval) - c.estimateCacheSize() - case <-ctx.Done(): - return - } - } - }() -} - -func (c *cache) flushSmallObjects(ctx context.Context) { - var lastKey []byte - for { - select { - case <-ctx.Done(): - return - default: - } - - var m []objectInfo - - c.modeMtx.RLock() - if c.readOnly() { - c.modeMtx.RUnlock() - time.Sleep(time.Second) - continue - } - - // We put objects in batches of fixed size to not interfere with main put cycle a lot. - _ = c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - cs := b.Cursor() - - var k, v []byte - - if len(lastKey) == 0 { - k, v = cs.First() - } else { - k, v = cs.Seek(lastKey) - if bytes.Equal(k, lastKey) { - k, v = cs.Next() - } - } - - for ; k != nil && len(m) < flushBatchSize; k, v = cs.Next() { - if len(lastKey) == len(k) { - copy(lastKey, k) - } else { - lastKey = bytes.Clone(k) - } - - m = append(m, objectInfo{ - addr: string(k), - data: bytes.Clone(v), - }) - } - return nil - }) - - var count int - for i := range m { - obj := objectSDK.New() - if err := obj.Unmarshal(m[i].data); err != nil { - continue - } - m[i].obj = obj - - count++ - select { - case c.flushCh <- m[i]: - case <-ctx.Done(): - c.modeMtx.RUnlock() - return - } - } - - c.modeMtx.RUnlock() - if count == 0 { - break - } - - c.log.Debug(logs.WritecacheTriedToFlushItemsFromWritecache, - zap.Int("count", count), - zap.String("start", base58.Encode(lastKey))) - } } func (c *cache) workerFlushBig(ctx context.Context) { @@ -197,9 +94,6 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { err = c.flushObject(ctx, &obj, e.ObjectData, StorageTypeFSTree) if err != nil { - if ignoreErrors { - return nil - } return err } @@ -211,29 +105,6 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } -// workerFlushSmall writes small objects to the main storage. -func (c *cache) workerFlushSmall(ctx context.Context) { - defer c.wg.Done() - - var objInfo objectInfo - for { - // Give priority to direct put. - select { - case objInfo = <-c.flushCh: - case <-ctx.Done(): - return - } - - err := c.flushObject(ctx, objInfo.obj, objInfo.data, StorageTypeDB) - if err != nil { - // Error is handled in flushObject. - continue - } - - c.deleteFromDB(objInfo.addr, true) - } -} - // flushObject is used to write object directly to the main storage. func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []byte, st StorageType) error { var err error @@ -300,13 +171,33 @@ func (c *cache) Flush(ctx context.Context, ignoreErrors, seal bool) error { } func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { - if err := c.flushFSTree(ctx, ignoreErrors); err != nil { - return err + return c.flushFSTree(ctx, ignoreErrors) +} + +type batchItem struct { + data []byte + address string +} + +func (c *cache) flushAndDropBBoltDB(ctx context.Context) error { + _, err := os.Stat(filepath.Join(c.path, dbName)) + if err != nil && os.IsNotExist(err) { + return nil } + if err != nil { + return fmt.Errorf("could not check write-cache database existence: %w", err) + } + db, err := OpenDB(c.path, true, os.OpenFile, c.pageSize) + if err != nil { + return fmt.Errorf("could not open write-cache database: %w", err) + } + defer func() { + _ = db.Close() + }() var last string for { - batch, err := c.readNextDBBatch(ignoreErrors, last) + batch, err := c.readNextDBBatch(db, last) if err != nil { return err } @@ -316,32 +207,27 @@ func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { for _, item := range batch { var obj objectSDK.Object if err := obj.Unmarshal(item.data); err != nil { - c.reportFlushError(logs.FSTreeCantDecodeDBObjectAddress, item.address, metaerr.Wrap(err)) - if ignoreErrors { - continue - } - return err + return fmt.Errorf("unmarshal object from database: %w", err) } - if err := c.flushObject(ctx, &obj, item.data, StorageTypeDB); err != nil { - return err + return fmt.Errorf("flush object from database: %w", err) } - c.deleteFromDB(item.address, false) } last = batch[len(batch)-1].address } + if err := db.Close(); err != nil { + return fmt.Errorf("close write-cache database: %w", err) + } + if err := os.Remove(filepath.Join(c.path, dbName)); err != nil { + return fmt.Errorf("remove write-cache database: %w", err) + } return nil } -type batchItem struct { - data []byte - address string -} - -func (c *cache) readNextDBBatch(ignoreErrors bool, last string) ([]batchItem, error) { +func (c *cache) readNextDBBatch(db *bbolt.DB, last string) ([]batchItem, error) { const batchSize = 100 var batch []batchItem - err := c.db.View(func(tx *bbolt.Tx) error { + err := db.View(func(tx *bbolt.Tx) error { var addr oid.Address b := tx.Bucket(defaultBucket) @@ -352,11 +238,7 @@ func (c *cache) readNextDBBatch(ignoreErrors bool, last string) ([]batchItem, er continue } if err := addr.DecodeString(sa); err != nil { - c.reportFlushError(logs.FSTreeCantDecodeDBObjectAddress, sa, metaerr.Wrap(err)) - if ignoreErrors { - continue - } - return err + return fmt.Errorf("decode address from database: %w", err) } batch = append(batch, batchItem{data: bytes.Clone(data), address: sa}) diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index a637da45d..9c7e240e0 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -19,7 +19,6 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" - "go.etcd.io/bbolt" "go.uber.org/zap" ) @@ -47,31 +46,6 @@ func TestFlush(t *testing.T) { } failures := []TestFailureInjector[Option]{ - { - Desc: "db, invalid address", - InjectFn: func(t *testing.T, wc Cache) { - c := wc.(*cache) - obj := testutil.GenerateObject() - data, err := obj.Marshal() - require.NoError(t, err) - require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - return b.Put([]byte{1, 2, 3}, data) - })) - }, - }, - { - Desc: "db, invalid object", - InjectFn: func(t *testing.T, wc Cache) { - c := wc.(*cache) - require.NoError(t, c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - k := []byte(oidtest.Address().EncodeToString()) - v := []byte{1, 2, 3} - return b.Put(k, v) - })) - }, - }, { Desc: "fs, read error", InjectFn: func(t *testing.T, wc Cache) { @@ -263,7 +237,7 @@ func check(t *testing.T, mb *meta.DB, bs *blobstor.BlobStor, objects []objectPai prm.StorageID = mRes.StorageID() res, err := bs.Get(context.Background(), prm) - require.NoError(t, err) + require.NoError(t, err, objects[i].addr) require.Equal(t, objects[i].obj, res.Object) } } diff --git a/pkg/local_object_storage/writecache/get.go b/pkg/local_object_storage/writecache/get.go index bf26833bd..c0847a65f 100644 --- a/pkg/local_object_storage/writecache/get.go +++ b/pkg/local_object_storage/writecache/get.go @@ -37,11 +37,11 @@ func (c *cache) Get(ctx context.Context, addr oid.Address) (*objectSDK.Object, e return nil, ErrDegraded } - obj, err := c.getInternal(ctx, saddr, addr) + obj, err := c.getInternal(ctx, addr) return obj, metaerr.Wrap(err) } -func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) (*objectSDK.Object, error) { +func (c *cache) getInternal(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { found := false storageType := StorageTypeUndefined startedAt := time.Now() @@ -49,14 +49,6 @@ func (c *cache) getInternal(ctx context.Context, saddr string, addr oid.Address) c.metrics.Get(time.Since(startedAt), found, storageType) }() - value, err := Get(c.db, []byte(saddr)) - if err == nil { - obj := objectSDK.New() - found = true - storageType = StorageTypeDB - return obj, obj.Unmarshal(value) - } - res, err := c.fsTree.Get(ctx, common.GetPrm{Address: addr}) if err != nil { return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) @@ -87,7 +79,7 @@ func (c *cache) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, return nil, ErrDegraded } - obj, err := c.getInternal(ctx, saddr, addr) + obj, err := c.getInternal(ctx, addr) if err != nil { return nil, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 44da9b36e..d12dd603b 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -5,13 +5,12 @@ import ( "errors" "fmt" "os" - "path/filepath" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -53,7 +52,7 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode, prm setModePrm) error } } - if err := c.closeDB(prm.shrink); err != nil { + if err := c.closeStorage(ctx, prm.shrink); err != nil { return err } @@ -78,33 +77,37 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode, prm setModePrm) error return nil } -func (c *cache) closeDB(shrink bool) error { - if c.db == nil { +func (c *cache) closeStorage(ctx context.Context, shrink bool) error { + if c.fsTree == nil { return nil } if !shrink { - if err := c.db.Close(); err != nil { - return fmt.Errorf("can't close write-cache database: %w", err) + if err := c.fsTree.Close(); err != nil { + return fmt.Errorf("can't close write-cache storage: %w", err) } return nil } - var empty bool - err := c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - empty = b == nil || b.Stats().KeyN == 0 - return nil + empty := true + _, err := c.fsTree.Iterate(ctx, common.IteratePrm{ + Handler: func(common.IterationElement) error { + return errIterationCompleted + }, }) - if err != nil && !errors.Is(err, bbolt.ErrDatabaseNotOpen) { - return fmt.Errorf("failed to check DB items: %w", err) + if err != nil { + if errors.Is(err, errIterationCompleted) { + empty = false + } else { + return fmt.Errorf("failed to check write-cache items: %w", err) + } } - if err := c.db.Close(); err != nil { - return fmt.Errorf("can't close write-cache database: %w", err) + if err := c.fsTree.Close(); err != nil { + return fmt.Errorf("can't close write-cache storage: %w", err) } if empty { - err := os.Remove(filepath.Join(c.path, dbName)) + err := os.RemoveAll(c.path) if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to remove DB file: %w", err) + return fmt.Errorf("failed to remove write-cache files: %w", err) } } else { c.log.Info(logs.WritecacheShrinkSkippedNotEmpty) diff --git a/pkg/local_object_storage/writecache/mode_test.go b/pkg/local_object_storage/writecache/mode_test.go index f684c15bc..70cfe8382 100644 --- a/pkg/local_object_storage/writecache/mode_test.go +++ b/pkg/local_object_storage/writecache/mode_test.go @@ -17,14 +17,14 @@ func TestMode(t *testing.T) { WithPath(t.TempDir())) require.NoError(t, wc.Open(context.Background(), mode.DegradedReadOnly)) - require.Nil(t, wc.(*cache).db) + require.Nil(t, wc.(*cache).fsTree) require.NoError(t, wc.Init()) - require.Nil(t, wc.(*cache).db) + require.Nil(t, wc.(*cache).fsTree) require.NoError(t, wc.Close()) require.NoError(t, wc.Open(context.Background(), mode.Degraded)) - require.Nil(t, wc.(*cache).db) + require.Nil(t, wc.(*cache).fsTree) require.NoError(t, wc.Init()) - require.Nil(t, wc.(*cache).db) + require.Nil(t, wc.(*cache).fsTree) require.NoError(t, wc.Close()) } diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index 980cf9303..7845c5da9 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -1,8 +1,6 @@ package writecache import ( - "io/fs" - "os" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -42,8 +40,6 @@ type options struct { noSync bool // reportError is the function called when encountering disk errors in background workers. reportError func(string, error) - // openFile is the function called internally by bbolt to open database files. Useful for hermetic testing. - openFile func(string, int, fs.FileMode) (*os.File, error) // metrics is metrics implementation metrics Metrics // disableBackgroundFlush is for testing purposes only. @@ -155,13 +151,6 @@ func WithReportErrorFunc(f func(string, error)) Option { } } -// WithOpenFile sets the OpenFile function to use internally by bolt. Useful for hermetic testing. -func WithOpenFile(f func(string, int, fs.FileMode) (*os.File, error)) Option { - return func(o *options) { - o.openFile = f - } -} - // WithMetrics sets metrics implementation. func WithMetrics(metrics Metrics) Option { return func(o *options) { diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index ae0e8b77a..c53067bea 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -8,7 +8,6 @@ import ( storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -50,62 +49,16 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro return common.PutRes{}, ErrBigObject } - oi := objectInfo{ - addr: prm.Address.EncodeToString(), - obj: prm.Object, - data: prm.RawData, - } - - if sz <= c.smallObjectSize { - storageType = StorageTypeDB - err := c.putSmall(oi) - if err == nil { - added = true - } - return common.PutRes{}, err - } - storageType = StorageTypeFSTree - err := c.putBig(ctx, oi.addr, prm) + err := c.putBig(ctx, prm) if err == nil { added = true } return common.PutRes{}, metaerr.Wrap(err) } -// putSmall persists small objects to the write-cache database and -// pushes the to the flush workers queue. -func (c *cache) putSmall(obj objectInfo) error { - if !c.hasEnoughSpaceDB() { - return ErrOutOfSpace - } - - var newRecord bool - err := c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - key := []byte(obj.addr) - newRecord = b.Get(key) == nil - if newRecord { - return b.Put(key, obj.data) - } - return nil - }) - if err == nil { - storagelog.Write(c.log, - storagelog.AddressField(obj.addr), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("db PUT"), - ) - if newRecord { - c.objCounters.cDB.Add(1) - c.estimateCacheSize() - } - } - return err -} - // putBig writes object to FSTree and pushes it to the flush workers queue. -func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) error { +func (c *cache) putBig(ctx context.Context, prm common.PutPrm) error { if !c.hasEnoughSpaceFS() { return ErrOutOfSpace } @@ -116,7 +69,7 @@ func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) erro } storagelog.Write(c.log, - storagelog.AddressField(addr), + storagelog.AddressField(prm.Address.EncodeToString()), storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree PUT"), ) diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index d03f4a63e..e4e22f404 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -1,29 +1,18 @@ package writecache import ( - "fmt" "math" "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" - "go.etcd.io/bbolt" ) func (c *cache) estimateCacheSize() (uint64, uint64) { - dbCount := c.objCounters.DB() fsCount := c.objCounters.FS() - if fsCount > 0 { - fsCount-- // db file - } - dbSize := dbCount * c.smallObjectSize fsSize := fsCount * c.maxObjectSize - c.metrics.SetEstimateSize(dbSize, fsSize) - c.metrics.SetActualCounters(dbCount, fsCount) - return dbCount + fsCount, dbSize + fsSize -} - -func (c *cache) hasEnoughSpaceDB() bool { - return c.hasEnoughSpace(c.smallObjectSize) + c.metrics.SetEstimateSize(0, fsSize) + c.metrics.SetActualCounters(0, fsCount) + return fsCount, fsSize } func (c *cache) hasEnoughSpaceFS() bool { @@ -41,11 +30,7 @@ func (c *cache) hasEnoughSpace(objectSize uint64) bool { var _ fstree.FileCounter = &counters{} type counters struct { - cDB, cFS atomic.Uint64 -} - -func (x *counters) DB() uint64 { - return x.cDB.Load() + cFS atomic.Uint64 } func (x *counters) FS() uint64 { @@ -68,18 +53,6 @@ func (x *counters) Dec() { } func (c *cache) initCounters() error { - var inDB uint64 - err := c.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - if b != nil { - inDB = uint64(b.Stats().KeyN) - } - return nil - }) - if err != nil { - return fmt.Errorf("could not read write-cache DB counter: %w", err) - } - c.objCounters.cDB.Store(inDB) c.estimateCacheSize() return nil } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index 57021cc17..309bd2a66 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -3,7 +3,6 @@ package writecache import ( "context" "fmt" - "math" "os" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -14,16 +13,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.etcd.io/bbolt" "go.uber.org/zap" ) -// store represents persistent storage with in-memory LRU cache -// for flushed items on top of it. -type store struct { - db *bbolt.DB -} - const dbName = "small.bolt" func (c *cache) openStore(mod mode.ComponentMode) error { @@ -32,24 +24,6 @@ func (c *cache) openStore(mod mode.ComponentMode) error { return err } - c.db, err = OpenDB(c.path, mod.ReadOnly(), c.openFile, c.pageSize) - if err != nil { - return fmt.Errorf("could not open database: %w", err) - } - - c.db.MaxBatchSize = c.maxBatchSize - c.db.MaxBatchDelay = c.maxBatchDelay - - if !mod.ReadOnly() { - err = c.db.Update(func(tx *bbolt.Tx) error { - _, err := tx.CreateBucketIfNotExists(defaultBucket) - return err - }) - if err != nil { - return fmt.Errorf("could not create default bucket: %w", err) - } - } - c.fsTree = fstree.New( fstree.WithPath(c.path), fstree.WithPerm(os.ModePerm), @@ -68,41 +42,6 @@ func (c *cache) openStore(mod mode.ComponentMode) error { return nil } -func (c *cache) deleteFromDB(key string, batched bool) { - var recordDeleted bool - var err error - if batched { - err = c.db.Batch(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - key := []byte(key) - recordDeleted = b.Get(key) != nil - return b.Delete(key) - }) - } else { - err = c.db.Update(func(tx *bbolt.Tx) error { - b := tx.Bucket(defaultBucket) - key := []byte(key) - recordDeleted = b.Get(key) != nil - return b.Delete(key) - }) - } - - if err == nil { - c.metrics.Evict(StorageTypeDB) - storagelog.Write(c.log, - storagelog.AddressField(key), - storagelog.StorageTypeField(wcStorageType), - storagelog.OpField("db DELETE"), - ) - if recordDeleted { - c.objCounters.cDB.Add(math.MaxUint64) - c.estimateCacheSize() - } - } else { - c.log.Error(logs.WritecacheCantRemoveObjectsFromTheDatabase, zap.Error(err)) - } -} - func (c *cache) deleteFromDisk(ctx context.Context, addr oid.Address) { _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err != nil && !client.IsErrObjectNotFound(err) { From b142b6f48e46210235a56840c79da358120cf0cc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Sep 2024 11:01:30 +0300 Subject: [PATCH 1340/1943] [#1367] fstree: Add size to file counter FSTree file counter used by writecache. As writecache has now only one storage, so it is required to use real object size to get writecache size more accurate than `count * max_object_size`. Signed-off-by: Dmitrii Stepanov --- .../blobstor/fstree/counter.go | 61 +++++++++++++++---- .../blobstor/fstree/fstree.go | 22 ++++--- .../blobstor/fstree/fstree_test.go | 15 +++-- .../blobstor/fstree/fstree_write_generic.go | 27 +++++--- .../blobstor/fstree/fstree_write_linux.go | 42 ++++++++++--- pkg/local_object_storage/writecache/cache.go | 3 + .../writecache/options.go | 2 - pkg/local_object_storage/writecache/state.go | 41 ++----------- .../writecache/storage.go | 2 +- 9 files changed, 130 insertions(+), 85 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/counter.go b/pkg/local_object_storage/blobstor/fstree/counter.go index 718104e2e..b5dbc9e40 100644 --- a/pkg/local_object_storage/blobstor/fstree/counter.go +++ b/pkg/local_object_storage/blobstor/fstree/counter.go @@ -1,22 +1,21 @@ package fstree import ( - "math" - "sync/atomic" + "sync" ) // FileCounter used to count files in FSTree. The implementation must be thread-safe. type FileCounter interface { - Set(v uint64) - Inc() - Dec() + Set(count, size uint64) + Inc(size uint64) + Dec(size uint64) } type noopCounter struct{} -func (c *noopCounter) Set(uint64) {} -func (c *noopCounter) Inc() {} -func (c *noopCounter) Dec() {} +func (c *noopCounter) Set(uint64, uint64) {} +func (c *noopCounter) Inc(uint64) {} +func (c *noopCounter) Dec(uint64) {} func counterEnabled(c FileCounter) bool { _, noop := c.(*noopCounter) @@ -24,14 +23,50 @@ func counterEnabled(c FileCounter) bool { } type SimpleCounter struct { - v atomic.Uint64 + mtx sync.RWMutex + count uint64 + size uint64 } func NewSimpleCounter() *SimpleCounter { return &SimpleCounter{} } -func (c *SimpleCounter) Set(v uint64) { c.v.Store(v) } -func (c *SimpleCounter) Inc() { c.v.Add(1) } -func (c *SimpleCounter) Dec() { c.v.Add(math.MaxUint64) } -func (c *SimpleCounter) Value() uint64 { return c.v.Load() } +func (c *SimpleCounter) Set(count, size uint64) { + c.mtx.Lock() + defer c.mtx.Unlock() + + c.count = count + c.size = size +} + +func (c *SimpleCounter) Inc(size uint64) { + c.mtx.Lock() + defer c.mtx.Unlock() + + c.count++ + c.size += size +} + +func (c *SimpleCounter) Dec(size uint64) { + c.mtx.Lock() + defer c.mtx.Unlock() + + if c.count > 0 { + c.count-- + } else { + panic("fstree.SimpleCounter: invalid count") + } + if c.size >= size { + c.size -= size + } else { + panic("fstree.SimpleCounter: invalid size") + } +} + +func (c *SimpleCounter) CountSize() (uint64, uint64) { + c.mtx.RLock() + defer c.mtx.RUnlock() + + return c.count, c.size +} diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 02580dbfa..bf6ba51e5 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -435,32 +435,38 @@ func (t *FSTree) initFileCounter() error { return nil } - counter, err := t.countFiles() + count, size, err := t.countFiles() if err != nil { return err } - t.fileCounter.Set(counter) + t.fileCounter.Set(count, size) return nil } -func (t *FSTree) countFiles() (uint64, error) { - var counter uint64 +func (t *FSTree) countFiles() (uint64, uint64, error) { + var count, size uint64 // it is simpler to just consider every file // that is not directory as an object err := filepath.WalkDir(t.RootPath, func(_ string, d fs.DirEntry, _ error) error { - if !d.IsDir() { - counter++ + if d.IsDir() { + return nil } + count++ + info, err := d.Info() + if err != nil { + return err + } + size += uint64(info.Size()) return nil }, ) if err != nil { - return 0, fmt.Errorf("could not walk through %s directory: %w", t.RootPath, err) + return 0, 0, fmt.Errorf("could not walk through %s directory: %w", t.RootPath, err) } - return counter, nil + return count, size, nil } func (t *FSTree) ObjectsCount(ctx context.Context) (uint64, error) { diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_test.go b/pkg/local_object_storage/blobstor/fstree/fstree_test.go index 5786dfd3b..f39c7296e 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_test.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_test.go @@ -47,8 +47,9 @@ func TestObjectCounter(t *testing.T) { require.NoError(t, fst.Open(mode.ComponentReadWrite)) require.NoError(t, fst.Init()) - counterValue := counter.Value() - require.Equal(t, uint64(0), counterValue) + count, size := counter.CountSize() + require.Equal(t, uint64(0), count) + require.Equal(t, uint64(0), size) defer func() { require.NoError(t, fst.Close()) @@ -64,9 +65,6 @@ func TestObjectCounter(t *testing.T) { putPrm.Address = addr putPrm.RawData, _ = obj.Marshal() - var getPrm common.GetPrm - getPrm.Address = putPrm.Address - var delPrm common.DeletePrm delPrm.Address = addr @@ -95,8 +93,9 @@ func TestObjectCounter(t *testing.T) { require.NoError(t, eg.Wait()) - counterValue = counter.Value() - realCount, err := fst.countFiles() + count, size = counter.CountSize() + realCount, realSize, err := fst.countFiles() require.NoError(t, err) - require.Equal(t, realCount, counterValue) + require.Equal(t, realCount, count, "real %d, actual %d", realCount, count) + require.Equal(t, realSize, size, "real %d, actual %d", realSize, size) } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 8b2622885..801fc4a22 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -78,14 +78,14 @@ func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error { } if w.fileCounterEnabled { - w.fileCounter.Inc() + w.fileCounter.Inc(uint64(len(data))) var targetFileExists bool if _, e := os.Stat(p); e == nil { targetFileExists = true } err = os.Rename(tmpPath, p) if err == nil && targetFileExists { - w.fileCounter.Dec() + w.fileCounter.Dec(uint64(len(data))) } } else { err = os.Rename(tmpPath, p) @@ -110,12 +110,7 @@ func (w *genericWriter) writeFile(p string, data []byte) error { func (w *genericWriter) removeFile(p string) error { var err error if w.fileCounterEnabled { - w.fileGuard.Lock(p) - err = os.Remove(p) - w.fileGuard.Unlock(p) - if err == nil { - w.fileCounter.Dec() - } + err = w.removeWithCounter(p) } else { err = os.Remove(p) } @@ -125,3 +120,19 @@ func (w *genericWriter) removeFile(p string) error { } return err } + +func (w *genericWriter) removeWithCounter(p string) error { + w.fileGuard.Lock(p) + defer w.fileGuard.Unlock(p) + + stat, err := os.Stat(p) + if err != nil { + return err + } + + if err := os.Remove(p); err != nil { + return err + } + w.fileCounter.Dec(uint64(stat.Size())) + return nil +} diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go index efc5a3d3d..3127579ac 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "golang.org/x/sys/unix" ) @@ -18,7 +19,9 @@ type linuxWriter struct { perm uint32 flags int - counter FileCounter + fileGuard keyLock + fileCounter FileCounter + fileCounterEnabled bool } func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync bool) writer { @@ -33,11 +36,18 @@ func newSpecificWriteData(c FileCounter, root string, perm fs.FileMode, noSync b return nil } _ = unix.Close(fd) // Don't care about error. + var fileGuard keyLock = &noopKeyLock{} + fileCounterEnabled := counterEnabled(c) + if fileCounterEnabled { + fileGuard = utilSync.NewKeyLocker[string]() + } w := &linuxWriter{ - root: root, - perm: uint32(perm), - flags: flags, - counter: c, + root: root, + perm: uint32(perm), + flags: flags, + fileGuard: fileGuard, + fileCounter: c, + fileCounterEnabled: fileCounterEnabled, } return w } @@ -51,6 +61,10 @@ func (w *linuxWriter) writeData(p string, data []byte) error { } func (w *linuxWriter) writeFile(p string, data []byte) error { + if w.fileCounterEnabled { + w.fileGuard.Lock(p) + defer w.fileGuard.Unlock(p) + } fd, err := unix.Open(w.root, w.flags, w.perm) if err != nil { return err @@ -61,7 +75,7 @@ func (w *linuxWriter) writeFile(p string, data []byte) error { if n == len(data) { err = unix.Linkat(unix.AT_FDCWD, tmpPath, unix.AT_FDCWD, p, unix.AT_SYMLINK_FOLLOW) if err == nil { - w.counter.Inc() + w.fileCounter.Inc(uint64(len(data))) } if errors.Is(err, unix.EEXIST) { err = nil @@ -78,12 +92,24 @@ func (w *linuxWriter) writeFile(p string, data []byte) error { } func (w *linuxWriter) removeFile(p string) error { - err := unix.Unlink(p) + if w.fileCounterEnabled { + w.fileGuard.Lock(p) + defer w.fileGuard.Unlock(p) + } + var stat unix.Stat_t + err := unix.Stat(p, &stat) + if err != nil { + if err == unix.ENOENT { + return logicerr.Wrap(new(apistatus.ObjectNotFound)) + } + return err + } + err = unix.Unlink(p) if err != nil && err == unix.ENOENT { return logicerr.Wrap(new(apistatus.ObjectNotFound)) } if err == nil { - w.counter.Dec() + w.fileCounter.Dec(uint64(stat.Size)) } return err } diff --git a/pkg/local_object_storage/writecache/cache.go b/pkg/local_object_storage/writecache/cache.go index ff38de407..f2280f2f4 100644 --- a/pkg/local_object_storage/writecache/cache.go +++ b/pkg/local_object_storage/writecache/cache.go @@ -29,6 +29,8 @@ type cache struct { wg sync.WaitGroup // fsTree contains big files stored directly on file-system. fsTree *fstree.FSTree + // counter contains atomic counters for the number of objects stored in cache. + counter *fstree.SimpleCounter } // wcStorageType is used for write-cache operations logging. @@ -56,6 +58,7 @@ func New(opts ...Option) Cache { c := &cache{ flushCh: make(chan objectInfo), mode: mode.Disabled, + counter: fstree.NewSimpleCounter(), options: options{ log: &logger.Logger{Logger: zap.NewNop()}, diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index 7845c5da9..0643faac0 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -30,8 +30,6 @@ type options struct { // maxCacheCount is the maximum total count of all object saved in cache. // 0 (no limit) by default. maxCacheCount uint64 - // objCounters contains atomic counters for the number of objects stored in cache. - objCounters counters // maxBatchSize is the maximum batch size for the small object database. maxBatchSize int // maxBatchDelay is the maximum batch wait time for the small object database. diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index e4e22f404..748c78bcb 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -1,18 +1,10 @@ package writecache -import ( - "math" - "sync/atomic" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" -) - func (c *cache) estimateCacheSize() (uint64, uint64) { - fsCount := c.objCounters.FS() - fsSize := fsCount * c.maxObjectSize - c.metrics.SetEstimateSize(0, fsSize) - c.metrics.SetActualCounters(0, fsCount) - return fsCount, fsSize + count, size := c.counter.CountSize() + c.metrics.SetEstimateSize(0, size) + c.metrics.SetActualCounters(0, count) + return count, size } func (c *cache) hasEnoughSpaceFS() bool { @@ -27,31 +19,6 @@ func (c *cache) hasEnoughSpace(objectSize uint64) bool { return c.maxCacheSize >= size+objectSize } -var _ fstree.FileCounter = &counters{} - -type counters struct { - cFS atomic.Uint64 -} - -func (x *counters) FS() uint64 { - return x.cFS.Load() -} - -// Set implements fstree.ObjectCounter. -func (x *counters) Set(v uint64) { - x.cFS.Store(v) -} - -// Inc implements fstree.ObjectCounter. -func (x *counters) Inc() { - x.cFS.Add(1) -} - -// Dec implements fstree.ObjectCounter. -func (x *counters) Dec() { - x.cFS.Add(math.MaxUint64) -} - func (c *cache) initCounters() error { c.estimateCacheSize() return nil diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index 309bd2a66..e708a529e 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -30,7 +30,7 @@ func (c *cache) openStore(mod mode.ComponentMode) error { fstree.WithDepth(1), fstree.WithDirNameLen(1), fstree.WithNoSync(c.noSync), - fstree.WithFileCounter(&c.objCounters), + fstree.WithFileCounter(c.counter), ) if err := c.fsTree.Open(mod); err != nil { return fmt.Errorf("could not open FSTree: %w", err) From 2dd3a6f7a85800452f38ddb3127dea455b6366dc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Sep 2024 11:47:42 +0300 Subject: [PATCH 1341/1943] [#1367] fstree: Add IterateInfo method Signed-off-by: Dmitrii Stepanov --- .../blobstor/fstree/fstree.go | 75 +++++++++++++++++++ .../blobstor/fstree/metrics.go | 2 + pkg/local_object_storage/metrics/fstree.go | 4 + 3 files changed, 81 insertions(+) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index bf6ba51e5..1c60ec340 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -222,6 +222,81 @@ func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, pr return nil } +type ObjectInfo struct { + Address oid.Address + DataSize uint64 +} +type IterateInfoHandler func(ObjectInfo) error + +func (t *FSTree) IterateInfo(ctx context.Context, handler IterateInfoHandler) error { + var ( + err error + startedAt = time.Now() + ) + defer func() { + t.metrics.IterateInfo(time.Since(startedAt), err == nil) + }() + _, span := tracing.StartSpanFromContext(ctx, "FSTree.IterateInfo") + defer span.End() + + return t.iterateInfo(ctx, 0, []string{t.RootPath}, handler) +} + +func (t *FSTree) iterateInfo(ctx context.Context, depth uint64, curPath []string, handler IterateInfoHandler) error { + curName := strings.Join(curPath[1:], "") + dirPath := filepath.Join(curPath...) + entries, err := os.ReadDir(dirPath) + if err != nil { + return fmt.Errorf("read fstree dir '%s': %w", dirPath, err) + } + + isLast := depth >= t.Depth + l := len(curPath) + curPath = append(curPath, "") + + for i := range entries { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + curPath[l] = entries[i].Name() + + if !isLast && entries[i].IsDir() { + err := t.iterateInfo(ctx, depth+1, curPath, handler) + if err != nil { + return err + } + } + + if depth != t.Depth { + continue + } + + addr, err := addressFromString(curName + entries[i].Name()) + if err != nil { + continue + } + info, err := entries[i].Info() + if err != nil { + if os.IsNotExist(err) { + continue + } + return err + } + + err = handler(ObjectInfo{ + Address: addr, + DataSize: uint64(info.Size()), + }) + if err != nil { + return err + } + } + + return nil +} + func (t *FSTree) treePath(addr oid.Address) string { sAddr := stringifyAddress(addr) diff --git a/pkg/local_object_storage/blobstor/fstree/metrics.go b/pkg/local_object_storage/blobstor/fstree/metrics.go index 10de935eb..4241beec9 100644 --- a/pkg/local_object_storage/blobstor/fstree/metrics.go +++ b/pkg/local_object_storage/blobstor/fstree/metrics.go @@ -13,6 +13,7 @@ type Metrics interface { Close() Iterate(d time.Duration, success bool) + IterateInfo(d time.Duration, success bool) Delete(d time.Duration, success bool) Exists(d time.Duration, success bool) Put(d time.Duration, size int, success bool) @@ -27,6 +28,7 @@ func (m *noopMetrics) SetParentID(string) {} func (m *noopMetrics) SetMode(mode.ComponentMode) {} func (m *noopMetrics) Close() {} func (m *noopMetrics) Iterate(time.Duration, bool) {} +func (m *noopMetrics) IterateInfo(time.Duration, bool) {} func (m *noopMetrics) Delete(time.Duration, bool) {} func (m *noopMetrics) Exists(time.Duration, bool) {} func (m *noopMetrics) Put(time.Duration, int, bool) {} diff --git a/pkg/local_object_storage/metrics/fstree.go b/pkg/local_object_storage/metrics/fstree.go index 76822ac2c..d93363fa3 100644 --- a/pkg/local_object_storage/metrics/fstree.go +++ b/pkg/local_object_storage/metrics/fstree.go @@ -38,6 +38,10 @@ func (m *fstreeMetrics) Iterate(d time.Duration, success bool) { m.m.MethodDuration(m.shardID, m.path, "Iterate", d, success) } +func (m *fstreeMetrics) IterateInfo(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "IterateInfo", d, success) +} + func (m *fstreeMetrics) Delete(d time.Duration, success bool) { m.m.MethodDuration(m.shardID, m.path, "Delete", d, success) } From 8a6e3025a07d9c4d80a6252f4ee8bb0e0aa2021d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Sep 2024 11:49:17 +0300 Subject: [PATCH 1342/1943] [#1367] writecache: Flush from FSTree concurrently Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + pkg/local_object_storage/writecache/cache.go | 7 +-- pkg/local_object_storage/writecache/flush.go | 65 +++++++++++++++++--- 3 files changed, 62 insertions(+), 11 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 87e4e0b43..7aef6873e 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -544,4 +544,5 @@ const ( FailedToSealWritecacheAsync = "failed to seal writecache async" WritecacheShrinkSkippedNotEmpty = "writecache shrink skipped: not empty" BlobovniczatreeFailedToRemoveRebuildTempFile = "failed to remove rebuild temp file" + WritecacheCantGetObject = "can't get an object from fstree" ) diff --git a/pkg/local_object_storage/writecache/cache.go b/pkg/local_object_storage/writecache/cache.go index f2280f2f4..b298f812a 100644 --- a/pkg/local_object_storage/writecache/cache.go +++ b/pkg/local_object_storage/writecache/cache.go @@ -10,7 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.uber.org/zap" ) @@ -37,9 +37,8 @@ type cache struct { const wcStorageType = "write-cache" type objectInfo struct { - addr string - data []byte - obj *objectSDK.Object + addr oid.Address + size uint64 } const ( diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 074756e32..d06896ed5 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -13,10 +13,12 @@ import ( objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -29,7 +31,7 @@ const ( // defaultFlushWorkersCount is number of workers for putting objects in main storage. defaultFlushWorkersCount = 20 // defaultFlushInterval is default time interval between successive flushes. - defaultFlushInterval = time.Second + defaultFlushInterval = 10 * time.Second ) var errIterationCompleted = errors.New("iteration completed") @@ -41,23 +43,41 @@ func (c *cache) runFlushLoop(ctx context.Context) { } c.wg.Add(1) go func() { - c.workerFlushBig(ctx) - c.wg.Done() + defer c.wg.Done() + c.pushToFlushQueue(ctx) }() + + for range c.workersCount { + c.wg.Add(1) + go c.workerFlush(ctx) + } } -func (c *cache) workerFlushBig(ctx context.Context) { - tick := time.NewTicker(defaultFlushInterval * 10) +func (c *cache) pushToFlushQueue(ctx context.Context) { + tick := time.NewTicker(defaultFlushInterval) for { select { case <-tick.C: c.modeMtx.RLock() if c.readOnly() || c.noMetabase() { c.modeMtx.RUnlock() - break + continue } - _ = c.flushFSTree(ctx, true) + err := c.fsTree.IterateInfo(ctx, func(oi fstree.ObjectInfo) error { + select { + case c.flushCh <- objectInfo{ + addr: oi.Address, + size: oi.DataSize, + }: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) + if err != nil { + c.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, zap.Error(err)) + } c.modeMtx.RUnlock() case <-ctx.Done(): @@ -66,6 +86,37 @@ func (c *cache) workerFlushBig(ctx context.Context) { } } +func (c *cache) workerFlush(ctx context.Context) { + defer c.wg.Done() + + var objInfo objectInfo + for { + select { + case objInfo = <-c.flushCh: + case <-ctx.Done(): + return + } + + res, err := c.fsTree.Get(ctx, common.GetPrm{ + Address: objInfo.addr, + }) + if err != nil { + if !errors.As(err, new(*apistatus.ObjectNotFound)) { + c.reportFlushError(logs.WritecacheCantGetObject, objInfo.addr.EncodeToString(), metaerr.Wrap(err)) + } + continue + } + + err = c.flushObject(ctx, res.Object, res.RawData, StorageTypeFSTree) + if err != nil { + // Error is handled in flushObject. + continue + } + + c.deleteFromDisk(ctx, objInfo.addr) + } +} + func (c *cache) reportFlushError(msg string, addr string, err error) { if c.reportError != nil { c.reportError(msg, err) From e39378b1c36d0d00864c4f5e7fcab44975ce506d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Sep 2024 12:56:29 +0300 Subject: [PATCH 1343/1943] [#1367] writecache: Add background flushing objects limiter To limit memory usage by background flush. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 + cmd/frostfs-node/config/engine/config_test.go | 2 + .../config/engine/shard/writecache/config.go | 18 +++++ config/example/node.env | 1 + config/example/node.json | 3 +- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 23 +++--- pkg/local_object_storage/writecache/cache.go | 1 + pkg/local_object_storage/writecache/flush.go | 61 ++++++++++------ .../writecache/limiter.go | 70 +++++++++++++++++++ .../writecache/limiter_test.go | 27 +++++++ .../writecache/options.go | 9 +++ 12 files changed, 184 insertions(+), 35 deletions(-) create mode 100644 pkg/local_object_storage/writecache/limiter.go create mode 100644 pkg/local_object_storage/writecache/limiter_test.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 16f49a082..ef2752872 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -154,6 +154,7 @@ type shardCfg struct { countLimit uint64 noSync bool pageSize int + flushSizeLimit uint64 } piloramaCfg struct { @@ -278,6 +279,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc.sizeLimit = writeCacheCfg.SizeLimit() wc.countLimit = writeCacheCfg.CountLimit() wc.noSync = writeCacheCfg.NoSync() + wc.flushSizeLimit = writeCacheCfg.MaxFlushingObjectsSize() } } @@ -865,6 +867,7 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { writecache.WithMaxBatchSize(wcRead.maxBatchSize), writecache.WithMaxBatchDelay(wcRead.maxBatchDelay), writecache.WithPageSize(wcRead.pageSize), + writecache.WithFlushSizeLimit(wcRead.flushSizeLimit), writecache.WithMaxObjectSize(wcRead.maxObjSize), writecache.WithSmallObjectSize(wcRead.smallObjectSize), writecache.WithFlushWorkersCount(wcRead.flushWorkerCount), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index ef6bf7f74..b952aca4c 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -79,6 +79,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 3221225472, wc.SizeLimit()) require.EqualValues(t, 4096, wc.BoltDB().PageSize()) require.EqualValues(t, 49, wc.CountLimit()) + require.EqualValues(t, uint64(100), wc.MaxFlushingObjectsSize()) require.Equal(t, "tmp/0/meta", meta.Path()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) @@ -136,6 +137,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 4294967296, wc.SizeLimit()) require.EqualValues(t, 0, wc.BoltDB().PageSize()) require.EqualValues(t, writecacheconfig.CountLimitDefault, wc.CountLimit()) + require.EqualValues(t, writecacheconfig.MaxFlushingObjectsSizeDefault, wc.MaxFlushingObjectsSize()) require.Equal(t, "tmp/1/meta", meta.Path()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) diff --git a/cmd/frostfs-node/config/engine/shard/writecache/config.go b/cmd/frostfs-node/config/engine/shard/writecache/config.go index bfe8144df..5a069e99f 100644 --- a/cmd/frostfs-node/config/engine/shard/writecache/config.go +++ b/cmd/frostfs-node/config/engine/shard/writecache/config.go @@ -24,6 +24,8 @@ const ( // CountLimitDefault is a default write-cache count limit. CountLimitDefault = 0 + + MaxFlushingObjectsSizeDefault = 128 << 20 ) // From wraps config section into Config. @@ -145,3 +147,19 @@ func (x *Config) NoSync() bool { func (x *Config) BoltDB() *boltdbconfig.Config { return (*boltdbconfig.Config)(x) } + +// MaxFlushingObjectsSize returns the value of "max_flushing_objects_size" config parameter. +// +// Returns MaxFlushingObjectsSizeDefault if the value is not a positive number. +func (x *Config) MaxFlushingObjectsSize() uint64 { + s := config.SizeInBytesSafe( + (*config.Config)(x), + "max_flushing_objects_size", + ) + + if s > 0 { + return s + } + + return MaxFlushingObjectsSizeDefault +} diff --git a/config/example/node.env b/config/example/node.env index 82553745e..c3fa85c13 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -106,6 +106,7 @@ FROSTFS_STORAGE_SHARD_0_WRITECACHE_FLUSH_WORKER_COUNT=30 FROSTFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472 FROSTFS_STORAGE_SHARD_0_WRITECACHE_PAGE_SIZE=4096 FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_COUNT=49 +FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_FLUSHING_OBJECTS_SIZE=100 ### Metabase config FROSTFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644 diff --git a/config/example/node.json b/config/example/node.json index da108c692..d7187250b 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -149,7 +149,8 @@ "flush_worker_count": 30, "capacity": 3221225472, "page_size": 4096, - "max_object_count": 49 + "max_object_count": 49, + "max_flushing_objects_size": 100 }, "metabase": { "path": "tmp/0/meta", diff --git a/config/example/node.yaml b/config/example/node.yaml index a79f48226..776b22bd0 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -172,6 +172,7 @@ storage: capacity: 3221225472 # approximate write-cache total size, bytes max_object_count: 49 page_size: 4k + max_flushing_objects_size: 100b metabase: path: tmp/0/meta # metabase path diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 5bf35cd65..c83828978 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -293,17 +293,18 @@ writecache: page_size: '4k' ``` -| Parameter | Type | Default value | Description | -|----------------------|------------|---------------|-------------------------------------------------------------------------------------------------------------------------------| -| `path` | `string` | | Path to the metabase file. | -| `capacity` | `size` | `1G` | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | -| `max_object_count` | `int` | unrestricted | Approximate maximum objects count in the writecache. If the writecache is full, objects are written to the blobstor directly. | -| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | -| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. | -| `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | -| `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | -| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | -| `page_size` | `size` | `0` | Page size overrides the default OS page size for small objects storage. Does not affect the existing storage. | +| Parameter | Type | Default value | Description | +| --------------------------- | ---------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `path` | `string` | | Path to the metabase file. | +| `capacity` | `size` | `1G` | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | +| `max_object_count` | `int` | unrestricted | Approximate maximum objects count in the writecache. If the writecache is full, objects are written to the blobstor directly. | +| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | +| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. | +| `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | +| `max_flushing_objects_size` | `size` | `512M` | Max total size of background flushing objects. | +| `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | +| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | +| `page_size` | `size` | `0` | Page size overrides the default OS page size for small objects storage. Does not affect the existing storage. | # `node` section diff --git a/pkg/local_object_storage/writecache/cache.go b/pkg/local_object_storage/writecache/cache.go index b298f812a..f0f10d8b5 100644 --- a/pkg/local_object_storage/writecache/cache.go +++ b/pkg/local_object_storage/writecache/cache.go @@ -68,6 +68,7 @@ func New(opts ...Option) Cache { maxBatchSize: bbolt.DefaultMaxBatchSize, maxBatchDelay: bbolt.DefaultMaxBatchDelay, metrics: DefaultMetrics(), + flushSizeLimit: defaultFlushWorkersCount * defaultMaxObjectSize, }, } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index d06896ed5..5d5fc13ab 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -18,7 +18,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -41,19 +41,25 @@ func (c *cache) runFlushLoop(ctx context.Context) { if c.disableBackgroundFlush { return } + fl := newFlushLimiter(c.flushSizeLimit) c.wg.Add(1) go func() { defer c.wg.Done() - c.pushToFlushQueue(ctx) + c.pushToFlushQueue(ctx, fl) }() for range c.workersCount { c.wg.Add(1) - go c.workerFlush(ctx) + go c.workerFlush(ctx, fl) } } -func (c *cache) pushToFlushQueue(ctx context.Context) { +func (c *cache) pushToFlushQueue(ctx context.Context, fl *flushLimiter) { + stopf := context.AfterFunc(ctx, func() { + fl.close() + }) + defer stopf() + tick := time.NewTicker(defaultFlushInterval) for { select { @@ -65,6 +71,9 @@ func (c *cache) pushToFlushQueue(ctx context.Context) { } err := c.fsTree.IterateInfo(ctx, func(oi fstree.ObjectInfo) error { + if err := fl.acquire(oi.DataSize); err != nil { + return err + } select { case c.flushCh <- objectInfo{ addr: oi.Address, @@ -72,6 +81,7 @@ func (c *cache) pushToFlushQueue(ctx context.Context) { }: return nil case <-ctx.Done(): + fl.release(oi.DataSize) return ctx.Err() } }) @@ -86,37 +96,42 @@ func (c *cache) pushToFlushQueue(ctx context.Context) { } } -func (c *cache) workerFlush(ctx context.Context) { +func (c *cache) workerFlush(ctx context.Context, fl *flushLimiter) { defer c.wg.Done() var objInfo objectInfo for { select { case objInfo = <-c.flushCh: + c.flushIfAnObjectExistsWorker(ctx, objInfo, fl) case <-ctx.Done(): return } - - res, err := c.fsTree.Get(ctx, common.GetPrm{ - Address: objInfo.addr, - }) - if err != nil { - if !errors.As(err, new(*apistatus.ObjectNotFound)) { - c.reportFlushError(logs.WritecacheCantGetObject, objInfo.addr.EncodeToString(), metaerr.Wrap(err)) - } - continue - } - - err = c.flushObject(ctx, res.Object, res.RawData, StorageTypeFSTree) - if err != nil { - // Error is handled in flushObject. - continue - } - - c.deleteFromDisk(ctx, objInfo.addr) } } +func (c *cache) flushIfAnObjectExistsWorker(ctx context.Context, objInfo objectInfo, fl *flushLimiter) { + defer fl.release(objInfo.size) + + res, err := c.fsTree.Get(ctx, common.GetPrm{ + Address: objInfo.addr, + }) + if err != nil { + if !client.IsErrObjectNotFound(err) { + c.reportFlushError(logs.WritecacheCantGetObject, objInfo.addr.EncodeToString(), metaerr.Wrap(err)) + } + return + } + + err = c.flushObject(ctx, res.Object, res.RawData, StorageTypeFSTree) + if err != nil { + // Error is handled in flushObject. + return + } + + c.deleteFromDisk(ctx, objInfo.addr) +} + func (c *cache) reportFlushError(msg string, addr string, err error) { if c.reportError != nil { c.reportError(msg, err) diff --git a/pkg/local_object_storage/writecache/limiter.go b/pkg/local_object_storage/writecache/limiter.go new file mode 100644 index 000000000..ddc4101be --- /dev/null +++ b/pkg/local_object_storage/writecache/limiter.go @@ -0,0 +1,70 @@ +package writecache + +import ( + "errors" + "sync" +) + +var errLimiterClosed = errors.New("acquire failed: limiter closed") + +// flushLimiter is used to limit the total size of objects +// being flushed to blobstore at the same time. This is a necessary +// limitation so that the flushing process does not have +// a strong impact on user requests. +type flushLimiter struct { + count, size uint64 + maxSize uint64 + cond *sync.Cond + closed bool +} + +func newFlushLimiter(maxSize uint64) *flushLimiter { + return &flushLimiter{ + maxSize: maxSize, + cond: sync.NewCond(&sync.Mutex{}), + } +} + +func (l *flushLimiter) acquire(size uint64) error { + l.cond.L.Lock() + defer l.cond.L.Unlock() + + // it is allowed to overflow maxSize to allow flushing objects with size > maxSize + for l.count > 0 && l.size+size > l.maxSize && !l.closed { + l.cond.Wait() + if l.closed { + return errLimiterClosed + } + } + l.count++ + l.size += size + return nil +} + +func (l *flushLimiter) release(size uint64) { + l.cond.L.Lock() + defer l.cond.L.Unlock() + + if l.size >= size { + l.size -= size + } else { + panic("flushLimiter: invalid size") + } + + if l.count > 0 { + l.count-- + } else { + panic("flushLimiter: invalid count") + } + + l.cond.Broadcast() +} + +func (l *flushLimiter) close() { + l.cond.L.Lock() + defer l.cond.L.Unlock() + + l.closed = true + + l.cond.Broadcast() +} diff --git a/pkg/local_object_storage/writecache/limiter_test.go b/pkg/local_object_storage/writecache/limiter_test.go new file mode 100644 index 000000000..db99b203a --- /dev/null +++ b/pkg/local_object_storage/writecache/limiter_test.go @@ -0,0 +1,27 @@ +package writecache + +import ( + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +func TestLimiter(t *testing.T) { + var maxSize uint64 = 10 + var single uint64 = 3 + l := newFlushLimiter(uint64(maxSize)) + var currSize atomic.Int64 + var eg errgroup.Group + for i := 0; i < 10_000; i++ { + eg.Go(func() error { + defer l.release(single) + defer currSize.Add(-1) + l.acquire(single) + require.True(t, currSize.Add(1) <= 3) + return nil + }) + } + require.NoError(t, eg.Wait()) +} diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index 0643faac0..edbb3d422 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -44,6 +44,8 @@ type options struct { disableBackgroundFlush bool // pageSize is bbolt's page size config value pageSize int + // flushSizeLimit is total size of flushing objects. + flushSizeLimit uint64 } // WithLogger sets logger. @@ -169,3 +171,10 @@ func WithPageSize(s int) Option { o.pageSize = s } } + +// WithFlushSizeLimit sets flush size limit. +func WithFlushSizeLimit(v uint64) Option { + return func(o *options) { + o.flushSizeLimit = v + } +} From 25d2ae8aaf22c12e9e625b0433f84a49f5f22b39 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Sep 2024 13:12:17 +0300 Subject: [PATCH 1344/1943] [#1367] writecache: Drop BBolt related config variables Signed-off-by: Dmitrii Stepanov --- .../internal/writecache/inspect.go | 2 +- cmd/frostfs-lens/internal/writecache/list.go | 2 +- cmd/frostfs-node/config.go | 12 ----- cmd/frostfs-node/config/engine/config_test.go | 4 -- .../config/engine/shard/writecache/config.go | 25 ----------- docs/storage-node-configuration.md | 6 --- .../writecache/benchmark/writecache_test.go | 1 - pkg/local_object_storage/writecache/cache.go | 21 ++++----- pkg/local_object_storage/writecache/flush.go | 2 +- .../writecache/flush_test.go | 13 +++--- .../writecache/options.go | 44 ------------------- pkg/local_object_storage/writecache/util.go | 3 +- 12 files changed, 17 insertions(+), 118 deletions(-) diff --git a/cmd/frostfs-lens/internal/writecache/inspect.go b/cmd/frostfs-lens/internal/writecache/inspect.go index 63c669a35..afc986c8b 100644 --- a/cmd/frostfs-lens/internal/writecache/inspect.go +++ b/cmd/frostfs-lens/internal/writecache/inspect.go @@ -25,7 +25,7 @@ func init() { func inspectFunc(cmd *cobra.Command, _ []string) { var data []byte - db, err := writecache.OpenDB(vPath, true, os.OpenFile, 0) + db, err := writecache.OpenDB(vPath, true, os.OpenFile) common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) defer db.Close() diff --git a/cmd/frostfs-lens/internal/writecache/list.go b/cmd/frostfs-lens/internal/writecache/list.go index 9c8fa6138..bcbae0ec9 100644 --- a/cmd/frostfs-lens/internal/writecache/list.go +++ b/cmd/frostfs-lens/internal/writecache/list.go @@ -31,7 +31,7 @@ func listFunc(cmd *cobra.Command, _ []string) { return err } - db, err := writecache.OpenDB(vPath, true, os.OpenFile, 0) + db, err := writecache.OpenDB(vPath, true, os.OpenFile) common.ExitOnErr(cmd, common.Errf("could not open write-cache db: %w", err)) defer db.Close() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index ef2752872..cdfabdebd 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -145,15 +145,11 @@ type shardCfg struct { writecacheCfg struct { enabled bool path string - maxBatchSize int - maxBatchDelay time.Duration - smallObjectSize uint64 maxObjSize uint64 flushWorkerCount int sizeLimit uint64 countLimit uint64 noSync bool - pageSize int flushSizeLimit uint64 } @@ -270,11 +266,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc.enabled = true wc.path = writeCacheCfg.Path() - wc.maxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize() - wc.maxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay() - wc.pageSize = writeCacheCfg.BoltDB().PageSize() wc.maxObjSize = writeCacheCfg.MaxObjectSize() - wc.smallObjectSize = writeCacheCfg.SmallObjectSize() wc.flushWorkerCount = writeCacheCfg.WorkerCount() wc.sizeLimit = writeCacheCfg.SizeLimit() wc.countLimit = writeCacheCfg.CountLimit() @@ -864,12 +856,8 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { if wcRead := shCfg.writecacheCfg; wcRead.enabled { writeCacheOpts = append(writeCacheOpts, writecache.WithPath(wcRead.path), - writecache.WithMaxBatchSize(wcRead.maxBatchSize), - writecache.WithMaxBatchDelay(wcRead.maxBatchDelay), - writecache.WithPageSize(wcRead.pageSize), writecache.WithFlushSizeLimit(wcRead.flushSizeLimit), writecache.WithMaxObjectSize(wcRead.maxObjSize), - writecache.WithSmallObjectSize(wcRead.smallObjectSize), writecache.WithFlushWorkersCount(wcRead.flushWorkerCount), writecache.WithMaxCacheSize(wcRead.sizeLimit), writecache.WithMaxCacheCount(wcRead.countLimit), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index b952aca4c..19ad0e7ac 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -73,11 +73,9 @@ func TestEngineSection(t *testing.T) { require.Equal(t, true, wc.NoSync()) require.Equal(t, "tmp/0/cache", wc.Path()) - require.EqualValues(t, 16384, wc.SmallObjectSize()) require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 3221225472, wc.SizeLimit()) - require.EqualValues(t, 4096, wc.BoltDB().PageSize()) require.EqualValues(t, 49, wc.CountLimit()) require.EqualValues(t, uint64(100), wc.MaxFlushingObjectsSize()) @@ -131,11 +129,9 @@ func TestEngineSection(t *testing.T) { require.Equal(t, false, wc.NoSync()) require.Equal(t, "tmp/1/cache", wc.Path()) - require.EqualValues(t, 16384, wc.SmallObjectSize()) require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 4294967296, wc.SizeLimit()) - require.EqualValues(t, 0, wc.BoltDB().PageSize()) require.EqualValues(t, writecacheconfig.CountLimitDefault, wc.CountLimit()) require.EqualValues(t, writecacheconfig.MaxFlushingObjectsSizeDefault, wc.MaxFlushingObjectsSize()) diff --git a/cmd/frostfs-node/config/engine/shard/writecache/config.go b/cmd/frostfs-node/config/engine/shard/writecache/config.go index 5a069e99f..6fff0308b 100644 --- a/cmd/frostfs-node/config/engine/shard/writecache/config.go +++ b/cmd/frostfs-node/config/engine/shard/writecache/config.go @@ -2,7 +2,6 @@ package writecacheconfig import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" - boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb" ) // Config is a wrapper over the config section @@ -10,9 +9,6 @@ import ( type Config config.Config const ( - // SmallSizeDefault is a default size of small objects. - SmallSizeDefault = 32 << 10 - // MaxSizeDefault is a default value of the object payload size limit. MaxSizeDefault = 64 << 20 @@ -56,22 +52,6 @@ func (x *Config) Path() string { return p } -// SmallObjectSize returns the value of "small_object_size" config parameter. -// -// Returns SmallSizeDefault if the value is not a positive number. -func (x *Config) SmallObjectSize() uint64 { - s := config.SizeInBytesSafe( - (*config.Config)(x), - "small_object_size", - ) - - if s > 0 { - return s - } - - return SmallSizeDefault -} - // MaxObjectSize returns the value of "max_object_size" config parameter. // // Returns MaxSizeDefault if the value is not a positive number. @@ -143,11 +123,6 @@ func (x *Config) NoSync() bool { return config.BoolSafe((*config.Config)(x), "no_sync") } -// BoltDB returns config instance for querying bolt db specific parameters. -func (x *Config) BoltDB() *boltdbconfig.Config { - return (*boltdbconfig.Config)(x) -} - // MaxFlushingObjectsSize returns the value of "max_flushing_objects_size" config parameter. // // Returns MaxFlushingObjectsSizeDefault if the value is not a positive number. diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index c83828978..c74695e2b 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -287,10 +287,8 @@ writecache: enabled: true path: /path/to/writecache capacity: 4294967296 - small_object_size: 16384 max_object_size: 134217728 flush_worker_count: 30 - page_size: '4k' ``` | Parameter | Type | Default value | Description | @@ -298,13 +296,9 @@ writecache: | `path` | `string` | | Path to the metabase file. | | `capacity` | `size` | `1G` | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | | `max_object_count` | `int` | unrestricted | Approximate maximum objects count in the writecache. If the writecache is full, objects are written to the blobstor directly. | -| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | | `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. | | `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | | `max_flushing_objects_size` | `size` | `512M` | Max total size of background flushing objects. | -| `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | -| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | -| `page_size` | `size` | `0` | Page size overrides the default OS page size for small objects storage. Does not affect the existing storage. | # `node` section diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index 4da9a26d7..79ab7d9c6 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -118,6 +118,5 @@ func newCache(b *testing.B) writecache.Cache { writecache.WithBlobstor(bs), writecache.WithMetabase(testMetabase{}), writecache.WithMaxCacheSize(256<<30), - writecache.WithSmallObjectSize(128<<10), ) } diff --git a/pkg/local_object_storage/writecache/cache.go b/pkg/local_object_storage/writecache/cache.go index f0f10d8b5..341071dc6 100644 --- a/pkg/local_object_storage/writecache/cache.go +++ b/pkg/local_object_storage/writecache/cache.go @@ -11,7 +11,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.etcd.io/bbolt" "go.uber.org/zap" ) @@ -42,9 +41,8 @@ type objectInfo struct { } const ( - defaultMaxObjectSize = 64 * 1024 * 1024 // 64 MiB - defaultSmallObjectSize = 32 * 1024 // 32 KiB - defaultMaxCacheSize = 1 << 30 // 1 GiB + defaultMaxObjectSize = 64 * 1024 * 1024 // 64 MiB + defaultMaxCacheSize = 1 << 30 // 1 GiB ) var ( @@ -60,15 +58,12 @@ func New(opts ...Option) Cache { counter: fstree.NewSimpleCounter(), options: options{ - log: &logger.Logger{Logger: zap.NewNop()}, - maxObjectSize: defaultMaxObjectSize, - smallObjectSize: defaultSmallObjectSize, - workersCount: defaultFlushWorkersCount, - maxCacheSize: defaultMaxCacheSize, - maxBatchSize: bbolt.DefaultMaxBatchSize, - maxBatchDelay: bbolt.DefaultMaxBatchDelay, - metrics: DefaultMetrics(), - flushSizeLimit: defaultFlushWorkersCount * defaultMaxObjectSize, + log: &logger.Logger{Logger: zap.NewNop()}, + maxObjectSize: defaultMaxObjectSize, + workersCount: defaultFlushWorkersCount, + maxCacheSize: defaultMaxCacheSize, + metrics: DefaultMetrics(), + flushSizeLimit: defaultFlushWorkersCount * defaultMaxObjectSize, }, } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 5d5fc13ab..10e4d68f0 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -253,7 +253,7 @@ func (c *cache) flushAndDropBBoltDB(ctx context.Context) error { if err != nil { return fmt.Errorf("could not check write-cache database existence: %w", err) } - db, err := OpenDB(c.path, true, os.OpenFile, c.pageSize) + db, err := OpenDB(c.path, true, os.OpenFile) if err != nil { return fmt.Errorf("could not open write-cache database: %w", err) } diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 9c7e240e0..59a4e4895 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -25,12 +25,11 @@ import ( func TestFlush(t *testing.T) { testlogger := test.NewLogger(t) - createCacheFn := func(t *testing.T, smallSize uint64, mb *meta.DB, bs MainStorage, opts ...Option) Cache { + createCacheFn := func(t *testing.T, mb *meta.DB, bs MainStorage, opts ...Option) Cache { return New( append([]Option{ WithLogger(testlogger), WithPath(filepath.Join(t.TempDir(), "writecache")), - WithSmallObjectSize(smallSize), WithMetabase(mb), WithBlobstor(bs), WithDisableBackgroundFlush(), @@ -92,7 +91,6 @@ const ( type CreateCacheFunc[Option any] func( t *testing.T, - smallSize uint64, meta *meta.DB, bs MainStorage, opts ...Option, @@ -115,7 +113,7 @@ func runFlushTest[Option any]( failures ...TestFailureInjector[Option], ) { t.Run("no errors", func(t *testing.T) { - wc, bs, mb := newCache(t, createCacheFn, smallSize) + wc, bs, mb := newCache(t, createCacheFn) defer func() { require.NoError(t, wc.Close()) }() objects := putObjects(t, wc) @@ -128,7 +126,7 @@ func runFlushTest[Option any]( }) t.Run("flush on moving to degraded mode", func(t *testing.T) { - wc, bs, mb := newCache(t, createCacheFn, smallSize) + wc, bs, mb := newCache(t, createCacheFn) defer func() { require.NoError(t, wc.Close()) }() objects := putObjects(t, wc) @@ -146,7 +144,7 @@ func runFlushTest[Option any]( for _, f := range failures { t.Run(f.Desc, func(t *testing.T) { errCountOpt, errCount := errCountOption() - wc, bs, mb := newCache(t, createCacheFn, smallSize, errCountOpt) + wc, bs, mb := newCache(t, createCacheFn, errCountOpt) defer func() { require.NoError(t, wc.Close()) }() objects := putObjects(t, wc) f.InjectFn(t, wc) @@ -168,7 +166,6 @@ func runFlushTest[Option any]( func newCache[Option any]( t *testing.T, createCacheFn CreateCacheFunc[Option], - smallSize uint64, opts ...Option, ) (Cache, *blobstor.BlobStor, *meta.DB) { dir := t.TempDir() @@ -189,7 +186,7 @@ func newCache[Option any]( require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) require.NoError(t, bs.Init()) - wc := createCacheFn(t, smallSize, mb, bs, opts...) + wc := createCacheFn(t, mb, bs, opts...) require.NoError(t, wc.Open(context.Background(), mode.ReadWrite)) require.NoError(t, wc.Init()) diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index edbb3d422..66ac7805c 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -1,8 +1,6 @@ package writecache import ( - "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -20,8 +18,6 @@ type options struct { metabase Metabase // maxObjectSize is the maximum size of the object stored in the write-cache. maxObjectSize uint64 - // smallObjectSize is the maximum size of the object stored in the database. - smallObjectSize uint64 // workersCount is the number of workers flushing objects in parallel. workersCount int // maxCacheSize is the maximum total size of all objects saved in cache (DB + FS). @@ -30,10 +26,6 @@ type options struct { // maxCacheCount is the maximum total count of all object saved in cache. // 0 (no limit) by default. maxCacheCount uint64 - // maxBatchSize is the maximum batch size for the small object database. - maxBatchSize int - // maxBatchDelay is the maximum batch wait time for the small object database. - maxBatchDelay time.Duration // noSync is true iff FSTree allows unsynchronized writes. noSync bool // reportError is the function called when encountering disk errors in background workers. @@ -42,8 +34,6 @@ type options struct { metrics Metrics // disableBackgroundFlush is for testing purposes only. disableBackgroundFlush bool - // pageSize is bbolt's page size config value - pageSize int // flushSizeLimit is total size of flushing objects. flushSizeLimit uint64 } @@ -85,15 +75,6 @@ func WithMaxObjectSize(sz uint64) Option { } } -// WithSmallObjectSize sets maximum object size to be stored in write-cache. -func WithSmallObjectSize(sz uint64) Option { - return func(o *options) { - if sz > 0 { - o.smallObjectSize = sz - } - } -} - func WithFlushWorkersCount(c int) Option { return func(o *options) { if c > 0 { @@ -116,24 +97,6 @@ func WithMaxCacheCount(v uint64) Option { } } -// WithMaxBatchSize sets max batch size for the small object database. -func WithMaxBatchSize(sz int) Option { - return func(o *options) { - if sz > 0 { - o.maxBatchSize = sz - } - } -} - -// WithMaxBatchDelay sets max batch delay for the small object database. -func WithMaxBatchDelay(d time.Duration) Option { - return func(o *options) { - if d > 0 { - o.maxBatchDelay = d - } - } -} - // WithNoSync sets an option to allow returning to caller on PUT before write is persisted. // Note, that we use this flag for FSTree only and DO NOT use it for a bolt DB because // we cannot yet properly handle the corrupted database during the startup. This SHOULD NOT @@ -165,13 +128,6 @@ func WithDisableBackgroundFlush() Option { } } -// WithPageSize sets bbolt's page size. -func WithPageSize(s int) Option { - return func(o *options) { - o.pageSize = s - } -} - // WithFlushSizeLimit sets flush size limit. func WithFlushSizeLimit(v uint64) Option { return func(o *options) { diff --git a/pkg/local_object_storage/writecache/util.go b/pkg/local_object_storage/writecache/util.go index ad3b443f3..0ed4a954e 100644 --- a/pkg/local_object_storage/writecache/util.go +++ b/pkg/local_object_storage/writecache/util.go @@ -10,12 +10,11 @@ import ( ) // OpenDB opens BoltDB instance for write-cache. Opens in read-only mode if ro is true. -func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error), pageSize int) (*bbolt.DB, error) { +func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error)) (*bbolt.DB, error) { return bbolt.Open(filepath.Join(p, dbName), os.ModePerm, &bbolt.Options{ NoFreelistSync: true, ReadOnly: ro, Timeout: 100 * time.Millisecond, OpenFile: openFile, - PageSize: pageSize, }) } From 3b236160a61c8257aae5bd7078035838606650b7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 11 Sep 2024 09:55:08 +0300 Subject: [PATCH 1345/1943] [#1367] writecache: Drop DB label from metrics Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/writecache.go | 10 ++++------ pkg/local_object_storage/writecache/metrics.go | 8 ++++---- pkg/local_object_storage/writecache/state.go | 4 ++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/local_object_storage/engine/writecache.go b/pkg/local_object_storage/engine/writecache.go index 7710bc7f4..e9ba3410f 100644 --- a/pkg/local_object_storage/engine/writecache.go +++ b/pkg/local_object_storage/engine/writecache.go @@ -169,18 +169,16 @@ func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.Sto m.metrics.AddMethodDuration(m.shardID, m.path, st.String(), "Put", success, d) } -func (m *writeCacheMetrics) SetEstimateSize(db, fstree uint64) { - m.metrics.SetEstimateSize(m.shardID, m.path, writecache.StorageTypeDB.String(), db) - m.metrics.SetEstimateSize(m.shardID, m.path, writecache.StorageTypeFSTree.String(), fstree) +func (m *writeCacheMetrics) SetEstimateSize(size uint64) { + m.metrics.SetEstimateSize(m.shardID, m.path, writecache.StorageTypeFSTree.String(), size) } func (m *writeCacheMetrics) SetMode(mod mode.ComponentMode) { m.metrics.SetMode(m.shardID, mod.String()) } -func (m *writeCacheMetrics) SetActualCounters(db, fstree uint64) { - m.metrics.SetActualCount(m.shardID, m.path, writecache.StorageTypeDB.String(), db) - m.metrics.SetActualCount(m.shardID, m.path, writecache.StorageTypeFSTree.String(), fstree) +func (m *writeCacheMetrics) SetActualCounters(count uint64) { + m.metrics.SetActualCount(m.shardID, m.path, writecache.StorageTypeFSTree.String(), count) } func (m *writeCacheMetrics) Flush(success bool, st writecache.StorageType) { diff --git a/pkg/local_object_storage/writecache/metrics.go b/pkg/local_object_storage/writecache/metrics.go index e68b6d8be..e3641f85e 100644 --- a/pkg/local_object_storage/writecache/metrics.go +++ b/pkg/local_object_storage/writecache/metrics.go @@ -26,9 +26,9 @@ type Metrics interface { Flush(success bool, st StorageType) Evict(st StorageType) - SetEstimateSize(db, fstree uint64) + SetEstimateSize(uint64) SetMode(m mode.ComponentMode) - SetActualCounters(db, fstree uint64) + SetActualCounters(uint64) SetPath(path string) Close() } @@ -47,11 +47,11 @@ func (metricsStub) Delete(time.Duration, bool, StorageType) {} func (metricsStub) Put(time.Duration, bool, StorageType) {} -func (metricsStub) SetEstimateSize(uint64, uint64) {} +func (metricsStub) SetEstimateSize(uint64) {} func (metricsStub) SetMode(mode.ComponentMode) {} -func (metricsStub) SetActualCounters(uint64, uint64) {} +func (metricsStub) SetActualCounters(uint64) {} func (metricsStub) Flush(bool, StorageType) {} diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index 748c78bcb..835686fbb 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -2,8 +2,8 @@ package writecache func (c *cache) estimateCacheSize() (uint64, uint64) { count, size := c.counter.CountSize() - c.metrics.SetEstimateSize(0, size) - c.metrics.SetActualCounters(0, count) + c.metrics.SetEstimateSize(size) + c.metrics.SetActualCounters(count) return count, size } From f345fe9a581762fcd5827719d1781e16ce133056 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 11 Sep 2024 17:08:52 +0300 Subject: [PATCH 1346/1943] [#1367] writecache: Move DB related code to upgrade.go This is done to drop this file in the future. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/writecache/cache.go | 5 +- pkg/local_object_storage/writecache/flush.go | 86 -------------- .../writecache/storage.go | 2 - .../writecache/upgrade.go | 110 ++++++++++++++++++ pkg/local_object_storage/writecache/util.go | 20 ---- 5 files changed, 111 insertions(+), 112 deletions(-) create mode 100644 pkg/local_object_storage/writecache/upgrade.go delete mode 100644 pkg/local_object_storage/writecache/util.go diff --git a/pkg/local_object_storage/writecache/cache.go b/pkg/local_object_storage/writecache/cache.go index 341071dc6..b97fc5856 100644 --- a/pkg/local_object_storage/writecache/cache.go +++ b/pkg/local_object_storage/writecache/cache.go @@ -45,10 +45,7 @@ const ( defaultMaxCacheSize = 1 << 30 // 1 GiB ) -var ( - defaultBucket = []byte{0} - dummyCanceler context.CancelFunc = func() {} -) +var dummyCanceler context.CancelFunc = func() {} // New creates new writecache instance. func New(opts ...Option) Cache { diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 10e4d68f0..83933375b 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -1,12 +1,8 @@ package writecache import ( - "bytes" "context" "errors" - "fmt" - "os" - "path/filepath" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -20,8 +16,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" @@ -239,83 +233,3 @@ func (c *cache) Flush(ctx context.Context, ignoreErrors, seal bool) error { func (c *cache) flush(ctx context.Context, ignoreErrors bool) error { return c.flushFSTree(ctx, ignoreErrors) } - -type batchItem struct { - data []byte - address string -} - -func (c *cache) flushAndDropBBoltDB(ctx context.Context) error { - _, err := os.Stat(filepath.Join(c.path, dbName)) - if err != nil && os.IsNotExist(err) { - return nil - } - if err != nil { - return fmt.Errorf("could not check write-cache database existence: %w", err) - } - db, err := OpenDB(c.path, true, os.OpenFile) - if err != nil { - return fmt.Errorf("could not open write-cache database: %w", err) - } - defer func() { - _ = db.Close() - }() - - var last string - for { - batch, err := c.readNextDBBatch(db, last) - if err != nil { - return err - } - if len(batch) == 0 { - break - } - for _, item := range batch { - var obj objectSDK.Object - if err := obj.Unmarshal(item.data); err != nil { - return fmt.Errorf("unmarshal object from database: %w", err) - } - if err := c.flushObject(ctx, &obj, item.data, StorageTypeDB); err != nil { - return fmt.Errorf("flush object from database: %w", err) - } - } - last = batch[len(batch)-1].address - } - if err := db.Close(); err != nil { - return fmt.Errorf("close write-cache database: %w", err) - } - if err := os.Remove(filepath.Join(c.path, dbName)); err != nil { - return fmt.Errorf("remove write-cache database: %w", err) - } - return nil -} - -func (c *cache) readNextDBBatch(db *bbolt.DB, last string) ([]batchItem, error) { - const batchSize = 100 - var batch []batchItem - err := db.View(func(tx *bbolt.Tx) error { - var addr oid.Address - - b := tx.Bucket(defaultBucket) - cs := b.Cursor() - for k, data := cs.Seek([]byte(last)); k != nil; k, data = cs.Next() { - sa := string(k) - if sa == last { - continue - } - if err := addr.DecodeString(sa); err != nil { - return fmt.Errorf("decode address from database: %w", err) - } - - batch = append(batch, batchItem{data: bytes.Clone(data), address: sa}) - if len(batch) == batchSize { - return errIterationCompleted - } - } - return nil - }) - if err == nil || errors.Is(err, errIterationCompleted) { - return batch, nil - } - return nil, err -} diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index e708a529e..6aface7a5 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -16,8 +16,6 @@ import ( "go.uber.org/zap" ) -const dbName = "small.bolt" - func (c *cache) openStore(mod mode.ComponentMode) error { err := util.MkdirAllX(c.path, os.ModePerm) if err != nil { diff --git a/pkg/local_object_storage/writecache/upgrade.go b/pkg/local_object_storage/writecache/upgrade.go new file mode 100644 index 000000000..3a100f1a3 --- /dev/null +++ b/pkg/local_object_storage/writecache/upgrade.go @@ -0,0 +1,110 @@ +package writecache + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.etcd.io/bbolt" +) + +const dbName = "small.bolt" + +var defaultBucket = []byte{0} + +func (c *cache) flushAndDropBBoltDB(ctx context.Context) error { + _, err := os.Stat(filepath.Join(c.path, dbName)) + if err != nil && os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("could not check write-cache database existence: %w", err) + } + db, err := OpenDB(c.path, true, os.OpenFile) + if err != nil { + return fmt.Errorf("could not open write-cache database: %w", err) + } + defer func() { + _ = db.Close() + }() + + var last string + for { + batch, err := c.readNextDBBatch(db, last) + if err != nil { + return err + } + if len(batch) == 0 { + break + } + for _, item := range batch { + var obj objectSDK.Object + if err := obj.Unmarshal(item.data); err != nil { + return fmt.Errorf("unmarshal object from database: %w", err) + } + if err := c.flushObject(ctx, &obj, item.data, StorageTypeDB); err != nil { + return fmt.Errorf("flush object from database: %w", err) + } + } + last = batch[len(batch)-1].address + } + if err := db.Close(); err != nil { + return fmt.Errorf("close write-cache database: %w", err) + } + if err := os.Remove(filepath.Join(c.path, dbName)); err != nil { + return fmt.Errorf("remove write-cache database: %w", err) + } + return nil +} + +type batchItem struct { + data []byte + address string +} + +func (c *cache) readNextDBBatch(db *bbolt.DB, last string) ([]batchItem, error) { + const batchSize = 100 + var batch []batchItem + err := db.View(func(tx *bbolt.Tx) error { + var addr oid.Address + + b := tx.Bucket(defaultBucket) + cs := b.Cursor() + for k, data := cs.Seek([]byte(last)); k != nil; k, data = cs.Next() { + sa := string(k) + if sa == last { + continue + } + if err := addr.DecodeString(sa); err != nil { + return fmt.Errorf("decode address from database: %w", err) + } + + batch = append(batch, batchItem{data: bytes.Clone(data), address: sa}) + if len(batch) == batchSize { + return errIterationCompleted + } + } + return nil + }) + if err == nil || errors.Is(err, errIterationCompleted) { + return batch, nil + } + return nil, err +} + +// OpenDB opens BoltDB instance for write-cache. Opens in read-only mode if ro is true. +func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error)) (*bbolt.DB, error) { + return bbolt.Open(filepath.Join(p, dbName), os.ModePerm, &bbolt.Options{ + NoFreelistSync: true, + ReadOnly: ro, + Timeout: 100 * time.Millisecond, + OpenFile: openFile, + }) +} diff --git a/pkg/local_object_storage/writecache/util.go b/pkg/local_object_storage/writecache/util.go deleted file mode 100644 index 0ed4a954e..000000000 --- a/pkg/local_object_storage/writecache/util.go +++ /dev/null @@ -1,20 +0,0 @@ -package writecache - -import ( - "io/fs" - "os" - "path/filepath" - "time" - - "go.etcd.io/bbolt" -) - -// OpenDB opens BoltDB instance for write-cache. Opens in read-only mode if ro is true. -func OpenDB(p string, ro bool, openFile func(string, int, fs.FileMode) (*os.File, error)) (*bbolt.DB, error) { - return bbolt.Open(filepath.Join(p, dbName), os.ModePerm, &bbolt.Options{ - NoFreelistSync: true, - ReadOnly: ro, - Timeout: 100 * time.Millisecond, - OpenFile: openFile, - }) -} From b33559754df994c4e2e37f1b5b6c8f29ac8f97f1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 12 Sep 2024 12:33:12 +0300 Subject: [PATCH 1347/1943] [#1367] fstree: Add size hint for Delete This allow to not to call `os.Stat` if caller already knows data size. Signed-off-by: Dmitrii Stepanov --- .../blobstor/common/delete.go | 1 + .../blobstor/fstree/fstree.go | 2 +- .../blobstor/fstree/fstree_test.go | 84 +++++++++++++------ .../blobstor/fstree/fstree_write_generic.go | 19 +++-- .../blobstor/fstree/fstree_write_linux.go | 25 +++--- pkg/local_object_storage/writecache/flush.go | 4 +- .../writecache/storage.go | 4 +- 7 files changed, 92 insertions(+), 47 deletions(-) diff --git a/pkg/local_object_storage/blobstor/common/delete.go b/pkg/local_object_storage/blobstor/common/delete.go index 1b04eab1a..c19e099cb 100644 --- a/pkg/local_object_storage/blobstor/common/delete.go +++ b/pkg/local_object_storage/blobstor/common/delete.go @@ -8,6 +8,7 @@ import ( type DeletePrm struct { Address oid.Address StorageID []byte + Size uint64 } // DeleteRes groups the resulting values of Delete operation. diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 1c60ec340..057796db2 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -338,7 +338,7 @@ func (t *FSTree) Delete(ctx context.Context, prm common.DeletePrm) (common.Delet } p := t.treePath(prm.Address) - err = t.writer.removeFile(p) + err = t.writer.removeFile(p, prm.Size) return common.DeleteRes{}, err } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_test.go b/pkg/local_object_storage/blobstor/fstree/fstree_test.go index f39c7296e..eb2126b6c 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_test.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_test.go @@ -68,34 +68,70 @@ func TestObjectCounter(t *testing.T) { var delPrm common.DeletePrm delPrm.Address = addr - eg, egCtx := errgroup.WithContext(context.Background()) + t.Run("without size hint", func(t *testing.T) { + eg, egCtx := errgroup.WithContext(context.Background()) - eg.Go(func() error { - for range 1_000 { - _, err := fst.Put(egCtx, putPrm) - if err != nil { - return err + eg.Go(func() error { + for range 1_000 { + _, err := fst.Put(egCtx, putPrm) + if err != nil { + return err + } } - } - return nil + return nil + }) + + eg.Go(func() error { + var le logicerr.Logical + for range 1_000 { + _, err := fst.Delete(egCtx, delPrm) + if err != nil && !errors.As(err, &le) { + return err + } + } + return nil + }) + + require.NoError(t, eg.Wait()) + + count, size = counter.CountSize() + realCount, realSize, err := fst.countFiles() + require.NoError(t, err) + require.Equal(t, realCount, count, "real %d, actual %d", realCount, count) + require.Equal(t, realSize, size, "real %d, actual %d", realSize, size) }) - eg.Go(func() error { - var le logicerr.Logical - for range 1_000 { - _, err := fst.Delete(egCtx, delPrm) - if err != nil && !errors.As(err, &le) { - return err + t.Run("with size hint", func(t *testing.T) { + delPrm.Size = uint64(len(putPrm.RawData)) + eg, egCtx := errgroup.WithContext(context.Background()) + + eg.Go(func() error { + for range 1_000 { + _, err := fst.Put(egCtx, putPrm) + if err != nil { + return err + } } - } - return nil + return nil + }) + + eg.Go(func() error { + var le logicerr.Logical + for range 1_000 { + _, err := fst.Delete(egCtx, delPrm) + if err != nil && !errors.As(err, &le) { + return err + } + } + return nil + }) + + require.NoError(t, eg.Wait()) + + count, size = counter.CountSize() + realCount, realSize, err := fst.countFiles() + require.NoError(t, err) + require.Equal(t, realCount, count, "real %d, actual %d", realCount, count) + require.Equal(t, realSize, size, "real %d, actual %d", realSize, size) }) - - require.NoError(t, eg.Wait()) - - count, size = counter.CountSize() - realCount, realSize, err := fst.countFiles() - require.NoError(t, err) - require.Equal(t, realCount, count, "real %d, actual %d", realCount, count) - require.Equal(t, realSize, size, "real %d, actual %d", realSize, size) } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 801fc4a22..4110ba7d7 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -16,7 +16,7 @@ import ( type writer interface { writeData(string, []byte) error - removeFile(string) error + removeFile(string, uint64) error } type genericWriter struct { @@ -107,10 +107,10 @@ func (w *genericWriter) writeFile(p string, data []byte) error { return err } -func (w *genericWriter) removeFile(p string) error { +func (w *genericWriter) removeFile(p string, size uint64) error { var err error if w.fileCounterEnabled { - err = w.removeWithCounter(p) + err = w.removeWithCounter(p, size) } else { err = os.Remove(p) } @@ -121,18 +121,21 @@ func (w *genericWriter) removeFile(p string) error { return err } -func (w *genericWriter) removeWithCounter(p string) error { +func (w *genericWriter) removeWithCounter(p string, size uint64) error { w.fileGuard.Lock(p) defer w.fileGuard.Unlock(p) - stat, err := os.Stat(p) - if err != nil { - return err + if size == 0 { + stat, err := os.Stat(p) + if err != nil { + return err + } + size = uint64(stat.Size()) } if err := os.Remove(p); err != nil { return err } - w.fileCounter.Dec(uint64(stat.Size())) + w.fileCounter.Dec(uint64(size)) return nil } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go index 3127579ac..3561c616b 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -91,25 +91,30 @@ func (w *linuxWriter) writeFile(p string, data []byte) error { return errClose } -func (w *linuxWriter) removeFile(p string) error { +func (w *linuxWriter) removeFile(p string, size uint64) error { if w.fileCounterEnabled { w.fileGuard.Lock(p) defer w.fileGuard.Unlock(p) - } - var stat unix.Stat_t - err := unix.Stat(p, &stat) - if err != nil { - if err == unix.ENOENT { - return logicerr.Wrap(new(apistatus.ObjectNotFound)) + + if size == 0 { + var stat unix.Stat_t + err := unix.Stat(p, &stat) + if err != nil { + if err == unix.ENOENT { + return logicerr.Wrap(new(apistatus.ObjectNotFound)) + } + return err + } + size = uint64(stat.Size) } - return err } - err = unix.Unlink(p) + + err := unix.Unlink(p) if err != nil && err == unix.ENOENT { return logicerr.Wrap(new(apistatus.ObjectNotFound)) } if err == nil { - w.fileCounter.Dec(uint64(stat.Size)) + w.fileCounter.Dec(uint64(size)) } return err } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 83933375b..bfa6aacb0 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -123,7 +123,7 @@ func (c *cache) flushIfAnObjectExistsWorker(ctx context.Context, objInfo objectI return } - c.deleteFromDisk(ctx, objInfo.addr) + c.deleteFromDisk(ctx, objInfo.addr, uint64(len(res.RawData))) } func (c *cache) reportFlushError(msg string, addr string, err error) { @@ -157,7 +157,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { return err } - c.deleteFromDisk(ctx, e.Address) + c.deleteFromDisk(ctx, e.Address, uint64(len(e.ObjectData))) return nil } diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index 6aface7a5..2e52e5b20 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -40,8 +40,8 @@ func (c *cache) openStore(mod mode.ComponentMode) error { return nil } -func (c *cache) deleteFromDisk(ctx context.Context, addr oid.Address) { - _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) +func (c *cache) deleteFromDisk(ctx context.Context, addr oid.Address, size uint64) { + _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr, Size: size}) if err != nil && !client.IsErrObjectNotFound(err) { c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) } else if err == nil { From e3764c51df237289431eacd0051bb3754e7682cc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 12 Sep 2024 09:53:21 +0300 Subject: [PATCH 1348/1943] [#1347] metabase: Fix EC search For EC chunks need to return EC parent object ID as EC chunks don't have own attributes but inherit parent's. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/select.go | 21 +++++++++------ .../metabase/select_test.go | 27 +++++++++++++++++++ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 1863fc25e..ed43fc41f 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -150,7 +150,8 @@ func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters continue // ignore removed objects } - if !db.matchSlowFilters(tx, addr, group.slowFilters, currEpoch) { + addr, match := db.matchSlowFilters(tx, addr, group.slowFilters, currEpoch) + if !match { continue // ignore objects with unmatched slow filters } @@ -382,15 +383,16 @@ func (db *DB) selectObjectID( } // matchSlowFilters return true if object header is matched by all slow filters. -func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.SearchFilters, currEpoch uint64) bool { +func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.SearchFilters, currEpoch uint64) (oid.Address, bool) { + result := addr if len(f) == 0 { - return true + return result, true } buf := make([]byte, addressKeySize) obj, err := db.get(tx, addr, buf, true, false, currEpoch) if err != nil { - return false + return result, false } for i := range f { @@ -415,23 +417,26 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.Searc default: // user attribute v, ok := attributeValue(obj, f[i].Header()) if ok { + if ech := obj.ECHeader(); ech != nil { + result.SetObject(ech.Parent()) + } data = []byte(v) } else { - return f[i].Operation() == objectSDK.MatchNotPresent + return result, f[i].Operation() == objectSDK.MatchNotPresent } } matchFunc, ok := db.matchers[f[i].Operation()] if !ok { - return false + return result, false } if !matchFunc.matchSlow(f[i].Header(), data, f[i].Value()) { - return false + return result, false } } - return true + return result, true } func attributeValue(obj *objectSDK.Object, attribute string) (string, bool) { diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 0fab3a108..6469bbdbc 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -70,6 +70,22 @@ func TestDB_SelectUserAttributes(t *testing.T) { err = putBig(db, raw6) require.NoError(t, err) + raw7 := testutil.GenerateObjectWithCID(cnr) + var attr objectSDK.Attribute + attr.SetKey("path") + attr.SetValue("test/3/4") + attrs := raw7.Attributes() + attrs = append(attrs, attr) + ech := objectSDK.NewECHeader(objectSDK.ECParentInfo{ + ID: oidtest.ID(), + Attributes: attrs, + }, 0, 3, []byte{}, 0) + raw7.SetECHeader(ech) + require.NoError(t, putBig(db, raw7)) + var raw7Parent oid.Address + raw7Parent.SetContainer(cnr) + raw7Parent.SetObject(ech.Parent()) + fs := objectSDK.SearchFilters{} fs.AddFilter("foo", "bar", objectSDK.MatchStringEqual) testSelect(t, db, cnr, fs, @@ -100,6 +116,7 @@ func TestDB_SelectUserAttributes(t *testing.T) { object.AddressOf(raw4), object.AddressOf(raw5), object.AddressOf(raw6), + object.AddressOf(raw7), ) fs = objectSDK.SearchFilters{} @@ -110,6 +127,7 @@ func TestDB_SelectUserAttributes(t *testing.T) { object.AddressOf(raw4), object.AddressOf(raw5), object.AddressOf(raw6), + object.AddressOf(raw7), ) fs = objectSDK.SearchFilters{} @@ -120,6 +138,7 @@ func TestDB_SelectUserAttributes(t *testing.T) { object.AddressOf(raw4), object.AddressOf(raw5), object.AddressOf(raw6), + object.AddressOf(raw7), ) fs = objectSDK.SearchFilters{} @@ -131,6 +150,7 @@ func TestDB_SelectUserAttributes(t *testing.T) { object.AddressOf(raw4), object.AddressOf(raw5), object.AddressOf(raw6), + object.AddressOf(raw7), ) fs = objectSDK.SearchFilters{} @@ -139,6 +159,7 @@ func TestDB_SelectUserAttributes(t *testing.T) { object.AddressOf(raw4), object.AddressOf(raw5), object.AddressOf(raw6), + raw7Parent, ) fs = objectSDK.SearchFilters{} @@ -147,6 +168,12 @@ func TestDB_SelectUserAttributes(t *testing.T) { object.AddressOf(raw4), object.AddressOf(raw5), ) + + fs = objectSDK.SearchFilters{} + fs.AddFilter("path", "test/3/4", objectSDK.MatchStringEqual) + testSelect(t, db, cnr, fs, + raw7Parent, + ) } func TestDB_SelectRootPhyParent(t *testing.T) { From 546d09660f52d262c3c053b72457170f5c4b68e7 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 12 Sep 2024 17:05:14 +0300 Subject: [PATCH 1349/1943] [#1283] Clear systemd-notify status on exit Signed-off-by: Aleksey Savchuk --- cmd/frostfs-ir/main.go | 5 +++++ cmd/frostfs-node/config.go | 4 ++++ pkg/util/sdnotify/sdnotify.go | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 5b852c37c..02936ae78 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -13,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sdnotify" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -124,4 +125,8 @@ func shutdown() { zap.String("error", err.Error()), ) } + + if err := sdnotify.ClearStatus(); err != nil { + log.Error(logs.FailedToReportStatusToSystemd, zap.Error(err)) + } } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index cdfabdebd..f98f1c1a3 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1398,4 +1398,8 @@ func (c *cfg) shutdown() { for i := range c.closers { c.closers[len(c.closers)-1-i].fn() } + + if err := sdnotify.ClearStatus(); err != nil { + c.log.Error(logs.FailedToReportStatusToSystemd, zap.Error(err)) + } } diff --git a/pkg/util/sdnotify/sdnotify.go b/pkg/util/sdnotify/sdnotify.go index 5235315cc..e94ff77ad 100644 --- a/pkg/util/sdnotify/sdnotify.go +++ b/pkg/util/sdnotify/sdnotify.go @@ -69,6 +69,11 @@ func Status(status string) error { return Send("STATUS=" + status) } +// ClearStatus resets the current service status previously set by Status. +func ClearStatus() error { + return Status("") +} + // Send state through the notify socket if any. // If the notify socket was not detected, it returns an error. func Send(state string) error { From bb448674918330ddd15df6462fbde860eca5bc64 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 13 Sep 2024 11:31:33 +0300 Subject: [PATCH 1350/1943] [#1374] go.mod: Upgrade grpc to v1.66.2 Signed-off-by: Dmitrii Stepanov --- go.mod | 14 +++++++------- go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 93eef5b8c..78fefc9ae 100644 --- a/go.mod +++ b/go.mod @@ -46,9 +46,9 @@ require ( golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/sync v0.7.0 golang.org/x/sys v0.22.0 - golang.org/x/term v0.18.0 - google.golang.org/grpc v1.63.2 - google.golang.org/protobuf v1.33.0 + golang.org/x/term v0.21.0 + google.golang.org/grpc v1.66.2 + google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -122,11 +122,11 @@ require ( go.opentelemetry.io/otel/sdk v1.22.0 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.23.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 102501484..2147f8988 100644 --- a/go.sum +++ b/go.sum @@ -306,8 +306,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -327,8 +327,8 @@ golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -370,8 +370,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -393,12 +393,12 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= -google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -407,8 +407,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 944160427bad682df038f75b94cfad1e3e23aa41 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 13 Sep 2024 11:32:03 +0300 Subject: [PATCH 1351/1943] [#1374] cli: Drop deprecated grpc connection method For `frostfs-cli` it is ok to use grpc-client without blocking, as `frostfs-cli` will perform RPC call anyway. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/tree/add.go | 5 +++-- cmd/frostfs-cli/modules/tree/add_by_path.go | 5 +++-- cmd/frostfs-cli/modules/tree/client.go | 22 +++++++++++---------- cmd/frostfs-cli/modules/tree/get_by_path.go | 5 +++-- cmd/frostfs-cli/modules/tree/get_op_log.go | 5 +++-- cmd/frostfs-cli/modules/tree/healthcheck.go | 5 +++-- cmd/frostfs-cli/modules/tree/list.go | 5 +++-- cmd/frostfs-cli/modules/tree/move.go | 5 +++-- cmd/frostfs-cli/modules/tree/remove.go | 5 +++-- cmd/frostfs-cli/modules/tree/subtree.go | 5 +++-- 10 files changed, 39 insertions(+), 28 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index 0b8dc292f..019feb0ec 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -47,9 +47,10 @@ func add(cmd *cobra.Command, _ []string) { meta, err := parseMeta(cmd) commonCmd.ExitOnErr(cmd, "meta data parsing: %w", err) - ctx := cmd.Context() + ctx, cancel := contextWithTimeout(cmd) + defer cancel() - cli, err := _client(ctx) + cli, err := _client() commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index ea815dbfe..5d5b00b7d 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -50,9 +50,10 @@ func addByPath(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) tid, _ := cmd.Flags().GetString(treeIDFlagKey) - ctx := cmd.Context() + ctx, cancel := contextWithTimeout(cmd) + defer cancel() - cli, err := _client(ctx) + cli, err := _client() commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index 4f4f54657..4e0099f02 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -3,13 +3,14 @@ package tree import ( "context" "strings" - "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + "github.com/spf13/cobra" "github.com/spf13/viper" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -17,7 +18,7 @@ import ( // _client returns grpc Tree service client. Should be removed // after making Tree API public. -func _client(ctx context.Context) (tree.TreeServiceClient, error) { +func _client() (tree.TreeServiceClient, error) { var netAddr network.Address err := netAddr.FromString(viper.GetString(commonflags.RPC)) if err != nil { @@ -25,7 +26,6 @@ func _client(ctx context.Context) (tree.TreeServiceClient, error) { } opts := []grpc.DialOption{ - grpc.WithBlock(), grpc.WithChainUnaryInterceptor( metrics.NewUnaryClientInterceptor(), tracing.NewUnaryClientInteceptor(), @@ -40,12 +40,14 @@ func _client(ctx context.Context) (tree.TreeServiceClient, error) { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } - // a default connection establishing timeout - const defaultClientConnectTimeout = time.Second * 2 - - ctx, cancel := context.WithTimeout(ctx, defaultClientConnectTimeout) - cc, err := grpc.DialContext(ctx, netAddr.URIAddr(), opts...) - cancel() - + cc, err := grpc.NewClient(netAddr.URIAddr(), opts...) return tree.NewTreeServiceClient(cc), err } + +func contextWithTimeout(cmd *cobra.Command) (context.Context, context.CancelFunc) { + if timeout := viper.GetDuration(commonflags.Timeout); timeout > 0 { + common.PrintVerbose(cmd, "Set request timeout to %s.", timeout) + return context.WithTimeout(cmd.Context(), timeout) + } + return context.WithTimeout(cmd.Context(), commonflags.TimeoutDefault) +} diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index f239066cd..7061723fd 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -50,9 +50,10 @@ func getByPath(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) tid, _ := cmd.Flags().GetString(treeIDFlagKey) - ctx := cmd.Context() + ctx, cancel := contextWithTimeout(cmd) + defer cancel() - cli, err := _client(ctx) + cli, err := _client() commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) diff --git a/cmd/frostfs-cli/modules/tree/get_op_log.go b/cmd/frostfs-cli/modules/tree/get_op_log.go index b1b307f62..376aa8e8d 100644 --- a/cmd/frostfs-cli/modules/tree/get_op_log.go +++ b/cmd/frostfs-cli/modules/tree/get_op_log.go @@ -44,9 +44,10 @@ func getOpLog(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) tid, _ := cmd.Flags().GetString(treeIDFlagKey) - ctx := cmd.Context() + ctx, cancel := contextWithTimeout(cmd) + defer cancel() - cli, err := _client(ctx) + cli, err := _client() commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) diff --git a/cmd/frostfs-cli/modules/tree/healthcheck.go b/cmd/frostfs-cli/modules/tree/healthcheck.go index f0506467e..b01bb2e77 100644 --- a/cmd/frostfs-cli/modules/tree/healthcheck.go +++ b/cmd/frostfs-cli/modules/tree/healthcheck.go @@ -26,9 +26,10 @@ func initHealthcheckCmd() { func healthcheck(cmd *cobra.Command, _ []string) { pk := key.GetOrGenerate(cmd) - ctx := cmd.Context() + ctx, cancel := contextWithTimeout(cmd) + defer cancel() - cli, err := _client(ctx) + cli, err := _client() commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) req := &tree.HealthcheckRequest{ diff --git a/cmd/frostfs-cli/modules/tree/list.go b/cmd/frostfs-cli/modules/tree/list.go index a25d066d5..f8c0e490f 100644 --- a/cmd/frostfs-cli/modules/tree/list.go +++ b/cmd/frostfs-cli/modules/tree/list.go @@ -38,9 +38,10 @@ func list(cmd *cobra.Command, _ []string) { err := cnr.DecodeString(cidString) commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) - ctx := cmd.Context() + ctx, cancel := contextWithTimeout(cmd) + defer cancel() - cli, err := _client(ctx) + cli, err := _client() commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) diff --git a/cmd/frostfs-cli/modules/tree/move.go b/cmd/frostfs-cli/modules/tree/move.go index 24abbd650..dc807d752 100644 --- a/cmd/frostfs-cli/modules/tree/move.go +++ b/cmd/frostfs-cli/modules/tree/move.go @@ -45,9 +45,10 @@ func move(cmd *cobra.Command, _ []string) { err := cnr.DecodeString(cidString) commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) - ctx := cmd.Context() + ctx, cancel := contextWithTimeout(cmd) + defer cancel() - cli, err := _client(ctx) + cli, err := _client() commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) diff --git a/cmd/frostfs-cli/modules/tree/remove.go b/cmd/frostfs-cli/modules/tree/remove.go index 74e9d9749..d0b6fab2f 100644 --- a/cmd/frostfs-cli/modules/tree/remove.go +++ b/cmd/frostfs-cli/modules/tree/remove.go @@ -41,9 +41,10 @@ func remove(cmd *cobra.Command, _ []string) { err := cnr.DecodeString(cidString) commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) - ctx := cmd.Context() + ctx, cancel := contextWithTimeout(cmd) + defer cancel() - cli, err := _client(ctx) + cli, err := _client() commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) diff --git a/cmd/frostfs-cli/modules/tree/subtree.go b/cmd/frostfs-cli/modules/tree/subtree.go index e88ef79cb..83a8909b6 100644 --- a/cmd/frostfs-cli/modules/tree/subtree.go +++ b/cmd/frostfs-cli/modules/tree/subtree.go @@ -46,9 +46,10 @@ func getSubTree(cmd *cobra.Command, _ []string) { err := cnr.DecodeString(cidString) commonCmd.ExitOnErr(cmd, "decode container ID string: %w", err) - ctx := cmd.Context() + ctx, cancel := contextWithTimeout(cmd) + defer cancel() - cli, err := _client(ctx) + cli, err := _client() commonCmd.ExitOnErr(cmd, "failed to create client: %w", err) rawCID := make([]byte, sha256.Size) From 54fe8383a4f274da8f1fd553097f3268c9f2d13a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 13 Sep 2024 11:39:25 +0300 Subject: [PATCH 1352/1943] [#1374] tree: Use NewClient to create grpc connection for sync Created connection will be used to sync trees, so it is ok to defer dial to the first RPC call. Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/sync.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index be22074a5..5bbc93978 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -294,7 +294,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return false } - cc, err := s.dialCtx(egCtx, a) + cc, err := s.createConnection(a) if err != nil { s.log.Warn(logs.TreeFailedToConnectForTreeSynchronization, zap.Error(err), zap.String("address", addr)) return false @@ -332,8 +332,8 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return from } -func (*Service) dialCtx(egCtx context.Context, a network.Address) (*grpc.ClientConn, error) { - return grpc.DialContext(egCtx, a.URIAddr(), +func (*Service) createConnection(a network.Address) (*grpc.ClientConn, error) { + return grpc.NewClient(a.URIAddr(), grpc.WithChainUnaryInterceptor( metrics.NewUnaryClientInterceptor(), tracing_grpc.NewUnaryClientInteceptor(), From 89d0435b1d59257d5bf15c926465193e53c11922 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 13 Sep 2024 11:59:17 +0300 Subject: [PATCH 1353/1943] [#1374] tree: Use NewClient to create grpc connection in cache Created grpc connection should be established, so perform Healthcheck request to check connection is ok. Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/cache.go | 33 +++++++++++++++++++++++++-------- pkg/services/tree/service.go | 2 +- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 1be1c2f83..38501b852 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -2,6 +2,7 @@ package tree import ( "context" + "crypto/ecdsa" "errors" "fmt" "sync" @@ -19,6 +20,7 @@ import ( type clientCache struct { sync.Mutex simplelru.LRU[string, cacheItem] + key *ecdsa.PrivateKey } type cacheItem struct { @@ -34,13 +36,14 @@ const ( var errRecentlyFailed = errors.New("client has recently failed") -func (c *clientCache) init() { +func (c *clientCache) init(pk *ecdsa.PrivateKey) { l, _ := simplelru.NewLRU(defaultClientCacheSize, func(_ string, value cacheItem) { if conn := value.cc; conn != nil { _ = conn.Close() } }) c.LRU = *l + c.key = pk } func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceClient, error) { @@ -63,7 +66,7 @@ func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceCl } } - cc, err := dialTreeService(ctx, netmapAddr) + cc, err := c.dialTreeService(ctx, netmapAddr) lastTry := time.Now() c.Lock() @@ -81,14 +84,13 @@ func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceCl return NewTreeServiceClient(cc), nil } -func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, error) { +func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, error) { var netAddr network.Address if err := netAddr.FromString(netmapAddr); err != nil { return nil, err } opts := []grpc.DialOption{ - grpc.WithBlock(), grpc.WithChainUnaryInterceptor( metrics.NewUnaryClientInterceptor(), tracing.NewUnaryClientInteceptor(), @@ -103,9 +105,24 @@ func dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } - ctx, cancel := context.WithTimeout(ctx, defaultClientConnectTimeout) - cc, err := grpc.DialContext(ctx, netAddr.URIAddr(), opts...) - cancel() + req := &HealthcheckRequest{ + Body: &HealthcheckRequest_Body{}, + } + if err := SignMessage(req, c.key); err != nil { + return nil, err + } - return cc, err + cc, err := grpc.NewClient(netAddr.URIAddr(), opts...) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(ctx, defaultClientConnectTimeout) + defer cancel() + // perform some request to check connection + if _, err := NewTreeServiceClient(cc).Healthcheck(ctx, req); err != nil { + _ = cc.Close() + return nil, err + } + return cc, nil } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 875e47ecb..60bb1a6ad 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -65,7 +65,7 @@ func New(opts ...Option) *Service { s.log = &logger.Logger{Logger: zap.NewNop()} } - s.cache.init() + s.cache.init(s.key) s.closeCh = make(chan struct{}) s.replicateCh = make(chan movePair, s.replicatorChannelCapacity) s.replicateLocalCh = make(chan applyOp) From 2be1aa781dbbc2462ca717f3b763053fb8cd2810 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 13 Sep 2024 15:44:23 +0300 Subject: [PATCH 1354/1943] [#1266] .forgejo: Make 'fumpt' job fail on changed files `gofumpt` always returns an exit code of 0, even when it finds misformatted files. To make `fumpt` action behave as expected we need to check if `gofumpt` changed any files. Signed-off-by: Ekaterina Lebedeva --- .forgejo/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index a908c6278..07ba5c268 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -106,4 +106,6 @@ jobs: run: make fumpt-install - name: Run gofumpt - run: make fumpt + run: | + make fumpt + git diff --exit-code --quiet From 74a6a1da7fb6a895583d72f188f31b8f067ce513 Mon Sep 17 00:00:00 2001 From: Aleksey Savaitan Date: Thu, 12 Sep 2024 17:40:10 +0300 Subject: [PATCH 1355/1943] [#1361] add root ca cert for telemetry configuration Signed-off-by: Aleksey Savaitan --- cmd/frostfs-node/config.go | 6 ++- cmd/frostfs-node/config/tracing/config.go | 23 +++++++- cmd/frostfs-node/tracing.go | 10 ++-- config/example/node.env | 1 + config/example/node.json | 3 +- config/example/node.yaml | 1 + go.mod | 30 +++++------ go.sum | 64 +++++++++++------------ 8 files changed, 84 insertions(+), 54 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index f98f1c1a3..bb6580a40 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1299,7 +1299,11 @@ func (c *cfg) reloadConfig(ctx context.Context) { }}) components = append(components, dCmp{"pools", c.reloadPools}) components = append(components, dCmp{"tracing", func() error { - updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg)) + traceConfig, err := tracingconfig.ToTracingConfig(c.appCfg) + if err != nil { + return err + } + updated, err := tracing.Setup(ctx, *traceConfig) if updated { c.log.Info(logs.FrostFSNodeTracingConfigationUpdated) } diff --git a/cmd/frostfs-node/config/tracing/config.go b/cmd/frostfs-node/config/tracing/config.go index e846be158..8544c672c 100644 --- a/cmd/frostfs-node/config/tracing/config.go +++ b/cmd/frostfs-node/config/tracing/config.go @@ -1,6 +1,11 @@ package tracing import ( + "crypto/x509" + "errors" + "fmt" + "os" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -11,8 +16,8 @@ const ( ) // ToTracingConfig extracts tracing config. -func ToTracingConfig(c *config.Config) *tracing.Config { - return &tracing.Config{ +func ToTracingConfig(c *config.Config) (*tracing.Config, error) { + conf := &tracing.Config{ Enabled: config.BoolSafe(c.Sub(subsection), "enabled"), Exporter: tracing.Exporter(config.StringSafe(c.Sub(subsection), "exporter")), Endpoint: config.StringSafe(c.Sub(subsection), "endpoint"), @@ -20,6 +25,20 @@ func ToTracingConfig(c *config.Config) *tracing.Config { InstanceID: getInstanceIDOrDefault(c), Version: misc.Version, } + + if trustedCa := config.StringSafe(c.Sub(subsection), "trusted_ca"); trustedCa != "" { + caBytes, err := os.ReadFile(trustedCa) + if err != nil { + return nil, fmt.Errorf("cannot read trusted ca cert by path: %w", err) + } + certPool := x509.NewCertPool() + ok := certPool.AppendCertsFromPEM(caBytes) + if !ok { + return nil, errors.New("can't fill cert pool by ca cert") + } + conf.ServerCaCertPool = certPool + } + return conf, nil } func getInstanceIDOrDefault(c *config.Config) string { diff --git a/cmd/frostfs-node/tracing.go b/cmd/frostfs-node/tracing.go index 675c31374..f550dd882 100644 --- a/cmd/frostfs-node/tracing.go +++ b/cmd/frostfs-node/tracing.go @@ -11,11 +11,15 @@ import ( ) func initTracing(ctx context.Context, c *cfg) { - conf := tracingconfig.ToTracingConfig(c.appCfg) - - _, err := tracing.Setup(ctx, *conf) + conf, err := tracingconfig.ToTracingConfig(c.appCfg) if err != nil { c.log.Error(logs.FrostFSNodeFailedInitTracing, zap.Error(err)) + return + } + _, err = tracing.Setup(ctx, *conf) + if err != nil { + c.log.Error(logs.FrostFSNodeFailedInitTracing, zap.Error(err)) + return } c.closers = append(c.closers, closer{ diff --git a/config/example/node.env b/config/example/node.env index c3fa85c13..030a79934 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -199,6 +199,7 @@ FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m FROSTFS_TRACING_ENABLED=true FROSTFS_TRACING_ENDPOINT="localhost" FROSTFS_TRACING_EXPORTER="otlp_grpc" +FROSTFS_TRACING_TRUSTED_CA="" FROSTFS_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 diff --git a/config/example/node.json b/config/example/node.json index d7187250b..4e6d239fe 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -255,7 +255,8 @@ "tracing": { "enabled": true, "endpoint": "localhost:9090", - "exporter": "otlp_grpc" + "exporter": "otlp_grpc", + "trusted_ca": "/etc/ssl/tracing.pem" }, "runtime": { "soft_memory_limit": 1073741824 diff --git a/config/example/node.yaml b/config/example/node.yaml index 776b22bd0..5a8e6a2a4 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -231,6 +231,7 @@ tracing: enabled: true exporter: "otlp_grpc" endpoint: "localhost" + trusted_ca: "" runtime: soft_memory_limit: 1gb diff --git a/go.mod b/go.mod index 78fefc9ae..621d2e85d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240903093628-8f751d9dd0ad git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 @@ -40,15 +40,15 @@ require ( github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.9.0 go.etcd.io/bbolt v1.3.10 - go.opentelemetry.io/otel v1.24.0 - go.opentelemetry.io/otel/trace v1.24.0 + go.opentelemetry.io/otel v1.28.0 + go.opentelemetry.io/otel/trace v1.28.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/sync v0.7.0 golang.org/x/sys v0.22.0 golang.org/x/term v0.21.0 google.golang.org/grpc v1.66.2 - google.golang.org/protobuf v1.34.1 + google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v3 v3.0.1 ) @@ -63,7 +63,7 @@ require ( github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 // indirect @@ -73,13 +73,13 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gdamore/encoding v1.0.0 // indirect github.com/go-fed/httpsig v1.1.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.4 // indirect @@ -115,18 +115,18 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/twmb/murmur3 v1.1.8 // indirect github.com/urfave/cli v1.22.14 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/sdk v1.22.0 // indirect - go.opentelemetry.io/proto/otlp v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 2147f8988..4d21d9bca 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d h1:uJ/wvuMdepbkaV8XMS5uN9B0FQWMep0CttSuDZiDhq0= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65 h1:PaZ8GpnUoXxUoNsc1qp36bT2u7FU+neU4Jn9cl8AWqI= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20231101111734-b3ad3335ff65/go.mod h1:6aAX80dvJ3r5fjN9CzzPglRptoiPgIC9KFGGsUA+1Hw= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240903093628-8f751d9dd0ad h1:cJGK/bXSF/0KMr6zkIy06qekQhQRU7eYzh+lWdQ0U8w= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240903093628-8f751d9dd0ad/go.mod h1:I4GzeEnftO++N2JHQn9QR88eV0rjQkGqlUCNMnCz1CY= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= @@ -33,8 +33,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= @@ -82,8 +82,8 @@ github.com/gdamore/tcell/v2 v2.7.4/go.mod h1:dSXtXTSK0VsW1biw65DZLZ2NKr7j0qP/0J7 github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-pkgz/expirable-cache/v3 v3.0.0 h1:u3/gcu3sabLYiTCevoRKv+WzjIn5oo7P8XtiXBeRDLw= @@ -116,8 +116,8 @@ github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 h1:f4tg github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0/go.mod h1:hKAkSgNkL0FII46ZkJcpVEAai4KV+swlIWCKfekd1pA= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 h1:HcUWd006luQPljE73d5sk+/VgYPGUReEVz2y1/qylwY= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -229,8 +229,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= @@ -278,22 +278,22 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 h1:H2JFgRcGiyHg7H7bwcwaQJYrNFqCqrbTQ8K4p1OvDu8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0/go.mod h1:WfCWp1bGoYK8MeULtI15MmQVczfR+bFkk0DF3h06QmQ= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0 h1:zr8ymM5OWWjjiWRzwTfZ67c905+2TMHYp2lMJ52QTyM= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0/go.mod h1:sQs7FT2iLVJ+67vYngGJkPe1qr39IzaBzaj9IDNNY8k= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= -go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= -go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= -go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -393,10 +393,10 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -407,8 +407,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 96308a26c6ab73c416464e3da2f0e859915abc93 Mon Sep 17 00:00:00 2001 From: Aleksey Savaitan Date: Thu, 12 Sep 2024 17:42:20 +0300 Subject: [PATCH 1356/1943] [#1361] linter: fix funlen Signed-off-by: Aleksey Savaitan --- cmd/frostfs-node/config.go | 55 +++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index bb6580a40..c0019d36a 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1278,7 +1278,6 @@ func (c *cfg) reloadConfig(ctx context.Context) { // all the components are expected to support // Logger's dynamic reconfiguration approach - var components []dCmp // Logger @@ -1288,6 +1287,36 @@ func (c *cfg) reloadConfig(ctx context.Context) { return } + components := c.getComponents(ctx, logPrm) + + // Storage Engine + + var rcfg engine.ReConfiguration + for _, optsWithID := range c.shardOpts(ctx) { + rcfg.AddShard(optsWithID.configID, append(optsWithID.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))) + } + + err = c.cfgObject.cfgLocalStorage.localStorage.Reload(ctx, rcfg) + if err != nil { + c.log.Error(logs.FrostFSNodeStorageEngineConfigurationUpdate, zap.Error(err)) + return + } + + for _, component := range components { + err = component.reloadFunc() + if err != nil { + c.log.Error(logs.FrostFSNodeUpdatedConfigurationApplying, + zap.String("component", component.name), + zap.Error(err)) + } + } + + c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) +} + +func (c *cfg) getComponents(ctx context.Context, logPrm *logger.Prm) []dCmp { + var components []dCmp + components = append(components, dCmp{"logger", logPrm.Reload}) components = append(components, dCmp{"runtime", func() error { setRuntimeParameters(c) @@ -1321,29 +1350,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { components = append(components, dCmp{cmp.name, func() error { return cmp.reload(ctx) }}) } - // Storage Engine - - var rcfg engine.ReConfiguration - for _, optsWithID := range c.shardOpts(ctx) { - rcfg.AddShard(optsWithID.configID, append(optsWithID.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))) - } - - err = c.cfgObject.cfgLocalStorage.localStorage.Reload(ctx, rcfg) - if err != nil { - c.log.Error(logs.FrostFSNodeStorageEngineConfigurationUpdate, zap.Error(err)) - return - } - - for _, component := range components { - err = component.reloadFunc() - if err != nil { - c.log.Error(logs.FrostFSNodeUpdatedConfigurationApplying, - zap.String("component", component.name), - zap.Error(err)) - } - } - - c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) + return components } func (c *cfg) reloadPools() error { From ea48e928c8d6c54662c72bd650b57a8b897e7879 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 16 Sep 2024 10:45:26 +0300 Subject: [PATCH 1357/1943] [#1366] logger: Make timestamp prepending optional Signed-off-by: Aleksey Savchuk --- pkg/util/logger/logger.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index e67afb36b..4b60f02de 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -40,6 +40,9 @@ type Prm struct { // do not support runtime rereading dest string + + // PrependTimestamp specifies whether to prepend a timestamp in the log + PrependTimestamp bool } const ( @@ -116,11 +119,16 @@ func newConsoleLogger(prm *Prm) (*Logger, error) { c := zap.NewProductionConfig() c.Level = lvl c.Encoding = "console" - c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder if prm.SamplingHook != nil { c.Sampling.Hook = prm.SamplingHook } + if prm.PrependTimestamp { + c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + } else { + c.EncoderConfig.TimeKey = "" + } + lZap, err := c.Build( zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), ) @@ -140,11 +148,16 @@ func newJournaldLogger(prm *Prm) (*Logger, error) { c := zap.NewProductionConfig() c.Level = lvl c.Encoding = "console" - c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder if prm.SamplingHook != nil { c.Sampling.Hook = prm.SamplingHook } + if prm.PrependTimestamp { + c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + } else { + c.EncoderConfig.TimeKey = "" + } + encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields) core := zapjournald.NewCore(lvl, encoder, &journald.Journal{}, zapjournald.SyslogFields) From d4bec24c9fa7ede0fcc4d5b879316f0a54c61bb9 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 16 Sep 2024 10:46:26 +0300 Subject: [PATCH 1358/1943] [#1366] node, ir: Support `timestamp` config option, update tests Signed-off-by: Aleksey Savchuk --- cmd/frostfs-ir/config.go | 2 ++ cmd/frostfs-ir/defaults.go | 1 + cmd/frostfs-ir/main.go | 2 ++ cmd/frostfs-node/config.go | 3 +++ cmd/frostfs-node/config/logger/config.go | 8 ++++++++ cmd/frostfs-node/config/logger/config_test.go | 2 ++ config/example/ir.env | 1 + config/example/ir.yaml | 1 + config/example/node.env | 1 + config/example/node.json | 3 ++- config/example/node.yaml | 1 + 11 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 4eaac845c..137e764ed 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -41,6 +41,8 @@ func reloadConfig() error { if err != nil { return err } + logPrm.PrependTimestamp = cfg.GetBool("logger.timestamp") + return logPrm.Reload() } diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index e703301ae..899918d22 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -9,6 +9,7 @@ import ( func defaultConfiguration(cfg *viper.Viper) { cfg.SetDefault("logger.level", "info") cfg.SetDefault("logger.destination", "stdout") + cfg.SetDefault("logger.timestamp", false) setPprofDefaults(cfg) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 02936ae78..4bc5923a0 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -79,6 +79,8 @@ func main() { ) exitErr(err) logPrm.SamplingHook = metrics.LogMetrics().GetSamplingHook() + logPrm.PrependTimestamp = cfg.GetBool("logger.timestamp") + log, err = logger.NewLogger(logPrm) exitErr(err) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c0019d36a..ed3a65c25 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -102,6 +102,7 @@ type applicationConfiguration struct { LoggerCfg struct { level string destination string + timestamp bool } EngineCfg struct { @@ -220,6 +221,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { a.LoggerCfg.level = loggerconfig.Level(c) a.LoggerCfg.destination = loggerconfig.Destination(c) + a.LoggerCfg.timestamp = loggerconfig.Timestamp(c) // Storage Engine @@ -1023,6 +1025,7 @@ func (c *cfg) loggerPrm() (*logger.Prm, error) { // not expected since validation should be performed before panic("incorrect log destination format: " + c.LoggerCfg.destination) } + c.dynamicConfiguration.logger.PrependTimestamp = c.LoggerCfg.timestamp return c.dynamicConfiguration.logger, nil } diff --git a/cmd/frostfs-node/config/logger/config.go b/cmd/frostfs-node/config/logger/config.go index 378b9d793..ba9eeea2b 100644 --- a/cmd/frostfs-node/config/logger/config.go +++ b/cmd/frostfs-node/config/logger/config.go @@ -52,6 +52,14 @@ func Destination(c *config.Config) string { return DestinationDefault } +// Timestamp returns the value of "timestamp" config parameter +// from "logger" section. +// +// Returns false if the value isn't specified. +func Timestamp(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "timestamp") +} + // ToLokiConfig extracts loki config. func ToLokiConfig(c *config.Config) loki.Config { hostname, _ := os.Hostname() diff --git a/cmd/frostfs-node/config/logger/config_test.go b/cmd/frostfs-node/config/logger/config_test.go index 3587a0ddb..ffe8ac693 100644 --- a/cmd/frostfs-node/config/logger/config_test.go +++ b/cmd/frostfs-node/config/logger/config_test.go @@ -13,6 +13,7 @@ func TestLoggerSection_Level(t *testing.T) { t.Run("defaults", func(t *testing.T) { require.Equal(t, loggerconfig.LevelDefault, loggerconfig.Level(configtest.EmptyConfig())) require.Equal(t, loggerconfig.DestinationDefault, loggerconfig.Destination(configtest.EmptyConfig())) + require.Equal(t, false, loggerconfig.Timestamp(configtest.EmptyConfig())) }) const path = "../../../../config/example/node" @@ -20,6 +21,7 @@ func TestLoggerSection_Level(t *testing.T) { fileConfigTest := func(c *config.Config) { require.Equal(t, "debug", loggerconfig.Level(c)) require.Equal(t, "journald", loggerconfig.Destination(c)) + require.Equal(t, true, loggerconfig.Timestamp(c)) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/config/example/ir.env b/config/example/ir.env index 3f9530ab6..7234a4b32 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -1,4 +1,5 @@ FROSTFS_IR_LOGGER_LEVEL=info +FROSTFS_IR_LOGGER_TIMESTAMP=true FROSTFS_IR_WALLET_PATH=/path/to/wallet.json FROSTFS_IR_WALLET_ADDRESS=NUHtW3eM6a4mmFCgyyr4rj4wygsTKB88XX diff --git a/config/example/ir.yaml b/config/example/ir.yaml index 401328e72..4c64f088b 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -2,6 +2,7 @@ logger: level: info # Logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" + timestamp: true wallet: path: /path/to/wallet.json # Path to NEP-6 NEO wallet file diff --git a/config/example/node.env b/config/example/node.env index 030a79934..6618a981a 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -1,5 +1,6 @@ FROSTFS_LOGGER_LEVEL=debug FROSTFS_LOGGER_DESTINATION=journald +FROSTFS_LOGGER_TIMESTAMP=true FROSTFS_PPROF_ENABLED=true FROSTFS_PPROF_ADDRESS=localhost:6060 diff --git a/config/example/node.json b/config/example/node.json index 4e6d239fe..0d100ed80 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -1,7 +1,8 @@ { "logger": { "level": "debug", - "destination": "journald" + "destination": "journald", + "timestamp": true }, "pprof": { "enabled": true, diff --git a/config/example/node.yaml b/config/example/node.yaml index 5a8e6a2a4..86be35ba8 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -1,6 +1,7 @@ logger: level: debug # logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" destination: journald # logger destination: one of "stdout" (default), "journald" + timestamp: true systemdnotify: enabled: true From b807d8c40066a90accf7dbdb5d8f31f6069f551c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 18 Sep 2024 12:20:55 +0300 Subject: [PATCH 1359/1943] [#1382] go.mod: Upgrade sdk-go and api-go versions Signed-off-by: Dmitrii Stepanov --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 621d2e85d..78dce0131 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240902111049-c11f50efeccb + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240903093628-8f751d9dd0ad + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240916123151-6009d089fc69 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index 4d21d9bca..dd0e31088 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240902111049-c11f50efeccb h1:p9ByDsw+H6p6LyYSx8LKFtAG/oPKQpDVMNfjPqdevTw= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240902111049-c11f50efeccb/go.mod h1:BDnEpkKMykCS8u1nLzR6SgNzCv6885RWlo5TnravQuI= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e h1:740ABnOBYx4o6jxULHdSSnVW2fYIO35ohg+Uz59sxd0= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240903093628-8f751d9dd0ad h1:cJGK/bXSF/0KMr6zkIy06qekQhQRU7eYzh+lWdQ0U8w= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240903093628-8f751d9dd0ad/go.mod h1:I4GzeEnftO++N2JHQn9QR88eV0rjQkGqlUCNMnCz1CY= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240916123151-6009d089fc69 h1:DJExzndXf6hztcQ8zHlBOJV/+FA6k2FpRGUcTDWqq2M= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240916123151-6009d089fc69/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= From 1e7f9909dade3ed905c07930c9a9f1bd9a8323b4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 18 Sep 2024 12:21:53 +0300 Subject: [PATCH 1360/1943] [#1382] policer: Replace deprecated methods Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/check.go | 2 +- pkg/services/policer/ec.go | 2 +- pkg/services/policer/policer_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index bf67ec4d4..06282bd8d 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -110,7 +110,7 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe requirements.needLocalCopy = true shortage-- - } else if nodes[i].IsMaintenance() { + } else if nodes[i].Status().IsMaintenance() { shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) } else { if status := checkedNodes.processStatus(nodes[i]); status.Processed() { diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index 61a65fc21..e822d1c09 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -106,7 +106,7 @@ func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, n validPlacement: true, } } - if requiredNode.IsMaintenance() { + if requiredNode.Status().IsMaintenance() { // consider maintenance mode has object, but do not drop local copy p.log.Debug(logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(requiredNode))) return ecChunkProcessResult{} diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index e353ea428..4e17e98a8 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -174,7 +174,7 @@ func TestProcessObject(t *testing.T) { nodes[i].SetPublicKey([]byte{byte(i)}) } for _, i := range ti.maintenanceNodes { - nodes[i].SetMaintenance() + nodes[i].SetStatus(netmap.Maintenance) } var policy netmap.PlacementPolicy From e5c8f7ff9f49b9e8d0f0a7ac4290aadeba356d6c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 18 Sep 2024 12:22:38 +0300 Subject: [PATCH 1361/1943] [#1382] controlSvc: Replace deprecated methods Signed-off-by: Dmitrii Stepanov --- pkg/services/control/ir/server/calls.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 2447a8a74..642932c91 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -91,7 +91,7 @@ func (s *Server) RemoveNode(_ context.Context, req *control.RemoveNodeRequest) ( if len(nodeInfo.PublicKey()) == 0 { return nil, status.Error(codes.NotFound, "no such node") } - if nodeInfo.IsOffline() { + if nodeInfo.Status().IsOffline() { return nil, status.Error(codes.FailedPrecondition, "node is already offline") } From d4be2f20d4a240dd5a46f09b07b432667ca52f24 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 18 Sep 2024 12:23:46 +0300 Subject: [PATCH 1362/1943] [#1382] morph: Replace deprecated methods Signed-off-by: Dmitrii Stepanov --- pkg/morph/client/netmap/netmap.go | 6 +++--- pkg/morph/client/netmap/netmap_test.go | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/morph/client/netmap/netmap.go b/pkg/morph/client/netmap/netmap.go index 61bbf5f17..f7b5c3ba4 100644 --- a/pkg/morph/client/netmap/netmap.go +++ b/pkg/morph/client/netmap/netmap.go @@ -136,11 +136,11 @@ func decodeNodeInfo(dst *netmap.NodeInfo, itemNode stackitem.Item) error { default: return fmt.Errorf("unsupported state %v", node.State) case netmapcontract.NodeStateOnline: - dst.SetOnline() + dst.SetStatus(netmap.Online) case netmapcontract.NodeStateOffline: - dst.SetOffline() + dst.SetStatus(netmap.Offline) case netmapcontract.NodeStateMaintenance: - dst.SetMaintenance() + dst.SetStatus(netmap.Maintenance) } return nil diff --git a/pkg/morph/client/netmap/netmap_test.go b/pkg/morph/client/netmap/netmap_test.go index a8a306197..e686e271e 100644 --- a/pkg/morph/client/netmap/netmap_test.go +++ b/pkg/morph/client/netmap/netmap_test.go @@ -20,11 +20,11 @@ func Test_stackItemsToNodeInfos(t *testing.T) { switch i % 3 { default: - expected[i].SetOffline() + expected[i].SetStatus(netmap.Offline) case int(netmapcontract.NodeStateOnline): - expected[i].SetOnline() + expected[i].SetStatus(netmap.Online) case int(netmapcontract.NodeStateMaintenance): - expected[i].SetMaintenance() + expected[i].SetStatus(netmap.Maintenance) } expected[i].SetPublicKey(pub) @@ -38,12 +38,12 @@ func Test_stackItemsToNodeInfos(t *testing.T) { var state int64 - switch { - case expected[i].IsOnline(): + switch expected[i].Status() { + case netmap.Online: state = int64(netmapcontract.NodeStateOnline) - case expected[i].IsOffline(): + case netmap.Offline: state = int64(netmapcontract.NodeStateOffline) - case expected[i].IsMaintenance(): + case netmap.Maintenance: state = int64(netmapcontract.NodeStateMaintenance) } From a603d14d080e2485fdedee4b92306b1ce4aee2b0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 18 Sep 2024 12:24:53 +0300 Subject: [PATCH 1363/1943] [#1382] ir: Replace deprecated methods Signed-off-by: Dmitrii Stepanov --- pkg/innerring/processors/netmap/cleanup_table.go | 2 +- pkg/innerring/processors/netmap/cleanup_table_test.go | 2 +- pkg/innerring/processors/netmap/handlers_test.go | 2 +- .../processors/netmap/nodevalidation/state/validator.go | 4 ++-- .../netmap/nodevalidation/state/validator_test.go | 8 ++++---- pkg/innerring/processors/netmap/process_peers.go | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/innerring/processors/netmap/cleanup_table.go b/pkg/innerring/processors/netmap/cleanup_table.go index c18611569..abd5b089a 100644 --- a/pkg/innerring/processors/netmap/cleanup_table.go +++ b/pkg/innerring/processors/netmap/cleanup_table.go @@ -60,7 +60,7 @@ func (c *cleanupTable) update(snapshot netmap.NetMap, now uint64) { } access.binNodeInfo = binNodeInfo - access.maintenance = nmNodes[i].IsMaintenance() + access.maintenance = nmNodes[i].Status().IsMaintenance() newMap[keyString] = access } diff --git a/pkg/innerring/processors/netmap/cleanup_table_test.go b/pkg/innerring/processors/netmap/cleanup_table_test.go index ae5620733..208bd5496 100644 --- a/pkg/innerring/processors/netmap/cleanup_table_test.go +++ b/pkg/innerring/processors/netmap/cleanup_table_test.go @@ -127,7 +127,7 @@ func TestCleanupTable(t *testing.T) { t.Run("skip maintenance nodes", func(t *testing.T) { cnt := 0 - infos[1].SetMaintenance() + infos[1].SetStatus(netmap.Maintenance) key := netmap.StringifyPublicKey(infos[1]) c.update(networkMap, 5) diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index b34abb78c..8875880bf 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -146,7 +146,7 @@ func TestAddPeer(t *testing.T) { require.Nil(t, nc.notaryInvokes, "invalid notary invokes") - node.SetOnline() + node.SetStatus(netmap.Online) ev = netmapEvent.AddPeer{ NodeBytes: node.Marshal(), Request: &payload.P2PNotaryRequest{ diff --git a/pkg/innerring/processors/netmap/nodevalidation/state/validator.go b/pkg/innerring/processors/netmap/nodevalidation/state/validator.go index 4094e50a5..e5165f618 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/state/validator.go +++ b/pkg/innerring/processors/netmap/nodevalidation/state/validator.go @@ -56,11 +56,11 @@ func (x *NetMapCandidateValidator) SetNetworkSettings(netSettings NetworkSetting // // See also netmap.NodeInfo.IsOnline/SetOnline and other similar methods. func (x *NetMapCandidateValidator) VerifyAndUpdate(node *netmap.NodeInfo) error { - if node.IsOnline() { + if node.Status().IsOnline() { return nil } - if node.IsMaintenance() { + if node.Status().IsMaintenance() { return x.netSettings.MaintenanceModeAllowed() } diff --git a/pkg/innerring/processors/netmap/nodevalidation/state/validator_test.go b/pkg/innerring/processors/netmap/nodevalidation/state/validator_test.go index a557628f0..b81d7243b 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/state/validator_test.go +++ b/pkg/innerring/processors/netmap/nodevalidation/state/validator_test.go @@ -41,22 +41,22 @@ func TestValidator_VerifyAndUpdate(t *testing.T) { }, { name: "ONLINE", - preparer: (*netmap.NodeInfo).SetOnline, + preparer: func(ni *netmap.NodeInfo) { ni.SetStatus(netmap.Online) }, valid: true, }, { name: "OFFLINE", - preparer: (*netmap.NodeInfo).SetOffline, + preparer: func(ni *netmap.NodeInfo) { ni.SetStatus(netmap.Offline) }, valid: false, }, { name: "MAINTENANCE/allowed", - preparer: (*netmap.NodeInfo).SetMaintenance, + preparer: func(ni *netmap.NodeInfo) { ni.SetStatus(netmap.Maintenance) }, valid: true, }, { name: "MAINTENANCE/disallowed", - preparer: (*netmap.NodeInfo).SetMaintenance, + preparer: func(ni *netmap.NodeInfo) { ni.SetStatus(netmap.Maintenance) }, valid: false, validatorPreparer: func(v *state.NetMapCandidateValidator) { var s testNetworkSettings diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index 9e6e8c283..c8c7928a3 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -62,7 +62,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { // But there is no guarantee that code will be executed in the same order. // That is why we need to perform `addPeerIR` only in case when node is online, // because in scope of this method, contract set state `ONLINE` for the node. - if updated && nodeInfo.IsOnline() { + if updated && nodeInfo.Status().IsOnline() { np.log.Info(logs.NetmapApprovingNetworkMapCandidate, zap.String("key", keyString)) From ac1eee091dfbb9193c407ac237cd53a26f4f83d9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 18 Sep 2024 12:27:10 +0300 Subject: [PATCH 1364/1943] [#1382] node: Replace deprecated methods Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 8 ++++++-- cmd/frostfs-node/netmap.go | 26 +++++++++++++++----------- cmd/internal/common/netmap.go | 8 ++++---- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index ed3a65c25..63f410b89 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1182,7 +1182,9 @@ func (c *cfg) bootstrapWithState(stateSetter func(*netmap.NodeInfo)) error { // bootstrapOnline calls cfg.bootstrapWithState with "online" state. func bootstrapOnline(c *cfg) error { - return c.bootstrapWithState((*netmap.NodeInfo).SetOnline) + return c.bootstrapWithState(func(ni *netmap.NodeInfo) { + ni.SetStatus(netmap.Online) + }) } // bootstrap calls bootstrapWithState with: @@ -1193,7 +1195,9 @@ func (c *cfg) bootstrap() error { st := c.cfgNetmap.state.controlNetmapStatus() if st == control.NetmapStatus_MAINTENANCE { c.log.Info(logs.FrostFSNodeBootstrappingWithTheMaintenanceState) - return c.bootstrapWithState((*netmap.NodeInfo).SetMaintenance) + return c.bootstrapWithState(func(ni *netmap.NodeInfo) { + ni.SetStatus(netmap.Maintenance) + }) } c.log.Info(logs.FrostFSNodeBootstrappingWithOnlineState, diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 8104b1dc1..c0b87492c 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -61,13 +61,15 @@ func (s *networkState) setNodeInfo(ni *netmapSDK.NodeInfo) { if ni != nil { s.nodeInfo.Store(*ni) - switch { - case ni.IsOnline(): + switch ni.Status() { + case netmapSDK.Online: ctrlNetSt = control.NetmapStatus_ONLINE - case ni.IsOffline(): + case netmapSDK.Offline: ctrlNetSt = control.NetmapStatus_OFFLINE - case ni.IsMaintenance(): + case netmapSDK.Maintenance: ctrlNetSt = control.NetmapStatus_MAINTENANCE + case netmapSDK.UnspecifiedState: + ctrlNetSt = control.NetmapStatus_STATUS_UNDEFINED } } else { ctrlNetSt = control.NetmapStatus_OFFLINE @@ -78,7 +80,7 @@ func (s *networkState) setNodeInfo(ni *netmapSDK.NodeInfo) { // nil ni means that the node is not included // in the netmap - niOld.SetOffline() + niOld.SetStatus(netmapSDK.Offline) s.nodeInfo.Store(niOld) } @@ -139,7 +141,7 @@ func initNetmapService(ctx context.Context, c *cfg) { network.WriteToNodeInfo(c.localAddr, &c.cfgNodeInfo.localInfo) c.cfgNodeInfo.localInfo.SetPublicKey(c.key.PublicKey().Bytes()) parseAttributes(c) - c.cfgNodeInfo.localInfo.SetOffline() + c.cfgNodeInfo.localInfo.SetStatus(netmapSDK.Offline) if c.cfgMorph.client == nil { initMorphComponents(ctx, c) @@ -252,7 +254,7 @@ func initNetmapState(c *cfg) { zap.String("state", stateWord), ) - if ni != nil && ni.IsMaintenance() { + if ni != nil && ni.Status().IsMaintenance() { c.isMaintenance.Store(true) } @@ -263,13 +265,15 @@ func initNetmapState(c *cfg) { func nodeState(ni *netmapSDK.NodeInfo) string { if ni != nil { - switch { - case ni.IsOnline(): + switch ni.Status() { + case netmapSDK.Online: return "online" - case ni.IsOffline(): + case netmapSDK.Offline: return "offline" - case ni.IsMaintenance(): + case netmapSDK.Maintenance: return "maintenance" + case netmapSDK.UnspecifiedState: + return "undefined" } } return "undefined" diff --git a/cmd/internal/common/netmap.go b/cmd/internal/common/netmap.go index 79b03a726..f550552d2 100644 --- a/cmd/internal/common/netmap.go +++ b/cmd/internal/common/netmap.go @@ -14,14 +14,14 @@ func PrettyPrintNodeInfo(cmd *cobra.Command, node netmap.NodeInfo, ) { var strState string - switch { + switch node.Status() { default: strState = "STATE_UNSUPPORTED" - case node.IsOnline(): + case netmap.Online: strState = "ONLINE" - case node.IsOffline(): + case netmap.Offline: strState = "OFFLINE" - case node.IsMaintenance(): + case netmap.Maintenance: strState = "MAINTENANCE" } From 3441fff05dd61647e7bd069db34f320e7e9efe9a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 18 Sep 2024 12:27:39 +0300 Subject: [PATCH 1365/1943] [#1382] cli: Replace deprecated methods Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/netmap/nodeinfo.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-cli/modules/netmap/nodeinfo.go b/cmd/frostfs-cli/modules/netmap/nodeinfo.go index b6ec48f35..ae4bb329a 100644 --- a/cmd/frostfs-cli/modules/netmap/nodeinfo.go +++ b/cmd/frostfs-cli/modules/netmap/nodeinfo.go @@ -49,14 +49,14 @@ func prettyPrintNodeInfo(cmd *cobra.Command, i netmap.NodeInfo) { cmd.Println("key:", hex.EncodeToString(i.PublicKey())) var stateWord string - switch { + switch i.Status() { default: stateWord = "" - case i.IsOnline(): + case netmap.Online: stateWord = "online" - case i.IsOffline(): + case netmap.Offline: stateWord = "offline" - case i.IsMaintenance(): + case netmap.Maintenance: stateWord = "maintenance" } From 61d5e140e051f92222fa9152d6bd807d505ca1e8 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 18 Sep 2024 12:13:15 +0300 Subject: [PATCH 1366/1943] [#1383] object: Add restrictions for `Patch` method * `Patch` can't be applied for non-regular type object (tombstones, locks etc.) * Complex object parts can't be patched. So, if an object has EC/Split header, it won't be patched. Signed-off-by: Airat Arifullin --- pkg/services/object/patch/streamer.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/services/object/patch/streamer.go b/pkg/services/object/patch/streamer.go index 85c28cda0..73def8c7c 100644 --- a/pkg/services/object/patch/streamer.go +++ b/pkg/services/object/patch/streamer.go @@ -57,12 +57,31 @@ func toFullObjectHeader(hdr *objectSDK.Object) objectV2.GetHeaderPart { return hs } +func isLinkObject(hdr *objectV2.HeaderWithSignature) bool { + split := hdr.GetHeader().GetSplit() + return len(split.GetChildren()) > 0 && split.GetParent() != nil +} + +func isComplexObjectPart(hdr *objectV2.HeaderWithSignature) bool { + return hdr.GetHeader().GetEC() != nil || hdr.GetHeader().GetSplit() != nil +} + func (s *Streamer) init(ctx context.Context, req *objectV2.PatchRequest) error { hdrWithSig, addr, err := s.readHeader(ctx, req) if err != nil { return err } + if hdrWithSig.GetHeader().GetObjectType() != objectV2.TypeRegular { + return errors.New("non-regular object can't be patched") + } + if isLinkObject(hdrWithSig) { + return errors.New("linking object can't be patched") + } + if isComplexObjectPart(hdrWithSig) { + return errors.New("complex object parts can't be patched") + } + commonPrm, err := util.CommonPrmFromV2(req) if err != nil { return err From 945b7c740b0deb4a2f16bb85f20efd8820762f53 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 18 Sep 2024 18:14:54 +0300 Subject: [PATCH 1367/1943] [#1372] adm/morph: Add delta flag to 'force-new-epoch' Signed-off-by: Alexander Chuprov --- cmd/frostfs-adm/internal/commonflags/flags.go | 1 + cmd/frostfs-adm/internal/modules/morph/helper/netmap.go | 8 ++++++-- cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go | 3 ++- cmd/frostfs-adm/internal/modules/morph/netmap/root.go | 2 ++ cmd/frostfs-adm/internal/modules/morph/node/remove.go | 2 +- 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-adm/internal/commonflags/flags.go b/cmd/frostfs-adm/internal/commonflags/flags.go index 81395edb0..b51d2e115 100644 --- a/cmd/frostfs-adm/internal/commonflags/flags.go +++ b/cmd/frostfs-adm/internal/commonflags/flags.go @@ -39,4 +39,5 @@ const ( CustomZoneFlag = "domain" AlphabetSizeFlag = "size" AllFlag = "all" + DeltaFlag = "delta" ) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go index 7a778f8c3..fb8f03783 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go @@ -72,13 +72,17 @@ func InvalidConfigValueErr(key string) error { return fmt.Errorf("invalid %s config value from netmap contract", key) } -func EmitNewEpochCall(bw *io.BufBinWriter, wCtx *InitializeContext, nmHash util.Uint160) error { +func EmitNewEpochCall(bw *io.BufBinWriter, wCtx *InitializeContext, nmHash util.Uint160, countEpoch int64) error { + if countEpoch <= 0 { + return errors.New("number of epochs cannot be less than 1") + } + curr, err := unwrap.Int64(wCtx.ReadOnlyInvoker.Call(nmHash, "epoch")) if err != nil { return errors.New("can't fetch current epoch from the netmap contract") } - newEpoch := curr + 1 + newEpoch := curr + countEpoch wCtx.Command.Printf("Current epoch: %d, increase to %d.\n", curr, newEpoch) // In NeoFS this is done via Notary contract. Here, however, we can form the diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go index df9a03fd1..5e4e9c725 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/nspcc-dev/neo-go/pkg/io" @@ -30,7 +31,7 @@ func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { } bw := io.NewBufBinWriter() - if err := helper.EmitNewEpochCall(bw, wCtx, nmHash); err != nil { + if err := helper.EmitNewEpochCall(bw, wCtx, nmHash, viper.GetInt64(commonflags.DeltaFlag)); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go index 31fda860e..0288bcdc5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go @@ -22,6 +22,7 @@ var ( PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.DeltaFlag, cmd.Flags().Lookup(commonflags.DeltaFlag)) }, RunE: ForceNewEpochCmd, } @@ -35,6 +36,7 @@ func initForceNewEpochCmd() { ForceNewEpoch.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) ForceNewEpoch.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) ForceNewEpoch.Flags().String(commonflags.LocalDumpFlag, "", "Path to the blocks dump file") + ForceNewEpoch.Flags().Int64(commonflags.DeltaFlag, 1, "Number of epochs to increase the current epoch") } func init() { diff --git a/cmd/frostfs-adm/internal/modules/morph/node/remove.go b/cmd/frostfs-adm/internal/modules/morph/node/remove.go index 0a19102ba..e47451e0c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/node/remove.go +++ b/cmd/frostfs-adm/internal/modules/morph/node/remove.go @@ -53,7 +53,7 @@ func RemoveNodesCmd(cmd *cobra.Command, args []string) error { int64(netmapcontract.NodeStateOffline), nodeKeys[i].Bytes()) } - if err := helper.EmitNewEpochCall(bw, wCtx, nmHash); err != nil { + if err := helper.EmitNewEpochCall(bw, wCtx, nmHash, 1); err != nil { return err } From 1361db91ee37d3da938dc5146cc3f15f9ee33517 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 16 Sep 2024 11:09:51 +0300 Subject: [PATCH 1368/1943] [#1301] adm/morph: Add flag -v to 'Tokens' Signed-off-by: Alexander Chuprov --- .../internal/modules/morph/nns/tokens.go | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/tokens.go b/cmd/frostfs-adm/internal/modules/morph/nns/tokens.go index 6e8ffb40a..3c7136e9d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/tokens.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/tokens.go @@ -1,15 +1,25 @@ package nns import ( + "math/big" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + client "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/spf13/cobra" ) +const ( + verboseDesc = "Include additional information about CNAME record." +) + func initTokensCmd() { Cmd.AddCommand(tokensCmd) tokensCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) tokensCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + tokensCmd.Flags().BoolP(commonflags.Verbose, commonflags.VerboseShorthand, false, verboseDesc) } func listTokens(cmd *cobra.Command, _ []string) { @@ -18,7 +28,39 @@ func listTokens(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "unable to get tokens: %w", err) for toks, err := it.Next(10); err == nil && len(toks) > 0; toks, err = it.Next(10) { for _, token := range toks { - cmd.Println(string(token)) + output := string(token) + if verbose, _ := cmd.Flags().GetBool(commonflags.Verbose); verbose { + cname, err := getCnameRecord(c, token) + commonCmd.ExitOnErr(cmd, "", err) + if cname != "" { + output += " (CNAME: " + cname + ")" + } + } + cmd.Println(output) } } } + +func getCnameRecord(c *client.Contract, token []byte) (string, error) { + items, err := c.GetRecords(string(token), big.NewInt(int64(nns.CNAME))) + + // GetRecords returns the error "not an array" if the domain does not contain records. + if err != nil && strings.Contains(err.Error(), "not an array") { + return "", nil + } + + if err != nil { + return "", err + } + + if len(items) == 0 { + return "", nil + } + + record, err := items[0].TryBytes() + if err != nil { + return "", err + } + + return string(record), nil +} From 5a53f9c4fd52243dd36c69e62d79f344342d4349 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 19 Sep 2024 14:19:16 +0300 Subject: [PATCH 1369/1943] [#1301] go.mod: Bump frostfs-contract Signed-off-by: Alexander Chuprov --- go.mod | 6 +++--- go.sum | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 78dce0131..502761866 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e - git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e + git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 @@ -28,7 +28,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.12.1 - github.com/nspcc-dev/neo-go v0.106.2 + github.com/nspcc-dev/neo-go v0.106.3 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 github.com/prometheus/client_golang v1.19.0 @@ -100,7 +100,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 // indirect - github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d // indirect + github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240727093519-1a48f1ce43ec // indirect github.com/nspcc-dev/rfc6979 v0.2.1 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect diff --git a/go.sum b/go.sum index dd0e31088..85d9df443 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e h1:740ABnOBYx4o6jxULHdSSnVW2fYIO35ohg+Uz59sxd0= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c h1:9JmXxWcqWaOXModBwtj/WlPYOfSk2jRSDoRsDRYvgnA= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c/go.mod h1:DVb1AqP323wiEwCtlfoJJTERF4Xji2L39X6T8kv26RE= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d h1:uJ/wvuMdepbkaV8XMS5uN9B0FQWMep0CttSuDZiDhq0= @@ -188,8 +188,8 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2 h1:mD9hU3v+zJcnHAVmHnZKt3I++tvn30gBj2rP2PocZMk= github.com/nspcc-dev/go-ordered-json v0.0.0-20240301084351-0246b013f8b2/go.mod h1:U5VfmPNM88P4RORFb6KSUVBdJBDhlqggJZYGXGPxOcc= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d h1:Vcb7YkZuUSSIC+WF/xV3UDfHbAxZgyT2zGleJP3Ig5k= -github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240521091047-78685785716d/go.mod h1:/vrbWSHc7YS1KSYhVOyyeucXW/e+1DkVBOgnBEXUCeY= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240727093519-1a48f1ce43ec h1:vDrbVXF2+2uP0RlkZmem3QYATcXCu9BzzGGCNsNcK7Q= +github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240727093519-1a48f1ce43ec/go.mod h1:/vrbWSHc7YS1KSYhVOyyeucXW/e+1DkVBOgnBEXUCeY= github.com/nspcc-dev/rfc6979 v0.2.1 h1:8wWxkamHWFmO790GsewSoKUSJjVnL1fmdRpokU/RgRM= github.com/nspcc-dev/rfc6979 v0.2.1/go.mod h1:Tk7h5kyUWkhjyO3zUgFFhy1v2vQv3BvQEntakdtqrWc= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= From 53a90634fc0a55be636a220b461be731f8a91b3f Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 19 Sep 2024 14:19:41 +0300 Subject: [PATCH 1370/1943] [#1301] adm/morph: Add 'delete' domains Signed-off-by: Alexander Chuprov --- .../morph/nns/{register.go => domains.go} | 20 +++++++++++++++++++ .../internal/modules/morph/nns/root.go | 10 ++++++++++ 2 files changed, 30 insertions(+) rename cmd/frostfs-adm/internal/modules/morph/nns/{register.go => domains.go} (73%) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/register.go b/cmd/frostfs-adm/internal/modules/morph/nns/domains.go similarity index 73% rename from cmd/frostfs-adm/internal/modules/morph/nns/register.go rename to cmd/frostfs-adm/internal/modules/morph/nns/domains.go index d05d9f171..3684db94a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/register.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/domains.go @@ -42,3 +42,23 @@ func registerDomain(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "register domain error: %w", err) cmd.Println("Domain registered successfully") } + +func initDeleteCmd() { + Cmd.AddCommand(deleteCmd) + deleteCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + deleteCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + deleteCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) + + _ = cobra.MarkFlagRequired(deleteCmd.Flags(), nnsNameFlag) +} + +func deleteDomain(cmd *cobra.Command, _ []string) { + c, actor, _ := getRPCClient(cmd) + + name, _ := cmd.Flags().GetString(nnsNameFlag) + h, vub, err := c.DeleteDomain(name) + + _, err = actor.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "delete domain error: %w", err) + cmd.Println("Domain deleted successfully") +} diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/root.go b/cmd/frostfs-adm/internal/modules/morph/nns/root.go index e528e4b7b..56774c292 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/root.go @@ -42,6 +42,15 @@ var ( }, Run: registerDomain, } + deleteCmd = &cobra.Command{ + Use: "delete", + Short: "Delete a domain by name", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + }, + Run: deleteDomain, + } renewCmd = &cobra.Command{ Use: "renew", Short: "Increases domain expiration date", @@ -91,6 +100,7 @@ var ( func init() { initTokensCmd() initRegisterCmd() + initDeleteCmd() initRenewCmd() initUpdateCmd() initAddRecordCmd() From c290d079fd71ddc851cce8d06f496d27ceedc168 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 20 Sep 2024 10:53:02 +0300 Subject: [PATCH 1371/1943] [#1312] go.mod: Update sdk-go Signed-off-by: Aleksey Savchuk --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 502761866..9817f8527 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240916123151-6009d089fc69 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index 85d9df443..3c6dd9a99 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240916123151-6009d089fc69 h1:DJExzndXf6hztcQ8zHlBOJV/+FA6k2FpRGUcTDWqq2M= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240916123151-6009d089fc69/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 h1:ijUci3thz0EwWkuRJDocW5D1RkVAJlt9xNG4CYepC90= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= From c34b8acedde282bbe81efccea772a923ee570a8f Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 20 Sep 2024 10:58:22 +0300 Subject: [PATCH 1372/1943] [#1312] Drop handling of system attributes with NeoFS prefix Signed-off-by: Aleksey Savchuk --- cmd/frostfs-cli/modules/container/list_objects.go | 13 ++----------- pkg/core/object/fmt.go | 2 +- pkg/local_object_storage/metabase/put.go | 4 ---- pkg/local_object_storage/metabase/upgrade.go | 2 +- pkg/services/object_manager/tombstone/checker.go | 2 +- 5 files changed, 5 insertions(+), 18 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/list_objects.go b/cmd/frostfs-cli/modules/container/list_objects.go index ff2f8cf45..d5850359d 100644 --- a/cmd/frostfs-cli/modules/container/list_objects.go +++ b/cmd/frostfs-cli/modules/container/list_objects.go @@ -1,9 +1,6 @@ package container import ( - "strings" - - v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" @@ -67,14 +64,8 @@ var listContainerObjectsCmd = &cobra.Command{ resHead, err := internalclient.HeadObject(cmd.Context(), prmHead) if err == nil { - attrs := resHead.Header().Attributes() - for i := range attrs { - attrKey := attrs[i].Key() - if !strings.HasPrefix(attrKey, v2object.SysAttributePrefix) && !strings.HasPrefix(attrKey, v2object.SysAttributePrefixNeoFS) { - // FIXME(@cthulhu-rider): https://git.frostfs.info/TrueCloudLab/frostfs-sdk-go/issues/97 - // Use dedicated method to skip system attributes. - cmd.Printf(" %s: %s\n", attrKey, attrs[i].Value()) - } + for _, attr := range resHead.Header().UserAttributes() { + cmd.Printf(" %s: %s\n", attr.Key(), attr.Value()) } } else { cmd.Printf(" failed to read attributes: %v\n", err) diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index 96f721806..317d62cb0 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -361,7 +361,7 @@ func (v *FormatValidator) checkIfExpired(ctx context.Context, obj *objectSDK.Obj func expirationEpochAttribute(obj *objectSDK.Object) (uint64, error) { for _, a := range obj.Attributes() { - if a.Key() != objectV2.SysAttributeExpEpoch && a.Key() != objectV2.SysAttributeExpEpochNeoFS { + if a.Key() != objectV2.SysAttributeExpEpoch { continue } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index ff79a0387..087529895 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -387,10 +387,6 @@ func hasExpirationEpoch(obj *objectSDK.Object) (uint64, bool) { attributes = ech.ParentAttributes() } for _, attr := range attributes { - if attr.Key() == objectV2.SysAttributeExpEpochNeoFS { - expEpoch, err := strconv.ParseUint(attr.Value(), 10, 64) - return expEpoch, err == nil - } if attr.Key() == objectV2.SysAttributeExpEpoch { expEpoch, err := strconv.ParseUint(attr.Value(), 10, 64) return expEpoch, err == nil diff --git a/pkg/local_object_storage/metabase/upgrade.go b/pkg/local_object_storage/metabase/upgrade.go index f677dcf8e..b5de430dc 100644 --- a/pkg/local_object_storage/metabase/upgrade.go +++ b/pkg/local_object_storage/metabase/upgrade.go @@ -252,7 +252,7 @@ func getNextObjectsWithExpirationEpochBatch(ctx context.Context, db *bbolt.DB, i continue } attributeKey := string(attrKey[1+cidSize:]) - if attributeKey != objectV2.SysAttributeExpEpochNeoFS && attributeKey != objectV2.SysAttributeExpEpoch { + if attributeKey != objectV2.SysAttributeExpEpoch { continue } var containerID cid.ID diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go index c3c810001..48a08b693 100644 --- a/pkg/services/object_manager/tombstone/checker.go +++ b/pkg/services/object_manager/tombstone/checker.go @@ -74,7 +74,7 @@ func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Addr func (g *ExpirationChecker) handleTS(addr string, ts *objectSDK.Object, reqEpoch uint64) bool { for _, atr := range ts.Attributes() { - if atr.Key() == objectV2.SysAttributeExpEpoch || atr.Key() == objectV2.SysAttributeExpEpochNeoFS { + if atr.Key() == objectV2.SysAttributeExpEpoch { epoch, err := strconv.ParseUint(atr.Value(), 10, 64) if err != nil { g.log.Warn( From f71418b73cfb49306ec1a191621b954a75105b18 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Fri, 20 Sep 2024 10:24:40 +0000 Subject: [PATCH 1373/1943] [#1386] frostfs-adm: Add info to error messages These error messages bubble up to human users - adding more context helps to find the cause of the issue faster. Signed-off-by: Vitaliy Potyarkin --- .../modules/morph/initialize/initialize_roles.go | 10 ++++++++-- .../morph/initialize/initialize_transfer.go | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_roles.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_roles.go index a6815ee13..05bc83a8b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_roles.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_roles.go @@ -1,6 +1,8 @@ package initialize import ( + "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" "github.com/nspcc-dev/neo-go/pkg/io" @@ -29,10 +31,14 @@ func setNotaryAndAlphabetNodes(c *helper.InitializeContext) error { callflag.States|callflag.AllowNotify, int64(noderoles.NeoFSAlphabet), pubs) if err := c.SendCommitteeTx(w.Bytes(), false); err != nil { - return err + return fmt.Errorf("send committee transaction: %w", err) } - return c.AwaitTx() + err := c.AwaitTx() + if err != nil { + err = fmt.Errorf("await committee transaction: %w", err) + } + return err } func setRolesFinished(c *helper.InitializeContext) (bool, error) { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go index b7102fa13..d7b0ec86c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go @@ -3,6 +3,7 @@ package initialize import ( "fmt" "math/big" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" @@ -144,5 +145,17 @@ func createNEP17MultiTransferTx(c helper.Client, acc *wallet.Account, recipients if err != nil { return nil, fmt.Errorf("can't create actor: %w", err) } - return act.MakeRun(w.Bytes()) + tx, err := act.MakeRun(w.Bytes()) + if err != nil { + sum := make(map[util.Uint160]int64) + for _, recipient := range recipients { + sum[recipient.Token] += recipient.Amount + } + detail := make([]string, 0, len(sum)) + for _, value := range sum { + detail = append(detail, fmt.Sprintf("amount=%v", value)) + } + err = fmt.Errorf("transfer failed: from=%s(%s) %s: %w", acc.Label, acc.Address, strings.Join(detail, " "), err) + } + return tx, err } From 0b87be804a63760fc7e43a51cc1b00c5aeedbb34 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 17 Sep 2024 11:24:48 +0300 Subject: [PATCH 1374/1943] [#1381] engine: Fix tests Drop not required `Eventually` calls. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/delete_test.go | 14 ++++------- pkg/local_object_storage/shard/get_test.go | 23 ++++--------------- pkg/local_object_storage/shard/head_test.go | 19 ++------------- pkg/local_object_storage/shard/inhume_test.go | 2 +- 4 files changed, 11 insertions(+), 47 deletions(-) diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index 9f205fa5d..574250a93 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -3,7 +3,6 @@ package shard import ( "context" "testing" - "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" @@ -58,19 +57,14 @@ func testShard(t *testing.T, hasWriteCache bool, payloadSize int) { _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) - _, err = testGet(t, sh, getPrm, hasWriteCache) + _, err = sh.Get(context.Background(), getPrm) require.NoError(t, err) if hasWriteCache { - sh.FlushWriteCache(context.Background(), FlushWriteCachePrm{ignoreErrors: false}) - require.Eventually(t, func() bool { - _, err = sh.Delete(context.Background(), delPrm) - return err == nil - }, 30*time.Second, 10*time.Millisecond) - } else { - _, err = sh.Delete(context.Background(), delPrm) - require.NoError(t, err) + require.NoError(t, sh.FlushWriteCache(context.Background(), FlushWriteCachePrm{ignoreErrors: false})) } + _, err = sh.Delete(context.Background(), delPrm) + require.NoError(t, err) _, err = sh.Get(context.Background(), getPrm) require.True(t, client.IsErrObjectNotFound(err)) diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index 8a7c6972d..d0eecf74e 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -5,11 +5,9 @@ import ( "context" "errors" "testing" - "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -49,7 +47,7 @@ func testShardGet(t *testing.T, hasWriteCache bool) { getPrm.SetAddress(object.AddressOf(obj)) - res, err := testGet(t, sh, getPrm, hasWriteCache) + res, err := sh.Get(context.Background(), getPrm) require.NoError(t, err) require.Equal(t, obj, res.Object()) }) @@ -67,7 +65,7 @@ func testShardGet(t *testing.T, hasWriteCache bool) { getPrm.SetAddress(object.AddressOf(obj)) - res, err := testGet(t, sh, getPrm, hasWriteCache) + res, err := sh.Get(context.Background(), getPrm) require.NoError(t, err) require.Equal(t, obj, res.Object()) }) @@ -95,13 +93,13 @@ func testShardGet(t *testing.T, hasWriteCache bool) { getPrm.SetAddress(object.AddressOf(child)) - res, err := testGet(t, sh, getPrm, hasWriteCache) + res, err := sh.Get(context.Background(), getPrm) require.NoError(t, err) require.True(t, binaryEqual(child, res.Object())) getPrm.SetAddress(object.AddressOf(parent)) - _, err = testGet(t, sh, getPrm, hasWriteCache) + _, err = sh.Get(context.Background(), getPrm) var si *objectSDK.SplitInfoError require.True(t, errors.As(err, &si)) @@ -115,19 +113,6 @@ func testShardGet(t *testing.T, hasWriteCache bool) { }) } -func testGet(t *testing.T, sh *Shard, getPrm GetPrm, hasWriteCache bool) (GetRes, error) { - res, err := sh.Get(context.Background(), getPrm) - if hasWriteCache { - require.Eventually(t, func() bool { - if client.IsErrObjectNotFound(err) { - res, err = sh.Get(context.Background(), getPrm) - } - return !client.IsErrObjectNotFound(err) - }, time.Second, time.Millisecond*100) - } - return res, err -} - // binary equal is used when object contains empty lists in the structure and // requre.Equal fails on comparing and []{} lists. func binaryEqual(a, b *objectSDK.Object) bool { diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index 1f4631993..c65bbb1e3 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -4,11 +4,9 @@ import ( "context" "errors" "testing" - "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" @@ -46,7 +44,7 @@ func testShardHead(t *testing.T, hasWriteCache bool) { headPrm.SetAddress(object.AddressOf(obj)) - res, err := testHead(t, sh, headPrm, hasWriteCache) + res, err := sh.Head(context.Background(), headPrm) require.NoError(t, err) require.Equal(t, obj.CutPayload(), res.Object()) }) @@ -74,7 +72,7 @@ func testShardHead(t *testing.T, hasWriteCache bool) { var siErr *objectSDK.SplitInfoError - _, err = testHead(t, sh, headPrm, hasWriteCache) + _, err = sh.Head(context.Background(), headPrm) require.True(t, errors.As(err, &siErr)) headPrm.SetAddress(object.AddressOf(parent)) @@ -85,16 +83,3 @@ func testShardHead(t *testing.T, hasWriteCache bool) { require.Equal(t, parent.CutPayload(), head.Object()) }) } - -func testHead(t *testing.T, sh *Shard, headPrm HeadPrm, hasWriteCache bool) (HeadRes, error) { - res, err := sh.Head(context.Background(), headPrm) - if hasWriteCache { - require.Eventually(t, func() bool { - if client.IsErrObjectNotFound(err) { - res, err = sh.Head(context.Background(), headPrm) - } - return !client.IsErrObjectNotFound(err) - }, time.Second, time.Millisecond*100) - } - return res, err -} diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 82754568f..1353d5d94 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -48,7 +48,7 @@ func testShardInhume(t *testing.T, hasWriteCache bool) { _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) - _, err = testGet(t, sh, getPrm, hasWriteCache) + _, err = sh.Get(context.Background(), getPrm) require.NoError(t, err) _, err = sh.Inhume(context.Background(), inhPrm) From d4493a6d082011cfa24df68b41d92c7b905fda27 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 15:07:10 +0300 Subject: [PATCH 1375/1943] [#1390] getSvc: Fix Head EC1.1 If local EC chunk found, but remote node is off, then `HEAD --raw` request returns object not found. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/container.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/services/object/get/container.go b/pkg/services/object/get/container.go index d22b14192..034768c81 100644 --- a/pkg/services/object/get/container.go +++ b/pkg/services/object/get/container.go @@ -26,8 +26,10 @@ func (r *request) executeOnContainer(ctx context.Context) { return } + localStatus := r.status + for { - if r.processCurrentEpoch(ctx) { + if r.processCurrentEpoch(ctx, localStatus) { break } @@ -43,7 +45,7 @@ func (r *request) executeOnContainer(ctx context.Context) { } } -func (r *request) processCurrentEpoch(ctx context.Context) bool { +func (r *request) processCurrentEpoch(ctx context.Context, localStatus int) bool { r.log.Debug(logs.ProcessEpoch, zap.Uint64("number", r.curProcEpoch), ) @@ -56,7 +58,11 @@ func (r *request) processCurrentEpoch(ctx context.Context) bool { ctx, cancel := context.WithCancel(ctx) defer cancel() - r.status = statusUndefined + if localStatus == statusEC { // possible only for raw == true and local == false + r.status = statusEC + } else { + r.status = statusUndefined + } for { addrs := traverser.Next() From 3bb65ba820274a2014b3abfe6e11a98047b2059f Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 24 Sep 2024 11:46:15 +0300 Subject: [PATCH 1376/1943] [#1392] object: Fix target initialization within put streamer * Remove `relay` field from put streamer as it's no longer used; * Fix initialization of `Relay` object writer parameter. Signed-off-by: Airat Arifullin --- pkg/services/object/put/streamer.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index f3803d433..d08e7fafa 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/target" objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" @@ -15,8 +14,6 @@ type Streamer struct { *objectwriter.Config target transformer.ChunkedObjectWriter - - relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error } var errNotInit = errors.New("stream not initialized") @@ -35,7 +32,7 @@ func (p *Streamer) Init(ctx context.Context, prm *PutInitPrm) error { Header: prm.hdr, Container: prm.cnr, TraverseOpts: prm.traverseOpts, - Relay: p.relay, + Relay: prm.relay, } var err error From 839dead226534887ffbb21b07327724e42cd2135 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 16 Sep 2024 12:38:45 +0300 Subject: [PATCH 1377/1943] [#1297] getSvc: Return AccessDenied instead of ObjectNotFound Do not replace the access denied error if it was received earlier. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/remote.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index 163767c43..f2639f8e6 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -41,7 +41,7 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { r.status = statusUndefined if errors.As(err, &errAccessDenied) { r.err = err - } else { + } else if r.err == nil || !errors.As(r.err, &errAccessDenied) { r.err = new(apistatus.ObjectNotFound) } } From bdf386366c4e268d9f151c38c4eb5c837a49ab25 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 16 Sep 2024 12:40:12 +0300 Subject: [PATCH 1378/1943] [#1297] dev: Bump neo-go version Signed-off-by: Dmitrii Stepanov --- dev/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 9d026797c..be1956e65 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -3,7 +3,7 @@ version: "2.4" services: neo-go: - image: nspccdev/neo-go:0.105.0 + image: nspccdev/neo-go:0.106.0 container_name: neo-go command: ["node", "--config-path", "/config", "--privnet", "--debug"] stop_signal: SIGKILL From 34e6a309c6b1cdd4e277f76b63a6b5d01b094115 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 18 Sep 2024 12:15:32 +0300 Subject: [PATCH 1379/1943] [#1356] engine: Evacuate object from shards concurrently Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/control/evacuation.go | 15 +- pkg/local_object_storage/engine/evacuate.go | 270 +++++++++++------- .../engine/evacuate_test.go | 41 ++- pkg/local_object_storage/metabase/list.go | 167 +++++++++++ pkg/local_object_storage/shard/list.go | 65 +++++ pkg/services/control/server/evacuate_async.go | 14 +- pkg/services/control/service.proto | 4 + pkg/services/control/service_frostfs.pb.go | 68 ++++- 8 files changed, 533 insertions(+), 111 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 6fa5ed75c..04a67e5b5 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -21,6 +21,9 @@ const ( noProgressFlag = "no-progress" scopeFlag = "scope" + containerWorkerCountFlag = "container-worker-count" + objectWorkerCountFlag = "object-worker-count" + scopeAll = "all" scopeObjects = "objects" scopeTrees = "trees" @@ -64,12 +67,16 @@ func startEvacuateShard(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag) + containerWorkerCount, _ := cmd.Flags().GetUint32(containerWorkerCountFlag) + objectWorkerCount, _ := cmd.Flags().GetUint32(objectWorkerCountFlag) req := &control.StartShardEvacuationRequest{ Body: &control.StartShardEvacuationRequest_Body{ - Shard_ID: getShardIDList(cmd), - IgnoreErrors: ignoreErrors, - Scope: getEvacuationScope(cmd), + Shard_ID: getShardIDList(cmd), + IgnoreErrors: ignoreErrors, + Scope: getEvacuationScope(cmd), + ContainerWorkerCount: containerWorkerCount, + ObjectWorkerCount: objectWorkerCount, }, } @@ -371,6 +378,8 @@ func initControlStartEvacuationShardCmd() { flags.String(scopeFlag, scopeAll, fmt.Sprintf("Evacuation scope; possible values: %s, %s, %s", scopeTrees, scopeObjects, scopeAll)) flags.Bool(awaitFlag, false, "Block execution until evacuation is completed") flags.Bool(noProgressFlag, false, fmt.Sprintf("Print progress if %s provided", awaitFlag)) + flags.Uint32(containerWorkerCountFlag, 0, "Count of concurrent container evacuation workers") + flags.Uint32(objectWorkerCountFlag, 0, "Count of concurrent object evacuation workers") startEvacuationShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 7bef6edfb..3db556a8f 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -10,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -24,6 +23,16 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +const ( + // containerWorkerCountDefault is a default value of the count of + // concurrent container evacuation workers. + containerWorkerCountDefault = 10 + // objectWorkerCountDefault is a default value of the count of + // concurrent object evacuation workers. + objectWorkerCountDefault = 10 ) var ( @@ -79,6 +88,9 @@ type EvacuateShardPrm struct { IgnoreErrors bool Async bool Scope EvacuateScope + + ContainerWorkerCount uint32 + ObjectWorkerCount uint32 } // EvacuateShardRes represents result of the EvacuateShard operation. @@ -189,8 +201,6 @@ func (p *EvacuateShardRes) DeepCopy() *EvacuateShardRes { return res } -const defaultEvacuateBatchSize = 100 - type pooledShard struct { hashedShard pool util.WorkerPool @@ -242,8 +252,16 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev return nil, err } + var mtx sync.RWMutex + copyShards := func() []pooledShard { + mtx.RLock() + defer mtx.RUnlock() + t := make([]pooledShard, len(shards)) + copy(t, shards) + return t + } eg.Go(func() error { - return e.evacuateShards(egCtx, shardIDs, prm, res, shards, shardsToEvacuate) + return e.evacuateShards(egCtx, shardIDs, prm, res, copyShards, shardsToEvacuate) }) if prm.Async { @@ -261,7 +279,7 @@ func ctxOrBackground(ctx context.Context, background bool) context.Context { } func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, + shards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) error { var err error ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShards", @@ -287,13 +305,39 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p return err } - for _, shardID := range shardIDs { - if err = e.evacuateShard(ctx, shardID, prm, res, shards, shardsToEvacuate); err != nil { - e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) - return err + ctx, cancel, egShard, egContainer, egObject := e.createErrorGroupsForEvacuation(ctx, prm) + continueLoop := true + for i := 0; continueLoop && i < len(shardIDs); i++ { + select { + case <-ctx.Done(): + continueLoop = false + default: + egShard.Go(func() error { + err := e.evacuateShard(ctx, cancel, shardIDs[i], prm, res, shards, shardsToEvacuate, egContainer, egObject) + if err != nil { + cancel(err) + } + return err + }) } } + err = egShard.Wait() + if err != nil { + err = fmt.Errorf("shard error: %w", err) + } + errContainer := egContainer.Wait() + errObject := egObject.Wait() + if errContainer != nil { + err = errors.Join(err, fmt.Errorf("container error: %w", errContainer)) + } + if errObject != nil { + err = errors.Join(err, fmt.Errorf("object error: %w", errObject)) + } + if err != nil { + e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) + return err + } e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, zap.Strings("shard_ids", shardIDs), @@ -309,6 +353,27 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p return nil } +func (e *StorageEngine) createErrorGroupsForEvacuation(ctx context.Context, prm EvacuateShardPrm) ( + context.Context, context.CancelCauseFunc, *errgroup.Group, *errgroup.Group, *errgroup.Group, +) { + operationCtx, cancel := context.WithCancelCause(ctx) + egObject, _ := errgroup.WithContext(operationCtx) + objectWorkerCount := prm.ObjectWorkerCount + if objectWorkerCount == 0 { + objectWorkerCount = objectWorkerCountDefault + } + egObject.SetLimit(int(objectWorkerCount)) + egContainer, _ := errgroup.WithContext(operationCtx) + containerWorkerCount := prm.ContainerWorkerCount + if containerWorkerCount == 0 { + containerWorkerCount = containerWorkerCountDefault + } + egContainer.SetLimit(int(containerWorkerCount)) + egShard, _ := errgroup.WithContext(operationCtx) + + return operationCtx, cancel, egShard, egContainer, egObject +} + func (e *StorageEngine) getTotals(ctx context.Context, prm EvacuateShardPrm, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.getTotals") defer span.End() @@ -335,8 +400,9 @@ func (e *StorageEngine) getTotals(ctx context.Context, prm EvacuateShardPrm, sha return nil } -func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, +func (e *StorageEngine) evacuateShard(ctx context.Context, cancel context.CancelCauseFunc, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, + shards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, + egContainer *errgroup.Group, egObject *errgroup.Group, ) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShard", trace.WithAttributes( @@ -345,11 +411,10 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E defer span.End() if prm.Scope.WithObjects() { - if err := e.evacuateShardObjects(ctx, shardID, prm, res, shards, shardsToEvacuate); err != nil { + if err := e.evacuateShardObjects(ctx, cancel, shardID, prm, res, shards, shardsToEvacuate, egContainer, egObject); err != nil { return err } } - if prm.Scope.WithTrees() && shardsToEvacuate[shardID].PiloramaEnabled() { if err := e.evacuateShardTrees(ctx, shardID, prm, res, shards, shardsToEvacuate); err != nil { return err @@ -359,44 +424,60 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E return nil } -func (e *StorageEngine) evacuateShardObjects(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, +func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context.CancelCauseFunc, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, + shards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, + egContainer *errgroup.Group, egObject *errgroup.Group, ) error { - var listPrm shard.ListWithCursorPrm - listPrm.WithCount(defaultEvacuateBatchSize) - sh := shardsToEvacuate[shardID] - sh.SetEvacuationInProgress(true) - - var c *meta.Cursor - for { - listPrm.WithCursor(c) - - // TODO (@fyrchik): #1731 this approach doesn't work in degraded modes - // because ListWithCursor works only with the metabase. - listRes, err := sh.ListWithCursor(ctx, listPrm) - if err != nil { - if errors.Is(err, meta.ErrEndOfListing) || errors.Is(err, shard.ErrDegradedMode) { - break + var cntPrm shard.IterateOverContainersPrm + cntPrm.Handler = func(ctx context.Context, name []byte, _ cid.ID) error { + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: + } + egContainer.Go(func() error { + var objPrm shard.IterateOverObjectsInContainerPrm + objPrm.BucketName = name + objPrm.Handler = func(ctx context.Context, objInfo *object.Info) error { + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: + } + egObject.Go(func() error { + err := e.evacuateObject(ctx, shardID, objInfo, prm, res, shards, shardsToEvacuate) + if err != nil { + cancel(err) + } + return err + }) + return nil + } + err := sh.IterateOverObjectsInContainer(ctx, objPrm) + if err != nil { + cancel(err) } - e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err - } - - if err = e.evacuateObjects(ctx, sh, listRes.AddressList(), prm, res, shards, shardsToEvacuate); err != nil { - return err - } - - c = listRes.Cursor() + }) + return nil } - return nil + + sh.SetEvacuationInProgress(true) + err := sh.IterateOverContainers(ctx, cntPrm) + if err != nil { + cancel(err) + e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } + return err } func (e *StorageEngine) evacuateShardTrees(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, + getShards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) error { sh := shardsToEvacuate[shardID] + shards := getShards() var listPrm pilorama.TreeListTreesPrm first := true @@ -637,68 +718,65 @@ func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) return shards, nil } -func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, toEvacuate []object.Info, prm EvacuateShardPrm, res *EvacuateShardRes, - shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, +func (e *StorageEngine) evacuateObject(ctx context.Context, shardID string, objInfo *object.Info, prm EvacuateShardPrm, res *EvacuateShardRes, + getShards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, ) error { - ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateObjects", - trace.WithAttributes( - attribute.Int("objects_count", len(toEvacuate)), - )) + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateObjects") defer span.End() - for i := range toEvacuate { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - addr := toEvacuate[i].Address + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: + } - var getPrm shard.GetPrm - getPrm.SetAddress(addr) - getPrm.SkipEvacCheck(true) + shards := getShards() + addr := objInfo.Address - getRes, err := sh.Get(ctx, getPrm) - if err != nil { - if prm.IgnoreErrors { - res.objFailed.Add(1) - continue - } - e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) - return err - } + var getPrm shard.GetPrm + getPrm.SetAddress(addr) + getPrm.SkipEvacCheck(true) - evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), sh, shards, shardsToEvacuate, res) - if err != nil { - return err - } - - if evacuatedLocal { - continue - } - - if prm.ObjectsHandler == nil { - // Do not check ignoreErrors flag here because - // ignoring errors on put make this command kinda useless. - return fmt.Errorf("%w: %s", errPutShard, toEvacuate[i]) - } - - moved, err := prm.ObjectsHandler(ctx, addr, getRes.Object()) - if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) - return err - } - if moved { - res.objEvacuated.Add(1) - } else if prm.IgnoreErrors { + getRes, err := shardsToEvacuate[shardID].Get(ctx, getPrm) + if err != nil { + if prm.IgnoreErrors { res.objFailed.Add(1) - e.log.Warn(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) - } else { - return fmt.Errorf("object %s was not replicated", addr) + return nil } + e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + return err + } + + evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), shardsToEvacuate[shardID], shards, shardsToEvacuate, res) + if err != nil { + return err + } + + if evacuatedLocal { + return nil + } + + if prm.ObjectsHandler == nil { + // Do not check ignoreErrors flag here because + // ignoring errors on put make this command kinda useless. + return fmt.Errorf("%w: %s", errPutShard, objInfo) + } + + moved, err := prm.ObjectsHandler(ctx, addr, getRes.Object()) + if err != nil { + e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + return err + } + if moved { + res.objEvacuated.Add(1) + } else if prm.IgnoreErrors { + res.objFailed.Add(1) + e.log.Warn(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } else { + return fmt.Errorf("object %s was not replicated", addr) } return nil } diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 28529fab9..f72333399 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -6,6 +6,8 @@ import ( "fmt" "path/filepath" "strconv" + "sync" + "sync/atomic" "testing" "time" @@ -174,13 +176,13 @@ func TestEvacuateObjectsNetwork(t *testing.T) { errReplication := errors.New("handler error") acceptOneOf := func(objects []*objectSDK.Object, max uint64) func(context.Context, oid.Address, *objectSDK.Object) (bool, error) { - var n uint64 + var n atomic.Uint64 return func(_ context.Context, addr oid.Address, obj *objectSDK.Object) (bool, error) { - if n == max { + if n.Load() == max { return false, errReplication } - n++ + n.Add(1) for i := range objects { if addr == objectCore.AddressOf(objects[i]) { require.Equal(t, objects[i], obj) @@ -314,6 +316,36 @@ func TestEvacuateCancellation(t *testing.T) { require.Equal(t, uint64(0), res.ObjectsEvacuated()) } +func TestEvacuateCancellationByError(t *testing.T) { + t.Parallel() + e, ids, _ := newEngineEvacuate(t, 2, 10) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + + var prm EvacuateShardPrm + prm.ShardID = ids[1:2] + var once atomic.Bool + prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) (bool, error) { + var err error + flag := true + if once.CompareAndSwap(false, true) { + err = errors.New("test error") + flag = false + } + return flag, err + } + prm.Scope = EvacuateScopeObjects + prm.ObjectWorkerCount = 2 + prm.ContainerWorkerCount = 2 + + _, err := e.Evacuate(context.Background(), prm) + require.ErrorContains(t, err, "test error") +} + func TestEvacuateSingleProcess(t *testing.T) { e, ids, _ := newEngineEvacuate(t, 2, 3) defer func() { @@ -531,6 +563,7 @@ func TestEvacuateTreesRemote(t *testing.T) { require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + mutex := sync.Mutex{} evacuatedTreeOps := make(map[string][]*pilorama.Move) var prm EvacuateShardPrm prm.ShardID = ids @@ -545,7 +578,9 @@ func TestEvacuateTreesRemote(t *testing.T) { if op.Time == 0 { return true, "", nil } + mutex.Lock() evacuatedTreeOps[key] = append(evacuatedTreeOps[key], &op) + mutex.Unlock() height = op.Time + 1 } } diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index b4326a92c..5943be7f4 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -1,6 +1,7 @@ package meta import ( + "bytes" "context" "time" @@ -61,6 +62,20 @@ func (l ListRes) Cursor() *Cursor { return l.cursor } +// IterateOverContainersPrm contains parameters for IterateOverContainers operation. +type IterateOverContainersPrm struct { + // Handler function executed upon containers in db. + Handler func(context.Context, []byte, cid.ID) error +} + +// IterateOverObjectsInContainerPrm contains parameters for IterateOverObjectsInContainer operation. +type IterateOverObjectsInContainerPrm struct { + // BucketName container's bucket name. + BucketName []byte + // Handler function executed upon objects in db. + Handler func(context.Context, *objectcore.Info) error +} + // ListWithCursor lists physical objects available in metabase starting from // cursor. Includes objects of all types. Does not include inhumed objects. // Use cursor value from response for consecutive requests. @@ -259,3 +274,155 @@ func parseContainerIDWithPrefix(containerID *cid.ID, name []byte) ([]byte, byte) return rawID, name[0] } + +// IterateOverContainers lists physical containers available in metabase starting from first. +func (db *DB) IterateOverContainers(ctx context.Context, prm IterateOverContainersPrm) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateOverContainers", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateOverContainers", + trace.WithAttributes( + attribute.Bool("has_handler", prm.Handler != nil), + )) + defer span.End() + + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return ErrDegradedMode + } + + err := db.boltDB.View(func(tx *bbolt.Tx) error { + return db.iterateOverContainers(ctx, tx, prm) + }) + success = err == nil + return metaerr.Wrap(err) +} + +func (db *DB) iterateOverContainers(ctx context.Context, tx *bbolt.Tx, prm IterateOverContainersPrm) error { + var containerID cid.ID + for _, prefix := range [][]byte{{byte(primaryPrefix)}, {byte(lockersPrefix)}, {byte(tombstonePrefix)}} { + c := tx.Cursor() + for name, _ := c.Seek(prefix); name != nil && bytes.HasPrefix(name, prefix); name, _ = c.Next() { + cidRaw, _ := parseContainerIDWithPrefix(&containerID, name) + if cidRaw == nil { + continue + } + + bktName := make([]byte, len(name)) + copy(bktName, name) + var cnt cid.ID + copy(cnt[:], containerID[:]) + err := prm.Handler(ctx, bktName, cnt) + if err != nil { + return err + } + } + } + + return nil +} + +// IterateOverObjectsInContainer iterate over physical objects available in metabase starting from first. +func (db *DB) IterateOverObjectsInContainer(ctx context.Context, prm IterateOverObjectsInContainerPrm) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("IterateOverObjectsInContainer", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateOverObjectsInContainer", + trace.WithAttributes( + attribute.Bool("has_handler", prm.Handler != nil), + )) + defer span.End() + + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return ErrDegradedMode + } + + var containerID cid.ID + cidRaw, prefix := parseContainerIDWithPrefix(&containerID, prm.BucketName) + if cidRaw == nil { + return nil + } + err := db.boltDB.View(func(tx *bbolt.Tx) error { + return db.iterateOverObjectsInContainer(ctx, tx, cidRaw, prefix, containerID, prm) + }) + success = err == nil + return metaerr.Wrap(err) +} + +func (db *DB) iterateOverObjectsInContainer(ctx context.Context, tx *bbolt.Tx, cidRaw []byte, prefix byte, + containerID cid.ID, prm IterateOverObjectsInContainerPrm, +) error { + bkt := tx.Bucket(prm.BucketName) + if bkt == nil { + return nil + } + graveyardBkt := tx.Bucket(graveyardBucketName) + garbageBkt := tx.Bucket(garbageBucketName) + c := bkt.Cursor() + k, v := c.First() + + var objType objectSDK.Type + + switch prefix { + case primaryPrefix: + objType = objectSDK.TypeRegular + case lockersPrefix: + objType = objectSDK.TypeLock + case tombstonePrefix: + objType = objectSDK.TypeTombstone + default: + return nil + } + + for ; k != nil; k, v = c.Next() { + var obj oid.ID + if err := obj.Decode(k); err != nil { + break + } + + if inGraveyardWithKey(append(cidRaw, k...), graveyardBkt, garbageBkt) > 0 { + continue + } + + var isLinkingObj bool + var ecInfo *objectcore.ECInfo + if objType == objectSDK.TypeRegular { + var o objectSDK.Object + if err := o.Unmarshal(v); err != nil { + return err + } + isLinkingObj = isLinkObject(&o) + ecHeader := o.ECHeader() + if ecHeader != nil { + ecInfo = &objectcore.ECInfo{ + ParentID: ecHeader.Parent(), + Index: ecHeader.Index(), + Total: ecHeader.Total(), + } + } + } + + var a oid.Address + a.SetContainer(containerID) + a.SetObject(obj) + objInfo := objectcore.Info{Address: a, Type: objType, IsLinkingObject: isLinkingObj, ECInfo: ecInfo} + err := prm.Handler(ctx, &objInfo) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index 08ea81a0c..9f56ec750 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -34,6 +34,20 @@ func (r ListContainersRes) Containers() []cid.ID { return r.containers } +// IterateOverContainersPrm contains parameters for IterateOverContainers operation. +type IterateOverContainersPrm struct { + // Handler function executed upon containers in db. + Handler func(context.Context, []byte, cid.ID) error +} + +// IterateOverObjectsInContainerPrm contains parameters for IterateOverObjectsInContainer operation. +type IterateOverObjectsInContainerPrm struct { + // BucketName container's bucket name. + BucketName []byte + // Handler function executed upon containers in db. + Handler func(context.Context, *objectcore.Info) error +} + // ListWithCursorPrm contains parameters for ListWithCursor operation. type ListWithCursorPrm struct { count uint32 @@ -164,3 +178,54 @@ func (s *Shard) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (List cursor: res.Cursor(), }, nil } + +// IterateOverContainers lists physical containers presented in shard. +func (s *Shard) IterateOverContainers(ctx context.Context, prm IterateOverContainersPrm) error { + _, span := tracing.StartSpanFromContext(ctx, "shard.IterateOverContainers", + trace.WithAttributes( + attribute.Bool("has_handler", prm.Handler != nil), + )) + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } + + var metaPrm meta.IterateOverContainersPrm + metaPrm.Handler = prm.Handler + err := s.metaBase.IterateOverContainers(ctx, metaPrm) + if err != nil { + return fmt.Errorf("could not iterate over containers: %w", err) + } + + return nil +} + +// IterateOverObjectsInContainer lists physical objects presented in shard for provided container's bucket name. +func (s *Shard) IterateOverObjectsInContainer(ctx context.Context, prm IterateOverObjectsInContainerPrm) error { + _, span := tracing.StartSpanFromContext(ctx, "shard.IterateOverObjectsInContainer", + trace.WithAttributes( + attribute.Bool("has_handler", prm.Handler != nil), + )) + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } + + var metaPrm meta.IterateOverObjectsInContainerPrm + metaPrm.BucketName = prm.BucketName + metaPrm.Handler = prm.Handler + err := s.metaBase.IterateOverObjectsInContainer(ctx, metaPrm) + if err != nil { + return fmt.Errorf("could not iterate over objects: %w", err) + } + + return nil +} diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index aacebe9e3..bdc6f7c38 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -23,12 +23,14 @@ func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartSha } prm := engine.EvacuateShardPrm{ - ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), - IgnoreErrors: req.GetBody().GetIgnoreErrors(), - ObjectsHandler: s.replicateObject, - TreeHandler: s.replicateTree, - Async: true, - Scope: engine.EvacuateScope(req.GetBody().GetScope()), + ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), + IgnoreErrors: req.GetBody().GetIgnoreErrors(), + ObjectsHandler: s.replicateObject, + TreeHandler: s.replicateTree, + Async: true, + Scope: engine.EvacuateScope(req.GetBody().GetScope()), + ContainerWorkerCount: req.GetBody().GetContainerWorkerCount(), + ObjectWorkerCount: req.GetBody().GetObjectWorkerCount(), } _, err = s.s.Evacuate(ctx, prm) diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 04994328a..88a06de22 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -394,6 +394,10 @@ message StartShardEvacuationRequest { bool ignore_errors = 2; // Evacuation scope. uint32 scope = 3; + // Count of concurrent container evacuation workers. + uint32 container_worker_count = 4; + // Count of concurrent object evacuation workers. + uint32 object_worker_count = 5; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 019cac290..e92a8acd1 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -6511,9 +6511,11 @@ func (x *StartShardEvacuationRequest_Body_Scope) FromString(s string) bool { } type StartShardEvacuationRequest_Body struct { - Shard_ID [][]byte `json:"shardID"` - IgnoreErrors bool `json:"ignoreErrors"` - Scope uint32 `json:"scope"` + Shard_ID [][]byte `json:"shardID"` + IgnoreErrors bool `json:"ignoreErrors"` + Scope uint32 `json:"scope"` + ContainerWorkerCount uint32 `json:"containerWorkerCount"` + ObjectWorkerCount uint32 `json:"objectWorkerCount"` } var ( @@ -6533,6 +6535,8 @@ func (x *StartShardEvacuationRequest_Body) StableSize() (size int) { size += proto.RepeatedBytesSize(1, x.Shard_ID) size += proto.BoolSize(2, x.IgnoreErrors) size += proto.UInt32Size(3, x.Scope) + size += proto.UInt32Size(4, x.ContainerWorkerCount) + size += proto.UInt32Size(5, x.ObjectWorkerCount) return size } @@ -6558,6 +6562,12 @@ func (x *StartShardEvacuationRequest_Body) EmitProtobuf(mm *easyproto.MessageMar if x.Scope != 0 { mm.AppendUint32(3, x.Scope) } + if x.ContainerWorkerCount != 0 { + mm.AppendUint32(4, x.ContainerWorkerCount) + } + if x.ObjectWorkerCount != 0 { + mm.AppendUint32(5, x.ObjectWorkerCount) + } } // UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. @@ -6587,6 +6597,18 @@ func (x *StartShardEvacuationRequest_Body) UnmarshalProtobuf(src []byte) (err er return fmt.Errorf("cannot unmarshal field %s", "Scope") } x.Scope = data + case 4: // ContainerWorkerCount + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerWorkerCount") + } + x.ContainerWorkerCount = data + case 5: // ObjectWorkerCount + data, ok := fc.Uint32() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ObjectWorkerCount") + } + x.ObjectWorkerCount = data } } return nil @@ -6618,6 +6640,24 @@ func (x *StartShardEvacuationRequest_Body) GetScope() uint32 { func (x *StartShardEvacuationRequest_Body) SetScope(v uint32) { x.Scope = v } +func (x *StartShardEvacuationRequest_Body) GetContainerWorkerCount() uint32 { + if x != nil { + return x.ContainerWorkerCount + } + return 0 +} +func (x *StartShardEvacuationRequest_Body) SetContainerWorkerCount(v uint32) { + x.ContainerWorkerCount = v +} +func (x *StartShardEvacuationRequest_Body) GetObjectWorkerCount() uint32 { + if x != nil { + return x.ObjectWorkerCount + } + return 0 +} +func (x *StartShardEvacuationRequest_Body) SetObjectWorkerCount(v uint32) { + x.ObjectWorkerCount = v +} // MarshalJSON implements the json.Marshaler interface. func (x *StartShardEvacuationRequest_Body) MarshalJSON() ([]byte, error) { @@ -6653,6 +6693,16 @@ func (x *StartShardEvacuationRequest_Body) MarshalEasyJSON(out *jwriter.Writer) out.RawString(prefix) out.Uint32(x.Scope) } + { + const prefix string = ",\"containerWorkerCount\":" + out.RawString(prefix) + out.Uint32(x.ContainerWorkerCount) + } + { + const prefix string = ",\"objectWorkerCount\":" + out.RawString(prefix) + out.Uint32(x.ObjectWorkerCount) + } out.RawByte('}') } @@ -6706,6 +6756,18 @@ func (x *StartShardEvacuationRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { f = in.Uint32() x.Scope = f } + case "containerWorkerCount": + { + var f uint32 + f = in.Uint32() + x.ContainerWorkerCount = f + } + case "objectWorkerCount": + { + var f uint32 + f = in.Uint32() + x.ObjectWorkerCount = f + } } in.WantComma() } From 8434f3dbfc850839c759430cea9640f3c87e5f95 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 19 Sep 2024 17:00:58 +0300 Subject: [PATCH 1380/1943] [#1385] metabase: Use `Batch` for delete-related operations Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/counter.go | 4 ++-- pkg/local_object_storage/metabase/delete.go | 2 +- pkg/local_object_storage/metabase/graveyard.go | 2 +- pkg/local_object_storage/metabase/inhume.go | 2 +- pkg/local_object_storage/metabase/lock.go | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index 275099ff2..3ead0d9a0 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -654,7 +654,7 @@ func (db *DB) DeleteContainerSize(ctx context.Context, id cid.ID) error { return ErrReadOnlyMode } - err := db.boltDB.Update(func(tx *bbolt.Tx) error { + err := db.boltDB.Batch(func(tx *bbolt.Tx) error { b := tx.Bucket(containerVolumeBucketName) key := make([]byte, cidSize) @@ -737,7 +737,7 @@ func (db *DB) DeleteContainerCount(ctx context.Context, id cid.ID) error { return ErrReadOnlyMode } - err := db.boltDB.Update(func(tx *bbolt.Tx) error { + err := db.boltDB.Batch(func(tx *bbolt.Tx) error { b := tx.Bucket(containerCounterBucketName) key := make([]byte, cidSize) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 00c8d06e0..e5e9840a0 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -112,7 +112,7 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { var err error var res DeleteRes - err = db.boltDB.Update(func(tx *bbolt.Tx) error { + err = db.boltDB.Batch(func(tx *bbolt.Tx) error { res, err = db.deleteGroup(tx, prm.addrs) return err }) diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index 80d40fb78..31f95d6ed 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -282,7 +282,7 @@ func (db *DB) DropGraves(ctx context.Context, tss []TombstonedObject) error { buf := make([]byte, addressKeySize) - return db.boltDB.Update(func(tx *bbolt.Tx) error { + return db.boltDB.Batch(func(tx *bbolt.Tx) error { bkt := tx.Bucket(graveyardBucketName) if bkt == nil { return nil diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index b62accc43..3aae15061 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -181,7 +181,7 @@ func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { inhumedByCnrID: make(map[cid.ID]ObjectCounters), } currEpoch := db.epochState.CurrentEpoch() - err := db.boltDB.Update(func(tx *bbolt.Tx) error { + err := db.boltDB.Batch(func(tx *bbolt.Tx) error { return db.inhumeTx(tx, currEpoch, prm, &res) }) success = err == nil diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 732ba426d..6b78ef392 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -78,7 +78,7 @@ func (db *DB) lockInternal(locked []oid.ID, cnr cid.ID, locker oid.ID) error { } key := make([]byte, cidSize) - return metaerr.Wrap(db.boltDB.Update(func(tx *bbolt.Tx) error { + return metaerr.Wrap(db.boltDB.Batch(func(tx *bbolt.Tx) error { if firstIrregularObjectType(tx, cnr, bucketKeysLocked...) != objectSDK.TypeRegular { return logicerr.Wrap(new(apistatus.LockNonRegularObject)) } @@ -143,7 +143,7 @@ func (db *DB) FreeLockedBy(lockers []oid.Address) ([]oid.Address, error) { var unlockedObjects []oid.Address - if err := db.boltDB.Update(func(tx *bbolt.Tx) error { + if err := db.boltDB.Batch(func(tx *bbolt.Tx) error { for i := range lockers { unlocked, err := freePotentialLocks(tx, lockers[i].Container(), lockers[i].Object()) if err != nil { From 76268e3ea2a73072119ea1963f914646c029e08a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 20 Sep 2024 13:28:21 +0300 Subject: [PATCH 1381/1943] [#1385] metabase: Validate that tombstone and target have the same container ID Target container ID is taken from tombstone: cmd/frostfs-node/object.go:507 Also object of type `TOMBSTONE` contains objectID, so tombstone and tombstoned object must have the same containerID. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/lock_test.go | 8 +++-- .../metabase/containers_test.go | 4 +-- .../metabase/control_test.go | 2 +- .../metabase/counter_test.go | 30 ++++++++++++------- .../metabase/delete_test.go | 6 ++-- .../metabase/exists_test.go | 2 +- pkg/local_object_storage/metabase/get_test.go | 3 +- .../metabase/graveyard_test.go | 27 ++++++++++------- pkg/local_object_storage/metabase/inhume.go | 18 +++++++++++ .../metabase/inhume_test.go | 21 +++++++++---- .../metabase/iterators_test.go | 6 ++++ .../metabase/list_test.go | 2 +- .../metabase/lock_test.go | 10 +++++-- .../metabase/select_test.go | 6 +--- .../metabase/storage_id_test.go | 2 +- .../shard/metrics_test.go | 13 ++++---- 16 files changed, 108 insertions(+), 52 deletions(-) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 7fa7c27ef..9e6758fb4 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -199,7 +199,9 @@ func TestLockExpiration(t *testing.T) { require.NoError(t, err) var inhumePrm InhumePrm - inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj)) + tombAddr := oidtest.Address() + tombAddr.SetContainer(cnr) + inhumePrm.WithTarget(tombAddr, objectcore.AddressOf(obj)) var objLockedErr *apistatus.ObjectLocked _, err = e.Inhume(context.Background(), inhumePrm) @@ -209,7 +211,9 @@ func TestLockExpiration(t *testing.T) { e.HandleNewEpoch(context.Background(), lockerExpiresAfter+1) // 4. - inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj)) + tombAddr = oidtest.Address() + tombAddr.SetContainer(cnr) + inhumePrm.WithTarget(tombAddr, objectcore.AddressOf(obj)) require.Eventually(t, func() bool { _, err = e.Inhume(context.Background(), inhumePrm) diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index 8b1874458..110be68ad 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -67,7 +67,7 @@ func TestDB_Containers(t *testing.T) { assertContains(cnrs, cnr) - require.NoError(t, metaInhume(db, object.AddressOf(obj), oidtest.Address())) + require.NoError(t, metaInhume(db, object.AddressOf(obj), oidtest.ID())) cnrs, err = db.Containers(context.Background()) require.NoError(t, err) @@ -164,7 +164,7 @@ func TestDB_ContainerSize(t *testing.T) { require.NoError(t, metaInhume( db, object.AddressOf(obj), - oidtest.Address(), + oidtest.ID(), )) volume -= int(obj.PayloadSize()) diff --git a/pkg/local_object_storage/metabase/control_test.go b/pkg/local_object_storage/metabase/control_test.go index 0354a5eb6..2a64881cb 100644 --- a/pkg/local_object_storage/metabase/control_test.go +++ b/pkg/local_object_storage/metabase/control_test.go @@ -41,7 +41,7 @@ func TestReset(t *testing.T) { err = putBig(db, obj) require.NoError(t, err) - err = metaInhume(db, addrToInhume, oidtest.Address()) + err = metaInhume(db, addrToInhume, oidtest.ID()) require.NoError(t, err) assertExists(addr, true, nil) diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index d1f808a63..dccccd456 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -156,13 +156,18 @@ func TestCounters(t *testing.T) { } var prm meta.InhumePrm - prm.SetTombstoneAddress(oidtest.Address()) - prm.SetAddresses(inhumedObjs...) + for _, o := range inhumedObjs { + tombAddr := oidtest.Address() + tombAddr.SetContainer(o.Container()) - res, err := db.Inhume(context.Background(), prm) - require.NoError(t, err) - require.Equal(t, uint64(len(inhumedObjs)), res.LogicInhumed()) - require.Equal(t, uint64(len(inhumedObjs)), res.UserInhumed()) + prm.SetTombstoneAddress(tombAddr) + prm.SetAddresses(o) + + res, err := db.Inhume(context.Background(), prm) + require.NoError(t, err) + require.Equal(t, uint64(1), res.LogicInhumed()) + require.Equal(t, uint64(1), res.UserInhumed()) + } c, err := db.ObjectCounters() require.NoError(t, err) @@ -296,11 +301,16 @@ func TestCounters(t *testing.T) { } var prm meta.InhumePrm - prm.SetTombstoneAddress(oidtest.Address()) - prm.SetAddresses(inhumedObjs...) + for _, o := range inhumedObjs { + tombAddr := oidtest.Address() + tombAddr.SetContainer(o.Container()) - _, err := db.Inhume(context.Background(), prm) - require.NoError(t, err) + prm.SetTombstoneAddress(tombAddr) + prm.SetAddresses(o) + + _, err := db.Inhume(context.Background(), prm) + require.NoError(t, err) + } c, err := db.ObjectCounters() require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/delete_test.go b/pkg/local_object_storage/metabase/delete_test.go index cb85157e7..fe5f7833b 100644 --- a/pkg/local_object_storage/metabase/delete_test.go +++ b/pkg/local_object_storage/metabase/delete_test.go @@ -40,12 +40,12 @@ func TestDB_Delete(t *testing.T) { // inhume parent and child so they will be on graveyard ts := testutil.GenerateObjectWithCID(cnr) - err = metaInhume(db, object.AddressOf(child), object.AddressOf(ts)) + err = metaInhume(db, object.AddressOf(child), object.AddressOf(ts).Object()) require.NoError(t, err) ts = testutil.GenerateObjectWithCID(cnr) - err = metaInhume(db, object.AddressOf(parent), object.AddressOf(ts)) + err = metaInhume(db, object.AddressOf(parent), object.AddressOf(ts).Object()) require.NoError(t, err) // delete object @@ -108,7 +108,7 @@ func TestGraveOnlyDelete(t *testing.T) { addr := oidtest.Address() // inhume non-existent object by address - require.NoError(t, metaInhume(db, addr, oidtest.Address())) + require.NoError(t, metaInhume(db, addr, oidtest.ID())) // delete the object data require.NoError(t, metaDelete(db, addr)) diff --git a/pkg/local_object_storage/metabase/exists_test.go b/pkg/local_object_storage/metabase/exists_test.go index 0087c1e31..1e4148eba 100644 --- a/pkg/local_object_storage/metabase/exists_test.go +++ b/pkg/local_object_storage/metabase/exists_test.go @@ -37,7 +37,7 @@ func TestDB_Exists(t *testing.T) { require.True(t, exists) t.Run("removed object", func(t *testing.T) { - err := metaInhume(db, object.AddressOf(regular), oidtest.Address()) + err := metaInhume(db, object.AddressOf(regular), oidtest.ID()) require.NoError(t, err) exists, err := metaExists(db, object.AddressOf(regular)) diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index 7654d2cd8..f0caaea70 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -150,9 +150,8 @@ func TestDB_Get(t *testing.T) { t.Run("get removed object", func(t *testing.T) { obj := oidtest.Address() - ts := oidtest.Address() - require.NoError(t, metaInhume(db, obj, ts)) + require.NoError(t, metaInhume(db, obj, oidtest.ID())) _, err := metaGet(db, obj, false) require.True(t, client.IsErrObjectAlreadyRemoved(err)) diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index 75c7e2852..b9c6ce28c 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -114,11 +115,12 @@ func TestDB_IterateDeletedObjects(t *testing.T) { db := newDB(t) defer func() { require.NoError(t, db.Close()) }() + cnr := cidtest.ID() // generate and put 4 objects - obj1 := testutil.GenerateObject() - obj2 := testutil.GenerateObject() - obj3 := testutil.GenerateObject() - obj4 := testutil.GenerateObject() + obj1 := testutil.GenerateObjectWithCID(cnr) + obj2 := testutil.GenerateObjectWithCID(cnr) + obj3 := testutil.GenerateObjectWithCID(cnr) + obj4 := testutil.GenerateObjectWithCID(cnr) var err error @@ -138,6 +140,7 @@ func TestDB_IterateDeletedObjects(t *testing.T) { // inhume with tombstone addrTombstone := oidtest.Address() + addrTombstone.SetContainer(cnr) inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) inhumePrm.SetTombstoneAddress(addrTombstone) @@ -201,11 +204,12 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { db := newDB(t) defer func() { require.NoError(t, db.Close()) }() + cnr := cidtest.ID() // generate and put 4 objects - obj1 := testutil.GenerateObject() - obj2 := testutil.GenerateObject() - obj3 := testutil.GenerateObject() - obj4 := testutil.GenerateObject() + obj1 := testutil.GenerateObjectWithCID(cnr) + obj2 := testutil.GenerateObjectWithCID(cnr) + obj3 := testutil.GenerateObjectWithCID(cnr) + obj4 := testutil.GenerateObjectWithCID(cnr) var err error @@ -223,6 +227,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { // inhume with tombstone addrTombstone := oidtest.Address() + addrTombstone.SetContainer(cnr) var inhumePrm meta.InhumePrm inhumePrm.SetAddresses( @@ -392,9 +397,10 @@ func TestDB_DropGraves(t *testing.T) { db := newDB(t) defer func() { require.NoError(t, db.Close()) }() + cnr := cidtest.ID() // generate and put 2 objects - obj1 := testutil.GenerateObject() - obj2 := testutil.GenerateObject() + obj1 := testutil.GenerateObjectWithCID(cnr) + obj2 := testutil.GenerateObjectWithCID(cnr) var err error @@ -406,6 +412,7 @@ func TestDB_DropGraves(t *testing.T) { // inhume with tombstone addrTombstone := oidtest.Address() + addrTombstone.SetContainer(cnr) var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 3aae15061..77bb84af1 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -143,6 +143,20 @@ func (p *InhumePrm) SetForceGCMark() { p.forceRemoval = true } +func (p *InhumePrm) validate() error { + if p == nil { + return nil + } + if p.tomb != nil { + for _, addr := range p.target { + if addr.Container() != p.tomb.Container() { + return fmt.Errorf("object %s and tombstone %s have different container ID", addr, p.tomb) + } + } + } + return nil +} + var errBreakBucketForEach = errors.New("bucket ForEach break") // ErrLockObjectRemoval is returned when inhume operation is being @@ -171,6 +185,10 @@ func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { db.modeMtx.RLock() defer db.modeMtx.RUnlock() + if err := prm.validate(); err != nil { + return InhumeRes{}, err + } + if db.mode.NoMetabase() { return InhumeRes{}, ErrDegradedMode } else if db.mode.ReadOnly() { diff --git a/pkg/local_object_storage/metabase/inhume_test.go b/pkg/local_object_storage/metabase/inhume_test.go index 163fbec2a..277316f7b 100644 --- a/pkg/local_object_storage/metabase/inhume_test.go +++ b/pkg/local_object_storage/metabase/inhume_test.go @@ -9,6 +9,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -21,12 +22,10 @@ func TestDB_Inhume(t *testing.T) { raw := testutil.GenerateObject() testutil.AddAttribute(raw, "foo", "bar") - tombstoneID := oidtest.Address() - err := putBig(db, raw) require.NoError(t, err) - err = metaInhume(db, object.AddressOf(raw), tombstoneID) + err = metaInhume(db, object.AddressOf(raw), oidtest.ID()) require.NoError(t, err) _, err = metaExists(db, object.AddressOf(raw)) @@ -43,13 +42,20 @@ func TestInhumeTombOnTomb(t *testing.T) { var ( err error + cnr = cidtest.ID() addr1 = oidtest.Address() addr2 = oidtest.Address() addr3 = oidtest.Address() + addr4 = oidtest.Address() inhumePrm meta.InhumePrm existsPrm meta.ExistsPrm ) + addr1.SetContainer(cnr) + addr2.SetContainer(cnr) + addr3.SetContainer(cnr) + addr4.SetContainer(cnr) + inhumePrm.SetAddresses(addr1) inhumePrm.SetTombstoneAddress(addr2) @@ -84,7 +90,7 @@ func TestInhumeTombOnTomb(t *testing.T) { require.True(t, client.IsErrObjectAlreadyRemoved(err)) inhumePrm.SetAddresses(addr1) - inhumePrm.SetTombstoneAddress(oidtest.Address()) + inhumePrm.SetTombstoneAddress(addr4) // try to inhume addr1 (which is already a tombstone in graveyard) _, err = db.Inhume(context.Background(), inhumePrm) @@ -117,10 +123,13 @@ func TestInhumeLocked(t *testing.T) { require.ErrorAs(t, err, &e) } -func metaInhume(db *meta.DB, target, tomb oid.Address) error { +func metaInhume(db *meta.DB, target oid.Address, tomb oid.ID) error { var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(target) - inhumePrm.SetTombstoneAddress(tomb) + var tombAddr oid.Address + tombAddr.SetContainer(target.Container()) + tombAddr.SetObject(tomb) + inhumePrm.SetTombstoneAddress(tombAddr) _, err := db.Inhume(context.Background(), inhumePrm) return err diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 54d56d923..777a94a6f 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -9,6 +9,7 @@ import ( object2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -71,11 +72,16 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { db := newDB(t) defer func() { require.NoError(t, db.Close()) }() + cnr := cidtest.ID() ts := oidtest.Address() protected1 := oidtest.Address() protected2 := oidtest.Address() protectedLocked := oidtest.Address() garbage := oidtest.Address() + ts.SetContainer(cnr) + protected1.SetContainer(cnr) + protected2.SetContainer(cnr) + protectedLocked.SetContainer(cnr) var prm meta.InhumePrm var err error diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 6207497b1..bc1726bd6 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -110,7 +110,7 @@ func TestLisObjectsWithCursor(t *testing.T) { err = putBig(db, obj) require.NoError(t, err) ts := testutil.GenerateObjectWithCID(containerID) - err = metaInhume(db, object.AddressOf(obj), object.AddressOf(ts)) + err = metaInhume(db, object.AddressOf(obj), object.AddressOf(ts).Object()) require.NoError(t, err) // add one child object (do not include parent into expected) diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 62a109b02..9601cb2be 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -73,7 +73,9 @@ func TestDB_Lock(t *testing.T) { _, err := db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) - inhumePrm.SetTombstoneAddress(oidtest.Address()) + tombAddr := oidtest.Address() + tombAddr.SetContainer(objAddr.Container()) + inhumePrm.SetTombstoneAddress(tombAddr) _, err = db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) @@ -89,7 +91,9 @@ func TestDB_Lock(t *testing.T) { _, err = db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) - inhumePrm.SetTombstoneAddress(oidtest.Address()) + tombAddr = oidtest.Address() + tombAddr.SetContainer(objAddr.Container()) + inhumePrm.SetTombstoneAddress(tombAddr) _, err = db.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) }) @@ -103,7 +107,7 @@ func TestDB_Lock(t *testing.T) { var objLockedErr *apistatus.ObjectLocked // try to inhume locked object using tombstone - err := metaInhume(db, objAddr, lockAddr) + err := metaInhume(db, objAddr, lockAddr.Object()) require.ErrorAs(t, err, &objLockedErr) // free locked object diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 6469bbdbc..fcd5d3a90 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -352,11 +352,7 @@ func TestDB_SelectInhume(t *testing.T) { object.AddressOf(raw2), ) - var tombstone oid.Address - tombstone.SetContainer(cnr) - tombstone.SetObject(oidtest.ID()) - - err = metaInhume(db, object.AddressOf(raw2), tombstone) + err = metaInhume(db, object.AddressOf(raw2), oidtest.ID()) require.NoError(t, err) fs = objectSDK.SearchFilters{} diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go index aaf6480ab..a86e42bd2 100644 --- a/pkg/local_object_storage/metabase/storage_id_test.go +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -43,7 +43,7 @@ func TestDB_StorageID(t *testing.T) { cnrID, ok := deleted.ContainerID() require.True(t, ok) ts := testutil.GenerateObjectWithCID(cnrID) - require.NoError(t, metaInhume(db, object.AddressOf(deleted), object.AddressOf(ts))) + require.NoError(t, metaInhume(db, object.AddressOf(deleted), object.AddressOf(ts).Object())) // check StorageID for object without storageID fetchedStorageID, err = metaStorageID(db, object.AddressOf(raw2)) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 01a85da97..56622326a 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -17,6 +17,7 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) @@ -308,17 +309,19 @@ func TestCounters(t *testing.T) { t.Run("inhume_TS", func(t *testing.T) { var prm InhumePrm - ts := objectcore.AddressOf(testutil.GenerateObject()) phy := mm.getObjectCounter(physical) logic := mm.getObjectCounter(logical) custom := mm.getObjectCounter(user) inhumedNumber := int(phy / 4) - prm.SetTarget(ts, addrFromObjs(oo[:inhumedNumber])...) - - _, err := sh.Inhume(context.Background(), prm) - require.NoError(t, err) + for _, o := range addrFromObjs(oo[:inhumedNumber]) { + ts := oidtest.Address() + ts.SetContainer(o.Container()) + prm.SetTarget(ts, o) + _, err := sh.Inhume(context.Background(), prm) + require.NoError(t, err) + } for i := range inhumedNumber { cid, ok := oo[i].ContainerID() From fd18aa363b7b33f8b662f9b4bffaf9f3099216a6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 20 Sep 2024 13:32:05 +0300 Subject: [PATCH 1382/1943] [#1385] metabase: Optimize `isTomb` check As tombstone and target must have the same containerID, do not iterate other containers. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/inhume.go | 29 ++++++++------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 77bb84af1..12f27d330 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -377,11 +377,8 @@ func (db *DB) getInhumeTargetBucketAndValue(garbageBKT, graveyardBKT *bbolt.Buck return targetBucket, value, nil } -func (db *DB) markAsGC(graveyardBKT, garbageBKT *bbolt.Bucket, key []byte) (bool, error) { - targetIsTomb, err := isTomb(graveyardBKT, key) - if err != nil { - return false, err - } +func (db *DB) markAsGC(graveyardBKT, garbageBKT *bbolt.Bucket, addressKey []byte) (bool, error) { + targetIsTomb := isTomb(graveyardBKT, addressKey) // do not add grave if target is a tombstone if targetIsTomb { @@ -390,7 +387,7 @@ func (db *DB) markAsGC(graveyardBKT, garbageBKT *bbolt.Bucket, key []byte) (bool // if tombstone appears object must be // additionally marked with GC - return false, garbageBKT.Put(key, zeroValue) + return false, garbageBKT.Put(addressKey, zeroValue) } func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Bucket, targetKey []byte, cnr cid.ID, obj *objectSDK.Object, res *InhumeRes) error { @@ -410,25 +407,21 @@ func (db *DB) updateDeleteInfo(tx *bbolt.Tx, garbageBKT, graveyardBKT *bbolt.Buc return nil } -func isTomb(graveyardBucket *bbolt.Bucket, key []byte) (bool, error) { +func isTomb(graveyardBucket *bbolt.Bucket, addressKey []byte) bool { targetIsTomb := false // iterate over graveyard and check if target address // is the address of tombstone in graveyard. - err := graveyardBucket.ForEach(func(_, v []byte) error { + // tombstone must have the same container ID as key. + c := graveyardBucket.Cursor() + containerPrefix := addressKey[:cidSize] + for k, v := c.Seek(containerPrefix); k != nil && bytes.HasPrefix(k, containerPrefix); k, v = c.Next() { // check if graveyard has record with key corresponding // to tombstone address (at least one) - targetIsTomb = bytes.Equal(v, key) - + targetIsTomb = bytes.Equal(v, addressKey) if targetIsTomb { - // break bucket iterator - return errBreakBucketForEach + break } - - return nil - }) - if err != nil && !errors.Is(err, errBreakBucketForEach) { - return false, err } - return targetIsTomb, nil + return targetIsTomb } From 95597d34371db6555739c4e92640cd8f8862ee7e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 08:56:02 +0300 Subject: [PATCH 1383/1943] [#1388] golangci: Make `unused` linter stricker Add aditional checks. The most important false positive - structs used as map keys. Signed-off-by: Dmitrii Stepanov --- .golangci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 971f0d0e7..33cf88d8a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -38,6 +38,10 @@ linters-settings: alias: pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object alias: objectSDK + unused: + field-writes-are-uses: false + exported-fields-are-used: false + local-variables-are-used: false custom: truecloudlab-linters: path: bin/linters/external_linters.so From 2bd560e52846b77d2902370cfaa80d54fcd77c46 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 08:57:18 +0300 Subject: [PATCH 1384/1943] [#1388] cli: Drop unused flag/parameter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/internal/client/client.go | 7 ------- cmd/frostfs-cli/modules/object/head.go | 3 --- 2 files changed, 10 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 03a987a57..dcd67f0d9 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -565,13 +565,6 @@ type HeadObjectPrm struct { commonObjectPrm objectAddressPrm rawPrm - - mainOnly bool -} - -// SetMainOnlyFlag sets flag to get only main fields of an object header in terms of FrostFS API. -func (x *HeadObjectPrm) SetMainOnlyFlag(v bool) { - x.mainOnly = v } // HeadObjectRes groups the resulting values of HeadObject operation. diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index 14797dc41..cf2e2d5e6 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -38,7 +38,6 @@ func initObjectHeadCmd() { _ = objectHeadCmd.MarkFlagRequired(commonflags.OIDFlag) flags.String(fileFlag, "", "File to write header to. Default: stdout.") - flags.Bool("main-only", false, "Return only main fields") flags.Bool(commonflags.JSON, false, "Marshal output in JSON") flags.Bool("proto", false, "Marshal output in Protobuf") flags.Bool(rawFlag, false, rawFlagDesc) @@ -49,7 +48,6 @@ func getObjectHeader(cmd *cobra.Command, _ []string) { var obj oid.ID objAddr := readObjectAddress(cmd, &cnr, &obj) - mainOnly, _ := cmd.Flags().GetBool("main-only") pk := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) @@ -62,7 +60,6 @@ func getObjectHeader(cmd *cobra.Command, _ []string) { raw, _ := cmd.Flags().GetBool(rawFlag) prm.SetRawFlag(raw) prm.SetAddress(objAddr) - prm.SetMainOnlyFlag(mainOnly) res, err := internalclient.HeadObject(cmd.Context(), prm) if err != nil { From b69e07da7af2c8167e02585a723008fa2753f848 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 09:02:00 +0300 Subject: [PATCH 1385/1943] [#1388] metrics: Mark nolint:unused metrics Although these fields could be deleted, I annotated them so that all the metrics used would be defined in one place. Signed-off-by: Dmitrii Stepanov --- internal/metrics/innerring.go | 3 ++- internal/metrics/node.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/metrics/innerring.go b/internal/metrics/innerring.go index f6b14a632..f3f529d05 100644 --- a/internal/metrics/innerring.go +++ b/internal/metrics/innerring.go @@ -17,7 +17,8 @@ type InnerRingServiceMetrics struct { eventDuration *prometheus.HistogramVec morphCacheMetrics *morphCacheMetrics logMetrics logger.LogMetrics - appInfo *ApplicationInfo + // nolint: unused + appInfo *ApplicationInfo } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. diff --git a/internal/metrics/node.go b/internal/metrics/node.go index d9e401446..711387875 100644 --- a/internal/metrics/node.go +++ b/internal/metrics/node.go @@ -25,7 +25,8 @@ type NodeMetrics struct { morphClient *morphClientMetrics morphCache *morphCacheMetrics log logger.LogMetrics - appInfo *ApplicationInfo + // nolint: unused + appInfo *ApplicationInfo } func NewNodeMetrics() *NodeMetrics { From aedb55f913d151669885aa6bc8ea5e83269a60b0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 09:06:19 +0300 Subject: [PATCH 1386/1943] [#1388] governance: Drop unused Signed-off-by: Dmitrii Stepanov --- pkg/innerring/initialization.go | 1 - .../processors/governance/handlers_test.go | 21 ------------------- .../processors/governance/processor.go | 3 --- 3 files changed, 25 deletions(-) diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 7da0a9794..c4aaeda56 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -163,7 +163,6 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli Log: s.log, Metrics: s.irMetrics, FrostFSClient: frostfsCli, - NetmapClient: s.netmapClient, AlphabetState: s, EpochState: s, Voter: s, diff --git a/pkg/innerring/processors/governance/handlers_test.go b/pkg/innerring/processors/governance/handlers_test.go index b73e24318..87040bdef 100644 --- a/pkg/innerring/processors/governance/handlers_test.go +++ b/pkg/innerring/processors/governance/handlers_test.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" - nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/rolemanagement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" @@ -38,7 +37,6 @@ func TestHandleAlphabetSyncEvent(t *testing.T) { alphabetKeys: testKeys.mainnetKeys, } f := &testFrostFSClient{} - nm := &testNetmapClient{} proc, err := New( &Params{ @@ -50,7 +48,6 @@ func TestHandleAlphabetSyncEvent(t *testing.T) { MorphClient: m, MainnetClient: mn, FrostFSClient: f, - NetmapClient: nm, }, ) @@ -73,10 +70,6 @@ func TestHandleAlphabetSyncEvent(t *testing.T) { }, }, v.votes, "invalid vote calls") - var irUpdateExp []nmClient.UpdateIRPrm - - require.EqualValues(t, irUpdateExp, nm.updates, "invalid IR updates") - var expAlphabetUpdate client.UpdateAlphabetListPrm expAlphabetUpdate.SetHash(ev.txHash) expAlphabetUpdate.SetList(testKeys.newInnerRingExp) @@ -119,7 +112,6 @@ func TestHandleAlphabetDesignateEvent(t *testing.T) { alphabetKeys: testKeys.mainnetKeys, } f := &testFrostFSClient{} - nm := &testNetmapClient{} proc, err := New( &Params{ @@ -131,7 +123,6 @@ func TestHandleAlphabetDesignateEvent(t *testing.T) { MorphClient: m, MainnetClient: mn, FrostFSClient: f, - NetmapClient: nm, }, ) @@ -155,9 +146,6 @@ func TestHandleAlphabetDesignateEvent(t *testing.T) { }, }, v.votes, "invalid vote calls") - var irUpdatesExp []nmClient.UpdateIRPrm - require.EqualValues(t, irUpdatesExp, nm.updates, "invalid IR updates") - var alpabetUpdExp client.UpdateAlphabetListPrm alpabetUpdExp.SetList(testKeys.newInnerRingExp) alpabetUpdExp.SetHash(ev.TxHash) @@ -293,12 +281,3 @@ func (c *testFrostFSClient) AlphabetUpdate(p frostfscontract.AlphabetUpdatePrm) c.updates = append(c.updates, p) return nil } - -type testNetmapClient struct { - updates []nmClient.UpdateIRPrm -} - -func (c *testNetmapClient) UpdateInnerRing(p nmClient.UpdateIRPrm) error { - c.updates = append(c.updates, p) - return nil -} diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index fa267eade..6daea417e 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -79,7 +79,6 @@ type ( metrics metrics.Register pool *ants.Pool frostfsClient FrostFSClient - netmapClient NetmapClient alphabetState AlphabetState epochState EpochState @@ -105,7 +104,6 @@ type ( MorphClient MorphClient MainnetClient MainnetClient FrostFSClient FrostFSClient - NetmapClient NetmapClient } ) @@ -146,7 +144,6 @@ func New(p *Params) (*Processor, error) { metrics: metricsRegister, pool: pool, frostfsClient: p.FrostFSClient, - netmapClient: p.NetmapClient, alphabetState: p.AlphabetState, epochState: p.EpochState, voter: p.Voter, From e319bf403e7ddd24d9527829a9d5863643635ff8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 09:08:11 +0300 Subject: [PATCH 1387/1943] [#1388] apeSvc: Drop unused and make annotations Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 - cmd/frostfs-node/object.go | 1 - cmd/frostfs-node/policy_engine.go | 4 +++- pkg/ape/chainbase/option.go | 10 ---------- pkg/services/object/ape/service.go | 6 +----- 5 files changed, 4 insertions(+), 18 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 63f410b89..0ffa8c45b 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1082,7 +1082,6 @@ func initAccessPolicyEngine(_ context.Context, c *cfg) { localOverrideDB = chainbase.NewInmemoryLocalOverrideDatabase() } else { localOverrideDB = chainbase.NewBoltLocalOverrideDatabase( - chainbase.WithLogger(c.log), chainbase.WithPath(nodeconfig.PersistentPolicyRules(c.appCfg).Path()), chainbase.WithPerm(nodeconfig.PersistentPolicyRules(c.appCfg).Perm()), chainbase.WithNoSync(nodeconfig.PersistentPolicyRules(c.appCfg).NoSync()), diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 610e2c363..9d4e35ca8 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -473,7 +473,6 @@ func createACLServiceV2(c *cfg, apeSvc *objectAPE.Service, irFetcher *cachedIRFe func createAPEService(c *cfg, splitSvc *objectService.TransportSplitter) *objectAPE.Service { return objectAPE.NewService( - c.log, objectAPE.NewChecker( c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalStorage(), c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.MorphRuleChainStorage(), diff --git a/cmd/frostfs-node/policy_engine.go b/cmd/frostfs-node/policy_engine.go index 22fda2b4c..55f76cc76 100644 --- a/cmd/frostfs-node/policy_engine.go +++ b/cmd/frostfs-node/policy_engine.go @@ -21,7 +21,9 @@ type accessPolicyEngine struct { var _ engine.MorphRuleChainStorageReader = (*morphAPEChainCache)(nil) type morphAPEChainCacheKey struct { - name chain.Name + // nolint:unused + name chain.Name + // nolint:unused target engine.Target } diff --git a/pkg/ape/chainbase/option.go b/pkg/ape/chainbase/option.go index e547701fb..590b7a885 100644 --- a/pkg/ape/chainbase/option.go +++ b/pkg/ape/chainbase/option.go @@ -5,9 +5,7 @@ import ( "os" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.etcd.io/bbolt" - "go.uber.org/zap" ) type Option func(*cfg) @@ -18,7 +16,6 @@ type cfg struct { noSync bool maxBatchDelay time.Duration maxBatchSize int - log *logger.Logger } func defaultCfg() *cfg { @@ -26,7 +23,6 @@ func defaultCfg() *cfg { perm: os.ModePerm, maxBatchDelay: bbolt.DefaultMaxBatchDelay, maxBatchSize: bbolt.DefaultMaxBatchSize, - log: &logger.Logger{Logger: zap.L()}, } } @@ -59,9 +55,3 @@ func WithMaxBatchSize(maxBatchSize int) Option { c.maxBatchSize = maxBatchSize } } - -func WithLogger(l *logger.Logger) Option { - return func(c *cfg) { - c.log = l - } -} diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index a1634e7c5..6eedaf99e 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -12,7 +12,6 @@ import ( objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -24,8 +23,6 @@ import ( var errFailedToCastToRequestContext = errors.New("failed cast to RequestContext") type Service struct { - log *logger.Logger - apeChecker Checker next objectSvc.ServiceServer @@ -67,9 +64,8 @@ func NewStorageEngineHeaderProvider(e *engine.StorageEngine, s *getsvc.Service) } } -func NewService(log *logger.Logger, apeChecker Checker, next objectSvc.ServiceServer) *Service { +func NewService(apeChecker Checker, next objectSvc.ServiceServer) *Service { return &Service{ - log: log, apeChecker: apeChecker, next: next, } From 580cd551807cea0ad2b9dfe9fbd21da0b55d6282 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 09:10:03 +0300 Subject: [PATCH 1388/1943] [#1388] getSvc: Drop unused Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assembleec.go | 2 +- pkg/services/object/get/assemblerec.go | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/services/object/get/assembleec.go b/pkg/services/object/get/assembleec.go index a58602bf7..03f913bbf 100644 --- a/pkg/services/object/get/assembleec.go +++ b/pkg/services/object/get/assembleec.go @@ -43,7 +43,7 @@ func (r *request) assembleEC(ctx context.Context) { } r.prm.common = r.prm.common.WithLocalOnly(false) - assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.localStorage, r.containerSource, r.log, r.headOnly(), r.traverserGenerator, r.curProcEpoch) + assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.localStorage, r.log, r.headOnly(), r.traverserGenerator, r.curProcEpoch) r.log.Debug(logs.GetAssemblingECObject, zap.Uint64("range_offset", r.ctxRange().GetOffset()), diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index dde0d7dad..44d9af3a2 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -34,7 +34,6 @@ type assemblerec struct { rng *objectSDK.Range remoteStorage ecRemoteStorage localStorage localStorage - cs container.Source log *logger.Logger head bool traverserGenerator traverserGenerator @@ -47,7 +46,6 @@ func newAssemblerEC( rng *objectSDK.Range, remoteStorage ecRemoteStorage, localStorage localStorage, - cs container.Source, log *logger.Logger, head bool, tg traverserGenerator, @@ -59,7 +57,6 @@ func newAssemblerEC( ecInfo: ecInfo, remoteStorage: remoteStorage, localStorage: localStorage, - cs: cs, log: log, head: head, traverserGenerator: tg, From 63a567a1de8d40b87c5e0cfcb99235eb4079f059 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 09:10:38 +0300 Subject: [PATCH 1389/1943] [#1388] engine: Drop unused Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/control.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 4778cf539..80fb3f9ed 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -249,23 +249,9 @@ func (e *StorageEngine) ResumeExecution() error { } type ReConfiguration struct { - errorsThreshold uint32 - shardPoolSize uint32 - shards map[string][]shard.Option // meta path -> shard opts } -// SetErrorsThreshold sets a size amount of errors after which -// shard is moved to read-only mode. -func (rCfg *ReConfiguration) SetErrorsThreshold(errorsThreshold uint32) { - rCfg.errorsThreshold = errorsThreshold -} - -// SetShardPoolSize sets a size of worker pool for each shard. -func (rCfg *ReConfiguration) SetShardPoolSize(shardPoolSize uint32) { - rCfg.shardPoolSize = shardPoolSize -} - // AddShard adds a shard for the reconfiguration. // Shard identifier is calculated from paths used in blobstor. func (rCfg *ReConfiguration) AddShard(id string, opts []shard.Option) { From 004ff9e9bf68174fbb64df6cbc81f98ced8755d3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 09:11:15 +0300 Subject: [PATCH 1390/1943] [#1388] blobstor: Drop unused Signed-off-by: Dmitrii Stepanov --- .../blobstor/memstore/control.go | 16 ++++++++-------- .../blobstor/memstore/memstore_test.go | 2 -- .../blobstor/memstore/option.go | 15 +-------------- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/pkg/local_object_storage/blobstor/memstore/control.go b/pkg/local_object_storage/blobstor/memstore/control.go index 449d4352a..83da52eb7 100644 --- a/pkg/local_object_storage/blobstor/memstore/control.go +++ b/pkg/local_object_storage/blobstor/memstore/control.go @@ -10,11 +10,11 @@ func (s *memstoreImpl) Open(mod mode.ComponentMode) error { return nil } -func (s *memstoreImpl) Init() error { return nil } -func (s *memstoreImpl) Close() error { return nil } -func (s *memstoreImpl) Type() string { return Type } -func (s *memstoreImpl) Path() string { return s.rootPath } -func (s *memstoreImpl) SetCompressor(cc *compression.Config) { s.compression = cc } -func (s *memstoreImpl) Compressor() *compression.Config { return s.compression } -func (s *memstoreImpl) SetReportErrorFunc(f func(string, error)) { s.reportError = f } -func (s *memstoreImpl) SetParentID(string) {} +func (s *memstoreImpl) Init() error { return nil } +func (s *memstoreImpl) Close() error { return nil } +func (s *memstoreImpl) Type() string { return Type } +func (s *memstoreImpl) Path() string { return s.rootPath } +func (s *memstoreImpl) SetCompressor(cc *compression.Config) { s.compression = cc } +func (s *memstoreImpl) Compressor() *compression.Config { return s.compression } +func (s *memstoreImpl) SetReportErrorFunc(func(string, error)) {} +func (s *memstoreImpl) SetParentID(string) {} diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index 8d1480dff..dd130e5f9 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/stretchr/testify/require" ) @@ -16,7 +15,6 @@ import ( func TestSimpleLifecycle(t *testing.T) { s := New( WithRootPath("memstore"), - WithLogger(test.NewLogger(t)), ) defer func() { require.NoError(t, s.Close()) }() require.NoError(t, s.Open(mode.ComponentReadWrite)) diff --git a/pkg/local_object_storage/blobstor/memstore/option.go b/pkg/local_object_storage/blobstor/memstore/option.go index 3d67b1e9c..97a03993d 100644 --- a/pkg/local_object_storage/blobstor/memstore/option.go +++ b/pkg/local_object_storage/blobstor/memstore/option.go @@ -2,33 +2,20 @@ package memstore import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" ) type cfg struct { - log *logger.Logger rootPath string readOnly bool compression *compression.Config - reportError func(string, error) } func defaultConfig() *cfg { - return &cfg{ - log: &logger.Logger{Logger: zap.L()}, - reportError: func(string, error) {}, - } + return &cfg{} } type Option func(*cfg) -func WithLogger(l *logger.Logger) Option { - return func(c *cfg) { - c.log = l - } -} - func WithRootPath(p string) Option { return func(c *cfg) { c.rootPath = p From 401c398704f15c1d516fbcc04f842d9d3fb8c2d3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 09:12:01 +0300 Subject: [PATCH 1391/1943] [#1388] metabase: Drop unused Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/delete.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index e5e9840a0..4ad11164f 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -77,8 +77,6 @@ func (p *DeletePrm) SetAddresses(addrs ...oid.Address) { type referenceNumber struct { all, cur int - addr oid.Address - obj *objectSDK.Object } @@ -295,9 +293,8 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter nRef, ok := refCounter[k] if !ok { nRef = &referenceNumber{ - all: parentLength(tx, parAddr), - addr: parAddr, - obj: parent, + all: parentLength(tx, parAddr), + obj: parent, } refCounter[k] = nRef From d1d6e3471c2e902c29480a091545f09c4daaf335 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 09:12:32 +0300 Subject: [PATCH 1392/1943] [#1388] signSvc: Drop unused Signed-off-by: Dmitrii Stepanov --- pkg/services/object/sign.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 35367aafe..f5ae97b62 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -10,8 +10,6 @@ import ( ) type SignService struct { - key *ecdsa.PrivateKey - sigSvc *util.SignService svc ServiceServer @@ -48,7 +46,6 @@ type getRangeStreamSigner struct { func NewSignService(key *ecdsa.PrivateKey, svc ServiceServer) *SignService { return &SignService{ - key: key, sigSvc: util.NewUnarySignService(key), svc: svc, } From bdd57c8b6b03f78ed74c31db41f5bbd0f3c84beb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 09:12:54 +0300 Subject: [PATCH 1393/1943] [#1388] sessionSvc: Add nolint annotations Used as map key. Signed-off-by: Dmitrii Stepanov --- pkg/services/session/storage/temporary/storage.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/services/session/storage/temporary/storage.go b/pkg/services/session/storage/temporary/storage.go index ee93dee71..9ae9db9dc 100644 --- a/pkg/services/session/storage/temporary/storage.go +++ b/pkg/services/session/storage/temporary/storage.go @@ -9,7 +9,9 @@ import ( ) type key struct { + // nolint:unused tokenID string + // nolint:unused ownerID string } From a2ab6d4942046c3bca59addd2b73ce3b58251b84 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 09:13:27 +0300 Subject: [PATCH 1394/1943] [#1388] node: Drop unused Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 - cmd/frostfs-node/container.go | 6 ------ cmd/frostfs-node/netmap.go | 1 - 3 files changed, 8 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 0ffa8c45b..c625b575f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -602,7 +602,6 @@ type cfgNetmap struct { needBootstrap bool reBoostrapTurnedOff *atomic.Bool // managed by control service in runtime - startEpoch uint64 // epoch number when application is started } type cfgNodeInfo struct { diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 5a29aac76..6733140d2 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -128,9 +128,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c cnrRdr.lister = client cnrRdr.eacl = c.cfgObject.eaclSource cnrRdr.src = c.cfgObject.cnrSource - - cnrWrt.cacheEnabled = true - cnrWrt.eacls = cachedEACLStorage } return cnrRdr, cnrWrt @@ -247,9 +244,6 @@ func (x *morphContainerReader) ContainersOf(id *user.ID) ([]cid.ID, error) { type morphContainerWriter struct { neoClient *cntClient.Client - - cacheEnabled bool - eacls ttlEACLStorage } func (m morphContainerWriter) Put(cnr containerCore.Container) (*cid.ID, error) { diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index c0b87492c..5e4585f85 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -259,7 +259,6 @@ func initNetmapState(c *cfg) { } c.cfgNetmap.state.setCurrentEpoch(epoch) - c.cfgNetmap.startEpoch = epoch c.setContractNodeInfo(ni) } From 29e4cf7ba1c88552172bdbb19dade34ea9ff5ba2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 23 Sep 2024 10:51:05 +0300 Subject: [PATCH 1395/1943] [#1388] ir: Annotate cmode as nolint Signed-off-by: Dmitrii Stepanov --- pkg/innerring/innerring.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 50a37845b..53a07e36c 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -103,6 +103,8 @@ type ( // to the application. runners []func(chan<- error) error + // cmode used for upgrade scenario. + // nolint:unused cmode *atomic.Bool } From 4fbfffd44c4e0f4aa7bc88052eff8400a0421f7c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 24 Sep 2024 12:13:11 +0300 Subject: [PATCH 1396/1943] [#1388] putSvc: Drop unused Signed-off-by: Dmitrii Stepanov --- pkg/services/object/put/prm.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pkg/services/object/put/prm.go b/pkg/services/object/put/prm.go index 0c8f12b45..52a7c102c 100644 --- a/pkg/services/object/put/prm.go +++ b/pkg/services/object/put/prm.go @@ -2,7 +2,6 @@ package putsvc import ( "context" - "crypto/ecdsa" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -21,8 +20,6 @@ type PutInitPrm struct { traverseOpts []placement.Option relay func(context.Context, client.NodeInfo, client.MultiAddressClient) error - - privateKey *ecdsa.PrivateKey } type PutChunkPrm struct { @@ -68,11 +65,3 @@ func (p *PutChunkPrm) WithChunk(v []byte) *PutChunkPrm { return p } - -func (p *PutInitPrm) WithPrivateKey(v *ecdsa.PrivateKey) *PutInitPrm { - if p != nil { - p.privateKey = v - } - - return p -} From 772b471aab53774e1d2cf11ae7db28166a47ec45 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 24 Sep 2024 15:58:52 +0300 Subject: [PATCH 1397/1943] [#1388] lens: Add nolint annotations Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-lens/internal/schema/common/raw.go | 2 ++ cmd/frostfs-lens/internal/schema/writecache/types.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cmd/frostfs-lens/internal/schema/common/raw.go b/cmd/frostfs-lens/internal/schema/common/raw.go index 0990e24c3..55051554c 100644 --- a/cmd/frostfs-lens/internal/schema/common/raw.go +++ b/cmd/frostfs-lens/internal/schema/common/raw.go @@ -7,6 +7,8 @@ import ( ) type RawEntry struct { + // key and value used for record dump. + // nolint:unused key, value []byte } diff --git a/cmd/frostfs-lens/internal/schema/writecache/types.go b/cmd/frostfs-lens/internal/schema/writecache/types.go index 3f71c5366..11e6f3fcd 100644 --- a/cmd/frostfs-lens/internal/schema/writecache/types.go +++ b/cmd/frostfs-lens/internal/schema/writecache/types.go @@ -16,6 +16,8 @@ type ( DefaultRecord struct { addr oid.Address + // data used for record dump. + // nolint:unused data []byte } ) From a5e1aa22c963fe612d6d2d3316ee7ca0482f0d09 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 25 Sep 2024 17:15:03 +0300 Subject: [PATCH 1398/1943] [#1394] putSvc: Fix relay Signed-off-by: Dmitrii Stepanov --- pkg/services/object/common/target/target.go | 9 +++++---- pkg/services/object/patch/streamer.go | 2 +- pkg/services/object/put/streamer.go | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/services/object/common/target/target.go b/pkg/services/object/common/target/target.go index 980c4c6bd..a2d6b4d39 100644 --- a/pkg/services/object/common/target/target.go +++ b/pkg/services/object/common/target/target.go @@ -13,16 +13,16 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) -func New(prm *objectwriter.Params) (transformer.ChunkedObjectWriter, error) { +func New(prm objectwriter.Params) (transformer.ChunkedObjectWriter, error) { // prepare needed put parameters - if err := preparePrm(prm); err != nil { + if err := preparePrm(&prm); err != nil { return nil, fmt.Errorf("could not prepare put parameters: %w", err) } if prm.Header.Signature() != nil { - return newUntrustedTarget(prm) + return newUntrustedTarget(&prm) } - return newTrustedTarget(prm) + return newTrustedTarget(&prm) } func newUntrustedTarget(prm *objectwriter.Params) (transformer.ChunkedObjectWriter, error) { @@ -49,6 +49,7 @@ func newUntrustedTarget(prm *objectwriter.Params) (transformer.ChunkedObjectWrit } func newTrustedTarget(prm *objectwriter.Params) (transformer.ChunkedObjectWriter, error) { + prm.Relay = nil // do not relay request without signature maxPayloadSz := prm.Config.MaxSizeSrc.MaxObjectSize() if maxPayloadSz == 0 { return nil, errors.New("could not obtain max object size parameter") diff --git a/pkg/services/object/patch/streamer.go b/pkg/services/object/patch/streamer.go index 73def8c7c..c8ed6fdbf 100644 --- a/pkg/services/object/patch/streamer.go +++ b/pkg/services/object/patch/streamer.go @@ -112,7 +112,7 @@ func (s *Streamer) init(ctx context.Context, req *objectV2.PatchRequest) error { } oV2.GetHeader().SetOwnerID(ownerID) - target, err := target.New(&objectwriter.Params{ + target, err := target.New(objectwriter.Params{ Config: s.Config, Common: commonPrm, Header: objectSDK.NewFromV2(oV2), diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index d08e7fafa..f71309d31 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -26,7 +26,7 @@ func (p *Streamer) Init(ctx context.Context, prm *PutInitPrm) error { } // initialize destination target - prmTarget := &objectwriter.Params{ + prmTarget := objectwriter.Params{ Config: p.Config, Common: prm.common, Header: prm.hdr, From 5f22ba6f380fd9d41be070f000b10cc4432981b9 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 27 Sep 2024 13:45:57 +0300 Subject: [PATCH 1399/1943] [#1397] object: Correctly set namespace before APE check Signed-off-by: Airat Arifullin --- pkg/services/object/ape/checker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 3688638d0..3f6cc7c20 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -108,7 +108,7 @@ func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { return c.checkerCore.CheckAPE(checkercore.CheckPrm{ Request: r, PublicKey: pub, - Namespace: prm.Method, + Namespace: prm.Namespace, Container: prm.Container, ContainerOwner: prm.ContainerOwner, BearerToken: prm.BearerToken, From d0ed29b3c73626f6bf881090f86bdc834d81acc1 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 3 Sep 2024 15:42:38 +0300 Subject: [PATCH 1400/1943] [#1350] node: Add ability to evacuate objects from `REP 1` only Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/control/evacuation.go | 4 + docs/evacuation.md | 7 +- pkg/local_object_storage/engine/evacuate.go | 39 +++- .../engine/evacuate_test.go | 181 +++++++++++++++++- pkg/local_object_storage/metabase/list.go | 51 +++++ pkg/local_object_storage/shard/list.go | 30 ++- pkg/services/control/server/evacuate_async.go | 1 + pkg/services/control/service.proto | 2 + pkg/services/control/service_frostfs.pb.go | 31 +++ 9 files changed, 340 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 04a67e5b5..fffc5e33e 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -20,6 +20,7 @@ const ( awaitFlag = "await" noProgressFlag = "no-progress" scopeFlag = "scope" + repOneOnlyFlag = "rep-one-only" containerWorkerCountFlag = "container-worker-count" objectWorkerCountFlag = "object-worker-count" @@ -69,6 +70,7 @@ func startEvacuateShard(cmd *cobra.Command, _ []string) { ignoreErrors, _ := cmd.Flags().GetBool(ignoreErrorsFlag) containerWorkerCount, _ := cmd.Flags().GetUint32(containerWorkerCountFlag) objectWorkerCount, _ := cmd.Flags().GetUint32(objectWorkerCountFlag) + repOneOnly, _ := cmd.Flags().GetBool(repOneOnlyFlag) req := &control.StartShardEvacuationRequest{ Body: &control.StartShardEvacuationRequest_Body{ @@ -77,6 +79,7 @@ func startEvacuateShard(cmd *cobra.Command, _ []string) { Scope: getEvacuationScope(cmd), ContainerWorkerCount: containerWorkerCount, ObjectWorkerCount: objectWorkerCount, + RepOneOnly: repOneOnly, }, } @@ -380,6 +383,7 @@ func initControlStartEvacuationShardCmd() { flags.Bool(noProgressFlag, false, fmt.Sprintf("Print progress if %s provided", awaitFlag)) flags.Uint32(containerWorkerCountFlag, 0, "Count of concurrent container evacuation workers") flags.Uint32(objectWorkerCountFlag, 0, "Count of concurrent object evacuation workers") + flags.Bool(repOneOnlyFlag, false, "Evacuate objects only from containers with policy 'REP 1 ...'") startEvacuationShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) } diff --git a/docs/evacuation.md b/docs/evacuation.md index 885ce169a..d47d56d15 100644 --- a/docs/evacuation.md +++ b/docs/evacuation.md @@ -20,7 +20,12 @@ Because it is necessary to prevent removing by policer objects with policy `REP ## Commands -`frostfs-cli control shards evacuation start` starts evacuation process for shards specified. To start evacuating all node shards, use the `--all` flag. By default, objects and trees are evacuated. To limit the evacuation scope, use `--scope` flag (possible values are `all`, `trees`, `objects`). +`frostfs-cli control shards evacuation start` starts evacuation process for shards specified. To start evacuating all node shards, use the `--all` flag. +By default, objects and trees are evacuated. To limit the evacuation scope, use `--scope` flag (possible values are `all`, `trees`, `objects`). +To evacuate objects only from containers with policy `REP 1` use option `--rep-one-only`. +To adjust resource consumption required for evacuation use options: + - `--container-worker-count` count of concurrent container evacuation workers + - `--object-worker-count` count of concurrent object evacuation workers `frostfs-cli control shards evacuation stop` stops running evacuation process. diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 3db556a8f..a618ff274 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" @@ -16,6 +17,7 @@ import ( tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -88,6 +90,7 @@ type EvacuateShardPrm struct { IgnoreErrors bool Async bool Scope EvacuateScope + RepOneOnly bool ContainerWorkerCount uint32 ObjectWorkerCount uint32 @@ -288,6 +291,7 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p attribute.Bool("async", prm.Async), attribute.Bool("ignoreErrors", prm.IgnoreErrors), attribute.Stringer("scope", prm.Scope), + attribute.Bool("repOneOnly", prm.RepOneOnly), )) defer func() { @@ -430,13 +434,34 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context ) error { sh := shardsToEvacuate[shardID] var cntPrm shard.IterateOverContainersPrm - cntPrm.Handler = func(ctx context.Context, name []byte, _ cid.ID) error { + cntPrm.Handler = func(ctx context.Context, name []byte, cnt cid.ID) error { select { case <-ctx.Done(): return context.Cause(ctx) default: } egContainer.Go(func() error { + var skip bool + c, err := e.containerSource.Load().cs.Get(cnt) + if err != nil { + if client.IsErrContainerNotFound(err) { + skip = true + } else { + return err + } + } + if !skip && prm.RepOneOnly { + skip = e.isNotRepOne(c) + } + if skip { + countPrm := shard.CountAliveObjectsInBucketPrm{BucketName: name} + count, err := sh.CountAliveObjectsInBucket(ctx, countPrm) + if err != nil { + return err + } + res.objSkipped.Add(count) + return nil + } var objPrm shard.IterateOverObjectsInContainerPrm objPrm.BucketName = name objPrm.Handler = func(ctx context.Context, objInfo *object.Info) error { @@ -454,7 +479,7 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context }) return nil } - err := sh.IterateOverObjectsInContainer(ctx, objPrm) + err = sh.IterateOverObjectsInContainer(ctx, objPrm) if err != nil { cancel(err) } @@ -781,6 +806,16 @@ func (e *StorageEngine) evacuateObject(ctx context.Context, shardID string, objI return nil } +func (e *StorageEngine) isNotRepOne(c *container.Container) bool { + p := c.Value.PlacementPolicy() + for i := range p.NumberOfReplicas() { + if p.ReplicaDescriptor(i).NumberOfObjects() > 1 { + return true + } + } + return false +} + func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes, ) (bool, error) { diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index f72333399..8498c9245 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + coreContainer "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" @@ -20,14 +21,38 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) +type containerStorage struct { + cntmap map[cid.ID]*container.Container + latency time.Duration +} + +func (cs *containerStorage) Get(id cid.ID) (*coreContainer.Container, error) { + time.Sleep(cs.latency) + v, ok := cs.cntmap[id] + if !ok { + return nil, new(apistatus.ContainerNotFound) + } + coreCnt := coreContainer.Container{ + Value: *v, + } + return &coreCnt, nil +} + +func (cs *containerStorage) DeletionInfo(cid.ID) (*coreContainer.DelInfo, error) { + return nil, nil +} + func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEngine, []*shard.ID, []*objectSDK.Object) { dir := t.TempDir() @@ -61,10 +86,15 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng {Key: pilorama.AttributeVersion, Value: []byte("XXX")}, {Key: pilorama.AttributeFilename, Value: []byte("file.txt")}, } - + cnrMap := make(map[cid.ID]*container.Container) for _, sh := range ids { - for range objPerShard { + for i := range objPerShard { + // Create dummy container + cnr1 := container.Container{} + cnr1.SetAttribute("cnr", "cnr"+strconv.Itoa(i)) contID := cidtest.ID() + cnrMap[contID] = &cnr1 + obj := testutil.GenerateObjectWithCID(contID) objects = append(objects, obj) @@ -78,6 +108,7 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng require.NoError(t, err) } } + e.SetContainerSource(&containerStorage{cntmap: cnrMap}) return e, ids, objects } @@ -177,7 +208,10 @@ func TestEvacuateObjectsNetwork(t *testing.T) { acceptOneOf := func(objects []*objectSDK.Object, max uint64) func(context.Context, oid.Address, *objectSDK.Object) (bool, error) { var n atomic.Uint64 + var mtx sync.Mutex return func(_ context.Context, addr oid.Address, obj *objectSDK.Object) (bool, error) { + mtx.Lock() + defer mtx.Unlock() if n.Load() == max { return false, errReplication } @@ -640,3 +674,146 @@ func TestEvacuateTreesRemote(t *testing.T) { require.Equal(t, expectedTreeOps, evacuatedTreeOps) } + +func TestEvacuateShardObjectsRepOneOnly(t *testing.T) { + e, ids, _ := newEngineEvacuate(t, 2, 0) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() + + // Create container with policy REP 2 + cnr1 := container.Container{} + p1 := netmap.PlacementPolicy{} + p1.SetContainerBackupFactor(1) + x1 := netmap.ReplicaDescriptor{} + x1.SetNumberOfObjects(2) + p1.AddReplicas(x1) + x1 = netmap.ReplicaDescriptor{} + x1.SetNumberOfObjects(1) + p1.AddReplicas(x1) + cnr1.SetPlacementPolicy(p1) + cnr1.SetAttribute("cnr", "cnr1") + + var idCnr1 cid.ID + container.CalculateID(&idCnr1, cnr1) + + cnrmap := make(map[cid.ID]*container.Container) + var cids []cid.ID + cnrmap[idCnr1] = &cnr1 + cids = append(cids, idCnr1) + + // Create container with policy REP 1 + cnr2 := container.Container{} + p2 := netmap.PlacementPolicy{} + p2.SetContainerBackupFactor(1) + x2 := netmap.ReplicaDescriptor{} + x2.SetNumberOfObjects(1) + p2.AddReplicas(x2) + x2 = netmap.ReplicaDescriptor{} + x2.SetNumberOfObjects(1) + p2.AddReplicas(x2) + cnr2.SetPlacementPolicy(p2) + cnr2.SetAttribute("cnr", "cnr2") + + var idCnr2 cid.ID + container.CalculateID(&idCnr2, cnr2) + cnrmap[idCnr2] = &cnr2 + cids = append(cids, idCnr2) + + // Create container for simulate removing + cnr3 := container.Container{} + p3 := netmap.PlacementPolicy{} + p3.SetContainerBackupFactor(1) + x3 := netmap.ReplicaDescriptor{} + x3.SetNumberOfObjects(1) + p3.AddReplicas(x3) + cnr3.SetPlacementPolicy(p3) + cnr3.SetAttribute("cnr", "cnr3") + + var idCnr3 cid.ID + container.CalculateID(&idCnr3, cnr3) + cids = append(cids, idCnr3) + + e.SetContainerSource(&containerStorage{cntmap: cnrmap}) + + for _, sh := range ids { + for j := range 3 { + for range 4 { + obj := testutil.GenerateObjectWithCID(cids[j]) + var putPrm shard.PutPrm + putPrm.SetObject(obj) + _, err := e.shards[sh.String()].Put(context.Background(), putPrm) + require.NoError(t, err) + } + } + } + + var prm EvacuateShardPrm + prm.ShardID = ids[0:1] + prm.Scope = EvacuateScopeObjects + prm.RepOneOnly = true + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + + res, err := e.Evacuate(context.Background(), prm) + require.NoError(t, err) + require.Equal(t, uint64(4), res.ObjectsEvacuated()) + require.Equal(t, uint64(8), res.ObjectsSkipped()) + require.Equal(t, uint64(0), res.ObjectsFailed()) +} + +func TestEvacuateShardObjectsRepOneOnlyBench(t *testing.T) { + t.Skip() + e, ids, _ := newEngineEvacuate(t, 2, 0) + defer func() { + require.NoError(t, e.Close(context.Background())) + }() + + cnrmap := make(map[cid.ID]*container.Container) + var cids []cid.ID + // Create containers with policy REP 1 + for i := range 10_000 { + cnr1 := container.Container{} + p1 := netmap.PlacementPolicy{} + p1.SetContainerBackupFactor(1) + x1 := netmap.ReplicaDescriptor{} + x1.SetNumberOfObjects(2) + p1.AddReplicas(x1) + cnr1.SetPlacementPolicy(p1) + cnr1.SetAttribute("i", strconv.Itoa(i)) + + var idCnr1 cid.ID + container.CalculateID(&idCnr1, cnr1) + + cnrmap[idCnr1] = &cnr1 + cids = append(cids, idCnr1) + } + + e.SetContainerSource(&containerStorage{ + cntmap: cnrmap, + latency: time.Millisecond * 100, + }) + + for _, cnt := range cids { + for range 1 { + obj := testutil.GenerateObjectWithCID(cnt) + var putPrm shard.PutPrm + putPrm.SetObject(obj) + _, err := e.shards[ids[0].String()].Put(context.Background(), putPrm) + require.NoError(t, err) + } + } + + var prm EvacuateShardPrm + prm.ShardID = ids[0:1] + prm.Scope = EvacuateScopeObjects + prm.RepOneOnly = true + prm.ContainerWorkerCount = 10 + + require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + + start := time.Now() + _, err := e.Evacuate(context.Background(), prm) + t.Logf("evacuate took %v\n", time.Since(start)) + require.NoError(t, err) +} diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 5943be7f4..44f25246e 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -76,6 +76,12 @@ type IterateOverObjectsInContainerPrm struct { Handler func(context.Context, *objectcore.Info) error } +// CountAliveObjectsInBucketPrm contains parameters for IterateOverObjectsInContainer operation. +type CountAliveObjectsInBucketPrm struct { + // BucketName container's bucket name. + BucketName []byte +} + // ListWithCursor lists physical objects available in metabase starting from // cursor. Includes objects of all types. Does not include inhumed objects. // Use cursor value from response for consecutive requests. @@ -426,3 +432,48 @@ func (db *DB) iterateOverObjectsInContainer(ctx context.Context, tx *bbolt.Tx, c } return nil } + +// CountAliveObjectsInBucket count objects in bucket which aren't in graveyard or garbage. +func (db *DB) CountAliveObjectsInBucket(ctx context.Context, prm CountAliveObjectsInBucketPrm) (uint64, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("CountAliveObjectsInBucket", time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "metabase.CountAliveObjectsInBucket") + defer span.End() + + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return 0, ErrDegradedMode + } + + cidRaw := prm.BucketName[1:bucketKeySize] + if cidRaw == nil { + return 0, nil + } + var count uint64 + err := db.boltDB.View(func(tx *bbolt.Tx) error { + bkt := tx.Bucket(prm.BucketName) + if bkt == nil { + return nil + } + graveyardBkt := tx.Bucket(graveyardBucketName) + garbageBkt := tx.Bucket(garbageBucketName) + c := bkt.Cursor() + k, _ := c.First() + for ; k != nil; k, _ = c.Next() { + if inGraveyardWithKey(append(cidRaw, k...), graveyardBkt, garbageBkt) > 0 { + continue + } + count++ + } + return nil + }) + success = err == nil + return count, metaerr.Wrap(err) +} diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index 9f56ec750..f5d633b77 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -44,10 +44,16 @@ type IterateOverContainersPrm struct { type IterateOverObjectsInContainerPrm struct { // BucketName container's bucket name. BucketName []byte - // Handler function executed upon containers in db. + // Handler function executed upon objects in db. Handler func(context.Context, *objectcore.Info) error } +// CountAliveObjectsInBucketPrm contains parameters for CountAliveObjectsInBucket operation. +type CountAliveObjectsInBucketPrm struct { + // BucketName container's bucket name. + BucketName []byte +} + // ListWithCursorPrm contains parameters for ListWithCursor operation. type ListWithCursorPrm struct { count uint32 @@ -229,3 +235,25 @@ func (s *Shard) IterateOverObjectsInContainer(ctx context.Context, prm IterateOv return nil } + +// CountAliveObjectsInBucket count objects in bucket which aren't in graveyard or garbage. +func (s *Shard) CountAliveObjectsInBucket(ctx context.Context, prm CountAliveObjectsInBucketPrm) (uint64, error) { + _, span := tracing.StartSpanFromContext(ctx, "shard.CountAliveObjectsInBucket") + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { + return 0, ErrDegradedMode + } + + var metaPrm meta.CountAliveObjectsInBucketPrm + metaPrm.BucketName = prm.BucketName + count, err := s.metaBase.CountAliveObjectsInBucket(ctx, metaPrm) + if err != nil { + return 0, fmt.Errorf("could not count alive objects in bucket: %w", err) + } + + return count, nil +} diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index bdc6f7c38..146ac7e16 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -31,6 +31,7 @@ func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartSha Scope: engine.EvacuateScope(req.GetBody().GetScope()), ContainerWorkerCount: req.GetBody().GetContainerWorkerCount(), ObjectWorkerCount: req.GetBody().GetObjectWorkerCount(), + RepOneOnly: req.GetBody().GetRepOneOnly(), } _, err = s.s.Evacuate(ctx, prm) diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 88a06de22..ae1939e13 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -398,6 +398,8 @@ message StartShardEvacuationRequest { uint32 container_worker_count = 4; // Count of concurrent object evacuation workers. uint32 object_worker_count = 5; + // Choose for evacuation objects in `REP 1` containers only. + bool rep_one_only = 6; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index e92a8acd1..e16f082b1 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -6516,6 +6516,7 @@ type StartShardEvacuationRequest_Body struct { Scope uint32 `json:"scope"` ContainerWorkerCount uint32 `json:"containerWorkerCount"` ObjectWorkerCount uint32 `json:"objectWorkerCount"` + RepOneOnly bool `json:"repOneOnly"` } var ( @@ -6537,6 +6538,7 @@ func (x *StartShardEvacuationRequest_Body) StableSize() (size int) { size += proto.UInt32Size(3, x.Scope) size += proto.UInt32Size(4, x.ContainerWorkerCount) size += proto.UInt32Size(5, x.ObjectWorkerCount) + size += proto.BoolSize(6, x.RepOneOnly) return size } @@ -6568,6 +6570,9 @@ func (x *StartShardEvacuationRequest_Body) EmitProtobuf(mm *easyproto.MessageMar if x.ObjectWorkerCount != 0 { mm.AppendUint32(5, x.ObjectWorkerCount) } + if x.RepOneOnly { + mm.AppendBool(6, x.RepOneOnly) + } } // UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. @@ -6609,6 +6614,12 @@ func (x *StartShardEvacuationRequest_Body) UnmarshalProtobuf(src []byte) (err er return fmt.Errorf("cannot unmarshal field %s", "ObjectWorkerCount") } x.ObjectWorkerCount = data + case 6: // RepOneOnly + data, ok := fc.Bool() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "RepOneOnly") + } + x.RepOneOnly = data } } return nil @@ -6658,6 +6669,15 @@ func (x *StartShardEvacuationRequest_Body) GetObjectWorkerCount() uint32 { func (x *StartShardEvacuationRequest_Body) SetObjectWorkerCount(v uint32) { x.ObjectWorkerCount = v } +func (x *StartShardEvacuationRequest_Body) GetRepOneOnly() bool { + if x != nil { + return x.RepOneOnly + } + return false +} +func (x *StartShardEvacuationRequest_Body) SetRepOneOnly(v bool) { + x.RepOneOnly = v +} // MarshalJSON implements the json.Marshaler interface. func (x *StartShardEvacuationRequest_Body) MarshalJSON() ([]byte, error) { @@ -6703,6 +6723,11 @@ func (x *StartShardEvacuationRequest_Body) MarshalEasyJSON(out *jwriter.Writer) out.RawString(prefix) out.Uint32(x.ObjectWorkerCount) } + { + const prefix string = ",\"repOneOnly\":" + out.RawString(prefix) + out.Bool(x.RepOneOnly) + } out.RawByte('}') } @@ -6768,6 +6793,12 @@ func (x *StartShardEvacuationRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { f = in.Uint32() x.ObjectWorkerCount = f } + case "repOneOnly": + { + var f bool + f = in.Bool() + x.RepOneOnly = f + } } in.WantComma() } From 7f8a1dcf8e238a08af84a1ef9e180541f783b71f Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 30 Sep 2024 14:15:13 +0300 Subject: [PATCH 1401/1943] [#1400] adm: Support flag `alphabet-wallets` for commands `proxy-add/remove-account` Signed-off-by: Anton Nikiforov --- cmd/frostfs-adm/internal/modules/morph/proxy/root.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy/root.go b/cmd/frostfs-adm/internal/modules/morph/proxy/root.go index 082bc57d1..1854c8d2b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/root.go @@ -30,11 +30,13 @@ var ( func initProxyAddAccount() { AddAccountCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) AddAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") + AddAccountCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initProxyRemoveAccount() { RemoveAccountCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) RemoveAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") + RemoveAccountCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func init() { From a13219808a42f30839fe87ba3ea88a8fdd54f0ac Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 27 Sep 2024 12:39:43 +0300 Subject: [PATCH 1402/1943] [#1375] node: Configure of the container cache size Signed-off-by: Alexander Chuprov --- cmd/frostfs-node/cache.go | 8 ++-- cmd/frostfs-node/config.go | 2 + cmd/frostfs-node/config/morph/config.go | 15 ++++++ cmd/frostfs-node/container.go | 63 +++++++++++++------------ cmd/frostfs-node/morph.go | 1 + config/example/node.yaml | 1 + 6 files changed, 55 insertions(+), 35 deletions(-) diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index 57f65d873..06142a46c 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -165,13 +165,11 @@ type ttlContainerStorage struct { delInfoCache *ttlNetCache[cid.ID, *container.DelInfo] } -func newCachedContainerStorage(v container.Source, ttl time.Duration) ttlContainerStorage { - const containerCacheSize = 100 - - lruCnrCache := newNetworkTTLCache(containerCacheSize, ttl, func(id cid.ID) (*container.Container, error) { +func newCachedContainerStorage(v container.Source, ttl time.Duration, containerCacheSize uint32) ttlContainerStorage { + lruCnrCache := newNetworkTTLCache(int(containerCacheSize), ttl, func(id cid.ID) (*container.Container, error) { return v.Get(id) }, metrics.NewCacheMetrics("container")) - lruDelInfoCache := newNetworkTTLCache(containerCacheSize, ttl, func(id cid.ID) (*container.DelInfo, error) { + lruDelInfoCache := newNetworkTTLCache(int(containerCacheSize), ttl, func(id cid.ID) (*container.DelInfo, error) { return v.DeletionInfo(id) }, metrics.NewCacheMetrics("container_deletion_info")) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c625b575f..58a96879f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -570,6 +570,8 @@ type cfgMorph struct { // TTL of Sidechain cached values. Non-positive value disables caching. cacheTTL time.Duration + containerCacheSize uint32 + proxyScriptHash neogoutil.Uint160 } diff --git a/cmd/frostfs-node/config/morph/config.go b/cmd/frostfs-node/config/morph/config.go index 1c536a0e2..d089870ea 100644 --- a/cmd/frostfs-node/config/morph/config.go +++ b/cmd/frostfs-node/config/morph/config.go @@ -30,6 +30,9 @@ const ( // FrostfsIDCacheSizeDefault is a default value of APE chain cache. FrostfsIDCacheSizeDefault = 10_000 + + // ContainerCacheSizeDefault represents the default size for the container cache. + ContainerCacheSizeDefault = 100 ) var errNoMorphEndpoints = errors.New("no morph chain RPC endpoints, see `morph.rpc_endpoint` section") @@ -103,6 +106,18 @@ func CacheTTL(c *config.Config) time.Duration { return CacheTTLDefault } +// ContainerCacheSize returns the value of "container_cache_size" config parameter +// from "morph" section. +// +// Returns 0 if the value is not positive integer. +// Returns ContainerCacheSizeDefault if the value is missing. +func ContainerCacheSize(c *config.Config) uint32 { + if c.Sub(subsection).Value("container_cache_size") == nil { + return ContainerCacheSizeDefault + } + return config.Uint32Safe(c.Sub(subsection), "container_cache_size") +} + // SwitchInterval returns the value of "switch_interval" config parameter // from "morph" section. // diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 6733140d2..729fcb8af 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -87,43 +87,46 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c cnrRdr.lister = client } else { // use RPC node as source of Container contract items (with caching) - cachedContainerStorage := newCachedContainerStorage(cnrSrc, c.cfgMorph.cacheTTL) - cachedEACLStorage := newCachedEACLStorage(eACLFetcher, c.cfgMorph.cacheTTL) + c.cfgObject.cnrSource = cnrSrc + if c.cfgMorph.containerCacheSize > 0 { + containerCache := newCachedContainerStorage(cnrSrc, c.cfgMorph.cacheTTL, c.cfgMorph.containerCacheSize) - subscribeToContainerCreation(c, func(e event.Event) { - ev := e.(containerEvent.PutSuccess) + subscribeToContainerCreation(c, func(e event.Event) { + ev := e.(containerEvent.PutSuccess) - // read owner of the created container in order to update the reading cache. - // TODO: use owner directly from the event after neofs-contract#256 will become resolved - // but don't forget about the profit of reading the new container and caching it: - // creation success are most commonly tracked by polling GET op. - cnr, err := cnrSrc.Get(ev.ID) - if err == nil { - cachedContainerStorage.containerCache.set(ev.ID, cnr, nil) - } else { - // unlike removal, we expect successful receive of the container - // after successful creation, so logging can be useful - c.log.Error(logs.FrostFSNodeReadNewlyCreatedContainerAfterTheNotification, + // read owner of the created container in order to update the reading cache. + // TODO: use owner directly from the event after neofs-contract#256 will become resolved + // but don't forget about the profit of reading the new container and caching it: + // creation success are most commonly tracked by polling GET op. + cnr, err := cnrSrc.Get(ev.ID) + if err == nil { + containerCache.containerCache.set(ev.ID, cnr, nil) + } else { + // unlike removal, we expect successful receive of the container + // after successful creation, so logging can be useful + c.log.Error(logs.FrostFSNodeReadNewlyCreatedContainerAfterTheNotification, + zap.Stringer("id", ev.ID), + zap.Error(err), + ) + } + + c.log.Debug(logs.FrostFSNodeContainerCreationEventsReceipt, zap.Stringer("id", ev.ID), - zap.Error(err), ) - } + }) - c.log.Debug(logs.FrostFSNodeContainerCreationEventsReceipt, - zap.Stringer("id", ev.ID), - ) - }) - - subscribeToContainerRemoval(c, func(e event.Event) { - ev := e.(containerEvent.DeleteSuccess) - cachedContainerStorage.handleRemoval(ev.ID) - c.log.Debug(logs.FrostFSNodeContainerRemovalEventsReceipt, - zap.Stringer("id", ev.ID), - ) - }) + subscribeToContainerRemoval(c, func(e event.Event) { + ev := e.(containerEvent.DeleteSuccess) + containerCache.handleRemoval(ev.ID) + c.log.Debug(logs.FrostFSNodeContainerRemovalEventsReceipt, + zap.Stringer("id", ev.ID), + ) + }) + c.cfgObject.cnrSource = containerCache + } + cachedEACLStorage := newCachedEACLStorage(eACLFetcher, c.cfgMorph.cacheTTL) c.cfgObject.eaclSource = cachedEACLStorage - c.cfgObject.cnrSource = cachedContainerStorage cnrRdr.lister = client cnrRdr.eacl = c.cfgObject.eaclSource diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 7178cd97d..1bfcb8ac9 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -90,6 +90,7 @@ func initMorphComponents(ctx context.Context, c *cfg) { var netmapSource netmap.Source + c.cfgMorph.containerCacheSize = morphconfig.ContainerCacheSize(c.appCfg) c.cfgMorph.cacheTTL = morphconfig.CacheTTL(c.appCfg) if c.cfgMorph.cacheTTL == 0 { diff --git a/config/example/node.yaml b/config/example/node.yaml index 86be35ba8..2a80fba18 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -81,6 +81,7 @@ morph: cache_ttl: 15s # Sidechain cache TTL value (min interval between similar calls). Negative value disables caching. # Default value: block time. It is recommended to have this value less or equal to block time. # Cached entities: containers, container lists, eACL tables. + container_cache_size: 100 # container_cache_size is is the maximum number of containers in the cache. switch_interval: 3m # interval b/w RPC switch attempts if the node is connected not to the highest priority node rpc_endpoint: # side chain NEO RPC endpoints; are shuffled and used one by one until the first success - address: wss://rpc1.morph.frostfs.info:40341/ws From 54eb0058229965b7ddd704fe4da2e24f41c20f3f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 1 Oct 2024 14:39:36 +0300 Subject: [PATCH 1403/1943] [#1404] go.mod: Update api-go Fix #1398 Fix #1399 Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9817f8527..1023948bc 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241001111852-d9a604fbc16f git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d diff --git a/go.sum b/go.sum index 3c6dd9a99..5d719a027 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e h1:740ABnOBYx4o6jxULHdSSnVW2fYIO35ohg+Uz59sxd0= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240916093537-13fa0da3741e/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241001111852-d9a604fbc16f h1:FZvX6CLzTQqMyMvOerIKMvIEJQbOImDjSooZx3AVRyE= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241001111852-d9a604fbc16f/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c h1:9JmXxWcqWaOXModBwtj/WlPYOfSk2jRSDoRsDRYvgnA= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c/go.mod h1:DVb1AqP323wiEwCtlfoJJTERF4Xji2L39X6T8kv26RE= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= From 9c5ddc4dfeb6447ae7d9cc0d74db551271ac6eb1 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 2 Oct 2024 10:09:10 +0300 Subject: [PATCH 1404/1943] [#1407] tree: Set `ContainerOwner` in parameter for `CheckAPE` Signed-off-by: Airat Arifullin --- pkg/services/tree/ape.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 693b16e60..69cf59405 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -75,12 +75,13 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, } return s.apeChecker.CheckAPE(checkercore.CheckPrm{ - Request: request, - Namespace: namespace, - Container: cid, - PublicKey: publicKey, - BearerToken: bt, - SoftAPECheck: false, + Request: request, + Namespace: namespace, + Container: cid, + ContainerOwner: container.Value.Owner(), + PublicKey: publicKey, + BearerToken: bt, + SoftAPECheck: false, }) } From 57c31e9802ad19b8d64388315cd53a05515e353e Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 1 Oct 2024 16:09:05 +0300 Subject: [PATCH 1405/1943] [#1306] node: Allow tombstone_lifetime config to be loaded on the fly Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-node/config.go | 17 +++++++++++++++-- cmd/frostfs-node/object.go | 6 ++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 58a96879f..4ad9ec6c6 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -105,6 +105,10 @@ type applicationConfiguration struct { timestamp bool } + ObjectCfg struct { + tombstoneLifetime uint64 + } + EngineCfg struct { errorThreshold uint32 shardPoolSize uint32 @@ -223,6 +227,10 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { a.LoggerCfg.destination = loggerconfig.Destination(c) a.LoggerCfg.timestamp = loggerconfig.Timestamp(c) + // Object + + a.ObjectCfg.tombstoneLifetime = objectconfig.TombstoneLifetime(c) + // Storage Engine a.EngineCfg.errorThreshold = engineconfig.ShardErrorThreshold(c) @@ -624,7 +632,7 @@ type cfgObject struct { cfgLocalStorage cfgLocalStorage - tombstoneLifetime uint64 + tombstoneLifetime *atomic.Uint64 skipSessionTokenIssuerVerification bool } @@ -815,9 +823,11 @@ func initCfgGRPC() cfgGRPC { } func initCfgObject(appCfg *config.Config) cfgObject { + var tsLifetime atomic.Uint64 + tsLifetime.Store(objectconfig.TombstoneLifetime(appCfg)) return cfgObject{ pool: initObjectPool(appCfg), - tombstoneLifetime: objectconfig.TombstoneLifetime(appCfg), + tombstoneLifetime: &tsLifetime, skipSessionTokenIssuerVerification: objectconfig.Put(appCfg).SkipSessionTokenIssuerVerification(), } } @@ -1296,6 +1306,9 @@ func (c *cfg) reloadConfig(ctx context.Context) { components := c.getComponents(ctx, logPrm) + // Object + c.cfgObject.tombstoneLifetime.Store(c.ObjectCfg.tombstoneLifetime) + // Storage Engine var rcfg engine.ReConfiguration diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 9d4e35ca8..47649c88b 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -109,13 +109,12 @@ func (s *objectSvc) GetRangeHash(ctx context.Context, req *object.GetRangeHashRe type delNetInfo struct { netmap.State - tsLifetime uint64 cfg *cfg } func (i *delNetInfo) TombstoneLifetime() (uint64, error) { - return i.tsLifetime, nil + return i.cfg.cfgObject.tombstoneLifetime.Load(), nil } // returns node owner ID calculated from configured private key. @@ -424,8 +423,7 @@ func createDeleteService(c *cfg, keyStorage *util.KeyStorage, sGet *getsvc.Servi sSearch, sPut, &delNetInfo{ - State: c.cfgNetmap.state, - tsLifetime: c.cfgObject.tombstoneLifetime, + State: c.cfgNetmap.state, cfg: c, }, From f45e75e3eb781662abbbc4f6820076c492fc1f0d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 2 Oct 2024 11:18:35 +0300 Subject: [PATCH 1406/1943] [#1409] adm: Do not bind DeltaFlag to viper We bind flag that could be specified in config. This is not a config flag, just a command option. Also fix TestInitialize failures: ``` Error: Received unexpected error: number of epochs cannot be less than 1 Test: TestInitialize/16_nodes/force-new-epoch ``` Refs #1372 (945b7c740b0deb4) Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go | 3 ++- cmd/frostfs-adm/internal/modules/morph/netmap/root.go | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go index 5e4e9c725..5c5fa9988 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go @@ -31,7 +31,8 @@ func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { } bw := io.NewBufBinWriter() - if err := helper.EmitNewEpochCall(bw, wCtx, nmHash, viper.GetInt64(commonflags.DeltaFlag)); err != nil { + delta, _ := cmd.Flags().GetInt64(commonflags.DeltaFlag) + if err := helper.EmitNewEpochCall(bw, wCtx, nmHash, delta); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go index 0288bcdc5..3300db36a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go @@ -22,7 +22,6 @@ var ( PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) - _ = viper.BindPFlag(commonflags.DeltaFlag, cmd.Flags().Lookup(commonflags.DeltaFlag)) }, RunE: ForceNewEpochCmd, } From 62028cd7ee0b5d825b71cfa11d1d87369b1da23d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 2 Oct 2024 11:20:09 +0300 Subject: [PATCH 1407/1943] [#1409] adm: Uncommonize DeltaFlag It is used only in `force-new-epoch`, it is not _common_ between multiple commands. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/commonflags/flags.go | 1 - cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go | 5 +++-- cmd/frostfs-adm/internal/modules/morph/netmap/root.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-adm/internal/commonflags/flags.go b/cmd/frostfs-adm/internal/commonflags/flags.go index b51d2e115..81395edb0 100644 --- a/cmd/frostfs-adm/internal/commonflags/flags.go +++ b/cmd/frostfs-adm/internal/commonflags/flags.go @@ -39,5 +39,4 @@ const ( CustomZoneFlag = "domain" AlphabetSizeFlag = "size" AllFlag = "all" - DeltaFlag = "delta" ) diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go index 5c5fa9988..94223dbd0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/epoch.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "github.com/nspcc-dev/neo-go/pkg/io" @@ -13,6 +12,8 @@ import ( "github.com/spf13/viper" ) +const deltaFlag = "delta" + func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { @@ -31,7 +32,7 @@ func ForceNewEpochCmd(cmd *cobra.Command, _ []string) error { } bw := io.NewBufBinWriter() - delta, _ := cmd.Flags().GetInt64(commonflags.DeltaFlag) + delta, _ := cmd.Flags().GetInt64(deltaFlag) if err := helper.EmitNewEpochCall(bw, wCtx, nmHash, delta); err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go index 3300db36a..55b7e64f0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go @@ -35,7 +35,7 @@ func initForceNewEpochCmd() { ForceNewEpoch.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) ForceNewEpoch.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) ForceNewEpoch.Flags().String(commonflags.LocalDumpFlag, "", "Path to the blocks dump file") - ForceNewEpoch.Flags().Int64(commonflags.DeltaFlag, 1, "Number of epochs to increase the current epoch") + ForceNewEpoch.Flags().Int64(deltaFlag, 1, "Number of epochs to increase the current epoch") } func init() { From f83f7feb8caa0ef5ab9a952a6a6d3e2f12a63486 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 2 Oct 2024 11:01:22 +0300 Subject: [PATCH 1408/1943] [#1391] adm: Properly check whether transfers were made Signed-off-by: Evgenii Stratonikov --- .../morph/initialize/initialize_transfer.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go index d7b0ec86c..7f1bfee2b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go @@ -27,12 +27,12 @@ const ( initialAlphabetGASAmount = 10_000 * native.GASFactor // initialProxyGASAmount represents the amount of GAS given to a proxy contract. initialProxyGASAmount = 50_000 * native.GASFactor - // alphabetGasRatio is a coefficient that defines the threshold below which - // the balance of the alphabet node is considered not replenished. The value - // of this coefficient is determined empirically. - alphabetGasRatio = 5 ) +func initialCommitteeGASAmount(c *helper.InitializeContext) int64 { + return (gasInitialTotalSupply - initialAlphabetGASAmount*int64(len(c.Wallets))) / 2 +} + func transferFunds(c *helper.InitializeContext) error { ok, err := transferFundsFinished(c) if ok || err != nil { @@ -59,7 +59,7 @@ func transferFunds(c *helper.InitializeContext) error { transferTarget{ Token: gas.Hash, Address: c.CommitteeAcc.Contract.ScriptHash(), - Amount: (gasInitialTotalSupply - initialAlphabetGASAmount*int64(len(c.Wallets))) / 2, + Amount: initialCommitteeGASAmount(c), }, transferTarget{ Token: neo.Hash, @@ -80,12 +80,19 @@ func transferFunds(c *helper.InitializeContext) error { return c.AwaitTx() } +// transferFundsFinished checks balances of accounts we transfer GAS to. +// The stage is considered finished if the balance is greater than the half of what we need to transfer. func transferFundsFinished(c *helper.InitializeContext) (bool, error) { acc := c.Accounts[0] r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) res, err := r.BalanceOf(acc.Contract.ScriptHash()) - return res.Cmp(big.NewInt(alphabetGasRatio*native.GASFactor)) == 1, err + if err != nil || res.Cmp(big.NewInt(initialAlphabetGASAmount/2)) != 1 { + return false, err + } + + res, err = r.BalanceOf(c.CommitteeAcc.ScriptHash()) + return res != nil && res.Cmp(big.NewInt(initialCommitteeGASAmount(c)/2)) == 1, err } func transferGASToProxy(c *helper.InitializeContext) error { From 434048e8d959b29375c0d63a112b8eb8df8792d8 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Oct 2024 11:28:00 +0300 Subject: [PATCH 1409/1943] [#1408] metabase: Fix EC search with slow and fast filters Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/select.go | 35 ++++++++++++- .../metabase/select_test.go | 50 +++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index ed43fc41f..85d1b08ba 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -389,8 +389,7 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.Searc return result, true } - buf := make([]byte, addressKeySize) - obj, err := db.get(tx, addr, buf, true, false, currEpoch) + obj, isECChunk, err := db.getObjectForSlowFilters(tx, addr, currEpoch) if err != nil { return result, false } @@ -401,17 +400,26 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.Searc case v2object.FilterHeaderVersion: data = []byte(obj.Version().String()) case v2object.FilterHeaderHomomorphicHash: + if isECChunk { + return result, false // EC chunk and EC parent hashes are incomparable + } cs, _ := obj.PayloadHomomorphicHash() data = cs.Value() case v2object.FilterHeaderCreationEpoch: data = make([]byte, 8) binary.LittleEndian.PutUint64(data, obj.CreationEpoch()) case v2object.FilterHeaderPayloadLength: + if isECChunk { + return result, false // EC chunk and EC parent payload lengths are incomparable + } data = make([]byte, 8) binary.LittleEndian.PutUint64(data, obj.PayloadSize()) case v2object.FilterHeaderOwnerID: data = []byte(obj.OwnerID().EncodeToString()) case v2object.FilterHeaderPayloadHash: + if isECChunk { + return result, false // EC chunk and EC parent payload hashes are incomparable + } cs, _ := obj.PayloadChecksum() data = cs.Value() default: // user attribute @@ -439,6 +447,29 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.Searc return result, true } +func (db *DB) getObjectForSlowFilters(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (*objectSDK.Object, bool, error) { + buf := make([]byte, addressKeySize) + obj, err := db.get(tx, addr, buf, true, false, currEpoch) + if err != nil { + var ecInfoError *objectSDK.ECInfoError + if errors.As(err, &ecInfoError) { + for _, chunk := range ecInfoError.ECInfo().Chunks { + var objID oid.ID + if err = objID.ReadFromV2(chunk.ID); err != nil { + continue + } + addr.SetObject(objID) + obj, err = db.get(tx, addr, buf, true, false, currEpoch) + if err == nil { + return obj, true, nil + } + } + } + return nil, false, err + } + return obj, false, nil +} + func attributeValue(obj *objectSDK.Object, attribute string) (string, bool) { objectAttributes := obj.Attributes() if ech := obj.ECHeader(); ech != nil { diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index fcd5d3a90..0c6ebc863 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -762,6 +762,56 @@ func TestDB_SelectOwnerID(t *testing.T) { }) } +func TestDB_SelectECWithFastAndSlowFilters(t *testing.T) { + t.Parallel() + + db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() + + cnr := cidtest.ID() + ecChunk1 := oidtest.ID() + ecChunk2 := oidtest.ID() + ecParent := oidtest.ID() + var ecParentAddr oid.Address + ecParentAddr.SetContainer(cnr) + ecParentAddr.SetObject(ecParent) + var ecParentAttr []objectSDK.Attribute + var attr objectSDK.Attribute + attr.SetKey(objectSDK.AttributeFilePath) + attr.SetValue("/1/2/3") + ecParentAttr = append(ecParentAttr, attr) + + chunkObj := testutil.GenerateObjectWithCID(cnr) + chunkObj.SetContainerID(cnr) + chunkObj.SetID(ecChunk1) + chunkObj.SetPayload([]byte{0, 1, 2, 3, 4}) + chunkObj.SetPayloadSize(uint64(5)) + chunkObj.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent, Attributes: ecParentAttr}, 0, 3, []byte{}, 0)) + + chunkObj2 := testutil.GenerateObjectWithCID(cnr) + chunkObj2.SetContainerID(cnr) + chunkObj2.SetID(ecChunk2) + chunkObj2.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) + chunkObj2.SetPayloadSize(uint64(10)) + chunkObj2.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent, Attributes: ecParentAttr}, 1, 3, []byte{}, 0)) + + // put object with EC + + var prm meta.PutPrm + prm.SetObject(chunkObj) + _, err := db.Put(context.Background(), prm) + require.NoError(t, err) + + prm.SetObject(chunkObj2) + _, err = db.Put(context.Background(), prm) + require.NoError(t, err) + + fs := objectSDK.SearchFilters{} + fs.AddRootFilter() + fs.AddFilter(objectSDK.AttributeFilePath, "/1/2/3", objectSDK.MatchCommonPrefix) + testSelect(t, db, cnr, fs, ecParentAddr) +} + type testTarget struct { objects []*objectSDK.Object } From 01e3944b31e7daed8ca855244b833302daabe9cc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Oct 2024 14:36:10 +0300 Subject: [PATCH 1410/1943] [#1408] metabase: Fix tests No need to specify container ID for objects created with `testutil.GenerateObjectWithCID`. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/delete_ec_test.go | 1 - pkg/local_object_storage/metabase/inhume_ec_test.go | 2 -- pkg/local_object_storage/metabase/select_test.go | 2 -- 3 files changed, 5 deletions(-) diff --git a/pkg/local_object_storage/metabase/delete_ec_test.go b/pkg/local_object_storage/metabase/delete_ec_test.go index 66c79ecd7..a25627990 100644 --- a/pkg/local_object_storage/metabase/delete_ec_test.go +++ b/pkg/local_object_storage/metabase/delete_ec_test.go @@ -39,7 +39,6 @@ func TestDeleteECObject_WithoutSplit(t *testing.T) { tombstoneID := oidtest.ID() chunkObj := testutil.GenerateObjectWithCID(cnr) - chunkObj.SetContainerID(cnr) chunkObj.SetID(ecChunk) chunkObj.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) chunkObj.SetPayloadSize(uint64(10)) diff --git a/pkg/local_object_storage/metabase/inhume_ec_test.go b/pkg/local_object_storage/metabase/inhume_ec_test.go index c3b1e72da..32e412c79 100644 --- a/pkg/local_object_storage/metabase/inhume_ec_test.go +++ b/pkg/local_object_storage/metabase/inhume_ec_test.go @@ -35,14 +35,12 @@ func TestInhumeECObject(t *testing.T) { tombstoneID := oidtest.ID() chunkObj := testutil.GenerateObjectWithCID(cnr) - chunkObj.SetContainerID(cnr) chunkObj.SetID(ecChunk) chunkObj.SetPayload([]byte{0, 1, 2, 3, 4}) chunkObj.SetPayloadSize(uint64(5)) chunkObj.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent}, 0, 3, []byte{}, 0)) chunkObj2 := testutil.GenerateObjectWithCID(cnr) - chunkObj2.SetContainerID(cnr) chunkObj2.SetID(ecChunk2) chunkObj2.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) chunkObj2.SetPayloadSize(uint64(10)) diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 0c6ebc863..bee778e2b 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -782,14 +782,12 @@ func TestDB_SelectECWithFastAndSlowFilters(t *testing.T) { ecParentAttr = append(ecParentAttr, attr) chunkObj := testutil.GenerateObjectWithCID(cnr) - chunkObj.SetContainerID(cnr) chunkObj.SetID(ecChunk1) chunkObj.SetPayload([]byte{0, 1, 2, 3, 4}) chunkObj.SetPayloadSize(uint64(5)) chunkObj.SetECHeader(objectSDK.NewECHeader(objectSDK.ECParentInfo{ID: ecParent, Attributes: ecParentAttr}, 0, 3, []byte{}, 0)) chunkObj2 := testutil.GenerateObjectWithCID(cnr) - chunkObj2.SetContainerID(cnr) chunkObj2.SetID(ecChunk2) chunkObj2.SetPayload([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) chunkObj2.SetPayloadSize(uint64(10)) From 6c46044c9cba5f2e20e105b3efa7abe166fbf577 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 3 Oct 2024 10:19:26 +0300 Subject: [PATCH 1411/1943] [#1410] shard: Move MetricsWriter interface to a separate file Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/metrics.go | 47 +++++++++++++++++++++++ pkg/local_object_storage/shard/shard.go | 44 --------------------- 2 files changed, 47 insertions(+), 44 deletions(-) create mode 100644 pkg/local_object_storage/shard/metrics.go diff --git a/pkg/local_object_storage/shard/metrics.go b/pkg/local_object_storage/shard/metrics.go new file mode 100644 index 000000000..568c0de5e --- /dev/null +++ b/pkg/local_object_storage/shard/metrics.go @@ -0,0 +1,47 @@ +package shard + +import "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + +// MetricsWriter is an interface that must store shard's metrics. +type MetricsWriter interface { + // SetObjectCounter must set object counter taking into account object type. + SetObjectCounter(objectType string, v uint64) + // AddToObjectCounter must update object counter taking into account object + // type. + // Negative parameter must decrease the counter. + AddToObjectCounter(objectType string, delta int) + // AddToContainerSize must add a value to the container size. + // Value can be negative. + AddToContainerSize(cnr string, value int64) + // AddToPayloadSize must add a value to the payload size. + // Value can be negative. + AddToPayloadSize(value int64) + // IncObjectCounter must increment shard's object counter taking into account + // object type. + IncObjectCounter(objectType string) + // SetShardID must set (update) the shard identifier that will be used in + // metrics. + SetShardID(id string) + // SetReadonly must set shard mode. + SetMode(mode mode.Mode) + // IncErrorCounter increment error counter. + IncErrorCounter() + // ClearErrorCounter clear error counter. + ClearErrorCounter() + // DeleteShardMetrics deletes shard metrics from registry. + DeleteShardMetrics() + // SetContainerObjectsCount sets container object count. + SetContainerObjectsCount(cnrID string, objectType string, value uint64) + // IncContainerObjectsCount increments container object count. + IncContainerObjectsCount(cnrID string, objectType string) + // SubContainerObjectsCount subtracts container object count. + SubContainerObjectsCount(cnrID string, objectType string, value uint64) + // IncRefillObjectsCount increments refill objects count. + IncRefillObjectsCount(path string, size int, success bool) + // SetRefillPercent sets refill percent. + SetRefillPercent(path string, percent uint32) + // SetRefillStatus sets refill status. + SetRefillStatus(path string, status string) + // SetEvacuationInProgress sets evacuation status + SetEvacuationInProgress(value bool) +} diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 7496fc352..f5317b16c 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -57,50 +57,6 @@ type DeletedLockCallback func(context.Context, []oid.Address) // EmptyContainersCallback is a callback hanfling list of zero-size and zero-count containers. type EmptyContainersCallback func(context.Context, []cid.ID) -// MetricsWriter is an interface that must store shard's metrics. -type MetricsWriter interface { - // SetObjectCounter must set object counter taking into account object type. - SetObjectCounter(objectType string, v uint64) - // AddToObjectCounter must update object counter taking into account object - // type. - // Negative parameter must decrease the counter. - AddToObjectCounter(objectType string, delta int) - // AddToContainerSize must add a value to the container size. - // Value can be negative. - AddToContainerSize(cnr string, value int64) - // AddToPayloadSize must add a value to the payload size. - // Value can be negative. - AddToPayloadSize(value int64) - // IncObjectCounter must increment shard's object counter taking into account - // object type. - IncObjectCounter(objectType string) - // SetShardID must set (update) the shard identifier that will be used in - // metrics. - SetShardID(id string) - // SetReadonly must set shard mode. - SetMode(mode mode.Mode) - // IncErrorCounter increment error counter. - IncErrorCounter() - // ClearErrorCounter clear error counter. - ClearErrorCounter() - // DeleteShardMetrics deletes shard metrics from registry. - DeleteShardMetrics() - // SetContainerObjectsCount sets container object count. - SetContainerObjectsCount(cnrID string, objectType string, value uint64) - // IncContainerObjectsCount increments container object count. - IncContainerObjectsCount(cnrID string, objectType string) - // SubContainerObjectsCount subtracts container object count. - SubContainerObjectsCount(cnrID string, objectType string, value uint64) - // IncRefillObjectsCount increments refill objects count. - IncRefillObjectsCount(path string, size int, success bool) - // SetRefillPercent sets refill percent. - SetRefillPercent(path string, percent uint32) - // SetRefillStatus sets refill status. - SetRefillStatus(path string, status string) - // SetEvacuationInProgress sets evacuation status - SetEvacuationInProgress(value bool) -} - type cfg struct { m sync.RWMutex From 9206ce5cd2ea973feef6a53ae0453736efacbe11 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 3 Oct 2024 10:23:59 +0300 Subject: [PATCH 1412/1943] [#1410] shard: Provide the default implementation for MetricsWriter Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/id.go | 4 +- pkg/local_object_storage/shard/metrics.go | 22 ++++++++++ pkg/local_object_storage/shard/mode.go | 4 +- pkg/local_object_storage/shard/shard.go | 49 +++++++++-------------- 4 files changed, 42 insertions(+), 37 deletions(-) diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index 2fe68d270..a72313498 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -45,9 +45,7 @@ func (s *Shard) UpdateID() (err error) { } shardID := s.info.ID.String() - if s.cfg.metricsWriter != nil { - s.cfg.metricsWriter.SetShardID(shardID) - } + s.cfg.metricsWriter.SetShardID(shardID) if s.writeCache != nil && s.writeCache.GetMetrics() != nil { s.writeCache.GetMetrics().SetShardID(shardID) } diff --git a/pkg/local_object_storage/shard/metrics.go b/pkg/local_object_storage/shard/metrics.go index 568c0de5e..6bf198048 100644 --- a/pkg/local_object_storage/shard/metrics.go +++ b/pkg/local_object_storage/shard/metrics.go @@ -45,3 +45,25 @@ type MetricsWriter interface { // SetEvacuationInProgress sets evacuation status SetEvacuationInProgress(value bool) } + +type noopMetrics struct{} + +var _ MetricsWriter = noopMetrics{} + +func (noopMetrics) SetObjectCounter(string, uint64) {} +func (noopMetrics) AddToObjectCounter(string, int) {} +func (noopMetrics) AddToContainerSize(string, int64) {} +func (noopMetrics) AddToPayloadSize(int64) {} +func (noopMetrics) IncObjectCounter(string) {} +func (noopMetrics) SetShardID(string) {} +func (noopMetrics) SetMode(mode.Mode) {} +func (noopMetrics) IncErrorCounter() {} +func (noopMetrics) ClearErrorCounter() {} +func (noopMetrics) DeleteShardMetrics() {} +func (noopMetrics) SetContainerObjectsCount(string, string, uint64) {} +func (noopMetrics) IncContainerObjectsCount(string, string) {} +func (noopMetrics) SubContainerObjectsCount(string, string, uint64) {} +func (noopMetrics) IncRefillObjectsCount(string, int, bool) {} +func (noopMetrics) SetRefillPercent(string, uint32) {} +func (noopMetrics) SetRefillStatus(string, string) {} +func (noopMetrics) SetEvacuationInProgress(bool) {} diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index 1bab57448..d90a5f4b6 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -65,9 +65,7 @@ func (s *Shard) setMode(m mode.Mode) error { } s.info.Mode = m - if s.metricsWriter != nil { - s.metricsWriter.SetMode(s.info.Mode) - } + s.metricsWriter.SetMode(s.info.Mode) s.log.Info(logs.ShardShardModeSetSuccessfully, zap.Stringer("mode", s.info.Mode)) diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index f5317b16c..a57b548be 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -105,6 +105,7 @@ func defaultCfg() *cfg { reportErrorFunc: func(string, string, error) {}, zeroSizeContainersCallback: func(context.Context, []cid.ID) {}, zeroCountContainersCallback: func(context.Context, []cid.ID) {}, + metricsWriter: noopMetrics{}, } } @@ -384,7 +385,7 @@ const ( ) func (s *Shard) updateMetrics(ctx context.Context) { - if s.cfg.metricsWriter == nil || s.GetMode().NoMetabase() { + if s.GetMode().NoMetabase() { return } @@ -439,35 +440,29 @@ func (s *Shard) updateMetrics(ctx context.Context) { // incObjectCounter increment both physical and logical object // counters. func (s *Shard) incObjectCounter(cnrID cid.ID, isUser bool) { - if s.cfg.metricsWriter != nil { - s.cfg.metricsWriter.IncObjectCounter(physical) - s.cfg.metricsWriter.IncObjectCounter(logical) - s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), physical) - s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), logical) - if isUser { - s.cfg.metricsWriter.IncObjectCounter(user) - s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), user) - } + s.cfg.metricsWriter.IncObjectCounter(physical) + s.cfg.metricsWriter.IncObjectCounter(logical) + s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), physical) + s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), logical) + if isUser { + s.cfg.metricsWriter.IncObjectCounter(user) + s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), user) } } func (s *Shard) decObjectCounterBy(typ string, v uint64) { - if s.cfg.metricsWriter != nil && v > 0 { + if v > 0 { s.cfg.metricsWriter.AddToObjectCounter(typ, -int(v)) } } func (s *Shard) setObjectCounterBy(typ string, v uint64) { - if s.cfg.metricsWriter != nil && v > 0 { + if v > 0 { s.cfg.metricsWriter.SetObjectCounter(typ, v) } } func (s *Shard) decContainerObjectCounter(byCnr map[cid.ID]meta.ObjectCounters) { - if s.cfg.metricsWriter == nil { - return - } - for cnrID, count := range byCnr { if count.Phy > 0 { s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), physical, count.Phy) @@ -482,46 +477,38 @@ func (s *Shard) decContainerObjectCounter(byCnr map[cid.ID]meta.ObjectCounters) } func (s *Shard) addToContainerSize(cnr string, size int64) { - if s.cfg.metricsWriter != nil && size != 0 { + if size != 0 { s.cfg.metricsWriter.AddToContainerSize(cnr, size) } } func (s *Shard) addToPayloadSize(size int64) { - if s.cfg.metricsWriter != nil && size != 0 { + if size != 0 { s.cfg.metricsWriter.AddToPayloadSize(size) } } func (s *Shard) setContainerObjectsCount(cnr string, typ string, v uint64) { - if s.cfg.metricsWriter != nil && v > 0 { + if v > 0 { s.metricsWriter.SetContainerObjectsCount(cnr, typ, v) } } func (s *Shard) IncErrorCounter() { - if s.cfg.metricsWriter != nil { - s.cfg.metricsWriter.IncErrorCounter() - } + s.cfg.metricsWriter.IncErrorCounter() } func (s *Shard) ClearErrorCounter() { - if s.cfg.metricsWriter != nil { - s.cfg.metricsWriter.ClearErrorCounter() - } + s.cfg.metricsWriter.ClearErrorCounter() } func (s *Shard) DeleteShardMetrics() { - if s.cfg.metricsWriter != nil { - s.cfg.metricsWriter.DeleteShardMetrics() - } + s.cfg.metricsWriter.DeleteShardMetrics() } func (s *Shard) SetEvacuationInProgress(val bool) { s.m.Lock() defer s.m.Unlock() s.info.EvacuationInProgress = val - if s.metricsWriter != nil { - s.metricsWriter.SetEvacuationInProgress(val) - } + s.metricsWriter.SetEvacuationInProgress(val) } From 9a87acb87ad243fcdd932e764a3f5f8d9c5c6657 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 3 Oct 2024 10:40:56 +0300 Subject: [PATCH 1413/1943] [#1410] engine: Provide the default implementation to MetricsRegister Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/container.go | 8 +--- pkg/local_object_storage/engine/delete.go | 4 +- pkg/local_object_storage/engine/engine.go | 1 + pkg/local_object_storage/engine/get.go | 4 +- pkg/local_object_storage/engine/head.go | 4 +- pkg/local_object_storage/engine/inhume.go | 4 +- pkg/local_object_storage/engine/metrics.go | 45 +++++++++++++++++++ pkg/local_object_storage/engine/put.go | 4 +- pkg/local_object_storage/engine/range.go | 4 +- pkg/local_object_storage/engine/select.go | 8 +--- pkg/local_object_storage/engine/shards.go | 46 +++++++++----------- 11 files changed, 77 insertions(+), 55 deletions(-) diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index e45f502ac..6def02f12 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -68,9 +68,7 @@ func ContainerSize(e *StorageEngine, id cid.ID) (uint64, error) { } func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRes, err error) { - if e.metrics != nil { - defer elapsed("EstimateContainerSize", e.metrics.AddMethodDuration)() - } + defer elapsed("EstimateContainerSize", e.metrics.AddMethodDuration)() e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { var csPrm shard.ContainerSizePrm @@ -116,9 +114,7 @@ func ListContainers(ctx context.Context, e *StorageEngine) ([]cid.ID, error) { } func (e *StorageEngine) listContainers(ctx context.Context) (ListContainersRes, error) { - if e.metrics != nil { - defer elapsed("ListContainers", e.metrics.AddMethodDuration)() - } + defer elapsed("ListContainers", e.metrics.AddMethodDuration)() uniqueIDs := make(map[string]cid.ID) diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 318f938fb..61cb6832d 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -68,9 +68,7 @@ func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRe } func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { - if e.metrics != nil { - defer elapsed("Delete", e.metrics.AddMethodDuration)() - } + defer elapsed("Delete", e.metrics.AddMethodDuration)() var locked struct { is bool diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 5e883a641..13efdcb84 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -219,6 +219,7 @@ func defaultCfg() *cfg { res := &cfg{ log: &logger.Logger{Logger: zap.L()}, shardPoolSize: 20, + metrics: noopMetrics{}, } res.containerSource.Store(&containerSource{}) return res diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 253256c34..4a9199be7 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -66,9 +66,7 @@ func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err er } func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { - if e.metrics != nil { - defer elapsed("Get", e.metrics.AddMethodDuration)() - } + defer elapsed("Get", e.metrics.AddMethodDuration)() errNotFound := new(apistatus.ObjectNotFound) diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index 6857a3631..d2e3cfd99 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -68,9 +68,7 @@ func (e *StorageEngine) Head(ctx context.Context, prm HeadPrm) (res HeadRes, err func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.head") defer span.End() - if e.metrics != nil { - defer elapsed("Head", e.metrics.AddMethodDuration)() - } + defer elapsed("Head", e.metrics.AddMethodDuration)() var ( head *objectSDK.Object diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 683713f94..35ce50f65 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -80,9 +80,7 @@ func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRe } func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { - if e.metrics != nil { - defer elapsed("Inhume", e.metrics.AddMethodDuration)() - } + defer elapsed("Inhume", e.metrics.AddMethodDuration)() var shPrm shard.InhumePrm if prm.forceRemoval { diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 1c088c754..75936206d 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -68,3 +68,48 @@ func (m *gcMetrics) AddExpiredObjectCollectionDuration(d time.Duration, success func (m *gcMetrics) AddInhumedObjectCount(count uint64, objectType string) { m.storage.AddInhumedObjectCount(m.shardID, count, objectType) } + +type ( + noopMetrics struct{} + noopWriteCacheMetrics struct{} + noopGCMetrics struct{} +) + +var ( + _ MetricRegister = noopMetrics{} + _ metrics.WriteCacheMetrics = noopWriteCacheMetrics{} + _ metrics.GCMetrics = noopGCMetrics{} +) + +func (noopMetrics) AddMethodDuration(string, time.Duration) {} +func (noopMetrics) SetObjectCounter(string, string, uint64) {} +func (noopMetrics) AddToObjectCounter(string, string, int) {} +func (noopMetrics) SetMode(string, mode.Mode) {} +func (noopMetrics) AddToContainerSize(string, int64) {} +func (noopMetrics) DeleteContainerSize(string) {} +func (noopMetrics) DeleteContainerCount(string) {} +func (noopMetrics) AddToPayloadCounter(string, int64) {} +func (noopMetrics) IncErrorCounter(string) {} +func (noopMetrics) ClearErrorCounter(string) {} +func (noopMetrics) DeleteShardMetrics(string) {} +func (noopMetrics) SetContainerObjectCounter(string, string, string, uint64) {} +func (noopMetrics) IncContainerObjectCounter(string, string, string) {} +func (noopMetrics) SubContainerObjectCounter(string, string, string, uint64) {} +func (noopMetrics) IncRefillObjectsCount(string, string, int, bool) {} +func (noopMetrics) SetRefillPercent(string, string, uint32) {} +func (noopMetrics) SetRefillStatus(string, string, string) {} +func (noopMetrics) SetEvacuationInProgress(string, bool) {} +func (noopMetrics) WriteCache() metrics.WriteCacheMetrics { return noopWriteCacheMetrics{} } +func (noopMetrics) GC() metrics.GCMetrics { return noopGCMetrics{} } + +func (noopWriteCacheMetrics) AddMethodDuration(string, string, string, string, bool, time.Duration) {} +func (noopWriteCacheMetrics) SetActualCount(string, string, string, uint64) {} +func (noopWriteCacheMetrics) SetEstimateSize(string, string, string, uint64) {} +func (noopWriteCacheMetrics) SetMode(string, string) {} +func (noopWriteCacheMetrics) IncOperationCounter(string, string, string, string, metrics.NullBool) {} +func (noopWriteCacheMetrics) Close(string, string) {} + +func (noopGCMetrics) AddRunDuration(string, time.Duration, bool) {} +func (noopGCMetrics) AddDeletedCount(string, uint64, uint64) {} +func (noopGCMetrics) AddExpiredObjectCollectionDuration(string, time.Duration, bool, string) {} +func (noopGCMetrics) AddInhumedObjectCount(string, uint64, string) {} diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index f92d83745..bf86402a7 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -72,9 +72,7 @@ func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) { } func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { - if e.metrics != nil { - defer elapsed("Put", e.metrics.AddMethodDuration)() - } + defer elapsed("Put", e.metrics.AddMethodDuration)() addr := object.AddressOf(prm.obj) diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index cbf26ff4e..498674fd2 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -82,9 +82,7 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error )) defer span.End() - if e.metrics != nil { - defer elapsed("GetRange", e.metrics.AddMethodDuration)() - } + defer elapsed("GetRange", e.metrics.AddMethodDuration)() var shPrm shard.RngPrm shPrm.SetAddress(prm.addr) diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 6a8c9fab9..972a4f52a 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -59,9 +59,7 @@ func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRe } func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) { - if e.metrics != nil { - defer elapsed("Search", e.metrics.AddMethodDuration)() - } + defer elapsed("Search", e.metrics.AddMethodDuration)() addrList := make([]oid.Address, 0) uniqueMap := make(map[string]struct{}) @@ -108,9 +106,7 @@ func (e *StorageEngine) List(ctx context.Context, limit uint64) (res SelectRes, } func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, error) { - if e.metrics != nil { - defer elapsed("ListObjects", e.metrics.AddMethodDuration)() - } + defer elapsed("ListObjects", e.metrics.AddMethodDuration)() addrList := make([]oid.Address, 0, limit) uniqueMap := make(map[string]struct{}) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 2ad6859e4..96f54369b 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -116,9 +116,7 @@ func (e *StorageEngine) AddShard(ctx context.Context, opts ...shard.Option) (*sh return nil, fmt.Errorf("could not add %s shard: %w", sh.ID().String(), err) } - if e.cfg.metrics != nil { - e.cfg.metrics.SetMode(sh.ID().String(), sh.GetMode()) - } + e.cfg.metrics.SetMode(sh.ID().String(), sh.GetMode()) return sh.ID(), nil } @@ -152,28 +150,26 @@ func (e *StorageEngine) appendMetrics(id *shard.ID, opts []shard.Option) []shard e.mtx.RLock() defer e.mtx.RUnlock() - if e.metrics != nil { - opts = append(opts, - shard.WithMetricsWriter( - &metricsWithID{ - id: id.String(), - mw: e.metrics, - }, - ), - shard.WithWriteCacheMetrics( - &writeCacheMetrics{ - shardID: id.String(), - metrics: e.metrics.WriteCache(), - }, - ), - shard.WithGCMetrics( - &gcMetrics{ - storage: e.metrics.GC(), - shardID: id.String(), - }, - ), - ) - } + opts = append(opts, + shard.WithMetricsWriter( + &metricsWithID{ + id: id.String(), + mw: e.metrics, + }, + ), + shard.WithWriteCacheMetrics( + &writeCacheMetrics{ + shardID: id.String(), + metrics: e.metrics.WriteCache(), + }, + ), + shard.WithGCMetrics( + &gcMetrics{ + storage: e.metrics.GC(), + shardID: id.String(), + }, + ), + ) return opts } From 963faa615ab0a70964821b3a3725c27ed5d7f60e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Oct 2024 14:58:45 +0300 Subject: [PATCH 1414/1943] [#1413] engine: Cleanup shard error reporting - `reportShardErrorBackground()` no longer differs from `reportShardError()`, reflect this in its name; - reuse common pieces of code to make it simpler. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/engine.go | 28 ++++------------------- pkg/local_object_storage/engine/shards.go | 2 +- 2 files changed, 5 insertions(+), 25 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 13efdcb84..f40c9cc04 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -115,10 +115,8 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } -// reportShardErrorBackground increases shard error counter and logs an error. -// It is intended to be used from background workers and -// doesn't change shard mode because of possible deadlocks. -func (e *StorageEngine) reportShardErrorBackground(id string, msg string, err error) { +// reportShardErrorByID increases shard error counter and logs an error. +func (e *StorageEngine) reportShardErrorByID(id string, msg string, err error) { e.mtx.RLock() sh, ok := e.shards[id] e.mtx.RUnlock() @@ -127,16 +125,7 @@ func (e *StorageEngine) reportShardErrorBackground(id string, msg string, err er return } - if isLogical(err) { - e.log.Warn(msg, - zap.Stringer("shard_id", sh.ID()), - zap.String("error", err.Error())) - return - } - - errCount := sh.errorCount.Add(1) - sh.Shard.IncErrorCounter() - e.reportShardErrorWithFlags(sh.Shard, errCount, msg, err) + e.reportShardError(sh, msg, err) } // reportShardError checks that the amount of errors doesn't exceed the configured threshold. @@ -156,16 +145,7 @@ func (e *StorageEngine) reportShardError( errCount := sh.errorCount.Add(1) sh.Shard.IncErrorCounter() - e.reportShardErrorWithFlags(sh.Shard, errCount, msg, err, fields...) -} -func (e *StorageEngine) reportShardErrorWithFlags( - sh *shard.Shard, - errCount uint32, - msg string, - err error, - fields ...zap.Field, -) { sid := sh.ID() e.log.Warn(msg, append([]zap.Field{ zap.Stringer("shard_id", sid), @@ -179,7 +159,7 @@ func (e *StorageEngine) reportShardErrorWithFlags( req := setModeRequest{ errorCount: errCount, - sh: sh, + sh: sh.Shard, isMeta: errors.As(err, new(metaerr.Error)), } diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 96f54369b..c3ccb5276 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -134,7 +134,7 @@ func (e *StorageEngine) createShard(_ context.Context, opts []shard.Option) (*sh shard.WithExpiredTombstonesCallback(e.processExpiredTombstones), shard.WithExpiredLocksCallback(e.processExpiredLocks), shard.WithDeletedLockCallback(e.processDeletedLocks), - shard.WithReportErrorFunc(e.reportShardErrorBackground), + shard.WithReportErrorFunc(e.reportShardErrorByID), shard.WithZeroSizeCallback(e.processZeroSizeContainers), shard.WithZeroCountCallback(e.processZeroCountContainers), )...) From 4dc9a1b300b2f22dbd9628713917347bc0d3a6ee Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Oct 2024 15:07:20 +0300 Subject: [PATCH 1415/1943] [#1413] engine: Remove error counting methods from Shard All error counting and hangling logic is present on the engine level. Currently, we pass engine metrics with shard ID metric to shard, then export 3 methods to manipulate these metrics. In this commits all methods are removed and error counter is tracked on the engine level exlusively. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/engine.go | 2 +- pkg/local_object_storage/engine/shards.go | 6 +++--- pkg/local_object_storage/shard/metrics.go | 9 --------- pkg/local_object_storage/shard/shard.go | 12 ------------ 4 files changed, 4 insertions(+), 25 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index f40c9cc04..3183d6137 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -144,7 +144,7 @@ func (e *StorageEngine) reportShardError( } errCount := sh.errorCount.Add(1) - sh.Shard.IncErrorCounter() + e.metrics.IncErrorCounter(sh.ID().String()) sid := sh.ID() e.log.Warn(msg, append([]zap.Field{ diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index c3ccb5276..aab2c423c 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -217,7 +217,7 @@ func (e *StorageEngine) removeShards(ids ...string) { continue } - sh.DeleteShardMetrics() + e.metrics.DeleteShardMetrics(id) ss = append(ss, sh) delete(e.shards, id) @@ -318,7 +318,7 @@ func (e *StorageEngine) SetShardMode(id *shard.ID, m mode.Mode, resetErrorCounte if id.String() == shID { if resetErrorCounter { sh.errorCount.Store(0) - sh.Shard.ClearErrorCounter() + e.metrics.ClearErrorCounter(shID) } return sh.SetMode(m) } @@ -422,7 +422,7 @@ func (e *StorageEngine) deleteShards(ids []*shard.ID) ([]hashedShard, error) { for _, sh := range ss { idStr := sh.ID().String() - sh.DeleteShardMetrics() + e.metrics.DeleteShardMetrics(idStr) delete(e.shards, idStr) diff --git a/pkg/local_object_storage/shard/metrics.go b/pkg/local_object_storage/shard/metrics.go index 6bf198048..91bf8d0ae 100644 --- a/pkg/local_object_storage/shard/metrics.go +++ b/pkg/local_object_storage/shard/metrics.go @@ -24,12 +24,6 @@ type MetricsWriter interface { SetShardID(id string) // SetReadonly must set shard mode. SetMode(mode mode.Mode) - // IncErrorCounter increment error counter. - IncErrorCounter() - // ClearErrorCounter clear error counter. - ClearErrorCounter() - // DeleteShardMetrics deletes shard metrics from registry. - DeleteShardMetrics() // SetContainerObjectsCount sets container object count. SetContainerObjectsCount(cnrID string, objectType string, value uint64) // IncContainerObjectsCount increments container object count. @@ -57,9 +51,6 @@ func (noopMetrics) AddToPayloadSize(int64) {} func (noopMetrics) IncObjectCounter(string) {} func (noopMetrics) SetShardID(string) {} func (noopMetrics) SetMode(mode.Mode) {} -func (noopMetrics) IncErrorCounter() {} -func (noopMetrics) ClearErrorCounter() {} -func (noopMetrics) DeleteShardMetrics() {} func (noopMetrics) SetContainerObjectsCount(string, string, uint64) {} func (noopMetrics) IncContainerObjectsCount(string, string) {} func (noopMetrics) SubContainerObjectsCount(string, string, uint64) {} diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index a57b548be..d7e723733 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -494,18 +494,6 @@ func (s *Shard) setContainerObjectsCount(cnr string, typ string, v uint64) { } } -func (s *Shard) IncErrorCounter() { - s.cfg.metricsWriter.IncErrorCounter() -} - -func (s *Shard) ClearErrorCounter() { - s.cfg.metricsWriter.ClearErrorCounter() -} - -func (s *Shard) DeleteShardMetrics() { - s.cfg.metricsWriter.DeleteShardMetrics() -} - func (s *Shard) SetEvacuationInProgress(val bool) { s.m.Lock() defer s.m.Unlock() From 2f710d8f945f90c5d65e4c9a0c53f0dfdcc4f291 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 4 Oct 2024 15:23:22 +0300 Subject: [PATCH 1416/1943] [#1414] metabase: Check parameter for `CountAliveObjectsInBucket` Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/metabase/list.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 44f25246e..74a529809 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -452,10 +452,11 @@ func (db *DB) CountAliveObjectsInBucket(ctx context.Context, prm CountAliveObjec return 0, ErrDegradedMode } - cidRaw := prm.BucketName[1:bucketKeySize] - if cidRaw == nil { + if len(prm.BucketName) != bucketKeySize { return 0, nil } + + cidRaw := prm.BucketName[1:bucketKeySize] var count uint64 err := db.boltDB.View(func(tx *bbolt.Tx) error { bkt := tx.Bucket(prm.BucketName) From fc032838c037c7c649f80181ca71d8c9f6847e7d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 7 Oct 2024 11:50:47 +0300 Subject: [PATCH 1417/1943] [#1215] blobstor/test: Cover iteration behaviour Signed-off-by: Evgenii Stratonikov --- .../blobstor/iterate_test.go | 172 ++++++------------ 1 file changed, 59 insertions(+), 113 deletions(-) diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index 079728380..195d0bd31 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -3,10 +3,13 @@ package blobstor import ( "context" "encoding/binary" + "errors" "os" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/memstore" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -90,117 +93,60 @@ func TestIterateObjects(t *testing.T) { } func TestIterate_IgnoreErrors(t *testing.T) { - t.Skip() - // dir := t.TempDir() - // - // const ( - // smallSize = 512 - // objCount = 5 - // ) - // bsOpts := []Option{ - // WithCompressObjects(true), - // WithRootPath(dir), - // WithSmallSizeLimit(smallSize * 2), // + header - // WithBlobovniczaOpenedCacheSize(1), - // WithBlobovniczaShallowWidth(1), - // WithBlobovniczaShallowDepth(1)} - // bs := New(bsOpts...) - // require.NoError(t, bs.Open(false)) - // require.NoError(t, bs.Init()) - // - // addrs := make([]oid.Address, objCount) - // for i := range addrs { - // addrs[i] = oidtest.Address() - // - // obj := object.New() - // obj.SetContainerID(addrs[i].Container()) - // obj.SetID(addrs[i].Object()) - // obj.SetPayload(make([]byte, smallSize<<(i%2))) - // - // objData, err := obj.Marshal() - // require.NoError(t, err) - // - // _, err = bs.PutRaw(addrs[i], objData, true) - // require.NoError(t, err) - // } - // - // // Construct corrupted compressed object. - // buf := bytes.NewBuffer(nil) - // badObject := make([]byte, smallSize/2+1) - // enc, err := zstd.NewWriter(buf) - // require.NoError(t, err) - // rawData := enc.EncodeAll(badObject, nil) - // for i := 4; /* magic size */ i < len(rawData); i += 2 { - // rawData[i] ^= 0xFF - // } - // // Will be put uncompressed but fetched as compressed because of magic. - // _, err = bs.PutRaw(oidtest.Address(), rawData, false) - // require.NoError(t, err) - // require.NoError(t, bs.fsTree.Put(oidtest.Address(), rawData)) - // - // require.NoError(t, bs.Close()) - // - // // Increase width to have blobovnicza which is definitely empty. - // b := New(append(bsOpts, WithBlobovniczaShallowWidth(2))...) - // require.NoError(t, b.Open(false)) - // require.NoError(t, b.Init()) - // - // var p string - // for i := 0; i < 2; i++ { - // bp := filepath.Join(bs.rootPath, "1", strconv.FormatUint(uint64(i), 10)) - // if _, ok := bs.blobovniczas.opened.Get(bp); !ok { - // p = bp - // break - // } - // } - // require.NotEqual(t, "", p, "expected to not have at least 1 blobovnicza in cache") - // require.NoError(t, os.Chmod(p, 0)) - // - // require.NoError(t, b.Close()) - // require.NoError(t, bs.Open(false)) - // require.NoError(t, bs.Init()) - // - // var prm IteratePrm - // prm.SetIterationHandler(func(e IterationElement) error { - // return nil - // }) - // _, err = bs.Iterate(prm) - // require.Error(t, err) - // - // prm.IgnoreErrors() - // - // t.Run("skip invalid objects", func(t *testing.T) { - // actual := make([]oid.Address, 0, len(addrs)) - // prm.SetIterationHandler(func(e IterationElement) error { - // obj := object.New() - // err := obj.Unmarshal(e.data) - // if err != nil { - // return err - // } - // - // var addr oid.Address - // cnr, _ := obj.ContainerID() - // addr.SetContainer(cnr) - // id, _ := obj.ID() - // addr.SetObject(id) - // actual = append(actual, addr) - // return nil - // }) - // - // _, err := bs.Iterate(prm) - // require.NoError(t, err) - // require.ElementsMatch(t, addrs, actual) - // }) - // t.Run("return errors from handler", func(t *testing.T) { - // n := 0 - // expectedErr := errors.New("expected error") - // prm.SetIterationHandler(func(e IterationElement) error { - // if n++; n == objCount/2 { - // return expectedErr - // } - // return nil - // }) - // _, err := bs.Iterate(prm) - // require.ErrorIs(t, err, expectedErr) - // }) + ctx := context.Background() + + myErr := errors.New("unique error") + nopIter := func(common.IteratePrm) (common.IterateRes, error) { return common.IterateRes{}, nil } + panicIter := func(common.IteratePrm) (common.IterateRes, error) { panic("unreachable") } + errIter := func(common.IteratePrm) (common.IterateRes, error) { return common.IterateRes{}, myErr } + + var s1iter, s2iter func(common.IteratePrm) (common.IterateRes, error) + st1 := teststore.New( + teststore.WithSubstorage(memstore.New()), + teststore.WithIterate(func(prm common.IteratePrm) (common.IterateRes, error) { + return s1iter(prm) + })) + st2 := teststore.New( + teststore.WithSubstorage(memstore.New()), + teststore.WithIterate(func(prm common.IteratePrm) (common.IterateRes, error) { + return s2iter(prm) + })) + + bsOpts := []Option{WithStorages([]SubStorage{ + {Storage: st1}, + {Storage: st2}, + })} + bs := New(bsOpts...) + require.NoError(t, bs.Open(ctx, mode.ReadWrite)) + require.NoError(t, bs.Init()) + + nopHandler := func(e common.IterationElement) error { + return nil + } + + t.Run("no errors", func(t *testing.T) { + s1iter = nopIter + s2iter = nopIter + _, err := bs.Iterate(ctx, common.IteratePrm{Handler: nopHandler}) + require.NoError(t, err) + }) + t.Run("error in the first sub storage, the second one is not iterated over", func(t *testing.T) { + s1iter = errIter + s2iter = panicIter + _, err := bs.Iterate(ctx, common.IteratePrm{Handler: nopHandler}) + require.ErrorIs(t, err, myErr) + }) + + t.Run("ignore errors, storage 1", func(t *testing.T) { + s1iter = errIter + s2iter = nopIter + _, err := bs.Iterate(ctx, common.IteratePrm{IgnoreErrors: true, Handler: nopHandler}) + require.NoError(t, err) + }) + t.Run("ignore errors, storage 2", func(t *testing.T) { + s1iter = nopIter + s2iter = errIter + _, err := bs.Iterate(ctx, common.IteratePrm{IgnoreErrors: true, Handler: nopHandler}) + require.NoError(t, err) + }) } From a5de74a2492deb9e969c1d722c82bb8b050130df Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 7 Oct 2024 15:13:44 +0300 Subject: [PATCH 1418/1943] [#1418] go.mod: Update api-go Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1023948bc..f81ba9cf7 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241001111852-d9a604fbc16f + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241007120543-29c522d5d8a3 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d diff --git a/go.sum b/go.sum index 5d719a027..8aa087de4 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241001111852-d9a604fbc16f h1:FZvX6CLzTQqMyMvOerIKMvIEJQbOImDjSooZx3AVRyE= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241001111852-d9a604fbc16f/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241007120543-29c522d5d8a3 h1:6QXNnfBgYx81UZsBdpPnQY+ZMSKGFbFc29wV7DJ/UG4= +git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241007120543-29c522d5d8a3/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c h1:9JmXxWcqWaOXModBwtj/WlPYOfSk2jRSDoRsDRYvgnA= git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c/go.mod h1:DVb1AqP323wiEwCtlfoJJTERF4Xji2L39X6T8kv26RE= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= From 5fbb2657ca9ca9bbc3aa2ca9239fbb55ea47cdc3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 8 Oct 2024 10:02:14 +0300 Subject: [PATCH 1419/1943] [#1419] mod: Bump sdk-go version Signed-off-by: Dmitrii Stepanov --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f81ba9cf7..91cc55a36 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241007135805-4c310ae1c7fa git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 @@ -60,7 +60,7 @@ require ( require ( git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect - github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect diff --git a/go.sum b/go.sum index 8aa087de4..728592ea5 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98 h1:ijUci3thz0EwWkuRJDocW5D1RkVAJlt9xNG4CYepC90= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240918095938-e580ee991d98/go.mod h1:GeNpo12HcEW4J412sH5yf8xFYapxlrt5fcYzRwg0Ino= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241007135805-4c310ae1c7fa h1:Jr8hXNNFECLhC7S45HuyQms4U/gim1xILoU3g4ZZnHg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241007135805-4c310ae1c7fa/go.mod h1:jmb7yxzZota9jWbC10p+7YR+6wwJPBj7J/Fl5VDkXys= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= @@ -27,8 +27,8 @@ git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02/go. github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/VictoriaMetrics/easyproto v0.1.4 h1:r8cNvo8o6sR4QShBXQd1bKw/VVLSQma/V2KhTBPf+Sc= github.com/VictoriaMetrics/easyproto v0.1.4/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710= -github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= From 0c49bca19c82d574c9a93681bda77362edd5b88c Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 7 Oct 2024 18:32:26 +0300 Subject: [PATCH 1420/1943] [#1415] lens/explorer: Add timeout for opening database Signed-off-by: Aleksey Savchuk --- cmd/frostfs-lens/internal/blobovnicza/tui.go | 13 +------------ cmd/frostfs-lens/internal/meta/tui.go | 13 +------------ cmd/frostfs-lens/internal/tui/util.go | 13 +++++++++++++ cmd/frostfs-lens/internal/writecache/tui.go | 13 +------------ 4 files changed, 16 insertions(+), 36 deletions(-) diff --git a/cmd/frostfs-lens/internal/blobovnicza/tui.go b/cmd/frostfs-lens/internal/blobovnicza/tui.go index eb4a5ff59..4aa281616 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/tui.go +++ b/cmd/frostfs-lens/internal/blobovnicza/tui.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui" "github.com/rivo/tview" "github.com/spf13/cobra" - "go.etcd.io/bbolt" ) var tuiCMD = &cobra.Command{ @@ -43,7 +42,7 @@ func tuiFunc(cmd *cobra.Command, _ []string) { } func runTUI(cmd *cobra.Command) error { - db, err := openDB(false) + db, err := tui.OpenDB(vPath, false) if err != nil { return fmt.Errorf("couldn't open database: %w", err) } @@ -67,13 +66,3 @@ func runTUI(cmd *cobra.Command) error { app.SetRoot(ui, true).SetFocus(ui) return app.Run() } - -func openDB(writable bool) (*bbolt.DB, error) { - db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{ - ReadOnly: !writable, - }) - if err != nil { - return nil, err - } - return db, nil -} diff --git a/cmd/frostfs-lens/internal/meta/tui.go b/cmd/frostfs-lens/internal/meta/tui.go index 00e8bf117..5a41f945c 100644 --- a/cmd/frostfs-lens/internal/meta/tui.go +++ b/cmd/frostfs-lens/internal/meta/tui.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui" "github.com/rivo/tview" "github.com/spf13/cobra" - "go.etcd.io/bbolt" ) var tuiCMD = &cobra.Command{ @@ -44,7 +43,7 @@ func tuiFunc(cmd *cobra.Command, _ []string) { } func runTUI(cmd *cobra.Command) error { - db, err := openDB(false) + db, err := tui.OpenDB(vPath, false) if err != nil { return fmt.Errorf("couldn't open database: %w", err) } @@ -70,13 +69,3 @@ func runTUI(cmd *cobra.Command) error { app.SetRoot(ui, true).SetFocus(ui) return app.Run() } - -func openDB(writable bool) (*bbolt.DB, error) { - db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{ - ReadOnly: !writable, - }) - if err != nil { - return nil, err - } - return db, nil -} diff --git a/cmd/frostfs-lens/internal/tui/util.go b/cmd/frostfs-lens/internal/tui/util.go index d4e13b2a9..2d1ab3e33 100644 --- a/cmd/frostfs-lens/internal/tui/util.go +++ b/cmd/frostfs-lens/internal/tui/util.go @@ -3,12 +3,25 @@ package tui import ( "errors" "strings" + "time" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/mr-tron/base58" + "go.etcd.io/bbolt" ) +func OpenDB(path string, writable bool) (*bbolt.DB, error) { + db, err := bbolt.Open(path, 0o600, &bbolt.Options{ + ReadOnly: !writable, + Timeout: 100 * time.Millisecond, + }) + if err != nil { + return nil, err + } + return db, nil +} + func CIDParser(s string) (any, error) { data, err := base58.Decode(s) if err != nil { diff --git a/cmd/frostfs-lens/internal/writecache/tui.go b/cmd/frostfs-lens/internal/writecache/tui.go index 6b7532b08..b7e4d7c96 100644 --- a/cmd/frostfs-lens/internal/writecache/tui.go +++ b/cmd/frostfs-lens/internal/writecache/tui.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui" "github.com/rivo/tview" "github.com/spf13/cobra" - "go.etcd.io/bbolt" ) var tuiCMD = &cobra.Command{ @@ -43,7 +42,7 @@ func tuiFunc(cmd *cobra.Command, _ []string) { } func runTUI(cmd *cobra.Command) error { - db, err := openDB(false) + db, err := tui.OpenDB(vPath, false) if err != nil { return fmt.Errorf("couldn't open database: %w", err) } @@ -67,13 +66,3 @@ func runTUI(cmd *cobra.Command) error { app.SetRoot(ui, true).SetFocus(ui) return app.Run() } - -func openDB(writable bool) (*bbolt.DB, error) { - db, err := bbolt.Open(vPath, 0o600, &bbolt.Options{ - ReadOnly: !writable, - }) - if err != nil { - return nil, err - } - return db, nil -} From 899cd55c277b04b974e67df29b81146528d5c293 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Oct 2024 13:28:46 +0300 Subject: [PATCH 1421/1943] [#1412] engine: PutPrm refactoring Use fields instead of methods. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/list_test.go | 6 +---- pkg/local_object_storage/engine/put.go | 24 ++++++-------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index 11a6c7841..d683b5475 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -82,11 +82,7 @@ func TestListWithCursor(t *testing.T) { for range tt.objectNum { containerID := cidtest.ID() obj := testutil.GenerateObjectWithCIDWithPayload(containerID, []byte{'a'}) - - var prm PutPrm - prm.WithObject(obj) - - err := e.Put(context.Background(), prm) + err := e.Put(context.Background(), PutPrm{Object: obj}) require.NoError(t, err) expected = append(expected, object.Info{Type: objectSDK.TypeRegular, Address: object.AddressOf(obj)}) } diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index bf86402a7..9ce31e791 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -22,7 +22,7 @@ import ( // PutPrm groups the parameters of Put operation. type PutPrm struct { - obj *objectSDK.Object + Object *objectSDK.Object } var errPutShard = errors.New("could not put object to any shard") @@ -41,13 +41,6 @@ type putToShardRes struct { err error } -// WithObject is a Put option to set object to save. -// -// Option is required. -func (p *PutPrm) WithObject(obj *objectSDK.Object) { - p.obj = obj -} - // Put saves the object to local storage. // // Returns any error encountered that @@ -59,7 +52,7 @@ func (p *PutPrm) WithObject(obj *objectSDK.Object) { func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Put", trace.WithAttributes( - attribute.String("address", object.AddressOf(prm.obj).EncodeToString()), + attribute.String("address", object.AddressOf(prm.Object).EncodeToString()), )) defer span.End() @@ -74,13 +67,13 @@ func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) { func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { defer elapsed("Put", e.metrics.AddMethodDuration)() - addr := object.AddressOf(prm.obj) + addr := object.AddressOf(prm.Object) // In #1146 this check was parallelized, however, it became // much slower on fast machines for 4 shards. var parent oid.Address - if prm.obj.ECHeader() != nil { - parent.SetObject(prm.obj.ECHeader().Parent()) + if prm.Object.ECHeader() != nil { + parent.SetObject(prm.Object.ECHeader().Parent()) parent.SetContainer(addr.Container()) } var shPrm shard.ExistsPrm @@ -113,7 +106,7 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { // Shard was concurrently removed, skip. return false } - shRes = e.putToShard(ctx, sh, pool, addr, prm.obj) + shRes = e.putToShard(ctx, sh, pool, addr, prm.Object) return shRes.status != putToShardUnknown }) switch shRes.status { @@ -202,8 +195,5 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti // Put writes provided object to local storage. func Put(ctx context.Context, storage *StorageEngine, obj *objectSDK.Object) error { - var putPrm PutPrm - putPrm.WithObject(obj) - - return storage.Put(ctx, putPrm) + return storage.Put(ctx, PutPrm{Object: obj}) } From 1b520f79733e3628af5d47b597b5baff60f3f36a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Oct 2024 15:27:06 +0300 Subject: [PATCH 1422/1943] [#1412] engine: Add `IsIndexedContainer` flag Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 4 +- pkg/core/container/info.go | 103 ++++++++++++++++++ pkg/core/container/util.go | 12 ++ .../engine/control_test.go | 2 +- .../engine/delete_test.go | 8 +- .../engine/engine_test.go | 2 +- .../engine/inhume_test.go | 2 +- pkg/local_object_storage/engine/lock_test.go | 14 +-- pkg/local_object_storage/engine/put.go | 7 +- pkg/local_object_storage/engine/tree_test.go | 2 +- pkg/services/object/common/writer/ec.go | 3 +- pkg/services/object/common/writer/local.go | 9 +- pkg/services/object/common/writer/writer.go | 3 +- pkg/services/object/put/single.go | 11 +- pkg/services/policer/check.go | 2 +- pkg/services/policer/ec.go | 39 ++++--- pkg/services/replicator/pull.go | 3 +- pkg/services/replicator/put.go | 3 +- pkg/services/replicator/task.go | 3 + 19 files changed, 182 insertions(+), 50 deletions(-) create mode 100644 pkg/core/container/info.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 47649c88b..5c322886b 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -535,6 +535,6 @@ func (e engineWithoutNotifications) Lock(ctx context.Context, locker oid.Address return e.engine.Lock(ctx, locker.Container(), locker.Object(), toLock) } -func (e engineWithoutNotifications) Put(ctx context.Context, o *objectSDK.Object) error { - return engine.Put(ctx, e.engine, o) +func (e engineWithoutNotifications) Put(ctx context.Context, o *objectSDK.Object, indexableContainer bool) error { + return engine.Put(ctx, e.engine, o, indexableContainer) } diff --git a/pkg/core/container/info.go b/pkg/core/container/info.go new file mode 100644 index 000000000..62cc21553 --- /dev/null +++ b/pkg/core/container/info.go @@ -0,0 +1,103 @@ +package container + +import ( + "sync" + + utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" +) + +type Info struct { + Indexed bool + Removed bool +} + +type infoValue struct { + info Info + err error +} + +type InfoProvider interface { + Info(id cid.ID) (Info, error) +} + +type infoProvider struct { + mtx *sync.RWMutex + cache map[cid.ID]infoValue + kl *utilSync.KeyLocker[cid.ID] + + source Source + sourceErr error + sourceOnce *sync.Once + sourceFactory func() (Source, error) +} + +func NewInfoProvider(sourceFactory func() (Source, error)) InfoProvider { + return &infoProvider{ + mtx: &sync.RWMutex{}, + cache: make(map[cid.ID]infoValue), + sourceOnce: &sync.Once{}, + kl: utilSync.NewKeyLocker[cid.ID](), + sourceFactory: sourceFactory, + } +} + +func (r *infoProvider) Info(id cid.ID) (Info, error) { + v, found := r.tryGetFromCache(id) + if found { + return v.info, v.err + } + + return r.getFromSource(id) +} + +func (r *infoProvider) tryGetFromCache(id cid.ID) (infoValue, bool) { + r.mtx.RLock() + defer r.mtx.RUnlock() + + value, found := r.cache[id] + return value, found +} + +func (r *infoProvider) getFromSource(id cid.ID) (Info, error) { + r.kl.Lock(id) + defer r.kl.Unlock(id) + + if v, ok := r.tryGetFromCache(id); ok { + return v.info, v.err + } + + r.sourceOnce.Do(func() { + r.source, r.sourceErr = r.sourceFactory() + }) + if r.sourceErr != nil { + return Info{}, r.sourceErr + } + + cnr, err := r.source.Get(id) + var civ infoValue + if err != nil { + if client.IsErrContainerNotFound(err) { + removed, err := WasRemoved(r.source, id) + if err != nil { + civ.err = err + } else { + civ.info.Removed = removed + } + } else { + civ.err = err + } + } else { + civ.info.Indexed = IsIndexedContainer(cnr.Value) + } + r.putToCache(id, civ) + return civ.info, civ.err +} + +func (r *infoProvider) putToCache(id cid.ID, ct infoValue) { + r.mtx.Lock() + defer r.mtx.Unlock() + + r.cache[id] = ct +} diff --git a/pkg/core/container/util.go b/pkg/core/container/util.go index 98919284e..d27556807 100644 --- a/pkg/core/container/util.go +++ b/pkg/core/container/util.go @@ -4,6 +4,7 @@ import ( "errors" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) @@ -20,3 +21,14 @@ func WasRemoved(s Source, cid cid.ID) (bool, error) { } return false, err } + +// IsIndexedContainer returns True if container attributes should be indexed. +func IsIndexedContainer(cnr containerSDK.Container) bool { + var isS3Container bool + cnr.IterateAttributes(func(key, _ string) { + if key == ".s3-location-constraint" { + isS3Container = true + } + }) + return !isS3Container +} diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 2de92ae84..83babeca3 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -171,7 +171,7 @@ func TestExecBlocks(t *testing.T) { addr := object.AddressOf(obj) - require.NoError(t, Put(context.Background(), e, obj)) + require.NoError(t, Put(context.Background(), e, obj, false)) // block executions errBlock := errors.New("block exec err") diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 4a6758012..0904c9820 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -58,9 +58,9 @@ func TestDeleteBigObject(t *testing.T) { defer e.Close(context.Background()) for i := range children { - require.NoError(t, Put(context.Background(), e, children[i])) + require.NoError(t, Put(context.Background(), e, children[i], false)) } - require.NoError(t, Put(context.Background(), e, link)) + require.NoError(t, Put(context.Background(), e, link, false)) addrParent := object.AddressOf(parent) checkGetError[*objectSDK.SplitInfoError](t, e, addrParent, true) @@ -126,9 +126,9 @@ func TestDeleteBigObjectWithoutGC(t *testing.T) { defer e.Close(context.Background()) for i := range children { - require.NoError(t, Put(context.Background(), e, children[i])) + require.NoError(t, Put(context.Background(), e, children[i], false)) } - require.NoError(t, Put(context.Background(), e, link)) + require.NoError(t, Put(context.Background(), e, link, false)) addrParent := object.AddressOf(parent) checkGetError[*objectSDK.SplitInfoError](t, e, addrParent, true) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 525e17f34..88c523b76 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -54,7 +54,7 @@ func benchmarkExists(b *testing.B, shardNum int) { addr := oidtest.Address() for range 100 { obj := testutil.GenerateObjectWithCID(cidtest.ID()) - err := Put(context.Background(), e, obj) + err := Put(context.Background(), e, obj, false) if err != nil { b.Fatal(err) } diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 9daa113f8..f87679253 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -40,7 +40,7 @@ func TestStorageEngine_Inhume(t *testing.T) { e := testNewEngine(t).setShardsNum(t, 1).engine defer e.Close(context.Background()) - err := Put(context.Background(), e, parent) + err := Put(context.Background(), e, parent, false) require.NoError(t, err) var inhumePrm InhumePrm diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 9e6758fb4..3702f567f 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -97,7 +97,7 @@ func TestLockUserScenario(t *testing.T) { id, _ := obj.ID() objAddr.SetObject(id) - err = Put(context.Background(), e, obj) + err = Put(context.Background(), e, obj, false) require.NoError(t, err) // 2. @@ -105,7 +105,7 @@ func TestLockUserScenario(t *testing.T) { locker.WriteMembers([]oid.ID{id}) objectSDK.WriteLock(lockerObj, locker) - err = Put(context.Background(), e, lockerObj) + err = Put(context.Background(), e, lockerObj, false) require.NoError(t, err) err = e.Lock(context.Background(), cnr, lockerID, []oid.ID{id}) @@ -124,7 +124,7 @@ func TestLockUserScenario(t *testing.T) { tombObj.SetID(tombForLockID) tombObj.SetAttributes(a) - err = Put(context.Background(), e, tombObj) + err = Put(context.Background(), e, tombObj, false) require.NoError(t, err) inhumePrm.WithTarget(tombForLockAddr, lockerAddr) @@ -177,7 +177,7 @@ func TestLockExpiration(t *testing.T) { // 1. obj := testutil.GenerateObjectWithCID(cnr) - err = Put(context.Background(), e, obj) + err = Put(context.Background(), e, obj, false) require.NoError(t, err) // 2. @@ -189,7 +189,7 @@ func TestLockExpiration(t *testing.T) { lock.SetType(objectSDK.TypeLock) lock.SetAttributes(a) - err = Put(context.Background(), e, lock) + err = Put(context.Background(), e, lock, false) require.NoError(t, err) id, _ := obj.ID() @@ -254,14 +254,14 @@ func TestLockForceRemoval(t *testing.T) { // 1. obj := testutil.GenerateObjectWithCID(cnr) - err = Put(context.Background(), e, obj) + err = Put(context.Background(), e, obj, false) require.NoError(t, err) // 2. lock := testutil.GenerateObjectWithCID(cnr) lock.SetType(objectSDK.TypeLock) - err = Put(context.Background(), e, lock) + err = Put(context.Background(), e, lock, false) require.NoError(t, err) id, _ := obj.ID() diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 9ce31e791..41e566560 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -22,7 +22,8 @@ import ( // PutPrm groups the parameters of Put operation. type PutPrm struct { - Object *objectSDK.Object + Object *objectSDK.Object + IsIndexedContainer bool } var errPutShard = errors.New("could not put object to any shard") @@ -194,6 +195,6 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti } // Put writes provided object to local storage. -func Put(ctx context.Context, storage *StorageEngine, obj *objectSDK.Object) error { - return storage.Put(ctx, PutPrm{Object: obj}) +func Put(ctx context.Context, storage *StorageEngine, obj *objectSDK.Object, indexedContainer bool) error { + return storage.Put(ctx, PutPrm{Object: obj, IsIndexedContainer: indexedContainer}) } diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index 6f694f082..21fcce415 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -37,7 +37,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { for i := range objCount { obj := testutil.GenerateObjectWithCID(cid) testutil.AddAttribute(obj, pilorama.AttributeFilename, strconv.Itoa(i)) - err := Put(context.Background(), te.ng, obj) + err := Put(context.Background(), te.ng, obj, false) if err != nil { b.Fatal(err) } diff --git a/pkg/services/object/common/writer/ec.go b/pkg/services/object/common/writer/ec.go index fb0a8e4e5..6b6a14cc0 100644 --- a/pkg/services/object/common/writer/ec.go +++ b/pkg/services/object/common/writer/ec.go @@ -310,7 +310,8 @@ func (e *ECWriter) putECPartToNode(ctx context.Context, obj *objectSDK.Object, n func (e *ECWriter) writePartLocal(ctx context.Context, obj *objectSDK.Object) error { var err error localTarget := LocalTarget{ - Storage: e.Config.LocalStore, + Storage: e.Config.LocalStore, + Container: e.Container, } completed := make(chan interface{}) if poolErr := e.Config.LocalPool.Submit(func() { diff --git a/pkg/services/object/common/writer/local.go b/pkg/services/object/common/writer/local.go index 02fd25b9e..e219b44dd 100644 --- a/pkg/services/object/common/writer/local.go +++ b/pkg/services/object/common/writer/local.go @@ -4,7 +4,9 @@ import ( "context" "fmt" + containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) @@ -13,7 +15,7 @@ import ( type ObjectStorage interface { // Put must save passed object // and return any appeared error. - Put(context.Context, *objectSDK.Object) error + Put(context.Context, *objectSDK.Object, bool) error // Delete must delete passed objects // and return any appeared error. Delete(ctx context.Context, tombstone oid.Address, toDelete []oid.ID) error @@ -25,7 +27,8 @@ type ObjectStorage interface { } type LocalTarget struct { - Storage ObjectStorage + Storage ObjectStorage + Container containerSDK.Container } func (t LocalTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, meta objectCore.ContentMeta) error { @@ -44,7 +47,7 @@ func (t LocalTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, met // objects that do not change meta storage } - if err := t.Storage.Put(ctx, obj); err != nil { + if err := t.Storage.Put(ctx, obj, containerCore.IsIndexedContainer(t.Container)); err != nil { return fmt.Errorf("(%T) could not put object to local storage: %w", t, err) } return nil diff --git a/pkg/services/object/common/writer/writer.go b/pkg/services/object/common/writer/writer.go index 3d50da988..0e4c4d9c6 100644 --- a/pkg/services/object/common/writer/writer.go +++ b/pkg/services/object/common/writer/writer.go @@ -150,7 +150,8 @@ func newDefaultObjectWriter(prm *Params, forECPlacement bool) transformer.Object nodeTargetInitializer: func(node NodeDescriptor) preparedObjectTarget { if node.Local { return LocalTarget{ - Storage: prm.Config.LocalStore, + Storage: prm.Config.LocalStore, + Container: prm.Container, } } diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 9b4163268..5f9b5d110 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -177,7 +177,7 @@ func (s *Service) saveToREPReplicas(ctx context.Context, placement putSinglePlac } return iter.ForEachNode(ctx, func(ctx context.Context, nd objectwriter.NodeDescriptor) error { - return s.saveToPlacementNode(ctx, &nd, obj, signer, meta) + return s.saveToPlacementNode(ctx, &nd, obj, signer, meta, placement.container) }) } @@ -263,10 +263,10 @@ func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumb } func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *objectwriter.NodeDescriptor, obj *objectSDK.Object, - signer *putSingleRequestSigner, meta object.ContentMeta, + signer *putSingleRequestSigner, meta object.ContentMeta, container containerSDK.Container, ) error { if nodeDesc.Local { - return s.saveLocal(ctx, obj, meta) + return s.saveLocal(ctx, obj, meta, container) } var info client.NodeInfo @@ -281,9 +281,10 @@ func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *objectwrite return s.redirectPutSingleRequest(ctx, signer, obj, info, c) } -func (s *Service) saveLocal(ctx context.Context, obj *objectSDK.Object, meta object.ContentMeta) error { +func (s *Service) saveLocal(ctx context.Context, obj *objectSDK.Object, meta object.ContentMeta, container containerSDK.Container) error { localTarget := &objectwriter.LocalTarget{ - Storage: s.Config.LocalStore, + Storage: s.Config.LocalStore, + Container: container, } return localTarget.WriteObject(ctx, obj, meta) } diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 06282bd8d..401977f66 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -37,7 +37,7 @@ func (p *Policer) processObject(ctx context.Context, objInfo objectcore.Info) er policy := cnr.Value.PlacementPolicy() if policycore.IsECPlacement(policy) { - return p.processECContainerObject(ctx, objInfo, policy) + return p.processECContainerObject(ctx, objInfo, cnr.Value) } return p.processRepContainerObject(ctx, objInfo, policy) } diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index e822d1c09..6d2c153c9 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -10,6 +10,7 @@ import ( objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" @@ -27,11 +28,11 @@ type ecChunkProcessResult struct { var errInvalidECPlacement = errors.New("invalid EC placement: EC placement must have one placement vector with at least one node") -func (p *Policer) processECContainerObject(ctx context.Context, objInfo objectcore.Info, policy netmap.PlacementPolicy) error { +func (p *Policer) processECContainerObject(ctx context.Context, objInfo objectcore.Info, cnr containerSDK.Container) error { if objInfo.ECInfo == nil { - return p.processECContainerRepObject(ctx, objInfo, policy) + return p.processECContainerRepObject(ctx, objInfo, cnr.PlacementPolicy()) } - return p.processECContainerECObject(ctx, objInfo, policy) + return p.processECContainerECObject(ctx, objInfo, cnr) } // processECContainerRepObject processes non erasure coded objects in EC container: tombstones, locks and linking objects. @@ -67,8 +68,8 @@ func (p *Policer) processECContainerRepObject(ctx context.Context, objInfo objec return nil } -func (p *Policer) processECContainerECObject(ctx context.Context, objInfo objectcore.Info, policy netmap.PlacementPolicy) error { - nn, err := p.placementBuilder.BuildPlacement(objInfo.Address.Container(), &objInfo.ECInfo.ParentID, policy) +func (p *Policer) processECContainerECObject(ctx context.Context, objInfo objectcore.Info, cnr containerSDK.Container) error { + nn, err := p.placementBuilder.BuildPlacement(objInfo.Address.Container(), &objInfo.ECInfo.ParentID, cnr.PlacementPolicy()) if err != nil { return fmt.Errorf("%s: %w", logs.PolicerCouldNotBuildPlacementVectorForObject, err) } @@ -85,9 +86,9 @@ func (p *Policer) processECContainerECObject(ctx context.Context, objInfo object res := p.processECChunk(ctx, objInfo, nn[0]) if !res.validPlacement { // drop local chunk only if all required chunks are in place - res.removeLocal = res.removeLocal && p.pullRequiredECChunks(ctx, objInfo, nn[0]) + res.removeLocal = res.removeLocal && p.pullRequiredECChunks(ctx, objInfo, nn[0], cnr) } - p.adjustECPlacement(ctx, objInfo, nn[0], policy) + p.adjustECPlacement(ctx, objInfo, nn[0], cnr) if res.removeLocal { p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, zap.Stringer("object", objInfo.Address)) @@ -138,7 +139,7 @@ func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, n } } -func (p *Policer) pullRequiredECChunks(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo) bool { +func (p *Policer) pullRequiredECChunks(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo, cnr containerSDK.Container) bool { var parentAddress oid.Address parentAddress.SetContainer(objInfo.Address.Container()) parentAddress.SetObject(objInfo.ECInfo.ParentID) @@ -169,8 +170,9 @@ func (p *Policer) pullRequiredECChunks(ctx context.Context, objInfo objectcore.I addr.SetContainer(objInfo.Address.Container()) addr.SetObject(indexToObjectID[index]) p.replicator.HandlePullTask(ctx, replicator.Task{ - Addr: addr, - Nodes: candidates, + Addr: addr, + Nodes: candidates, + Container: cnr, }) } // there was some missing chunks, it's not ok @@ -245,7 +247,7 @@ func (p *Policer) resolveRemoteECChunks(ctx context.Context, parentAddress oid.A return true } -func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo, policy netmap.PlacementPolicy) { +func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo, cnr containerSDK.Container) { var parentAddress oid.Address parentAddress.SetContainer(objInfo.Address.Container()) parentAddress.SetObject(objInfo.ECInfo.ParentID) @@ -292,7 +294,7 @@ func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info if !restore || uint32(len(resolved)) == objInfo.ECInfo.Total { return } - if objInfo.ECInfo.Total-uint32(len(resolved)) > policy.ReplicaDescriptor(0).GetECParityCount() { + if objInfo.ECInfo.Total-uint32(len(resolved)) > cnr.PlacementPolicy().ReplicaDescriptor(0).GetECParityCount() { var found []uint32 for i := range resolved { found = append(found, i) @@ -300,11 +302,13 @@ func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info p.log.Error(logs.PolicerCouldNotRestoreObjectNotEnoughChunks, zap.Stringer("object", parentAddress), zap.Uint32s("found_chunks", found)) return } - p.restoreECObject(ctx, objInfo, parentAddress, nodes, resolved, chunkIDs, policy) + p.restoreECObject(ctx, objInfo, parentAddress, nodes, resolved, chunkIDs, cnr) } -func (p *Policer) restoreECObject(ctx context.Context, objInfo objectcore.Info, parentAddress oid.Address, nodes []netmap.NodeInfo, existedChunks map[uint32][]netmap.NodeInfo, chunkIDs map[uint32]oid.ID, policy netmap.PlacementPolicy) { - c, err := erasurecode.NewConstructor(int(policy.ReplicaDescriptor(0).GetECDataCount()), int(policy.ReplicaDescriptor(0).GetECParityCount())) +func (p *Policer) restoreECObject(ctx context.Context, objInfo objectcore.Info, parentAddress oid.Address, nodes []netmap.NodeInfo, existedChunks map[uint32][]netmap.NodeInfo, chunkIDs map[uint32]oid.ID, + cnr containerSDK.Container, +) { + c, err := erasurecode.NewConstructor(int(cnr.PlacementPolicy().ReplicaDescriptor(0).GetECDataCount()), int(cnr.PlacementPolicy().ReplicaDescriptor(0).GetECParityCount())) if err != nil { p.log.Error(logs.PolicerFailedToRestoreObject, zap.Stringer("object", parentAddress), zap.Error(err)) return @@ -339,8 +343,9 @@ func (p *Policer) restoreECObject(ctx context.Context, objInfo objectcore.Info, targetNode := nodes[idx%len(nodes)] if p.cfg.netmapKeys.IsLocalKey(targetNode.PublicKey()) { p.replicator.HandleLocalPutTask(ctx, replicator.Task{ - Addr: addr, - Obj: part, + Addr: addr, + Obj: part, + Container: cnr, }) } else { p.replicator.HandleReplicationTask(ctx, replicator.Task{ diff --git a/pkg/services/replicator/pull.go b/pkg/services/replicator/pull.go index d178700f6..7e7090237 100644 --- a/pkg/services/replicator/pull.go +++ b/pkg/services/replicator/pull.go @@ -5,6 +5,7 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" @@ -62,7 +63,7 @@ func (p *Replicator) HandlePullTask(ctx context.Context, task Task) { return } - err := engine.Put(ctx, p.localStorage, obj) + err := engine.Put(ctx, p.localStorage, obj, containerCore.IsIndexedContainer(task.Container)) if err != nil { p.log.Error(logs.ReplicatorCouldNotPutObjectToLocalStorage, zap.Stringer("object", task.Addr), diff --git a/pkg/services/replicator/put.go b/pkg/services/replicator/put.go index c06ec3f65..537833516 100644 --- a/pkg/services/replicator/put.go +++ b/pkg/services/replicator/put.go @@ -5,6 +5,7 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -37,7 +38,7 @@ func (p *Replicator) HandleLocalPutTask(ctx context.Context, task Task) { return } - err := engine.Put(ctx, p.localStorage, task.Obj) + err := engine.Put(ctx, p.localStorage, task.Obj, containerCore.IsIndexedContainer(task.Container)) if err != nil { p.log.Error(logs.ReplicatorCouldNotPutObjectToLocalStorage, zap.Stringer("object", task.Addr), diff --git a/pkg/services/replicator/task.go b/pkg/services/replicator/task.go index d2b5b2506..a03f8dcaa 100644 --- a/pkg/services/replicator/task.go +++ b/pkg/services/replicator/task.go @@ -1,6 +1,7 @@ package replicator import ( + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -16,4 +17,6 @@ type Task struct { Obj *objectSDK.Object // Nodes is a list of potential object holders. Nodes []netmap.NodeInfo + + Container containerSDK.Container } From be744ae3e6eadb5b02952cbb110ef59f33f799bc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Oct 2024 18:19:12 +0300 Subject: [PATCH 1423/1943] [#1412] metabase: Index attributes for indexed containers Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 4 +- pkg/local_object_storage/engine/evacuate.go | 11 +-- pkg/local_object_storage/engine/put.go | 5 +- pkg/local_object_storage/metabase/delete.go | 45 ++++++++++ .../metabase/delete_meta_test.go | 85 +++++++++++++++++++ pkg/local_object_storage/metabase/put.go | 80 ++++++++++++++++- pkg/local_object_storage/metabase/util.go | 8 +- pkg/local_object_storage/shard/put.go | 8 +- 8 files changed, 231 insertions(+), 15 deletions(-) create mode 100644 pkg/local_object_storage/metabase/delete_meta_test.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 5c322886b..f2c4bff1d 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -535,6 +535,6 @@ func (e engineWithoutNotifications) Lock(ctx context.Context, locker oid.Address return e.engine.Lock(ctx, locker.Container(), locker.Object(), toLock) } -func (e engineWithoutNotifications) Put(ctx context.Context, o *objectSDK.Object, indexableContainer bool) error { - return engine.Put(ctx, e.engine, o, indexableContainer) +func (e engineWithoutNotifications) Put(ctx context.Context, o *objectSDK.Object, indexedContainer bool) error { + return engine.Put(ctx, e.engine, o, indexedContainer) } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index a618ff274..1baf237f9 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -18,6 +18,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -471,7 +472,7 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context default: } egObject.Go(func() error { - err := e.evacuateObject(ctx, shardID, objInfo, prm, res, shards, shardsToEvacuate) + err := e.evacuateObject(ctx, shardID, objInfo, prm, res, shards, shardsToEvacuate, c.Value) if err != nil { cancel(err) } @@ -744,7 +745,7 @@ func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) } func (e *StorageEngine) evacuateObject(ctx context.Context, shardID string, objInfo *object.Info, prm EvacuateShardPrm, res *EvacuateShardRes, - getShards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, + getShards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, cnr containerSDK.Container, ) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateObjects") defer span.End() @@ -773,7 +774,7 @@ func (e *StorageEngine) evacuateObject(ctx context.Context, shardID string, objI return err } - evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), shardsToEvacuate[shardID], shards, shardsToEvacuate, res) + evacuatedLocal, err := e.tryEvacuateObjectLocal(ctx, addr, getRes.Object(), shardsToEvacuate[shardID], shards, shardsToEvacuate, res, cnr) if err != nil { return err } @@ -817,7 +818,7 @@ func (e *StorageEngine) isNotRepOne(c *container.Container) bool { } func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, - shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes, + shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes, cnr containerSDK.Container, ) (bool, error) { hrw.SortHasherSliceByValue(shards, hrw.StringHash(addr.EncodeToString())) for j := range shards { @@ -830,7 +831,7 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add if _, ok := shardsToEvacuate[shards[j].ID().String()]; ok { continue } - switch e.putToShard(ctx, shards[j].hashedShard, shards[j].pool, addr, object).status { + switch e.putToShard(ctx, shards[j].hashedShard, shards[j].pool, addr, object, container.IsIndexedContainer(cnr)).status { case putToShardSuccess: res.objEvacuated.Add(1) e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 41e566560..a50d80b75 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -107,7 +107,7 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { // Shard was concurrently removed, skip. return false } - shRes = e.putToShard(ctx, sh, pool, addr, prm.Object) + shRes = e.putToShard(ctx, sh, pool, addr, prm.Object, prm.IsIndexedContainer) return shRes.status != putToShardUnknown }) switch shRes.status { @@ -125,7 +125,7 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { // putToShard puts object to sh. // Return putToShardStatus and error if it is necessary to propagate an error upper. func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool util.WorkerPool, - addr oid.Address, obj *objectSDK.Object, + addr oid.Address, obj *objectSDK.Object, isIndexedContainer bool, ) (res putToShardRes) { exitCh := make(chan struct{}) @@ -158,6 +158,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti var putPrm shard.PutPrm putPrm.SetObject(obj) + putPrm.SetIndexAttributes(isIndexedContainer) _, err = sh.Put(ctx, putPrm) if err != nil { diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 4ad11164f..3add1f268 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -338,6 +338,11 @@ func (db *DB) deleteObject( return fmt.Errorf("can't remove list indexes: %w", err) } + err = updateFKBTIndexes(tx, obj, delFKBTIndexItem) + if err != nil { + return fmt.Errorf("can't remove fake bucket tree indexes: %w", err) + } + if isParent { // remove record from the garbage bucket, because regular object deletion does nothing for virtual object garbageBKT := tx.Bucket(garbageBucketName) @@ -415,6 +420,46 @@ func delListIndexItem(tx *bbolt.Tx, item namedBucketItem) error { return nil } +func delFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { + bkt := tx.Bucket(item.name) + if bkt == nil { + return nil + } + + fkbtRoot := bkt.Bucket(item.key) + if fkbtRoot == nil { + return nil + } + + if err := fkbtRoot.Delete(item.val); err != nil { + return err + } + + if hasAnyItem(fkbtRoot) { + return nil + } + + if err := bkt.DeleteBucket(item.key); err != nil { + return err + } + + if hasAnyItem(bkt) { + return nil + } + + return tx.DeleteBucket(item.name) +} + +func hasAnyItem(b *bbolt.Bucket) bool { + var hasAnyItem bool + c := b.Cursor() + for k, _ := c.First(); k != nil; k, _ = c.Next() { + hasAnyItem = true + break + } + return hasAnyItem +} + func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error { addr := object.AddressOf(obj) diff --git a/pkg/local_object_storage/metabase/delete_meta_test.go b/pkg/local_object_storage/metabase/delete_meta_test.go new file mode 100644 index 000000000..cdfe2a203 --- /dev/null +++ b/pkg/local_object_storage/metabase/delete_meta_test.go @@ -0,0 +1,85 @@ +package meta + +import ( + "bytes" + "context" + "path/filepath" + "testing" + + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" +) + +func TestPutDeleteIndexAttributes(t *testing.T) { + db := New([]Option{ + WithPath(filepath.Join(t.TempDir(), "metabase")), + WithPermissions(0o600), + WithEpochState(epochState{}), + }...) + + require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) + require.NoError(t, db.Init()) + defer func() { require.NoError(t, db.Close()) }() + + cnr := cidtest.ID() + obj1 := testutil.GenerateObjectWithCID(cnr) + testutil.AddAttribute(obj1, "S3-Access-Box-CRDT-Name", "CRDT-Name") + testutil.AddAttribute(obj1, objectSDK.AttributeFilePath, "/path/to/object") + + var putPrm PutPrm + putPrm.SetObject(obj1) + + _, err := db.Put(context.Background(), putPrm) + require.NoError(t, err) + + require.NoError(t, db.boltDB.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(attributeBucketName(cnr, "S3-Access-Box-CRDT-Name", make([]byte, bucketKeySize))) + require.Nil(t, b) + b = tx.Bucket(attributeBucketName(cnr, objectSDK.AttributeFilePath, make([]byte, bucketKeySize))) + require.Nil(t, b) + return nil + })) + + obj2 := testutil.GenerateObjectWithCID(cnr) + testutil.AddAttribute(obj2, "S3-Access-Box-CRDT-Name", "CRDT-Name") + testutil.AddAttribute(obj2, objectSDK.AttributeFilePath, "/path/to/object") + + putPrm.SetObject(obj2) + putPrm.SetIndexAttributes(true) + + _, err = db.Put(context.Background(), putPrm) + require.NoError(t, err) + + objKey := objectKey(objectCore.AddressOf(obj2).Object(), make([]byte, objectKeySize)) + require.NoError(t, db.boltDB.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(attributeBucketName(cnr, "S3-Access-Box-CRDT-Name", make([]byte, bucketKeySize))) + require.NotNil(t, b) + b = b.Bucket([]byte("CRDT-Name")) + require.NotNil(t, b) + require.True(t, bytes.Equal(zeroValue, b.Get(objKey))) + b = tx.Bucket(attributeBucketName(cnr, objectSDK.AttributeFilePath, make([]byte, bucketKeySize))) + require.NotNil(t, b) + b = b.Bucket([]byte("/path/to/object")) + require.NotNil(t, b) + require.True(t, bytes.Equal(zeroValue, b.Get(objKey))) + return nil + })) + + var dPrm DeletePrm + dPrm.SetAddresses(objectCore.AddressOf(obj1), objectCore.AddressOf(obj2)) + _, err = db.Delete(context.Background(), dPrm) + require.NoError(t, err) + + require.NoError(t, db.boltDB.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(attributeBucketName(cnr, "S3-Access-Box-CRDT-Name", make([]byte, bucketKeySize))) + require.Nil(t, b) + b = tx.Bucket(attributeBucketName(cnr, objectSDK.AttributeFilePath, make([]byte, bucketKeySize))) + require.Nil(t, b) + return nil + })) +} diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 087529895..0c14196b7 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -35,6 +35,8 @@ type PutPrm struct { obj *objectSDK.Object id []byte + + indexAttributes bool } // PutRes groups the resulting values of Put operation. @@ -52,6 +54,10 @@ func (p *PutPrm) SetStorageID(id []byte) { p.id = id } +func (p *PutPrm) SetIndexAttributes(v bool) { + p.indexAttributes = v +} + var ( ErrUnknownObjectType = errors.New("unknown object type") ErrIncorrectSplitInfoUpdate = errors.New("updating split info on object without it") @@ -90,7 +96,7 @@ func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { err = db.boltDB.Batch(func(tx *bbolt.Tx) error { var e error - res, e = db.put(tx, prm.obj, prm.id, nil, currEpoch) + res, e = db.put(tx, prm.obj, prm.id, nil, currEpoch, prm.indexAttributes) return e }) if err == nil { @@ -108,6 +114,7 @@ func (db *DB) put(tx *bbolt.Tx, id []byte, si *objectSDK.SplitInfo, currEpoch uint64, + indexAttributes bool, ) (PutRes, error) { cnr, ok := obj.ContainerID() if !ok { @@ -129,7 +136,7 @@ func (db *DB) put(tx *bbolt.Tx, return PutRes{}, db.updateObj(tx, obj, id, si, isParent) } - return PutRes{Inserted: true}, db.insertObject(tx, obj, id, si, isParent, cnr, currEpoch) + return PutRes{Inserted: true}, db.insertObject(tx, obj, id, si, isParent, cnr, currEpoch, indexAttributes) } func (db *DB) updateObj(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *objectSDK.SplitInfo, isParent bool) error { @@ -152,14 +159,14 @@ func (db *DB) updateObj(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *obje return nil } -func (db *DB) insertObject(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *objectSDK.SplitInfo, isParent bool, cnr cid.ID, currEpoch uint64) error { +func (db *DB) insertObject(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *objectSDK.SplitInfo, isParent bool, cnr cid.ID, currEpoch uint64, indexAttributes bool) error { if par := obj.Parent(); par != nil && !isParent { // limit depth by two parentSI, err := splitInfoFromObject(obj) if err != nil { return err } - _, err = db.put(tx, par, id, parentSI, currEpoch) + _, err = db.put(tx, par, id, parentSI, currEpoch, indexAttributes) if err != nil { return err } @@ -175,6 +182,13 @@ func (db *DB) insertObject(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *o return fmt.Errorf("can't put list indexes: %w", err) } + if indexAttributes { + err = updateFKBTIndexes(tx, obj, putFKBTIndexItem) + if err != nil { + return fmt.Errorf("can't put fake bucket tree indexes: %w", err) + } + } + // update container volume size estimation if obj.Type() == objectSDK.TypeRegular && !isParent { err = changeContainerSize(tx, cnr, obj.PayloadSize(), true) @@ -381,6 +395,50 @@ func updateListIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFun return nil } +var indexedAttributes = map[string]struct{}{ + "S3-Access-Box-CRDT-Name": {}, + objectSDK.AttributeFilePath: {}, +} + +// IsAtrributeIndexed returns True if attribute is indexed by metabase. +func IsAtrributeIndexed(attr string) bool { + _, found := indexedAttributes[attr] + return found +} + +func updateFKBTIndexes(tx *bbolt.Tx, obj *objectSDK.Object, f updateIndexItemFunc) error { + id, _ := obj.ID() + cnr, _ := obj.ContainerID() + objKey := objectKey(id, make([]byte, objectKeySize)) + + key := make([]byte, bucketKeySize) + var attrs []objectSDK.Attribute + if obj.ECHeader() != nil { + attrs = obj.ECHeader().ParentAttributes() + objKey = objectKey(obj.ECHeader().Parent(), make([]byte, objectKeySize)) + } else { + attrs = obj.Attributes() + } + + // user specified attributes + for i := range attrs { + if !IsAtrributeIndexed(attrs[i].Key()) { + continue + } + key = attributeBucketName(cnr, attrs[i].Key(), key) + err := f(tx, namedBucketItem{ + name: key, + key: []byte(attrs[i].Value()), + val: objKey, + }) + if err != nil { + return err + } + } + + return nil +} + func hasExpirationEpoch(obj *objectSDK.Object) (uint64, bool) { attributes := obj.Attributes() if ech := obj.ECHeader(); ech != nil { @@ -425,6 +483,20 @@ func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error { return updateUniqueIndexItem(tx, item, func(_, val []byte) ([]byte, error) { return val, nil }) } +func putFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { + bkt, err := createBucketLikelyExists(tx, item.name) + if err != nil { + return fmt.Errorf("can't create index %v: %w", item.name, err) + } + + fkbtRoot, err := createBucketLikelyExists(bkt, item.key) + if err != nil { + return fmt.Errorf("can't create fake bucket tree index %v: %w", item.key, err) + } + + return fkbtRoot.Put(item.val, zeroValue) +} + func putListIndexItem(tx *bbolt.Tx, item namedBucketItem) error { bkt, err := createBucketLikelyExists(tx, item.name) if err != nil { diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index eef7210dc..4679de332 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -99,7 +99,6 @@ const ( // userAttributePrefix was used for prefixing FKBT index buckets containing objects. // Key: attribute value // Value: bucket containing object IDs as keys - // removed in version 3 userAttributePrefix // ==================== @@ -170,6 +169,13 @@ func smallBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, smallPrefix, key) } +// attributeBucketName returns _. +func attributeBucketName(cnr cid.ID, attributeKey string, key []byte) []byte { + key[0] = userAttributePrefix + cnr.Encode(key[1:]) + return append(key[:bucketKeySize], attributeKey...) +} + // rootBucketName returns _root. func rootBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, rootPrefix, key) diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index d7a9e7012..24cc75154 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -17,7 +17,8 @@ import ( // PutPrm groups the parameters of Put operation. type PutPrm struct { - obj *objectSDK.Object + obj *objectSDK.Object + indexAttributes bool } // PutRes groups the resulting values of Put operation. @@ -28,6 +29,10 @@ func (p *PutPrm) SetObject(obj *objectSDK.Object) { p.obj = obj } +func (p *PutPrm) SetIndexAttributes(v bool) { + p.indexAttributes = v +} + // Put saves the object in shard. // // Returns any error encountered that @@ -84,6 +89,7 @@ func (s *Shard) Put(ctx context.Context, prm PutPrm) (PutRes, error) { var pPrm meta.PutPrm pPrm.SetObject(prm.obj) pPrm.SetStorageID(res.StorageID) + pPrm.SetIndexAttributes(prm.indexAttributes) res, err := s.metaBase.Put(ctx, pPrm) if err != nil { // may we need to handle this case in a special way From 1efa64ee72851e53ca5eb2bf643e74141b41ca46 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Oct 2024 13:04:29 +0300 Subject: [PATCH 1424/1943] [#1412] metabase: Add search by indexed attributes Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/db_test.go | 12 +- pkg/local_object_storage/metabase/select.go | 115 ++++++++++++++++-- .../metabase/select_test.go | 81 +++++++----- 3 files changed, 169 insertions(+), 39 deletions(-) diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index 01e1ed2bc..0abb5ea89 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -32,7 +32,17 @@ func putBig(db *meta.DB, obj *objectSDK.Object) error { } func testSelect(t *testing.T, db *meta.DB, cnr cid.ID, fs objectSDK.SearchFilters, exp ...oid.Address) { - res, err := metaSelect(db, cnr, fs) + res, err := metaSelect(db, cnr, fs, false) + require.NoError(t, err) + require.Len(t, res, len(exp)) + + for i := range exp { + require.Contains(t, res, exp[i]) + } +} + +func testSelect2(t *testing.T, db *meta.DB, cnr cid.ID, fs objectSDK.SearchFilters, useAttrIndex bool, exp ...oid.Address) { + res, err := metaSelect(db, cnr, fs, useAttrIndex) require.NoError(t, err) require.Len(t, res, len(exp)) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 85d1b08ba..88ef7d5a4 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -37,8 +37,9 @@ type ( // SelectPrm groups the parameters of Select operation. type SelectPrm struct { - cnr cid.ID - filters objectSDK.SearchFilters + cnr cid.ID + filters objectSDK.SearchFilters + useAttributeIndex bool } // SelectRes groups the resulting values of Select operation. @@ -56,6 +57,10 @@ func (p *SelectPrm) SetFilters(fs objectSDK.SearchFilters) { p.filters = fs } +func (p *SelectPrm) SetUseAttributeIndex(v bool) { + p.useAttributeIndex = v +} + // AddressList returns list of addresses of the selected objects. func (r SelectRes) AddressList() []oid.Address { return r.addrList @@ -92,14 +97,14 @@ func (db *DB) Select(ctx context.Context, prm SelectPrm) (res SelectRes, err err currEpoch := db.epochState.CurrentEpoch() return res, metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { - res.addrList, err = db.selectObjects(tx, prm.cnr, prm.filters, currEpoch) + res.addrList, err = db.selectObjects(tx, prm.cnr, prm.filters, currEpoch, prm.useAttributeIndex) success = err == nil return err })) } -func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters, currEpoch uint64) ([]oid.Address, error) { - group, err := groupFilters(fs) +func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters, currEpoch uint64, useAttributeIndex bool) ([]oid.Address, error) { + group, err := groupFilters(fs, useAttributeIndex) if err != nil { return nil, err } @@ -218,7 +223,13 @@ func (db *DB) selectFastFilter( selectAllFromBucket(tx, primaryBucketName(cnr, bucketName), to, fNum) selectAllFromBucket(tx, tombstoneBucketName(cnr, bucketName), to, fNum) selectAllFromBucket(tx, bucketNameLockers(cnr, bucketName), to, fNum) - default: + default: // user attribute + bucketName := attributeBucketName(cnr, f.Header(), bucketName) + if f.Operation() == objectSDK.MatchNotPresent { + selectOutsideFKBT(tx, allBucketNames(cnr), bucketName, to, fNum) + } else { + db.selectFromFKBT(tx, bucketName, f, to, fNum) + } } } @@ -228,6 +239,15 @@ var mBucketNaming = map[string][]func(cid.ID, []byte) []byte{ v2object.TypeLock.String(): {bucketNameLockers}, } +func allBucketNames(cnr cid.ID) (names [][]byte) { + for _, fns := range mBucketNaming { + for _, fn := range fns { + names = append(names, fn(cnr, make([]byte, bucketKeySize))) + } + } + return +} + func bucketNamesForType(cnr cid.ID, mType objectSDK.SearchMatchType, typeVal string) (names [][]byte) { appendNames := func(key string) { fns, ok := mBucketNaming[key] @@ -259,6 +279,81 @@ func bucketNamesForType(cnr cid.ID, mType objectSDK.SearchMatchType, typeVal str return } +func (db *DB) selectFromFKBT( + tx *bbolt.Tx, + name []byte, // fkbt root bucket name + f objectSDK.SearchFilter, // filter for operation and value + to map[string]int, // resulting cache + fNum int, // index of filter +) { // + matchFunc, ok := db.matchers[f.Operation()] + if !ok { + db.log.Debug(logs.MetabaseMissingMatcher, zap.Stringer("operation", f.Operation())) + + return + } + + fkbtRoot := tx.Bucket(name) + if fkbtRoot == nil { + return + } + + err := matchFunc.matchBucket(fkbtRoot, f.Header(), f.Value(), func(k, _ []byte) error { + fkbtLeaf := fkbtRoot.Bucket(k) + if fkbtLeaf == nil { + return nil + } + + return fkbtLeaf.ForEach(func(k, _ []byte) error { + markAddressInCache(to, fNum, string(k)) + + return nil + }) + }) + if err != nil { + db.log.Debug(logs.MetabaseErrorInFKBTSelection, zap.String("error", err.Error())) + } +} + +// selectOutsideFKBT looks into all incl buckets to find list of addresses outside to add in +// resulting cache. +func selectOutsideFKBT( + tx *bbolt.Tx, + incl [][]byte, // buckets + name []byte, // fkbt root bucket name + to map[string]int, // resulting cache + fNum int, // index of filter +) { + mExcl := make(map[string]struct{}) + + bktExcl := tx.Bucket(name) + if bktExcl != nil { + _ = bktExcl.ForEachBucket(func(k []byte) error { + exclBktLeaf := bktExcl.Bucket(k) + return exclBktLeaf.ForEach(func(k, _ []byte) error { + mExcl[string(k)] = struct{}{} + + return nil + }) + }) + } + + for i := range incl { + bktIncl := tx.Bucket(incl[i]) + if bktIncl == nil { + continue + } + + _ = bktIncl.ForEach(func(k, _ []byte) error { + if _, ok := mExcl[string(k)]; !ok { + markAddressInCache(to, fNum, string(k)) + } + + return nil + }) + } +} + // selectFromList looks into index to find list of addresses to add in // resulting cache. func (db *DB) selectFromList( @@ -486,7 +581,7 @@ func attributeValue(obj *objectSDK.Object, attribute string) (string, bool) { // groupFilters divides filters in two groups: fast and slow. Fast filters // processed by indexes and slow filters processed after by unmarshaling // object headers. -func groupFilters(filters objectSDK.SearchFilters) (filterGroup, error) { +func groupFilters(filters objectSDK.SearchFilters, useAttributeIndex bool) (filterGroup, error) { res := filterGroup{ fastFilters: make(objectSDK.SearchFilters, 0, len(filters)), slowFilters: make(objectSDK.SearchFilters, 0, len(filters)), @@ -511,7 +606,11 @@ func groupFilters(filters objectSDK.SearchFilters) (filterGroup, error) { v2object.FilterPropertyPhy: res.fastFilters = append(res.fastFilters, filters[i]) default: - res.slowFilters = append(res.slowFilters, filters[i]) + if useAttributeIndex && IsAtrributeIndexed(filters[i].Header()) { + res.fastFilters = append(res.fastFilters, filters[i]) + } else { + res.slowFilters = append(res.slowFilters, filters[i]) + } } } diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index bee778e2b..5cc25a9f6 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -26,6 +26,16 @@ import ( func TestDB_SelectUserAttributes(t *testing.T) { t.Parallel() + t.Run("with_index", func(t *testing.T) { + testSelectUserAttributes(t, true) + }) + t.Run("without_index", func(t *testing.T) { + testSelectUserAttributes(t, false) + }) +} + +func testSelectUserAttributes(t *testing.T, index bool) { + t.Parallel() db := newDB(t) defer func() { require.NoError(t, db.Close()) }() @@ -36,44 +46,52 @@ func TestDB_SelectUserAttributes(t *testing.T) { testutil.AddAttribute(raw1, "foo", "bar") testutil.AddAttribute(raw1, "x", "y") - err := putBig(db, raw1) + var putPrm meta.PutPrm + putPrm.SetIndexAttributes(index) + putPrm.SetObject(raw1) + _, err := db.Put(context.Background(), putPrm) require.NoError(t, err) raw2 := testutil.GenerateObjectWithCID(cnr) testutil.AddAttribute(raw2, "foo", "bar") testutil.AddAttribute(raw2, "x", "z") - err = putBig(db, raw2) + putPrm.SetObject(raw2) + _, err = db.Put(context.Background(), putPrm) require.NoError(t, err) raw3 := testutil.GenerateObjectWithCID(cnr) testutil.AddAttribute(raw3, "a", "b") - err = putBig(db, raw3) + putPrm.SetObject(raw3) + _, err = db.Put(context.Background(), putPrm) require.NoError(t, err) raw4 := testutil.GenerateObjectWithCID(cnr) - testutil.AddAttribute(raw4, "path", "test/1/2") + testutil.AddAttribute(raw4, objectSDK.AttributeFilePath, "/test/1/2") - err = putBig(db, raw4) + putPrm.SetObject(raw4) + _, err = db.Put(context.Background(), putPrm) require.NoError(t, err) raw5 := testutil.GenerateObjectWithCID(cnr) - testutil.AddAttribute(raw5, "path", "test/1/3") + testutil.AddAttribute(raw5, objectSDK.AttributeFilePath, "/test/1/3") - err = putBig(db, raw5) + putPrm.SetObject(raw5) + _, err = db.Put(context.Background(), putPrm) require.NoError(t, err) raw6 := testutil.GenerateObjectWithCID(cnr) - testutil.AddAttribute(raw6, "path", "test/2/3") + testutil.AddAttribute(raw6, objectSDK.AttributeFilePath, "/test/2/3") - err = putBig(db, raw6) + putPrm.SetObject(raw6) + _, err = db.Put(context.Background(), putPrm) require.NoError(t, err) raw7 := testutil.GenerateObjectWithCID(cnr) var attr objectSDK.Attribute - attr.SetKey("path") - attr.SetValue("test/3/4") + attr.SetKey(objectSDK.AttributeFilePath) + attr.SetValue("/test/3/4") attrs := raw7.Attributes() attrs = append(attrs, attr) ech := objectSDK.NewECHeader(objectSDK.ECParentInfo{ @@ -81,37 +99,39 @@ func TestDB_SelectUserAttributes(t *testing.T) { Attributes: attrs, }, 0, 3, []byte{}, 0) raw7.SetECHeader(ech) - require.NoError(t, putBig(db, raw7)) + putPrm.SetObject(raw7) + _, err = db.Put(context.Background(), putPrm) + require.NoError(t, err) var raw7Parent oid.Address raw7Parent.SetContainer(cnr) raw7Parent.SetObject(ech.Parent()) fs := objectSDK.SearchFilters{} fs.AddFilter("foo", "bar", objectSDK.MatchStringEqual) - testSelect(t, db, cnr, fs, + testSelect2(t, db, cnr, fs, index, object.AddressOf(raw1), object.AddressOf(raw2), ) fs = objectSDK.SearchFilters{} fs.AddFilter("x", "y", objectSDK.MatchStringEqual) - testSelect(t, db, cnr, fs, object.AddressOf(raw1)) + testSelect2(t, db, cnr, fs, index, object.AddressOf(raw1)) fs = objectSDK.SearchFilters{} fs.AddFilter("x", "y", objectSDK.MatchStringNotEqual) - testSelect(t, db, cnr, fs, object.AddressOf(raw2)) + testSelect2(t, db, cnr, fs, index, object.AddressOf(raw2)) fs = objectSDK.SearchFilters{} fs.AddFilter("a", "b", objectSDK.MatchStringEqual) - testSelect(t, db, cnr, fs, object.AddressOf(raw3)) + testSelect2(t, db, cnr, fs, index, object.AddressOf(raw3)) fs = objectSDK.SearchFilters{} fs.AddFilter("c", "d", objectSDK.MatchStringEqual) - testSelect(t, db, cnr, fs) + testSelect2(t, db, cnr, fs, index) fs = objectSDK.SearchFilters{} fs.AddFilter("foo", "", objectSDK.MatchNotPresent) - testSelect(t, db, cnr, fs, + testSelect2(t, db, cnr, fs, index, object.AddressOf(raw3), object.AddressOf(raw4), object.AddressOf(raw5), @@ -121,7 +141,7 @@ func TestDB_SelectUserAttributes(t *testing.T) { fs = objectSDK.SearchFilters{} fs.AddFilter("a", "", objectSDK.MatchNotPresent) - testSelect(t, db, cnr, fs, + testSelect2(t, db, cnr, fs, index, object.AddressOf(raw1), object.AddressOf(raw2), object.AddressOf(raw4), @@ -131,7 +151,7 @@ func TestDB_SelectUserAttributes(t *testing.T) { ) fs = objectSDK.SearchFilters{} - testSelect(t, db, cnr, fs, + testSelect2(t, db, cnr, fs, index, object.AddressOf(raw1), object.AddressOf(raw2), object.AddressOf(raw3), @@ -143,7 +163,7 @@ func TestDB_SelectUserAttributes(t *testing.T) { fs = objectSDK.SearchFilters{} fs.AddFilter("key", "", objectSDK.MatchNotPresent) - testSelect(t, db, cnr, fs, + testSelect2(t, db, cnr, fs, index, object.AddressOf(raw1), object.AddressOf(raw2), object.AddressOf(raw3), @@ -154,8 +174,8 @@ func TestDB_SelectUserAttributes(t *testing.T) { ) fs = objectSDK.SearchFilters{} - fs.AddFilter("path", "test", objectSDK.MatchCommonPrefix) - testSelect(t, db, cnr, fs, + fs.AddFilter(objectSDK.AttributeFilePath, "/test", objectSDK.MatchCommonPrefix) + testSelect2(t, db, cnr, fs, index, object.AddressOf(raw4), object.AddressOf(raw5), object.AddressOf(raw6), @@ -163,15 +183,15 @@ func TestDB_SelectUserAttributes(t *testing.T) { ) fs = objectSDK.SearchFilters{} - fs.AddFilter("path", "test/1", objectSDK.MatchCommonPrefix) - testSelect(t, db, cnr, fs, + fs.AddFilter(objectSDK.AttributeFilePath, "/test/1", objectSDK.MatchCommonPrefix) + testSelect2(t, db, cnr, fs, index, object.AddressOf(raw4), object.AddressOf(raw5), ) fs = objectSDK.SearchFilters{} - fs.AddFilter("path", "test/3/4", objectSDK.MatchStringEqual) - testSelect(t, db, cnr, fs, + fs.AddFilter(objectSDK.AttributeFilePath, "/test/3/4", objectSDK.MatchStringEqual) + testSelect2(t, db, cnr, fs, index, raw7Parent, ) } @@ -1185,11 +1205,11 @@ func TestExpiredObjects(t *testing.T) { cidExp, _ := exp.ContainerID() cidNonExp, _ := nonExp.ContainerID() - objs, err := metaSelect(db, cidExp, objectSDK.SearchFilters{}) + objs, err := metaSelect(db, cidExp, objectSDK.SearchFilters{}, false) require.NoError(t, err) require.Empty(t, objs) // expired object should not be returned - objs, err = metaSelect(db, cidNonExp, objectSDK.SearchFilters{}) + objs, err = metaSelect(db, cidNonExp, objectSDK.SearchFilters{}, false) require.NoError(t, err) require.NotEmpty(t, objs) }) @@ -1211,10 +1231,11 @@ func benchmarkSelect(b *testing.B, db *meta.DB, cid cidSDK.ID, fs objectSDK.Sear } } -func metaSelect(db *meta.DB, cnr cidSDK.ID, fs objectSDK.SearchFilters) ([]oid.Address, error) { +func metaSelect(db *meta.DB, cnr cidSDK.ID, fs objectSDK.SearchFilters, useAttributeIndex bool) ([]oid.Address, error) { var prm meta.SelectPrm prm.SetFilters(fs) prm.SetContainerID(cnr) + prm.SetUseAttributeIndex(useAttributeIndex) res, err := db.Select(context.Background(), prm) return res.AddressList(), err From 4572fa487493f21cc5ebffcdc526270452a36e6a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Oct 2024 14:52:54 +0300 Subject: [PATCH 1425/1943] [#1412] searchSvc: Check container is indexed For non S3 containers it is expected to use attributes index for some attributes. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 5 +++-- pkg/local_object_storage/engine/delete.go | 2 +- pkg/local_object_storage/engine/inhume_test.go | 4 ++-- pkg/local_object_storage/engine/select.go | 14 ++++++++------ pkg/local_object_storage/engine/tree_test.go | 2 +- pkg/local_object_storage/shard/select.go | 9 ++++++--- pkg/services/object/search/container.go | 10 ++++++++++ pkg/services/object/search/service.go | 4 ++++ pkg/services/object/search/util.go | 7 ++++++- 9 files changed, 41 insertions(+), 16 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index f2c4bff1d..68acb05d3 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -174,7 +174,7 @@ func initObjectService(c *cfg) { sPutV2 := createPutSvcV2(sPut, keyStorage) - sSearch := createSearchSvc(c, keyStorage, traverseGen, c.clientCache) + sSearch := createSearchSvc(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource) sSearchV2 := createSearchSvcV2(sSearch, keyStorage) @@ -366,7 +366,7 @@ func createPatchSvc(sGet *getsvc.Service, sPut *putsvc.Service) *patchsvc.Servic return patchsvc.NewService(sPut.Config, sGet) } -func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache) *searchsvc.Service { +func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache, containerSource containercore.Source) *searchsvc.Service { ls := c.cfgObject.cfgLocalStorage.localStorage return searchsvc.New( @@ -377,6 +377,7 @@ func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Trav ), c.netMapSource, keyStorage, + containerSource, searchsvc.WithLogger(c.log), ) } diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 61cb6832d..3ec3f8f9b 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -143,7 +143,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo var selectPrm shard.SelectPrm selectPrm.SetFilters(fs) - selectPrm.SetContainerID(addr.Container()) + selectPrm.SetContainerID(addr.Container(), false) // doesn't matter for search by splitID var inhumePrm shard.InhumePrm if force { diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index f87679253..b4fbbd810 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -49,7 +49,7 @@ func TestStorageEngine_Inhume(t *testing.T) { _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) - addrs, err := Select(context.Background(), e, cnr, fs) + addrs, err := Select(context.Background(), e, cnr, false, fs) require.NoError(t, err) require.Empty(t, addrs) }) @@ -78,7 +78,7 @@ func TestStorageEngine_Inhume(t *testing.T) { _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) - addrs, err := Select(context.Background(), e, cnr, fs) + addrs, err := Select(context.Background(), e, cnr, false, fs) require.NoError(t, err) require.Empty(t, addrs) }) diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 972a4f52a..a85891f0c 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -14,8 +14,9 @@ import ( // SelectPrm groups the parameters of Select operation. type SelectPrm struct { - cnr cid.ID - filters objectSDK.SearchFilters + cnr cid.ID + filters objectSDK.SearchFilters + indexedContainer bool } // SelectRes groups the resulting values of Select operation. @@ -24,8 +25,9 @@ type SelectRes struct { } // WithContainerID is a Select option to set the container id to search in. -func (p *SelectPrm) WithContainerID(cnr cid.ID) { +func (p *SelectPrm) WithContainerID(cnr cid.ID, indexedContainer bool) { p.cnr = cnr + p.indexedContainer = indexedContainer } // WithFilters is a Select option to set the object filters. @@ -67,7 +69,7 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, var outError error var shPrm shard.SelectPrm - shPrm.SetContainerID(prm.cnr) + shPrm.SetContainerID(prm.cnr, prm.indexedContainer) shPrm.SetFilters(prm.filters) e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { @@ -140,9 +142,9 @@ func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, erro } // Select selects objects from local storage using provided filters. -func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, fs objectSDK.SearchFilters) ([]oid.Address, error) { +func Select(ctx context.Context, storage *StorageEngine, cnr cid.ID, isIndexedContainer bool, fs objectSDK.SearchFilters) ([]oid.Address, error) { var selectPrm SelectPrm - selectPrm.WithContainerID(cnr) + selectPrm.WithContainerID(cnr, isIndexedContainer) selectPrm.WithFilters(fs) res, err := storage.Select(ctx, selectPrm) diff --git a/pkg/local_object_storage/engine/tree_test.go b/pkg/local_object_storage/engine/tree_test.go index 21fcce415..ea0a9e74e 100644 --- a/pkg/local_object_storage/engine/tree_test.go +++ b/pkg/local_object_storage/engine/tree_test.go @@ -50,7 +50,7 @@ func benchmarkTreeVsSearch(b *testing.B, objCount int) { b.Run("search", func(b *testing.B) { var prm SelectPrm - prm.WithContainerID(cid) + prm.WithContainerID(cid, true) var fs objectSDK.SearchFilters fs.AddFilter(pilorama.AttributeFilename, strconv.Itoa(objCount/2), objectSDK.MatchStringEqual) diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index 1615f5fbe..184ca9b71 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -15,8 +15,9 @@ import ( // SelectPrm groups the parameters of Select operation. type SelectPrm struct { - cnr cid.ID - filters objectSDK.SearchFilters + cnr cid.ID + filters objectSDK.SearchFilters + isIndexedContainer bool } // SelectRes groups the resulting values of Select operation. @@ -25,8 +26,9 @@ type SelectRes struct { } // SetContainerID is a Select option to set the container id to search in. -func (p *SelectPrm) SetContainerID(cnr cid.ID) { +func (p *SelectPrm) SetContainerID(cnr cid.ID, isIndexedContainer bool) { p.cnr = cnr + p.isIndexedContainer = isIndexedContainer } // SetFilters is a Select option to set the object filters. @@ -61,6 +63,7 @@ func (s *Shard) Select(ctx context.Context, prm SelectPrm) (SelectRes, error) { var selectPrm meta.SelectPrm selectPrm.SetFilters(prm.filters) selectPrm.SetContainerID(prm.cnr) + selectPrm.SetUseAttributeIndex(prm.isIndexedContainer) mRes, err := s.metaBase.Select(ctx, selectPrm) if err != nil { diff --git a/pkg/services/object/search/container.go b/pkg/services/object/search/container.go index d70574156..39259b0ca 100644 --- a/pkg/services/object/search/container.go +++ b/pkg/services/object/search/container.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "go.uber.org/zap" ) @@ -112,3 +113,12 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { return nil } + +func (exec *execCtx) getContainer() (containerSDK.Container, error) { + cnrID := exec.containerID() + cnr, err := exec.svc.containerSource.Get(cnrID) + if err != nil { + return containerSDK.Container{}, err + } + return cnr.Value, nil +} diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index cc388c1b2..7700f78d8 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -54,6 +54,8 @@ type cfg struct { } keyStore *util.KeyStorage + + containerSource container.Source } // New creates, initializes and returns utility serving @@ -63,6 +65,7 @@ func New(e *engine.StorageEngine, tg *util.TraverserGenerator, ns netmap.Source, ks *util.KeyStorage, + cs container.Source, opts ...Option, ) *Service { c := &cfg{ @@ -76,6 +79,7 @@ func New(e *engine.StorageEngine, traverserGenerator: tg, currentEpochReceiver: ns, keyStore: ks, + containerSource: cs, } for i := range opts { diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index 67b6c0d01..910384a0b 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -5,6 +5,7 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" @@ -112,9 +113,13 @@ func (c *clientWrapper) searchObjects(ctx context.Context, exec *execCtx, info c } func (e *storageEngineWrapper) search(ctx context.Context, exec *execCtx) ([]oid.ID, error) { + cnr, err := exec.getContainer() + if err != nil { + return nil, err + } var selectPrm engine.SelectPrm selectPrm.WithFilters(exec.searchFilters()) - selectPrm.WithContainerID(exec.containerID()) + selectPrm.WithContainerID(exec.containerID(), container.IsIndexedContainer(cnr)) r, err := e.storage.Select(ctx, selectPrm) if err != nil { From 3da168f8cf96cb342c4cfe753e38c8a0bd7ad0a4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 2 Oct 2024 16:46:39 +0300 Subject: [PATCH 1426/1943] [#1412] shard: Resolve container is indexed on metabase resync Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 23 +++++++++++++++++++++-- internal/logs/logs.go | 1 + pkg/local_object_storage/shard/control.go | 17 +++++++++++++++++ pkg/local_object_storage/shard/shard.go | 10 ++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 4ad9ec6c6..3c7e310b4 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1058,7 +1058,9 @@ func initLocalStorage(ctx context.Context, c *cfg) { var shardsAttached int for _, optsWithMeta := range c.shardOpts(ctx) { - id, err := ls.AddShard(ctx, append(optsWithMeta.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))...) + id, err := ls.AddShard(ctx, append(optsWithMeta.shOpts, + shard.WithTombstoneSource(c.createTombstoneSource()), + shard.WithContainerInfoProvider(c.createContainerInfoProvider(ctx)))...) if err != nil { c.log.Error(logs.FrostFSNodeFailedToAttachShardToEngine, zap.Error(err)) } else { @@ -1313,7 +1315,10 @@ func (c *cfg) reloadConfig(ctx context.Context) { var rcfg engine.ReConfiguration for _, optsWithID := range c.shardOpts(ctx) { - rcfg.AddShard(optsWithID.configID, append(optsWithID.shOpts, shard.WithTombstoneSource(c.createTombstoneSource()))) + rcfg.AddShard(optsWithID.configID, append(optsWithID.shOpts, + shard.WithTombstoneSource(c.createTombstoneSource()), + shard.WithContainerInfoProvider(c.createContainerInfoProvider(ctx)), + )) } err = c.cfgObject.cfgLocalStorage.localStorage.Reload(ctx, rcfg) @@ -1414,6 +1419,20 @@ func (c *cfg) createTombstoneSource() *tombstone.ExpirationChecker { return tombstoneSource } +func (c *cfg) createContainerInfoProvider(ctx context.Context) container.InfoProvider { + return container.NewInfoProvider(func() (container.Source, error) { + // threadsafe: called on init or on sighup when morph initialized + if c.cfgMorph.client == nil { + initMorphComponents(ctx, c) + } + cc, err := containerClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0, containerClient.TryNotary()) + if err != nil { + return nil, err + } + return containerClient.AsContainerSource(cc), nil + }) +} + func (c *cfg) shutdown() { old := c.swapHealthStatus(control.HealthStatus_SHUTTING_DOWN) if old == control.HealthStatus_SHUTTING_DOWN { diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 7aef6873e..1ae4f0d3f 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -249,6 +249,7 @@ const ( ShardMetabaseFailureSwitchingMode = "metabase failure, switching mode" ShardCantMoveShardToReadonlySwitchMode = "can't move shard to readonly, switch mode" ShardCouldNotUnmarshalObject = "could not unmarshal object" + ShardSkipObjectFromResyncContainerDeleted = "object skipped from metabase resync: container deleted" ShardCouldNotCloseShardComponent = "could not close shard component" ShardCantOpenMetabaseMoveToADegradedMode = "can't open metabase, move to a degraded mode" ShardCantInitializeMetabaseMoveToADegradedreadonlyMode = "can't initialize metabase, move to a degraded-read-only mode" diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index de881654a..4f9f25608 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -275,6 +276,21 @@ func (s *Shard) refillObject(ctx context.Context, data []byte, addr oid.Address, return nil } + hasIndexedAttribute := slices.IndexFunc(obj.Attributes(), func(attr objectSDK.Attribute) bool { return meta.IsAtrributeIndexed(attr.Key()) }) > 0 + + var isIndexedContainer bool + if hasIndexedAttribute { + info, err := s.containerInfo.Info(addr.Container()) + if err != nil { + return err + } + if info.Removed { + s.log.Debug(logs.ShardSkipObjectFromResyncContainerDeleted, zap.Stringer("address", addr)) + return nil + } + isIndexedContainer = info.Indexed + } + var err error switch obj.Type() { case objectSDK.TypeTombstone: @@ -290,6 +306,7 @@ func (s *Shard) refillObject(ctx context.Context, data []byte, addr oid.Address, var mPrm meta.PutPrm mPrm.SetObject(obj) mPrm.SetStorageID(descriptor) + mPrm.SetIndexAttributes(hasIndexedAttribute && isIndexedContainer) _, err = s.metaBase.Put(ctx, mPrm) if err != nil && !client.IsErrObjectAlreadyRemoved(err) && !errors.Is(err, meta.ErrObjectIsExpired) { diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index d7e723733..413bfd2f7 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -7,6 +7,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -95,6 +96,8 @@ type cfg struct { metricsWriter MetricsWriter reportErrorFunc func(selfID string, message string, err error) + + containerInfo container.InfoProvider } func defaultCfg() *cfg { @@ -358,6 +361,13 @@ func WithZeroCountCallback(cb EmptyContainersCallback) Option { } } +// WithContainerInfoProvider returns option to set container info provider. +func WithContainerInfoProvider(containerInfo container.InfoProvider) Option { + return func(c *cfg) { + c.containerInfo = containerInfo + } +} + func (s *Shard) fillInfo() { s.cfg.info.MetaBaseInfo = s.metaBase.DumpInfo() s.cfg.info.BlobStorInfo = s.blobStor.DumpInfo() From 8093e145b316762977aff5b2c8babda64ae7283b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Oct 2024 11:06:31 +0300 Subject: [PATCH 1427/1943] [#1412] adm: Resolve container type by metabase upgrade Signed-off-by: Dmitrii Stepanov --- .../internal/modules/metabase/upgrade.go | 83 +++++++++++++---- pkg/local_object_storage/metabase/upgrade.go | 93 +++++++++++++++++-- .../metabase/upgrade_test.go | 21 +++-- pkg/local_object_storage/metabase/util.go | 15 +++ 4 files changed, 179 insertions(+), 33 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/metabase/upgrade.go b/cmd/frostfs-adm/internal/modules/metabase/upgrade.go index 96cb62f10..00b30c9b2 100644 --- a/cmd/frostfs-adm/internal/modules/metabase/upgrade.go +++ b/cmd/frostfs-adm/internal/modules/metabase/upgrade.go @@ -1,6 +1,7 @@ package metabase import ( + "context" "errors" "fmt" "sync" @@ -10,19 +11,24 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" engineconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine" shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard" + morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" + nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + morphcontainer "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" ) const ( - pathFlag = "path" noCompactFlag = "no-compact" ) -var errNoPathsFound = errors.New("no metabase paths found") - -var path string +var ( + errNoPathsFound = errors.New("no metabase paths found") + errNoMorphEndpointsFound = errors.New("no morph endpoints found") +) var UpgradeCmd = &cobra.Command{ Use: "upgrade", @@ -39,17 +45,10 @@ func upgrade(cmd *cobra.Command, _ []string) error { if err != nil { return err } - noCompact, _ := cmd.Flags().GetBool(noCompactFlag) - var paths []string - if path != "" { - paths = append(paths, path) - } appCfg := config.New(configFile, configDir, config.EnvPrefix) - if err := engineconfig.IterateShards(appCfg, false, func(sc *shardconfig.Config) error { - paths = append(paths, sc.Metabase().Path()) - return nil - }); err != nil { - return fmt.Errorf("failed to get metabase paths: %w", err) + paths, err := getMetabasePaths(appCfg) + if err != nil { + return err } if len(paths) == 0 { return errNoPathsFound @@ -58,6 +57,16 @@ func upgrade(cmd *cobra.Command, _ []string) error { for i, path := range paths { cmd.Println(i+1, ":", path) } + mc, err := createMorphClient(cmd.Context(), appCfg) + if err != nil { + return err + } + defer mc.Close() + civ, err := createContainerInfoProvider(mc) + if err != nil { + return err + } + noCompact, _ := cmd.Flags().GetBool(noCompactFlag) result := make(map[string]bool) var resultGuard sync.Mutex eg, ctx := errgroup.WithContext(cmd.Context()) @@ -65,7 +74,7 @@ func upgrade(cmd *cobra.Command, _ []string) error { eg.Go(func() error { var success bool cmd.Println("upgrading metabase", path, "...") - if err := meta.Upgrade(ctx, path, !noCompact, func(a ...any) { + if err := meta.Upgrade(ctx, path, !noCompact, civ, func(a ...any) { cmd.Println(append([]any{time.Now().Format(time.RFC3339), ":", path, ":"}, a...)...) }); err != nil { cmd.Println("error: failed to upgrade metabase", path, ":", err) @@ -92,8 +101,50 @@ func upgrade(cmd *cobra.Command, _ []string) error { return nil } +func getMetabasePaths(appCfg *config.Config) ([]string, error) { + var paths []string + if err := engineconfig.IterateShards(appCfg, false, func(sc *shardconfig.Config) error { + paths = append(paths, sc.Metabase().Path()) + return nil + }); err != nil { + return nil, fmt.Errorf("get metabase paths: %w", err) + } + return paths, nil +} + +func createMorphClient(ctx context.Context, appCfg *config.Config) (*client.Client, error) { + addresses := morphconfig.RPCEndpoint(appCfg) + if len(addresses) == 0 { + return nil, errNoMorphEndpointsFound + } + key := nodeconfig.Key(appCfg) + cli, err := client.New(ctx, + key, + client.WithDialTimeout(morphconfig.DialTimeout(appCfg)), + client.WithEndpoints(addresses...), + client.WithSwitchInterval(morphconfig.SwitchInterval(appCfg)), + ) + if err != nil { + return nil, fmt.Errorf("create morph client:%w", err) + } + return cli, nil +} + +func createContainerInfoProvider(cli *client.Client) (container.InfoProvider, error) { + sh, err := cli.NNSContractAddress(client.NNSContainerContractName) + if err != nil { + return nil, fmt.Errorf("resolve container contract hash: %w", err) + } + cc, err := morphcontainer.NewFromMorph(cli, sh, 0, morphcontainer.TryNotary()) + if err != nil { + return nil, fmt.Errorf("create morph container client: %w", err) + } + return container.NewInfoProvider(func() (container.Source, error) { + return morphcontainer.AsContainerSource(cc), nil + }), nil +} + func initUpgradeCommand() { flags := UpgradeCmd.Flags() - flags.StringVar(&path, pathFlag, "", "Path to metabase file") flags.Bool(noCompactFlag, false, "Do not compact upgraded metabase file") } diff --git a/pkg/local_object_storage/metabase/upgrade.go b/pkg/local_object_storage/metabase/upgrade.go index b5de430dc..f2a0107a1 100644 --- a/pkg/local_object_storage/metabase/upgrade.go +++ b/pkg/local_object_storage/metabase/upgrade.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/binary" + "encoding/hex" "errors" "fmt" "os" @@ -12,6 +13,7 @@ import ( "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" @@ -25,15 +27,15 @@ const ( upgradeTimeout = 1 * time.Second ) -var updates = map[uint64]func(ctx context.Context, db *bbolt.DB, log func(a ...any)) error{ +var updates = map[uint64]func(ctx context.Context, db *bbolt.DB, cs container.InfoProvider, log func(a ...any)) error{ 2: upgradeFromV2ToV3, - 3: func(_ context.Context, _ *bbolt.DB, log func(a ...any)) error { + 3: func(_ context.Context, _ *bbolt.DB, _ container.InfoProvider, log func(a ...any)) error { log("metabase already upgraded") return nil }, } -func Upgrade(ctx context.Context, path string, compact bool, log func(a ...any)) error { +func Upgrade(ctx context.Context, path string, compact bool, cs container.InfoProvider, log func(a ...any)) error { if _, err := os.Stat(path); err != nil { return fmt.Errorf("check metabase existence: %w", err) } @@ -61,7 +63,7 @@ func Upgrade(ctx context.Context, path string, compact bool, log func(a ...any)) }); err != nil { return fmt.Errorf("set upgrade key %w", err) } - if err := updater(ctx, db, log); err != nil { + if err := updater(ctx, db, cs, log); err != nil { return fmt.Errorf("update metabase schema: %w", err) } if err := db.Update(func(tx *bbolt.Tx) error { @@ -113,11 +115,11 @@ func compactDB(db *bbolt.DB) error { return nil } -func upgradeFromV2ToV3(ctx context.Context, db *bbolt.DB, log func(a ...any)) error { +func upgradeFromV2ToV3(ctx context.Context, db *bbolt.DB, cs container.InfoProvider, log func(a ...any)) error { if err := createExpirationEpochBuckets(ctx, db, log); err != nil { return err } - if err := dropUserAttributes(ctx, db, log); err != nil { + if err := dropUserAttributes(ctx, db, cs, log); err != nil { return err } if err := dropOwnerIDIndex(ctx, db, log); err != nil { @@ -323,10 +325,81 @@ func iterateExpirationAttributeKeyBucket(ctx context.Context, b *bbolt.Bucket, i return nil } -func dropUserAttributes(ctx context.Context, db *bbolt.DB, log func(a ...any)) error { - return dropBucketsByPrefix(ctx, db, []byte{userAttributePrefix}, func(a ...any) { - log(append([]any{"user attributes:"}, a...)...) - }) +func dropUserAttributes(ctx context.Context, db *bbolt.DB, cs container.InfoProvider, log func(a ...any)) error { + log("deleting user attribute buckets...") + const batch = 1000 + prefix := []byte{userAttributePrefix} + last := prefix + var count uint64 + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + var keys [][]byte + if err := db.View(func(tx *bbolt.Tx) error { + c := tx.Cursor() + for k, _ := c.Seek(last); k != nil && bytes.HasPrefix(k, prefix) && len(keys) < batch; k, _ = c.Next() { + if bytes.Equal(last, k) { + continue + } + keys = append(keys, bytes.Clone(k)) + } + return nil + }); err != nil { + log("deleting user attribute buckets completed with an error:", err) + return err + } + if len(keys) == 0 { + log("deleting user attribute buckets completed successfully, deleted", count, "buckets") + return nil + } + last = keys[len(keys)-1] + keysToDrop, err := selectUserAttributeKeysToDrop(keys, cs) + if err != nil { + return err + } + if err := db.Update(func(tx *bbolt.Tx) error { + for _, k := range keysToDrop { + if err := tx.DeleteBucket(k); err != nil { + return err + } + } + return nil + }); err != nil { + log("deleting buckets completed with an error:", err) + return err + } + count += uint64(len(keysToDrop)) + log("deleted", count, "buckets") + } +} + +func selectUserAttributeKeysToDrop(keys [][]byte, cs container.InfoProvider) ([][]byte, error) { + var keysToDrop [][]byte + for _, key := range keys { + attr, ok := attributeFromAttributeBucket(key) + if !ok { + return nil, fmt.Errorf("failed to parse attribute key from user attribute bucket key %s", hex.EncodeToString(key)) + } + if !IsAtrributeIndexed(attr) { + keysToDrop = append(keysToDrop, key) + continue + } + contID, ok := cidFromAttributeBucket(key) + if !ok { + return nil, fmt.Errorf("failed to parse container ID from user attribute bucket key %s", hex.EncodeToString(key)) + } + info, err := cs.Info(contID) + if err != nil { + return nil, err + } + if info.Removed || !info.Indexed { + keysToDrop = append(keysToDrop, key) + } + } + return keysToDrop, nil } func dropOwnerIDIndex(ctx context.Context, db *bbolt.DB, log func(a ...any)) error { diff --git a/pkg/local_object_storage/metabase/upgrade_test.go b/pkg/local_object_storage/metabase/upgrade_test.go index 3797de0a4..9c525291a 100644 --- a/pkg/local_object_storage/metabase/upgrade_test.go +++ b/pkg/local_object_storage/metabase/upgrade_test.go @@ -12,6 +12,7 @@ import ( "time" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -35,13 +36,19 @@ func TestUpgradeV2ToV3(t *testing.T) { require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.ErrorIs(t, db.Init(), ErrOutdatedVersion) require.NoError(t, db.Close()) - require.NoError(t, Upgrade(context.Background(), path, true, t.Log)) + require.NoError(t, Upgrade(context.Background(), path, true, &testContainerInfoProvider{}, t.Log)) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Init()) require.NoError(t, db.Close()) fmt.Println() } +type testContainerInfoProvider struct{} + +func (p *testContainerInfoProvider) Info(id cid.ID) (container.Info, error) { + return container.Info{}, nil +} + func createTempCopy(t *testing.T, path string) string { src, err := os.Open(path) require.NoError(t, err) @@ -95,7 +102,7 @@ func TestGenerateMetabaseFile(t *testing.T) { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) - testutil.AddAttribute(obj, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) + testutil.AddAttribute(obj, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) testutil.AddAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.FormatUint(uint64(i%minEpoch+minEpoch), 10)) _, err := db.Put(ctx, PutPrm{ obj: obj, @@ -118,8 +125,8 @@ func TestGenerateMetabaseFile(t *testing.T) { child.SetParent(parent) idParent, _ := parent.ID() child.SetParentID(idParent) - testutil.AddAttribute(child, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) - testutil.AddAttribute(parent, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) + testutil.AddAttribute(child, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) + testutil.AddAttribute(parent, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) testutil.AddAttribute(child, objectV2.SysAttributeExpEpoch, strconv.FormatUint(uint64(i%minEpoch+minEpoch), 10)) testutil.AddAttribute(parent, objectV2.SysAttributeExpEpoch, strconv.FormatUint(uint64(i%minEpoch+minEpoch), 10)) _, err := db.Put(ctx, PutPrm{ @@ -138,7 +145,7 @@ func TestGenerateMetabaseFile(t *testing.T) { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) - testutil.AddAttribute(obj, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) + testutil.AddAttribute(obj, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) _, err := db.Put(ctx, PutPrm{ obj: obj, id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), @@ -160,7 +167,7 @@ func TestGenerateMetabaseFile(t *testing.T) { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) - testutil.AddAttribute(obj, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) + testutil.AddAttribute(obj, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) _, err := db.Put(ctx, PutPrm{ obj: obj, id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), @@ -190,7 +197,7 @@ func TestGenerateMetabaseFile(t *testing.T) { i := i eg.Go(func() error { obj := testutil.GenerateObjectWithCID(containers[i%len(containers)]) - testutil.AddAttribute(obj, "FileName", strconv.FormatInt(int64(i%maxFilename), 10)) + testutil.AddAttribute(obj, objectSDK.AttributeFilePath, strconv.FormatInt(int64(i%maxFilename), 10)) _, err := db.Put(ctx, PutPrm{ obj: obj, id: []byte(strconv.FormatInt(int64(i%maxStorageID), 10) + "/" + strconv.FormatInt(int64(i%maxStorageID), 10)), diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index 4679de332..0a2f91a47 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -176,6 +176,21 @@ func attributeBucketName(cnr cid.ID, attributeKey string, key []byte) []byte { return append(key[:bucketKeySize], attributeKey...) } +func cidFromAttributeBucket(bucketName []byte) (cid.ID, bool) { + if len(bucketName) < bucketKeySize || bucketName[0] != userAttributePrefix { + return cid.ID{}, false + } + var result cid.ID + return result, result.Decode(bucketName[1:bucketKeySize]) == nil +} + +func attributeFromAttributeBucket(bucketName []byte) (string, bool) { + if len(bucketName) < bucketKeySize || bucketName[0] != userAttributePrefix { + return "", false + } + return string(bucketName[bucketKeySize:]), true +} + // rootBucketName returns _root. func rootBucketName(cnr cid.ID, key []byte) []byte { return bucketName(cnr, rootPrefix, key) From 87f4b934d1cca9a671dc93fcc4cdb5861be35915 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Oct 2024 17:57:21 +0300 Subject: [PATCH 1428/1943] [#1412] metabase: Run bucket drop steps on upgrade concurrently Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/upgrade.go | 22 ++++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/local_object_storage/metabase/upgrade.go b/pkg/local_object_storage/metabase/upgrade.go index f2a0107a1..a997b90a0 100644 --- a/pkg/local_object_storage/metabase/upgrade.go +++ b/pkg/local_object_storage/metabase/upgrade.go @@ -119,13 +119,17 @@ func upgradeFromV2ToV3(ctx context.Context, db *bbolt.DB, cs container.InfoProvi if err := createExpirationEpochBuckets(ctx, db, log); err != nil { return err } - if err := dropUserAttributes(ctx, db, cs, log); err != nil { - return err - } - if err := dropOwnerIDIndex(ctx, db, log); err != nil { - return err - } - if err := dropPayloadChecksumIndex(ctx, db, log); err != nil { + eg, ctx := errgroup.WithContext(ctx) + eg.Go(func() error { + return dropUserAttributes(ctx, db, cs, log) + }) + eg.Go(func() error { + return dropOwnerIDIndex(ctx, db, log) + }) + eg.Go(func() error { + return dropPayloadChecksumIndex(ctx, db, log) + }) + if err := eg.Wait(); err != nil { return err } return db.Update(func(tx *bbolt.Tx) error { @@ -360,7 +364,7 @@ func dropUserAttributes(ctx context.Context, db *bbolt.DB, cs container.InfoProv if err != nil { return err } - if err := db.Update(func(tx *bbolt.Tx) error { + if err := db.Batch(func(tx *bbolt.Tx) error { for _, k := range keysToDrop { if err := tx.DeleteBucket(k); err != nil { return err @@ -439,7 +443,7 @@ func dropBucketsByPrefix(ctx context.Context, db *bbolt.DB, prefix []byte, log f log("deleting buckets completed successfully, deleted", count, "buckets") return nil } - if err := db.Update(func(tx *bbolt.Tx) error { + if err := db.Batch(func(tx *bbolt.Tx) error { for _, k := range keys { if err := tx.DeleteBucket(k); err != nil { return err From fe9f664b577f1b51797e375ef736977ed61d9757 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 4 Oct 2024 10:49:39 +0300 Subject: [PATCH 1429/1943] [#1412] metabase: Drop empty user attribute buckets on upgrade Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/upgrade.go | 166 +++++++++++++++++-- 1 file changed, 154 insertions(+), 12 deletions(-) diff --git a/pkg/local_object_storage/metabase/upgrade.go b/pkg/local_object_storage/metabase/upgrade.go index a997b90a0..1f2c7956b 100644 --- a/pkg/local_object_storage/metabase/upgrade.go +++ b/pkg/local_object_storage/metabase/upgrade.go @@ -360,26 +360,40 @@ func dropUserAttributes(ctx context.Context, db *bbolt.DB, cs container.InfoProv return nil } last = keys[len(keys)-1] - keysToDrop, err := selectUserAttributeKeysToDrop(keys, cs) + cnt, err := dropNonIndexedUserAttributeBuckets(db, cs, keys) if err != nil { + log("deleting user attribute buckets completed with an error:", err) return err } - if err := db.Batch(func(tx *bbolt.Tx) error { - for _, k := range keysToDrop { - if err := tx.DeleteBucket(k); err != nil { - return err - } - } - return nil - }); err != nil { - log("deleting buckets completed with an error:", err) + count += cnt + cnt, err = dropEmptyUserAttributeBuckets(ctx, db, keys) + if err != nil { + log("deleting user attribute buckets completed with an error:", err) return err } - count += uint64(len(keysToDrop)) - log("deleted", count, "buckets") + count += cnt + log("deleted", count, "user attribute buckets") } } +func dropNonIndexedUserAttributeBuckets(db *bbolt.DB, cs container.InfoProvider, keys [][]byte) (uint64, error) { + keysToDrop, err := selectUserAttributeKeysToDrop(keys, cs) + if err != nil { + return 0, fmt.Errorf("select non indexed user attributes: %w", err) + } + if err := db.Batch(func(tx *bbolt.Tx) error { + for _, k := range keysToDrop { + if err := tx.DeleteBucket(k); err != nil { + return err + } + } + return nil + }); err != nil { + return 0, fmt.Errorf("drop non indexed user attributes: %w", err) + } + return uint64(len(keysToDrop)), nil +} + func selectUserAttributeKeysToDrop(keys [][]byte, cs container.InfoProvider) ([][]byte, error) { var keysToDrop [][]byte for _, key := range keys { @@ -406,6 +420,134 @@ func selectUserAttributeKeysToDrop(keys [][]byte, cs container.InfoProvider) ([] return keysToDrop, nil } +func dropEmptyUserAttributeBuckets(ctx context.Context, db *bbolt.DB, keys [][]byte) (uint64, error) { + var dropBuckets [][]byte + for _, key := range keys { + select { + case <-ctx.Done(): + return 0, ctx.Err() + default: + } + + if err := dropEmptyNestedBuckets(ctx, db, key); err != nil { + return 0, err + } + + empty, exists, err := bucketIsEmpty(db, key) + if err != nil { + return 0, err + } + if empty && exists { + dropBuckets = append(dropBuckets, key) + } + } + if len(dropBuckets) == 0 { + return 0, nil + } + if err := db.Batch(func(tx *bbolt.Tx) error { + for _, key := range dropBuckets { + if err := tx.DeleteBucket(key); err != nil { + return err + } + } + return nil + }); err != nil { + return 0, fmt.Errorf("drop empty user attributes buckets: %w", err) + } + return uint64(len(dropBuckets)), nil +} + +func bucketIsEmpty(db *bbolt.DB, bucketKey []byte) (bool, bool, error) { + var empty bool + var exists bool + if err := db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket(bucketKey) + if b == nil { + return nil + } + exists = true + empty = !hasAnyItem(b) + return nil + }); err != nil { + return false, false, fmt.Errorf("bucket empty check: %w", err) + } + return empty, exists, nil +} + +func dropEmptyNestedBuckets(ctx context.Context, db *bbolt.DB, rootBucketKey []byte) error { + var last []byte + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + var dropBuckets [][]byte + var err error + dropBuckets, last, err = selectEmptyNestedBuckets(ctx, db, rootBucketKey, last) + if err != nil { + return fmt.Errorf("select empty nested buckets: %w", err) + } + if len(dropBuckets) == 0 { + return nil + } + + if err := db.Batch(func(tx *bbolt.Tx) error { + rootBucket := tx.Bucket(rootBucketKey) + if rootBucket == nil { + return nil + } + for _, sb := range dropBuckets { + if err := rootBucket.DeleteBucket(sb); err != nil { + return err + } + } + return nil + }); err != nil { + return fmt.Errorf("drop empty nested buckets: %w", err) + } + } +} + +func selectEmptyNestedBuckets(ctx context.Context, db *bbolt.DB, rootBucketKey, last []byte) ([][]byte, []byte, error) { + const batchSize = 1000 + var result [][]byte + if err := db.View(func(tx *bbolt.Tx) error { + rootBucket := tx.Bucket(rootBucketKey) + if rootBucket == nil { + return nil + } + c := rootBucket.Cursor() + for k, v := c.Seek(last); k != nil && len(result) < batchSize; k, v = c.Next() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if bytes.Equal(last, k) { + continue + } + last = bytes.Clone(k) + if v != nil { // record + continue + } + nestedBucket := rootBucket.Bucket(k) + if nestedBucket == nil { + continue + } + if !hasAnyItem(nestedBucket) { + result = append(result, bytes.Clone(k)) + } + } + return nil + }); err != nil { + return nil, nil, err + } + return result, last, nil +} + func dropOwnerIDIndex(ctx context.Context, db *bbolt.DB, log func(a ...any)) error { return dropBucketsByPrefix(ctx, db, []byte{ownerPrefix}, func(a ...any) { log(append([]any{"owner ID index:"}, a...)...) From c065d55ca31c18fa48c8d8a173237095179be732 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 7 Oct 2024 17:19:04 +0300 Subject: [PATCH 1430/1943] [#1412] metabase: Drop logging inside transaction This could lead to hang the db. Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 6 ---- pkg/local_object_storage/metabase/select.go | 31 ++------------------- 2 files changed, 2 insertions(+), 35 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 1ae4f0d3f..84bd023f1 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -226,12 +226,6 @@ const ( EngineFinishedSuccessfullyShardsEvacuation = "shards evacuation finished successfully" EngineFinishedWithErrorShardsEvacuation = "shards evacuation finished with error" EngineObjectIsMovedToAnotherShard = "object is moved to another shard" - MetabaseMissingMatcher = "missing matcher" - MetabaseErrorInFKBTSelection = "error in FKBT selection" - MetabaseCantDecodeListBucketLeaf = "can't decode list bucket leaf" - MetabaseUnknownOperation = "unknown operation" - MetabaseCantIterateOverTheBucket = "can't iterate over the bucket" - MetabaseCouldNotIterateOverTheBuckets = "could not iterate over the buckets" MetabaseCreatedDirectoryForMetabase = "created directory for Metabase" MetabaseOpenedBoltDBInstanceForMetabase = "opened boltDB instance for Metabase" MetabaseCheckingMetabaseVersion = "checking metabase version" diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 88ef7d5a4..41f05b756 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -9,7 +9,6 @@ import ( "time" v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -18,7 +17,6 @@ import ( "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "go.uber.org/zap" ) type ( @@ -288,8 +286,6 @@ func (db *DB) selectFromFKBT( ) { // matchFunc, ok := db.matchers[f.Operation()] if !ok { - db.log.Debug(logs.MetabaseMissingMatcher, zap.Stringer("operation", f.Operation())) - return } @@ -298,7 +294,7 @@ func (db *DB) selectFromFKBT( return } - err := matchFunc.matchBucket(fkbtRoot, f.Header(), f.Value(), func(k, _ []byte) error { + _ = matchFunc.matchBucket(fkbtRoot, f.Header(), f.Value(), func(k, _ []byte) error { fkbtLeaf := fkbtRoot.Bucket(k) if fkbtLeaf == nil { return nil @@ -310,9 +306,6 @@ func (db *DB) selectFromFKBT( return nil }) }) - if err != nil { - db.log.Debug(logs.MetabaseErrorInFKBTSelection, zap.String("error", err.Error())) - } } // selectOutsideFKBT looks into all incl buckets to find list of addresses outside to add in @@ -377,24 +370,17 @@ func (db *DB) selectFromList( case objectSDK.MatchStringEqual: lst, err = decodeList(bkt.Get(bucketKeyHelper(f.Header(), f.Value()))) if err != nil { - db.log.Debug(logs.MetabaseCantDecodeListBucketLeaf, zap.String("error", err.Error())) return } default: fMatch, ok := db.matchers[op] if !ok { - db.log.Debug(logs.MetabaseUnknownOperation, zap.Uint32("operation", uint32(op))) - return } if err = fMatch.matchBucket(bkt, f.Header(), f.Value(), func(_, val []byte) error { l, err := decodeList(val) if err != nil { - db.log.Debug(logs.MetabaseCantDecodeListBucketLeaf, - zap.String("error", err.Error()), - ) - return err } @@ -402,10 +388,6 @@ func (db *DB) selectFromList( return nil }); err != nil { - db.log.Debug(logs.MetabaseCantIterateOverTheBucket, - zap.String("error", err.Error()), - ) - return } } @@ -447,10 +429,6 @@ func (db *DB) selectObjectID( default: fMatch, ok := db.matchers[op] if !ok { - db.log.Debug(logs.MetabaseUnknownOperation, - zap.Uint32("operation", uint32(f.Operation())), - ) - return } @@ -461,18 +439,13 @@ func (db *DB) selectObjectID( return } - err := fMatch.matchBucket(bkt, f.Header(), f.Value(), func(k, _ []byte) error { + _ = fMatch.matchBucket(bkt, f.Header(), f.Value(), func(k, _ []byte) error { var id oid.ID if err := id.Decode(k); err == nil { appendOID(id) } return nil }) - if err != nil { - db.log.Debug(logs.MetabaseCouldNotIterateOverTheBuckets, - zap.String("error", err.Error()), - ) - } } } } From 936ebbb8e5c1b1967e6fd0ec5ec50bd282c1f8bc Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 8 Oct 2024 18:39:52 +0300 Subject: [PATCH 1431/1943] [#1423] metabase: Hide `BucketName` form upper levels Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/engine/evacuate.go | 12 +- pkg/local_object_storage/metabase/list.go | 104 ++++++++++-------- .../metabase/list_test.go | 57 ++++++++++ pkg/local_object_storage/shard/list.go | 32 +++--- 4 files changed, 143 insertions(+), 62 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 1baf237f9..c1b9276f3 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -435,7 +435,7 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context ) error { sh := shardsToEvacuate[shardID] var cntPrm shard.IterateOverContainersPrm - cntPrm.Handler = func(ctx context.Context, name []byte, cnt cid.ID) error { + cntPrm.Handler = func(ctx context.Context, objType objectSDK.Type, cnt cid.ID) error { select { case <-ctx.Done(): return context.Cause(ctx) @@ -455,8 +455,11 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context skip = e.isNotRepOne(c) } if skip { - countPrm := shard.CountAliveObjectsInBucketPrm{BucketName: name} - count, err := sh.CountAliveObjectsInBucket(ctx, countPrm) + countPrm := shard.CountAliveObjectsInContainerPrm{ + ObjectType: objType, + ContainerID: cnt, + } + count, err := sh.CountAliveObjectsInContainer(ctx, countPrm) if err != nil { return err } @@ -464,7 +467,8 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context return nil } var objPrm shard.IterateOverObjectsInContainerPrm - objPrm.BucketName = name + objPrm.ObjectType = objType + objPrm.ContainerID = cnt objPrm.Handler = func(ctx context.Context, objInfo *object.Info) error { select { case <-ctx.Done(): diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 74a529809..b007ef0da 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -65,21 +65,25 @@ func (l ListRes) Cursor() *Cursor { // IterateOverContainersPrm contains parameters for IterateOverContainers operation. type IterateOverContainersPrm struct { // Handler function executed upon containers in db. - Handler func(context.Context, []byte, cid.ID) error + Handler func(context.Context, objectSDK.Type, cid.ID) error } // IterateOverObjectsInContainerPrm contains parameters for IterateOverObjectsInContainer operation. type IterateOverObjectsInContainerPrm struct { - // BucketName container's bucket name. - BucketName []byte + // ObjectType type of objects to iterate over. + ObjectType objectSDK.Type + // ContainerID container for objects to iterate over. + ContainerID cid.ID // Handler function executed upon objects in db. Handler func(context.Context, *objectcore.Info) error } -// CountAliveObjectsInBucketPrm contains parameters for IterateOverObjectsInContainer operation. -type CountAliveObjectsInBucketPrm struct { - // BucketName container's bucket name. - BucketName []byte +// CountAliveObjectsInContainerPrm contains parameters for IterateOverObjectsInContainer operation. +type CountAliveObjectsInContainerPrm struct { + // ObjectType type of objects to iterate over. + ObjectType objectSDK.Type + // ContainerID container for objects to iterate over. + ContainerID cid.ID } // ListWithCursor lists physical objects available in metabase starting from @@ -319,12 +323,20 @@ func (db *DB) iterateOverContainers(ctx context.Context, tx *bbolt.Tx, prm Itera if cidRaw == nil { continue } - - bktName := make([]byte, len(name)) - copy(bktName, name) var cnt cid.ID copy(cnt[:], containerID[:]) - err := prm.Handler(ctx, bktName, cnt) + var objType objectSDK.Type + switch prefix[0] { + case primaryPrefix: + objType = objectSDK.TypeRegular + case lockersPrefix: + objType = objectSDK.TypeLock + case tombstonePrefix: + objType = objectSDK.TypeTombstone + default: + continue + } + err := prm.Handler(ctx, objType, cnt) if err != nil { return err } @@ -356,22 +368,29 @@ func (db *DB) IterateOverObjectsInContainer(ctx context.Context, prm IterateOver return ErrDegradedMode } - var containerID cid.ID - cidRaw, prefix := parseContainerIDWithPrefix(&containerID, prm.BucketName) - if cidRaw == nil { - return nil - } err := db.boltDB.View(func(tx *bbolt.Tx) error { - return db.iterateOverObjectsInContainer(ctx, tx, cidRaw, prefix, containerID, prm) + return db.iterateOverObjectsInContainer(ctx, tx, prm) }) success = err == nil return metaerr.Wrap(err) } -func (db *DB) iterateOverObjectsInContainer(ctx context.Context, tx *bbolt.Tx, cidRaw []byte, prefix byte, - containerID cid.ID, prm IterateOverObjectsInContainerPrm, -) error { - bkt := tx.Bucket(prm.BucketName) +func (db *DB) iterateOverObjectsInContainer(ctx context.Context, tx *bbolt.Tx, prm IterateOverObjectsInContainerPrm) error { + var prefix byte + switch prm.ObjectType { + case objectSDK.TypeRegular: + prefix = primaryPrefix + case objectSDK.TypeLock: + prefix = lockersPrefix + case objectSDK.TypeTombstone: + prefix = tombstonePrefix + default: + return nil + } + bucketName := []byte{prefix} + bucketName = append(bucketName, prm.ContainerID[:]...) + + bkt := tx.Bucket(bucketName) if bkt == nil { return nil } @@ -380,32 +399,19 @@ func (db *DB) iterateOverObjectsInContainer(ctx context.Context, tx *bbolt.Tx, c c := bkt.Cursor() k, v := c.First() - var objType objectSDK.Type - - switch prefix { - case primaryPrefix: - objType = objectSDK.TypeRegular - case lockersPrefix: - objType = objectSDK.TypeLock - case tombstonePrefix: - objType = objectSDK.TypeTombstone - default: - return nil - } - for ; k != nil; k, v = c.Next() { var obj oid.ID if err := obj.Decode(k); err != nil { break } - if inGraveyardWithKey(append(cidRaw, k...), graveyardBkt, garbageBkt) > 0 { + if inGraveyardWithKey(append(prm.ContainerID[:], k...), graveyardBkt, garbageBkt) > 0 { continue } var isLinkingObj bool var ecInfo *objectcore.ECInfo - if objType == objectSDK.TypeRegular { + if prm.ObjectType == objectSDK.TypeRegular { var o objectSDK.Object if err := o.Unmarshal(v); err != nil { return err @@ -422,9 +428,9 @@ func (db *DB) iterateOverObjectsInContainer(ctx context.Context, tx *bbolt.Tx, c } var a oid.Address - a.SetContainer(containerID) + a.SetContainer(prm.ContainerID) a.SetObject(obj) - objInfo := objectcore.Info{Address: a, Type: objType, IsLinkingObject: isLinkingObj, ECInfo: ecInfo} + objInfo := objectcore.Info{Address: a, Type: prm.ObjectType, IsLinkingObject: isLinkingObj, ECInfo: ecInfo} err := prm.Handler(ctx, &objInfo) if err != nil { return err @@ -433,8 +439,8 @@ func (db *DB) iterateOverObjectsInContainer(ctx context.Context, tx *bbolt.Tx, c return nil } -// CountAliveObjectsInBucket count objects in bucket which aren't in graveyard or garbage. -func (db *DB) CountAliveObjectsInBucket(ctx context.Context, prm CountAliveObjectsInBucketPrm) (uint64, error) { +// CountAliveObjectsInContainer count objects in bucket which aren't in graveyard or garbage. +func (db *DB) CountAliveObjectsInContainer(ctx context.Context, prm CountAliveObjectsInContainerPrm) (uint64, error) { var ( startedAt = time.Now() success = false @@ -452,14 +458,22 @@ func (db *DB) CountAliveObjectsInBucket(ctx context.Context, prm CountAliveObjec return 0, ErrDegradedMode } - if len(prm.BucketName) != bucketKeySize { + var prefix byte + switch prm.ObjectType { + case objectSDK.TypeRegular: + prefix = primaryPrefix + case objectSDK.TypeLock: + prefix = lockersPrefix + case objectSDK.TypeTombstone: + prefix = tombstonePrefix + default: return 0, nil } - - cidRaw := prm.BucketName[1:bucketKeySize] + bucketName := []byte{prefix} + bucketName = append(bucketName, prm.ContainerID[:]...) var count uint64 err := db.boltDB.View(func(tx *bbolt.Tx) error { - bkt := tx.Bucket(prm.BucketName) + bkt := tx.Bucket(bucketName) if bkt == nil { return nil } @@ -468,7 +482,7 @@ func (db *DB) CountAliveObjectsInBucket(ctx context.Context, prm CountAliveObjec c := bkt.Cursor() k, _ := c.First() for ; k != nil; k, _ = c.Next() { - if inGraveyardWithKey(append(cidRaw, k...), graveyardBkt, garbageBkt) > 0 { + if inGraveyardWithKey(append(prm.ContainerID[:], k...), graveyardBkt, garbageBkt) > 0 { continue } count++ diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index bc1726bd6..203802ec0 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -219,3 +220,59 @@ func metaListWithCursor(db *meta.DB, count uint32, cursor *meta.Cursor) ([]objec r, err := db.ListWithCursor(context.Background(), listPrm) return r.AddressList(), r.Cursor(), err } + +func TestIterateOver(t *testing.T) { + t.Parallel() + + db := newDB(t) + defer func() { require.NoError(t, db.Close()) }() + + const total uint64 = 5 + for _, typ := range []objectSDK.Type{objectSDK.TypeRegular, objectSDK.TypeTombstone, objectSDK.TypeLock} { + var expected []*objectSDK.Object + // fill metabase with objects + cid := cidtest.ID() + for range total { + obj := testutil.GenerateObjectWithCID(cid) + obj.SetType(typ) + err := metaPut(db, obj, nil) + require.NoError(t, err) + expected = append(expected, obj) + } + + var metaIter meta.IterateOverObjectsInContainerPrm + var count uint64 + metaIter.Handler = func(context.Context, *object.Info) error { + count++ + return nil + } + metaIter.ContainerID = cid + metaIter.ObjectType = typ + err := db.IterateOverObjectsInContainer(context.Background(), metaIter) + require.NoError(t, err) + require.Equal(t, total, count) + + var metaCount meta.CountAliveObjectsInContainerPrm + metaCount.ContainerID = cid + metaCount.ObjectType = typ + res, err := db.CountAliveObjectsInContainer(context.Background(), metaCount) + require.NoError(t, err) + require.Equal(t, res, total) + + err = metaDelete(db, object.AddressOf(expected[0]), object.AddressOf(expected[1])) + require.NoError(t, err) + + res, err = db.CountAliveObjectsInContainer(context.Background(), metaCount) + require.NoError(t, err) + require.Equal(t, uint64(3), res) + } + var count int + var metaPrm meta.IterateOverContainersPrm + metaPrm.Handler = func(context.Context, objectSDK.Type, cidSDK.ID) error { + count++ + return nil + } + err := db.IterateOverContainers(context.Background(), metaPrm) + require.NoError(t, err) + require.Equal(t, 3, count) +} diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index f5d633b77..8d09974b8 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -37,21 +37,25 @@ func (r ListContainersRes) Containers() []cid.ID { // IterateOverContainersPrm contains parameters for IterateOverContainers operation. type IterateOverContainersPrm struct { // Handler function executed upon containers in db. - Handler func(context.Context, []byte, cid.ID) error + Handler func(context.Context, objectSDK.Type, cid.ID) error } // IterateOverObjectsInContainerPrm contains parameters for IterateOverObjectsInContainer operation. type IterateOverObjectsInContainerPrm struct { - // BucketName container's bucket name. - BucketName []byte + // ObjectType type of objects to iterate over. + ObjectType objectSDK.Type + // ContainerID container for objects to iterate over. + ContainerID cid.ID // Handler function executed upon objects in db. Handler func(context.Context, *objectcore.Info) error } -// CountAliveObjectsInBucketPrm contains parameters for CountAliveObjectsInBucket operation. -type CountAliveObjectsInBucketPrm struct { - // BucketName container's bucket name. - BucketName []byte +// CountAliveObjectsInContainerPrm contains parameters for CountAliveObjectsInContainer operation. +type CountAliveObjectsInContainerPrm struct { + // ObjectType type of objects to iterate over. + ObjectType objectSDK.Type + // ContainerID container for objects to iterate over. + ContainerID cid.ID } // ListWithCursorPrm contains parameters for ListWithCursor operation. @@ -226,7 +230,8 @@ func (s *Shard) IterateOverObjectsInContainer(ctx context.Context, prm IterateOv } var metaPrm meta.IterateOverObjectsInContainerPrm - metaPrm.BucketName = prm.BucketName + metaPrm.ContainerID = prm.ContainerID + metaPrm.ObjectType = prm.ObjectType metaPrm.Handler = prm.Handler err := s.metaBase.IterateOverObjectsInContainer(ctx, metaPrm) if err != nil { @@ -236,8 +241,8 @@ func (s *Shard) IterateOverObjectsInContainer(ctx context.Context, prm IterateOv return nil } -// CountAliveObjectsInBucket count objects in bucket which aren't in graveyard or garbage. -func (s *Shard) CountAliveObjectsInBucket(ctx context.Context, prm CountAliveObjectsInBucketPrm) (uint64, error) { +// CountAliveObjectsInContainer count objects in bucket which aren't in graveyard or garbage. +func (s *Shard) CountAliveObjectsInContainer(ctx context.Context, prm CountAliveObjectsInContainerPrm) (uint64, error) { _, span := tracing.StartSpanFromContext(ctx, "shard.CountAliveObjectsInBucket") defer span.End() @@ -248,9 +253,10 @@ func (s *Shard) CountAliveObjectsInBucket(ctx context.Context, prm CountAliveObj return 0, ErrDegradedMode } - var metaPrm meta.CountAliveObjectsInBucketPrm - metaPrm.BucketName = prm.BucketName - count, err := s.metaBase.CountAliveObjectsInBucket(ctx, metaPrm) + var metaPrm meta.CountAliveObjectsInContainerPrm + metaPrm.ObjectType = prm.ObjectType + metaPrm.ContainerID = prm.ContainerID + count, err := s.metaBase.CountAliveObjectsInContainer(ctx, metaPrm) if err != nil { return 0, fmt.Errorf("could not count alive objects in bucket: %w", err) } From 4190fba86d575ba6c05aeb919ae5bc30572ecbd2 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 8 Oct 2024 16:44:34 +0300 Subject: [PATCH 1432/1943] [#1425] Remove SetEACL-related code Signed-off-by: Evgenii Stratonikov --- .../modules/morph/container/container.go | 8 -- internal/logs/logs.go | 3 - .../processors/container/handlers.go | 4 - .../processors/container/processor.go | 10 --- pkg/morph/client/container/client.go | 1 - pkg/morph/event/container/eacl.go | 51 ------------- pkg/morph/event/container/eacl_notary.go | 75 ------------------- pkg/services/container/morph/executor_test.go | 4 - 8 files changed, 156 deletions(-) delete mode 100644 pkg/morph/event/container/eacl.go delete mode 100644 pkg/morph/event/container/eacl_notary.go diff --git a/cmd/frostfs-adm/internal/modules/morph/container/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go index a66438975..e280bc634 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -159,9 +159,6 @@ func dumpSingleContainer(bw *io.BufBinWriter, ch util.Uint160, inv *invoker.Invo if err != nil { return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err) } - if len(ea.Value) != 0 { - cnt.EACL = ea - } return cnt, nil } @@ -258,10 +255,6 @@ func restoreOrPutContainers(containers []Container, isOK func([]byte) bool, cmd func putContainer(bw *io.BufBinWriter, ch util.Uint160, cnt Container) { emit.AppCall(bw.BinWriter, ch, "put", callflag.All, cnt.Value, cnt.Signature, cnt.PublicKey, cnt.Token) - if ea := cnt.EACL; ea != nil { - emit.AppCall(bw.BinWriter, ch, "setEACL", callflag.All, - ea.Value, ea.Signature, ea.PublicKey, ea.Token) - } } func isContainerRestored(cmd *cobra.Command, wCtx *helper.InitializeContext, containerHash util.Uint160, bw *io.BufBinWriter, hashValue util.Uint256) (bool, error) { @@ -322,7 +315,6 @@ type Container struct { Signature []byte `json:"signature"` PublicKey []byte `json:"public_key"` Token []byte `json:"token"` - EACL *EACL `json:"eacl"` } // EACL represents extended ACL struct in contract storage. diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 84bd023f1..ca783a39d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -308,9 +308,6 @@ const ( ContainerNonAlphabetModeIgnoreContainerDelete = "non alphabet mode, ignore container delete" ContainerDeleteContainerCheckFailed = "delete container check failed" ContainerCouldNotApproveDeleteContainer = "could not approve delete container" - ContainerNonAlphabetModeIgnoreSetEACL = "non alphabet mode, ignore set EACL" - ContainerSetEACLCheckFailed = "set EACL check failed" - ContainerCouldNotApproveSetEACL = "could not approve set EACL" FrostFSNonAlphabetModeIgnoreConfig = "non alphabet mode, ignore config" FrostFSCantRelaySetConfigEvent = "can't relay set config event" FrostFSFrostfsWorkerPool = "frostfs worker pool" diff --git a/pkg/innerring/processors/container/handlers.go b/pkg/innerring/processors/container/handlers.go index 8bb89abe2..a54f3c772 100644 --- a/pkg/innerring/processors/container/handlers.go +++ b/pkg/innerring/processors/container/handlers.go @@ -48,7 +48,3 @@ func (cp *Processor) handleDelete(ev event.Event) { zap.Int("capacity", cp.pool.Cap())) } } - -func (cp *Processor) handleSetEACL(_ event.Event) { - cp.log.Warn(logs.SkipDeprecatedNotification, zap.String("type", "set EACL")) -} diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index 8fd9edfb8..a6fbdc707 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -157,11 +157,6 @@ func (cp *Processor) ListenerNotaryParsers() []event.NotaryParserInfo { p.SetParser(containerEvent.ParseDeleteNotary) pp = append(pp, p) - // set EACL - p.SetRequestType(containerEvent.SetEACLNotaryEvent) - p.SetParser(containerEvent.ParseSetEACLNotary) - pp = append(pp, p) - return pp } @@ -190,10 +185,5 @@ func (cp *Processor) ListenerNotaryHandlers() []event.NotaryHandlerInfo { h.SetHandler(cp.handleDelete) hh = append(hh, h) - // set eACL - h.SetRequestType(containerEvent.SetEACLNotaryEvent) - h.SetHandler(cp.handleSetEACL) - hh = append(hh, h) - return hh } diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index 9dd3a337b..fc892aafb 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -28,7 +28,6 @@ const ( listMethod = "list" containersOfMethod = "containersOf" eaclMethod = "eACL" - setEACLMethod = "setEACL" deletionInfoMethod = "deletionInfo" startEstimationMethod = "startContainerEstimation" diff --git a/pkg/morph/event/container/eacl.go b/pkg/morph/event/container/eacl.go deleted file mode 100644 index 4168d8842..000000000 --- a/pkg/morph/event/container/eacl.go +++ /dev/null @@ -1,51 +0,0 @@ -package container - -import ( - "github.com/nspcc-dev/neo-go/pkg/network/payload" -) - -// SetEACL represents structure of notification about -// modified eACL table coming from FrostFS Container contract. -type SetEACL struct { - TableValue []byte - SignatureValue []byte - PublicKeyValue []byte - TokenValue []byte - - // For notary notifications only. - // Contains raw transactions of notary request. - NotaryRequestValue *payload.P2PNotaryRequest -} - -// MorphEvent implements Neo:Morph Event interface. -func (SetEACL) MorphEvent() {} - -// Table returns returns eACL table in a binary FrostFS API format. -func (x SetEACL) Table() []byte { - return x.TableValue -} - -// Signature returns signature of the binary table. -func (x SetEACL) Signature() []byte { - return x.SignatureValue -} - -// PublicKey returns public keys of container -// owner in a binary format. -func (x SetEACL) PublicKey() []byte { - return x.PublicKeyValue -} - -// SessionToken returns binary token of the session -// within which the eACL was set. -func (x SetEACL) SessionToken() []byte { - return x.TokenValue -} - -// NotaryRequest returns raw notary request if notification -// was received via notary service. Otherwise, returns nil. -func (x SetEACL) NotaryRequest() *payload.P2PNotaryRequest { - return x.NotaryRequestValue -} - -const expectedItemNumEACL = 4 diff --git a/pkg/morph/event/container/eacl_notary.go b/pkg/morph/event/container/eacl_notary.go deleted file mode 100644 index a4fe7c966..000000000 --- a/pkg/morph/event/container/eacl_notary.go +++ /dev/null @@ -1,75 +0,0 @@ -package container - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/vm/opcode" -) - -func (x *SetEACL) setTable(v []byte) { - if v != nil { - x.TableValue = v - } -} - -func (x *SetEACL) setSignature(v []byte) { - if v != nil { - x.SignatureValue = v - } -} - -func (x *SetEACL) setPublicKey(v []byte) { - if v != nil { - x.PublicKeyValue = v - } -} - -func (x *SetEACL) setToken(v []byte) { - if v != nil { - x.TokenValue = v - } -} - -var setEACLFieldSetters = []func(*SetEACL, []byte){ - // order on stack is reversed - (*SetEACL).setToken, - (*SetEACL).setPublicKey, - (*SetEACL).setSignature, - (*SetEACL).setTable, -} - -const ( - // SetEACLNotaryEvent is method name for container EACL operations - // in `Container` contract. Is used as identificator for notary - // EACL changing requests. - SetEACLNotaryEvent = "setEACL" -) - -// ParseSetEACLNotary from NotaryEvent into container event structure. -func ParseSetEACLNotary(ne event.NotaryEvent) (event.Event, error) { - var ( - ev SetEACL - currentOp opcode.Opcode - ) - - fieldNum := 0 - - for _, op := range ne.Params() { - currentOp = op.Code() - - switch { - case opcode.PUSHDATA1 <= currentOp && currentOp <= opcode.PUSHDATA4: - if fieldNum == expectedItemNumEACL { - return nil, event.UnexpectedArgNumErr(SetEACLNotaryEvent) - } - - setEACLFieldSetters[fieldNum](&ev, op.Param()) - fieldNum++ - default: - return nil, event.UnexpectedOpcode(SetEACLNotaryEvent, op.Code()) - } - } - - ev.NotaryRequestValue = ne.Raw() - - return ev, nil -} diff --git a/pkg/services/container/morph/executor_test.go b/pkg/services/container/morph/executor_test.go index 560c69232..c64310eb3 100644 --- a/pkg/services/container/morph/executor_test.go +++ b/pkg/services/container/morph/executor_test.go @@ -32,10 +32,6 @@ func (m mock) Delete(_ containerCore.RemovalWitness) error { return nil } -func (m mock) PutEACL(_ containerCore.EACL) error { - return nil -} - func TestInvalidToken(t *testing.T) { m := mock{} e := containerSvcMorph.NewExecutor(m, m) From cc5360a57851e27dd51d72cf0ff5fa6ac44aba2f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 8 Oct 2024 16:46:41 +0300 Subject: [PATCH 1433/1943] [#1425] morph/event: Rename eacl_test.go Signed-off-by: Evgenii Stratonikov --- pkg/morph/event/container/{eacl_test.go => util_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/morph/event/container/{eacl_test.go => util_test.go} (100%) diff --git a/pkg/morph/event/container/eacl_test.go b/pkg/morph/event/container/util_test.go similarity index 100% rename from pkg/morph/event/container/eacl_test.go rename to pkg/morph/event/container/util_test.go From 94302235d03ad147c9d42791de39f75aaa7fe7f9 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 9 Oct 2024 10:03:58 +0300 Subject: [PATCH 1434/1943] [#1425] adm: Remove eACL fetching from dump-containers Signed-off-by: Evgenii Stratonikov --- .../modules/morph/container/container.go | 62 +------------------ 1 file changed, 2 insertions(+), 60 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/container/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go index e280bc634..6f08d1655 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -139,13 +139,12 @@ func dumpContainers(cmd *cobra.Command, _ []string) error { func dumpSingleContainer(bw *io.BufBinWriter, ch util.Uint160, inv *invoker.Invoker, id []byte) (*Container, error) { bw.Reset() emit.AppCall(bw.BinWriter, ch, "get", callflag.All, id) - emit.AppCall(bw.BinWriter, ch, "eACL", callflag.All, id) res, err := inv.Run(bw.Bytes()) if err != nil { return nil, fmt.Errorf("can't get container info: %w", err) } - if len(res.Stack) != 2 { - return nil, fmt.Errorf("%w: expected 2 items on stack", errInvalidContainerResponse) + if len(res.Stack) != 1 { + return nil, fmt.Errorf("%w: expected 1 items on stack", errInvalidContainerResponse) } cnt := new(Container) @@ -154,11 +153,6 @@ func dumpSingleContainer(bw *io.BufBinWriter, ch util.Uint160, inv *invoker.Invo return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err) } - ea := new(EACL) - err = ea.FromStackItem(res.Stack[1]) - if err != nil { - return nil, fmt.Errorf("%w: %v", errInvalidContainerResponse, err) - } return cnt, nil } @@ -317,14 +311,6 @@ type Container struct { Token []byte `json:"token"` } -// EACL represents extended ACL struct in contract storage. -type EACL struct { - Value []byte `json:"value"` - Signature []byte `json:"signature"` - PublicKey []byte `json:"public_key"` - Token []byte `json:"token"` -} - // ToStackItem implements stackitem.Convertible. func (c *Container) ToStackItem() (stackitem.Item, error) { return stackitem.NewStruct([]stackitem.Item{ @@ -369,50 +355,6 @@ func (c *Container) FromStackItem(item stackitem.Item) error { return nil } -// ToStackItem implements stackitem.Convertible. -func (c *EACL) ToStackItem() (stackitem.Item, error) { - return stackitem.NewStruct([]stackitem.Item{ - stackitem.NewByteArray(c.Value), - stackitem.NewByteArray(c.Signature), - stackitem.NewByteArray(c.PublicKey), - stackitem.NewByteArray(c.Token), - }), nil -} - -// FromStackItem implements stackitem.Convertible. -func (c *EACL) FromStackItem(item stackitem.Item) error { - arr, ok := item.Value().([]stackitem.Item) - if !ok || len(arr) != 4 { - return errors.New("invalid stack item type") - } - - value, err := arr[0].TryBytes() - if err != nil { - return errors.New("invalid eACL value") - } - - sig, err := arr[1].TryBytes() - if err != nil { - return errors.New("invalid eACL signature") - } - - pub, err := arr[2].TryBytes() - if err != nil { - return errors.New("invalid eACL public key") - } - - tok, err := arr[3].TryBytes() - if err != nil { - return errors.New("invalid eACL token") - } - - c.Value = value - c.Signature = sig - c.PublicKey = pub - c.Token = tok - return nil -} - // getCIDFilterFunc returns filtering function for container IDs. // Raw byte slices are used because it works with structures returned // from contract. From 02bb7159a54a9522ad0bc97d1a5456f5cfc425e4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 9 Oct 2024 10:50:30 +0300 Subject: [PATCH 1435/1943] [#1425] services/tree: Remove eACL processing Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/tree.go | 1 - pkg/services/tree/options.go | 9 -- pkg/services/tree/signature.go | 137 +------------------------ pkg/services/tree/signature_test.go | 151 ++++++++++++++++++---------- 4 files changed, 100 insertions(+), 198 deletions(-) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index d22e510de..192f08471 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -54,7 +54,6 @@ func initTreeService(c *cfg) { cli: c.shared.cnrClient, }), tree.WithFrostfsidSubjectProvider(c.shared.frostfsidClient), - tree.WithEACLSource(c.cfgObject.eaclSource), tree.WithNetmapSource(c.netMapSource), tree.WithPrivateKey(&c.key.PrivateKey), tree.WithLogger(c.log), diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index 6a20fe5cc..1db5607f6 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -33,7 +33,6 @@ type cfg struct { nmSource netmap.Source cnrSource ContainerSource frostfsidSubjectProvider frostfsidcore.SubjectProvider - eaclSource container.EACLSource forest pilorama.Forest // replication-related parameters replicatorChannelCapacity int @@ -65,14 +64,6 @@ func WithFrostfsidSubjectProvider(provider frostfsidcore.SubjectProvider) Option } } -// WithEACLSource sets a eACL table source for a tree service. -// This option is required. -func WithEACLSource(src container.EACLSource) Option { - return func(c *cfg) { - c.eaclSource = src - } -} - // WithNetmapSource sets a netmap source for a tree service. // This option is required. func WithNetmapSource(src netmap.Source) Option { diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 58cab659f..305adf2d7 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -9,10 +9,8 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" @@ -20,7 +18,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "go.uber.org/zap" ) type message interface { @@ -30,16 +27,11 @@ type message interface { SetSignature(*Signature) } -func basicACLErr(op acl.Op) error { - return fmt.Errorf("access to operation %s is denied by basic ACL check", op) -} - func eACLErr(op eacl.Operation, err error) error { return fmt.Errorf("access to operation %s is denied by extended ACL check: %w", op, err) } var ( - errBearerWrongOwner = errors.New("bearer token must be signed by the container owner") errBearerWrongContainer = errors.New("bearer token is created for another container") errBearerSignature = errors.New("invalid bearer token signature") ) @@ -77,56 +69,7 @@ func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, return fmt.Errorf("can't get request role: %w", err) } - basicACL := cnr.Value.BasicACL() - // Basic ACL mask can be unset, if a container operations are performed - // with strict APE checks only. - // - // FIXME(@aarifullin): tree service temporiraly performs APE checks on - // object verbs, because tree verbs have not been introduced yet. - if basicACL == 0x0 { - return s.checkAPE(ctx, bt, cnr, cid, op, role, pubKey) - } - - if !basicACL.IsOpAllowed(op, role) { - return basicACLErr(op) - } - - if !basicACL.Extendable() { - return nil - } - - var useBearer bool - if len(rawBearer) != 0 { - if !basicACL.AllowedBearerRules(op) { - s.log.Debug(logs.TreeBearerPresentedButNotAllowedByACL, - zap.String("cid", cid.EncodeToString()), - zap.Stringer("op", op), - ) - } else { - useBearer = true - } - } - - var tb eacl.Table - signer := req.GetSignature().GetKey() - if useBearer && !bt.Impersonate() { - if !bearer.ResolveIssuer(*bt).Equals(cnr.Value.Owner()) { - return eACLErr(eaclOp, errBearerWrongOwner) - } - tb = bt.EACLTable() - } else { - tbCore, err := s.eaclSource.GetEACL(cid) - if err != nil { - return handleGetEACLError(err) - } - tb = *tbCore.Value - - if useBearer && bt.Impersonate() { - signer = bt.SigningKeyBytes() - } - } - - return checkEACL(tb, signer, eACLRole(role), eaclOp) + return s.checkAPE(ctx, bt, cnr, cid, op, role, pubKey) } // Returns true iff the operation is read-only and request was signed @@ -168,14 +111,6 @@ func parseBearer(rawBearer []byte, cid cidSDK.ID, eaclOp eacl.Operation) (*beare return bt, nil } -func handleGetEACLError(err error) error { - if client.IsErrEACLNotFound(err) { - return nil - } - - return fmt.Errorf("get eACL table: %w", err) -} - func verifyMessage(m message) error { binBody, err := m.ReadSignedData(nil) if err != nil { @@ -260,73 +195,3 @@ func eACLOp(op acl.Op) eacl.Operation { panic(fmt.Sprintf("unexpected tree service ACL operation: %s", op)) } } - -func eACLRole(role acl.Role) eacl.Role { - switch role { - case acl.RoleOwner: - return eacl.RoleUser - case acl.RoleOthers: - return eacl.RoleOthers - default: - panic(fmt.Sprintf("unexpected tree service ACL role: %s", role)) - } -} - -var ( - errDENY = errors.New("DENY eACL rule") - errNoAllowRules = errors.New("not found allowing rules for the request") -) - -// checkEACL searches for the eACL rules that could be applied to the request -// (a tuple of a signer key, his FrostFS role and a request operation). -// It does not filter the request by the filters of the eACL table since tree -// requests do not contain any "object" information that could be filtered and, -// therefore, filtering leads to unexpected results. -// The code was copied with the minor updates from the SDK repo: -// https://github.com/nspcc-dev/frostfs-sdk-go/blob/43a57d42dd50dc60465bfd3482f7f12bcfcf3411/eacl/validator.go#L28. -func checkEACL(tb eacl.Table, signer []byte, role eacl.Role, op eacl.Operation) error { - for _, record := range tb.Records() { - // check type of operation - if record.Operation() != op { - continue - } - - // check target - if !targetMatches(record, role, signer) { - continue - } - - switch a := record.Action(); a { - case eacl.ActionAllow: - return nil - case eacl.ActionDeny: - return eACLErr(op, errDENY) - default: - return eACLErr(op, fmt.Errorf("unexpected action: %s", a)) - } - } - - return eACLErr(op, errNoAllowRules) -} - -func targetMatches(rec eacl.Record, role eacl.Role, signer []byte) bool { - for _, target := range rec.Targets() { - // check public key match - if pubs := target.BinaryKeys(); len(pubs) != 0 { - for _, key := range pubs { - if bytes.Equal(key, signer) { - return true - } - } - - continue - } - - // check target group match - if role == target.Role() { - return true - } - } - - return false -} diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 3c3ebfe89..939ff170d 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -4,22 +4,30 @@ import ( "context" "crypto/ecdsa" "crypto/sha256" + "encoding/hex" "errors" "testing" aclV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" + "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + checkercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" + "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" "github.com/stretchr/testify/require" ) @@ -27,6 +35,34 @@ type dummyNetmapSource struct { netmap.Source } +type dummySubjectProvider struct { + subjects map[util.Uint160]client.SubjectExtended +} + +func (s dummySubjectProvider) GetSubject(addr util.Uint160) (*client.Subject, error) { + res := s.subjects[addr] + return &client.Subject{ + PrimaryKey: res.PrimaryKey, + AdditionalKeys: res.AdditionalKeys, + Namespace: res.Namespace, + Name: res.Name, + KV: res.KV, + }, nil +} + +func (s dummySubjectProvider) GetSubjectExtended(addr util.Uint160) (*client.SubjectExtended, error) { + res := s.subjects[addr] + return &res, nil +} + +type dummyEpochSource struct { + epoch uint64 +} + +func (s dummyEpochSource) CurrentEpoch() uint64 { + return s.epoch +} + type dummyContainerSource map[string]*containercore.Container func (s dummyContainerSource) List() ([]cid.ID, error) { @@ -57,16 +93,6 @@ func (s dummyContainerSource) DeletionInfo(id cid.ID) (*containercore.DelInfo, e return &containercore.DelInfo{}, nil } -type dummyEACLSource map[string]*containercore.EACL - -func (s dummyEACLSource) GetEACL(id cid.ID) (*containercore.EACL, error) { - cntEACL, ok := s[id.String()] - if !ok { - return nil, errors.New("container not found") - } - return cntEACL, nil -} - func testContainer(owner user.ID) container.Container { var r netmapSDK.ReplicaDescriptor r.SetNumberOfObjects(1) @@ -81,6 +107,8 @@ func testContainer(owner user.ID) container.Container { return cnt } +const currentEpoch = 123 + func TestMessageSign(t *testing.T) { privs := make([]*keys.PrivateKey, 4) for i := range privs { @@ -99,6 +127,15 @@ func TestMessageSign(t *testing.T) { Value: testContainer(ownerID), } + e := inmemory.NewInMemoryLocalOverrides() + e.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.Target{ + Type: engine.Container, + Name: cid1.EncodeToString(), + }, testChain(privs[0].PublicKey(), privs[1].PublicKey())) + frostfsidProvider := dummySubjectProvider{ + subjects: make(map[util.Uint160]client.SubjectExtended), + } + s := &Service{ cfg: cfg{ log: test.NewLogger(t), @@ -107,12 +144,10 @@ func TestMessageSign(t *testing.T) { cnrSource: dummyContainerSource{ cid1.String(): cnr, }, - eaclSource: dummyEACLSource{ - cid1.String(): &containercore.EACL{ - Value: testTable(cid1, privs[0].PublicKey(), privs[1].PublicKey()), - }, - }, + frostfsidSubjectProvider: frostfsidProvider, + state: dummyEpochSource{epoch: currentEpoch}, }, + apeChecker: checkercore.New(e.LocalStorage(), e.MorphRuleChainStorage(), frostfsidProvider, dummyEpochSource{}), } rawCID1 := make([]byte, sha256.Size) @@ -235,46 +270,58 @@ func TestMessageSign(t *testing.T) { func testBearerToken(cid cid.ID, forPutGet, forGet *keys.PublicKey) bearer.Token { var b bearer.Token - b.SetEACLTable(*testTable(cid, forPutGet, forGet)) + b.SetExp(currentEpoch + 1) + b.SetAPEOverride(bearer.APEOverride{ + Target: ape.ChainTarget{ + TargetType: ape.TargetTypeContainer, + Name: cid.EncodeToString(), + }, + Chains: []ape.Chain{{Raw: testChain(forPutGet, forGet).Bytes()}}, + }) return b } -func testTable(cid cid.ID, forPutGet, forGet *keys.PublicKey) *eaclSDK.Table { - tgtGet := eaclSDK.NewTarget() - tgtGet.SetRole(eaclSDK.RoleUnknown) - tgtGet.SetBinaryKeys([][]byte{forPutGet.Bytes(), forGet.Bytes()}) - - rGet := eaclSDK.NewRecord() - rGet.SetAction(eaclSDK.ActionAllow) - rGet.SetOperation(eaclSDK.OperationGet) - rGet.SetTargets(*tgtGet) - - tgtPut := eaclSDK.NewTarget() - tgtPut.SetRole(eaclSDK.RoleUnknown) - tgtPut.SetBinaryKeys([][]byte{forPutGet.Bytes()}) - - rPut := eaclSDK.NewRecord() - rPut.SetAction(eaclSDK.ActionAllow) - rPut.SetOperation(eaclSDK.OperationPut) - rPut.SetTargets(*tgtPut) - - tb := eaclSDK.NewTable() - tb.AddRecord(rGet) - tb.AddRecord(rPut) - - tgt := eaclSDK.NewTarget() - tgt.SetRole(eaclSDK.RoleOthers) - - for _, op := range []eaclSDK.Operation{eaclSDK.OperationGet, eaclSDK.OperationPut} { - r := eaclSDK.NewRecord() - r.SetAction(eaclSDK.ActionDeny) - r.SetTargets(*tgt) - r.SetOperation(op) - tb.AddRecord(r) +func testChain(forPutGet, forGet *keys.PublicKey) *chain.Chain { + ruleGet := chain.Rule{ + Status: chain.Allow, + Resources: chain.Resources{Names: []string{native.ResourceFormatAllObjects}}, + Actions: chain.Actions{Names: []string{native.MethodGetObject}}, + Any: true, + Condition: []chain.Condition{ + { + Op: chain.CondStringEquals, + Kind: chain.KindRequest, + Key: native.PropertyKeyActorPublicKey, + Value: hex.EncodeToString(forPutGet.Bytes()), + }, + { + Op: chain.CondStringEquals, + Kind: chain.KindRequest, + Key: native.PropertyKeyActorPublicKey, + Value: hex.EncodeToString(forGet.Bytes()), + }, + }, + } + rulePut := chain.Rule{ + Status: chain.Allow, + Resources: chain.Resources{Names: []string{native.ResourceFormatAllObjects}}, + Actions: chain.Actions{Names: []string{native.MethodPutObject}}, + Any: true, + Condition: []chain.Condition{ + { + Op: chain.CondStringEquals, + Kind: chain.KindRequest, + Key: native.PropertyKeyActorPublicKey, + Value: hex.EncodeToString(forPutGet.Bytes()), + }, + }, } - tb.SetCID(cid) - - return tb + return &chain.Chain{ + Rules: []chain.Rule{ + ruleGet, + rulePut, + }, + } } From 11347602719a764179a74382076de4430936d7ad Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 9 Oct 2024 10:55:48 +0300 Subject: [PATCH 1436/1943] [#1425] services/tree: Remove eACL mentions from bearer token parsing errors Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/signature.go | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 305adf2d7..20a629fcc 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -15,7 +15,6 @@ import ( cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) @@ -27,10 +26,6 @@ type message interface { SetSignature(*Signature) } -func eACLErr(op eacl.Operation, err error) error { - return fmt.Errorf("access to operation %s is denied by extended ACL check: %w", op, err) -} - var ( errBearerWrongContainer = errors.New("bearer token is created for another container") errBearerSignature = errors.New("invalid bearer token signature") @@ -57,11 +52,9 @@ func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, return fmt.Errorf("can't get container %s: %w", cid, err) } - eaclOp := eACLOp(op) - - bt, err := parseBearer(rawBearer, cid, eaclOp) + bt, err := parseBearer(rawBearer, cid) if err != nil { - return err + return fmt.Errorf("access to operation %s is denied: %w", op, err) } role, pubKey, err := roleAndPubKeyFromReq(cnr, req, bt) @@ -93,20 +86,20 @@ func (s *Service) isAuthorized(req message, op acl.Op) (bool, error) { return false, nil } -func parseBearer(rawBearer []byte, cid cidSDK.ID, eaclOp eacl.Operation) (*bearer.Token, error) { +func parseBearer(rawBearer []byte, cid cidSDK.ID) (*bearer.Token, error) { if len(rawBearer) == 0 { return nil, nil } bt := new(bearer.Token) if err := bt.Unmarshal(rawBearer); err != nil { - return nil, eACLErr(eaclOp, fmt.Errorf("invalid bearer token: %w", err)) + return nil, fmt.Errorf("invalid bearer token: %w", err) } if !bt.AssertContainer(cid) { - return nil, eACLErr(eaclOp, errBearerWrongContainer) + return nil, errBearerWrongContainer } if !bt.VerifySignature() { - return nil, eACLErr(eaclOp, errBearerSignature) + return nil, errBearerSignature } return bt, nil } @@ -184,14 +177,3 @@ func roleAndPubKeyFromReq(cnr *core.Container, req message, bt *bearer.Token) (a return role, pub, nil } - -func eACLOp(op acl.Op) eacl.Operation { - switch op { - case acl.OpObjectGet: - return eacl.OperationGet - case acl.OpObjectPut: - return eacl.OperationPut - default: - panic(fmt.Sprintf("unexpected tree service ACL operation: %s", op)) - } -} From dfb00083d07499a0e3d89076cc3f08729c00cb71 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 10 Oct 2024 14:57:39 +0300 Subject: [PATCH 1437/1943] [#1426] go.mod: Update sdk-go Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 91cc55a36..9a64f0e81 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241007135805-4c310ae1c7fa + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241010110344-99c5c5836509 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 diff --git a/go.sum b/go.sum index 728592ea5..777d9b3ab 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241007135805-4c310ae1c7fa h1:Jr8hXNNFECLhC7S45HuyQms4U/gim1xILoU3g4ZZnHg= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241007135805-4c310ae1c7fa/go.mod h1:jmb7yxzZota9jWbC10p+7YR+6wwJPBj7J/Fl5VDkXys= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241010110344-99c5c5836509 h1:5gtEq4bjVgAbTOrbEquspyM3s+qsMtkpGC5m9FtfImk= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241010110344-99c5c5836509/go.mod h1:jmb7yxzZota9jWbC10p+7YR+6wwJPBj7J/Fl5VDkXys= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= From 5992ee901ae574d536a82bd7f1504852e17fb086 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 11 Oct 2024 11:33:36 +0300 Subject: [PATCH 1438/1943] [#1411] go.mod: Bump frostfs-contract to v0.20.0 Signed-off-by: Alexander Chuprov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9a64f0e81..1468c12b2 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241007120543-29c522d5d8a3 - git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c + git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.0 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 diff --git a/go.sum b/go.sum index 777d9b3ab..5ce81807a 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241007120543-29c522d5d8a3 h1:6QXNnfBgYx81UZsBdpPnQY+ZMSKGFbFc29wV7DJ/UG4= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241007120543-29c522d5d8a3/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c h1:9JmXxWcqWaOXModBwtj/WlPYOfSk2jRSDoRsDRYvgnA= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240909072709-3e221b973a3c/go.mod h1:DVb1AqP323wiEwCtlfoJJTERF4Xji2L39X6T8kv26RE= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.0 h1:8Z5iPhieCrbcdhxBuY/Bajh6V5fki7Whh0b4S2zYJYU= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.0/go.mod h1:Y2Xorxc8SBO4phoek7n3XxaPZz5rIrFgDsU4TOjmlGA= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d h1:uJ/wvuMdepbkaV8XMS5uN9B0FQWMep0CttSuDZiDhq0= From 42bf03e5cc04f2b6d67465608b842693171344ba Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 11 Oct 2024 11:33:56 +0300 Subject: [PATCH 1439/1943] [#1411] adm/nns: Add 'delRecord' Signed-off-by: Alexander Chuprov --- .../internal/modules/morph/nns/record.go | 29 +++++++++++++++++++ .../internal/modules/morph/nns/root.go | 10 +++++++ 2 files changed, 39 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/record.go b/cmd/frostfs-adm/internal/modules/morph/nns/record.go index 0e217eb61..66bb1b94f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/record.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/record.go @@ -47,6 +47,19 @@ func initDelRecordsCmd() { _ = cobra.MarkFlagRequired(delRecordsCmd.Flags(), nnsRecordTypeFlag) } +func initDelRecordCmd() { + Cmd.AddCommand(delRecordCmd) + delRecordCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + delRecordCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + delRecordCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) + delRecordCmd.Flags().String(nnsRecordTypeFlag, "", nnsRecordTypeFlagDesc) + delRecordCmd.Flags().String(nnsRecordDataFlag, "", nnsRecordDataFlagDesc) + + _ = cobra.MarkFlagRequired(delRecordCmd.Flags(), nnsNameFlag) + _ = cobra.MarkFlagRequired(delRecordCmd.Flags(), nnsRecordTypeFlag) + _ = cobra.MarkFlagRequired(delRecordCmd.Flags(), nnsRecordDataFlag) +} + func addRecord(cmd *cobra.Command, _ []string) { c, actor, _ := getRPCClient(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) @@ -115,6 +128,22 @@ func delRecords(cmd *cobra.Command, _ []string) { cmd.Println("Records removed successfully") } +func delRecord(cmd *cobra.Command, _ []string) { + c, actor, _ := getRPCClient(cmd) + name, _ := cmd.Flags().GetString(nnsNameFlag) + data, _ := cmd.Flags().GetString(nnsRecordDataFlag) + recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) + typ, err := getRecordType(recordType) + commonCmd.ExitOnErr(cmd, "unable to parse record type: %w", err) + h, vub, err := c.DeleteRecord(name, typ, data) + commonCmd.ExitOnErr(cmd, "unable to delete record: %w", err) + + cmd.Println("Waiting for transaction to persist...") + _, err = actor.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "delete records error: %w", err) + cmd.Println("Record removed successfully") +} + func getRecordType(recordType string) (*big.Int, error) { switch strings.ToUpper(recordType) { case "A": diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/root.go b/cmd/frostfs-adm/internal/modules/morph/nns/root.go index 56774c292..9bdeaccd9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/root.go @@ -95,6 +95,15 @@ var ( }, Run: delRecords, } + delRecordCmd = &cobra.Command{ + Use: "delete-record", + Short: "Removes domain record with the specified type and data", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + }, + Run: delRecord, + } ) func init() { @@ -106,4 +115,5 @@ func init() { initAddRecordCmd() initGetRecordsCmd() initDelRecordsCmd() + initDelRecordCmd() } From acd6eb18151d7e2fec413d97e273b13076a9b4bb Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 11 Oct 2024 15:40:01 +0300 Subject: [PATCH 1440/1943] [#1427] object: Fix `Put` for `EC` object when node unavailable There might be situation when context canceled earlier than traverser move to another part of the nodes. To avoid this, need to wait for the result from concurrent put at each traverser iteration. Signed-off-by: Anton Nikiforov --- pkg/services/object/common/writer/ec.go | 20 +- pkg/services/object/common/writer/ec_test.go | 191 +++++++++++++++++++ 2 files changed, 205 insertions(+), 6 deletions(-) create mode 100644 pkg/services/object/common/writer/ec_test.go diff --git a/pkg/services/object/common/writer/ec.go b/pkg/services/object/common/writer/ec.go index 6b6a14cc0..dffe52a6d 100644 --- a/pkg/services/object/common/writer/ec.go +++ b/pkg/services/object/common/writer/ec.go @@ -197,14 +197,15 @@ func (e *ECWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) er if err != nil { return err } + partsProcessed := make([]atomic.Bool, len(parts)) objID, _ := obj.ID() t, err := placement.NewTraverser(append(e.PlacementOpts, placement.ForObject(objID))...) if err != nil { return err } - eg, egCtx := errgroup.WithContext(ctx) for { + eg, egCtx := errgroup.WithContext(ctx) nodes := t.Next() if len(nodes) == 0 { break @@ -216,13 +217,20 @@ func (e *ECWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) er } for idx := range parts { - eg.Go(func() error { - return e.writePart(egCtx, parts[idx], idx, nodes, visited) - }) - t.SubmitSuccess() + if !partsProcessed[idx].Load() { + eg.Go(func() error { + err := e.writePart(egCtx, parts[idx], idx, nodes, visited) + if err == nil { + partsProcessed[idx].Store(true) + t.SubmitSuccess() + } + return err + }) + } } + err = eg.Wait() } - if err := eg.Wait(); err != nil { + if err != nil { return errIncompletePut{ singleErr: err, } diff --git a/pkg/services/object/common/writer/ec_test.go b/pkg/services/object/common/writer/ec_test.go new file mode 100644 index 000000000..32863d678 --- /dev/null +++ b/pkg/services/object/common/writer/ec_test.go @@ -0,0 +1,191 @@ +package writer + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "errors" + "fmt" + "strconv" + "testing" + + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + apiclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "git.frostfs.info/TrueCloudLab/tzhash/tz" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/panjf2000/ants/v2" + "github.com/stretchr/testify/require" +) + +type testPlacementBuilder struct { + vectors [][]netmap.NodeInfo +} + +func (p *testPlacementBuilder) BuildPlacement(_ cid.ID, _ *oid.ID, _ netmap.PlacementPolicy) ( + [][]netmap.NodeInfo, error, +) { + arr := make([]netmap.NodeInfo, len(p.vectors[0])) + copy(arr, p.vectors[0]) + return [][]netmap.NodeInfo{arr}, nil +} + +type nmKeys struct{} + +func (nmKeys) IsLocalKey(_ []byte) bool { + return false +} + +type clientConstructor struct { + vectors [][]netmap.NodeInfo +} + +func (c clientConstructor) Get(info client.NodeInfo) (client.MultiAddressClient, error) { + if bytes.Equal(info.PublicKey(), c.vectors[0][0].PublicKey()) || + bytes.Equal(info.PublicKey(), c.vectors[0][1].PublicKey()) { + return multiAddressClient{err: errors.New("node unavailable")}, nil + } + return multiAddressClient{}, nil +} + +type multiAddressClient struct { + client.MultiAddressClient + err error +} + +func (c multiAddressClient) ObjectPutSingle(_ context.Context, _ apiclient.PrmObjectPutSingle) (*apiclient.ResObjectPutSingle, error) { + if c.err != nil { + return nil, c.err + } + return &apiclient.ResObjectPutSingle{}, nil +} + +func (c multiAddressClient) ReportError(error) { +} + +func (multiAddressClient) RawForAddress(context.Context, network.Address, func(cli *rawclient.Client) error) error { + return nil +} + +func TestECWriter(t *testing.T) { + // Create container with policy EC 1.1 + cnr := container.Container{} + p1 := netmap.PlacementPolicy{} + p1.SetContainerBackupFactor(1) + x1 := netmap.ReplicaDescriptor{} + x1.SetECDataCount(1) + x1.SetECParityCount(1) + p1.AddReplicas(x1) + cnr.SetPlacementPolicy(p1) + cnr.SetAttribute("cnr", "cnr1") + + cid := cidtest.ID() + + // Create 4 nodes, 2 nodes for chunks, + // 2 nodes for the case when the first two will fail. + ns, _ := testNodeMatrix(t, []int{4}) + + data := make([]byte, 100) + _, _ = rand.Read(data) + ver := version.Current() + + var csum checksum.Checksum + csum.SetSHA256(sha256.Sum256(data)) + + var csumTZ checksum.Checksum + csumTZ.SetTillichZemor(tz.Sum(csum.Value())) + + obj := objectSDK.New() + obj.SetID(oidtest.ID()) + obj.SetOwnerID(usertest.ID()) + obj.SetContainerID(cid) + obj.SetVersion(&ver) + obj.SetPayload(data) + obj.SetPayloadSize(uint64(len(data))) + obj.SetPayloadChecksum(csum) + obj.SetPayloadHomomorphicHash(csumTZ) + + // Builder return nodes without sort by hrw + builder := &testPlacementBuilder{ + vectors: ns, + } + + ownerKey, err := keys.NewPrivateKey() + require.NoError(t, err) + + pool, err := ants.NewPool(4, ants.WithNonblocking(true)) + require.NoError(t, err) + + log, err := logger.NewLogger(nil) + require.NoError(t, err) + + var n nmKeys + ecw := ECWriter{ + Config: &Config{ + NetmapKeys: n, + RemotePool: pool, + Logger: log, + ClientConstructor: clientConstructor{vectors: ns}, + }, + PlacementOpts: append( + []placement.Option{placement.UseBuilder(builder), placement.ForContainer(cnr)}, + placement.WithCopyNumbers(nil)), // copies number ignored for EC + Container: cnr, + Key: &ownerKey.PrivateKey, + Relay: nil, + ObjectMetaValid: true, + } + + err = ecw.WriteObject(context.Background(), obj) + require.NoError(t, err) +} + +func testNodeMatrix(t testing.TB, dim []int) ([][]netmap.NodeInfo, [][]string) { + mNodes := make([][]netmap.NodeInfo, len(dim)) + mAddr := make([][]string, len(dim)) + + for i := range dim { + ns := make([]netmap.NodeInfo, dim[i]) + as := make([]string, dim[i]) + + for j := range dim[i] { + a := fmt.Sprintf("/ip4/192.168.0.%s/tcp/%s", + strconv.Itoa(i), + strconv.Itoa(60000+j), + ) + + var ni netmap.NodeInfo + ni.SetNetworkEndpoints(a) + ni.SetPublicKey([]byte(a)) + + var na network.AddressGroup + + err := na.FromIterator(netmapcore.Node(ni)) + require.NoError(t, err) + + as[j] = network.StringifyGroup(na) + + ns[j] = ni + } + + mNodes[i] = ns + mAddr[i] = as + } + + return mNodes, mAddr +} From d2a59b2de8572952df34e2b66c3bf51d03ce13d9 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 14 Oct 2024 15:51:37 +0300 Subject: [PATCH 1441/1943] [#1429] lens/explorer: Fix locked object records display text Display texts for a locked object and a list of it lockers were mistakenly swapped. Signed-off-by: Aleksey Savchuk --- cmd/frostfs-lens/internal/schema/metabase/records/string.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/string.go b/cmd/frostfs-lens/internal/schema/metabase/records/string.go index a6c70d537..ec0ab8e1a 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/records/string.go +++ b/cmd/frostfs-lens/internal/schema/metabase/records/string.go @@ -38,7 +38,7 @@ func (r *ContainerVolumeRecord) String() string { func (r *LockedRecord) String() string { return fmt.Sprintf( - "Locker OID %s %c Locked [%d]OID {...}", + "Object OID %s %c Lockers [%d]OID {...}", common.FormatSimple(fmt.Sprintf("%-44s", r.id), tcell.ColorAqua), tview.Borders.Vertical, len(r.ids), From 714ff784fa460767e82527b71fd520932b0256ed Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Mon, 14 Oct 2024 17:31:26 +0300 Subject: [PATCH 1442/1943] [#1431] objsvc: Use specific values in message about address mismatch This makes troubleshooting failed operations much easier Signed-off-by: Vitaliy Potyarkin --- pkg/services/object/common/target/target.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/common/target/target.go b/pkg/services/object/common/target/target.go index a2d6b4d39..9e0f49297 100644 --- a/pkg/services/object/common/target/target.go +++ b/pkg/services/object/common/target/target.go @@ -86,7 +86,7 @@ func newTrustedTarget(prm *objectwriter.Params) (transformer.ChunkedObjectWriter user.IDFromKey(&ownerSession, key.PublicKey) if !ownerObj.Equals(ownerSession) { - return nil, errors.New("session token is missing but object owner id is different from the default key") + return nil, fmt.Errorf("session token is missing but object owner id (%s) is different from the default key (%s)", ownerObj, ownerSession) } } else { if !ownerObj.Equals(sessionInfo.Owner) { From 3012286452e8b2bb04a6ad9b70e364b00b29919f Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 14 Oct 2024 17:10:03 +0300 Subject: [PATCH 1443/1943] [#1431] metabase: Fix unreachable code Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/metabase/delete.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 3add1f268..b5ac22017 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -453,7 +453,7 @@ func delFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { func hasAnyItem(b *bbolt.Bucket) bool { var hasAnyItem bool c := b.Cursor() - for k, _ := c.First(); k != nil; k, _ = c.Next() { + for k, _ := c.First(); k != nil; { hasAnyItem = true break } From d53732f663ce46ff29196782b5d836a37c1f6c7d Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 14 Oct 2024 17:17:00 +0300 Subject: [PATCH 1444/1943] [#1431] engine: Delete always false condition Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/engine/evacuate.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index c1b9276f3..940e30323 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -56,9 +56,6 @@ func (s EvacuateScope) String() string { var sb strings.Builder first := true if s&EvacuateScopeObjects == EvacuateScopeObjects { - if !first { - sb.WriteString(";") - } sb.WriteString("objects") first = false } From 63466d71b22d8b23eff4d0fc1eea5a16d36a138b Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 14 Oct 2024 17:20:14 +0300 Subject: [PATCH 1445/1943] [#1431] engine: Delete unused constants Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/modules/ape_manager/add_chain.go | 9 ++++----- internal/logs/logs.go | 14 -------------- pkg/morph/client/container/client.go | 1 - 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-cli/modules/ape_manager/add_chain.go b/cmd/frostfs-cli/modules/ape_manager/add_chain.go index c6622da25..a85f3c93e 100644 --- a/cmd/frostfs-cli/modules/ape_manager/add_chain.go +++ b/cmd/frostfs-cli/modules/ape_manager/add_chain.go @@ -31,11 +31,10 @@ const ( ) const ( - defaultNamespace = "" - namespaceTarget = "namespace" - containerTarget = "container" - userTarget = "user" - groupTarget = "group" + namespaceTarget = "namespace" + containerTarget = "container" + userTarget = "user" + groupTarget = "group" ) var errUnknownTargetType = errors.New("unknown target type") diff --git a/internal/logs/logs.go b/internal/logs/logs.go index ca783a39d..b4bc31b0c 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -14,8 +14,6 @@ const ( InterruptPlacementIterationByContext = "interrupt placement iteration by context" Notification = "notification" - - SkipDeprecatedNotification = "skip deprecated notification" ) const ( @@ -41,8 +39,6 @@ const ( InnerringCantUpdatePersistentState = "can't update persistent state" InnerringCloserError = "closer error" InnerringReadConfigFromBlockchain = "read config from blockchain" - NotificatorNotificatorStartProcessingObjectNotifications = "notificator: start processing object notifications" - NotificatorNotificatorProcessingObjectNotification = "notificator: processing object notification" PolicerCouldNotGetContainer = "could not get container" PolicerCouldNotConfirmContainerRemoval = "could not confirm container removal" PolicerCouldNotInhumeObjectWithMissingContainer = "could not inhume object with missing container" @@ -61,7 +57,6 @@ const ( ReplicatorCouldNotReplicateObject = "could not replicate object" ReplicatorObjectSuccessfullyReplicated = "object successfully replicated" TreeRedirectingTreeServiceQuery = "redirecting tree service query" - TreeBearerPresentedButNotAllowedByACL = "bearer presented but not allowed by ACL" TreeCouldNotGetLastSynchronizedHeightForATree = "could not get last synchronized height for a tree" TreeCouldNotUpdateLastSynchronizedHeightForATree = "could not update last synchronized height for a tree" TreeSynchronizeTree = "synchronize tree" @@ -107,7 +102,6 @@ const ( GetUnableToGetAllPartsECObject = "unable to get all parts, continue to reconstruct with existed" GetUnableToGetPartECObject = "unable to get part of the erasure-encoded object" GetUnableToHeadPartECObject = "unable to head part of the erasure-encoded object" - GetUnableToGetECObjectContainer = "unable to get container for erasure-coded object" GetUnableToHeadPartsECObject = "unable to head parts of the erasure-encoded object" GetAssemblingSplittedObjectCompleted = "assembling splitted object completed" GetAssemblingECObjectCompleted = "assembling erasure-coded object completed" @@ -271,9 +265,7 @@ const ( ShardFailureToMarkLockersAsGarbage = "failure to mark lockers as garbage" ShardFailureToGetExpiredUnlockedObjects = "failure to get expired unlocked objects" ShardCouldNotMarkObjectToDeleteInMetabase = "could not mark object to delete in metabase" - WritecacheTriedToFlushItemsFromWritecache = "tried to flush items from write-cache" WritecacheWaitingForChannelsToFlush = "waiting for channels to flush" - WritecacheCantRemoveObjectsFromTheDatabase = "can't remove objects from the database" WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" BlobovniczatreeCouldNotGetObjectFromLevel = "could not get object from level" BlobovniczatreeCouldNotCloseBlobovnicza = "could not close Blobovnicza" @@ -413,11 +405,6 @@ const ( FrostFSNodeReadNewlyCreatedContainerAfterTheNotification = "read newly created container after the notification" FrostFSNodeContainerCreationEventsReceipt = "container creation event's receipt" FrostFSNodeContainerRemovalEventsReceipt = "container removal event's receipt" - FrostFSNodeNotificatorCouldNotListContainers = "notificator: could not list containers" - FrostFSNodeNotificatorCouldNotSelectObjectsFromContainer = "notificator: could not select objects from container" - FrostFSNodeNotificatorCouldNotProcessObject = "notificator: could not process object" - FrostFSNodeNotificatorFinishedProcessingObjectNotifications = "notificator: finished processing object notifications" - FrostFSNodeCouldNotWriteObjectNotification = "could not write object notification" FrostFSNodeCouldNotGetMaxObjectSizeValue = "could not get max object size value" FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage = "could not inhume mark redundant copy as garbage" FrostFSNodeFailedInitTracing = "failed init tracing" @@ -461,7 +448,6 @@ const ( FSTreeCantUnmarshalObject = "can't unmarshal an object" FSTreeCantFushObjectBlobstor = "can't flush an object to blobstor" FSTreeCantUpdateID = "can't update object storage ID" - FSTreeCantDecodeDBObjectAddress = "can't decode object address from the DB" PutSingleRedirectFailure = "failed to redirect PutSingle request" StorageIDRetrievalFailure = "can't get storage ID from metabase" ObjectRemovalFailureBlobStor = "can't remove object from blobStor" diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index fc892aafb..f735a5ff7 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -33,7 +33,6 @@ const ( startEstimationMethod = "startContainerEstimation" stopEstimationMethod = "stopContainerEstimation" - putSizeMethod = "putContainerSize" listSizesMethod = "listContainerSizes" getSizeMethod = "getContainerSize" From 00b1cecfb7486aac93e8806caa6563fe75eabc1b Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 14 Oct 2024 17:25:00 +0300 Subject: [PATCH 1446/1943] [#1431] obj_storage/shard: Fix visibility of 'newMetricStore' Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/shard/control_test.go | 4 ++-- pkg/local_object_storage/shard/metrics_test.go | 4 ++-- pkg/local_object_storage/shard/reload_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 44fee1636..6b9eaa550 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -126,7 +126,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) { }), } - mm := NewMetricStore() + mm := newMetricStore() sh := New( WithID(NewIDFromBytes([]byte{})), @@ -190,7 +190,7 @@ func TestRefillMetabase(t *testing.T) { }), } - mm := NewMetricStore() + mm := newMetricStore() sh := New( WithID(NewIDFromBytes([]byte{})), diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 56622326a..cec5a12ad 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -35,7 +35,7 @@ type metricsStore struct { refillStatus string } -func NewMetricStore() *metricsStore { +func newMetricStore() *metricsStore { return &metricsStore{ objCounters: map[string]uint64{ "phy": 0, @@ -404,7 +404,7 @@ func shardWithMetrics(t *testing.T, path string) (*Shard, *metricsStore) { }), } - mm := NewMetricStore() + mm := newMetricStore() sh := New( WithID(NewIDFromBytes([]byte{})), diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 7dacbfa6c..7dd7189bb 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -51,7 +51,7 @@ func TestShardReload(t *testing.T) { WithMetaBaseOptions(metaOpts...), WithPiloramaOptions( pilorama.WithPath(filepath.Join(p, "pilorama"))), - WithMetricsWriter(NewMetricStore()), + WithMetricsWriter(newMetricStore()), } sh := New(opts...) From f6582081a4ee67e97773f655b8f18148946c5a0c Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 14 Oct 2024 17:31:15 +0300 Subject: [PATCH 1447/1943] [#1431] obj_storage/metabase: Delete unused variable Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/metabase/put.go | 5 ++--- pkg/services/object/remote_reader.go | 3 --- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 0c14196b7..b329e8032 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -59,9 +59,8 @@ func (p *PutPrm) SetIndexAttributes(v bool) { } var ( - ErrUnknownObjectType = errors.New("unknown object type") - ErrIncorrectSplitInfoUpdate = errors.New("updating split info on object without it") - ErrIncorrectRootObject = errors.New("invalid root object") + ErrUnknownObjectType = errors.New("unknown object type") + ErrIncorrectRootObject = errors.New("invalid root object") ) // Put saves object header in metabase. Object payload expected to be cut. diff --git a/pkg/services/object/remote_reader.go b/pkg/services/object/remote_reader.go index 18b6107cf..bc6ffd160 100644 --- a/pkg/services/object/remote_reader.go +++ b/pkg/services/object/remote_reader.go @@ -2,7 +2,6 @@ package object import ( "context" - "errors" "fmt" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" @@ -35,8 +34,6 @@ type RemoteRequestPrm struct { const remoteOpTTL = 1 -var ErrNotFound = errors.New("object header not found") - // NewRemoteReader creates, initializes and returns new RemoteHeader instance. func NewRemoteReader(keyStorage *util.KeyStorage, cache ClientConstructor) *RemoteReader { return &RemoteReader{ From d83879d4b859f016a9bfef808b19324ce593814e Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 14 Oct 2024 18:05:55 +0300 Subject: [PATCH 1448/1943] [#1431] node: Fix comment format Signed-off-by: Alexander Chuprov --- cmd/frostfs-node/config/profiler/config.go | 2 +- cmd/frostfs-node/object.go | 2 +- pkg/core/netmap/keys.go | 2 +- .../netmap/nodevalidation/locode/deps.go | 16 ++++++++-------- pkg/innerring/processors/netmap/processor.go | 2 +- pkg/local_object_storage/shard/metrics.go | 2 +- pkg/morph/client/actor.go | 2 +- pkg/morph/event/container/put_notary.go | 2 +- pkg/network/group.go | 4 ++-- pkg/services/control/ir/server/deps.go | 2 +- pkg/services/control/rpc.go | 2 +- pkg/services/control/server/server.go | 4 ++-- pkg/services/netmap/executor.go | 4 ++-- pkg/services/object/common/writer/distributed.go | 2 +- pkg/services/object/delete/service.go | 4 ++-- pkg/services/object/patch/service.go | 2 +- pkg/services/policer/option.go | 2 +- pkg/util/rand/rand.go | 2 +- pkg/util/sdnotify/sdnotify.go | 2 +- 19 files changed, 30 insertions(+), 30 deletions(-) diff --git a/cmd/frostfs-node/config/profiler/config.go b/cmd/frostfs-node/config/profiler/config.go index 191694970..6c3e8adab 100644 --- a/cmd/frostfs-node/config/profiler/config.go +++ b/cmd/frostfs-node/config/profiler/config.go @@ -52,7 +52,7 @@ func Address(c *config.Config) string { return AddressDefault } -// BlockRates returns the value of "block_rate" config parameter +// BlockRate returns the value of "block_rate" config parameter // from "pprof" section. func BlockRate(c *config.Config) int { s := c.Sub(subsection) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 68acb05d3..c484c5d8c 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -117,7 +117,7 @@ func (i *delNetInfo) TombstoneLifetime() (uint64, error) { return i.cfg.cfgObject.tombstoneLifetime.Load(), nil } -// returns node owner ID calculated from configured private key. +// LocalNodeID returns node owner ID calculated from configured private key. // // Implements method needed for Object.Delete service. func (i *delNetInfo) LocalNodeID() user.ID { diff --git a/pkg/core/netmap/keys.go b/pkg/core/netmap/keys.go index 29cb2dc94..0c64bb798 100644 --- a/pkg/core/netmap/keys.go +++ b/pkg/core/netmap/keys.go @@ -2,6 +2,6 @@ package netmap // AnnouncedKeys is an interface of utility for working with the announced public keys of the storage nodes. type AnnouncedKeys interface { - // Checks if the key was announced by a local node. + // IsLocalKey checks if the key was announced by a local node. IsLocalKey(key []byte) bool } diff --git a/pkg/innerring/processors/netmap/nodevalidation/locode/deps.go b/pkg/innerring/processors/netmap/nodevalidation/locode/deps.go index 8f6667933..ba5db9205 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/locode/deps.go +++ b/pkg/innerring/processors/netmap/nodevalidation/locode/deps.go @@ -8,38 +8,38 @@ import ( // Record is an interface of read-only // FrostFS LOCODE database single entry. type Record interface { - // Must return ISO 3166-1 alpha-2 + // CountryCode must return ISO 3166-1 alpha-2 // country code. // // Must not return nil. CountryCode() *locodedb.CountryCode - // Must return English short country name + // CountryName must return English short country name // officially used by the ISO 3166 // Maintenance Agency (ISO 3166/MA). CountryName() string - // Must return UN/LOCODE 3-character code + // LocationCode must return UN/LOCODE 3-character code // for the location (numerals 2-9 may also // be used). // // Must not return nil. LocationCode() *locodedb.LocationCode - // Must return name of the location which + // LocationName must return name of the location which // have been allocated a UN/LOCODE without // diacritic sign. LocationName() string - // Must return ISO 1-3 character alphabetic + // SubDivCode Must return ISO 1-3 character alphabetic // and/or numeric code for the administrative // division of the country concerned. SubDivCode() string - // Must return subdivision name. + // SubDivName must return subdivision name. SubDivName() string - // Must return existing continent where is + // Continent must return existing continent where is // the location. // // Must not return nil. @@ -49,7 +49,7 @@ type Record interface { // DB is an interface of read-only // FrostFS LOCODE database. type DB interface { - // Must find the record that corresponds to + // Get must find the record that corresponds to // LOCODE and provides the Record interface. // // Must return an error if Record is nil. diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index e8fb8721b..4cecda59c 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -43,7 +43,7 @@ type ( // of information about the node and its finalization for adding // to the network map. NodeValidator interface { - // Must verify and optionally update NodeInfo structure. + // VerifyAndUpdate must verify and optionally update NodeInfo structure. // // Must return an error if NodeInfo input is invalid. // Must return an error if it is not possible to correctly diff --git a/pkg/local_object_storage/shard/metrics.go b/pkg/local_object_storage/shard/metrics.go index 91bf8d0ae..087ba42ef 100644 --- a/pkg/local_object_storage/shard/metrics.go +++ b/pkg/local_object_storage/shard/metrics.go @@ -22,7 +22,7 @@ type MetricsWriter interface { // SetShardID must set (update) the shard identifier that will be used in // metrics. SetShardID(id string) - // SetReadonly must set shard mode. + // SetMode set mode of shard. SetMode(mode mode.Mode) // SetContainerObjectsCount sets container object count. SetContainerObjectsCount(cnrID string, objectType string, value uint64) diff --git a/pkg/morph/client/actor.go b/pkg/morph/client/actor.go index b6718dea5..2849f3052 100644 --- a/pkg/morph/client/actor.go +++ b/pkg/morph/client/actor.go @@ -16,7 +16,7 @@ type actorProvider interface { GetRPCActor() actor.RPCActor } -// Client switches an established connection with neo-go if it is broken. +// SwitchRPCGuardedActor switches an established connection with neo-go if it is broken. // This leads to an invalidation of an rpc actor within Client. That means the // components that are initilized with the rpc actor may unintentionally use // it when it is already invalidated. SwitchRPCGuardedActor is used to prevent diff --git a/pkg/morph/event/container/put_notary.go b/pkg/morph/event/container/put_notary.go index f5779ced6..6b2ee7b0a 100644 --- a/pkg/morph/event/container/put_notary.go +++ b/pkg/morph/event/container/put_notary.go @@ -46,7 +46,7 @@ const ( // put container requests. PutNotaryEvent = "put" - // PutNotaryEvent is an ID of notary "put named container" notification. + // PutNamedNotaryEvent is an ID of notary "put named container" notification. PutNamedNotaryEvent = "putNamed" ) diff --git a/pkg/network/group.go b/pkg/network/group.go index a6de0653e..9843b14d4 100644 --- a/pkg/network/group.go +++ b/pkg/network/group.go @@ -67,11 +67,11 @@ func (x AddressGroup) Swap(i, j int) { // MultiAddressIterator is an interface of network address group. type MultiAddressIterator interface { - // Must iterate over network addresses and pass each one + // IterateAddresses must iterate over network addresses and pass each one // to the handler until it returns true. IterateAddresses(func(string) bool) - // Must return number of addresses in group. + // NumberOfAddresses must return number of addresses in group. NumberOfAddresses() int } diff --git a/pkg/services/control/ir/server/deps.go b/pkg/services/control/ir/server/deps.go index 0c2de5300..9d5cfefc8 100644 --- a/pkg/services/control/ir/server/deps.go +++ b/pkg/services/control/ir/server/deps.go @@ -5,7 +5,7 @@ import control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ // HealthChecker is component interface for calculating // the current health status of a node. type HealthChecker interface { - // Must calculate and return current health status of the IR application. + // HealthStatus must calculate and return current health status of the IR application. // // If status can not be calculated for any reason, // control.HealthStatus_HEALTH_STATUS_UNDEFINED should be returned. diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 80aece008..04524a68c 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -292,7 +292,7 @@ func ListTargetsLocalOverrides(cli *client.Client, req *ListTargetsLocalOverride return wResp.message, nil } -// RemoveChainLocalOverride executes ControlService.RemoveChainLocalOverride RPC. +// GetChainLocalOverride executes ControlService.RemoveChainLocalOverride RPC. func GetChainLocalOverride(cli *client.Client, req *GetChainLocalOverrideRequest, opts ...client.CallOption) (*GetChainLocalOverrideResponse, error) { wResp := newResponseWrapper[GetChainLocalOverrideResponse]() wReq := &requestWrapper{m: req} diff --git a/pkg/services/control/server/server.go b/pkg/services/control/server/server.go index f3fe56a46..b6fdcb246 100644 --- a/pkg/services/control/server/server.go +++ b/pkg/services/control/server/server.go @@ -26,13 +26,13 @@ type Server struct { // HealthChecker is component interface for calculating // the current health status of a node. type HealthChecker interface { - // Must calculate and return current status of the node in FrostFS network map. + // NetmapStatus must calculate and return current status of the node in FrostFS network map. // // If status can not be calculated for any reason, // control.netmapStatus_STATUS_UNDEFINED should be returned. NetmapStatus() control.NetmapStatus - // Must calculate and return current health status of the node application. + // HealthStatus must calculate and return current health status of the node application. // // If status can not be calculated for any reason, // control.HealthStatus_HEALTH_STATUS_UNDEFINED should be returned. diff --git a/pkg/services/netmap/executor.go b/pkg/services/netmap/executor.go index 496b07a98..ae2044246 100644 --- a/pkg/services/netmap/executor.go +++ b/pkg/services/netmap/executor.go @@ -26,7 +26,7 @@ type executorSvc struct { // NodeState encapsulates information // about current node state. type NodeState interface { - // Must return current node state + // LocalNodeInfo must return current node state // in FrostFS API v2 NodeInfo structure. LocalNodeInfo() (*netmap.NodeInfo, error) @@ -39,7 +39,7 @@ type NodeState interface { // NetworkInfo encapsulates source of the // recent information about the FrostFS network. type NetworkInfo interface { - // Must return recent network information in FrostFS API v2 NetworkInfo structure. + // Dump must return recent network information in FrostFS API v2 NetworkInfo structure. // // If protocol version is <=2.9, MillisecondsPerBlock and network config should be unset. Dump(versionsdk.Version) (*netmapSDK.NetworkInfo, error) diff --git a/pkg/services/object/common/writer/distributed.go b/pkg/services/object/common/writer/distributed.go index f62934bed..f7486eae7 100644 --- a/pkg/services/object/common/writer/distributed.go +++ b/pkg/services/object/common/writer/distributed.go @@ -28,7 +28,7 @@ type distributedWriter struct { resetSuccessAfterOnBroadcast bool } -// parameters and state of container Traversal. +// Traversal parameters and state of container. type Traversal struct { Opts []placement.Option diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index 0ba21eee3..e4f7a8c50 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -27,11 +27,11 @@ type Option func(*cfg) type NetworkInfo interface { netmap.State - // Must return the lifespan of the tombstones + // TombstoneLifetime must return the lifespan of the tombstones // in the FrostFS epochs. TombstoneLifetime() (uint64, error) - // Returns user ID of the local storage node. Result must not be nil. + // LocalNodeID returns user ID of the local storage node. Result must not be nil. // New tombstone objects will have the result as an owner ID if removal is executed w/o a session. LocalNodeID() user.ID } diff --git a/pkg/services/object/patch/service.go b/pkg/services/object/patch/service.go index f1082dfff..953f82b48 100644 --- a/pkg/services/object/patch/service.go +++ b/pkg/services/object/patch/service.go @@ -26,7 +26,7 @@ func NewService(cfg *objectwriter.Config, } } -// Put calls internal service and returns v2 object streamer. +// Patch calls internal service and returns v2 object streamer. func (s *Service) Patch() (object.PatchObjectStream, error) { nodeKey, err := s.Config.KeyStorage.GetKey(nil) if err != nil { diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 9dbfd8b9f..336f7a0ab 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -143,7 +143,7 @@ func WithPlacementBuilder(v placement.Builder) Option { } } -// WithRemoteObjectHeader returns option to set remote object header receiver of Policer. +// WithRemoteObjectHeaderFunc returns option to set remote object header receiver of Policer. func WithRemoteObjectHeaderFunc(v RemoteObjectHeaderFunc) Option { return func(c *cfg) { c.remoteHeader = v diff --git a/pkg/util/rand/rand.go b/pkg/util/rand/rand.go index 97508f82a..a06296a07 100644 --- a/pkg/util/rand/rand.go +++ b/pkg/util/rand/rand.go @@ -13,7 +13,7 @@ func Uint64() uint64 { return source.Uint64() } -// Uint64 returns a random uint32 value. +// Uint32 returns a random uint32 value. func Uint32() uint32 { return source.Uint32() } diff --git a/pkg/util/sdnotify/sdnotify.go b/pkg/util/sdnotify/sdnotify.go index e94ff77ad..22549bc96 100644 --- a/pkg/util/sdnotify/sdnotify.go +++ b/pkg/util/sdnotify/sdnotify.go @@ -24,7 +24,7 @@ var ( errSocketIsNotInitialized = errors.New("socket is not initialized") ) -// Initializes socket with provided name of +// InitSocket initializes socket with provided name of // environment variable. func InitSocket() error { notifySocket := os.Getenv("NOTIFY_SOCKET") From 41038b2ec0fab0d9488f15330e8777f053a28c03 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 14 Oct 2024 18:16:59 +0300 Subject: [PATCH 1449/1943] [#1431] node: Fix 'empty slice declaration using a literal' Signed-off-by: Alexander Chuprov --- pkg/innerring/processors/alphabet/handlers_test.go | 6 +++--- pkg/local_object_storage/shard/control.go | 2 +- pkg/morph/event/notary_preparator_test.go | 2 +- scripts/populate-metabase/internal/generate.go | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/innerring/processors/alphabet/handlers_test.go b/pkg/innerring/processors/alphabet/handlers_test.go index dfda37472..c7a004b54 100644 --- a/pkg/innerring/processors/alphabet/handlers_test.go +++ b/pkg/innerring/processors/alphabet/handlers_test.go @@ -95,7 +95,7 @@ func TestProcessorEmitsGasToNetmapIfNoParsedWallets(t *testing.T) { t.Parallel() var emission uint64 = 100_000 var index int = 5 - var parsedWallets []util.Uint160 = []util.Uint160{} + var parsedWallets []util.Uint160 alphabetContracts := innerring.NewAlphabetContracts() for i := range index + 1 { @@ -167,7 +167,7 @@ func TestProcessorDoesntEmitGasIfNoNetmapOrParsedWallets(t *testing.T) { t.Parallel() var emission uint64 = 100_000 var index int = 5 - var parsedWallets []util.Uint160 = []util.Uint160{} + var parsedWallets []util.Uint160 alphabetContracts := innerring.NewAlphabetContracts() for i := range index + 1 { @@ -176,7 +176,7 @@ func TestProcessorDoesntEmitGasIfNoNetmapOrParsedWallets(t *testing.T) { morphClient := &testMorphClient{} - nodes := []netmap.NodeInfo{} + var nodes []netmap.NodeInfo network := &netmap.NetMap{} network.SetNodes(nodes) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 4f9f25608..62800dbd0 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -368,7 +368,7 @@ func (s *Shard) Close() error { if s.rb != nil { s.rb.Stop(s.log) } - components := []interface{ Close() error }{} + var components []interface{ Close() error } if s.pilorama != nil { components = append(components, s.pilorama) diff --git a/pkg/morph/event/notary_preparator_test.go b/pkg/morph/event/notary_preparator_test.go index 4c269bcbd..60ddb4601 100644 --- a/pkg/morph/event/notary_preparator_test.go +++ b/pkg/morph/event/notary_preparator_test.go @@ -25,7 +25,7 @@ var ( alphaKeys keys.PublicKeys wrongAlphaKeys keys.PublicKeys - dummyAlphabetInvocationScript = []byte{} // expected to be empty if generated by Notary Actor, as requester can't fill it in + dummyAlphabetInvocationScript []byte dummyAlphabetInvocationScriptOld = append([]byte{byte(opcode.PUSHDATA1), 64}, make([]byte, 64)...) // expected to be dummy if generated manually wrongDummyInvocationScript = append([]byte{byte(opcode.PUSHDATA1), 64, 1}, make([]byte, 63)...) diff --git a/scripts/populate-metabase/internal/generate.go b/scripts/populate-metabase/internal/generate.go index d2004b673..8a96dcaaa 100644 --- a/scripts/populate-metabase/internal/generate.go +++ b/scripts/populate-metabase/internal/generate.go @@ -18,7 +18,7 @@ import ( ) func GeneratePayloadPool(count uint, size uint) [][]byte { - pool := [][]byte{} + var pool [][]byte for i := uint(0); i < count; i++ { payload := make([]byte, size) _, _ = rand.Read(payload) @@ -29,7 +29,7 @@ func GeneratePayloadPool(count uint, size uint) [][]byte { } func GenerateAttributePool(count uint) []objectSDK.Attribute { - pool := []objectSDK.Attribute{} + var pool []objectSDK.Attribute for i := uint(0); i < count; i++ { for j := uint(0); j < count; j++ { attr := *objectSDK.NewAttribute() @@ -42,7 +42,7 @@ func GenerateAttributePool(count uint) []objectSDK.Attribute { } func GenerateOwnerPool(count uint) []user.ID { - pool := []user.ID{} + var pool []user.ID for i := uint(0); i < count; i++ { pool = append(pool, usertest.ID()) } @@ -117,7 +117,7 @@ func WithPayloadFromPool(pool [][]byte) ObjectOption { func WithAttributesFromPool(pool []objectSDK.Attribute, count uint) ObjectOption { return func(obj *objectSDK.Object) { - attrs := []objectSDK.Attribute{} + var attrs []objectSDK.Attribute for i := uint(0); i < count; i++ { attrs = append(attrs, pool[rand.Intn(len(pool))]) } From 07ce40e1196a44d305390dcea8e1e0040f6a16d2 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 15 Oct 2024 12:28:58 +0300 Subject: [PATCH 1450/1943] [#1430] adm/morph: Add NNS address display in 'deploy' Signed-off-by: Alexander Chuprov --- cmd/frostfs-adm/internal/modules/morph/helper/contract.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/contract.go b/cmd/frostfs-adm/internal/modules/morph/helper/contract.go index 2011301d1..eea3b040e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/contract.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/contract.go @@ -166,5 +166,6 @@ func DeployNNS(c *InitializeContext, method string) error { return fmt.Errorf("can't send deploy transaction: %w", err) } + c.Command.Println("NNS hash:", invokeHash.StringLE()) return c.AwaitTx() } From 90f36693995e1b411094686e4419bb7d11831f35 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 14 Oct 2024 16:07:38 +0300 Subject: [PATCH 1451/1943] [#1342] network/cache: Add node address to error multiClient Signed-off-by: Alexander Chuprov --- pkg/network/cache/multi.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 9305c143b..b83cbb217 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -169,15 +169,16 @@ func (x *multiClient) iterateClients(ctx context.Context, f func(clientcore.Clie var siErr *objectSDK.SplitInfoError var eiErr *objectSDK.ECInfoError + if err != nil { + err = fmt.Errorf("client connection error at %v: %w", addr, err) + x.ReportError(err) + } + success := err == nil || errors.Is(err, context.Canceled) || errors.As(err, &siErr) || errors.As(err, &eiErr) if success || firstErr == nil || errors.Is(firstErr, errRecentlyFailed) { firstErr = err } - if err != nil { - x.ReportError(err) - } - return success }) From b0c5def2d934ed5b79f54fb37160560f576785f4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 17 Oct 2024 14:16:03 +0300 Subject: [PATCH 1452/1943] [#1433] shard/test: Use WithDisabledGC() option where possible Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/gc_internal_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index 3993593ad..11db5e54e 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -73,10 +73,10 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { return pool }), WithGCRemoverSleepInterval(1 * time.Second), + WithDisabledGC(), } sh = New(opts...) - sh.gcCfg.testHookRemover = func(context.Context) gcRunResult { return gcRunResult{} } require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) defer func() { require.NoError(t, sh.Close()) }() From b42bcdc6fa6cca2cf8e5a5fbaf2c8cf82f957b37 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 17 Oct 2024 14:37:26 +0300 Subject: [PATCH 1453/1943] [#1433] services/object: Put object before auxiliary info Consider the following operations ordering: 1. Inhume(with tombstone A) --> add tombstone mark for an object 2. --> new epoch arives 3. --> GCMark is added for a tombstone A, because it is unavailable 4. Put(A) --> return error, because the object already has a GCMark It is possible, and I have successfully reproduced it with a test on the shard level. However, the error is related to the specific _ordering_ of operations with engine. And triggering race-conditions like this is only possible on a shard level currently, so no tests are written. Signed-off-by: Evgenii Stratonikov --- pkg/services/object/common/writer/local.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/services/object/common/writer/local.go b/pkg/services/object/common/writer/local.go index e219b44dd..cf3d03275 100644 --- a/pkg/services/object/common/writer/local.go +++ b/pkg/services/object/common/writer/local.go @@ -32,6 +32,10 @@ type LocalTarget struct { } func (t LocalTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, meta objectCore.ContentMeta) error { + if err := t.Storage.Put(ctx, obj, containerCore.IsIndexedContainer(t.Container)); err != nil { + return fmt.Errorf("(%T) could not put object to local storage: %w", t, err) + } + switch meta.Type() { case objectSDK.TypeTombstone: err := t.Storage.Delete(ctx, objectCore.AddressOf(obj), meta.Objects()) @@ -47,8 +51,5 @@ func (t LocalTarget) WriteObject(ctx context.Context, obj *objectSDK.Object, met // objects that do not change meta storage } - if err := t.Storage.Put(ctx, obj, containerCore.IsIndexedContainer(t.Container)); err != nil { - return fmt.Errorf("(%T) could not put object to local storage: %w", t, err) - } return nil } From 3304afa9d1f9893ad72bcd9445751798b6558c16 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 8 Oct 2024 15:24:01 +0300 Subject: [PATCH 1454/1943] [#1422] config: Add multinet config Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config/multinet/config.go | 62 +++++++++++++++++++ .../config/multinet/config_test.go | 52 ++++++++++++++++ config/example/ir.env | 9 +++ config/example/ir.yaml | 15 +++++ config/example/node.env | 10 +++ config/example/node.json | 22 +++++++ config/example/node.yaml | 15 +++++ docs/storage-node-configuration.md | 39 ++++++++++-- 8 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 cmd/frostfs-node/config/multinet/config.go create mode 100644 cmd/frostfs-node/config/multinet/config_test.go diff --git a/cmd/frostfs-node/config/multinet/config.go b/cmd/frostfs-node/config/multinet/config.go new file mode 100644 index 000000000..f598efc51 --- /dev/null +++ b/cmd/frostfs-node/config/multinet/config.go @@ -0,0 +1,62 @@ +package multinet + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" +) + +const ( + subsection = "multinet" + + FallbackDelayDefault = 300 * time.Millisecond +) + +// Enabled returns the value of "enabled" config parameter from "multinet" section. +func Enabled(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "enabled") +} + +type Subnet struct { + Mask string + SourceIPs []string +} + +// Subnets returns the value of "subnets" config parameter from "multinet" section. +func Subnets(c *config.Config) []Subnet { + var result []Subnet + sub := c.Sub(subsection).Sub("subnets") + for i := 0; ; i++ { + s := sub.Sub(strconv.FormatInt(int64(i), 10)) + mask := config.StringSafe(s, "mask") + if mask == "" { + break + } + sourceIPs := config.StringSliceSafe(s, "source_ips") + result = append(result, Subnet{ + Mask: mask, + SourceIPs: sourceIPs, + }) + } + return result +} + +// Balancer returns the value of "balancer" config parameter from "multinet" section. +func Balancer(c *config.Config) string { + return config.StringSafe(c.Sub(subsection), "balancer") +} + +// Restrict returns the value of "restrict" config parameter from "multinet" section. +func Restrict(c *config.Config) bool { + return config.BoolSafe(c.Sub(subsection), "restrict") +} + +// FallbackDelay returns the value of "fallback_delay" config parameter from "multinet" section. +func FallbackDelay(c *config.Config) time.Duration { + fd := config.DurationSafe(c.Sub(subsection), "fallback_delay") + if fd != 0 { // negative value means no fallback + return fd + } + return FallbackDelayDefault +} diff --git a/cmd/frostfs-node/config/multinet/config_test.go b/cmd/frostfs-node/config/multinet/config_test.go new file mode 100644 index 000000000..5f7dc6d53 --- /dev/null +++ b/cmd/frostfs-node/config/multinet/config_test.go @@ -0,0 +1,52 @@ +package multinet + +import ( + "testing" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "github.com/stretchr/testify/require" +) + +func TestMultinetSection(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + empty := configtest.EmptyConfig() + require.Equal(t, false, Enabled(empty)) + require.Equal(t, ([]Subnet)(nil), Subnets(empty)) + require.Equal(t, "", Balancer(empty)) + require.Equal(t, false, Restrict(empty)) + require.Equal(t, FallbackDelayDefault, FallbackDelay(empty)) + }) + + const path = "../../../../config/example/node" + + fileConfigTest := func(c *config.Config) { + require.Equal(t, true, Enabled(c)) + require.Equal(t, []Subnet{ + { + Mask: "192.168.219.174/24", + SourceIPs: []string{ + "192.168.218.185", + "192.168.219.185", + }, + }, + { + Mask: "10.78.70.74/24", + SourceIPs: []string{ + "10.78.70.185", + "10.78.71.185", + }, + }, + }, Subnets(c)) + require.Equal(t, "roundrobin", Balancer(c)) + require.Equal(t, false, Restrict(c)) + require.Equal(t, 350*time.Millisecond, FallbackDelay(c)) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) +} diff --git a/config/example/ir.env b/config/example/ir.env index 7234a4b32..ebd91c243 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -80,3 +80,12 @@ FROSTFS_IR_PPROF_MUTEX_RATE=10000 FROSTFS_IR_PROMETHEUS_ENABLED=true FROSTFS_IR_PROMETHEUS_ADDRESS=localhost:9090 FROSTFS_IR_PROMETHEUS_SHUTDOWN_TIMEOUT=30s + +FROSTFS_MULTINET_ENABLED=true +FROSTFS_MULTINET_SUBNETS_0_MASK="192.168.219.174/24" +FROSTFS_MULTINET_SUBNETS_0_SOURCE_IPS="192.168.218.185 192.168.219.185" +FROSTFS_MULTINET_SUBNETS_1_MASK="10.78.70.74/24" +FROSTFS_MULTINET_SUBNETS_1_SOURCE_IPS="10.78.70.185 10.78.71.185" +FROSTFS_MULTINET_BALANCER=roundrobin +FROSTFS_MULTINET_RESTRICT=false +FROSTFS_MULTINET_FALLBACK_DELAY=350ms diff --git a/config/example/ir.yaml b/config/example/ir.yaml index 4c64f088b..49f9fd324 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -123,3 +123,18 @@ prometheus: systemdnotify: enabled: true + +multinet: + enabled: true + subnets: + - mask: 192.168.219.174/24 + source_ips: + - 192.168.218.185 + - 192.168.219.185 + - mask: 10.78.70.74/24 + source_ips: + - 10.78.70.185 + - 10.78.71.185 + balancer: roundrobin + restrict: false + fallback_delay: 350ms diff --git a/config/example/node.env b/config/example/node.env index 6618a981a..580d343fb 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -206,3 +206,13 @@ FROSTFS_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 # AUDIT section FROSTFS_AUDIT_ENABLED=true + +# MULTINET section +FROSTFS_MULTINET_ENABLED=true +FROSTFS_MULTINET_SUBNETS_0_MASK="192.168.219.174/24" +FROSTFS_MULTINET_SUBNETS_0_SOURCE_IPS="192.168.218.185 192.168.219.185" +FROSTFS_MULTINET_SUBNETS_1_MASK="10.78.70.74/24" +FROSTFS_MULTINET_SUBNETS_1_SOURCE_IPS="10.78.70.185 10.78.71.185" +FROSTFS_MULTINET_BALANCER=roundrobin +FROSTFS_MULTINET_RESTRICT=false +FROSTFS_MULTINET_FALLBACK_DELAY=350ms diff --git a/config/example/node.json b/config/example/node.json index 0d100ed80..3470d2d12 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -264,5 +264,27 @@ }, "audit": { "enabled": true + }, + "multinet": { + "enabled": true, + "subnets": [ + { + "mask": "192.168.219.174/24", + "source_ips": [ + "192.168.218.185", + "192.168.219.185" + ] + }, + { + "mask": "10.78.70.74/24", + "source_ips":[ + "10.78.70.185", + "10.78.71.185" + ] + } + ], + "balancer": "roundrobin", + "restrict": false, + "fallback_delay": "350ms" } } diff --git a/config/example/node.yaml b/config/example/node.yaml index 2a80fba18..2a963fc0f 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -240,3 +240,18 @@ runtime: audit: enabled: true + +multinet: + enabled: true + subnets: + - mask: 192.168.219.174/24 + source_ips: + - 192.168.218.185 + - 192.168.219.185 + - mask: 10.78.70.74/24 + source_ips: + - 10.78.70.185 + - 10.78.71.185 + balancer: roundrobin + restrict: false + fallback_delay: 350ms diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index c74695e2b..2b94400df 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -25,8 +25,8 @@ There are some custom types used for brevity: | `replicator` | [Replicator service configuration](#replicator-section) | | `storage` | [Storage engine configuration](#storage-section) | | `runtime` | [Runtime configuration](#runtime-section) | -| `audit` | [Audit configuration](#audit-section) | - +| `audit` | [Audit configuration](#audit-section) | +| `multinet` | [Multinet configuration](#multinet-section) | # `control` section ```yaml @@ -435,6 +435,35 @@ audit: enabled: true ``` -| Parameter | Type | Default value | Description | -|---------------------|--------|---------------|---------------------------------------------------| -| `soft_memory_limit` | `bool` | false | If `true` then audit event logs will be recorded. | +| Parameter | Type | Default value | Description | +|-----------|--------|---------------|---------------------------------------------------| +| `enabled` | `bool` | false | If `true` then audit event logs will be recorded. | + + +# `multinet` section +Contains multinet parameters. + +```yaml +multinet: + enabled: true + subnets: + - mask: 192.168.219.174/24 + source_ips: + - 192.168.218.185 + - 192.168.219.185 + - mask: 10.78.70.74/24 + source_ips: + - 10.78.70.185 + - 10.78.71.185 + balancer: roundrobin + restrict: false + fallback_delay: 350ms +``` + +| Parameter | Type | Default value | Description | +| ---------------- | ---------- | ------------- | -------------------------------------------------------------------------------------------------------------------------- | +| `enabled` | `bool` | false | If `true` then source-based routing is enabled. | +| `subnets` | `subnet` | empty | Resulting subnets. | +| `balancer` | `string` | "" | Balancer to select network interfaces, allowed values are "" (no balancing, use first suitable interface) or "roundrobin". | +| `restrict` | `bool` | false | If `true` then any requests that do not match `subnets` will fail. | +| `fallback_delay` | `duration` | 350ms | Delay before fallback to secondary IP addresses in case of hostname resolve. | From 74db7352653b67e67e9345e0659fd37047fec710 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 8 Oct 2024 17:25:37 +0300 Subject: [PATCH 1455/1943] [#1422] node: Add dialer source to config Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 30 +++++++++++++++ go.mod | 1 + go.sum | 2 + internal/logs/logs.go | 1 + internal/net/config.go | 66 +++++++++++++++++++++++++++++++++ internal/net/dial_target.go | 54 +++++++++++++++++++++++++++ internal/net/dialer.go | 35 ++++++++++++++++++ internal/net/dialer_source.go | 69 +++++++++++++++++++++++++++++++++++ 8 files changed, 258 insertions(+) create mode 100644 internal/net/config.go create mode 100644 internal/net/dial_target.go create mode 100644 internal/net/dialer.go create mode 100644 internal/net/dialer_source.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 3c7e310b4..dc1bad485 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -26,12 +26,14 @@ import ( fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger" morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/multinet" nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" + internalNet "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" @@ -436,6 +438,8 @@ type shared struct { metricsCollector *metrics.NodeMetrics metricsSvc *objectService.MetricCollector + + dialerSource *internalNet.DialerSource } // dynamicConfiguration stores parameters of the @@ -760,6 +764,9 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt persistate, err := state.NewPersistentStorage(nodeconfig.PersistentState(appCfg).Path()) fatalOnErr(err) + ds, err := internalNet.NewDialerSource(internalNetConfig(appCfg)) + fatalOnErr(err) + cacheOpts := cache.ClientCacheOpts{ DialTimeout: apiclientconfig.DialTimeout(appCfg), StreamTimeout: apiclientconfig.StreamTimeout(appCfg), @@ -778,9 +785,27 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt putClientCache: cache.NewSDKClientCache(cacheOpts), persistate: persistate, metricsCollector: metrics.NewNodeMetrics(), + dialerSource: ds, } } +func internalNetConfig(appCfg *config.Config) internalNet.Config { + result := internalNet.Config{ + Enabled: multinet.Enabled(appCfg), + Balancer: multinet.Balancer(appCfg), + Restrict: multinet.Restrict(appCfg), + FallbackDelay: multinet.FallbackDelay(appCfg), + } + sn := multinet.Subnets(appCfg) + for _, s := range sn { + result.Subnets = append(result.Subnets, internalNet.Subnet{ + Prefix: s.Mask, + SourceIPs: s.SourceIPs, + }) + } + return result +} + func initNetmap(appCfg *config.Config, netState *networkState, relayOnly bool) cfgNetmap { netmapWorkerPool, err := ants.NewPool(notificationHandlerPoolSize) fatalOnErr(err) @@ -1336,6 +1361,11 @@ func (c *cfg) reloadConfig(ctx context.Context) { } } + if err := c.dialerSource.Update(internalNetConfig(c.appCfg)); err != nil { + c.log.Error(logs.FailedToUpdateMultinetConfiguration, zap.Error(err)) + return + } + c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } diff --git a/go.mod b/go.mod index 1468c12b2..a84d3122a 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241010110344-99c5c5836509 git.frostfs.info/TrueCloudLab/hrw v1.2.1 + git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 diff --git a/go.sum b/go.sum index 5ce81807a..43d53aa40 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241010110344-99c5c5836509 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241010110344-99c5c5836509/go.mod h1:jmb7yxzZota9jWbC10p+7YR+6wwJPBj7J/Fl5VDkXys= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= +git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= +git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972/go.mod h1:2hM42MBrlhvN6XToaW6OWNk5ZLcu1FhaukGgxtfpDDI= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 h1:vgbfkcnIexZUm3vREBBSa/Gv1Whjd1SFCUd0A+IaGPQ= diff --git a/internal/logs/logs.go b/internal/logs/logs.go index b4bc31b0c..0e9d58f32 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -523,4 +523,5 @@ const ( WritecacheShrinkSkippedNotEmpty = "writecache shrink skipped: not empty" BlobovniczatreeFailedToRemoveRebuildTempFile = "failed to remove rebuild temp file" WritecacheCantGetObject = "can't get an object from fstree" + FailedToUpdateMultinetConfiguration = "failed to update multinet configuration" ) diff --git a/internal/net/config.go b/internal/net/config.go new file mode 100644 index 000000000..10450db23 --- /dev/null +++ b/internal/net/config.go @@ -0,0 +1,66 @@ +package net + +import ( + "errors" + "fmt" + "net/netip" + "slices" + "time" + + "git.frostfs.info/TrueCloudLab/multinet" +) + +var errEmptySourceIPList = errors.New("empty source IP list") + +type Subnet struct { + Prefix string + SourceIPs []string +} + +type Config struct { + Enabled bool + Subnets []Subnet + Balancer string + Restrict bool + FallbackDelay time.Duration +} + +func (c Config) toMultinetConfig() (multinet.Config, error) { + var subnets []multinet.Subnet + for _, s := range c.Subnets { + var ms multinet.Subnet + p, err := netip.ParsePrefix(s.Prefix) + if err != nil { + return multinet.Config{}, fmt.Errorf("parse IP prefix '%s': %w", s.Prefix, err) + } + ms.Prefix = p + for _, ip := range s.SourceIPs { + addr, err := netip.ParseAddr(ip) + if err != nil { + return multinet.Config{}, fmt.Errorf("parse IP address '%s': %w", ip, err) + } + ms.SourceIPs = append(ms.SourceIPs, addr) + } + if len(ms.SourceIPs) == 0 { + return multinet.Config{}, errEmptySourceIPList + } + subnets = append(subnets, ms) + } + return multinet.Config{ + Subnets: subnets, + Balancer: multinet.BalancerType(c.Balancer), + Restrict: c.Restrict, + FallbackDelay: c.FallbackDelay, + Dialer: newDefaulDialer(), + }, nil +} + +func (c Config) equals(other Config) bool { + return c.Enabled == other.Enabled && + slices.EqualFunc(c.Subnets, other.Subnets, func(lhs, rhs Subnet) bool { + return lhs.Prefix == rhs.Prefix && slices.Equal(lhs.SourceIPs, rhs.SourceIPs) + }) && + c.Balancer == other.Balancer && + c.Restrict == other.Restrict && + c.FallbackDelay == other.FallbackDelay +} diff --git a/internal/net/dial_target.go b/internal/net/dial_target.go new file mode 100644 index 000000000..6265f1860 --- /dev/null +++ b/internal/net/dial_target.go @@ -0,0 +1,54 @@ +// NOTE: code is taken from https://github.com/grpc/grpc-go/blob/v1.68.x/internal/transport/http_util.go + +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package net + +import ( + "net/url" + "strings" +) + +// parseDialTarget returns the network and address to pass to dialer. +func parseDialTarget(target string) (string, string) { + net := "tcp" + m1 := strings.Index(target, ":") + m2 := strings.Index(target, ":/") + // handle unix:addr which will fail with url.Parse + if m1 >= 0 && m2 < 0 { + if n := target[0:m1]; n == "unix" { + return n, target[m1+1:] + } + } + if m2 >= 0 { + t, err := url.Parse(target) + if err != nil { + return net, target + } + scheme := t.Scheme + addr := t.Path + if scheme == "unix" { + if addr == "" { + addr = t.Host + } + return scheme, addr + } + } + return net, target +} diff --git a/internal/net/dialer.go b/internal/net/dialer.go new file mode 100644 index 000000000..4537490f6 --- /dev/null +++ b/internal/net/dialer.go @@ -0,0 +1,35 @@ +package net + +import ( + "context" + "net" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +type Dialer interface { + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} + +func newDefaulDialer() net.Dialer { + // From `grpc.WithContextDialer` comment: + // + // Note: All supported releases of Go (as of December 2023) override the OS + // defaults for TCP keepalive time and interval to 15s. To enable TCP keepalive + // with OS defaults for keepalive time and interval, use a net.Dialer that sets + // the KeepAlive field to a negative value, and sets the SO_KEEPALIVE socket + // option to true from the Control field. For a concrete example of how to do + // this, see internal.NetDialerWithTCPKeepalive(). + // + // https://github.com/grpc/grpc-go/blob/830135e6c5a351abf75f0c9cfdf978e5df8daeba/dialoptions.go#L432 + return net.Dialer{ + KeepAlive: time.Duration(-1), + Control: func(_, _ string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_KEEPALIVE, 1) + }) + }, + } +} diff --git a/internal/net/dialer_source.go b/internal/net/dialer_source.go new file mode 100644 index 000000000..e6a142a08 --- /dev/null +++ b/internal/net/dialer_source.go @@ -0,0 +1,69 @@ +package net + +import ( + "context" + "net" + "sync" + + "git.frostfs.info/TrueCloudLab/multinet" +) + +type DialerSource struct { + guard sync.RWMutex + + c Config + + md multinet.Dialer +} + +func NewDialerSource(c Config) (*DialerSource, error) { + result := &DialerSource{} + if err := result.build(c); err != nil { + return nil, err + } + return result, nil +} + +func (s *DialerSource) build(c Config) error { + if c.Enabled { + mc, err := c.toMultinetConfig() + if err != nil { + return err + } + md, err := multinet.NewDialer(mc) + if err != nil { + return err + } + s.md = md + s.c = c + return nil + } + s.md = nil + s.c = c + return nil +} + +// GrpcContextDialer returns grpc.WithContextDialer func. +// Returns nil if multinet disabled. +func (s *DialerSource) GrpcContextDialer() func(context.Context, string) (net.Conn, error) { + s.guard.RLock() + defer s.guard.RUnlock() + + if s.c.Enabled { + return func(ctx context.Context, address string) (net.Conn, error) { + network, address := parseDialTarget(address) + return s.md.DialContext(ctx, network, address) + } + } + return nil +} + +func (s *DialerSource) Update(c Config) error { + s.guard.Lock() + defer s.guard.Unlock() + + if s.c.equals(c) { + return nil + } + return s.build(c) +} From 6c96cc2af6eb9cb64e747c5b758fbec4d90c7287 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Oct 2024 11:11:44 +0300 Subject: [PATCH 1456/1943] [#1422] node: Use dialer source for SDK cache Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 + internal/net/dialer.go | 6 +++++- pkg/network/cache/client.go | 2 ++ pkg/network/cache/multi.go | 25 ++++++++++++++----------- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index dc1bad485..d44597857 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -773,6 +773,7 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt Key: &key.PrivateKey, AllowExternal: apiclientconfig.AllowExternal(appCfg), ReconnectTimeout: apiclientconfig.ReconnectTimeout(appCfg), + DialerSource: ds, } return shared{ diff --git a/internal/net/dialer.go b/internal/net/dialer.go index 4537490f6..daf0f815f 100644 --- a/internal/net/dialer.go +++ b/internal/net/dialer.go @@ -13,6 +13,10 @@ type Dialer interface { DialContext(ctx context.Context, network, address string) (net.Conn, error) } +func DialContextTCP(ctx context.Context, address string, d Dialer) (net.Conn, error) { + return d.DialContext(ctx, "tcp", address) +} + func newDefaulDialer() net.Dialer { // From `grpc.WithContextDialer` comment: // @@ -28,7 +32,7 @@ func newDefaulDialer() net.Dialer { KeepAlive: time.Duration(-1), Control: func(_, _ string, c syscall.RawConn) error { return c.Control(func(fd uintptr) { - unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_KEEPALIVE, 1) + _ = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_KEEPALIVE, 1) }) }, } diff --git a/pkg/network/cache/client.go b/pkg/network/cache/client.go index 371d3c76f..63ae0bfdb 100644 --- a/pkg/network/cache/client.go +++ b/pkg/network/cache/client.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" ) @@ -25,6 +26,7 @@ type ( Key *ecdsa.PrivateKey ResponseCallback func(client.ResponseMetaInfo) error AllowExternal bool + DialerSource *net.DialerSource } ) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index b83cbb217..e936ead65 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -60,18 +60,21 @@ func (x *multiClient) createForAddress(ctx context.Context, addr network.Address prmInit.Key = *x.opts.Key } + grpcOpts := []grpc.DialOption{ + grpc.WithChainUnaryInterceptor( + metrics.NewUnaryClientInterceptor(), + tracing.NewUnaryClientInteceptor(), + ), + grpc.WithChainStreamInterceptor( + metrics.NewStreamClientInterceptor(), + tracing.NewStreamClientInterceptor(), + ), + grpc.WithContextDialer(x.opts.DialerSource.GrpcContextDialer()), + } + prmDial := client.PrmDial{ - Endpoint: addr.URIAddr(), - GRPCDialOptions: []grpc.DialOption{ - grpc.WithChainUnaryInterceptor( - metrics.NewUnaryClientInterceptor(), - tracing.NewUnaryClientInteceptor(), - ), - grpc.WithChainStreamInterceptor( - metrics.NewStreamClientInterceptor(), - tracing.NewStreamClientInterceptor(), - ), - }, + Endpoint: addr.URIAddr(), + GRPCDialOptions: grpcOpts, } if x.opts.DialTimeout > 0 { prmDial.DialTimeout = x.opts.DialTimeout From e314f328c4806bf1b34b6e3c31abdc4afdfaaac4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Oct 2024 11:18:24 +0300 Subject: [PATCH 1457/1943] [#1422] tree: Use dialer source for tree service connections Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/tree.go | 1 + pkg/services/tree/cache.go | 6 +++++- pkg/services/tree/options.go | 8 ++++++++ pkg/services/tree/service.go | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index 192f08471..f188e2fbc 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -67,6 +67,7 @@ func initTreeService(c *cfg) { tree.WithAPELocalOverrideStorage(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalStorage()), tree.WithAPEMorphRuleStorage(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.MorphRuleChainStorage()), tree.WithNetmapState(c.cfgNetmap.state), + tree.WithDialerSource(c.dialerSource), ) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 38501b852..e490cb855 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -8,6 +8,7 @@ import ( "sync" "time" + internalNet "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" @@ -21,6 +22,7 @@ type clientCache struct { sync.Mutex simplelru.LRU[string, cacheItem] key *ecdsa.PrivateKey + ds *internalNet.DialerSource } type cacheItem struct { @@ -36,7 +38,7 @@ const ( var errRecentlyFailed = errors.New("client has recently failed") -func (c *clientCache) init(pk *ecdsa.PrivateKey) { +func (c *clientCache) init(pk *ecdsa.PrivateKey, ds *internalNet.DialerSource) { l, _ := simplelru.NewLRU(defaultClientCacheSize, func(_ string, value cacheItem) { if conn := value.cc; conn != nil { _ = conn.Close() @@ -44,6 +46,7 @@ func (c *clientCache) init(pk *ecdsa.PrivateKey) { }) c.LRU = *l c.key = pk + c.ds = ds } func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceClient, error) { @@ -99,6 +102,7 @@ func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (* metrics.NewStreamClientInterceptor(), tracing.NewStreamClientInterceptor(), ), + grpc.WithContextDialer(c.ds.GrpcContextDialer()), } if !netAddr.IsTLSEnabled() { diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index 1db5607f6..1633ae557 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -4,6 +4,7 @@ import ( "crypto/ecdsa" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -45,6 +46,7 @@ type cfg struct { morphChainStorage policyengine.MorphRuleChainStorageReader metrics MetricsRegister + ds *net.DialerSource } // Option represents configuration option for a tree service. @@ -161,3 +163,9 @@ func WithNetmapState(state netmap.State) Option { c.state = state } } + +func WithDialerSource(ds *net.DialerSource) Option { + return func(c *cfg) { + c.ds = ds + } +} diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 60bb1a6ad..2cb2af294 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -65,7 +65,7 @@ func New(opts ...Option) *Service { s.log = &logger.Logger{Logger: zap.NewNop()} } - s.cache.init(s.key) + s.cache.init(s.key, s.ds) s.closeCh = make(chan struct{}) s.replicateCh = make(chan movePair, s.replicatorChannelCapacity) s.replicateLocalCh = make(chan applyOp) From 5b653aa65fd65759ee733fd552d17870c6660b13 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Oct 2024 11:34:36 +0300 Subject: [PATCH 1458/1943] [#1422] morph: Drop single client as not used Signed-off-by: Dmitrii Stepanov --- pkg/morph/client/constructor.go | 59 +++++++++------------------------ 1 file changed, 15 insertions(+), 44 deletions(-) diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 78cb3e82f..60b5b4b97 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -41,8 +41,6 @@ type cfg struct { endpoints []Endpoint - singleCli *rpcclient.WSClient // neo-go client for single client mode - inactiveModeCb Callback switchInterval time.Duration @@ -124,40 +122,24 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er var err error var act *actor.Actor - if cfg.singleCli != nil { - // return client in single RPC node mode that uses - // predefined WS client - // - // in case of the closing web socket connection: - // if extra endpoints were provided via options, - // they will be used in switch process, otherwise - // inactive mode will be enabled - cli.client = cfg.singleCli - - act, err = newActor(cfg.singleCli, acc, *cfg) + var endpoint Endpoint + for cli.endpoints.curr, endpoint = range cli.endpoints.list { + cli.client, act, err = cli.newCli(ctx, endpoint) if err != nil { - return nil, fmt.Errorf("could not create RPC actor: %w", err) - } - } else { - var endpoint Endpoint - for cli.endpoints.curr, endpoint = range cli.endpoints.list { - cli.client, act, err = cli.newCli(ctx, endpoint) - if err != nil { - cli.logger.Warn(logs.FrostFSIRCouldntCreateRPCClientForEndpoint, - zap.Error(err), zap.String("endpoint", endpoint.Address)) - } else { - cli.logger.Info(logs.FrostFSIRCreatedRPCClientForEndpoint, - zap.String("endpoint", endpoint.Address)) - if cli.endpoints.curr > 0 && cli.cfg.switchInterval != 0 { - cli.switchIsActive.Store(true) - go cli.switchToMostPrioritized(ctx) - } - break + cli.logger.Warn(logs.FrostFSIRCouldntCreateRPCClientForEndpoint, + zap.Error(err), zap.String("endpoint", endpoint.Address)) + } else { + cli.logger.Info(logs.FrostFSIRCreatedRPCClientForEndpoint, + zap.String("endpoint", endpoint.Address)) + if cli.endpoints.curr > 0 && cli.cfg.switchInterval != 0 { + cli.switchIsActive.Store(true) + go cli.switchToMostPrioritized(ctx) } + break } - if cli.client == nil { - return nil, ErrNoHealthyEndpoint - } + } + if cli.client == nil { + return nil, ErrNoHealthyEndpoint } cli.setActor(act) @@ -285,17 +267,6 @@ func WithEndpoints(endpoints ...Endpoint) Option { } } -// WithSingleClient returns a client constructor option -// that specifies single neo-go client and forces Client -// to use it for requests. -// -// Passed client must already be initialized. -func WithSingleClient(cli *rpcclient.WSClient) Option { - return func(c *cfg) { - c.singleCli = cli - } -} - // WithConnLostCallback return a client constructor option // that specifies a callback that is called when Client // unsuccessfully tried to connect to all the specified From 67798bb50e57ecf77a6797a3a02751f54a2e02ee Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Oct 2024 11:48:16 +0300 Subject: [PATCH 1459/1943] [#1422] mod: Bump neoneo-go version Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a84d3122a..aefe2889a 100644 --- a/go.mod +++ b/go.mod @@ -133,4 +133,4 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/nspcc-dev/neo-go => git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 +replace github.com/nspcc-dev/neo-go => git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20241015133823-8aee80dbdc07 diff --git a/go.sum b/go.sum index 43d53aa40..4d44079d4 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8l git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972/go.mod h1:2hM42MBrlhvN6XToaW6OWNk5ZLcu1FhaukGgxtfpDDI= -git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928 h1:LK3mCkNZkY48eBA9jnk1N0eQZLsZhOG+XYw4EBoKUjM= -git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20240726093631-5481339d6928/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= +git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20241015133823-8aee80dbdc07 h1:gPaqGsk6gSWQyNVjaStydfUz6Z/loHc9XyvGrJ5qSPY= +git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20241015133823-8aee80dbdc07/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 h1:vgbfkcnIexZUm3vREBBSa/Gv1Whjd1SFCUd0A+IaGPQ= git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= From fbdfd503e4c2475b01728dbfc21c269a443c93f2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Oct 2024 11:48:59 +0300 Subject: [PATCH 1460/1943] [#1422] morph: Add dialer source support Signed-off-by: Dmitrii Stepanov --- internal/net/dialer_source.go | 14 ++++++++++++++ pkg/morph/client/constructor.go | 15 +++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/internal/net/dialer_source.go b/internal/net/dialer_source.go index e6a142a08..3d94dedc7 100644 --- a/internal/net/dialer_source.go +++ b/internal/net/dialer_source.go @@ -58,6 +58,20 @@ func (s *DialerSource) GrpcContextDialer() func(context.Context, string) (net.Co return nil } +// NetContextDialer returns net.DialContext dial function. +// Returns nil if multinet disabled. +func (s *DialerSource) NetContextDialer() func(context.Context, string, string) (net.Conn, error) { + s.guard.RLock() + defer s.guard.RUnlock() + + if s.c.Enabled { + return func(ctx context.Context, network, address string) (net.Conn, error) { + return s.md.DialContext(ctx, network, address) + } + } + return nil +} + func (s *DialerSource) Update(c Config) error { s.guard.Lock() defer s.guard.Unlock() diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 60b5b4b97..2313222f0 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -4,10 +4,12 @@ import ( "context" "errors" "fmt" + "net" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" + internalNet "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" @@ -46,6 +48,8 @@ type cfg struct { switchInterval time.Duration morphCacheMetrics metrics.MorphCacheMetrics + + dialerSource *internalNet.DialerSource } const ( @@ -153,10 +157,15 @@ func (c *Client) newCli(ctx context.Context, endpoint Endpoint) (*rpcclient.WSCl if err != nil { return nil, nil, fmt.Errorf("read mtls certificates: %w", err) } + var netDialContext func(ctx context.Context, network, addr string) (net.Conn, error) + if c.cfg.dialerSource != nil { // TODO fix after IR + netDialContext = c.cfg.dialerSource.NetContextDialer() + } cli, err := rpcclient.NewWS(ctx, endpoint.Address, rpcclient.WSOptions{ Options: rpcclient.Options{ DialTimeout: c.cfg.dialTimeout, TLSClientConfig: cfg, + NetDialContext: netDialContext, }, }) if err != nil { @@ -291,3 +300,9 @@ func WithMorphCacheMetrics(morphCacheMetrics metrics.MorphCacheMetrics) Option { c.morphCacheMetrics = morphCacheMetrics } } + +func WithDialerSource(ds *internalNet.DialerSource) Option { + return func(c *cfg) { + c.dialerSource = ds + } +} From f7caef355bee787ac34f2cd44a4f9c4eb32177a3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Oct 2024 11:49:23 +0300 Subject: [PATCH 1461/1943] [#1422] node: Use dialer source for morph Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/morph.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 1bfcb8ac9..f93f233eb 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -48,6 +48,7 @@ func initMorphComponents(ctx context.Context, c *cfg) { }), client.WithSwitchInterval(morphconfig.SwitchInterval(c.appCfg)), client.WithMorphCacheMetrics(c.metricsCollector.MorphCacheMetrics()), + client.WithDialerSource(c.dialerSource), ) if err != nil { c.log.Info(logs.FrostFSNodeFailedToCreateNeoRPCClient, From ef38420623c64e12aa00b7d4072bdf6b08dec247 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 14 Oct 2024 13:51:21 +0300 Subject: [PATCH 1462/1943] [#1422] ir: Add dialer source Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/defaults.go | 10 ++++++++++ pkg/innerring/innerring.go | 29 +++++++++++++++++++++++++++++ pkg/morph/client/constructor.go | 7 +------ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index 899918d22..9b775252f 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -48,6 +48,8 @@ func defaultConfiguration(cfg *viper.Viper) { cfg.SetDefault("node.kludge_compatibility_mode", false) cfg.SetDefault("audit.enabled", false) + + setMultinetDefaults(cfg) } func setControlDefaults(cfg *viper.Viper) { @@ -131,3 +133,11 @@ func setMorphDefaults(cfg *viper.Viper) { cfg.SetDefault("morph.validators", []string{}) cfg.SetDefault("morph.switch_interval", 2*time.Minute) } + +func setMultinetDefaults(cfg *viper.Viper) { + cfg.SetDefault("multinet.enabled", false) + cfg.SetDefault("multinet.balancer", "") + cfg.SetDefault("multinet.restrict", false) + cfg.SetDefault("multinet.fallback_delay", "0s") + cfg.SetDefault("multinet.subnets", "") +} diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 53a07e36c..a4a52edec 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" + internalNet "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" @@ -486,6 +487,12 @@ func createClient(ctx context.Context, p *chainParams, errChan chan<- error) (*c return nil, fmt.Errorf("%s chain client endpoints not provided", p.name) } + nc := parseMultinetConfig(p.cfg) + ds, err := internalNet.NewDialerSource(nc) + if err != nil { + return nil, fmt.Errorf("dialer source: %w", err) + } + return client.New( ctx, p.key, @@ -498,6 +505,7 @@ func createClient(ctx context.Context, p *chainParams, errChan chan<- error) (*c }), client.WithSwitchInterval(p.cfg.GetDuration(p.name+".switch_interval")), client.WithMorphCacheMetrics(p.morphCacheMetric), + client.WithDialerSource(ds), ) } @@ -542,6 +550,27 @@ func parseWalletAddressesFromStrings(wallets []string) ([]util.Uint160, error) { return extraWallets, nil } +func parseMultinetConfig(cfg *viper.Viper) internalNet.Config { + nc := internalNet.Config{ + Enabled: cfg.GetBool("multinet.enabled"), + Balancer: cfg.GetString("multinet.balancer"), + Restrict: cfg.GetBool("multinet.restrict"), + FallbackDelay: cfg.GetDuration("multinet.fallback_delay"), + } + for i := 0; ; i++ { + mask := cfg.GetString(fmt.Sprintf("multinet.subnets.%d.mask", i)) + if mask == "" { + break + } + sourceIPs := cfg.GetStringSlice(fmt.Sprintf("multinet.subnets.%d.source_ips", i)) + nc.Subnets = append(nc.Subnets, internalNet.Subnet{ + Prefix: mask, + SourceIPs: sourceIPs, + }) + } + return nc +} + func (s *Server) initConfigFromBlockchain() error { // get current epoch epoch, err := s.netmapClient.Epoch() diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 2313222f0..a8efa76e7 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "net" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -157,15 +156,11 @@ func (c *Client) newCli(ctx context.Context, endpoint Endpoint) (*rpcclient.WSCl if err != nil { return nil, nil, fmt.Errorf("read mtls certificates: %w", err) } - var netDialContext func(ctx context.Context, network, addr string) (net.Conn, error) - if c.cfg.dialerSource != nil { // TODO fix after IR - netDialContext = c.cfg.dialerSource.NetContextDialer() - } cli, err := rpcclient.NewWS(ctx, endpoint.Address, rpcclient.WSOptions{ Options: rpcclient.Options{ DialTimeout: c.cfg.dialTimeout, TLSClientConfig: cfg, - NetDialContext: netDialContext, + NetDialContext: c.cfg.dialerSource.NetContextDialer(), }, }) if err != nil { From 2d064d0bd87c9627aaeacb9c160cfe2bb029c56a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 14 Oct 2024 14:22:27 +0300 Subject: [PATCH 1463/1943] [#1422] morph: Resolve funlen linter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/morph.go | 84 ++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index f93f233eb..197e50371 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -29,6 +29,50 @@ const ( ) func initMorphComponents(ctx context.Context, c *cfg) { + initMorphClient(ctx, c) + + lookupScriptHashesInNNS(c) // smart contract auto negotiation + + if c.cfgMorph.notaryEnabled { + err := c.cfgMorph.client.EnableNotarySupport( + client.WithProxyContract( + c.cfgMorph.proxyScriptHash, + ), + ) + fatalOnErr(err) + } + + c.log.Info(logs.FrostFSNodeNotarySupport, + zap.Bool("sidechain_enabled", c.cfgMorph.notaryEnabled), + ) + + wrap, err := nmClient.NewFromMorph(c.cfgMorph.client, c.cfgNetmap.scriptHash, 0, nmClient.TryNotary()) + fatalOnErr(err) + + var netmapSource netmap.Source + + c.cfgMorph.containerCacheSize = morphconfig.ContainerCacheSize(c.appCfg) + c.cfgMorph.cacheTTL = morphconfig.CacheTTL(c.appCfg) + + if c.cfgMorph.cacheTTL == 0 { + msPerBlock, err := c.cfgMorph.client.MsPerBlock() + fatalOnErr(err) + c.cfgMorph.cacheTTL = time.Duration(msPerBlock) * time.Millisecond + c.log.Debug(logs.FrostFSNodeMorphcacheTTLFetchedFromNetwork, zap.Duration("value", c.cfgMorph.cacheTTL)) + } + + if c.cfgMorph.cacheTTL < 0 { + netmapSource = wrap + } else { + // use RPC node as source of netmap (with caching) + netmapSource = newCachedNetmapStorage(c.cfgNetmap.state, wrap) + } + + c.netMapSource = netmapSource + c.cfgNetmap.wrapper = wrap +} + +func initMorphClient(ctx context.Context, c *cfg) { addresses := morphconfig.RPCEndpoint(c.appCfg) // Morph client stable-sorts endpoints by priority. Shuffle here to randomize @@ -70,46 +114,6 @@ func initMorphComponents(ctx context.Context, c *cfg) { c.cfgMorph.client = cli c.cfgMorph.notaryEnabled = cli.ProbeNotary() - - lookupScriptHashesInNNS(c) // smart contract auto negotiation - - if c.cfgMorph.notaryEnabled { - err = c.cfgMorph.client.EnableNotarySupport( - client.WithProxyContract( - c.cfgMorph.proxyScriptHash, - ), - ) - fatalOnErr(err) - } - - c.log.Info(logs.FrostFSNodeNotarySupport, - zap.Bool("sidechain_enabled", c.cfgMorph.notaryEnabled), - ) - - wrap, err := nmClient.NewFromMorph(c.cfgMorph.client, c.cfgNetmap.scriptHash, 0, nmClient.TryNotary()) - fatalOnErr(err) - - var netmapSource netmap.Source - - c.cfgMorph.containerCacheSize = morphconfig.ContainerCacheSize(c.appCfg) - c.cfgMorph.cacheTTL = morphconfig.CacheTTL(c.appCfg) - - if c.cfgMorph.cacheTTL == 0 { - msPerBlock, err := c.cfgMorph.client.MsPerBlock() - fatalOnErr(err) - c.cfgMorph.cacheTTL = time.Duration(msPerBlock) * time.Millisecond - c.log.Debug(logs.FrostFSNodeMorphcacheTTLFetchedFromNetwork, zap.Duration("value", c.cfgMorph.cacheTTL)) - } - - if c.cfgMorph.cacheTTL < 0 { - netmapSource = wrap - } else { - // use RPC node as source of netmap (with caching) - netmapSource = newCachedNetmapStorage(c.cfgNetmap.state, wrap) - } - - c.netMapSource = netmapSource - c.cfgNetmap.wrapper = wrap } func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) { From c0a2f20eee5e6279e71cfc3d83f8bb89d0a2ef75 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 15 Oct 2024 14:46:00 +0300 Subject: [PATCH 1464/1943] [#1422] multinet: Add metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 11 +++++++---- internal/metrics/consts.go | 2 ++ internal/metrics/innerring.go | 6 ++++++ internal/metrics/multinet.go | 35 +++++++++++++++++++++++++++++++++ internal/metrics/node.go | 6 ++++++ internal/net/config.go | 3 +++ internal/net/event_handler.go | 29 +++++++++++++++++++++++++++ pkg/innerring/initialization.go | 1 + pkg/innerring/innerring.go | 6 ++++-- 9 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 internal/metrics/multinet.go create mode 100644 internal/net/event_handler.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d44597857..9d2b77210 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -764,7 +764,9 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt persistate, err := state.NewPersistentStorage(nodeconfig.PersistentState(appCfg).Path()) fatalOnErr(err) - ds, err := internalNet.NewDialerSource(internalNetConfig(appCfg)) + nodeMetrics := metrics.NewNodeMetrics() + + ds, err := internalNet.NewDialerSource(internalNetConfig(appCfg, nodeMetrics.MultinetMetrics())) fatalOnErr(err) cacheOpts := cache.ClientCacheOpts{ @@ -785,17 +787,18 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt bgClientCache: cache.NewSDKClientCache(cacheOpts), putClientCache: cache.NewSDKClientCache(cacheOpts), persistate: persistate, - metricsCollector: metrics.NewNodeMetrics(), + metricsCollector: nodeMetrics, dialerSource: ds, } } -func internalNetConfig(appCfg *config.Config) internalNet.Config { +func internalNetConfig(appCfg *config.Config, m metrics.MultinetMetrics) internalNet.Config { result := internalNet.Config{ Enabled: multinet.Enabled(appCfg), Balancer: multinet.Balancer(appCfg), Restrict: multinet.Restrict(appCfg), FallbackDelay: multinet.FallbackDelay(appCfg), + Metrics: m, } sn := multinet.Subnets(appCfg) for _, s := range sn { @@ -1362,7 +1365,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { } } - if err := c.dialerSource.Update(internalNetConfig(c.appCfg)); err != nil { + if err := c.dialerSource.Update(internalNetConfig(c.appCfg, c.metricsCollector.MultinetMetrics())); err != nil { c.log.Error(logs.FailedToUpdateMultinetConfiguration, zap.Error(err)) return } diff --git a/internal/metrics/consts.go b/internal/metrics/consts.go index 3aa51c0f0..cb165de69 100644 --- a/internal/metrics/consts.go +++ b/internal/metrics/consts.go @@ -22,6 +22,7 @@ const ( grpcServerSubsystem = "grpc_server" policerSubsystem = "policer" commonCacheSubsystem = "common_cache" + multinetSubsystem = "multinet" successLabel = "success" shardIDLabel = "shard_id" @@ -41,6 +42,7 @@ const ( endpointLabel = "endpoint" hitLabel = "hit" cacheLabel = "cache" + sourceIPLabel = "source_ip" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" diff --git a/internal/metrics/innerring.go b/internal/metrics/innerring.go index f3f529d05..d0cb8131f 100644 --- a/internal/metrics/innerring.go +++ b/internal/metrics/innerring.go @@ -17,6 +17,7 @@ type InnerRingServiceMetrics struct { eventDuration *prometheus.HistogramVec morphCacheMetrics *morphCacheMetrics logMetrics logger.LogMetrics + multinet *multinetMetrics // nolint: unused appInfo *ApplicationInfo } @@ -51,6 +52,7 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { morphCacheMetrics: newMorphCacheMetrics(innerRingNamespace), appInfo: NewApplicationInfo(misc.Version), logMetrics: logger.NewLogMetrics(innerRingNamespace), + multinet: newMultinetMetrics(innerRingNamespace), } } @@ -78,3 +80,7 @@ func (m *InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { func (m *InnerRingServiceMetrics) LogMetrics() logger.LogMetrics { return m.logMetrics } + +func (m *InnerRingServiceMetrics) Multinet() MultinetMetrics { + return m.multinet +} diff --git a/internal/metrics/multinet.go b/internal/metrics/multinet.go new file mode 100644 index 000000000..6b1f99d46 --- /dev/null +++ b/internal/metrics/multinet.go @@ -0,0 +1,35 @@ +package metrics + +import ( + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type multinetMetrics struct { + dials *prometheus.GaugeVec +} + +type MultinetMetrics interface { + Dial(sourceIP string, success bool) +} + +func newMultinetMetrics(ns string) *multinetMetrics { + return &multinetMetrics{ + dials: metrics.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: ns, + Subsystem: multinetSubsystem, + Name: "dial_count", + Help: "Dials count performed by multinet", + }, []string{sourceIPLabel, successLabel}), + } +} + +func (m *multinetMetrics) Dial(sourceIP string, success bool) { + m.dials.With(prometheus.Labels{ + sourceIPLabel: sourceIP, + successLabel: strconv.FormatBool(success), + }).Inc() +} diff --git a/internal/metrics/node.go b/internal/metrics/node.go index 711387875..4ea3c7c24 100644 --- a/internal/metrics/node.go +++ b/internal/metrics/node.go @@ -25,6 +25,7 @@ type NodeMetrics struct { morphClient *morphClientMetrics morphCache *morphCacheMetrics log logger.LogMetrics + multinet *multinetMetrics // nolint: unused appInfo *ApplicationInfo } @@ -53,6 +54,7 @@ func NewNodeMetrics() *NodeMetrics { morphCache: newMorphCacheMetrics(namespace), log: logger.NewLogMetrics(namespace), appInfo: NewApplicationInfo(misc.Version), + multinet: newMultinetMetrics(namespace), } } @@ -120,3 +122,7 @@ func (m *NodeMetrics) MorphCacheMetrics() MorphCacheMetrics { func (m *NodeMetrics) LogMetrics() logger.LogMetrics { return m.log } + +func (m *NodeMetrics) MultinetMetrics() MultinetMetrics { + return m.multinet +} diff --git a/internal/net/config.go b/internal/net/config.go index 10450db23..b84ac3b35 100644 --- a/internal/net/config.go +++ b/internal/net/config.go @@ -7,6 +7,7 @@ import ( "slices" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/multinet" ) @@ -23,6 +24,7 @@ type Config struct { Balancer string Restrict bool FallbackDelay time.Duration + Metrics metrics.MultinetMetrics } func (c Config) toMultinetConfig() (multinet.Config, error) { @@ -52,6 +54,7 @@ func (c Config) toMultinetConfig() (multinet.Config, error) { Restrict: c.Restrict, FallbackDelay: c.FallbackDelay, Dialer: newDefaulDialer(), + EventHandler: newEventHandler(c.Metrics), }, nil } diff --git a/internal/net/event_handler.go b/internal/net/event_handler.go new file mode 100644 index 000000000..024e5cf7c --- /dev/null +++ b/internal/net/event_handler.go @@ -0,0 +1,29 @@ +package net + +import ( + "net" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" + "git.frostfs.info/TrueCloudLab/multinet" +) + +var _ multinet.EventHandler = (*metricsEventHandler)(nil) + +type metricsEventHandler struct { + m metrics.MultinetMetrics +} + +func (m *metricsEventHandler) DialPerformed(sourceIP net.Addr, _ string, _ string, err error) { + sourceIPString := "undefined" + if sourceIP != nil { + sourceIPString = sourceIP.Network() + "://" + sourceIP.String() + } + m.m.Dial(sourceIPString, err == nil) +} + +func newEventHandler(m metrics.MultinetMetrics) multinet.EventHandler { + if m == nil { + return nil + } + return &metricsEventHandler{m: m} +} diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index c4aaeda56..cb0654b6e 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -463,6 +463,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- name: morphPrefix, from: fromSideChainBlock, morphCacheMetric: s.irMetrics.MorphCacheMetrics(), + multinetMetrics: s.irMetrics.Multinet(), } // create morph client diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index a4a52edec..b94312645 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -117,6 +117,7 @@ type ( sgn *transaction.Signer from uint32 // block height morphCacheMetric metrics.MorphCacheMetrics + multinetMetrics metrics.MultinetMetrics } ) @@ -487,7 +488,7 @@ func createClient(ctx context.Context, p *chainParams, errChan chan<- error) (*c return nil, fmt.Errorf("%s chain client endpoints not provided", p.name) } - nc := parseMultinetConfig(p.cfg) + nc := parseMultinetConfig(p.cfg, p.multinetMetrics) ds, err := internalNet.NewDialerSource(nc) if err != nil { return nil, fmt.Errorf("dialer source: %w", err) @@ -550,12 +551,13 @@ func parseWalletAddressesFromStrings(wallets []string) ([]util.Uint160, error) { return extraWallets, nil } -func parseMultinetConfig(cfg *viper.Viper) internalNet.Config { +func parseMultinetConfig(cfg *viper.Viper, m metrics.MultinetMetrics) internalNet.Config { nc := internalNet.Config{ Enabled: cfg.GetBool("multinet.enabled"), Balancer: cfg.GetString("multinet.balancer"), Restrict: cfg.GetBool("multinet.restrict"), FallbackDelay: cfg.GetDuration("multinet.fallback_delay"), + Metrics: m, } for i := 0; ; i++ { mask := cfg.GetString(fmt.Sprintf("multinet.subnets.%d.mask", i)) From 5afea62ec0c29b8b3422412e4cd8a5452785e6fd Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 18 Oct 2024 15:15:19 +0300 Subject: [PATCH 1465/1943] [#1438] debian: Remove package scripts Signed-off-by: Evgenii Stratonikov --- Makefile | 21 +------------ debian/changelog | 5 --- debian/clean | 2 -- debian/control | 39 ----------------------- debian/copyright | 23 -------------- debian/frostfs-cli.docs | 4 --- debian/frostfs-cli.install | 3 -- debian/frostfs-cli.manpages | 1 - debian/frostfs-ir.dirs | 2 -- debian/frostfs-ir.docs | 3 -- debian/frostfs-ir.install | 1 - debian/frostfs-ir.postinst | 51 ------------------------------ debian/frostfs-ir.postrm | 40 ------------------------ debian/frostfs-ir.preinst | 34 -------------------- debian/frostfs-ir.prerm | 37 ---------------------- debian/frostfs-ir.service | 17 ---------- debian/frostfs-storage.dirs | 3 -- debian/frostfs-storage.docs | 4 --- debian/frostfs-storage.install | 1 - debian/frostfs-storage.postinst | 55 --------------------------------- debian/frostfs-storage.postrm | 40 ------------------------ debian/frostfs-storage.preinst | 34 -------------------- debian/frostfs-storage.prerm | 37 ---------------------- debian/frostfs-storage.service | 17 ---------- debian/rules | 40 ------------------------ debian/source/format | 1 - docs/building-deb-package.md | 46 --------------------------- docs/release-instruction.md | 5 --- 28 files changed, 1 insertion(+), 565 deletions(-) delete mode 100644 debian/changelog delete mode 100644 debian/clean delete mode 100644 debian/control delete mode 100644 debian/copyright delete mode 100644 debian/frostfs-cli.docs delete mode 100644 debian/frostfs-cli.install delete mode 100644 debian/frostfs-cli.manpages delete mode 100644 debian/frostfs-ir.dirs delete mode 100644 debian/frostfs-ir.docs delete mode 100644 debian/frostfs-ir.install delete mode 100755 debian/frostfs-ir.postinst delete mode 100755 debian/frostfs-ir.postrm delete mode 100755 debian/frostfs-ir.preinst delete mode 100755 debian/frostfs-ir.prerm delete mode 100644 debian/frostfs-ir.service delete mode 100644 debian/frostfs-storage.dirs delete mode 100644 debian/frostfs-storage.docs delete mode 100644 debian/frostfs-storage.install delete mode 100755 debian/frostfs-storage.postinst delete mode 100755 debian/frostfs-storage.postrm delete mode 100755 debian/frostfs-storage.preinst delete mode 100755 debian/frostfs-storage.prerm delete mode 100644 debian/frostfs-storage.service delete mode 100755 debian/rules delete mode 100644 debian/source/format delete mode 100644 docs/building-deb-package.md diff --git a/Makefile b/Makefile index 2f29ac19c..d92844bb5 100755 --- a/Makefile +++ b/Makefile @@ -27,12 +27,6 @@ DIRS = $(BIN) $(RELEASE) CMDS = $(notdir $(basename $(wildcard cmd/frostfs-*))) BINS = $(addprefix $(BIN)/, $(CMDS)) -# .deb package versioning -OS_RELEASE = $(shell lsb_release -cs) -PKG_VERSION ?= $(shell echo $(VERSION) | sed "s/^v//" | \ - sed -E "s/(.*)-(g[a-fA-F0-9]{6,8})(.*)/\1\3~\2/" | \ - sed "s/-/~/")-${OS_RELEASE} - OUTPUT_LINT_DIR ?= $(abspath $(BIN))/linters LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION) TMP_DIR := .cache @@ -58,7 +52,7 @@ LOCODE_DB_PATH=$(abspath ./.cache/locode_db) LOCODE_DB_VERSION=v0.4.0 .PHONY: help all images dep clean fmts fumpt imports test lint docker/lint - prepare-release debpackage pre-commit unpre-commit + prepare-release pre-commit unpre-commit # To build a specific binary, use it's name prefix with bin/ as a target # For example `make bin/frostfs-node` will build only storage node binary @@ -263,19 +257,6 @@ clean: rm -rf $(BIN) rm -rf $(RELEASE) -# Package for Debian -debpackage: - dch -b --package frostfs-node \ - --controlmaint \ - --newversion $(PKG_VERSION) \ - --distribution $(OS_RELEASE) \ - "Please see CHANGELOG.md for code changes for $(VERSION)" - dpkg-buildpackage --no-sign -b - -# Cleanup deb package build directories -debclean: - dh clean - # Download locode database locode-download: mkdir -p $(TMP_DIR) diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index 47328c419..000000000 --- a/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -frostfs-node (0.0.1) stable; urgency=medium - - * Initial package build - - -- TrueCloudLab Tue, 25 Oct 2022 21:10:49 +0300 diff --git a/debian/clean b/debian/clean deleted file mode 100644 index 44dc05e0a..000000000 --- a/debian/clean +++ /dev/null @@ -1,2 +0,0 @@ -man/ -debian/*.bash-completion diff --git a/debian/control b/debian/control deleted file mode 100644 index f3f214bca..000000000 --- a/debian/control +++ /dev/null @@ -1,39 +0,0 @@ -Source: frostfs-node -Section: misc -Priority: optional -Maintainer: TrueCloudLab -Build-Depends: debhelper-compat (= 13), dh-sequence-bash-completion, devscripts -Standards-Version: 4.5.1 -Homepage: https://fs.neo.org/ -Vcs-Git: https://git.frostfs.info/TrueCloudLab/frostfs-node.git -Vcs-Browser: https://git.frostfs.info/TrueCloudLab/frostfs-node - -Package: frostfs-storage -Architecture: any -Depends: ${misc:Depends} -Description: FrostFS Storage node - FrostFS is a decentralized distributed object storage integrated with the NEO - Blockchain. FrostFS Nodes are organized in a peer-to-peer network that takes care - of storing and distributing user's data. Any Neo user may participate in the - network and get paid for providing storage resources to other users or store - their data in FrostFS and pay a competitive price for it. - -Package: frostfs-ir -Architecture: any -Depends: ${misc:Depends}, frostfs-locode-db -Description: FrostFS InnerRing node - FrostFS is a decentralized distributed object storage integrated with the NEO - Blockchain. FrostFS Nodes are organized in a peer-to-peer network that takes care - of storing and distributing user's data. Any Neo user may participate in the - network and get paid for providing storage resources to other users or store - their data in FrostFS and pay a competitive price for it. - -Package: frostfs-cli -Architecture: any -Depends: ${misc:Depends} -Description: CLI tools for FrostFS - FrostFS is a decentralized distributed object storage integrated with the NEO - Blockchain. FrostFS Nodes are organized in a peer-to-peer network that takes care - of storing and distributing user's data. Any Neo user may participate in the - network and get paid for providing storage resources to other users or store - their data in FrostFS and pay a competitive price for it. diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index 61dab665d..000000000 --- a/debian/copyright +++ /dev/null @@ -1,23 +0,0 @@ -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: frostfs-node -Upstream-Contact: tech@frostfs.info -Source: https://git.frostfs.info/TrueCloudLab/frostfs-node - -Files: * -Copyright: 2022-2023 TrueCloudLab (@TrueCloudLab), contributors of FrostFS project - 2018-2022 NeoSPCC (@nspcc-dev), contributors of NeoFS project - (https://git.frostfs.info/TrueCloudLab/frostfs-node/src/branch/master/CREDITS.md) - -License: GPL-3 - This program is free software: you can redistribute it and/or modify it - under the terms of the GNU General Public License as published - by the Free Software Foundation; version 3. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program or at /usr/share/common-licenses/GPL-3 - If not, see . diff --git a/debian/frostfs-cli.docs b/debian/frostfs-cli.docs deleted file mode 100644 index 58d4559cc..000000000 --- a/debian/frostfs-cli.docs +++ /dev/null @@ -1,4 +0,0 @@ -CONTRIBUTING.md -CREDITS.md -README.md -cmd/frostfs-adm/docs diff --git a/debian/frostfs-cli.install b/debian/frostfs-cli.install deleted file mode 100644 index 93025187b..000000000 --- a/debian/frostfs-cli.install +++ /dev/null @@ -1,3 +0,0 @@ -bin/frostfs-adm usr/bin -bin/frostfs-cli usr/bin -bin/frostfs-lens usr/bin diff --git a/debian/frostfs-cli.manpages b/debian/frostfs-cli.manpages deleted file mode 100644 index 85c5e001d..000000000 --- a/debian/frostfs-cli.manpages +++ /dev/null @@ -1 +0,0 @@ -man/* diff --git a/debian/frostfs-ir.dirs b/debian/frostfs-ir.dirs deleted file mode 100644 index 90da8fd27..000000000 --- a/debian/frostfs-ir.dirs +++ /dev/null @@ -1,2 +0,0 @@ -/etc/frostfs/ir -/var/lib/frostfs/ir diff --git a/debian/frostfs-ir.docs b/debian/frostfs-ir.docs deleted file mode 100644 index 38b0cef26..000000000 --- a/debian/frostfs-ir.docs +++ /dev/null @@ -1,3 +0,0 @@ -CONTRIBUTING.md -CREDITS.md -README.md diff --git a/debian/frostfs-ir.install b/debian/frostfs-ir.install deleted file mode 100644 index e052f5434..000000000 --- a/debian/frostfs-ir.install +++ /dev/null @@ -1 +0,0 @@ -bin/frostfs-ir usr/bin diff --git a/debian/frostfs-ir.postinst b/debian/frostfs-ir.postinst deleted file mode 100755 index eb9d381c9..000000000 --- a/debian/frostfs-ir.postinst +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `configure' -# * `abort-upgrade' -# * `abort-remove' `in-favour' -# -# * `abort-remove' -# * `abort-deconfigure' `in-favour' -# `removing' -# -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - -case "$1" in - configure) - USERNAME=ir - id -u frostfs-ir >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /var/lib/frostfs/ir --system -M -U -c "FrostFS InnerRing node" frostfs-ir - if ! dpkg-statoverride --list /etc/frostfs/$USERNAME >/dev/null; then - chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME - chmod -f 0750 /etc/frostfs/$USERNAME - chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/config.yml - chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/control.yml - chmod -f 0640 /etc/frostfs/$USERNAME/config.yml || true - chmod -f 0640 /etc/frostfs/$USERNAME/control.yml || true - fi - USERDIR="$(getent passwd frostfs-$USERNAME | cut -d: -f6)" - if ! dpkg-statoverride --list frostfs-"$USERDIR" >/dev/null; then - chown -f frostfs-$USERNAME: "$USERDIR" - fi - ;; - - abort-upgrade|abort-remove|abort-deconfigure) - ;; - - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/frostfs-ir.postrm b/debian/frostfs-ir.postrm deleted file mode 100755 index cbb7db2f2..000000000 --- a/debian/frostfs-ir.postrm +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `remove' -# * `purge' -# * `upgrade' -# * `failed-upgrade' -# * `abort-install' -# * `abort-install' -# * `abort-upgrade' -# * `disappear' -# -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - purge) - rm -rf /var/lib/frostfs/ir/* - ;; - - remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) - ;; - - *) - echo "postrm called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/frostfs-ir.preinst b/debian/frostfs-ir.preinst deleted file mode 100755 index 37f952537..000000000 --- a/debian/frostfs-ir.preinst +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `install' -# * `install' -# * `upgrade' -# * `abort-upgrade' -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - install|upgrade) - ;; - - abort-upgrade) - ;; - - *) - echo "preinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/frostfs-ir.prerm b/debian/frostfs-ir.prerm deleted file mode 100755 index 0da369d75..000000000 --- a/debian/frostfs-ir.prerm +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `remove' -# * `upgrade' -# * `failed-upgrade' -# * `remove' `in-favour' -# * `deconfigure' `in-favour' -# `removing' -# -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - remove|upgrade|deconfigure) - ;; - - failed-upgrade) - ;; - - *) - echo "prerm called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/frostfs-ir.service b/debian/frostfs-ir.service deleted file mode 100644 index 304017f68..000000000 --- a/debian/frostfs-ir.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=FrostFS InnerRing node -Requires=network.target - -[Service] -Type=notify -NotifyAccess=all -ExecStart=/usr/bin/frostfs-ir --config /etc/frostfs/ir/config.yml -User=frostfs-ir -Group=frostfs-ir -WorkingDirectory=/var/lib/frostfs/ir -Restart=always -RestartSec=5 -PrivateTmp=true - -[Install] -WantedBy=multi-user.target diff --git a/debian/frostfs-storage.dirs b/debian/frostfs-storage.dirs deleted file mode 100644 index 4142145ee..000000000 --- a/debian/frostfs-storage.dirs +++ /dev/null @@ -1,3 +0,0 @@ -/etc/frostfs/storage -/srv/frostfs -/var/lib/frostfs/storage diff --git a/debian/frostfs-storage.docs b/debian/frostfs-storage.docs deleted file mode 100644 index cd1f5f23f..000000000 --- a/debian/frostfs-storage.docs +++ /dev/null @@ -1,4 +0,0 @@ -docs/storage-node-configuration.md -CONTRIBUTING.md -CREDITS.md -README.md diff --git a/debian/frostfs-storage.install b/debian/frostfs-storage.install deleted file mode 100644 index 670935e7b..000000000 --- a/debian/frostfs-storage.install +++ /dev/null @@ -1 +0,0 @@ -bin/frostfs-node usr/bin diff --git a/debian/frostfs-storage.postinst b/debian/frostfs-storage.postinst deleted file mode 100755 index 88fa53be5..000000000 --- a/debian/frostfs-storage.postinst +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `configure' -# * `abort-upgrade' -# * `abort-remove' `in-favour' -# -# * `abort-remove' -# * `abort-deconfigure' `in-favour' -# `removing' -# -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - -case "$1" in - configure) - USERNAME=storage - id -u frostfs-$USERNAME >/dev/null 2>&1 || useradd -s /usr/sbin/nologin -d /var/lib/frostfs/$USERNAME --system -M -U -c "FrostFS Storage node" frostfs-$USERNAME - if ! dpkg-statoverride --list /etc/frostfs/$USERNAME >/dev/null; then - chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME - chmod -f 0750 /etc/frostfs/$USERNAME - chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/config.yml - chown -f root:frostfs-$USERNAME /etc/frostfs/$USERNAME/control.yml - chmod -f 0640 /etc/frostfs/$USERNAME/config.yml || true - chmod -f 0640 /etc/frostfs/$USERNAME/control.yml || true - fi - USERDIR=$(getent passwd frostfs-$USERNAME | cut -d: -f6) - if ! dpkg-statoverride --list frostfs-"$USERDIR" >/dev/null; then - chown -f frostfs-$USERNAME: "$USERDIR" - fi - USERDIR=/srv/frostfs - if ! dpkg-statoverride --list frostfs-$USERDIR >/dev/null; then - chown -f frostfs-$USERNAME: $USERDIR - fi - ;; - - abort-upgrade|abort-remove|abort-deconfigure) - ;; - - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/frostfs-storage.postrm b/debian/frostfs-storage.postrm deleted file mode 100755 index d9c8c9656..000000000 --- a/debian/frostfs-storage.postrm +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `remove' -# * `purge' -# * `upgrade' -# * `failed-upgrade' -# * `abort-install' -# * `abort-install' -# * `abort-upgrade' -# * `disappear' -# -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - purge) - rm -rf /var/lib/frostfs/storage/* - ;; - - remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) - ;; - - *) - echo "postrm called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/frostfs-storage.preinst b/debian/frostfs-storage.preinst deleted file mode 100755 index 37f952537..000000000 --- a/debian/frostfs-storage.preinst +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `install' -# * `install' -# * `upgrade' -# * `abort-upgrade' -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - install|upgrade) - ;; - - abort-upgrade) - ;; - - *) - echo "preinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/frostfs-storage.prerm b/debian/frostfs-storage.prerm deleted file mode 100755 index 0da369d75..000000000 --- a/debian/frostfs-storage.prerm +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `remove' -# * `upgrade' -# * `failed-upgrade' -# * `remove' `in-favour' -# * `deconfigure' `in-favour' -# `removing' -# -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - remove|upgrade|deconfigure) - ;; - - failed-upgrade) - ;; - - *) - echo "prerm called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/frostfs-storage.service b/debian/frostfs-storage.service deleted file mode 100644 index 573961756..000000000 --- a/debian/frostfs-storage.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=FrostFS Storage node -Requires=network.target - -[Service] -Type=notify -NotifyAccess=all -ExecStart=/usr/bin/frostfs-node --config /etc/frostfs/storage/config.yml -User=frostfs-storage -Group=frostfs-storage -WorkingDirectory=/srv/frostfs -Restart=always -RestartSec=5 -PrivateTmp=true - -[Install] -WantedBy=multi-user.target diff --git a/debian/rules b/debian/rules deleted file mode 100755 index 0dd8ee399..000000000 --- a/debian/rules +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/make -f - -# Do not try to strip Go binaries -export DEB_BUILD_OPTIONS := nostrip - -%: - dh $@ --with bash-completion - -override_dh_auto_test: - -override_dh_auto_install: - echo $(DEB_BUILD_OPTIONS) - dh_auto_install - - bin/frostfs-adm gendoc --type man man/ - bin/frostfs-cli gendoc --type man man/ - - bin/frostfs-adm completion bash > debian/frostfs-adm.bash-completion - bin/frostfs-cli completion bash > debian/frostfs-cli.bash-completion - install -m 0755 -d debian/frostfs-cli/usr/share/fish/completions/ - install -m 0755 -d debian/frostfs-cli/usr/share/zsh/vendor-completions/ - bin/frostfs-adm completion fish > debian/frostfs-cli/usr/share/fish/completions/frostfs-adm.fish - bin/frostfs-adm completion zsh > debian/frostfs-cli/usr/share/zsh/vendor-completions/_frostfs-adm - bin/frostfs-cli completion fish > debian/frostfs-cli/usr/share/fish/completions/frostfs-cli.fish - bin/frostfs-cli completion zsh > debian/frostfs-cli/usr/share/zsh/vendor-completions/_frostfs-cli - - install -T -m 0640 config/example/ir.yaml debian/frostfs-ir/etc/frostfs/ir/config.yml - install -T -m 0640 config/example/ir-control.yaml debian/frostfs-ir/etc/frostfs/ir/control.yml - install -T -m 0640 config/example/node.yaml debian/frostfs-storage/etc/frostfs/storage/config.yml - install -T -m 0640 config/example/node-control.yaml debian/frostfs-storage/etc/frostfs/storage/control.yml - -override_dh_installsystemd: - dh_installsystemd --no-enable --no-start --name=frostfs-ir - dh_installsystemd --no-enable --no-start --name=frostfs-storage - -override_dh_installchangelogs: - dh_installchangelogs -k CHANGELOG.md - -override_dh_installdocs: - dh_installdocs diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 163aaf8d8..000000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (quilt) diff --git a/docs/building-deb-package.md b/docs/building-deb-package.md deleted file mode 100644 index 26a77a27f..000000000 --- a/docs/building-deb-package.md +++ /dev/null @@ -1,46 +0,0 @@ -# Building Debian package on host - -## Prerequisites - -For now, we're assuming building for Debian 11 (stable) x86_64. - -Go version 18.4 or later should already be installed, i.e. this runs -successfully: - -* `make all` - -## Installing packaging dependencies - -```shell -$ sudo apt install debhelper-compat dh-sequence-bash-completion devscripts -``` - -Warining: number of package installed is pretty large considering dependecies. - -## Package building - -```shell -$ make debpackage -``` - -## Leftovers cleaning - -```shell -$ make debclean -``` -or -```shell -$ dh clean -``` - -# Package versioning - -By default, package version is based on product version and may also contain git -tags and hashes. - -Package version could be overwritten by setting `PKG_VERSION` variable before -build, Debian package versioning rules should be respected. - -```shell -$ PKG_VERSION=0.32.0 make debpackge -``` diff --git a/docs/release-instruction.md b/docs/release-instruction.md index 3aebc8e66..d000f10d0 100644 --- a/docs/release-instruction.md +++ b/docs/release-instruction.md @@ -43,11 +43,6 @@ Write new revision number into the root `VERSION` file: $ echo ${FROSTFS_TAG_PREFIX}${FROSTFS_REVISION} > VERSION ``` -Update version in Debian package changelog file -``` -$ cat debian/changelog -``` - Update the supported version of `TrueCloudLab/frostfs-contract` module in root `README.md` if needed. From ed13387c0e1406040898fc91f36c6d9370f58413 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 18 Oct 2024 15:19:29 +0300 Subject: [PATCH 1466/1943] [#1438] .docker: Use go1.23 for builders Signed-off-by: Evgenii Stratonikov --- .docker/Dockerfile.adm | 2 +- .docker/Dockerfile.ci | 2 +- .docker/Dockerfile.cli | 2 +- .docker/Dockerfile.ir | 2 +- .docker/Dockerfile.storage | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.docker/Dockerfile.adm b/.docker/Dockerfile.adm index 5d67a1d04..42aeebc48 100644 --- a/.docker/Dockerfile.adm +++ b/.docker/Dockerfile.adm @@ -1,4 +1,4 @@ -FROM golang:1.22 AS builder +FROM golang:1.23 AS builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ci b/.docker/Dockerfile.ci index e9077c831..9ddd8de59 100644 --- a/.docker/Dockerfile.ci +++ b/.docker/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM golang:1.22 +FROM golang:1.23 WORKDIR /tmp diff --git a/.docker/Dockerfile.cli b/.docker/Dockerfile.cli index 16f643b61..16f130056 100644 --- a/.docker/Dockerfile.cli +++ b/.docker/Dockerfile.cli @@ -1,4 +1,4 @@ -FROM golang:1.22 AS builder +FROM golang:1.23 AS builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.ir b/.docker/Dockerfile.ir index f2cb764e5..c119f8127 100644 --- a/.docker/Dockerfile.ir +++ b/.docker/Dockerfile.ir @@ -1,4 +1,4 @@ -FROM golang:1.22 AS builder +FROM golang:1.23 AS builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository diff --git a/.docker/Dockerfile.storage b/.docker/Dockerfile.storage index cf7f97748..854f7adea 100644 --- a/.docker/Dockerfile.storage +++ b/.docker/Dockerfile.storage @@ -1,4 +1,4 @@ -FROM golang:1.22 AS builder +FROM golang:1.23 AS builder ARG BUILD=now ARG VERSION=dev ARG REPO=repository From 8b6ec57c6147e5b784d78bc891144dd55493503d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 21 Oct 2024 14:12:02 +0300 Subject: [PATCH 1467/1943] [#1440] sdnotify: Fix status for `RELOADING` Before: ``` RELOADING=1 MONOTONIC_USEC=17951246687 STATUS=RELOADING=1 MONOTONIC_USEC=17951246687 ``` After: ``` RELOADING=1 MONOTONIC_USEC=17951246687 STATUS=RELOADING ``` Signed-off-by: Anton Nikiforov --- pkg/util/sdnotify/sdnotify.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/util/sdnotify/sdnotify.go b/pkg/util/sdnotify/sdnotify.go index 22549bc96..bd15d0e8f 100644 --- a/pkg/util/sdnotify/sdnotify.go +++ b/pkg/util/sdnotify/sdnotify.go @@ -59,6 +59,8 @@ func FlagAndStatus(status string) error { return fmt.Errorf("clock_gettime: %w", err) } status += "\nMONOTONIC_USEC=" + strconv.FormatInt(ts.Nano()/1000, 10) + status += "\nSTATUS=RELOADING" + return Send(status) } status += "\nSTATUS=" + strings.TrimSuffix(status, "=1") return Send(status) From e515dd458267b8cbaa05d09b0cf55ec07f8dcf5b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 23 Oct 2024 10:39:19 +0300 Subject: [PATCH 1468/1943] [#1444] config: Fix data race on morph component init It could be called for every shard on metabase resync concurrently and it is possible to get state with initialized client but not initialized contract hashes. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/accounting.go | 4 +--- cmd/frostfs-node/config.go | 8 ++++---- cmd/frostfs-node/morph.go | 8 +++++++- cmd/frostfs-node/netmap.go | 4 +--- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cmd/frostfs-node/accounting.go b/cmd/frostfs-node/accounting.go index ec737f8a0..1d065c227 100644 --- a/cmd/frostfs-node/accounting.go +++ b/cmd/frostfs-node/accounting.go @@ -13,9 +13,7 @@ import ( ) func initAccountingService(ctx context.Context, c *cfg) { - if c.cfgMorph.client == nil { - initMorphComponents(ctx, c) - } + c.initMorphComponents(ctx) balanceMorphWrapper, err := balance.NewFromMorph(c.cfgMorph.client, c.cfgAccounting.scriptHash, 0) fatalOnErr(err) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 9d2b77210..cf7e0da7e 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -575,6 +575,9 @@ func (c *cfgGRPC) dropConnection(endpoint string) { } type cfgMorph struct { + initialized bool + guard sync.Mutex + client *client.Client notaryEnabled bool @@ -1455,10 +1458,7 @@ func (c *cfg) createTombstoneSource() *tombstone.ExpirationChecker { func (c *cfg) createContainerInfoProvider(ctx context.Context) container.InfoProvider { return container.NewInfoProvider(func() (container.Source, error) { - // threadsafe: called on init or on sighup when morph initialized - if c.cfgMorph.client == nil { - initMorphComponents(ctx, c) - } + c.initMorphComponents(ctx) cc, err := containerClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0, containerClient.TryNotary()) if err != nil { return nil, err diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 197e50371..e85209059 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -28,7 +28,12 @@ const ( notaryDepositRetriesAmount = 300 ) -func initMorphComponents(ctx context.Context, c *cfg) { +func (c *cfg) initMorphComponents(ctx context.Context) { + c.cfgMorph.guard.Lock() + defer c.cfgMorph.guard.Unlock() + if c.cfgMorph.initialized { + return + } initMorphClient(ctx, c) lookupScriptHashesInNNS(c) // smart contract auto negotiation @@ -70,6 +75,7 @@ func initMorphComponents(ctx context.Context, c *cfg) { c.netMapSource = netmapSource c.cfgNetmap.wrapper = wrap + c.cfgMorph.initialized = true } func initMorphClient(ctx context.Context, c *cfg) { diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 5e4585f85..0e0571760 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -143,9 +143,7 @@ func initNetmapService(ctx context.Context, c *cfg) { parseAttributes(c) c.cfgNodeInfo.localInfo.SetStatus(netmapSDK.Offline) - if c.cfgMorph.client == nil { - initMorphComponents(ctx, c) - } + c.initMorphComponents(ctx) initNetmapState(c) From 6f798b9c4b4e244a0161e50dd5ee2e647835a949 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Tue, 22 Oct 2024 10:18:01 +0300 Subject: [PATCH 1469/1943] [#1441] cli: Use `grpc.WaitForReady` while initializing SDK client Before, when the target RPC server was unavailable, requests made by CLI didn't wait for a timeout specified by the `--timeout` option if the timeout was more than 20 seconds. It's because of the gRPC default backoff strategy. Adding this option fixes that behavior. Signed-off-by: Aleksey Savchuk --- cmd/frostfs-cli/internal/client/sdk.go | 1 + cmd/frostfs-cli/modules/tree/client.go | 1 + 2 files changed, 2 insertions(+) diff --git a/cmd/frostfs-cli/internal/client/sdk.go b/cmd/frostfs-cli/internal/client/sdk.go index f7c48b871..2d9c45cbd 100644 --- a/cmd/frostfs-cli/internal/client/sdk.go +++ b/cmd/frostfs-cli/internal/client/sdk.go @@ -58,6 +58,7 @@ func GetSDKClient(ctx context.Context, cmd *cobra.Command, key *ecdsa.PrivateKey GRPCDialOptions: []grpc.DialOption{ grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInteceptor()), grpc.WithChainStreamInterceptor(tracing.NewStreamClientInterceptor()), + grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), }, } if timeout := viper.GetDuration(commonflags.Timeout); timeout > 0 { diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index 4e0099f02..6891e711c 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -34,6 +34,7 @@ func _client() (tree.TreeServiceClient, error) { metrics.NewStreamClientInterceptor(), tracing.NewStreamClientInterceptor(), ), + grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), } if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") { From 9a260c2e64abd5309cce7e3e7fde93b131b3ffc0 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 23 Oct 2024 14:02:31 +0300 Subject: [PATCH 1470/1943] [#1441] network/cache: Use `grpc.WaitForReady` option when creating client Signed-off-by: Aleksey Savchuk --- pkg/network/cache/multi.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index e936ead65..2ecce3a01 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -70,6 +70,7 @@ func (x *multiClient) createForAddress(ctx context.Context, addr network.Address tracing.NewStreamClientInterceptor(), ), grpc.WithContextDialer(x.opts.DialerSource.GrpcContextDialer()), + grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), } prmDial := client.PrmDial{ From 65a4320c7539f4915e799f652475c739b7ad58b7 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 23 Oct 2024 14:04:37 +0300 Subject: [PATCH 1471/1943] [#1441] services/tree: Use `grpc.WaitForReady` option when creating client Signed-off-by: Aleksey Savchuk --- pkg/services/tree/cache.go | 1 + pkg/services/tree/sync.go | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index e490cb855..ac80d0e4c 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -103,6 +103,7 @@ func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (* tracing.NewStreamClientInterceptor(), ), grpc.WithContextDialer(c.ds.GrpcContextDialer()), + grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), } if !netAddr.IsTLSEnabled() { diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 5bbc93978..ce1e72104 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -342,7 +342,9 @@ func (*Service) createConnection(a network.Address) (*grpc.ClientConn, error) { metrics.NewStreamClientInterceptor(), tracing_grpc.NewStreamClientInterceptor(), ), - grpc.WithTransportCredentials(insecure.NewCredentials())) + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + ) } // ErrAlreadySyncing is returned when a service synchronization has already From b9284604d9a19ba4151e664439c6300894c14905 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 23 Oct 2024 15:36:49 +0300 Subject: [PATCH 1472/1943] [#1442] cli/tree: Allow to specify `rpc-endpoint` with config file We have several ways to specify the `rpc-endpoint`: with a flag, with a single config file or multiple files. Before, the `rpc-endpoint` flag was marked as required. Because `cobra` checked the required flag presence first, it prevented specifying `rpc-endpoint` with a config file. Signed-off-by: Aleksey Savchuk --- cmd/frostfs-cli/modules/tree/add.go | 2 -- cmd/frostfs-cli/modules/tree/add_by_path.go | 1 - cmd/frostfs-cli/modules/tree/get_by_path.go | 2 -- cmd/frostfs-cli/modules/tree/get_op_log.go | 2 -- cmd/frostfs-cli/modules/tree/healthcheck.go | 2 -- cmd/frostfs-cli/modules/tree/list.go | 2 -- cmd/frostfs-cli/modules/tree/move.go | 2 -- cmd/frostfs-cli/modules/tree/remove.go | 2 -- cmd/frostfs-cli/modules/tree/subtree.go | 2 -- 9 files changed, 17 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/add.go b/cmd/frostfs-cli/modules/tree/add.go index 019feb0ec..e2c05d486 100644 --- a/cmd/frostfs-cli/modules/tree/add.go +++ b/cmd/frostfs-cli/modules/tree/add.go @@ -30,8 +30,6 @@ func initAddCmd() { ff := addCmd.Flags() ff.StringSlice(metaFlagKey, nil, "Meta pairs in the form of Key1=[0x]Value1,Key2=[0x]Value2") ff.Uint64(parentIDFlagKey, 0, "Parent node ID") - - _ = cobra.MarkFlagRequired(ff, commonflags.RPC) } func add(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/tree/add_by_path.go b/cmd/frostfs-cli/modules/tree/add_by_path.go index 5d5b00b7d..7263bcd0d 100644 --- a/cmd/frostfs-cli/modules/tree/add_by_path.go +++ b/cmd/frostfs-cli/modules/tree/add_by_path.go @@ -36,7 +36,6 @@ func initAddByPathCmd() { ff.String(pathFlagKey, "", "Path to a node") ff.StringSlice(metaFlagKey, nil, "Meta pairs in the form of Key1=[0x]Value1,Key2=[0x]Value2") - _ = cobra.MarkFlagRequired(ff, commonflags.RPC) _ = cobra.MarkFlagRequired(ff, pathFlagKey) } diff --git a/cmd/frostfs-cli/modules/tree/get_by_path.go b/cmd/frostfs-cli/modules/tree/get_by_path.go index 7061723fd..210630e60 100644 --- a/cmd/frostfs-cli/modules/tree/get_by_path.go +++ b/cmd/frostfs-cli/modules/tree/get_by_path.go @@ -36,8 +36,6 @@ func initGetByPathCmd() { ff.String(pathFlagKey, "", "Path to a node") ff.Bool(latestOnlyFlagKey, false, "Look only for the latest version of a node") - - _ = cobra.MarkFlagRequired(ff, commonflags.RPC) } func getByPath(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/tree/get_op_log.go b/cmd/frostfs-cli/modules/tree/get_op_log.go index 376aa8e8d..9d767ab3e 100644 --- a/cmd/frostfs-cli/modules/tree/get_op_log.go +++ b/cmd/frostfs-cli/modules/tree/get_op_log.go @@ -30,8 +30,6 @@ func initGetOpLogCmd() { ff := getOpLogCmd.Flags() ff.Uint64(heightFlagKey, 0, "Height to start with") ff.Uint64(countFlagKey, 10, "Logged operations count") - - _ = cobra.MarkFlagRequired(ff, commonflags.RPC) } func getOpLog(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/tree/healthcheck.go b/cmd/frostfs-cli/modules/tree/healthcheck.go index b01bb2e77..c581b8e26 100644 --- a/cmd/frostfs-cli/modules/tree/healthcheck.go +++ b/cmd/frostfs-cli/modules/tree/healthcheck.go @@ -20,8 +20,6 @@ var healthcheckCmd = &cobra.Command{ func initHealthcheckCmd() { commonflags.Init(healthcheckCmd) - ff := healthcheckCmd.Flags() - _ = cobra.MarkFlagRequired(ff, commonflags.RPC) } func healthcheck(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/tree/list.go b/cmd/frostfs-cli/modules/tree/list.go index f8c0e490f..ee1db2a79 100644 --- a/cmd/frostfs-cli/modules/tree/list.go +++ b/cmd/frostfs-cli/modules/tree/list.go @@ -26,8 +26,6 @@ func initListCmd() { ff := listCmd.Flags() ff.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) _ = listCmd.MarkFlagRequired(commonflags.CIDFlag) - - _ = cobra.MarkFlagRequired(ff, commonflags.RPC) } func list(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/tree/move.go b/cmd/frostfs-cli/modules/tree/move.go index dc807d752..7a369bd02 100644 --- a/cmd/frostfs-cli/modules/tree/move.go +++ b/cmd/frostfs-cli/modules/tree/move.go @@ -33,8 +33,6 @@ func initMoveCmd() { _ = getSubtreeCmd.MarkFlagRequired(nodeIDFlagKey) _ = getSubtreeCmd.MarkFlagRequired(parentIDFlagKey) - - _ = cobra.MarkFlagRequired(ff, commonflags.RPC) } func move(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/tree/remove.go b/cmd/frostfs-cli/modules/tree/remove.go index d0b6fab2f..3c532fe26 100644 --- a/cmd/frostfs-cli/modules/tree/remove.go +++ b/cmd/frostfs-cli/modules/tree/remove.go @@ -29,8 +29,6 @@ func initRemoveCmd() { ff.Uint64(nodeIDFlagKey, 0, "Node ID.") _ = getSubtreeCmd.MarkFlagRequired(nodeIDFlagKey) - - _ = cobra.MarkFlagRequired(ff, commonflags.RPC) } func remove(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-cli/modules/tree/subtree.go b/cmd/frostfs-cli/modules/tree/subtree.go index 83a8909b6..c5f7ad401 100644 --- a/cmd/frostfs-cli/modules/tree/subtree.go +++ b/cmd/frostfs-cli/modules/tree/subtree.go @@ -34,8 +34,6 @@ func initGetSubtreeCmd() { _ = getSubtreeCmd.MarkFlagRequired(commonflags.CIDFlag) _ = getSubtreeCmd.MarkFlagRequired(treeIDFlagKey) - - _ = cobra.MarkFlagRequired(ff, commonflags.RPC) } func getSubTree(cmd *cobra.Command, _ []string) { From 29708b78d705958067a6f0d25b4e63572bf7e93d Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 23 Oct 2024 15:37:50 +0300 Subject: [PATCH 1473/1943] [#1442] cli/tree: Enchance error message if `rpc-endpoint` isn't defined Signed-off-by: Aleksey Savchuk --- cmd/frostfs-cli/modules/tree/client.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index 6891e711c..a70624ac8 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -2,6 +2,7 @@ package tree import ( "context" + "fmt" "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" @@ -20,7 +21,13 @@ import ( // after making Tree API public. func _client() (tree.TreeServiceClient, error) { var netAddr network.Address - err := netAddr.FromString(viper.GetString(commonflags.RPC)) + + rpcEndpoint := viper.GetString(commonflags.RPC) + if rpcEndpoint == "" { + return nil, fmt.Errorf("%s is not defined", commonflags.RPC) + } + + err := netAddr.FromString(rpcEndpoint) if err != nil { return nil, err } From bc8d79ddf949cb88546bc214d688a03f5ab9740e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 24 Oct 2024 09:45:57 +0300 Subject: [PATCH 1474/1943] [#1447] services/tree: Move relaying code to a separate function Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/redirect.go | 14 ++++++++ pkg/services/tree/service.go | 66 ++++------------------------------- 2 files changed, 20 insertions(+), 60 deletions(-) diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index ec41a60d4..5bde3ae38 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -12,10 +12,24 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + "google.golang.org/grpc" ) var errNoSuitableNode = errors.New("no node was found to execute the request") +func relayUnary[Req any, Resp any](ctx context.Context, s *Service, ns []netmapSDK.NodeInfo, req *Req, callback func(TreeServiceClient, context.Context, *Req, ...grpc.CallOption) (*Resp, error)) (*Resp, error) { + var resp *Resp + var outErr error + err := s.forEachNode(ctx, ns, func(c TreeServiceClient) bool { + resp, outErr = callback(c, ctx, req) + return true + }) + if err != nil { + return nil, err + } + return resp, outErr +} + // forEachNode executes callback for each node in the container until true is returned. // Returns errNoSuitableNode if there was no successful attempt to dial any node. func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo, f func(c TreeServiceClient) bool) error { diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 2cb2af294..acc2775e6 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -122,16 +122,7 @@ func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error return nil, err } if pos < 0 { - var resp *AddResponse - var outErr error - err = s.forEachNode(ctx, ns, func(c TreeServiceClient) bool { - resp, outErr = c.Add(ctx, req) - return true - }) - if err != nil { - return nil, err - } - return resp, outErr + return relayUnary(ctx, s, ns, req, (TreeServiceClient).Add) } d := pilorama.CIDDescriptor{CID: cid, Position: pos, Size: len(ns)} @@ -174,16 +165,7 @@ func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByP return nil, err } if pos < 0 { - var resp *AddByPathResponse - var outErr error - err = s.forEachNode(ctx, ns, func(c TreeServiceClient) bool { - resp, outErr = c.AddByPath(ctx, req) - return true - }) - if err != nil { - return nil, err - } - return resp, outErr + return relayUnary(ctx, s, ns, req, (TreeServiceClient).AddByPath) } meta := protoToMeta(b.GetMeta()) @@ -238,16 +220,7 @@ func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveRespon return nil, err } if pos < 0 { - var resp *RemoveResponse - var outErr error - err = s.forEachNode(ctx, ns, func(c TreeServiceClient) bool { - resp, outErr = c.Remove(ctx, req) - return true - }) - if err != nil { - return nil, err - } - return resp, outErr + return relayUnary(ctx, s, ns, req, (TreeServiceClient).Remove) } if b.GetNodeId() == pilorama.RootID { @@ -291,16 +264,7 @@ func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, er return nil, err } if pos < 0 { - var resp *MoveResponse - var outErr error - err = s.forEachNode(ctx, ns, func(c TreeServiceClient) bool { - resp, outErr = c.Move(ctx, req) - return true - }) - if err != nil { - return nil, err - } - return resp, outErr + return relayUnary(ctx, s, ns, req, (TreeServiceClient).Move) } if b.GetNodeId() == pilorama.RootID { @@ -343,16 +307,7 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) return nil, err } if pos < 0 { - var resp *GetNodeByPathResponse - var outErr error - err = s.forEachNode(ctx, ns, func(c TreeServiceClient) bool { - resp, outErr = c.GetNodeByPath(ctx, req) - return true - }) - if err != nil { - return nil, err - } - return resp, outErr + return relayUnary(ctx, s, ns, req, (TreeServiceClient).GetNodeByPath) } attr := b.GetPathAttribute() @@ -763,16 +718,7 @@ func (s *Service) TreeList(ctx context.Context, req *TreeListRequest) (*TreeList return nil, err } if pos < 0 { - var resp *TreeListResponse - var outErr error - err = s.forEachNode(ctx, ns, func(c TreeServiceClient) bool { - resp, outErr = c.TreeList(ctx, req) - return outErr == nil - }) - if err != nil { - return nil, err - } - return resp, outErr + return relayUnary(ctx, s, ns, req, (TreeServiceClient).TreeList) } ids, err := s.forest.TreeList(ctx, cid) From eb5336d5ff53f764edee4fcbd10e738db506a0e6 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 22 Oct 2024 15:47:15 +0300 Subject: [PATCH 1475/1943] [#1406] tree: Use delete verb instead put for Remove Signed-off-by: Airat Arifullin --- pkg/services/tree/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index acc2775e6..10c3b6ccc 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -210,7 +210,7 @@ func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveRespon return nil, err } - err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectPut) + err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectDelete) if err != nil { return nil, err } From 012af5cc38597d6bbd28b723a5d4190ece8c7cbb Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 28 Oct 2024 13:07:28 +0300 Subject: [PATCH 1476/1943] [#1406] tree: Add unit-tests for ape check Signed-off-by: Airat Arifullin --- pkg/services/tree/ape_test.go | 207 ++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 pkg/services/tree/ape_test.go diff --git a/pkg/services/tree/ape_test.go b/pkg/services/tree/ape_test.go new file mode 100644 index 000000000..3f94925b5 --- /dev/null +++ b/pkg/services/tree/ape_test.go @@ -0,0 +1,207 @@ +package tree + +import ( + "context" + "encoding/hex" + "fmt" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" + core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" + checkercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/common/ape" + containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine/inmemory" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/stretchr/testify/require" +) + +var ( + containerID = "73tQMTYyUkTgmvPR1HWib6pndbhSoBovbnMF7Pws8Rcy" + + senderPrivateKey, _ = keys.NewPrivateKey() + + senderKey = hex.EncodeToString(senderPrivateKey.PublicKey().Bytes()) + + rootCnr = &core.Container{Value: containerSDK.Container{}} +) + +type frostfsIDProviderMock struct { + subjects map[util.Uint160]*client.Subject + subjectsExtended map[util.Uint160]*client.SubjectExtended +} + +func (f *frostfsIDProviderMock) GetSubject(key util.Uint160) (*client.Subject, error) { + v, ok := f.subjects[key] + if !ok { + return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) + } + return v, nil +} + +func (f *frostfsIDProviderMock) GetSubjectExtended(key util.Uint160) (*client.SubjectExtended, error) { + v, ok := f.subjectsExtended[key] + if !ok { + return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) + } + return v, nil +} + +var _ frostfsidcore.SubjectProvider = (*frostfsIDProviderMock)(nil) + +func newFrostfsIDProviderMock(t *testing.T) *frostfsIDProviderMock { + return &frostfsIDProviderMock{ + subjects: map[util.Uint160]*client.Subject{ + scriptHashFromSenderKey(t, senderKey): { + Namespace: "testnamespace", + Name: "test", + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + }, + }, + subjectsExtended: map[util.Uint160]*client.SubjectExtended{ + scriptHashFromSenderKey(t, senderKey): { + Namespace: "testnamespace", + Name: "test", + KV: map[string]string{ + "tag-attr1": "value1", + "tag-attr2": "value2", + }, + Groups: []*client.Group{ + { + ID: 1, + Name: "test", + Namespace: "testnamespace", + KV: map[string]string{ + "attr1": "value1", + "attr2": "value2", + }, + }, + }, + }, + }, + } +} + +func scriptHashFromSenderKey(t *testing.T, senderKey string) util.Uint160 { + pk, err := keys.NewPublicKeyFromString(senderKey) + require.NoError(t, err) + return pk.GetScriptHash() +} + +type stMock struct{} + +func (m *stMock) CurrentEpoch() uint64 { + return 8 +} + +func TestCheckAPE(t *testing.T) { + cid := cid.ID{} + _ = cid.DecodeString(containerID) + + t.Run("put non-tombstone rule won't affect tree remove", func(t *testing.T) { + los := inmemory.NewInmemoryLocalStorage() + mcs := inmemory.NewInmemoryMorphRuleChainStorage() + fid := newFrostfsIDProviderMock(t) + s := Service{ + cfg: cfg{ + frostfsidSubjectProvider: fid, + }, + apeChecker: checkercore.New(los, mcs, fid, &stMock{}), + } + + los.AddOverride(chain.Ingress, engine.ContainerTarget(containerID), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + Condition: []chain.Condition{ + { + Op: chain.CondStringNotEquals, + Kind: chain.KindResource, + Key: nativeschema.PropertyKeyObjectType, + Value: "TOMBSTONE", + }, + }, + }, + }, + MatchType: chain.MatchTypeFirstMatch, + }) + + mcs.AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(containerID), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.Allow, + Actions: chain.Actions{Names: []string{nativeschema.MethodDeleteObject}}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + }, + }, + MatchType: chain.MatchTypeFirstMatch, + }) + + err := s.checkAPE(context.Background(), nil, rootCnr, cid, acl.OpObjectDelete, acl.RoleOwner, senderPrivateKey.PublicKey()) + require.NoError(t, err) + }) + + t.Run("delete rule won't affect tree add", func(t *testing.T) { + los := inmemory.NewInmemoryLocalStorage() + mcs := inmemory.NewInmemoryMorphRuleChainStorage() + fid := newFrostfsIDProviderMock(t) + s := Service{ + cfg: cfg{ + frostfsidSubjectProvider: fid, + }, + apeChecker: checkercore.New(los, mcs, fid, &stMock{}), + } + + los.AddOverride(chain.Ingress, engine.ContainerTarget(containerID), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.AccessDenied, + Actions: chain.Actions{Names: []string{nativeschema.MethodDeleteObject}}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + }, + }, + MatchType: chain.MatchTypeFirstMatch, + }) + + mcs.AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(containerID), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.Allow, + Actions: chain.Actions{Names: []string{nativeschema.MethodPutObject}}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + Condition: []chain.Condition{ + { + Op: chain.CondStringNotEquals, + Kind: chain.KindResource, + Key: nativeschema.PropertyKeyObjectType, + Value: "TOMBSTONE", + }, + }, + }, + }, + MatchType: chain.MatchTypeFirstMatch, + }) + + err := s.checkAPE(context.Background(), nil, rootCnr, cid, acl.OpObjectPut, acl.RoleOwner, senderPrivateKey.PublicKey()) + require.NoError(t, err) + }) +} From 3cd7d23f1095d2efe99e66a4dfe73ec4e5ff62d5 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 22 Oct 2024 10:04:34 +0300 Subject: [PATCH 1477/1943] [#1439] node: Reduce usage of `netmapAPI.NodeInfo` Remove outdated code from `netmap` service. Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config.go | 12 +++++------ pkg/services/netmap/executor.go | 36 ++++++--------------------------- 2 files changed, 11 insertions(+), 37 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index cf7e0da7e..b2dcafbd7 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1179,17 +1179,15 @@ func initObjectPool(cfg *config.Config) (pool cfgObjectRoutines) { return pool } -func (c *cfg) LocalNodeInfo() (*netmapV2.NodeInfo, error) { - var res netmapV2.NodeInfo - +func (c *cfg) LocalNodeInfo() *netmap.NodeInfo { + var res netmap.NodeInfo ni, ok := c.cfgNetmap.state.getNodeInfo() if ok { - ni.WriteToV2(&res) + res = ni } else { - c.cfgNodeInfo.localInfo.WriteToV2(&res) + res = c.cfgNodeInfo.localInfo } - - return &res, nil + return &res } // setContractNodeInfo rewrites local node info from the FrostFS network map. diff --git a/pkg/services/netmap/executor.go b/pkg/services/netmap/executor.go index ae2044246..f48357915 100644 --- a/pkg/services/netmap/executor.go +++ b/pkg/services/netmap/executor.go @@ -28,7 +28,7 @@ type executorSvc struct { type NodeState interface { // LocalNodeInfo must return current node state // in FrostFS API v2 NodeInfo structure. - LocalNodeInfo() (*netmap.NodeInfo, error) + LocalNodeInfo() *netmapSDK.NodeInfo // ReadCurrentNetMap reads current local network map of the storage node // into the given parameter. Returns any error encountered which prevented @@ -64,39 +64,15 @@ func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo, func (s *executorSvc) LocalNodeInfo( _ context.Context, - req *netmap.LocalNodeInfoRequest, + _ *netmap.LocalNodeInfoRequest, ) (*netmap.LocalNodeInfoResponse, error) { - verV2 := req.GetMetaHeader().GetVersion() - if verV2 == nil { - return nil, errors.New("missing version") - } - - var ver versionsdk.Version - if err := ver.ReadFromV2(*verV2); err != nil { - return nil, fmt.Errorf("can't read version: %w", err) - } - - ni, err := s.state.LocalNodeInfo() - if err != nil { - return nil, err - } - - if addrNum := ni.NumberOfAddresses(); addrNum > 0 && ver.Minor() <= 7 { - ni2 := new(netmap.NodeInfo) - ni2.SetPublicKey(ni.GetPublicKey()) - ni2.SetState(ni.GetState()) - ni2.SetAttributes(ni.GetAttributes()) - ni.IterateAddresses(func(s string) bool { - ni2.SetAddresses(s) - return true - }) - - ni = ni2 - } + ni := s.state.LocalNodeInfo() + var nodeInfo netmap.NodeInfo + ni.WriteToV2(&nodeInfo) body := new(netmap.LocalNodeInfoResponseBody) body.SetVersion(&s.version) - body.SetNodeInfo(ni) + body.SetNodeInfo(&nodeInfo) resp := new(netmap.LocalNodeInfoResponse) resp.SetBody(body) From 81f4cdbb91589ddab47ecde57992ef8de10e9ec0 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 22 Oct 2024 10:06:16 +0300 Subject: [PATCH 1478/1943] [#1439] object: Sort nodes by priority metrics to compute GET request Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config.go | 11 + cmd/frostfs-node/config/object/config.go | 20 ++ cmd/frostfs-node/object.go | 6 +- config/example/node.env | 1 + config/example/node.json | 3 + config/example/node.yaml | 4 + docs/storage-node-configuration.md | 14 +- .../object_manager/placement/metrics.go | 43 +++ .../object_manager/placement/traverser.go | 70 ++++- .../placement/traverser_test.go | 288 +++++++++++++++++- 10 files changed, 449 insertions(+), 11 deletions(-) create mode 100644 pkg/services/object_manager/placement/metrics.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index b2dcafbd7..800c49127 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -58,6 +58,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" objectService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/tombstone" tsourse "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/tombstone/source" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" @@ -109,6 +110,7 @@ type applicationConfiguration struct { ObjectCfg struct { tombstoneLifetime uint64 + priorityMetrics []placement.Metric } EngineCfg struct { @@ -232,6 +234,15 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { // Object a.ObjectCfg.tombstoneLifetime = objectconfig.TombstoneLifetime(c) + var pm []placement.Metric + for _, raw := range objectconfig.Get(c).Priority() { + m, err := placement.ParseMetric(raw) + if err != nil { + return err + } + pm = append(pm, m) + } + a.ObjectCfg.priorityMetrics = pm // Storage Engine diff --git a/cmd/frostfs-node/config/object/config.go b/cmd/frostfs-node/config/object/config.go index 876dc3ef1..6ff1fe2ab 100644 --- a/cmd/frostfs-node/config/object/config.go +++ b/cmd/frostfs-node/config/object/config.go @@ -10,10 +10,17 @@ type PutConfig struct { cfg *config.Config } +// GetConfig is a wrapper over "get" config section which provides access +// to object get pipeline configuration of object service. +type GetConfig struct { + cfg *config.Config +} + const ( subsection = "object" putSubsection = "put" + getSubsection = "get" // PutPoolSizeDefault is a default value of routine pool size to // process object.Put requests in object service. @@ -56,3 +63,16 @@ func (g PutConfig) PoolSizeLocal() int { func (g PutConfig) SkipSessionTokenIssuerVerification() bool { return config.BoolSafe(g.cfg, "skip_session_token_issuer_verification") } + +// Get returns structure that provides access to "get" subsection of +// "object" section. +func Get(c *config.Config) GetConfig { + return GetConfig{ + c.Sub(subsection).Sub(getSubsection), + } +} + +// Priority returns the value of "priority" config parameter. +func (g GetConfig) Priority() []string { + return config.StringSliceSafe(g.cfg, "priority") +} diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index c484c5d8c..c6bde2cff 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -178,7 +178,8 @@ func initObjectService(c *cfg) { sSearchV2 := createSearchSvcV2(sSearch, keyStorage) - sGet := createGetService(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource) + sGet := createGetService(c, keyStorage, traverseGen, c.clientCache, c.cfgObject.cnrSource, + c.ObjectCfg.priorityMetrics) *c.cfgObject.getSvc = *sGet // need smth better @@ -389,6 +390,7 @@ func createSearchSvcV2(sSearch *searchsvc.Service, keyStorage *util.KeyStorage) func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.TraverserGenerator, coreConstructor *cache.ClientCache, containerSource containercore.Source, + priorityMetrics []placement.Metric, ) *getsvc.Service { ls := c.cfgObject.cfgLocalStorage.localStorage @@ -398,6 +400,8 @@ func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Tra ls, traverseGen.WithTraverseOptions( placement.SuccessAfter(1), + placement.WithPriorityMetrics(priorityMetrics), + placement.WithNodeState(c), ), coreConstructor, containerSource, diff --git a/config/example/node.env b/config/example/node.env index 580d343fb..3979eb18f 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -87,6 +87,7 @@ FROSTFS_OBJECT_PUT_REMOTE_POOL_SIZE=100 FROSTFS_OBJECT_PUT_LOCAL_POOL_SIZE=200 FROSTFS_OBJECT_PUT_SKIP_SESSION_TOKEN_ISSUER_VERIFICATION=true FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10 +FROSTFS_OBJECT_GET_PRIORITY="$attribute:ClusterName $attribute:UN-LOCODE" # Storage engine section FROSTFS_STORAGE_SHARD_POOL_SIZE=15 diff --git a/config/example/node.json b/config/example/node.json index 3470d2d12..1ea28de6c 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -131,6 +131,9 @@ "remote_pool_size": 100, "local_pool_size": 200, "skip_session_token_issuer_verification": true + }, + "get": { + "priority": ["$attribute:ClusterName", "$attribute:UN-LOCODE"] } }, "storage": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 2a963fc0f..4a418dfcb 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -114,6 +114,10 @@ object: remote_pool_size: 100 # number of async workers for remote PUT operations local_pool_size: 200 # number of async workers for local PUT operations skip_session_token_issuer_verification: true # session token issuer verification will be skipped if true + get: + priority: # list of metrics of nodes for prioritization + - $attribute:ClusterName + - $attribute:UN-LOCODE storage: # note: shard configuration can be omitted for relay node (see `node.relay`) diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 2b94400df..363520481 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -407,13 +407,17 @@ Contains object-service related parameters. object: put: remote_pool_size: 100 + get: + priority: + - $attribute:ClusterName ``` -| Parameter | Type | Default value | Description | -|-----------------------------|-------|---------------|------------------------------------------------------------------------------------------------| -| `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | -| `put.remote_pool_size` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | -| `put.local_pool_size` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | +| Parameter | Type | Default value | Description | +|-----------------------------|------------|---------------|------------------------------------------------------------------------------------------------------| +| `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | +| `put.remote_pool_size` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | +| `put.local_pool_size` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | +| `get.priority` | `[]string` | | List of metrics of nodes for prioritization. Used for computing response on GET and SEARCH requests. | # `runtime` section Contains runtime parameters. diff --git a/pkg/services/object_manager/placement/metrics.go b/pkg/services/object_manager/placement/metrics.go new file mode 100644 index 000000000..45e6df339 --- /dev/null +++ b/pkg/services/object_manager/placement/metrics.go @@ -0,0 +1,43 @@ +package placement + +import ( + "errors" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" +) + +const ( + attrPrefix = "$attribute:" +) + +type Metric interface { + CalculateValue(*netmap.NodeInfo, *netmap.NodeInfo) int +} + +func ParseMetric(raw string) (Metric, error) { + if attr, found := strings.CutPrefix(raw, attrPrefix); found { + return NewAttributeMetric(attr), nil + } + return nil, errors.New("unsupported priority metric") +} + +// attributeMetric describes priority metric based on attribute. +type attributeMetric struct { + attribute string +} + +// CalculateValue return [0] if from and to contains attribute attributeMetric.attribute and +// the value of attribute is the same. In other case return [1]. +func (am *attributeMetric) CalculateValue(from *netmap.NodeInfo, to *netmap.NodeInfo) int { + fromAttr := from.Attribute(am.attribute) + toAttr := to.Attribute(am.attribute) + if len(fromAttr) > 0 && len(toAttr) > 0 && fromAttr == toAttr { + return 0 + } + return 1 +} + +func NewAttributeMetric(attr string) Metric { + return &attributeMetric{attribute: attr} +} diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 4e790628f..6440f187d 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -3,6 +3,7 @@ package placement import ( "errors" "fmt" + "slices" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" @@ -23,6 +24,11 @@ type Builder interface { BuildPlacement(cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) } +type NodeState interface { + // LocalNodeInfo return current node state in FrostFS API v2 NodeInfo structure. + LocalNodeInfo() *netmap.NodeInfo +} + // Option represents placement traverser option. type Option func(*cfg) @@ -50,6 +56,10 @@ type cfg struct { policy netmap.PlacementPolicy builder Builder + + metrics []Metric + + nodeState NodeState } const invalidOptsMsg = "invalid traverser options" @@ -99,7 +109,22 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } var rem []int - if cfg.flatSuccess != nil { + if len(cfg.metrics) > 0 && cfg.nodeState != nil { + rem = defaultCopiesVector(cfg.policy) + var unsortedVector []netmap.NodeInfo + var regularVector []netmap.NodeInfo + for i := range rem { + unsortedVector = append(unsortedVector, ns[i][:rem[i]]...) + regularVector = append(regularVector, ns[i][rem[i]:]...) + } + rem = []int{-1, -1} + + sortedVector, err := sortVector(cfg, unsortedVector) + if err != nil { + return nil, err + } + ns = [][]netmap.NodeInfo{sortedVector, regularVector} + } else if cfg.flatSuccess != nil { ns = flatNodes(ns) rem = []int{int(*cfg.flatSuccess)} } else { @@ -157,6 +182,35 @@ func flatNodes(ns [][]netmap.NodeInfo) [][]netmap.NodeInfo { return [][]netmap.NodeInfo{flat} } +type nodeMetrics struct { + index int + metrics []int +} + +func sortVector(cfg *cfg, unsortedVector []netmap.NodeInfo) ([]netmap.NodeInfo, error) { + nm := make([]nodeMetrics, len(unsortedVector)) + node := cfg.nodeState.LocalNodeInfo() + + for i := range unsortedVector { + m := make([]int, len(cfg.metrics)) + for j, pm := range cfg.metrics { + m[j] = pm.CalculateValue(node, &unsortedVector[i]) + } + nm[i] = nodeMetrics{ + index: i, + metrics: m, + } + } + slices.SortFunc(nm, func(a, b nodeMetrics) int { + return slices.Compare(a.metrics, b.metrics) + }) + sortedVector := make([]netmap.NodeInfo, len(unsortedVector)) + for i := range unsortedVector { + sortedVector[i] = unsortedVector[nm[i].index] + } + return sortedVector, nil +} + // Node is a descriptor of storage node with information required for intra-container communication. type Node struct { addresses network.AddressGroup @@ -322,3 +376,17 @@ func WithCopyNumbers(v []uint32) Option { c.copyNumbers = v } } + +// WithPriorityMetrics use provided priority metrics to sort nodes. +func WithPriorityMetrics(m []Metric) Option { + return func(c *cfg) { + c.metrics = m + } +} + +// WithNodeState provide state of the current node. +func WithNodeState(s NodeState) Option { + return func(c *cfg) { + c.nodeState = s + } +} diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index b3b57677d..38f62aa07 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -22,7 +22,9 @@ func (b testBuilder) BuildPlacement(cid.ID, *oid.ID, netmap.PlacementPolicy) ([] } func testNode(v uint32) (n netmap.NodeInfo) { - n.SetNetworkEndpoints("/ip4/0.0.0.0/tcp/" + strconv.Itoa(int(v))) + ip := "/ip4/0.0.0.0/tcp/" + strconv.Itoa(int(v)) + n.SetNetworkEndpoints(ip) + n.SetPublicKey([]byte(ip)) return n } @@ -40,7 +42,15 @@ func copyVectors(v [][]netmap.NodeInfo) [][]netmap.NodeInfo { return vc } -func testPlacement(ss, rs []int) ([][]netmap.NodeInfo, container.Container) { +func testPlacement(ss []int, rs []int) ([][]netmap.NodeInfo, container.Container) { + return placement(ss, rs, nil) +} + +func testECPlacement(ss []int, ec [][]int) ([][]netmap.NodeInfo, container.Container) { + return placement(ss, nil, ec) +} + +func placement(ss []int, rs []int, ec [][]int) ([][]netmap.NodeInfo, container.Container) { nodes := make([][]netmap.NodeInfo, 0, len(rs)) replicas := make([]netmap.ReplicaDescriptor, 0, len(rs)) num := uint32(0) @@ -56,7 +66,12 @@ func testPlacement(ss, rs []int) ([][]netmap.NodeInfo, container.Container) { nodes = append(nodes, ns) var rd netmap.ReplicaDescriptor - rd.SetNumberOfObjects(uint32(rs[i])) + if len(rs) > 0 { + rd.SetNumberOfObjects(uint32(rs[i])) + } else { + rd.SetECDataCount(uint32(ec[i][0])) + rd.SetECParityCount(uint32(ec[i][1])) + } replicas = append(replicas, rd) } @@ -134,7 +149,7 @@ func TestTraverserObjectScenarios(t *testing.T) { err = n.FromIterator(netmapcore.Node(nodes[1][0])) require.NoError(t, err) - require.Equal(t, []Node{{addresses: n}}, tr.Next()) + require.Equal(t, []Node{{addresses: n, key: []byte("/ip4/0.0.0.0/tcp/5")}}, tr.Next()) }) t.Run("put scenario", func(t *testing.T) { @@ -275,3 +290,268 @@ func TestTraverserRemValues(t *testing.T) { }) } } + +type nodeState struct { + node *netmap.NodeInfo +} + +func (n *nodeState) LocalNodeInfo() *netmap.NodeInfo { + return n.node +} + +func TestTraverserPriorityMetrics(t *testing.T) { + t.Run("one rep one metric", func(t *testing.T) { + selectors := []int{4} + replicas := []int{3} + + nodes, cnr := testPlacement(selectors, replicas) + + // Node_0, PK - ip4/0.0.0.0/tcp/0 + nodes[0][0].SetAttribute("ClusterName", "A") + // Node_1, PK - ip4/0.0.0.0/tcp/1 + nodes[0][1].SetAttribute("ClusterName", "A") + // Node_2, PK - ip4/0.0.0.0/tcp/2 + nodes[0][2].SetAttribute("ClusterName", "B") + // Node_3, PK - ip4/0.0.0.0/tcp/3 + nodes[0][3].SetAttribute("ClusterName", "B") + + sdkNode := testNode(5) + sdkNode.SetAttribute("ClusterName", "B") + + nodesCopy := copyVectors(nodes) + + m := []Metric{NewAttributeMetric("ClusterName")} + + tr, err := NewTraverser( + ForContainer(cnr), + UseBuilder(&testBuilder{ + vectors: nodesCopy, + }), + WithoutSuccessTracking(), + WithPriorityMetrics(m), + WithNodeState(&nodeState{ + node: &sdkNode, + }), + ) + require.NoError(t, err) + + // Without priority metric `ClusterName` the order will be: + // [ {Node_0 A}, {Node_1 A}, {Node_2 B}, {Node_3 B}] + // With priority metric `ClusterName` and current node in cluster B + // the order should be: + // [ {Node_2 B}, {Node_0 A}, {Node_1 A}, {Node_3 B}] + next := tr.Next() + require.NotNil(t, next) + require.Equal(t, 3, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[1].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[2].PublicKey())) + + next = tr.Next() + // The last node is + require.Equal(t, 1, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[0].PublicKey())) + + next = tr.Next() + require.Nil(t, next) + }) + + t.Run("two reps two metrics", func(t *testing.T) { + selectors := []int{3, 3} + replicas := []int{2, 2} + + nodes, cnr := testPlacement(selectors, replicas) + + // REPLICA #1 + // Node_0, PK - ip4/0.0.0.0/tcp/0 + nodes[0][0].SetAttribute("ClusterName", "A") + nodes[0][0].SetAttribute("UN-LOCODE", "RU LED") + + // Node_1, PK - ip4/0.0.0.0/tcp/1 + nodes[0][1].SetAttribute("ClusterName", "A") + nodes[0][1].SetAttribute("UN-LOCODE", "FI HEL") + + // Node_2, PK - ip4/0.0.0.0/tcp/2 + nodes[0][2].SetAttribute("ClusterName", "A") + nodes[0][2].SetAttribute("UN-LOCODE", "RU LED") + + // REPLICA #2 + // Node_3 ip4/0.0.0.0/tcp/3 + nodes[1][0].SetAttribute("ClusterName", "B") + nodes[1][0].SetAttribute("UN-LOCODE", "RU MOW") + + // Node_4, PK - ip4/0.0.0.0/tcp/4 + nodes[1][1].SetAttribute("ClusterName", "B") + nodes[1][1].SetAttribute("UN-LOCODE", "RU DME") + + // Node_5, PK - ip4/0.0.0.0/tcp/5 + nodes[1][2].SetAttribute("ClusterName", "B") + nodes[1][2].SetAttribute("UN-LOCODE", "RU MOW") + + sdkNode := testNode(9) + sdkNode.SetAttribute("ClusterName", "B") + sdkNode.SetAttribute("UN-LOCODE", "RU DME") + + nodesCopy := copyVectors(nodes) + + m := []Metric{ + NewAttributeMetric("ClusterName"), + NewAttributeMetric("UN-LOCODE"), + } + + tr, err := NewTraverser( + ForContainer(cnr), + UseBuilder(&testBuilder{ + vectors: nodesCopy, + }), + WithoutSuccessTracking(), + WithPriorityMetrics(m), + WithNodeState(&nodeState{ + node: &sdkNode, + }), + ) + require.NoError(t, err) + + // Check that nodes in the same cluster and + // in the same location should be the first in slice. + // Nodes which are follow criteria but stay outside the replica + // should be in the next slice. + + next := tr.Next() + require.Equal(t, 4, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/4", string(next[0].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[1].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[2].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[3].PublicKey())) + + next = tr.Next() + require.Equal(t, 2, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/5", string(next[1].PublicKey())) + + next = tr.Next() + require.Nil(t, next) + + sdkNode.SetAttribute("ClusterName", "B") + sdkNode.SetAttribute("UN-LOCODE", "RU MOW") + + nodesCopy = copyVectors(nodes) + + tr, err = NewTraverser( + ForContainer(cnr), + UseBuilder(&testBuilder{ + vectors: nodesCopy, + }), + WithoutSuccessTracking(), + WithPriorityMetrics(m), + WithNodeState(&nodeState{ + node: &sdkNode, + }), + ) + require.NoError(t, err) + + next = tr.Next() + require.Equal(t, 4, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[0].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/4", string(next[1].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[2].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[3].PublicKey())) + + next = tr.Next() + require.Equal(t, 2, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/5", string(next[1].PublicKey())) + + next = tr.Next() + require.Nil(t, next) + + sdkNode.SetAttribute("ClusterName", "A") + sdkNode.SetAttribute("UN-LOCODE", "RU LED") + + nodesCopy = copyVectors(nodes) + + tr, err = NewTraverser( + ForContainer(cnr), + UseBuilder(&testBuilder{ + vectors: nodesCopy, + }), + WithoutSuccessTracking(), + WithPriorityMetrics(m), + WithNodeState(&nodeState{ + node: &sdkNode, + }), + ) + require.NoError(t, err) + + next = tr.Next() + require.Equal(t, 4, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[0].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[1].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[2].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/4", string(next[3].PublicKey())) + + next = tr.Next() + require.Equal(t, 2, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/5", string(next[1].PublicKey())) + + next = tr.Next() + require.Nil(t, next) + }) + + t.Run("ec container", func(t *testing.T) { + selectors := []int{4} + ec := [][]int{{2, 1}} + + nodes, cnr := testECPlacement(selectors, ec) + + // Node_0, PK - ip4/0.0.0.0/tcp/0 + nodes[0][0].SetAttribute("ClusterName", "A") + // Node_1, PK - ip4/0.0.0.0/tcp/1 + nodes[0][1].SetAttribute("ClusterName", "A") + // Node_2, PK - ip4/0.0.0.0/tcp/2 + nodes[0][2].SetAttribute("ClusterName", "B") + // Node_3, PK - ip4/0.0.0.0/tcp/3 + nodes[0][3].SetAttribute("ClusterName", "B") + + sdkNode := testNode(5) + sdkNode.SetAttribute("ClusterName", "B") + + nodesCopy := copyVectors(nodes) + + m := []Metric{NewAttributeMetric("ClusterName")} + + tr, err := NewTraverser( + ForContainer(cnr), + UseBuilder(&testBuilder{ + vectors: nodesCopy, + }), + WithoutSuccessTracking(), + WithPriorityMetrics(m), + WithNodeState(&nodeState{ + node: &sdkNode, + }), + ) + require.NoError(t, err) + + // Without priority metric `ClusterName` the order will be: + // [ {Node_0 A}, {Node_1 A}, {Node_2 B}, {Node_3 B}] + // With priority metric `ClusterName` and current node in cluster B + // the order should be: + // [ {Node_2 B}, {Node_0 A}, {Node_1 A}, {Node_3 B}] + next := tr.Next() + require.NotNil(t, next) + require.Equal(t, 3, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/2", string(next[0].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[1].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[2].PublicKey())) + + next = tr.Next() + // The last node is + require.Equal(t, 1, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/3", string(next[0].PublicKey())) + + next = tr.Next() + require.Nil(t, next) + }) +} From 433aab12bb170832a8b6607e372d09dafdd6ba84 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 29 Oct 2024 16:14:40 +0300 Subject: [PATCH 1479/1943] [#1455] cli: Handle missing home directory go-homedir library incorrectly handles some of the errors that could occur. It is archived, so no PR, but let's fix it on our side. The scenario in case: executing command in an empty environment. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/root.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-cli/modules/root.go b/cmd/frostfs-cli/modules/root.go index 21c367d29..c0282586a 100644 --- a/cmd/frostfs-cli/modules/root.go +++ b/cmd/frostfs-cli/modules/root.go @@ -114,12 +114,14 @@ func initConfig() { } else { // Find home directory. home, err := homedir.Dir() - commonCmd.ExitOnErr(rootCmd, "", err) - - // Search config in `$HOME/.config/frostfs-cli/` with name "config.yaml" - viper.AddConfigPath(filepath.Join(home, ".config", "frostfs-cli")) - viper.SetConfigName("config") - viper.SetConfigType("yaml") + if err != nil { + common.PrintVerbose(rootCmd, "Get homedir: %s", err) + } else { + // Search config in `$HOME/.config/frostfs-cli/` with name "config.yaml" + viper.AddConfigPath(filepath.Join(home, ".config", "frostfs-cli")) + viper.SetConfigName("config") + viper.SetConfigType("yaml") + } } viper.SetEnvPrefix(envPrefix) From d5ee6d30390497b1c60245dc4cae782cad619995 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 29 Oct 2024 17:17:04 +0300 Subject: [PATCH 1480/1943] [#1456] morph: Use DialerSource interface instead of internal struct Signed-off-by: Dmitrii Stepanov --- pkg/morph/client/constructor.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index a8efa76e7..08d16deb4 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -4,11 +4,11 @@ import ( "context" "errors" "fmt" + "net" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" - internalNet "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" @@ -48,7 +48,7 @@ type cfg struct { morphCacheMetrics metrics.MorphCacheMetrics - dialerSource *internalNet.DialerSource + dialerSource DialerSource } const ( @@ -68,6 +68,7 @@ func defaultConfig() *cfg { Scopes: transaction.Global, }, morphCacheMetrics: &morphmetrics.NoopMorphCacheMetrics{}, + dialerSource: &noopDialerSource{}, } } @@ -296,7 +297,17 @@ func WithMorphCacheMetrics(morphCacheMetrics metrics.MorphCacheMetrics) Option { } } -func WithDialerSource(ds *internalNet.DialerSource) Option { +type DialerSource interface { + NetContextDialer() func(context.Context, string, string) (net.Conn, error) +} + +type noopDialerSource struct{} + +func (ds *noopDialerSource) NetContextDialer() func(context.Context, string, string) (net.Conn, error) { + return nil +} + +func WithDialerSource(ds DialerSource) Option { return func(c *cfg) { c.dialerSource = ds } From 87ac3c52797d65f1c46e255c937ffd30b8441d35 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 29 Oct 2024 17:16:21 +0300 Subject: [PATCH 1481/1943] [#1458] object: Make patch not set key before target construction * `SignRequestPrivateKey` field should be initialized either within `newUntrustedTarget` or within `newTrustedTarget`. Otherwise, all requests are signed by local node key that makes impossible to perform patch on non-container node. Signed-off-by: Airat Arifullin --- pkg/services/object/patch/streamer.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/services/object/patch/streamer.go b/pkg/services/object/patch/streamer.go index c8ed6fdbf..677c6610f 100644 --- a/pkg/services/object/patch/streamer.go +++ b/pkg/services/object/patch/streamer.go @@ -113,10 +113,9 @@ func (s *Streamer) init(ctx context.Context, req *objectV2.PatchRequest) error { oV2.GetHeader().SetOwnerID(ownerID) target, err := target.New(objectwriter.Params{ - Config: s.Config, - Common: commonPrm, - Header: objectSDK.NewFromV2(oV2), - SignRequestPrivateKey: s.localNodeKey, + Config: s.Config, + Common: commonPrm, + Header: objectSDK.NewFromV2(oV2), }) if err != nil { return fmt.Errorf("target creation: %w", err) From d28a5d2d7a48084a45b154183c450f219969db69 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Tue, 29 Oct 2024 15:47:19 +0300 Subject: [PATCH 1482/1943] [#1448] container/ape: Ignore an error when getting a role When getting a role in the APE checker for the container services, an error may be returned if network maps of the previous two epochs don't have enough nodes to fulfil a container placement policy. It's a logical error, so we should ignore it. Signed-off-by: Aleksey Savchuk --- pkg/services/container/ape.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index d92ecf58b..dd4878331 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -537,10 +537,7 @@ func (ac *apeChecker) isContainerKey(pk []byte, cnrID cid.ID, cont *containercor return false, err } - in, err := isContainerNode(nm, pk, binCnrID, cont) - if err != nil { - return false, err - } else if in { + if isContainerNode(nm, pk, binCnrID, cont) { return true, nil } @@ -551,24 +548,24 @@ func (ac *apeChecker) isContainerKey(pk []byte, cnrID cid.ID, cont *containercor return false, err } - return isContainerNode(nm, pk, binCnrID, cont) + return isContainerNode(nm, pk, binCnrID, cont), nil } -func isContainerNode(nm *netmapSDK.NetMap, pk, binCnrID []byte, cont *containercore.Container) (bool, error) { - cnrVectors, err := nm.ContainerNodes(cont.Value.PlacementPolicy(), binCnrID) - if err != nil { - return false, err - } +func isContainerNode(nm *netmapSDK.NetMap, pk, binCnrID []byte, cont *containercore.Container) bool { + // It could an error only if the network map doesn't have enough nodes to + // fulfil the policy. It's a logical error that doesn't affect an actor role + // determining, so we ignore it + cnrVectors, _ := nm.ContainerNodes(cont.Value.PlacementPolicy(), binCnrID) for i := range cnrVectors { for j := range cnrVectors[i] { if bytes.Equal(cnrVectors[i][j].PublicKey(), pk) { - return true, nil + return true } } } - return false, nil + return false } func (ac *apeChecker) namespaceByOwner(owner *refs.OwnerID) (string, error) { From 7ac08523647c7450366a9aa350f05eafecad8c7d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 30 Oct 2024 14:42:09 +0300 Subject: [PATCH 1483/1943] [#1459] .golangci.yml: Add intrange linter, fix issues Signed-off-by: Evgenii Stratonikov --- .golangci.yml | 1 + cmd/frostfs-adm/internal/modules/config/config.go | 2 +- .../internal/modules/morph/generate/generate_test.go | 2 +- .../internal/schema/metabase/records/util.go | 2 +- .../blobstor/blobovniczatree/rebuild_test.go | 6 +++--- pkg/local_object_storage/blobstor/iterate_test.go | 2 +- pkg/local_object_storage/engine/error_test.go | 2 +- pkg/local_object_storage/pilorama/forest_test.go | 2 +- pkg/local_object_storage/shard/control_test.go | 2 +- pkg/local_object_storage/writecache/limiter_test.go | 2 +- pkg/morph/timer/block_test.go | 2 +- pkg/services/object/acl/v2/util_test.go | 2 +- scripts/populate-metabase/internal/generate.go | 10 +++++----- scripts/populate-metabase/internal/populate.go | 8 ++++---- scripts/populate-metabase/main.go | 2 +- 15 files changed, 24 insertions(+), 23 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 33cf88d8a..1235c62b4 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -87,5 +87,6 @@ linters: - perfsprint - testifylint - protogetter + - intrange disable-all: true fast: false diff --git a/cmd/frostfs-adm/internal/modules/config/config.go b/cmd/frostfs-adm/internal/modules/config/config.go index a98245d01..69153f0d7 100644 --- a/cmd/frostfs-adm/internal/modules/config/config.go +++ b/cmd/frostfs-adm/internal/modules/config/config.go @@ -128,7 +128,7 @@ func generateConfigExample(appDir string, credSize int) (string, error) { tmpl.AlphabetDir = filepath.Join(appDir, "alphabet-wallets") var i innerring.GlagoliticLetter - for i = 0; i < innerring.GlagoliticLetter(credSize); i++ { + for i = range innerring.GlagoliticLetter(credSize) { tmpl.Glagolitics = append(tmpl.Glagolitics, i.String()) } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go index 1dd6420eb..15af5637b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate_test.go @@ -63,7 +63,7 @@ func TestGenerateAlphabet(t *testing.T) { buf.Reset() v.Set(commonflags.AlphabetWalletsFlag, walletDir) require.NoError(t, GenerateAlphabetCmd.Flags().Set(commonflags.AlphabetSizeFlag, strconv.FormatUint(size, 10))) - for i := uint64(0); i < size; i++ { + for i := range uint64(size) { buf.WriteString(strconv.FormatUint(i, 10) + "\r") } diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/util.go b/cmd/frostfs-lens/internal/schema/metabase/records/util.go index f50ebe951..d15d69146 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/records/util.go +++ b/cmd/frostfs-lens/internal/schema/metabase/records/util.go @@ -11,7 +11,7 @@ func DecodeOIDs(data []byte) ([]oid.ID, error) { size := r.ReadVarUint() oids := make([]oid.ID, size) - for i := uint64(0); i < size; i++ { + for i := range size { if err := oids[i].Decode(r.ReadVarBytes()); err != nil { return nil, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index dfd928aaf..dff4e9024 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -61,7 +61,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { require.NoError(t, b.Init()) storageIDs := make(map[oid.Address][]byte) - for i := 0; i < 100; i++ { + for range 100 { obj := blobstortest.NewObject(64 * 1024) // 64KB object data, err := obj.Marshal() require.NoError(t, err) @@ -168,7 +168,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { storageIDs := make(map[oid.Address][]byte) toDelete := make(map[oid.Address][]byte) - for i := 0; i < 100; i++ { // 2 objects for one blobovnicza, so 50 DBs total will be created + for i := range 100 { // 2 objects for one blobovnicza, so 50 DBs total will be created obj := blobstortest.NewObject(64 * 1024) data, err := obj.Marshal() require.NoError(t, err) @@ -236,7 +236,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { require.NoError(t, b.Init()) storageIDs := make(map[oid.Address][]byte) - for i := 0; i < 100; i++ { // 2 objects for one blobovnicza, so 50 DBs total will be created + for range 100 { // 2 objects for one blobovnicza, so 50 DBs total will be created obj := blobstortest.NewObject(64 * 1024) data, err := obj.Marshal() require.NoError(t, err) diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index 195d0bd31..3c9457db2 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -47,7 +47,7 @@ func TestIterateObjects(t *testing.T) { mObjs := make(map[string]addrData) - for i := uint64(0); i < objNum; i++ { + for i := range uint64(objNum) { sz := smalSz big := i < objNum/2 diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 535435ceb..1619003a1 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -151,7 +151,7 @@ func TestErrorReporting(t *testing.T) { checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } - for i := uint32(0); i < 2; i++ { + for i := range uint32(2) { _, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)}) require.Error(t, err) checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly) diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index 854fe0aad..fbcc53fb3 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -1081,7 +1081,7 @@ func prepareRandomTree(nodeCount, opCount int) []Move { } func compareForests(t *testing.T, expected, actual Forest, cid cidSDK.ID, treeID string, nodeCount int) { - for i := uint64(0); i < uint64(nodeCount); i++ { + for i := range uint64(nodeCount) { expectedMeta, expectedParent, err := expected.TreeGetMeta(context.Background(), cid, treeID, i) require.NoError(t, err) actualMeta, actualParent, err := actual.TreeGetMeta(context.Background(), cid, treeID, i) diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 6b9eaa550..b8f1d4417 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -216,7 +216,7 @@ func TestRefillMetabase(t *testing.T) { locked := make([]oid.ID, 1, 2) locked[0] = oidtest.ID() cnrLocked := cidtest.ID() - for i := uint64(0); i < objNum; i++ { + for range objNum { obj := objecttest.Object() obj.SetType(objectSDK.TypeRegular) diff --git a/pkg/local_object_storage/writecache/limiter_test.go b/pkg/local_object_storage/writecache/limiter_test.go index db99b203a..1ca3e1156 100644 --- a/pkg/local_object_storage/writecache/limiter_test.go +++ b/pkg/local_object_storage/writecache/limiter_test.go @@ -14,7 +14,7 @@ func TestLimiter(t *testing.T) { l := newFlushLimiter(uint64(maxSize)) var currSize atomic.Int64 var eg errgroup.Group - for i := 0; i < 10_000; i++ { + for range 10_000 { eg.Go(func() error { defer l.release(single) defer currSize.Add(-1) diff --git a/pkg/morph/timer/block_test.go b/pkg/morph/timer/block_test.go index 7929754c1..ee6091845 100644 --- a/pkg/morph/timer/block_test.go +++ b/pkg/morph/timer/block_test.go @@ -8,7 +8,7 @@ import ( ) func tickN(t *timer.BlockTimer, n uint32) { - for i := uint32(0); i < n; i++ { + for range n { t.Tick(0) } } diff --git a/pkg/services/object/acl/v2/util_test.go b/pkg/services/object/acl/v2/util_test.go index 435339683..8c6d01ae9 100644 --- a/pkg/services/object/acl/v2/util_test.go +++ b/pkg/services/object/acl/v2/util_test.go @@ -50,7 +50,7 @@ func testGenerateMetaHeader(depth uint32, b *acl.BearerToken, s *session.Token) metaHeader.SetBearerToken(b) metaHeader.SetSessionToken(s) - for i := uint32(0); i < depth; i++ { + for range depth { link := metaHeader metaHeader = new(session.RequestMetaHeader) metaHeader.SetOrigin(link) diff --git a/scripts/populate-metabase/internal/generate.go b/scripts/populate-metabase/internal/generate.go index 8a96dcaaa..f2f8881cf 100644 --- a/scripts/populate-metabase/internal/generate.go +++ b/scripts/populate-metabase/internal/generate.go @@ -19,7 +19,7 @@ import ( func GeneratePayloadPool(count uint, size uint) [][]byte { var pool [][]byte - for i := uint(0); i < count; i++ { + for range count { payload := make([]byte, size) _, _ = rand.Read(payload) @@ -30,8 +30,8 @@ func GeneratePayloadPool(count uint, size uint) [][]byte { func GenerateAttributePool(count uint) []objectSDK.Attribute { var pool []objectSDK.Attribute - for i := uint(0); i < count; i++ { - for j := uint(0); j < count; j++ { + for i := range count { + for j := range count { attr := *objectSDK.NewAttribute() attr.SetKey(fmt.Sprintf("key%d", i)) attr.SetValue(fmt.Sprintf("value%d", j)) @@ -43,7 +43,7 @@ func GenerateAttributePool(count uint) []objectSDK.Attribute { func GenerateOwnerPool(count uint) []user.ID { var pool []user.ID - for i := uint(0); i < count; i++ { + for range count { pool = append(pool, usertest.ID()) } return pool @@ -118,7 +118,7 @@ func WithPayloadFromPool(pool [][]byte) ObjectOption { func WithAttributesFromPool(pool []objectSDK.Attribute, count uint) ObjectOption { return func(obj *objectSDK.Object) { var attrs []objectSDK.Attribute - for i := uint(0); i < count; i++ { + for range count { attrs = append(attrs, pool[rand.Intn(len(pool))]) } obj.SetAttributes(attrs...) diff --git a/scripts/populate-metabase/internal/populate.go b/scripts/populate-metabase/internal/populate.go index 390c1cdc0..4da23a295 100644 --- a/scripts/populate-metabase/internal/populate.go +++ b/scripts/populate-metabase/internal/populate.go @@ -29,7 +29,7 @@ func PopulateWithObjects( ) { digits := "0123456789" - for i := uint(0); i < count; i++ { + for range count { obj := factory() id := []byte(fmt.Sprintf( @@ -59,7 +59,7 @@ func PopulateWithBigObjects( count uint, factory func() *objectSDK.Object, ) { - for i := uint(0); i < count; i++ { + for range count { group.Go(func() error { if err := populateWithBigObject(ctx, db, factory); err != nil { return fmt.Errorf("couldn't put a big object: %w", err) @@ -154,7 +154,7 @@ func PopulateGraveyard( wg := &sync.WaitGroup{} wg.Add(int(count)) - for i := uint(0); i < count; i++ { + for range count { obj := factory() prm := meta.PutPrm{} @@ -226,7 +226,7 @@ func PopulateLocked( wg := &sync.WaitGroup{} wg.Add(int(count)) - for i := uint(0); i < count; i++ { + for range count { defer wg.Done() obj := factory() diff --git a/scripts/populate-metabase/main.go b/scripts/populate-metabase/main.go index 2bc7a5553..6f6b233cf 100644 --- a/scripts/populate-metabase/main.go +++ b/scripts/populate-metabase/main.go @@ -116,7 +116,7 @@ func populate() (err error) { eg, ctx := errgroup.WithContext(ctx) eg.SetLimit(int(jobs)) - for i := uint(0); i < numContainers; i++ { + for range numContainers { cid := cidtest.ID() for _, typ := range types { From 89892d9754988fce2d826edee7a7c6af949ced64 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 30 Oct 2024 15:13:45 +0300 Subject: [PATCH 1484/1943] [#1459] cli: Simplify slice append Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/client/client.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index dcd67f0d9..ed9817b86 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -659,9 +659,7 @@ func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes for { n, ok = rdr.Read(buf) - for i := range n { - list = append(list, buf[i]) - } + list = append(list, buf[:n]...) if !ok { break } From 48862e0e63f0306cbe9556abc8e20bb7da27d399 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 30 Oct 2024 15:33:10 +0300 Subject: [PATCH 1485/1943] [#1459] .golanci.yml: Add tenv linter, fix issues Refs #1309 Signed-off-by: Evgenii Stratonikov --- .golangci.yml | 1 + cmd/frostfs-node/config/calls_test.go | 4 +--- cmd/frostfs-node/config/test/config.go | 14 +------------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 1235c62b4..57e3b4494 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -88,5 +88,6 @@ linters: - testifylint - protogetter - intrange + - tenv disable-all: true fast: false diff --git a/cmd/frostfs-node/config/calls_test.go b/cmd/frostfs-node/config/calls_test.go index 68bf1c679..bc149eb7d 100644 --- a/cmd/frostfs-node/config/calls_test.go +++ b/cmd/frostfs-node/config/calls_test.go @@ -1,7 +1,6 @@ package config_test import ( - "os" "strings" "testing" @@ -38,8 +37,7 @@ func TestConfigEnv(t *testing.T) { envName := strings.ToUpper( strings.Join([]string{config.EnvPrefix, section, name}, configViper.EnvSeparator)) - err := os.Setenv(envName, value) - require.NoError(t, err) + t.Setenv(envName, value) c := configtest.EmptyConfig() diff --git a/cmd/frostfs-node/config/test/config.go b/cmd/frostfs-node/config/test/config.go index a93d7e648..e98c032f0 100644 --- a/cmd/frostfs-node/config/test/config.go +++ b/cmd/frostfs-node/config/test/config.go @@ -11,8 +11,6 @@ import ( ) func fromFile(path string) *config.Config { - os.Clearenv() // ENVs have priority over config files, so we do this in tests - return config.New(path, "", "") } @@ -40,15 +38,6 @@ func ForEachFileType(pref string, f func(*config.Config)) { // ForEnvFileType creates config from `.env` file. func ForEnvFileType(t testing.TB, pref string, f func(*config.Config)) { - envs := os.Environ() - t.Cleanup(func() { - os.Clearenv() - for _, env := range envs { - keyValue := strings.Split(env, "=") - os.Setenv(keyValue[0], keyValue[1]) - } - }) - f(fromEnvFile(t, pref+".env")) } @@ -73,7 +62,6 @@ func loadEnv(t testing.TB, path string) { v = strings.Trim(v, `"`) - err = os.Setenv(k, v) - require.NoError(t, err, "can't set environment variable") + t.Setenv(k, v) } } From e74d05c03f6806cea12e69ce00d84bd69eab5fb7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 30 Oct 2024 17:43:54 +0300 Subject: [PATCH 1486/1943] [#1464] frostfsid: Add cache metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/container.go | 3 ++- cmd/frostfs-node/frostfsid.go | 24 +++++++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 729fcb8af..f95f671cd 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -8,6 +8,7 @@ import ( containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc" morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" @@ -42,7 +43,7 @@ func initContainerService(_ context.Context, c *cfg) { cacheSize := morphconfig.FrostfsIDCacheSize(c.appCfg) if cacheSize > 0 { - frostfsIDSubjectProvider = newMorphFrostfsIDCache(frostfsIDSubjectProvider, int(cacheSize), c.cfgMorph.cacheTTL) + frostfsIDSubjectProvider = newMorphFrostfsIDCache(frostfsIDSubjectProvider, int(cacheSize), c.cfgMorph.cacheTTL, metrics.NewCacheMetrics("frostfs_id")) } c.shared.frostfsidClient = frostfsIDSubjectProvider diff --git a/cmd/frostfs-node/frostfsid.go b/cmd/frostfs-node/frostfsid.go index fb55a6019..9039915f5 100644 --- a/cmd/frostfs-node/frostfsid.go +++ b/cmd/frostfs-node/frostfsid.go @@ -14,22 +14,29 @@ type morphFrostfsIDCache struct { subjCache *expirable.LRU[util.Uint160, *client.Subject] - subjExtCache *expirable.LRU[util.Uint160, *client.SubjectExtended] + metrics cacheMetrics } -func newMorphFrostfsIDCache(subjProvider frostfsidcore.SubjectProvider, size int, ttl time.Duration) frostfsidcore.SubjectProvider { +func newMorphFrostfsIDCache(subjProvider frostfsidcore.SubjectProvider, size int, ttl time.Duration, metrics cacheMetrics) frostfsidcore.SubjectProvider { return &morphFrostfsIDCache{ subjProvider: subjProvider, subjCache: expirable.NewLRU(size, func(util.Uint160, *client.Subject) {}, ttl), - subjExtCache: expirable.NewLRU(size, func(util.Uint160, *client.SubjectExtended) {}, ttl), + metrics: metrics, } } func (m *morphFrostfsIDCache) GetSubject(addr util.Uint160) (*client.Subject, error) { + hit := false + startedAt := time.Now() + defer func() { + m.metrics.AddMethodDuration("GetSubject", time.Since(startedAt), hit) + }() + result, found := m.subjCache.Get(addr) if found { + hit = true return result, nil } @@ -43,9 +50,16 @@ func (m *morphFrostfsIDCache) GetSubject(addr util.Uint160) (*client.Subject, er } func (m *morphFrostfsIDCache) GetSubjectExtended(addr util.Uint160) (*client.SubjectExtended, error) { - subjExt, found := m.subjExtCache.Get(addr) + hit := false + startedAt := time.Now() + defer func() { + m.metrics.AddMethodDuration("GetSubjectExtended", time.Since(startedAt), hit) + }() + + result, found := m.subjExtCache.Get(addr) if found { - return subjExt, nil + hit = true + return result, nil } var err error From 2285cfc36f1d815e722fb7ef246801a9029efb63 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 30 Oct 2024 17:44:31 +0300 Subject: [PATCH 1487/1943] [#1464] frostfsid: Cache `subject not found` error Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/frostfsid.go | 53 +++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/cmd/frostfs-node/frostfsid.go b/cmd/frostfs-node/frostfsid.go index 9039915f5..3cca09105 100644 --- a/cmd/frostfs-node/frostfsid.go +++ b/cmd/frostfs-node/frostfsid.go @@ -1,6 +1,7 @@ package main import ( + "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" @@ -9,10 +10,22 @@ import ( "github.com/nspcc-dev/neo-go/pkg/util" ) +type subjectWithError struct { + subject *client.Subject + err error +} + +type subjectExtWithError struct { + subject *client.SubjectExtended + err error +} + type morphFrostfsIDCache struct { subjProvider frostfsidcore.SubjectProvider - subjCache *expirable.LRU[util.Uint160, *client.Subject] + subjCache *expirable.LRU[util.Uint160, subjectWithError] + + subjExtCache *expirable.LRU[util.Uint160, subjectExtWithError] metrics cacheMetrics } @@ -21,7 +34,9 @@ func newMorphFrostfsIDCache(subjProvider frostfsidcore.SubjectProvider, size int return &morphFrostfsIDCache{ subjProvider: subjProvider, - subjCache: expirable.NewLRU(size, func(util.Uint160, *client.Subject) {}, ttl), + subjCache: expirable.NewLRU(size, func(util.Uint160, subjectWithError) {}, ttl), + + subjExtCache: expirable.NewLRU(size, func(util.Uint160, subjectExtWithError) {}, ttl), metrics: metrics, } @@ -37,16 +52,21 @@ func (m *morphFrostfsIDCache) GetSubject(addr util.Uint160) (*client.Subject, er result, found := m.subjCache.Get(addr) if found { hit = true - return result, nil + return result.subject, result.err } - result, err := m.subjProvider.GetSubject(addr) + subj, err := m.subjProvider.GetSubject(addr) if err != nil { + if m.isCacheableError(err) { + m.subjCache.Add(addr, subjectWithError{ + err: err, + }) + } return nil, err } - m.subjCache.Add(addr, result) - return result, nil + m.subjCache.Add(addr, subjectWithError{subject: subj}) + return subj, nil } func (m *morphFrostfsIDCache) GetSubjectExtended(addr util.Uint160) (*client.SubjectExtended, error) { @@ -59,21 +79,32 @@ func (m *morphFrostfsIDCache) GetSubjectExtended(addr util.Uint160) (*client.Sub result, found := m.subjExtCache.Get(addr) if found { hit = true - return result, nil + return result.subject, result.err } - var err error - subjExt, err = m.subjProvider.GetSubjectExtended(addr) + subjExt, err := m.subjProvider.GetSubjectExtended(addr) if err != nil { + if m.isCacheableError(err) { + m.subjExtCache.Add(addr, subjectExtWithError{ + err: err, + }) + m.subjCache.Add(addr, subjectWithError{ + err: err, + }) + } return nil, err } - m.subjExtCache.Add(addr, subjExt) - m.subjCache.Add(addr, subjectFromSubjectExtended(subjExt)) + m.subjExtCache.Add(addr, subjectExtWithError{subject: subjExt}) + m.subjCache.Add(addr, subjectWithError{subject: subjectFromSubjectExtended(subjExt)}) return subjExt, nil } +func (m *morphFrostfsIDCache) isCacheableError(err error) bool { + return strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) +} + func subjectFromSubjectExtended(subjExt *client.SubjectExtended) *client.Subject { return &client.Subject{ PrimaryKey: subjExt.PrimaryKey, From c2effcc61c5bcb10f58e9d039b12ea0b22a17bba Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 31 Oct 2024 09:13:50 +0300 Subject: [PATCH 1488/1943] [#1465] Makefile: Update golangci-lint, fix warnings Signed-off-by: Evgenii Stratonikov --- Makefile | 2 +- cmd/frostfs-cli/modules/object/nodes.go | 2 +- pkg/local_object_storage/pilorama/boltdb.go | 2 +- pkg/services/object/common/writer/ec.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index d92844bb5..94b1542d5 100755 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.22 -LINT_VERSION ?= 1.60.3 +LINT_VERSION ?= 1.61.0 TRUECLOUDLAB_LINT_VERSION ?= 0.0.7 PROTOC_VERSION ?= 25.0 PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 0eac4e6d2..e6918dfc9 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -195,7 +195,7 @@ func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, member prmHead.SetRawFlag(true) // to get an error instead of whole object eg, egCtx := errgroup.WithContext(cmd.Context()) - for idx := range len(members) { + for idx := range members { partObjID := members[idx] eg.Go(func() error { diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index e2d69cafa..09f2e1919 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -705,7 +705,7 @@ func (t *boltForest) applyOperation(logBucket, treeBucket *bbolt.Bucket, ms []*M key, value = c.Prev() } - for i := range len(ms) { + for i := range ms { // Loop invariant: key represents the next stored timestamp after ms[i].Time. // 2. Insert the operation. diff --git a/pkg/services/object/common/writer/ec.go b/pkg/services/object/common/writer/ec.go index dffe52a6d..ee0681349 100644 --- a/pkg/services/object/common/writer/ec.go +++ b/pkg/services/object/common/writer/ec.go @@ -284,7 +284,7 @@ func (e *ECWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx } // try to save to any node not visited by current part - for i := range len(nodes) { + for i := range nodes { select { case <-ctx.Done(): return ctx.Err() From 5bcf81d1cc20408a925c27a9f069bbba6e0a2993 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 31 Oct 2024 10:06:51 +0300 Subject: [PATCH 1489/1943] [#1466] Remove woodpecker CI We use forgejo actions now. Signed-off-by: Evgenii Stratonikov --- .woodpecker/pre-commit.yml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 .woodpecker/pre-commit.yml diff --git a/.woodpecker/pre-commit.yml b/.woodpecker/pre-commit.yml deleted file mode 100644 index bdf3402de..000000000 --- a/.woodpecker/pre-commit.yml +++ /dev/null @@ -1,11 +0,0 @@ -pipeline: - # Kludge for non-root containers under WoodPecker - fix-ownership: - image: alpine:latest - commands: chown -R 1234:1234 . - - pre-commit: - image: git.frostfs.info/truecloudlab/frostfs-ci:v0.36 - commands: - - export HOME="$(getent passwd $(id -u) | cut '-d:' -f6)" - - pre-commit run --hook-stage manual From d19ab43500c24795b9511f0644b706a010983a5a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 30 Oct 2024 16:42:09 +0300 Subject: [PATCH 1490/1943] [#1462] node: Add off-cpu profiler Signed-off-by: Dmitrii Stepanov --- go.mod | 2 ++ go.sum | 14 ++++++++++++++ pkg/util/http/pprof.go | 6 ++++++ 3 files changed, 22 insertions(+) diff --git a/go.mod b/go.mod index aefe2889a..ba9ec375c 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc + github.com/felixge/fgprof v0.9.5 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/gdamore/tcell/v2 v2.7.4 github.com/go-pkgz/expirable-cache/v3 v3.0.0 @@ -77,6 +78,7 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index 4d44079d4..a037a99af 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,9 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= +github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= @@ -68,6 +71,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= +github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -91,6 +96,9 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-pkgz/expirable-cache/v3 v3.0.0 h1:u3/gcu3sabLYiTCevoRKv+WzjIn5oo7P8XtiXBeRDLw= github.com/go-pkgz/expirable-cache/v3 v3.0.0/go.mod h1:2OQiDyEGQalYecLWmXprm3maPXeVb5/6/X7yRPYTzec= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -109,6 +117,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -130,6 +140,7 @@ github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= @@ -148,6 +159,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -210,6 +222,7 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/panjf2000/ants/v2 v2.9.0 h1:SztCLkVxBRigbg+vt0S5QvF5vxAbxbKt09/YfAJ0tEo= github.com/panjf2000/ants/v2 v2.9.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= @@ -361,6 +374,7 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/pkg/util/http/pprof.go b/pkg/util/http/pprof.go index 7a0413000..f85fd2ea9 100644 --- a/pkg/util/http/pprof.go +++ b/pkg/util/http/pprof.go @@ -3,8 +3,14 @@ package httputil import ( "net/http" "net/http/pprof" + + "github.com/felixge/fgprof" ) +func init() { + http.DefaultServeMux.Handle("/debug/fgprof", fgprof.Handler()) +} + // initializes pprof package in order to // register Prometheus handlers on http.DefaultServeMux. var _ = pprof.Handler("") From 6c45a17af66843ac5757fcd4b8f8e6acd0bca087 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 31 Oct 2024 13:13:03 +0300 Subject: [PATCH 1491/1943] [#1467] node: Break notary deposit wait after VUB Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/morph.go | 70 ++++++++++++++++++++++---------------- cmd/frostfs-node/netmap.go | 2 +- internal/logs/logs.go | 1 - pkg/innerring/notary.go | 5 +-- pkg/morph/client/client.go | 23 +++++++++++++ pkg/morph/client/notary.go | 17 ++++----- 6 files changed, 77 insertions(+), 41 deletions(-) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index e85209059..0969f5579 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -17,15 +17,16 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/waiter" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "go.uber.org/zap" ) const ( newEpochNotification = "NewEpoch" - - // amount of tries(blocks) before notary deposit timeout. - notaryDepositRetriesAmount = 300 ) func (c *cfg) initMorphComponents(ctx context.Context) { @@ -128,7 +129,7 @@ func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) { return } - tx, err := makeNotaryDeposit(c) + tx, vub, err := makeNotaryDeposit(c) fatalOnErr(err) if tx.Equals(util.Uint256{}) { @@ -139,11 +140,11 @@ func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) { return } - err = waitNotaryDeposit(ctx, c, tx) + err = waitNotaryDeposit(ctx, c, tx, vub) fatalOnErr(err) } -func makeNotaryDeposit(c *cfg) (util.Uint256, error) { +func makeNotaryDeposit(c *cfg) (util.Uint256, uint32, error) { const ( // gasMultiplier defines how many times more the notary // balance must be compared to the GAS balance of the node: @@ -157,7 +158,7 @@ func makeNotaryDeposit(c *cfg) (util.Uint256, error) { depositAmount, err := client.CalculateNotaryDepositAmount(c.cfgMorph.client, gasMultiplier, gasDivisor) if err != nil { - return util.Uint256{}, fmt.Errorf("could not calculate notary deposit: %w", err) + return util.Uint256{}, 0, fmt.Errorf("could not calculate notary deposit: %w", err) } return c.cfgMorph.client.DepositEndlessNotary(depositAmount) @@ -168,32 +169,43 @@ var ( errNotaryDepositTimeout = errors.New("notary deposit tx has not appeared in the network") ) -func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256) error { - for range notaryDepositRetriesAmount { - c.log.Debug(logs.ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted) - select { - case <-ctx.Done(): - return ctx.Err() - default: - } +type waiterClient struct { + c *client.Client +} - ok, err := c.cfgMorph.client.TxHalt(tx) - if err == nil { - if ok { - c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted) - return nil - } +func (w *waiterClient) Context() context.Context { + return context.Background() +} - return errNotaryDepositFail - } +func (w *waiterClient) GetApplicationLog(hash util.Uint256, trig *trigger.Type) (*result.ApplicationLog, error) { + return w.c.GetApplicationLog(hash, trig) +} - err = c.cfgMorph.client.Wait(ctx, 1) - if err != nil { - return fmt.Errorf("could not wait for one block in chain: %w", err) - } +func (w *waiterClient) GetBlockCount() (uint32, error) { + return w.c.BlockCount() +} + +func (w *waiterClient) GetVersion() (*result.Version, error) { + return w.c.GetVersion() +} + +func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256, vub uint32) error { + w, err := waiter.NewPollingBased(&waiterClient{c: c.cfgMorph.client}) + if err != nil { + return fmt.Errorf("could not create notary deposit waiter: %w", err) } - - return errNotaryDepositTimeout + res, err := w.WaitAny(ctx, vub, tx) + if err != nil { + if errors.Is(err, waiter.ErrTxNotAccepted) { + return errNotaryDepositTimeout + } + return fmt.Errorf("could not wait for notary deposit persists in chain: %w", err) + } + if res.Execution.VMState.HasFlag(vmstate.Halt) { + c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted) + return nil + } + return errNotaryDepositFail } func listenMorphNotifications(ctx context.Context, c *cfg) { diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 0e0571760..6e2a7c44a 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -192,7 +192,7 @@ func addNewEpochNotificationHandlers(c *cfg) { if c.cfgMorph.notaryEnabled { addNewEpochAsyncNotificationHandler(c, func(_ event.Event) { - _, err := makeNotaryDeposit(c) + _, _, err := makeNotaryDeposit(c) if err != nil { c.log.Error(logs.FrostFSNodeCouldNotMakeNotaryDeposit, zap.String("error", err.Error()), diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 0e9d58f32..e4bac4930 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -142,7 +142,6 @@ const ( ClientNotaryRequestWithPreparedMainTXInvoked = "notary request with prepared main TX invoked" ClientNotaryRequestInvoked = "notary request invoked" ClientNotaryDepositTransactionWasSuccessfullyPersisted = "notary deposit transaction was successfully persisted" - ClientAttemptToWaitForNotaryDepositTransactionToGetPersisted = "attempt to wait for notary deposit transaction to get persisted" ClientNeoClientInvoke = "neo client invoke" ClientNativeGasTransferInvoke = "native gas transfer invoke" ClientBatchGasTransferInvoke = "batch gas transfer invoke" diff --git a/pkg/innerring/notary.go b/pkg/innerring/notary.go index e6f2b1de4..5cdbb971c 100644 --- a/pkg/innerring/notary.go +++ b/pkg/innerring/notary.go @@ -40,13 +40,14 @@ func (s *Server) depositMainNotary() (tx util.Uint256, err error) { ) } -func (s *Server) depositSideNotary() (tx util.Uint256, err error) { +func (s *Server) depositSideNotary() (util.Uint256, error) { depositAmount, err := client.CalculateNotaryDepositAmount(s.morphClient, gasMultiplier, gasDivisor) if err != nil { return util.Uint256{}, fmt.Errorf("could not calculate side notary deposit amount: %w", err) } - return s.morphClient.DepositEndlessNotary(depositAmount) + tx, _, err := s.morphClient.DepositEndlessNotary(depositAmount) + return tx, err } func (s *Server) notaryHandler(_ event.Event) { diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index df521f56b..933f1039f 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -19,6 +19,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/gas" @@ -461,6 +462,28 @@ func (c *Client) TxHalt(h util.Uint256) (res bool, err error) { return len(aer.Executions) > 0 && aer.Executions[0].VMState.HasFlag(vmstate.Halt), nil } +func (c *Client) GetApplicationLog(hash util.Uint256, trig *trigger.Type) (*result.ApplicationLog, error) { + c.switchLock.RLock() + defer c.switchLock.RUnlock() + + if c.inactive { + return nil, ErrConnectionLost + } + + return c.client.GetApplicationLog(hash, trig) +} + +func (c *Client) GetVersion() (*result.Version, error) { + c.switchLock.RLock() + defer c.switchLock.RUnlock() + + if c.inactive { + return nil, ErrConnectionLost + } + + return c.client.GetVersion() +} + // TxHeight returns true if transaction has been successfully executed and persisted. func (c *Client) TxHeight(h util.Uint256) (res uint32, err error) { c.switchLock.RLock() diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 616b3b5c3..2a500b31b 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -140,7 +140,7 @@ func (c *Client) ProbeNotary() (res bool) { // use this function. // // This function must be invoked with notary enabled otherwise it throws panic. -func (c *Client) DepositNotary(amount fixedn.Fixed8, delta uint32) (res util.Uint256, err error) { +func (c *Client) DepositNotary(amount fixedn.Fixed8, delta uint32) (util.Uint256, error) { c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -163,7 +163,8 @@ func (c *Client) DepositNotary(amount fixedn.Fixed8, delta uint32) (res util.Uin } till := max(int64(bc+delta), currentTill) - return c.depositNotary(amount, till) + res, _, err := c.depositNotary(amount, till) + return res, err } // DepositEndlessNotary calls notary deposit method. Unlike `DepositNotary`, @@ -171,12 +172,12 @@ func (c *Client) DepositNotary(amount fixedn.Fixed8, delta uint32) (res util.Uin // This allows to avoid ValidAfterDeposit failures. // // This function must be invoked with notary enabled otherwise it throws panic. -func (c *Client) DepositEndlessNotary(amount fixedn.Fixed8) (res util.Uint256, err error) { +func (c *Client) DepositEndlessNotary(amount fixedn.Fixed8) (util.Uint256, uint32, error) { c.switchLock.RLock() defer c.switchLock.RUnlock() if c.inactive { - return util.Uint256{}, ErrConnectionLost + return util.Uint256{}, 0, ErrConnectionLost } if c.notary == nil { @@ -187,7 +188,7 @@ func (c *Client) DepositEndlessNotary(amount fixedn.Fixed8) (res util.Uint256, e return c.depositNotary(amount, math.MaxUint32) } -func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (res util.Uint256, err error) { +func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (util.Uint256, uint32, error) { txHash, vub, err := c.gasToken.Transfer( c.accAddr, c.notary.notary, @@ -195,7 +196,7 @@ func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (res util.Uint2 []any{c.acc.PrivateKey().GetScriptHash(), till}) if err != nil { if !errors.Is(err, neorpc.ErrAlreadyExists) { - return util.Uint256{}, fmt.Errorf("can't make notary deposit: %w", err) + return util.Uint256{}, 0, fmt.Errorf("can't make notary deposit: %w", err) } // Transaction is already in mempool waiting to be processed. @@ -205,7 +206,7 @@ func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (res util.Uint2 zap.Int64("expire_at", till), zap.Uint32("vub", vub), zap.Error(err)) - return util.Uint256{}, nil + return util.Uint256{}, 0, nil } c.logger.Info(logs.ClientNotaryDepositInvoke, @@ -214,7 +215,7 @@ func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (res util.Uint2 zap.Uint32("vub", vub), zap.Stringer("tx_hash", txHash.Reverse())) - return txHash, nil + return txHash, vub, nil } // GetNotaryDeposit returns deposit of client's account in notary contract. From 17ec84151b4f9f3d2ff593ecdad41b8c0c860e02 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 5 Nov 2024 16:15:59 +0300 Subject: [PATCH 1492/1943] [#532] cli: Respect XDG base directory spec XDG base directory specification defines where various files should be looked by an application. Hopefully, this makes `frostfs-cli` more predictable and pleasant to work with. Luckily for us, golang already has everything we need in the stdlib. This commit also gets rid of `github.com/mitchellh/go-homedir` dependency. Close #532 Refs #1455 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/root.go | 11 +++++------ go.mod | 1 - go.sum | 2 -- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-cli/modules/root.go b/cmd/frostfs-cli/modules/root.go index c0282586a..88acab341 100644 --- a/cmd/frostfs-cli/modules/root.go +++ b/cmd/frostfs-cli/modules/root.go @@ -21,7 +21,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/gendoc" - "github.com/mitchellh/go-homedir" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -112,13 +111,13 @@ func initConfig() { // Use config file from the flag. viper.SetConfigFile(cfgFile) } else { - // Find home directory. - home, err := homedir.Dir() + // Find config directory. + configDir, err := os.UserConfigDir() if err != nil { - common.PrintVerbose(rootCmd, "Get homedir: %s", err) + common.PrintVerbose(rootCmd, "Get config dir: %s", err) } else { - // Search config in `$HOME/.config/frostfs-cli/` with name "config.yaml" - viper.AddConfigPath(filepath.Join(home, ".config", "frostfs-cli")) + // Search config in `$XDG_CONFIG_HOME/frostfs-cli/` with name "config.yaml" + viper.AddConfigPath(filepath.Join(configDir, "frostfs-cli")) viper.SetConfigName("config") viper.SetConfigType("yaml") } diff --git a/go.mod b/go.mod index ba9ec375c..886fa958f 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,6 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/klauspost/compress v1.17.4 github.com/mailru/easyjson v0.7.7 - github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.12.1 github.com/nspcc-dev/neo-go v0.106.3 diff --git a/go.sum b/go.sum index a037a99af..6ed130cdb 100644 --- a/go.sum +++ b/go.sum @@ -179,8 +179,6 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= From 15102e6dfd3eddca34d83e2264c79d05d665939f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Nov 2024 10:34:16 +0300 Subject: [PATCH 1493/1943] [#1471] Replace sort.Slice in some places MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `slices.SortFunc` doesn't use reflection and is a bit faster. I have done some micro-benchmarks for `[]NodeInfo`: ``` $ benchstat -col "/func" out goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ sort.Slice │ slices.SortFunc │ │ sec/op │ sec/op vs base │ Sort-8 2.130µ ± 2% 1.253µ ± 2% -41.20% (p=0.000 n=10) ``` Haven't included them, though, as they I don't see them being used a lot. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/client/client.go | 5 ++--- cmd/frostfs-cli/internal/common/tracing.go | 10 +++------- pkg/local_object_storage/pilorama/batch.go | 6 +++--- pkg/local_object_storage/pilorama/boltdb.go | 12 ++++++++---- pkg/local_object_storage/pilorama/forest.go | 5 +---- pkg/local_object_storage/pilorama/inmemory.go | 13 ++++++++----- pkg/services/tree/service.go | 9 ++++----- 7 files changed, 29 insertions(+), 31 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index ed9817b86..948d61f36 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -670,9 +670,8 @@ func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes return nil, fmt.Errorf("read object list: %w", err) } - sort.Slice(list, func(i, j int) bool { - lhs, rhs := list[i].EncodeToString(), list[j].EncodeToString() - return strings.Compare(lhs, rhs) < 0 + slices.SortFunc(list, func(a, b oid.ID) int { + return strings.Compare(a.EncodeToString(), b.EncodeToString()) }) return &SearchObjectsRes{ diff --git a/cmd/frostfs-cli/internal/common/tracing.go b/cmd/frostfs-cli/internal/common/tracing.go index 30c2f2b1a..10863ed1e 100644 --- a/cmd/frostfs-cli/internal/common/tracing.go +++ b/cmd/frostfs-cli/internal/common/tracing.go @@ -2,7 +2,7 @@ package common import ( "context" - "sort" + "slices" "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" @@ -45,15 +45,11 @@ func StartClientCommandSpan(cmd *cobra.Command) { }) commonCmd.ExitOnErr(cmd, "init tracing: %w", err) - var components sort.StringSlice + var components []string for c := cmd; c != nil; c = c.Parent() { components = append(components, c.Name()) } - for i, j := 0, len(components)-1; i < j; { - components.Swap(i, j) - i++ - j-- - } + slices.Reverse(components) operation := strings.Join(components, ".") ctx, span := tracing.StartSpanFromContext(cmd.Context(), operation) diff --git a/pkg/local_object_storage/pilorama/batch.go b/pkg/local_object_storage/pilorama/batch.go index 520c6dfb4..4c5238921 100644 --- a/pkg/local_object_storage/pilorama/batch.go +++ b/pkg/local_object_storage/pilorama/batch.go @@ -1,9 +1,9 @@ package pilorama import ( + "cmp" "encoding/binary" "slices" - "sort" "sync" "time" @@ -48,8 +48,8 @@ func (b *batch) run() { // Sorting without a mutex is ok, because we append to this slice only if timer is non-nil. // See (*boltForest).addBatch for details. - sort.Slice(b.operations, func(i, j int) bool { - return b.operations[i].Time < b.operations[j].Time + slices.SortFunc(b.operations, func(mi, mj *Move) int { + return cmp.Compare(mi.Time, mj.Time) }) b.operations = slices.CompactFunc(b.operations, func(x, y *Move) bool { return x.Time == y.Time }) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 09f2e1919..7bce1f340 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "slices" - "sort" "strconv" "sync" "time" @@ -1093,14 +1092,19 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr return res, last, metaerr.Wrap(err) } +func sortByFilename(nodes []NodeInfo) { + slices.SortFunc(nodes, func(a, b NodeInfo) int { + return bytes.Compare(a.Meta.GetAttr(AttributeFilename), b.Meta.GetAttr(AttributeFilename)) + }) +} + func sortAndCut(result []NodeInfo, last *string) []NodeInfo { var lastBytes []byte if last != nil { lastBytes = []byte(*last) } - sort.Slice(result, func(i, j int) bool { - return bytes.Compare(result[i].Meta.GetAttr(AttributeFilename), result[j].Meta.GetAttr(AttributeFilename)) == -1 - }) + sortByFilename(result) + for i := range result { if lastBytes == nil || bytes.Compare(lastBytes, result[i].Meta.GetAttr(AttributeFilename)) == -1 { return result[i:] diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 78503bada..bb5c22e51 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -1,7 +1,6 @@ package pilorama import ( - "bytes" "context" "errors" "fmt" @@ -192,9 +191,7 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI return nil, start, nil } - sort.Slice(res, func(i, j int) bool { - return bytes.Compare(res[i].Meta.GetAttr(AttributeFilename), res[j].Meta.GetAttr(AttributeFilename)) == -1 - }) + sortByFilename(res) r := mergeNodeInfos(res) for i := range r { diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index c9f5df3b7..ce7b3db1e 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -1,6 +1,9 @@ package pilorama -import "sort" +import ( + "cmp" + "slices" +) // nodeInfo couples parent and metadata. type nodeInfo struct { @@ -131,10 +134,10 @@ func (t tree) getChildren(parent Node) []Node { } } - sort.Slice(children, func(i, j int) bool { - a := t.infoMap[children[i]] - b := t.infoMap[children[j]] - return a.Meta.Time < b.Meta.Time + slices.SortFunc(children, func(ci, cj uint64) int { + a := t.infoMap[ci] + b := t.infoMap[cj] + return cmp.Compare(a.Meta.Time, b.Meta.Time) }) return children } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 10c3b6ccc..8097d545c 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -5,7 +5,7 @@ import ( "context" "errors" "fmt" - "sort" + "slices" "sync" "sync/atomic" @@ -575,10 +575,9 @@ func sortByFilename(nodes []pilorama.NodeInfo, d GetSubTreeRequest_Body_Order_Di if len(nodes) == 0 { return nodes, nil } - less := func(i, j int) bool { - return bytes.Compare(nodes[i].Meta.GetAttr(pilorama.AttributeFilename), nodes[j].Meta.GetAttr(pilorama.AttributeFilename)) < 0 - } - sort.Slice(nodes, less) + slices.SortFunc(nodes, func(a, b pilorama.NodeInfo) int { + return bytes.Compare(a.Meta.GetAttr(pilorama.AttributeFilename), b.Meta.GetAttr(pilorama.AttributeFilename)) + }) return nodes, nil default: return nil, fmt.Errorf("unsupported order direction: %s", d.String()) From 33ad753302754c102247424b03ec4d1d8b9042c3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 6 Nov 2024 10:57:01 +0300 Subject: [PATCH 1494/1943] [#1473] policer: Add tracing span To filter HEAD requests from policer. Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/check.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 401977f66..dbc9ea53c 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -9,14 +9,25 @@ import ( objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" policycore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) func (p *Policer) processObject(ctx context.Context, objInfo objectcore.Info) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Policer.ProcessObject", trace.WithAttributes( + attribute.String("address", objInfo.Address.String()), + attribute.Bool("is_linking_object", objInfo.IsLinkingObject), + attribute.Bool("is_ec_part", objInfo.ECInfo != nil), + attribute.String("type", objInfo.Type.String()), + )) + defer span.End() + cnr, err := p.cnrSrc.Get(objInfo.Address.Container()) if err != nil { if client.IsErrContainerNotFound(err) { From 9902965ff49094c83b755e2f81bd386f2a74c347 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 28 Oct 2024 15:44:47 +0300 Subject: [PATCH 1495/1943] [#1451] writer: Sign EC parts with node's private key As EC put request may be processed only by container node, so sign requests with current node private to not to perform APE checks. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/common/writer/ec.go | 34 ++++++++++++++------ pkg/services/object/common/writer/ec_test.go | 4 +++ pkg/services/object/util/prm.go | 9 +++++- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/pkg/services/object/common/writer/ec.go b/pkg/services/object/common/writer/ec.go index ee0681349..3f7d4d49c 100644 --- a/pkg/services/object/common/writer/ec.go +++ b/pkg/services/object/common/writer/ec.go @@ -37,10 +37,12 @@ type ECWriter struct { ObjectMeta object.ContentMeta ObjectMetaValid bool + + remoteRequestSignKey *ecdsa.PrivateKey } func (e *ECWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error { - relayed, err := e.relayIfNotContainerNode(ctx, obj) + relayed, isContainerNode, err := e.relayIfNotContainerNode(ctx, obj) if err != nil { return err } @@ -60,23 +62,35 @@ func (e *ECWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error e.ObjectMetaValid = true } + if isContainerNode { + restoreTokens := e.CommonPrm.ForgetTokens() + defer restoreTokens() + // As request executed on container node, so sign request with container key. + e.remoteRequestSignKey, err = e.Config.KeyStorage.GetKey(nil) + if err != nil { + return err + } + } else { + e.remoteRequestSignKey = e.Key + } + if obj.ECHeader() != nil { return e.writeECPart(ctx, obj) } return e.writeRawObject(ctx, obj) } -func (e *ECWriter) relayIfNotContainerNode(ctx context.Context, obj *objectSDK.Object) (bool, error) { - if e.Relay == nil { - return false, nil - } +func (e *ECWriter) relayIfNotContainerNode(ctx context.Context, obj *objectSDK.Object) (bool, bool, error) { currentNodeIsContainerNode, err := e.currentNodeIsContainerNode() if err != nil { - return false, err + return false, false, err } if currentNodeIsContainerNode { // object can be splitted or saved local - return false, nil + return false, true, nil + } + if e.Relay == nil { + return false, currentNodeIsContainerNode, nil } objID := object.AddressOf(obj).Object() var index uint32 @@ -85,9 +99,9 @@ func (e *ECWriter) relayIfNotContainerNode(ctx context.Context, obj *objectSDK.O index = obj.ECHeader().Index() } if err := e.relayToContainerNode(ctx, objID, index); err != nil { - return false, err + return false, false, err } - return true, nil + return true, currentNodeIsContainerNode, nil } func (e *ECWriter) currentNodeIsContainerNode() (bool, error) { @@ -338,7 +352,7 @@ func (e *ECWriter) writePartRemote(ctx context.Context, obj *objectSDK.Object, n client.NodeInfoFromNetmapElement(&clientNodeInfo, node) remoteTaget := remoteWriter{ - privateKey: e.Key, + privateKey: e.remoteRequestSignKey, clientConstructor: e.Config.ClientConstructor, commonPrm: e.CommonPrm, nodeInfo: clientNodeInfo, diff --git a/pkg/services/object/common/writer/ec_test.go b/pkg/services/object/common/writer/ec_test.go index 32863d678..c828c79ba 100644 --- a/pkg/services/object/common/writer/ec_test.go +++ b/pkg/services/object/common/writer/ec_test.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" @@ -127,6 +128,8 @@ func TestECWriter(t *testing.T) { ownerKey, err := keys.NewPrivateKey() require.NoError(t, err) + nodeKey, err := keys.NewPrivateKey() + require.NoError(t, err) pool, err := ants.NewPool(4, ants.WithNonblocking(true)) require.NoError(t, err) @@ -141,6 +144,7 @@ func TestECWriter(t *testing.T) { RemotePool: pool, Logger: log, ClientConstructor: clientConstructor{vectors: ns}, + KeyStorage: util.NewKeyStorage(&nodeKey.PrivateKey, nil, nil), }, PlacementOpts: append( []placement.Option{placement.UseBuilder(builder), placement.ForContainer(cnr)}, diff --git a/pkg/services/object/util/prm.go b/pkg/services/object/util/prm.go index 022b9fe5b..80c0db39e 100644 --- a/pkg/services/object/util/prm.go +++ b/pkg/services/object/util/prm.go @@ -100,11 +100,18 @@ func (p *CommonPrm) SetNetmapLookupDepth(v uint64) { // ForgetTokens forgets all the tokens read from the request's // meta information before. -func (p *CommonPrm) ForgetTokens() { +func (p *CommonPrm) ForgetTokens() func() { if p != nil { + tk := p.token + br := p.bearer p.token = nil p.bearer = nil + return func() { + p.token = tk + p.bearer = br + } } + return func() {} } func CommonPrmFromV2(req interface { From 5b1ba8e23dfbeae687deb19aaebce316cc266ad9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 28 Oct 2024 15:46:38 +0300 Subject: [PATCH 1496/1943] [#1451] ape: Perform strict APE checks for EC parts Signed-off-by: Dmitrii Stepanov --- pkg/services/object/ape/checker_test.go | 1 + pkg/services/object/ape/request.go | 44 ++++++++++++++++--------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index 090f6a83c..7ebd147f3 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -695,6 +695,7 @@ func TestPutECChunk(t *testing.T) { nm := &netmapStub{ currentEpoch: 100, netmaps: map[uint64]*netmapSDK.NetMap{ + 99: netmap, 100: netmap, }, } diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index da5307ca7..d07e59067 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -3,6 +3,7 @@ package ape import ( "context" "crypto/sha256" + "errors" "fmt" "net" "strconv" @@ -11,6 +12,7 @@ import ( aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -24,6 +26,8 @@ import ( var defaultRequest = aperequest.Request{} +var errECMissingParentObjectID = errors.New("missing EC parent object ID") + func nativeSchemaRole(role acl.Role) string { switch role { case acl.RoleOwner: @@ -122,7 +126,10 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re header = headerObjSDK.ToV2().GetHeader() } } - header = c.fillHeaderWithECParent(ctx, prm, header) + header, err := c.fillHeaderWithECParent(ctx, prm, header) + if err != nil { + return defaultRequest, fmt.Errorf("get EC parent header: %w", err) + } reqProps := map[string]string{ nativeschema.PropertyKeyActorPublicKey: prm.SenderKey, nativeschema.PropertyKeyActorRole: prm.Role, @@ -133,7 +140,6 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re reqProps[xheadKey] = xhead.GetValue() } - var err error reqProps, err = c.fillWithUserClaimTags(reqProps, prm) if err != nil { return defaultRequest, err @@ -155,35 +161,43 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re ), nil } -func (c *checkerImpl) fillHeaderWithECParent(ctx context.Context, prm Prm, header *objectV2.Header) *objectV2.Header { +func (c *checkerImpl) fillHeaderWithECParent(ctx context.Context, prm Prm, header *objectV2.Header) (*objectV2.Header, error) { if header == nil { - return header + return header, nil } if header.GetEC() == nil { - return header - } - if prm.Role == nativeschema.PropertyValueContainerRoleContainer || - prm.Role == nativeschema.PropertyValueContainerRoleIR { - return header + return header, nil } parentObjRefID := header.GetEC().Parent if parentObjRefID == nil { - return header + return nil, errECMissingParentObjectID } var parentObjID oid.ID if err := parentObjID.ReadFromV2(*parentObjRefID); err != nil { - return header + return nil, fmt.Errorf("EC parent object ID format error: %w", err) } // only container node have access to collect parent object contNode, err := c.currentNodeIsContainerNode(prm.Container) - if err != nil || !contNode { - return header + if err != nil { + return nil, fmt.Errorf("check container node status: %w", err) + } + if !contNode { + return header, nil } parentObj, err := c.headerProvider.GetHeader(ctx, prm.Container, parentObjID, false) if err != nil { - return header + if isLogicalError(err) { + return header, nil + } + return nil, fmt.Errorf("EC parent header request: %w", err) } - return parentObj.ToV2().GetHeader() + return parentObj.ToV2().GetHeader(), nil +} + +func isLogicalError(err error) bool { + var errObjRemoved *apistatus.ObjectAlreadyRemoved + var errObjNotFound *apistatus.ObjectNotFound + return errors.As(err, &errObjRemoved) || errors.As(err, &errObjNotFound) } func (c *checkerImpl) currentNodeIsContainerNode(cnrID cid.ID) (bool, error) { From 9a77527f46f2db09628bef5b4567ae77cbdb3f69 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 28 Oct 2024 16:18:28 +0300 Subject: [PATCH 1497/1943] [#1451] ape: Drop unused Signed-off-by: Dmitrii Stepanov --- pkg/services/object/ape/checker.go | 3 --- pkg/services/object/ape/request.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 3f6cc7c20..8ce1b429d 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -67,9 +67,6 @@ type Prm struct { // If SoftAPECheck is set to true, then NoRuleFound is interpreted as allow. SoftAPECheck bool - // If true, object headers will not retrieved from storage engine. - WithoutHeaderRequest bool - // The request's bearer token. It is used in order to check APE overrides with the token. BearerToken *bearer.Token diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index d07e59067..e12fccb5e 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -120,7 +120,7 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re var header *objectV2.Header if prm.Header != nil { header = prm.Header - } else if prm.Object != nil && !prm.WithoutHeaderRequest { + } else if prm.Object != nil { headerObjSDK, err := c.headerProvider.GetHeader(ctx, prm.Container, *prm.Object, true) if err == nil { header = headerObjSDK.ToV2().GetHeader() From 3cf6ea745da38fe2dede263c88275fd4e96576e0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 29 Oct 2024 12:25:59 +0300 Subject: [PATCH 1498/1943] [#1451] ec: Check all parts are saved Signed-off-by: Dmitrii Stepanov --- pkg/services/object/common/writer/ec.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/services/object/common/writer/ec.go b/pkg/services/object/common/writer/ec.go index 3f7d4d49c..571bae7bb 100644 --- a/pkg/services/object/common/writer/ec.go +++ b/pkg/services/object/common/writer/ec.go @@ -25,7 +25,10 @@ import ( var _ transformer.ObjectWriter = (*ECWriter)(nil) -var errUnsupportedECObject = errors.New("object is not supported for erasure coding") +var ( + errUnsupportedECObject = errors.New("object is not supported for erasure coding") + errFailedToSaveAllECParts = errors.New("failed to save all EC parts") +) type ECWriter struct { Config *Config @@ -249,6 +252,13 @@ func (e *ECWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) er singleErr: err, } } + for idx := range partsProcessed { + if !partsProcessed[idx].Load() { + return errIncompletePut{ + singleErr: errFailedToSaveAllECParts, + } + } + } return nil } From 7edec9193cf7fd3d3f68a7cae3def81247b525eb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 29 Oct 2024 12:39:50 +0300 Subject: [PATCH 1499/1943] [#1451] placement: Return copy of slice from container nodes cache Nodes from cache could be changed by traverser, if no objectID specified. So it is required to return copy of cache's slice. Signed-off-by: Dmitrii Stepanov --- pkg/services/object_manager/placement/cache.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/services/object_manager/placement/cache.go b/pkg/services/object_manager/placement/cache.go index 217261877..2a8460ca5 100644 --- a/pkg/services/object_manager/placement/cache.go +++ b/pkg/services/object_manager/placement/cache.go @@ -3,6 +3,7 @@ package placement import ( "crypto/sha256" "fmt" + "slices" "sync" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -44,7 +45,7 @@ func (c *ContainerNodesCache) ContainerNodes(nm *netmapSDK.NetMap, cnr cid.ID, p raw, ok := c.containerCache.Get(cnr) c.mtx.Unlock() if ok { - return raw, nil + return c.cloneResult(raw), nil } } else { c.lastEpoch = nm.Epoch() @@ -65,5 +66,13 @@ func (c *ContainerNodesCache) ContainerNodes(nm *netmapSDK.NetMap, cnr cid.ID, p c.containerCache.Add(cnr, cn) } c.mtx.Unlock() - return cn, nil + return c.cloneResult(cn), nil +} + +func (c *ContainerNodesCache) cloneResult(nodes [][]netmapSDK.NodeInfo) [][]netmapSDK.NodeInfo { + result := make([][]netmapSDK.NodeInfo, len(nodes)) + for repIdx := range nodes { + result[repIdx] = slices.Clone(nodes[repIdx]) + } + return result } From c8fb154151c4a944c87bea373fe9828ac267f2ea Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 6 Nov 2024 15:25:02 +0300 Subject: [PATCH 1500/1943] [#1475] Remove container estimation code Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 3 - pkg/innerring/blocktimer.go | 53 +------ pkg/innerring/blocktimer_test.go | 42 +----- pkg/innerring/initialization.go | 15 +- pkg/innerring/innerring.go | 2 +- .../processors/netmap/handlers_test.go | 19 --- .../processors/netmap/process_epoch.go | 15 -- pkg/innerring/processors/netmap/processor.go | 10 -- pkg/morph/client/container/client.go | 6 - pkg/morph/client/container/estimations.go | 54 -------- pkg/morph/client/container/load.go | 131 ------------------ pkg/morph/event/container/estimates.go | 78 ----------- pkg/morph/event/container/estimates_test.go | 80 ----------- 13 files changed, 9 insertions(+), 499 deletions(-) delete mode 100644 pkg/morph/client/container/estimations.go delete mode 100644 pkg/morph/client/container/load.go delete mode 100644 pkg/morph/event/container/estimates.go delete mode 100644 pkg/morph/event/container/estimates_test.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index e4bac4930..d0bac4d11 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -17,8 +17,6 @@ const ( ) const ( - InnerringNonalphabetModeDoNotStopContainerEstimations = "non-alphabet mode, do not stop container estimations" - InnerringCantStopEpochEstimation = "can't stop epoch estimation" InnerringCantMakeNotaryDepositInMainChain = "can't make notary deposit in main chain" InnerringCantMakeNotaryDepositInSideChain = "can't make notary deposit in side chain" InnerringNotaryDepositHasAlreadyBeenMade = "notary deposit has already been made" @@ -343,7 +341,6 @@ const ( NetmapCantGetTransactionHeight = "can't get transaction height" NetmapCantResetEpochTimer = "can't reset epoch timer" NetmapCantGetNetmapSnapshotToPerformCleanup = "can't get netmap snapshot to perform cleanup" - NetmapCantStartContainerSizeEstimation = "can't start container size estimation" NetmapNonAlphabetModeIgnoreNewEpochTick = "non alphabet mode, ignore new epoch tick" NetmapNextEpoch = "next epoch" NetmapCantInvokeNetmapNewEpoch = "can't invoke netmap.NewEpoch" diff --git a/pkg/innerring/blocktimer.go b/pkg/innerring/blocktimer.go index ad69f207b..3db504368 100644 --- a/pkg/innerring/blocktimer.go +++ b/pkg/innerring/blocktimer.go @@ -3,14 +3,10 @@ package innerring import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/timer" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/util" - "go.uber.org/zap" ) type ( @@ -19,28 +15,12 @@ type ( EpochDuration() uint64 } - alphaState interface { - IsAlphabet() bool - } - newEpochHandler func() - containerEstimationStopper interface { - StopEstimation(p container.StopEstimationPrm) error - } - epochTimerArgs struct { - l *logger.Logger - - alphabetState alphaState - newEpochHandlers []newEpochHandler - cnrWrapper containerEstimationStopper // to invoke stop container estimation - epoch epochState // to specify which epoch to stop, and epoch duration - - stopEstimationDMul uint32 // X: X/Y of epoch in blocks - stopEstimationDDiv uint32 // Y: X/Y of epoch in blocks + epoch epochState // to specify which epoch to stop, and epoch duration } emitTimerArgs struct { @@ -74,7 +54,7 @@ func (s *Server) tickTimers(h uint32) { } func newEpochTimer(args *epochTimerArgs) *timer.BlockTimer { - epochTimer := timer.NewBlockTimer( + return timer.NewBlockTimer( func() (uint32, error) { return uint32(args.epoch.EpochDuration()), nil }, @@ -84,35 +64,6 @@ func newEpochTimer(args *epochTimerArgs) *timer.BlockTimer { } }, ) - - // sub-timer for epoch timer to tick stop container estimation events at - // some block in epoch - epochTimer.OnDelta( - args.stopEstimationDMul, - args.stopEstimationDDiv, - func() { - if !args.alphabetState.IsAlphabet() { - args.l.Debug(logs.InnerringNonalphabetModeDoNotStopContainerEstimations) - return - } - - epochN := args.epoch.EpochCounter() - if epochN == 0 { // estimates are invalid in genesis epoch - return - } - - prm := container.StopEstimationPrm{} - prm.SetEpoch(epochN - 1) - - err := args.cnrWrapper.StopEstimation(prm) - if err != nil { - args.l.Warn(logs.InnerringCantStopEpochEstimation, - zap.Uint64("epoch", epochN), - zap.String("error", err.Error())) - } - }) - - return epochTimer } func newEmissionTimer(args *emitTimerArgs) *timer.BlockTimer { diff --git a/pkg/innerring/blocktimer_test.go b/pkg/innerring/blocktimer_test.go index 242c0903b..4cbe7e394 100644 --- a/pkg/innerring/blocktimer_test.go +++ b/pkg/innerring/blocktimer_test.go @@ -3,29 +3,20 @@ package innerring import ( "testing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "github.com/stretchr/testify/require" ) func TestEpochTimer(t *testing.T) { t.Parallel() - alphaState := &testAlphabetState{isAlphabet: true} neh := &testNewEpochHandler{} - cnrStopper := &testContainerEstStopper{} epochState := &testEpochState{ counter: 99, duration: 10, } args := &epochTimerArgs{ - l: test.NewLogger(t), - alphabetState: alphaState, - newEpochHandlers: []newEpochHandler{neh.Handle}, - cnrWrapper: cnrStopper, - epoch: epochState, - stopEstimationDMul: 2, - stopEstimationDDiv: 10, + newEpochHandlers: []newEpochHandler{neh.Handle}, + epoch: epochState, } et := newEpochTimer(args) err := et.Reset() @@ -33,63 +24,43 @@ func TestEpochTimer(t *testing.T) { et.Tick(100) require.Equal(t, 0, neh.called, "invalid new epoch handler calls") - require.Equal(t, 0, cnrStopper.called, "invalid container stop handler calls") et.Tick(101) require.Equal(t, 0, neh.called, "invalid new epoch handler calls") - require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") et.Tick(102) require.Equal(t, 0, neh.called, "invalid new epoch handler calls") - require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") et.Tick(103) require.Equal(t, 0, neh.called, "invalid new epoch handler calls") - require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") var h uint32 for h = 104; h < 109; h++ { et.Tick(h) require.Equal(t, 0, neh.called, "invalid new epoch handler calls") - require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") } et.Tick(109) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") - require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") et.Tick(110) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") - require.Equal(t, 1, cnrStopper.called, "invalid container stop handler calls") et.Tick(111) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") - require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") et.Tick(112) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") - require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") et.Tick(113) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") - require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") for h = 114; h < 119; h++ { et.Tick(h) require.Equal(t, 1, neh.called, "invalid new epoch handler calls") - require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") } et.Tick(120) require.Equal(t, 2, neh.called, "invalid new epoch handler calls") - require.Equal(t, 2, cnrStopper.called, "invalid container stop handler calls") -} - -type testAlphabetState struct { - isAlphabet bool -} - -func (s *testAlphabetState) IsAlphabet() bool { - return s.isAlphabet } type testNewEpochHandler struct { @@ -100,15 +71,6 @@ func (h *testNewEpochHandler) Handle() { h.called++ } -type testContainerEstStopper struct { - called int -} - -func (s *testContainerEstStopper) StopEstimation(_ container.StopEstimationPrm) error { - s.called++ - return nil -} - type testEpochState struct { counter uint64 duration uint64 diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index cb0654b6e..d6b474c32 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -36,7 +36,6 @@ import ( ) func (s *Server) initNetmapProcessor(cfg *viper.Viper, - cnrClient *container.Client, alphaSync event.Handler, ) error { locodeValidator, err := s.newLocodeValidator(cfg) @@ -59,7 +58,6 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, AlphabetState: s, CleanupEnabled: cfg.GetBool("netmap_cleaner.enabled"), CleanupThreshold: cfg.GetUint64("netmap_cleaner.threshold"), - ContainerWrapper: cnrClient, NotaryDepositHandler: s.onlyAlphabetEventHandler( s.notaryHandler, ), @@ -198,15 +196,10 @@ func (s *Server) createIRFetcher() irFetcher { return irf } -func (s *Server) initTimers(cfg *viper.Viper, morphClients *serverMorphClients) { +func (s *Server) initTimers(cfg *viper.Viper) { s.epochTimer = newEpochTimer(&epochTimerArgs{ - l: s.log, - alphabetState: s, - newEpochHandlers: s.newEpochTickHandlers(), - cnrWrapper: morphClients.CnrClient, - epoch: s, - stopEstimationDMul: cfg.GetUint32("timers.stop_estimation.mul"), - stopEstimationDDiv: cfg.GetUint32("timers.stop_estimation.div"), + newEpochHandlers: s.newEpochTickHandlers(), + epoch: s, }) s.addBlockTimer(s.epochTimer) @@ -425,7 +418,7 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien return err } - err = s.initNetmapProcessor(cfg, morphClients.CnrClient, alphaSync) + err = s.initNetmapProcessor(cfg, alphaSync) if err != nil { return err } diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index b94312645..5fae302c4 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -405,7 +405,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - server.initTimers(cfg, morphClients) + server.initTimers(cfg) err = server.initGRPCServer(cfg, log, audit) if err != nil { diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index 8875880bf..35f4469b1 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -9,7 +9,6 @@ import ( netmapContract "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" @@ -68,7 +67,6 @@ func TestNewEpoch(t *testing.T) { duration: 10, } r := &testEpochResetter{} - cc := &testContainerClient{} nc := &testNetmapClient{ epochDuration: 20, txHeights: map[util.Uint256]uint32{ @@ -82,7 +80,6 @@ func TestNewEpoch(t *testing.T) { p.NotaryDepositHandler = eh.Handle p.AlphabetSyncHandler = eh.Handle p.NetmapClient = nc - p.ContainerWrapper = cc p.EpochTimer = r p.EpochState = es }) @@ -103,11 +100,6 @@ func TestNewEpoch(t *testing.T) { require.Equal(t, ev.Num, es.counter, "invalid epoch counter") require.EqualValues(t, []uint32{nc.txHeights[ev.Hash]}, r.timers, "invalid epoch timer resets") - var expEstimation cntClient.StartEstimationPrm - expEstimation.SetEpoch(ev.Num - 1) - expEstimation.SetHash(ev.Hash) - require.EqualValues(t, []cntClient.StartEstimationPrm{expEstimation}, cc.estimations, "invalid estimations") - require.EqualValues(t, []event.Event{ governance.NewSyncEvent(ev.TxHash()), ev, @@ -274,7 +266,6 @@ func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { as := &testAlphabetState{ isAlphabet: true, } - cc := &testContainerClient{} nc := &testNetmapClient{} eh := &testEventHandler{} @@ -288,7 +279,6 @@ func newTestProc(t *testing.T, nonDefault func(p *Params)) (*Processor, error) { EpochState: es, EpochTimer: r, AlphabetState: as, - ContainerWrapper: cc, NetmapClient: nc, NotaryDepositHandler: eh.Handle, AlphabetSyncHandler: eh.Handle, @@ -354,15 +344,6 @@ func (s *testAlphabetState) IsAlphabet() bool { return s.isAlphabet } -type testContainerClient struct { - estimations []cntClient.StartEstimationPrm -} - -func (c *testContainerClient) StartEstimation(p cntClient.StartEstimationPrm) error { - c.estimations = append(c.estimations, p) - return nil -} - type notaryInvoke struct { contract util.Uint160 fee fixedn.Fixed8 diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index 4dfa3997b..9522df26c 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -3,7 +3,6 @@ package netmap import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" "go.uber.org/zap" ) @@ -44,20 +43,6 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) bool { return false } - prm := cntClient.StartEstimationPrm{} - - prm.SetEpoch(epoch - 1) - prm.SetHash(ev.TxHash()) - - if epoch > 0 && np.alphabetState.IsAlphabet() { // estimates are invalid in genesis epoch - err = np.containerWrp.StartEstimation(prm) - if err != nil { - np.log.Warn(logs.NetmapCantStartContainerSizeEstimation, - zap.Uint64("epoch", epoch), - zap.String("error", err.Error())) - } - } - np.netmapSnapshot.update(*networkMap, epoch) np.handleCleanupTick(netmapCleanupTick{epoch: epoch, txHash: ev.TxHash()}) np.handleAlphabetSync(governance.NewSyncEvent(ev.TxHash())) diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index 4cecda59c..f5a91dee2 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -65,10 +64,6 @@ type ( MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error } - ContainerClient interface { - StartEstimation(p cntClient.StartEstimationPrm) error - } - // Processor of events produced by network map contract // and new epoch ticker, because it is related to contract. Processor struct { @@ -80,7 +75,6 @@ type ( alphabetState AlphabetState netmapClient Client - containerWrp ContainerClient netmapSnapshot cleanupTable @@ -103,7 +97,6 @@ type ( AlphabetState AlphabetState CleanupEnabled bool CleanupThreshold uint64 // in epochs - ContainerWrapper ContainerClient AlphabetSyncHandler event.Handler NotaryDepositHandler event.Handler @@ -133,8 +126,6 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/netmap: alphabet sync handler is not set") case p.NotaryDepositHandler == nil: return nil, errors.New("ir/netmap: notary deposit handler is not set") - case p.ContainerWrapper == nil: - return nil, errors.New("ir/netmap: container contract wrapper is not set") case p.NodeValidator == nil: return nil, errors.New("ir/netmap: node validator is not set") case p.NodeStateSettings == nil: @@ -161,7 +152,6 @@ func New(p *Params) (*Processor, error) { epochState: p.EpochState, alphabetState: p.AlphabetState, netmapClient: p.NetmapClient, - containerWrp: p.ContainerWrapper, netmapSnapshot: newCleanupTable(p.CleanupEnabled, p.CleanupThreshold), handleAlphabetSync: p.AlphabetSyncHandler, diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index f735a5ff7..b512a6594 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -30,12 +30,6 @@ const ( eaclMethod = "eACL" deletionInfoMethod = "deletionInfo" - startEstimationMethod = "startContainerEstimation" - stopEstimationMethod = "stopContainerEstimation" - - listSizesMethod = "listContainerSizes" - getSizeMethod = "getContainerSize" - // putNamedMethod is method name for container put with an alias. It is exported to provide custom fee. putNamedMethod = "putNamed" ) diff --git a/pkg/morph/client/container/estimations.go b/pkg/morph/client/container/estimations.go deleted file mode 100644 index f288c63cf..000000000 --- a/pkg/morph/client/container/estimations.go +++ /dev/null @@ -1,54 +0,0 @@ -package container - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" -) - -// StartEstimationPrm groups parameters of StartEstimation operation. -type StartEstimationPrm struct { - commonEstimationPrm -} - -// StopEstimationPrm groups parameters of StopEstimation operation. -type StopEstimationPrm struct { - commonEstimationPrm -} - -type commonEstimationPrm struct { - epoch uint64 - - client.InvokePrmOptional -} - -// SetEpoch sets epoch. -func (p *commonEstimationPrm) SetEpoch(epoch uint64) { - p.epoch = epoch -} - -// StartEstimation votes to produce start estimation notification. -func (c *Client) StartEstimation(p StartEstimationPrm) error { - prm := client.InvokePrm{} - prm.SetMethod(startEstimationMethod) - prm.SetArgs(p.epoch) - prm.InvokePrmOptional = p.InvokePrmOptional - - if _, err := c.client.Invoke(prm); err != nil { - return fmt.Errorf("could not invoke method (%s): %w", startEstimationMethod, err) - } - return nil -} - -// StopEstimation votes to produce stop estimation notification. -func (c *Client) StopEstimation(p StopEstimationPrm) error { - prm := client.InvokePrm{} - prm.SetMethod(stopEstimationMethod) - prm.SetArgs(p.epoch) - prm.InvokePrmOptional = p.InvokePrmOptional - - if _, err := c.client.Invoke(prm); err != nil { - return fmt.Errorf("could not invoke method (%s): %w", stopEstimationMethod, err) - } - return nil -} diff --git a/pkg/morph/client/container/load.go b/pkg/morph/client/container/load.go deleted file mode 100644 index 5e2c3c2c3..000000000 --- a/pkg/morph/client/container/load.go +++ /dev/null @@ -1,131 +0,0 @@ -package container - -import ( - "fmt" - - v2refs "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" -) - -// EstimationID is an identity of container load estimation inside Container contract. -type EstimationID []byte - -// ListLoadEstimationsByEpoch returns a list of container load estimations for to the specified epoch. -// The list is composed through Container contract call. -func (c *Client) ListLoadEstimationsByEpoch(epoch uint64) ([]EstimationID, error) { - invokePrm := client.TestInvokePrm{} - invokePrm.SetMethod(listSizesMethod) - invokePrm.SetArgs(epoch) - - prms, err := c.client.TestInvoke(invokePrm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", listSizesMethod, err) - } else if ln := len(prms); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", listSizesMethod, ln) - } - - prms, err = client.ArrayFromStackItem(prms[0]) - if err != nil { - return nil, fmt.Errorf("could not get stack item array from stack item (%s): %w", listSizesMethod, err) - } - - res := make([]EstimationID, 0, len(prms)) - for i := range prms { - id, err := client.BytesFromStackItem(prms[i]) - if err != nil { - return nil, fmt.Errorf("could not get ID byte array from stack item (%s): %w", listSizesMethod, err) - } - - res = append(res, id) - } - - return res, nil -} - -// Estimation is a structure of single container load estimation -// reported by storage node. -type Estimation struct { - Size uint64 - - Reporter []byte -} - -// Estimations is a structure of grouped container load estimation inside Container contract. -type Estimations struct { - ContainerID cid.ID - - Values []Estimation -} - -// GetUsedSpaceEstimations returns a list of container load estimations by ID. -// The list is composed through Container contract call. -func (c *Client) GetUsedSpaceEstimations(id EstimationID) (*Estimations, error) { - prm := client.TestInvokePrm{} - prm.SetMethod(getSizeMethod) - prm.SetArgs([]byte(id)) - - prms, err := c.client.TestInvoke(prm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", getSizeMethod, err) - } else if ln := len(prms); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", getSizeMethod, ln) - } - - prms, err = client.ArrayFromStackItem(prms[0]) - if err != nil { - return nil, fmt.Errorf("could not get stack items of estimation fields from stack item (%s): %w", getSizeMethod, err) - } else if ln := len(prms); ln != 2 { - return nil, fmt.Errorf("unexpected stack item count of estimations fields (%s)", getSizeMethod) - } - - rawCnr, err := client.BytesFromStackItem(prms[0]) - if err != nil { - return nil, fmt.Errorf("could not get container ID byte array from stack item (%s): %w", getSizeMethod, err) - } - - prms, err = client.ArrayFromStackItem(prms[1]) - if err != nil { - return nil, fmt.Errorf("could not get estimation list array from stack item (%s): %w", getSizeMethod, err) - } - - var cnr cid.ID - - err = cnr.Decode(rawCnr) - if err != nil { - return nil, fmt.Errorf("decode container ID: %w", err) - } - - v2 := new(v2refs.ContainerID) - v2.SetValue(rawCnr) - res := &Estimations{ - ContainerID: cnr, - Values: make([]Estimation, 0, len(prms)), - } - - for i := range prms { - arr, err := client.ArrayFromStackItem(prms[i]) - if err != nil { - return nil, fmt.Errorf("could not get estimation struct from stack item (%s): %w", getSizeMethod, err) - } else if ln := len(arr); ln != 2 { - return nil, fmt.Errorf("unexpected stack item count of estimation fields (%s)", getSizeMethod) - } - - reporter, err := client.BytesFromStackItem(arr[0]) - if err != nil { - return nil, fmt.Errorf("could not get reporter byte array from stack item (%s): %w", getSizeMethod, err) - } - - sz, err := client.IntFromStackItem(arr[1]) - if err != nil { - return nil, fmt.Errorf("could not get estimation size from stack item (%s): %w", getSizeMethod, err) - } - - res.Values = append(res.Values, Estimation{ - Reporter: reporter, - Size: uint64(sz), - }) - } - - return res, nil -} diff --git a/pkg/morph/event/container/estimates.go b/pkg/morph/event/container/estimates.go deleted file mode 100644 index 9fd21e2b5..000000000 --- a/pkg/morph/event/container/estimates.go +++ /dev/null @@ -1,78 +0,0 @@ -package container - -import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" -) - -// StartEstimation structure of container.StartEstimation notification from -// morph chain. -type StartEstimation struct { - epoch uint64 -} - -// StopEstimation structure of container.StopEstimation notification from -// morph chain. -type StopEstimation struct { - epoch uint64 -} - -// MorphEvent implements Neo:Morph Event interface. -func (StartEstimation) MorphEvent() {} - -// MorphEvent implements Neo:Morph Event interface. -func (StopEstimation) MorphEvent() {} - -// Epoch returns epoch value for which to start container size estimation. -func (s StartEstimation) Epoch() uint64 { return s.epoch } - -// Epoch returns epoch value for which to stop container size estimation. -func (s StopEstimation) Epoch() uint64 { return s.epoch } - -// ParseStartEstimation from notification into container event structure. -func ParseStartEstimation(e *state.ContainedNotificationEvent) (event.Event, error) { - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - epoch, err := parseEstimation(params) - if err != nil { - return nil, err - } - - return StartEstimation{epoch: epoch}, nil -} - -// ParseStopEstimation from notification into container event structure. -func ParseStopEstimation(e *state.ContainedNotificationEvent) (event.Event, error) { - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - epoch, err := parseEstimation(params) - if err != nil { - return nil, err - } - - return StopEstimation{epoch: epoch}, nil -} - -func parseEstimation(params []stackitem.Item) (uint64, error) { - if ln := len(params); ln != 1 { - return 0, event.WrongNumberOfParameters(1, ln) - } - - // parse container - epoch, err := client.IntFromStackItem(params[0]) - if err != nil { - return 0, fmt.Errorf("could not get estimation epoch: %w", err) - } - - return uint64(epoch), nil -} diff --git a/pkg/morph/event/container/estimates_test.go b/pkg/morph/event/container/estimates_test.go deleted file mode 100644 index be46e62c4..000000000 --- a/pkg/morph/event/container/estimates_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package container - -import ( - "math/big" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" -) - -func TestStartEstimation(t *testing.T) { - var epochNum uint64 = 100 - epochItem := stackitem.NewBigInteger(new(big.Int).SetUint64(epochNum)) - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParseStartEstimation(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(1, len(prms)).Error()) - }) - - t.Run("wrong estimation parameter", func(t *testing.T) { - _, err := ParseStartEstimation(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParseStartEstimation(createNotifyEventFromItems([]stackitem.Item{ - epochItem, - })) - - require.NoError(t, err) - - require.Equal(t, StartEstimation{ - epochNum, - }, ev) - }) -} - -func TestStopEstimation(t *testing.T) { - var epochNum uint64 = 100 - epochItem := stackitem.NewBigInteger(new(big.Int).SetUint64(epochNum)) - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParseStopEstimation(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(1, len(prms)).Error()) - }) - - t.Run("wrong estimation parameter", func(t *testing.T) { - _, err := ParseStopEstimation(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewMap(), - })) - - require.Error(t, err) - }) - - t.Run("correct behavior", func(t *testing.T) { - ev, err := ParseStopEstimation(createNotifyEventFromItems([]stackitem.Item{ - epochItem, - })) - - require.NoError(t, err) - - require.Equal(t, StopEstimation{ - epochNum, - }, ev) - }) -} From ef64930feff929fb613e2e90ce16c9fdc7f232ce Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 7 Nov 2024 16:00:20 +0300 Subject: [PATCH 1501/1943] [#1477] ape: Fix EC chunk test Initially, this test was a check that only the container node can assemble an EC object. But the implementation of this test was wrong. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/ape/checker_test.go | 56 +++++++++++++++---------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index 7ebd147f3..66f0822e4 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -652,7 +652,7 @@ func (s *testContainerSource) DeletionInfo(cid.ID) (*container.DelInfo, error) { return nil, nil } -func TestPutECChunk(t *testing.T) { +func TestGetECChunk(t *testing.T) { headerProvider := newHeaderProviderMock() frostfsidProvider := newFrostfsIDProviderMock(t) @@ -666,11 +666,10 @@ func TestPutECChunk(t *testing.T) { Rules: []chain.Rule{ { Status: chain.AccessDenied, - Actions: chain.Actions{Names: methodsOptionalOID}, + Actions: chain.Actions{Names: methodsRequiredOID}, Resources: chain.Resources{ Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, }, - Any: true, Condition: []chain.Condition{ { Op: chain.CondStringEquals, @@ -680,17 +679,27 @@ func TestPutECChunk(t *testing.T) { }, }, }, + { + Status: chain.Allow, + Actions: chain.Actions{Names: methodsRequiredOID}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + }, }, - MatchType: chain.MatchTypeFirstMatch, }) node1Key, err := keys.NewPrivateKey() require.NoError(t, err) node1 := netmapSDK.NodeInfo{} node1.SetPublicKey(node1Key.PublicKey().Bytes()) + node2Key, err := keys.NewPrivateKey() + require.NoError(t, err) + node2 := netmapSDK.NodeInfo{} + node2.SetPublicKey(node1Key.PublicKey().Bytes()) netmap := &netmapSDK.NetMap{} netmap.SetEpoch(100) - netmap.SetNodes([]netmapSDK.NodeInfo{node1}) + netmap.SetNodes([]netmapSDK.NodeInfo{node1, node2}) nm := &netmapStub{ currentEpoch: 100, @@ -703,7 +712,7 @@ func TestPutECChunk(t *testing.T) { cont := containerSDK.Container{} cont.Init() pp := netmapSDK.PlacementPolicy{} - require.NoError(t, pp.DecodeString("REP 1")) + require.NoError(t, pp.DecodeString("EC 1.1")) cont.SetPlacementPolicy(pp) cs := &testContainerSource{ containers: map[cid.ID]*container.Container{ @@ -719,7 +728,7 @@ func TestPutECChunk(t *testing.T) { chunkHeader := newHeaderObjectSDK(cnr, obj, nil).ToV2().GetHeader() ecHeader := object.ECHeader{ Index: 1, - Total: 5, + Total: 2, Parent: &refs.ObjectID{}, } chunkHeader.SetEC(&ecHeader) @@ -738,32 +747,33 @@ func TestPutECChunk(t *testing.T) { }) headerProvider.addHeader(cnr, ecParentID, parentHeader) - t.Run("access denied for container node", func(t *testing.T) { + // container node requests EC parent headers, so container node denies access by matching attribute key/value + t.Run("access denied on container node", func(t *testing.T) { prm := Prm{ - Method: nativeschema.MethodPutObject, - Container: cnr, - Object: obj, - Role: role, - SenderKey: senderKey, - Header: chunkHeader, - SoftAPECheck: true, + Method: nativeschema.MethodGetObject, + Container: cnr, + Object: obj, + Role: role, + SenderKey: hex.EncodeToString(node2Key.PublicKey().Bytes()), + Header: chunkHeader, } err = checker.CheckAPE(context.Background(), prm) require.Error(t, err) }) - t.Run("access allowed for non container node", func(t *testing.T) { + + // non container node has no access rights to collect EC parent header, so it uses EC chunk headers + t.Run("access allowed on non container node", func(t *testing.T) { otherKey, err := keys.NewPrivateKey() require.NoError(t, err) checker = NewChecker(ls, ms, headerProvider, frostfsidProvider, nm, &stMock{}, cs, otherKey.PublicKey().Bytes()) prm := Prm{ - Method: nativeschema.MethodPutObject, - Container: cnr, - Object: obj, - Role: nativeschema.PropertyValueContainerRoleOthers, - SenderKey: senderKey, - Header: chunkHeader, - SoftAPECheck: true, + Method: nativeschema.MethodGetObject, + Container: cnr, + Object: obj, + Role: nativeschema.PropertyValueContainerRoleOthers, + SenderKey: senderKey, + Header: chunkHeader, } err = checker.CheckAPE(context.Background(), prm) From 9b13a18aacf6591a6d165ac9b0d070a7940ba45f Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 7 Nov 2024 17:32:10 +0300 Subject: [PATCH 1502/1943] [#1479] go.mod: Bump frostfs-sdk-go version * Update version within go.mod; * Fix deprecated frostfs-api-go/v2 package and use frostfs-sdk-go/api instead. Signed-off-by: Airat Arifullin --- Makefile | 4 ++-- cmd/frostfs-cli/modules/container/create.go | 2 +- cmd/frostfs-cli/modules/control/add_rule.go | 2 +- cmd/frostfs-cli/modules/control/detach_shards.go | 2 +- cmd/frostfs-cli/modules/control/doctor.go | 2 +- cmd/frostfs-cli/modules/control/drop_objects.go | 2 +- cmd/frostfs-cli/modules/control/evacuate_shard.go | 2 +- cmd/frostfs-cli/modules/control/evacuation.go | 2 +- cmd/frostfs-cli/modules/control/flush_cache.go | 2 +- cmd/frostfs-cli/modules/control/get_rule.go | 2 +- cmd/frostfs-cli/modules/control/healthcheck.go | 2 +- cmd/frostfs-cli/modules/control/ir_healthcheck.go | 2 +- .../modules/control/ir_remove_container.go | 4 ++-- cmd/frostfs-cli/modules/control/ir_remove_node.go | 2 +- cmd/frostfs-cli/modules/control/ir_tick_epoch.go | 2 +- cmd/frostfs-cli/modules/control/list_rules.go | 2 +- cmd/frostfs-cli/modules/control/list_targets.go | 2 +- cmd/frostfs-cli/modules/control/rebuild_shards.go | 2 +- cmd/frostfs-cli/modules/control/remove_rule.go | 2 +- cmd/frostfs-cli/modules/control/set_netmap_status.go | 2 +- cmd/frostfs-cli/modules/control/shards_list.go | 2 +- cmd/frostfs-cli/modules/control/shards_set_mode.go | 2 +- cmd/frostfs-cli/modules/control/synchronize_tree.go | 2 +- cmd/frostfs-cli/modules/control/util.go | 2 +- cmd/frostfs-cli/modules/control/writecache.go | 2 +- cmd/frostfs-cli/modules/object/head.go | 2 +- cmd/frostfs-cli/modules/object/lock.go | 2 +- cmd/frostfs-cli/modules/object/put.go | 2 +- cmd/frostfs-node/accounting.go | 2 +- cmd/frostfs-node/apemanager.go | 2 +- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/container.go | 2 +- cmd/frostfs-node/netmap.go | 2 +- cmd/frostfs-node/object.go | 4 ++-- cmd/frostfs-node/session.go | 4 ++-- go.mod | 3 +-- go.sum | 6 ++---- internal/ape/converter.go | 2 +- internal/audit/request.go | 2 +- internal/audit/target.go | 2 +- pkg/core/client/client.go | 2 +- pkg/core/container/delete.go | 2 +- pkg/core/object/fmt.go | 4 ++-- pkg/core/object/fmt_test.go | 2 +- pkg/innerring/processors/netmap/handlers_test.go | 2 +- pkg/innerring/processors/netmap/process_cleanup.go | 2 +- pkg/local_object_storage/engine/lock_test.go | 2 +- pkg/local_object_storage/metabase/db.go | 2 +- pkg/local_object_storage/metabase/db_test.go | 2 +- pkg/local_object_storage/metabase/iterators_test.go | 2 +- pkg/local_object_storage/metabase/put.go | 2 +- pkg/local_object_storage/metabase/select.go | 2 +- pkg/local_object_storage/metabase/select_test.go | 2 +- pkg/local_object_storage/metabase/upgrade.go | 2 +- pkg/local_object_storage/metabase/upgrade_test.go | 2 +- pkg/local_object_storage/shard/gc_test.go | 2 +- pkg/morph/client/container/eacl.go | 2 +- pkg/morph/client/container/get.go | 2 +- pkg/morph/client/container/put.go | 2 +- pkg/network/address.go | 2 +- pkg/network/cache/multi.go | 2 +- pkg/network/transport/accounting/grpc/service.go | 4 ++-- pkg/network/transport/apemanager/grpc/service.go | 4 ++-- pkg/network/transport/container/grpc/service.go | 4 ++-- pkg/network/transport/netmap/grpc/service.go | 4 ++-- pkg/network/transport/object/grpc/get.go | 4 ++-- pkg/network/transport/object/grpc/range.go | 4 ++-- pkg/network/transport/object/grpc/search.go | 4 ++-- pkg/network/transport/object/grpc/service.go | 4 ++-- pkg/network/transport/session/grpc/service.go | 4 ++-- pkg/services/accounting/executor.go | 2 +- pkg/services/accounting/morph/executor.go | 2 +- pkg/services/accounting/server.go | 2 +- pkg/services/accounting/sign.go | 2 +- pkg/services/apemanager/audit.go | 4 ++-- pkg/services/apemanager/executor.go | 8 ++++---- pkg/services/apemanager/server.go | 2 +- pkg/services/apemanager/sign.go | 2 +- pkg/services/container/ape.go | 6 +++--- pkg/services/container/ape_test.go | 8 ++++---- pkg/services/container/audit.go | 4 ++-- pkg/services/container/executor.go | 4 ++-- pkg/services/container/morph/executor.go | 6 +++--- pkg/services/container/morph/executor_test.go | 6 +++--- pkg/services/container/server.go | 2 +- pkg/services/container/sign.go | 2 +- pkg/services/control/convert.go | 4 ++-- pkg/services/control/ir/convert.go | 4 ++-- pkg/services/control/ir/rpc.go | 6 +++--- pkg/services/control/ir/server/audit.go | 2 +- pkg/services/control/ir/server/calls.go | 2 +- pkg/services/control/ir/server/sign.go | 2 +- pkg/services/control/ir/service_frostfs.pb.go | 6 +++--- pkg/services/control/ir/types_frostfs.pb.go | 6 +++--- pkg/services/control/rpc.go | 4 ++-- pkg/services/control/server/ctrlmessage/sign.go | 2 +- pkg/services/control/server/sign.go | 2 +- pkg/services/control/service_frostfs.pb.go | 6 +++--- pkg/services/control/types_frostfs.pb.go | 6 +++--- pkg/services/netmap/executor.go | 4 ++-- pkg/services/netmap/server.go | 2 +- pkg/services/netmap/sign.go | 2 +- pkg/services/object/acl/eacl/v2/eacl_test.go | 6 +++--- pkg/services/object/acl/eacl/v2/headers.go | 8 ++++---- pkg/services/object/acl/eacl/v2/object.go | 2 +- pkg/services/object/acl/eacl/v2/xheader.go | 2 +- pkg/services/object/acl/v2/request.go | 2 +- pkg/services/object/acl/v2/request_test.go | 6 +++--- pkg/services/object/acl/v2/service.go | 4 ++-- pkg/services/object/acl/v2/util.go | 6 +++--- pkg/services/object/acl/v2/util_test.go | 4 ++-- pkg/services/object/ape/checker.go | 4 ++-- pkg/services/object/ape/checker_test.go | 6 +++--- pkg/services/object/ape/request.go | 2 +- pkg/services/object/ape/request_test.go | 2 +- pkg/services/object/ape/service.go | 4 ++-- pkg/services/object/audit.go | 6 +++--- pkg/services/object/common.go | 2 +- pkg/services/object/common/writer/ec_test.go | 2 +- pkg/services/object/delete/exec.go | 2 +- pkg/services/object/delete/v2/service.go | 2 +- pkg/services/object/delete/v2/util.go | 4 ++-- pkg/services/object/get/getrangeec_test.go | 2 +- pkg/services/object/get/v2/errors.go | 4 ++-- pkg/services/object/get/v2/get_forwarder.go | 10 +++++----- pkg/services/object/get/v2/get_range_forwarder.go | 10 +++++----- pkg/services/object/get/v2/get_range_hash.go | 10 +++++----- pkg/services/object/get/v2/head_forwarder.go | 12 ++++++------ pkg/services/object/get/v2/service.go | 2 +- pkg/services/object/get/v2/streamer.go | 2 +- pkg/services/object/get/v2/util.go | 10 +++++----- pkg/services/object/internal/key.go | 2 +- pkg/services/object/metrics.go | 2 +- pkg/services/object/patch/streamer.go | 4 ++-- pkg/services/object/patch/util.go | 4 ++-- pkg/services/object/put/single.go | 10 +++++----- pkg/services/object/put/v2/service.go | 2 +- pkg/services/object/put/v2/streamer.go | 10 +++++----- pkg/services/object/put/v2/util.go | 4 ++-- pkg/services/object/response.go | 2 +- pkg/services/object/search/search_test.go | 2 +- pkg/services/object/search/v2/request_forwarder.go | 10 +++++----- pkg/services/object/search/v2/service.go | 2 +- pkg/services/object/search/v2/streamer.go | 4 ++-- pkg/services/object/search/v2/util.go | 2 +- pkg/services/object/server.go | 2 +- pkg/services/object/sign.go | 2 +- pkg/services/object/transport_splitter.go | 2 +- pkg/services/object/util/key_test.go | 4 ++-- pkg/services/object/util/prm.go | 2 +- pkg/services/object_manager/tombstone/checker.go | 2 +- pkg/services/session/executor.go | 2 +- pkg/services/session/server.go | 2 +- pkg/services/session/sign.go | 2 +- pkg/services/session/storage/persistent/executor.go | 2 +- .../session/storage/persistent/executor_test.go | 4 ++-- pkg/services/session/storage/temporary/executor.go | 2 +- pkg/services/tree/service_frostfs.pb.go | 6 +++--- pkg/services/tree/signature.go | 2 +- pkg/services/tree/signature_test.go | 2 +- pkg/services/tree/types_frostfs.pb.go | 6 +++--- pkg/services/util/response/service.go | 4 ++-- pkg/services/util/sign.go | 4 ++-- 163 files changed, 271 insertions(+), 274 deletions(-) diff --git a/Makefile b/Makefile index 94b1542d5..68a31febe 100755 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ GO_VERSION ?= 1.22 LINT_VERSION ?= 1.61.0 TRUECLOUDLAB_LINT_VERSION ?= 0.0.7 PROTOC_VERSION ?= 25.0 -PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-api-go/v2) +PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-sdk-go) PROTOC_OS_VERSION=osx-x86_64 ifeq ($(shell uname), Linux) PROTOC_OS_VERSION=linux-x86_64 @@ -121,7 +121,7 @@ protoc-install: @unzip -q -o $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip -d $(PROTOC_DIR) @rm $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip @echo "⇒ Instaling protogen FrostFS plugin..." - @GOBIN=$(PROTOGEN_FROSTFS_DIR) go install -mod=mod -v git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/protogen@$(PROTOGEN_FROSTFS_VERSION) + @GOBIN=$(PROTOGEN_FROSTFS_DIR) go install -mod=mod -v git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/protogen@$(PROTOGEN_FROSTFS_VERSION) # Build FrostFS component's docker image image-%: diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index f37b169ce..017f4b5ce 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -7,12 +7,12 @@ import ( "strings" "time" - containerApi "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + containerApi "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index a22d0525d..c648377bd 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -4,11 +4,11 @@ import ( "encoding/hex" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/detach_shards.go b/cmd/frostfs-cli/modules/control/detach_shards.go index 5e5b60c3d..025a6e561 100644 --- a/cmd/frostfs-cli/modules/control/detach_shards.go +++ b/cmd/frostfs-cli/modules/control/detach_shards.go @@ -1,10 +1,10 @@ package control import ( - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/doctor.go b/cmd/frostfs-cli/modules/control/doctor.go index 13bb81a0a..632cdd6a7 100644 --- a/cmd/frostfs-cli/modules/control/doctor.go +++ b/cmd/frostfs-cli/modules/control/doctor.go @@ -1,10 +1,10 @@ package control import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/drop_objects.go b/cmd/frostfs-cli/modules/control/drop_objects.go index 8c0bb2332..dcc1c1229 100644 --- a/cmd/frostfs-cli/modules/control/drop_objects.go +++ b/cmd/frostfs-cli/modules/control/drop_objects.go @@ -1,10 +1,10 @@ package control import ( - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/evacuate_shard.go b/cmd/frostfs-cli/modules/control/evacuate_shard.go index 458e4cc0b..1e48c1df4 100644 --- a/cmd/frostfs-cli/modules/control/evacuate_shard.go +++ b/cmd/frostfs-cli/modules/control/evacuate_shard.go @@ -1,10 +1,10 @@ package control import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index fffc5e33e..73700e56d 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -7,11 +7,11 @@ import ( "sync/atomic" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/flush_cache.go b/cmd/frostfs-cli/modules/control/flush_cache.go index 541961903..280aacfad 100644 --- a/cmd/frostfs-cli/modules/control/flush_cache.go +++ b/cmd/frostfs-cli/modules/control/flush_cache.go @@ -1,10 +1,10 @@ package control import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/get_rule.go b/cmd/frostfs-cli/modules/control/get_rule.go index 050cf165c..4b4d6eef5 100644 --- a/cmd/frostfs-cli/modules/control/get_rule.go +++ b/cmd/frostfs-cli/modules/control/get_rule.go @@ -3,11 +3,11 @@ package control import ( "encoding/hex" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/healthcheck.go b/cmd/frostfs-cli/modules/control/healthcheck.go index 2241a403f..1d4441f1e 100644 --- a/cmd/frostfs-cli/modules/control/healthcheck.go +++ b/cmd/frostfs-cli/modules/control/healthcheck.go @@ -3,11 +3,11 @@ package control import ( "os" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/ir_healthcheck.go b/cmd/frostfs-cli/modules/control/ir_healthcheck.go index 4f272c1b4..373f21c30 100644 --- a/cmd/frostfs-cli/modules/control/ir_healthcheck.go +++ b/cmd/frostfs-cli/modules/control/ir_healthcheck.go @@ -3,12 +3,12 @@ package control import ( "os" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/ir_remove_container.go b/cmd/frostfs-cli/modules/control/ir_remove_container.go index a66d7e06d..460e299e5 100644 --- a/cmd/frostfs-cli/modules/control/ir_remove_container.go +++ b/cmd/frostfs-cli/modules/control/ir_remove_container.go @@ -1,13 +1,13 @@ package control import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/spf13/cobra" diff --git a/cmd/frostfs-cli/modules/control/ir_remove_node.go b/cmd/frostfs-cli/modules/control/ir_remove_node.go index 412dc7934..2fe686d63 100644 --- a/cmd/frostfs-cli/modules/control/ir_remove_node.go +++ b/cmd/frostfs-cli/modules/control/ir_remove_node.go @@ -4,11 +4,11 @@ import ( "encoding/hex" "errors" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/ir_tick_epoch.go b/cmd/frostfs-cli/modules/control/ir_tick_epoch.go index 6965b5dca..5f09e92c1 100644 --- a/cmd/frostfs-cli/modules/control/ir_tick_epoch.go +++ b/cmd/frostfs-cli/modules/control/ir_tick_epoch.go @@ -1,11 +1,11 @@ package control import ( - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index f5fc27bda..7162df5e0 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -5,11 +5,11 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/nspcc-dev/neo-go/cli/input" diff --git a/cmd/frostfs-cli/modules/control/list_targets.go b/cmd/frostfs-cli/modules/control/list_targets.go index 6a988c355..7c401eb17 100644 --- a/cmd/frostfs-cli/modules/control/list_targets.go +++ b/cmd/frostfs-cli/modules/control/list_targets.go @@ -7,10 +7,10 @@ import ( "strconv" "text/tabwriter" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/spf13/cobra" "google.golang.org/grpc/codes" diff --git a/cmd/frostfs-cli/modules/control/rebuild_shards.go b/cmd/frostfs-cli/modules/control/rebuild_shards.go index e2b408712..3df12a15d 100644 --- a/cmd/frostfs-cli/modules/control/rebuild_shards.go +++ b/cmd/frostfs-cli/modules/control/rebuild_shards.go @@ -3,10 +3,10 @@ package control import ( "fmt" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/mr-tron/base58" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/remove_rule.go b/cmd/frostfs-cli/modules/control/remove_rule.go index 4189ea76b..a996156a5 100644 --- a/cmd/frostfs-cli/modules/control/remove_rule.go +++ b/cmd/frostfs-cli/modules/control/remove_rule.go @@ -4,10 +4,10 @@ import ( "encoding/hex" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/set_netmap_status.go b/cmd/frostfs-cli/modules/control/set_netmap_status.go index a107b2b53..87c4f3b3d 100644 --- a/cmd/frostfs-cli/modules/control/set_netmap_status.go +++ b/cmd/frostfs-cli/modules/control/set_netmap_status.go @@ -6,12 +6,12 @@ import ( "fmt" "time" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/shards_list.go b/cmd/frostfs-cli/modules/control/shards_list.go index a81034a9e..40d6628ee 100644 --- a/cmd/frostfs-cli/modules/control/shards_list.go +++ b/cmd/frostfs-cli/modules/control/shards_list.go @@ -7,11 +7,11 @@ import ( "sort" "strings" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/mr-tron/base58" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/shards_set_mode.go b/cmd/frostfs-cli/modules/control/shards_set_mode.go index dd0d77748..8fe01ba30 100644 --- a/cmd/frostfs-cli/modules/control/shards_set_mode.go +++ b/cmd/frostfs-cli/modules/control/shards_set_mode.go @@ -6,10 +6,10 @@ import ( "slices" "strings" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/mr-tron/base58" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/synchronize_tree.go b/cmd/frostfs-cli/modules/control/synchronize_tree.go index 5f2e4da96..1e4575f49 100644 --- a/cmd/frostfs-cli/modules/control/synchronize_tree.go +++ b/cmd/frostfs-cli/modules/control/synchronize_tree.go @@ -4,12 +4,12 @@ import ( "crypto/sha256" "errors" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/util.go b/cmd/frostfs-cli/modules/control/util.go index ef547681f..41d9dbf8a 100644 --- a/cmd/frostfs-cli/modules/control/util.go +++ b/cmd/frostfs-cli/modules/control/util.go @@ -4,11 +4,11 @@ import ( "crypto/ecdsa" "errors" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" "github.com/spf13/cobra" diff --git a/cmd/frostfs-cli/modules/control/writecache.go b/cmd/frostfs-cli/modules/control/writecache.go index ffe9009ab..80e4a0c87 100644 --- a/cmd/frostfs-cli/modules/control/writecache.go +++ b/cmd/frostfs-cli/modules/control/writecache.go @@ -1,10 +1,10 @@ package control import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/mr-tron/base58" "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index cf2e2d5e6..70c273443 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -6,12 +6,12 @@ import ( "fmt" "os" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/cmd/frostfs-cli/modules/object/lock.go b/cmd/frostfs-cli/modules/object/lock.go index d2e9af24c..53dd01868 100644 --- a/cmd/frostfs-cli/modules/object/lock.go +++ b/cmd/frostfs-cli/modules/object/lock.go @@ -7,12 +7,12 @@ import ( "strconv" "time" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index 45e02edb3..affe9bbba 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -10,11 +10,11 @@ import ( "strings" "time" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" diff --git a/cmd/frostfs-node/accounting.go b/cmd/frostfs-node/accounting.go index 1d065c227..72a6e5331 100644 --- a/cmd/frostfs-node/accounting.go +++ b/cmd/frostfs-node/accounting.go @@ -4,11 +4,11 @@ import ( "context" "net" - accountingGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" accountingTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/accounting/grpc" accountingService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/accounting" accounting "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/accounting/morph" + accountingGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/accounting/grpc" "google.golang.org/grpc" ) diff --git a/cmd/frostfs-node/apemanager.go b/cmd/frostfs-node/apemanager.go index 79c45c254..c4d7725f5 100644 --- a/cmd/frostfs-node/apemanager.go +++ b/cmd/frostfs-node/apemanager.go @@ -3,11 +3,11 @@ package main import ( "net" - apemanager_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager/grpc" ape_contract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/contract_storage" morph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" apemanager_transport "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/apemanager/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/apemanager" + apemanager_grpc "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/apemanager/grpc" "google.golang.org/grpc" ) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 800c49127..40af23841 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -15,7 +15,6 @@ import ( "syscall" "time" - netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/audit" @@ -70,6 +69,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state" "git.frostfs.info/TrueCloudLab/frostfs-observability/logging/lokicore" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index f95f671cd..9c3505922 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -5,7 +5,6 @@ import ( "context" "net" - containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc" morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" @@ -18,6 +17,7 @@ import ( containerTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/container/grpc" containerService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container" containerMorph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/morph" + containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container/grpc" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 6e2a7c44a..58e066fc9 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -8,7 +8,6 @@ import ( "net" "sync/atomic" - netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -19,6 +18,7 @@ import ( netmapTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/netmap/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" netmapService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/netmap" + netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap/grpc" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "go.uber.org/zap" diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index c6bde2cff..7f26393a7 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -6,8 +6,6 @@ import ( "fmt" "net" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object/grpc" metricsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/metrics" policerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/policer" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" @@ -38,6 +36,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" diff --git a/cmd/frostfs-node/session.go b/cmd/frostfs-node/session.go index ee21ec230..20d2d318f 100644 --- a/cmd/frostfs-node/session.go +++ b/cmd/frostfs-node/session.go @@ -6,8 +6,6 @@ import ( "net" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - sessionGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session/grpc" nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" @@ -16,6 +14,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage/persistent" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage/temporary" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + sessionGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "google.golang.org/grpc" ) diff --git a/go.mod b/go.mod index 886fa958f..8a70c3819 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,11 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241007120543-29c522d5d8a3 git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.0 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241010110344-99c5c5836509 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241107121119-cb813e27a823 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 diff --git a/go.sum b/go.sum index 6ed130cdb..9778f91e2 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241007120543-29c522d5d8a3 h1:6QXNnfBgYx81UZsBdpPnQY+ZMSKGFbFc29wV7DJ/UG4= -git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20241007120543-29c522d5d8a3/go.mod h1:F5GS7hRb62PUy5sTYDC4ajVdeffoAfjHSSHTKUJEaYU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.0 h1:8Z5iPhieCrbcdhxBuY/Bajh6V5fki7Whh0b4S2zYJYU= git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.0/go.mod h1:Y2Xorxc8SBO4phoek7n3XxaPZz5rIrFgDsU4TOjmlGA= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= @@ -10,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241010110344-99c5c5836509 h1:5gtEq4bjVgAbTOrbEquspyM3s+qsMtkpGC5m9FtfImk= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241010110344-99c5c5836509/go.mod h1:jmb7yxzZota9jWbC10p+7YR+6wwJPBj7J/Fl5VDkXys= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241107121119-cb813e27a823 h1:sepm9FeuoInmygH1K/+3L+Yp5bJhGiVi/oGCH6Emp2c= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241107121119-cb813e27a823/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= diff --git a/internal/ape/converter.go b/internal/ape/converter.go index eb80e7ded..c706cf052 100644 --- a/internal/ape/converter.go +++ b/internal/ape/converter.go @@ -4,7 +4,7 @@ import ( "encoding/hex" "fmt" - v2acl "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" + v2acl "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/acl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" diff --git a/internal/audit/request.go b/internal/audit/request.go index cf0797300..3355087f1 100644 --- a/internal/audit/request.go +++ b/internal/audit/request.go @@ -1,10 +1,10 @@ package audit import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" crypto "git.frostfs.info/TrueCloudLab/frostfs-crypto" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.uber.org/zap" ) diff --git a/internal/audit/target.go b/internal/audit/target.go index 8bc87ee8e..2d6881e29 100644 --- a/internal/audit/target.go +++ b/internal/audit/target.go @@ -3,7 +3,7 @@ package audit import ( "strings" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) diff --git a/pkg/core/client/client.go b/pkg/core/client/client.go index 854fbc49f..98bdf99e7 100644 --- a/pkg/core/client/client.go +++ b/pkg/core/client/client.go @@ -3,8 +3,8 @@ package client import ( "context" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" ) diff --git a/pkg/core/container/delete.go b/pkg/core/container/delete.go index 8e0aaebb9..8c14bdf5e 100644 --- a/pkg/core/container/delete.go +++ b/pkg/core/container/delete.go @@ -1,7 +1,7 @@ package container import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" ) diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index 317d62cb0..5bc5c8bea 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -8,11 +8,11 @@ import ( "fmt" "strconv" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index 77afbfc45..b428b56da 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -7,9 +7,9 @@ import ( "strconv" "testing" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index 35f4469b1..a53458179 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - v2netmap "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" netmapContract "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" timerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" @@ -13,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + v2netmap "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" diff --git a/pkg/innerring/processors/netmap/process_cleanup.go b/pkg/innerring/processors/netmap/process_cleanup.go index 170c39e2c..269e79c5e 100644 --- a/pkg/innerring/processors/netmap/process_cleanup.go +++ b/pkg/innerring/processors/netmap/process_cleanup.go @@ -1,8 +1,8 @@ package netmap import ( - v2netmap "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + v2netmap "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.uber.org/zap" ) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 3702f567f..7e15c76f5 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -6,12 +6,12 @@ import ( "testing" "time" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index 1f444a3ef..2cd990814 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -11,9 +11,9 @@ import ( "sync" "time" - v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + v2object "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/mr-tron/base58" "go.etcd.io/bbolt" diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index 0abb5ea89..c61d762bc 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -6,10 +6,10 @@ import ( "strconv" "testing" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 777a94a6f..646dc196c 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -5,10 +5,10 @@ import ( "strconv" "testing" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" object2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index b329e8032..09c5e04ad 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -9,12 +9,12 @@ import ( "strconv" "time" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 41f05b756..f802036be 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -8,9 +8,9 @@ import ( "strings" "time" - v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + v2object "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 5cc25a9f6..6f48607be 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -7,10 +7,10 @@ import ( "strconv" "testing" - v2object "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + v2object "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/metabase/upgrade.go b/pkg/local_object_storage/metabase/upgrade.go index 1f2c7956b..bcf72f440 100644 --- a/pkg/local_object_storage/metabase/upgrade.go +++ b/pkg/local_object_storage/metabase/upgrade.go @@ -12,8 +12,8 @@ import ( "sync/atomic" "time" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" diff --git a/pkg/local_object_storage/metabase/upgrade_test.go b/pkg/local_object_storage/metabase/upgrade_test.go index 9c525291a..aeb14aeb6 100644 --- a/pkg/local_object_storage/metabase/upgrade_test.go +++ b/pkg/local_object_storage/metabase/upgrade_test.go @@ -11,12 +11,12 @@ import ( "testing" "time" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 90958cd35..2b97111e7 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -5,13 +5,13 @@ import ( "errors" "testing" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" diff --git a/pkg/morph/client/container/eacl.go b/pkg/morph/client/container/eacl.go index 8e9455050..9e604e091 100644 --- a/pkg/morph/client/container/eacl.go +++ b/pkg/morph/client/container/eacl.go @@ -4,9 +4,9 @@ import ( "crypto/sha256" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" diff --git a/pkg/morph/client/container/get.go b/pkg/morph/client/container/get.go index 6715f870f..ea57a3a95 100644 --- a/pkg/morph/client/container/get.go +++ b/pkg/morph/client/container/get.go @@ -5,10 +5,10 @@ import ( "fmt" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" containerContract "git.frostfs.info/TrueCloudLab/frostfs-contract/container" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" diff --git a/pkg/morph/client/container/put.go b/pkg/morph/client/container/put.go index ee323af00..777ae2d4e 100644 --- a/pkg/morph/client/container/put.go +++ b/pkg/morph/client/container/put.go @@ -3,9 +3,9 @@ package container import ( "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" ) diff --git a/pkg/network/address.go b/pkg/network/address.go index 88f4a571d..cb83a813d 100644 --- a/pkg/network/address.go +++ b/pkg/network/address.go @@ -7,7 +7,7 @@ import ( "net/url" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 2ecce3a01..481d1ea4a 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -7,11 +7,11 @@ import ( "sync" "time" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "google.golang.org/grpc" diff --git a/pkg/network/transport/accounting/grpc/service.go b/pkg/network/transport/accounting/grpc/service.go index 2144a3001..78129bfbe 100644 --- a/pkg/network/transport/accounting/grpc/service.go +++ b/pkg/network/transport/accounting/grpc/service.go @@ -3,9 +3,9 @@ package accounting import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" - accountingGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting/grpc" accountingsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/accounting" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/accounting" + accountingGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/accounting/grpc" ) // Server wraps FrostFS API Accounting service and diff --git a/pkg/network/transport/apemanager/grpc/service.go b/pkg/network/transport/apemanager/grpc/service.go index 59783cfc0..850d38a65 100644 --- a/pkg/network/transport/apemanager/grpc/service.go +++ b/pkg/network/transport/apemanager/grpc/service.go @@ -3,9 +3,9 @@ package apemanager import ( "context" - apemanager_v2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" - apemanager_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager/grpc" apemanager_svc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/apemanager" + apemanager_v2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/apemanager" + apemanager_grpc "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/apemanager/grpc" ) type Server struct { diff --git a/pkg/network/transport/container/grpc/service.go b/pkg/network/transport/container/grpc/service.go index 9fae22b45..49d083a90 100644 --- a/pkg/network/transport/container/grpc/service.go +++ b/pkg/network/transport/container/grpc/service.go @@ -3,9 +3,9 @@ package container import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" - containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc" containersvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" + containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container/grpc" ) // Server wraps FrostFS API Container service and diff --git a/pkg/network/transport/netmap/grpc/service.go b/pkg/network/transport/netmap/grpc/service.go index 406c77e58..4bc3a42f8 100644 --- a/pkg/network/transport/netmap/grpc/service.go +++ b/pkg/network/transport/netmap/grpc/service.go @@ -3,9 +3,9 @@ package grpc import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" - netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap/grpc" netmapsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap" + netmapGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap/grpc" ) // Server wraps FrostFS API Netmap service and diff --git a/pkg/network/transport/object/grpc/get.go b/pkg/network/transport/object/grpc/get.go index e1655c183..655b1f9fb 100644 --- a/pkg/network/transport/object/grpc/get.go +++ b/pkg/network/transport/object/grpc/get.go @@ -1,8 +1,8 @@ package object import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc" ) type getStreamerV2 struct { diff --git a/pkg/network/transport/object/grpc/range.go b/pkg/network/transport/object/grpc/range.go index 391536e8e..7d7ce0e4c 100644 --- a/pkg/network/transport/object/grpc/range.go +++ b/pkg/network/transport/object/grpc/range.go @@ -1,8 +1,8 @@ package object import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc" ) type getRangeStreamerV2 struct { diff --git a/pkg/network/transport/object/grpc/search.go b/pkg/network/transport/object/grpc/search.go index a151ced09..8432707f7 100644 --- a/pkg/network/transport/object/grpc/search.go +++ b/pkg/network/transport/object/grpc/search.go @@ -1,8 +1,8 @@ package object import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc" ) type searchStreamerV2 struct { diff --git a/pkg/network/transport/object/grpc/service.go b/pkg/network/transport/object/grpc/service.go index d55e3d87f..fa6252118 100644 --- a/pkg/network/transport/object/grpc/service.go +++ b/pkg/network/transport/object/grpc/service.go @@ -5,10 +5,10 @@ import ( "errors" "io" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object/grpc" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc" ) // Server wraps FrostFS API Object service and diff --git a/pkg/network/transport/session/grpc/service.go b/pkg/network/transport/session/grpc/service.go index e0dc74942..6fce397f3 100644 --- a/pkg/network/transport/session/grpc/service.go +++ b/pkg/network/transport/session/grpc/service.go @@ -3,9 +3,9 @@ package session import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - sessionGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session/grpc" sessionsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + sessionGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session/grpc" ) // Server wraps FrostFS API Session service and diff --git a/pkg/services/accounting/executor.go b/pkg/services/accounting/executor.go index b0722cf8a..93e44c52b 100644 --- a/pkg/services/accounting/executor.go +++ b/pkg/services/accounting/executor.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/accounting" ) type ServiceExecutor interface { diff --git a/pkg/services/accounting/morph/executor.go b/pkg/services/accounting/morph/executor.go index ac836b71d..b77d3e3e6 100644 --- a/pkg/services/accounting/morph/executor.go +++ b/pkg/services/accounting/morph/executor.go @@ -5,9 +5,9 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" accountingSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/accounting" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/accounting" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) diff --git a/pkg/services/accounting/server.go b/pkg/services/accounting/server.go index 72833c46c..a280416fb 100644 --- a/pkg/services/accounting/server.go +++ b/pkg/services/accounting/server.go @@ -3,7 +3,7 @@ package accounting import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/accounting" ) // Server is an interface of the FrostFS API Accounting service server. diff --git a/pkg/services/accounting/sign.go b/pkg/services/accounting/sign.go index cd6ff0307..d8feb76bd 100644 --- a/pkg/services/accounting/sign.go +++ b/pkg/services/accounting/sign.go @@ -4,8 +4,8 @@ import ( "context" "crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/accounting" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/accounting" ) type signService struct { diff --git a/pkg/services/apemanager/audit.go b/pkg/services/apemanager/audit.go index d132ae7db..b9bea07fb 100644 --- a/pkg/services/apemanager/audit.go +++ b/pkg/services/apemanager/audit.go @@ -4,10 +4,10 @@ import ( "context" "sync/atomic" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" - ape_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/apemanager" + ape_grpc "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/apemanager/grpc" ) var _ Server = (*auditService)(nil) diff --git a/pkg/services/apemanager/executor.go b/pkg/services/apemanager/executor.go index 25f43486a..86f9cb893 100644 --- a/pkg/services/apemanager/executor.go +++ b/pkg/services/apemanager/executor.go @@ -8,14 +8,14 @@ import ( "errors" "fmt" - apeV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/ape" - apemanagerV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" ape_contract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/contract_storage" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" apemanager_errors "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/apemanager/errors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + apeV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/ape" + apemanagerV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/apemanager" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + session "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" diff --git a/pkg/services/apemanager/server.go b/pkg/services/apemanager/server.go index 90b2d92ae..e624177ac 100644 --- a/pkg/services/apemanager/server.go +++ b/pkg/services/apemanager/server.go @@ -3,7 +3,7 @@ package apemanager import ( "context" - apemanager_v2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" + apemanager_v2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/apemanager" ) type Server interface { diff --git a/pkg/services/apemanager/sign.go b/pkg/services/apemanager/sign.go index eda2a7342..a172624ff 100644 --- a/pkg/services/apemanager/sign.go +++ b/pkg/services/apemanager/sign.go @@ -4,8 +4,8 @@ import ( "context" "crypto/ecdsa" - apemanager_v2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/apemanager" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + apemanager_v2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/apemanager" ) type signService struct { diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index dd4878331..2cdb30b45 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -12,14 +12,14 @@ import ( "net" "strings" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + session "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index d6f9b75ef..b6b42a559 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -9,13 +9,13 @@ import ( "net" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - session "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + session "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/signature" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" diff --git a/pkg/services/container/audit.go b/pkg/services/container/audit.go index b257272f5..03d3dc13d 100644 --- a/pkg/services/container/audit.go +++ b/pkg/services/container/audit.go @@ -4,10 +4,10 @@ import ( "context" "sync/atomic" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" - container_grpc "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" + container_grpc "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container/grpc" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) diff --git a/pkg/services/container/executor.go b/pkg/services/container/executor.go index 0917e3bd0..70234d3de 100644 --- a/pkg/services/container/executor.go +++ b/pkg/services/container/executor.go @@ -4,9 +4,9 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" ) type ServiceExecutor interface { diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index 05d8749cf..adb808af3 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -5,11 +5,11 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" containerSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" diff --git a/pkg/services/container/morph/executor_test.go b/pkg/services/container/morph/executor_test.go index c64310eb3..87d307385 100644 --- a/pkg/services/container/morph/executor_test.go +++ b/pkg/services/container/morph/executor_test.go @@ -4,12 +4,12 @@ import ( "context" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" containerSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container" containerSvcMorph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/morph" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" containertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/test" diff --git a/pkg/services/container/server.go b/pkg/services/container/server.go index a19d83c56..78fd3d34c 100644 --- a/pkg/services/container/server.go +++ b/pkg/services/container/server.go @@ -3,7 +3,7 @@ package container import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" ) // Server is an interface of the FrostFS API Container service server. diff --git a/pkg/services/container/sign.go b/pkg/services/container/sign.go index f7f5d6486..c478c0e1c 100644 --- a/pkg/services/container/sign.go +++ b/pkg/services/container/sign.go @@ -4,8 +4,8 @@ import ( "context" "crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" ) type signService struct { diff --git a/pkg/services/control/convert.go b/pkg/services/control/convert.go index fd6f020d1..37daf67be 100644 --- a/pkg/services/control/convert.go +++ b/pkg/services/control/convert.go @@ -1,8 +1,8 @@ package control import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/message" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/message" ) type requestWrapper struct { diff --git a/pkg/services/control/ir/convert.go b/pkg/services/control/ir/convert.go index c892c5b6c..024676b87 100644 --- a/pkg/services/control/ir/convert.go +++ b/pkg/services/control/ir/convert.go @@ -1,8 +1,8 @@ package control import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/message" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/message" ) type requestWrapper struct { diff --git a/pkg/services/control/ir/rpc.go b/pkg/services/control/ir/rpc.go index 0c9400f6c..62f800d99 100644 --- a/pkg/services/control/ir/rpc.go +++ b/pkg/services/control/ir/rpc.go @@ -1,9 +1,9 @@ package control import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/common" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/grpc" ) const serviceName = "ircontrol.ControlService" diff --git a/pkg/services/control/ir/server/audit.go b/pkg/services/control/ir/server/audit.go index 9f7a8b879..e54fa9824 100644 --- a/pkg/services/control/ir/server/audit.go +++ b/pkg/services/control/ir/server/audit.go @@ -6,10 +6,10 @@ import ( "strings" "sync/atomic" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 642932c91..63be22411 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -5,10 +5,10 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "google.golang.org/grpc/codes" diff --git a/pkg/services/control/ir/server/sign.go b/pkg/services/control/ir/server/sign.go index f72d51f9e..d39f6d5f9 100644 --- a/pkg/services/control/ir/server/sign.go +++ b/pkg/services/control/ir/server/sign.go @@ -6,8 +6,8 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" ) diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index 66d196617..ff4ce7245 100644 --- a/pkg/services/control/ir/service_frostfs.pb.go +++ b/pkg/services/control/ir/service_frostfs.pb.go @@ -5,9 +5,9 @@ package control import ( json "encoding/json" fmt "fmt" - pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" - proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" - encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + pool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto/encoding" easyproto "github.com/VictoriaMetrics/easyproto" jlexer "github.com/mailru/easyjson/jlexer" jwriter "github.com/mailru/easyjson/jwriter" diff --git a/pkg/services/control/ir/types_frostfs.pb.go b/pkg/services/control/ir/types_frostfs.pb.go index b230726a9..32bf457a1 100644 --- a/pkg/services/control/ir/types_frostfs.pb.go +++ b/pkg/services/control/ir/types_frostfs.pb.go @@ -5,9 +5,9 @@ package control import ( json "encoding/json" fmt "fmt" - pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" - proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" - encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + pool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto/encoding" easyproto "github.com/VictoriaMetrics/easyproto" jlexer "github.com/mailru/easyjson/jlexer" jwriter "github.com/mailru/easyjson/jwriter" diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 04524a68c..514061db4 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -1,8 +1,8 @@ package control import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/common" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/common" ) const serviceName = "control.ControlService" diff --git a/pkg/services/control/server/ctrlmessage/sign.go b/pkg/services/control/server/ctrlmessage/sign.go index 31425b337..d9d5c5f5e 100644 --- a/pkg/services/control/server/ctrlmessage/sign.go +++ b/pkg/services/control/server/ctrlmessage/sign.go @@ -4,8 +4,8 @@ import ( "crypto/ecdsa" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" ) diff --git a/pkg/services/control/server/sign.go b/pkg/services/control/server/sign.go index 514af273f..0e8e24b6e 100644 --- a/pkg/services/control/server/sign.go +++ b/pkg/services/control/server/sign.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" ) diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index e16f082b1..96b896478 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -5,9 +5,9 @@ package control import ( json "encoding/json" fmt "fmt" - pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" - proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" - encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + pool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto/encoding" easyproto "github.com/VictoriaMetrics/easyproto" jlexer "github.com/mailru/easyjson/jlexer" jwriter "github.com/mailru/easyjson/jwriter" diff --git a/pkg/services/control/types_frostfs.pb.go b/pkg/services/control/types_frostfs.pb.go index f92106589..50984f15a 100644 --- a/pkg/services/control/types_frostfs.pb.go +++ b/pkg/services/control/types_frostfs.pb.go @@ -5,9 +5,9 @@ package control import ( json "encoding/json" fmt "fmt" - pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" - proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" - encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + pool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto/encoding" easyproto "github.com/VictoriaMetrics/easyproto" jlexer "github.com/mailru/easyjson/jlexer" jwriter "github.com/mailru/easyjson/jwriter" diff --git a/pkg/services/netmap/executor.go b/pkg/services/netmap/executor.go index f48357915..5223047df 100644 --- a/pkg/services/netmap/executor.go +++ b/pkg/services/netmap/executor.go @@ -5,10 +5,10 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/version" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" versionsdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" ) diff --git a/pkg/services/netmap/server.go b/pkg/services/netmap/server.go index 0a09c9f44..eff880dbe 100644 --- a/pkg/services/netmap/server.go +++ b/pkg/services/netmap/server.go @@ -3,7 +3,7 @@ package netmap import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap" ) // Server is an interface of the FrostFS API Netmap service server. diff --git a/pkg/services/netmap/sign.go b/pkg/services/netmap/sign.go index 9a16ad8f1..5f184d5c0 100644 --- a/pkg/services/netmap/sign.go +++ b/pkg/services/netmap/sign.go @@ -4,8 +4,8 @@ import ( "context" "crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap" ) type signService struct { diff --git a/pkg/services/object/acl/eacl/v2/eacl_test.go b/pkg/services/object/acl/eacl/v2/eacl_test.go index 023b99239..94e015abe 100644 --- a/pkg/services/object/acl/eacl/v2/eacl_test.go +++ b/pkg/services/object/acl/eacl/v2/eacl_test.go @@ -6,9 +6,9 @@ import ( "errors" "testing" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/services/object/acl/eacl/v2/headers.go b/pkg/services/object/acl/eacl/v2/headers.go index 34975e1e6..ecb793df8 100644 --- a/pkg/services/object/acl/eacl/v2/headers.go +++ b/pkg/services/object/acl/eacl/v2/headers.go @@ -5,10 +5,10 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - refsV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/acl" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + refsV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/services/object/acl/eacl/v2/object.go b/pkg/services/object/acl/eacl/v2/object.go index 72bd4c2d2..92570a3c5 100644 --- a/pkg/services/object/acl/eacl/v2/object.go +++ b/pkg/services/object/acl/eacl/v2/object.go @@ -3,7 +3,7 @@ package v2 import ( "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/services/object/acl/eacl/v2/xheader.go b/pkg/services/object/acl/eacl/v2/xheader.go index c1fdea9d8..ce380c117 100644 --- a/pkg/services/object/acl/eacl/v2/xheader.go +++ b/pkg/services/object/acl/eacl/v2/xheader.go @@ -1,7 +1,7 @@ package v2 import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" ) diff --git a/pkg/services/object/acl/v2/request.go b/pkg/services/object/acl/v2/request.go index 74279e453..e35cd2e11 100644 --- a/pkg/services/object/acl/v2/request.go +++ b/pkg/services/object/acl/v2/request.go @@ -4,7 +4,7 @@ import ( "crypto/ecdsa" "fmt" - sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" diff --git a/pkg/services/object/acl/v2/request_test.go b/pkg/services/object/acl/v2/request_test.go index 980d1a2e5..618af3469 100644 --- a/pkg/services/object/acl/v2/request_test.go +++ b/pkg/services/object/acl/v2/request_test.go @@ -3,9 +3,9 @@ package v2 import ( "testing" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - sigutilV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/signature" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + sigutilV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/signature" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 5a8e8b065..e02a3be36 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -6,13 +6,13 @@ import ( "fmt" "strings" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" diff --git a/pkg/services/object/acl/v2/util.go b/pkg/services/object/acl/v2/util.go index c5225e8c4..e02f70771 100644 --- a/pkg/services/object/acl/v2/util.go +++ b/pkg/services/object/acl/v2/util.go @@ -6,9 +6,9 @@ import ( "errors" "fmt" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - refsV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + refsV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" diff --git a/pkg/services/object/acl/v2/util_test.go b/pkg/services/object/acl/v2/util_test.go index 8c6d01ae9..4b19cecfe 100644 --- a/pkg/services/object/acl/v2/util_test.go +++ b/pkg/services/object/acl/v2/util_test.go @@ -6,8 +6,8 @@ import ( "crypto/rand" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/acl" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test" aclsdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 8ce1b429d..abcd2f4bb 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -5,12 +5,12 @@ import ( "errors" "fmt" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" checkercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/common/ape" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index 66f0822e4..e03b5750c 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -8,13 +8,13 @@ import ( "fmt" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index e12fccb5e..cb9bbf1b8 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -8,10 +8,10 @@ import ( "net" "strconv" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" diff --git a/pkg/services/object/ape/request_test.go b/pkg/services/object/ape/request_test.go index 9dad69d17..787785b60 100644 --- a/pkg/services/object/ape/request_test.go +++ b/pkg/services/object/ape/request_test.go @@ -6,8 +6,8 @@ import ( "net" "testing" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" aperequest "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/request" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" checksumtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 6eedaf99e..c114f02f6 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -6,12 +6,12 @@ import ( "errors" "fmt" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/services/object/audit.go b/pkg/services/object/audit.go index 39e1f9f2d..b42084634 100644 --- a/pkg/services/object/audit.go +++ b/pkg/services/object/audit.go @@ -5,12 +5,12 @@ import ( "errors" "sync/atomic" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/audit" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) diff --git a/pkg/services/object/common.go b/pkg/services/object/common.go index f48cc5b3d..758156607 100644 --- a/pkg/services/object/common.go +++ b/pkg/services/object/common.go @@ -3,7 +3,7 @@ package object import ( "context" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) diff --git a/pkg/services/object/common/writer/ec_test.go b/pkg/services/object/common/writer/ec_test.go index c828c79ba..8b2599e5f 100644 --- a/pkg/services/object/common/writer/ec_test.go +++ b/pkg/services/object/common/writer/ec_test.go @@ -10,13 +10,13 @@ import ( "strconv" "testing" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" apiclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index c2f92950f..ec771320e 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -6,10 +6,10 @@ import ( "fmt" "strconv" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" apiclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/services/object/delete/v2/service.go b/pkg/services/object/delete/v2/service.go index 10dcd0e87..7146f0361 100644 --- a/pkg/services/object/delete/v2/service.go +++ b/pkg/services/object/delete/v2/service.go @@ -3,8 +3,8 @@ package deletesvc import ( "context" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" deletesvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/delete" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" ) // Service implements Delete operation of Object service v2. diff --git a/pkg/services/object/delete/v2/util.go b/pkg/services/object/delete/v2/util.go index d0db1f543..c57d4562a 100644 --- a/pkg/services/object/delete/v2/util.go +++ b/pkg/services/object/delete/v2/util.go @@ -4,10 +4,10 @@ import ( "errors" "fmt" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" deletesvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/delete" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) diff --git a/pkg/services/object/get/getrangeec_test.go b/pkg/services/object/get/getrangeec_test.go index a6882d4a8..599a6f176 100644 --- a/pkg/services/object/get/getrangeec_test.go +++ b/pkg/services/object/get/getrangeec_test.go @@ -6,12 +6,12 @@ import ( "fmt" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" coreContainer "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" diff --git a/pkg/services/object/get/v2/errors.go b/pkg/services/object/get/v2/errors.go index 213455e10..aaa09b891 100644 --- a/pkg/services/object/get/v2/errors.go +++ b/pkg/services/object/get/v2/errors.go @@ -4,8 +4,8 @@ import ( "errors" "fmt" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - refs "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + refs "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" ) var ( diff --git a/pkg/services/object/get/v2/get_forwarder.go b/pkg/services/object/get/v2/get_forwarder.go index 18194c740..60fcd7fbf 100644 --- a/pkg/services/object/get/v2/get_forwarder.go +++ b/pkg/services/object/get/v2/get_forwarder.go @@ -7,16 +7,16 @@ import ( "io" "sync" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" - rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc" + rpcclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/signature" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/services/object/get/v2/get_range_forwarder.go b/pkg/services/object/get/v2/get_range_forwarder.go index 10ecfc4a3..a44616fc9 100644 --- a/pkg/services/object/get/v2/get_range_forwarder.go +++ b/pkg/services/object/get/v2/get_range_forwarder.go @@ -7,15 +7,15 @@ import ( "io" "sync" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" - rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc" + rpcclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/signature" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/pkg/services/object/get/v2/get_range_hash.go b/pkg/services/object/get/v2/get_range_hash.go index e97b60f66..e8e82ddd9 100644 --- a/pkg/services/object/get/v2/get_range_hash.go +++ b/pkg/services/object/get/v2/get_range_hash.go @@ -5,15 +5,15 @@ import ( "encoding/hex" "fmt" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" - rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" clientCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc" + rpcclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/signature" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/services/object/get/v2/head_forwarder.go b/pkg/services/object/get/v2/head_forwarder.go index 5e16008b8..56056398d 100644 --- a/pkg/services/object/get/v2/head_forwarder.go +++ b/pkg/services/object/get/v2/head_forwarder.go @@ -5,15 +5,15 @@ import ( "crypto/ecdsa" "sync" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" - rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc" + rpcclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/signature" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index edd19b441..24b2f0099 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -4,7 +4,6 @@ import ( "context" "errors" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -12,6 +11,7 @@ import ( getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" objutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.uber.org/zap" ) diff --git a/pkg/services/object/get/v2/streamer.go b/pkg/services/object/get/v2/streamer.go index ce9a5c767..98207336c 100644 --- a/pkg/services/object/get/v2/streamer.go +++ b/pkg/services/object/get/v2/streamer.go @@ -3,8 +3,8 @@ package getsvc import ( "context" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index 852c2aec3..bfa7fd619 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -5,17 +5,17 @@ import ( "crypto/sha256" "hash" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/status" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/signature" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/status" clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/services/object/internal/key.go b/pkg/services/object/internal/key.go index eba716976..1e0a7ef90 100644 --- a/pkg/services/object/internal/key.go +++ b/pkg/services/object/internal/key.go @@ -3,8 +3,8 @@ package internal import ( "bytes" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" ) // VerifyResponseKeyV2 checks if response is signed with expected key. Returns client.ErrWrongPublicKey if not. diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index 61aed5003..377350fdd 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -4,8 +4,8 @@ import ( "context" "time" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" ) type ( diff --git a/pkg/services/object/patch/streamer.go b/pkg/services/object/patch/streamer.go index 677c6610f..91b4efdc1 100644 --- a/pkg/services/object/patch/streamer.go +++ b/pkg/services/object/patch/streamer.go @@ -7,13 +7,13 @@ import ( "fmt" "io" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - refsV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/target" objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + refsV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/patcher" diff --git a/pkg/services/object/patch/util.go b/pkg/services/object/patch/util.go index 4f3c3ef17..b9416789c 100644 --- a/pkg/services/object/patch/util.go +++ b/pkg/services/object/patch/util.go @@ -6,8 +6,8 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 5f9b5d110..3a0b3901f 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -9,11 +9,6 @@ import ( "hash" "sync" - objectAPI "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" @@ -28,6 +23,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + objectAPI "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/signature" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" diff --git a/pkg/services/object/put/v2/service.go b/pkg/services/object/put/v2/service.go index db902ae59..78d4c711d 100644 --- a/pkg/services/object/put/v2/service.go +++ b/pkg/services/object/put/v2/service.go @@ -4,10 +4,10 @@ import ( "context" "fmt" - objectAPI "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + objectAPI "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" ) // Service implements Put operation of Object service v2. diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 5bf15b4cd..36b514fbc 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -4,11 +4,6 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" - rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/target" @@ -17,6 +12,11 @@ import ( putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/signature" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) diff --git a/pkg/services/object/put/v2/util.go b/pkg/services/object/put/v2/util.go index a157a9542..5ec9ebe10 100644 --- a/pkg/services/object/put/v2/util.go +++ b/pkg/services/object/put/v2/util.go @@ -1,10 +1,10 @@ package putsvc import ( - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - refsV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + refsV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" ) diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index d7ba9f843..3787b4168 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" ) type ResponseService struct { diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index 44abcfe5b..0a40025e1 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -9,7 +9,6 @@ import ( "strconv" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" netmapcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -17,6 +16,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" diff --git a/pkg/services/object/search/v2/request_forwarder.go b/pkg/services/object/search/v2/request_forwarder.go index 5a2e9b936..7bb6e4d3c 100644 --- a/pkg/services/object/search/v2/request_forwarder.go +++ b/pkg/services/object/search/v2/request_forwarder.go @@ -8,14 +8,14 @@ import ( "io" "sync" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc" - rpcclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc" + rpcclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/signature" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) diff --git a/pkg/services/object/search/v2/service.go b/pkg/services/object/search/v2/service.go index 78b72ac79..856cd9f04 100644 --- a/pkg/services/object/search/v2/service.go +++ b/pkg/services/object/search/v2/service.go @@ -1,10 +1,10 @@ package searchsvc import ( - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" objutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" ) // Service implements Search operation of Object service v2. diff --git a/pkg/services/object/search/v2/streamer.go b/pkg/services/object/search/v2/streamer.go index 15e2d53d5..93b281343 100644 --- a/pkg/services/object/search/v2/streamer.go +++ b/pkg/services/object/search/v2/streamer.go @@ -1,9 +1,9 @@ package searchsvc import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" ) diff --git a/pkg/services/object/search/v2/util.go b/pkg/services/object/search/v2/util.go index e971fa8e5..48ae98958 100644 --- a/pkg/services/object/search/v2/util.go +++ b/pkg/services/object/search/v2/util.go @@ -5,12 +5,12 @@ import ( "errors" "fmt" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" searchsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" diff --git a/pkg/services/object/server.go b/pkg/services/object/server.go index da98ce245..c570e9d8e 100644 --- a/pkg/services/object/server.go +++ b/pkg/services/object/server.go @@ -3,8 +3,8 @@ package object import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" ) // GetObjectStream is an interface of FrostFS API v2 compatible object streamer. diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index f5ae97b62..2c5e794e9 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -5,8 +5,8 @@ import ( "crypto/ecdsa" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" ) type SignService struct { diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index e560d6d8c..1438a0ea2 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -4,8 +4,8 @@ import ( "bytes" "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" ) type ( diff --git a/pkg/services/object/util/key_test.go b/pkg/services/object/util/key_test.go index cb7ddfde5..1753a26f7 100644 --- a/pkg/services/object/util/key_test.go +++ b/pkg/services/object/util/key_test.go @@ -5,10 +5,10 @@ import ( "crypto/elliptic" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" tokenStorage "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage/temporary" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" diff --git a/pkg/services/object/util/prm.go b/pkg/services/object/util/prm.go index 80c0db39e..34d8ec704 100644 --- a/pkg/services/object/util/prm.go +++ b/pkg/services/object/util/prm.go @@ -4,7 +4,7 @@ import ( "fmt" "strconv" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" sessionsdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" ) diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go index 48a08b693..7476dbd48 100644 --- a/pkg/services/object_manager/tombstone/checker.go +++ b/pkg/services/object_manager/tombstone/checker.go @@ -4,9 +4,9 @@ import ( "context" "strconv" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" diff --git a/pkg/services/session/executor.go b/pkg/services/session/executor.go index 76c220fab..e914119b4 100644 --- a/pkg/services/session/executor.go +++ b/pkg/services/session/executor.go @@ -4,10 +4,10 @@ import ( "context" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "go.uber.org/zap" ) diff --git a/pkg/services/session/server.go b/pkg/services/session/server.go index 9e44ae667..e8555a7c9 100644 --- a/pkg/services/session/server.go +++ b/pkg/services/session/server.go @@ -3,7 +3,7 @@ package session import ( "context" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" ) // Server is an interface of the FrostFS API Session service server. diff --git a/pkg/services/session/sign.go b/pkg/services/session/sign.go index 690fff896..3664c1403 100644 --- a/pkg/services/session/sign.go +++ b/pkg/services/session/sign.go @@ -4,8 +4,8 @@ import ( "context" "crypto/ecdsa" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" ) type signService struct { diff --git a/pkg/services/session/storage/persistent/executor.go b/pkg/services/session/storage/persistent/executor.go index 21f55a7d1..ea0233f9a 100644 --- a/pkg/services/session/storage/persistent/executor.go +++ b/pkg/services/session/storage/persistent/executor.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.etcd.io/bbolt" diff --git a/pkg/services/session/storage/persistent/executor_test.go b/pkg/services/session/storage/persistent/executor_test.go index 124d36930..f80ecb591 100644 --- a/pkg/services/session/storage/persistent/executor_test.go +++ b/pkg/services/session/storage/persistent/executor_test.go @@ -8,8 +8,8 @@ import ( "path/filepath" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/stretchr/testify/require" diff --git a/pkg/services/session/storage/temporary/executor.go b/pkg/services/session/storage/temporary/executor.go index cd498709c..d531b25cb 100644 --- a/pkg/services/session/storage/temporary/executor.go +++ b/pkg/services/session/storage/temporary/executor.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/mr-tron/base58" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" diff --git a/pkg/services/tree/service_frostfs.pb.go b/pkg/services/tree/service_frostfs.pb.go index 7b6abb1dd..05076ee03 100644 --- a/pkg/services/tree/service_frostfs.pb.go +++ b/pkg/services/tree/service_frostfs.pb.go @@ -5,9 +5,9 @@ package tree import ( json "encoding/json" fmt "fmt" - pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" - proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" - encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + pool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto/encoding" easyproto "github.com/VictoriaMetrics/easyproto" jlexer "github.com/mailru/easyjson/jlexer" jwriter "github.com/mailru/easyjson/jwriter" diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 20a629fcc..4fd4a7e1e 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -8,8 +8,8 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 939ff170d..7bc5002dc 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -8,13 +8,13 @@ import ( "errors" "testing" - aclV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" checkercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" + aclV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/acl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" diff --git a/pkg/services/tree/types_frostfs.pb.go b/pkg/services/tree/types_frostfs.pb.go index 4399f8a8b..13f1a43be 100644 --- a/pkg/services/tree/types_frostfs.pb.go +++ b/pkg/services/tree/types_frostfs.pb.go @@ -5,9 +5,9 @@ package tree import ( json "encoding/json" fmt "fmt" - pool "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" - proto "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" - encoding "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/encoding" + pool "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/pool" + proto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto" + encoding "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/proto/encoding" easyproto "github.com/VictoriaMetrics/easyproto" jlexer "github.com/mailru/easyjson/jlexer" jwriter "github.com/mailru/easyjson/jwriter" diff --git a/pkg/services/util/response/service.go b/pkg/services/util/response/service.go index 005a643e5..5152a8ece 100644 --- a/pkg/services/util/response/service.go +++ b/pkg/services/util/response/service.go @@ -1,10 +1,10 @@ package response import ( - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" ) diff --git a/pkg/services/util/sign.go b/pkg/services/util/sign.go index bce43d6e8..348a45a94 100644 --- a/pkg/services/util/sign.go +++ b/pkg/services/util/sign.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" - "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/signature" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/signature" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) From 755cae3f19801437fd60dcbbbc2cda64969006ab Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 7 Nov 2024 17:50:14 +0300 Subject: [PATCH 1503/1943] [#1479] control: Regenerate protobufs for service Signed-off-by: Airat Arifullin --- pkg/services/control/ir/service_frostfs.pb.go | 340 ++- pkg/services/control/ir/types_frostfs.pb.go | 45 +- pkg/services/control/service_frostfs.pb.go | 1897 ++++++++++++++--- pkg/services/control/types_frostfs.pb.go | 283 ++- 4 files changed, 2144 insertions(+), 421 deletions(-) diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index ff4ce7245..d27746263 100644 --- a/pkg/services/control/ir/service_frostfs.pb.go +++ b/pkg/services/control/ir/service_frostfs.pb.go @@ -233,14 +233,25 @@ func (x *HealthCheckRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -375,11 +386,22 @@ func (x *HealthCheckResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"healthStatus\":" - out.RawString(prefix[1:]) - out.Int32(int32(x.HealthStatus)) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"healthStatus\":" + out.RawString(prefix) + v := int32(x.HealthStatus) + if vv, ok := HealthStatus_name[v]; ok { + out.String(vv) + } else { + out.Int32(v) + } } out.RawByte('}') } @@ -564,14 +586,25 @@ func (x *HealthCheckResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -706,10 +739,16 @@ func (x *TickEpochRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"vub\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"vub\":" + out.RawString(prefix) out.Uint32(x.Vub) } out.RawByte('}') @@ -743,7 +782,15 @@ func (x *TickEpochRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "vub": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.Vub = f } } @@ -879,14 +926,25 @@ func (x *TickEpochRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -1021,10 +1079,16 @@ func (x *TickEpochResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"vub\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"vub\":" + out.RawString(prefix) out.Uint32(x.Vub) } out.RawByte('}') @@ -1058,7 +1122,15 @@ func (x *TickEpochResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "vub": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.Vub = f } } @@ -1194,14 +1266,25 @@ func (x *TickEpochResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -1356,14 +1439,29 @@ func (x *RemoveNodeRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"key\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.Key) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"key\":" + out.RawString(prefix) + if x.Key != nil { + out.Base64Bytes(x.Key) + } else { + out.String("") + } } { - const prefix string = ",\"vub\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"vub\":" out.RawString(prefix) out.Uint32(x.Vub) } @@ -1398,13 +1496,27 @@ func (x *RemoveNodeRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "key": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Key = f } case "vub": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.Vub = f } } @@ -1540,14 +1652,25 @@ func (x *RemoveNodeRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -1682,10 +1805,16 @@ func (x *RemoveNodeResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"vub\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"vub\":" + out.RawString(prefix) out.Uint32(x.Vub) } out.RawByte('}') @@ -1719,7 +1848,15 @@ func (x *RemoveNodeResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "vub": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.Vub = f } } @@ -1855,14 +1992,25 @@ func (x *RemoveNodeResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -2037,19 +2185,43 @@ func (x *RemoveContainerRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"containerId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ContainerId) - } - { - const prefix string = ",\"owner\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" out.RawString(prefix) - out.Base64Bytes(x.Owner) + if x.ContainerId != nil { + out.Base64Bytes(x.ContainerId) + } else { + out.String("") + } } { - const prefix string = ",\"vub\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"owner\":" + out.RawString(prefix) + if x.Owner != nil { + out.Base64Bytes(x.Owner) + } else { + out.String("") + } + } + { + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"vub\":" out.RawString(prefix) out.Uint32(x.Vub) } @@ -2084,19 +2256,39 @@ func (x *RemoveContainerRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "containerId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ContainerId = f } case "owner": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Owner = f } case "vub": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.Vub = f } } @@ -2232,14 +2424,25 @@ func (x *RemoveContainerRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -2374,10 +2577,16 @@ func (x *RemoveContainerResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"vub\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"vub\":" + out.RawString(prefix) out.Uint32(x.Vub) } out.RawByte('}') @@ -2411,7 +2620,15 @@ func (x *RemoveContainerResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "vub": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.Vub = f } } @@ -2547,14 +2764,25 @@ func (x *RemoveContainerResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } diff --git a/pkg/services/control/ir/types_frostfs.pb.go b/pkg/services/control/ir/types_frostfs.pb.go index 32bf457a1..407eec6ad 100644 --- a/pkg/services/control/ir/types_frostfs.pb.go +++ b/pkg/services/control/ir/types_frostfs.pb.go @@ -155,16 +155,35 @@ func (x *Signature) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"key\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.Key) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"key\":" + out.RawString(prefix) + if x.Key != nil { + out.Base64Bytes(x.Key) + } else { + out.String("") + } } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) - out.Base64Bytes(x.Sign) + if x.Sign != nil { + out.Base64Bytes(x.Sign) + } else { + out.String("") + } } out.RawByte('}') } @@ -197,13 +216,25 @@ func (x *Signature) UnmarshalEasyJSON(in *jlexer.Lexer) { case "key": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Key = f } case "signature": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Sign = f } } diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 96b896478..0b4e3cf32 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -233,14 +233,25 @@ func (x *HealthCheckRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -395,16 +406,37 @@ func (x *HealthCheckResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"netmapStatus\":" - out.RawString(prefix[1:]) - out.Int32(int32(x.NetmapStatus)) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"netmapStatus\":" + out.RawString(prefix) + v := int32(x.NetmapStatus) + if vv, ok := NetmapStatus_name[v]; ok { + out.String(vv) + } else { + out.Int32(v) + } } { - const prefix string = ",\"healthStatus\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"healthStatus\":" out.RawString(prefix) - out.Int32(int32(x.HealthStatus)) + v := int32(x.HealthStatus) + if vv, ok := HealthStatus_name[v]; ok { + out.String(vv) + } else { + out.Int32(v) + } } out.RawByte('}') } @@ -611,14 +643,25 @@ func (x *HealthCheckResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -773,14 +816,30 @@ func (x *SetNetmapStatusRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"status\":" - out.RawString(prefix[1:]) - out.Int32(int32(x.Status)) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"status\":" + out.RawString(prefix) + v := int32(x.Status) + if vv, ok := NetmapStatus_name[v]; ok { + out.String(vv) + } else { + out.Int32(v) + } } { - const prefix string = ",\"forceMaintenance\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"forceMaintenance\":" out.RawString(prefix) out.Bool(x.ForceMaintenance) } @@ -973,14 +1032,25 @@ func (x *SetNetmapStatusRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -1254,14 +1324,25 @@ func (x *SetNetmapStatusResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -1535,14 +1616,25 @@ func (x *GetNetmapStatusRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -1697,16 +1789,34 @@ func (x *GetNetmapStatusResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"status\":" - out.RawString(prefix[1:]) - out.Int32(int32(x.Status)) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"status\":" + out.RawString(prefix) + v := int32(x.Status) + if vv, ok := NetmapStatus_name[v]; ok { + out.String(vv) + } else { + out.Int32(v) + } } { - const prefix string = ",\"epoch\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"epoch\":" out.RawString(prefix) - out.Uint64(x.Epoch) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.Epoch, 10) + out.RawByte('"') } out.RawByte('}') } @@ -1761,7 +1871,15 @@ func (x *GetNetmapStatusResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "epoch": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.Epoch = f } } @@ -1897,14 +2015,25 @@ func (x *GetNetmapStatusResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -2039,16 +2168,26 @@ func (x *DropObjectsRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"addressList\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"addressList\":" + out.RawString(prefix) out.RawByte('[') for i := range x.AddressList { if i != 0 { out.RawByte(',') } - out.Base64Bytes(x.AddressList[i]) + if x.AddressList[i] != nil { + out.Base64Bytes(x.AddressList[i]) + } else { + out.String("") + } } out.RawByte(']') } @@ -2086,7 +2225,13 @@ func (x *DropObjectsRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list [][]byte in.Delim('[') for !in.IsDelim(']') { - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } list = append(list, f) in.WantComma() } @@ -2226,14 +2371,25 @@ func (x *DropObjectsRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -2507,14 +2663,25 @@ func (x *DropObjectsResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -2788,14 +2955,25 @@ func (x *ListShardsRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -2936,10 +3114,16 @@ func (x *ListShardsResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"shards\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shards\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Shards { if i != 0 { @@ -3124,14 +3308,25 @@ func (x *ListShardsResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -3306,26 +3501,51 @@ func (x *SetShardModeRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"shardID\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Shard_ID { if i != 0 { out.RawByte(',') } - out.Base64Bytes(x.Shard_ID[i]) + if x.Shard_ID[i] != nil { + out.Base64Bytes(x.Shard_ID[i]) + } else { + out.String("") + } } out.RawByte(']') } { - const prefix string = ",\"mode\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"mode\":" out.RawString(prefix) - out.Int32(int32(x.Mode)) + v := int32(x.Mode) + if vv, ok := ShardMode_name[v]; ok { + out.String(vv) + } else { + out.Int32(v) + } } { - const prefix string = ",\"resetErrorCounter\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"resetErrorCounter\":" out.RawString(prefix) out.Bool(x.ResetErrorCounter) } @@ -3363,7 +3583,13 @@ func (x *SetShardModeRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list [][]byte in.Delim('[') for !in.IsDelim(']') { - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } list = append(list, f) in.WantComma() } @@ -3531,14 +3757,25 @@ func (x *SetShardModeRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -3812,14 +4049,25 @@ func (x *SetShardModeResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -3994,21 +4242,43 @@ func (x *SynchronizeTreeRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"containerId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ContainerId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" + out.RawString(prefix) + if x.ContainerId != nil { + out.Base64Bytes(x.ContainerId) + } else { + out.String("") + } } { - const prefix string = ",\"treeId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"treeId\":" out.RawString(prefix) out.String(x.TreeId) } { - const prefix string = ",\"height\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"height\":" out.RawString(prefix) - out.Uint64(x.Height) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.Height, 10) + out.RawByte('"') } out.RawByte('}') } @@ -4041,7 +4311,13 @@ func (x *SynchronizeTreeRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "containerId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ContainerId = f } case "treeId": @@ -4053,7 +4329,15 @@ func (x *SynchronizeTreeRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "height": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.Height = f } } @@ -4189,14 +4473,25 @@ func (x *SynchronizeTreeRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -4470,14 +4765,25 @@ func (x *SynchronizeTreeResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -4632,21 +4938,36 @@ func (x *EvacuateShardRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"shardID\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Shard_ID { if i != 0 { out.RawByte(',') } - out.Base64Bytes(x.Shard_ID[i]) + if x.Shard_ID[i] != nil { + out.Base64Bytes(x.Shard_ID[i]) + } else { + out.String("") + } } out.RawByte(']') } { - const prefix string = ",\"ignoreErrors\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"ignoreErrors\":" out.RawString(prefix) out.Bool(x.IgnoreErrors) } @@ -4684,7 +5005,13 @@ func (x *EvacuateShardRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list [][]byte in.Delim('[') for !in.IsDelim(']') { - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } list = append(list, f) in.WantComma() } @@ -4830,14 +5157,25 @@ func (x *EvacuateShardRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -4972,10 +5310,16 @@ func (x *EvacuateShardResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"count\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"count\":" + out.RawString(prefix) out.Uint32(x.Count) } out.RawByte('}') @@ -5009,7 +5353,15 @@ func (x *EvacuateShardResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "count": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.Count = f } } @@ -5145,14 +5497,25 @@ func (x *EvacuateShardResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -5307,21 +5670,36 @@ func (x *FlushCacheRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"shardID\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Shard_ID { if i != 0 { out.RawByte(',') } - out.Base64Bytes(x.Shard_ID[i]) + if x.Shard_ID[i] != nil { + out.Base64Bytes(x.Shard_ID[i]) + } else { + out.String("") + } } out.RawByte(']') } { - const prefix string = ",\"seal\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"seal\":" out.RawString(prefix) out.Bool(x.Seal) } @@ -5359,7 +5737,13 @@ func (x *FlushCacheRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list [][]byte in.Delim('[') for !in.IsDelim(']') { - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } list = append(list, f) in.WantComma() } @@ -5505,14 +5889,25 @@ func (x *FlushCacheRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -5786,14 +6181,25 @@ func (x *FlushCacheResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -5948,14 +6354,25 @@ func (x *DoctorRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"concurrency\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"concurrency\":" + out.RawString(prefix) out.Uint32(x.Concurrency) } { - const prefix string = ",\"removeDuplicates\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"removeDuplicates\":" out.RawString(prefix) out.Bool(x.RemoveDuplicates) } @@ -5990,7 +6407,15 @@ func (x *DoctorRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "concurrency": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.Concurrency = f } case "removeDuplicates": @@ -6132,14 +6557,25 @@ func (x *DoctorRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -6413,14 +6849,25 @@ func (x *DoctorResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -6690,41 +7137,76 @@ func (x *StartShardEvacuationRequest_Body) MarshalEasyJSON(out *jwriter.Writer) out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"shardID\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Shard_ID { if i != 0 { out.RawByte(',') } - out.Base64Bytes(x.Shard_ID[i]) + if x.Shard_ID[i] != nil { + out.Base64Bytes(x.Shard_ID[i]) + } else { + out.String("") + } } out.RawByte(']') } { - const prefix string = ",\"ignoreErrors\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"ignoreErrors\":" out.RawString(prefix) out.Bool(x.IgnoreErrors) } { - const prefix string = ",\"scope\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"scope\":" out.RawString(prefix) out.Uint32(x.Scope) } { - const prefix string = ",\"containerWorkerCount\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerWorkerCount\":" out.RawString(prefix) out.Uint32(x.ContainerWorkerCount) } { - const prefix string = ",\"objectWorkerCount\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"objectWorkerCount\":" out.RawString(prefix) out.Uint32(x.ObjectWorkerCount) } { - const prefix string = ",\"repOneOnly\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"repOneOnly\":" out.RawString(prefix) out.Bool(x.RepOneOnly) } @@ -6762,7 +7244,13 @@ func (x *StartShardEvacuationRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list [][]byte in.Delim('[') for !in.IsDelim(']') { - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } list = append(list, f) in.WantComma() } @@ -6778,19 +7266,43 @@ func (x *StartShardEvacuationRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "scope": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.Scope = f } case "containerWorkerCount": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.ContainerWorkerCount = f } case "objectWorkerCount": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.ObjectWorkerCount = f } case "repOneOnly": @@ -6932,14 +7444,25 @@ func (x *StartShardEvacuationRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -7213,14 +7736,25 @@ func (x *StartShardEvacuationResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -7494,14 +8028,25 @@ func (x *GetShardEvacuationStatusRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -7671,11 +8216,19 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) MarshalEasyJSON(ou out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"value\":" - out.RawString(prefix[1:]) - out.Int64(x.Value) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"value\":" + out.RawString(prefix) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendInt(out.Buffer.Buf, x.Value, 10) + out.RawByte('"') } out.RawByte('}') } @@ -7708,7 +8261,15 @@ func (x *GetShardEvacuationStatusResponse_Body_UnixTimestamp) UnmarshalEasyJSON( case "value": { var f int64 - f = in.Int64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseInt(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := int64(v) + f = pv x.Value = f } } @@ -7800,11 +8361,19 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) MarshalEasyJSON(out *jw out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"seconds\":" - out.RawString(prefix[1:]) - out.Int64(x.Seconds) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"seconds\":" + out.RawString(prefix) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendInt(out.Buffer.Buf, x.Seconds, 10) + out.RawByte('"') } out.RawByte('}') } @@ -7837,7 +8406,15 @@ func (x *GetShardEvacuationStatusResponse_Body_Duration) UnmarshalEasyJSON(in *j case "seconds": { var f int64 - f = in.Int64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseInt(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := int64(v) + f = pv x.Seconds = f } } @@ -8155,73 +8732,157 @@ func (x *GetShardEvacuationStatusResponse_Body) MarshalEasyJSON(out *jwriter.Wri out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"totalObjects\":" - out.RawString(prefix[1:]) - out.Uint64(x.TotalObjects) - } - { - const prefix string = ",\"evacuatedObjects\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"totalObjects\":" out.RawString(prefix) - out.Uint64(x.EvacuatedObjects) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.TotalObjects, 10) + out.RawByte('"') } { - const prefix string = ",\"failedObjects\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"evacuatedObjects\":" out.RawString(prefix) - out.Uint64(x.FailedObjects) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.EvacuatedObjects, 10) + out.RawByte('"') } { - const prefix string = ",\"shardID\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"failedObjects\":" + out.RawString(prefix) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.FailedObjects, 10) + out.RawByte('"') + } + { + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" out.RawString(prefix) out.RawByte('[') for i := range x.Shard_ID { if i != 0 { out.RawByte(',') } - out.Base64Bytes(x.Shard_ID[i]) + if x.Shard_ID[i] != nil { + out.Base64Bytes(x.Shard_ID[i]) + } else { + out.String("") + } } out.RawByte(']') } { - const prefix string = ",\"status\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"status\":" out.RawString(prefix) - out.Int32(int32(x.Status)) + v := int32(x.Status) + if vv, ok := GetShardEvacuationStatusResponse_Body_Status_name[v]; ok { + out.String(vv) + } else { + out.Int32(v) + } } { - const prefix string = ",\"duration\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"duration\":" out.RawString(prefix) x.Duration.MarshalEasyJSON(out) } { - const prefix string = ",\"startedAt\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"startedAt\":" out.RawString(prefix) x.StartedAt.MarshalEasyJSON(out) } { - const prefix string = ",\"errorMessage\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"errorMessage\":" out.RawString(prefix) out.String(x.ErrorMessage) } { - const prefix string = ",\"skippedObjects\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"skippedObjects\":" out.RawString(prefix) - out.Uint64(x.SkippedObjects) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.SkippedObjects, 10) + out.RawByte('"') } { - const prefix string = ",\"totalTrees\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"totalTrees\":" out.RawString(prefix) - out.Uint64(x.TotalTrees) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.TotalTrees, 10) + out.RawByte('"') } { - const prefix string = ",\"evacuatedTrees\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"evacuatedTrees\":" out.RawString(prefix) - out.Uint64(x.EvacuatedTrees) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.EvacuatedTrees, 10) + out.RawByte('"') } { - const prefix string = ",\"failedTrees\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"failedTrees\":" out.RawString(prefix) - out.Uint64(x.FailedTrees) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.FailedTrees, 10) + out.RawByte('"') } out.RawByte('}') } @@ -8254,19 +8915,43 @@ func (x *GetShardEvacuationStatusResponse_Body) UnmarshalEasyJSON(in *jlexer.Lex case "totalObjects": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.TotalObjects = f } case "evacuatedObjects": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.EvacuatedObjects = f } case "failedObjects": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.FailedObjects = f } case "shardID": @@ -8275,7 +8960,13 @@ func (x *GetShardEvacuationStatusResponse_Body) UnmarshalEasyJSON(in *jlexer.Lex var list [][]byte in.Delim('[') for !in.IsDelim(']') { - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } list = append(list, f) in.WantComma() } @@ -8327,25 +9018,57 @@ func (x *GetShardEvacuationStatusResponse_Body) UnmarshalEasyJSON(in *jlexer.Lex case "skippedObjects": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.SkippedObjects = f } case "totalTrees": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.TotalTrees = f } case "evacuatedTrees": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.EvacuatedTrees = f } case "failedTrees": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.FailedTrees = f } } @@ -8481,14 +9204,25 @@ func (x *GetShardEvacuationStatusResponse) MarshalEasyJSON(out *jwriter.Writer) out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -8762,14 +9496,25 @@ func (x *ResetShardEvacuationStatusRequest) MarshalEasyJSON(out *jwriter.Writer) out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -9043,14 +9788,25 @@ func (x *ResetShardEvacuationStatusResponse) MarshalEasyJSON(out *jwriter.Writer out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -9324,14 +10080,25 @@ func (x *StopShardEvacuationRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -9605,14 +10372,25 @@ func (x *StopShardEvacuationResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -9770,16 +10548,31 @@ func (x *AddChainLocalOverrideRequest_Body) MarshalEasyJSON(out *jwriter.Writer) out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"target\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"target\":" + out.RawString(prefix) x.Target.MarshalEasyJSON(out) } { - const prefix string = ",\"chain\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"chain\":" out.RawString(prefix) - out.Base64Bytes(x.Chain) + if x.Chain != nil { + out.Base64Bytes(x.Chain) + } else { + out.String("") + } } out.RawByte('}') } @@ -9819,7 +10612,13 @@ func (x *AddChainLocalOverrideRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) case "chain": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Chain = f } } @@ -9955,14 +10754,25 @@ func (x *AddChainLocalOverrideRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -10097,11 +10907,21 @@ func (x *AddChainLocalOverrideResponse_Body) MarshalEasyJSON(out *jwriter.Writer out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"chainId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ChainId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"chainId\":" + out.RawString(prefix) + if x.ChainId != nil { + out.Base64Bytes(x.ChainId) + } else { + out.String("") + } } out.RawByte('}') } @@ -10134,7 +10954,13 @@ func (x *AddChainLocalOverrideResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) case "chainId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ChainId = f } } @@ -10270,14 +11096,25 @@ func (x *AddChainLocalOverrideResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -10435,16 +11272,31 @@ func (x *GetChainLocalOverrideRequest_Body) MarshalEasyJSON(out *jwriter.Writer) out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"target\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"target\":" + out.RawString(prefix) x.Target.MarshalEasyJSON(out) } { - const prefix string = ",\"chainId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"chainId\":" out.RawString(prefix) - out.Base64Bytes(x.ChainId) + if x.ChainId != nil { + out.Base64Bytes(x.ChainId) + } else { + out.String("") + } } out.RawByte('}') } @@ -10484,7 +11336,13 @@ func (x *GetChainLocalOverrideRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) case "chainId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ChainId = f } } @@ -10620,14 +11478,25 @@ func (x *GetChainLocalOverrideRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -10762,11 +11631,21 @@ func (x *GetChainLocalOverrideResponse_Body) MarshalEasyJSON(out *jwriter.Writer out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"chain\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.Chain) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"chain\":" + out.RawString(prefix) + if x.Chain != nil { + out.Base64Bytes(x.Chain) + } else { + out.String("") + } } out.RawByte('}') } @@ -10799,7 +11678,13 @@ func (x *GetChainLocalOverrideResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) case "chain": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Chain = f } } @@ -10935,14 +11820,25 @@ func (x *GetChainLocalOverrideResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -11080,10 +11976,16 @@ func (x *ListChainLocalOverridesRequest_Body) MarshalEasyJSON(out *jwriter.Write out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"target\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"target\":" + out.RawString(prefix) x.Target.MarshalEasyJSON(out) } out.RawByte('}') @@ -11254,14 +12156,25 @@ func (x *ListChainLocalOverridesRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -11396,16 +12309,26 @@ func (x *ListChainLocalOverridesResponse_Body) MarshalEasyJSON(out *jwriter.Writ out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"chains\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"chains\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Chains { if i != 0 { out.RawByte(',') } - out.Base64Bytes(x.Chains[i]) + if x.Chains[i] != nil { + out.Base64Bytes(x.Chains[i]) + } else { + out.String("") + } } out.RawByte(']') } @@ -11443,7 +12366,13 @@ func (x *ListChainLocalOverridesResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexe var list [][]byte in.Delim('[') for !in.IsDelim(']') { - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } list = append(list, f) in.WantComma() } @@ -11583,14 +12512,25 @@ func (x *ListChainLocalOverridesResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -11725,10 +12665,16 @@ func (x *ListTargetsLocalOverridesRequest_Body) MarshalEasyJSON(out *jwriter.Wri out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"chainName\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"chainName\":" + out.RawString(prefix) out.String(x.ChainName) } out.RawByte('}') @@ -11898,14 +12844,25 @@ func (x *ListTargetsLocalOverridesRequest) MarshalEasyJSON(out *jwriter.Writer) out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -12046,10 +13003,16 @@ func (x *ListTargetsLocalOverridesResponse_Body) MarshalEasyJSON(out *jwriter.Wr out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"targets\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"targets\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Targets { if i != 0 { @@ -12234,14 +13197,25 @@ func (x *ListTargetsLocalOverridesResponse) MarshalEasyJSON(out *jwriter.Writer) out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -12399,16 +13373,31 @@ func (x *RemoveChainLocalOverrideRequest_Body) MarshalEasyJSON(out *jwriter.Writ out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"target\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"target\":" + out.RawString(prefix) x.Target.MarshalEasyJSON(out) } { - const prefix string = ",\"chainId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"chainId\":" out.RawString(prefix) - out.Base64Bytes(x.ChainId) + if x.ChainId != nil { + out.Base64Bytes(x.ChainId) + } else { + out.String("") + } } out.RawByte('}') } @@ -12448,7 +13437,13 @@ func (x *RemoveChainLocalOverrideRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexe case "chainId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ChainId = f } } @@ -12584,14 +13579,25 @@ func (x *RemoveChainLocalOverrideRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -12865,14 +13871,25 @@ func (x *RemoveChainLocalOverrideResponse) MarshalEasyJSON(out *jwriter.Writer) out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -13010,10 +14027,16 @@ func (x *RemoveChainLocalOverridesByTargetRequest_Body) MarshalEasyJSON(out *jwr out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"target\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"target\":" + out.RawString(prefix) x.Target.MarshalEasyJSON(out) } out.RawByte('}') @@ -13184,14 +14207,25 @@ func (x *RemoveChainLocalOverridesByTargetRequest) MarshalEasyJSON(out *jwriter. out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -13465,14 +14499,25 @@ func (x *RemoveChainLocalOverridesByTargetResponse) MarshalEasyJSON(out *jwriter out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -13687,36 +14732,66 @@ func (x *SealWriteCacheRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"shardID\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Shard_ID { if i != 0 { out.RawByte(',') } - out.Base64Bytes(x.Shard_ID[i]) + if x.Shard_ID[i] != nil { + out.Base64Bytes(x.Shard_ID[i]) + } else { + out.String("") + } } out.RawByte(']') } { - const prefix string = ",\"ignoreErrors\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"ignoreErrors\":" out.RawString(prefix) out.Bool(x.IgnoreErrors) } { - const prefix string = ",\"async\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"async\":" out.RawString(prefix) out.Bool(x.Async) } { - const prefix string = ",\"restoreMode\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"restoreMode\":" out.RawString(prefix) out.Bool(x.RestoreMode) } { - const prefix string = ",\"shrink\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shrink\":" out.RawString(prefix) out.Bool(x.Shrink) } @@ -13754,7 +14829,13 @@ func (x *SealWriteCacheRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list [][]byte in.Delim('[') for !in.IsDelim(']') { - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } list = append(list, f) in.WantComma() } @@ -13918,14 +14999,25 @@ func (x *SealWriteCacheRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -14100,19 +15192,39 @@ func (x *SealWriteCacheResponse_Body_Status) MarshalEasyJSON(out *jwriter.Writer out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"shardID\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.Shard_ID) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" + out.RawString(prefix) + if x.Shard_ID != nil { + out.Base64Bytes(x.Shard_ID) + } else { + out.String("") + } } { - const prefix string = ",\"success\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"success\":" out.RawString(prefix) out.Bool(x.Success) } { - const prefix string = ",\"error\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"error\":" out.RawString(prefix) out.String(x.Error) } @@ -14147,7 +15259,13 @@ func (x *SealWriteCacheResponse_Body_Status) UnmarshalEasyJSON(in *jlexer.Lexer) case "shardID": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Shard_ID = f } case "success": @@ -14257,10 +15375,16 @@ func (x *SealWriteCacheResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"results\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"results\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Results { if i != 0 { @@ -14445,14 +15569,25 @@ func (x *SealWriteCacheResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -14587,16 +15722,26 @@ func (x *DetachShardsRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"shardID\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Shard_ID { if i != 0 { out.RawByte(',') } - out.Base64Bytes(x.Shard_ID[i]) + if x.Shard_ID[i] != nil { + out.Base64Bytes(x.Shard_ID[i]) + } else { + out.String("") + } } out.RawByte(']') } @@ -14634,7 +15779,13 @@ func (x *DetachShardsRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list [][]byte in.Delim('[') for !in.IsDelim(']') { - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } list = append(list, f) in.WantComma() } @@ -14774,14 +15925,25 @@ func (x *DetachShardsRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -15055,14 +16217,25 @@ func (x *DetachShardsResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -15237,26 +16410,46 @@ func (x *StartShardRebuildRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"shardID\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Shard_ID { if i != 0 { out.RawByte(',') } - out.Base64Bytes(x.Shard_ID[i]) + if x.Shard_ID[i] != nil { + out.Base64Bytes(x.Shard_ID[i]) + } else { + out.String("") + } } out.RawByte(']') } { - const prefix string = ",\"targetFillPercent\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"targetFillPercent\":" out.RawString(prefix) out.Uint32(x.TargetFillPercent) } { - const prefix string = ",\"concurrencyLimit\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"concurrencyLimit\":" out.RawString(prefix) out.Uint32(x.ConcurrencyLimit) } @@ -15294,7 +16487,13 @@ func (x *StartShardRebuildRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list [][]byte in.Delim('[') for !in.IsDelim(']') { - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } list = append(list, f) in.WantComma() } @@ -15304,13 +16503,29 @@ func (x *StartShardRebuildRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "targetFillPercent": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.TargetFillPercent = f } case "concurrencyLimit": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.ConcurrencyLimit = f } } @@ -15446,14 +16661,25 @@ func (x *StartShardRebuildRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -15628,19 +16854,39 @@ func (x *StartShardRebuildResponse_Body_Status) MarshalEasyJSON(out *jwriter.Wri out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"shardID\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.Shard_ID) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" + out.RawString(prefix) + if x.Shard_ID != nil { + out.Base64Bytes(x.Shard_ID) + } else { + out.String("") + } } { - const prefix string = ",\"success\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"success\":" out.RawString(prefix) out.Bool(x.Success) } { - const prefix string = ",\"error\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"error\":" out.RawString(prefix) out.String(x.Error) } @@ -15675,7 +16921,13 @@ func (x *StartShardRebuildResponse_Body_Status) UnmarshalEasyJSON(in *jlexer.Lex case "shardID": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Shard_ID = f } case "success": @@ -15785,10 +17037,16 @@ func (x *StartShardRebuildResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"results\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"results\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Results { if i != 0 { @@ -15973,14 +17231,25 @@ func (x *StartShardRebuildResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } diff --git a/pkg/services/control/types_frostfs.pb.go b/pkg/services/control/types_frostfs.pb.go index 50984f15a..69d87292d 100644 --- a/pkg/services/control/types_frostfs.pb.go +++ b/pkg/services/control/types_frostfs.pb.go @@ -234,16 +234,35 @@ func (x *Signature) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"key\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.Key) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"key\":" + out.RawString(prefix) + if x.Key != nil { + out.Base64Bytes(x.Key) + } else { + out.String("") + } } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) - out.Base64Bytes(x.Sign) + if x.Sign != nil { + out.Base64Bytes(x.Sign) + } else { + out.String("") + } } out.RawByte('}') } @@ -276,13 +295,25 @@ func (x *Signature) UnmarshalEasyJSON(in *jlexer.Lexer) { case "key": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Key = f } case "signature": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Sign = f } } @@ -414,19 +445,35 @@ func (x *NodeInfo_Attribute) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"key\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"key\":" + out.RawString(prefix) out.String(x.Key) } { - const prefix string = ",\"value\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"value\":" out.RawString(prefix) out.String(x.Value) } { - const prefix string = ",\"parents\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"parents\":" out.RawString(prefix) out.RawByte('[') for i := range x.Parents { @@ -645,14 +692,29 @@ func (x *NodeInfo) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"publicKey\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.PublicKey) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"publicKey\":" + out.RawString(prefix) + if x.PublicKey != nil { + out.Base64Bytes(x.PublicKey) + } else { + out.String("") + } } { - const prefix string = ",\"addresses\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"addresses\":" out.RawString(prefix) out.RawByte('[') for i := range x.Addresses { @@ -664,7 +726,12 @@ func (x *NodeInfo) MarshalEasyJSON(out *jwriter.Writer) { out.RawByte(']') } { - const prefix string = ",\"attributes\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"attributes\":" out.RawString(prefix) out.RawByte('[') for i := range x.Attributes { @@ -676,9 +743,19 @@ func (x *NodeInfo) MarshalEasyJSON(out *jwriter.Writer) { out.RawByte(']') } { - const prefix string = ",\"state\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"state\":" out.RawString(prefix) - out.Int32(int32(x.State)) + v := int32(x.State) + if vv, ok := NetmapStatus_name[v]; ok { + out.String(vv) + } else { + out.Int32(v) + } } out.RawByte('}') } @@ -711,7 +788,13 @@ func (x *NodeInfo) UnmarshalEasyJSON(in *jlexer.Lexer) { case "publicKey": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.PublicKey = f } case "addresses": @@ -878,14 +961,27 @@ func (x *Netmap) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"epoch\":" - out.RawString(prefix[1:]) - out.Uint64(x.Epoch) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"epoch\":" + out.RawString(prefix) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.Epoch, 10) + out.RawByte('"') } { - const prefix string = ",\"nodes\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"nodes\":" out.RawString(prefix) out.RawByte('[') for i := range x.Nodes { @@ -927,7 +1023,15 @@ func (x *Netmap) UnmarshalEasyJSON(in *jlexer.Lexer) { case "epoch": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.Epoch = f } case "nodes": @@ -1179,19 +1283,39 @@ func (x *ShardInfo) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"shardID\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.Shard_ID) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" + out.RawString(prefix) + if x.Shard_ID != nil { + out.Base64Bytes(x.Shard_ID) + } else { + out.String("") + } } { - const prefix string = ",\"metabasePath\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"metabasePath\":" out.RawString(prefix) out.String(x.MetabasePath) } { - const prefix string = ",\"blobstor\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"blobstor\":" out.RawString(prefix) out.RawByte('[') for i := range x.Blobstor { @@ -1203,27 +1327,57 @@ func (x *ShardInfo) MarshalEasyJSON(out *jwriter.Writer) { out.RawByte(']') } { - const prefix string = ",\"writecachePath\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"writecachePath\":" out.RawString(prefix) out.String(x.WritecachePath) } { - const prefix string = ",\"mode\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"mode\":" out.RawString(prefix) - out.Int32(int32(x.Mode)) + v := int32(x.Mode) + if vv, ok := ShardMode_name[v]; ok { + out.String(vv) + } else { + out.Int32(v) + } } { - const prefix string = ",\"errorCount\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"errorCount\":" out.RawString(prefix) out.Uint32(x.ErrorCount) } { - const prefix string = ",\"piloramaPath\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"piloramaPath\":" out.RawString(prefix) out.String(x.PiloramaPath) } { - const prefix string = ",\"evacuationInProgress\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"evacuationInProgress\":" out.RawString(prefix) out.Bool(x.EvacuationInProgress) } @@ -1258,7 +1412,13 @@ func (x *ShardInfo) UnmarshalEasyJSON(in *jlexer.Lexer) { case "shardID": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Shard_ID = f } case "metabasePath": @@ -1312,7 +1472,15 @@ func (x *ShardInfo) UnmarshalEasyJSON(in *jlexer.Lexer) { case "errorCount": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.ErrorCount = f } case "piloramaPath": @@ -1436,14 +1604,25 @@ func (x *BlobstorInfo) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"path\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"path\":" + out.RawString(prefix) out.String(x.Path) } { - const prefix string = ",\"type\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"type\":" out.RawString(prefix) out.String(x.Type) } @@ -1637,14 +1816,30 @@ func (x *ChainTarget) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"type\":" - out.RawString(prefix[1:]) - out.Int32(int32(x.Type)) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"type\":" + out.RawString(prefix) + v := int32(x.Type) + if vv, ok := ChainTarget_TargetType_name[v]; ok { + out.String(vv) + } else { + out.Int32(v) + } } { - const prefix string = ",\"Name\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"Name\":" out.RawString(prefix) out.String(x.Name) } From 764450d04a38bedc3d1b38a098a7546983fcdf93 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 7 Nov 2024 17:51:33 +0300 Subject: [PATCH 1504/1943] [#1479] tree: Regenerate service protobufs Signed-off-by: Airat Arifullin --- pkg/services/tree/service_frostfs.pb.go | 1252 +++++++++++++++++++---- pkg/services/tree/types_frostfs.pb.go | 143 ++- 2 files changed, 1172 insertions(+), 223 deletions(-) diff --git a/pkg/services/tree/service_frostfs.pb.go b/pkg/services/tree/service_frostfs.pb.go index 05076ee03..88d002621 100644 --- a/pkg/services/tree/service_frostfs.pb.go +++ b/pkg/services/tree/service_frostfs.pb.go @@ -181,24 +181,51 @@ func (x *AddRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"containerId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ContainerId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" + out.RawString(prefix) + if x.ContainerId != nil { + out.Base64Bytes(x.ContainerId) + } else { + out.String("") + } } { - const prefix string = ",\"treeId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"treeId\":" out.RawString(prefix) out.String(x.TreeId) } { - const prefix string = ",\"parentId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"parentId\":" out.RawString(prefix) - out.Uint64(x.ParentId) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.ParentId, 10) + out.RawByte('"') } { - const prefix string = ",\"meta\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"meta\":" out.RawString(prefix) out.RawByte('[') for i := range x.Meta { @@ -210,9 +237,18 @@ func (x *AddRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawByte(']') } { - const prefix string = ",\"bearerToken\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"bearerToken\":" out.RawString(prefix) - out.Base64Bytes(x.BearerToken) + if x.BearerToken != nil { + out.Base64Bytes(x.BearerToken) + } else { + out.String("") + } } out.RawByte('}') } @@ -245,7 +281,13 @@ func (x *AddRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "containerId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ContainerId = f } case "treeId": @@ -257,7 +299,15 @@ func (x *AddRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "parentId": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.ParentId = f } case "meta": @@ -277,7 +327,13 @@ func (x *AddRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "bearerToken": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.BearerToken = f } } @@ -413,14 +469,25 @@ func (x *AddRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -555,11 +622,19 @@ func (x *AddResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"nodeId\":" - out.RawString(prefix[1:]) - out.Uint64(x.NodeId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"nodeId\":" + out.RawString(prefix) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.NodeId, 10) + out.RawByte('"') } out.RawByte('}') } @@ -592,7 +667,15 @@ func (x *AddResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "nodeId": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.NodeId = f } } @@ -728,14 +811,25 @@ func (x *AddResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -976,24 +1070,49 @@ func (x *AddByPathRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"containerId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ContainerId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" + out.RawString(prefix) + if x.ContainerId != nil { + out.Base64Bytes(x.ContainerId) + } else { + out.String("") + } } { - const prefix string = ",\"treeId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"treeId\":" out.RawString(prefix) out.String(x.TreeId) } { - const prefix string = ",\"pathAttribute\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"pathAttribute\":" out.RawString(prefix) out.String(x.PathAttribute) } { - const prefix string = ",\"path\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"path\":" out.RawString(prefix) out.RawByte('[') for i := range x.Path { @@ -1005,7 +1124,12 @@ func (x *AddByPathRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawByte(']') } { - const prefix string = ",\"meta\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"meta\":" out.RawString(prefix) out.RawByte('[') for i := range x.Meta { @@ -1017,9 +1141,18 @@ func (x *AddByPathRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawByte(']') } { - const prefix string = ",\"bearerToken\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"bearerToken\":" out.RawString(prefix) - out.Base64Bytes(x.BearerToken) + if x.BearerToken != nil { + out.Base64Bytes(x.BearerToken) + } else { + out.String("") + } } out.RawByte('}') } @@ -1052,7 +1185,13 @@ func (x *AddByPathRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "containerId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ContainerId = f } case "treeId": @@ -1097,7 +1236,13 @@ func (x *AddByPathRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "bearerToken": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.BearerToken = f } } @@ -1233,14 +1378,25 @@ func (x *AddByPathRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -1397,23 +1553,38 @@ func (x *AddByPathResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"nodes\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"nodes\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Nodes { if i != 0 { out.RawByte(',') } - out.Uint64(x.Nodes[i]) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.Nodes[i], 10) + out.RawByte('"') } out.RawByte(']') } { - const prefix string = ",\"parentId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"parentId\":" out.RawString(prefix) - out.Uint64(x.ParentId) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.ParentId, 10) + out.RawByte('"') } out.RawByte('}') } @@ -1449,7 +1620,15 @@ func (x *AddByPathResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list []uint64 in.Delim('[') for !in.IsDelim(']') { - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv list = append(list, f) in.WantComma() } @@ -1459,7 +1638,15 @@ func (x *AddByPathResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "parentId": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.ParentId = f } } @@ -1595,14 +1782,25 @@ func (x *AddByPathResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -1797,26 +1995,57 @@ func (x *RemoveRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"containerId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ContainerId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" + out.RawString(prefix) + if x.ContainerId != nil { + out.Base64Bytes(x.ContainerId) + } else { + out.String("") + } } { - const prefix string = ",\"treeId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"treeId\":" out.RawString(prefix) out.String(x.TreeId) } { - const prefix string = ",\"nodeId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"nodeId\":" out.RawString(prefix) - out.Uint64(x.NodeId) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.NodeId, 10) + out.RawByte('"') } { - const prefix string = ",\"bearerToken\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"bearerToken\":" out.RawString(prefix) - out.Base64Bytes(x.BearerToken) + if x.BearerToken != nil { + out.Base64Bytes(x.BearerToken) + } else { + out.String("") + } } out.RawByte('}') } @@ -1849,7 +2078,13 @@ func (x *RemoveRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "containerId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ContainerId = f } case "treeId": @@ -1861,13 +2096,27 @@ func (x *RemoveRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "nodeId": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.NodeId = f } case "bearerToken": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.BearerToken = f } } @@ -2003,14 +2252,25 @@ func (x *RemoveRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -2284,14 +2544,25 @@ func (x *RemoveResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -2532,29 +2803,63 @@ func (x *MoveRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"containerId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ContainerId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" + out.RawString(prefix) + if x.ContainerId != nil { + out.Base64Bytes(x.ContainerId) + } else { + out.String("") + } } { - const prefix string = ",\"treeId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"treeId\":" out.RawString(prefix) out.String(x.TreeId) } { - const prefix string = ",\"parentId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"parentId\":" out.RawString(prefix) - out.Uint64(x.ParentId) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.ParentId, 10) + out.RawByte('"') } { - const prefix string = ",\"nodeId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"nodeId\":" out.RawString(prefix) - out.Uint64(x.NodeId) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.NodeId, 10) + out.RawByte('"') } { - const prefix string = ",\"meta\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"meta\":" out.RawString(prefix) out.RawByte('[') for i := range x.Meta { @@ -2566,9 +2871,18 @@ func (x *MoveRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawByte(']') } { - const prefix string = ",\"bearerToken\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"bearerToken\":" out.RawString(prefix) - out.Base64Bytes(x.BearerToken) + if x.BearerToken != nil { + out.Base64Bytes(x.BearerToken) + } else { + out.String("") + } } out.RawByte('}') } @@ -2601,7 +2915,13 @@ func (x *MoveRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "containerId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ContainerId = f } case "treeId": @@ -2613,13 +2933,29 @@ func (x *MoveRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "parentId": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.ParentId = f } case "nodeId": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.NodeId = f } case "meta": @@ -2639,7 +2975,13 @@ func (x *MoveRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "bearerToken": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.BearerToken = f } } @@ -2775,14 +3117,25 @@ func (x *MoveRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -3056,14 +3409,25 @@ func (x *MoveResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -3338,24 +3702,49 @@ func (x *GetNodeByPathRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"containerId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ContainerId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" + out.RawString(prefix) + if x.ContainerId != nil { + out.Base64Bytes(x.ContainerId) + } else { + out.String("") + } } { - const prefix string = ",\"treeId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"treeId\":" out.RawString(prefix) out.String(x.TreeId) } { - const prefix string = ",\"pathAttribute\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"pathAttribute\":" out.RawString(prefix) out.String(x.PathAttribute) } { - const prefix string = ",\"path\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"path\":" out.RawString(prefix) out.RawByte('[') for i := range x.Path { @@ -3367,7 +3756,12 @@ func (x *GetNodeByPathRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawByte(']') } { - const prefix string = ",\"attributes\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"attributes\":" out.RawString(prefix) out.RawByte('[') for i := range x.Attributes { @@ -3379,19 +3773,38 @@ func (x *GetNodeByPathRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawByte(']') } { - const prefix string = ",\"latestOnly\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"latestOnly\":" out.RawString(prefix) out.Bool(x.LatestOnly) } { - const prefix string = ",\"allAttributes\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"allAttributes\":" out.RawString(prefix) out.Bool(x.AllAttributes) } { - const prefix string = ",\"bearerToken\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"bearerToken\":" out.RawString(prefix) - out.Base64Bytes(x.BearerToken) + if x.BearerToken != nil { + out.Base64Bytes(x.BearerToken) + } else { + out.String("") + } } out.RawByte('}') } @@ -3424,7 +3837,13 @@ func (x *GetNodeByPathRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "containerId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ContainerId = f } case "treeId": @@ -3480,7 +3899,13 @@ func (x *GetNodeByPathRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "bearerToken": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.BearerToken = f } } @@ -3616,14 +4041,25 @@ func (x *GetNodeByPathRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -3824,19 +4260,39 @@ func (x *GetNodeByPathResponse_Info) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"nodeId\":" - out.RawString(prefix[1:]) - out.Uint64(x.NodeId) - } - { - const prefix string = ",\"timestamp\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"nodeId\":" out.RawString(prefix) - out.Uint64(x.Timestamp) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.NodeId, 10) + out.RawByte('"') } { - const prefix string = ",\"meta\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"timestamp\":" + out.RawString(prefix) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.Timestamp, 10) + out.RawByte('"') + } + { + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"meta\":" out.RawString(prefix) out.RawByte('[') for i := range x.Meta { @@ -3848,9 +4304,16 @@ func (x *GetNodeByPathResponse_Info) MarshalEasyJSON(out *jwriter.Writer) { out.RawByte(']') } { - const prefix string = ",\"parentId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"parentId\":" out.RawString(prefix) - out.Uint64(x.ParentId) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.ParentId, 10) + out.RawByte('"') } out.RawByte('}') } @@ -3883,13 +4346,29 @@ func (x *GetNodeByPathResponse_Info) UnmarshalEasyJSON(in *jlexer.Lexer) { case "nodeId": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.NodeId = f } case "timestamp": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.Timestamp = f } case "meta": @@ -3909,7 +4388,15 @@ func (x *GetNodeByPathResponse_Info) UnmarshalEasyJSON(in *jlexer.Lexer) { case "parentId": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.ParentId = f } } @@ -4007,10 +4494,16 @@ func (x *GetNodeByPathResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"nodes\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"nodes\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Nodes { if i != 0 { @@ -4195,14 +4688,25 @@ func (x *GetNodeByPathResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -4369,11 +4873,22 @@ func (x *GetSubTreeRequest_Body_Order) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"direction\":" - out.RawString(prefix[1:]) - out.Int32(int32(x.Direction)) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"direction\":" + out.RawString(prefix) + v := int32(x.Direction) + if vv, ok := GetSubTreeRequest_Body_Order_Direction_name[v]; ok { + out.String(vv) + } else { + out.Int32(v) + } } out.RawByte('}') } @@ -4619,41 +5134,82 @@ func (x *GetSubTreeRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"containerId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ContainerId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" + out.RawString(prefix) + if x.ContainerId != nil { + out.Base64Bytes(x.ContainerId) + } else { + out.String("") + } } { - const prefix string = ",\"treeId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"treeId\":" out.RawString(prefix) out.String(x.TreeId) } { - const prefix string = ",\"rootId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"rootId\":" out.RawString(prefix) out.RawByte('[') for i := range x.RootId { if i != 0 { out.RawByte(',') } - out.Uint64(x.RootId[i]) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.RootId[i], 10) + out.RawByte('"') } out.RawByte(']') } { - const prefix string = ",\"depth\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"depth\":" out.RawString(prefix) out.Uint32(x.Depth) } { - const prefix string = ",\"bearerToken\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"bearerToken\":" out.RawString(prefix) - out.Base64Bytes(x.BearerToken) + if x.BearerToken != nil { + out.Base64Bytes(x.BearerToken) + } else { + out.String("") + } } { - const prefix string = ",\"orderBy\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"orderBy\":" out.RawString(prefix) x.OrderBy.MarshalEasyJSON(out) } @@ -4688,7 +5244,13 @@ func (x *GetSubTreeRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "containerId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ContainerId = f } case "treeId": @@ -4703,7 +5265,15 @@ func (x *GetSubTreeRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list []uint64 in.Delim('[') for !in.IsDelim(']') { - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv list = append(list, f) in.WantComma() } @@ -4713,13 +5283,27 @@ func (x *GetSubTreeRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "depth": { var f uint32 - f = in.Uint32() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 32) + if err != nil { + in.AddError(err) + return + } + pv := uint32(v) + f = pv x.Depth = f } case "bearerToken": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.BearerToken = f } case "orderBy": @@ -4862,14 +5446,25 @@ func (x *GetSubTreeRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -5076,45 +5671,72 @@ func (x *GetSubTreeResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"nodeId\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"nodeId\":" + out.RawString(prefix) out.RawByte('[') for i := range x.NodeId { if i != 0 { out.RawByte(',') } - out.Uint64(x.NodeId[i]) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.NodeId[i], 10) + out.RawByte('"') } out.RawByte(']') } { - const prefix string = ",\"parentId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"parentId\":" out.RawString(prefix) out.RawByte('[') for i := range x.ParentId { if i != 0 { out.RawByte(',') } - out.Uint64(x.ParentId[i]) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.ParentId[i], 10) + out.RawByte('"') } out.RawByte(']') } { - const prefix string = ",\"timestamp\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"timestamp\":" out.RawString(prefix) out.RawByte('[') for i := range x.Timestamp { if i != 0 { out.RawByte(',') } - out.Uint64(x.Timestamp[i]) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.Timestamp[i], 10) + out.RawByte('"') } out.RawByte(']') } { - const prefix string = ",\"meta\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"meta\":" out.RawString(prefix) out.RawByte('[') for i := range x.Meta { @@ -5159,7 +5781,15 @@ func (x *GetSubTreeResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list []uint64 in.Delim('[') for !in.IsDelim(']') { - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv list = append(list, f) in.WantComma() } @@ -5172,7 +5802,15 @@ func (x *GetSubTreeResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list []uint64 in.Delim('[') for !in.IsDelim(']') { - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv list = append(list, f) in.WantComma() } @@ -5185,7 +5823,15 @@ func (x *GetSubTreeResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { var list []uint64 in.Delim('[') for !in.IsDelim(']') { - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv list = append(list, f) in.WantComma() } @@ -5339,14 +5985,25 @@ func (x *GetSubTreeResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -5481,11 +6138,21 @@ func (x *TreeListRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"containerId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ContainerId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" + out.RawString(prefix) + if x.ContainerId != nil { + out.Base64Bytes(x.ContainerId) + } else { + out.String("") + } } out.RawByte('}') } @@ -5518,7 +6185,13 @@ func (x *TreeListRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "containerId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ContainerId = f } } @@ -5654,14 +6327,25 @@ func (x *TreeListRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -5796,10 +6480,16 @@ func (x *TreeListResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"ids\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"ids\":" + out.RawString(prefix) out.RawByte('[') for i := range x.Ids { if i != 0 { @@ -5983,14 +6673,25 @@ func (x *TreeListResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -6168,19 +6869,39 @@ func (x *ApplyRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"containerId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ContainerId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" + out.RawString(prefix) + if x.ContainerId != nil { + out.Base64Bytes(x.ContainerId) + } else { + out.String("") + } } { - const prefix string = ",\"treeId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"treeId\":" out.RawString(prefix) out.String(x.TreeId) } { - const prefix string = ",\"operation\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"operation\":" out.RawString(prefix) x.Operation.MarshalEasyJSON(out) } @@ -6215,7 +6936,13 @@ func (x *ApplyRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "containerId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ContainerId = f } case "treeId": @@ -6364,14 +7091,25 @@ func (x *ApplyRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -6645,14 +7383,25 @@ func (x *ApplyResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -6847,26 +7596,55 @@ func (x *GetOpLogRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"containerId\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.ContainerId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" + out.RawString(prefix) + if x.ContainerId != nil { + out.Base64Bytes(x.ContainerId) + } else { + out.String("") + } } { - const prefix string = ",\"treeId\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"treeId\":" out.RawString(prefix) out.String(x.TreeId) } { - const prefix string = ",\"height\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"height\":" out.RawString(prefix) - out.Uint64(x.Height) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.Height, 10) + out.RawByte('"') } { - const prefix string = ",\"count\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"count\":" out.RawString(prefix) - out.Uint64(x.Count) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.Count, 10) + out.RawByte('"') } out.RawByte('}') } @@ -6899,7 +7677,13 @@ func (x *GetOpLogRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "containerId": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.ContainerId = f } case "treeId": @@ -6911,13 +7695,29 @@ func (x *GetOpLogRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { case "height": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.Height = f } case "count": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.Count = f } } @@ -7053,14 +7853,25 @@ func (x *GetOpLogRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -7198,10 +8009,16 @@ func (x *GetOpLogResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"operation\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"operation\":" + out.RawString(prefix) x.Operation.MarshalEasyJSON(out) } out.RawByte('}') @@ -7372,14 +8189,25 @@ func (x *GetOpLogResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -7653,14 +8481,25 @@ func (x *HealthcheckResponse) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } @@ -7934,14 +8773,25 @@ func (x *HealthcheckRequest) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"body\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) x.Body.MarshalEasyJSON(out) } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) x.Signature.MarshalEasyJSON(out) } diff --git a/pkg/services/tree/types_frostfs.pb.go b/pkg/services/tree/types_frostfs.pb.go index 13f1a43be..2827b10a9 100644 --- a/pkg/services/tree/types_frostfs.pb.go +++ b/pkg/services/tree/types_frostfs.pb.go @@ -11,6 +11,7 @@ import ( easyproto "github.com/VictoriaMetrics/easyproto" jlexer "github.com/mailru/easyjson/jlexer" jwriter "github.com/mailru/easyjson/jwriter" + strconv "strconv" ) type KeyValue struct { @@ -113,16 +114,31 @@ func (x *KeyValue) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"key\":" - out.RawString(prefix[1:]) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"key\":" + out.RawString(prefix) out.String(x.Key) } { - const prefix string = ",\"value\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"value\":" out.RawString(prefix) - out.Base64Bytes(x.Value) + if x.Value != nil { + out.Base64Bytes(x.Value) + } else { + out.String("") + } } out.RawByte('}') } @@ -161,7 +177,13 @@ func (x *KeyValue) UnmarshalEasyJSON(in *jlexer.Lexer) { case "value": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Value = f } } @@ -293,21 +315,45 @@ func (x *LogMove) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"parentID\":" - out.RawString(prefix[1:]) - out.Uint64(x.ParentId) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"parentID\":" + out.RawString(prefix) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.ParentId, 10) + out.RawByte('"') } { - const prefix string = ",\"meta\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"meta\":" out.RawString(prefix) - out.Base64Bytes(x.Meta) + if x.Meta != nil { + out.Base64Bytes(x.Meta) + } else { + out.String("") + } } { - const prefix string = ",\"childID\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"childID\":" out.RawString(prefix) - out.Uint64(x.ChildId) + out.RawByte('"') + out.Buffer.Buf = strconv.AppendUint(out.Buffer.Buf, x.ChildId, 10) + out.RawByte('"') } out.RawByte('}') } @@ -340,19 +386,41 @@ func (x *LogMove) UnmarshalEasyJSON(in *jlexer.Lexer) { case "parentID": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.ParentId = f } case "meta": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Meta = f } case "childID": { var f uint64 - f = in.Uint64() + r := in.JsonNumber() + n := r.String() + v, err := strconv.ParseUint(n, 10, 64) + if err != nil { + in.AddError(err) + return + } + pv := uint64(v) + f = pv x.ChildId = f } } @@ -464,16 +532,35 @@ func (x *Signature) MarshalEasyJSON(out *jwriter.Writer) { out.RawString("null") return } + first := true out.RawByte('{') { - const prefix string = ",\"key\":" - out.RawString(prefix[1:]) - out.Base64Bytes(x.Key) + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"key\":" + out.RawString(prefix) + if x.Key != nil { + out.Base64Bytes(x.Key) + } else { + out.String("") + } } { - const prefix string = ",\"signature\":" + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" out.RawString(prefix) - out.Base64Bytes(x.Sign) + if x.Sign != nil { + out.Base64Bytes(x.Sign) + } else { + out.String("") + } } out.RawByte('}') } @@ -506,13 +593,25 @@ func (x *Signature) UnmarshalEasyJSON(in *jlexer.Lexer) { case "key": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Key = f } case "signature": { var f []byte - f = in.Bytes() + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } x.Sign = f } } From b1a31281e4901e73d93a5690ccb9ac6ae3e2aa4f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 8 Nov 2024 09:51:14 +0300 Subject: [PATCH 1505/1943] [#1480] ape: Remove SoftAPECheck flag Previous release was EACL-compatible. Starting from now all EACL should've been migrated to APE chains. Signed-off-by: Evgenii Stratonikov --- pkg/services/common/ape/checker.go | 5 +- pkg/services/object/acl/acl.go | 48 ------------ pkg/services/object/acl/acl_test.go | 89 ----------------------- pkg/services/object/acl/v2/errors.go | 7 -- pkg/services/object/acl/v2/errors_test.go | 10 --- pkg/services/object/acl/v2/request.go | 7 -- pkg/services/object/acl/v2/service.go | 67 ----------------- pkg/services/object/acl/v2/types.go | 11 --- pkg/services/object/ape/checker.go | 4 - pkg/services/object/ape/service.go | 12 --- pkg/services/object/request_context.go | 2 - pkg/services/tree/ape.go | 1 - 12 files changed, 1 insertion(+), 262 deletions(-) delete mode 100644 pkg/services/object/acl/acl_test.go diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index 278f6da31..eb4fd03c7 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -44,9 +44,6 @@ type CheckPrm struct { // The request's bearer token. It is used in order to check APE overrides with the token. BearerToken *bearer.Token - - // If SoftAPECheck is set to true, then NoRuleFound is interpreted as allow. - SoftAPECheck bool } // CheckCore provides methods to perform the common logic of APE check. @@ -104,7 +101,7 @@ func (c *checkerCoreImpl) CheckAPE(prm CheckPrm) error { if err != nil { return err } - if !found && prm.SoftAPECheck || status == apechain.Allow { + if found && status == apechain.Allow { return nil } err = fmt.Errorf("access to operation %s is denied by access policy engine: %s", prm.Request.Operation(), status.String()) diff --git a/pkg/services/object/acl/acl.go b/pkg/services/object/acl/acl.go index 921545c8b..53ba652e1 100644 --- a/pkg/services/object/acl/acl.go +++ b/pkg/services/object/acl/acl.go @@ -3,7 +3,6 @@ package acl import ( "context" "crypto/ecdsa" - "crypto/elliptic" "errors" "fmt" "io" @@ -22,7 +21,6 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" ) // Checker implements v2.ACLChecker interfaces and provides @@ -72,33 +70,6 @@ func NewChecker( } } -// CheckBasicACL is a main check function for basic ACL. -func (c *Checker) CheckBasicACL(info v2.RequestInfo) bool { - // check basic ACL permissions - return info.BasicACL().IsOpAllowed(info.Operation(), info.RequestRole()) -} - -// StickyBitCheck validates owner field in the request if sticky bit is enabled. -func (c *Checker) StickyBitCheck(info v2.RequestInfo, owner user.ID) bool { - // According to FrostFS specification sticky bit has no effect on system nodes - // for correct intra-container work with objects (in particular, replication). - if info.RequestRole() == acl.RoleContainer { - return true - } - - if !info.BasicACL().Sticky() { - return true - } - - if len(info.SenderKey()) == 0 { - return false - } - - requestSenderKey := unmarshalPublicKey(info.SenderKey()) - - return isOwnerFromKey(owner, requestSenderKey) -} - // CheckEACL is a main check function for extended ACL. func (c *Checker) CheckEACL(msg any, reqInfo v2.RequestInfo) error { basicACL := reqInfo.BasicACL() @@ -241,22 +212,3 @@ func isValidBearer(reqInfo v2.RequestInfo, st netmap.State) error { return nil } - -func isOwnerFromKey(id user.ID, key *keys.PublicKey) bool { - if key == nil { - return false - } - - var id2 user.ID - user.IDFromKey(&id2, (ecdsa.PublicKey)(*key)) - - return id.Equals(id2) -} - -func unmarshalPublicKey(bs []byte) *keys.PublicKey { - pub, err := keys.NewPublicKeyFromBytes(bs, elliptic.P256()) - if err != nil { - return nil - } - return pub -} diff --git a/pkg/services/object/acl/acl_test.go b/pkg/services/object/acl/acl_test.go deleted file mode 100644 index d63cb1285..000000000 --- a/pkg/services/object/acl/acl_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package acl - -import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" - "github.com/stretchr/testify/require" -) - -type emptyEACLSource struct{} - -func (e emptyEACLSource) GetEACL(_ cid.ID) (*container.EACL, error) { - return nil, nil -} - -type emptyNetmapState struct{} - -func (e emptyNetmapState) CurrentEpoch() uint64 { - return 0 -} - -func TestStickyCheck(t *testing.T) { - checker := NewChecker( - emptyNetmapState{}, - emptyEACLSource{}, - eaclSDK.NewValidator(), - &engine.StorageEngine{}) - - t.Run("system role", func(t *testing.T) { - var info v2.RequestInfo - - info.SetSenderKey(make([]byte, 33)) // any non-empty key - info.SetRequestRole(acl.RoleContainer) - - require.True(t, checker.StickyBitCheck(info, usertest.ID())) - - var basicACL acl.Basic - basicACL.MakeSticky() - - info.SetBasicACL(basicACL) - - require.True(t, checker.StickyBitCheck(info, usertest.ID())) - }) - - t.Run("owner ID and/or public key emptiness", func(t *testing.T) { - var info v2.RequestInfo - - info.SetRequestRole(acl.RoleOthers) // should be non-system role - - assertFn := func(isSticky, withKey, withOwner, expected bool) { - info := info - if isSticky { - var basicACL acl.Basic - basicACL.MakeSticky() - - info.SetBasicACL(basicACL) - } - - if withKey { - info.SetSenderKey(make([]byte, 33)) - } else { - info.SetSenderKey(nil) - } - - var ownerID user.ID - - if withOwner { - ownerID = usertest.ID() - } - - require.Equal(t, expected, checker.StickyBitCheck(info, ownerID)) - } - - assertFn(true, false, false, false) - assertFn(true, true, false, false) - assertFn(true, false, true, false) - assertFn(false, false, false, true) - assertFn(false, true, false, true) - assertFn(false, false, true, true) - assertFn(false, true, true, true) - }) -} diff --git a/pkg/services/object/acl/v2/errors.go b/pkg/services/object/acl/v2/errors.go index 11b9e6e5f..e969d37fa 100644 --- a/pkg/services/object/acl/v2/errors.go +++ b/pkg/services/object/acl/v2/errors.go @@ -26,13 +26,6 @@ const ( accessDeniedEACLReasonFmt = "access to operation %s is denied by extended ACL check: %v" ) -func basicACLErr(info RequestInfo) error { - errAccessDenied := &apistatus.ObjectAccessDenied{} - errAccessDenied.WriteReason(fmt.Sprintf(accessDeniedACLReasonFmt, info.operation)) - - return errAccessDenied -} - func eACLErr(info RequestInfo, err error) error { errAccessDenied := &apistatus.ObjectAccessDenied{} errAccessDenied.WriteReason(fmt.Sprintf(accessDeniedEACLReasonFmt, info.operation, err)) diff --git a/pkg/services/object/acl/v2/errors_test.go b/pkg/services/object/acl/v2/errors_test.go index 2d2b7bc8d..3cc74e6aa 100644 --- a/pkg/services/object/acl/v2/errors_test.go +++ b/pkg/services/object/acl/v2/errors_test.go @@ -8,16 +8,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestBasicACLErr(t *testing.T) { - var reqInfo RequestInfo - err := basicACLErr(reqInfo) - - var errAccessDenied *apistatus.ObjectAccessDenied - - require.ErrorAs(t, err, &errAccessDenied, - "basicACLErr must be able to be casted to apistatus.ObjectAccessDenied") -} - func TestEACLErr(t *testing.T) { var reqInfo RequestInfo testErr := errors.New("test-eacl") diff --git a/pkg/services/object/acl/v2/request.go b/pkg/services/object/acl/v2/request.go index e35cd2e11..8bd34ccb3 100644 --- a/pkg/services/object/acl/v2/request.go +++ b/pkg/services/object/acl/v2/request.go @@ -104,13 +104,6 @@ func (r RequestInfo) RequestRole() acl.Role { return r.requestRole } -// IsSoftAPECheck states if APE should perform soft checks. -// Soft APE check allows a request if CheckAPE returns NoRuleFound for it, -// otherwise it denies the request. -func (r RequestInfo) IsSoftAPECheck() bool { - return r.BasicACL().Bits() != 0 -} - // MetaWithToken groups session and bearer tokens, // verification header and raw API request. type MetaWithToken struct { diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index e02a3be36..9f5ac5a27 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -123,7 +123,6 @@ func (w *wrappedGetObjectStream) Context() context.Context { ContainerOwner: w.requestInfo.ContainerOwner(), SenderKey: w.requestInfo.SenderKey(), Role: w.requestInfo.RequestRole(), - SoftAPECheck: w.requestInfo.IsSoftAPECheck(), BearerToken: w.requestInfo.Bearer(), }) } @@ -149,7 +148,6 @@ func (w *wrappedRangeStream) Context() context.Context { ContainerOwner: w.requestInfo.ContainerOwner(), SenderKey: w.requestInfo.SenderKey(), Role: w.requestInfo.RequestRole(), - SoftAPECheck: w.requestInfo.IsSoftAPECheck(), BearerToken: w.requestInfo.Bearer(), }) } @@ -175,7 +173,6 @@ func (w *wrappedSearchStream) Context() context.Context { ContainerOwner: w.requestInfo.ContainerOwner(), SenderKey: w.requestInfo.SenderKey(), Role: w.requestInfo.RequestRole(), - SoftAPECheck: w.requestInfo.IsSoftAPECheck(), BearerToken: w.requestInfo.Bearer(), }) } @@ -231,14 +228,6 @@ func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream reqInfo.obj = obj - if reqInfo.IsSoftAPECheck() { - if !b.checker.CheckBasicACL(reqInfo) { - return basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return eACLErr(reqInfo, err) - } - } - return b.next.Get(request, &getStreamBasicChecker{ GetObjectStream: newWrappedGetObjectStreamStream(stream, reqInfo), info: reqInfo, @@ -309,14 +298,6 @@ func (b Service) Head( reqInfo.obj = obj - if reqInfo.IsSoftAPECheck() { - if !b.checker.CheckBasicACL(reqInfo) { - return nil, basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return nil, eACLErr(reqInfo, err) - } - } - resp, err := b.next.Head(requestContext(ctx, reqInfo), request) if err == nil { if err = b.checker.CheckEACL(resp, reqInfo); err != nil { @@ -362,14 +343,6 @@ func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStr return err } - if reqInfo.IsSoftAPECheck() { - if !b.checker.CheckBasicACL(reqInfo) { - return basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return eACLErr(reqInfo, err) - } - } - return b.next.Search(request, &searchStreamBasicChecker{ checker: b.checker, SearchStream: newWrappedSearchStream(stream, reqInfo), @@ -422,14 +395,6 @@ func (b Service) Delete( reqInfo.obj = obj - if reqInfo.IsSoftAPECheck() { - if !b.checker.CheckBasicACL(reqInfo) { - return nil, basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return nil, eACLErr(reqInfo, err) - } - } - return b.next.Delete(requestContext(ctx, reqInfo), request) } @@ -475,14 +440,6 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb reqInfo.obj = obj - if reqInfo.IsSoftAPECheck() { - if !b.checker.CheckBasicACL(reqInfo) { - return basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return eACLErr(reqInfo, err) - } - } - return b.next.GetRange(request, &rangeStreamBasicChecker{ checker: b.checker, GetObjectRangeStream: newWrappedRangeStream(stream, reqInfo), @@ -496,7 +453,6 @@ func requestContext(ctx context.Context, reqInfo RequestInfo) context.Context { ContainerOwner: reqInfo.ContainerOwner(), SenderKey: reqInfo.SenderKey(), Role: reqInfo.RequestRole(), - SoftAPECheck: reqInfo.IsSoftAPECheck(), BearerToken: reqInfo.Bearer(), }) } @@ -546,14 +502,6 @@ func (b Service) GetRangeHash( reqInfo.obj = obj - if reqInfo.IsSoftAPECheck() { - if !b.checker.CheckBasicACL(reqInfo) { - return nil, basicACLErr(reqInfo) - } else if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return nil, eACLErr(reqInfo, err) - } - } - return b.next.GetRangeHash(requestContext(ctx, reqInfo), request) } @@ -605,15 +553,6 @@ func (b Service) PutSingle(ctx context.Context, request *objectV2.PutSingleReque reqInfo.obj = obj - if reqInfo.IsSoftAPECheck() { - if !b.checker.CheckBasicACL(reqInfo) || !b.checker.StickyBitCheck(reqInfo, idOwner) { - return nil, basicACLErr(reqInfo) - } - if err := b.checker.CheckEACL(request, reqInfo); err != nil { - return nil, eACLErr(reqInfo, err) - } - } - return b.next.PutSingle(requestContext(ctx, reqInfo), request) } @@ -679,12 +618,6 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe reqInfo.obj = obj - if reqInfo.IsSoftAPECheck() { - if !p.source.checker.CheckBasicACL(reqInfo) || !p.source.checker.StickyBitCheck(reqInfo, idOwner) { - return basicACLErr(reqInfo) - } - } - ctx = requestContext(ctx, reqInfo) } diff --git a/pkg/services/object/acl/v2/types.go b/pkg/services/object/acl/v2/types.go index 061cd26b6..6ae80e9c2 100644 --- a/pkg/services/object/acl/v2/types.go +++ b/pkg/services/object/acl/v2/types.go @@ -1,22 +1,11 @@ package v2 -import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" -) - // ACLChecker is an interface that must provide // ACL related checks. type ACLChecker interface { - // CheckBasicACL must return true only if request - // passes basic ACL validation. - CheckBasicACL(RequestInfo) bool // CheckEACL must return non-nil error if request // doesn't pass extended ACL validation. CheckEACL(any, RequestInfo) error - // StickyBitCheck must return true only if sticky bit - // is disabled or enabled but request contains correct - // owner field. - StickyBitCheck(RequestInfo, user.ID) bool } // InnerRingFetcher is an interface that must provide diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index abcd2f4bb..4a3b5ba5e 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -64,9 +64,6 @@ type Prm struct { // An encoded container's owner user ID. ContainerOwner user.ID - // If SoftAPECheck is set to true, then NoRuleFound is interpreted as allow. - SoftAPECheck bool - // The request's bearer token. It is used in order to check APE overrides with the token. BearerToken *bearer.Token @@ -109,6 +106,5 @@ func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { Container: prm.Container, ContainerOwner: prm.ContainerOwner, BearerToken: prm.BearerToken, - SoftAPECheck: prm.SoftAPECheck, }) } diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index c114f02f6..558c48da8 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -84,8 +84,6 @@ type getStreamBasicChecker struct { role string - softAPECheck bool - bearerToken *bearer.Token } @@ -105,7 +103,6 @@ func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { SenderKey: hex.EncodeToString(g.senderKey), ContainerOwner: g.containerOwner, Role: g.role, - SoftAPECheck: g.softAPECheck, BearerToken: g.bearerToken, XHeaders: resp.GetMetaHeader().GetXHeaders(), } @@ -142,7 +139,6 @@ func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectSt senderKey: reqCtx.SenderKey, containerOwner: reqCtx.ContainerOwner, role: nativeSchemaRole(reqCtx.Role), - softAPECheck: reqCtx.SoftAPECheck, bearerToken: reqCtx.BearerToken, }) } @@ -174,7 +170,6 @@ func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutR SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, Role: nativeSchemaRole(reqCtx.Role), - SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, XHeaders: request.GetMetaHeader().GetXHeaders(), } @@ -230,7 +225,6 @@ func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.Pa SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, Role: nativeSchemaRole(reqCtx.Role), - SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, XHeaders: request.GetMetaHeader().GetXHeaders(), } @@ -300,7 +294,6 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj Role: nativeSchemaRole(reqCtx.Role), SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, - SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, XHeaders: request.GetMetaHeader().GetXHeaders(), }) @@ -330,7 +323,6 @@ func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.Searc Role: nativeSchemaRole(reqCtx.Role), SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, - SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, XHeaders: request.GetMetaHeader().GetXHeaders(), }) @@ -360,7 +352,6 @@ func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) ( Role: nativeSchemaRole(reqCtx.Role), SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, - SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, XHeaders: request.GetMetaHeader().GetXHeaders(), }) @@ -395,7 +386,6 @@ func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.G Role: nativeSchemaRole(reqCtx.Role), SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, - SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, XHeaders: request.GetMetaHeader().GetXHeaders(), }) @@ -425,7 +415,6 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa Role: nativeSchemaRole(reqCtx.Role), SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, - SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, XHeaders: request.GetMetaHeader().GetXHeaders(), } @@ -461,7 +450,6 @@ func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequ Role: nativeSchemaRole(reqCtx.Role), SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, - SoftAPECheck: reqCtx.SoftAPECheck, BearerToken: reqCtx.BearerToken, XHeaders: request.GetMetaHeader().GetXHeaders(), } diff --git a/pkg/services/object/request_context.go b/pkg/services/object/request_context.go index 95d4c9d93..eb4041f80 100644 --- a/pkg/services/object/request_context.go +++ b/pkg/services/object/request_context.go @@ -20,7 +20,5 @@ type RequestContext struct { Role acl.Role - SoftAPECheck bool - BearerToken *bearer.Token } diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 69cf59405..606044f8e 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -81,7 +81,6 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, ContainerOwner: container.Value.Owner(), PublicKey: publicKey, BearerToken: bt, - SoftAPECheck: false, }) } From f666898e5d19e34a571848ecd0a78dbbea3a1b34 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 8 Nov 2024 10:05:37 +0300 Subject: [PATCH 1506/1943] [#1480] objsvc: Remove EACL checks Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/object.go | 9 - pkg/services/object/acl/acl.go | 214 ---------------------- pkg/services/object/acl/v2/errors.go | 14 -- pkg/services/object/acl/v2/errors_test.go | 20 -- pkg/services/object/acl/v2/service.go | 45 +---- pkg/services/object/acl/v2/types.go | 8 - 6 files changed, 1 insertion(+), 309 deletions(-) delete mode 100644 pkg/services/object/acl/acl.go delete mode 100644 pkg/services/object/acl/v2/errors_test.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 7f26393a7..629f79207 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -19,7 +19,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/cache" objectTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/object/grpc" objectService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl" v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" objectAPE "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/ape" objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" @@ -39,7 +38,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -458,17 +456,10 @@ func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Servi } func createACLServiceV2(c *cfg, apeSvc *objectAPE.Service, irFetcher *cachedIRFetcher) v2.Service { - ls := c.cfgObject.cfgLocalStorage.localStorage - return v2.New( apeSvc, c.netMapSource, irFetcher, - acl.NewChecker( - c.cfgNetmap.state, - c.cfgObject.eaclSource, - eaclSDK.NewValidator(), - ls), c.cfgObject.cnrSource, v2.WithLogger(c.log), ) diff --git a/pkg/services/object/acl/acl.go b/pkg/services/object/acl/acl.go deleted file mode 100644 index 53ba652e1..000000000 --- a/pkg/services/object/acl/acl.go +++ /dev/null @@ -1,214 +0,0 @@ -package acl - -import ( - "context" - "crypto/ecdsa" - "errors" - "fmt" - "io" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - eaclV2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/eacl/v2" - v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" - bearerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" - eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" -) - -// Checker implements v2.ACLChecker interfaces and provides -// ACL/eACL validation functionality. -type Checker struct { - eaclSrc container.EACLSource - validator *eaclSDK.Validator - localStorage *engine.StorageEngine - state netmap.State -} - -type localStorage struct { - ls *engine.StorageEngine -} - -func (s *localStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { - if s.ls == nil { - return nil, io.ErrUnexpectedEOF - } - - return engine.Head(ctx, s.ls, addr) -} - -// Various EACL check errors. -var ( - errEACLDeniedByRule = errors.New("denied by rule") - errBearerExpired = errors.New("bearer token has expired") - errBearerInvalidSignature = errors.New("bearer token has invalid signature") - errBearerInvalidContainerID = errors.New("bearer token was created for another container") - errBearerNotSignedByOwner = errors.New("bearer token is not signed by the container owner") - errBearerInvalidOwner = errors.New("bearer token owner differs from the request sender") -) - -// NewChecker creates Checker. -// Panics if at least one of the parameter is nil. -func NewChecker( - state netmap.State, - eaclSrc container.EACLSource, - validator *eaclSDK.Validator, - localStorage *engine.StorageEngine, -) *Checker { - return &Checker{ - eaclSrc: eaclSrc, - validator: validator, - localStorage: localStorage, - state: state, - } -} - -// CheckEACL is a main check function for extended ACL. -func (c *Checker) CheckEACL(msg any, reqInfo v2.RequestInfo) error { - basicACL := reqInfo.BasicACL() - if !basicACL.Extendable() { - return nil - } - - bearerTok := reqInfo.Bearer() - impersonate := bearerTok != nil && bearerTok.Impersonate() - - // if bearer token is not allowed, then ignore it - if impersonate || !basicACL.AllowedBearerRules(reqInfo.Operation()) { - reqInfo.CleanBearer() - } - - var table eaclSDK.Table - cnr := reqInfo.ContainerID() - - if bearerTok == nil { - eaclInfo, err := c.eaclSrc.GetEACL(cnr) - if err != nil { - if client.IsErrEACLNotFound(err) { - return nil - } - return err - } - - table = *eaclInfo.Value - } else { - table = bearerTok.EACLTable() - } - - // if bearer token is not present, isValidBearer returns true - if err := isValidBearer(reqInfo, c.state); err != nil { - return err - } - - hdrSrc, err := c.getHeaderSource(cnr, msg, reqInfo) - if err != nil { - return err - } - - eaclRole := getRole(reqInfo) - - action, _ := c.validator.CalculateAction(new(eaclSDK.ValidationUnit). - WithRole(eaclRole). - WithOperation(eaclSDK.Operation(reqInfo.Operation())). - WithContainerID(&cnr). - WithSenderKey(reqInfo.SenderKey()). - WithHeaderSource(hdrSrc). - WithEACLTable(&table), - ) - - if action != eaclSDK.ActionAllow { - return errEACLDeniedByRule - } - return nil -} - -func getRole(reqInfo v2.RequestInfo) eaclSDK.Role { - var eaclRole eaclSDK.Role - switch op := reqInfo.RequestRole(); op { - default: - eaclRole = eaclSDK.Role(op) - case acl.RoleOwner: - eaclRole = eaclSDK.RoleUser - case acl.RoleInnerRing, acl.RoleContainer: - eaclRole = eaclSDK.RoleSystem - case acl.RoleOthers: - eaclRole = eaclSDK.RoleOthers - } - return eaclRole -} - -func (c *Checker) getHeaderSource(cnr cid.ID, msg any, reqInfo v2.RequestInfo) (eaclSDK.TypedHeaderSource, error) { - var xHeaderSource eaclV2.XHeaderSource - if req, ok := msg.(eaclV2.Request); ok { - xHeaderSource = eaclV2.NewRequestXHeaderSource(req) - } else { - xHeaderSource = eaclV2.NewResponseXHeaderSource(msg.(eaclV2.Response), reqInfo.Request().(eaclV2.Request)) - } - - hdrSrc, err := eaclV2.NewMessageHeaderSource(&localStorage{ls: c.localStorage}, xHeaderSource, cnr, eaclV2.WithOID(reqInfo.ObjectID())) - if err != nil { - return nil, fmt.Errorf("can't parse headers: %w", err) - } - return hdrSrc, nil -} - -// isValidBearer checks whether bearer token was correctly signed by authorized -// entity. This method might be defined on whole ACL service because it will -// require fetching current epoch to check lifetime. -func isValidBearer(reqInfo v2.RequestInfo, st netmap.State) error { - ownerCnr := reqInfo.ContainerOwner() - - token := reqInfo.Bearer() - - // 0. Check if bearer token is present in reqInfo. - if token == nil { - return nil - } - - // 1. First check token lifetime. Simplest verification. - if token.InvalidAt(st.CurrentEpoch()) { - return errBearerExpired - } - - // 2. Then check if bearer token is signed correctly. - if !token.VerifySignature() { - return errBearerInvalidSignature - } - - // 3. Then check if container is either empty or equal to the container in the request. - cnr, isSet := token.EACLTable().CID() - if isSet && !cnr.Equals(reqInfo.ContainerID()) { - return errBearerInvalidContainerID - } - - // 4. Then check if container owner signed this token. - if !bearerSDK.ResolveIssuer(*token).Equals(ownerCnr) { - // TODO: #767 in this case we can issue all owner keys from frostfs.id and check once again - return errBearerNotSignedByOwner - } - - // 5. Then check if request sender has rights to use this token. - var keySender frostfsecdsa.PublicKey - - err := keySender.Decode(reqInfo.SenderKey()) - if err != nil { - return fmt.Errorf("decode sender public key: %w", err) - } - - var usrSender user.ID - user.IDFromKey(&usrSender, ecdsa.PublicKey(keySender)) - - if !token.AssertUser(usrSender) { - // TODO: #767 in this case we can issue all owner keys from frostfs.id and check once again - return errBearerInvalidOwner - } - - return nil -} diff --git a/pkg/services/object/acl/v2/errors.go b/pkg/services/object/acl/v2/errors.go index e969d37fa..cd2de174a 100644 --- a/pkg/services/object/acl/v2/errors.go +++ b/pkg/services/object/acl/v2/errors.go @@ -2,8 +2,6 @@ package v2 import ( "fmt" - - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) const invalidRequestMessage = "malformed request" @@ -20,15 +18,3 @@ var ( errInvalidSessionOwner = malformedRequestError("invalid session token owner") errInvalidVerb = malformedRequestError("session token verb is invalid") ) - -const ( - accessDeniedACLReasonFmt = "access to operation %s is denied by basic ACL check" - accessDeniedEACLReasonFmt = "access to operation %s is denied by extended ACL check: %v" -) - -func eACLErr(info RequestInfo, err error) error { - errAccessDenied := &apistatus.ObjectAccessDenied{} - errAccessDenied.WriteReason(fmt.Sprintf(accessDeniedEACLReasonFmt, info.operation, err)) - - return errAccessDenied -} diff --git a/pkg/services/object/acl/v2/errors_test.go b/pkg/services/object/acl/v2/errors_test.go deleted file mode 100644 index 3cc74e6aa..000000000 --- a/pkg/services/object/acl/v2/errors_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package v2 - -import ( - "errors" - "testing" - - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - "github.com/stretchr/testify/require" -) - -func TestEACLErr(t *testing.T) { - var reqInfo RequestInfo - testErr := errors.New("test-eacl") - err := eACLErr(reqInfo, testErr) - - var errAccessDenied *apistatus.ObjectAccessDenied - - require.ErrorAs(t, err, &errAccessDenied, - "eACLErr must be able to be casted to apistatus.ObjectAccessDenied") -} diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 9f5ac5a27..69406e5b0 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -42,27 +42,15 @@ type patchStreamBasicChecker struct { } type getStreamBasicChecker struct { - checker ACLChecker - object.GetObjectStream - - info RequestInfo } type rangeStreamBasicChecker struct { - checker ACLChecker - object.GetObjectRangeStream - - info RequestInfo } type searchStreamBasicChecker struct { - checker ACLChecker - object.SearchStream - - info RequestInfo } // Option represents Service constructor option. @@ -73,8 +61,6 @@ type cfg struct { containers container.Source - checker ACLChecker - irFetcher InnerRingFetcher nm netmap.Source @@ -86,7 +72,6 @@ type cfg struct { func New(next object.ServiceServer, nm netmap.Source, irf InnerRingFetcher, - acl ACLChecker, cs container.Source, opts ...Option, ) Service { @@ -95,7 +80,6 @@ func New(next object.ServiceServer, next: next, nm: nm, irFetcher: irf, - checker: acl, containers: cs, } @@ -230,8 +214,6 @@ func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream return b.next.Get(request, &getStreamBasicChecker{ GetObjectStream: newWrappedGetObjectStreamStream(stream, reqInfo), - info: reqInfo, - checker: b.checker, }) } @@ -298,14 +280,7 @@ func (b Service) Head( reqInfo.obj = obj - resp, err := b.next.Head(requestContext(ctx, reqInfo), request) - if err == nil { - if err = b.checker.CheckEACL(resp, reqInfo); err != nil { - err = eACLErr(reqInfo, err) - } - } - - return resp, err + return b.next.Head(requestContext(ctx, reqInfo), request) } func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStream) error { @@ -344,9 +319,7 @@ func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStr } return b.next.Search(request, &searchStreamBasicChecker{ - checker: b.checker, SearchStream: newWrappedSearchStream(stream, reqInfo), - info: reqInfo, }) } @@ -441,9 +414,7 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb reqInfo.obj = obj return b.next.GetRange(request, &rangeStreamBasicChecker{ - checker: b.checker, GetObjectRangeStream: newWrappedRangeStream(stream, reqInfo), - info: reqInfo, }) } @@ -657,28 +628,14 @@ func (p putStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PutR } func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { - if _, ok := resp.GetBody().GetObjectPart().(*objectV2.GetObjectPartInit); ok { - if err := g.checker.CheckEACL(resp, g.info); err != nil { - return eACLErr(g.info, err) - } - } - return g.GetObjectStream.Send(resp) } func (g *rangeStreamBasicChecker) Send(resp *objectV2.GetRangeResponse) error { - if err := g.checker.CheckEACL(resp, g.info); err != nil { - return eACLErr(g.info, err) - } - return g.GetObjectRangeStream.Send(resp) } func (g *searchStreamBasicChecker) Send(resp *objectV2.SearchResponse) error { - if err := g.checker.CheckEACL(resp, g.info); err != nil { - return eACLErr(g.info, err) - } - return g.SearchStream.Send(resp) } diff --git a/pkg/services/object/acl/v2/types.go b/pkg/services/object/acl/v2/types.go index 6ae80e9c2..b03261b90 100644 --- a/pkg/services/object/acl/v2/types.go +++ b/pkg/services/object/acl/v2/types.go @@ -1,13 +1,5 @@ package v2 -// ACLChecker is an interface that must provide -// ACL related checks. -type ACLChecker interface { - // CheckEACL must return non-nil error if request - // doesn't pass extended ACL validation. - CheckEACL(any, RequestInfo) error -} - // InnerRingFetcher is an interface that must provide // Inner Ring information. type InnerRingFetcher interface { From c82c753e9f67c70dfa99f98a34623fae3dfa5fce Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 8 Nov 2024 10:10:06 +0300 Subject: [PATCH 1507/1943] [#1480] objsvc: Remove useless stream wrappers Signed-off-by: Evgenii Stratonikov --- pkg/services/object/acl/v2/service.go | 36 +++------------------------ 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 69406e5b0..56748b08c 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -41,18 +41,6 @@ type patchStreamBasicChecker struct { nonFirstSend bool } -type getStreamBasicChecker struct { - object.GetObjectStream -} - -type rangeStreamBasicChecker struct { - object.GetObjectRangeStream -} - -type searchStreamBasicChecker struct { - object.SearchStream -} - // Option represents Service constructor option. type Option func(*cfg) @@ -212,9 +200,7 @@ func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream reqInfo.obj = obj - return b.next.Get(request, &getStreamBasicChecker{ - GetObjectStream: newWrappedGetObjectStreamStream(stream, reqInfo), - }) + return b.next.Get(request, newWrappedGetObjectStreamStream(stream, reqInfo)) } func (b Service) Put() (object.PutObjectStream, error) { @@ -318,9 +304,7 @@ func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStr return err } - return b.next.Search(request, &searchStreamBasicChecker{ - SearchStream: newWrappedSearchStream(stream, reqInfo), - }) + return b.next.Search(request, newWrappedSearchStream(stream, reqInfo)) } func (b Service) Delete( @@ -413,9 +397,7 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb reqInfo.obj = obj - return b.next.GetRange(request, &rangeStreamBasicChecker{ - GetObjectRangeStream: newWrappedRangeStream(stream, reqInfo), - }) + return b.next.GetRange(request, newWrappedRangeStream(stream, reqInfo)) } func requestContext(ctx context.Context, reqInfo RequestInfo) context.Context { @@ -627,18 +609,6 @@ func (p putStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PutR return p.next.CloseAndRecv(ctx) } -func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { - return g.GetObjectStream.Send(resp) -} - -func (g *rangeStreamBasicChecker) Send(resp *objectV2.GetRangeResponse) error { - return g.GetObjectRangeStream.Send(resp) -} - -func (g *searchStreamBasicChecker) Send(resp *objectV2.SearchResponse) error { - return g.SearchStream.Send(resp) -} - func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.PatchRequest) error { body := request.GetBody() if body == nil { From d336f2d487abd5fd102d24d130749a6a6ddeb8c4 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 30 Oct 2024 17:39:02 +0300 Subject: [PATCH 1508/1943] [#1393] adm: Make `NewLocalActor` receive accout name * Some RPC-clients for contracts require different wallet account types. Since, `Policy` contract gets `consensus` accounts while `NNS` gets `committee` accounts. Signed-off-by: Airat Arifullin --- .../internal/modules/morph/ape/ape_util.go | 2 +- .../internal/modules/morph/helper/actor.go | 11 +++++++---- cmd/frostfs-adm/internal/modules/morph/nns/helper.go | 3 ++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index d4aedda2e..f4373c535 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -139,7 +139,7 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *he c, err := helper.GetN3Client(viper.GetViper()) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) - ac, err := helper.NewLocalActor(cmd, c) + ac, err := helper.NewLocalActor(cmd, c, constants.ConsensusAccountName) commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) var ch util.Uint160 diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/actor.go b/cmd/frostfs-adm/internal/modules/morph/helper/actor.go index 1ca246f9f..ff0421335 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/actor.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/actor.go @@ -5,7 +5,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -31,7 +30,11 @@ type LocalActor struct { // NewLocalActor create LocalActor with accounts form provided wallets. // In case of empty wallets provided created actor with dummy account only for read operation. -func NewLocalActor(cmd *cobra.Command, c actor.RPCActor) (*LocalActor, error) { +// +// If wallets are provided, the contract client will use accounts with accName name from these wallets. +// To determine which account name should be used in a contract client, refer to how the contract +// verifies the transaction signature. +func NewLocalActor(cmd *cobra.Command, c actor.RPCActor, accName string) (*LocalActor, error) { walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) var act *actor.Actor var accounts []*wallet.Account @@ -53,8 +56,8 @@ func NewLocalActor(cmd *cobra.Command, c actor.RPCActor) (*LocalActor, error) { commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) for _, w := range wallets { - acc, err := GetWalletAccount(w, constants.CommitteeAccountName) - commonCmd.ExitOnErr(cmd, "can't find committee account: %w", err) + acc, err := GetWalletAccount(w, accName) + commonCmd.ExitOnErr(cmd, fmt.Sprintf("can't find %s account: %%w", accName), err) accounts = append(accounts, acc) } act, err = actor.New(c, []actor.SignerAccount{{ diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go index 0eaa5ac58..de439acd1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go @@ -2,6 +2,7 @@ package nns import ( client "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" @@ -15,7 +16,7 @@ func getRPCClient(cmd *cobra.Command) (*client.Contract, *helper.LocalActor, uti c, err := helper.GetN3Client(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) - ac, err := helper.NewLocalActor(cmd, c) + ac, err := helper.NewLocalActor(cmd, c, constants.CommitteeAccountName) commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) r := management.NewReader(ac.Invoker) From ad01fb958afd2c48e2efba03e5d22acdf8be7849 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Wed, 6 Nov 2024 13:39:46 +0300 Subject: [PATCH 1509/1943] [#1474] Stop using obsolete .github directory This commit is a part of multi-repo cleanup effort: https://git.frostfs.info/TrueCloudLab/frostfs-infra/issues/136 Signed-off-by: Vitaliy Potyarkin --- {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md | 0 {.github => .forgejo}/ISSUE_TEMPLATE/config.yml | 0 {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md | 0 {.github => .forgejo}/logo.svg | 0 README.md | 2 +- docs/release-instruction.md | 8 ++++---- docs/update-go-instruction.md | 2 +- 7 files changed, 6 insertions(+), 6 deletions(-) rename {.github => .forgejo}/ISSUE_TEMPLATE/bug_report.md (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/config.yml (100%) rename {.github => .forgejo}/ISSUE_TEMPLATE/feature_request.md (100%) rename {.github => .forgejo}/logo.svg (100%) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.forgejo/ISSUE_TEMPLATE/bug_report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.md rename to .forgejo/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.forgejo/ISSUE_TEMPLATE/config.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/config.yml rename to .forgejo/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.forgejo/ISSUE_TEMPLATE/feature_request.md similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .forgejo/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/logo.svg b/.forgejo/logo.svg similarity index 100% rename from .github/logo.svg rename to .forgejo/logo.svg diff --git a/README.md b/README.md index 47d812b18..6998b6726 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- FrostFS + FrostFS

diff --git a/docs/release-instruction.md b/docs/release-instruction.md index d000f10d0..18659c699 100644 --- a/docs/release-instruction.md +++ b/docs/release-instruction.md @@ -55,7 +55,7 @@ Add an entry to the `CHANGELOG.md` following the style established there. * update `Unreleased...new` and `new...old` diff-links at the bottom of the file * add optional codename and release date in the heading * remove all empty sections such as `Added`, `Removed`, etc. -* make sure all changes have references to GitHub issues in `#123` format (if possible) +* make sure all changes have references to relevant issues in `#123` format (if possible) * clean up all `Unreleased` sections and leave them empty ### Make release commit @@ -110,9 +110,9 @@ $ docker push truecloudlab/frostfs-cli:${FROSTFS_REVISION} $ docker push truecloudlab/frostfs-adm:${FROSTFS_REVISION} ``` -### Make a proper GitHub release (if not automated) +### Make a proper release (if not automated) -Edit an automatically-created release on GitHub, copy things from `CHANGELOG.md`. +Edit an automatically-created release on git.frostfs.info, copy things from `CHANGELOG.md`. Build and tar release binaries with `make prepare-release`, attach them to the release. Publish the release. @@ -121,7 +121,7 @@ the release. Publish the release. Prepare pull-request in [frostfs-devenv](https://git.frostfs.info/TrueCloudLab/frostfs-dev-env) with new versions. -### Close GitHub milestone +### Close milestone Look up [milestones](https://git.frostfs.info/TrueCloudLab/frostfs-node/milestones) and close the release one if exists. diff --git a/docs/update-go-instruction.md b/docs/update-go-instruction.md index f99225046..195e0c6b3 100644 --- a/docs/update-go-instruction.md +++ b/docs/update-go-instruction.md @@ -7,7 +7,7 @@ ## Update CI Change Golang versions for unit test in CI. -There is `go` section in `.github/workflows/go.yaml` file: +There is `go` section in `.forgejo/workflows/*.yml` files: ```yaml jobs: test: From 8a57c78f5f5e2098ec1b0085c709a2473d40adbb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Nov 2024 12:46:55 +0300 Subject: [PATCH 1510/1943] [#1484] engine: Fix engine metrics 1. Add forgotten metrics for client requests 2. Include execIfNotBlocked into metrics Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/container.go | 8 ++++---- pkg/local_object_storage/engine/delete.go | 3 +-- pkg/local_object_storage/engine/get.go | 3 +-- pkg/local_object_storage/engine/inhume.go | 3 +-- pkg/local_object_storage/engine/list.go | 5 +++++ pkg/local_object_storage/engine/lock.go | 1 + pkg/local_object_storage/engine/put.go | 3 +-- pkg/local_object_storage/engine/range.go | 19 +++++++++---------- pkg/local_object_storage/engine/select.go | 6 ++---- 9 files changed, 25 insertions(+), 26 deletions(-) diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 6def02f12..a52436175 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -45,6 +45,8 @@ func (r ListContainersRes) Containers() []cid.ID { // // Returns an error if executions are blocked (see BlockExecution). func (e *StorageEngine) ContainerSize(prm ContainerSizePrm) (res ContainerSizeRes, err error) { + defer elapsed("ContainerSize", e.metrics.AddMethodDuration)() + err = e.execIfNotBlocked(func() error { res, err = e.containerSize(prm) return err @@ -68,8 +70,6 @@ func ContainerSize(e *StorageEngine, id cid.ID) (uint64, error) { } func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRes, err error) { - defer elapsed("EstimateContainerSize", e.metrics.AddMethodDuration)() - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { var csPrm shard.ContainerSizePrm csPrm.SetContainerID(prm.cnr) @@ -93,6 +93,8 @@ func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRe // // Returns an error if executions are blocked (see BlockExecution). func (e *StorageEngine) ListContainers(ctx context.Context, _ ListContainersPrm) (res ListContainersRes, err error) { + defer elapsed("ListContainers", e.metrics.AddMethodDuration)() + err = e.execIfNotBlocked(func() error { res, err = e.listContainers(ctx) return err @@ -114,8 +116,6 @@ func ListContainers(ctx context.Context, e *StorageEngine) ([]cid.ID, error) { } func (e *StorageEngine) listContainers(ctx context.Context) (ListContainersRes, error) { - defer elapsed("ListContainers", e.metrics.AddMethodDuration)() - uniqueIDs := make(map[string]cid.ID) e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 3ec3f8f9b..9ca3a7cee 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -58,6 +58,7 @@ func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRe attribute.Bool("force_removal", prm.forceRemoval), )) defer span.End() + defer elapsed("Delete", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { res, err = e.delete(ctx, prm) @@ -68,8 +69,6 @@ func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRe } func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { - defer elapsed("Delete", e.metrics.AddMethodDuration)() - var locked struct { is bool } diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 4a9199be7..a1fe8a010 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -56,6 +56,7 @@ func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err er attribute.String("address", prm.addr.EncodeToString()), )) defer span.End() + defer elapsed("Get", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { res, err = e.get(ctx, prm) @@ -66,8 +67,6 @@ func (e *StorageEngine) Get(ctx context.Context, prm GetPrm) (res GetRes, err er } func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { - defer elapsed("Get", e.metrics.AddMethodDuration)() - errNotFound := new(apistatus.ObjectNotFound) var shPrm shard.GetPrm diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 35ce50f65..b8959b534 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -70,6 +70,7 @@ var errInhumeFailure = errors.New("inhume operation failed") func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Inhume") defer span.End() + defer elapsed("Inhume", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { res, err = e.inhume(ctx, prm) @@ -80,8 +81,6 @@ func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRe } func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { - defer elapsed("Inhume", e.metrics.AddMethodDuration)() - var shPrm shard.InhumePrm if prm.forceRemoval { shPrm.ForceRemoval() diff --git a/pkg/local_object_storage/engine/list.go b/pkg/local_object_storage/engine/list.go index cb3830b7c..073248862 100644 --- a/pkg/local_object_storage/engine/list.go +++ b/pkg/local_object_storage/engine/list.go @@ -7,6 +7,7 @@ import ( objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" ) // ErrEndOfListing is returned from an object listing with cursor @@ -98,6 +99,10 @@ func (l ListWithCursorRes) Cursor() *Cursor { // Returns ErrEndOfListing if there are no more objects to return or count // parameter set to zero. func (e *StorageEngine) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (ListWithCursorRes, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.ListWithCursor") + defer span.End() + defer elapsed("ListWithCursor", e.metrics.AddMethodDuration)() + result := make([]objectcore.Info, 0, prm.count) // Set initial cursors diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index ac8fa9c6f..bbab59bfa 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -32,6 +32,7 @@ func (e *StorageEngine) Lock(ctx context.Context, idCnr cid.ID, locker oid.ID, l attribute.Int("locked_count", len(locked)), )) defer span.End() + defer elapsed("Lock", e.metrics.AddMethodDuration)() return e.execIfNotBlocked(func() error { return e.lock(ctx, idCnr, locker, locked) diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index a50d80b75..56d3ef490 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -56,6 +56,7 @@ func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) { attribute.String("address", object.AddressOf(prm.Object).EncodeToString()), )) defer span.End() + defer elapsed("Put", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { err = e.put(ctx, prm) @@ -66,8 +67,6 @@ func (e *StorageEngine) Put(ctx context.Context, prm PutPrm) (err error) { } func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { - defer elapsed("Put", e.metrics.AddMethodDuration)() - addr := object.AddressOf(prm.Object) // In #1146 this check was parallelized, however, it became diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 498674fd2..fde6052ae 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -65,6 +65,15 @@ func (r RngRes) Object() *objectSDK.Object { // // Returns an error if executions are blocked (see BlockExecution). func (e *StorageEngine) GetRange(ctx context.Context, prm RngPrm) (res RngRes, err error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.getRange", + trace.WithAttributes( + attribute.String("address", prm.addr.EncodeToString()), + attribute.String("offset", strconv.FormatUint(prm.off, 10)), + attribute.String("length", strconv.FormatUint(prm.ln, 10)), + )) + defer span.End() + defer elapsed("GetRange", e.metrics.AddMethodDuration)() + err = e.execIfNotBlocked(func() error { res, err = e.getRange(ctx, prm) return err @@ -74,16 +83,6 @@ func (e *StorageEngine) GetRange(ctx context.Context, prm RngPrm) (res RngRes, e } func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.getRange", - trace.WithAttributes( - attribute.String("address", prm.addr.EncodeToString()), - attribute.String("offset", strconv.FormatUint(prm.off, 10)), - attribute.String("length", strconv.FormatUint(prm.ln, 10)), - )) - defer span.End() - - defer elapsed("GetRange", e.metrics.AddMethodDuration)() - var shPrm shard.RngPrm shPrm.SetAddress(prm.addr) shPrm.SetRange(prm.off, prm.ln) diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index a85891f0c..0f1341f85 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -51,6 +51,7 @@ func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRe attribute.String("container_id", prm.cnr.EncodeToString()), )) defer span.End() + defer elapsed("Select", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { res, err = e._select(ctx, prm) @@ -61,8 +62,6 @@ func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRe } func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) { - defer elapsed("Search", e.metrics.AddMethodDuration)() - addrList := make([]oid.Address, 0) uniqueMap := make(map[string]struct{}) @@ -99,6 +98,7 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, // // Returns an error if executions are blocked (see BlockExecution). func (e *StorageEngine) List(ctx context.Context, limit uint64) (res SelectRes, err error) { + defer elapsed("List", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { res, err = e.list(ctx, limit) return err @@ -108,8 +108,6 @@ func (e *StorageEngine) List(ctx context.Context, limit uint64) (res SelectRes, } func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, error) { - defer elapsed("ListObjects", e.metrics.AddMethodDuration)() - addrList := make([]oid.Address, 0, limit) uniqueMap := make(map[string]struct{}) ln := uint64(0) From 2f3bc6eb8421eb742d1304371d077015a74e218c Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 12 Nov 2024 11:07:48 +0300 Subject: [PATCH 1511/1943] [#1396] cli/playground: Improve terminal control key handling Signed-off-by: Alexander Chuprov --- .../modules/container/policy_playground.go | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 095ab6438..6885d35f2 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -1,11 +1,10 @@ package container import ( - "bufio" "encoding/hex" "encoding/json" + "errors" "fmt" - "io" "os" "strings" @@ -14,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/chzyer/readline" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -163,6 +163,16 @@ func (repl *policyPlaygroundREPL) netMap() netmap.NetMap { return nm } +var policyPlaygroundCompleter = readline.NewPrefixCompleter( + readline.PcItem("list"), + readline.PcItem("ls"), + readline.PcItem("add"), + readline.PcItem("load"), + readline.PcItem("remove"), + readline.PcItem("rm"), + readline.PcItem("eval"), +) + func (repl *policyPlaygroundREPL) run() error { if len(viper.GetString(commonflags.RPC)) > 0 { key := key.GetOrGenerate(repl.cmd) @@ -189,15 +199,32 @@ func (repl *policyPlaygroundREPL) run() error { "rm": repl.handleRemove, "eval": repl.handleEval, } - for reader := bufio.NewReader(os.Stdin); ; { - fmt.Print("> ") - line, err := reader.ReadString('\n') + + rl, err := readline.NewEx(&readline.Config{ + Prompt: "> ", + InterruptPrompt: "^C", + AutoComplete: policyPlaygroundCompleter, + }) + if err != nil { + return fmt.Errorf("error initializing readline: %w", err) + } + defer rl.Close() + + var exit bool + for { + line, err := rl.Readline() if err != nil { - if err == io.EOF { - return nil + if errors.Is(err, readline.ErrInterrupt) { + if exit { + return nil + } + exit = true + continue } return fmt.Errorf("reading line: %v", err) } + exit = false + parts := strings.Fields(line) if len(parts) == 0 { continue From 80f8a8fd3abd0cf9a605c4002cbaced6f2fd9b8c Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 12 Nov 2024 11:09:27 +0300 Subject: [PATCH 1512/1943] [#1396] cli/playground: Refactor Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/modules/container/policy_playground.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 6885d35f2..40bd4110b 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -221,7 +221,7 @@ func (repl *policyPlaygroundREPL) run() error { exit = true continue } - return fmt.Errorf("reading line: %v", err) + return fmt.Errorf("reading line: %w", err) } exit = false @@ -230,8 +230,7 @@ func (repl *policyPlaygroundREPL) run() error { continue } cmd := parts[0] - handler, exists := cmdHandlers[cmd] - if exists { + if handler, exists := cmdHandlers[cmd]; exists { if err := handler(parts[1:]); err != nil { fmt.Printf("error: %v\n", err) } From b543569c3f0ae2feab378ad3568bab27528d1035 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 11 Nov 2024 18:54:07 +0300 Subject: [PATCH 1513/1943] [#1486] node: Introduce dual service support * Register GRPC services for both neo.fs.v2 and frost.fs namespaces * Use this temporary solution until all nodes are updated Signed-off-by: Airat Arifullin --- cmd/frostfs-node/accounting.go | 23 +++++++++++++++++++++++ cmd/frostfs-node/apemanager.go | 3 +++ cmd/frostfs-node/container.go | 3 +++ cmd/frostfs-node/netmap.go | 3 +++ cmd/frostfs-node/object.go | 3 +++ cmd/frostfs-node/session.go | 3 +++ 6 files changed, 38 insertions(+) diff --git a/cmd/frostfs-node/accounting.go b/cmd/frostfs-node/accounting.go index 72a6e5331..2d52e0c56 100644 --- a/cmd/frostfs-node/accounting.go +++ b/cmd/frostfs-node/accounting.go @@ -3,6 +3,7 @@ package main import ( "context" "net" + "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" accountingTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/accounting/grpc" @@ -30,5 +31,27 @@ func initAccountingService(ctx context.Context, c *cfg) { c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { accountingGRPC.RegisterAccountingServiceServer(s, server) + + // TODO(@aarifullin): #1487 remove the dual service support. + s.RegisterService(frostFSServiceDesc(accountingGRPC.AccountingService_ServiceDesc), server) }) } + +// frostFSServiceDesc creates a service descriptor with the new namespace for dual service support. +func frostFSServiceDesc(sd grpc.ServiceDesc) *grpc.ServiceDesc { + sdLegacy := new(grpc.ServiceDesc) + *sdLegacy = sd + + const ( + legacyNamespace = "neo.fs.v2" + apemanagerLegacyNamespace = "frostfs.v2" + newNamespace = "frost.fs" + ) + + if strings.HasPrefix(sd.ServiceName, legacyNamespace) { + sdLegacy.ServiceName = strings.ReplaceAll(sd.ServiceName, legacyNamespace, newNamespace) + } else if strings.HasPrefix(sd.ServiceName, apemanagerLegacyNamespace) { + sdLegacy.ServiceName = strings.ReplaceAll(sd.ServiceName, apemanagerLegacyNamespace, newNamespace) + } + return sdLegacy +} diff --git a/cmd/frostfs-node/apemanager.go b/cmd/frostfs-node/apemanager.go index c4d7725f5..de3aed660 100644 --- a/cmd/frostfs-node/apemanager.go +++ b/cmd/frostfs-node/apemanager.go @@ -26,5 +26,8 @@ func initAPEManagerService(c *cfg) { c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { apemanager_grpc.RegisterAPEManagerServiceServer(s, server) + + // TODO(@aarifullin): #1487 remove the dual service support. + s.RegisterService(frostFSServiceDesc(apemanager_grpc.APEManagerService_ServiceDesc), server) }) } diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 9c3505922..de840ec80 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -64,6 +64,9 @@ func initContainerService(_ context.Context, c *cfg) { c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { containerGRPC.RegisterContainerServiceServer(s, server) + + // TODO(@aarifullin): #1487 remove the dual service support. + s.RegisterService(frostFSServiceDesc(containerGRPC.ContainerService_ServiceDesc), server) }) c.cfgObject.cfgLocalStorage.localStorage.SetContainerSource(cnrRdr) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 58e066fc9..73871bfc9 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -166,6 +166,9 @@ func initNetmapService(ctx context.Context, c *cfg) { c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { netmapGRPC.RegisterNetmapServiceServer(s, server) + + // TODO(@aarifullin): #1487 remove the dual service support. + s.RegisterService(frostFSServiceDesc(netmapGRPC.NetmapService_ServiceDesc), server) }) addNewEpochNotificationHandlers(c) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 629f79207..ec521ab01 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -218,6 +218,9 @@ func initObjectService(c *cfg) { c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { objectGRPC.RegisterObjectServiceServer(s, server) + + // TODO(@aarifullin): #1487 remove the dual service support. + s.RegisterService(frostFSServiceDesc(objectGRPC.ObjectService_ServiceDesc), server) }) } diff --git a/cmd/frostfs-node/session.go b/cmd/frostfs-node/session.go index 20d2d318f..a35d4e470 100644 --- a/cmd/frostfs-node/session.go +++ b/cmd/frostfs-node/session.go @@ -61,5 +61,8 @@ func initSessionService(c *cfg) { c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { sessionGRPC.RegisterSessionServiceServer(s, server) + + // TODO(@aarifullin): #1487 remove the dual service support. + s.RegisterService(frostFSServiceDesc(sessionGRPC.SessionService_ServiceDesc), server) }) } From be2753de0032c2eb4c0abb5b00096c7e6697413f Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 12 Nov 2024 13:31:13 +0300 Subject: [PATCH 1514/1943] [#1490] docs: Update description for `object.get.priority` Signed-off-by: Anton Nikiforov --- docs/storage-node-configuration.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 363520481..98d72cb69 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -412,12 +412,12 @@ object: - $attribute:ClusterName ``` -| Parameter | Type | Default value | Description | -|-----------------------------|------------|---------------|------------------------------------------------------------------------------------------------------| -| `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | -| `put.remote_pool_size` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | -| `put.local_pool_size` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | -| `get.priority` | `[]string` | | List of metrics of nodes for prioritization. Used for computing response on GET and SEARCH requests. | +| Parameter | Type | Default value | Description | +|-----------------------------|------------|---------------|------------------------------------------------------------------------------------------------| +| `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | +| `put.remote_pool_size` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | +| `put.local_pool_size` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | +| `get.priority` | `[]string` | | List of metrics of nodes for prioritization. Used for computing response on GET requests. | # `runtime` section Contains runtime parameters. From a69229853366e192296e2a287c49adc66c647bb3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 11 Nov 2024 09:24:09 +0300 Subject: [PATCH 1515/1943] [#1483] node: Remove eACL cache Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/cache.go | 25 ----------------------- cmd/frostfs-node/config.go | 2 -- cmd/frostfs-node/container.go | 16 --------------- cmd/frostfs-node/object.go | 26 ------------------------ pkg/core/container/storage.go | 13 ------------ pkg/services/container/morph/executor.go | 1 - 6 files changed, 83 deletions(-) diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index 06142a46c..b90641799 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -196,31 +196,6 @@ func (s ttlContainerStorage) DeletionInfo(cnr cid.ID) (*container.DelInfo, error return s.delInfoCache.get(cnr) } -type ttlEACLStorage struct { - *ttlNetCache[cid.ID, *container.EACL] -} - -func newCachedEACLStorage(v container.EACLSource, ttl time.Duration) ttlEACLStorage { - const eaclCacheSize = 100 - - lruCnrCache := newNetworkTTLCache(eaclCacheSize, ttl, func(id cid.ID) (*container.EACL, error) { - return v.GetEACL(id) - }, metrics.NewCacheMetrics("eacl")) - - return ttlEACLStorage{lruCnrCache} -} - -// GetEACL returns eACL value from the cache. If value is missing in the cache -// or expired, then it returns value from side chain and updates cache. -func (s ttlEACLStorage) GetEACL(cnr cid.ID) (*container.EACL, error) { - return s.get(cnr) -} - -// InvalidateEACL removes cached eACL value. -func (s ttlEACLStorage) InvalidateEACL(cnr cid.ID) { - s.remove(cnr) -} - type lruNetmapSource struct { netState netmap.State diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 40af23841..902187560 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -642,8 +642,6 @@ type cfgObject struct { cnrSource container.Source - eaclSource container.EACLSource - cfgAccessPolicyEngine cfgAccessPolicyEngine pool cfgObjectRoutines diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index de840ec80..cc38876ee 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -73,10 +73,6 @@ func initContainerService(_ context.Context, c *cfg) { } func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc containerCore.Source) (*morphContainerReader, *morphContainerWriter) { - eACLFetcher := &morphEACLFetcher{ - w: client, - } - cnrRdr := new(morphContainerReader) cnrWrt := &morphContainerWriter{ @@ -84,8 +80,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c } if c.cfgMorph.cacheTTL <= 0 { - c.cfgObject.eaclSource = eACLFetcher - cnrRdr.eacl = eACLFetcher c.cfgObject.cnrSource = cnrSrc cnrRdr.src = cnrSrc cnrRdr.lister = client @@ -129,11 +123,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c c.cfgObject.cnrSource = containerCache } - cachedEACLStorage := newCachedEACLStorage(eACLFetcher, c.cfgMorph.cacheTTL) - c.cfgObject.eaclSource = cachedEACLStorage - cnrRdr.lister = client - cnrRdr.eacl = c.cfgObject.eaclSource cnrRdr.src = c.cfgObject.cnrSource } @@ -224,8 +214,6 @@ func (c *cfg) ExternalAddresses() []string { // implements interface required by container service provided by morph executor. type morphContainerReader struct { - eacl containerCore.EACLSource - src containerCore.Source lister interface { @@ -241,10 +229,6 @@ func (x *morphContainerReader) DeletionInfo(id cid.ID) (*containerCore.DelInfo, return x.src.DeletionInfo(id) } -func (x *morphContainerReader) GetEACL(id cid.ID) (*containerCore.EACL, error) { - return x.eacl.GetEACL(id) -} - func (x *morphContainerReader) ContainersOf(id *user.ID) ([]cid.ID, error) { return x.lister.ContainersOf(id) } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index ec521ab01..aeab1d6cb 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "fmt" "net" @@ -14,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/cache" objectTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/object/grpc" @@ -37,7 +35,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -484,29 +481,6 @@ func createAPEService(c *cfg, splitSvc *objectService.TransportSplitter) *object ) } -type morphEACLFetcher struct { - w *cntClient.Client -} - -func (s *morphEACLFetcher) GetEACL(cnr cid.ID) (*containercore.EACL, error) { - eaclInfo, err := s.w.GetEACL(cnr) - if err != nil { - return nil, err - } - - binTable, err := eaclInfo.Value.Marshal() - if err != nil { - return nil, fmt.Errorf("marshal eACL table: %w", err) - } - - if !eaclInfo.Signature.Verify(binTable) { - // TODO(@cthulhu-rider): #468 use "const" error - return nil, errors.New("invalid signature of the eACL table") - } - - return eaclInfo, nil -} - type engineWithoutNotifications struct { engine *engine.StorageEngine } diff --git a/pkg/core/container/storage.go b/pkg/core/container/storage.go index 69854f495..ba4404546 100644 --- a/pkg/core/container/storage.go +++ b/pkg/core/container/storage.go @@ -58,16 +58,3 @@ type EACL struct { // Session within which Value was set. Nil means session absence. Session *session.Container } - -// EACLSource is the interface that wraps -// basic methods of extended ACL table source. -type EACLSource interface { - // GetEACL reads the table from the source by identifier. - // It returns any error encountered. - // - // GetEACL must return exactly one non-nil value. - // - // Must return apistatus.ErrEACLNotFound if requested - // eACL table is not in source. - GetEACL(cid.ID) (*EACL, error) -} diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index adb808af3..eb43eab70 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -25,7 +25,6 @@ type morphExecutor struct { // Reader is an interface of read-only container storage. type Reader interface { containercore.Source - containercore.EACLSource // ContainersOf returns a list of container identifiers belonging // to the specified user of FrostFS system. Returns the identifiers From 3324c26fd8f8b68ca7feea07263bfc91b8587817 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 11 Nov 2024 09:26:00 +0300 Subject: [PATCH 1516/1943] [#1483] morph: Remove container.GetEACL() Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/container/client.go | 1 - pkg/morph/client/container/eacl.go | 95 ---------------------------- 2 files changed, 96 deletions(-) delete mode 100644 pkg/morph/client/container/eacl.go diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index b512a6594..bdbcce917 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -27,7 +27,6 @@ const ( getMethod = "get" listMethod = "list" containersOfMethod = "containersOf" - eaclMethod = "eACL" deletionInfoMethod = "deletionInfo" // putNamedMethod is method name for container put with an alias. It is exported to provide custom fee. diff --git a/pkg/morph/client/container/eacl.go b/pkg/morph/client/container/eacl.go deleted file mode 100644 index 9e604e091..000000000 --- a/pkg/morph/client/container/eacl.go +++ /dev/null @@ -1,95 +0,0 @@ -package container - -import ( - "crypto/sha256" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" -) - -// GetEACL reads the extended ACL table from FrostFS system -// through Container contract call. -// -// Returns apistatus.EACLNotFound if eACL table is missing in the contract. -func (c *Client) GetEACL(cnr cid.ID) (*container.EACL, error) { - binCnr := make([]byte, sha256.Size) - cnr.Encode(binCnr) - - prm := client.TestInvokePrm{} - prm.SetMethod(eaclMethod) - prm.SetArgs(binCnr) - - prms, err := c.client.TestInvoke(prm) - if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", eaclMethod, err) - } else if ln := len(prms); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", eaclMethod, ln) - } - - arr, err := client.ArrayFromStackItem(prms[0]) - if err != nil { - return nil, fmt.Errorf("could not get item array of eACL (%s): %w", eaclMethod, err) - } - - if len(arr) != 4 { - return nil, fmt.Errorf("unexpected eacl stack item count (%s): %d", eaclMethod, len(arr)) - } - - rawEACL, err := client.BytesFromStackItem(arr[0]) - if err != nil { - return nil, fmt.Errorf("could not get byte array of eACL (%s): %w", eaclMethod, err) - } - - sig, err := client.BytesFromStackItem(arr[1]) - if err != nil { - return nil, fmt.Errorf("could not get byte array of eACL signature (%s): %w", eaclMethod, err) - } - - // Client may not return errors if the table is missing, so check this case additionally. - // The absence of a signature in the response can be taken as an eACL absence criterion, - // since unsigned table cannot be approved in the storage by design. - if len(sig) == 0 { - return nil, new(apistatus.EACLNotFound) - } - - pub, err := client.BytesFromStackItem(arr[2]) - if err != nil { - return nil, fmt.Errorf("could not get byte array of eACL public key (%s): %w", eaclMethod, err) - } - - binToken, err := client.BytesFromStackItem(arr[3]) - if err != nil { - return nil, fmt.Errorf("could not get byte array of eACL session token (%s): %w", eaclMethod, err) - } - - var res container.EACL - - res.Value = eacl.NewTable() - if err = res.Value.Unmarshal(rawEACL); err != nil { - return nil, err - } - - if len(binToken) > 0 { - res.Session = new(session.Container) - - err = res.Session.Unmarshal(binToken) - if err != nil { - return nil, fmt.Errorf("could not unmarshal session token: %w", err) - } - } - - // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion - var sigV2 refs.Signature - sigV2.SetKey(pub) - sigV2.SetSign(sig) - sigV2.SetScheme(refs.ECDSA_RFC6979_SHA256) - - err = res.Signature.ReadFromV2(sigV2) - return &res, err -} From 1cf51a80799b93cc679be7ea1d1f4022ae019ade Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 11 Nov 2024 09:37:30 +0300 Subject: [PATCH 1517/1943] [#1483] cli/docs: Remove set-eacl mention Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/docs/sessions.md | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/frostfs-cli/docs/sessions.md b/cmd/frostfs-cli/docs/sessions.md index 04563b7af..52c0e9b9b 100644 --- a/cmd/frostfs-cli/docs/sessions.md +++ b/cmd/frostfs-cli/docs/sessions.md @@ -72,4 +72,3 @@ All other `object` sub-commands support only static sessions (2). List of commands supporting sessions (static only): - `create` - `delete` -- `set-eacl` From 16830033f82cfa2936a721b22ab65469896ea240 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 11 Nov 2024 09:41:55 +0300 Subject: [PATCH 1518/1943] [#1483] cli: Remove --basic-acl flag Signed-off-by: Evgenii Stratonikov --- README.md | 2 +- cmd/frostfs-cli/modules/container/create.go | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/README.md b/README.md index 6998b6726..0109ed0e5 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ See `frostfs-contract`'s README.md for build instructions. 4. To create container and put object into it run (container and object IDs will be different): ``` -./bin/frostfs-cli container create -r 127.0.0.1:8080 --wallet ./dev/wallet.json --policy "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" --basic-acl public-read-write --await +./bin/frostfs-cli container create -r 127.0.0.1:8080 --wallet ./dev/wallet.json --policy "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" --await Enter password > <- press ENTER, the is no password for wallet CID: CfPhEuHQ2PRvM4gfBQDC4dWZY3NccovyfcnEdiq2ixju diff --git a/cmd/frostfs-cli/modules/container/create.go b/cmd/frostfs-cli/modules/container/create.go index 017f4b5ce..30f995180 100644 --- a/cmd/frostfs-cli/modules/container/create.go +++ b/cmd/frostfs-cli/modules/container/create.go @@ -15,14 +15,12 @@ import ( containerApi "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/spf13/cobra" ) var ( - containerACL string containerPolicy string containerAttributes []string containerAwait bool @@ -89,9 +87,6 @@ It will be stored in sidechain when inner ring will accepts it.`, err = parseAttributes(&cnr, containerAttributes) commonCmd.ExitOnErr(cmd, "", err) - var basicACL acl.Basic - commonCmd.ExitOnErr(cmd, "decode basic ACL string: %w", basicACL.DecodeString(containerACL)) - tok := getSession(cmd) if tok != nil { @@ -105,7 +100,6 @@ It will be stored in sidechain when inner ring will accepts it.`, } cnr.SetPlacementPolicy(*placementPolicy) - cnr.SetBasicACL(basicACL) var syncContainerPrm internalclient.SyncContainerPrm syncContainerPrm.SetClient(cli) @@ -163,10 +157,6 @@ func initContainerCreateCmd() { flags.DurationP(commonflags.Timeout, commonflags.TimeoutShorthand, commonflags.TimeoutDefault, commonflags.TimeoutUsage) flags.StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, commonflags.WalletPathDefault, commonflags.WalletPathUsage) flags.StringP(commonflags.Account, commonflags.AccountShorthand, commonflags.AccountDefault, commonflags.AccountUsage) - - flags.StringVar(&containerACL, "basic-acl", acl.NamePrivate, fmt.Sprintf("HEX encoded basic ACL value or keywords like '%s', '%s', '%s'", - acl.NamePublicRW, acl.NamePrivate, acl.NamePublicROExtended, - )) flags.StringVarP(&containerPolicy, "policy", "p", "", "QL-encoded or JSON-encoded placement policy or path to file with it") flags.StringSliceVarP(&containerAttributes, "attributes", "a", nil, "Comma separated pairs of container attributes in form of Key1=Value1,Key2=Value2") flags.BoolVar(&containerAwait, "await", false, "Block execution until container is persisted") From 9bd05e94c8e1dee1a63d343938abf77cc40eb51d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 30 Oct 2024 11:02:52 +0300 Subject: [PATCH 1519/1943] [#1449] tree: Add ApplyBatch method Concurrent Apply can lead to child node applies before parent, so undo/redo operations will perform. This leads to performance degradation in case of tree with many sublevels. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config/tree/config.go | 13 ++++ cmd/frostfs-node/config/tree/config_test.go | 1 + cmd/frostfs-node/tree.go | 1 + config/example/node.env | 1 + config/example/node.json | 1 + config/example/node.yaml | 1 + pkg/local_object_storage/engine/tree.go | 28 +++++++ pkg/local_object_storage/pilorama/boltdb.go | 74 +++++++++++++++++++ pkg/local_object_storage/pilorama/forest.go | 9 +++ .../pilorama/interface.go | 2 + pkg/local_object_storage/shard/tree.go | 27 +++++++ pkg/services/tree/options.go | 7 ++ pkg/services/tree/replicator.go | 1 + pkg/services/tree/service.go | 1 + pkg/services/tree/sync.go | 34 ++++----- 15 files changed, 180 insertions(+), 21 deletions(-) diff --git a/cmd/frostfs-node/config/tree/config.go b/cmd/frostfs-node/config/tree/config.go index 8a8919999..da877791e 100644 --- a/cmd/frostfs-node/config/tree/config.go +++ b/cmd/frostfs-node/config/tree/config.go @@ -10,6 +10,8 @@ import ( const ( subsection = "tree" + + SyncBatchSizeDefault = 1000 ) // TreeConfig is a wrapper over "tree" config section @@ -74,6 +76,17 @@ func (c TreeConfig) SyncInterval() time.Duration { return config.DurationSafe(c.cfg, "sync_interval") } +// SyncBatchSize returns the value of "sync_batch_size" +// config parameter from the "tree" section. +// +// Returns `SyncBatchSizeDefault` if config value is not specified. +func (c TreeConfig) SyncBatchSize() int { + if v := config.IntSafe(c.cfg, "sync_batch_size"); v > 0 { + return int(v) + } + return SyncBatchSizeDefault +} + // AuthorizedKeys parses and returns an array of "authorized_keys" config // parameter from "tree" section. // diff --git a/cmd/frostfs-node/config/tree/config_test.go b/cmd/frostfs-node/config/tree/config_test.go index 285ea0725..6628b8878 100644 --- a/cmd/frostfs-node/config/tree/config_test.go +++ b/cmd/frostfs-node/config/tree/config_test.go @@ -44,6 +44,7 @@ func TestTreeSection(t *testing.T) { require.Equal(t, 32, treeSec.ReplicationWorkerCount()) require.Equal(t, 5*time.Second, treeSec.ReplicationTimeout()) require.Equal(t, time.Hour, treeSec.SyncInterval()) + require.Equal(t, 2000, treeSec.SyncBatchSize()) require.Equal(t, expectedKeys, treeSec.AuthorizedKeys()) } diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index f188e2fbc..a92979daf 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -62,6 +62,7 @@ func initTreeService(c *cfg) { tree.WithReplicationTimeout(treeConfig.ReplicationTimeout()), tree.WithReplicationChannelCapacity(treeConfig.ReplicationChannelCapacity()), tree.WithReplicationWorkerCount(treeConfig.ReplicationWorkerCount()), + tree.WithSyncBatchSize(treeConfig.SyncBatchSize()), tree.WithAuthorizedKeys(treeConfig.AuthorizedKeys()), tree.WithMetrics(c.metricsCollector.TreeService()), tree.WithAPELocalOverrideStorage(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalStorage()), diff --git a/config/example/node.env b/config/example/node.env index 3979eb18f..e21328b60 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -31,6 +31,7 @@ FROSTFS_TREE_REPLICATION_CHANNEL_CAPACITY=32 FROSTFS_TREE_REPLICATION_WORKER_COUNT=32 FROSTFS_TREE_REPLICATION_TIMEOUT=5s FROSTFS_TREE_SYNC_INTERVAL=1h +FROSTFS_TREE_SYNC_BATCH_SIZE=2000 FROSTFS_TREE_AUTHORIZED_KEYS="0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0 02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56" # gRPC section diff --git a/config/example/node.json b/config/example/node.json index 1ea28de6c..ac7125949 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -69,6 +69,7 @@ "replication_worker_count": 32, "replication_timeout": "5s", "sync_interval": "1h", + "sync_batch_size": 2000, "authorized_keys": [ "0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0", "02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56" diff --git a/config/example/node.yaml b/config/example/node.yaml index 4a418dfcb..d547f5cee 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -59,6 +59,7 @@ tree: replication_channel_capacity: 32 replication_timeout: 5s sync_interval: 1h + sync_batch_size: 2000 authorized_keys: # list of hex-encoded public keys that have rights to use the Tree Service with frostfs-cli - 0397d207ea77909f7d66fa6f36d08daae22ace672be7ea4f53513484dde8a142a0 - 02053819235c20d784132deba10bb3061629e3a5c819a039ef091841d9d35dad56 diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 39122628f..6bb5e3a41 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -110,6 +110,34 @@ func (e *StorageEngine) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID str return nil } +// TreeApplyBatch implements the pilorama.Forest interface. +func (e *StorageEngine) TreeApplyBatch(ctx context.Context, cnr cidSDK.ID, treeID string, m []*pilorama.Move) error { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeApplyBatch", + trace.WithAttributes( + attribute.String("container_id", cnr.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + index, lst, err := e.getTreeShard(ctx, cnr, treeID) + if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { + return err + } + + err = lst[index].TreeApplyBatch(ctx, cnr, treeID, m) + if err != nil { + if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { + e.reportShardError(lst[index], "can't perform `TreeApplyBatch`", err, + zap.Stringer("cid", cnr), + zap.String("tree", treeID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } + return err + } + return nil +} + // TreeGetByPath implements the pilorama.Forest interface. func (e *StorageEngine) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]pilorama.Node, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeGetByPath", diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 7bce1f340..a778434dd 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -558,6 +558,80 @@ func (t *boltForest) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string return metaerr.Wrap(err) } +func (t *boltForest) TreeApplyBatch(ctx context.Context, cnr cidSDK.ID, treeID string, m []*Move) error { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + t.metrics.AddMethodDuration("TreeApplyBatch", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "boltForest.TreeApplyBatch", + trace.WithAttributes( + attribute.String("container_id", cnr.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + m, err := t.filterSeen(cnr, treeID, m) + if err != nil { + return err + } + if len(m) == 0 { + success = true + return nil + } + + ch := make(chan error) + b := &batch{ + forest: t, + cid: cnr, + treeID: treeID, + results: []chan<- error{ch}, + operations: m, + } + go func() { + b.run() + }() + err = <-ch + success = err == nil + return metaerr.Wrap(err) +} + +func (t *boltForest) filterSeen(cnr cidSDK.ID, treeID string, m []*Move) ([]*Move, error) { + t.modeMtx.RLock() + defer t.modeMtx.RUnlock() + + if t.mode.NoMetabase() { + return nil, ErrDegradedMode + } + + ops := make([]*Move, 0, len(m)) + err := t.db.View(func(tx *bbolt.Tx) error { + treeRoot := tx.Bucket(bucketName(cnr, treeID)) + if treeRoot == nil { + ops = m + return nil + } + b := treeRoot.Bucket(logBucket) + for _, op := range m { + var logKey [8]byte + binary.BigEndian.PutUint64(logKey[:], op.Time) + seen := b.Get(logKey[:]) != nil + if !seen { + ops = append(ops, op) + } + } + return nil + }) + if err != nil { + return nil, metaerr.Wrap(err) + } + return ops, nil +} + // TreeApplyStream should be used with caution: this method locks other write transactions while `source` is not closed. func (t *boltForest) TreeApplyStream(ctx context.Context, cnr cidSDK.ID, treeID string, source <-chan *Move) error { var ( diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index bb5c22e51..374943745 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -111,6 +111,15 @@ func (f *memoryForest) TreeApply(_ context.Context, cnr cid.ID, treeID string, o return s.Apply(op) } +func (f *memoryForest) TreeApplyBatch(ctx context.Context, cnr cid.ID, treeID string, ops []*Move) error { + for _, op := range ops { + if err := f.TreeApply(ctx, cnr, treeID, op, true); err != nil { + return err + } + } + return nil +} + func (f *memoryForest) Init() error { return nil } diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 61a3849bf..b6ca246f2 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -21,6 +21,8 @@ type Forest interface { // TreeApply applies replicated operation from another node. // If background is true, TreeApply will first check whether an operation exists. TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string, m *Move, backgroundSync bool) error + // TreeApplyBatch applies replicated operations from another node. + TreeApplyBatch(ctx context.Context, cnr cidSDK.ID, treeID string, m []*Move) error // TreeGetByPath returns all nodes corresponding to the path. // The path is constructed by descending from the root using the values of the // AttributeFilename in meta. diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 26dc8ec1e..01a014cec 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -106,6 +106,33 @@ func (s *Shard) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string, m * return s.pilorama.TreeApply(ctx, cnr, treeID, m, backgroundSync) } +// TreeApplyBatch implements the pilorama.Forest interface. +func (s *Shard) TreeApplyBatch(ctx context.Context, cnr cidSDK.ID, treeID string, m []*pilorama.Move) error { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeApplyBatch", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.String("container_id", cnr.EncodeToString()), + attribute.String("tree_id", treeID), + ), + ) + defer span.End() + + if s.pilorama == nil { + return ErrPiloramaDisabled + } + + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.ReadOnly() { + return ErrReadOnlyMode + } + if s.info.Mode.NoMetabase() { + return ErrDegradedMode + } + return s.pilorama.TreeApplyBatch(ctx, cnr, treeID, m) +} + // TreeGetByPath implements the pilorama.Forest interface. func (s *Shard) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID string, attr string, path []string, latest bool) ([]pilorama.Node, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeGetByPath", diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index 1633ae557..a3f488009 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -41,6 +41,7 @@ type cfg struct { replicatorTimeout time.Duration containerCacheSize int authorizedKeys [][]byte + syncBatchSize int localOverrideStorage policyengine.LocalOverrideStorage morphChainStorage policyengine.MorphRuleChainStorageReader @@ -113,6 +114,12 @@ func WithReplicationWorkerCount(n int) Option { } } +func WithSyncBatchSize(n int) Option { + return func(c *cfg) { + c.syncBatchSize = n + } +} + func WithContainerCacheSize(n int) Option { return func(c *cfg) { if n > 0 { diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 95c8f8013..84e376cf7 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -40,6 +40,7 @@ const ( defaultReplicatorCapacity = 64 defaultReplicatorWorkerCount = 64 defaultReplicatorSendTimeout = time.Second * 5 + defaultSyncBatchSize = 1000 ) func (s *Service) localReplicationWorker(ctx context.Context) { diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 8097d545c..b63338d25 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -55,6 +55,7 @@ func New(opts ...Option) *Service { s.replicatorChannelCapacity = defaultReplicatorCapacity s.replicatorWorkerCount = defaultReplicatorWorkerCount s.replicatorTimeout = defaultReplicatorSendTimeout + s.syncBatchSize = defaultSyncBatchSize s.metrics = defaultMetricsRegister{} for i := range opts { diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index ce1e72104..b93410616 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -177,37 +177,29 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID string, operationStream <-chan *pilorama.Move, ) uint64 { - errGroup, _ := errgroup.WithContext(ctx) - const workersCount = 1024 - errGroup.SetLimit(workersCount) - - // We run TreeApply concurrently for the operation batch. Let's consider two operations - // in the batch m1 and m2 such that m1.Time < m2.Time. The engine may apply m2 and fail - // on m1. That means the service must start sync from m1.Time in the next iteration and - // this height is stored in unappliedOperationHeight. - var unappliedOperationHeight uint64 = math.MaxUint64 - var heightMtx sync.Mutex - var prev *pilorama.Move + var batch []*pilorama.Move for m := range operationStream { // skip already applied op if prev != nil && prev.Time == m.Time { continue } prev = m + batch = append(batch, m) - errGroup.Go(func() error { - if err := s.forest.TreeApply(ctx, cid, treeID, m, true); err != nil { - heightMtx.Lock() - unappliedOperationHeight = min(unappliedOperationHeight, m.Time) - heightMtx.Unlock() - return err + if len(batch) == s.syncBatchSize { + if err := s.forest.TreeApplyBatch(ctx, cid, treeID, batch); err != nil { + return batch[0].Time } - return nil - }) + batch = batch[:0] + } } - _ = errGroup.Wait() - return unappliedOperationHeight + if len(batch) > 0 { + if err := s.forest.TreeApplyBatch(ctx, cid, treeID, batch); err != nil { + return batch[0].Time + } + } + return math.MaxUint64 } func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, From 46fef276b474ba5c5ee66d0dee2a7595d47bc4f0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 11 Nov 2024 12:12:45 +0300 Subject: [PATCH 1520/1943] [#1449] tree: Log tree sync with Info level Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index b93410616..2c6deeb78 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -376,7 +376,7 @@ func (s *Service) syncLoop(ctx context.Context) { return case <-s.syncChan: ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.sync") - s.log.Debug(logs.TreeSyncingTrees) + s.log.Info(logs.TreeSyncingTrees) start := time.Now() @@ -394,7 +394,7 @@ func (s *Service) syncLoop(ctx context.Context) { s.removeContainers(ctx, newMap) - s.log.Debug(logs.TreeTreesHaveBeenSynchronized) + s.log.Info(logs.TreeTreesHaveBeenSynchronized) s.metrics.AddSyncDuration(time.Since(start), true) span.End() From c00f4bab18854b88f8c87d5b3dfff859d7505a26 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 12 Nov 2024 11:32:33 +0300 Subject: [PATCH 1521/1943] [#1488] go.mod: Bump observability version Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8a70c3819..84f9f9a66 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.0 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241107121119-cb813e27a823 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 diff --git a/go.sum b/go.sum index 9778f91e2..d93e9b74e 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d h1:uJ/wvuMdepbkaV8XMS5uN9B0FQWMep0CttSuDZiDhq0= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573 h1:6qCcm1oqFbmf9C5AauXzrL5OPGnTbI9HoB/jAtD9274= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20240909114314-666d326cc573/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241107121119-cb813e27a823 h1:sepm9FeuoInmygH1K/+3L+Yp5bJhGiVi/oGCH6Emp2c= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241107121119-cb813e27a823/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= From e2658c75194fb8b04b0ec763dda92bfa94967e55 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 12 Nov 2024 11:33:23 +0300 Subject: [PATCH 1522/1943] [#1488] tracing: KV attributes for spans from config Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config/tracing/config.go | 23 +++++++++- .../config/tracing/config_test.go | 46 +++++++++++++++++++ config/example/node.env | 4 ++ config/example/node.json | 14 +++++- config/example/node.yaml | 5 ++ 5 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 cmd/frostfs-node/config/tracing/config_test.go diff --git a/cmd/frostfs-node/config/tracing/config.go b/cmd/frostfs-node/config/tracing/config.go index 8544c672c..91ef669ee 100644 --- a/cmd/frostfs-node/config/tracing/config.go +++ b/cmd/frostfs-node/config/tracing/config.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" @@ -24,6 +25,7 @@ func ToTracingConfig(c *config.Config) (*tracing.Config, error) { Service: "frostfs-node", InstanceID: getInstanceIDOrDefault(c), Version: misc.Version, + Attributes: make(map[string]string), } if trustedCa := config.StringSafe(c.Sub(subsection), "trusted_ca"); trustedCa != "" { @@ -38,11 +40,30 @@ func ToTracingConfig(c *config.Config) (*tracing.Config, error) { } conf.ServerCaCertPool = certPool } + + i := uint64(0) + for ; ; i++ { + si := strconv.FormatUint(i, 10) + ac := c.Sub(subsection).Sub("attributes").Sub(si) + k := config.StringSafe(ac, "key") + if k == "" { + break + } + v := config.StringSafe(ac, "value") + if v == "" { + return nil, fmt.Errorf("empty tracing attribute value for key %s", k) + } + if _, ok := conf.Attributes[k]; ok { + return nil, fmt.Errorf("tracing attribute key %s defined more than once", k) + } + conf.Attributes[k] = v + } + return conf, nil } func getInstanceIDOrDefault(c *config.Config) string { - s := config.StringSlice(c.Sub("node"), "addresses") + s := config.StringSliceSafe(c.Sub("node"), "addresses") if len(s) > 0 { return s[0] } diff --git a/cmd/frostfs-node/config/tracing/config_test.go b/cmd/frostfs-node/config/tracing/config_test.go new file mode 100644 index 000000000..8e485ca6e --- /dev/null +++ b/cmd/frostfs-node/config/tracing/config_test.go @@ -0,0 +1,46 @@ +package tracing + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "github.com/stretchr/testify/require" +) + +func TestTracingSection(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + tc, err := ToTracingConfig(configtest.EmptyConfig()) + require.NoError(t, err) + require.Equal(t, false, tc.Enabled) + require.Equal(t, tracing.Exporter(""), tc.Exporter) + require.Equal(t, "", tc.Endpoint) + require.Equal(t, "frostfs-node", tc.Service) + require.Equal(t, "", tc.InstanceID) + require.Nil(t, tc.ServerCaCertPool) + require.Empty(t, tc.Attributes) + }) + + const path = "../../../../config/example/node" + + fileConfigTest := func(c *config.Config) { + tc, err := ToTracingConfig(c) + require.NoError(t, err) + require.Equal(t, true, tc.Enabled) + require.Equal(t, tracing.OTLPgRPCExporter, tc.Exporter) + require.Equal(t, "localhost", tc.Endpoint) + require.Equal(t, "frostfs-node", tc.Service) + require.Nil(t, tc.ServerCaCertPool) + require.EqualValues(t, map[string]string{ + "key0": "value", + "key1": "value", + }, tc.Attributes) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) +} diff --git a/config/example/node.env b/config/example/node.env index e21328b60..f470acf3e 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -203,6 +203,10 @@ FROSTFS_TRACING_ENABLED=true FROSTFS_TRACING_ENDPOINT="localhost" FROSTFS_TRACING_EXPORTER="otlp_grpc" FROSTFS_TRACING_TRUSTED_CA="" +FROSTFS_TRACING_ATTRIBUTES_0_KEY=key0 +FROSTFS_TRACING_ATTRIBUTES_0_VALUE=value +FROSTFS_TRACING_ATTRIBUTES_1_KEY=key1 +FROSTFS_TRACING_ATTRIBUTES_1_VALUE=value FROSTFS_RUNTIME_SOFT_MEMORY_LIMIT=1073741824 diff --git a/config/example/node.json b/config/example/node.json index ac7125949..dba3bad8b 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -259,9 +259,19 @@ }, "tracing": { "enabled": true, - "endpoint": "localhost:9090", + "endpoint": "localhost", "exporter": "otlp_grpc", - "trusted_ca": "/etc/ssl/tracing.pem" + "trusted_ca": "", + "attributes":[ + { + "key": "key0", + "value": "value" + }, + { + "key": "key1", + "value": "value" + } + ] }, "runtime": { "soft_memory_limit": 1073741824 diff --git a/config/example/node.yaml b/config/example/node.yaml index d547f5cee..8f9300b4a 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -239,6 +239,11 @@ tracing: exporter: "otlp_grpc" endpoint: "localhost" trusted_ca: "" + attributes: + - key: key0 + value: value + - key: key1 + value: value runtime: soft_memory_limit: 1gb From e122ff6013f63898471ecb95c06073d925f50698 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 12 Nov 2024 11:34:58 +0300 Subject: [PATCH 1523/1943] [#1488] dev: Add Jaeger image and enable tracing on debug Signed-off-by: Dmitrii Stepanov --- dev/.vscode-example/launch.json | 28 ++++++++++++++++++++++++---- dev/docker-compose.yml | 12 ++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/dev/.vscode-example/launch.json b/dev/.vscode-example/launch.json index 990fd42a8..6abf5ecdc 100644 --- a/dev/.vscode-example/launch.json +++ b/dev/.vscode-example/launch.json @@ -78,7 +78,12 @@ "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s1/pilorama1", "FROSTFS_PROMETHEUS_ENABLED":"true", "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9090", - "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s", + "FROSTFS_TRACING_ENABLED":"true", + "FROSTFS_TRACING_EXPORTER":"otlp_grpc", + "FROSTFS_TRACING_ENDPOINT":"127.0.0.1:4317", + "FROSTFS_TRACING_ATTRIBUTES_0_KEY":"host.ip", + "FROSTFS_TRACING_ATTRIBUTES_0_VALUE":"127.0.0.1:8080" }, "postDebugTask": "env-down" }, @@ -129,7 +134,12 @@ "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s2/pilorama1", "FROSTFS_PROMETHEUS_ENABLED":"true", "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9091", - "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s", + "FROSTFS_TRACING_ENABLED":"true", + "FROSTFS_TRACING_EXPORTER":"otlp_grpc", + "FROSTFS_TRACING_ENDPOINT":"127.0.0.1:4317", + "FROSTFS_TRACING_ATTRIBUTES_0_KEY":"host.ip", + "FROSTFS_TRACING_ATTRIBUTES_0_VALUE":"127.0.0.1:8082" }, "postDebugTask": "env-down" }, @@ -180,7 +190,12 @@ "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s3/pilorama1", "FROSTFS_PROMETHEUS_ENABLED":"true", "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9092", - "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s", + "FROSTFS_TRACING_ENABLED":"true", + "FROSTFS_TRACING_EXPORTER":"otlp_grpc", + "FROSTFS_TRACING_ENDPOINT":"127.0.0.1:4317", + "FROSTFS_TRACING_ATTRIBUTES_0_KEY":"host.ip", + "FROSTFS_TRACING_ATTRIBUTES_0_VALUE":"127.0.0.1:8084" }, "postDebugTask": "env-down" }, @@ -231,7 +246,12 @@ "FROSTFS_STORAGE_SHARD_1_PILORAMA_PATH":"${workspaceFolder}/.cache/storage/s4/pilorama1", "FROSTFS_PROMETHEUS_ENABLED":"true", "FROSTFS_PROMETHEUS_ADDRESS":"127.0.0.1:9093", - "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s" + "FROSTFS_PROMETHEUS_SHUTDOWN_TIMEOUT":"15s", + "FROSTFS_TRACING_ENABLED":"true", + "FROSTFS_TRACING_EXPORTER":"otlp_grpc", + "FROSTFS_TRACING_ENDPOINT":"127.0.0.1:4317", + "FROSTFS_TRACING_ATTRIBUTES_0_KEY":"host.ip", + "FROSTFS_TRACING_ATTRIBUTES_0_VALUE":"127.0.0.1:8086" }, "postDebugTask": "env-down" } diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index be1956e65..40ed35aeb 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -14,3 +14,15 @@ services: - ./neo-go/node-wallet.json:/wallets/node-wallet.json - ./neo-go/config.yml:/wallets/config.yml - ./neo-go/wallet.json:/wallets/wallet.json + jaeger: + image: jaegertracing/all-in-one:latest + container_name: jaeger + ports: + - '4317:4317' #OTLP over gRPC + - '4318:4318' #OTLP over HTTP + - '16686:16686' #frontend + stop_signal: SIGKILL + environment: + - COLLECTOR_OTLP_ENABLED=true + - SPAN_STORAGE_TYPE=badger + - BADGER_EPHEMERAL=true From f1556e3c42764499c2a7f4f9c75f8b13a7d643d7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 12 Nov 2024 11:35:20 +0300 Subject: [PATCH 1524/1943] [#1488] Makefile: Drop all containers created on env-up Signed-off-by: Dmitrii Stepanov --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 68a31febe..c54601c73 100755 --- a/Makefile +++ b/Makefile @@ -282,7 +282,6 @@ env-up: all # Shutdown dev environment env-down: - docker compose -f dev/docker-compose.yml down - docker volume rm -f frostfs-node_neo-go + docker compose -f dev/docker-compose.yml down -v rm -rf ./$(TMP_DIR)/state rm -rf ./$(TMP_DIR)/storage From b451de94c81a7a40f2afa712dd56f9c84f5be1c1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 13 Nov 2024 10:02:38 +0300 Subject: [PATCH 1525/1943] [#1492] metabase: Fix typo in objData Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/get.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index 776f5d27c..6a35d47f3 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -216,10 +216,10 @@ func getECInfoError(tx *bbolt.Tx, cnr cid.ID, data []byte) error { ecInfo := objectSDK.NewECInfo() for _, key := range keys { // check in primary index - ojbData := getFromBucket(tx, primaryBucketName(cnr, make([]byte, bucketKeySize)), key) - if len(ojbData) != 0 { + objData := getFromBucket(tx, primaryBucketName(cnr, make([]byte, bucketKeySize)), key) + if len(objData) != 0 { obj := objectSDK.New() - if err := obj.Unmarshal(ojbData); err != nil { + if err := obj.Unmarshal(objData); err != nil { return err } chunk := objectSDK.ECChunk{} From 8ed7a676d50e24489e2abeb5269d6eb3332df1f8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 13 Nov 2024 10:07:32 +0300 Subject: [PATCH 1526/1943] [#1492] metabase: Ensure Unmarshal() is called on a cloned slice The slice returned from bucket.Get() is only valid during the tx lifetime. Cloning it is not necessary everywhere, but better safe than sorry. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/exists.go | 3 ++- pkg/local_object_storage/metabase/get.go | 4 ++-- pkg/local_object_storage/metabase/iterators.go | 3 ++- pkg/local_object_storage/metabase/list.go | 4 ++-- pkg/local_object_storage/metabase/put.go | 3 ++- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index 2e1b1dce8..0294dd3ba 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -1,6 +1,7 @@ package meta import ( + "bytes" "context" "fmt" "time" @@ -215,7 +216,7 @@ func getSplitInfo(tx *bbolt.Tx, cnr cid.ID, key []byte) (*objectSDK.SplitInfo, e splitInfo := objectSDK.NewSplitInfo() - err := splitInfo.Unmarshal(rawSplitInfo) + err := splitInfo.Unmarshal(bytes.Clone(rawSplitInfo)) if err != nil { return nil, fmt.Errorf("can't unmarshal split info from root index: %w", err) } diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index 6a35d47f3..1cbf78ab2 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -185,7 +185,7 @@ func getVirtualObject(tx *bbolt.Tx, cnr cid.ID, key []byte, raw bool) (*objectSD child := objectSDK.New() - err = child.Unmarshal(data) + err = child.Unmarshal(bytes.Clone(data)) if err != nil { return nil, fmt.Errorf("can't unmarshal child with parent: %w", err) } @@ -219,7 +219,7 @@ func getECInfoError(tx *bbolt.Tx, cnr cid.ID, data []byte) error { objData := getFromBucket(tx, primaryBucketName(cnr, make([]byte, bucketKeySize)), key) if len(objData) != 0 { obj := objectSDK.New() - if err := obj.Unmarshal(objData); err != nil { + if err := obj.Unmarshal(bytes.Clone(objData)); err != nil { return err } chunk := objectSDK.ECChunk{} diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index d44c51fb2..1809bfd2d 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -1,6 +1,7 @@ package meta import ( + "bytes" "context" "errors" "fmt" @@ -195,7 +196,7 @@ func iteratePhyObjects(tx *bbolt.Tx, f func(cid.ID, oid.ID, *objectSDK.Object) e } return b.ForEach(func(k, v []byte) error { - if oid.Decode(k) == nil && obj.Unmarshal(v) == nil { + if oid.Decode(k) == nil && obj.Unmarshal(bytes.Clone(v)) == nil { return f(cid, oid, obj) } diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index b007ef0da..a7ff2222f 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -247,7 +247,7 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket var ecInfo *objectcore.ECInfo if objType == objectSDK.TypeRegular { var o objectSDK.Object - if err := o.Unmarshal(v); err != nil { + if err := o.Unmarshal(bytes.Clone(v)); err != nil { return nil, nil, nil, err } isLinkingObj = isLinkObject(&o) @@ -413,7 +413,7 @@ func (db *DB) iterateOverObjectsInContainer(ctx context.Context, tx *bbolt.Tx, p var ecInfo *objectcore.ECInfo if prm.ObjectType == objectSDK.TypeRegular { var o objectSDK.Object - if err := o.Unmarshal(v); err != nil { + if err := o.Unmarshal(bytes.Clone(v)); err != nil { return err } isLinkingObj = isLinkObject(&o) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 09c5e04ad..2d94e7ae1 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -1,6 +1,7 @@ package meta import ( + "bytes" "context" "encoding/binary" "errors" @@ -313,7 +314,7 @@ func updateSplitInfoIndex(tx *bbolt.Tx, objKey []byte, cnr cid.ID, bucketName [] return si.Marshal() default: oldSI := objectSDK.NewSplitInfo() - if err := oldSI.Unmarshal(old); err != nil { + if err := oldSI.Unmarshal(bytes.Clone(old)); err != nil { return nil, err } si = util.MergeSplitInfo(si, oldSI) From fd004add00fd35924e70dd450221dc74bc285cb9 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 13 Nov 2024 10:08:17 +0300 Subject: [PATCH 1527/1943] [#1492] metabase: Fix import formatting Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/iterators.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index 1809bfd2d..5d42e4125 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" "go.opentelemetry.io/otel/attribute" From c16dae8b4d76c55037a56d9382d359fa91dc27f6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 18 Oct 2024 13:31:53 +0300 Subject: [PATCH 1528/1943] [#1437] logger: Use context to log trace id Signed-off-by: Dmitrii Stepanov --- pkg/util/logger/log.go | 40 ++++++++++++++++++++++++++++++++++ pkg/util/logger/logger.go | 16 +++++++++++--- pkg/util/logger/test/logger.go | 11 +++++----- 3 files changed, 59 insertions(+), 8 deletions(-) create mode 100644 pkg/util/logger/log.go diff --git a/pkg/util/logger/log.go b/pkg/util/logger/log.go new file mode 100644 index 000000000..269e07d90 --- /dev/null +++ b/pkg/util/logger/log.go @@ -0,0 +1,40 @@ +package logger + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" + "go.uber.org/zap" +) + +func (l *Logger) Debug(ctx context.Context, msg string, fields ...zap.Field) { + if traceID := tracing.GetTraceID(ctx); traceID != "" { + l.z.Debug(msg, append(fields, zap.String("trace_id", traceID))...) + return + } + l.z.Debug(msg, fields...) +} + +func (l *Logger) Info(ctx context.Context, msg string, fields ...zap.Field) { + if traceID := tracing.GetTraceID(ctx); traceID != "" { + l.z.Info(msg, append(fields, zap.String("trace_id", traceID))...) + return + } + l.z.Info(msg, fields...) +} + +func (l *Logger) Warn(ctx context.Context, msg string, fields ...zap.Field) { + if traceID := tracing.GetTraceID(ctx); traceID != "" { + l.z.Warn(msg, append(fields, zap.String("trace_id", traceID))...) + return + } + l.z.Warn(msg, fields...) +} + +func (l *Logger) Error(ctx context.Context, msg string, fields ...zap.Field) { + if traceID := tracing.GetTraceID(ctx); traceID != "" { + l.z.Error(msg, append(fields, zap.String("trace_id", traceID))...) + return + } + l.z.Error(msg, fields...) +} diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index 4b60f02de..d4ac2ab02 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -12,7 +12,7 @@ import ( // Logger represents a component // for writing messages to log. type Logger struct { - *zap.Logger + z *zap.Logger lvl zap.AtomicLevel } @@ -136,7 +136,7 @@ func newConsoleLogger(prm *Prm) (*Logger, error) { return nil, err } - l := &Logger{Logger: lZap, lvl: lvl} + l := &Logger{z: lZap, lvl: lvl} prm._log = l return l, nil @@ -169,7 +169,7 @@ func newJournaldLogger(prm *Prm) (*Logger, error) { lZap := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) - l := &Logger{Logger: lZap, lvl: lvl} + l := &Logger{z: lZap, lvl: lvl} prm._log = l return l, nil @@ -179,3 +179,13 @@ func (l *Logger) reload(prm Prm) error { l.lvl.SetLevel(prm.level) return nil } + +func (l *Logger) WithOptions(options ...zap.Option) { + l.z = l.z.WithOptions(options...) +} + +func NewLoggerWrapper(z *zap.Logger) *Logger { + return &Logger{ + z: z, + } +} diff --git a/pkg/util/logger/test/logger.go b/pkg/util/logger/test/logger.go index f93756d17..b5b0a31eb 100644 --- a/pkg/util/logger/test/logger.go +++ b/pkg/util/logger/test/logger.go @@ -11,9 +11,10 @@ import ( // NewLogger creates a new logger. func NewLogger(t testing.TB) *logger.Logger { - var l logger.Logger - l.Logger = zaptest.NewLogger(t, - zaptest.Level(zapcore.DebugLevel), - zaptest.WrapOptions(zap.Development(), zap.AddCaller())) - return &l + return logger.NewLoggerWrapper( + zaptest.NewLogger(t, + zaptest.Level(zapcore.DebugLevel), + zaptest.WrapOptions(zap.Development(), zap.AddCaller()), + ), + ) } From 6db46257c0d30ddba71138de248aa86c18611730 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 21 Oct 2024 10:22:54 +0300 Subject: [PATCH 1529/1943] [#1437] node: Use ctx for logging Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/config.go | 25 +++---- cmd/frostfs-ir/httpcomponent.go | 15 ++-- cmd/frostfs-ir/main.go | 12 ++-- cmd/frostfs-ir/pprof.go | 7 +- cmd/frostfs-node/config.go | 68 +++++++++--------- cmd/frostfs-node/container.go | 6 +- cmd/frostfs-node/control.go | 6 +- cmd/frostfs-node/grpc.go | 29 ++++---- cmd/frostfs-node/httpcomponent.go | 4 +- cmd/frostfs-node/main.go | 20 +++--- cmd/frostfs-node/morph.go | 22 +++--- cmd/frostfs-node/netmap.go | 10 +-- cmd/frostfs-node/object.go | 6 +- cmd/frostfs-node/runtime.go | 5 +- cmd/frostfs-node/tracing.go | 6 +- cmd/frostfs-node/tree.go | 10 +-- internal/audit/request.go | 4 +- pkg/core/object/fmt_test.go | 12 ++-- pkg/core/object/sender_classifier.go | 5 +- pkg/innerring/initialization.go | 12 ++-- pkg/innerring/innerring.go | 14 ++-- pkg/innerring/notary.go | 8 +-- pkg/innerring/processors/alphabet/handlers.go | 6 +- .../processors/alphabet/process_emit.go | 19 ++--- .../processors/alphabet/processor.go | 3 +- pkg/innerring/processors/balance/handlers.go | 5 +- .../processors/balance/process_assets.go | 6 +- pkg/innerring/processors/balance/processor.go | 3 +- .../processors/container/handlers.go | 9 +-- .../processors/container/process_container.go | 13 ++-- .../processors/container/processor.go | 3 +- pkg/innerring/processors/frostfs/handlers.go | 17 ++--- .../processors/frostfs/process_assets.go | 24 ++++--- .../processors/frostfs/process_config.go | 6 +- pkg/innerring/processors/frostfs/processor.go | 3 +- .../processors/governance/handlers.go | 6 +- .../processors/governance/process_update.go | 29 ++++---- pkg/innerring/processors/netmap/handlers.go | 23 +++--- .../processors/netmap/process_cleanup.go | 12 ++-- .../processors/netmap/process_epoch.go | 16 +++-- .../processors/netmap/process_peers.go | 19 ++--- pkg/innerring/processors/netmap/processor.go | 3 +- pkg/innerring/state.go | 15 ++-- .../blobovnicza/blobovnicza.go | 4 +- .../blobovnicza/control.go | 19 ++--- .../blobovnicza/delete.go | 2 +- .../blobstor/blobovniczatree/control.go | 6 +- .../blobstor/blobovniczatree/delete.go | 2 +- .../blobstor/blobovniczatree/exists.go | 2 +- .../blobstor/blobovniczatree/get.go | 2 +- .../blobstor/blobovniczatree/get_range.go | 2 +- .../blobstor/blobovniczatree/iterate.go | 4 +- .../blobstor/blobovniczatree/manager.go | 7 +- .../blobstor/blobovniczatree/option.go | 2 +- .../blobstor/blobovniczatree/put.go | 6 +- .../blobstor/blobovniczatree/rebuild.go | 50 ++++++------- pkg/local_object_storage/blobstor/blobstor.go | 4 +- pkg/local_object_storage/blobstor/control.go | 8 +-- pkg/local_object_storage/blobstor/delete.go | 4 +- pkg/local_object_storage/blobstor/exists.go | 2 +- .../blobstor/fstree/fstree.go | 6 +- .../blobstor/fstree/option.go | 2 +- pkg/local_object_storage/blobstor/iterate.go | 2 +- pkg/local_object_storage/blobstor/logger.go | 6 +- pkg/local_object_storage/blobstor/put.go | 2 +- pkg/local_object_storage/blobstor/rebuild.go | 4 +- pkg/local_object_storage/engine/control.go | 14 ++-- pkg/local_object_storage/engine/delete.go | 6 +- pkg/local_object_storage/engine/engine.go | 20 +++--- pkg/local_object_storage/engine/evacuate.go | 28 ++++---- pkg/local_object_storage/engine/get.go | 2 +- pkg/local_object_storage/engine/inhume.go | 24 +++---- pkg/local_object_storage/engine/put.go | 8 +-- pkg/local_object_storage/engine/range.go | 2 +- .../engine/remove_copies.go | 8 +-- pkg/local_object_storage/engine/shards.go | 16 ++--- pkg/local_object_storage/internal/log/log.go | 6 +- pkg/local_object_storage/metabase/control.go | 6 +- pkg/local_object_storage/metabase/db.go | 2 +- pkg/local_object_storage/metabase/delete.go | 2 +- pkg/local_object_storage/metabase/inhume.go | 2 +- pkg/local_object_storage/metabase/put.go | 2 +- .../metabase/upgrade_test.go | 10 +-- pkg/local_object_storage/shard/control.go | 16 ++--- pkg/local_object_storage/shard/delete.go | 6 +- pkg/local_object_storage/shard/gc.go | 70 +++++++++---------- pkg/local_object_storage/shard/get.go | 6 +- pkg/local_object_storage/shard/id.go | 3 +- pkg/local_object_storage/shard/inhume.go | 2 +- pkg/local_object_storage/shard/list.go | 2 +- pkg/local_object_storage/shard/lock_test.go | 2 +- pkg/local_object_storage/shard/mode.go | 6 +- pkg/local_object_storage/shard/put.go | 2 +- pkg/local_object_storage/shard/rebuild.go | 8 +-- pkg/local_object_storage/shard/shard.go | 10 +-- pkg/local_object_storage/shard/writecache.go | 6 +- pkg/local_object_storage/writecache/cache.go | 2 +- pkg/local_object_storage/writecache/delete.go | 2 +- pkg/local_object_storage/writecache/flush.go | 4 +- .../writecache/flush_test.go | 2 +- pkg/local_object_storage/writecache/mode.go | 4 +- .../writecache/options.go | 2 +- pkg/local_object_storage/writecache/put.go | 2 +- .../writecache/storage.go | 4 +- pkg/morph/client/client.go | 10 +-- pkg/morph/client/constructor.go | 6 +- pkg/morph/client/multi.go | 8 +-- pkg/morph/client/notary.go | 9 +-- pkg/morph/event/listener.go | 70 +++++++++---------- pkg/morph/event/utils.go | 3 +- pkg/morph/subscriber/subscriber.go | 10 +-- pkg/services/apemanager/executor.go | 2 +- pkg/services/object/acl/v2/service.go | 2 +- pkg/services/object/common/writer/common.go | 2 +- pkg/services/object/common/writer/ec.go | 8 +-- pkg/services/object/delete/delete.go | 6 +- pkg/services/object/delete/exec.go | 18 ++--- pkg/services/object/delete/local.go | 6 +- pkg/services/object/delete/service.go | 4 +- pkg/services/object/get/assemble.go | 10 +-- pkg/services/object/get/assembleec.go | 10 +-- pkg/services/object/get/assemblerec.go | 18 ++--- pkg/services/object/get/container.go | 12 ++-- pkg/services/object/get/get.go | 14 ++-- pkg/services/object/get/local.go | 2 +- pkg/services/object/get/remote.go | 4 +- pkg/services/object/get/request.go | 14 ++-- pkg/services/object/get/service.go | 4 +- pkg/services/object/get/v2/get_range_hash.go | 8 +-- pkg/services/object/get/v2/service.go | 4 +- pkg/services/object/put/service.go | 2 +- pkg/services/object/put/single.go | 2 +- pkg/services/object/search/container.go | 16 ++--- pkg/services/object/search/exec.go | 4 +- pkg/services/object/search/local.go | 2 +- pkg/services/object/search/search.go | 8 +-- pkg/services/object/search/service.go | 4 +- pkg/services/object/util/log.go | 6 +- .../object_manager/tombstone/checker.go | 4 +- .../object_manager/tombstone/constructor.go | 2 +- pkg/services/policer/check.go | 10 +-- pkg/services/policer/ec.go | 40 +++++------ pkg/services/policer/option.go | 2 +- pkg/services/policer/policer.go | 3 +- pkg/services/policer/process.go | 8 +-- pkg/services/replicator/process.go | 8 +-- pkg/services/replicator/pull.go | 8 +-- pkg/services/replicator/put.go | 6 +- pkg/services/replicator/replicator.go | 2 +- pkg/services/session/executor.go | 2 +- .../session/storage/persistent/options.go | 2 +- .../session/storage/persistent/storage.go | 7 +- pkg/services/tree/redirect.go | 2 +- pkg/services/tree/replicator.go | 8 +-- pkg/services/tree/service.go | 2 +- pkg/services/tree/sync.go | 36 +++++----- pkg/util/logger/logger.go | 4 ++ 157 files changed, 764 insertions(+), 713 deletions(-) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 137e764ed..34d9d5595 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -1,6 +1,7 @@ package main import ( + "context" "os" "os/signal" "syscall" @@ -58,13 +59,13 @@ func watchForSignal(cancel func()) { // signals causing application to shut down should have priority over // reconfiguration signal case <-ch: - log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + log.Info(context.Background(), logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) cancel() shutdown() - log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) + log.Info(context.Background(), logs.FrostFSNodeTerminationSignalProcessingIsComplete) return case err := <-intErr: // internal application error - log.Info(logs.FrostFSIRInternalError, zap.String("msg", err.Error())) + log.Info(context.Background(), logs.FrostFSIRInternalError, zap.String("msg", err.Error())) cancel() shutdown() return @@ -72,35 +73,35 @@ func watchForSignal(cancel func()) { // block until any signal is receieved select { case <-ch: - log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + log.Info(context.Background(), logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) cancel() shutdown() - log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) + log.Info(context.Background(), logs.FrostFSNodeTerminationSignalProcessingIsComplete) return case err := <-intErr: // internal application error - log.Info(logs.FrostFSIRInternalError, zap.String("msg", err.Error())) + log.Info(context.Background(), logs.FrostFSIRInternalError, zap.String("msg", err.Error())) cancel() shutdown() return case <-sighupCh: - log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) + log.Info(context.Background(), logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) if !innerRing.CompareAndSwapHealthStatus(control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) { - log.Info(logs.FrostFSNodeSIGHUPSkip) + log.Info(context.Background(), logs.FrostFSNodeSIGHUPSkip) break } err := reloadConfig() if err != nil { - log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) + log.Error(context.Background(), logs.FrostFSNodeConfigurationReading, zap.Error(err)) } pprofCmp.reload() metricsCmp.reload() - log.Info(logs.FrostFSIRReloadExtraWallets) + log.Info(context.Background(), logs.FrostFSIRReloadExtraWallets) err = innerRing.SetExtraWallets(cfg) if err != nil { - log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) + log.Error(context.Background(), logs.FrostFSNodeConfigurationReading, zap.Error(err)) } innerRing.CompareAndSwapHealthStatus(control.HealthStatus_RECONFIGURING, control.HealthStatus_READY) - log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) + log.Info(context.Background(), logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } } } diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index 2792c3548..685ef61ad 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -1,6 +1,7 @@ package main import ( + "context" "net/http" "time" @@ -25,7 +26,7 @@ const ( ) func (c *httpComponent) init() { - log.Info("init " + c.name) + log.Info(context.Background(), "init "+c.name) c.enabled = cfg.GetBool(c.name + enabledKeyPostfix) c.address = cfg.GetString(c.name + addressKeyPostfix) c.shutdownDur = cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) @@ -39,14 +40,14 @@ func (c *httpComponent) init() { httputil.WithShutdownTimeout(c.shutdownDur), ) } else { - log.Info(c.name + " is disabled, skip") + log.Info(context.Background(), c.name+" is disabled, skip") c.srv = nil } } func (c *httpComponent) start() { if c.srv != nil { - log.Info("start " + c.name) + log.Info(context.Background(), "start "+c.name) wg.Add(1) go func() { defer wg.Done() @@ -57,7 +58,7 @@ func (c *httpComponent) start() { func (c *httpComponent) shutdown() error { if c.srv != nil { - log.Info("shutdown " + c.name) + log.Info(context.Background(), "shutdown "+c.name) return c.srv.Shutdown() } return nil @@ -71,11 +72,11 @@ func (c *httpComponent) needReload() bool { } func (c *httpComponent) reload() { - log.Info("reload " + c.name) + log.Info(context.Background(), "reload "+c.name) if c.needReload() { - log.Info(c.name + " config updated") + log.Info(context.Background(), c.name+" config updated") if err := c.shutdown(); err != nil { - log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + log.Debug(context.Background(), logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error()), ) } else { diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 4bc5923a0..55a8ce00d 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -103,32 +103,32 @@ func main() { err = innerRing.Start(ctx, intErr) exitErr(err) - log.Info(logs.CommonApplicationStarted, + log.Info(ctx, logs.CommonApplicationStarted, zap.String("version", misc.Version)) watchForSignal(cancel) <-ctx.Done() // graceful shutdown - log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop) + log.Debug(ctx, logs.FrostFSNodeWaitingForAllProcessesToStop) wg.Wait() - log.Info(logs.FrostFSIRApplicationStopped) + log.Info(ctx, logs.FrostFSIRApplicationStopped) } func shutdown() { innerRing.Stop() if err := metricsCmp.shutdown(); err != nil { - log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + log.Debug(context.Background(), logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error()), ) } if err := pprofCmp.shutdown(); err != nil { - log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + log.Debug(context.Background(), logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error()), ) } if err := sdnotify.ClearStatus(); err != nil { - log.Error(logs.FailedToReportStatusToSystemd, zap.Error(err)) + log.Error(context.Background(), logs.FailedToReportStatusToSystemd, zap.Error(err)) } } diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go index ff5642008..e95fd117f 100644 --- a/cmd/frostfs-ir/pprof.go +++ b/cmd/frostfs-ir/pprof.go @@ -1,6 +1,7 @@ package main import ( + "context" "runtime" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -52,11 +53,11 @@ func (c *pprofComponent) needReload() bool { } func (c *pprofComponent) reload() { - log.Info("reload " + c.name) + log.Info(context.Background(), "reload "+c.name) if c.needReload() { - log.Info(c.name + " config updated") + log.Info(context.Background(), c.name+" config updated") if err := c.shutdown(); err != nil { - log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + log.Debug(context.Background(), logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error())) return } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 902187560..bd1b99095 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -400,13 +400,13 @@ type internals struct { func (c *cfg) startMaintenance() { c.isMaintenance.Store(true) c.cfgNetmap.state.setControlNetmapStatus(control.NetmapStatus_MAINTENANCE) - c.log.Info(logs.FrostFSNodeStartedLocalNodesMaintenance) + c.log.Info(context.Background(), logs.FrostFSNodeStartedLocalNodesMaintenance) } // stops node's maintenance. func (c *internals) stopMaintenance() { if c.isMaintenance.CompareAndSwap(true, false) { - c.log.Info(logs.FrostFSNodeStoppedLocalNodesMaintenance) + c.log.Info(context.Background(), logs.FrostFSNodeStoppedLocalNodesMaintenance) } } @@ -705,7 +705,7 @@ func initCfg(appCfg *config.Config) *cfg { log, err := logger.NewLogger(logPrm) fatalOnErr(err) if loggerconfig.ToLokiConfig(appCfg).Enabled { - log.Logger = log.Logger.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + log.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { lokiCore := lokicore.New(core, loggerconfig.ToLokiConfig(appCfg)) return lokiCore })) @@ -1103,10 +1103,10 @@ func initLocalStorage(ctx context.Context, c *cfg) { shard.WithTombstoneSource(c.createTombstoneSource()), shard.WithContainerInfoProvider(c.createContainerInfoProvider(ctx)))...) if err != nil { - c.log.Error(logs.FrostFSNodeFailedToAttachShardToEngine, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeFailedToAttachShardToEngine, zap.Error(err)) } else { shardsAttached++ - c.log.Info(logs.FrostFSNodeShardAttachedToEngine, zap.Stringer("id", id)) + c.log.Info(ctx, logs.FrostFSNodeShardAttachedToEngine, zap.Stringer("id", id)) } } if shardsAttached == 0 { @@ -1116,15 +1116,15 @@ func initLocalStorage(ctx context.Context, c *cfg) { c.cfgObject.cfgLocalStorage.localStorage = ls c.onShutdown(func() { - c.log.Info(logs.FrostFSNodeClosingComponentsOfTheStorageEngine) + c.log.Info(ctx, logs.FrostFSNodeClosingComponentsOfTheStorageEngine) err := ls.Close(context.WithoutCancel(ctx)) if err != nil { - c.log.Info(logs.FrostFSNodeStorageEngineClosingFailure, + c.log.Info(ctx, logs.FrostFSNodeStorageEngineClosingFailure, zap.String("error", err.Error()), ) } else { - c.log.Info(logs.FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully) + c.log.Info(ctx, logs.FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully) } }) } @@ -1132,7 +1132,7 @@ func initLocalStorage(ctx context.Context, c *cfg) { func initAccessPolicyEngine(_ context.Context, c *cfg) { var localOverrideDB chainbase.LocalOverrideDatabase if nodeconfig.PersistentPolicyRules(c.appCfg).Path() == "" { - c.log.Warn(logs.FrostFSNodePersistentRuleStorageDBPathIsNotSetInmemoryWillBeUsed) + c.log.Warn(context.Background(), logs.FrostFSNodePersistentRuleStorageDBPathIsNotSetInmemoryWillBeUsed) localOverrideDB = chainbase.NewInmemoryLocalOverrideDatabase() } else { localOverrideDB = chainbase.NewBoltLocalOverrideDatabase( @@ -1157,7 +1157,7 @@ func initAccessPolicyEngine(_ context.Context, c *cfg) { c.onShutdown(func() { if err := ape.LocalOverrideDatabaseCore().Close(); err != nil { - c.log.Warn(logs.FrostFSNodeAccessPolicyEngineClosingFailure, + c.log.Warn(context.Background(), logs.FrostFSNodeAccessPolicyEngineClosingFailure, zap.Error(err), ) } @@ -1209,7 +1209,7 @@ func (c *cfg) setContractNodeInfo(ni *netmap.NodeInfo) { func (c *cfg) updateContractNodeInfo(epoch uint64) { ni, err := c.netmapLocalNodeState(epoch) if err != nil { - c.log.Error(logs.FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch, + c.log.Error(context.Background(), logs.FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch, zap.Uint64("epoch", epoch), zap.String("error", err.Error())) return @@ -1245,13 +1245,13 @@ func (c *cfg) bootstrap() error { // switch to online except when under maintenance st := c.cfgNetmap.state.controlNetmapStatus() if st == control.NetmapStatus_MAINTENANCE { - c.log.Info(logs.FrostFSNodeBootstrappingWithTheMaintenanceState) + c.log.Info(context.Background(), logs.FrostFSNodeBootstrappingWithTheMaintenanceState) return c.bootstrapWithState(func(ni *netmap.NodeInfo) { ni.SetStatus(netmap.Maintenance) }) } - c.log.Info(logs.FrostFSNodeBootstrappingWithOnlineState, + c.log.Info(context.Background(), logs.FrostFSNodeBootstrappingWithOnlineState, zap.Stringer("previous", st), ) @@ -1280,19 +1280,19 @@ func (c *cfg) signalWatcher(ctx context.Context) { // signals causing application to shut down should have priority over // reconfiguration signal case <-ch: - c.log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + c.log.Info(ctx, logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) c.shutdown() - c.log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) + c.log.Info(ctx, logs.FrostFSNodeTerminationSignalProcessingIsComplete) return case err := <-c.internalErr: // internal application error - c.log.Warn(logs.FrostFSNodeInternalApplicationError, + c.log.Warn(ctx, logs.FrostFSNodeInternalApplicationError, zap.String("message", err.Error())) c.shutdown() - c.log.Info(logs.FrostFSNodeInternalErrorProcessingIsComplete) + c.log.Info(ctx, logs.FrostFSNodeInternalErrorProcessingIsComplete) return default: // block until any signal is receieved @@ -1300,19 +1300,19 @@ func (c *cfg) signalWatcher(ctx context.Context) { case <-sighupCh: c.reloadConfig(ctx) case <-ch: - c.log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + c.log.Info(ctx, logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) c.shutdown() - c.log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) + c.log.Info(ctx, logs.FrostFSNodeTerminationSignalProcessingIsComplete) return case err := <-c.internalErr: // internal application error - c.log.Warn(logs.FrostFSNodeInternalApplicationError, + c.log.Warn(ctx, logs.FrostFSNodeInternalApplicationError, zap.String("message", err.Error())) c.shutdown() - c.log.Info(logs.FrostFSNodeInternalErrorProcessingIsComplete) + c.log.Info(ctx, logs.FrostFSNodeInternalErrorProcessingIsComplete) return } } @@ -1320,17 +1320,17 @@ func (c *cfg) signalWatcher(ctx context.Context) { } func (c *cfg) reloadConfig(ctx context.Context) { - c.log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) + c.log.Info(ctx, logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) if !c.compareAndSwapHealthStatus(control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) { - c.log.Info(logs.FrostFSNodeSIGHUPSkip) + c.log.Info(ctx, logs.FrostFSNodeSIGHUPSkip) return } defer c.compareAndSwapHealthStatus(control.HealthStatus_RECONFIGURING, control.HealthStatus_READY) err := c.reloadAppConfig() if err != nil { - c.log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeConfigurationReading, zap.Error(err)) return } @@ -1341,7 +1341,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { logPrm, err := c.loggerPrm() if err != nil { - c.log.Error(logs.FrostFSNodeLoggerConfigurationPreparation, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeLoggerConfigurationPreparation, zap.Error(err)) return } @@ -1362,25 +1362,25 @@ func (c *cfg) reloadConfig(ctx context.Context) { err = c.cfgObject.cfgLocalStorage.localStorage.Reload(ctx, rcfg) if err != nil { - c.log.Error(logs.FrostFSNodeStorageEngineConfigurationUpdate, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeStorageEngineConfigurationUpdate, zap.Error(err)) return } for _, component := range components { err = component.reloadFunc() if err != nil { - c.log.Error(logs.FrostFSNodeUpdatedConfigurationApplying, + c.log.Error(ctx, logs.FrostFSNodeUpdatedConfigurationApplying, zap.String("component", component.name), zap.Error(err)) } } if err := c.dialerSource.Update(internalNetConfig(c.appCfg, c.metricsCollector.MultinetMetrics())); err != nil { - c.log.Error(logs.FailedToUpdateMultinetConfiguration, zap.Error(err)) + c.log.Error(ctx, logs.FailedToUpdateMultinetConfiguration, zap.Error(err)) return } - c.log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) + c.log.Info(ctx, logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } func (c *cfg) getComponents(ctx context.Context, logPrm *logger.Prm) []dCmp { @@ -1403,7 +1403,7 @@ func (c *cfg) getComponents(ctx context.Context, logPrm *logger.Prm) []dCmp { } updated, err := tracing.Setup(ctx, *traceConfig) if updated { - c.log.Info(logs.FrostFSNodeTracingConfigationUpdated) + c.log.Info(ctx, logs.FrostFSNodeTracingConfigationUpdated) } return err }}) @@ -1438,7 +1438,7 @@ func (c *cfg) reloadPools() error { func (c *cfg) reloadPool(p *ants.Pool, newSize int, name string) { oldSize := p.Cap() if oldSize != newSize { - c.log.Info(logs.FrostFSNodePoolConfigurationUpdate, zap.String("field", name), + c.log.Info(context.Background(), logs.FrostFSNodePoolConfigurationUpdate, zap.String("field", name), zap.Int("old", oldSize), zap.Int("new", newSize)) p.Tune(newSize) } @@ -1477,11 +1477,11 @@ func (c *cfg) createContainerInfoProvider(ctx context.Context) container.InfoPro func (c *cfg) shutdown() { old := c.swapHealthStatus(control.HealthStatus_SHUTTING_DOWN) if old == control.HealthStatus_SHUTTING_DOWN { - c.log.Info(logs.FrostFSNodeShutdownSkip) + c.log.Info(context.Background(), logs.FrostFSNodeShutdownSkip) return } if old == control.HealthStatus_STARTING { - c.log.Warn(logs.FrostFSNodeShutdownWhenNotReady) + c.log.Warn(context.Background(), logs.FrostFSNodeShutdownWhenNotReady) } c.ctxCancel() @@ -1491,6 +1491,6 @@ func (c *cfg) shutdown() { } if err := sdnotify.ClearStatus(); err != nil { - c.log.Error(logs.FailedToReportStatusToSystemd, zap.Error(err)) + c.log.Error(context.Background(), logs.FailedToReportStatusToSystemd, zap.Error(err)) } } diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index cc38876ee..1a54f9ffc 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -102,13 +102,13 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c } else { // unlike removal, we expect successful receive of the container // after successful creation, so logging can be useful - c.log.Error(logs.FrostFSNodeReadNewlyCreatedContainerAfterTheNotification, + c.log.Error(context.Background(), logs.FrostFSNodeReadNewlyCreatedContainerAfterTheNotification, zap.Stringer("id", ev.ID), zap.Error(err), ) } - c.log.Debug(logs.FrostFSNodeContainerCreationEventsReceipt, + c.log.Debug(context.Background(), logs.FrostFSNodeContainerCreationEventsReceipt, zap.Stringer("id", ev.ID), ) }) @@ -116,7 +116,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c subscribeToContainerRemoval(c, func(e event.Event) { ev := e.(containerEvent.DeleteSuccess) containerCache.handleRemoval(ev.ID) - c.log.Debug(logs.FrostFSNodeContainerRemovalEventsReceipt, + c.log.Debug(context.Background(), logs.FrostFSNodeContainerRemovalEventsReceipt, zap.Stringer("id", ev.ID), ) }) diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index e1e6e3ac9..ffac23eec 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -46,7 +46,7 @@ func initControlService(c *cfg) { lis, err := net.Listen("tcp", endpoint) if err != nil { - c.log.Error(logs.FrostFSNodeCantListenGRPCEndpointControl, zap.Error(err)) + c.log.Error(context.Background(), logs.FrostFSNodeCantListenGRPCEndpointControl, zap.Error(err)) return } @@ -60,7 +60,7 @@ func initControlService(c *cfg) { c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { runAndLog(ctx, c, serviceNameControl, false, func(context.Context, *cfg) { - c.log.Info(logs.FrostFSNodeStartListeningEndpoint, + c.log.Info(ctx, logs.FrostFSNodeStartListeningEndpoint, zap.String("service", serviceNameControl), zap.String("endpoint", endpoint)) fatalOnErr(c.cfgControlService.server.Serve(lis)) @@ -113,6 +113,6 @@ func (c *cfg) notifySystemd(st control.HealthStatus) { err = sdnotify.Status(fmt.Sprintf("%v", st)) } if err != nil { - c.log.Error(logs.FailedToReportStatusToSystemd, zap.Error(err)) + c.log.Error(context.Background(), logs.FailedToReportStatusToSystemd, zap.Error(err)) } } diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index 3a38b2cca..271810ee6 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/tls" "errors" "net" @@ -30,7 +31,7 @@ func initGRPC(c *cfg) { lis, err := net.Listen("tcp", sc.Endpoint()) if err != nil { c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(sc.Endpoint()) - c.log.Error(logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) + c.log.Error(context.Background(), logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) endpointsToReconnect = append(endpointsToReconnect, sc.Endpoint()) return } @@ -76,19 +77,19 @@ func scheduleReconnect(endpoint string, c *cfg) { } func tryReconnect(endpoint string, c *cfg) bool { - c.log.Info(logs.FrostFSNodeGRPCReconnecting, zap.String("endpoint", endpoint)) + c.log.Info(context.Background(), logs.FrostFSNodeGRPCReconnecting, zap.String("endpoint", endpoint)) serverOpts, found := getGRPCEndpointOpts(endpoint, c) if !found { - c.log.Warn(logs.FrostFSNodeGRPCServerConfigNotFound, zap.String("endpoint", endpoint)) + c.log.Warn(context.Background(), logs.FrostFSNodeGRPCServerConfigNotFound, zap.String("endpoint", endpoint)) return true } lis, err := net.Listen("tcp", endpoint) if err != nil { c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(endpoint) - c.log.Error(logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) - c.log.Warn(logs.FrostFSNodeGRPCReconnectFailed, zap.Duration("next_try_in", c.cfgGRPC.reconnectTimeout)) + c.log.Error(context.Background(), logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) + c.log.Warn(context.Background(), logs.FrostFSNodeGRPCReconnectFailed, zap.Duration("next_try_in", c.cfgGRPC.reconnectTimeout)) return false } c.metricsCollector.GrpcServerMetrics().MarkHealthy(endpoint) @@ -101,7 +102,7 @@ func tryReconnect(endpoint string, c *cfg) bool { c.cfgGRPC.appendAndHandle(endpoint, lis, srv) - c.log.Info(logs.FrostFSNodeGRPCReconnectedSuccessfully, zap.String("endpoint", endpoint)) + c.log.Info(context.Background(), logs.FrostFSNodeGRPCReconnectedSuccessfully, zap.String("endpoint", endpoint)) return true } @@ -143,7 +144,7 @@ func getGrpcServerOpts(c *cfg, sc *grpcconfig.Config) ([]grpc.ServerOption, bool if tlsCfg != nil { cert, err := tls.LoadX509KeyPair(tlsCfg.CertificateFile(), tlsCfg.KeyFile()) if err != nil { - c.log.Error(logs.FrostFSNodeCouldNotReadCertificateFromFile, zap.Error(err)) + c.log.Error(context.Background(), logs.FrostFSNodeCouldNotReadCertificateFromFile, zap.Error(err)) return nil, false } @@ -180,21 +181,21 @@ func serveGRPC(c *cfg) { go func() { defer func() { - c.log.Info(logs.FrostFSNodeStopListeningGRPCEndpoint, + c.log.Info(context.Background(), logs.FrostFSNodeStopListeningGRPCEndpoint, zap.Stringer("endpoint", l.Addr()), ) c.wg.Done() }() - c.log.Info(logs.FrostFSNodeStartListeningEndpoint, + c.log.Info(context.Background(), logs.FrostFSNodeStartListeningEndpoint, zap.String("service", "gRPC"), zap.Stringer("endpoint", l.Addr()), ) if err := s.Serve(l); err != nil { c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(e) - c.log.Error(logs.FrostFSNodeGRPCServerError, zap.Error(err)) + c.log.Error(context.Background(), logs.FrostFSNodeGRPCServerError, zap.Error(err)) c.cfgGRPC.dropConnection(e) scheduleReconnect(e, c) } @@ -203,9 +204,9 @@ func serveGRPC(c *cfg) { } func stopGRPC(name string, s *grpc.Server, l *logger.Logger) { - l = &logger.Logger{Logger: l.With(zap.String("name", name))} + l = l.With(zap.String("name", name)) - l.Info(logs.FrostFSNodeStoppingGRPCServer) + l.Info(context.Background(), logs.FrostFSNodeStoppingGRPCServer) // GracefulStop() may freeze forever, see #1270 done := make(chan struct{}) @@ -217,9 +218,9 @@ func stopGRPC(name string, s *grpc.Server, l *logger.Logger) { select { case <-done: case <-time.After(1 * time.Minute): - l.Info(logs.FrostFSNodeGRPCCannotShutdownGracefullyForcingStop) + l.Info(context.Background(), logs.FrostFSNodeGRPCCannotShutdownGracefullyForcingStop) s.Stop() } - l.Info(logs.FrostFSNodeGRPCServerStoppedSuccessfully) + l.Info(context.Background(), logs.FrostFSNodeGRPCServerStoppedSuccessfully) } diff --git a/cmd/frostfs-node/httpcomponent.go b/cmd/frostfs-node/httpcomponent.go index 2ec20d848..a699a72a1 100644 --- a/cmd/frostfs-node/httpcomponent.go +++ b/cmd/frostfs-node/httpcomponent.go @@ -22,7 +22,7 @@ type httpComponent struct { func (cmp *httpComponent) init(c *cfg) { if !cmp.enabled { - c.log.Info(cmp.name + " is disabled") + c.log.Info(context.Background(), cmp.name+" is disabled") return } // Init server with parameters @@ -39,7 +39,7 @@ func (cmp *httpComponent) init(c *cfg) { go func() { defer c.wg.Done() - c.log.Info(logs.FrostFSNodeStartListeningEndpoint, + c.log.Info(context.Background(), logs.FrostFSNodeStartListeningEndpoint, zap.String("service", cmp.name), zap.String("endpoint", cmp.address)) fatalOnErr(srv.Serve()) diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index e4f0a434c..cd42d5f1d 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -73,9 +73,9 @@ func main() { } func initAndLog(c *cfg, name string, initializer func(*cfg)) { - c.log.Info(fmt.Sprintf("initializing %s service...", name)) + c.log.Info(context.Background(), fmt.Sprintf("initializing %s service...", name)) initializer(c) - c.log.Info(name + " service has been successfully initialized") + c.log.Info(context.Background(), name+" service has been successfully initialized") } func initApp(ctx context.Context, c *cfg) { @@ -120,25 +120,25 @@ func initApp(ctx context.Context, c *cfg) { } func runAndLog(ctx context.Context, c *cfg, name string, logSuccess bool, starter func(context.Context, *cfg)) { - c.log.Info(fmt.Sprintf("starting %s service...", name)) + c.log.Info(ctx, fmt.Sprintf("starting %s service...", name)) starter(ctx, c) if logSuccess { - c.log.Info(name + " service started successfully") + c.log.Info(ctx, name+" service started successfully") } } func stopAndLog(c *cfg, name string, stopper func() error) { - c.log.Debug(fmt.Sprintf("shutting down %s service", name)) + c.log.Debug(context.Background(), fmt.Sprintf("shutting down %s service", name)) err := stopper() if err != nil { - c.log.Debug(fmt.Sprintf("could not shutdown %s server", name), + c.log.Debug(context.Background(), fmt.Sprintf("could not shutdown %s server", name), zap.String("error", err.Error()), ) } - c.log.Debug(name + " service has been stopped") + c.log.Debug(context.Background(), name+" service has been stopped") } func bootUp(ctx context.Context, c *cfg) { @@ -150,7 +150,7 @@ func bootUp(ctx context.Context, c *cfg) { } func wait(c *cfg) { - c.log.Info(logs.CommonApplicationStarted, + c.log.Info(context.Background(), logs.CommonApplicationStarted, zap.String("version", misc.Version)) <-c.done // graceful shutdown @@ -160,12 +160,12 @@ func wait(c *cfg) { go func() { defer drain.Done() for err := range c.internalErr { - c.log.Warn(logs.FrostFSNodeInternalApplicationError, + c.log.Warn(context.Background(), logs.FrostFSNodeInternalApplicationError, zap.String("message", err.Error())) } }() - c.log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop) + c.log.Debug(context.Background(), logs.FrostFSNodeWaitingForAllProcessesToStop) c.wg.Wait() diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 0969f5579..3e010b181 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -48,7 +48,7 @@ func (c *cfg) initMorphComponents(ctx context.Context) { fatalOnErr(err) } - c.log.Info(logs.FrostFSNodeNotarySupport, + c.log.Info(ctx, logs.FrostFSNodeNotarySupport, zap.Bool("sidechain_enabled", c.cfgMorph.notaryEnabled), ) @@ -64,7 +64,7 @@ func (c *cfg) initMorphComponents(ctx context.Context) { msPerBlock, err := c.cfgMorph.client.MsPerBlock() fatalOnErr(err) c.cfgMorph.cacheTTL = time.Duration(msPerBlock) * time.Millisecond - c.log.Debug(logs.FrostFSNodeMorphcacheTTLFetchedFromNetwork, zap.Duration("value", c.cfgMorph.cacheTTL)) + c.log.Debug(ctx, logs.FrostFSNodeMorphcacheTTLFetchedFromNetwork, zap.Duration("value", c.cfgMorph.cacheTTL)) } if c.cfgMorph.cacheTTL < 0 { @@ -102,7 +102,7 @@ func initMorphClient(ctx context.Context, c *cfg) { client.WithDialerSource(c.dialerSource), ) if err != nil { - c.log.Info(logs.FrostFSNodeFailedToCreateNeoRPCClient, + c.log.Info(ctx, logs.FrostFSNodeFailedToCreateNeoRPCClient, zap.Any("endpoints", addresses), zap.String("error", err.Error()), ) @@ -111,12 +111,12 @@ func initMorphClient(ctx context.Context, c *cfg) { } c.onShutdown(func() { - c.log.Info(logs.FrostFSNodeClosingMorphComponents) + c.log.Info(ctx, logs.FrostFSNodeClosingMorphComponents) cli.Close() }) if err := cli.SetGroupSignerScope(); err != nil { - c.log.Info(logs.FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal, zap.Error(err)) + c.log.Info(ctx, logs.FrostFSNodeFailedToSetGroupSignerScopeContinueWithGlobal, zap.Error(err)) } c.cfgMorph.client = cli @@ -136,7 +136,7 @@ func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) { // non-error deposit with an empty TX hash means // that the deposit has already been made; no // need to wait it. - c.log.Info(logs.FrostFSNodeNotaryDepositHasAlreadyBeenMade) + c.log.Info(ctx, logs.FrostFSNodeNotaryDepositHasAlreadyBeenMade) return } @@ -202,7 +202,7 @@ func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256, vub uint32) return fmt.Errorf("could not wait for notary deposit persists in chain: %w", err) } if res.Execution.VMState.HasFlag(vmstate.Halt) { - c.log.Info(logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted) + c.log.Info(ctx, logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted) return nil } return errNotaryDepositFail @@ -217,7 +217,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { fromSideChainBlock, err := c.persistate.UInt32(persistateSideChainLastBlockKey) if err != nil { fromSideChainBlock = 0 - c.log.Warn(logs.FrostFSNodeCantGetLastProcessedSideChainBlockNumber, zap.String("error", err.Error())) + c.log.Warn(ctx, logs.FrostFSNodeCantGetLastProcessedSideChainBlockNumber, zap.String("error", err.Error())) } subs, err = subscriber.New(ctx, &subscriber.Params{ @@ -246,7 +246,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { setNetmapNotificationParser(c, newEpochNotification, func(src *state.ContainedNotificationEvent) (event.Event, error) { res, err := netmapEvent.ParseNewEpoch(src) if err == nil { - c.log.Info(logs.FrostFSNodeNewEpochEventFromSidechain, + c.log.Info(ctx, logs.FrostFSNodeNewEpochEventFromSidechain, zap.Uint64("number", res.(netmapEvent.NewEpoch).EpochNumber()), ) } @@ -257,11 +257,11 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { registerNotificationHandlers(c.cfgContainer.scriptHash, lis, c.cfgContainer.parsers, c.cfgContainer.subscribers) registerBlockHandler(lis, func(block *block.Block) { - c.log.Debug(logs.FrostFSNodeNewBlock, zap.Uint32("index", block.Index)) + c.log.Debug(ctx, logs.FrostFSNodeNewBlock, zap.Uint32("index", block.Index)) err = c.persistate.SetUInt32(persistateSideChainLastBlockKey, block.Index) if err != nil { - c.log.Warn(logs.FrostFSNodeCantUpdatePersistentState, + c.log.Warn(ctx, logs.FrostFSNodeCantUpdatePersistentState, zap.String("chain", "side"), zap.Uint32("block_index", block.Index)) } diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 73871bfc9..18667e636 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -189,7 +189,7 @@ func addNewEpochNotificationHandlers(c *cfg) { } if err := c.bootstrap(); err != nil { - c.log.Warn(logs.FrostFSNodeCantSendRebootstrapTx, zap.Error(err)) + c.log.Warn(context.Background(), logs.FrostFSNodeCantSendRebootstrapTx, zap.Error(err)) } }) @@ -197,7 +197,7 @@ func addNewEpochNotificationHandlers(c *cfg) { addNewEpochAsyncNotificationHandler(c, func(_ event.Event) { _, _, err := makeNotaryDeposit(c) if err != nil { - c.log.Error(logs.FrostFSNodeCouldNotMakeNotaryDeposit, + c.log.Error(context.Background(), logs.FrostFSNodeCouldNotMakeNotaryDeposit, zap.String("error", err.Error()), ) } @@ -210,7 +210,7 @@ func addNewEpochNotificationHandlers(c *cfg) { func bootstrapNode(c *cfg) { if c.needBootstrap() { if c.IsMaintenance() { - c.log.Info(logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) + c.log.Info(context.Background(), logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) return } err := c.bootstrap() @@ -250,7 +250,7 @@ func initNetmapState(c *cfg) { stateWord := nodeState(ni) - c.log.Info(logs.FrostFSNodeInitialNetworkState, + c.log.Info(context.Background(), logs.FrostFSNodeInitialNetworkState, zap.Uint64("epoch", epoch), zap.String("state", stateWord), ) @@ -307,7 +307,7 @@ func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error if nmState != candidateState { // This happens when the node was switched to maintenance without epoch tick. // We expect it to continue staying in maintenance. - c.log.Info(logs.CandidateStatusPriority, + c.log.Info(context.Background(), logs.CandidateStatusPriority, zap.String("netmap", nmState), zap.String("candidate", candidateState)) } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index aeab1d6cb..cad6d5ee3 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -58,7 +58,7 @@ type objectSvc struct { func (c *cfg) MaxObjectSize() uint64 { sz, err := c.cfgNetmap.wrapper.MaxObjectSize() if err != nil { - c.log.Error(logs.FrostFSNodeCouldNotGetMaxObjectSizeValue, + c.log.Error(context.Background(), logs.FrostFSNodeCouldNotGetMaxObjectSizeValue, zap.String("error", err.Error()), ) } @@ -223,7 +223,7 @@ func initObjectService(c *cfg) { func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.ClientCache) { if policerconfig.UnsafeDisable(c.appCfg) { - c.log.Warn(logs.FrostFSNodePolicerIsDisabled) + c.log.Warn(context.Background(), logs.FrostFSNodePolicerIsDisabled) return } @@ -287,7 +287,7 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl _, err := ls.Inhume(ctx, inhumePrm) if err != nil { - c.log.Warn(logs.FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage, + c.log.Warn(ctx, logs.FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage, zap.String("error", err.Error()), ) } diff --git a/cmd/frostfs-node/runtime.go b/cmd/frostfs-node/runtime.go index d858ba490..0b1000e70 100644 --- a/cmd/frostfs-node/runtime.go +++ b/cmd/frostfs-node/runtime.go @@ -1,6 +1,7 @@ package main import ( + "context" "os" "runtime/debug" @@ -12,14 +13,14 @@ import ( func setRuntimeParameters(c *cfg) { if len(os.Getenv("GOMEMLIMIT")) != 0 { // default limit < yaml limit < app env limit < GOMEMLIMIT - c.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) + c.log.Warn(context.Background(), logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) return } memLimitBytes := runtime.GCMemoryLimitBytes(c.appCfg) previous := debug.SetMemoryLimit(memLimitBytes) if memLimitBytes != previous { - c.log.Info(logs.RuntimeSoftMemoryLimitUpdated, + c.log.Info(context.Background(), logs.RuntimeSoftMemoryLimitUpdated, zap.Int64("new_value", memLimitBytes), zap.Int64("old_value", previous)) } diff --git a/cmd/frostfs-node/tracing.go b/cmd/frostfs-node/tracing.go index f550dd882..65f5aec15 100644 --- a/cmd/frostfs-node/tracing.go +++ b/cmd/frostfs-node/tracing.go @@ -13,12 +13,12 @@ import ( func initTracing(ctx context.Context, c *cfg) { conf, err := tracingconfig.ToTracingConfig(c.appCfg) if err != nil { - c.log.Error(logs.FrostFSNodeFailedInitTracing, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeFailedInitTracing, zap.Error(err)) return } _, err = tracing.Setup(ctx, *conf) if err != nil { - c.log.Error(logs.FrostFSNodeFailedInitTracing, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeFailedInitTracing, zap.Error(err)) return } @@ -29,7 +29,7 @@ func initTracing(ctx context.Context, c *cfg) { defer cancel() err := tracing.Shutdown(ctx) // cfg context cancels before close if err != nil { - c.log.Error(logs.FrostFSNodeFailedShutdownTracing, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeFailedShutdownTracing, zap.Error(err)) } }, }) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index a92979daf..59923ee2f 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -44,7 +44,7 @@ func (c cnrSource) List() ([]cid.ID, error) { func initTreeService(c *cfg) { treeConfig := treeconfig.Tree(c.appCfg) if !treeConfig.Enabled() { - c.log.Info(logs.FrostFSNodeTreeServiceIsNotEnabledSkipInitialization) + c.log.Info(context.Background(), logs.FrostFSNodeTreeServiceIsNotEnabledSkipInitialization) return } @@ -83,7 +83,7 @@ func initTreeService(c *cfg) { addNewEpochNotificationHandler(c, func(_ event.Event) { err := c.treeService.SynchronizeAll() if err != nil { - c.log.Error(logs.FrostFSNodeCouldNotSynchronizeTreeService, zap.Error(err)) + c.log.Error(context.Background(), logs.FrostFSNodeCouldNotSynchronizeTreeService, zap.Error(err)) } }) } else { @@ -94,7 +94,7 @@ func initTreeService(c *cfg) { for range tick.C { err := c.treeService.SynchronizeAll() if err != nil { - c.log.Error(logs.FrostFSNodeCouldNotSynchronizeTreeService, zap.Error(err)) + c.log.Error(context.Background(), logs.FrostFSNodeCouldNotSynchronizeTreeService, zap.Error(err)) if errors.Is(err, tree.ErrShuttingDown) { return } @@ -107,11 +107,11 @@ func initTreeService(c *cfg) { ev := e.(containerEvent.DeleteSuccess) // This is executed asynchronously, so we don't care about the operation taking some time. - c.log.Debug(logs.FrostFSNodeRemovingAllTreesForContainer, zap.Stringer("cid", ev.ID)) + c.log.Debug(context.Background(), logs.FrostFSNodeRemovingAllTreesForContainer, zap.Stringer("cid", ev.ID)) err := c.treeService.DropTree(context.Background(), ev.ID, "") if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { // Ignore pilorama.ErrTreeNotFound but other errors, including shard.ErrReadOnly, should be logged. - c.log.Error(logs.FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved, + c.log.Error(context.Background(), logs.FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved, zap.Stringer("cid", ev.ID), zap.String("error", err.Error())) } diff --git a/internal/audit/request.go b/internal/audit/request.go index 3355087f1..15a4a7960 100644 --- a/internal/audit/request.go +++ b/internal/audit/request.go @@ -1,6 +1,8 @@ package audit import ( + "context" + crypto "git.frostfs.info/TrueCloudLab/frostfs-crypto" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -37,7 +39,7 @@ func LogRequestWithKey(log *logger.Logger, operation string, key []byte, target object = target.String() } - log.Info(logs.AuditEventLogRecord, + log.Info(context.Background(), logs.AuditEventLogRecord, zap.String("operation", operation), zap.String("object", object), zap.String("subject", subject), diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index b428b56da..20560cf3a 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -65,7 +65,7 @@ func TestFormatValidator_Validate(t *testing.T) { epoch: curEpoch, }), WithLockSource(ls), - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(logger.NewLoggerWrapper(zaptest.NewLogger(t))), ) ownerKey, err := keys.NewPrivateKey() @@ -290,7 +290,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { }), WithLockSource(ls), WithVerifySessionTokenIssuer(false), - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(logger.NewLoggerWrapper(zaptest.NewLogger(t))), ) tok := sessiontest.Object() @@ -339,7 +339,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { }, }, ), - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(logger.NewLoggerWrapper(zaptest.NewLogger(t))), ) tok := sessiontest.Object() @@ -417,7 +417,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { currentEpoch: curEpoch, }, ), - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(logger.NewLoggerWrapper(zaptest.NewLogger(t))), ) require.NoError(t, v.Validate(context.Background(), obj, false)) @@ -491,7 +491,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { currentEpoch: curEpoch, }, ), - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(logger.NewLoggerWrapper(zaptest.NewLogger(t))), ) require.NoError(t, v.Validate(context.Background(), obj, false)) @@ -567,7 +567,7 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { currentEpoch: curEpoch, }, ), - WithLogger(&logger.Logger{Logger: zaptest.NewLogger(t)}), + WithLogger(logger.NewLoggerWrapper(zaptest.NewLogger(t))), ) require.Error(t, v.Validate(context.Background(), obj, false)) diff --git a/pkg/core/object/sender_classifier.go b/pkg/core/object/sender_classifier.go index 13d0ebfb1..ed438c0b9 100644 --- a/pkg/core/object/sender_classifier.go +++ b/pkg/core/object/sender_classifier.go @@ -2,6 +2,7 @@ package object import ( "bytes" + "context" "crypto/sha256" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -64,7 +65,7 @@ func (c SenderClassifier) IsInnerRingOrContainerNode(ownerKeyInBytes []byte, idC isInnerRingNode, err := c.isInnerRingKey(ownerKeyInBytes) if err != nil { // do not throw error, try best case matching - c.log.Debug(logs.V2CantCheckIfRequestFromInnerRing, + c.log.Debug(context.Background(), logs.V2CantCheckIfRequestFromInnerRing, zap.String("error", err.Error())) } else if isInnerRingNode { return &ClassifyResult{ @@ -81,7 +82,7 @@ func (c SenderClassifier) IsInnerRingOrContainerNode(ownerKeyInBytes []byte, idC // error might happen if request has `RoleOther` key and placement // is not possible for previous epoch, so // do not throw error, try best case matching - c.log.Debug(logs.V2CantCheckIfRequestFromContainerNode, + c.log.Debug(context.Background(), logs.V2CantCheckIfRequestFromContainerNode, zap.String("error", err.Error())) } else if isContainerNode { return &ClassifyResult{ diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index d6b474c32..b8812819e 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -97,7 +97,7 @@ func (s *Server) initMainnet(ctx context.Context, cfg *viper.Viper, morphChain * fromMainChainBlock, err := s.persistate.UInt32(persistateMainChainLastBlockKey) if err != nil { fromMainChainBlock = 0 - s.log.Warn(logs.InnerringCantGetLastProcessedMainChainBlockNumber, zap.String("error", err.Error())) + s.log.Warn(ctx, logs.InnerringCantGetLastProcessedMainChainBlockNumber, zap.String("error", err.Error())) } mainnetChain.from = fromMainChainBlock @@ -142,7 +142,7 @@ func (s *Server) initNotaryConfig() { !s.withoutMainNet && s.mainnetClient.ProbeNotary(), // if mainnet disabled then notary flag must be disabled too ) - s.log.Info(logs.InnerringNotarySupport, + s.log.Info(context.Background(), logs.InnerringNotarySupport, zap.Bool("sidechain_enabled", true), zap.Bool("mainchain_enabled", !s.mainNotaryConfig.disabled), ) @@ -153,7 +153,7 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli if s.withoutMainNet || cfg.GetBool("governance.disable") { alphaSync = func(event.Event) { - s.log.Debug(logs.InnerringAlphabetKeysSyncIsDisabled) + s.log.Debug(context.Background(), logs.InnerringAlphabetKeysSyncIsDisabled) } } else { // create governance processor @@ -307,7 +307,7 @@ func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper) error { func (s *Server) initGRPCServer(cfg *viper.Viper, log *logger.Logger, audit *atomic.Bool) error { controlSvcEndpoint := cfg.GetString("control.grpc.endpoint") if controlSvcEndpoint == "" { - s.log.Info(logs.InnerringNoControlServerEndpointSpecified) + s.log.Info(context.Background(), logs.InnerringNoControlServerEndpointSpecified) return nil } @@ -446,7 +446,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- fromSideChainBlock, err := s.persistate.UInt32(persistateSideChainLastBlockKey) if err != nil { fromSideChainBlock = 0 - s.log.Warn(logs.InnerringCantGetLastProcessedSideChainBlockNumber, zap.String("error", err.Error())) + s.log.Warn(ctx, logs.InnerringCantGetLastProcessedSideChainBlockNumber, zap.String("error", err.Error())) } morphChain := &chainParams{ @@ -471,7 +471,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- return nil, err } if err := s.morphClient.SetGroupSignerScope(); err != nil { - morphChain.log.Info(logs.InnerringFailedToSetGroupSignerScope, zap.Error(err)) + morphChain.log.Info(ctx, logs.InnerringFailedToSetGroupSignerScope, zap.Error(err)) } return morphChain, nil diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 5fae302c4..63a4cb1cb 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -176,7 +176,7 @@ func (s *Server) Start(ctx context.Context, intError chan<- error) (err error) { err = s.voteForSidechainValidator(prm) if err != nil { // we don't stop inner ring execution on this error - s.log.Warn(logs.InnerringCantVoteForPreparedValidators, + s.log.Warn(ctx, logs.InnerringCantVoteForPreparedValidators, zap.String("error", err.Error())) } @@ -218,13 +218,13 @@ func (s *Server) Start(ctx context.Context, intError chan<- error) (err error) { func (s *Server) registerMorphNewBlockEventHandler() { s.morphListener.RegisterBlockHandler(func(b *block.Block) { - s.log.Debug(logs.InnerringNewBlock, + s.log.Debug(context.Background(), logs.InnerringNewBlock, zap.Uint32("index", b.Index), ) err := s.persistate.SetUInt32(persistateSideChainLastBlockKey, b.Index) if err != nil { - s.log.Warn(logs.InnerringCantUpdatePersistentState, + s.log.Warn(context.Background(), logs.InnerringCantUpdatePersistentState, zap.String("chain", "side"), zap.Uint32("block_index", b.Index)) } @@ -238,7 +238,7 @@ func (s *Server) registerMainnetNewBlockEventHandler() { s.mainnetListener.RegisterBlockHandler(func(b *block.Block) { err := s.persistate.SetUInt32(persistateMainChainLastBlockKey, b.Index) if err != nil { - s.log.Warn(logs.InnerringCantUpdatePersistentState, + s.log.Warn(context.Background(), logs.InnerringCantUpdatePersistentState, zap.String("chain", "main"), zap.Uint32("block_index", b.Index)) } @@ -307,7 +307,7 @@ func (s *Server) Stop() { for _, c := range s.closers { if err := c(); err != nil { - s.log.Warn(logs.InnerringCloserError, + s.log.Warn(context.Background(), logs.InnerringCloserError, zap.String("error", err.Error()), ) } @@ -438,7 +438,7 @@ func createListener(ctx context.Context, cli *client.Client, p *chainParams) (ev } listener, err := event.NewListener(event.ListenerParams{ - Logger: &logger.Logger{Logger: p.log.With(zap.String("chain", p.name))}, + Logger: p.log.With(zap.String("chain", p.name)), Subscriber: sub, }) if err != nil { @@ -602,7 +602,7 @@ func (s *Server) initConfigFromBlockchain() error { return err } - s.log.Debug(logs.InnerringReadConfigFromBlockchain, + s.log.Debug(context.Background(), logs.InnerringReadConfigFromBlockchain, zap.Bool("active", s.IsActive()), zap.Bool("alphabet", s.IsAlphabet()), zap.Uint64("epoch", epoch), diff --git a/pkg/innerring/notary.go b/pkg/innerring/notary.go index 5cdbb971c..902a4c30a 100644 --- a/pkg/innerring/notary.go +++ b/pkg/innerring/notary.go @@ -54,12 +54,12 @@ func (s *Server) notaryHandler(_ event.Event) { if !s.mainNotaryConfig.disabled { _, err := s.depositMainNotary() if err != nil { - s.log.Error(logs.InnerringCantMakeNotaryDepositInMainChain, zap.Error(err)) + s.log.Error(context.Background(), logs.InnerringCantMakeNotaryDepositInMainChain, zap.Error(err)) } } if _, err := s.depositSideNotary(); err != nil { - s.log.Error(logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) + s.log.Error(context.Background(), logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) } } @@ -81,11 +81,11 @@ func (s *Server) initNotary(ctx context.Context, deposit depositor, await awaite // non-error deposit with an empty TX hash means // that the deposit has already been made; no // need to wait it. - s.log.Info(logs.InnerringNotaryDepositHasAlreadyBeenMade) + s.log.Info(ctx, logs.InnerringNotaryDepositHasAlreadyBeenMade) return nil } - s.log.Info(msg) + s.log.Info(ctx, msg) return await(ctx, tx) } diff --git a/pkg/innerring/processors/alphabet/handlers.go b/pkg/innerring/processors/alphabet/handlers.go index 9de075f17..0cc2a5f39 100644 --- a/pkg/innerring/processors/alphabet/handlers.go +++ b/pkg/innerring/processors/alphabet/handlers.go @@ -1,6 +1,8 @@ package alphabet import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" @@ -10,14 +12,14 @@ import ( func (ap *Processor) HandleGasEmission(ev event.Event) { _ = ev.(timers.NewAlphabetEmitTick) - ap.log.Info(logs.AlphabetTick, zap.String("type", "alphabet gas emit")) + ap.log.Info(context.Background(), logs.AlphabetTick, zap.String("type", "alphabet gas emit")) // send event to the worker pool err := processors.SubmitEvent(ap.pool, ap.metrics, "alphabet_emit_gas", ap.processEmit) if err != nil { // there system can be moved into controlled degradation stage - ap.log.Warn(logs.AlphabetAlphabetProcessorWorkerPoolDrained, + ap.log.Warn(context.Background(), logs.AlphabetAlphabetProcessorWorkerPoolDrained, zap.Int("capacity", ap.pool.Cap())) } } diff --git a/pkg/innerring/processors/alphabet/process_emit.go b/pkg/innerring/processors/alphabet/process_emit.go index 2317f3e98..142409631 100644 --- a/pkg/innerring/processors/alphabet/process_emit.go +++ b/pkg/innerring/processors/alphabet/process_emit.go @@ -1,6 +1,7 @@ package alphabet import ( + "context" "crypto/elliptic" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -16,14 +17,14 @@ const emitMethod = "emit" func (ap *Processor) processEmit() bool { index := ap.irList.AlphabetIndex() if index < 0 { - ap.log.Info(logs.AlphabetNonAlphabetModeIgnoreGasEmissionEvent) + ap.log.Info(context.Background(), logs.AlphabetNonAlphabetModeIgnoreGasEmissionEvent) return true } contract, ok := ap.alphabetContracts.GetByIndex(index) if !ok { - ap.log.Debug(logs.AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent, + ap.log.Debug(context.Background(), logs.AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent, zap.Int("index", index)) return false @@ -32,20 +33,20 @@ func (ap *Processor) processEmit() bool { // there is no signature collecting, so we don't need extra fee _, err := ap.morphClient.Invoke(contract, 0, emitMethod) if err != nil { - ap.log.Warn(logs.AlphabetCantInvokeAlphabetEmitMethod, zap.String("error", err.Error())) + ap.log.Warn(context.Background(), logs.AlphabetCantInvokeAlphabetEmitMethod, zap.String("error", err.Error())) return false } if ap.storageEmission == 0 { - ap.log.Info(logs.AlphabetStorageNodeEmissionIsOff) + ap.log.Info(context.Background(), logs.AlphabetStorageNodeEmissionIsOff) return true } networkMap, err := ap.netmapClient.NetMap() if err != nil { - ap.log.Warn(logs.AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes, + ap.log.Warn(context.Background(), logs.AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes, zap.String("error", err.Error())) return false @@ -58,7 +59,7 @@ func (ap *Processor) processEmit() bool { ap.pwLock.RUnlock() extraLen := len(pw) - ap.log.Debug(logs.AlphabetGasEmission, + ap.log.Debug(context.Background(), logs.AlphabetGasEmission, zap.Int("network_map", nmLen), zap.Int("extra_wallets", extraLen)) @@ -81,7 +82,7 @@ func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerN key, err := keys.NewPublicKeyFromBytes(keyBytes, elliptic.P256()) if err != nil { - ap.log.Warn(logs.AlphabetCantParseNodePublicKey, + ap.log.Warn(context.Background(), logs.AlphabetCantParseNodePublicKey, zap.String("error", err.Error())) continue @@ -89,7 +90,7 @@ func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerN err = ap.morphClient.TransferGas(key.GetScriptHash(), gasPerNode) if err != nil { - ap.log.Warn(logs.AlphabetCantTransferGas, + ap.log.Warn(context.Background(), logs.AlphabetCantTransferGas, zap.String("receiver", key.Address()), zap.Int64("amount", int64(gasPerNode)), zap.String("error", err.Error()), @@ -106,7 +107,7 @@ func (ap *Processor) transferGasToExtraNodes(pw []util.Uint160, gasPerNode fixed for i, addr := range pw { receiversLog[i] = addr.StringLE() } - ap.log.Warn(logs.AlphabetCantTransferGasToWallet, + ap.log.Warn(context.Background(), logs.AlphabetCantTransferGasToWallet, zap.Strings("receivers", receiversLog), zap.Int64("amount", int64(gasPerNode)), zap.String("error", err.Error()), diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index ce6679969..8dbef1e20 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -1,6 +1,7 @@ package alphabet import ( + "context" "errors" "fmt" "sync" @@ -85,7 +86,7 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/alphabet: global state is not set") } - p.Log.Debug(logs.AlphabetAlphabetWorkerPool, zap.Int("size", p.PoolSize)) + p.Log.Debug(context.Background(), logs.AlphabetAlphabetWorkerPool, zap.Int("size", p.PoolSize)) pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { diff --git a/pkg/innerring/processors/balance/handlers.go b/pkg/innerring/processors/balance/handlers.go index e39f3abbd..3792fc2af 100644 --- a/pkg/innerring/processors/balance/handlers.go +++ b/pkg/innerring/processors/balance/handlers.go @@ -1,6 +1,7 @@ package balance import ( + "context" "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -12,7 +13,7 @@ import ( func (bp *Processor) handleLock(ev event.Event) { lock := ev.(balanceEvent.Lock) - bp.log.Info(logs.Notification, + bp.log.Info(context.Background(), logs.Notification, zap.String("type", "lock"), zap.String("value", hex.EncodeToString(lock.ID()))) @@ -23,7 +24,7 @@ func (bp *Processor) handleLock(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - bp.log.Warn(logs.BalanceBalanceWorkerPoolDrained, + bp.log.Warn(context.Background(), logs.BalanceBalanceWorkerPoolDrained, zap.Int("capacity", bp.pool.Cap())) } } diff --git a/pkg/innerring/processors/balance/process_assets.go b/pkg/innerring/processors/balance/process_assets.go index 1d94fa454..ac6a1e493 100644 --- a/pkg/innerring/processors/balance/process_assets.go +++ b/pkg/innerring/processors/balance/process_assets.go @@ -1,6 +1,8 @@ package balance import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" frostfsContract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" balanceEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/balance" @@ -11,7 +13,7 @@ import ( // back to the withdraw issuer. func (bp *Processor) processLock(lock *balanceEvent.Lock) bool { if !bp.alphabetState.IsAlphabet() { - bp.log.Info(logs.BalanceNonAlphabetModeIgnoreBalanceLock) + bp.log.Info(context.Background(), logs.BalanceNonAlphabetModeIgnoreBalanceLock) return true } @@ -25,7 +27,7 @@ func (bp *Processor) processLock(lock *balanceEvent.Lock) bool { err := bp.frostfsClient.Cheque(prm) if err != nil { - bp.log.Error(logs.BalanceCantSendLockAssetTx, zap.Error(err)) + bp.log.Error(context.Background(), logs.BalanceCantSendLockAssetTx, zap.Error(err)) return false } diff --git a/pkg/innerring/processors/balance/processor.go b/pkg/innerring/processors/balance/processor.go index 5cc849b5c..c4078461e 100644 --- a/pkg/innerring/processors/balance/processor.go +++ b/pkg/innerring/processors/balance/processor.go @@ -1,6 +1,7 @@ package balance import ( + "context" "errors" "fmt" @@ -68,7 +69,7 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/balance: balance precision converter is not set") } - p.Log.Debug(logs.BalanceBalanceWorkerPool, zap.Int("size", p.PoolSize)) + p.Log.Debug(context.Background(), logs.BalanceBalanceWorkerPool, zap.Int("size", p.PoolSize)) pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { diff --git a/pkg/innerring/processors/container/handlers.go b/pkg/innerring/processors/container/handlers.go index a54f3c772..b3d50d9d0 100644 --- a/pkg/innerring/processors/container/handlers.go +++ b/pkg/innerring/processors/container/handlers.go @@ -1,6 +1,7 @@ package container import ( + "context" "crypto/sha256" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -15,7 +16,7 @@ func (cp *Processor) handlePut(ev event.Event) { put := ev.(putEvent) id := sha256.Sum256(put.Container()) - cp.log.Info(logs.Notification, + cp.log.Info(context.Background(), logs.Notification, zap.String("type", "container put"), zap.String("id", base58.Encode(id[:]))) @@ -26,14 +27,14 @@ func (cp *Processor) handlePut(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, + cp.log.Warn(context.Background(), logs.ContainerContainerProcessorWorkerPoolDrained, zap.Int("capacity", cp.pool.Cap())) } } func (cp *Processor) handleDelete(ev event.Event) { del := ev.(containerEvent.Delete) - cp.log.Info(logs.Notification, + cp.log.Info(context.Background(), logs.Notification, zap.String("type", "container delete"), zap.String("id", base58.Encode(del.ContainerID()))) @@ -44,7 +45,7 @@ func (cp *Processor) handleDelete(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - cp.log.Warn(logs.ContainerContainerProcessorWorkerPoolDrained, + cp.log.Warn(context.Background(), logs.ContainerContainerProcessorWorkerPoolDrained, zap.Int("capacity", cp.pool.Cap())) } } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index d89b63e82..2b9c5995c 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -1,6 +1,7 @@ package container import ( + "context" "errors" "fmt" "strings" @@ -38,7 +39,7 @@ var errContainerAndOwnerNamespaceDontMatch = errors.New("container and owner nam // and sending approve tx back to the morph. func (cp *Processor) processContainerPut(put putEvent) bool { if !cp.alphabetState.IsAlphabet() { - cp.log.Info(logs.ContainerNonAlphabetModeIgnoreContainerPut) + cp.log.Info(context.Background(), logs.ContainerNonAlphabetModeIgnoreContainerPut) return true } @@ -48,7 +49,7 @@ func (cp *Processor) processContainerPut(put putEvent) bool { err := cp.checkPutContainer(ctx) if err != nil { - cp.log.Error(logs.ContainerPutContainerCheckFailed, + cp.log.Error(context.Background(), logs.ContainerPutContainerCheckFailed, zap.String("error", err.Error()), ) @@ -56,7 +57,7 @@ func (cp *Processor) processContainerPut(put putEvent) bool { } if err := cp.morphClient.NotarySignAndInvokeTX(ctx.e.NotaryRequest().MainTransaction); err != nil { - cp.log.Error(logs.ContainerCouldNotApprovePutContainer, + cp.log.Error(context.Background(), logs.ContainerCouldNotApprovePutContainer, zap.String("error", err.Error()), ) return false @@ -105,13 +106,13 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { // and sending approve tx back to morph. func (cp *Processor) processContainerDelete(e containerEvent.Delete) bool { if !cp.alphabetState.IsAlphabet() { - cp.log.Info(logs.ContainerNonAlphabetModeIgnoreContainerDelete) + cp.log.Info(context.Background(), logs.ContainerNonAlphabetModeIgnoreContainerDelete) return true } err := cp.checkDeleteContainer(e) if err != nil { - cp.log.Error(logs.ContainerDeleteContainerCheckFailed, + cp.log.Error(context.Background(), logs.ContainerDeleteContainerCheckFailed, zap.String("error", err.Error()), ) @@ -119,7 +120,7 @@ func (cp *Processor) processContainerDelete(e containerEvent.Delete) bool { } if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { - cp.log.Error(logs.ContainerCouldNotApproveDeleteContainer, + cp.log.Error(context.Background(), logs.ContainerCouldNotApproveDeleteContainer, zap.String("error", err.Error()), ) diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index a6fbdc707..7a50ca773 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -1,6 +1,7 @@ package container import ( + "context" "errors" "fmt" @@ -97,7 +98,7 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/container: FrostFSID client is not set") } - p.Log.Debug(logs.ContainerContainerWorkerPool, zap.Int("size", p.PoolSize)) + p.Log.Debug(context.Background(), logs.ContainerContainerWorkerPool, zap.Int("size", p.PoolSize)) pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index c80f9fdc5..02dfbaf60 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -2,6 +2,7 @@ package frostfs import ( "bytes" + "context" "encoding/hex" "slices" @@ -16,7 +17,7 @@ func (np *Processor) handleDeposit(ev event.Event) { deposit := ev.(frostfsEvent.Deposit) depositIDBin := bytes.Clone(deposit.ID()) slices.Reverse(depositIDBin) - np.log.Info(logs.Notification, + np.log.Info(context.Background(), logs.Notification, zap.String("type", "deposit"), zap.String("id", hex.EncodeToString(depositIDBin))) @@ -27,7 +28,7 @@ func (np *Processor) handleDeposit(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, + np.log.Warn(context.Background(), logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } @@ -36,7 +37,7 @@ func (np *Processor) handleWithdraw(ev event.Event) { withdraw := ev.(frostfsEvent.Withdraw) withdrawBin := bytes.Clone(withdraw.ID()) slices.Reverse(withdrawBin) - np.log.Info(logs.Notification, + np.log.Info(context.Background(), logs.Notification, zap.String("type", "withdraw"), zap.String("id", hex.EncodeToString(withdrawBin))) @@ -47,14 +48,14 @@ func (np *Processor) handleWithdraw(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, + np.log.Warn(context.Background(), logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleCheque(ev event.Event) { cheque := ev.(frostfsEvent.Cheque) - np.log.Info(logs.Notification, + np.log.Info(context.Background(), logs.Notification, zap.String("type", "cheque"), zap.String("id", hex.EncodeToString(cheque.ID()))) @@ -65,14 +66,14 @@ func (np *Processor) handleCheque(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, + np.log.Warn(context.Background(), logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleConfig(ev event.Event) { cfg := ev.(frostfsEvent.Config) - np.log.Info(logs.Notification, + np.log.Info(context.Background(), logs.Notification, zap.String("type", "set config"), zap.String("key", hex.EncodeToString(cfg.Key())), zap.String("value", hex.EncodeToString(cfg.Value()))) @@ -84,7 +85,7 @@ func (np *Processor) handleConfig(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(logs.FrostFSFrostfsProcessorWorkerPoolDrained, + np.log.Warn(context.Background(), logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } diff --git a/pkg/innerring/processors/frostfs/process_assets.go b/pkg/innerring/processors/frostfs/process_assets.go index c72aeceee..3bee6ed96 100644 --- a/pkg/innerring/processors/frostfs/process_assets.go +++ b/pkg/innerring/processors/frostfs/process_assets.go @@ -1,6 +1,8 @@ package frostfs import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" @@ -17,7 +19,7 @@ const ( // gas in the sidechain. func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { if !np.alphabetState.IsAlphabet() { - np.log.Info(logs.FrostFSNonAlphabetModeIgnoreDeposit) + np.log.Info(context.Background(), logs.FrostFSNonAlphabetModeIgnoreDeposit) return true } @@ -30,7 +32,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { // send transferX to a balance contract err := np.balanceClient.Mint(prm) if err != nil { - np.log.Error(logs.FrostFSCantTransferAssetsToBalanceContract, zap.Error(err)) + np.log.Error(context.Background(), logs.FrostFSCantTransferAssetsToBalanceContract, zap.Error(err)) } curEpoch := np.epochState.EpochCounter() @@ -44,7 +46,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { val, ok := np.mintEmitCache.Get(receiver.String()) if ok && val+np.mintEmitThreshold >= curEpoch { - np.log.Warn(logs.FrostFSDoubleMintEmissionDeclined, + np.log.Warn(context.Background(), logs.FrostFSDoubleMintEmissionDeclined, zap.Stringer("receiver", receiver), zap.Uint64("last_emission", val), zap.Uint64("current_epoch", curEpoch)) @@ -56,12 +58,12 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { // before gas transfer check if the balance is greater than the threshold balance, err := np.morphClient.GasBalance() if err != nil { - np.log.Error(logs.FrostFSCantGetGasBalanceOfTheNode, zap.Error(err)) + np.log.Error(context.Background(), logs.FrostFSCantGetGasBalanceOfTheNode, zap.Error(err)) return false } if balance < np.gasBalanceThreshold { - np.log.Warn(logs.FrostFSGasBalanceThresholdHasBeenReached, + np.log.Warn(context.Background(), logs.FrostFSGasBalanceThresholdHasBeenReached, zap.Int64("balance", balance), zap.Int64("threshold", np.gasBalanceThreshold)) @@ -70,7 +72,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { err = np.morphClient.TransferGas(receiver, np.mintEmitValue) if err != nil { - np.log.Error(logs.FrostFSCantTransferNativeGasToReceiver, + np.log.Error(context.Background(), logs.FrostFSCantTransferNativeGasToReceiver, zap.String("error", err.Error())) return false @@ -84,14 +86,14 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { // Process withdraw event by locking assets in the balance account. func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) bool { if !np.alphabetState.IsAlphabet() { - np.log.Info(logs.FrostFSNonAlphabetModeIgnoreWithdraw) + np.log.Info(context.Background(), logs.FrostFSNonAlphabetModeIgnoreWithdraw) return true } // create lock account lock, err := util.Uint160DecodeBytesBE(withdraw.ID()[:util.Uint160Size]) if err != nil { - np.log.Error(logs.FrostFSCantCreateLockAccount, zap.Error(err)) + np.log.Error(context.Background(), logs.FrostFSCantCreateLockAccount, zap.Error(err)) return false } @@ -107,7 +109,7 @@ func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) bool { err = np.balanceClient.Lock(prm) if err != nil { - np.log.Error(logs.FrostFSCantLockAssetsForWithdraw, zap.Error(err)) + np.log.Error(context.Background(), logs.FrostFSCantLockAssetsForWithdraw, zap.Error(err)) return false } @@ -118,7 +120,7 @@ func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) bool { // the reserve account. func (np *Processor) processCheque(cheque frostfsEvent.Cheque) bool { if !np.alphabetState.IsAlphabet() { - np.log.Info(logs.FrostFSNonAlphabetModeIgnoreCheque) + np.log.Info(context.Background(), logs.FrostFSNonAlphabetModeIgnoreCheque) return true } @@ -130,7 +132,7 @@ func (np *Processor) processCheque(cheque frostfsEvent.Cheque) bool { err := np.balanceClient.Burn(prm) if err != nil { - np.log.Error(logs.FrostFSCantTransferAssetsToFedContract, zap.Error(err)) + np.log.Error(context.Background(), logs.FrostFSCantTransferAssetsToFedContract, zap.Error(err)) return false } diff --git a/pkg/innerring/processors/frostfs/process_config.go b/pkg/innerring/processors/frostfs/process_config.go index 2ae3e6ced..814dd40b4 100644 --- a/pkg/innerring/processors/frostfs/process_config.go +++ b/pkg/innerring/processors/frostfs/process_config.go @@ -1,6 +1,8 @@ package frostfs import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" frostfsEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/frostfs" @@ -11,7 +13,7 @@ import ( // the sidechain. func (np *Processor) processConfig(config frostfsEvent.Config) bool { if !np.alphabetState.IsAlphabet() { - np.log.Info(logs.FrostFSNonAlphabetModeIgnoreConfig) + np.log.Info(context.Background(), logs.FrostFSNonAlphabetModeIgnoreConfig) return true } @@ -24,7 +26,7 @@ func (np *Processor) processConfig(config frostfsEvent.Config) bool { err := np.netmapClient.SetConfig(prm) if err != nil { - np.log.Error(logs.FrostFSCantRelaySetConfigEvent, zap.Error(err)) + np.log.Error(context.Background(), logs.FrostFSCantRelaySetConfigEvent, zap.Error(err)) return false } diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index 2019857ac..fdc31d82e 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -1,6 +1,7 @@ package frostfs import ( + "context" "errors" "fmt" "sync" @@ -110,7 +111,7 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/frostfs: balance precision converter is not set") } - p.Log.Debug(logs.FrostFSFrostfsWorkerPool, zap.Int("size", p.PoolSize)) + p.Log.Debug(context.Background(), logs.FrostFSFrostfsWorkerPool, zap.Int("size", p.PoolSize)) pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { diff --git a/pkg/innerring/processors/governance/handlers.go b/pkg/innerring/processors/governance/handlers.go index fd7f539c3..dee8c13e2 100644 --- a/pkg/innerring/processors/governance/handlers.go +++ b/pkg/innerring/processors/governance/handlers.go @@ -1,6 +1,8 @@ package governance import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -32,7 +34,7 @@ func (gp *Processor) HandleAlphabetSync(e event.Event) { return } - gp.log.Info(logs.GovernanceNewEvent, zap.String("type", typ)) + gp.log.Info(context.Background(), logs.GovernanceNewEvent, zap.String("type", typ)) // send event to the worker pool @@ -41,7 +43,7 @@ func (gp *Processor) HandleAlphabetSync(e event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - gp.log.Warn(logs.GovernanceGovernanceWorkerPoolDrained, + gp.log.Warn(context.Background(), logs.GovernanceGovernanceWorkerPoolDrained, zap.Int("capacity", gp.pool.Cap())) } } diff --git a/pkg/innerring/processors/governance/process_update.go b/pkg/innerring/processors/governance/process_update.go index 50ba58e77..faca22f67 100644 --- a/pkg/innerring/processors/governance/process_update.go +++ b/pkg/innerring/processors/governance/process_update.go @@ -1,6 +1,7 @@ package governance import ( + "context" "encoding/binary" "encoding/hex" "sort" @@ -20,37 +21,37 @@ const ( func (gp *Processor) processAlphabetSync(txHash util.Uint256) bool { if !gp.alphabetState.IsAlphabet() { - gp.log.Info(logs.GovernanceNonAlphabetModeIgnoreAlphabetSync) + gp.log.Info(context.Background(), logs.GovernanceNonAlphabetModeIgnoreAlphabetSync) return true } mainnetAlphabet, err := gp.mainnetClient.NeoFSAlphabetList() if err != nil { - gp.log.Error(logs.GovernanceCantFetchAlphabetListFromMainNet, + gp.log.Error(context.Background(), logs.GovernanceCantFetchAlphabetListFromMainNet, zap.String("error", err.Error())) return false } sidechainAlphabet, err := gp.morphClient.Committee() if err != nil { - gp.log.Error(logs.GovernanceCantFetchAlphabetListFromSideChain, + gp.log.Error(context.Background(), logs.GovernanceCantFetchAlphabetListFromSideChain, zap.String("error", err.Error())) return false } newAlphabet, err := newAlphabetList(sidechainAlphabet, mainnetAlphabet) if err != nil { - gp.log.Error(logs.GovernanceCantMergeAlphabetListsFromMainNetAndSideChain, + gp.log.Error(context.Background(), logs.GovernanceCantMergeAlphabetListsFromMainNetAndSideChain, zap.String("error", err.Error())) return false } if newAlphabet == nil { - gp.log.Info(logs.GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged) + gp.log.Info(context.Background(), logs.GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged) return true } - gp.log.Info(logs.GovernanceAlphabetListHasBeenChangedStartingUpdate, + gp.log.Info(context.Background(), logs.GovernanceAlphabetListHasBeenChangedStartingUpdate, zap.String("side_chain_alphabet", prettyKeys(sidechainAlphabet)), zap.String("new_alphabet", prettyKeys(newAlphabet)), ) @@ -63,7 +64,7 @@ func (gp *Processor) processAlphabetSync(txHash util.Uint256) bool { // 1. Vote to sidechain committee via alphabet contracts. err = gp.voter.VoteForSidechainValidator(votePrm) if err != nil { - gp.log.Error(logs.GovernanceCantVoteForSideChainCommittee, + gp.log.Error(context.Background(), logs.GovernanceCantVoteForSideChainCommittee, zap.String("error", err.Error())) } @@ -76,7 +77,7 @@ func (gp *Processor) processAlphabetSync(txHash util.Uint256) bool { // 4. Update FrostFS contract in the mainnet. gp.updateFrostFSContractInMainnet(newAlphabet) - gp.log.Info(logs.GovernanceFinishedAlphabetListUpdate) + gp.log.Info(context.Background(), logs.GovernanceFinishedAlphabetListUpdate) return true } @@ -96,21 +97,21 @@ func prettyKeys(keys keys.PublicKeys) string { func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(sidechainAlphabet, newAlphabet keys.PublicKeys, txHash util.Uint256) { innerRing, err := gp.irFetcher.InnerRingKeys() if err != nil { - gp.log.Error(logs.GovernanceCantFetchInnerRingListFromSideChain, + gp.log.Error(context.Background(), logs.GovernanceCantFetchInnerRingListFromSideChain, zap.String("error", err.Error())) return } newInnerRing, err := updateInnerRing(innerRing, sidechainAlphabet, newAlphabet) if err != nil { - gp.log.Error(logs.GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys, + gp.log.Error(context.Background(), logs.GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys, zap.String("error", err.Error())) return } sort.Sort(newInnerRing) - gp.log.Info(logs.GovernanceUpdateOfTheInnerRingList, + gp.log.Info(context.Background(), logs.GovernanceUpdateOfTheInnerRingList, zap.String("before", prettyKeys(innerRing)), zap.String("after", prettyKeys(newInnerRing)), ) @@ -120,7 +121,7 @@ func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(sidechainAlphabet, newAl updPrm.SetHash(txHash) if err = gp.morphClient.UpdateNeoFSAlphabetList(updPrm); err != nil { - gp.log.Error(logs.GovernanceCantUpdateInnerRingListWithNewAlphabetKeys, + gp.log.Error(context.Background(), logs.GovernanceCantUpdateInnerRingListWithNewAlphabetKeys, zap.String("error", err.Error())) } } @@ -133,7 +134,7 @@ func (gp *Processor) updateNotaryRoleInSidechain(newAlphabet keys.PublicKeys, tx err := gp.morphClient.UpdateNotaryList(updPrm) if err != nil { - gp.log.Error(logs.GovernanceCantUpdateListOfNotaryNodesInSideChain, + gp.log.Error(context.Background(), logs.GovernanceCantUpdateListOfNotaryNodesInSideChain, zap.String("error", err.Error())) } } @@ -153,7 +154,7 @@ func (gp *Processor) updateFrostFSContractInMainnet(newAlphabet keys.PublicKeys) err := gp.frostfsClient.AlphabetUpdate(prm) if err != nil { - gp.log.Error(logs.GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract, + gp.log.Error(context.Background(), logs.GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract, zap.String("error", err.Error())) } } diff --git a/pkg/innerring/processors/netmap/handlers.go b/pkg/innerring/processors/netmap/handlers.go index c6053e281..478ab5eab 100644 --- a/pkg/innerring/processors/netmap/handlers.go +++ b/pkg/innerring/processors/netmap/handlers.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -13,21 +14,21 @@ import ( func (np *Processor) HandleNewEpochTick(ev event.Event) { _ = ev.(timerEvent.NewEpochTick) - np.log.Info(logs.NetmapTick, zap.String("type", "epoch")) + np.log.Info(context.Background(), logs.NetmapTick, zap.String("type", "epoch")) // send an event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "netmap_new_epoch_tick", np.processNewEpochTick) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, + np.log.Warn(context.Background(), logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleNewEpoch(ev event.Event) { epochEvent := ev.(netmapEvent.NewEpoch) - np.log.Info(logs.Notification, + np.log.Info(context.Background(), logs.Notification, zap.String("type", "new epoch"), zap.Uint64("value", epochEvent.EpochNumber())) @@ -38,7 +39,7 @@ func (np *Processor) handleNewEpoch(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, + np.log.Warn(context.Background(), logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } @@ -46,7 +47,7 @@ func (np *Processor) handleNewEpoch(ev event.Event) { func (np *Processor) handleAddPeer(ev event.Event) { newPeer := ev.(netmapEvent.AddPeer) - np.log.Info(logs.Notification, + np.log.Info(context.Background(), logs.Notification, zap.String("type", "add peer"), ) @@ -57,14 +58,14 @@ func (np *Processor) handleAddPeer(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, + np.log.Warn(context.Background(), logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleUpdateState(ev event.Event) { updPeer := ev.(netmapEvent.UpdatePeer) - np.log.Info(logs.Notification, + np.log.Info(context.Background(), logs.Notification, zap.String("type", "update peer state"), zap.String("key", hex.EncodeToString(updPeer.PublicKey().Bytes()))) @@ -75,21 +76,21 @@ func (np *Processor) handleUpdateState(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, + np.log.Warn(context.Background(), logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } func (np *Processor) handleCleanupTick(ev event.Event) { if !np.netmapSnapshot.enabled { - np.log.Debug(logs.NetmapNetmapCleanUpRoutineIsDisabled518) + np.log.Debug(context.Background(), logs.NetmapNetmapCleanUpRoutineIsDisabled518) return } cleanup := ev.(netmapCleanupTick) - np.log.Info(logs.NetmapTick, zap.String("type", "netmap cleaner")) + np.log.Info(context.Background(), logs.NetmapTick, zap.String("type", "netmap cleaner")) // send event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "netmap_cleanup_tick", func() bool { @@ -97,7 +98,7 @@ func (np *Processor) handleCleanupTick(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(logs.NetmapNetmapWorkerPoolDrained, + np.log.Warn(context.Background(), logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } diff --git a/pkg/innerring/processors/netmap/process_cleanup.go b/pkg/innerring/processors/netmap/process_cleanup.go index 269e79c5e..9529d3a0c 100644 --- a/pkg/innerring/processors/netmap/process_cleanup.go +++ b/pkg/innerring/processors/netmap/process_cleanup.go @@ -1,6 +1,8 @@ package netmap import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" v2netmap "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -9,7 +11,7 @@ import ( func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) bool { if !np.alphabetState.IsAlphabet() { - np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick) + np.log.Info(context.Background(), logs.NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick) return true } @@ -17,13 +19,13 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) bool { err := np.netmapSnapshot.forEachRemoveCandidate(ev.epoch, func(s string) error { key, err := keys.NewPublicKeyFromString(s) if err != nil { - np.log.Warn(logs.NetmapCantDecodePublicKeyOfNetmapNode, + np.log.Warn(context.Background(), logs.NetmapCantDecodePublicKeyOfNetmapNode, zap.String("key", s)) return nil } - np.log.Info(logs.NetmapVoteToRemoveNodeFromNetmap, zap.String("key", s)) + np.log.Info(context.Background(), logs.NetmapVoteToRemoveNodeFromNetmap, zap.String("key", s)) // In notary environments we call UpdateStateIR method instead of UpdateState. // It differs from UpdateState only by name, so we can do this in the same form. @@ -39,13 +41,13 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) bool { int64(v2netmap.Offline), key.Bytes(), ) if err != nil { - np.log.Error(logs.NetmapCantInvokeNetmapUpdateState, zap.Error(err)) + np.log.Error(context.Background(), logs.NetmapCantInvokeNetmapUpdateState, zap.Error(err)) } return nil }) if err != nil { - np.log.Warn(logs.NetmapCantIterateOnNetmapCleanerCache, + np.log.Warn(context.Background(), logs.NetmapCantIterateOnNetmapCleanerCache, zap.String("error", err.Error())) return false } diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index 9522df26c..8ad295a74 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -1,6 +1,8 @@ package netmap import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/governance" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" @@ -14,7 +16,7 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) bool { epochDuration, err := np.netmapClient.EpochDuration() if err != nil { - np.log.Warn(logs.NetmapCantGetEpochDuration, + np.log.Warn(context.Background(), logs.NetmapCantGetEpochDuration, zap.String("error", err.Error())) } else { np.epochState.SetEpochDuration(epochDuration) @@ -24,20 +26,20 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) bool { h, err := np.netmapClient.MorphTxHeight(ev.TxHash()) if err != nil { - np.log.Warn(logs.NetmapCantGetTransactionHeight, + np.log.Warn(context.Background(), logs.NetmapCantGetTransactionHeight, zap.String("hash", ev.TxHash().StringLE()), zap.String("error", err.Error())) } if err := np.epochTimer.ResetEpochTimer(h); err != nil { - np.log.Warn(logs.NetmapCantResetEpochTimer, + np.log.Warn(context.Background(), logs.NetmapCantResetEpochTimer, zap.String("error", err.Error())) } // get new netmap snapshot networkMap, err := np.netmapClient.NetMap() if err != nil { - np.log.Warn(logs.NetmapCantGetNetmapSnapshotToPerformCleanup, + np.log.Warn(context.Background(), logs.NetmapCantGetNetmapSnapshotToPerformCleanup, zap.String("error", err.Error())) return false @@ -54,16 +56,16 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) bool { // Process new epoch tick by invoking new epoch method in network map contract. func (np *Processor) processNewEpochTick() bool { if !np.alphabetState.IsAlphabet() { - np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewEpochTick) + np.log.Info(context.Background(), logs.NetmapNonAlphabetModeIgnoreNewEpochTick) return true } nextEpoch := np.epochState.EpochCounter() + 1 - np.log.Debug(logs.NetmapNextEpoch, zap.Uint64("value", nextEpoch)) + np.log.Debug(context.Background(), logs.NetmapNextEpoch, zap.Uint64("value", nextEpoch)) err := np.netmapClient.NewEpoch(nextEpoch) if err != nil { - np.log.Error(logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) + np.log.Error(context.Background(), logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) return false } diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index c8c7928a3..42d1b5ec6 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -14,7 +15,7 @@ import ( // local epoch timer. func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { if !np.alphabetState.IsAlphabet() { - np.log.Info(logs.NetmapNonAlphabetModeIgnoreNewPeerNotification) + np.log.Info(context.Background(), logs.NetmapNonAlphabetModeIgnoreNewPeerNotification) return true } @@ -22,7 +23,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { tx := ev.NotaryRequest().MainTransaction ok, err := np.netmapClient.MorphIsValidScript(tx.Script, tx.Signers) if err != nil || !ok { - np.log.Warn(logs.NetmapNonhaltNotaryTransaction, + np.log.Warn(context.Background(), logs.NetmapNonhaltNotaryTransaction, zap.String("method", "netmap.AddPeer"), zap.String("hash", tx.Hash().StringLE()), zap.Error(err)) @@ -33,14 +34,14 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { var nodeInfo netmap.NodeInfo if err := nodeInfo.Unmarshal(ev.Node()); err != nil { // it will be nice to have tx id at event structure to log it - np.log.Warn(logs.NetmapCantParseNetworkMapCandidate) + np.log.Warn(context.Background(), logs.NetmapCantParseNetworkMapCandidate) return false } // validate and update node info err = np.nodeValidator.VerifyAndUpdate(&nodeInfo) if err != nil { - np.log.Warn(logs.NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate, + np.log.Warn(context.Background(), logs.NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate, zap.String("error", err.Error()), ) @@ -63,7 +64,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { // That is why we need to perform `addPeerIR` only in case when node is online, // because in scope of this method, contract set state `ONLINE` for the node. if updated && nodeInfo.Status().IsOnline() { - np.log.Info(logs.NetmapApprovingNetworkMapCandidate, + np.log.Info(context.Background(), logs.NetmapApprovingNetworkMapCandidate, zap.String("key", keyString)) prm := netmapclient.AddPeerPrm{} @@ -84,7 +85,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { nodeInfoBinary, ) if err != nil { - np.log.Error(logs.NetmapCantInvokeNetmapAddPeer, zap.Error(err)) + np.log.Error(context.Background(), logs.NetmapCantInvokeNetmapAddPeer, zap.Error(err)) return false } } @@ -95,7 +96,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { // Process update peer notification by sending approval tx to the smart contract. func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) bool { if !np.alphabetState.IsAlphabet() { - np.log.Info(logs.NetmapNonAlphabetModeIgnoreUpdatePeerNotification) + np.log.Info(context.Background(), logs.NetmapNonAlphabetModeIgnoreUpdatePeerNotification) return true } @@ -108,7 +109,7 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) bool { if ev.Maintenance() { err = np.nodeStateSettings.MaintenanceModeAllowed() if err != nil { - np.log.Info(logs.NetmapPreventSwitchingNodeToMaintenanceState, + np.log.Info(context.Background(), logs.NetmapPreventSwitchingNodeToMaintenanceState, zap.Error(err), ) @@ -117,7 +118,7 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) bool { } if err = np.netmapClient.MorphNotarySignAndInvokeTX(ev.NotaryRequest().MainTransaction); err != nil { - np.log.Error(logs.NetmapCantInvokeNetmapUpdatePeer, zap.Error(err)) + np.log.Error(context.Background(), logs.NetmapCantInvokeNetmapUpdatePeer, zap.Error(err)) return false } diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index f5a91dee2..bbd60c1e1 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "errors" "fmt" @@ -132,7 +133,7 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/netmap: node state settings is not set") } - p.Log.Debug(logs.NetmapNetmapWorkerPool, zap.Int("size", p.PoolSize)) + p.Log.Debug(context.Background(), logs.NetmapNetmapWorkerPool, zap.Int("size", p.PoolSize)) pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index d3071faad..250f41e5f 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -1,6 +1,7 @@ package innerring import ( + "context" "fmt" "sort" @@ -61,7 +62,7 @@ func (s *Server) IsAlphabet() bool { func (s *Server) InnerRingIndex() int { index, err := s.statusIndex.InnerRingIndex() if err != nil { - s.log.Error(logs.InnerringCantGetInnerRingIndex, zap.String("error", err.Error())) + s.log.Error(context.Background(), logs.InnerringCantGetInnerRingIndex, zap.String("error", err.Error())) return -1 } @@ -73,7 +74,7 @@ func (s *Server) InnerRingIndex() int { func (s *Server) InnerRingSize() int { size, err := s.statusIndex.InnerRingSize() if err != nil { - s.log.Error(logs.InnerringCantGetInnerRingSize, zap.String("error", err.Error())) + s.log.Error(context.Background(), logs.InnerringCantGetInnerRingSize, zap.String("error", err.Error())) return 0 } @@ -85,7 +86,7 @@ func (s *Server) InnerRingSize() int { func (s *Server) AlphabetIndex() int { index, err := s.statusIndex.AlphabetIndex() if err != nil { - s.log.Error(logs.InnerringCantGetAlphabetIndex, zap.String("error", err.Error())) + s.log.Error(context.Background(), logs.InnerringCantGetAlphabetIndex, zap.String("error", err.Error())) return -1 } @@ -97,13 +98,13 @@ func (s *Server) voteForSidechainValidator(prm governance.VoteValidatorPrm) erro index := s.InnerRingIndex() if s.contracts.alphabet.indexOutOfRange(index) { - s.log.Info(logs.InnerringIgnoreValidatorVoteNodeNotInAlphabetRange) + s.log.Info(context.Background(), logs.InnerringIgnoreValidatorVoteNodeNotInAlphabetRange) return nil } if len(validators) == 0 { - s.log.Info(logs.InnerringIgnoreValidatorVoteEmptyValidatorsList) + s.log.Info(context.Background(), logs.InnerringIgnoreValidatorVoteEmptyValidatorsList) return nil } @@ -128,7 +129,7 @@ func (s *Server) voteForSidechainValidator(prm governance.VoteValidatorPrm) erro s.contracts.alphabet.iterate(func(letter GlagoliticLetter, contract util.Uint160) { _, err := s.morphClient.NotaryInvoke(contract, s.feeConfig.SideChainFee(), nonce, vubP, voteMethod, epoch, validators) if err != nil { - s.log.Warn(logs.InnerringCantInvokeVoteMethodInAlphabetContract, + s.log.Warn(context.Background(), logs.InnerringCantInvokeVoteMethodInAlphabetContract, zap.Int8("alphabet_index", int8(letter)), zap.Uint64("epoch", epoch), zap.String("error", err.Error())) @@ -202,6 +203,6 @@ func (s *Server) notifySystemd(st control.HealthStatus) { err = sdnotify.Status(fmt.Sprintf("%v", st)) } if err != nil { - s.log.Error(logs.FailedToReportStatusToSystemd, zap.Error(err)) + s.log.Error(context.Background(), logs.FailedToReportStatusToSystemd, zap.Error(err)) } } diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index c787f9d5e..08ef8b86c 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -58,7 +58,7 @@ func defaultCfg(c *cfg) { }, fullSizeLimit: 1 << 30, // 1GB objSizeLimit: 1 << 20, // 1MB - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), metrics: &NoopMetrics{}, } } @@ -110,7 +110,7 @@ func WithFullSizeLimit(lim uint64) Option { // WithLogger returns an option to specify Blobovnicza's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = &logger.Logger{Logger: l.With(zap.String("component", "Blobovnicza"))} + c.log = l.With(zap.String("component", "Blobovnicza")) } } diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index aeaa4e1d5..5d7135741 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -1,6 +1,7 @@ package blobovnicza import ( + "context" "errors" "fmt" "path/filepath" @@ -23,7 +24,7 @@ func (b *Blobovnicza) Open() error { return nil } - b.log.Debug(logs.BlobovniczaCreatingDirectoryForBoltDB, + b.log.Debug(context.Background(), logs.BlobovniczaCreatingDirectoryForBoltDB, zap.String("path", b.path), zap.Bool("ro", b.boltOptions.ReadOnly), ) @@ -37,7 +38,7 @@ func (b *Blobovnicza) Open() error { } } - b.log.Debug(logs.BlobovniczaOpeningBoltDB, + b.log.Debug(context.Background(), logs.BlobovniczaOpeningBoltDB, zap.String("path", b.path), zap.Stringer("permissions", b.perm), ) @@ -63,7 +64,7 @@ func (b *Blobovnicza) Init() error { return errors.New("blobovnicza is not open") } - b.log.Debug(logs.BlobovniczaInitializing, + b.log.Debug(context.Background(), logs.BlobovniczaInitializing, zap.Uint64("object size limit", b.objSizeLimit), zap.Uint64("storage size limit", b.fullSizeLimit), ) @@ -71,7 +72,7 @@ func (b *Blobovnicza) Init() error { size := b.dataSize.Load() items := b.itemsCount.Load() if size != 0 || items != 0 { - b.log.Debug(logs.BlobovniczaAlreadyInitialized, zap.Uint64("size", size), zap.Uint64("items", items)) + b.log.Debug(context.Background(), logs.BlobovniczaAlreadyInitialized, zap.Uint64("size", size), zap.Uint64("items", items)) return nil } @@ -81,7 +82,7 @@ func (b *Blobovnicza) Init() error { // create size range bucket rangeStr := stringifyBounds(lower, upper) - b.log.Debug(logs.BlobovniczaCreatingBucketForSizeRange, + b.log.Debug(context.Background(), logs.BlobovniczaCreatingBucketForSizeRange, zap.String("range", rangeStr)) _, err := tx.CreateBucketIfNotExists(key) @@ -131,17 +132,17 @@ func (b *Blobovnicza) initializeCounters() error { return fmt.Errorf("can't determine DB size: %w", err) } if (!sizeExists || !itemsCountExists) && !b.boltOptions.ReadOnly { - b.log.Debug(logs.BlobovniczaSavingCountersToMeta, zap.Uint64("size", size), zap.Uint64("items", items)) + b.log.Debug(context.Background(), logs.BlobovniczaSavingCountersToMeta, zap.Uint64("size", size), zap.Uint64("items", items)) if err := b.boltDB.Update(func(tx *bbolt.Tx) error { if err := saveDataSize(tx, size); err != nil { return err } return saveItemsCount(tx, items) }); err != nil { - b.log.Debug(logs.BlobovniczaSavingCountersToMetaFailed, zap.Uint64("size", size), zap.Uint64("items", items)) + b.log.Debug(context.Background(), logs.BlobovniczaSavingCountersToMetaFailed, zap.Uint64("size", size), zap.Uint64("items", items)) return fmt.Errorf("can't save blobovnicza's size and items count: %w", err) } - b.log.Debug(logs.BlobovniczaSavingCountersToMetaSuccess, zap.Uint64("size", size), zap.Uint64("items", items)) + b.log.Debug(context.Background(), logs.BlobovniczaSavingCountersToMetaSuccess, zap.Uint64("size", size), zap.Uint64("items", items)) } b.dataSize.Store(size) @@ -162,7 +163,7 @@ func (b *Blobovnicza) Close() error { return nil } - b.log.Debug(logs.BlobovniczaClosingBoltDB, + b.log.Debug(context.Background(), logs.BlobovniczaClosingBoltDB, zap.String("path", b.path), ) diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index 5d6787897..d821b2991 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -91,7 +91,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err } if err == nil && found { - b.log.Debug(logs.BlobovniczaObjectWasRemovedFromBucket, + b.log.Debug(ctx, logs.BlobovniczaObjectWasRemovedFromBucket, zap.String("binary size", stringifyByteSize(dataSize)), zap.String("range", stringifyBounds(sizeLowerBound, sizeUpperBound)), zap.String("trace_id", tracingPkg.GetTraceID(ctx)), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 681cf876c..55c9d6630 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -24,10 +24,10 @@ func (b *Blobovniczas) Open(mode mode.ComponentMode) error { // // Should be called exactly once. func (b *Blobovniczas) Init() error { - b.log.Debug(logs.BlobovniczatreeInitializingBlobovniczas) + b.log.Debug(context.Background(), logs.BlobovniczatreeInitializingBlobovniczas) if b.readOnly { - b.log.Debug(logs.BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization) + b.log.Debug(context.Background(), logs.BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization) return nil } @@ -60,7 +60,7 @@ func (b *Blobovniczas) initializeDBs(ctx context.Context) error { b.deleteProtectedObjects.Add(move.Address) } - b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) + b.log.Debug(egCtx, logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) return nil }) return false, nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 298de3ad6..dd5258042 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -80,7 +80,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co res, err = b.deleteObjectFromLevel(ctx, bPrm, p) if err != nil { if !client.IsErrObjectNotFound(err) { - b.log.Debug(logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, + b.log.Debug(ctx, logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx)), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index a64b2bbb1..2149b17c0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -55,7 +55,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common _, err := b.getObjectFromLevel(ctx, gPrm, p) if err != nil { if !client.IsErrObjectNotFound(err) { - b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, + b.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 08cacda8a..e79480095 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -67,7 +67,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G res, err = b.getObjectFromLevel(ctx, bPrm, p) if err != nil { if !client.IsErrObjectNotFound(err) { - b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, + b.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx)), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index d237ae439..20f2be2ba 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -69,7 +69,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re if err != nil { outOfBounds := isErrOutOfRange(err) if !outOfBounds && !client.IsErrObjectNotFound(err) { - b.log.Debug(logs.BlobovniczatreeCouldNotGetObjectFromLevel, + b.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index f6acb46aa..7f0453410 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -42,7 +42,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm data, err := b.compression.Decompress(elem.ObjectData()) if err != nil { if prm.IgnoreErrors { - b.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, + b.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, zap.Stringer("address", elem.Address()), zap.String("err", err.Error()), zap.String("storage_id", p), @@ -76,7 +76,7 @@ func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors boo blz, err := shBlz.Open() if err != nil { if ignoreErrors { - b.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, + b.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, zap.String("err", err.Error()), zap.String("storage_id", p), zap.String("root_path", b.rootPath)) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index 4fdde15a9..235c9f65d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "context" "errors" "fmt" "os" @@ -86,7 +87,7 @@ func (b *sharedDB) Close() { defer b.cond.L.Unlock() if b.refCount == 0 { - b.log.Error(logs.AttemtToCloseAlreadyClosedBlobovnicza, zap.String("id", b.path)) + b.log.Error(context.Background(), logs.AttemtToCloseAlreadyClosedBlobovnicza, zap.String("id", b.path)) b.cond.Broadcast() return } @@ -94,7 +95,7 @@ func (b *sharedDB) Close() { if b.refCount == 1 { b.refCount = 0 if err := b.blcza.Close(); err != nil { - b.log.Error(logs.BlobovniczatreeCouldNotCloseBlobovnicza, + b.log.Error(context.Background(), logs.BlobovniczatreeCouldNotCloseBlobovnicza, zap.String("id", b.path), zap.String("error", err.Error()), ) @@ -122,7 +123,7 @@ func (b *sharedDB) CloseAndRemoveFile() error { } if err := b.blcza.Close(); err != nil { - b.log.Error(logs.BlobovniczatreeCouldNotCloseBlobovnicza, + b.log.Error(context.Background(), logs.BlobovniczatreeCouldNotCloseBlobovnicza, zap.String("id", b.path), zap.String("error", err.Error()), ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 008be9543..b56251772 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -47,7 +47,7 @@ const ( func initConfig(c *cfg) { *c = cfg{ - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), perm: defaultPerm, openedCacheSize: defaultOpenedCacheSize, openedCacheTTL: defaultOpenedCacheTTL, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 76c4953e4..844b43151 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -82,7 +82,7 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) if !isLogical(err) { i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) } else { - i.B.log.Debug(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, + i.B.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } @@ -91,7 +91,7 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) } if active == nil { - i.B.log.Debug(logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath), + i.B.log.Debug(ctx, logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return false, nil } @@ -104,7 +104,7 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) if !isLogical(err) { i.B.reportError(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err) } else { - i.B.log.Debug(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, + i.B.log.Debug(ctx, logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", active.SystemPath()), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index 202d38cd7..fee67a0a8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -49,25 +49,25 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm var res common.RebuildRes - b.log.Debug(logs.BlobovniczaTreeCompletingPreviousRebuild) + b.log.Debug(ctx, logs.BlobovniczaTreeCompletingPreviousRebuild) completedPreviosMoves, err := b.completeIncompletedMove(ctx, prm.MetaStorage) res.ObjectsMoved += completedPreviosMoves if err != nil { - b.log.Warn(logs.BlobovniczaTreeCompletedPreviousRebuildFailed, zap.Error(err)) + b.log.Warn(ctx, logs.BlobovniczaTreeCompletedPreviousRebuildFailed, zap.Error(err)) success = false return res, err } - b.log.Debug(logs.BlobovniczaTreeCompletedPreviousRebuildSuccess) + b.log.Debug(ctx, logs.BlobovniczaTreeCompletedPreviousRebuildSuccess) - b.log.Debug(logs.BlobovniczaTreeCollectingDBToRebuild) + b.log.Debug(ctx, logs.BlobovniczaTreeCollectingDBToRebuild) dbsToMigrate, err := b.getDBsToRebuild(ctx, prm.FillPercent) if err != nil { - b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err)) + b.log.Warn(ctx, logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err)) success = false return res, err } - b.log.Info(logs.BlobovniczaTreeCollectingDBToRebuildSuccess, zap.Int("blobovniczas_to_rebuild", len(dbsToMigrate))) + b.log.Info(ctx, logs.BlobovniczaTreeCollectingDBToRebuildSuccess, zap.Int("blobovniczas_to_rebuild", len(dbsToMigrate))) res, err = b.migrateDBs(ctx, dbsToMigrate, prm, res) if err != nil { success = false @@ -78,14 +78,14 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common.RebuildPrm, res common.RebuildRes) (common.RebuildRes, error) { var completedDBCount uint32 for _, db := range dbs { - b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db)) + b.log.Debug(ctx, logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db)) movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage, prm.WorkerLimiter) res.ObjectsMoved += movedObjects if err != nil { - b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err)) + b.log.Warn(ctx, logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err)) return res, err } - b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) + b.log.Debug(ctx, logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects)) res.FilesRemoved++ completedDBCount++ b.metrics.SetRebuildPercent((100 * completedDBCount) / uint32(len(dbs))) @@ -233,7 +233,7 @@ func (b *Blobovniczas) addRebuildTempFile(path string) (func(), error) { } return func() { if err := os.Remove(sysPath); err != nil { - b.log.Warn(logs.BlobovniczatreeFailedToRemoveRebuildTempFile, zap.Error(err)) + b.log.Warn(context.Background(), logs.BlobovniczatreeFailedToRemoveRebuildTempFile, zap.Error(err)) } }, nil } @@ -389,7 +389,7 @@ func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore co }) for _, tmp := range rebuildTempFilesToRemove { if err := os.Remove(filepath.Join(b.rootPath, tmp)); err != nil { - b.log.Warn(logs.BlobovniczatreeFailedToRemoveRebuildTempFile, zap.Error(err)) + b.log.Warn(ctx, logs.BlobovniczatreeFailedToRemoveRebuildTempFile, zap.Error(err)) } } return count, err @@ -413,14 +413,14 @@ func (b *Blobovniczas) performMove(ctx context.Context, source *blobovnicza.Blob if client.IsErrObjectNotFound(err) { existsInSource = false } else { - b.log.Warn(logs.BlobovniczatreeCouldNotCheckExistenceInTargetDB, zap.Error(err)) + b.log.Warn(ctx, logs.BlobovniczatreeCouldNotCheckExistenceInTargetDB, zap.Error(err)) return err } } if !existsInSource { // object was deleted by Rebuild, need to delete move info if err = source.DropMoveInfo(ctx, move.Address); err != nil { - b.log.Warn(logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", sourcePath), zap.Error(err)) + b.log.Warn(ctx, logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", sourcePath), zap.Error(err)) return err } b.deleteProtectedObjects.Delete(move.Address) @@ -429,7 +429,7 @@ func (b *Blobovniczas) performMove(ctx context.Context, source *blobovnicza.Blob existsInTarget, err := target.Exists(ctx, move.Address) if err != nil { - b.log.Warn(logs.BlobovniczatreeCouldNotCheckExistenceInTargetDB, zap.Error(err)) + b.log.Warn(ctx, logs.BlobovniczatreeCouldNotCheckExistenceInTargetDB, zap.Error(err)) return err } @@ -439,25 +439,25 @@ func (b *Blobovniczas) performMove(ctx context.Context, source *blobovnicza.Blob putPrm.SetMarshaledObject(gRes.Object()) _, err = target.Put(ctx, putPrm) if err != nil { - b.log.Warn(logs.BlobovniczatreeCouldNotPutObjectToTargetDB, zap.String("path", targetDB.SystemPath()), zap.Error(err)) + b.log.Warn(ctx, logs.BlobovniczatreeCouldNotPutObjectToTargetDB, zap.String("path", targetDB.SystemPath()), zap.Error(err)) return err } } if err = metaStore.UpdateStorageID(ctx, move.Address, move.TargetStorageID); err != nil { - b.log.Warn(logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err), zap.Stringer("address", move.Address)) + b.log.Warn(ctx, logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err), zap.Stringer("address", move.Address)) return err } var deletePrm blobovnicza.DeletePrm deletePrm.SetAddress(move.Address) if _, err = source.Delete(ctx, deletePrm); err != nil { - b.log.Warn(logs.BlobovniczatreeCouldNotDeleteFromSource, zap.String("path", sourcePath), zap.Error(err)) + b.log.Warn(ctx, logs.BlobovniczatreeCouldNotDeleteFromSource, zap.String("path", sourcePath), zap.Error(err)) return err } if err = source.DropMoveInfo(ctx, move.Address); err != nil { - b.log.Warn(logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", sourcePath), zap.Error(err)) + b.log.Warn(ctx, logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", sourcePath), zap.Error(err)) return err } @@ -482,13 +482,13 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, if !isLogical(err) { i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) } else { - i.B.log.Warn(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, zap.Error(err)) + i.B.log.Warn(ctx, logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, zap.Error(err)) } return false, nil } if target == nil { - i.B.log.Warn(logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath)) + i.B.log.Warn(ctx, logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath)) return false, nil } defer target.Close() @@ -505,7 +505,7 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, if !isLogical(err) { i.B.reportError(logs.BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza, err) } else { - i.B.log.Warn(logs.BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza, zap.String("path", i.SourceSysPath), zap.Error(err)) + i.B.log.Warn(ctx, logs.BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza, zap.String("path", i.SourceSysPath), zap.Error(err)) } return true, nil } @@ -521,13 +521,13 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, if !isLogical(err) { i.B.reportError(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err) } else { - i.B.log.Warn(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", target.SystemPath()), zap.Error(err)) + i.B.log.Warn(ctx, logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", target.SystemPath()), zap.Error(err)) } return true, nil } if err = i.MetaStore.UpdateStorageID(ctx, i.Address, targetStorageID.Bytes()); err != nil { - i.B.log.Warn(logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err), zap.Stringer("address", i.Address)) + i.B.log.Warn(ctx, logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err), zap.Stringer("address", i.Address)) return true, nil } @@ -537,7 +537,7 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, if !isLogical(err) { i.B.reportError(logs.BlobovniczatreeCouldNotDeleteFromSource, err) } else { - i.B.log.Warn(logs.BlobovniczatreeCouldNotDeleteFromSource, zap.String("path", i.SourceSysPath), zap.Error(err)) + i.B.log.Warn(ctx, logs.BlobovniczatreeCouldNotDeleteFromSource, zap.String("path", i.SourceSysPath), zap.Error(err)) } return true, nil } @@ -546,7 +546,7 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, if !isLogical(err) { i.B.reportError(logs.BlobovniczatreeCouldNotDropMoveInfo, err) } else { - i.B.log.Warn(logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", i.SourceSysPath), zap.Error(err)) + i.B.log.Warn(ctx, logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", i.SourceSysPath), zap.Error(err)) } return true, nil } diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index 6f579a8ca..41c6cf161 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -47,7 +47,7 @@ type cfg struct { } func initConfig(c *cfg) { - c.log = &logger.Logger{Logger: zap.L()} + c.log = logger.NewLoggerWrapper(zap.L()) c.metrics = &noopMetrics{} } @@ -90,7 +90,7 @@ func WithStorages(st []SubStorage) Option { // WithLogger returns option to specify BlobStor's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = &logger.Logger{Logger: l.With(zap.String("component", "BlobStor"))} + c.log = l.With(zap.String("component", "BlobStor")) } } diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 9b414a9be..43436b4eb 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -12,7 +12,7 @@ import ( // Open opens BlobStor. func (b *BlobStor) Open(ctx context.Context, mode mode.Mode) error { - b.log.Debug(logs.BlobstorOpening) + b.log.Debug(ctx, logs.BlobstorOpening) b.modeMtx.Lock() defer b.modeMtx.Unlock() @@ -51,7 +51,7 @@ var ErrInitBlobovniczas = errors.New("failure on blobovnicza initialization stag // // Returns wrapped ErrInitBlobovniczas on blobovnicza tree's initializaiton failure. func (b *BlobStor) Init() error { - b.log.Debug(logs.BlobstorInitializing) + b.log.Debug(context.Background(), logs.BlobstorInitializing) if err := b.compression.Init(); err != nil { return err @@ -68,13 +68,13 @@ func (b *BlobStor) Init() error { // Close releases all internal resources of BlobStor. func (b *BlobStor) Close() error { - b.log.Debug(logs.BlobstorClosing) + b.log.Debug(context.Background(), logs.BlobstorClosing) var firstErr error for i := range b.storage { err := b.storage[i].Storage.Close() if err != nil { - b.log.Info(logs.BlobstorCouldntCloseStorage, zap.String("error", err.Error())) + b.log.Info(context.Background(), logs.BlobstorCouldntCloseStorage, zap.String("error", err.Error())) if firstErr == nil { firstErr = err } diff --git a/pkg/local_object_storage/blobstor/delete.go b/pkg/local_object_storage/blobstor/delete.go index c91508e6d..86d8f15e3 100644 --- a/pkg/local_object_storage/blobstor/delete.go +++ b/pkg/local_object_storage/blobstor/delete.go @@ -39,7 +39,7 @@ func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.Del if err == nil || !client.IsErrObjectNotFound(err) { if err == nil { success = true - logOp(b.log, deleteOp, prm.Address, b.storage[i].Storage.Type(), prm.StorageID) + logOp(ctx, b.log, deleteOp, prm.Address, b.storage[i].Storage.Type(), prm.StorageID) } return res, err } @@ -58,7 +58,7 @@ func (b *BlobStor) Delete(ctx context.Context, prm common.DeletePrm) (common.Del res, err := st.Delete(ctx, prm) if err == nil { success = true - logOp(b.log, deleteOp, prm.Address, st.Type(), prm.StorageID) + logOp(ctx, b.log, deleteOp, prm.Address, st.Type(), prm.StorageID) } return res, err diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 43feec7c9..556f53e12 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -73,7 +73,7 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi } for _, err := range errors[:len(errors)-1] { - b.log.Warn(logs.BlobstorErrorOccurredDuringObjectExistenceChecking, + b.log.Warn(ctx, logs.BlobstorErrorOccurredDuringObjectExistenceChecking, zap.Stringer("address", prm.Address), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 057796db2..7f52762a7 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -87,7 +87,7 @@ func New(opts ...Option) *FSTree { DirNameLen: DirNameLen, metrics: &noopMetrics{}, fileCounter: &noopCounter{}, - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), } for i := range opts { opts[i](f) @@ -152,7 +152,7 @@ func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, pr des, err := os.ReadDir(dirPath) if err != nil { if prm.IgnoreErrors { - t.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, + t.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, zap.String("err", err.Error()), zap.String("directory_path", dirPath)) return nil @@ -200,7 +200,7 @@ func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, pr } if err != nil { if prm.IgnoreErrors { - t.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, + t.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, zap.Stringer("address", addr), zap.String("err", err.Error()), zap.String("path", path)) diff --git a/pkg/local_object_storage/blobstor/fstree/option.go b/pkg/local_object_storage/blobstor/fstree/option.go index 4d1f8fc22..7155ddcbb 100644 --- a/pkg/local_object_storage/blobstor/fstree/option.go +++ b/pkg/local_object_storage/blobstor/fstree/option.go @@ -53,6 +53,6 @@ func WithFileCounter(c FileCounter) Option { func WithLogger(l *logger.Logger) Option { return func(f *FSTree) { - f.log = &logger.Logger{Logger: l.With(zap.String("component", "FSTree"))} + f.log = l.With(zap.String("component", "FSTree")) } } diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index f213d7547..1ba835a95 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -42,7 +42,7 @@ func (b *BlobStor) Iterate(ctx context.Context, prm common.IteratePrm) (common.I _, err := b.storage[i].Storage.Iterate(ctx, prm) if err != nil { if prm.IgnoreErrors { - b.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, + b.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, zap.String("storage_path", b.storage[i].Storage.Path()), zap.String("storage_type", b.storage[i].Storage.Type()), zap.String("err", err.Error())) diff --git a/pkg/local_object_storage/blobstor/logger.go b/pkg/local_object_storage/blobstor/logger.go index 7e057a0e3..070b1eac9 100644 --- a/pkg/local_object_storage/blobstor/logger.go +++ b/pkg/local_object_storage/blobstor/logger.go @@ -1,6 +1,8 @@ package blobstor import ( + "context" + storagelog "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/log" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -11,8 +13,8 @@ const ( putOp = "PUT" ) -func logOp(l *logger.Logger, op string, addr oid.Address, typ string, sID []byte) { - storagelog.Write(l, +func logOp(ctx context.Context, l *logger.Logger, op string, addr oid.Address, typ string, sID []byte) { + storagelog.Write(ctx, l, storagelog.AddressField(addr), storagelog.OpField(op), storagelog.StorageTypeField(typ), diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index 1adae303d..342da28bf 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -63,7 +63,7 @@ func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e res, err := b.storage[i].Storage.Put(ctx, prm) if err == nil { success = true - logOp(b.log, putOp, prm.Address, b.storage[i].Storage.Type(), res.StorageID) + logOp(ctx, b.log, putOp, prm.Address, b.storage[i].Storage.Type(), res.StorageID) } return res, err } diff --git a/pkg/local_object_storage/blobstor/rebuild.go b/pkg/local_object_storage/blobstor/rebuild.go index 7b2786ba2..2a6b94789 100644 --- a/pkg/local_object_storage/blobstor/rebuild.go +++ b/pkg/local_object_storage/blobstor/rebuild.go @@ -30,7 +30,7 @@ func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate, limiter Con summary.FilesRemoved += res.FilesRemoved summary.ObjectsMoved += res.ObjectsMoved if err != nil { - b.log.Error(logs.BlobstorRebuildFailedToRebuildStorages, + b.log.Error(ctx, logs.BlobstorRebuildFailedToRebuildStorages, zap.String("failed_storage_path", storage.Storage.Path()), zap.String("failed_storage_type", storage.Storage.Type()), zap.Error(err)) @@ -38,7 +38,7 @@ func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate, limiter Con break } } - b.log.Info(logs.BlobstorRebuildRebuildStoragesCompleted, + b.log.Info(ctx, logs.BlobstorRebuildRebuildStoragesCompleted, zap.Bool("success", rErr == nil), zap.Uint64("total_files_removed", summary.FilesRemoved), zap.Uint64("total_objects_moved", summary.ObjectsMoved)) diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 80fb3f9ed..98ec73ae9 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -49,7 +49,7 @@ func (e *StorageEngine) open(ctx context.Context) error { for res := range errCh { if res.err != nil { - e.log.Error(logs.EngineCouldNotOpenShardClosingAndSkipping, + e.log.Error(ctx, logs.EngineCouldNotOpenShardClosingAndSkipping, zap.String("id", res.id), zap.Error(res.err)) @@ -58,7 +58,7 @@ func (e *StorageEngine) open(ctx context.Context) error { err := sh.Close() if err != nil { - e.log.Error(logs.EngineCouldNotClosePartiallyInitializedShard, + e.log.Error(ctx, logs.EngineCouldNotClosePartiallyInitializedShard, zap.String("id", res.id), zap.Error(res.err)) } @@ -101,7 +101,7 @@ func (e *StorageEngine) Init(ctx context.Context) error { for res := range errCh { if res.err != nil { if errors.Is(res.err, blobstor.ErrInitBlobovniczas) { - e.log.Error(logs.EngineCouldNotInitializeShardClosingAndSkipping, + e.log.Error(ctx, logs.EngineCouldNotInitializeShardClosingAndSkipping, zap.String("id", res.id), zap.Error(res.err)) @@ -110,7 +110,7 @@ func (e *StorageEngine) Init(ctx context.Context) error { err := sh.Close() if err != nil { - e.log.Error(logs.EngineCouldNotClosePartiallyInitializedShard, + e.log.Error(ctx, logs.EngineCouldNotClosePartiallyInitializedShard, zap.String("id", res.id), zap.Error(res.err)) } @@ -165,7 +165,7 @@ func (e *StorageEngine) close(releasePools bool) error { for id, sh := range e.shards { if err := sh.Close(); err != nil { - e.log.Debug(logs.EngineCouldNotCloseShard, + e.log.Debug(context.Background(), logs.EngineCouldNotCloseShard, zap.String("id", id), zap.String("error", err.Error()), ) @@ -311,7 +311,7 @@ loop: for _, p := range shardsToReload { err := p.sh.Reload(ctx, p.opts...) if err != nil { - e.log.Error(logs.EngineCouldNotReloadAShard, + e.log.Error(ctx, logs.EngineCouldNotReloadAShard, zap.Stringer("shard id", p.sh.ID()), zap.Error(err)) } @@ -340,7 +340,7 @@ loop: return fmt.Errorf("could not add %s shard: %w", idStr, err) } - e.log.Info(logs.EngineAddedNewShard, zap.String("id", idStr)) + e.log.Info(ctx, logs.EngineAddedNewShard, zap.String("id", idStr)) } return nil diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 9ca3a7cee..2e957eb04 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -152,7 +152,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { res, err := sh.Select(ctx, selectPrm) if err != nil { - e.log.Warn(logs.EngineErrorDuringSearchingForObjectChildren, + e.log.Warn(ctx, logs.EngineErrorDuringSearchingForObjectChildren, zap.Stringer("addr", addr), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -164,7 +164,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo _, err = sh.Inhume(ctx, inhumePrm) if err != nil { - e.log.Debug(logs.EngineCouldNotInhumeObjectInShard, + e.log.Debug(ctx, logs.EngineCouldNotInhumeObjectInShard, zap.Stringer("addr", addr), zap.String("err", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -194,7 +194,7 @@ func (e *StorageEngine) deleteChunks( inhumePrm.MarkAsGarbage(addr) _, err = sh.Inhume(ctx, inhumePrm) if err != nil { - e.log.Debug(logs.EngineCouldNotInhumeObjectInShard, + e.log.Debug(ctx, logs.EngineCouldNotInhumeObjectInShard, zap.Stringer("addr", addr), zap.String("err", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 3183d6137..8963ec099 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -99,20 +99,20 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta if isMeta { err := sh.SetMode(mode.DegradedReadOnly) if err == nil { - log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) + log.Info(context.Background(), logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) return } - log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, + log.Error(context.Background(), logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, zap.Error(err)) } err := sh.SetMode(mode.ReadOnly) if err != nil { - log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) + log.Error(context.Background(), logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) return } - log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) + log.Info(context.Background(), logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } // reportShardErrorByID increases shard error counter and logs an error. @@ -137,7 +137,7 @@ func (e *StorageEngine) reportShardError( fields ...zap.Field, ) { if isLogical(err) { - e.log.Warn(msg, + e.log.Warn(context.Background(), msg, zap.Stringer("shard_id", sh.ID()), zap.String("error", err.Error())) return @@ -147,7 +147,7 @@ func (e *StorageEngine) reportShardError( e.metrics.IncErrorCounter(sh.ID().String()) sid := sh.ID() - e.log.Warn(msg, append([]zap.Field{ + e.log.Warn(context.Background(), msg, append([]zap.Field{ zap.Stringer("shard_id", sid), zap.Uint32("error count", errCount), zap.String("error", err.Error()), @@ -168,7 +168,7 @@ func (e *StorageEngine) reportShardError( default: // For background workers we can have a lot of such errors, // thus logging is done with DEBUG level. - e.log.Debug(logs.EngineModeChangeIsInProgressIgnoringSetmodeRequest, + e.log.Debug(context.Background(), logs.EngineModeChangeIsInProgressIgnoringSetmodeRequest, zap.Stringer("shard_id", sid), zap.Uint32("error_count", errCount)) } @@ -197,7 +197,7 @@ type cfg struct { func defaultCfg() *cfg { res := &cfg{ - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), shardPoolSize: 20, metrics: noopMetrics{}, } @@ -269,8 +269,8 @@ type containerSource struct { func (s *containerSource) IsContainerAvailable(ctx context.Context, id cid.ID) (bool, error) { select { - case <-ctx.Done(): - return false, ctx.Err() + case <-context.Background().Done(): + return false, context.Background().Err() default: } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 940e30323..b88c249b1 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -297,12 +297,12 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p e.evacuateLimiter.Complete(err) }() - e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, + e.log.Info(ctx, logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) err = e.getTotals(ctx, prm, shardsToEvacuate, res) if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField, + e.log.Error(ctx, logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) return err } @@ -336,12 +336,12 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p err = errors.Join(err, fmt.Errorf("object error: %w", errObject)) } if err != nil { - e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, + e.log.Error(ctx, logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) return err } - e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, + e.log.Info(ctx, logs.EngineFinishedSuccessfullyShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, zap.Uint64("total_objects", res.ObjectsTotal()), @@ -494,7 +494,7 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context err := sh.IterateOverContainers(ctx, cntPrm) if err != nil { cancel(err) - e.log.Error(logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField, + e.log.Error(ctx, logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } return err @@ -551,7 +551,7 @@ func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, tree return err } if success { - e.log.Debug(logs.EngineShardsEvacuationTreeEvacuatedLocal, + e.log.Debug(ctx, logs.EngineShardsEvacuationTreeEvacuatedLocal, zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), zap.String("from_shard_id", sh.ID().String()), zap.String("to_shard_id", shardID), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -561,26 +561,26 @@ func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, tree moved, nodePK, err := e.evacuateTreeToOtherNode(ctx, sh, contTree, prm) if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToMoveTree, + e.log.Error(ctx, logs.EngineShardsEvacuationFailedToMoveTree, zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), zap.String("from_shard_id", sh.ID().String()), evacuationOperationLogField, zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } if moved { - e.log.Debug(logs.EngineShardsEvacuationTreeEvacuatedRemote, + e.log.Debug(ctx, logs.EngineShardsEvacuationTreeEvacuatedRemote, zap.String("cid", contTree.CID.EncodeToString()), zap.String("treeID", contTree.TreeID), zap.String("from_shardID", sh.ID().String()), zap.String("to_node", nodePK), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) res.trEvacuated.Add(1) } else if prm.IgnoreErrors { res.trFailed.Add(1) - e.log.Warn(logs.EngineShardsEvacuationFailedToMoveTree, + e.log.Warn(ctx, logs.EngineShardsEvacuationFailedToMoveTree, zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), zap.String("from_shard_id", sh.ID().String()), evacuationOperationLogField, zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } else { - e.log.Error(logs.EngineShardsEvacuationFailedToMoveTree, + e.log.Error(ctx, logs.EngineShardsEvacuationFailedToMoveTree, zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), zap.String("from_shard_id", sh.ID().String()), evacuationOperationLogField, zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -770,7 +770,7 @@ func (e *StorageEngine) evacuateObject(ctx context.Context, shardID string, objI res.objFailed.Add(1) return nil } - e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, + e.log.Error(ctx, logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } @@ -792,7 +792,7 @@ func (e *StorageEngine) evacuateObject(ctx context.Context, shardID string, objI moved, err := prm.ObjectsHandler(ctx, addr, getRes.Object()) if err != nil { - e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, + e.log.Error(ctx, logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } @@ -800,7 +800,7 @@ func (e *StorageEngine) evacuateObject(ctx context.Context, shardID string, objI res.objEvacuated.Add(1) } else if prm.IgnoreErrors { res.objFailed.Add(1) - e.log.Warn(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, + e.log.Warn(ctx, logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } else { return fmt.Errorf("object %s was not replicated", addr) @@ -835,7 +835,7 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add switch e.putToShard(ctx, shards[j].hashedShard, shards[j].pool, addr, object, container.IsIndexedContainer(cnr)).status { case putToShardSuccess: res.objEvacuated.Add(1) - e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, + e.log.Debug(ctx, logs.EngineObjectIsMovedToAnotherShard, zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), zap.Stringer("addr", addr), diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index a1fe8a010..d6827e6c3 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -104,7 +104,7 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { return GetRes{}, it.OutError } if it.ShardWithMeta.Shard != nil && it.MetaError != nil { - e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound, + e.log.Warn(ctx, logs.ShardMetaInfoPresentButObjectNotFound, zap.Stringer("shard_id", it.ShardWithMeta.ID()), zap.String("error", it.MetaError.Error()), zap.Stringer("address", prm.addr), diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index b8959b534..1dc64c174 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -90,7 +90,7 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e if !prm.forceRemoval { locked, err := e.IsLocked(ctx, prm.addrs[i]) if err != nil { - e.log.Warn(logs.EngineRemovingAnObjectWithoutFullLockingCheck, + e.log.Warn(ctx, logs.EngineRemovingAnObjectWithoutFullLockingCheck, zap.Error(err), zap.Stringer("addr", prm.addrs[i]), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -264,7 +264,7 @@ func (e *StorageEngine) processExpiredLocks(ctx context.Context, epoch uint64, l select { case <-ctx.Done(): - e.log.Info(logs.EngineInterruptProcessingTheExpiredLocks, zap.Error(ctx.Err())) + e.log.Info(ctx, logs.EngineInterruptProcessingTheExpiredLocks, zap.Error(ctx.Err())) return true default: return false @@ -278,7 +278,7 @@ func (e *StorageEngine) processDeletedLocks(ctx context.Context, lockers []oid.A select { case <-ctx.Done(): - e.log.Info(logs.EngineInterruptProcessingTheDeletedLocks, zap.Error(ctx.Err())) + e.log.Info(ctx, logs.EngineInterruptProcessingTheDeletedLocks, zap.Error(ctx.Err())) return true default: return false @@ -305,7 +305,7 @@ func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid e.iterateOverUnsortedShards(func(sh hashedShard) bool { select { case <-ctx.Done(): - e.log.Info(logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(ctx.Err())) + e.log.Info(ctx, logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(ctx.Err())) failed = true return true default: @@ -316,7 +316,7 @@ func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid prm.SetContainerID(id) s, err := sh.ContainerSize(prm) if err != nil { - e.log.Warn(logs.EngineFailedToGetContainerSize, zap.Stringer("container_id", id), zap.Error(err)) + e.log.Warn(ctx, logs.EngineFailedToGetContainerSize, zap.Stringer("container_id", id), zap.Error(err)) failed = true return true } @@ -338,7 +338,7 @@ func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid e.iterateOverUnsortedShards(func(sh hashedShard) bool { select { case <-ctx.Done(): - e.log.Info(logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(ctx.Err())) + e.log.Info(ctx, logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(ctx.Err())) failed = true return true default: @@ -346,7 +346,7 @@ func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid for id := range idMap { if err := sh.DeleteContainerSize(ctx, id); err != nil { - e.log.Warn(logs.EngineFailedToDeleteContainerSize, zap.Stringer("container_id", id), zap.Error(err)) + e.log.Warn(ctx, logs.EngineFailedToDeleteContainerSize, zap.Stringer("container_id", id), zap.Error(err)) failed = true return true } @@ -383,7 +383,7 @@ func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []ci e.iterateOverUnsortedShards(func(sh hashedShard) bool { select { case <-ctx.Done(): - e.log.Info(logs.EngineInterruptProcessingZeroCountContainers, zap.Error(ctx.Err())) + e.log.Info(ctx, logs.EngineInterruptProcessingZeroCountContainers, zap.Error(ctx.Err())) failed = true return true default: @@ -394,7 +394,7 @@ func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []ci prm.ContainerID = id s, err := sh.ContainerCount(ctx, prm) if err != nil { - e.log.Warn(logs.EngineFailedToGetContainerCounters, zap.Stringer("container_id", id), zap.Error(err)) + e.log.Warn(ctx, logs.EngineFailedToGetContainerCounters, zap.Stringer("container_id", id), zap.Error(err)) failed = true return true } @@ -416,7 +416,7 @@ func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []ci e.iterateOverUnsortedShards(func(sh hashedShard) bool { select { case <-ctx.Done(): - e.log.Info(logs.EngineInterruptProcessingZeroCountContainers, zap.Error(ctx.Err())) + e.log.Info(ctx, logs.EngineInterruptProcessingZeroCountContainers, zap.Error(ctx.Err())) failed = true return true default: @@ -424,7 +424,7 @@ func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []ci for id := range idMap { if err := sh.DeleteContainerCount(ctx, id); err != nil { - e.log.Warn(logs.EngineFailedToDeleteContainerSize, zap.Stringer("container_id", id), zap.Error(err)) + e.log.Warn(ctx, logs.EngineFailedToDeleteContainerSize, zap.Stringer("container_id", id), zap.Error(err)) failed = true return true } @@ -449,7 +449,7 @@ func (e *StorageEngine) selectNonExistentIDs(ctx context.Context, ids []cid.ID) for _, id := range ids { isAvailable, err := cs.IsContainerAvailable(ctx, id) if err != nil { - e.log.Warn(logs.EngineFailedToCheckContainerAvailability, zap.Stringer("container_id", id), zap.Error(err)) + e.log.Warn(ctx, logs.EngineFailedToCheckContainerAvailability, zap.Stringer("container_id", id), zap.Error(err)) return nil, err } if isAvailable { diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 56d3ef490..635f0e302 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -141,7 +141,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti // expired => do nothing with it res.status = putToShardExists } else { - e.log.Warn(logs.EngineCouldNotCheckObjectExistence, + e.log.Warn(ctx, logs.EngineCouldNotCheckObjectExistence, zap.Stringer("shard_id", sh.ID()), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -163,14 +163,14 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti if err != nil { if errors.Is(err, shard.ErrReadOnlyMode) || errors.Is(err, blobstor.ErrNoPlaceFound) || errors.Is(err, common.ErrReadOnly) || errors.Is(err, common.ErrNoSpace) { - e.log.Warn(logs.EngineCouldNotPutObjectToShard, + e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, zap.Stringer("shard_id", sh.ID()), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return } if client.IsErrObjectAlreadyRemoved(err) { - e.log.Warn(logs.EngineCouldNotPutObjectToShard, + e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, zap.Stringer("shard_id", sh.ID()), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -185,7 +185,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti res.status = putToShardSuccess }); err != nil { - e.log.Warn(logs.EngineCouldNotPutObjectToShard, zap.Error(err)) + e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, zap.Error(err)) close(exitCh) } diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index fde6052ae..c5c94eef7 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -116,7 +116,7 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error return RngRes{}, it.OutError } if it.ShardWithMeta.Shard != nil && it.MetaError != nil { - e.log.Warn(logs.ShardMetaInfoPresentButObjectNotFound, + e.log.Warn(ctx, logs.ShardMetaInfoPresentButObjectNotFound, zap.Stringer("shard_id", it.ShardWithMeta.ID()), zap.String("error", it.MetaError.Error()), zap.Stringer("address", prm.addr), diff --git a/pkg/local_object_storage/engine/remove_copies.go b/pkg/local_object_storage/engine/remove_copies.go index 5e1ced56a..8ab3c5217 100644 --- a/pkg/local_object_storage/engine/remove_copies.go +++ b/pkg/local_object_storage/engine/remove_copies.go @@ -43,7 +43,7 @@ func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicat prm.Concurrency = defaultRemoveDuplicatesConcurrency } - e.log.Info(logs.EngineStartingRemovalOfLocallyredundantCopies, + e.log.Info(ctx, logs.EngineStartingRemovalOfLocallyredundantCopies, zap.Int("concurrency", prm.Concurrency)) // The mutext must be taken for the whole duration to avoid target shard being removed @@ -55,7 +55,7 @@ func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicat // This is not currently the case, because `FreeSpace` metric used by weight sorting is always 0. // However we could change weights in future and easily forget this function. for _, sh := range e.shards { - e.log.Debug(logs.EngineStartedDuplicatesRemovalRoutine, zap.Stringer("shard_id", sh.ID())) + e.log.Debug(ctx, logs.EngineStartedDuplicatesRemovalRoutine, zap.Stringer("shard_id", sh.ID())) ch := make(chan oid.Address) errG, ctx := errgroup.WithContext(ctx) @@ -93,12 +93,12 @@ func (e *StorageEngine) RemoveDuplicates(ctx context.Context, prm RemoveDuplicat }) } if err := errG.Wait(); err != nil { - e.log.Error(logs.EngineFinishedRemovalOfLocallyredundantCopies, zap.Error(err)) + e.log.Error(ctx, logs.EngineFinishedRemovalOfLocallyredundantCopies, zap.Error(err)) return err } } - e.log.Info(logs.EngineFinishedRemovalOfLocallyredundantCopies) + e.log.Info(ctx, logs.EngineFinishedRemovalOfLocallyredundantCopies) return nil } diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index aab2c423c..e172706e3 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -140,7 +140,7 @@ func (e *StorageEngine) createShard(_ context.Context, opts []shard.Option) (*sh )...) if err := sh.UpdateID(); err != nil { - e.log.Warn(logs.FailedToUpdateShardID, zap.Stringer("shard_id", sh.ID()), zap.String("metabase_path", sh.DumpInfo().MetaBaseInfo.Path), zap.Error(err)) + e.log.Warn(context.Background(), logs.FailedToUpdateShardID, zap.Stringer("shard_id", sh.ID()), zap.String("metabase_path", sh.DumpInfo().MetaBaseInfo.Path), zap.Error(err)) } return sh, nil @@ -228,7 +228,7 @@ func (e *StorageEngine) removeShards(ids ...string) { delete(e.shardPools, id) } - e.log.Info(logs.EngineShardHasBeenRemoved, + e.log.Info(context.Background(), logs.EngineShardHasBeenRemoved, zap.String("id", id)) } e.mtx.Unlock() @@ -236,14 +236,14 @@ func (e *StorageEngine) removeShards(ids ...string) { for _, sh := range ss { err := sh.SetMode(mode.Disabled) if err != nil { - e.log.Error(logs.EngineCouldNotChangeShardModeToDisabled, + e.log.Error(context.Background(), logs.EngineCouldNotChangeShardModeToDisabled, zap.Stringer("id", sh.ID()), zap.Error(err), ) } err = sh.Close() if err != nil { - e.log.Error(logs.EngineCouldNotCloseRemovedShard, + e.log.Error(context.Background(), logs.EngineCouldNotCloseRemovedShard, zap.Stringer("id", sh.ID()), zap.Error(err), ) @@ -340,7 +340,7 @@ func (e *StorageEngine) HandleNewEpoch(ctx context.Context, epoch uint64) { return case sh.NotificationChannel() <- ev: default: - e.log.Debug(logs.ShardEventProcessingInProgress, + e.log.Debug(ctx, logs.ShardEventProcessingInProgress, zap.Uint64("epoch", epoch), zap.Stringer("shard", sh.ID())) } } @@ -369,7 +369,7 @@ func (e *StorageEngine) closeShards(deletedShards []hashedShard) error { eg.Go(func() error { err := sh.SetMode(mode.Disabled) if err != nil { - e.log.Error(logs.EngineCouldNotChangeShardModeToDisabled, + e.log.Error(context.Background(), logs.EngineCouldNotChangeShardModeToDisabled, zap.Stringer("id", sh.ID()), zap.Error(err), ) @@ -380,7 +380,7 @@ func (e *StorageEngine) closeShards(deletedShards []hashedShard) error { err = sh.Close() if err != nil { - e.log.Error(logs.EngineCouldNotCloseRemovedShard, + e.log.Error(context.Background(), logs.EngineCouldNotCloseRemovedShard, zap.Stringer("id", sh.ID()), zap.Error(err), ) @@ -432,7 +432,7 @@ func (e *StorageEngine) deleteShards(ids []*shard.ID) ([]hashedShard, error) { delete(e.shardPools, idStr) } - e.log.Info(logs.EngineShardHasBeenRemoved, + e.log.Info(context.Background(), logs.EngineShardHasBeenRemoved, zap.String("id", idStr)) } diff --git a/pkg/local_object_storage/internal/log/log.go b/pkg/local_object_storage/internal/log/log.go index 23740868d..6b101fa60 100644 --- a/pkg/local_object_storage/internal/log/log.go +++ b/pkg/local_object_storage/internal/log/log.go @@ -1,14 +1,16 @@ package storagelog import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) // Write writes message about storage engine's operation to logger. -func Write(logger *logger.Logger, fields ...zap.Field) { - logger.Debug(logs.StorageOperation, fields...) +func Write(ctx context.Context, logger *logger.Logger, fields ...zap.Field) { + logger.Debug(ctx, logs.StorageOperation, fields...) } // AddressField returns logger's field for object address. diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index d6546d922..68e065a0a 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -57,7 +57,7 @@ func (db *DB) openDB(mode mode.Mode) error { return fmt.Errorf("can't create dir %s for metabase: %w", db.info.Path, err) } - db.log.Debug(logs.MetabaseCreatedDirectoryForMetabase, zap.String("path", db.info.Path)) + db.log.Debug(context.Background(), logs.MetabaseCreatedDirectoryForMetabase, zap.String("path", db.info.Path)) if db.boltOptions == nil { opts := *bbolt.DefaultOptions @@ -78,9 +78,9 @@ func (db *DB) openBolt() error { db.boltDB.MaxBatchDelay = db.boltBatchDelay db.boltDB.MaxBatchSize = db.boltBatchSize - db.log.Debug(logs.MetabaseOpenedBoltDBInstanceForMetabase) + db.log.Debug(context.Background(), logs.MetabaseOpenedBoltDBInstanceForMetabase) - db.log.Debug(logs.MetabaseCheckingMetabaseVersion) + db.log.Debug(context.Background(), logs.MetabaseCheckingMetabaseVersion) return db.boltDB.View(func(tx *bbolt.Tx) error { // The safest way to check if the metabase is fresh is to check if it has no buckets. // However, shard info can be present. So here we check that the number of buckets is diff --git a/pkg/local_object_storage/metabase/db.go b/pkg/local_object_storage/metabase/db.go index 2cd990814..4474aa229 100644 --- a/pkg/local_object_storage/metabase/db.go +++ b/pkg/local_object_storage/metabase/db.go @@ -70,7 +70,7 @@ func defaultCfg() *cfg { }, boltBatchDelay: bbolt.DefaultMaxBatchDelay, boltBatchSize: bbolt.DefaultMaxBatchSize, - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), metrics: &noopMetrics{}, } } diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index b5ac22017..62ab1056d 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -117,7 +117,7 @@ func (db *DB) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { if err == nil { deleted = true for i := range prm.addrs { - storagelog.Write(db.log, + storagelog.Write(ctx, db.log, storagelog.AddressField(prm.addrs[i]), storagelog.OpField("metabase DELETE")) } diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 12f27d330..8d1e18729 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -205,7 +205,7 @@ func (db *DB) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { success = err == nil if success { for _, addr := range prm.target { - storagelog.Write(db.log, + storagelog.Write(ctx, db.log, storagelog.AddressField(addr), storagelog.OpField("metabase INHUME")) } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 2d94e7ae1..d7675869f 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -101,7 +101,7 @@ func (db *DB) Put(ctx context.Context, prm PutPrm) (res PutRes, err error) { }) if err == nil { success = true - storagelog.Write(db.log, + storagelog.Write(ctx, db.log, storagelog.AddressField(objectCore.AddressOf(prm.obj)), storagelog.OpField("metabase PUT")) } diff --git a/pkg/local_object_storage/metabase/upgrade_test.go b/pkg/local_object_storage/metabase/upgrade_test.go index aeb14aeb6..e2eee86b0 100644 --- a/pkg/local_object_storage/metabase/upgrade_test.go +++ b/pkg/local_object_storage/metabase/upgrade_test.go @@ -113,7 +113,7 @@ func TestGenerateMetabaseFile(t *testing.T) { }) } require.NoError(t, eg.Wait()) - db.log.Info("simple objects generated") + db.log.Info(ctx, "simple objects generated") eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(generateWorkersCount) // complex objects @@ -137,7 +137,7 @@ func TestGenerateMetabaseFile(t *testing.T) { }) } require.NoError(t, eg.Wait()) - db.log.Info("complex objects generated") + db.log.Info(ctx, "complex objects generated") eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(generateWorkersCount) // simple objects deleted by gc marks @@ -159,7 +159,7 @@ func TestGenerateMetabaseFile(t *testing.T) { }) } require.NoError(t, eg.Wait()) - db.log.Info("simple objects deleted by gc marks generated") + db.log.Info(ctx, "simple objects deleted by gc marks generated") eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(10000) // simple objects deleted by tombstones @@ -189,7 +189,7 @@ func TestGenerateMetabaseFile(t *testing.T) { }) } require.NoError(t, eg.Wait()) - db.log.Info("simple objects deleted by tombstones generated") + db.log.Info(ctx, "simple objects deleted by tombstones generated") eg, ctx = errgroup.WithContext(context.Background()) eg.SetLimit(generateWorkersCount) // simple objects locked by locks @@ -216,7 +216,7 @@ func TestGenerateMetabaseFile(t *testing.T) { }) } require.NoError(t, eg.Wait()) - db.log.Info("simple objects locked by locks generated") + db.log.Info(ctx, "simple objects locked by locks generated") require.NoError(t, db.boltDB.Sync()) require.NoError(t, db.Close()) } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 62800dbd0..eb3aa61c0 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -21,7 +21,7 @@ import ( ) func (s *Shard) handleMetabaseFailure(stage string, err error) error { - s.log.Error(logs.ShardMetabaseFailureSwitchingMode, + s.log.Error(context.Background(), logs.ShardMetabaseFailureSwitchingMode, zap.String("stage", stage), zap.Stringer("mode", mode.ReadOnly), zap.Error(err)) @@ -31,7 +31,7 @@ func (s *Shard) handleMetabaseFailure(stage string, err error) error { return nil } - s.log.Error(logs.ShardCantMoveShardToReadonlySwitchMode, + s.log.Error(context.Background(), logs.ShardCantMoveShardToReadonlySwitchMode, zap.String("stage", stage), zap.Stringer("mode", mode.DegradedReadOnly), zap.Error(err)) @@ -211,7 +211,7 @@ func (s *Shard) refillMetabase(ctx context.Context) error { withCount := true totalObjects, err := s.blobStor.ObjectsCount(ctx) if err != nil { - s.log.Warn(logs.EngineRefillFailedToGetObjectsCount, zap.Error(err)) + s.log.Warn(ctx, logs.EngineRefillFailedToGetObjectsCount, zap.Error(err)) withCount = false } @@ -270,7 +270,7 @@ func (s *Shard) refillMetabase(ctx context.Context) error { func (s *Shard) refillObject(ctx context.Context, data []byte, addr oid.Address, descriptor []byte) error { obj := objectSDK.New() if err := obj.Unmarshal(data); err != nil { - s.log.Warn(logs.ShardCouldNotUnmarshalObject, + s.log.Warn(ctx, logs.ShardCouldNotUnmarshalObject, zap.Stringer("address", addr), zap.String("err", err.Error())) return nil @@ -285,7 +285,7 @@ func (s *Shard) refillObject(ctx context.Context, data []byte, addr oid.Address, return err } if info.Removed { - s.log.Debug(logs.ShardSkipObjectFromResyncContainerDeleted, zap.Stringer("address", addr)) + s.log.Debug(ctx, logs.ShardSkipObjectFromResyncContainerDeleted, zap.Stringer("address", addr)) return nil } isIndexedContainer = info.Indexed @@ -386,7 +386,7 @@ func (s *Shard) Close() error { for _, component := range components { if err := component.Close(); err != nil { lastErr = err - s.log.Error(logs.ShardCouldNotCloseShardComponent, zap.Error(err)) + s.log.Error(context.Background(), logs.ShardCouldNotCloseShardComponent, zap.Error(err)) } } @@ -424,7 +424,7 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { ok, err := s.metaBase.Reload(c.metaOpts...) if err != nil { if errors.Is(err, meta.ErrDegradedMode) { - s.log.Error(logs.ShardCantOpenMetabaseMoveToADegradedMode, zap.Error(err)) + s.log.Error(ctx, logs.ShardCantOpenMetabaseMoveToADegradedMode, zap.Error(err)) _ = s.setMode(mode.DegradedReadOnly) } return err @@ -440,7 +440,7 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { err = s.metaBase.Init() } if err != nil { - s.log.Error(logs.ShardCantInitializeMetabaseMoveToADegradedreadonlyMode, zap.Error(err)) + s.log.Error(ctx, logs.ShardCantInitializeMetabaseMoveToADegradedreadonlyMode, zap.Error(err)) _ = s.setMode(mode.DegradedReadOnly) return err } diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index c898fdf41..f62cecd56 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -95,7 +95,7 @@ func (s *Shard) validateWritecacheDoesntContainObject(ctx context.Context, addr } _, err := s.writeCache.Head(ctx, addr) if err == nil { - s.log.Warn(logs.ObjectRemovalFailureExistsInWritecache, zap.Stringer("object_address", addr)) + s.log.Warn(ctx, logs.ObjectRemovalFailureExistsInWritecache, zap.Stringer("object_address", addr)) return fmt.Errorf("object %s must be flushed from writecache", addr) } if client.IsErrObjectNotFound(err) { @@ -110,7 +110,7 @@ func (s *Shard) deleteFromBlobstor(ctx context.Context, addr oid.Address) error res, err := s.metaBase.StorageID(ctx, sPrm) if err != nil { - s.log.Debug(logs.StorageIDRetrievalFailure, + s.log.Debug(ctx, logs.StorageIDRetrievalFailure, zap.Stringer("object", addr), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -130,7 +130,7 @@ func (s *Shard) deleteFromBlobstor(ctx context.Context, addr oid.Address) error _, err = s.blobStor.Delete(ctx, delPrm) if err != nil && !client.IsErrObjectNotFound(err) { - s.log.Debug(logs.ObjectRemovalFailureBlobStor, + s.log.Debug(ctx, logs.ObjectRemovalFailureBlobStor, zap.Stringer("object_address", addr), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index d605746e8..6fabf7103 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -131,7 +131,7 @@ type gcCfg struct { func defaultGCCfg() gcCfg { return gcCfg{ removerInterval: 10 * time.Second, - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), workerPoolInit: func(int) util.WorkerPool { return nil }, @@ -161,14 +161,14 @@ func (gc *gc) listenEvents(ctx context.Context) { for { select { case <-gc.stopChannel: - gc.log.Warn(logs.ShardStopEventListenerByClosedStopChannel) + gc.log.Warn(ctx, logs.ShardStopEventListenerByClosedStopChannel) return case <-ctx.Done(): - gc.log.Warn(logs.ShardStopEventListenerByContext) + gc.log.Warn(ctx, logs.ShardStopEventListenerByContext) return case event, ok := <-gc.eventChan: if !ok { - gc.log.Warn(logs.ShardStopEventListenerByClosedEventChannel) + gc.log.Warn(ctx, logs.ShardStopEventListenerByClosedEventChannel) return } @@ -204,7 +204,7 @@ func (gc *gc) handleEvent(ctx context.Context, event Event) { h(runCtx, event) }) if err != nil { - gc.log.Warn(logs.ShardCouldNotSubmitGCJobToWorkerPool, + gc.log.Warn(ctx, logs.ShardCouldNotSubmitGCJobToWorkerPool, zap.String("error", err.Error()), ) @@ -222,7 +222,7 @@ func (gc *gc) releaseResources() { // because it is possible that we are close it earlier than stop writing. // It is ok to keep it opened. - gc.log.Debug(logs.ShardGCIsStopped) + gc.log.Debug(context.Background(), logs.ShardGCIsStopped) } func (gc *gc) tickRemover(ctx context.Context) { @@ -263,7 +263,7 @@ func (gc *gc) stop() { close(gc.stopChannel) }) - gc.log.Info(logs.ShardWaitingForGCWorkersToStop) + gc.log.Info(context.Background(), logs.ShardWaitingForGCWorkersToStop) gc.wg.Wait() } @@ -286,8 +286,8 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { return } - s.log.Debug(logs.ShardGCRemoveGarbageStarted) - defer s.log.Debug(logs.ShardGCRemoveGarbageCompleted) + s.log.Debug(ctx, logs.ShardGCRemoveGarbageStarted) + defer s.log.Debug(ctx, logs.ShardGCRemoveGarbageCompleted) buf := make([]oid.Address, 0, s.rmBatchSize) @@ -312,7 +312,7 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { // (no more than s.rmBatchSize objects) err := s.metaBase.IterateOverGarbage(ctx, iterPrm) if err != nil { - s.log.Warn(logs.ShardIteratorOverMetabaseGraveyardFailed, + s.log.Warn(ctx, logs.ShardIteratorOverMetabaseGraveyardFailed, zap.String("error", err.Error()), ) @@ -333,7 +333,7 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { result.success = true if err != nil { - s.log.Warn(logs.ShardCouldNotDeleteTheObjects, + s.log.Warn(ctx, logs.ShardCouldNotDeleteTheObjects, zap.String("error", err.Error()), ) result.success = false @@ -356,8 +356,8 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeRegular) }() - s.log.Debug(logs.ShardGCCollectingExpiredObjectsStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) - defer s.log.Debug(logs.ShardGCCollectingExpiredObjectsCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) + s.log.Debug(ctx, logs.ShardGCCollectingExpiredObjectsStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) + defer s.log.Debug(ctx, logs.ShardGCCollectingExpiredObjectsCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) workersCount, batchSize := s.getExpiredObjectsParameters() @@ -396,7 +396,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { }) if err = errGroup.Wait(); err != nil { - s.log.Warn(logs.ShardIteratorOverExpiredObjectsFailed, zap.String("error", err.Error())) + s.log.Warn(ctx, logs.ShardIteratorOverExpiredObjectsFailed, zap.String("error", err.Error())) } } @@ -416,7 +416,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) expired, err := s.getExpiredWithLinked(ctx, expired) if err != nil { - s.log.Warn(logs.ShardGCFailedToGetExpiredWithLinked, zap.Error(err)) + s.log.Warn(ctx, logs.ShardGCFailedToGetExpiredWithLinked, zap.Error(err)) return } @@ -428,7 +428,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) // inhume the collected objects res, err := s.metaBase.Inhume(ctx, inhumePrm) if err != nil { - s.log.Warn(logs.ShardCouldNotInhumeTheObjects, + s.log.Warn(ctx, logs.ShardCouldNotInhumeTheObjects, zap.String("error", err.Error()), ) @@ -473,8 +473,8 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { epoch := e.(newEpoch).epoch log := s.log.With(zap.Uint64("epoch", epoch)) - log.Debug(logs.ShardStartedExpiredTombstonesHandling) - defer log.Debug(logs.ShardFinishedExpiredTombstonesHandling) + log.Debug(ctx, logs.ShardStartedExpiredTombstonesHandling) + defer log.Debug(ctx, logs.ShardFinishedExpiredTombstonesHandling) const tssDeleteBatch = 50 tss := make([]meta.TombstonedObject, 0, tssDeleteBatch) @@ -492,12 +492,12 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { }) for { - log.Debug(logs.ShardIteratingTombstones) + log.Debug(ctx, logs.ShardIteratingTombstones) s.m.RLock() if s.info.Mode.NoMetabase() { - s.log.Debug(logs.ShardShardIsInADegradedModeSkipCollectingExpiredTombstones) + s.log.Debug(ctx, logs.ShardShardIsInADegradedModeSkipCollectingExpiredTombstones) s.m.RUnlock() return @@ -505,7 +505,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { err = s.metaBase.IterateOverGraveyard(ctx, iterPrm) if err != nil { - log.Error(logs.ShardIteratorOverGraveyardFailed, zap.Error(err)) + log.Error(ctx, logs.ShardIteratorOverGraveyardFailed, zap.Error(err)) s.m.RUnlock() return @@ -524,7 +524,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { } } - log.Debug(logs.ShardHandlingExpiredTombstonesBatch, zap.Int("number", len(tssExp))) + log.Debug(ctx, logs.ShardHandlingExpiredTombstonesBatch, zap.Int("number", len(tssExp))) if len(tssExp) > 0 { s.expiredTombstonesCallback(ctx, tssExp) } @@ -543,8 +543,8 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeLock) }() - s.log.Debug(logs.ShardGCCollectingExpiredLocksStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) - defer s.log.Debug(logs.ShardGCCollectingExpiredLocksCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) + s.log.Debug(ctx, logs.ShardGCCollectingExpiredLocksStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) + defer s.log.Debug(ctx, logs.ShardGCCollectingExpiredLocksCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) workersCount, batchSize := s.getExpiredObjectsParameters() @@ -584,7 +584,7 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { }) if err = errGroup.Wait(); err != nil { - s.log.Warn(logs.ShardIteratorOverExpiredLocksFailed, zap.String("error", err.Error())) + s.log.Warn(ctx, logs.ShardIteratorOverExpiredLocksFailed, zap.String("error", err.Error())) } } @@ -645,7 +645,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston // inhume tombstones res, err := s.metaBase.Inhume(ctx, pInhume) if err != nil { - s.log.Warn(logs.ShardCouldNotMarkTombstonesAsGarbage, + s.log.Warn(ctx, logs.ShardCouldNotMarkTombstonesAsGarbage, zap.String("error", err.Error()), ) @@ -668,7 +668,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston // from graveyard err = s.metaBase.DropGraves(ctx, tss) if err != nil { - s.log.Warn(logs.ShardCouldNotDropExpiredGraveRecords, zap.Error(err)) + s.log.Warn(ctx, logs.ShardCouldNotDropExpiredGraveRecords, zap.Error(err)) } } @@ -680,7 +680,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] } unlocked, err := s.metaBase.FreeLockedBy(lockers) if err != nil { - s.log.Warn(logs.ShardFailureToUnlockObjects, + s.log.Warn(ctx, logs.ShardFailureToUnlockObjects, zap.String("error", err.Error()), ) @@ -693,7 +693,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] res, err := s.metaBase.Inhume(ctx, pInhume) if err != nil { - s.log.Warn(logs.ShardFailureToMarkLockersAsGarbage, + s.log.Warn(ctx, logs.ShardFailureToMarkLockersAsGarbage, zap.String("error", err.Error()), ) @@ -718,7 +718,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] func (s *Shard) inhumeUnlockedIfExpired(ctx context.Context, epoch uint64, unlocked []oid.Address) { expiredUnlocked, err := s.selectExpired(ctx, epoch, unlocked) if err != nil { - s.log.Warn(logs.ShardFailureToGetExpiredUnlockedObjects, zap.Error(err)) + s.log.Warn(ctx, logs.ShardFailureToGetExpiredUnlockedObjects, zap.Error(err)) return } @@ -737,7 +737,7 @@ func (s *Shard) HandleDeletedLocks(lockers []oid.Address) { _, err := s.metaBase.FreeLockedBy(lockers) if err != nil { - s.log.Warn(logs.ShardFailureToUnlockObjects, + s.log.Warn(context.Background(), logs.ShardFailureToUnlockObjects, zap.String("error", err.Error()), ) @@ -756,8 +756,8 @@ func (s *Shard) collectExpiredMetrics(ctx context.Context, e Event) { epoch := e.(newEpoch).epoch - s.log.Debug(logs.ShardGCCollectingExpiredMetricsStarted, zap.Uint64("epoch", epoch)) - defer s.log.Debug(logs.ShardGCCollectingExpiredMetricsCompleted, zap.Uint64("epoch", epoch)) + s.log.Debug(ctx, logs.ShardGCCollectingExpiredMetricsStarted, zap.Uint64("epoch", epoch)) + defer s.log.Debug(ctx, logs.ShardGCCollectingExpiredMetricsCompleted, zap.Uint64("epoch", epoch)) s.collectExpiredContainerSizeMetrics(ctx, epoch) s.collectExpiredContainerCountMetrics(ctx, epoch) @@ -766,7 +766,7 @@ func (s *Shard) collectExpiredMetrics(ctx context.Context, e Event) { func (s *Shard) collectExpiredContainerSizeMetrics(ctx context.Context, epoch uint64) { ids, err := s.metaBase.ZeroSizeContainers(ctx) if err != nil { - s.log.Warn(logs.ShardGCFailedToCollectZeroSizeContainers, zap.Uint64("epoch", epoch), zap.Error(err)) + s.log.Warn(ctx, logs.ShardGCFailedToCollectZeroSizeContainers, zap.Uint64("epoch", epoch), zap.Error(err)) return } if len(ids) == 0 { @@ -778,7 +778,7 @@ func (s *Shard) collectExpiredContainerSizeMetrics(ctx context.Context, epoch ui func (s *Shard) collectExpiredContainerCountMetrics(ctx context.Context, epoch uint64) { ids, err := s.metaBase.ZeroCountContainers(ctx) if err != nil { - s.log.Warn(logs.ShardGCFailedToCollectZeroCountContainers, zap.Uint64("epoch", epoch), zap.Error(err)) + s.log.Warn(ctx, logs.ShardGCFailedToCollectZeroCountContainers, zap.Uint64("epoch", epoch), zap.Error(err)) return } if len(ids) == 0 { diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index d1c393613..7a31a705e 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -144,7 +144,7 @@ func (s *Shard) fetchObjectData(ctx context.Context, addr oid.Address, skipMeta return nil, false, logicerr.Wrap(new(apistatus.ObjectNotFound)) } } else { - s.log.Warn(logs.ShardFetchingObjectWithoutMeta, zap.Stringer("addr", addr)) + s.log.Warn(ctx, logs.ShardFetchingObjectWithoutMeta, zap.Stringer("addr", addr)) } if s.hasWriteCache() { @@ -153,12 +153,12 @@ func (s *Shard) fetchObjectData(ctx context.Context, addr oid.Address, skipMeta return res, false, err } if client.IsErrObjectNotFound(err) { - s.log.Debug(logs.ShardObjectIsMissingInWritecache, + s.log.Debug(ctx, logs.ShardObjectIsMissingInWritecache, zap.Stringer("addr", addr), zap.Bool("skip_meta", skipMeta), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } else { - s.log.Error(logs.ShardFailedToFetchObjectFromWritecache, + s.log.Error(ctx, logs.ShardFailedToFetchObjectFromWritecache, zap.Error(err), zap.Stringer("addr", addr), zap.Bool("skip_meta", skipMeta), diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index a72313498..e27dc0733 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -5,7 +5,6 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/mr-tron/base58" "go.uber.org/zap" ) @@ -50,7 +49,7 @@ func (s *Shard) UpdateID() (err error) { s.writeCache.GetMetrics().SetShardID(shardID) } - s.log = &logger.Logger{Logger: s.log.With(zap.Stringer("shard_id", s.info.ID))} + s.log = s.log.With(zap.Stringer("shard_id", s.info.ID)) s.metaBase.SetLogger(s.log) s.blobStor.SetLogger(s.log) if s.hasWriteCache() { diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 746177c3a..984c54fbc 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -109,7 +109,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { return InhumeRes{}, ErrLockObjectRemoval } - s.log.Debug(logs.ShardCouldNotMarkObjectToDeleteInMetabase, + s.log.Debug(ctx, logs.ShardCouldNotMarkObjectToDeleteInMetabase, zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index 8d09974b8..7b267d2e4 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -122,7 +122,7 @@ func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { sRes, err := s.metaBase.Select(ctx, sPrm) // consider making List in metabase if err != nil { - s.log.Debug(logs.ShardCantSelectAllObjects, + s.log.Debug(ctx, logs.ShardCantSelectAllObjects, zap.Stringer("cid", lst[i]), zap.String("error", err.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 9ce95feb1..595afb60e 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -30,7 +30,7 @@ func TestShard_Lock(t *testing.T) { rootPath := t.TempDir() opts := []Option{ WithID(NewIDFromBytes([]byte{})), - WithLogger(&logger.Logger{Logger: zap.NewNop()}), + WithLogger(logger.NewLoggerWrapper(zap.NewNop())), WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{ { diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index d90a5f4b6..98b4c37b2 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -1,6 +1,8 @@ package shard import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -26,7 +28,7 @@ func (s *Shard) SetMode(m mode.Mode) error { } func (s *Shard) setMode(m mode.Mode) error { - s.log.Info(logs.ShardSettingShardMode, + s.log.Info(context.Background(), logs.ShardSettingShardMode, zap.Stringer("old_mode", s.info.Mode), zap.Stringer("new_mode", m)) @@ -67,7 +69,7 @@ func (s *Shard) setMode(m mode.Mode) error { s.info.Mode = m s.metricsWriter.SetMode(s.info.Mode) - s.log.Info(logs.ShardShardModeSetSuccessfully, + s.log.Info(context.Background(), logs.ShardShardModeSetSuccessfully, zap.Stringer("mode", s.info.Mode)) return nil } diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index 24cc75154..50125a88d 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -75,7 +75,7 @@ func (s *Shard) Put(ctx context.Context, prm PutPrm) (PutRes, error) { } if err != nil || !tryCache { if err != nil { - s.log.Debug(logs.ShardCantPutObjectToTheWritecacheTryingBlobstor, + s.log.Debug(ctx, logs.ShardCantPutObjectToTheWritecacheTryingBlobstor, zap.String("err", err.Error())) } diff --git a/pkg/local_object_storage/shard/rebuild.go b/pkg/local_object_storage/shard/rebuild.go index 0d83caa0c..124b72a5c 100644 --- a/pkg/local_object_storage/shard/rebuild.go +++ b/pkg/local_object_storage/shard/rebuild.go @@ -102,11 +102,11 @@ func runRebuild(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *lo return default: } - log.Info(logs.BlobstoreRebuildStarted) + log.Info(ctx, logs.BlobstoreRebuildStarted) if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, limiter, fillPercent); err != nil { - log.Warn(logs.FailedToRebuildBlobstore, zap.Error(err)) + log.Warn(ctx, logs.FailedToRebuildBlobstore, zap.Error(err)) } else { - log.Info(logs.BlobstoreRebuildCompletedSuccessfully) + log.Info(ctx, logs.BlobstoreRebuildCompletedSuccessfully) } } @@ -138,7 +138,7 @@ func (r *rebuilder) Stop(log *logger.Logger) { r.wg.Wait() r.cancel = nil r.done = nil - log.Info(logs.BlobstoreRebuildStopped) + log.Info(context.Background(), logs.BlobstoreRebuildStopped) } var errMBIsNotAvailable = errors.New("metabase is not available") diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 413bfd2f7..3a06fe8a7 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -103,7 +103,7 @@ type cfg struct { func defaultCfg() *cfg { return &cfg{ rmBatchSize: 100, - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), gcCfg: defaultGCCfg(), reportErrorFunc: func(string, string, error) {}, zeroSizeContainersCallback: func(context.Context, []cid.ID) {}, @@ -401,7 +401,7 @@ func (s *Shard) updateMetrics(ctx context.Context) { cc, err := s.metaBase.ObjectCounters() if err != nil { - s.log.Warn(logs.ShardMetaObjectCounterRead, + s.log.Warn(ctx, logs.ShardMetaObjectCounterRead, zap.Error(err), ) @@ -414,7 +414,7 @@ func (s *Shard) updateMetrics(ctx context.Context) { cnrList, err := s.metaBase.Containers(ctx) if err != nil { - s.log.Warn(logs.ShardMetaCantReadContainerList, zap.Error(err)) + s.log.Warn(ctx, logs.ShardMetaCantReadContainerList, zap.Error(err)) return } @@ -423,7 +423,7 @@ func (s *Shard) updateMetrics(ctx context.Context) { for i := range cnrList { size, err := s.metaBase.ContainerSize(cnrList[i]) if err != nil { - s.log.Warn(logs.ShardMetaCantReadContainerSize, + s.log.Warn(ctx, logs.ShardMetaCantReadContainerSize, zap.String("cid", cnrList[i].EncodeToString()), zap.Error(err)) continue @@ -436,7 +436,7 @@ func (s *Shard) updateMetrics(ctx context.Context) { contCount, err := s.metaBase.ContainerCounters(ctx) if err != nil { - s.log.Warn(logs.FailedToGetContainerCounters, zap.Error(err)) + s.log.Warn(ctx, logs.FailedToGetContainerCounters, zap.Error(err)) return } for contID, count := range contCount.Counts { diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index a6de07f03..f655e477a 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -124,12 +124,12 @@ func (s *Shard) SealWriteCache(ctx context.Context, p SealWriteCachePrm) error { close(started) defer cleanup() - s.log.Info(logs.StartedWritecacheSealAsync) + s.log.Info(ctx, logs.StartedWritecacheSealAsync) if err := s.writeCache.Seal(ctx, prm); err != nil { - s.log.Warn(logs.FailedToSealWritecacheAsync, zap.Error(err)) + s.log.Warn(ctx, logs.FailedToSealWritecacheAsync, zap.Error(err)) return } - s.log.Info(logs.WritecacheSealCompletedAsync) + s.log.Info(ctx, logs.WritecacheSealCompletedAsync) }() select { case <-ctx.Done(): diff --git a/pkg/local_object_storage/writecache/cache.go b/pkg/local_object_storage/writecache/cache.go index b97fc5856..098872e08 100644 --- a/pkg/local_object_storage/writecache/cache.go +++ b/pkg/local_object_storage/writecache/cache.go @@ -55,7 +55,7 @@ func New(opts ...Option) Cache { counter: fstree.NewSimpleCounter(), options: options{ - log: &logger.Logger{Logger: zap.NewNop()}, + log: logger.NewLoggerWrapper(zap.NewNop()), maxObjectSize: defaultMaxObjectSize, workersCount: defaultFlushWorkersCount, maxCacheSize: defaultMaxCacheSize, diff --git a/pkg/local_object_storage/writecache/delete.go b/pkg/local_object_storage/writecache/delete.go index dda284439..94a0a40db 100644 --- a/pkg/local_object_storage/writecache/delete.go +++ b/pkg/local_object_storage/writecache/delete.go @@ -46,7 +46,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error { storageType = StorageTypeFSTree _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr}) if err == nil { - storagelog.Write(c.log, + storagelog.Write(ctx, c.log, storagelog.AddressField(addr.EncodeToString()), storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree DELETE"), diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index bfa6aacb0..123eb4abc 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -80,7 +80,7 @@ func (c *cache) pushToFlushQueue(ctx context.Context, fl *flushLimiter) { } }) if err != nil { - c.log.Warn(logs.BlobstorErrorOccurredDuringTheIteration, zap.Error(err)) + c.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, zap.Error(err)) } c.modeMtx.RUnlock() @@ -130,7 +130,7 @@ func (c *cache) reportFlushError(msg string, addr string, err error) { if c.reportError != nil { c.reportError(msg, err) } else { - c.log.Error(msg, + c.log.Error(context.Background(), msg, zap.String("address", addr), zap.Error(err)) } diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 59a4e4895..26f47e82e 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -40,7 +40,7 @@ func TestFlush(t *testing.T) { cnt := &atomic.Uint32{} return WithReportErrorFunc(func(msg string, err error) { cnt.Add(1) - testlogger.Warn(msg, zap.Uint32("error_count", cnt.Load()), zap.Error(err)) + testlogger.Warn(context.Background(), msg, zap.Uint32("error_count", cnt.Load()), zap.Error(err)) }), cnt } diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index d12dd603b..26658e9b8 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -60,7 +60,7 @@ func (c *cache) setMode(ctx context.Context, m mode.Mode, prm setModePrm) error // flushCh is populated by `flush` with `modeMtx` taken, thus waiting until it is empty // guarantees that there are no in-fly operations. for len(c.flushCh) != 0 { - c.log.Info(logs.WritecacheWaitingForChannelsToFlush) + c.log.Info(ctx, logs.WritecacheWaitingForChannelsToFlush) time.Sleep(time.Second) } @@ -110,7 +110,7 @@ func (c *cache) closeStorage(ctx context.Context, shrink bool) error { return fmt.Errorf("failed to remove write-cache files: %w", err) } } else { - c.log.Info(logs.WritecacheShrinkSkippedNotEmpty) + c.log.Info(ctx, logs.WritecacheShrinkSkippedNotEmpty) } return nil } diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index 66ac7805c..25c1694a8 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -41,7 +41,7 @@ type options struct { // WithLogger sets logger. func WithLogger(log *logger.Logger) Option { return func(o *options) { - o.log = &logger.Logger{Logger: log.With(zap.String("component", "WriteCache"))} + o.log = log.With(zap.String("component", "WriteCache")) } } diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index c53067bea..7da5c4d3a 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -68,7 +68,7 @@ func (c *cache) putBig(ctx context.Context, prm common.PutPrm) error { return err } - storagelog.Write(c.log, + storagelog.Write(ctx, c.log, storagelog.AddressField(prm.Address.EncodeToString()), storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree PUT"), diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index 2e52e5b20..a0e236cb7 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -43,9 +43,9 @@ func (c *cache) openStore(mod mode.ComponentMode) error { func (c *cache) deleteFromDisk(ctx context.Context, addr oid.Address, size uint64) { _, err := c.fsTree.Delete(ctx, common.DeletePrm{Address: addr, Size: size}) if err != nil && !client.IsErrObjectNotFound(err) { - c.log.Error(logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) + c.log.Error(ctx, logs.WritecacheCantRemoveObjectFromWritecache, zap.Error(err)) } else if err == nil { - storagelog.Write(c.log, + storagelog.Write(ctx, c.log, storagelog.AddressField(addr.EncodeToString()), storagelog.StorageTypeField(wcStorageType), storagelog.OpField("fstree DELETE"), diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 933f1039f..12c0e0842 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -199,7 +199,7 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, return 0, fmt.Errorf("could not invoke %s: %w", method, err) } - c.logger.Debug(logs.ClientNeoClientInvoke, + c.logger.Debug(context.Background(), logs.ClientNeoClientInvoke, zap.String("method", method), zap.Uint32("vub", vub), zap.Stringer("tx_hash", txHash.Reverse())) @@ -328,7 +328,7 @@ func (c *Client) TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error return err } - c.logger.Debug(logs.ClientNativeGasTransferInvoke, + c.logger.Debug(context.Background(), logs.ClientNativeGasTransferInvoke, zap.String("to", receiver.StringLE()), zap.Stringer("tx_hash", txHash.Reverse()), zap.Uint32("vub", vub)) @@ -362,7 +362,7 @@ func (c *Client) BatchTransferGas(receivers []util.Uint160, amount fixedn.Fixed8 return err } - c.logger.Debug(logs.ClientBatchGasTransferInvoke, + c.logger.Debug(context.Background(), logs.ClientBatchGasTransferInvoke, zap.Strings("to", receiversLog), zap.Stringer("tx_hash", txHash.Reverse()), zap.Uint32("vub", vub)) @@ -389,7 +389,7 @@ func (c *Client) Wait(ctx context.Context, n uint32) error { height, err = c.rpcActor.GetBlockCount() if err != nil { - c.logger.Error(logs.ClientCantGetBlockchainHeight, + c.logger.Error(context.Background(), logs.ClientCantGetBlockchainHeight, zap.String("error", err.Error())) return nil } @@ -403,7 +403,7 @@ func (c *Client) Wait(ctx context.Context, n uint32) error { newHeight, err = c.rpcActor.GetBlockCount() if err != nil { - c.logger.Error(logs.ClientCantGetBlockchainHeight243, + c.logger.Error(context.Background(), logs.ClientCantGetBlockchainHeight243, zap.String("error", err.Error())) return nil } diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 08d16deb4..d061747bb 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -61,7 +61,7 @@ var ErrNoHealthyEndpoint = errors.New("no healthy endpoint") func defaultConfig() *cfg { return &cfg{ dialTimeout: defaultDialTimeout, - logger: &logger.Logger{Logger: zap.L()}, + logger: logger.NewLoggerWrapper(zap.L()), metrics: morphmetrics.NoopRegister{}, waitInterval: defaultWaitInterval, signer: &transaction.Signer{ @@ -130,10 +130,10 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er for cli.endpoints.curr, endpoint = range cli.endpoints.list { cli.client, act, err = cli.newCli(ctx, endpoint) if err != nil { - cli.logger.Warn(logs.FrostFSIRCouldntCreateRPCClientForEndpoint, + cli.logger.Warn(ctx, logs.FrostFSIRCouldntCreateRPCClientForEndpoint, zap.Error(err), zap.String("endpoint", endpoint.Address)) } else { - cli.logger.Info(logs.FrostFSIRCreatedRPCClientForEndpoint, + cli.logger.Info(ctx, logs.FrostFSIRCreatedRPCClientForEndpoint, zap.String("endpoint", endpoint.Address)) if cli.endpoints.curr > 0 && cli.cfg.switchInterval != 0 { cli.switchIsActive.Store(true) diff --git a/pkg/morph/client/multi.go b/pkg/morph/client/multi.go index 10ed21582..708d3b39f 100644 --- a/pkg/morph/client/multi.go +++ b/pkg/morph/client/multi.go @@ -42,7 +42,7 @@ func (c *Client) SwitchRPC(ctx context.Context) bool { newEndpoint := c.endpoints.list[c.endpoints.curr] cli, act, err := c.newCli(ctx, newEndpoint) if err != nil { - c.logger.Warn(logs.ClientCouldNotEstablishConnectionToTheSwitchedRPCNode, + c.logger.Warn(ctx, logs.ClientCouldNotEstablishConnectionToTheSwitchedRPCNode, zap.String("endpoint", newEndpoint.Address), zap.Error(err), ) @@ -52,7 +52,7 @@ func (c *Client) SwitchRPC(ctx context.Context) bool { c.cache.invalidate() - c.logger.Info(logs.ClientConnectionToTheNewRPCNodeHasBeenEstablished, + c.logger.Info(ctx, logs.ClientConnectionToTheNewRPCNodeHasBeenEstablished, zap.String("endpoint", newEndpoint.Address)) c.client = cli @@ -122,7 +122,7 @@ mainLoop: cli, act, err := c.newCli(ctx, e) if err != nil { - c.logger.Warn(logs.ClientCouldNotCreateClientToTheHigherPriorityNode, + c.logger.Warn(ctx, logs.ClientCouldNotCreateClientToTheHigherPriorityNode, zap.String("endpoint", tryE), zap.Error(err), ) @@ -147,7 +147,7 @@ mainLoop: c.switchLock.Unlock() - c.logger.Info(logs.ClientSwitchedToTheHigherPriorityRPC, + c.logger.Info(ctx, logs.ClientSwitchedToTheHigherPriorityRPC, zap.String("endpoint", tryE)) return diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 2a500b31b..58c417fb1 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -1,6 +1,7 @@ package client import ( + "context" "crypto/elliptic" "encoding/binary" "errors" @@ -201,7 +202,7 @@ func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (util.Uint256, // Transaction is already in mempool waiting to be processed. // This is an expected situation if we restart the service. - c.logger.Info(logs.ClientNotaryDepositHasAlreadyBeenMade, + c.logger.Info(context.Background(), logs.ClientNotaryDepositHasAlreadyBeenMade, zap.Int64("amount", int64(amount)), zap.Int64("expire_at", till), zap.Uint32("vub", vub), @@ -209,7 +210,7 @@ func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (util.Uint256, return util.Uint256{}, 0, nil } - c.logger.Info(logs.ClientNotaryDepositInvoke, + c.logger.Info(context.Background(), logs.ClientNotaryDepositInvoke, zap.Int64("amount", int64(amount)), zap.Int64("expire_at", till), zap.Uint32("vub", vub), @@ -429,7 +430,7 @@ func (c *Client) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { return err } - c.logger.Debug(logs.ClientNotaryRequestWithPreparedMainTXInvoked, + c.logger.Debug(context.Background(), logs.ClientNotaryRequestWithPreparedMainTXInvoked, zap.String("tx_hash", mainH.StringLE()), zap.Uint32("valid_until_block", untilActual), zap.String("fallback_hash", fbH.StringLE())) @@ -485,7 +486,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint return 0, err } - c.logger.Debug(logs.ClientNotaryRequestInvoked, + c.logger.Debug(context.Background(), logs.ClientNotaryRequestInvoked, zap.String("method", method), zap.Uint32("valid_until_block", untilActual), zap.String("tx_hash", mainH.StringLE()), diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index dd3c7d216..03bba8ab9 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -144,7 +144,7 @@ func (l *listener) Listen(ctx context.Context) { l.wg.Add(1) defer l.wg.Done() if err := l.listen(ctx, nil); err != nil { - l.log.Error(logs.EventCouldNotStartListenToEvents, + l.log.Error(ctx, logs.EventCouldNotStartListenToEvents, zap.String("error", err.Error()), ) } @@ -162,7 +162,7 @@ func (l *listener) ListenWithError(ctx context.Context, intError chan<- error) { l.wg.Add(1) defer l.wg.Done() if err := l.listen(ctx, intError); err != nil { - l.log.Error(logs.EventCouldNotStartListenToEvents, + l.log.Error(ctx, logs.EventCouldNotStartListenToEvents, zap.String("error", err.Error()), ) l.sendError(ctx, intError, err) @@ -234,7 +234,7 @@ func (l *listener) sendError(ctx context.Context, intErr chan<- error, err error // in the same routine when shutting down node. select { case <-ctx.Done(): - l.log.Info(logs.EventStopEventListenerByContext, + l.log.Info(ctx, logs.EventStopEventListenerByContext, zap.String("reason", ctx.Err().Error()), ) return false @@ -251,43 +251,43 @@ loop: select { case err := <-subErrCh: if !l.sendError(ctx, intErr, err) { - l.log.Error(logs.EventStopEventListenerByError, zap.Error(err)) + l.log.Error(ctx, logs.EventStopEventListenerByError, zap.Error(err)) } break loop case <-ctx.Done(): - l.log.Info(logs.EventStopEventListenerByContext, + l.log.Info(ctx, logs.EventStopEventListenerByContext, zap.String("reason", ctx.Err().Error()), ) break loop case notifyEvent, ok := <-chs.NotificationsCh: if !ok { - l.log.Warn(logs.EventStopEventListenerByNotificationChannel) + l.log.Warn(ctx, logs.EventStopEventListenerByNotificationChannel) l.sendError(ctx, intErr, errNotificationSubscrConnectionTerminated) break loop } else if notifyEvent == nil { - l.log.Warn(logs.EventNilNotificationEventWasCaught) + l.log.Warn(ctx, logs.EventNilNotificationEventWasCaught) continue loop } l.handleNotifyEvent(notifyEvent) case notaryEvent, ok := <-chs.NotaryRequestsCh: if !ok { - l.log.Warn(logs.EventStopEventListenerByNotaryChannel) + l.log.Warn(ctx, logs.EventStopEventListenerByNotaryChannel) l.sendError(ctx, intErr, errNotarySubscrConnectionTerminated) break loop } else if notaryEvent == nil { - l.log.Warn(logs.EventNilNotaryEventWasCaught) + l.log.Warn(ctx, logs.EventNilNotaryEventWasCaught) continue loop } l.handleNotaryEvent(notaryEvent) case b, ok := <-chs.BlockCh: if !ok { - l.log.Warn(logs.EventStopEventListenerByBlockChannel) + l.log.Warn(ctx, logs.EventStopEventListenerByBlockChannel) l.sendError(ctx, intErr, errBlockNotificationChannelClosed) break loop } else if b == nil { - l.log.Warn(logs.EventNilBlockWasCaught) + l.log.Warn(ctx, logs.EventNilBlockWasCaught) continue loop } @@ -302,7 +302,7 @@ func (l *listener) handleBlockEvent(b *block.Block) { l.blockHandlers[i](b) } }); err != nil { - l.log.Warn(logs.EventListenerWorkerPoolDrained, + l.log.Warn(context.Background(), logs.EventListenerWorkerPoolDrained, zap.Int("capacity", l.pool.Cap())) } } @@ -311,7 +311,7 @@ func (l *listener) handleNotaryEvent(notaryEvent *result.NotaryRequestEvent) { if err := l.pool.Submit(func() { l.parseAndHandleNotary(notaryEvent) }); err != nil { - l.log.Warn(logs.EventListenerWorkerPoolDrained, + l.log.Warn(context.Background(), logs.EventListenerWorkerPoolDrained, zap.Int("capacity", l.pool.Cap())) } } @@ -320,7 +320,7 @@ func (l *listener) handleNotifyEvent(notifyEvent *state.ContainedNotificationEve if err := l.pool.Submit(func() { l.parseAndHandleNotification(notifyEvent) }); err != nil { - l.log.Warn(logs.EventListenerWorkerPoolDrained, + l.log.Warn(context.Background(), logs.EventListenerWorkerPoolDrained, zap.Int("capacity", l.pool.Cap())) } } @@ -347,7 +347,7 @@ func (l *listener) parseAndHandleNotification(notifyEvent *state.ContainedNotifi l.mtx.RUnlock() if !ok { - log.Debug(logs.EventEventParserNotSet) + log.Debug(context.Background(), logs.EventEventParserNotSet) return } @@ -355,7 +355,7 @@ func (l *listener) parseAndHandleNotification(notifyEvent *state.ContainedNotifi // parse the notification event event, err := parser(notifyEvent) if err != nil { - log.Warn(logs.EventCouldNotParseNotificationEvent, + log.Warn(context.Background(), logs.EventCouldNotParseNotificationEvent, zap.String("error", err.Error()), ) @@ -368,7 +368,7 @@ func (l *listener) parseAndHandleNotification(notifyEvent *state.ContainedNotifi l.mtx.RUnlock() if len(handlers) == 0 { - log.Info(logs.EventNotificationHandlersForParsedNotificationEventWereNotRegistered, + log.Info(context.Background(), logs.EventNotificationHandlersForParsedNotificationEventWereNotRegistered, zap.Any("event", event), ) @@ -388,13 +388,13 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { switch { case errors.Is(err, ErrTXAlreadyHandled): case errors.As(err, &expErr): - l.log.Warn(logs.EventSkipExpiredMainTXNotaryEvent, + l.log.Warn(context.Background(), logs.EventSkipExpiredMainTXNotaryEvent, zap.String("error", err.Error()), zap.Uint32("current_block_height", expErr.CurrentBlockHeight), zap.Uint32("fallback_tx_not_valid_before_height", expErr.FallbackTXNotValidBeforeHeight), ) default: - l.log.Warn(logs.EventCouldNotPrepareAndValidateNotaryEvent, + l.log.Warn(context.Background(), logs.EventCouldNotPrepareAndValidateNotaryEvent, zap.String("error", err.Error()), ) } @@ -418,7 +418,7 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { l.mtx.RUnlock() if !ok { - log.Debug(logs.EventNotaryParserNotSet) + log.Debug(context.Background(), logs.EventNotaryParserNotSet) return } @@ -426,7 +426,7 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { // parse the notary event event, err := parser(notaryEvent) if err != nil { - log.Warn(logs.EventCouldNotParseNotaryEvent, + log.Warn(context.Background(), logs.EventCouldNotParseNotaryEvent, zap.String("error", err.Error()), ) @@ -439,7 +439,7 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { l.mtx.RUnlock() if !ok { - log.Info(logs.EventNotaryHandlersForParsedNotificationEventWereNotRegistered, + log.Info(context.Background(), logs.EventNotaryHandlersForParsedNotificationEventWereNotRegistered, zap.Any("event", event), ) @@ -461,7 +461,7 @@ func (l *listener) SetNotificationParser(pi NotificationParserInfo) { parser := pi.parser() if parser == nil { - log.Info(logs.EventIgnoreNilEventParser) + log.Info(context.Background(), logs.EventIgnoreNilEventParser) return } @@ -470,7 +470,7 @@ func (l *listener) SetNotificationParser(pi NotificationParserInfo) { // check if the listener was started if l.started { - log.Warn(logs.EventListenerHasBeenAlreadyStartedIgnoreParser) + log.Warn(context.Background(), logs.EventListenerHasBeenAlreadyStartedIgnoreParser) return } @@ -479,7 +479,7 @@ func (l *listener) SetNotificationParser(pi NotificationParserInfo) { l.notificationParsers[pi.scriptHashWithType] = pi.parser() } - log.Debug(logs.EventRegisteredNewEventParser) + log.Debug(context.Background(), logs.EventRegisteredNewEventParser) } // RegisterNotificationHandler registers the handler for particular notification event of contract. @@ -494,7 +494,7 @@ func (l *listener) RegisterNotificationHandler(hi NotificationHandlerInfo) { handler := hi.Handler() if handler == nil { - log.Warn(logs.EventIgnoreNilEventHandler) + log.Warn(context.Background(), logs.EventIgnoreNilEventHandler) return } @@ -504,7 +504,7 @@ func (l *listener) RegisterNotificationHandler(hi NotificationHandlerInfo) { l.mtx.RUnlock() if !ok { - log.Warn(logs.EventIgnoreHandlerOfEventWoParser) + log.Warn(context.Background(), logs.EventIgnoreHandlerOfEventWoParser) return } @@ -516,7 +516,7 @@ func (l *listener) RegisterNotificationHandler(hi NotificationHandlerInfo) { ) l.mtx.Unlock() - log.Debug(logs.EventRegisteredNewEventHandler) + log.Debug(context.Background(), logs.EventRegisteredNewEventHandler) } // EnableNotarySupport enables notary request listening. Passed hash is @@ -557,7 +557,7 @@ func (l *listener) SetNotaryParser(pi NotaryParserInfo) { parser := pi.parser() if parser == nil { - log.Info(logs.EventIgnoreNilNotaryEventParser) + log.Info(context.Background(), logs.EventIgnoreNilNotaryEventParser) return } @@ -566,7 +566,7 @@ func (l *listener) SetNotaryParser(pi NotaryParserInfo) { // check if the listener was started if l.started { - log.Warn(logs.EventListenerHasBeenAlreadyStartedIgnoreNotaryParser) + log.Warn(context.Background(), logs.EventListenerHasBeenAlreadyStartedIgnoreNotaryParser) return } @@ -575,7 +575,7 @@ func (l *listener) SetNotaryParser(pi NotaryParserInfo) { l.notaryParsers[pi.notaryRequestTypes] = pi.parser() } - log.Info(logs.EventRegisteredNewEventParser) + log.Info(context.Background(), logs.EventRegisteredNewEventParser) } // RegisterNotaryHandler registers the handler for particular notification notary request event. @@ -595,7 +595,7 @@ func (l *listener) RegisterNotaryHandler(hi NotaryHandlerInfo) { handler := hi.Handler() if handler == nil { - log.Warn(logs.EventIgnoreNilNotaryEventHandler) + log.Warn(context.Background(), logs.EventIgnoreNilNotaryEventHandler) return } @@ -605,7 +605,7 @@ func (l *listener) RegisterNotaryHandler(hi NotaryHandlerInfo) { l.mtx.RUnlock() if !ok { - log.Warn(logs.EventIgnoreHandlerOfNotaryEventWoParser) + log.Warn(context.Background(), logs.EventIgnoreHandlerOfNotaryEventWoParser) return } @@ -614,7 +614,7 @@ func (l *listener) RegisterNotaryHandler(hi NotaryHandlerInfo) { l.notaryHandlers[hi.notaryRequestTypes] = hi.Handler() l.mtx.Unlock() - log.Info(logs.EventRegisteredNewEventHandler) + log.Info(context.Background(), logs.EventRegisteredNewEventHandler) } // Stop closes subscription channel with remote neo node. @@ -628,7 +628,7 @@ func (l *listener) Stop() { func (l *listener) RegisterBlockHandler(handler BlockHandler) { if handler == nil { - l.log.Warn(logs.EventIgnoreNilBlockHandler) + l.log.Warn(context.Background(), logs.EventIgnoreNilBlockHandler) return } diff --git a/pkg/morph/event/utils.go b/pkg/morph/event/utils.go index f3b6443fb..31bbf4432 100644 --- a/pkg/morph/event/utils.go +++ b/pkg/morph/event/utils.go @@ -1,6 +1,7 @@ package event import ( + "context" "errors" "fmt" @@ -89,7 +90,7 @@ func WorkerPoolHandler(w util2.WorkerPool, h Handler, log *logger.Logger) Handle h(e) }) if err != nil { - log.Warn(logs.EventCouldNotSubmitHandlerToWorkerPool, + log.Warn(context.Background(), logs.EventCouldNotSubmitHandlerToWorkerPool, zap.String("error", err.Error()), ) } diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index ee5466a7d..3a2da6757 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -245,9 +245,9 @@ routeloop: } func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) bool { - s.log.Info(logs.RPConnectionLost) + s.log.Info(ctx, logs.RPConnectionLost) if !s.client.SwitchRPC(ctx) { - s.log.Error(logs.RPCNodeSwitchFailure) + s.log.Error(ctx, logs.RPCNodeSwitchFailure) return false } @@ -304,7 +304,7 @@ func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotific if s.subscribedToNewBlocks { _, err = s.client.ReceiveBlocks(blCh) if err != nil { - s.log.Error(logs.ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch, zap.Error(err)) + s.log.Error(context.Background(), logs.ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch, zap.Error(err)) return false } } @@ -313,7 +313,7 @@ func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotific for contract := range s.subscribedEvents { _, err = s.client.ReceiveExecutionNotifications(contract, notifCh) if err != nil { - s.log.Error(logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) + s.log.Error(context.Background(), logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) return false } } @@ -322,7 +322,7 @@ func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotific for signer := range s.subscribedNotaryEvents { _, err = s.client.ReceiveNotaryRequests(signer, notaryCh) if err != nil { - s.log.Error(logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) + s.log.Error(context.Background(), logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) return false } } diff --git a/pkg/services/apemanager/executor.go b/pkg/services/apemanager/executor.go index 86f9cb893..cc792e23d 100644 --- a/pkg/services/apemanager/executor.go +++ b/pkg/services/apemanager/executor.go @@ -53,7 +53,7 @@ func New(cnrSrc containercore.Source, contractStorage ape_contract.ProxyAdaptedC } if s.log == nil { - s.log = &logger.Logger{Logger: zap.NewNop()} + s.log = logger.NewLoggerWrapper(zap.NewNop()) } return s diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 56748b08c..93ad3dc46 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -64,7 +64,7 @@ func New(next object.ServiceServer, opts ...Option, ) Service { cfg := &cfg{ - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), next: next, nm: nm, irFetcher: irf, diff --git a/pkg/services/object/common/writer/common.go b/pkg/services/object/common/writer/common.go index 6689557ee..8b92d34ed 100644 --- a/pkg/services/object/common/writer/common.go +++ b/pkg/services/object/common/writer/common.go @@ -59,7 +59,7 @@ func (n *NodeIterator) ForEachNode(ctx context.Context, f func(context.Context, if n.Traversal.submitPrimaryPlacementFinish() { err := n.ForEachNode(ctx, f) if err != nil { - n.cfg.Logger.Error(logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) + n.cfg.Logger.Error(ctx, logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) // we don't fail primary operation because of broadcast failure } } diff --git a/pkg/services/object/common/writer/ec.go b/pkg/services/object/common/writer/ec.go index 571bae7bb..64115b86b 100644 --- a/pkg/services/object/common/writer/ec.go +++ b/pkg/services/object/common/writer/ec.go @@ -162,7 +162,7 @@ func (e *ECWriter) relayToContainerNode(ctx context.Context, objID oid.ID, index if err == nil { return nil } - e.Config.Logger.Logger.Warn(logs.ECFailedToSendToContainerNode, zap.Stringers("address_group", info.AddressGroup())) + e.Config.Logger.Warn(ctx, logs.ECFailedToSendToContainerNode, zap.Stringers("address_group", info.AddressGroup())) lastErr = err } } @@ -275,7 +275,7 @@ func (e *ECWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx if err == nil { return nil } - e.Config.Logger.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), + e.Config.Logger.Warn(ctx, logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), zap.Stringer("parent_address", obj.ECHeader().Parent()), zap.Int("part_index", partIdx), zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Error(err)) @@ -299,7 +299,7 @@ func (e *ECWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx if err == nil { return nil } - e.Config.Logger.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), + e.Config.Logger.Warn(ctx, logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), zap.Stringer("parent_address", obj.ECHeader().Parent()), zap.Int("part_index", partIdx), zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Error(err)) @@ -323,7 +323,7 @@ func (e *ECWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx if err == nil { return nil } - e.Config.Logger.Warn(logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), + e.Config.Logger.Warn(ctx, logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), zap.Stringer("parent_address", obj.ECHeader().Parent()), zap.Int("part_index", partIdx), zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Error(err)) diff --git a/pkg/services/object/delete/delete.go b/pkg/services/object/delete/delete.go index 88454625d..8aaff670c 100644 --- a/pkg/services/object/delete/delete.go +++ b/pkg/services/object/delete/delete.go @@ -33,13 +33,13 @@ func (s *Service) Delete(ctx context.Context, prm Prm) error { } func (exec *execCtx) execute(ctx context.Context) error { - exec.log.Debug(logs.ServingRequest) + exec.log.Debug(ctx, logs.ServingRequest) if err := exec.executeLocal(ctx); err != nil { - exec.log.Debug(logs.OperationFinishedWithError, zap.String("error", err.Error())) + exec.log.Debug(ctx, logs.OperationFinishedWithError, zap.String("error", err.Error())) return err } - exec.log.Debug(logs.OperationFinishedSuccessfully) + exec.log.Debug(ctx, logs.OperationFinishedSuccessfully) return nil } diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index ec771320e..36a17bde2 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -34,13 +34,13 @@ type execCtx struct { } func (exec *execCtx) setLogger(l *logger.Logger) { - exec.log = &logger.Logger{Logger: l.With( + exec.log = l.With( zap.String("request", "DELETE"), zap.Stringer("address", exec.address()), zap.Bool("local", exec.isLocal()), zap.Bool("with session", exec.prm.common.SessionToken() != nil), zap.Bool("with bearer", exec.prm.common.BearerToken() != nil), - )} + ) } func (exec *execCtx) isLocal() bool { @@ -83,16 +83,16 @@ func (exec *execCtx) formExtendedInfo(ctx context.Context) error { exec.splitInfo = errSplitInfo.SplitInfo() exec.tombstone.SetSplitID(exec.splitInfo.SplitID()) - exec.log.Debug(logs.DeleteSplitInfoSuccessfullyFormedCollectingMembers) + exec.log.Debug(ctx, logs.DeleteSplitInfoSuccessfullyFormedCollectingMembers) if err := exec.collectMembers(ctx); err != nil { return err } - exec.log.Debug(logs.DeleteMembersSuccessfullyCollected) + exec.log.Debug(ctx, logs.DeleteMembersSuccessfullyCollected) return nil case errors.As(err, &errECInfo): - exec.log.Debug(logs.DeleteECObjectReceived) + exec.log.Debug(ctx, logs.DeleteECObjectReceived) return nil } @@ -108,7 +108,7 @@ func (exec *execCtx) formExtendedInfo(ctx context.Context) error { func (exec *execCtx) collectMembers(ctx context.Context) error { if exec.splitInfo == nil { - exec.log.Debug(logs.DeleteNoSplitInfoObjectIsPHY) + exec.log.Debug(ctx, logs.DeleteNoSplitInfoObjectIsPHY) return nil } @@ -131,7 +131,7 @@ func (exec *execCtx) collectMembers(ctx context.Context) error { func (exec *execCtx) collectChain(ctx context.Context) error { var chain []oid.ID - exec.log.Debug(logs.DeleteAssemblingChain) + exec.log.Debug(ctx, logs.DeleteAssemblingChain) for prev, withPrev := exec.splitInfo.LastPart(); withPrev; { chain = append(chain, prev) @@ -152,7 +152,7 @@ func (exec *execCtx) collectChain(ctx context.Context) error { } func (exec *execCtx) collectChildren(ctx context.Context) error { - exec.log.Debug(logs.DeleteCollectingChildren) + exec.log.Debug(ctx, logs.DeleteCollectingChildren) children, err := exec.svc.header.children(ctx, exec) if err != nil { @@ -165,7 +165,7 @@ func (exec *execCtx) collectChildren(ctx context.Context) error { } func (exec *execCtx) supplementBySplitID(ctx context.Context) error { - exec.log.Debug(logs.DeleteSupplementBySplitID) + exec.log.Debug(ctx, logs.DeleteSupplementBySplitID) chain, err := exec.svc.searcher.splitMembers(ctx, exec) if err != nil { diff --git a/pkg/services/object/delete/local.go b/pkg/services/object/delete/local.go index 2c3c47f49..01b2d9b3f 100644 --- a/pkg/services/object/delete/local.go +++ b/pkg/services/object/delete/local.go @@ -10,13 +10,13 @@ import ( ) func (exec *execCtx) executeLocal(ctx context.Context) error { - exec.log.Debug(logs.DeleteFormingTombstoneStructure) + exec.log.Debug(ctx, logs.DeleteFormingTombstoneStructure) if err := exec.formTombstone(ctx); err != nil { return err } - exec.log.Debug(logs.DeleteTombstoneStructureSuccessfullyFormedSaving) + exec.log.Debug(ctx, logs.DeleteTombstoneStructureSuccessfullyFormedSaving) return exec.saveTombstone(ctx) } @@ -33,7 +33,7 @@ func (exec *execCtx) formTombstone(ctx context.Context) error { ) exec.addMembers([]oid.ID{exec.address().Object()}) - exec.log.Debug(logs.DeleteFormingSplitInfo) + exec.log.Debug(ctx, logs.DeleteFormingSplitInfo) if err := exec.formExtendedInfo(ctx); err != nil { return fmt.Errorf("form extended info: %w", err) diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index e4f7a8c50..867d3f4ef 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -72,7 +72,7 @@ func New(gs *getsvc.Service, opts ...Option, ) *Service { c := &cfg{ - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), header: &headSvcWrapper{s: gs}, searcher: &searchSvcWrapper{s: ss}, placer: &putSvcWrapper{s: ps}, @@ -92,6 +92,6 @@ func New(gs *getsvc.Service, // WithLogger returns option to specify Delete service's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = &logger.Logger{Logger: l.With(zap.String("component", "objectSDK.Delete service"))} + c.log = l.With(zap.String("component", "objectSDK.Delete service")) } } diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index 9f17f1e4c..e164627d2 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -13,7 +13,7 @@ import ( func (r *request) assemble(ctx context.Context) { if !r.canAssembleComplexObject() { - r.log.Debug(logs.GetCanNotAssembleTheObject) + r.log.Debug(ctx, logs.GetCanNotAssembleTheObject) return } @@ -35,23 +35,23 @@ func (r *request) assemble(ctx context.Context) { // `execCtx` so it should be disabled there. r.disableForwarding() - r.log.Debug(logs.GetTryingToAssembleTheObject) + r.log.Debug(ctx, logs.GetTryingToAssembleTheObject) r.prm.common = r.prm.common.WithLocalOnly(false) assembler := newAssembler(r.address(), r.splitInfo(), r.ctxRange(), r, r.headOnly()) - r.log.Debug(logs.GetAssemblingSplittedObject, + r.log.Debug(ctx, logs.GetAssemblingSplittedObject, zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), ) - defer r.log.Debug(logs.GetAssemblingSplittedObjectCompleted, + defer r.log.Debug(ctx, logs.GetAssemblingSplittedObjectCompleted, zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), ) obj, err := assembler.Assemble(ctx, r.prm.objWriter) if err != nil { - r.log.Warn(logs.GetFailedToAssembleSplittedObject, + r.log.Warn(ctx, logs.GetFailedToAssembleSplittedObject, zap.Error(err), zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), diff --git a/pkg/services/object/get/assembleec.go b/pkg/services/object/get/assembleec.go index 03f913bbf..8ab423c87 100644 --- a/pkg/services/object/get/assembleec.go +++ b/pkg/services/object/get/assembleec.go @@ -12,7 +12,7 @@ import ( func (r *request) assembleEC(ctx context.Context) { if r.isRaw() { - r.log.Debug(logs.GetCanNotAssembleTheObject) + r.log.Debug(ctx, logs.GetCanNotAssembleTheObject) return } @@ -34,7 +34,7 @@ func (r *request) assembleEC(ctx context.Context) { // `execCtx` so it should be disabled there. r.disableForwarding() - r.log.Debug(logs.GetTryingToAssembleTheECObject) + r.log.Debug(ctx, logs.GetTryingToAssembleTheECObject) // initialize epoch number ok := r.initEpoch() @@ -45,18 +45,18 @@ func (r *request) assembleEC(ctx context.Context) { r.prm.common = r.prm.common.WithLocalOnly(false) assembler := newAssemblerEC(r.address(), r.infoEC, r.ctxRange(), r, r.localStorage, r.log, r.headOnly(), r.traverserGenerator, r.curProcEpoch) - r.log.Debug(logs.GetAssemblingECObject, + r.log.Debug(ctx, logs.GetAssemblingECObject, zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), ) - defer r.log.Debug(logs.GetAssemblingECObjectCompleted, + defer r.log.Debug(ctx, logs.GetAssemblingECObjectCompleted, zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), ) obj, err := assembler.Assemble(ctx, r.prm.objWriter) if err != nil && !errors.As(err, new(*objectSDK.ECInfoError)) { - r.log.Warn(logs.GetFailedToAssembleECObject, + r.log.Warn(ctx, logs.GetFailedToAssembleECObject, zap.Error(err), zap.Uint64("range_offset", r.ctxRange().GetOffset()), zap.Uint64("range_length", r.ctxRange().GetLength()), diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index 44d9af3a2..b0895e13e 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -155,7 +155,7 @@ func (a *assemblerec) retrieveParts(ctx context.Context, trav *placement.Travers parts, err := a.processECNodesRequests(ctx, remoteNodes, dataCount, parityCount) if err != nil { - a.log.Debug(logs.GetUnableToGetAllPartsECObject, zap.Error(err)) + a.log.Debug(ctx, logs.GetUnableToGetAllPartsECObject, zap.Error(err)) } return parts } @@ -229,7 +229,7 @@ func (a *assemblerec) tryGetChunkFromLocalStorage(ctx context.Context, ch object var objID oid.ID err := objID.ReadFromV2(ch.ID) if err != nil { - a.log.Error(logs.GetUnableToHeadPartECObject, zap.String("node", "local"), zap.Uint32("part_index", ch.Index), zap.Error(fmt.Errorf("invalid object ID: %w", err))) + a.log.Error(ctx, logs.GetUnableToHeadPartECObject, zap.String("node", "local"), zap.Uint32("part_index", ch.Index), zap.Error(fmt.Errorf("invalid object ID: %w", err))) return nil } var addr oid.Address @@ -239,13 +239,13 @@ func (a *assemblerec) tryGetChunkFromLocalStorage(ctx context.Context, ch object if a.head { object, err = a.localStorage.Head(ctx, addr, false) if err != nil { - a.log.Warn(logs.GetUnableToHeadPartECObject, zap.String("node", "local"), zap.Stringer("part_id", objID), zap.Error(err)) + a.log.Warn(ctx, logs.GetUnableToHeadPartECObject, zap.String("node", "local"), zap.Stringer("part_id", objID), zap.Error(err)) return nil } } else { object, err = a.localStorage.Get(ctx, addr) if err != nil { - a.log.Warn(logs.GetUnableToGetPartECObject, zap.String("node", "local"), zap.Stringer("part_id", objID), zap.Error(err)) + a.log.Warn(ctx, logs.GetUnableToGetPartECObject, zap.String("node", "local"), zap.Stringer("part_id", objID), zap.Error(err)) return nil } } @@ -259,11 +259,11 @@ func (a *assemblerec) tryGetChunkListFromNode(ctx context.Context, node client.N var errECInfo *objectSDK.ECInfoError _, err := a.remoteStorage.headObjectFromNode(ctx, a.addr, node, true) if err == nil { - a.log.Error(logs.GetUnexpectedECObject, zap.String("node", hex.EncodeToString(node.PublicKey()))) + a.log.Error(ctx, logs.GetUnexpectedECObject, zap.String("node", hex.EncodeToString(node.PublicKey()))) return nil } if !errors.As(err, &errECInfo) { - a.log.Warn(logs.GetUnableToHeadPartsECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Error(err)) + a.log.Warn(ctx, logs.GetUnableToHeadPartsECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Error(err)) return nil } result := make([]objectSDK.ECChunk, 0, len(errECInfo.ECInfo().Chunks)) @@ -277,7 +277,7 @@ func (a *assemblerec) tryGetChunkFromRemoteStorage(ctx context.Context, node cli var objID oid.ID err := objID.ReadFromV2(ch.ID) if err != nil { - a.log.Error(logs.GetUnableToHeadPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Uint32("part_index", ch.Index), zap.Error(fmt.Errorf("invalid object ID: %w", err))) + a.log.Error(ctx, logs.GetUnableToHeadPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Uint32("part_index", ch.Index), zap.Error(fmt.Errorf("invalid object ID: %w", err))) return nil } var addr oid.Address @@ -287,13 +287,13 @@ func (a *assemblerec) tryGetChunkFromRemoteStorage(ctx context.Context, node cli if a.head { object, err = a.remoteStorage.headObjectFromNode(ctx, addr, node, false) if err != nil { - a.log.Warn(logs.GetUnableToHeadPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Stringer("part_id", objID), zap.Error(err)) + a.log.Warn(ctx, logs.GetUnableToHeadPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Stringer("part_id", objID), zap.Error(err)) return nil } } else { object, err = a.remoteStorage.getObjectFromNode(ctx, addr, node) if err != nil { - a.log.Warn(logs.GetUnableToGetPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Stringer("part_id", objID), zap.Error(err)) + a.log.Warn(ctx, logs.GetUnableToGetPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Stringer("part_id", objID), zap.Error(err)) return nil } } diff --git a/pkg/services/object/get/container.go b/pkg/services/object/get/container.go index 034768c81..2b84c5b32 100644 --- a/pkg/services/object/get/container.go +++ b/pkg/services/object/get/container.go @@ -10,13 +10,13 @@ import ( func (r *request) executeOnContainer(ctx context.Context) { if r.isLocal() { - r.log.Debug(logs.GetReturnResultDirectly) + r.log.Debug(ctx, logs.GetReturnResultDirectly) return } lookupDepth := r.netmapLookupDepth() - r.log.Debug(logs.TryingToExecuteInContainer, + r.log.Debug(ctx, logs.TryingToExecuteInContainer, zap.Uint64("netmap lookup depth", lookupDepth), ) @@ -46,7 +46,7 @@ func (r *request) executeOnContainer(ctx context.Context) { } func (r *request) processCurrentEpoch(ctx context.Context, localStatus int) bool { - r.log.Debug(logs.ProcessEpoch, + r.log.Debug(ctx, logs.ProcessEpoch, zap.Uint64("number", r.curProcEpoch), ) @@ -67,7 +67,7 @@ func (r *request) processCurrentEpoch(ctx context.Context, localStatus int) bool for { addrs := traverser.Next() if len(addrs) == 0 { - r.log.Debug(logs.NoMoreNodesAbortPlacementIteration) + r.log.Debug(ctx, logs.NoMoreNodesAbortPlacementIteration) return false } @@ -75,7 +75,7 @@ func (r *request) processCurrentEpoch(ctx context.Context, localStatus int) bool for i := range addrs { select { case <-ctx.Done(): - r.log.Debug(logs.InterruptPlacementIterationByContext, + r.log.Debug(ctx, logs.InterruptPlacementIterationByContext, zap.Error(ctx.Err()), ) @@ -91,7 +91,7 @@ func (r *request) processCurrentEpoch(ctx context.Context, localStatus int) bool client.NodeInfoFromNetmapElement(&info, addrs[i]) if r.processNode(ctx, info) { - r.log.Debug(logs.GetCompletingTheOperation) + r.log.Debug(ctx, logs.GetCompletingTheOperation) return true } } diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 03b7f8bf2..557e9a028 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -91,7 +91,7 @@ func (s *Service) get(ctx context.Context, prm RequestParameters) error { } func (exec *request) execute(ctx context.Context) { - exec.log.Debug(logs.ServingRequest) + exec.log.Debug(ctx, logs.ServingRequest) // perform local operation exec.executeLocal(ctx) @@ -103,23 +103,23 @@ func (exec *request) analyzeStatus(ctx context.Context, execCnr bool) { // analyze local result switch exec.status { case statusOK: - exec.log.Debug(logs.OperationFinishedSuccessfully) + exec.log.Debug(ctx, logs.OperationFinishedSuccessfully) case statusINHUMED: - exec.log.Debug(logs.GetRequestedObjectWasMarkedAsRemoved) + exec.log.Debug(ctx, logs.GetRequestedObjectWasMarkedAsRemoved) case statusVIRTUAL: - exec.log.Debug(logs.GetRequestedObjectIsVirtual) + exec.log.Debug(ctx, logs.GetRequestedObjectIsVirtual) exec.assemble(ctx) case statusOutOfRange: - exec.log.Debug(logs.GetRequestedRangeIsOutOfObjectBounds) + exec.log.Debug(ctx, logs.GetRequestedRangeIsOutOfObjectBounds) case statusEC: - exec.log.Debug(logs.GetRequestedObjectIsEC) + exec.log.Debug(ctx, logs.GetRequestedObjectIsEC) if exec.isRaw() && execCnr { exec.executeOnContainer(ctx) exec.analyzeStatus(ctx, false) } exec.assembleEC(ctx) default: - exec.log.Debug(logs.OperationFinishedWithError, + exec.log.Debug(ctx, logs.OperationFinishedWithError, zap.Error(exec.err), ) var errAccessDenied *apistatus.ObjectAccessDenied diff --git a/pkg/services/object/get/local.go b/pkg/services/object/get/local.go index 1cd5e549c..cfabb082f 100644 --- a/pkg/services/object/get/local.go +++ b/pkg/services/object/get/local.go @@ -31,7 +31,7 @@ func (r *request) executeLocal(ctx context.Context) { r.status = statusUndefined r.err = err - r.log.Debug(logs.GetLocalGetFailed, zap.Error(err)) + r.log.Debug(ctx, logs.GetLocalGetFailed, zap.Error(err)) case err == nil: r.status = statusOK r.err = nil diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index f2639f8e6..b6a83fd0c 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -18,7 +18,7 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { ctx, span := tracing.StartSpanFromContext(ctx, "getService.processNode") defer span.End() - r.log.Debug(logs.ProcessingNode, zap.String("node_key", hex.EncodeToString(info.PublicKey()))) + r.log.Debug(ctx, logs.ProcessingNode, zap.String("node_key", hex.EncodeToString(info.PublicKey()))) rs, ok := r.getRemoteStorage(info) if !ok { @@ -35,7 +35,7 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { switch { default: - r.log.Debug(logs.GetRemoteCallFailed, zap.Error(err)) + r.log.Debug(ctx, logs.GetRemoteCallFailed, zap.Error(err)) if r.status != statusEC { // for raw requests, continue to collect other parts r.status = statusUndefined diff --git a/pkg/services/object/get/request.go b/pkg/services/object/get/request.go index 1a7a43a35..bba767d2d 100644 --- a/pkg/services/object/get/request.go +++ b/pkg/services/object/get/request.go @@ -47,14 +47,14 @@ func (r *request) setLogger(l *logger.Logger) { req = "GET_RANGE" } - r.log = &logger.Logger{Logger: l.With( + r.log = l.With( zap.String("request", req), zap.Stringer("address", r.address()), zap.Bool("raw", r.isRaw()), zap.Bool("local", r.isLocal()), zap.Bool("with session", r.prm.common.SessionToken() != nil), zap.Bool("with bearer", r.prm.common.BearerToken() != nil), - )} + ) } func (r *request) isLocal() bool { @@ -129,7 +129,7 @@ func (r *request) initEpoch() bool { r.status = statusUndefined r.err = err - r.log.Debug(logs.CouldNotGetCurrentEpochNumber, zap.Error(err)) + r.log.Debug(context.Background(), logs.CouldNotGetCurrentEpochNumber, zap.Error(err)) return false case err == nil: @@ -148,7 +148,7 @@ func (r *request) generateTraverser(addr oid.Address) (*placement.Traverser, boo r.status = statusUndefined r.err = err - r.log.Debug(logs.GetCouldNotGenerateContainerTraverser, zap.Error(err)) + r.log.Debug(context.Background(), logs.GetCouldNotGenerateContainerTraverser, zap.Error(err)) return nil, false case err == nil: @@ -162,7 +162,7 @@ func (r *request) getRemoteStorage(info clientcore.NodeInfo) (remoteStorage, boo r.status = statusUndefined r.err = err - r.log.Debug(logs.GetCouldNotConstructRemoteNodeClient) + r.log.Debug(context.Background(), logs.GetCouldNotConstructRemoteNodeClient) return nil, false } @@ -185,7 +185,7 @@ func (r *request) writeCollectedHeader(ctx context.Context) bool { r.status = statusUndefined r.err = err - r.log.Debug(logs.GetCouldNotWriteHeader, zap.Error(err)) + r.log.Debug(ctx, logs.GetCouldNotWriteHeader, zap.Error(err)) case err == nil: r.status = statusOK r.err = nil @@ -206,7 +206,7 @@ func (r *request) writeObjectPayload(ctx context.Context, obj *objectSDK.Object) r.status = statusUndefined r.err = err - r.log.Debug(logs.GetCouldNotWritePayloadChunk, zap.Error(err)) + r.log.Debug(ctx, logs.GetCouldNotWritePayloadChunk, zap.Error(err)) case err == nil: r.status = statusOK r.err = nil diff --git a/pkg/services/object/get/service.go b/pkg/services/object/get/service.go index 3413abeb7..9ec10b5f2 100644 --- a/pkg/services/object/get/service.go +++ b/pkg/services/object/get/service.go @@ -34,7 +34,7 @@ func New( result := &Service{ keyStore: ks, epochSource: es, - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), localStorage: &engineLocalStorage{ engine: e, }, @@ -53,6 +53,6 @@ func New( // WithLogger returns option to specify Get service's logger. func WithLogger(l *logger.Logger) Option { return func(s *Service) { - s.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Get service"))} + s.log = l.With(zap.String("component", "Object.Get service")) } } diff --git a/pkg/services/object/get/v2/get_range_hash.go b/pkg/services/object/get/v2/get_range_hash.go index e8e82ddd9..7d26a38c3 100644 --- a/pkg/services/object/get/v2/get_range_hash.go +++ b/pkg/services/object/get/v2/get_range_hash.go @@ -125,14 +125,14 @@ func (s *Service) forwardGetRangeHashRequest(ctx context.Context, req *objectV2. var addrGr network.AddressGroup if err := addrGr.FromIterator(network.NodeEndpointsIterator(node)); err != nil { - s.log.Warn(logs.GetSvcV2FailedToParseNodeEndpoints, zap.String("node_public_key", hex.EncodeToString(node.PublicKey()))) + s.log.Warn(ctx, logs.GetSvcV2FailedToParseNodeEndpoints, zap.String("node_public_key", hex.EncodeToString(node.PublicKey()))) continue } var extAddr network.AddressGroup if len(node.ExternalAddresses()) > 0 { if err := extAddr.FromStringSlice(node.ExternalAddresses()); err != nil { - s.log.Warn(logs.GetSvcV2FailedToParseNodeExternalAddresses, zap.String("node_public_key", hex.EncodeToString(node.PublicKey()))) + s.log.Warn(ctx, logs.GetSvcV2FailedToParseNodeExternalAddresses, zap.String("node_public_key", hex.EncodeToString(node.PublicKey()))) continue } } @@ -150,12 +150,12 @@ func (s *Service) forwardGetRangeHashRequest(ctx context.Context, req *objectV2. if firstErr == nil { firstErr = err } - s.log.Debug(logs.GetSvcV2FailedToGetRangeHashFromNode, + s.log.Debug(ctx, logs.GetSvcV2FailedToGetRangeHashFromNode, zap.String("node_public_key", hex.EncodeToString(node.PublicKey())), zap.Stringer("address", params.address), zap.Error(err)) } - s.log.Debug(logs.GetSvcV2FailedToGetRangeHashFromAllOfContainerNodes, zap.Stringer("address", params.address), zap.Error(firstErr)) + s.log.Debug(ctx, logs.GetSvcV2FailedToGetRangeHashFromAllOfContainerNodes, zap.Stringer("address", params.address), zap.Error(firstErr)) if firstErr != nil { return nil, firstErr } diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index 24b2f0099..fc483b74b 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -60,7 +60,7 @@ func NewService(svc *getsvc.Service, netmapSource: netmapSource, announcedKeys: announcedKeys, contSource: contSource, - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), } for i := range opts { @@ -145,6 +145,6 @@ func (s *Service) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Get V2 service"))} + c.log = l.With(zap.String("component", "Object.Get V2 service")) } } diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 8cf4f0d62..5cc0a5722 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -29,7 +29,7 @@ func NewService(ks *objutil.KeyStorage, c := &objectwriter.Config{ RemotePool: util.NewPseudoWorkerPool(), LocalPool: util.NewPseudoWorkerPool(), - Logger: &logger.Logger{Logger: zap.L()}, + Logger: logger.NewLoggerWrapper(zap.L()), KeyStorage: ks, ClientConstructor: cc, MaxSizeSrc: ms, diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 3a0b3901f..36b0bd54c 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -317,7 +317,7 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, if err != nil { objID, _ := obj.ID() cnrID, _ := obj.ContainerID() - s.Config.Logger.Warn(logs.PutSingleRedirectFailure, + s.Config.Logger.Warn(ctx, logs.PutSingleRedirectFailure, zap.Error(err), zap.Stringer("address", addr), zap.Stringer("object_id", objID), diff --git a/pkg/services/object/search/container.go b/pkg/services/object/search/container.go index 39259b0ca..999a3cc9e 100644 --- a/pkg/services/object/search/container.go +++ b/pkg/services/object/search/container.go @@ -15,7 +15,7 @@ import ( func (exec *execCtx) executeOnContainer(ctx context.Context) error { lookupDepth := exec.netmapLookupDepth() - exec.log.Debug(logs.TryingToExecuteInContainer, + exec.log.Debug(ctx, logs.TryingToExecuteInContainer, zap.Uint64("netmap lookup depth", lookupDepth), ) @@ -44,7 +44,7 @@ func (exec *execCtx) executeOnContainer(ctx context.Context) error { } func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { - exec.log.Debug(logs.ProcessEpoch, + exec.log.Debug(ctx, logs.ProcessEpoch, zap.Uint64("number", exec.curProcEpoch), ) @@ -59,7 +59,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { for { addrs := traverser.Next() if len(addrs) == 0 { - exec.log.Debug(logs.NoMoreNodesAbortPlacementIteration) + exec.log.Debug(ctx, logs.NoMoreNodesAbortPlacementIteration) break } @@ -72,7 +72,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { defer wg.Done() select { case <-ctx.Done(): - exec.log.Debug(logs.InterruptPlacementIterationByContext, + exec.log.Debug(ctx, logs.InterruptPlacementIterationByContext, zap.String("error", ctx.Err().Error())) return default: @@ -82,17 +82,17 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { client.NodeInfoFromNetmapElement(&info, addrs[i]) - exec.log.Debug(logs.ProcessingNode, zap.String("key", hex.EncodeToString(addrs[i].PublicKey()))) + exec.log.Debug(ctx, logs.ProcessingNode, zap.String("key", hex.EncodeToString(addrs[i].PublicKey()))) c, err := exec.svc.clientConstructor.get(info) if err != nil { - exec.log.Debug(logs.SearchCouldNotConstructRemoteNodeClient, zap.String("error", err.Error())) + exec.log.Debug(ctx, logs.SearchCouldNotConstructRemoteNodeClient, zap.String("error", err.Error())) return } ids, err := c.searchObjects(ctx, exec, info) if err != nil { - exec.log.Debug(logs.SearchRemoteOperationFailed, + exec.log.Debug(ctx, logs.SearchRemoteOperationFailed, zap.String("error", err.Error())) return @@ -102,7 +102,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { err = exec.writeIDList(ids) mtx.Unlock() if err != nil { - exec.log.Debug(logs.SearchCouldNotWriteObjectIdentifiers, zap.String("error", err.Error())) + exec.log.Debug(ctx, logs.SearchCouldNotWriteObjectIdentifiers, zap.String("error", err.Error())) return } }(i) diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index 4a2c04ecd..eb9635f14 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -19,13 +19,13 @@ type execCtx struct { } func (exec *execCtx) setLogger(l *logger.Logger) { - exec.log = &logger.Logger{Logger: l.With( + exec.log = l.With( zap.String("request", "SEARCH"), zap.Stringer("container", exec.containerID()), zap.Bool("local", exec.isLocal()), zap.Bool("with session", exec.prm.common.SessionToken() != nil), zap.Bool("with bearer", exec.prm.common.BearerToken() != nil), - )} + ) } func (exec *execCtx) isLocal() bool { diff --git a/pkg/services/object/search/local.go b/pkg/services/object/search/local.go index cfaed13b8..bc59d0394 100644 --- a/pkg/services/object/search/local.go +++ b/pkg/services/object/search/local.go @@ -11,7 +11,7 @@ import ( func (exec *execCtx) executeLocal(ctx context.Context) error { ids, err := exec.svc.localStorage.search(ctx, exec) if err != nil { - exec.log.Debug(logs.SearchLocalOperationFailed, zap.String("error", err.Error())) + exec.log.Debug(ctx, logs.SearchLocalOperationFailed, zap.String("error", err.Error())) return err } diff --git a/pkg/services/object/search/search.go b/pkg/services/object/search/search.go index 4a5c414d5..bb5c720ff 100644 --- a/pkg/services/object/search/search.go +++ b/pkg/services/object/search/search.go @@ -20,13 +20,13 @@ func (s *Service) Search(ctx context.Context, prm Prm) error { } func (exec *execCtx) execute(ctx context.Context) error { - exec.log.Debug(logs.ServingRequest) + exec.log.Debug(ctx, logs.ServingRequest) err := exec.executeLocal(ctx) exec.logResult(err) if exec.isLocal() { - exec.log.Debug(logs.SearchReturnResultDirectly) + exec.log.Debug(ctx, logs.SearchReturnResultDirectly) return err } @@ -38,8 +38,8 @@ func (exec *execCtx) execute(ctx context.Context) error { func (exec *execCtx) logResult(err error) { switch { default: - exec.log.Debug(logs.OperationFinishedWithError, zap.String("error", err.Error())) + exec.log.Debug(context.Background(), logs.OperationFinishedWithError, zap.String("error", err.Error())) case err == nil: - exec.log.Debug(logs.OperationFinishedSuccessfully) + exec.log.Debug(context.Background(), logs.OperationFinishedSuccessfully) } } diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index 7700f78d8..77d25357a 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -69,7 +69,7 @@ func New(e *engine.StorageEngine, opts ...Option, ) *Service { c := &cfg{ - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), clientConstructor: &clientConstructorWrapper{ constructor: cc, }, @@ -94,6 +94,6 @@ func New(e *engine.StorageEngine, // WithLogger returns option to specify Get service's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = &logger.Logger{Logger: l.With(zap.String("component", "Object.Search service"))} + c.log = l.With(zap.String("component", "Object.Search service")) } } diff --git a/pkg/services/object/util/log.go b/pkg/services/object/util/log.go index 92beedaa7..5075344a4 100644 --- a/pkg/services/object/util/log.go +++ b/pkg/services/object/util/log.go @@ -1,6 +1,8 @@ package util import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -9,7 +11,7 @@ import ( // LogServiceError writes error message of object service to provided logger. func LogServiceError(l *logger.Logger, req string, node network.AddressGroup, err error) { - l.Error(logs.UtilObjectServiceError, + l.Error(context.Background(), logs.UtilObjectServiceError, zap.String("node", network.StringifyGroup(node)), zap.String("request", req), zap.String("error", err.Error()), @@ -18,7 +20,7 @@ func LogServiceError(l *logger.Logger, req string, node network.AddressGroup, er // LogWorkerPoolError writes debug error message of object worker pool to provided logger. func LogWorkerPoolError(l *logger.Logger, req string, err error) { - l.Error(logs.UtilCouldNotPushTaskToWorkerPool, + l.Error(context.Background(), logs.UtilCouldNotPushTaskToWorkerPool, zap.String("request", req), zap.String("error", err.Error()), ) diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go index 7476dbd48..6a9706b9e 100644 --- a/pkg/services/object_manager/tombstone/checker.go +++ b/pkg/services/object_manager/tombstone/checker.go @@ -57,7 +57,7 @@ func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Addr ts, err := g.tsSource.Tombstone(ctx, a, epoch) if err != nil { - log.Warn( + log.Warn(ctx, logs.TombstoneCouldNotGetTheTombstoneTheSource, zap.Error(err), ) @@ -77,7 +77,7 @@ func (g *ExpirationChecker) handleTS(addr string, ts *objectSDK.Object, reqEpoch if atr.Key() == objectV2.SysAttributeExpEpoch { epoch, err := strconv.ParseUint(atr.Value(), 10, 64) if err != nil { - g.log.Warn( + g.log.Warn(context.Background(), logs.TombstoneExpirationParseFailure, zap.Error(err), ) diff --git a/pkg/services/object_manager/tombstone/constructor.go b/pkg/services/object_manager/tombstone/constructor.go index 9d33e8179..67ddf316f 100644 --- a/pkg/services/object_manager/tombstone/constructor.go +++ b/pkg/services/object_manager/tombstone/constructor.go @@ -23,7 +23,7 @@ type Option func(*cfg) func defaultCfg() *cfg { return &cfg{ - log: &logger.Logger{Logger: zap.NewNop()}, + log: logger.NewLoggerWrapper(zap.NewNop()), cacheSize: defaultLRUCacheSize, } } diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index dbc9ea53c..c82680a1e 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -86,7 +86,7 @@ func (p *Policer) processRepContainerObject(ctx context.Context, objInfo objectc } if !c.needLocalCopy && c.removeLocalCopy { - p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, + p.log.Info(ctx, logs.PolicerRedundantLocalObjectCopyDetected, zap.Stringer("object", objInfo.Address), ) @@ -151,7 +151,7 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe } else if client.IsErrNodeUnderMaintenance(err) { shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) } else { - p.log.Error(logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, + p.log.Error(ctx, logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, zap.Stringer("object", addr), zap.String("error", err.Error()), ) @@ -178,7 +178,7 @@ func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes nodeCache shortage-- uncheckedCopies++ - p.log.Debug(logs.PolicerConsiderNodeUnderMaintenanceAsOK, + p.log.Debug(context.Background(), logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(node)), ) return shortage, uncheckedCopies @@ -189,7 +189,7 @@ func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address ) { switch { case shortage > 0: - p.log.Debug(logs.PolicerShortageOfObjectCopiesDetected, + p.log.Debug(ctx, logs.PolicerShortageOfObjectCopiesDetected, zap.Stringer("object", addr), zap.Uint32("shortage", shortage), ) @@ -205,7 +205,7 @@ func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address case uncheckedCopies > 0: // If we have more copies than needed, but some of them are from the maintenance nodes, // save the local copy. - p.log.Debug(logs.PolicerSomeOfTheCopiesAreStoredOnNodesUnderMaintenance, + p.log.Debug(ctx, logs.PolicerSomeOfTheCopiesAreStoredOnNodesUnderMaintenance, zap.Int("count", uncheckedCopies)) case uncheckedCopies == 0: diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index 6d2c153c9..cb583f1d3 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -59,7 +59,7 @@ func (p *Policer) processECContainerRepObject(ctx context.Context, objInfo objec p.processRepNodes(ctx, c, objInfo, nn[0], uint32(len(nn[0])), checkedNodes) if !c.needLocalCopy && c.removeLocalCopy { - p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, + p.log.Info(ctx, logs.PolicerRedundantLocalObjectCopyDetected, zap.Stringer("object", objInfo.Address), ) @@ -91,7 +91,7 @@ func (p *Policer) processECContainerECObject(ctx context.Context, objInfo object p.adjustECPlacement(ctx, objInfo, nn[0], cnr) if res.removeLocal { - p.log.Info(logs.PolicerRedundantLocalObjectCopyDetected, zap.Stringer("object", objInfo.Address)) + p.log.Info(ctx, logs.PolicerRedundantLocalObjectCopyDetected, zap.Stringer("object", objInfo.Address)) p.cbRedundantCopy(ctx, objInfo.Address) } return nil @@ -109,7 +109,7 @@ func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, n } if requiredNode.Status().IsMaintenance() { // consider maintenance mode has object, but do not drop local copy - p.log.Debug(logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(requiredNode))) + p.log.Debug(ctx, logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(requiredNode))) return ecChunkProcessResult{} } @@ -120,7 +120,7 @@ func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, n if err == nil { removeLocalChunk = true } else if client.IsErrObjectNotFound(err) { - p.log.Debug(logs.PolicerShortageOfObjectCopiesDetected, zap.Stringer("object", objInfo.Address), zap.Uint32("shortage", 1)) + p.log.Debug(ctx, logs.PolicerShortageOfObjectCopiesDetected, zap.Stringer("object", objInfo.Address), zap.Uint32("shortage", 1)) task := replicator.Task{ NumCopies: 1, Addr: objInfo.Address, @@ -129,9 +129,9 @@ func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, n p.replicator.HandleReplicationTask(ctx, task, newNodeCache()) } else if client.IsErrNodeUnderMaintenance(err) { // consider maintenance mode has object, but do not drop local copy - p.log.Debug(logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(requiredNode))) + p.log.Debug(ctx, logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(requiredNode))) } else { - p.log.Error(logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, zap.Stringer("object", objInfo.Address), zap.String("error", err.Error())) + p.log.Error(ctx, logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, zap.Stringer("object", objInfo.Address), zap.String("error", err.Error())) } return ecChunkProcessResult{ @@ -146,13 +146,13 @@ func (p *Policer) pullRequiredECChunks(ctx context.Context, objInfo objectcore.I requiredChunkIndexes := p.collectRequiredECChunks(nodes, objInfo) if len(requiredChunkIndexes) == 0 { - p.log.Info(logs.PolicerNodeIsNotECObjectNode, zap.Stringer("object", objInfo.ECInfo.ParentID)) + p.log.Info(ctx, logs.PolicerNodeIsNotECObjectNode, zap.Stringer("object", objInfo.ECInfo.ParentID)) return true } err := p.resolveLocalECChunks(ctx, parentAddress, requiredChunkIndexes) if err != nil { - p.log.Error(logs.PolicerFailedToGetLocalECChunks, zap.Error(err), zap.Stringer("object", parentAddress)) + p.log.Error(ctx, logs.PolicerFailedToGetLocalECChunks, zap.Error(err), zap.Stringer("object", parentAddress)) return false } if len(requiredChunkIndexes) == 0 { @@ -224,11 +224,11 @@ func (p *Policer) resolveRemoteECChunks(ctx context.Context, parentAddress oid.A var chunkID oid.ID if err := chunkID.ReadFromV2(ch.ID); err != nil { - p.log.Error(logs.PolicerFailedToDecodeECChunkID, zap.Error(err), zap.Stringer("object", parentAddress)) + p.log.Error(ctx, logs.PolicerFailedToDecodeECChunkID, zap.Error(err), zap.Stringer("object", parentAddress)) return false } if existed, ok := indexToObjectID[ch.Index]; ok && existed != chunkID { - p.log.Error(logs.PolicerDifferentObjectIDForTheSameECChunk, zap.Stringer("first", existed), + p.log.Error(ctx, logs.PolicerDifferentObjectIDForTheSameECChunk, zap.Stringer("first", existed), zap.Stringer("second", chunkID), zap.Stringer("object", parentAddress), zap.Uint32("index", ch.Index)) return false } @@ -239,7 +239,7 @@ func (p *Policer) resolveRemoteECChunks(ctx context.Context, parentAddress oid.A for index, candidates := range required { if len(candidates) == 0 { - p.log.Error(logs.PolicerMissingECChunk, zap.Stringer("object", parentAddress), zap.Uint32("index", index)) + p.log.Error(ctx, logs.PolicerMissingECChunk, zap.Stringer("object", parentAddress), zap.Uint32("index", index)) return false } } @@ -271,18 +271,18 @@ func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info resolved[ch.Index] = append(resolved[ch.Index], n) var ecInfoChunkID oid.ID if err := ecInfoChunkID.ReadFromV2(ch.ID); err != nil { - p.log.Error(logs.PolicerFailedToDecodeECChunkID, zap.Error(err), zap.Stringer("object", parentAddress)) + p.log.Error(ctx, logs.PolicerFailedToDecodeECChunkID, zap.Error(err), zap.Stringer("object", parentAddress)) return } if chunkID, exist := chunkIDs[ch.Index]; exist && chunkID != ecInfoChunkID { - p.log.Error(logs.PolicerDifferentObjectIDForTheSameECChunk, zap.Stringer("first", chunkID), + p.log.Error(ctx, logs.PolicerDifferentObjectIDForTheSameECChunk, zap.Stringer("first", chunkID), zap.Stringer("second", ecInfoChunkID), zap.Stringer("object", parentAddress), zap.Uint32("index", ch.Index)) return } chunkIDs[ch.Index] = ecInfoChunkID } } else if !p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) && uint32(idx) < objInfo.ECInfo.Total { - p.log.Warn(logs.PolicerCouldNotGetObjectFromNodeMoving, zap.String("node", hex.EncodeToString(n.PublicKey())), zap.Stringer("object", parentAddress), zap.Error(err)) + p.log.Warn(ctx, logs.PolicerCouldNotGetObjectFromNodeMoving, zap.String("node", hex.EncodeToString(n.PublicKey())), zap.Stringer("object", parentAddress), zap.Error(err)) p.replicator.HandleReplicationTask(ctx, replicator.Task{ NumCopies: 1, Addr: objInfo.Address, @@ -299,7 +299,7 @@ func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info for i := range resolved { found = append(found, i) } - p.log.Error(logs.PolicerCouldNotRestoreObjectNotEnoughChunks, zap.Stringer("object", parentAddress), zap.Uint32s("found_chunks", found)) + p.log.Error(ctx, logs.PolicerCouldNotRestoreObjectNotEnoughChunks, zap.Stringer("object", parentAddress), zap.Uint32s("found_chunks", found)) return } p.restoreECObject(ctx, objInfo, parentAddress, nodes, resolved, chunkIDs, cnr) @@ -310,7 +310,7 @@ func (p *Policer) restoreECObject(ctx context.Context, objInfo objectcore.Info, ) { c, err := erasurecode.NewConstructor(int(cnr.PlacementPolicy().ReplicaDescriptor(0).GetECDataCount()), int(cnr.PlacementPolicy().ReplicaDescriptor(0).GetECParityCount())) if err != nil { - p.log.Error(logs.PolicerFailedToRestoreObject, zap.Stringer("object", parentAddress), zap.Error(err)) + p.log.Error(ctx, logs.PolicerFailedToRestoreObject, zap.Stringer("object", parentAddress), zap.Error(err)) return } parts := p.collectExistedChunks(ctx, objInfo, existedChunks, parentAddress, chunkIDs) @@ -319,7 +319,7 @@ func (p *Policer) restoreECObject(ctx context.Context, objInfo objectcore.Info, } key, err := p.keyStorage.GetKey(nil) if err != nil { - p.log.Error(logs.PolicerFailedToRestoreObject, zap.Stringer("object", parentAddress), zap.Error(err)) + p.log.Error(ctx, logs.PolicerFailedToRestoreObject, zap.Stringer("object", parentAddress), zap.Error(err)) return } required := make([]bool, len(parts)) @@ -329,7 +329,7 @@ func (p *Policer) restoreECObject(ctx context.Context, objInfo objectcore.Info, } } if err := c.ReconstructParts(parts, required, key); err != nil { - p.log.Error(logs.PolicerFailedToRestoreObject, zap.Stringer("object", parentAddress), zap.Error(err)) + p.log.Error(ctx, logs.PolicerFailedToRestoreObject, zap.Stringer("object", parentAddress), zap.Error(err)) return } for idx, part := range parts { @@ -377,7 +377,7 @@ func (p *Policer) collectExistedChunks(ctx context.Context, objInfo objectcore.I if err == nil { break } - p.log.Warn(logs.PolicerCouldNotGetChunk, zap.Stringer("object", parentAddress), zap.Stringer("chunkID", objID), zap.Error(err), zap.String("node", hex.EncodeToString(node.PublicKey()))) + p.log.Warn(ctx, logs.PolicerCouldNotGetChunk, zap.Stringer("object", parentAddress), zap.Stringer("chunkID", objID), zap.Error(err), zap.String("node", hex.EncodeToString(node.PublicKey()))) } if obj != nil { parts[idx] = obj @@ -386,7 +386,7 @@ func (p *Policer) collectExistedChunks(ctx context.Context, objInfo objectcore.I }) } if err := errGroup.Wait(); err != nil { - p.log.Error(logs.PolicerCouldNotGetChunks, zap.Stringer("object", parentAddress), zap.Error(err)) + p.log.Error(ctx, logs.PolicerCouldNotGetChunks, zap.Stringer("object", parentAddress), zap.Error(err)) return nil } return parts diff --git a/pkg/services/policer/option.go b/pkg/services/policer/option.go index 336f7a0ab..5d59604c2 100644 --- a/pkg/services/policer/option.go +++ b/pkg/services/policer/option.go @@ -91,7 +91,7 @@ type cfg struct { func defaultCfg() *cfg { return &cfg{ - log: &logger.Logger{Logger: zap.L()}, + log: logger.NewLoggerWrapper(zap.L()), batchSize: 10, cacheSize: 1024, // 1024 * address size = 1024 * 64 = 64 MiB sleepDuration: 1 * time.Second, diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index 363c0b922..4e8bacfec 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -4,7 +4,6 @@ import ( "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" "go.uber.org/zap" @@ -55,7 +54,7 @@ func New(opts ...Option) *Policer { opts[i](c) } - c.log = &logger.Logger{Logger: c.log.With(zap.String("component", "Object Policer"))} + c.log = c.log.With(zap.String("component", "Object Policer")) cache, err := lru.New[oid.Address, time.Time](int(c.cacheSize)) if err != nil { diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index a5ebb0010..80a87ade9 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -14,7 +14,7 @@ import ( func (p *Policer) Run(ctx context.Context) { p.shardPolicyWorker(ctx) - p.log.Info(logs.PolicerRoutineStopped) + p.log.Info(ctx, logs.PolicerRoutineStopped) } func (p *Policer) shardPolicyWorker(ctx context.Context) { @@ -33,7 +33,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { time.Sleep(p.sleepDuration) // finished whole cycle, sleep a bit continue } - p.log.Warn(logs.PolicerFailureAtObjectSelectForReplication, zap.Error(err)) + p.log.Warn(ctx, logs.PolicerFailureAtObjectSelectForReplication, zap.Error(err)) } skipMap := newSkipMap() @@ -59,7 +59,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { if p.objsInWork.add(addr.Address) { err := p.processObject(ctx, addr) if err != nil && !skipMap.addSeenError(addr.Address.Container(), err) { - p.log.Error(logs.PolicerUnableToProcessObj, + p.log.Error(ctx, logs.PolicerUnableToProcessObj, zap.Stringer("object", addr.Address), zap.String("error", err.Error())) } @@ -69,7 +69,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { } }) if err != nil { - p.log.Warn(logs.PolicerPoolSubmission, zap.Error(err)) + p.log.Warn(ctx, logs.PolicerPoolSubmission, zap.Error(err)) } } } diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 7e5c6e093..2120312f6 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -27,7 +27,7 @@ func (p *Replicator) HandleReplicationTask(ctx context.Context, task Task, res T p.metrics.IncInFlightRequest() defer p.metrics.DecInFlightRequest() defer func() { - p.log.Debug(logs.ReplicatorFinishWork, + p.log.Debug(ctx, logs.ReplicatorFinishWork, zap.Uint32("amount of unfinished replicas", task.NumCopies), ) }() @@ -43,7 +43,7 @@ func (p *Replicator) HandleReplicationTask(ctx context.Context, task Task, res T var err error task.Obj, err = engine.Get(ctx, p.localStorage, task.Addr) if err != nil { - p.log.Error(logs.ReplicatorCouldNotGetObjectFromLocalStorage, + p.log.Error(ctx, logs.ReplicatorCouldNotGetObjectFromLocalStorage, zap.Stringer("object", task.Addr), zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -75,11 +75,11 @@ func (p *Replicator) HandleReplicationTask(ctx context.Context, task Task, res T cancel() if err != nil { - log.Error(logs.ReplicatorCouldNotReplicateObject, + log.Error(ctx, logs.ReplicatorCouldNotReplicateObject, zap.String("error", err.Error()), ) } else { - log.Debug(logs.ReplicatorObjectSuccessfullyReplicated) + log.Debug(ctx, logs.ReplicatorObjectSuccessfullyReplicated) task.NumCopies-- diff --git a/pkg/services/replicator/pull.go b/pkg/services/replicator/pull.go index 7e7090237..5ce929342 100644 --- a/pkg/services/replicator/pull.go +++ b/pkg/services/replicator/pull.go @@ -22,7 +22,7 @@ func (p *Replicator) HandlePullTask(ctx context.Context, task Task) { p.metrics.IncInFlightRequest() defer p.metrics.DecInFlightRequest() defer func() { - p.log.Debug(logs.ReplicatorFinishWork, zap.String("type", "pull")) + p.log.Debug(ctx, logs.ReplicatorFinishWork, zap.String("type", "pull")) }() ctx, span := tracing.StartSpanFromContext(ctx, "Replicator.HandlePullTask", @@ -48,7 +48,7 @@ func (p *Replicator) HandlePullTask(ctx context.Context, task Task) { endpoints = append(endpoints, s) return false }) - p.log.Error(logs.ReplicatorCouldNotGetObjectFromRemoteStorage, + p.log.Error(ctx, logs.ReplicatorCouldNotGetObjectFromRemoteStorage, zap.Stringer("object", task.Addr), zap.Error(err), zap.Strings("endpoints", endpoints), @@ -56,7 +56,7 @@ func (p *Replicator) HandlePullTask(ctx context.Context, task Task) { } if obj == nil { - p.log.Error(logs.ReplicatorCouldNotGetObjectFromRemoteStorage, + p.log.Error(ctx, logs.ReplicatorCouldNotGetObjectFromRemoteStorage, zap.Stringer("object", task.Addr), zap.Error(errFailedToGetObjectFromAnyNode), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -65,7 +65,7 @@ func (p *Replicator) HandlePullTask(ctx context.Context, task Task) { err := engine.Put(ctx, p.localStorage, obj, containerCore.IsIndexedContainer(task.Container)) if err != nil { - p.log.Error(logs.ReplicatorCouldNotPutObjectToLocalStorage, + p.log.Error(ctx, logs.ReplicatorCouldNotPutObjectToLocalStorage, zap.Stringer("object", task.Addr), zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/services/replicator/put.go b/pkg/services/replicator/put.go index 537833516..489f66ae5 100644 --- a/pkg/services/replicator/put.go +++ b/pkg/services/replicator/put.go @@ -20,7 +20,7 @@ func (p *Replicator) HandleLocalPutTask(ctx context.Context, task Task) { p.metrics.IncInFlightRequest() defer p.metrics.DecInFlightRequest() defer func() { - p.log.Debug(logs.ReplicatorFinishWork, zap.String("type", "pull")) + p.log.Debug(ctx, logs.ReplicatorFinishWork, zap.String("type", "pull")) }() ctx, span := tracing.StartSpanFromContext(ctx, "Replicator.HandleLocalPutTask", @@ -31,7 +31,7 @@ func (p *Replicator) HandleLocalPutTask(ctx context.Context, task Task) { defer span.End() if task.Obj == nil { - p.log.Error(logs.ReplicatorCouldNotPutObjectToLocalStorage, + p.log.Error(ctx, logs.ReplicatorCouldNotPutObjectToLocalStorage, zap.Stringer("object", task.Addr), zap.Error(errObjectNotDefined), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -40,7 +40,7 @@ func (p *Replicator) HandleLocalPutTask(ctx context.Context, task Task) { err := engine.Put(ctx, p.localStorage, task.Obj, containerCore.IsIndexedContainer(task.Container)) if err != nil { - p.log.Error(logs.ReplicatorCouldNotPutObjectToLocalStorage, + p.log.Error(ctx, logs.ReplicatorCouldNotPutObjectToLocalStorage, zap.Stringer("object", task.Addr), zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/services/replicator/replicator.go b/pkg/services/replicator/replicator.go index f2f86daf0..6910fa5af 100644 --- a/pkg/services/replicator/replicator.go +++ b/pkg/services/replicator/replicator.go @@ -45,7 +45,7 @@ func New(opts ...Option) *Replicator { opts[i](c) } - c.log = &logger.Logger{Logger: c.log.With(zap.String("component", "Object Replicator"))} + c.log = c.log.With(zap.String("component", "Object Replicator")) return &Replicator{ cfg: c, diff --git a/pkg/services/session/executor.go b/pkg/services/session/executor.go index e914119b4..12b221613 100644 --- a/pkg/services/session/executor.go +++ b/pkg/services/session/executor.go @@ -33,7 +33,7 @@ func NewExecutionService(exec ServiceExecutor, respSvc *response.Service, l *log } func (s *executorSvc) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - s.log.Debug(logs.ServingRequest, + s.log.Debug(ctx, logs.ServingRequest, zap.String("component", "SessionService"), zap.String("request", "Create"), ) diff --git a/pkg/services/session/storage/persistent/options.go b/pkg/services/session/storage/persistent/options.go index 411734ea1..60db97f90 100644 --- a/pkg/services/session/storage/persistent/options.go +++ b/pkg/services/session/storage/persistent/options.go @@ -19,7 +19,7 @@ type Option func(*cfg) func defaultCfg() *cfg { return &cfg{ - l: &logger.Logger{Logger: zap.L()}, + l: logger.NewLoggerWrapper(zap.L()), timeout: 100 * time.Millisecond, } } diff --git a/pkg/services/session/storage/persistent/storage.go b/pkg/services/session/storage/persistent/storage.go index 71711e371..d312ea0ea 100644 --- a/pkg/services/session/storage/persistent/storage.go +++ b/pkg/services/session/storage/persistent/storage.go @@ -1,6 +1,7 @@ package persistent import ( + "context" "crypto/aes" "crypto/cipher" "encoding/hex" @@ -105,7 +106,7 @@ func (s *TokenStore) Get(ownerID user.ID, tokenID []byte) (t *storage.PrivateTok return err }) if err != nil { - s.l.Error(logs.PersistentCouldNotGetSessionFromPersistentStorage, + s.l.Error(context.Background(), logs.PersistentCouldNotGetSessionFromPersistentStorage, zap.Error(err), zap.Stringer("ownerID", ownerID), zap.String("tokenID", hex.EncodeToString(tokenID)), @@ -130,7 +131,7 @@ func (s *TokenStore) RemoveOld(epoch uint64) { if epochFromToken(v) <= epoch { err = c.Delete() if err != nil { - s.l.Error(logs.PersistentCouldNotDeleteSToken, + s.l.Error(context.Background(), logs.PersistentCouldNotDeleteSToken, zap.String("token_id", hex.EncodeToString(k)), ) } @@ -141,7 +142,7 @@ func (s *TokenStore) RemoveOld(epoch uint64) { }) }) if err != nil { - s.l.Error(logs.PersistentCouldNotCleanUpExpiredTokens, + s.l.Error(context.Background(), logs.PersistentCouldNotCleanUpExpiredTokens, zap.Uint64("epoch", epoch), ) } diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index 5bde3ae38..416a0fafe 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -54,7 +54,7 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo return false } - s.log.Debug(logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint), + s.log.Debug(ctx, logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) called = true diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 84e376cf7..0c5bde078 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -58,7 +58,7 @@ func (s *Service) localReplicationWorker(ctx context.Context) { err := s.forest.TreeApply(ctx, op.cid, op.treeID, &op.Move, false) if err != nil { - s.log.Error(logs.TreeFailedToApplyReplicatedOperation, + s.log.Error(ctx, logs.TreeFailedToApplyReplicatedOperation, zap.String("err", err.Error())) } span.End() @@ -116,11 +116,11 @@ func (s *Service) ReplicateTreeOp(ctx context.Context, n netmapSDK.NodeInfo, req if lastErr != nil { if errors.Is(lastErr, errRecentlyFailed) { - s.log.Debug(logs.TreeDoNotSendUpdateToTheNode, + s.log.Debug(ctx, logs.TreeDoNotSendUpdateToTheNode, zap.String("last_error", lastErr.Error()), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } else { - s.log.Warn(logs.TreeFailedToSentUpdateToTheNode, + s.log.Warn(ctx, logs.TreeFailedToSentUpdateToTheNode, zap.String("last_error", lastErr.Error()), zap.String("address", lastAddr), zap.String("key", hex.EncodeToString(n.PublicKey())), @@ -154,7 +154,7 @@ func (s *Service) replicateLoop(ctx context.Context) { start := time.Now() err := s.replicate(op) if err != nil { - s.log.Error(logs.TreeErrorDuringReplication, + s.log.Error(ctx, logs.TreeErrorDuringReplication, zap.String("err", err.Error()), zap.Stringer("cid", op.cid), zap.String("treeID", op.treeID)) diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index b63338d25..2df3c08e6 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -63,7 +63,7 @@ func New(opts ...Option) *Service { } if s.log == nil { - s.log = &logger.Logger{Logger: zap.NewNop()} + s.log = logger.NewLoggerWrapper(zap.NewNop()) } s.cache.init(s.key, s.ds) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 2c6deeb78..e2249c9fb 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -92,7 +92,7 @@ func (s *Service) synchronizeAllTrees(ctx context.Context, cid cid.ID) error { for _, tid := range treesToSync { h, err := s.forest.TreeLastSyncHeight(ctx, cid, tid) if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { - s.log.Warn(logs.TreeCouldNotGetLastSynchronizedHeightForATree, + s.log.Warn(ctx, logs.TreeCouldNotGetLastSynchronizedHeightForATree, zap.Stringer("cid", cid), zap.String("tree", tid)) continue @@ -100,7 +100,7 @@ func (s *Service) synchronizeAllTrees(ctx context.Context, cid cid.ID) error { newHeight := s.synchronizeTree(ctx, cid, h, tid, nodes) if h < newHeight { if err := s.forest.TreeUpdateLastSyncHeight(ctx, cid, tid, newHeight); err != nil { - s.log.Warn(logs.TreeCouldNotUpdateLastSynchronizedHeightForATree, + s.log.Warn(ctx, logs.TreeCouldNotUpdateLastSynchronizedHeightForATree, zap.Stringer("cid", cid), zap.String("tree", tid)) } @@ -251,7 +251,7 @@ func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, treeID string, nodes []netmapSDK.NodeInfo, ) uint64 { - s.log.Debug(logs.TreeSynchronizeTree, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.Uint64("from", from)) + s.log.Debug(ctx, logs.TreeSynchronizeTree, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.Uint64("from", from)) errGroup, egCtx := errgroup.WithContext(ctx) const workersCount = 1024 @@ -282,20 +282,20 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, n.IterateNetworkEndpoints(func(addr string) bool { var a network.Address if err := a.FromString(addr); err != nil { - s.log.Warn(logs.TreeFailedToParseAddressForTreeSynchronization, zap.Error(err), zap.String("address", addr)) + s.log.Warn(ctx, logs.TreeFailedToParseAddressForTreeSynchronization, zap.Error(err), zap.String("address", addr)) return false } cc, err := s.createConnection(a) if err != nil { - s.log.Warn(logs.TreeFailedToConnectForTreeSynchronization, zap.Error(err), zap.String("address", addr)) + s.log.Warn(ctx, logs.TreeFailedToConnectForTreeSynchronization, zap.Error(err), zap.String("address", addr)) return false } defer cc.Close() err = s.startStream(egCtx, cid, treeID, from, cc, nodeOperationStreams[i]) if err != nil { - s.log.Warn(logs.TreeFailedToRunTreeSynchronizationForSpecificNode, zap.Error(err), zap.String("address", addr)) + s.log.Warn(ctx, logs.TreeFailedToRunTreeSynchronizationForSpecificNode, zap.Error(err), zap.String("address", addr)) } nodeSynced = err == nil return true @@ -309,7 +309,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, } if err := errGroup.Wait(); err != nil { allNodesSynced.Store(false) - s.log.Warn(logs.TreeFailedToRunTreeSynchronizationOverAllNodes, zap.Error(err)) + s.log.Warn(ctx, logs.TreeFailedToRunTreeSynchronizationOverAllNodes, zap.Error(err)) } newHeight := minStreamedLastHeight @@ -376,13 +376,13 @@ func (s *Service) syncLoop(ctx context.Context) { return case <-s.syncChan: ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.sync") - s.log.Info(logs.TreeSyncingTrees) + s.log.Info(ctx, logs.TreeSyncingTrees) start := time.Now() cnrs, err := s.cfg.cnrSource.List() if err != nil { - s.log.Error(logs.TreeCouldNotFetchContainers, zap.Error(err)) + s.log.Error(ctx, logs.TreeCouldNotFetchContainers, zap.Error(err)) s.metrics.AddSyncDuration(time.Since(start), false) span.End() break @@ -394,7 +394,7 @@ func (s *Service) syncLoop(ctx context.Context) { s.removeContainers(ctx, newMap) - s.log.Info(logs.TreeTreesHaveBeenSynchronized) + s.log.Info(ctx, logs.TreeTreesHaveBeenSynchronized) s.metrics.AddSyncDuration(time.Since(start), true) span.End() @@ -414,19 +414,19 @@ func (s *Service) syncContainers(ctx context.Context, cnrs []cid.ID) { err := s.syncPool.Submit(func() { defer wg.Done() - s.log.Debug(logs.TreeSyncingContainerTrees, zap.Stringer("cid", cnr)) + s.log.Debug(ctx, logs.TreeSyncingContainerTrees, zap.Stringer("cid", cnr)) err := s.synchronizeAllTrees(ctx, cnr) if err != nil { - s.log.Error(logs.TreeCouldNotSyncTrees, zap.Stringer("cid", cnr), zap.Error(err)) + s.log.Error(ctx, logs.TreeCouldNotSyncTrees, zap.Stringer("cid", cnr), zap.Error(err)) return } - s.log.Debug(logs.TreeContainerTreesHaveBeenSynced, zap.Stringer("cid", cnr)) + s.log.Debug(ctx, logs.TreeContainerTreesHaveBeenSynced, zap.Stringer("cid", cnr)) }) if err != nil { wg.Done() - s.log.Error(logs.TreeCouldNotQueryTreesForSynchronization, + s.log.Error(ctx, logs.TreeCouldNotQueryTreesForSynchronization, zap.Stringer("cid", cnr), zap.Error(err)) if errors.Is(err, ants.ErrPoolClosed) { @@ -452,7 +452,7 @@ func (s *Service) removeContainers(ctx context.Context, newContainers map[cid.ID existed, err := containerCore.WasRemoved(s.cnrSource, cnr) if err != nil { - s.log.Error(logs.TreeCouldNotCheckIfContainerExisted, + s.log.Error(ctx, logs.TreeCouldNotCheckIfContainerExisted, zap.Stringer("cid", cnr), zap.Error(err)) } else if existed { @@ -464,11 +464,11 @@ func (s *Service) removeContainers(ctx context.Context, newContainers map[cid.ID } for _, cnr := range removed { - s.log.Debug(logs.TreeRemovingRedundantTrees, zap.Stringer("cid", cnr)) + s.log.Debug(ctx, logs.TreeRemovingRedundantTrees, zap.Stringer("cid", cnr)) err := s.DropTree(ctx, cnr, "") if err != nil { - s.log.Error(logs.TreeCouldNotRemoveRedundantTree, + s.log.Error(ctx, logs.TreeCouldNotRemoveRedundantTree, zap.Stringer("cid", cnr), zap.Error(err)) } @@ -482,7 +482,7 @@ func (s *Service) containersToSync(cnrs []cid.ID) (map[cid.ID]struct{}, []cid.ID for _, cnr := range cnrs { _, pos, err := s.getContainerNodes(cnr) if err != nil { - s.log.Error(logs.TreeCouldNotCalculateContainerNodes, + s.log.Error(context.Background(), logs.TreeCouldNotCalculateContainerNodes, zap.Stringer("cid", cnr), zap.Error(err)) continue diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index d4ac2ab02..b3a1b9b94 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -184,6 +184,10 @@ func (l *Logger) WithOptions(options ...zap.Option) { l.z = l.z.WithOptions(options...) } +func (l *Logger) With(fields ...zap.Field) *Logger { + return &Logger{z: l.z.With(fields...)} +} + func NewLoggerWrapper(z *zap.Logger) *Logger { return &Logger{ z: z, From 62b5181618f194f024ede60070e240bcd8fe7a31 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 21 Oct 2024 11:20:17 +0300 Subject: [PATCH 1530/1943] [#1437] blobovnicza: Fix contextcheck linter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-lens/internal/blobovnicza/root.go | 2 +- .../blobovnicza/blobovnicza_test.go | 2 +- .../blobovnicza/control.go | 6 ++--- .../blobovnicza/get_test.go | 2 +- .../blobovnicza/iterate_test.go | 2 +- .../blobstor/blobovniczatree/active.go | 23 ++++++++++--------- .../blobstor/blobovniczatree/cache.go | 8 +++---- .../blobstor/blobovniczatree/control.go | 6 ++--- .../blobstor/blobovniczatree/count.go | 4 ++-- .../blobstor/blobovniczatree/delete.go | 8 +++---- .../blobstor/blobovniczatree/exists.go | 4 ++-- .../blobstor/blobovniczatree/get.go | 8 +++---- .../blobstor/blobovniczatree/get_range.go | 8 +++---- .../blobstor/blobovniczatree/iterate.go | 4 ++-- .../blobstor/blobovniczatree/manager.go | 4 ++-- .../blobstor/blobovniczatree/put.go | 2 +- .../blobstor/blobovniczatree/rebuild.go | 22 +++++++++--------- .../blobovniczatree/rebuild_failover_test.go | 14 +++++------ pkg/local_object_storage/engine/engine.go | 4 ++-- pkg/morph/client/client.go | 4 ++-- pkg/morph/event/listener.go | 16 ++++++------- pkg/morph/subscriber/subscriber.go | 10 ++++---- 22 files changed, 82 insertions(+), 81 deletions(-) diff --git a/cmd/frostfs-lens/internal/blobovnicza/root.go b/cmd/frostfs-lens/internal/blobovnicza/root.go index 9d8ef3dad..2819981d6 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/root.go +++ b/cmd/frostfs-lens/internal/blobovnicza/root.go @@ -27,7 +27,7 @@ func openBlobovnicza(cmd *cobra.Command) *blobovnicza.Blobovnicza { blobovnicza.WithPath(vPath), blobovnicza.WithReadOnly(true), ) - common.ExitOnErr(cmd, blz.Open()) + common.ExitOnErr(cmd, blz.Open(cmd.Context())) return blz } diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index caee770e8..10cb6f368 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -69,7 +69,7 @@ func TestBlobovnicza(t *testing.T) { defer os.Remove(p) // open Blobovnicza - require.NoError(t, blz.Open()) + require.NoError(t, blz.Open(context.Background())) // initialize Blobovnicza require.NoError(t, blz.Init()) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index 5d7135741..a317279a4 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -16,7 +16,7 @@ import ( // // If the database file does not exist, it will be created automatically. // If blobovnicza is already open, does nothing. -func (b *Blobovnicza) Open() error { +func (b *Blobovnicza) Open(ctx context.Context) error { b.controlMtx.Lock() defer b.controlMtx.Unlock() @@ -24,7 +24,7 @@ func (b *Blobovnicza) Open() error { return nil } - b.log.Debug(context.Background(), logs.BlobovniczaCreatingDirectoryForBoltDB, + b.log.Debug(ctx, logs.BlobovniczaCreatingDirectoryForBoltDB, zap.String("path", b.path), zap.Bool("ro", b.boltOptions.ReadOnly), ) @@ -38,7 +38,7 @@ func (b *Blobovnicza) Open() error { } } - b.log.Debug(context.Background(), logs.BlobovniczaOpeningBoltDB, + b.log.Debug(ctx, logs.BlobovniczaOpeningBoltDB, zap.String("path", b.path), zap.Stringer("permissions", b.perm), ) diff --git a/pkg/local_object_storage/blobovnicza/get_test.go b/pkg/local_object_storage/blobovnicza/get_test.go index c464abc87..8c435af89 100644 --- a/pkg/local_object_storage/blobovnicza/get_test.go +++ b/pkg/local_object_storage/blobovnicza/get_test.go @@ -26,7 +26,7 @@ func TestBlobovnicza_Get(t *testing.T) { WithObjectSizeLimit(szLimit), ) - require.NoError(t, blz.Open()) + require.NoError(t, blz.Open(context.Background())) require.NoError(t, blz.Init()) } diff --git a/pkg/local_object_storage/blobovnicza/iterate_test.go b/pkg/local_object_storage/blobovnicza/iterate_test.go index 5db1e4165..bec23bb96 100644 --- a/pkg/local_object_storage/blobovnicza/iterate_test.go +++ b/pkg/local_object_storage/blobovnicza/iterate_test.go @@ -15,7 +15,7 @@ import ( func TestBlobovniczaIterate(t *testing.T) { filename := filepath.Join(t.TempDir(), "blob") b := New(WithPath(filename)) - require.NoError(t, b.Open()) + require.NoError(t, b.Open(context.Background())) require.NoError(t, b.Init()) data := [][]byte{{0, 1, 2, 3}, {5, 6, 7, 8}} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/active.go b/pkg/local_object_storage/blobstor/blobovniczatree/active.go index 603c6abe3..0ac15df82 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/active.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/active.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "context" "path/filepath" "sync" @@ -53,8 +54,8 @@ func newActiveDBManager(dbManager *dbManager, rootPath string) *activeDBManager // GetOpenedActiveDBForLevel returns active DB for level. // DB must be closed after use. -func (m *activeDBManager) GetOpenedActiveDBForLevel(lvlPath string) (*activeDB, error) { - activeDB, err := m.getCurrentActiveIfOk(lvlPath) +func (m *activeDBManager) GetOpenedActiveDBForLevel(ctx context.Context, lvlPath string) (*activeDB, error) { + activeDB, err := m.getCurrentActiveIfOk(ctx, lvlPath) if err != nil { return nil, err } @@ -62,7 +63,7 @@ func (m *activeDBManager) GetOpenedActiveDBForLevel(lvlPath string) (*activeDB, return activeDB, nil } - return m.updateAndGetActive(lvlPath) + return m.updateAndGetActive(ctx, lvlPath) } func (m *activeDBManager) Open() { @@ -83,7 +84,7 @@ func (m *activeDBManager) Close() { m.closed = true } -func (m *activeDBManager) getCurrentActiveIfOk(lvlPath string) (*activeDB, error) { +func (m *activeDBManager) getCurrentActiveIfOk(ctx context.Context, lvlPath string) (*activeDB, error) { m.levelToActiveDBGuard.RLock() defer m.levelToActiveDBGuard.RUnlock() @@ -96,7 +97,7 @@ func (m *activeDBManager) getCurrentActiveIfOk(lvlPath string) (*activeDB, error return nil, nil } - blz, err := db.Open() // open db for usage, will be closed on activeDB.Close() + blz, err := db.Open(ctx) // open db for usage, will be closed on activeDB.Close() if err != nil { return nil, err } @@ -112,11 +113,11 @@ func (m *activeDBManager) getCurrentActiveIfOk(lvlPath string) (*activeDB, error }, nil } -func (m *activeDBManager) updateAndGetActive(lvlPath string) (*activeDB, error) { +func (m *activeDBManager) updateAndGetActive(ctx context.Context, lvlPath string) (*activeDB, error) { m.levelLock.Lock(lvlPath) defer m.levelLock.Unlock(lvlPath) - current, err := m.getCurrentActiveIfOk(lvlPath) + current, err := m.getCurrentActiveIfOk(ctx, lvlPath) if err != nil { return nil, err } @@ -124,7 +125,7 @@ func (m *activeDBManager) updateAndGetActive(lvlPath string) (*activeDB, error) return current, nil } - nextShDB, err := m.getNextSharedDB(lvlPath) + nextShDB, err := m.getNextSharedDB(ctx, lvlPath) if err != nil { return nil, err } @@ -133,7 +134,7 @@ func (m *activeDBManager) updateAndGetActive(lvlPath string) (*activeDB, error) return nil, nil } - blz, err := nextShDB.Open() // open db for client, client must call Close() after usage + blz, err := nextShDB.Open(ctx) // open db for client, client must call Close() after usage if err != nil { return nil, err } @@ -143,7 +144,7 @@ func (m *activeDBManager) updateAndGetActive(lvlPath string) (*activeDB, error) }, nil } -func (m *activeDBManager) getNextSharedDB(lvlPath string) (*sharedDB, error) { +func (m *activeDBManager) getNextSharedDB(ctx context.Context, lvlPath string) (*sharedDB, error) { var nextActiveDBIdx uint64 hasActive, currentIdx := m.hasActiveDB(lvlPath) if hasActive { @@ -160,7 +161,7 @@ func (m *activeDBManager) getNextSharedDB(lvlPath string) (*sharedDB, error) { path := filepath.Join(lvlPath, u64ToHexStringExt(nextActiveDBIdx)) next := m.dbManager.GetByPath(path) - _, err := next.Open() // open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close() + _, err := next.Open(ctx) // open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close() if err != nil { return nil, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go index 5c103c1bb..e8016781a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -81,12 +81,12 @@ func (c *dbCache) Close() { c.closed = true } -func (c *dbCache) GetOrCreate(path string) *sharedDB { +func (c *dbCache) GetOrCreate(ctx context.Context, path string) *sharedDB { value := c.getExisted(path) if value != nil { return value } - return c.create(path) + return c.create(ctx, path) } func (c *dbCache) EvictAndMarkNonCached(path string) { @@ -122,7 +122,7 @@ func (c *dbCache) getExisted(path string) *sharedDB { return nil } -func (c *dbCache) create(path string) *sharedDB { +func (c *dbCache) create(ctx context.Context, path string) *sharedDB { c.pathLock.Lock(path) defer c.pathLock.Unlock(path) @@ -133,7 +133,7 @@ func (c *dbCache) create(path string) *sharedDB { value = c.dbManager.GetByPath(path) - _, err := value.Open() // open db to hold reference, closed by evictedDB.Close() or if cache closed + _, err := value.Open(ctx) // open db to hold reference, closed by evictedDB.Close() or if cache closed if err != nil { return value } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 55c9d6630..7c0a9edd6 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -46,7 +46,7 @@ func (b *Blobovniczas) initializeDBs(ctx context.Context) error { eg.Go(func() error { p = strings.TrimSuffix(p, rebuildSuffix) shBlz := b.getBlobovniczaWithoutCaching(p) - blz, err := shBlz.Open() + blz, err := shBlz.Open(egCtx) if err != nil { return err } @@ -91,8 +91,8 @@ func (b *Blobovniczas) Close() error { // returns blobovnicza with path p // // If blobovnicza is already cached, instance from cache is returned w/o changes. -func (b *Blobovniczas) getBlobovnicza(p string) *sharedDB { - return b.dbCache.GetOrCreate(p) +func (b *Blobovniczas) getBlobovnicza(ctx context.Context, p string) *sharedDB { + return b.dbCache.GetOrCreate(ctx, p) } func (b *Blobovniczas) getBlobovniczaWithoutCaching(p string) *sharedDB { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/count.go b/pkg/local_object_storage/blobstor/blobovniczatree/count.go index cf91637d7..1137b9eb2 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/count.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/count.go @@ -16,13 +16,13 @@ func (b *Blobovniczas) ObjectsCount(ctx context.Context) (uint64, error) { b.metrics.ObjectsCount(time.Since(startedAt), success) }() - _, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.ObjectsCount") + ctx, span := tracing.StartSpanFromContext(ctx, "Blobovniczas.ObjectsCount") defer span.End() var result uint64 err := b.iterateExistingDBPaths(ctx, func(p string) (bool, error) { shDB := b.getBlobovniczaWithoutCaching(p) - blz, err := shDB.Open() + blz, err := shDB.Open(ctx) if err != nil { return true, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index dd5258042..b26e44144 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -61,8 +61,8 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co if prm.StorageID != nil { id := NewIDFromBytes(prm.StorageID) - shBlz := b.getBlobovnicza(id.Path()) - blz, err := shBlz.Open() + shBlz := b.getBlobovnicza(ctx, id.Path()) + blz, err := shBlz.Open(ctx) if err != nil { return res, err } @@ -109,8 +109,8 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co // // returns no error if object was removed from some blobovnicza of the same level. func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicza.DeletePrm, blzPath string) (common.DeleteRes, error) { - shBlz := b.getBlobovnicza(blzPath) - blz, err := shBlz.Open() + shBlz := b.getBlobovnicza(ctx, blzPath) + blz, err := shBlz.Open(ctx) if err != nil { return common.DeleteRes{}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index 2149b17c0..528dbfed7 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -37,8 +37,8 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common if prm.StorageID != nil { id := NewIDFromBytes(prm.StorageID) - shBlz := b.getBlobovnicza(id.Path()) - blz, err := shBlz.Open() + shBlz := b.getBlobovnicza(ctx, id.Path()) + blz, err := shBlz.Open(ctx) if err != nil { return common.ExistsRes{}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index e79480095..fc017f22d 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -48,8 +48,8 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G if prm.StorageID != nil { id := NewIDFromBytes(prm.StorageID) - shBlz := b.getBlobovnicza(id.Path()) - blz, err := shBlz.Open() + shBlz := b.getBlobovnicza(ctx, id.Path()) + blz, err := shBlz.Open(ctx) if err != nil { return res, err } @@ -95,8 +95,8 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G // returns error if object could not be read from any blobovnicza of the same level. func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.GetPrm, blzPath string) (common.GetRes, error) { // open blobovnicza (cached inside) - shBlz := b.getBlobovnicza(blzPath) - blz, err := shBlz.Open() + shBlz := b.getBlobovnicza(ctx, blzPath) + blz, err := shBlz.Open(ctx) if err != nil { return common.GetRes{}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 20f2be2ba..384544d7b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -47,8 +47,8 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re if prm.StorageID != nil { id := NewIDFromBytes(prm.StorageID) - shBlz := b.getBlobovnicza(id.Path()) - blz, err := shBlz.Open() + shBlz := b.getBlobovnicza(ctx, id.Path()) + blz, err := shBlz.Open(ctx) if err != nil { return common.GetRangeRes{}, err } @@ -103,8 +103,8 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re // returns error if object could not be read from any blobovnicza of the same level. func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRangePrm, blzPath string) (common.GetRangeRes, error) { // open blobovnicza (cached inside) - shBlz := b.getBlobovnicza(blzPath) - blz, err := shBlz.Open() + shBlz := b.getBlobovnicza(ctx, blzPath) + blz, err := shBlz.Open(ctx) if err != nil { return common.GetRangeRes{}, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 7f0453410..049a61d72 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -72,8 +72,8 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm // iterator over all Blobovniczas in unsorted order. Break on f's error return. func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error { return b.iterateExistingDBPaths(ctx, func(p string) (bool, error) { - shBlz := b.getBlobovnicza(p) - blz, err := shBlz.Open() + shBlz := b.getBlobovnicza(ctx, p) + blz, err := shBlz.Open(ctx) if err != nil { if ignoreErrors { b.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index 235c9f65d..502202d68 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -49,7 +49,7 @@ func newSharedDB(options []blobovnicza.Option, path string, readOnly bool, } } -func (b *sharedDB) Open() (*blobovnicza.Blobovnicza, error) { +func (b *sharedDB) Open(ctx context.Context) (*blobovnicza.Blobovnicza, error) { if b.closedFlag.Load() { return nil, errClosed } @@ -68,7 +68,7 @@ func (b *sharedDB) Open() (*blobovnicza.Blobovnicza, error) { blobovnicza.WithMetrics(b.metrics), )...) - if err := blz.Open(); err != nil { + if err := blz.Open(ctx); err != nil { return nil, fmt.Errorf("could not open blobovnicza %s: %w", b.path, err) } if err := blz.Init(); err != nil { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 844b43151..8dff436d3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -77,7 +77,7 @@ type putIterator struct { } func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) { - active, err := i.B.activeDBManager.GetOpenedActiveDBForLevel(lvlPath) + active, err := i.B.activeDBManager.GetOpenedActiveDBForLevel(ctx, lvlPath) if err != nil { if !isLogical(err) { i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index fee67a0a8..e137bdd99 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -165,7 +165,7 @@ func (b *Blobovniczas) selectDBsDoNotMatchFillPercent(ctx context.Context, targe continue } path := filepath.Join(lvlPath, e.Name()) - resettlementRequired, err := b.rebuildBySize(path, target) + resettlementRequired, err := b.rebuildBySize(ctx, path, target) if err != nil { return false, err } @@ -180,9 +180,9 @@ func (b *Blobovniczas) selectDBsDoNotMatchFillPercent(ctx context.Context, targe return result, nil } -func (b *Blobovniczas) rebuildBySize(path string, targetFillPercent int) (bool, error) { - shDB := b.getBlobovnicza(path) - blz, err := shDB.Open() +func (b *Blobovniczas) rebuildBySize(ctx context.Context, path string, targetFillPercent int) (bool, error) { + shDB := b.getBlobovnicza(ctx, path) + blz, err := shDB.Open(ctx) if err != nil { return false, err } @@ -196,8 +196,8 @@ func (b *Blobovniczas) rebuildBySize(path string, targetFillPercent int) (bool, } func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.MetaStorage, limiter common.ConcurrentWorkersLimiter) (uint64, error) { - shDB := b.getBlobovnicza(path) - blz, err := shDB.Open() + shDB := b.getBlobovnicza(ctx, path) + blz, err := shDB.Open(ctx) if err != nil { return 0, err } @@ -365,8 +365,8 @@ func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore co err := b.iterateIncompletedRebuildDBPaths(ctx, func(s string) (bool, error) { rebuildTmpFilePath := s s = strings.TrimSuffix(s, rebuildSuffix) - shDB := b.getBlobovnicza(s) - blz, err := shDB.Open() + shDB := b.getBlobovnicza(ctx, s) + blz, err := shDB.Open(ctx) if err != nil { return true, err } @@ -398,8 +398,8 @@ func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore co func (b *Blobovniczas) performMove(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string, move blobovnicza.MoveInfo, metaStore common.MetaStorage, ) error { - targetDB := b.getBlobovnicza(NewIDFromBytes(move.TargetStorageID).Path()) - target, err := targetDB.Open() + targetDB := b.getBlobovnicza(ctx, NewIDFromBytes(move.TargetStorageID).Path()) + target, err := targetDB.Open(ctx) if err != nil { return err } @@ -477,7 +477,7 @@ type moveIterator struct { } func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, error) { - target, err := i.B.activeDBManager.GetOpenedActiveDBForLevel(lvlPath) + target, err := i.B.activeDBManager.GetOpenedActiveDBForLevel(ctx, lvlPath) if err != nil { if !isLogical(err) { i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index b177d20fc..bfea97afe 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -35,7 +35,7 @@ func testRebuildFailoverOnlyMoveInfoSaved(t *testing.T) { dir := t.TempDir() blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) - require.NoError(t, blz.Open()) + require.NoError(t, blz.Open(context.Background())) require.NoError(t, blz.Init()) obj := blobstortest.NewObject(1024) @@ -65,7 +65,7 @@ func testRebuildFailoverObjectSavedToTarget(t *testing.T) { dir := t.TempDir() blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) - require.NoError(t, blz.Open()) + require.NoError(t, blz.Open(context.Background())) require.NoError(t, blz.Init()) obj := blobstortest.NewObject(1024) @@ -89,7 +89,7 @@ func testRebuildFailoverObjectSavedToTarget(t *testing.T) { require.NoError(t, err) blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db"))) - require.NoError(t, blz.Open()) + require.NoError(t, blz.Open(context.Background())) require.NoError(t, blz.Init()) _, err = blz.Put(context.Background(), pPrm) @@ -105,7 +105,7 @@ func testRebuildFailoverObjectDeletedFromSource(t *testing.T) { dir := t.TempDir() blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) - require.NoError(t, blz.Open()) + require.NoError(t, blz.Open(context.Background())) require.NoError(t, blz.Init()) obj := blobstortest.NewObject(1024) @@ -123,7 +123,7 @@ func testRebuildFailoverObjectDeletedFromSource(t *testing.T) { require.NoError(t, err) blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db"))) - require.NoError(t, blz.Open()) + require.NoError(t, blz.Open(context.Background())) require.NoError(t, blz.Init()) var pPrm blobovnicza.PutPrm @@ -173,7 +173,7 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object require.NoError(t, b.Close()) blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) - require.NoError(t, blz.Open()) + require.NoError(t, blz.Open(context.Background())) require.NoError(t, blz.Init()) moveInfo, err := blz.ListMoveInfo(context.Background()) @@ -188,7 +188,7 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object require.NoError(t, blz.Close()) blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db"))) - require.NoError(t, blz.Open()) + require.NoError(t, blz.Open(context.Background())) require.NoError(t, blz.Init()) moveInfo, err = blz.ListMoveInfo(context.Background()) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 8963ec099..a8caa215a 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -269,8 +269,8 @@ type containerSource struct { func (s *containerSource) IsContainerAvailable(ctx context.Context, id cid.ID) (bool, error) { select { - case <-context.Background().Done(): - return false, context.Background().Err() + case <-ctx.Done(): + return false, ctx.Err() default: } diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 12c0e0842..ef6a009e4 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -389,7 +389,7 @@ func (c *Client) Wait(ctx context.Context, n uint32) error { height, err = c.rpcActor.GetBlockCount() if err != nil { - c.logger.Error(context.Background(), logs.ClientCantGetBlockchainHeight, + c.logger.Error(ctx, logs.ClientCantGetBlockchainHeight, zap.String("error", err.Error())) return nil } @@ -403,7 +403,7 @@ func (c *Client) Wait(ctx context.Context, n uint32) error { newHeight, err = c.rpcActor.GetBlockCount() if err != nil { - c.logger.Error(context.Background(), logs.ClientCantGetBlockchainHeight243, + c.logger.Error(ctx, logs.ClientCantGetBlockchainHeight243, zap.String("error", err.Error())) return nil } diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 03bba8ab9..3d3d806a4 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -269,7 +269,7 @@ loop: continue loop } - l.handleNotifyEvent(notifyEvent) + l.handleNotifyEvent(ctx, notifyEvent) case notaryEvent, ok := <-chs.NotaryRequestsCh: if !ok { l.log.Warn(ctx, logs.EventStopEventListenerByNotaryChannel) @@ -316,16 +316,16 @@ func (l *listener) handleNotaryEvent(notaryEvent *result.NotaryRequestEvent) { } } -func (l *listener) handleNotifyEvent(notifyEvent *state.ContainedNotificationEvent) { +func (l *listener) handleNotifyEvent(ctx context.Context, notifyEvent *state.ContainedNotificationEvent) { if err := l.pool.Submit(func() { - l.parseAndHandleNotification(notifyEvent) + l.parseAndHandleNotification(ctx, notifyEvent) }); err != nil { - l.log.Warn(context.Background(), logs.EventListenerWorkerPoolDrained, + l.log.Warn(ctx, logs.EventListenerWorkerPoolDrained, zap.Int("capacity", l.pool.Cap())) } } -func (l *listener) parseAndHandleNotification(notifyEvent *state.ContainedNotificationEvent) { +func (l *listener) parseAndHandleNotification(ctx context.Context, notifyEvent *state.ContainedNotificationEvent) { log := l.log.With( zap.String("script hash LE", notifyEvent.ScriptHash.StringLE()), ) @@ -347,7 +347,7 @@ func (l *listener) parseAndHandleNotification(notifyEvent *state.ContainedNotifi l.mtx.RUnlock() if !ok { - log.Debug(context.Background(), logs.EventEventParserNotSet) + log.Debug(ctx, logs.EventEventParserNotSet) return } @@ -355,7 +355,7 @@ func (l *listener) parseAndHandleNotification(notifyEvent *state.ContainedNotifi // parse the notification event event, err := parser(notifyEvent) if err != nil { - log.Warn(context.Background(), logs.EventCouldNotParseNotificationEvent, + log.Warn(ctx, logs.EventCouldNotParseNotificationEvent, zap.String("error", err.Error()), ) @@ -368,7 +368,7 @@ func (l *listener) parseAndHandleNotification(notifyEvent *state.ContainedNotifi l.mtx.RUnlock() if len(handlers) == 0 { - log.Info(context.Background(), logs.EventNotificationHandlersForParsedNotificationEventWereNotRegistered, + log.Info(ctx, logs.EventNotificationHandlersForParsedNotificationEventWereNotRegistered, zap.Any("event", event), ) diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 3a2da6757..4ef59ed6a 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -254,7 +254,7 @@ func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) b s.Lock() chs := newSubChannels() go func() { - finishCh <- s.restoreSubscriptions(chs.NotifyChan, chs.BlockChan, chs.NotaryChan) + finishCh <- s.restoreSubscriptions(ctx, chs.NotifyChan, chs.BlockChan, chs.NotaryChan) }() s.current = chs s.Unlock() @@ -295,7 +295,7 @@ drainloop: // restoreSubscriptions restores subscriptions according to // cached information about them. -func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotificationEvent, +func (s *subscriber) restoreSubscriptions(ctx context.Context, notifCh chan<- *state.ContainedNotificationEvent, blCh chan<- *block.Block, notaryCh chan<- *result.NotaryRequestEvent, ) bool { var err error @@ -304,7 +304,7 @@ func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotific if s.subscribedToNewBlocks { _, err = s.client.ReceiveBlocks(blCh) if err != nil { - s.log.Error(context.Background(), logs.ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch, zap.Error(err)) + s.log.Error(ctx, logs.ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch, zap.Error(err)) return false } } @@ -313,7 +313,7 @@ func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotific for contract := range s.subscribedEvents { _, err = s.client.ReceiveExecutionNotifications(contract, notifCh) if err != nil { - s.log.Error(context.Background(), logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) + s.log.Error(ctx, logs.ClientCouldNotRestoreNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) return false } } @@ -322,7 +322,7 @@ func (s *subscriber) restoreSubscriptions(notifCh chan<- *state.ContainedNotific for signer := range s.subscribedNotaryEvents { _, err = s.client.ReceiveNotaryRequests(signer, notaryCh) if err != nil { - s.log.Error(context.Background(), logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) + s.log.Error(ctx, logs.ClientCouldNotRestoreNotaryNotificationSubscriptionAfterRPCSwitch, zap.Error(err)) return false } } From c139892117c3ebd6405ea1362cd7576edda1ac00 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 21 Oct 2024 11:30:31 +0300 Subject: [PATCH 1531/1943] [#1437] ir: Fix contextcheck linter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/config.go | 38 ++++++++++++++++++------------------- cmd/frostfs-ir/main.go | 12 ++++++------ pkg/innerring/innerring.go | 12 ++++++------ pkg/innerring/state.go | 12 ++++++------ pkg/innerring/state_test.go | 3 ++- 5 files changed, 39 insertions(+), 38 deletions(-) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 34d9d5595..7415e8e70 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -47,7 +47,7 @@ func reloadConfig() error { return logPrm.Reload() } -func watchForSignal(cancel func()) { +func watchForSignal(ctx context.Context, cancel func()) { ch := make(chan os.Signal, 1) signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) @@ -59,49 +59,49 @@ func watchForSignal(cancel func()) { // signals causing application to shut down should have priority over // reconfiguration signal case <-ch: - log.Info(context.Background(), logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + log.Info(ctx, logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) cancel() - shutdown() - log.Info(context.Background(), logs.FrostFSNodeTerminationSignalProcessingIsComplete) + shutdown(ctx) + log.Info(ctx, logs.FrostFSNodeTerminationSignalProcessingIsComplete) return case err := <-intErr: // internal application error - log.Info(context.Background(), logs.FrostFSIRInternalError, zap.String("msg", err.Error())) + log.Info(ctx, logs.FrostFSIRInternalError, zap.String("msg", err.Error())) cancel() - shutdown() + shutdown(ctx) return default: // block until any signal is receieved select { case <-ch: - log.Info(context.Background(), logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + log.Info(ctx, logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) cancel() - shutdown() - log.Info(context.Background(), logs.FrostFSNodeTerminationSignalProcessingIsComplete) + shutdown(ctx) + log.Info(ctx, logs.FrostFSNodeTerminationSignalProcessingIsComplete) return case err := <-intErr: // internal application error - log.Info(context.Background(), logs.FrostFSIRInternalError, zap.String("msg", err.Error())) + log.Info(ctx, logs.FrostFSIRInternalError, zap.String("msg", err.Error())) cancel() - shutdown() + shutdown(ctx) return case <-sighupCh: - log.Info(context.Background(), logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) - if !innerRing.CompareAndSwapHealthStatus(control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) { - log.Info(context.Background(), logs.FrostFSNodeSIGHUPSkip) + log.Info(ctx, logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) + if !innerRing.CompareAndSwapHealthStatus(ctx, control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) { + log.Info(ctx, logs.FrostFSNodeSIGHUPSkip) break } err := reloadConfig() if err != nil { - log.Error(context.Background(), logs.FrostFSNodeConfigurationReading, zap.Error(err)) + log.Error(ctx, logs.FrostFSNodeConfigurationReading, zap.Error(err)) } pprofCmp.reload() metricsCmp.reload() - log.Info(context.Background(), logs.FrostFSIRReloadExtraWallets) + log.Info(ctx, logs.FrostFSIRReloadExtraWallets) err = innerRing.SetExtraWallets(cfg) if err != nil { - log.Error(context.Background(), logs.FrostFSNodeConfigurationReading, zap.Error(err)) + log.Error(ctx, logs.FrostFSNodeConfigurationReading, zap.Error(err)) } - innerRing.CompareAndSwapHealthStatus(control.HealthStatus_RECONFIGURING, control.HealthStatus_READY) - log.Info(context.Background(), logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) + innerRing.CompareAndSwapHealthStatus(ctx, control.HealthStatus_RECONFIGURING, control.HealthStatus_READY) + log.Info(ctx, logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } } } diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 55a8ce00d..bcb2c5dd8 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -106,7 +106,7 @@ func main() { log.Info(ctx, logs.CommonApplicationStarted, zap.String("version", misc.Version)) - watchForSignal(cancel) + watchForSignal(ctx, cancel) <-ctx.Done() // graceful shutdown log.Debug(ctx, logs.FrostFSNodeWaitingForAllProcessesToStop) @@ -115,20 +115,20 @@ func main() { log.Info(ctx, logs.FrostFSIRApplicationStopped) } -func shutdown() { - innerRing.Stop() +func shutdown(ctx context.Context) { + innerRing.Stop(ctx) if err := metricsCmp.shutdown(); err != nil { - log.Debug(context.Background(), logs.FrostFSIRCouldNotShutdownHTTPServer, + log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error()), ) } if err := pprofCmp.shutdown(); err != nil { - log.Debug(context.Background(), logs.FrostFSIRCouldNotShutdownHTTPServer, + log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error()), ) } if err := sdnotify.ClearStatus(); err != nil { - log.Error(context.Background(), logs.FailedToReportStatusToSystemd, zap.Error(err)) + log.Error(ctx, logs.FailedToReportStatusToSystemd, zap.Error(err)) } } diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 63a4cb1cb..67927c10c 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -140,10 +140,10 @@ var ( // Start runs all event providers. func (s *Server) Start(ctx context.Context, intError chan<- error) (err error) { - s.setHealthStatus(control.HealthStatus_STARTING) + s.setHealthStatus(ctx, control.HealthStatus_STARTING) defer func() { if err == nil { - s.setHealthStatus(control.HealthStatus_READY) + s.setHealthStatus(ctx, control.HealthStatus_READY) } }() @@ -299,15 +299,15 @@ func (s *Server) startWorkers(ctx context.Context) { } // Stop closes all subscription channels. -func (s *Server) Stop() { - s.setHealthStatus(control.HealthStatus_SHUTTING_DOWN) +func (s *Server) Stop(ctx context.Context) { + s.setHealthStatus(ctx, control.HealthStatus_SHUTTING_DOWN) go s.morphListener.Stop() go s.mainnetListener.Stop() for _, c := range s.closers { if err := c(); err != nil { - s.log.Warn(context.Background(), logs.InnerringCloserError, + s.log.Warn(ctx, logs.InnerringCloserError, zap.String("error", err.Error()), ) } @@ -349,7 +349,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED) + server.setHealthStatus(ctx, control.HealthStatus_HEALTH_STATUS_UNDEFINED) // parse notary support server.feeConfig = config.NewFeeConfig(cfg) diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 250f41e5f..2dbcd7494 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -154,17 +154,17 @@ func (s *Server) ResetEpochTimer(h uint32) error { return s.epochTimer.Reset() } -func (s *Server) setHealthStatus(hs control.HealthStatus) { +func (s *Server) setHealthStatus(ctx context.Context, hs control.HealthStatus) { s.healthStatus.Store(int32(hs)) - s.notifySystemd(hs) + s.notifySystemd(ctx, hs) if s.irMetrics != nil { s.irMetrics.SetHealth(int32(hs)) } } -func (s *Server) CompareAndSwapHealthStatus(oldSt, newSt control.HealthStatus) (swapped bool) { +func (s *Server) CompareAndSwapHealthStatus(ctx context.Context, oldSt, newSt control.HealthStatus) (swapped bool) { if swapped = s.healthStatus.CompareAndSwap(int32(oldSt), int32(newSt)); swapped { - s.notifySystemd(newSt) + s.notifySystemd(ctx, newSt) if s.irMetrics != nil { s.irMetrics.SetHealth(int32(newSt)) } @@ -187,7 +187,7 @@ func initPersistentStateStorage(cfg *viper.Viper) (*state.PersistentStorage, err return persistStorage, nil } -func (s *Server) notifySystemd(st control.HealthStatus) { +func (s *Server) notifySystemd(ctx context.Context, st control.HealthStatus) { if !s.sdNotify { return } @@ -203,6 +203,6 @@ func (s *Server) notifySystemd(st control.HealthStatus) { err = sdnotify.Status(fmt.Sprintf("%v", st)) } if err != nil { - s.log.Error(context.Background(), logs.FailedToReportStatusToSystemd, zap.Error(err)) + s.log.Error(ctx, logs.FailedToReportStatusToSystemd, zap.Error(err)) } } diff --git a/pkg/innerring/state_test.go b/pkg/innerring/state_test.go index fe09f8f2d..9313edf78 100644 --- a/pkg/innerring/state_test.go +++ b/pkg/innerring/state_test.go @@ -1,6 +1,7 @@ package innerring import ( + "context" "testing" "time" @@ -42,7 +43,7 @@ func TestServerState(t *testing.T) { require.Equal(t, epochDuration, srv.EpochDuration(), "invalid epoch duration") var healthStatus control.HealthStatus = control.HealthStatus_READY - srv.setHealthStatus(healthStatus) + srv.setHealthStatus(context.Background(), healthStatus) require.Equal(t, healthStatus, srv.HealthStatus(), "invalid health status") require.True(t, srv.IsActive(), "invalid IsActive result") From 16598553d9603cdf7d5b349db250041a80f9ec14 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 21 Oct 2024 11:56:38 +0300 Subject: [PATCH 1532/1943] [#1437] shard: Fix contextcheck linter Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobstor/mode.go | 4 +-- pkg/local_object_storage/engine/control.go | 20 +++++------ pkg/local_object_storage/engine/engine.go | 10 +++--- pkg/local_object_storage/engine/error_test.go | 4 +-- .../engine/evacuate_test.go | 36 +++++++++---------- pkg/local_object_storage/engine/shards.go | 36 +++++++++---------- .../engine/shards_test.go | 10 +++--- .../internal/storagetest/storage.go | 12 +++---- pkg/local_object_storage/metabase/control.go | 14 ++++---- pkg/local_object_storage/metabase/mode.go | 5 +-- pkg/local_object_storage/metabase/shard_id.go | 9 ++--- .../metabase/version_test.go | 2 +- pkg/local_object_storage/pilorama/boltdb.go | 2 +- pkg/local_object_storage/pilorama/forest.go | 2 +- .../pilorama/interface.go | 2 +- pkg/local_object_storage/shard/control.go | 34 +++++++++--------- .../shard/control_test.go | 16 ++++----- pkg/local_object_storage/shard/delete_test.go | 2 +- .../shard/gc_internal_test.go | 2 +- pkg/local_object_storage/shard/gc_test.go | 8 ++--- pkg/local_object_storage/shard/get_test.go | 2 +- pkg/local_object_storage/shard/head_test.go | 2 +- pkg/local_object_storage/shard/id.go | 7 ++-- pkg/local_object_storage/shard/inhume_test.go | 2 +- pkg/local_object_storage/shard/list_test.go | 4 +-- pkg/local_object_storage/shard/lock_test.go | 4 +-- .../shard/metrics_test.go | 6 ++-- pkg/local_object_storage/shard/mode.go | 16 +++++---- pkg/local_object_storage/shard/range_test.go | 2 +- pkg/local_object_storage/shard/rebuild.go | 4 +-- pkg/local_object_storage/shard/refill_test.go | 6 ++-- pkg/local_object_storage/shard/reload_test.go | 2 +- .../shard/shutdown_test.go | 4 +-- .../writecache/flush_test.go | 20 +++++------ pkg/local_object_storage/writecache/mode.go | 4 +-- .../writecache/writecache.go | 2 +- pkg/services/control/server/detach_shards.go | 4 +-- pkg/services/control/server/set_shard_mode.go | 4 +-- 38 files changed, 165 insertions(+), 160 deletions(-) diff --git a/pkg/local_object_storage/blobstor/mode.go b/pkg/local_object_storage/blobstor/mode.go index a579a6f92..f081ff645 100644 --- a/pkg/local_object_storage/blobstor/mode.go +++ b/pkg/local_object_storage/blobstor/mode.go @@ -8,7 +8,7 @@ import ( ) // SetMode sets the blobstor mode of operation. -func (b *BlobStor) SetMode(m mode.Mode) error { +func (b *BlobStor) SetMode(ctx context.Context, m mode.Mode) error { b.modeMtx.Lock() defer b.modeMtx.Unlock() @@ -22,7 +22,7 @@ func (b *BlobStor) SetMode(m mode.Mode) error { err := b.Close() if err == nil { - if err = b.openBlobStor(context.TODO(), m); err == nil { + if err = b.openBlobStor(ctx, m); err == nil { err = b.Init() } } diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 98ec73ae9..a5c53dcad 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -56,7 +56,7 @@ func (e *StorageEngine) open(ctx context.Context) error { sh := e.shards[res.id] delete(e.shards, res.id) - err := sh.Close() + err := sh.Close(ctx) if err != nil { e.log.Error(ctx, logs.EngineCouldNotClosePartiallyInitializedShard, zap.String("id", res.id), @@ -108,7 +108,7 @@ func (e *StorageEngine) Init(ctx context.Context) error { sh := e.shards[res.id] delete(e.shards, res.id) - err := sh.Close() + err := sh.Close(ctx) if err != nil { e.log.Error(ctx, logs.EngineCouldNotClosePartiallyInitializedShard, zap.String("id", res.id), @@ -126,7 +126,7 @@ func (e *StorageEngine) Init(ctx context.Context) error { } e.wg.Add(1) - go e.setModeLoop() + go e.setModeLoop(ctx) return nil } @@ -153,7 +153,7 @@ func (e *StorageEngine) Close(ctx context.Context) error { } // closes all shards. Never returns an error, shard errors are logged. -func (e *StorageEngine) close(releasePools bool) error { +func (e *StorageEngine) close(ctx context.Context, releasePools bool) error { e.mtx.RLock() defer e.mtx.RUnlock() @@ -164,8 +164,8 @@ func (e *StorageEngine) close(releasePools bool) error { } for id, sh := range e.shards { - if err := sh.Close(); err != nil { - e.log.Debug(context.Background(), logs.EngineCouldNotCloseShard, + if err := sh.Close(ctx); err != nil { + e.log.Debug(ctx, logs.EngineCouldNotCloseShard, zap.String("id", id), zap.String("error", err.Error()), ) @@ -213,7 +213,7 @@ func (e *StorageEngine) setBlockExecErr(ctx context.Context, err error) error { return e.open(ctx) } } else if prevErr == nil { // ok -> block - return e.close(errors.Is(err, errClosed)) + return e.close(ctx, errors.Is(err, errClosed)) } // otherwise do nothing @@ -306,7 +306,7 @@ loop: e.mtx.RUnlock() - e.removeShards(shardsToRemove...) + e.removeShards(ctx, shardsToRemove...) for _, p := range shardsToReload { err := p.sh.Reload(ctx, p.opts...) @@ -330,13 +330,13 @@ loop: err = sh.Init(ctx) } if err != nil { - _ = sh.Close() + _ = sh.Close(ctx) return fmt.Errorf("could not init %s shard: %w", idStr, err) } err = e.addShard(sh) if err != nil { - _ = sh.Close() + _ = sh.Close(ctx) return fmt.Errorf("could not add %s shard: %w", idStr, err) } diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index a8caa215a..6e30ee9de 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -55,7 +55,7 @@ type setModeRequest struct { // setModeLoop listens setModeCh to perform degraded mode transition of a single shard. // Instead of creating a worker per single shard we use a single goroutine. -func (e *StorageEngine) setModeLoop() { +func (e *StorageEngine) setModeLoop(ctx context.Context) { defer e.wg.Done() var ( @@ -75,7 +75,7 @@ func (e *StorageEngine) setModeLoop() { if !ok { inProgress[sid] = struct{}{} go func() { - e.moveToDegraded(r.sh, r.errorCount, r.isMeta) + e.moveToDegraded(ctx, r.sh, r.errorCount, r.isMeta) mtx.Lock() delete(inProgress, sid) @@ -87,7 +87,7 @@ func (e *StorageEngine) setModeLoop() { } } -func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) { +func (e *StorageEngine) moveToDegraded(ctx context.Context, sh *shard.Shard, errCount uint32, isMeta bool) { sid := sh.ID() log := e.log.With( zap.Stringer("shard_id", sid), @@ -97,7 +97,7 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta defer e.mtx.RUnlock() if isMeta { - err := sh.SetMode(mode.DegradedReadOnly) + err := sh.SetMode(ctx, mode.DegradedReadOnly) if err == nil { log.Info(context.Background(), logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) return @@ -106,7 +106,7 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta zap.Error(err)) } - err := sh.SetMode(mode.ReadOnly) + err := sh.SetMode(ctx, mode.ReadOnly) if err != nil { log.Error(context.Background(), logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) return diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 1619003a1..57c423764 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -158,10 +158,10 @@ func TestErrorReporting(t *testing.T) { checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite) } - require.NoError(t, te.ng.SetShardMode(te.shards[0].id, mode.ReadWrite, false)) + require.NoError(t, te.ng.SetShardMode(context.Background(), te.shards[0].id, mode.ReadWrite, false)) checkShardState(t, te.ng, te.shards[0].id, errThreshold+1, mode.ReadWrite) - require.NoError(t, te.ng.SetShardMode(te.shards[0].id, mode.ReadWrite, true)) + require.NoError(t, te.ng.SetShardMode(context.Background(), te.shards[0].id, mode.ReadWrite, true)) checkShardState(t, te.ng, te.shards[0].id, 0, mode.ReadWrite) require.NoError(t, te.ng.Close(context.Background())) }) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 8498c9245..54eacc3f2 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -146,7 +146,7 @@ func TestEvacuateShardObjects(t *testing.T) { require.Equal(t, uint64(0), res.ObjectsEvacuated()) }) - require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[evacuateShardID].SetMode(context.Background(), mode.ReadOnly)) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) @@ -237,7 +237,7 @@ func TestEvacuateObjectsNetwork(t *testing.T) { evacuateShardID := ids[0].String() - require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[evacuateShardID].SetMode(context.Background(), mode.ReadOnly)) var prm EvacuateShardPrm prm.ShardID = ids[0:1] @@ -260,8 +260,8 @@ func TestEvacuateObjectsNetwork(t *testing.T) { require.NoError(t, e.Close(context.Background())) }() - require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) - require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(context.Background(), mode.ReadOnly)) var prm EvacuateShardPrm prm.ShardID = ids[1:2] @@ -298,7 +298,7 @@ func TestEvacuateObjectsNetwork(t *testing.T) { } for i := range ids { - require.NoError(t, e.shards[ids[i].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[i].String()].SetMode(context.Background(), mode.ReadOnly)) } var prm EvacuateShardPrm @@ -327,8 +327,8 @@ func TestEvacuateCancellation(t *testing.T) { require.NoError(t, e.Close(context.Background())) }() - require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) - require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(context.Background(), mode.ReadOnly)) var prm EvacuateShardPrm prm.ShardID = ids[1:2] @@ -357,8 +357,8 @@ func TestEvacuateCancellationByError(t *testing.T) { require.NoError(t, e.Close(context.Background())) }() - require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) - require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(context.Background(), mode.ReadOnly)) var prm EvacuateShardPrm prm.ShardID = ids[1:2] @@ -386,8 +386,8 @@ func TestEvacuateSingleProcess(t *testing.T) { require.NoError(t, e.Close(context.Background())) }() - require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) - require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(context.Background(), mode.ReadOnly)) blocker := make(chan interface{}) running := make(chan interface{}) @@ -429,8 +429,8 @@ func TestEvacuateObjectsAsync(t *testing.T) { require.NoError(t, e.Close(context.Background())) }() - require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) - require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(context.Background(), mode.ReadOnly)) blocker := make(chan interface{}) running := make(chan interface{}) @@ -515,7 +515,7 @@ func TestEvacuateTreesLocal(t *testing.T) { require.NoError(t, e.Close(context.Background())) }() - require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) var prm EvacuateShardPrm prm.ShardID = ids[0:1] @@ -594,8 +594,8 @@ func TestEvacuateTreesRemote(t *testing.T) { require.NoError(t, e.Close(context.Background())) }() - require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) - require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) + require.NoError(t, e.shards[ids[1].String()].SetMode(context.Background(), mode.ReadOnly)) mutex := sync.Mutex{} evacuatedTreeOps := make(map[string][]*pilorama.Move) @@ -753,7 +753,7 @@ func TestEvacuateShardObjectsRepOneOnly(t *testing.T) { prm.Scope = EvacuateScopeObjects prm.RepOneOnly = true - require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) @@ -810,7 +810,7 @@ func TestEvacuateShardObjectsRepOneOnlyBench(t *testing.T) { prm.RepOneOnly = true prm.ContainerWorkerCount = 10 - require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly)) + require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) start := time.Now() _, err := e.Evacuate(context.Background(), prm) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index e172706e3..2b94103e9 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -121,7 +121,7 @@ func (e *StorageEngine) AddShard(ctx context.Context, opts ...shard.Option) (*sh return sh.ID(), nil } -func (e *StorageEngine) createShard(_ context.Context, opts []shard.Option) (*shard.Shard, error) { +func (e *StorageEngine) createShard(ctx context.Context, opts []shard.Option) (*shard.Shard, error) { id, err := generateShardID() if err != nil { return nil, fmt.Errorf("could not generate shard ID: %w", err) @@ -139,8 +139,8 @@ func (e *StorageEngine) createShard(_ context.Context, opts []shard.Option) (*sh shard.WithZeroCountCallback(e.processZeroCountContainers), )...) - if err := sh.UpdateID(); err != nil { - e.log.Warn(context.Background(), logs.FailedToUpdateShardID, zap.Stringer("shard_id", sh.ID()), zap.String("metabase_path", sh.DumpInfo().MetaBaseInfo.Path), zap.Error(err)) + if err := sh.UpdateID(ctx); err != nil { + e.log.Warn(ctx, logs.FailedToUpdateShardID, zap.Stringer("shard_id", sh.ID()), zap.String("metabase_path", sh.DumpInfo().MetaBaseInfo.Path), zap.Error(err)) } return sh, nil @@ -203,7 +203,7 @@ func (e *StorageEngine) addShard(sh *shard.Shard) error { // removeShards removes specified shards. Skips non-existent shards. // Logs errors about shards that it could not Close after the removal. -func (e *StorageEngine) removeShards(ids ...string) { +func (e *StorageEngine) removeShards(ctx context.Context, ids ...string) { if len(ids) == 0 { return } @@ -228,22 +228,22 @@ func (e *StorageEngine) removeShards(ids ...string) { delete(e.shardPools, id) } - e.log.Info(context.Background(), logs.EngineShardHasBeenRemoved, + e.log.Info(ctx, logs.EngineShardHasBeenRemoved, zap.String("id", id)) } e.mtx.Unlock() for _, sh := range ss { - err := sh.SetMode(mode.Disabled) + err := sh.SetMode(ctx, mode.Disabled) if err != nil { - e.log.Error(context.Background(), logs.EngineCouldNotChangeShardModeToDisabled, + e.log.Error(ctx, logs.EngineCouldNotChangeShardModeToDisabled, zap.Stringer("id", sh.ID()), zap.Error(err), ) } - err = sh.Close() + err = sh.Close(ctx) if err != nil { - e.log.Error(context.Background(), logs.EngineCouldNotCloseRemovedShard, + e.log.Error(ctx, logs.EngineCouldNotCloseRemovedShard, zap.Stringer("id", sh.ID()), zap.Error(err), ) @@ -310,7 +310,7 @@ func (e *StorageEngine) iterateOverUnsortedShards(handler func(hashedShard) (sto // SetShardMode sets mode of the shard with provided identifier. // // Returns an error if shard mode was not set, or shard was not found in storage engine. -func (e *StorageEngine) SetShardMode(id *shard.ID, m mode.Mode, resetErrorCounter bool) error { +func (e *StorageEngine) SetShardMode(ctx context.Context, id *shard.ID, m mode.Mode, resetErrorCounter bool) error { e.mtx.RLock() defer e.mtx.RUnlock() @@ -320,7 +320,7 @@ func (e *StorageEngine) SetShardMode(id *shard.ID, m mode.Mode, resetErrorCounte sh.errorCount.Store(0) e.metrics.ClearErrorCounter(shID) } - return sh.SetMode(m) + return sh.SetMode(ctx, m) } } @@ -346,7 +346,7 @@ func (e *StorageEngine) HandleNewEpoch(ctx context.Context, epoch uint64) { } } -func (e *StorageEngine) DetachShards(ids []*shard.ID) error { +func (e *StorageEngine) DetachShards(ctx context.Context, ids []*shard.ID) error { if len(ids) == 0 { return logicerr.New("ids must be non-empty") } @@ -356,20 +356,20 @@ func (e *StorageEngine) DetachShards(ids []*shard.ID) error { return err } - return e.closeShards(deletedShards) + return e.closeShards(ctx, deletedShards) } // closeShards closes deleted shards. Tries to close all shards. // Returns single error with joined shard errors. -func (e *StorageEngine) closeShards(deletedShards []hashedShard) error { +func (e *StorageEngine) closeShards(ctx context.Context, deletedShards []hashedShard) error { var multiErr error var multiErrGuard sync.Mutex var eg errgroup.Group for _, sh := range deletedShards { eg.Go(func() error { - err := sh.SetMode(mode.Disabled) + err := sh.SetMode(ctx, mode.Disabled) if err != nil { - e.log.Error(context.Background(), logs.EngineCouldNotChangeShardModeToDisabled, + e.log.Error(ctx, logs.EngineCouldNotChangeShardModeToDisabled, zap.Stringer("id", sh.ID()), zap.Error(err), ) @@ -378,9 +378,9 @@ func (e *StorageEngine) closeShards(deletedShards []hashedShard) error { multiErrGuard.Unlock() } - err = sh.Close() + err = sh.Close(ctx) if err != nil { - e.log.Error(context.Background(), logs.EngineCouldNotCloseRemovedShard, + e.log.Error(ctx, logs.EngineCouldNotCloseRemovedShard, zap.Stringer("id", sh.ID()), zap.Error(err), ) diff --git a/pkg/local_object_storage/engine/shards_test.go b/pkg/local_object_storage/engine/shards_test.go index 3347d58f1..207491bd4 100644 --- a/pkg/local_object_storage/engine/shards_test.go +++ b/pkg/local_object_storage/engine/shards_test.go @@ -33,7 +33,7 @@ func TestRemoveShard(t *testing.T) { for id, remove := range mSh { if remove { - e.removeShards(id) + e.removeShards(context.Background(), id) } } @@ -55,11 +55,11 @@ func TestDisableShards(t *testing.T) { e, ids := te.engine, te.shardIDs defer func() { require.NoError(t, e.Close(context.Background())) }() - require.ErrorAs(t, e.DetachShards(ids), new(logicerr.Logical)) - require.ErrorAs(t, e.DetachShards(nil), new(logicerr.Logical)) - require.ErrorAs(t, e.DetachShards([]*shard.ID{}), new(logicerr.Logical)) + require.ErrorAs(t, e.DetachShards(context.Background(), ids), new(logicerr.Logical)) + require.ErrorAs(t, e.DetachShards(context.Background(), nil), new(logicerr.Logical)) + require.ErrorAs(t, e.DetachShards(context.Background(), []*shard.ID{}), new(logicerr.Logical)) - require.NoError(t, e.DetachShards([]*shard.ID{ids[0]})) + require.NoError(t, e.DetachShards(context.Background(), []*shard.ID{ids[0]})) require.Equal(t, 1, len(e.shards)) } diff --git a/pkg/local_object_storage/internal/storagetest/storage.go b/pkg/local_object_storage/internal/storagetest/storage.go index 586b3dcc6..d8ac106dd 100644 --- a/pkg/local_object_storage/internal/storagetest/storage.go +++ b/pkg/local_object_storage/internal/storagetest/storage.go @@ -11,7 +11,7 @@ import ( // Component represents single storage component. type Component interface { Open(context.Context, mode.Mode) error - SetMode(mode.Mode) error + SetMode(context.Context, mode.Mode) error Init() error Close() error } @@ -91,12 +91,12 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { // call `SetMode` on all not-yet-initialized components. s := cons(t) require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, s.SetMode(m)) + require.NoError(t, s.SetMode(context.Background(), m)) t.Run("after open in RO", func(t *testing.T) { require.NoError(t, s.Close()) require.NoError(t, s.Open(context.Background(), mode.ReadOnly)) - require.NoError(t, s.SetMode(m)) + require.NoError(t, s.SetMode(context.Background(), m)) }) require.NoError(t, s.Close()) @@ -106,7 +106,7 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { // Use-case: notmal node operation. require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Init()) - require.NoError(t, s.SetMode(m)) + require.NoError(t, s.SetMode(context.Background(), m)) require.NoError(t, s.Close()) }) } @@ -116,7 +116,7 @@ func TestModeTransition(t *testing.T, cons Constructor, from, to mode.Mode) { s := cons(t) require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) require.NoError(t, s.Init()) - require.NoError(t, s.SetMode(from)) - require.NoError(t, s.SetMode(to)) + require.NoError(t, s.SetMode(context.Background(), from)) + require.NoError(t, s.SetMode(context.Background(), to)) require.NoError(t, s.Close()) } diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 68e065a0a..54bea4204 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -39,7 +39,7 @@ var ( ) // Open boltDB instance for metabase. -func (db *DB) Open(_ context.Context, m mode.Mode) error { +func (db *DB) Open(ctx context.Context, m mode.Mode) error { db.modeMtx.Lock() defer db.modeMtx.Unlock() db.mode = m @@ -48,10 +48,10 @@ func (db *DB) Open(_ context.Context, m mode.Mode) error { if m.NoMetabase() { return nil } - return db.openDB(m) + return db.openDB(ctx, m) } -func (db *DB) openDB(mode mode.Mode) error { +func (db *DB) openDB(ctx context.Context, mode mode.Mode) error { err := util.MkdirAllX(filepath.Dir(db.info.Path), db.info.Permission) if err != nil { return fmt.Errorf("can't create dir %s for metabase: %w", db.info.Path, err) @@ -65,10 +65,10 @@ func (db *DB) openDB(mode mode.Mode) error { } db.boltOptions.ReadOnly = mode.ReadOnly() - return metaerr.Wrap(db.openBolt()) + return metaerr.Wrap(db.openBolt(ctx)) } -func (db *DB) openBolt() error { +func (db *DB) openBolt(ctx context.Context) error { var err error db.boltDB, err = bbolt.Open(db.info.Path, db.info.Permission, db.boltOptions) @@ -226,7 +226,7 @@ func (db *DB) close() error { // If there was a problem with applying new configuration, an error is returned. // // If a metabase was couldn't be reopened because of an error, ErrDegradedMode is returned. -func (db *DB) Reload(opts ...Option) (bool, error) { +func (db *DB) Reload(ctx context.Context, opts ...Option) (bool, error) { var c cfg for i := range opts { opts[i](&c) @@ -243,7 +243,7 @@ func (db *DB) Reload(opts ...Option) (bool, error) { db.mode = mode.Disabled db.metrics.SetMode(mode.ComponentDisabled) db.info.Path = c.info.Path - if err := db.openBolt(); err != nil { + if err := db.openBolt(ctx); err != nil { return false, metaerr.Wrap(fmt.Errorf("%w: %v", ErrDegradedMode, err)) } diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index 2032ed6b2..f99262be4 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -1,6 +1,7 @@ package meta import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -8,7 +9,7 @@ import ( // SetMode sets the metabase mode of operation. // If the mode assumes no operation metabase, the database is closed. -func (db *DB) SetMode(m mode.Mode) error { +func (db *DB) SetMode(ctx context.Context, m mode.Mode) error { db.modeMtx.Lock() defer db.modeMtx.Unlock() @@ -25,7 +26,7 @@ func (db *DB) SetMode(m mode.Mode) error { if m.NoMetabase() { db.boltDB = nil } else { - err := db.openDB(m) + err := db.openDB(ctx, m) if err == nil && !m.ReadOnly() { err = db.Init() } diff --git a/pkg/local_object_storage/metabase/shard_id.go b/pkg/local_object_storage/metabase/shard_id.go index 88446494e..e58115bc8 100644 --- a/pkg/local_object_storage/metabase/shard_id.go +++ b/pkg/local_object_storage/metabase/shard_id.go @@ -2,6 +2,7 @@ package meta import ( "bytes" + "context" "errors" "fmt" "os" @@ -21,7 +22,7 @@ var ( // If id is missing, returns nil, nil. // // GetShardID does not report any metrics. -func (db *DB) GetShardID(mode metamode.Mode) ([]byte, error) { +func (db *DB) GetShardID(ctx context.Context, mode metamode.Mode) ([]byte, error) { db.modeMtx.Lock() defer db.modeMtx.Unlock() db.mode = mode @@ -30,7 +31,7 @@ func (db *DB) GetShardID(mode metamode.Mode) ([]byte, error) { return nil, nil } - if err := db.openDB(mode); err != nil { + if err := db.openDB(ctx, mode); err != nil { return nil, fmt.Errorf("failed to open metabase: %w", err) } @@ -59,7 +60,7 @@ func (db *DB) readShardID() ([]byte, error) { // SetShardID sets metabase operation mode // and writes shard id to db. -func (db *DB) SetShardID(id []byte, mode metamode.Mode) error { +func (db *DB) SetShardID(ctx context.Context, id []byte, mode metamode.Mode) error { db.modeMtx.Lock() defer db.modeMtx.Unlock() db.mode = mode @@ -68,7 +69,7 @@ func (db *DB) SetShardID(id []byte, mode metamode.Mode) error { return ErrReadOnlyMode } - if err := db.openDB(mode); err != nil { + if err := db.openDB(ctx, mode); err != nil { return fmt.Errorf("failed to open metabase: %w", err) } diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index 75229a1b4..509e72479 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -58,7 +58,7 @@ func TestVersion(t *testing.T) { }) t.Run("old data", func(t *testing.T) { db := newDB(t) - require.NoError(t, db.SetShardID([]byte{1, 2, 3, 4}, mode.ReadWrite)) + require.NoError(t, db.SetShardID(context.Background(), []byte{1, 2, 3, 4}, mode.ReadWrite)) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Init()) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index a778434dd..9ffcf1e83 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -91,7 +91,7 @@ func NewBoltForest(opts ...Option) ForestStorage { return &b } -func (t *boltForest) SetMode(m mode.Mode) error { +func (t *boltForest) SetMode(_ context.Context, m mode.Mode) error { t.modeMtx.Lock() defer t.modeMtx.Unlock() diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 374943745..76da1c0c2 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -128,7 +128,7 @@ func (f *memoryForest) Open(context.Context, mode.Mode) error { return nil } -func (f *memoryForest) SetMode(mode.Mode) error { +func (f *memoryForest) SetMode(context.Context, mode.Mode) error { return nil } diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index b6ca246f2..9717b2401 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -65,7 +65,7 @@ type ForestStorage interface { Init() error Open(context.Context, mode.Mode) error Close() error - SetMode(m mode.Mode) error + SetMode(context.Context, mode.Mode) error SetParentID(id string) Forest diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index eb3aa61c0..056737a9d 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -20,23 +20,23 @@ import ( "golang.org/x/sync/errgroup" ) -func (s *Shard) handleMetabaseFailure(stage string, err error) error { - s.log.Error(context.Background(), logs.ShardMetabaseFailureSwitchingMode, +func (s *Shard) handleMetabaseFailure(ctx context.Context, stage string, err error) error { + s.log.Error(ctx, logs.ShardMetabaseFailureSwitchingMode, zap.String("stage", stage), zap.Stringer("mode", mode.ReadOnly), zap.Error(err)) - err = s.SetMode(mode.ReadOnly) + err = s.SetMode(ctx, mode.ReadOnly) if err == nil { return nil } - s.log.Error(context.Background(), logs.ShardCantMoveShardToReadonlySwitchMode, + s.log.Error(ctx, logs.ShardCantMoveShardToReadonlySwitchMode, zap.String("stage", stage), zap.Stringer("mode", mode.DegradedReadOnly), zap.Error(err)) - err = s.SetMode(mode.DegradedReadOnly) + err = s.SetMode(ctx, mode.DegradedReadOnly) if err != nil { return fmt.Errorf("could not switch to mode %s", mode.Mode(mode.DegradedReadOnly)) } @@ -75,7 +75,7 @@ func (s *Shard) Open(ctx context.Context) error { return fmt.Errorf("could not open %T: %w", components[j], err) } } - err = s.handleMetabaseFailure("open", err) + err = s.handleMetabaseFailure(ctx, "open", err) if err != nil { return err } @@ -101,7 +101,7 @@ func (x *metabaseSynchronizer) Init() error { // Init initializes all Shard's components. func (s *Shard) Init(ctx context.Context) error { m := s.GetMode() - if err := s.initializeComponents(m); err != nil { + if err := s.initializeComponents(ctx, m); err != nil { return err } @@ -138,7 +138,7 @@ func (s *Shard) Init(ctx context.Context) error { return nil } -func (s *Shard) initializeComponents(m mode.Mode) error { +func (s *Shard) initializeComponents(ctx context.Context, m mode.Mode) error { type initializer interface { Init() error } @@ -176,7 +176,7 @@ func (s *Shard) initializeComponents(m mode.Mode) error { return fmt.Errorf("metabase initialization: %w", err) } - err = s.handleMetabaseFailure("init", err) + err = s.handleMetabaseFailure(ctx, "init", err) if err != nil { return err } @@ -364,9 +364,9 @@ func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object } // Close releases all Shard's components. -func (s *Shard) Close() error { +func (s *Shard) Close(ctx context.Context) error { if s.rb != nil { - s.rb.Stop(s.log) + s.rb.Stop(ctx, s.log) } var components []interface{ Close() error } @@ -386,7 +386,7 @@ func (s *Shard) Close() error { for _, component := range components { if err := component.Close(); err != nil { lastErr = err - s.log.Error(context.Background(), logs.ShardCouldNotCloseShardComponent, zap.Error(err)) + s.log.Error(ctx, logs.ShardCouldNotCloseShardComponent, zap.Error(err)) } } @@ -414,18 +414,18 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { unlock := s.lockExclusive() defer unlock() - s.rb.Stop(s.log) + s.rb.Stop(ctx, s.log) if !s.info.Mode.NoMetabase() { defer func() { s.rb.Start(ctx, s.blobStor, s.metaBase, s.log) }() } - ok, err := s.metaBase.Reload(c.metaOpts...) + ok, err := s.metaBase.Reload(ctx, c.metaOpts...) if err != nil { if errors.Is(err, meta.ErrDegradedMode) { s.log.Error(ctx, logs.ShardCantOpenMetabaseMoveToADegradedMode, zap.Error(err)) - _ = s.setMode(mode.DegradedReadOnly) + _ = s.setMode(ctx, mode.DegradedReadOnly) } return err } @@ -441,11 +441,11 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { } if err != nil { s.log.Error(ctx, logs.ShardCantInitializeMetabaseMoveToADegradedreadonlyMode, zap.Error(err)) - _ = s.setMode(mode.DegradedReadOnly) + _ = s.setMode(ctx, mode.DegradedReadOnly) return err } } - return s.setMode(c.info.Mode) + return s.setMode(ctx, c.info.Mode) } func (s *Shard) lockExclusive() func() { diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index b8f1d4417..6d2cd7137 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -86,7 +86,7 @@ func TestShardOpen(t *testing.T) { require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) require.Equal(t, mode.ReadWrite, sh.GetMode()) - require.NoError(t, sh.Close()) + require.NoError(t, sh.Close(context.Background())) // Metabase can be opened in read-only => start in ReadOnly mode. allowedMode.Store(int64(os.O_RDONLY)) @@ -95,9 +95,9 @@ func TestShardOpen(t *testing.T) { require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) require.Equal(t, mode.ReadOnly, sh.GetMode()) - require.Error(t, sh.SetMode(mode.ReadWrite)) + require.Error(t, sh.SetMode(context.Background(), mode.ReadWrite)) require.Equal(t, mode.ReadOnly, sh.GetMode()) - require.NoError(t, sh.Close()) + require.NoError(t, sh.Close(context.Background())) // Metabase is corrupted => start in DegradedReadOnly mode. allowedMode.Store(math.MaxInt64) @@ -106,7 +106,7 @@ func TestShardOpen(t *testing.T) { require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) require.Equal(t, mode.DegradedReadOnly, sh.GetMode()) - require.NoError(t, sh.Close()) + require.NoError(t, sh.Close(context.Background())) } func TestRefillMetabaseCorrupted(t *testing.T) { @@ -146,7 +146,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) { putPrm.SetObject(obj) _, err := sh.Put(context.Background(), putPrm) require.NoError(t, err) - require.NoError(t, sh.Close()) + require.NoError(t, sh.Close(context.Background())) addr := object.AddressOf(obj) // This is copied from `fstree.treePath()` to avoid exporting function just for tests. @@ -170,7 +170,7 @@ func TestRefillMetabaseCorrupted(t *testing.T) { getPrm.SetAddress(addr) _, err = sh.Get(context.Background(), getPrm) require.True(t, client.IsErrObjectNotFound(err)) - require.NoError(t, sh.Close()) + require.NoError(t, sh.Close(context.Background())) } func TestRefillMetabase(t *testing.T) { @@ -358,7 +358,7 @@ func TestRefillMetabase(t *testing.T) { phyBefore := c.Phy logicalBefore := c.Logic - err = sh.Close() + err = sh.Close(context.Background()) require.NoError(t, err) sh = New( @@ -379,7 +379,7 @@ func TestRefillMetabase(t *testing.T) { // initialize Blobstor require.NoError(t, sh.Init(context.Background())) - defer sh.Close() + defer sh.Close(context.Background()) checkAllObjs(false) checkObj(object.AddressOf(tombObj), nil) diff --git a/pkg/local_object_storage/shard/delete_test.go b/pkg/local_object_storage/shard/delete_test.go index 574250a93..c9ce93bc5 100644 --- a/pkg/local_object_storage/shard/delete_test.go +++ b/pkg/local_object_storage/shard/delete_test.go @@ -37,7 +37,7 @@ func TestShard_Delete_BigObject(t *testing.T) { func testShard(t *testing.T, hasWriteCache bool, payloadSize int) { sh := newShard(t, hasWriteCache) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() cnr := cidtest.ID() diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index 11db5e54e..39073a529 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -79,7 +79,7 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { sh = New(opts...) require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() cnr := cidtest.ID() obj := testutil.GenerateObjectWithCID(cnr) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 2b97111e7..e3670b441 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -34,7 +34,7 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { return util.NewPseudoWorkerPool() // synchronous event processing })}, }) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() cnr := cidtest.ID() @@ -131,7 +131,7 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { return util.NewPseudoWorkerPool() // synchronous event processing })}, }) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() lock := testutil.GenerateObjectWithCID(cnr) lock.SetType(objectSDK.TypeLock) @@ -190,7 +190,7 @@ func testGCDropsObjectInhumedFromWritecache(t *testing.T, flushbeforeInhume bool additionalShardOptions: []Option{WithDisabledGC()}, wcOpts: []writecache.Option{writecache.WithDisableBackgroundFlush()}, }) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() obj := testutil.GenerateObjectWithSize(1024) @@ -254,7 +254,7 @@ func TestGCDontDeleteObjectFromWritecache(t *testing.T) { additionalShardOptions: []Option{WithDisabledGC()}, wcOpts: []writecache.Option{writecache.WithDisableBackgroundFlush()}, }) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() obj := testutil.GenerateObjectWithSize(1024) diff --git a/pkg/local_object_storage/shard/get_test.go b/pkg/local_object_storage/shard/get_test.go index d0eecf74e..837991b73 100644 --- a/pkg/local_object_storage/shard/get_test.go +++ b/pkg/local_object_storage/shard/get_test.go @@ -30,7 +30,7 @@ func TestShard_Get(t *testing.T) { func testShardGet(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() var putPrm PutPrm var getPrm GetPrm diff --git a/pkg/local_object_storage/shard/head_test.go b/pkg/local_object_storage/shard/head_test.go index c65bbb1e3..deb3019df 100644 --- a/pkg/local_object_storage/shard/head_test.go +++ b/pkg/local_object_storage/shard/head_test.go @@ -28,7 +28,7 @@ func TestShard_Head(t *testing.T) { func testShardHead(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() var putPrm PutPrm var headPrm HeadPrm diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index e27dc0733..6ccae3f53 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -1,6 +1,7 @@ package shard import ( + "context" "errors" "fmt" @@ -30,11 +31,11 @@ func (s *Shard) ID() *ID { } // UpdateID reads shard ID saved in the metabase and updates it if it is missing. -func (s *Shard) UpdateID() (err error) { +func (s *Shard) UpdateID(ctx context.Context) (err error) { var idFromMetabase []byte modeDegraded := s.GetMode().NoMetabase() if !modeDegraded { - if idFromMetabase, err = s.metaBase.GetShardID(mode.ReadOnly); err != nil { + if idFromMetabase, err = s.metaBase.GetShardID(ctx, mode.ReadOnly); err != nil { err = fmt.Errorf("failed to read shard id from metabase: %w", err) } } @@ -62,7 +63,7 @@ func (s *Shard) UpdateID() (err error) { } if len(idFromMetabase) == 0 && !modeDegraded { - if setErr := s.metaBase.SetShardID(*s.info.ID, s.GetMode()); setErr != nil { + if setErr := s.metaBase.SetShardID(ctx, *s.info.ID, s.GetMode()); setErr != nil { err = errors.Join(err, fmt.Errorf("failed to write shard id to metabase: %w", setErr)) } } diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 1353d5d94..1421f0e18 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -27,7 +27,7 @@ func TestShard_Inhume(t *testing.T) { func testShardInhume(t *testing.T, hasWriteCache bool) { sh := newShard(t, hasWriteCache) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() cnr := cidtest.ID() diff --git a/pkg/local_object_storage/shard/list_test.go b/pkg/local_object_storage/shard/list_test.go index 3414dc76a..139b2e316 100644 --- a/pkg/local_object_storage/shard/list_test.go +++ b/pkg/local_object_storage/shard/list_test.go @@ -18,14 +18,14 @@ func TestShard_List(t *testing.T) { t.Run("without write cache", func(t *testing.T) { t.Parallel() sh := newShard(t, false) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() testShardList(t, sh) }) t.Run("with write cache", func(t *testing.T) { t.Parallel() shWC := newShard(t, true) - defer func() { require.NoError(t, shWC.Close()) }() + defer func() { require.NoError(t, shWC.Close(context.Background())) }() testShardList(t, shWC) }) } diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 595afb60e..7da8b8c28 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -62,7 +62,7 @@ func TestShard_Lock(t *testing.T) { require.NoError(t, sh.Open(context.Background())) require.NoError(t, sh.Init(context.Background())) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() cnr := cidtest.ID() obj := testutil.GenerateObjectWithCID(cnr) @@ -148,7 +148,7 @@ func TestShard_Lock(t *testing.T) { func TestShard_IsLocked(t *testing.T) { sh := newShard(t, false) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() cnr := cidtest.ID() obj := testutil.GenerateObjectWithCID(cnr) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index cec5a12ad..5230dcad0 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -201,11 +201,11 @@ func TestCounters(t *testing.T) { dir := t.TempDir() sh, mm := shardWithMetrics(t, dir) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() - sh.SetMode(mode.ReadOnly) + sh.SetMode(context.Background(), mode.ReadOnly) require.Equal(t, mode.ReadOnly, mm.mode) - sh.SetMode(mode.ReadWrite) + sh.SetMode(context.Background(), mode.ReadWrite) require.Equal(t, mode.ReadWrite, mm.mode) const objNumber = 10 diff --git a/pkg/local_object_storage/shard/mode.go b/pkg/local_object_storage/shard/mode.go index 98b4c37b2..901528976 100644 --- a/pkg/local_object_storage/shard/mode.go +++ b/pkg/local_object_storage/shard/mode.go @@ -20,19 +20,21 @@ var ErrDegradedMode = logicerr.New("shard is in degraded mode") // // Returns any error encountered that did not allow // setting shard mode. -func (s *Shard) SetMode(m mode.Mode) error { +func (s *Shard) SetMode(ctx context.Context, m mode.Mode) error { unlock := s.lockExclusive() defer unlock() - return s.setMode(m) + return s.setMode(ctx, m) } -func (s *Shard) setMode(m mode.Mode) error { - s.log.Info(context.Background(), logs.ShardSettingShardMode, +func (s *Shard) setMode(ctx context.Context, m mode.Mode) error { + s.log.Info(ctx, logs.ShardSettingShardMode, zap.Stringer("old_mode", s.info.Mode), zap.Stringer("new_mode", m)) - components := []interface{ SetMode(mode.Mode) error }{ + components := []interface { + SetMode(context.Context, mode.Mode) error + }{ s.metaBase, s.blobStor, } @@ -60,7 +62,7 @@ func (s *Shard) setMode(m mode.Mode) error { if !m.Disabled() { for i := range components { - if err := components[i].SetMode(m); err != nil { + if err := components[i].SetMode(ctx, m); err != nil { return err } } @@ -69,7 +71,7 @@ func (s *Shard) setMode(m mode.Mode) error { s.info.Mode = m s.metricsWriter.SetMode(s.info.Mode) - s.log.Info(context.Background(), logs.ShardShardModeSetSuccessfully, + s.log.Info(ctx, logs.ShardShardModeSetSuccessfully, zap.Stringer("mode", s.info.Mode)) return nil } diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index cc73db316..146e834cc 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -94,7 +94,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { }), }, }) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/pkg/local_object_storage/shard/rebuild.go b/pkg/local_object_storage/shard/rebuild.go index 124b72a5c..10eb51a28 100644 --- a/pkg/local_object_storage/shard/rebuild.go +++ b/pkg/local_object_storage/shard/rebuild.go @@ -125,7 +125,7 @@ func (r *rebuilder) ScheduleRebuild(ctx context.Context, limiter RebuildWorkerLi } } -func (r *rebuilder) Stop(log *logger.Logger) { +func (r *rebuilder) Stop(ctx context.Context, log *logger.Logger) { r.mtx.Lock() defer r.mtx.Unlock() @@ -138,7 +138,7 @@ func (r *rebuilder) Stop(log *logger.Logger) { r.wg.Wait() r.cancel = nil r.done = nil - log.Info(context.Background(), logs.BlobstoreRebuildStopped) + log.Info(ctx, logs.BlobstoreRebuildStopped) } var errMBIsNotAvailable = errors.New("metabase is not available") diff --git a/pkg/local_object_storage/shard/refill_test.go b/pkg/local_object_storage/shard/refill_test.go index 0025bb45a..d90343265 100644 --- a/pkg/local_object_storage/shard/refill_test.go +++ b/pkg/local_object_storage/shard/refill_test.go @@ -34,7 +34,7 @@ func benchRefillMetabase(b *testing.B, objectsCount int) { additionalShardOptions: []Option{WithRefillMetabaseWorkersCount(shardconfig.RefillMetabaseWorkersCountDefault)}, }) - defer func() { require.NoError(b, sh.Close()) }() + defer func() { require.NoError(b, sh.Close(context.Background())) }() var putPrm PutPrm @@ -61,7 +61,7 @@ func benchRefillMetabase(b *testing.B, objectsCount int) { require.NoError(b, err) } - require.NoError(b, sh.Close()) + require.NoError(b, sh.Close(context.Background())) require.NoError(b, os.Remove(sh.metaBase.DumpInfo().Path)) require.NoError(b, sh.Open(context.Background())) @@ -72,5 +72,5 @@ func benchRefillMetabase(b *testing.B, objectsCount int) { require.NoError(b, sh.Init(context.Background())) - require.NoError(b, sh.Close()) + require.NoError(b, sh.Close(context.Background())) } diff --git a/pkg/local_object_storage/shard/reload_test.go b/pkg/local_object_storage/shard/reload_test.go index 7dd7189bb..e563f390b 100644 --- a/pkg/local_object_storage/shard/reload_test.go +++ b/pkg/local_object_storage/shard/reload_test.go @@ -59,7 +59,7 @@ func TestShardReload(t *testing.T) { require.NoError(t, sh.Init(context.Background())) defer func() { - require.NoError(t, sh.Close()) + require.NoError(t, sh.Close(context.Background())) }() objects := make([]objAddr, 5) diff --git a/pkg/local_object_storage/shard/shutdown_test.go b/pkg/local_object_storage/shard/shutdown_test.go index de00eabd1..b1232707f 100644 --- a/pkg/local_object_storage/shard/shutdown_test.go +++ b/pkg/local_object_storage/shard/shutdown_test.go @@ -52,10 +52,10 @@ func TestWriteCacheObjectLoss(t *testing.T) { }) } require.NoError(t, errG.Wait()) - require.NoError(t, sh.Close()) + require.NoError(t, sh.Close(context.Background())) sh = newCustomShard(t, true, shardOptions{rootPath: dir, wcOpts: wcOpts}) - defer func() { require.NoError(t, sh.Close()) }() + defer func() { require.NoError(t, sh.Close(context.Background())) }() var getPrm GetPrm diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 26f47e82e..92fb493e0 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -117,8 +117,8 @@ func runFlushTest[Option any]( defer func() { require.NoError(t, wc.Close()) }() objects := putObjects(t, wc) - require.NoError(t, bs.SetMode(mode.ReadWrite)) - require.NoError(t, mb.SetMode(mode.ReadWrite)) + require.NoError(t, bs.SetMode(context.Background(), mode.ReadWrite)) + require.NoError(t, mb.SetMode(context.Background(), mode.ReadWrite)) require.NoError(t, wc.Flush(context.Background(), false, false)) @@ -131,11 +131,11 @@ func runFlushTest[Option any]( objects := putObjects(t, wc) // Blobstor is read-only, so we expect en error from `flush` here. - require.Error(t, wc.SetMode(mode.Degraded)) + require.Error(t, wc.SetMode(context.Background(), mode.Degraded)) - require.NoError(t, bs.SetMode(mode.ReadWrite)) - require.NoError(t, mb.SetMode(mode.ReadWrite)) - require.NoError(t, wc.SetMode(mode.Degraded)) + require.NoError(t, bs.SetMode(context.Background(), mode.ReadWrite)) + require.NoError(t, mb.SetMode(context.Background(), mode.ReadWrite)) + require.NoError(t, wc.SetMode(context.Background(), mode.Degraded)) check(t, mb, bs, objects) }) @@ -149,8 +149,8 @@ func runFlushTest[Option any]( objects := putObjects(t, wc) f.InjectFn(t, wc) - require.NoError(t, bs.SetMode(mode.ReadWrite)) - require.NoError(t, mb.SetMode(mode.ReadWrite)) + require.NoError(t, bs.SetMode(context.Background(), mode.ReadWrite)) + require.NoError(t, mb.SetMode(context.Background(), mode.ReadWrite)) require.Equal(t, uint32(0), errCount.Load()) require.Error(t, wc.Flush(context.Background(), false, false)) @@ -191,8 +191,8 @@ func newCache[Option any]( require.NoError(t, wc.Init()) // First set mode for metabase and blobstor to prevent background flushes. - require.NoError(t, mb.SetMode(mode.ReadOnly)) - require.NoError(t, bs.SetMode(mode.ReadOnly)) + require.NoError(t, mb.SetMode(context.Background(), mode.ReadOnly)) + require.NoError(t, bs.SetMode(context.Background(), mode.ReadOnly)) return wc, bs, mb } diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 26658e9b8..db789d994 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -23,8 +23,8 @@ type setModePrm struct { // SetMode sets write-cache mode of operation. // When shard is put in read-only mode all objects in memory are flushed to disk // and all background jobs are suspended. -func (c *cache) SetMode(m mode.Mode) error { - ctx, span := tracing.StartSpanFromContext(context.TODO(), "writecache.SetMode", +func (c *cache) SetMode(ctx context.Context, m mode.Mode) error { + ctx, span := tracing.StartSpanFromContext(ctx, "writecache.SetMode", trace.WithAttributes( attribute.String("mode", m.String()), )) diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index a973df604..d07220b68 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -38,7 +38,7 @@ type Cache interface { // Returns ErrReadOnly if the Cache is currently in the read-only mode. Delete(context.Context, oid.Address) error Put(context.Context, common.PutPrm) (common.PutRes, error) - SetMode(mode.Mode) error + SetMode(context.Context, mode.Mode) error SetLogger(*logger.Logger) DumpInfo() Info Flush(context.Context, bool, bool) error diff --git a/pkg/services/control/server/detach_shards.go b/pkg/services/control/server/detach_shards.go index a4111bddb..ffd36962b 100644 --- a/pkg/services/control/server/detach_shards.go +++ b/pkg/services/control/server/detach_shards.go @@ -11,7 +11,7 @@ import ( "google.golang.org/grpc/status" ) -func (s *Server) DetachShards(_ context.Context, req *control.DetachShardsRequest) (*control.DetachShardsResponse, error) { +func (s *Server) DetachShards(ctx context.Context, req *control.DetachShardsRequest) (*control.DetachShardsResponse, error) { err := s.isValidRequest(req) if err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) @@ -19,7 +19,7 @@ func (s *Server) DetachShards(_ context.Context, req *control.DetachShardsReques shardIDs := s.getShardIDList(req.GetBody().GetShard_ID()) - if err := s.s.DetachShards(shardIDs); err != nil { + if err := s.s.DetachShards(ctx, shardIDs); err != nil { if errors.As(err, new(logicerr.Logical)) { return nil, status.Error(codes.InvalidArgument, err.Error()) } diff --git a/pkg/services/control/server/set_shard_mode.go b/pkg/services/control/server/set_shard_mode.go index 52835c41d..4f8796263 100644 --- a/pkg/services/control/server/set_shard_mode.go +++ b/pkg/services/control/server/set_shard_mode.go @@ -11,7 +11,7 @@ import ( "google.golang.org/grpc/status" ) -func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeRequest) (*control.SetShardModeResponse, error) { +func (s *Server) SetShardMode(ctx context.Context, req *control.SetShardModeRequest) (*control.SetShardModeResponse, error) { // verify request err := s.isValidRequest(req) if err != nil { @@ -38,7 +38,7 @@ func (s *Server) SetShardMode(_ context.Context, req *control.SetShardModeReques } for _, shardID := range s.getShardIDList(req.GetBody().GetShard_ID()) { - err = s.s.SetShardMode(shardID, m, req.GetBody().GetResetErrorCounter()) + err = s.s.SetShardMode(ctx, shardID, m, req.GetBody().GetResetErrorCounter()) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } From 6921a890619cc23a2eb2d9e5b4e92cc1bc45ee41 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 21 Oct 2024 12:21:01 +0300 Subject: [PATCH 1533/1943] [#1437] ir: Fix contextcheck linters Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/container.go | 10 +++--- cmd/frostfs-node/netmap.go | 10 +++--- cmd/frostfs-node/session.go | 2 +- cmd/frostfs-node/tree.go | 12 +++---- pkg/innerring/initialization.go | 12 +++---- pkg/innerring/innerring.go | 28 ++++++++-------- pkg/innerring/notary.go | 6 ++-- pkg/innerring/processors/balance/handlers.go | 6 ++-- .../processors/balance/handlers_test.go | 5 +-- .../processors/container/handlers.go | 12 +++---- .../processors/container/handlers_test.go | 5 +-- pkg/innerring/processors/frostfs/handlers.go | 24 +++++++------- .../processors/frostfs/handlers_test.go | 11 ++++--- .../processors/governance/handlers.go | 8 ++--- .../processors/governance/handlers_test.go | 7 ++-- .../processors/governance/process_update.go | 20 ++++++------ .../processors/governance/processor.go | 3 +- pkg/innerring/processors/netmap/handlers.go | 32 +++++++++---------- .../processors/netmap/handlers_test.go | 15 +++++---- .../processors/netmap/process_epoch.go | 24 +++++++------- pkg/innerring/state.go | 26 +++++++-------- pkg/innerring/state_test.go | 6 ++-- pkg/morph/event/handlers.go | 4 ++- pkg/morph/event/listener.go | 24 +++++++------- pkg/morph/event/listener_test.go | 2 +- pkg/morph/event/utils.go | 6 ++-- 27 files changed, 165 insertions(+), 157 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index bd1b99095..aa92e5ec5 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1089,7 +1089,7 @@ func (c *cfg) LocalAddress() network.AddressGroup { func initLocalStorage(ctx context.Context, c *cfg) { ls := engine.New(c.engineOpts()...) - addNewEpochAsyncNotificationHandler(c, func(ev event.Event) { + addNewEpochAsyncNotificationHandler(c, func(ctx context.Context, ev event.Event) { ls.HandleNewEpoch(ctx, ev.(netmap2.NewEpoch).EpochNumber()) }) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 1a54f9ffc..3f75be235 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -89,7 +89,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c if c.cfgMorph.containerCacheSize > 0 { containerCache := newCachedContainerStorage(cnrSrc, c.cfgMorph.cacheTTL, c.cfgMorph.containerCacheSize) - subscribeToContainerCreation(c, func(e event.Event) { + subscribeToContainerCreation(c, func(ctx context.Context, e event.Event) { ev := e.(containerEvent.PutSuccess) // read owner of the created container in order to update the reading cache. @@ -102,21 +102,21 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c } else { // unlike removal, we expect successful receive of the container // after successful creation, so logging can be useful - c.log.Error(context.Background(), logs.FrostFSNodeReadNewlyCreatedContainerAfterTheNotification, + c.log.Error(ctx, logs.FrostFSNodeReadNewlyCreatedContainerAfterTheNotification, zap.Stringer("id", ev.ID), zap.Error(err), ) } - c.log.Debug(context.Background(), logs.FrostFSNodeContainerCreationEventsReceipt, + c.log.Debug(ctx, logs.FrostFSNodeContainerCreationEventsReceipt, zap.Stringer("id", ev.ID), ) }) - subscribeToContainerRemoval(c, func(e event.Event) { + subscribeToContainerRemoval(c, func(ctx context.Context, e event.Event) { ev := e.(containerEvent.DeleteSuccess) containerCache.handleRemoval(ev.ID) - c.log.Debug(context.Background(), logs.FrostFSNodeContainerRemovalEventsReceipt, + c.log.Debug(ctx, logs.FrostFSNodeContainerRemovalEventsReceipt, zap.Stringer("id", ev.ID), ) }) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 18667e636..35ab4d575 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -175,11 +175,11 @@ func initNetmapService(ctx context.Context, c *cfg) { } func addNewEpochNotificationHandlers(c *cfg) { - addNewEpochNotificationHandler(c, func(ev event.Event) { + addNewEpochNotificationHandler(c, func(_ context.Context, ev event.Event) { c.cfgNetmap.state.setCurrentEpoch(ev.(netmapEvent.NewEpoch).EpochNumber()) }) - addNewEpochAsyncNotificationHandler(c, func(ev event.Event) { + addNewEpochAsyncNotificationHandler(c, func(ctx context.Context, ev event.Event) { e := ev.(netmapEvent.NewEpoch).EpochNumber() c.updateContractNodeInfo(e) @@ -189,15 +189,15 @@ func addNewEpochNotificationHandlers(c *cfg) { } if err := c.bootstrap(); err != nil { - c.log.Warn(context.Background(), logs.FrostFSNodeCantSendRebootstrapTx, zap.Error(err)) + c.log.Warn(ctx, logs.FrostFSNodeCantSendRebootstrapTx, zap.Error(err)) } }) if c.cfgMorph.notaryEnabled { - addNewEpochAsyncNotificationHandler(c, func(_ event.Event) { + addNewEpochAsyncNotificationHandler(c, func(ctx context.Context, _ event.Event) { _, _, err := makeNotaryDeposit(c) if err != nil { - c.log.Error(context.Background(), logs.FrostFSNodeCouldNotMakeNotaryDeposit, + c.log.Error(ctx, logs.FrostFSNodeCouldNotMakeNotaryDeposit, zap.String("error", err.Error()), ) } diff --git a/cmd/frostfs-node/session.go b/cmd/frostfs-node/session.go index a35d4e470..2f3c9cbfe 100644 --- a/cmd/frostfs-node/session.go +++ b/cmd/frostfs-node/session.go @@ -48,7 +48,7 @@ func initSessionService(c *cfg) { _ = c.privateTokenStore.Close() }) - addNewEpochNotificationHandler(c, func(ev event.Event) { + addNewEpochNotificationHandler(c, func(_ context.Context, ev event.Event) { c.privateTokenStore.RemoveOld(ev.(netmap.NewEpoch).EpochNumber()) }) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index 59923ee2f..c423c0660 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -80,10 +80,10 @@ func initTreeService(c *cfg) { })) if d := treeConfig.SyncInterval(); d == 0 { - addNewEpochNotificationHandler(c, func(_ event.Event) { + addNewEpochNotificationHandler(c, func(ctx context.Context, _ event.Event) { err := c.treeService.SynchronizeAll() if err != nil { - c.log.Error(context.Background(), logs.FrostFSNodeCouldNotSynchronizeTreeService, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeCouldNotSynchronizeTreeService, zap.Error(err)) } }) } else { @@ -103,15 +103,15 @@ func initTreeService(c *cfg) { }() } - subscribeToContainerRemoval(c, func(e event.Event) { + subscribeToContainerRemoval(c, func(ctx context.Context, e event.Event) { ev := e.(containerEvent.DeleteSuccess) // This is executed asynchronously, so we don't care about the operation taking some time. - c.log.Debug(context.Background(), logs.FrostFSNodeRemovingAllTreesForContainer, zap.Stringer("cid", ev.ID)) - err := c.treeService.DropTree(context.Background(), ev.ID, "") + c.log.Debug(ctx, logs.FrostFSNodeRemovingAllTreesForContainer, zap.Stringer("cid", ev.ID)) + err := c.treeService.DropTree(ctx, ev.ID, "") if err != nil && !errors.Is(err, pilorama.ErrTreeNotFound) { // Ignore pilorama.ErrTreeNotFound but other errors, including shard.ErrReadOnly, should be logged. - c.log.Error(context.Background(), logs.FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved, + c.log.Error(ctx, logs.FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved, zap.Stringer("cid", ev.ID), zap.String("error", err.Error())) } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index b8812819e..e08a613c3 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -137,12 +137,12 @@ func (s *Server) enableNotarySupport() error { return nil } -func (s *Server) initNotaryConfig() { +func (s *Server) initNotaryConfig(ctx context.Context) { s.mainNotaryConfig = notaryConfigs( !s.withoutMainNet && s.mainnetClient.ProbeNotary(), // if mainnet disabled then notary flag must be disabled too ) - s.log.Info(context.Background(), logs.InnerringNotarySupport, + s.log.Info(ctx, logs.InnerringNotarySupport, zap.Bool("sidechain_enabled", true), zap.Bool("mainchain_enabled", !s.mainNotaryConfig.disabled), ) @@ -152,8 +152,8 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli var alphaSync event.Handler if s.withoutMainNet || cfg.GetBool("governance.disable") { - alphaSync = func(event.Event) { - s.log.Debug(context.Background(), logs.InnerringAlphabetKeysSyncIsDisabled) + alphaSync = func(ctx context.Context, _ event.Event) { + s.log.Debug(ctx, logs.InnerringAlphabetKeysSyncIsDisabled) } } else { // create governance processor @@ -196,9 +196,9 @@ func (s *Server) createIRFetcher() irFetcher { return irf } -func (s *Server) initTimers(cfg *viper.Viper) { +func (s *Server) initTimers(ctx context.Context, cfg *viper.Viper) { s.epochTimer = newEpochTimer(&epochTimerArgs{ - newEpochHandlers: s.newEpochTickHandlers(), + newEpochHandlers: s.newEpochTickHandlers(ctx), epoch: s, }) diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 67927c10c..e81ec6bca 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -152,7 +152,7 @@ func (s *Server) Start(ctx context.Context, intError chan<- error) (err error) { return err } - err = s.initConfigFromBlockchain() + err = s.initConfigFromBlockchain(ctx) if err != nil { return err } @@ -173,14 +173,14 @@ func (s *Server) Start(ctx context.Context, intError chan<- error) (err error) { prm.Validators = s.predefinedValidators // vote for sidechain validator if it is prepared in config - err = s.voteForSidechainValidator(prm) + err = s.voteForSidechainValidator(ctx, prm) if err != nil { // we don't stop inner ring execution on this error s.log.Warn(ctx, logs.InnerringCantVoteForPreparedValidators, zap.String("error", err.Error())) } - s.tickInitialExpoch() + s.tickInitialExpoch(ctx) morphErr := make(chan error) mainnnetErr := make(chan error) @@ -283,11 +283,11 @@ func (s *Server) initSideNotary(ctx context.Context) error { ) } -func (s *Server) tickInitialExpoch() { +func (s *Server) tickInitialExpoch(ctx context.Context) { initialEpochTicker := timer.NewOneTickTimer( timer.StaticBlockMeter(s.initialEpochTickDelta), func() { - s.netmapProcessor.HandleNewEpochTick(timerEvent.NewEpochTick{}) + s.netmapProcessor.HandleNewEpochTick(ctx, timerEvent.NewEpochTick{}) }) s.addBlockTimer(initialEpochTicker) } @@ -376,7 +376,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - server.initNotaryConfig() + server.initNotaryConfig(ctx) err = server.initContracts(cfg) if err != nil { @@ -405,7 +405,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - server.initTimers(cfg) + server.initTimers(ctx, cfg) err = server.initGRPCServer(cfg, log, audit) if err != nil { @@ -573,7 +573,7 @@ func parseMultinetConfig(cfg *viper.Viper, m metrics.MultinetMetrics) internalNe return nc } -func (s *Server) initConfigFromBlockchain() error { +func (s *Server) initConfigFromBlockchain(ctx context.Context) error { // get current epoch epoch, err := s.netmapClient.Epoch() if err != nil { @@ -602,8 +602,8 @@ func (s *Server) initConfigFromBlockchain() error { return err } - s.log.Debug(context.Background(), logs.InnerringReadConfigFromBlockchain, - zap.Bool("active", s.IsActive()), + s.log.Debug(ctx, logs.InnerringReadConfigFromBlockchain, + zap.Bool("active", s.IsActive(ctx)), zap.Bool("alphabet", s.IsAlphabet()), zap.Uint64("epoch", epoch), zap.Uint32("precision", balancePrecision), @@ -635,17 +635,17 @@ func (s *Server) nextEpochBlockDelta() (uint32, error) { // onlyAlphabet wrapper around event handler that executes it // only if inner ring node is alphabet node. func (s *Server) onlyAlphabetEventHandler(f event.Handler) event.Handler { - return func(ev event.Event) { + return func(ctx context.Context, ev event.Event) { if s.IsAlphabet() { - f(ev) + f(ctx, ev) } } } -func (s *Server) newEpochTickHandlers() []newEpochHandler { +func (s *Server) newEpochTickHandlers(ctx context.Context) []newEpochHandler { newEpochHandlers := []newEpochHandler{ func() { - s.netmapProcessor.HandleNewEpochTick(timerEvent.NewEpochTick{}) + s.netmapProcessor.HandleNewEpochTick(ctx, timerEvent.NewEpochTick{}) }, } diff --git a/pkg/innerring/notary.go b/pkg/innerring/notary.go index 902a4c30a..dd3afa2c2 100644 --- a/pkg/innerring/notary.go +++ b/pkg/innerring/notary.go @@ -50,16 +50,16 @@ func (s *Server) depositSideNotary() (util.Uint256, error) { return tx, err } -func (s *Server) notaryHandler(_ event.Event) { +func (s *Server) notaryHandler(ctx context.Context, _ event.Event) { if !s.mainNotaryConfig.disabled { _, err := s.depositMainNotary() if err != nil { - s.log.Error(context.Background(), logs.InnerringCantMakeNotaryDepositInMainChain, zap.Error(err)) + s.log.Error(ctx, logs.InnerringCantMakeNotaryDepositInMainChain, zap.Error(err)) } } if _, err := s.depositSideNotary(); err != nil { - s.log.Error(context.Background(), logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) + s.log.Error(ctx, logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) } } diff --git a/pkg/innerring/processors/balance/handlers.go b/pkg/innerring/processors/balance/handlers.go index 3792fc2af..5a89e6f7c 100644 --- a/pkg/innerring/processors/balance/handlers.go +++ b/pkg/innerring/processors/balance/handlers.go @@ -11,9 +11,9 @@ import ( "go.uber.org/zap" ) -func (bp *Processor) handleLock(ev event.Event) { +func (bp *Processor) handleLock(ctx context.Context, ev event.Event) { lock := ev.(balanceEvent.Lock) - bp.log.Info(context.Background(), logs.Notification, + bp.log.Info(ctx, logs.Notification, zap.String("type", "lock"), zap.String("value", hex.EncodeToString(lock.ID()))) @@ -24,7 +24,7 @@ func (bp *Processor) handleLock(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - bp.log.Warn(context.Background(), logs.BalanceBalanceWorkerPoolDrained, + bp.log.Warn(ctx, logs.BalanceBalanceWorkerPoolDrained, zap.Int("capacity", bp.pool.Cap())) } } diff --git a/pkg/innerring/processors/balance/handlers_test.go b/pkg/innerring/processors/balance/handlers_test.go index 86a9e15d0..3ef4959cc 100644 --- a/pkg/innerring/processors/balance/handlers_test.go +++ b/pkg/innerring/processors/balance/handlers_test.go @@ -1,6 +1,7 @@ package balance import ( + "context" "testing" "time" @@ -30,7 +31,7 @@ func TestProcessorCallsFrostFSContractForLockEvent(t *testing.T) { }) require.NoError(t, err, "failed to create processor") - processor.handleLock(balanceEvent.Lock{}) + processor.handleLock(context.Background(), balanceEvent.Lock{}) for processor.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -56,7 +57,7 @@ func TestProcessorDoesntCallFrostFSContractIfNotAlphabet(t *testing.T) { }) require.NoError(t, err, "failed to create processor") - processor.handleLock(balanceEvent.Lock{}) + processor.handleLock(context.Background(), balanceEvent.Lock{}) for processor.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) diff --git a/pkg/innerring/processors/container/handlers.go b/pkg/innerring/processors/container/handlers.go index b3d50d9d0..45cac513a 100644 --- a/pkg/innerring/processors/container/handlers.go +++ b/pkg/innerring/processors/container/handlers.go @@ -12,11 +12,11 @@ import ( "go.uber.org/zap" ) -func (cp *Processor) handlePut(ev event.Event) { +func (cp *Processor) handlePut(ctx context.Context, ev event.Event) { put := ev.(putEvent) id := sha256.Sum256(put.Container()) - cp.log.Info(context.Background(), logs.Notification, + cp.log.Info(ctx, logs.Notification, zap.String("type", "container put"), zap.String("id", base58.Encode(id[:]))) @@ -27,14 +27,14 @@ func (cp *Processor) handlePut(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - cp.log.Warn(context.Background(), logs.ContainerContainerProcessorWorkerPoolDrained, + cp.log.Warn(ctx, logs.ContainerContainerProcessorWorkerPoolDrained, zap.Int("capacity", cp.pool.Cap())) } } -func (cp *Processor) handleDelete(ev event.Event) { +func (cp *Processor) handleDelete(ctx context.Context, ev event.Event) { del := ev.(containerEvent.Delete) - cp.log.Info(context.Background(), logs.Notification, + cp.log.Info(ctx, logs.Notification, zap.String("type", "container delete"), zap.String("id", base58.Encode(del.ContainerID()))) @@ -45,7 +45,7 @@ func (cp *Processor) handleDelete(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - cp.log.Warn(context.Background(), logs.ContainerContainerProcessorWorkerPoolDrained, + cp.log.Warn(ctx, logs.ContainerContainerProcessorWorkerPoolDrained, zap.Int("capacity", cp.pool.Cap())) } } diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index dc1e919bb..a2fe50fa8 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -1,6 +1,7 @@ package container import ( + "context" "crypto/ecdsa" "encoding/hex" "testing" @@ -71,7 +72,7 @@ func TestPutEvent(t *testing.T) { nr: nr, } - proc.handlePut(event) + proc.handlePut(context.Background(), event) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -143,7 +144,7 @@ func TestDeleteEvent(t *testing.T) { Signature: signature, } - proc.handleDelete(ev) + proc.handleDelete(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index 02dfbaf60..d11ad0f5c 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -13,11 +13,11 @@ import ( "go.uber.org/zap" ) -func (np *Processor) handleDeposit(ev event.Event) { +func (np *Processor) handleDeposit(ctx context.Context, ev event.Event) { deposit := ev.(frostfsEvent.Deposit) depositIDBin := bytes.Clone(deposit.ID()) slices.Reverse(depositIDBin) - np.log.Info(context.Background(), logs.Notification, + np.log.Info(ctx, logs.Notification, zap.String("type", "deposit"), zap.String("id", hex.EncodeToString(depositIDBin))) @@ -28,16 +28,16 @@ func (np *Processor) handleDeposit(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(context.Background(), logs.FrostFSFrostfsProcessorWorkerPoolDrained, + np.log.Warn(ctx, logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } -func (np *Processor) handleWithdraw(ev event.Event) { +func (np *Processor) handleWithdraw(ctx context.Context, ev event.Event) { withdraw := ev.(frostfsEvent.Withdraw) withdrawBin := bytes.Clone(withdraw.ID()) slices.Reverse(withdrawBin) - np.log.Info(context.Background(), logs.Notification, + np.log.Info(ctx, logs.Notification, zap.String("type", "withdraw"), zap.String("id", hex.EncodeToString(withdrawBin))) @@ -48,14 +48,14 @@ func (np *Processor) handleWithdraw(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(context.Background(), logs.FrostFSFrostfsProcessorWorkerPoolDrained, + np.log.Warn(ctx, logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } -func (np *Processor) handleCheque(ev event.Event) { +func (np *Processor) handleCheque(ctx context.Context, ev event.Event) { cheque := ev.(frostfsEvent.Cheque) - np.log.Info(context.Background(), logs.Notification, + np.log.Info(ctx, logs.Notification, zap.String("type", "cheque"), zap.String("id", hex.EncodeToString(cheque.ID()))) @@ -66,14 +66,14 @@ func (np *Processor) handleCheque(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(context.Background(), logs.FrostFSFrostfsProcessorWorkerPoolDrained, + np.log.Warn(ctx, logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } -func (np *Processor) handleConfig(ev event.Event) { +func (np *Processor) handleConfig(ctx context.Context, ev event.Event) { cfg := ev.(frostfsEvent.Config) - np.log.Info(context.Background(), logs.Notification, + np.log.Info(ctx, logs.Notification, zap.String("type", "set config"), zap.String("key", hex.EncodeToString(cfg.Key())), zap.String("value", hex.EncodeToString(cfg.Value()))) @@ -85,7 +85,7 @@ func (np *Processor) handleConfig(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(context.Background(), logs.FrostFSFrostfsProcessorWorkerPoolDrained, + np.log.Warn(ctx, logs.FrostFSFrostfsProcessorWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } diff --git a/pkg/innerring/processors/frostfs/handlers_test.go b/pkg/innerring/processors/frostfs/handlers_test.go index 6425172bd..c1541ca40 100644 --- a/pkg/innerring/processors/frostfs/handlers_test.go +++ b/pkg/innerring/processors/frostfs/handlers_test.go @@ -1,6 +1,7 @@ package frostfs import ( + "context" "testing" "time" @@ -36,7 +37,7 @@ func TestHandleDeposit(t *testing.T) { AmountValue: 1000, } - proc.handleDeposit(ev) + proc.handleDeposit(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -57,7 +58,7 @@ func TestHandleDeposit(t *testing.T) { es.epochCounter = 109 - proc.handleDeposit(ev) + proc.handleDeposit(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -98,7 +99,7 @@ func TestHandleWithdraw(t *testing.T) { AmountValue: 1000, } - proc.handleWithdraw(ev) + proc.handleWithdraw(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -139,7 +140,7 @@ func TestHandleCheque(t *testing.T) { LockValue: util.Uint160{200}, } - proc.handleCheque(ev) + proc.handleCheque(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -176,7 +177,7 @@ func TestHandleConfig(t *testing.T) { TxHashValue: util.Uint256{100}, } - proc.handleConfig(ev) + proc.handleConfig(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) diff --git a/pkg/innerring/processors/governance/handlers.go b/pkg/innerring/processors/governance/handlers.go index dee8c13e2..7e8ab629d 100644 --- a/pkg/innerring/processors/governance/handlers.go +++ b/pkg/innerring/processors/governance/handlers.go @@ -13,7 +13,7 @@ import ( "go.uber.org/zap" ) -func (gp *Processor) HandleAlphabetSync(e event.Event) { +func (gp *Processor) HandleAlphabetSync(ctx context.Context, e event.Event) { var ( typ string hash util.Uint256 @@ -34,16 +34,16 @@ func (gp *Processor) HandleAlphabetSync(e event.Event) { return } - gp.log.Info(context.Background(), logs.GovernanceNewEvent, zap.String("type", typ)) + gp.log.Info(ctx, logs.GovernanceNewEvent, zap.String("type", typ)) // send event to the worker pool err := processors.SubmitEvent(gp.pool, gp.metrics, "alphabet_sync", func() bool { - return gp.processAlphabetSync(hash) + return gp.processAlphabetSync(ctx, hash) }) if err != nil { // there system can be moved into controlled degradation stage - gp.log.Warn(context.Background(), logs.GovernanceGovernanceWorkerPoolDrained, + gp.log.Warn(ctx, logs.GovernanceGovernanceWorkerPoolDrained, zap.Int("capacity", gp.pool.Cap())) } } diff --git a/pkg/innerring/processors/governance/handlers_test.go b/pkg/innerring/processors/governance/handlers_test.go index 87040bdef..286935129 100644 --- a/pkg/innerring/processors/governance/handlers_test.go +++ b/pkg/innerring/processors/governance/handlers_test.go @@ -1,6 +1,7 @@ package governance import ( + "context" "encoding/binary" "sort" "testing" @@ -57,7 +58,7 @@ func TestHandleAlphabetSyncEvent(t *testing.T) { txHash: util.Uint256{100}, } - proc.HandleAlphabetSync(ev) + proc.HandleAlphabetSync(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -133,7 +134,7 @@ func TestHandleAlphabetDesignateEvent(t *testing.T) { Role: noderoles.NeoFSAlphabet, } - proc.HandleAlphabetSync(ev) + proc.HandleAlphabetSync(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -226,7 +227,7 @@ type testVoter struct { votes []VoteValidatorPrm } -func (v *testVoter) VoteForSidechainValidator(prm VoteValidatorPrm) error { +func (v *testVoter) VoteForSidechainValidator(_ context.Context, prm VoteValidatorPrm) error { v.votes = append(v.votes, prm) return nil } diff --git a/pkg/innerring/processors/governance/process_update.go b/pkg/innerring/processors/governance/process_update.go index faca22f67..fdfdfa479 100644 --- a/pkg/innerring/processors/governance/process_update.go +++ b/pkg/innerring/processors/governance/process_update.go @@ -19,39 +19,39 @@ const ( alphabetUpdateIDPrefix = "AlphabetUpdate" ) -func (gp *Processor) processAlphabetSync(txHash util.Uint256) bool { +func (gp *Processor) processAlphabetSync(ctx context.Context, txHash util.Uint256) bool { if !gp.alphabetState.IsAlphabet() { - gp.log.Info(context.Background(), logs.GovernanceNonAlphabetModeIgnoreAlphabetSync) + gp.log.Info(ctx, logs.GovernanceNonAlphabetModeIgnoreAlphabetSync) return true } mainnetAlphabet, err := gp.mainnetClient.NeoFSAlphabetList() if err != nil { - gp.log.Error(context.Background(), logs.GovernanceCantFetchAlphabetListFromMainNet, + gp.log.Error(ctx, logs.GovernanceCantFetchAlphabetListFromMainNet, zap.String("error", err.Error())) return false } sidechainAlphabet, err := gp.morphClient.Committee() if err != nil { - gp.log.Error(context.Background(), logs.GovernanceCantFetchAlphabetListFromSideChain, + gp.log.Error(ctx, logs.GovernanceCantFetchAlphabetListFromSideChain, zap.String("error", err.Error())) return false } newAlphabet, err := newAlphabetList(sidechainAlphabet, mainnetAlphabet) if err != nil { - gp.log.Error(context.Background(), logs.GovernanceCantMergeAlphabetListsFromMainNetAndSideChain, + gp.log.Error(ctx, logs.GovernanceCantMergeAlphabetListsFromMainNetAndSideChain, zap.String("error", err.Error())) return false } if newAlphabet == nil { - gp.log.Info(context.Background(), logs.GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged) + gp.log.Info(ctx, logs.GovernanceNoGovernanceUpdateAlphabetListHasNotBeenChanged) return true } - gp.log.Info(context.Background(), logs.GovernanceAlphabetListHasBeenChangedStartingUpdate, + gp.log.Info(ctx, logs.GovernanceAlphabetListHasBeenChangedStartingUpdate, zap.String("side_chain_alphabet", prettyKeys(sidechainAlphabet)), zap.String("new_alphabet", prettyKeys(newAlphabet)), ) @@ -62,9 +62,9 @@ func (gp *Processor) processAlphabetSync(txHash util.Uint256) bool { } // 1. Vote to sidechain committee via alphabet contracts. - err = gp.voter.VoteForSidechainValidator(votePrm) + err = gp.voter.VoteForSidechainValidator(ctx, votePrm) if err != nil { - gp.log.Error(context.Background(), logs.GovernanceCantVoteForSideChainCommittee, + gp.log.Error(ctx, logs.GovernanceCantVoteForSideChainCommittee, zap.String("error", err.Error())) } @@ -77,7 +77,7 @@ func (gp *Processor) processAlphabetSync(txHash util.Uint256) bool { // 4. Update FrostFS contract in the mainnet. gp.updateFrostFSContractInMainnet(newAlphabet) - gp.log.Info(context.Background(), logs.GovernanceFinishedAlphabetListUpdate) + gp.log.Info(ctx, logs.GovernanceFinishedAlphabetListUpdate) return true } diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index 6daea417e..eaadfdb4f 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -1,6 +1,7 @@ package governance import ( + "context" "errors" "fmt" @@ -38,7 +39,7 @@ type VoteValidatorPrm struct { // Voter is a callback interface for alphabet contract voting. type Voter interface { - VoteForSidechainValidator(VoteValidatorPrm) error + VoteForSidechainValidator(context.Context, VoteValidatorPrm) error } type ( diff --git a/pkg/innerring/processors/netmap/handlers.go b/pkg/innerring/processors/netmap/handlers.go index 478ab5eab..61547e0ba 100644 --- a/pkg/innerring/processors/netmap/handlers.go +++ b/pkg/innerring/processors/netmap/handlers.go @@ -12,13 +12,13 @@ import ( "go.uber.org/zap" ) -func (np *Processor) HandleNewEpochTick(ev event.Event) { +func (np *Processor) HandleNewEpochTick(ctx context.Context, ev event.Event) { _ = ev.(timerEvent.NewEpochTick) np.log.Info(context.Background(), logs.NetmapTick, zap.String("type", "epoch")) // send an event to the worker pool - err := processors.SubmitEvent(np.pool, np.metrics, "netmap_new_epoch_tick", np.processNewEpochTick) + err := processors.SubmitEvent(np.pool, np.metrics, "netmap_new_epoch_tick", func() bool { return np.processNewEpochTick(ctx) }) if err != nil { // there system can be moved into controlled degradation stage np.log.Warn(context.Background(), logs.NetmapNetmapWorkerPoolDrained, @@ -26,28 +26,28 @@ func (np *Processor) HandleNewEpochTick(ev event.Event) { } } -func (np *Processor) handleNewEpoch(ev event.Event) { +func (np *Processor) handleNewEpoch(ctx context.Context, ev event.Event) { epochEvent := ev.(netmapEvent.NewEpoch) - np.log.Info(context.Background(), logs.Notification, + np.log.Info(ctx, logs.Notification, zap.String("type", "new epoch"), zap.Uint64("value", epochEvent.EpochNumber())) // send an event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "netmap_new_epoch", func() bool { - return np.processNewEpoch(epochEvent) + return np.processNewEpoch(ctx, epochEvent) }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(context.Background(), logs.NetmapNetmapWorkerPoolDrained, + np.log.Warn(ctx, logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } -func (np *Processor) handleAddPeer(ev event.Event) { +func (np *Processor) handleAddPeer(ctx context.Context, ev event.Event) { newPeer := ev.(netmapEvent.AddPeer) - np.log.Info(context.Background(), logs.Notification, + np.log.Info(ctx, logs.Notification, zap.String("type", "add peer"), ) @@ -58,14 +58,14 @@ func (np *Processor) handleAddPeer(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(context.Background(), logs.NetmapNetmapWorkerPoolDrained, + np.log.Warn(ctx, logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } -func (np *Processor) handleUpdateState(ev event.Event) { +func (np *Processor) handleUpdateState(ctx context.Context, ev event.Event) { updPeer := ev.(netmapEvent.UpdatePeer) - np.log.Info(context.Background(), logs.Notification, + np.log.Info(ctx, logs.Notification, zap.String("type", "update peer state"), zap.String("key", hex.EncodeToString(updPeer.PublicKey().Bytes()))) @@ -76,21 +76,21 @@ func (np *Processor) handleUpdateState(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(context.Background(), logs.NetmapNetmapWorkerPoolDrained, + np.log.Warn(ctx, logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } -func (np *Processor) handleCleanupTick(ev event.Event) { +func (np *Processor) handleCleanupTick(ctx context.Context, ev event.Event) { if !np.netmapSnapshot.enabled { - np.log.Debug(context.Background(), logs.NetmapNetmapCleanUpRoutineIsDisabled518) + np.log.Debug(ctx, logs.NetmapNetmapCleanUpRoutineIsDisabled518) return } cleanup := ev.(netmapCleanupTick) - np.log.Info(context.Background(), logs.NetmapTick, zap.String("type", "netmap cleaner")) + np.log.Info(ctx, logs.NetmapTick, zap.String("type", "netmap cleaner")) // send event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "netmap_cleanup_tick", func() bool { @@ -98,7 +98,7 @@ func (np *Processor) handleCleanupTick(ev event.Event) { }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(context.Background(), logs.NetmapNetmapWorkerPoolDrained, + np.log.Warn(ctx, logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index a53458179..1e8be4095 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "fmt" "testing" "time" @@ -38,7 +39,7 @@ func TestNewEpochTick(t *testing.T) { require.NoError(t, err, "failed to create processor") ev := timerEvent.NewEpochTick{} - proc.HandleNewEpochTick(ev) + proc.HandleNewEpochTick(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -90,7 +91,7 @@ func TestNewEpoch(t *testing.T) { Num: 101, Hash: util.Uint256{101}, } - proc.handleNewEpoch(ev) + proc.handleNewEpoch(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -130,7 +131,7 @@ func TestAddPeer(t *testing.T) { MainTransaction: &transaction.Transaction{}, }, } - proc.handleAddPeer(ev) + proc.handleAddPeer(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -145,7 +146,7 @@ func TestAddPeer(t *testing.T) { MainTransaction: &transaction.Transaction{}, }, } - proc.handleAddPeer(ev) + proc.handleAddPeer(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -188,7 +189,7 @@ func TestUpdateState(t *testing.T) { MainTransaction: &transaction.Transaction{}, }, } - proc.handleUpdateState(ev) + proc.handleUpdateState(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -232,7 +233,7 @@ func TestCleanupTick(t *testing.T) { txHash: util.Uint256{123}, } - proc.handleCleanupTick(ev) + proc.handleCleanupTick(context.Background(), ev) for proc.pool.Running() > 0 { time.Sleep(10 * time.Millisecond) @@ -413,6 +414,6 @@ type testEventHandler struct { handledEvents []event.Event } -func (h *testEventHandler) Handle(e event.Event) { +func (h *testEventHandler) Handle(_ context.Context, e event.Event) { h.handledEvents = append(h.handledEvents, e) } diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index 8ad295a74..e401ef4f2 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -11,12 +11,12 @@ import ( // Process new epoch notification by setting global epoch value and resetting // local epoch timer. -func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) bool { +func (np *Processor) processNewEpoch(ctx context.Context, ev netmapEvent.NewEpoch) bool { epoch := ev.EpochNumber() epochDuration, err := np.netmapClient.EpochDuration() if err != nil { - np.log.Warn(context.Background(), logs.NetmapCantGetEpochDuration, + np.log.Warn(ctx, logs.NetmapCantGetEpochDuration, zap.String("error", err.Error())) } else { np.epochState.SetEpochDuration(epochDuration) @@ -26,46 +26,46 @@ func (np *Processor) processNewEpoch(ev netmapEvent.NewEpoch) bool { h, err := np.netmapClient.MorphTxHeight(ev.TxHash()) if err != nil { - np.log.Warn(context.Background(), logs.NetmapCantGetTransactionHeight, + np.log.Warn(ctx, logs.NetmapCantGetTransactionHeight, zap.String("hash", ev.TxHash().StringLE()), zap.String("error", err.Error())) } if err := np.epochTimer.ResetEpochTimer(h); err != nil { - np.log.Warn(context.Background(), logs.NetmapCantResetEpochTimer, + np.log.Warn(ctx, logs.NetmapCantResetEpochTimer, zap.String("error", err.Error())) } // get new netmap snapshot networkMap, err := np.netmapClient.NetMap() if err != nil { - np.log.Warn(context.Background(), logs.NetmapCantGetNetmapSnapshotToPerformCleanup, + np.log.Warn(ctx, logs.NetmapCantGetNetmapSnapshotToPerformCleanup, zap.String("error", err.Error())) return false } np.netmapSnapshot.update(*networkMap, epoch) - np.handleCleanupTick(netmapCleanupTick{epoch: epoch, txHash: ev.TxHash()}) - np.handleAlphabetSync(governance.NewSyncEvent(ev.TxHash())) - np.handleNotaryDeposit(ev) + np.handleCleanupTick(ctx, netmapCleanupTick{epoch: epoch, txHash: ev.TxHash()}) + np.handleAlphabetSync(ctx, governance.NewSyncEvent(ev.TxHash())) + np.handleNotaryDeposit(ctx, ev) return true } // Process new epoch tick by invoking new epoch method in network map contract. -func (np *Processor) processNewEpochTick() bool { +func (np *Processor) processNewEpochTick(ctx context.Context) bool { if !np.alphabetState.IsAlphabet() { - np.log.Info(context.Background(), logs.NetmapNonAlphabetModeIgnoreNewEpochTick) + np.log.Info(ctx, logs.NetmapNonAlphabetModeIgnoreNewEpochTick) return true } nextEpoch := np.epochState.EpochCounter() + 1 - np.log.Debug(context.Background(), logs.NetmapNextEpoch, zap.Uint64("value", nextEpoch)) + np.log.Debug(ctx, logs.NetmapNextEpoch, zap.Uint64("value", nextEpoch)) err := np.netmapClient.NewEpoch(nextEpoch) if err != nil { - np.log.Error(context.Background(), logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) + np.log.Error(ctx, logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) return false } diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 2dbcd7494..85f332fb6 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -48,8 +48,8 @@ func (s *Server) SetEpochDuration(val uint64) { } // IsActive is a getter for a global active flag state. -func (s *Server) IsActive() bool { - return s.InnerRingIndex() >= 0 +func (s *Server) IsActive(ctx context.Context) bool { + return s.InnerRingIndex(ctx) >= 0 } // IsAlphabet is a getter for a global alphabet flag state. @@ -59,10 +59,10 @@ func (s *Server) IsAlphabet() bool { // InnerRingIndex is a getter for a global index of node in inner ring list. Negative // index means that node is not in the inner ring list. -func (s *Server) InnerRingIndex() int { +func (s *Server) InnerRingIndex(ctx context.Context) int { index, err := s.statusIndex.InnerRingIndex() if err != nil { - s.log.Error(context.Background(), logs.InnerringCantGetInnerRingIndex, zap.String("error", err.Error())) + s.log.Error(ctx, logs.InnerringCantGetInnerRingIndex, zap.String("error", err.Error())) return -1 } @@ -71,10 +71,10 @@ func (s *Server) InnerRingIndex() int { // InnerRingSize is a getter for a global size of inner ring list. This value // paired with inner ring index. -func (s *Server) InnerRingSize() int { +func (s *Server) InnerRingSize(ctx context.Context) int { size, err := s.statusIndex.InnerRingSize() if err != nil { - s.log.Error(context.Background(), logs.InnerringCantGetInnerRingSize, zap.String("error", err.Error())) + s.log.Error(ctx, logs.InnerringCantGetInnerRingSize, zap.String("error", err.Error())) return 0 } @@ -93,18 +93,18 @@ func (s *Server) AlphabetIndex() int { return int(index) } -func (s *Server) voteForSidechainValidator(prm governance.VoteValidatorPrm) error { +func (s *Server) voteForSidechainValidator(ctx context.Context, prm governance.VoteValidatorPrm) error { validators := prm.Validators - index := s.InnerRingIndex() + index := s.InnerRingIndex(ctx) if s.contracts.alphabet.indexOutOfRange(index) { - s.log.Info(context.Background(), logs.InnerringIgnoreValidatorVoteNodeNotInAlphabetRange) + s.log.Info(ctx, logs.InnerringIgnoreValidatorVoteNodeNotInAlphabetRange) return nil } if len(validators) == 0 { - s.log.Info(context.Background(), logs.InnerringIgnoreValidatorVoteEmptyValidatorsList) + s.log.Info(ctx, logs.InnerringIgnoreValidatorVoteEmptyValidatorsList) return nil } @@ -129,7 +129,7 @@ func (s *Server) voteForSidechainValidator(prm governance.VoteValidatorPrm) erro s.contracts.alphabet.iterate(func(letter GlagoliticLetter, contract util.Uint160) { _, err := s.morphClient.NotaryInvoke(contract, s.feeConfig.SideChainFee(), nonce, vubP, voteMethod, epoch, validators) if err != nil { - s.log.Warn(context.Background(), logs.InnerringCantInvokeVoteMethodInAlphabetContract, + s.log.Warn(ctx, logs.InnerringCantInvokeVoteMethodInAlphabetContract, zap.Int8("alphabet_index", int8(letter)), zap.Uint64("epoch", epoch), zap.String("error", err.Error())) @@ -141,9 +141,9 @@ func (s *Server) voteForSidechainValidator(prm governance.VoteValidatorPrm) erro // VoteForSidechainValidator calls vote method on alphabet contracts with // the provided list of keys. -func (s *Server) VoteForSidechainValidator(prm governance.VoteValidatorPrm) error { +func (s *Server) VoteForSidechainValidator(ctx context.Context, prm governance.VoteValidatorPrm) error { sort.Sort(prm.Validators) - return s.voteForSidechainValidator(prm) + return s.voteForSidechainValidator(ctx, prm) } // ResetEpochTimer resets the block timer that produces events to update epoch diff --git a/pkg/innerring/state_test.go b/pkg/innerring/state_test.go index 9313edf78..17ab995af 100644 --- a/pkg/innerring/state_test.go +++ b/pkg/innerring/state_test.go @@ -46,9 +46,9 @@ func TestServerState(t *testing.T) { srv.setHealthStatus(context.Background(), healthStatus) require.Equal(t, healthStatus, srv.HealthStatus(), "invalid health status") - require.True(t, srv.IsActive(), "invalid IsActive result") + require.True(t, srv.IsActive(context.Background()), "invalid IsActive result") require.True(t, srv.IsAlphabet(), "invalid IsAlphabet result") - require.Equal(t, 0, srv.InnerRingIndex(), "invalid IR index") - require.Equal(t, 1, srv.InnerRingSize(), "invalid IR index") + require.Equal(t, 0, srv.InnerRingIndex(context.Background()), "invalid IR index") + require.Equal(t, 1, srv.InnerRingSize(context.Background()), "invalid IR index") require.Equal(t, 0, srv.AlphabetIndex(), "invalid alphabet index") } diff --git a/pkg/morph/event/handlers.go b/pkg/morph/event/handlers.go index 182b4667e..bda83ba54 100644 --- a/pkg/morph/event/handlers.go +++ b/pkg/morph/event/handlers.go @@ -1,11 +1,13 @@ package event import ( + "context" + "github.com/nspcc-dev/neo-go/pkg/core/block" ) // Handler is an Event processing function. -type Handler func(Event) +type Handler func(context.Context, Event) // BlockHandler is a chain block processing function. type BlockHandler func(*block.Block) diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 3d3d806a4..eeec46540 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -280,7 +280,7 @@ loop: continue loop } - l.handleNotaryEvent(notaryEvent) + l.handleNotaryEvent(ctx, notaryEvent) case b, ok := <-chs.BlockCh: if !ok { l.log.Warn(ctx, logs.EventStopEventListenerByBlockChannel) @@ -307,11 +307,11 @@ func (l *listener) handleBlockEvent(b *block.Block) { } } -func (l *listener) handleNotaryEvent(notaryEvent *result.NotaryRequestEvent) { +func (l *listener) handleNotaryEvent(ctx context.Context, notaryEvent *result.NotaryRequestEvent) { if err := l.pool.Submit(func() { - l.parseAndHandleNotary(notaryEvent) + l.parseAndHandleNotary(ctx, notaryEvent) }); err != nil { - l.log.Warn(context.Background(), logs.EventListenerWorkerPoolDrained, + l.log.Warn(ctx, logs.EventListenerWorkerPoolDrained, zap.Int("capacity", l.pool.Cap())) } } @@ -376,11 +376,11 @@ func (l *listener) parseAndHandleNotification(ctx context.Context, notifyEvent * } for _, handler := range handlers { - handler(event) + handler(ctx, event) } } -func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { +func (l *listener) parseAndHandleNotary(ctx context.Context, nr *result.NotaryRequestEvent) { // prepare the notary event notaryEvent, err := l.notaryEventsPreparator.Prepare(nr.NotaryRequest) if err != nil { @@ -388,13 +388,13 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { switch { case errors.Is(err, ErrTXAlreadyHandled): case errors.As(err, &expErr): - l.log.Warn(context.Background(), logs.EventSkipExpiredMainTXNotaryEvent, + l.log.Warn(ctx, logs.EventSkipExpiredMainTXNotaryEvent, zap.String("error", err.Error()), zap.Uint32("current_block_height", expErr.CurrentBlockHeight), zap.Uint32("fallback_tx_not_valid_before_height", expErr.FallbackTXNotValidBeforeHeight), ) default: - l.log.Warn(context.Background(), logs.EventCouldNotPrepareAndValidateNotaryEvent, + l.log.Warn(ctx, logs.EventCouldNotPrepareAndValidateNotaryEvent, zap.String("error", err.Error()), ) } @@ -418,7 +418,7 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { l.mtx.RUnlock() if !ok { - log.Debug(context.Background(), logs.EventNotaryParserNotSet) + log.Debug(ctx, logs.EventNotaryParserNotSet) return } @@ -426,7 +426,7 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { // parse the notary event event, err := parser(notaryEvent) if err != nil { - log.Warn(context.Background(), logs.EventCouldNotParseNotaryEvent, + log.Warn(ctx, logs.EventCouldNotParseNotaryEvent, zap.String("error", err.Error()), ) @@ -439,14 +439,14 @@ func (l *listener) parseAndHandleNotary(nr *result.NotaryRequestEvent) { l.mtx.RUnlock() if !ok { - log.Info(context.Background(), logs.EventNotaryHandlersForParsedNotificationEventWereNotRegistered, + log.Info(ctx, logs.EventNotaryHandlersForParsedNotificationEventWereNotRegistered, zap.Any("event", event), ) return } - handler(event) + handler(ctx, event) } // SetNotificationParser sets the parser of particular contract event. diff --git a/pkg/morph/event/listener_test.go b/pkg/morph/event/listener_test.go index 5f7cf9f43..214daf694 100644 --- a/pkg/morph/event/listener_test.go +++ b/pkg/morph/event/listener_test.go @@ -59,7 +59,7 @@ func TestEventHandling(t *testing.T) { handledNotifications := make([]Event, 0) l.RegisterNotificationHandler(NotificationHandlerInfo{ scriptHashWithType: key, - h: func(e Event) { + h: func(_ context.Context, e Event) { handledNotifications = append(handledNotifications, e) notificationHandled <- true }, diff --git a/pkg/morph/event/utils.go b/pkg/morph/event/utils.go index 31bbf4432..99ea9a7f0 100644 --- a/pkg/morph/event/utils.go +++ b/pkg/morph/event/utils.go @@ -85,12 +85,12 @@ func (s typeValue) GetType() Type { // WorkerPoolHandler sets closure over worker pool w with passed handler h. func WorkerPoolHandler(w util2.WorkerPool, h Handler, log *logger.Logger) Handler { - return func(e Event) { + return func(ctx context.Context, e Event) { err := w.Submit(func() { - h(e) + h(ctx, e) }) if err != nil { - log.Warn(context.Background(), logs.EventCouldNotSubmitHandlerToWorkerPool, + log.Warn(ctx, logs.EventCouldNotSubmitHandlerToWorkerPool, zap.String("error", err.Error()), ) } From 74295532665e32efc80fcdcde639b2d9bc872a88 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 21 Oct 2024 16:27:28 +0300 Subject: [PATCH 1534/1943] [#1437] node: Fix contextcheck linter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-ir/config.go | 4 +- cmd/frostfs-ir/httpcomponent.go | 30 +++++----- cmd/frostfs-ir/main.go | 12 ++-- cmd/frostfs-ir/pprof.go | 18 +++--- .../internal/blobovnicza/inspect.go | 2 +- cmd/frostfs-lens/internal/blobovnicza/list.go | 2 +- cmd/frostfs-lens/internal/meta/inspect.go | 2 +- .../internal/meta/list-garbage.go | 2 +- .../internal/meta/list-graveyard.go | 2 +- cmd/frostfs-node/config.go | 60 +++++++++---------- cmd/frostfs-node/container.go | 8 +-- cmd/frostfs-node/control.go | 22 +++---- cmd/frostfs-node/grpc.go | 56 ++++++++--------- cmd/frostfs-node/httpcomponent.go | 10 ++-- cmd/frostfs-node/main.go | 56 ++++++++--------- cmd/frostfs-node/morph.go | 8 +-- cmd/frostfs-node/netmap.go | 48 +++++++-------- cmd/frostfs-node/object.go | 4 +- cmd/frostfs-node/pprof.go | 5 +- cmd/frostfs-node/runtime.go | 6 +- internal/audit/request.go | 8 +-- pkg/core/object/fmt.go | 12 ++-- pkg/core/object/sender_classifier.go | 9 +-- pkg/innerring/blocktimer.go | 6 +- pkg/innerring/initialization.go | 49 +++++++++------ pkg/innerring/innerring.go | 20 +++---- pkg/innerring/notary.go | 13 ++-- pkg/innerring/processors/alphabet/handlers.go | 8 +-- .../processors/alphabet/handlers_test.go | 11 ++-- .../processors/alphabet/process_emit.go | 32 +++++----- .../processors/alphabet/processor.go | 8 +-- pkg/innerring/processors/balance/handlers.go | 2 +- .../processors/balance/handlers_test.go | 4 +- .../processors/balance/process_assets.go | 10 ++-- pkg/innerring/processors/balance/processor.go | 8 +-- .../processors/container/handlers.go | 4 +- .../processors/container/handlers_test.go | 2 +- .../processors/container/process_container.go | 26 ++++---- .../processors/container/processor.go | 6 +- pkg/innerring/processors/frostfs/handlers.go | 8 +-- .../processors/frostfs/handlers_test.go | 10 ++-- .../processors/frostfs/process_assets.go | 40 ++++++------- .../processors/frostfs/process_config.go | 10 ++-- pkg/innerring/processors/frostfs/processor.go | 14 ++--- .../processors/governance/handlers_test.go | 8 +-- .../processors/governance/process_update.go | 32 +++++----- .../processors/governance/processor.go | 8 +-- pkg/innerring/processors/netmap/handlers.go | 10 ++-- .../processors/netmap/handlers_test.go | 6 +- .../processors/netmap/process_cleanup.go | 15 ++--- .../processors/netmap/process_epoch.go | 4 +- .../processors/netmap/process_peers.go | 27 +++++---- pkg/innerring/processors/netmap/processor.go | 10 +--- pkg/innerring/processors/netmap/wrappers.go | 18 +++--- pkg/innerring/state.go | 10 ++-- pkg/innerring/state_test.go | 4 +- .../blobovnicza/blobovnicza_test.go | 4 +- .../blobovnicza/control.go | 22 +++---- .../blobovnicza/get_test.go | 6 +- .../blobovnicza/iterate_test.go | 2 +- .../blobstor/blobovniczatree/active.go | 14 ++--- .../blobstor/blobovniczatree/blobovnicza.go | 2 +- .../blobstor/blobovniczatree/cache.go | 4 +- .../blobovniczatree/concurrency_test.go | 2 +- .../blobstor/blobovniczatree/control.go | 6 +- .../blobstor/blobovniczatree/control_test.go | 6 +- .../blobstor/blobovniczatree/count.go | 2 +- .../blobstor/blobovniczatree/delete.go | 4 +- .../blobstor/blobovniczatree/exists.go | 2 +- .../blobstor/blobovniczatree/exists_test.go | 2 +- .../blobstor/blobovniczatree/get.go | 4 +- .../blobstor/blobovniczatree/get_range.go | 4 +- .../blobstor/blobovniczatree/iterate.go | 2 +- .../blobstor/blobovniczatree/manager.go | 16 ++--- .../blobstor/blobovniczatree/option.go | 5 +- .../blobstor/blobovniczatree/put.go | 6 +- .../blobstor/blobovniczatree/rebuild.go | 28 ++++----- .../blobovniczatree/rebuild_failover_test.go | 30 +++++----- .../blobstor/blobovniczatree/rebuild_test.go | 18 +++--- pkg/local_object_storage/blobstor/blobstor.go | 3 +- .../blobstor/blobstor_test.go | 14 ++--- .../blobstor/common/storage.go | 4 +- pkg/local_object_storage/blobstor/control.go | 12 ++-- .../blobstor/exists_test.go | 2 +- .../blobstor/fstree/control.go | 4 +- .../blobstor/fstree/fstree.go | 2 +- .../blobstor/fstree/fstree_test.go | 2 +- .../blobstor/internal/blobstortest/control.go | 2 +- .../blobstor/internal/blobstortest/delete.go | 2 +- .../blobstor/internal/blobstortest/exists.go | 2 +- .../blobstor/internal/blobstortest/get.go | 2 +- .../internal/blobstortest/get_range.go | 2 +- .../blobstor/internal/blobstortest/iterate.go | 2 +- .../blobstor/iterate_test.go | 6 +- .../blobstor/memstore/control.go | 18 +++--- .../blobstor/memstore/memstore_test.go | 2 +- pkg/local_object_storage/blobstor/mode.go | 4 +- .../blobstor/perf_test.go | 6 +- .../blobstor/teststore/option.go | 6 +- .../blobstor/teststore/teststore.go | 6 +- pkg/local_object_storage/engine/container.go | 14 ++--- pkg/local_object_storage/engine/delete.go | 6 +- pkg/local_object_storage/engine/engine.go | 19 +++--- pkg/local_object_storage/engine/exists.go | 2 +- pkg/local_object_storage/engine/get.go | 2 +- pkg/local_object_storage/engine/head.go | 2 +- pkg/local_object_storage/engine/inhume.go | 10 ++-- pkg/local_object_storage/engine/lock.go | 8 +-- pkg/local_object_storage/engine/put.go | 2 +- pkg/local_object_storage/engine/range.go | 2 +- pkg/local_object_storage/engine/select.go | 4 +- pkg/local_object_storage/engine/shards.go | 6 +- pkg/local_object_storage/engine/tree.go | 26 ++++---- .../internal/storagetest/storage.go | 30 +++++----- .../metabase/containers_test.go | 6 +- pkg/local_object_storage/metabase/control.go | 12 ++-- .../metabase/control_test.go | 2 +- .../metabase/counter_test.go | 18 +++--- pkg/local_object_storage/metabase/db_test.go | 2 +- .../metabase/delete_ec_test.go | 8 +-- .../metabase/delete_meta_test.go | 4 +- .../metabase/delete_test.go | 12 ++-- .../metabase/exists_test.go | 3 +- .../metabase/expired_test.go | 2 +- pkg/local_object_storage/metabase/get_test.go | 6 +- .../metabase/graveyard_test.go | 12 ++-- .../metabase/inhume_ec_test.go | 4 +- .../metabase/inhume_test.go | 6 +- .../metabase/iterators_test.go | 4 +- .../metabase/list_test.go | 8 +-- .../metabase/lock_test.go | 6 +- pkg/local_object_storage/metabase/mode.go | 4 +- .../metabase/mode_test.go | 8 +-- pkg/local_object_storage/metabase/put_test.go | 6 +- .../metabase/reset_test.go | 4 +- .../metabase/select_test.go | 34 +++++------ .../metabase/storage_id_test.go | 4 +- .../metabase/upgrade_test.go | 12 ++-- .../metabase/version_test.go | 32 +++++----- .../pilorama/bench_test.go | 4 +- pkg/local_object_storage/pilorama/boltdb.go | 10 ++-- pkg/local_object_storage/pilorama/forest.go | 4 +- .../pilorama/forest_test.go | 54 ++++++++--------- .../pilorama/interface.go | 4 +- .../pilorama/mode_test.go | 8 +-- pkg/local_object_storage/shard/control.go | 16 ++--- pkg/local_object_storage/shard/gc.go | 16 ++--- .../shard/gc_internal_test.go | 4 +- pkg/local_object_storage/shard/lock_test.go | 4 +- pkg/local_object_storage/shard/shard.go | 10 ++-- pkg/local_object_storage/shard/shard_test.go | 4 +- .../writecache/benchmark/writecache_test.go | 8 +-- pkg/local_object_storage/writecache/cache.go | 10 ++-- pkg/local_object_storage/writecache/flush.go | 14 ++--- .../writecache/flush_test.go | 16 ++--- pkg/local_object_storage/writecache/mode.go | 4 +- .../writecache/mode_test.go | 8 +-- .../writecache/options.go | 6 +- .../writecache/writecache.go | 4 +- pkg/morph/client/balance/burn.go | 6 +- pkg/morph/client/balance/lock.go | 6 +- pkg/morph/client/balance/mint.go | 6 +- pkg/morph/client/balance/transfer.go | 5 +- pkg/morph/client/client.go | 4 +- pkg/morph/client/container/delete.go | 9 +-- pkg/morph/client/container/put.go | 9 +-- pkg/morph/client/frostfs/cheque.go | 10 ++-- pkg/morph/client/netmap/config.go | 5 +- pkg/morph/client/netmap/innerring.go | 5 +- pkg/morph/client/netmap/new_epoch.go | 9 +-- pkg/morph/client/netmap/peer.go | 9 +-- pkg/morph/client/netmap/update_state.go | 5 +- pkg/morph/client/notary.go | 40 +++++++------ pkg/morph/client/static.go | 8 ++- pkg/morph/event/handlers.go | 2 +- pkg/morph/event/listener.go | 8 +-- pkg/morph/event/listener_test.go | 4 +- pkg/network/transport/object/grpc/service.go | 4 +- pkg/services/apemanager/audit.go | 6 +- pkg/services/container/audit.go | 8 +-- pkg/services/container/morph/executor.go | 12 ++-- pkg/services/container/morph/executor_test.go | 4 +- pkg/services/control/ir/server/audit.go | 8 +-- pkg/services/control/ir/server/calls.go | 18 +++--- pkg/services/control/server/server.go | 5 +- .../control/server/set_netmap_status.go | 6 +- pkg/services/object/acl/v2/service.go | 34 +++++------ pkg/services/object/ape/service.go | 8 +-- pkg/services/object/audit.go | 34 +++++------ pkg/services/object/common.go | 8 +-- pkg/services/object/common/writer/common.go | 4 +- pkg/services/object/common/writer/ec.go | 2 +- pkg/services/object/get/assembleec.go | 2 +- pkg/services/object/get/container.go | 4 +- pkg/services/object/get/remote.go | 2 +- pkg/services/object/get/request.go | 12 ++-- pkg/services/object/metrics.go | 12 ++-- pkg/services/object/response.go | 8 +-- pkg/services/object/search/search.go | 10 ++-- pkg/services/object/server.go | 4 +- pkg/services/object/sign.go | 8 +-- pkg/services/object/transport_splitter.go | 8 +-- pkg/services/object/util/log.go | 8 +-- .../object_manager/tombstone/checker.go | 6 +- pkg/services/policer/check.go | 8 +-- pkg/services/tree/getsubtree_test.go | 2 +- pkg/services/tree/sync.go | 6 +- pkg/util/http/calls.go | 4 +- scripts/populate-metabase/main.go | 6 +- 209 files changed, 1068 insertions(+), 1036 deletions(-) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 7415e8e70..09af08525 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -93,8 +93,8 @@ func watchForSignal(ctx context.Context, cancel func()) { if err != nil { log.Error(ctx, logs.FrostFSNodeConfigurationReading, zap.Error(err)) } - pprofCmp.reload() - metricsCmp.reload() + pprofCmp.reload(ctx) + metricsCmp.reload(ctx) log.Info(ctx, logs.FrostFSIRReloadExtraWallets) err = innerRing.SetExtraWallets(cfg) if err != nil { diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index 685ef61ad..a8eef6010 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -25,8 +25,8 @@ const ( shutdownTimeoutKeyPostfix = ".shutdown_timeout" ) -func (c *httpComponent) init() { - log.Info(context.Background(), "init "+c.name) +func (c *httpComponent) init(ctx context.Context) { + log.Info(ctx, "init "+c.name) c.enabled = cfg.GetBool(c.name + enabledKeyPostfix) c.address = cfg.GetString(c.name + addressKeyPostfix) c.shutdownDur = cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) @@ -40,14 +40,14 @@ func (c *httpComponent) init() { httputil.WithShutdownTimeout(c.shutdownDur), ) } else { - log.Info(context.Background(), c.name+" is disabled, skip") + log.Info(ctx, c.name+" is disabled, skip") c.srv = nil } } -func (c *httpComponent) start() { +func (c *httpComponent) start(ctx context.Context) { if c.srv != nil { - log.Info(context.Background(), "start "+c.name) + log.Info(ctx, "start "+c.name) wg.Add(1) go func() { defer wg.Done() @@ -56,10 +56,10 @@ func (c *httpComponent) start() { } } -func (c *httpComponent) shutdown() error { +func (c *httpComponent) shutdown(ctx context.Context) error { if c.srv != nil { - log.Info(context.Background(), "shutdown "+c.name) - return c.srv.Shutdown() + log.Info(ctx, "shutdown "+c.name) + return c.srv.Shutdown(ctx) } return nil } @@ -71,17 +71,17 @@ func (c *httpComponent) needReload() bool { return enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) } -func (c *httpComponent) reload() { - log.Info(context.Background(), "reload "+c.name) +func (c *httpComponent) reload(ctx context.Context) { + log.Info(ctx, "reload "+c.name) if c.needReload() { - log.Info(context.Background(), c.name+" config updated") - if err := c.shutdown(); err != nil { - log.Debug(context.Background(), logs.FrostFSIRCouldNotShutdownHTTPServer, + log.Info(ctx, c.name+" config updated") + if err := c.shutdown(ctx); err != nil { + log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error()), ) } else { - c.init() - c.start() + c.init(ctx) + c.start(ctx) } } } diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index bcb2c5dd8..e86c04b9e 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -87,17 +87,17 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) pprofCmp = newPprofComponent() - pprofCmp.init() + pprofCmp.init(ctx) metricsCmp = newMetricsComponent() - metricsCmp.init() + metricsCmp.init(ctx) audit.Store(cfg.GetBool("audit.enabled")) innerRing, err = innerring.New(ctx, log, cfg, intErr, metrics, cmode, audit) exitErr(err) - pprofCmp.start() - metricsCmp.start() + pprofCmp.start(ctx) + metricsCmp.start(ctx) // start inner ring err = innerRing.Start(ctx, intErr) @@ -117,12 +117,12 @@ func main() { func shutdown(ctx context.Context) { innerRing.Stop(ctx) - if err := metricsCmp.shutdown(); err != nil { + if err := metricsCmp.shutdown(ctx); err != nil { log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error()), ) } - if err := pprofCmp.shutdown(); err != nil { + if err := pprofCmp.shutdown(ctx); err != nil { log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error()), ) diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go index e95fd117f..8e81d8b85 100644 --- a/cmd/frostfs-ir/pprof.go +++ b/cmd/frostfs-ir/pprof.go @@ -29,8 +29,8 @@ func newPprofComponent() *pprofComponent { } } -func (c *pprofComponent) init() { - c.httpComponent.init() +func (c *pprofComponent) init(ctx context.Context) { + c.httpComponent.init(ctx) if c.enabled { c.blockRate = cfg.GetInt(pprofBlockRateKey) @@ -52,17 +52,17 @@ func (c *pprofComponent) needReload() bool { c.enabled && (c.blockRate != blockRate || c.mutexRate != mutexRate) } -func (c *pprofComponent) reload() { - log.Info(context.Background(), "reload "+c.name) +func (c *pprofComponent) reload(ctx context.Context) { + log.Info(ctx, "reload "+c.name) if c.needReload() { - log.Info(context.Background(), c.name+" config updated") - if err := c.shutdown(); err != nil { - log.Debug(context.Background(), logs.FrostFSIRCouldNotShutdownHTTPServer, + log.Info(ctx, c.name+" config updated") + if err := c.shutdown(ctx); err != nil { + log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer, zap.String("error", err.Error())) return } - c.init() - c.start() + c.init(ctx) + c.start(ctx) } } diff --git a/cmd/frostfs-lens/internal/blobovnicza/inspect.go b/cmd/frostfs-lens/internal/blobovnicza/inspect.go index b1a6e3fd2..e7e2c0769 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/inspect.go +++ b/cmd/frostfs-lens/internal/blobovnicza/inspect.go @@ -28,7 +28,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { common.ExitOnErr(cmd, common.Errf("invalid address argument: %w", err)) blz := openBlobovnicza(cmd) - defer blz.Close() + defer blz.Close(cmd.Context()) var prm blobovnicza.GetPrm prm.SetAddress(addr) diff --git a/cmd/frostfs-lens/internal/blobovnicza/list.go b/cmd/frostfs-lens/internal/blobovnicza/list.go index d327dbc41..d41a15bcf 100644 --- a/cmd/frostfs-lens/internal/blobovnicza/list.go +++ b/cmd/frostfs-lens/internal/blobovnicza/list.go @@ -32,7 +32,7 @@ func listFunc(cmd *cobra.Command, _ []string) { } blz := openBlobovnicza(cmd) - defer blz.Close() + defer blz.Close(cmd.Context()) err := blobovnicza.IterateAddresses(context.Background(), blz, wAddr) common.ExitOnErr(cmd, common.Errf("blobovnicza iterator failure: %w", err)) diff --git a/cmd/frostfs-lens/internal/meta/inspect.go b/cmd/frostfs-lens/internal/meta/inspect.go index 9eb60f966..f436343c7 100644 --- a/cmd/frostfs-lens/internal/meta/inspect.go +++ b/cmd/frostfs-lens/internal/meta/inspect.go @@ -31,7 +31,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) { common.ExitOnErr(cmd, common.Errf("invalid address argument: %w", err)) db := openMeta(cmd) - defer db.Close() + defer db.Close(cmd.Context()) storageID := meta.StorageIDPrm{} storageID.SetAddress(addr) diff --git a/cmd/frostfs-lens/internal/meta/list-garbage.go b/cmd/frostfs-lens/internal/meta/list-garbage.go index 61b10ca1f..6b27a232f 100644 --- a/cmd/frostfs-lens/internal/meta/list-garbage.go +++ b/cmd/frostfs-lens/internal/meta/list-garbage.go @@ -19,7 +19,7 @@ func init() { func listGarbageFunc(cmd *cobra.Command, _ []string) { db := openMeta(cmd) - defer db.Close() + defer db.Close(cmd.Context()) var garbPrm meta.GarbageIterationPrm garbPrm.SetHandler( diff --git a/cmd/frostfs-lens/internal/meta/list-graveyard.go b/cmd/frostfs-lens/internal/meta/list-graveyard.go index 19a93691c..45642e74b 100644 --- a/cmd/frostfs-lens/internal/meta/list-graveyard.go +++ b/cmd/frostfs-lens/internal/meta/list-graveyard.go @@ -19,7 +19,7 @@ func init() { func listGraveyardFunc(cmd *cobra.Command, _ []string) { db := openMeta(cmd) - defer db.Close() + defer db.Close(cmd.Context()) var gravePrm meta.GraveyardIterationPrm gravePrm.SetHandler( diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index aa92e5ec5..5af37865f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -397,16 +397,16 @@ type internals struct { } // starts node's maintenance. -func (c *cfg) startMaintenance() { +func (c *cfg) startMaintenance(ctx context.Context) { c.isMaintenance.Store(true) c.cfgNetmap.state.setControlNetmapStatus(control.NetmapStatus_MAINTENANCE) - c.log.Info(context.Background(), logs.FrostFSNodeStartedLocalNodesMaintenance) + c.log.Info(ctx, logs.FrostFSNodeStartedLocalNodesMaintenance) } // stops node's maintenance. -func (c *internals) stopMaintenance() { +func (c *internals) stopMaintenance(ctx context.Context) { if c.isMaintenance.CompareAndSwap(true, false) { - c.log.Info(context.Background(), logs.FrostFSNodeStoppedLocalNodesMaintenance) + c.log.Info(ctx, logs.FrostFSNodeStoppedLocalNodesMaintenance) } } @@ -1129,10 +1129,10 @@ func initLocalStorage(ctx context.Context, c *cfg) { }) } -func initAccessPolicyEngine(_ context.Context, c *cfg) { +func initAccessPolicyEngine(ctx context.Context, c *cfg) { var localOverrideDB chainbase.LocalOverrideDatabase if nodeconfig.PersistentPolicyRules(c.appCfg).Path() == "" { - c.log.Warn(context.Background(), logs.FrostFSNodePersistentRuleStorageDBPathIsNotSetInmemoryWillBeUsed) + c.log.Warn(ctx, logs.FrostFSNodePersistentRuleStorageDBPathIsNotSetInmemoryWillBeUsed) localOverrideDB = chainbase.NewInmemoryLocalOverrideDatabase() } else { localOverrideDB = chainbase.NewBoltLocalOverrideDatabase( @@ -1157,7 +1157,7 @@ func initAccessPolicyEngine(_ context.Context, c *cfg) { c.onShutdown(func() { if err := ape.LocalOverrideDatabaseCore().Close(); err != nil { - c.log.Warn(context.Background(), logs.FrostFSNodeAccessPolicyEngineClosingFailure, + c.log.Warn(ctx, logs.FrostFSNodeAccessPolicyEngineClosingFailure, zap.Error(err), ) } @@ -1206,10 +1206,10 @@ func (c *cfg) setContractNodeInfo(ni *netmap.NodeInfo) { c.cfgNetmap.state.setNodeInfo(ni) } -func (c *cfg) updateContractNodeInfo(epoch uint64) { +func (c *cfg) updateContractNodeInfo(ctx context.Context, epoch uint64) { ni, err := c.netmapLocalNodeState(epoch) if err != nil { - c.log.Error(context.Background(), logs.FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch, + c.log.Error(ctx, logs.FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch, zap.Uint64("epoch", epoch), zap.String("error", err.Error())) return @@ -1221,19 +1221,19 @@ func (c *cfg) updateContractNodeInfo(epoch uint64) { // bootstrapWithState calls "addPeer" method of the Sidechain Netmap contract // with the binary-encoded information from the current node's configuration. // The state is set using the provided setter which MUST NOT be nil. -func (c *cfg) bootstrapWithState(stateSetter func(*netmap.NodeInfo)) error { +func (c *cfg) bootstrapWithState(ctx context.Context, stateSetter func(*netmap.NodeInfo)) error { ni := c.cfgNodeInfo.localInfo stateSetter(&ni) prm := nmClient.AddPeerPrm{} prm.SetNodeInfo(ni) - return c.cfgNetmap.wrapper.AddPeer(prm) + return c.cfgNetmap.wrapper.AddPeer(ctx, prm) } // bootstrapOnline calls cfg.bootstrapWithState with "online" state. -func bootstrapOnline(c *cfg) error { - return c.bootstrapWithState(func(ni *netmap.NodeInfo) { +func bootstrapOnline(ctx context.Context, c *cfg) error { + return c.bootstrapWithState(ctx, func(ni *netmap.NodeInfo) { ni.SetStatus(netmap.Online) }) } @@ -1241,21 +1241,21 @@ func bootstrapOnline(c *cfg) error { // bootstrap calls bootstrapWithState with: // - "maintenance" state if maintenance is in progress on the current node // - "online", otherwise -func (c *cfg) bootstrap() error { +func (c *cfg) bootstrap(ctx context.Context) error { // switch to online except when under maintenance st := c.cfgNetmap.state.controlNetmapStatus() if st == control.NetmapStatus_MAINTENANCE { - c.log.Info(context.Background(), logs.FrostFSNodeBootstrappingWithTheMaintenanceState) - return c.bootstrapWithState(func(ni *netmap.NodeInfo) { + c.log.Info(ctx, logs.FrostFSNodeBootstrappingWithTheMaintenanceState) + return c.bootstrapWithState(ctx, func(ni *netmap.NodeInfo) { ni.SetStatus(netmap.Maintenance) }) } - c.log.Info(context.Background(), logs.FrostFSNodeBootstrappingWithOnlineState, + c.log.Info(ctx, logs.FrostFSNodeBootstrappingWithOnlineState, zap.Stringer("previous", st), ) - return bootstrapOnline(c) + return bootstrapOnline(ctx, c) } // needBootstrap checks if local node should be registered in network on bootup. @@ -1282,7 +1282,7 @@ func (c *cfg) signalWatcher(ctx context.Context) { case <-ch: c.log.Info(ctx, logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) - c.shutdown() + c.shutdown(ctx) c.log.Info(ctx, logs.FrostFSNodeTerminationSignalProcessingIsComplete) return @@ -1290,7 +1290,7 @@ func (c *cfg) signalWatcher(ctx context.Context) { c.log.Warn(ctx, logs.FrostFSNodeInternalApplicationError, zap.String("message", err.Error())) - c.shutdown() + c.shutdown(ctx) c.log.Info(ctx, logs.FrostFSNodeInternalErrorProcessingIsComplete) return @@ -1302,7 +1302,7 @@ func (c *cfg) signalWatcher(ctx context.Context) { case <-ch: c.log.Info(ctx, logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) - c.shutdown() + c.shutdown(ctx) c.log.Info(ctx, logs.FrostFSNodeTerminationSignalProcessingIsComplete) return @@ -1310,7 +1310,7 @@ func (c *cfg) signalWatcher(ctx context.Context) { c.log.Warn(ctx, logs.FrostFSNodeInternalApplicationError, zap.String("message", err.Error())) - c.shutdown() + c.shutdown(ctx) c.log.Info(ctx, logs.FrostFSNodeInternalErrorProcessingIsComplete) return @@ -1322,11 +1322,11 @@ func (c *cfg) signalWatcher(ctx context.Context) { func (c *cfg) reloadConfig(ctx context.Context) { c.log.Info(ctx, logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) - if !c.compareAndSwapHealthStatus(control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) { + if !c.compareAndSwapHealthStatus(ctx, control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) { c.log.Info(ctx, logs.FrostFSNodeSIGHUPSkip) return } - defer c.compareAndSwapHealthStatus(control.HealthStatus_RECONFIGURING, control.HealthStatus_READY) + defer c.compareAndSwapHealthStatus(ctx, control.HealthStatus_RECONFIGURING, control.HealthStatus_READY) err := c.reloadAppConfig() if err != nil { @@ -1388,7 +1388,7 @@ func (c *cfg) getComponents(ctx context.Context, logPrm *logger.Prm) []dCmp { components = append(components, dCmp{"logger", logPrm.Reload}) components = append(components, dCmp{"runtime", func() error { - setRuntimeParameters(c) + setRuntimeParameters(ctx, c) return nil }}) components = append(components, dCmp{"audit", func() error { @@ -1474,14 +1474,14 @@ func (c *cfg) createContainerInfoProvider(ctx context.Context) container.InfoPro }) } -func (c *cfg) shutdown() { - old := c.swapHealthStatus(control.HealthStatus_SHUTTING_DOWN) +func (c *cfg) shutdown(ctx context.Context) { + old := c.swapHealthStatus(ctx, control.HealthStatus_SHUTTING_DOWN) if old == control.HealthStatus_SHUTTING_DOWN { - c.log.Info(context.Background(), logs.FrostFSNodeShutdownSkip) + c.log.Info(ctx, logs.FrostFSNodeShutdownSkip) return } if old == control.HealthStatus_STARTING { - c.log.Warn(context.Background(), logs.FrostFSNodeShutdownWhenNotReady) + c.log.Warn(ctx, logs.FrostFSNodeShutdownWhenNotReady) } c.ctxCancel() @@ -1491,6 +1491,6 @@ func (c *cfg) shutdown() { } if err := sdnotify.ClearStatus(); err != nil { - c.log.Error(context.Background(), logs.FailedToReportStatusToSystemd, zap.Error(err)) + c.log.Error(ctx, logs.FailedToReportStatusToSystemd, zap.Error(err)) } } diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 3f75be235..d3e1b2766 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -237,10 +237,10 @@ type morphContainerWriter struct { neoClient *cntClient.Client } -func (m morphContainerWriter) Put(cnr containerCore.Container) (*cid.ID, error) { - return cntClient.Put(m.neoClient, cnr) +func (m morphContainerWriter) Put(ctx context.Context, cnr containerCore.Container) (*cid.ID, error) { + return cntClient.Put(ctx, m.neoClient, cnr) } -func (m morphContainerWriter) Delete(witness containerCore.RemovalWitness) error { - return cntClient.Delete(m.neoClient, witness) +func (m morphContainerWriter) Delete(ctx context.Context, witness containerCore.RemovalWitness) error { + return cntClient.Delete(ctx, m.neoClient, witness) } diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index ffac23eec..ecd82bba5 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -16,7 +16,7 @@ import ( const serviceNameControl = "control" -func initControlService(c *cfg) { +func initControlService(ctx context.Context, c *cfg) { endpoint := controlconfig.GRPC(c.appCfg).Endpoint() if endpoint == controlconfig.GRPCEndpointDefault { return @@ -46,14 +46,14 @@ func initControlService(c *cfg) { lis, err := net.Listen("tcp", endpoint) if err != nil { - c.log.Error(context.Background(), logs.FrostFSNodeCantListenGRPCEndpointControl, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeCantListenGRPCEndpointControl, zap.Error(err)) return } c.cfgControlService.server = grpc.NewServer() c.onShutdown(func() { - stopGRPC("FrostFS Control API", c.cfgControlService.server, c.log) + stopGRPC(ctx, "FrostFS Control API", c.cfgControlService.server, c.log) }) control.RegisterControlServiceServer(c.cfgControlService.server, ctlSvc) @@ -72,23 +72,23 @@ func (c *cfg) NetmapStatus() control.NetmapStatus { return c.cfgNetmap.state.controlNetmapStatus() } -func (c *cfg) setHealthStatus(st control.HealthStatus) { - c.notifySystemd(st) +func (c *cfg) setHealthStatus(ctx context.Context, st control.HealthStatus) { + c.notifySystemd(ctx, st) c.healthStatus.Store(int32(st)) c.metricsCollector.State().SetHealth(int32(st)) } -func (c *cfg) compareAndSwapHealthStatus(oldSt, newSt control.HealthStatus) (swapped bool) { +func (c *cfg) compareAndSwapHealthStatus(ctx context.Context, oldSt, newSt control.HealthStatus) (swapped bool) { if swapped = c.healthStatus.CompareAndSwap(int32(oldSt), int32(newSt)); swapped { - c.notifySystemd(newSt) + c.notifySystemd(ctx, newSt) c.metricsCollector.State().SetHealth(int32(newSt)) } return } -func (c *cfg) swapHealthStatus(st control.HealthStatus) (old control.HealthStatus) { +func (c *cfg) swapHealthStatus(ctx context.Context, st control.HealthStatus) (old control.HealthStatus) { old = control.HealthStatus(c.healthStatus.Swap(int32(st))) - c.notifySystemd(st) + c.notifySystemd(ctx, st) c.metricsCollector.State().SetHealth(int32(st)) return } @@ -97,7 +97,7 @@ func (c *cfg) HealthStatus() control.HealthStatus { return control.HealthStatus(c.healthStatus.Load()) } -func (c *cfg) notifySystemd(st control.HealthStatus) { +func (c *cfg) notifySystemd(ctx context.Context, st control.HealthStatus) { if !c.sdNotify { return } @@ -113,6 +113,6 @@ func (c *cfg) notifySystemd(st control.HealthStatus) { err = sdnotify.Status(fmt.Sprintf("%v", st)) } if err != nil { - c.log.Error(context.Background(), logs.FailedToReportStatusToSystemd, zap.Error(err)) + c.log.Error(ctx, logs.FailedToReportStatusToSystemd, zap.Error(err)) } } diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index 271810ee6..6105be861 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -19,11 +19,11 @@ import ( const maxRecvMsgSize = 256 << 20 -func initGRPC(c *cfg) { +func initGRPC(ctx context.Context, c *cfg) { var endpointsToReconnect []string var successCount int grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) { - serverOpts, ok := getGrpcServerOpts(c, sc) + serverOpts, ok := getGrpcServerOpts(ctx, c, sc) if !ok { return } @@ -31,7 +31,7 @@ func initGRPC(c *cfg) { lis, err := net.Listen("tcp", sc.Endpoint()) if err != nil { c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(sc.Endpoint()) - c.log.Error(context.Background(), logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) endpointsToReconnect = append(endpointsToReconnect, sc.Endpoint()) return } @@ -40,7 +40,7 @@ func initGRPC(c *cfg) { srv := grpc.NewServer(serverOpts...) c.onShutdown(func() { - stopGRPC("FrostFS Public API", srv, c.log) + stopGRPC(ctx, "FrostFS Public API", srv, c.log) }) c.cfgGRPC.append(sc.Endpoint(), lis, srv) @@ -53,11 +53,11 @@ func initGRPC(c *cfg) { c.cfgGRPC.reconnectTimeout = grpcconfig.ReconnectTimeout(c.appCfg) for _, endpoint := range endpointsToReconnect { - scheduleReconnect(endpoint, c) + scheduleReconnect(ctx, endpoint, c) } } -func scheduleReconnect(endpoint string, c *cfg) { +func scheduleReconnect(ctx context.Context, endpoint string, c *cfg) { c.wg.Add(1) go func() { defer c.wg.Done() @@ -66,7 +66,7 @@ func scheduleReconnect(endpoint string, c *cfg) { for { select { case <-t.C: - if tryReconnect(endpoint, c) { + if tryReconnect(ctx, endpoint, c) { return } case <-c.done: @@ -76,20 +76,20 @@ func scheduleReconnect(endpoint string, c *cfg) { }() } -func tryReconnect(endpoint string, c *cfg) bool { - c.log.Info(context.Background(), logs.FrostFSNodeGRPCReconnecting, zap.String("endpoint", endpoint)) +func tryReconnect(ctx context.Context, endpoint string, c *cfg) bool { + c.log.Info(ctx, logs.FrostFSNodeGRPCReconnecting, zap.String("endpoint", endpoint)) - serverOpts, found := getGRPCEndpointOpts(endpoint, c) + serverOpts, found := getGRPCEndpointOpts(ctx, endpoint, c) if !found { - c.log.Warn(context.Background(), logs.FrostFSNodeGRPCServerConfigNotFound, zap.String("endpoint", endpoint)) + c.log.Warn(ctx, logs.FrostFSNodeGRPCServerConfigNotFound, zap.String("endpoint", endpoint)) return true } lis, err := net.Listen("tcp", endpoint) if err != nil { c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(endpoint) - c.log.Error(context.Background(), logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) - c.log.Warn(context.Background(), logs.FrostFSNodeGRPCReconnectFailed, zap.Duration("next_try_in", c.cfgGRPC.reconnectTimeout)) + c.log.Error(ctx, logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err)) + c.log.Warn(ctx, logs.FrostFSNodeGRPCReconnectFailed, zap.Duration("next_try_in", c.cfgGRPC.reconnectTimeout)) return false } c.metricsCollector.GrpcServerMetrics().MarkHealthy(endpoint) @@ -97,16 +97,16 @@ func tryReconnect(endpoint string, c *cfg) bool { srv := grpc.NewServer(serverOpts...) c.onShutdown(func() { - stopGRPC("FrostFS Public API", srv, c.log) + stopGRPC(ctx, "FrostFS Public API", srv, c.log) }) c.cfgGRPC.appendAndHandle(endpoint, lis, srv) - c.log.Info(context.Background(), logs.FrostFSNodeGRPCReconnectedSuccessfully, zap.String("endpoint", endpoint)) + c.log.Info(ctx, logs.FrostFSNodeGRPCReconnectedSuccessfully, zap.String("endpoint", endpoint)) return true } -func getGRPCEndpointOpts(endpoint string, c *cfg) (result []grpc.ServerOption, found bool) { +func getGRPCEndpointOpts(ctx context.Context, endpoint string, c *cfg) (result []grpc.ServerOption, found bool) { unlock := c.LockAppConfigShared() defer unlock() grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) { @@ -117,7 +117,7 @@ func getGRPCEndpointOpts(endpoint string, c *cfg) (result []grpc.ServerOption, f return } var ok bool - result, ok = getGrpcServerOpts(c, sc) + result, ok = getGrpcServerOpts(ctx, c, sc) if !ok { return } @@ -126,7 +126,7 @@ func getGRPCEndpointOpts(endpoint string, c *cfg) (result []grpc.ServerOption, f return } -func getGrpcServerOpts(c *cfg, sc *grpcconfig.Config) ([]grpc.ServerOption, bool) { +func getGrpcServerOpts(ctx context.Context, c *cfg, sc *grpcconfig.Config) ([]grpc.ServerOption, bool) { serverOpts := []grpc.ServerOption{ grpc.MaxRecvMsgSize(maxRecvMsgSize), grpc.ChainUnaryInterceptor( @@ -144,7 +144,7 @@ func getGrpcServerOpts(c *cfg, sc *grpcconfig.Config) ([]grpc.ServerOption, bool if tlsCfg != nil { cert, err := tls.LoadX509KeyPair(tlsCfg.CertificateFile(), tlsCfg.KeyFile()) if err != nil { - c.log.Error(context.Background(), logs.FrostFSNodeCouldNotReadCertificateFromFile, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeCouldNotReadCertificateFromFile, zap.Error(err)) return nil, false } @@ -175,38 +175,38 @@ func getGrpcServerOpts(c *cfg, sc *grpcconfig.Config) ([]grpc.ServerOption, bool return serverOpts, true } -func serveGRPC(c *cfg) { +func serveGRPC(ctx context.Context, c *cfg) { c.cfgGRPC.performAndSave(func(e string, l net.Listener, s *grpc.Server) { c.wg.Add(1) go func() { defer func() { - c.log.Info(context.Background(), logs.FrostFSNodeStopListeningGRPCEndpoint, + c.log.Info(ctx, logs.FrostFSNodeStopListeningGRPCEndpoint, zap.Stringer("endpoint", l.Addr()), ) c.wg.Done() }() - c.log.Info(context.Background(), logs.FrostFSNodeStartListeningEndpoint, + c.log.Info(ctx, logs.FrostFSNodeStartListeningEndpoint, zap.String("service", "gRPC"), zap.Stringer("endpoint", l.Addr()), ) if err := s.Serve(l); err != nil { c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(e) - c.log.Error(context.Background(), logs.FrostFSNodeGRPCServerError, zap.Error(err)) + c.log.Error(ctx, logs.FrostFSNodeGRPCServerError, zap.Error(err)) c.cfgGRPC.dropConnection(e) - scheduleReconnect(e, c) + scheduleReconnect(ctx, e, c) } }() }) } -func stopGRPC(name string, s *grpc.Server, l *logger.Logger) { +func stopGRPC(ctx context.Context, name string, s *grpc.Server, l *logger.Logger) { l = l.With(zap.String("name", name)) - l.Info(context.Background(), logs.FrostFSNodeStoppingGRPCServer) + l.Info(ctx, logs.FrostFSNodeStoppingGRPCServer) // GracefulStop() may freeze forever, see #1270 done := make(chan struct{}) @@ -218,9 +218,9 @@ func stopGRPC(name string, s *grpc.Server, l *logger.Logger) { select { case <-done: case <-time.After(1 * time.Minute): - l.Info(context.Background(), logs.FrostFSNodeGRPCCannotShutdownGracefullyForcingStop) + l.Info(ctx, logs.FrostFSNodeGRPCCannotShutdownGracefullyForcingStop) s.Stop() } - l.Info(context.Background(), logs.FrostFSNodeGRPCServerStoppedSuccessfully) + l.Info(ctx, logs.FrostFSNodeGRPCServerStoppedSuccessfully) } diff --git a/cmd/frostfs-node/httpcomponent.go b/cmd/frostfs-node/httpcomponent.go index a699a72a1..7346206ef 100644 --- a/cmd/frostfs-node/httpcomponent.go +++ b/cmd/frostfs-node/httpcomponent.go @@ -20,9 +20,9 @@ type httpComponent struct { preReload func(c *cfg) } -func (cmp *httpComponent) init(c *cfg) { +func (cmp *httpComponent) init(ctx context.Context, c *cfg) { if !cmp.enabled { - c.log.Info(context.Background(), cmp.name+" is disabled") + c.log.Info(ctx, cmp.name+" is disabled") return } // Init server with parameters @@ -39,14 +39,14 @@ func (cmp *httpComponent) init(c *cfg) { go func() { defer c.wg.Done() - c.log.Info(context.Background(), logs.FrostFSNodeStartListeningEndpoint, + c.log.Info(ctx, logs.FrostFSNodeStartListeningEndpoint, zap.String("service", cmp.name), zap.String("endpoint", cmp.address)) fatalOnErr(srv.Serve()) }() c.closers = append(c.closers, closer{ cmp.name, - func() { stopAndLog(c, cmp.name, srv.Shutdown) }, + func() { stopAndLog(ctx, c, cmp.name, srv.Shutdown) }, }) } @@ -62,7 +62,7 @@ func (cmp *httpComponent) reload(ctx context.Context) error { // Cleanup delCloser(cmp.cfg, cmp.name) // Init server with new parameters - cmp.init(cmp.cfg) + cmp.init(ctx, cmp.cfg) // Start worker if cmp.enabled { startWorker(ctx, cmp.cfg, *getWorker(cmp.cfg, cmp.name)) diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index cd42d5f1d..f8854ab3c 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -61,21 +61,21 @@ func main() { var ctx context.Context ctx, c.ctxCancel = context.WithCancel(context.Background()) - c.setHealthStatus(control.HealthStatus_STARTING) + c.setHealthStatus(ctx, control.HealthStatus_STARTING) initApp(ctx, c) bootUp(ctx, c) - c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY) + c.compareAndSwapHealthStatus(ctx, control.HealthStatus_STARTING, control.HealthStatus_READY) wait(c) } -func initAndLog(c *cfg, name string, initializer func(*cfg)) { - c.log.Info(context.Background(), fmt.Sprintf("initializing %s service...", name)) +func initAndLog(ctx context.Context, c *cfg, name string, initializer func(*cfg)) { + c.log.Info(ctx, fmt.Sprintf("initializing %s service...", name)) initializer(c) - c.log.Info(context.Background(), name+" service has been successfully initialized") + c.log.Info(ctx, name+" service has been successfully initialized") } func initApp(ctx context.Context, c *cfg) { @@ -85,38 +85,38 @@ func initApp(ctx context.Context, c *cfg) { c.wg.Done() }() - setRuntimeParameters(c) + setRuntimeParameters(ctx, c) metrics, _ := metricsComponent(c) - initAndLog(c, "profiler", initProfilerService) - initAndLog(c, metrics.name, metrics.init) + initAndLog(ctx, c, "profiler", func(c *cfg) { initProfilerService(ctx, c) }) + initAndLog(ctx, c, metrics.name, func(c *cfg) { metrics.init(ctx, c) }) - initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) }) + initAndLog(ctx, c, "tracing", func(c *cfg) { initTracing(ctx, c) }) initLocalStorage(ctx, c) - initAndLog(c, "storage engine", func(c *cfg) { + initAndLog(ctx, c, "storage engine", func(c *cfg) { fatalOnErr(c.cfgObject.cfgLocalStorage.localStorage.Open(ctx)) fatalOnErr(c.cfgObject.cfgLocalStorage.localStorage.Init(ctx)) }) - initAndLog(c, "gRPC", initGRPC) - initAndLog(c, "netmap", func(c *cfg) { initNetmapService(ctx, c) }) + initAndLog(ctx, c, "gRPC", func(c *cfg) { initGRPC(ctx, c) }) + initAndLog(ctx, c, "netmap", func(c *cfg) { initNetmapService(ctx, c) }) initAccessPolicyEngine(ctx, c) - initAndLog(c, "access policy engine", func(c *cfg) { + initAndLog(ctx, c, "access policy engine", func(c *cfg) { fatalOnErr(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalOverrideDatabaseCore().Open(ctx)) fatalOnErr(c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalOverrideDatabaseCore().Init()) }) - initAndLog(c, "accounting", func(c *cfg) { initAccountingService(ctx, c) }) - initAndLog(c, "container", func(c *cfg) { initContainerService(ctx, c) }) - initAndLog(c, "session", initSessionService) - initAndLog(c, "object", initObjectService) - initAndLog(c, "tree", initTreeService) - initAndLog(c, "apemanager", initAPEManagerService) - initAndLog(c, "control", initControlService) + initAndLog(ctx, c, "accounting", func(c *cfg) { initAccountingService(ctx, c) }) + initAndLog(ctx, c, "container", func(c *cfg) { initContainerService(ctx, c) }) + initAndLog(ctx, c, "session", initSessionService) + initAndLog(ctx, c, "object", initObjectService) + initAndLog(ctx, c, "tree", initTreeService) + initAndLog(ctx, c, "apemanager", initAPEManagerService) + initAndLog(ctx, c, "control", func(c *cfg) { initControlService(ctx, c) }) - initAndLog(c, "morph notifications", func(c *cfg) { listenMorphNotifications(ctx, c) }) + initAndLog(ctx, c, "morph notifications", func(c *cfg) { listenMorphNotifications(ctx, c) }) } func runAndLog(ctx context.Context, c *cfg, name string, logSuccess bool, starter func(context.Context, *cfg)) { @@ -128,24 +128,24 @@ func runAndLog(ctx context.Context, c *cfg, name string, logSuccess bool, starte } } -func stopAndLog(c *cfg, name string, stopper func() error) { - c.log.Debug(context.Background(), fmt.Sprintf("shutting down %s service", name)) +func stopAndLog(ctx context.Context, c *cfg, name string, stopper func(context.Context) error) { + c.log.Debug(ctx, fmt.Sprintf("shutting down %s service", name)) - err := stopper() + err := stopper(ctx) if err != nil { - c.log.Debug(context.Background(), fmt.Sprintf("could not shutdown %s server", name), + c.log.Debug(ctx, fmt.Sprintf("could not shutdown %s server", name), zap.String("error", err.Error()), ) } - c.log.Debug(context.Background(), name+" service has been stopped") + c.log.Debug(ctx, name+" service has been stopped") } func bootUp(ctx context.Context, c *cfg) { - runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(c) }) + runAndLog(ctx, c, "gRPC", false, func(_ context.Context, c *cfg) { serveGRPC(ctx, c) }) runAndLog(ctx, c, "notary", true, makeAndWaitNotaryDeposit) - bootstrapNode(c) + bootstrapNode(ctx, c) startWorkers(ctx, c) } diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 3e010b181..ca9f4fe3e 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -129,7 +129,7 @@ func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) { return } - tx, vub, err := makeNotaryDeposit(c) + tx, vub, err := makeNotaryDeposit(ctx, c) fatalOnErr(err) if tx.Equals(util.Uint256{}) { @@ -144,7 +144,7 @@ func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) { fatalOnErr(err) } -func makeNotaryDeposit(c *cfg) (util.Uint256, uint32, error) { +func makeNotaryDeposit(ctx context.Context, c *cfg) (util.Uint256, uint32, error) { const ( // gasMultiplier defines how many times more the notary // balance must be compared to the GAS balance of the node: @@ -161,7 +161,7 @@ func makeNotaryDeposit(c *cfg) (util.Uint256, uint32, error) { return util.Uint256{}, 0, fmt.Errorf("could not calculate notary deposit: %w", err) } - return c.cfgMorph.client.DepositEndlessNotary(depositAmount) + return c.cfgMorph.client.DepositEndlessNotary(ctx, depositAmount) } var ( @@ -256,7 +256,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { registerNotificationHandlers(c.cfgNetmap.scriptHash, lis, c.cfgNetmap.parsers, c.cfgNetmap.subscribers) registerNotificationHandlers(c.cfgContainer.scriptHash, lis, c.cfgContainer.parsers, c.cfgContainer.subscribers) - registerBlockHandler(lis, func(block *block.Block) { + registerBlockHandler(lis, func(ctx context.Context, block *block.Block) { c.log.Debug(ctx, logs.FrostFSNodeNewBlock, zap.Uint32("index", block.Index)) err = c.persistate.SetUInt32(persistateSideChainLastBlockKey, block.Index) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 35ab4d575..6df947954 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -145,7 +145,7 @@ func initNetmapService(ctx context.Context, c *cfg) { c.initMorphComponents(ctx) - initNetmapState(c) + initNetmapState(ctx, c) server := netmapTransportGRPC.New( netmapService.NewSignService( @@ -182,20 +182,20 @@ func addNewEpochNotificationHandlers(c *cfg) { addNewEpochAsyncNotificationHandler(c, func(ctx context.Context, ev event.Event) { e := ev.(netmapEvent.NewEpoch).EpochNumber() - c.updateContractNodeInfo(e) + c.updateContractNodeInfo(ctx, e) if !c.needBootstrap() || c.cfgNetmap.reBoostrapTurnedOff.Load() { // fixes #470 return } - if err := c.bootstrap(); err != nil { + if err := c.bootstrap(ctx); err != nil { c.log.Warn(ctx, logs.FrostFSNodeCantSendRebootstrapTx, zap.Error(err)) } }) if c.cfgMorph.notaryEnabled { addNewEpochAsyncNotificationHandler(c, func(ctx context.Context, _ event.Event) { - _, _, err := makeNotaryDeposit(c) + _, _, err := makeNotaryDeposit(ctx, c) if err != nil { c.log.Error(ctx, logs.FrostFSNodeCouldNotMakeNotaryDeposit, zap.String("error", err.Error()), @@ -207,13 +207,13 @@ func addNewEpochNotificationHandlers(c *cfg) { // bootstrapNode adds current node to the Network map. // Must be called after initNetmapService. -func bootstrapNode(c *cfg) { +func bootstrapNode(ctx context.Context, c *cfg) { if c.needBootstrap() { if c.IsMaintenance() { - c.log.Info(context.Background(), logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) + c.log.Info(ctx, logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) return } - err := c.bootstrap() + err := c.bootstrap(ctx) fatalOnErrDetails("bootstrap error", err) } } @@ -240,17 +240,17 @@ func setNetmapNotificationParser(c *cfg, sTyp string, p event.NotificationParser // initNetmapState inits current Network map state. // Must be called after Morph components initialization. -func initNetmapState(c *cfg) { +func initNetmapState(ctx context.Context, c *cfg) { epoch, err := c.cfgNetmap.wrapper.Epoch() fatalOnErrDetails("could not initialize current epoch number", err) var ni *netmapSDK.NodeInfo - ni, err = c.netmapInitLocalNodeState(epoch) + ni, err = c.netmapInitLocalNodeState(ctx, epoch) fatalOnErrDetails("could not init network state", err) stateWord := nodeState(ni) - c.log.Info(context.Background(), logs.FrostFSNodeInitialNetworkState, + c.log.Info(ctx, logs.FrostFSNodeInitialNetworkState, zap.Uint64("epoch", epoch), zap.String("state", stateWord), ) @@ -279,7 +279,7 @@ func nodeState(ni *netmapSDK.NodeInfo) string { return "undefined" } -func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { +func (c *cfg) netmapInitLocalNodeState(ctx context.Context, epoch uint64) (*netmapSDK.NodeInfo, error) { nmNodes, err := c.cfgNetmap.wrapper.GetCandidates() if err != nil { return nil, err @@ -307,7 +307,7 @@ func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error if nmState != candidateState { // This happens when the node was switched to maintenance without epoch tick. // We expect it to continue staying in maintenance. - c.log.Info(context.Background(), logs.CandidateStatusPriority, + c.log.Info(ctx, logs.CandidateStatusPriority, zap.String("netmap", nmState), zap.String("candidate", candidateState)) } @@ -353,16 +353,16 @@ func addNewEpochAsyncNotificationHandler(c *cfg, h event.Handler) { var errRelayBootstrap = errors.New("setting netmap status is forbidden in relay mode") -func (c *cfg) SetNetmapStatus(st control.NetmapStatus) error { +func (c *cfg) SetNetmapStatus(ctx context.Context, st control.NetmapStatus) error { switch st { default: return fmt.Errorf("unsupported status %v", st) case control.NetmapStatus_MAINTENANCE: - return c.setMaintenanceStatus(false) + return c.setMaintenanceStatus(ctx, false) case control.NetmapStatus_ONLINE, control.NetmapStatus_OFFLINE: } - c.stopMaintenance() + c.stopMaintenance(ctx) if !c.needBootstrap() { return errRelayBootstrap @@ -370,12 +370,12 @@ func (c *cfg) SetNetmapStatus(st control.NetmapStatus) error { if st == control.NetmapStatus_ONLINE { c.cfgNetmap.reBoostrapTurnedOff.Store(false) - return bootstrapOnline(c) + return bootstrapOnline(ctx, c) } c.cfgNetmap.reBoostrapTurnedOff.Store(true) - return c.updateNetMapState(func(*nmClient.UpdatePeerPrm) {}) + return c.updateNetMapState(ctx, func(*nmClient.UpdatePeerPrm) {}) } func (c *cfg) GetNetmapStatus() (control.NetmapStatus, uint64, error) { @@ -387,11 +387,11 @@ func (c *cfg) GetNetmapStatus() (control.NetmapStatus, uint64, error) { return st, epoch, nil } -func (c *cfg) ForceMaintenance() error { - return c.setMaintenanceStatus(true) +func (c *cfg) ForceMaintenance(ctx context.Context) error { + return c.setMaintenanceStatus(ctx, true) } -func (c *cfg) setMaintenanceStatus(force bool) error { +func (c *cfg) setMaintenanceStatus(ctx context.Context, force bool) error { netSettings, err := c.cfgNetmap.wrapper.ReadNetworkConfiguration() if err != nil { err = fmt.Errorf("read network settings to check maintenance allowance: %w", err) @@ -400,10 +400,10 @@ func (c *cfg) setMaintenanceStatus(force bool) error { } if err == nil || force { - c.startMaintenance() + c.startMaintenance(ctx) if err == nil { - err = c.updateNetMapState((*nmClient.UpdatePeerPrm).SetMaintenance) + err = c.updateNetMapState(ctx, (*nmClient.UpdatePeerPrm).SetMaintenance) } if err != nil { @@ -416,12 +416,12 @@ func (c *cfg) setMaintenanceStatus(force bool) error { // calls UpdatePeerState operation of Netmap contract's client for the local node. // State setter is used to specify node state to switch to. -func (c *cfg) updateNetMapState(stateSetter func(*nmClient.UpdatePeerPrm)) error { +func (c *cfg) updateNetMapState(ctx context.Context, stateSetter func(*nmClient.UpdatePeerPrm)) error { var prm nmClient.UpdatePeerPrm prm.SetKey(c.key.PublicKey().Bytes()) stateSetter(&prm) - _, err := c.cfgNetmap.wrapper.UpdatePeerState(prm) + _, err := c.cfgNetmap.wrapper.UpdatePeerState(ctx, prm) return err } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index cad6d5ee3..c4205a620 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -66,11 +66,11 @@ func (c *cfg) MaxObjectSize() uint64 { return sz } -func (s *objectSvc) Put() (objectService.PutObjectStream, error) { +func (s *objectSvc) Put(_ context.Context) (objectService.PutObjectStream, error) { return s.put.Put() } -func (s *objectSvc) Patch() (objectService.PatchObjectStream, error) { +func (s *objectSvc) Patch(_ context.Context) (objectService.PatchObjectStream, error) { return s.patch.Patch() } diff --git a/cmd/frostfs-node/pprof.go b/cmd/frostfs-node/pprof.go index dcd320146..5b40c8a88 100644 --- a/cmd/frostfs-node/pprof.go +++ b/cmd/frostfs-node/pprof.go @@ -1,17 +1,18 @@ package main import ( + "context" "runtime" profilerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/profiler" httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" ) -func initProfilerService(c *cfg) { +func initProfilerService(ctx context.Context, c *cfg) { tuneProfilers(c) pprof, _ := pprofComponent(c) - pprof.init(c) + pprof.init(ctx, c) } func pprofComponent(c *cfg) (*httpComponent, bool) { diff --git a/cmd/frostfs-node/runtime.go b/cmd/frostfs-node/runtime.go index 0b1000e70..f6d398574 100644 --- a/cmd/frostfs-node/runtime.go +++ b/cmd/frostfs-node/runtime.go @@ -10,17 +10,17 @@ import ( "go.uber.org/zap" ) -func setRuntimeParameters(c *cfg) { +func setRuntimeParameters(ctx context.Context, c *cfg) { if len(os.Getenv("GOMEMLIMIT")) != 0 { // default limit < yaml limit < app env limit < GOMEMLIMIT - c.log.Warn(context.Background(), logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) + c.log.Warn(ctx, logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT) return } memLimitBytes := runtime.GCMemoryLimitBytes(c.appCfg) previous := debug.SetMemoryLimit(memLimitBytes) if memLimitBytes != previous { - c.log.Info(context.Background(), logs.RuntimeSoftMemoryLimitUpdated, + c.log.Info(ctx, logs.RuntimeSoftMemoryLimitUpdated, zap.Int64("new_value", memLimitBytes), zap.Int64("old_value", previous)) } diff --git a/internal/audit/request.go b/internal/audit/request.go index 15a4a7960..17666ab4b 100644 --- a/internal/audit/request.go +++ b/internal/audit/request.go @@ -19,15 +19,15 @@ type Target interface { String() string } -func LogRequest(log *logger.Logger, operation string, req Request, target Target, status bool) { +func LogRequest(ctx context.Context, log *logger.Logger, operation string, req Request, target Target, status bool) { var key []byte if req != nil { key = req.GetVerificationHeader().GetBodySignature().GetKey() } - LogRequestWithKey(log, operation, key, target, status) + LogRequestWithKey(ctx, log, operation, key, target, status) } -func LogRequestWithKey(log *logger.Logger, operation string, key []byte, target Target, status bool) { +func LogRequestWithKey(ctx context.Context, log *logger.Logger, operation string, key []byte, target Target, status bool) { object, subject := NotDefined, NotDefined publicKey := crypto.UnmarshalPublicKey(key) @@ -39,7 +39,7 @@ func LogRequestWithKey(log *logger.Logger, operation string, key []byte, target object = target.String() } - log.Info(context.Background(), logs.AuditEventLogRecord, + log.Info(ctx, logs.AuditEventLogRecord, zap.String("operation", operation), zap.String("object", object), zap.String("subject", subject), diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index 5bc5c8bea..19b5d34e4 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -117,7 +117,7 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *objectSDK.Object, u } if !unprepared { - if err := v.validateSignatureKey(obj); err != nil { + if err := v.validateSignatureKey(ctx, obj); err != nil { return fmt.Errorf("(%T) could not validate signature key: %w", v, err) } @@ -134,7 +134,7 @@ func (v *FormatValidator) Validate(ctx context.Context, obj *objectSDK.Object, u return nil } -func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { +func (v *FormatValidator) validateSignatureKey(ctx context.Context, obj *objectSDK.Object) error { sig := obj.Signature() if sig == nil { return errMissingSignature @@ -156,7 +156,7 @@ func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { ownerID := obj.OwnerID() if token == nil && obj.ECHeader() != nil { - role, err := v.isIROrContainerNode(obj, binKey) + role, err := v.isIROrContainerNode(ctx, obj, binKey) if err != nil { return err } @@ -172,7 +172,7 @@ func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { } if v.verifyTokenIssuer { - role, err := v.isIROrContainerNode(obj, binKey) + role, err := v.isIROrContainerNode(ctx, obj, binKey) if err != nil { return err } @@ -190,7 +190,7 @@ func (v *FormatValidator) validateSignatureKey(obj *objectSDK.Object) error { return nil } -func (v *FormatValidator) isIROrContainerNode(obj *objectSDK.Object, signerKey []byte) (acl.Role, error) { +func (v *FormatValidator) isIROrContainerNode(ctx context.Context, obj *objectSDK.Object, signerKey []byte) (acl.Role, error) { cnrID, containerIDSet := obj.ContainerID() if !containerIDSet { return acl.RoleOthers, errNilCID @@ -204,7 +204,7 @@ func (v *FormatValidator) isIROrContainerNode(obj *objectSDK.Object, signerKey [ return acl.RoleOthers, fmt.Errorf("failed to get container (id=%s): %w", cnrID.EncodeToString(), err) } - res, err := v.senderClassifier.IsInnerRingOrContainerNode(signerKey, cnrID, cnr.Value) + res, err := v.senderClassifier.IsInnerRingOrContainerNode(ctx, signerKey, cnrID, cnr.Value) if err != nil { return acl.RoleOthers, err } diff --git a/pkg/core/object/sender_classifier.go b/pkg/core/object/sender_classifier.go index ed438c0b9..3b3650134 100644 --- a/pkg/core/object/sender_classifier.go +++ b/pkg/core/object/sender_classifier.go @@ -41,6 +41,7 @@ type ClassifyResult struct { } func (c SenderClassifier) Classify( + ctx context.Context, ownerID *user.ID, ownerKey *keys.PublicKey, idCnr cid.ID, @@ -58,14 +59,14 @@ func (c SenderClassifier) Classify( }, nil } - return c.IsInnerRingOrContainerNode(ownerKeyInBytes, idCnr, cnr) + return c.IsInnerRingOrContainerNode(ctx, ownerKeyInBytes, idCnr, cnr) } -func (c SenderClassifier) IsInnerRingOrContainerNode(ownerKeyInBytes []byte, idCnr cid.ID, cnr container.Container) (*ClassifyResult, error) { +func (c SenderClassifier) IsInnerRingOrContainerNode(ctx context.Context, ownerKeyInBytes []byte, idCnr cid.ID, cnr container.Container) (*ClassifyResult, error) { isInnerRingNode, err := c.isInnerRingKey(ownerKeyInBytes) if err != nil { // do not throw error, try best case matching - c.log.Debug(context.Background(), logs.V2CantCheckIfRequestFromInnerRing, + c.log.Debug(ctx, logs.V2CantCheckIfRequestFromInnerRing, zap.String("error", err.Error())) } else if isInnerRingNode { return &ClassifyResult{ @@ -82,7 +83,7 @@ func (c SenderClassifier) IsInnerRingOrContainerNode(ownerKeyInBytes []byte, idC // error might happen if request has `RoleOther` key and placement // is not possible for previous epoch, so // do not throw error, try best case matching - c.log.Debug(context.Background(), logs.V2CantCheckIfRequestFromContainerNode, + c.log.Debug(ctx, logs.V2CantCheckIfRequestFromContainerNode, zap.String("error", err.Error())) } else if isContainerNode { return &ClassifyResult{ diff --git a/pkg/innerring/blocktimer.go b/pkg/innerring/blocktimer.go index 3db504368..3f9d8df5f 100644 --- a/pkg/innerring/blocktimer.go +++ b/pkg/innerring/blocktimer.go @@ -29,7 +29,7 @@ type ( emitDuration uint32 // in blocks } - depositor func() (util.Uint256, error) + depositor func(context.Context) (util.Uint256, error) awaiter func(context.Context, util.Uint256) error ) @@ -66,11 +66,11 @@ func newEpochTimer(args *epochTimerArgs) *timer.BlockTimer { ) } -func newEmissionTimer(args *emitTimerArgs) *timer.BlockTimer { +func newEmissionTimer(ctx context.Context, args *emitTimerArgs) *timer.BlockTimer { return timer.NewBlockTimer( timer.StaticBlockMeter(args.emitDuration), func() { - args.ap.HandleGasEmission(timerEvent.NewAlphabetEmitTick{}) + args.ap.HandleGasEmission(ctx, timerEvent.NewAlphabetEmitTick{}) }, ) } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index e08a613c3..25f4ff034 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -35,7 +35,7 @@ import ( "google.golang.org/grpc" ) -func (s *Server) initNetmapProcessor(cfg *viper.Viper, +func (s *Server) initNetmapProcessor(ctx context.Context, cfg *viper.Viper, alphaSync event.Handler, ) error { locodeValidator, err := s.newLocodeValidator(cfg) @@ -48,10 +48,13 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, var netMapCandidateStateValidator statevalidation.NetMapCandidateValidator netMapCandidateStateValidator.SetNetworkSettings(netSettings) + poolSize := cfg.GetInt("workers.netmap") + s.log.Debug(ctx, logs.NetmapNetmapWorkerPool, zap.Int("size", poolSize)) + s.netmapProcessor, err = netmap.New(&netmap.Params{ Log: s.log, Metrics: s.irMetrics, - PoolSize: cfg.GetInt("workers.netmap"), + PoolSize: poolSize, NetmapClient: netmap.NewNetmapClient(s.netmapClient), EpochTimer: s, EpochState: s, @@ -205,7 +208,7 @@ func (s *Server) initTimers(ctx context.Context, cfg *viper.Viper) { s.addBlockTimer(s.epochTimer) // initialize emission timer - emissionTimer := newEmissionTimer(&emitTimerArgs{ + emissionTimer := newEmissionTimer(ctx, &emitTimerArgs{ ap: s.alphabetProcessor, emitDuration: cfg.GetUint32("timers.emit"), }) @@ -213,18 +216,20 @@ func (s *Server) initTimers(ctx context.Context, cfg *viper.Viper) { s.addBlockTimer(emissionTimer) } -func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { +func (s *Server) initAlphabetProcessor(ctx context.Context, cfg *viper.Viper) error { parsedWallets, err := parseWalletAddressesFromStrings(cfg.GetStringSlice("emit.extra_wallets")) if err != nil { return err } + poolSize := cfg.GetInt("workers.alphabet") + s.log.Debug(ctx, logs.AlphabetAlphabetWorkerPool, zap.Int("size", poolSize)) // create alphabet processor s.alphabetProcessor, err = alphabet.New(&alphabet.Params{ ParsedWallets: parsedWallets, Log: s.log, Metrics: s.irMetrics, - PoolSize: cfg.GetInt("workers.alphabet"), + PoolSize: poolSize, AlphabetContracts: s.contracts.alphabet, NetmapClient: s.netmapClient, MorphClient: s.morphClient, @@ -239,12 +244,14 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { return err } -func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.Client, frostfsIDClient *frostfsid.Client) error { +func (s *Server) initContainerProcessor(ctx context.Context, cfg *viper.Viper, cnrClient *container.Client, frostfsIDClient *frostfsid.Client) error { + poolSize := cfg.GetInt("workers.container") + s.log.Debug(ctx, logs.ContainerContainerWorkerPool, zap.Int("size", poolSize)) // container processor containerProcessor, err := cont.New(&cont.Params{ Log: s.log, Metrics: s.irMetrics, - PoolSize: cfg.GetInt("workers.container"), + PoolSize: poolSize, AlphabetState: s, ContainerClient: cnrClient, MorphClient: cnrClient.Morph(), @@ -258,12 +265,14 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C return bindMorphProcessor(containerProcessor, s) } -func (s *Server) initBalanceProcessor(cfg *viper.Viper, frostfsCli *frostfsClient.Client) error { +func (s *Server) initBalanceProcessor(ctx context.Context, cfg *viper.Viper, frostfsCli *frostfsClient.Client) error { + poolSize := cfg.GetInt("workers.balance") + s.log.Debug(ctx, logs.BalanceBalanceWorkerPool, zap.Int("size", poolSize)) // create balance processor balanceProcessor, err := balance.New(&balance.Params{ Log: s.log, Metrics: s.irMetrics, - PoolSize: cfg.GetInt("workers.balance"), + PoolSize: poolSize, FrostFSClient: frostfsCli, BalanceSC: s.contracts.balance, AlphabetState: s, @@ -276,15 +285,17 @@ func (s *Server) initBalanceProcessor(cfg *viper.Viper, frostfsCli *frostfsClien return bindMorphProcessor(balanceProcessor, s) } -func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper) error { +func (s *Server) initFrostFSMainnetProcessor(ctx context.Context, cfg *viper.Viper) error { if s.withoutMainNet { return nil } + poolSize := cfg.GetInt("workers.frostfs") + s.log.Debug(ctx, logs.FrostFSFrostfsWorkerPool, zap.Int("size", poolSize)) frostfsProcessor, err := frostfs.New(&frostfs.Params{ Log: s.log, Metrics: s.irMetrics, - PoolSize: cfg.GetInt("workers.frostfs"), + PoolSize: poolSize, FrostFSContract: s.contracts.frostfs, BalanceClient: s.balanceClient, NetmapClient: s.netmapClient, @@ -304,10 +315,10 @@ func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper) error { return bindMainnetProcessor(frostfsProcessor, s) } -func (s *Server) initGRPCServer(cfg *viper.Viper, log *logger.Logger, audit *atomic.Bool) error { +func (s *Server) initGRPCServer(ctx context.Context, cfg *viper.Viper, log *logger.Logger, audit *atomic.Bool) error { controlSvcEndpoint := cfg.GetString("control.grpc.endpoint") if controlSvcEndpoint == "" { - s.log.Info(context.Background(), logs.InnerringNoControlServerEndpointSpecified) + s.log.Info(ctx, logs.InnerringNoControlServerEndpointSpecified) return nil } @@ -403,7 +414,7 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { return result, nil } -func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClients) error { +func (s *Server) initProcessors(ctx context.Context, cfg *viper.Viper, morphClients *serverMorphClients) error { irf := s.createIRFetcher() s.statusIndex = newInnerRingIndexer( @@ -418,27 +429,27 @@ func (s *Server) initProcessors(cfg *viper.Viper, morphClients *serverMorphClien return err } - err = s.initNetmapProcessor(cfg, alphaSync) + err = s.initNetmapProcessor(ctx, cfg, alphaSync) if err != nil { return err } - err = s.initContainerProcessor(cfg, morphClients.CnrClient, morphClients.FrostFSIDClient) + err = s.initContainerProcessor(ctx, cfg, morphClients.CnrClient, morphClients.FrostFSIDClient) if err != nil { return err } - err = s.initBalanceProcessor(cfg, morphClients.FrostFSClient) + err = s.initBalanceProcessor(ctx, cfg, morphClients.FrostFSClient) if err != nil { return err } - err = s.initFrostFSMainnetProcessor(cfg) + err = s.initFrostFSMainnetProcessor(ctx, cfg) if err != nil { return err } - err = s.initAlphabetProcessor(cfg) + err = s.initAlphabetProcessor(ctx, cfg) return err } diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index e81ec6bca..4fe9cc084 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -157,7 +157,7 @@ func (s *Server) Start(ctx context.Context, intError chan<- error) (err error) { return err } - if s.IsAlphabet() { + if s.IsAlphabet(ctx) { err = s.initMainNotary(ctx) if err != nil { return err @@ -217,14 +217,14 @@ func (s *Server) Start(ctx context.Context, intError chan<- error) (err error) { } func (s *Server) registerMorphNewBlockEventHandler() { - s.morphListener.RegisterBlockHandler(func(b *block.Block) { - s.log.Debug(context.Background(), logs.InnerringNewBlock, + s.morphListener.RegisterBlockHandler(func(ctx context.Context, b *block.Block) { + s.log.Debug(ctx, logs.InnerringNewBlock, zap.Uint32("index", b.Index), ) err := s.persistate.SetUInt32(persistateSideChainLastBlockKey, b.Index) if err != nil { - s.log.Warn(context.Background(), logs.InnerringCantUpdatePersistentState, + s.log.Warn(ctx, logs.InnerringCantUpdatePersistentState, zap.String("chain", "side"), zap.Uint32("block_index", b.Index)) } @@ -235,10 +235,10 @@ func (s *Server) registerMorphNewBlockEventHandler() { func (s *Server) registerMainnetNewBlockEventHandler() { if !s.withoutMainNet { - s.mainnetListener.RegisterBlockHandler(func(b *block.Block) { + s.mainnetListener.RegisterBlockHandler(func(ctx context.Context, b *block.Block) { err := s.persistate.SetUInt32(persistateMainChainLastBlockKey, b.Index) if err != nil { - s.log.Warn(context.Background(), logs.InnerringCantUpdatePersistentState, + s.log.Warn(ctx, logs.InnerringCantUpdatePersistentState, zap.String("chain", "main"), zap.Uint32("block_index", b.Index)) } @@ -400,14 +400,14 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - err = server.initProcessors(cfg, morphClients) + err = server.initProcessors(ctx, cfg, morphClients) if err != nil { return nil, err } server.initTimers(ctx, cfg) - err = server.initGRPCServer(cfg, log, audit) + err = server.initGRPCServer(ctx, cfg, log, audit) if err != nil { return nil, err } @@ -604,7 +604,7 @@ func (s *Server) initConfigFromBlockchain(ctx context.Context) error { s.log.Debug(ctx, logs.InnerringReadConfigFromBlockchain, zap.Bool("active", s.IsActive(ctx)), - zap.Bool("alphabet", s.IsAlphabet()), + zap.Bool("alphabet", s.IsAlphabet(ctx)), zap.Uint64("epoch", epoch), zap.Uint32("precision", balancePrecision), zap.Uint32("init_epoch_tick_delta", s.initialEpochTickDelta), @@ -636,7 +636,7 @@ func (s *Server) nextEpochBlockDelta() (uint32, error) { // only if inner ring node is alphabet node. func (s *Server) onlyAlphabetEventHandler(f event.Handler) event.Handler { return func(ctx context.Context, ev event.Event) { - if s.IsAlphabet() { + if s.IsAlphabet(ctx) { f(ctx, ev) } } diff --git a/pkg/innerring/notary.go b/pkg/innerring/notary.go index dd3afa2c2..c8a69962f 100644 --- a/pkg/innerring/notary.go +++ b/pkg/innerring/notary.go @@ -28,37 +28,38 @@ const ( gasDivisor = 2 ) -func (s *Server) depositMainNotary() (tx util.Uint256, err error) { +func (s *Server) depositMainNotary(ctx context.Context) (tx util.Uint256, err error) { depositAmount, err := client.CalculateNotaryDepositAmount(s.mainnetClient, gasMultiplier, gasDivisor) if err != nil { return util.Uint256{}, fmt.Errorf("could not calculate main notary deposit amount: %w", err) } return s.mainnetClient.DepositNotary( + ctx, depositAmount, uint32(s.epochDuration.Load())+notaryExtraBlocks, ) } -func (s *Server) depositSideNotary() (util.Uint256, error) { +func (s *Server) depositSideNotary(ctx context.Context) (util.Uint256, error) { depositAmount, err := client.CalculateNotaryDepositAmount(s.morphClient, gasMultiplier, gasDivisor) if err != nil { return util.Uint256{}, fmt.Errorf("could not calculate side notary deposit amount: %w", err) } - tx, _, err := s.morphClient.DepositEndlessNotary(depositAmount) + tx, _, err := s.morphClient.DepositEndlessNotary(ctx, depositAmount) return tx, err } func (s *Server) notaryHandler(ctx context.Context, _ event.Event) { if !s.mainNotaryConfig.disabled { - _, err := s.depositMainNotary() + _, err := s.depositMainNotary(ctx) if err != nil { s.log.Error(ctx, logs.InnerringCantMakeNotaryDepositInMainChain, zap.Error(err)) } } - if _, err := s.depositSideNotary(); err != nil { + if _, err := s.depositSideNotary(ctx); err != nil { s.log.Error(ctx, logs.InnerringCantMakeNotaryDepositInSideChain, zap.Error(err)) } } @@ -72,7 +73,7 @@ func (s *Server) awaitSideNotaryDeposit(ctx context.Context, tx util.Uint256) er } func (s *Server) initNotary(ctx context.Context, deposit depositor, await awaiter, msg string) error { - tx, err := deposit() + tx, err := deposit(ctx) if err != nil { return err } diff --git a/pkg/innerring/processors/alphabet/handlers.go b/pkg/innerring/processors/alphabet/handlers.go index 0cc2a5f39..d6b653282 100644 --- a/pkg/innerring/processors/alphabet/handlers.go +++ b/pkg/innerring/processors/alphabet/handlers.go @@ -10,16 +10,16 @@ import ( "go.uber.org/zap" ) -func (ap *Processor) HandleGasEmission(ev event.Event) { +func (ap *Processor) HandleGasEmission(ctx context.Context, ev event.Event) { _ = ev.(timers.NewAlphabetEmitTick) - ap.log.Info(context.Background(), logs.AlphabetTick, zap.String("type", "alphabet gas emit")) + ap.log.Info(ctx, logs.AlphabetTick, zap.String("type", "alphabet gas emit")) // send event to the worker pool - err := processors.SubmitEvent(ap.pool, ap.metrics, "alphabet_emit_gas", ap.processEmit) + err := processors.SubmitEvent(ap.pool, ap.metrics, "alphabet_emit_gas", func() bool { return ap.processEmit(ctx) }) if err != nil { // there system can be moved into controlled degradation stage - ap.log.Warn(context.Background(), logs.AlphabetAlphabetProcessorWorkerPoolDrained, + ap.log.Warn(ctx, logs.AlphabetAlphabetProcessorWorkerPoolDrained, zap.Int("capacity", ap.pool.Cap())) } } diff --git a/pkg/innerring/processors/alphabet/handlers_test.go b/pkg/innerring/processors/alphabet/handlers_test.go index c7a004b54..036b8055c 100644 --- a/pkg/innerring/processors/alphabet/handlers_test.go +++ b/pkg/innerring/processors/alphabet/handlers_test.go @@ -1,6 +1,7 @@ package alphabet_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" @@ -60,7 +61,7 @@ func TestProcessorEmitsGasToNetmapAndAlphabet(t *testing.T) { processor, err := alphabet.New(params) require.NoError(t, err, "failed to create processor instance") - processor.HandleGasEmission(timers.NewAlphabetEmitTick{}) + processor.HandleGasEmission(context.Background(), timers.NewAlphabetEmitTick{}) processor.WaitPoolRunning() @@ -137,7 +138,7 @@ func TestProcessorEmitsGasToNetmapIfNoParsedWallets(t *testing.T) { processor, err := alphabet.New(params) require.NoError(t, err, "failed to create processor instance") - processor.HandleGasEmission(timers.NewAlphabetEmitTick{}) + processor.HandleGasEmission(context.Background(), timers.NewAlphabetEmitTick{}) processor.WaitPoolRunning() @@ -198,7 +199,7 @@ func TestProcessorDoesntEmitGasIfNoNetmapOrParsedWallets(t *testing.T) { processor, err := alphabet.New(params) require.NoError(t, err, "failed to create processor instance") - processor.HandleGasEmission(timers.NewAlphabetEmitTick{}) + processor.HandleGasEmission(context.Background(), timers.NewAlphabetEmitTick{}) processor.WaitPoolRunning() @@ -219,7 +220,7 @@ type testIndexer struct { index int } -func (i *testIndexer) AlphabetIndex() int { +func (i *testIndexer) AlphabetIndex(context.Context) int { return i.index } @@ -246,7 +247,7 @@ type testMorphClient struct { batchTransferedGas []batchTransferGas } -func (c *testMorphClient) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) { +func (c *testMorphClient) Invoke(_ context.Context, contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) { c.invokedMethods = append(c.invokedMethods, invokedMethod{ contract: contract, diff --git a/pkg/innerring/processors/alphabet/process_emit.go b/pkg/innerring/processors/alphabet/process_emit.go index 142409631..229261250 100644 --- a/pkg/innerring/processors/alphabet/process_emit.go +++ b/pkg/innerring/processors/alphabet/process_emit.go @@ -14,39 +14,39 @@ import ( const emitMethod = "emit" -func (ap *Processor) processEmit() bool { - index := ap.irList.AlphabetIndex() +func (ap *Processor) processEmit(ctx context.Context) bool { + index := ap.irList.AlphabetIndex(ctx) if index < 0 { - ap.log.Info(context.Background(), logs.AlphabetNonAlphabetModeIgnoreGasEmissionEvent) + ap.log.Info(ctx, logs.AlphabetNonAlphabetModeIgnoreGasEmissionEvent) return true } contract, ok := ap.alphabetContracts.GetByIndex(index) if !ok { - ap.log.Debug(context.Background(), logs.AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent, + ap.log.Debug(ctx, logs.AlphabetNodeIsOutOfAlphabetRangeIgnoreGasEmissionEvent, zap.Int("index", index)) return false } // there is no signature collecting, so we don't need extra fee - _, err := ap.morphClient.Invoke(contract, 0, emitMethod) + _, err := ap.morphClient.Invoke(ctx, contract, 0, emitMethod) if err != nil { - ap.log.Warn(context.Background(), logs.AlphabetCantInvokeAlphabetEmitMethod, zap.String("error", err.Error())) + ap.log.Warn(ctx, logs.AlphabetCantInvokeAlphabetEmitMethod, zap.String("error", err.Error())) return false } if ap.storageEmission == 0 { - ap.log.Info(context.Background(), logs.AlphabetStorageNodeEmissionIsOff) + ap.log.Info(ctx, logs.AlphabetStorageNodeEmissionIsOff) return true } networkMap, err := ap.netmapClient.NetMap() if err != nil { - ap.log.Warn(context.Background(), logs.AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes, + ap.log.Warn(ctx, logs.AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes, zap.String("error", err.Error())) return false @@ -59,7 +59,7 @@ func (ap *Processor) processEmit() bool { ap.pwLock.RUnlock() extraLen := len(pw) - ap.log.Debug(context.Background(), logs.AlphabetGasEmission, + ap.log.Debug(ctx, logs.AlphabetGasEmission, zap.Int("network_map", nmLen), zap.Int("extra_wallets", extraLen)) @@ -69,20 +69,20 @@ func (ap *Processor) processEmit() bool { gasPerNode := fixedn.Fixed8(ap.storageEmission / uint64(nmLen+extraLen)) - ap.transferGasToNetmapNodes(nmNodes, gasPerNode) + ap.transferGasToNetmapNodes(ctx, nmNodes, gasPerNode) - ap.transferGasToExtraNodes(pw, gasPerNode) + ap.transferGasToExtraNodes(ctx, pw, gasPerNode) return true } -func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerNode fixedn.Fixed8) { +func (ap *Processor) transferGasToNetmapNodes(ctx context.Context, nmNodes []netmap.NodeInfo, gasPerNode fixedn.Fixed8) { for i := range nmNodes { keyBytes := nmNodes[i].PublicKey() key, err := keys.NewPublicKeyFromBytes(keyBytes, elliptic.P256()) if err != nil { - ap.log.Warn(context.Background(), logs.AlphabetCantParseNodePublicKey, + ap.log.Warn(ctx, logs.AlphabetCantParseNodePublicKey, zap.String("error", err.Error())) continue @@ -90,7 +90,7 @@ func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerN err = ap.morphClient.TransferGas(key.GetScriptHash(), gasPerNode) if err != nil { - ap.log.Warn(context.Background(), logs.AlphabetCantTransferGas, + ap.log.Warn(ctx, logs.AlphabetCantTransferGas, zap.String("receiver", key.Address()), zap.Int64("amount", int64(gasPerNode)), zap.String("error", err.Error()), @@ -99,7 +99,7 @@ func (ap *Processor) transferGasToNetmapNodes(nmNodes []netmap.NodeInfo, gasPerN } } -func (ap *Processor) transferGasToExtraNodes(pw []util.Uint160, gasPerNode fixedn.Fixed8) { +func (ap *Processor) transferGasToExtraNodes(ctx context.Context, pw []util.Uint160, gasPerNode fixedn.Fixed8) { if len(pw) > 0 { err := ap.morphClient.BatchTransferGas(pw, gasPerNode) if err != nil { @@ -107,7 +107,7 @@ func (ap *Processor) transferGasToExtraNodes(pw []util.Uint160, gasPerNode fixed for i, addr := range pw { receiversLog[i] = addr.StringLE() } - ap.log.Warn(context.Background(), logs.AlphabetCantTransferGasToWallet, + ap.log.Warn(ctx, logs.AlphabetCantTransferGasToWallet, zap.Strings("receivers", receiversLog), zap.Int64("amount", int64(gasPerNode)), zap.String("error", err.Error()), diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index 8dbef1e20..bf74834ed 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -15,13 +14,12 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" - "go.uber.org/zap" ) type ( // Indexer is a callback interface for inner ring global state. Indexer interface { - AlphabetIndex() int + AlphabetIndex(context.Context) int } // Contracts is an interface of the storage @@ -41,7 +39,7 @@ type ( } morphClient interface { - Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) + Invoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error BatchTransferGas(receivers []util.Uint160, amount fixedn.Fixed8) error } @@ -86,8 +84,6 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/alphabet: global state is not set") } - p.Log.Debug(context.Background(), logs.AlphabetAlphabetWorkerPool, zap.Int("size", p.PoolSize)) - pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { return nil, fmt.Errorf("ir/frostfs: can't create worker pool: %w", err) diff --git a/pkg/innerring/processors/balance/handlers.go b/pkg/innerring/processors/balance/handlers.go index 5a89e6f7c..b5d05a02e 100644 --- a/pkg/innerring/processors/balance/handlers.go +++ b/pkg/innerring/processors/balance/handlers.go @@ -20,7 +20,7 @@ func (bp *Processor) handleLock(ctx context.Context, ev event.Event) { // send an event to the worker pool err := processors.SubmitEvent(bp.pool, bp.metrics, "lock", func() bool { - return bp.processLock(&lock) + return bp.processLock(ctx, &lock) }) if err != nil { // there system can be moved into controlled degradation stage diff --git a/pkg/innerring/processors/balance/handlers_test.go b/pkg/innerring/processors/balance/handlers_test.go index 3ef4959cc..0fd23d8ab 100644 --- a/pkg/innerring/processors/balance/handlers_test.go +++ b/pkg/innerring/processors/balance/handlers_test.go @@ -70,7 +70,7 @@ type testAlphabetState struct { isAlphabet bool } -func (s *testAlphabetState) IsAlphabet() bool { +func (s *testAlphabetState) IsAlphabet(context.Context) bool { return s.isAlphabet } @@ -84,7 +84,7 @@ type testFrostFSContractClient struct { chequeCalls int } -func (c *testFrostFSContractClient) Cheque(p frostfscontract.ChequePrm) error { +func (c *testFrostFSContractClient) Cheque(_ context.Context, p frostfscontract.ChequePrm) error { c.chequeCalls++ return nil } diff --git a/pkg/innerring/processors/balance/process_assets.go b/pkg/innerring/processors/balance/process_assets.go index ac6a1e493..60475908c 100644 --- a/pkg/innerring/processors/balance/process_assets.go +++ b/pkg/innerring/processors/balance/process_assets.go @@ -11,9 +11,9 @@ import ( // Process lock event by invoking Cheque method in main net to send assets // back to the withdraw issuer. -func (bp *Processor) processLock(lock *balanceEvent.Lock) bool { - if !bp.alphabetState.IsAlphabet() { - bp.log.Info(context.Background(), logs.BalanceNonAlphabetModeIgnoreBalanceLock) +func (bp *Processor) processLock(ctx context.Context, lock *balanceEvent.Lock) bool { + if !bp.alphabetState.IsAlphabet(ctx) { + bp.log.Info(ctx, logs.BalanceNonAlphabetModeIgnoreBalanceLock) return true } @@ -25,9 +25,9 @@ func (bp *Processor) processLock(lock *balanceEvent.Lock) bool { prm.SetLock(lock.LockAccount()) prm.SetHash(lock.TxHash()) - err := bp.frostfsClient.Cheque(prm) + err := bp.frostfsClient.Cheque(ctx, prm) if err != nil { - bp.log.Error(context.Background(), logs.BalanceCantSendLockAssetTx, zap.Error(err)) + bp.log.Error(ctx, logs.BalanceCantSendLockAssetTx, zap.Error(err)) return false } diff --git a/pkg/innerring/processors/balance/processor.go b/pkg/innerring/processors/balance/processor.go index c4078461e..e2f649600 100644 --- a/pkg/innerring/processors/balance/processor.go +++ b/pkg/innerring/processors/balance/processor.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" frostfscontract "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -13,13 +12,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" - "go.uber.org/zap" ) type ( // AlphabetState is a callback interface for inner ring global state. AlphabetState interface { - IsAlphabet() bool + IsAlphabet(context.Context) bool } // PrecisionConverter converts balance amount values. @@ -28,7 +26,7 @@ type ( } FrostFSClient interface { - Cheque(p frostfscontract.ChequePrm) error + Cheque(ctx context.Context, p frostfscontract.ChequePrm) error } // Processor of events produced by balance contract in the morphchain. @@ -69,8 +67,6 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/balance: balance precision converter is not set") } - p.Log.Debug(context.Background(), logs.BalanceBalanceWorkerPool, zap.Int("size", p.PoolSize)) - pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { return nil, fmt.Errorf("ir/balance: can't create worker pool: %w", err) diff --git a/pkg/innerring/processors/container/handlers.go b/pkg/innerring/processors/container/handlers.go index 45cac513a..bb038a3cb 100644 --- a/pkg/innerring/processors/container/handlers.go +++ b/pkg/innerring/processors/container/handlers.go @@ -23,7 +23,7 @@ func (cp *Processor) handlePut(ctx context.Context, ev event.Event) { // send an event to the worker pool err := processors.SubmitEvent(cp.pool, cp.metrics, "container_put", func() bool { - return cp.processContainerPut(put) + return cp.processContainerPut(ctx, put) }) if err != nil { // there system can be moved into controlled degradation stage @@ -41,7 +41,7 @@ func (cp *Processor) handleDelete(ctx context.Context, ev event.Event) { // send an event to the worker pool err := processors.SubmitEvent(cp.pool, cp.metrics, "container_delete", func() bool { - return cp.processContainerDelete(del) + return cp.processContainerDelete(ctx, del) }) if err != nil { // there system can be moved into controlled degradation stage diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index a2fe50fa8..f28e5372a 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -161,7 +161,7 @@ type testAlphabetState struct { isAlphabet bool } -func (s *testAlphabetState) IsAlphabet() bool { +func (s *testAlphabetState) IsAlphabet(context.Context) bool { return s.isAlphabet } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 2b9c5995c..16c450166 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -37,27 +37,27 @@ var errContainerAndOwnerNamespaceDontMatch = errors.New("container and owner nam // Process a new container from the user by checking the container sanity // and sending approve tx back to the morph. -func (cp *Processor) processContainerPut(put putEvent) bool { - if !cp.alphabetState.IsAlphabet() { - cp.log.Info(context.Background(), logs.ContainerNonAlphabetModeIgnoreContainerPut) +func (cp *Processor) processContainerPut(ctx context.Context, put putEvent) bool { + if !cp.alphabetState.IsAlphabet(ctx) { + cp.log.Info(ctx, logs.ContainerNonAlphabetModeIgnoreContainerPut) return true } - ctx := &putContainerContext{ + pctx := &putContainerContext{ e: put, } - err := cp.checkPutContainer(ctx) + err := cp.checkPutContainer(pctx) if err != nil { - cp.log.Error(context.Background(), logs.ContainerPutContainerCheckFailed, + cp.log.Error(ctx, logs.ContainerPutContainerCheckFailed, zap.String("error", err.Error()), ) return false } - if err := cp.morphClient.NotarySignAndInvokeTX(ctx.e.NotaryRequest().MainTransaction); err != nil { - cp.log.Error(context.Background(), logs.ContainerCouldNotApprovePutContainer, + if err := cp.morphClient.NotarySignAndInvokeTX(pctx.e.NotaryRequest().MainTransaction); err != nil { + cp.log.Error(ctx, logs.ContainerCouldNotApprovePutContainer, zap.String("error", err.Error()), ) return false @@ -104,15 +104,15 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { // Process delete container operation from the user by checking container sanity // and sending approve tx back to morph. -func (cp *Processor) processContainerDelete(e containerEvent.Delete) bool { - if !cp.alphabetState.IsAlphabet() { - cp.log.Info(context.Background(), logs.ContainerNonAlphabetModeIgnoreContainerDelete) +func (cp *Processor) processContainerDelete(ctx context.Context, e containerEvent.Delete) bool { + if !cp.alphabetState.IsAlphabet(ctx) { + cp.log.Info(ctx, logs.ContainerNonAlphabetModeIgnoreContainerDelete) return true } err := cp.checkDeleteContainer(e) if err != nil { - cp.log.Error(context.Background(), logs.ContainerDeleteContainerCheckFailed, + cp.log.Error(ctx, logs.ContainerDeleteContainerCheckFailed, zap.String("error", err.Error()), ) @@ -120,7 +120,7 @@ func (cp *Processor) processContainerDelete(e containerEvent.Delete) bool { } if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { - cp.log.Error(context.Background(), logs.ContainerCouldNotApproveDeleteContainer, + cp.log.Error(ctx, logs.ContainerCouldNotApproveDeleteContainer, zap.String("error", err.Error()), ) diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index 7a50ca773..58b90457c 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -6,7 +6,6 @@ import ( "fmt" frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -16,13 +15,12 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" - "go.uber.org/zap" ) type ( // AlphabetState is a callback interface for inner ring global state. AlphabetState interface { - IsAlphabet() bool + IsAlphabet(context.Context) bool } ContClient interface { @@ -98,8 +96,6 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/container: FrostFSID client is not set") } - p.Log.Debug(context.Background(), logs.ContainerContainerWorkerPool, zap.Int("size", p.PoolSize)) - pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { return nil, fmt.Errorf("ir/container: can't create worker pool: %w", err) diff --git a/pkg/innerring/processors/frostfs/handlers.go b/pkg/innerring/processors/frostfs/handlers.go index d11ad0f5c..936de2e77 100644 --- a/pkg/innerring/processors/frostfs/handlers.go +++ b/pkg/innerring/processors/frostfs/handlers.go @@ -24,7 +24,7 @@ func (np *Processor) handleDeposit(ctx context.Context, ev event.Event) { // send event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_deposit", func() bool { - return np.processDeposit(deposit) + return np.processDeposit(ctx, deposit) }) if err != nil { // there system can be moved into controlled degradation stage @@ -44,7 +44,7 @@ func (np *Processor) handleWithdraw(ctx context.Context, ev event.Event) { // send event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_withdraw", func() bool { - return np.processWithdraw(withdraw) + return np.processWithdraw(ctx, withdraw) }) if err != nil { // there system can be moved into controlled degradation stage @@ -62,7 +62,7 @@ func (np *Processor) handleCheque(ctx context.Context, ev event.Event) { // send event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_cheque", func() bool { - return np.processCheque(cheque) + return np.processCheque(ctx, cheque) }) if err != nil { // there system can be moved into controlled degradation stage @@ -81,7 +81,7 @@ func (np *Processor) handleConfig(ctx context.Context, ev event.Event) { // send event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "frostfs_config", func() bool { - return np.processConfig(cfg) + return np.processConfig(ctx, cfg) }) if err != nil { // there system can be moved into controlled degradation stage diff --git a/pkg/innerring/processors/frostfs/handlers_test.go b/pkg/innerring/processors/frostfs/handlers_test.go index c1541ca40..72310f6f9 100644 --- a/pkg/innerring/processors/frostfs/handlers_test.go +++ b/pkg/innerring/processors/frostfs/handlers_test.go @@ -226,7 +226,7 @@ type testAlphabetState struct { isAlphabet bool } -func (s *testAlphabetState) IsAlphabet() bool { +func (s *testAlphabetState) IsAlphabet(context.Context) bool { return s.isAlphabet } @@ -242,17 +242,17 @@ type testBalaceClient struct { burn []balance.BurnPrm } -func (c *testBalaceClient) Mint(p balance.MintPrm) error { +func (c *testBalaceClient) Mint(_ context.Context, p balance.MintPrm) error { c.mint = append(c.mint, p) return nil } -func (c *testBalaceClient) Lock(p balance.LockPrm) error { +func (c *testBalaceClient) Lock(_ context.Context, p balance.LockPrm) error { c.lock = append(c.lock, p) return nil } -func (c *testBalaceClient) Burn(p balance.BurnPrm) error { +func (c *testBalaceClient) Burn(_ context.Context, p balance.BurnPrm) error { c.burn = append(c.burn, p) return nil } @@ -261,7 +261,7 @@ type testNetmapClient struct { config []nmClient.SetConfigPrm } -func (c *testNetmapClient) SetConfig(p nmClient.SetConfigPrm) error { +func (c *testNetmapClient) SetConfig(_ context.Context, p nmClient.SetConfigPrm) error { c.config = append(c.config, p) return nil } diff --git a/pkg/innerring/processors/frostfs/process_assets.go b/pkg/innerring/processors/frostfs/process_assets.go index 3bee6ed96..ee824ea31 100644 --- a/pkg/innerring/processors/frostfs/process_assets.go +++ b/pkg/innerring/processors/frostfs/process_assets.go @@ -17,9 +17,9 @@ const ( // Process deposit event by invoking a balance contract and sending native // gas in the sidechain. -func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { - if !np.alphabetState.IsAlphabet() { - np.log.Info(context.Background(), logs.FrostFSNonAlphabetModeIgnoreDeposit) +func (np *Processor) processDeposit(ctx context.Context, deposit frostfsEvent.Deposit) bool { + if !np.alphabetState.IsAlphabet(ctx) { + np.log.Info(ctx, logs.FrostFSNonAlphabetModeIgnoreDeposit) return true } @@ -30,9 +30,9 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { prm.SetID(deposit.ID()) // send transferX to a balance contract - err := np.balanceClient.Mint(prm) + err := np.balanceClient.Mint(ctx, prm) if err != nil { - np.log.Error(context.Background(), logs.FrostFSCantTransferAssetsToBalanceContract, zap.Error(err)) + np.log.Error(ctx, logs.FrostFSCantTransferAssetsToBalanceContract, zap.Error(err)) } curEpoch := np.epochState.EpochCounter() @@ -46,7 +46,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { val, ok := np.mintEmitCache.Get(receiver.String()) if ok && val+np.mintEmitThreshold >= curEpoch { - np.log.Warn(context.Background(), logs.FrostFSDoubleMintEmissionDeclined, + np.log.Warn(ctx, logs.FrostFSDoubleMintEmissionDeclined, zap.Stringer("receiver", receiver), zap.Uint64("last_emission", val), zap.Uint64("current_epoch", curEpoch)) @@ -58,12 +58,12 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { // before gas transfer check if the balance is greater than the threshold balance, err := np.morphClient.GasBalance() if err != nil { - np.log.Error(context.Background(), logs.FrostFSCantGetGasBalanceOfTheNode, zap.Error(err)) + np.log.Error(ctx, logs.FrostFSCantGetGasBalanceOfTheNode, zap.Error(err)) return false } if balance < np.gasBalanceThreshold { - np.log.Warn(context.Background(), logs.FrostFSGasBalanceThresholdHasBeenReached, + np.log.Warn(ctx, logs.FrostFSGasBalanceThresholdHasBeenReached, zap.Int64("balance", balance), zap.Int64("threshold", np.gasBalanceThreshold)) @@ -72,7 +72,7 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { err = np.morphClient.TransferGas(receiver, np.mintEmitValue) if err != nil { - np.log.Error(context.Background(), logs.FrostFSCantTransferNativeGasToReceiver, + np.log.Error(ctx, logs.FrostFSCantTransferNativeGasToReceiver, zap.String("error", err.Error())) return false @@ -84,16 +84,16 @@ func (np *Processor) processDeposit(deposit frostfsEvent.Deposit) bool { } // Process withdraw event by locking assets in the balance account. -func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) bool { - if !np.alphabetState.IsAlphabet() { - np.log.Info(context.Background(), logs.FrostFSNonAlphabetModeIgnoreWithdraw) +func (np *Processor) processWithdraw(ctx context.Context, withdraw frostfsEvent.Withdraw) bool { + if !np.alphabetState.IsAlphabet(ctx) { + np.log.Info(ctx, logs.FrostFSNonAlphabetModeIgnoreWithdraw) return true } // create lock account lock, err := util.Uint160DecodeBytesBE(withdraw.ID()[:util.Uint160Size]) if err != nil { - np.log.Error(context.Background(), logs.FrostFSCantCreateLockAccount, zap.Error(err)) + np.log.Error(ctx, logs.FrostFSCantCreateLockAccount, zap.Error(err)) return false } @@ -107,9 +107,9 @@ func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) bool { prm.SetAmount(np.converter.ToBalancePrecision(withdraw.Amount())) prm.SetDueEpoch(int64(curEpoch + lockAccountLifetime)) - err = np.balanceClient.Lock(prm) + err = np.balanceClient.Lock(ctx, prm) if err != nil { - np.log.Error(context.Background(), logs.FrostFSCantLockAssetsForWithdraw, zap.Error(err)) + np.log.Error(ctx, logs.FrostFSCantLockAssetsForWithdraw, zap.Error(err)) return false } @@ -118,9 +118,9 @@ func (np *Processor) processWithdraw(withdraw frostfsEvent.Withdraw) bool { // Process cheque event by transferring assets from the lock account back to // the reserve account. -func (np *Processor) processCheque(cheque frostfsEvent.Cheque) bool { - if !np.alphabetState.IsAlphabet() { - np.log.Info(context.Background(), logs.FrostFSNonAlphabetModeIgnoreCheque) +func (np *Processor) processCheque(ctx context.Context, cheque frostfsEvent.Cheque) bool { + if !np.alphabetState.IsAlphabet(ctx) { + np.log.Info(ctx, logs.FrostFSNonAlphabetModeIgnoreCheque) return true } @@ -130,9 +130,9 @@ func (np *Processor) processCheque(cheque frostfsEvent.Cheque) bool { prm.SetAmount(np.converter.ToBalancePrecision(cheque.Amount())) prm.SetID(cheque.ID()) - err := np.balanceClient.Burn(prm) + err := np.balanceClient.Burn(ctx, prm) if err != nil { - np.log.Error(context.Background(), logs.FrostFSCantTransferAssetsToFedContract, zap.Error(err)) + np.log.Error(ctx, logs.FrostFSCantTransferAssetsToFedContract, zap.Error(err)) return false } diff --git a/pkg/innerring/processors/frostfs/process_config.go b/pkg/innerring/processors/frostfs/process_config.go index 814dd40b4..dc579f6bb 100644 --- a/pkg/innerring/processors/frostfs/process_config.go +++ b/pkg/innerring/processors/frostfs/process_config.go @@ -11,9 +11,9 @@ import ( // Process config event by setting configuration value from the mainchain in // the sidechain. -func (np *Processor) processConfig(config frostfsEvent.Config) bool { - if !np.alphabetState.IsAlphabet() { - np.log.Info(context.Background(), logs.FrostFSNonAlphabetModeIgnoreConfig) +func (np *Processor) processConfig(ctx context.Context, config frostfsEvent.Config) bool { + if !np.alphabetState.IsAlphabet(ctx) { + np.log.Info(ctx, logs.FrostFSNonAlphabetModeIgnoreConfig) return true } @@ -24,9 +24,9 @@ func (np *Processor) processConfig(config frostfsEvent.Config) bool { prm.SetValue(config.Value()) prm.SetHash(config.TxHash()) - err := np.netmapClient.SetConfig(prm) + err := np.netmapClient.SetConfig(ctx, prm) if err != nil { - np.log.Error(context.Background(), logs.FrostFSCantRelaySetConfigEvent, zap.Error(err)) + np.log.Error(ctx, logs.FrostFSCantRelaySetConfigEvent, zap.Error(err)) return false } diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index fdc31d82e..6c29d330d 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -6,7 +6,6 @@ import ( "fmt" "sync" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -17,7 +16,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" - "go.uber.org/zap" ) type ( @@ -28,7 +26,7 @@ type ( // AlphabetState is a callback interface for inner ring global state. AlphabetState interface { - IsAlphabet() bool + IsAlphabet(context.Context) bool } // PrecisionConverter converts balance amount values. @@ -37,13 +35,13 @@ type ( } BalanceClient interface { - Mint(p balance.MintPrm) error - Lock(p balance.LockPrm) error - Burn(p balance.BurnPrm) error + Mint(ctx context.Context, p balance.MintPrm) error + Lock(ctx context.Context, p balance.LockPrm) error + Burn(ctx context.Context, p balance.BurnPrm) error } NetmapClient interface { - SetConfig(p nmClient.SetConfigPrm) error + SetConfig(ctx context.Context, p nmClient.SetConfigPrm) error } MorphClient interface { @@ -111,8 +109,6 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/frostfs: balance precision converter is not set") } - p.Log.Debug(context.Background(), logs.FrostFSFrostfsWorkerPool, zap.Int("size", p.PoolSize)) - pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { return nil, fmt.Errorf("ir/frostfs: can't create worker pool: %w", err) diff --git a/pkg/innerring/processors/governance/handlers_test.go b/pkg/innerring/processors/governance/handlers_test.go index 286935129..5a6126249 100644 --- a/pkg/innerring/processors/governance/handlers_test.go +++ b/pkg/innerring/processors/governance/handlers_test.go @@ -219,7 +219,7 @@ type testAlphabetState struct { isAlphabet bool } -func (s *testAlphabetState) IsAlphabet() bool { +func (s *testAlphabetState) IsAlphabet(context.Context) bool { return s.isAlphabet } @@ -251,12 +251,12 @@ func (c *testMorphClient) Committee() (res keys.PublicKeys, err error) { return c.commiteeKeys, nil } -func (c *testMorphClient) UpdateNeoFSAlphabetList(prm client.UpdateAlphabetListPrm) error { +func (c *testMorphClient) UpdateNeoFSAlphabetList(_ context.Context, prm client.UpdateAlphabetListPrm) error { c.alphabetUpdates = append(c.alphabetUpdates, prm) return nil } -func (c *testMorphClient) UpdateNotaryList(prm client.UpdateNotaryListPrm) error { +func (c *testMorphClient) UpdateNotaryList(_ context.Context, prm client.UpdateNotaryListPrm) error { c.notaryUpdates = append(c.notaryUpdates, prm) return nil } @@ -278,7 +278,7 @@ type testFrostFSClient struct { updates []frostfscontract.AlphabetUpdatePrm } -func (c *testFrostFSClient) AlphabetUpdate(p frostfscontract.AlphabetUpdatePrm) error { +func (c *testFrostFSClient) AlphabetUpdate(_ context.Context, p frostfscontract.AlphabetUpdatePrm) error { c.updates = append(c.updates, p) return nil } diff --git a/pkg/innerring/processors/governance/process_update.go b/pkg/innerring/processors/governance/process_update.go index fdfdfa479..73d21a7d2 100644 --- a/pkg/innerring/processors/governance/process_update.go +++ b/pkg/innerring/processors/governance/process_update.go @@ -20,7 +20,7 @@ const ( ) func (gp *Processor) processAlphabetSync(ctx context.Context, txHash util.Uint256) bool { - if !gp.alphabetState.IsAlphabet() { + if !gp.alphabetState.IsAlphabet(ctx) { gp.log.Info(ctx, logs.GovernanceNonAlphabetModeIgnoreAlphabetSync) return true } @@ -69,13 +69,13 @@ func (gp *Processor) processAlphabetSync(ctx context.Context, txHash util.Uint25 } // 2. Update NeoFSAlphabet role in the sidechain. - gp.updateNeoFSAlphabetRoleInSidechain(sidechainAlphabet, newAlphabet, txHash) + gp.updateNeoFSAlphabetRoleInSidechain(ctx, sidechainAlphabet, newAlphabet, txHash) // 3. Update notary role in the sidechain. - gp.updateNotaryRoleInSidechain(newAlphabet, txHash) + gp.updateNotaryRoleInSidechain(ctx, newAlphabet, txHash) // 4. Update FrostFS contract in the mainnet. - gp.updateFrostFSContractInMainnet(newAlphabet) + gp.updateFrostFSContractInMainnet(ctx, newAlphabet) gp.log.Info(ctx, logs.GovernanceFinishedAlphabetListUpdate) @@ -94,24 +94,24 @@ func prettyKeys(keys keys.PublicKeys) string { return strings.TrimRight(sb.String(), delimiter) } -func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(sidechainAlphabet, newAlphabet keys.PublicKeys, txHash util.Uint256) { +func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(ctx context.Context, sidechainAlphabet, newAlphabet keys.PublicKeys, txHash util.Uint256) { innerRing, err := gp.irFetcher.InnerRingKeys() if err != nil { - gp.log.Error(context.Background(), logs.GovernanceCantFetchInnerRingListFromSideChain, + gp.log.Error(ctx, logs.GovernanceCantFetchInnerRingListFromSideChain, zap.String("error", err.Error())) return } newInnerRing, err := updateInnerRing(innerRing, sidechainAlphabet, newAlphabet) if err != nil { - gp.log.Error(context.Background(), logs.GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys, + gp.log.Error(ctx, logs.GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys, zap.String("error", err.Error())) return } sort.Sort(newInnerRing) - gp.log.Info(context.Background(), logs.GovernanceUpdateOfTheInnerRingList, + gp.log.Info(ctx, logs.GovernanceUpdateOfTheInnerRingList, zap.String("before", prettyKeys(innerRing)), zap.String("after", prettyKeys(newInnerRing)), ) @@ -120,26 +120,26 @@ func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(sidechainAlphabet, newAl updPrm.SetList(newInnerRing) updPrm.SetHash(txHash) - if err = gp.morphClient.UpdateNeoFSAlphabetList(updPrm); err != nil { - gp.log.Error(context.Background(), logs.GovernanceCantUpdateInnerRingListWithNewAlphabetKeys, + if err = gp.morphClient.UpdateNeoFSAlphabetList(ctx, updPrm); err != nil { + gp.log.Error(ctx, logs.GovernanceCantUpdateInnerRingListWithNewAlphabetKeys, zap.String("error", err.Error())) } } -func (gp *Processor) updateNotaryRoleInSidechain(newAlphabet keys.PublicKeys, txHash util.Uint256) { +func (gp *Processor) updateNotaryRoleInSidechain(ctx context.Context, newAlphabet keys.PublicKeys, txHash util.Uint256) { updPrm := client.UpdateNotaryListPrm{} updPrm.SetList(newAlphabet) updPrm.SetHash(txHash) - err := gp.morphClient.UpdateNotaryList(updPrm) + err := gp.morphClient.UpdateNotaryList(ctx, updPrm) if err != nil { - gp.log.Error(context.Background(), logs.GovernanceCantUpdateListOfNotaryNodesInSideChain, + gp.log.Error(ctx, logs.GovernanceCantUpdateListOfNotaryNodesInSideChain, zap.String("error", err.Error())) } } -func (gp *Processor) updateFrostFSContractInMainnet(newAlphabet keys.PublicKeys) { +func (gp *Processor) updateFrostFSContractInMainnet(ctx context.Context, newAlphabet keys.PublicKeys) { epoch := gp.epochState.EpochCounter() buf := make([]byte, 8) @@ -152,9 +152,9 @@ func (gp *Processor) updateFrostFSContractInMainnet(newAlphabet keys.PublicKeys) prm.SetID(id) prm.SetPubs(newAlphabet) - err := gp.frostfsClient.AlphabetUpdate(prm) + err := gp.frostfsClient.AlphabetUpdate(ctx, prm) if err != nil { - gp.log.Error(context.Background(), logs.GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract, + gp.log.Error(ctx, logs.GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract, zap.String("error", err.Error())) } } diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index eaadfdb4f..565f4c27d 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -26,7 +26,7 @@ const ProcessorPoolSize = 1 type ( // AlphabetState is a callback interface for innerring global state. AlphabetState interface { - IsAlphabet() bool + IsAlphabet(context.Context) bool } ) @@ -56,7 +56,7 @@ type ( } FrostFSClient interface { - AlphabetUpdate(p frostfscontract.AlphabetUpdatePrm) error + AlphabetUpdate(ctx context.Context, p frostfscontract.AlphabetUpdatePrm) error } NetmapClient interface { @@ -70,8 +70,8 @@ type ( MorphClient interface { Committee() (res keys.PublicKeys, err error) - UpdateNeoFSAlphabetList(prm client.UpdateAlphabetListPrm) error - UpdateNotaryList(prm client.UpdateNotaryListPrm) error + UpdateNeoFSAlphabetList(ctx context.Context, prm client.UpdateAlphabetListPrm) error + UpdateNotaryList(ctx context.Context, prm client.UpdateNotaryListPrm) error } // Processor of events related to governance in the network. diff --git a/pkg/innerring/processors/netmap/handlers.go b/pkg/innerring/processors/netmap/handlers.go index 61547e0ba..4c7199a49 100644 --- a/pkg/innerring/processors/netmap/handlers.go +++ b/pkg/innerring/processors/netmap/handlers.go @@ -14,14 +14,14 @@ import ( func (np *Processor) HandleNewEpochTick(ctx context.Context, ev event.Event) { _ = ev.(timerEvent.NewEpochTick) - np.log.Info(context.Background(), logs.NetmapTick, zap.String("type", "epoch")) + np.log.Info(ctx, logs.NetmapTick, zap.String("type", "epoch")) // send an event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "netmap_new_epoch_tick", func() bool { return np.processNewEpochTick(ctx) }) if err != nil { // there system can be moved into controlled degradation stage - np.log.Warn(context.Background(), logs.NetmapNetmapWorkerPoolDrained, + np.log.Warn(ctx, logs.NetmapNetmapWorkerPoolDrained, zap.Int("capacity", np.pool.Cap())) } } @@ -54,7 +54,7 @@ func (np *Processor) handleAddPeer(ctx context.Context, ev event.Event) { // send an event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "netmap_add_peer", func() bool { - return np.processAddPeer(newPeer) + return np.processAddPeer(ctx, newPeer) }) if err != nil { // there system can be moved into controlled degradation stage @@ -72,7 +72,7 @@ func (np *Processor) handleUpdateState(ctx context.Context, ev event.Event) { // send event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "netmap_update_peer", func() bool { - return np.processUpdatePeer(updPeer) + return np.processUpdatePeer(ctx, updPeer) }) if err != nil { // there system can be moved into controlled degradation stage @@ -94,7 +94,7 @@ func (np *Processor) handleCleanupTick(ctx context.Context, ev event.Event) { // send event to the worker pool err := processors.SubmitEvent(np.pool, np.metrics, "netmap_cleanup_tick", func() bool { - return np.processNetmapCleanupTick(cleanup) + return np.processNetmapCleanupTick(ctx, cleanup) }) if err != nil { // there system can be moved into controlled degradation stage diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index 1e8be4095..5a5adfb2d 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -341,7 +341,7 @@ type testAlphabetState struct { isAlphabet bool } -func (s *testAlphabetState) IsAlphabet() bool { +func (s *testAlphabetState) IsAlphabet(context.Context) bool { return s.isAlphabet } @@ -365,7 +365,7 @@ type testNetmapClient struct { invokedTxs []*transaction.Transaction } -func (c *testNetmapClient) MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { +func (c *testNetmapClient) MorphNotaryInvoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { c.notaryInvokes = append(c.notaryInvokes, notaryInvoke{ contract: contract, fee: fee, @@ -396,7 +396,7 @@ func (c *testNetmapClient) NetMap() (*netmap.NetMap, error) { return c.netmap, nil } -func (c *testNetmapClient) NewEpoch(epoch uint64) error { +func (c *testNetmapClient) NewEpoch(_ context.Context, epoch uint64) error { c.newEpochs = append(c.newEpochs, epoch) return nil } diff --git a/pkg/innerring/processors/netmap/process_cleanup.go b/pkg/innerring/processors/netmap/process_cleanup.go index 9529d3a0c..a43005ffb 100644 --- a/pkg/innerring/processors/netmap/process_cleanup.go +++ b/pkg/innerring/processors/netmap/process_cleanup.go @@ -9,9 +9,9 @@ import ( "go.uber.org/zap" ) -func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) bool { - if !np.alphabetState.IsAlphabet() { - np.log.Info(context.Background(), logs.NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick) +func (np *Processor) processNetmapCleanupTick(ctx context.Context, ev netmapCleanupTick) bool { + if !np.alphabetState.IsAlphabet(ctx) { + np.log.Info(ctx, logs.NetmapNonAlphabetModeIgnoreNewNetmapCleanupTick) return true } @@ -19,13 +19,13 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) bool { err := np.netmapSnapshot.forEachRemoveCandidate(ev.epoch, func(s string) error { key, err := keys.NewPublicKeyFromString(s) if err != nil { - np.log.Warn(context.Background(), logs.NetmapCantDecodePublicKeyOfNetmapNode, + np.log.Warn(ctx, logs.NetmapCantDecodePublicKeyOfNetmapNode, zap.String("key", s)) return nil } - np.log.Info(context.Background(), logs.NetmapVoteToRemoveNodeFromNetmap, zap.String("key", s)) + np.log.Info(ctx, logs.NetmapVoteToRemoveNodeFromNetmap, zap.String("key", s)) // In notary environments we call UpdateStateIR method instead of UpdateState. // It differs from UpdateState only by name, so we can do this in the same form. @@ -33,6 +33,7 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) bool { const methodUpdateStateNotary = "updateStateIR" err = np.netmapClient.MorphNotaryInvoke( + ctx, np.netmapClient.ContractAddress(), 0, uint32(ev.epoch), @@ -41,13 +42,13 @@ func (np *Processor) processNetmapCleanupTick(ev netmapCleanupTick) bool { int64(v2netmap.Offline), key.Bytes(), ) if err != nil { - np.log.Error(context.Background(), logs.NetmapCantInvokeNetmapUpdateState, zap.Error(err)) + np.log.Error(ctx, logs.NetmapCantInvokeNetmapUpdateState, zap.Error(err)) } return nil }) if err != nil { - np.log.Warn(context.Background(), logs.NetmapCantIterateOnNetmapCleanerCache, + np.log.Warn(ctx, logs.NetmapCantIterateOnNetmapCleanerCache, zap.String("error", err.Error())) return false } diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index e401ef4f2..237c4e512 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -55,7 +55,7 @@ func (np *Processor) processNewEpoch(ctx context.Context, ev netmapEvent.NewEpoc // Process new epoch tick by invoking new epoch method in network map contract. func (np *Processor) processNewEpochTick(ctx context.Context) bool { - if !np.alphabetState.IsAlphabet() { + if !np.alphabetState.IsAlphabet(ctx) { np.log.Info(ctx, logs.NetmapNonAlphabetModeIgnoreNewEpochTick) return true } @@ -63,7 +63,7 @@ func (np *Processor) processNewEpochTick(ctx context.Context) bool { nextEpoch := np.epochState.EpochCounter() + 1 np.log.Debug(ctx, logs.NetmapNextEpoch, zap.Uint64("value", nextEpoch)) - err := np.netmapClient.NewEpoch(nextEpoch) + err := np.netmapClient.NewEpoch(ctx, nextEpoch) if err != nil { np.log.Error(ctx, logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) return false diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index 42d1b5ec6..72aa08f76 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -13,9 +13,9 @@ import ( // Process add peer notification by sanity check of new node // local epoch timer. -func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { - if !np.alphabetState.IsAlphabet() { - np.log.Info(context.Background(), logs.NetmapNonAlphabetModeIgnoreNewPeerNotification) +func (np *Processor) processAddPeer(ctx context.Context, ev netmapEvent.AddPeer) bool { + if !np.alphabetState.IsAlphabet(ctx) { + np.log.Info(ctx, logs.NetmapNonAlphabetModeIgnoreNewPeerNotification) return true } @@ -23,7 +23,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { tx := ev.NotaryRequest().MainTransaction ok, err := np.netmapClient.MorphIsValidScript(tx.Script, tx.Signers) if err != nil || !ok { - np.log.Warn(context.Background(), logs.NetmapNonhaltNotaryTransaction, + np.log.Warn(ctx, logs.NetmapNonhaltNotaryTransaction, zap.String("method", "netmap.AddPeer"), zap.String("hash", tx.Hash().StringLE()), zap.Error(err)) @@ -34,14 +34,14 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { var nodeInfo netmap.NodeInfo if err := nodeInfo.Unmarshal(ev.Node()); err != nil { // it will be nice to have tx id at event structure to log it - np.log.Warn(context.Background(), logs.NetmapCantParseNetworkMapCandidate) + np.log.Warn(ctx, logs.NetmapCantParseNetworkMapCandidate) return false } // validate and update node info err = np.nodeValidator.VerifyAndUpdate(&nodeInfo) if err != nil { - np.log.Warn(context.Background(), logs.NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate, + np.log.Warn(ctx, logs.NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate, zap.String("error", err.Error()), ) @@ -64,7 +64,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { // That is why we need to perform `addPeerIR` only in case when node is online, // because in scope of this method, contract set state `ONLINE` for the node. if updated && nodeInfo.Status().IsOnline() { - np.log.Info(context.Background(), logs.NetmapApprovingNetworkMapCandidate, + np.log.Info(ctx, logs.NetmapApprovingNetworkMapCandidate, zap.String("key", keyString)) prm := netmapclient.AddPeerPrm{} @@ -77,6 +77,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { // create new notary request with the original nonce err = np.netmapClient.MorphNotaryInvoke( + ctx, np.netmapClient.ContractAddress(), 0, ev.NotaryRequest().MainTransaction.Nonce, @@ -85,7 +86,7 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { nodeInfoBinary, ) if err != nil { - np.log.Error(context.Background(), logs.NetmapCantInvokeNetmapAddPeer, zap.Error(err)) + np.log.Error(ctx, logs.NetmapCantInvokeNetmapAddPeer, zap.Error(err)) return false } } @@ -94,9 +95,9 @@ func (np *Processor) processAddPeer(ev netmapEvent.AddPeer) bool { } // Process update peer notification by sending approval tx to the smart contract. -func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) bool { - if !np.alphabetState.IsAlphabet() { - np.log.Info(context.Background(), logs.NetmapNonAlphabetModeIgnoreUpdatePeerNotification) +func (np *Processor) processUpdatePeer(ctx context.Context, ev netmapEvent.UpdatePeer) bool { + if !np.alphabetState.IsAlphabet(ctx) { + np.log.Info(ctx, logs.NetmapNonAlphabetModeIgnoreUpdatePeerNotification) return true } @@ -109,7 +110,7 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) bool { if ev.Maintenance() { err = np.nodeStateSettings.MaintenanceModeAllowed() if err != nil { - np.log.Info(context.Background(), logs.NetmapPreventSwitchingNodeToMaintenanceState, + np.log.Info(ctx, logs.NetmapPreventSwitchingNodeToMaintenanceState, zap.Error(err), ) @@ -118,7 +119,7 @@ func (np *Processor) processUpdatePeer(ev netmapEvent.UpdatePeer) bool { } if err = np.netmapClient.MorphNotarySignAndInvokeTX(ev.NotaryRequest().MainTransaction); err != nil { - np.log.Error(context.Background(), logs.NetmapCantInvokeNetmapUpdatePeer, zap.Error(err)) + np.log.Error(ctx, logs.NetmapCantInvokeNetmapUpdatePeer, zap.Error(err)) return false } diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index bbd60c1e1..b3d57e85b 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -17,7 +16,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/panjf2000/ants/v2" - "go.uber.org/zap" ) type ( @@ -36,7 +34,7 @@ type ( // AlphabetState is a callback interface for inner ring global state. AlphabetState interface { - IsAlphabet() bool + IsAlphabet(context.Context) bool } // NodeValidator wraps basic method of checking the correctness @@ -55,12 +53,12 @@ type ( } Client interface { - MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error + MorphNotaryInvoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error ContractAddress() util.Uint160 EpochDuration() (uint64, error) MorphTxHeight(h util.Uint256) (res uint32, err error) NetMap() (*netmap.NetMap, error) - NewEpoch(epoch uint64) error + NewEpoch(ctx context.Context, epoch uint64) error MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error } @@ -133,8 +131,6 @@ func New(p *Params) (*Processor, error) { return nil, errors.New("ir/netmap: node state settings is not set") } - p.Log.Debug(context.Background(), logs.NetmapNetmapWorkerPool, zap.Int("size", p.PoolSize)) - pool, err := ants.NewPool(p.PoolSize, ants.WithNonblocking(true)) if err != nil { return nil, fmt.Errorf("ir/netmap: can't create worker pool: %w", err) diff --git a/pkg/innerring/processors/netmap/wrappers.go b/pkg/innerring/processors/netmap/wrappers.go index e75fdaf40..9cd71ae48 100644 --- a/pkg/innerring/processors/netmap/wrappers.go +++ b/pkg/innerring/processors/netmap/wrappers.go @@ -1,6 +1,8 @@ package netmap import ( + "context" + netmapclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -18,13 +20,13 @@ type netmapClientWrapper struct { netmapClient *netmapclient.Client } -func (w *netmapClientWrapper) UpdatePeerState(p netmapclient.UpdatePeerPrm) error { - _, err := w.netmapClient.UpdatePeerState(p) +func (w *netmapClientWrapper) UpdatePeerState(ctx context.Context, p netmapclient.UpdatePeerPrm) error { + _, err := w.netmapClient.UpdatePeerState(ctx, p) return err } -func (w *netmapClientWrapper) MorphNotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { - _, err := w.netmapClient.Morph().NotaryInvoke(contract, fee, nonce, vub, method, args...) +func (w *netmapClientWrapper) MorphNotaryInvoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error { + _, err := w.netmapClient.Morph().NotaryInvoke(ctx, contract, fee, nonce, vub, method, args...) return err } @@ -44,16 +46,16 @@ func (w *netmapClientWrapper) NetMap() (*netmap.NetMap, error) { return w.netmapClient.NetMap() } -func (w *netmapClientWrapper) NewEpoch(epoch uint64) error { - return w.netmapClient.NewEpoch(epoch) +func (w *netmapClientWrapper) NewEpoch(ctx context.Context, epoch uint64) error { + return w.netmapClient.NewEpoch(ctx, epoch) } func (w *netmapClientWrapper) MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) { return w.netmapClient.Morph().IsValidScript(script, signers) } -func (w *netmapClientWrapper) AddPeer(p netmapclient.AddPeerPrm) error { - return w.netmapClient.AddPeer(p) +func (w *netmapClientWrapper) AddPeer(ctx context.Context, p netmapclient.AddPeerPrm) error { + return w.netmapClient.AddPeer(ctx, p) } func (w *netmapClientWrapper) MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error { diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 85f332fb6..77c2af2ce 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -53,8 +53,8 @@ func (s *Server) IsActive(ctx context.Context) bool { } // IsAlphabet is a getter for a global alphabet flag state. -func (s *Server) IsAlphabet() bool { - return s.AlphabetIndex() >= 0 +func (s *Server) IsAlphabet(ctx context.Context) bool { + return s.AlphabetIndex(ctx) >= 0 } // InnerRingIndex is a getter for a global index of node in inner ring list. Negative @@ -83,10 +83,10 @@ func (s *Server) InnerRingSize(ctx context.Context) int { // AlphabetIndex is a getter for a global index of node in alphabet list. // Negative index means that node is not in the alphabet list. -func (s *Server) AlphabetIndex() int { +func (s *Server) AlphabetIndex(ctx context.Context) int { index, err := s.statusIndex.AlphabetIndex() if err != nil { - s.log.Error(context.Background(), logs.InnerringCantGetAlphabetIndex, zap.String("error", err.Error())) + s.log.Error(ctx, logs.InnerringCantGetAlphabetIndex, zap.String("error", err.Error())) return -1 } @@ -127,7 +127,7 @@ func (s *Server) voteForSidechainValidator(ctx context.Context, prm governance.V } s.contracts.alphabet.iterate(func(letter GlagoliticLetter, contract util.Uint160) { - _, err := s.morphClient.NotaryInvoke(contract, s.feeConfig.SideChainFee(), nonce, vubP, voteMethod, epoch, validators) + _, err := s.morphClient.NotaryInvoke(ctx, contract, s.feeConfig.SideChainFee(), nonce, vubP, voteMethod, epoch, validators) if err != nil { s.log.Warn(ctx, logs.InnerringCantInvokeVoteMethodInAlphabetContract, zap.Int8("alphabet_index", int8(letter)), diff --git a/pkg/innerring/state_test.go b/pkg/innerring/state_test.go index 17ab995af..f60ca87c4 100644 --- a/pkg/innerring/state_test.go +++ b/pkg/innerring/state_test.go @@ -47,8 +47,8 @@ func TestServerState(t *testing.T) { require.Equal(t, healthStatus, srv.HealthStatus(), "invalid health status") require.True(t, srv.IsActive(context.Background()), "invalid IsActive result") - require.True(t, srv.IsAlphabet(), "invalid IsAlphabet result") + require.True(t, srv.IsAlphabet(context.Background()), "invalid IsAlphabet result") require.Equal(t, 0, srv.InnerRingIndex(context.Background()), "invalid IR index") require.Equal(t, 1, srv.InnerRingSize(context.Background()), "invalid IR index") - require.Equal(t, 0, srv.AlphabetIndex(), "invalid alphabet index") + require.Equal(t, 0, srv.AlphabetIndex(context.Background()), "invalid alphabet index") } diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go index 10cb6f368..95fdd844b 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza_test.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza_test.go @@ -72,7 +72,7 @@ func TestBlobovnicza(t *testing.T) { require.NoError(t, blz.Open(context.Background())) // initialize Blobovnicza - require.NoError(t, blz.Init()) + require.NoError(t, blz.Init(context.Background())) // try to read non-existent address testGet(t, blz, oidtest.Address(), nil, client.IsErrObjectNotFound) @@ -102,5 +102,5 @@ func TestBlobovnicza(t *testing.T) { return err == nil }, nil) - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) } diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index a317279a4..d0e71a876 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -56,7 +56,7 @@ func (b *Blobovnicza) Open(ctx context.Context) error { // // If Blobovnicza is already initialized, no action is taken. // Blobovnicza must be open, otherwise an error will return. -func (b *Blobovnicza) Init() error { +func (b *Blobovnicza) Init(ctx context.Context) error { b.controlMtx.Lock() defer b.controlMtx.Unlock() @@ -64,7 +64,7 @@ func (b *Blobovnicza) Init() error { return errors.New("blobovnicza is not open") } - b.log.Debug(context.Background(), logs.BlobovniczaInitializing, + b.log.Debug(ctx, logs.BlobovniczaInitializing, zap.Uint64("object size limit", b.objSizeLimit), zap.Uint64("storage size limit", b.fullSizeLimit), ) @@ -72,7 +72,7 @@ func (b *Blobovnicza) Init() error { size := b.dataSize.Load() items := b.itemsCount.Load() if size != 0 || items != 0 { - b.log.Debug(context.Background(), logs.BlobovniczaAlreadyInitialized, zap.Uint64("size", size), zap.Uint64("items", items)) + b.log.Debug(ctx, logs.BlobovniczaAlreadyInitialized, zap.Uint64("size", size), zap.Uint64("items", items)) return nil } @@ -82,7 +82,7 @@ func (b *Blobovnicza) Init() error { // create size range bucket rangeStr := stringifyBounds(lower, upper) - b.log.Debug(context.Background(), logs.BlobovniczaCreatingBucketForSizeRange, + b.log.Debug(ctx, logs.BlobovniczaCreatingBucketForSizeRange, zap.String("range", rangeStr)) _, err := tx.CreateBucketIfNotExists(key) @@ -99,14 +99,14 @@ func (b *Blobovnicza) Init() error { } } - return b.initializeCounters() + return b.initializeCounters(ctx) } func (b *Blobovnicza) ObjectsCount() uint64 { return b.itemsCount.Load() } -func (b *Blobovnicza) initializeCounters() error { +func (b *Blobovnicza) initializeCounters(ctx context.Context) error { var size uint64 var items uint64 var sizeExists bool @@ -132,17 +132,17 @@ func (b *Blobovnicza) initializeCounters() error { return fmt.Errorf("can't determine DB size: %w", err) } if (!sizeExists || !itemsCountExists) && !b.boltOptions.ReadOnly { - b.log.Debug(context.Background(), logs.BlobovniczaSavingCountersToMeta, zap.Uint64("size", size), zap.Uint64("items", items)) + b.log.Debug(ctx, logs.BlobovniczaSavingCountersToMeta, zap.Uint64("size", size), zap.Uint64("items", items)) if err := b.boltDB.Update(func(tx *bbolt.Tx) error { if err := saveDataSize(tx, size); err != nil { return err } return saveItemsCount(tx, items) }); err != nil { - b.log.Debug(context.Background(), logs.BlobovniczaSavingCountersToMetaFailed, zap.Uint64("size", size), zap.Uint64("items", items)) + b.log.Debug(ctx, logs.BlobovniczaSavingCountersToMetaFailed, zap.Uint64("size", size), zap.Uint64("items", items)) return fmt.Errorf("can't save blobovnicza's size and items count: %w", err) } - b.log.Debug(context.Background(), logs.BlobovniczaSavingCountersToMetaSuccess, zap.Uint64("size", size), zap.Uint64("items", items)) + b.log.Debug(ctx, logs.BlobovniczaSavingCountersToMetaSuccess, zap.Uint64("size", size), zap.Uint64("items", items)) } b.dataSize.Store(size) @@ -155,7 +155,7 @@ func (b *Blobovnicza) initializeCounters() error { // Close releases all internal database resources. // // If blobovnicza is already closed, does nothing. -func (b *Blobovnicza) Close() error { +func (b *Blobovnicza) Close(ctx context.Context) error { b.controlMtx.Lock() defer b.controlMtx.Unlock() @@ -163,7 +163,7 @@ func (b *Blobovnicza) Close() error { return nil } - b.log.Debug(context.Background(), logs.BlobovniczaClosingBoltDB, + b.log.Debug(ctx, logs.BlobovniczaClosingBoltDB, zap.String("path", b.path), ) diff --git a/pkg/local_object_storage/blobovnicza/get_test.go b/pkg/local_object_storage/blobovnicza/get_test.go index 8c435af89..5a382c159 100644 --- a/pkg/local_object_storage/blobovnicza/get_test.go +++ b/pkg/local_object_storage/blobovnicza/get_test.go @@ -14,11 +14,11 @@ func TestBlobovnicza_Get(t *testing.T) { filename := filepath.Join(t.TempDir(), "blob") var blz *Blobovnicza - defer func() { require.NoError(t, blz.Close()) }() + defer func() { require.NoError(t, blz.Close(context.Background())) }() fnInit := func(szLimit uint64) { if blz != nil { - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) } blz = New( @@ -27,7 +27,7 @@ func TestBlobovnicza_Get(t *testing.T) { ) require.NoError(t, blz.Open(context.Background())) - require.NoError(t, blz.Init()) + require.NoError(t, blz.Init(context.Background())) } // initial distribution: [0:32K] (32K:64K] diff --git a/pkg/local_object_storage/blobovnicza/iterate_test.go b/pkg/local_object_storage/blobovnicza/iterate_test.go index bec23bb96..717274781 100644 --- a/pkg/local_object_storage/blobovnicza/iterate_test.go +++ b/pkg/local_object_storage/blobovnicza/iterate_test.go @@ -16,7 +16,7 @@ func TestBlobovniczaIterate(t *testing.T) { filename := filepath.Join(t.TempDir(), "blob") b := New(WithPath(filename)) require.NoError(t, b.Open(context.Background())) - require.NoError(t, b.Init()) + require.NoError(t, b.Init(context.Background())) data := [][]byte{{0, 1, 2, 3}, {5, 6, 7, 8}} addr := oidtest.Address() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/active.go b/pkg/local_object_storage/blobstor/blobovniczatree/active.go index 0ac15df82..dbaa7387a 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/active.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/active.go @@ -18,8 +18,8 @@ func (db *activeDB) Blobovnicza() *blobovnicza.Blobovnicza { return db.blz } -func (db *activeDB) Close() { - db.shDB.Close() +func (db *activeDB) Close(ctx context.Context) { + db.shDB.Close(ctx) } func (db *activeDB) SystemPath() string { @@ -73,12 +73,12 @@ func (m *activeDBManager) Open() { m.closed = false } -func (m *activeDBManager) Close() { +func (m *activeDBManager) Close(ctx context.Context) { m.levelToActiveDBGuard.Lock() defer m.levelToActiveDBGuard.Unlock() for _, db := range m.levelToActiveDB { - db.Close() + db.Close(ctx) } m.levelToActiveDB = make(map[string]*sharedDB) m.closed = true @@ -103,7 +103,7 @@ func (m *activeDBManager) getCurrentActiveIfOk(ctx context.Context, lvlPath stri } if blz.IsFull() { - db.Close() + db.Close(ctx) return nil, nil } @@ -168,10 +168,10 @@ func (m *activeDBManager) getNextSharedDB(ctx context.Context, lvlPath string) ( previous, updated := m.replace(lvlPath, next) if !updated && next != nil { - next.Close() // manager is closed, so don't hold active DB open + next.Close(ctx) // manager is closed, so don't hold active DB open } if updated && previous != nil { - previous.Close() + previous.Close(ctx) } return next, nil } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index c909113c7..d9e99d0d1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -167,7 +167,7 @@ func (b *Blobovniczas) Compressor() *compression.Config { } // SetReportErrorFunc implements common.Storage. -func (b *Blobovniczas) SetReportErrorFunc(f func(string, error)) { +func (b *Blobovniczas) SetReportErrorFunc(f func(context.Context, string, error)) { b.reportError = f } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go index e8016781a..04ff5120c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/cache.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/cache.go @@ -32,7 +32,7 @@ func newDBCache(parentCtx context.Context, size int, ch := cache.NewCache[string, *sharedDB](). WithTTL(ttl).WithLRU().WithMaxKeys(size). WithOnEvicted(func(_ string, db *sharedDB) { - db.Close() + db.Close(parentCtx) }) ctx, cancel := context.WithCancel(parentCtx) res := &dbCache{ @@ -138,7 +138,7 @@ func (c *dbCache) create(ctx context.Context, path string) *sharedDB { return value } if added := c.put(path, value); !added { - value.Close() + value.Close(ctx) } return value } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go index cc8a52d03..ec9743b57 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go @@ -27,7 +27,7 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) { require.NoError(t, st.Open(mode.ComponentReadWrite)) require.NoError(t, st.Init()) defer func() { - require.NoError(t, st.Close()) + require.NoError(t, st.Close(context.Background())) }() objGen := &testutil.SeqObjGenerator{ObjSize: 1} diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index 7c0a9edd6..c77df63bf 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -50,7 +50,7 @@ func (b *Blobovniczas) initializeDBs(ctx context.Context) error { if err != nil { return err } - defer shBlz.Close() + defer shBlz.Close(egCtx) moveInfo, err := blz.ListMoveInfo(egCtx) if err != nil { @@ -80,9 +80,9 @@ func (b *Blobovniczas) openManagers() { } // Close implements common.Storage. -func (b *Blobovniczas) Close() error { +func (b *Blobovniczas) Close(ctx context.Context) error { b.dbCache.Close() // order important - b.activeDBManager.Close() + b.activeDBManager.Close(ctx) b.commondbManager.Close() return nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go index f0a32ded1..b26323bd0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go @@ -51,7 +51,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { require.NoError(t, err) require.EqualValues(t, obj35, gRes.Object) - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) // change depth and width blz = NewBlobovniczaTree( @@ -89,7 +89,7 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { }) require.NoError(t, err) - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) // change depth and width back blz = NewBlobovniczaTree( @@ -127,5 +127,5 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { require.NoError(t, err) require.EqualValues(t, obj52, gRes.Object) - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/count.go b/pkg/local_object_storage/blobstor/blobovniczatree/count.go index 1137b9eb2..b83849c77 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/count.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/count.go @@ -26,7 +26,7 @@ func (b *Blobovniczas) ObjectsCount(ctx context.Context) (uint64, error) { if err != nil { return true, err } - defer shDB.Close() + defer shDB.Close(ctx) result += blz.ObjectsCount() return false, nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index b26e44144..8c2d7aa67 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -66,7 +66,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co if err != nil { return res, err } - defer shBlz.Close() + defer shBlz.Close(ctx) if res, err = b.deleteObject(ctx, blz, bPrm); err == nil { success = true @@ -114,7 +114,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicz if err != nil { return common.DeleteRes{}, err } - defer shBlz.Close() + defer shBlz.Close(ctx) return b.deleteObject(ctx, blz, prm) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index 528dbfed7..63d2f21e1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -42,7 +42,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common if err != nil { return common.ExistsRes{}, err } - defer shBlz.Close() + defer shBlz.Close(ctx) exists, err := blz.Exists(ctx, prm.Address) return common.ExistsRes{Exists: exists}, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index d6ffd8bce..5414140f0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -27,7 +27,7 @@ func TestExistsInvalidStorageID(t *testing.T) { WithBlobovniczaSize(1<<20)) require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) - defer func() { require.NoError(t, b.Close()) }() + defer func() { require.NoError(t, b.Close(context.Background())) }() obj := blobstortest.NewObject(1024) addr := object.AddressOf(obj) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index fc017f22d..b7ef8d8a5 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -53,7 +53,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G if err != nil { return res, err } - defer shBlz.Close() + defer shBlz.Close(ctx) res, err = b.getObject(ctx, blz, bPrm) if err == nil { @@ -100,7 +100,7 @@ func (b *Blobovniczas) getObjectFromLevel(ctx context.Context, prm blobovnicza.G if err != nil { return common.GetRes{}, err } - defer shBlz.Close() + defer shBlz.Close(ctx) return b.getObject(ctx, blz, prm) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 384544d7b..b24f1b881 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -52,7 +52,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re if err != nil { return common.GetRangeRes{}, err } - defer shBlz.Close() + defer shBlz.Close(ctx) res, err := b.getObjectRange(ctx, blz, prm) if err == nil { @@ -108,7 +108,7 @@ func (b *Blobovniczas) getRangeFromLevel(ctx context.Context, prm common.GetRang if err != nil { return common.GetRangeRes{}, err } - defer shBlz.Close() + defer shBlz.Close(ctx) return b.getObjectRange(ctx, blz, prm) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 049a61d72..b120c22f7 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -84,7 +84,7 @@ func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors boo } return false, fmt.Errorf("could not open blobovnicza %s: %w", p, err) } - defer shBlz.Close() + defer shBlz.Close(ctx) err = f(p, blz) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index 502202d68..b35e052cf 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -71,7 +71,7 @@ func (b *sharedDB) Open(ctx context.Context) (*blobovnicza.Blobovnicza, error) { if err := blz.Open(ctx); err != nil { return nil, fmt.Errorf("could not open blobovnicza %s: %w", b.path, err) } - if err := blz.Init(); err != nil { + if err := blz.Init(ctx); err != nil { return nil, fmt.Errorf("could not init blobovnicza %s: %w", b.path, err) } @@ -82,20 +82,20 @@ func (b *sharedDB) Open(ctx context.Context) (*blobovnicza.Blobovnicza, error) { return blz, nil } -func (b *sharedDB) Close() { +func (b *sharedDB) Close(ctx context.Context) { b.cond.L.Lock() defer b.cond.L.Unlock() if b.refCount == 0 { - b.log.Error(context.Background(), logs.AttemtToCloseAlreadyClosedBlobovnicza, zap.String("id", b.path)) + b.log.Error(ctx, logs.AttemtToCloseAlreadyClosedBlobovnicza, zap.String("id", b.path)) b.cond.Broadcast() return } if b.refCount == 1 { b.refCount = 0 - if err := b.blcza.Close(); err != nil { - b.log.Error(context.Background(), logs.BlobovniczatreeCouldNotCloseBlobovnicza, + if err := b.blcza.Close(ctx); err != nil { + b.log.Error(ctx, logs.BlobovniczatreeCouldNotCloseBlobovnicza, zap.String("id", b.path), zap.String("error", err.Error()), ) @@ -111,7 +111,7 @@ func (b *sharedDB) Close() { } } -func (b *sharedDB) CloseAndRemoveFile() error { +func (b *sharedDB) CloseAndRemoveFile(ctx context.Context) error { b.cond.L.Lock() if b.refCount > 1 { b.cond.Wait() @@ -122,8 +122,8 @@ func (b *sharedDB) CloseAndRemoveFile() error { return errClosingClosedBlobovnicza } - if err := b.blcza.Close(); err != nil { - b.log.Error(context.Background(), logs.BlobovniczatreeCouldNotCloseBlobovnicza, + if err := b.blcza.Close(ctx); err != nil { + b.log.Error(ctx, logs.BlobovniczatreeCouldNotCloseBlobovnicza, zap.String("id", b.path), zap.String("error", err.Error()), ) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index b56251772..0e1b2022e 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -1,6 +1,7 @@ package blobovniczatree import ( + "context" "io/fs" "time" @@ -20,7 +21,7 @@ type cfg struct { blzShallowWidth uint64 compression *compression.Config blzOpts []blobovnicza.Option - reportError func(string, error) // reportError is the function called when encountering disk errors. + reportError func(context.Context, string, error) // reportError is the function called when encountering disk errors. metrics Metrics waitBeforeDropDB time.Duration blzInitWorkerCount int @@ -54,7 +55,7 @@ func initConfig(c *cfg) { openedCacheExpInterval: defaultOpenedCacheInterval, blzShallowDepth: defaultBlzShallowDepth, blzShallowWidth: defaultBlzShallowWidth, - reportError: func(string, error) {}, + reportError: func(context.Context, string, error) {}, metrics: &noopMetrics{}, waitBeforeDropDB: defaultWaitBeforeDropDB, blzInitWorkerCount: defaultBlzInitWorkerCount, diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 8dff436d3..1678e578c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -80,7 +80,7 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) active, err := i.B.activeDBManager.GetOpenedActiveDBForLevel(ctx, lvlPath) if err != nil { if !isLogical(err) { - i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) + i.B.reportError(ctx, logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) } else { i.B.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, zap.String("error", err.Error()), @@ -95,14 +95,14 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return false, nil } - defer active.Close() + defer active.Close(ctx) i.AllFull = false _, err = active.Blobovnicza().Put(ctx, i.PutPrm) if err != nil { if !isLogical(err) { - i.B.reportError(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err) + i.B.reportError(ctx, logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err) } else { i.B.log.Debug(ctx, logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", active.SystemPath()), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index e137bdd99..16ef2b180 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -186,7 +186,7 @@ func (b *Blobovniczas) rebuildBySize(ctx context.Context, path string, targetFil if err != nil { return false, err } - defer shDB.Close() + defer shDB.Close(ctx) fp := blz.FillPercent() // accepted fill percent defines as // |----|+++++++++++++++++|+++++++++++++++++|--------------- @@ -206,9 +206,9 @@ func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.M if shDBClosed { return } - shDB.Close() + shDB.Close(ctx) }() - dropTempFile, err := b.addRebuildTempFile(path) + dropTempFile, err := b.addRebuildTempFile(ctx, path) if err != nil { return 0, err } @@ -224,7 +224,7 @@ func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.M return migratedObjects, err } -func (b *Blobovniczas) addRebuildTempFile(path string) (func(), error) { +func (b *Blobovniczas) addRebuildTempFile(ctx context.Context, path string) (func(), error) { sysPath := filepath.Join(b.rootPath, path) sysPath = sysPath + rebuildSuffix _, err := os.OpenFile(sysPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, b.perm) @@ -233,7 +233,7 @@ func (b *Blobovniczas) addRebuildTempFile(path string) (func(), error) { } return func() { if err := os.Remove(sysPath); err != nil { - b.log.Warn(context.Background(), logs.BlobovniczatreeFailedToRemoveRebuildTempFile, zap.Error(err)) + b.log.Warn(ctx, logs.BlobovniczatreeFailedToRemoveRebuildTempFile, zap.Error(err)) } }, nil } @@ -330,7 +330,7 @@ func (b *Blobovniczas) dropDB(ctx context.Context, path string, shDb *sharedDB) b.dbFilesGuard.Lock() defer b.dbFilesGuard.Unlock() - if err := shDb.CloseAndRemoveFile(); err != nil { + if err := shDb.CloseAndRemoveFile(ctx); err != nil { return false, err } b.commondbManager.CleanResources(path) @@ -370,7 +370,7 @@ func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore co if err != nil { return true, err } - defer shDB.Close() + defer shDB.Close(ctx) incompletedMoves, err := blz.ListMoveInfo(ctx) if err != nil { @@ -403,7 +403,7 @@ func (b *Blobovniczas) performMove(ctx context.Context, source *blobovnicza.Blob if err != nil { return err } - defer targetDB.Close() + defer targetDB.Close(ctx) existsInSource := true var gPrm blobovnicza.GetPrm @@ -480,7 +480,7 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, target, err := i.B.activeDBManager.GetOpenedActiveDBForLevel(ctx, lvlPath) if err != nil { if !isLogical(err) { - i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) + i.B.reportError(ctx, logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) } else { i.B.log.Warn(ctx, logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, zap.Error(err)) } @@ -491,7 +491,7 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, i.B.log.Warn(ctx, logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath)) return false, nil } - defer target.Close() + defer target.Close(ctx) i.AllFull = false @@ -503,7 +503,7 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, TargetStorageID: targetStorageID.Bytes(), }); err != nil { if !isLogical(err) { - i.B.reportError(logs.BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza, err) + i.B.reportError(ctx, logs.BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza, err) } else { i.B.log.Warn(ctx, logs.BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza, zap.String("path", i.SourceSysPath), zap.Error(err)) } @@ -519,7 +519,7 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, _, err = target.Blobovnicza().Put(ctx, putPrm) if err != nil { if !isLogical(err) { - i.B.reportError(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err) + i.B.reportError(ctx, logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err) } else { i.B.log.Warn(ctx, logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", target.SystemPath()), zap.Error(err)) } @@ -535,7 +535,7 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, deletePrm.SetAddress(i.Address) if _, err = i.Source.Delete(ctx, deletePrm); err != nil { if !isLogical(err) { - i.B.reportError(logs.BlobovniczatreeCouldNotDeleteFromSource, err) + i.B.reportError(ctx, logs.BlobovniczatreeCouldNotDeleteFromSource, err) } else { i.B.log.Warn(ctx, logs.BlobovniczatreeCouldNotDeleteFromSource, zap.String("path", i.SourceSysPath), zap.Error(err)) } @@ -544,7 +544,7 @@ func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, if err = i.Source.DropMoveInfo(ctx, i.Address); err != nil { if !isLogical(err) { - i.B.reportError(logs.BlobovniczatreeCouldNotDropMoveInfo, err) + i.B.reportError(ctx, logs.BlobovniczatreeCouldNotDropMoveInfo, err) } else { i.B.log.Warn(ctx, logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", i.SourceSysPath), zap.Error(err)) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index bfea97afe..2f58624aa 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -36,7 +36,7 @@ func testRebuildFailoverOnlyMoveInfoSaved(t *testing.T) { blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) require.NoError(t, blz.Open(context.Background())) - require.NoError(t, blz.Init()) + require.NoError(t, blz.Init(context.Background())) obj := blobstortest.NewObject(1024) data, err := obj.Marshal() @@ -53,7 +53,7 @@ func testRebuildFailoverOnlyMoveInfoSaved(t *testing.T) { TargetStorageID: []byte("0/0/0"), })) - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) _, err = os.OpenFile(filepath.Join(dir, "0", "0", "1.db.rebuild"), os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, defaultPerm) require.NoError(t, err) @@ -66,7 +66,7 @@ func testRebuildFailoverObjectSavedToTarget(t *testing.T) { blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) require.NoError(t, blz.Open(context.Background())) - require.NoError(t, blz.Init()) + require.NoError(t, blz.Init(context.Background())) obj := blobstortest.NewObject(1024) data, err := obj.Marshal() @@ -83,19 +83,19 @@ func testRebuildFailoverObjectSavedToTarget(t *testing.T) { TargetStorageID: []byte("0/0/0"), })) - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) _, err = os.OpenFile(filepath.Join(dir, "0", "0", "1.db.rebuild"), os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, defaultPerm) require.NoError(t, err) blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db"))) require.NoError(t, blz.Open(context.Background())) - require.NoError(t, blz.Init()) + require.NoError(t, blz.Init(context.Background())) _, err = blz.Put(context.Background(), pPrm) require.NoError(t, err) - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) testRebuildFailoverValidate(t, dir, obj, true) } @@ -106,7 +106,7 @@ func testRebuildFailoverObjectDeletedFromSource(t *testing.T) { blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) require.NoError(t, blz.Open(context.Background())) - require.NoError(t, blz.Init()) + require.NoError(t, blz.Init(context.Background())) obj := blobstortest.NewObject(1024) data, err := obj.Marshal() @@ -117,14 +117,14 @@ func testRebuildFailoverObjectDeletedFromSource(t *testing.T) { TargetStorageID: []byte("0/0/0"), })) - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) _, err = os.OpenFile(filepath.Join(dir, "0", "0", "1.db.rebuild"), os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, defaultPerm) require.NoError(t, err) blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db"))) require.NoError(t, blz.Open(context.Background())) - require.NoError(t, blz.Init()) + require.NoError(t, blz.Init(context.Background())) var pPrm blobovnicza.PutPrm pPrm.SetAddress(object.AddressOf(obj)) @@ -132,7 +132,7 @@ func testRebuildFailoverObjectDeletedFromSource(t *testing.T) { _, err = blz.Put(context.Background(), pPrm) require.NoError(t, err) - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) testRebuildFailoverValidate(t, dir, obj, false) } @@ -170,11 +170,11 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object require.Equal(t, uint64(1), rRes.ObjectsMoved) require.Equal(t, uint64(0), rRes.FilesRemoved) - require.NoError(t, b.Close()) + require.NoError(t, b.Close(context.Background())) blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) require.NoError(t, blz.Open(context.Background())) - require.NoError(t, blz.Init()) + require.NoError(t, blz.Init(context.Background())) moveInfo, err := blz.ListMoveInfo(context.Background()) require.NoError(t, err) @@ -185,11 +185,11 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object _, err = blz.Get(context.Background(), gPrm) require.True(t, client.IsErrObjectNotFound(err)) - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db"))) require.NoError(t, blz.Open(context.Background())) - require.NoError(t, blz.Init()) + require.NoError(t, blz.Init(context.Background())) moveInfo, err = blz.ListMoveInfo(context.Background()) require.NoError(t, err) @@ -203,7 +203,7 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object require.True(t, bytes.Equal([]byte("0/0/0"), metaStub.storageIDs[object.AddressOf(obj)])) } - require.NoError(t, blz.Close()) + require.NoError(t, blz.Close(context.Background())) _, err = os.Stat(filepath.Join(dir, "0", "0", "1.db.rebuild")) require.True(t, os.IsNotExist(err)) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index dff4e9024..aae72b5ff 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -93,7 +93,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { require.NoError(t, err) } - require.NoError(t, b.Close()) + require.NoError(t, b.Close(context.Background())) }) t.Run("no rebuild single db", func(t *testing.T) { @@ -145,7 +145,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { require.NoError(t, err) } - require.NoError(t, b.Close()) + require.NoError(t, b.Close(context.Background())) }) t.Run("rebuild by fill percent", func(t *testing.T) { @@ -214,7 +214,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { require.NoError(t, err) } - require.NoError(t, b.Close()) + require.NoError(t, b.Close(context.Background())) }) t.Run("rebuild by overflow", func(t *testing.T) { @@ -251,7 +251,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { storageIDs: storageIDs, guard: &sync.Mutex{}, } - require.NoError(t, b.Close()) + require.NoError(t, b.Close(context.Background())) b = NewBlobovniczaTree( context.Background(), WithLogger(test.NewLogger(t)), @@ -284,7 +284,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { require.NoError(t, err) } - require.NoError(t, b.Close()) + require.NoError(t, b.Close(context.Background())) }) } @@ -318,7 +318,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { storageIDs := make(map[oid.Address][]byte) storageIDs[prm.Address] = res.StorageID - require.NoError(t, b.Close()) + require.NoError(t, b.Close(context.Background())) b = NewBlobovniczaTree( context.Background(), @@ -355,7 +355,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { require.NoError(t, err) } - require.NoError(t, b.Close()) + require.NoError(t, b.Close(context.Background())) } func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, targetDepth, targetWidth uint64, shouldMigrate bool) { @@ -399,7 +399,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta } require.NoError(t, eg.Wait()) - require.NoError(t, b.Close()) + require.NoError(t, b.Close(context.Background())) b = NewBlobovniczaTree( context.Background(), @@ -444,7 +444,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta require.NoError(t, err) } - require.NoError(t, b.Close()) + require.NoError(t, b.Close(context.Background())) } type storageIDUpdateStub struct { diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index 41c6cf161..f850f48b4 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -1,6 +1,7 @@ package blobstor import ( + "context" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -139,7 +140,7 @@ func WithUncompressableContentTypes(values []string) Option { // SetReportErrorFunc allows to provide a function to be called on disk errors. // This function MUST be called before Open. -func (b *BlobStor) SetReportErrorFunc(f func(string, error)) { +func (b *BlobStor) SetReportErrorFunc(f func(context.Context, string, error)) { for i := range b.storage { b.storage[i].Storage.SetReportErrorFunc(f) } diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index bed5e0eb9..6cc56fa3b 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -54,7 +54,7 @@ func TestCompression(t *testing.T) { WithCompressObjects(compress), WithStorages(defaultStorages(dir, smallSizeLimit))) require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, bs.Init()) + require.NoError(t, bs.Init(context.Background())) return bs } @@ -91,20 +91,20 @@ func TestCompression(t *testing.T) { blobStor := newBlobStor(t, false) testPut(t, blobStor, 0) testGet(t, blobStor, 0) - require.NoError(t, blobStor.Close()) + require.NoError(t, blobStor.Close(context.Background())) blobStor = newBlobStor(t, true) testGet(t, blobStor, 0) // get uncompressed object with compress enabled testPut(t, blobStor, 1) testGet(t, blobStor, 1) - require.NoError(t, blobStor.Close()) + require.NoError(t, blobStor.Close(context.Background())) blobStor = newBlobStor(t, false) testGet(t, blobStor, 0) // get old uncompressed object testGet(t, blobStor, 1) // get compressed object with compression disabled testPut(t, blobStor, 2) testGet(t, blobStor, 2) - require.NoError(t, blobStor.Close()) + require.NoError(t, blobStor.Close(context.Background())) } func TestBlobstor_needsCompression(t *testing.T) { @@ -130,7 +130,7 @@ func TestBlobstor_needsCompression(t *testing.T) { }, })) require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, bs.Init()) + require.NoError(t, bs.Init(context.Background())) return bs } @@ -192,7 +192,7 @@ func TestConcurrentPut(t *testing.T) { blobStor := New( WithStorages(defaultStorages(dir, smallSizeLimit))) require.NoError(t, blobStor.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, blobStor.Init()) + require.NoError(t, blobStor.Init(context.Background())) testGet := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { res, err := b.Get(context.Background(), common.GetPrm{Address: object.AddressOf(obj)}) @@ -272,7 +272,7 @@ func TestConcurrentDelete(t *testing.T) { blobStor := New( WithStorages(defaultStorages(dir, smallSizeLimit))) require.NoError(t, blobStor.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, blobStor.Init()) + require.NoError(t, blobStor.Init(context.Background())) testPut := func(t *testing.T, b *BlobStor, obj *objectSDK.Object) { var prm common.PutPrm diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 4f3a20993..6ecef48cd 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -12,7 +12,7 @@ import ( type Storage interface { Open(mode mode.ComponentMode) error Init() error - Close() error + Close(context.Context) error Type() string Path() string @@ -23,7 +23,7 @@ type Storage interface { // SetReportErrorFunc allows to provide a function to be called on disk errors. // This function MUST be called before Open. - SetReportErrorFunc(f func(string, error)) + SetReportErrorFunc(f func(context.Context, string, error)) SetParentID(parentID string) Get(context.Context, GetPrm) (GetRes, error) diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 43436b4eb..44685524f 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -50,8 +50,8 @@ var ErrInitBlobovniczas = errors.New("failure on blobovnicza initialization stag // If BlobStor is already initialized, no action is taken. // // Returns wrapped ErrInitBlobovniczas on blobovnicza tree's initializaiton failure. -func (b *BlobStor) Init() error { - b.log.Debug(context.Background(), logs.BlobstorInitializing) +func (b *BlobStor) Init(ctx context.Context) error { + b.log.Debug(ctx, logs.BlobstorInitializing) if err := b.compression.Init(); err != nil { return err @@ -67,14 +67,14 @@ func (b *BlobStor) Init() error { } // Close releases all internal resources of BlobStor. -func (b *BlobStor) Close() error { - b.log.Debug(context.Background(), logs.BlobstorClosing) +func (b *BlobStor) Close(ctx context.Context) error { + b.log.Debug(ctx, logs.BlobstorClosing) var firstErr error for i := range b.storage { - err := b.storage[i].Storage.Close() + err := b.storage[i].Storage.Close(ctx) if err != nil { - b.log.Info(context.Background(), logs.BlobstorCouldntCloseStorage, zap.String("error", err.Error())) + b.log.Info(ctx, logs.BlobstorCouldntCloseStorage, zap.String("error", err.Error())) if firstErr == nil { firstErr = err } diff --git a/pkg/local_object_storage/blobstor/exists_test.go b/pkg/local_object_storage/blobstor/exists_test.go index 783c198b2..7eb7d49bf 100644 --- a/pkg/local_object_storage/blobstor/exists_test.go +++ b/pkg/local_object_storage/blobstor/exists_test.go @@ -22,7 +22,7 @@ func TestExists(t *testing.T) { b := New(WithStorages(storages)) require.NoError(t, b.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, b.Init()) + require.NoError(t, b.Init(context.Background())) objects := []*objectSDK.Object{ testObject(smallSizeLimit / 2), diff --git a/pkg/local_object_storage/blobstor/fstree/control.go b/pkg/local_object_storage/blobstor/fstree/control.go index c21d79f09..2544729f7 100644 --- a/pkg/local_object_storage/blobstor/fstree/control.go +++ b/pkg/local_object_storage/blobstor/fstree/control.go @@ -1,6 +1,8 @@ package fstree import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" ) @@ -28,7 +30,7 @@ func (t *FSTree) Init() error { } // Close implements common.Storage. -func (t *FSTree) Close() error { +func (t *FSTree) Close(_ context.Context) error { t.metrics.Close() return nil } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 7f52762a7..53eb0395a 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -606,7 +606,7 @@ func (t *FSTree) Compressor() *compression.Config { } // SetReportErrorFunc implements common.Storage. -func (t *FSTree) SetReportErrorFunc(_ func(string, error)) { +func (t *FSTree) SetReportErrorFunc(_ func(context.Context, string, error)) { // Do nothing, FSTree can encounter only one error which is returned. } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_test.go b/pkg/local_object_storage/blobstor/fstree/fstree_test.go index eb2126b6c..50dae46a7 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_test.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_test.go @@ -52,7 +52,7 @@ func TestObjectCounter(t *testing.T) { require.Equal(t, uint64(0), size) defer func() { - require.NoError(t, fst.Close()) + require.NoError(t, fst.Close(context.Background())) }() addr := oidtest.Address() diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go index 21c80b089..b8e88f84a 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/control.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/control.go @@ -19,7 +19,7 @@ func TestControl(t *testing.T, cons Constructor, minSize, maxSize uint64) { require.NoError(t, s.Init()) objects := prepare(t, 10, s, minSize, maxSize) - require.NoError(t, s.Close()) + require.NoError(t, s.Close(context.Background())) require.NoError(t, s.Open(mode.ComponentReadOnly)) for i := range objects { diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go index cf4e76513..3a163f6b1 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/delete.go @@ -15,7 +15,7 @@ func TestDelete(t *testing.T, cons Constructor, minSize, maxSize uint64) { s := cons(t) require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() objects := prepare(t, 4, s, minSize, maxSize) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go index 08465ed5e..f34fe5f97 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/exists.go @@ -14,7 +14,7 @@ func TestExists(t *testing.T, cons Constructor, minSize, maxSize uint64) { s := cons(t) require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() objects := prepare(t, 1, s, minSize, maxSize) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go index d1f709b0c..af0f4b45d 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get.go @@ -15,7 +15,7 @@ func TestGet(t *testing.T, cons Constructor, minSize, maxSize uint64) { s := cons(t) require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() objects := prepare(t, 2, s, minSize, maxSize) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go index fcbeddac7..13032048c 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/get_range.go @@ -17,7 +17,7 @@ func TestGetRange(t *testing.T, cons Constructor, minSize, maxSize uint64) { s := cons(t) require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() objects := prepare(t, 1, s, minSize, maxSize) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index 3a6c8b699..36b2c33f8 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -14,7 +14,7 @@ func TestIterate(t *testing.T, cons Constructor, minSize, maxSize uint64) { s := cons(t) require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() objects := prepare(t, 10, s, minSize, maxSize) diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index 3c9457db2..ccfa510fe 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -33,9 +33,9 @@ func TestIterateObjects(t *testing.T) { require.NoError(t, blobStor.Open(context.Background(), mode.ReadWrite)) // initialize Blobstor - require.NoError(t, blobStor.Init()) + require.NoError(t, blobStor.Init(context.Background())) - defer blobStor.Close() + defer blobStor.Close(context.Background()) const objNum = 5 @@ -118,7 +118,7 @@ func TestIterate_IgnoreErrors(t *testing.T) { })} bs := New(bsOpts...) require.NoError(t, bs.Open(ctx, mode.ReadWrite)) - require.NoError(t, bs.Init()) + require.NoError(t, bs.Init(ctx)) nopHandler := func(e common.IterationElement) error { return nil diff --git a/pkg/local_object_storage/blobstor/memstore/control.go b/pkg/local_object_storage/blobstor/memstore/control.go index 83da52eb7..95a916662 100644 --- a/pkg/local_object_storage/blobstor/memstore/control.go +++ b/pkg/local_object_storage/blobstor/memstore/control.go @@ -1,6 +1,8 @@ package memstore import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) @@ -10,11 +12,11 @@ func (s *memstoreImpl) Open(mod mode.ComponentMode) error { return nil } -func (s *memstoreImpl) Init() error { return nil } -func (s *memstoreImpl) Close() error { return nil } -func (s *memstoreImpl) Type() string { return Type } -func (s *memstoreImpl) Path() string { return s.rootPath } -func (s *memstoreImpl) SetCompressor(cc *compression.Config) { s.compression = cc } -func (s *memstoreImpl) Compressor() *compression.Config { return s.compression } -func (s *memstoreImpl) SetReportErrorFunc(func(string, error)) {} -func (s *memstoreImpl) SetParentID(string) {} +func (s *memstoreImpl) Init() error { return nil } +func (s *memstoreImpl) Close(context.Context) error { return nil } +func (s *memstoreImpl) Type() string { return Type } +func (s *memstoreImpl) Path() string { return s.rootPath } +func (s *memstoreImpl) SetCompressor(cc *compression.Config) { s.compression = cc } +func (s *memstoreImpl) Compressor() *compression.Config { return s.compression } +func (s *memstoreImpl) SetReportErrorFunc(func(context.Context, string, error)) {} +func (s *memstoreImpl) SetParentID(string) {} diff --git a/pkg/local_object_storage/blobstor/memstore/memstore_test.go b/pkg/local_object_storage/blobstor/memstore/memstore_test.go index dd130e5f9..f904d4232 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore_test.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore_test.go @@ -16,7 +16,7 @@ func TestSimpleLifecycle(t *testing.T) { s := New( WithRootPath("memstore"), ) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() require.NoError(t, s.Open(mode.ComponentReadWrite)) require.NoError(t, s.Init()) diff --git a/pkg/local_object_storage/blobstor/mode.go b/pkg/local_object_storage/blobstor/mode.go index f081ff645..af19e398e 100644 --- a/pkg/local_object_storage/blobstor/mode.go +++ b/pkg/local_object_storage/blobstor/mode.go @@ -20,10 +20,10 @@ func (b *BlobStor) SetMode(ctx context.Context, m mode.Mode) error { return nil } - err := b.Close() + err := b.Close(ctx) if err == nil { if err = b.openBlobStor(ctx, m); err == nil { - err = b.Init() + err = b.Init(ctx) } } if err != nil { diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index 1ac769e36..64e3c8da1 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -106,7 +106,7 @@ func BenchmarkSubstorageReadPerf(b *testing.B) { b.Run(fmt.Sprintf("%s-%s", stEntry.desc, tt.desc), func(b *testing.B) { objGen := tt.objGen() st := stEntry.open(b) - defer func() { require.NoError(b, st.Close()) }() + defer func() { require.NoError(b, st.Close(context.Background())) }() // Fill database var errG errgroup.Group @@ -161,7 +161,7 @@ func BenchmarkSubstorageWritePerf(b *testing.B) { b.Run(fmt.Sprintf("%s-%s", stEntry.desc, genEntry.desc), func(b *testing.B) { gen := genEntry.create() st := stEntry.open(b) - defer func() { require.NoError(b, st.Close()) }() + defer func() { require.NoError(b, st.Close(context.Background())) }() b.ResetTimer() b.RunParallel(func(pb *testing.PB) { @@ -200,7 +200,7 @@ func BenchmarkSubstorageIteratePerf(b *testing.B) { b.Run(fmt.Sprintf("%s-%s", stEntry.desc, tt.desc), func(b *testing.B) { objGen := tt.objGen() st := stEntry.open(b) - defer func() { require.NoError(b, st.Close()) }() + defer func() { require.NoError(b, st.Close(context.Background())) }() // Fill database for range tt.size { diff --git a/pkg/local_object_storage/blobstor/teststore/option.go b/pkg/local_object_storage/blobstor/teststore/option.go index bc0bed49d..fb1188751 100644 --- a/pkg/local_object_storage/blobstor/teststore/option.go +++ b/pkg/local_object_storage/blobstor/teststore/option.go @@ -1,6 +1,8 @@ package teststore import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -17,7 +19,7 @@ type cfg struct { Path func() string SetCompressor func(cc *compression.Config) Compressor func() *compression.Config - SetReportErrorFunc func(f func(string, error)) + SetReportErrorFunc func(f func(context.Context, string, error)) Get func(common.GetPrm) (common.GetRes, error) GetRange func(common.GetRangePrm) (common.GetRangeRes, error) @@ -51,7 +53,7 @@ func WithCompressor(f func() *compression.Config) Option { return func(c *cfg) { c.overrides.Compressor = f } } -func WithReportErrorFunc(f func(func(string, error))) Option { +func WithReportErrorFunc(f func(func(context.Context, string, error))) Option { return func(c *cfg) { c.overrides.SetReportErrorFunc = f } } diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index fea4a2d49..626ba0023 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -77,14 +77,14 @@ func (s *TestStore) Init() error { } } -func (s *TestStore) Close() error { +func (s *TestStore) Close(ctx context.Context) error { s.mu.RLock() defer s.mu.RUnlock() switch { case s.overrides.Close != nil: return s.overrides.Close() case s.st != nil: - return s.st.Close() + return s.st.Close(ctx) default: panic("unexpected storage call: Close()") } @@ -142,7 +142,7 @@ func (s *TestStore) Compressor() *compression.Config { } } -func (s *TestStore) SetReportErrorFunc(f func(string, error)) { +func (s *TestStore) SetReportErrorFunc(f func(context.Context, string, error)) { s.mu.RLock() defer s.mu.RUnlock() switch { diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index a52436175..24059a3f9 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -44,11 +44,11 @@ func (r ListContainersRes) Containers() []cid.ID { // ContainerSize returns the sum of estimation container sizes among all shards. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) ContainerSize(prm ContainerSizePrm) (res ContainerSizeRes, err error) { +func (e *StorageEngine) ContainerSize(ctx context.Context, prm ContainerSizePrm) (res ContainerSizeRes, err error) { defer elapsed("ContainerSize", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res, err = e.containerSize(prm) + res, err = e.containerSize(ctx, prm) return err }) @@ -56,12 +56,12 @@ func (e *StorageEngine) ContainerSize(prm ContainerSizePrm) (res ContainerSizeRe } // ContainerSize calls ContainerSize method on engine to calculate sum of estimation container sizes among all shards. -func ContainerSize(e *StorageEngine, id cid.ID) (uint64, error) { +func ContainerSize(ctx context.Context, e *StorageEngine, id cid.ID) (uint64, error) { var prm ContainerSizePrm prm.SetContainerID(id) - res, err := e.ContainerSize(prm) + res, err := e.ContainerSize(ctx, prm) if err != nil { return 0, err } @@ -69,14 +69,14 @@ func ContainerSize(e *StorageEngine, id cid.ID) (uint64, error) { return res.Size(), nil } -func (e *StorageEngine) containerSize(prm ContainerSizePrm) (res ContainerSizeRes, err error) { +func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) (res ContainerSizeRes, err error) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { var csPrm shard.ContainerSizePrm csPrm.SetContainerID(prm.cnr) csRes, err := sh.Shard.ContainerSize(csPrm) if err != nil { - e.reportShardError(sh, "can't get container size", err, + e.reportShardError(ctx, sh, "can't get container size", err, zap.Stringer("container_id", prm.cnr)) return false } @@ -121,7 +121,7 @@ func (e *StorageEngine) listContainers(ctx context.Context) (ListContainersRes, e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { res, err := sh.Shard.ListContainers(ctx, shard.ListContainersPrm{}) if err != nil { - e.reportShardError(sh, "can't get list of containers", err) + e.reportShardError(ctx, sh, "can't get list of containers", err) return false } diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 2e957eb04..7164ff21f 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -97,7 +97,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e return false } else { if !client.IsErrObjectNotFound(err) { - e.reportShardError(sh, "could not check object existence", err, zap.Stringer("address", prm.addr)) + e.reportShardError(ctx, sh, "could not check object existence", err, zap.Stringer("address", prm.addr)) } return false } @@ -113,7 +113,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e _, err = sh.Inhume(ctx, shPrm) if err != nil { - e.reportShardError(sh, "could not inhume object in shard", err, zap.Stringer("address", prm.addr)) + e.reportShardError(ctx, sh, "could not inhume object in shard", err, zap.Stringer("address", prm.addr)) var target *apistatus.ObjectLocked locked.is = errors.As(err, &target) @@ -188,7 +188,7 @@ func (e *StorageEngine) deleteChunks( var objID oid.ID err := objID.ReadFromV2(chunk.ID) if err != nil { - e.reportShardError(sh, "could not delete EC chunk", err, zap.Stringer("address", prm.addr)) + e.reportShardError(ctx, sh, "could not delete EC chunk", err, zap.Stringer("address", prm.addr)) } addr.SetObject(objID) inhumePrm.MarkAsGarbage(addr) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 6e30ee9de..029904046 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -99,24 +99,24 @@ func (e *StorageEngine) moveToDegraded(ctx context.Context, sh *shard.Shard, err if isMeta { err := sh.SetMode(ctx, mode.DegradedReadOnly) if err == nil { - log.Info(context.Background(), logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) + log.Info(ctx, logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold) return } - log.Error(context.Background(), logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, + log.Error(ctx, logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly, zap.Error(err)) } err := sh.SetMode(ctx, mode.ReadOnly) if err != nil { - log.Error(context.Background(), logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) + log.Error(ctx, logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err)) return } - log.Info(context.Background(), logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) + log.Info(ctx, logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold) } // reportShardErrorByID increases shard error counter and logs an error. -func (e *StorageEngine) reportShardErrorByID(id string, msg string, err error) { +func (e *StorageEngine) reportShardErrorByID(ctx context.Context, id string, msg string, err error) { e.mtx.RLock() sh, ok := e.shards[id] e.mtx.RUnlock() @@ -125,19 +125,20 @@ func (e *StorageEngine) reportShardErrorByID(id string, msg string, err error) { return } - e.reportShardError(sh, msg, err) + e.reportShardError(ctx, sh, msg, err) } // reportShardError checks that the amount of errors doesn't exceed the configured threshold. // If it does, shard is set to read-only mode. func (e *StorageEngine) reportShardError( + ctx context.Context, sh hashedShard, msg string, err error, fields ...zap.Field, ) { if isLogical(err) { - e.log.Warn(context.Background(), msg, + e.log.Warn(ctx, msg, zap.Stringer("shard_id", sh.ID()), zap.String("error", err.Error())) return @@ -147,7 +148,7 @@ func (e *StorageEngine) reportShardError( e.metrics.IncErrorCounter(sh.ID().String()) sid := sh.ID() - e.log.Warn(context.Background(), msg, append([]zap.Field{ + e.log.Warn(ctx, msg, append([]zap.Field{ zap.Stringer("shard_id", sid), zap.Uint32("error count", errCount), zap.String("error", err.Error()), @@ -168,7 +169,7 @@ func (e *StorageEngine) reportShardError( default: // For background workers we can have a lot of such errors, // thus logging is done with DEBUG level. - e.log.Debug(context.Background(), logs.EngineModeChangeIsInProgressIgnoringSetmodeRequest, + e.log.Debug(ctx, logs.EngineModeChangeIsInProgressIgnoringSetmodeRequest, zap.Stringer("shard_id", sid), zap.Uint32("error_count", errCount)) } diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index d98101306..9d2b1c1b7 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -37,7 +37,7 @@ func (e *StorageEngine) exists(ctx context.Context, shPrm shard.ExistsPrm) (bool } if !client.IsErrObjectNotFound(err) { - e.reportShardError(sh, "could not check existence of object in shard", err, zap.Stringer("address", shPrm.Address)) + e.reportShardError(ctx, sh, "could not check existence of object in shard", err, zap.Stringer("address", shPrm.Address)) } return false } diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index d6827e6c3..c7145889b 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -183,7 +183,7 @@ func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { i.ObjectExpired = true return true default: - i.Engine.reportShardError(sh, "could not get object from shard", err, zap.Stringer("address", i.Address)) + i.Engine.reportShardError(ctx, sh, "could not get object from shard", err, zap.Stringer("address", i.Address)) return false } }) diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index d2e3cfd99..d6892f129 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -117,7 +117,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) outError = new(apistatus.ObjectNotFound) return true default: - e.reportShardError(sh, "could not head object from shard", err, zap.Stringer("address", prm.addr)) + e.reportShardError(ctx, sh, "could not head object from shard", err, zap.Stringer("address", prm.addr)) return false } } diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 1dc64c174..e89a8d048 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -151,7 +151,7 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh var siErr *objectSDK.SplitInfoError var ecErr *objectSDK.ECInfoError if !(errors.As(err, &siErr) || errors.As(err, &ecErr)) { - e.reportShardError(sh, "could not check for presents in shard", err, zap.Stringer("address", addr)) + e.reportShardError(ctx, sh, "could not check for presents in shard", err, zap.Stringer("address", addr)) return } @@ -176,7 +176,7 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh return true } - e.reportShardError(sh, "could not inhume object in shard", err, zap.Stringer("address", addr)) + e.reportShardError(ctx, sh, "could not inhume object in shard", err, zap.Stringer("address", addr)) return false } @@ -202,7 +202,7 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { locked, err = h.Shard.IsLocked(ctx, addr) if err != nil { - e.reportShardError(h, "can't check object's lockers", err, zap.Stringer("address", addr), + e.reportShardError(ctx, h, "can't check object's lockers", err, zap.Stringer("address", addr), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) outErr = err return false @@ -232,7 +232,7 @@ func (e *StorageEngine) GetLocked(ctx context.Context, addr oid.Address) ([]oid. e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { ld, err := h.Shard.GetLocked(ctx, addr) if err != nil { - e.reportShardError(h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr), + e.reportShardError(ctx, h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) outErr = err } @@ -274,7 +274,7 @@ func (e *StorageEngine) processExpiredLocks(ctx context.Context, epoch uint64, l func (e *StorageEngine) processDeletedLocks(ctx context.Context, lockers []oid.Address) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - sh.HandleDeletedLocks(lockers) + sh.HandleDeletedLocks(ctx, lockers) select { case <-ctx.Done(): diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index bbab59bfa..5d43e59df 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -89,7 +89,7 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo var objID oid.ID err = objID.ReadFromV2(chunk.ID) if err != nil { - e.reportShardError(sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), + e.reportShardError(ctx, sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) return false } @@ -97,7 +97,7 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo } err = sh.Lock(ctx, idCnr, locker, eclocked) if err != nil { - e.reportShardError(sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), + e.reportShardError(ctx, sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) return false } @@ -109,7 +109,7 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo // do not lock it return true } - e.reportShardError(sh, "could not check locked object for presence in shard", err, zap.Stringer("container_id", idCnr), + e.reportShardError(ctx, sh, "could not check locked object for presence in shard", err, zap.Stringer("container_id", idCnr), zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) return } @@ -122,7 +122,7 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo err := sh.Lock(ctx, idCnr, locker, []oid.ID{locked}) if err != nil { - e.reportShardError(sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), + e.reportShardError(ctx, sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) var errIrregular *apistatus.LockNonRegularObject diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 635f0e302..e080191ae 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -179,7 +179,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti return } - e.reportShardError(sh, "could not put object to shard", err, zap.Stringer("address", addr)) + e.reportShardError(ctx, sh, "could not put object to shard", err, zap.Stringer("address", addr)) return } diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index c5c94eef7..0c9cea903 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -205,7 +205,7 @@ func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) { return true // stop, return it back default: - i.Engine.reportShardError(sh, "could not get object from shard", err, zap.Stringer("address", i.Address)) + i.Engine.reportShardError(ctx, sh, "could not get object from shard", err, zap.Stringer("address", i.Address)) return false } }) diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 0f1341f85..02149b4c8 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -74,7 +74,7 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { res, err := sh.Select(ctx, shPrm) if err != nil { - e.reportShardError(sh, "could not select objects from shard", err) + e.reportShardError(ctx, sh, "could not select objects from shard", err) return false } @@ -116,7 +116,7 @@ func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, erro e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { res, err := sh.List(ctx) // consider limit result of shard iterator if err != nil { - e.reportShardError(sh, "could not select objects from shard", err) + e.reportShardError(ctx, sh, "could not select objects from shard", err) } else { for _, addr := range res.AddressList() { // save only unique values if _, ok := uniqueMap[addr.EncodeToString()]; !ok { diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 2b94103e9..898f685ec 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -351,7 +351,7 @@ func (e *StorageEngine) DetachShards(ctx context.Context, ids []*shard.ID) error return logicerr.New("ids must be non-empty") } - deletedShards, err := e.deleteShards(ids) + deletedShards, err := e.deleteShards(ctx, ids) if err != nil { return err } @@ -400,7 +400,7 @@ func (e *StorageEngine) closeShards(ctx context.Context, deletedShards []hashedS // deleteShards deletes shards with specified ids from engine shard list // and releases all engine resources associated with shards. // Returns deleted shards or error if some shard could not be deleted. -func (e *StorageEngine) deleteShards(ids []*shard.ID) ([]hashedShard, error) { +func (e *StorageEngine) deleteShards(ctx context.Context, ids []*shard.ID) ([]hashedShard, error) { ss := make([]hashedShard, 0, len(ids)) e.mtx.Lock() @@ -432,7 +432,7 @@ func (e *StorageEngine) deleteShards(ids []*shard.ID) ([]hashedShard, error) { delete(e.shardPools, idStr) } - e.log.Info(context.Background(), logs.EngineShardHasBeenRemoved, + e.log.Info(ctx, logs.EngineShardHasBeenRemoved, zap.String("id", idStr)) } diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 6bb5e3a41..268b4adfa 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -37,7 +37,7 @@ func (e *StorageEngine) TreeMove(ctx context.Context, d pilorama.CIDDescriptor, lm, err := lst[index].TreeMove(ctx, d, treeID, m) if err != nil { if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { - e.reportShardError(lst[index], "can't perform `TreeMove`", err, + e.reportShardError(ctx, lst[index], "can't perform `TreeMove`", err, zap.Stringer("cid", d.CID), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -71,7 +71,7 @@ func (e *StorageEngine) TreeAddByPath(ctx context.Context, d pilorama.CIDDescrip lm, err := lst[index].TreeAddByPath(ctx, d, treeID, attr, path, m) if err != nil { if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { - e.reportShardError(lst[index], "can't perform `TreeAddByPath`", err, + e.reportShardError(ctx, lst[index], "can't perform `TreeAddByPath`", err, zap.Stringer("cid", d.CID), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -100,7 +100,7 @@ func (e *StorageEngine) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID str err = lst[index].TreeApply(ctx, cnr, treeID, m, backgroundSync) if err != nil { if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { - e.reportShardError(lst[index], "can't perform `TreeApply`", err, + e.reportShardError(ctx, lst[index], "can't perform `TreeApply`", err, zap.Stringer("cid", cnr), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -128,7 +128,7 @@ func (e *StorageEngine) TreeApplyBatch(ctx context.Context, cnr cidSDK.ID, treeI err = lst[index].TreeApplyBatch(ctx, cnr, treeID, m) if err != nil { if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { - e.reportShardError(lst[index], "can't perform `TreeApplyBatch`", err, + e.reportShardError(ctx, lst[index], "can't perform `TreeApplyBatch`", err, zap.Stringer("cid", cnr), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -160,7 +160,7 @@ func (e *StorageEngine) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID break } if !errors.Is(err, pilorama.ErrTreeNotFound) { - e.reportShardError(sh, "can't perform `TreeGetByPath`", err, + e.reportShardError(ctx, sh, "can't perform `TreeGetByPath`", err, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -193,7 +193,7 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s break } if !errors.Is(err, pilorama.ErrTreeNotFound) { - e.reportShardError(sh, "can't perform `TreeGetMeta`", err, + e.reportShardError(ctx, sh, "can't perform `TreeGetMeta`", err, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -225,7 +225,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree break } if !errors.Is(err, pilorama.ErrTreeNotFound) { - e.reportShardError(sh, "can't perform `TreeGetChildren`", err, + e.reportShardError(ctx, sh, "can't perform `TreeGetChildren`", err, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -257,7 +257,7 @@ func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, break } if !errors.Is(err, pilorama.ErrTreeNotFound) { - e.reportShardError(sh, "can't perform `TreeSortedByFilename`", err, + e.reportShardError(ctx, sh, "can't perform `TreeSortedByFilename`", err, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -289,7 +289,7 @@ func (e *StorageEngine) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID break } if !errors.Is(err, pilorama.ErrTreeNotFound) { - e.reportShardError(sh, "can't perform `TreeGetOpLog`", err, + e.reportShardError(ctx, sh, "can't perform `TreeGetOpLog`", err, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -319,7 +319,7 @@ func (e *StorageEngine) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID stri break } if !errors.Is(err, pilorama.ErrTreeNotFound) && !errors.Is(err, shard.ErrReadOnlyMode) { - e.reportShardError(sh, "can't perform `TreeDrop`", err, + e.reportShardError(ctx, sh, "can't perform `TreeDrop`", err, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -349,7 +349,7 @@ func (e *StorageEngine) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, return nil, err } - e.reportShardError(sh, "can't perform `TreeList`", err, + e.reportShardError(ctx, sh, "can't perform `TreeList`", err, zap.Stringer("cid", cid), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -415,7 +415,7 @@ func (e *StorageEngine) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK err = lst[index].TreeUpdateLastSyncHeight(ctx, cid, treeID, height) if err != nil && !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { - e.reportShardError(lst[index], "can't update tree synchronization height", err, + e.reportShardError(ctx, lst[index], "can't update tree synchronization height", err, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) @@ -442,7 +442,7 @@ func (e *StorageEngine) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, t break } if !errors.Is(err, pilorama.ErrTreeNotFound) { - e.reportShardError(sh, "can't read tree synchronization height", err, + e.reportShardError(ctx, sh, "can't read tree synchronization height", err, zap.Stringer("cid", cid), zap.String("tree", treeID), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/local_object_storage/internal/storagetest/storage.go b/pkg/local_object_storage/internal/storagetest/storage.go index d8ac106dd..d46365296 100644 --- a/pkg/local_object_storage/internal/storagetest/storage.go +++ b/pkg/local_object_storage/internal/storagetest/storage.go @@ -12,8 +12,8 @@ import ( type Component interface { Open(context.Context, mode.Mode) error SetMode(context.Context, mode.Mode) error - Init() error - Close() error + Init(context.Context) error + Close(context.Context) error } // Constructor constructs storage component. @@ -59,18 +59,18 @@ func TestCloseAfterOpen(t *testing.T, cons Constructor) { // Use-case: irrecoverable error on some components, close everything. s := cons(t) require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, s.Close()) + require.NoError(t, s.Close(context.Background())) }) t.Run("RO", func(t *testing.T) { // Use-case: irrecoverable error on some components, close everything. // Open in read-only must be done after the db is here. s := cons(t) require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, s.Init()) - require.NoError(t, s.Close()) + require.NoError(t, s.Init(context.Background())) + require.NoError(t, s.Close(context.Background())) require.NoError(t, s.Open(context.Background(), mode.ReadOnly)) - require.NoError(t, s.Close()) + require.NoError(t, s.Close(context.Background())) }) } @@ -79,9 +79,9 @@ func TestCloseTwice(t *testing.T, cons Constructor) { // Use-case: move to maintenance mode twice, first time failed. s := cons(t) require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, s.Init()) - require.NoError(t, s.Close()) - require.NoError(t, s.Close()) // already closed, no-op + require.NoError(t, s.Init(context.Background())) + require.NoError(t, s.Close(context.Background())) + require.NoError(t, s.Close(context.Background())) // already closed, no-op } // TestSetMode checks that any mode transition can be done safely. @@ -94,20 +94,20 @@ func TestSetMode(t *testing.T, cons Constructor, m mode.Mode) { require.NoError(t, s.SetMode(context.Background(), m)) t.Run("after open in RO", func(t *testing.T) { - require.NoError(t, s.Close()) + require.NoError(t, s.Close(context.Background())) require.NoError(t, s.Open(context.Background(), mode.ReadOnly)) require.NoError(t, s.SetMode(context.Background(), m)) }) - require.NoError(t, s.Close()) + require.NoError(t, s.Close(context.Background())) }) t.Run("after init", func(t *testing.T) { s := cons(t) // Use-case: notmal node operation. require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, s.Init()) + require.NoError(t, s.Init(context.Background())) require.NoError(t, s.SetMode(context.Background(), m)) - require.NoError(t, s.Close()) + require.NoError(t, s.Close(context.Background())) }) } @@ -115,8 +115,8 @@ func TestModeTransition(t *testing.T, cons Constructor, from, to mode.Mode) { // Use-case: normal node operation. s := cons(t) require.NoError(t, s.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, s.Init()) + require.NoError(t, s.Init(context.Background())) require.NoError(t, s.SetMode(context.Background(), from)) require.NoError(t, s.SetMode(context.Background(), to)) - require.NoError(t, s.Close()) + require.NoError(t, s.Close(context.Background())) } diff --git a/pkg/local_object_storage/metabase/containers_test.go b/pkg/local_object_storage/metabase/containers_test.go index 110be68ad..8d8d91dc7 100644 --- a/pkg/local_object_storage/metabase/containers_test.go +++ b/pkg/local_object_storage/metabase/containers_test.go @@ -18,7 +18,7 @@ func TestDB_Containers(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() const N = 10 @@ -79,7 +79,7 @@ func TestDB_ContainersCount(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() const R, T, SG, L = 10, 11, 12, 13 // amount of object per type @@ -116,7 +116,7 @@ func TestDB_ContainerSize(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() const ( C = 3 diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 54bea4204..07fa7e9cf 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -57,7 +57,7 @@ func (db *DB) openDB(ctx context.Context, mode mode.Mode) error { return fmt.Errorf("can't create dir %s for metabase: %w", db.info.Path, err) } - db.log.Debug(context.Background(), logs.MetabaseCreatedDirectoryForMetabase, zap.String("path", db.info.Path)) + db.log.Debug(ctx, logs.MetabaseCreatedDirectoryForMetabase, zap.String("path", db.info.Path)) if db.boltOptions == nil { opts := *bbolt.DefaultOptions @@ -78,9 +78,9 @@ func (db *DB) openBolt(ctx context.Context) error { db.boltDB.MaxBatchDelay = db.boltBatchDelay db.boltDB.MaxBatchSize = db.boltBatchSize - db.log.Debug(context.Background(), logs.MetabaseOpenedBoltDBInstanceForMetabase) + db.log.Debug(ctx, logs.MetabaseOpenedBoltDBInstanceForMetabase) - db.log.Debug(context.Background(), logs.MetabaseCheckingMetabaseVersion) + db.log.Debug(ctx, logs.MetabaseCheckingMetabaseVersion) return db.boltDB.View(func(tx *bbolt.Tx) error { // The safest way to check if the metabase is fresh is to check if it has no buckets. // However, shard info can be present. So here we check that the number of buckets is @@ -109,7 +109,7 @@ func (db *DB) openBolt(ctx context.Context) error { // // Does nothing if metabase has already been initialized and filled. To roll back the database to its initial state, // use Reset. -func (db *DB) Init() error { +func (db *DB) Init(_ context.Context) error { return metaerr.Wrap(db.init(false)) } @@ -205,7 +205,7 @@ func (db *DB) SyncCounters() error { // Close closes boltDB instance // and reports metabase metric. -func (db *DB) Close() error { +func (db *DB) Close(context.Context) error { var err error if db.boltDB != nil { err = db.close() @@ -236,7 +236,7 @@ func (db *DB) Reload(ctx context.Context, opts ...Option) (bool, error) { defer db.modeMtx.Unlock() if db.mode.NoMetabase() || c.info.Path != "" && filepath.Clean(db.info.Path) != filepath.Clean(c.info.Path) { - if err := db.Close(); err != nil { + if err := db.Close(ctx); err != nil { return false, err } diff --git a/pkg/local_object_storage/metabase/control_test.go b/pkg/local_object_storage/metabase/control_test.go index 2a64881cb..d26402675 100644 --- a/pkg/local_object_storage/metabase/control_test.go +++ b/pkg/local_object_storage/metabase/control_test.go @@ -15,7 +15,7 @@ import ( func TestReset(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() err := db.Reset() require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index dccccd456..950385a29 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -22,7 +22,7 @@ func TestCounters(t *testing.T) { t.Run("defaults", func(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() c, err := db.ObjectCounters() require.NoError(t, err) require.Zero(t, c.Phy) @@ -37,7 +37,7 @@ func TestCounters(t *testing.T) { t.Run("put", func(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() oo := make([]*objectSDK.Object, 0, objCount) for range objCount { oo = append(oo, testutil.GenerateObject()) @@ -75,7 +75,7 @@ func TestCounters(t *testing.T) { t.Run("delete", func(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() oo := putObjs(t, db, objCount, false) exp := make(map[cid.ID]meta.ObjectCounters) @@ -120,7 +120,7 @@ func TestCounters(t *testing.T) { t.Run("inhume", func(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() oo := putObjs(t, db, objCount, false) exp := make(map[cid.ID]meta.ObjectCounters) @@ -185,7 +185,7 @@ func TestCounters(t *testing.T) { t.Run("put_split", func(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() parObj := testutil.GenerateObject() exp := make(map[cid.ID]meta.ObjectCounters) @@ -223,7 +223,7 @@ func TestCounters(t *testing.T) { t.Run("delete_split", func(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() oo := putObjs(t, db, objCount, true) exp := make(map[cid.ID]meta.ObjectCounters) @@ -265,7 +265,7 @@ func TestCounters(t *testing.T) { t.Run("inhume_split", func(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() oo := putObjs(t, db, objCount, true) exp := make(map[cid.ID]meta.ObjectCounters) @@ -329,7 +329,7 @@ func TestCounters(t *testing.T) { func TestDoublePut(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() obj := testutil.GenerateObject() exp := make(map[cid.ID]meta.ObjectCounters) @@ -387,7 +387,7 @@ func TestCounters_Expired(t *testing.T) { es := &epochState{epoch} db := newDB(t, meta.WithEpochState(es)) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() oo := make([]oid.Address, objCount) for i := range oo { diff --git a/pkg/local_object_storage/metabase/db_test.go b/pkg/local_object_storage/metabase/db_test.go index c61d762bc..edaeb13c5 100644 --- a/pkg/local_object_storage/metabase/db_test.go +++ b/pkg/local_object_storage/metabase/db_test.go @@ -61,7 +61,7 @@ func newDB(t testing.TB, opts ...meta.Option) *meta.DB { ) require.NoError(t, bdb.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, bdb.Init()) + require.NoError(t, bdb.Init(context.Background())) return bdb } diff --git a/pkg/local_object_storage/metabase/delete_ec_test.go b/pkg/local_object_storage/metabase/delete_ec_test.go index a25627990..9f1f91e14 100644 --- a/pkg/local_object_storage/metabase/delete_ec_test.go +++ b/pkg/local_object_storage/metabase/delete_ec_test.go @@ -30,8 +30,8 @@ func TestDeleteECObject_WithoutSplit(t *testing.T) { ) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, db.Init()) - defer func() { require.NoError(t, db.Close()) }() + require.NoError(t, db.Init(context.Background())) + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() ecChunk := oidtest.ID() @@ -194,8 +194,8 @@ func testDeleteECObjectWithSplit(t *testing.T, chunksCount int, withLinking bool ) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, db.Init()) - defer func() { require.NoError(t, db.Close()) }() + require.NoError(t, db.Init(context.Background())) + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() ecChunks := make([]oid.ID, chunksCount) diff --git a/pkg/local_object_storage/metabase/delete_meta_test.go b/pkg/local_object_storage/metabase/delete_meta_test.go index cdfe2a203..0329e3a73 100644 --- a/pkg/local_object_storage/metabase/delete_meta_test.go +++ b/pkg/local_object_storage/metabase/delete_meta_test.go @@ -23,8 +23,8 @@ func TestPutDeleteIndexAttributes(t *testing.T) { }...) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, db.Init()) - defer func() { require.NoError(t, db.Close()) }() + require.NoError(t, db.Init(context.Background())) + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() obj1 := testutil.GenerateObjectWithCID(cnr) diff --git a/pkg/local_object_storage/metabase/delete_test.go b/pkg/local_object_storage/metabase/delete_test.go index fe5f7833b..c0762a377 100644 --- a/pkg/local_object_storage/metabase/delete_test.go +++ b/pkg/local_object_storage/metabase/delete_test.go @@ -18,7 +18,7 @@ import ( func TestDB_Delete(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() parent := testutil.GenerateObjectWithCID(cnr) @@ -65,7 +65,7 @@ func TestDB_Delete(t *testing.T) { func TestDeleteAllChildren(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -103,7 +103,7 @@ func TestDeleteAllChildren(t *testing.T) { func TestGraveOnlyDelete(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() addr := oidtest.Address() @@ -116,7 +116,7 @@ func TestGraveOnlyDelete(t *testing.T) { func TestExpiredObject(t *testing.T) { db := newDB(t, meta.WithEpochState(epochState{currEpoch})) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() checkExpiredObjects(t, db, func(exp, nonExp *objectSDK.Object) { // removing expired object should be error-free @@ -128,7 +128,7 @@ func TestExpiredObject(t *testing.T) { func TestDelete(t *testing.T) { db := newDB(t, meta.WithEpochState(epochState{currEpoch})) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() for range 10 { @@ -170,7 +170,7 @@ func TestDelete(t *testing.T) { func TestDeleteDropsGCMarkIfObjectNotFound(t *testing.T) { db := newDB(t, meta.WithEpochState(epochState{currEpoch})) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() addr := oidtest.Address() diff --git a/pkg/local_object_storage/metabase/exists_test.go b/pkg/local_object_storage/metabase/exists_test.go index 1e4148eba..3045e17f1 100644 --- a/pkg/local_object_storage/metabase/exists_test.go +++ b/pkg/local_object_storage/metabase/exists_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "errors" "testing" @@ -18,7 +19,7 @@ const currEpoch = 1000 func TestDB_Exists(t *testing.T) { db := newDB(t, meta.WithEpochState(epochState{currEpoch})) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() t.Run("no object", func(t *testing.T) { nonExist := testutil.GenerateObject() diff --git a/pkg/local_object_storage/metabase/expired_test.go b/pkg/local_object_storage/metabase/expired_test.go index bb98745ee..495c1eee7 100644 --- a/pkg/local_object_storage/metabase/expired_test.go +++ b/pkg/local_object_storage/metabase/expired_test.go @@ -13,7 +13,7 @@ import ( func TestDB_SelectExpired(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() containerID1 := cidtest.ID() diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index f0caaea70..c93d2c992 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -25,7 +25,7 @@ import ( func TestDB_Get(t *testing.T) { db := newDB(t, meta.WithEpochState(epochState{currEpoch})) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() raw := testutil.GenerateObject() @@ -219,7 +219,7 @@ func benchmarkGet(b *testing.B, numOfObj int) { meta.WithMaxBatchSize(batchSize), meta.WithMaxBatchDelay(10*time.Millisecond), ) - defer func() { require.NoError(b, db.Close()) }() + defer func() { require.NoError(b, db.Close(context.Background())) }() addrs := make([]oid.Address, 0, numOfObj) for range numOfObj { @@ -253,7 +253,7 @@ func benchmarkGet(b *testing.B, numOfObj int) { }) }) - require.NoError(b, db.Close()) + require.NoError(b, db.Close(context.Background())) require.NoError(b, os.RemoveAll(b.Name())) db, addrs = prepareDb(1) diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index b9c6ce28c..99794e609 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -15,7 +15,7 @@ import ( func TestDB_IterateDeletedObjects_EmptyDB(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() var counter int var iterGravePRM meta.GraveyardIterationPrm @@ -42,7 +42,7 @@ func TestDB_IterateDeletedObjects_EmptyDB(t *testing.T) { func TestDB_Iterate_OffsetNotFound(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() obj1 := testutil.GenerateObject() obj2 := testutil.GenerateObject() @@ -113,7 +113,7 @@ func TestDB_Iterate_OffsetNotFound(t *testing.T) { func TestDB_IterateDeletedObjects(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() // generate and put 4 objects @@ -202,7 +202,7 @@ func TestDB_IterateDeletedObjects(t *testing.T) { func TestDB_IterateOverGraveyard_Offset(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() // generate and put 4 objects @@ -303,7 +303,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { func TestDB_IterateOverGarbage_Offset(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() // generate and put 4 objects obj1 := testutil.GenerateObject() @@ -395,7 +395,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { func TestDB_DropGraves(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() // generate and put 2 objects diff --git a/pkg/local_object_storage/metabase/inhume_ec_test.go b/pkg/local_object_storage/metabase/inhume_ec_test.go index 32e412c79..180713287 100644 --- a/pkg/local_object_storage/metabase/inhume_ec_test.go +++ b/pkg/local_object_storage/metabase/inhume_ec_test.go @@ -25,8 +25,8 @@ func TestInhumeECObject(t *testing.T) { ) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, db.Init()) - defer func() { require.NoError(t, db.Close()) }() + require.NoError(t, db.Init(context.Background())) + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() ecChunk := oidtest.ID() diff --git a/pkg/local_object_storage/metabase/inhume_test.go b/pkg/local_object_storage/metabase/inhume_test.go index 277316f7b..786d10396 100644 --- a/pkg/local_object_storage/metabase/inhume_test.go +++ b/pkg/local_object_storage/metabase/inhume_test.go @@ -17,7 +17,7 @@ import ( func TestDB_Inhume(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() raw := testutil.GenerateObject() testutil.AddAttribute(raw, "foo", "bar") @@ -37,7 +37,7 @@ func TestDB_Inhume(t *testing.T) { func TestInhumeTombOnTomb(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() var ( err error @@ -107,7 +107,7 @@ func TestInhumeTombOnTomb(t *testing.T) { func TestInhumeLocked(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() locked := oidtest.Address() diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 646dc196c..7eed32c55 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -18,7 +18,7 @@ import ( func TestDB_IterateExpired(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() const epoch = 13 @@ -70,7 +70,7 @@ func putWithExpiration(t *testing.T, db *meta.DB, typ objectSDK.Type, expiresAt func TestDB_IterateCoveredByTombstones(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() ts := oidtest.Address() diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 203802ec0..6f6463071 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -33,7 +33,7 @@ func listWithCursorPrepareDB(b *testing.B) *meta.DB { db := newDB(b, meta.WithMaxBatchSize(1), meta.WithBoltDBOptions(&bbolt.Options{ NoSync: true, })) // faster single-thread generation - defer func() { require.NoError(b, db.Close()) }() + defer func() { require.NoError(b, db.Close(context.Background())) }() obj := testutil.GenerateObject() for i := range 100_000 { // should be a multiple of all batch sizes @@ -71,7 +71,7 @@ func TestLisObjectsWithCursor(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() const ( containers = 5 @@ -163,7 +163,7 @@ func TestAddObjectDuringListingWithCursor(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() const total = 5 @@ -225,7 +225,7 @@ func TestIterateOver(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() const total uint64 = 5 for _, typ := range []objectSDK.Type{objectSDK.TypeRegular, objectSDK.TypeTombstone, objectSDK.TypeLock} { diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 9601cb2be..341ff9ad1 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -21,7 +21,7 @@ func TestDB_Lock(t *testing.T) { cnr := cidtest.ID() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() t.Run("empty locked list", func(t *testing.T) { require.Panics(t, func() { _ = db.Lock(context.Background(), cnr, oid.ID{}, nil) }) @@ -187,7 +187,7 @@ func TestDB_Lock_Expired(t *testing.T) { es := &epochState{e: 123} db := newDB(t, meta.WithEpochState(es)) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() // put an object addr := putWithExpiration(t, db, objectSDK.TypeRegular, 124) @@ -209,7 +209,7 @@ func TestDB_IsLocked(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() // existing and locked objs diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index f99262be4..ce6ae1004 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -18,7 +18,7 @@ func (db *DB) SetMode(ctx context.Context, m mode.Mode) error { } if !db.mode.NoMetabase() { - if err := db.Close(); err != nil { + if err := db.Close(ctx); err != nil { return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err) } } @@ -28,7 +28,7 @@ func (db *DB) SetMode(ctx context.Context, m mode.Mode) error { } else { err := db.openDB(ctx, m) if err == nil && !m.ReadOnly() { - err = db.Init() + err = db.Init(ctx) } if err != nil { return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err) diff --git a/pkg/local_object_storage/metabase/mode_test.go b/pkg/local_object_storage/metabase/mode_test.go index 1b9f60055..28b42283f 100644 --- a/pkg/local_object_storage/metabase/mode_test.go +++ b/pkg/local_object_storage/metabase/mode_test.go @@ -25,13 +25,13 @@ func Test_Mode(t *testing.T) { require.NoError(t, bdb.Open(context.Background(), mode.DegradedReadOnly)) require.Nil(t, bdb.boltDB) - require.NoError(t, bdb.Init()) + require.NoError(t, bdb.Init(context.Background())) require.Nil(t, bdb.boltDB) - require.NoError(t, bdb.Close()) + require.NoError(t, bdb.Close(context.Background())) require.NoError(t, bdb.Open(context.Background(), mode.Degraded)) require.Nil(t, bdb.boltDB) - require.NoError(t, bdb.Init()) + require.NoError(t, bdb.Init(context.Background())) require.Nil(t, bdb.boltDB) - require.NoError(t, bdb.Close()) + require.NoError(t, bdb.Close(context.Background())) } diff --git a/pkg/local_object_storage/metabase/put_test.go b/pkg/local_object_storage/metabase/put_test.go index 914f5ef06..f37ed4cf2 100644 --- a/pkg/local_object_storage/metabase/put_test.go +++ b/pkg/local_object_storage/metabase/put_test.go @@ -46,7 +46,7 @@ func BenchmarkPut(b *testing.B) { db := newDB(b, meta.WithMaxBatchDelay(time.Millisecond*10), meta.WithMaxBatchSize(runtime.NumCPU())) - defer func() { require.NoError(b, db.Close()) }() + defer func() { require.NoError(b, db.Close(context.Background())) }() // Ensure the benchmark is bound by CPU and not waiting batch-delay time. b.SetParallelism(1) @@ -68,7 +68,7 @@ func BenchmarkPut(b *testing.B) { db := newDB(b, meta.WithMaxBatchDelay(time.Millisecond*10), meta.WithMaxBatchSize(1)) - defer func() { require.NoError(b, db.Close()) }() + defer func() { require.NoError(b, db.Close(context.Background())) }() var index atomic.Int64 index.Store(-1) objs := prepareObjects(b.N) @@ -84,7 +84,7 @@ func BenchmarkPut(b *testing.B) { func TestDB_PutBlobovniczaUpdate(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() raw1 := testutil.GenerateObject() storageID := []byte{1, 2, 3, 4} diff --git a/pkg/local_object_storage/metabase/reset_test.go b/pkg/local_object_storage/metabase/reset_test.go index 993079dce..45faecc13 100644 --- a/pkg/local_object_storage/metabase/reset_test.go +++ b/pkg/local_object_storage/metabase/reset_test.go @@ -30,9 +30,9 @@ func TestResetDropsContainerBuckets(t *testing.T) { ) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, db.Init()) + require.NoError(t, db.Init(context.Background())) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() for idx := range 100 { var putPrm PutPrm diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 6f48607be..5cc998311 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -38,7 +38,7 @@ func testSelectUserAttributes(t *testing.T, index bool) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -200,7 +200,7 @@ func TestDB_SelectRootPhyParent(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -354,7 +354,7 @@ func TestDB_SelectInhume(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -385,7 +385,7 @@ func TestDB_SelectPayloadHash(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -456,7 +456,7 @@ func TestDB_SelectWithSlowFilters(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -564,7 +564,7 @@ func TestDB_SelectObjectID(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -680,7 +680,7 @@ func TestDB_SelectOwnerID(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -786,7 +786,7 @@ func TestDB_SelectECWithFastAndSlowFilters(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() ecChunk1 := oidtest.ID() @@ -865,7 +865,7 @@ func TestDB_RawHead_SplitInfo(t *testing.T) { ) db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -906,7 +906,7 @@ func testGetRawSplitInfo(t *testing.T, cnr cidSDK.ID, ids *transformer.AccessIde t.Run("first last, then linking", func(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() require.NoError(t, metaPut(db, lastPart, nil)) require.NoError(t, metaPut(db, linking, nil)) @@ -930,7 +930,7 @@ func testGetRawSplitInfo(t *testing.T, cnr cidSDK.ID, ids *transformer.AccessIde }) t.Run("first linking, then last", func(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() require.NoError(t, metaPut(db, linking, nil)) require.NoError(t, metaPut(db, lastPart, nil)) @@ -954,7 +954,7 @@ func testGetRawSplitInfo(t *testing.T, cnr cidSDK.ID, ids *transformer.AccessIde }) t.Run("only last part", func(t *testing.T) { db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() require.NoError(t, metaPut(db, lastPart, nil)) @@ -984,7 +984,7 @@ func TestDB_SelectSplitID_EC(t *testing.T) { ) db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -1052,7 +1052,7 @@ func TestDB_SelectSplitID(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -1109,7 +1109,7 @@ func TestDB_SelectContainerID(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() cnr := cidtest.ID() @@ -1157,7 +1157,7 @@ func TestDB_SelectContainerID(t *testing.T) { func BenchmarkSelect(b *testing.B) { const objCount = 1000 db := newDB(b) - defer func() { require.NoError(b, db.Close()) }() + defer func() { require.NoError(b, db.Close(context.Background())) }() cid := cidtest.ID() @@ -1199,7 +1199,7 @@ func TestExpiredObjects(t *testing.T) { t.Parallel() db := newDB(t, meta.WithEpochState(epochState{currEpoch})) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() checkExpiredObjects(t, db, func(exp, nonExp *objectSDK.Object) { cidExp, _ := exp.ContainerID() diff --git a/pkg/local_object_storage/metabase/storage_id_test.go b/pkg/local_object_storage/metabase/storage_id_test.go index a86e42bd2..fef680159 100644 --- a/pkg/local_object_storage/metabase/storage_id_test.go +++ b/pkg/local_object_storage/metabase/storage_id_test.go @@ -15,7 +15,7 @@ func TestDB_StorageID(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() raw1 := testutil.GenerateObject() raw2 := testutil.GenerateObject() @@ -79,7 +79,7 @@ func TestPutWritecacheDataRace(t *testing.T) { t.Parallel() db := newDB(t) - defer func() { require.NoError(t, db.Close()) }() + defer func() { require.NoError(t, db.Close(context.Background())) }() putStorageID := []byte{1, 2, 3} wcStorageID := []byte{1, 2, 3, 4, 5} diff --git a/pkg/local_object_storage/metabase/upgrade_test.go b/pkg/local_object_storage/metabase/upgrade_test.go index e2eee86b0..5444264be 100644 --- a/pkg/local_object_storage/metabase/upgrade_test.go +++ b/pkg/local_object_storage/metabase/upgrade_test.go @@ -34,12 +34,12 @@ func TestUpgradeV2ToV3(t *testing.T) { }() db := New(WithPath(path), WithEpochState(epochState{e: 1000}), WithLogger(test.NewLogger(t))) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.ErrorIs(t, db.Init(), ErrOutdatedVersion) - require.NoError(t, db.Close()) + require.ErrorIs(t, db.Init(context.Background()), ErrOutdatedVersion) + require.NoError(t, db.Close(context.Background())) require.NoError(t, Upgrade(context.Background(), path, true, &testContainerInfoProvider{}, t.Log)) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, db.Init()) - require.NoError(t, db.Close()) + require.NoError(t, db.Init(context.Background())) + require.NoError(t, db.Close(context.Background())) fmt.Println() } @@ -87,7 +87,7 @@ func TestGenerateMetabaseFile(t *testing.T) { require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) db.boltDB.AllocSize = allocSize db.boltDB.NoSync = true - require.NoError(t, db.Init()) + require.NoError(t, db.Init(context.Background())) containers := make([]cid.ID, containersCount) for i := range containers { containers[i] = cidtest.ID() @@ -218,5 +218,5 @@ func TestGenerateMetabaseFile(t *testing.T) { require.NoError(t, eg.Wait()) db.log.Info(ctx, "simple objects locked by locks generated") require.NoError(t, db.boltDB.Sync()) - require.NoError(t, db.Close()) + require.NoError(t, db.Close(context.Background())) } diff --git a/pkg/local_object_storage/metabase/version_test.go b/pkg/local_object_storage/metabase/version_test.go index 509e72479..b373fb32e 100644 --- a/pkg/local_object_storage/metabase/version_test.go +++ b/pkg/local_object_storage/metabase/version_test.go @@ -45,15 +45,15 @@ func TestVersion(t *testing.T) { t.Run("simple", func(t *testing.T) { db := newDB(t) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, db.Init()) + require.NoError(t, db.Init(context.Background())) check(t, db) - require.NoError(t, db.Close()) + require.NoError(t, db.Close(context.Background())) t.Run("reopen", func(t *testing.T) { require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, db.Init()) + require.NoError(t, db.Init(context.Background())) check(t, db) - require.NoError(t, db.Close()) + require.NoError(t, db.Close(context.Background())) }) }) t.Run("old data", func(t *testing.T) { @@ -61,9 +61,9 @@ func TestVersion(t *testing.T) { require.NoError(t, db.SetShardID(context.Background(), []byte{1, 2, 3, 4}, mode.ReadWrite)) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, db.Init()) + require.NoError(t, db.Init(context.Background())) check(t, db) - require.NoError(t, db.Close()) + require.NoError(t, db.Close(context.Background())) }) t.Run("invalid version", func(t *testing.T) { db := newDB(t) @@ -71,37 +71,37 @@ func TestVersion(t *testing.T) { require.NoError(t, db.boltDB.Update(func(tx *bbolt.Tx) error { return updateVersion(tx, version+1) })) - require.NoError(t, db.Close()) + require.NoError(t, db.Close(context.Background())) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.Error(t, db.Init()) - require.NoError(t, db.Close()) + require.Error(t, db.Init(context.Background())) + require.NoError(t, db.Close(context.Background())) t.Run("reset", func(t *testing.T) { require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.Reset()) check(t, db) - require.NoError(t, db.Close()) + require.NoError(t, db.Close(context.Background())) }) }) t.Run("incompleted upgrade", func(t *testing.T) { db := newDB(t) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, db.Init()) - require.NoError(t, db.Close()) + require.NoError(t, db.Init(context.Background())) + require.NoError(t, db.Close(context.Background())) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.boltDB.Update(func(tx *bbolt.Tx) error { return tx.Bucket(shardInfoBucket).Put(upgradeKey, zeroValue) })) - require.ErrorIs(t, db.Init(), ErrIncompletedUpgrade) - require.NoError(t, db.Close()) + require.ErrorIs(t, db.Init(context.Background()), ErrIncompletedUpgrade) + require.NoError(t, db.Close(context.Background())) require.NoError(t, db.Open(context.Background(), mode.ReadWrite)) require.NoError(t, db.boltDB.Update(func(tx *bbolt.Tx) error { return tx.Bucket(shardInfoBucket).Delete(upgradeKey) })) - require.NoError(t, db.Init()) - require.NoError(t, db.Close()) + require.NoError(t, db.Init(context.Background())) + require.NoError(t, db.Close(context.Background())) }) } diff --git a/pkg/local_object_storage/pilorama/bench_test.go b/pkg/local_object_storage/pilorama/bench_test.go index 22b951a41..3156751f2 100644 --- a/pkg/local_object_storage/pilorama/bench_test.go +++ b/pkg/local_object_storage/pilorama/bench_test.go @@ -28,8 +28,8 @@ func BenchmarkCreate(b *testing.B) { WithPath(filepath.Join(tmpDir, "test.db")), WithMaxBatchSize(runtime.GOMAXPROCS(0))) require.NoError(b, f.Open(context.Background(), mode.ReadWrite)) - require.NoError(b, f.Init()) - defer func() { require.NoError(b, f.Close()) }() + require.NoError(b, f.Init(context.Background())) + defer func() { require.NoError(b, f.Close(context.Background())) }() b.Cleanup(func() { require.NoError(b, os.RemoveAll(tmpDir)) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 9ffcf1e83..c62d728b1 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -91,7 +91,7 @@ func NewBoltForest(opts ...Option) ForestStorage { return &b } -func (t *boltForest) SetMode(_ context.Context, m mode.Mode) error { +func (t *boltForest) SetMode(ctx context.Context, m mode.Mode) error { t.modeMtx.Lock() defer t.modeMtx.Unlock() @@ -99,10 +99,10 @@ func (t *boltForest) SetMode(_ context.Context, m mode.Mode) error { return nil } - err := t.Close() + err := t.Close(ctx) if err == nil && !m.NoMetabase() { if err = t.openBolt(m); err == nil { - err = t.Init() + err = t.Init(ctx) } } if err != nil { @@ -148,7 +148,7 @@ func (t *boltForest) openBolt(m mode.Mode) error { return nil } -func (t *boltForest) Init() error { +func (t *boltForest) Init(context.Context) error { if t.mode.NoMetabase() || t.db.IsReadOnly() { return nil } @@ -162,7 +162,7 @@ func (t *boltForest) Init() error { }) } -func (t *boltForest) Close() error { +func (t *boltForest) Close(context.Context) error { var err error if t.db != nil { err = t.db.Close() diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 76da1c0c2..f31504e2b 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -120,7 +120,7 @@ func (f *memoryForest) TreeApplyBatch(ctx context.Context, cnr cid.ID, treeID st return nil } -func (f *memoryForest) Init() error { +func (f *memoryForest) Init(context.Context) error { return nil } @@ -132,7 +132,7 @@ func (f *memoryForest) SetMode(context.Context, mode.Mode) error { return nil } -func (f *memoryForest) Close() error { +func (f *memoryForest) Close(context.Context) error { return nil } func (f *memoryForest) SetParentID(string) {} diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index fbcc53fb3..de56fc82b 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -30,7 +30,7 @@ var providers = []struct { {"inmemory", func(t testing.TB, _ ...Option) ForestStorage { f := NewMemoryForest() require.NoError(t, f.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, f.Init()) + require.NoError(t, f.Init(context.Background())) return f }}, {"bbolt", func(t testing.TB, opts ...Option) ForestStorage { @@ -40,7 +40,7 @@ var providers = []struct { WithMaxBatchSize(1), }, opts...)...) require.NoError(t, f.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, f.Init()) + require.NoError(t, f.Init(context.Background())) return f }}, } @@ -61,7 +61,7 @@ func TestForest_TreeMove(t *testing.T) { } func testForestTreeMove(t *testing.T, s ForestStorage) { - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() cid := cidtest.ID() d := CIDDescriptor{cid, 0, 1} @@ -125,7 +125,7 @@ func TestMemoryForest_TreeGetChildren(t *testing.T) { } func testForestTreeGetChildren(t *testing.T, s ForestStorage) { - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() cid := cidtest.ID() d := CIDDescriptor{cid, 0, 1} @@ -247,7 +247,7 @@ func TestForest_TreeSortedIterationBugWithSkip(t *testing.T) { } func testForestTreeSortedIterationBugWithSkip(t *testing.T, s ForestStorage) { - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() cid := cidtest.ID() d := CIDDescriptor{cid, 0, 1} @@ -302,7 +302,7 @@ func TestForest_TreeSortedIteration(t *testing.T) { } func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() cid := cidtest.ID() d := CIDDescriptor{cid, 0, 1} @@ -361,7 +361,7 @@ func TestForest_TreeSortedFilename(t *testing.T) { } func testForestTreeSortedByFilename(t *testing.T, s ForestStorage) { - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() const controlAttr = "control_attr" cid := cidtest.ID() @@ -453,7 +453,7 @@ func TestForest_TreeDrop(t *testing.T) { } func testForestTreeDrop(t *testing.T, s ForestStorage) { - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() const cidsSize = 3 var cids [cidsSize]cidSDK.ID @@ -523,7 +523,7 @@ func TestForest_TreeAdd(t *testing.T) { } func testForestTreeAdd(t *testing.T, s ForestStorage) { - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() cid := cidtest.ID() d := CIDDescriptor{cid, 0, 1} @@ -571,7 +571,7 @@ func TestForest_TreeAddByPath(t *testing.T) { } func testForestTreeAddByPath(t *testing.T, s ForestStorage) { - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() cid := cidtest.ID() d := CIDDescriptor{cid, 0, 1} @@ -709,7 +709,7 @@ func testForestTreeApply(t *testing.T, constructor func(t testing.TB, _ ...Optio t.Run("add a child, then insert a parent removal", func(t *testing.T) { s := constructor(t) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() testApply(t, s, 10, 0, Meta{Time: 1, Items: []KeyValue{{"grand", []byte{1}}}}) @@ -722,7 +722,7 @@ func testForestTreeApply(t *testing.T, constructor func(t testing.TB, _ ...Optio }) t.Run("add a child to non-existent parent, then add a parent", func(t *testing.T) { s := constructor(t) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() meta := Meta{Time: 1, Items: []KeyValue{{"child", []byte{3}}}} testApply(t, s, 11, 10, meta) @@ -792,7 +792,7 @@ func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ t.Run("expected", func(t *testing.T) { s := constructor(t) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() for i := range logs { require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[i], false)) @@ -801,7 +801,7 @@ func testForestApplySameOperation(t *testing.T, constructor func(t testing.TB, _ }) s := constructor(t, WithMaxBatchSize(batchSize)) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() require.NoError(t, s.TreeApply(ctx, cid, treeID, &logs[0], false)) for range batchSize { @@ -842,7 +842,7 @@ func testForestTreeGetOpLog(t *testing.T, constructor func(t testing.TB, _ ...Op } s := constructor(t) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() t.Run("empty log, no panic", func(t *testing.T) { _, err := s.TreeGetOpLog(context.Background(), cid, treeID, 0) @@ -883,7 +883,7 @@ func TestForest_TreeExists(t *testing.T) { func testForestTreeExists(t *testing.T, constructor func(t testing.TB, opts ...Option) ForestStorage) { s := constructor(t) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() checkExists := func(t *testing.T, expected bool, cid cidSDK.ID, treeID string) { actual, err := s.TreeExists(context.Background(), cid, treeID) @@ -942,7 +942,7 @@ func TestApplyTricky1(t *testing.T) { for i := range providers { t.Run(providers[i].name, func(t *testing.T) { s := providers[i].construct(t) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() for i := range ops { require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &ops[i], false)) @@ -1005,7 +1005,7 @@ func TestApplyTricky2(t *testing.T) { for i := range providers { t.Run(providers[i].name, func(t *testing.T) { s := providers[i].construct(t) - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() for i := range ops { require.NoError(t, s.TreeApply(context.Background(), cid, treeID, &ops[i], false)) @@ -1115,7 +1115,7 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ treeID := "version" expected := constructor(t, WithNoSync(true)) - defer func() { require.NoError(t, expected.Close()) }() + defer func() { require.NoError(t, expected.Close(context.Background())) }() for i := range ops { require.NoError(t, expected.TreeApply(context.Background(), cid, treeID, &ops[i], false)) @@ -1145,7 +1145,7 @@ func testForestTreeParallelApply(t *testing.T, constructor func(t testing.TB, _ wg.Wait() compareForests(t, expected, actual, cid, treeID, nodeCount) - require.NoError(t, actual.Close()) + require.NoError(t, actual.Close(context.Background())) } } @@ -1163,7 +1163,7 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ .. treeID := "version" expected := constructor(t, WithNoSync(true)) - defer func() { require.NoError(t, expected.Close()) }() + defer func() { require.NoError(t, expected.Close(context.Background())) }() for i := range ops { require.NoError(t, expected.TreeApply(context.Background(), cid, treeID, &ops[i], false)) @@ -1179,7 +1179,7 @@ func testForestTreeApplyRandom(t *testing.T, constructor func(t testing.TB, _ .. require.NoError(t, actual.TreeApply(context.Background(), cid, treeID, &ops[i], false)) } compareForests(t, expected, actual, cid, treeID, nodeCount) - require.NoError(t, actual.Close()) + require.NoError(t, actual.Close(context.Background())) } } @@ -1197,7 +1197,7 @@ func BenchmarkApplySequential(b *testing.B) { b.Run("batchsize="+strconv.Itoa(bs), func(b *testing.B) { r := mrand.New(mrand.NewSource(time.Now().Unix())) s := providers[i].construct(b, WithMaxBatchSize(bs)) - defer func() { require.NoError(b, s.Close()) }() + defer func() { require.NoError(b, s.Close(context.Background())) }() benchmarkApply(b, s, func(opCount int) []Move { ops := make([]Move, opCount) @@ -1233,7 +1233,7 @@ func BenchmarkApplyReorderLast(b *testing.B) { b.Run("batchsize="+strconv.Itoa(bs), func(b *testing.B) { r := mrand.New(mrand.NewSource(time.Now().Unix())) s := providers[i].construct(b, WithMaxBatchSize(bs)) - defer func() { require.NoError(b, s.Close()) }() + defer func() { require.NoError(b, s.Close(context.Background())) }() benchmarkApply(b, s, func(opCount int) []Move { ops := make([]Move, opCount) @@ -1290,7 +1290,7 @@ func TestTreeGetByPath(t *testing.T) { } func testTreeGetByPath(t *testing.T, s ForestStorage) { - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() cid := cidtest.ID() treeID := "version" @@ -1369,7 +1369,7 @@ func TestGetTrees(t *testing.T) { } func testTreeGetTrees(t *testing.T, s ForestStorage) { - defer func() { require.NoError(t, s.Close()) }() + defer func() { require.NoError(t, s.Close(context.Background())) }() cids := []cidSDK.ID{cidtest.ID(), cidtest.ID()} d := CIDDescriptor{Position: 0, Size: 1} @@ -1415,7 +1415,7 @@ func TestTreeLastSyncHeight(t *testing.T) { } func testTreeLastSyncHeight(t *testing.T, f ForestStorage) { - defer func() { require.NoError(t, f.Close()) }() + defer func() { require.NoError(t, f.Close(context.Background())) }() cnr := cidtest.ID() treeID := "someTree" diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 9717b2401..1f7e742a2 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -62,9 +62,9 @@ type Forest interface { type ForestStorage interface { // DumpInfo returns information about the pilorama. DumpInfo() Info - Init() error + Init(context.Context) error Open(context.Context, mode.Mode) error - Close() error + Close(context.Context) error SetMode(context.Context, mode.Mode) error SetParentID(id string) Forest diff --git a/pkg/local_object_storage/pilorama/mode_test.go b/pkg/local_object_storage/pilorama/mode_test.go index 01d3da9f0..0c042aa56 100644 --- a/pkg/local_object_storage/pilorama/mode_test.go +++ b/pkg/local_object_storage/pilorama/mode_test.go @@ -19,13 +19,13 @@ func Test_Mode(t *testing.T) { require.NoError(t, f.Open(context.Background(), mode.DegradedReadOnly)) require.Nil(t, f.(*boltForest).db) - require.NoError(t, f.Init()) + require.NoError(t, f.Init(context.Background())) require.Nil(t, f.(*boltForest).db) - require.NoError(t, f.Close()) + require.NoError(t, f.Close(context.Background())) require.NoError(t, f.Open(context.Background(), mode.Degraded)) require.Nil(t, f.(*boltForest).db) - require.NoError(t, f.Init()) + require.NoError(t, f.Init(context.Background())) require.Nil(t, f.(*boltForest).db) - require.NoError(t, f.Close()) + require.NoError(t, f.Close(context.Background())) } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 056737a9d..5a9e26155 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -91,8 +91,8 @@ func (s *Shard) Open(ctx context.Context) error { type metabaseSynchronizer Shard -func (x *metabaseSynchronizer) Init() error { - ctx, span := tracing.StartSpanFromContext(context.TODO(), "metabaseSynchronizer.Init") +func (x *metabaseSynchronizer) Init(ctx context.Context) error { + ctx, span := tracing.StartSpanFromContext(ctx, "metabaseSynchronizer.Init") defer span.End() return (*Shard)(x).refillMetabase(ctx) @@ -140,7 +140,7 @@ func (s *Shard) Init(ctx context.Context) error { func (s *Shard) initializeComponents(ctx context.Context, m mode.Mode) error { type initializer interface { - Init() error + Init(context.Context) error } var components []initializer @@ -170,7 +170,7 @@ func (s *Shard) initializeComponents(ctx context.Context, m mode.Mode) error { } for _, component := range components { - if err := component.Init(); err != nil { + if err := component.Init(ctx); err != nil { if component == s.metaBase { if errors.Is(err, meta.ErrOutdatedVersion) || errors.Is(err, meta.ErrIncompletedUpgrade) { return fmt.Errorf("metabase initialization: %w", err) @@ -368,7 +368,7 @@ func (s *Shard) Close(ctx context.Context) error { if s.rb != nil { s.rb.Stop(ctx, s.log) } - var components []interface{ Close() error } + var components []interface{ Close(context.Context) error } if s.pilorama != nil { components = append(components, s.pilorama) @@ -384,7 +384,7 @@ func (s *Shard) Close(ctx context.Context) error { var lastErr error for _, component := range components { - if err := component.Close(); err != nil { + if err := component.Close(ctx); err != nil { lastErr = err s.log.Error(ctx, logs.ShardCouldNotCloseShardComponent, zap.Error(err)) } @@ -392,7 +392,7 @@ func (s *Shard) Close(ctx context.Context) error { // If Init/Open was unsuccessful gc can be nil. if s.gc != nil { - s.gc.stop() + s.gc.stop(ctx) } return lastErr @@ -437,7 +437,7 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { // config after the node was updated. err = s.refillMetabase(ctx) } else { - err = s.metaBase.Init() + err = s.metaBase.Init(ctx) } if err != nil { s.log.Error(ctx, logs.ShardCantInitializeMetabaseMoveToADegradedreadonlyMode, zap.Error(err)) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 6fabf7103..a987d3d14 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -213,7 +213,7 @@ func (gc *gc) handleEvent(ctx context.Context, event Event) { } } -func (gc *gc) releaseResources() { +func (gc *gc) releaseResources(ctx context.Context) { if gc.workerPool != nil { gc.workerPool.Release() } @@ -222,7 +222,7 @@ func (gc *gc) releaseResources() { // because it is possible that we are close it earlier than stop writing. // It is ok to keep it opened. - gc.log.Debug(context.Background(), logs.ShardGCIsStopped) + gc.log.Debug(ctx, logs.ShardGCIsStopped) } func (gc *gc) tickRemover(ctx context.Context) { @@ -236,10 +236,10 @@ func (gc *gc) tickRemover(ctx context.Context) { case <-ctx.Done(): // Context canceled earlier than we start to close shards. // It make sense to stop collecting garbage by context too. - gc.releaseResources() + gc.releaseResources(ctx) return case <-gc.stopChannel: - gc.releaseResources() + gc.releaseResources(ctx) return case <-timer.C: startedAt := time.Now() @@ -258,12 +258,12 @@ func (gc *gc) tickRemover(ctx context.Context) { } } -func (gc *gc) stop() { +func (gc *gc) stop(ctx context.Context) { gc.onceStop.Do(func() { close(gc.stopChannel) }) - gc.log.Info(context.Background(), logs.ShardWaitingForGCWorkersToStop) + gc.log.Info(ctx, logs.ShardWaitingForGCWorkersToStop) gc.wg.Wait() } @@ -730,14 +730,14 @@ func (s *Shard) inhumeUnlockedIfExpired(ctx context.Context, epoch uint64, unloc } // HandleDeletedLocks unlocks all objects which were locked by lockers. -func (s *Shard) HandleDeletedLocks(lockers []oid.Address) { +func (s *Shard) HandleDeletedLocks(ctx context.Context, lockers []oid.Address) { if s.GetMode().NoMetabase() { return } _, err := s.metaBase.FreeLockedBy(lockers) if err != nil { - s.log.Warn(context.Background(), logs.ShardFailureToUnlockObjects, + s.log.Warn(ctx, logs.ShardFailureToUnlockObjects, zap.String("error", err.Error()), ) diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index 39073a529..9998bbae2 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -61,8 +61,8 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { meta.WithEpochState(epochState{}), ), WithPiloramaOptions(pilorama.WithPath(filepath.Join(rootPath, "pilorama"))), - WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { - sh.HandleDeletedLocks(addresses) + WithDeletedLockCallback(func(ctx context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(ctx, addresses) }), WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { sh.HandleExpiredLocks(ctx, epoch, a) diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 7da8b8c28..5caf3641f 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -53,8 +53,8 @@ func TestShard_Lock(t *testing.T) { meta.WithPath(filepath.Join(rootPath, "meta")), meta.WithEpochState(epochState{}), ), - WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { - sh.HandleDeletedLocks(addresses) + WithDeletedLockCallback(func(ctx context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(ctx, addresses) }), } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 3a06fe8a7..1eb7f14d0 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -95,7 +95,7 @@ type cfg struct { metricsWriter MetricsWriter - reportErrorFunc func(selfID string, message string, err error) + reportErrorFunc func(ctx context.Context, selfID string, message string, err error) containerInfo container.InfoProvider } @@ -105,7 +105,7 @@ func defaultCfg() *cfg { rmBatchSize: 100, log: logger.NewLoggerWrapper(zap.L()), gcCfg: defaultGCCfg(), - reportErrorFunc: func(string, string, error) {}, + reportErrorFunc: func(context.Context, string, string, error) {}, zeroSizeContainersCallback: func(context.Context, []cid.ID) {}, zeroCountContainersCallback: func(context.Context, []cid.ID) {}, metricsWriter: noopMetrics{}, @@ -130,8 +130,8 @@ func New(opts ...Option) *Shard { tsSource: c.tsSource, } - reportFunc := func(msg string, err error) { - s.reportErrorFunc(s.ID().String(), msg, err) + reportFunc := func(ctx context.Context, msg string, err error) { + s.reportErrorFunc(ctx, s.ID().String(), msg, err) } s.blobStor.SetReportErrorFunc(reportFunc) @@ -317,7 +317,7 @@ func WithGCMetrics(v GCMectrics) Option { // WithReportErrorFunc returns option to specify callback for handling storage-related errors // in the background workers. -func WithReportErrorFunc(f func(selfID string, message string, err error)) Option { +func WithReportErrorFunc(f func(ctx context.Context, selfID string, message string, err error)) Option { return func(c *cfg) { c.reportErrorFunc = f } diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index 73ba2e82b..f9ee34488 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -89,8 +89,8 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard WithPiloramaOptions(pilorama.WithPath(filepath.Join(o.rootPath, "pilorama"))), WithWriteCache(enableWriteCache), WithWriteCacheOptions(o.wcOpts), - WithDeletedLockCallback(func(_ context.Context, addresses []oid.Address) { - sh.HandleDeletedLocks(addresses) + WithDeletedLockCallback(func(ctx context.Context, addresses []oid.Address) { + sh.HandleDeletedLocks(ctx, addresses) }), WithExpiredLocksCallback(func(ctx context.Context, epoch uint64, a []oid.Address) { sh.HandleExpiredLocks(ctx, epoch, a) diff --git a/pkg/local_object_storage/writecache/benchmark/writecache_test.go b/pkg/local_object_storage/writecache/benchmark/writecache_test.go index 79ab7d9c6..fd85b4501 100644 --- a/pkg/local_object_storage/writecache/benchmark/writecache_test.go +++ b/pkg/local_object_storage/writecache/benchmark/writecache_test.go @@ -43,12 +43,12 @@ func BenchmarkWriteAfterDelete(b *testing.B) { b.SetParallelism(parallel) benchmarkRunPar(b, cache, payloadSize) }) - require.NoError(b, cache.Close()) + require.NoError(b, cache.Close(context.Background())) } func benchmarkPutSeq(b *testing.B, cache writecache.Cache, size uint64) { benchmarkPutPrepare(b, cache) - defer func() { require.NoError(b, cache.Close()) }() + defer func() { require.NoError(b, cache.Close(context.Background())) }() ctx := context.Background() objGen := testutil.RandObjGenerator{ObjSize: size} @@ -71,7 +71,7 @@ func benchmarkPutSeq(b *testing.B, cache writecache.Cache, size uint64) { func benchmarkPutPar(b *testing.B, cache writecache.Cache, size uint64) { benchmarkPutPrepare(b, cache) - defer func() { require.NoError(b, cache.Close()) }() + defer func() { require.NoError(b, cache.Close(context.Background())) }() benchmarkRunPar(b, cache, size) } @@ -100,7 +100,7 @@ func benchmarkRunPar(b *testing.B, cache writecache.Cache, size uint64) { func benchmarkPutPrepare(b *testing.B, cache writecache.Cache) { require.NoError(b, cache.Open(context.Background(), mode.ReadWrite), "opening") - require.NoError(b, cache.Init(), "initializing") + require.NoError(b, cache.Init(context.Background()), "initializing") } type testMetabase struct{} diff --git a/pkg/local_object_storage/writecache/cache.go b/pkg/local_object_storage/writecache/cache.go index 098872e08..e829d013c 100644 --- a/pkg/local_object_storage/writecache/cache.go +++ b/pkg/local_object_storage/writecache/cache.go @@ -98,19 +98,19 @@ func (c *cache) Open(_ context.Context, mod mode.Mode) error { } // Init runs necessary services. -func (c *cache) Init() error { +func (c *cache) Init(ctx context.Context) error { c.metrics.SetMode(mode.ConvertToComponentModeDegraded(c.mode)) - if err := c.flushAndDropBBoltDB(context.Background()); err != nil { + if err := c.flushAndDropBBoltDB(ctx); err != nil { return fmt.Errorf("flush previous version write-cache database: %w", err) } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.WithoutCancel(ctx)) // canceling performed by cache c.cancel.Store(cancel) c.runFlushLoop(ctx) return nil } // Close closes db connection and stops services. Executes ObjectCounters.FlushAndClose op. -func (c *cache) Close() error { +func (c *cache) Close(ctx context.Context) error { if cancelValue := c.cancel.Swap(dummyCanceler); cancelValue != nil { cancelValue.(context.CancelFunc)() } @@ -127,7 +127,7 @@ func (c *cache) Close() error { var err error if c.fsTree != nil { - err = c.fsTree.Close() + err = c.fsTree.Close(ctx) if err != nil { c.fsTree = nil } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 123eb4abc..d9e34ceab 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -112,7 +112,7 @@ func (c *cache) flushIfAnObjectExistsWorker(ctx context.Context, objInfo objectI }) if err != nil { if !client.IsErrObjectNotFound(err) { - c.reportFlushError(logs.WritecacheCantGetObject, objInfo.addr.EncodeToString(), metaerr.Wrap(err)) + c.reportFlushError(ctx, logs.WritecacheCantGetObject, objInfo.addr.EncodeToString(), metaerr.Wrap(err)) } return } @@ -126,11 +126,11 @@ func (c *cache) flushIfAnObjectExistsWorker(ctx context.Context, objInfo objectI c.deleteFromDisk(ctx, objInfo.addr, uint64(len(res.RawData))) } -func (c *cache) reportFlushError(msg string, addr string, err error) { +func (c *cache) reportFlushError(ctx context.Context, msg string, addr string, err error) { if c.reportError != nil { - c.reportError(msg, err) + c.reportError(ctx, msg, err) } else { - c.log.Error(context.Background(), msg, + c.log.Error(ctx, msg, zap.String("address", addr), zap.Error(err)) } @@ -145,7 +145,7 @@ func (c *cache) flushFSTree(ctx context.Context, ignoreErrors bool) error { var obj objectSDK.Object err := obj.Unmarshal(e.ObjectData) if err != nil { - c.reportFlushError(logs.FSTreeCantUnmarshalObject, sAddr, metaerr.Wrap(err)) + c.reportFlushError(ctx, logs.FSTreeCantUnmarshalObject, sAddr, metaerr.Wrap(err)) if ignoreErrors { return nil } @@ -183,7 +183,7 @@ func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []b if err != nil { if !errors.Is(err, common.ErrNoSpace) && !errors.Is(err, common.ErrReadOnly) && !errors.Is(err, blobstor.ErrNoPlaceFound) { - c.reportFlushError(logs.FSTreeCantFushObjectBlobstor, + c.reportFlushError(ctx, logs.FSTreeCantFushObjectBlobstor, addr.EncodeToString(), err) } return err @@ -195,7 +195,7 @@ func (c *cache) flushObject(ctx context.Context, obj *objectSDK.Object, data []b _, err = c.metabase.UpdateStorageID(ctx, updPrm) if err != nil { - c.reportFlushError(logs.FSTreeCantUpdateID, + c.reportFlushError(ctx, logs.FSTreeCantUpdateID, addr.EncodeToString(), err) } return err diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 92fb493e0..7fc84657c 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -38,9 +38,9 @@ func TestFlush(t *testing.T) { errCountOpt := func() (Option, *atomic.Uint32) { cnt := &atomic.Uint32{} - return WithReportErrorFunc(func(msg string, err error) { + return WithReportErrorFunc(func(ctx context.Context, msg string, err error) { cnt.Add(1) - testlogger.Warn(context.Background(), msg, zap.Uint32("error_count", cnt.Load()), zap.Error(err)) + testlogger.Warn(ctx, msg, zap.Uint32("error_count", cnt.Load()), zap.Error(err)) }), cnt } @@ -114,7 +114,7 @@ func runFlushTest[Option any]( ) { t.Run("no errors", func(t *testing.T) { wc, bs, mb := newCache(t, createCacheFn) - defer func() { require.NoError(t, wc.Close()) }() + defer func() { require.NoError(t, wc.Close(context.Background())) }() objects := putObjects(t, wc) require.NoError(t, bs.SetMode(context.Background(), mode.ReadWrite)) @@ -127,7 +127,7 @@ func runFlushTest[Option any]( t.Run("flush on moving to degraded mode", func(t *testing.T) { wc, bs, mb := newCache(t, createCacheFn) - defer func() { require.NoError(t, wc.Close()) }() + defer func() { require.NoError(t, wc.Close(context.Background())) }() objects := putObjects(t, wc) // Blobstor is read-only, so we expect en error from `flush` here. @@ -145,7 +145,7 @@ func runFlushTest[Option any]( t.Run(f.Desc, func(t *testing.T) { errCountOpt, errCount := errCountOption() wc, bs, mb := newCache(t, createCacheFn, errCountOpt) - defer func() { require.NoError(t, wc.Close()) }() + defer func() { require.NoError(t, wc.Close(context.Background())) }() objects := putObjects(t, wc) f.InjectFn(t, wc) @@ -173,7 +173,7 @@ func newCache[Option any]( meta.WithPath(filepath.Join(dir, "meta")), meta.WithEpochState(dummyEpoch{})) require.NoError(t, mb.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, mb.Init()) + require.NoError(t, mb.Init(context.Background())) bs := blobstor.New(blobstor.WithStorages([]blobstor.SubStorage{ { @@ -184,11 +184,11 @@ func newCache[Option any]( }, })) require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, bs.Init()) + require.NoError(t, bs.Init(context.Background())) wc := createCacheFn(t, mb, bs, opts...) require.NoError(t, wc.Open(context.Background(), mode.ReadWrite)) - require.NoError(t, wc.Init()) + require.NoError(t, wc.Init(context.Background())) // First set mode for metabase and blobstor to prevent background flushes. require.NoError(t, mb.SetMode(context.Background(), mode.ReadOnly)) diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index db789d994..73d12fd33 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -82,7 +82,7 @@ func (c *cache) closeStorage(ctx context.Context, shrink bool) error { return nil } if !shrink { - if err := c.fsTree.Close(); err != nil { + if err := c.fsTree.Close(ctx); err != nil { return fmt.Errorf("can't close write-cache storage: %w", err) } return nil @@ -101,7 +101,7 @@ func (c *cache) closeStorage(ctx context.Context, shrink bool) error { return fmt.Errorf("failed to check write-cache items: %w", err) } } - if err := c.fsTree.Close(); err != nil { + if err := c.fsTree.Close(ctx); err != nil { return fmt.Errorf("can't close write-cache storage: %w", err) } if empty { diff --git a/pkg/local_object_storage/writecache/mode_test.go b/pkg/local_object_storage/writecache/mode_test.go index 70cfe8382..4fbadbc64 100644 --- a/pkg/local_object_storage/writecache/mode_test.go +++ b/pkg/local_object_storage/writecache/mode_test.go @@ -18,13 +18,13 @@ func TestMode(t *testing.T) { require.NoError(t, wc.Open(context.Background(), mode.DegradedReadOnly)) require.Nil(t, wc.(*cache).fsTree) - require.NoError(t, wc.Init()) + require.NoError(t, wc.Init(context.Background())) require.Nil(t, wc.(*cache).fsTree) - require.NoError(t, wc.Close()) + require.NoError(t, wc.Close(context.Background())) require.NoError(t, wc.Open(context.Background(), mode.Degraded)) require.Nil(t, wc.(*cache).fsTree) - require.NoError(t, wc.Init()) + require.NoError(t, wc.Init(context.Background())) require.Nil(t, wc.(*cache).fsTree) - require.NoError(t, wc.Close()) + require.NoError(t, wc.Close(context.Background())) } diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index 25c1694a8..f2957fe98 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -1,6 +1,8 @@ package writecache import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -29,7 +31,7 @@ type options struct { // noSync is true iff FSTree allows unsynchronized writes. noSync bool // reportError is the function called when encountering disk errors in background workers. - reportError func(string, error) + reportError func(context.Context, string, error) // metrics is metrics implementation metrics Metrics // disableBackgroundFlush is for testing purposes only. @@ -108,7 +110,7 @@ func WithNoSync(noSync bool) Option { } // WithReportErrorFunc sets error reporting function. -func WithReportErrorFunc(f func(string, error)) Option { +func WithReportErrorFunc(f func(context.Context, string, error)) Option { return func(o *options) { o.reportError = f } diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index d07220b68..70b17eb8e 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -44,9 +44,9 @@ type Cache interface { Flush(context.Context, bool, bool) error Seal(context.Context, SealPrm) error - Init() error + Init(context.Context) error Open(ctx context.Context, mode mode.Mode) error - Close() error + Close(context.Context) error GetMetrics() Metrics } diff --git a/pkg/morph/client/balance/burn.go b/pkg/morph/client/balance/burn.go index 4befbef45..f4685b0ab 100644 --- a/pkg/morph/client/balance/burn.go +++ b/pkg/morph/client/balance/burn.go @@ -1,6 +1,8 @@ package balance import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/util" ) @@ -30,12 +32,12 @@ func (b *BurnPrm) SetID(id []byte) { } // Burn destroys funds from the account. -func (c *Client) Burn(p BurnPrm) error { +func (c *Client) Burn(ctx context.Context, p BurnPrm) error { prm := client.InvokePrm{} prm.SetMethod(burnMethod) prm.SetArgs(p.to, p.amount, p.id) prm.InvokePrmOptional = p.InvokePrmOptional - _, err := c.client.Invoke(prm) + _, err := c.client.Invoke(ctx, prm) return err } diff --git a/pkg/morph/client/balance/lock.go b/pkg/morph/client/balance/lock.go index a5b206799..83e8b0586 100644 --- a/pkg/morph/client/balance/lock.go +++ b/pkg/morph/client/balance/lock.go @@ -1,6 +1,8 @@ package balance import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/util" ) @@ -42,12 +44,12 @@ func (l *LockPrm) SetDueEpoch(dueEpoch int64) { } // Lock locks fund on the user account. -func (c *Client) Lock(p LockPrm) error { +func (c *Client) Lock(ctx context.Context, p LockPrm) error { prm := client.InvokePrm{} prm.SetMethod(lockMethod) prm.SetArgs(p.id, p.user, p.lock, p.amount, p.dueEpoch) prm.InvokePrmOptional = p.InvokePrmOptional - _, err := c.client.Invoke(prm) + _, err := c.client.Invoke(ctx, prm) return err } diff --git a/pkg/morph/client/balance/mint.go b/pkg/morph/client/balance/mint.go index 73448da31..082ade85e 100644 --- a/pkg/morph/client/balance/mint.go +++ b/pkg/morph/client/balance/mint.go @@ -1,6 +1,8 @@ package balance import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/util" ) @@ -30,12 +32,12 @@ func (m *MintPrm) SetID(id []byte) { } // Mint sends funds to the account. -func (c *Client) Mint(p MintPrm) error { +func (c *Client) Mint(ctx context.Context, p MintPrm) error { prm := client.InvokePrm{} prm.SetMethod(mintMethod) prm.SetArgs(p.to, p.amount, p.id) prm.InvokePrmOptional = p.InvokePrmOptional - _, err := c.client.Invoke(prm) + _, err := c.client.Invoke(ctx, prm) return err } diff --git a/pkg/morph/client/balance/transfer.go b/pkg/morph/client/balance/transfer.go index 08fb05289..65a0b70a6 100644 --- a/pkg/morph/client/balance/transfer.go +++ b/pkg/morph/client/balance/transfer.go @@ -1,6 +1,7 @@ package balance import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" @@ -23,7 +24,7 @@ type TransferPrm struct { // with details p.Details through direct smart contract call. // // If TryNotary is provided, calls notary contract. -func (c *Client) TransferX(p TransferPrm) error { +func (c *Client) TransferX(ctx context.Context, p TransferPrm) error { from, err := address.StringToUint160(p.From.EncodeToString()) if err != nil { return err @@ -39,7 +40,7 @@ func (c *Client) TransferX(p TransferPrm) error { prm.SetArgs(from, to, p.Amount, p.Details) prm.InvokePrmOptional = p.InvokePrmOptional - _, err = c.client.Invoke(prm) + _, err = c.client.Invoke(ctx, prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", transferXMethod, err) } diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index ef6a009e4..f61c6e9f9 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -180,7 +180,7 @@ func wrapFrostFSError(err error) error { // Invoke invokes contract method by sending transaction into blockchain. // Returns valid until block value. // Supported args types: int64, string, util.Uint160, []byte and bool. -func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) { +func (c *Client) Invoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) { start := time.Now() success := false defer func() { @@ -199,7 +199,7 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, return 0, fmt.Errorf("could not invoke %s: %w", method, err) } - c.logger.Debug(context.Background(), logs.ClientNeoClientInvoke, + c.logger.Debug(ctx, logs.ClientNeoClientInvoke, zap.String("method", method), zap.Uint32("vub", vub), zap.Stringer("tx_hash", txHash.Reverse())) diff --git a/pkg/morph/client/container/delete.go b/pkg/morph/client/container/delete.go index 20351b570..5696645b2 100644 --- a/pkg/morph/client/container/delete.go +++ b/pkg/morph/client/container/delete.go @@ -1,6 +1,7 @@ package container import ( + "context" "crypto/sha256" "fmt" @@ -12,7 +13,7 @@ import ( // along with signature and session token. // // Returns error if container ID is nil. -func Delete(c *Client, witness core.RemovalWitness) error { +func Delete(ctx context.Context, c *Client, witness core.RemovalWitness) error { binCnr := make([]byte, sha256.Size) witness.ContainerID.Encode(binCnr) @@ -26,7 +27,7 @@ func Delete(c *Client, witness core.RemovalWitness) error { prm.SetToken(tok.Marshal()) } - _, err := c.Delete(prm) + _, err := c.Delete(ctx, prm) return err } @@ -67,7 +68,7 @@ func (d *DeletePrm) SetKey(key []byte) { // the removal to interrupt. // // If TryNotary is provided, calls notary contract. -func (c *Client) Delete(p DeletePrm) (uint32, error) { +func (c *Client) Delete(ctx context.Context, p DeletePrm) (uint32, error) { if len(p.signature) == 0 && !p.IsControl() { return 0, errNilArgument } @@ -77,7 +78,7 @@ func (c *Client) Delete(p DeletePrm) (uint32, error) { prm.SetArgs(p.cnr, p.signature, p.key, p.token) prm.InvokePrmOptional = p.InvokePrmOptional - res, err := c.client.Invoke(prm) + res, err := c.client.Invoke(ctx, prm) if err != nil { return 0, fmt.Errorf("could not invoke method (%s): %w", deleteMethod, err) } diff --git a/pkg/morph/client/container/put.go b/pkg/morph/client/container/put.go index 777ae2d4e..74d9f6da8 100644 --- a/pkg/morph/client/container/put.go +++ b/pkg/morph/client/container/put.go @@ -1,6 +1,7 @@ package container import ( + "context" "fmt" containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" @@ -14,7 +15,7 @@ import ( // along with sig.Key() and sig.Sign(). // // Returns error if container is nil. -func Put(c *Client, cnr containercore.Container) (*cid.ID, error) { +func Put(ctx context.Context, c *Client, cnr containercore.Container) (*cid.ID, error) { data := cnr.Value.Marshal() d := container.ReadDomain(cnr.Value) @@ -35,7 +36,7 @@ func Put(c *Client, cnr containercore.Container) (*cid.ID, error) { prm.SetKey(sigV2.GetKey()) prm.SetSignature(sigV2.GetSign()) - err := c.Put(prm) + err := c.Put(ctx, prm) if err != nil { return nil, err } @@ -95,7 +96,7 @@ func (p *PutPrm) SetZone(zone string) { // encountered that caused the saving to interrupt. // // If TryNotary is provided, calls notary contract. -func (c *Client) Put(p PutPrm) error { +func (c *Client) Put(ctx context.Context, p PutPrm) error { if len(p.sig) == 0 || len(p.key) == 0 { return errNilArgument } @@ -116,7 +117,7 @@ func (c *Client) Put(p PutPrm) error { prm.SetMethod(method) prm.InvokePrmOptional = p.InvokePrmOptional - _, err := c.client.Invoke(prm) + _, err := c.client.Invoke(ctx, prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", method, err) } diff --git a/pkg/morph/client/frostfs/cheque.go b/pkg/morph/client/frostfs/cheque.go index 016b56f8f..d3eba7639 100644 --- a/pkg/morph/client/frostfs/cheque.go +++ b/pkg/morph/client/frostfs/cheque.go @@ -1,6 +1,8 @@ package frostfscontract import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/util" @@ -37,13 +39,13 @@ func (c *ChequePrm) SetLock(lock util.Uint160) { } // Cheque invokes `cheque` method of FrostFS contract. -func (x *Client) Cheque(p ChequePrm) error { +func (x *Client) Cheque(ctx context.Context, p ChequePrm) error { prm := client.InvokePrm{} prm.SetMethod(chequeMethod) prm.SetArgs(p.id, p.user, p.amount, p.lock) prm.InvokePrmOptional = p.InvokePrmOptional - _, err := x.client.Invoke(prm) + _, err := x.client.Invoke(ctx, prm) return err } @@ -66,12 +68,12 @@ func (a *AlphabetUpdatePrm) SetPubs(pubs keys.PublicKeys) { } // AlphabetUpdate update list of alphabet nodes. -func (x *Client) AlphabetUpdate(p AlphabetUpdatePrm) error { +func (x *Client) AlphabetUpdate(ctx context.Context, p AlphabetUpdatePrm) error { prm := client.InvokePrm{} prm.SetMethod(alphabetUpdateMethod) prm.SetArgs(p.id, p.pubs) prm.InvokePrmOptional = p.InvokePrmOptional - _, err := x.client.Invoke(prm) + _, err := x.client.Invoke(ctx, prm) return err } diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 2d19a8193..0a3c351db 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "errors" "fmt" @@ -155,13 +156,13 @@ func (s *SetConfigPrm) SetValue(value any) { } // SetConfig sets config field. -func (c *Client) SetConfig(p SetConfigPrm) error { +func (c *Client) SetConfig(ctx context.Context, p SetConfigPrm) error { prm := client.InvokePrm{} prm.SetMethod(setConfigMethod) prm.SetArgs(p.id, p.key, p.value) prm.InvokePrmOptional = p.InvokePrmOptional - _, err := c.client.Invoke(prm) + _, err := c.client.Invoke(ctx, prm) return err } diff --git a/pkg/morph/client/netmap/innerring.go b/pkg/morph/client/netmap/innerring.go index d6f8c56b2..c9dc7d2fc 100644 --- a/pkg/morph/client/netmap/innerring.go +++ b/pkg/morph/client/netmap/innerring.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "crypto/elliptic" "fmt" @@ -23,7 +24,7 @@ func (u *UpdateIRPrm) SetKeys(keys keys.PublicKeys) { } // UpdateInnerRing updates inner ring keys. -func (c *Client) UpdateInnerRing(p UpdateIRPrm) error { +func (c *Client) UpdateInnerRing(ctx context.Context, p UpdateIRPrm) error { args := make([][]byte, len(p.keys)) for i := range args { args[i] = p.keys[i].Bytes() @@ -34,7 +35,7 @@ func (c *Client) UpdateInnerRing(p UpdateIRPrm) error { prm.SetArgs(args) prm.InvokePrmOptional = p.InvokePrmOptional - _, err := c.client.Invoke(prm) + _, err := c.client.Invoke(ctx, prm) return err } diff --git a/pkg/morph/client/netmap/new_epoch.go b/pkg/morph/client/netmap/new_epoch.go index ded386c86..efcdfd7b6 100644 --- a/pkg/morph/client/netmap/new_epoch.go +++ b/pkg/morph/client/netmap/new_epoch.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" @@ -8,12 +9,12 @@ import ( // NewEpoch updates FrostFS epoch number through // Netmap contract call. -func (c *Client) NewEpoch(epoch uint64) error { +func (c *Client) NewEpoch(ctx context.Context, epoch uint64) error { prm := client.InvokePrm{} prm.SetMethod(newEpochMethod) prm.SetArgs(epoch) - _, err := c.client.Invoke(prm) + _, err := c.client.Invoke(ctx, prm) if err != nil { return fmt.Errorf("could not invoke method (%s): %w", newEpochMethod, err) } @@ -24,14 +25,14 @@ func (c *Client) NewEpoch(epoch uint64) error { // control notary transaction internally to ensure all // nodes produce the same transaction with high probability. // If vub > 0, vub will be used as valid until block value. -func (c *Client) NewEpochControl(epoch uint64, vub uint32) (uint32, error) { +func (c *Client) NewEpochControl(ctx context.Context, epoch uint64, vub uint32) (uint32, error) { prm := client.InvokePrm{} prm.SetMethod(newEpochMethod) prm.SetArgs(epoch) prm.SetControlTX(true) prm.SetVUB(vub) - res, err := c.client.Invoke(prm) + res, err := c.client.Invoke(ctx, prm) if err != nil { return 0, fmt.Errorf("could not invoke method (%s): %w", newEpochMethod, err) } diff --git a/pkg/morph/client/netmap/peer.go b/pkg/morph/client/netmap/peer.go index 764bbc899..9617d018c 100644 --- a/pkg/morph/client/netmap/peer.go +++ b/pkg/morph/client/netmap/peer.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "errors" "fmt" @@ -24,7 +25,7 @@ func (a *AddPeerPrm) SetNodeInfo(nodeInfo netmap.NodeInfo) { // AddPeer registers peer in FrostFS network through // Netmap contract call. -func (c *Client) AddPeer(p AddPeerPrm) error { +func (c *Client) AddPeer(ctx context.Context, p AddPeerPrm) error { method := addPeerMethod if c.client.WithNotary() && c.client.IsAlpha() { @@ -39,7 +40,7 @@ func (c *Client) AddPeer(p AddPeerPrm) error { prm.SetArgs(p.nodeInfo.Marshal()) prm.InvokePrmOptional = p.InvokePrmOptional - if _, err := c.client.Invoke(prm); err != nil { + if _, err := c.client.Invoke(ctx, prm); err != nil { return fmt.Errorf("could not invoke method (%s): %w", method, err) } return nil @@ -47,7 +48,7 @@ func (c *Client) AddPeer(p AddPeerPrm) error { // ForceRemovePeer marks the given peer as offline via a notary control transaction. // If vub > 0, vub will be used as valid until block value. -func (c *Client) ForceRemovePeer(nodeInfo netmap.NodeInfo, vub uint32) (uint32, error) { +func (c *Client) ForceRemovePeer(ctx context.Context, nodeInfo netmap.NodeInfo, vub uint32) (uint32, error) { if !c.client.WithNotary() { return 0, errFailedToRemovePeerWithoutNotary } @@ -57,7 +58,7 @@ func (c *Client) ForceRemovePeer(nodeInfo netmap.NodeInfo, vub uint32) (uint32, prm.SetControlTX(true) prm.SetVUB(vub) - vub, err := c.UpdatePeerState(prm) + vub, err := c.UpdatePeerState(ctx, prm) if err != nil { return 0, fmt.Errorf("updating peer state: %v", err) } diff --git a/pkg/morph/client/netmap/update_state.go b/pkg/morph/client/netmap/update_state.go index 7c3a4e8cd..971a55d33 100644 --- a/pkg/morph/client/netmap/update_state.go +++ b/pkg/morph/client/netmap/update_state.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" @@ -36,7 +37,7 @@ func (u *UpdatePeerPrm) SetMaintenance() { } // UpdatePeerState changes peer status through Netmap contract call. -func (c *Client) UpdatePeerState(p UpdatePeerPrm) (uint32, error) { +func (c *Client) UpdatePeerState(ctx context.Context, p UpdatePeerPrm) (uint32, error) { method := updateStateMethod if c.client.WithNotary() && c.client.IsAlpha() { @@ -55,7 +56,7 @@ func (c *Client) UpdatePeerState(p UpdatePeerPrm) (uint32, error) { prm.SetArgs(int64(p.state), p.key) prm.InvokePrmOptional = p.InvokePrmOptional - res, err := c.client.Invoke(prm) + res, err := c.client.Invoke(ctx, prm) if err != nil { return 0, fmt.Errorf("could not invoke smart contract: %w", err) } diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 58c417fb1..65a5e77a6 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -141,7 +141,7 @@ func (c *Client) ProbeNotary() (res bool) { // use this function. // // This function must be invoked with notary enabled otherwise it throws panic. -func (c *Client) DepositNotary(amount fixedn.Fixed8, delta uint32) (util.Uint256, error) { +func (c *Client) DepositNotary(ctx context.Context, amount fixedn.Fixed8, delta uint32) (util.Uint256, error) { c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -164,7 +164,7 @@ func (c *Client) DepositNotary(amount fixedn.Fixed8, delta uint32) (util.Uint256 } till := max(int64(bc+delta), currentTill) - res, _, err := c.depositNotary(amount, till) + res, _, err := c.depositNotary(ctx, amount, till) return res, err } @@ -173,7 +173,7 @@ func (c *Client) DepositNotary(amount fixedn.Fixed8, delta uint32) (util.Uint256 // This allows to avoid ValidAfterDeposit failures. // // This function must be invoked with notary enabled otherwise it throws panic. -func (c *Client) DepositEndlessNotary(amount fixedn.Fixed8) (util.Uint256, uint32, error) { +func (c *Client) DepositEndlessNotary(ctx context.Context, amount fixedn.Fixed8) (util.Uint256, uint32, error) { c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -186,10 +186,10 @@ func (c *Client) DepositEndlessNotary(amount fixedn.Fixed8) (util.Uint256, uint3 } // till value refers to a block height and it is uint32 value in neo-go - return c.depositNotary(amount, math.MaxUint32) + return c.depositNotary(ctx, amount, math.MaxUint32) } -func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (util.Uint256, uint32, error) { +func (c *Client) depositNotary(ctx context.Context, amount fixedn.Fixed8, till int64) (util.Uint256, uint32, error) { txHash, vub, err := c.gasToken.Transfer( c.accAddr, c.notary.notary, @@ -202,7 +202,7 @@ func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (util.Uint256, // Transaction is already in mempool waiting to be processed. // This is an expected situation if we restart the service. - c.logger.Info(context.Background(), logs.ClientNotaryDepositHasAlreadyBeenMade, + c.logger.Info(ctx, logs.ClientNotaryDepositHasAlreadyBeenMade, zap.Int64("amount", int64(amount)), zap.Int64("expire_at", till), zap.Uint32("vub", vub), @@ -210,7 +210,7 @@ func (c *Client) depositNotary(amount fixedn.Fixed8, till int64) (util.Uint256, return util.Uint256{}, 0, nil } - c.logger.Info(context.Background(), logs.ClientNotaryDepositInvoke, + c.logger.Info(ctx, logs.ClientNotaryDepositInvoke, zap.Int64("amount", int64(amount)), zap.Int64("expire_at", till), zap.Uint32("vub", vub), @@ -275,7 +275,7 @@ func (u *UpdateNotaryListPrm) SetHash(hash util.Uint256) { // committee multi signature. // // This function must be invoked with notary enabled otherwise it throws panic. -func (c *Client) UpdateNotaryList(prm UpdateNotaryListPrm) error { +func (c *Client) UpdateNotaryList(ctx context.Context, prm UpdateNotaryListPrm) error { c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -293,6 +293,7 @@ func (c *Client) UpdateNotaryList(prm UpdateNotaryListPrm) error { } return c.notaryInvokeAsCommittee( + ctx, setDesignateMethod, nonce, vub, @@ -323,7 +324,7 @@ func (u *UpdateAlphabetListPrm) SetHash(hash util.Uint256) { // Requires committee multi signature. // // This function must be invoked with notary enabled otherwise it throws panic. -func (c *Client) UpdateNeoFSAlphabetList(prm UpdateAlphabetListPrm) error { +func (c *Client) UpdateNeoFSAlphabetList(ctx context.Context, prm UpdateAlphabetListPrm) error { c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -341,6 +342,7 @@ func (c *Client) UpdateNeoFSAlphabetList(prm UpdateAlphabetListPrm) error { } return c.notaryInvokeAsCommittee( + ctx, setDesignateMethod, nonce, vub, @@ -356,7 +358,7 @@ func (c *Client) UpdateNeoFSAlphabetList(prm UpdateAlphabetListPrm) error { // Returns valid until block value. // // `nonce` and `vub` are used only if notary is enabled. -func (c *Client) NotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) (uint32, error) { +func (c *Client) NotaryInvoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) (uint32, error) { c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -365,10 +367,10 @@ func (c *Client) NotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce ui } if c.notary == nil { - return c.Invoke(contract, fee, method, args...) + return c.Invoke(ctx, contract, fee, method, args...) } - return c.notaryInvoke(false, true, contract, nonce, vub, method, args...) + return c.notaryInvoke(ctx, false, true, contract, nonce, vub, method, args...) } // NotaryInvokeNotAlpha does the same as NotaryInvoke but does not use client's @@ -376,7 +378,7 @@ func (c *Client) NotaryInvoke(contract util.Uint160, fee fixedn.Fixed8, nonce ui // not expected to be signed by the current node. // // Considered to be used by non-IR nodes. -func (c *Client) NotaryInvokeNotAlpha(contract util.Uint160, fee fixedn.Fixed8, vubP *uint32, method string, args ...any) (uint32, error) { +func (c *Client) NotaryInvokeNotAlpha(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, vubP *uint32, method string, args ...any) (uint32, error) { c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -385,10 +387,10 @@ func (c *Client) NotaryInvokeNotAlpha(contract util.Uint160, fee fixedn.Fixed8, } if c.notary == nil { - return c.Invoke(contract, fee, method, args...) + return c.Invoke(ctx, contract, fee, method, args...) } - return c.notaryInvoke(false, false, contract, rand.Uint32(), vubP, method, args...) + return c.notaryInvoke(ctx, false, false, contract, rand.Uint32(), vubP, method, args...) } // NotarySignAndInvokeTX signs and sends notary request that was received from @@ -438,13 +440,13 @@ func (c *Client) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { return nil } -func (c *Client) notaryInvokeAsCommittee(method string, nonce, vub uint32, args ...any) error { +func (c *Client) notaryInvokeAsCommittee(ctx context.Context, method string, nonce, vub uint32, args ...any) error { designate := c.GetDesignateHash() - _, err := c.notaryInvoke(true, true, designate, nonce, &vub, method, args...) + _, err := c.notaryInvoke(ctx, true, true, designate, nonce, &vub, method, args...) return err } -func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint160, nonce uint32, vub *uint32, method string, args ...any) (uint32, error) { +func (c *Client) notaryInvoke(ctx context.Context, committee, invokedByAlpha bool, contract util.Uint160, nonce uint32, vub *uint32, method string, args ...any) (uint32, error) { start := time.Now() success := false defer func() { @@ -486,7 +488,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint return 0, err } - c.logger.Debug(context.Background(), logs.ClientNotaryRequestInvoked, + c.logger.Debug(ctx, logs.ClientNotaryRequestInvoked, zap.String("method", method), zap.Uint32("valid_until_block", untilActual), zap.String("tx_hash", mainH.StringLE()), diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index dfcf62b83..1e091936f 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -1,6 +1,7 @@ package client import ( + "context" "fmt" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" @@ -140,7 +141,7 @@ type InvokeRes struct { // // If fee for the operation executed using specified method is customized, then StaticClient uses it. // Otherwise, default fee is used. -func (s StaticClient) Invoke(prm InvokePrm) (InvokeRes, error) { +func (s StaticClient) Invoke(ctx context.Context, prm InvokePrm) (InvokeRes, error) { var res InvokeRes var err error var vubP *uint32 @@ -169,7 +170,7 @@ func (s StaticClient) Invoke(prm InvokePrm) (InvokeRes, error) { vubP = &prm.vub } - res.VUB, err = s.client.NotaryInvoke(s.scScriptHash, s.fee, nonce, vubP, prm.method, prm.args...) + res.VUB, err = s.client.NotaryInvoke(ctx, s.scScriptHash, s.fee, nonce, vubP, prm.method, prm.args...) return res, err } @@ -177,11 +178,12 @@ func (s StaticClient) Invoke(prm InvokePrm) (InvokeRes, error) { vubP = &prm.vub } - res.VUB, err = s.client.NotaryInvokeNotAlpha(s.scScriptHash, s.fee, vubP, prm.method, prm.args...) + res.VUB, err = s.client.NotaryInvokeNotAlpha(ctx, s.scScriptHash, s.fee, vubP, prm.method, prm.args...) return res, err } res.VUB, err = s.client.Invoke( + ctx, s.scScriptHash, s.fee, prm.method, diff --git a/pkg/morph/event/handlers.go b/pkg/morph/event/handlers.go index bda83ba54..822335329 100644 --- a/pkg/morph/event/handlers.go +++ b/pkg/morph/event/handlers.go @@ -10,7 +10,7 @@ import ( type Handler func(context.Context, Event) // BlockHandler is a chain block processing function. -type BlockHandler func(*block.Block) +type BlockHandler func(context.Context, *block.Block) // NotificationHandlerInfo is a structure that groups // the parameters of the handler of particular diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index eeec46540..6e6184e77 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -291,18 +291,18 @@ loop: continue loop } - l.handleBlockEvent(b) + l.handleBlockEvent(ctx, b) } } } -func (l *listener) handleBlockEvent(b *block.Block) { +func (l *listener) handleBlockEvent(ctx context.Context, b *block.Block) { if err := l.pool.Submit(func() { for i := range l.blockHandlers { - l.blockHandlers[i](b) + l.blockHandlers[i](ctx, b) } }); err != nil { - l.log.Warn(context.Background(), logs.EventListenerWorkerPoolDrained, + l.log.Warn(ctx, logs.EventListenerWorkerPoolDrained, zap.Int("capacity", l.pool.Cap())) } } diff --git a/pkg/morph/event/listener_test.go b/pkg/morph/event/listener_test.go index 214daf694..c0f9722d7 100644 --- a/pkg/morph/event/listener_test.go +++ b/pkg/morph/event/listener_test.go @@ -34,7 +34,7 @@ func TestEventHandling(t *testing.T) { blockHandled := make(chan bool) handledBlocks := make([]*block.Block, 0) - l.RegisterBlockHandler(func(b *block.Block) { + l.RegisterBlockHandler(func(_ context.Context, b *block.Block) { handledBlocks = append(handledBlocks, b) blockHandled <- true }) @@ -137,7 +137,7 @@ func TestErrorPassing(t *testing.T) { WorkerPoolCapacity: 10, }) require.NoError(t, err, "failed to create listener") - l.RegisterBlockHandler(func(b *block.Block) {}) + l.RegisterBlockHandler(func(context.Context, *block.Block) {}) errCh := make(chan error) diff --git a/pkg/network/transport/object/grpc/service.go b/pkg/network/transport/object/grpc/service.go index fa6252118..15dacd553 100644 --- a/pkg/network/transport/object/grpc/service.go +++ b/pkg/network/transport/object/grpc/service.go @@ -26,7 +26,7 @@ func New(c objectSvc.ServiceServer) *Server { // Patch opens internal Object patch stream and feeds it by the data read from gRPC stream. func (s *Server) Patch(gStream objectGRPC.ObjectService_PatchServer) error { - stream, err := s.srv.Patch() + stream, err := s.srv.Patch(gStream.Context()) if err != nil { return err } @@ -68,7 +68,7 @@ func (s *Server) Patch(gStream objectGRPC.ObjectService_PatchServer) error { // Put opens internal Object service Put stream and overtakes data from gRPC stream to it. func (s *Server) Put(gStream objectGRPC.ObjectService_PutServer) error { - stream, err := s.srv.Put() + stream, err := s.srv.Put(gStream.Context()) if err != nil { return err } diff --git a/pkg/services/apemanager/audit.go b/pkg/services/apemanager/audit.go index b9bea07fb..61fb025b8 100644 --- a/pkg/services/apemanager/audit.go +++ b/pkg/services/apemanager/audit.go @@ -33,7 +33,7 @@ func (a *auditService) AddChain(ctx context.Context, req *apemanager.AddChainReq return res, err } - audit.LogRequest(a.log, ape_grpc.APEManagerService_AddChain_FullMethodName, req, + audit.LogRequest(ctx, a.log, ape_grpc.APEManagerService_AddChain_FullMethodName, req, audit.TargetFromChainID(req.GetBody().GetTarget().GetTargetType().String(), req.GetBody().GetTarget().GetName(), res.GetBody().GetChainID()), @@ -49,7 +49,7 @@ func (a *auditService) ListChains(ctx context.Context, req *apemanager.ListChain return res, err } - audit.LogRequest(a.log, ape_grpc.APEManagerService_ListChains_FullMethodName, req, + audit.LogRequest(ctx, a.log, ape_grpc.APEManagerService_ListChains_FullMethodName, req, audit.TargetFromChainID(req.GetBody().GetTarget().GetTargetType().String(), req.GetBody().GetTarget().GetName(), nil), @@ -65,7 +65,7 @@ func (a *auditService) RemoveChain(ctx context.Context, req *apemanager.RemoveCh return res, err } - audit.LogRequest(a.log, ape_grpc.APEManagerService_RemoveChain_FullMethodName, req, + audit.LogRequest(ctx, a.log, ape_grpc.APEManagerService_RemoveChain_FullMethodName, req, audit.TargetFromChainID(req.GetBody().GetTarget().GetTargetType().String(), req.GetBody().GetTarget().GetName(), req.GetBody().GetChainID()), diff --git a/pkg/services/container/audit.go b/pkg/services/container/audit.go index 03d3dc13d..411eb4863 100644 --- a/pkg/services/container/audit.go +++ b/pkg/services/container/audit.go @@ -35,7 +35,7 @@ func (a *auditService) Delete(ctx context.Context, req *container.DeleteRequest) return res, err } - audit.LogRequest(a.log, container_grpc.ContainerService_Delete_FullMethodName, req, + audit.LogRequest(ctx, a.log, container_grpc.ContainerService_Delete_FullMethodName, req, audit.TargetFromRef(req.GetBody().GetContainerID(), &cid.ID{}), err == nil) return res, err @@ -47,7 +47,7 @@ func (a *auditService) Get(ctx context.Context, req *container.GetRequest) (*con if !a.enabled.Load() { return res, err } - audit.LogRequest(a.log, container_grpc.ContainerService_Get_FullMethodName, req, + audit.LogRequest(ctx, a.log, container_grpc.ContainerService_Get_FullMethodName, req, audit.TargetFromRef(req.GetBody().GetContainerID(), &cid.ID{}), err == nil) return res, err } @@ -58,7 +58,7 @@ func (a *auditService) List(ctx context.Context, req *container.ListRequest) (*c if !a.enabled.Load() { return res, err } - audit.LogRequest(a.log, container_grpc.ContainerService_List_FullMethodName, req, + audit.LogRequest(ctx, a.log, container_grpc.ContainerService_List_FullMethodName, req, audit.TargetFromRef(req.GetBody().GetOwnerID(), &user.ID{}), err == nil) return res, err } @@ -69,7 +69,7 @@ func (a *auditService) Put(ctx context.Context, req *container.PutRequest) (*con if !a.enabled.Load() { return res, err } - audit.LogRequest(a.log, container_grpc.ContainerService_Put_FullMethodName, req, + audit.LogRequest(ctx, a.log, container_grpc.ContainerService_Put_FullMethodName, req, audit.TargetFromRef(res.GetBody().GetContainerID(), &cid.ID{}), err == nil) return res, err } diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index eb43eab70..211f469f3 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -35,9 +35,9 @@ type Reader interface { // Writer is an interface of container storage updater. type Writer interface { // Put stores specified container in the side chain. - Put(containercore.Container) (*cid.ID, error) + Put(context.Context, containercore.Container) (*cid.ID, error) // Delete removes specified container from the side chain. - Delete(containercore.RemovalWitness) error + Delete(context.Context, containercore.RemovalWitness) error } func NewExecutor(rdr Reader, wrt Writer) containerSvc.ServiceExecutor { @@ -47,7 +47,7 @@ func NewExecutor(rdr Reader, wrt Writer) containerSvc.ServiceExecutor { } } -func (s *morphExecutor) Put(_ context.Context, tokV2 *sessionV2.Token, body *container.PutRequestBody) (*container.PutResponseBody, error) { +func (s *morphExecutor) Put(ctx context.Context, tokV2 *sessionV2.Token, body *container.PutRequestBody) (*container.PutResponseBody, error) { sigV2 := body.GetSignature() if sigV2 == nil { // TODO(@cthulhu-rider): #468 use "const" error @@ -80,7 +80,7 @@ func (s *morphExecutor) Put(_ context.Context, tokV2 *sessionV2.Token, body *con } } - idCnr, err := s.wrt.Put(cnr) + idCnr, err := s.wrt.Put(ctx, cnr) if err != nil { return nil, err } @@ -94,7 +94,7 @@ func (s *morphExecutor) Put(_ context.Context, tokV2 *sessionV2.Token, body *con return res, nil } -func (s *morphExecutor) Delete(_ context.Context, tokV2 *sessionV2.Token, body *container.DeleteRequestBody) (*container.DeleteResponseBody, error) { +func (s *morphExecutor) Delete(ctx context.Context, tokV2 *sessionV2.Token, body *container.DeleteRequestBody) (*container.DeleteResponseBody, error) { idV2 := body.GetContainerID() if idV2 == nil { return nil, errors.New("missing container ID") @@ -124,7 +124,7 @@ func (s *morphExecutor) Delete(_ context.Context, tokV2 *sessionV2.Token, body * rmWitness.Signature = body.GetSignature() rmWitness.SessionToken = tok - err = s.wrt.Delete(rmWitness) + err = s.wrt.Delete(ctx, rmWitness) if err != nil { return nil, err } diff --git a/pkg/services/container/morph/executor_test.go b/pkg/services/container/morph/executor_test.go index 87d307385..1f6fdb0be 100644 --- a/pkg/services/container/morph/executor_test.go +++ b/pkg/services/container/morph/executor_test.go @@ -24,11 +24,11 @@ type mock struct { containerSvcMorph.Reader } -func (m mock) Put(_ containerCore.Container) (*cid.ID, error) { +func (m mock) Put(_ context.Context, _ containerCore.Container) (*cid.ID, error) { return new(cid.ID), nil } -func (m mock) Delete(_ containerCore.RemovalWitness) error { +func (m mock) Delete(_ context.Context, _ containerCore.RemovalWitness) error { return nil } diff --git a/pkg/services/control/ir/server/audit.go b/pkg/services/control/ir/server/audit.go index e54fa9824..d9f65a2fc 100644 --- a/pkg/services/control/ir/server/audit.go +++ b/pkg/services/control/ir/server/audit.go @@ -36,7 +36,7 @@ func (a *auditService) HealthCheck(ctx context.Context, req *control.HealthCheck if !a.enabled.Load() { return res, err } - audit.LogRequestWithKey(a.log, control.ControlService_HealthCheck_FullMethodName, req.GetSignature().GetKey(), nil, err == nil) + audit.LogRequestWithKey(ctx, a.log, control.ControlService_HealthCheck_FullMethodName, req.GetSignature().GetKey(), nil, err == nil) return res, err } @@ -79,7 +79,7 @@ func (a *auditService) RemoveContainer(ctx context.Context, req *control.RemoveC } } - audit.LogRequestWithKey(a.log, control.ControlService_RemoveContainer_FullMethodName, req.GetSignature().GetKey(), sb, err == nil) + audit.LogRequestWithKey(ctx, a.log, control.ControlService_RemoveContainer_FullMethodName, req.GetSignature().GetKey(), sb, err == nil) return res, err } @@ -90,7 +90,7 @@ func (a *auditService) RemoveNode(ctx context.Context, req *control.RemoveNodeRe return res, err } - audit.LogRequestWithKey(a.log, control.ControlService_RemoveNode_FullMethodName, req.GetSignature().GetKey(), + audit.LogRequestWithKey(ctx, a.log, control.ControlService_RemoveNode_FullMethodName, req.GetSignature().GetKey(), audit.TargetFromString(hex.EncodeToString(req.GetBody().GetKey())), err == nil) return res, err } @@ -102,7 +102,7 @@ func (a *auditService) TickEpoch(ctx context.Context, req *control.TickEpochRequ return res, err } - audit.LogRequestWithKey(a.log, control.ControlService_TickEpoch_FullMethodName, req.GetSignature().GetKey(), + audit.LogRequestWithKey(ctx, a.log, control.ControlService_TickEpoch_FullMethodName, req.GetSignature().GetKey(), nil, err == nil) return res, err } diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 63be22411..e2c385c6a 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -40,7 +40,7 @@ func (s *Server) HealthCheck(_ context.Context, req *control.HealthCheckRequest) // TickEpoch forces a new epoch. // // If request is not signed with a key from white list, permission error returns. -func (s *Server) TickEpoch(_ context.Context, req *control.TickEpochRequest) (*control.TickEpochResponse, error) { +func (s *Server) TickEpoch(ctx context.Context, req *control.TickEpochRequest) (*control.TickEpochResponse, error) { if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) } @@ -53,7 +53,7 @@ func (s *Server) TickEpoch(_ context.Context, req *control.TickEpochRequest) (*c return nil, fmt.Errorf("getting current epoch: %w", err) } - vub, err := s.netmapClient.NewEpochControl(epoch+1, req.GetBody().GetVub()) + vub, err := s.netmapClient.NewEpochControl(ctx, epoch+1, req.GetBody().GetVub()) if err != nil { return nil, fmt.Errorf("forcing new epoch: %w", err) } @@ -69,7 +69,7 @@ func (s *Server) TickEpoch(_ context.Context, req *control.TickEpochRequest) (*c // RemoveNode forces a node removal. // // If request is not signed with a key from white list, permission error returns. -func (s *Server) RemoveNode(_ context.Context, req *control.RemoveNodeRequest) (*control.RemoveNodeResponse, error) { +func (s *Server) RemoveNode(ctx context.Context, req *control.RemoveNodeRequest) (*control.RemoveNodeResponse, error) { if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) } @@ -95,7 +95,7 @@ func (s *Server) RemoveNode(_ context.Context, req *control.RemoveNodeRequest) ( return nil, status.Error(codes.FailedPrecondition, "node is already offline") } - vub, err := s.netmapClient.ForceRemovePeer(nodeInfo, req.GetBody().GetVub()) + vub, err := s.netmapClient.ForceRemovePeer(ctx, nodeInfo, req.GetBody().GetVub()) if err != nil { return nil, fmt.Errorf("forcing node removal: %w", err) } @@ -109,7 +109,7 @@ func (s *Server) RemoveNode(_ context.Context, req *control.RemoveNodeRequest) ( } // RemoveContainer forces a container removal. -func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainerRequest) (*control.RemoveContainerResponse, error) { +func (s *Server) RemoveContainer(ctx context.Context, req *control.RemoveContainerRequest) (*control.RemoveContainerResponse, error) { if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) } @@ -124,7 +124,7 @@ func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainer return nil, status.Error(codes.InvalidArgument, "failed to parse container ID: "+err.Error()) } var err error - vub, err = s.removeContainer(containerID, req.GetBody().GetVub()) + vub, err = s.removeContainer(ctx, containerID, req.GetBody().GetVub()) if err != nil { return nil, err } @@ -144,7 +144,7 @@ func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainer } for _, containerID := range cids { - vub, err = s.removeContainer(containerID, req.GetBody().GetVub()) + vub, err = s.removeContainer(ctx, containerID, req.GetBody().GetVub()) if err != nil { return nil, err } @@ -162,13 +162,13 @@ func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainer return resp, nil } -func (s *Server) removeContainer(containerID cid.ID, vub uint32) (uint32, error) { +func (s *Server) removeContainer(ctx context.Context, containerID cid.ID, vub uint32) (uint32, error) { var prm container.DeletePrm prm.SetCID(containerID[:]) prm.SetControlTX(true) prm.SetVUB(vub) - vub, err := s.containerClient.Delete(prm) + vub, err := s.containerClient.Delete(ctx, prm) if err != nil { return 0, fmt.Errorf("forcing container removal: %w", err) } diff --git a/pkg/services/control/server/server.go b/pkg/services/control/server/server.go index b6fdcb246..94aa1ff5b 100644 --- a/pkg/services/control/server/server.go +++ b/pkg/services/control/server/server.go @@ -1,6 +1,7 @@ package control import ( + "context" "crypto/ecdsa" "sync/atomic" @@ -45,11 +46,11 @@ type NodeState interface { // // If status is control.NetmapStatus_MAINTENANCE and maintenance is allowed // in the network settings, the node additionally starts local maintenance. - SetNetmapStatus(st control.NetmapStatus) error + SetNetmapStatus(ctx context.Context, st control.NetmapStatus) error // ForceMaintenance works like SetNetmapStatus(control.NetmapStatus_MAINTENANCE) // but starts local maintenance regardless of the network settings. - ForceMaintenance() error + ForceMaintenance(ctx context.Context) error GetNetmapStatus() (control.NetmapStatus, uint64, error) } diff --git a/pkg/services/control/server/set_netmap_status.go b/pkg/services/control/server/set_netmap_status.go index 3fd69df12..529041dca 100644 --- a/pkg/services/control/server/set_netmap_status.go +++ b/pkg/services/control/server/set_netmap_status.go @@ -12,7 +12,7 @@ import ( // SetNetmapStatus sets node status in FrostFS network. // // If request is unsigned or signed by disallowed key, permission error returns. -func (s *Server) SetNetmapStatus(_ context.Context, req *control.SetNetmapStatusRequest) (*control.SetNetmapStatusResponse, error) { +func (s *Server) SetNetmapStatus(ctx context.Context, req *control.SetNetmapStatusRequest) (*control.SetNetmapStatusResponse, error) { // verify request if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) @@ -29,9 +29,9 @@ func (s *Server) SetNetmapStatus(_ context.Context, req *control.SetNetmapStatus "force_maintenance MUST be set for %s status only", control.NetmapStatus_MAINTENANCE) } - err = s.nodeState.ForceMaintenance() + err = s.nodeState.ForceMaintenance(ctx) } else { - err = s.nodeState.SetNetmapStatus(st) + err = s.nodeState.SetNetmapStatus(ctx, st) } if err != nil { diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 93ad3dc46..db0f13ee7 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -193,7 +193,7 @@ func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream src: request, } - reqInfo, err := b.findRequestInfo(req, cnr, acl.OpObjectGet) + reqInfo, err := b.findRequestInfo(stream.Context(), req, cnr, acl.OpObjectGet) if err != nil { return err } @@ -203,8 +203,8 @@ func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream return b.next.Get(request, newWrappedGetObjectStreamStream(stream, reqInfo)) } -func (b Service) Put() (object.PutObjectStream, error) { - streamer, err := b.next.Put() +func (b Service) Put(ctx context.Context) (object.PutObjectStream, error) { + streamer, err := b.next.Put(ctx) return putStreamBasicChecker{ source: &b, @@ -212,8 +212,8 @@ func (b Service) Put() (object.PutObjectStream, error) { }, err } -func (b Service) Patch() (object.PatchObjectStream, error) { - streamer, err := b.next.Patch() +func (b Service) Patch(ctx context.Context) (object.PatchObjectStream, error) { + streamer, err := b.next.Patch(ctx) return &patchStreamBasicChecker{ source: &b, @@ -259,7 +259,7 @@ func (b Service) Head( src: request, } - reqInfo, err := b.findRequestInfo(req, cnr, acl.OpObjectHead) + reqInfo, err := b.findRequestInfo(ctx, req, cnr, acl.OpObjectHead) if err != nil { return nil, err } @@ -299,7 +299,7 @@ func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStr src: request, } - reqInfo, err := b.findRequestInfo(req, id, acl.OpObjectSearch) + reqInfo, err := b.findRequestInfo(stream.Context(), req, id, acl.OpObjectSearch) if err != nil { return err } @@ -345,7 +345,7 @@ func (b Service) Delete( src: request, } - reqInfo, err := b.findRequestInfo(req, cnr, acl.OpObjectDelete) + reqInfo, err := b.findRequestInfo(ctx, req, cnr, acl.OpObjectDelete) if err != nil { return nil, err } @@ -390,7 +390,7 @@ func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetOb src: request, } - reqInfo, err := b.findRequestInfo(req, cnr, acl.OpObjectRange) + reqInfo, err := b.findRequestInfo(stream.Context(), req, cnr, acl.OpObjectRange) if err != nil { return err } @@ -448,7 +448,7 @@ func (b Service) GetRangeHash( src: request, } - reqInfo, err := b.findRequestInfo(req, cnr, acl.OpObjectHash) + reqInfo, err := b.findRequestInfo(ctx, req, cnr, acl.OpObjectHash) if err != nil { return nil, err } @@ -499,7 +499,7 @@ func (b Service) PutSingle(ctx context.Context, request *objectV2.PutSingleReque src: request, } - reqInfo, err := b.findRequestInfo(req, cnr, acl.OpObjectPut) + reqInfo, err := b.findRequestInfo(ctx, req, cnr, acl.OpObjectPut) if err != nil { return nil, err } @@ -564,7 +564,7 @@ func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRe src: request, } - reqInfo, err := p.source.findRequestInfo(req, cnr, acl.OpObjectPut) + reqInfo, err := p.source.findRequestInfo(ctx, req, cnr, acl.OpObjectPut) if err != nil { return err } @@ -651,7 +651,7 @@ func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.Pa src: request, } - reqInfo, err := p.source.findRequestInfoWithoutACLOperationAssert(req, cnr) + reqInfo, err := p.source.findRequestInfoWithoutACLOperationAssert(ctx, req, cnr) if err != nil { return err } @@ -668,7 +668,7 @@ func (p patchStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.Pa return p.next.CloseAndRecv(ctx) } -func (b Service) findRequestInfo(req MetaWithToken, idCnr cid.ID, op acl.Op) (info RequestInfo, err error) { +func (b Service) findRequestInfo(ctx context.Context, req MetaWithToken, idCnr cid.ID, op acl.Op) (info RequestInfo, err error) { cnr, err := b.containers.Get(idCnr) // fetch actual container if err != nil { return info, err @@ -697,7 +697,7 @@ func (b Service) findRequestInfo(req MetaWithToken, idCnr cid.ID, op acl.Op) (in if err != nil { return info, err } - res, err := b.c.Classify(ownerID, ownerKey, idCnr, cnr.Value) + res, err := b.c.Classify(ctx, ownerID, ownerKey, idCnr, cnr.Value) if err != nil { return info, err } @@ -726,7 +726,7 @@ func (b Service) findRequestInfo(req MetaWithToken, idCnr cid.ID, op acl.Op) (in } // findRequestInfoWithoutACLOperationAssert is findRequestInfo without session token verb assert. -func (b Service) findRequestInfoWithoutACLOperationAssert(req MetaWithToken, idCnr cid.ID) (info RequestInfo, err error) { +func (b Service) findRequestInfoWithoutACLOperationAssert(ctx context.Context, req MetaWithToken, idCnr cid.ID) (info RequestInfo, err error) { cnr, err := b.containers.Get(idCnr) // fetch actual container if err != nil { return info, err @@ -751,7 +751,7 @@ func (b Service) findRequestInfoWithoutACLOperationAssert(req MetaWithToken, idC if err != nil { return info, err } - res, err := b.c.Classify(ownerID, ownerKey, idCnr, cnr.Value) + res, err := b.c.Classify(ctx, ownerID, ownerKey, idCnr, cnr.Value) if err != nil { return info, err } diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index 558c48da8..c6d152e0f 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -186,8 +186,8 @@ func (p putStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PutR return p.next.CloseAndRecv(ctx) } -func (c *Service) Put() (objectSvc.PutObjectStream, error) { - streamer, err := c.next.Put() +func (c *Service) Put(ctx context.Context) (objectSvc.PutObjectStream, error) { + streamer, err := c.next.Put(ctx) return &putStreamBasicChecker{ apeChecker: c.apeChecker, @@ -241,8 +241,8 @@ func (p patchStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.Pa return p.next.CloseAndRecv(ctx) } -func (c *Service) Patch() (objectSvc.PatchObjectStream, error) { - streamer, err := c.next.Patch() +func (c *Service) Patch(ctx context.Context) (objectSvc.PatchObjectStream, error) { + streamer, err := c.next.Patch(ctx) return &patchStreamBasicChecker{ apeChecker: c.apeChecker, diff --git a/pkg/services/object/audit.go b/pkg/services/object/audit.go index b42084634..dde9f8fc0 100644 --- a/pkg/services/object/audit.go +++ b/pkg/services/object/audit.go @@ -37,7 +37,7 @@ func (a *auditService) Delete(ctx context.Context, req *object.DeleteRequest) (* if !a.enabled.Load() { return res, err } - audit.LogRequest(a.log, objectGRPC.ObjectService_Delete_FullMethodName, req, + audit.LogRequest(ctx, a.log, objectGRPC.ObjectService_Delete_FullMethodName, req, audit.TargetFromRef(req.GetBody().GetAddress(), &oid.Address{}), err == nil) return res, err } @@ -48,7 +48,7 @@ func (a *auditService) Get(req *object.GetRequest, stream GetObjectStream) error if !a.enabled.Load() { return err } - audit.LogRequest(a.log, objectGRPC.ObjectService_Get_FullMethodName, req, + audit.LogRequest(stream.Context(), a.log, objectGRPC.ObjectService_Get_FullMethodName, req, audit.TargetFromRef(req.GetBody().GetAddress(), &oid.Address{}), err == nil) return err } @@ -59,7 +59,7 @@ func (a *auditService) GetRange(req *object.GetRangeRequest, stream GetObjectRan if !a.enabled.Load() { return err } - audit.LogRequest(a.log, objectGRPC.ObjectService_GetRange_FullMethodName, req, + audit.LogRequest(stream.Context(), a.log, objectGRPC.ObjectService_GetRange_FullMethodName, req, audit.TargetFromRef(req.GetBody().GetAddress(), &oid.Address{}), err == nil) return err } @@ -70,7 +70,7 @@ func (a *auditService) GetRangeHash(ctx context.Context, req *object.GetRangeHas if !a.enabled.Load() { return resp, err } - audit.LogRequest(a.log, objectGRPC.ObjectService_GetRangeHash_FullMethodName, req, + audit.LogRequest(ctx, a.log, objectGRPC.ObjectService_GetRangeHash_FullMethodName, req, audit.TargetFromRef(req.GetBody().GetAddress(), &oid.Address{}), err == nil) return resp, err } @@ -81,19 +81,19 @@ func (a *auditService) Head(ctx context.Context, req *object.HeadRequest) (*obje if !a.enabled.Load() { return resp, err } - audit.LogRequest(a.log, objectGRPC.ObjectService_Head_FullMethodName, req, + audit.LogRequest(ctx, a.log, objectGRPC.ObjectService_Head_FullMethodName, req, audit.TargetFromRef(req.GetBody().GetAddress(), &oid.Address{}), err == nil) return resp, err } // Put implements ServiceServer. -func (a *auditService) Put() (PutObjectStream, error) { - res, err := a.next.Put() +func (a *auditService) Put(ctx context.Context) (PutObjectStream, error) { + res, err := a.next.Put(ctx) if !a.enabled.Load() { return res, err } if err != nil { - audit.LogRequest(a.log, objectGRPC.ObjectService_Put_FullMethodName, nil, nil, false) + audit.LogRequest(ctx, a.log, objectGRPC.ObjectService_Put_FullMethodName, nil, nil, false) return res, err } return &auditPutStream{ @@ -108,7 +108,7 @@ func (a *auditService) PutSingle(ctx context.Context, req *object.PutSingleReque if !a.enabled.Load() { return resp, err } - audit.LogRequest(a.log, objectGRPC.ObjectService_PutSingle_FullMethodName, req, + audit.LogRequest(ctx, a.log, objectGRPC.ObjectService_PutSingle_FullMethodName, req, audit.TargetFromContainerIDObjectID(req.GetBody().GetObject().GetHeader().GetContainerID(), req.GetBody().GetObject().GetObjectID()), err == nil) @@ -121,7 +121,7 @@ func (a *auditService) Search(req *object.SearchRequest, stream SearchStream) er if !a.enabled.Load() { return err } - audit.LogRequest(a.log, objectGRPC.ObjectService_Search_FullMethodName, req, + audit.LogRequest(stream.Context(), a.log, objectGRPC.ObjectService_Search_FullMethodName, req, audit.TargetFromRef(req.GetBody().GetContainerID(), &cid.ID{}), err == nil) return err } @@ -145,7 +145,7 @@ func (a *auditPutStream) CloseAndRecv(ctx context.Context) (*object.PutResponse, a.failed = true } a.objectID = resp.GetBody().GetObjectID() - audit.LogRequestWithKey(a.log, objectGRPC.ObjectService_Put_FullMethodName, a.key, + audit.LogRequestWithKey(ctx, a.log, objectGRPC.ObjectService_Put_FullMethodName, a.key, audit.TargetFromContainerIDObjectID(a.containerID, a.objectID), !a.failed) return resp, err @@ -164,7 +164,7 @@ func (a *auditPutStream) Send(ctx context.Context, req *object.PutRequest) error a.failed = true } if !errors.Is(err, util.ErrAbortStream) { // CloseAndRecv will not be called, so log here - audit.LogRequestWithKey(a.log, objectGRPC.ObjectService_Put_FullMethodName, a.key, + audit.LogRequestWithKey(ctx, a.log, objectGRPC.ObjectService_Put_FullMethodName, a.key, audit.TargetFromContainerIDObjectID(a.containerID, a.objectID), !a.failed) } @@ -183,13 +183,13 @@ type auditPatchStream struct { nonFirstSend bool } -func (a *auditService) Patch() (PatchObjectStream, error) { - res, err := a.next.Patch() +func (a *auditService) Patch(ctx context.Context) (PatchObjectStream, error) { + res, err := a.next.Patch(ctx) if !a.enabled.Load() { return res, err } if err != nil { - audit.LogRequest(a.log, objectGRPC.ObjectService_Patch_FullMethodName, nil, nil, false) + audit.LogRequest(ctx, a.log, objectGRPC.ObjectService_Patch_FullMethodName, nil, nil, false) return res, err } return &auditPatchStream{ @@ -205,7 +205,7 @@ func (a *auditPatchStream) CloseAndRecv(ctx context.Context) (*object.PatchRespo a.failed = true } a.objectID = resp.GetBody().GetObjectID() - audit.LogRequestWithKey(a.log, objectGRPC.ObjectService_Patch_FullMethodName, a.key, + audit.LogRequestWithKey(ctx, a.log, objectGRPC.ObjectService_Patch_FullMethodName, a.key, audit.TargetFromContainerIDObjectID(a.containerID, a.objectID), !a.failed) return resp, err @@ -225,7 +225,7 @@ func (a *auditPatchStream) Send(ctx context.Context, req *object.PatchRequest) e a.failed = true } if !errors.Is(err, util.ErrAbortStream) { // CloseAndRecv will not be called, so log here - audit.LogRequestWithKey(a.log, objectGRPC.ObjectService_Patch_FullMethodName, a.key, + audit.LogRequestWithKey(ctx, a.log, objectGRPC.ObjectService_Patch_FullMethodName, a.key, audit.TargetFromContainerIDObjectID(a.containerID, a.objectID), !a.failed) } diff --git a/pkg/services/object/common.go b/pkg/services/object/common.go index 758156607..ef65e78bc 100644 --- a/pkg/services/object/common.go +++ b/pkg/services/object/common.go @@ -40,20 +40,20 @@ func (x *Common) Get(req *objectV2.GetRequest, stream GetObjectStream) error { return x.nextHandler.Get(req, stream) } -func (x *Common) Put() (PutObjectStream, error) { +func (x *Common) Put(ctx context.Context) (PutObjectStream, error) { if x.state.IsMaintenance() { return nil, new(apistatus.NodeUnderMaintenance) } - return x.nextHandler.Put() + return x.nextHandler.Put(ctx) } -func (x *Common) Patch() (PatchObjectStream, error) { +func (x *Common) Patch(ctx context.Context) (PatchObjectStream, error) { if x.state.IsMaintenance() { return nil, new(apistatus.NodeUnderMaintenance) } - return x.nextHandler.Patch() + return x.nextHandler.Patch(ctx) } func (x *Common) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { diff --git a/pkg/services/object/common/writer/common.go b/pkg/services/object/common/writer/common.go index 8b92d34ed..3b68efab4 100644 --- a/pkg/services/object/common/writer/common.go +++ b/pkg/services/object/common/writer/common.go @@ -89,7 +89,7 @@ func (n *NodeIterator) forEachAddress(ctx context.Context, traverser *placement. err := f(ctx, NodeDescriptor{Local: isLocal, Info: addr}) if err != nil { resErr.Store(err) - svcutil.LogServiceError(n.cfg.Logger, "PUT", addr.Addresses(), err) + svcutil.LogServiceError(ctx, n.cfg.Logger, "PUT", addr.Addresses(), err) return } @@ -97,7 +97,7 @@ func (n *NodeIterator) forEachAddress(ctx context.Context, traverser *placement. *item = true }); err != nil { wg.Done() - svcutil.LogWorkerPoolError(n.cfg.Logger, "PUT", err) + svcutil.LogWorkerPoolError(ctx, n.cfg.Logger, "PUT", err) return true } diff --git a/pkg/services/object/common/writer/ec.go b/pkg/services/object/common/writer/ec.go index 64115b86b..fdaa569da 100644 --- a/pkg/services/object/common/writer/ec.go +++ b/pkg/services/object/common/writer/ec.go @@ -154,7 +154,7 @@ func (e *ECWriter) relayToContainerNode(ctx context.Context, objID oid.ID, index err = e.Relay(ctx, info, c) }); poolErr != nil { close(completed) - svcutil.LogWorkerPoolError(e.Config.Logger, "PUT", poolErr) + svcutil.LogWorkerPoolError(ctx, e.Config.Logger, "PUT", poolErr) return poolErr } <-completed diff --git a/pkg/services/object/get/assembleec.go b/pkg/services/object/get/assembleec.go index 8ab423c87..59dd7fd93 100644 --- a/pkg/services/object/get/assembleec.go +++ b/pkg/services/object/get/assembleec.go @@ -37,7 +37,7 @@ func (r *request) assembleEC(ctx context.Context) { r.log.Debug(ctx, logs.GetTryingToAssembleTheECObject) // initialize epoch number - ok := r.initEpoch() + ok := r.initEpoch(ctx) if !ok { return } diff --git a/pkg/services/object/get/container.go b/pkg/services/object/get/container.go index 2b84c5b32..0ee8aed53 100644 --- a/pkg/services/object/get/container.go +++ b/pkg/services/object/get/container.go @@ -21,7 +21,7 @@ func (r *request) executeOnContainer(ctx context.Context) { ) // initialize epoch number - ok := r.initEpoch() + ok := r.initEpoch(ctx) if !ok { return } @@ -50,7 +50,7 @@ func (r *request) processCurrentEpoch(ctx context.Context, localStatus int) bool zap.Uint64("number", r.curProcEpoch), ) - traverser, ok := r.generateTraverser(r.address()) + traverser, ok := r.generateTraverser(ctx, r.address()) if !ok { return true } diff --git a/pkg/services/object/get/remote.go b/pkg/services/object/get/remote.go index b6a83fd0c..78ca5b5e3 100644 --- a/pkg/services/object/get/remote.go +++ b/pkg/services/object/get/remote.go @@ -20,7 +20,7 @@ func (r *request) processNode(ctx context.Context, info client.NodeInfo) bool { r.log.Debug(ctx, logs.ProcessingNode, zap.String("node_key", hex.EncodeToString(info.PublicKey()))) - rs, ok := r.getRemoteStorage(info) + rs, ok := r.getRemoteStorage(ctx, info) if !ok { return true } diff --git a/pkg/services/object/get/request.go b/pkg/services/object/get/request.go index bba767d2d..be0950c60 100644 --- a/pkg/services/object/get/request.go +++ b/pkg/services/object/get/request.go @@ -116,7 +116,7 @@ func (r *request) netmapLookupDepth() uint64 { return r.prm.common.NetmapLookupDepth() } -func (r *request) initEpoch() bool { +func (r *request) initEpoch(ctx context.Context) bool { r.curProcEpoch = r.netmapEpoch() if r.curProcEpoch > 0 { return true @@ -129,7 +129,7 @@ func (r *request) initEpoch() bool { r.status = statusUndefined r.err = err - r.log.Debug(context.Background(), logs.CouldNotGetCurrentEpochNumber, zap.Error(err)) + r.log.Debug(ctx, logs.CouldNotGetCurrentEpochNumber, zap.Error(err)) return false case err == nil: @@ -138,7 +138,7 @@ func (r *request) initEpoch() bool { } } -func (r *request) generateTraverser(addr oid.Address) (*placement.Traverser, bool) { +func (r *request) generateTraverser(ctx context.Context, addr oid.Address) (*placement.Traverser, bool) { obj := addr.Object() t, _, err := r.traverserGenerator.GenerateTraverser(addr.Container(), &obj, r.curProcEpoch) @@ -148,7 +148,7 @@ func (r *request) generateTraverser(addr oid.Address) (*placement.Traverser, boo r.status = statusUndefined r.err = err - r.log.Debug(context.Background(), logs.GetCouldNotGenerateContainerTraverser, zap.Error(err)) + r.log.Debug(ctx, logs.GetCouldNotGenerateContainerTraverser, zap.Error(err)) return nil, false case err == nil: @@ -156,13 +156,13 @@ func (r *request) generateTraverser(addr oid.Address) (*placement.Traverser, boo } } -func (r *request) getRemoteStorage(info clientcore.NodeInfo) (remoteStorage, bool) { +func (r *request) getRemoteStorage(ctx context.Context, info clientcore.NodeInfo) (remoteStorage, bool) { rs, err := r.remoteStorageConstructor.Get(info) if err != nil { r.status = statusUndefined r.err = err - r.log.Debug(context.Background(), logs.GetCouldNotConstructRemoteNodeClient) + r.log.Debug(ctx, logs.GetCouldNotConstructRemoteNodeClient) return nil, false } diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index 377350fdd..19748e938 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -64,11 +64,11 @@ func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (er return } -func (m MetricCollector) Put() (PutObjectStream, error) { +func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) { if m.enabled { t := time.Now() - stream, err := m.next.Put() + stream, err := m.next.Put(ctx) if err != nil { return nil, err } @@ -79,14 +79,14 @@ func (m MetricCollector) Put() (PutObjectStream, error) { start: t, }, nil } - return m.next.Put() + return m.next.Put(ctx) } -func (m MetricCollector) Patch() (PatchObjectStream, error) { +func (m MetricCollector) Patch(ctx context.Context) (PatchObjectStream, error) { if m.enabled { t := time.Now() - stream, err := m.next.Patch() + stream, err := m.next.Patch(ctx) if err != nil { return nil, err } @@ -97,7 +97,7 @@ func (m MetricCollector) Patch() (PatchObjectStream, error) { start: t, }, nil } - return m.next.Patch() + return m.next.Patch(ctx) } func (m MetricCollector) PutSingle(ctx context.Context, request *object.PutSingleRequest) (*object.PutSingleResponse, error) { diff --git a/pkg/services/object/response.go b/pkg/services/object/response.go index 3787b4168..80c971e8f 100644 --- a/pkg/services/object/response.go +++ b/pkg/services/object/response.go @@ -80,8 +80,8 @@ func (s *putStreamResponser) CloseAndRecv(ctx context.Context) (*object.PutRespo return r, nil } -func (s *ResponseService) Put() (PutObjectStream, error) { - stream, err := s.svc.Put() +func (s *ResponseService) Put(ctx context.Context) (PutObjectStream, error) { + stream, err := s.svc.Put(ctx) if err != nil { return nil, fmt.Errorf("could not create Put object streamer: %w", err) } @@ -109,8 +109,8 @@ func (s *patchStreamResponser) CloseAndRecv(ctx context.Context) (*object.PatchR return r, nil } -func (s *ResponseService) Patch() (PatchObjectStream, error) { - stream, err := s.svc.Patch() +func (s *ResponseService) Patch(ctx context.Context) (PatchObjectStream, error) { + stream, err := s.svc.Patch(ctx) if err != nil { return nil, fmt.Errorf("could not create Put object streamer: %w", err) } diff --git a/pkg/services/object/search/search.go b/pkg/services/object/search/search.go index bb5c720ff..e24da975d 100644 --- a/pkg/services/object/search/search.go +++ b/pkg/services/object/search/search.go @@ -23,7 +23,7 @@ func (exec *execCtx) execute(ctx context.Context) error { exec.log.Debug(ctx, logs.ServingRequest) err := exec.executeLocal(ctx) - exec.logResult(err) + exec.logResult(ctx, err) if exec.isLocal() { exec.log.Debug(ctx, logs.SearchReturnResultDirectly) @@ -31,15 +31,15 @@ func (exec *execCtx) execute(ctx context.Context) error { } err = exec.executeOnContainer(ctx) - exec.logResult(err) + exec.logResult(ctx, err) return err } -func (exec *execCtx) logResult(err error) { +func (exec *execCtx) logResult(ctx context.Context, err error) { switch { default: - exec.log.Debug(context.Background(), logs.OperationFinishedWithError, zap.String("error", err.Error())) + exec.log.Debug(ctx, logs.OperationFinishedWithError, zap.String("error", err.Error())) case err == nil: - exec.log.Debug(context.Background(), logs.OperationFinishedSuccessfully) + exec.log.Debug(ctx, logs.OperationFinishedSuccessfully) } } diff --git a/pkg/services/object/server.go b/pkg/services/object/server.go index c570e9d8e..e65293977 100644 --- a/pkg/services/object/server.go +++ b/pkg/services/object/server.go @@ -41,8 +41,8 @@ type PatchObjectStream interface { // serving v2 Object service. type ServiceServer interface { Get(*object.GetRequest, GetObjectStream) error - Put() (PutObjectStream, error) - Patch() (PatchObjectStream, error) + Put(context.Context) (PutObjectStream, error) + Patch(context.Context) (PatchObjectStream, error) Head(context.Context, *object.HeadRequest) (*object.HeadResponse, error) Search(*object.SearchRequest, SearchStream) error Delete(context.Context, *object.DeleteRequest) (*object.DeleteResponse, error) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 2c5e794e9..2b44227a5 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -103,8 +103,8 @@ func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.PutRes return resp, s.sigSvc.SignResponse(resp, err) } -func (s *SignService) Put() (PutObjectStream, error) { - stream, err := s.svc.Put() +func (s *SignService) Put(ctx context.Context) (PutObjectStream, error) { + stream, err := s.svc.Put(ctx) if err != nil { return nil, fmt.Errorf("could not create Put object streamer: %w", err) } @@ -139,8 +139,8 @@ func (s *patchStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.Patc return resp, s.sigSvc.SignResponse(resp, err) } -func (s *SignService) Patch() (PatchObjectStream, error) { - stream, err := s.svc.Patch() +func (s *SignService) Patch(ctx context.Context) (PatchObjectStream, error) { + stream, err := s.svc.Patch(ctx) if err != nil { return nil, fmt.Errorf("could not create Put object streamer: %w", err) } diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index 1438a0ea2..0b3676edb 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -87,12 +87,12 @@ func (c *TransportSplitter) Get(req *object.GetRequest, stream GetObjectStream) }) } -func (c TransportSplitter) Put() (PutObjectStream, error) { - return c.next.Put() +func (c TransportSplitter) Put(ctx context.Context) (PutObjectStream, error) { + return c.next.Put(ctx) } -func (c TransportSplitter) Patch() (PatchObjectStream, error) { - return c.next.Patch() +func (c TransportSplitter) Patch(ctx context.Context) (PatchObjectStream, error) { + return c.next.Patch(ctx) } func (c TransportSplitter) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) { diff --git a/pkg/services/object/util/log.go b/pkg/services/object/util/log.go index 5075344a4..a9f875d8d 100644 --- a/pkg/services/object/util/log.go +++ b/pkg/services/object/util/log.go @@ -10,8 +10,8 @@ import ( ) // LogServiceError writes error message of object service to provided logger. -func LogServiceError(l *logger.Logger, req string, node network.AddressGroup, err error) { - l.Error(context.Background(), logs.UtilObjectServiceError, +func LogServiceError(ctx context.Context, l *logger.Logger, req string, node network.AddressGroup, err error) { + l.Error(ctx, logs.UtilObjectServiceError, zap.String("node", network.StringifyGroup(node)), zap.String("request", req), zap.String("error", err.Error()), @@ -19,8 +19,8 @@ func LogServiceError(l *logger.Logger, req string, node network.AddressGroup, er } // LogWorkerPoolError writes debug error message of object worker pool to provided logger. -func LogWorkerPoolError(l *logger.Logger, req string, err error) { - l.Error(context.Background(), logs.UtilCouldNotPushTaskToWorkerPool, +func LogWorkerPoolError(ctx context.Context, l *logger.Logger, req string, err error) { + l.Error(ctx, logs.UtilCouldNotPushTaskToWorkerPool, zap.String("request", req), zap.String("error", err.Error()), ) diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go index 6a9706b9e..a4e36c2dc 100644 --- a/pkg/services/object_manager/tombstone/checker.go +++ b/pkg/services/object_manager/tombstone/checker.go @@ -63,7 +63,7 @@ func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Addr ) } else { if ts != nil { - return g.handleTS(addrStr, ts, epoch) + return g.handleTS(ctx, addrStr, ts, epoch) } } @@ -72,12 +72,12 @@ func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Addr return false } -func (g *ExpirationChecker) handleTS(addr string, ts *objectSDK.Object, reqEpoch uint64) bool { +func (g *ExpirationChecker) handleTS(ctx context.Context, addr string, ts *objectSDK.Object, reqEpoch uint64) bool { for _, atr := range ts.Attributes() { if atr.Key() == objectV2.SysAttributeExpEpoch { epoch, err := strconv.ParseUint(atr.Value(), 10, 64) if err != nil { - g.log.Warn(context.Background(), + g.log.Warn(ctx, logs.TombstoneExpirationParseFailure, zap.Error(err), ) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index c82680a1e..2e5e54dfd 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -122,7 +122,7 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe shortage-- } else if nodes[i].Status().IsMaintenance() { - shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) + shortage, uncheckedCopies = p.handleMaintenance(ctx, nodes[i], checkedNodes, shortage, uncheckedCopies) } else { if status := checkedNodes.processStatus(nodes[i]); status.Processed() { if status == nodeHoldsObject { @@ -149,7 +149,7 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe checkedNodes.submitReplicaCandidate(nodes[i]) continue } else if client.IsErrNodeUnderMaintenance(err) { - shortage, uncheckedCopies = p.handleMaintenance(nodes[i], checkedNodes, shortage, uncheckedCopies) + shortage, uncheckedCopies = p.handleMaintenance(ctx, nodes[i], checkedNodes, shortage, uncheckedCopies) } else { p.log.Error(ctx, logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, zap.Stringer("object", addr), @@ -173,12 +173,12 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe // prevent spam with new replicas. // However, additional copies should not be removed in this case, // because we can remove the only copy this way. -func (p *Policer) handleMaintenance(node netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { +func (p *Policer) handleMaintenance(ctx context.Context, node netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { checkedNodes.submitReplicaHolder(node) shortage-- uncheckedCopies++ - p.log.Debug(context.Background(), logs.PolicerConsiderNodeUnderMaintenanceAsOK, + p.log.Debug(ctx, logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(node)), ) return shortage, uncheckedCopies diff --git a/pkg/services/tree/getsubtree_test.go b/pkg/services/tree/getsubtree_test.go index 95bdda34b..e7a13827e 100644 --- a/pkg/services/tree/getsubtree_test.go +++ b/pkg/services/tree/getsubtree_test.go @@ -131,7 +131,7 @@ func TestGetSubTreeOrderAsc(t *testing.T) { t.Run("boltdb forest", func(t *testing.T) { p := pilorama.NewBoltForest(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))) require.NoError(t, p.Open(context.Background(), 0o644)) - require.NoError(t, p.Init()) + require.NoError(t, p.Init(context.Background())) testGetSubTreeOrderAsc(t, p) }) } diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index e2249c9fb..c48a312fb 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -388,7 +388,7 @@ func (s *Service) syncLoop(ctx context.Context) { break } - newMap, cnrsToSync := s.containersToSync(cnrs) + newMap, cnrsToSync := s.containersToSync(ctx, cnrs) s.syncContainers(ctx, cnrsToSync) @@ -475,14 +475,14 @@ func (s *Service) removeContainers(ctx context.Context, newContainers map[cid.ID } } -func (s *Service) containersToSync(cnrs []cid.ID) (map[cid.ID]struct{}, []cid.ID) { +func (s *Service) containersToSync(ctx context.Context, cnrs []cid.ID) (map[cid.ID]struct{}, []cid.ID) { newMap := make(map[cid.ID]struct{}, len(s.cnrMap)) cnrsToSync := make([]cid.ID, 0, len(cnrs)) for _, cnr := range cnrs { _, pos, err := s.getContainerNodes(cnr) if err != nil { - s.log.Error(context.Background(), logs.TreeCouldNotCalculateContainerNodes, + s.log.Error(ctx, logs.TreeCouldNotCalculateContainerNodes, zap.Stringer("cid", cnr), zap.Error(err)) continue diff --git a/pkg/util/http/calls.go b/pkg/util/http/calls.go index a9877e007..8569ec734 100644 --- a/pkg/util/http/calls.go +++ b/pkg/util/http/calls.go @@ -32,8 +32,8 @@ func (x *Server) Serve() error { // // Once Shutdown has been called on a server, it may not be reused; // future calls to Serve method will have no effect. -func (x *Server) Shutdown() error { - ctx, cancel := context.WithTimeout(context.Background(), x.shutdownTimeout) +func (x *Server) Shutdown(ctx context.Context) error { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), x.shutdownTimeout) err := x.srv.Shutdown(ctx) diff --git a/scripts/populate-metabase/main.go b/scripts/populate-metabase/main.go index 6f6b233cf..8c4ea41ad 100644 --- a/scripts/populate-metabase/main.go +++ b/scripts/populate-metabase/main.go @@ -91,15 +91,15 @@ func populate() (err error) { return fmt.Errorf("couldn't open the metabase: %w", err) } defer func() { - if errOnClose := db.Close(); errOnClose != nil { + if errOnClose := db.Close(ctx); errOnClose != nil { err = errors.Join( err, - fmt.Errorf("couldn't close the metabase: %w", db.Close()), + fmt.Errorf("couldn't close the metabase: %w", db.Close(ctx)), ) } }() - if err = db.Init(); err != nil { + if err = db.Init(ctx); err != nil { return fmt.Errorf("couldn't init the metabase: %w", err) } From 612b34d5708c23f888f75f6b6b1e5e87efdeede1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 7 Nov 2024 14:37:42 +0300 Subject: [PATCH 1535/1943] [#1437] logger: Add caller skip to log original caller position Signed-off-by: Dmitrii Stepanov --- pkg/util/logger/logger.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index b3a1b9b94..19d3f1ed1 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -131,6 +131,7 @@ func newConsoleLogger(prm *Prm) (*Logger, error) { lZap, err := c.Build( zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), + zap.AddCallerSkip(1), ) if err != nil { return nil, err @@ -167,7 +168,7 @@ func newJournaldLogger(prm *Prm) (*Logger, error) { zapjournald.SyslogPid(), }) - lZap := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) + lZap := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), zap.AddCallerSkip(1)) l := &Logger{z: lZap, lvl: lvl} prm._log = l @@ -190,6 +191,6 @@ func (l *Logger) With(fields ...zap.Field) *Logger { func NewLoggerWrapper(z *zap.Logger) *Logger { return &Logger{ - z: z, + z: z.WithOptions(zap.AddCallerSkip(1)), } } From c6066d6ee4da0e1ac193ad986ca2b8c470f070f7 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Tue, 12 Nov 2024 17:15:23 +0300 Subject: [PATCH 1536/1943] [#1491] engine/test: Use more suitable testing utils here and there Use `setShardsNum` instead of `setInitializedShards` wherever possible. Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/engine/delete_test.go | 6 +----- pkg/local_object_storage/engine/engine_test.go | 7 +------ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 0904c9820..e095e4bbd 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -49,11 +49,7 @@ func TestDeleteBigObject(t *testing.T) { link.SetSplitID(splitID) link.SetChildren(childIDs...) - s1 := testNewShard(t) - s2 := testNewShard(t) - s3 := testNewShard(t) - - e := testNewEngine(t).setInitializedShards(t, s1, s2, s3).engine + e := testNewEngine(t).setShardsNum(t, 3).engine e.log = test.NewLogger(t) defer e.Close(context.Background()) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 88c523b76..44bda2cbc 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -43,12 +43,7 @@ func BenchmarkExists(b *testing.B) { } func benchmarkExists(b *testing.B, shardNum int) { - shards := make([]*shard.Shard, shardNum) - for i := range shardNum { - shards[i] = testNewShard(b) - } - - e := testNewEngine(b).setInitializedShards(b, shards...).engine + e := testNewEngine(b).setShardsNum(b, shardNum).engine defer func() { require.NoError(b, e.Close(context.Background())) }() addr := oidtest.Address() From 7ef36749d0c17d7c4f1da0e684f3db7e7ab69e61 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 13 Nov 2024 11:59:42 +0300 Subject: [PATCH 1537/1943] [#1491] engine/test: Move `BenchmarkExists` to `exists_test.go` Move `BenchmarkExists` from `engine_test.go` to `exists_test.go` for better organization and clarity. Signed-off-by: Aleksey Savchuk --- .../engine/engine_test.go | 44 ---------------- .../engine/exists_test.go | 51 +++++++++++++++++++ 2 files changed, 51 insertions(+), 44 deletions(-) create mode 100644 pkg/local_object_storage/engine/exists_test.go diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 44bda2cbc..bac35917c 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -10,17 +10,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - "git.frostfs.info/TrueCloudLab/hrw" - "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" ) @@ -30,44 +24,6 @@ func (s epochState) CurrentEpoch() uint64 { return 0 } -func BenchmarkExists(b *testing.B) { - b.Run("2 shards", func(b *testing.B) { - benchmarkExists(b, 2) - }) - b.Run("4 shards", func(b *testing.B) { - benchmarkExists(b, 4) - }) - b.Run("8 shards", func(b *testing.B) { - benchmarkExists(b, 8) - }) -} - -func benchmarkExists(b *testing.B, shardNum int) { - e := testNewEngine(b).setShardsNum(b, shardNum).engine - defer func() { require.NoError(b, e.Close(context.Background())) }() - - addr := oidtest.Address() - for range 100 { - obj := testutil.GenerateObjectWithCID(cidtest.ID()) - err := Put(context.Background(), e, obj, false) - if err != nil { - b.Fatal(err) - } - } - - b.ReportAllocs() - b.ResetTimer() - for range b.N { - var shPrm shard.ExistsPrm - shPrm.Address = addr - shPrm.ParentAddress = oid.Address{} - ok, _, err := e.exists(context.Background(), shPrm) - if err != nil || ok { - b.Fatalf("%t %v", ok, err) - } - } -} - type testEngineWrapper struct { engine *StorageEngine shardIDs []*shard.ID diff --git a/pkg/local_object_storage/engine/exists_test.go b/pkg/local_object_storage/engine/exists_test.go new file mode 100644 index 000000000..e2e5ff13e --- /dev/null +++ b/pkg/local_object_storage/engine/exists_test.go @@ -0,0 +1,51 @@ +package engine + +import ( + "context" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +func BenchmarkExists(b *testing.B) { + b.Run("2 shards", func(b *testing.B) { + benchmarkExists(b, 2) + }) + b.Run("4 shards", func(b *testing.B) { + benchmarkExists(b, 4) + }) + b.Run("8 shards", func(b *testing.B) { + benchmarkExists(b, 8) + }) +} + +func benchmarkExists(b *testing.B, shardNum int) { + e := testNewEngine(b).setShardsNum(b, shardNum).engine + defer func() { require.NoError(b, e.Close(context.Background())) }() + + addr := oidtest.Address() + for range 100 { + obj := testutil.GenerateObjectWithCID(cidtest.ID()) + err := Put(context.Background(), e, obj, false) + if err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + for range b.N { + var shPrm shard.ExistsPrm + shPrm.Address = addr + shPrm.ParentAddress = oid.Address{} + ok, _, err := e.exists(context.Background(), shPrm) + if err != nil || ok { + b.Fatalf("%t %v", ok, err) + } + } +} From 7fc6101bec418f6a5540a1ecf3626be8251d2696 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 13 Nov 2024 13:30:16 +0300 Subject: [PATCH 1538/1943] [#1491] engine/test: Rework engine test utils - Remove `testNewShard` and `setInitializedShards` because they violated the default engine workflow. The correct workflow is: first use `New()`, followed by `Open()`, and then `Init()`. As a result, adding new logic to `(*StorageEngine).Init` caused several tests to fail with a panic when attempting to access uninitialized resources. Now, all engines created with the test utils must be initialized manually. The new helper method `prepare` can be used for that purpose. - Additionally, `setInitializedShards` hardcoded the shard worker pool size, which prevented it from being configured in tests and benchmarks. This has been fixed as well. - Ensure engine initialization is done wherever it was missing. - Refactor `setShardsNumOpts`, `setShardsNumAdditionalOpts`, and `setShardsNum`. Make them all depend on `setShardsNumOpts`. Signed-off-by: Aleksey Savchuk --- .../engine/control_test.go | 8 +- .../engine/delete_test.go | 16 +-- .../engine/engine_test.go | 129 +++++++----------- pkg/local_object_storage/engine/error_test.go | 4 +- .../engine/evacuate_test.go | 5 +- .../engine/exists_test.go | 2 +- pkg/local_object_storage/engine/head_test.go | 8 +- .../engine/inhume_test.go | 13 +- pkg/local_object_storage/engine/list_test.go | 5 +- pkg/local_object_storage/engine/lock_test.go | 17 +-- .../engine/shards_test.go | 4 +- 11 files changed, 88 insertions(+), 123 deletions(-) diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index 83babeca3..c9efc312c 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -164,7 +164,7 @@ func testEngineFailInitAndReload(t *testing.T, degradedMode bool, opts []shard.O } func TestExecBlocks(t *testing.T) { - e := testNewEngine(t).setShardsNum(t, 2).engine // number doesn't matter in this test, 2 is several but not many + e := testNewEngine(t).setShardsNum(t, 2).prepare(t).engine // number doesn't matter in this test, 2 is several but not many // put some object obj := testutil.GenerateObjectWithCID(cidtest.ID()) @@ -302,7 +302,8 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str meta.WithEpochState(epochState{}), ), } - }) + }). + prepare(t) e, ids := te.engine, te.shardIDs for _, id := range ids { @@ -312,8 +313,5 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str require.Equal(t, num, len(e.shards)) require.Equal(t, num, len(e.shardPools)) - require.NoError(t, e.Open(context.Background())) - require.NoError(t, e.Init(context.Background())) - return e, currShards } diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index e095e4bbd..0dd2e94bb 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -49,9 +48,8 @@ func TestDeleteBigObject(t *testing.T) { link.SetSplitID(splitID) link.SetChildren(childIDs...) - e := testNewEngine(t).setShardsNum(t, 3).engine - e.log = test.NewLogger(t) - defer e.Close(context.Background()) + e := testNewEngine(t).setShardsNum(t, 3).prepare(t).engine + defer func() { require.NoError(t, e.Close(context.Background())) }() for i := range children { require.NoError(t, Put(context.Background(), e, children[i], false)) @@ -115,11 +113,13 @@ func TestDeleteBigObjectWithoutGC(t *testing.T) { link.SetSplitID(splitID) link.SetChildren(childIDs...) - s1 := testNewShard(t, shard.WithDisabledGC()) + te := testNewEngine(t).setShardsNumAdditionalOpts(t, 1, func(_ int) []shard.Option { + return []shard.Option{shard.WithDisabledGC()} + }).prepare(t) + e := te.engine + defer func() { require.NoError(t, e.Close(context.Background())) }() - e := testNewEngine(t).setInitializedShards(t, s1).engine - e.log = test.NewLogger(t) - defer e.Close(context.Background()) + s1 := te.shards[0] for i := range children { require.NoError(t, Put(context.Background(), e, children[i], false)) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index bac35917c..a7cb90bae 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -3,7 +3,6 @@ package engine import ( "context" "path/filepath" - "sync/atomic" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" @@ -26,68 +25,77 @@ func (s epochState) CurrentEpoch() uint64 { type testEngineWrapper struct { engine *StorageEngine + shards []*shard.Shard shardIDs []*shard.ID } func testNewEngine(t testing.TB, opts ...Option) *testEngineWrapper { - engine := New(WithLogger(test.NewLogger(t))) - for _, opt := range opts { - opt(engine.cfg) - } - return &testEngineWrapper{ - engine: engine, - } -} - -func (te *testEngineWrapper) setInitializedShards(t testing.TB, shards ...*shard.Shard) *testEngineWrapper { - for _, s := range shards { - pool, err := ants.NewPool(10, ants.WithNonblocking(true)) - require.NoError(t, err) - - te.engine.shards[s.ID().String()] = hashedShard{ - shardWrapper: shardWrapper{ - errorCount: new(atomic.Uint32), - Shard: s, - }, - hash: hrw.StringHash(s.ID().String()), - } - te.engine.shardPools[s.ID().String()] = pool - te.shardIDs = append(te.shardIDs, s.ID()) - } - return te + opts = append(testGetDefaultEngineOptions(t), opts...) + return &testEngineWrapper{engine: New(opts...)} } func (te *testEngineWrapper) setShardsNum(t testing.TB, num int) *testEngineWrapper { - shards := make([]*shard.Shard, 0, num) - - for range num { - shards = append(shards, testNewShard(t)) - } - - return te.setInitializedShards(t, shards...) + return te.setShardsNumOpts(t, num, func(_ int) []shard.Option { + return testGetDefaultShardOptions(t) + }) } -func (te *testEngineWrapper) setShardsNumOpts(t testing.TB, num int, shardOpts func(id int) []shard.Option) *testEngineWrapper { +func (te *testEngineWrapper) setShardsNumOpts( + t testing.TB, num int, shardOpts func(id int) []shard.Option, +) *testEngineWrapper { + te.shards = make([]*shard.Shard, num) + te.shardIDs = make([]*shard.ID, num) for i := range num { - opts := shardOpts(i) - id, err := te.engine.AddShard(context.Background(), opts...) + shard, err := te.engine.createShard(context.Background(), shardOpts(i)) require.NoError(t, err) - te.shardIDs = append(te.shardIDs, id) + require.NoError(t, te.engine.addShard(shard)) + te.shards[i] = shard + te.shardIDs[i] = shard.ID() } + require.Len(t, te.engine.shards, num) + require.Len(t, te.engine.shardPools, num) return te } -func (te *testEngineWrapper) setShardsNumAdditionalOpts(t testing.TB, num int, shardOpts func(id int) []shard.Option) *testEngineWrapper { - for i := range num { - defaultOpts := testDefaultShardOptions(t) - opts := append(defaultOpts, shardOpts(i)...) - id, err := te.engine.AddShard(context.Background(), opts...) - require.NoError(t, err) - te.shardIDs = append(te.shardIDs, id) - } +func (te *testEngineWrapper) setShardsNumAdditionalOpts( + t testing.TB, num int, shardOpts func(id int) []shard.Option, +) *testEngineWrapper { + return te.setShardsNumOpts(t, num, func(id int) []shard.Option { + return append(testGetDefaultShardOptions(t), shardOpts(id)...) + }) +} + +// prepare calls Open and Init on the created engine. +func (te *testEngineWrapper) prepare(t testing.TB) *testEngineWrapper { + require.NoError(t, te.engine.Open(context.Background())) + require.NoError(t, te.engine.Init(context.Background())) return te } +func testGetDefaultEngineOptions(t testing.TB) []Option { + return []Option{ + WithLogger(test.NewLogger(t)), + } +} + +func testGetDefaultShardOptions(t testing.TB) []shard.Option { + return []shard.Option{ + shard.WithLogger(test.NewLogger(t)), + shard.WithBlobStorOptions( + blobstor.WithStorages( + newStorages(t, t.TempDir(), 1<<20)), + blobstor.WithLogger(test.NewLogger(t)), + ), + shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))), + shard.WithMetaBaseOptions( + meta.WithPath(filepath.Join(t.TempDir(), "metabase")), + meta.WithPermissions(0o700), + meta.WithEpochState(epochState{}), + meta.WithLogger(test.NewLogger(t)), + ), + } +} + func newStorages(t testing.TB, root string, smallSize uint64) []blobstor.SubStorage { return []blobstor.SubStorage{ { @@ -137,34 +145,3 @@ func newTestStorages(root string, smallSize uint64) ([]blobstor.SubStorage, *tes }, }, smallFileStorage, largeFileStorage } - -func testNewShard(t testing.TB, opts ...shard.Option) *shard.Shard { - sid, err := generateShardID() - require.NoError(t, err) - - shardOpts := append([]shard.Option{shard.WithID(sid)}, testDefaultShardOptions(t)...) - s := shard.New(append(shardOpts, opts...)...) - - require.NoError(t, s.Open(context.Background())) - require.NoError(t, s.Init(context.Background())) - - return s -} - -func testDefaultShardOptions(t testing.TB) []shard.Option { - return []shard.Option{ - shard.WithLogger(test.NewLogger(t)), - shard.WithBlobStorOptions( - blobstor.WithStorages( - newStorages(t, t.TempDir(), 1<<20)), - blobstor.WithLogger(test.NewLogger(t)), - ), - shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))), - shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(t.TempDir(), "metabase")), - meta.WithPermissions(0o700), - meta.WithEpochState(epochState{}), - meta.WithLogger(test.NewLogger(t)), - ), - } -} diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index 57c423764..d68a7e826 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -67,10 +67,8 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) pilorama.WithPath(filepath.Join(dir, fmt.Sprintf("%d.pilorama", id))), pilorama.WithPerm(0o700)), } - }) + }).prepare(t) e := te.engine - require.NoError(t, e.Open(context.Background())) - require.NoError(t, e.Init(context.Background())) for i, id := range te.shardIDs { testShards[i].id = id diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 54eacc3f2..beab8384e 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -75,10 +75,9 @@ func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEng pilorama.WithPerm(0o700), ), } - }) + }). + prepare(t) e, ids := te.engine, te.shardIDs - require.NoError(t, e.Open(context.Background())) - require.NoError(t, e.Init(context.Background())) objects := make([]*objectSDK.Object, 0, objPerShard*len(ids)) treeID := "version" diff --git a/pkg/local_object_storage/engine/exists_test.go b/pkg/local_object_storage/engine/exists_test.go index e2e5ff13e..1b51c10dc 100644 --- a/pkg/local_object_storage/engine/exists_test.go +++ b/pkg/local_object_storage/engine/exists_test.go @@ -25,7 +25,7 @@ func BenchmarkExists(b *testing.B) { } func benchmarkExists(b *testing.B, shardNum int) { - e := testNewEngine(b).setShardsNum(b, shardNum).engine + e := testNewEngine(b).setShardsNum(b, shardNum).prepare(b).engine defer func() { require.NoError(b, e.Close(context.Background())) }() addr := oidtest.Address() diff --git a/pkg/local_object_storage/engine/head_test.go b/pkg/local_object_storage/engine/head_test.go index 5afc50f07..f9db81f16 100644 --- a/pkg/local_object_storage/engine/head_test.go +++ b/pkg/local_object_storage/engine/head_test.go @@ -39,11 +39,11 @@ func TestHeadRaw(t *testing.T) { link.SetSplitID(splitID) t.Run("virtual object split in different shards", func(t *testing.T) { - s1 := testNewShard(t) - s2 := testNewShard(t) + te := testNewEngine(t).setShardsNum(t, 2).prepare(t) + e := te.engine + defer func() { require.NoError(t, e.Close(context.Background())) }() - e := testNewEngine(t).setInitializedShards(t, s1, s2).engine - defer e.Close(context.Background()) + s1, s2 := te.shards[0], te.shards[1] var putPrmLeft shard.PutPrm putPrmLeft.SetObject(child) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index b4fbbd810..6980afb07 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -37,8 +37,8 @@ func TestStorageEngine_Inhume(t *testing.T) { t.Run("delete small object", func(t *testing.T) { t.Parallel() - e := testNewEngine(t).setShardsNum(t, 1).engine - defer e.Close(context.Background()) + e := testNewEngine(t).setShardsNum(t, 1).prepare(t).engine + defer func() { require.NoError(t, e.Close(context.Background())) }() err := Put(context.Background(), e, parent, false) require.NoError(t, err) @@ -56,11 +56,12 @@ func TestStorageEngine_Inhume(t *testing.T) { t.Run("delete big object", func(t *testing.T) { t.Parallel() - s1 := testNewShard(t) - s2 := testNewShard(t) - e := testNewEngine(t).setInitializedShards(t, s1, s2).engine - defer e.Close(context.Background()) + te := testNewEngine(t).setShardsNum(t, 2).prepare(t) + e := te.engine + defer func() { require.NoError(t, e.Close(context.Background())) }() + + s1, s2 := te.shards[0], te.shards[1] var putChild shard.PutPrm putChild.SetObject(child) diff --git a/pkg/local_object_storage/engine/list_test.go b/pkg/local_object_storage/engine/list_test.go index d683b5475..6cfa546f8 100644 --- a/pkg/local_object_storage/engine/list_test.go +++ b/pkg/local_object_storage/engine/list_test.go @@ -68,10 +68,7 @@ func TestListWithCursor(t *testing.T) { meta.WithEpochState(epochState{}), ), } - }).engine - require.NoError(t, e.Open(context.Background())) - require.NoError(t, e.Init(context.Background())) - + }).prepare(t).engine defer func() { require.NoError(t, e.Close(context.Background())) }() diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 7e15c76f5..feca9cb69 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -57,11 +57,9 @@ func TestLockUserScenario(t *testing.T) { }), shard.WithTombstoneSource(tss{lockerExpiresAfter}), } - }) + }). + prepare(t) e := testEngine.engine - require.NoError(t, e.Open(context.Background())) - require.NoError(t, e.Init(context.Background())) - defer func() { require.NoError(t, e.Close(context.Background())) }() lockerID := oidtest.ID() @@ -162,11 +160,9 @@ func TestLockExpiration(t *testing.T) { return pool }), } - }) + }). + prepare(t) e := testEngine.engine - require.NoError(t, e.Open(context.Background())) - require.NoError(t, e.Init(context.Background())) - defer func() { require.NoError(t, e.Close(context.Background())) }() const lockerExpiresAfter = 13 @@ -243,9 +239,8 @@ func TestLockForceRemoval(t *testing.T) { }), shard.WithDeletedLockCallback(e.processDeletedLocks), } - }).engine - require.NoError(t, e.Open(context.Background())) - require.NoError(t, e.Init(context.Background())) + }). + prepare(t).engine defer func() { require.NoError(t, e.Close(context.Background())) }() cnr := cidtest.ID() diff --git a/pkg/local_object_storage/engine/shards_test.go b/pkg/local_object_storage/engine/shards_test.go index 207491bd4..0bbc7563c 100644 --- a/pkg/local_object_storage/engine/shards_test.go +++ b/pkg/local_object_storage/engine/shards_test.go @@ -13,7 +13,7 @@ import ( func TestRemoveShard(t *testing.T) { const numOfShards = 6 - te := testNewEngine(t).setShardsNum(t, numOfShards) + te := testNewEngine(t).setShardsNum(t, numOfShards).prepare(t) e, ids := te.engine, te.shardIDs defer func() { require.NoError(t, e.Close(context.Background())) }() @@ -51,7 +51,7 @@ func TestDisableShards(t *testing.T) { const numOfShards = 2 - te := testNewEngine(t).setShardsNum(t, numOfShards) + te := testNewEngine(t).setShardsNum(t, numOfShards).prepare(t) e, ids := te.engine, te.shardIDs defer func() { require.NoError(t, e.Close(context.Background())) }() From f7cb6b4d87ec9869d371e569b49cdf418d26a756 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 13 Nov 2024 16:26:40 +0300 Subject: [PATCH 1539/1943] [#1482] Makefile: Update golangci-lint Signed-off-by: Alexander Chuprov --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c54601c73..ecac760e9 100755 --- a/Makefile +++ b/Makefile @@ -8,8 +8,8 @@ HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.22 -LINT_VERSION ?= 1.61.0 -TRUECLOUDLAB_LINT_VERSION ?= 0.0.7 +LINT_VERSION ?= 1.62.0 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.8 PROTOC_VERSION ?= 25.0 PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-sdk-go) PROTOC_OS_VERSION=osx-x86_64 From 6dc0dc66919a19e08b6542cbdb5569b78f432121 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 18 Oct 2024 13:45:06 +0300 Subject: [PATCH 1540/1943] [#1493] shard: Take mode mutex in HandleExpiredTombstones() Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/shard/gc.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index a987d3d14..57c21459c 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -627,7 +627,10 @@ func (s *Shard) selectExpired(ctx context.Context, epoch uint64, addresses []oid // // Does not modify tss. func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.TombstonedObject) { - if s.GetMode().NoMetabase() { + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { return } From 1e6f132b4e59bfc7c341e44a4612de6b2924a555 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 18 Oct 2024 13:57:16 +0300 Subject: [PATCH 1541/1943] [#1493] metabase: Pass InhumePrm by value Unify with the other code, no functional changes. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/inhume.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 8d1e18729..915de5262 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -217,7 +217,7 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes garbageBKT := tx.Bucket(garbageBucketName) graveyardBKT := tx.Bucket(graveyardBucketName) - bkt, value, err := db.getInhumeTargetBucketAndValue(garbageBKT, graveyardBKT, &prm) + bkt, value, err := db.getInhumeTargetBucketAndValue(garbageBKT, graveyardBKT, prm) if err != nil { return err } @@ -354,7 +354,7 @@ func (db *DB) applyInhumeResToCounters(tx *bbolt.Tx, res *InhumeRes) error { // 1. tombstone address if Inhume was called with // a Tombstone // 2. zeroValue if Inhume was called with a GC mark -func (db *DB) getInhumeTargetBucketAndValue(garbageBKT, graveyardBKT *bbolt.Bucket, prm *InhumePrm) (targetBucket *bbolt.Bucket, value []byte, err error) { +func (db *DB) getInhumeTargetBucketAndValue(garbageBKT, graveyardBKT *bbolt.Bucket, prm InhumePrm) (targetBucket *bbolt.Bucket, value []byte, err error) { if prm.tomb != nil { targetBucket = graveyardBKT tombKey := addressKey(*prm.tomb, make([]byte, addressKeySize)) From 44df67492f8f933c2f923e7660d9a58949e21a57 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 18 Oct 2024 14:08:57 +0300 Subject: [PATCH 1542/1943] [#1493] metabase: Split inhumeTx() into 2 functions No functional changes. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/inhume.go | 138 +++++++++++--------- 1 file changed, 73 insertions(+), 65 deletions(-) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 915de5262..5ac0c0be5 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -224,78 +224,86 @@ func (db *DB) inhumeTx(tx *bbolt.Tx, epoch uint64, prm InhumePrm, res *InhumeRes buf := make([]byte, addressKeySize) for i := range prm.target { - id := prm.target[i].Object() - cnr := prm.target[i].Container() - - // prevent locked objects to be inhumed - if !prm.forceRemoval && objectLocked(tx, cnr, id) { - return new(apistatus.ObjectLocked) - } - - var lockWasChecked bool - - // prevent lock objects to be inhumed - // if `Inhume` was called not with the - // `WithForceGCMark` option - if !prm.forceRemoval { - if isLockObject(tx, cnr, id) { - return ErrLockObjectRemoval - } - - lockWasChecked = true - } - - obj, err := db.get(tx, prm.target[i], buf, false, true, epoch) - targetKey := addressKey(prm.target[i], buf) - var ecErr *objectSDK.ECInfoError - if err == nil { - err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, targetKey, cnr, obj, res) - if err != nil { - return err - } - } else if errors.As(err, &ecErr) { - err = db.inhumeECInfo(tx, epoch, prm.tomb, res, garbageBKT, graveyardBKT, ecErr.ECInfo(), cnr, bkt, value) - if err != nil { - return err - } - } - - if prm.tomb != nil { - var isTomb bool - isTomb, err = db.markAsGC(graveyardBKT, garbageBKT, targetKey) - if err != nil { - return err - } - - if isTomb { - continue - } - } - - // consider checking if target is already in graveyard? - err = bkt.Put(targetKey, value) - if err != nil { + if err := db.inhumeTxSingle(bkt, value, graveyardBKT, garbageBKT, prm.target[i], buf, epoch, prm, res); err != nil { return err } - - if prm.lockObjectHandling { - // do not perform lock check if - // it was already called - if lockWasChecked { - // inhumed object is not of - // the LOCK type - continue - } - - if isLockObject(tx, cnr, id) { - res.deletedLockObj = append(res.deletedLockObj, prm.target[i]) - } - } } return db.applyInhumeResToCounters(tx, res) } +func (db *DB) inhumeTxSingle(bkt *bbolt.Bucket, value []byte, graveyardBKT, garbageBKT *bbolt.Bucket, addr oid.Address, buf []byte, epoch uint64, prm InhumePrm, res *InhumeRes) error { + id := addr.Object() + cnr := addr.Container() + tx := bkt.Tx() + + // prevent locked objects to be inhumed + if !prm.forceRemoval && objectLocked(tx, cnr, id) { + return new(apistatus.ObjectLocked) + } + + var lockWasChecked bool + + // prevent lock objects to be inhumed + // if `Inhume` was called not with the + // `WithForceGCMark` option + if !prm.forceRemoval { + if isLockObject(tx, cnr, id) { + return ErrLockObjectRemoval + } + + lockWasChecked = true + } + + obj, err := db.get(tx, addr, buf, false, true, epoch) + targetKey := addressKey(addr, buf) + var ecErr *objectSDK.ECInfoError + if err == nil { + err = db.updateDeleteInfo(tx, garbageBKT, graveyardBKT, targetKey, cnr, obj, res) + if err != nil { + return err + } + } else if errors.As(err, &ecErr) { + err = db.inhumeECInfo(tx, epoch, prm.tomb, res, garbageBKT, graveyardBKT, ecErr.ECInfo(), cnr, bkt, value) + if err != nil { + return err + } + } + + if prm.tomb != nil { + var isTomb bool + isTomb, err = db.markAsGC(graveyardBKT, garbageBKT, targetKey) + if err != nil { + return err + } + + if isTomb { + return nil + } + } + + // consider checking if target is already in graveyard? + err = bkt.Put(targetKey, value) + if err != nil { + return err + } + + if prm.lockObjectHandling { + // do not perform lock check if + // it was already called + if lockWasChecked { + // inhumed object is not of + // the LOCK type + return nil + } + + if isLockObject(tx, cnr, id) { + res.deletedLockObj = append(res.deletedLockObj, addr) + } + } + return nil +} + func (db *DB) inhumeECInfo(tx *bbolt.Tx, epoch uint64, tomb *oid.Address, res *InhumeRes, garbageBKT *bbolt.Bucket, graveyardBKT *bbolt.Bucket, ecInfo *objectSDK.ECInfo, cnr cid.ID, targetBucket *bbolt.Bucket, value []byte, From d77a218f7c1a449369eb6d63e00ae1906984aed4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 13 Nov 2024 15:56:16 +0300 Subject: [PATCH 1543/1943] [#1493] metabase: Merge Inhume() and DropGraves() for tombstones DropGraves() is only used to drop gravemarks after a tombstone removal. Thus, it makes sense to do Inhume() and DropGraves() in one transaction. It has less overhead and no unexpected problems in case of sudden power failure. Signed-off-by: Evgenii Stratonikov --- .../metabase/delete_ec_test.go | 20 +--------- .../metabase/graveyard.go | 39 ++++++++++++------- .../metabase/graveyard_test.go | 26 ++++++++++--- pkg/local_object_storage/shard/gc.go | 21 +--------- 4 files changed, 50 insertions(+), 56 deletions(-) diff --git a/pkg/local_object_storage/metabase/delete_ec_test.go b/pkg/local_object_storage/metabase/delete_ec_test.go index 9f1f91e14..884da23ff 100644 --- a/pkg/local_object_storage/metabase/delete_ec_test.go +++ b/pkg/local_object_storage/metabase/delete_ec_test.go @@ -130,17 +130,9 @@ func TestDeleteECObject_WithoutSplit(t *testing.T) { require.NoError(t, db.IterateOverGraveyard(context.Background(), graveyardIterationPrm)) require.Equal(t, 2, len(tombstonedObjects)) - var tombstones []oid.Address - for _, tss := range tombstonedObjects { - tombstones = append(tombstones, tss.tomb) - } - inhumePrm.SetAddresses(tombstones...) - inhumePrm.SetGCMark() - _, err = db.Inhume(context.Background(), inhumePrm) + _, err = db.InhumeTombstones(context.Background(), tombstonedObjects) require.NoError(t, err) - require.NoError(t, db.DropGraves(context.Background(), tombstonedObjects)) - // GC finds tombstone as garbage and deletes it garbageAddresses = nil @@ -374,17 +366,9 @@ func testDeleteECObjectWithSplit(t *testing.T, chunksCount int, withLinking bool require.NoError(t, db.IterateOverGraveyard(context.Background(), graveyardIterationPrm)) require.True(t, len(tombstonedObjects) == parentCount+chunksCount) - var tombstones []oid.Address - for _, tss := range tombstonedObjects { - tombstones = append(tombstones, tss.tomb) - } - inhumePrm.SetAddresses(tombstones...) - inhumePrm.SetGCMark() - _, err = db.Inhume(context.Background(), inhumePrm) + _, err = db.InhumeTombstones(context.Background(), tombstonedObjects) require.NoError(t, err) - require.NoError(t, db.DropGraves(context.Background(), tombstonedObjects)) - // GC finds tombstone as garbage and deletes it garbageAddresses = nil diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index 31f95d6ed..b0db952b2 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.etcd.io/bbolt" ) @@ -255,46 +256,58 @@ func graveFromKV(k, v []byte) (res TombstonedObject, err error) { return } -// DropGraves deletes tombstoned objects from the +// InhumeTombstones deletes tombstoned objects from the // graveyard bucket. // // Returns any error appeared during deletion process. -func (db *DB) DropGraves(ctx context.Context, tss []TombstonedObject) error { +func (db *DB) InhumeTombstones(ctx context.Context, tss []TombstonedObject) (InhumeRes, error) { var ( startedAt = time.Now() success = false ) defer func() { - db.metrics.AddMethodDuration("DropGraves", time.Since(startedAt), success) + db.metrics.AddMethodDuration("InhumeTombstones", time.Since(startedAt), success) }() - _, span := tracing.StartSpanFromContext(ctx, "metabase.DropGraves") + _, span := tracing.StartSpanFromContext(ctx, "metabase.InhumeTombstones") defer span.End() db.modeMtx.RLock() defer db.modeMtx.RUnlock() if db.mode.NoMetabase() { - return ErrDegradedMode + return InhumeRes{}, ErrDegradedMode } else if db.mode.ReadOnly() { - return ErrReadOnlyMode + return InhumeRes{}, ErrReadOnlyMode } buf := make([]byte, addressKeySize) + prm := InhumePrm{forceRemoval: true} + currEpoch := db.epochState.CurrentEpoch() - return db.boltDB.Batch(func(tx *bbolt.Tx) error { - bkt := tx.Bucket(graveyardBucketName) - if bkt == nil { - return nil + var res InhumeRes + + err := db.boltDB.Batch(func(tx *bbolt.Tx) error { + res = InhumeRes{inhumedByCnrID: make(map[cid.ID]ObjectCounters)} + + garbageBKT := tx.Bucket(garbageBucketName) + graveyardBKT := tx.Bucket(graveyardBucketName) + + bkt, value, err := db.getInhumeTargetBucketAndValue(garbageBKT, graveyardBKT, prm) + if err != nil { + return err } - for _, ts := range tss { - err := bkt.Delete(addressKey(ts.Address(), buf)) - if err != nil { + for i := range tss { + if err := db.inhumeTxSingle(bkt, value, graveyardBKT, garbageBKT, tss[i].Tombstone(), buf, currEpoch, prm, &res); err != nil { + return err + } + if err := graveyardBKT.Delete(addressKey(tss[i].Address(), buf)); err != nil { return err } } return nil }) + return res, err } diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index 99794e609..ebadecc04 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -7,7 +7,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" @@ -393,7 +395,7 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { require.False(t, iWasCalled) } -func TestDB_DropGraves(t *testing.T) { +func TestDB_InhumeTombstones(t *testing.T) { db := newDB(t) defer func() { require.NoError(t, db.Close(context.Background())) }() @@ -410,9 +412,20 @@ func TestDB_DropGraves(t *testing.T) { err = putBig(db, obj2) require.NoError(t, err) - // inhume with tombstone - addrTombstone := oidtest.Address() - addrTombstone.SetContainer(cnr) + id1, _ := obj1.ID() + id2, _ := obj2.ID() + ts := objectSDK.NewTombstone() + ts.SetMembers([]oid.ID{id1, id2}) + objTs := objectSDK.New() + objTs.SetContainerID(cnr) + objTs.SetType(objectSDK.TypeTombstone) + + data, _ := ts.Marshal() + objTs.SetPayload(data) + require.NoError(t, objectSDK.CalculateAndSetID(objTs)) + require.NoError(t, putBig(db, objTs)) + + addrTombstone := object.AddressOf(objTs) var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) @@ -435,8 +448,11 @@ func TestDB_DropGraves(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, counter) - err = db.DropGraves(context.Background(), buriedTS) + res, err := db.InhumeTombstones(context.Background(), buriedTS) require.NoError(t, err) + require.EqualValues(t, 1, res.LogicInhumed()) + require.EqualValues(t, 0, res.UserInhumed()) + require.EqualValues(t, map[cid.ID]meta.ObjectCounters{cnr: {Logic: 1}}, res.InhumedByCnrID()) counter = 0 iterGravePRM.SetHandler(func(_ meta.TombstonedObject) error { diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 57c21459c..c212f8c36 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -634,19 +634,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston return } - // Mark tombstones as garbage. - var pInhume meta.InhumePrm - - tsAddrs := make([]oid.Address, 0, len(tss)) - for _, ts := range tss { - tsAddrs = append(tsAddrs, ts.Tombstone()) - } - - pInhume.SetGCMark() - pInhume.SetAddresses(tsAddrs...) - - // inhume tombstones - res, err := s.metaBase.Inhume(ctx, pInhume) + res, err := s.metaBase.InhumeTombstones(ctx, tss) if err != nil { s.log.Warn(ctx, logs.ShardCouldNotMarkTombstonesAsGarbage, zap.String("error", err.Error()), @@ -666,13 +654,6 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston s.addToContainerSize(delInfo.CID.EncodeToString(), -int64(delInfo.Size)) i++ } - - // drop just processed expired tombstones - // from graveyard - err = s.metaBase.DropGraves(ctx, tss) - if err != nil { - s.log.Warn(ctx, logs.ShardCouldNotDropExpiredGraveRecords, zap.Error(err)) - } } // HandleExpiredLocks unlocks all objects which were locked by lockers. From 69c63006da57d182618eb1ef498e3aaac9888783 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 14 Nov 2024 09:52:35 +0300 Subject: [PATCH 1544/1943] [#1496] morph: Move tx waiter to morph package Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/morph.go | 49 ++++---------------------------- pkg/morph/client/waiter.go | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 44 deletions(-) create mode 100644 pkg/morph/client/waiter.go diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index ca9f4fe3e..4e33ad6e2 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -17,11 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/neorpc/result" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/waiter" - "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "go.uber.org/zap" ) @@ -164,48 +160,13 @@ func makeNotaryDeposit(ctx context.Context, c *cfg) (util.Uint256, uint32, error return c.cfgMorph.client.DepositEndlessNotary(ctx, depositAmount) } -var ( - errNotaryDepositFail = errors.New("notary deposit tx has faulted") - errNotaryDepositTimeout = errors.New("notary deposit tx has not appeared in the network") -) - -type waiterClient struct { - c *client.Client -} - -func (w *waiterClient) Context() context.Context { - return context.Background() -} - -func (w *waiterClient) GetApplicationLog(hash util.Uint256, trig *trigger.Type) (*result.ApplicationLog, error) { - return w.c.GetApplicationLog(hash, trig) -} - -func (w *waiterClient) GetBlockCount() (uint32, error) { - return w.c.BlockCount() -} - -func (w *waiterClient) GetVersion() (*result.Version, error) { - return w.c.GetVersion() -} - func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256, vub uint32) error { - w, err := waiter.NewPollingBased(&waiterClient{c: c.cfgMorph.client}) - if err != nil { - return fmt.Errorf("could not create notary deposit waiter: %w", err) + if err := c.cfgMorph.client.WaitTxHalt(ctx, client.WaitParams{Hash: tx, ValidUntilBlock: vub}); err != nil { + return err } - res, err := w.WaitAny(ctx, vub, tx) - if err != nil { - if errors.Is(err, waiter.ErrTxNotAccepted) { - return errNotaryDepositTimeout - } - return fmt.Errorf("could not wait for notary deposit persists in chain: %w", err) - } - if res.Execution.VMState.HasFlag(vmstate.Halt) { - c.log.Info(ctx, logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted) - return nil - } - return errNotaryDepositFail + + c.log.Info(ctx, logs.ClientNotaryDepositTransactionWasSuccessfullyPersisted) + return nil } func listenMorphNotifications(ctx context.Context, c *cfg) { diff --git a/pkg/morph/client/waiter.go b/pkg/morph/client/waiter.go new file mode 100644 index 000000000..8211deac4 --- /dev/null +++ b/pkg/morph/client/waiter.go @@ -0,0 +1,57 @@ +package client + +import ( + "context" + "fmt" + + "github.com/nspcc-dev/neo-go/pkg/neorpc/result" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/waiter" + "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" +) + +// WaitParams represents transaction to wait for. +type WaitParams struct { + Hash util.Uint256 + ValidUntilBlock uint32 +} + +type waiterClient struct { + c *Client +} + +func (w *waiterClient) Context() context.Context { + return context.Background() +} + +func (w *waiterClient) GetApplicationLog(hash util.Uint256, trig *trigger.Type) (*result.ApplicationLog, error) { + return w.c.GetApplicationLog(hash, trig) +} + +func (w *waiterClient) GetBlockCount() (uint32, error) { + return w.c.BlockCount() +} + +func (w *waiterClient) GetVersion() (*result.Version, error) { + return w.c.GetVersion() +} + +// WaitTxHalt waits until transaction with the specified hash persists on the blockchain. +// It also checks execution result to finish in HALT state. +func (c *Client) WaitTxHalt(ctx context.Context, p WaitParams) error { + w, err := waiter.NewPollingBased(&waiterClient{c: c}) + if err != nil { + return fmt.Errorf("create tx waiter: %w", err) + } + + res, err := w.WaitAny(ctx, p.ValidUntilBlock, p.Hash) + if err != nil { + return fmt.Errorf("wait until tx persists: %w", err) + } + + if res.VMState.HasFlag(vmstate.Halt) { + return nil + } + return wrapFrostFSError(¬HaltStateError{state: res.VMState.String(), exception: res.FaultException}) +} From b65874d1c35fe194a20d25bf19c8d82c633a04b6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 14 Nov 2024 10:01:59 +0300 Subject: [PATCH 1545/1943] [#1496] morph: Return `InvokeRes` from all invoke*() methods Signed-off-by: Evgenii Stratonikov --- .../processors/alphabet/handlers_test.go | 5 +++-- .../processors/alphabet/processor.go | 3 ++- pkg/morph/client/client.go | 8 +++---- pkg/morph/client/notary.go | 22 +++++++++---------- pkg/morph/client/static.go | 14 +++++------- 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/pkg/innerring/processors/alphabet/handlers_test.go b/pkg/innerring/processors/alphabet/handlers_test.go index 036b8055c..ac3e2a14d 100644 --- a/pkg/innerring/processors/alphabet/handlers_test.go +++ b/pkg/innerring/processors/alphabet/handlers_test.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/alphabet" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/timers" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -247,7 +248,7 @@ type testMorphClient struct { batchTransferedGas []batchTransferGas } -func (c *testMorphClient) Invoke(_ context.Context, contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) { +func (c *testMorphClient) Invoke(_ context.Context, contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (client.InvokeRes, error) { c.invokedMethods = append(c.invokedMethods, invokedMethod{ contract: contract, @@ -255,7 +256,7 @@ func (c *testMorphClient) Invoke(_ context.Context, contract util.Uint160, fee f method: method, args: args, }) - return 0, nil + return client.InvokeRes{}, nil } func (c *testMorphClient) TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error { diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index bf74834ed..3992e00f3 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -8,6 +8,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -39,7 +40,7 @@ type ( } morphClient interface { - Invoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) + Invoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (client.InvokeRes, error) TransferGas(receiver util.Uint160, amount fixedn.Fixed8) error BatchTransferGas(receivers []util.Uint160, amount fixedn.Fixed8) error } diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index f61c6e9f9..a0c29141b 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -180,7 +180,7 @@ func wrapFrostFSError(err error) error { // Invoke invokes contract method by sending transaction into blockchain. // Returns valid until block value. // Supported args types: int64, string, util.Uint160, []byte and bool. -func (c *Client) Invoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (uint32, error) { +func (c *Client) Invoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) (InvokeRes, error) { start := time.Now() success := false defer func() { @@ -191,12 +191,12 @@ func (c *Client) Invoke(ctx context.Context, contract util.Uint160, fee fixedn.F defer c.switchLock.RUnlock() if c.inactive { - return 0, ErrConnectionLost + return InvokeRes{}, ErrConnectionLost } txHash, vub, err := c.rpcActor.SendTunedCall(contract, method, nil, addFeeCheckerModifier(int64(fee)), args...) if err != nil { - return 0, fmt.Errorf("could not invoke %s: %w", method, err) + return InvokeRes{}, fmt.Errorf("could not invoke %s: %w", method, err) } c.logger.Debug(ctx, logs.ClientNeoClientInvoke, @@ -205,7 +205,7 @@ func (c *Client) Invoke(ctx context.Context, contract util.Uint160, fee fixedn.F zap.Stringer("tx_hash", txHash.Reverse())) success = true - return vub, nil + return InvokeRes{Hash: txHash, VUB: vub}, nil } // TestInvokeIterator invokes contract method returning an iterator and executes cb on each element. diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 65a5e77a6..71232cb33 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -358,12 +358,12 @@ func (c *Client) UpdateNeoFSAlphabetList(ctx context.Context, prm UpdateAlphabet // Returns valid until block value. // // `nonce` and `vub` are used only if notary is enabled. -func (c *Client) NotaryInvoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) (uint32, error) { +func (c *Client) NotaryInvoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) (InvokeRes, error) { c.switchLock.RLock() defer c.switchLock.RUnlock() if c.inactive { - return 0, ErrConnectionLost + return InvokeRes{}, ErrConnectionLost } if c.notary == nil { @@ -378,12 +378,12 @@ func (c *Client) NotaryInvoke(ctx context.Context, contract util.Uint160, fee fi // not expected to be signed by the current node. // // Considered to be used by non-IR nodes. -func (c *Client) NotaryInvokeNotAlpha(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, vubP *uint32, method string, args ...any) (uint32, error) { +func (c *Client) NotaryInvokeNotAlpha(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, vubP *uint32, method string, args ...any) (InvokeRes, error) { c.switchLock.RLock() defer c.switchLock.RUnlock() if c.inactive { - return 0, ErrConnectionLost + return InvokeRes{}, ErrConnectionLost } if c.notary == nil { @@ -446,7 +446,7 @@ func (c *Client) notaryInvokeAsCommittee(ctx context.Context, method string, non return err } -func (c *Client) notaryInvoke(ctx context.Context, committee, invokedByAlpha bool, contract util.Uint160, nonce uint32, vub *uint32, method string, args ...any) (uint32, error) { +func (c *Client) notaryInvoke(ctx context.Context, committee, invokedByAlpha bool, contract util.Uint160, nonce uint32, vub *uint32, method string, args ...any) (InvokeRes, error) { start := time.Now() success := false defer func() { @@ -455,22 +455,22 @@ func (c *Client) notaryInvoke(ctx context.Context, committee, invokedByAlpha boo alphabetList, err := c.notary.alphabetSource() if err != nil { - return 0, err + return InvokeRes{}, err } until, err := c.getUntilValue(vub) if err != nil { - return 0, err + return InvokeRes{}, err } cosigners, err := c.notaryCosigners(invokedByAlpha, alphabetList, committee) if err != nil { - return 0, err + return InvokeRes{}, err } nAct, err := notary.NewActor(c.client, cosigners, c.acc) if err != nil { - return 0, err + return InvokeRes{}, err } mainH, fbH, untilActual, err := nAct.Notarize(nAct.MakeTunedCall(contract, method, nil, func(r *result.Invoke, t *transaction.Transaction) error { @@ -485,7 +485,7 @@ func (c *Client) notaryInvoke(ctx context.Context, committee, invokedByAlpha boo }, args...)) if err != nil && !alreadyOnChainError(err) { - return 0, err + return InvokeRes{}, err } c.logger.Debug(ctx, logs.ClientNotaryRequestInvoked, @@ -495,7 +495,7 @@ func (c *Client) notaryInvoke(ctx context.Context, committee, invokedByAlpha boo zap.String("fallback_hash", fbH.StringLE())) success = true - return until, nil + return InvokeRes{Hash: mainH, VUB: until}, nil } func (c *Client) notaryCosignersFromTx(mainTx *transaction.Transaction, alphabetList keys.PublicKeys) ([]actor.SignerAccount, error) { diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index 1e091936f..be4c09182 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -129,7 +129,8 @@ func (i *InvokePrmOptional) SetVUB(v uint32) { } type InvokeRes struct { - VUB uint32 + Hash util.Uint256 + VUB uint32 } // Invoke calls Invoke method of Client with static internal script hash and fee. @@ -142,8 +143,6 @@ type InvokeRes struct { // If fee for the operation executed using specified method is customized, then StaticClient uses it. // Otherwise, default fee is used. func (s StaticClient) Invoke(ctx context.Context, prm InvokePrm) (InvokeRes, error) { - var res InvokeRes - var err error var vubP *uint32 if s.tryNotary { if s.alpha { @@ -170,26 +169,23 @@ func (s StaticClient) Invoke(ctx context.Context, prm InvokePrm) (InvokeRes, err vubP = &prm.vub } - res.VUB, err = s.client.NotaryInvoke(ctx, s.scScriptHash, s.fee, nonce, vubP, prm.method, prm.args...) - return res, err + return s.client.NotaryInvoke(ctx, s.scScriptHash, s.fee, nonce, vubP, prm.method, prm.args...) } if prm.vub > 0 { vubP = &prm.vub } - res.VUB, err = s.client.NotaryInvokeNotAlpha(ctx, s.scScriptHash, s.fee, vubP, prm.method, prm.args...) - return res, err + return s.client.NotaryInvokeNotAlpha(ctx, s.scScriptHash, s.fee, vubP, prm.method, prm.args...) } - res.VUB, err = s.client.Invoke( + return s.client.Invoke( ctx, s.scScriptHash, s.fee, prm.method, prm.args..., ) - return res, err } // TestInvokePrm groups parameters of the TestInvoke operation. From acd5babd86ec98f227a30573497af0cdcd53df87 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 14 Nov 2024 10:04:02 +0300 Subject: [PATCH 1546/1943] [#1496] morph: Merge `InvokeRes` and `WaitParams` Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/morph.go | 2 +- pkg/morph/client/waiter.go | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 4e33ad6e2..67d2d1c06 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -161,7 +161,7 @@ func makeNotaryDeposit(ctx context.Context, c *cfg) (util.Uint256, uint32, error } func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256, vub uint32) error { - if err := c.cfgMorph.client.WaitTxHalt(ctx, client.WaitParams{Hash: tx, ValidUntilBlock: vub}); err != nil { + if err := c.cfgMorph.client.WaitTxHalt(ctx, client.InvokeRes{Hash: tx, VUB: vub}); err != nil { return err } diff --git a/pkg/morph/client/waiter.go b/pkg/morph/client/waiter.go index 8211deac4..962ec1bc2 100644 --- a/pkg/morph/client/waiter.go +++ b/pkg/morph/client/waiter.go @@ -11,12 +11,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" ) -// WaitParams represents transaction to wait for. -type WaitParams struct { - Hash util.Uint256 - ValidUntilBlock uint32 -} - type waiterClient struct { c *Client } @@ -39,13 +33,13 @@ func (w *waiterClient) GetVersion() (*result.Version, error) { // WaitTxHalt waits until transaction with the specified hash persists on the blockchain. // It also checks execution result to finish in HALT state. -func (c *Client) WaitTxHalt(ctx context.Context, p WaitParams) error { +func (c *Client) WaitTxHalt(ctx context.Context, p InvokeRes) error { w, err := waiter.NewPollingBased(&waiterClient{c: c}) if err != nil { return fmt.Errorf("create tx waiter: %w", err) } - res, err := w.WaitAny(ctx, p.ValidUntilBlock, p.Hash) + res, err := w.WaitAny(ctx, p.VUB, p.Hash) if err != nil { return fmt.Errorf("wait until tx persists: %w", err) } From d82f0d192691f33eb63e2d71beac433a4232b163 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 14 Nov 2024 15:09:42 +0300 Subject: [PATCH 1547/1943] [#1496] node/control: Await until SetNetmapStatus() persists Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/netmap.go | 7 +++++-- pkg/morph/client/netmap/peer.go | 4 ++-- pkg/morph/client/netmap/update_state.go | 9 ++------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 6df947954..9127d1123 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -421,8 +421,11 @@ func (c *cfg) updateNetMapState(ctx context.Context, stateSetter func(*nmClient. prm.SetKey(c.key.PublicKey().Bytes()) stateSetter(&prm) - _, err := c.cfgNetmap.wrapper.UpdatePeerState(ctx, prm) - return err + res, err := c.cfgNetmap.wrapper.UpdatePeerState(ctx, prm) + if err != nil { + return err + } + return c.cfgNetmap.wrapper.Morph().WaitTxHalt(ctx, res) } type netInfo struct { diff --git a/pkg/morph/client/netmap/peer.go b/pkg/morph/client/netmap/peer.go index 9617d018c..949e8cb63 100644 --- a/pkg/morph/client/netmap/peer.go +++ b/pkg/morph/client/netmap/peer.go @@ -58,9 +58,9 @@ func (c *Client) ForceRemovePeer(ctx context.Context, nodeInfo netmap.NodeInfo, prm.SetControlTX(true) prm.SetVUB(vub) - vub, err := c.UpdatePeerState(ctx, prm) + res, err := c.UpdatePeerState(ctx, prm) if err != nil { return 0, fmt.Errorf("updating peer state: %v", err) } - return vub, nil + return res.VUB, nil } diff --git a/pkg/morph/client/netmap/update_state.go b/pkg/morph/client/netmap/update_state.go index 971a55d33..f9f639c19 100644 --- a/pkg/morph/client/netmap/update_state.go +++ b/pkg/morph/client/netmap/update_state.go @@ -2,7 +2,6 @@ package netmap import ( "context" - "fmt" "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" @@ -37,7 +36,7 @@ func (u *UpdatePeerPrm) SetMaintenance() { } // UpdatePeerState changes peer status through Netmap contract call. -func (c *Client) UpdatePeerState(ctx context.Context, p UpdatePeerPrm) (uint32, error) { +func (c *Client) UpdatePeerState(ctx context.Context, p UpdatePeerPrm) (client.InvokeRes, error) { method := updateStateMethod if c.client.WithNotary() && c.client.IsAlpha() { @@ -56,9 +55,5 @@ func (c *Client) UpdatePeerState(ctx context.Context, p UpdatePeerPrm) (uint32, prm.SetArgs(int64(p.state), p.key) prm.InvokePrmOptional = p.InvokePrmOptional - res, err := c.client.Invoke(ctx, prm) - if err != nil { - return 0, fmt.Errorf("could not invoke smart contract: %w", err) - } - return res.VUB, nil + return c.client.Invoke(ctx, prm) } From 9f4ce600ac98bd34bb097b73d45b148ec4f75b26 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 19 Nov 2024 16:25:16 +0300 Subject: [PATCH 1548/1943] [#1505] adm: Allow to manage additional keys in frostfsid Signed-off-by: Evgenii Stratonikov --- .../morph/frostfsid/additional_keys.go | 83 +++++++++++++++++++ .../internal/modules/morph/frostfsid/root.go | 2 + 2 files changed, 85 insertions(+) create mode 100644 cmd/frostfs-adm/internal/modules/morph/frostfsid/additional_keys.go diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/additional_keys.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/additional_keys.go new file mode 100644 index 000000000..4046e85e3 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/additional_keys.go @@ -0,0 +1,83 @@ +package frostfsid + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + frostfsidAddSubjectKeyCmd = &cobra.Command{ + Use: "add-subject-key", + Short: "Add a public key to the subject in frostfsid contract", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + }, + Run: frostfsidAddSubjectKey, + } + frostfsidRemoveSubjectKeyCmd = &cobra.Command{ + Use: "remove-subject-key", + Short: "Remove a public key from the subject in frostfsid contract", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + }, + Run: frostfsidRemoveSubjectKey, + } +) + +func initFrostfsIDAddSubjectKeyCmd() { + Cmd.AddCommand(frostfsidAddSubjectKeyCmd) + + ff := frostfsidAddSubjectKeyCmd.Flags() + ff.StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + ff.String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + + ff.String(subjectAddressFlag, "", "Subject address") + _ = frostfsidAddSubjectKeyCmd.MarkFlagRequired(subjectAddressFlag) + + ff.String(subjectKeyFlag, "", "Public key to add") + _ = frostfsidAddSubjectKeyCmd.MarkFlagRequired(subjectKeyFlag) +} + +func initFrostfsIDRemoveSubjectKeyCmd() { + Cmd.AddCommand(frostfsidRemoveSubjectKeyCmd) + + ff := frostfsidRemoveSubjectKeyCmd.Flags() + ff.StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + ff.String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + + ff.String(subjectAddressFlag, "", "Subject address") + _ = frostfsidAddSubjectKeyCmd.MarkFlagRequired(subjectAddressFlag) + + ff.String(subjectKeyFlag, "", "Public key to remove") + _ = frostfsidAddSubjectKeyCmd.MarkFlagRequired(subjectKeyFlag) +} + +func frostfsidAddSubjectKey(cmd *cobra.Command, _ []string) { + addr := getFrostfsIDSubjectAddress(cmd) + pub := getFrostfsIDSubjectKey(cmd) + + ffsid, err := newFrostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + + ffsid.addCall(ffsid.roCli.AddSubjectKeyCall(addr, pub)) + + err = ffsid.sendWait() + commonCmd.ExitOnErr(cmd, "add subject key: %w", err) +} + +func frostfsidRemoveSubjectKey(cmd *cobra.Command, _ []string) { + addr := getFrostfsIDSubjectAddress(cmd) + pub := getFrostfsIDSubjectKey(cmd) + + ffsid, err := newFrostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + + ffsid.addCall(ffsid.roCli.RemoveSubjectKeyCall(addr, pub)) + + err = ffsid.sendWait() + commonCmd.ExitOnErr(cmd, "remove subject key: %w", err) +} diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go index 850474794..6ffcaa487 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go @@ -12,4 +12,6 @@ func init() { initFrostfsIDAddSubjectToGroupCmd() initFrostfsIDRemoveSubjectFromGroupCmd() initFrostfsIDListGroupSubjectsCmd() + initFrostfsIDAddSubjectKeyCmd() + initFrostfsIDRemoveSubjectKeyCmd() } From e2cb0640f1236ea1874daeb18472ed77da30df90 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 18 Nov 2024 12:19:13 +0300 Subject: [PATCH 1549/1943] [#1501] util: Move eACL-to-APE converter to `pkg/util` * `ConvertEACLToAPE` is useful method which couldn't be imported out of frostfs-node so far as it has been in `internal` * Since `ConvertEACLToAPE` and related structures and unit-tests are placed in `pkg/util` Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/util/convert_eacl.go | 2 +- {internal => pkg/util}/ape/converter.go | 0 {internal => pkg/util}/ape/converter_test.go | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {internal => pkg/util}/ape/converter.go (100%) rename {internal => pkg/util}/ape/converter_test.go (100%) diff --git a/cmd/frostfs-cli/modules/util/convert_eacl.go b/cmd/frostfs-cli/modules/util/convert_eacl.go index d588ba35d..caa6dfcfe 100644 --- a/cmd/frostfs-cli/modules/util/convert_eacl.go +++ b/cmd/frostfs-cli/modules/util/convert_eacl.go @@ -6,7 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - apeutil "git.frostfs.info/TrueCloudLab/frostfs-node/internal/ape" + apeutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/ape" "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" ) diff --git a/internal/ape/converter.go b/pkg/util/ape/converter.go similarity index 100% rename from internal/ape/converter.go rename to pkg/util/ape/converter.go diff --git a/internal/ape/converter_test.go b/pkg/util/ape/converter_test.go similarity index 100% rename from internal/ape/converter_test.go rename to pkg/util/ape/converter_test.go From ae31ef36029e0f8d6e7934faa5ccfd9a32d2a42a Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 18 Nov 2024 12:37:55 +0300 Subject: [PATCH 1550/1943] [#1501] cli: Move `PrintHumanReadableAPEChain` to a common package * Both `frostfs-cli` and `frostfs-adm` APE-related subcommands use `PrintHumanReadableAPEChain` to print a parsed APE-chain. So, it's more correct to have it in a common package over `frostfs-cli` and `frostfs-adm` folders. Signed-off-by: Airat Arifullin --- .../internal/modules/morph/ape/ape.go | 4 +- .../internal/modules/morph/ape/ape_util.go | 3 +- .../modules/ape_manager/add_chain.go | 3 +- .../modules/ape_manager/list_chain.go | 4 +- .../modules/bearer/generate_override.go | 3 +- cmd/frostfs-cli/modules/control/add_rule.go | 3 +- cmd/frostfs-cli/modules/control/get_rule.go | 4 +- cmd/frostfs-cli/modules/control/list_rules.go | 4 +- cmd/frostfs-cli/modules/util/ape.go | 34 --------------- cmd/internal/common/ape/commands.go | 41 +++++++++++++++++++ 10 files changed, 57 insertions(+), 46 deletions(-) create mode 100644 cmd/internal/common/ape/commands.go diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go index 077e03737..fb363f903 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go @@ -5,8 +5,8 @@ import ( "encoding/json" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" - parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/cobra" @@ -210,7 +210,7 @@ func listRuleChains(cmd *cobra.Command, _ []string) { prettyJSONFormat(cmd, chains) } else { for _, c := range chains { - parseutil.PrintHumanReadableAPEChain(cmd, c) + apeCmd.PrintHumanReadableAPEChain(cmd, c) } } } diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index f4373c535..df358ff69 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" morph "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy" @@ -90,7 +91,7 @@ func parseChain(cmd *cobra.Command) *apechain.Chain { chain.ID = parseChainID(cmd) cmd.Println("Parsed chain:") - parseutil.PrintHumanReadableAPEChain(cmd, chain) + apeCmd.PrintHumanReadableAPEChain(cmd, chain) return chain } diff --git a/cmd/frostfs-cli/modules/ape_manager/add_chain.go b/cmd/frostfs-cli/modules/ape_manager/add_chain.go index a85f3c93e..d9cfc304c 100644 --- a/cmd/frostfs-cli/modules/ape_manager/add_chain.go +++ b/cmd/frostfs-cli/modules/ape_manager/add_chain.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -95,7 +96,7 @@ func parseChain(cmd *cobra.Command) apeSDK.Chain { } cmd.Println("Parsed chain:") - util.PrintHumanReadableAPEChain(cmd, chain) + apeCmd.PrintHumanReadableAPEChain(cmd, chain) serialized := chain.Bytes() return apeSDK.Chain{ diff --git a/cmd/frostfs-cli/modules/ape_manager/list_chain.go b/cmd/frostfs-cli/modules/ape_manager/list_chain.go index a5dd44614..9955c8c03 100644 --- a/cmd/frostfs-cli/modules/ape_manager/list_chain.go +++ b/cmd/frostfs-cli/modules/ape_manager/list_chain.go @@ -4,8 +4,8 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - apeutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" @@ -35,7 +35,7 @@ func list(cmd *cobra.Command, _ []string) { for _, respChain := range resp.Chains { var chain apechain.Chain commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(respChain.Raw)) - apeutil.PrintHumanReadableAPEChain(cmd, &chain) + apeCmd.PrintHumanReadableAPEChain(cmd, &chain) } } diff --git a/cmd/frostfs-cli/modules/bearer/generate_override.go b/cmd/frostfs-cli/modules/bearer/generate_override.go index 482c0027e..31dbdbdbd 100644 --- a/cmd/frostfs-cli/modules/bearer/generate_override.go +++ b/cmd/frostfs-cli/modules/bearer/generate_override.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -109,7 +110,7 @@ func parseChain(cmd *cobra.Command) *apechain.Chain { chain.ID = parseChainID(cmd) cmd.Println("Parsed chain:") - parseutil.PrintHumanReadableAPEChain(cmd, chain) + apeCmd.PrintHumanReadableAPEChain(cmd, chain) return chain } diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index c648377bd..88291f0ac 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -55,7 +56,7 @@ func parseChain(cmd *cobra.Command) *apechain.Chain { } cmd.Println("Parsed chain:") - util.PrintHumanReadableAPEChain(cmd, chain) + apeCmd.PrintHumanReadableAPEChain(cmd, chain) return chain } diff --git a/cmd/frostfs-cli/modules/control/get_rule.go b/cmd/frostfs-cli/modules/control/get_rule.go index 4b4d6eef5..9bbbb02f7 100644 --- a/cmd/frostfs-cli/modules/control/get_rule.go +++ b/cmd/frostfs-cli/modules/control/get_rule.go @@ -4,8 +4,8 @@ import ( "encoding/hex" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -56,7 +56,7 @@ func getRule(cmd *cobra.Command, _ []string) { var chain apechain.Chain commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(resp.GetBody().GetChain())) - util.PrintHumanReadableAPEChain(cmd, &chain) + apeCmd.PrintHumanReadableAPEChain(cmd, &chain) } func initControGetRuleCmd() { diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index 7162df5e0..a7b9f9ef5 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -6,8 +6,8 @@ import ( "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -117,7 +117,7 @@ func listRules(cmd *cobra.Command, _ []string) { for _, c := range chains { var chain apechain.Chain commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(c)) - util.PrintHumanReadableAPEChain(cmd, &chain) + apeCmd.PrintHumanReadableAPEChain(cmd, &chain) } } diff --git a/cmd/frostfs-cli/modules/util/ape.go b/cmd/frostfs-cli/modules/util/ape.go index 73c368510..0e963c0a3 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/cmd/frostfs-cli/modules/util/ape.go @@ -4,13 +4,11 @@ import ( "errors" "fmt" "os" - "strconv" "strings" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" "github.com/flynn-archive/go-shlex" - "github.com/spf13/cobra" ) var ( @@ -27,38 +25,6 @@ var ( errFailedToParseAllAny = errors.New("any/all is not parsed") ) -// PrintHumanReadableAPEChain print APE chain rules. -func PrintHumanReadableAPEChain(cmd *cobra.Command, chain *apechain.Chain) { - cmd.Println("Chain ID: " + string(chain.ID)) - cmd.Printf(" HEX: %x\n", chain.ID) - cmd.Println("Rules:") - for _, rule := range chain.Rules { - cmd.Println("\n\tStatus: " + rule.Status.String()) - cmd.Println("\tAny: " + strconv.FormatBool(rule.Any)) - cmd.Println("\tConditions:") - for _, c := range rule.Condition { - var ot string - switch c.Kind { - case apechain.KindResource: - ot = "Resource" - case apechain.KindRequest: - ot = "Request" - default: - panic("unknown object type") - } - cmd.Println(fmt.Sprintf("\t\t%s %s %s %s", ot, c.Key, c.Op, c.Value)) - } - cmd.Println("\tActions:\tInverted:" + strconv.FormatBool(rule.Actions.Inverted)) - for _, name := range rule.Actions.Names { - cmd.Println("\t\t" + name) - } - cmd.Println("\tResources:\tInverted:" + strconv.FormatBool(rule.Resources.Inverted)) - for _, name := range rule.Resources.Names { - cmd.Println("\t\t" + name) - } - } -} - func ParseAPEChainBinaryOrJSON(chain *apechain.Chain, path string) error { data, err := os.ReadFile(path) if err != nil { diff --git a/cmd/internal/common/ape/commands.go b/cmd/internal/common/ape/commands.go new file mode 100644 index 000000000..a8f50b29e --- /dev/null +++ b/cmd/internal/common/ape/commands.go @@ -0,0 +1,41 @@ +package ape + +import ( + "fmt" + "strconv" + + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "github.com/spf13/cobra" +) + +// PrintHumanReadableAPEChain print APE chain rules. +func PrintHumanReadableAPEChain(cmd *cobra.Command, chain *apechain.Chain) { + cmd.Println("Chain ID: " + string(chain.ID)) + cmd.Printf(" HEX: %x\n", chain.ID) + cmd.Println("Rules:") + for _, rule := range chain.Rules { + cmd.Println("\n\tStatus: " + rule.Status.String()) + cmd.Println("\tAny: " + strconv.FormatBool(rule.Any)) + cmd.Println("\tConditions:") + for _, c := range rule.Condition { + var ot string + switch c.Kind { + case apechain.KindResource: + ot = "Resource" + case apechain.KindRequest: + ot = "Request" + default: + panic("unknown object type") + } + cmd.Println(fmt.Sprintf("\t\t%s %s %s %s", ot, c.Key, c.Op, c.Value)) + } + cmd.Println("\tActions:\tInverted:" + strconv.FormatBool(rule.Actions.Inverted)) + for _, name := range rule.Actions.Names { + cmd.Println("\t\t" + name) + } + cmd.Println("\tResources:\tInverted:" + strconv.FormatBool(rule.Resources.Inverted)) + for _, name := range rule.Resources.Names { + cmd.Println("\t\t" + name) + } + } +} From ffe9906266044ca3d8a1f2baf6256490a787cada Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 18 Nov 2024 12:54:02 +0300 Subject: [PATCH 1551/1943] [#1501] cli: Move APE-chain parser methods to `pkg/util` Signed-off-by: Airat Arifullin --- cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go | 2 +- cmd/frostfs-cli/modules/ape_manager/add_chain.go | 6 +++--- cmd/frostfs-cli/modules/bearer/generate_override.go | 2 +- cmd/frostfs-cli/modules/control/add_rule.go | 6 +++--- .../modules/util/ape.go => pkg/util/ape/parser.go | 2 +- .../modules/util/ape_test.go => pkg/util/ape/parser_test.go | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) rename cmd/frostfs-cli/modules/util/ape.go => pkg/util/ape/parser.go (99%) rename cmd/frostfs-cli/modules/util/ape_test.go => pkg/util/ape/parser_test.go (99%) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index df358ff69..1a70dd4a2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -6,9 +6,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" - parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" + parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/ape" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" morph "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy" diff --git a/cmd/frostfs-cli/modules/ape_manager/add_chain.go b/cmd/frostfs-cli/modules/ape_manager/add_chain.go index d9cfc304c..7a49dee68 100644 --- a/cmd/frostfs-cli/modules/ape_manager/add_chain.go +++ b/cmd/frostfs-cli/modules/ape_manager/add_chain.go @@ -7,9 +7,9 @@ import ( internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" + parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/ape" apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -88,9 +88,9 @@ func parseChain(cmd *cobra.Command) apeSDK.Chain { chain.ID = apechain.ID(chainIDRaw) if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 { - commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, rules)) + commonCmd.ExitOnErr(cmd, "parser error: %w", parseutil.ParseAPEChain(chain, rules)) } else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" { - commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", util.ParseAPEChainBinaryOrJSON(chain, encPath)) + commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", parseutil.ParseAPEChainBinaryOrJSON(chain, encPath)) } else { commonCmd.ExitOnErr(cmd, "parser error: %w", errors.New("rule is not passed")) } diff --git a/cmd/frostfs-cli/modules/bearer/generate_override.go b/cmd/frostfs-cli/modules/bearer/generate_override.go index 31dbdbdbd..c3df0294e 100644 --- a/cmd/frostfs-cli/modules/bearer/generate_override.go +++ b/cmd/frostfs-cli/modules/bearer/generate_override.go @@ -6,9 +6,9 @@ import ( "os" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" - parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" + parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/ape" apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index 88291f0ac..25960f439 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -5,10 +5,10 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/util" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/ape" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" @@ -48,9 +48,9 @@ func parseChain(cmd *cobra.Command) *apechain.Chain { chain.ID = apechain.ID(chainIDRaw) if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 { - commonCmd.ExitOnErr(cmd, "parser error: %w", util.ParseAPEChain(chain, rules)) + commonCmd.ExitOnErr(cmd, "parser error: %w", parseutil.ParseAPEChain(chain, rules)) } else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" { - commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", util.ParseAPEChainBinaryOrJSON(chain, encPath)) + commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", parseutil.ParseAPEChainBinaryOrJSON(chain, encPath)) } else { commonCmd.ExitOnErr(cmd, "parser error", errors.New("rule is not passed")) } diff --git a/cmd/frostfs-cli/modules/util/ape.go b/pkg/util/ape/parser.go similarity index 99% rename from cmd/frostfs-cli/modules/util/ape.go rename to pkg/util/ape/parser.go index 0e963c0a3..b4a31fd8d 100644 --- a/cmd/frostfs-cli/modules/util/ape.go +++ b/pkg/util/ape/parser.go @@ -1,4 +1,4 @@ -package util +package ape import ( "errors" diff --git a/cmd/frostfs-cli/modules/util/ape_test.go b/pkg/util/ape/parser_test.go similarity index 99% rename from cmd/frostfs-cli/modules/util/ape_test.go rename to pkg/util/ape/parser_test.go index b275803df..21649fd24 100644 --- a/cmd/frostfs-cli/modules/util/ape_test.go +++ b/pkg/util/ape/parser_test.go @@ -1,4 +1,4 @@ -package util +package ape import ( "fmt" From 7a7ee71a4d076325306e1bae754bdb8c40f5f026 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 18 Nov 2024 13:12:22 +0300 Subject: [PATCH 1552/1943] [#1501] cmd: Introduce common APE-chain parser commands * Introduce common parsing commands to use them in `frostfs-cli` and `frostfs-adm` APE-related subcommands * Introduce common flags for these parsing commands Signed-off-by: Airat Arifullin --- cmd/internal/common/ape/commands.go | 125 ++++++++++++++++++++++++++++ cmd/internal/common/ape/flags.go | 19 +++++ 2 files changed, 144 insertions(+) create mode 100644 cmd/internal/common/ape/flags.go diff --git a/cmd/internal/common/ape/commands.go b/cmd/internal/common/ape/commands.go index a8f50b29e..354747330 100644 --- a/cmd/internal/common/ape/commands.go +++ b/cmd/internal/common/ape/commands.go @@ -1,13 +1,43 @@ package ape import ( + "encoding/hex" + "errors" "fmt" "strconv" + "strings" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/ape" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" + "github.com/nspcc-dev/neo-go/cli/input" "github.com/spf13/cobra" ) +const ( + defaultNamespace = "root" + namespaceTarget = "namespace" + containerTarget = "container" + userTarget = "user" + groupTarget = "group" + + Ingress = "ingress" + S3 = "s3" +) + +var mChainName = map[string]apechain.Name{ + Ingress: apechain.Ingress, + S3: apechain.S3, +} + +var ( + errSettingDefaultValueWasDeclined = errors.New("setting default value was declined") + errUnknownTargetType = errors.New("unknown target type") + errUnsupportedChainName = errors.New("unsupported chain name") +) + // PrintHumanReadableAPEChain print APE chain rules. func PrintHumanReadableAPEChain(cmd *cobra.Command, chain *apechain.Chain) { cmd.Println("Chain ID: " + string(chain.ID)) @@ -39,3 +69,98 @@ func PrintHumanReadableAPEChain(cmd *cobra.Command, chain *apechain.Chain) { } } } + +// ParseTarget handles target parsing of an APE chain. +func ParseTarget(cmd *cobra.Command) engine.Target { + typ := ParseTargetType(cmd) + name, _ := cmd.Flags().GetString(TargetNameFlag) + switch typ { + case engine.Namespace: + if name == "" { + ln, err := input.ReadLine(fmt.Sprintf("Target name is not set. Confirm to use %s namespace (n|Y)> ", defaultNamespace)) + commonCmd.ExitOnErr(cmd, "read line error: %w", err) + ln = strings.ToLower(ln) + if len(ln) > 0 && (ln[0] == 'n') { + commonCmd.ExitOnErr(cmd, "read namespace error: %w", errSettingDefaultValueWasDeclined) + } + name = defaultNamespace + } + return engine.NamespaceTarget(name) + case engine.Container: + var cnr cid.ID + commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(name)) + return engine.ContainerTarget(name) + case engine.User: + return engine.UserTarget(name) + case engine.Group: + return engine.GroupTarget(name) + default: + commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType) + } + return engine.Target{} +} + +// ParseTargetType handles target type parsing of an APE chain. +func ParseTargetType(cmd *cobra.Command) engine.TargetType { + typ, _ := cmd.Flags().GetString(TargetTypeFlag) + switch typ { + case namespaceTarget: + return engine.Namespace + case containerTarget: + return engine.Container + case userTarget: + return engine.User + case groupTarget: + return engine.Group + } + commonCmd.ExitOnErr(cmd, "parse target type error: %w", errUnknownTargetType) + return engine.TargetType(0) +} + +// ParseChainID handles the parsing of APE-chain identifier. +// For some subcommands, chain ID is optional as an input parameter and should be generated by +// the service instead. +func ParseChainID(cmd *cobra.Command) (id apechain.ID) { + chainID, _ := cmd.Flags().GetString(ChainIDFlag) + id = apechain.ID(chainID) + + hexEncoded, _ := cmd.Flags().GetBool(ChainIDHexFlag) + if !hexEncoded { + return + } + + chainIDRaw, err := hex.DecodeString(chainID) + commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err) + id = apechain.ID(chainIDRaw) + return +} + +// ParseChain parses an APE chain which can be provided either as a rule statement +// or loaded from a binary/JSON file path. +func ParseChain(cmd *cobra.Command) *apechain.Chain { + chain := new(apechain.Chain) + chain.ID = ParseChainID(cmd) + + if rules, _ := cmd.Flags().GetStringArray(RuleFlag); len(rules) > 0 { + commonCmd.ExitOnErr(cmd, "parser error: %w", apeutil.ParseAPEChain(chain, rules)) + } else if encPath, _ := cmd.Flags().GetString(PathFlag); encPath != "" { + commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", apeutil.ParseAPEChainBinaryOrJSON(chain, encPath)) + } else { + commonCmd.ExitOnErr(cmd, "parser error", errors.New("rule is not passed")) + } + + cmd.Println("Parsed chain:") + PrintHumanReadableAPEChain(cmd, chain) + + return chain +} + +// ParseChainName parses chain name: the place in the request lifecycle where policy is applied. +func ParseChainName(cmd *cobra.Command) apechain.Name { + chainName, _ := cmd.Flags().GetString(ChainNameFlag) + apeChainName, ok := mChainName[strings.ToLower(chainName)] + if !ok { + commonCmd.ExitOnErr(cmd, "", errUnsupportedChainName) + } + return apeChainName +} diff --git a/cmd/internal/common/ape/flags.go b/cmd/internal/common/ape/flags.go new file mode 100644 index 000000000..c5e2a3a99 --- /dev/null +++ b/cmd/internal/common/ape/flags.go @@ -0,0 +1,19 @@ +package ape + +const ( + RuleFlag = "rule" + RuleFlagDesc = "Rule statement" + PathFlag = "path" + PathFlagDesc = "Path to encoded chain in JSON or binary format" + TargetNameFlag = "target-name" + TargetNameFlagDesc = "Resource name in APE resource name format" + TargetTypeFlag = "target-type" + TargetTypeFlagDesc = "Resource type(container/namespace)" + ChainIDFlag = "chain-id" + ChainIDFlagDesc = "Chain id" + ChainIDHexFlag = "chain-id-hex" + ChainIDHexFlagDesc = "Flag to parse chain ID as hex" + ChainNameFlag = "chain-name" + ChainNameFlagDesc = "Chain name(ingress|s3)" + AllFlag = "all" +) From daff77b2737cdf2c5257cac608794e87c67ebb60 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 18 Nov 2024 13:33:47 +0300 Subject: [PATCH 1553/1943] [#1501] cli: Refactor local override managing subcommands * Refactor local override managing subcommands * Use `cmd/internal/common/ape` parser commands within local override subcommands * Use flag names from `cmd/internal/common/ape` Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/control/add_rule.go | 57 ++----------- cmd/frostfs-cli/modules/control/get_rule.go | 18 ++-- cmd/frostfs-cli/modules/control/list_rules.go | 82 +++++-------------- .../modules/control/list_targets.go | 20 ++--- .../modules/control/remove_rule.go | 27 +++--- 5 files changed, 54 insertions(+), 150 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/add_rule.go b/cmd/frostfs-cli/modules/control/add_rule.go index 25960f439..42f229ad9 100644 --- a/cmd/frostfs-cli/modules/control/add_rule.go +++ b/cmd/frostfs-cli/modules/control/add_rule.go @@ -1,24 +1,14 @@ package control import ( - "encoding/hex" - "errors" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/ape" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" - apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" ) -const ( - ruleFlag = "rule" - pathFlag = "path" -) - var addRuleCmd = &cobra.Command{ Use: "add-rule", Short: "Add local override", @@ -32,41 +22,12 @@ control add-rule --endpoint ... -w ... --address ... --chain-id ChainID --cid .. Run: addRule, } -func parseChain(cmd *cobra.Command) *apechain.Chain { - chainID, _ := cmd.Flags().GetString(chainIDFlag) - hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) - - chainIDRaw := []byte(chainID) - - if hexEncoded { - var err error - chainIDRaw, err = hex.DecodeString(chainID) - commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err) - } - - chain := new(apechain.Chain) - chain.ID = apechain.ID(chainIDRaw) - - if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 { - commonCmd.ExitOnErr(cmd, "parser error: %w", parseutil.ParseAPEChain(chain, rules)) - } else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" { - commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", parseutil.ParseAPEChainBinaryOrJSON(chain, encPath)) - } else { - commonCmd.ExitOnErr(cmd, "parser error", errors.New("rule is not passed")) - } - - cmd.Println("Parsed chain:") - apeCmd.PrintHumanReadableAPEChain(cmd, chain) - - return chain -} - func addRule(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) target := parseTarget(cmd) - parsed := parseChain(cmd) + parsed := apeCmd.ParseChain(cmd) req := &control.AddChainLocalOverrideRequest{ Body: &control.AddChainLocalOverrideRequest_Body{ @@ -95,13 +56,13 @@ func initControlAddRuleCmd() { initControlFlags(addRuleCmd) ff := addRuleCmd.Flags() - ff.StringArray(ruleFlag, []string{}, "Rule statement") - ff.String(pathFlag, "", "Path to encoded chain in JSON or binary format") - ff.String(chainIDFlag, "", "Assign ID to the parsed chain") - ff.String(targetNameFlag, "", targetNameDesc) - ff.String(targetTypeFlag, "", targetTypeDesc) - _ = addRuleCmd.MarkFlagRequired(targetTypeFlag) - ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") + ff.StringArray(apeCmd.RuleFlag, []string{}, "Rule statement") + ff.String(apeCmd.PathFlag, "", "Path to encoded chain in JSON or binary format") + ff.String(apeCmd.ChainIDFlag, "", "Assign ID to the parsed chain") + ff.String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc) + ff.String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc) + _ = addRuleCmd.MarkFlagRequired(apeCmd.TargetTypeFlag) + ff.Bool(apeCmd.ChainIDHexFlag, false, "Flag to parse chain ID as hex") - addRuleCmd.MarkFlagsMutuallyExclusive(pathFlag, ruleFlag) + addRuleCmd.MarkFlagsMutuallyExclusive(apeCmd.PathFlag, apeCmd.RuleFlag) } diff --git a/cmd/frostfs-cli/modules/control/get_rule.go b/cmd/frostfs-cli/modules/control/get_rule.go index 9bbbb02f7..4da903a9a 100644 --- a/cmd/frostfs-cli/modules/control/get_rule.go +++ b/cmd/frostfs-cli/modules/control/get_rule.go @@ -5,7 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" + apecmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -24,8 +24,8 @@ func getRule(cmd *cobra.Command, _ []string) { target := parseTarget(cmd) - chainID, _ := cmd.Flags().GetString(chainIDFlag) - hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) + chainID, _ := cmd.Flags().GetString(apecmd.ChainIDFlag) + hexEncoded, _ := cmd.Flags().GetBool(apecmd.ChainIDHexFlag) if hexEncoded { chainIDBytes, err := hex.DecodeString(chainID) @@ -56,16 +56,16 @@ func getRule(cmd *cobra.Command, _ []string) { var chain apechain.Chain commonCmd.ExitOnErr(cmd, "decode error: %w", chain.DecodeBytes(resp.GetBody().GetChain())) - apeCmd.PrintHumanReadableAPEChain(cmd, &chain) + apecmd.PrintHumanReadableAPEChain(cmd, &chain) } func initControGetRuleCmd() { initControlFlags(getRuleCmd) ff := getRuleCmd.Flags() - ff.String(targetNameFlag, "", targetNameDesc) - ff.String(targetTypeFlag, "", targetTypeDesc) - _ = getRuleCmd.MarkFlagRequired(targetTypeFlag) - ff.String(chainIDFlag, "", "Chain id") - ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") + ff.String(apecmd.TargetNameFlag, "", apecmd.TargetNameFlagDesc) + ff.String(apecmd.TargetTypeFlag, "", apecmd.TargetTypeFlagDesc) + _ = getRuleCmd.MarkFlagRequired(apecmd.TargetTypeFlag) + ff.String(apecmd.ChainIDFlag, "", "Chain id") + ff.Bool(apecmd.ChainIDHexFlag, false, "Flag to parse chain ID as hex") } diff --git a/cmd/frostfs-cli/modules/control/list_rules.go b/cmd/frostfs-cli/modules/control/list_rules.go index a7b9f9ef5..a6c65d083 100644 --- a/cmd/frostfs-cli/modules/control/list_rules.go +++ b/cmd/frostfs-cli/modules/control/list_rules.go @@ -1,7 +1,6 @@ package control import ( - "errors" "fmt" "strings" @@ -10,9 +9,8 @@ import ( apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" - "github.com/nspcc-dev/neo-go/cli/input" + policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "github.com/spf13/cobra" ) @@ -23,65 +21,25 @@ var listRulesCmd = &cobra.Command{ Run: listRules, } -const ( - defaultNamespace = "root" - namespaceTarget = "namespace" - containerTarget = "container" - userTarget = "user" - groupTarget = "group" -) - -const ( - targetNameFlag = "target-name" - targetNameDesc = "Resource name in APE resource name format" - targetTypeFlag = "target-type" - targetTypeDesc = "Resource type(container/namespace)" -) - -var ( - errSettingDefaultValueWasDeclined = errors.New("setting default value was declined") - errUnknownTargetType = errors.New("unknown target type") -) +var engineToControlSvcType = map[policyengine.TargetType]control.ChainTarget_TargetType{ + policyengine.Namespace: control.ChainTarget_NAMESPACE, + policyengine.Container: control.ChainTarget_CONTAINER, + policyengine.User: control.ChainTarget_USER, + policyengine.Group: control.ChainTarget_GROUP, +} func parseTarget(cmd *cobra.Command) *control.ChainTarget { - typ, _ := cmd.Flags().GetString(targetTypeFlag) - name, _ := cmd.Flags().GetString(targetNameFlag) - switch typ { - case namespaceTarget: - if name == "" { - ln, err := input.ReadLine(fmt.Sprintf("Target name is not set. Confirm to use %s namespace (n|Y)> ", defaultNamespace)) - commonCmd.ExitOnErr(cmd, "read line error: %w", err) - ln = strings.ToLower(ln) - if len(ln) > 0 && (ln[0] == 'n') { - commonCmd.ExitOnErr(cmd, "read namespace error: %w", errSettingDefaultValueWasDeclined) - } - name = defaultNamespace - } - return &control.ChainTarget{ - Name: name, - Type: control.ChainTarget_NAMESPACE, - } - case containerTarget: - var cnr cid.ID - commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(name)) - return &control.ChainTarget{ - Name: name, - Type: control.ChainTarget_CONTAINER, - } - case userTarget: - return &control.ChainTarget{ - Name: name, - Type: control.ChainTarget_USER, - } - case groupTarget: - return &control.ChainTarget{ - Name: name, - Type: control.ChainTarget_GROUP, - } - default: - commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType) + target := apeCmd.ParseTarget(cmd) + + typ, ok := engineToControlSvcType[target.Type] + if !ok { + commonCmd.ExitOnErr(cmd, "%w", fmt.Errorf("unknown type '%c", target.Type)) + } + + return &control.ChainTarget{ + Name: target.Name, + Type: typ, } - return nil } func listRules(cmd *cobra.Command, _ []string) { @@ -125,7 +83,7 @@ func initControlListRulesCmd() { initControlFlags(listRulesCmd) ff := listRulesCmd.Flags() - ff.String(targetNameFlag, "", targetNameDesc) - ff.String(targetTypeFlag, "", targetTypeDesc) - _ = listRulesCmd.MarkFlagRequired(targetTypeFlag) + ff.String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc) + ff.String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc) + _ = listRulesCmd.MarkFlagRequired(apeCmd.TargetTypeFlag) } diff --git a/cmd/frostfs-cli/modules/control/list_targets.go b/cmd/frostfs-cli/modules/control/list_targets.go index 7c401eb17..8bd2dc9cd 100644 --- a/cmd/frostfs-cli/modules/control/list_targets.go +++ b/cmd/frostfs-cli/modules/control/list_targets.go @@ -2,26 +2,20 @@ package control import ( "bytes" - "crypto/sha256" "fmt" "strconv" "text/tabwriter" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/spf13/cobra" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) -const ( - chainNameFlag = "chain-name" - chainNameFlagUsage = "Chain name(ingress|s3)" -) - var listTargetsCmd = &cobra.Command{ Use: "list-targets", Short: "List local targets", @@ -32,15 +26,11 @@ var listTargetsCmd = &cobra.Command{ func listTargets(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) - var cnr cid.ID - chainName, _ := cmd.Flags().GetString(chainNameFlag) - - rawCID := make([]byte, sha256.Size) - cnr.Encode(rawCID) + chainName := apeCmd.ParseChainName(cmd) req := &control.ListTargetsLocalOverridesRequest{ Body: &control.ListTargetsLocalOverridesRequest_Body{ - ChainName: chainName, + ChainName: string(chainName), }, } @@ -82,7 +72,7 @@ func initControlListTargetsCmd() { initControlFlags(listTargetsCmd) ff := listTargetsCmd.Flags() - ff.String(chainNameFlag, "", chainNameFlagUsage) + ff.String(apeCmd.ChainNameFlag, "", apeCmd.ChainNameFlagDesc) - _ = cobra.MarkFlagRequired(ff, chainNameFlag) + _ = cobra.MarkFlagRequired(ff, apeCmd.ChainNameFlag) } diff --git a/cmd/frostfs-cli/modules/control/remove_rule.go b/cmd/frostfs-cli/modules/control/remove_rule.go index a996156a5..036317bcb 100644 --- a/cmd/frostfs-cli/modules/control/remove_rule.go +++ b/cmd/frostfs-cli/modules/control/remove_rule.go @@ -6,17 +6,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apecmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" ) -const ( - chainIDFlag = "chain-id" - chainIDHexFlag = "chain-id-hex" - allFlag = "all" -) - var ( errEmptyChainID = errors.New("chain id cannot be empty") @@ -30,8 +25,8 @@ var ( func removeRule(cmd *cobra.Command, _ []string) { pk := key.Get(cmd) - hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) - removeAll, _ := cmd.Flags().GetBool(allFlag) + hexEncoded, _ := cmd.Flags().GetBool(apecmd.ChainIDHexFlag) + removeAll, _ := cmd.Flags().GetBool(apecmd.AllFlag) if removeAll { req := &control.RemoveChainLocalOverridesByTargetRequest{ Body: &control.RemoveChainLocalOverridesByTargetRequest_Body{ @@ -52,7 +47,7 @@ func removeRule(cmd *cobra.Command, _ []string) { return } - chainID, _ := cmd.Flags().GetString(chainIDFlag) + chainID, _ := cmd.Flags().GetString(apecmd.ChainIDFlag) if chainID == "" { commonCmd.ExitOnErr(cmd, "read chain id error: %w", errEmptyChainID) } @@ -92,11 +87,11 @@ func initControlRemoveRuleCmd() { initControlFlags(removeRuleCmd) ff := removeRuleCmd.Flags() - ff.String(targetNameFlag, "", targetNameDesc) - ff.String(targetTypeFlag, "", targetTypeDesc) - _ = removeRuleCmd.MarkFlagRequired(targetTypeFlag) - ff.String(chainIDFlag, "", "Chain id") - ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") - ff.Bool(allFlag, false, "Remove all chains") - removeRuleCmd.MarkFlagsMutuallyExclusive(allFlag, chainIDFlag) + ff.String(apecmd.TargetNameFlag, "", apecmd.TargetNameFlagDesc) + ff.String(apecmd.TargetTypeFlag, "", apecmd.TargetTypeFlagDesc) + _ = removeRuleCmd.MarkFlagRequired(apecmd.TargetTypeFlag) + ff.String(apecmd.ChainIDFlag, "", apecmd.ChainIDFlagDesc) + ff.Bool(apecmd.ChainIDHexFlag, false, apecmd.ChainIDHexFlagDesc) + ff.Bool(apecmd.AllFlag, false, "Remove all chains") + removeRuleCmd.MarkFlagsMutuallyExclusive(apecmd.AllFlag, apecmd.ChainIDFlag) } From 3b1364e4cf9276b944b9a48610a11989e74aa75b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 18 Nov 2024 14:01:27 +0300 Subject: [PATCH 1554/1943] [#1501] cli: Refactor ape-manager subcommands * Refactor ape-manager subcommands * Use `cmd/internal/common/ape` parser commands within ape-manager subcommands * Use flag names from `cmd/internal/common/ape` Signed-off-by: Airat Arifullin --- .../modules/ape_manager/add_chain.go | 93 ++++--------------- .../modules/ape_manager/list_chain.go | 6 +- .../modules/ape_manager/remove_chain.go | 47 ++++------ 3 files changed, 39 insertions(+), 107 deletions(-) diff --git a/cmd/frostfs-cli/modules/ape_manager/add_chain.go b/cmd/frostfs-cli/modules/ape_manager/add_chain.go index 7a49dee68..f4039283f 100644 --- a/cmd/frostfs-cli/modules/ape_manager/add_chain.go +++ b/cmd/frostfs-cli/modules/ape_manager/add_chain.go @@ -1,45 +1,19 @@ package apemanager import ( - "encoding/hex" - "errors" + "fmt" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" - parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/ape" apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "github.com/spf13/cobra" ) -const ( - chainIDFlag = "chain-id" - chainIDHexFlag = "chain-id-hex" - ruleFlag = "rule" - pathFlag = "path" -) - -const ( - targetNameFlag = "target-name" - targetNameDesc = "Resource name in APE resource name format" - targetTypeFlag = "target-type" - targetTypeDesc = "Resource type(container/namespace)" -) - -const ( - namespaceTarget = "namespace" - containerTarget = "container" - userTarget = "user" - groupTarget = "group" -) - -var errUnknownTargetType = errors.New("unknown target type") - var addCmd = &cobra.Command{ Use: "add", Short: "Add rule chain for a target", @@ -50,55 +24,28 @@ var addCmd = &cobra.Command{ } func parseTarget(cmd *cobra.Command) (ct apeSDK.ChainTarget) { - typ, _ := cmd.Flags().GetString(targetTypeFlag) - name, _ := cmd.Flags().GetString(targetNameFlag) + t := apeCmd.ParseTarget(cmd) - ct.Name = name + ct.Name = t.Name - switch typ { - case namespaceTarget: + switch t.Type { + case engine.Namespace: ct.TargetType = apeSDK.TargetTypeNamespace - case containerTarget: - var cnr cid.ID - commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(name)) + case engine.Container: ct.TargetType = apeSDK.TargetTypeContainer - case userTarget: + case engine.User: ct.TargetType = apeSDK.TargetTypeUser - case groupTarget: + case engine.Group: ct.TargetType = apeSDK.TargetTypeGroup default: - commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType) + commonCmd.ExitOnErr(cmd, "conversion error: %w", fmt.Errorf("unknown type '%c'", t.Type)) } return ct } func parseChain(cmd *cobra.Command) apeSDK.Chain { - chainID, _ := cmd.Flags().GetString(chainIDFlag) - hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) - - chainIDRaw := []byte(chainID) - - if hexEncoded { - var err error - chainIDRaw, err = hex.DecodeString(chainID) - commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err) - } - - chain := new(apechain.Chain) - chain.ID = apechain.ID(chainIDRaw) - - if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 { - commonCmd.ExitOnErr(cmd, "parser error: %w", parseutil.ParseAPEChain(chain, rules)) - } else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" { - commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", parseutil.ParseAPEChainBinaryOrJSON(chain, encPath)) - } else { - commonCmd.ExitOnErr(cmd, "parser error: %w", errors.New("rule is not passed")) - } - - cmd.Println("Parsed chain:") - apeCmd.PrintHumanReadableAPEChain(cmd, chain) - - serialized := chain.Bytes() + c := apeCmd.ParseChain(cmd) + serialized := c.Bytes() return apeSDK.Chain{ Raw: serialized, } @@ -127,13 +74,13 @@ func initAddCmd() { commonflags.Init(addCmd) ff := addCmd.Flags() - ff.StringArray(ruleFlag, []string{}, "Rule statement") - ff.String(pathFlag, "", "Path to encoded chain in JSON or binary format") - ff.String(chainIDFlag, "", "Assign ID to the parsed chain") - ff.String(targetNameFlag, "", targetNameDesc) - ff.String(targetTypeFlag, "", targetTypeDesc) - _ = addCmd.MarkFlagRequired(targetTypeFlag) - ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") + ff.StringArray(apeCmd.RuleFlag, []string{}, apeCmd.RuleFlagDesc) + ff.String(apeCmd.PathFlag, "", apeCmd.PathFlagDesc) + ff.String(apeCmd.ChainIDFlag, "", apeCmd.ChainIDFlagDesc) + ff.String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc) + ff.String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc) + _ = addCmd.MarkFlagRequired(apeCmd.TargetTypeFlag) + ff.Bool(apeCmd.ChainIDHexFlag, false, apeCmd.ChainIDHexFlagDesc) - addCmd.MarkFlagsMutuallyExclusive(pathFlag, ruleFlag) + addCmd.MarkFlagsMutuallyExclusive(apeCmd.PathFlag, apeCmd.RuleFlag) } diff --git a/cmd/frostfs-cli/modules/ape_manager/list_chain.go b/cmd/frostfs-cli/modules/ape_manager/list_chain.go index 9955c8c03..b07ecc52f 100644 --- a/cmd/frostfs-cli/modules/ape_manager/list_chain.go +++ b/cmd/frostfs-cli/modules/ape_manager/list_chain.go @@ -43,7 +43,7 @@ func initListCmd() { commonflags.Init(listCmd) ff := listCmd.Flags() - ff.String(targetNameFlag, "", targetNameDesc) - ff.String(targetTypeFlag, "", targetTypeDesc) - _ = listCmd.MarkFlagRequired(targetTypeFlag) + ff.String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc) + ff.String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc) + _ = listCmd.MarkFlagRequired(apeCmd.TargetTypeFlag) } diff --git a/cmd/frostfs-cli/modules/ape_manager/remove_chain.go b/cmd/frostfs-cli/modules/ape_manager/remove_chain.go index 179bd5c9e..136ca81c3 100644 --- a/cmd/frostfs-cli/modules/ape_manager/remove_chain.go +++ b/cmd/frostfs-cli/modules/ape_manager/remove_chain.go @@ -1,29 +1,23 @@ package apemanager import ( - "encoding/hex" - "errors" - internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" client_sdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" ) -var ( - errEmptyChainID = errors.New("chain id cannot be empty") - - removeCmd = &cobra.Command{ - Use: "remove", - Short: "Remove rule chain for a target", - Run: remove, - PersistentPreRun: func(cmd *cobra.Command, _ []string) { - commonflags.Bind(cmd) - }, - } -) +var removeCmd = &cobra.Command{ + Use: "remove", + Short: "Remove rule chain for a target", + Run: remove, + PersistentPreRun: func(cmd *cobra.Command, _ []string) { + commonflags.Bind(cmd) + }, +} func remove(cmd *cobra.Command, _ []string) { target := parseTarget(cmd) @@ -31,19 +25,9 @@ func remove(cmd *cobra.Command, _ []string) { key := key.Get(cmd) cli := internalclient.GetSDKClientByFlag(cmd, key, commonflags.RPC) - chainID, _ := cmd.Flags().GetString(chainIDFlag) - if chainID == "" { - commonCmd.ExitOnErr(cmd, "read chain id error: %w", errEmptyChainID) - } + chainID := apeCmd.ParseChainID(cmd) chainIDRaw := []byte(chainID) - hexEncoded, _ := cmd.Flags().GetBool(chainIDHexFlag) - if hexEncoded { - var err error - chainIDRaw, err = hex.DecodeString(chainID) - commonCmd.ExitOnErr(cmd, "can't decode chain ID as hex: %w", err) - } - _, err := cli.APEManagerRemoveChain(cmd.Context(), client_sdk.PrmAPEManagerRemoveChain{ ChainTarget: target, ChainID: chainIDRaw, @@ -58,9 +42,10 @@ func initRemoveCmd() { commonflags.Init(removeCmd) ff := removeCmd.Flags() - ff.String(targetNameFlag, "", targetNameDesc) - ff.String(targetTypeFlag, "", targetTypeDesc) - _ = removeCmd.MarkFlagRequired(targetTypeFlag) - ff.String(chainIDFlag, "", "Chain id") - ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") + ff.String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc) + ff.String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc) + _ = removeCmd.MarkFlagRequired(apeCmd.TargetTypeFlag) + ff.String(apeCmd.ChainIDFlag, "", apeCmd.ChainIDFlagDesc) + _ = removeCmd.MarkFlagRequired(apeCmd.ChainIDFlag) + ff.Bool(apeCmd.ChainIDHexFlag, false, apeCmd.ChainIDHexFlagDesc) } From 4ab4ed6f96e3538a25110e1eb5229e90e438de49 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 18 Nov 2024 14:07:14 +0300 Subject: [PATCH 1555/1943] [#1501] cli: Refactor bearer subcommand * Use `cmd/internal/common/ape` parser commands within `generate-ape-override` subcommand * Use flag names from `cmd/internal/common/ape` Signed-off-by: Airat Arifullin --- .../modules/bearer/generate_override.go | 52 +++---------------- 1 file changed, 6 insertions(+), 46 deletions(-) diff --git a/cmd/frostfs-cli/modules/bearer/generate_override.go b/cmd/frostfs-cli/modules/bearer/generate_override.go index c3df0294e..13fe07995 100644 --- a/cmd/frostfs-cli/modules/bearer/generate_override.go +++ b/cmd/frostfs-cli/modules/bearer/generate_override.go @@ -1,32 +1,20 @@ package bearer import ( - "errors" "fmt" "os" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" - parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/ape" apeSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" "github.com/spf13/cobra" ) -var ( - errChainIDCannotBeEmpty = errors.New("chain id cannot be empty") - errRuleIsNotParsed = errors.New("rule is not passed") -) - const ( - chainIDFlag = "chain-id" - chainIDHexFlag = "chain-id-hex" - ruleFlag = "rule" - pathFlag = "path" - outputFlag = "output" + outputFlag = "output" ) var generateAPEOverrideCmd = &cobra.Command{ @@ -41,7 +29,7 @@ Generated APE override can be dumped to a file in JSON format that is passed to } func genereateAPEOverride(cmd *cobra.Command, _ []string) { - c := parseChain(cmd) + c := apeCmd.ParseChain(cmd) targetCID, _ := cmd.Flags().GetString(commonflags.CIDFlag) var cid cidSDK.ID @@ -78,39 +66,11 @@ func init() { ff.StringP(commonflags.CIDFlag, "", "", "Target container ID.") _ = cobra.MarkFlagRequired(createCmd.Flags(), commonflags.CIDFlag) - ff.StringArray(ruleFlag, []string{}, "Rule statement") - ff.String(pathFlag, "", "Path to encoded chain in JSON or binary format") - ff.String(chainIDFlag, "", "Assign ID to the parsed chain") - ff.Bool(chainIDHexFlag, false, "Flag to parse chain ID as hex") + ff.StringArray(apeCmd.RuleFlag, []string{}, "Rule statement") + ff.String(apeCmd.PathFlag, "", "Path to encoded chain in JSON or binary format") + ff.String(apeCmd.ChainIDFlag, "", "Assign ID to the parsed chain") + ff.Bool(apeCmd.ChainIDHexFlag, false, "Flag to parse chain ID as hex") ff.String(outputFlag, "", "Output path to dump result JSON-encoded APE override") _ = cobra.MarkFlagFilename(createCmd.Flags(), outputFlag) } - -func parseChainID(cmd *cobra.Command) apechain.ID { - chainID, _ := cmd.Flags().GetString(chainIDFlag) - if chainID == "" { - commonCmd.ExitOnErr(cmd, "read chain id error: %w", - errChainIDCannotBeEmpty) - } - return apechain.ID(chainID) -} - -func parseChain(cmd *cobra.Command) *apechain.Chain { - chain := new(apechain.Chain) - - if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 { - commonCmd.ExitOnErr(cmd, "parser error: %w", parseutil.ParseAPEChain(chain, rules)) - } else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" { - commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", parseutil.ParseAPEChainBinaryOrJSON(chain, encPath)) - } else { - commonCmd.ExitOnErr(cmd, "parser error: %w", errRuleIsNotParsed) - } - - chain.ID = parseChainID(cmd) - - cmd.Println("Parsed chain:") - apeCmd.PrintHumanReadableAPEChain(cmd, chain) - - return chain -} From a339b52a6038f0746a54acd8f668ea57a8541fd5 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 18 Nov 2024 14:11:21 +0300 Subject: [PATCH 1556/1943] [#1501] adm: Refactor APE-chains managing subcommands * Use `cmd/internal/common/ape` parser commands within `ape` subcommands * Use flag names from `cmd/internal/common/ape Signed-off-by: Airat Arifullin --- .../internal/modules/morph/ape/ape.go | 86 ++++++--------- .../internal/modules/morph/ape/ape_util.go | 103 ++++-------------- cmd/internal/common/ape/commands.go | 7 +- 3 files changed, 58 insertions(+), 138 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go index fb363f903..8fcd4a441 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go @@ -14,26 +14,10 @@ import ( ) const ( - namespaceTarget = "namespace" - containerTarget = "container" - userTarget = "user" - groupTarget = "group" - jsonFlag = "json" - jsonFlagDesc = "Output rule chains in JSON format" - chainIDFlag = "chain-id" - chainIDDesc = "Rule chain ID" - ruleFlag = "rule" - ruleFlagDesc = "Rule chain in text format" - pathFlag = "path" - pathFlagDesc = "path to encoded chain in JSON or binary format" - targetNameFlag = "target-name" - targetNameDesc = "Resource name in APE resource name format" - targetTypeFlag = "target-type" - targetTypeDesc = "Resource type(container/namespace)" - addrAdminFlag = "addr" - addrAdminDesc = "The address of the admins wallet" - chainNameFlag = "chain-name" - chainNameFlagDesc = "Chain name(ingress|s3)" + jsonFlag = "json" + jsonFlagDesc = "Output rule chains in JSON format" + addrAdminFlag = "addr" + addrAdminDesc = "The address of the admins wallet" ) var ( @@ -101,17 +85,17 @@ func initAddRuleChainCmd() { addRuleChainCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) addRuleChainCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) - addRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc) - _ = addRuleChainCmd.MarkFlagRequired(targetTypeFlag) - addRuleChainCmd.Flags().String(targetNameFlag, "", targetNameDesc) - _ = addRuleChainCmd.MarkFlagRequired(targetNameFlag) + addRuleChainCmd.Flags().String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc) + _ = addRuleChainCmd.MarkFlagRequired(apeCmd.TargetTypeFlag) + addRuleChainCmd.Flags().String(apeCmd.TargetNameFlag, "", apeCmd.TargetTypeFlagDesc) + _ = addRuleChainCmd.MarkFlagRequired(apeCmd.TargetNameFlag) - addRuleChainCmd.Flags().String(chainIDFlag, "", chainIDDesc) - _ = addRuleChainCmd.MarkFlagRequired(chainIDFlag) - addRuleChainCmd.Flags().StringArray(ruleFlag, []string{}, ruleFlagDesc) - addRuleChainCmd.Flags().String(pathFlag, "", pathFlagDesc) - addRuleChainCmd.Flags().String(chainNameFlag, ingress, chainNameFlagDesc) - addRuleChainCmd.MarkFlagsMutuallyExclusive(ruleFlag, pathFlag) + addRuleChainCmd.Flags().String(apeCmd.ChainIDFlag, "", apeCmd.ChainIDFlagDesc) + _ = addRuleChainCmd.MarkFlagRequired(apeCmd.ChainIDFlag) + addRuleChainCmd.Flags().StringArray(apeCmd.RuleFlag, []string{}, apeCmd.RuleFlagDesc) + addRuleChainCmd.Flags().String(apeCmd.PathFlag, "", apeCmd.PathFlagDesc) + addRuleChainCmd.Flags().String(apeCmd.ChainNameFlag, apeCmd.Ingress, apeCmd.ChainNameFlagDesc) + addRuleChainCmd.MarkFlagsMutuallyExclusive(apeCmd.RuleFlag, apeCmd.PathFlag) } func initRemoveRuleChainCmd() { @@ -120,26 +104,25 @@ func initRemoveRuleChainCmd() { removeRuleChainCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) removeRuleChainCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) - removeRuleChainCmd.Flags().String(targetTypeFlag, "", targetTypeDesc) - _ = removeRuleChainCmd.MarkFlagRequired(targetTypeFlag) - removeRuleChainCmd.Flags().String(targetNameFlag, "", targetNameDesc) - _ = removeRuleChainCmd.MarkFlagRequired(targetNameFlag) - removeRuleChainCmd.Flags().String(chainIDFlag, "", chainIDDesc) - removeRuleChainCmd.Flags().String(chainNameFlag, ingress, chainNameFlagDesc) + removeRuleChainCmd.Flags().String(apeCmd.TargetTypeFlag, "", apeCmd.TargetTypeFlagDesc) + _ = removeRuleChainCmd.MarkFlagRequired(apeCmd.TargetTypeFlag) + removeRuleChainCmd.Flags().String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc) + _ = removeRuleChainCmd.MarkFlagRequired(apeCmd.TargetNameFlag) + removeRuleChainCmd.Flags().String(apeCmd.ChainIDFlag, "", apeCmd.ChainIDFlagDesc) + removeRuleChainCmd.Flags().String(apeCmd.ChainNameFlag, apeCmd.Ingress, apeCmd.ChainNameFlagDesc) removeRuleChainCmd.Flags().Bool(commonflags.AllFlag, false, "Remove all chains for target") - removeRuleChainCmd.MarkFlagsMutuallyExclusive(commonflags.AllFlag, chainIDFlag) + removeRuleChainCmd.MarkFlagsMutuallyExclusive(commonflags.AllFlag, apeCmd.ChainIDFlag) } func initListRuleChainsCmd() { Cmd.AddCommand(listRuleChainsCmd) listRuleChainsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) - listRuleChainsCmd.Flags().StringP(targetTypeFlag, "t", "", targetTypeDesc) - _ = listRuleChainsCmd.MarkFlagRequired(targetTypeFlag) - listRuleChainsCmd.Flags().String(targetNameFlag, "", targetNameDesc) - _ = listRuleChainsCmd.MarkFlagRequired(targetNameFlag) + listRuleChainsCmd.Flags().StringP(apeCmd.TargetTypeFlag, "t", "", apeCmd.TargetTypeFlagDesc) + _ = listRuleChainsCmd.MarkFlagRequired(apeCmd.TargetTypeFlag) + listRuleChainsCmd.Flags().String(apeCmd.TargetNameFlag, "", apeCmd.TargetNameFlagDesc) listRuleChainsCmd.Flags().Bool(jsonFlag, false, jsonFlagDesc) - listRuleChainsCmd.Flags().String(chainNameFlag, ingress, chainNameFlagDesc) + listRuleChainsCmd.Flags().String(apeCmd.ChainNameFlag, apeCmd.Ingress, apeCmd.ChainNameFlagDesc) } func initSetAdminCmd() { @@ -161,15 +144,15 @@ func initListTargetsCmd() { Cmd.AddCommand(listTargetsCmd) listTargetsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) - listTargetsCmd.Flags().StringP(targetTypeFlag, "t", "", targetTypeDesc) - _ = listTargetsCmd.MarkFlagRequired(targetTypeFlag) + listTargetsCmd.Flags().StringP(apeCmd.TargetTypeFlag, "t", "", apeCmd.TargetTypeFlagDesc) + _ = listTargetsCmd.MarkFlagRequired(apeCmd.TargetTypeFlag) } func addRuleChain(cmd *cobra.Command, _ []string) { - chain := parseChain(cmd) + chain := apeCmd.ParseChain(cmd) target := parseTarget(cmd) pci, ac := newPolicyContractInterface(cmd) - h, vub, err := pci.AddMorphRuleChain(parseChainName(cmd), target, chain) + h, vub, err := pci.AddMorphRuleChain(apeCmd.ParseChainName(cmd), target, chain) cmd.Println("Waiting for transaction to persist...") _, err = ac.Wait(h, vub, err) commonCmd.ExitOnErr(cmd, "add rule chain error: %w", err) @@ -181,14 +164,14 @@ func removeRuleChain(cmd *cobra.Command, _ []string) { pci, ac := newPolicyContractInterface(cmd) removeAll, _ := cmd.Flags().GetBool(commonflags.AllFlag) if removeAll { - h, vub, err := pci.RemoveMorphRuleChainsByTarget(parseChainName(cmd), target) + h, vub, err := pci.RemoveMorphRuleChainsByTarget(apeCmd.ParseChainName(cmd), target) cmd.Println("Waiting for transaction to persist...") _, err = ac.Wait(h, vub, err) commonCmd.ExitOnErr(cmd, "remove rule chain error: %w", err) cmd.Println("All chains for target removed successfully") } else { - chainID := parseChainID(cmd) - h, vub, err := pci.RemoveMorphRuleChain(parseChainName(cmd), target, chainID) + chainID := apeCmd.ParseChainID(cmd) + h, vub, err := pci.RemoveMorphRuleChain(apeCmd.ParseChainName(cmd), target, chainID) cmd.Println("Waiting for transaction to persist...") _, err = ac.Wait(h, vub, err) commonCmd.ExitOnErr(cmd, "remove rule chain error: %w", err) @@ -199,7 +182,7 @@ func removeRuleChain(cmd *cobra.Command, _ []string) { func listRuleChains(cmd *cobra.Command, _ []string) { target := parseTarget(cmd) pci, _ := newPolicyContractReaderInterface(cmd) - chains, err := pci.ListMorphRuleChains(parseChainName(cmd), target) + chains, err := pci.ListMorphRuleChains(apeCmd.ParseChainName(cmd), target) commonCmd.ExitOnErr(cmd, "list rule chains error: %w", err) if len(chains) == 0 { return @@ -235,8 +218,7 @@ func getAdmin(cmd *cobra.Command, _ []string) { } func listTargets(cmd *cobra.Command, _ []string) { - typ, err := parseTargetType(cmd) - commonCmd.ExitOnErr(cmd, "parse target type error: %w", err) + typ := apeCmd.ParseTargetType(cmd) pci, inv := newPolicyContractReaderInterface(cmd) sid, it, err := pci.ListTargetsIterator(typ) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 1a70dd4a2..6780e6dd3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -2,14 +2,12 @@ package ape import ( "errors" - "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" - parseutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/ape" - apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" policyengine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" morph "git.frostfs.info/TrueCloudLab/policy-engine/pkg/morph/policy" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -19,90 +17,29 @@ import ( "github.com/spf13/viper" ) -const ( - ingress = "ingress" - s3 = "s3" -) - -var mChainName = map[string]apechain.Name{ - ingress: apechain.Ingress, - s3: apechain.S3, -} - -var ( - errUnknownTargetType = errors.New("unknown target type") - errChainIDCannotBeEmpty = errors.New("chain id cannot be empty") - errRuleIsNotParsed = errors.New("rule is not passed") - errUnsupportedChainName = errors.New("unsupported chain name") -) +var errUnknownTargetType = errors.New("unknown target type") func parseTarget(cmd *cobra.Command) policyengine.Target { - name, _ := cmd.Flags().GetString(targetNameFlag) - typ, err := parseTargetType(cmd) - - // interpret "root" namespace as empty - if typ == policyengine.Namespace && name == "root" { - name = "" - } - - commonCmd.ExitOnErr(cmd, "read target type error: %w", err) - - return policyengine.Target{ - Name: name, - Type: typ, - } -} - -func parseTargetType(cmd *cobra.Command) (policyengine.TargetType, error) { - typ, _ := cmd.Flags().GetString(targetTypeFlag) + typ := apeCmd.ParseTargetType(cmd) + name, _ := cmd.Flags().GetString(apeCmd.TargetNameFlag) switch typ { - case namespaceTarget: - return policyengine.Namespace, nil - case containerTarget: - return policyengine.Container, nil - case userTarget: - return policyengine.User, nil - case groupTarget: - return policyengine.Group, nil + case policyengine.Namespace: + if name == "root" { + name = "" + } + return policyengine.NamespaceTarget(name) + case policyengine.Container: + var cnr cid.ID + commonCmd.ExitOnErr(cmd, "can't decode container ID: %w", cnr.DecodeString(name)) + return policyengine.ContainerTarget(name) + case policyengine.User: + return policyengine.UserTarget(name) + case policyengine.Group: + return policyengine.GroupTarget(name) + default: + commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType) } - return -1, errUnknownTargetType -} - -func parseChainID(cmd *cobra.Command) apechain.ID { - chainID, _ := cmd.Flags().GetString(chainIDFlag) - if chainID == "" { - commonCmd.ExitOnErr(cmd, "read chain id error: %w", - errChainIDCannotBeEmpty) - } - return apechain.ID(chainID) -} - -func parseChain(cmd *cobra.Command) *apechain.Chain { - chain := new(apechain.Chain) - - if rules, _ := cmd.Flags().GetStringArray(ruleFlag); len(rules) > 0 { - commonCmd.ExitOnErr(cmd, "parser error: %w", parseutil.ParseAPEChain(chain, rules)) - } else if encPath, _ := cmd.Flags().GetString(pathFlag); encPath != "" { - commonCmd.ExitOnErr(cmd, "decode binary or json error: %w", parseutil.ParseAPEChainBinaryOrJSON(chain, encPath)) - } else { - commonCmd.ExitOnErr(cmd, "parser error: %w", errRuleIsNotParsed) - } - - chain.ID = parseChainID(cmd) - - cmd.Println("Parsed chain:") - apeCmd.PrintHumanReadableAPEChain(cmd, chain) - - return chain -} - -func parseChainName(cmd *cobra.Command) apechain.Name { - chainName, _ := cmd.Flags().GetString(chainNameFlag) - apeChainName, ok := mChainName[strings.ToLower(chainName)] - if !ok { - commonCmd.ExitOnErr(cmd, "", errUnsupportedChainName) - } - return apeChainName + panic("unreachable") } // invokerAdapter adapats invoker.Invoker to ContractStorageInvoker interface. diff --git a/cmd/internal/common/ape/commands.go b/cmd/internal/common/ape/commands.go index 354747330..e5a35ab71 100644 --- a/cmd/internal/common/ape/commands.go +++ b/cmd/internal/common/ape/commands.go @@ -97,7 +97,7 @@ func ParseTarget(cmd *cobra.Command) engine.Target { default: commonCmd.ExitOnErr(cmd, "read target type error: %w", errUnknownTargetType) } - return engine.Target{} + panic("unreachable") } // ParseTargetType handles target type parsing of an APE chain. @@ -112,9 +112,10 @@ func ParseTargetType(cmd *cobra.Command) engine.TargetType { return engine.User case groupTarget: return engine.Group + default: + commonCmd.ExitOnErr(cmd, "parse target type error: %w", errUnknownTargetType) } - commonCmd.ExitOnErr(cmd, "parse target type error: %w", errUnknownTargetType) - return engine.TargetType(0) + panic("unreachable") } // ParseChainID handles the parsing of APE-chain identifier. From 3042490340fcfe223fa8069006876f84e1721082 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 20 Nov 2024 10:43:38 +0300 Subject: [PATCH 1557/1943] [#1507] timer: Remove unused OnDelta() method Signed-off-by: Evgenii Stratonikov --- pkg/morph/timer/block.go | 73 +---------------- pkg/morph/timer/block_test.go | 142 ++++------------------------------ 2 files changed, 17 insertions(+), 198 deletions(-) diff --git a/pkg/morph/timer/block.go b/pkg/morph/timer/block.go index be20d3571..974be1120 100644 --- a/pkg/morph/timer/block.go +++ b/pkg/morph/timer/block.go @@ -15,41 +15,19 @@ type BlockTickHandler func() // It can tick the blocks and perform certain actions // on block time intervals. type BlockTimer struct { - rolledBack bool - mtx sync.Mutex dur BlockMeter baseDur uint32 - mul, div uint32 - cur, tgt uint32 last uint32 h BlockTickHandler - ps []BlockTimer - once bool - - deltaCfg -} - -// DeltaOption is an option of delta-interval handler. -type DeltaOption func(*deltaCfg) - -type deltaCfg struct { - pulse bool -} - -// WithPulse returns option to call delta-interval handler multiple times. -func WithPulse() DeltaOption { - return func(c *deltaCfg) { - c.pulse = true - } } // StaticBlockMeter returns BlockMeters that always returns (d, nil). @@ -65,52 +43,19 @@ func StaticBlockMeter(d uint32) BlockMeter { func NewBlockTimer(dur BlockMeter, h BlockTickHandler) *BlockTimer { return &BlockTimer{ dur: dur, - mul: 1, - div: 1, h: h, - deltaCfg: deltaCfg{ - pulse: true, - }, } } // NewOneTickTimer creates a new BlockTimer that ticks only once. -// -// Do not use delta handlers with pulse in this timer. func NewOneTickTimer(dur BlockMeter, h BlockTickHandler) *BlockTimer { return &BlockTimer{ dur: dur, - mul: 1, - div: 1, h: h, once: true, } } -// OnDelta registers handler which is executed on (mul / div * BlockMeter()) block -// after basic interval reset. -// -// If WithPulse option is provided, handler is executed (mul / div * BlockMeter()) block -// during base interval. -func (t *BlockTimer) OnDelta(mul, div uint32, h BlockTickHandler, opts ...DeltaOption) { - c := deltaCfg{ - pulse: false, - } - - for i := range opts { - opts[i](&c) - } - - t.ps = append(t.ps, BlockTimer{ - mul: mul, - div: div, - h: h, - once: t.once, - - deltaCfg: c, - }) -} - // Reset resets previous ticks of the BlockTimer. // // Returns BlockMeter's error upon occurrence. @@ -124,29 +69,18 @@ func (t *BlockTimer) Reset() error { t.resetWithBaseInterval(d) - for i := range t.ps { - t.ps[i].resetWithBaseInterval(d) - } - t.mtx.Unlock() return nil } func (t *BlockTimer) resetWithBaseInterval(d uint32) { - t.rolledBack = false t.baseDur = d t.reset() } func (t *BlockTimer) reset() { - mul, div := t.mul, t.div - - if !t.pulse && t.rolledBack && mul < div { - mul, div = 1, 1 - } - - delta := mul * t.baseDur / div + delta := t.baseDur if delta == 0 { delta = 1 } @@ -180,12 +114,7 @@ func (t *BlockTimer) tick(h uint32) { if !t.once { t.cur = 0 - t.rolledBack = true t.reset() } } - - for i := range t.ps { - t.ps[i].tick(h) - } } diff --git a/pkg/morph/timer/block_test.go b/pkg/morph/timer/block_test.go index ee6091845..c0af6c5c3 100644 --- a/pkg/morph/timer/block_test.go +++ b/pkg/morph/timer/block_test.go @@ -63,85 +63,6 @@ func TestBlockTimer(t *testing.T) { tickN(bt, intervalNum*blockDur) require.Equal(t, intervalNum, uint32(baseCallCounter)) - - // add half-interval handler - halfCallCounter := uint32(0) - - bt.OnDelta(1, 2, func() { - halfCallCounter++ - }) - - // add double interval handler - doubleCallCounter := uint32(0) - - bt.OnDelta(2, 1, func() { - doubleCallCounter++ - }) - - require.NoError(t, bt.Reset()) - - baseCallCounter = 0 - intervalNum = 20 - - tickN(bt, intervalNum*blockDur) - - require.Equal(t, intervalNum, uint32(halfCallCounter)) - require.Equal(t, intervalNum, uint32(baseCallCounter)) - require.Equal(t, intervalNum/2, uint32(doubleCallCounter)) -} - -func TestDeltaPulse(t *testing.T) { - blockDur := uint32(9) - baseCallCounter := uint32(0) - - bt := timer.NewBlockTimer(timer.StaticBlockMeter(blockDur), func() { - baseCallCounter++ - }) - - deltaCallCounter := uint32(0) - - div := uint32(3) - - bt.OnDelta(1, div, func() { - deltaCallCounter++ - }, timer.WithPulse()) - - require.NoError(t, bt.Reset()) - - intervalNum := uint32(7) - - tickN(bt, intervalNum*blockDur) - - require.Equal(t, intervalNum, uint32(baseCallCounter)) - require.Equal(t, intervalNum*div, uint32(deltaCallCounter)) -} - -func TestDeltaReset(t *testing.T) { - blockDur := uint32(6) - baseCallCounter := 0 - - bt := timer.NewBlockTimer(timer.StaticBlockMeter(blockDur), func() { - baseCallCounter++ - }) - - detlaCallCounter := 0 - - bt.OnDelta(1, 3, func() { - detlaCallCounter++ - }) - - require.NoError(t, bt.Reset()) - - tickN(bt, 6) - - require.Equal(t, 1, baseCallCounter) - require.Equal(t, 1, detlaCallCounter) - - require.NoError(t, bt.Reset()) - - tickN(bt, 3) - - require.Equal(t, 2, detlaCallCounter) } func TestNewOneTickTimer(t *testing.T) { @@ -168,82 +89,51 @@ func TestNewOneTickTimer(t *testing.T) { tickN(bt, 10) require.Equal(t, 1, baseCallCounter) }) - - t.Run("delta without pulse", func(t *testing.T) { - blockDur = uint32(10) - baseCallCounter = 0 - - bt = timer.NewOneTickTimer(timer.StaticBlockMeter(blockDur), func() { - baseCallCounter++ - }) - - detlaCallCounter := 0 - - bt.OnDelta(1, 10, func() { - detlaCallCounter++ - }) - - require.NoError(t, bt.Reset()) - - tickN(bt, 10) - require.Equal(t, 1, baseCallCounter) - require.Equal(t, 1, detlaCallCounter) - - tickN(bt, 10) // 10 more ticks must not affect counters - require.Equal(t, 1, baseCallCounter) - require.Equal(t, 1, detlaCallCounter) - }) } func TestBlockTimer_TickSameHeight(t *testing.T) { - var baseCounter, deltaCounter int + var baseCounter int blockDur := uint32(2) bt := timer.NewBlockTimer( func() (uint32, error) { return blockDur, nil }, func() { baseCounter++ }) - bt.OnDelta(2, 1, func() { - deltaCounter++ - }) require.NoError(t, bt.Reset()) - check := func(t *testing.T, h uint32, base, delta int) { + check := func(t *testing.T, h uint32, base int) { for range 2 * int(blockDur) { bt.Tick(h) require.Equal(t, base, baseCounter) - require.Equal(t, delta, deltaCounter) } } - check(t, 1, 0, 0) - check(t, 2, 1, 0) - check(t, 3, 1, 0) - check(t, 4, 2, 1) + check(t, 1, 0) + check(t, 2, 1) + check(t, 3, 1) + check(t, 4, 2) t.Run("works the same way after `Reset()`", func(t *testing.T) { t.Run("same block duration", func(t *testing.T) { require.NoError(t, bt.Reset()) baseCounter = 0 - deltaCounter = 0 - check(t, 1, 0, 0) - check(t, 2, 1, 0) - check(t, 3, 1, 0) - check(t, 4, 2, 1) + check(t, 1, 0) + check(t, 2, 1) + check(t, 3, 1) + check(t, 4, 2) }) t.Run("different block duration", func(t *testing.T) { blockDur = 3 require.NoError(t, bt.Reset()) baseCounter = 0 - deltaCounter = 0 - check(t, 1, 0, 0) - check(t, 2, 0, 0) - check(t, 3, 1, 0) - check(t, 4, 1, 0) - check(t, 5, 1, 0) - check(t, 6, 2, 1) + check(t, 1, 0) + check(t, 2, 0) + check(t, 3, 1) + check(t, 4, 1) + check(t, 5, 1) + check(t, 6, 2) }) }) } From 2e974f734c38161c251e6556daacbc8fbf4fbfd3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 20 Nov 2024 10:53:33 +0300 Subject: [PATCH 1558/1943] [#1507] timer/test: Improve test coverage Signed-off-by: Evgenii Stratonikov --- pkg/morph/timer/block_test.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/pkg/morph/timer/block_test.go b/pkg/morph/timer/block_test.go index c0af6c5c3..615631dcb 100644 --- a/pkg/morph/timer/block_test.go +++ b/pkg/morph/timer/block_test.go @@ -1,6 +1,7 @@ package timer_test import ( + "errors" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/timer" @@ -48,6 +49,38 @@ func TestIRBlockTimer_Reset(t *testing.T) { require.Equal(t, baseCounter[0], baseCounter[1]) } +func TestBlockTimer_ResetChangeDuration(t *testing.T) { + var dur uint32 = 2 + var err error + var counter int + + bt := timer.NewBlockTimer( + func() (uint32, error) { return dur, err }, + func() { counter++ }) + + require.NoError(t, bt.Reset()) + + tickN(bt, 2) + require.Equal(t, 1, counter) + + t.Run("return error", func(t *testing.T) { + dur = 5 + err = errors.New("my awesome error") + require.ErrorIs(t, bt.Reset(), err) + + tickN(bt, 2) + require.Equal(t, 2, counter) + }) + t.Run("change duration", func(t *testing.T) { + dur = 5 + err = nil + require.NoError(t, bt.Reset()) + + tickN(bt, 5) + require.Equal(t, 3, counter) + }) +} + func TestBlockTimer(t *testing.T) { blockDur := uint32(10) baseCallCounter := uint32(0) From 49a4e727fd864da271c72ee6a53f1b315a959efd Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 20 Nov 2024 10:55:24 +0300 Subject: [PATCH 1559/1943] [#1507] timer/test: Use const for constants Make it easy to see what the test is about. Signed-off-by: Evgenii Stratonikov --- pkg/morph/timer/block_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/morph/timer/block_test.go b/pkg/morph/timer/block_test.go index 615631dcb..a144b3db6 100644 --- a/pkg/morph/timer/block_test.go +++ b/pkg/morph/timer/block_test.go @@ -18,7 +18,7 @@ func tickN(t *timer.BlockTimer, n uint32) { // "resetting" consists of ticking the current height as well and invoking `Reset`. func TestIRBlockTimer_Reset(t *testing.T) { var baseCounter [2]int - blockDur := uint32(3) + const blockDur = uint32(3) bt1 := timer.NewBlockTimer( func() (uint32, error) { return blockDur, nil }, @@ -82,7 +82,7 @@ func TestBlockTimer_ResetChangeDuration(t *testing.T) { } func TestBlockTimer(t *testing.T) { - blockDur := uint32(10) + const blockDur = uint32(10) baseCallCounter := uint32(0) bt := timer.NewBlockTimer(timer.StaticBlockMeter(blockDur), func() { From 6ae8667fb4a7c00fc5d3a72ff181a8da54a0eba3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 20 Nov 2024 11:40:13 +0300 Subject: [PATCH 1560/1943] [#1509] .forgejo: Run actions on push to master Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/build.yml | 6 +++++- .forgejo/workflows/pre-commit.yml | 7 ++++++- .forgejo/workflows/tests.yml | 7 ++++++- .forgejo/workflows/vulncheck.yml | 7 ++++++- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index ce2d64dd9..9129d136e 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -1,6 +1,10 @@ name: Build -on: [pull_request] +on: + pull_request: + push: + branches: + - master jobs: build: diff --git a/.forgejo/workflows/pre-commit.yml b/.forgejo/workflows/pre-commit.yml index 8b06a2fdf..b27e7a39a 100644 --- a/.forgejo/workflows/pre-commit.yml +++ b/.forgejo/workflows/pre-commit.yml @@ -1,5 +1,10 @@ name: Pre-commit hooks -on: [pull_request] + +on: + pull_request: + push: + branches: + - master jobs: precommit: diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 07ba5c268..4f1bebe61 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -1,5 +1,10 @@ name: Tests and linters -on: [pull_request] + +on: + pull_request: + push: + branches: + - master jobs: lint: diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 2951a8059..cf15005b1 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -1,5 +1,10 @@ name: Vulncheck -on: [pull_request] + +on: + pull_request: + push: + branches: + - master jobs: vulncheck: From 2e2c62147db434c3834be92e7ebcc3ed80a44a57 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 20 Nov 2024 14:16:17 +0300 Subject: [PATCH 1561/1943] [#1513] adm: Move ProtoConfigPath from `constants` to `commonflags` package Refs #932 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/commonflags/flags.go | 1 + cmd/frostfs-adm/internal/modules/morph/constants/const.go | 1 - .../internal/modules/morph/helper/local_client.go | 3 ++- .../internal/modules/morph/initialize/initialize_test.go | 2 +- cmd/frostfs-adm/internal/modules/morph/initialize/root.go | 5 ++--- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-adm/internal/commonflags/flags.go b/cmd/frostfs-adm/internal/commonflags/flags.go index 81395edb0..87692d013 100644 --- a/cmd/frostfs-adm/internal/commonflags/flags.go +++ b/cmd/frostfs-adm/internal/commonflags/flags.go @@ -20,6 +20,7 @@ const ( AlphabetWalletsFlagDesc = "Path to alphabet wallets dir" LocalDumpFlag = "local-dump" + ProtoConfigPath = "protocol" ContractsInitFlag = "contracts" ContractsInitFlagDesc = "Path to archive with compiled FrostFS contracts (the default is to fetch the latest release from the official repository)" ContractsURLFlag = "contracts-url" diff --git a/cmd/frostfs-adm/internal/modules/morph/constants/const.go b/cmd/frostfs-adm/internal/modules/morph/constants/const.go index a3b4f129a..be4041a86 100644 --- a/cmd/frostfs-adm/internal/modules/morph/constants/const.go +++ b/cmd/frostfs-adm/internal/modules/morph/constants/const.go @@ -4,7 +4,6 @@ import "time" const ( ConsensusAccountName = "consensus" - ProtoConfigPath = "protocol" // MaxAlphabetNodes is the maximum number of candidates allowed, which is currently limited by the size // of the invocation script. diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go index 375fa84d7..ed028fb7c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go @@ -8,6 +8,7 @@ import ( "sort" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/config" @@ -47,7 +48,7 @@ type LocalClient struct { } func NewLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet, dumpPath string) (*LocalClient, error) { - cfg, err := config.LoadFile(v.GetString(constants.ProtoConfigPath)) + cfg, err := config.LoadFile(v.GetString(commonflags.ProtoConfigPath)) if err != nil { return nil, err } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go index 74f5d3e88..9bc51c055 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_test.go @@ -62,7 +62,7 @@ func testInitialize(t *testing.T, committeeSize int) { v := viper.GetViper() require.NoError(t, generateTestData(testdataDir, committeeSize)) - v.Set(constants.ProtoConfigPath, filepath.Join(testdataDir, protoFileName)) + v.Set(commonflags.ProtoConfigPath, filepath.Join(testdataDir, protoFileName)) // Set to the path or remove the next statement to download from the network. require.NoError(t, Cmd.Flags().Set(commonflags.ContractsInitFlag, contractsPath)) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/root.go b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go index b7885c512..50f14e728 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/root.go @@ -2,7 +2,6 @@ package initialize import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -32,7 +31,7 @@ var Cmd = &cobra.Command{ _ = viper.BindPFlag(commonflags.ContainerFeeInitFlag, cmd.Flags().Lookup(containerFeeCLIFlag)) _ = viper.BindPFlag(commonflags.ContainerAliasFeeInitFlag, cmd.Flags().Lookup(containerAliasFeeCLIFlag)) _ = viper.BindPFlag(commonflags.WithdrawFeeInitFlag, cmd.Flags().Lookup(withdrawFeeCLIFlag)) - _ = viper.BindPFlag(constants.ProtoConfigPath, cmd.Flags().Lookup(constants.ProtoConfigPath)) + _ = viper.BindPFlag(commonflags.ProtoConfigPath, cmd.Flags().Lookup(commonflags.ProtoConfigPath)) }, RunE: initializeSideChainCmd, } @@ -48,7 +47,7 @@ func initInitCmd() { // Defaults are taken from neo-preodolenie. Cmd.Flags().Uint64(containerFeeCLIFlag, 1000, "Container registration fee") Cmd.Flags().Uint64(containerAliasFeeCLIFlag, 500, "Container alias fee") - Cmd.Flags().String(constants.ProtoConfigPath, "", "Path to the consensus node configuration") + Cmd.Flags().String(commonflags.ProtoConfigPath, "", "Path to the consensus node configuration") Cmd.Flags().String(commonflags.LocalDumpFlag, "", "Path to the blocks dump file") Cmd.MarkFlagsMutuallyExclusive(commonflags.ContractsInitFlag, commonflags.ContractsURLFlag) } From f12f04199e8437a9a6f440cac7ef5dfdd4454efc Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 21 Nov 2024 13:17:16 +0300 Subject: [PATCH 1562/1943] [#1516] traverser: Check for placement vector out of range Placement vector may contain fewer nodes count than it required by policy due to the outage of the one of the node. Signed-off-by: Anton Nikiforov --- .../object_manager/placement/traverser.go | 5 +- .../placement/traverser_test.go | 46 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 6440f187d..7c720b204 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -114,8 +114,9 @@ func NewTraverser(opts ...Option) (*Traverser, error) { var unsortedVector []netmap.NodeInfo var regularVector []netmap.NodeInfo for i := range rem { - unsortedVector = append(unsortedVector, ns[i][:rem[i]]...) - regularVector = append(regularVector, ns[i][rem[i]:]...) + pivot := min(len(ns[i]), rem[i]) + unsortedVector = append(unsortedVector, ns[i][:pivot]...) + regularVector = append(regularVector, ns[i][pivot:]...) } rem = []int{-1, -1} diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index 38f62aa07..f96e5c8a7 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -356,6 +356,52 @@ func TestTraverserPriorityMetrics(t *testing.T) { require.Nil(t, next) }) + t.Run("one rep one metric fewer nodes", func(t *testing.T) { + selectors := []int{2} + replicas := []int{3} + + nodes, cnr := testPlacement(selectors, replicas) + + // Node_0, PK - ip4/0.0.0.0/tcp/0 + nodes[0][0].SetAttribute("ClusterName", "A") + // Node_1, PK - ip4/0.0.0.0/tcp/1 + nodes[0][1].SetAttribute("ClusterName", "B") + + sdkNode := testNode(5) + sdkNode.SetAttribute("ClusterName", "B") + + nodesCopy := copyVectors(nodes) + + m := []Metric{NewAttributeMetric("ClusterName")} + + tr, err := NewTraverser( + ForContainer(cnr), + UseBuilder(&testBuilder{ + vectors: nodesCopy, + }), + WithoutSuccessTracking(), + WithPriorityMetrics(m), + WithNodeState(&nodeState{ + node: &sdkNode, + }), + ) + require.NoError(t, err) + + // Without priority metric `ClusterName` the order will be: + // [ {Node_0 A}, {Node_1 A} ] + // With priority metric `ClusterName` and current node in cluster B + // the order should be: + // [ {Node_1 B}, {Node_0 A} ] + next := tr.Next() + require.NotNil(t, next) + require.Equal(t, 2, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[0].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[1].PublicKey())) + + next = tr.Next() + require.Nil(t, next) + }) + t.Run("two reps two metrics", func(t *testing.T) { selectors := []int{3, 3} replicas := []int{2, 2} From efa4ce00b8eef3b896d94f0119edcdf7cbbc24b8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 21 Nov 2024 10:10:56 +0300 Subject: [PATCH 1563/1943] [#1514] go.mod: Update frostfs-contract to v0.21.0-rc.3 Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 84f9f9a66..c538a3178 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.0 + git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.0-rc.4 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 diff --git a/go.sum b/go.sum index d93e9b74e..064f3274e 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.0 h1:8Z5iPhieCrbcdhxBuY/Bajh6V5fki7Whh0b4S2zYJYU= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.20.0/go.mod h1:Y2Xorxc8SBO4phoek7n3XxaPZz5rIrFgDsU4TOjmlGA= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.0-rc.4 h1:o3iqVmbvFsfe8kpB2Hvuix6Q/tAhbiPLP91xK4lmoBQ= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.0-rc.4/go.mod h1:5fSm/l5xSjGWqsPUffSdboiGFUHa7y/1S0fvxzQowN8= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d h1:uJ/wvuMdepbkaV8XMS5uN9B0FQWMep0CttSuDZiDhq0= @@ -267,6 +267,7 @@ github.com/ssgreg/journald v1.0.0/go.mod h1:RUckwmTM8ghGWPslq2+ZBZzbb9/2KgjzYZ4J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= From 2771fdb8c7916c39dc855a0ee1a8eae97bb10205 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 21 Nov 2024 10:39:04 +0300 Subject: [PATCH 1564/1943] [#1514] adm/nns: Use nns.GetAllRecords() wrapper It was not possible previously, because GetAllRecords() was not declared safe in frostfs-contract. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/nns/record.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/record.go b/cmd/frostfs-adm/internal/modules/morph/nns/record.go index 66bb1b94f..32c88481f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/record.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/record.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/spf13/cobra" ) @@ -77,11 +76,11 @@ func addRecord(cmd *cobra.Command, _ []string) { } func getRecords(cmd *cobra.Command, _ []string) { - c, act, hash := getRPCClient(cmd) + c, act, _ := getRPCClient(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) if recordType == "" { - sid, r, err := unwrap.SessionIterator(act.Invoker.Call(hash, "getAllRecords", name)) + sid, r, err := c.GetAllRecords(name) commonCmd.ExitOnErr(cmd, "unable to get records: %w", err) defer func() { _ = act.Invoker.TerminateSession(sid) From 9073e555db7f0f5e0a389ad3ac9c23885c2df67f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 21 Nov 2024 10:43:53 +0300 Subject: [PATCH 1565/1943] [#1514] adm/nns: Do not create actor for readonly commands `nns get-records` and `nns tokens` command do not need to sign anything, so remove useless actor and use invoker directly. `NewLocalActor()` is only used in `ape` and `nns` packages. `ape` package seem to use it correctly, only when alphabet wallets are provided, so no changes there. Also, remove --alphabet-wallets flag from commands that do not need it. Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/helper/actor.go | 48 +++++++------------ .../internal/modules/morph/nns/helper.go | 13 +++++ .../internal/modules/morph/nns/record.go | 9 ++-- .../internal/modules/morph/nns/tokens.go | 5 +- 4 files changed, 36 insertions(+), 39 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/actor.go b/cmd/frostfs-adm/internal/modules/morph/helper/actor.go index ff0421335..eb0444408 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/actor.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/actor.go @@ -38,38 +38,24 @@ func NewLocalActor(cmd *cobra.Command, c actor.RPCActor, accName string) (*Local walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) var act *actor.Actor var accounts []*wallet.Account - if walletDir == "" { - account, err := wallet.NewAccount() - commonCmd.ExitOnErr(cmd, "unable to create dummy account: %w", err) - act, err = actor.New(c, []actor.SignerAccount{{ - Signer: transaction.Signer{ - Account: account.Contract.ScriptHash(), - Scopes: transaction.Global, - }, - Account: account, - }}) - if err != nil { - return nil, err - } - } else { - wallets, err := GetAlphabetWallets(viper.GetViper(), walletDir) - commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) - for _, w := range wallets { - acc, err := GetWalletAccount(w, accName) - commonCmd.ExitOnErr(cmd, fmt.Sprintf("can't find %s account: %%w", accName), err) - accounts = append(accounts, acc) - } - act, err = actor.New(c, []actor.SignerAccount{{ - Signer: transaction.Signer{ - Account: accounts[0].Contract.ScriptHash(), - Scopes: transaction.Global, - }, - Account: accounts[0], - }}) - if err != nil { - return nil, err - } + wallets, err := GetAlphabetWallets(viper.GetViper(), walletDir) + commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) + + for _, w := range wallets { + acc, err := GetWalletAccount(w, accName) + commonCmd.ExitOnErr(cmd, fmt.Sprintf("can't find %s account: %%w", accName), err) + accounts = append(accounts, acc) + } + act, err = actor.New(c, []actor.SignerAccount{{ + Signer: transaction.Signer{ + Account: accounts[0].Contract.ScriptHash(), + Scopes: transaction.Global, + }, + Account: accounts[0], + }}) + if err != nil { + return nil, err } return &LocalActor{ neoActor: act, diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go index de439acd1..68a629c3c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/cobra" @@ -24,3 +25,15 @@ func getRPCClient(cmd *cobra.Command) (*client.Contract, *helper.LocalActor, uti commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) return client.New(ac, nnsCs.Hash), ac, nnsCs.Hash } + +func nnsReader(cmd *cobra.Command) (*client.ContractReader, *invoker.Invoker) { + c, err := helper.GetN3Client(viper.GetViper()) + commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) + + inv := invoker.New(c, nil) + r := management.NewReader(inv) + nnsCs, err := helper.GetContractByID(r, 1) + commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) + + return client.NewReader(inv, nnsCs.Hash), inv +} diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/record.go b/cmd/frostfs-adm/internal/modules/morph/nns/record.go index 32c88481f..5a6db6e72 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/record.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/record.go @@ -28,7 +28,6 @@ func initAddRecordCmd() { func initGetRecordsCmd() { Cmd.AddCommand(getRecordsCmd) getRecordsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) - getRecordsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) getRecordsCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) getRecordsCmd.Flags().String(nnsRecordTypeFlag, "", nnsRecordTypeFlagDesc) @@ -76,16 +75,16 @@ func addRecord(cmd *cobra.Command, _ []string) { } func getRecords(cmd *cobra.Command, _ []string) { - c, act, _ := getRPCClient(cmd) + c, inv := nnsReader(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) if recordType == "" { sid, r, err := c.GetAllRecords(name) commonCmd.ExitOnErr(cmd, "unable to get records: %w", err) defer func() { - _ = act.Invoker.TerminateSession(sid) + _ = inv.TerminateSession(sid) }() - items, err := act.Invoker.TraverseIterator(sid, &r, 0) + items, err := inv.TraverseIterator(sid, &r, 0) commonCmd.ExitOnErr(cmd, "unable to get records: %w", err) for len(items) != 0 { for j := range items { @@ -96,7 +95,7 @@ func getRecords(cmd *cobra.Command, _ []string) { recordTypeToString(nns.RecordType(rs[1].Value().(*big.Int).Int64())), string(bs)) } - items, err = act.Invoker.TraverseIterator(sid, &r, 0) + items, err = inv.TraverseIterator(sid, &r, 0) commonCmd.ExitOnErr(cmd, "unable to get records: %w", err) } } else { diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/tokens.go b/cmd/frostfs-adm/internal/modules/morph/nns/tokens.go index 3c7136e9d..4ccbb1677 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/tokens.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/tokens.go @@ -18,12 +18,11 @@ const ( func initTokensCmd() { Cmd.AddCommand(tokensCmd) tokensCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) - tokensCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) tokensCmd.Flags().BoolP(commonflags.Verbose, commonflags.VerboseShorthand, false, verboseDesc) } func listTokens(cmd *cobra.Command, _ []string) { - c, _, _ := getRPCClient(cmd) + c, _ := nnsReader(cmd) it, err := c.Tokens() commonCmd.ExitOnErr(cmd, "unable to get tokens: %w", err) for toks, err := it.Next(10); err == nil && len(toks) > 0; toks, err = it.Next(10) { @@ -41,7 +40,7 @@ func listTokens(cmd *cobra.Command, _ []string) { } } -func getCnameRecord(c *client.Contract, token []byte) (string, error) { +func getCnameRecord(c *client.ContractReader, token []byte) (string, error) { items, err := c.GetRecords(string(token), big.NewInt(int64(nns.CNAME))) // GetRecords returns the error "not an array" if the domain does not contain records. From e5ea95c045fa5de03a699204102943cdb4bd7e1e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 21 Nov 2024 10:51:38 +0300 Subject: [PATCH 1566/1943] [#1514] adm/nns: Do not return hash from getRPCClient() It was unused and we employ better abstractions now. gopatch: ``` @@ var a, b expression @@ -a, b, _ := getRPCClient(...) +a, b := getRPCClient(...) ``` Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/nns/domains.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/nns/helper.go | 5 ++--- cmd/frostfs-adm/internal/modules/morph/nns/record.go | 6 +++--- cmd/frostfs-adm/internal/modules/morph/nns/renew.go | 2 +- cmd/frostfs-adm/internal/modules/morph/nns/update.go | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/domains.go b/cmd/frostfs-adm/internal/modules/morph/nns/domains.go index 3684db94a..c11154595 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/domains.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/domains.go @@ -24,7 +24,7 @@ func initRegisterCmd() { } func registerDomain(cmd *cobra.Command, _ []string) { - c, actor, _ := getRPCClient(cmd) + c, actor := getRPCClient(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) email, _ := cmd.Flags().GetString(nnsEmailFlag) @@ -53,7 +53,7 @@ func initDeleteCmd() { } func deleteDomain(cmd *cobra.Command, _ []string) { - c, actor, _ := getRPCClient(cmd) + c, actor := getRPCClient(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) h, vub, err := c.DeleteDomain(name) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go index 68a629c3c..4b7ae4664 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go @@ -7,12 +7,11 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" - "github.com/nspcc-dev/neo-go/pkg/util" "github.com/spf13/cobra" "github.com/spf13/viper" ) -func getRPCClient(cmd *cobra.Command) (*client.Contract, *helper.LocalActor, util.Uint160) { +func getRPCClient(cmd *cobra.Command) (*client.Contract, *helper.LocalActor) { v := viper.GetViper() c, err := helper.GetN3Client(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) @@ -23,7 +22,7 @@ func getRPCClient(cmd *cobra.Command) (*client.Contract, *helper.LocalActor, uti r := management.NewReader(ac.Invoker) nnsCs, err := helper.GetContractByID(r, 1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) - return client.New(ac, nnsCs.Hash), ac, nnsCs.Hash + return client.New(ac, nnsCs.Hash), ac } func nnsReader(cmd *cobra.Command) (*client.ContractReader, *invoker.Invoker) { diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/record.go b/cmd/frostfs-adm/internal/modules/morph/nns/record.go index 5a6db6e72..cb6c5e094 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/record.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/record.go @@ -59,7 +59,7 @@ func initDelRecordCmd() { } func addRecord(cmd *cobra.Command, _ []string) { - c, actor, _ := getRPCClient(cmd) + c, actor := getRPCClient(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) data, _ := cmd.Flags().GetString(nnsRecordDataFlag) recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) @@ -112,7 +112,7 @@ func getRecords(cmd *cobra.Command, _ []string) { } func delRecords(cmd *cobra.Command, _ []string) { - c, actor, _ := getRPCClient(cmd) + c, actor := getRPCClient(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) typ, err := getRecordType(recordType) @@ -127,7 +127,7 @@ func delRecords(cmd *cobra.Command, _ []string) { } func delRecord(cmd *cobra.Command, _ []string) { - c, actor, _ := getRPCClient(cmd) + c, actor := getRPCClient(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) data, _ := cmd.Flags().GetString(nnsRecordDataFlag) recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/renew.go b/cmd/frostfs-adm/internal/modules/morph/nns/renew.go index b13092240..80105ded2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/renew.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/renew.go @@ -14,7 +14,7 @@ func initRenewCmd() { } func renewDomain(cmd *cobra.Command, _ []string) { - c, actor, _ := getRPCClient(cmd) + c, actor := getRPCClient(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) h, vub, err := c.Renew(name) commonCmd.ExitOnErr(cmd, "unable to renew domain: %w", err) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/update.go b/cmd/frostfs-adm/internal/modules/morph/nns/update.go index 3437316e3..dc8dcb3a3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/update.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/update.go @@ -30,7 +30,7 @@ func initUpdateCmd() { } func updateSOA(cmd *cobra.Command, _ []string) { - c, actor, _ := getRPCClient(cmd) + c, actor := getRPCClient(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) email, _ := cmd.Flags().GetString(nnsEmailFlag) From 256f96e252f52b9dafd7b14d1608498d622df350 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 21 Nov 2024 10:54:14 +0300 Subject: [PATCH 1567/1943] [#1514] adm/nns: Rename getRPCClient() to nnsWriter() Make it more specific and similar to nnsReader(). Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/nns/domains.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/nns/helper.go | 2 +- cmd/frostfs-adm/internal/modules/morph/nns/record.go | 6 +++--- cmd/frostfs-adm/internal/modules/morph/nns/renew.go | 2 +- cmd/frostfs-adm/internal/modules/morph/nns/update.go | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/domains.go b/cmd/frostfs-adm/internal/modules/morph/nns/domains.go index c11154595..1668bb327 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/domains.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/domains.go @@ -24,7 +24,7 @@ func initRegisterCmd() { } func registerDomain(cmd *cobra.Command, _ []string) { - c, actor := getRPCClient(cmd) + c, actor := nnsWriter(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) email, _ := cmd.Flags().GetString(nnsEmailFlag) @@ -53,7 +53,7 @@ func initDeleteCmd() { } func deleteDomain(cmd *cobra.Command, _ []string) { - c, actor := getRPCClient(cmd) + c, actor := nnsWriter(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) h, vub, err := c.DeleteDomain(name) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go index 4b7ae4664..29b0a24ae 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/viper" ) -func getRPCClient(cmd *cobra.Command) (*client.Contract, *helper.LocalActor) { +func nnsWriter(cmd *cobra.Command) (*client.Contract, *helper.LocalActor) { v := viper.GetViper() c, err := helper.GetN3Client(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/record.go b/cmd/frostfs-adm/internal/modules/morph/nns/record.go index cb6c5e094..09ed92ab3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/record.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/record.go @@ -59,7 +59,7 @@ func initDelRecordCmd() { } func addRecord(cmd *cobra.Command, _ []string) { - c, actor := getRPCClient(cmd) + c, actor := nnsWriter(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) data, _ := cmd.Flags().GetString(nnsRecordDataFlag) recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) @@ -112,7 +112,7 @@ func getRecords(cmd *cobra.Command, _ []string) { } func delRecords(cmd *cobra.Command, _ []string) { - c, actor := getRPCClient(cmd) + c, actor := nnsWriter(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) typ, err := getRecordType(recordType) @@ -127,7 +127,7 @@ func delRecords(cmd *cobra.Command, _ []string) { } func delRecord(cmd *cobra.Command, _ []string) { - c, actor := getRPCClient(cmd) + c, actor := nnsWriter(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) data, _ := cmd.Flags().GetString(nnsRecordDataFlag) recordType, _ := cmd.Flags().GetString(nnsRecordTypeFlag) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/renew.go b/cmd/frostfs-adm/internal/modules/morph/nns/renew.go index 80105ded2..53bd943f0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/renew.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/renew.go @@ -14,7 +14,7 @@ func initRenewCmd() { } func renewDomain(cmd *cobra.Command, _ []string) { - c, actor := getRPCClient(cmd) + c, actor := nnsWriter(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) h, vub, err := c.Renew(name) commonCmd.ExitOnErr(cmd, "unable to renew domain: %w", err) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/update.go b/cmd/frostfs-adm/internal/modules/morph/nns/update.go index dc8dcb3a3..c6d77ead6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/update.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/update.go @@ -30,7 +30,7 @@ func initUpdateCmd() { } func updateSOA(cmd *cobra.Command, _ []string) { - c, actor := getRPCClient(cmd) + c, actor := nnsWriter(cmd) name, _ := cmd.Flags().GetString(nnsNameFlag) email, _ := cmd.Flags().GetString(nnsEmailFlag) From 99f9e59de974e1219ad78befd89331826f3712f3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 21 Nov 2024 11:52:50 +0300 Subject: [PATCH 1568/1943] [#1514] adm: Remove --alphabet-wallets flag from readonly commands Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/frostfsid/frostfsid.go | 8 -------- cmd/frostfs-adm/internal/modules/morph/netmap/root.go | 1 - 2 files changed, 9 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 091d6634a..74da52a8f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -61,7 +61,6 @@ var ( Use: "list-namespaces", Short: "List all namespaces in frostfsid", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidListNamespaces, @@ -91,7 +90,6 @@ var ( Use: "list-subjects", Short: "List subjects in namespace", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidListSubjects, @@ -121,7 +119,6 @@ var ( Use: "list-groups", Short: "List groups in namespace", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidListGroups, @@ -151,7 +148,6 @@ var ( Use: "list-group-subjects", Short: "List subjects in group", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) }, Run: frostfsidListGroupSubjects, @@ -169,7 +165,6 @@ func initFrostfsIDCreateNamespaceCmd() { func initFrostfsIDListNamespacesCmd() { Cmd.AddCommand(frostfsidListNamespacesCmd) frostfsidListNamespacesCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) - frostfsidListNamespacesCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDCreateSubjectCmd() { @@ -193,7 +188,6 @@ func initFrostfsIDListSubjectsCmd() { frostfsidListSubjectsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidListSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace to list subjects") frostfsidListSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") - frostfsidListSubjectsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDCreateGroupCmd() { @@ -217,7 +211,6 @@ func initFrostfsIDListGroupsCmd() { Cmd.AddCommand(frostfsidListGroupsCmd) frostfsidListGroupsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidListGroupsCmd.Flags().String(namespaceFlag, "", "Namespace to list groups") - frostfsidListGroupsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initFrostfsIDAddSubjectToGroupCmd() { @@ -242,7 +235,6 @@ func initFrostfsIDListGroupSubjectsCmd() { frostfsidListGroupSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace name") frostfsidListGroupSubjectsCmd.Flags().Int64(groupIDFlag, 0, "Group id") frostfsidListGroupSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") - frostfsidListGroupSubjectsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func frostfsidCreateNamespace(cmd *cobra.Command, _ []string) { diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go index 55b7e64f0..291482e0f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/root.go @@ -12,7 +12,6 @@ var ( Short: "List netmap candidates nodes", PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) - _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) }, Run: listNetmapCandidatesNodes, } From 1ed7ab75fbb62f293e17ecd6afc62b76c543206e Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 22 Nov 2024 15:19:51 +0300 Subject: [PATCH 1569/1943] [#1517] cli: Print the reason of ape manager error Signed-off-by: Airat Arifullin --- cmd/internal/common/exit.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/internal/common/exit.go b/cmd/internal/common/exit.go index 9e4fa3098..b8acf0143 100644 --- a/cmd/internal/common/exit.go +++ b/cmd/internal/common/exit.go @@ -26,13 +26,15 @@ func ExitOnErr(cmd *cobra.Command, errFmt string, err error) { _ = iota internal aclDenied + apemanagerDenied ) var ( code int - internalErr = new(sdkstatus.ServerInternal) - accessErr = new(sdkstatus.ObjectAccessDenied) + internalErr = new(sdkstatus.ServerInternal) + accessErr = new(sdkstatus.ObjectAccessDenied) + apemanagerErr = new(sdkstatus.APEManagerAccessDenied) ) switch { @@ -41,6 +43,9 @@ func ExitOnErr(cmd *cobra.Command, errFmt string, err error) { case errors.As(err, &accessErr): code = aclDenied err = fmt.Errorf("%w: %s", err, accessErr.Reason()) + case errors.As(err, &apemanagerErr): + code = apemanagerDenied + err = fmt.Errorf("%w: %s", err, apemanagerErr.Reason()) default: code = internal } From 3ebd560f4225b96788ca12cc6662245486d54d88 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 22 Nov 2024 15:50:47 +0300 Subject: [PATCH 1570/1943] [#1519] cli: Make descriptive help for`--rule` option Signed-off-by: Airat Arifullin --- cmd/internal/common/ape/flags.go | 62 +++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/cmd/internal/common/ape/flags.go b/cmd/internal/common/ape/flags.go index c5e2a3a99..d8b2e88a2 100644 --- a/cmd/internal/common/ape/flags.go +++ b/cmd/internal/common/ape/flags.go @@ -2,7 +2,6 @@ package ape const ( RuleFlag = "rule" - RuleFlagDesc = "Rule statement" PathFlag = "path" PathFlagDesc = "Path to encoded chain in JSON or binary format" TargetNameFlag = "target-name" @@ -17,3 +16,64 @@ const ( ChainNameFlagDesc = "Chain name(ingress|s3)" AllFlag = "all" ) + +const RuleFlagDesc = `Defines an Access Policy Engine (APE) rule in the format: + [:status_detail] ... ... ... + +Status: + - allow Permits specified actions + - deny Prohibits specified actions + - deny:QuotaLimitReached Denies access due to quota limits + +Actions: + Object operations: + - Object.Put, Object.Get, etc. + - Object.* (all object operations) + Container operations: + - Container.Put, Container.Get, etc. + - Container.* (all container operations) + +Conditions: + ResourceCondition: + Format: ResourceCondition:"key"=value, "key"!=value + Reserved properties (use '\' before '$'): + - $Object:version + - $Object:objectID + - $Object:containerID + - $Object:ownerID + - $Object:creationEpoch + - $Object:payloadLength + - $Object:payloadHash + - $Object:objectType + - $Object:homomorphicHash + +RequestCondition: + Format: RequestCondition:"key"=value, "key"!=value + Reserved properties (use '\' before '$'): + - $Actor:publicKey + - $Actor:role + + Example: + ResourceCondition:"check_key"!="check_value" RequestCondition:"$Actor:role"=others + +Resources: + For objects: + - namespace/cid/oid (specific object) + - namespace/cid/* (all objects in container) + - namespace/* (all objects in namespace) + - * (all objects) + - /* (all objects in root namespace) + - /cid/* (all objects in root container) + - /cid/oid (specific object in root container) + + For containers: + - namespace/cid (specific container) + - namespace/* (all containers in namespace) + - * (all containers) + - /cid (root container) + - /* (all root containers) + +Notes: + - Cannot mix object and container operations in one rule + - Default behavior is Any=false unless 'any' is specified + - Use 'all' keyword to explicitly set Any=false` From 0e5524dac75b3e8a287ba0b820dc4f8fccdecfb4 Mon Sep 17 00:00:00 2001 From: George Bartolomey Date: Sat, 23 Nov 2024 15:08:50 +0300 Subject: [PATCH 1571/1943] [#1515] adm: Print address in base58 format in morph ape get-admin Signed-off-by: George Bartolomey --- cmd/frostfs-adm/internal/modules/morph/ape/ape.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go index 8fcd4a441..1960faab4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape.go @@ -8,7 +8,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" apeCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/ape" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" - "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -200,7 +200,7 @@ func listRuleChains(cmd *cobra.Command, _ []string) { func setAdmin(cmd *cobra.Command, _ []string) { s, _ := cmd.Flags().GetString(addrAdminFlag) - addr, err := util.Uint160DecodeStringLE(s) + addr, err := address.StringToUint160(s) commonCmd.ExitOnErr(cmd, "can't decode admin addr: %w", err) pci, ac := newPolicyContractInterface(cmd) h, vub, err := pci.SetAdmin(addr) @@ -214,7 +214,7 @@ func getAdmin(cmd *cobra.Command, _ []string) { pci, _ := newPolicyContractReaderInterface(cmd) addr, err := pci.GetAdmin() commonCmd.ExitOnErr(cmd, "unable to get admin: %w", err) - cmd.Println(addr.StringLE()) + cmd.Println(address.Uint160ToString(addr)) } func listTargets(cmd *cobra.Command, _ []string) { From 7eac5fb18bd1d234b65a311bacdf1326d256e283 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 25 Nov 2024 13:55:14 +0300 Subject: [PATCH 1572/1943] Release v0.44.0 Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 24 ++++++++++++++++++++++++ VERSION | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4ba6a5d6..92c84ab16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,30 @@ Changelog for FrostFS Node ### Removed ### Updated +## [v0.44.0] - 2024-25-11 - Rongbuk + +### Added +- Allow to prioritize nodes during GET traversal via attributes (#1439) +- Add metrics for the frostfsid cache (#1464) +- Customize constant attributes attached to every tracing span (#1488) +- Manage additional keys in the `frostfsid` contract (#1505) +- Describe `--rule` flag in detail for `frostfs-cli ape-manager` subcommands (#1519) + +### Changed +- Support richer interaction with the console in `frostfs-cli container policy-playground` (#1396) +- Print address in base58 format in `frostfs-adm morph policy set-admin` (#1515) + +### Fixed +- Fix EC object search (#1408) +- Fix EC object put when one of the nodes is unavailable (#1427) + +### Removed +- Drop most of the eACL-related code (#1425) +- Remove `--basic-acl` flag from `frostfs-cli container create` (#1483) + +### Upgrading from v0.43.0 +The metabase schema has changed completely, resync is required. + ## [v0.42.0] ### Added diff --git a/VERSION b/VERSION index 01efe7f3a..9052dab96 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.42.0 +v0.44.0 From caa4253249a5713d201d262939343a28df70a620 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 22 Nov 2024 09:59:43 +0300 Subject: [PATCH 1573/1943] [#1522] adm: Remove unnecessary variable declaration It is better to have small scope. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 6780e6dd3..0d28383a2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -57,12 +57,11 @@ func newPolicyContractReaderInterface(cmd *cobra.Command) (*morph.ContractStorag commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) inv := invoker.New(c, nil) - var ch util.Uint160 r := management.NewReader(inv) nnsCs, err := helper.GetContractByID(r, 1) commonCmd.ExitOnErr(cmd, "can't get NNS contract state: %w", err) - ch, err = helper.NNSResolveHash(inv, nnsCs.Hash, helper.DomainOf(constants.PolicyContract)) + ch, err := helper.NNSResolveHash(inv, nnsCs.Hash, helper.DomainOf(constants.PolicyContract)) commonCmd.ExitOnErr(cmd, "unable to resolve policy contract hash: %w", err) invokerAdapter := &invokerAdapter{ From b1766e47c766ed2591e8f8a6b97dbfc13ad6976c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 22 Nov 2024 10:12:54 +0300 Subject: [PATCH 1574/1943] [#1522] adm/helper: Remove unused GetCommittee() method from the Client interface Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/helper/local_client.go | 5 ----- cmd/frostfs-adm/internal/modules/morph/helper/n3client.go | 2 -- 2 files changed, 7 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go index ed028fb7c..17bee7858 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go @@ -128,11 +128,6 @@ func (l *LocalClient) GetApplicationLog(h util.Uint256, t *trigger.Type) (*resul return &a, nil } -func (l *LocalClient) GetCommittee() (keys.PublicKeys, error) { - // not used by `morph init` command - panic("unexpected call") -} - // InvokeFunction is implemented via `InvokeScript`. func (l *LocalClient) InvokeFunction(h util.Uint160, method string, sPrm []smartcontract.Parameter, ss []transaction.Signer) (*result.Invoke, error) { var err error diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go index e62a21b3f..5563b1fd9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go @@ -10,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/neorpc/result" "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" @@ -32,7 +31,6 @@ type Client interface { GetApplicationLog(util.Uint256, *trigger.Type) (*result.ApplicationLog, error) GetVersion() (*result.Version, error) SendRawTransaction(*transaction.Transaction) (util.Uint256, error) - GetCommittee() (keys.PublicKeys, error) CalculateNetworkFee(tx *transaction.Transaction) (int64, error) } From 16053916286fd9cc3e7badc068514cb4d267c03f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 22 Nov 2024 10:14:00 +0300 Subject: [PATCH 1575/1943] [#1522] adm/helper: Simplify Client interface Just reuse `actor.RPCActor`. No functional changes. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/helper/n3client.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go index 5563b1fd9..066ce01fc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go @@ -24,14 +24,10 @@ import ( // Client represents N3 client interface capable of test-invoking scripts // and sending signed transactions to chain. type Client interface { - invoker.RPCInvoke + actor.RPCActor - GetBlockCount() (uint32, error) GetNativeContracts() ([]state.Contract, error) GetApplicationLog(util.Uint256, *trigger.Type) (*result.ApplicationLog, error) - GetVersion() (*result.Version, error) - SendRawTransaction(*transaction.Transaction) (util.Uint256, error) - CalculateNetworkFee(tx *transaction.Transaction) (int64, error) } type HashVUBPair struct { From b10c9543772b9ac4b00abf06a28b9da62f0ef022 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 22 Nov 2024 10:22:54 +0300 Subject: [PATCH 1576/1943] [#1522] adm: Split NewLocalClient() into functions No functional changes. Signed-off-by: Evgenii Stratonikov --- .../modules/morph/helper/local_client.go | 87 +++++++++++-------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go index 17bee7858..34ce5938a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go @@ -58,17 +58,59 @@ func NewLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet return nil, err } - m := smartcontract.GetDefaultHonestNodeCount(int(cfg.ProtocolConfiguration.ValidatorsCount)) - accounts := make([]*wallet.Account, len(wallets)) - for i := range accounts { - accounts[i], err = GetWalletAccount(wallets[i], constants.ConsensusAccountName) - if err != nil { - return nil, err + go bc.Run() + + accounts, err := getBlockSigningAccounts(cfg.ProtocolConfiguration, wallets) + if err != nil { + return nil, err + } + + if cmd.Name() != "init" { + if err := restoreDump(bc, dumpPath); err != nil { + return nil, fmt.Errorf("restore dump: %w", err) } } + return &LocalClient{ + bc: bc, + dumpPath: dumpPath, + accounts: accounts, + }, nil +} + +func restoreDump(bc *core.Blockchain, dumpPath string) error { + f, err := os.OpenFile(dumpPath, os.O_RDONLY, 0o600) + if err != nil { + return fmt.Errorf("can't open local dump: %w", err) + } + defer f.Close() + + r := io.NewBinReaderFromIO(f) + + var skip uint32 + if bc.BlockHeight() != 0 { + skip = bc.BlockHeight() + 1 + } + + count := r.ReadU32LE() - skip + if err := chaindump.Restore(bc, r, skip, count, nil); err != nil { + return err + } + return nil +} + +func getBlockSigningAccounts(cfg config.ProtocolConfiguration, wallets []*wallet.Wallet) ([]*wallet.Account, error) { + accounts := make([]*wallet.Account, len(wallets)) + for i := range accounts { + acc, err := GetWalletAccount(wallets[i], constants.ConsensusAccountName) + if err != nil { + return nil, err + } + accounts[i] = acc + } + indexMap := make(map[string]int) - for i, pub := range cfg.ProtocolConfiguration.StandbyCommittee { + for i, pub := range cfg.StandbyCommittee { indexMap[pub] = i } @@ -77,37 +119,12 @@ func NewLocalClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet pj := accounts[j].PrivateKey().PublicKey().Bytes() return indexMap[string(pi)] < indexMap[string(pj)] }) - sort.Slice(accounts[:cfg.ProtocolConfiguration.ValidatorsCount], func(i, j int) bool { + sort.Slice(accounts[:cfg.ValidatorsCount], func(i, j int) bool { return accounts[i].PublicKey().Cmp(accounts[j].PublicKey()) == -1 }) - go bc.Run() - - if cmd.Name() != "init" { - f, err := os.OpenFile(dumpPath, os.O_RDONLY, 0o600) - if err != nil { - return nil, fmt.Errorf("can't open local dump: %w", err) - } - defer f.Close() - - r := io.NewBinReaderFromIO(f) - - var skip uint32 - if bc.BlockHeight() != 0 { - skip = bc.BlockHeight() + 1 - } - - count := r.ReadU32LE() - skip - if err := chaindump.Restore(bc, r, skip, count, nil); err != nil { - return nil, fmt.Errorf("can't restore local dump: %w", err) - } - } - - return &LocalClient{ - bc: bc, - dumpPath: dumpPath, - accounts: accounts[:m], - }, nil + m := smartcontract.GetDefaultHonestNodeCount(int(cfg.ValidatorsCount)) + return accounts[:m], nil } func (l *LocalClient) GetBlockCount() (uint32, error) { From 61ee1b56103bd77ea3318f23dceebb1928b846a3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 22 Nov 2024 10:33:24 +0300 Subject: [PATCH 1577/1943] [#1522] adm: Simplify LocalClient.SendRawTransaction() The old code was there before Copy() method was introduced. It was also supposed to check errors, however, they are already checked server-side. Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/helper/local_client.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go index 34ce5938a..d0a05d5c7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go @@ -308,13 +308,7 @@ func (l *LocalClient) InvokeScript(script []byte, signers []transaction.Signer) } func (l *LocalClient) SendRawTransaction(tx *transaction.Transaction) (util.Uint256, error) { - // We need to test that transaction was formed correctly to catch as many errors as we can. - bs := tx.Bytes() - _, err := transaction.NewTransactionFromBytes(bs) - if err != nil { - return tx.Hash(), fmt.Errorf("invalid transaction: %w", err) - } - + tx = tx.Copy() l.transactions = append(l.transactions, tx) return tx.Hash(), nil } From 49959c4166d92cd46df04d352c7928f92fbf1aa9 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 22 Nov 2024 10:45:47 +0300 Subject: [PATCH 1578/1943] [#1522] adm/helper: Unexport GetFrostfsIDAdmin() It is used in `helper` package only, besides unit-tests. Move unit-tests to the same package, where they belong. Signed-off-by: Evgenii Stratonikov --- .../morph/frostfsid/frostfsid_util_test.go | 47 ---------------- .../internal/modules/morph/helper/contract.go | 2 +- .../modules/morph/helper/frostfsid.go | 2 +- .../modules/morph/helper/frostfsid_test.go | 53 +++++++++++++++++++ 4 files changed, 55 insertions(+), 49 deletions(-) create mode 100644 cmd/frostfs-adm/internal/modules/morph/helper/frostfsid_test.go diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go index cce859d2f..1d0bc8441 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid_util_test.go @@ -1,59 +1,12 @@ package frostfsid import ( - "encoding/hex" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/ape" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/encoding/address" - "github.com/spf13/viper" "github.com/stretchr/testify/require" ) -func TestFrostfsIDConfig(t *testing.T) { - pks := make([]*keys.PrivateKey, 4) - for i := range pks { - pk, err := keys.NewPrivateKey() - require.NoError(t, err) - pks[i] = pk - } - - fmts := []string{ - pks[0].GetScriptHash().StringLE(), - address.Uint160ToString(pks[1].GetScriptHash()), - hex.EncodeToString(pks[2].PublicKey().UncompressedBytes()), - hex.EncodeToString(pks[3].PublicKey().Bytes()), - } - - for i := range fmts { - v := viper.New() - v.Set("frostfsid.admin", fmts[i]) - - actual, found, err := helper.GetFrostfsIDAdmin(v) - require.NoError(t, err) - require.True(t, found) - require.Equal(t, pks[i].GetScriptHash(), actual) - } - - t.Run("bad key", func(t *testing.T) { - v := viper.New() - v.Set("frostfsid.admin", "abc") - - _, found, err := helper.GetFrostfsIDAdmin(v) - require.Error(t, err) - require.True(t, found) - }) - t.Run("missing key", func(t *testing.T) { - v := viper.New() - - _, found, err := helper.GetFrostfsIDAdmin(v) - require.NoError(t, err) - require.False(t, found) - }) -} - func TestNamespaceRegexp(t *testing.T) { for _, tc := range []struct { name string diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/contract.go b/cmd/frostfs-adm/internal/modules/morph/helper/contract.go index eea3b040e..64d1c6393 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/contract.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/contract.go @@ -82,7 +82,7 @@ func GetContractDeployData(c *InitializeContext, ctrName string, keysParam []any h, found, err = getFrostfsIDAdminFromContract(c.ReadOnlyInvoker) } if method != constants.UpdateMethodName || err == nil && !found { - h, found, err = GetFrostfsIDAdmin(viper.GetViper()) + h, found, err = getFrostfsIDAdmin(viper.GetViper()) } if err != nil { return nil, err diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/helper/frostfsid.go index f29042b82..fce2dfb74 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/frostfsid.go @@ -11,7 +11,7 @@ import ( const frostfsIDAdminConfigKey = "frostfsid.admin" -func GetFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) { +func getFrostfsIDAdmin(v *viper.Viper) (util.Uint160, bool, error) { admin := v.GetString(frostfsIDAdminConfigKey) if admin == "" { return util.Uint160{}, false, nil diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/frostfsid_test.go b/cmd/frostfs-adm/internal/modules/morph/helper/frostfsid_test.go new file mode 100644 index 000000000..38991e962 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/morph/helper/frostfsid_test.go @@ -0,0 +1,53 @@ +package helper + +import ( + "encoding/hex" + "testing" + + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestFrostfsIDConfig(t *testing.T) { + pks := make([]*keys.PrivateKey, 4) + for i := range pks { + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + pks[i] = pk + } + + fmts := []string{ + pks[0].GetScriptHash().StringLE(), + address.Uint160ToString(pks[1].GetScriptHash()), + hex.EncodeToString(pks[2].PublicKey().UncompressedBytes()), + hex.EncodeToString(pks[3].PublicKey().Bytes()), + } + + for i := range fmts { + v := viper.New() + v.Set("frostfsid.admin", fmts[i]) + + actual, found, err := getFrostfsIDAdmin(v) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, pks[i].GetScriptHash(), actual) + } + + t.Run("bad key", func(t *testing.T) { + v := viper.New() + v.Set("frostfsid.admin", "abc") + + _, found, err := getFrostfsIDAdmin(v) + require.Error(t, err) + require.True(t, found) + }) + t.Run("missing key", func(t *testing.T) { + v := viper.New() + + _, found, err := getFrostfsIDAdmin(v) + require.NoError(t, err) + require.False(t, found) + }) +} From a6ef4ab52437802432d0ce022d42ad7fbafd844b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 22 Nov 2024 10:53:09 +0300 Subject: [PATCH 1579/1943] [#1522] adm/helper: Rename GetN3Client() -> NewRemoteClient() Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/balance/balance.go | 2 +- cmd/frostfs-adm/internal/modules/morph/config/config.go | 2 +- cmd/frostfs-adm/internal/modules/morph/container/container.go | 4 ++-- .../internal/modules/morph/contract/dump_hashes.go | 2 +- cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go | 2 +- .../internal/modules/morph/helper/initialize_ctx.go | 2 +- cmd/frostfs-adm/internal/modules/morph/helper/n3client.go | 2 +- .../internal/modules/morph/netmap/netmap_candidates.go | 2 +- cmd/frostfs-adm/internal/modules/morph/nns/helper.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/notary/notary.go | 2 +- cmd/frostfs-adm/internal/modules/morph/policy/policy.go | 2 +- 12 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 0d28383a2..914682647 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -53,7 +53,7 @@ func (n *invokerAdapter) GetRPCInvoker() invoker.RPCInvoke { } func newPolicyContractReaderInterface(cmd *cobra.Command) (*morph.ContractStorageReader, *invoker.Invoker) { - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) inv := invoker.New(c, nil) @@ -73,7 +73,7 @@ func newPolicyContractReaderInterface(cmd *cobra.Command) (*morph.ContractStorag } func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *helper.LocalActor) { - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) ac, err := helper.NewLocalActor(cmd, c, constants.ConsensusAccountName) diff --git a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go index 5519705d4..be42f2aa5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go @@ -51,7 +51,7 @@ func dumpBalances(cmd *cobra.Command, _ []string) error { nmHash util.Uint160 ) - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index 3a7f84acb..65ccc9f9f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -26,7 +26,7 @@ import ( const forceConfigSet = "force" func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/container/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go index 6f08d1655..e72dc15e9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -76,7 +76,7 @@ func dumpContainers(cmd *cobra.Command, _ []string) error { return fmt.Errorf("invalid filename: %w", err) } - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } @@ -157,7 +157,7 @@ func dumpSingleContainer(bw *io.BufBinWriter, ch util.Uint160, inv *invoker.Invo } func listContainers(cmd *cobra.Command, _ []string) error { - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index be2134b77..437e2480d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -36,7 +36,7 @@ type contractDumpInfo struct { } func dumpContractHashes(cmd *cobra.Command, _ []string) error { - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) if err != nil { return fmt.Errorf("can't create N3 client: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 74da52a8f..c3a232710 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -514,7 +514,7 @@ func readIterator(inv *invoker.Invoker, iter *result.Iterator, batchSize int, se } func initInvoker(cmd *cobra.Command) (*invoker.Invoker, *state.Contract, util.Uint160) { - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) commonCmd.ExitOnErr(cmd, "can't create N3 client: %w", err) inv := invoker.New(c, nil) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go index b5b6adf05..335ec5ac9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go @@ -191,7 +191,7 @@ func createClient(cmd *cobra.Command, v *viper.Viper, wallets []*wallet.Wallet) } c, err = NewLocalClient(cmd, v, wallets, ldf.Value.String()) } else { - c, err = GetN3Client(v) + c, err = NewRemoteClient(v) } if err != nil { return nil, fmt.Errorf("can't create N3 client: %w", err) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go index 066ce01fc..25160305d 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go @@ -42,7 +42,7 @@ type ClientContext struct { SentTxs []HashVUBPair } -func GetN3Client(v *viper.Viper) (Client, error) { +func NewRemoteClient(v *viper.Viper) (Client, error) { // number of opened connections // by neo-go client per one host const ( diff --git a/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go b/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go index d8471bb9a..a689e0ec1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go +++ b/cmd/frostfs-adm/internal/modules/morph/netmap/netmap_candidates.go @@ -13,7 +13,7 @@ import ( ) func listNetmapCandidatesNodes(cmd *cobra.Command, _ []string) { - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) commonCmd.ExitOnErr(cmd, "can't create N3 client: %w", err) inv := invoker.New(c, nil) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go index 29b0a24ae..b13cbc8a1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go @@ -13,7 +13,7 @@ import ( func nnsWriter(cmd *cobra.Command) (*client.Contract, *helper.LocalActor) { v := viper.GetViper() - c, err := helper.GetN3Client(v) + c, err := helper.NewRemoteClient(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) ac, err := helper.NewLocalActor(cmd, c, constants.CommitteeAccountName) @@ -26,7 +26,7 @@ func nnsWriter(cmd *cobra.Command) (*client.Contract, *helper.LocalActor) { } func nnsReader(cmd *cobra.Command) (*client.ContractReader, *invoker.Invoker) { - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) inv := invoker.New(c, nil) diff --git a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go index 9b213da4e..fd42d5a4a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go @@ -89,7 +89,7 @@ func depositNotary(cmd *cobra.Command, _ []string) error { } func transferGas(cmd *cobra.Command, acc *wallet.Account, accHash util.Uint160, gasAmount fixedn.Fixed8, till int64) error { - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) if err != nil { return err } diff --git a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go index 36547e22c..686a244f0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go @@ -62,7 +62,7 @@ func SetPolicyCmd(cmd *cobra.Command, args []string) error { } func dumpPolicyCmd(cmd *cobra.Command, _ []string) error { - c, err := helper.GetN3Client(viper.GetViper()) + c, err := helper.NewRemoteClient(viper.GetViper()) commonCmd.ExitOnErr(cmd, "can't create N3 client:", err) inv := invoker.New(c, nil) From 2469e0c6831604b70e11b4418b42ddf8f1fc2593 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 22 Nov 2024 10:55:51 +0300 Subject: [PATCH 1580/1943] [#1522] adm/helper: Remove NewActor() helper It is used once, it is used only internally and it is single-statement. I see no justification in having it as a separate function. It introduces confusion, because we also have NewLocalActor(). Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/helper/n3client.go | 8 +++++++- .../internal/modules/morph/helper/util.go | 12 ------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go index 25160305d..03009ebe3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go @@ -83,7 +83,13 @@ func NewRemoteClient(v *viper.Viper) (Client, error) { } func DefaultClientContext(c Client, committeeAcc *wallet.Account) (*ClientContext, error) { - commAct, err := NewActor(c, committeeAcc) + commAct, err := actor.New(c, []actor.SignerAccount{{ + Signer: transaction.Signer{ + Account: committeeAcc.Contract.ScriptHash(), + Scopes: transaction.Global, + }, + Account: committeeAcc, + }}) if err != nil { return nil, err } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/util.go b/cmd/frostfs-adm/internal/modules/morph/helper/util.go index 8c6b90539..c26aa447b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/util.go @@ -15,10 +15,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/core/state" - "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/viper" @@ -87,16 +85,6 @@ func openAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, er return wallets, nil } -func NewActor(c actor.RPCActor, committeeAcc *wallet.Account) (*actor.Actor, error) { - return actor.New(c, []actor.SignerAccount{{ - Signer: transaction.Signer{ - Account: committeeAcc.Contract.ScriptHash(), - Scopes: transaction.Global, - }, - Account: committeeAcc, - }}) -} - func ReadContract(ctrPath, ctrName string) (*ContractState, error) { rawNef, err := os.ReadFile(filepath.Join(ctrPath, ctrName+"_contract.nef")) if err != nil { From 9e275d44c8bb9c32134c4dabeb2757580da7ea75 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 22 Nov 2024 13:11:43 +0300 Subject: [PATCH 1581/1943] [#1522] adm/helper: Unexport DefaultClientContext() Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go | 2 +- cmd/frostfs-adm/internal/modules/morph/helper/n3client.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go index 335ec5ac9..08a87b999 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go @@ -139,7 +139,7 @@ func NewInitializeContext(cmd *cobra.Command, v *viper.Viper) (*InitializeContex return nil, err } - cliCtx, err := DefaultClientContext(c, committeeAcc) + cliCtx, err := defaultClientContext(c, committeeAcc) if err != nil { return nil, fmt.Errorf("client context: %w", err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go index 03009ebe3..3f3a66cb6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go @@ -82,7 +82,7 @@ func NewRemoteClient(v *viper.Viper) (Client, error) { return c, nil } -func DefaultClientContext(c Client, committeeAcc *wallet.Account) (*ClientContext, error) { +func defaultClientContext(c Client, committeeAcc *wallet.Account) (*ClientContext, error) { commAct, err := actor.New(c, []actor.SignerAccount{{ Signer: transaction.Signer{ Account: committeeAcc.Contract.ScriptHash(), From 11703707533e085d37cd7d77be41801da5def702 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 22 Nov 2024 13:14:02 +0300 Subject: [PATCH 1582/1943] [#1522] adm/helper: Rename createSingleAccounts() -> getSingleAccounts() It doesn't create any accounts, purely finds them in the wallet. Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/helper/initialize_ctx.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go index 08a87b999..8e5615baa 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go @@ -134,7 +134,7 @@ func NewInitializeContext(cmd *cobra.Command, v *viper.Viper) (*InitializeContex return nil, err } - accounts, err := createWalletAccounts(wallets) + accounts, err := getSingleAccounts(wallets) if err != nil { return nil, err } @@ -211,7 +211,7 @@ func getContractsPath(cmd *cobra.Command, needContracts bool) (string, error) { return ctrPath, nil } -func createWalletAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) { +func getSingleAccounts(wallets []*wallet.Wallet) ([]*wallet.Account, error) { accounts := make([]*wallet.Account, len(wallets)) for i, w := range wallets { acc, err := GetWalletAccount(w, constants.SingleAccountName) From aac65001e524b789c6e8f1d55c70bca6f47293d1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 22 Nov 2024 13:19:58 +0300 Subject: [PATCH 1583/1943] [#1522] adm/frostfsid: Remove unreachable condition SendConsensusTx() modifies SendTxs field, if it is not the case, there is a bug in code. Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/frostfsid/frostfsid.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index c3a232710..db98bb8ad 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -1,7 +1,6 @@ package frostfsid import ( - "errors" "fmt" "math/big" "sort" @@ -489,10 +488,6 @@ func (f *frostfsidClient) sendWaitRes() (*state.AppExecResult, error) { } f.bw.Reset() - if len(f.wCtx.SentTxs) == 0 { - return nil, errors.New("no transactions to wait") - } - f.wCtx.Command.Println("Waiting for transactions to persist...") return f.roCli.Wait(f.wCtx.SentTxs[0].Hash, f.wCtx.SentTxs[0].Vub, nil) } From 01acec708fa8f0c536ebc79e8f03ee2420ef3731 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 28 Nov 2024 09:15:29 +0300 Subject: [PATCH 1584/1943] [#1525] pilorama: Use AppendUint* helpers from stdlib gopatch: ``` @@ var slice, e expression @@ +import "encoding/binary" -append(slice, byte(e), byte(e >> 8)) +binary.LittleEndian.AppendUint16(slice, e) ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/boltdb.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index c62d728b1..6e68e9986 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1240,7 +1240,7 @@ func (t *boltForest) fillSortedChildren(b *bbolt.Bucket, nodeIDs MultiNode, h *f nodes = nil length = actualLength + 1 count = 0 - c.Seek(append(prefix, byte(length), byte(length>>8))) + c.Seek(binary.LittleEndian.AppendUint16(prefix, length)) c.Prev() // c.Next() will be performed by for loop } } @@ -1664,7 +1664,7 @@ func internalKeyPrefix(key []byte, k string) []byte { key = append(key, 'i') l := len(k) - key = append(key, byte(l), byte(l>>8)) + key = binary.LittleEndian.AppendUint16(key, uint16(l)) key = append(key, k...) return key } @@ -1679,14 +1679,10 @@ func internalKey(key []byte, k, v string, parent, node Node) []byte { key = internalKeyPrefix(key, k) l := len(v) - key = append(key, byte(l), byte(l>>8)) + key = binary.LittleEndian.AppendUint16(key, uint16(l)) key = append(key, v...) - var raw [8]byte - binary.LittleEndian.PutUint64(raw[:], parent) - key = append(key, raw[:]...) - - binary.LittleEndian.PutUint64(raw[:], node) - key = append(key, raw[:]...) + key = binary.LittleEndian.AppendUint64(key, parent) + key = binary.LittleEndian.AppendUint64(key, node) return key } From bba1892fa13d0c8aa904dc6b6e992a6407f57b64 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 27 Nov 2024 15:48:16 +0300 Subject: [PATCH 1585/1943] [#1524] ape: Make APE checker return error without status Signed-off-by: Airat Arifullin --- pkg/services/common/ape/checker.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index eb4fd03c7..30580da12 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -11,7 +11,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" @@ -104,14 +103,7 @@ func (c *checkerCoreImpl) CheckAPE(prm CheckPrm) error { if found && status == apechain.Allow { return nil } - err = fmt.Errorf("access to operation %s is denied by access policy engine: %s", prm.Request.Operation(), status.String()) - return apeErr(err) -} - -func apeErr(err error) error { - errAccessDenied := &apistatus.ObjectAccessDenied{} - errAccessDenied.WriteReason(err.Error()) - return errAccessDenied + return fmt.Errorf("access to operation %s is denied by access policy engine: %s", prm.Request.Operation(), status.String()) } // isValidBearer checks whether bearer token was correctly signed by authorized From 00c608c05e3c6574b5258f9c63b5ed8bfca264a7 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 27 Nov 2024 15:52:23 +0300 Subject: [PATCH 1586/1943] [#1524] tree: Make check APE error get wrapped to api status Signed-off-by: Airat Arifullin --- pkg/services/tree/signature.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 4fd4a7e1e..80f5b3590 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -11,6 +11,7 @@ import ( core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" @@ -62,7 +63,16 @@ func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, return fmt.Errorf("can't get request role: %w", err) } - return s.checkAPE(ctx, bt, cnr, cid, op, role, pubKey) + if err = s.checkAPE(ctx, bt, cnr, cid, op, role, pubKey); err != nil { + return apeErr(err) + } + return nil +} + +func apeErr(err error) error { + errAccessDenied := &apistatus.ObjectAccessDenied{} + errAccessDenied.WriteReason(err.Error()) + return errAccessDenied } // Returns true iff the operation is read-only and request was signed From e0ac3a583f9207684d0cbfa12ecd59f5c4b83642 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 28 Nov 2024 10:41:59 +0300 Subject: [PATCH 1587/1943] [#1523] metabase: Remove `(*DB).IterateCoveredByTombstones` Remove this method because it isn't used anywhere since 7799f8e4c. Signed-off-by: Aleksey Savchuk --- .../metabase/iterators.go | 65 ------------------- .../metabase/iterators_test.go | 63 ------------------ 2 files changed, 128 deletions(-) diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index 5d42e4125..0d438e102 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "errors" - "fmt" "strconv" "time" @@ -111,70 +110,6 @@ func (db *DB) iterateExpired(tx *bbolt.Tx, epoch uint64, h ExpiredObjectHandler) return nil } -// IterateCoveredByTombstones iterates over all objects in DB which are covered -// by tombstone with string address from tss. Locked objects are not included -// (do not confuse with objects of type LOCK). -// -// If h returns ErrInterruptIterator, nil returns immediately. -// Returns other errors of h directly. -// -// Does not modify tss. -func (db *DB) IterateCoveredByTombstones(ctx context.Context, tss map[string]oid.Address, h func(oid.Address) error) error { - var ( - startedAt = time.Now() - success = false - ) - defer func() { - db.metrics.AddMethodDuration("IterateCoveredByTombstones", time.Since(startedAt), success) - }() - _, span := tracing.StartSpanFromContext(ctx, "metabase.IterateCoveredByTombstones") - defer span.End() - - db.modeMtx.RLock() - defer db.modeMtx.RUnlock() - - if db.mode.NoMetabase() { - return ErrDegradedMode - } - - return db.boltDB.View(func(tx *bbolt.Tx) error { - return db.iterateCoveredByTombstones(tx, tss, h) - }) -} - -func (db *DB) iterateCoveredByTombstones(tx *bbolt.Tx, tss map[string]oid.Address, h func(oid.Address) error) error { - bktGraveyard := tx.Bucket(graveyardBucketName) - - err := bktGraveyard.ForEach(func(k, v []byte) error { - var addr oid.Address - if err := decodeAddressFromKey(&addr, v); err != nil { - return err - } - if _, ok := tss[addr.EncodeToString()]; ok { - var addr oid.Address - - err := decodeAddressFromKey(&addr, k) - if err != nil { - return fmt.Errorf("could not parse address of the object under tombstone: %w", err) - } - - if objectLocked(tx, addr.Container(), addr.Object()) { - return nil - } - - return h(addr) - } - - return nil - }) - - if errors.Is(err, ErrInterruptIterator) { - err = nil - } - - return err -} - func iteratePhyObjects(tx *bbolt.Tx, f func(cid.ID, oid.ID, *objectSDK.Object) error) error { var cid cid.ID var oid oid.ID diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 7eed32c55..4c9579965 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" @@ -67,65 +66,3 @@ func putWithExpiration(t *testing.T, db *meta.DB, typ objectSDK.Type, expiresAt return object2.AddressOf(obj) } - -func TestDB_IterateCoveredByTombstones(t *testing.T) { - db := newDB(t) - defer func() { require.NoError(t, db.Close(context.Background())) }() - - cnr := cidtest.ID() - ts := oidtest.Address() - protected1 := oidtest.Address() - protected2 := oidtest.Address() - protectedLocked := oidtest.Address() - garbage := oidtest.Address() - ts.SetContainer(cnr) - protected1.SetContainer(cnr) - protected2.SetContainer(cnr) - protectedLocked.SetContainer(cnr) - - var prm meta.InhumePrm - var err error - - prm.SetAddresses(protected1, protected2, protectedLocked) - prm.SetTombstoneAddress(ts) - - _, err = db.Inhume(context.Background(), prm) - require.NoError(t, err) - - prm.SetAddresses(garbage) - prm.SetGCMark() - - _, err = db.Inhume(context.Background(), prm) - require.NoError(t, err) - - var handled []oid.Address - - tss := map[string]oid.Address{ - ts.EncodeToString(): ts, - } - - err = db.IterateCoveredByTombstones(context.Background(), tss, func(addr oid.Address) error { - handled = append(handled, addr) - return nil - }) - require.NoError(t, err) - - require.Len(t, handled, 3) - require.Contains(t, handled, protected1) - require.Contains(t, handled, protected2) - require.Contains(t, handled, protectedLocked) - - err = db.Lock(context.Background(), protectedLocked.Container(), oidtest.ID(), []oid.ID{protectedLocked.Object()}) - require.NoError(t, err) - - handled = handled[:0] - - err = db.IterateCoveredByTombstones(context.Background(), tss, func(addr oid.Address) error { - handled = append(handled, addr) - return nil - }) - require.NoError(t, err) - - require.Len(t, handled, 2) - require.NotContains(t, handled, protectedLocked) -} From edfa3f4825ecefd40f9dbda0dfa0356c062a4fa7 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 2 Dec 2024 13:31:32 +0300 Subject: [PATCH 1588/1943] [#1528] node: Keep order for equal elements when sort priority metrics Signed-off-by: Anton Nikiforov --- pkg/services/object_manager/placement/traverser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 7c720b204..6a949e938 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -202,7 +202,7 @@ func sortVector(cfg *cfg, unsortedVector []netmap.NodeInfo) ([]netmap.NodeInfo, metrics: m, } } - slices.SortFunc(nm, func(a, b nodeMetrics) int { + slices.SortStableFunc(nm, func(a, b nodeMetrics) int { return slices.Compare(a.metrics, b.metrics) }) sortedVector := make([]netmap.NodeInfo, len(unsortedVector)) From 635a292ae49a9e6bfdb82575cd8c1421c03c4e37 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 2 Dec 2024 13:35:59 +0300 Subject: [PATCH 1589/1943] [#1528] cli: Keep order for required nodes in the result of `object nodes` Signed-off-by: Anton Nikiforov --- cmd/frostfs-cli/modules/object/nodes.go | 32 ------------------------- 1 file changed, 32 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index e6918dfc9..31682c0e1 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -1,15 +1,12 @@ package object import ( - "bytes" - "cmp" "context" "crypto/ecdsa" "encoding/hex" "encoding/json" "errors" "fmt" - "slices" "sync" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" @@ -507,7 +504,6 @@ func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, } func printPlacement(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) { - normilizeObjectNodesResult(objects, result) if json, _ := cmd.Flags().GetBool(commonflags.JSON); json { printObjectNodesAsJSON(cmd, objID, objects, result) } else { @@ -515,34 +511,6 @@ func printPlacement(cmd *cobra.Command, objID oid.ID, objects []phyObject, resul } } -func normilizeObjectNodesResult(objects []phyObject, result *objectNodesResult) { - slices.SortFunc(objects, func(lhs, rhs phyObject) int { - if lhs.ecHeader == nil && rhs.ecHeader == nil { - return bytes.Compare(lhs.objectID[:], rhs.objectID[:]) - } - if lhs.ecHeader == nil { - return -1 - } - if rhs.ecHeader == nil { - return 1 - } - if lhs.ecHeader.parent == rhs.ecHeader.parent { - return cmp.Compare(lhs.ecHeader.index, rhs.ecHeader.index) - } - return bytes.Compare(lhs.ecHeader.parent[:], rhs.ecHeader.parent[:]) - }) - for _, obj := range objects { - op := result.placements[obj.objectID] - slices.SortFunc(op.confirmedNodes, func(lhs, rhs netmapSDK.NodeInfo) int { - return bytes.Compare(lhs.PublicKey(), rhs.PublicKey()) - }) - slices.SortFunc(op.requiredNodes, func(lhs, rhs netmapSDK.NodeInfo) int { - return bytes.Compare(lhs.PublicKey(), rhs.PublicKey()) - }) - result.placements[obj.objectID] = op - } -} - func printObjectNodesAsText(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) { fmt.Fprintf(cmd.OutOrStdout(), "Object %s stores payload in %d data objects:\n", objID.EncodeToString(), len(objects)) From 60feed3b5f71f58e8694873721cb1c115bcf6bfd Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 2 Dec 2024 15:37:25 +0300 Subject: [PATCH 1590/1943] [#1527] engine/test: Allow to specify current epoch in `epochState` Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/engine/engine_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index a7cb90bae..9a3eaadc0 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -17,10 +17,12 @@ import ( "github.com/stretchr/testify/require" ) -type epochState struct{} +type epochState struct { + currEpoch uint64 +} func (s epochState) CurrentEpoch() uint64 { - return 0 + return s.currEpoch } type testEngineWrapper struct { From 9cabca9dfe73999b4b68f19ef9422d1fafc0b23a Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 2 Dec 2024 16:12:12 +0300 Subject: [PATCH 1591/1943] [#1527] engine/test: Move default metabase options to separate function Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/engine/engine_test.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 9a3eaadc0..926ff43f3 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -89,12 +89,16 @@ func testGetDefaultShardOptions(t testing.TB) []shard.Option { blobstor.WithLogger(test.NewLogger(t)), ), shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))), - shard.WithMetaBaseOptions( - meta.WithPath(filepath.Join(t.TempDir(), "metabase")), - meta.WithPermissions(0o700), - meta.WithEpochState(epochState{}), - meta.WithLogger(test.NewLogger(t)), - ), + shard.WithMetaBaseOptions(testGetDefaultMetabaseOptions(t)...), + } +} + +func testGetDefaultMetabaseOptions(t testing.TB) []meta.Option { + return []meta.Option{ + meta.WithPath(filepath.Join(t.TempDir(), "metabase")), + meta.WithPermissions(0o700), + meta.WithEpochState(epochState{}), + meta.WithLogger(test.NewLogger(t)), } } From 432042c534ec3f8774b2d75e4236e3419f466482 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 2 Dec 2024 16:16:50 +0300 Subject: [PATCH 1592/1943] [#1527] engine: Add tests for handling expired objects on inhume and lock Currently, it's allowed to inhume or lock an expired object. Consider the following scenario: 1) An user inhumes or locks an object 2) The object expires 3) GC hasn't yet deleted the object 4) The node loses the associated tombstone or lock 5) Another node replicates tombstone or lock to the first node In this case, the second node succeeds, which is the desired behavior. Signed-off-by: Aleksey Savchuk --- .../engine/inhume_test.go | 53 +++++++++++++++++++ pkg/local_object_storage/engine/lock_test.go | 53 +++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 6980afb07..b89cf09a8 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -2,13 +2,17 @@ package engine import ( "context" + "strconv" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" ) @@ -84,3 +88,52 @@ func TestStorageEngine_Inhume(t *testing.T) { require.Empty(t, addrs) }) } + +func TestInhumeExpiredRegularObject(t *testing.T) { + t.Parallel() + + const currEpoch = 42 + const objectExpiresAfter = currEpoch - 1 + + engine := testNewEngine(t).setShardsNumAdditionalOpts(t, 1, func(_ int) []shard.Option { + return []shard.Option{ + shard.WithDisabledGC(), + shard.WithMetaBaseOptions(append( + testGetDefaultMetabaseOptions(t), + meta.WithEpochState(epochState{currEpoch}), + )...), + } + }).prepare(t).engine + + cnr := cidtest.ID() + + generateAndPutObject := func() *objectSDK.Object { + obj := testutil.GenerateObjectWithCID(cnr) + testutil.AddAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.Itoa(objectExpiresAfter)) + + var putPrm PutPrm + putPrm.Object = obj + require.NoError(t, engine.Put(context.Background(), putPrm)) + return obj + } + + t.Run("inhume with tombstone", func(t *testing.T) { + obj := generateAndPutObject() + ts := oidtest.Address() + ts.SetContainer(cnr) + + var prm InhumePrm + prm.WithTarget(ts, object.AddressOf(obj)) + _, err := engine.Inhume(context.Background(), prm) + require.NoError(t, err) + }) + + t.Run("inhume without tombstone", func(t *testing.T) { + obj := generateAndPutObject() + + var prm InhumePrm + prm.MarkAsGarbage(object.AddressOf(obj)) + _, err := engine.Inhume(context.Background(), prm) + require.NoError(t, err) + }) +} diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index feca9cb69..7bb9e3934 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -292,3 +292,56 @@ func TestLockForceRemoval(t *testing.T) { _, err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) } + +func TestLockExpiredRegularObject(t *testing.T) { + const currEpoch = 42 + const objectExpiresAfter = currEpoch - 1 + + engine := testNewEngine(t).setShardsNumAdditionalOpts(t, 1, func(_ int) []shard.Option { + return []shard.Option{ + shard.WithDisabledGC(), + shard.WithMetaBaseOptions(append( + testGetDefaultMetabaseOptions(t), + meta.WithEpochState(epochState{currEpoch}), + )...), + } + }).prepare(t).engine + + cnr := cidtest.ID() + + object := testutil.GenerateObjectWithCID(cnr) + testutil.AddAttribute(object, objectV2.SysAttributeExpEpoch, strconv.Itoa(objectExpiresAfter)) + + address := objectcore.AddressOf(object) + + var putPrm PutPrm + putPrm.Object = object + require.NoError(t, engine.Put(context.Background(), putPrm)) + + var getPrm GetPrm + var errNotFound *apistatus.ObjectNotFound + + getPrm.WithAddress(address) + _, err := engine.Get(context.Background(), getPrm) + require.ErrorAs(t, err, &errNotFound) + + t.Run("lock expired regular object", func(t *testing.T) { + engine.Lock(context.Background(), + address.Container(), + oidtest.ID(), + []oid.ID{address.Object()}, + ) + + res, err := engine.IsLocked(context.Background(), objectcore.AddressOf(object)) + require.NoError(t, err) + require.True(t, res) + }) + + t.Run("get expired and locked regular object", func(t *testing.T) { + getPrm.WithAddress(objectcore.AddressOf(object)) + + res, err := engine.Get(context.Background(), getPrm) + require.NoError(t, err) + require.Equal(t, res.Object(), object) + }) +} From 47dfd8840c14fc47ebc621aea1980d8fe002fce6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 3 Dec 2024 15:34:26 +0300 Subject: [PATCH 1593/1943] [#1532] node: Allow to omit metabase.path if shard is disabled Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config/engine/config.go | 8 ++++---- cmd/frostfs-node/config/engine/config_test.go | 16 ++++++++++++++++ .../config/engine/testdata/shards.env | 3 +++ .../config/engine/testdata/shards.json | 13 +++++++++++++ .../config/engine/testdata/shards.yaml | 7 +++++++ 5 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 cmd/frostfs-node/config/engine/testdata/shards.env create mode 100644 cmd/frostfs-node/config/engine/testdata/shards.json create mode 100644 cmd/frostfs-node/config/engine/testdata/shards.yaml diff --git a/cmd/frostfs-node/config/engine/config.go b/cmd/frostfs-node/config/engine/config.go index c944d1c58..e5735e88b 100644 --- a/cmd/frostfs-node/config/engine/config.go +++ b/cmd/frostfs-node/config/engine/config.go @@ -41,6 +41,10 @@ func IterateShards(c *config.Config, required bool, f func(*shardconfig.Config) c.Sub(si), ) + if sc.Mode() == mode.Disabled { + continue + } + // Path for the blobstor can't be present in the default section, because different shards // must have different paths, so if it is missing, the shard is not here. // At the same time checking for "blobstor" section doesn't work proper @@ -50,10 +54,6 @@ func IterateShards(c *config.Config, required bool, f func(*shardconfig.Config) } (*config.Config)(sc).SetDefault(def) - if sc.Mode() == mode.Disabled { - continue - } - if err := f(sc); err != nil { return err } diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 19ad0e7ac..ef6380a62 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -18,6 +18,22 @@ import ( "github.com/stretchr/testify/require" ) +func TestIterateShards(t *testing.T) { + fileConfigTest := func(c *config.Config) { + var res []string + require.NoError(t, + engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error { + res = append(res, sc.Metabase().Path()) + return nil + })) + require.Equal(t, []string{"abc", "xyz"}, res) + } + + const cfgDir = "./testdata/shards" + configtest.ForEachFileType(cfgDir, fileConfigTest) + configtest.ForEnvFileType(t, cfgDir, fileConfigTest) +} + func TestEngineSection(t *testing.T) { t.Run("defaults", func(t *testing.T) { empty := configtest.EmptyConfig() diff --git a/cmd/frostfs-node/config/engine/testdata/shards.env b/cmd/frostfs-node/config/engine/testdata/shards.env new file mode 100644 index 000000000..079789b0f --- /dev/null +++ b/cmd/frostfs-node/config/engine/testdata/shards.env @@ -0,0 +1,3 @@ +FROSTFS_STORAGE_SHARD_0_METABASE_PATH=abc +FROSTFS_STORAGE_SHARD_1_MODE=disabled +FROSTFS_STORAGE_SHARD_2_METABASE_PATH=xyz diff --git a/cmd/frostfs-node/config/engine/testdata/shards.json b/cmd/frostfs-node/config/engine/testdata/shards.json new file mode 100644 index 000000000..b3d6abe85 --- /dev/null +++ b/cmd/frostfs-node/config/engine/testdata/shards.json @@ -0,0 +1,13 @@ +{ + "storage.shard": { + "0": { + "metabase.path": "abc" + }, + "1": { + "mode": "disabled" + }, + "2": { + "metabase.path": "xyz" + } + } +} diff --git a/cmd/frostfs-node/config/engine/testdata/shards.yaml b/cmd/frostfs-node/config/engine/testdata/shards.yaml new file mode 100644 index 000000000..bbbba3af8 --- /dev/null +++ b/cmd/frostfs-node/config/engine/testdata/shards.yaml @@ -0,0 +1,7 @@ +storage.shard: + 0: + metabase.path: abc + 1: + mode: disabled + 2: + metabase.path: xyz From 748edd19993e6cc70329f5eeb782f850e2cef87b Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 2 Dec 2024 15:15:50 +0300 Subject: [PATCH 1594/1943] [#1450] engine: Return shard-level error if object is expired on inhume Since we have errors defined on the shard-level, it looks strage that we check an error againt the shard-level error `ErrLockObjectRemoval`, but then return the metabase-level error. Let's return the same shard-level error instead. Since we have errors defined on the shard-level Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/engine/inhume.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index e89a8d048..74e5b5660 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -169,7 +169,7 @@ func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm sh retErr = new(apistatus.ObjectLocked) return true case errors.Is(err, shard.ErrLockObjectRemoval): - retErr = meta.ErrLockObjectRemoval + retErr = shard.ErrLockObjectRemoval return true case errors.Is(err, shard.ErrReadOnlyMode) || errors.Is(err, shard.ErrDegradedMode): retErr = err From b348b20289e6bc08fc923a8882d7dc24409ee0d6 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 4 Dec 2024 10:08:34 +0300 Subject: [PATCH 1595/1943] [#1450] engine: Add benchmark for `Inhume` operation Signed-off-by: Aleksey Savchuk --- .../engine/inhume_test.go | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index b89cf09a8..9d7196d94 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -2,6 +2,7 @@ package engine import ( "context" + "fmt" "strconv" "testing" @@ -12,8 +13,11 @@ import ( objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + objecttest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/test" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) func TestStorageEngine_Inhume(t *testing.T) { @@ -137,3 +141,59 @@ func TestInhumeExpiredRegularObject(t *testing.T) { require.NoError(t, err) }) } + +func BenchmarkInhumeMultipart(b *testing.B) { + // The benchmark result insignificantly depends on the number of shards, + // so do not use it as a benchmark parameter, just set it big enough. + numShards := 100 + + for numObjects := 1; numObjects <= 10000; numObjects *= 10 { + b.Run( + fmt.Sprintf("objects=%d", numObjects), + func(b *testing.B) { + benchmarkInhumeMultipart(b, numShards, numObjects) + }, + ) + } +} + +func benchmarkInhumeMultipart(b *testing.B, numShards, numObjects int) { + b.StopTimer() + + engine := testNewEngine(b, WithShardPoolSize(uint32(numObjects))). + setShardsNum(b, numShards).prepare(b).engine + defer func() { require.NoError(b, engine.Close(context.Background())) }() + + cnt := cidtest.ID() + eg := errgroup.Group{} + + for range b.N { + addrs := make([]oid.Address, numObjects) + + for i := range numObjects { + prm := PutPrm{} + + prm.Object = objecttest.Object().Parent() + prm.Object.SetContainerID(cnt) + prm.Object.SetType(objectSDK.TypeRegular) + + addrs[i] = object.AddressOf(prm.Object) + + eg.Go(func() error { + return engine.Put(context.Background(), prm) + }) + } + require.NoError(b, eg.Wait()) + + ts := oidtest.Address() + ts.SetContainer(cnt) + + prm := InhumePrm{} + prm.WithTarget(ts, addrs...) + + b.StartTimer() + _, err := engine.Inhume(context.Background(), prm) + require.NoError(b, err) + b.StopTimer() + } +} From 281d65435e05c6b6b37224c46c755a939b486f99 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 18 Nov 2024 14:40:10 +0300 Subject: [PATCH 1596/1943] [#1450] engine: Group object by shard before `Inhume` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine cpu: 12th Gen Intel(R) Core(TM) i5-1235U │ old.txt │ new.txt │ │ sec/op │ sec/op vs base │ InhumeMultipart/objects=1-12 11.42m ± 1% 10.71m ± 0% -6.27% (p=0.000 n=10) InhumeMultipart/objects=10-12 113.5m ± 0% 100.9m ± 3% -11.08% (p=0.000 n=10) InhumeMultipart/objects=100-12 1135.4m ± 1% 681.3m ± 2% -40.00% (p=0.000 n=10) InhumeMultipart/objects=1000-12 11.358 ± 0% 1.089 ± 1% -90.41% (p=0.000 n=10) InhumeMultipart/objects=10000-12 113.251 ± 0% 1.645 ± 1% -98.55% (p=0.000 n=10) geomean 1.136 265.5m -76.63% ``` Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/engine/inhume.go | 184 +++++++++++++--------- 1 file changed, 110 insertions(+), 74 deletions(-) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 74e5b5660..80c77af54 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -81,110 +81,146 @@ func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRe } func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { + addrsPerShard, err := e.groupObjectsByShard(ctx, prm.addrs, !prm.forceRemoval) + if err != nil { + return InhumeRes{}, err + } + var shPrm shard.InhumePrm if prm.forceRemoval { shPrm.ForceRemoval() } - for i := range prm.addrs { - if !prm.forceRemoval { - locked, err := e.IsLocked(ctx, prm.addrs[i]) - if err != nil { - e.log.Warn(ctx, logs.EngineRemovingAnObjectWithoutFullLockingCheck, - zap.Error(err), - zap.Stringer("addr", prm.addrs[i]), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) - } else if locked { - return InhumeRes{}, new(apistatus.ObjectLocked) - } - } + var errLocked *apistatus.ObjectLocked + for shardID, addrs := range addrsPerShard { if prm.tombstone != nil { - shPrm.SetTarget(*prm.tombstone, prm.addrs[i]) + shPrm.SetTarget(*prm.tombstone, addrs...) } else { - shPrm.MarkAsGarbage(prm.addrs[i]) + shPrm.MarkAsGarbage(addrs...) } - ok, err := e.inhumeAddr(ctx, prm.addrs[i], shPrm, true) - if err != nil { - return InhumeRes{}, err + sh, exists := e.shards[shardID] + if !exists { + e.log.Warn(ctx, logs.EngineCouldNotInhumeObjectInShard, + zap.Error(errors.New("this shard was expected to exist")), + zap.String("shard_id", shardID), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), + ) + return InhumeRes{}, errInhumeFailure } - if !ok { - ok, err := e.inhumeAddr(ctx, prm.addrs[i], shPrm, false) - if err != nil { - return InhumeRes{}, err - } else if !ok { - return InhumeRes{}, errInhumeFailure + + if _, err := sh.Inhume(ctx, shPrm); err != nil { + switch { + case errors.As(err, &errLocked): + case errors.Is(err, shard.ErrLockObjectRemoval): + case errors.Is(err, shard.ErrReadOnlyMode): + case errors.Is(err, shard.ErrDegradedMode): + default: + e.reportShardError(ctx, sh, "couldn't inhume object in shard", err) } + return InhumeRes{}, err } } return InhumeRes{}, nil } -// Returns ok if object was inhumed during this invocation or before. -func (e *StorageEngine) inhumeAddr(ctx context.Context, addr oid.Address, prm shard.InhumePrm, checkExists bool) (bool, error) { - root := false - var existPrm shard.ExistsPrm - var retErr error - var ok bool +// groupObjectsByShard groups objects based on the shard(s) they are stored on. +// +// If checkLocked is set, [apistatus.ObjectLocked] will be returned if any of +// the objects are locked. +func (e *StorageEngine) groupObjectsByShard(ctx context.Context, addrs []oid.Address, checkLocked bool) (map[string][]oid.Address, error) { + groups := make(map[string][]oid.Address) + + for _, addr := range addrs { + ids, err := e.findShards(ctx, addr, checkLocked) + if err != nil { + return nil, err + } + for _, id := range ids { + groups[id] = append(groups[id], addr) + } + } + + return groups, nil +} + +// findShards determines the shard(s) where the object is stored. +// +// If the object is a root object, multiple shards will be returned. +// +// If checkLocked is set, [apistatus.ObjectLocked] will be returned if any of +// the objects are locked. +func (e *StorageEngine) findShards(ctx context.Context, addr oid.Address, checkLocked bool) ([]string, error) { + var ( + ids []string + retErr error + + prm shard.ExistsPrm + + siErr *objectSDK.SplitInfoError + ecErr *objectSDK.ECInfoError + + isRootObject bool + objectExists bool + ) e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { - defer func() { - // if object is root we continue since information about it - // can be presented in other shards - if checkExists && root { - stop = false - } - }() + objectExists = false - if checkExists { - existPrm.Address = addr - exRes, err := sh.Exists(ctx, existPrm) - if err != nil { - if client.IsErrObjectAlreadyRemoved(err) || shard.IsErrObjectExpired(err) { - // inhumed once - no need to be inhumed again - ok = true - return true - } - - var siErr *objectSDK.SplitInfoError - var ecErr *objectSDK.ECInfoError - if !(errors.As(err, &siErr) || errors.As(err, &ecErr)) { - e.reportShardError(ctx, sh, "could not check for presents in shard", err, zap.Stringer("address", addr)) - return - } - - root = true - } else if !exRes.Exists() { - return - } + prm.Address = addr + switch res, err := sh.Exists(ctx, prm); { + case client.IsErrObjectAlreadyRemoved(err) || shard.IsErrObjectExpired(err): + // NOTE(@a-savchuk): there were some considerations that we can stop + // immediately if the object is already removed or expired. However, + // the previous method behavior was: + // - keep iterating if it's a root object and already removed, + // - stop iterating if it's not a root object and removed. + // + // Since my task was only improving method speed, let's keep the + // previous method behavior. Continue if it's a root object. + return !isRootObject + case errors.As(err, &siErr) || errors.As(err, &ecErr): + isRootObject = true + objectExists = true + case err != nil: + e.reportShardError( + ctx, sh, "couldn't check for presence in shard", + err, zap.Stringer("address", addr), + ) + case res.Exists(): + objectExists = true + default: } - _, err := sh.Inhume(ctx, prm) - if err != nil { - var errLocked *apistatus.ObjectLocked - switch { - case errors.As(err, &errLocked): + if !objectExists { + return + } + + if checkLocked { + if isLocked, err := sh.IsLocked(ctx, addr); err != nil { + e.log.Warn(ctx, logs.EngineRemovingAnObjectWithoutFullLockingCheck, + zap.Error(err), + zap.Stringer("address", addr), + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), + ) + } else if isLocked { retErr = new(apistatus.ObjectLocked) return true - case errors.Is(err, shard.ErrLockObjectRemoval): - retErr = shard.ErrLockObjectRemoval - return true - case errors.Is(err, shard.ErrReadOnlyMode) || errors.Is(err, shard.ErrDegradedMode): - retErr = err - return true } - - e.reportShardError(ctx, sh, "could not inhume object in shard", err, zap.Stringer("address", addr)) - return false } - ok = true - return true + ids = append(ids, sh.ID().String()) + + // Continue if it's a root object. + return !isRootObject }) - return ok, retErr + if retErr != nil { + return nil, retErr + } + return ids, nil } // IsLocked checks whether an object is locked according to StorageEngine's state. From 6c679d15350e8d28b25ede822d682c3354b06ebe Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 4 Dec 2024 11:03:42 +0300 Subject: [PATCH 1597/1943] [#1535] morph: Unify client creation error messages Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/balance/client.go | 2 +- pkg/morph/client/container/client.go | 2 +- pkg/morph/client/frostfs/client.go | 2 +- pkg/morph/client/frostfsid/client.go | 2 +- pkg/morph/client/netmap/client.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/morph/client/balance/client.go b/pkg/morph/client/balance/client.go index b05c526dc..d229e5900 100644 --- a/pkg/morph/client/balance/client.go +++ b/pkg/morph/client/balance/client.go @@ -39,7 +39,7 @@ func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8, staticClient, err := client.NewStatic(cli, contract, fee, ([]client.StaticClientOption)(*o)...) if err != nil { - return nil, fmt.Errorf("could not create static client of Balance contract: %w", err) + return nil, fmt.Errorf("create 'balance' contract client: %w", err) } return &Client{ diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index bdbcce917..b95b1ea06 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -48,7 +48,7 @@ func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8, sc, err := client.NewStatic(cli, contract, fee, o.staticOpts...) if err != nil { - return nil, fmt.Errorf("can't create container static client: %w", err) + return nil, fmt.Errorf("create 'container' contract client: %w", err) } return &Client{client: sc}, nil diff --git a/pkg/morph/client/frostfs/client.go b/pkg/morph/client/frostfs/client.go index 571915c27..cd6a9849e 100644 --- a/pkg/morph/client/frostfs/client.go +++ b/pkg/morph/client/frostfs/client.go @@ -35,7 +35,7 @@ func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8, sc, err := client.NewStatic(cli, contract, fee, ([]client.StaticClientOption)(*o)...) if err != nil { - return nil, fmt.Errorf("could not create client of FrostFS contract: %w", err) + return nil, fmt.Errorf("create 'frostfs' contract client: %w", err) } return &Client{client: sc}, nil diff --git a/pkg/morph/client/frostfsid/client.go b/pkg/morph/client/frostfsid/client.go index 4c31f42de..61eb03f09 100644 --- a/pkg/morph/client/frostfsid/client.go +++ b/pkg/morph/client/frostfsid/client.go @@ -27,7 +27,7 @@ var _ frostfsidcore.SubjectProvider = (*Client)(nil) func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8) (*Client, error) { sc, err := client.NewStatic(cli, contract, fee, client.TryNotary(), client.AsAlphabet()) if err != nil { - return nil, fmt.Errorf("could not create client of FrostFS ID contract: %w", err) + return nil, fmt.Errorf("create 'frostfsid' contract client: %w", err) } return &Client{client: sc}, nil diff --git a/pkg/morph/client/netmap/client.go b/pkg/morph/client/netmap/client.go index eafa097e9..da7eb0719 100644 --- a/pkg/morph/client/netmap/client.go +++ b/pkg/morph/client/netmap/client.go @@ -52,7 +52,7 @@ func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8, sc, err := client.NewStatic(cli, contract, fee, ([]client.StaticClientOption)(*o)...) if err != nil { - return nil, fmt.Errorf("can't create netmap static client: %w", err) + return nil, fmt.Errorf("create 'netmap' contract client: %w", err) } return &Client{client: sc}, nil From e37dcdf88bfb6e159b1e2812b43774227e7d3366 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 4 Dec 2024 11:07:43 +0300 Subject: [PATCH 1598/1943] [#1535] morph/netmap: Unify error messages for config retrieval Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/netmap/config.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 0a3c351db..29bd1517d 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -28,7 +28,7 @@ const ( func (c *Client) MaxObjectSize() (uint64, error) { objectSize, err := c.readUInt64Config(MaxObjectSizeConfig) if err != nil { - return 0, fmt.Errorf("(%T) could not get epoch number: %w", c, err) + return 0, err } return objectSize, nil @@ -38,7 +38,7 @@ func (c *Client) MaxObjectSize() (uint64, error) { func (c *Client) EpochDuration() (uint64, error) { epochDuration, err := c.readUInt64Config(EpochDurationConfig) if err != nil { - return 0, fmt.Errorf("(%T) could not get epoch duration: %w", c, err) + return 0, err } return epochDuration, nil @@ -49,7 +49,7 @@ func (c *Client) EpochDuration() (uint64, error) { func (c *Client) ContainerFee() (uint64, error) { fee, err := c.readUInt64Config(ContainerFeeConfig) if err != nil { - return 0, fmt.Errorf("(%T) could not get container fee: %w", c, err) + return 0, err } return fee, nil @@ -60,7 +60,7 @@ func (c *Client) ContainerFee() (uint64, error) { func (c *Client) ContainerAliasFee() (uint64, error) { fee, err := c.readUInt64Config(ContainerAliasFeeConfig) if err != nil { - return 0, fmt.Errorf("(%T) could not get container alias fee: %w", c, err) + return 0, err } return fee, nil @@ -79,7 +79,7 @@ func (c *Client) HomomorphicHashDisabled() (bool, error) { func (c *Client) InnerRingCandidateFee() (uint64, error) { fee, err := c.readUInt64Config(IrCandidateFeeConfig) if err != nil { - return 0, fmt.Errorf("(%T) could not get inner ring candidate fee: %w", c, err) + return 0, err } return fee, nil @@ -90,7 +90,7 @@ func (c *Client) InnerRingCandidateFee() (uint64, error) { func (c *Client) WithdrawFee() (uint64, error) { fee, err := c.readUInt64Config(WithdrawFeeConfig) if err != nil { - return 0, fmt.Errorf("(%T) could not get withdraw fee: %w", c, err) + return 0, err } return fee, nil @@ -108,7 +108,7 @@ func (c *Client) MaintenanceModeAllowed() (bool, error) { func (c *Client) readUInt64Config(key string) (uint64, error) { v, err := c.config([]byte(key), IntegerAssert) if err != nil { - return 0, err + return 0, fmt.Errorf("read netconfig value '%s': %w", key, err) } // IntegerAssert is guaranteed to return int64 if the error is nil. @@ -124,7 +124,7 @@ func (c *Client) readBoolConfig(key string) (bool, error) { return false, nil } - return false, fmt.Errorf("read boolean configuration value %s from the Sidechain: %w", key, err) + return false, fmt.Errorf("read netconfig value '%s': %w", key, err) } // BoolAssert is guaranteed to return bool if the error is nil. From e3487d5af5d513115659294a1699bde18269887f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 4 Dec 2024 11:17:13 +0300 Subject: [PATCH 1599/1943] [#1535] morph: Unify test invoke error messages Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/balance/balanceOf.go | 2 +- pkg/morph/client/balance/decimals.go | 2 +- pkg/morph/client/container/deletion_info.go | 2 +- pkg/morph/client/container/get.go | 2 +- pkg/morph/client/container/list.go | 2 +- pkg/morph/client/frostfsid/subject.go | 4 ++-- pkg/morph/client/netmap/config.go | 4 ++-- pkg/morph/client/netmap/epoch.go | 4 ++-- pkg/morph/client/netmap/innerring.go | 2 +- pkg/morph/client/netmap/netmap.go | 6 +++--- pkg/morph/client/netmap/snapshot.go | 4 +++- pkg/morph/client/notary.go | 4 ++-- 12 files changed, 20 insertions(+), 18 deletions(-) diff --git a/pkg/morph/client/balance/balanceOf.go b/pkg/morph/client/balance/balanceOf.go index aae245acd..73aab1736 100644 --- a/pkg/morph/client/balance/balanceOf.go +++ b/pkg/morph/client/balance/balanceOf.go @@ -23,7 +23,7 @@ func (c *Client) BalanceOf(id user.ID) (*big.Int, error) { prms, err := c.client.TestInvoke(invokePrm) if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", balanceOfMethod, err) + return nil, fmt.Errorf("test invoke (%s): %w", balanceOfMethod, err) } else if ln := len(prms); ln != 1 { return nil, fmt.Errorf("unexpected stack item count (%s): %d", balanceOfMethod, ln) } diff --git a/pkg/morph/client/balance/decimals.go b/pkg/morph/client/balance/decimals.go index 39e4b28e5..28329ee6e 100644 --- a/pkg/morph/client/balance/decimals.go +++ b/pkg/morph/client/balance/decimals.go @@ -14,7 +14,7 @@ func (c *Client) Decimals() (uint32, error) { prms, err := c.client.TestInvoke(invokePrm) if err != nil { - return 0, fmt.Errorf("could not perform test invocation (%s): %w", decimalsMethod, err) + return 0, fmt.Errorf("test invoke (%s): %w", decimalsMethod, err) } else if ln := len(prms); ln != 1 { return 0, fmt.Errorf("unexpected stack item count (%s): %d", decimalsMethod, ln) } diff --git a/pkg/morph/client/container/deletion_info.go b/pkg/morph/client/container/deletion_info.go index dda6bf98c..436ca3c01 100644 --- a/pkg/morph/client/container/deletion_info.go +++ b/pkg/morph/client/container/deletion_info.go @@ -39,7 +39,7 @@ func (c *Client) DeletionInfo(cid []byte) (*containercore.DelInfo, error) { if strings.Contains(err.Error(), containerContract.NotFoundError) { return nil, new(apistatus.ContainerNotFound) } - return nil, fmt.Errorf("could not perform test invocation (%s): %w", deletionInfoMethod, err) + return nil, fmt.Errorf("test invoke (%s): %w", deletionInfoMethod, err) } else if ln := len(res); ln != 1 { return nil, fmt.Errorf("unexpected stack item count (%s): %d", deletionInfoMethod, ln) } diff --git a/pkg/morph/client/container/get.go b/pkg/morph/client/container/get.go index ea57a3a95..1d84e9109 100644 --- a/pkg/morph/client/container/get.go +++ b/pkg/morph/client/container/get.go @@ -53,7 +53,7 @@ func (c *Client) Get(cid []byte) (*containercore.Container, error) { if strings.Contains(err.Error(), containerContract.NotFoundError) { return nil, new(apistatus.ContainerNotFound) } - return nil, fmt.Errorf("could not perform test invocation (%s): %w", getMethod, err) + return nil, fmt.Errorf("test invoke (%s): %w", getMethod, err) } else if ln := len(res); ln != 1 { return nil, fmt.Errorf("unexpected stack item count (%s): %d", getMethod, ln) } diff --git a/pkg/morph/client/container/list.go b/pkg/morph/client/container/list.go index 6fed46c1a..55317375a 100644 --- a/pkg/morph/client/container/list.go +++ b/pkg/morph/client/container/list.go @@ -27,7 +27,7 @@ func (c *Client) list(idUser *user.ID) ([]cid.ID, error) { res, err := c.client.TestInvoke(prm) if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", listMethod, err) + return nil, fmt.Errorf("test invoke (%s): %w", listMethod, err) } else if ln := len(res); ln != 1 { return nil, fmt.Errorf("unexpected stack item count (%s): %d", listMethod, ln) } diff --git a/pkg/morph/client/frostfsid/subject.go b/pkg/morph/client/frostfsid/subject.go index 0852f536c..7c93f4922 100644 --- a/pkg/morph/client/frostfsid/subject.go +++ b/pkg/morph/client/frostfsid/subject.go @@ -21,7 +21,7 @@ func (c *Client) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) res, err := c.client.TestInvoke(prm) if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", methodGetSubject, err) + return nil, fmt.Errorf("test invoke (%s): %w", methodGetSubject, err) } structArr, err := checkStackItem(res) @@ -44,7 +44,7 @@ func (c *Client) GetSubjectExtended(addr util.Uint160) (*frostfsidclient.Subject res, err := c.client.TestInvoke(prm) if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", methodGetSubjectExtended, err) + return nil, fmt.Errorf("test invoke (%s): %w", methodGetSubjectExtended, err) } structArr, err := checkStackItem(res) diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 29bd1517d..2b87df6f7 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -206,7 +206,7 @@ func (c *Client) ReadNetworkConfiguration() (NetworkConfiguration, error) { items, err := c.client.TestInvoke(prm) if err != nil { - return res, fmt.Errorf("could not perform test invocation (%s): %w", + return res, fmt.Errorf("test invoke (%s): %w", configListMethod, err) } @@ -292,7 +292,7 @@ func (c *Client) config(key []byte, assert func(stackitem.Item) (any, error)) (a items, err := c.client.TestInvoke(prm) if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", + return nil, fmt.Errorf("test invoke (%s): %w", configMethod, err) } diff --git a/pkg/morph/client/netmap/epoch.go b/pkg/morph/client/netmap/epoch.go index 92d569ae2..6d909a7a1 100644 --- a/pkg/morph/client/netmap/epoch.go +++ b/pkg/morph/client/netmap/epoch.go @@ -14,7 +14,7 @@ func (c *Client) Epoch() (uint64, error) { items, err := c.client.TestInvoke(prm) if err != nil { - return 0, fmt.Errorf("could not perform test invocation (%s): %w", + return 0, fmt.Errorf("test invoke (%s): %w", epochMethod, err) } @@ -38,7 +38,7 @@ func (c *Client) LastEpochBlock() (uint32, error) { items, err := c.client.TestInvoke(prm) if err != nil { - return 0, fmt.Errorf("could not perform test invocation (%s): %w", + return 0, fmt.Errorf("test invoke (%s): %w", lastEpochBlockMethod, err) } diff --git a/pkg/morph/client/netmap/innerring.go b/pkg/morph/client/netmap/innerring.go index c9dc7d2fc..824827d6f 100644 --- a/pkg/morph/client/netmap/innerring.go +++ b/pkg/morph/client/netmap/innerring.go @@ -46,7 +46,7 @@ func (c *Client) GetInnerRingList() (keys.PublicKeys, error) { prms, err := c.client.TestInvoke(invokePrm) if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", innerRingListMethod, err) + return nil, fmt.Errorf("test invoke (%s): %w", innerRingListMethod, err) } return irKeysFromStackItem(prms, innerRingListMethod) diff --git a/pkg/morph/client/netmap/netmap.go b/pkg/morph/client/netmap/netmap.go index f7b5c3ba4..a0009ea73 100644 --- a/pkg/morph/client/netmap/netmap.go +++ b/pkg/morph/client/netmap/netmap.go @@ -18,7 +18,7 @@ func (c *Client) GetNetMapByEpoch(epoch uint64) (*netmap.NetMap, error) { res, err := c.client.TestInvoke(invokePrm) if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", + return nil, fmt.Errorf("test invoke (%s): %w", epochSnapshotMethod, err) } @@ -40,7 +40,7 @@ func (c *Client) GetCandidates() ([]netmap.NodeInfo, error) { res, err := c.client.TestInvoke(invokePrm) if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", netMapCandidatesMethod, err) + return nil, fmt.Errorf("test invoke (%s): %w", netMapCandidatesMethod, err) } if len(res) > 0 { @@ -57,7 +57,7 @@ func (c *Client) NetMap() (*netmap.NetMap, error) { res, err := c.client.TestInvoke(invokePrm) if err != nil { - return nil, fmt.Errorf("could not perform test invocation (%s): %w", + return nil, fmt.Errorf("test invoke (%s): %w", netMapMethod, err) } diff --git a/pkg/morph/client/netmap/snapshot.go b/pkg/morph/client/netmap/snapshot.go index ba2c26af7..a5134bcef 100644 --- a/pkg/morph/client/netmap/snapshot.go +++ b/pkg/morph/client/netmap/snapshot.go @@ -1,6 +1,8 @@ package netmap import ( + "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) @@ -13,7 +15,7 @@ func (c *Client) GetNetMap(diff uint64) (*netmap.NetMap, error) { res, err := c.client.TestInvoke(prm) if err != nil { - return nil, err + return nil, fmt.Errorf("test invoke (%s): %w", snapshotMethod, err) } return DecodeNetMap(res) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 71232cb33..8516c89a9 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -239,7 +239,7 @@ func (c *Client) GetNotaryDeposit() (res int64, err error) { items, err := c.TestInvoke(c.notary.notary, notaryBalanceOfMethod, sh) if err != nil { - return 0, fmt.Errorf("%v: %w", notaryBalanceErrMsg, err) + return 0, fmt.Errorf("test invoke (%s): %w", notaryBalanceOfMethod, err) } if len(items) != 1 { @@ -654,7 +654,7 @@ func (c *Client) notaryTxValidationLimit() (uint32, error) { func (c *Client) depositExpirationOf() (int64, error) { expirationRes, err := c.TestInvoke(c.notary.notary, notaryExpirationOfMethod, c.acc.PrivateKey().GetScriptHash()) if err != nil { - return 0, fmt.Errorf("can't invoke method: %w", err) + return 0, fmt.Errorf("test invoke (%s): %w", notaryExpirationOfMethod, err) } if len(expirationRes) != 1 { From 2d5d4093be45cd26660ddcd9ea28d7b625a6e203 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 4 Dec 2024 13:19:37 +0300 Subject: [PATCH 1600/1943] [#1537] morph: Use `(user.ID).ScriptHash()` where possible Pick up changes from TrueCloudLab/frostfs-sdk-go#198. gopatch: ``` @@ var user expression @@ -address.StringToUint160(user.EncodeToString()) +user.ScriptHash() ``` Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/balance/balanceOf.go | 3 +-- pkg/morph/client/balance/transfer.go | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/morph/client/balance/balanceOf.go b/pkg/morph/client/balance/balanceOf.go index 73aab1736..12a499ffb 100644 --- a/pkg/morph/client/balance/balanceOf.go +++ b/pkg/morph/client/balance/balanceOf.go @@ -6,13 +6,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/encoding/address" ) // BalanceOf receives the amount of funds in the client's account // through the Balance contract call, and returns it. func (c *Client) BalanceOf(id user.ID) (*big.Int, error) { - h, err := address.StringToUint160(id.EncodeToString()) + h, err := id.ScriptHash() if err != nil { return nil, err } diff --git a/pkg/morph/client/balance/transfer.go b/pkg/morph/client/balance/transfer.go index 65a0b70a6..9638b5c45 100644 --- a/pkg/morph/client/balance/transfer.go +++ b/pkg/morph/client/balance/transfer.go @@ -6,7 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/encoding/address" ) // TransferPrm groups parameters of TransferX method. @@ -25,12 +24,12 @@ type TransferPrm struct { // // If TryNotary is provided, calls notary contract. func (c *Client) TransferX(ctx context.Context, p TransferPrm) error { - from, err := address.StringToUint160(p.From.EncodeToString()) + from, err := p.From.ScriptHash() if err != nil { return err } - to, err := address.StringToUint160(p.To.EncodeToString()) + to, err := p.To.ScriptHash() if err != nil { return err } From 5c3b2d95ba860327963fce069553f5ec953982a1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 4 Dec 2024 15:17:50 +0300 Subject: [PATCH 1601/1943] [#1538] node: Assume notary is enabled Notaryless environments are not tested at all since a while. We use neo-go only and it has notary contract enabled. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config.go | 2 -- cmd/frostfs-node/morph.go | 28 +++++++--------------------- cmd/frostfs-node/netmap.go | 18 ++++++++---------- cmd/frostfs-node/object.go | 28 ++-------------------------- 4 files changed, 17 insertions(+), 59 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 5af37865f..e63786b7d 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -591,8 +591,6 @@ type cfgMorph struct { client *client.Client - notaryEnabled bool - // TTL of Sidechain cached values. Non-positive value disables caching. cacheTTL time.Duration diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 67d2d1c06..3b9175981 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -35,18 +35,14 @@ func (c *cfg) initMorphComponents(ctx context.Context) { lookupScriptHashesInNNS(c) // smart contract auto negotiation - if c.cfgMorph.notaryEnabled { - err := c.cfgMorph.client.EnableNotarySupport( - client.WithProxyContract( - c.cfgMorph.proxyScriptHash, - ), - ) - fatalOnErr(err) - } - - c.log.Info(ctx, logs.FrostFSNodeNotarySupport, - zap.Bool("sidechain_enabled", c.cfgMorph.notaryEnabled), + err := c.cfgMorph.client.EnableNotarySupport( + client.WithProxyContract( + c.cfgMorph.proxyScriptHash, + ), ) + fatalOnErr(err) + + c.log.Info(ctx, logs.FrostFSNodeNotarySupport) wrap, err := nmClient.NewFromMorph(c.cfgMorph.client, c.cfgNetmap.scriptHash, 0, nmClient.TryNotary()) fatalOnErr(err) @@ -116,15 +112,9 @@ func initMorphClient(ctx context.Context, c *cfg) { } c.cfgMorph.client = cli - c.cfgMorph.notaryEnabled = cli.ProbeNotary() } func makeAndWaitNotaryDeposit(ctx context.Context, c *cfg) { - // skip notary deposit in non-notary environments - if !c.cfgMorph.notaryEnabled { - return - } - tx, vub, err := makeNotaryDeposit(ctx, c) fatalOnErr(err) @@ -282,10 +272,6 @@ func lookupScriptHashesInNNS(c *cfg) { ) for _, t := range targets { - if t.nnsName == client.NNSProxyContractName && !c.cfgMorph.notaryEnabled { - continue // ignore proxy contract if notary disabled - } - if emptyHash.Equals(*t.h) { *t.h, err = c.cfgMorph.client.NNSContractAddress(t.nnsName) fatalOnErrDetails(fmt.Sprintf("can't resolve %s in NNS", t.nnsName), err) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 9127d1123..e94428fcb 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -193,16 +193,14 @@ func addNewEpochNotificationHandlers(c *cfg) { } }) - if c.cfgMorph.notaryEnabled { - addNewEpochAsyncNotificationHandler(c, func(ctx context.Context, _ event.Event) { - _, _, err := makeNotaryDeposit(ctx, c) - if err != nil { - c.log.Error(ctx, logs.FrostFSNodeCouldNotMakeNotaryDeposit, - zap.String("error", err.Error()), - ) - } - }) - } + addNewEpochAsyncNotificationHandler(c, func(ctx context.Context, _ event.Event) { + _, _, err := makeNotaryDeposit(ctx, c) + if err != nil { + c.log.Error(ctx, logs.FrostFSNodeCouldNotMakeNotaryDeposit, + zap.String("error", err.Error()), + ) + } + }) } // bootstrapNode adds current node to the Network map. diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index c4205a620..6804aae59 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/cache" objectTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/object/grpc" objectService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" @@ -137,24 +136,6 @@ func (fn *innerRingFetcherWithNotary) InnerRingKeys() ([][]byte, error) { return result, nil } -type innerRingFetcherWithoutNotary struct { - nm *nmClient.Client -} - -func (f *innerRingFetcherWithoutNotary) InnerRingKeys() ([][]byte, error) { - keys, err := f.nm.GetInnerRingList() - if err != nil { - return nil, fmt.Errorf("can't get inner ring keys from netmap contract: %w", err) - } - - result := make([][]byte, 0, len(keys)) - for i := range keys { - result = append(result, keys[i].Bytes()) - } - - return result, nil -} - func initObjectService(c *cfg) { keyStorage := util.NewKeyStorage(&c.key.PrivateKey, c.privateTokenStore, c.cfgNetmap.state) @@ -305,13 +286,8 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl } func createInnerRingFetcher(c *cfg) v2.InnerRingFetcher { - if c.cfgMorph.client.ProbeNotary() { - return &innerRingFetcherWithNotary{ - sidechain: c.cfgMorph.client, - } - } - return &innerRingFetcherWithoutNotary{ - nm: c.cfgNetmap.wrapper, + return &innerRingFetcherWithNotary{ + sidechain: c.cfgMorph.client, } } From 6a51086030bc9ae8f6403928579313d629549110 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 4 Dec 2024 15:25:36 +0300 Subject: [PATCH 1602/1943] [#1538] morph/client: Remove TryNotary() option from side-chain contracts The notary is always enabled and this option does always work. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/metabase/upgrade.go | 2 +- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/container.go | 2 +- cmd/frostfs-node/morph.go | 2 +- pkg/innerring/initialization.go | 5 ++--- pkg/morph/client/balance/client.go | 10 +--------- pkg/morph/client/balance/transfer.go | 2 -- pkg/morph/client/container/client.go | 10 +--------- pkg/morph/client/container/delete.go | 2 -- pkg/morph/client/container/put.go | 2 -- pkg/morph/client/netmap/client.go | 10 +--------- 11 files changed, 9 insertions(+), 40 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/metabase/upgrade.go b/cmd/frostfs-adm/internal/modules/metabase/upgrade.go index 00b30c9b2..beced0d7a 100644 --- a/cmd/frostfs-adm/internal/modules/metabase/upgrade.go +++ b/cmd/frostfs-adm/internal/modules/metabase/upgrade.go @@ -135,7 +135,7 @@ func createContainerInfoProvider(cli *client.Client) (container.InfoProvider, er if err != nil { return nil, fmt.Errorf("resolve container contract hash: %w", err) } - cc, err := morphcontainer.NewFromMorph(cli, sh, 0, morphcontainer.TryNotary()) + cc, err := morphcontainer.NewFromMorph(cli, sh, 0) if err != nil { return nil, fmt.Errorf("create morph container client: %w", err) } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e63786b7d..9b727e41a 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1464,7 +1464,7 @@ func (c *cfg) createTombstoneSource() *tombstone.ExpirationChecker { func (c *cfg) createContainerInfoProvider(ctx context.Context) container.InfoProvider { return container.NewInfoProvider(func() (container.Source, error) { c.initMorphComponents(ctx) - cc, err := containerClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0, containerClient.TryNotary()) + cc, err := containerClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0) if err != nil { return nil, err } diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index d3e1b2766..7d558dacb 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -28,7 +28,7 @@ import ( func initContainerService(_ context.Context, c *cfg) { // container wrapper that tries to invoke notary // requests if chain is configured so - wrap, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0, cntClient.TryNotary()) + wrap, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0) fatalOnErr(err) c.shared.cnrClient = wrap diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 3b9175981..81579c7fc 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -44,7 +44,7 @@ func (c *cfg) initMorphComponents(ctx context.Context) { c.log.Info(ctx, logs.FrostFSNodeNotarySupport) - wrap, err := nmClient.NewFromMorph(c.cfgMorph.client, c.cfgNetmap.scriptHash, 0, nmClient.TryNotary()) + wrap, err := nmClient.NewFromMorph(c.cfgMorph.client, c.cfgNetmap.scriptHash, 0) fatalOnErr(err) var netmapSource netmap.Source diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 25f4ff034..5481354e1 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -380,7 +380,6 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { // form morph container client's options morphCnrOpts := make([]container.Option, 0, 3) morphCnrOpts = append(morphCnrOpts, - container.TryNotary(), container.AsAlphabet(), ) @@ -390,12 +389,12 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { } s.containerClient = result.CnrClient - s.netmapClient, err = nmClient.NewFromMorph(s.morphClient, s.contracts.netmap, fee, nmClient.TryNotary(), nmClient.AsAlphabet()) + s.netmapClient, err = nmClient.NewFromMorph(s.morphClient, s.contracts.netmap, fee, nmClient.AsAlphabet()) if err != nil { return nil, err } - s.balanceClient, err = balanceClient.NewFromMorph(s.morphClient, s.contracts.balance, fee, balanceClient.TryNotary(), balanceClient.AsAlphabet()) + s.balanceClient, err = balanceClient.NewFromMorph(s.morphClient, s.contracts.balance, fee, balanceClient.AsAlphabet()) if err != nil { return nil, err } diff --git a/pkg/morph/client/balance/client.go b/pkg/morph/client/balance/client.go index d229e5900..1dacb9574 100644 --- a/pkg/morph/client/balance/client.go +++ b/pkg/morph/client/balance/client.go @@ -54,15 +54,7 @@ type Option func(*opts) type opts []client.StaticClientOption func defaultOpts() *opts { - return new(opts) -} - -// TryNotary returns option to enable -// notary invocation tries. -func TryNotary() Option { - return func(o *opts) { - *o = append(*o, client.TryNotary()) - } + return &opts{client.TryNotary()} } // AsAlphabet returns option to sign main TX diff --git a/pkg/morph/client/balance/transfer.go b/pkg/morph/client/balance/transfer.go index 9638b5c45..16c8f3982 100644 --- a/pkg/morph/client/balance/transfer.go +++ b/pkg/morph/client/balance/transfer.go @@ -21,8 +21,6 @@ type TransferPrm struct { // TransferX transfers p.Amount of GASe-12 from p.From to p.To // with details p.Details through direct smart contract call. -// -// If TryNotary is provided, calls notary contract. func (c *Client) TransferX(ctx context.Context, p TransferPrm) error { from, err := p.From.ScriptHash() if err != nil { diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index b95b1ea06..51aa1a93a 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -73,15 +73,7 @@ type opts struct { } func defaultOpts() *opts { - return new(opts) -} - -// TryNotary returns option to enable -// notary invocation tries. -func TryNotary() Option { - return func(o *opts) { - o.staticOpts = append(o.staticOpts, client.TryNotary()) - } + return &opts{staticOpts: []client.StaticClientOption{client.TryNotary()}} } // AsAlphabet returns option to sign main TX diff --git a/pkg/morph/client/container/delete.go b/pkg/morph/client/container/delete.go index 5696645b2..ef5cc8c38 100644 --- a/pkg/morph/client/container/delete.go +++ b/pkg/morph/client/container/delete.go @@ -66,8 +66,6 @@ func (d *DeletePrm) SetKey(key []byte) { // // Returns valid until block and any error encountered that caused // the removal to interrupt. -// -// If TryNotary is provided, calls notary contract. func (c *Client) Delete(ctx context.Context, p DeletePrm) (uint32, error) { if len(p.signature) == 0 && !p.IsControl() { return 0, errNilArgument diff --git a/pkg/morph/client/container/put.go b/pkg/morph/client/container/put.go index 74d9f6da8..1eb03694f 100644 --- a/pkg/morph/client/container/put.go +++ b/pkg/morph/client/container/put.go @@ -94,8 +94,6 @@ func (p *PutPrm) SetZone(zone string) { // // Returns calculated container identifier and any error // encountered that caused the saving to interrupt. -// -// If TryNotary is provided, calls notary contract. func (c *Client) Put(ctx context.Context, p PutPrm) error { if len(p.sig) == 0 || len(p.key) == 0 { return errNilArgument diff --git a/pkg/morph/client/netmap/client.go b/pkg/morph/client/netmap/client.go index da7eb0719..de8afbfb5 100644 --- a/pkg/morph/client/netmap/client.go +++ b/pkg/morph/client/netmap/client.go @@ -65,15 +65,7 @@ type Option func(*opts) type opts []client.StaticClientOption func defaultOpts() *opts { - return new(opts) -} - -// TryNotary returns option to enable -// notary invocation tries. -func TryNotary() Option { - return func(o *opts) { - *o = append(*o, client.TryNotary()) - } + return &opts{client.TryNotary()} } // AsAlphabet returns option to sign main TX From 84b4051b4da05e3a0433793c6de89fcb5b5e2e9c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 4 Dec 2024 15:28:33 +0300 Subject: [PATCH 1603/1943] [#1538] morph/container: Make opts struct similar to that of other contracts Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/container/client.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/morph/client/container/client.go b/pkg/morph/client/container/client.go index 51aa1a93a..be684619b 100644 --- a/pkg/morph/client/container/client.go +++ b/pkg/morph/client/container/client.go @@ -46,7 +46,7 @@ func NewFromMorph(cli *client.Client, contract util.Uint160, fee fixedn.Fixed8, opts[i](o) } - sc, err := client.NewStatic(cli, contract, fee, o.staticOpts...) + sc, err := client.NewStatic(cli, contract, fee, *o...) if err != nil { return nil, fmt.Errorf("create 'container' contract client: %w", err) } @@ -68,12 +68,10 @@ func (c Client) ContractAddress() util.Uint160 { // parameter of Wrapper. type Option func(*opts) -type opts struct { - staticOpts []client.StaticClientOption -} +type opts []client.StaticClientOption func defaultOpts() *opts { - return &opts{staticOpts: []client.StaticClientOption{client.TryNotary()}} + return &opts{client.TryNotary()} } // AsAlphabet returns option to sign main TX @@ -83,6 +81,6 @@ func defaultOpts() *opts { // Considered to be used by IR nodes only. func AsAlphabet() Option { return func(o *opts) { - o.staticOpts = append(o.staticOpts, client.AsAlphabet()) + *o = append(*o, client.AsAlphabet()) } } From 5fe78e51d1c4cc19d53e2106de4c7577f6d4f272 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 5 Dec 2024 10:32:20 +0300 Subject: [PATCH 1604/1943] [#1540] getSvc: Do not log context canceled errors during EC assemble Those errors are fired when it is enough chunks retrieved and error group cancels other requests. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assemblerec.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index b0895e13e..127be2b52 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -238,13 +238,13 @@ func (a *assemblerec) tryGetChunkFromLocalStorage(ctx context.Context, ch object var object *objectSDK.Object if a.head { object, err = a.localStorage.Head(ctx, addr, false) - if err != nil { + if err != nil && !errors.Is(err, context.Canceled) { a.log.Warn(ctx, logs.GetUnableToHeadPartECObject, zap.String("node", "local"), zap.Stringer("part_id", objID), zap.Error(err)) return nil } } else { object, err = a.localStorage.Get(ctx, addr) - if err != nil { + if err != nil && !errors.Is(err, context.Canceled) { a.log.Warn(ctx, logs.GetUnableToGetPartECObject, zap.String("node", "local"), zap.Stringer("part_id", objID), zap.Error(err)) return nil } @@ -286,13 +286,13 @@ func (a *assemblerec) tryGetChunkFromRemoteStorage(ctx context.Context, node cli var object *objectSDK.Object if a.head { object, err = a.remoteStorage.headObjectFromNode(ctx, addr, node, false) - if err != nil { + if err != nil && !errors.Is(err, context.Canceled) { a.log.Warn(ctx, logs.GetUnableToHeadPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Stringer("part_id", objID), zap.Error(err)) return nil } } else { object, err = a.remoteStorage.getObjectFromNode(ctx, addr, node) - if err != nil { + if err != nil && !errors.Is(err, context.Canceled) { a.log.Warn(ctx, logs.GetUnableToGetPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Stringer("part_id", objID), zap.Error(err)) return nil } From 7df3520d486555a0211f7e37ee3e0fa9a96cf92c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 5 Dec 2024 10:32:30 +0300 Subject: [PATCH 1605/1943] [#1540] getSvc: Drop redundant returns Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/assemblerec.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index 127be2b52..a53299480 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -240,13 +240,11 @@ func (a *assemblerec) tryGetChunkFromLocalStorage(ctx context.Context, ch object object, err = a.localStorage.Head(ctx, addr, false) if err != nil && !errors.Is(err, context.Canceled) { a.log.Warn(ctx, logs.GetUnableToHeadPartECObject, zap.String("node", "local"), zap.Stringer("part_id", objID), zap.Error(err)) - return nil } } else { object, err = a.localStorage.Get(ctx, addr) if err != nil && !errors.Is(err, context.Canceled) { a.log.Warn(ctx, logs.GetUnableToGetPartECObject, zap.String("node", "local"), zap.Stringer("part_id", objID), zap.Error(err)) - return nil } } return object @@ -288,13 +286,11 @@ func (a *assemblerec) tryGetChunkFromRemoteStorage(ctx context.Context, node cli object, err = a.remoteStorage.headObjectFromNode(ctx, addr, node, false) if err != nil && !errors.Is(err, context.Canceled) { a.log.Warn(ctx, logs.GetUnableToHeadPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Stringer("part_id", objID), zap.Error(err)) - return nil } } else { object, err = a.remoteStorage.getObjectFromNode(ctx, addr, node) if err != nil && !errors.Is(err, context.Canceled) { a.log.Warn(ctx, logs.GetUnableToGetPartECObject, zap.String("node", hex.EncodeToString(node.PublicKey())), zap.Stringer("part_id", objID), zap.Error(err)) - return nil } } return object From d5d5ce2074d1a5ef60ebe04465c35f047dfdca15 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 5 Dec 2024 11:17:12 +0300 Subject: [PATCH 1606/1943] [#1541] morph/event: Simplify balance contract event parsing Signed-off-by: Evgenii Stratonikov --- pkg/morph/event/balance/lock.go | 68 +++++----------------------- pkg/morph/event/balance/lock_test.go | 3 +- 2 files changed, 13 insertions(+), 58 deletions(-) diff --git a/pkg/morph/event/balance/lock.go b/pkg/morph/event/balance/lock.go index 062a2a886..99f80584a 100644 --- a/pkg/morph/event/balance/lock.go +++ b/pkg/morph/event/balance/lock.go @@ -3,7 +3,7 @@ package balance import ( "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/util" @@ -47,61 +47,17 @@ func (l Lock) TxHash() util.Uint256 { return l.txHash } // ParseLock from notification into lock structure. func ParseLock(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Lock - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) + var le balance.LockEvent + if err := le.FromStackItem(e.Item); err != nil { + return nil, fmt.Errorf("parse balance.LockEvent: %w", err) } - if ln := len(params); ln != 5 { - return nil, event.WrongNumberOfParameters(5, ln) - } - - // parse id - ev.id, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get lock id: %w", err) - } - - // parse user - user, err := client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get lock user value: %w", err) - } - - ev.user, err = util.Uint160DecodeBytesBE(user) - if err != nil { - return nil, fmt.Errorf("could not convert lock user value to uint160: %w", err) - } - - // parse lock account - lock, err := client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get lock account value: %w", err) - } - - ev.lock, err = util.Uint160DecodeBytesBE(lock) - if err != nil { - return nil, fmt.Errorf("could not convert lock account value to uint160: %w", err) - } - - // parse amount - ev.amount, err = client.IntFromStackItem(params[3]) - if err != nil { - return nil, fmt.Errorf("could not get lock amount: %w", err) - } - - // parse until deadline - ev.until, err = client.IntFromStackItem(params[4]) - if err != nil { - return nil, fmt.Errorf("could not get lock deadline: %w", err) - } - - ev.txHash = e.Container - - return ev, nil + return Lock{ + id: le.TxID, + user: le.From, + lock: le.To, + amount: le.Amount.Int64(), + until: le.Until.Int64(), + txHash: e.Container, + }, nil } diff --git a/pkg/morph/event/balance/lock_test.go b/pkg/morph/event/balance/lock_test.go index 9199bcd55..87b91aede 100644 --- a/pkg/morph/event/balance/lock_test.go +++ b/pkg/morph/event/balance/lock_test.go @@ -4,7 +4,6 @@ import ( "math/big" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" @@ -28,7 +27,7 @@ func TestParseLock(t *testing.T) { } _, err := ParseLock(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(5, len(prms)).Error()) + require.Error(t, err) }) t.Run("wrong id parameter", func(t *testing.T) { From d5c46d812a641a93e299d468b0daa49aa2f9e323 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 5 Dec 2024 11:43:24 +0300 Subject: [PATCH 1607/1943] [#1541] go.mod: Update frostfs-contract New version contains more idiomatic types in the auto-generated code. Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c538a3178..7e7fa584e 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.0-rc.4 + git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1-0.20241205083807-762d7f9f9f08 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 diff --git a/go.sum b/go.sum index 064f3274e..dec34ff6e 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.0-rc.4 h1:o3iqVmbvFsfe8kpB2Hvuix6Q/tAhbiPLP91xK4lmoBQ= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.0-rc.4/go.mod h1:5fSm/l5xSjGWqsPUffSdboiGFUHa7y/1S0fvxzQowN8= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1-0.20241205083807-762d7f9f9f08 h1:tl1TT+zNk1lF/J5EaD3syDrTaYbQwvJKVOVENM4oQ+k= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1-0.20241205083807-762d7f9f9f08/go.mod h1:5fSm/l5xSjGWqsPUffSdboiGFUHa7y/1S0fvxzQowN8= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d h1:uJ/wvuMdepbkaV8XMS5uN9B0FQWMep0CttSuDZiDhq0= From a353d45742e9ac48260a9d2f32b81c4b3c7c769c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 5 Dec 2024 11:52:30 +0300 Subject: [PATCH 1608/1943] [#1541] morph/event: Simplify container contract event parsing Signed-off-by: Evgenii Stratonikov --- pkg/morph/event/container/delete.go | 32 ++++++-------------- pkg/morph/event/container/delete_test.go | 3 +- pkg/morph/event/container/put.go | 37 ++++++------------------ pkg/morph/event/container/put_test.go | 28 +++++++++++++----- 4 files changed, 39 insertions(+), 61 deletions(-) diff --git a/pkg/morph/event/container/delete.go b/pkg/morph/event/container/delete.go index a206307f8..d28f6d521 100644 --- a/pkg/morph/event/container/delete.go +++ b/pkg/morph/event/container/delete.go @@ -3,7 +3,7 @@ package container import ( "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -58,28 +58,14 @@ func (DeleteSuccess) MorphEvent() {} // ParseDeleteSuccess decodes notification event thrown by Container contract into // DeleteSuccess and returns it as event.Event. func ParseDeleteSuccess(e *state.ContainedNotificationEvent) (event.Event, error) { - items, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("parse stack array from raw notification event: %w", err) + var dse container.DeleteSuccessEvent + if err := dse.FromStackItem(e.Item); err != nil { + return nil, fmt.Errorf("parse container.DeleteSuccessEvent: %w", err) } - const expectedItemNumDeleteSuccess = 1 - - if ln := len(items); ln != expectedItemNumDeleteSuccess { - return nil, event.WrongNumberOfParameters(expectedItemNumDeleteSuccess, ln) - } - - binID, err := client.BytesFromStackItem(items[0]) - if err != nil { - return nil, fmt.Errorf("parse container ID item: %w", err) - } - - var res DeleteSuccess - - err = res.ID.Decode(binID) - if err != nil { - return nil, fmt.Errorf("decode container ID: %w", err) - } - - return res, nil + var cnr cid.ID + cnr.SetSHA256(dse.ContainerID) + return DeleteSuccess{ + ID: cnr, + }, nil } diff --git a/pkg/morph/event/container/delete_test.go b/pkg/morph/event/container/delete_test.go index 627c5fcf5..62e7d7277 100644 --- a/pkg/morph/event/container/delete_test.go +++ b/pkg/morph/event/container/delete_test.go @@ -4,7 +4,6 @@ import ( "crypto/sha256" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/stretchr/testify/require" @@ -18,7 +17,7 @@ func TestParseDeleteSuccess(t *testing.T) { } _, err := ParseDeleteSuccess(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(1, len(prms)).Error()) + require.Error(t, err) }) t.Run("wrong container parameter", func(t *testing.T) { diff --git a/pkg/morph/event/container/put.go b/pkg/morph/event/container/put.go index 335034bf3..b09394ba4 100644 --- a/pkg/morph/event/container/put.go +++ b/pkg/morph/event/container/put.go @@ -3,7 +3,7 @@ package container import ( "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -78,33 +78,14 @@ func (PutSuccess) MorphEvent() {} // ParsePutSuccess decodes notification event thrown by Container contract into // PutSuccess and returns it as event.Event. func ParsePutSuccess(e *state.ContainedNotificationEvent) (event.Event, error) { - items, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("parse stack array from raw notification event: %w", err) + var pse container.PutSuccessEvent + if err := pse.FromStackItem(e.Item); err != nil { + return nil, fmt.Errorf("parse container.PutSuccessEvent: %w", err) } - const expectedItemNumPutSuccess = 2 - - if ln := len(items); ln != expectedItemNumPutSuccess { - return nil, event.WrongNumberOfParameters(expectedItemNumPutSuccess, ln) - } - - binID, err := client.BytesFromStackItem(items[0]) - if err != nil { - return nil, fmt.Errorf("parse container ID item: %w", err) - } - - _, err = client.BytesFromStackItem(items[1]) - if err != nil { - return nil, fmt.Errorf("parse public key item: %w", err) - } - - var res PutSuccess - - err = res.ID.Decode(binID) - if err != nil { - return nil, fmt.Errorf("decode container ID: %w", err) - } - - return res, nil + var cnr cid.ID + cnr.SetSHA256(pse.ContainerID) + return PutSuccess{ + ID: cnr, + }, nil } diff --git a/pkg/morph/event/container/put_test.go b/pkg/morph/event/container/put_test.go index 3622f9943..dd5c7ea93 100644 --- a/pkg/morph/event/container/put_test.go +++ b/pkg/morph/event/container/put_test.go @@ -4,8 +4,8 @@ import ( "crypto/sha256" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/stretchr/testify/require" ) @@ -17,7 +17,7 @@ func TestParsePutSuccess(t *testing.T) { } _, err := ParsePutSuccess(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(2, len(prms)).Error()) + require.Error(t, err) }) t.Run("wrong container ID parameter", func(t *testing.T) { @@ -35,18 +35,30 @@ func TestParsePutSuccess(t *testing.T) { id.Encode(binID) t.Run("wrong public key parameter", func(t *testing.T) { - _, err := ParsePutSuccess(createNotifyEventFromItems([]stackitem.Item{ - stackitem.NewByteArray(binID), - stackitem.NewMap(), - })) + t.Run("wrong type", func(t *testing.T) { + _, err := ParsePutSuccess(createNotifyEventFromItems([]stackitem.Item{ + stackitem.NewByteArray(binID), + stackitem.NewMap(), + })) - require.Error(t, err) + require.Error(t, err) + }) + t.Run("garbage data", func(t *testing.T) { + _, err := ParsePutSuccess(createNotifyEventFromItems([]stackitem.Item{ + stackitem.NewByteArray(binID), + stackitem.NewByteArray([]byte("key")), + })) + require.Error(t, err) + }) }) t.Run("correct behavior", func(t *testing.T) { + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + ev, err := ParsePutSuccess(createNotifyEventFromItems([]stackitem.Item{ stackitem.NewByteArray(binID), - stackitem.NewByteArray([]byte("key")), + stackitem.NewByteArray(pk.PublicKey().Bytes()), })) require.NoError(t, err) From 1c12f23b841511b95efac32b62470a7b1e316fb3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 5 Dec 2024 11:58:38 +0300 Subject: [PATCH 1609/1943] [#1541] morph/event: Simplify netmap contract event parsing Signed-off-by: Evgenii Stratonikov --- pkg/morph/event/netmap/epoch.go | 21 +++++---------------- pkg/morph/event/netmap/epoch_test.go | 3 +-- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/pkg/morph/event/netmap/epoch.go b/pkg/morph/event/netmap/epoch.go index e454e2a6a..4dcc0d035 100644 --- a/pkg/morph/event/netmap/epoch.go +++ b/pkg/morph/event/netmap/epoch.go @@ -1,9 +1,7 @@ package netmap import ( - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/util" @@ -37,22 +35,13 @@ func (s NewEpoch) TxHash() util.Uint256 { // // Result is type of NewEpoch. func ParseNewEpoch(e *state.ContainedNotificationEvent) (event.Event, error) { - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) - } - - if ln := len(params); ln != 1 { - return nil, event.WrongNumberOfParameters(1, ln) - } - - prmEpochNum, err := client.IntFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get integer epoch number: %w", err) + var nee netmap.NewEpochEvent + if err := nee.FromStackItem(e.Item); err != nil { + return nil, err } return NewEpoch{ - Num: uint64(prmEpochNum), + Num: uint64(nee.Epoch.Uint64()), Hash: e.Container, }, nil } diff --git a/pkg/morph/event/netmap/epoch_test.go b/pkg/morph/event/netmap/epoch_test.go index bc267ecb6..6ff692327 100644 --- a/pkg/morph/event/netmap/epoch_test.go +++ b/pkg/morph/event/netmap/epoch_test.go @@ -4,7 +4,6 @@ import ( "math/big" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/stretchr/testify/require" ) @@ -17,7 +16,7 @@ func TestParseNewEpoch(t *testing.T) { } _, err := ParseNewEpoch(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(1, len(prms)).Error()) + require.Error(t, err) }) t.Run("wrong first parameter type", func(t *testing.T) { From d1bc4351c3d2904bedfad970d13c9bcde55e3d31 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 6 Dec 2024 13:59:15 +0300 Subject: [PATCH 1610/1943] [#1545] morph/event: Simplify frostfs contract event parsing Signed-off-by: Evgenii Stratonikov --- pkg/morph/event/frostfs/cheque.go | 59 ++++++------------------ pkg/morph/event/frostfs/cheque_test.go | 3 +- pkg/morph/event/frostfs/config.go | 44 ++++-------------- pkg/morph/event/frostfs/config_test.go | 3 +- pkg/morph/event/frostfs/deposit.go | 55 ++++------------------ pkg/morph/event/frostfs/deposit_test.go | 9 ++-- pkg/morph/event/frostfs/withdraw.go | 43 ++++------------- pkg/morph/event/frostfs/withdraw_test.go | 9 ++-- 8 files changed, 52 insertions(+), 173 deletions(-) diff --git a/pkg/morph/event/frostfs/cheque.go b/pkg/morph/event/frostfs/cheque.go index eae2a23f5..cf56464b8 100644 --- a/pkg/morph/event/frostfs/cheque.go +++ b/pkg/morph/event/frostfs/cheque.go @@ -3,7 +3,7 @@ package frostfs import ( "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/util" @@ -34,53 +34,20 @@ func (c Cheque) LockAccount() util.Uint160 { return c.LockValue } // ParseCheque from notification into cheque structure. func ParseCheque(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Cheque - err error - ) + var ce frostfs.ChequeEvent + if err := ce.FromStackItem(e.Item); err != nil { + return nil, fmt.Errorf("parse frostfs.ChequeEvent: %w", err) + } - params, err := event.ParseStackArray(e) + lock, err := util.Uint160DecodeBytesBE(ce.LockAccount) if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) + return nil, fmt.Errorf("parse frostfs.ChequeEvent: field LockAccount: %w", err) } - if ln := len(params); ln != 4 { - return nil, event.WrongNumberOfParameters(4, ln) - } - - // parse id - ev.IDValue, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get cheque id: %w", err) - } - - // parse user - user, err := client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get cheque user: %w", err) - } - - ev.UserValue, err = util.Uint160DecodeBytesBE(user) - if err != nil { - return nil, fmt.Errorf("could not convert cheque user to uint160: %w", err) - } - - // parse amount - ev.AmountValue, err = client.IntFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get cheque amount: %w", err) - } - - // parse lock account - lock, err := client.BytesFromStackItem(params[3]) - if err != nil { - return nil, fmt.Errorf("could not get cheque lock account: %w", err) - } - - ev.LockValue, err = util.Uint160DecodeBytesBE(lock) - if err != nil { - return nil, fmt.Errorf("could not convert cheque lock account to uint160: %w", err) - } - - return ev, nil + return Cheque{ + IDValue: ce.Id, + AmountValue: ce.Amount.Int64(), + UserValue: ce.User, + LockValue: lock, + }, nil } diff --git a/pkg/morph/event/frostfs/cheque_test.go b/pkg/morph/event/frostfs/cheque_test.go index ab177757f..d92b7922b 100644 --- a/pkg/morph/event/frostfs/cheque_test.go +++ b/pkg/morph/event/frostfs/cheque_test.go @@ -4,7 +4,6 @@ import ( "math/big" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" @@ -27,7 +26,7 @@ func TestParseCheque(t *testing.T) { } _, err := ParseCheque(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(4, len(prms)).Error()) + require.Error(t, err) }) t.Run("wrong id parameter", func(t *testing.T) { diff --git a/pkg/morph/event/frostfs/config.go b/pkg/morph/event/frostfs/config.go index 4c87634c2..805e80f3c 100644 --- a/pkg/morph/event/frostfs/config.go +++ b/pkg/morph/event/frostfs/config.go @@ -3,7 +3,7 @@ package frostfs import ( "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/util" @@ -36,39 +36,15 @@ func (u Config) Key() []byte { return u.KeyValue } func (u Config) Value() []byte { return u.ValueValue } func ParseConfig(e *state.ContainedNotificationEvent) (event.Event, error) { - var ( - ev Config - err error - ) - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) + var sce frostfs.SetConfigEvent + if err := sce.FromStackItem(e.Item); err != nil { + return nil, fmt.Errorf("parse frostfs.SetConfigEvent: %w", err) } - if ln := len(params); ln != 3 { - return nil, event.WrongNumberOfParameters(3, ln) - } - - // parse id - ev.IDValue, err = client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get config update id: %w", err) - } - - // parse key - ev.KeyValue, err = client.BytesFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get config key: %w", err) - } - - // parse value - ev.ValueValue, err = client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get config value: %w", err) - } - - ev.TxHashValue = e.Container - - return ev, nil + return Config{ + KeyValue: sce.Key, + ValueValue: sce.Value, + IDValue: sce.Id, + TxHashValue: e.Container, + }, nil } diff --git a/pkg/morph/event/frostfs/config_test.go b/pkg/morph/event/frostfs/config_test.go index dcd4201e4..8acc8c15c 100644 --- a/pkg/morph/event/frostfs/config_test.go +++ b/pkg/morph/event/frostfs/config_test.go @@ -3,7 +3,6 @@ package frostfs import ( "testing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/stretchr/testify/require" ) @@ -21,7 +20,7 @@ func TestParseConfig(t *testing.T) { } _, err := ParseConfig(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(3, len(prms)).Error()) + require.Error(t, err) }) t.Run("wrong first parameter", func(t *testing.T) { diff --git a/pkg/morph/event/frostfs/deposit.go b/pkg/morph/event/frostfs/deposit.go index d8a3b82f0..fcb01577e 100644 --- a/pkg/morph/event/frostfs/deposit.go +++ b/pkg/morph/event/frostfs/deposit.go @@ -3,7 +3,7 @@ package frostfs import ( "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/util" @@ -34,50 +34,15 @@ func (d Deposit) Amount() int64 { return d.AmountValue } // ParseDeposit notification into deposit structure. func ParseDeposit(e *state.ContainedNotificationEvent) (event.Event, error) { - var ev Deposit - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) + var de frostfs.DepositEvent + if err := de.FromStackItem(e.Item); err != nil { + return nil, fmt.Errorf("parse frostfs.DepositEvent: %w", err) } - if ln := len(params); ln != 4 { - return nil, event.WrongNumberOfParameters(4, ln) - } - - // parse from - from, err := client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get deposit sender: %w", err) - } - - ev.FromValue, err = util.Uint160DecodeBytesBE(from) - if err != nil { - return nil, fmt.Errorf("could not convert deposit sender to uint160: %w", err) - } - - // parse amount - ev.AmountValue, err = client.IntFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get deposit amount: %w", err) - } - - // parse to - to, err := client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get deposit receiver: %w", err) - } - - ev.ToValue, err = util.Uint160DecodeBytesBE(to) - if err != nil { - return nil, fmt.Errorf("could not convert deposit receiver to uint160: %w", err) - } - - // parse id - ev.IDValue, err = client.BytesFromStackItem(params[3]) - if err != nil { - return nil, fmt.Errorf("could not get deposit id: %w", err) - } - - return ev, nil + return Deposit{ + IDValue: de.TxHash[:], + AmountValue: de.Amount.Int64(), + FromValue: de.From, + ToValue: de.Receiver, + }, nil } diff --git a/pkg/morph/event/frostfs/deposit_test.go b/pkg/morph/event/frostfs/deposit_test.go index f279a7f9c..38d3e61f6 100644 --- a/pkg/morph/event/frostfs/deposit_test.go +++ b/pkg/morph/event/frostfs/deposit_test.go @@ -4,7 +4,6 @@ import ( "math/big" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/stretchr/testify/require" @@ -12,7 +11,7 @@ import ( func TestParseDeposit(t *testing.T) { var ( - id = []byte("Hello World") + id = util.Uint256{0, 1, 2, 3} from = util.Uint160{0x1, 0x2, 0x3} to = util.Uint160{0x3, 0x2, 0x1} @@ -26,7 +25,7 @@ func TestParseDeposit(t *testing.T) { } _, err := ParseDeposit(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(4, len(prms)).Error()) + require.Error(t, err) }) t.Run("wrong from parameter", func(t *testing.T) { @@ -72,12 +71,12 @@ func TestParseDeposit(t *testing.T) { stackitem.NewByteArray(from.BytesBE()), stackitem.NewBigInteger(new(big.Int).SetInt64(amount)), stackitem.NewByteArray(to.BytesBE()), - stackitem.NewByteArray(id), + stackitem.NewByteArray(id[:]), })) require.NoError(t, err) require.Equal(t, Deposit{ - IDValue: id, + IDValue: id[:], AmountValue: amount, FromValue: from, ToValue: to, diff --git a/pkg/morph/event/frostfs/withdraw.go b/pkg/morph/event/frostfs/withdraw.go index f48067f86..2568b6512 100644 --- a/pkg/morph/event/frostfs/withdraw.go +++ b/pkg/morph/event/frostfs/withdraw.go @@ -3,7 +3,7 @@ package frostfs import ( "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/util" @@ -30,39 +30,14 @@ func (w Withdraw) Amount() int64 { return w.AmountValue } // ParseWithdraw notification into withdraw structure. func ParseWithdraw(e *state.ContainedNotificationEvent) (event.Event, error) { - var ev Withdraw - - params, err := event.ParseStackArray(e) - if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) + var we frostfs.WithdrawEvent + if err := we.FromStackItem(e.Item); err != nil { + return nil, fmt.Errorf("parse frostfs.WithdrawEvent: %w", err) } - if ln := len(params); ln != 3 { - return nil, event.WrongNumberOfParameters(3, ln) - } - - // parse user - user, err := client.BytesFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get withdraw user: %w", err) - } - - ev.UserValue, err = util.Uint160DecodeBytesBE(user) - if err != nil { - return nil, fmt.Errorf("could not convert withdraw user to uint160: %w", err) - } - - // parse amount - ev.AmountValue, err = client.IntFromStackItem(params[1]) - if err != nil { - return nil, fmt.Errorf("could not get withdraw amount: %w", err) - } - - // parse id - ev.IDValue, err = client.BytesFromStackItem(params[2]) - if err != nil { - return nil, fmt.Errorf("could not get withdraw id: %w", err) - } - - return ev, nil + return Withdraw{ + IDValue: we.TxHash[:], + AmountValue: we.Amount.Int64(), + UserValue: we.User, + }, nil } diff --git a/pkg/morph/event/frostfs/withdraw_test.go b/pkg/morph/event/frostfs/withdraw_test.go index 33435d19a..e382305e6 100644 --- a/pkg/morph/event/frostfs/withdraw_test.go +++ b/pkg/morph/event/frostfs/withdraw_test.go @@ -4,7 +4,6 @@ import ( "math/big" "testing" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/stretchr/testify/require" @@ -12,7 +11,7 @@ import ( func TestParseWithdraw(t *testing.T) { var ( - id = []byte("Hello World") + id = util.Uint256{1, 2, 3} user = util.Uint160{0x1, 0x2, 0x3} amount int64 = 10 @@ -25,7 +24,7 @@ func TestParseWithdraw(t *testing.T) { } _, err := ParseWithdraw(createNotifyEventFromItems(prms)) - require.EqualError(t, err, event.WrongNumberOfParameters(3, len(prms)).Error()) + require.Error(t, err) }) t.Run("wrong user parameter", func(t *testing.T) { @@ -59,12 +58,12 @@ func TestParseWithdraw(t *testing.T) { ev, err := ParseWithdraw(createNotifyEventFromItems([]stackitem.Item{ stackitem.NewByteArray(user.BytesBE()), stackitem.NewBigInteger(new(big.Int).SetInt64(amount)), - stackitem.NewByteArray(id), + stackitem.NewByteArray(id[:]), })) require.NoError(t, err) require.Equal(t, Withdraw{ - IDValue: id, + IDValue: id[:], AmountValue: amount, UserValue: user, }, ev) From 7e542906ef9085a792e5ec5f7185bfdd9acecedb Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 6 Dec 2024 14:48:44 +0300 Subject: [PATCH 1611/1943] [#1539] go.mod: Bump `frostfs-sdk-go` version * Also fix placement unit-test in object manager Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 ++-- pkg/services/object_manager/placement/cache_test.go | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 7e7fa584e..6ac37d343 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241107121119-cb813e27a823 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 diff --git a/go.sum b/go.sum index dec34ff6e..e084c2445 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241107121119-cb813e27a823 h1:sepm9FeuoInmygH1K/+3L+Yp5bJhGiVi/oGCH6Emp2c= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241107121119-cb813e27a823/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d h1:FpXI+mOrmJk3t2MKQFZuhLjCHDyDeo5rtP1WXl7gUWc= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= diff --git a/pkg/services/object_manager/placement/cache_test.go b/pkg/services/object_manager/placement/cache_test.go index a890d5357..7242970b5 100644 --- a/pkg/services/object_manager/placement/cache_test.go +++ b/pkg/services/object_manager/placement/cache_test.go @@ -85,7 +85,10 @@ func TestContainerNodesCache(t *testing.T) { }) t.Run("the error is propagated", func(t *testing.T) { var pp netmapSDK.PlacementPolicy - require.NoError(t, pp.DecodeString("REP 1 SELECT 1 FROM X FILTER ATTR EQ 42 AS X")) + r := netmapSDK.ReplicaDescriptor{} + r.SetNumberOfObjects(1) + r.SetSelectorName("Missing") + pp.AddReplicas(r) c := placement.NewContainerNodesCache(size) _, err := c.ContainerNodes(nm(1, nodes[0:1]), cidtest.ID(), pp) From ac0511d21423df1d191539f8272f962c23dfb62b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Dec 2024 16:52:46 +0300 Subject: [PATCH 1612/1943] [#1549] controlSvc: Drop deprecated EvacuateShard rpc Signed-off-by: Dmitrii Stepanov --- .../modules/control/evacuate_shard.go | 56 ------ cmd/frostfs-cli/modules/control/evacuation.go | 9 +- cmd/frostfs-cli/modules/control/shards.go | 2 - pkg/services/control/rpc.go | 14 -- pkg/services/control/server/evacuate.go | 188 ------------------ pkg/services/control/server/evacuate_async.go | 148 +++++++++++++- pkg/services/control/service.proto | 5 - pkg/services/control/service_grpc.pb.go | 43 ---- 8 files changed, 151 insertions(+), 314 deletions(-) delete mode 100644 cmd/frostfs-cli/modules/control/evacuate_shard.go delete mode 100644 pkg/services/control/server/evacuate.go diff --git a/cmd/frostfs-cli/modules/control/evacuate_shard.go b/cmd/frostfs-cli/modules/control/evacuate_shard.go deleted file mode 100644 index 1e48c1df4..000000000 --- a/cmd/frostfs-cli/modules/control/evacuate_shard.go +++ /dev/null @@ -1,56 +0,0 @@ -package control - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" - "github.com/spf13/cobra" -) - -const ignoreErrorsFlag = "no-errors" - -var evacuateShardCmd = &cobra.Command{ - Use: "evacuate", - Short: "Evacuate objects from shard", - Long: "Evacuate objects from shard to other shards", - Run: evacuateShard, - Deprecated: "use frostfs-cli control shards evacuation start", -} - -func evacuateShard(cmd *cobra.Command, _ []string) { - pk := key.Get(cmd) - - req := &control.EvacuateShardRequest{Body: new(control.EvacuateShardRequest_Body)} - req.Body.Shard_ID = getShardIDList(cmd) - req.Body.IgnoreErrors, _ = cmd.Flags().GetBool(ignoreErrorsFlag) - - signRequest(cmd, pk, req) - - cli := getClient(cmd, pk) - - var resp *control.EvacuateShardResponse - var err error - err = cli.ExecRaw(func(client *client.Client) error { - resp, err = control.EvacuateShard(client, req) - return err - }) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - cmd.Printf("Objects moved: %d\n", resp.GetBody().GetCount()) - - verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) - - cmd.Println("Shard has successfully been evacuated.") -} - -func initControlEvacuateShardCmd() { - initControlFlags(evacuateShardCmd) - - flags := evacuateShardCmd.Flags() - flags.StringSlice(shardIDFlag, nil, "List of shard IDs in base58 encoding") - flags.Bool(shardAllFlag, false, "Process all shards") - flags.Bool(ignoreErrorsFlag, false, "Skip invalid/unreadable objects") - - evacuateShardCmd.MarkFlagsMutuallyExclusive(shardIDFlag, shardAllFlag) -} diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 73700e56d..8032bf09a 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -17,10 +17,11 @@ import ( ) const ( - awaitFlag = "await" - noProgressFlag = "no-progress" - scopeFlag = "scope" - repOneOnlyFlag = "rep-one-only" + awaitFlag = "await" + noProgressFlag = "no-progress" + scopeFlag = "scope" + repOneOnlyFlag = "rep-one-only" + ignoreErrorsFlag = "no-errors" containerWorkerCountFlag = "container-worker-count" objectWorkerCountFlag = "object-worker-count" diff --git a/cmd/frostfs-cli/modules/control/shards.go b/cmd/frostfs-cli/modules/control/shards.go index 329cb9100..3483f5d62 100644 --- a/cmd/frostfs-cli/modules/control/shards.go +++ b/cmd/frostfs-cli/modules/control/shards.go @@ -13,7 +13,6 @@ var shardsCmd = &cobra.Command{ func initControlShardsCmd() { shardsCmd.AddCommand(listShardsCmd) shardsCmd.AddCommand(setShardModeCmd) - shardsCmd.AddCommand(evacuateShardCmd) shardsCmd.AddCommand(evacuationShardCmd) shardsCmd.AddCommand(flushCacheCmd) shardsCmd.AddCommand(doctorCmd) @@ -23,7 +22,6 @@ func initControlShardsCmd() { initControlShardsListCmd() initControlSetShardModeCmd() - initControlEvacuateShardCmd() initControlEvacuationShardCmd() initControlFlushCacheCmd() initControlDoctorCmd() diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 514061db4..6982d780d 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -15,7 +15,6 @@ const ( rpcListShards = "ListShards" rpcSetShardMode = "SetShardMode" rpcSynchronizeTree = "SynchronizeTree" - rpcEvacuateShard = "EvacuateShard" rpcStartShardEvacuation = "StartShardEvacuation" rpcGetShardEvacuationStatus = "GetShardEvacuationStatus" rpcResetShardEvacuationStatus = "ResetShardEvacuationStatus" @@ -162,19 +161,6 @@ func SynchronizeTree(cli *client.Client, req *SynchronizeTreeRequest, opts ...cl return wResp.message, nil } -// EvacuateShard executes ControlService.EvacuateShard RPC. -func EvacuateShard(cli *client.Client, req *EvacuateShardRequest, opts ...client.CallOption) (*EvacuateShardResponse, error) { - wResp := newResponseWrapper[EvacuateShardResponse]() - wReq := &requestWrapper{m: req} - - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcEvacuateShard), wReq, wResp, opts...) - if err != nil { - return nil, err - } - - return wResp.message, nil -} - // StartShardEvacuation executes ControlService.StartShardEvacuation RPC. func StartShardEvacuation(cli *client.Client, req *StartShardEvacuationRequest, opts ...client.CallOption) (*StartShardEvacuationResponse, error) { wResp := newResponseWrapper[StartShardEvacuationResponse]() diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go deleted file mode 100644 index ae3413373..000000000 --- a/pkg/services/control/server/evacuate.go +++ /dev/null @@ -1,188 +0,0 @@ -package control - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -var errFailedToBuildListOfContainerNodes = errors.New("can't build a list of container nodes") - -func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRequest) (*control.EvacuateShardResponse, error) { - err := s.isValidRequest(req) - if err != nil { - return nil, status.Error(codes.PermissionDenied, err.Error()) - } - - prm := engine.EvacuateShardPrm{ - ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), - IgnoreErrors: req.GetBody().GetIgnoreErrors(), - ObjectsHandler: s.replicateObject, - Scope: engine.EvacuateScopeObjects, - } - - res, err := s.s.Evacuate(ctx, prm) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - - resp := &control.EvacuateShardResponse{ - Body: &control.EvacuateShardResponse_Body{ - Count: uint32(res.ObjectsEvacuated()), - }, - } - - err = ctrlmessage.Sign(s.key, resp) - if err != nil { - return nil, status.Error(codes.Internal, err.Error()) - } - return resp, nil -} - -func (s *Server) replicateObject(ctx context.Context, addr oid.Address, obj *objectSDK.Object) (bool, error) { - cid, ok := obj.ContainerID() - if !ok { - // Return nil to prevent situations where a shard can't be evacuated - // because of a single bad/corrupted object. - return false, nil - } - - nodes, err := s.getContainerNodes(cid) - if err != nil { - return false, err - } - - if len(nodes) == 0 { - return false, nil - } - - var res replicatorResult - task := replicator.Task{ - NumCopies: 1, - Addr: addr, - Obj: obj, - Nodes: nodes, - } - s.replicator.HandleReplicationTask(ctx, task, &res) - - if res.count == 0 { - return false, errors.New("object was not replicated") - } - return true, nil -} - -func (s *Server) replicateTree(ctx context.Context, contID cid.ID, treeID string, forest pilorama.Forest) (bool, string, error) { - nodes, err := s.getContainerNodes(contID) - if err != nil { - return false, "", err - } - if len(nodes) == 0 { - return false, "", nil - } - - for _, node := range nodes { - err = s.replicateTreeToNode(ctx, forest, contID, treeID, node) - if err == nil { - return true, hex.EncodeToString(node.PublicKey()), nil - } - } - return false, "", err -} - -func (s *Server) replicateTreeToNode(ctx context.Context, forest pilorama.Forest, contID cid.ID, treeID string, node netmap.NodeInfo) error { - rawCID := make([]byte, sha256.Size) - contID.Encode(rawCID) - - var height uint64 - for { - op, err := forest.TreeGetOpLog(ctx, contID, treeID, height) - if err != nil { - return err - } - - if op.Time == 0 { - return nil - } - - req := &tree.ApplyRequest{ - Body: &tree.ApplyRequest_Body{ - ContainerId: rawCID, - TreeId: treeID, - Operation: &tree.LogMove{ - ParentId: op.Parent, - Meta: op.Meta.Bytes(), - ChildId: op.Child, - }, - }, - } - - err = tree.SignMessage(req, s.key) - if err != nil { - return fmt.Errorf("can't message apply request: %w", err) - } - - err = s.treeService.ReplicateTreeOp(ctx, node, req) - if err != nil { - return err - } - - height = op.Time + 1 - } -} - -func (s *Server) getContainerNodes(contID cid.ID) ([]netmap.NodeInfo, error) { - nm, err := s.netMapSrc.GetNetMap(0) - if err != nil { - return nil, err - } - - c, err := s.cnrSrc.Get(contID) - if err != nil { - return nil, err - } - - binCnr := make([]byte, sha256.Size) - contID.Encode(binCnr) - - ns, err := nm.ContainerNodes(c.Value.PlacementPolicy(), binCnr) - if err != nil { - return nil, errFailedToBuildListOfContainerNodes - } - - nodes := placement.FlattenNodes(ns) - bs := (*keys.PublicKey)(&s.key.PublicKey).Bytes() - for i := 0; i < len(nodes); i++ { // don't use range, slice mutates in body - if bytes.Equal(nodes[i].PublicKey(), bs) { - copy(nodes[i:], nodes[i+1:]) - nodes = nodes[:len(nodes)-1] - } - } - return nodes, nil -} - -type replicatorResult struct { - count int -} - -// SubmitSuccessfulReplication implements the replicator.TaskResult interface. -func (r *replicatorResult) SubmitSuccessfulReplication(_ netmap.NodeInfo) { - r.count++ -} diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index 146ac7e16..04465eb26 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -1,17 +1,32 @@ package control import ( + "bytes" "context" + "crypto/sha256" + "encoding/hex" "errors" + "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) +var errFailedToBuildListOfContainerNodes = errors.New("can't build a list of container nodes") + func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartShardEvacuationRequest) (*control.StartShardEvacuationResponse, error) { err := s.isValidRequest(req) if err != nil { @@ -34,8 +49,7 @@ func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartSha RepOneOnly: req.GetBody().GetRepOneOnly(), } - _, err = s.s.Evacuate(ctx, prm) - if err != nil { + if err = s.s.Evacuate(ctx, prm); err != nil { var logicalErr logicerr.Logical if errors.As(err, &logicalErr) { return nil, status.Error(codes.Aborted, err.Error()) @@ -135,3 +149,133 @@ func (s *Server) ResetShardEvacuationStatus(ctx context.Context, req *control.Re } return resp, nil } + +func (s *Server) replicateObject(ctx context.Context, addr oid.Address, obj *objectSDK.Object) (bool, error) { + cid, ok := obj.ContainerID() + if !ok { + // Return nil to prevent situations where a shard can't be evacuated + // because of a single bad/corrupted object. + return false, nil + } + + nodes, err := s.getContainerNodes(cid) + if err != nil { + return false, err + } + + if len(nodes) == 0 { + return false, nil + } + + var res replicatorResult + task := replicator.Task{ + NumCopies: 1, + Addr: addr, + Obj: obj, + Nodes: nodes, + } + s.replicator.HandleReplicationTask(ctx, task, &res) + + if res.count == 0 { + return false, errors.New("object was not replicated") + } + return true, nil +} + +func (s *Server) replicateTree(ctx context.Context, contID cid.ID, treeID string, forest pilorama.Forest) (bool, string, error) { + nodes, err := s.getContainerNodes(contID) + if err != nil { + return false, "", err + } + if len(nodes) == 0 { + return false, "", nil + } + + for _, node := range nodes { + err = s.replicateTreeToNode(ctx, forest, contID, treeID, node) + if err == nil { + return true, hex.EncodeToString(node.PublicKey()), nil + } + } + return false, "", err +} + +func (s *Server) replicateTreeToNode(ctx context.Context, forest pilorama.Forest, contID cid.ID, treeID string, node netmap.NodeInfo) error { + rawCID := make([]byte, sha256.Size) + contID.Encode(rawCID) + + var height uint64 + for { + op, err := forest.TreeGetOpLog(ctx, contID, treeID, height) + if err != nil { + return err + } + + if op.Time == 0 { + return nil + } + + req := &tree.ApplyRequest{ + Body: &tree.ApplyRequest_Body{ + ContainerId: rawCID, + TreeId: treeID, + Operation: &tree.LogMove{ + ParentId: op.Parent, + Meta: op.Meta.Bytes(), + ChildId: op.Child, + }, + }, + } + + err = tree.SignMessage(req, s.key) + if err != nil { + return fmt.Errorf("can't message apply request: %w", err) + } + + err = s.treeService.ReplicateTreeOp(ctx, node, req) + if err != nil { + return err + } + + height = op.Time + 1 + } +} + +func (s *Server) getContainerNodes(contID cid.ID) ([]netmap.NodeInfo, error) { + nm, err := s.netMapSrc.GetNetMap(0) + if err != nil { + return nil, err + } + + c, err := s.cnrSrc.Get(contID) + if err != nil { + return nil, err + } + + binCnr := make([]byte, sha256.Size) + contID.Encode(binCnr) + + ns, err := nm.ContainerNodes(c.Value.PlacementPolicy(), binCnr) + if err != nil { + return nil, errFailedToBuildListOfContainerNodes + } + + nodes := placement.FlattenNodes(ns) + bs := (*keys.PublicKey)(&s.key.PublicKey).Bytes() + for i := 0; i < len(nodes); i++ { // don't use range, slice mutates in body + if bytes.Equal(nodes[i].PublicKey(), bs) { + copy(nodes[i:], nodes[i+1:]) + nodes = nodes[:len(nodes)-1] + } + } + return nodes, nil +} + +type replicatorResult struct { + count int +} + +// SubmitSuccessfulReplication implements the replicator.TaskResult interface. +func (r *replicatorResult) SubmitSuccessfulReplication(_ netmap.NodeInfo) { + r.count++ +} diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index ae1939e13..97ecf9a8c 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -30,11 +30,6 @@ service ControlService { // Synchronizes all log operations for the specified tree. rpc SynchronizeTree(SynchronizeTreeRequest) returns (SynchronizeTreeResponse); - // EvacuateShard moves all data from one shard to the others. - // Deprecated: Use - // StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation - rpc EvacuateShard(EvacuateShardRequest) returns (EvacuateShardResponse); - // StartShardEvacuation starts moving all data from one shard to the others. rpc StartShardEvacuation(StartShardEvacuationRequest) returns (StartShardEvacuationResponse); diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index f5cfefa85..987e08c59 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -26,7 +26,6 @@ const ( ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards" ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode" ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree" - ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard" ControlService_StartShardEvacuation_FullMethodName = "/control.ControlService/StartShardEvacuation" ControlService_GetShardEvacuationStatus_FullMethodName = "/control.ControlService/GetShardEvacuationStatus" ControlService_ResetShardEvacuationStatus_FullMethodName = "/control.ControlService/ResetShardEvacuationStatus" @@ -62,10 +61,6 @@ type ControlServiceClient interface { SetShardMode(ctx context.Context, in *SetShardModeRequest, opts ...grpc.CallOption) (*SetShardModeResponse, error) // Synchronizes all log operations for the specified tree. SynchronizeTree(ctx context.Context, in *SynchronizeTreeRequest, opts ...grpc.CallOption) (*SynchronizeTreeResponse, error) - // EvacuateShard moves all data from one shard to the others. - // Deprecated: Use - // StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation - EvacuateShard(ctx context.Context, in *EvacuateShardRequest, opts ...grpc.CallOption) (*EvacuateShardResponse, error) // StartShardEvacuation starts moving all data from one shard to the others. StartShardEvacuation(ctx context.Context, in *StartShardEvacuationRequest, opts ...grpc.CallOption) (*StartShardEvacuationResponse, error) // GetShardEvacuationStatus returns evacuation status. @@ -173,15 +168,6 @@ func (c *controlServiceClient) SynchronizeTree(ctx context.Context, in *Synchron return out, nil } -func (c *controlServiceClient) EvacuateShard(ctx context.Context, in *EvacuateShardRequest, opts ...grpc.CallOption) (*EvacuateShardResponse, error) { - out := new(EvacuateShardResponse) - err := c.cc.Invoke(ctx, ControlService_EvacuateShard_FullMethodName, in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *controlServiceClient) StartShardEvacuation(ctx context.Context, in *StartShardEvacuationRequest, opts ...grpc.CallOption) (*StartShardEvacuationResponse, error) { out := new(StartShardEvacuationResponse) err := c.cc.Invoke(ctx, ControlService_StartShardEvacuation_FullMethodName, in, out, opts...) @@ -335,10 +321,6 @@ type ControlServiceServer interface { SetShardMode(context.Context, *SetShardModeRequest) (*SetShardModeResponse, error) // Synchronizes all log operations for the specified tree. SynchronizeTree(context.Context, *SynchronizeTreeRequest) (*SynchronizeTreeResponse, error) - // EvacuateShard moves all data from one shard to the others. - // Deprecated: Use - // StartShardEvacuation/GetShardEvacuationStatus/StopShardEvacuation - EvacuateShard(context.Context, *EvacuateShardRequest) (*EvacuateShardResponse, error) // StartShardEvacuation starts moving all data from one shard to the others. StartShardEvacuation(context.Context, *StartShardEvacuationRequest) (*StartShardEvacuationResponse, error) // GetShardEvacuationStatus returns evacuation status. @@ -400,9 +382,6 @@ func (UnimplementedControlServiceServer) SetShardMode(context.Context, *SetShard func (UnimplementedControlServiceServer) SynchronizeTree(context.Context, *SynchronizeTreeRequest) (*SynchronizeTreeResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method SynchronizeTree not implemented") } -func (UnimplementedControlServiceServer) EvacuateShard(context.Context, *EvacuateShardRequest) (*EvacuateShardResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method EvacuateShard not implemented") -} func (UnimplementedControlServiceServer) StartShardEvacuation(context.Context, *StartShardEvacuationRequest) (*StartShardEvacuationResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method StartShardEvacuation not implemented") } @@ -586,24 +565,6 @@ func _ControlService_SynchronizeTree_Handler(srv interface{}, ctx context.Contex return interceptor(ctx, in, info, handler) } -func _ControlService_EvacuateShard_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(EvacuateShardRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ControlServiceServer).EvacuateShard(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: ControlService_EvacuateShard_FullMethodName, - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ControlServiceServer).EvacuateShard(ctx, req.(*EvacuateShardRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _ControlService_StartShardEvacuation_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(StartShardEvacuationRequest) if err := dec(in); err != nil { @@ -909,10 +870,6 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "SynchronizeTree", Handler: _ControlService_SynchronizeTree_Handler, }, - { - MethodName: "EvacuateShard", - Handler: _ControlService_EvacuateShard_Handler, - }, { MethodName: "StartShardEvacuation", Handler: _ControlService_StartShardEvacuation_Handler, From 41da27dad51b2906b15157c0e3dc4e87c991da02 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Dec 2024 16:53:19 +0300 Subject: [PATCH 1613/1943] [#1549] engine: Drop Async flag from evacuation parameters Now it is only async evacuation. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 30 ++-- .../engine/evacuate_test.go | 130 +++++++++--------- pkg/services/control/server/evacuate_async.go | 1 - 3 files changed, 74 insertions(+), 87 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index b88c249b1..2e0344bfb 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -86,7 +86,6 @@ type EvacuateShardPrm struct { ObjectsHandler func(context.Context, oid.Address, *objectSDK.Object) (bool, error) TreeHandler func(context.Context, cid.ID, string, pilorama.Forest) (bool, string, error) IgnoreErrors bool - Async bool Scope EvacuateScope RepOneOnly bool @@ -211,10 +210,10 @@ var errMustHaveTwoShards = errors.New("must have at least 1 spare shard") // Evacuate moves data from one shard to the others. // The shard being moved must be in read-only mode. -func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*EvacuateShardRes, error) { +func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) error { select { case <-ctx.Done(): - return nil, ctx.Err() + return ctx.Err() default: } @@ -226,7 +225,6 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Evacuate", trace.WithAttributes( attribute.StringSlice("shardIDs", shardIDs), - attribute.Bool("async", prm.Async), attribute.Bool("ignoreErrors", prm.IgnoreErrors), attribute.Stringer("scope", prm.Scope), )) @@ -234,7 +232,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev shards, err := e.getActualShards(shardIDs, prm) if err != nil { - return nil, err + return err } shardsToEvacuate := make(map[string]*shard.Shard) @@ -247,10 +245,10 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev } res := NewEvacuateShardRes() - ctx = ctxOrBackground(ctx, prm.Async) - eg, egCtx, err := e.evacuateLimiter.TryStart(ctx, shardIDs, res) + ctx = context.WithoutCancel(ctx) + eg, ctx, err := e.evacuateLimiter.TryStart(ctx, shardIDs, res) if err != nil { - return nil, err + return err } var mtx sync.RWMutex @@ -262,21 +260,10 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev return t } eg.Go(func() error { - return e.evacuateShards(egCtx, shardIDs, prm, res, copyShards, shardsToEvacuate) + return e.evacuateShards(ctx, shardIDs, prm, res, copyShards, shardsToEvacuate) }) - if prm.Async { - return nil, nil - } - - return res, eg.Wait() -} - -func ctxOrBackground(ctx context.Context, background bool) context.Context { - if background { - return context.Background() - } - return ctx + return nil } func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, prm EvacuateShardPrm, res *EvacuateShardRes, @@ -286,7 +273,6 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShards", trace.WithAttributes( attribute.StringSlice("shardIDs", shardIDs), - attribute.Bool("async", prm.Async), attribute.Bool("ignoreErrors", prm.IgnoreErrors), attribute.Stringer("scope", prm.Scope), attribute.Bool("repOneOnly", prm.RepOneOnly), diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index beab8384e..248c39155 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -140,16 +140,17 @@ func TestEvacuateShardObjects(t *testing.T) { prm.Scope = EvacuateScopeObjects t.Run("must be read-only", func(t *testing.T) { - res, err := e.Evacuate(context.Background(), prm) + err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, ErrMustBeReadOnly) - require.Equal(t, uint64(0), res.ObjectsEvacuated()) }) require.NoError(t, e.shards[evacuateShardID].SetMode(context.Background(), mode.ReadOnly)) - res, err := e.Evacuate(context.Background(), prm) + err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, uint64(objPerShard), res.ObjectsEvacuated()) + st := testWaitForEvacuationCompleted(t, e) + require.Equal(t, st.ErrorMessage(), "") + require.Equal(t, uint64(objPerShard), st.ObjectsEvacuated()) // We check that all objects are available both before and after shard removal. // First case is a real-world use-case. It ensures that an object can be put in presense @@ -186,9 +187,10 @@ func TestEvacuateShardObjects(t *testing.T) { } // Calling it again is OK, but all objects are already moved, so no new PUTs should be done. - res, err = e.Evacuate(context.Background(), prm) - require.NoError(t, err) - require.Equal(t, uint64(0), res.ObjectsEvacuated()) + require.NoError(t, e.Evacuate(context.Background(), prm)) + st = testWaitForEvacuationCompleted(t, e) + require.Equal(t, st.ErrorMessage(), "") + require.Equal(t, uint64(0), st.ObjectsEvacuated()) checkHasObjects(t) @@ -200,6 +202,17 @@ func TestEvacuateShardObjects(t *testing.T) { checkHasObjects(t) } +func testWaitForEvacuationCompleted(t *testing.T, e *StorageEngine) *EvacuationState { + var st *EvacuationState + var err error + require.Eventually(t, func() bool { + st, err = e.GetEvacuationState(context.Background()) + require.NoError(t, err) + return st.ProcessingStatus() == EvacuateProcessStateCompleted + }, 3*time.Second, 10*time.Millisecond) + return st +} + func TestEvacuateObjectsNetwork(t *testing.T) { t.Parallel() @@ -242,15 +255,15 @@ func TestEvacuateObjectsNetwork(t *testing.T) { prm.ShardID = ids[0:1] prm.Scope = EvacuateScopeObjects - res, err := e.Evacuate(context.Background(), prm) + err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errMustHaveTwoShards) - require.Equal(t, uint64(0), res.ObjectsEvacuated()) prm.ObjectsHandler = acceptOneOf(objects, 2) - res, err = e.Evacuate(context.Background(), prm) - require.ErrorIs(t, err, errReplication) - require.Equal(t, uint64(2), res.ObjectsEvacuated()) + require.NoError(t, e.Evacuate(context.Background(), prm)) + st := testWaitForEvacuationCompleted(t, e) + require.Contains(t, st.ErrorMessage(), errReplication.Error()) + require.Equal(t, uint64(2), st.ObjectsEvacuated()) }) t.Run("multiple shards, evacuate one", func(t *testing.T) { t.Parallel() @@ -267,16 +280,18 @@ func TestEvacuateObjectsNetwork(t *testing.T) { prm.ObjectsHandler = acceptOneOf(objects, 2) prm.Scope = EvacuateScopeObjects - res, err := e.Evacuate(context.Background(), prm) - require.ErrorIs(t, err, errReplication) - require.Equal(t, uint64(2), res.ObjectsEvacuated()) + require.NoError(t, e.Evacuate(context.Background(), prm)) + st := testWaitForEvacuationCompleted(t, e) + require.Contains(t, st.ErrorMessage(), errReplication.Error()) + require.Equal(t, uint64(2), st.ObjectsEvacuated()) t.Run("no errors", func(t *testing.T) { prm.ObjectsHandler = acceptOneOf(objects, 3) - res, err := e.Evacuate(context.Background(), prm) - require.NoError(t, err) - require.Equal(t, uint64(3), res.ObjectsEvacuated()) + require.NoError(t, e.Evacuate(context.Background(), prm)) + st := testWaitForEvacuationCompleted(t, e) + require.Equal(t, st.ErrorMessage(), "") + require.Equal(t, uint64(3), st.ObjectsEvacuated()) }) }) t.Run("multiple shards, evacuate many", func(t *testing.T) { @@ -305,16 +320,18 @@ func TestEvacuateObjectsNetwork(t *testing.T) { prm.ObjectsHandler = acceptOneOf(objects, totalCount-1) prm.Scope = EvacuateScopeObjects - res, err := e.Evacuate(context.Background(), prm) - require.ErrorIs(t, err, errReplication) - require.Equal(t, totalCount-1, res.ObjectsEvacuated()) + require.NoError(t, e.Evacuate(context.Background(), prm)) + st := testWaitForEvacuationCompleted(t, e) + require.Contains(t, st.ErrorMessage(), errReplication.Error()) + require.Equal(t, totalCount-1, st.ObjectsEvacuated()) t.Run("no errors", func(t *testing.T) { prm.ObjectsHandler = acceptOneOf(objects, totalCount) - res, err := e.Evacuate(context.Background(), prm) - require.NoError(t, err) - require.Equal(t, totalCount, res.ObjectsEvacuated()) + require.NoError(t, e.Evacuate(context.Background(), prm)) + st := testWaitForEvacuationCompleted(t, e) + require.Equal(t, st.ErrorMessage(), "") + require.Equal(t, totalCount, st.ObjectsEvacuated()) }) }) } @@ -344,9 +361,8 @@ func TestEvacuateCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - res, err := e.Evacuate(ctx, prm) + err := e.Evacuate(ctx, prm) require.ErrorContains(t, err, "context canceled") - require.Equal(t, uint64(0), res.ObjectsEvacuated()) } func TestEvacuateCancellationByError(t *testing.T) { @@ -375,8 +391,9 @@ func TestEvacuateCancellationByError(t *testing.T) { prm.ObjectWorkerCount = 2 prm.ContainerWorkerCount = 2 - _, err := e.Evacuate(context.Background(), prm) - require.ErrorContains(t, err, "test error") + require.NoError(t, e.Evacuate(context.Background(), prm)) + st := testWaitForEvacuationCompleted(t, e) + require.Contains(t, st.ErrorMessage(), "test error") } func TestEvacuateSingleProcess(t *testing.T) { @@ -406,20 +423,19 @@ func TestEvacuateSingleProcess(t *testing.T) { eg, egCtx := errgroup.WithContext(context.Background()) eg.Go(func() error { - res, err := e.Evacuate(egCtx, prm) - require.NoError(t, err, "first evacuation failed") - require.Equal(t, uint64(3), res.ObjectsEvacuated()) + require.NoError(t, e.Evacuate(egCtx, prm), "first evacuation failed") return nil }) eg.Go(func() error { <-running - res, err := e.Evacuate(egCtx, prm) - require.ErrorContains(t, err, "evacuate is already running for shard ids", "second evacuation not failed") - require.Equal(t, uint64(0), res.ObjectsEvacuated()) + require.ErrorContains(t, e.Evacuate(egCtx, prm), "evacuate is already running for shard ids", "second evacuation not failed") close(blocker) return nil }) require.NoError(t, eg.Wait()) + st := testWaitForEvacuationCompleted(t, e) + require.Equal(t, uint64(3), st.ObjectsEvacuated()) + require.Equal(t, st.ErrorMessage(), "") } func TestEvacuateObjectsAsync(t *testing.T) { @@ -458,9 +474,9 @@ func TestEvacuateObjectsAsync(t *testing.T) { eg, egCtx := errgroup.WithContext(context.Background()) eg.Go(func() error { - res, err := e.Evacuate(egCtx, prm) - require.NoError(t, err, "first evacuation failed") - require.Equal(t, uint64(3), res.ObjectsEvacuated()) + require.NoError(t, e.Evacuate(egCtx, prm), "first evacuation failed") + st = testWaitForEvacuationCompleted(t, e) + require.Equal(t, uint64(3), st.ObjectsEvacuated(), "invalid final count") return nil }) @@ -483,12 +499,7 @@ func TestEvacuateObjectsAsync(t *testing.T) { close(blocker) - require.Eventually(t, func() bool { - st, err = e.GetEvacuationState(context.Background()) - return st.ProcessingStatus() == EvacuateProcessStateCompleted - }, 3*time.Second, 10*time.Millisecond, "invalid final state") - - require.NoError(t, err, "get final state failed") + st = testWaitForEvacuationCompleted(t, e) require.Equal(t, uint64(3), st.ObjectsEvacuated(), "invalid final count") require.NotNil(t, st.StartedAt(), "invalid final started at") require.NotNil(t, st.FinishedAt(), "invalid final finished at") @@ -534,14 +545,9 @@ func TestEvacuateTreesLocal(t *testing.T) { require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids") require.Equal(t, "", st.ErrorMessage(), "invalid init error message") - res, err := e.Evacuate(context.Background(), prm) - require.NotNil(t, res, "sync evacuation result must be not nil") - require.NoError(t, err, "evacuation failed") - - st, err = e.GetEvacuationState(context.Background()) - require.NoError(t, err, "get evacuation state failed") - require.Equal(t, EvacuateProcessStateCompleted, st.ProcessingStatus()) + require.NoError(t, e.Evacuate(context.Background(), prm), "evacuation failed") + st = testWaitForEvacuationCompleted(t, e) require.Equal(t, uint64(3), st.TreesTotal(), "invalid trees total count") require.Equal(t, uint64(3), st.TreesEvacuated(), "invalid trees evacuated count") require.Equal(t, uint64(0), st.TreesFailed(), "invalid trees failed count") @@ -632,15 +638,9 @@ func TestEvacuateTreesRemote(t *testing.T) { require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids") require.Equal(t, "", st.ErrorMessage(), "invalid init error message") - res, err := e.Evacuate(context.Background(), prm) - require.NotNil(t, res, "sync evacuation must return not nil") - require.NoError(t, err, "evacuation failed") + require.NoError(t, e.Evacuate(context.Background(), prm), "evacuation failed") + st = testWaitForEvacuationCompleted(t, e) - st, err = e.GetEvacuationState(context.Background()) - require.NoError(t, err, "get evacuation state failed") - require.Equal(t, EvacuateProcessStateCompleted, st.ProcessingStatus()) - - require.NoError(t, err, "get final state failed") require.Equal(t, uint64(6), st.TreesTotal(), "invalid trees total count") require.Equal(t, uint64(6), st.TreesEvacuated(), "invalid trees evacuated count") require.Equal(t, uint64(0), st.TreesFailed(), "invalid trees failed count") @@ -754,11 +754,12 @@ func TestEvacuateShardObjectsRepOneOnly(t *testing.T) { require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) - res, err := e.Evacuate(context.Background(), prm) - require.NoError(t, err) - require.Equal(t, uint64(4), res.ObjectsEvacuated()) - require.Equal(t, uint64(8), res.ObjectsSkipped()) - require.Equal(t, uint64(0), res.ObjectsFailed()) + require.NoError(t, e.Evacuate(context.Background(), prm)) + st := testWaitForEvacuationCompleted(t, e) + require.Equal(t, "", st.ErrorMessage()) + require.Equal(t, uint64(4), st.ObjectsEvacuated()) + require.Equal(t, uint64(8), st.ObjectsSkipped()) + require.Equal(t, uint64(0), st.ObjectsFailed()) } func TestEvacuateShardObjectsRepOneOnlyBench(t *testing.T) { @@ -812,7 +813,8 @@ func TestEvacuateShardObjectsRepOneOnlyBench(t *testing.T) { require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) start := time.Now() - _, err := e.Evacuate(context.Background(), prm) + err := e.Evacuate(context.Background(), prm) + testWaitForEvacuationCompleted(t, e) t.Logf("evacuate took %v\n", time.Since(start)) require.NoError(t, err) } diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index 04465eb26..da5401515 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -42,7 +42,6 @@ func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartSha IgnoreErrors: req.GetBody().GetIgnoreErrors(), ObjectsHandler: s.replicateObject, TreeHandler: s.replicateTree, - Async: true, Scope: engine.EvacuateScope(req.GetBody().GetScope()), ContainerWorkerCount: req.GetBody().GetContainerWorkerCount(), ObjectWorkerCount: req.GetBody().GetObjectWorkerCount(), From 15139d80c9e78ca5a9023eb3f6466d31d40942f3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Dec 2024 12:47:20 +0300 Subject: [PATCH 1614/1943] [#1548] policer: Do not replicate EC chunk if object already removed Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/ec.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index cb583f1d3..db640e323 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -281,6 +281,8 @@ func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info } chunkIDs[ch.Index] = ecInfoChunkID } + } else if client.IsErrObjectAlreadyRemoved(err) { + restore = false } else if !p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) && uint32(idx) < objInfo.ECInfo.Total { p.log.Warn(ctx, logs.PolicerCouldNotGetObjectFromNodeMoving, zap.String("node", hex.EncodeToString(n.PublicKey())), zap.Stringer("object", parentAddress), zap.Error(err)) p.replicator.HandleReplicationTask(ctx, replicator.Task{ From 386a12eea45d5af072dfbba307e5a079ee9ebdb4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Dec 2024 12:52:26 +0300 Subject: [PATCH 1615/1943] [#1548] engine: Rename parent -> ecParent Parent could mean split parent or EC parent. In this case it is EC parent only. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/exists_test.go | 2 +- pkg/local_object_storage/engine/put.go | 10 +++++----- pkg/local_object_storage/metabase/exists.go | 18 +++++++++--------- pkg/local_object_storage/shard/exists.go | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/local_object_storage/engine/exists_test.go b/pkg/local_object_storage/engine/exists_test.go index 1b51c10dc..9b3c0833f 100644 --- a/pkg/local_object_storage/engine/exists_test.go +++ b/pkg/local_object_storage/engine/exists_test.go @@ -42,7 +42,7 @@ func benchmarkExists(b *testing.B, shardNum int) { for range b.N { var shPrm shard.ExistsPrm shPrm.Address = addr - shPrm.ParentAddress = oid.Address{} + shPrm.ECParentAddress = oid.Address{} ok, _, err := e.exists(context.Background(), shPrm) if err != nil || ok { b.Fatalf("%t %v", ok, err) diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index e080191ae..ba4a144d1 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -71,21 +71,21 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { // In #1146 this check was parallelized, however, it became // much slower on fast machines for 4 shards. - var parent oid.Address + var ecParent oid.Address if prm.Object.ECHeader() != nil { - parent.SetObject(prm.Object.ECHeader().Parent()) - parent.SetContainer(addr.Container()) + ecParent.SetObject(prm.Object.ECHeader().Parent()) + ecParent.SetContainer(addr.Container()) } var shPrm shard.ExistsPrm shPrm.Address = addr - shPrm.ParentAddress = parent + shPrm.ECParentAddress = ecParent existed, locked, err := e.exists(ctx, shPrm) if err != nil { return err } if !existed && locked { - lockers, err := e.GetLocked(ctx, parent) + lockers, err := e.GetLocked(ctx, ecParent) if err != nil { return err } diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index 0294dd3ba..f6596e830 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -20,8 +20,8 @@ import ( // ExistsPrm groups the parameters of Exists operation. type ExistsPrm struct { - addr oid.Address - paddr oid.Address + addr oid.Address + ecParentAddr oid.Address } // ExistsRes groups the resulting values of Exists operation. @@ -37,9 +37,9 @@ func (p *ExistsPrm) SetAddress(addr oid.Address) { p.addr = addr } -// SetParent is an Exists option to set objects parent. -func (p *ExistsPrm) SetParent(addr oid.Address) { - p.paddr = addr +// SetECParent is an Exists option to set objects parent. +func (p *ExistsPrm) SetECParent(addr oid.Address) { + p.ecParentAddr = addr } // Exists returns the fact that the object is in the metabase. @@ -82,7 +82,7 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err currEpoch := db.epochState.CurrentEpoch() err = db.boltDB.View(func(tx *bbolt.Tx) error { - res.exists, res.locked, err = db.exists(tx, prm.addr, prm.paddr, currEpoch) + res.exists, res.locked, err = db.exists(tx, prm.addr, prm.ecParentAddr, currEpoch) return err }) @@ -90,10 +90,10 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err return res, metaerr.Wrap(err) } -func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, parent oid.Address, currEpoch uint64) (bool, bool, error) { +func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, ecParent oid.Address, currEpoch uint64) (bool, bool, error) { var locked bool - if !parent.Equals(oid.Address{}) { - locked = objectLocked(tx, parent.Container(), parent.Object()) + if !ecParent.Equals(oid.Address{}) { + locked = objectLocked(tx, ecParent.Container(), ecParent.Object()) } // check graveyard and object expiration first st, err := objectStatus(tx, addr, currEpoch) diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 784bf293a..82ce48dde 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -18,7 +18,7 @@ type ExistsPrm struct { // Exists option to set object checked for existence. Address oid.Address // Exists option to set parent object checked for existence. - ParentAddress oid.Address + ECParentAddress oid.Address } // ExistsRes groups the resulting values of Exists operation. @@ -74,7 +74,7 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { } else { var existsPrm meta.ExistsPrm existsPrm.SetAddress(prm.Address) - existsPrm.SetParent(prm.ParentAddress) + existsPrm.SetECParent(prm.ECParentAddress) var res meta.ExistsRes res, err = s.metaBase.Exists(ctx, existsPrm) From 1f6cf57e308eab9b2659989eb31f1bee1a56ef9b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 10 Dec 2024 12:53:54 +0300 Subject: [PATCH 1616/1943] [#1548] metabase: Check if EC parent is removed or expired Signed-off-by: Dmitrii Stepanov --- .../engine/inhume_test.go | 45 +++++++++++++++++++ pkg/local_object_storage/metabase/exists.go | 11 +++++ pkg/local_object_storage/metabase/put.go | 8 +++- 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 9d7196d94..2d083a58c 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -11,6 +11,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -93,6 +94,50 @@ func TestStorageEngine_Inhume(t *testing.T) { }) } +func TestStorageEngine_ECInhume(t *testing.T) { + parentObjectAddress := oidtest.Address() + containerID := parentObjectAddress.Container() + + chunkObject0 := testutil.GenerateObjectWithCID(containerID) + chunkObject0.SetECHeader(objectSDK.NewECHeader( + objectSDK.ECParentInfo{ + ID: parentObjectAddress.Object(), + }, 0, 4, []byte{}, 0)) + + chunkObject1 := testutil.GenerateObjectWithCID(containerID) + chunkObject1.SetECHeader(objectSDK.NewECHeader( + objectSDK.ECParentInfo{ + ID: parentObjectAddress.Object(), + }, 1, 4, []byte{}, 0)) + + tombstone := objectSDK.NewTombstone() + tombstone.SetMembers([]oid.ID{parentObjectAddress.Object()}) + payload, err := tombstone.Marshal() + require.NoError(t, err) + tombstoneObject := testutil.GenerateObjectWithCID(containerID) + tombstoneObject.SetType(objectSDK.TypeTombstone) + tombstoneObject.SetPayload(payload) + tombstoneObjectAddress := object.AddressOf(tombstoneObject) + + e := testNewEngine(t).setShardsNum(t, 5).prepare(t).engine + defer func() { require.NoError(t, e.Close(context.Background())) }() + + require.NoError(t, Put(context.Background(), e, chunkObject0, false)) + + require.NoError(t, Put(context.Background(), e, tombstoneObject, false)) + + var inhumePrm InhumePrm + inhumePrm.WithTarget(tombstoneObjectAddress, parentObjectAddress) + _, err = e.Inhume(context.Background(), inhumePrm) + require.NoError(t, err) + + var alreadyRemoved *apistatus.ObjectAlreadyRemoved + + require.ErrorAs(t, Put(context.Background(), e, chunkObject0, false), &alreadyRemoved) + + require.ErrorAs(t, Put(context.Background(), e, chunkObject1, false), &alreadyRemoved) +} + func TestInhumeExpiredRegularObject(t *testing.T) { t.Parallel() diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index f6596e830..411beb6b3 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -93,6 +93,17 @@ func (db *DB) Exists(ctx context.Context, prm ExistsPrm) (res ExistsRes, err err func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, ecParent oid.Address, currEpoch uint64) (bool, bool, error) { var locked bool if !ecParent.Equals(oid.Address{}) { + st, err := objectStatus(tx, ecParent, currEpoch) + if err != nil { + return false, false, err + } + switch st { + case 2: + return false, locked, logicerr.Wrap(new(apistatus.ObjectAlreadyRemoved)) + case 3: + return false, locked, ErrObjectIsExpired + } + locked = objectLocked(tx, ecParent.Container(), ecParent.Object()) } // check graveyard and object expiration first diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index d7675869f..6f9dc1bf0 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -121,9 +121,15 @@ func (db *DB) put(tx *bbolt.Tx, return PutRes{}, errors.New("missing container in object") } + var ecParentAddress oid.Address + if ecHeader := obj.ECHeader(); ecHeader != nil { + ecParentAddress.SetContainer(cnr) + ecParentAddress.SetObject(ecHeader.Parent()) + } + isParent := si != nil - exists, _, err := db.exists(tx, objectCore.AddressOf(obj), oid.Address{}, currEpoch) + exists, _, err := db.exists(tx, objectCore.AddressOf(obj), ecParentAddress, currEpoch) var splitInfoError *objectSDK.SplitInfoError if errors.As(err, &splitInfoError) { From 670305a7216bf3a5304b6256eed8dc1efe285f9f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 6 Dec 2024 15:50:57 +0300 Subject: [PATCH 1617/1943] [#1546] morph/event: Remove nil checks from event handler registrar This codepath hides possible bugs in code. We would rather panic then silently fail. Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 5 ----- pkg/morph/event/listener.go | 29 ----------------------------- 2 files changed, 34 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index d0bac4d11..893be7006 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -164,17 +164,12 @@ const ( EventNotaryParserNotSet = "notary parser not set" EventCouldNotParseNotaryEvent = "could not parse notary event" EventNotaryHandlersForParsedNotificationEventWereNotRegistered = "notary handlers for parsed notification event were not registered" - EventIgnoreNilEventParser = "ignore nil event parser" EventListenerHasBeenAlreadyStartedIgnoreParser = "listener has been already started, ignore parser" EventRegisteredNewEventParser = "registered new event parser" - EventIgnoreNilEventHandler = "ignore nil event handler" EventIgnoreHandlerOfEventWoParser = "ignore handler of event w/o parser" EventRegisteredNewEventHandler = "registered new event handler" - EventIgnoreNilNotaryEventParser = "ignore nil notary event parser" EventListenerHasBeenAlreadyStartedIgnoreNotaryParser = "listener has been already started, ignore notary parser" - EventIgnoreNilNotaryEventHandler = "ignore nil notary event handler" EventIgnoreHandlerOfNotaryEventWoParser = "ignore handler of notary event w/o parser" - EventIgnoreNilBlockHandler = "ignore nil block handler" StorageOperation = "local object storage operation" BlobovniczaCreatingDirectoryForBoltDB = "creating directory for BoltDB" BlobovniczaOpeningBoltDB = "opening BoltDB" diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 6e6184e77..b67546804 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -459,12 +459,6 @@ func (l *listener) SetNotificationParser(pi NotificationParserInfo) { zap.Stringer("event_type", pi.getType()), ) - parser := pi.parser() - if parser == nil { - log.Info(context.Background(), logs.EventIgnoreNilEventParser) - return - } - l.mtx.Lock() defer l.mtx.Unlock() @@ -492,12 +486,6 @@ func (l *listener) RegisterNotificationHandler(hi NotificationHandlerInfo) { zap.Stringer("event_type", hi.GetType()), ) - handler := hi.Handler() - if handler == nil { - log.Warn(context.Background(), logs.EventIgnoreNilEventHandler) - return - } - // check if parser was set l.mtx.RLock() _, ok := l.notificationParsers[hi.scriptHashWithType] @@ -555,12 +543,6 @@ func (l *listener) SetNotaryParser(pi NotaryParserInfo) { zap.Stringer("notary_type", pi.RequestType()), ) - parser := pi.parser() - if parser == nil { - log.Info(context.Background(), logs.EventIgnoreNilNotaryEventParser) - return - } - l.mtx.Lock() defer l.mtx.Unlock() @@ -593,12 +575,6 @@ func (l *listener) RegisterNotaryHandler(hi NotaryHandlerInfo) { zap.Stringer("notary type", hi.RequestType()), ) - handler := hi.Handler() - if handler == nil { - log.Warn(context.Background(), logs.EventIgnoreNilNotaryEventHandler) - return - } - // check if parser was set l.mtx.RLock() _, ok := l.notaryParsers[hi.notaryRequestTypes] @@ -627,11 +603,6 @@ func (l *listener) Stop() { } func (l *listener) RegisterBlockHandler(handler BlockHandler) { - if handler == nil { - l.log.Warn(context.Background(), logs.EventIgnoreNilBlockHandler) - return - } - l.blockHandlers = append(l.blockHandlers, handler) } From dfa51048a852aa9a8c88a67b6914d671137dc3f7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 6 Dec 2024 15:56:41 +0300 Subject: [PATCH 1618/1943] [#1546] morph/event: Remove "is started" checks from event handler registrar This codepath hides possible bugs in code. All initialization function should run before init stage. Signed-off-by: Evgenii Stratonikov --- internal/logs/logs.go | 2 -- pkg/morph/event/listener.go | 17 ----------------- 2 files changed, 19 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 893be7006..535802a20 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -164,11 +164,9 @@ const ( EventNotaryParserNotSet = "notary parser not set" EventCouldNotParseNotaryEvent = "could not parse notary event" EventNotaryHandlersForParsedNotificationEventWereNotRegistered = "notary handlers for parsed notification event were not registered" - EventListenerHasBeenAlreadyStartedIgnoreParser = "listener has been already started, ignore parser" EventRegisteredNewEventParser = "registered new event parser" EventIgnoreHandlerOfEventWoParser = "ignore handler of event w/o parser" EventRegisteredNewEventHandler = "registered new event handler" - EventListenerHasBeenAlreadyStartedIgnoreNotaryParser = "listener has been already started, ignore notary parser" EventIgnoreHandlerOfNotaryEventWoParser = "ignore handler of notary event w/o parser" StorageOperation = "local object storage operation" BlobovniczaCreatingDirectoryForBoltDB = "creating directory for BoltDB" diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index b67546804..64ff205be 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -100,8 +100,6 @@ type listener struct { startOnce, stopOnce sync.Once - started bool - notificationParsers map[scriptHashWithType]NotificationParser notificationHandlers map[scriptHashWithType][]Handler @@ -171,9 +169,6 @@ func (l *listener) ListenWithError(ctx context.Context, intError chan<- error) { } func (l *listener) listen(ctx context.Context, intError chan<- error) error { - // mark listener as started - l.started = true - subErrCh := make(chan error) go l.subscribe(subErrCh) @@ -462,12 +457,6 @@ func (l *listener) SetNotificationParser(pi NotificationParserInfo) { l.mtx.Lock() defer l.mtx.Unlock() - // check if the listener was started - if l.started { - log.Warn(context.Background(), logs.EventListenerHasBeenAlreadyStartedIgnoreParser) - return - } - // add event parser if _, ok := l.notificationParsers[pi.scriptHashWithType]; !ok { l.notificationParsers[pi.scriptHashWithType] = pi.parser() @@ -546,12 +535,6 @@ func (l *listener) SetNotaryParser(pi NotaryParserInfo) { l.mtx.Lock() defer l.mtx.Unlock() - // check if the listener was started - if l.started { - log.Warn(context.Background(), logs.EventListenerHasBeenAlreadyStartedIgnoreNotaryParser) - return - } - // add event parser if _, ok := l.notaryParsers[pi.notaryRequestTypes]; !ok { l.notaryParsers[pi.notaryRequestTypes] = pi.parser() From d0ce835fbf75658271613a40741443405b86f7eb Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 6 Dec 2024 16:01:16 +0300 Subject: [PATCH 1619/1943] [#1546] morph/event: Merge notification parser and handlers They are decoupled, but it is an error to have a handler without a corresponding parser. Register them together on the code level and get rid of unreachable code. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/morph.go | 18 +++----- internal/logs/logs.go | 1 - pkg/innerring/bindings.go | 6 --- .../processors/alphabet/processor.go | 5 --- pkg/innerring/processors/balance/processor.go | 17 +------ .../processors/container/processor.go | 5 --- pkg/innerring/processors/frostfs/processor.go | 45 ++++--------------- .../processors/governance/processor.go | 12 +---- pkg/innerring/processors/netmap/processor.go | 19 +------- pkg/morph/event/handlers.go | 18 +++++--- pkg/morph/event/listener.go | 44 ++---------------- pkg/morph/event/listener_test.go | 18 ++++---- pkg/morph/event/parsers.go | 27 ----------- 13 files changed, 44 insertions(+), 191 deletions(-) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 81579c7fc..e80dda80e 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -223,27 +223,21 @@ func registerNotificationHandlers(scHash util.Uint160, lis event.Listener, parse subs map[event.Type][]event.Handler, ) { for typ, handlers := range subs { - pi := event.NotificationParserInfo{} - pi.SetType(typ) - pi.SetScriptHash(scHash) + hi := event.NotificationHandlerInfo{} + hi.SetType(typ) + hi.SetScriptHash(scHash) p, ok := parsers[typ] if !ok { panic(fmt.Sprintf("missing parser for event %s", typ)) } - pi.SetParser(p) - - lis.SetNotificationParser(pi) + hi.SetParser(p) for _, h := range handlers { - hi := event.NotificationHandlerInfo{} - hi.SetType(typ) - hi.SetScriptHash(scHash) - hi.SetHandler(h) - - lis.RegisterNotificationHandler(hi) + hi.AddHandler(h) } + lis.RegisterNotificationHandler(hi) } } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 535802a20..b24f3593d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -165,7 +165,6 @@ const ( EventCouldNotParseNotaryEvent = "could not parse notary event" EventNotaryHandlersForParsedNotificationEventWereNotRegistered = "notary handlers for parsed notification event were not registered" EventRegisteredNewEventParser = "registered new event parser" - EventIgnoreHandlerOfEventWoParser = "ignore handler of event w/o parser" EventRegisteredNewEventHandler = "registered new event handler" EventIgnoreHandlerOfNotaryEventWoParser = "ignore handler of notary event w/o parser" StorageOperation = "local object storage operation" diff --git a/pkg/innerring/bindings.go b/pkg/innerring/bindings.go index c4de07a5f..dfada764a 100644 --- a/pkg/innerring/bindings.go +++ b/pkg/innerring/bindings.go @@ -8,7 +8,6 @@ type ( // ContractProcessor interface defines functions for binding event producers // such as event.Listener and Timers with contract processor. ContractProcessor interface { - ListenerNotificationParsers() []event.NotificationParserInfo ListenerNotificationHandlers() []event.NotificationHandlerInfo ListenerNotaryParsers() []event.NotaryParserInfo ListenerNotaryHandlers() []event.NotaryHandlerInfo @@ -16,11 +15,6 @@ type ( ) func connectListenerWithProcessor(l event.Listener, p ContractProcessor) { - // register notification parsers - for _, parser := range p.ListenerNotificationParsers() { - l.SetNotificationParser(parser) - } - // register notification handlers for _, handler := range p.ListenerNotificationHandlers() { l.RegisterNotificationHandler(handler) diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index 3992e00f3..2c4654e7c 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -114,11 +114,6 @@ func (ap *Processor) SetParsedWallets(parsedWallets []util.Uint160) { ap.pwLock.Unlock() } -// ListenerNotificationParsers for the 'event.Listener' event producer. -func (ap *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { - return nil -} - // ListenerNotificationHandlers for the 'event.Listener' event producer. func (ap *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { return nil diff --git a/pkg/innerring/processors/balance/processor.go b/pkg/innerring/processors/balance/processor.go index e2f649600..d323238c7 100644 --- a/pkg/innerring/processors/balance/processor.go +++ b/pkg/innerring/processors/balance/processor.go @@ -88,20 +88,6 @@ func New(p *Params) (*Processor, error) { }, nil } -// ListenerNotificationParsers for the 'event.Listener' event producer. -func (bp *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { - var parsers []event.NotificationParserInfo - - // new lock event - lock := event.NotificationParserInfo{} - lock.SetType(lockNotification) - lock.SetScriptHash(bp.balanceSC) - lock.SetParser(balanceEvent.ParseLock) - parsers = append(parsers, lock) - - return parsers -} - // ListenerNotificationHandlers for the 'event.Listener' event producer. func (bp *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { var handlers []event.NotificationHandlerInfo @@ -110,7 +96,8 @@ func (bp *Processor) ListenerNotificationHandlers() []event.NotificationHandlerI lock := event.NotificationHandlerInfo{} lock.SetType(lockNotification) lock.SetScriptHash(bp.balanceSC) - lock.SetHandler(bp.handleLock) + lock.SetParser(balanceEvent.ParseLock) + lock.AddHandler(bp.handleLock) handlers = append(handlers, lock) return handlers diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index 58b90457c..a0b7491e1 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -118,11 +118,6 @@ func New(p *Params) (*Processor, error) { }, nil } -// ListenerNotificationParsers for the 'event.Listener' event producer. -func (cp *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { - return nil -} - // ListenerNotificationHandlers for the 'event.Listener' event producer. func (cp *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { return nil diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index 6c29d330d..64b171f7f 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -142,39 +142,6 @@ func New(p *Params) (*Processor, error) { }, nil } -// ListenerNotificationParsers for the 'event.Listener' event producer. -func (np *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { - var ( - parsers = make([]event.NotificationParserInfo, 0, 6) - - p event.NotificationParserInfo - ) - - p.SetScriptHash(np.frostfsContract) - - // deposit event - p.SetType(event.TypeFromString(depositNotification)) - p.SetParser(frostfsEvent.ParseDeposit) - parsers = append(parsers, p) - - // withdraw event - p.SetType(event.TypeFromString(withdrawNotification)) - p.SetParser(frostfsEvent.ParseWithdraw) - parsers = append(parsers, p) - - // cheque event - p.SetType(event.TypeFromString(chequeNotification)) - p.SetParser(frostfsEvent.ParseCheque) - parsers = append(parsers, p) - - // config event - p.SetType(event.TypeFromString(configNotification)) - p.SetParser(frostfsEvent.ParseConfig) - parsers = append(parsers, p) - - return parsers -} - // ListenerNotificationHandlers for the 'event.Listener' event producer. func (np *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { var ( @@ -187,22 +154,26 @@ func (np *Processor) ListenerNotificationHandlers() []event.NotificationHandlerI // deposit handler h.SetType(event.TypeFromString(depositNotification)) - h.SetHandler(np.handleDeposit) + h.SetParser(frostfsEvent.ParseDeposit) + h.AddHandler(np.handleDeposit) handlers = append(handlers, h) // withdraw handler h.SetType(event.TypeFromString(withdrawNotification)) - h.SetHandler(np.handleWithdraw) + h.SetParser(frostfsEvent.ParseWithdraw) + h.AddHandler(np.handleWithdraw) handlers = append(handlers, h) // cheque handler h.SetType(event.TypeFromString(chequeNotification)) - h.SetHandler(np.handleCheque) + h.SetParser(frostfsEvent.ParseCheque) + h.AddHandler(np.handleCheque) handlers = append(handlers, h) // config handler h.SetType(event.TypeFromString(configNotification)) - h.SetHandler(np.handleConfig) + h.SetParser(frostfsEvent.ParseConfig) + h.AddHandler(np.handleConfig) handlers = append(handlers, h) return handlers diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index 565f4c27d..313a0baea 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -155,21 +155,13 @@ func New(p *Params) (*Processor, error) { }, nil } -// ListenerNotificationParsers for the 'event.Listener' event producer. -func (gp *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { - var pi event.NotificationParserInfo - pi.SetScriptHash(gp.designate) - pi.SetType(event.TypeFromString(native.DesignationEventName)) - pi.SetParser(rolemanagement.ParseDesignate) - return []event.NotificationParserInfo{pi} -} - // ListenerNotificationHandlers for the 'event.Listener' event producer. func (gp *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { var hi event.NotificationHandlerInfo hi.SetScriptHash(gp.designate) hi.SetType(event.TypeFromString(native.DesignationEventName)) - hi.SetHandler(gp.HandleAlphabetSync) + hi.SetParser(rolemanagement.ParseDesignate) + hi.AddHandler(gp.HandleAlphabetSync) return []event.NotificationHandlerInfo{hi} } diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index b3d57e85b..c726df955 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -161,22 +161,6 @@ func New(p *Params) (*Processor, error) { }, nil } -// ListenerNotificationParsers for the 'event.Listener' event producer. -func (np *Processor) ListenerNotificationParsers() []event.NotificationParserInfo { - parsers := make([]event.NotificationParserInfo, 0, 3) - - var p event.NotificationParserInfo - - p.SetScriptHash(np.netmapClient.ContractAddress()) - - // new epoch event - p.SetType(newEpochNotification) - p.SetParser(netmapEvent.ParseNewEpoch) - parsers = append(parsers, p) - - return parsers -} - // ListenerNotificationHandlers for the 'event.Listener' event producer. func (np *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { handlers := make([]event.NotificationHandlerInfo, 0, 3) @@ -187,7 +171,8 @@ func (np *Processor) ListenerNotificationHandlers() []event.NotificationHandlerI // new epoch handler i.SetType(newEpochNotification) - i.SetHandler(np.handleNewEpoch) + i.SetParser(netmapEvent.ParseNewEpoch) + i.AddHandler(np.handleNewEpoch) handlers = append(handlers, i) return handlers diff --git a/pkg/morph/event/handlers.go b/pkg/morph/event/handlers.go index 822335329..e96abb846 100644 --- a/pkg/morph/event/handlers.go +++ b/pkg/morph/event/handlers.go @@ -18,17 +18,23 @@ type BlockHandler func(context.Context, *block.Block) type NotificationHandlerInfo struct { scriptHashWithType - h Handler + parser NotificationParser + handlers []Handler } -// SetHandler is an event handler setter. -func (s *NotificationHandlerInfo) SetHandler(v Handler) { - s.h = v +// SetParser is an event handler setter. +func (s *NotificationHandlerInfo) SetParser(p NotificationParser) { + s.parser = p +} + +// AddHandler adds an event handler. +func (s *NotificationHandlerInfo) AddHandler(v Handler) { + s.handlers = append(s.handlers, v) } // Handler returns an event handler. -func (s NotificationHandlerInfo) Handler() Handler { - return s.h +func (s NotificationHandlerInfo) Handlers() []Handler { + return s.handlers } // NotaryHandlerInfo is a structure that groups diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 64ff205be..8fe95cf49 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -33,13 +33,6 @@ type Listener interface { // it could not be started. ListenWithError(context.Context, chan<- error) - // SetNotificationParser must set the parser of particular contract event. - // - // Parser of each event must be set once. All parsers must be set before Listen call. - // - // Must ignore nil parsers and all calls after listener has been started. - SetNotificationParser(NotificationParserInfo) - // RegisterNotificationHandler must register the event handler for particular notification event of contract. // // The specified handler must be called after each capture and parsing of the event. @@ -444,27 +437,6 @@ func (l *listener) parseAndHandleNotary(ctx context.Context, nr *result.NotaryRe handler(ctx, event) } -// SetNotificationParser sets the parser of particular contract event. -// -// Ignores nil and already set parsers. -// Ignores the parser if listener is started. -func (l *listener) SetNotificationParser(pi NotificationParserInfo) { - log := l.log.With( - zap.String("contract", pi.ScriptHash().StringLE()), - zap.Stringer("event_type", pi.getType()), - ) - - l.mtx.Lock() - defer l.mtx.Unlock() - - // add event parser - if _, ok := l.notificationParsers[pi.scriptHashWithType]; !ok { - l.notificationParsers[pi.scriptHashWithType] = pi.parser() - } - - log.Debug(context.Background(), logs.EventRegisteredNewEventParser) -} - // RegisterNotificationHandler registers the handler for particular notification event of contract. // // Ignores nil handlers. @@ -476,22 +448,14 @@ func (l *listener) RegisterNotificationHandler(hi NotificationHandlerInfo) { ) // check if parser was set - l.mtx.RLock() - _, ok := l.notificationParsers[hi.scriptHashWithType] - l.mtx.RUnlock() - - if !ok { - log.Warn(context.Background(), logs.EventIgnoreHandlerOfEventWoParser) - return - } - - // add event handler l.mtx.Lock() + defer l.mtx.Unlock() + + l.notificationParsers[hi.scriptHashWithType] = hi.parser l.notificationHandlers[hi.scriptHashWithType] = append( l.notificationHandlers[hi.scriptHashWithType], - hi.Handler(), + hi.handlers..., ) - l.mtx.Unlock() log.Debug(context.Background(), logs.EventRegisteredNewEventHandler) } diff --git a/pkg/morph/event/listener_test.go b/pkg/morph/event/listener_test.go index c0f9722d7..02dad2fd3 100644 --- a/pkg/morph/event/listener_test.go +++ b/pkg/morph/event/listener_test.go @@ -48,20 +48,18 @@ func TestEventHandling(t *testing.T) { }, } - l.SetNotificationParser(NotificationParserInfo{ - scriptHashWithType: key, - p: func(cne *state.ContainedNotificationEvent) (Event, error) { - return testNotificationEvent{source: cne}, nil - }, - }) - notificationHandled := make(chan bool) handledNotifications := make([]Event, 0) l.RegisterNotificationHandler(NotificationHandlerInfo{ scriptHashWithType: key, - h: func(_ context.Context, e Event) { - handledNotifications = append(handledNotifications, e) - notificationHandled <- true + parser: func(cne *state.ContainedNotificationEvent) (Event, error) { + return testNotificationEvent{source: cne}, nil + }, + handlers: []Handler{ + func(_ context.Context, e Event) { + handledNotifications = append(handledNotifications, e) + notificationHandled <- true + }, }, }) diff --git a/pkg/morph/event/parsers.go b/pkg/morph/event/parsers.go index 90eff0bd2..5adeb4b30 100644 --- a/pkg/morph/event/parsers.go +++ b/pkg/morph/event/parsers.go @@ -11,15 +11,6 @@ import ( // from the StackItem list. type NotificationParser func(*state.ContainedNotificationEvent) (Event, error) -// NotificationParserInfo is a structure that groups -// the parameters of particular contract -// notification event parser. -type NotificationParserInfo struct { - scriptHashWithType - - p NotificationParser -} - // NotaryPreparator constructs NotaryEvent // from the NotaryRequest event. type NotaryPreparator interface { @@ -47,24 +38,6 @@ func (n *NotaryParserInfo) SetParser(p NotaryParser) { n.p = p } -// SetParser is an event parser setter. -func (s *NotificationParserInfo) SetParser(v NotificationParser) { - s.p = v -} - -func (s NotificationParserInfo) parser() NotificationParser { - return s.p -} - -// SetType is an event type setter. -func (s *NotificationParserInfo) SetType(v Type) { - s.typ = v -} - -func (s NotificationParserInfo) getType() Type { - return s.typ -} - type wrongPrmNumber struct { exp, act int } From b1614a284d42521dfb1d5b7b46dd7e45acad4f7b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 6 Dec 2024 16:12:47 +0300 Subject: [PATCH 1620/1943] [#1546] morph/event: Export NotificationHandlerInfo fields Hiding them achieves nothing, as the struct has no methods and is not used concurrently. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/morph.go | 16 ++--- pkg/innerring/processors/balance/processor.go | 19 +++--- pkg/innerring/processors/frostfs/processor.go | 59 ++++++++----------- .../processors/governance/processor.go | 14 +++-- pkg/innerring/processors/netmap/processor.go | 21 +++---- pkg/morph/event/handlers.go | 24 ++------ pkg/morph/event/listener.go | 16 +++-- pkg/morph/event/listener_test.go | 16 ++--- 8 files changed, 75 insertions(+), 110 deletions(-) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index e80dda80e..0d4e8a1be 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -223,21 +223,17 @@ func registerNotificationHandlers(scHash util.Uint160, lis event.Listener, parse subs map[event.Type][]event.Handler, ) { for typ, handlers := range subs { - hi := event.NotificationHandlerInfo{} - hi.SetType(typ) - hi.SetScriptHash(scHash) - p, ok := parsers[typ] if !ok { panic(fmt.Sprintf("missing parser for event %s", typ)) } - hi.SetParser(p) - - for _, h := range handlers { - hi.AddHandler(h) - } - lis.RegisterNotificationHandler(hi) + lis.RegisterNotificationHandler(event.NotificationHandlerInfo{ + Contract: scHash, + Type: typ, + Parser: p, + Handlers: handlers, + }) } } diff --git a/pkg/innerring/processors/balance/processor.go b/pkg/innerring/processors/balance/processor.go index d323238c7..34203b74f 100644 --- a/pkg/innerring/processors/balance/processor.go +++ b/pkg/innerring/processors/balance/processor.go @@ -90,17 +90,14 @@ func New(p *Params) (*Processor, error) { // ListenerNotificationHandlers for the 'event.Listener' event producer. func (bp *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { - var handlers []event.NotificationHandlerInfo - - // lock handler - lock := event.NotificationHandlerInfo{} - lock.SetType(lockNotification) - lock.SetScriptHash(bp.balanceSC) - lock.SetParser(balanceEvent.ParseLock) - lock.AddHandler(bp.handleLock) - handlers = append(handlers, lock) - - return handlers + return []event.NotificationHandlerInfo{ + { + Contract: bp.balanceSC, + Type: lockNotification, + Parser: balanceEvent.ParseLock, + Handlers: []event.Handler{bp.handleLock}, + }, + } } // ListenerNotaryParsers for the 'event.Listener' event producer. diff --git a/pkg/innerring/processors/frostfs/processor.go b/pkg/innerring/processors/frostfs/processor.go index 64b171f7f..9d3bf65cd 100644 --- a/pkg/innerring/processors/frostfs/processor.go +++ b/pkg/innerring/processors/frostfs/processor.go @@ -144,39 +144,32 @@ func New(p *Params) (*Processor, error) { // ListenerNotificationHandlers for the 'event.Listener' event producer. func (np *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { - var ( - handlers = make([]event.NotificationHandlerInfo, 0, 6) - - h event.NotificationHandlerInfo - ) - - h.SetScriptHash(np.frostfsContract) - - // deposit handler - h.SetType(event.TypeFromString(depositNotification)) - h.SetParser(frostfsEvent.ParseDeposit) - h.AddHandler(np.handleDeposit) - handlers = append(handlers, h) - - // withdraw handler - h.SetType(event.TypeFromString(withdrawNotification)) - h.SetParser(frostfsEvent.ParseWithdraw) - h.AddHandler(np.handleWithdraw) - handlers = append(handlers, h) - - // cheque handler - h.SetType(event.TypeFromString(chequeNotification)) - h.SetParser(frostfsEvent.ParseCheque) - h.AddHandler(np.handleCheque) - handlers = append(handlers, h) - - // config handler - h.SetType(event.TypeFromString(configNotification)) - h.SetParser(frostfsEvent.ParseConfig) - h.AddHandler(np.handleConfig) - handlers = append(handlers, h) - - return handlers + return []event.NotificationHandlerInfo{ + { + Contract: np.frostfsContract, + Type: event.TypeFromString(depositNotification), + Parser: frostfsEvent.ParseDeposit, + Handlers: []event.Handler{np.handleDeposit}, + }, + { + Contract: np.frostfsContract, + Type: event.TypeFromString(withdrawNotification), + Parser: frostfsEvent.ParseWithdraw, + Handlers: []event.Handler{np.handleWithdraw}, + }, + { + Contract: np.frostfsContract, + Type: event.TypeFromString(chequeNotification), + Parser: frostfsEvent.ParseCheque, + Handlers: []event.Handler{np.handleCheque}, + }, + { + Contract: np.frostfsContract, + Type: event.TypeFromString(configNotification), + Parser: frostfsEvent.ParseConfig, + Handlers: []event.Handler{np.handleConfig}, + }, + } } // ListenerNotaryParsers for the 'event.Listener' event producer. diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index 313a0baea..7859ebee1 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -157,12 +157,14 @@ func New(p *Params) (*Processor, error) { // ListenerNotificationHandlers for the 'event.Listener' event producer. func (gp *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { - var hi event.NotificationHandlerInfo - hi.SetScriptHash(gp.designate) - hi.SetType(event.TypeFromString(native.DesignationEventName)) - hi.SetParser(rolemanagement.ParseDesignate) - hi.AddHandler(gp.HandleAlphabetSync) - return []event.NotificationHandlerInfo{hi} + return []event.NotificationHandlerInfo{ + { + Contract: gp.designate, + Type: event.TypeFromString(native.DesignationEventName), + Parser: rolemanagement.ParseDesignate, + Handlers: []event.Handler{gp.HandleAlphabetSync}, + }, + } } // ListenerNotaryParsers for the 'event.Listener' event producer. diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index c726df955..36df57afe 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -163,19 +163,14 @@ func New(p *Params) (*Processor, error) { // ListenerNotificationHandlers for the 'event.Listener' event producer. func (np *Processor) ListenerNotificationHandlers() []event.NotificationHandlerInfo { - handlers := make([]event.NotificationHandlerInfo, 0, 3) - - var i event.NotificationHandlerInfo - - i.SetScriptHash(np.netmapClient.ContractAddress()) - - // new epoch handler - i.SetType(newEpochNotification) - i.SetParser(netmapEvent.ParseNewEpoch) - i.AddHandler(np.handleNewEpoch) - handlers = append(handlers, i) - - return handlers + return []event.NotificationHandlerInfo{ + { + Contract: np.netmapClient.ContractAddress(), + Type: newEpochNotification, + Parser: netmapEvent.ParseNewEpoch, + Handlers: []event.Handler{np.handleNewEpoch}, + }, + } } // ListenerNotaryParsers for the 'event.Listener' event producer. diff --git a/pkg/morph/event/handlers.go b/pkg/morph/event/handlers.go index e96abb846..55a514ff1 100644 --- a/pkg/morph/event/handlers.go +++ b/pkg/morph/event/handlers.go @@ -4,6 +4,7 @@ import ( "context" "github.com/nspcc-dev/neo-go/pkg/core/block" + "github.com/nspcc-dev/neo-go/pkg/util" ) // Handler is an Event processing function. @@ -16,25 +17,10 @@ type BlockHandler func(context.Context, *block.Block) // the parameters of the handler of particular // contract event. type NotificationHandlerInfo struct { - scriptHashWithType - - parser NotificationParser - handlers []Handler -} - -// SetParser is an event handler setter. -func (s *NotificationHandlerInfo) SetParser(p NotificationParser) { - s.parser = p -} - -// AddHandler adds an event handler. -func (s *NotificationHandlerInfo) AddHandler(v Handler) { - s.handlers = append(s.handlers, v) -} - -// Handler returns an event handler. -func (s NotificationHandlerInfo) Handlers() []Handler { - return s.handlers + Contract util.Uint160 + Type Type + Parser NotificationParser + Handlers []Handler } // NotaryHandlerInfo is a structure that groups diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 8fe95cf49..9852a3d62 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -443,18 +443,22 @@ func (l *listener) parseAndHandleNotary(ctx context.Context, nr *result.NotaryRe // Ignores handlers of event without parser. func (l *listener) RegisterNotificationHandler(hi NotificationHandlerInfo) { log := l.log.With( - zap.String("contract", hi.ScriptHash().StringLE()), - zap.Stringer("event_type", hi.GetType()), + zap.String("contract", hi.Contract.StringLE()), + zap.Stringer("event_type", hi.Type), ) // check if parser was set l.mtx.Lock() defer l.mtx.Unlock() - l.notificationParsers[hi.scriptHashWithType] = hi.parser - l.notificationHandlers[hi.scriptHashWithType] = append( - l.notificationHandlers[hi.scriptHashWithType], - hi.handlers..., + var k scriptHashWithType + k.hash = hi.Contract + k.typ = hi.Type + + l.notificationParsers[k] = hi.Parser + l.notificationHandlers[k] = append( + l.notificationHandlers[k], + hi.Handlers..., ) log.Debug(context.Background(), logs.EventRegisteredNewEventHandler) diff --git a/pkg/morph/event/listener_test.go b/pkg/morph/event/listener_test.go index 02dad2fd3..87f37305f 100644 --- a/pkg/morph/event/listener_test.go +++ b/pkg/morph/event/listener_test.go @@ -39,23 +39,15 @@ func TestEventHandling(t *testing.T) { blockHandled <- true }) - key := scriptHashWithType{ - scriptHashValue: scriptHashValue{ - hash: util.Uint160{100}, - }, - typeValue: typeValue{ - typ: TypeFromString("notification type"), - }, - } - notificationHandled := make(chan bool) handledNotifications := make([]Event, 0) l.RegisterNotificationHandler(NotificationHandlerInfo{ - scriptHashWithType: key, - parser: func(cne *state.ContainedNotificationEvent) (Event, error) { + Contract: util.Uint160{100}, + Type: TypeFromString("notification type"), + Parser: func(cne *state.ContainedNotificationEvent) (Event, error) { return testNotificationEvent{source: cne}, nil }, - handlers: []Handler{ + Handlers: []Handler{ func(_ context.Context, e Event) { handledNotifications = append(handledNotifications, e) notificationHandled <- true From a641c91594dcf48c567ae312a7b7db8ccabf121c Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 10 Dec 2024 18:04:22 +0300 Subject: [PATCH 1621/1943] [#1550] Add CODEOWNERS Signed-off-by: Vitaliy Potyarkin --- CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 000000000..d19c96a5c --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,3 @@ +.* @TrueCloudLab/storage-core-committers @TrueCloudLab/storage-core-developers +.forgejo/.* @potyarkin +Makefile @potyarkin From e9837bbcf9b02ed9218f7ebdb088f0099fd04b86 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 11 Dec 2024 14:39:53 +0300 Subject: [PATCH 1622/1943] [#1554] morph/event: Remove unused AlphabetUpdate event Refs TrueCloudLab/frostfs-contract#138. Signed-off-by: Evgenii Stratonikov --- pkg/morph/event/frostfs/ir_update.go | 54 --------------------- pkg/morph/event/frostfs/ir_update_test.go | 57 ----------------------- 2 files changed, 111 deletions(-) delete mode 100644 pkg/morph/event/frostfs/ir_update.go delete mode 100644 pkg/morph/event/frostfs/ir_update_test.go diff --git a/pkg/morph/event/frostfs/ir_update.go b/pkg/morph/event/frostfs/ir_update.go deleted file mode 100644 index 62203540f..000000000 --- a/pkg/morph/event/frostfs/ir_update.go +++ /dev/null @@ -1,54 +0,0 @@ -package frostfs - -import ( - "crypto/elliptic" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" -) - -type UpdateInnerRing struct { - keys []*keys.PublicKey -} - -// MorphEvent implements Neo:Morph Event interface. -func (UpdateInnerRing) MorphEvent() {} - -func (u UpdateInnerRing) Keys() []*keys.PublicKey { return u.keys } - -func ParseUpdateInnerRing(params []stackitem.Item) (event.Event, error) { - var ( - ev UpdateInnerRing - err error - ) - - if ln := len(params); ln != 1 { - return nil, event.WrongNumberOfParameters(1, ln) - } - - // parse keys - irKeys, err := client.ArrayFromStackItem(params[0]) - if err != nil { - return nil, fmt.Errorf("could not get updated inner ring keys: %w", err) - } - - ev.keys = make([]*keys.PublicKey, 0, len(irKeys)) - for i := range irKeys { - rawKey, err := client.BytesFromStackItem(irKeys[i]) - if err != nil { - return nil, fmt.Errorf("could not get updated inner ring public key: %w", err) - } - - key, err := keys.NewPublicKeyFromBytes(rawKey, elliptic.P256()) - if err != nil { - return nil, fmt.Errorf("could not parse updated inner ring public key: %w", err) - } - - ev.keys = append(ev.keys, key) - } - - return ev, nil -} diff --git a/pkg/morph/event/frostfs/ir_update_test.go b/pkg/morph/event/frostfs/ir_update_test.go deleted file mode 100644 index fae87e5f9..000000000 --- a/pkg/morph/event/frostfs/ir_update_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package frostfs - -import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/stretchr/testify/require" -) - -func genKey(t *testing.T) *keys.PrivateKey { - priv, err := keys.NewPrivateKey() - require.NoError(t, err) - return priv -} - -func TestParseUpdateInnerRing(t *testing.T) { - publicKeys := []*keys.PublicKey{ - genKey(t).PublicKey(), - genKey(t).PublicKey(), - genKey(t).PublicKey(), - } - - t.Run("wrong number of parameters", func(t *testing.T) { - prms := []stackitem.Item{ - stackitem.NewMap(), - stackitem.NewMap(), - } - - _, err := ParseUpdateInnerRing(prms) - require.EqualError(t, err, event.WrongNumberOfParameters(1, len(prms)).Error()) - }) - - t.Run("wrong first parameter", func(t *testing.T) { - _, err := ParseUpdateInnerRing([]stackitem.Item{ - stackitem.NewMap(), - }) - - require.Error(t, err) - }) - - t.Run("correct", func(t *testing.T) { - ev, err := ParseUpdateInnerRing([]stackitem.Item{ - stackitem.NewArray([]stackitem.Item{ - stackitem.NewByteArray(publicKeys[0].Bytes()), - stackitem.NewByteArray(publicKeys[1].Bytes()), - stackitem.NewByteArray(publicKeys[2].Bytes()), - }), - }) - require.NoError(t, err) - - require.Equal(t, UpdateInnerRing{ - keys: publicKeys, - }, ev) - }) -} From 72470d6b4884780978768add9c3496768cb6ee9c Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 11 Dec 2024 14:43:36 +0300 Subject: [PATCH 1623/1943] [#1555] local_object_storage: Rename method `GetLocked` -> `GetLocks` Renamed to better reflect the method's purpose of returning locks for the specified object. Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/engine/inhume.go | 8 ++++---- pkg/local_object_storage/engine/put.go | 2 +- pkg/local_object_storage/metabase/lock.go | 12 ++++++------ pkg/local_object_storage/shard/lock.go | 8 ++++---- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 80c77af54..0eb9d05c9 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -254,9 +254,9 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e return locked, outErr } -// GetLocked return lock id's if object is locked according to StorageEngine's state. -func (e *StorageEngine) GetLocked(ctx context.Context, addr oid.Address) ([]oid.ID, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.GetLocked", +// GetLocks return lock id's if object is locked according to StorageEngine's state. +func (e *StorageEngine) GetLocks(ctx context.Context, addr oid.Address) ([]oid.ID, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.GetLocks", trace.WithAttributes( attribute.String("address", addr.EncodeToString()), )) @@ -266,7 +266,7 @@ func (e *StorageEngine) GetLocked(ctx context.Context, addr oid.Address) ([]oid. var outErr error e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { - ld, err := h.Shard.GetLocked(ctx, addr) + ld, err := h.Shard.GetLocks(ctx, addr) if err != nil { e.reportShardError(ctx, h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index ba4a144d1..62671f433 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -85,7 +85,7 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { } if !existed && locked { - lockers, err := e.GetLocked(ctx, ecParent) + lockers, err := e.GetLocks(ctx, ecParent) if err != nil { return err } diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index 6b78ef392..b930a0141 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -176,7 +176,7 @@ func objectLocked(tx *bbolt.Tx, idCnr cid.ID, idObj oid.ID) bool { } // return `LOCK` id's if specified object is locked in the specified container. -func getLocked(tx *bbolt.Tx, idCnr cid.ID, idObj oid.ID) ([]oid.ID, error) { +func getLocks(tx *bbolt.Tx, idCnr cid.ID, idObj oid.ID) ([]oid.ID, error) { var lockers []oid.ID bucketLocked := tx.Bucket(bucketNameLocked) if bucketLocked != nil { @@ -351,20 +351,20 @@ func (db *DB) IsLocked(ctx context.Context, prm IsLockedPrm) (res IsLockedRes, e return res, err } -// GetLocked return `LOCK` id's if provided object is locked by any `LOCK`. Not found +// GetLocks return `LOCK` id's if provided object is locked by any `LOCK`. Not found // object is considered as non-locked. // // Returns only non-logical errors related to underlying database. -func (db *DB) GetLocked(ctx context.Context, addr oid.Address) (res []oid.ID, err error) { +func (db *DB) GetLocks(ctx context.Context, addr oid.Address) (res []oid.ID, err error) { var ( startedAt = time.Now() success = false ) defer func() { - db.metrics.AddMethodDuration("GetLocked", time.Since(startedAt), success) + db.metrics.AddMethodDuration("GetLocks", time.Since(startedAt), success) }() - _, span := tracing.StartSpanFromContext(ctx, "metabase.GetLocked", + _, span := tracing.StartSpanFromContext(ctx, "metabase.GetLocks", trace.WithAttributes( attribute.String("address", addr.EncodeToString()), )) @@ -377,7 +377,7 @@ func (db *DB) GetLocked(ctx context.Context, addr oid.Address) (res []oid.ID, er return res, ErrDegradedMode } err = metaerr.Wrap(db.boltDB.View(func(tx *bbolt.Tx) error { - res, err = getLocked(tx, addr.Container(), addr.Object()) + res, err = getLocks(tx, addr.Container(), addr.Object()) return nil })) success = err == nil diff --git a/pkg/local_object_storage/shard/lock.go b/pkg/local_object_storage/shard/lock.go index 4a8d89d63..31ca16aa1 100644 --- a/pkg/local_object_storage/shard/lock.go +++ b/pkg/local_object_storage/shard/lock.go @@ -72,10 +72,10 @@ func (s *Shard) IsLocked(ctx context.Context, addr oid.Address) (bool, error) { return res.Locked(), nil } -// GetLocked return lock id's of the provided object. Not found object is +// GetLocks return lock id's of the provided object. Not found object is // considered as not locked. Requires healthy metabase, returns ErrDegradedMode otherwise. -func (s *Shard) GetLocked(ctx context.Context, addr oid.Address) ([]oid.ID, error) { - ctx, span := tracing.StartSpanFromContext(ctx, "Shard.GetLocked", +func (s *Shard) GetLocks(ctx context.Context, addr oid.Address) ([]oid.ID, error) { + ctx, span := tracing.StartSpanFromContext(ctx, "Shard.GetLocks", trace.WithAttributes( attribute.String("shard_id", s.ID().String()), attribute.String("address", addr.EncodeToString()), @@ -86,5 +86,5 @@ func (s *Shard) GetLocked(ctx context.Context, addr oid.Address) ([]oid.ID, erro if m.NoMetabase() { return nil, ErrDegradedMode } - return s.metaBase.GetLocked(ctx, addr) + return s.metaBase.GetLocks(ctx, addr) } From 3821645085986c116b2f19c721775e6921ad76a0 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 11 Dec 2024 14:47:18 +0300 Subject: [PATCH 1624/1943] [#1555] engine: Refactor `(*StorageEngine).GetLocks` Refactored after renaming the method to replace the confusing `locked` variable with `locks`. Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/engine/inhume.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 0eb9d05c9..bae784064 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -262,23 +262,23 @@ func (e *StorageEngine) GetLocks(ctx context.Context, addr oid.Address) ([]oid.I )) defer span.End() - var locked []oid.ID + var allLocks []oid.ID var outErr error e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { - ld, err := h.Shard.GetLocks(ctx, addr) + locks, err := h.Shard.GetLocks(ctx, addr) if err != nil { e.reportShardError(ctx, h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) outErr = err } - locked = append(locked, ld...) + allLocks = append(allLocks, locks...) return false }) - if len(locked) > 0 { - return locked, nil + if len(allLocks) > 0 { + return allLocks, nil } - return locked, outErr + return allLocks, outErr } func (e *StorageEngine) processExpiredTombstones(ctx context.Context, addrs []meta.TombstonedObject) { From 7853dbc315a7a9d7d450d92e4342b59c155a514c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 11 Dec 2024 15:20:59 +0300 Subject: [PATCH 1625/1943] [#1557] morph/event: Remove embedded structs from scriptHashWithValue Also, make them public, because otherwise `unused` linter complains. ``` pkg/morph/event/utils.go:25:2 unused field `typ` is unused ``` This complain is wrong, though: we _use_ `typ` field because the whole struct is used as a map key. Signed-off-by: Evgenii Stratonikov --- pkg/morph/event/listener.go | 12 ++++-------- pkg/morph/event/utils.go | 18 ++---------------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 9852a3d62..7a16eb991 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -180,7 +180,7 @@ func (l *listener) subscribe(errCh chan error) { // fill the list with the contracts with set event parsers. l.mtx.RLock() for hashType := range l.notificationParsers { - scHash := hashType.ScriptHash() + scHash := hashType.Hash // prevent repetitions for _, hash := range hashes { @@ -189,7 +189,7 @@ func (l *listener) subscribe(errCh chan error) { } } - hashes = append(hashes, hashType.ScriptHash()) + hashes = append(hashes, hashType.Hash) } l.mtx.RUnlock() @@ -326,9 +326,7 @@ func (l *listener) parseAndHandleNotification(ctx context.Context, notifyEvent * ) // get the event parser - keyEvent := scriptHashWithType{} - keyEvent.SetScriptHash(notifyEvent.ScriptHash) - keyEvent.SetType(typEvent) + keyEvent := scriptHashWithType{Hash: notifyEvent.ScriptHash, Type: typEvent} l.mtx.RLock() parser, ok := l.notificationParsers[keyEvent] @@ -451,9 +449,7 @@ func (l *listener) RegisterNotificationHandler(hi NotificationHandlerInfo) { l.mtx.Lock() defer l.mtx.Unlock() - var k scriptHashWithType - k.hash = hi.Contract - k.typ = hi.Type + k := scriptHashWithType{Hash: hi.Contract, Type: hi.Type} l.notificationParsers[k] = hi.Parser l.notificationHandlers[k] = append( diff --git a/pkg/morph/event/utils.go b/pkg/morph/event/utils.go index 99ea9a7f0..058959c63 100644 --- a/pkg/morph/event/utils.go +++ b/pkg/morph/event/utils.go @@ -20,13 +20,9 @@ type scriptHashValue struct { hash util.Uint160 } -type typeValue struct { - typ Type -} - type scriptHashWithType struct { - scriptHashValue - typeValue + Hash util.Uint160 + Type Type } type notaryRequestTypes struct { @@ -73,16 +69,6 @@ func (s scriptHashValue) ScriptHash() util.Uint160 { return s.hash } -// SetType is an event type setter. -func (s *typeValue) SetType(v Type) { - s.typ = v -} - -// GetType is an event type getter. -func (s typeValue) GetType() Type { - return s.typ -} - // WorkerPoolHandler sets closure over worker pool w with passed handler h. func WorkerPoolHandler(w util2.WorkerPool, h Handler, log *logger.Logger) Handler { return func(ctx context.Context, e Event) { From 91d9dc2676fa7739ae5ba6763995ae92fab81d29 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 12 Dec 2024 14:59:49 +0300 Subject: [PATCH 1626/1943] [#1558] morph/event: Remove "could not" from error messages Signed-off-by: Evgenii Stratonikov --- pkg/morph/event/listener.go | 10 +++++----- pkg/morph/event/netmap/update_peer_notary.go | 4 ++-- pkg/morph/event/notary_preparator.go | 18 +++++++++--------- pkg/morph/event/rolemanagement/designate.go | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 7a16eb991..ed77352ec 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -111,7 +111,7 @@ type listener struct { pool *ants.Pool } -const newListenerFailMsg = "could not instantiate Listener" +const newListenerFailMsg = "instantiate Listener" var ( errNilLogger = errors.New("nil logger") @@ -195,20 +195,20 @@ func (l *listener) subscribe(errCh chan error) { err := l.subscriber.SubscribeForNotification(hashes...) if err != nil { - errCh <- fmt.Errorf("could not subscribe for notifications: %w", err) + errCh <- fmt.Errorf("subscribe for notifications: %w", err) return } if len(l.blockHandlers) > 0 { if err = l.subscriber.BlockNotifications(); err != nil { - errCh <- fmt.Errorf("could not subscribe for blocks: %w", err) + errCh <- fmt.Errorf("subscribe for blocks: %w", err) return } } if l.listenNotary { if err = l.subscriber.SubscribeForNotaryRequests(l.notaryMainTXSigner); err != nil { - errCh <- fmt.Errorf("could not subscribe for notary requests: %w", err) + errCh <- fmt.Errorf("subscribe for notary requests: %w", err) return } } @@ -566,7 +566,7 @@ func NewListener(p ListenerParams) (Listener, error) { // The default capacity is 0, which means "infinite". pool, err := ants.NewPool(p.WorkerPoolCapacity) if err != nil { - return nil, fmt.Errorf("could not init worker pool: %w", err) + return nil, fmt.Errorf("init worker pool: %w", err) } return &listener{ diff --git a/pkg/morph/event/netmap/update_peer_notary.go b/pkg/morph/event/netmap/update_peer_notary.go index 0260810b8..993182ab4 100644 --- a/pkg/morph/event/netmap/update_peer_notary.go +++ b/pkg/morph/event/netmap/update_peer_notary.go @@ -10,7 +10,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/vm/opcode" ) -var errNilPubKey = errors.New("could not parse public key: public key is nil") +var errNilPubKey = errors.New("public key is nil") func (s *UpdatePeer) setPublicKey(v []byte) (err error) { if v == nil { @@ -19,7 +19,7 @@ func (s *UpdatePeer) setPublicKey(v []byte) (err error) { s.PubKey, err = keys.NewPublicKeyFromBytes(v, elliptic.P256()) if err != nil { - return fmt.Errorf("could not parse public key: %w", err) + return fmt.Errorf("parse public key: %w", err) } return diff --git a/pkg/morph/event/notary_preparator.go b/pkg/morph/event/notary_preparator.go index 37091f768..40f5984a9 100644 --- a/pkg/morph/event/notary_preparator.go +++ b/pkg/morph/event/notary_preparator.go @@ -127,7 +127,7 @@ func (p Preparator) Prepare(nr *payload.P2PNotaryRequest) (NotaryEvent, error) { for { opCode, param, err = ctx.Next() if err != nil { - return nil, fmt.Errorf("could not get next opcode in script: %w", err) + return nil, fmt.Errorf("get next opcode in script: %w", err) } if opCode == opcode.RET { @@ -147,7 +147,7 @@ func (p Preparator) Prepare(nr *payload.P2PNotaryRequest) (NotaryEvent, error) { // retrieve contract's script hash contractHash, err := util.Uint160DecodeBytesBE(ops[opsLen-2].param) if err != nil { - return nil, fmt.Errorf("could not decode contract hash: %w", err) + return nil, fmt.Errorf("decode contract hash: %w", err) } // retrieve contract's method @@ -164,7 +164,7 @@ func (p Preparator) Prepare(nr *payload.P2PNotaryRequest) (NotaryEvent, error) { if len(args) != 0 { err = p.validateParameterOpcodes(args) if err != nil { - return nil, fmt.Errorf("could not validate arguments: %w", err) + return nil, fmt.Errorf("validate arguments: %w", err) } // without args packing opcodes @@ -206,7 +206,7 @@ func (p Preparator) validateNotaryRequest(nr *payload.P2PNotaryRequest) error { currentAlphabet, err := p.alphaKeys() if err != nil { - return fmt.Errorf("could not fetch Alphabet public keys: %w", err) + return fmt.Errorf("fetch Alphabet public keys: %w", err) } err = p.validateCosigners(ln, nr.MainTransaction.Signers, currentAlphabet) @@ -239,7 +239,7 @@ func (p Preparator) validateParameterOpcodes(ops []Op) error { argsLen, err := IntFromOpcode(ops[l-2]) if err != nil { - return fmt.Errorf("could not parse argument len: %w", err) + return fmt.Errorf("parse argument len: %w", err) } err = validateNestedArgs(argsLen, ops[:l-2]) @@ -273,7 +273,7 @@ func validateNestedArgs(expArgLen int64, ops []Op) error { argsLen, err := IntFromOpcode(ops[i-1]) if err != nil { - return fmt.Errorf("could not parse argument len: %w", err) + return fmt.Errorf("parse argument len: %w", err) } expArgLen += argsLen + 1 @@ -307,7 +307,7 @@ func (p Preparator) validateExpiration(fbTX *transaction.Transaction) error { currBlock, err := p.blockCounter.BlockCount() if err != nil { - return fmt.Errorf("could not fetch current chain height: %w", err) + return fmt.Errorf("fetch current chain height: %w", err) } if currBlock >= nvb.Height { @@ -327,7 +327,7 @@ func (p Preparator) validateCosigners(expected int, s []transaction.Signer, alph alphaVerificationScript, err := smartcontract.CreateMultiSigRedeemScript(len(alphaKeys)*2/3+1, alphaKeys) if err != nil { - return fmt.Errorf("could not get Alphabet verification script: %w", err) + return fmt.Errorf("get Alphabet verification script: %w", err) } if !s[1].Account.Equals(hash.Hash160(alphaVerificationScript)) { @@ -346,7 +346,7 @@ func (p Preparator) validateWitnesses(w []transaction.Witness, alphaKeys keys.Pu alphaVerificationScript, err := smartcontract.CreateMultiSigRedeemScript(len(alphaKeys)*2/3+1, alphaKeys) if err != nil { - return fmt.Errorf("could not get Alphabet verification script: %w", err) + return fmt.Errorf("get Alphabet verification script: %w", err) } // the second one must be witness of the current diff --git a/pkg/morph/event/rolemanagement/designate.go b/pkg/morph/event/rolemanagement/designate.go index 28c968046..b384e436b 100644 --- a/pkg/morph/event/rolemanagement/designate.go +++ b/pkg/morph/event/rolemanagement/designate.go @@ -26,7 +26,7 @@ func (Designate) MorphEvent() {} func ParseDesignate(e *state.ContainedNotificationEvent) (event.Event, error) { params, err := event.ParseStackArray(e) if err != nil { - return nil, fmt.Errorf("could not parse stack items from notify event: %w", err) + return nil, fmt.Errorf("parse stack items from notify event: %w", err) } if len(params) != 2 { From 7151c71d51615648233acda1479feb7f4ab86ce6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 12 Dec 2024 15:06:20 +0300 Subject: [PATCH 1627/1943] [#1558] morph/client: Remove "could not"/"can't"/"failed to" from error messages Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/balance/balanceOf.go | 2 +- pkg/morph/client/balance/decimals.go | 2 +- pkg/morph/client/balance/transfer.go | 2 +- pkg/morph/client/client.go | 6 ++-- pkg/morph/client/container/containers_of.go | 2 +- pkg/morph/client/container/delete.go | 2 +- pkg/morph/client/container/deletion_info.go | 8 ++--- pkg/morph/client/container/get.go | 14 ++++---- pkg/morph/client/container/list.go | 4 +-- pkg/morph/client/container/put.go | 2 +- pkg/morph/client/frostfsid/subject.go | 6 ++-- pkg/morph/client/netmap/epoch.go | 4 +-- pkg/morph/client/netmap/innerring.go | 6 ++-- pkg/morph/client/netmap/new_epoch.go | 4 +-- pkg/morph/client/netmap/peer.go | 2 +- pkg/morph/client/nns.go | 2 +- pkg/morph/client/notary.go | 38 ++++++++++----------- pkg/morph/client/static.go | 2 +- pkg/morph/client/util.go | 2 +- 19 files changed, 55 insertions(+), 55 deletions(-) diff --git a/pkg/morph/client/balance/balanceOf.go b/pkg/morph/client/balance/balanceOf.go index 12a499ffb..a5fb8e82a 100644 --- a/pkg/morph/client/balance/balanceOf.go +++ b/pkg/morph/client/balance/balanceOf.go @@ -29,7 +29,7 @@ func (c *Client) BalanceOf(id user.ID) (*big.Int, error) { amount, err := client.BigIntFromStackItem(prms[0]) if err != nil { - return nil, fmt.Errorf("could not get integer stack item from stack item (%s): %w", balanceOfMethod, err) + return nil, fmt.Errorf("get integer stack item from stack item (%s): %w", balanceOfMethod, err) } return amount, nil } diff --git a/pkg/morph/client/balance/decimals.go b/pkg/morph/client/balance/decimals.go index 28329ee6e..c2a66dded 100644 --- a/pkg/morph/client/balance/decimals.go +++ b/pkg/morph/client/balance/decimals.go @@ -21,7 +21,7 @@ func (c *Client) Decimals() (uint32, error) { decimals, err := client.IntFromStackItem(prms[0]) if err != nil { - return 0, fmt.Errorf("could not get integer stack item from stack item (%s): %w", decimalsMethod, err) + return 0, fmt.Errorf("get integer stack item from stack item (%s): %w", decimalsMethod, err) } return uint32(decimals), nil } diff --git a/pkg/morph/client/balance/transfer.go b/pkg/morph/client/balance/transfer.go index 16c8f3982..52d69dccb 100644 --- a/pkg/morph/client/balance/transfer.go +++ b/pkg/morph/client/balance/transfer.go @@ -39,7 +39,7 @@ func (c *Client) TransferX(ctx context.Context, p TransferPrm) error { _, err = c.client.Invoke(ctx, prm) if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", transferXMethod, err) + return fmt.Errorf("invoke method (%s): %w", transferXMethod, err) } return nil } diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index a0c29141b..5a5d24900 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -196,7 +196,7 @@ func (c *Client) Invoke(ctx context.Context, contract util.Uint160, fee fixedn.F txHash, vub, err := c.rpcActor.SendTunedCall(contract, method, nil, addFeeCheckerModifier(int64(fee)), args...) if err != nil { - return InvokeRes{}, fmt.Errorf("could not invoke %s: %w", method, err) + return InvokeRes{}, fmt.Errorf("invoke %s: %w", method, err) } c.logger.Debug(ctx, logs.ClientNeoClientInvoke, @@ -509,7 +509,7 @@ func (c *Client) NeoFSAlphabetList() (res keys.PublicKeys, err error) { list, err := c.roleList(noderoles.NeoFSAlphabet) if err != nil { - return nil, fmt.Errorf("can't get alphabet nodes role list: %w", err) + return nil, fmt.Errorf("get alphabet nodes role list: %w", err) } return list, nil @@ -523,7 +523,7 @@ func (c *Client) GetDesignateHash() util.Uint160 { func (c *Client) roleList(r noderoles.Role) (keys.PublicKeys, error) { height, err := c.rpcActor.GetBlockCount() if err != nil { - return nil, fmt.Errorf("can't get chain height: %w", err) + return nil, fmt.Errorf("get chain height: %w", err) } return c.rolemgmt.GetDesignatedByRole(r, height) diff --git a/pkg/morph/client/container/containers_of.go b/pkg/morph/client/container/containers_of.go index c4db0fe6e..5fe15be0d 100644 --- a/pkg/morph/client/container/containers_of.go +++ b/pkg/morph/client/container/containers_of.go @@ -26,7 +26,7 @@ func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) { cb := func(item stackitem.Item) error { rawID, err := client.BytesFromStackItem(item) if err != nil { - return fmt.Errorf("could not get byte array from stack item (%s): %w", containersOfMethod, err) + return fmt.Errorf("get byte array from stack item (%s): %w", containersOfMethod, err) } var id cid.ID diff --git a/pkg/morph/client/container/delete.go b/pkg/morph/client/container/delete.go index ef5cc8c38..09912efa5 100644 --- a/pkg/morph/client/container/delete.go +++ b/pkg/morph/client/container/delete.go @@ -78,7 +78,7 @@ func (c *Client) Delete(ctx context.Context, p DeletePrm) (uint32, error) { res, err := c.client.Invoke(ctx, prm) if err != nil { - return 0, fmt.Errorf("could not invoke method (%s): %w", deleteMethod, err) + return 0, fmt.Errorf("invoke method (%s): %w", deleteMethod, err) } return res.VUB, nil } diff --git a/pkg/morph/client/container/deletion_info.go b/pkg/morph/client/container/deletion_info.go index 436ca3c01..b86e0ce9c 100644 --- a/pkg/morph/client/container/deletion_info.go +++ b/pkg/morph/client/container/deletion_info.go @@ -46,7 +46,7 @@ func (c *Client) DeletionInfo(cid []byte) (*containercore.DelInfo, error) { arr, err := client.ArrayFromStackItem(res[0]) if err != nil { - return nil, fmt.Errorf("could not get item array of container (%s): %w", deletionInfoMethod, err) + return nil, fmt.Errorf("get item array of container (%s): %w", deletionInfoMethod, err) } if len(arr) != 2 { @@ -55,17 +55,17 @@ func (c *Client) DeletionInfo(cid []byte) (*containercore.DelInfo, error) { rawOwner, err := client.BytesFromStackItem(arr[0]) if err != nil { - return nil, fmt.Errorf("could not get byte array of container (%s): %w", deletionInfoMethod, err) + return nil, fmt.Errorf("get byte array of container (%s): %w", deletionInfoMethod, err) } var owner user.ID if err := owner.DecodeString(base58.Encode(rawOwner)); err != nil { - return nil, fmt.Errorf("could not decode container owner id (%s): %w", deletionInfoMethod, err) + return nil, fmt.Errorf("decode container owner id (%s): %w", deletionInfoMethod, err) } epoch, err := client.BigIntFromStackItem(arr[1]) if err != nil { - return nil, fmt.Errorf("could not get byte array of container signature (%s): %w", deletionInfoMethod, err) + return nil, fmt.Errorf("get byte array of container signature (%s): %w", deletionInfoMethod, err) } return &containercore.DelInfo{ diff --git a/pkg/morph/client/container/get.go b/pkg/morph/client/container/get.go index 1d84e9109..2ab58bf01 100644 --- a/pkg/morph/client/container/get.go +++ b/pkg/morph/client/container/get.go @@ -60,7 +60,7 @@ func (c *Client) Get(cid []byte) (*containercore.Container, error) { arr, err := client.ArrayFromStackItem(res[0]) if err != nil { - return nil, fmt.Errorf("could not get item array of container (%s): %w", getMethod, err) + return nil, fmt.Errorf("get item array of container (%s): %w", getMethod, err) } if len(arr) != 4 { @@ -69,29 +69,29 @@ func (c *Client) Get(cid []byte) (*containercore.Container, error) { cnrBytes, err := client.BytesFromStackItem(arr[0]) if err != nil { - return nil, fmt.Errorf("could not get byte array of container (%s): %w", getMethod, err) + return nil, fmt.Errorf("get byte array of container (%s): %w", getMethod, err) } sigBytes, err := client.BytesFromStackItem(arr[1]) if err != nil { - return nil, fmt.Errorf("could not get byte array of container signature (%s): %w", getMethod, err) + return nil, fmt.Errorf("get byte array of container signature (%s): %w", getMethod, err) } pub, err := client.BytesFromStackItem(arr[2]) if err != nil { - return nil, fmt.Errorf("could not get byte array of public key (%s): %w", getMethod, err) + return nil, fmt.Errorf("get byte array of public key (%s): %w", getMethod, err) } tokBytes, err := client.BytesFromStackItem(arr[3]) if err != nil { - return nil, fmt.Errorf("could not get byte array of session token (%s): %w", getMethod, err) + return nil, fmt.Errorf("get byte array of session token (%s): %w", getMethod, err) } var cnr containercore.Container if err := cnr.Value.Unmarshal(cnrBytes); err != nil { // use other major version if there any - return nil, fmt.Errorf("can't unmarshal container: %w", err) + return nil, fmt.Errorf("unmarshal container: %w", err) } if len(tokBytes) > 0 { @@ -99,7 +99,7 @@ func (c *Client) Get(cid []byte) (*containercore.Container, error) { err = cnr.Session.Unmarshal(tokBytes) if err != nil { - return nil, fmt.Errorf("could not unmarshal session token: %w", err) + return nil, fmt.Errorf("unmarshal session token: %w", err) } } diff --git a/pkg/morph/client/container/list.go b/pkg/morph/client/container/list.go index 55317375a..d9719aedd 100644 --- a/pkg/morph/client/container/list.go +++ b/pkg/morph/client/container/list.go @@ -34,14 +34,14 @@ func (c *Client) list(idUser *user.ID) ([]cid.ID, error) { res, err = client.ArrayFromStackItem(res[0]) if err != nil { - return nil, fmt.Errorf("could not get stack item array from stack item (%s): %w", listMethod, err) + return nil, fmt.Errorf("get stack item array from stack item (%s): %w", listMethod, err) } cidList := make([]cid.ID, 0, len(res)) for i := range res { rawID, err := client.BytesFromStackItem(res[i]) if err != nil { - return nil, fmt.Errorf("could not get byte array from stack item (%s): %w", listMethod, err) + return nil, fmt.Errorf("get byte array from stack item (%s): %w", listMethod, err) } var id cid.ID diff --git a/pkg/morph/client/container/put.go b/pkg/morph/client/container/put.go index 1eb03694f..3bb84eb87 100644 --- a/pkg/morph/client/container/put.go +++ b/pkg/morph/client/container/put.go @@ -117,7 +117,7 @@ func (c *Client) Put(ctx context.Context, p PutPrm) error { _, err := c.client.Invoke(ctx, prm) if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", method, err) + return fmt.Errorf("invoke method (%s): %w", method, err) } return nil } diff --git a/pkg/morph/client/frostfsid/subject.go b/pkg/morph/client/frostfsid/subject.go index 7c93f4922..305f3ce09 100644 --- a/pkg/morph/client/frostfsid/subject.go +++ b/pkg/morph/client/frostfsid/subject.go @@ -31,7 +31,7 @@ func (c *Client) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) subj, err := frostfsidclient.ParseSubject(structArr) if err != nil { - return nil, fmt.Errorf("could not parse test invocation result (%s): %w", methodGetSubject, err) + return nil, fmt.Errorf("parse test invocation result (%s): %w", methodGetSubject, err) } return subj, nil @@ -54,7 +54,7 @@ func (c *Client) GetSubjectExtended(addr util.Uint160) (*frostfsidclient.Subject subj, err := frostfsidclient.ParseSubjectExtended(structArr) if err != nil { - return nil, fmt.Errorf("could not parse test invocation result (%s): %w", methodGetSubject, err) + return nil, fmt.Errorf("parse test invocation result (%s): %w", methodGetSubject, err) } return subj, nil @@ -67,7 +67,7 @@ func checkStackItem(res []stackitem.Item) (structArr []stackitem.Item, err error structArr, err = client.ArrayFromStackItem(res[0]) if err != nil { - return nil, fmt.Errorf("could not get item array of container (%s): %w", methodGetSubject, err) + return nil, fmt.Errorf("get item array of container (%s): %w", methodGetSubject, err) } return } diff --git a/pkg/morph/client/netmap/epoch.go b/pkg/morph/client/netmap/epoch.go index 6d909a7a1..6d7394998 100644 --- a/pkg/morph/client/netmap/epoch.go +++ b/pkg/morph/client/netmap/epoch.go @@ -25,7 +25,7 @@ func (c *Client) Epoch() (uint64, error) { num, err := client.IntFromStackItem(items[0]) if err != nil { - return 0, fmt.Errorf("could not get number from stack item (%s): %w", epochMethod, err) + return 0, fmt.Errorf("get number from stack item (%s): %w", epochMethod, err) } return uint64(num), nil } @@ -49,7 +49,7 @@ func (c *Client) LastEpochBlock() (uint32, error) { block, err := client.IntFromStackItem(items[0]) if err != nil { - return 0, fmt.Errorf("could not get number from stack item (%s): %w", + return 0, fmt.Errorf("get number from stack item (%s): %w", lastEpochBlockMethod, err) } return uint32(block), nil diff --git a/pkg/morph/client/netmap/innerring.go b/pkg/morph/client/netmap/innerring.go index 824827d6f..0cfad4c82 100644 --- a/pkg/morph/client/netmap/innerring.go +++ b/pkg/morph/client/netmap/innerring.go @@ -59,7 +59,7 @@ func irKeysFromStackItem(stack []stackitem.Item, method string) (keys.PublicKeys irs, err := client.ArrayFromStackItem(stack[0]) if err != nil { - return nil, fmt.Errorf("could not get stack item array from stack item (%s): %w", method, err) + return nil, fmt.Errorf("get stack item array from stack item (%s): %w", method, err) } irKeys := make(keys.PublicKeys, len(irs)) @@ -79,7 +79,7 @@ const irNodeFixedPrmNumber = 1 func irKeyFromStackItem(prm stackitem.Item) (*keys.PublicKey, error) { prms, err := client.ArrayFromStackItem(prm) if err != nil { - return nil, fmt.Errorf("could not get stack item array (IRNode): %w", err) + return nil, fmt.Errorf("get stack item array (IRNode): %w", err) } else if ln := len(prms); ln != irNodeFixedPrmNumber { return nil, fmt.Errorf( "unexpected stack item count (IRNode): expected %d, has %d", @@ -90,7 +90,7 @@ func irKeyFromStackItem(prm stackitem.Item) (*keys.PublicKey, error) { byteKey, err := client.BytesFromStackItem(prms[0]) if err != nil { - return nil, fmt.Errorf("could not parse bytes from stack item (IRNode): %w", err) + return nil, fmt.Errorf("parse bytes from stack item (IRNode): %w", err) } return keys.NewPublicKeyFromBytes(byteKey, elliptic.P256()) diff --git a/pkg/morph/client/netmap/new_epoch.go b/pkg/morph/client/netmap/new_epoch.go index efcdfd7b6..341b20935 100644 --- a/pkg/morph/client/netmap/new_epoch.go +++ b/pkg/morph/client/netmap/new_epoch.go @@ -16,7 +16,7 @@ func (c *Client) NewEpoch(ctx context.Context, epoch uint64) error { _, err := c.client.Invoke(ctx, prm) if err != nil { - return fmt.Errorf("could not invoke method (%s): %w", newEpochMethod, err) + return fmt.Errorf("invoke method (%s): %w", newEpochMethod, err) } return nil } @@ -34,7 +34,7 @@ func (c *Client) NewEpochControl(ctx context.Context, epoch uint64, vub uint32) res, err := c.client.Invoke(ctx, prm) if err != nil { - return 0, fmt.Errorf("could not invoke method (%s): %w", newEpochMethod, err) + return 0, fmt.Errorf("invoke method (%s): %w", newEpochMethod, err) } return res.VUB, nil } diff --git a/pkg/morph/client/netmap/peer.go b/pkg/morph/client/netmap/peer.go index 949e8cb63..e83acde39 100644 --- a/pkg/morph/client/netmap/peer.go +++ b/pkg/morph/client/netmap/peer.go @@ -41,7 +41,7 @@ func (c *Client) AddPeer(ctx context.Context, p AddPeerPrm) error { prm.InvokePrmOptional = p.InvokePrmOptional if _, err := c.client.Invoke(ctx, prm); err != nil { - return fmt.Errorf("could not invoke method (%s): %w", method, err) + return fmt.Errorf("invoke method (%s): %w", method, err) } return nil } diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index 218f7ad8e..f292dccf1 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -107,7 +107,7 @@ func (c *Client) NNSHash() (util.Uint160, error) { func nnsResolveItem(c *rpcclient.WSClient, nnsHash util.Uint160, domain string) (stackitem.Item, error) { found, err := exists(c, nnsHash, domain) if err != nil { - return nil, fmt.Errorf("could not check presence in NNS contract for %s: %w", domain, err) + return nil, fmt.Errorf("check presence in NNS contract for %s: %w", domain, err) } if !found { diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 8516c89a9..1e80be921 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -62,7 +62,7 @@ const ( notaryExpirationOfMethod = "expirationOf" setDesignateMethod = "designateAsRole" - notaryBalanceErrMsg = "can't fetch notary balance" + notaryBalanceErrMsg = "fetch notary balance" notaryNotEnabledPanicMsg = "notary support was not enabled on this client" ) @@ -155,12 +155,12 @@ func (c *Client) DepositNotary(ctx context.Context, amount fixedn.Fixed8, delta bc, err := c.rpcActor.GetBlockCount() if err != nil { - return util.Uint256{}, fmt.Errorf("can't get blockchain height: %w", err) + return util.Uint256{}, fmt.Errorf("get blockchain height: %w", err) } currentTill, err := c.depositExpirationOf() if err != nil { - return util.Uint256{}, fmt.Errorf("can't get previous expiration value: %w", err) + return util.Uint256{}, fmt.Errorf("get previous expiration value: %w", err) } till := max(int64(bc+delta), currentTill) @@ -197,7 +197,7 @@ func (c *Client) depositNotary(ctx context.Context, amount fixedn.Fixed8, till i []any{c.acc.PrivateKey().GetScriptHash(), till}) if err != nil { if !errors.Is(err, neorpc.ErrAlreadyExists) { - return util.Uint256{}, 0, fmt.Errorf("can't make notary deposit: %w", err) + return util.Uint256{}, 0, fmt.Errorf("make notary deposit: %w", err) } // Transaction is already in mempool waiting to be processed. @@ -289,7 +289,7 @@ func (c *Client) UpdateNotaryList(ctx context.Context, prm UpdateNotaryListPrm) nonce, vub, err := c.CalculateNonceAndVUB(&prm.hash) if err != nil { - return fmt.Errorf("could not calculate nonce and `valicUntilBlock` values: %w", err) + return fmt.Errorf("calculate nonce and `valicUntilBlock` values: %w", err) } return c.notaryInvokeAsCommittee( @@ -338,7 +338,7 @@ func (c *Client) UpdateNeoFSAlphabetList(ctx context.Context, prm UpdateAlphabet nonce, vub, err := c.CalculateNonceAndVUB(&prm.hash) if err != nil { - return fmt.Errorf("could not calculate nonce and `valicUntilBlock` values: %w", err) + return fmt.Errorf("calculate nonce and `valicUntilBlock` values: %w", err) } return c.notaryInvokeAsCommittee( @@ -407,7 +407,7 @@ func (c *Client) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error { alphabetList, err := c.notary.alphabetSource() if err != nil { - return fmt.Errorf("could not fetch current alphabet keys: %w", err) + return fmt.Errorf("fetch current alphabet keys: %w", err) } cosigners, err := c.notaryCosignersFromTx(mainTx, alphabetList) @@ -529,24 +529,24 @@ func (c *Client) notaryCosignersFromTx(mainTx *transaction.Transaction, alphabet if ok { pub, err := keys.NewPublicKeyFromBytes(pubBytes, elliptic.P256()) if err != nil { - return nil, fmt.Errorf("failed to parse verification script of signer #2: invalid public key: %w", err) + return nil, fmt.Errorf("parse verification script of signer #2: invalid public key: %w", err) } acc = notary.FakeSimpleAccount(pub) } else { m, pubsBytes, ok := vm.ParseMultiSigContract(script) if !ok { - return nil, errors.New("failed to parse verification script of signer #2: unknown witness type") + return nil, errors.New("parse verification script of signer #2: unknown witness type") } pubs := make(keys.PublicKeys, len(pubsBytes)) for i := range pubs { pubs[i], err = keys.NewPublicKeyFromBytes(pubsBytes[i], elliptic.P256()) if err != nil { - return nil, fmt.Errorf("failed to parse verification script of signer #2: invalid public key #%d: %w", i, err) + return nil, fmt.Errorf("parse verification script of signer #2: invalid public key #%d: %w", i, err) } } acc, err = notary.FakeMultisigAccount(m, pubs) if err != nil { - return nil, fmt.Errorf("failed to create fake account for signer #2: %w", err) + return nil, fmt.Errorf("create fake account for signer #2: %w", err) } } } @@ -623,7 +623,7 @@ func (c *Client) notaryMultisigAccount(ir []*keys.PublicKey, committee, invokedB err := multisigAccount.ConvertMultisig(m, ir) if err != nil { // wrap error as FrostFS-specific since the call is not related to any client - return nil, wrapFrostFSError(fmt.Errorf("can't convert account to inner ring multisig wallet: %w", err)) + return nil, wrapFrostFSError(fmt.Errorf("convert account to inner ring multisig wallet: %w", err)) } } else { // alphabet multisig redeem script is @@ -632,7 +632,7 @@ func (c *Client) notaryMultisigAccount(ir []*keys.PublicKey, committee, invokedB multisigAccount, err = notary.FakeMultisigAccount(m, ir) if err != nil { // wrap error as FrostFS-specific since the call is not related to any client - return nil, wrapFrostFSError(fmt.Errorf("can't make inner ring multisig wallet: %w", err)) + return nil, wrapFrostFSError(fmt.Errorf("make inner ring multisig wallet: %w", err)) } } @@ -642,7 +642,7 @@ func (c *Client) notaryMultisigAccount(ir []*keys.PublicKey, committee, invokedB func (c *Client) notaryTxValidationLimit() (uint32, error) { bc, err := c.rpcActor.GetBlockCount() if err != nil { - return 0, fmt.Errorf("can't get current blockchain height: %w", err) + return 0, fmt.Errorf("get current blockchain height: %w", err) } minTime := bc + c.notary.txValidTime @@ -663,7 +663,7 @@ func (c *Client) depositExpirationOf() (int64, error) { currentTillBig, err := expirationRes[0].TryInteger() if err != nil { - return 0, fmt.Errorf("can't parse deposit till value: %w", err) + return 0, fmt.Errorf("parse deposit till value: %w", err) } return currentTillBig.Int64(), nil @@ -742,12 +742,12 @@ func alreadyOnChainError(err error) bool { func CalculateNotaryDepositAmount(c *Client, gasMul, gasDiv int64) (fixedn.Fixed8, error) { notaryBalance, err := c.GetNotaryDeposit() if err != nil { - return 0, fmt.Errorf("could not get notary balance: %w", err) + return 0, fmt.Errorf("get notary balance: %w", err) } gasBalance, err := c.GasBalance() if err != nil { - return 0, fmt.Errorf("could not get GAS balance: %w", err) + return 0, fmt.Errorf("get GAS balance: %w", err) } if gasBalance == 0 { @@ -796,12 +796,12 @@ func (c *Client) calculateNonceAndVUB(hash *util.Uint256, roundBlockHeight bool) if hash != nil { height, err = c.getTransactionHeight(*hash) if err != nil { - return 0, 0, fmt.Errorf("could not get transaction height: %w", err) + return 0, 0, fmt.Errorf("get transaction height: %w", err) } } else { height, err = c.rpcActor.GetBlockCount() if err != nil { - return 0, 0, fmt.Errorf("could not get chain height: %w", err) + return 0, 0, fmt.Errorf("get chain height: %w", err) } } diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index be4c09182..21adebd9e 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -159,7 +159,7 @@ func (s StaticClient) Invoke(ctx context.Context, prm InvokePrm) (InvokeRes, err nonce, vub, err = s.client.CalculateNonceAndVUB(prm.hash) } if err != nil { - return InvokeRes{}, fmt.Errorf("could not calculate nonce and VUB for notary alphabet invoke: %w", err) + return InvokeRes{}, fmt.Errorf("calculate nonce and VUB for notary alphabet invoke: %w", err) } vubP = &vub diff --git a/pkg/morph/client/util.go b/pkg/morph/client/util.go index cd55d6bd2..f68d39beb 100644 --- a/pkg/morph/client/util.go +++ b/pkg/morph/client/util.go @@ -53,7 +53,7 @@ func BytesFromStackItem(param stackitem.Item) ([]byte, error) { case stackitem.IntegerT: n, err := param.TryInteger() if err != nil { - return nil, fmt.Errorf("can't parse integer bytes: %w", err) + return nil, fmt.Errorf("parse integer bytes: %w", err) } return n.Bytes(), nil From d165ac042cf9f34a54117ea002d789fae0b0905b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 12 Dec 2024 15:27:17 +0300 Subject: [PATCH 1628/1943] [#1558] morph/client: Reuse notary rpcclient wrapper Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/notary.go | 44 ++++++-------------------------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 1e80be921..dbd58a53a 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -58,16 +58,11 @@ const ( defaultNotaryValidTime = 50 defaultNotaryRoundTime = 100 - notaryBalanceOfMethod = "balanceOf" - notaryExpirationOfMethod = "expirationOf" - setDesignateMethod = "designateAsRole" + setDesignateMethod = "designateAsRole" - notaryBalanceErrMsg = "fetch notary balance" notaryNotEnabledPanicMsg = "notary support was not enabled on this client" ) -var errUnexpectedItems = errors.New("invalid number of NEO VM arguments on stack") - func defaultNotaryConfig(c *Client) *notaryCfg { return ¬aryCfg{ txValidTime: defaultNotaryValidTime, @@ -158,12 +153,13 @@ func (c *Client) DepositNotary(ctx context.Context, amount fixedn.Fixed8, delta return util.Uint256{}, fmt.Errorf("get blockchain height: %w", err) } - currentTill, err := c.depositExpirationOf() + r := notary.NewReader(c.rpcActor) + currentTill, err := r.ExpirationOf(c.acc.PrivateKey().GetScriptHash()) if err != nil { return util.Uint256{}, fmt.Errorf("get previous expiration value: %w", err) } - till := max(int64(bc+delta), currentTill) + till := max(int64(bc+delta), int64(currentTill)) res, _, err := c.depositNotary(ctx, amount, till) return res, err } @@ -237,18 +233,10 @@ func (c *Client) GetNotaryDeposit() (res int64, err error) { sh := c.acc.PrivateKey().PublicKey().GetScriptHash() - items, err := c.TestInvoke(c.notary.notary, notaryBalanceOfMethod, sh) + r := notary.NewReader(c.rpcActor) + bigIntDeposit, err := r.BalanceOf(sh) if err != nil { - return 0, fmt.Errorf("test invoke (%s): %w", notaryBalanceOfMethod, err) - } - - if len(items) != 1 { - return 0, wrapFrostFSError(fmt.Errorf("%v: %w", notaryBalanceErrMsg, errUnexpectedItems)) - } - - bigIntDeposit, err := items[0].TryInteger() - if err != nil { - return 0, wrapFrostFSError(fmt.Errorf("%v: %w", notaryBalanceErrMsg, err)) + return 0, fmt.Errorf("get notary deposit: %w", err) } return bigIntDeposit.Int64(), nil @@ -651,24 +639,6 @@ func (c *Client) notaryTxValidationLimit() (uint32, error) { return rounded, nil } -func (c *Client) depositExpirationOf() (int64, error) { - expirationRes, err := c.TestInvoke(c.notary.notary, notaryExpirationOfMethod, c.acc.PrivateKey().GetScriptHash()) - if err != nil { - return 0, fmt.Errorf("test invoke (%s): %w", notaryExpirationOfMethod, err) - } - - if len(expirationRes) != 1 { - return 0, fmt.Errorf("method returned unexpected item count: %d", len(expirationRes)) - } - - currentTillBig, err := expirationRes[0].TryInteger() - if err != nil { - return 0, fmt.Errorf("parse deposit till value: %w", err) - } - - return currentTillBig.Int64(), nil -} - // sigCount returns the number of required signature. // For FrostFS Alphabet M is a 2/3+1 of it (like in dBFT). // If committee is true, returns M as N/2+1. From 2af3409d39f173e749fb6b1396cc2f4c03d1b179 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 13 Dec 2024 12:59:35 +0300 Subject: [PATCH 1629/1943] [#1510] metabase/test: Fix `BenchmarkGet` Fix misplaced `(*DB).Close` (broken after 47dcfa20f3) Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/metabase/get_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/get_test.go b/pkg/local_object_storage/metabase/get_test.go index c93d2c992..98c428410 100644 --- a/pkg/local_object_storage/metabase/get_test.go +++ b/pkg/local_object_storage/metabase/get_test.go @@ -219,7 +219,6 @@ func benchmarkGet(b *testing.B, numOfObj int) { meta.WithMaxBatchSize(batchSize), meta.WithMaxBatchDelay(10*time.Millisecond), ) - defer func() { require.NoError(b, db.Close(context.Background())) }() addrs := make([]oid.Address, 0, numOfObj) for range numOfObj { @@ -234,6 +233,7 @@ func benchmarkGet(b *testing.B, numOfObj int) { } db, addrs := prepareDb(runtime.NumCPU()) + defer func() { require.NoError(b, db.Close(context.Background())) }() b.Run("parallel", func(b *testing.B) { b.ReportAllocs() From 8ba9f31fca82ca8c0ccabdbb7254b2ad67a2f6fe Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 13 Dec 2024 13:04:12 +0300 Subject: [PATCH 1630/1943] [#1510] metabase/test: Fix `BenchmarkListWithCursor` - Fix misplaced `(*DB).Close` (broken after 47dcfa20f3) - Use `errors.Is` for error checking (broken after fcdbf5e509) Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/metabase/list_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 6f6463071..1d8beb175 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -18,6 +18,8 @@ import ( func BenchmarkListWithCursor(b *testing.B) { db := listWithCursorPrepareDB(b) + defer func() { require.NoError(b, db.Close(context.Background())) }() + b.Run("1 item", func(b *testing.B) { benchmarkListWithCursor(b, db, 1) }) @@ -33,7 +35,6 @@ func listWithCursorPrepareDB(b *testing.B) *meta.DB { db := newDB(b, meta.WithMaxBatchSize(1), meta.WithBoltDBOptions(&bbolt.Options{ NoSync: true, })) // faster single-thread generation - defer func() { require.NoError(b, db.Close(context.Background())) }() obj := testutil.GenerateObject() for i := range 100_000 { // should be a multiple of all batch sizes @@ -55,7 +56,7 @@ func benchmarkListWithCursor(b *testing.B, db *meta.DB, batchSize int) { for range b.N { res, err := db.ListWithCursor(context.Background(), prm) if err != nil { - if err != meta.ErrEndOfListing { + if errors.Is(err, meta.ErrEndOfListing) { b.Fatalf("error: %v", err) } prm.SetCursor(nil) From f0c43c8d80e79b12111e8e8d1574e847f5162ac1 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 13 Dec 2024 11:44:56 +0300 Subject: [PATCH 1631/1943] [#1502] Use `zap.Error` for logging errors Use `zap.Error` instead of `zap.String` for logging errors: change all expressions like `zap.String("error", err.Error())` or `zap.String("err", err.Error())` to `zap.Error(err)`. Leave similar expressions with other messages unchanged, for example, `zap.String("last_error", lastErr.Error())` or `zap.String("reason", ctx.Err().Error())`. This change was made by applying the following patch: ```diff @@ var err expression @@ -zap.String("error", err.Error()) +zap.Error(err) @@ var err expression @@ -zap.String("err", err.Error()) +zap.Error(err) ``` Signed-off-by: Aleksey Savchuk --- cmd/frostfs-ir/httpcomponent.go | 2 +- cmd/frostfs-ir/main.go | 4 ++-- cmd/frostfs-ir/pprof.go | 2 +- cmd/frostfs-node/config.go | 4 ++-- cmd/frostfs-node/main.go | 2 +- cmd/frostfs-node/morph.go | 4 ++-- cmd/frostfs-node/netmap.go | 2 +- cmd/frostfs-node/object.go | 4 ++-- cmd/frostfs-node/tree.go | 2 +- pkg/core/object/sender_classifier.go | 4 ++-- pkg/innerring/initialization.go | 4 ++-- pkg/innerring/innerring.go | 4 ++-- .../processors/alphabet/process_emit.go | 10 +++++----- .../processors/container/process_container.go | 8 ++++---- .../processors/frostfs/process_assets.go | 2 +- .../processors/governance/process_update.go | 18 ++++++++--------- .../processors/netmap/process_cleanup.go | 2 +- .../processors/netmap/process_epoch.go | 8 ++++---- .../processors/netmap/process_peers.go | 2 +- pkg/innerring/state.go | 8 ++++---- .../blobstor/blobovniczatree/delete.go | 2 +- .../blobstor/blobovniczatree/exists.go | 2 +- .../blobstor/blobovniczatree/get.go | 2 +- .../blobstor/blobovniczatree/get_range.go | 2 +- .../blobstor/blobovniczatree/iterate.go | 4 ++-- .../blobstor/blobovniczatree/manager.go | 4 ++-- .../blobstor/blobovniczatree/put.go | 4 ++-- pkg/local_object_storage/blobstor/control.go | 2 +- pkg/local_object_storage/blobstor/exists.go | 2 +- .../blobstor/fstree/fstree.go | 4 ++-- pkg/local_object_storage/blobstor/iterate.go | 2 +- pkg/local_object_storage/engine/control.go | 2 +- pkg/local_object_storage/engine/delete.go | 6 +++--- pkg/local_object_storage/engine/engine.go | 4 ++-- pkg/local_object_storage/engine/get.go | 2 +- pkg/local_object_storage/engine/put.go | 6 +++--- pkg/local_object_storage/engine/range.go | 2 +- pkg/local_object_storage/shard/control.go | 2 +- pkg/local_object_storage/shard/delete.go | 4 ++-- pkg/local_object_storage/shard/gc.go | 20 +++++++++---------- pkg/local_object_storage/shard/inhume.go | 2 +- pkg/local_object_storage/shard/list.go | 2 +- pkg/local_object_storage/shard/put.go | 2 +- pkg/morph/client/client.go | 4 ++-- pkg/morph/event/listener.go | 12 +++++------ pkg/morph/event/utils.go | 2 +- pkg/services/object/delete/delete.go | 2 +- pkg/services/object/search/container.go | 8 ++++---- pkg/services/object/search/local.go | 2 +- pkg/services/object/search/search.go | 2 +- pkg/services/object/util/log.go | 4 ++-- pkg/services/policer/check.go | 2 +- pkg/services/policer/ec.go | 2 +- pkg/services/policer/process.go | 2 +- pkg/services/replicator/process.go | 2 +- pkg/services/tree/replicator.go | 4 ++-- 56 files changed, 114 insertions(+), 114 deletions(-) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index a8eef6010..dd70fc91c 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -77,7 +77,7 @@ func (c *httpComponent) reload(ctx context.Context) { log.Info(ctx, c.name+" config updated") if err := c.shutdown(ctx); err != nil { log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer, - zap.String("error", err.Error()), + zap.Error(err), ) } else { c.init(ctx) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index e86c04b9e..ade64ba84 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -119,12 +119,12 @@ func shutdown(ctx context.Context) { innerRing.Stop(ctx) if err := metricsCmp.shutdown(ctx); err != nil { log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer, - zap.String("error", err.Error()), + zap.Error(err), ) } if err := pprofCmp.shutdown(ctx); err != nil { log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer, - zap.String("error", err.Error()), + zap.Error(err), ) } diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go index 8e81d8b85..2aebcde7f 100644 --- a/cmd/frostfs-ir/pprof.go +++ b/cmd/frostfs-ir/pprof.go @@ -58,7 +58,7 @@ func (c *pprofComponent) reload(ctx context.Context) { log.Info(ctx, c.name+" config updated") if err := c.shutdown(ctx); err != nil { log.Debug(ctx, logs.FrostFSIRCouldNotShutdownHTTPServer, - zap.String("error", err.Error())) + zap.Error(err)) return } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 9b727e41a..6950c9e24 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1119,7 +1119,7 @@ func initLocalStorage(ctx context.Context, c *cfg) { err := ls.Close(context.WithoutCancel(ctx)) if err != nil { c.log.Info(ctx, logs.FrostFSNodeStorageEngineClosingFailure, - zap.String("error", err.Error()), + zap.Error(err), ) } else { c.log.Info(ctx, logs.FrostFSNodeAllComponentsOfTheStorageEngineClosedSuccessfully) @@ -1209,7 +1209,7 @@ func (c *cfg) updateContractNodeInfo(ctx context.Context, epoch uint64) { if err != nil { c.log.Error(ctx, logs.FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch, zap.Uint64("epoch", epoch), - zap.String("error", err.Error())) + zap.Error(err)) return } diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index f8854ab3c..3c15dc439 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -134,7 +134,7 @@ func stopAndLog(ctx context.Context, c *cfg, name string, stopper func(context.C err := stopper(ctx) if err != nil { c.log.Debug(ctx, fmt.Sprintf("could not shutdown %s server", name), - zap.String("error", err.Error()), + zap.Error(err), ) } diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 0d4e8a1be..5415da12a 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -96,7 +96,7 @@ func initMorphClient(ctx context.Context, c *cfg) { if err != nil { c.log.Info(ctx, logs.FrostFSNodeFailedToCreateNeoRPCClient, zap.Any("endpoints", addresses), - zap.String("error", err.Error()), + zap.Error(err), ) fatalOnErr(err) @@ -168,7 +168,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { fromSideChainBlock, err := c.persistate.UInt32(persistateSideChainLastBlockKey) if err != nil { fromSideChainBlock = 0 - c.log.Warn(ctx, logs.FrostFSNodeCantGetLastProcessedSideChainBlockNumber, zap.String("error", err.Error())) + c.log.Warn(ctx, logs.FrostFSNodeCantGetLastProcessedSideChainBlockNumber, zap.Error(err)) } subs, err = subscriber.New(ctx, &subscriber.Params{ diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index e94428fcb..2eb4cd132 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -197,7 +197,7 @@ func addNewEpochNotificationHandlers(c *cfg) { _, _, err := makeNotaryDeposit(ctx, c) if err != nil { c.log.Error(ctx, logs.FrostFSNodeCouldNotMakeNotaryDeposit, - zap.String("error", err.Error()), + zap.Error(err), ) } }) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 6804aae59..f82a8e533 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -58,7 +58,7 @@ func (c *cfg) MaxObjectSize() uint64 { sz, err := c.cfgNetmap.wrapper.MaxObjectSize() if err != nil { c.log.Error(context.Background(), logs.FrostFSNodeCouldNotGetMaxObjectSizeValue, - zap.String("error", err.Error()), + zap.Error(err), ) } @@ -269,7 +269,7 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl _, err := ls.Inhume(ctx, inhumePrm) if err != nil { c.log.Warn(ctx, logs.FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage, - zap.String("error", err.Error()), + zap.Error(err), ) } }), diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index c423c0660..f3ddc8cbe 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -113,7 +113,7 @@ func initTreeService(c *cfg) { // Ignore pilorama.ErrTreeNotFound but other errors, including shard.ErrReadOnly, should be logged. c.log.Error(ctx, logs.FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved, zap.Stringer("cid", ev.ID), - zap.String("error", err.Error())) + zap.Error(err)) } }) diff --git a/pkg/core/object/sender_classifier.go b/pkg/core/object/sender_classifier.go index 3b3650134..a1a5fcac1 100644 --- a/pkg/core/object/sender_classifier.go +++ b/pkg/core/object/sender_classifier.go @@ -67,7 +67,7 @@ func (c SenderClassifier) IsInnerRingOrContainerNode(ctx context.Context, ownerK if err != nil { // do not throw error, try best case matching c.log.Debug(ctx, logs.V2CantCheckIfRequestFromInnerRing, - zap.String("error", err.Error())) + zap.Error(err)) } else if isInnerRingNode { return &ClassifyResult{ Role: acl.RoleInnerRing, @@ -84,7 +84,7 @@ func (c SenderClassifier) IsInnerRingOrContainerNode(ctx context.Context, ownerK // is not possible for previous epoch, so // do not throw error, try best case matching c.log.Debug(ctx, logs.V2CantCheckIfRequestFromContainerNode, - zap.String("error", err.Error())) + zap.Error(err)) } else if isContainerNode { return &ClassifyResult{ Role: acl.RoleContainer, diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 5481354e1..ecaf8ae86 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -100,7 +100,7 @@ func (s *Server) initMainnet(ctx context.Context, cfg *viper.Viper, morphChain * fromMainChainBlock, err := s.persistate.UInt32(persistateMainChainLastBlockKey) if err != nil { fromMainChainBlock = 0 - s.log.Warn(ctx, logs.InnerringCantGetLastProcessedMainChainBlockNumber, zap.String("error", err.Error())) + s.log.Warn(ctx, logs.InnerringCantGetLastProcessedMainChainBlockNumber, zap.Error(err)) } mainnetChain.from = fromMainChainBlock @@ -456,7 +456,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- fromSideChainBlock, err := s.persistate.UInt32(persistateSideChainLastBlockKey) if err != nil { fromSideChainBlock = 0 - s.log.Warn(ctx, logs.InnerringCantGetLastProcessedSideChainBlockNumber, zap.String("error", err.Error())) + s.log.Warn(ctx, logs.InnerringCantGetLastProcessedSideChainBlockNumber, zap.Error(err)) } morphChain := &chainParams{ diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 4fe9cc084..0b9e83443 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -177,7 +177,7 @@ func (s *Server) Start(ctx context.Context, intError chan<- error) (err error) { if err != nil { // we don't stop inner ring execution on this error s.log.Warn(ctx, logs.InnerringCantVoteForPreparedValidators, - zap.String("error", err.Error())) + zap.Error(err)) } s.tickInitialExpoch(ctx) @@ -308,7 +308,7 @@ func (s *Server) Stop(ctx context.Context) { for _, c := range s.closers { if err := c(); err != nil { s.log.Warn(ctx, logs.InnerringCloserError, - zap.String("error", err.Error()), + zap.Error(err), ) } } diff --git a/pkg/innerring/processors/alphabet/process_emit.go b/pkg/innerring/processors/alphabet/process_emit.go index 229261250..8e11d2d61 100644 --- a/pkg/innerring/processors/alphabet/process_emit.go +++ b/pkg/innerring/processors/alphabet/process_emit.go @@ -33,7 +33,7 @@ func (ap *Processor) processEmit(ctx context.Context) bool { // there is no signature collecting, so we don't need extra fee _, err := ap.morphClient.Invoke(ctx, contract, 0, emitMethod) if err != nil { - ap.log.Warn(ctx, logs.AlphabetCantInvokeAlphabetEmitMethod, zap.String("error", err.Error())) + ap.log.Warn(ctx, logs.AlphabetCantInvokeAlphabetEmitMethod, zap.Error(err)) return false } @@ -47,7 +47,7 @@ func (ap *Processor) processEmit(ctx context.Context) bool { networkMap, err := ap.netmapClient.NetMap() if err != nil { ap.log.Warn(ctx, logs.AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes, - zap.String("error", err.Error())) + zap.Error(err)) return false } @@ -83,7 +83,7 @@ func (ap *Processor) transferGasToNetmapNodes(ctx context.Context, nmNodes []net key, err := keys.NewPublicKeyFromBytes(keyBytes, elliptic.P256()) if err != nil { ap.log.Warn(ctx, logs.AlphabetCantParseNodePublicKey, - zap.String("error", err.Error())) + zap.Error(err)) continue } @@ -93,7 +93,7 @@ func (ap *Processor) transferGasToNetmapNodes(ctx context.Context, nmNodes []net ap.log.Warn(ctx, logs.AlphabetCantTransferGas, zap.String("receiver", key.Address()), zap.Int64("amount", int64(gasPerNode)), - zap.String("error", err.Error()), + zap.Error(err), ) } } @@ -110,7 +110,7 @@ func (ap *Processor) transferGasToExtraNodes(ctx context.Context, pw []util.Uint ap.log.Warn(ctx, logs.AlphabetCantTransferGasToWallet, zap.Strings("receivers", receiversLog), zap.Int64("amount", int64(gasPerNode)), - zap.String("error", err.Error()), + zap.Error(err), ) } } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 16c450166..ffaea653a 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -50,7 +50,7 @@ func (cp *Processor) processContainerPut(ctx context.Context, put putEvent) bool err := cp.checkPutContainer(pctx) if err != nil { cp.log.Error(ctx, logs.ContainerPutContainerCheckFailed, - zap.String("error", err.Error()), + zap.Error(err), ) return false @@ -58,7 +58,7 @@ func (cp *Processor) processContainerPut(ctx context.Context, put putEvent) bool if err := cp.morphClient.NotarySignAndInvokeTX(pctx.e.NotaryRequest().MainTransaction); err != nil { cp.log.Error(ctx, logs.ContainerCouldNotApprovePutContainer, - zap.String("error", err.Error()), + zap.Error(err), ) return false } @@ -113,7 +113,7 @@ func (cp *Processor) processContainerDelete(ctx context.Context, e containerEven err := cp.checkDeleteContainer(e) if err != nil { cp.log.Error(ctx, logs.ContainerDeleteContainerCheckFailed, - zap.String("error", err.Error()), + zap.Error(err), ) return false @@ -121,7 +121,7 @@ func (cp *Processor) processContainerDelete(ctx context.Context, e containerEven if err := cp.morphClient.NotarySignAndInvokeTX(e.NotaryRequest().MainTransaction); err != nil { cp.log.Error(ctx, logs.ContainerCouldNotApproveDeleteContainer, - zap.String("error", err.Error()), + zap.Error(err), ) return false diff --git a/pkg/innerring/processors/frostfs/process_assets.go b/pkg/innerring/processors/frostfs/process_assets.go index ee824ea31..d10eb9660 100644 --- a/pkg/innerring/processors/frostfs/process_assets.go +++ b/pkg/innerring/processors/frostfs/process_assets.go @@ -73,7 +73,7 @@ func (np *Processor) processDeposit(ctx context.Context, deposit frostfsEvent.De err = np.morphClient.TransferGas(receiver, np.mintEmitValue) if err != nil { np.log.Error(ctx, logs.FrostFSCantTransferNativeGasToReceiver, - zap.String("error", err.Error())) + zap.Error(err)) return false } diff --git a/pkg/innerring/processors/governance/process_update.go b/pkg/innerring/processors/governance/process_update.go index 73d21a7d2..245679656 100644 --- a/pkg/innerring/processors/governance/process_update.go +++ b/pkg/innerring/processors/governance/process_update.go @@ -28,21 +28,21 @@ func (gp *Processor) processAlphabetSync(ctx context.Context, txHash util.Uint25 mainnetAlphabet, err := gp.mainnetClient.NeoFSAlphabetList() if err != nil { gp.log.Error(ctx, logs.GovernanceCantFetchAlphabetListFromMainNet, - zap.String("error", err.Error())) + zap.Error(err)) return false } sidechainAlphabet, err := gp.morphClient.Committee() if err != nil { gp.log.Error(ctx, logs.GovernanceCantFetchAlphabetListFromSideChain, - zap.String("error", err.Error())) + zap.Error(err)) return false } newAlphabet, err := newAlphabetList(sidechainAlphabet, mainnetAlphabet) if err != nil { gp.log.Error(ctx, logs.GovernanceCantMergeAlphabetListsFromMainNetAndSideChain, - zap.String("error", err.Error())) + zap.Error(err)) return false } @@ -65,7 +65,7 @@ func (gp *Processor) processAlphabetSync(ctx context.Context, txHash util.Uint25 err = gp.voter.VoteForSidechainValidator(ctx, votePrm) if err != nil { gp.log.Error(ctx, logs.GovernanceCantVoteForSideChainCommittee, - zap.String("error", err.Error())) + zap.Error(err)) } // 2. Update NeoFSAlphabet role in the sidechain. @@ -98,14 +98,14 @@ func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(ctx context.Context, sid innerRing, err := gp.irFetcher.InnerRingKeys() if err != nil { gp.log.Error(ctx, logs.GovernanceCantFetchInnerRingListFromSideChain, - zap.String("error", err.Error())) + zap.Error(err)) return } newInnerRing, err := updateInnerRing(innerRing, sidechainAlphabet, newAlphabet) if err != nil { gp.log.Error(ctx, logs.GovernanceCantCreateNewInnerRingListWithNewAlphabetKeys, - zap.String("error", err.Error())) + zap.Error(err)) return } @@ -122,7 +122,7 @@ func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(ctx context.Context, sid if err = gp.morphClient.UpdateNeoFSAlphabetList(ctx, updPrm); err != nil { gp.log.Error(ctx, logs.GovernanceCantUpdateInnerRingListWithNewAlphabetKeys, - zap.String("error", err.Error())) + zap.Error(err)) } } @@ -135,7 +135,7 @@ func (gp *Processor) updateNotaryRoleInSidechain(ctx context.Context, newAlphabe err := gp.morphClient.UpdateNotaryList(ctx, updPrm) if err != nil { gp.log.Error(ctx, logs.GovernanceCantUpdateListOfNotaryNodesInSideChain, - zap.String("error", err.Error())) + zap.Error(err)) } } @@ -155,6 +155,6 @@ func (gp *Processor) updateFrostFSContractInMainnet(ctx context.Context, newAlph err := gp.frostfsClient.AlphabetUpdate(ctx, prm) if err != nil { gp.log.Error(ctx, logs.GovernanceCantUpdateListOfAlphabetNodesInFrostfsContract, - zap.String("error", err.Error())) + zap.Error(err)) } } diff --git a/pkg/innerring/processors/netmap/process_cleanup.go b/pkg/innerring/processors/netmap/process_cleanup.go index a43005ffb..8f8cc17ff 100644 --- a/pkg/innerring/processors/netmap/process_cleanup.go +++ b/pkg/innerring/processors/netmap/process_cleanup.go @@ -49,7 +49,7 @@ func (np *Processor) processNetmapCleanupTick(ctx context.Context, ev netmapClea }) if err != nil { np.log.Warn(ctx, logs.NetmapCantIterateOnNetmapCleanerCache, - zap.String("error", err.Error())) + zap.Error(err)) return false } diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index 237c4e512..93e00bbaa 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -17,7 +17,7 @@ func (np *Processor) processNewEpoch(ctx context.Context, ev netmapEvent.NewEpoc epochDuration, err := np.netmapClient.EpochDuration() if err != nil { np.log.Warn(ctx, logs.NetmapCantGetEpochDuration, - zap.String("error", err.Error())) + zap.Error(err)) } else { np.epochState.SetEpochDuration(epochDuration) } @@ -28,19 +28,19 @@ func (np *Processor) processNewEpoch(ctx context.Context, ev netmapEvent.NewEpoc if err != nil { np.log.Warn(ctx, logs.NetmapCantGetTransactionHeight, zap.String("hash", ev.TxHash().StringLE()), - zap.String("error", err.Error())) + zap.Error(err)) } if err := np.epochTimer.ResetEpochTimer(h); err != nil { np.log.Warn(ctx, logs.NetmapCantResetEpochTimer, - zap.String("error", err.Error())) + zap.Error(err)) } // get new netmap snapshot networkMap, err := np.netmapClient.NetMap() if err != nil { np.log.Warn(ctx, logs.NetmapCantGetNetmapSnapshotToPerformCleanup, - zap.String("error", err.Error())) + zap.Error(err)) return false } diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index 72aa08f76..5b565ffd1 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -42,7 +42,7 @@ func (np *Processor) processAddPeer(ctx context.Context, ev netmapEvent.AddPeer) err = np.nodeValidator.VerifyAndUpdate(&nodeInfo) if err != nil { np.log.Warn(ctx, logs.NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate, - zap.String("error", err.Error()), + zap.Error(err), ) return false diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 77c2af2ce..3e9880e70 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -62,7 +62,7 @@ func (s *Server) IsAlphabet(ctx context.Context) bool { func (s *Server) InnerRingIndex(ctx context.Context) int { index, err := s.statusIndex.InnerRingIndex() if err != nil { - s.log.Error(ctx, logs.InnerringCantGetInnerRingIndex, zap.String("error", err.Error())) + s.log.Error(ctx, logs.InnerringCantGetInnerRingIndex, zap.Error(err)) return -1 } @@ -74,7 +74,7 @@ func (s *Server) InnerRingIndex(ctx context.Context) int { func (s *Server) InnerRingSize(ctx context.Context) int { size, err := s.statusIndex.InnerRingSize() if err != nil { - s.log.Error(ctx, logs.InnerringCantGetInnerRingSize, zap.String("error", err.Error())) + s.log.Error(ctx, logs.InnerringCantGetInnerRingSize, zap.Error(err)) return 0 } @@ -86,7 +86,7 @@ func (s *Server) InnerRingSize(ctx context.Context) int { func (s *Server) AlphabetIndex(ctx context.Context) int { index, err := s.statusIndex.AlphabetIndex() if err != nil { - s.log.Error(ctx, logs.InnerringCantGetAlphabetIndex, zap.String("error", err.Error())) + s.log.Error(ctx, logs.InnerringCantGetAlphabetIndex, zap.Error(err)) return -1 } @@ -132,7 +132,7 @@ func (s *Server) voteForSidechainValidator(ctx context.Context, prm governance.V s.log.Warn(ctx, logs.InnerringCantInvokeVoteMethodInAlphabetContract, zap.Int8("alphabet_index", int8(letter)), zap.Uint64("epoch", epoch), - zap.String("error", err.Error())) + zap.Error(err)) } }) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 8c2d7aa67..47e12bafb 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -82,7 +82,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co if !client.IsErrObjectNotFound(err) { b.log.Debug(ctx, logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, zap.String("level", p), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index 63d2f21e1..d2c99945f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -57,7 +57,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common if !client.IsErrObjectNotFound(err) { b.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index b7ef8d8a5..1a4f11c29 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -69,7 +69,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G if !client.IsErrObjectNotFound(err) { b.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index b24f1b881..6d06b8e6f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -71,7 +71,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re if !outOfBounds && !client.IsErrObjectNotFound(err) { b.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } if outOfBounds { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index b120c22f7..a710cf988 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -44,7 +44,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm if prm.IgnoreErrors { b.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, zap.Stringer("address", elem.Address()), - zap.String("err", err.Error()), + zap.Error(err), zap.String("storage_id", p), zap.String("root_path", b.rootPath)) return nil @@ -77,7 +77,7 @@ func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors boo if err != nil { if ignoreErrors { b.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, - zap.String("err", err.Error()), + zap.Error(err), zap.String("storage_id", p), zap.String("root_path", b.rootPath)) return false, nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index b35e052cf..7d44aa5c6 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -97,7 +97,7 @@ func (b *sharedDB) Close(ctx context.Context) { if err := b.blcza.Close(ctx); err != nil { b.log.Error(ctx, logs.BlobovniczatreeCouldNotCloseBlobovnicza, zap.String("id", b.path), - zap.String("error", err.Error()), + zap.Error(err), ) } b.blcza = nil @@ -125,7 +125,7 @@ func (b *sharedDB) CloseAndRemoveFile(ctx context.Context) error { if err := b.blcza.Close(ctx); err != nil { b.log.Error(ctx, logs.BlobovniczatreeCouldNotCloseBlobovnicza, zap.String("id", b.path), - zap.String("error", err.Error()), + zap.Error(err), ) return fmt.Errorf("failed to close blobovnicza (path = %s): %w", b.path, err) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 1678e578c..8276a25ef 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -83,7 +83,7 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) i.B.reportError(ctx, logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) } else { i.B.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } @@ -106,7 +106,7 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) } else { i.B.log.Debug(ctx, logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", active.SystemPath()), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } if errors.Is(err, blobovnicza.ErrNoSpace) { diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 44685524f..93316be02 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -74,7 +74,7 @@ func (b *BlobStor) Close(ctx context.Context) error { for i := range b.storage { err := b.storage[i].Storage.Close(ctx) if err != nil { - b.log.Info(ctx, logs.BlobstorCouldntCloseStorage, zap.String("error", err.Error())) + b.log.Info(ctx, logs.BlobstorCouldntCloseStorage, zap.Error(err)) if firstErr == nil { firstErr = err } diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index 556f53e12..f1e45fe10 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -75,7 +75,7 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi for _, err := range errors[:len(errors)-1] { b.log.Warn(ctx, logs.BlobstorErrorOccurredDuringObjectExistenceChecking, zap.Stringer("address", prm.Address), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 53eb0395a..a77ad2f93 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -153,7 +153,7 @@ func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, pr if err != nil { if prm.IgnoreErrors { t.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, - zap.String("err", err.Error()), + zap.Error(err), zap.String("directory_path", dirPath)) return nil } @@ -202,7 +202,7 @@ func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, pr if prm.IgnoreErrors { t.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, zap.Stringer("address", addr), - zap.String("err", err.Error()), + zap.Error(err), zap.String("path", path)) continue } diff --git a/pkg/local_object_storage/blobstor/iterate.go b/pkg/local_object_storage/blobstor/iterate.go index 1ba835a95..ff1aa9d64 100644 --- a/pkg/local_object_storage/blobstor/iterate.go +++ b/pkg/local_object_storage/blobstor/iterate.go @@ -45,7 +45,7 @@ func (b *BlobStor) Iterate(ctx context.Context, prm common.IteratePrm) (common.I b.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, zap.String("storage_path", b.storage[i].Storage.Path()), zap.String("storage_type", b.storage[i].Storage.Type()), - zap.String("err", err.Error())) + zap.Error(err)) continue } return common.IterateRes{}, fmt.Errorf("blobstor iterator failure: %w", err) diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index a5c53dcad..bd9eb1021 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -167,7 +167,7 @@ func (e *StorageEngine) close(ctx context.Context, releasePools bool) error { if err := sh.Close(ctx); err != nil { e.log.Debug(ctx, logs.EngineCouldNotCloseShard, zap.String("id", id), - zap.String("error", err.Error()), + zap.Error(err), ) } } diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 7164ff21f..65ccbdb9e 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -154,7 +154,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo if err != nil { e.log.Warn(ctx, logs.EngineErrorDuringSearchingForObjectChildren, zap.Stringer("addr", addr), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return false } @@ -166,7 +166,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo if err != nil { e.log.Debug(ctx, logs.EngineCouldNotInhumeObjectInShard, zap.Stringer("addr", addr), - zap.String("err", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) continue } @@ -196,7 +196,7 @@ func (e *StorageEngine) deleteChunks( if err != nil { e.log.Debug(ctx, logs.EngineCouldNotInhumeObjectInShard, zap.Stringer("addr", addr), - zap.String("err", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) continue } diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 029904046..f82268d1d 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -140,7 +140,7 @@ func (e *StorageEngine) reportShardError( if isLogical(err) { e.log.Warn(ctx, msg, zap.Stringer("shard_id", sh.ID()), - zap.String("error", err.Error())) + zap.Error(err)) return } @@ -151,7 +151,7 @@ func (e *StorageEngine) reportShardError( e.log.Warn(ctx, msg, append([]zap.Field{ zap.Stringer("shard_id", sid), zap.Uint32("error count", errCount), - zap.String("error", err.Error()), + zap.Error(err), }, fields...)...) if e.errorsThreshold == 0 || errCount < e.errorsThreshold { diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index c7145889b..81b027c26 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -106,7 +106,7 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { if it.ShardWithMeta.Shard != nil && it.MetaError != nil { e.log.Warn(ctx, logs.ShardMetaInfoPresentButObjectNotFound, zap.Stringer("shard_id", it.ShardWithMeta.ID()), - zap.String("error", it.MetaError.Error()), + zap.Error(it.MetaError), zap.Stringer("address", prm.addr), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 62671f433..c79b6e251 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -143,7 +143,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti } else { e.log.Warn(ctx, logs.EngineCouldNotCheckObjectExistence, zap.Stringer("shard_id", sh.ID()), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } @@ -165,14 +165,14 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti errors.Is(err, common.ErrReadOnly) || errors.Is(err, common.ErrNoSpace) { e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, zap.Stringer("shard_id", sh.ID()), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return } if client.IsErrObjectAlreadyRemoved(err) { e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, zap.Stringer("shard_id", sh.ID()), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) res.status = putToShardRemoved res.err = err diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 0c9cea903..600e7266c 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -118,7 +118,7 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error if it.ShardWithMeta.Shard != nil && it.MetaError != nil { e.log.Warn(ctx, logs.ShardMetaInfoPresentButObjectNotFound, zap.Stringer("shard_id", it.ShardWithMeta.ID()), - zap.String("error", it.MetaError.Error()), + zap.Error(it.MetaError), zap.Stringer("address", prm.addr), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 5a9e26155..78ce241fe 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -272,7 +272,7 @@ func (s *Shard) refillObject(ctx context.Context, data []byte, addr oid.Address, if err := obj.Unmarshal(data); err != nil { s.log.Warn(ctx, logs.ShardCouldNotUnmarshalObject, zap.Stringer("address", addr), - zap.String("err", err.Error())) + zap.Error(err)) return nil } diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index f62cecd56..fb6769b51 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -112,7 +112,7 @@ func (s *Shard) deleteFromBlobstor(ctx context.Context, addr oid.Address) error if err != nil { s.log.Debug(ctx, logs.StorageIDRetrievalFailure, zap.Stringer("object", addr), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } @@ -132,7 +132,7 @@ func (s *Shard) deleteFromBlobstor(ctx context.Context, addr oid.Address) error if err != nil && !client.IsErrObjectNotFound(err) { s.log.Debug(ctx, logs.ObjectRemovalFailureBlobStor, zap.Stringer("object_address", addr), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index c212f8c36..1b218a372 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -205,7 +205,7 @@ func (gc *gc) handleEvent(ctx context.Context, event Event) { }) if err != nil { gc.log.Warn(ctx, logs.ShardCouldNotSubmitGCJobToWorkerPool, - zap.String("error", err.Error()), + zap.Error(err), ) v.prevGroup.Done() @@ -313,7 +313,7 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { err := s.metaBase.IterateOverGarbage(ctx, iterPrm) if err != nil { s.log.Warn(ctx, logs.ShardIteratorOverMetabaseGraveyardFailed, - zap.String("error", err.Error()), + zap.Error(err), ) return @@ -334,7 +334,7 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { if err != nil { s.log.Warn(ctx, logs.ShardCouldNotDeleteTheObjects, - zap.String("error", err.Error()), + zap.Error(err), ) result.success = false } @@ -396,7 +396,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { }) if err = errGroup.Wait(); err != nil { - s.log.Warn(ctx, logs.ShardIteratorOverExpiredObjectsFailed, zap.String("error", err.Error())) + s.log.Warn(ctx, logs.ShardIteratorOverExpiredObjectsFailed, zap.Error(err)) } } @@ -429,7 +429,7 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) res, err := s.metaBase.Inhume(ctx, inhumePrm) if err != nil { s.log.Warn(ctx, logs.ShardCouldNotInhumeTheObjects, - zap.String("error", err.Error()), + zap.Error(err), ) return @@ -584,7 +584,7 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { }) if err = errGroup.Wait(); err != nil { - s.log.Warn(ctx, logs.ShardIteratorOverExpiredLocksFailed, zap.String("error", err.Error())) + s.log.Warn(ctx, logs.ShardIteratorOverExpiredLocksFailed, zap.Error(err)) } } @@ -637,7 +637,7 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston res, err := s.metaBase.InhumeTombstones(ctx, tss) if err != nil { s.log.Warn(ctx, logs.ShardCouldNotMarkTombstonesAsGarbage, - zap.String("error", err.Error()), + zap.Error(err), ) return @@ -665,7 +665,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] unlocked, err := s.metaBase.FreeLockedBy(lockers) if err != nil { s.log.Warn(ctx, logs.ShardFailureToUnlockObjects, - zap.String("error", err.Error()), + zap.Error(err), ) return @@ -678,7 +678,7 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] res, err := s.metaBase.Inhume(ctx, pInhume) if err != nil { s.log.Warn(ctx, logs.ShardFailureToMarkLockersAsGarbage, - zap.String("error", err.Error()), + zap.Error(err), ) return @@ -722,7 +722,7 @@ func (s *Shard) HandleDeletedLocks(ctx context.Context, lockers []oid.Address) { _, err := s.metaBase.FreeLockedBy(lockers) if err != nil { s.log.Warn(ctx, logs.ShardFailureToUnlockObjects, - zap.String("error", err.Error()), + zap.Error(err), ) return diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 984c54fbc..d46400869 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -110,7 +110,7 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { } s.log.Debug(ctx, logs.ShardCouldNotMarkObjectToDeleteInMetabase, - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index 7b267d2e4..f583ef5d9 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -124,7 +124,7 @@ func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { if err != nil { s.log.Debug(ctx, logs.ShardCantSelectAllObjects, zap.Stringer("cid", lst[i]), - zap.String("error", err.Error()), + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) continue diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index 50125a88d..1e4643db5 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -76,7 +76,7 @@ func (s *Shard) Put(ctx context.Context, prm PutPrm) (PutRes, error) { if err != nil || !tryCache { if err != nil { s.log.Debug(ctx, logs.ShardCantPutObjectToTheWritecacheTryingBlobstor, - zap.String("err", err.Error())) + zap.Error(err)) } res, err = s.blobStor.Put(ctx, putPrm) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 5a5d24900..37599e696 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -390,7 +390,7 @@ func (c *Client) Wait(ctx context.Context, n uint32) error { height, err = c.rpcActor.GetBlockCount() if err != nil { c.logger.Error(ctx, logs.ClientCantGetBlockchainHeight, - zap.String("error", err.Error())) + zap.Error(err)) return nil } @@ -404,7 +404,7 @@ func (c *Client) Wait(ctx context.Context, n uint32) error { newHeight, err = c.rpcActor.GetBlockCount() if err != nil { c.logger.Error(ctx, logs.ClientCantGetBlockchainHeight243, - zap.String("error", err.Error())) + zap.Error(err)) return nil } diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index ed77352ec..83f8bee07 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -136,7 +136,7 @@ func (l *listener) Listen(ctx context.Context) { defer l.wg.Done() if err := l.listen(ctx, nil); err != nil { l.log.Error(ctx, logs.EventCouldNotStartListenToEvents, - zap.String("error", err.Error()), + zap.Error(err), ) } }) @@ -154,7 +154,7 @@ func (l *listener) ListenWithError(ctx context.Context, intError chan<- error) { defer l.wg.Done() if err := l.listen(ctx, intError); err != nil { l.log.Error(ctx, logs.EventCouldNotStartListenToEvents, - zap.String("error", err.Error()), + zap.Error(err), ) l.sendError(ctx, intError, err) } @@ -342,7 +342,7 @@ func (l *listener) parseAndHandleNotification(ctx context.Context, notifyEvent * event, err := parser(notifyEvent) if err != nil { log.Warn(ctx, logs.EventCouldNotParseNotificationEvent, - zap.String("error", err.Error()), + zap.Error(err), ) return @@ -375,13 +375,13 @@ func (l *listener) parseAndHandleNotary(ctx context.Context, nr *result.NotaryRe case errors.Is(err, ErrTXAlreadyHandled): case errors.As(err, &expErr): l.log.Warn(ctx, logs.EventSkipExpiredMainTXNotaryEvent, - zap.String("error", err.Error()), + zap.Error(err), zap.Uint32("current_block_height", expErr.CurrentBlockHeight), zap.Uint32("fallback_tx_not_valid_before_height", expErr.FallbackTXNotValidBeforeHeight), ) default: l.log.Warn(ctx, logs.EventCouldNotPrepareAndValidateNotaryEvent, - zap.String("error", err.Error()), + zap.Error(err), ) } @@ -413,7 +413,7 @@ func (l *listener) parseAndHandleNotary(ctx context.Context, nr *result.NotaryRe event, err := parser(notaryEvent) if err != nil { log.Warn(ctx, logs.EventCouldNotParseNotaryEvent, - zap.String("error", err.Error()), + zap.Error(err), ) return diff --git a/pkg/morph/event/utils.go b/pkg/morph/event/utils.go index 058959c63..0088be400 100644 --- a/pkg/morph/event/utils.go +++ b/pkg/morph/event/utils.go @@ -77,7 +77,7 @@ func WorkerPoolHandler(w util2.WorkerPool, h Handler, log *logger.Logger) Handle }) if err != nil { log.Warn(ctx, logs.EventCouldNotSubmitHandlerToWorkerPool, - zap.String("error", err.Error()), + zap.Error(err), ) } } diff --git a/pkg/services/object/delete/delete.go b/pkg/services/object/delete/delete.go index 8aaff670c..57e33fde7 100644 --- a/pkg/services/object/delete/delete.go +++ b/pkg/services/object/delete/delete.go @@ -36,7 +36,7 @@ func (exec *execCtx) execute(ctx context.Context) error { exec.log.Debug(ctx, logs.ServingRequest) if err := exec.executeLocal(ctx); err != nil { - exec.log.Debug(ctx, logs.OperationFinishedWithError, zap.String("error", err.Error())) + exec.log.Debug(ctx, logs.OperationFinishedWithError, zap.Error(err)) return err } diff --git a/pkg/services/object/search/container.go b/pkg/services/object/search/container.go index 999a3cc9e..e82f999cf 100644 --- a/pkg/services/object/search/container.go +++ b/pkg/services/object/search/container.go @@ -73,7 +73,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { select { case <-ctx.Done(): exec.log.Debug(ctx, logs.InterruptPlacementIterationByContext, - zap.String("error", ctx.Err().Error())) + zap.Error(ctx.Err())) return default: } @@ -86,14 +86,14 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { c, err := exec.svc.clientConstructor.get(info) if err != nil { - exec.log.Debug(ctx, logs.SearchCouldNotConstructRemoteNodeClient, zap.String("error", err.Error())) + exec.log.Debug(ctx, logs.SearchCouldNotConstructRemoteNodeClient, zap.Error(err)) return } ids, err := c.searchObjects(ctx, exec, info) if err != nil { exec.log.Debug(ctx, logs.SearchRemoteOperationFailed, - zap.String("error", err.Error())) + zap.Error(err)) return } @@ -102,7 +102,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { err = exec.writeIDList(ids) mtx.Unlock() if err != nil { - exec.log.Debug(ctx, logs.SearchCouldNotWriteObjectIdentifiers, zap.String("error", err.Error())) + exec.log.Debug(ctx, logs.SearchCouldNotWriteObjectIdentifiers, zap.Error(err)) return } }(i) diff --git a/pkg/services/object/search/local.go b/pkg/services/object/search/local.go index bc59d0394..ec65ab06a 100644 --- a/pkg/services/object/search/local.go +++ b/pkg/services/object/search/local.go @@ -11,7 +11,7 @@ import ( func (exec *execCtx) executeLocal(ctx context.Context) error { ids, err := exec.svc.localStorage.search(ctx, exec) if err != nil { - exec.log.Debug(ctx, logs.SearchLocalOperationFailed, zap.String("error", err.Error())) + exec.log.Debug(ctx, logs.SearchLocalOperationFailed, zap.Error(err)) return err } diff --git a/pkg/services/object/search/search.go b/pkg/services/object/search/search.go index e24da975d..76c091f85 100644 --- a/pkg/services/object/search/search.go +++ b/pkg/services/object/search/search.go @@ -38,7 +38,7 @@ func (exec *execCtx) execute(ctx context.Context) error { func (exec *execCtx) logResult(ctx context.Context, err error) { switch { default: - exec.log.Debug(ctx, logs.OperationFinishedWithError, zap.String("error", err.Error())) + exec.log.Debug(ctx, logs.OperationFinishedWithError, zap.Error(err)) case err == nil: exec.log.Debug(ctx, logs.OperationFinishedSuccessfully) } diff --git a/pkg/services/object/util/log.go b/pkg/services/object/util/log.go index a9f875d8d..2c1e053ac 100644 --- a/pkg/services/object/util/log.go +++ b/pkg/services/object/util/log.go @@ -14,7 +14,7 @@ func LogServiceError(ctx context.Context, l *logger.Logger, req string, node net l.Error(ctx, logs.UtilObjectServiceError, zap.String("node", network.StringifyGroup(node)), zap.String("request", req), - zap.String("error", err.Error()), + zap.Error(err), ) } @@ -22,6 +22,6 @@ func LogServiceError(ctx context.Context, l *logger.Logger, req string, node net func LogWorkerPoolError(ctx context.Context, l *logger.Logger, req string, err error) { l.Error(ctx, logs.UtilCouldNotPushTaskToWorkerPool, zap.String("request", req), - zap.String("error", err.Error()), + zap.Error(err), ) } diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 2e5e54dfd..bdfc4344b 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -153,7 +153,7 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe } else { p.log.Error(ctx, logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, zap.Stringer("object", addr), - zap.String("error", err.Error()), + zap.Error(err), ) } } diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index db640e323..f6d3b9ea1 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -131,7 +131,7 @@ func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, n // consider maintenance mode has object, but do not drop local copy p.log.Debug(ctx, logs.PolicerConsiderNodeUnderMaintenanceAsOK, zap.String("node", netmap.StringifyPublicKey(requiredNode))) } else { - p.log.Error(ctx, logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, zap.Stringer("object", objInfo.Address), zap.String("error", err.Error())) + p.log.Error(ctx, logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, zap.Stringer("object", objInfo.Address), zap.Error(err)) } return ecChunkProcessResult{ diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index 80a87ade9..bd830d04e 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -61,7 +61,7 @@ func (p *Policer) shardPolicyWorker(ctx context.Context) { if err != nil && !skipMap.addSeenError(addr.Address.Container(), err) { p.log.Error(ctx, logs.PolicerUnableToProcessObj, zap.Stringer("object", addr.Address), - zap.String("error", err.Error())) + zap.Error(err)) } p.cache.Add(addr.Address, time.Now()) p.objsInWork.remove(addr.Address) diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 2120312f6..69395bb02 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -76,7 +76,7 @@ func (p *Replicator) HandleReplicationTask(ctx context.Context, task Task, res T if err != nil { log.Error(ctx, logs.ReplicatorCouldNotReplicateObject, - zap.String("error", err.Error()), + zap.Error(err), ) } else { log.Debug(ctx, logs.ReplicatorObjectSuccessfullyReplicated) diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 0c5bde078..e0085d73a 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -59,7 +59,7 @@ func (s *Service) localReplicationWorker(ctx context.Context) { err := s.forest.TreeApply(ctx, op.cid, op.treeID, &op.Move, false) if err != nil { s.log.Error(ctx, logs.TreeFailedToApplyReplicatedOperation, - zap.String("err", err.Error())) + zap.Error(err)) } span.End() } @@ -155,7 +155,7 @@ func (s *Service) replicateLoop(ctx context.Context) { err := s.replicate(op) if err != nil { s.log.Error(ctx, logs.TreeErrorDuringReplication, - zap.String("err", err.Error()), + zap.Error(err), zap.Stringer("cid", op.cid), zap.String("treeID", op.treeID)) } From 7ac354271462ec6ec59d5d02f2e4cfad0c693572 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 16 Dec 2024 13:59:08 +0300 Subject: [PATCH 1632/1943] [#1563] ape: Introduce `ChainRouterError` error type Signed-off-by: Airat Arifullin --- pkg/services/common/ape/checker.go | 2 +- pkg/services/common/ape/error.go | 33 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 pkg/services/common/ape/error.go diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index 30580da12..86021c3db 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -103,7 +103,7 @@ func (c *checkerCoreImpl) CheckAPE(prm CheckPrm) error { if found && status == apechain.Allow { return nil } - return fmt.Errorf("access to operation %s is denied by access policy engine: %s", prm.Request.Operation(), status.String()) + return newChainRouterError(prm.Request.Operation(), status) } // isValidBearer checks whether bearer token was correctly signed by authorized diff --git a/pkg/services/common/ape/error.go b/pkg/services/common/ape/error.go new file mode 100644 index 000000000..d3c381de7 --- /dev/null +++ b/pkg/services/common/ape/error.go @@ -0,0 +1,33 @@ +package ape + +import ( + "fmt" + + apechain "git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain" +) + +// ChainRouterError is returned when chain router validation prevents +// the APE request from being processed (no rule found, access denied, etc.). +type ChainRouterError struct { + operation string + status apechain.Status +} + +func (e *ChainRouterError) Error() string { + return fmt.Sprintf("access to operation %s is denied by access policy engine: %s", e.Operation(), e.Status()) +} + +func (e *ChainRouterError) Operation() string { + return e.operation +} + +func (e *ChainRouterError) Status() apechain.Status { + return e.status +} + +func newChainRouterError(operation string, status apechain.Status) *ChainRouterError { + return &ChainRouterError{ + operation: operation, + status: status, + } +} From 1a091ea7bbc2c1f01bd3026b7be338261338a4d9 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 16 Dec 2024 14:10:18 +0300 Subject: [PATCH 1633/1943] [#1563] object: Wrap only `ChainRouterError` erros with `ObjectAccessDenied` * Such wrapping helps to differentiate logical check errors and server internal errors. Signed-off-by: Airat Arifullin --- pkg/services/object/ape/errors.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/services/object/ape/errors.go b/pkg/services/object/ape/errors.go index 1b2024ed5..6e458b384 100644 --- a/pkg/services/object/ape/errors.go +++ b/pkg/services/object/ape/errors.go @@ -1,10 +1,19 @@ package ape import ( + "errors" + + checkercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/common/ape" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) func toStatusErr(err error) error { + var chRouterErr *checkercore.ChainRouterError + if !errors.As(err, &chRouterErr) { + errServerInternal := &apistatus.ServerInternal{} + apistatus.WriteInternalServerErr(errServerInternal, err) + return errServerInternal + } errAccessDenied := &apistatus.ObjectAccessDenied{} errAccessDenied.WriteReason("ape denied request: " + err.Error()) return errAccessDenied From 6e82661c3556fa64ee11310258c7b980870a649e Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 16 Dec 2024 14:17:37 +0300 Subject: [PATCH 1634/1943] [#1563] tree: Wrap only `ChainRouterError` erros with `ObjectAccessDenied` * Such wrapping helps to differentiate logical check errors and server internal errors. Signed-off-by: Airat Arifullin --- pkg/services/tree/signature.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 80f5b3590..b0f00615a 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -9,6 +9,7 @@ import ( "fmt" core "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + checkercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -70,6 +71,12 @@ func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, } func apeErr(err error) error { + var chRouterErr *checkercore.ChainRouterError + if !errors.As(err, &chRouterErr) { + errServerInternal := &apistatus.ServerInternal{} + apistatus.WriteInternalServerErr(errServerInternal, err) + return errServerInternal + } errAccessDenied := &apistatus.ObjectAccessDenied{} errAccessDenied.WriteReason(err.Error()) return errAccessDenied From b6c8ebf4930174367a08ff546fb2354162fc6d4f Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Thu, 12 Dec 2024 17:46:30 +0300 Subject: [PATCH 1635/1943] [#1453] container: Replace `sort.Slice` with `slices.SortFunc` * Replaced `sort.Slice` with `slices.SortFunc` in `ListContainersRes.SortedIDList()` as it is a bit faster, according to 15102e6dfd. Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/internal/client/client.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 948d61f36..851cf5ccc 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -9,7 +9,6 @@ import ( "io" "os" "slices" - "sort" "strings" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/accounting" @@ -78,9 +77,8 @@ func ListContainers(ctx context.Context, prm ListContainersPrm) (res ListContain // SortedIDList returns sorted list of identifiers of user's containers. func (x ListContainersRes) SortedIDList() []cid.ID { list := x.cliRes.Containers() - sort.Slice(list, func(i, j int) bool { - lhs, rhs := list[i].EncodeToString(), list[j].EncodeToString() - return strings.Compare(lhs, rhs) < 0 + slices.SortFunc(list, func(lhs, rhs cid.ID) int { + return strings.Compare(lhs.EncodeToString(), rhs.EncodeToString()) }) return list } From df05057ed46632e7746fcaa26731987a9070b2e5 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 28 Oct 2024 18:10:07 +0300 Subject: [PATCH 1636/1943] [#1452] container: Add ListStream method * Added new method for listing containers to container service. It opens stream and sends containers in batches. * Added TransportSplitter wrapper around ExecutionService to split container ID list read from contract in parts that are smaller than grpc max message size. Batch size can be changed in node configuration file (as in example config file). * Changed `container list` implementaion in cli: now ListStream is called by default. Old List is called only if ListStream is not implemented. * Changed `internalclient.ListContainersPrm`.`Account` to `OwnerID` since `client.PrmContainerList`.`Account` was renamed to `OwnerID` in sdk. Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/internal/client/client.go | 19 ++++ cmd/frostfs-cli/modules/container/list.go | 77 ++++++++++------ cmd/frostfs-node/config.go | 7 +- .../config/container/container.go | 27 ++++++ .../config/container/container_test.go | 27 ++++++ cmd/frostfs-node/container.go | 6 +- config/example/node.env | 3 + config/example/node.json | 5 + config/example/node.yaml | 4 + go.mod | 2 +- go.sum | 4 +- .../transport/container/grpc/service.go | 23 +++++ pkg/services/container/ape.go | 73 +++++++++++++++ pkg/services/container/ape_test.go | 5 + pkg/services/container/audit.go | 11 +++ pkg/services/container/executor.go | 9 ++ pkg/services/container/morph/executor.go | 32 +++++++ pkg/services/container/server.go | 8 ++ pkg/services/container/sign.go | 37 ++++++++ pkg/services/container/transport_splitter.go | 92 +++++++++++++++++++ 20 files changed, 435 insertions(+), 36 deletions(-) create mode 100644 cmd/frostfs-node/config/container/container.go create mode 100644 cmd/frostfs-node/config/container/container_test.go create mode 100644 pkg/services/container/transport_splitter.go diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 851cf5ccc..ceae36ae7 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -83,6 +83,25 @@ func (x ListContainersRes) SortedIDList() []cid.ID { return list } +func ListContainersStream(ctx context.Context, prm ListContainersPrm, processCnr func(id cid.ID) bool) (err error) { + cliPrm := &client.PrmContainerListStream{ + XHeaders: prm.XHeaders, + OwnerID: prm.OwnerID, + Session: prm.Session, + } + rdr, err := prm.cli.ContainerListInit(ctx, *cliPrm) + if err != nil { + return fmt.Errorf("init container list: %w", err) + } + + err = rdr.Iterate(processCnr) + if err != nil { + return fmt.Errorf("read container list: %w", err) + } + + return +} + // PutContainerPrm groups parameters of PutContainer operation. type PutContainerPrm struct { Client *client.Client diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index f01e4db4d..bbb8da840 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -6,8 +6,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/spf13/cobra" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // flags of list command. @@ -51,44 +54,60 @@ var listContainersCmd = &cobra.Command{ var prm internalclient.ListContainersPrm prm.SetClient(cli) - prm.Account = idUser - - res, err := internalclient.ListContainers(cmd.Context(), prm) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - + prm.OwnerID = idUser prmGet := internalclient.GetContainerPrm{ Client: cli, } + var containerIDs []cid.ID + + err := internalclient.ListContainersStream(cmd.Context(), prm, func(id cid.ID) bool { + printContainer(cmd, prmGet, id) + return false + }) + if err == nil { + return + } + + if e, ok := status.FromError(err); ok && e.Code() == codes.Unimplemented { + res, err := internalclient.ListContainers(cmd.Context(), prm) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + containerIDs = res.SortedIDList() + } else { + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + } - containerIDs := res.SortedIDList() for _, cnrID := range containerIDs { - if flagVarListName == "" && !flagVarListPrintAttr { - cmd.Println(cnrID.String()) - continue - } - - prmGet.ClientParams.ContainerID = &cnrID - res, err := internalclient.GetContainer(cmd.Context(), prmGet) - if err != nil { - cmd.Printf(" failed to read attributes: %v\n", err) - continue - } - - cnr := res.Container() - if cnrName := containerSDK.Name(cnr); flagVarListName != "" && cnrName != flagVarListName { - continue - } - cmd.Println(cnrID.String()) - - if flagVarListPrintAttr { - cnr.IterateUserAttributes(func(key, val string) { - cmd.Printf(" %s: %s\n", key, val) - }) - } + printContainer(cmd, prmGet, cnrID) } }, } +func printContainer(cmd *cobra.Command, prmGet internalclient.GetContainerPrm, id cid.ID) { + if flagVarListName == "" && !flagVarListPrintAttr { + cmd.Println(id.String()) + return + } + + prmGet.ClientParams.ContainerID = &id + res, err := internalclient.GetContainer(cmd.Context(), prmGet) + if err != nil { + cmd.Printf(" failed to read attributes: %v\n", err) + return + } + + cnr := res.Container() + if cnrName := containerSDK.Name(cnr); flagVarListName != "" && cnrName != flagVarListName { + return + } + cmd.Println(id.String()) + + if flagVarListPrintAttr { + cnr.IterateUserAttributes(func(key, val string) { + cmd.Printf(" %s: %s\n", key, val) + }) + } +} + func initContainerListContainersCmd() { commonflags.Init(listContainersCmd) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 6950c9e24..40be8f45f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -606,9 +606,10 @@ type cfgAccounting struct { type cfgContainer struct { scriptHash neogoutil.Uint160 - parsers map[event.Type]event.NotificationParser - subscribers map[event.Type][]event.Handler - workerPool util.WorkerPool // pool for asynchronous handlers + parsers map[event.Type]event.NotificationParser + subscribers map[event.Type][]event.Handler + workerPool util.WorkerPool // pool for asynchronous handlers + containerBatchSize uint32 } type cfgFrostfsID struct { diff --git a/cmd/frostfs-node/config/container/container.go b/cmd/frostfs-node/config/container/container.go new file mode 100644 index 000000000..b0b8043d6 --- /dev/null +++ b/cmd/frostfs-node/config/container/container.go @@ -0,0 +1,27 @@ +package containerconfig + +import "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + +const ( + subsection = "container" + listStreamSubsection = "list_stream" + + // ContainerBatchSizeDefault represents he maximum amount of containers to send via stream at once. + ContainerBatchSizeDefault = 1000 +) + +// ContainerBatchSize returns the value of "batch_size" config parameter +// from "list_stream" subsection of "container" section. +// +// Returns ContainerBatchSizeDefault if the value is missing or if +// the value is not positive integer. +func ContainerBatchSize(c *config.Config) uint32 { + if c.Sub(subsection).Sub(listStreamSubsection).Value("batch_size") == nil { + return ContainerBatchSizeDefault + } + size := config.Uint32Safe(c.Sub(subsection).Sub(listStreamSubsection), "batch_size") + if size == 0 { + return ContainerBatchSizeDefault + } + return size +} diff --git a/cmd/frostfs-node/config/container/container_test.go b/cmd/frostfs-node/config/container/container_test.go new file mode 100644 index 000000000..744cd3295 --- /dev/null +++ b/cmd/frostfs-node/config/container/container_test.go @@ -0,0 +1,27 @@ +package containerconfig_test + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + containerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/container" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "github.com/stretchr/testify/require" +) + +func TestContainerSection(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + empty := configtest.EmptyConfig() + require.Equal(t, uint32(containerconfig.ContainerBatchSizeDefault), containerconfig.ContainerBatchSize(empty)) + }) + + const path = "../../../../config/example/node" + fileConfigTest := func(c *config.Config) { + require.Equal(t, uint32(500), containerconfig.ContainerBatchSize(c)) + } + + configtest.ForEachFileType(path, fileConfigTest) + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) +} diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 7d558dacb..fb2550a03 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -5,6 +5,7 @@ import ( "context" "net" + containerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/container" morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" @@ -47,6 +48,7 @@ func initContainerService(_ context.Context, c *cfg) { } c.shared.frostfsidClient = frostfsIDSubjectProvider + c.cfgContainer.containerBatchSize = containerconfig.ContainerBatchSize(c.appCfg) defaultChainRouter := engine.NewDefaultChainRouterWithLocalOverrides( c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.MorphRuleChainStorage(), @@ -56,7 +58,9 @@ func initContainerService(_ context.Context, c *cfg) { &c.key.PrivateKey, containerService.NewAPEServer(defaultChainRouter, cnrRdr, newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, c.shared.frostfsidClient, - containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc), + containerService.NewSplitterService( + c.cfgContainer.containerBatchSize, c.respSvc, + containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc)), ), ) service = containerService.NewAuditService(service, c.log, c.audit) diff --git a/config/example/node.env b/config/example/node.env index f470acf3e..b2a0633a9 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -83,6 +83,9 @@ FROSTFS_POLICER_HEAD_TIMEOUT=15s FROSTFS_REPLICATOR_PUT_TIMEOUT=15s FROSTFS_REPLICATOR_POOL_SIZE=10 +# Container service section +FROSTFS_CONTAINER_LIST_STREAM_BATCH_SIZE=500 + # Object service section FROSTFS_OBJECT_PUT_REMOTE_POOL_SIZE=100 FROSTFS_OBJECT_PUT_LOCAL_POOL_SIZE=200 diff --git a/config/example/node.json b/config/example/node.json index dba3bad8b..f3192ac2f 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -124,6 +124,11 @@ "pool_size": 10, "put_timeout": "15s" }, + "container": { + "list_stream": { + "batch_size": "500" + } + }, "object": { "delete": { "tombstone_lifetime": 10 diff --git a/config/example/node.yaml b/config/example/node.yaml index 8f9300b4a..a179b4704 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -108,6 +108,10 @@ replicator: put_timeout: 15s # timeout for the Replicator PUT remote operation pool_size: 10 # maximum amount of concurrent replications +container: + list_stream: + batch_size: 500 # container_batch_size is the maximum amount of containers to send via stream at once + object: delete: tombstone_lifetime: 10 # tombstone "local" lifetime in epochs diff --git a/go.mod b/go.mod index 6ac37d343..8f4053872 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241210104938-c4463df8d467 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 diff --git a/go.sum b/go.sum index e084c2445..d63396202 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d h1:FpXI+mOrmJk3t2MKQFZuhLjCHDyDeo5rtP1WXl7gUWc= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241206094944-81c423e7094d/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241210104938-c4463df8d467 h1:MH9uHZFZNyUCL+YChiDcVeXPjhTDcFDeoGr8Mc8NY9M= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241210104938-c4463df8d467/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= diff --git a/pkg/network/transport/container/grpc/service.go b/pkg/network/transport/container/grpc/service.go index 49d083a90..8cbf8d9c3 100644 --- a/pkg/network/transport/container/grpc/service.go +++ b/pkg/network/transport/container/grpc/service.go @@ -80,3 +80,26 @@ func (s *Server) List(ctx context.Context, req *containerGRPC.ListRequest) (*con return resp.ToGRPCMessage().(*containerGRPC.ListResponse), nil } + +type containerStreamerV2 struct { + containerGRPC.ContainerService_ListStreamServer +} + +func (s *containerStreamerV2) Send(resp *container.ListStreamResponse) error { + return s.ContainerService_ListStreamServer.Send( + resp.ToGRPCMessage().(*containerGRPC.ListStreamResponse), + ) +} + +// ListStream converts gRPC ListRequest message and server-side stream and overtakes its data +// to gRPC stream. +func (s *Server) ListStream(req *containerGRPC.ListStreamRequest, gStream containerGRPC.ContainerService_ListStreamServer) error { + listReq := new(container.ListStreamRequest) + if err := listReq.FromGRPCMessage(req); err != nil { + return err + } + + return s.srv.ListStream(listReq, &containerStreamerV2{ + ContainerService_ListStreamServer: gStream, + }) +} diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 2cdb30b45..493452fa6 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -175,6 +175,79 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co return nil, apeErr(nativeschema.MethodListContainers, s) } +func (ac *apeChecker) ListStream(req *container.ListStreamRequest, stream ListStream) error { + ctx, span := tracing.StartSpanFromContext(stream.Context(), "apeChecker.ListStream") + defer span.End() + + role, pk, err := ac.getRoleWithoutContainerID(req.GetBody().GetOwnerID(), req.GetMetaHeader(), req.GetVerificationHeader()) + if err != nil { + return err + } + + reqProps := map[string]string{ + nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(pk.Bytes()), + nativeschema.PropertyKeyActorRole: role, + } + + reqProps, err = ac.fillWithUserClaimTags(reqProps, pk) + if err != nil { + return err + } + if p, ok := peer.FromContext(ctx); ok { + if tcpAddr, ok := p.Addr.(*net.TCPAddr); ok { + reqProps[commonschema.PropertyKeyFrostFSSourceIP] = tcpAddr.IP.String() + } + } + + namespace, err := ac.namespaceByOwner(req.GetBody().GetOwnerID()) + if err != nil { + return fmt.Errorf("could not get owner namespace: %w", err) + } + if err := ac.validateNamespaceByPublicKey(pk, namespace); err != nil { + return err + } + + request := aperequest.NewRequest( + nativeschema.MethodListContainers, + aperequest.NewResource( + resourceName(namespace, ""), + make(map[string]string), + ), + reqProps, + ) + + groups, err := aperequest.Groups(ac.frostFSIDClient, pk) + if err != nil { + return fmt.Errorf("failed to get group ids: %w", err) + } + + // Policy contract keeps group related chains as namespace-group pair. + for i := range groups { + groups[i] = fmt.Sprintf("%s:%s", namespace, groups[i]) + } + + rt := policyengine.NewRequestTargetWithNamespace(namespace) + rt.User = &policyengine.Target{ + Type: policyengine.User, + Name: fmt.Sprintf("%s:%s", namespace, pk.Address()), + } + rt.Groups = make([]policyengine.Target, len(groups)) + for i := range groups { + rt.Groups[i] = policyengine.GroupTarget(groups[i]) + } + + s, found, err := ac.router.IsAllowed(apechain.Ingress, rt, request) + if err != nil { + return err + } + + if found && s == apechain.Allow { + return ac.next.ListStream(req, stream) + } + + return apeErr(nativeschema.MethodListContainers, s) +} + func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.Put") defer span.End() diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index b6b42a559..513ffff02 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -1079,6 +1079,11 @@ func (s *srvStub) List(context.Context, *container.ListRequest) (*container.List return &container.ListResponse{}, nil } +func (s *srvStub) ListStream(*container.ListStreamRequest, ListStream) error { + s.calls["ListStream"]++ + return nil +} + func (s *srvStub) Put(context.Context, *container.PutRequest) (*container.PutResponse, error) { s.calls["Put"]++ return &container.PutResponse{}, nil diff --git a/pkg/services/container/audit.go b/pkg/services/container/audit.go index 411eb4863..b235efa3c 100644 --- a/pkg/services/container/audit.go +++ b/pkg/services/container/audit.go @@ -63,6 +63,17 @@ func (a *auditService) List(ctx context.Context, req *container.ListRequest) (*c return res, err } +// ListStream implements Server. +func (a *auditService) ListStream(req *container.ListStreamRequest, stream ListStream) error { + err := a.next.ListStream(req, stream) + if !a.enabled.Load() { + return err + } + audit.LogRequest(stream.Context(), a.log, container_grpc.ContainerService_ListStream_FullMethodName, req, + audit.TargetFromRef(req.GetBody().GetOwnerID(), &user.ID{}), err == nil) + return err +} + // Put implements Server. func (a *auditService) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { res, err := a.next.Put(ctx, req) diff --git a/pkg/services/container/executor.go b/pkg/services/container/executor.go index 70234d3de..cdd0d2514 100644 --- a/pkg/services/container/executor.go +++ b/pkg/services/container/executor.go @@ -14,6 +14,7 @@ type ServiceExecutor interface { Delete(context.Context, *session.Token, *container.DeleteRequestBody) (*container.DeleteResponseBody, error) Get(context.Context, *container.GetRequestBody) (*container.GetResponseBody, error) List(context.Context, *container.ListRequestBody) (*container.ListResponseBody, error) + ListStream(context.Context, *container.ListStreamRequest, ListStream) error } type executorSvc struct { @@ -93,3 +94,11 @@ func (s *executorSvc) List(ctx context.Context, req *container.ListRequest) (*co s.respSvc.SetMeta(resp) return resp, nil } + +func (s *executorSvc) ListStream(req *container.ListStreamRequest, stream ListStream) error { + err := s.exec.ListStream(stream.Context(), req, stream) + if err != nil { + return fmt.Errorf("could not execute ListStream request: %w", err) + } + return nil +} diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index 211f469f3..e9d1606f1 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -200,3 +200,35 @@ func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) return res, nil } + +func (s *morphExecutor) ListStream(_ context.Context, req *container.ListStreamRequest, stream containerSvc.ListStream) error { + body := req.GetBody() + idV2 := body.GetOwnerID() + if idV2 == nil { + return errMissingUserID + } + + var id user.ID + + err := id.ReadFromV2(*idV2) + if err != nil { + return fmt.Errorf("invalid user ID: %w", err) + } + + cnrs, err := s.rdr.ContainersOf(&id) + if err != nil { + return err + } + + cidList := make([]refs.ContainerID, len(cnrs)) + for i := range cnrs { + cnrs[i].WriteToV2(&cidList[i]) + } + + resBody := new(container.ListStreamResponseBody) + resBody.SetContainerIDs(cidList) + r := new(container.ListStreamResponse) + r.SetBody(resBody) + + return stream.Send(r) +} diff --git a/pkg/services/container/server.go b/pkg/services/container/server.go index 78fd3d34c..d9208077d 100644 --- a/pkg/services/container/server.go +++ b/pkg/services/container/server.go @@ -3,6 +3,7 @@ package container import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" ) @@ -12,4 +13,11 @@ type Server interface { Get(context.Context, *container.GetRequest) (*container.GetResponse, error) Delete(context.Context, *container.DeleteRequest) (*container.DeleteResponse, error) List(context.Context, *container.ListRequest) (*container.ListResponse, error) + ListStream(*container.ListStreamRequest, ListStream) error +} + +// ListStream is an interface of FrostFS API v2 compatible search streamer. +type ListStream interface { + util.ServerStream + Send(*container.ListStreamResponse) error } diff --git a/pkg/services/container/sign.go b/pkg/services/container/sign.go index c478c0e1c..85fe7ae87 100644 --- a/pkg/services/container/sign.go +++ b/pkg/services/container/sign.go @@ -56,3 +56,40 @@ func (s *signService) List(ctx context.Context, req *container.ListRequest) (*co resp, err := util.EnsureNonNilResponse(s.svc.List(ctx, req)) return resp, s.sigSvc.SignResponse(resp, err) } + +func (s *signService) ListStream(req *container.ListStreamRequest, stream ListStream) error { + if err := s.sigSvc.VerifyRequest(req); err != nil { + resp := new(container.ListStreamResponse) + _ = s.sigSvc.SignResponse(resp, err) + return stream.Send(resp) + } + + ss := &listStreamSigner{ + ListStream: stream, + sigSvc: s.sigSvc, + } + err := s.svc.ListStream(req, ss) + if err != nil || !ss.nonEmptyResp { + return ss.send(new(container.ListStreamResponse), err) + } + return nil +} + +type listStreamSigner struct { + ListStream + sigSvc *util.SignService + + nonEmptyResp bool // set on first Send call +} + +func (s *listStreamSigner) Send(resp *container.ListStreamResponse) error { + s.nonEmptyResp = true + return s.send(resp, nil) +} + +func (s *listStreamSigner) send(resp *container.ListStreamResponse, err error) error { + if err := s.sigSvc.SignResponse(resp, err); err != nil { + return err + } + return s.ListStream.Send(resp) +} diff --git a/pkg/services/container/transport_splitter.go b/pkg/services/container/transport_splitter.go new file mode 100644 index 000000000..4f8708da7 --- /dev/null +++ b/pkg/services/container/transport_splitter.go @@ -0,0 +1,92 @@ +package container + +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container" +) + +type ( + TransportSplitter struct { + next Server + + respSvc *response.Service + cnrAmount uint32 + } + + listStreamMsgSizeCtrl struct { + util.ServerStream + stream ListStream + respSvc *response.Service + cnrAmount uint32 + } +) + +func NewSplitterService(cnrAmount uint32, respSvc *response.Service, next Server) Server { + return &TransportSplitter{ + next: next, + respSvc: respSvc, + cnrAmount: cnrAmount, + } +} + +func (s *TransportSplitter) Put(ctx context.Context, req *container.PutRequest) (*container.PutResponse, error) { + return s.next.Put(ctx, req) +} + +func (s *TransportSplitter) Delete(ctx context.Context, req *container.DeleteRequest) (*container.DeleteResponse, error) { + return s.next.Delete(ctx, req) +} + +func (s *TransportSplitter) Get(ctx context.Context, req *container.GetRequest) (*container.GetResponse, error) { + return s.next.Get(ctx, req) +} + +func (s *TransportSplitter) List(ctx context.Context, req *container.ListRequest) (*container.ListResponse, error) { + return s.next.List(ctx, req) +} + +func (s *TransportSplitter) ListStream(req *container.ListStreamRequest, stream ListStream) error { + return s.next.ListStream(req, &listStreamMsgSizeCtrl{ + ServerStream: stream, + stream: stream, + respSvc: s.respSvc, + cnrAmount: s.cnrAmount, + }) +} + +func (s *listStreamMsgSizeCtrl) Send(resp *container.ListStreamResponse) error { + s.respSvc.SetMeta(resp) + body := resp.GetBody() + ids := body.GetContainerIDs() + + var newResp *container.ListStreamResponse + + for { + if newResp == nil { + newResp = new(container.ListStreamResponse) + newResp.SetBody(body) + } + + cut := min(s.cnrAmount, uint32(len(ids))) + + body.SetContainerIDs(ids[:cut]) + newResp.SetMetaHeader(resp.GetMetaHeader()) + newResp.SetVerificationHeader(resp.GetVerificationHeader()) + + if err := s.stream.Send(newResp); err != nil { + return fmt.Errorf("TransportSplitter: %w", err) + } + + ids = ids[cut:] + + if len(ids) == 0 { + break + } + } + + return nil +} From bed49e6ace1b975e0c0d4a53d82540ad35e90bf3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 18 Dec 2024 10:01:19 +0300 Subject: [PATCH 1637/1943] [#1569] cli: Make `--range` flag required in `object hash` Previously, `object head` was used if no range was provided. This is wrong on multiple levels: 1. We print an error if the checksum is missing in header, even though taking hash is possible. 2. We silently ignore --salt parameter. 3. `--range` is required for Object.RANGEHASH RPC, custom logic for one specific usecase has no value. So we make it required and make CLI command follow more closely the FrostFS API. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/object/hash.go | 31 ++------------------------ 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/hash.go b/cmd/frostfs-cli/modules/object/hash.go index 26243e7e7..b18fab5e0 100644 --- a/cmd/frostfs-cli/modules/object/hash.go +++ b/cmd/frostfs-cli/modules/object/hash.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/spf13/cobra" @@ -43,6 +42,8 @@ func initObjectHashCmd() { _ = objectHashCmd.MarkFlagRequired(commonflags.OIDFlag) flags.String("range", "", "Range to take hash from in the form offset1:length1,...") + _ = objectHashCmd.MarkFlagRequired("range") + flags.String("type", hashSha256, "Hash type. Either 'sha256' or 'tz'") flags.String(getRangeHashSaltFlag, "", "Salt in hex format") } @@ -67,34 +68,6 @@ func getObjectHash(cmd *cobra.Command, _ []string) { cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) tz := typ == hashTz - fullHash := len(ranges) == 0 - if fullHash { - var headPrm internalclient.HeadObjectPrm - headPrm.SetClient(cli) - Prepare(cmd, &headPrm) - headPrm.SetAddress(objAddr) - - // get hash of full payload through HEAD (may be user can do it through dedicated command?) - res, err := internalclient.HeadObject(cmd.Context(), headPrm) - commonCmd.ExitOnErr(cmd, "rpc error: %w", err) - - var cs checksum.Checksum - var csSet bool - - if tz { - cs, csSet = res.Header().PayloadHomomorphicHash() - } else { - cs, csSet = res.Header().PayloadChecksum() - } - - if csSet { - cmd.Println(hex.EncodeToString(cs.Value())) - } else { - cmd.Println("Missing checksum in object header.") - } - - return - } var hashPrm internalclient.HashPayloadRangesPrm hashPrm.SetClient(cli) From e44b84c18c428a691b36ec100a41cfa100e4ad8a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 18 Dec 2024 10:07:22 +0300 Subject: [PATCH 1638/1943] [#1569] cli: Remove unnecessary variable after refactoring Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/object/hash.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/hash.go b/cmd/frostfs-cli/modules/object/hash.go index b18fab5e0..461c35f30 100644 --- a/cmd/frostfs-cli/modules/object/hash.go +++ b/cmd/frostfs-cli/modules/object/hash.go @@ -67,8 +67,6 @@ func getObjectHash(cmd *cobra.Command, _ []string) { pk := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - tz := typ == hashTz - var hashPrm internalclient.HashPayloadRangesPrm hashPrm.SetClient(cli) Prepare(cmd, &hashPrm) @@ -77,7 +75,7 @@ func getObjectHash(cmd *cobra.Command, _ []string) { hashPrm.SetSalt(salt) hashPrm.SetRanges(ranges) - if tz { + if typ == hashTz { hashPrm.TZ() } From bd0197eaa8579ac4aa54bc2ebc7b38a610a38aa2 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 18 Dec 2024 09:38:14 +0300 Subject: [PATCH 1639/1943] [#1568] storage: Remove "could not/can't/failed to" from error messages Signed-off-by: Evgenii Stratonikov --- .../blobovnicza/control.go | 4 ++-- .../blobovnicza/iterate.go | 2 +- .../blobstor/blobovniczatree/get.go | 4 ++-- .../blobstor/blobovniczatree/get_range.go | 4 ++-- .../blobstor/blobovniczatree/iterate.go | 4 ++-- .../blobstor/blobovniczatree/manager.go | 6 ++--- .../blobstor/fstree/fstree.go | 4 ++-- .../blobstor/memstore/memstore.go | 4 ++-- pkg/local_object_storage/blobstor/mode.go | 2 +- pkg/local_object_storage/blobstor/put.go | 2 +- pkg/local_object_storage/engine/control.go | 10 ++++---- pkg/local_object_storage/engine/evacuate.go | 2 +- pkg/local_object_storage/engine/shards.go | 12 +++++----- pkg/local_object_storage/metabase/control.go | 12 +++++----- pkg/local_object_storage/metabase/counter.go | 22 ++++++++--------- pkg/local_object_storage/metabase/delete.go | 24 +++++++++---------- pkg/local_object_storage/metabase/exists.go | 2 +- pkg/local_object_storage/metabase/get.go | 2 +- .../metabase/graveyard.go | 6 ++--- pkg/local_object_storage/metabase/inhume.go | 2 +- pkg/local_object_storage/metabase/mode.go | 4 ++-- pkg/local_object_storage/metabase/put.go | 20 ++++++++-------- pkg/local_object_storage/metabase/select.go | 2 +- pkg/local_object_storage/metabase/shard_id.go | 8 +++---- pkg/local_object_storage/metabase/upgrade.go | 8 +++---- pkg/local_object_storage/metabase/util.go | 4 ++-- pkg/local_object_storage/metabase/version.go | 2 +- pkg/local_object_storage/pilorama/boltdb.go | 10 ++++---- pkg/local_object_storage/shard/container.go | 4 ++-- pkg/local_object_storage/shard/control.go | 22 ++++++++--------- pkg/local_object_storage/shard/get.go | 2 +- pkg/local_object_storage/shard/id.go | 4 ++-- pkg/local_object_storage/shard/list.go | 12 +++++----- pkg/local_object_storage/shard/put.go | 4 ++-- pkg/local_object_storage/shard/select.go | 2 +- .../writecache/iterate.go | 2 +- pkg/local_object_storage/writecache/mode.go | 8 +++---- .../writecache/storage.go | 4 ++-- .../writecache/upgrade.go | 4 ++-- 39 files changed, 128 insertions(+), 128 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/control.go b/pkg/local_object_storage/blobovnicza/control.go index d0e71a876..4947512cc 100644 --- a/pkg/local_object_storage/blobovnicza/control.go +++ b/pkg/local_object_storage/blobovnicza/control.go @@ -129,7 +129,7 @@ func (b *Blobovnicza) initializeCounters(ctx context.Context) error { }) }) if err != nil { - return fmt.Errorf("can't determine DB size: %w", err) + return fmt.Errorf("determine DB size: %w", err) } if (!sizeExists || !itemsCountExists) && !b.boltOptions.ReadOnly { b.log.Debug(ctx, logs.BlobovniczaSavingCountersToMeta, zap.Uint64("size", size), zap.Uint64("items", items)) @@ -140,7 +140,7 @@ func (b *Blobovnicza) initializeCounters(ctx context.Context) error { return saveItemsCount(tx, items) }); err != nil { b.log.Debug(ctx, logs.BlobovniczaSavingCountersToMetaFailed, zap.Uint64("size", size), zap.Uint64("items", items)) - return fmt.Errorf("can't save blobovnicza's size and items count: %w", err) + return fmt.Errorf("save blobovnicza's size and items count: %w", err) } b.log.Debug(ctx, logs.BlobovniczaSavingCountersToMetaSuccess, zap.Uint64("size", size), zap.Uint64("items", items)) } diff --git a/pkg/local_object_storage/blobovnicza/iterate.go b/pkg/local_object_storage/blobovnicza/iterate.go index 01e5529da..cd33b263c 100644 --- a/pkg/local_object_storage/blobovnicza/iterate.go +++ b/pkg/local_object_storage/blobovnicza/iterate.go @@ -146,7 +146,7 @@ func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes, if prm.ignoreErrors { return nil } - return fmt.Errorf("could not decode address key: %w", err) + return fmt.Errorf("decode address key: %w", err) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 1a4f11c29..5d158644e 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -115,13 +115,13 @@ func (b *Blobovniczas) getObject(ctx context.Context, blz *blobovnicza.Blobovnic // decompress the data data, err := b.compression.Decompress(res.Object()) if err != nil { - return common.GetRes{}, fmt.Errorf("could not decompress object data: %w", err) + return common.GetRes{}, fmt.Errorf("decompress object data: %w", err) } // unmarshal the object obj := objectSDK.New() if err := obj.Unmarshal(data); err != nil { - return common.GetRes{}, fmt.Errorf("could not unmarshal the object: %w", err) + return common.GetRes{}, fmt.Errorf("unmarshal the object: %w", err) } return common.GetRes{Object: obj, RawData: data}, nil diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 6d06b8e6f..84b9bc55f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -130,13 +130,13 @@ func (b *Blobovniczas) getObjectRange(ctx context.Context, blz *blobovnicza.Blob // decompress the data data, err := b.compression.Decompress(res.Object()) if err != nil { - return common.GetRangeRes{}, fmt.Errorf("could not decompress object data: %w", err) + return common.GetRangeRes{}, fmt.Errorf("decompress object data: %w", err) } // unmarshal the object obj := objectSDK.New() if err := obj.Unmarshal(data); err != nil { - return common.GetRangeRes{}, fmt.Errorf("could not unmarshal the object: %w", err) + return common.GetRangeRes{}, fmt.Errorf("unmarshal the object: %w", err) } from := prm.Range.GetOffset() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index a710cf988..5c2d58ca1 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -49,7 +49,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm zap.String("root_path", b.rootPath)) return nil } - return fmt.Errorf("could not decompress object data: %w", err) + return fmt.Errorf("decompress object data: %w", err) } if prm.Handler != nil { @@ -82,7 +82,7 @@ func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors boo zap.String("root_path", b.rootPath)) return false, nil } - return false, fmt.Errorf("could not open blobovnicza %s: %w", p, err) + return false, fmt.Errorf("open blobovnicza %s: %w", p, err) } defer shBlz.Close(ctx) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index 7d44aa5c6..f2f9509ad 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -69,10 +69,10 @@ func (b *sharedDB) Open(ctx context.Context) (*blobovnicza.Blobovnicza, error) { )...) if err := blz.Open(ctx); err != nil { - return nil, fmt.Errorf("could not open blobovnicza %s: %w", b.path, err) + return nil, fmt.Errorf("open blobovnicza %s: %w", b.path, err) } if err := blz.Init(ctx); err != nil { - return nil, fmt.Errorf("could not init blobovnicza %s: %w", b.path, err) + return nil, fmt.Errorf("init blobovnicza %s: %w", b.path, err) } b.refCount++ @@ -127,7 +127,7 @@ func (b *sharedDB) CloseAndRemoveFile(ctx context.Context) error { zap.String("id", b.path), zap.Error(err), ) - return fmt.Errorf("failed to close blobovnicza (path = %s): %w", b.path, err) + return fmt.Errorf("close blobovnicza (path = %s): %w", b.path, err) } b.refCount = 0 diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index a77ad2f93..031b385b2 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -538,7 +538,7 @@ func (t *FSTree) countFiles() (uint64, uint64, error) { }, ) if err != nil { - return 0, 0, fmt.Errorf("could not walk through %s directory: %w", t.RootPath, err) + return 0, 0, fmt.Errorf("walk through %s directory: %w", t.RootPath, err) } return count, size, nil @@ -577,7 +577,7 @@ func (t *FSTree) ObjectsCount(ctx context.Context) (uint64, error) { }, ) if err != nil { - return 0, fmt.Errorf("could not walk through %s directory: %w", t.RootPath, err) + return 0, fmt.Errorf("walk through %s directory: %w", t.RootPath, err) } success = true return result, nil diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index 0252c7983..3afef7d18 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -47,13 +47,13 @@ func (s *memstoreImpl) Get(_ context.Context, req common.GetPrm) (common.GetRes, // Decompress the data. var err error if data, err = s.compression.Decompress(data); err != nil { - return common.GetRes{}, fmt.Errorf("could not decompress object data: %w", err) + return common.GetRes{}, fmt.Errorf("decompress object data: %w", err) } // Unmarshal the SDK object. obj := objectSDK.New() if err := obj.Unmarshal(data); err != nil { - return common.GetRes{}, fmt.Errorf("could not unmarshal the object: %w", err) + return common.GetRes{}, fmt.Errorf("unmarshal the object: %w", err) } return common.GetRes{Object: obj, RawData: data}, nil diff --git a/pkg/local_object_storage/blobstor/mode.go b/pkg/local_object_storage/blobstor/mode.go index af19e398e..80268fa7a 100644 --- a/pkg/local_object_storage/blobstor/mode.go +++ b/pkg/local_object_storage/blobstor/mode.go @@ -27,7 +27,7 @@ func (b *BlobStor) SetMode(ctx context.Context, m mode.Mode) error { } } if err != nil { - return fmt.Errorf("can't set blobstor mode (old=%s, new=%s): %w", b.mode, m, err) + return fmt.Errorf("set blobstor mode (old=%s, new=%s): %w", b.mode, m, err) } b.mode = m diff --git a/pkg/local_object_storage/blobstor/put.go b/pkg/local_object_storage/blobstor/put.go index 342da28bf..fe9c109dd 100644 --- a/pkg/local_object_storage/blobstor/put.go +++ b/pkg/local_object_storage/blobstor/put.go @@ -52,7 +52,7 @@ func (b *BlobStor) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, e // marshal object data, err := prm.Object.Marshal() if err != nil { - return common.PutRes{}, fmt.Errorf("could not marshal the object: %w", err) + return common.PutRes{}, fmt.Errorf("marshal the object: %w", err) } prm.RawData = data } diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index bd9eb1021..6a416cfd9 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -95,7 +95,7 @@ func (e *StorageEngine) Init(ctx context.Context) error { err := eg.Wait() close(errCh) if err != nil { - return fmt.Errorf("failed to initialize shards: %w", err) + return fmt.Errorf("initialize shards: %w", err) } for res := range errCh { @@ -117,7 +117,7 @@ func (e *StorageEngine) Init(ctx context.Context) error { continue } - return fmt.Errorf("could not initialize shard %s: %w", res.id, res.err) + return fmt.Errorf("initialize shard %s: %w", res.id, res.err) } } @@ -320,7 +320,7 @@ loop: for _, newID := range shardsToAdd { sh, err := e.createShard(ctx, rcfg.shards[newID]) if err != nil { - return fmt.Errorf("could not add new shard with '%s' metabase path: %w", newID, err) + return fmt.Errorf("add new shard with '%s' metabase path: %w", newID, err) } idStr := sh.ID().String() @@ -331,13 +331,13 @@ loop: } if err != nil { _ = sh.Close(ctx) - return fmt.Errorf("could not init %s shard: %w", idStr, err) + return fmt.Errorf("init %s shard: %w", idStr, err) } err = e.addShard(sh) if err != nil { _ = sh.Close(ctx) - return fmt.Errorf("could not add %s shard: %w", idStr, err) + return fmt.Errorf("add %s shard: %w", idStr, err) } e.log.Info(ctx, logs.EngineAddedNewShard, zap.String("id", idStr)) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 2e0344bfb..623f5c941 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -578,7 +578,7 @@ func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, tree func (e *StorageEngine) evacuateTreeToOtherNode(ctx context.Context, sh *shard.Shard, tree pilorama.ContainerIDTreeID, prm EvacuateShardPrm) (bool, string, error) { if prm.TreeHandler == nil { - return false, "", fmt.Errorf("failed to evacuate tree '%s' for container %s from shard %s: local evacuation failed, but no remote evacuation available", tree.TreeID, tree.CID, sh.ID()) + return false, "", fmt.Errorf("evacuate tree '%s' for container %s from shard %s: local evacuation failed, but no remote evacuation available", tree.TreeID, tree.CID, sh.ID()) } return prm.TreeHandler(ctx, tree.CID, tree.TreeID, sh) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 898f685ec..6d4844b75 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -108,12 +108,12 @@ func (m *metricsWithID) SetEvacuationInProgress(value bool) { func (e *StorageEngine) AddShard(ctx context.Context, opts ...shard.Option) (*shard.ID, error) { sh, err := e.createShard(ctx, opts) if err != nil { - return nil, fmt.Errorf("could not create a shard: %w", err) + return nil, fmt.Errorf("create a shard: %w", err) } err = e.addShard(sh) if err != nil { - return nil, fmt.Errorf("could not add %s shard: %w", sh.ID().String(), err) + return nil, fmt.Errorf("add %s shard: %w", sh.ID().String(), err) } e.cfg.metrics.SetMode(sh.ID().String(), sh.GetMode()) @@ -124,7 +124,7 @@ func (e *StorageEngine) AddShard(ctx context.Context, opts ...shard.Option) (*sh func (e *StorageEngine) createShard(ctx context.Context, opts []shard.Option) (*shard.Shard, error) { id, err := generateShardID() if err != nil { - return nil, fmt.Errorf("could not generate shard ID: %w", err) + return nil, fmt.Errorf("generate shard ID: %w", err) } opts = e.appendMetrics(id, opts) @@ -180,7 +180,7 @@ func (e *StorageEngine) addShard(sh *shard.Shard) error { pool, err := ants.NewPool(int(e.shardPoolSize), ants.WithNonblocking(true)) if err != nil { - return fmt.Errorf("could not create pool: %w", err) + return fmt.Errorf("create pool: %w", err) } strID := sh.ID().String() @@ -374,7 +374,7 @@ func (e *StorageEngine) closeShards(ctx context.Context, deletedShards []hashedS zap.Error(err), ) multiErrGuard.Lock() - multiErr = errors.Join(multiErr, fmt.Errorf("could not change shard (id:%s) mode to disabled: %w", sh.ID(), err)) + multiErr = errors.Join(multiErr, fmt.Errorf("change shard (id:%s) mode to disabled: %w", sh.ID(), err)) multiErrGuard.Unlock() } @@ -385,7 +385,7 @@ func (e *StorageEngine) closeShards(ctx context.Context, deletedShards []hashedS zap.Error(err), ) multiErrGuard.Lock() - multiErr = errors.Join(multiErr, fmt.Errorf("could not close removed shard (id:%s): %w", sh.ID(), err)) + multiErr = errors.Join(multiErr, fmt.Errorf("close removed shard (id:%s): %w", sh.ID(), err)) multiErrGuard.Unlock() } return nil diff --git a/pkg/local_object_storage/metabase/control.go b/pkg/local_object_storage/metabase/control.go index 07fa7e9cf..c19c65224 100644 --- a/pkg/local_object_storage/metabase/control.go +++ b/pkg/local_object_storage/metabase/control.go @@ -54,7 +54,7 @@ func (db *DB) Open(ctx context.Context, m mode.Mode) error { func (db *DB) openDB(ctx context.Context, mode mode.Mode) error { err := util.MkdirAllX(filepath.Dir(db.info.Path), db.info.Permission) if err != nil { - return fmt.Errorf("can't create dir %s for metabase: %w", db.info.Path, err) + return fmt.Errorf("create dir %s for metabase: %w", db.info.Path, err) } db.log.Debug(ctx, logs.MetabaseCreatedDirectoryForMetabase, zap.String("path", db.info.Path)) @@ -73,7 +73,7 @@ func (db *DB) openBolt(ctx context.Context) error { db.boltDB, err = bbolt.Open(db.info.Path, db.info.Permission, db.boltOptions) if err != nil { - return fmt.Errorf("can't open boltDB database: %w", err) + return fmt.Errorf("open boltDB database: %w", err) } db.boltDB.MaxBatchDelay = db.boltBatchDelay db.boltDB.MaxBatchSize = db.boltBatchSize @@ -145,27 +145,27 @@ func (db *DB) init(reset bool) error { if reset { err := tx.DeleteBucket(name) if err != nil && !errors.Is(err, bbolt.ErrBucketNotFound) { - return fmt.Errorf("could not delete static bucket %s: %w", k, err) + return fmt.Errorf("delete static bucket %s: %w", k, err) } } _, err := tx.CreateBucketIfNotExists(name) if err != nil { - return fmt.Errorf("could not create static bucket %s: %w", k, err) + return fmt.Errorf("create static bucket %s: %w", k, err) } } for _, b := range deprecatedBuckets { err := tx.DeleteBucket(b) if err != nil && !errors.Is(err, bbolt.ErrBucketNotFound) { - return fmt.Errorf("could not delete deprecated bucket %s: %w", string(b), err) + return fmt.Errorf("delete deprecated bucket %s: %w", string(b), err) } } if !reset { // counters will be recalculated by refill metabase err = syncCounter(tx, false) if err != nil { - return fmt.Errorf("could not sync object counter: %w", err) + return fmt.Errorf("sync object counter: %w", err) } return nil diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index 3ead0d9a0..f29dafe77 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -238,14 +238,14 @@ func (db *DB) incCounters(tx *bbolt.Tx, cnrID cid.ID, isUserObject bool) error { } if err := db.updateShardObjectCounterBucket(b, phy, 1, true); err != nil { - return fmt.Errorf("could not increase phy object counter: %w", err) + return fmt.Errorf("increase phy object counter: %w", err) } if err := db.updateShardObjectCounterBucket(b, logical, 1, true); err != nil { - return fmt.Errorf("could not increase logical object counter: %w", err) + return fmt.Errorf("increase logical object counter: %w", err) } if isUserObject { if err := db.updateShardObjectCounterBucket(b, user, 1, true); err != nil { - return fmt.Errorf("could not increase user object counter: %w", err) + return fmt.Errorf("increase user object counter: %w", err) } } return db.incContainerObjectCounter(tx, cnrID, isUserObject) @@ -362,7 +362,7 @@ func (db *DB) incContainerObjectCounter(tx *bbolt.Tx, cnrID cid.ID, isUserObject func syncCounter(tx *bbolt.Tx, force bool) error { shardInfoB, err := createBucketLikelyExists(tx, shardInfoBucket) if err != nil { - return fmt.Errorf("could not get shard info bucket: %w", err) + return fmt.Errorf("get shard info bucket: %w", err) } shardObjectCounterInitialized := len(shardInfoB.Get(objectPhyCounterKey)) == 8 && len(shardInfoB.Get(objectLogicCounterKey)) == 8 && @@ -375,7 +375,7 @@ func syncCounter(tx *bbolt.Tx, force bool) error { containerCounterB, err := createBucketLikelyExists(tx, containerCounterBucketName) if err != nil { - return fmt.Errorf("could not get container counter bucket: %w", err) + return fmt.Errorf("get container counter bucket: %w", err) } var addr oid.Address @@ -428,7 +428,7 @@ func syncCounter(tx *bbolt.Tx, force bool) error { return nil }) if err != nil { - return fmt.Errorf("could not iterate objects: %w", err) + return fmt.Errorf("iterate objects: %w", err) } return setObjectCounters(counters, shardInfoB, containerCounterB) @@ -448,7 +448,7 @@ func setObjectCounters(counters map[cid.ID]ObjectCounters, shardInfoB, container value := containerCounterValue(count) err := containerCounterB.Put(key, value) if err != nil { - return fmt.Errorf("could not update phy container object counter: %w", err) + return fmt.Errorf("update phy container object counter: %w", err) } } phyData := make([]byte, 8) @@ -456,7 +456,7 @@ func setObjectCounters(counters map[cid.ID]ObjectCounters, shardInfoB, container err := shardInfoB.Put(objectPhyCounterKey, phyData) if err != nil { - return fmt.Errorf("could not update phy object counter: %w", err) + return fmt.Errorf("update phy object counter: %w", err) } logData := make([]byte, 8) @@ -464,7 +464,7 @@ func setObjectCounters(counters map[cid.ID]ObjectCounters, shardInfoB, container err = shardInfoB.Put(objectLogicCounterKey, logData) if err != nil { - return fmt.Errorf("could not update logic object counter: %w", err) + return fmt.Errorf("update logic object counter: %w", err) } userData := make([]byte, 8) @@ -472,7 +472,7 @@ func setObjectCounters(counters map[cid.ID]ObjectCounters, shardInfoB, container err = shardInfoB.Put(objectUserCounterKey, userData) if err != nil { - return fmt.Errorf("could not update user object counter: %w", err) + return fmt.Errorf("update user object counter: %w", err) } return nil @@ -492,7 +492,7 @@ func parseContainerCounterKey(buf []byte) (cid.ID, error) { } var cnrID cid.ID if err := cnrID.Decode(buf); err != nil { - return cid.ID{}, fmt.Errorf("failed to decode container ID: %w", err) + return cid.ID{}, fmt.Errorf("decode container ID: %w", err) } return cnrID, nil } diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 62ab1056d..00ee2baa3 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -163,26 +163,26 @@ func (db *DB) updateCountersDelete(tx *bbolt.Tx, res DeleteRes) error { if res.phyCount > 0 { err := db.updateShardObjectCounter(tx, phy, res.phyCount, false) if err != nil { - return fmt.Errorf("could not decrease phy object counter: %w", err) + return fmt.Errorf("decrease phy object counter: %w", err) } } if res.logicCount > 0 { err := db.updateShardObjectCounter(tx, logical, res.logicCount, false) if err != nil { - return fmt.Errorf("could not decrease logical object counter: %w", err) + return fmt.Errorf("decrease logical object counter: %w", err) } } if res.userCount > 0 { err := db.updateShardObjectCounter(tx, user, res.userCount, false) if err != nil { - return fmt.Errorf("could not decrease user object counter: %w", err) + return fmt.Errorf("decrease user object counter: %w", err) } } if err := db.updateContainerCounter(tx, res.removedByCnrID, false); err != nil { - return fmt.Errorf("could not decrease container object counter: %w", err) + return fmt.Errorf("decrease container object counter: %w", err) } return nil } @@ -259,7 +259,7 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter if garbageBKT != nil { err := garbageBKT.Delete(addrKey) if err != nil { - return deleteSingleResult{}, fmt.Errorf("could not remove from garbage bucket: %w", err) + return deleteSingleResult{}, fmt.Errorf("remove from garbage bucket: %w", err) } } return deleteSingleResult{}, nil @@ -280,7 +280,7 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter if garbageBKT != nil { err := garbageBKT.Delete(addrKey) if err != nil { - return deleteSingleResult{}, fmt.Errorf("could not remove from garbage bucket: %w", err) + return deleteSingleResult{}, fmt.Errorf("remove from garbage bucket: %w", err) } } @@ -308,7 +308,7 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, refCounter referenceCounter // remove object err = db.deleteObject(tx, obj, false) if err != nil { - return deleteSingleResult{}, fmt.Errorf("could not remove object: %w", err) + return deleteSingleResult{}, fmt.Errorf("remove object: %w", err) } if err := deleteECRelatedInfo(tx, garbageBKT, obj, addr.Container(), refCounter); err != nil { @@ -335,12 +335,12 @@ func (db *DB) deleteObject( err = updateListIndexes(tx, obj, delListIndexItem) if err != nil { - return fmt.Errorf("can't remove list indexes: %w", err) + return fmt.Errorf("remove list indexes: %w", err) } err = updateFKBTIndexes(tx, obj, delFKBTIndexItem) if err != nil { - return fmt.Errorf("can't remove fake bucket tree indexes: %w", err) + return fmt.Errorf("remove fake bucket tree indexes: %w", err) } if isParent { @@ -351,7 +351,7 @@ func (db *DB) deleteObject( addrKey := addressKey(object.AddressOf(obj), key) err := garbageBKT.Delete(addrKey) if err != nil { - return fmt.Errorf("could not remove from garbage bucket: %w", err) + return fmt.Errorf("remove from garbage bucket: %w", err) } } } @@ -529,7 +529,7 @@ func deleteECRelatedInfo(tx *bbolt.Tx, garbageBKT *bbolt.Bucket, obj *objectSDK. addrKey := addressKey(ecParentAddress, make([]byte, addressKeySize)) err := garbageBKT.Delete(addrKey) if err != nil { - return fmt.Errorf("could not remove EC parent from garbage bucket: %w", err) + return fmt.Errorf("remove EC parent from garbage bucket: %w", err) } } @@ -567,7 +567,7 @@ func deleteECRelatedInfo(tx *bbolt.Tx, garbageBKT *bbolt.Bucket, obj *objectSDK. addrKey := addressKey(splitParentAddress, make([]byte, addressKeySize)) err := garbageBKT.Delete(addrKey) if err != nil { - return fmt.Errorf("could not remove EC parent from garbage bucket: %w", err) + return fmt.Errorf("remove EC parent from garbage bucket: %w", err) } } diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index 411beb6b3..3133c5480 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -229,7 +229,7 @@ func getSplitInfo(tx *bbolt.Tx, cnr cid.ID, key []byte) (*objectSDK.SplitInfo, e err := splitInfo.Unmarshal(bytes.Clone(rawSplitInfo)) if err != nil { - return nil, fmt.Errorf("can't unmarshal split info from root index: %w", err) + return nil, fmt.Errorf("unmarshal split info from root index: %w", err) } return splitInfo, nil diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index 1cbf78ab2..af274b245 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -187,7 +187,7 @@ func getVirtualObject(tx *bbolt.Tx, cnr cid.ID, key []byte, raw bool) (*objectSD err = child.Unmarshal(bytes.Clone(data)) if err != nil { - return nil, fmt.Errorf("can't unmarshal child with parent: %w", err) + return nil, fmt.Errorf("unmarshal child with parent: %w", err) } par := child.Parent() diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index b0db952b2..2f23d424c 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -177,7 +177,7 @@ type gcHandler struct { func (g gcHandler) handleKV(k, _ []byte) error { o, err := garbageFromKV(k) if err != nil { - return fmt.Errorf("could not parse garbage object: %w", err) + return fmt.Errorf("parse garbage object: %w", err) } return g.h(o) @@ -190,7 +190,7 @@ type graveyardHandler struct { func (g graveyardHandler) handleKV(k, v []byte) error { o, err := graveFromKV(k, v) if err != nil { - return fmt.Errorf("could not parse grave: %w", err) + return fmt.Errorf("parse grave: %w", err) } return g.h(o) @@ -240,7 +240,7 @@ func (db *DB) iterateDeletedObj(tx *bbolt.Tx, h kvHandler, offset *oid.Address) func garbageFromKV(k []byte) (res GarbageObject, err error) { err = decodeAddressFromKey(&res.addr, k) if err != nil { - err = fmt.Errorf("could not parse address: %w", err) + err = fmt.Errorf("parse address: %w", err) } return diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 5ac0c0be5..99fdec310 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -373,7 +373,7 @@ func (db *DB) getInhumeTargetBucketAndValue(garbageBKT, graveyardBKT *bbolt.Buck if data != nil { err := targetBucket.Delete(tombKey) if err != nil { - return nil, nil, fmt.Errorf("could not remove grave with tombstone key: %w", err) + return nil, nil, fmt.Errorf("remove grave with tombstone key: %w", err) } } diff --git a/pkg/local_object_storage/metabase/mode.go b/pkg/local_object_storage/metabase/mode.go index ce6ae1004..7edb96384 100644 --- a/pkg/local_object_storage/metabase/mode.go +++ b/pkg/local_object_storage/metabase/mode.go @@ -19,7 +19,7 @@ func (db *DB) SetMode(ctx context.Context, m mode.Mode) error { if !db.mode.NoMetabase() { if err := db.Close(ctx); err != nil { - return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err) + return fmt.Errorf("set metabase mode (old=%s, new=%s): %w", db.mode, m, err) } } @@ -31,7 +31,7 @@ func (db *DB) SetMode(ctx context.Context, m mode.Mode) error { err = db.Init(ctx) } if err != nil { - return fmt.Errorf("can't set metabase mode (old=%s, new=%s): %w", db.mode, m, err) + return fmt.Errorf("set metabase mode (old=%s, new=%s): %w", db.mode, m, err) } } diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 6f9dc1bf0..16918c4d9 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -180,18 +180,18 @@ func (db *DB) insertObject(tx *bbolt.Tx, obj *objectSDK.Object, id []byte, si *o err := putUniqueIndexes(tx, obj, si, id) if err != nil { - return fmt.Errorf("can't put unique indexes: %w", err) + return fmt.Errorf("put unique indexes: %w", err) } err = updateListIndexes(tx, obj, putListIndexItem) if err != nil { - return fmt.Errorf("can't put list indexes: %w", err) + return fmt.Errorf("put list indexes: %w", err) } if indexAttributes { err = updateFKBTIndexes(tx, obj, putFKBTIndexItem) if err != nil { - return fmt.Errorf("can't put fake bucket tree indexes: %w", err) + return fmt.Errorf("put fake bucket tree indexes: %w", err) } } @@ -250,7 +250,7 @@ func putRawObjectData(tx *bbolt.Tx, obj *objectSDK.Object, bucketName []byte, ad } rawObject, err := obj.CutPayload().Marshal() if err != nil { - return fmt.Errorf("can't marshal object header: %w", err) + return fmt.Errorf("marshal object header: %w", err) } return putUniqueIndexItem(tx, namedBucketItem{ name: bucketName, @@ -475,7 +475,7 @@ func createBucketLikelyExists[T bucketContainer](tx T, name []byte) (*bbolt.Buck func updateUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem, update func(oldData, newData []byte) ([]byte, error)) error { bkt, err := createBucketLikelyExists(tx, item.name) if err != nil { - return fmt.Errorf("can't create index %v: %w", item.name, err) + return fmt.Errorf("create index %v: %w", item.name, err) } data, err := update(bkt.Get(item.key), item.val) @@ -492,12 +492,12 @@ func putUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error { func putFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { bkt, err := createBucketLikelyExists(tx, item.name) if err != nil { - return fmt.Errorf("can't create index %v: %w", item.name, err) + return fmt.Errorf("create index %v: %w", item.name, err) } fkbtRoot, err := createBucketLikelyExists(bkt, item.key) if err != nil { - return fmt.Errorf("can't create fake bucket tree index %v: %w", item.key, err) + return fmt.Errorf("create fake bucket tree index %v: %w", item.key, err) } return fkbtRoot.Put(item.val, zeroValue) @@ -506,19 +506,19 @@ func putFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { func putListIndexItem(tx *bbolt.Tx, item namedBucketItem) error { bkt, err := createBucketLikelyExists(tx, item.name) if err != nil { - return fmt.Errorf("can't create index %v: %w", item.name, err) + return fmt.Errorf("create index %v: %w", item.name, err) } lst, err := decodeList(bkt.Get(item.key)) if err != nil { - return fmt.Errorf("can't decode leaf list %v: %w", item.key, err) + return fmt.Errorf("decode leaf list %v: %w", item.key, err) } lst = append(lst, item.val) encodedLst, err := encodeList(lst) if err != nil { - return fmt.Errorf("can't encode leaf list %v: %w", item.key, err) + return fmt.Errorf("encode leaf list %v: %w", item.key, err) } return bkt.Put(item.key, encodedLst) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index f802036be..9f1b8b060 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -565,7 +565,7 @@ func groupFilters(filters objectSDK.SearchFilters, useAttributeIndex bool) (filt case v2object.FilterHeaderContainerID: // support deprecated field err := res.cnr.DecodeString(filters[i].Value()) if err != nil { - return filterGroup{}, fmt.Errorf("can't parse container id: %w", err) + return filterGroup{}, fmt.Errorf("parse container id: %w", err) } res.withCnrFilter = true diff --git a/pkg/local_object_storage/metabase/shard_id.go b/pkg/local_object_storage/metabase/shard_id.go index e58115bc8..72618b1a0 100644 --- a/pkg/local_object_storage/metabase/shard_id.go +++ b/pkg/local_object_storage/metabase/shard_id.go @@ -32,13 +32,13 @@ func (db *DB) GetShardID(ctx context.Context, mode metamode.Mode) ([]byte, error } if err := db.openDB(ctx, mode); err != nil { - return nil, fmt.Errorf("failed to open metabase: %w", err) + return nil, fmt.Errorf("open metabase: %w", err) } id, err := db.readShardID() if cErr := db.close(); cErr != nil { - err = errors.Join(err, fmt.Errorf("failed to close metabase: %w", cErr)) + err = errors.Join(err, fmt.Errorf("close metabase: %w", cErr)) } return id, metaerr.Wrap(err) @@ -70,7 +70,7 @@ func (db *DB) SetShardID(ctx context.Context, id []byte, mode metamode.Mode) err } if err := db.openDB(ctx, mode); err != nil { - return fmt.Errorf("failed to open metabase: %w", err) + return fmt.Errorf("open metabase: %w", err) } err := db.writeShardID(id) @@ -79,7 +79,7 @@ func (db *DB) SetShardID(ctx context.Context, id []byte, mode metamode.Mode) err } if cErr := db.close(); cErr != nil { - err = errors.Join(err, fmt.Errorf("failed to close metabase: %w", cErr)) + err = errors.Join(err, fmt.Errorf("close metabase: %w", cErr)) } return metaerr.Wrap(err) diff --git a/pkg/local_object_storage/metabase/upgrade.go b/pkg/local_object_storage/metabase/upgrade.go index bcf72f440..6eba58c69 100644 --- a/pkg/local_object_storage/metabase/upgrade.go +++ b/pkg/local_object_storage/metabase/upgrade.go @@ -95,7 +95,7 @@ func compactDB(db *bbolt.DB) error { NoSync: true, }) if err != nil { - return fmt.Errorf("can't open new metabase to compact: %w", err) + return fmt.Errorf("open new metabase to compact: %w", err) } if err := bbolt.Compact(dst, db, compactMaxTxSize); err != nil { return fmt.Errorf("compact metabase: %w", errors.Join(err, dst.Close(), os.Remove(tmpFileName))) @@ -292,7 +292,7 @@ func iterateExpirationAttributeKeyBucket(ctx context.Context, b *bbolt.Bucket, i } expirationEpoch, err := strconv.ParseUint(string(attrValue), 10, 64) if err != nil { - return fmt.Errorf("could not parse expiration epoch: %w", err) + return fmt.Errorf("parse expiration epoch: %w", err) } expirationEpochBucket := b.Bucket(attrValue) attrKeyValueC := expirationEpochBucket.Cursor() @@ -399,7 +399,7 @@ func selectUserAttributeKeysToDrop(keys [][]byte, cs container.InfoProvider) ([] for _, key := range keys { attr, ok := attributeFromAttributeBucket(key) if !ok { - return nil, fmt.Errorf("failed to parse attribute key from user attribute bucket key %s", hex.EncodeToString(key)) + return nil, fmt.Errorf("parse attribute key from user attribute bucket key %s", hex.EncodeToString(key)) } if !IsAtrributeIndexed(attr) { keysToDrop = append(keysToDrop, key) @@ -407,7 +407,7 @@ func selectUserAttributeKeysToDrop(keys [][]byte, cs container.InfoProvider) ([] } contID, ok := cidFromAttributeBucket(key) if !ok { - return nil, fmt.Errorf("failed to parse container ID from user attribute bucket key %s", hex.EncodeToString(key)) + return nil, fmt.Errorf("parse container ID from user attribute bucket key %s", hex.EncodeToString(key)) } info, err := cs.Info(contID) if err != nil { diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index 0a2f91a47..80851f1c4 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -231,11 +231,11 @@ func parseExpirationEpochKey(key []byte) (uint64, cid.ID, oid.ID, error) { epoch := binary.BigEndian.Uint64(key) var cnr cid.ID if err := cnr.Decode(key[epochSize : epochSize+cidSize]); err != nil { - return 0, cid.ID{}, oid.ID{}, fmt.Errorf("failed to decode expiration epoch to object key (container ID): %w", err) + return 0, cid.ID{}, oid.ID{}, fmt.Errorf("decode expiration epoch to object key (container ID): %w", err) } var obj oid.ID if err := obj.Decode(key[epochSize+cidSize:]); err != nil { - return 0, cid.ID{}, oid.ID{}, fmt.Errorf("failed to decode expiration epoch to object key (object ID): %w", err) + return 0, cid.ID{}, oid.ID{}, fmt.Errorf("decode expiration epoch to object key (object ID): %w", err) } return epoch, cnr, obj, nil } diff --git a/pkg/local_object_storage/metabase/version.go b/pkg/local_object_storage/metabase/version.go index 048bb9af6..fbc0f1ad9 100644 --- a/pkg/local_object_storage/metabase/version.go +++ b/pkg/local_object_storage/metabase/version.go @@ -67,7 +67,7 @@ func updateVersion(tx *bbolt.Tx, version uint64) error { b, err := tx.CreateBucketIfNotExists(shardInfoBucket) if err != nil { - return fmt.Errorf("can't create auxiliary bucket: %w", err) + return fmt.Errorf("create auxiliary bucket: %w", err) } return b.Put(versionKey, data) } diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 6e68e9986..fecf96f66 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -106,7 +106,7 @@ func (t *boltForest) SetMode(ctx context.Context, m mode.Mode) error { } } if err != nil { - return fmt.Errorf("can't set pilorama mode (old=%s, new=%s): %w", t.mode, m, err) + return fmt.Errorf("set pilorama mode (old=%s, new=%s): %w", t.mode, m, err) } t.mode = m @@ -128,7 +128,7 @@ func (t *boltForest) openBolt(m mode.Mode) error { readOnly := m.ReadOnly() err := util.MkdirAllX(filepath.Dir(t.path), t.perm) if err != nil { - return metaerr.Wrap(fmt.Errorf("can't create dir %s for the pilorama: %w", t.path, err)) + return metaerr.Wrap(fmt.Errorf("create dir %s for the pilorama: %w", t.path, err)) } opts := *bbolt.DefaultOptions @@ -139,7 +139,7 @@ func (t *boltForest) openBolt(m mode.Mode) error { t.db, err = bbolt.Open(t.path, t.perm, &opts) if err != nil { - return metaerr.Wrap(fmt.Errorf("can't open the pilorama DB: %w", err)) + return metaerr.Wrap(fmt.Errorf("open the pilorama DB: %w", err)) } t.db.MaxBatchSize = t.maxBatchSize @@ -1360,7 +1360,7 @@ func (t *boltForest) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, err return nil }) if err != nil { - return nil, metaerr.Wrap(fmt.Errorf("could not list trees: %w", err)) + return nil, metaerr.Wrap(fmt.Errorf("list trees: %w", err)) } success = true return ids, nil @@ -1504,7 +1504,7 @@ func (t *boltForest) TreeListTrees(ctx context.Context, prm TreeListTreesPrm) (* var contID cidSDK.ID if err := contID.Decode(k[:32]); err != nil { - return fmt.Errorf("failed to decode containerID: %w", err) + return fmt.Errorf("decode containerID: %w", err) } res.Items = append(res.Items, ContainerIDTreeID{ CID: contID, diff --git a/pkg/local_object_storage/shard/container.go b/pkg/local_object_storage/shard/container.go index 364649b50..0309f0c81 100644 --- a/pkg/local_object_storage/shard/container.go +++ b/pkg/local_object_storage/shard/container.go @@ -36,7 +36,7 @@ func (s *Shard) ContainerSize(prm ContainerSizePrm) (ContainerSizeRes, error) { size, err := s.metaBase.ContainerSize(prm.cnr) if err != nil { - return ContainerSizeRes{}, fmt.Errorf("could not get container size: %w", err) + return ContainerSizeRes{}, fmt.Errorf("get container size: %w", err) } return ContainerSizeRes{ @@ -71,7 +71,7 @@ func (s *Shard) ContainerCount(ctx context.Context, prm ContainerCountPrm) (Cont counters, err := s.metaBase.ContainerCount(ctx, prm.ContainerID) if err != nil { - return ContainerCountRes{}, fmt.Errorf("could not get container counters: %w", err) + return ContainerCountRes{}, fmt.Errorf("get container counters: %w", err) } return ContainerCountRes{ diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 78ce241fe..1c1933af5 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -38,7 +38,7 @@ func (s *Shard) handleMetabaseFailure(ctx context.Context, stage string, err err err = s.SetMode(ctx, mode.DegradedReadOnly) if err != nil { - return fmt.Errorf("could not switch to mode %s", mode.Mode(mode.DegradedReadOnly)) + return fmt.Errorf("switch to mode %s", mode.Mode(mode.DegradedReadOnly)) } return nil } @@ -72,7 +72,7 @@ func (s *Shard) Open(ctx context.Context) error { for j := i + 1; j < len(components); j++ { if err := components[j].Open(ctx, m); err != nil { // Other components must be opened, fail. - return fmt.Errorf("could not open %T: %w", components[j], err) + return fmt.Errorf("open %T: %w", components[j], err) } } err = s.handleMetabaseFailure(ctx, "open", err) @@ -83,7 +83,7 @@ func (s *Shard) Open(ctx context.Context) error { break } - return fmt.Errorf("could not open %T: %w", component, err) + return fmt.Errorf("open %T: %w", component, err) } } return nil @@ -184,7 +184,7 @@ func (s *Shard) initializeComponents(ctx context.Context, m mode.Mode) error { break } - return fmt.Errorf("could not initialize %T: %w", component, err) + return fmt.Errorf("initialize %T: %w", component, err) } } return nil @@ -205,7 +205,7 @@ func (s *Shard) refillMetabase(ctx context.Context) error { err := s.metaBase.Reset() if err != nil { - return fmt.Errorf("could not reset metabase: %w", err) + return fmt.Errorf("reset metabase: %w", err) } withCount := true @@ -254,12 +254,12 @@ func (s *Shard) refillMetabase(ctx context.Context) error { err = errors.Join(egErr, itErr) if err != nil { - return fmt.Errorf("could not put objects to the meta: %w", err) + return fmt.Errorf("put objects to the meta: %w", err) } err = s.metaBase.SyncCounters() if err != nil { - return fmt.Errorf("could not sync object counters: %w", err) + return fmt.Errorf("sync object counters: %w", err) } success = true @@ -318,7 +318,7 @@ func (s *Shard) refillObject(ctx context.Context, data []byte, addr oid.Address, func (s *Shard) refillLockObject(ctx context.Context, obj *objectSDK.Object) error { var lock objectSDK.Lock if err := lock.Unmarshal(obj.Payload()); err != nil { - return fmt.Errorf("could not unmarshal lock content: %w", err) + return fmt.Errorf("unmarshal lock content: %w", err) } locked := make([]oid.ID, lock.NumberOfMembers()) @@ -328,7 +328,7 @@ func (s *Shard) refillLockObject(ctx context.Context, obj *objectSDK.Object) err id, _ := obj.ID() err := s.metaBase.Lock(ctx, cnr, id, locked) if err != nil { - return fmt.Errorf("could not lock objects: %w", err) + return fmt.Errorf("lock objects: %w", err) } return nil } @@ -337,7 +337,7 @@ func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object tombstone := objectSDK.NewTombstone() if err := tombstone.Unmarshal(obj.Payload()); err != nil { - return fmt.Errorf("could not unmarshal tombstone content: %w", err) + return fmt.Errorf("unmarshal tombstone content: %w", err) } tombAddr := object.AddressOf(obj) @@ -358,7 +358,7 @@ func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object _, err := s.metaBase.Inhume(ctx, inhumePrm) if err != nil { - return fmt.Errorf("could not inhume objects: %w", err) + return fmt.Errorf("inhume objects: %w", err) } return nil } diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 7a31a705e..15d1eb6ba 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -175,7 +175,7 @@ func (s *Shard) fetchObjectData(ctx context.Context, addr oid.Address, skipMeta mExRes, err := s.metaBase.StorageID(ctx, mPrm) if err != nil { - return nil, true, fmt.Errorf("can't fetch blobovnicza id from metabase: %w", err) + return nil, true, fmt.Errorf("fetch blobovnicza id from metabase: %w", err) } storageID := mExRes.StorageID() diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index 6ccae3f53..26492cf01 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -36,7 +36,7 @@ func (s *Shard) UpdateID(ctx context.Context) (err error) { modeDegraded := s.GetMode().NoMetabase() if !modeDegraded { if idFromMetabase, err = s.metaBase.GetShardID(ctx, mode.ReadOnly); err != nil { - err = fmt.Errorf("failed to read shard id from metabase: %w", err) + err = fmt.Errorf("read shard id from metabase: %w", err) } } @@ -64,7 +64,7 @@ func (s *Shard) UpdateID(ctx context.Context) (err error) { if len(idFromMetabase) == 0 && !modeDegraded { if setErr := s.metaBase.SetShardID(ctx, *s.info.ID, s.GetMode()); setErr != nil { - err = errors.Join(err, fmt.Errorf("failed to write shard id to metabase: %w", setErr)) + err = errors.Join(err, fmt.Errorf("write shard id to metabase: %w", setErr)) } } return diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index f583ef5d9..c5275dafd 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -109,7 +109,7 @@ func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { lst, err := s.metaBase.Containers(ctx) if err != nil { - return res, fmt.Errorf("can't list stored containers: %w", err) + return res, fmt.Errorf("list stored containers: %w", err) } filters := objectSDK.NewSearchFilters() @@ -149,7 +149,7 @@ func (s *Shard) ListContainers(ctx context.Context, _ ListContainersPrm) (ListCo containers, err := s.metaBase.Containers(ctx) if err != nil { - return ListContainersRes{}, fmt.Errorf("could not get list of containers: %w", err) + return ListContainersRes{}, fmt.Errorf("get list of containers: %w", err) } return ListContainersRes{ @@ -180,7 +180,7 @@ func (s *Shard) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (List metaPrm.SetCursor(prm.cursor) res, err := s.metaBase.ListWithCursor(ctx, metaPrm) if err != nil { - return ListWithCursorRes{}, fmt.Errorf("could not get list of objects: %w", err) + return ListWithCursorRes{}, fmt.Errorf("get list of objects: %w", err) } return ListWithCursorRes{ @@ -208,7 +208,7 @@ func (s *Shard) IterateOverContainers(ctx context.Context, prm IterateOverContai metaPrm.Handler = prm.Handler err := s.metaBase.IterateOverContainers(ctx, metaPrm) if err != nil { - return fmt.Errorf("could not iterate over containers: %w", err) + return fmt.Errorf("iterate over containers: %w", err) } return nil @@ -235,7 +235,7 @@ func (s *Shard) IterateOverObjectsInContainer(ctx context.Context, prm IterateOv metaPrm.Handler = prm.Handler err := s.metaBase.IterateOverObjectsInContainer(ctx, metaPrm) if err != nil { - return fmt.Errorf("could not iterate over objects: %w", err) + return fmt.Errorf("iterate over objects: %w", err) } return nil @@ -258,7 +258,7 @@ func (s *Shard) CountAliveObjectsInContainer(ctx context.Context, prm CountAlive metaPrm.ContainerID = prm.ContainerID count, err := s.metaBase.CountAliveObjectsInContainer(ctx, metaPrm) if err != nil { - return 0, fmt.Errorf("could not count alive objects in bucket: %w", err) + return 0, fmt.Errorf("count alive objects in bucket: %w", err) } return count, nil diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index 1e4643db5..3f23111af 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -81,7 +81,7 @@ func (s *Shard) Put(ctx context.Context, prm PutPrm) (PutRes, error) { res, err = s.blobStor.Put(ctx, putPrm) if err != nil { - return PutRes{}, fmt.Errorf("could not put object to BLOB storage: %w", err) + return PutRes{}, fmt.Errorf("put object to BLOB storage: %w", err) } } @@ -94,7 +94,7 @@ func (s *Shard) Put(ctx context.Context, prm PutPrm) (PutRes, error) { if err != nil { // may we need to handle this case in a special way // since the object has been successfully written to BlobStor - return PutRes{}, fmt.Errorf("could not put object to metabase: %w", err) + return PutRes{}, fmt.Errorf("put object to metabase: %w", err) } if res.Inserted { diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index 184ca9b71..c7c7e11c2 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -67,7 +67,7 @@ func (s *Shard) Select(ctx context.Context, prm SelectPrm) (SelectRes, error) { mRes, err := s.metaBase.Select(ctx, selectPrm) if err != nil { - return SelectRes{}, fmt.Errorf("could not select objects from metabase: %w", err) + return SelectRes{}, fmt.Errorf("select objects from metabase: %w", err) } return SelectRes{ diff --git a/pkg/local_object_storage/writecache/iterate.go b/pkg/local_object_storage/writecache/iterate.go index 9ec039f91..e369fbd50 100644 --- a/pkg/local_object_storage/writecache/iterate.go +++ b/pkg/local_object_storage/writecache/iterate.go @@ -30,7 +30,7 @@ func IterateDB(db *bbolt.DB, f func(oid.Address) error) error { return b.ForEach(func(k, _ []byte) error { err := addr.DecodeString(string(k)) if err != nil { - return fmt.Errorf("could not parse object address: %w", err) + return fmt.Errorf("parse object address: %w", err) } return f(addr) diff --git a/pkg/local_object_storage/writecache/mode.go b/pkg/local_object_storage/writecache/mode.go index 73d12fd33..c491be60b 100644 --- a/pkg/local_object_storage/writecache/mode.go +++ b/pkg/local_object_storage/writecache/mode.go @@ -83,7 +83,7 @@ func (c *cache) closeStorage(ctx context.Context, shrink bool) error { } if !shrink { if err := c.fsTree.Close(ctx); err != nil { - return fmt.Errorf("can't close write-cache storage: %w", err) + return fmt.Errorf("close write-cache storage: %w", err) } return nil } @@ -98,16 +98,16 @@ func (c *cache) closeStorage(ctx context.Context, shrink bool) error { if errors.Is(err, errIterationCompleted) { empty = false } else { - return fmt.Errorf("failed to check write-cache items: %w", err) + return fmt.Errorf("check write-cache items: %w", err) } } if err := c.fsTree.Close(ctx); err != nil { - return fmt.Errorf("can't close write-cache storage: %w", err) + return fmt.Errorf("close write-cache storage: %w", err) } if empty { err := os.RemoveAll(c.path) if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to remove write-cache files: %w", err) + return fmt.Errorf("remove write-cache files: %w", err) } } else { c.log.Info(ctx, logs.WritecacheShrinkSkippedNotEmpty) diff --git a/pkg/local_object_storage/writecache/storage.go b/pkg/local_object_storage/writecache/storage.go index a0e236cb7..e88566cdf 100644 --- a/pkg/local_object_storage/writecache/storage.go +++ b/pkg/local_object_storage/writecache/storage.go @@ -31,10 +31,10 @@ func (c *cache) openStore(mod mode.ComponentMode) error { fstree.WithFileCounter(c.counter), ) if err := c.fsTree.Open(mod); err != nil { - return fmt.Errorf("could not open FSTree: %w", err) + return fmt.Errorf("open FSTree: %w", err) } if err := c.fsTree.Init(); err != nil { - return fmt.Errorf("could not init FSTree: %w", err) + return fmt.Errorf("init FSTree: %w", err) } return nil diff --git a/pkg/local_object_storage/writecache/upgrade.go b/pkg/local_object_storage/writecache/upgrade.go index 3a100f1a3..5eb341ba4 100644 --- a/pkg/local_object_storage/writecache/upgrade.go +++ b/pkg/local_object_storage/writecache/upgrade.go @@ -25,11 +25,11 @@ func (c *cache) flushAndDropBBoltDB(ctx context.Context) error { return nil } if err != nil { - return fmt.Errorf("could not check write-cache database existence: %w", err) + return fmt.Errorf("check write-cache database existence: %w", err) } db, err := OpenDB(c.path, true, os.OpenFile) if err != nil { - return fmt.Errorf("could not open write-cache database: %w", err) + return fmt.Errorf("open write-cache database: %w", err) } defer func() { _ = db.Close() From 226dd25dd0f464f440248d0d4c7180db0ca4be56 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 18 Dec 2024 09:40:19 +0300 Subject: [PATCH 1640/1943] [#1568] pilorama: Replace "containerID" with "container ID" in the error message It is "container ID" in every other place. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/boltdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index fecf96f66..86b19e3af 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1504,7 +1504,7 @@ func (t *boltForest) TreeListTrees(ctx context.Context, prm TreeListTreesPrm) (* var contID cidSDK.ID if err := contID.Decode(k[:32]); err != nil { - return fmt.Errorf("decode containerID: %w", err) + return fmt.Errorf("decode container ID: %w", err) } res.Items = append(res.Items, ContainerIDTreeID{ CID: contID, From 51ee132ea311b6227d1f0642ab736b5b8e21a0f7 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 18 Dec 2024 19:27:14 +0300 Subject: [PATCH 1641/1943] [#1342] network/cache: Add node address to error multiClient Signed-off-by: Alexander Chuprov --- pkg/network/cache/multi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 481d1ea4a..1bcb83259 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -155,7 +155,7 @@ func (x *multiClient) iterateClients(ctx context.Context, f func(clientcore.Clie group.IterateAddresses(func(addr network.Address) bool { select { case <-ctx.Done(): - firstErr = context.Canceled + firstErr = fmt.Errorf("try %v: %w", addr, context.Canceled) return true default: } From 148d68933bb1f3fda17db2c7f4fe073336554f67 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 19 Dec 2024 16:07:22 +0300 Subject: [PATCH 1642/1943] [#1573] node: Simplify bootstrapWithState() After #1382 we have no need to use lambdas. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 40be8f45f..18d3e2454 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1220,9 +1220,9 @@ func (c *cfg) updateContractNodeInfo(ctx context.Context, epoch uint64) { // bootstrapWithState calls "addPeer" method of the Sidechain Netmap contract // with the binary-encoded information from the current node's configuration. // The state is set using the provided setter which MUST NOT be nil. -func (c *cfg) bootstrapWithState(ctx context.Context, stateSetter func(*netmap.NodeInfo)) error { +func (c *cfg) bootstrapWithState(ctx context.Context, state netmap.NodeState) error { ni := c.cfgNodeInfo.localInfo - stateSetter(&ni) + ni.SetStatus(state) prm := nmClient.AddPeerPrm{} prm.SetNodeInfo(ni) @@ -1232,9 +1232,7 @@ func (c *cfg) bootstrapWithState(ctx context.Context, stateSetter func(*netmap.N // bootstrapOnline calls cfg.bootstrapWithState with "online" state. func bootstrapOnline(ctx context.Context, c *cfg) error { - return c.bootstrapWithState(ctx, func(ni *netmap.NodeInfo) { - ni.SetStatus(netmap.Online) - }) + return c.bootstrapWithState(ctx, netmap.Online) } // bootstrap calls bootstrapWithState with: @@ -1245,9 +1243,7 @@ func (c *cfg) bootstrap(ctx context.Context) error { st := c.cfgNetmap.state.controlNetmapStatus() if st == control.NetmapStatus_MAINTENANCE { c.log.Info(ctx, logs.FrostFSNodeBootstrappingWithTheMaintenanceState) - return c.bootstrapWithState(ctx, func(ni *netmap.NodeInfo) { - ni.SetStatus(netmap.Maintenance) - }) + return c.bootstrapWithState(ctx, netmap.Maintenance) } c.log.Info(ctx, logs.FrostFSNodeBootstrappingWithOnlineState, From db03742d332681ab3d4626d50e85f5da6c482629 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 20 Dec 2024 15:04:17 +0300 Subject: [PATCH 1643/1943] [#1578] adm: Reword help message for `morph refill-gas` Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/generate/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go index 7af776797..37c5d4a4a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go @@ -159,7 +159,7 @@ func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error } } else { if storageWalletPath == "" { - return fmt.Errorf("missing wallet path (use '--%s ')", commonflags.StorageWalletFlag) + return fmt.Errorf("missing wallet path (use '--%s ')", commonflags.StorageWalletFlag) } var w *wallet.Wallet From bb9ba1bce2101e44999aeae10815688aec8d7a01 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 23 Dec 2024 09:55:09 +0300 Subject: [PATCH 1644/1943] [#1578] adm: Remove bool flag from refillGas() Signed-off-by: Evgenii Stratonikov --- .../modules/morph/generate/generate.go | 74 ++++++------------- .../internal/modules/morph/generate/root.go | 32 +++++++- 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go index 37c5d4a4a..388d5c060 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/gas" "github.com/nspcc-dev/neo-go/pkg/smartcontract" @@ -141,60 +140,29 @@ func addMultisigAccount(w *wallet.Wallet, m int, name, password string, pubs key } func generateStorageCreds(cmd *cobra.Command, _ []string) error { - return refillGas(cmd, storageGasConfigFlag, true) -} - -func refillGas(cmd *cobra.Command, gasFlag string, createWallet bool) (err error) { - // storage wallet path is not part of the config - storageWalletPath, _ := cmd.Flags().GetString(commonflags.StorageWalletFlag) - // wallet address is not part of the config - walletAddress, _ := cmd.Flags().GetString(walletAddressFlag) - - var gasReceiver util.Uint160 - - if len(walletAddress) != 0 { - gasReceiver, err = address.StringToUint160(walletAddress) - if err != nil { - return fmt.Errorf("invalid wallet address %s: %w", walletAddress, err) - } - } else { - if storageWalletPath == "" { - return fmt.Errorf("missing wallet path (use '--%s ')", commonflags.StorageWalletFlag) - } - - var w *wallet.Wallet - - if createWallet { - w, err = wallet.NewWallet(storageWalletPath) - } else { - w, err = wallet.NewWalletFromFile(storageWalletPath) - } - - if err != nil { - return fmt.Errorf("can't create wallet: %w", err) - } - - if createWallet { - var password string - - label, _ := cmd.Flags().GetString(storageWalletLabelFlag) - password, err := config.GetStoragePassword(viper.GetViper(), label) - if err != nil { - return fmt.Errorf("can't fetch password: %w", err) - } - - if label == "" { - label = constants.SingleAccountName - } - - if err := w.CreateAccount(label, password); err != nil { - return fmt.Errorf("can't create account: %w", err) - } - } - - gasReceiver = w.Accounts[0].Contract.ScriptHash() + walletPath, _ := cmd.Flags().GetString(commonflags.StorageWalletFlag) + w, err := wallet.NewWallet(walletPath) + if err != nil { + return fmt.Errorf("create wallet: %w", err) } + label, _ := cmd.Flags().GetString(storageWalletLabelFlag) + password, err := config.GetStoragePassword(viper.GetViper(), label) + if err != nil { + return fmt.Errorf("can't fetch password: %w", err) + } + + if label == "" { + label = constants.SingleAccountName + } + + if err := w.CreateAccount(label, password); err != nil { + return fmt.Errorf("can't create account: %w", err) + } + return refillGas(cmd, storageGasConfigFlag, w.Accounts[0].ScriptHash()) +} + +func refillGas(cmd *cobra.Command, gasFlag string, gasReceiver util.Uint160) (err error) { gasStr := viper.GetString(gasFlag) gasAmount, err := helper.ParseGASAmount(gasStr) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/root.go b/cmd/frostfs-adm/internal/modules/morph/generate/root.go index 3633d9a8e..da9665d22 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/root.go @@ -1,7 +1,12 @@ package generate import ( + "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "github.com/nspcc-dev/neo-go/pkg/encoding/address" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -33,7 +38,32 @@ var ( _ = viper.BindPFlag(commonflags.RefillGasAmountFlag, cmd.Flags().Lookup(commonflags.RefillGasAmountFlag)) }, RunE: func(cmd *cobra.Command, _ []string) error { - return refillGas(cmd, commonflags.RefillGasAmountFlag, false) + // storage wallet path is not part of the config + storageWalletPath, _ := cmd.Flags().GetString(commonflags.StorageWalletFlag) + // wallet address is not part of the config + walletAddress, _ := cmd.Flags().GetString(walletAddressFlag) + + var gasReceiver util.Uint160 + + if len(walletAddress) != 0 { + var err error + gasReceiver, err = address.StringToUint160(walletAddress) + if err != nil { + return fmt.Errorf("invalid wallet address %s: %w", walletAddress, err) + } + } else { + if storageWalletPath == "" { + return fmt.Errorf("missing wallet path (use '--%s ')", commonflags.StorageWalletFlag) + } + + w, err := wallet.NewWalletFromFile(storageWalletPath) + if err != nil { + return fmt.Errorf("can't create wallet: %w", err) + } + + gasReceiver = w.Accounts[0].Contract.ScriptHash() + } + return refillGas(cmd, commonflags.RefillGasAmountFlag, gasReceiver) }, } GenerateAlphabetCmd = &cobra.Command{ From 303cd35a019f110f1f47d264ac30827977643b91 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 23 Dec 2024 09:55:54 +0300 Subject: [PATCH 1645/1943] [#1578] adm: Remove unnecessary comments in RefillGasCmd Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/generate/root.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/root.go b/cmd/frostfs-adm/internal/modules/morph/generate/root.go index da9665d22..bdf4dc3d8 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/root.go @@ -38,9 +38,7 @@ var ( _ = viper.BindPFlag(commonflags.RefillGasAmountFlag, cmd.Flags().Lookup(commonflags.RefillGasAmountFlag)) }, RunE: func(cmd *cobra.Command, _ []string) error { - // storage wallet path is not part of the config storageWalletPath, _ := cmd.Flags().GetString(commonflags.StorageWalletFlag) - // wallet address is not part of the config walletAddress, _ := cmd.Flags().GetString(walletAddressFlag) var gasReceiver util.Uint160 From e64871c3fdcd1180b673ac7eb96e2ff6dffe6b87 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 23 Dec 2024 10:20:26 +0300 Subject: [PATCH 1646/1943] [#1578] adm: Allow to transfer GAS to multiple recepients Signed-off-by: Evgenii Stratonikov --- .../modules/morph/generate/generate.go | 10 ++++--- .../internal/modules/morph/generate/root.go | 29 +++++++++---------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go index 388d5c060..78f8617f1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/generate.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/generate.go @@ -162,7 +162,7 @@ func generateStorageCreds(cmd *cobra.Command, _ []string) error { return refillGas(cmd, storageGasConfigFlag, w.Accounts[0].ScriptHash()) } -func refillGas(cmd *cobra.Command, gasFlag string, gasReceiver util.Uint160) (err error) { +func refillGas(cmd *cobra.Command, gasFlag string, gasReceivers ...util.Uint160) (err error) { gasStr := viper.GetString(gasFlag) gasAmount, err := helper.ParseGASAmount(gasStr) @@ -176,9 +176,11 @@ func refillGas(cmd *cobra.Command, gasFlag string, gasReceiver util.Uint160) (er } bw := io.NewBufBinWriter() - emit.AppCall(bw.BinWriter, gas.Hash, "transfer", callflag.All, - wCtx.CommitteeAcc.Contract.ScriptHash(), gasReceiver, int64(gasAmount), nil) - emit.Opcodes(bw.BinWriter, opcode.ASSERT) + for _, gasReceiver := range gasReceivers { + emit.AppCall(bw.BinWriter, gas.Hash, "transfer", callflag.All, + wCtx.CommitteeAcc.Contract.ScriptHash(), gasReceiver, int64(gasAmount), nil) + emit.Opcodes(bw.BinWriter, opcode.ASSERT) + } if bw.Err != nil { return fmt.Errorf("BUG: invalid transfer arguments: %w", bw.Err) } diff --git a/cmd/frostfs-adm/internal/modules/morph/generate/root.go b/cmd/frostfs-adm/internal/modules/morph/generate/root.go index bdf4dc3d8..73c986713 100644 --- a/cmd/frostfs-adm/internal/modules/morph/generate/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/generate/root.go @@ -38,30 +38,27 @@ var ( _ = viper.BindPFlag(commonflags.RefillGasAmountFlag, cmd.Flags().Lookup(commonflags.RefillGasAmountFlag)) }, RunE: func(cmd *cobra.Command, _ []string) error { - storageWalletPath, _ := cmd.Flags().GetString(commonflags.StorageWalletFlag) - walletAddress, _ := cmd.Flags().GetString(walletAddressFlag) + storageWalletPaths, _ := cmd.Flags().GetStringArray(commonflags.StorageWalletFlag) + walletAddresses, _ := cmd.Flags().GetStringArray(walletAddressFlag) - var gasReceiver util.Uint160 - - if len(walletAddress) != 0 { - var err error - gasReceiver, err = address.StringToUint160(walletAddress) + var gasReceivers []util.Uint160 + for _, walletAddress := range walletAddresses { + addr, err := address.StringToUint160(walletAddress) if err != nil { return fmt.Errorf("invalid wallet address %s: %w", walletAddress, err) } - } else { - if storageWalletPath == "" { - return fmt.Errorf("missing wallet path (use '--%s ')", commonflags.StorageWalletFlag) - } + gasReceivers = append(gasReceivers, addr) + } + for _, storageWalletPath := range storageWalletPaths { w, err := wallet.NewWalletFromFile(storageWalletPath) if err != nil { return fmt.Errorf("can't create wallet: %w", err) } - gasReceiver = w.Accounts[0].Contract.ScriptHash() + gasReceivers = append(gasReceivers, w.Accounts[0].Contract.ScriptHash()) } - return refillGas(cmd, commonflags.RefillGasAmountFlag, gasReceiver) + return refillGas(cmd, commonflags.RefillGasAmountFlag, gasReceivers...) }, } GenerateAlphabetCmd = &cobra.Command{ @@ -78,10 +75,10 @@ var ( func initRefillGasCmd() { RefillGasCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) RefillGasCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) - RefillGasCmd.Flags().String(commonflags.StorageWalletFlag, "", "Path to storage node wallet") - RefillGasCmd.Flags().String(walletAddressFlag, "", "Address of wallet") + RefillGasCmd.Flags().StringArray(commonflags.StorageWalletFlag, nil, "Path to storage node wallet") + RefillGasCmd.Flags().StringArray(walletAddressFlag, nil, "Address of wallet") RefillGasCmd.Flags().String(commonflags.RefillGasAmountFlag, "", "Additional amount of GAS to transfer") - RefillGasCmd.MarkFlagsMutuallyExclusive(walletAddressFlag, commonflags.StorageWalletFlag) + RefillGasCmd.MarkFlagsOneRequired(walletAddressFlag, commonflags.StorageWalletFlag) } func initGenerateStorageCmd() { From 7c3bcb0f44cca381ae9c430d032f685265ed6938 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 23 Dec 2024 10:22:09 +0300 Subject: [PATCH 1647/1943] [#1578] Makefile: Refill GAS with a single command in env-up Signed-off-by: Evgenii Stratonikov --- Makefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index ecac760e9..f0cdc273c 100755 --- a/Makefile +++ b/Makefile @@ -270,10 +270,12 @@ env-up: all echo "Frostfs contracts not found"; exit 1; \ fi ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph init --contracts ${FROSTFS_CONTRACTS_PATH} - ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet01.json --gas 10.0 - ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet02.json --gas 10.0 - ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet03.json --gas 10.0 - ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --storage-wallet ./dev/storage/wallet04.json --gas 10.0 + ${BIN}/frostfs-adm --config ./dev/adm/frostfs-adm.yml morph refill-gas --gas 10.0 \ + --storage-wallet ./dev/storage/wallet01.json \ + --storage-wallet ./dev/storage/wallet02.json \ + --storage-wallet ./dev/storage/wallet03.json \ + --storage-wallet ./dev/storage/wallet04.json + @if [ ! -f "$(LOCODE_DB_PATH)" ]; then \ make locode-download; \ fi From 2832f4443750fb11ee827be5491f34f749c9122a Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 20 Dec 2024 12:30:06 +0300 Subject: [PATCH 1648/1943] [#1531] metrics: Rename `app_info` metric Signed-off-by: Ekaterina Lebedeva --- internal/metrics/application.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/metrics/application.go b/internal/metrics/application.go index 8bc408ab6..53acf9b7f 100644 --- a/internal/metrics/application.go +++ b/internal/metrics/application.go @@ -12,8 +12,9 @@ type ApplicationInfo struct { func NewApplicationInfo(version string) *ApplicationInfo { appInfo := &ApplicationInfo{ versionValue: metrics.NewGaugeVec(prometheus.GaugeOpts{ - Name: "app_info", - Help: "General information about the application.", + Namespace: namespace, + Name: "app_info", + Help: "General information about the application.", }, []string{"version"}), } appInfo.versionValue.With(prometheus.Labels{"version": version}) From b5b4f78b4925f71e6a774a4b0b7753a6c3833cbe Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Dec 2024 14:14:57 +0300 Subject: [PATCH 1649/1943] [#1582] adm: Allow using the default account in `deposit-notary` It has never worked, actually. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/notary/notary.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go index fd42d5a4a..6e159f11e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go @@ -41,7 +41,8 @@ func depositNotary(cmd *cobra.Command, _ []string) error { } accHash := w.GetChangeAddress() - if addr, err := cmd.Flags().GetString(walletAccountFlag); err == nil { + addr, _ := cmd.Flags().GetString(walletAccountFlag) + if addr != "" { accHash, err = address.StringToUint160(addr) if err != nil { return fmt.Errorf("invalid address: %s", addr) From 31d3d299bfd03b9de5b3d3f7e973ea6e3718f98a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Dec 2024 14:17:28 +0300 Subject: [PATCH 1650/1943] [#1582] adm: Unify promps for reading a password Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/notary/notary.go | 2 +- cmd/frostfs-adm/internal/modules/storagecfg/root.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go index 6e159f11e..7058818c0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go @@ -54,7 +54,7 @@ func depositNotary(cmd *cobra.Command, _ []string) error { return fmt.Errorf("can't find account for %s", accHash) } - prompt := fmt.Sprintf("Enter password for %s >", address.Uint160ToString(accHash)) + prompt := fmt.Sprintf("Enter password for %s > ", address.Uint160ToString(accHash)) pass, err := input.ReadPassword(prompt) if err != nil { return fmt.Errorf("can't get password: %v", err) diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/root.go b/cmd/frostfs-adm/internal/modules/storagecfg/root.go index 127272da5..8e6a8354e 100644 --- a/cmd/frostfs-adm/internal/modules/storagecfg/root.go +++ b/cmd/frostfs-adm/internal/modules/storagecfg/root.go @@ -105,7 +105,7 @@ func storageConfig(cmd *cobra.Command, args []string) { fatalOnErr(errors.New("can't find account in wallet")) } - c.Wallet.Password, err = input.ReadPassword(fmt.Sprintf("Account password for %s: ", c.Wallet.Account)) + c.Wallet.Password, err = input.ReadPassword(fmt.Sprintf("Enter password for %s > ", c.Wallet.Account)) fatalOnErr(err) err = acc.Decrypt(c.Wallet.Password, keys.NEP2ScryptParams()) From ea868e09f84e0be28acb6c403006900f68ad8bf0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Dec 2024 14:21:36 +0300 Subject: [PATCH 1651/1943] [#1582] adm: Use int64 type and the default value for `--till` flag Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/notary/notary.go | 14 +++----------- .../internal/modules/morph/notary/root.go | 2 +- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go index 7058818c0..3435926c0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary/notary.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary/notary.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "math/big" - "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" @@ -74,16 +73,9 @@ func depositNotary(cmd *cobra.Command, _ []string) error { return err } - till := int64(defaultNotaryDepositLifetime) - tillStr, err := cmd.Flags().GetString(notaryDepositTillFlag) - if err != nil { - return err - } - if tillStr != "" { - till, err = strconv.ParseInt(tillStr, 10, 64) - if err != nil || till <= 0 { - return errInvalidNotaryDepositLifetime - } + till, _ := cmd.Flags().GetInt64(notaryDepositTillFlag) + if till <= 0 { + return errInvalidNotaryDepositLifetime } return transferGas(cmd, acc, accHash, gasAmount, till) diff --git a/cmd/frostfs-adm/internal/modules/morph/notary/root.go b/cmd/frostfs-adm/internal/modules/morph/notary/root.go index 497ff8ea1..d7be2e503 100644 --- a/cmd/frostfs-adm/internal/modules/morph/notary/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/notary/root.go @@ -20,7 +20,7 @@ func initDepositoryNotaryCmd() { DepositCmd.Flags().String(commonflags.StorageWalletFlag, "", "Path to storage node wallet") DepositCmd.Flags().String(walletAccountFlag, "", "Wallet account address") DepositCmd.Flags().String(commonflags.RefillGasAmountFlag, "", "Amount of GAS to deposit") - DepositCmd.Flags().String(notaryDepositTillFlag, "", "Notary deposit duration in blocks") + DepositCmd.Flags().Int64(notaryDepositTillFlag, defaultNotaryDepositLifetime, "Notary deposit duration in blocks") } func init() { From f6c5222952ebce32569b4ff3801943895b56258f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Dec 2024 12:15:56 +0300 Subject: [PATCH 1652/1943] [#1581] services/session: Use user.ID.EncodeToString() where possible gopatch: ``` @@ var id expression @@ -base58.Encode(id.WalletBytes()) +id.EncodeToString() ``` Signed-off-by: Evgenii Stratonikov --- pkg/services/session/storage/temporary/executor.go | 2 +- pkg/services/session/storage/temporary/storage.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/services/session/storage/temporary/executor.go b/pkg/services/session/storage/temporary/executor.go index d531b25cb..423e579d7 100644 --- a/pkg/services/session/storage/temporary/executor.go +++ b/pkg/services/session/storage/temporary/executor.go @@ -38,7 +38,7 @@ func (s *TokenStore) Create(_ context.Context, body *session.CreateRequestBody) s.mtx.Lock() s.tokens[key{ tokenID: base58.Encode(uidBytes), - ownerID: base58.Encode(id.WalletBytes()), + ownerID: id.EncodeToString(), }] = storage.NewPrivateToken(&sk.PrivateKey, body.GetExpiration()) s.mtx.Unlock() diff --git a/pkg/services/session/storage/temporary/storage.go b/pkg/services/session/storage/temporary/storage.go index 9ae9db9dc..c9da6b842 100644 --- a/pkg/services/session/storage/temporary/storage.go +++ b/pkg/services/session/storage/temporary/storage.go @@ -41,7 +41,7 @@ func (s *TokenStore) Get(ownerID user.ID, tokenID []byte) *storage.PrivateToken s.mtx.RLock() t := s.tokens[key{ tokenID: base58.Encode(tokenID), - ownerID: base58.Encode(ownerID.WalletBytes()), + ownerID: ownerID.EncodeToString(), }] s.mtx.RUnlock() From ca0a33ea0febe0a44f5ca70a61c92a23b6cbd4d5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 25 Dec 2024 12:04:08 +0300 Subject: [PATCH 1653/1943] [#465] objsvc: Set NETMAP_EPOCH xheader for auxiliary requests Signed-off-by: Evgenii Stratonikov --- pkg/services/object/internal/client/client.go | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/pkg/services/object/internal/client/client.go b/pkg/services/object/internal/client/client.go index 2c405070d..3e8832640 100644 --- a/pkg/services/object/internal/client/client.go +++ b/pkg/services/object/internal/client/client.go @@ -7,9 +7,11 @@ import ( "errors" "fmt" "io" + "strconv" coreclient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + sessionAPI "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -31,6 +33,8 @@ type commonPrm struct { local bool xHeaders []string + + netmapEpoch uint64 } // SetClient sets base client for ForstFS API communication. @@ -73,6 +77,14 @@ func (x *commonPrm) SetXHeaders(hs []string) { x.xHeaders = hs } +func (x *commonPrm) calculateXHeaders() []string { + hs := x.xHeaders + if x.netmapEpoch != 0 { + hs = append(hs, sessionAPI.XHeaderNetmapEpoch, strconv.FormatUint(x.netmapEpoch, 10)) + } + return hs +} + type readPrmCommon struct { commonPrm } @@ -80,8 +92,8 @@ type readPrmCommon struct { // SetNetmapEpoch sets the epoch number to be used to locate the objectSDK. // // By default current epoch on the server will be used. -func (x *readPrmCommon) SetNetmapEpoch(_ uint64) { - // FIXME(@fyrchik): https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/465 +func (x *readPrmCommon) SetNetmapEpoch(epoch uint64) { + x.netmapEpoch = epoch } // GetObjectPrm groups parameters of GetObject operation. @@ -139,7 +151,7 @@ func GetObject(ctx context.Context, prm GetObjectPrm) (*GetObjectRes, error) { prm.ClientParams.Session = prm.tokenSession } - prm.ClientParams.XHeaders = prm.xHeaders + prm.ClientParams.XHeaders = prm.calculateXHeaders() prm.ClientParams.BearerToken = prm.tokenBearer prm.ClientParams.Local = prm.local prm.ClientParams.Key = prm.key @@ -233,7 +245,7 @@ func HeadObject(ctx context.Context, prm HeadObjectPrm) (*HeadObjectRes, error) prm.ClientParams.BearerToken = prm.tokenBearer prm.ClientParams.Local = prm.local - prm.ClientParams.XHeaders = prm.xHeaders + prm.ClientParams.XHeaders = prm.calculateXHeaders() cliRes, err := prm.cli.ObjectHead(ctx, prm.ClientParams) if err == nil { @@ -326,7 +338,7 @@ func PayloadRange(ctx context.Context, prm PayloadRangePrm) (*PayloadRangeRes, e prm.ClientParams.Session = prm.tokenSession } - prm.ClientParams.XHeaders = prm.xHeaders + prm.ClientParams.XHeaders = prm.calculateXHeaders() prm.ClientParams.BearerToken = prm.tokenBearer prm.ClientParams.Local = prm.local prm.ClientParams.Length = prm.ln @@ -390,7 +402,7 @@ func PutObject(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, error) { defer span.End() prmCli := client.PrmObjectPutInit{ - XHeaders: prm.xHeaders, + XHeaders: prm.calculateXHeaders(), BearerToken: prm.tokenBearer, Session: prm.tokenSession, Local: true, @@ -437,7 +449,7 @@ func PutObjectSingle(ctx context.Context, prm PutObjectPrm) (*PutObjectRes, erro } prmCli := client.PrmObjectPutSingle{ - XHeaders: prm.xHeaders, + XHeaders: prm.calculateXHeaders(), BearerToken: prm.tokenBearer, Session: prm.tokenSession, Local: true, @@ -496,7 +508,7 @@ func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes prm.cliPrm.Local = prm.local prm.cliPrm.Session = prm.tokenSession prm.cliPrm.BearerToken = prm.tokenBearer - prm.cliPrm.XHeaders = prm.xHeaders + prm.cliPrm.XHeaders = prm.calculateXHeaders() prm.cliPrm.Key = prm.key rdr, err := prm.cli.ObjectSearchInit(ctx, prm.cliPrm) From 9cd1bcef06b1c116910f6d9d2e4895332a628b92 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 23 Dec 2024 14:53:04 +0300 Subject: [PATCH 1654/1943] [#1512] object: Make raw `PutSingle` check status within response Signed-off-by: Airat Arifullin --- pkg/services/object/put/single.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 36b0bd54c..19d025ff8 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -29,6 +29,7 @@ import ( sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/signature" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/tzhash/tz" @@ -353,6 +354,9 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, err = fmt.Errorf("response verification failed: %w", err) } + st := apistatus.FromStatusV2(resp.GetMetaHeader().GetStatus()) + err = apistatus.ErrFromStatus(st) + return }) From e44782473a737daac7f05b0b491dc756de31391d Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 24 Dec 2024 21:27:45 +0300 Subject: [PATCH 1655/1943] [#1512] object: Fix `writePart` for EC-container * Immediatly return after `ObjectAlreadyRemoved` error. Signed-off-by: Airat Arifullin --- pkg/services/object/common/writer/ec.go | 3 +++ pkg/services/object/put/single.go | 1 + 2 files changed, 4 insertions(+) diff --git a/pkg/services/object/common/writer/ec.go b/pkg/services/object/common/writer/ec.go index fdaa569da..94bcf6a32 100644 --- a/pkg/services/object/common/writer/ec.go +++ b/pkg/services/object/common/writer/ec.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" + clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" @@ -274,6 +275,8 @@ func (e *ECWriter) writePart(ctx context.Context, obj *objectSDK.Object, partIdx err := e.putECPartToNode(ctx, obj, node) if err == nil { return nil + } else if clientSDK.IsErrObjectAlreadyRemoved(err) { + return err } e.Config.Logger.Warn(ctx, logs.ECFailedToSaveECPart, zap.Stringer("part_address", object.AddressOf(obj)), zap.Stringer("parent_address", obj.ECHeader().Parent()), zap.Int("part_index", partIdx), diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 19d025ff8..5219e64d5 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -352,6 +352,7 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, err = signature.VerifyServiceMessage(resp) if err != nil { err = fmt.Errorf("response verification failed: %w", err) + return } st := apistatus.FromStatusV2(resp.GetMetaHeader().GetStatus()) From 0da998ef50472d8de4589bfd9105ce1892636c94 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 25 Dec 2024 22:12:41 +0300 Subject: [PATCH 1656/1943] [#1583] metabase: Skip expired objects in `ListWithCursor` Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/metabase/list.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index a7ff2222f..375d1cb1a 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -87,7 +87,8 @@ type CountAliveObjectsInContainerPrm struct { } // ListWithCursor lists physical objects available in metabase starting from -// cursor. Includes objects of all types. Does not include inhumed objects. +// cursor. Includes objects of all types. Does not include inhumed and expired +// objects. // Use cursor value from response for consecutive requests. // // Returns ErrEndOfListing if there are no more objects to return or count @@ -143,6 +144,8 @@ func (db *DB) listWithCursor(tx *bbolt.Tx, result []objectcore.Info, count int, rawAddr := make([]byte, cidSize, addressKeySize) + currEpoch := db.epochState.CurrentEpoch() + loop: for ; name != nil; name, _ = c.Next() { cidRaw, prefix := parseContainerIDWithPrefix(&containerID, name) @@ -167,7 +170,7 @@ loop: if bkt != nil { copy(rawAddr, cidRaw) result, offset, cursor, err = selectNFromBucket(bkt, objType, graveyardBkt, garbageBkt, rawAddr, containerID, - result, count, cursor, threshold) + result, count, cursor, threshold, currEpoch) if err != nil { return nil, nil, err } @@ -212,6 +215,7 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket limit int, // stop listing at `limit` items in result cursor *Cursor, // start from cursor object threshold bool, // ignore cursor and start immediately + currEpoch uint64, ) ([]objectcore.Info, []byte, *Cursor, error) { if cursor == nil { cursor = new(Cursor) @@ -243,13 +247,19 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket continue } + var o objectSDK.Object + if err := o.Unmarshal(bytes.Clone(v)); err != nil { + return nil, nil, nil, err + } + + expEpoch, hasExpEpoch := hasExpirationEpoch(&o) + if !objectLocked(bkt.Tx(), cnt, obj) && hasExpEpoch && expEpoch < currEpoch { + continue + } + var isLinkingObj bool var ecInfo *objectcore.ECInfo if objType == objectSDK.TypeRegular { - var o objectSDK.Object - if err := o.Unmarshal(bytes.Clone(v)); err != nil { - return nil, nil, nil, err - } isLinkingObj = isLinkObject(&o) ecHeader := o.ECHeader() if ecHeader != nil { From fa08bfa553ee013b65076c38146f83134ae2f06c Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 25 Dec 2024 22:16:11 +0300 Subject: [PATCH 1657/1943] [#1583] metabase/test: Update `TestLisObjectsWithCursor` Update this test following recent changes to ensure that `(*DB).ListWithCursor` skips expired objects. Signed-off-by: Aleksey Savchuk --- .../metabase/list_test.go | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 1d8beb175..817b22010 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -3,14 +3,17 @@ package meta_test import ( "context" "errors" + "strconv" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" @@ -71,14 +74,16 @@ func benchmarkListWithCursor(b *testing.B, db *meta.DB, batchSize int) { func TestLisObjectsWithCursor(t *testing.T) { t.Parallel() - db := newDB(t) - defer func() { require.NoError(t, db.Close(context.Background())) }() - const ( + currEpoch = 100 + expEpoch = currEpoch - 1 containers = 5 - total = containers * 4 // regular + ts + child + lock + total = containers * 6 // regular + ts + child + lock + non-expired regular + locked expired ) + db := newDB(t, meta.WithEpochState(epochState{currEpoch})) + defer func() { require.NoError(t, db.Close(context.Background())) }() + expected := make([]object.Info, 0, total) // fill metabase with objects @@ -127,6 +132,26 @@ func TestLisObjectsWithCursor(t *testing.T) { err = putBig(db, child) require.NoError(t, err) expected = append(expected, object.Info{Address: object.AddressOf(child), Type: objectSDK.TypeRegular}) + + // add expired object (do not include into expected) + obj = testutil.GenerateObjectWithCID(containerID) + testutil.AddAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.Itoa(expEpoch)) + require.NoError(t, metaPut(db, obj, nil)) + + // add non-expired object (include into expected) + obj = testutil.GenerateObjectWithCID(containerID) + testutil.AddAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.Itoa(currEpoch)) + require.NoError(t, metaPut(db, obj, nil)) + expected = append(expected, object.Info{Address: object.AddressOf(obj), Type: objectSDK.TypeRegular}) + + // add locked expired object (include into expected) + obj = testutil.GenerateObjectWithCID(containerID) + objID := oidtest.ID() + obj.SetID(objID) + testutil.AddAttribute(obj, objectV2.SysAttributeExpEpoch, strconv.Itoa(expEpoch)) + require.NoError(t, metaPut(db, obj, nil)) + require.NoError(t, db.Lock(context.Background(), containerID, oidtest.ID(), []oid.ID{objID})) + expected = append(expected, object.Info{Address: object.AddressOf(obj), Type: objectSDK.TypeRegular}) } t.Run("success with various count", func(t *testing.T) { From 6fe34d266a2cd8ff35929053610cb36851263cb4 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Wed, 25 Dec 2024 15:32:24 +0300 Subject: [PATCH 1658/1943] [#1577] morph: Fix typo Signed-off-by: Ekaterina Lebedeva --- pkg/morph/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 37599e696..01fcc98e5 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -210,7 +210,7 @@ func (c *Client) Invoke(ctx context.Context, contract util.Uint160, fee fixedn.F // TestInvokeIterator invokes contract method returning an iterator and executes cb on each element. // If cb returns an error, the session is closed and this error is returned as-is. -// If the remove neo-go node does not support sessions, `unwrap.ErrNoSessionID` is returned. +// If the remote neo-go node does not support sessions, `unwrap.ErrNoSessionID` is returned. // batchSize is the number of items to prefetch: if the number of items in the iterator is less than batchSize, no session will be created. // The default batchSize is 100, the default limit from neo-go. func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int, contract util.Uint160, method string, args ...interface{}) error { From 242f0095d08e54124807f14b30f5ee3adfe9eb6b Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Wed, 25 Dec 2024 14:25:29 +0300 Subject: [PATCH 1659/1943] [#1577] container: Reduce iterations through container list * Separated iteration through container ids from `ContainersOf()` so that it could be reused. * When listing containers we used to iterate through the the whole list of containers twice: first when reading from a contract, then when sending them. Now we can send batches of containers when reading from the contract. Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-node/container.go | 5 +++ pkg/morph/client/container/containers_of.go | 50 +++++++++++---------- pkg/morph/client/container/list.go | 44 +++++++++++------- pkg/services/container/morph/executor.go | 48 ++++++++++++++------ 4 files changed, 94 insertions(+), 53 deletions(-) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index fb2550a03..be0acf738 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -222,6 +222,7 @@ type morphContainerReader struct { lister interface { ContainersOf(*user.ID) ([]cid.ID, error) + IterateContainersOf(*user.ID, func(cid.ID) error) error } } @@ -237,6 +238,10 @@ func (x *morphContainerReader) ContainersOf(id *user.ID) ([]cid.ID, error) { return x.lister.ContainersOf(id) } +func (x *morphContainerReader) IterateContainersOf(id *user.ID, processCID func(cid.ID) error) error { + return x.lister.IterateContainersOf(id, processCID) +} + type morphContainerWriter struct { neoClient *cntClient.Client } diff --git a/pkg/morph/client/container/containers_of.go b/pkg/morph/client/container/containers_of.go index 5fe15be0d..6381a14c0 100644 --- a/pkg/morph/client/container/containers_of.go +++ b/pkg/morph/client/container/containers_of.go @@ -2,9 +2,7 @@ package container import ( "errors" - "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" @@ -16,27 +14,37 @@ import ( // // If remote RPC does not support neo-go session API, fallback to List() method. func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) { - var rawID []byte + var cidList []cid.ID + var err error + cb := func(id cid.ID) error { + cidList = append(cidList, id) + return nil + } + if err = c.IterateContainersOf(idUser, cb); err != nil { + return nil, err + } + return cidList, nil +} + +// iterateContainers iterates over a list of container identifiers +// belonging to the specified user of FrostFS system and executes +// `cb` on each element. If idUser is nil, calls it on the list of all containers. +func (c *Client) IterateContainersOf(idUser *user.ID, cb func(item cid.ID) error) error { + var rawID []byte if idUser != nil { rawID = idUser.WalletBytes() } - var cidList []cid.ID - cb := func(item stackitem.Item) error { - rawID, err := client.BytesFromStackItem(item) + cnrHash := c.client.ContractAddress() + itemCb := func(item stackitem.Item) error { + id, err := getCIDfromStackItem(item) if err != nil { - return fmt.Errorf("get byte array from stack item (%s): %w", containersOfMethod, err) + return err } - - var id cid.ID - - err = id.Decode(rawID) - if err != nil { - return fmt.Errorf("decode container ID: %w", err) + if err = cb(id); err != nil { + return err } - - cidList = append(cidList, id) return nil } @@ -49,14 +57,10 @@ func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) { // 512 is big enough value and it is beautiful. const batchSize = 512 - cnrHash := c.client.ContractAddress() - err := c.client.Morph().TestInvokeIterator(cb, batchSize, cnrHash, containersOfMethod, rawID) - if err != nil { - if errors.Is(err, unwrap.ErrNoSessionID) { - return c.list(idUser) - } - return nil, err + err := c.client.Morph().TestInvokeIterator(itemCb, batchSize, cnrHash, containersOfMethod, rawID) + if err != nil && errors.Is(err, unwrap.ErrNoSessionID) { + return c.iterate(idUser, cb) } - return cidList, nil + return err } diff --git a/pkg/morph/client/container/list.go b/pkg/morph/client/container/list.go index d9719aedd..78ea8278f 100644 --- a/pkg/morph/client/container/list.go +++ b/pkg/morph/client/container/list.go @@ -6,15 +6,16 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" ) -// list returns a list of container identifiers belonging +// iterate iterates through a list of container identifiers belonging // to the specified user of FrostFS system. The list is composed // through Container contract call. // -// Returns the identifiers of all FrostFS containers if pointer +// Iterates through the identifiers of all FrostFS containers if pointer // to user identifier is nil. -func (c *Client) list(idUser *user.ID) ([]cid.ID, error) { +func (c *Client) iterate(idUser *user.ID, cb func(cid.ID) error) error { var rawID []byte if idUser != nil { @@ -27,32 +28,41 @@ func (c *Client) list(idUser *user.ID) ([]cid.ID, error) { res, err := c.client.TestInvoke(prm) if err != nil { - return nil, fmt.Errorf("test invoke (%s): %w", listMethod, err) + return fmt.Errorf("test invoke (%s): %w", listMethod, err) } else if ln := len(res); ln != 1 { - return nil, fmt.Errorf("unexpected stack item count (%s): %d", listMethod, ln) + return fmt.Errorf("unexpected stack item count (%s): %d", listMethod, ln) } res, err = client.ArrayFromStackItem(res[0]) if err != nil { - return nil, fmt.Errorf("get stack item array from stack item (%s): %w", listMethod, err) + return fmt.Errorf("get stack item array from stack item (%s): %w", listMethod, err) } - cidList := make([]cid.ID, 0, len(res)) for i := range res { - rawID, err := client.BytesFromStackItem(res[i]) + id, err := getCIDfromStackItem(res[i]) if err != nil { - return nil, fmt.Errorf("get byte array from stack item (%s): %w", listMethod, err) + return err } - var id cid.ID - - err = id.Decode(rawID) - if err != nil { - return nil, fmt.Errorf("decode container ID: %w", err) + if err = cb(id); err != nil { + return err } - - cidList = append(cidList, id) } - return cidList, nil + return nil +} + +func getCIDfromStackItem(item stackitem.Item) (cid.ID, error) { + rawID, err := client.BytesFromStackItem(item) + if err != nil { + return cid.ID{}, fmt.Errorf("get byte array from stack item (%s): %w", listMethod, err) + } + + var id cid.ID + + err = id.Decode(rawID) + if err != nil { + return cid.ID{}, fmt.Errorf("decode container ID: %w", err) + } + return id, nil } diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index e9d1606f1..cadf92e19 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -30,6 +30,7 @@ type Reader interface { // to the specified user of FrostFS system. Returns the identifiers // of all FrostFS containers if pointer to owner identifier is nil. ContainersOf(*user.ID) ([]cid.ID, error) + IterateContainersOf(*user.ID, func(cid.ID) error) error } // Writer is an interface of container storage updater. @@ -201,7 +202,7 @@ func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) return res, nil } -func (s *morphExecutor) ListStream(_ context.Context, req *container.ListStreamRequest, stream containerSvc.ListStream) error { +func (s *morphExecutor) ListStream(ctx context.Context, req *container.ListStreamRequest, stream containerSvc.ListStream) error { body := req.GetBody() idV2 := body.GetOwnerID() if idV2 == nil { @@ -215,20 +216,41 @@ func (s *morphExecutor) ListStream(_ context.Context, req *container.ListStreamR return fmt.Errorf("invalid user ID: %w", err) } - cnrs, err := s.rdr.ContainersOf(&id) - if err != nil { - return err - } - - cidList := make([]refs.ContainerID, len(cnrs)) - for i := range cnrs { - cnrs[i].WriteToV2(&cidList[i]) - } - resBody := new(container.ListStreamResponseBody) - resBody.SetContainerIDs(cidList) r := new(container.ListStreamResponse) r.SetBody(resBody) - return stream.Send(r) + var cidList []refs.ContainerID + + // Amount of containers to send at once. + const batchSize = 1000 + + processCID := func(id cid.ID) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + var refID refs.ContainerID + id.WriteToV2(&refID) + cidList = append(cidList, refID) + if len(cidList) == batchSize { + r.GetBody().SetContainerIDs(cidList) + cidList = cidList[:0] + return stream.Send(r) + } + return nil + } + + if err = s.rdr.IterateContainersOf(&id, processCID); err != nil { + return err + } + + if len(cidList) > 0 { + r.GetBody().SetContainerIDs(cidList) + return stream.Send(r) + } + + return nil } From c0221d76e6534a7c264c24a7b1c6af73dad2b5ed Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 27 Dec 2024 15:31:06 +0300 Subject: [PATCH 1660/1943] [#1577] node/container: Fix typo Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-node/config/container/container.go | 2 +- pkg/morph/client/container/containers_of.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-node/config/container/container.go b/cmd/frostfs-node/config/container/container.go index b0b8043d6..1cd64a6f8 100644 --- a/cmd/frostfs-node/config/container/container.go +++ b/cmd/frostfs-node/config/container/container.go @@ -6,7 +6,7 @@ const ( subsection = "container" listStreamSubsection = "list_stream" - // ContainerBatchSizeDefault represents he maximum amount of containers to send via stream at once. + // ContainerBatchSizeDefault represents the maximum amount of containers to send via stream at once. ContainerBatchSizeDefault = 1000 ) diff --git a/pkg/morph/client/container/containers_of.go b/pkg/morph/client/container/containers_of.go index 6381a14c0..074a586be 100644 --- a/pkg/morph/client/container/containers_of.go +++ b/pkg/morph/client/container/containers_of.go @@ -36,7 +36,6 @@ func (c *Client) IterateContainersOf(idUser *user.ID, cb func(item cid.ID) error rawID = idUser.WalletBytes() } - cnrHash := c.client.ContractAddress() itemCb := func(item stackitem.Item) error { id, err := getCIDfromStackItem(item) if err != nil { @@ -57,6 +56,7 @@ func (c *Client) IterateContainersOf(idUser *user.ID, cb func(item cid.ID) error // 512 is big enough value and it is beautiful. const batchSize = 512 + cnrHash := c.client.ContractAddress() err := c.client.Morph().TestInvokeIterator(itemCb, batchSize, cnrHash, containersOfMethod, rawID) if err != nil && errors.Is(err, unwrap.ErrNoSessionID) { return c.iterate(idUser, cb) From d7fcc5ce304c0c8e971c4edccf778cf6b8693d52 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 28 Dec 2024 09:03:07 +0300 Subject: [PATCH 1661/1943] [#1586] objsvc: Allow to send search response in multiple messages Previously, `ln` was only set once, so search has really worked for small number of objects. Fix panic: ``` panic: runtime error: slice bounds out of range [:43690] with capacity 21238 goroutine 6859775 [running]: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object.(*searchStreamMsgSizeCtrl).Send(0xc001eec8d0, 0xc005734000) git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/transport_splitter.go:173 +0x1f0 git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search/v2.(*streamWriter).WriteIDs(0xc000520320, {0xc00eb1a000, 0x4fd9c, 0x7fd6475a9a68?}) git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search/v2/streamer.go:28 +0x155 git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search.(*uniqueIDWriter).WriteIDs(0xc001386420, {0xc00eb1a000?, 0xc0013ea9c0?, 0x113eef3?}) git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search/util.go:62 +0x202 git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search.(*execCtx).writeIDList(0xc00011aa38?, {0xc00eb1a000?, 0xc001eec9f0?, 0xc0008f4380?}) git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search/exec.go:68 +0x91 git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search.(*execCtx).executeLocal(0xc0008f4380, {0x176c538, 0xc001eec9f0}) git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/search/local.go:18 +0x16b ``` Signed-off-by: Evgenii Stratonikov --- pkg/services/object/transport_splitter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/services/object/transport_splitter.go b/pkg/services/object/transport_splitter.go index 0b3676edb..b446d3605 100644 --- a/pkg/services/object/transport_splitter.go +++ b/pkg/services/object/transport_splitter.go @@ -162,13 +162,13 @@ func (s *searchStreamMsgSizeCtrl) Send(resp *object.SearchResponse) error { var newResp *object.SearchResponse - for ln := uint64(len(ids)); ; { + for { if newResp == nil { newResp = new(object.SearchResponse) newResp.SetBody(body) } - cut := min(s.addrAmount, ln) + cut := min(s.addrAmount, uint64(len(ids))) body.SetIDList(ids[:cut]) newResp.SetMetaHeader(resp.GetMetaHeader()) From cddcd73f04cbb1c1894e45dab62dc2070f3931d5 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 28 Dec 2024 14:36:18 +0300 Subject: [PATCH 1662/1943] [#1590] adm: Make `--account` flag required in `proxy-*` commands Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/proxy/root.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy/root.go b/cmd/frostfs-adm/internal/modules/morph/proxy/root.go index 1854c8d2b..5f07e5862 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/root.go @@ -30,12 +30,14 @@ var ( func initProxyAddAccount() { AddAccountCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) AddAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") + _ = AddAccountCmd.MarkFlagRequired(accountAddressFlag) AddAccountCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initProxyRemoveAccount() { RemoveAccountCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) RemoveAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") + _ = AddAccountCmd.MarkFlagRequired(accountAddressFlag) RemoveAccountCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } From dc410fca901dc0f4628fb3d64d83be32525414bd Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 28 Dec 2024 14:37:35 +0300 Subject: [PATCH 1663/1943] [#1590] adm: Accept many accounts in `proxy-*` commands Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/proxy/proxy.go | 31 +++++++++++++------ .../internal/modules/morph/proxy/root.go | 4 +-- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go index cb575b657..24cda45a6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/proxy.go @@ -20,23 +20,32 @@ const ( accountAddressFlag = "account" ) +func parseAddresses(cmd *cobra.Command) []util.Uint160 { + var addrs []util.Uint160 + + accs, _ := cmd.Flags().GetStringArray(accountAddressFlag) + for _, acc := range accs { + addr, err := address.StringToUint160(acc) + commonCmd.ExitOnErr(cmd, "invalid account: %w", err) + + addrs = append(addrs, addr) + } + return addrs +} + func addProxyAccount(cmd *cobra.Command, _ []string) { - acc, _ := cmd.Flags().GetString(accountAddressFlag) - addr, err := address.StringToUint160(acc) - commonCmd.ExitOnErr(cmd, "invalid account: %w", err) - err = processAccount(cmd, addr, "addAccount") + addrs := parseAddresses(cmd) + err := processAccount(cmd, addrs, "addAccount") commonCmd.ExitOnErr(cmd, "processing error: %w", err) } func removeProxyAccount(cmd *cobra.Command, _ []string) { - acc, _ := cmd.Flags().GetString(accountAddressFlag) - addr, err := address.StringToUint160(acc) - commonCmd.ExitOnErr(cmd, "invalid account: %w", err) - err = processAccount(cmd, addr, "removeAccount") + addrs := parseAddresses(cmd) + err := processAccount(cmd, addrs, "removeAccount") commonCmd.ExitOnErr(cmd, "processing error: %w", err) } -func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error { +func processAccount(cmd *cobra.Command, addrs []util.Uint160, method string) error { wCtx, err := helper.NewInitializeContext(cmd, viper.GetViper()) if err != nil { return fmt.Errorf("can't initialize context: %w", err) @@ -54,7 +63,9 @@ func processAccount(cmd *cobra.Command, addr util.Uint160, method string) error } bw := io.NewBufBinWriter() - emit.AppCall(bw.BinWriter, proxyHash, method, callflag.All, addr) + for _, addr := range addrs { + emit.AppCall(bw.BinWriter, proxyHash, method, callflag.All, addr) + } if err := wCtx.SendConsensusTx(bw.Bytes()); err != nil { return err diff --git a/cmd/frostfs-adm/internal/modules/morph/proxy/root.go b/cmd/frostfs-adm/internal/modules/morph/proxy/root.go index 5f07e5862..ad89af2b5 100644 --- a/cmd/frostfs-adm/internal/modules/morph/proxy/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/proxy/root.go @@ -29,14 +29,14 @@ var ( func initProxyAddAccount() { AddAccountCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) - AddAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") + AddAccountCmd.Flags().StringArray(accountAddressFlag, nil, "Wallet address string") _ = AddAccountCmd.MarkFlagRequired(accountAddressFlag) AddAccountCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } func initProxyRemoveAccount() { RemoveAccountCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) - RemoveAccountCmd.Flags().String(accountAddressFlag, "", "Wallet address string") + RemoveAccountCmd.Flags().StringArray(accountAddressFlag, nil, "Wallet address string") _ = AddAccountCmd.MarkFlagRequired(accountAddressFlag) RemoveAccountCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) } From 5ccb3394b49c29ea0e9186849bfc9a909e74f5b8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 9 Jan 2025 14:30:17 +0300 Subject: [PATCH 1664/1943] [#1592] go.mod: Update sdk-go Signed-off-by: Evgenii Stratonikov --- go.mod | 32 +++++++++++++-------------- go.sum | 68 +++++++++++++++++++++++++++++++--------------------------- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 8f4053872..267e411fb 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241210104938-c4463df8d467 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250109084609-328d214d2d76 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 @@ -27,7 +27,7 @@ require ( github.com/klauspost/compress v1.17.4 github.com/mailru/easyjson v0.7.7 github.com/mr-tron/base58 v1.2.0 - github.com/multiformats/go-multiaddr v0.12.1 + github.com/multiformats/go-multiaddr v0.14.0 github.com/nspcc-dev/neo-go v0.106.3 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 @@ -40,15 +40,15 @@ require ( github.com/ssgreg/journald v1.0.0 github.com/stretchr/testify v1.9.0 go.etcd.io/bbolt v1.3.10 - go.opentelemetry.io/otel v1.28.0 - go.opentelemetry.io/otel/trace v1.28.0 + go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/sync v0.7.0 - golang.org/x/sys v0.22.0 - golang.org/x/term v0.21.0 - google.golang.org/grpc v1.66.2 - google.golang.org/protobuf v1.34.2 + golang.org/x/sync v0.10.0 + golang.org/x/sys v0.28.0 + golang.org/x/term v0.27.0 + google.golang.org/grpc v1.69.2 + google.golang.org/protobuf v1.36.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -119,15 +119,15 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/text v0.21.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect gopkg.in/ini.v1 v1.67.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index d63396202..935d3c56a 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241210104938-c4463df8d467 h1:MH9uHZFZNyUCL+YChiDcVeXPjhTDcFDeoGr8Mc8NY9M= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20241210104938-c4463df8d467/go.mod h1:eoK7+KZQ9GJxbzIs6vTnoUJqFDppavInLRHaN4MYgZg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250109084609-328d214d2d76 h1:wzvSJIiS+p9qKfl3eg1oH6qlrjaEWiqTc/iMDKG3Ml4= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250109084609-328d214d2d76/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= @@ -106,6 +106,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -188,8 +190,8 @@ github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aG github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.12.1 h1:vm+BA/WZA8QZDp1pF1FWhi5CT3g1tbi5GJmqpb6wnlk= -github.com/multiformats/go-multiaddr v0.12.1/go.mod h1:7mPkiBMmLeFipt+nNSq9pHZUeJSt8lHBgH6yhj0YQzE= +github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU= +github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= @@ -290,20 +292,22 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 h1:EVSnY9JbEEW92bEkIYOVMw4q1WJxIAGoFTrtYOzWuRQ= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0/go.mod h1:Ea1N1QQryNXpCD0I1fdLibBAIpQuBkznMmkdKrapk1Y= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -318,8 +322,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -339,16 +343,16 @@ golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -375,16 +379,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -392,8 +396,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -406,12 +410,12 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= -google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= -google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= +google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -420,8 +424,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 3900b92927bb0343cf7282bf91e817f8cd44f827 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 26 Dec 2024 14:12:35 +0300 Subject: [PATCH 1665/1943] Revert "[#1492] metabase: Ensure Unmarshal() is called on a cloned slice" This reverts commit 8ed7a676d50e24489e2abeb5269d6eb3332df1f8. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/exists.go | 3 +-- pkg/local_object_storage/metabase/get.go | 11 +++++------ pkg/local_object_storage/metabase/iterators.go | 3 +-- pkg/local_object_storage/metabase/list.go | 4 ++-- pkg/local_object_storage/metabase/put.go | 3 +-- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index 3133c5480..962108a76 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -1,7 +1,6 @@ package meta import ( - "bytes" "context" "fmt" "time" @@ -227,7 +226,7 @@ func getSplitInfo(tx *bbolt.Tx, cnr cid.ID, key []byte) (*objectSDK.SplitInfo, e splitInfo := objectSDK.NewSplitInfo() - err := splitInfo.Unmarshal(bytes.Clone(rawSplitInfo)) + err := splitInfo.Unmarshal(rawSplitInfo) if err != nil { return nil, fmt.Errorf("unmarshal split info from root index: %w", err) } diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index af274b245..615add1af 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -1,7 +1,6 @@ package meta import ( - "bytes" "context" "fmt" "time" @@ -112,7 +111,7 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b // check in primary index data := getFromBucket(tx, primaryBucketName(cnr, bucketName), key) if len(data) != 0 { - return obj, obj.Unmarshal(bytes.Clone(data)) + return obj, obj.Unmarshal(data) } data = getFromBucket(tx, ecInfoBucketName(cnr, bucketName), key) @@ -123,13 +122,13 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b // if not found then check in tombstone index data = getFromBucket(tx, tombstoneBucketName(cnr, bucketName), key) if len(data) != 0 { - return obj, obj.Unmarshal(bytes.Clone(data)) + return obj, obj.Unmarshal(data) } // if not found then check in locker index data = getFromBucket(tx, bucketNameLockers(cnr, bucketName), key) if len(data) != 0 { - return obj, obj.Unmarshal(bytes.Clone(data)) + return obj, obj.Unmarshal(data) } // if not found then check if object is a virtual @@ -185,7 +184,7 @@ func getVirtualObject(tx *bbolt.Tx, cnr cid.ID, key []byte, raw bool) (*objectSD child := objectSDK.New() - err = child.Unmarshal(bytes.Clone(data)) + err = child.Unmarshal(data) if err != nil { return nil, fmt.Errorf("unmarshal child with parent: %w", err) } @@ -219,7 +218,7 @@ func getECInfoError(tx *bbolt.Tx, cnr cid.ID, data []byte) error { objData := getFromBucket(tx, primaryBucketName(cnr, make([]byte, bucketKeySize)), key) if len(objData) != 0 { obj := objectSDK.New() - if err := obj.Unmarshal(bytes.Clone(objData)); err != nil { + if err := obj.Unmarshal(objData); err != nil { return err } chunk := objectSDK.ECChunk{} diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index 0d438e102..9cccd7dad 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -1,7 +1,6 @@ package meta import ( - "bytes" "context" "errors" "strconv" @@ -130,7 +129,7 @@ func iteratePhyObjects(tx *bbolt.Tx, f func(cid.ID, oid.ID, *objectSDK.Object) e } return b.ForEach(func(k, v []byte) error { - if oid.Decode(k) == nil && obj.Unmarshal(bytes.Clone(v)) == nil { + if oid.Decode(k) == nil && obj.Unmarshal(v) == nil { return f(cid, oid, obj) } diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 375d1cb1a..f488c3ced 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -248,7 +248,7 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket } var o objectSDK.Object - if err := o.Unmarshal(bytes.Clone(v)); err != nil { + if err := o.Unmarshal(v); err != nil { return nil, nil, nil, err } @@ -423,7 +423,7 @@ func (db *DB) iterateOverObjectsInContainer(ctx context.Context, tx *bbolt.Tx, p var ecInfo *objectcore.ECInfo if prm.ObjectType == objectSDK.TypeRegular { var o objectSDK.Object - if err := o.Unmarshal(bytes.Clone(v)); err != nil { + if err := o.Unmarshal(v); err != nil { return err } isLinkingObj = isLinkObject(&o) diff --git a/pkg/local_object_storage/metabase/put.go b/pkg/local_object_storage/metabase/put.go index 16918c4d9..5e1bbfe9e 100644 --- a/pkg/local_object_storage/metabase/put.go +++ b/pkg/local_object_storage/metabase/put.go @@ -1,7 +1,6 @@ package meta import ( - "bytes" "context" "encoding/binary" "errors" @@ -320,7 +319,7 @@ func updateSplitInfoIndex(tx *bbolt.Tx, objKey []byte, cnr cid.ID, bucketName [] return si.Marshal() default: oldSI := objectSDK.NewSplitInfo() - if err := oldSI.Unmarshal(bytes.Clone(old)); err != nil { + if err := oldSI.Unmarshal(old); err != nil { return nil, err } si = util.MergeSplitInfo(si, oldSI) From 8a658de0b28835fcf55a7c611eed497dd1e253ee Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 20 Dec 2024 11:40:34 +0300 Subject: [PATCH 1666/1943] [#1506] ape: Do not create cosigners slice on each contract invocation Signed-off-by: Evgenii Stratonikov --- pkg/ape/contract_storage/proxy.go | 47 +++++++++++++------------------ 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/pkg/ape/contract_storage/proxy.go b/pkg/ape/contract_storage/proxy.go index 953b91a79..8ae3f5203 100644 --- a/pkg/ape/contract_storage/proxy.go +++ b/pkg/ape/contract_storage/proxy.go @@ -31,9 +31,7 @@ type RPCActorProvider interface { type ProxyVerificationContractStorage struct { rpcActorProvider RPCActorProvider - acc *wallet.Account - - proxyScriptHash util.Uint160 + cosigners []actor.SignerAccount policyScriptHash util.Uint160 } @@ -41,12 +39,27 @@ type ProxyVerificationContractStorage struct { var _ ProxyAdaptedContractStorage = (*ProxyVerificationContractStorage)(nil) func NewProxyVerificationContractStorage(rpcActorProvider RPCActorProvider, key *keys.PrivateKey, proxyScriptHash, policyScriptHash util.Uint160) *ProxyVerificationContractStorage { + acc := wallet.NewAccountFromPrivateKey(key) return &ProxyVerificationContractStorage{ rpcActorProvider: rpcActorProvider, - acc: wallet.NewAccountFromPrivateKey(key), - - proxyScriptHash: proxyScriptHash, + cosigners: []actor.SignerAccount{ + { + Signer: transaction.Signer{ + Account: proxyScriptHash, + Scopes: transaction.CustomContracts, + AllowedContracts: []util.Uint160{policyScriptHash}, + }, + Account: notary.FakeContractAccount(proxyScriptHash), + }, + { + Signer: transaction.Signer{ + Account: acc.Contract.ScriptHash(), + Scopes: transaction.CalledByEntry, + }, + Account: acc, + }, + }, policyScriptHash: policyScriptHash, } @@ -64,7 +77,7 @@ func (n *contractStorageActorAdapter) GetRPCInvoker() invoker.RPCInvoke { func (contractStorage *ProxyVerificationContractStorage) newContractStorageActor() (policy_morph.ContractStorageActor, error) { rpcActor := contractStorage.rpcActorProvider.GetRPCActor() - act, err := actor.New(rpcActor, cosigners(contractStorage.acc, contractStorage.proxyScriptHash, contractStorage.policyScriptHash)) + act, err := actor.New(rpcActor, contractStorage.cosigners) if err != nil { return nil, err } @@ -106,23 +119,3 @@ func (contractStorage *ProxyVerificationContractStorage) ListMorphRuleChains(nam } return policy_morph.NewContractStorage(contractStorageActor, contractStorage.policyScriptHash).ListMorphRuleChains(name, target) } - -func cosigners(acc *wallet.Account, proxyScriptHash, policyScriptHash util.Uint160) []actor.SignerAccount { - return []actor.SignerAccount{ - { - Signer: transaction.Signer{ - Account: proxyScriptHash, - Scopes: transaction.CustomContracts, - AllowedContracts: []util.Uint160{policyScriptHash}, - }, - Account: notary.FakeContractAccount(proxyScriptHash), - }, - { - Signer: transaction.Signer{ - Account: acc.Contract.ScriptHash(), - Scopes: transaction.CalledByEntry, - }, - Account: acc, - }, - } -} From 85af6bcd5c31f43672d88345f730e7edb2b1de2a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 20 Dec 2024 11:47:28 +0300 Subject: [PATCH 1667/1943] [#1506] ape: Use contract reader in ListMorphRuleChains() Signed-off-by: Evgenii Stratonikov --- pkg/ape/contract_storage/proxy.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/ape/contract_storage/proxy.go b/pkg/ape/contract_storage/proxy.go index 8ae3f5203..8cbb1cce9 100644 --- a/pkg/ape/contract_storage/proxy.go +++ b/pkg/ape/contract_storage/proxy.go @@ -111,11 +111,16 @@ func (contractStorage *ProxyVerificationContractStorage) RemoveMorphRuleChain(na // ListMorphRuleChains lists morph rule chains from Policy contract using both Proxy contract and storage account as consigners. func (contractStorage *ProxyVerificationContractStorage) ListMorphRuleChains(name chain.Name, target engine.Target) ([]*chain.Chain, error) { - // contractStorageActor is reconstructed per each method invocation because RPCActor's (that is, basically, WSClient) connection may get invalidated, but - // ProxyVerificationContractStorage does not manage reconnections. - contractStorageActor, err := contractStorage.newContractStorageActor() - if err != nil { - return nil, err - } - return policy_morph.NewContractStorage(contractStorageActor, contractStorage.policyScriptHash).ListMorphRuleChains(name, target) + rpcActor := contractStorage.rpcActorProvider.GetRPCActor() + inv := &invokerAdapter{Invoker: invoker.New(rpcActor, nil), rpcInvoker: rpcActor} + return policy_morph.NewContractStorageReader(inv, contractStorage.policyScriptHash).ListMorphRuleChains(name, target) +} + +type invokerAdapter struct { + *invoker.Invoker + rpcInvoker invoker.RPCInvoke +} + +func (n *invokerAdapter) GetRPCInvoker() invoker.RPCInvoke { + return n.rpcInvoker } From 198aaebc94495b992a2eb143f501b14fde4ca641 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 28 Dec 2024 10:20:53 +0300 Subject: [PATCH 1668/1943] [#1506] morph: Simplify WaitTxHalt() signature Avoid dependency on `morph/client` package because of `InvokeRes`. Make signature resemble `WaitAny()` method of `waiter.Waiter` from neo-go. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/morph.go | 2 +- cmd/frostfs-node/netmap.go | 2 +- pkg/morph/client/waiter.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 5415da12a..657e22389 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -151,7 +151,7 @@ func makeNotaryDeposit(ctx context.Context, c *cfg) (util.Uint256, uint32, error } func waitNotaryDeposit(ctx context.Context, c *cfg, tx util.Uint256, vub uint32) error { - if err := c.cfgMorph.client.WaitTxHalt(ctx, client.InvokeRes{Hash: tx, VUB: vub}); err != nil { + if err := c.cfgMorph.client.WaitTxHalt(ctx, vub, tx); err != nil { return err } diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 2eb4cd132..a26fdd798 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -423,7 +423,7 @@ func (c *cfg) updateNetMapState(ctx context.Context, stateSetter func(*nmClient. if err != nil { return err } - return c.cfgNetmap.wrapper.Morph().WaitTxHalt(ctx, res) + return c.cfgNetmap.wrapper.Morph().WaitTxHalt(ctx, res.VUB, res.Hash) } type netInfo struct { diff --git a/pkg/morph/client/waiter.go b/pkg/morph/client/waiter.go index 962ec1bc2..5b9d2cbe0 100644 --- a/pkg/morph/client/waiter.go +++ b/pkg/morph/client/waiter.go @@ -33,13 +33,13 @@ func (w *waiterClient) GetVersion() (*result.Version, error) { // WaitTxHalt waits until transaction with the specified hash persists on the blockchain. // It also checks execution result to finish in HALT state. -func (c *Client) WaitTxHalt(ctx context.Context, p InvokeRes) error { +func (c *Client) WaitTxHalt(ctx context.Context, vub uint32, h util.Uint256) error { w, err := waiter.NewPollingBased(&waiterClient{c: c}) if err != nil { return fmt.Errorf("create tx waiter: %w", err) } - res, err := w.WaitAny(ctx, p.VUB, p.Hash) + res, err := w.WaitAny(ctx, vub, h) if err != nil { return fmt.Errorf("wait until tx persists: %w", err) } From f7e75b13b0b3796852d061c06a4d40cc6f5501d7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 20 Dec 2024 15:01:44 +0300 Subject: [PATCH 1669/1943] [#1506] ape_manager: Await tx persist before returning response Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/apemanager.go | 1 + pkg/services/apemanager/executor.go | 27 ++++++++++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-node/apemanager.go b/cmd/frostfs-node/apemanager.go index de3aed660..e761a1b14 100644 --- a/cmd/frostfs-node/apemanager.go +++ b/cmd/frostfs-node/apemanager.go @@ -19,6 +19,7 @@ func initAPEManagerService(c *cfg) { c.cfgObject.cfgAccessPolicyEngine.policyContractHash) execsvc := apemanager.New(c.cfgObject.cnrSource, contractStorage, + c.cfgMorph.client, apemanager.WithLogger(c.log)) sigsvc := apemanager.NewSignService(&c.key.PrivateKey, execsvc) auditSvc := apemanager.NewAuditService(sigsvc, c.log, c.audit) diff --git a/pkg/services/apemanager/executor.go b/pkg/services/apemanager/executor.go index cc792e23d..9d8f665af 100644 --- a/pkg/services/apemanager/executor.go +++ b/pkg/services/apemanager/executor.go @@ -22,6 +22,7 @@ import ( policy_engine "git.frostfs.info/TrueCloudLab/policy-engine/pkg/engine" "github.com/mr-tron/base58/base58" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" "go.uber.org/zap" ) @@ -34,6 +35,8 @@ type cfg struct { type Service struct { cfg + waiter Waiter + cnrSrc containercore.Source contractStorage ape_contract.ProxyAdaptedContractStorage @@ -41,11 +44,17 @@ type Service struct { type Option func(*cfg) -func New(cnrSrc containercore.Source, contractStorage ape_contract.ProxyAdaptedContractStorage, opts ...Option) *Service { +type Waiter interface { + WaitTxHalt(context.Context, uint32, util.Uint256) error +} + +func New(cnrSrc containercore.Source, contractStorage ape_contract.ProxyAdaptedContractStorage, waiter Waiter, opts ...Option) *Service { s := &Service{ cnrSrc: cnrSrc, contractStorage: contractStorage, + + waiter: waiter, } for i := range opts { @@ -84,7 +93,7 @@ func (s *Service) validateContainerTargetRequest(cid string, pubKey *keys.Public return nil } -func (s *Service) AddChain(_ context.Context, req *apemanagerV2.AddChainRequest) (*apemanagerV2.AddChainResponse, error) { +func (s *Service) AddChain(ctx context.Context, req *apemanagerV2.AddChainRequest) (*apemanagerV2.AddChainResponse, error) { pub, err := getSignaturePublicKey(req.GetVerificationHeader()) if err != nil { return nil, err @@ -116,7 +125,11 @@ func (s *Service) AddChain(_ context.Context, req *apemanagerV2.AddChainRequest) return nil, fmt.Errorf("unsupported target type: %s", targetType) } - if _, _, err = s.contractStorage.AddMorphRuleChain(apechain.Ingress, target, &chain); err != nil { + txHash, vub, err := s.contractStorage.AddMorphRuleChain(apechain.Ingress, target, &chain) + if err != nil { + return nil, err + } + if err := s.waiter.WaitTxHalt(ctx, vub, txHash); err != nil { return nil, err } @@ -129,7 +142,7 @@ func (s *Service) AddChain(_ context.Context, req *apemanagerV2.AddChainRequest) return resp, nil } -func (s *Service) RemoveChain(_ context.Context, req *apemanagerV2.RemoveChainRequest) (*apemanagerV2.RemoveChainResponse, error) { +func (s *Service) RemoveChain(ctx context.Context, req *apemanagerV2.RemoveChainRequest) (*apemanagerV2.RemoveChainResponse, error) { pub, err := getSignaturePublicKey(req.GetVerificationHeader()) if err != nil { return nil, err @@ -148,7 +161,11 @@ func (s *Service) RemoveChain(_ context.Context, req *apemanagerV2.RemoveChainRe return nil, fmt.Errorf("unsupported target type: %s", targetType) } - if _, _, err = s.contractStorage.RemoveMorphRuleChain(apechain.Ingress, target, req.GetBody().GetChainID()); err != nil { + txHash, vub, err := s.contractStorage.RemoveMorphRuleChain(apechain.Ingress, target, req.GetBody().GetChainID()) + if err != nil { + return nil, err + } + if err := s.waiter.WaitTxHalt(ctx, vub, txHash); err != nil { return nil, err } From ceac1c870915898770603208de2c1cb2d9a23db0 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 9 Jan 2025 20:52:24 +0300 Subject: [PATCH 1670/1943] [#1594] dev: Remove unused parameter 'FROSTFS_MORPH_INACTIVITY_TIMEOUT' Signed-off-by: Alexander Chuprov --- dev/.vscode-example/launch.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dev/.vscode-example/launch.json b/dev/.vscode-example/launch.json index 6abf5ecdc..b68ce4fa3 100644 --- a/dev/.vscode-example/launch.json +++ b/dev/.vscode-example/launch.json @@ -42,7 +42,6 @@ "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", - "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet01.json", "FROSTFS_NODE_WALLET_PASSWORD":"", "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8080", @@ -98,7 +97,6 @@ "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", - "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet02.json", "FROSTFS_NODE_WALLET_PASSWORD":"", "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8082", @@ -154,7 +152,6 @@ "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", - "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet03.json", "FROSTFS_NODE_WALLET_PASSWORD":"", "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8084", @@ -210,7 +207,6 @@ "FROSTFS_MORPH_DIAL_TIMEOUT":"30s", "FROSTFS_MORPH_RPC_ENDPOINT_0_ADDRESS":"ws://127.0.0.1:30333/ws", "FROSTFS_MORPH_RPC_ENDPOINT_0_PRIORITY":"0", - "FROSTFS_MORPH_INACTIVITY_TIMEOUT":"60s", "FROSTFS_NODE_WALLET_PATH":"${workspaceFolder}/dev/storage/wallet04.json", "FROSTFS_NODE_WALLET_PASSWORD":"", "FROSTFS_NODE_ADDRESSES":"127.0.0.1:8086", From 09faca034c089b99da140519ef723e03f2c2238b Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 10 Jan 2025 14:53:10 +0300 Subject: [PATCH 1671/1943] [#1593] node: Fix initialization of frostfsid cache Signed-off-by: Alexander Chuprov --- cmd/frostfs-node/container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index be0acf738..98fea9f41 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -43,7 +43,7 @@ func initContainerService(_ context.Context, c *cfg) { fatalOnErr(err) cacheSize := morphconfig.FrostfsIDCacheSize(c.appCfg) - if cacheSize > 0 { + if cacheSize > 0 && c.cfgMorph.cacheTTL > 0 { frostfsIDSubjectProvider = newMorphFrostfsIDCache(frostfsIDSubjectProvider, int(cacheSize), c.cfgMorph.cacheTTL, metrics.NewCacheMetrics("frostfs_id")) } From a2485637bb91fcf976ce21cfa09af95f12c27c94 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Fri, 10 Jan 2025 15:13:10 +0300 Subject: [PATCH 1672/1943] [#1593] node/config_example: Add description of morph/cache_ttl=0 behavior Signed-off-by: Alexander Chuprov --- config/example/node.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/example/node.yaml b/config/example/node.yaml index a179b4704..c5acf5386 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -79,7 +79,8 @@ contracts: # side chain NEOFS contract script hashes; optional, override values morph: dial_timeout: 30s # timeout for side chain NEO RPC client connection - cache_ttl: 15s # Sidechain cache TTL value (min interval between similar calls). Negative value disables caching. + cache_ttl: 15s # Sidechain cache TTL value (min interval between similar calls). + # Negative value disables caching. A zero value sets the default value. # Default value: block time. It is recommended to have this value less or equal to block time. # Cached entities: containers, container lists, eACL tables. container_cache_size: 100 # container_cache_size is is the maximum number of containers in the cache. From 6c51f48aab6932b035a42f1982a531eed8251174 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 13 Jan 2025 12:52:02 +0300 Subject: [PATCH 1673/1943] [#1596] metrics: Create public aliases for internal `engine` metrics Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/engine/metrics.go | 44 ++++++---------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/pkg/local_object_storage/engine/metrics.go b/pkg/local_object_storage/engine/metrics.go index 75936206d..963292d83 100644 --- a/pkg/local_object_storage/engine/metrics.go +++ b/pkg/local_object_storage/engine/metrics.go @@ -7,34 +7,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) -type MetricRegister interface { - AddMethodDuration(method string, d time.Duration) - - SetObjectCounter(shardID, objectType string, v uint64) - AddToObjectCounter(shardID, objectType string, delta int) - - SetMode(shardID string, mode mode.Mode) - - AddToContainerSize(cnrID string, size int64) - DeleteContainerSize(cnrID string) - DeleteContainerCount(cnrID string) - AddToPayloadCounter(shardID string, size int64) - IncErrorCounter(shardID string) - ClearErrorCounter(shardID string) - DeleteShardMetrics(shardID string) - - SetContainerObjectCounter(shardID, contID, objectType string, v uint64) - IncContainerObjectCounter(shardID, contID, objectType string) - SubContainerObjectCounter(shardID, contID, objectType string, v uint64) - - IncRefillObjectsCount(shardID, path string, size int, success bool) - SetRefillPercent(shardID, path string, percent uint32) - SetRefillStatus(shardID, path, status string) - SetEvacuationInProgress(shardID string, value bool) - - WriteCache() metrics.WriteCacheMetrics - GC() metrics.GCMetrics -} +type ( + MetricRegister = metrics.EngineMetrics + GCMetrics = metrics.GCMetrics + WriteCacheMetrics = metrics.WriteCacheMetrics + NullBool = metrics.NullBool +) func elapsed(method string, addFunc func(method string, d time.Duration)) func() { t := time.Now() @@ -76,9 +54,9 @@ type ( ) var ( - _ MetricRegister = noopMetrics{} - _ metrics.WriteCacheMetrics = noopWriteCacheMetrics{} - _ metrics.GCMetrics = noopGCMetrics{} + _ MetricRegister = noopMetrics{} + _ WriteCacheMetrics = noopWriteCacheMetrics{} + _ GCMetrics = noopGCMetrics{} ) func (noopMetrics) AddMethodDuration(string, time.Duration) {} @@ -99,8 +77,8 @@ func (noopMetrics) IncRefillObjectsCount(string, string, int, bool) {} func (noopMetrics) SetRefillPercent(string, string, uint32) {} func (noopMetrics) SetRefillStatus(string, string, string) {} func (noopMetrics) SetEvacuationInProgress(string, bool) {} -func (noopMetrics) WriteCache() metrics.WriteCacheMetrics { return noopWriteCacheMetrics{} } -func (noopMetrics) GC() metrics.GCMetrics { return noopGCMetrics{} } +func (noopMetrics) WriteCache() WriteCacheMetrics { return noopWriteCacheMetrics{} } +func (noopMetrics) GC() GCMetrics { return noopGCMetrics{} } func (noopWriteCacheMetrics) AddMethodDuration(string, string, string, string, bool, time.Duration) {} func (noopWriteCacheMetrics) SetActualCount(string, string, string, uint64) {} From a9f27e074bbf085016f82e1b4802c87ede452283 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 10 Jan 2025 16:27:08 +0300 Subject: [PATCH 1674/1943] [#1243] object: Look for X-Headers within origin before APE check * X-Headers can be found in `origin` field of `MetaHeader` if the request has been forwarded from non-container node. Signed-off-by: Airat Arifullin --- pkg/services/object/ape/service.go | 56 +++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index c6d152e0f..d9594a3fc 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -150,6 +150,11 @@ type putStreamBasicChecker struct { } func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { + meta := request.GetMetaHeader() + for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { + meta = origin + } + if partInit, ok := request.GetBody().GetObjectPart().(*objectV2.PutObjectPartInit); ok { reqCtx, err := requestContext(ctx) if err != nil { @@ -171,7 +176,7 @@ func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutR ContainerOwner: reqCtx.ContainerOwner, Role: nativeSchemaRole(reqCtx.Role), BearerToken: reqCtx.BearerToken, - XHeaders: request.GetMetaHeader().GetXHeaders(), + XHeaders: meta.GetXHeaders(), } if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -204,6 +209,11 @@ type patchStreamBasicChecker struct { } func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.PatchRequest) error { + meta := request.GetMetaHeader() + for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { + meta = origin + } + if !p.nonFirstSend { p.nonFirstSend = true @@ -226,7 +236,7 @@ func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.Pa ContainerOwner: reqCtx.ContainerOwner, Role: nativeSchemaRole(reqCtx.Role), BearerToken: reqCtx.BearerToken, - XHeaders: request.GetMetaHeader().GetXHeaders(), + XHeaders: meta.GetXHeaders(), } if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -251,6 +261,11 @@ func (c *Service) Patch(ctx context.Context) (objectSvc.PatchObjectStream, error } func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { + meta := request.GetMetaHeader() + for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { + meta = origin + } + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { return nil, err @@ -295,7 +310,7 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, BearerToken: reqCtx.BearerToken, - XHeaders: request.GetMetaHeader().GetXHeaders(), + XHeaders: meta.GetXHeaders(), }) if err != nil { return nil, toStatusErr(err) @@ -304,6 +319,11 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj } func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.SearchStream) error { + meta := request.GetMetaHeader() + for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { + meta = origin + } + var cnrID cid.ID if cnrV2 := request.GetBody().GetContainerID(); cnrV2 != nil { if err := cnrID.ReadFromV2(*cnrV2); err != nil { @@ -324,7 +344,7 @@ func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.Searc SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, BearerToken: reqCtx.BearerToken, - XHeaders: request.GetMetaHeader().GetXHeaders(), + XHeaders: meta.GetXHeaders(), }) if err != nil { return toStatusErr(err) @@ -334,6 +354,11 @@ func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.Searc } func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) (*objectV2.DeleteResponse, error) { + meta := request.GetMetaHeader() + for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { + meta = origin + } + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { return nil, err @@ -353,7 +378,7 @@ func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) ( SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, BearerToken: reqCtx.BearerToken, - XHeaders: request.GetMetaHeader().GetXHeaders(), + XHeaders: meta.GetXHeaders(), }) if err != nil { return nil, toStatusErr(err) @@ -368,6 +393,11 @@ func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) ( } func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.GetObjectRangeStream) error { + meta := request.GetMetaHeader() + for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { + meta = origin + } + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { return toStatusErr(err) @@ -387,7 +417,7 @@ func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.G SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, BearerToken: reqCtx.BearerToken, - XHeaders: request.GetMetaHeader().GetXHeaders(), + XHeaders: meta.GetXHeaders(), }) if err != nil { return toStatusErr(err) @@ -397,6 +427,11 @@ func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.G } func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHashRequest) (*objectV2.GetRangeHashResponse, error) { + meta := request.GetMetaHeader() + for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { + meta = origin + } + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { return nil, err @@ -416,7 +451,7 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, BearerToken: reqCtx.BearerToken, - XHeaders: request.GetMetaHeader().GetXHeaders(), + XHeaders: meta.GetXHeaders(), } resp, err := c.next.GetRangeHash(ctx, request) @@ -431,6 +466,11 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa } func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { + meta := request.GetMetaHeader() + for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { + meta = origin + } + cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetObject().GetHeader().GetContainerID(), request.GetBody().GetObject().GetObjectID()) if err != nil { return nil, err @@ -451,7 +491,7 @@ func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequ SenderKey: hex.EncodeToString(reqCtx.SenderKey), ContainerOwner: reqCtx.ContainerOwner, BearerToken: reqCtx.BearerToken, - XHeaders: request.GetMetaHeader().GetXHeaders(), + XHeaders: meta.GetXHeaders(), } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { From 4d5ae59a52e7ab0ce89e2c5cc1f9c6c88a193de2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 13 Jan 2025 16:55:43 +0300 Subject: [PATCH 1675/1943] [#1598] golangci: Enable unconvert linters To drop unnecessary conversions. Signed-off-by: Dmitrii Stepanov --- .golangci.yml | 1 + cmd/frostfs-cli/modules/bearer/generate_override.go | 2 +- cmd/frostfs-node/config.go | 4 ++-- cmd/frostfs-node/config/node/config.go | 4 ++-- cmd/frostfs-node/netmap.go | 2 +- .../blobstor/fstree/fstree_write_generic.go | 2 +- .../blobstor/fstree/fstree_write_linux.go | 2 +- pkg/local_object_storage/blobstor/memstore/memstore.go | 4 ++-- pkg/local_object_storage/engine/evacuate.go | 2 +- pkg/local_object_storage/engine/shards.go | 4 ++-- pkg/local_object_storage/shard/control.go | 2 +- pkg/morph/event/netmap/epoch.go | 2 +- pkg/services/object/put/v2/streamer.go | 2 +- 13 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 57e3b4494..d0e45aa75 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -89,5 +89,6 @@ linters: - protogetter - intrange - tenv + - unconvert disable-all: true fast: false diff --git a/cmd/frostfs-cli/modules/bearer/generate_override.go b/cmd/frostfs-cli/modules/bearer/generate_override.go index 13fe07995..9632061f1 100644 --- a/cmd/frostfs-cli/modules/bearer/generate_override.go +++ b/cmd/frostfs-cli/modules/bearer/generate_override.go @@ -52,7 +52,7 @@ func genereateAPEOverride(cmd *cobra.Command, _ []string) { outputPath, _ := cmd.Flags().GetString(outputFlag) if outputPath != "" { - err := os.WriteFile(outputPath, []byte(overrideMarshalled), 0o644) + err := os.WriteFile(outputPath, overrideMarshalled, 0o644) commonCmd.ExitOnErr(cmd, "dump error: %w", err) } else { fmt.Print("\n") diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 18d3e2454..3392589bf 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -853,8 +853,8 @@ func initFrostfsID(appCfg *config.Config) cfgFrostfsID { } func initCfgGRPC() cfgGRPC { - maxChunkSize := uint64(maxMsgSize) * 3 / 4 // 25% to meta, 75% to payload - maxAddrAmount := uint64(maxChunkSize) / addressSize // each address is about 72 bytes + maxChunkSize := uint64(maxMsgSize) * 3 / 4 // 25% to meta, 75% to payload + maxAddrAmount := maxChunkSize / addressSize // each address is about 72 bytes return cfgGRPC{ maxChunkSize: maxChunkSize, diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index 4d063245b..969d77396 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -198,7 +198,7 @@ func (l PersistentPolicyRulesConfig) Path() string { // // Returns PermDefault if the value is not a positive number. func (l PersistentPolicyRulesConfig) Perm() fs.FileMode { - p := config.UintSafe((*config.Config)(l.cfg), "perm") + p := config.UintSafe(l.cfg, "perm") if p == 0 { p = PermDefault } @@ -210,7 +210,7 @@ func (l PersistentPolicyRulesConfig) Perm() fs.FileMode { // // Returns false if the value is not a boolean. func (l PersistentPolicyRulesConfig) NoSync() bool { - return config.BoolSafe((*config.Config)(l.cfg), "no_sync") + return config.BoolSafe(l.cfg, "no_sync") } // CompatibilityMode returns true if need to run node in compatibility with previous versions mode. diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index a26fdd798..34cd00ac8 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -86,7 +86,7 @@ func (s *networkState) setNodeInfo(ni *netmapSDK.NodeInfo) { } } - s.setControlNetmapStatus(control.NetmapStatus(ctrlNetSt)) + s.setControlNetmapStatus(ctrlNetSt) } // sets the current node state to the given value. Subsequent cfg.bootstrap diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 4110ba7d7..07a618b0a 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -136,6 +136,6 @@ func (w *genericWriter) removeWithCounter(p string, size uint64) error { if err := os.Remove(p); err != nil { return err } - w.fileCounter.Dec(uint64(size)) + w.fileCounter.Dec(size) return nil } diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go index 3561c616b..c62654028 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -114,7 +114,7 @@ func (w *linuxWriter) removeFile(p string, size uint64) error { return logicerr.Wrap(new(apistatus.ObjectNotFound)) } if err == nil { - w.fileCounter.Dec(uint64(size)) + w.fileCounter.Dec(size) } return err } diff --git a/pkg/local_object_storage/blobstor/memstore/memstore.go b/pkg/local_object_storage/blobstor/memstore/memstore.go index 3afef7d18..7ef7e37a4 100644 --- a/pkg/local_object_storage/blobstor/memstore/memstore.go +++ b/pkg/local_object_storage/blobstor/memstore/memstore.go @@ -133,11 +133,11 @@ func (s *memstoreImpl) Iterate(_ context.Context, req common.IteratePrm) (common elem := common.IterationElement{ ObjectData: v, } - if err := elem.Address.DecodeString(string(k)); err != nil { + if err := elem.Address.DecodeString(k); err != nil { if req.IgnoreErrors { continue } - return common.IterateRes{}, logicerr.Wrap(fmt.Errorf("(%T) decoding address string %q: %v", s, string(k), err)) + return common.IterateRes{}, logicerr.Wrap(fmt.Errorf("(%T) decoding address string %q: %v", s, k, err)) } var err error if elem.ObjectData, err = s.compression.Decompress(elem.ObjectData); err != nil { diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 623f5c941..682f23dff 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -724,7 +724,7 @@ func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) shards := make([]pooledShard, 0, len(e.shards)) for id := range e.shards { shards = append(shards, pooledShard{ - hashedShard: hashedShard(e.shards[id]), + hashedShard: e.shards[id], pool: e.shardPools[id], }) } diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 6d4844b75..8e191f72c 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -272,7 +272,7 @@ func (e *StorageEngine) sortShards(objAddr interface{ EncodeToString() string }) h := hrw.StringHash(objAddr.EncodeToString()) shards := make([]hashedShard, 0, len(e.shards)) for _, sh := range e.shards { - shards = append(shards, hashedShard(sh)) + shards = append(shards, sh) } hrw.SortHasherSliceByValue(shards, h) return shards @@ -285,7 +285,7 @@ func (e *StorageEngine) unsortedShards() []hashedShard { shards := make([]hashedShard, 0, len(e.shards)) for _, sh := range e.shards { - shards = append(shards, hashedShard(sh)) + shards = append(shards, sh) } return shards diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 1c1933af5..3136ddfcc 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -38,7 +38,7 @@ func (s *Shard) handleMetabaseFailure(ctx context.Context, stage string, err err err = s.SetMode(ctx, mode.DegradedReadOnly) if err != nil { - return fmt.Errorf("switch to mode %s", mode.Mode(mode.DegradedReadOnly)) + return fmt.Errorf("switch to mode %s", mode.DegradedReadOnly) } return nil } diff --git a/pkg/morph/event/netmap/epoch.go b/pkg/morph/event/netmap/epoch.go index 4dcc0d035..39c8f6237 100644 --- a/pkg/morph/event/netmap/epoch.go +++ b/pkg/morph/event/netmap/epoch.go @@ -41,7 +41,7 @@ func ParseNewEpoch(e *state.ContainedNotificationEvent) (event.Event, error) { } return NewEpoch{ - Num: uint64(nee.Epoch.Uint64()), + Num: nee.Epoch.Uint64(), Hash: e.Container, }, nil } diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 36b514fbc..1cd10cd7f 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -59,7 +59,7 @@ func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) maxSz := s.stream.MaxSizeSrc.MaxObjectSize() s.sizes = &sizes{ - payloadSz: uint64(v.GetHeader().GetPayloadLength()), + payloadSz: v.GetHeader().GetPayloadLength(), } // check payload size limit overflow From fb928616ccade3c704dd98dccd7f2e521d27e5c5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 13 Jan 2025 17:43:39 +0300 Subject: [PATCH 1676/1943] [#1598] golangci: Enable unparam linter To drop unnecessary parameters and return values. Signed-off-by: Dmitrii Stepanov --- .golangci.yml | 1 + .../modules/morph/frostfsid/frostfsid.go | 16 +++++----- .../modules/container/policy_playground.go | 7 ++--- cmd/frostfs-lens/internal/tui/buckets.go | 18 ++++------- cmd/frostfs-lens/internal/tui/db.go | 31 +++++++------------ cmd/frostfs-lens/internal/tui/records.go | 8 ++--- cmd/frostfs-node/config.go | 13 +++----- internal/logs/logs.go | 2 -- pkg/innerring/initialization.go | 6 ++-- pkg/innerring/locode.go | 4 +-- .../blobstor/blobovniczatree/delete.go | 17 +++++----- .../blobstor/blobovniczatree/iterate.go | 6 ++++ pkg/local_object_storage/engine/container.go | 14 ++++----- pkg/local_object_storage/engine/delete.go | 6 ++-- pkg/local_object_storage/engine/inhume.go | 10 +++--- pkg/local_object_storage/engine/select.go | 18 +++++------ .../metabase/containers.go | 13 ++++---- pkg/local_object_storage/metabase/counter.go | 4 +-- pkg/local_object_storage/metabase/delete.go | 6 ++-- pkg/local_object_storage/metabase/inhume.go | 4 +-- .../metabase/storage_id.go | 18 +++++------ pkg/local_object_storage/pilorama/boltdb.go | 22 +++++-------- pkg/local_object_storage/writecache/cache.go | 3 +- pkg/local_object_storage/writecache/state.go | 3 +- pkg/morph/event/listener.go | 19 +++--------- .../object_manager/placement/traverser.go | 9 ++---- 26 files changed, 123 insertions(+), 155 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index d0e45aa75..d83f36de8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -90,5 +90,6 @@ linters: - intrange - tenv - unconvert + - unparam disable-all: true fast: false diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index db98bb8ad..b229d0436 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -253,7 +253,7 @@ func frostfsidListNamespaces(cmd *cobra.Command, _ []string) { reader := frostfsidrpclient.NewReader(inv, hash) sessionID, it, err := reader.ListNamespaces() commonCmd.ExitOnErr(cmd, "can't get namespace: %w", err) - items, err := readIterator(inv, &it, iteratorBatchSize, sessionID) + items, err := readIterator(inv, &it, sessionID) commonCmd.ExitOnErr(cmd, "can't read iterator: %w", err) namespaces, err := frostfsidclient.ParseNamespaces(items) @@ -305,7 +305,7 @@ func frostfsidListSubjects(cmd *cobra.Command, _ []string) { sessionID, it, err := reader.ListNamespaceSubjects(ns) commonCmd.ExitOnErr(cmd, "can't get namespace: %w", err) - subAddresses, err := frostfsidclient.UnwrapArrayOfUint160(readIterator(inv, &it, iteratorBatchSize, sessionID)) + subAddresses, err := frostfsidclient.UnwrapArrayOfUint160(readIterator(inv, &it, sessionID)) commonCmd.ExitOnErr(cmd, "can't unwrap: %w", err) sort.Slice(subAddresses, func(i, j int) bool { return subAddresses[i].Less(subAddresses[j]) }) @@ -319,7 +319,7 @@ func frostfsidListSubjects(cmd *cobra.Command, _ []string) { sessionID, it, err := reader.ListSubjects() commonCmd.ExitOnErr(cmd, "can't get subject: %w", err) - items, err := readIterator(inv, &it, iteratorBatchSize, sessionID) + items, err := readIterator(inv, &it, sessionID) commonCmd.ExitOnErr(cmd, "can't read iterator: %w", err) subj, err := frostfsidclient.ParseSubject(items) @@ -365,7 +365,7 @@ func frostfsidListGroups(cmd *cobra.Command, _ []string) { sessionID, it, err := reader.ListGroups(ns) commonCmd.ExitOnErr(cmd, "can't get namespace: %w", err) - items, err := readIterator(inv, &it, iteratorBatchSize, sessionID) + items, err := readIterator(inv, &it, sessionID) commonCmd.ExitOnErr(cmd, "can't list groups: %w", err) groups, err := frostfsidclient.ParseGroups(items) commonCmd.ExitOnErr(cmd, "can't parse groups: %w", err) @@ -415,7 +415,7 @@ func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { sessionID, it, err := reader.ListGroupSubjects(ns, big.NewInt(groupID)) commonCmd.ExitOnErr(cmd, "can't list groups: %w", err) - items, err := readIterator(inv, &it, iteratorBatchSize, sessionID) + items, err := readIterator(inv, &it, sessionID) commonCmd.ExitOnErr(cmd, "can't read iterator: %w", err) subjects, err := frostfsidclient.UnwrapArrayOfUint160(items, err) @@ -492,17 +492,17 @@ func (f *frostfsidClient) sendWaitRes() (*state.AppExecResult, error) { return f.roCli.Wait(f.wCtx.SentTxs[0].Hash, f.wCtx.SentTxs[0].Vub, nil) } -func readIterator(inv *invoker.Invoker, iter *result.Iterator, batchSize int, sessionID uuid.UUID) ([]stackitem.Item, error) { +func readIterator(inv *invoker.Invoker, iter *result.Iterator, sessionID uuid.UUID) ([]stackitem.Item, error) { var shouldStop bool res := make([]stackitem.Item, 0) for !shouldStop { - items, err := inv.TraverseIterator(sessionID, iter, batchSize) + items, err := inv.TraverseIterator(sessionID, iter, iteratorBatchSize) if err != nil { return nil, err } res = append(res, items...) - shouldStop = len(items) < batchSize + shouldStop = len(items) < iteratorBatchSize } return res, nil diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 40bd4110b..dcd755510 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -23,11 +23,11 @@ type policyPlaygroundREPL struct { nodes map[string]netmap.NodeInfo } -func newPolicyPlaygroundREPL(cmd *cobra.Command) (*policyPlaygroundREPL, error) { +func newPolicyPlaygroundREPL(cmd *cobra.Command) *policyPlaygroundREPL { return &policyPlaygroundREPL{ cmd: cmd, nodes: map[string]netmap.NodeInfo{}, - }, nil + } } func (repl *policyPlaygroundREPL) handleLs(args []string) error { @@ -246,8 +246,7 @@ var policyPlaygroundCmd = &cobra.Command{ Long: `A REPL for testing placement policies. If a wallet and endpoint is provided, the initial netmap data will be loaded from the snapshot of the node. Otherwise, an empty playground is created.`, Run: func(cmd *cobra.Command, _ []string) { - repl, err := newPolicyPlaygroundREPL(cmd) - commonCmd.ExitOnErr(cmd, "could not create policy playground: %w", err) + repl := newPolicyPlaygroundREPL(cmd) commonCmd.ExitOnErr(cmd, "policy playground failed: %w", repl.run()) }, } diff --git a/cmd/frostfs-lens/internal/tui/buckets.go b/cmd/frostfs-lens/internal/tui/buckets.go index 3f5088e7a..2d3b20792 100644 --- a/cmd/frostfs-lens/internal/tui/buckets.go +++ b/cmd/frostfs-lens/internal/tui/buckets.go @@ -124,10 +124,7 @@ func (v *BucketsView) loadNodeChildren( path := parentBucket.Path parser := parentBucket.NextParser - buffer, err := LoadBuckets(ctx, v.ui.db, path, v.ui.loadBufferSize) - if err != nil { - return err - } + buffer := LoadBuckets(ctx, v.ui.db, path, v.ui.loadBufferSize) for item := range buffer { if item.err != nil { @@ -135,6 +132,7 @@ func (v *BucketsView) loadNodeChildren( } bucket := item.val + var err error bucket.Entry, bucket.NextParser, err = parser(bucket.Name, nil) if err != nil { return err @@ -180,10 +178,7 @@ func (v *BucketsView) bucketSatisfiesFilter( defer cancel() // Check the current bucket's nested buckets if exist - bucketsBuffer, err := LoadBuckets(ctx, v.ui.db, bucket.Path, v.ui.loadBufferSize) - if err != nil { - return false, err - } + bucketsBuffer := LoadBuckets(ctx, v.ui.db, bucket.Path, v.ui.loadBufferSize) for item := range bucketsBuffer { if item.err != nil { @@ -191,6 +186,7 @@ func (v *BucketsView) bucketSatisfiesFilter( } b := item.val + var err error b.Entry, b.NextParser, err = bucket.NextParser(b.Name, nil) if err != nil { return false, err @@ -206,10 +202,7 @@ func (v *BucketsView) bucketSatisfiesFilter( } // Check the current bucket's nested records if exist - recordsBuffer, err := LoadRecords(ctx, v.ui.db, bucket.Path, v.ui.loadBufferSize) - if err != nil { - return false, err - } + recordsBuffer := LoadRecords(ctx, v.ui.db, bucket.Path, v.ui.loadBufferSize) for item := range recordsBuffer { if item.err != nil { @@ -217,6 +210,7 @@ func (v *BucketsView) bucketSatisfiesFilter( } r := item.val + var err error r.Entry, _, err = bucket.NextParser(r.Key, r.Value) if err != nil { return false, err diff --git a/cmd/frostfs-lens/internal/tui/db.go b/cmd/frostfs-lens/internal/tui/db.go index d0cf611d4..94fa87f98 100644 --- a/cmd/frostfs-lens/internal/tui/db.go +++ b/cmd/frostfs-lens/internal/tui/db.go @@ -35,7 +35,7 @@ func resolvePath(tx *bbolt.Tx, path [][]byte) (*bbolt.Bucket, error) { func load[T any]( ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int, filter func(key, value []byte) bool, transform func(key, value []byte) T, -) (<-chan Item[T], error) { +) <-chan Item[T] { buffer := make(chan Item[T], bufferSize) go func() { @@ -77,13 +77,13 @@ func load[T any]( } }() - return buffer, nil + return buffer } func LoadBuckets( ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int, -) (<-chan Item[*Bucket], error) { - buffer, err := load( +) <-chan Item[*Bucket] { + buffer := load( ctx, db, path, bufferSize, func(_, value []byte) bool { return value == nil @@ -98,17 +98,14 @@ func LoadBuckets( } }, ) - if err != nil { - return nil, fmt.Errorf("can't start iterating bucket: %w", err) - } - return buffer, nil + return buffer } func LoadRecords( ctx context.Context, db *bbolt.DB, path [][]byte, bufferSize int, -) (<-chan Item[*Record], error) { - buffer, err := load( +) <-chan Item[*Record] { + buffer := load( ctx, db, path, bufferSize, func(_, value []byte) bool { return value != nil @@ -124,11 +121,8 @@ func LoadRecords( } }, ) - if err != nil { - return nil, fmt.Errorf("can't start iterating bucket: %w", err) - } - return buffer, nil + return buffer } // HasBuckets checks if a bucket has nested buckets. It relies on assumption @@ -137,24 +131,21 @@ func HasBuckets(ctx context.Context, db *bbolt.DB, path [][]byte) (bool, error) ctx, cancel := context.WithCancel(ctx) defer cancel() - buffer, err := load( + buffer := load( ctx, db, path, 1, nil, func(_, value []byte) []byte { return value }, ) - if err != nil { - return false, err - } x, ok := <-buffer if !ok { return false, nil } if x.err != nil { - return false, err + return false, x.err } if x.val != nil { - return false, err + return false, nil } return true, nil } diff --git a/cmd/frostfs-lens/internal/tui/records.go b/cmd/frostfs-lens/internal/tui/records.go index 5f53ed287..5f61df884 100644 --- a/cmd/frostfs-lens/internal/tui/records.go +++ b/cmd/frostfs-lens/internal/tui/records.go @@ -62,10 +62,7 @@ func (v *RecordsView) Mount(ctx context.Context) error { ctx, v.onUnmount = context.WithCancel(ctx) - tempBuffer, err := LoadRecords(ctx, v.ui.db, v.bucket.Path, v.ui.loadBufferSize) - if err != nil { - return err - } + tempBuffer := LoadRecords(ctx, v.ui.db, v.bucket.Path, v.ui.loadBufferSize) v.buffer = make(chan *Record, v.ui.loadBufferSize) go func() { @@ -73,11 +70,12 @@ func (v *RecordsView) Mount(ctx context.Context) error { for item := range tempBuffer { if item.err != nil { - v.ui.stopOnError(err) + v.ui.stopOnError(item.err) break } record := item.val + var err error record.Entry, _, err = v.bucket.NextParser(record.Key, record.Value) if err != nil { v.ui.stopOnError(err) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 3392589bf..9931d0dc8 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -698,8 +698,7 @@ func initCfg(appCfg *config.Config) *cfg { netState.metrics = c.metricsCollector - logPrm, err := c.loggerPrm() - fatalOnErr(err) + logPrm := c.loggerPrm() logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook() log, err := logger.NewLogger(logPrm) fatalOnErr(err) @@ -1059,7 +1058,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID return sh } -func (c *cfg) loggerPrm() (*logger.Prm, error) { +func (c *cfg) loggerPrm() *logger.Prm { // check if it has been inited before if c.dynamicConfiguration.logger == nil { c.dynamicConfiguration.logger = new(logger.Prm) @@ -1078,7 +1077,7 @@ func (c *cfg) loggerPrm() (*logger.Prm, error) { } c.dynamicConfiguration.logger.PrependTimestamp = c.LoggerCfg.timestamp - return c.dynamicConfiguration.logger, nil + return c.dynamicConfiguration.logger } func (c *cfg) LocalAddress() network.AddressGroup { @@ -1334,11 +1333,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { // Logger - logPrm, err := c.loggerPrm() - if err != nil { - c.log.Error(ctx, logs.FrostFSNodeLoggerConfigurationPreparation, zap.Error(err)) - return - } + logPrm := c.loggerPrm() components := c.getComponents(ctx, logPrm) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index b24f3593d..0610dc175 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -146,7 +146,6 @@ const ( ClientCantGetBlockchainHeight = "can't get blockchain height" ClientCantGetBlockchainHeight243 = "can't get blockchain height" EventCouldNotSubmitHandlerToWorkerPool = "could not Submit handler to worker pool" - EventCouldNotStartListenToEvents = "could not start listen to events" EventStopEventListenerByError = "stop event listener by error" EventStopEventListenerByContext = "stop event listener by context" EventStopEventListenerByNotificationChannel = "stop event listener by notification channel" @@ -384,7 +383,6 @@ const ( FrostFSNodeShutdownSkip = "node is already shutting down, skipped shutdown" FrostFSNodeShutdownWhenNotReady = "node is going to shut down when subsystems are still initializing" FrostFSNodeConfigurationReading = "configuration reading" - FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" FrostFSNodeTracingConfigationUpdated = "tracing configation updated" FrostFSNodeStorageEngineConfigurationUpdate = "storage engine configuration update" FrostFSNodePoolConfigurationUpdate = "adjust pool configuration" diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index ecaf8ae86..f7b71dbe6 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -38,10 +38,7 @@ import ( func (s *Server) initNetmapProcessor(ctx context.Context, cfg *viper.Viper, alphaSync event.Handler, ) error { - locodeValidator, err := s.newLocodeValidator(cfg) - if err != nil { - return err - } + locodeValidator := s.newLocodeValidator(cfg) netSettings := (*networkSettings)(s.netmapClient) @@ -51,6 +48,7 @@ func (s *Server) initNetmapProcessor(ctx context.Context, cfg *viper.Viper, poolSize := cfg.GetInt("workers.netmap") s.log.Debug(ctx, logs.NetmapNetmapWorkerPool, zap.Int("size", poolSize)) + var err error s.netmapProcessor, err = netmap.New(&netmap.Params{ Log: s.log, Metrics: s.irMetrics, diff --git a/pkg/innerring/locode.go b/pkg/innerring/locode.go index a0c3ea751..ae4c85168 100644 --- a/pkg/innerring/locode.go +++ b/pkg/innerring/locode.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/viper" ) -func (s *Server) newLocodeValidator(cfg *viper.Viper) (netmap.NodeValidator, error) { +func (s *Server) newLocodeValidator(cfg *viper.Viper) netmap.NodeValidator { locodeDB := locodebolt.New(locodebolt.Prm{ Path: cfg.GetString("locode.db.path"), }, @@ -21,7 +21,7 @@ func (s *Server) newLocodeValidator(cfg *viper.Viper) (netmap.NodeValidator, err return irlocode.New(irlocode.Prm{ DB: (*locodeBoltDBWrapper)(locodeDB), - }), nil + }) } type locodeBoltEntryWrapper struct { diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 47e12bafb..8d17fc4b0 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -19,7 +19,10 @@ import ( "go.uber.org/zap" ) -var errObjectIsDeleteProtected = errors.New("object is delete protected") +var ( + errObjectIsDeleteProtected = errors.New("object is delete protected") + deleteRes = common.DeleteRes{} +) // Delete deletes object from blobovnicza tree. // @@ -43,17 +46,17 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co defer span.End() if b.readOnly { - return common.DeleteRes{}, common.ErrReadOnly + return deleteRes, common.ErrReadOnly } if b.rebuildGuard.TryRLock() { defer b.rebuildGuard.RUnlock() } else { - return common.DeleteRes{}, errRebuildInProgress + return deleteRes, errRebuildInProgress } if b.deleteProtectedObjects.Contains(prm.Address) { - return common.DeleteRes{}, errObjectIsDeleteProtected + return deleteRes, errObjectIsDeleteProtected } var bPrm blobovnicza.DeletePrm @@ -98,7 +101,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co if err == nil && !objectFound { // not found in any blobovnicza - return common.DeleteRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) + return deleteRes, logicerr.Wrap(new(apistatus.ObjectNotFound)) } success = err == nil @@ -112,7 +115,7 @@ func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicz shBlz := b.getBlobovnicza(ctx, blzPath) blz, err := shBlz.Open(ctx) if err != nil { - return common.DeleteRes{}, err + return deleteRes, err } defer shBlz.Close(ctx) @@ -122,5 +125,5 @@ func (b *Blobovniczas) deleteObjectFromLevel(ctx context.Context, prm blobovnicz // removes object from blobovnicza and returns common.DeleteRes. func (b *Blobovniczas) deleteObject(ctx context.Context, blz *blobovnicza.Blobovnicza, prm blobovnicza.DeletePrm) (common.DeleteRes, error) { _, err := blz.Delete(ctx, prm) - return common.DeleteRes{}, err + return deleteRes, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 5c2d58ca1..ceb8fb7e3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -249,6 +249,12 @@ func (b *Blobovniczas) iterateSortedDBPaths(ctx context.Context, addr oid.Addres } func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path string, addr oid.Address, f func(string) (bool, error)) (bool, error) { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } + sysPath := filepath.Join(b.rootPath, path) entries, err := os.ReadDir(sysPath) if os.IsNotExist(err) && b.readOnly && path == "" { // non initialized tree in read only mode diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 24059a3f9..b2d7a1037 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -48,8 +48,8 @@ func (e *StorageEngine) ContainerSize(ctx context.Context, prm ContainerSizePrm) defer elapsed("ContainerSize", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res, err = e.containerSize(ctx, prm) - return err + res = e.containerSize(ctx, prm) + return nil }) return @@ -69,7 +69,7 @@ func ContainerSize(ctx context.Context, e *StorageEngine, id cid.ID) (uint64, er return res.Size(), nil } -func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) (res ContainerSizeRes, err error) { +func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) (res ContainerSizeRes) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { var csPrm shard.ContainerSizePrm csPrm.SetContainerID(prm.cnr) @@ -96,8 +96,8 @@ func (e *StorageEngine) ListContainers(ctx context.Context, _ ListContainersPrm) defer elapsed("ListContainers", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res, err = e.listContainers(ctx) - return err + res = e.listContainers(ctx) + return nil }) return @@ -115,7 +115,7 @@ func ListContainers(ctx context.Context, e *StorageEngine) ([]cid.ID, error) { return res.Containers(), nil } -func (e *StorageEngine) listContainers(ctx context.Context) (ListContainersRes, error) { +func (e *StorageEngine) listContainers(ctx context.Context) ListContainersRes { uniqueIDs := make(map[string]cid.ID) e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { @@ -142,5 +142,5 @@ func (e *StorageEngine) listContainers(ctx context.Context) (ListContainersRes, return ListContainersRes{ containers: result, - }, nil + } } diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 65ccbdb9e..c735cc41d 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -27,6 +27,8 @@ type DeletePrm struct { // DeleteRes groups the resulting values of Delete operation. type DeleteRes struct{} +var deleteRes = DeleteRes{} + // WithAddress is a Delete option to set the addresses of the objects to delete. // // Option is required. @@ -126,14 +128,14 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e }) if locked.is { - return DeleteRes{}, new(apistatus.ObjectLocked) + return deleteRes, new(apistatus.ObjectLocked) } if splitInfo != nil { e.deleteChildren(ctx, prm.addr, prm.forceRemoval, splitInfo.SplitID()) } - return DeleteRes{}, nil + return deleteRes, nil } func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, force bool, splitID *objectSDK.SplitID) { diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index bae784064..a0c38cd5d 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -30,6 +30,8 @@ type InhumePrm struct { // InhumeRes encapsulates results of inhume operation. type InhumeRes struct{} +var inhumeRes = InhumeRes{} + // WithTarget sets a list of objects that should be inhumed and tombstone address // as the reason for inhume operation. // @@ -83,7 +85,7 @@ func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRe func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { addrsPerShard, err := e.groupObjectsByShard(ctx, prm.addrs, !prm.forceRemoval) if err != nil { - return InhumeRes{}, err + return inhumeRes, err } var shPrm shard.InhumePrm @@ -107,7 +109,7 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e zap.String("shard_id", shardID), zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) - return InhumeRes{}, errInhumeFailure + return inhumeRes, errInhumeFailure } if _, err := sh.Inhume(ctx, shPrm); err != nil { @@ -119,11 +121,11 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e default: e.reportShardError(ctx, sh, "couldn't inhume object in shard", err) } - return InhumeRes{}, err + return inhumeRes, err } } - return InhumeRes{}, nil + return inhumeRes, nil } // groupObjectsByShard groups objects based on the shard(s) they are stored on. diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index 02149b4c8..fc8b4a9a7 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -54,19 +54,17 @@ func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRe defer elapsed("Select", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res, err = e._select(ctx, prm) - return err + res = e._select(ctx, prm) + return nil }) return } -func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) { +func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) SelectRes { addrList := make([]oid.Address, 0) uniqueMap := make(map[string]struct{}) - var outError error - var shPrm shard.SelectPrm shPrm.SetContainerID(prm.cnr, prm.indexedContainer) shPrm.SetFilters(prm.filters) @@ -90,7 +88,7 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, return SelectRes{ addrList: addrList, - }, outError + } } // List returns `limit` available physically storage object addresses in engine. @@ -100,14 +98,14 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, func (e *StorageEngine) List(ctx context.Context, limit uint64) (res SelectRes, err error) { defer elapsed("List", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res, err = e.list(ctx, limit) - return err + res = e.list(ctx, limit) + return nil }) return } -func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, error) { +func (e *StorageEngine) list(ctx context.Context, limit uint64) SelectRes { addrList := make([]oid.Address, 0, limit) uniqueMap := make(map[string]struct{}) ln := uint64(0) @@ -136,7 +134,7 @@ func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, erro return SelectRes{ addrList: addrList, - }, nil + } } // Select selects objects from local storage using provided filters. diff --git a/pkg/local_object_storage/metabase/containers.go b/pkg/local_object_storage/metabase/containers.go index 472b2affc..da27e6085 100644 --- a/pkg/local_object_storage/metabase/containers.go +++ b/pkg/local_object_storage/metabase/containers.go @@ -56,7 +56,7 @@ func (db *DB) containers(tx *bbolt.Tx) ([]cid.ID, error) { return result, err } -func (db *DB) ContainerSize(id cid.ID) (size uint64, err error) { +func (db *DB) ContainerSize(id cid.ID) (uint64, error) { db.modeMtx.RLock() defer db.modeMtx.RUnlock() @@ -64,21 +64,22 @@ func (db *DB) ContainerSize(id cid.ID) (size uint64, err error) { return 0, ErrDegradedMode } - err = db.boltDB.View(func(tx *bbolt.Tx) error { - size, err = db.containerSize(tx, id) + var size uint64 + err := db.boltDB.View(func(tx *bbolt.Tx) error { + size = db.containerSize(tx, id) - return err + return nil }) return size, metaerr.Wrap(err) } -func (db *DB) containerSize(tx *bbolt.Tx, id cid.ID) (uint64, error) { +func (db *DB) containerSize(tx *bbolt.Tx, id cid.ID) uint64 { containerVolume := tx.Bucket(containerVolumeBucketName) key := make([]byte, cidSize) id.Encode(key) - return parseContainerSize(containerVolume.Get(key)), nil + return parseContainerSize(containerVolume.Get(key)) } func parseContainerID(dst *cid.ID, name []byte, ignore map[string]struct{}) bool { diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index f29dafe77..732f99519 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -251,13 +251,13 @@ func (db *DB) incCounters(tx *bbolt.Tx, cnrID cid.ID, isUserObject bool) error { return db.incContainerObjectCounter(tx, cnrID, isUserObject) } -func (db *DB) updateShardObjectCounter(tx *bbolt.Tx, typ objectType, delta uint64, inc bool) error { +func (db *DB) decShardObjectCounter(tx *bbolt.Tx, typ objectType, delta uint64) error { b := tx.Bucket(shardInfoBucket) if b == nil { return nil } - return db.updateShardObjectCounterBucket(b, typ, delta, inc) + return db.updateShardObjectCounterBucket(b, typ, delta, false) } func (*DB) updateShardObjectCounterBucket(b *bbolt.Bucket, typ objectType, delta uint64, inc bool) error { diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index 00ee2baa3..d338e228f 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -161,21 +161,21 @@ func (db *DB) deleteGroup(tx *bbolt.Tx, addrs []oid.Address) (DeleteRes, error) func (db *DB) updateCountersDelete(tx *bbolt.Tx, res DeleteRes) error { if res.phyCount > 0 { - err := db.updateShardObjectCounter(tx, phy, res.phyCount, false) + err := db.decShardObjectCounter(tx, phy, res.phyCount) if err != nil { return fmt.Errorf("decrease phy object counter: %w", err) } } if res.logicCount > 0 { - err := db.updateShardObjectCounter(tx, logical, res.logicCount, false) + err := db.decShardObjectCounter(tx, logical, res.logicCount) if err != nil { return fmt.Errorf("decrease logical object counter: %w", err) } } if res.userCount > 0 { - err := db.updateShardObjectCounter(tx, user, res.userCount, false) + err := db.decShardObjectCounter(tx, user, res.userCount) if err != nil { return fmt.Errorf("decrease user object counter: %w", err) } diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 99fdec310..76018fb61 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -342,10 +342,10 @@ func (db *DB) inhumeECInfo(tx *bbolt.Tx, epoch uint64, tomb *oid.Address, res *I } func (db *DB) applyInhumeResToCounters(tx *bbolt.Tx, res *InhumeRes) error { - if err := db.updateShardObjectCounter(tx, logical, res.LogicInhumed(), false); err != nil { + if err := db.decShardObjectCounter(tx, logical, res.LogicInhumed()); err != nil { return err } - if err := db.updateShardObjectCounter(tx, user, res.UserInhumed(), false); err != nil { + if err := db.decShardObjectCounter(tx, user, res.UserInhumed()); err != nil { return err } diff --git a/pkg/local_object_storage/metabase/storage_id.go b/pkg/local_object_storage/metabase/storage_id.go index 6d620b41a..8f2376503 100644 --- a/pkg/local_object_storage/metabase/storage_id.go +++ b/pkg/local_object_storage/metabase/storage_id.go @@ -35,7 +35,7 @@ func (r StorageIDRes) StorageID() []byte { // StorageID returns storage descriptor for objects from the blobstor. // It is put together with the object can makes get/delete operation faster. -func (db *DB) StorageID(ctx context.Context, prm StorageIDPrm) (res StorageIDRes, err error) { +func (db *DB) StorageID(ctx context.Context, prm StorageIDPrm) (StorageIDRes, error) { var ( startedAt = time.Now() success = false @@ -53,32 +53,32 @@ func (db *DB) StorageID(ctx context.Context, prm StorageIDPrm) (res StorageIDRes db.modeMtx.RLock() defer db.modeMtx.RUnlock() + var res StorageIDRes if db.mode.NoMetabase() { return res, ErrDegradedMode } - err = db.boltDB.View(func(tx *bbolt.Tx) error { - res.id, err = db.storageID(tx, prm.addr) - - return err + err := db.boltDB.View(func(tx *bbolt.Tx) error { + res.id = db.storageID(tx, prm.addr) + return nil }) success = err == nil return res, metaerr.Wrap(err) } -func (db *DB) storageID(tx *bbolt.Tx, addr oid.Address) ([]byte, error) { +func (db *DB) storageID(tx *bbolt.Tx, addr oid.Address) []byte { key := make([]byte, bucketKeySize) smallBucket := tx.Bucket(smallBucketName(addr.Container(), key)) if smallBucket == nil { - return nil, nil + return nil } storageID := smallBucket.Get(objectKey(addr.Object(), key)) if storageID == nil { - return nil, nil + return nil } - return bytes.Clone(storageID), nil + return bytes.Clone(storageID) } // UpdateStorageIDPrm groups the parameters of UpdateStorageID operation. diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 86b19e3af..1d55d1c13 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -419,10 +419,7 @@ func (t *boltForest) addByPathInternal(d CIDDescriptor, attr string, treeID stri return err } - i, node, err := t.getPathPrefix(bTree, attr, path) - if err != nil { - return err - } + i, node := t.getPathPrefix(bTree, attr, path) ts := t.getLatestTimestamp(bLog, d.Position, d.Size) lm = make([]Move, len(path)-i+1) @@ -980,10 +977,7 @@ func (t *boltForest) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID st b := treeRoot.Bucket(dataBucket) - i, curNodes, err := t.getPathPrefixMultiTraversal(b, attr, path[:len(path)-1]) - if err != nil { - return err - } + i, curNodes := t.getPathPrefixMultiTraversal(b, attr, path[:len(path)-1]) if i < len(path)-1 { return nil } @@ -1526,7 +1520,7 @@ func (t *boltForest) TreeListTrees(ctx context.Context, prm TreeListTreesPrm) (* return &res, nil } -func (t *boltForest) getPathPrefixMultiTraversal(bTree *bbolt.Bucket, attr string, path []string) (int, []Node, error) { +func (t *boltForest) getPathPrefixMultiTraversal(bTree *bbolt.Bucket, attr string, path []string) (int, []Node) { c := bTree.Cursor() var curNodes []Node @@ -1549,14 +1543,14 @@ func (t *boltForest) getPathPrefixMultiTraversal(bTree *bbolt.Bucket, attr strin } if len(nextNodes) == 0 { - return i, curNodes, nil + return i, curNodes } } - return len(path), nextNodes, nil + return len(path), nextNodes } -func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node, error) { +func (t *boltForest) getPathPrefix(bTree *bbolt.Bucket, attr string, path []string) (int, Node) { c := bTree.Cursor() var curNode Node @@ -1576,10 +1570,10 @@ loop: childKey, value = c.Next() } - return i, curNode, nil + return i, curNode } - return len(path), curNode, nil + return len(path), curNode } func (t *boltForest) moveFromBytes(m *Move, data []byte) error { diff --git a/pkg/local_object_storage/writecache/cache.go b/pkg/local_object_storage/writecache/cache.go index e829d013c..b99d73d3a 100644 --- a/pkg/local_object_storage/writecache/cache.go +++ b/pkg/local_object_storage/writecache/cache.go @@ -94,7 +94,8 @@ func (c *cache) Open(_ context.Context, mod mode.Mode) error { if err != nil { return metaerr.Wrap(err) } - return metaerr.Wrap(c.initCounters()) + c.initCounters() + return nil } // Init runs necessary services. diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index 835686fbb..44caa2603 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -19,7 +19,6 @@ func (c *cache) hasEnoughSpace(objectSize uint64) bool { return c.maxCacheSize >= size+objectSize } -func (c *cache) initCounters() error { +func (c *cache) initCounters() { c.estimateCacheSize() - return nil } diff --git a/pkg/morph/event/listener.go b/pkg/morph/event/listener.go index 83f8bee07..e5cdfeef7 100644 --- a/pkg/morph/event/listener.go +++ b/pkg/morph/event/listener.go @@ -134,11 +134,8 @@ func (l *listener) Listen(ctx context.Context) { l.startOnce.Do(func() { l.wg.Add(1) defer l.wg.Done() - if err := l.listen(ctx, nil); err != nil { - l.log.Error(ctx, logs.EventCouldNotStartListenToEvents, - zap.Error(err), - ) - } + + l.listen(ctx, nil) }) } @@ -152,23 +149,17 @@ func (l *listener) ListenWithError(ctx context.Context, intError chan<- error) { l.startOnce.Do(func() { l.wg.Add(1) defer l.wg.Done() - if err := l.listen(ctx, intError); err != nil { - l.log.Error(ctx, logs.EventCouldNotStartListenToEvents, - zap.Error(err), - ) - l.sendError(ctx, intError, err) - } + + l.listen(ctx, intError) }) } -func (l *listener) listen(ctx context.Context, intError chan<- error) error { +func (l *listener) listen(ctx context.Context, intError chan<- error) { subErrCh := make(chan error) go l.subscribe(subErrCh) l.listenLoop(ctx, intError, subErrCh) - - return nil } func (l *listener) subscribe(errCh chan error) { diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 6a949e938..8daf38217 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -120,10 +120,7 @@ func NewTraverser(opts ...Option) (*Traverser, error) { } rem = []int{-1, -1} - sortedVector, err := sortVector(cfg, unsortedVector) - if err != nil { - return nil, err - } + sortedVector := sortVector(cfg, unsortedVector) ns = [][]netmap.NodeInfo{sortedVector, regularVector} } else if cfg.flatSuccess != nil { ns = flatNodes(ns) @@ -188,7 +185,7 @@ type nodeMetrics struct { metrics []int } -func sortVector(cfg *cfg, unsortedVector []netmap.NodeInfo) ([]netmap.NodeInfo, error) { +func sortVector(cfg *cfg, unsortedVector []netmap.NodeInfo) []netmap.NodeInfo { nm := make([]nodeMetrics, len(unsortedVector)) node := cfg.nodeState.LocalNodeInfo() @@ -209,7 +206,7 @@ func sortVector(cfg *cfg, unsortedVector []netmap.NodeInfo) ([]netmap.NodeInfo, for i := range unsortedVector { sortedVector[i] = unsortedVector[nm[i].index] } - return sortedVector, nil + return sortedVector } // Node is a descriptor of storage node with information required for intra-container communication. From eff95bd632dc11435b8356555bfbdebe35b6d1d6 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 14 Jan 2025 11:15:21 +0300 Subject: [PATCH 1677/1943] [#1598] engine: Drop unnecessary result structs Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 9 +++---- pkg/local_object_storage/engine/delete.go | 20 +++++----------- .../engine/delete_test.go | 8 +++---- pkg/local_object_storage/engine/inhume.go | 24 +++++++------------ .../engine/inhume_test.go | 12 +++++----- pkg/local_object_storage/engine/lock_test.go | 19 +++++++-------- pkg/services/control/server/gc.go | 3 +-- 7 files changed, 36 insertions(+), 59 deletions(-) diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index f82a8e533..939241168 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -215,8 +215,7 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl prm.MarkAsGarbage(addr) prm.WithForceRemoval() - _, err := ls.Inhume(ctx, prm) - return err + return ls.Inhume(ctx, prm) } remoteReader := objectService.NewRemoteReader(keyStorage, clientConstructor) @@ -266,8 +265,7 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl var inhumePrm engine.InhumePrm inhumePrm.MarkAsGarbage(addr) - _, err := ls.Inhume(ctx, inhumePrm) - if err != nil { + if err := ls.Inhume(ctx, inhumePrm); err != nil { c.log.Warn(ctx, logs.FrostFSNodeCouldNotInhumeMarkRedundantCopyAsGarbage, zap.Error(err), ) @@ -476,8 +474,7 @@ func (e engineWithoutNotifications) Delete(ctx context.Context, tombstone oid.Ad prm.WithTarget(tombstone, addrs...) - _, err := e.engine.Inhume(ctx, prm) - return err + return e.engine.Inhume(ctx, prm) } func (e engineWithoutNotifications) Lock(ctx context.Context, locker oid.Address, toLock []oid.ID) error { diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index c735cc41d..20c915da6 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -24,11 +24,6 @@ type DeletePrm struct { forceRemoval bool } -// DeleteRes groups the resulting values of Delete operation. -type DeleteRes struct{} - -var deleteRes = DeleteRes{} - // WithAddress is a Delete option to set the addresses of the objects to delete. // // Option is required. @@ -53,7 +48,7 @@ func (p *DeletePrm) WithForceRemoval() { // NOTE: Marks any object to be deleted (despite any prohibitions // on operations with that object) if WithForceRemoval option has // been provided. -func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRes, err error) { +func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Delete", trace.WithAttributes( attribute.String("address", prm.addr.EncodeToString()), @@ -62,15 +57,12 @@ func (e *StorageEngine) Delete(ctx context.Context, prm DeletePrm) (res DeleteRe defer span.End() defer elapsed("Delete", e.metrics.AddMethodDuration)() - err = e.execIfNotBlocked(func() error { - res, err = e.delete(ctx, prm) - return err + return e.execIfNotBlocked(func() error { + return e.delete(ctx, prm) }) - - return } -func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) { +func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) error { var locked struct { is bool } @@ -128,14 +120,14 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) (DeleteRes, e }) if locked.is { - return deleteRes, new(apistatus.ObjectLocked) + return new(apistatus.ObjectLocked) } if splitInfo != nil { e.deleteChildren(ctx, prm.addr, prm.forceRemoval, splitInfo.SplitID()) } - return deleteRes, nil + return nil } func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, force bool, splitID *objectSDK.SplitID) { diff --git a/pkg/local_object_storage/engine/delete_test.go b/pkg/local_object_storage/engine/delete_test.go index 0dd2e94bb..a56598c09 100644 --- a/pkg/local_object_storage/engine/delete_test.go +++ b/pkg/local_object_storage/engine/delete_test.go @@ -70,8 +70,7 @@ func TestDeleteBigObject(t *testing.T) { deletePrm.WithForceRemoval() deletePrm.WithAddress(addrParent) - _, err := e.Delete(context.Background(), deletePrm) - require.NoError(t, err) + require.NoError(t, e.Delete(context.Background(), deletePrm)) checkGetError[*apistatus.ObjectNotFound](t, e, addrParent, true) checkGetError[*apistatus.ObjectNotFound](t, e, addrLink, true) @@ -141,8 +140,7 @@ func TestDeleteBigObjectWithoutGC(t *testing.T) { deletePrm.WithForceRemoval() deletePrm.WithAddress(addrParent) - _, err := e.Delete(context.Background(), deletePrm) - require.NoError(t, err) + require.NoError(t, e.Delete(context.Background(), deletePrm)) checkGetError[*apistatus.ObjectNotFound](t, e, addrParent, true) checkGetError[*apistatus.ObjectNotFound](t, e, addrLink, true) @@ -153,7 +151,7 @@ func TestDeleteBigObjectWithoutGC(t *testing.T) { // delete physical var delPrm shard.DeletePrm delPrm.SetAddresses(addrParent) - _, err = s1.Delete(context.Background(), delPrm) + _, err := s1.Delete(context.Background(), delPrm) require.NoError(t, err) delPrm.SetAddresses(addrLink) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index a0c38cd5d..75bd15c8b 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -27,11 +27,6 @@ type InhumePrm struct { forceRemoval bool } -// InhumeRes encapsulates results of inhume operation. -type InhumeRes struct{} - -var inhumeRes = InhumeRes{} - // WithTarget sets a list of objects that should be inhumed and tombstone address // as the reason for inhume operation. // @@ -69,23 +64,20 @@ var errInhumeFailure = errors.New("inhume operation failed") // with that object) if WithForceRemoval option has been provided. // // Returns an error if executions are blocked (see BlockExecution). -func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) (res InhumeRes, err error) { +func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.Inhume") defer span.End() defer elapsed("Inhume", e.metrics.AddMethodDuration)() - err = e.execIfNotBlocked(func() error { - res, err = e.inhume(ctx, prm) - return err + return e.execIfNotBlocked(func() error { + return e.inhume(ctx, prm) }) - - return } -func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { +func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) error { addrsPerShard, err := e.groupObjectsByShard(ctx, prm.addrs, !prm.forceRemoval) if err != nil { - return inhumeRes, err + return err } var shPrm shard.InhumePrm @@ -109,7 +101,7 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e zap.String("shard_id", shardID), zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) - return inhumeRes, errInhumeFailure + return errInhumeFailure } if _, err := sh.Inhume(ctx, shPrm); err != nil { @@ -121,11 +113,11 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) (InhumeRes, e default: e.reportShardError(ctx, sh, "couldn't inhume object in shard", err) } - return inhumeRes, err + return err } } - return inhumeRes, nil + return nil } // groupObjectsByShard groups objects based on the shard(s) they are stored on. diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 2d083a58c..8c5d28b15 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -55,7 +55,7 @@ func TestStorageEngine_Inhume(t *testing.T) { var inhumePrm InhumePrm inhumePrm.WithTarget(tombstoneID, object.AddressOf(parent)) - _, err = e.Inhume(context.Background(), inhumePrm) + err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) addrs, err := Select(context.Background(), e, cnr, false, fs) @@ -85,7 +85,7 @@ func TestStorageEngine_Inhume(t *testing.T) { var inhumePrm InhumePrm inhumePrm.WithTarget(tombstoneID, object.AddressOf(parent)) - _, err = e.Inhume(context.Background(), inhumePrm) + err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) addrs, err := Select(context.Background(), e, cnr, false, fs) @@ -128,7 +128,7 @@ func TestStorageEngine_ECInhume(t *testing.T) { var inhumePrm InhumePrm inhumePrm.WithTarget(tombstoneObjectAddress, parentObjectAddress) - _, err = e.Inhume(context.Background(), inhumePrm) + err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) var alreadyRemoved *apistatus.ObjectAlreadyRemoved @@ -173,7 +173,7 @@ func TestInhumeExpiredRegularObject(t *testing.T) { var prm InhumePrm prm.WithTarget(ts, object.AddressOf(obj)) - _, err := engine.Inhume(context.Background(), prm) + err := engine.Inhume(context.Background(), prm) require.NoError(t, err) }) @@ -182,7 +182,7 @@ func TestInhumeExpiredRegularObject(t *testing.T) { var prm InhumePrm prm.MarkAsGarbage(object.AddressOf(obj)) - _, err := engine.Inhume(context.Background(), prm) + err := engine.Inhume(context.Background(), prm) require.NoError(t, err) }) } @@ -237,7 +237,7 @@ func benchmarkInhumeMultipart(b *testing.B, numShards, numObjects int) { prm.WithTarget(ts, addrs...) b.StartTimer() - _, err := engine.Inhume(context.Background(), prm) + err := engine.Inhume(context.Background(), prm) require.NoError(b, err) b.StopTimer() } diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 7bb9e3934..b8c9d6b1d 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -114,7 +114,7 @@ func TestLockUserScenario(t *testing.T) { inhumePrm.WithTarget(tombAddr, objAddr) var objLockedErr *apistatus.ObjectLocked - _, err = e.Inhume(context.Background(), inhumePrm) + err = e.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) // 4. @@ -127,7 +127,7 @@ func TestLockUserScenario(t *testing.T) { inhumePrm.WithTarget(tombForLockAddr, lockerAddr) - _, err = e.Inhume(context.Background(), inhumePrm) + err = e.Inhume(context.Background(), inhumePrm) require.ErrorIs(t, err, meta.ErrLockObjectRemoval) // 5. @@ -136,7 +136,7 @@ func TestLockUserScenario(t *testing.T) { inhumePrm.WithTarget(tombAddr, objAddr) require.Eventually(t, func() bool { - _, err = e.Inhume(context.Background(), inhumePrm) + err = e.Inhume(context.Background(), inhumePrm) return err == nil }, 30*time.Second, time.Second) } @@ -200,7 +200,7 @@ func TestLockExpiration(t *testing.T) { inhumePrm.WithTarget(tombAddr, objectcore.AddressOf(obj)) var objLockedErr *apistatus.ObjectLocked - _, err = e.Inhume(context.Background(), inhumePrm) + err = e.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) // 3. @@ -212,7 +212,7 @@ func TestLockExpiration(t *testing.T) { inhumePrm.WithTarget(tombAddr, objectcore.AddressOf(obj)) require.Eventually(t, func() bool { - _, err = e.Inhume(context.Background(), inhumePrm) + err = e.Inhume(context.Background(), inhumePrm) return err == nil }, 30*time.Second, time.Second) } @@ -270,12 +270,12 @@ func TestLockForceRemoval(t *testing.T) { inhumePrm.MarkAsGarbage(objectcore.AddressOf(obj)) var objLockedErr *apistatus.ObjectLocked - _, err = e.Inhume(context.Background(), inhumePrm) + err = e.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) inhumePrm.WithTarget(oidtest.Address(), objectcore.AddressOf(obj)) - _, err = e.Inhume(context.Background(), inhumePrm) + err = e.Inhume(context.Background(), inhumePrm) require.ErrorAs(t, err, &objLockedErr) // 4. @@ -283,13 +283,12 @@ func TestLockForceRemoval(t *testing.T) { deletePrm.WithAddress(objectcore.AddressOf(lock)) deletePrm.WithForceRemoval() - _, err = e.Delete(context.Background(), deletePrm) - require.NoError(t, err) + require.NoError(t, e.Delete(context.Background(), deletePrm)) // 5. inhumePrm.MarkAsGarbage(objectcore.AddressOf(obj)) - _, err = e.Inhume(context.Background(), inhumePrm) + err = e.Inhume(context.Background(), inhumePrm) require.NoError(t, err) } diff --git a/pkg/services/control/server/gc.go b/pkg/services/control/server/gc.go index d9fefc38e..a8ef7809e 100644 --- a/pkg/services/control/server/gc.go +++ b/pkg/services/control/server/gc.go @@ -42,8 +42,7 @@ func (s *Server) DropObjects(ctx context.Context, req *control.DropObjectsReques prm.WithForceRemoval() prm.WithAddress(addrList[i]) - _, err := s.s.Delete(ctx, prm) - if err != nil && firstErr == nil { + if err := s.s.Delete(ctx, prm); err != nil && firstErr == nil { firstErr = err } } From 05fd999162f30042a0021f14a250aa3305b79f29 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 14 Jan 2025 10:54:53 +0300 Subject: [PATCH 1678/1943] [#1600] fstree: Handle incomplete writes Signed-off-by: Evgenii Stratonikov --- .../blobstor/fstree/fstree_write_linux.go | 25 +++++++++-- .../fstree/fstree_write_linux_test.go | 42 +++++++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/fstree/fstree_write_linux_test.go diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go index c62654028..49cbda344 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux.go @@ -69,10 +69,13 @@ func (w *linuxWriter) writeFile(p string, data []byte) error { if err != nil { return err } + written := 0 tmpPath := "/proc/self/fd/" + strconv.FormatUint(uint64(fd), 10) n, err := unix.Write(fd, data) - if err == nil { - if n == len(data) { + for err == nil { + written += n + + if written == len(data) { err = unix.Linkat(unix.AT_FDCWD, tmpPath, unix.AT_FDCWD, p, unix.AT_SYMLINK_FOLLOW) if err == nil { w.fileCounter.Inc(uint64(len(data))) @@ -80,9 +83,23 @@ func (w *linuxWriter) writeFile(p string, data []byte) error { if errors.Is(err, unix.EEXIST) { err = nil } - } else { - err = errors.New("incomplete write") + break } + + // From man 2 write: + // https://www.man7.org/linux/man-pages/man2/write.2.html + // + // Note that a successful write() may transfer fewer than count + // bytes. Such partial writes can occur for various reasons; for + // example, because there was insufficient space on the disk device + // to write all of the requested bytes, or because a blocked write() + // to a socket, pipe, or similar was interrupted by a signal handler + // after it had transferred some, but before it had transferred all + // of the requested bytes. In the event of a partial write, the + // caller can make another write() call to transfer the remaining + // bytes. The subsequent call will either transfer further bytes or + // may result in an error (e.g., if the disk is now full). + n, err = unix.Write(fd, data[written:]) } errClose := unix.Close(fd) if err != nil { diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_linux_test.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux_test.go new file mode 100644 index 000000000..7fae2e695 --- /dev/null +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_linux_test.go @@ -0,0 +1,42 @@ +//go:build linux && integration + +package fstree + +import ( + "context" + "errors" + "os" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestENOSPC(t *testing.T) { + dir, err := os.MkdirTemp(t.TempDir(), "ramdisk") + require.NoError(t, err) + + f, err := os.CreateTemp(t.TempDir(), "ramdisk_*") + require.NoError(t, err) + + err = unix.Mount(f.Name(), dir, "tmpfs", 0, "size=1M") + if errors.Is(err, unix.EPERM) { + t.Skipf("skip size tests: no permission to mount: %v", err) + return + } + require.NoError(t, err) + defer func() { + require.NoError(t, unix.Unmount(dir, 0)) + }() + + fst := New(WithPath(dir), WithDepth(1)) + require.NoError(t, fst.Open(mode.ComponentReadWrite)) + require.NoError(t, fst.Init()) + + _, err = fst.Put(context.Background(), common.PutPrm{ + RawData: make([]byte, 10<<20), + }) + require.ErrorIs(t, err, common.ErrNoSpace) +} From c3c034eccac1297a15645d1ff7995855cc45fc74 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 15 Jan 2025 14:19:56 +0300 Subject: [PATCH 1679/1943] [#1601] util: Correctly parse 'root' name for container resources * Convert `root/*` to `//`; * Add unit-test case for parses to check parsing correctness. Signed-off-by: Airat Arifullin --- pkg/util/ape/parser.go | 2 +- pkg/util/ape/parser_test.go | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/util/ape/parser.go b/pkg/util/ape/parser.go index b4a31fd8d..a34a17f6f 100644 --- a/pkg/util/ape/parser.go +++ b/pkg/util/ape/parser.go @@ -261,7 +261,7 @@ func parseResource(lexeme string, isObj bool) (string, error) { } else { if lexeme == "*" { return nativeschema.ResourceFormatAllContainers, nil - } else if lexeme == "/*" { + } else if lexeme == "/*" || lexeme == "root/*" { return nativeschema.ResourceFormatRootContainers, nil } else if strings.HasPrefix(lexeme, "/") && len(lexeme) > 1 { lexeme = lexeme[1:] diff --git a/pkg/util/ape/parser_test.go b/pkg/util/ape/parser_test.go index 21649fd24..c236c4603 100644 --- a/pkg/util/ape/parser_test.go +++ b/pkg/util/ape/parser_test.go @@ -43,6 +43,15 @@ func TestParseAPERule(t *testing.T) { Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootObjects}}, }, }, + { + name: "Valid rule for all containers in explicit root namespace", + rule: "allow Container.Put root/*", + expectRule: policyengine.Rule{ + Status: policyengine.Allow, + Actions: policyengine.Actions{Names: []string{nativeschema.MethodPutContainer}}, + Resources: policyengine.Resources{Names: []string{nativeschema.ResourceFormatRootContainers}}, + }, + }, { name: "Valid rule for all objects in root namespace and container", rule: "allow Object.Put /cid/*", From 436d65d784876248585edb94be9cf397a874caba Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Thu, 9 Jan 2025 12:07:17 +0300 Subject: [PATCH 1680/1943] [#1591] Build and host OCI images on our own infra Similar to https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/issues/587 this PR introduces a CI pipeline that builds Docker images and pushes them to our selfhosted registry. Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/oci-image.yml | 28 ++++++++++++++++++++++++++++ Makefile | 9 +++++++++ docs/release-instruction.md | 12 ++++-------- 3 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 .forgejo/workflows/oci-image.yml diff --git a/.forgejo/workflows/oci-image.yml b/.forgejo/workflows/oci-image.yml new file mode 100644 index 000000000..fe91d65f9 --- /dev/null +++ b/.forgejo/workflows/oci-image.yml @@ -0,0 +1,28 @@ +name: OCI image + +on: + push: + workflow_dispatch: + +jobs: + image: + name: Build container images + runs-on: docker + container: git.frostfs.info/truecloudlab/env:oci-image-builder-bookworm + steps: + - name: Clone git repo + uses: actions/checkout@v3 + + - name: Build OCI image + run: make images + + - name: Push image to OCI registry + run: | + echo "$REGISTRY_PASSWORD" \ + | docker login --username truecloudlab --password-stdin git.frostfs.info + make push-images + if: >- + startsWith(github.ref, 'refs/tags/v') && + (github.event_name == 'workflow_dispatch' || github.event_name == 'push') + env: + REGISTRY_PASSWORD: ${{secrets.FORGEJO_OCI_REGISTRY_PUSH_TOKEN}} diff --git a/Makefile b/Makefile index f0cdc273c..e9601a87e 100755 --- a/Makefile +++ b/Makefile @@ -139,6 +139,15 @@ images: image-storage image-ir image-cli image-adm # Build dirty local Docker images dirty-images: image-dirty-storage image-dirty-ir image-dirty-cli image-dirty-adm +# Push FrostFS components' docker image to the registry +push-image-%: + @echo "⇒ Publish FrostFS $* docker image " + @docker push $(HUB_IMAGE)-$*:$(HUB_TAG) + +# Push all Docker images to the registry +.PHONY: push-images +push-images: push-image-storage push-image-ir push-image-cli push-image-adm + # Run `make %` in Golang container docker/%: docker run --rm -t \ diff --git a/docs/release-instruction.md b/docs/release-instruction.md index 18659c699..aa867e83c 100644 --- a/docs/release-instruction.md +++ b/docs/release-instruction.md @@ -95,19 +95,15 @@ $ git push origin ${FROSTFS_TAG_PREFIX}${FROSTFS_REVISION} ## Post-release -### Prepare and push images to a Docker Hub (if not automated) +### Prepare and push images to a Docker registry (automated) -Create Docker images for all applications and push them into Docker Hub -(requires [organization](https://hub.docker.com/u/truecloudlab) privileges) +Create Docker images for all applications and push them into container registry +(executed automatically in Forgejo Actions upon pushing a release tag): ```shell $ git checkout ${FROSTFS_TAG_PREFIX}${FROSTFS_REVISION} $ make images -$ docker push truecloudlab/frostfs-storage:${FROSTFS_REVISION} -$ docker push truecloudlab/frostfs-storage-testnet:${FROSTFS_REVISION} -$ docker push truecloudlab/frostfs-ir:${FROSTFS_REVISION} -$ docker push truecloudlab/frostfs-cli:${FROSTFS_REVISION} -$ docker push truecloudlab/frostfs-adm:${FROSTFS_REVISION} +$ make push-images ``` ### Make a proper release (if not automated) From 5a270e2e61fc245130c5fc663e035a574ec6bc5e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 16 Jan 2025 15:08:35 +0300 Subject: [PATCH 1681/1943] [#1604] policer: Use status instead of bool value in node cache Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/nodecache.go | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/pkg/services/policer/nodecache.go b/pkg/services/policer/nodecache.go index cd47cb0fc..410ef7d86 100644 --- a/pkg/services/policer/nodecache.go +++ b/pkg/services/policer/nodecache.go @@ -15,37 +15,30 @@ func (st nodeProcessStatus) Processed() bool { } // nodeCache tracks Policer's check progress. -type nodeCache map[uint64]bool +type nodeCache map[uint64]nodeProcessStatus func newNodeCache() nodeCache { - return make(map[uint64]bool) + return make(map[uint64]nodeProcessStatus) } -func (n nodeCache) set(node netmap.NodeInfo, val bool) { +func (n nodeCache) set(node netmap.NodeInfo, val nodeProcessStatus) { n[node.Hash()] = val } // submits storage node as a candidate to store the object replica in case of // shortage. func (n nodeCache) submitReplicaCandidate(node netmap.NodeInfo) { - n.set(node, false) + n.set(node, nodeDoesNotHoldObject) } // submits storage node as a current object replica holder. func (n nodeCache) submitReplicaHolder(node netmap.NodeInfo) { - n.set(node, true) + n.set(node, nodeHoldsObject) } // processStatus returns current processing status of the storage node. func (n nodeCache) processStatus(node netmap.NodeInfo) nodeProcessStatus { - switch val, ok := n[node.Hash()]; { - case !ok: - return nodeNotProcessed - case val: - return nodeHoldsObject - default: - return nodeDoesNotHoldObject - } + return n[node.Hash()] } // SubmitSuccessfulReplication marks given storage node as a current object From 84e1599997c3d4674c418556dccbeaeb1d2414ed Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 16 Jan 2025 15:52:21 +0300 Subject: [PATCH 1682/1943] [#1604] policer: Remove one-line helpers Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/check.go | 6 +++--- pkg/services/policer/check_test.go | 4 ++-- pkg/services/policer/nodecache.go | 13 +------------ 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index bdfc4344b..9335a75d8 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -143,10 +143,10 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe if err == nil { shortage-- - checkedNodes.submitReplicaHolder(nodes[i]) + checkedNodes.set(nodes[i], nodeHoldsObject) } else { if client.IsErrObjectNotFound(err) { - checkedNodes.submitReplicaCandidate(nodes[i]) + checkedNodes.set(nodes[i], nodeDoesNotHoldObject) continue } else if client.IsErrNodeUnderMaintenance(err) { shortage, uncheckedCopies = p.handleMaintenance(ctx, nodes[i], checkedNodes, shortage, uncheckedCopies) @@ -174,7 +174,7 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe // However, additional copies should not be removed in this case, // because we can remove the only copy this way. func (p *Policer) handleMaintenance(ctx context.Context, node netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { - checkedNodes.submitReplicaHolder(node) + checkedNodes.set(node, nodeHoldsObject) shortage-- uncheckedCopies++ diff --git a/pkg/services/policer/check_test.go b/pkg/services/policer/check_test.go index d4c7ccbf9..69879c439 100644 --- a/pkg/services/policer/check_test.go +++ b/pkg/services/policer/check_test.go @@ -16,9 +16,9 @@ func TestNodeCache(t *testing.T) { cache.SubmitSuccessfulReplication(node) require.Equal(t, cache.processStatus(node), nodeHoldsObject) - cache.submitReplicaCandidate(node) + cache.set(node, nodeDoesNotHoldObject) require.Equal(t, cache.processStatus(node), nodeDoesNotHoldObject) - cache.submitReplicaHolder(node) + cache.set(node, nodeHoldsObject) require.Equal(t, cache.processStatus(node), nodeHoldsObject) } diff --git a/pkg/services/policer/nodecache.go b/pkg/services/policer/nodecache.go index 410ef7d86..7a8217858 100644 --- a/pkg/services/policer/nodecache.go +++ b/pkg/services/policer/nodecache.go @@ -25,17 +25,6 @@ func (n nodeCache) set(node netmap.NodeInfo, val nodeProcessStatus) { n[node.Hash()] = val } -// submits storage node as a candidate to store the object replica in case of -// shortage. -func (n nodeCache) submitReplicaCandidate(node netmap.NodeInfo) { - n.set(node, nodeDoesNotHoldObject) -} - -// submits storage node as a current object replica holder. -func (n nodeCache) submitReplicaHolder(node netmap.NodeInfo) { - n.set(node, nodeHoldsObject) -} - // processStatus returns current processing status of the storage node. func (n nodeCache) processStatus(node netmap.NodeInfo) nodeProcessStatus { return n[node.Hash()] @@ -46,5 +35,5 @@ func (n nodeCache) processStatus(node netmap.NodeInfo) nodeProcessStatus { // // SubmitSuccessfulReplication implements replicator.TaskResult. func (n nodeCache) SubmitSuccessfulReplication(node netmap.NodeInfo) { - n.submitReplicaHolder(node) + n.set(node, nodeHoldsObject) } From 4538ccb12a4769eeb397417f0b755dc1bc7bd712 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 16 Jan 2025 15:59:17 +0300 Subject: [PATCH 1683/1943] [#1604] policer: Do not process the same node twice Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/check.go | 6 +++--- pkg/services/policer/nodecache.go | 1 + pkg/services/policer/policer_test.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 9335a75d8..f79ffbece 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -126,12 +126,11 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe } else { if status := checkedNodes.processStatus(nodes[i]); status.Processed() { if status == nodeHoldsObject { - // node already contains replica, no need to replicate - nodes = append(nodes[:i], nodes[i+1:]...) - i-- shortage-- } + nodes = append(nodes[:i], nodes[i+1:]...) + i-- continue } @@ -155,6 +154,7 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe zap.Stringer("object", addr), zap.Error(err), ) + checkedNodes.set(nodes[i], nodeStatusUnknown) } } } diff --git a/pkg/services/policer/nodecache.go b/pkg/services/policer/nodecache.go index 7a8217858..84a333278 100644 --- a/pkg/services/policer/nodecache.go +++ b/pkg/services/policer/nodecache.go @@ -8,6 +8,7 @@ const ( nodeNotProcessed nodeProcessStatus = iota nodeDoesNotHoldObject nodeHoldsObject + nodeStatusUnknown ) func (st nodeProcessStatus) Processed() bool { diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 4e17e98a8..2ed71c404 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -127,7 +127,7 @@ func TestProcessObject(t *testing.T) { nodeCount: 2, policy: `REP 2 REP 2`, placement: [][]int{{0, 1}, {0, 1}}, - wantReplicateTo: []int{1, 1}, // is this actually good? + wantReplicateTo: []int{1}, }, { desc: "lock object must be replicated to all nodes", From 26e0c82fb8ad0d74782e2c96cbf4d7bcdd76e428 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 16 Jan 2025 16:22:37 +0300 Subject: [PATCH 1684/1943] [#1604] policer/test: Add test for MAINTENANCE runtime status The node can have MAINTENANCE status in the network map, but can also be ONLINE while responding with MAINTENANCE. These are 2 different code paths, let's test them separately. Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/policer_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 2ed71c404..ca6bff944 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -78,6 +78,7 @@ func TestProcessObject(t *testing.T) { maintenanceNodes []int wantRemoveRedundant bool wantReplicateTo []int + headResult map[int]error ecInfo *objectcore.ECInfo }{ { @@ -145,6 +146,14 @@ func TestProcessObject(t *testing.T) { objHolders: []int{1}, maintenanceNodes: []int{2}, }, + { + desc: "preserve local copy when node response with MAINTENANCE", + nodeCount: 3, + policy: `REP 2`, + placement: [][]int{{1, 2}}, + objHolders: []int{1}, + headResult: map[int]error{2: new(apistatus.NodeUnderMaintenance)}, + }, { desc: "lock object must be replicated to all EC nodes", objType: objectSDK.TypeLock, @@ -204,6 +213,11 @@ func TestProcessObject(t *testing.T) { t.Errorf("unexpected remote object head: node=%+v addr=%v", ni, a) return nil, errors.New("unexpected object head") } + if ti.headResult != nil { + if err, ok := ti.headResult[index]; ok { + return nil, err + } + } for _, i := range ti.objHolders { if index == i { return nil, nil From 57efa0bc8eebbc7b26ef9d0c14b4dbbdb7b59de7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 16 Jan 2025 16:04:40 +0300 Subject: [PATCH 1685/1943] [#1604] policer: Properly handle maintenance nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consider `REP 1 REP 1` placement (selects/filters are omitted). The placement is `[1, 2], [1, 0]`. We are the 0-th node. Node 1 is under maintenance, so we do not replicate object on the node 2. In the second replication group node 1 is under maintenance, but current caching logic considers it as "replica holder" and removes local copy. Voilà, we have DL if the object is missing from the node 1. Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/check.go | 6 +++++- pkg/services/policer/nodecache.go | 1 + pkg/services/policer/policer_test.go | 8 ++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index f79ffbece..7ac5fc9e0 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -128,6 +128,10 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe if status == nodeHoldsObject { shortage-- } + if status == nodeIsUnderMaintenance { + shortage-- + uncheckedCopies++ + } nodes = append(nodes[:i], nodes[i+1:]...) i-- @@ -174,7 +178,7 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe // However, additional copies should not be removed in this case, // because we can remove the only copy this way. func (p *Policer) handleMaintenance(ctx context.Context, node netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { - checkedNodes.set(node, nodeHoldsObject) + checkedNodes.set(node, nodeIsUnderMaintenance) shortage-- uncheckedCopies++ diff --git a/pkg/services/policer/nodecache.go b/pkg/services/policer/nodecache.go index 84a333278..53b64d3fa 100644 --- a/pkg/services/policer/nodecache.go +++ b/pkg/services/policer/nodecache.go @@ -9,6 +9,7 @@ const ( nodeDoesNotHoldObject nodeHoldsObject nodeStatusUnknown + nodeIsUnderMaintenance ) func (st nodeProcessStatus) Processed() bool { diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index ca6bff944..9b9ab99ac 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -170,6 +170,14 @@ func TestProcessObject(t *testing.T) { placement: [][]int{{0, 1, 2}}, wantReplicateTo: []int{1, 2}, }, + { + desc: "do not remove local copy when MAINTENANCE status is cached", + objType: objectSDK.TypeRegular, + nodeCount: 3, + policy: `REP 1 REP 1`, + placement: [][]int{{1, 2}, {1, 0}}, + headResult: map[int]error{1: new(apistatus.NodeUnderMaintenance)}, + }, } for i := range tests { From 80de5d70bfd5bca16b9c891b8fdef25165e0bba1 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 16 Jan 2025 15:25:05 +0300 Subject: [PATCH 1686/1943] [#1593] node: Fix initialization of ape_chain cache Signed-off-by: Alexander Chuprov --- cmd/frostfs-node/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 9931d0dc8..3e9cd4e11 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1146,7 +1146,7 @@ func initAccessPolicyEngine(ctx context.Context, c *cfg) { c.cfgObject.cfgAccessPolicyEngine.policyContractHash) cacheSize := morphconfig.APEChainCacheSize(c.appCfg) - if cacheSize > 0 { + if cacheSize > 0 && c.cfgMorph.cacheTTL > 0 { morphRuleStorage = newMorphCache(morphRuleStorage, int(cacheSize), c.cfgMorph.cacheTTL) } From c98357606b4e387f7a8063331438dccfb24d255e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 17 Jan 2025 13:51:18 +0300 Subject: [PATCH 1687/1943] [#1606] Use slices.Clone()/bytes.Clone() where possible gopatch: ``` @@ var from, to expression @@ +import "bytes" -to := make([]byte, len(from)) -copy(to, from) +to := bytes.Clone(from) @@ var from, to expression @@ +import "bytes" -to = make([]byte, len(from)) -copy(to, from) +to = bytes.Clone(from) @@ var from, to, typ expression @@ +import "slices" -to := make([]typ, len(from)) -copy(to, from) +to := slices.Clone(from) @@ var from, to, typ expression @@ +import "slices" -to = make([]typ, len(from)) -copy(to, from) +to = slices.Clone(from) ``` Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/storagecfg/root.go | 4 ++-- cmd/frostfs-node/config/calls.go | 4 ++-- pkg/local_object_storage/engine/evacuate.go | 4 ++-- pkg/local_object_storage/engine/evacuate_limiter.go | 4 ++-- pkg/local_object_storage/metabase/list.go | 6 ++---- pkg/local_object_storage/pilorama/boltdb.go | 3 +-- pkg/local_object_storage/pilorama/forest.go | 4 ++-- pkg/morph/client/multi.go | 4 ++-- pkg/services/object/common/writer/ec_test.go | 4 ++-- pkg/services/object/search/search_test.go | 4 ++-- pkg/services/object_manager/placement/traverser_test.go | 4 ++-- 11 files changed, 21 insertions(+), 24 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/root.go b/cmd/frostfs-adm/internal/modules/storagecfg/root.go index 8e6a8354e..8acbc4579 100644 --- a/cmd/frostfs-adm/internal/modules/storagecfg/root.go +++ b/cmd/frostfs-adm/internal/modules/storagecfg/root.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "path/filepath" + "slices" "strconv" "strings" "text/template" @@ -410,8 +411,7 @@ func initClient(rpc []string) *rpcclient.Client { var c *rpcclient.Client var err error - shuffled := make([]string, len(rpc)) - copy(shuffled, rpc) + shuffled := slices.Clone(rpc) rand.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] }) for _, endpoint := range shuffled { diff --git a/cmd/frostfs-node/config/calls.go b/cmd/frostfs-node/config/calls.go index 36e53ea7c..c40bf3620 100644 --- a/cmd/frostfs-node/config/calls.go +++ b/cmd/frostfs-node/config/calls.go @@ -1,6 +1,7 @@ package config import ( + "slices" "strings" configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" @@ -52,6 +53,5 @@ func (x *Config) Value(name string) any { // It supports only one level of nesting and is intended to be used // to provide default values. func (x *Config) SetDefault(from *Config) { - x.defaultPath = make([]string, len(from.path)) - copy(x.defaultPath, from.path) + x.defaultPath = slices.Clone(from.path) } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 682f23dff..b37f6b68e 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "strings" "sync" "sync/atomic" @@ -255,8 +256,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) erro copyShards := func() []pooledShard { mtx.RLock() defer mtx.RUnlock() - t := make([]pooledShard, len(shards)) - copy(t, shards) + t := slices.Clone(shards) return t } eg.Go(func() error { diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go index 1e6b9ccb1..c74134500 100644 --- a/pkg/local_object_storage/engine/evacuate_limiter.go +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -3,6 +3,7 @@ package engine import ( "context" "fmt" + "slices" "sync" "time" @@ -123,8 +124,7 @@ func (s *EvacuationState) DeepCopy() *EvacuationState { if s == nil { return nil } - shardIDs := make([]string, len(s.shardIDs)) - copy(shardIDs, s.shardIDs) + shardIDs := slices.Clone(s.shardIDs) return &EvacuationState{ shardIDs: shardIDs, diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index f488c3ced..eaef3b9ba 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -188,8 +188,7 @@ loop: if offset != nil { // new slice is much faster but less memory efficient // we need to copy, because offset exists during bbolt tx - cursor.inBucketOffset = make([]byte, len(offset)) - copy(cursor.inBucketOffset, offset) + cursor.inBucketOffset = bytes.Clone(offset) } if len(result) == 0 { @@ -198,8 +197,7 @@ loop: // new slice is much faster but less memory efficient // we need to copy, because bucketName exists during bbolt tx - cursor.bucketName = make([]byte, len(bucketName)) - copy(cursor.bucketName, bucketName) + cursor.bucketName = bytes.Clone(bucketName) return result, cursor, nil } diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 1d55d1c13..9d71d9fda 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1506,8 +1506,7 @@ func (t *boltForest) TreeListTrees(ctx context.Context, prm TreeListTreesPrm) (* }) if len(res.Items) == batchSize { - res.NextPageToken = make([]byte, len(k)) - copy(res.NextPageToken, k) + res.NextPageToken = bytes.Clone(k) break } } diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index f31504e2b..92183716c 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "sort" "strings" @@ -84,8 +85,7 @@ func (f *memoryForest) TreeAddByPath(_ context.Context, d CIDDescriptor, treeID s.operations = append(s.operations, op) } - mCopy := make([]KeyValue, len(m)) - copy(mCopy, m) + mCopy := slices.Clone(m) op := s.do(&Move{ Parent: node, Meta: Meta{ diff --git a/pkg/morph/client/multi.go b/pkg/morph/client/multi.go index 708d3b39f..b9e39c25e 100644 --- a/pkg/morph/client/multi.go +++ b/pkg/morph/client/multi.go @@ -2,6 +2,7 @@ package client import ( "context" + "slices" "sort" "time" @@ -99,8 +100,7 @@ mainLoop: case <-t.C: c.switchLock.RLock() - endpointsCopy := make([]Endpoint, len(c.endpoints.list)) - copy(endpointsCopy, c.endpoints.list) + endpointsCopy := slices.Clone(c.endpoints.list) currPriority := c.endpoints.list[c.endpoints.curr].Priority highestPriority := c.endpoints.list[0].Priority diff --git a/pkg/services/object/common/writer/ec_test.go b/pkg/services/object/common/writer/ec_test.go index 8b2599e5f..8ad7e641a 100644 --- a/pkg/services/object/common/writer/ec_test.go +++ b/pkg/services/object/common/writer/ec_test.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "errors" "fmt" + "slices" "strconv" "testing" @@ -41,8 +42,7 @@ type testPlacementBuilder struct { func (p *testPlacementBuilder) BuildPlacement(_ cid.ID, _ *oid.ID, _ netmap.PlacementPolicy) ( [][]netmap.NodeInfo, error, ) { - arr := make([]netmap.NodeInfo, len(p.vectors[0])) - copy(arr, p.vectors[0]) + arr := slices.Clone(p.vectors[0]) return [][]netmap.NodeInfo{arr}, nil } diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index 0a40025e1..05643eb2b 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "errors" "fmt" + "slices" "strconv" "testing" @@ -103,8 +104,7 @@ func (p *testPlacementBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, _ netmap. return nil, errors.New("vectors for address not found") } - res := make([][]netmap.NodeInfo, len(vs)) - copy(res, vs) + res := slices.Clone(vs) return res, nil } diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index f96e5c8a7..624efb007 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -1,6 +1,7 @@ package placement import ( + "slices" "strconv" "testing" @@ -33,8 +34,7 @@ func copyVectors(v [][]netmap.NodeInfo) [][]netmap.NodeInfo { vc := make([][]netmap.NodeInfo, 0, len(v)) for i := range v { - ns := make([]netmap.NodeInfo, len(v[i])) - copy(ns, v[i]) + ns := slices.Clone(v[i]) vc = append(vc, ns) } From 0bcbeb26b2b79d210d4b99360e1ad58ae809faab Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 17 Jan 2025 12:38:44 +0300 Subject: [PATCH 1688/1943] [#1605] policer: Simplify processRepNodes() checks Current flow is hard to reason about, #1601 is a notorious example of accidental complexity. 1. Remove multiple nested ifs, use depth=1. 2. Process each status exactly once, hopefully preventing bugs like #1601. Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/check.go | 106 ++++++++++++++---------------- pkg/services/policer/nodecache.go | 1 + 2 files changed, 52 insertions(+), 55 deletions(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 7ac5fc9e0..3e536f105 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -117,50 +117,40 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe default: } - if p.netmapKeys.IsLocalKey(nodes[i].PublicKey()) { + var err error + st := checkedNodes.processStatus(nodes[i]) + if !st.Processed() { + st, err = p.checkStatus(ctx, addr, nodes[i]) + checkedNodes.set(nodes[i], st) + if st == nodeDoesNotHoldObject { + // 1. This is the first time the node is encountered (`!st.Processed()`). + // 2. The node does not hold object (`st == nodeDoesNotHoldObject`). + // So we leave the node in the list and skip its removal + // at the end of the loop body. + continue + } + } + + switch st { + case nodeIsLocal: requirements.needLocalCopy = true shortage-- - } else if nodes[i].Status().IsMaintenance() { - shortage, uncheckedCopies = p.handleMaintenance(ctx, nodes[i], checkedNodes, shortage, uncheckedCopies) - } else { - if status := checkedNodes.processStatus(nodes[i]); status.Processed() { - if status == nodeHoldsObject { - shortage-- - } - if status == nodeIsUnderMaintenance { - shortage-- - uncheckedCopies++ - } + case nodeIsUnderMaintenance: + shortage-- + uncheckedCopies++ - nodes = append(nodes[:i], nodes[i+1:]...) - i-- - continue - } - - callCtx, cancel := context.WithTimeout(ctx, p.headTimeout) - - _, err := p.remoteHeader(callCtx, nodes[i], addr, false) - - cancel() - - if err == nil { - shortage-- - checkedNodes.set(nodes[i], nodeHoldsObject) - } else { - if client.IsErrObjectNotFound(err) { - checkedNodes.set(nodes[i], nodeDoesNotHoldObject) - continue - } else if client.IsErrNodeUnderMaintenance(err) { - shortage, uncheckedCopies = p.handleMaintenance(ctx, nodes[i], checkedNodes, shortage, uncheckedCopies) - } else { - p.log.Error(ctx, logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, - zap.Stringer("object", addr), - zap.Error(err), - ) - checkedNodes.set(nodes[i], nodeStatusUnknown) - } - } + p.log.Debug(ctx, logs.PolicerConsiderNodeUnderMaintenanceAsOK, + zap.String("node", netmap.StringifyPublicKey(nodes[i]))) + case nodeHoldsObject: + shortage-- + case nodeDoesNotHoldObject: + case nodeStatusUnknown: + p.log.Error(ctx, logs.PolicerReceiveObjectHeaderToCheckPolicyCompliance, + zap.Stringer("object", addr), + zap.Error(err)) + default: + panic("unreachable") } nodes = append(nodes[:i], nodes[i+1:]...) @@ -170,22 +160,28 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe p.handleProcessNodesResult(ctx, addr, requirements, nodes, checkedNodes, shortage, uncheckedCopies) } -// handleMaintenance handles node in maintenance mode and returns new shortage and uncheckedCopies values -// -// consider remote nodes under maintenance as problem OK. Such -// nodes MAY not respond with object, however, this is how we -// prevent spam with new replicas. -// However, additional copies should not be removed in this case, -// because we can remove the only copy this way. -func (p *Policer) handleMaintenance(ctx context.Context, node netmap.NodeInfo, checkedNodes nodeCache, shortage uint32, uncheckedCopies int) (uint32, int) { - checkedNodes.set(node, nodeIsUnderMaintenance) - shortage-- - uncheckedCopies++ +func (p *Policer) checkStatus(ctx context.Context, addr oid.Address, node netmap.NodeInfo) (nodeProcessStatus, error) { + if p.netmapKeys.IsLocalKey(node.PublicKey()) { + return nodeIsLocal, nil + } + if node.Status().IsMaintenance() { + return nodeIsUnderMaintenance, nil + } - p.log.Debug(ctx, logs.PolicerConsiderNodeUnderMaintenanceAsOK, - zap.String("node", netmap.StringifyPublicKey(node)), - ) - return shortage, uncheckedCopies + callCtx, cancel := context.WithTimeout(ctx, p.headTimeout) + _, err := p.remoteHeader(callCtx, node, addr, false) + cancel() + + if err == nil { + return nodeHoldsObject, nil + } + if client.IsErrObjectNotFound(err) { + return nodeDoesNotHoldObject, nil + } + if client.IsErrNodeUnderMaintenance(err) { + return nodeIsUnderMaintenance, nil + } + return nodeStatusUnknown, err } func (p *Policer) handleProcessNodesResult(ctx context.Context, addr oid.Address, requirements *placementRequirements, diff --git a/pkg/services/policer/nodecache.go b/pkg/services/policer/nodecache.go index 53b64d3fa..c2157de5d 100644 --- a/pkg/services/policer/nodecache.go +++ b/pkg/services/policer/nodecache.go @@ -10,6 +10,7 @@ const ( nodeHoldsObject nodeStatusUnknown nodeIsUnderMaintenance + nodeIsLocal ) func (st nodeProcessStatus) Processed() bool { From 951a7ee1c7fa9ffa96041e77731e3bacea16242b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 20 Jan 2025 16:09:35 +0300 Subject: [PATCH 1689/1943] [#1605] policer: Do not mutate slice under iteration Nothing wrong with it, besides being difficult to read. Signed-off-by: Evgenii Stratonikov --- pkg/services/policer/check.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 3e536f105..7df372476 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -110,6 +110,7 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe // Number of copies that are stored on maintenance nodes. var uncheckedCopies int + var candidates []netmap.NodeInfo for i := 0; shortage > 0 && i < len(nodes); i++ { select { case <-ctx.Done(): @@ -125,8 +126,8 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe if st == nodeDoesNotHoldObject { // 1. This is the first time the node is encountered (`!st.Processed()`). // 2. The node does not hold object (`st == nodeDoesNotHoldObject`). - // So we leave the node in the list and skip its removal - // at the end of the loop body. + // So we need to try to put an object to it. + candidates = append(candidates, nodes[i]) continue } } @@ -152,12 +153,9 @@ func (p *Policer) processRepNodes(ctx context.Context, requirements *placementRe default: panic("unreachable") } - - nodes = append(nodes[:i], nodes[i+1:]...) - i-- } - p.handleProcessNodesResult(ctx, addr, requirements, nodes, checkedNodes, shortage, uncheckedCopies) + p.handleProcessNodesResult(ctx, addr, requirements, candidates, checkedNodes, shortage, uncheckedCopies) } func (p *Policer) checkStatus(ctx context.Context, addr oid.Address, node netmap.NodeInfo) (nodeProcessStatus, error) { From 30e14d50ef97e04758b5fb529fb24a5f912d247b Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 23 Jan 2025 17:56:52 +0300 Subject: [PATCH 1690/1943] [#1612] Makefile: Update golangci-lint Signed-off-by: Alexander Chuprov --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e9601a87e..497dce115 100755 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.22 -LINT_VERSION ?= 1.62.0 +LINT_VERSION ?= 1.62.2 TRUECLOUDLAB_LINT_VERSION ?= 0.0.8 PROTOC_VERSION ?= 25.0 PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-sdk-go) From 603015d029ba15dcc9f84e19b14eea1e38e55309 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 24 Jan 2025 16:33:32 +0300 Subject: [PATCH 1691/1943] [#1570] cli: Use array type for --range parameter to object hash Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/modules/object/hash.go | 2 +- cmd/frostfs-cli/modules/object/range.go | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/hash.go b/cmd/frostfs-cli/modules/object/hash.go index 461c35f30..d8ea449eb 100644 --- a/cmd/frostfs-cli/modules/object/hash.go +++ b/cmd/frostfs-cli/modules/object/hash.go @@ -41,7 +41,7 @@ func initObjectHashCmd() { flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage) _ = objectHashCmd.MarkFlagRequired(commonflags.OIDFlag) - flags.String("range", "", "Range to take hash from in the form offset1:length1,...") + flags.StringSlice("range", nil, "Range to take hash from in the form offset1:length1,...") _ = objectHashCmd.MarkFlagRequired("range") flags.String("type", hashSha256, "Hash type. Either 'sha256' or 'tz'") diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index ad4bc3d59..8f59906ca 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -38,7 +38,7 @@ func initObjectRangeCmd() { flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage) _ = objectRangeCmd.MarkFlagRequired(commonflags.OIDFlag) - flags.String("range", "", "Range to take data from in the form offset:length") + flags.StringSlice("range", nil, "Range to take data from in the form offset:length") flags.String(fileFlag, "", "File to write object payload to. Default: stdout.") flags.Bool(rawFlag, false, rawFlagDesc) } @@ -195,11 +195,10 @@ func marshalECInfo(cmd *cobra.Command, info *objectSDK.ECInfo) ([]byte, error) { } func getRangeList(cmd *cobra.Command) ([]objectSDK.Range, error) { - v := cmd.Flag("range").Value.String() - if len(v) == 0 { - return nil, nil + vs, err := cmd.Flags().GetStringSlice("range") + if len(vs) == 0 || err != nil { + return nil, err } - vs := strings.Split(v, ",") rs := make([]objectSDK.Range, len(vs)) for i := range vs { before, after, found := strings.Cut(vs[i], rangeSep) From a788d4477342b92f7fa9c3bc9c34be8a9c9b45df Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 24 Jan 2025 16:54:27 +0300 Subject: [PATCH 1692/1943] [#1570] cli: Use array type for attributes parameters Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/modules/object/patch.go | 10 ++++------ cmd/frostfs-cli/modules/object/put.go | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/patch.go b/cmd/frostfs-cli/modules/object/patch.go index 8f03885ab..ebc415b2f 100644 --- a/cmd/frostfs-cli/modules/object/patch.go +++ b/cmd/frostfs-cli/modules/object/patch.go @@ -46,7 +46,7 @@ func initObjectPatchCmd() { flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage) _ = objectRangeCmd.MarkFlagRequired(commonflags.OIDFlag) - flags.String(newAttrsFlagName, "", "New object attributes in form of Key1=Value1,Key2=Value2") + flags.StringSlice(newAttrsFlagName, nil, "New object attributes in form of Key1=Value1,Key2=Value2") flags.Bool(replaceAttrsFlagName, false, "Replace object attributes by new ones.") flags.StringSlice(rangeFlagName, []string{}, "Range to which patch payload is applied. Format: offset:length") flags.StringSlice(payloadFlagName, []string{}, "Path to file with patch payload.") @@ -99,11 +99,9 @@ func patch(cmd *cobra.Command, _ []string) { } func parseNewObjectAttrs(cmd *cobra.Command) ([]objectSDK.Attribute, error) { - var rawAttrs []string - - raw := cmd.Flag(newAttrsFlagName).Value.String() - if len(raw) != 0 { - rawAttrs = strings.Split(raw, ",") + rawAttrs, err := cmd.Flags().GetStringSlice(newAttrsFlagName) + if err != nil { + return nil, err } attrs := make([]objectSDK.Attribute, len(rawAttrs), len(rawAttrs)+2) // name + timestamp attributes diff --git a/cmd/frostfs-cli/modules/object/put.go b/cmd/frostfs-cli/modules/object/put.go index affe9bbba..9e8a7cc6f 100644 --- a/cmd/frostfs-cli/modules/object/put.go +++ b/cmd/frostfs-cli/modules/object/put.go @@ -50,7 +50,7 @@ func initObjectPutCmd() { flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) - flags.String("attributes", "", "User attributes in form of Key1=Value1,Key2=Value2") + flags.StringSlice("attributes", nil, "User attributes in form of Key1=Value1,Key2=Value2") flags.Bool("disable-filename", false, "Do not set well-known filename attribute") flags.Bool("disable-timestamp", false, "Do not set well-known timestamp attribute") flags.Uint64VarP(&putExpiredOn, commonflags.ExpireAt, "e", 0, "The last active epoch in the life of the object") @@ -214,11 +214,9 @@ func getAllObjectAttributes(cmd *cobra.Command) []objectSDK.Attribute { } func parseObjectAttrs(cmd *cobra.Command) ([]objectSDK.Attribute, error) { - var rawAttrs []string - - raw := cmd.Flag("attributes").Value.String() - if len(raw) != 0 { - rawAttrs = strings.Split(raw, ",") + rawAttrs, err := cmd.Flags().GetStringSlice("attributes") + if err != nil { + return nil, err } attrs := make([]objectSDK.Attribute, len(rawAttrs), len(rawAttrs)+2) // name + timestamp attributes From abfd9657f97f2d7b8e9227befeadf08b3fc75f03 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 28 Jan 2025 17:43:14 +0300 Subject: [PATCH 1693/1943] [#1617] govulncheck: Use patch release with security fixes https://go.dev/doc/devel/release#go1.23.minor Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/vulncheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index cf15005b1..8a5a818aa 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -18,7 +18,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.23' + go-version: '1.23.5' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest From e1a984e9d8bb0cedd5597f6176b865dea5e2f3b5 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 29 Jan 2025 16:09:53 +0300 Subject: [PATCH 1694/1943] [#1620] logs: Drop redundant trace_id fields `trace_id` is taken from context. Signed-off-by: Dmitrii Stepanov --- .../blobovnicza/delete.go | 2 - .../blobstor/blobovniczatree/delete.go | 2 - .../blobstor/blobovniczatree/exists.go | 4 +- .../blobstor/blobovniczatree/get.go | 2 - .../blobstor/blobovniczatree/get_range.go | 4 +- .../blobstor/blobovniczatree/put.go | 10 ++--- pkg/local_object_storage/blobstor/exists.go | 4 +- pkg/local_object_storage/engine/delete.go | 10 ++--- pkg/local_object_storage/engine/evacuate.go | 32 ++++++--------- pkg/local_object_storage/engine/get.go | 4 +- pkg/local_object_storage/engine/inhume.go | 9 +---- pkg/local_object_storage/engine/put.go | 10 ++--- pkg/local_object_storage/engine/range.go | 4 +- pkg/local_object_storage/engine/tree.go | 40 ++++++------------- pkg/local_object_storage/shard/delete.go | 7 +--- pkg/local_object_storage/shard/get.go | 7 +--- pkg/local_object_storage/shard/inhume.go | 2 - pkg/local_object_storage/shard/list.go | 4 +- pkg/services/object/put/single.go | 2 - pkg/services/replicator/process.go | 5 +-- pkg/services/replicator/pull.go | 10 ++--- pkg/services/replicator/put.go | 7 +--- pkg/services/tree/redirect.go | 4 +- pkg/services/tree/replicator.go | 7 +--- 24 files changed, 56 insertions(+), 136 deletions(-) diff --git a/pkg/local_object_storage/blobovnicza/delete.go b/pkg/local_object_storage/blobovnicza/delete.go index d821b2991..8f24b5675 100644 --- a/pkg/local_object_storage/blobovnicza/delete.go +++ b/pkg/local_object_storage/blobovnicza/delete.go @@ -6,7 +6,6 @@ import ( "syscall" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -94,7 +93,6 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err b.log.Debug(ctx, logs.BlobovniczaObjectWasRemovedFromBucket, zap.String("binary size", stringifyByteSize(dataSize)), zap.String("range", stringifyBounds(sizeLowerBound, sizeUpperBound)), - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) b.itemDeleted(recordSize) } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go index 8d17fc4b0..d096791c3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/delete.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/delete.go @@ -10,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -86,7 +85,6 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co b.log.Debug(ctx, logs.BlobovniczatreeCouldNotRemoveObjectFromLevel, zap.String("level", p), zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go index d2c99945f..0c5e48821 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "go.opentelemetry.io/otel/attribute" @@ -57,8 +56,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common if !client.IsErrObjectNotFound(err) { b.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get.go b/pkg/local_object_storage/blobstor/blobovniczatree/get.go index 5d158644e..e5c83e5f2 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get.go @@ -10,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -70,7 +69,6 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G b.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go index 84b9bc55f..27d13f4f3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/get_range.go @@ -11,7 +11,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -71,8 +70,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re if !outOfBounds && !client.IsErrObjectNotFound(err) { b.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetObjectFromLevel, zap.String("level", p), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) } if outOfBounds { return true, err diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/put.go b/pkg/local_object_storage/blobstor/blobovniczatree/put.go index 8276a25ef..37c49d741 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/put.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/put.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -83,16 +82,14 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) i.B.reportError(ctx, logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err) } else { i.B.log.Debug(ctx, logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) } return false, nil } if active == nil { - i.B.log.Debug(ctx, logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + i.B.log.Debug(ctx, logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath)) return false, nil } defer active.Close(ctx) @@ -106,8 +103,7 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error) } else { i.B.log.Debug(ctx, logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", active.SystemPath()), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) } if errors.Is(err, blobovnicza.ErrNoSpace) { i.AllFull = true diff --git a/pkg/local_object_storage/blobstor/exists.go b/pkg/local_object_storage/blobstor/exists.go index f1e45fe10..c155e15b8 100644 --- a/pkg/local_object_storage/blobstor/exists.go +++ b/pkg/local_object_storage/blobstor/exists.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -75,8 +74,7 @@ func (b *BlobStor) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exi for _, err := range errors[:len(errors)-1] { b.log.Warn(ctx, logs.BlobstorErrorOccurredDuringObjectExistenceChecking, zap.Stringer("address", prm.Address), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) } return common.ExistsRes{}, errors[len(errors)-1] diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 20c915da6..5e5f65fa2 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -6,7 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -148,8 +147,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo if err != nil { e.log.Warn(ctx, logs.EngineErrorDuringSearchingForObjectChildren, zap.Stringer("addr", addr), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) return false } @@ -160,8 +158,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo if err != nil { e.log.Debug(ctx, logs.EngineCouldNotInhumeObjectInShard, zap.Stringer("addr", addr), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) continue } } @@ -190,8 +187,7 @@ func (e *StorageEngine) deleteChunks( if err != nil { e.log.Debug(ctx, logs.EngineCouldNotInhumeObjectInShard, zap.Stringer("addr", addr), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) continue } } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index b37f6b68e..fd1530f53 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -15,7 +15,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -284,12 +283,12 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p }() e.log.Info(ctx, logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) + zap.Stringer("scope", prm.Scope)) err = e.getTotals(ctx, prm, shardsToEvacuate, res) if err != nil { e.log.Error(ctx, logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) + zap.Stringer("scope", prm.Scope)) return err } @@ -323,7 +322,7 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p } if err != nil { e.log.Error(ctx, logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) + zap.Stringer("scope", prm.Scope)) return err } @@ -480,8 +479,7 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context err := sh.IterateOverContainers(ctx, cntPrm) if err != nil { cancel(err) - e.log.Error(ctx, logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + e.log.Error(ctx, logs.EngineShardsEvacuationFailedToListObjects, zap.String("shard_id", shardID), zap.Error(err), evacuationOperationLogField) } return err } @@ -540,7 +538,7 @@ func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, tree e.log.Debug(ctx, logs.EngineShardsEvacuationTreeEvacuatedLocal, zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), zap.String("from_shard_id", sh.ID().String()), zap.String("to_shard_id", shardID), - evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + evacuationOperationLogField) res.trEvacuated.Add(1) continue } @@ -550,26 +548,26 @@ func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, tree e.log.Error(ctx, logs.EngineShardsEvacuationFailedToMoveTree, zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), zap.String("from_shard_id", sh.ID().String()), evacuationOperationLogField, - zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) return err } if moved { e.log.Debug(ctx, logs.EngineShardsEvacuationTreeEvacuatedRemote, zap.String("cid", contTree.CID.EncodeToString()), zap.String("treeID", contTree.TreeID), zap.String("from_shardID", sh.ID().String()), zap.String("to_node", nodePK), - evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + evacuationOperationLogField) res.trEvacuated.Add(1) } else if prm.IgnoreErrors { res.trFailed.Add(1) e.log.Warn(ctx, logs.EngineShardsEvacuationFailedToMoveTree, zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), zap.String("from_shard_id", sh.ID().String()), evacuationOperationLogField, - zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) } else { e.log.Error(ctx, logs.EngineShardsEvacuationFailedToMoveTree, zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), zap.String("from_shard_id", sh.ID().String()), evacuationOperationLogField, - zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) return fmt.Errorf("no remote nodes available to replicate tree '%s' of container %s", contTree.TreeID, contTree.CID) } } @@ -756,8 +754,7 @@ func (e *StorageEngine) evacuateObject(ctx context.Context, shardID string, objI res.objFailed.Add(1) return nil } - e.log.Error(ctx, logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + e.log.Error(ctx, logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) return err } @@ -778,16 +775,14 @@ func (e *StorageEngine) evacuateObject(ctx context.Context, shardID string, objI moved, err := prm.ObjectsHandler(ctx, addr, getRes.Object()) if err != nil { - e.log.Error(ctx, logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + e.log.Error(ctx, logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) return err } if moved { res.objEvacuated.Add(1) } else if prm.IgnoreErrors { res.objFailed.Add(1) - e.log.Warn(ctx, logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + e.log.Warn(ctx, logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField) } else { return fmt.Errorf("object %s was not replicated", addr) } @@ -825,8 +820,7 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), zap.Stringer("addr", addr), - evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + evacuationOperationLogField) return true, nil case putToShardExists, putToShardRemoved: res.objSkipped.Add(1) diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 81b027c26..74c64bbb6 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -107,8 +106,7 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { e.log.Warn(ctx, logs.ShardMetaInfoPresentButObjectNotFound, zap.Stringer("shard_id", it.ShardWithMeta.ID()), zap.Error(it.MetaError), - zap.Stringer("address", prm.addr), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Stringer("address", prm.addr)) } } diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 75bd15c8b..fb802ef2a 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -99,7 +98,6 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) error { e.log.Warn(ctx, logs.EngineCouldNotInhumeObjectInShard, zap.Error(errors.New("this shard was expected to exist")), zap.String("shard_id", shardID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) return errInhumeFailure } @@ -197,7 +195,6 @@ func (e *StorageEngine) findShards(ctx context.Context, addr oid.Address, checkL e.log.Warn(ctx, logs.EngineRemovingAnObjectWithoutFullLockingCheck, zap.Error(err), zap.Stringer("address", addr), - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } else if isLocked { retErr = new(apistatus.ObjectLocked) @@ -232,8 +229,7 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { locked, err = h.Shard.IsLocked(ctx, addr) if err != nil { - e.reportShardError(ctx, h, "can't check object's lockers", err, zap.Stringer("address", addr), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + e.reportShardError(ctx, h, "can't check object's lockers", err, zap.Stringer("address", addr)) outErr = err return false } @@ -262,8 +258,7 @@ func (e *StorageEngine) GetLocks(ctx context.Context, addr oid.Address) ([]oid.I e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { locks, err := h.Shard.GetLocks(ctx, addr) if err != nil { - e.reportShardError(ctx, h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + e.reportShardError(ctx, h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr)) outErr = err } allLocks = append(allLocks, locks...) diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index c79b6e251..64288a511 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -143,8 +142,7 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti } else { e.log.Warn(ctx, logs.EngineCouldNotCheckObjectExistence, zap.Stringer("shard_id", sh.ID()), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) } return // this is not ErrAlreadyRemoved error so we can go to the next shard @@ -165,15 +163,13 @@ func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool uti errors.Is(err, common.ErrReadOnly) || errors.Is(err, common.ErrNoSpace) { e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, zap.Stringer("shard_id", sh.ID()), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) return } if client.IsErrObjectAlreadyRemoved(err) { e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, zap.Stringer("shard_id", sh.ID()), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) res.status = putToShardRemoved res.err = err return diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index 600e7266c..a468cf594 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -119,8 +118,7 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error e.log.Warn(ctx, logs.ShardMetaInfoPresentButObjectNotFound, zap.Stringer("shard_id", it.ShardWithMeta.ID()), zap.Error(it.MetaError), - zap.Stringer("address", prm.addr), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Stringer("address", prm.addr)) } } diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 268b4adfa..7f70d36f7 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.opentelemetry.io/otel/attribute" @@ -39,8 +38,7 @@ func (e *StorageEngine) TreeMove(ctx context.Context, d pilorama.CIDDescriptor, if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(ctx, lst[index], "can't perform `TreeMove`", err, zap.Stringer("cid", d.CID), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } return nil, err @@ -73,8 +71,7 @@ func (e *StorageEngine) TreeAddByPath(ctx context.Context, d pilorama.CIDDescrip if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(ctx, lst[index], "can't perform `TreeAddByPath`", err, zap.Stringer("cid", d.CID), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } return nil, err } @@ -102,8 +99,7 @@ func (e *StorageEngine) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID str if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(ctx, lst[index], "can't perform `TreeApply`", err, zap.Stringer("cid", cnr), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } return err } @@ -130,8 +126,7 @@ func (e *StorageEngine) TreeApplyBatch(ctx context.Context, cnr cidSDK.ID, treeI if !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(ctx, lst[index], "can't perform `TreeApplyBatch`", err, zap.Stringer("cid", cnr), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } return err } @@ -162,8 +157,7 @@ func (e *StorageEngine) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(ctx, sh, "can't perform `TreeGetByPath`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } continue } @@ -195,8 +189,7 @@ func (e *StorageEngine) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID s if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(ctx, sh, "can't perform `TreeGetMeta`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } continue } @@ -227,8 +220,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(ctx, sh, "can't perform `TreeGetChildren`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } continue } @@ -259,8 +251,7 @@ func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(ctx, sh, "can't perform `TreeSortedByFilename`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } continue } @@ -291,8 +282,7 @@ func (e *StorageEngine) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(ctx, sh, "can't perform `TreeGetOpLog`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } continue } @@ -321,8 +311,7 @@ func (e *StorageEngine) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID stri if !errors.Is(err, pilorama.ErrTreeNotFound) && !errors.Is(err, shard.ErrReadOnlyMode) { e.reportShardError(ctx, sh, "can't perform `TreeDrop`", err, zap.Stringer("cid", cid), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } continue } @@ -350,8 +339,7 @@ func (e *StorageEngine) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, } e.reportShardError(ctx, sh, "can't perform `TreeList`", err, - zap.Stringer("cid", cid), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Stringer("cid", cid)) // returns as much info about // trees as possible @@ -417,8 +405,7 @@ func (e *StorageEngine) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK if err != nil && !errors.Is(err, shard.ErrReadOnlyMode) && err != shard.ErrPiloramaDisabled { e.reportShardError(ctx, lst[index], "can't update tree synchronization height", err, zap.Stringer("cid", cid), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } return err } @@ -444,8 +431,7 @@ func (e *StorageEngine) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, t if !errors.Is(err, pilorama.ErrTreeNotFound) { e.reportShardError(ctx, sh, "can't read tree synchronization height", err, zap.Stringer("cid", cid), - zap.String("tree", treeID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("tree", treeID)) } continue } diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index fb6769b51..55231b032 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -112,8 +111,7 @@ func (s *Shard) deleteFromBlobstor(ctx context.Context, addr oid.Address) error if err != nil { s.log.Debug(ctx, logs.StorageIDRetrievalFailure, zap.Stringer("object", addr), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) return err } storageID := res.StorageID() @@ -132,8 +130,7 @@ func (s *Shard) deleteFromBlobstor(ctx context.Context, addr oid.Address) error if err != nil && !client.IsErrObjectNotFound(err) { s.log.Debug(ctx, logs.ObjectRemovalFailureBlobStor, zap.Stringer("object_address", addr), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) return err } return nil diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 15d1eb6ba..05823c62b 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -10,7 +10,6 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -155,14 +154,12 @@ func (s *Shard) fetchObjectData(ctx context.Context, addr oid.Address, skipMeta if client.IsErrObjectNotFound(err) { s.log.Debug(ctx, logs.ShardObjectIsMissingInWritecache, zap.Stringer("addr", addr), - zap.Bool("skip_meta", skipMeta), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Bool("skip_meta", skipMeta)) } else { s.log.Error(ctx, logs.ShardFailedToFetchObjectFromWritecache, zap.Error(err), zap.Stringer("addr", addr), - zap.Bool("skip_meta", skipMeta), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Bool("skip_meta", skipMeta)) } } if skipMeta || mErr != nil { diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index d46400869..9d5f66063 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" @@ -111,7 +110,6 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { s.log.Debug(ctx, logs.ShardCouldNotMarkObjectToDeleteInMetabase, zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) s.m.RUnlock() diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index c5275dafd..7bc5ead1d 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -124,8 +123,7 @@ func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { if err != nil { s.log.Debug(ctx, logs.ShardCantSelectAllObjects, zap.Stringer("cid", lst[i]), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) continue } diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index 5219e64d5..fec50b1d9 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -21,7 +21,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/internal" svcutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectAPI "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc" @@ -323,7 +322,6 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, zap.Stringer("address", addr), zap.Stringer("object_id", objID), zap.Stringer("container_id", cnrID), - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) } diff --git a/pkg/services/replicator/process.go b/pkg/services/replicator/process.go index 69395bb02..8c6f0df06 100644 --- a/pkg/services/replicator/process.go +++ b/pkg/services/replicator/process.go @@ -6,7 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" @@ -45,8 +44,7 @@ func (p *Replicator) HandleReplicationTask(ctx context.Context, task Task, res T if err != nil { p.log.Error(ctx, logs.ReplicatorCouldNotGetObjectFromLocalStorage, zap.Stringer("object", task.Addr), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) return } @@ -65,7 +63,6 @@ func (p *Replicator) HandleReplicationTask(ctx context.Context, task Task, res T log := p.log.With( zap.String("node", netmap.StringifyPublicKey(task.Nodes[i])), zap.Stringer("object", task.Addr), - zap.String("trace_id", tracingPkg.GetTraceID(ctx)), ) callCtx, cancel := context.WithTimeout(ctx, p.putTimeout) diff --git a/pkg/services/replicator/pull.go b/pkg/services/replicator/pull.go index 5ce929342..bb38c72ad 100644 --- a/pkg/services/replicator/pull.go +++ b/pkg/services/replicator/pull.go @@ -8,7 +8,6 @@ import ( containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" @@ -51,15 +50,13 @@ func (p *Replicator) HandlePullTask(ctx context.Context, task Task) { p.log.Error(ctx, logs.ReplicatorCouldNotGetObjectFromRemoteStorage, zap.Stringer("object", task.Addr), zap.Error(err), - zap.Strings("endpoints", endpoints), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Strings("endpoints", endpoints)) } if obj == nil { p.log.Error(ctx, logs.ReplicatorCouldNotGetObjectFromRemoteStorage, zap.Stringer("object", task.Addr), - zap.Error(errFailedToGetObjectFromAnyNode), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(errFailedToGetObjectFromAnyNode)) return } @@ -67,7 +64,6 @@ func (p *Replicator) HandlePullTask(ctx context.Context, task Task) { if err != nil { p.log.Error(ctx, logs.ReplicatorCouldNotPutObjectToLocalStorage, zap.Stringer("object", task.Addr), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) } } diff --git a/pkg/services/replicator/put.go b/pkg/services/replicator/put.go index 489f66ae5..bcad8471d 100644 --- a/pkg/services/replicator/put.go +++ b/pkg/services/replicator/put.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -33,8 +32,7 @@ func (p *Replicator) HandleLocalPutTask(ctx context.Context, task Task) { if task.Obj == nil { p.log.Error(ctx, logs.ReplicatorCouldNotPutObjectToLocalStorage, zap.Stringer("object", task.Addr), - zap.Error(errObjectNotDefined), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(errObjectNotDefined)) return } @@ -42,7 +40,6 @@ func (p *Replicator) HandleLocalPutTask(ctx context.Context, task Task) { if err != nil { p.log.Error(ctx, logs.ReplicatorCouldNotPutObjectToLocalStorage, zap.Stringer("object", task.Addr), - zap.Error(err), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.Error(err)) } } diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index 416a0fafe..d92c749a8 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -6,7 +6,6 @@ import ( "errors" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "go.opentelemetry.io/otel/attribute" @@ -54,8 +53,7 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo return false } - s.log.Debug(ctx, logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + s.log.Debug(ctx, logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint)) called = true stop = f(c) diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index e0085d73a..bcbb73589 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -10,7 +10,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" - tracingPkg "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -117,14 +116,12 @@ func (s *Service) ReplicateTreeOp(ctx context.Context, n netmapSDK.NodeInfo, req if lastErr != nil { if errors.Is(lastErr, errRecentlyFailed) { s.log.Debug(ctx, logs.TreeDoNotSendUpdateToTheNode, - zap.String("last_error", lastErr.Error()), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("last_error", lastErr.Error())) } else { s.log.Warn(ctx, logs.TreeFailedToSentUpdateToTheNode, zap.String("last_error", lastErr.Error()), zap.String("address", lastAddr), - zap.String("key", hex.EncodeToString(n.PublicKey())), - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("key", hex.EncodeToString(n.PublicKey()))) } s.metrics.AddReplicateTaskDuration(time.Since(start), false) return lastErr From c0a341a7f6f02cc5e5a419287fab841251f4f4cc Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 27 Jan 2025 14:17:58 +0300 Subject: [PATCH 1695/1943] [#1616] getsvc: Move head assembling to a separate file Signed-off-by: Evgenii Stratonikov --- pkg/services/object/get/assembler.go | 37 ------------------- pkg/services/object/get/assembler_head.go | 45 +++++++++++++++++++++++ 2 files changed, 45 insertions(+), 37 deletions(-) create mode 100644 pkg/services/object/get/assembler_head.go diff --git a/pkg/services/object/get/assembler.go b/pkg/services/object/get/assembler.go index ff3f90bf2..23fc187f5 100644 --- a/pkg/services/object/get/assembler.go +++ b/pkg/services/object/get/assembler.go @@ -71,43 +71,6 @@ func (a *assembler) Assemble(ctx context.Context, writer ObjectWriter) (*objectS return a.parentObject, nil } -func (a *assembler) assembleHeader(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { - var sourceObjectIDs []oid.ID - sourceObjectID, ok := a.splitInfo.Link() - if ok { - sourceObjectIDs = append(sourceObjectIDs, sourceObjectID) - } - sourceObjectID, ok = a.splitInfo.LastPart() - if ok { - sourceObjectIDs = append(sourceObjectIDs, sourceObjectID) - } - if len(sourceObjectIDs) == 0 { - return nil, objectSDK.NewSplitInfoError(a.splitInfo) - } - for _, sourceObjectID = range sourceObjectIDs { - obj, err := a.getParent(ctx, sourceObjectID, writer) - if err == nil { - return obj, nil - } - } - return nil, objectSDK.NewSplitInfoError(a.splitInfo) -} - -func (a *assembler) getParent(ctx context.Context, sourceObjectID oid.ID, writer ObjectWriter) (*objectSDK.Object, error) { - obj, err := a.objGetter.HeadObject(ctx, sourceObjectID) - if err != nil { - return nil, err - } - parent := obj.Parent() - if parent == nil { - return nil, objectSDK.NewSplitInfoError(a.splitInfo) - } - if err := writer.WriteHeader(ctx, parent); err != nil { - return nil, err - } - return obj, nil -} - func (a *assembler) getLastPartOrLinkObjectID() (oid.ID, bool) { sourceObjectID, ok := a.splitInfo.Link() if ok { diff --git a/pkg/services/object/get/assembler_head.go b/pkg/services/object/get/assembler_head.go new file mode 100644 index 000000000..ff213cb82 --- /dev/null +++ b/pkg/services/object/get/assembler_head.go @@ -0,0 +1,45 @@ +package getsvc + +import ( + "context" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +func (a *assembler) assembleHeader(ctx context.Context, writer ObjectWriter) (*objectSDK.Object, error) { + var sourceObjectIDs []oid.ID + sourceObjectID, ok := a.splitInfo.Link() + if ok { + sourceObjectIDs = append(sourceObjectIDs, sourceObjectID) + } + sourceObjectID, ok = a.splitInfo.LastPart() + if ok { + sourceObjectIDs = append(sourceObjectIDs, sourceObjectID) + } + if len(sourceObjectIDs) == 0 { + return nil, objectSDK.NewSplitInfoError(a.splitInfo) + } + for _, sourceObjectID = range sourceObjectIDs { + obj, err := a.getParent(ctx, sourceObjectID, writer) + if err == nil { + return obj, nil + } + } + return nil, objectSDK.NewSplitInfoError(a.splitInfo) +} + +func (a *assembler) getParent(ctx context.Context, sourceObjectID oid.ID, writer ObjectWriter) (*objectSDK.Object, error) { + obj, err := a.objGetter.HeadObject(ctx, sourceObjectID) + if err != nil { + return nil, err + } + parent := obj.Parent() + if parent == nil { + return nil, objectSDK.NewSplitInfoError(a.splitInfo) + } + if err := writer.WriteHeader(ctx, parent); err != nil { + return nil, err + } + return obj, nil +} From 6410542d1933fa35b765f4cf324b60e1f6584714 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 27 Jan 2025 14:22:46 +0300 Subject: [PATCH 1696/1943] [#1616] getsvc: Move range assembling to a separate file Signed-off-by: Evgenii Stratonikov --- pkg/services/object/get/assembler.go | 102 ++++++-------------- pkg/services/object/get/assembler_range.go | 103 +++++++++++++++++++++ 2 files changed, 129 insertions(+), 76 deletions(-) create mode 100644 pkg/services/object/get/assembler_range.go diff --git a/pkg/services/object/get/assembler.go b/pkg/services/object/get/assembler.go index 23fc187f5..886f0aabb 100644 --- a/pkg/services/object/get/assembler.go +++ b/pkg/services/object/get/assembler.go @@ -59,15 +59,23 @@ func (a *assembler) Assemble(ctx context.Context, writer ObjectWriter) (*objectS if previousID == nil && len(childrenIDs) == 0 { return nil, objectSDK.NewSplitInfoError(a.splitInfo) } + if len(childrenIDs) > 0 { - if err := a.assembleObjectByChildrenList(ctx, childrenIDs, writer); err != nil { - return nil, err + if a.rng != nil { + err = a.assembleObjectByChildrenListRange(ctx, childrenIDs, writer) + } else { + err = a.assembleObjectByChildrenList(ctx, childrenIDs, writer) } } else { - if err := a.assemleObjectByPreviousIDInReverse(ctx, *previousID, writer); err != nil { - return nil, err + if a.rng != nil { + err = a.assemleObjectByPreviousIDInReverseRange(ctx, *previousID, writer) + } else { + err = a.assemleObjectByPreviousIDInReverse(ctx, *previousID, writer) } } + if err != nil { + return nil, err + } return a.parentObject, nil } @@ -153,26 +161,16 @@ func (a *assembler) getChildObject(ctx context.Context, id oid.ID, rng *objectSD } func (a *assembler) assembleObjectByChildrenList(ctx context.Context, childrenIDs []oid.ID, writer ObjectWriter) error { - if a.rng == nil { - if err := writer.WriteHeader(ctx, a.parentObject.CutPayload()); err != nil { - return err - } - return a.assemblePayloadByObjectIDs(ctx, writer, childrenIDs, nil, true) - } - - if err := a.assemblePayloadInReverse(ctx, writer, childrenIDs[len(childrenIDs)-1]); err != nil { + if err := writer.WriteHeader(ctx, a.parentObject.CutPayload()); err != nil { return err } - return writer.WriteChunk(ctx, a.parentObject.Payload()) + return a.assemblePayloadByObjectIDs(ctx, writer, childrenIDs, true) } func (a *assembler) assemleObjectByPreviousIDInReverse(ctx context.Context, prevID oid.ID, writer ObjectWriter) error { - if a.rng == nil { - if err := writer.WriteHeader(ctx, a.parentObject.CutPayload()); err != nil { - return err - } + if err := writer.WriteHeader(ctx, a.parentObject.CutPayload()); err != nil { + return err } - if err := a.assemblePayloadInReverse(ctx, writer, prevID); err != nil { return err } @@ -182,16 +180,9 @@ func (a *assembler) assemleObjectByPreviousIDInReverse(ctx context.Context, prev return nil } -func (a *assembler) assemblePayloadByObjectIDs(ctx context.Context, writer ObjectWriter, partIDs []oid.ID, partRanges []objectSDK.Range, verifyIsChild bool) error { - withRng := len(partRanges) > 0 && a.rng != nil - +func (a *assembler) assemblePayloadByObjectIDs(ctx context.Context, writer ObjectWriter, partIDs []oid.ID, verifyIsChild bool) error { for i := range partIDs { - var r *objectSDK.Range - if withRng { - r = &partRanges[i] - } - - _, err := a.getChildObject(ctx, partIDs[i], r, verifyIsChild, writer) + _, err := a.getChildObject(ctx, partIDs[i], nil, verifyIsChild, writer) if err != nil { return err } @@ -200,22 +191,16 @@ func (a *assembler) assemblePayloadByObjectIDs(ctx context.Context, writer Objec } func (a *assembler) assemblePayloadInReverse(ctx context.Context, writer ObjectWriter, prevID oid.ID) error { - chain, rngs, err := a.buildChain(ctx, prevID) + chain, err := a.buildChain(ctx, prevID) if err != nil { return err } - reverseRngs := len(rngs) > 0 - for left, right := 0, len(chain)-1; left < right; left, right = left+1, right-1 { chain[left], chain[right] = chain[right], chain[left] - - if reverseRngs { - rngs[left], rngs[right] = rngs[right], rngs[left] - } } - return a.assemblePayloadByObjectIDs(ctx, writer, chain, rngs, false) + return a.assemblePayloadByObjectIDs(ctx, writer, chain, false) } func (a *assembler) isChild(obj *objectSDK.Object) bool { @@ -223,63 +208,28 @@ func (a *assembler) isChild(obj *objectSDK.Object) bool { return parent == nil || equalAddresses(a.addr, object.AddressOf(parent)) } -func (a *assembler) buildChain(ctx context.Context, prevID oid.ID) ([]oid.ID, []objectSDK.Range, error) { +func (a *assembler) buildChain(ctx context.Context, prevID oid.ID) ([]oid.ID, error) { var ( chain []oid.ID - rngs []objectSDK.Range - from = a.rng.GetOffset() - to = from + a.rng.GetLength() hasPrev = true ) // fill the chain end-to-start for hasPrev { - // check that only for "range" requests, - // for `GET` it stops via the false `withPrev` - if a.rng != nil && a.currentOffset <= from { - break - } - head, err := a.objGetter.HeadObject(ctx, prevID) if err != nil { - return nil, nil, err + return nil, err } if !a.isChild(head) { - return nil, nil, errParentAddressDiffers + return nil, errParentAddressDiffers } - if a.rng != nil { - sz := head.PayloadSize() - - a.currentOffset -= sz - - if a.currentOffset < to { - off := uint64(0) - if from > a.currentOffset { - off = from - a.currentOffset - sz -= from - a.currentOffset - } - - if to < a.currentOffset+off+sz { - sz = to - off - a.currentOffset - } - - index := len(rngs) - rngs = append(rngs, objectSDK.Range{}) - rngs[index].SetOffset(off) - rngs[index].SetLength(sz) - - id, _ := head.ID() - chain = append(chain, id) - } - } else { - id, _ := head.ID() - chain = append(chain, id) - } + id, _ := head.ID() + chain = append(chain, id) prevID, hasPrev = head.PreviousID() } - return chain, rngs, nil + return chain, nil } diff --git a/pkg/services/object/get/assembler_range.go b/pkg/services/object/get/assembler_range.go new file mode 100644 index 000000000..638db0c76 --- /dev/null +++ b/pkg/services/object/get/assembler_range.go @@ -0,0 +1,103 @@ +package getsvc + +import ( + "context" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +func (a *assembler) assembleObjectByChildrenListRange(ctx context.Context, childrenIDs []oid.ID, writer ObjectWriter) error { + if err := a.assemblePayloadInReverseRange(ctx, writer, childrenIDs[len(childrenIDs)-1]); err != nil { + return err + } + return writer.WriteChunk(ctx, a.parentObject.Payload()) +} + +func (a *assembler) assemleObjectByPreviousIDInReverseRange(ctx context.Context, prevID oid.ID, writer ObjectWriter) error { + if err := a.assemblePayloadInReverseRange(ctx, writer, prevID); err != nil { + return err + } + if err := writer.WriteChunk(ctx, a.parentObject.Payload()); err != nil { // last part + return err + } + return nil +} + +func (a *assembler) assemblePayloadByObjectIDsRange(ctx context.Context, writer ObjectWriter, partIDs []oid.ID, partRanges []objectSDK.Range) error { + for i := range partIDs { + _, err := a.getChildObject(ctx, partIDs[i], &partRanges[i], false, writer) + if err != nil { + return err + } + } + return nil +} + +func (a *assembler) assemblePayloadInReverseRange(ctx context.Context, writer ObjectWriter, prevID oid.ID) error { + chain, rngs, err := a.buildChainRange(ctx, prevID) + if err != nil { + return err + } + + for left, right := 0, len(chain)-1; left < right; left, right = left+1, right-1 { + chain[left], chain[right] = chain[right], chain[left] + rngs[left], rngs[right] = rngs[right], rngs[left] + } + + return a.assemblePayloadByObjectIDsRange(ctx, writer, chain, rngs) +} + +func (a *assembler) buildChainRange(ctx context.Context, prevID oid.ID) ([]oid.ID, []objectSDK.Range, error) { + var ( + chain []oid.ID + rngs []objectSDK.Range + from = a.rng.GetOffset() + to = from + a.rng.GetLength() + + hasPrev = true + ) + + // fill the chain end-to-start + for hasPrev { + if a.currentOffset <= from { + break + } + + head, err := a.objGetter.HeadObject(ctx, prevID) + if err != nil { + return nil, nil, err + } + if !a.isChild(head) { + return nil, nil, errParentAddressDiffers + } + + sz := head.PayloadSize() + + a.currentOffset -= sz + + if a.currentOffset < to { + off := uint64(0) + if from > a.currentOffset { + off = from - a.currentOffset + sz -= from - a.currentOffset + } + + if to < a.currentOffset+off+sz { + sz = to - off - a.currentOffset + } + + index := len(rngs) + rngs = append(rngs, objectSDK.Range{}) + rngs[index].SetOffset(off) + rngs[index].SetLength(sz) + + id, _ := head.ID() + chain = append(chain, id) + } + + prevID, hasPrev = head.PreviousID() + } + + return chain, rngs, nil +} From c8acdf40bbd18970fb39130f78ea64959f072f4b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 27 Jan 2025 14:24:06 +0300 Subject: [PATCH 1697/1943] [#1616] getsvc: Use slices.Reverse() where possible Signed-off-by: Evgenii Stratonikov --- pkg/services/object/get/assembler.go | 6 ++---- pkg/services/object/get/assembler_range.go | 8 +++----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/pkg/services/object/get/assembler.go b/pkg/services/object/get/assembler.go index 886f0aabb..b24c9417b 100644 --- a/pkg/services/object/get/assembler.go +++ b/pkg/services/object/get/assembler.go @@ -2,6 +2,7 @@ package getsvc import ( "context" + "slices" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -196,10 +197,7 @@ func (a *assembler) assemblePayloadInReverse(ctx context.Context, writer ObjectW return err } - for left, right := 0, len(chain)-1; left < right; left, right = left+1, right-1 { - chain[left], chain[right] = chain[right], chain[left] - } - + slices.Reverse(chain) return a.assemblePayloadByObjectIDs(ctx, writer, chain, false) } diff --git a/pkg/services/object/get/assembler_range.go b/pkg/services/object/get/assembler_range.go index 638db0c76..748a499ef 100644 --- a/pkg/services/object/get/assembler_range.go +++ b/pkg/services/object/get/assembler_range.go @@ -2,6 +2,7 @@ package getsvc import ( "context" + "slices" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -40,11 +41,8 @@ func (a *assembler) assemblePayloadInReverseRange(ctx context.Context, writer Ob return err } - for left, right := 0, len(chain)-1; left < right; left, right = left+1, right-1 { - chain[left], chain[right] = chain[right], chain[left] - rngs[left], rngs[right] = rngs[right], rngs[left] - } - + slices.Reverse(chain) + slices.Reverse(rngs) return a.assemblePayloadByObjectIDsRange(ctx, writer, chain, rngs) } From b309b34bfc77db01c77a6b98b6143aefc1673fcc Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 28 Jan 2025 15:01:02 +0300 Subject: [PATCH 1698/1943] [#1616] getsvc: Simplify buildChainRange() Signed-off-by: Evgenii Stratonikov --- pkg/services/object/get/assembler_range.go | 24 +++++++--------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/pkg/services/object/get/assembler_range.go b/pkg/services/object/get/assembler_range.go index 748a499ef..844a471a6 100644 --- a/pkg/services/object/get/assembler_range.go +++ b/pkg/services/object/get/assembler_range.go @@ -70,30 +70,20 @@ func (a *assembler) buildChainRange(ctx context.Context, prevID oid.ID) ([]oid.I return nil, nil, errParentAddressDiffers } - sz := head.PayloadSize() - - a.currentOffset -= sz - - if a.currentOffset < to { - off := uint64(0) - if from > a.currentOffset { - off = from - a.currentOffset - sz -= from - a.currentOffset - } - - if to < a.currentOffset+off+sz { - sz = to - off - a.currentOffset - } - + nextOffset := a.currentOffset - head.PayloadSize() + clampedFrom := max(from, nextOffset) + clampedTo := min(to, a.currentOffset) + if clampedFrom < clampedTo { index := len(rngs) rngs = append(rngs, objectSDK.Range{}) - rngs[index].SetOffset(off) - rngs[index].SetLength(sz) + rngs[index].SetOffset(clampedFrom - nextOffset) + rngs[index].SetLength(clampedTo - clampedFrom) id, _ := head.ID() chain = append(chain, id) } + a.currentOffset = nextOffset prevID, hasPrev = head.PreviousID() } From 57dc0a8e9e69567213f87616d0080a6c12182019 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 29 Jan 2025 12:23:05 +0300 Subject: [PATCH 1699/1943] [#1616] getsvc: Move break condition from body to the loop condition Signed-off-by: Evgenii Stratonikov --- pkg/services/object/get/assembler_range.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/services/object/get/assembler_range.go b/pkg/services/object/get/assembler_range.go index 844a471a6..780693c40 100644 --- a/pkg/services/object/get/assembler_range.go +++ b/pkg/services/object/get/assembler_range.go @@ -57,11 +57,7 @@ func (a *assembler) buildChainRange(ctx context.Context, prevID oid.ID) ([]oid.I ) // fill the chain end-to-start - for hasPrev { - if a.currentOffset <= from { - break - } - + for hasPrev && from < a.currentOffset { head, err := a.objGetter.HeadObject(ctx, prevID) if err != nil { return nil, nil, err From 78bfd1222908d4ff4e8163f3e045fa2d7e478fcb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 30 Jan 2025 15:29:38 +0300 Subject: [PATCH 1700/1943] [#1622] adm: Return non zero return code if metabase upgrade failed Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-adm/internal/modules/metabase/upgrade.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/metabase/upgrade.go b/cmd/frostfs-adm/internal/modules/metabase/upgrade.go index beced0d7a..c0c290c5e 100644 --- a/cmd/frostfs-adm/internal/modules/metabase/upgrade.go +++ b/cmd/frostfs-adm/internal/modules/metabase/upgrade.go @@ -28,6 +28,7 @@ const ( var ( errNoPathsFound = errors.New("no metabase paths found") errNoMorphEndpointsFound = errors.New("no morph endpoints found") + errUpgradeFailed = errors.New("upgrade failed") ) var UpgradeCmd = &cobra.Command{ @@ -91,14 +92,19 @@ func upgrade(cmd *cobra.Command, _ []string) error { if err := eg.Wait(); err != nil { return err } + allSuccess := true for mb, ok := range result { if ok { cmd.Println(mb, ": success") } else { cmd.Println(mb, ": failed") + allSuccess = false } } - return nil + if allSuccess { + return nil + } + return errUpgradeFailed } func getMetabasePaths(appCfg *config.Config) ([]string, error) { From 9ee3dd4e91bea626eedee35f751fddaedf9c9621 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 28 Jan 2025 18:32:14 +0300 Subject: [PATCH 1701/1943] [#1618] ape: Fix object service request tests Constant string `testOwnerID` for these tests has an invalid format. It has 11 bytes instead of required 25 for `user.ID`. It worked because: 1. `user.ID` was a byte slice and didn't check length and format of byte slices decoded from strings. 2. in these tests `testOwnerID` was used only to decode container owner id and to compare it with owner id encoded back to string. Since `user.ID implementation has changed`, the problem arised. Now `testOwnerID` is valid. Signed-off-by: Ekaterina Lebedeva --- pkg/services/object/ape/request_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/ape/request_test.go b/pkg/services/object/ape/request_test.go index 787785b60..f270bf97d 100644 --- a/pkg/services/object/ape/request_test.go +++ b/pkg/services/object/ape/request_test.go @@ -19,7 +19,7 @@ import ( ) const ( - testOwnerID = "FPPtmAi9TCX329" + testOwnerID = "NURFM8PWbLA2aLt2vrD8q4FyfAdgESwM8y" incomingIP = "192.92.33.1" ) From 24054cf6f4c9b2cae443ea5b9741602eaca57a22 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 28 Jan 2025 18:49:45 +0300 Subject: [PATCH 1702/1943] [#1618] pkg: Refactor `user.ID.ScriptHash()` usage `user.ID.ScriptHash()` does not return an error anymore. Signed-off-by: Ekaterina Lebedeva --- go.mod | 2 +- go.sum | 4 ++-- pkg/morph/client/balance/balanceOf.go | 5 +---- pkg/morph/client/balance/transfer.go | 13 +++---------- pkg/services/container/ape.go | 10 ++-------- pkg/services/container/ape_test.go | 22 ++++++++-------------- 6 files changed, 17 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 267e411fb..95efa584e 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250109084609-328d214d2d76 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250129133430-d195cb510401 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 diff --git a/go.sum b/go.sum index 935d3c56a..3be3f2246 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250109084609-328d214d2d76 h1:wzvSJIiS+p9qKfl3eg1oH6qlrjaEWiqTc/iMDKG3Ml4= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250109084609-328d214d2d76/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250129133430-d195cb510401 h1:5EN6YsbF5u2BffTKo5kVwe+Aqcy6CRTTVKcwZOI+Ym0= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250129133430-d195cb510401/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= diff --git a/pkg/morph/client/balance/balanceOf.go b/pkg/morph/client/balance/balanceOf.go index a5fb8e82a..449d49e5a 100644 --- a/pkg/morph/client/balance/balanceOf.go +++ b/pkg/morph/client/balance/balanceOf.go @@ -11,10 +11,7 @@ import ( // BalanceOf receives the amount of funds in the client's account // through the Balance contract call, and returns it. func (c *Client) BalanceOf(id user.ID) (*big.Int, error) { - h, err := id.ScriptHash() - if err != nil { - return nil, err - } + h := id.ScriptHash() invokePrm := client.TestInvokePrm{} invokePrm.SetMethod(balanceOfMethod) diff --git a/pkg/morph/client/balance/transfer.go b/pkg/morph/client/balance/transfer.go index 52d69dccb..870bed166 100644 --- a/pkg/morph/client/balance/transfer.go +++ b/pkg/morph/client/balance/transfer.go @@ -22,22 +22,15 @@ type TransferPrm struct { // TransferX transfers p.Amount of GASe-12 from p.From to p.To // with details p.Details through direct smart contract call. func (c *Client) TransferX(ctx context.Context, p TransferPrm) error { - from, err := p.From.ScriptHash() - if err != nil { - return err - } - - to, err := p.To.ScriptHash() - if err != nil { - return err - } + from := p.From.ScriptHash() + to := p.To.ScriptHash() prm := client.InvokePrm{} prm.SetMethod(transferXMethod) prm.SetArgs(from, to, p.Amount, p.Details) prm.InvokePrmOptional = p.InvokePrmOptional - _, err = c.client.Invoke(ctx, prm) + _, err := c.client.Invoke(ctx, prm) if err != nil { return fmt.Errorf("invoke method (%s): %w", transferXMethod, err) } diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 493452fa6..2c240412b 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -649,10 +649,7 @@ func (ac *apeChecker) namespaceByOwner(owner *refs.OwnerID) (string, error) { if err := ownerSDK.ReadFromV2(*owner); err != nil { return "", err } - addr, err := ownerSDK.ScriptHash() - if err != nil { - return "", err - } + addr := ownerSDK.ScriptHash() namespace := "" subject, err := ac.frostFSIDClient.GetSubject(addr) @@ -674,10 +671,7 @@ func (ac *apeChecker) namespaceByKnownOwner(owner *refs.OwnerID) (string, error) if err := ownerSDK.ReadFromV2(*owner); err != nil { return "", err } - addr, err := ownerSDK.ScriptHash() - if err != nil { - return "", err - } + addr := ownerSDK.ScriptHash() subject, err := ac.frostFSIDClient.GetSubject(addr) if err != nil { return "", fmt.Errorf("get subject error: %w", err) diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index 513ffff02..38b240d94 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -678,8 +678,7 @@ func testDenyPutContainerForOthersSessionToken(t *testing.T) { testContainer := containertest.Container() owner := testContainer.Owner() - ownerAddr, err := owner.ScriptHash() - require.NoError(t, err) + ownerAddr := owner.ScriptHash() frostfsIDSubjectReader := &frostfsidStub{ subjects: map[util.Uint160]*client.Subject{ ownerAddr: {}, @@ -690,7 +689,7 @@ func testDenyPutContainerForOthersSessionToken(t *testing.T) { nm.currentEpoch = 100 nm.netmaps = map[uint64]*netmap.NetMap{} - _, _, err = router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(""), &chain.Chain{ + _, _, err := router.MorphRuleChainStorage().AddMorphRuleChain(chain.Ingress, engine.NamespaceTarget(""), &chain.Chain{ Rules: []chain.Rule{ { Status: chain.AccessDenied, @@ -773,7 +772,7 @@ func testDenyPutContainerReadNamespaceFromFrostfsID(t *testing.T) { require.NoError(t, err) req := initPutRequest(t, testContainer) - ownerScriptHash := initOwnerIDScriptHash(t, testContainer) + ownerScriptHash := initOwnerIDScriptHash(testContainer) frostfsIDSubjectReader := &frostfsidStub{ subjects: map[util.Uint160]*client.Subject{ @@ -857,7 +856,7 @@ func testDenyPutContainerInvalidNamespace(t *testing.T) { require.NoError(t, err) req := initPutRequest(t, testContainer) - ownerScriptHash := initOwnerIDScriptHash(t, testContainer) + ownerScriptHash := initOwnerIDScriptHash(testContainer) frostfsIDSubjectReader := &frostfsidStub{ subjects: map[util.Uint160]*client.Subject{ @@ -1532,26 +1531,21 @@ func initPutRequest(t *testing.T, testContainer cnrSDK.Container) *container.Put return req } -func initOwnerIDScriptHash(t *testing.T, testContainer cnrSDK.Container) util.Uint160 { +func initOwnerIDScriptHash(testContainer cnrSDK.Container) util.Uint160 { var ownerSDK *user.ID owner := testContainer.Owner() ownerSDK = &owner - sc, err := ownerSDK.ScriptHash() - require.NoError(t, err) - return sc + return ownerSDK.ScriptHash() } func initActorOwnerScriptHashes(t *testing.T, actorPK *keys.PrivateKey, ownerPK *keys.PrivateKey) (actorScriptHash util.Uint160, ownerScriptHash util.Uint160) { var actorUserID user.ID user.IDFromKey(&actorUserID, ecdsa.PublicKey(*actorPK.PublicKey())) - var err error - actorScriptHash, err = actorUserID.ScriptHash() - require.NoError(t, err) + actorScriptHash = actorUserID.ScriptHash() var ownerUserID user.ID user.IDFromKey(&ownerUserID, ecdsa.PublicKey(*ownerPK.PublicKey())) - ownerScriptHash, err = ownerUserID.ScriptHash() - require.NoError(t, err) + ownerScriptHash = ownerUserID.ScriptHash() require.NotEqual(t, ownerScriptHash.String(), actorScriptHash.String()) return } From 6a580db55effc5d824d3f6f53b2c3bab07781f93 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 30 Jan 2025 16:06:14 +0300 Subject: [PATCH 1703/1943] [#1626] go.mod: Bump frostfs-sdk-go Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 95efa584e..e3f44569e 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250129133430-d195cb510401 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 diff --git a/go.sum b/go.sum index 3be3f2246..e0aacb454 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250129133430-d195cb510401 h1:5EN6YsbF5u2BffTKo5kVwe+Aqcy6CRTTVKcwZOI+Ym0= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250129133430-d195cb510401/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 1df64c5caba97eb0b03a4a2220517cd8c649dac1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 30 Jan 2025 10:23:30 +0300 Subject: [PATCH 1704/1943] [#1621] treesvc: Move variable initialization to top in mergeOperationStreams() Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/sync.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index c48a312fb..4f3262113 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -134,11 +134,6 @@ func (s *Service) SynchronizeTree(ctx context.Context, cid cid.ID, treeID string func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *pilorama.Move) uint64 { defer close(merged) - ms := make([]*pilorama.Move, len(streams)) - for i := range streams { - ms[i] = <-streams[i] - } - // Merging different node streams shuffles incoming operations like that: // // x - operation from the stream A @@ -150,6 +145,11 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram // operation height from the stream B. This height is stored in minStreamedLastHeight. var minStreamedLastHeight uint64 = math.MaxUint64 + ms := make([]*pilorama.Move, len(streams)) + for i := range streams { + ms[i] = <-streams[i] + } + for { var minTimeMoveTime uint64 = math.MaxUint64 minTimeMoveIndex := -1 From 6fcae9f75ab822a67a7a0462cfe3cedaac155b92 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 30 Jan 2025 10:36:02 +0300 Subject: [PATCH 1705/1943] [#1621] treesvc: Cancel background sync on failure If applyOperationStream() exits prematurely, other goroutines will block on send and errgroup will never finish waiting. In this commit we also check whether context is cancelled. Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/sync.go | 35 +++++++++++++++++++++++----------- pkg/services/tree/sync_test.go | 3 ++- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 4f3262113..1a455def9 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -131,7 +131,7 @@ func (s *Service) SynchronizeTree(ctx context.Context, cid cid.ID, treeID string } // mergeOperationStreams performs merge sort for node operation streams to one stream. -func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *pilorama.Move) uint64 { +func mergeOperationStreams(ctx context.Context, streams []chan *pilorama.Move, merged chan<- *pilorama.Move) uint64 { defer close(merged) // Merging different node streams shuffles incoming operations like that: @@ -147,7 +147,11 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram ms := make([]*pilorama.Move, len(streams)) for i := range streams { - ms[i] = <-streams[i] + select { + case ms[i] = <-streams[i]: + case <-ctx.Done(): + return minStreamedLastHeight + } } for { @@ -164,7 +168,11 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram break } - merged <- ms[minTimeMoveIndex] + select { + case merged <- ms[minTimeMoveIndex]: + case <-ctx.Done(): + return minStreamedLastHeight + } height := ms[minTimeMoveIndex].Time if ms[minTimeMoveIndex] = <-streams[minTimeMoveIndex]; ms[minTimeMoveIndex] == nil { minStreamedLastHeight = min(minStreamedLastHeight, height) @@ -176,7 +184,7 @@ func mergeOperationStreams(streams []chan *pilorama.Move, merged chan<- *piloram func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID string, operationStream <-chan *pilorama.Move, -) uint64 { +) (uint64, error) { var prev *pilorama.Move var batch []*pilorama.Move for m := range operationStream { @@ -189,17 +197,17 @@ func (s *Service) applyOperationStream(ctx context.Context, cid cid.ID, treeID s if len(batch) == s.syncBatchSize { if err := s.forest.TreeApplyBatch(ctx, cid, treeID, batch); err != nil { - return batch[0].Time + return batch[0].Time, err } batch = batch[:0] } } if len(batch) > 0 { if err := s.forest.TreeApplyBatch(ctx, cid, treeID, batch); err != nil { - return batch[0].Time + return batch[0].Time, err } } - return math.MaxUint64 + return math.MaxUint64, nil } func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, @@ -235,7 +243,11 @@ func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, if err := m.Meta.FromBytes(lm.GetMeta()); err != nil { return err } - opsCh <- m + select { + case opsCh <- m: + case <-ctx.Done(): + return ctx.Err() + } } if !errors.Is(err, io.EOF) { return err @@ -264,13 +276,14 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, merged := make(chan *pilorama.Move) var minStreamedLastHeight uint64 errGroup.Go(func() error { - minStreamedLastHeight = mergeOperationStreams(nodeOperationStreams, merged) + minStreamedLastHeight = mergeOperationStreams(egCtx, nodeOperationStreams, merged) return nil }) var minUnappliedHeight uint64 errGroup.Go(func() error { - minUnappliedHeight = s.applyOperationStream(ctx, cid, treeID, merged) - return nil + var err error + minUnappliedHeight, err = s.applyOperationStream(egCtx, cid, treeID, merged) + return err }) var allNodesSynced atomic.Bool diff --git a/pkg/services/tree/sync_test.go b/pkg/services/tree/sync_test.go index 497d90554..87d419408 100644 --- a/pkg/services/tree/sync_test.go +++ b/pkg/services/tree/sync_test.go @@ -1,6 +1,7 @@ package tree import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -64,7 +65,7 @@ func Test_mergeOperationStreams(t *testing.T) { merged := make(chan *pilorama.Move, 1) min := make(chan uint64) go func() { - min <- mergeOperationStreams(nodeOpChans, merged) + min <- mergeOperationStreams(context.Background(), nodeOpChans, merged) }() var res []uint64 From 9c4c5a52625180ae8491cee184d2f8583f541e75 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Tue, 14 Jan 2025 16:37:48 +0300 Subject: [PATCH 1706/1943] [#1602] go.mod: Update sdk-go Signed-off-by: Aleksey Savchuk --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e3f44569e..cc6b0a202 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250202151421-8389887a3421 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 diff --git a/go.sum b/go.sum index e0aacb454..eae467b31 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a h1:Ud+3zz4WP9HPxEQxDPJZPpiPdm30nDNSKucsWP9L54M= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250130095343-593dd77d841a/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250202151421-8389887a3421 h1:pP19IawSdsLCKFv7HMNfWAeH6E3uSnntKZkwka+/2+4= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250202151421-8389887a3421/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 4de5fca547359398cd4b6bbf801f15d0bd603b83 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 3 Feb 2025 17:36:21 +0300 Subject: [PATCH 1707/1943] [#1624] blobovniczatree: Return source error from Init() As it was before: when the database was opened, an error returned, but along with the original error, the `context cancelled`` error returned, because `iterateIncompletedRebuildDBPaths` method has `ctx.Done()` check and egCtx passed to `iterateIncompletedRebuildDBPaths` method. Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/control.go | 53 +++++++++---------- .../blobstor/blobovniczatree/control_test.go | 34 ++++++++++++ 2 files changed, 60 insertions(+), 27 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control.go b/pkg/local_object_storage/blobstor/blobovniczatree/control.go index c77df63bf..a6c1ce368 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control.go @@ -41,35 +41,34 @@ func (b *Blobovniczas) initializeDBs(ctx context.Context) error { } eg, egCtx := errgroup.WithContext(ctx) - eg.SetLimit(b.blzInitWorkerCount) - err = b.iterateIncompletedRebuildDBPaths(egCtx, func(p string) (bool, error) { - eg.Go(func() error { - p = strings.TrimSuffix(p, rebuildSuffix) - shBlz := b.getBlobovniczaWithoutCaching(p) - blz, err := shBlz.Open(egCtx) - if err != nil { - return err - } - defer shBlz.Close(egCtx) - - moveInfo, err := blz.ListMoveInfo(egCtx) - if err != nil { - return err - } - for _, move := range moveInfo { - b.deleteProtectedObjects.Add(move.Address) - } - - b.log.Debug(egCtx, logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) - return nil - }) - return false, nil - }) - if err != nil { - _ = eg.Wait() - return err + if b.blzInitWorkerCount > 0 { + eg.SetLimit(b.blzInitWorkerCount + 1) } + eg.Go(func() error { + return b.iterateIncompletedRebuildDBPaths(egCtx, func(p string) (bool, error) { + eg.Go(func() error { + p = strings.TrimSuffix(p, rebuildSuffix) + shBlz := b.getBlobovniczaWithoutCaching(p) + blz, err := shBlz.Open(egCtx) + if err != nil { + return err + } + defer shBlz.Close(egCtx) + moveInfo, err := blz.ListMoveInfo(egCtx) + if err != nil { + return err + } + for _, move := range moveInfo { + b.deleteProtectedObjects.Add(move.Address) + } + + b.log.Debug(egCtx, logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p)) + return nil + }) + return false, nil + }) + }) return eg.Wait() } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go index b26323bd0..7db1891f9 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/control_test.go @@ -2,6 +2,9 @@ package blobovniczatree import ( "context" + "os" + "path" + "strconv" "testing" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -129,3 +132,34 @@ func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) { require.NoError(t, blz.Close(context.Background())) } + +func TestInitBlobovniczasInitErrorType(t *testing.T) { + t.Parallel() + + rootDir := t.TempDir() + + for idx := 0; idx < 10; idx++ { + f, err := os.Create(path.Join(rootDir, strconv.FormatInt(int64(idx), 10)+".db")) + require.NoError(t, err) + _, err = f.Write([]byte("invalid db")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + f, err = os.Create(path.Join(rootDir, strconv.FormatInt(int64(idx), 10)+".db"+rebuildSuffix)) + require.NoError(t, err) + require.NoError(t, f.Close()) + } + + blz := NewBlobovniczaTree( + context.Background(), + WithBlobovniczaShallowDepth(1), + WithBlobovniczaShallowWidth(1), + WithRootPath(rootDir), + ) + + require.NoError(t, blz.Open(mode.ComponentReadWrite)) + err := blz.Init() + require.Contains(t, err.Error(), "open blobovnicza") + require.Contains(t, err.Error(), "invalid database") + require.NoError(t, blz.Close(context.Background())) +} From 9b113c315668fd26a9d43f8b29c9139fcf3fbb5e Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 5 Feb 2025 16:37:11 +0300 Subject: [PATCH 1708/1943] [#1613] morph: Add tracing for morph queries to neo-go Signed-off-by: Alexander Chuprov --- .../modules/control/set_netmap_status.go | 2 +- cmd/frostfs-cli/modules/object/nodes.go | 4 +- cmd/frostfs-node/cache.go | 62 ++++++++------- cmd/frostfs-node/cache_test.go | 11 +-- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/container.go | 22 +++--- cmd/frostfs-node/frostfsid.go | 9 ++- cmd/frostfs-node/netmap.go | 20 ++--- cmd/frostfs-node/object.go | 10 +-- cmd/frostfs-node/tree.go | 12 +-- pkg/ape/request/frostfsid.go | 9 ++- pkg/core/container/info.go | 13 ++-- pkg/core/container/storage.go | 6 +- pkg/core/container/util.go | 5 +- pkg/core/frostfsid/subject_provider.go | 6 +- pkg/core/netmap/storage.go | 16 ++-- pkg/core/object/fmt.go | 2 +- pkg/core/object/fmt_test.go | 14 ++-- pkg/core/object/sender_classifier.go | 15 ++-- pkg/innerring/fetcher.go | 10 ++- pkg/innerring/indexer.go | 19 ++--- pkg/innerring/indexer_test.go | 47 ++++++------ pkg/innerring/innerring.go | 12 +-- pkg/innerring/netmap.go | 5 +- .../processors/alphabet/handlers_test.go | 2 +- .../processors/alphabet/process_emit.go | 2 +- .../processors/alphabet/processor.go | 2 +- pkg/innerring/processors/container/common.go | 13 ++-- .../processors/container/handlers_test.go | 8 +- .../processors/container/process_container.go | 48 ++++++------ .../processors/container/processor.go | 8 +- .../processors/governance/handlers_test.go | 4 +- .../processors/governance/process_update.go | 4 +- .../processors/governance/processor.go | 4 +- .../processors/netmap/handlers_test.go | 8 +- .../netmap/nodevalidation/locode/calls.go | 3 +- .../nodevalidation/locode/calls_test.go | 9 ++- .../netmap/nodevalidation/maddress/calls.go | 3 +- .../netmap/nodevalidation/state/validator.go | 7 +- .../nodevalidation/state/validator_test.go | 5 +- .../netmap/nodevalidation/validator.go | 6 +- .../processors/netmap/process_epoch.go | 4 +- .../processors/netmap/process_peers.go | 4 +- pkg/innerring/processors/netmap/processor.go | 6 +- pkg/innerring/processors/netmap/wrappers.go | 8 +- pkg/innerring/state.go | 6 +- pkg/local_object_storage/engine/engine.go | 2 +- pkg/local_object_storage/engine/evacuate.go | 2 +- .../engine/evacuate_test.go | 4 +- pkg/local_object_storage/metabase/upgrade.go | 10 +-- pkg/local_object_storage/shard/control.go | 2 +- pkg/morph/client/balance/balanceOf.go | 5 +- pkg/morph/client/balance/decimals.go | 5 +- pkg/morph/client/client.go | 2 +- pkg/morph/client/container/containers_of.go | 9 ++- pkg/morph/client/container/deletion_info.go | 15 ++-- pkg/morph/client/container/get.go | 15 ++-- pkg/morph/client/container/list.go | 5 +- pkg/morph/client/frostfsid/subject.go | 9 ++- pkg/morph/client/netmap/config.go | 48 ++++++------ pkg/morph/client/netmap/epoch.go | 9 ++- pkg/morph/client/netmap/innerring.go | 4 +- pkg/morph/client/netmap/netmap.go | 13 ++-- pkg/morph/client/netmap/snapshot.go | 5 +- pkg/morph/client/static.go | 5 +- pkg/services/accounting/morph/executor.go | 6 +- pkg/services/apemanager/executor.go | 16 ++-- pkg/services/common/ape/checker.go | 7 +- pkg/services/container/ape.go | 76 +++++++++---------- pkg/services/container/ape_test.go | 16 ++-- pkg/services/container/morph/executor.go | 14 ++-- pkg/services/control/ir/server/calls.go | 6 +- pkg/services/control/rpc.go | 3 + pkg/services/control/server/evacuate_async.go | 10 +-- .../control/server/get_netmap_status.go | 4 +- pkg/services/control/server/server.go | 2 +- pkg/services/netmap/executor.go | 6 +- pkg/services/object/acl/v2/service.go | 8 +- pkg/services/object/acl/v2/types.go | 4 +- pkg/services/object/ape/checker.go | 2 +- pkg/services/object/ape/checker_test.go | 16 ++-- pkg/services/object/ape/request.go | 16 ++-- pkg/services/object/common/target/target.go | 23 +++--- pkg/services/object/common/writer/common.go | 2 +- pkg/services/object/common/writer/ec.go | 12 +-- pkg/services/object/common/writer/ec_test.go | 2 +- pkg/services/object/common/writer/writer.go | 4 +- pkg/services/object/get/assemblerec.go | 2 +- pkg/services/object/get/get_test.go | 8 +- pkg/services/object/get/getrangeec_test.go | 4 +- pkg/services/object/get/remote_getter.go | 2 +- pkg/services/object/get/request.go | 4 +- pkg/services/object/get/types.go | 4 +- pkg/services/object/get/v2/get_range_hash.go | 12 +-- pkg/services/object/patch/streamer.go | 2 +- pkg/services/object/put/single.go | 14 ++-- pkg/services/object/put/streamer.go | 2 +- pkg/services/object/put/v2/streamer.go | 2 +- pkg/services/object/search/container.go | 8 +- pkg/services/object/search/exec.go | 6 +- pkg/services/object/search/search_test.go | 8 +- pkg/services/object/search/service.go | 4 +- pkg/services/object/search/util.go | 2 +- pkg/services/object/util/placement.go | 17 +++-- .../object_manager/placement/netmap.go | 7 +- .../object_manager/placement/traverser.go | 7 +- .../placement/traverser_test.go | 25 +++--- pkg/services/policer/check.go | 6 +- pkg/services/policer/ec.go | 4 +- pkg/services/policer/ec_test.go | 8 +- pkg/services/policer/policer_test.go | 30 ++++---- pkg/services/tree/ape.go | 8 +- pkg/services/tree/ape_test.go | 4 +- pkg/services/tree/container.go | 7 +- pkg/services/tree/options.go | 5 +- pkg/services/tree/replicator.go | 6 +- pkg/services/tree/service.go | 24 +++--- pkg/services/tree/signature.go | 2 +- pkg/services/tree/signature_test.go | 10 +-- pkg/services/tree/sync.go | 10 +-- 120 files changed, 623 insertions(+), 562 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/set_netmap_status.go b/cmd/frostfs-cli/modules/control/set_netmap_status.go index 87c4f3b3d..26a1ba883 100644 --- a/cmd/frostfs-cli/modules/control/set_netmap_status.go +++ b/cmd/frostfs-cli/modules/control/set_netmap_status.go @@ -127,7 +127,7 @@ func awaitSetNetmapStatus(cmd *cobra.Command, pk *ecdsa.PrivateKey, cli *client. var resp *control.GetNetmapStatusResponse var err error err = cli.ExecRaw(func(client *rawclient.Client) error { - resp, err = control.GetNetmapStatus(client, req) + resp, err = control.GetNetmapStatus(cmd.Context(), client, req) return err }) commonCmd.ExitOnErr(cmd, "failed to get current netmap status: %w", err) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 31682c0e1..1500830a2 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -320,7 +320,7 @@ func getReplicaRequiredPlacement(cmd *cobra.Command, objects []phyObject, placem } placementBuilder := placement.NewNetworkMapBuilder(netmap) for _, object := range objects { - placement, err := placementBuilder.BuildPlacement(object.containerID, &object.objectID, placementPolicy) + placement, err := placementBuilder.BuildPlacement(cmd.Context(), object.containerID, &object.objectID, placementPolicy) commonCmd.ExitOnErr(cmd, "failed to get required placement for object: %w", err) for repIdx, rep := range placement { numOfReplicas := placementPolicy.ReplicaDescriptor(repIdx).NumberOfObjects() @@ -358,7 +358,7 @@ func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placem placementObjectID = object.ecHeader.parent } placementBuilder := placement.NewNetworkMapBuilder(netmap) - placement, err := placementBuilder.BuildPlacement(object.containerID, &placementObjectID, placementPolicy) + placement, err := placementBuilder.BuildPlacement(cmd.Context(), object.containerID, &placementObjectID, placementPolicy) commonCmd.ExitOnErr(cmd, "failed to get required placement: %w", err) for _, vector := range placement { diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index b90641799..0fe56d2b0 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -1,6 +1,7 @@ package main import ( + "context" "sync" "time" @@ -16,7 +17,7 @@ import ( "github.com/hashicorp/golang-lru/v2/expirable" ) -type netValueReader[K any, V any] func(K) (V, error) +type netValueReader[K any, V any] func(ctx context.Context, cid K) (V, error) type valueWithError[V any] struct { v V @@ -49,7 +50,7 @@ func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr n // updates the value from the network on cache miss or by TTL. // // returned value should not be modified. -func (c *ttlNetCache[K, V]) get(key K) (V, error) { +func (c *ttlNetCache[K, V]) get(ctx context.Context, key K) (V, error) { hit := false startedAt := time.Now() defer func() { @@ -71,7 +72,7 @@ func (c *ttlNetCache[K, V]) get(key K) (V, error) { return val.v, val.e } - v, err := c.netRdr(key) + v, err := c.netRdr(ctx, key) c.cache.Add(key, &valueWithError[V]{ v: v, @@ -135,7 +136,7 @@ func newNetworkLRUCache(sz int, netRdr netValueReader[uint64, *netmapSDK.NetMap] // updates the value from the network on cache miss. // // returned value should not be modified. -func (c *lruNetCache) get(key uint64) (*netmapSDK.NetMap, error) { +func (c *lruNetCache) get(ctx context.Context, key uint64) (*netmapSDK.NetMap, error) { hit := false startedAt := time.Now() defer func() { @@ -148,7 +149,7 @@ func (c *lruNetCache) get(key uint64) (*netmapSDK.NetMap, error) { return val, nil } - val, err := c.netRdr(key) + val, err := c.netRdr(ctx, key) if err != nil { return nil, err } @@ -166,11 +167,11 @@ type ttlContainerStorage struct { } func newCachedContainerStorage(v container.Source, ttl time.Duration, containerCacheSize uint32) ttlContainerStorage { - lruCnrCache := newNetworkTTLCache(int(containerCacheSize), ttl, func(id cid.ID) (*container.Container, error) { - return v.Get(id) + lruCnrCache := newNetworkTTLCache(int(containerCacheSize), ttl, func(ctx context.Context, id cid.ID) (*container.Container, error) { + return v.Get(ctx, id) }, metrics.NewCacheMetrics("container")) - lruDelInfoCache := newNetworkTTLCache(int(containerCacheSize), ttl, func(id cid.ID) (*container.DelInfo, error) { - return v.DeletionInfo(id) + lruDelInfoCache := newNetworkTTLCache(int(containerCacheSize), ttl, func(ctx context.Context, id cid.ID) (*container.DelInfo, error) { + return v.DeletionInfo(ctx, id) }, metrics.NewCacheMetrics("container_deletion_info")) return ttlContainerStorage{ @@ -188,12 +189,12 @@ func (s ttlContainerStorage) handleRemoval(cnr cid.ID) { // Get returns container value from the cache. If value is missing in the cache // or expired, then it returns value from side chain and updates the cache. -func (s ttlContainerStorage) Get(cnr cid.ID) (*container.Container, error) { - return s.containerCache.get(cnr) +func (s ttlContainerStorage) Get(ctx context.Context, cnr cid.ID) (*container.Container, error) { + return s.containerCache.get(ctx, cnr) } -func (s ttlContainerStorage) DeletionInfo(cnr cid.ID) (*container.DelInfo, error) { - return s.delInfoCache.get(cnr) +func (s ttlContainerStorage) DeletionInfo(ctx context.Context, cnr cid.ID) (*container.DelInfo, error) { + return s.delInfoCache.get(ctx, cnr) } type lruNetmapSource struct { @@ -205,8 +206,8 @@ type lruNetmapSource struct { func newCachedNetmapStorage(s netmap.State, v netmap.Source) netmap.Source { const netmapCacheSize = 10 - lruNetmapCache := newNetworkLRUCache(netmapCacheSize, func(key uint64) (*netmapSDK.NetMap, error) { - return v.GetNetMapByEpoch(key) + lruNetmapCache := newNetworkLRUCache(netmapCacheSize, func(ctx context.Context, key uint64) (*netmapSDK.NetMap, error) { + return v.GetNetMapByEpoch(ctx, key) }, metrics.NewCacheMetrics("netmap")) return &lruNetmapSource{ @@ -215,16 +216,16 @@ func newCachedNetmapStorage(s netmap.State, v netmap.Source) netmap.Source { } } -func (s *lruNetmapSource) GetNetMap(diff uint64) (*netmapSDK.NetMap, error) { - return s.getNetMapByEpoch(s.netState.CurrentEpoch() - diff) +func (s *lruNetmapSource) GetNetMap(ctx context.Context, diff uint64) (*netmapSDK.NetMap, error) { + return s.getNetMapByEpoch(ctx, s.netState.CurrentEpoch()-diff) } -func (s *lruNetmapSource) GetNetMapByEpoch(epoch uint64) (*netmapSDK.NetMap, error) { - return s.getNetMapByEpoch(epoch) +func (s *lruNetmapSource) GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmapSDK.NetMap, error) { + return s.getNetMapByEpoch(ctx, epoch) } -func (s *lruNetmapSource) getNetMapByEpoch(epoch uint64) (*netmapSDK.NetMap, error) { - val, err := s.cache.get(epoch) +func (s *lruNetmapSource) getNetMapByEpoch(ctx context.Context, epoch uint64) (*netmapSDK.NetMap, error) { + val, err := s.cache.get(ctx, epoch) if err != nil { return nil, err } @@ -232,7 +233,7 @@ func (s *lruNetmapSource) getNetMapByEpoch(epoch uint64) (*netmapSDK.NetMap, err return val, nil } -func (s *lruNetmapSource) Epoch() (uint64, error) { +func (s *lruNetmapSource) Epoch(_ context.Context) (uint64, error) { return s.netState.CurrentEpoch(), nil } @@ -240,7 +241,10 @@ type cachedIRFetcher struct { *ttlNetCache[struct{}, [][]byte] } -func newCachedIRFetcher(f interface{ InnerRingKeys() ([][]byte, error) }) cachedIRFetcher { +func newCachedIRFetcher(f interface { + InnerRingKeys(ctx context.Context) ([][]byte, error) +}, +) cachedIRFetcher { const ( irFetcherCacheSize = 1 // we intend to store only one value @@ -254,8 +258,8 @@ func newCachedIRFetcher(f interface{ InnerRingKeys() ([][]byte, error) }) cached ) irFetcherCache := newNetworkTTLCache(irFetcherCacheSize, irFetcherCacheTTL, - func(_ struct{}) ([][]byte, error) { - return f.InnerRingKeys() + func(ctx context.Context, _ struct{}) ([][]byte, error) { + return f.InnerRingKeys(ctx) }, metrics.NewCacheMetrics("ir_keys"), ) @@ -265,8 +269,8 @@ func newCachedIRFetcher(f interface{ InnerRingKeys() ([][]byte, error) }) cached // InnerRingKeys returns cached list of Inner Ring keys. If keys are missing in // the cache or expired, then it returns keys from side chain and updates // the cache. -func (f cachedIRFetcher) InnerRingKeys() ([][]byte, error) { - val, err := f.get(struct{}{}) +func (f cachedIRFetcher) InnerRingKeys(ctx context.Context) ([][]byte, error) { + val, err := f.get(ctx, struct{}{}) if err != nil { return nil, err } @@ -289,7 +293,7 @@ func newCachedMaxObjectSizeSource(src objectwriter.MaxSizeSource) objectwriter.M } } -func (c *ttlMaxObjectSizeCache) MaxObjectSize() uint64 { +func (c *ttlMaxObjectSizeCache) MaxObjectSize(ctx context.Context) uint64 { const ttl = time.Second * 30 hit := false @@ -311,7 +315,7 @@ func (c *ttlMaxObjectSizeCache) MaxObjectSize() uint64 { c.mtx.Lock() size = c.lastSize if !c.lastUpdated.After(prevUpdated) { - size = c.src.MaxObjectSize() + size = c.src.MaxObjectSize(ctx) c.lastSize = size c.lastUpdated = time.Now() } diff --git a/cmd/frostfs-node/cache_test.go b/cmd/frostfs-node/cache_test.go index f8c324a2f..b1601aa67 100644 --- a/cmd/frostfs-node/cache_test.go +++ b/cmd/frostfs-node/cache_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "testing" "time" @@ -17,7 +18,7 @@ func TestTTLNetCache(t *testing.T) { t.Run("Test Add and Get", func(t *testing.T) { ti := time.Now() cache.set(key, ti, nil) - val, err := cache.get(key) + val, err := cache.get(context.Background(), key) require.NoError(t, err) require.Equal(t, ti, val) }) @@ -26,7 +27,7 @@ func TestTTLNetCache(t *testing.T) { ti := time.Now() cache.set(key, ti, nil) time.Sleep(2 * ttlDuration) - val, err := cache.get(key) + val, err := cache.get(context.Background(), key) require.NoError(t, err) require.NotEqual(t, val, ti) }) @@ -35,20 +36,20 @@ func TestTTLNetCache(t *testing.T) { ti := time.Now() cache.set(key, ti, nil) cache.remove(key) - val, err := cache.get(key) + val, err := cache.get(context.Background(), key) require.NoError(t, err) require.NotEqual(t, val, ti) }) t.Run("Test Cache Error", func(t *testing.T) { cache.set("error", time.Now(), errors.New("mock error")) - _, err := cache.get("error") + _, err := cache.get(context.Background(), "error") require.Error(t, err) require.Equal(t, "mock error", err.Error()) }) } -func testNetValueReader(key string) (time.Time, error) { +func testNetValueReader(_ context.Context, key string) (time.Time, error) { if key == "error" { return time.Now(), errors.New("mock error") } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 3e9cd4e11..75d6f6dec 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1205,7 +1205,7 @@ func (c *cfg) setContractNodeInfo(ni *netmap.NodeInfo) { } func (c *cfg) updateContractNodeInfo(ctx context.Context, epoch uint64) { - ni, err := c.netmapLocalNodeState(epoch) + ni, err := c.netmapLocalNodeState(ctx, epoch) if err != nil { c.log.Error(ctx, logs.FrostFSNodeCouldNotUpdateNodeStateOnNewEpoch, zap.Uint64("epoch", epoch), diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 98fea9f41..012012297 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -100,7 +100,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c // TODO: use owner directly from the event after neofs-contract#256 will become resolved // but don't forget about the profit of reading the new container and caching it: // creation success are most commonly tracked by polling GET op. - cnr, err := cnrSrc.Get(ev.ID) + cnr, err := cnrSrc.Get(ctx, ev.ID) if err == nil { containerCache.containerCache.set(ev.ID, cnr, nil) } else { @@ -221,25 +221,25 @@ type morphContainerReader struct { src containerCore.Source lister interface { - ContainersOf(*user.ID) ([]cid.ID, error) - IterateContainersOf(*user.ID, func(cid.ID) error) error + ContainersOf(context.Context, *user.ID) ([]cid.ID, error) + IterateContainersOf(context.Context, *user.ID, func(cid.ID) error) error } } -func (x *morphContainerReader) Get(id cid.ID) (*containerCore.Container, error) { - return x.src.Get(id) +func (x *morphContainerReader) Get(ctx context.Context, id cid.ID) (*containerCore.Container, error) { + return x.src.Get(ctx, id) } -func (x *morphContainerReader) DeletionInfo(id cid.ID) (*containerCore.DelInfo, error) { - return x.src.DeletionInfo(id) +func (x *morphContainerReader) DeletionInfo(ctx context.Context, id cid.ID) (*containerCore.DelInfo, error) { + return x.src.DeletionInfo(ctx, id) } -func (x *morphContainerReader) ContainersOf(id *user.ID) ([]cid.ID, error) { - return x.lister.ContainersOf(id) +func (x *morphContainerReader) ContainersOf(ctx context.Context, id *user.ID) ([]cid.ID, error) { + return x.lister.ContainersOf(ctx, id) } -func (x *morphContainerReader) IterateContainersOf(id *user.ID, processCID func(cid.ID) error) error { - return x.lister.IterateContainersOf(id, processCID) +func (x *morphContainerReader) IterateContainersOf(ctx context.Context, id *user.ID, processCID func(cid.ID) error) error { + return x.lister.IterateContainersOf(ctx, id, processCID) } type morphContainerWriter struct { diff --git a/cmd/frostfs-node/frostfsid.go b/cmd/frostfs-node/frostfsid.go index 3cca09105..d2d4e9785 100644 --- a/cmd/frostfs-node/frostfsid.go +++ b/cmd/frostfs-node/frostfsid.go @@ -1,6 +1,7 @@ package main import ( + "context" "strings" "time" @@ -42,7 +43,7 @@ func newMorphFrostfsIDCache(subjProvider frostfsidcore.SubjectProvider, size int } } -func (m *morphFrostfsIDCache) GetSubject(addr util.Uint160) (*client.Subject, error) { +func (m *morphFrostfsIDCache) GetSubject(ctx context.Context, addr util.Uint160) (*client.Subject, error) { hit := false startedAt := time.Now() defer func() { @@ -55,7 +56,7 @@ func (m *morphFrostfsIDCache) GetSubject(addr util.Uint160) (*client.Subject, er return result.subject, result.err } - subj, err := m.subjProvider.GetSubject(addr) + subj, err := m.subjProvider.GetSubject(ctx, addr) if err != nil { if m.isCacheableError(err) { m.subjCache.Add(addr, subjectWithError{ @@ -69,7 +70,7 @@ func (m *morphFrostfsIDCache) GetSubject(addr util.Uint160) (*client.Subject, er return subj, nil } -func (m *morphFrostfsIDCache) GetSubjectExtended(addr util.Uint160) (*client.SubjectExtended, error) { +func (m *morphFrostfsIDCache) GetSubjectExtended(ctx context.Context, addr util.Uint160) (*client.SubjectExtended, error) { hit := false startedAt := time.Now() defer func() { @@ -82,7 +83,7 @@ func (m *morphFrostfsIDCache) GetSubjectExtended(addr util.Uint160) (*client.Sub return result.subject, result.err } - subjExt, err := m.subjProvider.GetSubjectExtended(addr) + subjExt, err := m.subjProvider.GetSubjectExtended(ctx, addr) if err != nil { if m.isCacheableError(err) { m.subjExtCache.Add(addr, subjectExtWithError{ diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 34cd00ac8..0e90e7707 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -239,7 +239,7 @@ func setNetmapNotificationParser(c *cfg, sTyp string, p event.NotificationParser // initNetmapState inits current Network map state. // Must be called after Morph components initialization. func initNetmapState(ctx context.Context, c *cfg) { - epoch, err := c.cfgNetmap.wrapper.Epoch() + epoch, err := c.cfgNetmap.wrapper.Epoch(ctx) fatalOnErrDetails("could not initialize current epoch number", err) var ni *netmapSDK.NodeInfo @@ -278,7 +278,7 @@ func nodeState(ni *netmapSDK.NodeInfo) string { } func (c *cfg) netmapInitLocalNodeState(ctx context.Context, epoch uint64) (*netmapSDK.NodeInfo, error) { - nmNodes, err := c.cfgNetmap.wrapper.GetCandidates() + nmNodes, err := c.cfgNetmap.wrapper.GetCandidates(ctx) if err != nil { return nil, err } @@ -291,7 +291,7 @@ func (c *cfg) netmapInitLocalNodeState(ctx context.Context, epoch uint64) (*netm } } - node, err := c.netmapLocalNodeState(epoch) + node, err := c.netmapLocalNodeState(ctx, epoch) if err != nil { return nil, err } @@ -312,9 +312,9 @@ func (c *cfg) netmapInitLocalNodeState(ctx context.Context, epoch uint64) (*netm return candidate, nil } -func (c *cfg) netmapLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) { +func (c *cfg) netmapLocalNodeState(ctx context.Context, epoch uint64) (*netmapSDK.NodeInfo, error) { // calculate current network state - nm, err := c.cfgNetmap.wrapper.GetNetMapByEpoch(epoch) + nm, err := c.cfgNetmap.wrapper.GetNetMapByEpoch(ctx, epoch) if err != nil { return nil, err } @@ -376,8 +376,8 @@ func (c *cfg) SetNetmapStatus(ctx context.Context, st control.NetmapStatus) erro return c.updateNetMapState(ctx, func(*nmClient.UpdatePeerPrm) {}) } -func (c *cfg) GetNetmapStatus() (control.NetmapStatus, uint64, error) { - epoch, err := c.netMapSource.Epoch() +func (c *cfg) GetNetmapStatus(ctx context.Context) (control.NetmapStatus, uint64, error) { + epoch, err := c.netMapSource.Epoch(ctx) if err != nil { return control.NetmapStatus_STATUS_UNDEFINED, 0, fmt.Errorf("failed to get current epoch: %w", err) } @@ -390,7 +390,7 @@ func (c *cfg) ForceMaintenance(ctx context.Context) error { } func (c *cfg) setMaintenanceStatus(ctx context.Context, force bool) error { - netSettings, err := c.cfgNetmap.wrapper.ReadNetworkConfiguration() + netSettings, err := c.cfgNetmap.wrapper.ReadNetworkConfiguration(ctx) if err != nil { err = fmt.Errorf("read network settings to check maintenance allowance: %w", err) } else if !netSettings.MaintenanceModeAllowed { @@ -438,7 +438,7 @@ type netInfo struct { msPerBlockRdr func() (int64, error) } -func (n *netInfo) Dump(ver version.Version) (*netmapSDK.NetworkInfo, error) { +func (n *netInfo) Dump(ctx context.Context, ver version.Version) (*netmapSDK.NetworkInfo, error) { magic, err := n.magic.MagicNumber() if err != nil { return nil, err @@ -448,7 +448,7 @@ func (n *netInfo) Dump(ver version.Version) (*netmapSDK.NetworkInfo, error) { ni.SetCurrentEpoch(n.netState.CurrentEpoch()) ni.SetMagicNumber(magic) - netInfoMorph, err := n.morphClientNetMap.ReadNetworkConfiguration() + netInfoMorph, err := n.morphClientNetMap.ReadNetworkConfiguration(ctx) if err != nil { return nil, fmt.Errorf("read network configuration using netmap contract client: %w", err) } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 939241168..77446b81c 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -54,10 +54,10 @@ type objectSvc struct { patch *patchsvc.Service } -func (c *cfg) MaxObjectSize() uint64 { - sz, err := c.cfgNetmap.wrapper.MaxObjectSize() +func (c *cfg) MaxObjectSize(ctx context.Context) uint64 { + sz, err := c.cfgNetmap.wrapper.MaxObjectSize(ctx) if err != nil { - c.log.Error(context.Background(), logs.FrostFSNodeCouldNotGetMaxObjectSizeValue, + c.log.Error(ctx, logs.FrostFSNodeCouldNotGetMaxObjectSizeValue, zap.Error(err), ) } @@ -122,8 +122,8 @@ type innerRingFetcherWithNotary struct { sidechain *morphClient.Client } -func (fn *innerRingFetcherWithNotary) InnerRingKeys() ([][]byte, error) { - keys, err := fn.sidechain.NeoFSAlphabetList() +func (fn *innerRingFetcherWithNotary) InnerRingKeys(ctx context.Context) ([][]byte, error) { + keys, err := fn.sidechain.NeoFSAlphabetList(ctx) if err != nil { return nil, fmt.Errorf("can't get inner ring keys from alphabet role: %w", err) } diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index f3ddc8cbe..f8330a25e 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -29,16 +29,16 @@ type cnrSource struct { cli *containerClient.Client } -func (c cnrSource) Get(id cid.ID) (*container.Container, error) { - return c.src.Get(id) +func (c cnrSource) Get(ctx context.Context, id cid.ID) (*container.Container, error) { + return c.src.Get(ctx, id) } -func (c cnrSource) DeletionInfo(cid cid.ID) (*container.DelInfo, error) { - return c.src.DeletionInfo(cid) +func (c cnrSource) DeletionInfo(ctx context.Context, cid cid.ID) (*container.DelInfo, error) { + return c.src.DeletionInfo(ctx, cid) } -func (c cnrSource) List() ([]cid.ID, error) { - return c.cli.ContainersOf(nil) +func (c cnrSource) List(ctx context.Context) ([]cid.ID, error) { + return c.cli.ContainersOf(ctx, nil) } func initTreeService(c *cfg) { diff --git a/pkg/ape/request/frostfsid.go b/pkg/ape/request/frostfsid.go index c0413678d..d32bd4a07 100644 --- a/pkg/ape/request/frostfsid.go +++ b/pkg/ape/request/frostfsid.go @@ -1,6 +1,7 @@ package request import ( + "context" "fmt" "strconv" "strings" @@ -12,9 +13,9 @@ import ( ) // FormFrostfsIDRequestProperties forms frostfsid specific request properties like user-claim tags and group ID. -func FormFrostfsIDRequestProperties(frostFSIDClient frostfsidcore.SubjectProvider, pk *keys.PublicKey) (map[string]string, error) { +func FormFrostfsIDRequestProperties(ctx context.Context, frostFSIDClient frostfsidcore.SubjectProvider, pk *keys.PublicKey) (map[string]string, error) { reqProps := make(map[string]string) - subj, err := frostFSIDClient.GetSubjectExtended(pk.GetScriptHash()) + subj, err := frostFSIDClient.GetSubjectExtended(ctx, pk.GetScriptHash()) if err != nil { if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { return nil, fmt.Errorf("get subject error: %w", err) @@ -36,8 +37,8 @@ func FormFrostfsIDRequestProperties(frostFSIDClient frostfsidcore.SubjectProvide } // Groups return the actor's group ids from frostfsid contract. -func Groups(frostFSIDClient frostfsidcore.SubjectProvider, pk *keys.PublicKey) ([]string, error) { - subj, err := frostFSIDClient.GetSubjectExtended(pk.GetScriptHash()) +func Groups(ctx context.Context, frostFSIDClient frostfsidcore.SubjectProvider, pk *keys.PublicKey) ([]string, error) { + subj, err := frostFSIDClient.GetSubjectExtended(ctx, pk.GetScriptHash()) if err != nil { if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { return nil, fmt.Errorf("get subject error: %w", err) diff --git a/pkg/core/container/info.go b/pkg/core/container/info.go index 62cc21553..1c52d93e7 100644 --- a/pkg/core/container/info.go +++ b/pkg/core/container/info.go @@ -1,6 +1,7 @@ package container import ( + "context" "sync" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" @@ -19,7 +20,7 @@ type infoValue struct { } type InfoProvider interface { - Info(id cid.ID) (Info, error) + Info(ctx context.Context, id cid.ID) (Info, error) } type infoProvider struct { @@ -43,13 +44,13 @@ func NewInfoProvider(sourceFactory func() (Source, error)) InfoProvider { } } -func (r *infoProvider) Info(id cid.ID) (Info, error) { +func (r *infoProvider) Info(ctx context.Context, id cid.ID) (Info, error) { v, found := r.tryGetFromCache(id) if found { return v.info, v.err } - return r.getFromSource(id) + return r.getFromSource(ctx, id) } func (r *infoProvider) tryGetFromCache(id cid.ID) (infoValue, bool) { @@ -60,7 +61,7 @@ func (r *infoProvider) tryGetFromCache(id cid.ID) (infoValue, bool) { return value, found } -func (r *infoProvider) getFromSource(id cid.ID) (Info, error) { +func (r *infoProvider) getFromSource(ctx context.Context, id cid.ID) (Info, error) { r.kl.Lock(id) defer r.kl.Unlock(id) @@ -75,11 +76,11 @@ func (r *infoProvider) getFromSource(id cid.ID) (Info, error) { return Info{}, r.sourceErr } - cnr, err := r.source.Get(id) + cnr, err := r.source.Get(ctx, id) var civ infoValue if err != nil { if client.IsErrContainerNotFound(err) { - removed, err := WasRemoved(r.source, id) + removed, err := WasRemoved(ctx, r.source, id) if err != nil { civ.err = err } else { diff --git a/pkg/core/container/storage.go b/pkg/core/container/storage.go index ba4404546..4eb14e53c 100644 --- a/pkg/core/container/storage.go +++ b/pkg/core/container/storage.go @@ -1,6 +1,8 @@ package container import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" @@ -41,9 +43,9 @@ type Source interface { // // Implementations must not retain the container pointer and modify // the container through it. - Get(cid.ID) (*Container, error) + Get(ctx context.Context, cid cid.ID) (*Container, error) - DeletionInfo(cid.ID) (*DelInfo, error) + DeletionInfo(ctx context.Context, cid cid.ID) (*DelInfo, error) } // EACL groups information about the FrostFS container's extended ACL stored in diff --git a/pkg/core/container/util.go b/pkg/core/container/util.go index d27556807..a24b36944 100644 --- a/pkg/core/container/util.go +++ b/pkg/core/container/util.go @@ -1,6 +1,7 @@ package container import ( + "context" "errors" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -10,8 +11,8 @@ import ( // WasRemoved checks whether the container ever existed or // it just has not been created yet at the current epoch. -func WasRemoved(s Source, cid cid.ID) (bool, error) { - _, err := s.DeletionInfo(cid) +func WasRemoved(ctx context.Context, s Source, cid cid.ID) (bool, error) { + _, err := s.DeletionInfo(ctx, cid) if err == nil { return true, nil } diff --git a/pkg/core/frostfsid/subject_provider.go b/pkg/core/frostfsid/subject_provider.go index ecfd0eb15..e752043d3 100644 --- a/pkg/core/frostfsid/subject_provider.go +++ b/pkg/core/frostfsid/subject_provider.go @@ -1,6 +1,8 @@ package frostfsid import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" "github.com/nspcc-dev/neo-go/pkg/util" ) @@ -11,6 +13,6 @@ const ( // SubjectProvider interface provides methods to get subject from FrostfsID contract. type SubjectProvider interface { - GetSubject(util.Uint160) (*client.Subject, error) - GetSubjectExtended(util.Uint160) (*client.SubjectExtended, error) + GetSubject(ctx context.Context, addr util.Uint160) (*client.Subject, error) + GetSubjectExtended(ctx context.Context, addr util.Uint160) (*client.SubjectExtended, error) } diff --git a/pkg/core/netmap/storage.go b/pkg/core/netmap/storage.go index 7770c61c7..97313da84 100644 --- a/pkg/core/netmap/storage.go +++ b/pkg/core/netmap/storage.go @@ -1,6 +1,8 @@ package netmap import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) @@ -16,7 +18,7 @@ type Source interface { // // Implementations must not retain the network map pointer and modify // the network map through it. - GetNetMap(diff uint64) (*netmap.NetMap, error) + GetNetMap(ctx context.Context, diff uint64) (*netmap.NetMap, error) // GetNetMapByEpoch reads network map by the epoch number from the storage. // It returns the pointer to the requested network map and any error encountered. @@ -25,21 +27,21 @@ type Source interface { // // Implementations must not retain the network map pointer and modify // the network map through it. - GetNetMapByEpoch(epoch uint64) (*netmap.NetMap, error) + GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmap.NetMap, error) // Epoch reads the current epoch from the storage. // It returns thw number of the current epoch and any error encountered. // // Must return exactly one non-default value. - Epoch() (uint64, error) + Epoch(ctx context.Context) (uint64, error) } // GetLatestNetworkMap requests and returns the latest network map from the storage. -func GetLatestNetworkMap(src Source) (*netmap.NetMap, error) { - return src.GetNetMap(0) +func GetLatestNetworkMap(ctx context.Context, src Source) (*netmap.NetMap, error) { + return src.GetNetMap(ctx, 0) } // GetPreviousNetworkMap requests and returns previous from the latest network map from the storage. -func GetPreviousNetworkMap(src Source) (*netmap.NetMap, error) { - return src.GetNetMap(1) +func GetPreviousNetworkMap(ctx context.Context, src Source) (*netmap.NetMap, error) { + return src.GetNetMap(ctx, 1) } diff --git a/pkg/core/object/fmt.go b/pkg/core/object/fmt.go index 19b5d34e4..cf090eb37 100644 --- a/pkg/core/object/fmt.go +++ b/pkg/core/object/fmt.go @@ -199,7 +199,7 @@ func (v *FormatValidator) isIROrContainerNode(ctx context.Context, obj *objectSD cnrIDBin := make([]byte, sha256.Size) cnrID.Encode(cnrIDBin) - cnr, err := v.containers.Get(cnrID) + cnr, err := v.containers.Get(ctx, cnrID) if err != nil { return acl.RoleOthers, fmt.Errorf("failed to get container (id=%s): %w", cnrID.EncodeToString(), err) } diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index 20560cf3a..239a9f389 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -578,7 +578,7 @@ type testIRSource struct { irNodes [][]byte } -func (s *testIRSource) InnerRingKeys() ([][]byte, error) { +func (s *testIRSource) InnerRingKeys(_ context.Context) ([][]byte, error) { return s.irNodes, nil } @@ -586,14 +586,14 @@ type testContainerSource struct { containers map[cid.ID]*container.Container } -func (s *testContainerSource) Get(cnrID cid.ID) (*container.Container, error) { +func (s *testContainerSource) Get(ctx context.Context, cnrID cid.ID) (*container.Container, error) { if cnr, found := s.containers[cnrID]; found { return cnr, nil } return nil, fmt.Errorf("container not found") } -func (s *testContainerSource) DeletionInfo(cid.ID) (*container.DelInfo, error) { +func (s *testContainerSource) DeletionInfo(context.Context, cid.ID) (*container.DelInfo, error) { return nil, nil } @@ -602,20 +602,20 @@ type testNetmapSource struct { currentEpoch uint64 } -func (s *testNetmapSource) GetNetMap(diff uint64) (*netmap.NetMap, error) { +func (s *testNetmapSource) GetNetMap(ctx context.Context, diff uint64) (*netmap.NetMap, error) { if diff >= s.currentEpoch { return nil, fmt.Errorf("invalid diff") } - return s.GetNetMapByEpoch(s.currentEpoch - diff) + return s.GetNetMapByEpoch(ctx, s.currentEpoch-diff) } -func (s *testNetmapSource) GetNetMapByEpoch(epoch uint64) (*netmap.NetMap, error) { +func (s *testNetmapSource) GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmap.NetMap, error) { if nm, found := s.netmaps[epoch]; found { return nm, nil } return nil, fmt.Errorf("netmap not found") } -func (s *testNetmapSource) Epoch() (uint64, error) { +func (s *testNetmapSource) Epoch(ctx context.Context) (uint64, error) { return s.currentEpoch, nil } diff --git a/pkg/core/object/sender_classifier.go b/pkg/core/object/sender_classifier.go index a1a5fcac1..3733ed507 100644 --- a/pkg/core/object/sender_classifier.go +++ b/pkg/core/object/sender_classifier.go @@ -18,7 +18,7 @@ import ( ) type InnerRing interface { - InnerRingKeys() ([][]byte, error) + InnerRingKeys(ctx context.Context) ([][]byte, error) } type SenderClassifier struct { @@ -63,7 +63,7 @@ func (c SenderClassifier) Classify( } func (c SenderClassifier) IsInnerRingOrContainerNode(ctx context.Context, ownerKeyInBytes []byte, idCnr cid.ID, cnr container.Container) (*ClassifyResult, error) { - isInnerRingNode, err := c.isInnerRingKey(ownerKeyInBytes) + isInnerRingNode, err := c.isInnerRingKey(ctx, ownerKeyInBytes) if err != nil { // do not throw error, try best case matching c.log.Debug(ctx, logs.V2CantCheckIfRequestFromInnerRing, @@ -78,7 +78,7 @@ func (c SenderClassifier) IsInnerRingOrContainerNode(ctx context.Context, ownerK binCnr := make([]byte, sha256.Size) idCnr.Encode(binCnr) - isContainerNode, err := c.isContainerKey(ownerKeyInBytes, binCnr, cnr) + isContainerNode, err := c.isContainerKey(ctx, ownerKeyInBytes, binCnr, cnr) if err != nil { // error might happen if request has `RoleOther` key and placement // is not possible for previous epoch, so @@ -99,8 +99,8 @@ func (c SenderClassifier) IsInnerRingOrContainerNode(ctx context.Context, ownerK }, nil } -func (c SenderClassifier) isInnerRingKey(owner []byte) (bool, error) { - innerRingKeys, err := c.innerRing.InnerRingKeys() +func (c SenderClassifier) isInnerRingKey(ctx context.Context, owner []byte) (bool, error) { + innerRingKeys, err := c.innerRing.InnerRingKeys(ctx) if err != nil { return false, err } @@ -116,10 +116,11 @@ func (c SenderClassifier) isInnerRingKey(owner []byte) (bool, error) { } func (c SenderClassifier) isContainerKey( + ctx context.Context, owner, idCnr []byte, cnr container.Container, ) (bool, error) { - nm, err := core.GetLatestNetworkMap(c.netmap) // first check current netmap + nm, err := core.GetLatestNetworkMap(ctx, c.netmap) // first check current netmap if err != nil { return false, err } @@ -133,7 +134,7 @@ func (c SenderClassifier) isContainerKey( // then check previous netmap, this can happen in-between epoch change // when node migrates data from last epoch container - nm, err = core.GetPreviousNetworkMap(c.netmap) + nm, err = core.GetPreviousNetworkMap(ctx, c.netmap) if err != nil { return false, err } diff --git a/pkg/innerring/fetcher.go b/pkg/innerring/fetcher.go index 4a80ebf3b..7deec3f31 100644 --- a/pkg/innerring/fetcher.go +++ b/pkg/innerring/fetcher.go @@ -1,6 +1,8 @@ package innerring import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -47,12 +49,12 @@ type IrFetcherWithoutNotary struct { // InnerRingKeys fetches list of innerring keys from NeoFSAlphabet // role in the sidechain. -func (fN IrFetcherWithNotary) InnerRingKeys() (keys.PublicKeys, error) { - return fN.cli.NeoFSAlphabetList() +func (fN IrFetcherWithNotary) InnerRingKeys(ctx context.Context) (keys.PublicKeys, error) { + return fN.cli.NeoFSAlphabetList(ctx) } // InnerRingKeys fetches list of innerring keys from netmap contract // in the sidechain. -func (f IrFetcherWithoutNotary) InnerRingKeys() (keys.PublicKeys, error) { - return f.nm.GetInnerRingList() +func (f IrFetcherWithoutNotary) InnerRingKeys(ctx context.Context) (keys.PublicKeys, error) { + return f.nm.GetInnerRingList(ctx) } diff --git a/pkg/innerring/indexer.go b/pkg/innerring/indexer.go index 45135a57b..439400bac 100644 --- a/pkg/innerring/indexer.go +++ b/pkg/innerring/indexer.go @@ -1,6 +1,7 @@ package innerring import ( + "context" "fmt" "sync" "time" @@ -10,7 +11,7 @@ import ( type ( irFetcher interface { - InnerRingKeys() (keys.PublicKeys, error) + InnerRingKeys(ctx context.Context) (keys.PublicKeys, error) } committeeFetcher interface { @@ -45,7 +46,7 @@ func newInnerRingIndexer(comf committeeFetcher, irf irFetcher, key *keys.PublicK } } -func (s *innerRingIndexer) update() (ind indexes, err error) { +func (s *innerRingIndexer) update(ctx context.Context) (ind indexes, err error) { s.RLock() if time.Since(s.lastAccess) < s.timeout { @@ -62,7 +63,7 @@ func (s *innerRingIndexer) update() (ind indexes, err error) { return s.ind, nil } - innerRing, err := s.irFetcher.InnerRingKeys() + innerRing, err := s.irFetcher.InnerRingKeys(ctx) if err != nil { return indexes{}, err } @@ -81,8 +82,8 @@ func (s *innerRingIndexer) update() (ind indexes, err error) { return s.ind, nil } -func (s *innerRingIndexer) InnerRingIndex() (int32, error) { - ind, err := s.update() +func (s *innerRingIndexer) InnerRingIndex(ctx context.Context) (int32, error) { + ind, err := s.update(ctx) if err != nil { return 0, fmt.Errorf("can't update index state: %w", err) } @@ -90,8 +91,8 @@ func (s *innerRingIndexer) InnerRingIndex() (int32, error) { return ind.innerRingIndex, nil } -func (s *innerRingIndexer) InnerRingSize() (int32, error) { - ind, err := s.update() +func (s *innerRingIndexer) InnerRingSize(ctx context.Context) (int32, error) { + ind, err := s.update(ctx) if err != nil { return 0, fmt.Errorf("can't update index state: %w", err) } @@ -99,8 +100,8 @@ func (s *innerRingIndexer) InnerRingSize() (int32, error) { return ind.innerRingSize, nil } -func (s *innerRingIndexer) AlphabetIndex() (int32, error) { - ind, err := s.update() +func (s *innerRingIndexer) AlphabetIndex(ctx context.Context) (int32, error) { + ind, err := s.update(ctx) if err != nil { return 0, fmt.Errorf("can't update index state: %w", err) } diff --git a/pkg/innerring/indexer_test.go b/pkg/innerring/indexer_test.go index c8a819b5b..f8201b7df 100644 --- a/pkg/innerring/indexer_test.go +++ b/pkg/innerring/indexer_test.go @@ -1,6 +1,7 @@ package innerring import ( + "context" "fmt" "sync/atomic" "testing" @@ -37,15 +38,15 @@ func TestIndexerReturnsIndexes(t *testing.T) { indexer := newInnerRingIndexer(cf, irf, key, time.Second) - idx, err := indexer.AlphabetIndex() + idx, err := indexer.AlphabetIndex(context.Background()) require.NoError(t, err, "failed to get alphabet index") require.Equal(t, int32(1), idx, "invalid alphabet index") - idx, err = indexer.InnerRingIndex() + idx, err = indexer.InnerRingIndex(context.Background()) require.NoError(t, err, "failed to get IR index") require.Equal(t, int32(2), idx, "invalid IR index") - size, err := indexer.InnerRingSize() + size, err := indexer.InnerRingSize(context.Background()) require.NoError(t, err, "failed to get IR size") require.Equal(t, int32(3), size, "invalid IR size") }) @@ -56,11 +57,11 @@ func TestIndexerReturnsIndexes(t *testing.T) { indexer := newInnerRingIndexer(cf, irf, key, time.Second) - idx, err := indexer.AlphabetIndex() + idx, err := indexer.AlphabetIndex(context.Background()) require.NoError(t, err, "failed to get alphabet index") require.Equal(t, int32(-1), idx, "invalid alphabet index") - idx, err = indexer.InnerRingIndex() + idx, err = indexer.InnerRingIndex(context.Background()) require.NoError(t, err, "failed to get IR index") require.Equal(t, int32(0), idx, "invalid IR index") }) @@ -71,11 +72,11 @@ func TestIndexerReturnsIndexes(t *testing.T) { indexer := newInnerRingIndexer(cf, irf, key, time.Second) - idx, err := indexer.AlphabetIndex() + idx, err := indexer.AlphabetIndex(context.Background()) require.NoError(t, err, "failed to get alphabet index") require.Equal(t, int32(0), idx, "invalid alphabet index") - idx, err = indexer.InnerRingIndex() + idx, err = indexer.InnerRingIndex(context.Background()) require.NoError(t, err, "failed to get IR index") require.Equal(t, int32(-1), idx, "invalid IR index") }) @@ -100,30 +101,30 @@ func TestIndexerCachesIndexes(t *testing.T) { indexer := newInnerRingIndexer(cf, irf, key, time.Second) - idx, err := indexer.AlphabetIndex() + idx, err := indexer.AlphabetIndex(context.Background()) require.NoError(t, err, "failed to get alphabet index") require.Equal(t, int32(-1), idx, "invalid alphabet index") - idx, err = indexer.InnerRingIndex() + idx, err = indexer.InnerRingIndex(context.Background()) require.NoError(t, err, "failed to get IR index") require.Equal(t, int32(-1), idx, "invalid IR index") - size, err := indexer.InnerRingSize() + size, err := indexer.InnerRingSize(context.Background()) require.NoError(t, err, "failed to get IR size") require.Equal(t, int32(0), size, "invalid IR size") require.Equal(t, int32(1), cf.calls.Load(), "invalid commitee calls count") require.Equal(t, int32(1), irf.calls.Load(), "invalid IR calls count") - idx, err = indexer.AlphabetIndex() + idx, err = indexer.AlphabetIndex(context.Background()) require.NoError(t, err, "failed to get alphabet index") require.Equal(t, int32(-1), idx, "invalid alphabet index") - idx, err = indexer.InnerRingIndex() + idx, err = indexer.InnerRingIndex(context.Background()) require.NoError(t, err, "failed to get IR index") require.Equal(t, int32(-1), idx, "invalid IR index") - size, err = indexer.InnerRingSize() + size, err = indexer.InnerRingSize(context.Background()) require.NoError(t, err, "failed to get IR size") require.Equal(t, int32(0), size, "invalid IR size") @@ -132,15 +133,15 @@ func TestIndexerCachesIndexes(t *testing.T) { time.Sleep(2 * time.Second) - idx, err = indexer.AlphabetIndex() + idx, err = indexer.AlphabetIndex(context.Background()) require.NoError(t, err, "failed to get alphabet index") require.Equal(t, int32(-1), idx, "invalid alphabet index") - idx, err = indexer.InnerRingIndex() + idx, err = indexer.InnerRingIndex(context.Background()) require.NoError(t, err, "failed to get IR index") require.Equal(t, int32(-1), idx, "invalid IR index") - size, err = indexer.InnerRingSize() + size, err = indexer.InnerRingSize(context.Background()) require.NoError(t, err, "failed to get IR size") require.Equal(t, int32(0), size, "invalid IR size") @@ -165,15 +166,15 @@ func TestIndexerThrowsErrors(t *testing.T) { indexer := newInnerRingIndexer(cf, irf, key, time.Second) - idx, err := indexer.AlphabetIndex() + idx, err := indexer.AlphabetIndex(context.Background()) require.ErrorContains(t, err, "test commitee error", "error from commitee not throwed") require.Equal(t, int32(0), idx, "invalid alphabet index") - idx, err = indexer.InnerRingIndex() + idx, err = indexer.InnerRingIndex(context.Background()) require.ErrorContains(t, err, "test commitee error", "error from IR not throwed") require.Equal(t, int32(0), idx, "invalid IR index") - size, err := indexer.InnerRingSize() + size, err := indexer.InnerRingSize(context.Background()) require.ErrorContains(t, err, "test commitee error", "error from IR not throwed") require.Equal(t, int32(0), size, "invalid IR size") @@ -189,15 +190,15 @@ func TestIndexerThrowsErrors(t *testing.T) { indexer = newInnerRingIndexer(cf, irf, key, time.Second) - idx, err = indexer.AlphabetIndex() + idx, err = indexer.AlphabetIndex(context.Background()) require.ErrorContains(t, err, "test IR error", "error from commitee not throwed") require.Equal(t, int32(0), idx, "invalid alphabet index") - idx, err = indexer.InnerRingIndex() + idx, err = indexer.InnerRingIndex(context.Background()) require.ErrorContains(t, err, "test IR error", "error from IR not throwed") require.Equal(t, int32(0), idx, "invalid IR index") - size, err = indexer.InnerRingSize() + size, err = indexer.InnerRingSize(context.Background()) require.ErrorContains(t, err, "test IR error", "error from IR not throwed") require.Equal(t, int32(0), size, "invalid IR size") } @@ -219,7 +220,7 @@ type testIRFetcher struct { calls atomic.Int32 } -func (f *testIRFetcher) InnerRingKeys() (keys.PublicKeys, error) { +func (f *testIRFetcher) InnerRingKeys(context.Context) (keys.PublicKeys, error) { f.calls.Add(1) return f.keys, f.err } diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 0b9e83443..ae5661905 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -575,19 +575,19 @@ func parseMultinetConfig(cfg *viper.Viper, m metrics.MultinetMetrics) internalNe func (s *Server) initConfigFromBlockchain(ctx context.Context) error { // get current epoch - epoch, err := s.netmapClient.Epoch() + epoch, err := s.netmapClient.Epoch(ctx) if err != nil { return fmt.Errorf("can't read epoch number: %w", err) } // get current epoch duration - epochDuration, err := s.netmapClient.EpochDuration() + epochDuration, err := s.netmapClient.EpochDuration(ctx) if err != nil { return fmt.Errorf("can't read epoch duration: %w", err) } // get balance precision - balancePrecision, err := s.balanceClient.Decimals() + balancePrecision, err := s.balanceClient.Decimals(ctx) if err != nil { return fmt.Errorf("can't read balance contract precision: %w", err) } @@ -597,7 +597,7 @@ func (s *Server) initConfigFromBlockchain(ctx context.Context) error { s.precision.SetBalancePrecision(balancePrecision) // get next epoch delta tick - s.initialEpochTickDelta, err = s.nextEpochBlockDelta() + s.initialEpochTickDelta, err = s.nextEpochBlockDelta(ctx) if err != nil { return err } @@ -613,8 +613,8 @@ func (s *Server) initConfigFromBlockchain(ctx context.Context) error { return nil } -func (s *Server) nextEpochBlockDelta() (uint32, error) { - epochBlock, err := s.netmapClient.LastEpochBlock() +func (s *Server) nextEpochBlockDelta(ctx context.Context) (uint32, error) { + epochBlock, err := s.netmapClient.LastEpochBlock(ctx) if err != nil { return 0, fmt.Errorf("can't read last epoch block: %w", err) } diff --git a/pkg/innerring/netmap.go b/pkg/innerring/netmap.go index 9961710ca..fb11e9426 100644 --- a/pkg/innerring/netmap.go +++ b/pkg/innerring/netmap.go @@ -1,6 +1,7 @@ package innerring import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" @@ -17,8 +18,8 @@ type networkSettings netmapclient.Client // MaintenanceModeAllowed requests network configuration from the Sidechain // and check allowance of storage node's maintenance mode according to it. // Always returns state.ErrMaintenanceModeDisallowed. -func (s *networkSettings) MaintenanceModeAllowed() error { - allowed, err := (*netmapclient.Client)(s).MaintenanceModeAllowed() +func (s *networkSettings) MaintenanceModeAllowed(ctx context.Context) error { + allowed, err := (*netmapclient.Client)(s).MaintenanceModeAllowed(ctx) if err != nil { return fmt.Errorf("read maintenance mode's allowance from the Sidechain: %w", err) } else if allowed { diff --git a/pkg/innerring/processors/alphabet/handlers_test.go b/pkg/innerring/processors/alphabet/handlers_test.go index ac3e2a14d..1da3c401d 100644 --- a/pkg/innerring/processors/alphabet/handlers_test.go +++ b/pkg/innerring/processors/alphabet/handlers_test.go @@ -279,6 +279,6 @@ type testNetmapClient struct { netmap *netmap.NetMap } -func (c *testNetmapClient) NetMap() (*netmap.NetMap, error) { +func (c *testNetmapClient) NetMap(context.Context) (*netmap.NetMap, error) { return c.netmap, nil } diff --git a/pkg/innerring/processors/alphabet/process_emit.go b/pkg/innerring/processors/alphabet/process_emit.go index 8e11d2d61..d3d0f83f2 100644 --- a/pkg/innerring/processors/alphabet/process_emit.go +++ b/pkg/innerring/processors/alphabet/process_emit.go @@ -44,7 +44,7 @@ func (ap *Processor) processEmit(ctx context.Context) bool { return true } - networkMap, err := ap.netmapClient.NetMap() + networkMap, err := ap.netmapClient.NetMap(ctx) if err != nil { ap.log.Warn(ctx, logs.AlphabetCantGetNetmapSnapshotToEmitGasToStorageNodes, zap.Error(err)) diff --git a/pkg/innerring/processors/alphabet/processor.go b/pkg/innerring/processors/alphabet/processor.go index 2c4654e7c..0aea74003 100644 --- a/pkg/innerring/processors/alphabet/processor.go +++ b/pkg/innerring/processors/alphabet/processor.go @@ -36,7 +36,7 @@ type ( } netmapClient interface { - NetMap() (*netmap.NetMap, error) + NetMap(ctx context.Context) (*netmap.NetMap, error) } morphClient interface { diff --git a/pkg/innerring/processors/container/common.go b/pkg/innerring/processors/container/common.go index ba12ebb37..5334b9a1f 100644 --- a/pkg/innerring/processors/container/common.go +++ b/pkg/innerring/processors/container/common.go @@ -1,6 +1,7 @@ package container import ( + "context" "crypto/ecdsa" "errors" "fmt" @@ -45,7 +46,7 @@ type signatureVerificationData struct { // - v.binPublicKey is a public session key // - session context corresponds to the container and verb in v // - session is "alive" -func (cp *Processor) verifySignature(v signatureVerificationData) error { +func (cp *Processor) verifySignature(ctx context.Context, v signatureVerificationData) error { var err error var key frostfsecdsa.PublicKeyRFC6979 keyProvided := v.binPublicKey != nil @@ -58,7 +59,7 @@ func (cp *Processor) verifySignature(v signatureVerificationData) error { } if len(v.binTokenSession) > 0 { - return cp.verifyByTokenSession(v, &key, keyProvided) + return cp.verifyByTokenSession(ctx, v, &key, keyProvided) } if keyProvided { @@ -77,8 +78,8 @@ func (cp *Processor) verifySignature(v signatureVerificationData) error { return errors.New("signature is invalid or calculated with the key not bound to the container owner") } -func (cp *Processor) checkTokenLifetime(token session.Container) error { - curEpoch, err := cp.netState.Epoch() +func (cp *Processor) checkTokenLifetime(ctx context.Context, token session.Container) error { + curEpoch, err := cp.netState.Epoch(ctx) if err != nil { return fmt.Errorf("could not read current epoch: %w", err) } @@ -90,7 +91,7 @@ func (cp *Processor) checkTokenLifetime(token session.Container) error { return nil } -func (cp *Processor) verifyByTokenSession(v signatureVerificationData, key *frostfsecdsa.PublicKeyRFC6979, keyProvided bool) error { +func (cp *Processor) verifyByTokenSession(ctx context.Context, v signatureVerificationData, key *frostfsecdsa.PublicKeyRFC6979, keyProvided bool) error { var tok session.Container err := tok.Unmarshal(v.binTokenSession) @@ -118,7 +119,7 @@ func (cp *Processor) verifyByTokenSession(v signatureVerificationData, key *fros return errors.New("owner differs with token owner") } - err = cp.checkTokenLifetime(tok) + err = cp.checkTokenLifetime(ctx, tok) if err != nil { return fmt.Errorf("check session lifetime: %w", err) } diff --git a/pkg/innerring/processors/container/handlers_test.go b/pkg/innerring/processors/container/handlers_test.go index f28e5372a..1b3842eb0 100644 --- a/pkg/innerring/processors/container/handlers_test.go +++ b/pkg/innerring/processors/container/handlers_test.go @@ -170,11 +170,11 @@ type testNetworkState struct { epoch uint64 } -func (s *testNetworkState) HomomorphicHashDisabled() (bool, error) { +func (s *testNetworkState) HomomorphicHashDisabled(context.Context) (bool, error) { return s.homHashDisabled, nil } -func (s *testNetworkState) Epoch() (uint64, error) { +func (s *testNetworkState) Epoch(context.Context) (uint64, error) { return s.epoch, nil } @@ -187,7 +187,7 @@ func (c *testContainerClient) ContractAddress() util.Uint160 { return c.contractAddress } -func (c *testContainerClient) Get(cid []byte) (*containercore.Container, error) { +func (c *testContainerClient) Get(ctx context.Context, cid []byte) (*containercore.Container, error) { key := hex.EncodeToString(cid) if cont, found := c.get[key]; found { return cont, nil @@ -237,6 +237,6 @@ func (c *testMorphClient) NotarySignAndInvokeTX(mainTx *transaction.Transaction) type testFrostFSIDClient struct{} -func (c *testFrostFSIDClient) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) { +func (c *testFrostFSIDClient) GetSubject(ctx context.Context, addr util.Uint160) (*frostfsidclient.Subject, error) { return &frostfsidclient.Subject{}, nil } diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index ffaea653a..854e2c779 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -47,7 +47,7 @@ func (cp *Processor) processContainerPut(ctx context.Context, put putEvent) bool e: put, } - err := cp.checkPutContainer(pctx) + err := cp.checkPutContainer(ctx, pctx) if err != nil { cp.log.Error(ctx, logs.ContainerPutContainerCheckFailed, zap.Error(err), @@ -66,8 +66,8 @@ func (cp *Processor) processContainerPut(ctx context.Context, put putEvent) bool return true } -func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { - binCnr := ctx.e.Container() +func (cp *Processor) checkPutContainer(ctx context.Context, pctx *putContainerContext) error { + binCnr := pctx.e.Container() var cnr containerSDK.Container err := cnr.Unmarshal(binCnr) @@ -75,12 +75,12 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { return fmt.Errorf("invalid binary container: %w", err) } - err = cp.verifySignature(signatureVerificationData{ + err = cp.verifySignature(ctx, signatureVerificationData{ ownerContainer: cnr.Owner(), verb: session.VerbContainerPut, - binTokenSession: ctx.e.SessionToken(), - binPublicKey: ctx.e.PublicKey(), - signature: ctx.e.Signature(), + binTokenSession: pctx.e.SessionToken(), + binPublicKey: pctx.e.PublicKey(), + signature: pctx.e.Signature(), signedData: binCnr, }) if err != nil { @@ -88,13 +88,13 @@ func (cp *Processor) checkPutContainer(ctx *putContainerContext) error { } // check homomorphic hashing setting - err = checkHomomorphicHashing(cp.netState, cnr) + err = checkHomomorphicHashing(ctx, cp.netState, cnr) if err != nil { return fmt.Errorf("incorrect homomorphic hashing setting: %w", err) } // check native name and zone - err = cp.checkNNS(ctx, cnr) + err = cp.checkNNS(ctx, pctx, cnr) if err != nil { return fmt.Errorf("NNS: %w", err) } @@ -110,7 +110,7 @@ func (cp *Processor) processContainerDelete(ctx context.Context, e containerEven return true } - err := cp.checkDeleteContainer(e) + err := cp.checkDeleteContainer(ctx, e) if err != nil { cp.log.Error(ctx, logs.ContainerDeleteContainerCheckFailed, zap.Error(err), @@ -130,7 +130,7 @@ func (cp *Processor) processContainerDelete(ctx context.Context, e containerEven return true } -func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { +func (cp *Processor) checkDeleteContainer(ctx context.Context, e containerEvent.Delete) error { binCnr := e.ContainerID() var idCnr cid.ID @@ -141,12 +141,12 @@ func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { } // receive owner of the related container - cnr, err := cp.cnrClient.Get(binCnr) + cnr, err := cp.cnrClient.Get(ctx, binCnr) if err != nil { return fmt.Errorf("could not receive the container: %w", err) } - err = cp.verifySignature(signatureVerificationData{ + err = cp.verifySignature(ctx, signatureVerificationData{ ownerContainer: cnr.Value.Owner(), verb: session.VerbContainerDelete, idContainerSet: true, @@ -163,21 +163,21 @@ func (cp *Processor) checkDeleteContainer(e containerEvent.Delete) error { return nil } -func (cp *Processor) checkNNS(ctx *putContainerContext, cnr containerSDK.Container) error { +func (cp *Processor) checkNNS(ctx context.Context, pctx *putContainerContext, cnr containerSDK.Container) error { // fetch domain info - ctx.d = containerSDK.ReadDomain(cnr) + pctx.d = containerSDK.ReadDomain(cnr) // if PutNamed event => check if values in container correspond to args - if named, ok := ctx.e.(interface { + if named, ok := pctx.e.(interface { Name() string Zone() string }); ok { - if name := named.Name(); name != ctx.d.Name() { - return fmt.Errorf("names differ %s/%s", name, ctx.d.Name()) + if name := named.Name(); name != pctx.d.Name() { + return fmt.Errorf("names differ %s/%s", name, pctx.d.Name()) } - if zone := named.Zone(); zone != ctx.d.Zone() { - return fmt.Errorf("zones differ %s/%s", zone, ctx.d.Zone()) + if zone := named.Zone(); zone != pctx.d.Zone() { + return fmt.Errorf("zones differ %s/%s", zone, pctx.d.Zone()) } } @@ -186,12 +186,12 @@ func (cp *Processor) checkNNS(ctx *putContainerContext, cnr containerSDK.Contain return fmt.Errorf("could not get container owner address: %w", err) } - subject, err := cp.frostFSIDClient.GetSubject(addr) + subject, err := cp.frostFSIDClient.GetSubject(ctx, addr) if err != nil { return fmt.Errorf("could not get subject from FrostfsID contract: %w", err) } - namespace, hasNamespace := strings.CutSuffix(ctx.d.Zone(), ".ns") + namespace, hasNamespace := strings.CutSuffix(pctx.d.Zone(), ".ns") if !hasNamespace { return nil } @@ -203,8 +203,8 @@ func (cp *Processor) checkNNS(ctx *putContainerContext, cnr containerSDK.Contain return nil } -func checkHomomorphicHashing(ns NetworkState, cnr containerSDK.Container) error { - netSetting, err := ns.HomomorphicHashDisabled() +func checkHomomorphicHashing(ctx context.Context, ns NetworkState, cnr containerSDK.Container) error { + netSetting, err := ns.HomomorphicHashDisabled(ctx) if err != nil { return fmt.Errorf("could not get setting in contract: %w", err) } diff --git a/pkg/innerring/processors/container/processor.go b/pkg/innerring/processors/container/processor.go index a0b7491e1..9be93baa4 100644 --- a/pkg/innerring/processors/container/processor.go +++ b/pkg/innerring/processors/container/processor.go @@ -25,7 +25,7 @@ type ( ContClient interface { ContractAddress() util.Uint160 - Get(cid []byte) (*containercore.Container, error) + Get(ctx context.Context, cid []byte) (*containercore.Container, error) } MorphClient interface { @@ -33,7 +33,7 @@ type ( } FrostFSIDClient interface { - GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) + GetSubject(ctx context.Context, addr util.Uint160) (*frostfsidclient.Subject, error) } // Processor of events produced by container contract in the sidechain. @@ -68,7 +68,7 @@ type NetworkState interface { // // Must return any error encountered // which did not allow reading the value. - Epoch() (uint64, error) + Epoch(ctx context.Context) (uint64, error) // HomomorphicHashDisabled must return boolean that // represents homomorphic network state: @@ -76,7 +76,7 @@ type NetworkState interface { // * false if hashing is enabled. // // which did not allow reading the value. - HomomorphicHashDisabled() (bool, error) + HomomorphicHashDisabled(ctx context.Context) (bool, error) } // New creates a container contract processor instance. diff --git a/pkg/innerring/processors/governance/handlers_test.go b/pkg/innerring/processors/governance/handlers_test.go index 5a6126249..864c5da67 100644 --- a/pkg/innerring/processors/governance/handlers_test.go +++ b/pkg/innerring/processors/governance/handlers_test.go @@ -236,7 +236,7 @@ type testIRFetcher struct { publicKeys keys.PublicKeys } -func (f *testIRFetcher) InnerRingKeys() (keys.PublicKeys, error) { +func (f *testIRFetcher) InnerRingKeys(context.Context) (keys.PublicKeys, error) { return f.publicKeys, nil } @@ -266,7 +266,7 @@ type testMainnetClient struct { designateHash util.Uint160 } -func (c *testMainnetClient) NeoFSAlphabetList() (res keys.PublicKeys, err error) { +func (c *testMainnetClient) NeoFSAlphabetList(context.Context) (res keys.PublicKeys, err error) { return c.alphabetKeys, nil } diff --git a/pkg/innerring/processors/governance/process_update.go b/pkg/innerring/processors/governance/process_update.go index 245679656..6e22abb3c 100644 --- a/pkg/innerring/processors/governance/process_update.go +++ b/pkg/innerring/processors/governance/process_update.go @@ -25,7 +25,7 @@ func (gp *Processor) processAlphabetSync(ctx context.Context, txHash util.Uint25 return true } - mainnetAlphabet, err := gp.mainnetClient.NeoFSAlphabetList() + mainnetAlphabet, err := gp.mainnetClient.NeoFSAlphabetList(ctx) if err != nil { gp.log.Error(ctx, logs.GovernanceCantFetchAlphabetListFromMainNet, zap.Error(err)) @@ -95,7 +95,7 @@ func prettyKeys(keys keys.PublicKeys) string { } func (gp *Processor) updateNeoFSAlphabetRoleInSidechain(ctx context.Context, sidechainAlphabet, newAlphabet keys.PublicKeys, txHash util.Uint256) { - innerRing, err := gp.irFetcher.InnerRingKeys() + innerRing, err := gp.irFetcher.InnerRingKeys(ctx) if err != nil { gp.log.Error(ctx, logs.GovernanceCantFetchInnerRingListFromSideChain, zap.Error(err)) diff --git a/pkg/innerring/processors/governance/processor.go b/pkg/innerring/processors/governance/processor.go index 7859ebee1..2d131edda 100644 --- a/pkg/innerring/processors/governance/processor.go +++ b/pkg/innerring/processors/governance/processor.go @@ -52,7 +52,7 @@ type ( // Implementation must take into account availability of // the notary contract. IRFetcher interface { - InnerRingKeys() (keys.PublicKeys, error) + InnerRingKeys(ctx context.Context) (keys.PublicKeys, error) } FrostFSClient interface { @@ -64,7 +64,7 @@ type ( } MainnetClient interface { - NeoFSAlphabetList() (res keys.PublicKeys, err error) + NeoFSAlphabetList(context.Context) (res keys.PublicKeys, err error) GetDesignateHash() util.Uint160 } diff --git a/pkg/innerring/processors/netmap/handlers_test.go b/pkg/innerring/processors/netmap/handlers_test.go index 5a5adfb2d..934c3790d 100644 --- a/pkg/innerring/processors/netmap/handlers_test.go +++ b/pkg/innerring/processors/netmap/handlers_test.go @@ -294,7 +294,7 @@ type testNodeStateSettings struct { maintAllowed bool } -func (s *testNodeStateSettings) MaintenanceModeAllowed() error { +func (s *testNodeStateSettings) MaintenanceModeAllowed(context.Context) error { if s.maintAllowed { return nil } @@ -303,7 +303,7 @@ func (s *testNodeStateSettings) MaintenanceModeAllowed() error { type testValidator struct{} -func (v *testValidator) VerifyAndUpdate(*netmap.NodeInfo) error { +func (v *testValidator) VerifyAndUpdate(context.Context, *netmap.NodeInfo) error { return nil } @@ -381,7 +381,7 @@ func (c *testNetmapClient) ContractAddress() util.Uint160 { return c.contractAddress } -func (c *testNetmapClient) EpochDuration() (uint64, error) { +func (c *testNetmapClient) EpochDuration(context.Context) (uint64, error) { return c.epochDuration, nil } @@ -392,7 +392,7 @@ func (c *testNetmapClient) MorphTxHeight(h util.Uint256) (uint32, error) { return 0, fmt.Errorf("not found") } -func (c *testNetmapClient) NetMap() (*netmap.NetMap, error) { +func (c *testNetmapClient) NetMap(context.Context) (*netmap.NetMap, error) { return c.netmap, nil } diff --git a/pkg/innerring/processors/netmap/nodevalidation/locode/calls.go b/pkg/innerring/processors/netmap/nodevalidation/locode/calls.go index 5e0558344..b81dc9989 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/locode/calls.go +++ b/pkg/innerring/processors/netmap/nodevalidation/locode/calls.go @@ -1,6 +1,7 @@ package locode import ( + "context" "errors" "fmt" @@ -29,7 +30,7 @@ var errMissingRequiredAttr = errors.New("missing required attribute in DB record // - Continent: R.Continent().String(). // // UN-LOCODE attribute remains untouched. -func (v *Validator) VerifyAndUpdate(n *netmap.NodeInfo) error { +func (v *Validator) VerifyAndUpdate(_ context.Context, n *netmap.NodeInfo) error { attrLocode := n.LOCODE() if attrLocode == "" { return nil diff --git a/pkg/innerring/processors/netmap/nodevalidation/locode/calls_test.go b/pkg/innerring/processors/netmap/nodevalidation/locode/calls_test.go index 8ab174dfd..fa2dd1ac1 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/locode/calls_test.go +++ b/pkg/innerring/processors/netmap/nodevalidation/locode/calls_test.go @@ -1,6 +1,7 @@ package locode_test import ( + "context" "errors" "fmt" "testing" @@ -92,7 +93,7 @@ func TestValidator_VerifyAndUpdate(t *testing.T) { t.Run("w/o locode", func(t *testing.T) { n := nodeInfoWithSomeAttrs() - err := validator.VerifyAndUpdate(n) + err := validator.VerifyAndUpdate(context.Background(), n) require.NoError(t, err) }) @@ -102,7 +103,7 @@ func TestValidator_VerifyAndUpdate(t *testing.T) { addLocodeAttrValue(n, "WRONG LOCODE") - err := validator.VerifyAndUpdate(n) + err := validator.VerifyAndUpdate(context.Background(), n) require.Error(t, err) }) @@ -111,7 +112,7 @@ func TestValidator_VerifyAndUpdate(t *testing.T) { addLocodeAttr(n, locodestd.LOCODE{"RU", "SPB"}) - err := validator.VerifyAndUpdate(n) + err := validator.VerifyAndUpdate(context.Background(), n) require.Error(t, err) }) @@ -119,7 +120,7 @@ func TestValidator_VerifyAndUpdate(t *testing.T) { addLocodeAttr(n, r.LOCODE) - err := validator.VerifyAndUpdate(n) + err := validator.VerifyAndUpdate(context.Background(), n) require.NoError(t, err) require.Equal(t, rec.CountryCode().String(), n.Attribute("CountryCode")) diff --git a/pkg/innerring/processors/netmap/nodevalidation/maddress/calls.go b/pkg/innerring/processors/netmap/nodevalidation/maddress/calls.go index 126f36582..0e4628ac7 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/maddress/calls.go +++ b/pkg/innerring/processors/netmap/nodevalidation/maddress/calls.go @@ -1,6 +1,7 @@ package maddress import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" @@ -8,7 +9,7 @@ import ( ) // VerifyAndUpdate calls network.VerifyAddress. -func (v *Validator) VerifyAndUpdate(n *netmap.NodeInfo) error { +func (v *Validator) VerifyAndUpdate(_ context.Context, n *netmap.NodeInfo) error { err := network.VerifyMultiAddress(*n) if err != nil { return fmt.Errorf("could not verify multiaddress: %w", err) diff --git a/pkg/innerring/processors/netmap/nodevalidation/state/validator.go b/pkg/innerring/processors/netmap/nodevalidation/state/validator.go index e5165f618..03c41a451 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/state/validator.go +++ b/pkg/innerring/processors/netmap/nodevalidation/state/validator.go @@ -7,6 +7,7 @@ map candidates. package state import ( + "context" "errors" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -23,7 +24,7 @@ type NetworkSettings interface { // no error if allowed; // ErrMaintenanceModeDisallowed if disallowed; // other error if there are any problems with the check. - MaintenanceModeAllowed() error + MaintenanceModeAllowed(ctx context.Context) error } // NetMapCandidateValidator represents tool which checks state of nodes which @@ -55,13 +56,13 @@ func (x *NetMapCandidateValidator) SetNetworkSettings(netSettings NetworkSetting // MUST NOT be called before SetNetworkSettings. // // See also netmap.NodeInfo.IsOnline/SetOnline and other similar methods. -func (x *NetMapCandidateValidator) VerifyAndUpdate(node *netmap.NodeInfo) error { +func (x *NetMapCandidateValidator) VerifyAndUpdate(ctx context.Context, node *netmap.NodeInfo) error { if node.Status().IsOnline() { return nil } if node.Status().IsMaintenance() { - return x.netSettings.MaintenanceModeAllowed() + return x.netSettings.MaintenanceModeAllowed(ctx) } return errors.New("invalid status: MUST be either ONLINE or MAINTENANCE") diff --git a/pkg/innerring/processors/netmap/nodevalidation/state/validator_test.go b/pkg/innerring/processors/netmap/nodevalidation/state/validator_test.go index b81d7243b..cbf48a710 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/state/validator_test.go +++ b/pkg/innerring/processors/netmap/nodevalidation/state/validator_test.go @@ -1,6 +1,7 @@ package state_test import ( + "context" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" @@ -13,7 +14,7 @@ type testNetworkSettings struct { disallowed bool } -func (x testNetworkSettings) MaintenanceModeAllowed() error { +func (x testNetworkSettings) MaintenanceModeAllowed(context.Context) error { if x.disallowed { return state.ErrMaintenanceModeDisallowed } @@ -81,7 +82,7 @@ func TestValidator_VerifyAndUpdate(t *testing.T) { testCase.validatorPreparer(&v) } - err := v.VerifyAndUpdate(&node) + err := v.VerifyAndUpdate(context.Background(), &node) if testCase.valid { require.NoError(t, err, testCase.name) diff --git a/pkg/innerring/processors/netmap/nodevalidation/validator.go b/pkg/innerring/processors/netmap/nodevalidation/validator.go index e9b24e024..3dbe98a8d 100644 --- a/pkg/innerring/processors/netmap/nodevalidation/validator.go +++ b/pkg/innerring/processors/netmap/nodevalidation/validator.go @@ -1,6 +1,8 @@ package nodevalidation import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap" apinetmap "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) @@ -26,9 +28,9 @@ func New(validators ...netmap.NodeValidator) *CompositeValidator { // VerifyAndUpdate passes apinetmap.NodeInfo to wrapped validators. // // If error appears, returns it immediately. -func (c *CompositeValidator) VerifyAndUpdate(ni *apinetmap.NodeInfo) error { +func (c *CompositeValidator) VerifyAndUpdate(ctx context.Context, ni *apinetmap.NodeInfo) error { for _, v := range c.validators { - if err := v.VerifyAndUpdate(ni); err != nil { + if err := v.VerifyAndUpdate(ctx, ni); err != nil { return err } } diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index 93e00bbaa..7c78d24a5 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -14,7 +14,7 @@ import ( func (np *Processor) processNewEpoch(ctx context.Context, ev netmapEvent.NewEpoch) bool { epoch := ev.EpochNumber() - epochDuration, err := np.netmapClient.EpochDuration() + epochDuration, err := np.netmapClient.EpochDuration(ctx) if err != nil { np.log.Warn(ctx, logs.NetmapCantGetEpochDuration, zap.Error(err)) @@ -37,7 +37,7 @@ func (np *Processor) processNewEpoch(ctx context.Context, ev netmapEvent.NewEpoc } // get new netmap snapshot - networkMap, err := np.netmapClient.NetMap() + networkMap, err := np.netmapClient.NetMap(ctx) if err != nil { np.log.Warn(ctx, logs.NetmapCantGetNetmapSnapshotToPerformCleanup, zap.Error(err)) diff --git a/pkg/innerring/processors/netmap/process_peers.go b/pkg/innerring/processors/netmap/process_peers.go index 5b565ffd1..b5c727cc7 100644 --- a/pkg/innerring/processors/netmap/process_peers.go +++ b/pkg/innerring/processors/netmap/process_peers.go @@ -39,7 +39,7 @@ func (np *Processor) processAddPeer(ctx context.Context, ev netmapEvent.AddPeer) } // validate and update node info - err = np.nodeValidator.VerifyAndUpdate(&nodeInfo) + err = np.nodeValidator.VerifyAndUpdate(ctx, &nodeInfo) if err != nil { np.log.Warn(ctx, logs.NetmapCouldNotVerifyAndUpdateInformationAboutNetworkMapCandidate, zap.Error(err), @@ -108,7 +108,7 @@ func (np *Processor) processUpdatePeer(ctx context.Context, ev netmapEvent.Updat var err error if ev.Maintenance() { - err = np.nodeStateSettings.MaintenanceModeAllowed() + err = np.nodeStateSettings.MaintenanceModeAllowed(ctx) if err != nil { np.log.Info(ctx, logs.NetmapPreventSwitchingNodeToMaintenanceState, zap.Error(err), diff --git a/pkg/innerring/processors/netmap/processor.go b/pkg/innerring/processors/netmap/processor.go index 36df57afe..277bca1c3 100644 --- a/pkg/innerring/processors/netmap/processor.go +++ b/pkg/innerring/processors/netmap/processor.go @@ -49,15 +49,15 @@ type ( // // If no error occurs, the parameter must point to the // ready-made NodeInfo structure. - VerifyAndUpdate(*netmap.NodeInfo) error + VerifyAndUpdate(context.Context, *netmap.NodeInfo) error } Client interface { MorphNotaryInvoke(ctx context.Context, contract util.Uint160, fee fixedn.Fixed8, nonce uint32, vub *uint32, method string, args ...any) error ContractAddress() util.Uint160 - EpochDuration() (uint64, error) + EpochDuration(ctx context.Context) (uint64, error) MorphTxHeight(h util.Uint256) (res uint32, err error) - NetMap() (*netmap.NetMap, error) + NetMap(ctx context.Context) (*netmap.NetMap, error) NewEpoch(ctx context.Context, epoch uint64) error MorphIsValidScript(script []byte, signers []transaction.Signer) (valid bool, err error) MorphNotarySignAndInvokeTX(mainTx *transaction.Transaction) error diff --git a/pkg/innerring/processors/netmap/wrappers.go b/pkg/innerring/processors/netmap/wrappers.go index 9cd71ae48..310f12248 100644 --- a/pkg/innerring/processors/netmap/wrappers.go +++ b/pkg/innerring/processors/netmap/wrappers.go @@ -34,16 +34,16 @@ func (w *netmapClientWrapper) ContractAddress() util.Uint160 { return w.netmapClient.ContractAddress() } -func (w *netmapClientWrapper) EpochDuration() (uint64, error) { - return w.netmapClient.EpochDuration() +func (w *netmapClientWrapper) EpochDuration(ctx context.Context) (uint64, error) { + return w.netmapClient.EpochDuration(ctx) } func (w *netmapClientWrapper) MorphTxHeight(h util.Uint256) (res uint32, err error) { return w.netmapClient.Morph().TxHeight(h) } -func (w *netmapClientWrapper) NetMap() (*netmap.NetMap, error) { - return w.netmapClient.NetMap() +func (w *netmapClientWrapper) NetMap(ctx context.Context) (*netmap.NetMap, error) { + return w.netmapClient.NetMap(ctx) } func (w *netmapClientWrapper) NewEpoch(ctx context.Context, epoch uint64) error { diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 3e9880e70..0ef771359 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -60,7 +60,7 @@ func (s *Server) IsAlphabet(ctx context.Context) bool { // InnerRingIndex is a getter for a global index of node in inner ring list. Negative // index means that node is not in the inner ring list. func (s *Server) InnerRingIndex(ctx context.Context) int { - index, err := s.statusIndex.InnerRingIndex() + index, err := s.statusIndex.InnerRingIndex(ctx) if err != nil { s.log.Error(ctx, logs.InnerringCantGetInnerRingIndex, zap.Error(err)) return -1 @@ -72,7 +72,7 @@ func (s *Server) InnerRingIndex(ctx context.Context) int { // InnerRingSize is a getter for a global size of inner ring list. This value // paired with inner ring index. func (s *Server) InnerRingSize(ctx context.Context) int { - size, err := s.statusIndex.InnerRingSize() + size, err := s.statusIndex.InnerRingSize(ctx) if err != nil { s.log.Error(ctx, logs.InnerringCantGetInnerRingSize, zap.Error(err)) return 0 @@ -84,7 +84,7 @@ func (s *Server) InnerRingSize(ctx context.Context) int { // AlphabetIndex is a getter for a global index of node in alphabet list. // Negative index means that node is not in the alphabet list. func (s *Server) AlphabetIndex(ctx context.Context) int { - index, err := s.statusIndex.AlphabetIndex() + index, err := s.statusIndex.AlphabetIndex(ctx) if err != nil { s.log.Error(ctx, logs.InnerringCantGetAlphabetIndex, zap.Error(err)) return -1 diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index f82268d1d..85652b3ae 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -279,7 +279,7 @@ func (s *containerSource) IsContainerAvailable(ctx context.Context, id cid.ID) ( return true, nil } - wasRemoved, err := container.WasRemoved(s.cs, id) + wasRemoved, err := container.WasRemoved(ctx, s.cs, id) if err != nil { return false, err } diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index fd1530f53..27eaea768 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -425,7 +425,7 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context } egContainer.Go(func() error { var skip bool - c, err := e.containerSource.Load().cs.Get(cnt) + c, err := e.containerSource.Load().cs.Get(ctx, cnt) if err != nil { if client.IsErrContainerNotFound(err) { skip = true diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 248c39155..b9d7888e7 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -37,7 +37,7 @@ type containerStorage struct { latency time.Duration } -func (cs *containerStorage) Get(id cid.ID) (*coreContainer.Container, error) { +func (cs *containerStorage) Get(ctx context.Context, id cid.ID) (*coreContainer.Container, error) { time.Sleep(cs.latency) v, ok := cs.cntmap[id] if !ok { @@ -49,7 +49,7 @@ func (cs *containerStorage) Get(id cid.ID) (*coreContainer.Container, error) { return &coreCnt, nil } -func (cs *containerStorage) DeletionInfo(cid.ID) (*coreContainer.DelInfo, error) { +func (cs *containerStorage) DeletionInfo(context.Context, cid.ID) (*coreContainer.DelInfo, error) { return nil, nil } diff --git a/pkg/local_object_storage/metabase/upgrade.go b/pkg/local_object_storage/metabase/upgrade.go index 6eba58c69..4948f3424 100644 --- a/pkg/local_object_storage/metabase/upgrade.go +++ b/pkg/local_object_storage/metabase/upgrade.go @@ -360,7 +360,7 @@ func dropUserAttributes(ctx context.Context, db *bbolt.DB, cs container.InfoProv return nil } last = keys[len(keys)-1] - cnt, err := dropNonIndexedUserAttributeBuckets(db, cs, keys) + cnt, err := dropNonIndexedUserAttributeBuckets(ctx, db, cs, keys) if err != nil { log("deleting user attribute buckets completed with an error:", err) return err @@ -376,8 +376,8 @@ func dropUserAttributes(ctx context.Context, db *bbolt.DB, cs container.InfoProv } } -func dropNonIndexedUserAttributeBuckets(db *bbolt.DB, cs container.InfoProvider, keys [][]byte) (uint64, error) { - keysToDrop, err := selectUserAttributeKeysToDrop(keys, cs) +func dropNonIndexedUserAttributeBuckets(ctx context.Context, db *bbolt.DB, cs container.InfoProvider, keys [][]byte) (uint64, error) { + keysToDrop, err := selectUserAttributeKeysToDrop(ctx, keys, cs) if err != nil { return 0, fmt.Errorf("select non indexed user attributes: %w", err) } @@ -394,7 +394,7 @@ func dropNonIndexedUserAttributeBuckets(db *bbolt.DB, cs container.InfoProvider, return uint64(len(keysToDrop)), nil } -func selectUserAttributeKeysToDrop(keys [][]byte, cs container.InfoProvider) ([][]byte, error) { +func selectUserAttributeKeysToDrop(ctx context.Context, keys [][]byte, cs container.InfoProvider) ([][]byte, error) { var keysToDrop [][]byte for _, key := range keys { attr, ok := attributeFromAttributeBucket(key) @@ -409,7 +409,7 @@ func selectUserAttributeKeysToDrop(keys [][]byte, cs container.InfoProvider) ([] if !ok { return nil, fmt.Errorf("parse container ID from user attribute bucket key %s", hex.EncodeToString(key)) } - info, err := cs.Info(contID) + info, err := cs.Info(ctx, contID) if err != nil { return nil, err } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 3136ddfcc..fedde2206 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -280,7 +280,7 @@ func (s *Shard) refillObject(ctx context.Context, data []byte, addr oid.Address, var isIndexedContainer bool if hasIndexedAttribute { - info, err := s.containerInfo.Info(addr.Container()) + info, err := s.containerInfo.Info(ctx, addr.Container()) if err != nil { return err } diff --git a/pkg/morph/client/balance/balanceOf.go b/pkg/morph/client/balance/balanceOf.go index 449d49e5a..4462daab4 100644 --- a/pkg/morph/client/balance/balanceOf.go +++ b/pkg/morph/client/balance/balanceOf.go @@ -1,6 +1,7 @@ package balance import ( + "context" "fmt" "math/big" @@ -10,14 +11,14 @@ import ( // BalanceOf receives the amount of funds in the client's account // through the Balance contract call, and returns it. -func (c *Client) BalanceOf(id user.ID) (*big.Int, error) { +func (c *Client) BalanceOf(ctx context.Context, id user.ID) (*big.Int, error) { h := id.ScriptHash() invokePrm := client.TestInvokePrm{} invokePrm.SetMethod(balanceOfMethod) invokePrm.SetArgs(h) - prms, err := c.client.TestInvoke(invokePrm) + prms, err := c.client.TestInvoke(ctx, invokePrm) if err != nil { return nil, fmt.Errorf("test invoke (%s): %w", balanceOfMethod, err) } else if ln := len(prms); ln != 1 { diff --git a/pkg/morph/client/balance/decimals.go b/pkg/morph/client/balance/decimals.go index c2a66dded..57e61d62b 100644 --- a/pkg/morph/client/balance/decimals.go +++ b/pkg/morph/client/balance/decimals.go @@ -1,6 +1,7 @@ package balance import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" @@ -8,11 +9,11 @@ import ( // Decimals decimal precision of currency transactions // through the Balance contract call, and returns it. -func (c *Client) Decimals() (uint32, error) { +func (c *Client) Decimals(ctx context.Context) (uint32, error) { invokePrm := client.TestInvokePrm{} invokePrm.SetMethod(decimalsMethod) - prms, err := c.client.TestInvoke(invokePrm) + prms, err := c.client.TestInvoke(ctx, invokePrm) if err != nil { return 0, fmt.Errorf("test invoke (%s): %w", decimalsMethod, err) } else if ln := len(prms); ln != 1 { diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 01fcc98e5..10ded5142 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -499,7 +499,7 @@ func (c *Client) TxHeight(h util.Uint256) (res uint32, err error) { // NeoFSAlphabetList returns keys that stored in NeoFS Alphabet role. Main chain // stores alphabet node keys of inner ring there, however the sidechain stores both // alphabet and non alphabet node keys of inner ring. -func (c *Client) NeoFSAlphabetList() (res keys.PublicKeys, err error) { +func (c *Client) NeoFSAlphabetList(_ context.Context) (res keys.PublicKeys, err error) { c.switchLock.RLock() defer c.switchLock.RUnlock() diff --git a/pkg/morph/client/container/containers_of.go b/pkg/morph/client/container/containers_of.go index 074a586be..60fb8ad7c 100644 --- a/pkg/morph/client/container/containers_of.go +++ b/pkg/morph/client/container/containers_of.go @@ -1,6 +1,7 @@ package container import ( + "context" "errors" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -13,7 +14,7 @@ import ( // to the specified user of FrostFS system. If idUser is nil, returns the list of all containers. // // If remote RPC does not support neo-go session API, fallback to List() method. -func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) { +func (c *Client) ContainersOf(ctx context.Context, idUser *user.ID) ([]cid.ID, error) { var cidList []cid.ID var err error @@ -21,7 +22,7 @@ func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) { cidList = append(cidList, id) return nil } - if err = c.IterateContainersOf(idUser, cb); err != nil { + if err = c.IterateContainersOf(ctx, idUser, cb); err != nil { return nil, err } return cidList, nil @@ -30,7 +31,7 @@ func (c *Client) ContainersOf(idUser *user.ID) ([]cid.ID, error) { // iterateContainers iterates over a list of container identifiers // belonging to the specified user of FrostFS system and executes // `cb` on each element. If idUser is nil, calls it on the list of all containers. -func (c *Client) IterateContainersOf(idUser *user.ID, cb func(item cid.ID) error) error { +func (c *Client) IterateContainersOf(ctx context.Context, idUser *user.ID, cb func(item cid.ID) error) error { var rawID []byte if idUser != nil { rawID = idUser.WalletBytes() @@ -59,7 +60,7 @@ func (c *Client) IterateContainersOf(idUser *user.ID, cb func(item cid.ID) error cnrHash := c.client.ContractAddress() err := c.client.Morph().TestInvokeIterator(itemCb, batchSize, cnrHash, containersOfMethod, rawID) if err != nil && errors.Is(err, unwrap.ErrNoSessionID) { - return c.iterate(idUser, cb) + return c.iterate(ctx, idUser, cb) } return err diff --git a/pkg/morph/client/container/deletion_info.go b/pkg/morph/client/container/deletion_info.go index b86e0ce9c..90bcdd7d5 100644 --- a/pkg/morph/client/container/deletion_info.go +++ b/pkg/morph/client/container/deletion_info.go @@ -1,6 +1,7 @@ package container import ( + "context" "crypto/sha256" "fmt" "strings" @@ -14,27 +15,27 @@ import ( "github.com/mr-tron/base58" ) -func (x *containerSource) DeletionInfo(cnr cid.ID) (*containercore.DelInfo, error) { - return DeletionInfo((*Client)(x), cnr) +func (x *containerSource) DeletionInfo(ctx context.Context, cnr cid.ID) (*containercore.DelInfo, error) { + return DeletionInfo(ctx, (*Client)(x), cnr) } type deletionInfo interface { - DeletionInfo(cid []byte) (*containercore.DelInfo, error) + DeletionInfo(ctx context.Context, cid []byte) (*containercore.DelInfo, error) } -func DeletionInfo(c deletionInfo, cnr cid.ID) (*containercore.DelInfo, error) { +func DeletionInfo(ctx context.Context, c deletionInfo, cnr cid.ID) (*containercore.DelInfo, error) { binCnr := make([]byte, sha256.Size) cnr.Encode(binCnr) - return c.DeletionInfo(binCnr) + return c.DeletionInfo(ctx, binCnr) } -func (c *Client) DeletionInfo(cid []byte) (*containercore.DelInfo, error) { +func (c *Client) DeletionInfo(ctx context.Context, cid []byte) (*containercore.DelInfo, error) { prm := client.TestInvokePrm{} prm.SetMethod(deletionInfoMethod) prm.SetArgs(cid) - res, err := c.client.TestInvoke(prm) + res, err := c.client.TestInvoke(ctx, prm) if err != nil { if strings.Contains(err.Error(), containerContract.NotFoundError) { return nil, new(apistatus.ContainerNotFound) diff --git a/pkg/morph/client/container/get.go b/pkg/morph/client/container/get.go index 2ab58bf01..8622d2cdd 100644 --- a/pkg/morph/client/container/get.go +++ b/pkg/morph/client/container/get.go @@ -1,6 +1,7 @@ package container import ( + "context" "crypto/sha256" "fmt" "strings" @@ -16,8 +17,8 @@ import ( type containerSource Client -func (x *containerSource) Get(cnr cid.ID) (*containercore.Container, error) { - return Get((*Client)(x), cnr) +func (x *containerSource) Get(ctx context.Context, cnr cid.ID) (*containercore.Container, error) { + return Get(ctx, (*Client)(x), cnr) } // AsContainerSource provides container Source interface @@ -27,15 +28,15 @@ func AsContainerSource(w *Client) containercore.Source { } type getContainer interface { - Get(cid []byte) (*containercore.Container, error) + Get(ctx context.Context, cid []byte) (*containercore.Container, error) } // Get marshals container ID, and passes it to Wrapper's Get method. -func Get(c getContainer, cnr cid.ID) (*containercore.Container, error) { +func Get(ctx context.Context, c getContainer, cnr cid.ID) (*containercore.Container, error) { binCnr := make([]byte, sha256.Size) cnr.Encode(binCnr) - return c.Get(binCnr) + return c.Get(ctx, binCnr) } // Get reads the container from FrostFS system by binary identifier @@ -43,12 +44,12 @@ func Get(c getContainer, cnr cid.ID) (*containercore.Container, error) { // // If an empty slice is returned for the requested identifier, // storage.ErrNotFound error is returned. -func (c *Client) Get(cid []byte) (*containercore.Container, error) { +func (c *Client) Get(ctx context.Context, cid []byte) (*containercore.Container, error) { prm := client.TestInvokePrm{} prm.SetMethod(getMethod) prm.SetArgs(cid) - res, err := c.client.TestInvoke(prm) + res, err := c.client.TestInvoke(ctx, prm) if err != nil { if strings.Contains(err.Error(), containerContract.NotFoundError) { return nil, new(apistatus.ContainerNotFound) diff --git a/pkg/morph/client/container/list.go b/pkg/morph/client/container/list.go index 78ea8278f..fc63d1beb 100644 --- a/pkg/morph/client/container/list.go +++ b/pkg/morph/client/container/list.go @@ -1,6 +1,7 @@ package container import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" @@ -15,7 +16,7 @@ import ( // // Iterates through the identifiers of all FrostFS containers if pointer // to user identifier is nil. -func (c *Client) iterate(idUser *user.ID, cb func(cid.ID) error) error { +func (c *Client) iterate(ctx context.Context, idUser *user.ID, cb func(cid.ID) error) error { var rawID []byte if idUser != nil { @@ -26,7 +27,7 @@ func (c *Client) iterate(idUser *user.ID, cb func(cid.ID) error) error { prm.SetMethod(listMethod) prm.SetArgs(rawID) - res, err := c.client.TestInvoke(prm) + res, err := c.client.TestInvoke(ctx, prm) if err != nil { return fmt.Errorf("test invoke (%s): %w", listMethod, err) } else if ln := len(res); ln != 1 { diff --git a/pkg/morph/client/frostfsid/subject.go b/pkg/morph/client/frostfsid/subject.go index 305f3ce09..3a789672a 100644 --- a/pkg/morph/client/frostfsid/subject.go +++ b/pkg/morph/client/frostfsid/subject.go @@ -1,6 +1,7 @@ package frostfsid import ( + "context" "fmt" frostfsidclient "git.frostfs.info/TrueCloudLab/frostfs-contract/frostfsid/client" @@ -14,12 +15,12 @@ const ( methodGetSubjectExtended = "getSubjectExtended" ) -func (c *Client) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) { +func (c *Client) GetSubject(ctx context.Context, addr util.Uint160) (*frostfsidclient.Subject, error) { prm := client.TestInvokePrm{} prm.SetMethod(methodGetSubject) prm.SetArgs(addr) - res, err := c.client.TestInvoke(prm) + res, err := c.client.TestInvoke(ctx, prm) if err != nil { return nil, fmt.Errorf("test invoke (%s): %w", methodGetSubject, err) } @@ -37,12 +38,12 @@ func (c *Client) GetSubject(addr util.Uint160) (*frostfsidclient.Subject, error) return subj, nil } -func (c *Client) GetSubjectExtended(addr util.Uint160) (*frostfsidclient.SubjectExtended, error) { +func (c *Client) GetSubjectExtended(ctx context.Context, addr util.Uint160) (*frostfsidclient.SubjectExtended, error) { prm := client.TestInvokePrm{} prm.SetMethod(methodGetSubjectExtended) prm.SetArgs(addr) - res, err := c.client.TestInvoke(prm) + res, err := c.client.TestInvoke(ctx, prm) if err != nil { return nil, fmt.Errorf("test invoke (%s): %w", methodGetSubjectExtended, err) } diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 2b87df6f7..fcdb70e3f 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -25,8 +25,8 @@ const ( // MaxObjectSize receives max object size configuration // value through the Netmap contract call. -func (c *Client) MaxObjectSize() (uint64, error) { - objectSize, err := c.readUInt64Config(MaxObjectSizeConfig) +func (c *Client) MaxObjectSize(ctx context.Context) (uint64, error) { + objectSize, err := c.readUInt64Config(ctx, MaxObjectSizeConfig) if err != nil { return 0, err } @@ -35,8 +35,8 @@ func (c *Client) MaxObjectSize() (uint64, error) { } // EpochDuration returns number of sidechain blocks per one FrostFS epoch. -func (c *Client) EpochDuration() (uint64, error) { - epochDuration, err := c.readUInt64Config(EpochDurationConfig) +func (c *Client) EpochDuration(ctx context.Context) (uint64, error) { + epochDuration, err := c.readUInt64Config(ctx, EpochDurationConfig) if err != nil { return 0, err } @@ -46,8 +46,8 @@ func (c *Client) EpochDuration() (uint64, error) { // ContainerFee returns fee paid by container owner to each alphabet node // for container registration. -func (c *Client) ContainerFee() (uint64, error) { - fee, err := c.readUInt64Config(ContainerFeeConfig) +func (c *Client) ContainerFee(ctx context.Context) (uint64, error) { + fee, err := c.readUInt64Config(ctx, ContainerFeeConfig) if err != nil { return 0, err } @@ -57,8 +57,8 @@ func (c *Client) ContainerFee() (uint64, error) { // ContainerAliasFee returns additional fee paid by container owner to each // alphabet node for container nice name registration. -func (c *Client) ContainerAliasFee() (uint64, error) { - fee, err := c.readUInt64Config(ContainerAliasFeeConfig) +func (c *Client) ContainerAliasFee(ctx context.Context) (uint64, error) { + fee, err := c.readUInt64Config(ctx, ContainerAliasFeeConfig) if err != nil { return 0, err } @@ -70,14 +70,14 @@ func (c *Client) ContainerAliasFee() (uint64, error) { // settings. // // Returns (false, nil) if config key is not found in the contract. -func (c *Client) HomomorphicHashDisabled() (bool, error) { - return c.readBoolConfig(HomomorphicHashingDisabledKey) +func (c *Client) HomomorphicHashDisabled(ctx context.Context) (bool, error) { + return c.readBoolConfig(ctx, HomomorphicHashingDisabledKey) } // InnerRingCandidateFee returns global configuration value of fee paid by // node to be in inner ring candidates list. -func (c *Client) InnerRingCandidateFee() (uint64, error) { - fee, err := c.readUInt64Config(IrCandidateFeeConfig) +func (c *Client) InnerRingCandidateFee(ctx context.Context) (uint64, error) { + fee, err := c.readUInt64Config(ctx, IrCandidateFeeConfig) if err != nil { return 0, err } @@ -87,8 +87,8 @@ func (c *Client) InnerRingCandidateFee() (uint64, error) { // WithdrawFee returns global configuration value of fee paid by user to // withdraw assets from FrostFS contract. -func (c *Client) WithdrawFee() (uint64, error) { - fee, err := c.readUInt64Config(WithdrawFeeConfig) +func (c *Client) WithdrawFee(ctx context.Context) (uint64, error) { + fee, err := c.readUInt64Config(ctx, WithdrawFeeConfig) if err != nil { return 0, err } @@ -101,12 +101,12 @@ func (c *Client) WithdrawFee() (uint64, error) { // that storage nodes are allowed to switch their state to "maintenance". // // By default, maintenance state is disallowed. -func (c *Client) MaintenanceModeAllowed() (bool, error) { - return c.readBoolConfig(MaintenanceModeAllowedConfig) +func (c *Client) MaintenanceModeAllowed(ctx context.Context) (bool, error) { + return c.readBoolConfig(ctx, MaintenanceModeAllowedConfig) } -func (c *Client) readUInt64Config(key string) (uint64, error) { - v, err := c.config([]byte(key), IntegerAssert) +func (c *Client) readUInt64Config(ctx context.Context, key string) (uint64, error) { + v, err := c.config(ctx, []byte(key), IntegerAssert) if err != nil { return 0, fmt.Errorf("read netconfig value '%s': %w", key, err) } @@ -117,8 +117,8 @@ func (c *Client) readUInt64Config(key string) (uint64, error) { // reads boolean value by the given key from the FrostFS network configuration // stored in the Sidechain. Returns false if key is not presented. -func (c *Client) readBoolConfig(key string) (bool, error) { - v, err := c.config([]byte(key), BoolAssert) +func (c *Client) readBoolConfig(ctx context.Context, key string) (bool, error) { + v, err := c.config(ctx, []byte(key), BoolAssert) if err != nil { if errors.Is(err, ErrConfigNotFound) { return false, nil @@ -199,12 +199,12 @@ type NetworkConfiguration struct { } // ReadNetworkConfiguration reads NetworkConfiguration from the FrostFS Sidechain. -func (c *Client) ReadNetworkConfiguration() (NetworkConfiguration, error) { +func (c *Client) ReadNetworkConfiguration(ctx context.Context) (NetworkConfiguration, error) { var res NetworkConfiguration prm := client.TestInvokePrm{} prm.SetMethod(configListMethod) - items, err := c.client.TestInvoke(prm) + items, err := c.client.TestInvoke(ctx, prm) if err != nil { return res, fmt.Errorf("test invoke (%s): %w", configListMethod, err) @@ -285,12 +285,12 @@ var ErrConfigNotFound = errors.New("config value not found") // method of FrostFS Netmap contract. // // Returns ErrConfigNotFound if config key is not found in the contract. -func (c *Client) config(key []byte, assert func(stackitem.Item) (any, error)) (any, error) { +func (c *Client) config(ctx context.Context, key []byte, assert func(stackitem.Item) (any, error)) (any, error) { prm := client.TestInvokePrm{} prm.SetMethod(configMethod) prm.SetArgs(key) - items, err := c.client.TestInvoke(prm) + items, err := c.client.TestInvoke(ctx, prm) if err != nil { return nil, fmt.Errorf("test invoke (%s): %w", configMethod, err) diff --git a/pkg/morph/client/netmap/epoch.go b/pkg/morph/client/netmap/epoch.go index 6d7394998..8561329ec 100644 --- a/pkg/morph/client/netmap/epoch.go +++ b/pkg/morph/client/netmap/epoch.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" @@ -8,11 +9,11 @@ import ( // Epoch receives number of current FrostFS epoch // through the Netmap contract call. -func (c *Client) Epoch() (uint64, error) { +func (c *Client) Epoch(ctx context.Context) (uint64, error) { prm := client.TestInvokePrm{} prm.SetMethod(epochMethod) - items, err := c.client.TestInvoke(prm) + items, err := c.client.TestInvoke(ctx, prm) if err != nil { return 0, fmt.Errorf("test invoke (%s): %w", epochMethod, err) @@ -32,11 +33,11 @@ func (c *Client) Epoch() (uint64, error) { // LastEpochBlock receives block number of current FrostFS epoch // through the Netmap contract call. -func (c *Client) LastEpochBlock() (uint32, error) { +func (c *Client) LastEpochBlock(ctx context.Context) (uint32, error) { prm := client.TestInvokePrm{} prm.SetMethod(lastEpochBlockMethod) - items, err := c.client.TestInvoke(prm) + items, err := c.client.TestInvoke(ctx, prm) if err != nil { return 0, fmt.Errorf("test invoke (%s): %w", lastEpochBlockMethod, err) diff --git a/pkg/morph/client/netmap/innerring.go b/pkg/morph/client/netmap/innerring.go index 0cfad4c82..0e1f9186b 100644 --- a/pkg/morph/client/netmap/innerring.go +++ b/pkg/morph/client/netmap/innerring.go @@ -40,11 +40,11 @@ func (c *Client) UpdateInnerRing(ctx context.Context, p UpdateIRPrm) error { } // GetInnerRingList return current IR list. -func (c *Client) GetInnerRingList() (keys.PublicKeys, error) { +func (c *Client) GetInnerRingList(ctx context.Context) (keys.PublicKeys, error) { invokePrm := client.TestInvokePrm{} invokePrm.SetMethod(innerRingListMethod) - prms, err := c.client.TestInvoke(invokePrm) + prms, err := c.client.TestInvoke(ctx, invokePrm) if err != nil { return nil, fmt.Errorf("test invoke (%s): %w", innerRingListMethod, err) } diff --git a/pkg/morph/client/netmap/netmap.go b/pkg/morph/client/netmap/netmap.go index a0009ea73..97782fc25 100644 --- a/pkg/morph/client/netmap/netmap.go +++ b/pkg/morph/client/netmap/netmap.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "fmt" netmapcontract "git.frostfs.info/TrueCloudLab/frostfs-contract/netmap" @@ -11,12 +12,12 @@ import ( // GetNetMapByEpoch calls "snapshotByEpoch" method with the given epoch and // decodes netmap.NetMap from the response. -func (c *Client) GetNetMapByEpoch(epoch uint64) (*netmap.NetMap, error) { +func (c *Client) GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmap.NetMap, error) { invokePrm := client.TestInvokePrm{} invokePrm.SetMethod(epochSnapshotMethod) invokePrm.SetArgs(epoch) - res, err := c.client.TestInvoke(invokePrm) + res, err := c.client.TestInvoke(ctx, invokePrm) if err != nil { return nil, fmt.Errorf("test invoke (%s): %w", epochSnapshotMethod, err) @@ -34,11 +35,11 @@ func (c *Client) GetNetMapByEpoch(epoch uint64) (*netmap.NetMap, error) { // GetCandidates calls "netmapCandidates" method and decodes []netmap.NodeInfo // from the response. -func (c *Client) GetCandidates() ([]netmap.NodeInfo, error) { +func (c *Client) GetCandidates(ctx context.Context) ([]netmap.NodeInfo, error) { invokePrm := client.TestInvokePrm{} invokePrm.SetMethod(netMapCandidatesMethod) - res, err := c.client.TestInvoke(invokePrm) + res, err := c.client.TestInvoke(ctx, invokePrm) if err != nil { return nil, fmt.Errorf("test invoke (%s): %w", netMapCandidatesMethod, err) } @@ -51,11 +52,11 @@ func (c *Client) GetCandidates() ([]netmap.NodeInfo, error) { } // NetMap calls "netmap" method and decode netmap.NetMap from the response. -func (c *Client) NetMap() (*netmap.NetMap, error) { +func (c *Client) NetMap(ctx context.Context) (*netmap.NetMap, error) { invokePrm := client.TestInvokePrm{} invokePrm.SetMethod(netMapMethod) - res, err := c.client.TestInvoke(invokePrm) + res, err := c.client.TestInvoke(ctx, invokePrm) if err != nil { return nil, fmt.Errorf("test invoke (%s): %w", netMapMethod, err) diff --git a/pkg/morph/client/netmap/snapshot.go b/pkg/morph/client/netmap/snapshot.go index a5134bcef..9dbec1a90 100644 --- a/pkg/morph/client/netmap/snapshot.go +++ b/pkg/morph/client/netmap/snapshot.go @@ -1,6 +1,7 @@ package netmap import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" @@ -8,12 +9,12 @@ import ( ) // GetNetMap calls "snapshot" method and decodes netmap.NetMap from the response. -func (c *Client) GetNetMap(diff uint64) (*netmap.NetMap, error) { +func (c *Client) GetNetMap(ctx context.Context, diff uint64) (*netmap.NetMap, error) { prm := client.TestInvokePrm{} prm.SetMethod(snapshotMethod) prm.SetArgs(diff) - res, err := c.client.TestInvoke(prm) + res, err := c.client.TestInvoke(ctx, prm) if err != nil { return nil, fmt.Errorf("test invoke (%s): %w", snapshotMethod, err) } diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index 21adebd9e..c4eb120d2 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" @@ -205,7 +206,9 @@ func (ti *TestInvokePrm) SetArgs(args ...any) { } // TestInvoke calls TestInvoke method of Client with static internal script hash. -func (s StaticClient) TestInvoke(prm TestInvokePrm) ([]stackitem.Item, error) { +func (s StaticClient) TestInvoke(ctx context.Context, prm TestInvokePrm) ([]stackitem.Item, error) { + _, span := tracing.StartSpanFromContext(ctx, "Morph.TestInvoke."+prm.method) + defer span.End() return s.client.TestInvoke( s.scScriptHash, prm.method, diff --git a/pkg/services/accounting/morph/executor.go b/pkg/services/accounting/morph/executor.go index b77d3e3e6..6c2df8428 100644 --- a/pkg/services/accounting/morph/executor.go +++ b/pkg/services/accounting/morph/executor.go @@ -21,7 +21,7 @@ func NewExecutor(client *balance.Client) accountingSvc.ServiceExecutor { } } -func (s *morphExecutor) Balance(_ context.Context, body *accounting.BalanceRequestBody) (*accounting.BalanceResponseBody, error) { +func (s *morphExecutor) Balance(ctx context.Context, body *accounting.BalanceRequestBody) (*accounting.BalanceResponseBody, error) { idV2 := body.GetOwnerID() if idV2 == nil { return nil, errors.New("missing account") @@ -34,12 +34,12 @@ func (s *morphExecutor) Balance(_ context.Context, body *accounting.BalanceReque return nil, fmt.Errorf("invalid account: %w", err) } - amount, err := s.client.BalanceOf(id) + amount, err := s.client.BalanceOf(ctx, id) if err != nil { return nil, err } - balancePrecision, err := s.client.Decimals() + balancePrecision, err := s.client.Decimals(ctx) if err != nil { return nil, err } diff --git a/pkg/services/apemanager/executor.go b/pkg/services/apemanager/executor.go index 9d8f665af..58922fede 100644 --- a/pkg/services/apemanager/executor.go +++ b/pkg/services/apemanager/executor.go @@ -78,12 +78,12 @@ var _ Server = (*Service)(nil) // validateContainerTargetRequest validates request for the container target. // It checks if request actor is the owner of the container, otherwise it denies the request. -func (s *Service) validateContainerTargetRequest(cid string, pubKey *keys.PublicKey) error { +func (s *Service) validateContainerTargetRequest(ctx context.Context, cid string, pubKey *keys.PublicKey) error { var cidSDK cidSDK.ID if err := cidSDK.DecodeString(cid); err != nil { return fmt.Errorf("invalid CID format: %w", err) } - isOwner, err := s.isActorContainerOwner(cidSDK, pubKey) + isOwner, err := s.isActorContainerOwner(ctx, cidSDK, pubKey) if err != nil { return fmt.Errorf("failed to check owner: %w", err) } @@ -117,7 +117,7 @@ func (s *Service) AddChain(ctx context.Context, req *apemanagerV2.AddChainReques switch targetType := req.GetBody().GetTarget().GetTargetType(); targetType { case apeV2.TargetTypeContainer: reqCID := req.GetBody().GetTarget().GetName() - if err = s.validateContainerTargetRequest(reqCID, pub); err != nil { + if err = s.validateContainerTargetRequest(ctx, reqCID, pub); err != nil { return nil, err } target = policy_engine.ContainerTarget(reqCID) @@ -153,7 +153,7 @@ func (s *Service) RemoveChain(ctx context.Context, req *apemanagerV2.RemoveChain switch targetType := req.GetBody().GetTarget().GetTargetType(); targetType { case apeV2.TargetTypeContainer: reqCID := req.GetBody().GetTarget().GetName() - if err = s.validateContainerTargetRequest(reqCID, pub); err != nil { + if err = s.validateContainerTargetRequest(ctx, reqCID, pub); err != nil { return nil, err } target = policy_engine.ContainerTarget(reqCID) @@ -177,7 +177,7 @@ func (s *Service) RemoveChain(ctx context.Context, req *apemanagerV2.RemoveChain return resp, nil } -func (s *Service) ListChains(_ context.Context, req *apemanagerV2.ListChainsRequest) (*apemanagerV2.ListChainsResponse, error) { +func (s *Service) ListChains(ctx context.Context, req *apemanagerV2.ListChainsRequest) (*apemanagerV2.ListChainsResponse, error) { pub, err := getSignaturePublicKey(req.GetVerificationHeader()) if err != nil { return nil, err @@ -188,7 +188,7 @@ func (s *Service) ListChains(_ context.Context, req *apemanagerV2.ListChainsRequ switch targetType := req.GetBody().GetTarget().GetTargetType(); targetType { case apeV2.TargetTypeContainer: reqCID := req.GetBody().GetTarget().GetName() - if err = s.validateContainerTargetRequest(reqCID, pub); err != nil { + if err = s.validateContainerTargetRequest(ctx, reqCID, pub); err != nil { return nil, err } target = policy_engine.ContainerTarget(reqCID) @@ -237,13 +237,13 @@ func getSignaturePublicKey(vh *session.RequestVerificationHeader) (*keys.PublicK return key, nil } -func (s *Service) isActorContainerOwner(cid cidSDK.ID, pk *keys.PublicKey) (bool, error) { +func (s *Service) isActorContainerOwner(ctx context.Context, cid cidSDK.ID, pk *keys.PublicKey) (bool, error) { var actor user.ID user.IDFromKey(&actor, (ecdsa.PublicKey)(*pk)) actorOwnerID := new(refs.OwnerID) actor.WriteToV2(actorOwnerID) - cnr, err := s.cnrSrc.Get(cid) + cnr, err := s.cnrSrc.Get(ctx, cid) if err != nil { return false, fmt.Errorf("get container error: %w", err) } diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index 86021c3db..c9b0b7363 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -1,6 +1,7 @@ package ape import ( + "context" "crypto/ecdsa" "errors" "fmt" @@ -48,7 +49,7 @@ type CheckPrm struct { // CheckCore provides methods to perform the common logic of APE check. type CheckCore interface { // CheckAPE performs the common policy-engine check logic on a prepared request. - CheckAPE(prm CheckPrm) error + CheckAPE(ctx context.Context, prm CheckPrm) error } type checkerCoreImpl struct { @@ -70,7 +71,7 @@ func New(localOverrideStorage policyengine.LocalOverrideStorage, morphChainStora } // CheckAPE performs the common policy-engine check logic on a prepared request. -func (c *checkerCoreImpl) CheckAPE(prm CheckPrm) error { +func (c *checkerCoreImpl) CheckAPE(ctx context.Context, prm CheckPrm) error { var cr policyengine.ChainRouter if prm.BearerToken != nil && !prm.BearerToken.Impersonate() { var err error @@ -85,7 +86,7 @@ func (c *checkerCoreImpl) CheckAPE(prm CheckPrm) error { cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.MorphChainStorage, c.LocalOverrideStorage) } - groups, err := aperequest.Groups(c.FrostFSSubjectProvider, prm.PublicKey) + groups, err := aperequest.Groups(ctx, c.FrostFSSubjectProvider, prm.PublicKey) if err != nil { return fmt.Errorf("failed to get group ids: %w", err) } diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index 2c240412b..e1fbe3960 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -49,11 +49,11 @@ var ( ) type ir interface { - InnerRingKeys() ([][]byte, error) + InnerRingKeys(ctx context.Context) ([][]byte, error) } type containers interface { - Get(cid.ID) (*containercore.Container, error) + Get(context.Context, cid.ID) (*containercore.Container, error) } type apeChecker struct { @@ -106,7 +106,7 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.List") defer span.End() - role, pk, err := ac.getRoleWithoutContainerID(req.GetBody().GetOwnerID(), req.GetMetaHeader(), req.GetVerificationHeader()) + role, pk, err := ac.getRoleWithoutContainerID(ctx, req.GetBody().GetOwnerID(), req.GetMetaHeader(), req.GetVerificationHeader()) if err != nil { return nil, err } @@ -116,7 +116,7 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co nativeschema.PropertyKeyActorRole: role, } - reqProps, err = ac.fillWithUserClaimTags(reqProps, pk) + reqProps, err = ac.fillWithUserClaimTags(ctx, reqProps, pk) if err != nil { return nil, err } @@ -126,11 +126,11 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co } } - namespace, err := ac.namespaceByOwner(req.GetBody().GetOwnerID()) + namespace, err := ac.namespaceByOwner(ctx, req.GetBody().GetOwnerID()) if err != nil { return nil, fmt.Errorf("could not get owner namespace: %w", err) } - if err := ac.validateNamespaceByPublicKey(pk, namespace); err != nil { + if err := ac.validateNamespaceByPublicKey(ctx, pk, namespace); err != nil { return nil, err } @@ -143,7 +143,7 @@ func (ac *apeChecker) List(ctx context.Context, req *container.ListRequest) (*co reqProps, ) - groups, err := aperequest.Groups(ac.frostFSIDClient, pk) + groups, err := aperequest.Groups(ctx, ac.frostFSIDClient, pk) if err != nil { return nil, fmt.Errorf("failed to get group ids: %w", err) } @@ -179,7 +179,7 @@ func (ac *apeChecker) ListStream(req *container.ListStreamRequest, stream ListSt ctx, span := tracing.StartSpanFromContext(stream.Context(), "apeChecker.ListStream") defer span.End() - role, pk, err := ac.getRoleWithoutContainerID(req.GetBody().GetOwnerID(), req.GetMetaHeader(), req.GetVerificationHeader()) + role, pk, err := ac.getRoleWithoutContainerID(stream.Context(), req.GetBody().GetOwnerID(), req.GetMetaHeader(), req.GetVerificationHeader()) if err != nil { return err } @@ -189,7 +189,7 @@ func (ac *apeChecker) ListStream(req *container.ListStreamRequest, stream ListSt nativeschema.PropertyKeyActorRole: role, } - reqProps, err = ac.fillWithUserClaimTags(reqProps, pk) + reqProps, err = ac.fillWithUserClaimTags(ctx, reqProps, pk) if err != nil { return err } @@ -199,11 +199,11 @@ func (ac *apeChecker) ListStream(req *container.ListStreamRequest, stream ListSt } } - namespace, err := ac.namespaceByOwner(req.GetBody().GetOwnerID()) + namespace, err := ac.namespaceByOwner(ctx, req.GetBody().GetOwnerID()) if err != nil { return fmt.Errorf("could not get owner namespace: %w", err) } - if err := ac.validateNamespaceByPublicKey(pk, namespace); err != nil { + if err := ac.validateNamespaceByPublicKey(ctx, pk, namespace); err != nil { return err } @@ -216,7 +216,7 @@ func (ac *apeChecker) ListStream(req *container.ListStreamRequest, stream ListSt reqProps, ) - groups, err := aperequest.Groups(ac.frostFSIDClient, pk) + groups, err := aperequest.Groups(ctx, ac.frostFSIDClient, pk) if err != nil { return fmt.Errorf("failed to get group ids: %w", err) } @@ -252,7 +252,7 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont ctx, span := tracing.StartSpanFromContext(ctx, "apeChecker.Put") defer span.End() - role, pk, err := ac.getRoleWithoutContainerID(req.GetBody().GetContainer().GetOwnerID(), req.GetMetaHeader(), req.GetVerificationHeader()) + role, pk, err := ac.getRoleWithoutContainerID(ctx, req.GetBody().GetContainer().GetOwnerID(), req.GetMetaHeader(), req.GetVerificationHeader()) if err != nil { return nil, err } @@ -262,7 +262,7 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont nativeschema.PropertyKeyActorRole: role, } - reqProps, err = ac.fillWithUserClaimTags(reqProps, pk) + reqProps, err = ac.fillWithUserClaimTags(ctx, reqProps, pk) if err != nil { return nil, err } @@ -272,7 +272,7 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont } } - namespace, err := ac.namespaceByKnownOwner(req.GetBody().GetContainer().GetOwnerID()) + namespace, err := ac.namespaceByKnownOwner(ctx, req.GetBody().GetContainer().GetOwnerID()) if err != nil { return nil, fmt.Errorf("get namespace error: %w", err) } @@ -289,7 +289,7 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont reqProps, ) - groups, err := aperequest.Groups(ac.frostFSIDClient, pk) + groups, err := aperequest.Groups(ctx, ac.frostFSIDClient, pk) if err != nil { return nil, fmt.Errorf("failed to get group ids: %w", err) } @@ -321,7 +321,7 @@ func (ac *apeChecker) Put(ctx context.Context, req *container.PutRequest) (*cont return nil, apeErr(nativeschema.MethodPutContainer, s) } -func (ac *apeChecker) getRoleWithoutContainerID(oID *refs.OwnerID, mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader) (string, *keys.PublicKey, error) { +func (ac *apeChecker) getRoleWithoutContainerID(ctx context.Context, oID *refs.OwnerID, mh *session.RequestMetaHeader, vh *session.RequestVerificationHeader) (string, *keys.PublicKey, error) { if vh == nil { return "", nil, errMissingVerificationHeader } @@ -344,7 +344,7 @@ func (ac *apeChecker) getRoleWithoutContainerID(oID *refs.OwnerID, mh *session.R } pkBytes := pk.Bytes() - isIR, err := ac.isInnerRingKey(pkBytes) + isIR, err := ac.isInnerRingKey(ctx, pkBytes) if err != nil { return "", nil, err } @@ -365,7 +365,7 @@ func (ac *apeChecker) validateContainerBoundedOperation(ctx context.Context, con return err } - cont, err := ac.reader.Get(id) + cont, err := ac.reader.Get(ctx, id) if err != nil { return err } @@ -381,7 +381,7 @@ func (ac *apeChecker) validateContainerBoundedOperation(ctx context.Context, con namespace = cntNamespace } - groups, err := aperequest.Groups(ac.frostFSIDClient, pk) + groups, err := aperequest.Groups(ctx, ac.frostFSIDClient, pk) if err != nil { return fmt.Errorf("failed to get group ids: %w", err) } @@ -458,7 +458,7 @@ func (ac *apeChecker) getRequestProps(ctx context.Context, mh *session.RequestMe if err != nil { return nil, nil, err } - role, err := ac.getRole(actor, pk, cont, cnrID) + role, err := ac.getRole(ctx, actor, pk, cont, cnrID) if err != nil { return nil, nil, err } @@ -466,7 +466,7 @@ func (ac *apeChecker) getRequestProps(ctx context.Context, mh *session.RequestMe nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(pk.Bytes()), nativeschema.PropertyKeyActorRole: role, } - reqProps, err = ac.fillWithUserClaimTags(reqProps, pk) + reqProps, err = ac.fillWithUserClaimTags(ctx, reqProps, pk) if err != nil { return nil, nil, err } @@ -478,13 +478,13 @@ func (ac *apeChecker) getRequestProps(ctx context.Context, mh *session.RequestMe return reqProps, pk, nil } -func (ac *apeChecker) getRole(actor *user.ID, pk *keys.PublicKey, cont *containercore.Container, cnrID cid.ID) (string, error) { +func (ac *apeChecker) getRole(ctx context.Context, actor *user.ID, pk *keys.PublicKey, cont *containercore.Container, cnrID cid.ID) (string, error) { if cont.Value.Owner().Equals(*actor) { return nativeschema.PropertyValueContainerRoleOwner, nil } pkBytes := pk.Bytes() - isIR, err := ac.isInnerRingKey(pkBytes) + isIR, err := ac.isInnerRingKey(ctx, pkBytes) if err != nil { return "", err } @@ -492,7 +492,7 @@ func (ac *apeChecker) getRole(actor *user.ID, pk *keys.PublicKey, cont *containe return nativeschema.PropertyValueContainerRoleIR, nil } - isContainer, err := ac.isContainerKey(pkBytes, cnrID, cont) + isContainer, err := ac.isContainerKey(ctx, pkBytes, cnrID, cont) if err != nil { return "", err } @@ -586,8 +586,8 @@ func isOwnerFromKey(id user.ID, key *keys.PublicKey) bool { return id2.Equals(id) } -func (ac *apeChecker) isInnerRingKey(pk []byte) (bool, error) { - innerRingKeys, err := ac.ir.InnerRingKeys() +func (ac *apeChecker) isInnerRingKey(ctx context.Context, pk []byte) (bool, error) { + innerRingKeys, err := ac.ir.InnerRingKeys(ctx) if err != nil { return false, err } @@ -601,11 +601,11 @@ func (ac *apeChecker) isInnerRingKey(pk []byte) (bool, error) { return false, nil } -func (ac *apeChecker) isContainerKey(pk []byte, cnrID cid.ID, cont *containercore.Container) (bool, error) { +func (ac *apeChecker) isContainerKey(ctx context.Context, pk []byte, cnrID cid.ID, cont *containercore.Container) (bool, error) { binCnrID := make([]byte, sha256.Size) cnrID.Encode(binCnrID) - nm, err := netmap.GetLatestNetworkMap(ac.nm) + nm, err := netmap.GetLatestNetworkMap(ctx, ac.nm) if err != nil { return false, err } @@ -616,7 +616,7 @@ func (ac *apeChecker) isContainerKey(pk []byte, cnrID cid.ID, cont *containercor // then check previous netmap, this can happen in-between epoch change // when node migrates data from last epoch container - nm, err = netmap.GetPreviousNetworkMap(ac.nm) + nm, err = netmap.GetPreviousNetworkMap(ctx, ac.nm) if err != nil { return false, err } @@ -641,7 +641,7 @@ func isContainerNode(nm *netmapSDK.NetMap, pk, binCnrID []byte, cont *containerc return false } -func (ac *apeChecker) namespaceByOwner(owner *refs.OwnerID) (string, error) { +func (ac *apeChecker) namespaceByOwner(ctx context.Context, owner *refs.OwnerID) (string, error) { var ownerSDK user.ID if owner == nil { return "", errOwnerIDIsNotSet @@ -652,7 +652,7 @@ func (ac *apeChecker) namespaceByOwner(owner *refs.OwnerID) (string, error) { addr := ownerSDK.ScriptHash() namespace := "" - subject, err := ac.frostFSIDClient.GetSubject(addr) + subject, err := ac.frostFSIDClient.GetSubject(ctx, addr) if err == nil { namespace = subject.Namespace } else { @@ -663,7 +663,7 @@ func (ac *apeChecker) namespaceByOwner(owner *refs.OwnerID) (string, error) { return namespace, nil } -func (ac *apeChecker) namespaceByKnownOwner(owner *refs.OwnerID) (string, error) { +func (ac *apeChecker) namespaceByKnownOwner(ctx context.Context, owner *refs.OwnerID) (string, error) { var ownerSDK user.ID if owner == nil { return "", errOwnerIDIsNotSet @@ -672,7 +672,7 @@ func (ac *apeChecker) namespaceByKnownOwner(owner *refs.OwnerID) (string, error) return "", err } addr := ownerSDK.ScriptHash() - subject, err := ac.frostFSIDClient.GetSubject(addr) + subject, err := ac.frostFSIDClient.GetSubject(ctx, addr) if err != nil { return "", fmt.Errorf("get subject error: %w", err) } @@ -706,12 +706,12 @@ func validateNamespace(cnrV2 *container.Container, ownerIDNamespace string) erro // validateNamespace validates if a namespace of a request actor equals to owner's namespace. // An actor's namespace is calculated by a public key. -func (ac *apeChecker) validateNamespaceByPublicKey(pk *keys.PublicKey, ownerIDNamespace string) error { +func (ac *apeChecker) validateNamespaceByPublicKey(ctx context.Context, pk *keys.PublicKey, ownerIDNamespace string) error { var actor user.ID user.IDFromKey(&actor, (ecdsa.PublicKey)(*pk)) actorOwnerID := new(refs.OwnerID) actor.WriteToV2(actorOwnerID) - actorNamespace, err := ac.namespaceByOwner(actorOwnerID) + actorNamespace, err := ac.namespaceByOwner(ctx, actorOwnerID) if err != nil { return fmt.Errorf("could not get actor namespace: %w", err) } @@ -722,11 +722,11 @@ func (ac *apeChecker) validateNamespaceByPublicKey(pk *keys.PublicKey, ownerIDNa } // fillWithUserClaimTags fills ape request properties with user claim tags getting them from frostfsid contract by actor public key. -func (ac *apeChecker) fillWithUserClaimTags(reqProps map[string]string, pk *keys.PublicKey) (map[string]string, error) { +func (ac *apeChecker) fillWithUserClaimTags(ctx context.Context, reqProps map[string]string, pk *keys.PublicKey) (map[string]string, error) { if reqProps == nil { reqProps = make(map[string]string) } - props, err := aperequest.FormFrostfsIDRequestProperties(ac.frostFSIDClient, pk) + props, err := aperequest.FormFrostfsIDRequestProperties(ctx, ac.frostFSIDClient, pk) if err != nil { return reqProps, err } diff --git a/pkg/services/container/ape_test.go b/pkg/services/container/ape_test.go index 38b240d94..77a981d1a 100644 --- a/pkg/services/container/ape_test.go +++ b/pkg/services/container/ape_test.go @@ -1092,7 +1092,7 @@ type irStub struct { keys [][]byte } -func (s *irStub) InnerRingKeys() ([][]byte, error) { +func (s *irStub) InnerRingKeys(_ context.Context) ([][]byte, error) { return s.keys, nil } @@ -1100,7 +1100,7 @@ type containerStub struct { c map[cid.ID]*containercore.Container } -func (s *containerStub) Get(id cid.ID) (*containercore.Container, error) { +func (s *containerStub) Get(_ context.Context, id cid.ID) (*containercore.Container, error) { if v, ok := s.c[id]; ok { return v, nil } @@ -1112,21 +1112,21 @@ type netmapStub struct { currentEpoch uint64 } -func (s *netmapStub) GetNetMap(diff uint64) (*netmap.NetMap, error) { +func (s *netmapStub) GetNetMap(ctx context.Context, diff uint64) (*netmap.NetMap, error) { if diff >= s.currentEpoch { return nil, errors.New("invalid diff") } - return s.GetNetMapByEpoch(s.currentEpoch - diff) + return s.GetNetMapByEpoch(ctx, s.currentEpoch-diff) } -func (s *netmapStub) GetNetMapByEpoch(epoch uint64) (*netmap.NetMap, error) { +func (s *netmapStub) GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmap.NetMap, error) { if nm, found := s.netmaps[epoch]; found { return nm, nil } return nil, errors.New("netmap not found") } -func (s *netmapStub) Epoch() (uint64, error) { +func (s *netmapStub) Epoch(ctx context.Context) (uint64, error) { return s.currentEpoch, nil } @@ -1135,7 +1135,7 @@ type frostfsidStub struct { subjectsExt map[util.Uint160]*client.SubjectExtended } -func (f *frostfsidStub) GetSubject(owner util.Uint160) (*client.Subject, error) { +func (f *frostfsidStub) GetSubject(ctx context.Context, owner util.Uint160) (*client.Subject, error) { s, ok := f.subjects[owner] if !ok { return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) @@ -1143,7 +1143,7 @@ func (f *frostfsidStub) GetSubject(owner util.Uint160) (*client.Subject, error) return s, nil } -func (f *frostfsidStub) GetSubjectExtended(owner util.Uint160) (*client.SubjectExtended, error) { +func (f *frostfsidStub) GetSubjectExtended(ctx context.Context, owner util.Uint160) (*client.SubjectExtended, error) { s, ok := f.subjectsExt[owner] if !ok { return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) diff --git a/pkg/services/container/morph/executor.go b/pkg/services/container/morph/executor.go index cadf92e19..eaa608eba 100644 --- a/pkg/services/container/morph/executor.go +++ b/pkg/services/container/morph/executor.go @@ -29,8 +29,8 @@ type Reader interface { // ContainersOf returns a list of container identifiers belonging // to the specified user of FrostFS system. Returns the identifiers // of all FrostFS containers if pointer to owner identifier is nil. - ContainersOf(*user.ID) ([]cid.ID, error) - IterateContainersOf(*user.ID, func(cid.ID) error) error + ContainersOf(context.Context, *user.ID) ([]cid.ID, error) + IterateContainersOf(context.Context, *user.ID, func(cid.ID) error) error } // Writer is an interface of container storage updater. @@ -133,7 +133,7 @@ func (s *morphExecutor) Delete(ctx context.Context, tokV2 *sessionV2.Token, body return new(container.DeleteResponseBody), nil } -func (s *morphExecutor) Get(_ context.Context, body *container.GetRequestBody) (*container.GetResponseBody, error) { +func (s *morphExecutor) Get(ctx context.Context, body *container.GetRequestBody) (*container.GetResponseBody, error) { idV2 := body.GetContainerID() if idV2 == nil { return nil, errors.New("missing container ID") @@ -146,7 +146,7 @@ func (s *morphExecutor) Get(_ context.Context, body *container.GetRequestBody) ( return nil, fmt.Errorf("invalid container ID: %w", err) } - cnr, err := s.rdr.Get(id) + cnr, err := s.rdr.Get(ctx, id) if err != nil { return nil, err } @@ -173,7 +173,7 @@ func (s *morphExecutor) Get(_ context.Context, body *container.GetRequestBody) ( return res, nil } -func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) (*container.ListResponseBody, error) { +func (s *morphExecutor) List(ctx context.Context, body *container.ListRequestBody) (*container.ListResponseBody, error) { idV2 := body.GetOwnerID() if idV2 == nil { return nil, errMissingUserID @@ -186,7 +186,7 @@ func (s *morphExecutor) List(_ context.Context, body *container.ListRequestBody) return nil, fmt.Errorf("invalid user ID: %w", err) } - cnrs, err := s.rdr.ContainersOf(&id) + cnrs, err := s.rdr.ContainersOf(ctx, &id) if err != nil { return nil, err } @@ -243,7 +243,7 @@ func (s *morphExecutor) ListStream(ctx context.Context, req *container.ListStrea return nil } - if err = s.rdr.IterateContainersOf(&id, processCID); err != nil { + if err = s.rdr.IterateContainersOf(ctx, &id, processCID); err != nil { return err } diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index e2c385c6a..0509d2646 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -48,7 +48,7 @@ func (s *Server) TickEpoch(ctx context.Context, req *control.TickEpochRequest) ( resp := new(control.TickEpochResponse) resp.SetBody(new(control.TickEpochResponse_Body)) - epoch, err := s.netmapClient.Epoch() + epoch, err := s.netmapClient.Epoch(ctx) if err != nil { return nil, fmt.Errorf("getting current epoch: %w", err) } @@ -77,7 +77,7 @@ func (s *Server) RemoveNode(ctx context.Context, req *control.RemoveNodeRequest) resp := new(control.RemoveNodeResponse) resp.SetBody(new(control.RemoveNodeResponse_Body)) - nm, err := s.netmapClient.NetMap() + nm, err := s.netmapClient.NetMap(ctx) if err != nil { return nil, fmt.Errorf("getting netmap: %w", err) } @@ -138,7 +138,7 @@ func (s *Server) RemoveContainer(ctx context.Context, req *control.RemoveContain return nil, status.Error(codes.InvalidArgument, "failed to read owner: "+err.Error()) } - cids, err := s.containerClient.ContainersOf(&owner) + cids, err := s.containerClient.ContainersOf(ctx, &owner) if err != nil { return nil, fmt.Errorf("failed to get owner's containers: %w", err) } diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index 6982d780d..bbf2cf0cc 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -1,6 +1,8 @@ package control import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/common" ) @@ -73,6 +75,7 @@ func SetNetmapStatus( // GetNetmapStatus executes ControlService.GetNetmapStatus RPC. func GetNetmapStatus( + _ context.Context, cli *client.Client, req *GetNetmapStatusRequest, opts ...client.CallOption, diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index da5401515..7469ea74e 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -157,7 +157,7 @@ func (s *Server) replicateObject(ctx context.Context, addr oid.Address, obj *obj return false, nil } - nodes, err := s.getContainerNodes(cid) + nodes, err := s.getContainerNodes(ctx, cid) if err != nil { return false, err } @@ -182,7 +182,7 @@ func (s *Server) replicateObject(ctx context.Context, addr oid.Address, obj *obj } func (s *Server) replicateTree(ctx context.Context, contID cid.ID, treeID string, forest pilorama.Forest) (bool, string, error) { - nodes, err := s.getContainerNodes(contID) + nodes, err := s.getContainerNodes(ctx, contID) if err != nil { return false, "", err } @@ -240,13 +240,13 @@ func (s *Server) replicateTreeToNode(ctx context.Context, forest pilorama.Forest } } -func (s *Server) getContainerNodes(contID cid.ID) ([]netmap.NodeInfo, error) { - nm, err := s.netMapSrc.GetNetMap(0) +func (s *Server) getContainerNodes(ctx context.Context, contID cid.ID) ([]netmap.NodeInfo, error) { + nm, err := s.netMapSrc.GetNetMap(ctx, 0) if err != nil { return nil, err } - c, err := s.cnrSrc.Get(contID) + c, err := s.cnrSrc.Get(ctx, contID) if err != nil { return nil, err } diff --git a/pkg/services/control/server/get_netmap_status.go b/pkg/services/control/server/get_netmap_status.go index 1c038253a..5e0496910 100644 --- a/pkg/services/control/server/get_netmap_status.go +++ b/pkg/services/control/server/get_netmap_status.go @@ -10,12 +10,12 @@ import ( ) // GetNetmapStatus gets node status in FrostFS network. -func (s *Server) GetNetmapStatus(_ context.Context, req *control.GetNetmapStatusRequest) (*control.GetNetmapStatusResponse, error) { +func (s *Server) GetNetmapStatus(ctx context.Context, req *control.GetNetmapStatusRequest) (*control.GetNetmapStatusResponse, error) { if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) } - st, epoch, err := s.nodeState.GetNetmapStatus() + st, epoch, err := s.nodeState.GetNetmapStatus(ctx) if err != nil { return nil, err } diff --git a/pkg/services/control/server/server.go b/pkg/services/control/server/server.go index 94aa1ff5b..59d701bc6 100644 --- a/pkg/services/control/server/server.go +++ b/pkg/services/control/server/server.go @@ -52,7 +52,7 @@ type NodeState interface { // but starts local maintenance regardless of the network settings. ForceMaintenance(ctx context.Context) error - GetNetmapStatus() (control.NetmapStatus, uint64, error) + GetNetmapStatus(ctx context.Context) (control.NetmapStatus, uint64, error) } // LocalOverrideStorageDecorator interface provides methods to decorate LocalOverrideEngine diff --git a/pkg/services/netmap/executor.go b/pkg/services/netmap/executor.go index 5223047df..44101a153 100644 --- a/pkg/services/netmap/executor.go +++ b/pkg/services/netmap/executor.go @@ -42,7 +42,7 @@ type NetworkInfo interface { // Dump must return recent network information in FrostFS API v2 NetworkInfo structure. // // If protocol version is <=2.9, MillisecondsPerBlock and network config should be unset. - Dump(versionsdk.Version) (*netmapSDK.NetworkInfo, error) + Dump(context.Context, versionsdk.Version) (*netmapSDK.NetworkInfo, error) } func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo, respSvc *response.Service) Server { @@ -82,7 +82,7 @@ func (s *executorSvc) LocalNodeInfo( } func (s *executorSvc) NetworkInfo( - _ context.Context, + ctx context.Context, req *netmap.NetworkInfoRequest, ) (*netmap.NetworkInfoResponse, error) { verV2 := req.GetMetaHeader().GetVersion() @@ -95,7 +95,7 @@ func (s *executorSvc) NetworkInfo( return nil, fmt.Errorf("can't read version: %w", err) } - ni, err := s.netInfo.Dump(ver) + ni, err := s.netInfo.Dump(ctx, ver) if err != nil { return nil, err } diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index db0f13ee7..86daec6cc 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -669,13 +669,13 @@ func (p patchStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.Pa } func (b Service) findRequestInfo(ctx context.Context, req MetaWithToken, idCnr cid.ID, op acl.Op) (info RequestInfo, err error) { - cnr, err := b.containers.Get(idCnr) // fetch actual container + cnr, err := b.containers.Get(ctx, idCnr) // fetch actual container if err != nil { return info, err } if req.token != nil { - currentEpoch, err := b.nm.Epoch() + currentEpoch, err := b.nm.Epoch(ctx) if err != nil { return info, errors.New("can't fetch current epoch") } @@ -727,13 +727,13 @@ func (b Service) findRequestInfo(ctx context.Context, req MetaWithToken, idCnr c // findRequestInfoWithoutACLOperationAssert is findRequestInfo without session token verb assert. func (b Service) findRequestInfoWithoutACLOperationAssert(ctx context.Context, req MetaWithToken, idCnr cid.ID) (info RequestInfo, err error) { - cnr, err := b.containers.Get(idCnr) // fetch actual container + cnr, err := b.containers.Get(ctx, idCnr) // fetch actual container if err != nil { return info, err } if req.token != nil { - currentEpoch, err := b.nm.Epoch() + currentEpoch, err := b.nm.Epoch(ctx) if err != nil { return info, errors.New("can't fetch current epoch") } diff --git a/pkg/services/object/acl/v2/types.go b/pkg/services/object/acl/v2/types.go index b03261b90..3cf10eb56 100644 --- a/pkg/services/object/acl/v2/types.go +++ b/pkg/services/object/acl/v2/types.go @@ -1,9 +1,11 @@ package v2 +import "context" + // InnerRingFetcher is an interface that must provide // Inner Ring information. type InnerRingFetcher interface { // InnerRingKeys must return list of public keys of // the actual inner ring. - InnerRingKeys() ([][]byte, error) + InnerRingKeys(ctx context.Context) ([][]byte, error) } diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index 4a3b5ba5e..ee46a6fe4 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -99,7 +99,7 @@ func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { return err } - return c.checkerCore.CheckAPE(checkercore.CheckPrm{ + return c.checkerCore.CheckAPE(ctx, checkercore.CheckPrm{ Request: r, PublicKey: pub, Namespace: prm.Namespace, diff --git a/pkg/services/object/ape/checker_test.go b/pkg/services/object/ape/checker_test.go index e03b5750c..97eb2b2d7 100644 --- a/pkg/services/object/ape/checker_test.go +++ b/pkg/services/object/ape/checker_test.go @@ -219,7 +219,7 @@ func scriptHashFromSenderKey(t *testing.T, senderKey string) util.Uint160 { return pk.GetScriptHash() } -func (f *frostfsIDProviderMock) GetSubject(key util.Uint160) (*client.Subject, error) { +func (f *frostfsIDProviderMock) GetSubject(ctx context.Context, key util.Uint160) (*client.Subject, error) { v, ok := f.subjects[key] if !ok { return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) @@ -227,7 +227,7 @@ func (f *frostfsIDProviderMock) GetSubject(key util.Uint160) (*client.Subject, e return v, nil } -func (f *frostfsIDProviderMock) GetSubjectExtended(key util.Uint160) (*client.SubjectExtended, error) { +func (f *frostfsIDProviderMock) GetSubjectExtended(ctx context.Context, key util.Uint160) (*client.SubjectExtended, error) { v, ok := f.subjectsExtended[key] if !ok { return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) @@ -619,21 +619,21 @@ type netmapStub struct { currentEpoch uint64 } -func (s *netmapStub) GetNetMap(diff uint64) (*netmapSDK.NetMap, error) { +func (s *netmapStub) GetNetMap(ctx context.Context, diff uint64) (*netmapSDK.NetMap, error) { if diff >= s.currentEpoch { return nil, errors.New("invalid diff") } - return s.GetNetMapByEpoch(s.currentEpoch - diff) + return s.GetNetMapByEpoch(ctx, s.currentEpoch-diff) } -func (s *netmapStub) GetNetMapByEpoch(epoch uint64) (*netmapSDK.NetMap, error) { +func (s *netmapStub) GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmapSDK.NetMap, error) { if nm, found := s.netmaps[epoch]; found { return nm, nil } return nil, errors.New("netmap not found") } -func (s *netmapStub) Epoch() (uint64, error) { +func (s *netmapStub) Epoch(ctx context.Context) (uint64, error) { return s.currentEpoch, nil } @@ -641,14 +641,14 @@ type testContainerSource struct { containers map[cid.ID]*container.Container } -func (s *testContainerSource) Get(cnrID cid.ID) (*container.Container, error) { +func (s *testContainerSource) Get(ctx context.Context, cnrID cid.ID) (*container.Container, error) { if cnr, found := s.containers[cnrID]; found { return cnr, nil } return nil, fmt.Errorf("container not found") } -func (s *testContainerSource) DeletionInfo(cid.ID) (*container.DelInfo, error) { +func (s *testContainerSource) DeletionInfo(context.Context, cid.ID) (*container.DelInfo, error) { return nil, nil } diff --git a/pkg/services/object/ape/request.go b/pkg/services/object/ape/request.go index cb9bbf1b8..001a5f71e 100644 --- a/pkg/services/object/ape/request.go +++ b/pkg/services/object/ape/request.go @@ -140,7 +140,7 @@ func (c *checkerImpl) newAPERequest(ctx context.Context, prm Prm) (aperequest.Re reqProps[xheadKey] = xhead.GetValue() } - reqProps, err = c.fillWithUserClaimTags(reqProps, prm) + reqProps, err = c.fillWithUserClaimTags(ctx, reqProps, prm) if err != nil { return defaultRequest, err } @@ -177,7 +177,7 @@ func (c *checkerImpl) fillHeaderWithECParent(ctx context.Context, prm Prm, heade return nil, fmt.Errorf("EC parent object ID format error: %w", err) } // only container node have access to collect parent object - contNode, err := c.currentNodeIsContainerNode(prm.Container) + contNode, err := c.currentNodeIsContainerNode(ctx, prm.Container) if err != nil { return nil, fmt.Errorf("check container node status: %w", err) } @@ -200,13 +200,13 @@ func isLogicalError(err error) bool { return errors.As(err, &errObjRemoved) || errors.As(err, &errObjNotFound) } -func (c *checkerImpl) currentNodeIsContainerNode(cnrID cid.ID) (bool, error) { - cnr, err := c.cnrSource.Get(cnrID) +func (c *checkerImpl) currentNodeIsContainerNode(ctx context.Context, cnrID cid.ID) (bool, error) { + cnr, err := c.cnrSource.Get(ctx, cnrID) if err != nil { return false, err } - nm, err := netmap.GetLatestNetworkMap(c.nm) + nm, err := netmap.GetLatestNetworkMap(ctx, c.nm) if err != nil { return false, err } @@ -220,7 +220,7 @@ func (c *checkerImpl) currentNodeIsContainerNode(cnrID cid.ID) (bool, error) { return true, nil } - nm, err = netmap.GetPreviousNetworkMap(c.nm) + nm, err = netmap.GetPreviousNetworkMap(ctx, c.nm) if err != nil { return false, err } @@ -229,7 +229,7 @@ func (c *checkerImpl) currentNodeIsContainerNode(cnrID cid.ID) (bool, error) { } // fillWithUserClaimTags fills ape request properties with user claim tags getting them from frostfsid contract by actor public key. -func (c *checkerImpl) fillWithUserClaimTags(reqProps map[string]string, prm Prm) (map[string]string, error) { +func (c *checkerImpl) fillWithUserClaimTags(ctx context.Context, reqProps map[string]string, prm Prm) (map[string]string, error) { if reqProps == nil { reqProps = make(map[string]string) } @@ -237,7 +237,7 @@ func (c *checkerImpl) fillWithUserClaimTags(reqProps map[string]string, prm Prm) if err != nil { return nil, err } - props, err := aperequest.FormFrostfsIDRequestProperties(c.frostFSIDClient, pk) + props, err := aperequest.FormFrostfsIDRequestProperties(ctx, c.frostFSIDClient, pk) if err != nil { return reqProps, err } diff --git a/pkg/services/object/common/target/target.go b/pkg/services/object/common/target/target.go index 9e0f49297..b2ae79dbc 100644 --- a/pkg/services/object/common/target/target.go +++ b/pkg/services/object/common/target/target.go @@ -1,6 +1,7 @@ package target import ( + "context" "errors" "fmt" @@ -13,20 +14,20 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" ) -func New(prm objectwriter.Params) (transformer.ChunkedObjectWriter, error) { +func New(ctx context.Context, prm objectwriter.Params) (transformer.ChunkedObjectWriter, error) { // prepare needed put parameters - if err := preparePrm(&prm); err != nil { + if err := preparePrm(ctx, &prm); err != nil { return nil, fmt.Errorf("could not prepare put parameters: %w", err) } if prm.Header.Signature() != nil { - return newUntrustedTarget(&prm) + return newUntrustedTarget(ctx, &prm) } - return newTrustedTarget(&prm) + return newTrustedTarget(ctx, &prm) } -func newUntrustedTarget(prm *objectwriter.Params) (transformer.ChunkedObjectWriter, error) { - maxPayloadSz := prm.Config.MaxSizeSrc.MaxObjectSize() +func newUntrustedTarget(ctx context.Context, prm *objectwriter.Params) (transformer.ChunkedObjectWriter, error) { + maxPayloadSz := prm.Config.MaxSizeSrc.MaxObjectSize(ctx) if maxPayloadSz == 0 { return nil, errors.New("could not obtain max object size parameter") } @@ -48,9 +49,9 @@ func newUntrustedTarget(prm *objectwriter.Params) (transformer.ChunkedObjectWrit }, nil } -func newTrustedTarget(prm *objectwriter.Params) (transformer.ChunkedObjectWriter, error) { +func newTrustedTarget(ctx context.Context, prm *objectwriter.Params) (transformer.ChunkedObjectWriter, error) { prm.Relay = nil // do not relay request without signature - maxPayloadSz := prm.Config.MaxSizeSrc.MaxObjectSize() + maxPayloadSz := prm.Config.MaxSizeSrc.MaxObjectSize(ctx) if maxPayloadSz == 0 { return nil, errors.New("could not obtain max object size parameter") } @@ -111,11 +112,11 @@ func newTrustedTarget(prm *objectwriter.Params) (transformer.ChunkedObjectWriter }, nil } -func preparePrm(prm *objectwriter.Params) error { +func preparePrm(ctx context.Context, prm *objectwriter.Params) error { var err error // get latest network map - nm, err := netmap.GetLatestNetworkMap(prm.Config.NetmapSource) + nm, err := netmap.GetLatestNetworkMap(ctx, prm.Config.NetmapSource) if err != nil { return fmt.Errorf("could not get latest network map: %w", err) } @@ -126,7 +127,7 @@ func preparePrm(prm *objectwriter.Params) error { } // get container to store the object - cnrInfo, err := prm.Config.ContainerSource.Get(idCnr) + cnrInfo, err := prm.Config.ContainerSource.Get(ctx, idCnr) if err != nil { return fmt.Errorf("could not get container by ID: %w", err) } diff --git a/pkg/services/object/common/writer/common.go b/pkg/services/object/common/writer/common.go index 3b68efab4..dae168baf 100644 --- a/pkg/services/object/common/writer/common.go +++ b/pkg/services/object/common/writer/common.go @@ -29,7 +29,7 @@ func (c *Config) NewNodeIterator(opts []placement.Option) *NodeIterator { } func (n *NodeIterator) ForEachNode(ctx context.Context, f func(context.Context, NodeDescriptor) error) error { - traverser, err := placement.NewTraverser(n.Traversal.Opts...) + traverser, err := placement.NewTraverser(ctx, n.Traversal.Opts...) if err != nil { return fmt.Errorf("could not create object placement traverser: %w", err) } diff --git a/pkg/services/object/common/writer/ec.go b/pkg/services/object/common/writer/ec.go index 94bcf6a32..8f269ec21 100644 --- a/pkg/services/object/common/writer/ec.go +++ b/pkg/services/object/common/writer/ec.go @@ -85,7 +85,7 @@ func (e *ECWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error } func (e *ECWriter) relayIfNotContainerNode(ctx context.Context, obj *objectSDK.Object) (bool, bool, error) { - currentNodeIsContainerNode, err := e.currentNodeIsContainerNode() + currentNodeIsContainerNode, err := e.currentNodeIsContainerNode(ctx) if err != nil { return false, false, err } @@ -108,8 +108,8 @@ func (e *ECWriter) relayIfNotContainerNode(ctx context.Context, obj *objectSDK.O return true, currentNodeIsContainerNode, nil } -func (e *ECWriter) currentNodeIsContainerNode() (bool, error) { - t, err := placement.NewTraverser(e.PlacementOpts...) +func (e *ECWriter) currentNodeIsContainerNode(ctx context.Context) (bool, error) { + t, err := placement.NewTraverser(ctx, e.PlacementOpts...) if err != nil { return false, err } @@ -128,7 +128,7 @@ func (e *ECWriter) currentNodeIsContainerNode() (bool, error) { } func (e *ECWriter) relayToContainerNode(ctx context.Context, objID oid.ID, index uint32) error { - t, err := placement.NewTraverser(append(e.PlacementOpts, placement.ForObject(objID))...) + t, err := placement.NewTraverser(ctx, append(e.PlacementOpts, placement.ForObject(objID))...) if err != nil { return err } @@ -180,7 +180,7 @@ func (e *ECWriter) writeECPart(ctx context.Context, obj *objectSDK.Object) error return e.writePartLocal(ctx, obj) } - t, err := placement.NewTraverser(append(e.PlacementOpts, placement.ForObject(obj.ECHeader().Parent()))...) + t, err := placement.NewTraverser(ctx, append(e.PlacementOpts, placement.ForObject(obj.ECHeader().Parent()))...) if err != nil { return err } @@ -217,7 +217,7 @@ func (e *ECWriter) writeRawObject(ctx context.Context, obj *objectSDK.Object) er } partsProcessed := make([]atomic.Bool, len(parts)) objID, _ := obj.ID() - t, err := placement.NewTraverser(append(e.PlacementOpts, placement.ForObject(objID))...) + t, err := placement.NewTraverser(ctx, append(e.PlacementOpts, placement.ForObject(objID))...) if err != nil { return err } diff --git a/pkg/services/object/common/writer/ec_test.go b/pkg/services/object/common/writer/ec_test.go index 8ad7e641a..b7764661f 100644 --- a/pkg/services/object/common/writer/ec_test.go +++ b/pkg/services/object/common/writer/ec_test.go @@ -39,7 +39,7 @@ type testPlacementBuilder struct { vectors [][]netmap.NodeInfo } -func (p *testPlacementBuilder) BuildPlacement(_ cid.ID, _ *oid.ID, _ netmap.PlacementPolicy) ( +func (p *testPlacementBuilder) BuildPlacement(ctx context.Context, _ cid.ID, _ *oid.ID, _ netmap.PlacementPolicy) ( [][]netmap.NodeInfo, error, ) { arr := slices.Clone(p.vectors[0]) diff --git a/pkg/services/object/common/writer/writer.go b/pkg/services/object/common/writer/writer.go index 0e4c4d9c6..adaf1945b 100644 --- a/pkg/services/object/common/writer/writer.go +++ b/pkg/services/object/common/writer/writer.go @@ -24,7 +24,7 @@ type MaxSizeSource interface { // of physically stored object in system. // // Must return 0 if value can not be obtained. - MaxObjectSize() uint64 + MaxObjectSize(context.Context) uint64 } type ClientConstructor interface { @@ -32,7 +32,7 @@ type ClientConstructor interface { } type InnerRing interface { - InnerRingKeys() ([][]byte, error) + InnerRingKeys(ctx context.Context) ([][]byte, error) } type FormatValidatorConfig interface { diff --git a/pkg/services/object/get/assemblerec.go b/pkg/services/object/get/assemblerec.go index a53299480..e0a7e1da6 100644 --- a/pkg/services/object/get/assemblerec.go +++ b/pkg/services/object/get/assemblerec.go @@ -125,7 +125,7 @@ func (a *assemblerec) reconstructObject(ctx context.Context, writer ObjectWriter func (a *assemblerec) reconstructObjectFromParts(ctx context.Context, headers bool) (*objectSDK.Object, error) { objID := a.addr.Object() - trav, cnr, err := a.traverserGenerator.GenerateTraverser(a.addr.Container(), &objID, a.epoch) + trav, cnr, err := a.traverserGenerator.GenerateTraverser(ctx, a.addr.Container(), &objID, a.epoch) if err != nil { return nil, err } diff --git a/pkg/services/object/get/get_test.go b/pkg/services/object/get/get_test.go index 6827018dc..3efc72065 100644 --- a/pkg/services/object/get/get_test.go +++ b/pkg/services/object/get/get_test.go @@ -63,7 +63,7 @@ type testClient struct { type testEpochReceiver uint64 -func (e testEpochReceiver) Epoch() (uint64, error) { +func (e testEpochReceiver) Epoch(ctx context.Context) (uint64, error) { return uint64(e), nil } @@ -79,7 +79,7 @@ func newTestStorage() *testStorage { } } -func (g *testTraverserGenerator) GenerateTraverser(cnr cid.ID, obj *oid.ID, e uint64) (*placement.Traverser, *containerCore.Container, error) { +func (g *testTraverserGenerator) GenerateTraverser(ctx context.Context, cnr cid.ID, obj *oid.ID, e uint64) (*placement.Traverser, *containerCore.Container, error) { opts := make([]placement.Option, 0, 4) opts = append(opts, placement.ForContainer(g.c), @@ -91,13 +91,13 @@ func (g *testTraverserGenerator) GenerateTraverser(cnr cid.ID, obj *oid.ID, e ui opts = append(opts, placement.ForObject(*obj)) } - t, err := placement.NewTraverser(opts...) + t, err := placement.NewTraverser(context.Background(), opts...) return t, &containerCore.Container{ Value: g.c, }, err } -func (p *testPlacementBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, _ netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { +func (p *testPlacementBuilder) BuildPlacement(ctx context.Context, cnr cid.ID, obj *oid.ID, _ netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { var addr oid.Address addr.SetContainer(cnr) diff --git a/pkg/services/object/get/getrangeec_test.go b/pkg/services/object/get/getrangeec_test.go index 599a6f176..83ef54744 100644 --- a/pkg/services/object/get/getrangeec_test.go +++ b/pkg/services/object/get/getrangeec_test.go @@ -28,14 +28,14 @@ type containerStorage struct { cnt *container.Container } -func (cs *containerStorage) Get(cid.ID) (*coreContainer.Container, error) { +func (cs *containerStorage) Get(context.Context, cid.ID) (*coreContainer.Container, error) { coreCnt := coreContainer.Container{ Value: *cs.cnt, } return &coreCnt, nil } -func (cs *containerStorage) DeletionInfo(cid.ID) (*coreContainer.DelInfo, error) { +func (cs *containerStorage) DeletionInfo(context.Context, cid.ID) (*coreContainer.DelInfo, error) { return nil, nil } diff --git a/pkg/services/object/get/remote_getter.go b/pkg/services/object/get/remote_getter.go index 0df67dec9..2c64244cf 100644 --- a/pkg/services/object/get/remote_getter.go +++ b/pkg/services/object/get/remote_getter.go @@ -30,7 +30,7 @@ func (g *RemoteGetter) Get(ctx context.Context, prm RemoteGetPrm) (*objectSDK.Ob if err != nil { return nil, err } - epoch, err := g.es.Epoch() + epoch, err := g.es.Epoch(ctx) if err != nil { return nil, err } diff --git a/pkg/services/object/get/request.go b/pkg/services/object/get/request.go index be0950c60..268080486 100644 --- a/pkg/services/object/get/request.go +++ b/pkg/services/object/get/request.go @@ -122,7 +122,7 @@ func (r *request) initEpoch(ctx context.Context) bool { return true } - e, err := r.epochSource.Epoch() + e, err := r.epochSource.Epoch(ctx) switch { default: @@ -141,7 +141,7 @@ func (r *request) initEpoch(ctx context.Context) bool { func (r *request) generateTraverser(ctx context.Context, addr oid.Address) (*placement.Traverser, bool) { obj := addr.Object() - t, _, err := r.traverserGenerator.GenerateTraverser(addr.Container(), &obj, r.curProcEpoch) + t, _, err := r.traverserGenerator.GenerateTraverser(ctx, addr.Container(), &obj, r.curProcEpoch) switch { default: diff --git a/pkg/services/object/get/types.go b/pkg/services/object/get/types.go index 9669afdba..664366d1b 100644 --- a/pkg/services/object/get/types.go +++ b/pkg/services/object/get/types.go @@ -20,11 +20,11 @@ import ( ) type epochSource interface { - Epoch() (uint64, error) + Epoch(ctx context.Context) (uint64, error) } type traverserGenerator interface { - GenerateTraverser(cid.ID, *oid.ID, uint64) (*placement.Traverser, *container.Container, error) + GenerateTraverser(context.Context, cid.ID, *oid.ID, uint64) (*placement.Traverser, *container.Container, error) } type keyStorage interface { diff --git a/pkg/services/object/get/v2/get_range_hash.go b/pkg/services/object/get/v2/get_range_hash.go index 7d26a38c3..308ccd512 100644 --- a/pkg/services/object/get/v2/get_range_hash.go +++ b/pkg/services/object/get/v2/get_range_hash.go @@ -22,7 +22,7 @@ import ( // GetRangeHash calls internal service and returns v2 response. func (s *Service) GetRangeHash(ctx context.Context, req *objectV2.GetRangeHashRequest) (*objectV2.GetRangeHashResponse, error) { - forward, err := s.needToForwardGetRangeHashRequest(req) + forward, err := s.needToForwardGetRangeHashRequest(ctx, req) if err != nil { return nil, err } @@ -48,7 +48,7 @@ type getRangeForwardParams struct { address oid.Address } -func (s *Service) needToForwardGetRangeHashRequest(req *objectV2.GetRangeHashRequest) (getRangeForwardParams, error) { +func (s *Service) needToForwardGetRangeHashRequest(ctx context.Context, req *objectV2.GetRangeHashRequest) (getRangeForwardParams, error) { if req.GetMetaHeader().GetTTL() <= 1 { return getRangeForwardParams{}, nil } @@ -66,17 +66,17 @@ func (s *Service) needToForwardGetRangeHashRequest(req *objectV2.GetRangeHashReq } result.address = addr - cont, err := s.contSource.Get(addr.Container()) + cont, err := s.contSource.Get(ctx, addr.Container()) if err != nil { return result, fmt.Errorf("(%T) could not get container: %w", s, err) } - epoch, err := s.netmapSource.Epoch() + epoch, err := s.netmapSource.Epoch(ctx) if err != nil { return result, fmt.Errorf("(%T) could not get epoch: %w", s, err) } - nm, err := s.netmapSource.GetNetMapByEpoch(epoch) + nm, err := s.netmapSource.GetNetMapByEpoch(ctx, epoch) if err != nil { return result, fmt.Errorf("(%T) could not get netmap: %w", s, err) } @@ -84,7 +84,7 @@ func (s *Service) needToForwardGetRangeHashRequest(req *objectV2.GetRangeHashReq builder := placement.NewNetworkMapBuilder(nm) objectID := addr.Object() - nodesVector, err := builder.BuildPlacement(addr.Container(), &objectID, cont.Value.PlacementPolicy()) + nodesVector, err := builder.BuildPlacement(ctx, addr.Container(), &objectID, cont.Value.PlacementPolicy()) if err != nil { return result, fmt.Errorf("(%T) could not build object placement: %w", s, err) } diff --git a/pkg/services/object/patch/streamer.go b/pkg/services/object/patch/streamer.go index 91b4efdc1..5aba13f66 100644 --- a/pkg/services/object/patch/streamer.go +++ b/pkg/services/object/patch/streamer.go @@ -112,7 +112,7 @@ func (s *Streamer) init(ctx context.Context, req *objectV2.PatchRequest) error { } oV2.GetHeader().SetOwnerID(ownerID) - target, err := target.New(objectwriter.Params{ + target, err := target.New(ctx, objectwriter.Params{ Config: s.Config, Common: commonPrm, Header: objectSDK.NewFromV2(oV2), diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index fec50b1d9..f3c2dca1a 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -86,7 +86,7 @@ func (s *Service) PutSingle(ctx context.Context, req *objectAPI.PutSingleRequest } func (s *Service) validatePutSingle(ctx context.Context, obj *objectSDK.Object) (object.ContentMeta, error) { - if err := s.validarePutSingleSize(obj); err != nil { + if err := s.validarePutSingleSize(ctx, obj); err != nil { return object.ContentMeta{}, err } @@ -97,12 +97,12 @@ func (s *Service) validatePutSingle(ctx context.Context, obj *objectSDK.Object) return s.validatePutSingleObject(ctx, obj) } -func (s *Service) validarePutSingleSize(obj *objectSDK.Object) error { +func (s *Service) validarePutSingleSize(ctx context.Context, obj *objectSDK.Object) error { if uint64(len(obj.Payload())) != obj.PayloadSize() { return target.ErrWrongPayloadSize } - maxAllowedSize := s.Config.MaxSizeSrc.MaxObjectSize() + maxAllowedSize := s.Config.MaxSizeSrc.MaxObjectSize(ctx) if obj.PayloadSize() > maxAllowedSize { return target.ErrExceedingMaxSize } @@ -153,7 +153,7 @@ func (s *Service) validatePutSingleObject(ctx context.Context, obj *objectSDK.Ob func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { localOnly := req.GetMetaHeader().GetTTL() <= 1 - placement, err := s.getPutSinglePlacementOptions(obj, req.GetBody().GetCopiesNumber(), localOnly) + placement, err := s.getPutSinglePlacementOptions(ctx, obj, req.GetBody().GetCopiesNumber(), localOnly) if err != nil { return err } @@ -218,14 +218,14 @@ type putSinglePlacement struct { resetSuccessAfterOnBroadcast bool } -func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) (putSinglePlacement, error) { +func (s *Service) getPutSinglePlacementOptions(ctx context.Context, obj *objectSDK.Object, copiesNumber []uint32, localOnly bool) (putSinglePlacement, error) { var result putSinglePlacement cnrID, ok := obj.ContainerID() if !ok { return result, errors.New("missing container ID") } - cnrInfo, err := s.Config.ContainerSource.Get(cnrID) + cnrInfo, err := s.Config.ContainerSource.Get(ctx, cnrID) if err != nil { return result, fmt.Errorf("could not get container by ID: %w", err) } @@ -249,7 +249,7 @@ func (s *Service) getPutSinglePlacementOptions(obj *objectSDK.Object, copiesNumb } result.placementOptions = append(result.placementOptions, placement.ForObject(objID)) - latestNetmap, err := netmap.GetLatestNetworkMap(s.Config.NetmapSource) + latestNetmap, err := netmap.GetLatestNetworkMap(ctx, s.Config.NetmapSource) if err != nil { return result, fmt.Errorf("could not get latest network map: %w", err) } diff --git a/pkg/services/object/put/streamer.go b/pkg/services/object/put/streamer.go index f71309d31..19768b7fa 100644 --- a/pkg/services/object/put/streamer.go +++ b/pkg/services/object/put/streamer.go @@ -36,7 +36,7 @@ func (p *Streamer) Init(ctx context.Context, prm *PutInitPrm) error { } var err error - p.target, err = target.New(prmTarget) + p.target, err = target.New(ctx, prmTarget) if err != nil { return fmt.Errorf("(%T) could not initialize object target: %w", p, err) } diff --git a/pkg/services/object/put/v2/streamer.go b/pkg/services/object/put/v2/streamer.go index 1cd10cd7f..f0c648187 100644 --- a/pkg/services/object/put/v2/streamer.go +++ b/pkg/services/object/put/v2/streamer.go @@ -56,7 +56,7 @@ func (s *streamer) Send(ctx context.Context, req *object.PutRequest) (err error) s.saveChunks = v.GetSignature() != nil if s.saveChunks { - maxSz := s.stream.MaxSizeSrc.MaxObjectSize() + maxSz := s.stream.MaxSizeSrc.MaxObjectSize(ctx) s.sizes = &sizes{ payloadSz: v.GetHeader().GetPayloadLength(), diff --git a/pkg/services/object/search/container.go b/pkg/services/object/search/container.go index e82f999cf..60d469b11 100644 --- a/pkg/services/object/search/container.go +++ b/pkg/services/object/search/container.go @@ -20,7 +20,7 @@ func (exec *execCtx) executeOnContainer(ctx context.Context) error { ) // initialize epoch number - if err := exec.initEpoch(); err != nil { + if err := exec.initEpoch(ctx); err != nil { return fmt.Errorf("%s: %w", logs.CouldNotGetCurrentEpochNumber, err) } @@ -48,7 +48,7 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { zap.Uint64("number", exec.curProcEpoch), ) - traverser, _, err := exec.svc.traverserGenerator.GenerateTraverser(exec.containerID(), nil, exec.curProcEpoch) + traverser, _, err := exec.svc.traverserGenerator.GenerateTraverser(ctx, exec.containerID(), nil, exec.curProcEpoch) if err != nil { return fmt.Errorf("%s: %w", logs.SearchCouldNotGenerateContainerTraverser, err) } @@ -114,9 +114,9 @@ func (exec *execCtx) processCurrentEpoch(ctx context.Context) error { return nil } -func (exec *execCtx) getContainer() (containerSDK.Container, error) { +func (exec *execCtx) getContainer(ctx context.Context) (containerSDK.Container, error) { cnrID := exec.containerID() - cnr, err := exec.svc.containerSource.Get(cnrID) + cnr, err := exec.svc.containerSource.Get(ctx, cnrID) if err != nil { return containerSDK.Container{}, err } diff --git a/pkg/services/object/search/exec.go b/pkg/services/object/search/exec.go index eb9635f14..ced51ecce 100644 --- a/pkg/services/object/search/exec.go +++ b/pkg/services/object/search/exec.go @@ -1,6 +1,8 @@ package searchsvc import ( + "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -48,13 +50,13 @@ func (exec *execCtx) netmapLookupDepth() uint64 { return exec.prm.common.NetmapLookupDepth() } -func (exec *execCtx) initEpoch() error { +func (exec *execCtx) initEpoch(ctx context.Context) error { exec.curProcEpoch = exec.netmapEpoch() if exec.curProcEpoch > 0 { return nil } - e, err := exec.svc.currentEpochReceiver.Epoch() + e, err := exec.svc.currentEpochReceiver.Epoch(ctx) if err != nil { return err } diff --git a/pkg/services/object/search/search_test.go b/pkg/services/object/search/search_test.go index 05643eb2b..918ad421f 100644 --- a/pkg/services/object/search/search_test.go +++ b/pkg/services/object/search/search_test.go @@ -59,7 +59,7 @@ type simpleIDWriter struct { type testEpochReceiver uint64 -func (e testEpochReceiver) Epoch() (uint64, error) { +func (e testEpochReceiver) Epoch(ctx context.Context) (uint64, error) { return uint64(e), nil } @@ -82,8 +82,8 @@ func newTestStorage() *testStorage { } } -func (g *testTraverserGenerator) GenerateTraverser(_ cid.ID, _ *oid.ID, epoch uint64) (*placement.Traverser, *containerCore.Container, error) { - t, err := placement.NewTraverser( +func (g *testTraverserGenerator) GenerateTraverser(ctx context.Context, _ cid.ID, _ *oid.ID, epoch uint64) (*placement.Traverser, *containerCore.Container, error) { + t, err := placement.NewTraverser(context.Background(), placement.ForContainer(g.c), placement.UseBuilder(g.b[epoch]), placement.WithoutSuccessTracking(), @@ -91,7 +91,7 @@ func (g *testTraverserGenerator) GenerateTraverser(_ cid.ID, _ *oid.ID, epoch ui return t, &containerCore.Container{Value: g.c}, err } -func (p *testPlacementBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, _ netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { +func (p *testPlacementBuilder) BuildPlacement(ctx context.Context, cnr cid.ID, obj *oid.ID, _ netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { var addr oid.Address addr.SetContainer(cnr) diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index 77d25357a..e1aeca957 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -46,11 +46,11 @@ type cfg struct { } traverserGenerator interface { - GenerateTraverser(cid.ID, *oid.ID, uint64) (*placement.Traverser, *container.Container, error) + GenerateTraverser(context.Context, cid.ID, *oid.ID, uint64) (*placement.Traverser, *container.Container, error) } currentEpochReceiver interface { - Epoch() (uint64, error) + Epoch(ctx context.Context) (uint64, error) } keyStore *util.KeyStorage diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index 910384a0b..fed168187 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -113,7 +113,7 @@ func (c *clientWrapper) searchObjects(ctx context.Context, exec *execCtx, info c } func (e *storageEngineWrapper) search(ctx context.Context, exec *execCtx) ([]oid.ID, error) { - cnr, err := exec.getContainer() + cnr, err := exec.getContainer(ctx) if err != nil { return nil, err } diff --git a/pkg/services/object/util/placement.go b/pkg/services/object/util/placement.go index 1bd39f9ea..195944f92 100644 --- a/pkg/services/object/util/placement.go +++ b/pkg/services/object/util/placement.go @@ -1,6 +1,7 @@ package util import ( + "context" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" @@ -43,8 +44,8 @@ func NewLocalPlacement(b placement.Builder, s netmap.AnnouncedKeys) placement.Bu } } -func (p *localPlacement) BuildPlacement(cnr cid.ID, obj *oid.ID, policy netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) { - vs, err := p.builder.BuildPlacement(cnr, obj, policy) +func (p *localPlacement) BuildPlacement(ctx context.Context, cnr cid.ID, obj *oid.ID, policy netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) { + vs, err := p.builder.BuildPlacement(ctx, cnr, obj, policy) if err != nil { return nil, fmt.Errorf("(%T) could not build object placement: %w", p, err) } @@ -76,8 +77,8 @@ func NewRemotePlacementBuilder(b placement.Builder, s netmap.AnnouncedKeys) plac } } -func (p *remotePlacement) BuildPlacement(cnr cid.ID, obj *oid.ID, policy netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) { - vs, err := p.builder.BuildPlacement(cnr, obj, policy) +func (p *remotePlacement) BuildPlacement(ctx context.Context, cnr cid.ID, obj *oid.ID, policy netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) { + vs, err := p.builder.BuildPlacement(ctx, cnr, obj, policy) if err != nil { return nil, fmt.Errorf("(%T) could not build object placement: %w", p, err) } @@ -122,15 +123,15 @@ func (g *TraverserGenerator) WithTraverseOptions(opts ...placement.Option) *Trav // GenerateTraverser generates placement Traverser for provided object address // using epoch-th network map. -func (g *TraverserGenerator) GenerateTraverser(idCnr cid.ID, idObj *oid.ID, epoch uint64) (*placement.Traverser, *container.Container, error) { +func (g *TraverserGenerator) GenerateTraverser(ctx context.Context, idCnr cid.ID, idObj *oid.ID, epoch uint64) (*placement.Traverser, *container.Container, error) { // get network map by epoch - nm, err := g.netMapSrc.GetNetMapByEpoch(epoch) + nm, err := g.netMapSrc.GetNetMapByEpoch(ctx, epoch) if err != nil { return nil, nil, fmt.Errorf("could not get network map #%d: %w", epoch, err) } // get container related container - cnr, err := g.cnrSrc.Get(idCnr) + cnr, err := g.cnrSrc.Get(ctx, idCnr) if err != nil { return nil, nil, fmt.Errorf("could not get container: %w", err) } @@ -160,7 +161,7 @@ func (g *TraverserGenerator) GenerateTraverser(idCnr cid.ID, idObj *oid.ID, epoc ) } - t, err := placement.NewTraverser(traverseOpts...) + t, err := placement.NewTraverser(ctx, traverseOpts...) if err != nil { return nil, nil, err } diff --git a/pkg/services/object_manager/placement/netmap.go b/pkg/services/object_manager/placement/netmap.go index 1782e27ea..b3f8d9c03 100644 --- a/pkg/services/object_manager/placement/netmap.go +++ b/pkg/services/object_manager/placement/netmap.go @@ -1,6 +1,7 @@ package placement import ( + "context" "crypto/sha256" "fmt" @@ -35,12 +36,12 @@ func NewNetworkMapSourceBuilder(nmSrc netmap.Source) Builder { } } -func (s *netMapSrc) GetNetMap(_ uint64) (*netmapSDK.NetMap, error) { +func (s *netMapSrc) GetNetMap(_ context.Context, _ uint64) (*netmapSDK.NetMap, error) { return s.nm, nil } -func (b *netMapBuilder) BuildPlacement(cnr cid.ID, obj *oid.ID, p netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) { - nm, err := netmap.GetLatestNetworkMap(b.nmSrc) +func (b *netMapBuilder) BuildPlacement(ctx context.Context, cnr cid.ID, obj *oid.ID, p netmapSDK.PlacementPolicy) ([][]netmapSDK.NodeInfo, error) { + nm, err := netmap.GetLatestNetworkMap(ctx, b.nmSrc) if err != nil { return nil, fmt.Errorf("could not get network map: %w", err) } diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index 8daf38217..efa4a5b06 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -1,6 +1,7 @@ package placement import ( + "context" "errors" "fmt" "slices" @@ -21,7 +22,7 @@ type Builder interface { // // Must return all container nodes if object identifier // is nil. - BuildPlacement(cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) + BuildPlacement(context.Context, cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) } type NodeState interface { @@ -78,7 +79,7 @@ func defaultCfg() *cfg { } // NewTraverser creates, initializes with options and returns Traverser instance. -func NewTraverser(opts ...Option) (*Traverser, error) { +func NewTraverser(ctx context.Context, opts ...Option) (*Traverser, error) { cfg := defaultCfg() for i := range opts { @@ -98,7 +99,7 @@ func NewTraverser(opts ...Option) (*Traverser, error) { return nil, fmt.Errorf("%s: %w", invalidOptsMsg, errNilPolicy) } - ns, err := cfg.builder.BuildPlacement(cfg.cnr, cfg.obj, cfg.policy) + ns, err := cfg.builder.BuildPlacement(ctx, cfg.cnr, cfg.obj, cfg.policy) if err != nil { return nil, fmt.Errorf("could not build placement: %w", err) } diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index 624efb007..9c825bf19 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -1,6 +1,7 @@ package placement import ( + "context" "slices" "strconv" "testing" @@ -18,7 +19,7 @@ type testBuilder struct { vectors [][]netmap.NodeInfo } -func (b testBuilder) BuildPlacement(cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { +func (b testBuilder) BuildPlacement(context.Context, cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { return b.vectors, nil } @@ -102,7 +103,7 @@ func TestTraverserObjectScenarios(t *testing.T) { nodesCopy := copyVectors(nodes) - tr, err := NewTraverser( + tr, err := NewTraverser(context.Background(), ForContainer(cnr), UseBuilder(&testBuilder{vectors: nodesCopy}), WithoutSuccessTracking(), @@ -131,7 +132,7 @@ func TestTraverserObjectScenarios(t *testing.T) { nodesCopy := copyVectors(nodes) - tr, err := NewTraverser( + tr, err := NewTraverser(context.Background(), ForContainer(cnr), UseBuilder(&testBuilder{ vectors: nodesCopy, @@ -160,7 +161,7 @@ func TestTraverserObjectScenarios(t *testing.T) { nodesCopy := copyVectors(nodes) - tr, err := NewTraverser( + tr, err := NewTraverser(context.Background(), ForContainer(cnr), UseBuilder(&testBuilder{vectors: nodesCopy}), ) @@ -201,7 +202,7 @@ func TestTraverserObjectScenarios(t *testing.T) { nodes, cnr := testPlacement(selectors, replicas) - tr, err := NewTraverser( + tr, err := NewTraverser(context.Background(), ForContainer(cnr), UseBuilder(&testBuilder{ vectors: [][]netmap.NodeInfo{{nodes[1][1]}}, // single node (local) @@ -276,7 +277,7 @@ func TestTraverserRemValues(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - tr, err := NewTraverser( + tr, err := NewTraverser(context.Background(), ForContainer(cnr), UseBuilder(&testBuilder{vectors: nodesCopy}), WithCopyNumbers(testCase.copyNumbers), @@ -322,7 +323,7 @@ func TestTraverserPriorityMetrics(t *testing.T) { m := []Metric{NewAttributeMetric("ClusterName")} - tr, err := NewTraverser( + tr, err := NewTraverser(context.Background(), ForContainer(cnr), UseBuilder(&testBuilder{ vectors: nodesCopy, @@ -374,7 +375,7 @@ func TestTraverserPriorityMetrics(t *testing.T) { m := []Metric{NewAttributeMetric("ClusterName")} - tr, err := NewTraverser( + tr, err := NewTraverser(context.Background(), ForContainer(cnr), UseBuilder(&testBuilder{ vectors: nodesCopy, @@ -445,7 +446,7 @@ func TestTraverserPriorityMetrics(t *testing.T) { NewAttributeMetric("UN-LOCODE"), } - tr, err := NewTraverser( + tr, err := NewTraverser(context.Background(), ForContainer(cnr), UseBuilder(&testBuilder{ vectors: nodesCopy, @@ -483,7 +484,7 @@ func TestTraverserPriorityMetrics(t *testing.T) { nodesCopy = copyVectors(nodes) - tr, err = NewTraverser( + tr, err = NewTraverser(context.Background(), ForContainer(cnr), UseBuilder(&testBuilder{ vectors: nodesCopy, @@ -516,7 +517,7 @@ func TestTraverserPriorityMetrics(t *testing.T) { nodesCopy = copyVectors(nodes) - tr, err = NewTraverser( + tr, err = NewTraverser(context.Background(), ForContainer(cnr), UseBuilder(&testBuilder{ vectors: nodesCopy, @@ -567,7 +568,7 @@ func TestTraverserPriorityMetrics(t *testing.T) { m := []Metric{NewAttributeMetric("ClusterName")} - tr, err := NewTraverser( + tr, err := NewTraverser(context.Background(), ForContainer(cnr), UseBuilder(&testBuilder{ vectors: nodesCopy, diff --git a/pkg/services/policer/check.go b/pkg/services/policer/check.go index 7df372476..dcaaec0b4 100644 --- a/pkg/services/policer/check.go +++ b/pkg/services/policer/check.go @@ -28,10 +28,10 @@ func (p *Policer) processObject(ctx context.Context, objInfo objectcore.Info) er )) defer span.End() - cnr, err := p.cnrSrc.Get(objInfo.Address.Container()) + cnr, err := p.cnrSrc.Get(ctx, objInfo.Address.Container()) if err != nil { if client.IsErrContainerNotFound(err) { - existed, errWasRemoved := containercore.WasRemoved(p.cnrSrc, objInfo.Address.Container()) + existed, errWasRemoved := containercore.WasRemoved(ctx, p.cnrSrc, objInfo.Address.Container()) if errWasRemoved != nil { return fmt.Errorf("%s: %w", logs.PolicerCouldNotConfirmContainerRemoval, errWasRemoved) } else if existed { @@ -56,7 +56,7 @@ func (p *Policer) processObject(ctx context.Context, objInfo objectcore.Info) er func (p *Policer) processRepContainerObject(ctx context.Context, objInfo objectcore.Info, policy netmap.PlacementPolicy) error { idObj := objInfo.Address.Object() idCnr := objInfo.Address.Container() - nn, err := p.placementBuilder.BuildPlacement(idCnr, &idObj, policy) + nn, err := p.placementBuilder.BuildPlacement(ctx, idCnr, &idObj, policy) if err != nil { return fmt.Errorf("%s: %w", logs.PolicerCouldNotBuildPlacementVectorForObject, err) } diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index f6d3b9ea1..1ee31d480 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -39,7 +39,7 @@ func (p *Policer) processECContainerObject(ctx context.Context, objInfo objectco // All of them must be stored on all of the container nodes. func (p *Policer) processECContainerRepObject(ctx context.Context, objInfo objectcore.Info, policy netmap.PlacementPolicy) error { objID := objInfo.Address.Object() - nn, err := p.placementBuilder.BuildPlacement(objInfo.Address.Container(), &objID, policy) + nn, err := p.placementBuilder.BuildPlacement(ctx, objInfo.Address.Container(), &objID, policy) if err != nil { return fmt.Errorf("%s: %w", logs.PolicerCouldNotBuildPlacementVectorForObject, err) } @@ -69,7 +69,7 @@ func (p *Policer) processECContainerRepObject(ctx context.Context, objInfo objec } func (p *Policer) processECContainerECObject(ctx context.Context, objInfo objectcore.Info, cnr containerSDK.Container) error { - nn, err := p.placementBuilder.BuildPlacement(objInfo.Address.Container(), &objInfo.ECInfo.ParentID, cnr.PlacementPolicy()) + nn, err := p.placementBuilder.BuildPlacement(ctx, objInfo.Address.Container(), &objInfo.ECInfo.ParentID, cnr.PlacementPolicy()) if err != nil { return fmt.Errorf("%s: %w", logs.PolicerCouldNotBuildPlacementVectorForObject, err) } diff --git a/pkg/services/policer/ec_test.go b/pkg/services/policer/ec_test.go index e230153f9..c6980536b 100644 --- a/pkg/services/policer/ec_test.go +++ b/pkg/services/policer/ec_test.go @@ -36,7 +36,7 @@ func TestECChunkHasValidPlacement(t *testing.T) { cnr.Value.Init() cnr.Value.SetPlacementPolicy(policy) containerSrc := containerSrc{ - get: func(id cid.ID) (*container.Container, error) { + get: func(ctx context.Context, id cid.ID) (*container.Container, error) { if id.Equals(chunkAddress.Container()) { return cnr, nil } @@ -123,7 +123,7 @@ func TestECChunkHasInvalidPlacement(t *testing.T) { cnr.Value.Init() cnr.Value.SetPlacementPolicy(policy) containerSrc := containerSrc{ - get: func(id cid.ID) (*container.Container, error) { + get: func(ctx context.Context, id cid.ID) (*container.Container, error) { if id.Equals(chunkAddress.Container()) { return cnr, nil } @@ -448,7 +448,7 @@ func TestECChunkRestore(t *testing.T) { cnr.Value.Init() cnr.Value.SetPlacementPolicy(policy) containerSrc := containerSrc{ - get: func(id cid.ID) (*container.Container, error) { + get: func(ctx context.Context, id cid.ID) (*container.Container, error) { if id.Equals(parentAddress.Container()) { return cnr, nil } @@ -599,7 +599,7 @@ func TestECChunkRestoreNodeOff(t *testing.T) { cnr.Value.Init() cnr.Value.SetPlacementPolicy(policy) containerSrc := containerSrc{ - get: func(id cid.ID) (*container.Container, error) { + get: func(ctx context.Context, id cid.ID) (*container.Container, error) { if id.Equals(parentAddress.Container()) { return cnr, nil } diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index 9b9ab99ac..cef4c36d9 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -36,10 +36,10 @@ func TestBuryObjectWithoutContainer(t *testing.T) { // Container source and bury function buryCh := make(chan oid.Address) containerSrc := containerSrc{ - get: func(id cid.ID) (*container.Container, error) { + get: func(ctx context.Context, id cid.ID) (*container.Container, error) { return nil, new(apistatus.ContainerNotFound) }, - deletionInfo: func(id cid.ID) (*container.DelInfo, error) { + deletionInfo: func(ctx context.Context, id cid.ID) (*container.DelInfo, error) { return &container.DelInfo{}, nil }, } @@ -239,14 +239,14 @@ func TestProcessObject(t *testing.T) { cnr.Value.Init() cnr.Value.SetPlacementPolicy(policy) containerSrc := containerSrc{ - get: func(id cid.ID) (*container.Container, error) { + get: func(ctx context.Context, id cid.ID) (*container.Container, error) { if id.Equals(addr.Container()) { return cnr, nil } t.Errorf("unexpected container requested: got=%v, want=%v", id, addr.Container()) return nil, new(apistatus.ContainerNotFound) }, - deletionInfo: func(id cid.ID) (*container.DelInfo, error) { + deletionInfo: func(ctx context.Context, id cid.ID) (*container.DelInfo, error) { return &container.DelInfo{}, nil }, } @@ -304,10 +304,10 @@ func TestProcessObjectError(t *testing.T) { cnr := &container.Container{} cnr.Value.Init() source := containerSrc{ - get: func(id cid.ID) (*container.Container, error) { + get: func(ctx context.Context, id cid.ID) (*container.Container, error) { return nil, new(apistatus.ContainerNotFound) }, - deletionInfo: func(id cid.ID) (*container.DelInfo, error) { + deletionInfo: func(ctx context.Context, id cid.ID) (*container.DelInfo, error) { return nil, new(apistatus.ContainerNotFound) }, } @@ -352,10 +352,10 @@ func TestIteratorContract(t *testing.T) { } containerSrc := containerSrc{ - get: func(id cid.ID) (*container.Container, error) { + get: func(ctx context.Context, id cid.ID) (*container.Container, error) { return nil, new(apistatus.ContainerNotFound) }, - deletionInfo: func(id cid.ID) (*container.DelInfo, error) { + deletionInfo: func(ctx context.Context, id cid.ID) (*container.DelInfo, error) { return &container.DelInfo{}, nil }, } @@ -444,18 +444,22 @@ func (it *sliceKeySpaceIterator) Rewind() { } type containerSrc struct { - get func(id cid.ID) (*container.Container, error) - deletionInfo func(id cid.ID) (*container.DelInfo, error) + get func(ctx context.Context, id cid.ID) (*container.Container, error) + deletionInfo func(ctx context.Context, id cid.ID) (*container.DelInfo, error) } -func (f containerSrc) Get(id cid.ID) (*container.Container, error) { return f.get(id) } +func (f containerSrc) Get(ctx context.Context, id cid.ID) (*container.Container, error) { + return f.get(ctx, id) +} -func (f containerSrc) DeletionInfo(id cid.ID) (*container.DelInfo, error) { return f.deletionInfo(id) } +func (f containerSrc) DeletionInfo(ctx context.Context, id cid.ID) (*container.DelInfo, error) { + return f.deletionInfo(ctx, id) +} // placementBuilderFunc is a placement.Builder backed by a function type placementBuilderFunc func(cid.ID, *oid.ID, netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) -func (f placementBuilderFunc) BuildPlacement(c cid.ID, o *oid.ID, p netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { +func (f placementBuilderFunc) BuildPlacement(ctx context.Context, c cid.ID, o *oid.ID, p netmap.PlacementPolicy) ([][]netmap.NodeInfo, error) { return f(c, o, p) } diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index 606044f8e..c4b03cbe6 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -36,7 +36,7 @@ func (s *Service) newAPERequest(ctx context.Context, namespace string, nativeschema.PropertyKeyActorPublicKey: hex.EncodeToString(publicKey.Bytes()), nativeschema.PropertyKeyActorRole: schemaRole, } - reqProps, err = s.fillWithUserClaimTags(reqProps, publicKey) + reqProps, err = s.fillWithUserClaimTags(ctx, reqProps, publicKey) if err != nil { return aperequest.Request{}, err } @@ -74,7 +74,7 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, return fmt.Errorf("failed to create ape request: %w", err) } - return s.apeChecker.CheckAPE(checkercore.CheckPrm{ + return s.apeChecker.CheckAPE(ctx, checkercore.CheckPrm{ Request: request, Namespace: namespace, Container: cid, @@ -85,11 +85,11 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, } // fillWithUserClaimTags fills ape request properties with user claim tags getting them from frostfsid contract by actor public key. -func (s *Service) fillWithUserClaimTags(reqProps map[string]string, publicKey *keys.PublicKey) (map[string]string, error) { +func (s *Service) fillWithUserClaimTags(ctx context.Context, reqProps map[string]string, publicKey *keys.PublicKey) (map[string]string, error) { if reqProps == nil { reqProps = make(map[string]string) } - props, err := aperequest.FormFrostfsIDRequestProperties(s.frostfsidSubjectProvider, publicKey) + props, err := aperequest.FormFrostfsIDRequestProperties(ctx, s.frostfsidSubjectProvider, publicKey) if err != nil { return reqProps, err } diff --git a/pkg/services/tree/ape_test.go b/pkg/services/tree/ape_test.go index 3f94925b5..0afc7660a 100644 --- a/pkg/services/tree/ape_test.go +++ b/pkg/services/tree/ape_test.go @@ -37,7 +37,7 @@ type frostfsIDProviderMock struct { subjectsExtended map[util.Uint160]*client.SubjectExtended } -func (f *frostfsIDProviderMock) GetSubject(key util.Uint160) (*client.Subject, error) { +func (f *frostfsIDProviderMock) GetSubject(ctx context.Context, key util.Uint160) (*client.Subject, error) { v, ok := f.subjects[key] if !ok { return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) @@ -45,7 +45,7 @@ func (f *frostfsIDProviderMock) GetSubject(key util.Uint160) (*client.Subject, e return v, nil } -func (f *frostfsIDProviderMock) GetSubjectExtended(key util.Uint160) (*client.SubjectExtended, error) { +func (f *frostfsIDProviderMock) GetSubjectExtended(ctx context.Context, key util.Uint160) (*client.SubjectExtended, error) { v, ok := f.subjectsExtended[key] if !ok { return nil, fmt.Errorf("%s", frostfsidcore.SubjectNotFoundErrorMessage) diff --git a/pkg/services/tree/container.go b/pkg/services/tree/container.go index 435257550..c641a21a2 100644 --- a/pkg/services/tree/container.go +++ b/pkg/services/tree/container.go @@ -2,6 +2,7 @@ package tree import ( "bytes" + "context" "crypto/sha256" "fmt" "sync" @@ -32,13 +33,13 @@ type containerCacheItem struct { const defaultContainerCacheSize = 10 // getContainerNodes returns nodes in the container and a position of local key in the list. -func (s *Service) getContainerNodes(cid cidSDK.ID) ([]netmapSDK.NodeInfo, int, error) { - nm, err := s.nmSource.GetNetMap(0) +func (s *Service) getContainerNodes(ctx context.Context, cid cidSDK.ID) ([]netmapSDK.NodeInfo, int, error) { + nm, err := s.nmSource.GetNetMap(ctx, 0) if err != nil { return nil, -1, fmt.Errorf("can't get netmap: %w", err) } - cnr, err := s.cnrSource.Get(cid) + cnr, err := s.cnrSource.Get(ctx, cid) if err != nil { return nil, -1, fmt.Errorf("can't get container: %w", err) } diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index a3f488009..4ad760846 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -1,6 +1,7 @@ package tree import ( + "context" "crypto/ecdsa" "time" @@ -18,12 +19,12 @@ import ( type ContainerSource interface { container.Source - DeletionInfo(cid.ID) (*container.DelInfo, error) + DeletionInfo(ctx context.Context, cid cid.ID) (*container.DelInfo, error) // List must return list of all the containers in the FrostFS network // at the moment of a call and any error that does not allow fetching // container information. - List() ([]cid.ID, error) + List(ctx context.Context) ([]cid.ID, error) } type cfg struct { diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index bcbb73589..164815c76 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -149,7 +149,7 @@ func (s *Service) replicateLoop(ctx context.Context) { return case op := <-s.replicateCh: start := time.Now() - err := s.replicate(op) + err := s.replicate(ctx, op) if err != nil { s.log.Error(ctx, logs.TreeErrorDuringReplication, zap.Error(err), @@ -161,14 +161,14 @@ func (s *Service) replicateLoop(ctx context.Context) { } } -func (s *Service) replicate(op movePair) error { +func (s *Service) replicate(ctx context.Context, op movePair) error { req := newApplyRequest(&op) err := SignMessage(req, s.key) if err != nil { return fmt.Errorf("can't sign data: %w", err) } - nodes, localIndex, err := s.getContainerNodes(op.cid) + nodes, localIndex, err := s.getContainerNodes(ctx, op.cid) if err != nil { return fmt.Errorf("can't get container nodes: %w", err) } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 2df3c08e6..3c0214a98 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -118,7 +118,7 @@ func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error return nil, err } - ns, pos, err := s.getContainerNodes(cid) + ns, pos, err := s.getContainerNodes(ctx, cid) if err != nil { return nil, err } @@ -161,7 +161,7 @@ func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByP return nil, err } - ns, pos, err := s.getContainerNodes(cid) + ns, pos, err := s.getContainerNodes(ctx, cid) if err != nil { return nil, err } @@ -216,7 +216,7 @@ func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveRespon return nil, err } - ns, pos, err := s.getContainerNodes(cid) + ns, pos, err := s.getContainerNodes(ctx, cid) if err != nil { return nil, err } @@ -260,7 +260,7 @@ func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, er return nil, err } - ns, pos, err := s.getContainerNodes(cid) + ns, pos, err := s.getContainerNodes(ctx, cid) if err != nil { return nil, err } @@ -303,7 +303,7 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) return nil, err } - ns, pos, err := s.getContainerNodes(cid) + ns, pos, err := s.getContainerNodes(ctx, cid) if err != nil { return nil, err } @@ -376,7 +376,7 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS return err } - ns, pos, err := s.getContainerNodes(cid) + ns, pos, err := s.getContainerNodes(srv.Context(), cid) if err != nil { return err } @@ -586,7 +586,7 @@ func sortByFilename(nodes []pilorama.NodeInfo, d GetSubTreeRequest_Body_Order_Di } // Apply locally applies operation from the remote node to the tree. -func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, error) { +func (s *Service) Apply(ctx context.Context, req *ApplyRequest) (*ApplyResponse, error) { err := verifyMessage(req) if err != nil { return nil, err @@ -599,7 +599,7 @@ func (s *Service) Apply(_ context.Context, req *ApplyRequest) (*ApplyResponse, e key := req.GetSignature().GetKey() - _, pos, _, err := s.getContainerInfo(cid, key) + _, pos, _, err := s.getContainerInfo(ctx, cid, key) if err != nil { return nil, err } @@ -641,7 +641,7 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) return err } - ns, pos, err := s.getContainerNodes(cid) + ns, pos, err := s.getContainerNodes(srv.Context(), cid) if err != nil { return err } @@ -713,7 +713,7 @@ func (s *Service) TreeList(ctx context.Context, req *TreeListRequest) (*TreeList return nil, err } - ns, pos, err := s.getContainerNodes(cid) + ns, pos, err := s.getContainerNodes(ctx, cid) if err != nil { return nil, err } @@ -755,8 +755,8 @@ func metaToProto(arr []pilorama.KeyValue) []KeyValue { // getContainerInfo returns the list of container nodes, position in the container for the node // with pub key and total amount of nodes in all replicas. -func (s *Service) getContainerInfo(cid cidSDK.ID, pub []byte) ([]netmapSDK.NodeInfo, int, int, error) { - cntNodes, _, err := s.getContainerNodes(cid) +func (s *Service) getContainerInfo(ctx context.Context, cid cidSDK.ID, pub []byte) ([]netmapSDK.NodeInfo, int, int, error) { + cntNodes, _, err := s.getContainerNodes(ctx, cid) if err != nil { return nil, 0, 0, err } diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index b0f00615a..d15438e81 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -49,7 +49,7 @@ func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, return err } - cnr, err := s.cnrSource.Get(cid) + cnr, err := s.cnrSource.Get(ctx, cid) if err != nil { return fmt.Errorf("can't get container %s: %w", cid, err) } diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 7bc5002dc..97f8a727a 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -39,7 +39,7 @@ type dummySubjectProvider struct { subjects map[util.Uint160]client.SubjectExtended } -func (s dummySubjectProvider) GetSubject(addr util.Uint160) (*client.Subject, error) { +func (s dummySubjectProvider) GetSubject(ctx context.Context, addr util.Uint160) (*client.Subject, error) { res := s.subjects[addr] return &client.Subject{ PrimaryKey: res.PrimaryKey, @@ -50,7 +50,7 @@ func (s dummySubjectProvider) GetSubject(addr util.Uint160) (*client.Subject, er }, nil } -func (s dummySubjectProvider) GetSubjectExtended(addr util.Uint160) (*client.SubjectExtended, error) { +func (s dummySubjectProvider) GetSubjectExtended(ctx context.Context, addr util.Uint160) (*client.SubjectExtended, error) { res := s.subjects[addr] return &res, nil } @@ -65,7 +65,7 @@ func (s dummyEpochSource) CurrentEpoch() uint64 { type dummyContainerSource map[string]*containercore.Container -func (s dummyContainerSource) List() ([]cid.ID, error) { +func (s dummyContainerSource) List(context.Context) ([]cid.ID, error) { res := make([]cid.ID, 0, len(s)) var cnr cid.ID @@ -81,7 +81,7 @@ func (s dummyContainerSource) List() ([]cid.ID, error) { return res, nil } -func (s dummyContainerSource) Get(id cid.ID) (*containercore.Container, error) { +func (s dummyContainerSource) Get(ctx context.Context, id cid.ID) (*containercore.Container, error) { cnt, ok := s[id.String()] if !ok { return nil, errors.New("container not found") @@ -89,7 +89,7 @@ func (s dummyContainerSource) Get(id cid.ID) (*containercore.Container, error) { return cnt, nil } -func (s dummyContainerSource) DeletionInfo(id cid.ID) (*containercore.DelInfo, error) { +func (s dummyContainerSource) DeletionInfo(ctx context.Context, id cid.ID) (*containercore.DelInfo, error) { return &containercore.DelInfo{}, nil } diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 1a455def9..9b177d6b6 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -39,7 +39,7 @@ const defaultSyncWorkerCount = 20 // tree IDs from the other container nodes. Returns ErrNotInContainer if the node // is not included in the container. func (s *Service) synchronizeAllTrees(ctx context.Context, cid cid.ID) error { - nodes, pos, err := s.getContainerNodes(cid) + nodes, pos, err := s.getContainerNodes(ctx, cid) if err != nil { return fmt.Errorf("can't get container nodes: %w", err) } @@ -112,7 +112,7 @@ func (s *Service) synchronizeAllTrees(ctx context.Context, cid cid.ID) error { // SynchronizeTree tries to synchronize log starting from the last stored height. func (s *Service) SynchronizeTree(ctx context.Context, cid cid.ID, treeID string) error { - nodes, pos, err := s.getContainerNodes(cid) + nodes, pos, err := s.getContainerNodes(ctx, cid) if err != nil { return fmt.Errorf("can't get container nodes: %w", err) } @@ -393,7 +393,7 @@ func (s *Service) syncLoop(ctx context.Context) { start := time.Now() - cnrs, err := s.cfg.cnrSource.List() + cnrs, err := s.cfg.cnrSource.List(ctx) if err != nil { s.log.Error(ctx, logs.TreeCouldNotFetchContainers, zap.Error(err)) s.metrics.AddSyncDuration(time.Since(start), false) @@ -463,7 +463,7 @@ func (s *Service) removeContainers(ctx context.Context, newContainers map[cid.ID continue } - existed, err := containerCore.WasRemoved(s.cnrSource, cnr) + existed, err := containerCore.WasRemoved(ctx, s.cnrSource, cnr) if err != nil { s.log.Error(ctx, logs.TreeCouldNotCheckIfContainerExisted, zap.Stringer("cid", cnr), @@ -493,7 +493,7 @@ func (s *Service) containersToSync(ctx context.Context, cnrs []cid.ID) (map[cid. cnrsToSync := make([]cid.ID, 0, len(cnrs)) for _, cnr := range cnrs { - _, pos, err := s.getContainerNodes(cnr) + _, pos, err := s.getContainerNodes(ctx, cnr) if err != nil { s.log.Error(ctx, logs.TreeCouldNotCalculateContainerNodes, zap.Stringer("cid", cnr), From 69c35b1d61fad2d08edf008f709f19ad45fdf250 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 6 Feb 2025 21:19:24 +0300 Subject: [PATCH 1709/1943] [#1637] govulncheck: Use patch release with security fixes Signed-off-by: Alexander Chuprov --- .forgejo/workflows/vulncheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 8a5a818aa..7c89a3555 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -18,7 +18,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.23.5' + go-version: '1.23.6' - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest From 155f9eecb0159ac98ea85de0fefe04bbe24148c7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 21 Jan 2025 14:15:59 +0300 Subject: [PATCH 1710/1943] [#1608] config: Add QoS section and config Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 + cmd/frostfs-node/config/qos/config.go | 46 ++++++++++++++++++++++ cmd/frostfs-node/config/qos/config_test.go | 40 +++++++++++++++++++ cmd/frostfs-node/qos.go | 20 ++++++++++ config/example/node.env | 3 ++ config/example/node.json | 14 +++++++ config/example/node.yaml | 10 +++++ docs/storage-node-configuration.md | 20 +++++++++- 8 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 cmd/frostfs-node/config/qos/config.go create mode 100644 cmd/frostfs-node/config/qos/config_test.go create mode 100644 cmd/frostfs-node/qos.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 75d6f6dec..511777566 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -493,6 +493,7 @@ type cfg struct { cfgNetmap cfgNetmap cfgControlService cfgControlService cfgObject cfgObject + cfgQoSService cfgQoSService } // ReadCurrentNetMap reads network map which has been cached at the diff --git a/cmd/frostfs-node/config/qos/config.go b/cmd/frostfs-node/config/qos/config.go new file mode 100644 index 000000000..85f8180ed --- /dev/null +++ b/cmd/frostfs-node/config/qos/config.go @@ -0,0 +1,46 @@ +package qos + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +const ( + subsection = "qos" + criticalSubSection = "critical" + internalSubSection = "internal" +) + +// CriticalAuthorizedKeys parses and returns an array of "critical.authorized_keys" config +// parameter from "qos" section. +// +// Returns an empty list if not set. +func CriticalAuthorizedKeys(c *config.Config) keys.PublicKeys { + return authorizedKeys(c, criticalSubSection) +} + +// InternalAuthorizedKeys parses and returns an array of "internal.authorized_keys" config +// parameter from "qos" section. +// +// Returns an empty list if not set. +func InternalAuthorizedKeys(c *config.Config) keys.PublicKeys { + return authorizedKeys(c, internalSubSection) +} + +func authorizedKeys(c *config.Config, sub string) keys.PublicKeys { + strKeys := config.StringSliceSafe(c.Sub(subsection).Sub(sub), "authorized_keys") + pubs := make(keys.PublicKeys, 0, len(strKeys)) + + for i := range strKeys { + pub, err := keys.NewPublicKeyFromString(strKeys[i]) + if err != nil { + panic(fmt.Errorf("invalid authorized key %s for qos.%s: %w", strKeys[i], sub, err)) + } + + pubs = append(pubs, pub) + } + + return pubs +} diff --git a/cmd/frostfs-node/config/qos/config_test.go b/cmd/frostfs-node/config/qos/config_test.go new file mode 100644 index 000000000..b3b6019cc --- /dev/null +++ b/cmd/frostfs-node/config/qos/config_test.go @@ -0,0 +1,40 @@ +package qos + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestQoSSection(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + empty := configtest.EmptyConfig() + + require.Empty(t, CriticalAuthorizedKeys(empty)) + require.Empty(t, InternalAuthorizedKeys(empty)) + }) + + const path = "../../../../config/example/node" + + criticalPubs := make(keys.PublicKeys, 2) + criticalPubs[0], _ = keys.NewPublicKeyFromString("035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11") + criticalPubs[1], _ = keys.NewPublicKeyFromString("028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6") + + internalPubs := make(keys.PublicKeys, 2) + internalPubs[0], _ = keys.NewPublicKeyFromString("02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2") + internalPubs[1], _ = keys.NewPublicKeyFromString("031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a") + + fileConfigTest := func(c *config.Config) { + require.Equal(t, criticalPubs, CriticalAuthorizedKeys(c)) + require.Equal(t, internalPubs, InternalAuthorizedKeys(c)) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) +} diff --git a/cmd/frostfs-node/qos.go b/cmd/frostfs-node/qos.go new file mode 100644 index 000000000..20ef43230 --- /dev/null +++ b/cmd/frostfs-node/qos.go @@ -0,0 +1,20 @@ +package main + +import qosconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/qos" + +type cfgQoSService struct{} + +func initQoSService(c *cfg) { + criticalPubs := qosconfig.CriticalAuthorizedKeys(c.appCfg) + internalPubs := qosconfig.InternalAuthorizedKeys(c.appCfg) + rawCriticalPubs := make([][]byte, 0, len(criticalPubs)) + rawInternalPubs := make([][]byte, 0, len(internalPubs)) + for i := range criticalPubs { + rawCriticalPubs = append(rawCriticalPubs, criticalPubs[i].Bytes()) + } + for i := range internalPubs { + rawInternalPubs = append(rawInternalPubs, internalPubs[i].Bytes()) + } + + c.cfgQoSService = cfgQoSService{} +} diff --git a/config/example/node.env b/config/example/node.env index b2a0633a9..2ba432b1b 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -225,3 +225,6 @@ FROSTFS_MULTINET_SUBNETS_1_SOURCE_IPS="10.78.70.185 10.78.71.185" FROSTFS_MULTINET_BALANCER=roundrobin FROSTFS_MULTINET_RESTRICT=false FROSTFS_MULTINET_FALLBACK_DELAY=350ms + +FROSTFS_QOS_CRITICAL_AUTHORIZED_KEYS="035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6" +FROSTFS_QOS_INTERNAL_AUTHORIZED_KEYS="02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a" diff --git a/config/example/node.json b/config/example/node.json index f3192ac2f..cfde8bcc7 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -305,5 +305,19 @@ "balancer": "roundrobin", "restrict": false, "fallback_delay": "350ms" + }, + "qos": { + "critical": { + "authorized_keys": [ + "035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11", + "028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6" + ] + }, + "internal": { + "authorized_keys": [ + "02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2", + "031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a" + ] + } } } diff --git a/config/example/node.yaml b/config/example/node.yaml index c5acf5386..1f8ec843d 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -270,3 +270,13 @@ multinet: balancer: roundrobin restrict: false fallback_delay: 350ms + +qos: + critical: + authorized_keys: # list of hex-encoded public keys that have rights to use `critical` IO tag + - 035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 + - 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6 + internal: + authorized_keys: # list of hex-encoded public keys that have rights to use `internal` IO tag + - 02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2 + - 031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 98d72cb69..de2729c68 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -26,7 +26,8 @@ There are some custom types used for brevity: | `storage` | [Storage engine configuration](#storage-section) | | `runtime` | [Runtime configuration](#runtime-section) | | `audit` | [Audit configuration](#audit-section) | -| `multinet` | [Multinet configuration](#multinet-section) | +| `multinet` | [Multinet configuration](#multinet-section) | +| `qos` | [QoS configuration](#qos-section) | # `control` section ```yaml @@ -471,3 +472,20 @@ multinet: | `balancer` | `string` | "" | Balancer to select network interfaces, allowed values are "" (no balancing, use first suitable interface) or "roundrobin". | | `restrict` | `bool` | false | If `true` then any requests that do not match `subnets` will fail. | | `fallback_delay` | `duration` | 350ms | Delay before fallback to secondary IP addresses in case of hostname resolve. | + +# `qos` section +```yaml +qos: + critical: + authorized_keys: + - 035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 + - 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6 + internal: + authorized_keys: + - 035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 + - 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6 +``` +| Parameter | Type | Default value | Description | +| -------------------------- | -------------- | ------------- | --------------------------------------------------------------------------- | +| `critical.authorized_keys` | `[]public key` | empty | List of public keys for which requests with the tag `critical` are allowed. | +| `internal.authorized_keys` | `[]public key` | empty | List of public keys for which requests with the tag `internal` are allowed. | From 9729f31e5c96028a434a175cb4c2ce51b74875a9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 29 Jan 2025 11:26:09 +0300 Subject: [PATCH 1711/1943] [#1608] grpc: Add QoS interceptors for server and clients Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/internal/client/sdk.go | 5 +++-- cmd/frostfs-cli/modules/tree/client.go | 3 +++ cmd/frostfs-node/grpc.go | 3 +++ go.mod | 1 + go.sum | 2 ++ pkg/network/cache/multi.go | 3 +++ pkg/services/tree/cache.go | 3 +++ pkg/services/tree/sync.go | 3 +++ 8 files changed, 21 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/sdk.go b/cmd/frostfs-cli/internal/client/sdk.go index 2d9c45cbd..71c35bd2c 100644 --- a/cmd/frostfs-cli/internal/client/sdk.go +++ b/cmd/frostfs-cli/internal/client/sdk.go @@ -13,6 +13,7 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -56,8 +57,8 @@ func GetSDKClient(ctx context.Context, cmd *cobra.Command, key *ecdsa.PrivateKey prmDial := client.PrmDial{ Endpoint: addr.URIAddr(), GRPCDialOptions: []grpc.DialOption{ - grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInteceptor()), - grpc.WithChainStreamInterceptor(tracing.NewStreamClientInterceptor()), + grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInteceptor(), qos.NewUnaryClientInteceptor()), + grpc.WithChainStreamInterceptor(tracing.NewStreamClientInterceptor(), qos.NewStreamClientInterceptor()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), }, } diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index a70624ac8..421f11532 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "github.com/spf13/cobra" "github.com/spf13/viper" "google.golang.org/grpc" @@ -36,10 +37,12 @@ func _client() (tree.TreeServiceClient, error) { grpc.WithChainUnaryInterceptor( metrics.NewUnaryClientInterceptor(), tracing.NewUnaryClientInteceptor(), + qos.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( metrics.NewStreamClientInterceptor(), tracing.NewStreamClientInterceptor(), + qos.NewStreamClientInterceptor(), ), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), } diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index 6105be861..4d679e4cc 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -12,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -130,10 +131,12 @@ func getGrpcServerOpts(ctx context.Context, c *cfg, sc *grpcconfig.Config) ([]gr serverOpts := []grpc.ServerOption{ grpc.MaxRecvMsgSize(maxRecvMsgSize), grpc.ChainUnaryInterceptor( + qos.NewUnaryServerInterceptor(), metrics.NewUnaryServerInterceptor(), tracing.NewUnaryServerInterceptor(), ), grpc.ChainStreamInterceptor( + qos.NewStreamServerInterceptor(), metrics.NewStreamServerInterceptor(), tracing.NewStreamServerInterceptor(), ), diff --git a/go.mod b/go.mod index cc6b0a202..8f2e7bc51 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 + git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250202151421-8389887a3421 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 diff --git a/go.sum b/go.sum index eae467b31..e92b64886 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe h1:81gDNdWNLP24oMQukRiCE9R1wGSh0l0dRq3F1W+Oesc= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250202151421-8389887a3421 h1:pP19IawSdsLCKFv7HMNfWAeH6E3uSnntKZkwka+/2+4= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250202151421-8389887a3421/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 1bcb83259..5c3beb553 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -64,10 +65,12 @@ func (x *multiClient) createForAddress(ctx context.Context, addr network.Address grpc.WithChainUnaryInterceptor( metrics.NewUnaryClientInterceptor(), tracing.NewUnaryClientInteceptor(), + qos.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( metrics.NewStreamClientInterceptor(), tracing.NewStreamClientInterceptor(), + qos.NewStreamClientInterceptor(), ), grpc.WithContextDialer(x.opts.DialerSource.GrpcContextDialer()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index ac80d0e4c..7f1dcf07c 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -12,6 +12,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "github.com/hashicorp/golang-lru/v2/simplelru" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" @@ -97,10 +98,12 @@ func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (* grpc.WithChainUnaryInterceptor( metrics.NewUnaryClientInterceptor(), tracing.NewUnaryClientInteceptor(), + qos.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( metrics.NewStreamClientInterceptor(), tracing.NewStreamClientInterceptor(), + qos.NewStreamClientInterceptor(), ), grpc.WithContextDialer(c.ds.GrpcContextDialer()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 9b177d6b6..a0485d9d8 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -20,6 +20,7 @@ import ( metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" tracing_grpc "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" @@ -342,10 +343,12 @@ func (*Service) createConnection(a network.Address) (*grpc.ClientConn, error) { grpc.WithChainUnaryInterceptor( metrics.NewUnaryClientInterceptor(), tracing_grpc.NewUnaryClientInteceptor(), + qos.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( metrics.NewStreamClientInterceptor(), tracing_grpc.NewStreamClientInterceptor(), + qos.NewStreamClientInterceptor(), ), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), From f6b3f79e8916f826f55bb28aff799699d1465be0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 29 Jan 2025 11:28:52 +0300 Subject: [PATCH 1712/1943] [#1608] qos: Add qos service to adjust incoming IO tags Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/main.go | 1 + cmd/frostfs-node/qos.go | 81 ++++++++++++++++++- internal/logs/logs.go | 3 + .../metabase/upgrade_test.go | 2 +- 4 files changed, 83 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 3c15dc439..b53bd8823 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -101,6 +101,7 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(ctx, c, "gRPC", func(c *cfg) { initGRPC(ctx, c) }) initAndLog(ctx, c, "netmap", func(c *cfg) { initNetmapService(ctx, c) }) + initAndLog(ctx, c, "qos", func(c *cfg) { initQoSService(c) }) initAccessPolicyEngine(ctx, c) initAndLog(ctx, c, "access policy engine", func(c *cfg) { diff --git a/cmd/frostfs-node/qos.go b/cmd/frostfs-node/qos.go index 20ef43230..bfc278333 100644 --- a/cmd/frostfs-node/qos.go +++ b/cmd/frostfs-node/qos.go @@ -1,8 +1,24 @@ package main -import qosconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/qos" +import ( + "bytes" + "context" -type cfgQoSService struct{} + qosconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/qos" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + qosTagging "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + "go.uber.org/zap" +) + +type cfgQoSService struct { + netmapSource netmap.Source + logger *logger.Logger + allowedCriticalPubs [][]byte + allowedInternalPubs [][]byte +} func initQoSService(c *cfg) { criticalPubs := qosconfig.CriticalAuthorizedKeys(c.appCfg) @@ -16,5 +32,64 @@ func initQoSService(c *cfg) { rawInternalPubs = append(rawInternalPubs, internalPubs[i].Bytes()) } - c.cfgQoSService = cfgQoSService{} + c.cfgQoSService = cfgQoSService{ + netmapSource: c.netMapSource, + logger: c.log, + allowedCriticalPubs: rawCriticalPubs, + allowedInternalPubs: rawInternalPubs, + } +} + +func (s *cfgQoSService) AdjustIncomingTag(ctx context.Context, requestSignPublicKey []byte) context.Context { + rawTag, defined := qosTagging.IOTagFromContext(ctx) + if !defined { + return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) + } + ioTag, err := qos.FromRawString(rawTag) + if err != nil { + s.logger.Warn(ctx, logs.FailedToParseIncomingIOTag, zap.Error(err)) + return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) + } + + switch ioTag { + case qos.IOTagClient: + return ctx + case qos.IOTagCritical: + for _, pk := range s.allowedCriticalPubs { + if bytes.Equal(pk, requestSignPublicKey) { + return ctx + } + } + nm, err := s.netmapSource.GetNetMap(ctx, 0) + if err != nil { + s.logger.Debug(ctx, logs.FailedToGetNetmapToAdjustIOTag, zap.Error(err)) + return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) + } + for _, node := range nm.Nodes() { + if bytes.Equal(node.PublicKey(), requestSignPublicKey) { + return ctx + } + } + return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) + case qos.IOTagInternal: + for _, pk := range s.allowedInternalPubs { + if bytes.Equal(pk, requestSignPublicKey) { + return ctx + } + } + nm, err := s.netmapSource.GetNetMap(ctx, 0) + if err != nil { + s.logger.Debug(ctx, logs.FailedToGetNetmapToAdjustIOTag, zap.Error(err)) + return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) + } + for _, node := range nm.Nodes() { + if bytes.Equal(node.PublicKey(), requestSignPublicKey) { + return ctx + } + } + return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) + default: + s.logger.Warn(ctx, logs.NotSupportedIncomingIOTagReplacedWithClient, zap.Stringer("io_tag", ioTag)) + return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) + } } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 0610dc175..6a72644e5 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -510,4 +510,7 @@ const ( BlobovniczatreeFailedToRemoveRebuildTempFile = "failed to remove rebuild temp file" WritecacheCantGetObject = "can't get an object from fstree" FailedToUpdateMultinetConfiguration = "failed to update multinet configuration" + FailedToParseIncomingIOTag = "failed to parse incoming IO tag" + NotSupportedIncomingIOTagReplacedWithClient = "incoming IO tag is not supported, replaced with `client`" + FailedToGetNetmapToAdjustIOTag = "failed to get netmap to adjust IO tag, replaced with `client`" ) diff --git a/pkg/local_object_storage/metabase/upgrade_test.go b/pkg/local_object_storage/metabase/upgrade_test.go index 5444264be..c90de4dd6 100644 --- a/pkg/local_object_storage/metabase/upgrade_test.go +++ b/pkg/local_object_storage/metabase/upgrade_test.go @@ -45,7 +45,7 @@ func TestUpgradeV2ToV3(t *testing.T) { type testContainerInfoProvider struct{} -func (p *testContainerInfoProvider) Info(id cid.ID) (container.Info, error) { +func (p *testContainerInfoProvider) Info(ctx context.Context, id cid.ID) (container.Info, error) { return container.Info{}, nil } From 12da2f826278762e8879994c2788e88b9084afe4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 29 Jan 2025 11:29:22 +0300 Subject: [PATCH 1713/1943] [#1608] object: Add IO tag adjustment layer Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/object.go | 5 +- internal/qos/tags.go | 39 +++++++++++ pkg/services/object/qos.go | 132 +++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 internal/qos/tags.go create mode 100644 pkg/services/object/qos.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 77446b81c..40d3cc1cd 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -168,7 +168,7 @@ func initObjectService(c *cfg) { sPatch := createPatchSvc(sGet, sPut) // build service pipeline - // grpc | audit | | signature | response | acl | ape | split + // grpc | audit | qos | | signature | response | acl | ape | split splitSvc := createSplitService(c, sPutV2, sGetV2, sSearchV2, sDeleteV2, sPatch) @@ -191,7 +191,8 @@ func initObjectService(c *cfg) { c.shared.metricsSvc = objectService.NewMetricCollector( signSvc, c.metricsCollector.ObjectService(), metricsconfig.Enabled(c.appCfg)) - auditSvc := objectService.NewAuditService(c.shared.metricsSvc, c.log, c.audit) + qosService := objectService.NewQoSObjectService(c.shared.metricsSvc, &c.cfgQoSService) + auditSvc := objectService.NewAuditService(qosService, c.log, c.audit) server := objectTransportGRPC.New(auditSvc) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { diff --git a/internal/qos/tags.go b/internal/qos/tags.go new file mode 100644 index 000000000..6a9a7f7a4 --- /dev/null +++ b/internal/qos/tags.go @@ -0,0 +1,39 @@ +package qos + +import "fmt" + +type IOTag string + +const ( + IOTagClient IOTag = "client" + IOTagInternal IOTag = "internal" + IOTagBackground IOTag = "background" + IOTagWritecache IOTag = "writecache" + IOTagPolicer IOTag = "policer" + IOTagCritical IOTag = "critical" + + ioTagUnknown IOTag = "" +) + +func FromRawString(s string) (IOTag, error) { + switch s { + case string(IOTagCritical): + return IOTagCritical, nil + case string(IOTagClient): + return IOTagClient, nil + case string(IOTagInternal): + return IOTagInternal, nil + case string(IOTagBackground): + return IOTagBackground, nil + case string(IOTagWritecache): + return IOTagWritecache, nil + case string(IOTagPolicer): + return IOTagPolicer, nil + default: + return ioTagUnknown, fmt.Errorf("unknown tag %s", s) + } +} + +func (t IOTag) String() string { + return string(t) +} diff --git a/pkg/services/object/qos.go b/pkg/services/object/qos.go new file mode 100644 index 000000000..145a316e2 --- /dev/null +++ b/pkg/services/object/qos.go @@ -0,0 +1,132 @@ +package object + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" +) + +var _ ServiceServer = (*qosObjectService)(nil) + +type AdjustIOTag interface { + AdjustIncomingTag(ctx context.Context, requestSignPublicKey []byte) context.Context +} + +type qosObjectService struct { + next ServiceServer + adj AdjustIOTag +} + +func NewQoSObjectService(next ServiceServer, adjIOTag AdjustIOTag) ServiceServer { + return &qosObjectService{ + next: next, + adj: adjIOTag, + } +} + +func (q *qosObjectService) Delete(ctx context.Context, req *object.DeleteRequest) (*object.DeleteResponse, error) { + ctx = q.adj.AdjustIncomingTag(ctx, req.GetVerificationHeader().GetBodySignature().GetKey()) + return q.next.Delete(ctx, req) +} + +func (q *qosObjectService) Get(req *object.GetRequest, s GetObjectStream) error { + ctx := q.adj.AdjustIncomingTag(s.Context(), req.GetVerificationHeader().GetBodySignature().GetKey()) + return q.next.Get(req, &qosReadStream[*object.GetResponse]{ + ctxF: func() context.Context { return ctx }, + sender: s, + }) +} + +func (q *qosObjectService) GetRange(req *object.GetRangeRequest, s GetObjectRangeStream) error { + ctx := q.adj.AdjustIncomingTag(s.Context(), req.GetVerificationHeader().GetBodySignature().GetKey()) + return q.next.GetRange(req, &qosReadStream[*object.GetRangeResponse]{ + ctxF: func() context.Context { return ctx }, + sender: s, + }) +} + +func (q *qosObjectService) GetRangeHash(ctx context.Context, req *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { + ctx = q.adj.AdjustIncomingTag(ctx, req.GetVerificationHeader().GetBodySignature().GetKey()) + return q.next.GetRangeHash(ctx, req) +} + +func (q *qosObjectService) Head(ctx context.Context, req *object.HeadRequest) (*object.HeadResponse, error) { + ctx = q.adj.AdjustIncomingTag(ctx, req.GetVerificationHeader().GetBodySignature().GetKey()) + return q.next.Head(ctx, req) +} + +func (q *qosObjectService) Patch(ctx context.Context) (PatchObjectStream, error) { + s, err := q.next.Patch(ctx) + if err != nil { + return nil, err + } + return &qosWriteStream[*object.PatchRequest, *object.PatchResponse]{ + s: s, + adj: q.adj, + }, nil +} + +func (q *qosObjectService) Put(ctx context.Context) (PutObjectStream, error) { + s, err := q.next.Put(ctx) + if err != nil { + return nil, err + } + return &qosWriteStream[*object.PutRequest, *object.PutResponse]{ + s: s, + adj: q.adj, + }, nil +} + +func (q *qosObjectService) PutSingle(ctx context.Context, req *object.PutSingleRequest) (*object.PutSingleResponse, error) { + ctx = q.adj.AdjustIncomingTag(ctx, req.GetVerificationHeader().GetBodySignature().GetKey()) + return q.next.PutSingle(ctx, req) +} + +func (q *qosObjectService) Search(req *object.SearchRequest, s SearchStream) error { + ctx := q.adj.AdjustIncomingTag(s.Context(), req.GetVerificationHeader().GetBodySignature().GetKey()) + return q.next.Search(req, &qosReadStream[*object.SearchResponse]{ + ctxF: func() context.Context { return ctx }, + sender: s, + }) +} + +type qosSend[T any] interface { + Send(T) error +} + +type qosReadStream[T any] struct { + sender qosSend[T] + ctxF func() context.Context +} + +func (g *qosReadStream[T]) Context() context.Context { + return g.ctxF() +} + +func (g *qosReadStream[T]) Send(resp T) error { + return g.sender.Send(resp) +} + +type qosVerificationHeader interface { + GetVerificationHeader() *session.RequestVerificationHeader +} + +type qosSendRecv[TReq qosVerificationHeader, TResp any] interface { + Send(context.Context, TReq) error + CloseAndRecv(context.Context) (TResp, error) +} + +type qosWriteStream[TReq qosVerificationHeader, TResp any] struct { + s qosSendRecv[TReq, TResp] + adj AdjustIOTag +} + +func (q *qosWriteStream[TReq, TResp]) CloseAndRecv(ctx context.Context) (TResp, error) { + return q.s.CloseAndRecv(ctx) +} + +func (q *qosWriteStream[TReq, TResp]) Send(ctx context.Context, req TReq) error { + ctx = q.adj.AdjustIncomingTag(ctx, req.GetVerificationHeader().GetBodySignature().GetKey()) + return q.s.Send(ctx, req) +} From 170860c14a7778ff141a136899e52ba20ee6607b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 29 Jan 2025 11:44:20 +0300 Subject: [PATCH 1714/1943] [#1608] logger: Add IO tag logging Signed-off-by: Dmitrii Stepanov --- pkg/util/logger/log.go | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/pkg/util/logger/log.go b/pkg/util/logger/log.go index 269e07d90..413b1d9aa 100644 --- a/pkg/util/logger/log.go +++ b/pkg/util/logger/log.go @@ -4,37 +4,32 @@ import ( "context" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/tracing" + qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "go.uber.org/zap" ) func (l *Logger) Debug(ctx context.Context, msg string, fields ...zap.Field) { - if traceID := tracing.GetTraceID(ctx); traceID != "" { - l.z.Debug(msg, append(fields, zap.String("trace_id", traceID))...) - return - } - l.z.Debug(msg, fields...) + l.z.Debug(msg, appendContext(ctx, fields...)...) } func (l *Logger) Info(ctx context.Context, msg string, fields ...zap.Field) { - if traceID := tracing.GetTraceID(ctx); traceID != "" { - l.z.Info(msg, append(fields, zap.String("trace_id", traceID))...) - return - } - l.z.Info(msg, fields...) + l.z.Info(msg, appendContext(ctx, fields...)...) } func (l *Logger) Warn(ctx context.Context, msg string, fields ...zap.Field) { - if traceID := tracing.GetTraceID(ctx); traceID != "" { - l.z.Warn(msg, append(fields, zap.String("trace_id", traceID))...) - return - } - l.z.Warn(msg, fields...) + l.z.Warn(msg, appendContext(ctx, fields...)...) } func (l *Logger) Error(ctx context.Context, msg string, fields ...zap.Field) { - if traceID := tracing.GetTraceID(ctx); traceID != "" { - l.z.Error(msg, append(fields, zap.String("trace_id", traceID))...) - return - } - l.z.Error(msg, fields...) + l.z.Error(msg, appendContext(ctx, fields...)...) +} + +func appendContext(ctx context.Context, fields ...zap.Field) []zap.Field { + if traceID := tracing.GetTraceID(ctx); traceID != "" { + fields = append(fields, zap.String("trace_id", traceID)) + } + if ioTag, ioTagDefined := qos.IOTagFromContext(ctx); ioTagDefined { + fields = append(fields, zap.String("io_tag", ioTag)) + } + return fields } From dc6aea7b7956bfca222288fa057eecdaca35ca4d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 29 Jan 2025 15:56:31 +0300 Subject: [PATCH 1715/1943] [#1608] control: Use IO tag `critical` Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/control.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index ecd82bba5..1d9ac3df0 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -7,9 +7,13 @@ import ( controlconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/control" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" controlSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sdnotify" + metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" + tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -50,7 +54,17 @@ func initControlService(ctx context.Context, c *cfg) { return } - c.cfgControlService.server = grpc.NewServer() + c.cfgControlService.server = grpc.NewServer( + grpc.ChainUnaryInterceptor( + func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + ctx = tagging.ContextWithIOTag(ctx, qos.IOTagCritical.String()) + return handler(ctx, req) + }, + metrics.NewUnaryServerInterceptor(), + tracing.NewUnaryServerInterceptor(), + ), + // control service has no stream methods, so no stream interceptors added + ) c.onShutdown(func() { stopGRPC(ctx, "FrostFS Control API", c.cfgControlService.server, c.log) From acec938b2df8705c008efa2f47b4b2ecd9767245 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 29 Jan 2025 17:43:53 +0300 Subject: [PATCH 1716/1943] [#1608] qos: Add client grpc interceptors `qos` client interceptors replace internal IO tags `writecache`, `policer` and `background` with `internal` IO tag for outcomming RPC. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/control.go | 6 +---- internal/qos/grpc.go | 51 +++++++++++++++++++++++++++++++++++++ pkg/network/cache/multi.go | 9 ++++--- pkg/services/tree/cache.go | 9 ++++--- pkg/services/tree/sync.go | 9 ++++--- 5 files changed, 70 insertions(+), 14 deletions(-) create mode 100644 internal/qos/grpc.go diff --git a/cmd/frostfs-node/control.go b/cmd/frostfs-node/control.go index 1d9ac3df0..1825013c7 100644 --- a/cmd/frostfs-node/control.go +++ b/cmd/frostfs-node/control.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sdnotify" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -56,10 +55,7 @@ func initControlService(ctx context.Context, c *cfg) { c.cfgControlService.server = grpc.NewServer( grpc.ChainUnaryInterceptor( - func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { - ctx = tagging.ContextWithIOTag(ctx, qos.IOTagCritical.String()) - return handler(ctx, req) - }, + qos.NewSetCriticalIOTagUnaryServerInterceptor(), metrics.NewUnaryServerInterceptor(), tracing.NewUnaryServerInterceptor(), ), diff --git a/internal/qos/grpc.go b/internal/qos/grpc.go new file mode 100644 index 000000000..c253f1e9d --- /dev/null +++ b/internal/qos/grpc.go @@ -0,0 +1,51 @@ +package qos + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + "google.golang.org/grpc" +) + +func NewSetCriticalIOTagUnaryServerInterceptor() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + ctx = tagging.ContextWithIOTag(ctx, IOTagCritical.String()) + return handler(ctx, req) + } +} + +func NewAdjustOutgoingIOTagUnaryClientInterceptor() grpc.UnaryClientInterceptor { + return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + rawTag, ok := tagging.IOTagFromContext(ctx) + if !ok { + return invoker(ctx, method, req, reply, cc, opts...) + } + tag, err := FromRawString(rawTag) + if err != nil { + tag = IOTagClient + } + if tag == IOTagBackground || tag == IOTagPolicer || tag == IOTagWritecache { + tag = IOTagInternal + } + ctx = tagging.ContextWithIOTag(ctx, tag.String()) + return invoker(ctx, method, req, reply, cc, opts...) + } +} + +func NewAdjustOutgoingIOTagStreamClientInterceptor() grpc.StreamClientInterceptor { + return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + rawTag, ok := tagging.IOTagFromContext(ctx) + if !ok { + return streamer(ctx, desc, cc, method, opts...) + } + tag, err := FromRawString(rawTag) + if err != nil { + tag = IOTagClient + } + if tag == IOTagBackground || tag == IOTagPolicer || tag == IOTagWritecache { + tag = IOTagInternal + } + ctx = tagging.ContextWithIOTag(ctx, tag.String()) + return streamer(ctx, desc, cc, method, opts...) + } +} diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 5c3beb553..e94fa580a 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -7,11 +7,12 @@ import ( "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" clientcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" - qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -63,14 +64,16 @@ func (x *multiClient) createForAddress(ctx context.Context, addr network.Address grpcOpts := []grpc.DialOption{ grpc.WithChainUnaryInterceptor( + qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), metrics.NewUnaryClientInterceptor(), tracing.NewUnaryClientInteceptor(), - qos.NewUnaryClientInteceptor(), + tagging.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( + qos.NewAdjustOutgoingIOTagStreamClientInterceptor(), metrics.NewStreamClientInterceptor(), tracing.NewStreamClientInterceptor(), - qos.NewStreamClientInterceptor(), + tagging.NewStreamClientInterceptor(), ), grpc.WithContextDialer(x.opts.DialerSource.GrpcContextDialer()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 7f1dcf07c..125871fc4 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -9,10 +9,11 @@ import ( "time" internalNet "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" - qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "github.com/hashicorp/golang-lru/v2/simplelru" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" @@ -96,14 +97,16 @@ func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (* opts := []grpc.DialOption{ grpc.WithChainUnaryInterceptor( + qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), metrics.NewUnaryClientInterceptor(), tracing.NewUnaryClientInteceptor(), - qos.NewUnaryClientInteceptor(), + tagging.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( + qos.NewAdjustOutgoingIOTagStreamClientInterceptor(), metrics.NewStreamClientInterceptor(), tracing.NewStreamClientInterceptor(), - qos.NewStreamClientInterceptor(), + tagging.NewStreamClientInterceptor(), ), grpc.WithContextDialer(c.ds.GrpcContextDialer()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index a0485d9d8..3e0a45385 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -13,6 +13,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" @@ -20,7 +21,7 @@ import ( metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" tracing_grpc "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" - qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" @@ -341,14 +342,16 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, func (*Service) createConnection(a network.Address) (*grpc.ClientConn, error) { return grpc.NewClient(a.URIAddr(), grpc.WithChainUnaryInterceptor( + qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), metrics.NewUnaryClientInterceptor(), tracing_grpc.NewUnaryClientInteceptor(), - qos.NewUnaryClientInteceptor(), + tagging.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( + qos.NewAdjustOutgoingIOTagStreamClientInterceptor(), metrics.NewStreamClientInterceptor(), tracing_grpc.NewStreamClientInterceptor(), - qos.NewStreamClientInterceptor(), + tagging.NewStreamClientInterceptor(), ), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), From bfe325e61d79cd297b8c7a3a153dd59a67238f18 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 29 Jan 2025 17:45:40 +0300 Subject: [PATCH 1717/1943] [#1608] policer: Add IO tag to context Signed-off-by: Dmitrii Stepanov --- pkg/services/policer/process.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/services/policer/process.go b/pkg/services/policer/process.go index bd830d04e..635a5683b 100644 --- a/pkg/services/policer/process.go +++ b/pkg/services/policer/process.go @@ -7,7 +7,9 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" ) @@ -18,6 +20,7 @@ func (p *Policer) Run(ctx context.Context) { } func (p *Policer) shardPolicyWorker(ctx context.Context) { + ctx = tagging.ContextWithIOTag(ctx, qos.IOTagPolicer.String()) for { select { case <-ctx.Done(): From fd0c6c461dd83daacd9e5d7e4427366ac2982f15 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 29 Jan 2025 17:47:10 +0300 Subject: [PATCH 1718/1943] [#1608] cli: Drop redundant interceptors There is no metrics or IO tags in cli. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/internal/client/sdk.go | 5 ++--- cmd/frostfs-cli/modules/tree/client.go | 6 ------ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/sdk.go b/cmd/frostfs-cli/internal/client/sdk.go index 71c35bd2c..2d9c45cbd 100644 --- a/cmd/frostfs-cli/internal/client/sdk.go +++ b/cmd/frostfs-cli/internal/client/sdk.go @@ -13,7 +13,6 @@ import ( commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" - qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -57,8 +56,8 @@ func GetSDKClient(ctx context.Context, cmd *cobra.Command, key *ecdsa.PrivateKey prmDial := client.PrmDial{ Endpoint: addr.URIAddr(), GRPCDialOptions: []grpc.DialOption{ - grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInteceptor(), qos.NewUnaryClientInteceptor()), - grpc.WithChainStreamInterceptor(tracing.NewStreamClientInterceptor(), qos.NewStreamClientInterceptor()), + grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInteceptor()), + grpc.WithChainStreamInterceptor(tracing.NewStreamClientInterceptor()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), }, } diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index 421f11532..c6953f126 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -9,9 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" - metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" - qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "github.com/spf13/cobra" "github.com/spf13/viper" "google.golang.org/grpc" @@ -35,14 +33,10 @@ func _client() (tree.TreeServiceClient, error) { opts := []grpc.DialOption{ grpc.WithChainUnaryInterceptor( - metrics.NewUnaryClientInterceptor(), tracing.NewUnaryClientInteceptor(), - qos.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( - metrics.NewStreamClientInterceptor(), tracing.NewStreamClientInterceptor(), - qos.NewStreamClientInterceptor(), ), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), } From abba5b20890d157305ab8581b8ae3c9dadf0da47 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 31 Jan 2025 10:12:46 +0300 Subject: [PATCH 1719/1943] [#1608] writecache: Add IO tag to flush worker Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/writecache/flush.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index d9e34ceab..3f9b36f9d 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -6,6 +6,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -14,6 +15,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "go.opentelemetry.io/otel/attribute" @@ -35,6 +37,7 @@ func (c *cache) runFlushLoop(ctx context.Context) { if c.disableBackgroundFlush { return } + ctx = tagging.ContextWithIOTag(ctx, qos.IOTagWritecache.String()) fl := newFlushLimiter(c.flushSizeLimit) c.wg.Add(1) go func() { From 92450a76ba11cb5c75eeffa1fe2513e49f09a4df Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 31 Jan 2025 11:00:28 +0300 Subject: [PATCH 1720/1943] [#1608] shard: Add IO tag to GC Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 1b218a372..4a5ec7a71 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -6,11 +6,13 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.uber.org/zap" @@ -149,7 +151,7 @@ func (gc *gc) init(ctx context.Context) { if sz > 0 { gc.workerPool = gc.workerPoolInit(sz) } - + ctx = tagging.ContextWithIOTag(ctx, qos.IOTagBackground.String()) gc.wg.Add(2) go gc.tickRemover(ctx) go gc.listenEvents(ctx) From 0671c277db2837fab94fdef2249b05f64caf2b73 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 31 Jan 2025 12:14:51 +0300 Subject: [PATCH 1721/1943] [#1608] tree: Add IO tag for tree sync requests Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/tree.go | 2 +- pkg/services/tree/qos.go | 101 +++++++++++++++++++++++++++++++++++ pkg/services/tree/service.go | 3 ++ 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 pkg/services/tree/qos.go diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index f8330a25e..65414f0ca 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -72,7 +72,7 @@ func initTreeService(c *cfg) { ) c.cfgGRPC.performAndSave(func(_ string, _ net.Listener, s *grpc.Server) { - tree.RegisterTreeServiceServer(s, c.treeService) + tree.RegisterTreeServiceServer(s, tree.NewIOTagAdjustServer(c.treeService, &c.cfgQoSService)) }) c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { diff --git a/pkg/services/tree/qos.go b/pkg/services/tree/qos.go new file mode 100644 index 000000000..8f21686df --- /dev/null +++ b/pkg/services/tree/qos.go @@ -0,0 +1,101 @@ +package tree + +import ( + "context" + + "google.golang.org/grpc" +) + +var _ TreeServiceServer = (*ioTagAdjust)(nil) + +type AdjustIOTag interface { + AdjustIncomingTag(ctx context.Context, requestSignPublicKey []byte) context.Context +} + +type ioTagAdjust struct { + s TreeServiceServer + a AdjustIOTag +} + +func NewIOTagAdjustServer(s TreeServiceServer, a AdjustIOTag) TreeServiceServer { + return &ioTagAdjust{ + s: s, + a: a, + } +} + +func (i *ioTagAdjust) Add(ctx context.Context, req *AddRequest) (*AddResponse, error) { + ctx = i.a.AdjustIncomingTag(ctx, req.GetSignature().GetKey()) + return i.s.Add(ctx, req) +} + +func (i *ioTagAdjust) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByPathResponse, error) { + ctx = i.a.AdjustIncomingTag(ctx, req.GetSignature().GetKey()) + return i.s.AddByPath(ctx, req) +} + +func (i *ioTagAdjust) Apply(ctx context.Context, req *ApplyRequest) (*ApplyResponse, error) { + ctx = i.a.AdjustIncomingTag(ctx, req.GetSignature().GetKey()) + return i.s.Apply(ctx, req) +} + +func (i *ioTagAdjust) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) (*GetNodeByPathResponse, error) { + ctx = i.a.AdjustIncomingTag(ctx, req.GetSignature().GetKey()) + return i.s.GetNodeByPath(ctx, req) +} + +func (i *ioTagAdjust) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) error { + ctx := i.a.AdjustIncomingTag(srv.Context(), req.GetSignature().GetKey()) + return i.s.GetOpLog(req, &qosServerWrapper[*GetOpLogResponse]{ + sender: srv, + ServerStream: srv, + ctxF: func() context.Context { return ctx }, + }) +} + +func (i *ioTagAdjust) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeServer) error { + ctx := i.a.AdjustIncomingTag(srv.Context(), req.GetSignature().GetKey()) + return i.s.GetSubTree(req, &qosServerWrapper[*GetSubTreeResponse]{ + sender: srv, + ServerStream: srv, + ctxF: func() context.Context { return ctx }, + }) +} + +func (i *ioTagAdjust) Healthcheck(ctx context.Context, req *HealthcheckRequest) (*HealthcheckResponse, error) { + ctx = i.a.AdjustIncomingTag(ctx, req.GetSignature().GetKey()) + return i.s.Healthcheck(ctx, req) +} + +func (i *ioTagAdjust) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, error) { + ctx = i.a.AdjustIncomingTag(ctx, req.GetSignature().GetKey()) + return i.s.Move(ctx, req) +} + +func (i *ioTagAdjust) Remove(ctx context.Context, req *RemoveRequest) (*RemoveResponse, error) { + ctx = i.a.AdjustIncomingTag(ctx, req.GetSignature().GetKey()) + return i.s.Remove(ctx, req) +} + +func (i *ioTagAdjust) TreeList(ctx context.Context, req *TreeListRequest) (*TreeListResponse, error) { + ctx = i.a.AdjustIncomingTag(ctx, req.GetSignature().GetKey()) + return i.s.TreeList(ctx, req) +} + +type qosSend[T any] interface { + Send(T) error +} + +type qosServerWrapper[T any] struct { + grpc.ServerStream + sender qosSend[T] + ctxF func() context.Context +} + +func (w *qosServerWrapper[T]) Send(resp T) error { + return w.sender.Send(resp) +} + +func (w *qosServerWrapper[T]) Context() context.Context { + return w.ctxF() +} diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 3c0214a98..2e9722e79 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -9,9 +9,11 @@ import ( "sync" "sync/atomic" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" checkercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/common/ape" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -83,6 +85,7 @@ func New(opts ...Option) *Service { // Start starts the service. func (s *Service) Start(ctx context.Context) { + ctx = tagging.ContextWithIOTag(ctx, qos.IOTagBackground.String()) go s.replicateLoop(ctx) go s.syncLoop(ctx) From 5d79abe523b0254fe0484220d6d2c57a172090e1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Feb 2025 15:32:20 +0300 Subject: [PATCH 1722/1943] [#1608] shard: Add IO tag for rebuild Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/rebuild.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/local_object_storage/shard/rebuild.go b/pkg/local_object_storage/shard/rebuild.go index 10eb51a28..0593f5894 100644 --- a/pkg/local_object_storage/shard/rebuild.go +++ b/pkg/local_object_storage/shard/rebuild.go @@ -6,10 +6,12 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -103,6 +105,7 @@ func runRebuild(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *lo default: } log.Info(ctx, logs.BlobstoreRebuildStarted) + ctx = tagging.ContextWithIOTag(ctx, qos.IOTagBackground.String()) if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, limiter, fillPercent); err != nil { log.Warn(ctx, logs.FailedToRebuildBlobstore, zap.Error(err)) } else { From a7145ca9bff4ef15abf3f7f170e420f806ebc310 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 10 Feb 2025 15:26:12 +0300 Subject: [PATCH 1723/1943] [#1614] adm/frostfsid: Add 'set-kv' Signed-off-by: Alexander Chuprov --- .../modules/morph/frostfsid/frostfsid.go | 43 +++++++++++++++++++ .../internal/modules/morph/frostfsid/root.go | 1 + 2 files changed, 44 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index b229d0436..974299154 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -1,6 +1,7 @@ package frostfsid import ( + "errors" "fmt" "math/big" "sort" @@ -38,6 +39,11 @@ const ( groupIDFlag = "group-id" rootNamespacePlaceholder = "" + + keyFlag = "key" + keyDescFlag = "Key for storing a value in the subject's KV storage" + valueFlag = "value" + valueDescFlag = "Value to be stored in the subject's KV storage" ) var ( @@ -151,6 +157,15 @@ var ( }, Run: frostfsidListGroupSubjects, } + + frostfsidSetKVCmd = &cobra.Command{ + Use: "set-kv", + Short: "Store a key-value pair in the subject's KV storage", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + }, + Run: frostfsidSetKV, + } ) func initFrostfsIDCreateNamespaceCmd() { @@ -236,6 +251,14 @@ func initFrostfsIDListGroupSubjectsCmd() { frostfsidListGroupSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") } +func initFrostfsIDSetKVCmd() { + Cmd.AddCommand(frostfsidSetKVCmd) + frostfsidSetKVCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + frostfsidSetKVCmd.Flags().String(subjectAddressFlag, "", "Subject address") + frostfsidSetKVCmd.Flags().String(keyFlag, "", keyDescFlag) + frostfsidSetKVCmd.Flags().String(valueFlag, "", valueDescFlag) +} + func frostfsidCreateNamespace(cmd *cobra.Command, _ []string) { ns := getFrostfsIDNamespace(cmd) @@ -403,6 +426,26 @@ func frostfsidRemoveSubjectFromGroup(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "remove subject from group error: %w", err) } +func frostfsidSetKV(cmd *cobra.Command, _ []string) { + subjectAddress := getFrostfsIDSubjectAddress(cmd) + key, _ := cmd.Flags().GetString(keyFlag) + value, _ := cmd.Flags().GetString(valueFlag) + + if key == "" { + commonCmd.ExitOnErr(cmd, "", errors.New("key can't be empty")) + } + + ffsid, err := newFrostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + + method, args := ffsid.roCli.SetSubjectKVCall(subjectAddress, key, value) + + ffsid.addCall(method, args) + + err = ffsid.sendWait() + commonCmd.ExitOnErr(cmd, "set KV: %w", err) +} + func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { ns := getFrostfsIDNamespace(cmd) groupID := getFrostfsIDGroupID(cmd) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go index 6ffcaa487..930865f81 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go @@ -12,6 +12,7 @@ func init() { initFrostfsIDAddSubjectToGroupCmd() initFrostfsIDRemoveSubjectFromGroupCmd() initFrostfsIDListGroupSubjectsCmd() + initFrostfsIDSetKVCmd() initFrostfsIDAddSubjectKeyCmd() initFrostfsIDRemoveSubjectKeyCmd() } From 076952f4c73a8a2d7388f658344d8090a91f64a9 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 10 Feb 2025 15:28:29 +0300 Subject: [PATCH 1724/1943] [#1614] adm/frostfsid: Add 'delete-kv' Signed-off-by: Alexander Chuprov --- .../modules/morph/frostfsid/frostfsid.go | 34 +++++++++++++++++++ .../internal/modules/morph/frostfsid/root.go | 1 + 2 files changed, 35 insertions(+) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 974299154..4fbd0bfe1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -166,6 +166,14 @@ var ( }, Run: frostfsidSetKV, } + frostfsidDeleteKVCmd = &cobra.Command{ + Use: "delete-kv", + Short: "Delete a value from the subject's KV storage", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + }, + Run: frostfsidDeleteKV, + } ) func initFrostfsIDCreateNamespaceCmd() { @@ -259,6 +267,13 @@ func initFrostfsIDSetKVCmd() { frostfsidSetKVCmd.Flags().String(valueFlag, "", valueDescFlag) } +func initFrostfsIDDeleteKVCmd() { + Cmd.AddCommand(frostfsidDeleteKVCmd) + frostfsidDeleteKVCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + frostfsidDeleteKVCmd.Flags().String(subjectAddressFlag, "", "Subject address") + frostfsidDeleteKVCmd.Flags().String(keyFlag, "", keyDescFlag) +} + func frostfsidCreateNamespace(cmd *cobra.Command, _ []string) { ns := getFrostfsIDNamespace(cmd) @@ -446,6 +461,25 @@ func frostfsidSetKV(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "set KV: %w", err) } +func frostfsidDeleteKV(cmd *cobra.Command, _ []string) { + subjectAddress := getFrostfsIDSubjectAddress(cmd) + key, _ := cmd.Flags().GetString(keyFlag) + + if key == "" { + commonCmd.ExitOnErr(cmd, "", errors.New("key can't be empty")) + } + + ffsid, err := newFrostfsIDClient(cmd) + commonCmd.ExitOnErr(cmd, "init contract client: %w", err) + + method, args := ffsid.roCli.DeleteSubjectKVCall(subjectAddress, key) + + ffsid.addCall(method, args) + + err = ffsid.sendWait() + commonCmd.ExitOnErr(cmd, "delete KV: %w", err) +} + func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { ns := getFrostfsIDNamespace(cmd) groupID := getFrostfsIDGroupID(cmd) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go index 930865f81..8aad5c5c1 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/root.go @@ -13,6 +13,7 @@ func init() { initFrostfsIDRemoveSubjectFromGroupCmd() initFrostfsIDListGroupSubjectsCmd() initFrostfsIDSetKVCmd() + initFrostfsIDDeleteKVCmd() initFrostfsIDAddSubjectKeyCmd() initFrostfsIDRemoveSubjectKeyCmd() } From b2163ff44c584f1f00bfc78272492b6e33fa3132 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 11 Feb 2025 12:00:25 +0300 Subject: [PATCH 1725/1943] [#1614] adm: Allow use any wallets Signed-off-by: Alexander Chuprov --- cmd/frostfs-adm/internal/commonflags/flags.go | 4 + .../internal/modules/morph/ape/ape_util.go | 5 +- .../internal/modules/morph/helper/actor.go | 88 +++++++++++++++---- .../internal/modules/morph/helper/util.go | 24 ++++- .../internal/modules/morph/nns/domains.go | 2 + .../internal/modules/morph/nns/helper.go | 26 +++++- .../internal/modules/morph/nns/record.go | 3 + .../internal/modules/morph/nns/root.go | 5 ++ 8 files changed, 135 insertions(+), 22 deletions(-) diff --git a/cmd/frostfs-adm/internal/commonflags/flags.go b/cmd/frostfs-adm/internal/commonflags/flags.go index 87692d013..c25d902c2 100644 --- a/cmd/frostfs-adm/internal/commonflags/flags.go +++ b/cmd/frostfs-adm/internal/commonflags/flags.go @@ -16,6 +16,10 @@ const ( EndpointFlagDesc = "N3 RPC node endpoint" EndpointFlagShort = "r" + WalletPath = "wallet" + WalletPathShorthand = "w" + WalletPathUsage = "Path to the wallet" + AlphabetWalletsFlag = "alphabet-wallets" AlphabetWalletsFlagDesc = "Path to alphabet wallets dir" diff --git a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go index 914682647..3c332c3f0 100644 --- a/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go +++ b/cmd/frostfs-adm/internal/modules/morph/ape/ape_util.go @@ -3,6 +3,8 @@ package ape import ( "errors" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -76,7 +78,8 @@ func newPolicyContractInterface(cmd *cobra.Command) (*morph.ContractStorage, *he c, err := helper.NewRemoteClient(viper.GetViper()) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) - ac, err := helper.NewLocalActor(cmd, c, constants.ConsensusAccountName) + walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) + ac, err := helper.NewLocalActor(c, &helper.AlphabetWallets{Path: walletDir, Label: constants.ConsensusAccountName}) commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) var ch util.Uint160 diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/actor.go b/cmd/frostfs-adm/internal/modules/morph/helper/actor.go index eb0444408..6499ace5f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/actor.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/actor.go @@ -3,9 +3,6 @@ package helper import ( "fmt" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" - commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -16,7 +13,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" "github.com/nspcc-dev/neo-go/pkg/wallet" - "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -28,32 +24,86 @@ type LocalActor struct { rpcInvoker invoker.RPCInvoke } +type AlphabetWallets struct { + Label string + Path string +} + +func (a *AlphabetWallets) GetAccount(v *viper.Viper) ([]*wallet.Account, error) { + w, err := GetAlphabetWallets(v, a.Path) + if err != nil { + return nil, err + } + + var accounts []*wallet.Account + for _, wall := range w { + acc, err := GetWalletAccount(wall, a.Label) + if err != nil { + return nil, err + } + accounts = append(accounts, acc) + } + return accounts, nil +} + +type RegularWallets struct{ Path string } + +func (r *RegularWallets) GetAccount() ([]*wallet.Account, error) { + w, err := getRegularWallet(r.Path) + if err != nil { + return nil, err + } + + return []*wallet.Account{w.GetAccount(w.GetChangeAddress())}, nil +} + // NewLocalActor create LocalActor with accounts form provided wallets. // In case of empty wallets provided created actor with dummy account only for read operation. // // If wallets are provided, the contract client will use accounts with accName name from these wallets. // To determine which account name should be used in a contract client, refer to how the contract // verifies the transaction signature. -func NewLocalActor(cmd *cobra.Command, c actor.RPCActor, accName string) (*LocalActor, error) { - walletDir := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) +func NewLocalActor(c actor.RPCActor, alphabet *AlphabetWallets, regularWallets ...*RegularWallets) (*LocalActor, error) { var act *actor.Actor var accounts []*wallet.Account + var signers []actor.SignerAccount - wallets, err := GetAlphabetWallets(viper.GetViper(), walletDir) - commonCmd.ExitOnErr(cmd, "unable to get alphabet wallets: %w", err) + if alphabet != nil { + account, err := alphabet.GetAccount(viper.GetViper()) + if err != nil { + return nil, err + } - for _, w := range wallets { - acc, err := GetWalletAccount(w, accName) - commonCmd.ExitOnErr(cmd, fmt.Sprintf("can't find %s account: %%w", accName), err) - accounts = append(accounts, acc) + accounts = append(accounts, account...) + signers = append(signers, actor.SignerAccount{ + Signer: transaction.Signer{ + Account: account[0].Contract.ScriptHash(), + Scopes: transaction.Global, + }, + Account: account[0], + }) } - act, err = actor.New(c, []actor.SignerAccount{{ - Signer: transaction.Signer{ - Account: accounts[0].Contract.ScriptHash(), - Scopes: transaction.Global, - }, - Account: accounts[0], - }}) + + for _, w := range regularWallets { + if w == nil { + continue + } + account, err := w.GetAccount() + if err != nil { + return nil, err + } + + accounts = append(accounts, account...) + signers = append(signers, actor.SignerAccount{ + Signer: transaction.Signer{ + Account: account[0].Contract.ScriptHash(), + Scopes: transaction.Global, + }, + Account: account[0], + }) + } + + act, err := actor.New(c, signers) if err != nil { return nil, err } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/util.go b/cmd/frostfs-adm/internal/modules/morph/helper/util.go index c26aa447b..be6b2c6dd 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/util.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/util.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" + "github.com/nspcc-dev/neo-go/cli/input" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" @@ -22,6 +23,27 @@ import ( "github.com/spf13/viper" ) +func getRegularWallet(walletPath string) (*wallet.Wallet, error) { + w, err := wallet.NewWalletFromFile(walletPath) + if err != nil { + return nil, err + } + + password, err := input.ReadPassword("Enter password for wallet:") + if err != nil { + return nil, fmt.Errorf("can't fetch password: %w", err) + } + + for i := range w.Accounts { + if err = w.Accounts[i].Decrypt(password, keys.NEP2ScryptParams()); err != nil { + err = fmt.Errorf("can't unlock wallet: %w", err) + break + } + } + + return w, err +} + func GetAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, error) { wallets, err := openAlphabetWallets(v, walletDir) if err != nil { @@ -51,7 +73,7 @@ func openAlphabetWallets(v *viper.Viper, walletDir string) ([]*wallet.Wallet, er if errors.Is(err, os.ErrNotExist) { err = nil } else { - err = fmt.Errorf("can't open wallet: %w", err) + err = fmt.Errorf("can't open alphabet wallet: %w", err) } break } diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/domains.go b/cmd/frostfs-adm/internal/modules/morph/nns/domains.go index 1668bb327..e25d0db49 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/domains.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/domains.go @@ -19,6 +19,7 @@ func initRegisterCmd() { registerCmd.Flags().Int64(nnsRetryFlag, constants.NNSRetryDefVal, "SOA record RETRY parameter") registerCmd.Flags().Int64(nnsExpireFlag, int64(constants.DefaultExpirationTime), "SOA record EXPIRE parameter") registerCmd.Flags().Int64(nnsTTLFlag, constants.NNSTtlDefVal, "SOA record TTL parameter") + registerCmd.Flags().StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, "", commonflags.WalletPathUsage) _ = cobra.MarkFlagRequired(registerCmd.Flags(), nnsNameFlag) } @@ -48,6 +49,7 @@ func initDeleteCmd() { deleteCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) deleteCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) deleteCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) + deleteCmd.Flags().StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, "", commonflags.WalletPathUsage) _ = cobra.MarkFlagRequired(deleteCmd.Flags(), nnsNameFlag) } diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go index b13cbc8a1..5cf02e866 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go @@ -1,7 +1,11 @@ package nns import ( + "errors" + client "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/nns" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" @@ -16,7 +20,27 @@ func nnsWriter(cmd *cobra.Command) (*client.Contract, *helper.LocalActor) { c, err := helper.NewRemoteClient(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) - ac, err := helper.NewLocalActor(cmd, c, constants.CommitteeAccountName) + alphabetWalletPath := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) + walletPath := config.ResolveHomePath(viper.GetString(commonflags.WalletPath)) + + var ( + alphabet *helper.AlphabetWallets + regularWallets []*helper.RegularWallets + ) + + if alphabetWalletPath != "" { + alphabet = &helper.AlphabetWallets{Path: alphabetWalletPath, Label: constants.ConsensusAccountName} + } + + if walletPath != "" { + regularWallets = append(regularWallets, &helper.RegularWallets{Path: walletPath}) + } + + if alphabet == nil && regularWallets == nil { + commonCmd.ExitOnErr(cmd, "", errors.New("no wallets provided")) + } + + ac, err := helper.NewLocalActor(c, alphabet, regularWallets...) commonCmd.ExitOnErr(cmd, "can't create actor: %w", err) r := management.NewReader(ac.Invoker) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/record.go b/cmd/frostfs-adm/internal/modules/morph/nns/record.go index 09ed92ab3..9cb47356f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/record.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/record.go @@ -19,6 +19,7 @@ func initAddRecordCmd() { addRecordCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) addRecordCmd.Flags().String(nnsRecordTypeFlag, "", nnsRecordTypeFlagDesc) addRecordCmd.Flags().String(nnsRecordDataFlag, "", nnsRecordDataFlagDesc) + addRecordCmd.Flags().StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, "", commonflags.WalletPathUsage) _ = cobra.MarkFlagRequired(addRecordCmd.Flags(), nnsNameFlag) _ = cobra.MarkFlagRequired(addRecordCmd.Flags(), nnsRecordTypeFlag) @@ -40,6 +41,7 @@ func initDelRecordsCmd() { delRecordsCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) delRecordsCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) delRecordsCmd.Flags().String(nnsRecordTypeFlag, "", nnsRecordTypeFlagDesc) + delRecordsCmd.Flags().StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, "", commonflags.WalletPathUsage) _ = cobra.MarkFlagRequired(delRecordsCmd.Flags(), nnsNameFlag) _ = cobra.MarkFlagRequired(delRecordsCmd.Flags(), nnsRecordTypeFlag) @@ -52,6 +54,7 @@ func initDelRecordCmd() { delRecordCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) delRecordCmd.Flags().String(nnsRecordTypeFlag, "", nnsRecordTypeFlagDesc) delRecordCmd.Flags().String(nnsRecordDataFlag, "", nnsRecordDataFlagDesc) + delRecordCmd.Flags().StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, "", commonflags.WalletPathUsage) _ = cobra.MarkFlagRequired(delRecordCmd.Flags(), nnsNameFlag) _ = cobra.MarkFlagRequired(delRecordCmd.Flags(), nnsRecordTypeFlag) diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/root.go b/cmd/frostfs-adm/internal/modules/morph/nns/root.go index 9bdeaccd9..64a279288 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/root.go @@ -39,6 +39,7 @@ var ( PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.WalletPath, cmd.Flags().Lookup(commonflags.WalletPath)) }, Run: registerDomain, } @@ -48,6 +49,7 @@ var ( PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.WalletPath, cmd.Flags().Lookup(commonflags.WalletPath)) }, Run: deleteDomain, } @@ -75,6 +77,7 @@ var ( PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.WalletPath, cmd.Flags().Lookup(commonflags.WalletPath)) }, Run: addRecord, } @@ -92,6 +95,7 @@ var ( PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.WalletPath, cmd.Flags().Lookup(commonflags.WalletPath)) }, Run: delRecords, } @@ -101,6 +105,7 @@ var ( PreRun: func(cmd *cobra.Command, _ []string) { _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.WalletPath, cmd.Flags().Lookup(commonflags.WalletPath)) }, Run: delRecord, } From 304bee938ba7fbd1ebbbef2b9996a0ae87950be6 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 11 Feb 2025 12:00:51 +0300 Subject: [PATCH 1726/1943] [#1614] adm/nns: Add 'set-admin' Signed-off-by: Alexander Chuprov --- cmd/frostfs-adm/internal/commonflags/flags.go | 3 +++ .../internal/modules/morph/nns/domains.go | 27 +++++++++++++++++++ .../internal/modules/morph/nns/helper.go | 9 +++++-- .../internal/modules/morph/nns/root.go | 12 +++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/commonflags/flags.go b/cmd/frostfs-adm/internal/commonflags/flags.go index c25d902c2..f194e97f5 100644 --- a/cmd/frostfs-adm/internal/commonflags/flags.go +++ b/cmd/frostfs-adm/internal/commonflags/flags.go @@ -23,6 +23,9 @@ const ( AlphabetWalletsFlag = "alphabet-wallets" AlphabetWalletsFlagDesc = "Path to alphabet wallets dir" + AdminWalletPath = "wallet-admin" + AdminWalletUsage = "Path to the admin wallet" + LocalDumpFlag = "local-dump" ProtoConfigPath = "protocol" ContractsInitFlag = "contracts" diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/domains.go b/cmd/frostfs-adm/internal/modules/morph/nns/domains.go index e25d0db49..14f6eb390 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/domains.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/domains.go @@ -6,7 +6,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" + "github.com/spf13/viper" ) func initRegisterCmd() { @@ -64,3 +66,28 @@ func deleteDomain(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "delete domain error: %w", err) cmd.Println("Domain deleted successfully") } + +func initSetAdminCmd() { + Cmd.AddCommand(setAdminCmd) + setAdminCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + setAdminCmd.Flags().String(commonflags.AlphabetWalletsFlag, "", commonflags.AlphabetWalletsFlagDesc) + setAdminCmd.Flags().String(nnsNameFlag, "", nnsNameFlagDesc) + setAdminCmd.Flags().StringP(commonflags.WalletPath, commonflags.WalletPathShorthand, "", commonflags.WalletPathUsage) + setAdminCmd.Flags().String(commonflags.AdminWalletPath, "", commonflags.AdminWalletUsage) + _ = setAdminCmd.MarkFlagRequired(commonflags.AdminWalletPath) + + _ = cobra.MarkFlagRequired(setAdminCmd.Flags(), nnsNameFlag) +} + +func setAdmin(cmd *cobra.Command, _ []string) { + c, actor := nnsWriter(cmd) + + name, _ := cmd.Flags().GetString(nnsNameFlag) + w, err := wallet.NewWalletFromFile(viper.GetString(commonflags.AdminWalletPath)) + commonCmd.ExitOnErr(cmd, "can't get admin wallet: %w", err) + h, vub, err := c.SetAdmin(name, w.GetAccount(w.GetChangeAddress()).ScriptHash()) + + _, err = actor.Wait(h, vub, err) + commonCmd.ExitOnErr(cmd, "Set admin error: %w", err) + cmd.Println("Set admin successfully") +} diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go index 5cf02e866..e49f62256 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/helper.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/helper.go @@ -20,8 +20,9 @@ func nnsWriter(cmd *cobra.Command) (*client.Contract, *helper.LocalActor) { c, err := helper.NewRemoteClient(v) commonCmd.ExitOnErr(cmd, "unable to create NEO rpc client: %w", err) - alphabetWalletPath := config.ResolveHomePath(viper.GetString(commonflags.AlphabetWalletsFlag)) - walletPath := config.ResolveHomePath(viper.GetString(commonflags.WalletPath)) + alphabetWalletPath := config.ResolveHomePath(v.GetString(commonflags.AlphabetWalletsFlag)) + walletPath := config.ResolveHomePath(v.GetString(commonflags.WalletPath)) + adminWalletPath := config.ResolveHomePath(v.GetString(commonflags.AdminWalletPath)) var ( alphabet *helper.AlphabetWallets @@ -36,6 +37,10 @@ func nnsWriter(cmd *cobra.Command) (*client.Contract, *helper.LocalActor) { regularWallets = append(regularWallets, &helper.RegularWallets{Path: walletPath}) } + if adminWalletPath != "" { + regularWallets = append(regularWallets, &helper.RegularWallets{Path: adminWalletPath}) + } + if alphabet == nil && regularWallets == nil { commonCmd.ExitOnErr(cmd, "", errors.New("no wallets provided")) } diff --git a/cmd/frostfs-adm/internal/modules/morph/nns/root.go b/cmd/frostfs-adm/internal/modules/morph/nns/root.go index 64a279288..bb84933c6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/nns/root.go +++ b/cmd/frostfs-adm/internal/modules/morph/nns/root.go @@ -109,6 +109,17 @@ var ( }, Run: delRecord, } + setAdminCmd = &cobra.Command{ + Use: "set-admin", + Short: "Sets admin for domain", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.AlphabetWalletsFlag, cmd.Flags().Lookup(commonflags.AlphabetWalletsFlag)) + _ = viper.BindPFlag(commonflags.WalletPath, cmd.Flags().Lookup(commonflags.WalletPath)) + _ = viper.BindPFlag(commonflags.AdminWalletPath, cmd.Flags().Lookup(commonflags.AdminWalletPath)) + }, + Run: setAdmin, + } ) func init() { @@ -121,4 +132,5 @@ func init() { initGetRecordsCmd() initDelRecordsCmd() initDelRecordCmd() + initSetAdminCmd() } From 1bcaa1af1f4ff32b7a7f7bdfbef1ca62954361a2 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Mon, 10 Feb 2025 18:30:18 +0300 Subject: [PATCH 1727/1943] [#1641] govulncheck: Fix minor toolchain updates for good Signed-off-by: Vitaliy Potyarkin --- .forgejo/workflows/vulncheck.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 7c89a3555..140434dfc 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -18,7 +18,8 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.23.6' + go-version: '1.23' + check-latest: true - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest From fe0cf86dc681e221706d9dcb4b25413bfdf065e9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 12 Feb 2025 15:37:38 +0300 Subject: [PATCH 1728/1943] [#1643] go.mod: Bump frostfs-observability version To add `grpc_client_msg_send_handling_seconds` metric. Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8f2e7bc51..5df7a5cc1 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1-0.20241205083807-762d7f9f9f08 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250202151421-8389887a3421 git.frostfs.info/TrueCloudLab/hrw v1.2.1 diff --git a/go.sum b/go.sum index e92b64886..39a6a98e6 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d h1:uJ/wvuMdepbkaV8XMS5uN9B0FQWMep0CttSuDZiDhq0= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88 h1:9bvBDLApbbO5sXBKdODpE9tzy3HV99nXxkDWNn22rdI= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20241112082307-f17779933e88/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 h1:Mxw1c/8t96vFIUOffl28lFaHKi413oCBfLMGJmF9cFA= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe h1:81gDNdWNLP24oMQukRiCE9R1wGSh0l0dRq3F1W+Oesc= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250202151421-8389887a3421 h1:pP19IawSdsLCKFv7HMNfWAeH6E3uSnntKZkwka+/2+4= From 9b29e7392fb960f7d3c923f17ce29d5475ef9562 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 18 Feb 2025 09:38:00 +0300 Subject: [PATCH 1729/1943] [#1647] go.mod: Bump frostfs-sdk-go version Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5df7a5cc1..23ddad276 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250202151421-8389887a3421 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 diff --git a/go.sum b/go.sum index 39a6a98e6..ecd2ab525 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e13 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe h1:81gDNdWNLP24oMQukRiCE9R1wGSh0l0dRq3F1W+Oesc= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250202151421-8389887a3421 h1:pP19IawSdsLCKFv7HMNfWAeH6E3uSnntKZkwka+/2+4= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250202151421-8389887a3421/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4 h1:dOZHuOywvH1ms8U38lDCWpysgkCCeJ02RLI7zDhPcyw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 02f3a7f65c0d649f29db04f5c49703e48621b95b Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Tue, 18 Feb 2025 10:51:43 +0300 Subject: [PATCH 1730/1943] [#1648] writecache: Fix race condition when reporting cache size metrics There is a race condition when multiple cache operation try to report the cache size metrics simultaneously. Consider the following example: - the initial total size of objects stored in the cache size is 2 - worker X deletes an object and reads the cache size, which is 1 - worker Y deletes an object and reads the cache size, which is 0 - worker Y reports the cache size it learnt, which is 0 - worker X reports the cache size it learnt, which is 1 As a result, the observed cache size is 1 (i. e. one object remains in the cache), which is incorrect because the actual cache size is 0. To fix this, let's report the metrics periodically in the flush loop. Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/writecache/flush.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 3f9b36f9d..2d07d8b32 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -87,6 +87,9 @@ func (c *cache) pushToFlushQueue(ctx context.Context, fl *flushLimiter) { } c.modeMtx.RUnlock() + + // counter changed by fstree + c.estimateCacheSize() case <-ctx.Done(): return } From b2adf1109e380c364b8132ee7fa1947094c65c86 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 20 Feb 2025 12:27:07 +0300 Subject: [PATCH 1731/1943] [#1646] cli: Use Cmp() functions for ID-like structs Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/internal/client/client.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index ceae36ae7..3f235f070 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -9,7 +9,6 @@ import ( "io" "os" "slices" - "strings" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/accounting" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" @@ -77,9 +76,7 @@ func ListContainers(ctx context.Context, prm ListContainersPrm) (res ListContain // SortedIDList returns sorted list of identifiers of user's containers. func (x ListContainersRes) SortedIDList() []cid.ID { list := x.cliRes.Containers() - slices.SortFunc(list, func(lhs, rhs cid.ID) int { - return strings.Compare(lhs.EncodeToString(), rhs.EncodeToString()) - }) + slices.SortFunc(list, cid.ID.Cmp) return list } @@ -687,9 +684,7 @@ func SearchObjects(ctx context.Context, prm SearchObjectsPrm) (*SearchObjectsRes return nil, fmt.Errorf("read object list: %w", err) } - slices.SortFunc(list, func(a, b oid.ID) int { - return strings.Compare(a.EncodeToString(), b.EncodeToString()) - }) + slices.SortFunc(list, oid.ID.Cmp) return &SearchObjectsRes{ ids: list, From 003d568ae2361e7b675091b2a0dff36b490a6013 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 30 Jan 2025 16:27:48 +0300 Subject: [PATCH 1732/1943] [#1628] innerring: Relax container homomorphic hashing check Our initial desire was to prohibit using homomorphic hashing on the network level because of the resource consumption. However, the ability to use it, doesn't mean that we must. So only fail validation if container wants to have homomorphic hashing, but the network prohibits it. Signed-off-by: Evgenii Stratonikov --- pkg/innerring/processors/container/process_container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/innerring/processors/container/process_container.go b/pkg/innerring/processors/container/process_container.go index 854e2c779..8e4ab2623 100644 --- a/pkg/innerring/processors/container/process_container.go +++ b/pkg/innerring/processors/container/process_container.go @@ -209,7 +209,7 @@ func checkHomomorphicHashing(ctx context.Context, ns NetworkState, cnr container return fmt.Errorf("could not get setting in contract: %w", err) } - if cnrSetting := containerSDK.IsHomomorphicHashingDisabled(cnr); netSetting != cnrSetting { + if cnrSetting := containerSDK.IsHomomorphicHashingDisabled(cnr); netSetting && !cnrSetting { return fmt.Errorf("network setting: %t, container setting: %t", netSetting, cnrSetting) } From 37972a91c19c6aa1b44c322bb5e485d553f031a9 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 24 Feb 2025 12:47:09 +0300 Subject: [PATCH 1733/1943] [#1654] adm: Make 'morph-init' idempotent Signed-off-by: Alexander Chuprov --- .../modules/morph/initialize/initialize_transfer.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go index 7f1bfee2b..d379204a6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go @@ -83,11 +83,14 @@ func transferFunds(c *helper.InitializeContext) error { // transferFundsFinished checks balances of accounts we transfer GAS to. // The stage is considered finished if the balance is greater than the half of what we need to transfer. func transferFundsFinished(c *helper.InitializeContext) (bool, error) { - acc := c.Accounts[0] - r := nep17.NewReader(c.ReadOnlyInvoker, gas.Hash) - res, err := r.BalanceOf(acc.Contract.ScriptHash()) - if err != nil || res.Cmp(big.NewInt(initialAlphabetGASAmount/2)) != 1 { + res, err := r.BalanceOf(c.ConsensusAcc.ScriptHash()) + if err != nil { + return false, err + } + + version, err := c.Client.GetVersion() + if err != nil || res.Cmp(big.NewInt(int64(version.Protocol.InitialGasDistribution))) != -1 { return false, err } From 2ff032db90de1c1c2af24a9231eeafaabbe5d643 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 24 Feb 2025 13:22:23 +0300 Subject: [PATCH 1734/1943] [#1654] adm: Get 'gasInitialTotalSupply' from network Signed-off-by: Alexander Chuprov --- .../morph/initialize/initialize_transfer.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go index d379204a6..bb684b3a9 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_transfer.go @@ -22,15 +22,14 @@ import ( ) const ( - gasInitialTotalSupply = 30000000 * native.GASFactor // initialAlphabetGASAmount represents the amount of GAS given to each alphabet node. initialAlphabetGASAmount = 10_000 * native.GASFactor // initialProxyGASAmount represents the amount of GAS given to a proxy contract. initialProxyGASAmount = 50_000 * native.GASFactor ) -func initialCommitteeGASAmount(c *helper.InitializeContext) int64 { - return (gasInitialTotalSupply - initialAlphabetGASAmount*int64(len(c.Wallets))) / 2 +func initialCommitteeGASAmount(c *helper.InitializeContext, initialGasDistribution int64) int64 { + return (initialGasDistribution - initialAlphabetGASAmount*int64(len(c.Wallets))) / 2 } func transferFunds(c *helper.InitializeContext) error { @@ -42,6 +41,11 @@ func transferFunds(c *helper.InitializeContext) error { return err } + version, err := c.Client.GetVersion() + if err != nil { + return err + } + var transfers []transferTarget for _, acc := range c.Accounts { to := acc.Contract.ScriptHash() @@ -59,7 +63,7 @@ func transferFunds(c *helper.InitializeContext) error { transferTarget{ Token: gas.Hash, Address: c.CommitteeAcc.Contract.ScriptHash(), - Amount: initialCommitteeGASAmount(c), + Amount: initialCommitteeGASAmount(c, int64(version.Protocol.InitialGasDistribution)), }, transferTarget{ Token: neo.Hash, @@ -95,7 +99,11 @@ func transferFundsFinished(c *helper.InitializeContext) (bool, error) { } res, err = r.BalanceOf(c.CommitteeAcc.ScriptHash()) - return res != nil && res.Cmp(big.NewInt(initialCommitteeGASAmount(c)/2)) == 1, err + if err != nil { + return false, err + } + + return res != nil && res.Cmp(big.NewInt(initialCommitteeGASAmount(c, int64(version.Protocol.InitialGasDistribution)))) == 1, err } func transferGASToProxy(c *helper.InitializeContext) error { From 9a0507704ae0b5069ab6555730fb758f71d04618 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 20 Feb 2025 17:47:28 +0300 Subject: [PATCH 1735/1943] [#1651] apemanager: Wrap some errors with `InvalidArgument` status Signed-off-by: Airat Arifullin --- pkg/services/apemanager/errors/errors.go | 6 ++++++ pkg/services/apemanager/executor.go | 14 +++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/services/apemanager/errors/errors.go b/pkg/services/apemanager/errors/errors.go index e64f9a8d1..1d485321c 100644 --- a/pkg/services/apemanager/errors/errors.go +++ b/pkg/services/apemanager/errors/errors.go @@ -9,3 +9,9 @@ func ErrAPEManagerAccessDenied(reason string) error { err.WriteReason(reason) return err } + +func ErrAPEManagerInvalidArgument(msg string) error { + err := new(apistatus.InvalidArgument) + err.SetMessage(msg) + return err +} diff --git a/pkg/services/apemanager/executor.go b/pkg/services/apemanager/executor.go index 58922fede..fc08fe569 100644 --- a/pkg/services/apemanager/executor.go +++ b/pkg/services/apemanager/executor.go @@ -81,7 +81,7 @@ var _ Server = (*Service)(nil) func (s *Service) validateContainerTargetRequest(ctx context.Context, cid string, pubKey *keys.PublicKey) error { var cidSDK cidSDK.ID if err := cidSDK.DecodeString(cid); err != nil { - return fmt.Errorf("invalid CID format: %w", err) + return apemanager_errors.ErrAPEManagerInvalidArgument(fmt.Sprintf("invalid CID format: %v", err)) } isOwner, err := s.isActorContainerOwner(ctx, cidSDK, pubKey) if err != nil { @@ -101,7 +101,7 @@ func (s *Service) AddChain(ctx context.Context, req *apemanagerV2.AddChainReques chain, err := decodeAndValidateChain(req.GetBody().GetChain().GetKind().(*apeV2.ChainRaw).GetRaw()) if err != nil { - return nil, err + return nil, apemanager_errors.ErrAPEManagerInvalidArgument(err.Error()) } if len(chain.ID) == 0 { const randomIDLength = 10 @@ -122,7 +122,7 @@ func (s *Service) AddChain(ctx context.Context, req *apemanagerV2.AddChainReques } target = policy_engine.ContainerTarget(reqCID) default: - return nil, fmt.Errorf("unsupported target type: %s", targetType) + return nil, apemanager_errors.ErrAPEManagerInvalidArgument(fmt.Sprintf("unsupported target type: %s", targetType)) } txHash, vub, err := s.contractStorage.AddMorphRuleChain(apechain.Ingress, target, &chain) @@ -158,7 +158,7 @@ func (s *Service) RemoveChain(ctx context.Context, req *apemanagerV2.RemoveChain } target = policy_engine.ContainerTarget(reqCID) default: - return nil, fmt.Errorf("unsupported target type: %s", targetType) + return nil, apemanager_errors.ErrAPEManagerInvalidArgument(fmt.Sprintf("unsupported target type: %s", targetType)) } txHash, vub, err := s.contractStorage.RemoveMorphRuleChain(apechain.Ingress, target, req.GetBody().GetChainID()) @@ -193,7 +193,7 @@ func (s *Service) ListChains(ctx context.Context, req *apemanagerV2.ListChainsRe } target = policy_engine.ContainerTarget(reqCID) default: - return nil, fmt.Errorf("unsupported target type: %s", targetType) + return nil, apemanager_errors.ErrAPEManagerInvalidArgument(fmt.Sprintf("unsupported target type: %s", targetType)) } chs, err := s.contractStorage.ListMorphRuleChains(apechain.Ingress, target) @@ -227,11 +227,11 @@ func getSignaturePublicKey(vh *session.RequestVerificationHeader) (*keys.PublicK } sig := vh.GetBodySignature() if sig == nil { - return nil, errEmptyBodySignature + return nil, apemanager_errors.ErrAPEManagerInvalidArgument(errEmptyBodySignature.Error()) } key, err := keys.NewPublicKeyFromBytes(sig.GetKey(), elliptic.P256()) if err != nil { - return nil, fmt.Errorf("invalid signature key: %w", err) + return nil, apemanager_errors.ErrAPEManagerInvalidArgument(fmt.Sprintf("invalid signature key: %v", err)) } return key, nil From a97bded440c345c5212f7aaa9c9a89785b22cd32 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 6 Feb 2025 11:09:33 +0300 Subject: [PATCH 1736/1943] [#1639] config: Separate `replicator.pool_size` from other settings Separated `replicator.pool_size` and `object.put.remote_pool_size` settings. Signed-off-by: Aleksey Savchuk --- cmd/frostfs-node/config.go | 4 ---- cmd/frostfs-node/config/replicator/config.go | 11 ++++++++++- cmd/frostfs-node/config/replicator/config_test.go | 2 +- docs/storage-node-configuration.md | 8 ++++---- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 511777566..d575c7228 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1177,10 +1177,6 @@ func initObjectPool(cfg *config.Config) (pool cfgObjectRoutines) { fatalOnErr(err) replicatorPoolSize := replicatorconfig.PoolSize(cfg) - if replicatorPoolSize <= 0 { - replicatorPoolSize = putRemoteCapacity - } - pool.replication, err = ants.NewPool(replicatorPoolSize) fatalOnErr(err) diff --git a/cmd/frostfs-node/config/replicator/config.go b/cmd/frostfs-node/config/replicator/config.go index 0fbac935c..e954bf19d 100644 --- a/cmd/frostfs-node/config/replicator/config.go +++ b/cmd/frostfs-node/config/replicator/config.go @@ -11,6 +11,8 @@ const ( // PutTimeoutDefault is a default timeout of object put request in replicator. PutTimeoutDefault = 5 * time.Second + // PoolSizeDefault is a default pool size for put request in replicator. + PoolSizeDefault = 10 ) // PutTimeout returns the value of "put_timeout" config parameter @@ -28,6 +30,13 @@ func PutTimeout(c *config.Config) time.Duration { // PoolSize returns the value of "pool_size" config parameter // from "replicator" section. +// +// Returns PoolSizeDefault if the value is non-positive integer. func PoolSize(c *config.Config) int { - return int(config.IntSafe(c.Sub(subsection), "pool_size")) + v := int(config.IntSafe(c.Sub(subsection), "pool_size")) + if v > 0 { + return v + } + + return PoolSizeDefault } diff --git a/cmd/frostfs-node/config/replicator/config_test.go b/cmd/frostfs-node/config/replicator/config_test.go index 2129c01b4..2aa490946 100644 --- a/cmd/frostfs-node/config/replicator/config_test.go +++ b/cmd/frostfs-node/config/replicator/config_test.go @@ -15,7 +15,7 @@ func TestReplicatorSection(t *testing.T) { empty := configtest.EmptyConfig() require.Equal(t, replicatorconfig.PutTimeoutDefault, replicatorconfig.PutTimeout(empty)) - require.Equal(t, 0, replicatorconfig.PoolSize(empty)) + require.Equal(t, replicatorconfig.PoolSizeDefault, replicatorconfig.PoolSize(empty)) }) const path = "../../../../config/example/node" diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index de2729c68..aef05d589 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -396,10 +396,10 @@ replicator: pool_size: 10 ``` -| Parameter | Type | Default value | Description | -|---------------|------------|----------------------------------------|---------------------------------------------| -| `put_timeout` | `duration` | `5s` | Timeout for performing the `PUT` operation. | -| `pool_size` | `int` | Equal to `object.put.remote_pool_size` | Maximum amount of concurrent replications. | +| Parameter | Type | Default value | Description | +|---------------|------------|---------------|---------------------------------------------| +| `put_timeout` | `duration` | `5s` | Timeout for performing the `PUT` operation. | +| `pool_size` | `int` | `10` | Maximum amount of concurrent replications. | # `object` section Contains object-service related parameters. From dce269c62e555a08eb6e768b8fb25d7307564991 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 6 Feb 2025 11:50:12 +0300 Subject: [PATCH 1737/1943] [#1639] services/object: Remove limiting pools for Put operation Signed-off-by: Aleksey Savchuk --- cmd/frostfs-node/config.go | 22 +----------- cmd/frostfs-node/config/object/config.go | 28 --------------- cmd/frostfs-node/config/object/config_test.go | 4 --- cmd/frostfs-node/object.go | 1 - config/example/node.env | 2 -- config/example/node.json | 2 -- config/example/node.yaml | 2 -- docs/storage-node-configuration.md | 4 --- internal/logs/logs.go | 1 - pkg/services/object/common/writer/common.go | 10 ++---- pkg/services/object/common/writer/ec.go | 36 ++----------------- pkg/services/object/common/writer/ec_test.go | 5 --- pkg/services/object/common/writer/writer.go | 16 --------- pkg/services/object/put/service.go | 3 -- pkg/services/object/util/log.go | 8 ----- 15 files changed, 7 insertions(+), 137 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d575c7228..43ec40d1d 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -664,10 +664,6 @@ type cfgAccessPolicyEngine struct { } type cfgObjectRoutines struct { - putRemote *ants.Pool - - putLocal *ants.Pool - replication *ants.Pool } @@ -1166,16 +1162,6 @@ func initAccessPolicyEngine(ctx context.Context, c *cfg) { func initObjectPool(cfg *config.Config) (pool cfgObjectRoutines) { var err error - optNonBlocking := ants.WithNonblocking(true) - - putRemoteCapacity := objectconfig.Put(cfg).PoolSizeRemote() - pool.putRemote, err = ants.NewPool(putRemoteCapacity, optNonBlocking) - fatalOnErr(err) - - putLocalCapacity := objectconfig.Put(cfg).PoolSizeLocal() - pool.putLocal, err = ants.NewPool(putLocalCapacity, optNonBlocking) - fatalOnErr(err) - replicatorPoolSize := replicatorconfig.PoolSize(cfg) pool.replication, err = ants.NewPool(replicatorPoolSize) fatalOnErr(err) @@ -1410,13 +1396,7 @@ func (c *cfg) getComponents(ctx context.Context, logPrm *logger.Prm) []dCmp { } func (c *cfg) reloadPools() error { - newSize := objectconfig.Put(c.appCfg).PoolSizeLocal() - c.reloadPool(c.cfgObject.pool.putLocal, newSize, "object.put.local_pool_size") - - newSize = objectconfig.Put(c.appCfg).PoolSizeRemote() - c.reloadPool(c.cfgObject.pool.putRemote, newSize, "object.put.remote_pool_size") - - newSize = replicatorconfig.PoolSize(c.appCfg) + newSize := replicatorconfig.PoolSize(c.appCfg) c.reloadPool(c.cfgObject.pool.replication, newSize, "replicator.pool_size") return nil diff --git a/cmd/frostfs-node/config/object/config.go b/cmd/frostfs-node/config/object/config.go index 6ff1fe2ab..c8c967d30 100644 --- a/cmd/frostfs-node/config/object/config.go +++ b/cmd/frostfs-node/config/object/config.go @@ -21,10 +21,6 @@ const ( putSubsection = "put" getSubsection = "get" - - // PutPoolSizeDefault is a default value of routine pool size to - // process object.Put requests in object service. - PutPoolSizeDefault = 10 ) // Put returns structure that provides access to "put" subsection of @@ -35,30 +31,6 @@ func Put(c *config.Config) PutConfig { } } -// PoolSizeRemote returns the value of "remote_pool_size" config parameter. -// -// Returns PutPoolSizeDefault if the value is not a positive number. -func (g PutConfig) PoolSizeRemote() int { - v := config.Int(g.cfg, "remote_pool_size") - if v > 0 { - return int(v) - } - - return PutPoolSizeDefault -} - -// PoolSizeLocal returns the value of "local_pool_size" config parameter. -// -// Returns PutPoolSizeDefault if the value is not a positive number. -func (g PutConfig) PoolSizeLocal() int { - v := config.Int(g.cfg, "local_pool_size") - if v > 0 { - return int(v) - } - - return PutPoolSizeDefault -} - // SkipSessionTokenIssuerVerification returns the value of "skip_session_token_issuer_verification" config parameter or `false“ if is not defined. func (g PutConfig) SkipSessionTokenIssuerVerification() bool { return config.BoolSafe(g.cfg, "skip_session_token_issuer_verification") diff --git a/cmd/frostfs-node/config/object/config_test.go b/cmd/frostfs-node/config/object/config_test.go index e2bb105d9..1c525ef55 100644 --- a/cmd/frostfs-node/config/object/config_test.go +++ b/cmd/frostfs-node/config/object/config_test.go @@ -13,8 +13,6 @@ func TestObjectSection(t *testing.T) { t.Run("defaults", func(t *testing.T) { empty := configtest.EmptyConfig() - require.Equal(t, objectconfig.PutPoolSizeDefault, objectconfig.Put(empty).PoolSizeRemote()) - require.Equal(t, objectconfig.PutPoolSizeDefault, objectconfig.Put(empty).PoolSizeLocal()) require.EqualValues(t, objectconfig.DefaultTombstoneLifetime, objectconfig.TombstoneLifetime(empty)) require.False(t, objectconfig.Put(empty).SkipSessionTokenIssuerVerification()) }) @@ -22,8 +20,6 @@ func TestObjectSection(t *testing.T) { const path = "../../../../config/example/node" fileConfigTest := func(c *config.Config) { - require.Equal(t, 100, objectconfig.Put(c).PoolSizeRemote()) - require.Equal(t, 200, objectconfig.Put(c).PoolSizeLocal()) require.EqualValues(t, 10, objectconfig.TombstoneLifetime(c)) require.True(t, objectconfig.Put(c).SkipSessionTokenIssuerVerification()) } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 40d3cc1cd..ad6f4140a 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -326,7 +326,6 @@ func createPutSvc(c *cfg, keyStorage *util.KeyStorage, irFetcher *cachedIRFetche c, c.cfgNetmap.state, irFetcher, - objectwriter.WithWorkerPools(c.cfgObject.pool.putRemote, c.cfgObject.pool.putLocal), objectwriter.WithLogger(c.log), objectwriter.WithVerifySessionTokenIssuer(!c.cfgObject.skipSessionTokenIssuerVerification), ) diff --git a/config/example/node.env b/config/example/node.env index 2ba432b1b..aa3c72a91 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -87,8 +87,6 @@ FROSTFS_REPLICATOR_POOL_SIZE=10 FROSTFS_CONTAINER_LIST_STREAM_BATCH_SIZE=500 # Object service section -FROSTFS_OBJECT_PUT_REMOTE_POOL_SIZE=100 -FROSTFS_OBJECT_PUT_LOCAL_POOL_SIZE=200 FROSTFS_OBJECT_PUT_SKIP_SESSION_TOKEN_ISSUER_VERIFICATION=true FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10 FROSTFS_OBJECT_GET_PRIORITY="$attribute:ClusterName $attribute:UN-LOCODE" diff --git a/config/example/node.json b/config/example/node.json index cfde8bcc7..afa815bc3 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -134,8 +134,6 @@ "tombstone_lifetime": 10 }, "put": { - "remote_pool_size": 100, - "local_pool_size": 200, "skip_session_token_issuer_verification": true }, "get": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 1f8ec843d..f63cc514b 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -117,8 +117,6 @@ object: delete: tombstone_lifetime: 10 # tombstone "local" lifetime in epochs put: - remote_pool_size: 100 # number of async workers for remote PUT operations - local_pool_size: 200 # number of async workers for local PUT operations skip_session_token_issuer_verification: true # session token issuer verification will be skipped if true get: priority: # list of metrics of nodes for prioritization diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index aef05d589..b2ab75b7e 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -406,8 +406,6 @@ Contains object-service related parameters. ```yaml object: - put: - remote_pool_size: 100 get: priority: - $attribute:ClusterName @@ -416,8 +414,6 @@ object: | Parameter | Type | Default value | Description | |-----------------------------|------------|---------------|------------------------------------------------------------------------------------------------| | `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | -| `put.remote_pool_size` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | -| `put.local_pool_size` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | | `get.priority` | `[]string` | | List of metrics of nodes for prioritization. Used for computing response on GET requests. | # `runtime` section diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 6a72644e5..d48a4da9b 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -125,7 +125,6 @@ const ( SearchCouldNotWriteObjectIdentifiers = "could not write object identifiers" SearchLocalOperationFailed = "local operation failed" UtilObjectServiceError = "object service error" - UtilCouldNotPushTaskToWorkerPool = "could not push task to worker pool" V2CantCheckIfRequestFromInnerRing = "can't check if request from inner ring" V2CantCheckIfRequestFromContainerNode = "can't check if request from container node" ClientCouldNotRestoreBlockSubscriptionAfterRPCSwitch = "could not restore block subscription after RPC switch" diff --git a/pkg/services/object/common/writer/common.go b/pkg/services/object/common/writer/common.go index dae168baf..1998e9638 100644 --- a/pkg/services/object/common/writer/common.go +++ b/pkg/services/object/common/writer/common.go @@ -79,11 +79,11 @@ func (n *NodeIterator) forEachAddress(ctx context.Context, traverser *placement. continue } - workerPool, isLocal := n.cfg.getWorkerPool(addr.PublicKey()) + isLocal := n.cfg.NetmapKeys.IsLocalKey(addr.PublicKey()) item := new(bool) wg.Add(1) - if err := workerPool.Submit(func() { + go func() { defer wg.Done() err := f(ctx, NodeDescriptor{Local: isLocal, Info: addr}) @@ -95,11 +95,7 @@ func (n *NodeIterator) forEachAddress(ctx context.Context, traverser *placement. traverser.SubmitSuccess() *item = true - }); err != nil { - wg.Done() - svcutil.LogWorkerPoolError(ctx, n.cfg.Logger, "PUT", err) - return true - } + }() // Mark the container node as processed in order to exclude it // in subsequent container broadcast. Note that we don't diff --git a/pkg/services/object/common/writer/ec.go b/pkg/services/object/common/writer/ec.go index 8f269ec21..26a53e315 100644 --- a/pkg/services/object/common/writer/ec.go +++ b/pkg/services/object/common/writer/ec.go @@ -149,17 +149,7 @@ func (e *ECWriter) relayToContainerNode(ctx context.Context, objID oid.ID, index return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) } - completed := make(chan interface{}) - if poolErr := e.Config.RemotePool.Submit(func() { - defer close(completed) - err = e.Relay(ctx, info, c) - }); poolErr != nil { - close(completed) - svcutil.LogWorkerPoolError(ctx, e.Config.Logger, "PUT", poolErr) - return poolErr - } - <-completed - + err = e.Relay(ctx, info, c) if err == nil { return nil } @@ -343,21 +333,11 @@ func (e *ECWriter) putECPartToNode(ctx context.Context, obj *objectSDK.Object, n } func (e *ECWriter) writePartLocal(ctx context.Context, obj *objectSDK.Object) error { - var err error localTarget := LocalTarget{ Storage: e.Config.LocalStore, Container: e.Container, } - completed := make(chan interface{}) - if poolErr := e.Config.LocalPool.Submit(func() { - defer close(completed) - err = localTarget.WriteObject(ctx, obj, e.ObjectMeta) - }); poolErr != nil { - close(completed) - return poolErr - } - <-completed - return err + return localTarget.WriteObject(ctx, obj, e.ObjectMeta) } func (e *ECWriter) writePartRemote(ctx context.Context, obj *objectSDK.Object, node placement.Node) error { @@ -371,15 +351,5 @@ func (e *ECWriter) writePartRemote(ctx context.Context, obj *objectSDK.Object, n nodeInfo: clientNodeInfo, } - var err error - completed := make(chan interface{}) - if poolErr := e.Config.RemotePool.Submit(func() { - defer close(completed) - err = remoteTaget.WriteObject(ctx, obj, e.ObjectMeta) - }); poolErr != nil { - close(completed) - return poolErr - } - <-completed - return err + return remoteTaget.WriteObject(ctx, obj, e.ObjectMeta) } diff --git a/pkg/services/object/common/writer/ec_test.go b/pkg/services/object/common/writer/ec_test.go index b7764661f..2458e352f 100644 --- a/pkg/services/object/common/writer/ec_test.go +++ b/pkg/services/object/common/writer/ec_test.go @@ -31,7 +31,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "git.frostfs.info/TrueCloudLab/tzhash/tz" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" ) @@ -131,9 +130,6 @@ func TestECWriter(t *testing.T) { nodeKey, err := keys.NewPrivateKey() require.NoError(t, err) - pool, err := ants.NewPool(4, ants.WithNonblocking(true)) - require.NoError(t, err) - log, err := logger.NewLogger(nil) require.NoError(t, err) @@ -141,7 +137,6 @@ func TestECWriter(t *testing.T) { ecw := ECWriter{ Config: &Config{ NetmapKeys: n, - RemotePool: pool, Logger: log, ClientConstructor: clientConstructor{vectors: ns}, KeyStorage: util.NewKeyStorage(&nodeKey.PrivateKey, nil, nil), diff --git a/pkg/services/object/common/writer/writer.go b/pkg/services/object/common/writer/writer.go index adaf1945b..d3d2b41b4 100644 --- a/pkg/services/object/common/writer/writer.go +++ b/pkg/services/object/common/writer/writer.go @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/policy" objutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -52,8 +51,6 @@ type Config struct { NetmapSource netmap.Source - RemotePool, LocalPool util.WorkerPool - NetmapKeys netmap.AnnouncedKeys FormatValidator *object.FormatValidator @@ -69,12 +66,6 @@ type Config struct { type Option func(*Config) -func WithWorkerPools(remote, local util.WorkerPool) Option { - return func(c *Config) { - c.RemotePool, c.LocalPool = remote, local - } -} - func WithLogger(l *logger.Logger) Option { return func(c *Config) { c.Logger = l @@ -87,13 +78,6 @@ func WithVerifySessionTokenIssuer(v bool) Option { } } -func (c *Config) getWorkerPool(pub []byte) (util.WorkerPool, bool) { - if c.NetmapKeys.IsLocalKey(pub) { - return c.LocalPool, true - } - return c.RemotePool, false -} - type Params struct { Config *Config diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 5cc0a5722..099486b3f 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -6,7 +6,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" objutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -27,8 +26,6 @@ func NewService(ks *objutil.KeyStorage, opts ...objectwriter.Option, ) *Service { c := &objectwriter.Config{ - RemotePool: util.NewPseudoWorkerPool(), - LocalPool: util.NewPseudoWorkerPool(), Logger: logger.NewLoggerWrapper(zap.L()), KeyStorage: ks, ClientConstructor: cc, diff --git a/pkg/services/object/util/log.go b/pkg/services/object/util/log.go index 2c1e053ac..b10826226 100644 --- a/pkg/services/object/util/log.go +++ b/pkg/services/object/util/log.go @@ -17,11 +17,3 @@ func LogServiceError(ctx context.Context, l *logger.Logger, req string, node net zap.Error(err), ) } - -// LogWorkerPoolError writes debug error message of object worker pool to provided logger. -func LogWorkerPoolError(ctx context.Context, l *logger.Logger, req string, err error) { - l.Error(ctx, logs.UtilCouldNotPushTaskToWorkerPool, - zap.String("request", req), - zap.Error(err), - ) -} From f0b2017057841693612ef0e8239443afd230ce47 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Thu, 20 Feb 2025 11:18:02 +0300 Subject: [PATCH 1738/1943] [#1639] go.mod: Update sdk-go and qos Signed-off-by: Aleksey Savchuk --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 23ddad276..2bfc3abfe 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 - git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe + git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250213125059-356851eed3bf git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 diff --git a/go.sum b/go.sum index ecd2ab525..4a7dfd4dc 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 h1:Mxw1c/8t96vFIUOffl28lFaHKi413oCBfLMGJmF9cFA= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe h1:81gDNdWNLP24oMQukRiCE9R1wGSh0l0dRq3F1W+Oesc= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250128150313-cfbca7fa1dfe/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250213125059-356851eed3bf h1:ik2aMBpTJJpoZe2ffcGShXRkrvny65NEPLVt67KmH/A= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250213125059-356851eed3bf/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4 h1:dOZHuOywvH1ms8U38lDCWpysgkCCeJ02RLI7zDhPcyw= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= From 55908865996def33858ca6e7274d0e0e892a9bf1 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 7 Feb 2025 17:23:10 +0300 Subject: [PATCH 1739/1943] [#1639] qos: Add interceptors for limiting active RPCs Signed-off-by: Aleksey Savchuk --- internal/qos/grpc.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/internal/qos/grpc.go b/internal/qos/grpc.go index c253f1e9d..534a1f74b 100644 --- a/internal/qos/grpc.go +++ b/internal/qos/grpc.go @@ -3,7 +3,9 @@ package qos import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-qos/limiting" "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" "google.golang.org/grpc" ) @@ -49,3 +51,36 @@ func NewAdjustOutgoingIOTagStreamClientInterceptor() grpc.StreamClientIntercepto return streamer(ctx, desc, cc, method, opts...) } } + +func NewMaxActiveRPCLimiterUnaryServerInterceptor(getLimiter func() limiting.Limiter) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + if tag, ok := tagging.IOTagFromContext(ctx); ok && tag == IOTagCritical.String() { + return handler(ctx, req) + } + + release, ok := getLimiter().Acquire(info.FullMethod) + if !ok { + return nil, new(apistatus.ResourceExhausted) + } + defer release() + + return handler(ctx, req) + } +} + +//nolint:contextcheck (grpc.ServerStream manages the context itself) +func NewMaxActiveRPCLimiterStreamServerInterceptor(getLimiter func() limiting.Limiter) grpc.StreamServerInterceptor { + return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if tag, ok := tagging.IOTagFromContext(ss.Context()); ok && tag == IOTagCritical.String() { + return handler(srv, ss) + } + + release, ok := getLimiter().Acquire(info.FullMethod) + if !ok { + return new(apistatus.ResourceExhausted) + } + defer release() + + return handler(srv, ss) + } +} From dae0949f6eb8af3677f6552f52a00c2fdaa6bf79 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 7 Feb 2025 15:17:37 +0300 Subject: [PATCH 1740/1943] [#1639] node: Support active RPC limiting - Allow configuration of active RPC limits for method groups - Apply RPC limiting for all services except the control service Signed-off-by: Aleksey Savchuk --- cmd/frostfs-node/config.go | 40 ++++++++++++-- cmd/frostfs-node/config/rpc/config.go | 43 +++++++++++++++ cmd/frostfs-node/config/rpc/config_test.go | 53 +++++++++++++++++++ cmd/frostfs-node/config/rpc/testdata/node.env | 3 ++ .../config/rpc/testdata/node.json | 18 +++++++ .../config/rpc/testdata/node.yaml | 8 +++ cmd/frostfs-node/grpc.go | 4 ++ config/example/node.env | 5 ++ config/example/node.json | 17 ++++++ config/example/node.yaml | 10 ++++ docs/storage-node-configuration.md | 21 ++++++++ 11 files changed, 217 insertions(+), 5 deletions(-) create mode 100644 cmd/frostfs-node/config/rpc/config.go create mode 100644 cmd/frostfs-node/config/rpc/config_test.go create mode 100644 cmd/frostfs-node/config/rpc/testdata/node.env create mode 100644 cmd/frostfs-node/config/rpc/testdata/node.json create mode 100644 cmd/frostfs-node/config/rpc/testdata/node.yaml diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 43ec40d1d..a7aeedc21 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -29,6 +29,7 @@ import ( nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" + rpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/rpc" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" @@ -69,6 +70,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/state" "git.frostfs.info/TrueCloudLab/frostfs-observability/logging/lokicore" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "git.frostfs.info/TrueCloudLab/frostfs-qos/limiting" netmapV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -528,6 +530,8 @@ type cfgGRPC struct { maxChunkSize uint64 maxAddrAmount uint64 reconnectTimeout time.Duration + + limiter atomic.Pointer[limiting.SemaphoreLimiter] } func (c *cfgGRPC) append(e string, l net.Listener, s *grpc.Server) { @@ -717,7 +721,7 @@ func initCfg(appCfg *config.Config) *cfg { c.cfgNetmap = initNetmap(appCfg, netState, relayOnly) - c.cfgGRPC = initCfgGRPC() + c.cfgGRPC = initCfgGRPC(appCfg) c.cfgMorph = cfgMorph{ proxyScriptHash: contractsconfig.Proxy(appCfg), @@ -848,14 +852,23 @@ func initFrostfsID(appCfg *config.Config) cfgFrostfsID { } } -func initCfgGRPC() cfgGRPC { +func initCfgGRPC(appCfg *config.Config) (cfg cfgGRPC) { maxChunkSize := uint64(maxMsgSize) * 3 / 4 // 25% to meta, 75% to payload maxAddrAmount := maxChunkSize / addressSize // each address is about 72 bytes - return cfgGRPC{ - maxChunkSize: maxChunkSize, - maxAddrAmount: maxAddrAmount, + var limits []limiting.KeyLimit + for _, l := range rpcconfig.Limits(appCfg) { + limits = append(limits, limiting.KeyLimit{Keys: l.Methods, Limit: l.MaxOps}) } + + limiter, err := limiting.NewSemaphoreLimiter(limits) + fatalOnErr(err) + + cfg.maxChunkSize = maxChunkSize + cfg.maxAddrAmount = maxAddrAmount + cfg.limiter.Store(limiter) + + return } func initCfgObject(appCfg *config.Config) cfgObject { @@ -1392,9 +1405,26 @@ func (c *cfg) getComponents(ctx context.Context, logPrm *logger.Prm) []dCmp { components = append(components, dCmp{cmp.name, func() error { return cmp.reload(ctx) }}) } + components = append(components, dCmp{"rpc_limiter", c.reloadLimits}) + return components } +func (c *cfg) reloadLimits() error { + var limits []limiting.KeyLimit + for _, l := range rpcconfig.Limits(c.appCfg) { + limits = append(limits, limiting.KeyLimit{Keys: l.Methods, Limit: l.MaxOps}) + } + + limiter, err := limiting.NewSemaphoreLimiter(limits) + if err != nil { + return err + } + + c.cfgGRPC.limiter.Store(limiter) + return nil +} + func (c *cfg) reloadPools() error { newSize := replicatorconfig.PoolSize(c.appCfg) c.reloadPool(c.cfgObject.pool.replication, newSize, "replicator.pool_size") diff --git a/cmd/frostfs-node/config/rpc/config.go b/cmd/frostfs-node/config/rpc/config.go new file mode 100644 index 000000000..197990d07 --- /dev/null +++ b/cmd/frostfs-node/config/rpc/config.go @@ -0,0 +1,43 @@ +package rpcconfig + +import ( + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" +) + +const ( + subsection = "rpc" + limitsSubsection = "limits" +) + +type LimitConfig struct { + Methods []string + MaxOps int64 +} + +// Limits returns the "limits" config from "rpc" section. +func Limits(c *config.Config) []LimitConfig { + c = c.Sub(subsection).Sub(limitsSubsection) + + var limits []LimitConfig + + for i := uint64(0); ; i++ { + si := strconv.FormatUint(i, 10) + sc := c.Sub(si) + + methods := config.StringSliceSafe(sc, "methods") + if len(methods) == 0 { + break + } + + maxOps := config.IntSafe(sc, "max_ops") + if maxOps == 0 { + panic("no max operations for method group") + } + + limits = append(limits, LimitConfig{methods, maxOps}) + } + + return limits +} diff --git a/cmd/frostfs-node/config/rpc/config_test.go b/cmd/frostfs-node/config/rpc/config_test.go new file mode 100644 index 000000000..31a837cee --- /dev/null +++ b/cmd/frostfs-node/config/rpc/config_test.go @@ -0,0 +1,53 @@ +package rpcconfig + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "github.com/stretchr/testify/require" +) + +func TestRPCSection(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + require.Empty(t, Limits(configtest.EmptyConfig())) + }) + + t.Run("correct config", func(t *testing.T) { + const path = "../../../../config/example/node" + + fileConfigTest := func(c *config.Config) { + limits := Limits(c) + require.Len(t, limits, 2) + + limit0 := limits[0] + limit1 := limits[1] + + require.ElementsMatch(t, limit0.Methods, []string{"/neo.fs.v2.object.ObjectService/PutSingle", "/neo.fs.v2.object.ObjectService/Put"}) + require.Equal(t, limit0.MaxOps, int64(1000)) + + require.ElementsMatch(t, limit1.Methods, []string{"/neo.fs.v2.object.ObjectService/Get"}) + require.Equal(t, limit1.MaxOps, int64(10000)) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) + }) + + t.Run("no max operations", func(t *testing.T) { + const path = "testdata/node" + + fileConfigTest := func(c *config.Config) { + require.Panics(t, func() { _ = Limits(c) }) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) + }) +} diff --git a/cmd/frostfs-node/config/rpc/testdata/node.env b/cmd/frostfs-node/config/rpc/testdata/node.env new file mode 100644 index 000000000..2fed4c5bc --- /dev/null +++ b/cmd/frostfs-node/config/rpc/testdata/node.env @@ -0,0 +1,3 @@ +FROSTFS_RPC_LIMITS_0_METHODS="/neo.fs.v2.object.ObjectService/PutSingle /neo.fs.v2.object.ObjectService/Put" +FROSTFS_RPC_LIMITS_1_METHODS="/neo.fs.v2.object.ObjectService/Get" +FROSTFS_RPC_LIMITS_1_MAX_OPS=10000 diff --git a/cmd/frostfs-node/config/rpc/testdata/node.json b/cmd/frostfs-node/config/rpc/testdata/node.json new file mode 100644 index 000000000..6156aa71d --- /dev/null +++ b/cmd/frostfs-node/config/rpc/testdata/node.json @@ -0,0 +1,18 @@ +{ + "rpc": { + "limits": [ + { + "methods": [ + "/neo.fs.v2.object.ObjectService/PutSingle", + "/neo.fs.v2.object.ObjectService/Put" + ] + }, + { + "methods": [ + "/neo.fs.v2.object.ObjectService/Get" + ], + "max_ops": 10000 + } + ] + } +} diff --git a/cmd/frostfs-node/config/rpc/testdata/node.yaml b/cmd/frostfs-node/config/rpc/testdata/node.yaml new file mode 100644 index 000000000..e50b7ae93 --- /dev/null +++ b/cmd/frostfs-node/config/rpc/testdata/node.yaml @@ -0,0 +1,8 @@ +rpc: + limits: + - methods: + - /neo.fs.v2.object.ObjectService/PutSingle + - /neo.fs.v2.object.ObjectService/Put + - methods: + - /neo.fs.v2.object.ObjectService/Get + max_ops: 10000 diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index 4d679e4cc..e1a273ce4 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -9,9 +9,11 @@ import ( grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + qosInternal "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-qos/limiting" qos "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "go.uber.org/zap" "google.golang.org/grpc" @@ -134,11 +136,13 @@ func getGrpcServerOpts(ctx context.Context, c *cfg, sc *grpcconfig.Config) ([]gr qos.NewUnaryServerInterceptor(), metrics.NewUnaryServerInterceptor(), tracing.NewUnaryServerInterceptor(), + qosInternal.NewMaxActiveRPCLimiterUnaryServerInterceptor(func() limiting.Limiter { return c.cfgGRPC.limiter.Load() }), ), grpc.ChainStreamInterceptor( qos.NewStreamServerInterceptor(), metrics.NewStreamServerInterceptor(), tracing.NewStreamServerInterceptor(), + qosInternal.NewMaxActiveRPCLimiterStreamServerInterceptor(func() limiting.Limiter { return c.cfgGRPC.limiter.Load() }), ), } diff --git a/config/example/node.env b/config/example/node.env index aa3c72a91..2ebef181a 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -91,6 +91,11 @@ FROSTFS_OBJECT_PUT_SKIP_SESSION_TOKEN_ISSUER_VERIFICATION=true FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10 FROSTFS_OBJECT_GET_PRIORITY="$attribute:ClusterName $attribute:UN-LOCODE" +FROSTFS_RPC_LIMITS_0_METHODS="/neo.fs.v2.object.ObjectService/PutSingle /neo.fs.v2.object.ObjectService/Put" +FROSTFS_RPC_LIMITS_0_MAX_OPS=1000 +FROSTFS_RPC_LIMITS_1_METHODS="/neo.fs.v2.object.ObjectService/Get" +FROSTFS_RPC_LIMITS_1_MAX_OPS=10000 + # Storage engine section FROSTFS_STORAGE_SHARD_POOL_SIZE=15 FROSTFS_STORAGE_SHARD_RO_ERROR_THRESHOLD=100 diff --git a/config/example/node.json b/config/example/node.json index afa815bc3..0ed72effc 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -140,6 +140,23 @@ "priority": ["$attribute:ClusterName", "$attribute:UN-LOCODE"] } }, + "rpc": { + "limits": [ + { + "methods": [ + "/neo.fs.v2.object.ObjectService/PutSingle", + "/neo.fs.v2.object.ObjectService/Put" + ], + "max_ops": 1000 + }, + { + "methods": [ + "/neo.fs.v2.object.ObjectService/Get" + ], + "max_ops": 10000 + } + ] + }, "storage": { "shard_pool_size": 15, "shard_ro_error_threshold": 100, diff --git a/config/example/node.yaml b/config/example/node.yaml index f63cc514b..6b810653e 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -123,6 +123,16 @@ object: - $attribute:ClusterName - $attribute:UN-LOCODE +rpc: + limits: + - methods: + - /neo.fs.v2.object.ObjectService/PutSingle + - /neo.fs.v2.object.ObjectService/Put + max_ops: 1000 + - methods: + - /neo.fs.v2.object.ObjectService/Get + max_ops: 10000 + storage: # note: shard configuration can be omitted for relay node (see `node.relay`) shard_pool_size: 15 # size of per-shard worker pools used for PUT operations diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index b2ab75b7e..1eb5437ba 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -416,6 +416,27 @@ object: | `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | | `get.priority` | `[]string` | | List of metrics of nodes for prioritization. Used for computing response on GET requests. | + +# `rpc` section +Contains limits on the number of active RPC for specified method(s). + +```yaml +rpc: + limits: + - methods: + - /neo.fs.v2.object.ObjectService/PutSingle + - /neo.fs.v2.object.ObjectService/Put + max_ops: 1000 + - methods: + - /neo.fs.v2.object.ObjectService/Get + max_ops: 10000 +``` + +| Parameter | Type | Default value | Description | +|------------------|------------|---------------|--------------------------------------------------------------| +| `limits.max_ops` | `int` | | Maximum number of active RPC allowed for the given method(s) | +| `limits.methods` | `[]string` | | List of RPC methods sharing the given limit | + # `runtime` section Contains runtime parameters. From 92ab58984b6bbe7722ef8306ca9718e9543d5d9c Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 28 Feb 2025 15:07:24 +0300 Subject: [PATCH 1741/1943] [#1658] node: Simplify RPC limiter initialization - Move all initialization logic to one place - Initialize the limiter after all RPC services are registered to be able to validate that configured limits match the methods registered earlier Signed-off-by: Aleksey Savchuk --- cmd/frostfs-node/config.go | 31 +++---------------------------- cmd/frostfs-node/grpc.go | 17 +++++++++++++++++ cmd/frostfs-node/main.go | 2 ++ 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index a7aeedc21..3e1bfb4f2 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -29,7 +29,6 @@ import ( nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" - rpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/rpc" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" @@ -721,7 +720,7 @@ func initCfg(appCfg *config.Config) *cfg { c.cfgNetmap = initNetmap(appCfg, netState, relayOnly) - c.cfgGRPC = initCfgGRPC(appCfg) + c.cfgGRPC = initCfgGRPC() c.cfgMorph = cfgMorph{ proxyScriptHash: contractsconfig.Proxy(appCfg), @@ -852,21 +851,12 @@ func initFrostfsID(appCfg *config.Config) cfgFrostfsID { } } -func initCfgGRPC(appCfg *config.Config) (cfg cfgGRPC) { +func initCfgGRPC() (cfg cfgGRPC) { maxChunkSize := uint64(maxMsgSize) * 3 / 4 // 25% to meta, 75% to payload maxAddrAmount := maxChunkSize / addressSize // each address is about 72 bytes - var limits []limiting.KeyLimit - for _, l := range rpcconfig.Limits(appCfg) { - limits = append(limits, limiting.KeyLimit{Keys: l.Methods, Limit: l.MaxOps}) - } - - limiter, err := limiting.NewSemaphoreLimiter(limits) - fatalOnErr(err) - cfg.maxChunkSize = maxChunkSize cfg.maxAddrAmount = maxAddrAmount - cfg.limiter.Store(limiter) return } @@ -1405,26 +1395,11 @@ func (c *cfg) getComponents(ctx context.Context, logPrm *logger.Prm) []dCmp { components = append(components, dCmp{cmp.name, func() error { return cmp.reload(ctx) }}) } - components = append(components, dCmp{"rpc_limiter", c.reloadLimits}) + components = append(components, dCmp{"rpc_limiter", func() error { return initRPCLimiter(c) }}) return components } -func (c *cfg) reloadLimits() error { - var limits []limiting.KeyLimit - for _, l := range rpcconfig.Limits(c.appCfg) { - limits = append(limits, limiting.KeyLimit{Keys: l.Methods, Limit: l.MaxOps}) - } - - limiter, err := limiting.NewSemaphoreLimiter(limits) - if err != nil { - return err - } - - c.cfgGRPC.limiter.Store(limiter) - return nil -} - func (c *cfg) reloadPools() error { newSize := replicatorconfig.PoolSize(c.appCfg) c.reloadPool(c.cfgObject.pool.replication, newSize, "replicator.pool_size") diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index e1a273ce4..9ea3dd8a2 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -4,10 +4,12 @@ import ( "context" "crypto/tls" "errors" + "fmt" "net" "time" grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc" + rpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/rpc" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" qosInternal "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -231,3 +233,18 @@ func stopGRPC(ctx context.Context, name string, s *grpc.Server, l *logger.Logger l.Info(ctx, logs.FrostFSNodeGRPCServerStoppedSuccessfully) } + +func initRPCLimiter(c *cfg) error { + var limits []limiting.KeyLimit + for _, l := range rpcconfig.Limits(c.appCfg) { + limits = append(limits, limiting.KeyLimit{Keys: l.Methods, Limit: l.MaxOps}) + } + + limiter, err := limiting.NewSemaphoreLimiter(limits) + if err != nil { + return fmt.Errorf("create RPC limiter: %w", err) + } + + c.cfgGRPC.limiter.Store(limiter) + return nil +} diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index b53bd8823..0228d2a10 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -117,6 +117,8 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(ctx, c, "apemanager", initAPEManagerService) initAndLog(ctx, c, "control", func(c *cfg) { initControlService(ctx, c) }) + initAndLog(ctx, c, "RPC limiter", func(c *cfg) { fatalOnErr(initRPCLimiter(c)) }) + initAndLog(ctx, c, "morph notifications", func(c *cfg) { listenMorphNotifications(ctx, c) }) } From c66027103910ab413a51dede50f64f767ef582ae Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 28 Feb 2025 15:14:15 +0300 Subject: [PATCH 1742/1943] [#1658] node: Validate RPC limiter configuration Validate that configured limits match the methods registered earlier. Signed-off-by: Aleksey Savchuk --- cmd/frostfs-node/grpc.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/cmd/frostfs-node/grpc.go b/cmd/frostfs-node/grpc.go index 9ea3dd8a2..6b6d44750 100644 --- a/cmd/frostfs-node/grpc.go +++ b/cmd/frostfs-node/grpc.go @@ -240,6 +240,10 @@ func initRPCLimiter(c *cfg) error { limits = append(limits, limiting.KeyLimit{Keys: l.Methods, Limit: l.MaxOps}) } + if err := validateRPCLimits(c, limits); err != nil { + return fmt.Errorf("validate RPC limits: %w", err) + } + limiter, err := limiting.NewSemaphoreLimiter(limits) if err != nil { return fmt.Errorf("create RPC limiter: %w", err) @@ -248,3 +252,35 @@ func initRPCLimiter(c *cfg) error { c.cfgGRPC.limiter.Store(limiter) return nil } + +func validateRPCLimits(c *cfg, limits []limiting.KeyLimit) error { + availableMethods := getAvailableMethods(c.cfgGRPC.servers) + for _, limit := range limits { + for _, method := range limit.Keys { + if _, ok := availableMethods[method]; !ok { + return fmt.Errorf("set limit on an unknown method %q", method) + } + } + } + return nil +} + +func getAvailableMethods(servers []grpcServer) map[string]struct{} { + res := make(map[string]struct{}) + for _, server := range servers { + for _, method := range getMethodsForServer(server.Server) { + res[method] = struct{}{} + } + } + return res +} + +func getMethodsForServer(server *grpc.Server) []string { + var res []string + for service, info := range server.GetServiceInfo() { + for _, method := range info.Methods { + res = append(res, fmt.Sprintf("/%s/%s", service, method.Name)) + } + } + return res +} From 0991077cb364c94e40f6838035acab4754b4d8af Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Feb 2025 14:51:35 +0300 Subject: [PATCH 1743/1943] [#1657] engine: Fix data race in evacuation tests Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index b9d7888e7..45c4b696b 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -475,7 +475,7 @@ func TestEvacuateObjectsAsync(t *testing.T) { eg, egCtx := errgroup.WithContext(context.Background()) eg.Go(func() error { require.NoError(t, e.Evacuate(egCtx, prm), "first evacuation failed") - st = testWaitForEvacuationCompleted(t, e) + st := testWaitForEvacuationCompleted(t, e) require.Equal(t, uint64(3), st.ObjectsEvacuated(), "invalid final count") return nil }) From 98d61250292bf3304043da4fe4007dc7bff9c99f Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 4 Feb 2025 18:20:48 +0300 Subject: [PATCH 1744/1943] [#1636] config: Add `shard.limits` config Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 12 ++ cmd/frostfs-node/config/engine/config_test.go | 86 ++++++++++++ .../config/engine/shard/config.go | 9 ++ .../config/engine/shard/limits/config.go | 130 ++++++++++++++++++ config/example/node.env | 41 ++++++ config/example/node.json | 70 ++++++++++ config/example/node.yaml | 46 +++++++ docs/storage-node-configuration.md | 59 ++++++++ internal/qos/validate.go | 92 +++++++++++++ 9 files changed, 545 insertions(+) create mode 100644 cmd/frostfs-node/config/engine/shard/limits/config.go create mode 100644 internal/qos/validate.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 3e1bfb4f2..004c8f128 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -33,6 +33,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" internalNet "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/ape/chainbase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" frostfsidcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/frostfsid" @@ -277,6 +278,9 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, oldConfig a.setMetabaseConfig(&newConfig, oldConfig) a.setGCConfig(&newConfig, oldConfig) + if err := a.setLimits(&newConfig, oldConfig); err != nil { + return err + } a.EngineCfg.shards = append(a.EngineCfg.shards, newConfig) @@ -370,6 +374,14 @@ func (a *applicationConfiguration) setGCConfig(newConfig *shardCfg, oldConfig *s newConfig.gcCfg.expiredCollectorWorkerCount = gcCfg.ExpiredCollectorWorkerCount() } +func (a *applicationConfiguration) setLimits(newConfig *shardCfg, oldConfig *shardconfig.Config) error { + limitsConfig := oldConfig.Limits() + if err := qos.ValidateConfig(limitsConfig); err != nil { + return err + } + return nil +} + // internals contains application-specific internals that are created // on application startup and are shared b/w the components during // the application life cycle. diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index ef6380a62..b912b5d7d 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -11,6 +11,7 @@ import ( blobovniczaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza" fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" gcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/gc" + limitsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" @@ -76,6 +77,7 @@ func TestEngineSection(t *testing.T) { ss := blob.Storages() pl := sc.Pilorama() gc := sc.GC() + limits := sc.Limits() switch num { case 0: @@ -134,6 +136,75 @@ func TestEngineSection(t *testing.T) { require.Equal(t, false, sc.RefillMetabase()) require.Equal(t, mode.ReadOnly, sc.Mode()) require.Equal(t, 100, sc.RefillMetabaseWorkersCount()) + + readLimits := limits.Read() + writeLimits := limits.Write() + require.Equal(t, 30*time.Second, readLimits.IdleTimeout) + require.Equal(t, int64(10_000), readLimits.MaxRunningOps) + require.Equal(t, int64(1_000), readLimits.MaxWaitingOps) + require.Equal(t, 45*time.Second, writeLimits.IdleTimeout) + require.Equal(t, int64(1_000), writeLimits.MaxRunningOps) + require.Equal(t, int64(100), writeLimits.MaxWaitingOps) + require.ElementsMatch(t, readLimits.Tags, + []limitsconfig.IOTagConfig{ + { + Tag: "internal", + Weight: toPtr(20), + ReservedOps: toPtr(1000), + LimitOps: toPtr(0), + }, + { + Tag: "client", + Weight: toPtr(70), + ReservedOps: toPtr(10000), + }, + { + Tag: "background", + Weight: toPtr(5), + LimitOps: toPtr(10000), + ReservedOps: toPtr(0), + }, + { + Tag: "writecache", + Weight: toPtr(5), + LimitOps: toPtr(25000), + }, + { + Tag: "policer", + Weight: toPtr(5), + LimitOps: toPtr(25000), + }, + }) + require.ElementsMatch(t, writeLimits.Tags, + []limitsconfig.IOTagConfig{ + { + Tag: "internal", + Weight: toPtr(200), + ReservedOps: toPtr(100), + LimitOps: toPtr(0), + }, + { + Tag: "client", + Weight: toPtr(700), + ReservedOps: toPtr(1000), + }, + { + Tag: "background", + Weight: toPtr(50), + LimitOps: toPtr(1000), + ReservedOps: toPtr(0), + }, + { + Tag: "writecache", + Weight: toPtr(50), + LimitOps: toPtr(2500), + }, + { + Tag: "policer", + Weight: toPtr(50), + LimitOps: toPtr(2500), + }, + }) case 1: require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path()) require.Equal(t, fs.FileMode(0o644), pl.Perm()) @@ -188,6 +259,17 @@ func TestEngineSection(t *testing.T) { require.Equal(t, true, sc.RefillMetabase()) require.Equal(t, mode.ReadWrite, sc.Mode()) require.Equal(t, shardconfig.RefillMetabaseWorkersCountDefault, sc.RefillMetabaseWorkersCount()) + + readLimits := limits.Read() + writeLimits := limits.Write() + require.Equal(t, limitsconfig.DefaultIdleTimeout, readLimits.IdleTimeout) + require.Equal(t, limitsconfig.NoLimit, readLimits.MaxRunningOps) + require.Equal(t, limitsconfig.NoLimit, readLimits.MaxWaitingOps) + require.Equal(t, limitsconfig.DefaultIdleTimeout, writeLimits.IdleTimeout) + require.Equal(t, limitsconfig.NoLimit, writeLimits.MaxRunningOps) + require.Equal(t, limitsconfig.NoLimit, writeLimits.MaxWaitingOps) + require.Equal(t, 0, len(readLimits.Tags)) + require.Equal(t, 0, len(writeLimits.Tags)) } return nil }) @@ -201,3 +283,7 @@ func TestEngineSection(t *testing.T) { configtest.ForEnvFileType(t, path, fileConfigTest) }) } + +func toPtr(v float64) *float64 { + return &v +} diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go index 0620c9f63..e50d56b95 100644 --- a/cmd/frostfs-node/config/engine/shard/config.go +++ b/cmd/frostfs-node/config/engine/shard/config.go @@ -4,6 +4,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" blobstorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor" gcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/gc" + limitsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" metabaseconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/metabase" piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache" @@ -125,6 +126,14 @@ func (x *Config) GC() *gcconfig.Config { ) } +// Limits returns "limits" subsection as a limitsconfig.Config. +func (x *Config) Limits() *limitsconfig.Config { + return limitsconfig.From( + (*config.Config)(x). + Sub("limits"), + ) +} + // RefillMetabase returns the value of "resync_metabase" config parameter. // // Returns false if the value is not a valid bool. diff --git a/cmd/frostfs-node/config/engine/shard/limits/config.go b/cmd/frostfs-node/config/engine/shard/limits/config.go new file mode 100644 index 000000000..b9b5c4382 --- /dev/null +++ b/cmd/frostfs-node/config/engine/shard/limits/config.go @@ -0,0 +1,130 @@ +package limits + +import ( + "math" + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "github.com/spf13/cast" +) + +const ( + NoLimit int64 = math.MaxInt64 + DefaultIdleTimeout = 5 * time.Minute +) + +// From wraps config section into Config. +func From(c *config.Config) *Config { + return (*Config)(c) +} + +// Config is a wrapper over the config section +// which provides access to Shard's limits configurations. +type Config config.Config + +// Read returns the value of "read" limits config section. +func (x *Config) Read() OpConfig { + return x.parse("read") +} + +// Write returns the value of "write" limits config section. +func (x *Config) Write() OpConfig { + return x.parse("write") +} + +func (x *Config) parse(sub string) OpConfig { + c := (*config.Config)(x).Sub(sub) + var result OpConfig + + if s := config.Int(c, "max_waiting_ops"); s > 0 { + result.MaxWaitingOps = s + } else { + result.MaxWaitingOps = NoLimit + } + + if s := config.Int(c, "max_running_ops"); s > 0 { + result.MaxRunningOps = s + } else { + result.MaxRunningOps = NoLimit + } + + if s := config.DurationSafe(c, "idle_timeout"); s > 0 { + result.IdleTimeout = s + } else { + result.IdleTimeout = DefaultIdleTimeout + } + + result.Tags = tags(c) + + return result +} + +type OpConfig struct { + // MaxWaitingOps returns the value of "max_waiting_ops" config parameter. + // + // Equals NoLimit if the value is not a positive number. + MaxWaitingOps int64 + // MaxRunningOps returns the value of "max_running_ops" config parameter. + // + // Equals NoLimit if the value is not a positive number. + MaxRunningOps int64 + // IdleTimeout returns the value of "idle_timeout" config parameter. + // + // Equals DefaultIdleTimeout if the value is not a valid duration. + IdleTimeout time.Duration + // Tags returns the value of "tags" config parameter. + // + // Equals nil if the value is not a valid tags config slice. + Tags []IOTagConfig +} + +type IOTagConfig struct { + Tag string + Weight *float64 + LimitOps *float64 + ReservedOps *float64 +} + +func tags(c *config.Config) []IOTagConfig { + c = c.Sub("tags") + var result []IOTagConfig + for i := 0; ; i++ { + tag := config.String(c, strconv.Itoa(i)+".tag") + if tag == "" { + return result + } + + var tagConfig IOTagConfig + tagConfig.Tag = tag + + v := c.Value(strconv.Itoa(i) + ".weight") + if v != nil { + w, err := cast.ToFloat64E(v) + panicOnErr(err) + tagConfig.Weight = &w + } + + v = c.Value(strconv.Itoa(i) + ".limit_ops") + if v != nil { + l, err := cast.ToFloat64E(v) + panicOnErr(err) + tagConfig.LimitOps = &l + } + + v = c.Value(strconv.Itoa(i) + ".reserved_ops") + if v != nil { + r, err := cast.ToFloat64E(v) + panicOnErr(err) + tagConfig.ReservedOps = &r + } + + result = append(result, tagConfig) + } +} + +func panicOnErr(err error) { + if err != nil { + panic(err) + } +} diff --git a/config/example/node.env b/config/example/node.env index 2ebef181a..9bd645344 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -157,6 +157,47 @@ FROSTFS_STORAGE_SHARD_0_GC_REMOVER_SLEEP_INTERVAL=2m FROSTFS_STORAGE_SHARD_0_GC_EXPIRED_COLLECTOR_BATCH_SIZE=1500 #### Limit of concurrent workers collecting expired objects by the garbage collector FROSTFS_STORAGE_SHARD_0_GC_EXPIRED_COLLECTOR_WORKER_COUNT=15 +#### Limits config +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_MAX_RUNNING_OPS=10000 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_MAX_WAITING_OPS=1000 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_MAX_RUNNING_OPS=1000 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_MAX_WAITING_OPS=100 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_IDLE_TIMEOUT=45s +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_IDLE_TIMEOUT=30s +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_0_TAG=internal +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_0_WEIGHT=20 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_0_LIMIT_OPS=0 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_0_RESERVED_OPS=1000 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_1_TAG=client +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_1_WEIGHT=70 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_1_RESERVED_OPS=10000 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_2_TAG=background +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_2_WEIGHT=5 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_2_LIMIT_OPS=10000 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_2_RESERVED_OPS=0 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_3_TAG=writecache +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_3_WEIGHT=5 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_3_LIMIT_OPS=25000 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_TAG=policer +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_WEIGHT=5 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_LIMIT_OPS=25000 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_TAG=internal +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_WEIGHT=200 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_LIMIT_OPS=0 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_RESERVED_OPS=100 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_1_TAG=client +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_1_WEIGHT=700 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_1_RESERVED_OPS=1000 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_2_TAG=background +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_2_WEIGHT=50 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_2_LIMIT_OPS=1000 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_2_RESERVED_OPS=0 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_3_TAG=writecache +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_3_WEIGHT=50 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_3_LIMIT_OPS=2500 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_4_TAG=policer +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_4_WEIGHT=50 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_4_LIMIT_OPS=2500 ## 1 shard ### Flag to refill Metabase from BlobStor diff --git a/config/example/node.json b/config/example/node.json index 0ed72effc..6b799b318 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -221,6 +221,76 @@ "remover_sleep_interval": "2m", "expired_collector_batch_size": 1500, "expired_collector_worker_count": 15 + }, + "limits": { + "read": { + "max_running_ops": 10000, + "max_waiting_ops": 1000, + "idle_timeout": "30s", + "tags": [ + { + "tag": "internal", + "weight": 20, + "limit_ops": 0, + "reserved_ops": 1000 + }, + { + "tag": "client", + "weight": 70, + "reserved_ops": 10000 + }, + { + "tag": "background", + "weight": 5, + "limit_ops": 10000, + "reserved_ops": 0 + }, + { + "tag": "writecache", + "weight": 5, + "limit_ops": 25000 + }, + { + "tag": "policer", + "weight": 5, + "limit_ops": 25000 + } + ] + }, + "write": { + "max_running_ops": 1000, + "max_waiting_ops": 100, + "idle_timeout": "45s", + "tags": [ + { + "tag": "internal", + "weight": 200, + "limit_ops": 0, + "reserved_ops": 100 + }, + { + "tag": "client", + "weight": 700, + "reserved_ops": 1000 + }, + { + "tag": "background", + "weight": 50, + "limit_ops": 1000, + "reserved_ops": 0 + }, + { + "tag": "writecache", + "weight": 50, + "limit_ops": 2500 + }, + { + "tag": "policer", + "weight": 50, + "limit_ops": 2500 + } + ] + } } }, "1": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 6b810653e..2552a419c 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -227,6 +227,52 @@ storage: expired_collector_batch_size: 1500 # number of objects to be marked expired by the garbage collector expired_collector_worker_count: 15 # number of concurrent workers collecting expired objects by the garbage collector + limits: + read: + max_running_ops: 10000 + max_waiting_ops: 1000 + idle_timeout: 30s + tags: + - tag: internal + weight: 20 + limit_ops: 0 + reserved_ops: 1000 + - tag: client + weight: 70 + reserved_ops: 10000 + - tag: background + weight: 5 + limit_ops: 10000 + reserved_ops: 0 + - tag: writecache + weight: 5 + limit_ops: 25000 + - tag: policer + weight: 5 + limit_ops: 25000 + write: + max_running_ops: 1000 + max_waiting_ops: 100 + idle_timeout: 45s + tags: + - tag: internal + weight: 200 + limit_ops: 0 + reserved_ops: 100 + - tag: client + weight: 700 + reserved_ops: 1000 + - tag: background + weight: 50 + limit_ops: 1000 + reserved_ops: 0 + - tag: writecache + weight: 50 + limit_ops: 2500 + - tag: policer + weight: 50 + limit_ops: 2500 + 1: writecache: path: tmp/1/cache # write-cache root directory diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 1eb5437ba..271cc6532 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -195,6 +195,7 @@ The following table describes configuration for each shard. | `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | | `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. | | `gc` | [GC config](#gc-subsection) | | GC configuration. | +| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. | ### `blobstor` subsection @@ -301,6 +302,64 @@ writecache: | `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | | `max_flushing_objects_size` | `size` | `512M` | Max total size of background flushing objects. | +### `limits` subsection + +```yaml +limits: + max_read_running_ops: 10000 + max_read_waiting_ops: 1000 + max_write_running_ops: 1000 + max_write_waiting_ops: 100 + read: + - tag: internal + weight: 20 + limit_ops: 0 + reserved_ops: 1000 + - tag: client + weight: 70 + reserved_ops: 10000 + - tag: background + weight: 5 + limit_ops: 10000 + reserved_ops: 0 + - tag: writecache + weight: 5 + limit_ops: 25000 + - tag: policer + weight: 5 + limit_ops: 25000 + write: + - tag: internal + weight: 200 + limit_ops: 0 + reserved_ops: 100 + - tag: client + weight: 700 + reserved_ops: 1000 + - tag: background + weight: 50 + limit_ops: 1000 + reserved_ops: 0 + - tag: writecache + weight: 50 + limit_ops: 2500 + - tag: policer + weight: 50 + limit_ops: 2500 +``` + +| Parameter | Type | Default value | Description | +| ----------------------- | -------- | -------------- | --------------------------------------------------------------------------------------------------------------- | +| `max_read_running_ops` | `int` | 0 (no limit) | The maximum number of runnig read operations. | +| `max_read_waiting_ops` | `int` | 0 (no limit) | The maximum number of waiting read operations. | +| `max_write_running_ops` | `int` | 0 (no limit) | The maximum number of running write operations. | +| `max_write_waiting_ops` | `int` | 0 (no limit) | The maximum number of running write operations. | +| `read` | `[]tag` | empty | Array of shard read settings for tags. | +| `write` | `[]tag` | empty | Array of shard write settings for tags. | +| `tag.tag` | `string` | empty | Tag name. Allowed values: `client`, `internal`, `background`, `writecache`, `policer`. | +| `tag.weight` | `float` | 0 (no weight) | Weight for queries with the specified tag. Weights must be specified for all tags or not specified for any one. | +| `tag.limit_ops` | `float` | 0 (no limit) | Operations per second rate limit for queries with the specified tag. | +| `tag.reserved_ops` | `float` | 0 (no reserve) | Reserved operations per second rate for queries with the specified tag. | # `node` section diff --git a/internal/qos/validate.go b/internal/qos/validate.go new file mode 100644 index 000000000..afced345b --- /dev/null +++ b/internal/qos/validate.go @@ -0,0 +1,92 @@ +package qos + +import ( + "errors" + "fmt" + "math" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" +) + +var errWeightsMustBeSpecified = errors.New("invalid weights: weights must be specified for all tags or not specified for any") + +type tagConfig struct { + Shares, Limit, Reserved *float64 +} + +func ValidateConfig(c *limits.Config) error { + if err := validateOpConfig(c.Read()); err != nil { + return fmt.Errorf("limits 'read' section validation error: %w", err) + } + if err := validateOpConfig(c.Write()); err != nil { + return fmt.Errorf("limits 'write' section validation error: %w", err) + } + return nil +} + +func validateOpConfig(c limits.OpConfig) error { + if c.MaxRunningOps <= 0 { + return fmt.Errorf("invalid 'max_running_ops = %d': must be greater than zero", c.MaxRunningOps) + } + if c.MaxWaitingOps <= 0 { + return fmt.Errorf("invalid 'max_waiting_ops = %d': must be greater than zero", c.MaxWaitingOps) + } + if c.IdleTimeout <= 0 { + return fmt.Errorf("invalid 'idle_timeout = %s': must be greater than zero", c.IdleTimeout.String()) + } + if err := validateTags(c.Tags); err != nil { + return fmt.Errorf("'tags' config section validation error: %w", err) + } + return nil +} + +func validateTags(configTags []limits.IOTagConfig) error { + tags := map[IOTag]tagConfig{ + IOTagClient: {}, + IOTagInternal: {}, + IOTagBackground: {}, + IOTagWritecache: {}, + IOTagPolicer: {}, + } + for _, t := range configTags { + tag, err := FromRawString(t.Tag) + if err != nil { + return fmt.Errorf("invalid tag %s: %w", t.Tag, err) + } + if _, ok := tags[tag]; !ok { + return fmt.Errorf("tag %s is not configurable", t.Tag) + } + tags[tag] = tagConfig{ + Shares: t.Weight, + Limit: t.LimitOps, + Reserved: t.ReservedOps, + } + } + idx := 0 + var shares float64 + for t, v := range tags { + if idx == 0 { + idx++ + shares = float64Value(v.Shares) + } else if (shares != 0 && float64Value(v.Shares) == 0) || (shares == 0 && float64Value(v.Shares) != 0) { + return errWeightsMustBeSpecified + } + if float64Value(v.Shares) < 0 || math.IsNaN(float64Value(v.Shares)) { + return fmt.Errorf("invalid weight for tag %s: must be positive value", t.String()) + } + if float64Value(v.Limit) < 0 || math.IsNaN(float64Value(v.Limit)) { + return fmt.Errorf("invalid limit_ops for tag %s: must be positive value", t.String()) + } + if float64Value(v.Reserved) < 0 || math.IsNaN(float64Value(v.Reserved)) { + return fmt.Errorf("invalid reserved_ops for tag %s: must be positive value", t.String()) + } + } + return nil +} + +func float64Value(f *float64) float64 { + if f == nil { + return 0.0 + } + return *f +} From 92a67a6716723f81dabb4f809ff040ef7014903a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Feb 2025 12:31:01 +0300 Subject: [PATCH 1745/1943] [#1636] qos: Add Limiter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 12 +++- internal/assert/cond.go | 9 +++ internal/qos/limiter.go | 132 +++++++++++++++++++++++++++++++++++++ internal/qos/validate.go | 11 +++- 4 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 internal/assert/cond.go create mode 100644 internal/qos/limiter.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 004c8f128..c625a041f 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -136,6 +136,7 @@ type shardCfg struct { refillMetabase bool refillMetabaseWorkersCount int mode shardmode.Mode + limiter qos.Limiter metaCfg struct { path string @@ -278,7 +279,7 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, oldConfig a.setMetabaseConfig(&newConfig, oldConfig) a.setGCConfig(&newConfig, oldConfig) - if err := a.setLimits(&newConfig, oldConfig); err != nil { + if err := a.setLimiter(&newConfig, oldConfig); err != nil { return err } @@ -374,11 +375,16 @@ func (a *applicationConfiguration) setGCConfig(newConfig *shardCfg, oldConfig *s newConfig.gcCfg.expiredCollectorWorkerCount = gcCfg.ExpiredCollectorWorkerCount() } -func (a *applicationConfiguration) setLimits(newConfig *shardCfg, oldConfig *shardconfig.Config) error { +func (a *applicationConfiguration) setLimiter(newConfig *shardCfg, oldConfig *shardconfig.Config) error { limitsConfig := oldConfig.Limits() - if err := qos.ValidateConfig(limitsConfig); err != nil { + limiter, err := qos.NewLimiter(limitsConfig) + if err != nil { return err } + if newConfig.limiter != nil { + newConfig.limiter.Close() + } + newConfig.limiter = limiter return nil } diff --git a/internal/assert/cond.go b/internal/assert/cond.go new file mode 100644 index 000000000..701036fa8 --- /dev/null +++ b/internal/assert/cond.go @@ -0,0 +1,9 @@ +package assert + +import "strings" + +func True(cond bool, details ...string) { + if !cond { + panic(strings.Join(details, " ")) + } +} diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go new file mode 100644 index 000000000..d2a1919f1 --- /dev/null +++ b/internal/qos/limiter.go @@ -0,0 +1,132 @@ +package qos + +import ( + "context" + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" + "git.frostfs.info/TrueCloudLab/frostfs-qos/scheduling" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" +) + +const ( + defaultIdleTimeout time.Duration = 0 + defaultShare float64 = 1.0 +) + +type ReleaseFunc scheduling.ReleaseFunc + +type Limiter interface { + ReadRequest(context.Context) (ReleaseFunc, error) + WriteRequest(context.Context) (ReleaseFunc, error) + Close() +} + +func NewLimiter(c *limits.Config) (Limiter, error) { + if err := validateConfig(c); err != nil { + return nil, err + } + read, write := c.Read(), c.Write() + if isNoop(read, write) { + return noopLimiterInstance, nil + } + readScheduler, err := scheduling.NewMClock( + uint64(read.MaxRunningOps), uint64(read.MaxWaitingOps), + converToSchedulingTags(read.Tags), read.IdleTimeout) + if err != nil { + return nil, fmt.Errorf("create read scheduler: %w", err) + } + writeScheduler, err := scheduling.NewMClock( + uint64(write.MaxRunningOps), uint64(write.MaxWaitingOps), + converToSchedulingTags(write.Tags), write.IdleTimeout) + if err != nil { + return nil, fmt.Errorf("create write scheduler: %w", err) + } + return &mClockLimiter{ + readScheduler: readScheduler, + writeScheduler: writeScheduler, + }, nil +} + +func converToSchedulingTags(limits []limits.IOTagConfig) map[string]scheduling.TagInfo { + result := make(map[string]scheduling.TagInfo) + for _, tag := range []IOTag{IOTagClient, IOTagBackground, IOTagInternal, IOTagPolicer, IOTagWritecache} { + result[tag.String()] = scheduling.TagInfo{ + Share: defaultShare, + } + } + for _, l := range limits { + v := result[l.Tag] + if l.Weight != nil && *l.Weight != 0 { + v.Share = *l.Weight + } + if l.LimitOps != nil && *l.LimitOps != 0 { + v.LimitIOPS = l.LimitOps + } + if l.ReservedOps != nil && *l.ReservedOps != 0 { + v.ReservedIOPS = l.ReservedOps + } + result[l.Tag] = v + } + return result +} + +var ( + _ Limiter = (*noopLimiter)(nil) + releaseStub ReleaseFunc = func() {} + noopLimiterInstance = &noopLimiter{} +) + +type noopLimiter struct{} + +func (n *noopLimiter) ReadRequest(context.Context) (ReleaseFunc, error) { + return releaseStub, nil +} + +func (n *noopLimiter) WriteRequest(context.Context) (ReleaseFunc, error) { + return releaseStub, nil +} + +func (n *noopLimiter) Close() {} + +var _ Limiter = (*mClockLimiter)(nil) + +type mClockLimiter struct { + readScheduler *scheduling.MClock + writeScheduler *scheduling.MClock +} + +func (n *mClockLimiter) ReadRequest(ctx context.Context) (ReleaseFunc, error) { + tag, ok := tagging.IOTagFromContext(ctx) + assert.True(ok, "request has no tag") + if tag == IOTagCritical.String() { + return releaseStub, nil + } + rel, err := n.readScheduler.RequestArrival(ctx, tag) + if err != nil { + return nil, err + } + return ReleaseFunc(rel), nil +} + +func (n *mClockLimiter) WriteRequest(ctx context.Context) (ReleaseFunc, error) { + tag, ok := tagging.IOTagFromContext(ctx) + if !ok { + tag = IOTagClient.String() + } + if tag == IOTagCritical.String() { + return releaseStub, nil + } + rel, err := n.writeScheduler.RequestArrival(ctx, tag) + if err != nil { + return nil, err + } + return ReleaseFunc(rel), nil +} + +func (n *mClockLimiter) Close() { + n.readScheduler.Close() + n.writeScheduler.Close() +} diff --git a/internal/qos/validate.go b/internal/qos/validate.go index afced345b..43aa74942 100644 --- a/internal/qos/validate.go +++ b/internal/qos/validate.go @@ -14,7 +14,7 @@ type tagConfig struct { Shares, Limit, Reserved *float64 } -func ValidateConfig(c *limits.Config) error { +func validateConfig(c *limits.Config) error { if err := validateOpConfig(c.Read()); err != nil { return fmt.Errorf("limits 'read' section validation error: %w", err) } @@ -90,3 +90,12 @@ func float64Value(f *float64) float64 { } return *f } + +func isNoop(read, write limits.OpConfig) bool { + return read.MaxRunningOps == limits.NoLimit && + read.MaxWaitingOps == limits.NoLimit && + write.MaxRunningOps == limits.NoLimit && + write.MaxWaitingOps == limits.NoLimit && + len(read.Tags) == 0 && + len(write.Tags) == 0 +} From e0dc3c3d0c4b948152cd054bda5545c64f413153 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Feb 2025 15:57:27 +0300 Subject: [PATCH 1746/1943] [#1636] shard: Add limiter usage Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 + internal/qos/limiter.go | 4 + pkg/local_object_storage/engine/container.go | 2 +- pkg/local_object_storage/engine/inhume.go | 2 +- pkg/local_object_storage/shard/container.go | 26 ++- pkg/local_object_storage/shard/control.go | 4 + pkg/local_object_storage/shard/count.go | 6 + pkg/local_object_storage/shard/delete.go | 6 + pkg/local_object_storage/shard/exists.go | 17 +- pkg/local_object_storage/shard/gc.go | 177 +++++++++++++------ pkg/local_object_storage/shard/get.go | 6 + pkg/local_object_storage/shard/head.go | 6 + pkg/local_object_storage/shard/inhume.go | 6 + pkg/local_object_storage/shard/list.go | 40 ++++- pkg/local_object_storage/shard/lock.go | 21 ++- pkg/local_object_storage/shard/put.go | 6 + pkg/local_object_storage/shard/range.go | 6 + pkg/local_object_storage/shard/rebuild.go | 1 + pkg/local_object_storage/shard/select.go | 6 + pkg/local_object_storage/shard/shard.go | 10 ++ pkg/local_object_storage/shard/tree.go | 85 +++++++++ pkg/local_object_storage/shard/writecache.go | 19 ++ 22 files changed, 390 insertions(+), 67 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c625a041f..b42275538 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1072,6 +1072,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID return pool }), + shard.WithLimiter(shCfg.limiter), } return sh } diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index d2a1919f1..996cebea1 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -79,6 +79,10 @@ var ( noopLimiterInstance = &noopLimiter{} ) +func NewNoopLimiter() Limiter { + return &noopLimiter{} +} + type noopLimiter struct{} func (n *noopLimiter) ReadRequest(context.Context) (ReleaseFunc, error) { diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index b2d7a1037..3160d7f83 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -74,7 +74,7 @@ func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) var csPrm shard.ContainerSizePrm csPrm.SetContainerID(prm.cnr) - csRes, err := sh.Shard.ContainerSize(csPrm) + csRes, err := sh.Shard.ContainerSize(ctx, csPrm) if err != nil { e.reportShardError(ctx, sh, "can't get container size", err, zap.Stringer("container_id", prm.cnr)) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index fb802ef2a..c8ee33b53 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -339,7 +339,7 @@ func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid var drop []cid.ID for id := range idMap { prm.SetContainerID(id) - s, err := sh.ContainerSize(prm) + s, err := sh.ContainerSize(ctx, prm) if err != nil { e.log.Warn(ctx, logs.EngineFailedToGetContainerSize, zap.Stringer("container_id", id), zap.Error(err)) failed = true diff --git a/pkg/local_object_storage/shard/container.go b/pkg/local_object_storage/shard/container.go index 0309f0c81..b4015ae8d 100644 --- a/pkg/local_object_storage/shard/container.go +++ b/pkg/local_object_storage/shard/container.go @@ -26,7 +26,7 @@ func (r ContainerSizeRes) Size() uint64 { return r.size } -func (s *Shard) ContainerSize(prm ContainerSizePrm) (ContainerSizeRes, error) { +func (s *Shard) ContainerSize(ctx context.Context, prm ContainerSizePrm) (ContainerSizeRes, error) { s.m.RLock() defer s.m.RUnlock() @@ -34,6 +34,12 @@ func (s *Shard) ContainerSize(prm ContainerSizePrm) (ContainerSizeRes, error) { return ContainerSizeRes{}, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return ContainerSizeRes{}, err + } + defer release() + size, err := s.metaBase.ContainerSize(prm.cnr) if err != nil { return ContainerSizeRes{}, fmt.Errorf("get container size: %w", err) @@ -69,6 +75,12 @@ func (s *Shard) ContainerCount(ctx context.Context, prm ContainerCountPrm) (Cont return ContainerCountRes{}, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return ContainerCountRes{}, err + } + defer release() + counters, err := s.metaBase.ContainerCount(ctx, prm.ContainerID) if err != nil { return ContainerCountRes{}, fmt.Errorf("get container counters: %w", err) @@ -100,6 +112,12 @@ func (s *Shard) DeleteContainerSize(ctx context.Context, id cid.ID) error { return ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return err + } + defer release() + return s.metaBase.DeleteContainerSize(ctx, id) } @@ -122,5 +140,11 @@ func (s *Shard) DeleteContainerCount(ctx context.Context, id cid.ID) error { return ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return err + } + defer release() + return s.metaBase.DeleteContainerCount(ctx, id) } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index fedde2206..3520277c0 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -395,6 +395,10 @@ func (s *Shard) Close(ctx context.Context) error { s.gc.stop(ctx) } + if s.opsLimiter != nil { + s.opsLimiter.Close() + } + return lastErr } diff --git a/pkg/local_object_storage/shard/count.go b/pkg/local_object_storage/shard/count.go index b3bc6a30b..8dc1f0522 100644 --- a/pkg/local_object_storage/shard/count.go +++ b/pkg/local_object_storage/shard/count.go @@ -23,6 +23,12 @@ func (s *Shard) LogicalObjectsCount(ctx context.Context) (uint64, error) { return 0, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return 0, err + } + defer release() + cc, err := s.metaBase.ObjectCounters() if err != nil { return 0, err diff --git a/pkg/local_object_storage/shard/delete.go b/pkg/local_object_storage/shard/delete.go index 55231b032..0101817a8 100644 --- a/pkg/local_object_storage/shard/delete.go +++ b/pkg/local_object_storage/shard/delete.go @@ -54,6 +54,12 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm, skipFailed bool) (Del return DeleteRes{}, ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return DeleteRes{}, err + } + defer release() + result := DeleteRes{} for _, addr := range prm.addr { select { diff --git a/pkg/local_object_storage/shard/exists.go b/pkg/local_object_storage/shard/exists.go index 82ce48dde..2c11b6b01 100644 --- a/pkg/local_object_storage/shard/exists.go +++ b/pkg/local_object_storage/shard/exists.go @@ -53,10 +53,6 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { )) defer span.End() - var exists bool - var locked bool - var err error - s.m.RLock() defer s.m.RUnlock() @@ -64,7 +60,18 @@ func (s *Shard) Exists(ctx context.Context, prm ExistsPrm) (ExistsRes, error) { return ExistsRes{}, ErrShardDisabled } else if s.info.EvacuationInProgress { return ExistsRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) - } else if s.info.Mode.NoMetabase() { + } + + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return ExistsRes{}, err + } + defer release() + + var exists bool + var locked bool + + if s.info.Mode.NoMetabase() { var p common.ExistsPrm p.Address = prm.Address diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 4a5ec7a71..32a377cd5 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -291,28 +291,7 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { s.log.Debug(ctx, logs.ShardGCRemoveGarbageStarted) defer s.log.Debug(ctx, logs.ShardGCRemoveGarbageCompleted) - buf := make([]oid.Address, 0, s.rmBatchSize) - - var iterPrm meta.GarbageIterationPrm - iterPrm.SetHandler(func(g meta.GarbageObject) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - buf = append(buf, g.Address()) - - if len(buf) == s.rmBatchSize { - return meta.ErrInterruptIterator - } - - return nil - }) - - // iterate over metabase's objects with GC mark - // (no more than s.rmBatchSize objects) - err := s.metaBase.IterateOverGarbage(ctx, iterPrm) + buf, err := s.getGarbage(ctx) if err != nil { s.log.Warn(ctx, logs.ShardIteratorOverMetabaseGraveyardFailed, zap.Error(err), @@ -344,6 +323,39 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) { return } +func (s *Shard) getGarbage(ctx context.Context) ([]oid.Address, error) { + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return nil, err + } + defer release() + + buf := make([]oid.Address, 0, s.rmBatchSize) + + var iterPrm meta.GarbageIterationPrm + iterPrm.SetHandler(func(g meta.GarbageObject) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + buf = append(buf, g.Address()) + + if len(buf) == s.rmBatchSize { + return meta.ErrInterruptIterator + } + + return nil + }) + + if err := s.metaBase.IterateOverGarbage(ctx, iterPrm); err != nil { + return nil, err + } + + return buf, nil +} + func (s *Shard) getExpiredObjectsParameters() (workerCount, batchSize int) { workerCount = max(minExpiredWorkers, s.gc.gcCfg.expiredCollectorWorkerCount) batchSize = max(minExpiredBatchSize, s.gc.gcCfg.expiredCollectorBatchSize) @@ -422,18 +434,9 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } - var inhumePrm meta.InhumePrm - - inhumePrm.SetAddresses(expired...) - inhumePrm.SetGCMark() - - // inhume the collected objects - res, err := s.metaBase.Inhume(ctx, inhumePrm) + res, err := s.inhumeGC(ctx, expired) if err != nil { - s.log.Warn(ctx, logs.ShardCouldNotInhumeTheObjects, - zap.Error(err), - ) - + s.log.Warn(ctx, logs.ShardCouldNotInhumeTheObjects, zap.Error(err)) return } @@ -451,6 +454,12 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) } func (s *Shard) getExpiredWithLinked(ctx context.Context, source []oid.Address) ([]oid.Address, error) { + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return nil, err + } + defer release() + result := make([]oid.Address, 0, len(source)) parentToChildren, err := s.metaBase.GetChildren(ctx, source) if err != nil { @@ -464,6 +473,19 @@ func (s *Shard) getExpiredWithLinked(ctx context.Context, source []oid.Address) return result, nil } +func (s *Shard) inhumeGC(ctx context.Context, addrs []oid.Address) (meta.InhumeRes, error) { + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return meta.InhumeRes{}, err + } + defer release() + + var inhumePrm meta.InhumePrm + inhumePrm.SetAddresses(addrs...) + inhumePrm.SetGCMark() + return s.metaBase.Inhume(ctx, inhumePrm) +} + func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { var err error startedAt := time.Now() @@ -505,11 +527,17 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { return } - err = s.metaBase.IterateOverGraveyard(ctx, iterPrm) + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + log.Error(ctx, logs.ShardIteratorOverGraveyardFailed, zap.Error(err)) + s.m.RUnlock() + return + } + err = s.metaBase.IterateOverGraveyard(ctx, iterPrm) + release() if err != nil { log.Error(ctx, logs.ShardIteratorOverGraveyardFailed, zap.Error(err)) s.m.RUnlock() - return } @@ -598,7 +626,13 @@ func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, onExpiredFo return ErrDegradedMode } - err := s.metaBase.IterateExpired(ctx, epoch, func(expiredObject *meta.ExpiredObject) error { + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return err + } + defer release() + + err = s.metaBase.IterateExpired(ctx, epoch, func(expiredObject *meta.ExpiredObject) error { select { case <-ctx.Done(): return meta.ErrInterruptIterator @@ -621,6 +655,12 @@ func (s *Shard) selectExpired(ctx context.Context, epoch uint64, addresses []oid return nil, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return nil, err + } + defer release() + return s.metaBase.FilterExpired(ctx, epoch, addresses) } @@ -636,12 +676,15 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston return } - res, err := s.metaBase.InhumeTombstones(ctx, tss) + release, err := s.opsLimiter.WriteRequest(ctx) if err != nil { - s.log.Warn(ctx, logs.ShardCouldNotMarkTombstonesAsGarbage, - zap.Error(err), - ) - + s.log.Warn(ctx, logs.ShardCouldNotMarkTombstonesAsGarbage, zap.Error(err)) + return + } + res, err := s.metaBase.InhumeTombstones(ctx, tss) + release() + if err != nil { + s.log.Warn(ctx, logs.ShardCouldNotMarkTombstonesAsGarbage, zap.Error(err)) return } @@ -664,11 +707,16 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] if s.GetMode().NoMetabase() { return } - unlocked, err := s.metaBase.FreeLockedBy(lockers) + + release, err := s.opsLimiter.WriteRequest(ctx) if err != nil { - s.log.Warn(ctx, logs.ShardFailureToUnlockObjects, - zap.Error(err), - ) + s.log.Warn(ctx, logs.ShardFailureToUnlockObjects, zap.Error(err)) + return + } + unlocked, err := s.metaBase.FreeLockedBy(lockers) + release() + if err != nil { + s.log.Warn(ctx, logs.ShardFailureToUnlockObjects, zap.Error(err)) return } @@ -676,13 +724,15 @@ func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers [] var pInhume meta.InhumePrm pInhume.SetAddresses(lockers...) pInhume.SetForceGCMark() - - res, err := s.metaBase.Inhume(ctx, pInhume) + release, err = s.opsLimiter.WriteRequest(ctx) if err != nil { - s.log.Warn(ctx, logs.ShardFailureToMarkLockersAsGarbage, - zap.Error(err), - ) - + s.log.Warn(ctx, logs.ShardFailureToMarkLockersAsGarbage, zap.Error(err)) + return + } + res, err := s.metaBase.Inhume(ctx, pInhume) + release() + if err != nil { + s.log.Warn(ctx, logs.ShardFailureToMarkLockersAsGarbage, zap.Error(err)) return } @@ -721,12 +771,15 @@ func (s *Shard) HandleDeletedLocks(ctx context.Context, lockers []oid.Address) { return } - _, err := s.metaBase.FreeLockedBy(lockers) + release, err := s.opsLimiter.WriteRequest(ctx) if err != nil { - s.log.Warn(ctx, logs.ShardFailureToUnlockObjects, - zap.Error(err), - ) - + s.log.Warn(ctx, logs.ShardFailureToUnlockObjects, zap.Error(err)) + return + } + _, err = s.metaBase.FreeLockedBy(lockers) + release() + if err != nil { + s.log.Warn(ctx, logs.ShardFailureToUnlockObjects, zap.Error(err)) return } } @@ -750,7 +803,13 @@ func (s *Shard) collectExpiredMetrics(ctx context.Context, e Event) { } func (s *Shard) collectExpiredContainerSizeMetrics(ctx context.Context, epoch uint64) { + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + s.log.Warn(ctx, logs.ShardGCFailedToCollectZeroSizeContainers, zap.Uint64("epoch", epoch), zap.Error(err)) + return + } ids, err := s.metaBase.ZeroSizeContainers(ctx) + release() if err != nil { s.log.Warn(ctx, logs.ShardGCFailedToCollectZeroSizeContainers, zap.Uint64("epoch", epoch), zap.Error(err)) return @@ -762,7 +821,13 @@ func (s *Shard) collectExpiredContainerSizeMetrics(ctx context.Context, epoch ui } func (s *Shard) collectExpiredContainerCountMetrics(ctx context.Context, epoch uint64) { + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + s.log.Warn(ctx, logs.ShardGCFailedToCollectZeroCountContainers, zap.Uint64("epoch", epoch), zap.Error(err)) + return + } ids, err := s.metaBase.ZeroCountContainers(ctx) + release() if err != nil { s.log.Warn(ctx, logs.ShardGCFailedToCollectZeroCountContainers, zap.Uint64("epoch", epoch), zap.Error(err)) return diff --git a/pkg/local_object_storage/shard/get.go b/pkg/local_object_storage/shard/get.go index 05823c62b..28f8912be 100644 --- a/pkg/local_object_storage/shard/get.go +++ b/pkg/local_object_storage/shard/get.go @@ -111,6 +111,12 @@ func (s *Shard) Get(ctx context.Context, prm GetPrm) (GetRes, error) { return c.Get(ctx, prm.addr) } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return GetRes{}, err + } + defer release() + skipMeta := prm.skipMeta || s.info.Mode.NoMetabase() obj, hasMeta, err := s.fetchObjectData(ctx, prm.addr, skipMeta, cb, wc) diff --git a/pkg/local_object_storage/shard/head.go b/pkg/local_object_storage/shard/head.go index ff57e3bf9..34b8290d6 100644 --- a/pkg/local_object_storage/shard/head.go +++ b/pkg/local_object_storage/shard/head.go @@ -81,6 +81,12 @@ func (s *Shard) Head(ctx context.Context, prm HeadPrm) (HeadRes, error) { headParams.SetAddress(prm.addr) headParams.SetRaw(prm.raw) + release, limitErr := s.opsLimiter.ReadRequest(ctx) + if limitErr != nil { + return HeadRes{}, limitErr + } + defer release() + var res meta.GetRes res, err = s.metaBase.Get(ctx, headParams) obj = res.Header() diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 9d5f66063..c0fd65f4b 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -81,6 +81,12 @@ func (s *Shard) Inhume(ctx context.Context, prm InhumePrm) (InhumeRes, error) { return InhumeRes{}, ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return InhumeRes{}, err + } + defer release() + if s.hasWriteCache() { for i := range prm.target { _ = s.writeCache.Delete(ctx, prm.target[i]) diff --git a/pkg/local_object_storage/shard/list.go b/pkg/local_object_storage/shard/list.go index 7bc5ead1d..af87981ca 100644 --- a/pkg/local_object_storage/shard/list.go +++ b/pkg/local_object_storage/shard/list.go @@ -106,6 +106,12 @@ func (s *Shard) List(ctx context.Context) (res SelectRes, err error) { return SelectRes{}, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return SelectRes{}, err + } + defer release() + lst, err := s.metaBase.Containers(ctx) if err != nil { return res, fmt.Errorf("list stored containers: %w", err) @@ -145,6 +151,12 @@ func (s *Shard) ListContainers(ctx context.Context, _ ListContainersPrm) (ListCo return ListContainersRes{}, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return ListContainersRes{}, err + } + defer release() + containers, err := s.metaBase.Containers(ctx) if err != nil { return ListContainersRes{}, fmt.Errorf("get list of containers: %w", err) @@ -173,6 +185,12 @@ func (s *Shard) ListWithCursor(ctx context.Context, prm ListWithCursorPrm) (List return ListWithCursorRes{}, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return ListWithCursorRes{}, err + } + defer release() + var metaPrm meta.ListPrm metaPrm.SetCount(prm.count) metaPrm.SetCursor(prm.cursor) @@ -202,9 +220,15 @@ func (s *Shard) IterateOverContainers(ctx context.Context, prm IterateOverContai return ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return err + } + defer release() + var metaPrm meta.IterateOverContainersPrm metaPrm.Handler = prm.Handler - err := s.metaBase.IterateOverContainers(ctx, metaPrm) + err = s.metaBase.IterateOverContainers(ctx, metaPrm) if err != nil { return fmt.Errorf("iterate over containers: %w", err) } @@ -227,11 +251,17 @@ func (s *Shard) IterateOverObjectsInContainer(ctx context.Context, prm IterateOv return ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return err + } + defer release() + var metaPrm meta.IterateOverObjectsInContainerPrm metaPrm.ContainerID = prm.ContainerID metaPrm.ObjectType = prm.ObjectType metaPrm.Handler = prm.Handler - err := s.metaBase.IterateOverObjectsInContainer(ctx, metaPrm) + err = s.metaBase.IterateOverObjectsInContainer(ctx, metaPrm) if err != nil { return fmt.Errorf("iterate over objects: %w", err) } @@ -251,6 +281,12 @@ func (s *Shard) CountAliveObjectsInContainer(ctx context.Context, prm CountAlive return 0, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return 0, err + } + defer release() + var metaPrm meta.CountAliveObjectsInContainerPrm metaPrm.ObjectType = prm.ObjectType metaPrm.ContainerID = prm.ContainerID diff --git a/pkg/local_object_storage/shard/lock.go b/pkg/local_object_storage/shard/lock.go index 31ca16aa1..9c392fdac 100644 --- a/pkg/local_object_storage/shard/lock.go +++ b/pkg/local_object_storage/shard/lock.go @@ -38,7 +38,13 @@ func (s *Shard) Lock(ctx context.Context, idCnr cid.ID, locker oid.ID, locked [] return ErrDegradedMode } - err := s.metaBase.Lock(ctx, idCnr, locker, locked) + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return err + } + defer release() + + err = s.metaBase.Lock(ctx, idCnr, locker, locked) if err != nil { return fmt.Errorf("metabase lock: %w", err) } @@ -61,6 +67,12 @@ func (s *Shard) IsLocked(ctx context.Context, addr oid.Address) (bool, error) { return false, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return false, err + } + defer release() + var prm meta.IsLockedPrm prm.SetAddress(addr) @@ -86,5 +98,12 @@ func (s *Shard) GetLocks(ctx context.Context, addr oid.Address) ([]oid.ID, error if m.NoMetabase() { return nil, ErrDegradedMode } + + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return nil, err + } + defer release() + return s.metaBase.GetLocks(ctx, addr) } diff --git a/pkg/local_object_storage/shard/put.go b/pkg/local_object_storage/shard/put.go index 3f23111af..f8cb00a31 100644 --- a/pkg/local_object_storage/shard/put.go +++ b/pkg/local_object_storage/shard/put.go @@ -67,6 +67,12 @@ func (s *Shard) Put(ctx context.Context, prm PutPrm) (PutRes, error) { var res common.PutRes + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return PutRes{}, err + } + defer release() + // exist check are not performed there, these checks should be executed // ahead of `Put` by storage engine tryCache := s.hasWriteCache() && !m.NoMetabase() diff --git a/pkg/local_object_storage/shard/range.go b/pkg/local_object_storage/shard/range.go index 701268820..443689104 100644 --- a/pkg/local_object_storage/shard/range.go +++ b/pkg/local_object_storage/shard/range.go @@ -131,6 +131,12 @@ func (s *Shard) GetRange(ctx context.Context, prm RngPrm) (RngRes, error) { return obj, nil } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return RngRes{}, err + } + defer release() + skipMeta := prm.skipMeta || s.info.Mode.NoMetabase() obj, hasMeta, err := s.fetchObjectData(ctx, prm.addr, skipMeta, cb, wc) diff --git a/pkg/local_object_storage/shard/rebuild.go b/pkg/local_object_storage/shard/rebuild.go index 0593f5894..9fe1bbe8c 100644 --- a/pkg/local_object_storage/shard/rebuild.go +++ b/pkg/local_object_storage/shard/rebuild.go @@ -106,6 +106,7 @@ func runRebuild(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *lo } log.Info(ctx, logs.BlobstoreRebuildStarted) ctx = tagging.ContextWithIOTag(ctx, qos.IOTagBackground.String()) + // TODO use shard limiter if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, limiter, fillPercent); err != nil { log.Warn(ctx, logs.FailedToRebuildBlobstore, zap.Error(err)) } else { diff --git a/pkg/local_object_storage/shard/select.go b/pkg/local_object_storage/shard/select.go index c7c7e11c2..fbc751e26 100644 --- a/pkg/local_object_storage/shard/select.go +++ b/pkg/local_object_storage/shard/select.go @@ -60,6 +60,12 @@ func (s *Shard) Select(ctx context.Context, prm SelectPrm) (SelectRes, error) { return SelectRes{}, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return SelectRes{}, nil + } + defer release() + var selectPrm meta.SelectPrm selectPrm.SetFilters(prm.filters) selectPrm.SetContainerID(prm.cnr) diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 1eb7f14d0..b9ec05f01 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -7,6 +7,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -98,6 +99,8 @@ type cfg struct { reportErrorFunc func(ctx context.Context, selfID string, message string, err error) containerInfo container.InfoProvider + + opsLimiter qos.Limiter } func defaultCfg() *cfg { @@ -109,6 +112,7 @@ func defaultCfg() *cfg { zeroSizeContainersCallback: func(context.Context, []cid.ID) {}, zeroCountContainersCallback: func(context.Context, []cid.ID) {}, metricsWriter: noopMetrics{}, + opsLimiter: qos.NewNoopLimiter(), } } @@ -368,6 +372,12 @@ func WithContainerInfoProvider(containerInfo container.InfoProvider) Option { } } +func WithLimiter(l qos.Limiter) Option { + return func(c *cfg) { + c.opsLimiter = l + } +} + func (s *Shard) fillInfo() { s.cfg.info.MetaBaseInfo = s.metaBase.DumpInfo() s.cfg.info.BlobStorInfo = s.blobStor.DumpInfo() diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index 01a014cec..e9cd5f8c1 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -43,6 +43,11 @@ func (s *Shard) TreeMove(ctx context.Context, d pilorama.CIDDescriptor, treeID s if s.info.Mode.NoMetabase() { return nil, ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return nil, err + } + defer release() return s.pilorama.TreeMove(ctx, d, treeID, m) } @@ -75,6 +80,11 @@ func (s *Shard) TreeAddByPath(ctx context.Context, d pilorama.CIDDescriptor, tre if s.info.Mode.NoMetabase() { return nil, ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return nil, err + } + defer release() return s.pilorama.TreeAddByPath(ctx, d, treeID, attr, path, meta) } @@ -103,6 +113,11 @@ func (s *Shard) TreeApply(ctx context.Context, cnr cidSDK.ID, treeID string, m * if s.info.Mode.NoMetabase() { return ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return err + } + defer release() return s.pilorama.TreeApply(ctx, cnr, treeID, m, backgroundSync) } @@ -130,6 +145,11 @@ func (s *Shard) TreeApplyBatch(ctx context.Context, cnr cidSDK.ID, treeID string if s.info.Mode.NoMetabase() { return ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return err + } + defer release() return s.pilorama.TreeApplyBatch(ctx, cnr, treeID, m) } @@ -157,6 +177,11 @@ func (s *Shard) TreeGetByPath(ctx context.Context, cid cidSDK.ID, treeID string, if s.info.Mode.NoMetabase() { return nil, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return nil, err + } + defer release() return s.pilorama.TreeGetByPath(ctx, cid, treeID, attr, path, latest) } @@ -182,6 +207,11 @@ func (s *Shard) TreeGetMeta(ctx context.Context, cid cidSDK.ID, treeID string, n if s.info.Mode.NoMetabase() { return pilorama.Meta{}, 0, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return pilorama.Meta{}, 0, err + } + defer release() return s.pilorama.TreeGetMeta(ctx, cid, treeID, nodeID) } @@ -207,6 +237,11 @@ func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID strin if s.info.Mode.NoMetabase() { return nil, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return nil, err + } + defer release() return s.pilorama.TreeGetChildren(ctx, cid, treeID, nodeID) } @@ -231,6 +266,11 @@ func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID if s.info.Mode.NoMetabase() { return nil, last, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return nil, last, err + } + defer release() return s.pilorama.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count) } @@ -256,6 +296,11 @@ func (s *Shard) TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, if s.info.Mode.NoMetabase() { return pilorama.Move{}, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return pilorama.Move{}, err + } + defer release() return s.pilorama.TreeGetOpLog(ctx, cid, treeID, height) } @@ -280,6 +325,11 @@ func (s *Shard) TreeDrop(ctx context.Context, cid cidSDK.ID, treeID string) erro if s.info.Mode.NoMetabase() { return ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return err + } + defer release() return s.pilorama.TreeDrop(ctx, cid, treeID) } @@ -303,6 +353,11 @@ func (s *Shard) TreeList(ctx context.Context, cid cidSDK.ID) ([]string, error) { if s.info.Mode.NoMetabase() { return nil, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return nil, err + } + defer release() return s.pilorama.TreeList(ctx, cid) } @@ -326,6 +381,11 @@ func (s *Shard) TreeHeight(ctx context.Context, cid cidSDK.ID, treeID string) (u if s.pilorama == nil { return 0, ErrPiloramaDisabled } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return 0, err + } + defer release() return s.pilorama.TreeHeight(ctx, cid, treeID) } @@ -350,6 +410,11 @@ func (s *Shard) TreeExists(ctx context.Context, cid cidSDK.ID, treeID string) (b if s.info.Mode.NoMetabase() { return false, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return false, err + } + defer release() return s.pilorama.TreeExists(ctx, cid, treeID) } @@ -378,6 +443,11 @@ func (s *Shard) TreeUpdateLastSyncHeight(ctx context.Context, cid cidSDK.ID, tre if s.info.Mode.NoMetabase() { return ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return err + } + defer release() return s.pilorama.TreeUpdateLastSyncHeight(ctx, cid, treeID, height) } @@ -402,6 +472,11 @@ func (s *Shard) TreeLastSyncHeight(ctx context.Context, cid cidSDK.ID, treeID st if s.info.Mode.NoMetabase() { return 0, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return 0, err + } + defer release() return s.pilorama.TreeLastSyncHeight(ctx, cid, treeID) } @@ -423,6 +498,11 @@ func (s *Shard) TreeListTrees(ctx context.Context, prm pilorama.TreeListTreesPrm if s.info.Mode.NoMetabase() { return nil, ErrDegradedMode } + release, err := s.opsLimiter.ReadRequest(ctx) + if err != nil { + return nil, err + } + defer release() return s.pilorama.TreeListTrees(ctx, prm) } @@ -452,5 +532,10 @@ func (s *Shard) TreeApplyStream(ctx context.Context, cnr cidSDK.ID, treeID strin if s.info.Mode.NoMetabase() { return ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return err + } + defer release() return s.pilorama.TreeApplyStream(ctx, cnr, treeID, source) } diff --git a/pkg/local_object_storage/shard/writecache.go b/pkg/local_object_storage/shard/writecache.go index f655e477a..9edb89df8 100644 --- a/pkg/local_object_storage/shard/writecache.go +++ b/pkg/local_object_storage/shard/writecache.go @@ -67,6 +67,12 @@ func (s *Shard) FlushWriteCache(ctx context.Context, p FlushWriteCachePrm) error return ErrDegradedMode } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return err + } + defer release() + return s.writeCache.Flush(ctx, p.ignoreErrors, p.seal) } @@ -124,6 +130,13 @@ func (s *Shard) SealWriteCache(ctx context.Context, p SealWriteCachePrm) error { close(started) defer cleanup() + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + s.log.Warn(ctx, logs.FailedToSealWritecacheAsync, zap.Error(err)) + return + } + defer release() + s.log.Info(ctx, logs.StartedWritecacheSealAsync) if err := s.writeCache.Seal(ctx, prm); err != nil { s.log.Warn(ctx, logs.FailedToSealWritecacheAsync, zap.Error(err)) @@ -138,5 +151,11 @@ func (s *Shard) SealWriteCache(ctx context.Context, p SealWriteCachePrm) error { return nil } } + release, err := s.opsLimiter.WriteRequest(ctx) + if err != nil { + return err + } + defer release() + return s.writeCache.Seal(ctx, prm) } From ceff5e1f6a08c12d84a04fcfa2e23f097bfa97fa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 6 Feb 2025 17:24:23 +0300 Subject: [PATCH 1747/1943] [#1636] storage: Refactor shard rebuild Drop redundant interfaces. Rename fields. Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/rebuild.go | 13 ++--- .../blobovniczatree/rebuild_failover_test.go | 6 +-- .../blobstor/blobovniczatree/rebuild_test.go | 33 ++++++------- .../blobstor/common/rebuild.go | 17 ++++--- pkg/local_object_storage/blobstor/rebuild.go | 13 ++--- pkg/local_object_storage/engine/rebuild.go | 22 ++++++++- pkg/local_object_storage/shard/rebuild.go | 47 ++++--------------- 7 files changed, 72 insertions(+), 79 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index 16ef2b180..cbd45c3b4 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -79,7 +79,7 @@ func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common. var completedDBCount uint32 for _, db := range dbs { b.log.Debug(ctx, logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db)) - movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage, prm.WorkerLimiter) + movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage, prm.Limiter) res.ObjectsMoved += movedObjects if err != nil { b.log.Warn(ctx, logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err)) @@ -195,7 +195,7 @@ func (b *Blobovniczas) rebuildBySize(ctx context.Context, path string, targetFil return fp < targetFillPercent || fp > 100+(100-targetFillPercent), nil } -func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.MetaStorage, limiter common.ConcurrentWorkersLimiter) (uint64, error) { +func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.MetaStorage, concLimiter common.RebuildLimiter) (uint64, error) { shDB := b.getBlobovnicza(ctx, path) blz, err := shDB.Open(ctx) if err != nil { @@ -212,7 +212,7 @@ func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.M if err != nil { return 0, err } - migratedObjects, err := b.moveObjects(ctx, blz, shDB.SystemPath(), meta, limiter) + migratedObjects, err := b.moveObjects(ctx, blz, shDB.SystemPath(), meta, concLimiter) if err != nil { return migratedObjects, err } @@ -238,7 +238,7 @@ func (b *Blobovniczas) addRebuildTempFile(ctx context.Context, path string) (fun }, nil } -func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovnicza, blzPath string, meta common.MetaStorage, limiter common.ConcurrentWorkersLimiter) (uint64, error) { +func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovnicza, blzPath string, meta common.MetaStorage, concLimiter common.RebuildLimiter) (uint64, error) { var result atomic.Uint64 batch := make(map[oid.Address][]byte) @@ -265,12 +265,13 @@ func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovn eg, egCtx := errgroup.WithContext(ctx) for addr, data := range batch { - if err := limiter.AcquireWorkSlot(egCtx); err != nil { + release, err := concLimiter.AcquireWorkSlot(egCtx) + if err != nil { _ = eg.Wait() return result.Load(), err } eg.Go(func() error { - defer limiter.ReleaseWorkSlot() + defer release() err := b.moveObject(egCtx, blz, blzPath, addr, data, meta) if err == nil { result.Add(1) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index 2f58624aa..91578d5e8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -162,9 +162,9 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object guard: &sync.Mutex{}, } rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ - MetaStorage: metaStub, - WorkerLimiter: &rebuildLimiterStub{}, - FillPercent: 1, + MetaStorage: metaStub, + Limiter: &rebuildLimiterStub{}, + FillPercent: 1, }) require.NoError(t, err) require.Equal(t, uint64(1), rRes.ObjectsMoved) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index aae72b5ff..e26c485ba 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -77,9 +77,9 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { guard: &sync.Mutex{}, } rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ - MetaStorage: metaStub, - WorkerLimiter: &rebuildLimiterStub{}, - FillPercent: 60, + MetaStorage: metaStub, + Limiter: &rebuildLimiterStub{}, + FillPercent: 60, }) require.NoError(t, err) dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 @@ -129,9 +129,9 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { guard: &sync.Mutex{}, } rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ - MetaStorage: metaStub, - WorkerLimiter: &rebuildLimiterStub{}, - FillPercent: 90, // 64KB / 100KB = 64% + MetaStorage: metaStub, + Limiter: &rebuildLimiterStub{}, + FillPercent: 90, // 64KB / 100KB = 64% }) require.NoError(t, err) dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0 @@ -194,9 +194,9 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { guard: &sync.Mutex{}, } rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ - MetaStorage: metaStub, - WorkerLimiter: &rebuildLimiterStub{}, - FillPercent: 80, + MetaStorage: metaStub, + Limiter: &rebuildLimiterStub{}, + FillPercent: 80, }) require.NoError(t, err) require.Equal(t, uint64(49), rRes.FilesRemoved) @@ -267,9 +267,9 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { require.NoError(t, b.Init()) rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ - MetaStorage: metaStub, - WorkerLimiter: &rebuildLimiterStub{}, - FillPercent: 80, + MetaStorage: metaStub, + Limiter: &rebuildLimiterStub{}, + FillPercent: 80, }) require.NoError(t, err) require.Equal(t, uint64(49), rRes.FilesRemoved) @@ -340,7 +340,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { } var rPrm common.RebuildPrm rPrm.MetaStorage = metaStub - rPrm.WorkerLimiter = &rebuildLimiterStub{} + rPrm.Limiter = &rebuildLimiterStub{} rPrm.FillPercent = 1 rRes, err := b.Rebuild(context.Background(), rPrm) require.NoError(t, err) @@ -429,7 +429,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta } var rPrm common.RebuildPrm rPrm.MetaStorage = metaStub - rPrm.WorkerLimiter = &rebuildLimiterStub{} + rPrm.Limiter = &rebuildLimiterStub{} rPrm.FillPercent = 1 rRes, err := b.Rebuild(context.Background(), rPrm) require.NoError(t, err) @@ -464,5 +464,6 @@ func (s *storageIDUpdateStub) UpdateStorageID(ctx context.Context, addr oid.Addr type rebuildLimiterStub struct{} -func (s *rebuildLimiterStub) AcquireWorkSlot(context.Context) error { return nil } -func (s *rebuildLimiterStub) ReleaseWorkSlot() {} +func (s *rebuildLimiterStub) AcquireWorkSlot(context.Context) (common.ReleaseFunc, error) { + return func() {}, nil +} diff --git a/pkg/local_object_storage/blobstor/common/rebuild.go b/pkg/local_object_storage/blobstor/common/rebuild.go index 19e181ee7..4615190f7 100644 --- a/pkg/local_object_storage/blobstor/common/rebuild.go +++ b/pkg/local_object_storage/blobstor/common/rebuild.go @@ -12,16 +12,21 @@ type RebuildRes struct { } type RebuildPrm struct { - MetaStorage MetaStorage - WorkerLimiter ConcurrentWorkersLimiter - FillPercent int + MetaStorage MetaStorage + Limiter RebuildLimiter + FillPercent int } type MetaStorage interface { UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error } -type ConcurrentWorkersLimiter interface { - AcquireWorkSlot(ctx context.Context) error - ReleaseWorkSlot() +type ReleaseFunc func() + +type ConcurrencyLimiter interface { + AcquireWorkSlot(ctx context.Context) (ReleaseFunc, error) +} + +type RebuildLimiter interface { + ConcurrencyLimiter } diff --git a/pkg/local_object_storage/blobstor/rebuild.go b/pkg/local_object_storage/blobstor/rebuild.go index 2a6b94789..f28816555 100644 --- a/pkg/local_object_storage/blobstor/rebuild.go +++ b/pkg/local_object_storage/blobstor/rebuild.go @@ -13,19 +13,14 @@ type StorageIDUpdate interface { UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error } -type ConcurrentWorkersLimiter interface { - AcquireWorkSlot(ctx context.Context) error - ReleaseWorkSlot() -} - -func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate, limiter ConcurrentWorkersLimiter, fillPercent int) error { +func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate, concLimiter common.RebuildLimiter, fillPercent int) error { var summary common.RebuildRes var rErr error for _, storage := range b.storage { res, err := storage.Storage.Rebuild(ctx, common.RebuildPrm{ - MetaStorage: upd, - WorkerLimiter: limiter, - FillPercent: fillPercent, + MetaStorage: upd, + Limiter: concLimiter, + FillPercent: fillPercent, }) summary.FilesRemoved += res.FilesRemoved summary.ObjectsMoved += res.ObjectsMoved diff --git a/pkg/local_object_storage/engine/rebuild.go b/pkg/local_object_storage/engine/rebuild.go index 83c6a54ed..a29dd7ed9 100644 --- a/pkg/local_object_storage/engine/rebuild.go +++ b/pkg/local_object_storage/engine/rebuild.go @@ -4,6 +4,7 @@ import ( "context" "sync" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "go.opentelemetry.io/otel/attribute" @@ -41,7 +42,7 @@ func (e *StorageEngine) Rebuild(ctx context.Context, prm RebuildPrm) (RebuildRes } resGuard := &sync.Mutex{} - limiter := shard.NewRebuildLimiter(prm.ConcurrencyLimit) + concLimiter := &concurrencyLimiter{semaphore: make(chan struct{}, prm.ConcurrencyLimit)} eg, egCtx := errgroup.WithContext(ctx) for _, shardID := range prm.ShardIDs { @@ -61,7 +62,7 @@ func (e *StorageEngine) Rebuild(ctx context.Context, prm RebuildPrm) (RebuildRes } err := sh.ScheduleRebuild(egCtx, shard.RebuildPrm{ - ConcurrencyLimiter: limiter, + ConcurrencyLimiter: concLimiter, TargetFillPercent: prm.TargetFillPercent, }) @@ -88,3 +89,20 @@ func (e *StorageEngine) Rebuild(ctx context.Context, prm RebuildPrm) (RebuildRes } return res, nil } + +type concurrencyLimiter struct { + semaphore chan struct{} +} + +func (l *concurrencyLimiter) AcquireWorkSlot(ctx context.Context) (common.ReleaseFunc, error) { + select { + case l.semaphore <- struct{}{}: + return l.releaseWorkSlot, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (l *concurrencyLimiter) releaseWorkSlot() { + <-l.semaphore +} diff --git a/pkg/local_object_storage/shard/rebuild.go b/pkg/local_object_storage/shard/rebuild.go index 9fe1bbe8c..3aa94d5a3 100644 --- a/pkg/local_object_storage/shard/rebuild.go +++ b/pkg/local_object_storage/shard/rebuild.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -20,37 +21,9 @@ import ( var ErrRebuildInProgress = errors.New("shard rebuild in progress") -type RebuildWorkerLimiter interface { - AcquireWorkSlot(ctx context.Context) error - ReleaseWorkSlot() -} - -type rebuildLimiter struct { - semaphore chan struct{} -} - -func NewRebuildLimiter(workersCount uint32) RebuildWorkerLimiter { - return &rebuildLimiter{ - semaphore: make(chan struct{}, workersCount), - } -} - -func (l *rebuildLimiter) AcquireWorkSlot(ctx context.Context) error { - select { - case l.semaphore <- struct{}{}: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - -func (l *rebuildLimiter) ReleaseWorkSlot() { - <-l.semaphore -} - type rebuildTask struct { - limiter RebuildWorkerLimiter - fillPercent int + concurrencyLimiter common.RebuildLimiter + fillPercent int } type rebuilder struct { @@ -90,14 +63,14 @@ func (r *rebuilder) Start(ctx context.Context, bs *blobstor.BlobStor, mb *meta.D if !ok { continue } - runRebuild(ctx, bs, mb, log, t.fillPercent, t.limiter) + runRebuild(ctx, bs, mb, log, t.fillPercent, t.concurrencyLimiter) } } }() } func runRebuild(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *logger.Logger, - fillPercent int, limiter RebuildWorkerLimiter, + fillPercent int, concLimiter common.RebuildLimiter, ) { select { case <-ctx.Done(): @@ -107,21 +80,21 @@ func runRebuild(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *lo log.Info(ctx, logs.BlobstoreRebuildStarted) ctx = tagging.ContextWithIOTag(ctx, qos.IOTagBackground.String()) // TODO use shard limiter - if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, limiter, fillPercent); err != nil { + if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, concLimiter, fillPercent); err != nil { log.Warn(ctx, logs.FailedToRebuildBlobstore, zap.Error(err)) } else { log.Info(ctx, logs.BlobstoreRebuildCompletedSuccessfully) } } -func (r *rebuilder) ScheduleRebuild(ctx context.Context, limiter RebuildWorkerLimiter, fillPercent int, +func (r *rebuilder) ScheduleRebuild(ctx context.Context, limiter common.RebuildLimiter, fillPercent int, ) error { select { case <-ctx.Done(): return ctx.Err() case r.tasks <- rebuildTask{ - limiter: limiter, - fillPercent: fillPercent, + concurrencyLimiter: limiter, + fillPercent: fillPercent, }: return nil default: @@ -170,7 +143,7 @@ func (u *mbStorageIDUpdate) UpdateStorageID(ctx context.Context, addr oid.Addres } type RebuildPrm struct { - ConcurrencyLimiter RebuildWorkerLimiter + ConcurrencyLimiter common.ConcurrencyLimiter TargetFillPercent uint32 } From b9360be1dcd8b08ab20e2a73a150ceadeeeea966 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 7 Feb 2025 17:25:47 +0300 Subject: [PATCH 1748/1943] [#1636] blobovniczatree: Use RebuildLimiter Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/rebuild.go | 40 +++++++++++++++---- .../blobstor/blobovniczatree/rebuild_test.go | 8 ++++ .../blobstor/common/rebuild.go | 6 +++ pkg/local_object_storage/shard/rebuild.go | 28 ++++++++++++- 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index cbd45c3b4..7ef3317fd 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -50,7 +50,7 @@ func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (comm var res common.RebuildRes b.log.Debug(ctx, logs.BlobovniczaTreeCompletingPreviousRebuild) - completedPreviosMoves, err := b.completeIncompletedMove(ctx, prm.MetaStorage) + completedPreviosMoves, err := b.completeIncompletedMove(ctx, prm.MetaStorage, prm.Limiter) res.ObjectsMoved += completedPreviosMoves if err != nil { b.log.Warn(ctx, logs.BlobovniczaTreeCompletedPreviousRebuildFailed, zap.Error(err)) @@ -238,7 +238,7 @@ func (b *Blobovniczas) addRebuildTempFile(ctx context.Context, path string) (fun }, nil } -func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovnicza, blzPath string, meta common.MetaStorage, concLimiter common.RebuildLimiter) (uint64, error) { +func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovnicza, blzPath string, meta common.MetaStorage, limiter common.RebuildLimiter) (uint64, error) { var result atomic.Uint64 batch := make(map[oid.Address][]byte) @@ -253,7 +253,12 @@ func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovn }) for { - _, err := blz.Iterate(ctx, prm) + release, err := limiter.ReadRequest(ctx) + if err != nil { + return result.Load(), err + } + _, err = blz.Iterate(ctx, prm) + release() if err != nil && !errors.Is(err, errBatchFull) { return result.Load(), err } @@ -265,14 +270,19 @@ func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovn eg, egCtx := errgroup.WithContext(ctx) for addr, data := range batch { - release, err := concLimiter.AcquireWorkSlot(egCtx) + release, err := limiter.AcquireWorkSlot(egCtx) if err != nil { _ = eg.Wait() return result.Load(), err } eg.Go(func() error { defer release() - err := b.moveObject(egCtx, blz, blzPath, addr, data, meta) + moveRelease, err := limiter.WriteRequest(ctx) + if err != nil { + return err + } + err = b.moveObject(egCtx, blz, blzPath, addr, data, meta) + moveRelease() if err == nil { result.Add(1) } @@ -360,7 +370,7 @@ func (b *Blobovniczas) dropDirectoryIfEmpty(path string) error { return b.dropDirectoryIfEmpty(filepath.Dir(path)) } -func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore common.MetaStorage) (uint64, error) { +func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore common.MetaStorage, rateLimiter common.RateLimiter) (uint64, error) { var count uint64 var rebuildTempFilesToRemove []string err := b.iterateIncompletedRebuildDBPaths(ctx, func(s string) (bool, error) { @@ -373,13 +383,24 @@ func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore co } defer shDB.Close(ctx) + release, err := rateLimiter.ReadRequest(ctx) + if err != nil { + return false, err + } incompletedMoves, err := blz.ListMoveInfo(ctx) + release() if err != nil { return true, err } for _, move := range incompletedMoves { - if err := b.performMove(ctx, blz, shDB.SystemPath(), move, metaStore); err != nil { + release, err := rateLimiter.WriteRequest(ctx) + if err != nil { + return false, err + } + err = b.performMove(ctx, blz, shDB.SystemPath(), move, metaStore) + release() + if err != nil { return true, err } count++ @@ -389,9 +410,14 @@ func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore co return false, nil }) for _, tmp := range rebuildTempFilesToRemove { + release, err := rateLimiter.WriteRequest(ctx) + if err != nil { + return count, err + } if err := os.Remove(filepath.Join(b.rootPath, tmp)); err != nil { b.log.Warn(ctx, logs.BlobovniczatreeFailedToRemoveRebuildTempFile, zap.Error(err)) } + release() } return count, err } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index e26c485ba..865d04a8f 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -467,3 +467,11 @@ type rebuildLimiterStub struct{} func (s *rebuildLimiterStub) AcquireWorkSlot(context.Context) (common.ReleaseFunc, error) { return func() {}, nil } + +func (s *rebuildLimiterStub) ReadRequest(context.Context) (common.ReleaseFunc, error) { + return func() {}, nil +} + +func (s *rebuildLimiterStub) WriteRequest(context.Context) (common.ReleaseFunc, error) { + return func() {}, nil +} diff --git a/pkg/local_object_storage/blobstor/common/rebuild.go b/pkg/local_object_storage/blobstor/common/rebuild.go index 4615190f7..788fe66f2 100644 --- a/pkg/local_object_storage/blobstor/common/rebuild.go +++ b/pkg/local_object_storage/blobstor/common/rebuild.go @@ -27,6 +27,12 @@ type ConcurrencyLimiter interface { AcquireWorkSlot(ctx context.Context) (ReleaseFunc, error) } +type RateLimiter interface { + ReadRequest(context.Context) (ReleaseFunc, error) + WriteRequest(context.Context) (ReleaseFunc, error) +} + type RebuildLimiter interface { ConcurrencyLimiter + RateLimiter } diff --git a/pkg/local_object_storage/shard/rebuild.go b/pkg/local_object_storage/shard/rebuild.go index 3aa94d5a3..20f1f2b6f 100644 --- a/pkg/local_object_storage/shard/rebuild.go +++ b/pkg/local_object_storage/shard/rebuild.go @@ -79,7 +79,6 @@ func runRebuild(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *lo } log.Info(ctx, logs.BlobstoreRebuildStarted) ctx = tagging.ContextWithIOTag(ctx, qos.IOTagBackground.String()) - // TODO use shard limiter if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, concLimiter, fillPercent); err != nil { log.Warn(ctx, logs.FailedToRebuildBlobstore, zap.Error(err)) } else { @@ -165,5 +164,30 @@ func (s *Shard) ScheduleRebuild(ctx context.Context, p RebuildPrm) error { return ErrDegradedMode } - return s.rb.ScheduleRebuild(ctx, p.ConcurrencyLimiter, int(p.TargetFillPercent)) + limiter := &rebuildLimiter{ + concurrencyLimiter: p.ConcurrencyLimiter, + rateLimiter: s.opsLimiter, + } + return s.rb.ScheduleRebuild(ctx, limiter, int(p.TargetFillPercent)) +} + +var _ common.RebuildLimiter = (*rebuildLimiter)(nil) + +type rebuildLimiter struct { + concurrencyLimiter common.ConcurrencyLimiter + rateLimiter qos.Limiter +} + +func (r *rebuildLimiter) AcquireWorkSlot(ctx context.Context) (common.ReleaseFunc, error) { + return r.concurrencyLimiter.AcquireWorkSlot(ctx) +} + +func (r *rebuildLimiter) ReadRequest(ctx context.Context) (common.ReleaseFunc, error) { + release, err := r.rateLimiter.ReadRequest(ctx) + return common.ReleaseFunc(release), err +} + +func (r *rebuildLimiter) WriteRequest(ctx context.Context) (common.ReleaseFunc, error) { + release, err := r.rateLimiter.WriteRequest(ctx) + return common.ReleaseFunc(release), err } From 2162f8e189a318550f55825755862a4e2a6a3e7b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 10 Feb 2025 17:55:34 +0300 Subject: [PATCH 1749/1943] [#1636] object: Fix IO tag adjustment for Put/Patch There was no tag adjustment for CloseAndRecv. Signed-off-by: Dmitrii Stepanov --- pkg/services/object/qos.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/services/object/qos.go b/pkg/services/object/qos.go index 145a316e2..01eb1ea8d 100644 --- a/pkg/services/object/qos.go +++ b/pkg/services/object/qos.go @@ -3,6 +3,8 @@ package object import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" ) @@ -120,13 +122,24 @@ type qosSendRecv[TReq qosVerificationHeader, TResp any] interface { type qosWriteStream[TReq qosVerificationHeader, TResp any] struct { s qosSendRecv[TReq, TResp] adj AdjustIOTag + + ioTag string + ioTagDefined bool } func (q *qosWriteStream[TReq, TResp]) CloseAndRecv(ctx context.Context) (TResp, error) { + if q.ioTagDefined { + ctx = tagging.ContextWithIOTag(ctx, q.ioTag) + } return q.s.CloseAndRecv(ctx) } func (q *qosWriteStream[TReq, TResp]) Send(ctx context.Context, req TReq) error { - ctx = q.adj.AdjustIncomingTag(ctx, req.GetVerificationHeader().GetBodySignature().GetKey()) + if !q.ioTagDefined { + ctx = q.adj.AdjustIncomingTag(ctx, req.GetVerificationHeader().GetBodySignature().GetKey()) + q.ioTag, q.ioTagDefined = tagging.IOTagFromContext(ctx) + } + assert.True(q.ioTagDefined, "io tag undefined after incoming tag adjustment") + ctx = tagging.ContextWithIOTag(ctx, q.ioTag) return q.s.Send(ctx, req) } From c2d855aedd11258c37126c5920e7e0ffde42425d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 12 Feb 2025 16:45:39 +0300 Subject: [PATCH 1750/1943] [#1636] qos: Return Resource Exhausted error Signed-off-by: Dmitrii Stepanov --- internal/qos/limiter.go | 8 ++++++++ pkg/local_object_storage/engine/engine.go | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index 996cebea1..3b6c6547c 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -2,6 +2,7 @@ package qos import ( "context" + "errors" "fmt" "time" @@ -9,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-qos/scheduling" "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) const ( @@ -110,6 +112,9 @@ func (n *mClockLimiter) ReadRequest(ctx context.Context) (ReleaseFunc, error) { } rel, err := n.readScheduler.RequestArrival(ctx, tag) if err != nil { + if errors.Is(err, scheduling.ErrMClockSchedulerRequestLimitExceeded) { + return nil, &apistatus.ResourceExhausted{} + } return nil, err } return ReleaseFunc(rel), nil @@ -125,6 +130,9 @@ func (n *mClockLimiter) WriteRequest(ctx context.Context) (ReleaseFunc, error) { } rel, err := n.writeScheduler.RequestArrival(ctx, tag) if err != nil { + if errors.Is(err, scheduling.ErrMClockSchedulerRequestLimitExceeded) { + return nil, &apistatus.ResourceExhausted{} + } return nil, err } return ReleaseFunc(rel), nil diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 85652b3ae..e13252b82 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" ) @@ -176,7 +177,10 @@ func (e *StorageEngine) reportShardError( } func isLogical(err error) bool { - return errors.As(err, &logicerr.Logical{}) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) + return errors.As(err, &logicerr.Logical{}) || + errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) || + errors.As(err, new(*apistatus.ResourceExhausted)) } // Option represents StorageEngine's constructor option. From 8ed71a969e256ee2410a1f089c8e41103ac63143 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Feb 2025 10:05:06 +0300 Subject: [PATCH 1751/1943] [#1636] qos: Add semaphore limiter If no tags specified, then limiter could be optimized to use atomic semaphore. Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- internal/qos/limiter.go | 50 ++++++++++++++++++++------------------- internal/qos/semaphore.go | 39 ++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 27 deletions(-) create mode 100644 internal/qos/semaphore.go diff --git a/go.mod b/go.mod index 2bfc3abfe..bbb817957 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 - git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250213125059-356851eed3bf + git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 diff --git a/go.sum b/go.sum index 4a7dfd4dc..0c66f4555 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 h1:Mxw1c/8t96vFIUOffl28lFaHKi413oCBfLMGJmF9cFA= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250213125059-356851eed3bf h1:ik2aMBpTJJpoZe2ffcGShXRkrvny65NEPLVt67KmH/A= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250213125059-356851eed3bf/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3 h1:QnAt5b2R6+hQthMOIn5ECfLAlVD8IAE5JRm1NCCOmuE= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4 h1:dOZHuOywvH1ms8U38lDCWpysgkCCeJ02RLI7zDhPcyw= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index 3b6c6547c..b1406a7f3 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -7,7 +7,6 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-qos/scheduling" "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -26,6 +25,11 @@ type Limiter interface { Close() } +type scheduler interface { + RequestArrival(ctx context.Context, tag string) (scheduling.ReleaseFunc, error) + Close() +} + func NewLimiter(c *limits.Config) (Limiter, error) { if err := validateConfig(c); err != nil { return nil, err @@ -34,15 +38,11 @@ func NewLimiter(c *limits.Config) (Limiter, error) { if isNoop(read, write) { return noopLimiterInstance, nil } - readScheduler, err := scheduling.NewMClock( - uint64(read.MaxRunningOps), uint64(read.MaxWaitingOps), - converToSchedulingTags(read.Tags), read.IdleTimeout) + readScheduler, err := createScheduler(c.Read()) if err != nil { return nil, fmt.Errorf("create read scheduler: %w", err) } - writeScheduler, err := scheduling.NewMClock( - uint64(write.MaxRunningOps), uint64(write.MaxWaitingOps), - converToSchedulingTags(write.Tags), write.IdleTimeout) + writeScheduler, err := createScheduler(c.Write()) if err != nil { return nil, fmt.Errorf("create write scheduler: %w", err) } @@ -52,6 +52,15 @@ func NewLimiter(c *limits.Config) (Limiter, error) { }, nil } +func createScheduler(config limits.OpConfig) (scheduler, error) { + if len(config.Tags) == 0 && config.MaxWaitingOps == limits.NoLimit { + return newSemaphoreScheduler(config.MaxRunningOps), nil + } + return scheduling.NewMClock( + uint64(config.MaxRunningOps), uint64(config.MaxWaitingOps), + converToSchedulingTags(config.Tags), config.IdleTimeout) +} + func converToSchedulingTags(limits []limits.IOTagConfig) map[string]scheduling.TagInfo { result := make(map[string]scheduling.TagInfo) for _, tag := range []IOTag{IOTagClient, IOTagBackground, IOTagInternal, IOTagPolicer, IOTagWritecache} { @@ -100,27 +109,19 @@ func (n *noopLimiter) Close() {} var _ Limiter = (*mClockLimiter)(nil) type mClockLimiter struct { - readScheduler *scheduling.MClock - writeScheduler *scheduling.MClock + readScheduler scheduler + writeScheduler scheduler } func (n *mClockLimiter) ReadRequest(ctx context.Context) (ReleaseFunc, error) { - tag, ok := tagging.IOTagFromContext(ctx) - assert.True(ok, "request has no tag") - if tag == IOTagCritical.String() { - return releaseStub, nil - } - rel, err := n.readScheduler.RequestArrival(ctx, tag) - if err != nil { - if errors.Is(err, scheduling.ErrMClockSchedulerRequestLimitExceeded) { - return nil, &apistatus.ResourceExhausted{} - } - return nil, err - } - return ReleaseFunc(rel), nil + return requestArrival(ctx, n.readScheduler) } func (n *mClockLimiter) WriteRequest(ctx context.Context) (ReleaseFunc, error) { + return requestArrival(ctx, n.writeScheduler) +} + +func requestArrival(ctx context.Context, s scheduler) (ReleaseFunc, error) { tag, ok := tagging.IOTagFromContext(ctx) if !ok { tag = IOTagClient.String() @@ -128,9 +129,10 @@ func (n *mClockLimiter) WriteRequest(ctx context.Context) (ReleaseFunc, error) { if tag == IOTagCritical.String() { return releaseStub, nil } - rel, err := n.writeScheduler.RequestArrival(ctx, tag) + rel, err := s.RequestArrival(ctx, tag) if err != nil { - if errors.Is(err, scheduling.ErrMClockSchedulerRequestLimitExceeded) { + if errors.Is(err, scheduling.ErrMClockSchedulerRequestLimitExceeded) || + errors.Is(err, errSemaphoreLimitExceeded) { return nil, &apistatus.ResourceExhausted{} } return nil, err diff --git a/internal/qos/semaphore.go b/internal/qos/semaphore.go new file mode 100644 index 000000000..74e6928f3 --- /dev/null +++ b/internal/qos/semaphore.go @@ -0,0 +1,39 @@ +package qos + +import ( + "context" + "errors" + + qosSemaphore "git.frostfs.info/TrueCloudLab/frostfs-qos/limiting/semaphore" + "git.frostfs.info/TrueCloudLab/frostfs-qos/scheduling" +) + +var ( + _ scheduler = (*semaphore)(nil) + errSemaphoreLimitExceeded = errors.New("semaphore limit exceeded") +) + +type semaphore struct { + s *qosSemaphore.Semaphore +} + +func newSemaphoreScheduler(size int64) *semaphore { + return &semaphore{ + s: qosSemaphore.NewSemaphore(size), + } +} + +func (s *semaphore) Close() {} + +func (s *semaphore) RequestArrival(ctx context.Context, _ string) (scheduling.ReleaseFunc, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + if s.s.Acquire() { + return s.s.Release, nil + } + return nil, errSemaphoreLimitExceeded +} From 401d96a89e179026b210729d261399b7d5874905 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 21 Feb 2025 16:37:00 +0300 Subject: [PATCH 1752/1943] [#1636] config: Refactor newConfig and oldConfig `newConfig` is actually target config to set config values from source (which is called `oldConfig`). Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 80 +++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index b42275538..afde0bbc0 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -256,42 +256,42 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { return engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error { return a.updateShardConfig(c, sc) }) } -func (a *applicationConfiguration) updateShardConfig(c *config.Config, oldConfig *shardconfig.Config) error { - var newConfig shardCfg +func (a *applicationConfiguration) updateShardConfig(c *config.Config, source *shardconfig.Config) error { + var target shardCfg - newConfig.refillMetabase = oldConfig.RefillMetabase() - newConfig.refillMetabaseWorkersCount = oldConfig.RefillMetabaseWorkersCount() - newConfig.mode = oldConfig.Mode() - newConfig.compress = oldConfig.Compress() - newConfig.estimateCompressibility = oldConfig.EstimateCompressibility() - newConfig.estimateCompressibilityThreshold = oldConfig.EstimateCompressibilityThreshold() - newConfig.uncompressableContentType = oldConfig.UncompressableContentTypes() - newConfig.smallSizeObjectLimit = oldConfig.SmallSizeLimit() + target.refillMetabase = source.RefillMetabase() + target.refillMetabaseWorkersCount = source.RefillMetabaseWorkersCount() + target.mode = source.Mode() + target.compress = source.Compress() + target.estimateCompressibility = source.EstimateCompressibility() + target.estimateCompressibilityThreshold = source.EstimateCompressibilityThreshold() + target.uncompressableContentType = source.UncompressableContentTypes() + target.smallSizeObjectLimit = source.SmallSizeLimit() - a.setShardWriteCacheConfig(&newConfig, oldConfig) + a.setShardWriteCacheConfig(&target, source) - a.setShardPiloramaConfig(c, &newConfig, oldConfig) + a.setShardPiloramaConfig(c, &target, source) - if err := a.setShardStorageConfig(&newConfig, oldConfig); err != nil { + if err := a.setShardStorageConfig(&target, source); err != nil { return err } - a.setMetabaseConfig(&newConfig, oldConfig) + a.setMetabaseConfig(&target, source) - a.setGCConfig(&newConfig, oldConfig) - if err := a.setLimiter(&newConfig, oldConfig); err != nil { + a.setGCConfig(&target, source) + if err := a.setLimiter(&target, source); err != nil { return err } - a.EngineCfg.shards = append(a.EngineCfg.shards, newConfig) + a.EngineCfg.shards = append(a.EngineCfg.shards, target) return nil } -func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, oldConfig *shardconfig.Config) { - writeCacheCfg := oldConfig.WriteCache() +func (a *applicationConfiguration) setShardWriteCacheConfig(target *shardCfg, source *shardconfig.Config) { + writeCacheCfg := source.WriteCache() if writeCacheCfg.Enabled() { - wc := &newConfig.writecacheCfg + wc := &target.writecacheCfg wc.enabled = true wc.path = writeCacheCfg.Path() @@ -304,10 +304,10 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, } } -func (a *applicationConfiguration) setShardPiloramaConfig(c *config.Config, newConfig *shardCfg, oldConfig *shardconfig.Config) { +func (a *applicationConfiguration) setShardPiloramaConfig(c *config.Config, target *shardCfg, source *shardconfig.Config) { if config.BoolSafe(c.Sub("tree"), "enabled") { - piloramaCfg := oldConfig.Pilorama() - pr := &newConfig.piloramaCfg + piloramaCfg := source.Pilorama() + pr := &target.piloramaCfg pr.enabled = true pr.path = piloramaCfg.Path() @@ -318,8 +318,8 @@ func (a *applicationConfiguration) setShardPiloramaConfig(c *config.Config, newC } } -func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, oldConfig *shardconfig.Config) error { - blobStorCfg := oldConfig.BlobStor() +func (a *applicationConfiguration) setShardStorageConfig(target *shardCfg, source *shardconfig.Config) error { + blobStorCfg := source.BlobStor() storagesCfg := blobStorCfg.Storages() ss := make([]subStorageCfg, 0, len(storagesCfg)) @@ -353,13 +353,13 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol ss = append(ss, sCfg) } - newConfig.subStorages = ss + target.subStorages = ss return nil } -func (a *applicationConfiguration) setMetabaseConfig(newConfig *shardCfg, oldConfig *shardconfig.Config) { - metabaseCfg := oldConfig.Metabase() - m := &newConfig.metaCfg +func (a *applicationConfiguration) setMetabaseConfig(target *shardCfg, source *shardconfig.Config) { + metabaseCfg := source.Metabase() + m := &target.metaCfg m.path = metabaseCfg.Path() m.perm = metabaseCfg.BoltDB().Perm() @@ -367,24 +367,24 @@ func (a *applicationConfiguration) setMetabaseConfig(newConfig *shardCfg, oldCon m.maxBatchSize = metabaseCfg.BoltDB().MaxBatchSize() } -func (a *applicationConfiguration) setGCConfig(newConfig *shardCfg, oldConfig *shardconfig.Config) { - gcCfg := oldConfig.GC() - newConfig.gcCfg.removerBatchSize = gcCfg.RemoverBatchSize() - newConfig.gcCfg.removerSleepInterval = gcCfg.RemoverSleepInterval() - newConfig.gcCfg.expiredCollectorBatchSize = gcCfg.ExpiredCollectorBatchSize() - newConfig.gcCfg.expiredCollectorWorkerCount = gcCfg.ExpiredCollectorWorkerCount() +func (a *applicationConfiguration) setGCConfig(target *shardCfg, source *shardconfig.Config) { + gcCfg := source.GC() + target.gcCfg.removerBatchSize = gcCfg.RemoverBatchSize() + target.gcCfg.removerSleepInterval = gcCfg.RemoverSleepInterval() + target.gcCfg.expiredCollectorBatchSize = gcCfg.ExpiredCollectorBatchSize() + target.gcCfg.expiredCollectorWorkerCount = gcCfg.ExpiredCollectorWorkerCount() } -func (a *applicationConfiguration) setLimiter(newConfig *shardCfg, oldConfig *shardconfig.Config) error { - limitsConfig := oldConfig.Limits() +func (a *applicationConfiguration) setLimiter(target *shardCfg, source *shardconfig.Config) error { + limitsConfig := source.Limits() limiter, err := qos.NewLimiter(limitsConfig) if err != nil { return err } - if newConfig.limiter != nil { - newConfig.limiter.Close() + if target.limiter != nil { + target.limiter.Close() } - newConfig.limiter = limiter + target.limiter = limiter return nil } From 6c6e463b73d0d7afbdd7f7142d0c9f20a59f871c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Feb 2025 16:06:07 +0300 Subject: [PATCH 1753/1943] [#1636] shard: Change ops limiter on shard reload Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/control.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 3520277c0..6dee2da3f 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -449,6 +449,10 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error { return err } } + if c.opsLimiter != nil { + s.opsLimiter.Close() + s.opsLimiter = c.opsLimiter + } return s.setMode(ctx, c.info.Mode) } From eb8b9b2b3bd933e4fa7bfc7ffc8ea80beb49d522 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Feb 2025 10:04:24 +0300 Subject: [PATCH 1754/1943] [#1636] blobovniczatree: Validate limiter release in rebuild unit tests Signed-off-by: Dmitrii Stepanov --- .../blobovniczatree/rebuild_failover_test.go | 4 +- .../blobstor/blobovniczatree/rebuild_test.go | 54 +++++++++++++++---- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index 91578d5e8..8832603c4 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -161,9 +161,10 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object storageIDs: make(map[oid.Address][]byte), guard: &sync.Mutex{}, } + limiter := &rebuildLimiterStub{} rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ MetaStorage: metaStub, - Limiter: &rebuildLimiterStub{}, + Limiter: limiter, FillPercent: 1, }) require.NoError(t, err) @@ -171,6 +172,7 @@ func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object require.Equal(t, uint64(0), rRes.FilesRemoved) require.NoError(t, b.Close(context.Background())) + require.NoError(t, limiter.ValidateReleased()) blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db"))) require.NoError(t, blz.Open(context.Background())) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 865d04a8f..9c971bfb6 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -2,7 +2,9 @@ package blobovniczatree import ( "context" + "fmt" "sync" + "sync/atomic" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" @@ -76,9 +78,10 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { storageIDs: storageIDs, guard: &sync.Mutex{}, } + limiter := &rebuildLimiterStub{} rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ MetaStorage: metaStub, - Limiter: &rebuildLimiterStub{}, + Limiter: limiter, FillPercent: 60, }) require.NoError(t, err) @@ -94,6 +97,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { } require.NoError(t, b.Close(context.Background())) + require.NoError(t, limiter.ValidateReleased()) }) t.Run("no rebuild single db", func(t *testing.T) { @@ -128,9 +132,10 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { storageIDs: storageIDs, guard: &sync.Mutex{}, } + limiter := &rebuildLimiterStub{} rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ MetaStorage: metaStub, - Limiter: &rebuildLimiterStub{}, + Limiter: limiter, FillPercent: 90, // 64KB / 100KB = 64% }) require.NoError(t, err) @@ -146,6 +151,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { } require.NoError(t, b.Close(context.Background())) + require.NoError(t, limiter.ValidateReleased()) }) t.Run("rebuild by fill percent", func(t *testing.T) { @@ -193,9 +199,10 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { storageIDs: storageIDs, guard: &sync.Mutex{}, } + limiter := &rebuildLimiterStub{} rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ MetaStorage: metaStub, - Limiter: &rebuildLimiterStub{}, + Limiter: limiter, FillPercent: 80, }) require.NoError(t, err) @@ -215,6 +222,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { } require.NoError(t, b.Close(context.Background())) + require.NoError(t, limiter.ValidateReleased()) }) t.Run("rebuild by overflow", func(t *testing.T) { @@ -266,9 +274,10 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { require.NoError(t, b.Open(mode.ComponentReadWrite)) require.NoError(t, b.Init()) + limiter := &rebuildLimiterStub{} rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{ MetaStorage: metaStub, - Limiter: &rebuildLimiterStub{}, + Limiter: limiter, FillPercent: 80, }) require.NoError(t, err) @@ -285,6 +294,7 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { } require.NoError(t, b.Close(context.Background())) + require.NoError(t, limiter.ValidateReleased()) }) } @@ -338,9 +348,10 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { storageIDs: storageIDs, guard: &sync.Mutex{}, } + limiter := &rebuildLimiterStub{} var rPrm common.RebuildPrm rPrm.MetaStorage = metaStub - rPrm.Limiter = &rebuildLimiterStub{} + rPrm.Limiter = limiter rPrm.FillPercent = 1 rRes, err := b.Rebuild(context.Background(), rPrm) require.NoError(t, err) @@ -356,6 +367,7 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { } require.NoError(t, b.Close(context.Background())) + require.NoError(t, limiter.ValidateReleased()) } func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, targetDepth, targetWidth uint64, shouldMigrate bool) { @@ -427,9 +439,10 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta storageIDs: storageIDs, guard: &sync.Mutex{}, } + limiter := &rebuildLimiterStub{} var rPrm common.RebuildPrm rPrm.MetaStorage = metaStub - rPrm.Limiter = &rebuildLimiterStub{} + rPrm.Limiter = limiter rPrm.FillPercent = 1 rRes, err := b.Rebuild(context.Background(), rPrm) require.NoError(t, err) @@ -445,6 +458,7 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta } require.NoError(t, b.Close(context.Background())) + require.NoError(t, limiter.ValidateReleased()) } type storageIDUpdateStub struct { @@ -462,16 +476,36 @@ func (s *storageIDUpdateStub) UpdateStorageID(ctx context.Context, addr oid.Addr return nil } -type rebuildLimiterStub struct{} +type rebuildLimiterStub struct { + slots atomic.Int64 + readRequests atomic.Int64 + writeRequests atomic.Int64 +} func (s *rebuildLimiterStub) AcquireWorkSlot(context.Context) (common.ReleaseFunc, error) { - return func() {}, nil + s.slots.Add(1) + return func() { s.slots.Add(-1) }, nil } func (s *rebuildLimiterStub) ReadRequest(context.Context) (common.ReleaseFunc, error) { - return func() {}, nil + s.readRequests.Add(1) + return func() { s.readRequests.Add(-1) }, nil } func (s *rebuildLimiterStub) WriteRequest(context.Context) (common.ReleaseFunc, error) { - return func() {}, nil + s.writeRequests.Add(1) + return func() { s.writeRequests.Add(-1) }, nil +} + +func (s *rebuildLimiterStub) ValidateReleased() error { + if v := s.slots.Load(); v != 0 { + return fmt.Errorf("invalid slots value %d", v) + } + if v := s.readRequests.Load(); v != 0 { + return fmt.Errorf("invalid read requests value %d", v) + } + if v := s.writeRequests.Load(); v != 0 { + return fmt.Errorf("invalid write requests value %d", v) + } + return nil } From 4685afb1dc5737a70833231031582fda29476835 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Feb 2025 10:21:30 +0300 Subject: [PATCH 1755/1943] [#1636] engine: Validate limiter release in unit tests Signed-off-by: Dmitrii Stepanov --- .../engine/engine_test.go | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 926ff43f3..7ddde1f02 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -3,8 +3,10 @@ package engine import ( "context" "path/filepath" + "sync/atomic" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" @@ -90,6 +92,7 @@ func testGetDefaultShardOptions(t testing.TB) []shard.Option { ), shard.WithPiloramaOptions(pilorama.WithPath(filepath.Join(t.TempDir(), "pilorama"))), shard.WithMetaBaseOptions(testGetDefaultMetabaseOptions(t)...), + shard.WithLimiter(&testQoSLimiter{t: t}), } } @@ -151,3 +154,26 @@ func newTestStorages(root string, smallSize uint64) ([]blobstor.SubStorage, *tes }, }, smallFileStorage, largeFileStorage } + +var _ qos.Limiter = (*testQoSLimiter)(nil) + +type testQoSLimiter struct { + t testing.TB + read atomic.Int64 + write atomic.Int64 +} + +func (t *testQoSLimiter) Close() { + require.Equal(t.t, int64(0), t.read.Load(), "read requests count after limiter close must be 0") + require.Equal(t.t, int64(0), t.write.Load(), "write requests count after limiter close must be 0") +} + +func (t *testQoSLimiter) ReadRequest(context.Context) (qos.ReleaseFunc, error) { + t.read.Add(1) + return func() { t.read.Add(-1) }, nil +} + +func (t *testQoSLimiter) WriteRequest(context.Context) (qos.ReleaseFunc, error) { + t.write.Add(1) + return func() { t.write.Add(-1) }, nil +} From 5ba0e2918e60146fe64d9e5c159629f279fe079d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 4 Mar 2025 10:16:20 +0300 Subject: [PATCH 1756/1943] [#1661] cli: Clarify `--rpc-endpoint` values `multiaddr` is not something an average user knows. Personally, I have never used it in CLI. On the other hand, we need to connect with TLS quite often, so it needs to be mentioned in help. Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/internal/commonflags/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/internal/commonflags/flags.go b/cmd/frostfs-cli/internal/commonflags/flags.go index cd46d63eb..fad1f6183 100644 --- a/cmd/frostfs-cli/internal/commonflags/flags.go +++ b/cmd/frostfs-cli/internal/commonflags/flags.go @@ -28,7 +28,7 @@ const ( RPC = "rpc-endpoint" RPCShorthand = "r" RPCDefault = "" - RPCUsage = "Remote node address (as 'multiaddr' or ':')" + RPCUsage = "Remote node address (':' or 'grpcs://:')" Timeout = "timeout" TimeoutShorthand = "t" From bf8914fedc401fc7a8d17e7a8dc23f6776102a66 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 11 Nov 2024 12:33:31 +0300 Subject: [PATCH 1757/1943] [#1416] lens/explorer: Support metabase schema v3 Signed-off-by: Aleksey Savchuk --- cmd/frostfs-lens/internal/meta/tui.go | 49 ++++++++++++++++++- .../schema/metabase/buckets/parsers.go | 17 ++++++- .../schema/metabase/buckets/prefix.go | 40 ++++++++------- .../schema/metabase/buckets/string.go | 6 +-- .../internal/schema/metabase/buckets/types.go | 19 +++++-- .../internal/schema/metabase/parser.go | 27 +++++++++- .../schema/metabase/records/detailed.go | 8 +++ .../schema/metabase/records/filter.go | 23 +++++++++ .../schema/metabase/records/parsers.go | 42 ++++++++++++++++ .../schema/metabase/records/string.go | 20 ++++++++ .../internal/schema/metabase/records/types.go | 11 +++++ 11 files changed, 233 insertions(+), 29 deletions(-) diff --git a/cmd/frostfs-lens/internal/meta/tui.go b/cmd/frostfs-lens/internal/meta/tui.go index 5a41f945c..7b0e25f3d 100644 --- a/cmd/frostfs-lens/internal/meta/tui.go +++ b/cmd/frostfs-lens/internal/meta/tui.go @@ -2,13 +2,17 @@ package meta import ( "context" + "encoding/binary" + "errors" "fmt" common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal" + schemaCommon "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" schema "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/tui" "github.com/rivo/tview" "github.com/spf13/cobra" + "go.etcd.io/bbolt" ) var tuiCMD = &cobra.Command{ @@ -27,6 +31,11 @@ Available search filters: var initialPrompt string +var parserPerSchemaVersion = map[uint64]schemaCommon.Parser{ + 2: schema.MetabaseParserV2, + 3: schema.MetabaseParserV3, +} + func init() { common.AddComponentPathFlag(tuiCMD, &vPath) @@ -49,12 +58,22 @@ func runTUI(cmd *cobra.Command) error { } defer db.Close() + schemaVersion, hasVersion := lookupSchemaVersion(cmd, db) + if !hasVersion { + return errors.New("couldn't detect schema version") + } + + metabaseParser, ok := parserPerSchemaVersion[schemaVersion] + if !ok { + return fmt.Errorf("unknown schema version %d", schemaVersion) + } + // Need if app was stopped with Ctrl-C. ctx, cancel := context.WithCancel(cmd.Context()) defer cancel() app := tview.NewApplication() - ui := tui.NewUI(ctx, app, db, schema.MetabaseParser, nil) + ui := tui.NewUI(ctx, app, db, metabaseParser, nil) _ = ui.AddFilter("cid", tui.CIDParser, "CID") _ = ui.AddFilter("oid", tui.OIDParser, "OID") @@ -69,3 +88,31 @@ func runTUI(cmd *cobra.Command) error { app.SetRoot(ui, true).SetFocus(ui) return app.Run() } + +var ( + shardInfoBucket = []byte{5} + versionRecord = []byte("version") +) + +func lookupSchemaVersion(cmd *cobra.Command, db *bbolt.DB) (version uint64, ok bool) { + err := db.View(func(tx *bbolt.Tx) error { + bkt := tx.Bucket(shardInfoBucket) + if bkt == nil { + return nil + } + rec := bkt.Get(versionRecord) + if rec == nil { + return nil + } + + version = binary.LittleEndian.Uint64(rec) + ok = true + + return nil + }) + if err != nil { + common.ExitOnErr(cmd, fmt.Errorf("couldn't lookup version: %w", err)) + } + + return +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go index 24cc0e52d..4e6bbf08a 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/parsers.go @@ -80,10 +80,15 @@ var ( }, ) - UserAttributeParser = NewUserAttributeKeyBucketParser( + UserAttributeParserV2 = NewUserAttributeKeyBucketParser( NewUserAttributeValueBucketParser(records.UserAttributeRecordParser), ) + UserAttributeParserV3 = NewUserAttributeKeyBucketParserWithSpecificKeys( + NewUserAttributeValueBucketParser(records.UserAttributeRecordParser), + []string{"FilePath", "S3-Access-Box-CRDT-Name"}, + ) + PayloadHashParser = NewPrefixContainerBucketParser(PayloadHash, records.PayloadHashRecordParser, Resolvers{ cidResolver: StrictResolver, oidResolver: StrictResolver, @@ -108,4 +113,14 @@ var ( cidResolver: StrictResolver, oidResolver: LenientResolver, }) + + ExpirationEpochToObjectParser = NewPrefixBucketParser(ExpirationEpochToObject, records.ExpirationEpochToObjectRecordParser, Resolvers{ + cidResolver: LenientResolver, + oidResolver: LenientResolver, + }) + + ObjectToExpirationEpochParser = NewPrefixContainerBucketParser(ObjectToExpirationEpoch, records.ObjectToExpirationEpochRecordParser, Resolvers{ + cidResolver: StrictResolver, + oidResolver: LenientResolver, + }) ) diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go index 2fb122940..42a24c594 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/prefix.go @@ -22,27 +22,31 @@ const ( Split ContainerCounters ECInfo + ExpirationEpochToObject + ObjectToExpirationEpoch ) var x = map[Prefix]string{ - Graveyard: "Graveyard", - Garbage: "Garbage", - ToMoveIt: "To Move It", - ContainerVolume: "Container Volume", - Locked: "Locked", - ShardInfo: "Shard Info", - Primary: "Primary", - Lockers: "Lockers", - Tombstone: "Tombstone", - Small: "Small", - Root: "Root", - Owner: "Owner", - UserAttribute: "User Attribute", - PayloadHash: "Payload Hash", - Parent: "Parent", - Split: "Split", - ContainerCounters: "Container Counters", - ECInfo: "EC Info", + Graveyard: "Graveyard", + Garbage: "Garbage", + ToMoveIt: "To Move It", + ContainerVolume: "Container Volume", + Locked: "Locked", + ShardInfo: "Shard Info", + Primary: "Primary", + Lockers: "Lockers", + Tombstone: "Tombstone", + Small: "Small", + Root: "Root", + Owner: "Owner", + UserAttribute: "User Attribute", + PayloadHash: "Payload Hash", + Parent: "Parent", + Split: "Split", + ContainerCounters: "Container Counters", + ECInfo: "EC Info", + ExpirationEpochToObject: "Exp. Epoch to Object", + ObjectToExpirationEpoch: "Object to Exp. Epoch", } func (p Prefix) String() string { diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/string.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/string.go index db90bddbd..62d126f88 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/buckets/string.go +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/string.go @@ -9,7 +9,7 @@ import ( func (b *PrefixBucket) String() string { return common.FormatSimple( - fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime, + fmt.Sprintf("(%2d %-20s)", b.prefix, b.prefix), tcell.ColorLime, ) } @@ -17,7 +17,7 @@ func (b *PrefixContainerBucket) String() string { return fmt.Sprintf( "%s CID %s", common.FormatSimple( - fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime, + fmt.Sprintf("(%2d %-20s)", b.prefix, b.prefix), tcell.ColorLime, ), common.FormatSimple(b.id.String(), tcell.ColorAqua), ) @@ -34,7 +34,7 @@ func (b *ContainerBucket) String() string { func (b *UserAttributeKeyBucket) String() string { return fmt.Sprintf("%s CID %s ATTR-KEY %s", common.FormatSimple( - fmt.Sprintf("(%2d %-18s)", b.prefix, b.prefix), tcell.ColorLime, + fmt.Sprintf("(%2d %-20s)", b.prefix, b.prefix), tcell.ColorLime, ), common.FormatSimple( fmt.Sprintf("%-44s", b.id), tcell.ColorAqua, diff --git a/cmd/frostfs-lens/internal/schema/metabase/buckets/types.go b/cmd/frostfs-lens/internal/schema/metabase/buckets/types.go index 82b47dd85..7355c3d9e 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/buckets/types.go +++ b/cmd/frostfs-lens/internal/schema/metabase/buckets/types.go @@ -2,6 +2,7 @@ package buckets import ( "errors" + "slices" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -57,10 +58,11 @@ var ( ) var ( - ErrNotBucket = errors.New("not a bucket") - ErrInvalidKeyLength = errors.New("invalid key length") - ErrInvalidValueLength = errors.New("invalid value length") - ErrInvalidPrefix = errors.New("invalid prefix") + ErrNotBucket = errors.New("not a bucket") + ErrInvalidKeyLength = errors.New("invalid key length") + ErrInvalidValueLength = errors.New("invalid value length") + ErrInvalidPrefix = errors.New("invalid prefix") + ErrUnexpectedAttributeKey = errors.New("unexpected attribute key") ) func NewPrefixBucketParser(prefix Prefix, next common.Parser, resolvers Resolvers) common.Parser { @@ -132,6 +134,10 @@ func NewContainerBucketParser(next common.Parser, resolvers Resolvers) common.Pa } func NewUserAttributeKeyBucketParser(next common.Parser) common.Parser { + return NewUserAttributeKeyBucketParserWithSpecificKeys(next, nil) +} + +func NewUserAttributeKeyBucketParserWithSpecificKeys(next common.Parser, keys []string) common.Parser { return func(key, value []byte) (common.SchemaEntry, common.Parser, error) { if value != nil { return nil, nil, ErrNotBucket @@ -147,6 +153,11 @@ func NewUserAttributeKeyBucketParser(next common.Parser) common.Parser { return nil, nil, err } b.key = string(key[33:]) + + if len(keys) != 0 && !slices.Contains(keys, b.key) { + return nil, nil, ErrUnexpectedAttributeKey + } + return &b, next, nil } } diff --git a/cmd/frostfs-lens/internal/schema/metabase/parser.go b/cmd/frostfs-lens/internal/schema/metabase/parser.go index ea095e207..4cc9e8765 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/parser.go +++ b/cmd/frostfs-lens/internal/schema/metabase/parser.go @@ -5,7 +5,30 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/metabase/buckets" ) -var MetabaseParser = common.WithFallback( +var MetabaseParserV3 = common.WithFallback( + common.Any( + buckets.GraveyardParser, + buckets.GarbageParser, + buckets.ContainerVolumeParser, + buckets.LockedParser, + buckets.ShardInfoParser, + buckets.PrimaryParser, + buckets.LockersParser, + buckets.TombstoneParser, + buckets.SmallParser, + buckets.RootParser, + buckets.UserAttributeParserV3, + buckets.ParentParser, + buckets.SplitParser, + buckets.ContainerCountersParser, + buckets.ECInfoParser, + buckets.ExpirationEpochToObjectParser, + buckets.ObjectToExpirationEpochParser, + ), + common.RawParser.ToFallbackParser(), +) + +var MetabaseParserV2 = common.WithFallback( common.Any( buckets.GraveyardParser, buckets.GarbageParser, @@ -18,7 +41,7 @@ var MetabaseParser = common.WithFallback( buckets.SmallParser, buckets.RootParser, buckets.OwnerParser, - buckets.UserAttributeParser, + buckets.UserAttributeParserV2, buckets.PayloadHashParser, buckets.ParentParser, buckets.SplitParser, diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/detailed.go b/cmd/frostfs-lens/internal/schema/metabase/records/detailed.go index 2dda15b4f..477c4fc9d 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/records/detailed.go +++ b/cmd/frostfs-lens/internal/schema/metabase/records/detailed.go @@ -63,3 +63,11 @@ func (r *ContainerCountersRecord) DetailedString() string { func (r *ECInfoRecord) DetailedString() string { return spew.Sdump(*r) } + +func (r *ExpirationEpochToObjectRecord) DetailedString() string { + return spew.Sdump(*r) +} + +func (r *ObjectToExpirationEpochRecord) DetailedString() string { + return spew.Sdump(*r) +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/filter.go b/cmd/frostfs-lens/internal/schema/metabase/records/filter.go index 880a7a8ff..e038911d7 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/records/filter.go +++ b/cmd/frostfs-lens/internal/schema/metabase/records/filter.go @@ -143,3 +143,26 @@ func (r *ECInfoRecord) Filter(typ string, val any) common.FilterResult { return common.No } } + +func (r *ExpirationEpochToObjectRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "cid": + id := val.(cid.ID) + return common.IfThenElse(r.cnt.Equals(id), common.Yes, common.No) + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.obj.Equals(id), common.Yes, common.No) + default: + return common.No + } +} + +func (r *ObjectToExpirationEpochRecord) Filter(typ string, val any) common.FilterResult { + switch typ { + case "oid": + id := val.(oid.ID) + return common.IfThenElse(r.obj.Equals(id), common.Yes, common.No) + default: + return common.No + } +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/parsers.go b/cmd/frostfs-lens/internal/schema/metabase/records/parsers.go index 1b070e2a0..5d846cb75 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/records/parsers.go +++ b/cmd/frostfs-lens/internal/schema/metabase/records/parsers.go @@ -249,3 +249,45 @@ func ECInfoRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, e } return &r, nil, nil } + +func ExpirationEpochToObjectRecordParser(key, _ []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 72 { + return nil, nil, ErrInvalidKeyLength + } + + var ( + r ExpirationEpochToObjectRecord + err error + ) + + r.epoch = binary.BigEndian.Uint64(key[:8]) + if err = r.cnt.Decode(key[8:40]); err != nil { + return nil, nil, err + } + if err = r.obj.Decode(key[40:]); err != nil { + return nil, nil, err + } + + return &r, nil, nil +} + +func ObjectToExpirationEpochRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, error) { + if len(key) != 32 { + return nil, nil, ErrInvalidKeyLength + } + if len(value) != 8 { + return nil, nil, ErrInvalidValueLength + } + + var ( + r ObjectToExpirationEpochRecord + err error + ) + + if err = r.obj.Decode(key); err != nil { + return nil, nil, err + } + r.epoch = binary.LittleEndian.Uint64(value) + + return &r, nil, nil +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/string.go b/cmd/frostfs-lens/internal/schema/metabase/records/string.go index ec0ab8e1a..f71244625 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/records/string.go +++ b/cmd/frostfs-lens/internal/schema/metabase/records/string.go @@ -2,6 +2,7 @@ package records import ( "fmt" + "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" "github.com/gdamore/tcell/v2" @@ -133,3 +134,22 @@ func (r *ECInfoRecord) String() string { len(r.ids), ) } + +func (r *ExpirationEpochToObjectRecord) String() string { + return fmt.Sprintf( + "exp. epoch %s %c CID %s OID %s", + common.FormatSimple(fmt.Sprintf("%-20d", r.epoch), tcell.ColorAqua), + tview.Borders.Vertical, + common.FormatSimple(fmt.Sprintf("%-44s", r.cnt), tcell.ColorAqua), + common.FormatSimple(fmt.Sprintf("%-44s", r.obj), tcell.ColorAqua), + ) +} + +func (r *ObjectToExpirationEpochRecord) String() string { + return fmt.Sprintf( + "OID %s %c exp. epoch %s", + common.FormatSimple(fmt.Sprintf("%-44s", r.obj), tcell.ColorAqua), + tview.Borders.Vertical, + common.FormatSimple(strconv.FormatUint(r.epoch, 10), tcell.ColorAqua), + ) +} diff --git a/cmd/frostfs-lens/internal/schema/metabase/records/types.go b/cmd/frostfs-lens/internal/schema/metabase/records/types.go index 34c1c29fd..0809cad1a 100644 --- a/cmd/frostfs-lens/internal/schema/metabase/records/types.go +++ b/cmd/frostfs-lens/internal/schema/metabase/records/types.go @@ -79,4 +79,15 @@ type ( id oid.ID ids []oid.ID } + + ExpirationEpochToObjectRecord struct { + epoch uint64 + cnt cid.ID + obj oid.ID + } + + ObjectToExpirationEpochRecord struct { + obj oid.ID + epoch uint64 + } ) From 4c8f9580a16eb5d30320d670849585fe25c1d04d Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 5 Mar 2025 17:09:02 +0300 Subject: [PATCH 1758/1943] [#1662] object: Fix `CloseAndRecv` for patch streamer * A client may open stream to server, not send anything and close the open stream immediatly. This shouldn't cause a panic; * Return the error if `s.patcher` is uninitialized. Uninitialized patcher cannot be closed, this causes a panic. Signed-off-by: Airat Arifullin --- pkg/services/object/patch/streamer.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/services/object/patch/streamer.go b/pkg/services/object/patch/streamer.go index 5aba13f66..642b9f9fa 100644 --- a/pkg/services/object/patch/streamer.go +++ b/pkg/services/object/patch/streamer.go @@ -214,6 +214,9 @@ func (s *Streamer) Send(ctx context.Context, req *objectV2.PatchRequest) error { } func (s *Streamer) CloseAndRecv(ctx context.Context) (*objectV2.PatchResponse, error) { + if s.patcher == nil { + return nil, errors.New("uninitialized patch streamer") + } patcherResp, err := s.patcher.Close(ctx) if err != nil { return nil, err From 471aeeaff3bdb8df231416bd69e659ca38e1e0c2 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 3 Mar 2025 04:21:53 +0300 Subject: [PATCH 1759/1943] [#1659] audit: Fix duplicated request logs When we do `object put` with audit enabled we get several entries in logs: with and without object id. `object put` request is logged in 2 places: 1. `(*auditPutStream) CloseAndRecv()` - when the client closes the request stream or when stream gets aborted. 2. `(*auditPutStream) Send()` - when stream was NOT aborted. `Send()` does error check for `ErrAbortStream` because if there is any other error - CloseAndRecv will not be called and there won't be any audit log about failed request. It led to logging on every object chunck put, even if `err == nil`. Added check for `err != nil` in `Send()` to fix it. Signed-off-by: Ekaterina Lebedeva --- pkg/services/object/audit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/audit.go b/pkg/services/object/audit.go index dde9f8fc0..367be0c0c 100644 --- a/pkg/services/object/audit.go +++ b/pkg/services/object/audit.go @@ -163,7 +163,7 @@ func (a *auditPutStream) Send(ctx context.Context, req *object.PutRequest) error if err != nil { a.failed = true } - if !errors.Is(err, util.ErrAbortStream) { // CloseAndRecv will not be called, so log here + if err != nil && !errors.Is(err, util.ErrAbortStream) { // CloseAndRecv will not be called, so log here audit.LogRequestWithKey(ctx, a.log, objectGRPC.ObjectService_Put_FullMethodName, a.key, audit.TargetFromContainerIDObjectID(a.containerID, a.objectID), !a.failed) From a17c3356fa762c3d558ed9d6fcd76f44db2f1e40 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 6 Mar 2025 15:07:37 +0300 Subject: [PATCH 1760/1943] [#1665] go.mod: Update sdk-go Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bbb817957..18378466e 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 diff --git a/go.sum b/go.sum index 0c66f4555..5205dddef 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e13 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3 h1:QnAt5b2R6+hQthMOIn5ECfLAlVD8IAE5JRm1NCCOmuE= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4 h1:dOZHuOywvH1ms8U38lDCWpysgkCCeJ02RLI7zDhPcyw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250217152255-c3f7378887a4/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9 h1:svCl6NDAPZ/KuQPjdVKo74RkCIANesxUPM45zQZDhSw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 6260d703ce02a12a812d5071b601a3c925eb8860 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 6 Mar 2025 15:13:38 +0300 Subject: [PATCH 1761/1943] [#1665] treesvc: Disable service config query By default, gRPC fetches TXT report while resolving a domain. https://github.com/grpc/grpc-go/blob/0914bba6c5c5a545d34bd11e5dee0bbb8eaadd3f/internal/resolver/dns/dns_resolver.go#L336 This leads to a hanging dial if DNS is unavailable, even though the host may be specified in `/etc/hosts` (hello, localhost!). SDK client for the main API uses these options by default. Refs TrueCloudLab/frostfs-sdk-go#342 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/tree/client.go | 1 + pkg/services/tree/cache.go | 1 + pkg/services/tree/sync.go | 1 + 3 files changed, 3 insertions(+) diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index c6953f126..933378df6 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -39,6 +39,7 @@ func _client() (tree.TreeServiceClient, error) { tracing.NewStreamClientInterceptor(), ), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + grpc.WithDisableServiceConfig(), } if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") { diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 125871fc4..70f4a843b 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -110,6 +110,7 @@ func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (* ), grpc.WithContextDialer(c.ds.GrpcContextDialer()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + grpc.WithDisableServiceConfig(), } if !netAddr.IsTLSEnabled() { diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 3e0a45385..89450b739 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -355,6 +355,7 @@ func (*Service) createConnection(a network.Address) (*grpc.ClientConn, error) { ), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + grpc.WithDisableServiceConfig(), ) } From 9e31cb249f10c06cd0908db1269f600f4bd51cda Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 4 Feb 2025 21:21:31 +0300 Subject: [PATCH 1762/1943] [#1635] control: Add method to search shards by object Added method `ListShardsForObject` to ControlService and to StorageEngine. It returns information about shards storing object on the node. Signed-off-by: Ekaterina Lebedeva --- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/shards.go | 46 ++ pkg/services/control/rpc.go | 20 + .../control/server/list_shards_for_object.go | 66 ++ pkg/services/control/service.proto | 23 + pkg/services/control/service_frostfs.pb.go | 724 ++++++++++++++++++ pkg/services/control/service_grpc.pb.go | 39 + 7 files changed, 919 insertions(+) create mode 100644 pkg/services/control/server/list_shards_for_object.go diff --git a/internal/logs/logs.go b/internal/logs/logs.go index d48a4da9b..d07f47fbf 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -252,6 +252,7 @@ const ( ShardFailureToMarkLockersAsGarbage = "failure to mark lockers as garbage" ShardFailureToGetExpiredUnlockedObjects = "failure to get expired unlocked objects" ShardCouldNotMarkObjectToDeleteInMetabase = "could not mark object to delete in metabase" + ShardCouldNotFindObject = "could not find object" WritecacheWaitingForChannelsToFlush = "waiting for channels to flush" WritecacheCantRemoveObjectFromWritecache = "can't remove object from write-cache" BlobovniczatreeCouldNotGetObjectFromLevel = "could not get object from level" diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 8e191f72c..28f0287bc 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -11,6 +11,9 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" "github.com/google/uuid" @@ -442,3 +445,46 @@ func (e *StorageEngine) deleteShards(ctx context.Context, ids []*shard.ID) ([]ha func (s hashedShard) Hash() uint64 { return s.hash } + +func (e *StorageEngine) ListShardsForObject(ctx context.Context, obj oid.Address) ([]shard.Info, error) { + var err error + var info []shard.Info + prm := shard.ExistsPrm{ + Address: obj, + } + var siErr *objectSDK.SplitInfoError + var ecErr *objectSDK.ECInfoError + + e.iterateOverUnsortedShards(func(hs hashedShard) (stop bool) { + res, exErr := hs.Exists(ctx, prm) + if exErr != nil { + if client.IsErrObjectAlreadyRemoved(exErr) { + err = new(apistatus.ObjectAlreadyRemoved) + return true + } + + // Check if error is either SplitInfoError or ECInfoError. + // True means the object is virtual. + if errors.As(exErr, &siErr) || errors.As(exErr, &ecErr) { + info = append(info, hs.DumpInfo()) + return false + } + + if shard.IsErrObjectExpired(exErr) { + err = exErr + return true + } + + if !client.IsErrObjectNotFound(exErr) { + e.reportShardError(ctx, hs, "could not check existence of object in shard", exErr, zap.Stringer("address", prm.Address)) + } + + return false + } + if res.Exists() { + info = append(info, hs.DumpInfo()) + } + return false + }) + return info, err +} diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index bbf2cf0cc..0c4236d0e 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -32,6 +32,7 @@ const ( rpcListTargetsLocalOverrides = "ListTargetsLocalOverrides" rpcDetachShards = "DetachShards" rpcStartShardRebuild = "StartShardRebuild" + rpcListShardsForObject = "ListShardsForObject" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -364,3 +365,22 @@ func StartShardRebuild(cli *client.Client, req *StartShardRebuildRequest, opts . return wResp.message, nil } + +// ListShardsForObject executes ControlService.ListShardsForObject RPC. +func ListShardsForObject( + cli *client.Client, + req *ListShardsForObjectRequest, + opts ...client.CallOption, +) (*ListShardsForObjectResponse, error) { + wResp := newResponseWrapper[ListShardsForObjectResponse]() + + wReq := &requestWrapper{ + m: req, + } + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcListShardsForObject), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.message, nil +} diff --git a/pkg/services/control/server/list_shards_for_object.go b/pkg/services/control/server/list_shards_for_object.go new file mode 100644 index 000000000..84469772f --- /dev/null +++ b/pkg/services/control/server/list_shards_for_object.go @@ -0,0 +1,66 @@ +package control + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" + + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *Server) ListShardsForObject(ctx context.Context, req *control.ListShardsForObjectRequest) (*control.ListShardsForObjectResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + var obj oid.ID + err = obj.DecodeString(req.GetBody().GetObjectId()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + var cnr cid.ID + err = cnr.DecodeString(req.GetBody().GetContainerId()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + resp := new(control.ListShardsForObjectResponse) + body := new(control.ListShardsForObjectResponse_Body) + resp.SetBody(body) + + var objAddr oid.Address + objAddr.SetContainer(cnr) + objAddr.SetObject(obj) + info, err := s.s.ListShardsForObject(ctx, objAddr) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + if len(info) == 0 { + return nil, status.Error(codes.NotFound, logs.ShardCouldNotFindObject) + } + + body.SetShard_ID(shardInfoToProto(info)) + + // Sign the response + if err := ctrlmessage.Sign(s.key, resp); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil +} + +func shardInfoToProto(infos []shard.Info) [][]byte { + shardInfos := make([][]byte, 0, len(infos)) + for _, info := range infos { + shardInfos = append(shardInfos, *info.ID) + } + + return shardInfos +} diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 97ecf9a8c..4c539acfc 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -89,6 +89,9 @@ service ControlService { // StartShardRebuild starts shard rebuild process. rpc StartShardRebuild(StartShardRebuildRequest) returns (StartShardRebuildResponse); + + // ListShardsForObject returns shard info where object is stored. + rpc ListShardsForObject(ListShardsForObjectRequest) returns (ListShardsForObjectResponse); } // Health check request. @@ -729,3 +732,23 @@ message StartShardRebuildResponse { Signature signature = 2; } + +message ListShardsForObjectRequest { + message Body { + string object_id = 1; + string container_id = 2; + } + + Body body = 1; + Signature signature = 2; +} + +message ListShardsForObjectResponse { + message Body { + // List of the node's shards storing object. + repeated bytes shard_ID = 1; + } + + Body body = 1; + Signature signature = 2; +} diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index 0b4e3cf32..44849d591 100644 --- a/pkg/services/control/service_frostfs.pb.go +++ b/pkg/services/control/service_frostfs.pb.go @@ -17303,3 +17303,727 @@ func (x *StartShardRebuildResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { in.Consumed() } } + +type ListShardsForObjectRequest_Body struct { + ObjectId string `json:"objectId"` + ContainerId string `json:"containerId"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListShardsForObjectRequest_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ListShardsForObjectRequest_Body)(nil) + _ json.Marshaler = (*ListShardsForObjectRequest_Body)(nil) + _ json.Unmarshaler = (*ListShardsForObjectRequest_Body)(nil) +) + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListShardsForObjectRequest_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.StringSize(1, x.ObjectId) + size += proto.StringSize(2, x.ContainerId) + return size +} + +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListShardsForObjectRequest_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *ListShardsForObjectRequest_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if len(x.ObjectId) != 0 { + mm.AppendString(1, x.ObjectId) + } + if len(x.ContainerId) != 0 { + mm.AppendString(2, x.ContainerId) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListShardsForObjectRequest_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListShardsForObjectRequest_Body") + } + switch fc.FieldNum { + case 1: // ObjectId + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ObjectId") + } + x.ObjectId = data + case 2: // ContainerId + data, ok := fc.String() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "ContainerId") + } + x.ContainerId = data + } + } + return nil +} +func (x *ListShardsForObjectRequest_Body) GetObjectId() string { + if x != nil { + return x.ObjectId + } + return "" +} +func (x *ListShardsForObjectRequest_Body) SetObjectId(v string) { + x.ObjectId = v +} +func (x *ListShardsForObjectRequest_Body) GetContainerId() string { + if x != nil { + return x.ContainerId + } + return "" +} +func (x *ListShardsForObjectRequest_Body) SetContainerId(v string) { + x.ContainerId = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListShardsForObjectRequest_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListShardsForObjectRequest_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + first := true + out.RawByte('{') + { + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"objectId\":" + out.RawString(prefix) + out.String(x.ObjectId) + } + { + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"containerId\":" + out.RawString(prefix) + out.String(x.ContainerId) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListShardsForObjectRequest_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListShardsForObjectRequest_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "objectId": + { + var f string + f = in.String() + x.ObjectId = f + } + case "containerId": + { + var f string + f = in.String() + x.ContainerId = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListShardsForObjectRequest struct { + Body *ListShardsForObjectRequest_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListShardsForObjectRequest)(nil) + _ encoding.ProtoUnmarshaler = (*ListShardsForObjectRequest)(nil) + _ json.Marshaler = (*ListShardsForObjectRequest)(nil) + _ json.Unmarshaler = (*ListShardsForObjectRequest)(nil) +) + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListShardsForObjectRequest) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *ListShardsForObjectRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *ListShardsForObjectRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().MarshalProtobuf(buf), nil +} + +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListShardsForObjectRequest) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *ListShardsForObjectRequest) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListShardsForObjectRequest) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListShardsForObjectRequest") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ListShardsForObjectRequest_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ListShardsForObjectRequest) GetBody() *ListShardsForObjectRequest_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ListShardsForObjectRequest) SetBody(v *ListShardsForObjectRequest_Body) { + x.Body = v +} +func (x *ListShardsForObjectRequest) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ListShardsForObjectRequest) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListShardsForObjectRequest) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListShardsForObjectRequest) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + first := true + out.RawByte('{') + { + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) + x.Body.MarshalEasyJSON(out) + } + { + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListShardsForObjectRequest) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListShardsForObjectRequest) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ListShardsForObjectRequest_Body + f = new(ListShardsForObjectRequest_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListShardsForObjectResponse_Body struct { + Shard_ID [][]byte `json:"shardID"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListShardsForObjectResponse_Body)(nil) + _ encoding.ProtoUnmarshaler = (*ListShardsForObjectResponse_Body)(nil) + _ json.Marshaler = (*ListShardsForObjectResponse_Body)(nil) + _ json.Unmarshaler = (*ListShardsForObjectResponse_Body)(nil) +) + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListShardsForObjectResponse_Body) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.RepeatedBytesSize(1, x.Shard_ID) + return size +} + +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListShardsForObjectResponse_Body) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *ListShardsForObjectResponse_Body) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + for j := range x.Shard_ID { + mm.AppendBytes(1, x.Shard_ID[j]) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListShardsForObjectResponse_Body) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListShardsForObjectResponse_Body") + } + switch fc.FieldNum { + case 1: // Shard_ID + data, ok := fc.Bytes() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Shard_ID") + } + x.Shard_ID = append(x.Shard_ID, data) + } + } + return nil +} +func (x *ListShardsForObjectResponse_Body) GetShard_ID() [][]byte { + if x != nil { + return x.Shard_ID + } + return nil +} +func (x *ListShardsForObjectResponse_Body) SetShard_ID(v [][]byte) { + x.Shard_ID = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListShardsForObjectResponse_Body) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListShardsForObjectResponse_Body) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + first := true + out.RawByte('{') + { + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"shardID\":" + out.RawString(prefix) + out.RawByte('[') + for i := range x.Shard_ID { + if i != 0 { + out.RawByte(',') + } + if x.Shard_ID[i] != nil { + out.Base64Bytes(x.Shard_ID[i]) + } else { + out.String("") + } + } + out.RawByte(']') + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListShardsForObjectResponse_Body) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListShardsForObjectResponse_Body) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "shardID": + { + var f []byte + var list [][]byte + in.Delim('[') + for !in.IsDelim(']') { + { + tmp := in.Bytes() + if len(tmp) == 0 { + tmp = nil + } + f = tmp + } + list = append(list, f) + in.WantComma() + } + x.Shard_ID = list + in.Delim(']') + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} + +type ListShardsForObjectResponse struct { + Body *ListShardsForObjectResponse_Body `json:"body"` + Signature *Signature `json:"signature"` +} + +var ( + _ encoding.ProtoMarshaler = (*ListShardsForObjectResponse)(nil) + _ encoding.ProtoUnmarshaler = (*ListShardsForObjectResponse)(nil) + _ json.Marshaler = (*ListShardsForObjectResponse)(nil) + _ json.Unmarshaler = (*ListShardsForObjectResponse)(nil) +) + +// StableSize returns the size of x in protobuf format. +// +// Structures with the same field values have the same binary size. +func (x *ListShardsForObjectResponse) StableSize() (size int) { + if x == nil { + return 0 + } + size += proto.NestedStructureSize(1, x.Body) + size += proto.NestedStructureSize(2, x.Signature) + return size +} + +// ReadSignedData fills buf with signed data of x. +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *ListShardsForObjectResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *ListShardsForObjectResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().MarshalProtobuf(buf), nil +} + +// MarshalProtobuf implements the encoding.ProtoMarshaler interface. +func (x *ListShardsForObjectResponse) MarshalProtobuf(dst []byte) []byte { + m := pool.MarshalerPool.Get() + defer pool.MarshalerPool.Put(m) + x.EmitProtobuf(m.MessageMarshaler()) + dst = m.Marshal(dst) + return dst +} + +func (x *ListShardsForObjectResponse) EmitProtobuf(mm *easyproto.MessageMarshaler) { + if x == nil { + return + } + if x.Body != nil { + x.Body.EmitProtobuf(mm.AppendMessage(1)) + } + if x.Signature != nil { + x.Signature.EmitProtobuf(mm.AppendMessage(2)) + } +} + +// UnmarshalProtobuf implements the encoding.ProtoUnmarshaler interface. +func (x *ListShardsForObjectResponse) UnmarshalProtobuf(src []byte) (err error) { + var fc easyproto.FieldContext + for len(src) > 0 { + src, err = fc.NextField(src) + if err != nil { + return fmt.Errorf("cannot read next field in %s", "ListShardsForObjectResponse") + } + switch fc.FieldNum { + case 1: // Body + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Body") + } + x.Body = new(ListShardsForObjectResponse_Body) + if err := x.Body.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + case 2: // Signature + data, ok := fc.MessageData() + if !ok { + return fmt.Errorf("cannot unmarshal field %s", "Signature") + } + x.Signature = new(Signature) + if err := x.Signature.UnmarshalProtobuf(data); err != nil { + return fmt.Errorf("unmarshal: %w", err) + } + } + } + return nil +} +func (x *ListShardsForObjectResponse) GetBody() *ListShardsForObjectResponse_Body { + if x != nil { + return x.Body + } + return nil +} +func (x *ListShardsForObjectResponse) SetBody(v *ListShardsForObjectResponse_Body) { + x.Body = v +} +func (x *ListShardsForObjectResponse) GetSignature() *Signature { + if x != nil { + return x.Signature + } + return nil +} +func (x *ListShardsForObjectResponse) SetSignature(v *Signature) { + x.Signature = v +} + +// MarshalJSON implements the json.Marshaler interface. +func (x *ListShardsForObjectResponse) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + x.MarshalEasyJSON(&w) + return w.Buffer.BuildBytes(), w.Error +} +func (x *ListShardsForObjectResponse) MarshalEasyJSON(out *jwriter.Writer) { + if x == nil { + out.RawString("null") + return + } + first := true + out.RawByte('{') + { + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"body\":" + out.RawString(prefix) + x.Body.MarshalEasyJSON(out) + } + { + if !first { + out.RawByte(',') + } else { + first = false + } + const prefix string = "\"signature\":" + out.RawString(prefix) + x.Signature.MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (x *ListShardsForObjectResponse) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + x.UnmarshalEasyJSON(&r) + return r.Error() +} +func (x *ListShardsForObjectResponse) UnmarshalEasyJSON(in *jlexer.Lexer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "body": + { + var f *ListShardsForObjectResponse_Body + f = new(ListShardsForObjectResponse_Body) + f.UnmarshalEasyJSON(in) + x.Body = f + } + case "signature": + { + var f *Signature + f = new(Signature) + f.UnmarshalEasyJSON(in) + x.Signature = f + } + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 987e08c59..045662ccf 100644 --- a/pkg/services/control/service_grpc.pb.go +++ b/pkg/services/control/service_grpc.pb.go @@ -41,6 +41,7 @@ const ( ControlService_SealWriteCache_FullMethodName = "/control.ControlService/SealWriteCache" ControlService_DetachShards_FullMethodName = "/control.ControlService/DetachShards" ControlService_StartShardRebuild_FullMethodName = "/control.ControlService/StartShardRebuild" + ControlService_ListShardsForObject_FullMethodName = "/control.ControlService/ListShardsForObject" ) // ControlServiceClient is the client API for ControlService service. @@ -95,6 +96,8 @@ type ControlServiceClient interface { DetachShards(ctx context.Context, in *DetachShardsRequest, opts ...grpc.CallOption) (*DetachShardsResponse, error) // StartShardRebuild starts shard rebuild process. StartShardRebuild(ctx context.Context, in *StartShardRebuildRequest, opts ...grpc.CallOption) (*StartShardRebuildResponse, error) + // ListShardsForObject returns shard info where object is stored. + ListShardsForObject(ctx context.Context, in *ListShardsForObjectRequest, opts ...grpc.CallOption) (*ListShardsForObjectResponse, error) } type controlServiceClient struct { @@ -303,6 +306,15 @@ func (c *controlServiceClient) StartShardRebuild(ctx context.Context, in *StartS return out, nil } +func (c *controlServiceClient) ListShardsForObject(ctx context.Context, in *ListShardsForObjectRequest, opts ...grpc.CallOption) (*ListShardsForObjectResponse, error) { + out := new(ListShardsForObjectResponse) + err := c.cc.Invoke(ctx, ControlService_ListShardsForObject_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // ControlServiceServer is the server API for ControlService service. // All implementations should embed UnimplementedControlServiceServer // for forward compatibility @@ -355,6 +367,8 @@ type ControlServiceServer interface { DetachShards(context.Context, *DetachShardsRequest) (*DetachShardsResponse, error) // StartShardRebuild starts shard rebuild process. StartShardRebuild(context.Context, *StartShardRebuildRequest) (*StartShardRebuildResponse, error) + // ListShardsForObject returns shard info where object is stored. + ListShardsForObject(context.Context, *ListShardsForObjectRequest) (*ListShardsForObjectResponse, error) } // UnimplementedControlServiceServer should be embedded to have forward compatible implementations. @@ -427,6 +441,9 @@ func (UnimplementedControlServiceServer) DetachShards(context.Context, *DetachSh func (UnimplementedControlServiceServer) StartShardRebuild(context.Context, *StartShardRebuildRequest) (*StartShardRebuildResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method StartShardRebuild not implemented") } +func (UnimplementedControlServiceServer) ListShardsForObject(context.Context, *ListShardsForObjectRequest) (*ListShardsForObjectResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListShardsForObject not implemented") +} // UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ControlServiceServer will @@ -835,6 +852,24 @@ func _ControlService_StartShardRebuild_Handler(srv interface{}, ctx context.Cont return interceptor(ctx, in, info, handler) } +func _ControlService_ListShardsForObject_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListShardsForObjectRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServiceServer).ListShardsForObject(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ControlService_ListShardsForObject_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServiceServer).ListShardsForObject(ctx, req.(*ListShardsForObjectRequest)) + } + return interceptor(ctx, in, info, handler) +} + // ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -930,6 +965,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{ MethodName: "StartShardRebuild", Handler: _ControlService_StartShardRebuild_Handler, }, + { + MethodName: "ListShardsForObject", + Handler: _ControlService_ListShardsForObject_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "pkg/services/control/service.proto", From aab8addae006ea7667d0851170b278c65f2643e8 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 18 Feb 2025 13:04:48 +0300 Subject: [PATCH 1763/1943] [#1635] cli: Make `object.readObjectAddress()` public This method will be useful for upcoming control command. Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/modules/object/delete.go | 2 +- cmd/frostfs-cli/modules/object/get.go | 2 +- cmd/frostfs-cli/modules/object/hash.go | 2 +- cmd/frostfs-cli/modules/object/head.go | 2 +- cmd/frostfs-cli/modules/object/nodes.go | 2 +- cmd/frostfs-cli/modules/object/patch.go | 2 +- cmd/frostfs-cli/modules/object/range.go | 2 +- cmd/frostfs-cli/modules/object/util.go | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/delete.go b/cmd/frostfs-cli/modules/object/delete.go index e4e9cddb8..08a9ac4c8 100644 --- a/cmd/frostfs-cli/modules/object/delete.go +++ b/cmd/frostfs-cli/modules/object/delete.go @@ -55,7 +55,7 @@ func deleteObject(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "", fmt.Errorf("required flag \"%s\" not set", commonflags.OIDFlag)) } - objAddr = readObjectAddress(cmd, &cnr, &obj) + objAddr = ReadObjectAddress(cmd, &cnr, &obj) } pk := key.GetOrGenerate(cmd) diff --git a/cmd/frostfs-cli/modules/object/get.go b/cmd/frostfs-cli/modules/object/get.go index f1edccba2..7312f5384 100644 --- a/cmd/frostfs-cli/modules/object/get.go +++ b/cmd/frostfs-cli/modules/object/get.go @@ -46,7 +46,7 @@ func getObject(cmd *cobra.Command, _ []string) { var cnr cid.ID var obj oid.ID - objAddr := readObjectAddress(cmd, &cnr, &obj) + objAddr := ReadObjectAddress(cmd, &cnr, &obj) filename := cmd.Flag(fileFlag).Value.String() out, closer := createOutWriter(cmd, filename) diff --git a/cmd/frostfs-cli/modules/object/hash.go b/cmd/frostfs-cli/modules/object/hash.go index d8ea449eb..25df375d4 100644 --- a/cmd/frostfs-cli/modules/object/hash.go +++ b/cmd/frostfs-cli/modules/object/hash.go @@ -52,7 +52,7 @@ func getObjectHash(cmd *cobra.Command, _ []string) { var cnr cid.ID var obj oid.ID - objAddr := readObjectAddress(cmd, &cnr, &obj) + objAddr := ReadObjectAddress(cmd, &cnr, &obj) ranges, err := getRangeList(cmd) commonCmd.ExitOnErr(cmd, "", err) diff --git a/cmd/frostfs-cli/modules/object/head.go b/cmd/frostfs-cli/modules/object/head.go index 70c273443..97e996cad 100644 --- a/cmd/frostfs-cli/modules/object/head.go +++ b/cmd/frostfs-cli/modules/object/head.go @@ -47,7 +47,7 @@ func getObjectHeader(cmd *cobra.Command, _ []string) { var cnr cid.ID var obj oid.ID - objAddr := readObjectAddress(cmd, &cnr, &obj) + objAddr := ReadObjectAddress(cmd, &cnr, &obj) pk := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 1500830a2..bc34b370d 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -101,7 +101,7 @@ func initObjectNodesCmd() { func objectNodes(cmd *cobra.Command, _ []string) { var cnrID cid.ID var objID oid.ID - readObjectAddress(cmd, &cnrID, &objID) + ReadObjectAddress(cmd, &cnrID, &objID) pk := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) diff --git a/cmd/frostfs-cli/modules/object/patch.go b/cmd/frostfs-cli/modules/object/patch.go index ebc415b2f..d98182679 100644 --- a/cmd/frostfs-cli/modules/object/patch.go +++ b/cmd/frostfs-cli/modules/object/patch.go @@ -56,7 +56,7 @@ func patch(cmd *cobra.Command, _ []string) { var cnr cid.ID var obj oid.ID - objAddr := readObjectAddress(cmd, &cnr, &obj) + objAddr := ReadObjectAddress(cmd, &cnr, &obj) ranges, err := getRangeSlice(cmd) commonCmd.ExitOnErr(cmd, "", err) diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index 8f59906ca..be4fee4cf 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -47,7 +47,7 @@ func getObjectRange(cmd *cobra.Command, _ []string) { var cnr cid.ID var obj oid.ID - objAddr := readObjectAddress(cmd, &cnr, &obj) + objAddr := ReadObjectAddress(cmd, &cnr, &obj) ranges, err := getRangeList(cmd) commonCmd.ExitOnErr(cmd, "", err) diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index b090c9f8c..3955f8ee1 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -74,7 +74,7 @@ func parseXHeaders(cmd *cobra.Command) []string { return xs } -func readObjectAddress(cmd *cobra.Command, cnr *cid.ID, obj *oid.ID) oid.Address { +func ReadObjectAddress(cmd *cobra.Command, cnr *cid.ID, obj *oid.ID) oid.Address { readCID(cmd, cnr) readOID(cmd, obj) From df6d9da82aa965dbbf45114149b8279655276127 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 4 Feb 2025 21:23:00 +0300 Subject: [PATCH 1764/1943] [#1635] cli: Add command to get object's shard info Added `frostfs-cli object locate` subcommand. It lists info about shards storing an object. Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/modules/control/locate.go | 118 ++++++++++++++++++++++ cmd/frostfs-cli/modules/control/root.go | 2 + go.mod | 4 +- go.sum | 8 +- 4 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 cmd/frostfs-cli/modules/control/locate.go diff --git a/cmd/frostfs-cli/modules/control/locate.go b/cmd/frostfs-cli/modules/control/locate.go new file mode 100644 index 000000000..d10e2a896 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/locate.go @@ -0,0 +1,118 @@ +package control + +import ( + "bytes" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + object "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/modules/object" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + + "github.com/mr-tron/base58" + "github.com/spf13/cobra" +) + +const ( + FullInfoFlag = "full" + FullInfoFlagUsage = "Print full ShardInfo." +) + +var locateObjectCmd = &cobra.Command{ + Use: "locate-object", + Short: "List shards storing the object", + Long: "List shards storing the object", + Run: locateObject, +} + +func initControlLocateObjectCmd() { + initControlFlags(locateObjectCmd) + + flags := locateObjectCmd.Flags() + + flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + _ = locateObjectCmd.MarkFlagRequired(commonflags.CIDFlag) + + flags.String(commonflags.OIDFlag, "", commonflags.OIDFlagUsage) + _ = locateObjectCmd.MarkFlagRequired(commonflags.OIDFlag) + + flags.Bool(commonflags.JSON, false, "Print shard info as a JSON array. Requires --full flag.") + flags.Bool(FullInfoFlag, false, FullInfoFlagUsage) +} + +func locateObject(cmd *cobra.Command, _ []string) { + var cnr cid.ID + var obj oid.ID + + _ = object.ReadObjectAddress(cmd, &cnr, &obj) + + pk := key.Get(cmd) + + body := new(control.ListShardsForObjectRequest_Body) + body.SetContainerId(cnr.EncodeToString()) + body.SetObjectId(obj.EncodeToString()) + req := new(control.ListShardsForObjectRequest) + req.SetBody(body) + signRequest(cmd, pk, req) + + cli := getClient(cmd, pk) + + var err error + var resp *control.ListShardsForObjectResponse + err = cli.ExecRaw(func(client *rawclient.Client) error { + resp, err = control.ListShardsForObject(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + shardIDs := resp.GetBody().GetShard_ID() + + isFull, _ := cmd.Flags().GetBool(FullInfoFlag) + if !isFull { + for _, id := range shardIDs { + cmd.Println(base58.Encode(id)) + } + return + } + + // get full shard info + listShardsReq := new(control.ListShardsRequest) + listShardsReq.SetBody(new(control.ListShardsRequest_Body)) + signRequest(cmd, pk, listShardsReq) + var listShardsResp *control.ListShardsResponse + err = cli.ExecRaw(func(client *rawclient.Client) error { + listShardsResp, err = control.ListShards(client, listShardsReq) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, listShardsResp.GetSignature(), listShardsResp.GetBody()) + + shards := listShardsResp.GetBody().GetShards() + sortShardsByID(shards) + shards = filterShards(shards, shardIDs) + + isJSON, _ := cmd.Flags().GetBool(commonflags.JSON) + if isJSON { + prettyPrintShardsJSON(cmd, shards) + } else { + prettyPrintShards(cmd, shards) + } +} + +func filterShards(info []control.ShardInfo, ids [][]byte) []control.ShardInfo { + var res []control.ShardInfo + for _, id := range ids { + for _, inf := range info { + if bytes.Equal(inf.Shard_ID, id) { + res = append(res, inf) + } + } + } + return res +} diff --git a/cmd/frostfs-cli/modules/control/root.go b/cmd/frostfs-cli/modules/control/root.go index b20d3618e..3abfe80cb 100644 --- a/cmd/frostfs-cli/modules/control/root.go +++ b/cmd/frostfs-cli/modules/control/root.go @@ -39,6 +39,7 @@ func init() { listRulesCmd, getRuleCmd, listTargetsCmd, + locateObjectCmd, ) initControlHealthCheckCmd() @@ -52,4 +53,5 @@ func init() { initControlListRulesCmd() initControGetRuleCmd() initControlListTargetsCmd() + initControlLocateObjectCmd() } diff --git a/go.mod b/go.mod index 18378466e..9d0988bcd 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( code.gitea.io/sdk/gitea v0.17.1 - git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1-0.20241205083807-762d7f9f9f08 + git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 @@ -12,7 +12,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/VictoriaMetrics/easyproto v0.1.4 diff --git a/go.sum b/go.sum index 5205dddef..3ec679ee7 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1-0.20241205083807-762d7f9f9f08 h1:tl1TT+zNk1lF/J5EaD3syDrTaYbQwvJKVOVENM4oQ+k= -git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1-0.20241205083807-762d7f9f9f08/go.mod h1:5fSm/l5xSjGWqsPUffSdboiGFUHa7y/1S0fvxzQowN8= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1 h1:k1Qw8dWUQczfo0eVXlhrq9eXEbUMyDLW8jEMzY+gxMc= +git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1/go.mod h1:5fSm/l5xSjGWqsPUffSdboiGFUHa7y/1S0fvxzQowN8= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d h1:uJ/wvuMdepbkaV8XMS5uN9B0FQWMep0CttSuDZiDhq0= @@ -18,8 +18,8 @@ git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/96 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972/go.mod h1:2hM42MBrlhvN6XToaW6OWNk5ZLcu1FhaukGgxtfpDDI= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20241015133823-8aee80dbdc07 h1:gPaqGsk6gSWQyNVjaStydfUz6Z/loHc9XyvGrJ5qSPY= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20241015133823-8aee80dbdc07/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88 h1:vgbfkcnIexZUm3vREBBSa/Gv1Whjd1SFCUd0A+IaGPQ= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240814080254-96225afacb88/go.mod h1:SgioiGhQNWqiV5qpFAXRDJF81SEFRBhtwGEiU0FViyA= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b h1:M50kdfrf/h8c3cz0bJ2AEUcbXvAlPFVC1Wp1WkfZ/8E= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b/go.mod h1:GZTk55RI4dKzsK6BCn5h2xxE28UHNfgoq/NJxW/LQ6A= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= From bd61f7bf0a8e5a102175da73d0b79e9218de69f8 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Fri, 7 Mar 2025 12:29:28 +0300 Subject: [PATCH 1765/1943] [#1666] audit: Fix duplicated log in `Patch` method When we do `object patch` with audit enabled we get several duplicated entries in logs. `object patch` request is logged in 2 places: 1. `(*auditPatchStream) CloseAndRecv()` - when the client closes the request stream or when stream gets aborted. 2. `(*auditPatchStream) Send()` - when stream was NOT aborted. `Send()` doesn't check if `err != nil` before logging. It led to to logging on every `Send()` call. Signed-off-by: Ekaterina Lebedeva --- pkg/services/object/audit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/object/audit.go b/pkg/services/object/audit.go index 367be0c0c..f8ee089fe 100644 --- a/pkg/services/object/audit.go +++ b/pkg/services/object/audit.go @@ -224,7 +224,7 @@ func (a *auditPatchStream) Send(ctx context.Context, req *object.PatchRequest) e if err != nil { a.failed = true } - if !errors.Is(err, util.ErrAbortStream) { // CloseAndRecv will not be called, so log here + if err != nil && !errors.Is(err, util.ErrAbortStream) { // CloseAndRecv will not be called, so log here audit.LogRequestWithKey(ctx, a.log, objectGRPC.ObjectService_Patch_FullMethodName, a.key, audit.TargetFromContainerIDObjectID(a.containerID, a.objectID), !a.failed) From 8643e0abc569f9f540abac5b9d0fba9e05ae5640 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 10 Mar 2025 17:51:06 +0300 Subject: [PATCH 1766/1943] [#1668] writecache: Use object size to check free space Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/writecache/put.go | 11 ++++++++++- pkg/local_object_storage/writecache/state.go | 4 ---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 7da5c4d3a..2fbf50913 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -2,6 +2,7 @@ package writecache import ( "context" + "fmt" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -59,7 +60,15 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro // putBig writes object to FSTree and pushes it to the flush workers queue. func (c *cache) putBig(ctx context.Context, prm common.PutPrm) error { - if !c.hasEnoughSpaceFS() { + if prm.RawData == nil { // foolproof: RawData should be marshalled by shard. + data, err := prm.Object.Marshal() + if err != nil { + return fmt.Errorf("cannot marshal object: %w", err) + } + prm.RawData = data + } + size := uint64(len(prm.RawData)) + if !c.hasEnoughSpace(size) { return ErrOutOfSpace } diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index 44caa2603..7a52d3672 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -7,10 +7,6 @@ func (c *cache) estimateCacheSize() (uint64, uint64) { return count, size } -func (c *cache) hasEnoughSpaceFS() bool { - return c.hasEnoughSpace(c.maxObjectSize) -} - func (c *cache) hasEnoughSpace(objectSize uint64) bool { count, size := c.estimateCacheSize() if c.maxCacheCount > 0 && count+1 > c.maxCacheCount { From d36afa31c7b4d0a878ecbe8e41657f6f955fbdc9 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 18 Feb 2025 10:57:17 +0300 Subject: [PATCH 1767/1943] [#1653] qos: Fix logging Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/qos.go | 6 ++++-- internal/logs/logs.go | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-node/qos.go b/cmd/frostfs-node/qos.go index bfc278333..9663fc6ae 100644 --- a/cmd/frostfs-node/qos.go +++ b/cmd/frostfs-node/qos.go @@ -47,7 +47,7 @@ func (s *cfgQoSService) AdjustIncomingTag(ctx context.Context, requestSignPublic } ioTag, err := qos.FromRawString(rawTag) if err != nil { - s.logger.Warn(ctx, logs.FailedToParseIncomingIOTag, zap.Error(err)) + s.logger.Debug(ctx, logs.FailedToParseIncomingIOTag, zap.Error(err)) return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) } @@ -70,6 +70,7 @@ func (s *cfgQoSService) AdjustIncomingTag(ctx context.Context, requestSignPublic return ctx } } + s.logger.Debug(ctx, logs.FailedToValidateIncomingIOTag) return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) case qos.IOTagInternal: for _, pk := range s.allowedInternalPubs { @@ -87,9 +88,10 @@ func (s *cfgQoSService) AdjustIncomingTag(ctx context.Context, requestSignPublic return ctx } } + s.logger.Debug(ctx, logs.FailedToValidateIncomingIOTag) return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) default: - s.logger.Warn(ctx, logs.NotSupportedIncomingIOTagReplacedWithClient, zap.Stringer("io_tag", ioTag)) + s.logger.Debug(ctx, logs.NotSupportedIncomingIOTagReplacedWithClient, zap.Stringer("io_tag", ioTag)) return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) } } diff --git a/internal/logs/logs.go b/internal/logs/logs.go index d07f47fbf..6115cdf90 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -513,4 +513,5 @@ const ( FailedToParseIncomingIOTag = "failed to parse incoming IO tag" NotSupportedIncomingIOTagReplacedWithClient = "incoming IO tag is not supported, replaced with `client`" FailedToGetNetmapToAdjustIOTag = "failed to get netmap to adjust IO tag, replaced with `client`" + FailedToValidateIncomingIOTag = "failed to validate incoming IO tag, replaced with `client`" ) From 3727d60331d3d410087a146da1df2dfbef7fa7cd Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Mar 2025 13:53:32 +0300 Subject: [PATCH 1768/1943] [#1653] qos: Add metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 + internal/metrics/consts.go | 2 + internal/metrics/node.go | 6 ++ internal/metrics/qos.go | 52 +++++++++ internal/qos/limiter.go | 101 +++++++++++++++--- internal/qos/metrics.go | 31 ++++++ internal/qos/stats.go | 28 +++++ internal/qos/validate.go | 9 -- .../engine/engine_test.go | 4 + pkg/local_object_storage/shard/id.go | 1 + 10 files changed, 214 insertions(+), 21 deletions(-) create mode 100644 internal/metrics/qos.go create mode 100644 internal/qos/metrics.go create mode 100644 internal/qos/stats.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index afde0bbc0..92aa827f2 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1048,6 +1048,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID } if c.metricsCollector != nil { mbOptions = append(mbOptions, meta.WithMetrics(lsmetrics.NewMetabaseMetrics(shCfg.metaCfg.path, c.metricsCollector.MetabaseMetrics()))) + shCfg.limiter.SetMetrics(c.metricsCollector.QoSMetrics()) } var sh shardOptsWithID diff --git a/internal/metrics/consts.go b/internal/metrics/consts.go index cb165de69..9123541ff 100644 --- a/internal/metrics/consts.go +++ b/internal/metrics/consts.go @@ -23,6 +23,7 @@ const ( policerSubsystem = "policer" commonCacheSubsystem = "common_cache" multinetSubsystem = "multinet" + qosSubsystem = "qos" successLabel = "success" shardIDLabel = "shard_id" @@ -43,6 +44,7 @@ const ( hitLabel = "hit" cacheLabel = "cache" sourceIPLabel = "source_ip" + ioTagLabel = "io_tag" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" diff --git a/internal/metrics/node.go b/internal/metrics/node.go index 4ea3c7c24..8ade19eb2 100644 --- a/internal/metrics/node.go +++ b/internal/metrics/node.go @@ -26,6 +26,7 @@ type NodeMetrics struct { morphCache *morphCacheMetrics log logger.LogMetrics multinet *multinetMetrics + qos *QoSMetrics // nolint: unused appInfo *ApplicationInfo } @@ -55,6 +56,7 @@ func NewNodeMetrics() *NodeMetrics { log: logger.NewLogMetrics(namespace), appInfo: NewApplicationInfo(misc.Version), multinet: newMultinetMetrics(namespace), + qos: newQoSMetrics(), } } @@ -126,3 +128,7 @@ func (m *NodeMetrics) LogMetrics() logger.LogMetrics { func (m *NodeMetrics) MultinetMetrics() MultinetMetrics { return m.multinet } + +func (m *NodeMetrics) QoSMetrics() *QoSMetrics { + return m.qos +} diff --git a/internal/metrics/qos.go b/internal/metrics/qos.go new file mode 100644 index 000000000..17fb67a27 --- /dev/null +++ b/internal/metrics/qos.go @@ -0,0 +1,52 @@ +package metrics + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type QoSMetrics struct { + opsCounter *prometheus.GaugeVec +} + +func newQoSMetrics() *QoSMetrics { + return &QoSMetrics{ + opsCounter: metrics.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: qosSubsystem, + Name: "operations_total", + Help: "Count of pending, in progree, completed and failed due of resource exhausted error operations for each shard", + }, []string{shardIDLabel, operationLabel, ioTagLabel, typeLabel}), + } +} + +func (m *QoSMetrics) SetOperationTagCounters(shardID, operation, tag string, pending, inProgress, completed, resourceExhausted uint64) { + m.opsCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + operationLabel: operation, + ioTagLabel: tag, + typeLabel: "pending", + }).Set(float64(pending)) + m.opsCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + operationLabel: operation, + ioTagLabel: tag, + typeLabel: "in_progress", + }).Set(float64(inProgress)) + m.opsCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + operationLabel: operation, + ioTagLabel: tag, + typeLabel: "completed", + }).Set(float64(completed)) + m.opsCounter.With(prometheus.Labels{ + shardIDLabel: shardID, + operationLabel: operation, + ioTagLabel: tag, + typeLabel: "resource_exhausted", + }).Set(float64(resourceExhausted)) +} + +func (m *QoSMetrics) Close(shardID string) { + m.opsCounter.DeletePartialMatch(prometheus.Labels{shardIDLabel: shardID}) +} diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index b1406a7f3..8f00791c5 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "sync" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" @@ -15,6 +17,9 @@ import ( const ( defaultIdleTimeout time.Duration = 0 defaultShare float64 = 1.0 + minusOne = ^uint64(0) + + defaultMetricsCollectTimeout = 5 * time.Second ) type ReleaseFunc scheduling.ReleaseFunc @@ -22,6 +27,8 @@ type ReleaseFunc scheduling.ReleaseFunc type Limiter interface { ReadRequest(context.Context) (ReleaseFunc, error) WriteRequest(context.Context) (ReleaseFunc, error) + SetParentID(string) + SetMetrics(Metrics) Close() } @@ -34,10 +41,6 @@ func NewLimiter(c *limits.Config) (Limiter, error) { if err := validateConfig(c); err != nil { return nil, err } - read, write := c.Read(), c.Write() - if isNoop(read, write) { - return noopLimiterInstance, nil - } readScheduler, err := createScheduler(c.Read()) if err != nil { return nil, fmt.Errorf("create read scheduler: %w", err) @@ -46,10 +49,18 @@ func NewLimiter(c *limits.Config) (Limiter, error) { if err != nil { return nil, fmt.Errorf("create write scheduler: %w", err) } - return &mClockLimiter{ + l := &mClockLimiter{ readScheduler: readScheduler, writeScheduler: writeScheduler, - }, nil + closeCh: make(chan struct{}), + wg: &sync.WaitGroup{}, + readStats: createStats(), + writeStats: createStats(), + } + l.shardID.Store(&shardID{}) + l.metrics.Store(&metricsHolder{metrics: &noopMetrics{}}) + l.startMetricsCollect() + return l, nil } func createScheduler(config limits.OpConfig) (scheduler, error) { @@ -91,7 +102,7 @@ var ( ) func NewNoopLimiter() Limiter { - return &noopLimiter{} + return noopLimiterInstance } type noopLimiter struct{} @@ -104,43 +115,109 @@ func (n *noopLimiter) WriteRequest(context.Context) (ReleaseFunc, error) { return releaseStub, nil } +func (n *noopLimiter) SetParentID(string) {} + func (n *noopLimiter) Close() {} +func (n *noopLimiter) SetMetrics(Metrics) {} + var _ Limiter = (*mClockLimiter)(nil) +type shardID struct { + id string +} + type mClockLimiter struct { readScheduler scheduler writeScheduler scheduler + + readStats map[string]*stat + writeStats map[string]*stat + + shardID atomic.Pointer[shardID] + metrics atomic.Pointer[metricsHolder] + closeCh chan struct{} + wg *sync.WaitGroup } func (n *mClockLimiter) ReadRequest(ctx context.Context) (ReleaseFunc, error) { - return requestArrival(ctx, n.readScheduler) + return requestArrival(ctx, n.readScheduler, n.readStats) } func (n *mClockLimiter) WriteRequest(ctx context.Context) (ReleaseFunc, error) { - return requestArrival(ctx, n.writeScheduler) + return requestArrival(ctx, n.writeScheduler, n.writeStats) } -func requestArrival(ctx context.Context, s scheduler) (ReleaseFunc, error) { +func requestArrival(ctx context.Context, s scheduler, stats map[string]*stat) (ReleaseFunc, error) { tag, ok := tagging.IOTagFromContext(ctx) if !ok { tag = IOTagClient.String() } + stat := getStat(tag, stats) + stat.pending.Add(1) if tag == IOTagCritical.String() { - return releaseStub, nil + stat.inProgress.Add(1) + return func() { + stat.completed.Add(1) + }, nil } rel, err := s.RequestArrival(ctx, tag) + stat.inProgress.Add(1) if err != nil { if errors.Is(err, scheduling.ErrMClockSchedulerRequestLimitExceeded) || errors.Is(err, errSemaphoreLimitExceeded) { + stat.resourceExhausted.Add(1) return nil, &apistatus.ResourceExhausted{} } + stat.completed.Add(1) return nil, err } - return ReleaseFunc(rel), nil + return func() { + rel() + stat.completed.Add(1) + }, nil } func (n *mClockLimiter) Close() { n.readScheduler.Close() n.writeScheduler.Close() + close(n.closeCh) + n.wg.Wait() + n.metrics.Load().metrics.Close(n.shardID.Load().id) +} + +func (n *mClockLimiter) SetParentID(parentID string) { + n.shardID.Store(&shardID{id: parentID}) +} + +func (n *mClockLimiter) SetMetrics(m Metrics) { + n.metrics.Store(&metricsHolder{metrics: m}) +} + +func (n *mClockLimiter) startMetricsCollect() { + n.wg.Add(1) + go func() { + defer n.wg.Done() + + ticker := time.NewTicker(defaultMetricsCollectTimeout) + defer ticker.Stop() + for { + select { + case <-n.closeCh: + return + case <-ticker.C: + shardID := n.shardID.Load().id + if shardID == "" { + continue + } + metrics := n.metrics.Load().metrics + for tag, s := range n.readStats { + metrics.SetOperationTagCounters(shardID, "read", tag, s.pending.Load(), s.inProgress.Load(), s.completed.Load(), s.resourceExhausted.Load()) + } + for tag, s := range n.writeStats { + metrics.SetOperationTagCounters(shardID, "write", tag, s.pending.Load(), s.inProgress.Load(), s.completed.Load(), s.resourceExhausted.Load()) + } + } + } + }() } diff --git a/internal/qos/metrics.go b/internal/qos/metrics.go new file mode 100644 index 000000000..c00da51b7 --- /dev/null +++ b/internal/qos/metrics.go @@ -0,0 +1,31 @@ +package qos + +import "sync/atomic" + +type Metrics interface { + SetOperationTagCounters(shardID, operation, tag string, pending, inProgress, completed, resourceExhausted uint64) + Close(shardID string) +} + +var _ Metrics = (*noopMetrics)(nil) + +type noopMetrics struct{} + +func (n *noopMetrics) SetOperationTagCounters(string, string, string, uint64, uint64, uint64, uint64) { +} + +func (n *noopMetrics) Close(string) {} + +// stat presents limiter statistics cumulative counters. +// +// Each operation changes its status as follows: `pending` -> `in_progress` -> `completed` or `resource_exhausted`. +type stat struct { + completed atomic.Uint64 + pending atomic.Uint64 + resourceExhausted atomic.Uint64 + inProgress atomic.Uint64 +} + +type metricsHolder struct { + metrics Metrics +} diff --git a/internal/qos/stats.go b/internal/qos/stats.go new file mode 100644 index 000000000..f077f552b --- /dev/null +++ b/internal/qos/stats.go @@ -0,0 +1,28 @@ +package qos + +const unknownStatsTag = "unknown" + +var statTags = map[string]struct{}{ + IOTagClient.String(): {}, + IOTagBackground.String(): {}, + IOTagInternal.String(): {}, + IOTagPolicer.String(): {}, + IOTagWritecache.String(): {}, + IOTagCritical.String(): {}, + unknownStatsTag: {}, +} + +func createStats() map[string]*stat { + result := make(map[string]*stat) + for tag := range statTags { + result[tag] = &stat{} + } + return result +} + +func getStat(tag string, stats map[string]*stat) *stat { + if v, ok := stats[tag]; ok { + return v + } + return stats[unknownStatsTag] +} diff --git a/internal/qos/validate.go b/internal/qos/validate.go index 43aa74942..3fa4ebbd1 100644 --- a/internal/qos/validate.go +++ b/internal/qos/validate.go @@ -90,12 +90,3 @@ func float64Value(f *float64) float64 { } return *f } - -func isNoop(read, write limits.OpConfig) bool { - return read.MaxRunningOps == limits.NoLimit && - read.MaxWaitingOps == limits.NoLimit && - write.MaxRunningOps == limits.NoLimit && - write.MaxWaitingOps == limits.NoLimit && - len(read.Tags) == 0 && - len(write.Tags) == 0 -} diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 7ddde1f02..3f9196128 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -163,6 +163,8 @@ type testQoSLimiter struct { write atomic.Int64 } +func (t *testQoSLimiter) SetMetrics(qos.Metrics) {} + func (t *testQoSLimiter) Close() { require.Equal(t.t, int64(0), t.read.Load(), "read requests count after limiter close must be 0") require.Equal(t.t, int64(0), t.write.Load(), "write requests count after limiter close must be 0") @@ -177,3 +179,5 @@ func (t *testQoSLimiter) WriteRequest(context.Context) (qos.ReleaseFunc, error) t.write.Add(1) return func() { t.write.Add(-1) }, nil } + +func (t *testQoSLimiter) SetParentID(string) {} diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index 26492cf01..b233b705c 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -61,6 +61,7 @@ func (s *Shard) UpdateID(ctx context.Context) (err error) { if s.pilorama != nil { s.pilorama.SetParentID(s.info.ID.String()) } + s.opsLimiter.SetParentID(s.info.ID.String()) if len(idFromMetabase) == 0 && !modeDegraded { if setErr := s.metaBase.SetShardID(ctx, *s.info.ID, s.GetMode()); setErr != nil { From 4ed2bbdb0f72589e59b5127ce237b65151f13b57 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Mar 2025 15:25:35 +0300 Subject: [PATCH 1769/1943] [#1653] objectSvc: Add operations by IO tag metric Signed-off-by: Dmitrii Stepanov --- internal/metrics/object.go | 19 +++++++++++++++---- internal/qos/tags.go | 15 ++++++++++++++- pkg/services/object/metrics.go | 21 +++++++++++---------- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/internal/metrics/object.go b/internal/metrics/object.go index 0ba994ed3..e4f6dfde1 100644 --- a/internal/metrics/object.go +++ b/internal/metrics/object.go @@ -9,13 +9,14 @@ import ( ) type ObjectServiceMetrics interface { - AddRequestDuration(method string, d time.Duration, success bool) + AddRequestDuration(method string, d time.Duration, success bool, ioTag string) AddPayloadSize(method string, size int) } type objectServiceMetrics struct { - methodDuration *prometheus.HistogramVec - payloadCounter *prometheus.CounterVec + methodDuration *prometheus.HistogramVec + payloadCounter *prometheus.CounterVec + ioTagOpsCounter *prometheus.CounterVec } func newObjectServiceMetrics() *objectServiceMetrics { @@ -32,14 +33,24 @@ func newObjectServiceMetrics() *objectServiceMetrics { Name: "request_payload_bytes", Help: "Object Service request payload", }, []string{methodLabel}), + ioTagOpsCounter: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: objectSubsystem, + Name: "requests_total", + Help: "Count of requests for each IO tag", + }, []string{methodLabel, ioTagLabel}), } } -func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool) { +func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool, ioTag string) { m.methodDuration.With(prometheus.Labels{ methodLabel: method, successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) + m.ioTagOpsCounter.With(prometheus.Labels{ + ioTagLabel: ioTag, + methodLabel: method, + }).Inc() } func (m *objectServiceMetrics) AddPayloadSize(method string, size int) { diff --git a/internal/qos/tags.go b/internal/qos/tags.go index 6a9a7f7a4..9db45f190 100644 --- a/internal/qos/tags.go +++ b/internal/qos/tags.go @@ -1,6 +1,11 @@ package qos -import "fmt" +import ( + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" +) type IOTag string @@ -37,3 +42,11 @@ func FromRawString(s string) (IOTag, error) { func (t IOTag) String() string { return string(t) } + +func IOTagFromContext(ctx context.Context) string { + tag, ok := tagging.IOTagFromContext(ctx) + if !ok { + tag = "undefined" + } + return tag +} diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index 19748e938..6a6ee0f0f 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -4,6 +4,7 @@ import ( "context" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" ) @@ -34,7 +35,7 @@ type ( } MetricRegister interface { - AddRequestDuration(string, time.Duration, bool) + AddRequestDuration(string, time.Duration, bool, string) AddPayloadSize(string, int) } ) @@ -51,7 +52,7 @@ func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (er if m.enabled { t := time.Now() defer func() { - m.metrics.AddRequestDuration("Get", time.Since(t), err == nil) + m.metrics.AddRequestDuration("Get", time.Since(t), err == nil, qos.IOTagFromContext(stream.Context())) }() err = m.next.Get(req, &getStreamMetric{ ServerStream: stream, @@ -106,7 +107,7 @@ func (m MetricCollector) PutSingle(ctx context.Context, request *object.PutSingl res, err := m.next.PutSingle(ctx, request) - m.metrics.AddRequestDuration("PutSingle", time.Since(t), err == nil) + m.metrics.AddRequestDuration("PutSingle", time.Since(t), err == nil, qos.IOTagFromContext(ctx)) if err == nil { m.metrics.AddPayloadSize("PutSingle", len(request.GetBody().GetObject().GetPayload())) } @@ -122,7 +123,7 @@ func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) res, err := m.next.Head(ctx, request) - m.metrics.AddRequestDuration("Head", time.Since(t), err == nil) + m.metrics.AddRequestDuration("Head", time.Since(t), err == nil, qos.IOTagFromContext(ctx)) return res, err } @@ -135,7 +136,7 @@ func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream) err := m.next.Search(req, stream) - m.metrics.AddRequestDuration("Search", time.Since(t), err == nil) + m.metrics.AddRequestDuration("Search", time.Since(t), err == nil, qos.IOTagFromContext(stream.Context())) return err } @@ -148,7 +149,7 @@ func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteReque res, err := m.next.Delete(ctx, request) - m.metrics.AddRequestDuration("Delete", time.Since(t), err == nil) + m.metrics.AddRequestDuration("Delete", time.Since(t), err == nil, qos.IOTagFromContext(ctx)) return res, err } return m.next.Delete(ctx, request) @@ -160,7 +161,7 @@ func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectR err := m.next.GetRange(req, stream) - m.metrics.AddRequestDuration("GetRange", time.Since(t), err == nil) + m.metrics.AddRequestDuration("GetRange", time.Since(t), err == nil, qos.IOTagFromContext(stream.Context())) return err } @@ -173,7 +174,7 @@ func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRa res, err := m.next.GetRangeHash(ctx, request) - m.metrics.AddRequestDuration("GetRangeHash", time.Since(t), err == nil) + m.metrics.AddRequestDuration("GetRangeHash", time.Since(t), err == nil, qos.IOTagFromContext(ctx)) return res, err } @@ -209,7 +210,7 @@ func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) { res, err := s.stream.CloseAndRecv(ctx) - s.metrics.AddRequestDuration("Put", time.Since(s.start), err == nil) + s.metrics.AddRequestDuration("Put", time.Since(s.start), err == nil, qos.IOTagFromContext(ctx)) return res, err } @@ -223,7 +224,7 @@ func (s patchStreamMetric) Send(ctx context.Context, req *object.PatchRequest) e func (s patchStreamMetric) CloseAndRecv(ctx context.Context) (*object.PatchResponse, error) { res, err := s.stream.CloseAndRecv(ctx) - s.metrics.AddRequestDuration("Patch", time.Since(s.start), err == nil) + s.metrics.AddRequestDuration("Patch", time.Since(s.start), err == nil, qos.IOTagFromContext(ctx)) return res, err } From 597bce7a879f2c5258b819014ca80b5815f6a168 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 5 Mar 2025 15:44:15 +0300 Subject: [PATCH 1770/1943] [#1653] treeSvc: Add operations by IO tag metric Signed-off-by: Dmitrii Stepanov --- internal/metrics/treeservice.go | 15 +++++++++++++++ pkg/services/tree/metrics.go | 2 ++ pkg/services/tree/service.go | 9 +++++++++ 3 files changed, 26 insertions(+) diff --git a/internal/metrics/treeservice.go b/internal/metrics/treeservice.go index 6702aa83c..e192c4398 100644 --- a/internal/metrics/treeservice.go +++ b/internal/metrics/treeservice.go @@ -12,12 +12,14 @@ type TreeMetricsRegister interface { AddReplicateTaskDuration(time.Duration, bool) AddReplicateWaitDuration(time.Duration, bool) AddSyncDuration(time.Duration, bool) + AddOperation(string, string) } type treeServiceMetrics struct { replicateTaskDuration *prometheus.HistogramVec replicateWaitDuration *prometheus.HistogramVec syncOpDuration *prometheus.HistogramVec + ioTagOpsCounter *prometheus.CounterVec } var _ TreeMetricsRegister = (*treeServiceMetrics)(nil) @@ -42,6 +44,12 @@ func newTreeServiceMetrics() *treeServiceMetrics { Name: "sync_duration_seconds", Help: "Duration of synchronization operations", }, []string{successLabel}), + ioTagOpsCounter: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: treeServiceSubsystem, + Name: "requests_total", + Help: "Count of requests for each IO tag", + }, []string{methodLabel, ioTagLabel}), } } @@ -62,3 +70,10 @@ func (m *treeServiceMetrics) AddSyncDuration(d time.Duration, success bool) { successLabel: strconv.FormatBool(success), }).Observe(d.Seconds()) } + +func (m *treeServiceMetrics) AddOperation(op string, ioTag string) { + m.ioTagOpsCounter.With(prometheus.Labels{ + ioTagLabel: ioTag, + methodLabel: op, + }).Inc() +} diff --git a/pkg/services/tree/metrics.go b/pkg/services/tree/metrics.go index 0f0e4ee57..07503f8c3 100644 --- a/pkg/services/tree/metrics.go +++ b/pkg/services/tree/metrics.go @@ -6,6 +6,7 @@ type MetricsRegister interface { AddReplicateTaskDuration(time.Duration, bool) AddReplicateWaitDuration(time.Duration, bool) AddSyncDuration(time.Duration, bool) + AddOperation(string, string) } type defaultMetricsRegister struct{} @@ -13,3 +14,4 @@ type defaultMetricsRegister struct{} func (defaultMetricsRegister) AddReplicateTaskDuration(time.Duration, bool) {} func (defaultMetricsRegister) AddReplicateWaitDuration(time.Duration, bool) {} func (defaultMetricsRegister) AddSyncDuration(time.Duration, bool) {} +func (defaultMetricsRegister) AddOperation(string, string) {} diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 2e9722e79..f9b7395e7 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -105,6 +105,7 @@ func (s *Service) Shutdown() { } func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error) { + defer s.metrics.AddOperation("Add", qos.IOTagFromContext(ctx)) if !s.initialSyncDone.Load() { return nil, ErrAlreadySyncing } @@ -148,6 +149,7 @@ func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error } func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByPathResponse, error) { + defer s.metrics.AddOperation("AddByPath", qos.IOTagFromContext(ctx)) if !s.initialSyncDone.Load() { return nil, ErrAlreadySyncing } @@ -203,6 +205,7 @@ func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByP } func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveResponse, error) { + defer s.metrics.AddOperation("Remove", qos.IOTagFromContext(ctx)) if !s.initialSyncDone.Load() { return nil, ErrAlreadySyncing } @@ -247,6 +250,7 @@ func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveRespon // Move applies client operation to the specified tree and pushes in queue // for replication on other nodes. func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, error) { + defer s.metrics.AddOperation("Move", qos.IOTagFromContext(ctx)) if !s.initialSyncDone.Load() { return nil, ErrAlreadySyncing } @@ -290,6 +294,7 @@ func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, er } func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) (*GetNodeByPathResponse, error) { + defer s.metrics.AddOperation("GetNodeByPath", qos.IOTagFromContext(ctx)) if !s.initialSyncDone.Load() { return nil, ErrAlreadySyncing } @@ -363,6 +368,7 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) } func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeServer) error { + defer s.metrics.AddOperation("GetSubTree", qos.IOTagFromContext(srv.Context())) if !s.initialSyncDone.Load() { return ErrAlreadySyncing } @@ -590,6 +596,7 @@ func sortByFilename(nodes []pilorama.NodeInfo, d GetSubTreeRequest_Body_Order_Di // Apply locally applies operation from the remote node to the tree. func (s *Service) Apply(ctx context.Context, req *ApplyRequest) (*ApplyResponse, error) { + defer s.metrics.AddOperation("Apply", qos.IOTagFromContext(ctx)) err := verifyMessage(req) if err != nil { return nil, err @@ -633,6 +640,7 @@ func (s *Service) Apply(ctx context.Context, req *ApplyRequest) (*ApplyResponse, } func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) error { + defer s.metrics.AddOperation("GetOpLog", qos.IOTagFromContext(srv.Context())) if !s.initialSyncDone.Load() { return ErrAlreadySyncing } @@ -697,6 +705,7 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) } func (s *Service) TreeList(ctx context.Context, req *TreeListRequest) (*TreeListResponse, error) { + defer s.metrics.AddOperation("TreeList", qos.IOTagFromContext(ctx)) if !s.initialSyncDone.Load() { return nil, ErrAlreadySyncing } From 2005fdda0982f7aa23d5938dfe7a40dd2a72fdb4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 10 Mar 2025 13:04:39 +0300 Subject: [PATCH 1771/1943] [#1667] shard: Drop shard pool After adding an ops limiter, shard's `put` pool is redundant. Signed-off-by: Dmitrii Stepanov --- .../internal/modules/storagecfg/config.go | 2 - cmd/frostfs-node/config.go | 3 - cmd/frostfs-node/config/configdir_test.go | 7 +- cmd/frostfs-node/config/engine/config.go | 16 --- cmd/frostfs-node/config/engine/config_test.go | 2 - config/example/node.env | 1 - config/example/node.json | 1 - config/example/node.yaml | 1 - docs/storage-node-configuration.md | 1 - pkg/local_object_storage/engine/control.go | 10 +- .../engine/control_test.go | 4 - pkg/local_object_storage/engine/engine.go | 18 +-- .../engine/engine_test.go | 1 - pkg/local_object_storage/engine/error_test.go | 1 - pkg/local_object_storage/engine/evacuate.go | 43 +++---- .../engine/evacuate_test.go | 1 - .../engine/inhume_test.go | 2 +- pkg/local_object_storage/engine/put.go | 106 ++++++++---------- pkg/local_object_storage/engine/shards.go | 20 ---- .../engine/shards_test.go | 2 - 20 files changed, 71 insertions(+), 171 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/config.go b/cmd/frostfs-adm/internal/modules/storagecfg/config.go index 77183fb49..67e3414c2 100644 --- a/cmd/frostfs-adm/internal/modules/storagecfg/config.go +++ b/cmd/frostfs-adm/internal/modules/storagecfg/config.go @@ -40,8 +40,6 @@ morph: - address: wss://{{.}}/ws{{end}} {{if not .Relay }} storage: - shard_pool_size: 15 # size of per-shard worker pools used for PUT operations - shard: default: # section with the default shard parameters metabase: diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 92aa827f2..e2fe23135 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -117,7 +117,6 @@ type applicationConfiguration struct { EngineCfg struct { errorThreshold uint32 - shardPoolSize uint32 shards []shardCfg lowMem bool } @@ -250,7 +249,6 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { // Storage Engine a.EngineCfg.errorThreshold = engineconfig.ShardErrorThreshold(c) - a.EngineCfg.shardPoolSize = engineconfig.ShardPoolSize(c) a.EngineCfg.lowMem = engineconfig.EngineLowMemoryConsumption(c) return engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error { return a.updateShardConfig(c, sc) }) @@ -893,7 +891,6 @@ func (c *cfg) engineOpts() []engine.Option { var opts []engine.Option opts = append(opts, - engine.WithShardPoolSize(c.EngineCfg.shardPoolSize), engine.WithErrorThreshold(c.EngineCfg.errorThreshold), engine.WithLogger(c.log), engine.WithLowMemoryConsumption(c.EngineCfg.lowMem), diff --git a/cmd/frostfs-node/config/configdir_test.go b/cmd/frostfs-node/config/configdir_test.go index 35dae97d9..ee9d4268b 100644 --- a/cmd/frostfs-node/config/configdir_test.go +++ b/cmd/frostfs-node/config/configdir_test.go @@ -12,13 +12,10 @@ import ( func TestConfigDir(t *testing.T) { dir := t.TempDir() - cfgFileName0 := path.Join(dir, "cfg_00.json") - cfgFileName1 := path.Join(dir, "cfg_01.yml") + cfgFileName := path.Join(dir, "cfg_01.yml") - require.NoError(t, os.WriteFile(cfgFileName0, []byte(`{"storage":{"shard_pool_size":15}}`), 0o777)) - require.NoError(t, os.WriteFile(cfgFileName1, []byte("logger:\n level: debug"), 0o777)) + require.NoError(t, os.WriteFile(cfgFileName, []byte("logger:\n level: debug"), 0o777)) c := New("", dir, "") require.Equal(t, "debug", cast.ToString(c.Sub("logger").Value("level"))) - require.EqualValues(t, 15, cast.ToUint32(c.Sub("storage").Value("shard_pool_size"))) } diff --git a/cmd/frostfs-node/config/engine/config.go b/cmd/frostfs-node/config/engine/config.go index e5735e88b..7994e7809 100644 --- a/cmd/frostfs-node/config/engine/config.go +++ b/cmd/frostfs-node/config/engine/config.go @@ -11,10 +11,6 @@ import ( const ( subsection = "storage" - - // ShardPoolSizeDefault is a default value of routine pool size per-shard to - // process object PUT operations in a storage engine. - ShardPoolSizeDefault = 20 ) // ErrNoShardConfigured is returned when at least 1 shard is required but none are found. @@ -65,18 +61,6 @@ func IterateShards(c *config.Config, required bool, f func(*shardconfig.Config) return nil } -// ShardPoolSize returns the value of "shard_pool_size" config parameter from "storage" section. -// -// Returns ShardPoolSizeDefault if the value is not a positive number. -func ShardPoolSize(c *config.Config) uint32 { - v := config.Uint32Safe(c.Sub(subsection), "shard_pool_size") - if v > 0 { - return v - } - - return ShardPoolSizeDefault -} - // ShardErrorThreshold returns the value of "shard_ro_error_threshold" config parameter from "storage" section. // // Returns 0 if the the value is missing. diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index b912b5d7d..eaf2a294e 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -54,7 +54,6 @@ func TestEngineSection(t *testing.T) { require.False(t, handlerCalled) require.EqualValues(t, 0, engineconfig.ShardErrorThreshold(empty)) - require.EqualValues(t, engineconfig.ShardPoolSizeDefault, engineconfig.ShardPoolSize(empty)) require.EqualValues(t, mode.ReadWrite, shardconfig.From(empty).Mode()) }) @@ -64,7 +63,6 @@ func TestEngineSection(t *testing.T) { num := 0 require.EqualValues(t, 100, engineconfig.ShardErrorThreshold(c)) - require.EqualValues(t, 15, engineconfig.ShardPoolSize(c)) err := engineconfig.IterateShards(c, true, func(sc *shardconfig.Config) error { defer func() { diff --git a/config/example/node.env b/config/example/node.env index 9bd645344..010b6840c 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -97,7 +97,6 @@ FROSTFS_RPC_LIMITS_1_METHODS="/neo.fs.v2.object.ObjectService/Get" FROSTFS_RPC_LIMITS_1_MAX_OPS=10000 # Storage engine section -FROSTFS_STORAGE_SHARD_POOL_SIZE=15 FROSTFS_STORAGE_SHARD_RO_ERROR_THRESHOLD=100 ## 0 shard ### Flag to refill Metabase from BlobStor diff --git a/config/example/node.json b/config/example/node.json index 6b799b318..b26c35d2c 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -158,7 +158,6 @@ ] }, "storage": { - "shard_pool_size": 15, "shard_ro_error_threshold": 100, "shard": { "0": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 2552a419c..58b687d5c 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -135,7 +135,6 @@ rpc: storage: # note: shard configuration can be omitted for relay node (see `node.relay`) - shard_pool_size: 15 # size of per-shard worker pools used for PUT operations shard_ro_error_threshold: 100 # amount of errors to occur before shard is made read-only (default: 0, ignore errors) shard: diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 271cc6532..51f0a9669 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -170,7 +170,6 @@ Local storage engine configuration. | Parameter | Type | Default value | Description | |----------------------------|-----------------------------------|---------------|------------------------------------------------------------------------------------------------------------------| -| `shard_pool_size` | `int` | `20` | Pool size for shard workers. Limits the amount of concurrent `PUT` operations on each shard. | | `shard_ro_error_threshold` | `int` | `0` | Maximum amount of storage errors to encounter before shard automatically moves to `Degraded` or `ReadOnly` mode. | | `low_mem` | `bool` | `false` | Reduce memory consumption by reducing performance. | | `shard` | [Shard config](#shard-subsection) | | Configuration for separate shards. | diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 6a416cfd9..7caa515d4 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -153,16 +153,10 @@ func (e *StorageEngine) Close(ctx context.Context) error { } // closes all shards. Never returns an error, shard errors are logged. -func (e *StorageEngine) close(ctx context.Context, releasePools bool) error { +func (e *StorageEngine) close(ctx context.Context) error { e.mtx.RLock() defer e.mtx.RUnlock() - if releasePools { - for _, p := range e.shardPools { - p.Release() - } - } - for id, sh := range e.shards { if err := sh.Close(ctx); err != nil { e.log.Debug(ctx, logs.EngineCouldNotCloseShard, @@ -213,7 +207,7 @@ func (e *StorageEngine) setBlockExecErr(ctx context.Context, err error) error { return e.open(ctx) } } else if prevErr == nil { // ok -> block - return e.close(ctx, errors.Is(err, errClosed)) + return e.close(ctx) } // otherwise do nothing diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index c9efc312c..a0e658aeb 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -245,7 +245,6 @@ func TestReload(t *testing.T) { // no new paths => no new shards require.Equal(t, shardNum, len(e.shards)) - require.Equal(t, shardNum, len(e.shardPools)) newMeta := filepath.Join(addPath, fmt.Sprintf("%d.metabase", shardNum)) @@ -257,7 +256,6 @@ func TestReload(t *testing.T) { require.NoError(t, e.Reload(context.Background(), rcfg)) require.Equal(t, shardNum+1, len(e.shards)) - require.Equal(t, shardNum+1, len(e.shardPools)) require.NoError(t, e.Close(context.Background())) }) @@ -277,7 +275,6 @@ func TestReload(t *testing.T) { // removed one require.Equal(t, shardNum-1, len(e.shards)) - require.Equal(t, shardNum-1, len(e.shardPools)) require.NoError(t, e.Close(context.Background())) }) @@ -311,7 +308,6 @@ func engineWithShards(t *testing.T, path string, num int) (*StorageEngine, []str } require.Equal(t, num, len(e.shards)) - require.Equal(t, num, len(e.shardPools)) return e, currShards } diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index e13252b82..a915c9bd6 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -12,7 +12,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -29,8 +28,6 @@ type StorageEngine struct { shards map[string]hashedShard - shardPools map[string]util.WorkerPool - closeCh chan struct{} setModeCh chan setModeRequest wg sync.WaitGroup @@ -193,8 +190,6 @@ type cfg struct { metrics MetricRegister - shardPoolSize uint32 - lowMem bool containerSource atomic.Pointer[containerSource] @@ -202,9 +197,8 @@ type cfg struct { func defaultCfg() *cfg { res := &cfg{ - log: logger.NewLoggerWrapper(zap.L()), - shardPoolSize: 20, - metrics: noopMetrics{}, + log: logger.NewLoggerWrapper(zap.L()), + metrics: noopMetrics{}, } res.containerSource.Store(&containerSource{}) return res @@ -221,7 +215,6 @@ func New(opts ...Option) *StorageEngine { return &StorageEngine{ cfg: c, shards: make(map[string]hashedShard), - shardPools: make(map[string]util.WorkerPool), closeCh: make(chan struct{}), setModeCh: make(chan setModeRequest), evacuateLimiter: &evacuationLimiter{}, @@ -241,13 +234,6 @@ func WithMetrics(v MetricRegister) Option { } } -// WithShardPoolSize returns option to specify size of worker pool for each shard. -func WithShardPoolSize(sz uint32) Option { - return func(c *cfg) { - c.shardPoolSize = sz - } -} - // WithErrorThreshold returns an option to specify size amount of errors after which // shard is moved to read-only mode. func WithErrorThreshold(sz uint32) Option { diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 3f9196128..6ef3846ee 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -57,7 +57,6 @@ func (te *testEngineWrapper) setShardsNumOpts( te.shardIDs[i] = shard.ID() } require.Len(t, te.engine.shards, num) - require.Len(t, te.engine.shardPools, num) return te } diff --git a/pkg/local_object_storage/engine/error_test.go b/pkg/local_object_storage/engine/error_test.go index d68a7e826..57029dd5f 100644 --- a/pkg/local_object_storage/engine/error_test.go +++ b/pkg/local_object_storage/engine/error_test.go @@ -46,7 +46,6 @@ func newEngineWithErrorThreshold(t testing.TB, dir string, errThreshold uint32) var testShards [2]*testShard te := testNewEngine(t, - WithShardPoolSize(1), WithErrorThreshold(errThreshold), ). setShardsNumOpts(t, 2, func(id int) []shard.Option { diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 27eaea768..c08dfbf03 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -15,7 +15,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" @@ -201,11 +200,6 @@ func (p *EvacuateShardRes) DeepCopy() *EvacuateShardRes { return res } -type pooledShard struct { - hashedShard - pool util.WorkerPool -} - var errMustHaveTwoShards = errors.New("must have at least 1 spare shard") // Evacuate moves data from one shard to the others. @@ -252,7 +246,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) erro } var mtx sync.RWMutex - copyShards := func() []pooledShard { + copyShards := func() []hashedShard { mtx.RLock() defer mtx.RUnlock() t := slices.Clone(shards) @@ -266,7 +260,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) erro } func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, + shards func() []hashedShard, shardsToEvacuate map[string]*shard.Shard, ) error { var err error ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShards", @@ -388,7 +382,7 @@ func (e *StorageEngine) getTotals(ctx context.Context, prm EvacuateShardPrm, sha } func (e *StorageEngine) evacuateShard(ctx context.Context, cancel context.CancelCauseFunc, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, + shards func() []hashedShard, shardsToEvacuate map[string]*shard.Shard, egContainer *errgroup.Group, egObject *errgroup.Group, ) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateShard", @@ -412,7 +406,7 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, cancel context.Cancel } func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context.CancelCauseFunc, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, - shards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, + shards func() []hashedShard, shardsToEvacuate map[string]*shard.Shard, egContainer *errgroup.Group, egObject *errgroup.Group, ) error { sh := shardsToEvacuate[shardID] @@ -485,7 +479,7 @@ func (e *StorageEngine) evacuateShardObjects(ctx context.Context, cancel context } func (e *StorageEngine) evacuateShardTrees(ctx context.Context, shardID string, prm EvacuateShardPrm, res *EvacuateShardRes, - getShards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, + getShards func() []hashedShard, shardsToEvacuate map[string]*shard.Shard, ) error { sh := shardsToEvacuate[shardID] shards := getShards() @@ -515,7 +509,7 @@ func (e *StorageEngine) evacuateShardTrees(ctx context.Context, shardID string, } func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, trees []pilorama.ContainerIDTreeID, - prm EvacuateShardPrm, res *EvacuateShardRes, shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, + prm EvacuateShardPrm, res *EvacuateShardRes, shards []hashedShard, shardsToEvacuate map[string]*shard.Shard, ) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateTrees", trace.WithAttributes( @@ -583,7 +577,7 @@ func (e *StorageEngine) evacuateTreeToOtherNode(ctx context.Context, sh *shard.S } func (e *StorageEngine) tryEvacuateTreeLocal(ctx context.Context, sh *shard.Shard, tree pilorama.ContainerIDTreeID, - prm EvacuateShardPrm, shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, + prm EvacuateShardPrm, shards []hashedShard, shardsToEvacuate map[string]*shard.Shard, ) (bool, string, error) { target, found, err := e.findShardToEvacuateTree(ctx, tree, shards, shardsToEvacuate) if err != nil { @@ -653,15 +647,15 @@ func (e *StorageEngine) tryEvacuateTreeLocal(ctx context.Context, sh *shard.Shar // findShardToEvacuateTree returns first shard according HRW or first shard with tree exists. func (e *StorageEngine) findShardToEvacuateTree(ctx context.Context, tree pilorama.ContainerIDTreeID, - shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, -) (pooledShard, bool, error) { + shards []hashedShard, shardsToEvacuate map[string]*shard.Shard, +) (hashedShard, bool, error) { hrw.SortHasherSliceByValue(shards, hrw.StringHash(tree.CID.EncodeToString())) - var result pooledShard + var result hashedShard var found bool for _, target := range shards { select { case <-ctx.Done(): - return pooledShard{}, false, ctx.Err() + return hashedShard{}, false, ctx.Err() default: } @@ -689,7 +683,7 @@ func (e *StorageEngine) findShardToEvacuateTree(ctx context.Context, tree pilora return result, found, nil } -func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) ([]pooledShard, error) { +func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) ([]hashedShard, error) { e.mtx.RLock() defer e.mtx.RUnlock() @@ -719,18 +713,15 @@ func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) // We must have all shards, to have correct information about their // indexes in a sorted slice and set appropriate marks in the metabase. // Evacuated shard is skipped during put. - shards := make([]pooledShard, 0, len(e.shards)) + shards := make([]hashedShard, 0, len(e.shards)) for id := range e.shards { - shards = append(shards, pooledShard{ - hashedShard: e.shards[id], - pool: e.shardPools[id], - }) + shards = append(shards, e.shards[id]) } return shards, nil } func (e *StorageEngine) evacuateObject(ctx context.Context, shardID string, objInfo *object.Info, prm EvacuateShardPrm, res *EvacuateShardRes, - getShards func() []pooledShard, shardsToEvacuate map[string]*shard.Shard, cnr containerSDK.Container, + getShards func() []hashedShard, shardsToEvacuate map[string]*shard.Shard, cnr containerSDK.Container, ) error { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.evacuateObjects") defer span.End() @@ -800,7 +791,7 @@ func (e *StorageEngine) isNotRepOne(c *container.Container) bool { } func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Address, object *objectSDK.Object, sh *shard.Shard, - shards []pooledShard, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes, cnr containerSDK.Container, + shards []hashedShard, shardsToEvacuate map[string]*shard.Shard, res *EvacuateShardRes, cnr containerSDK.Container, ) (bool, error) { hrw.SortHasherSliceByValue(shards, hrw.StringHash(addr.EncodeToString())) for j := range shards { @@ -813,7 +804,7 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add if _, ok := shardsToEvacuate[shards[j].ID().String()]; ok { continue } - switch e.putToShard(ctx, shards[j].hashedShard, shards[j].pool, addr, object, container.IsIndexedContainer(cnr)).status { + switch e.putToShard(ctx, shards[j], addr, object, container.IsIndexedContainer(cnr)).status { case putToShardSuccess: res.objEvacuated.Add(1) e.log.Debug(ctx, logs.EngineObjectIsMovedToAnotherShard, diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 45c4b696b..ec7923297 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -196,7 +196,6 @@ func TestEvacuateShardObjects(t *testing.T) { e.mtx.Lock() delete(e.shards, evacuateShardID) - delete(e.shardPools, evacuateShardID) e.mtx.Unlock() checkHasObjects(t) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 8c5d28b15..10cebfb52 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -205,7 +205,7 @@ func BenchmarkInhumeMultipart(b *testing.B) { func benchmarkInhumeMultipart(b *testing.B, numShards, numObjects int) { b.StopTimer() - engine := testNewEngine(b, WithShardPoolSize(uint32(numObjects))). + engine := testNewEngine(b). setShardsNum(b, numShards).prepare(b).engine defer func() { require.NoError(b, engine.Close(context.Background())) }() diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index 64288a511..b348d13a2 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -9,7 +9,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -99,13 +98,13 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { var shRes putToShardRes e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { e.mtx.RLock() - pool, ok := e.shardPools[sh.ID().String()] + _, ok := e.shards[sh.ID().String()] e.mtx.RUnlock() if !ok { // Shard was concurrently removed, skip. return false } - shRes = e.putToShard(ctx, sh, pool, addr, prm.Object, prm.IsIndexedContainer) + shRes = e.putToShard(ctx, sh, addr, prm.Object, prm.IsIndexedContainer) return shRes.status != putToShardUnknown }) switch shRes.status { @@ -122,70 +121,59 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { // putToShard puts object to sh. // Return putToShardStatus and error if it is necessary to propagate an error upper. -func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, pool util.WorkerPool, +func (e *StorageEngine) putToShard(ctx context.Context, sh hashedShard, addr oid.Address, obj *objectSDK.Object, isIndexedContainer bool, ) (res putToShardRes) { - exitCh := make(chan struct{}) + var existPrm shard.ExistsPrm + existPrm.Address = addr - if err := pool.Submit(func() { - defer close(exitCh) - - var existPrm shard.ExistsPrm - existPrm.Address = addr - - exists, err := sh.Exists(ctx, existPrm) - if err != nil { - if shard.IsErrObjectExpired(err) { - // object is already found but - // expired => do nothing with it - res.status = putToShardExists - } else { - e.log.Warn(ctx, logs.EngineCouldNotCheckObjectExistence, - zap.Stringer("shard_id", sh.ID()), - zap.Error(err)) - } - - return // this is not ErrAlreadyRemoved error so we can go to the next shard - } - - if exists.Exists() { + exists, err := sh.Exists(ctx, existPrm) + if err != nil { + if shard.IsErrObjectExpired(err) { + // object is already found but + // expired => do nothing with it res.status = putToShardExists - return + } else { + e.log.Warn(ctx, logs.EngineCouldNotCheckObjectExistence, + zap.Stringer("shard_id", sh.ID()), + zap.Error(err)) } - var putPrm shard.PutPrm - putPrm.SetObject(obj) - putPrm.SetIndexAttributes(isIndexedContainer) - - _, err = sh.Put(ctx, putPrm) - if err != nil { - if errors.Is(err, shard.ErrReadOnlyMode) || errors.Is(err, blobstor.ErrNoPlaceFound) || - errors.Is(err, common.ErrReadOnly) || errors.Is(err, common.ErrNoSpace) { - e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, - zap.Stringer("shard_id", sh.ID()), - zap.Error(err)) - return - } - if client.IsErrObjectAlreadyRemoved(err) { - e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, - zap.Stringer("shard_id", sh.ID()), - zap.Error(err)) - res.status = putToShardRemoved - res.err = err - return - } - - e.reportShardError(ctx, sh, "could not put object to shard", err, zap.Stringer("address", addr)) - return - } - - res.status = putToShardSuccess - }); err != nil { - e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, zap.Error(err)) - close(exitCh) + return // this is not ErrAlreadyRemoved error so we can go to the next shard } - <-exitCh + if exists.Exists() { + res.status = putToShardExists + return + } + + var putPrm shard.PutPrm + putPrm.SetObject(obj) + putPrm.SetIndexAttributes(isIndexedContainer) + + _, err = sh.Put(ctx, putPrm) + if err != nil { + if errors.Is(err, shard.ErrReadOnlyMode) || errors.Is(err, blobstor.ErrNoPlaceFound) || + errors.Is(err, common.ErrReadOnly) || errors.Is(err, common.ErrNoSpace) { + e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, + zap.Stringer("shard_id", sh.ID()), + zap.Error(err)) + return + } + if client.IsErrObjectAlreadyRemoved(err) { + e.log.Warn(ctx, logs.EngineCouldNotPutObjectToShard, + zap.Stringer("shard_id", sh.ID()), + zap.Error(err)) + res.status = putToShardRemoved + res.err = err + return + } + + e.reportShardError(ctx, sh, "could not put object to shard", err, zap.Stringer("address", addr)) + return + } + + res.status = putToShardSuccess return } diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 28f0287bc..a38c85151 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -17,7 +17,6 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/hrw" "github.com/google/uuid" - "github.com/panjf2000/ants/v2" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) @@ -181,11 +180,6 @@ func (e *StorageEngine) addShard(sh *shard.Shard) error { e.mtx.Lock() defer e.mtx.Unlock() - pool, err := ants.NewPool(int(e.shardPoolSize), ants.WithNonblocking(true)) - if err != nil { - return fmt.Errorf("create pool: %w", err) - } - strID := sh.ID().String() if _, ok := e.shards[strID]; ok { return fmt.Errorf("shard with id %s was already added", strID) @@ -199,8 +193,6 @@ func (e *StorageEngine) addShard(sh *shard.Shard) error { hash: hrw.StringHash(strID), } - e.shardPools[strID] = pool - return nil } @@ -225,12 +217,6 @@ func (e *StorageEngine) removeShards(ctx context.Context, ids ...string) { ss = append(ss, sh) delete(e.shards, id) - pool, ok := e.shardPools[id] - if ok { - pool.Release() - delete(e.shardPools, id) - } - e.log.Info(ctx, logs.EngineShardHasBeenRemoved, zap.String("id", id)) } @@ -429,12 +415,6 @@ func (e *StorageEngine) deleteShards(ctx context.Context, ids []*shard.ID) ([]ha delete(e.shards, idStr) - pool, ok := e.shardPools[idStr] - if ok { - pool.Release() - delete(e.shardPools, idStr) - } - e.log.Info(ctx, logs.EngineShardHasBeenRemoved, zap.String("id", idStr)) } diff --git a/pkg/local_object_storage/engine/shards_test.go b/pkg/local_object_storage/engine/shards_test.go index 0bbc7563c..3aa9629b0 100644 --- a/pkg/local_object_storage/engine/shards_test.go +++ b/pkg/local_object_storage/engine/shards_test.go @@ -17,7 +17,6 @@ func TestRemoveShard(t *testing.T) { e, ids := te.engine, te.shardIDs defer func() { require.NoError(t, e.Close(context.Background())) }() - require.Equal(t, numOfShards, len(e.shardPools)) require.Equal(t, numOfShards, len(e.shards)) removedNum := numOfShards / 2 @@ -37,7 +36,6 @@ func TestRemoveShard(t *testing.T) { } } - require.Equal(t, numOfShards-removedNum, len(e.shardPools)) require.Equal(t, numOfShards-removedNum, len(e.shards)) for id, removed := range mSh { From 737788b35f173fb9cd2838fb53858d8c8b22ec25 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 12 Mar 2025 09:21:15 +0300 Subject: [PATCH 1772/1943] [#1669] go.mod: Bump frostfs-qos version Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9d0988bcd..69273fda2 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 - git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3 + git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250310135838-3e7ca9403529 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 diff --git a/go.sum b/go.sum index 3ec679ee7..a8f7216a5 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 h1:Mxw1c/8t96vFIUOffl28lFaHKi413oCBfLMGJmF9cFA= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3 h1:QnAt5b2R6+hQthMOIn5ECfLAlVD8IAE5JRm1NCCOmuE= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250227072915-25102d1e1aa3/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250310135838-3e7ca9403529 h1:CBreXSxGoYJAdZ1QdJPsDs1UCXGF5psinII0lxtohsc= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250310135838-3e7ca9403529/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9 h1:svCl6NDAPZ/KuQPjdVKo74RkCIANesxUPM45zQZDhSw= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= From bcc84c85a0bdc9095f511ba2d03094723b48c7dd Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 13 Mar 2025 08:57:35 +0300 Subject: [PATCH 1773/1943] [#1671] Replace `interface{}` with `any` gopatch: ``` @@ @@ -interface{} +any ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/evacuate_test.go | 8 ++++---- pkg/morph/client/client.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index ec7923297..bd5222b78 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -404,8 +404,8 @@ func TestEvacuateSingleProcess(t *testing.T) { require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) require.NoError(t, e.shards[ids[1].String()].SetMode(context.Background(), mode.ReadOnly)) - blocker := make(chan interface{}) - running := make(chan interface{}) + blocker := make(chan any) + running := make(chan any) var prm EvacuateShardPrm prm.ShardID = ids[1:2] @@ -446,8 +446,8 @@ func TestEvacuateObjectsAsync(t *testing.T) { require.NoError(t, e.shards[ids[0].String()].SetMode(context.Background(), mode.ReadOnly)) require.NoError(t, e.shards[ids[1].String()].SetMode(context.Background(), mode.ReadOnly)) - blocker := make(chan interface{}) - running := make(chan interface{}) + blocker := make(chan any) + running := make(chan any) var prm EvacuateShardPrm prm.ShardID = ids[1:2] diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 10ded5142..19349ccd5 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -213,7 +213,7 @@ func (c *Client) Invoke(ctx context.Context, contract util.Uint160, fee fixedn.F // If the remote neo-go node does not support sessions, `unwrap.ErrNoSessionID` is returned. // batchSize is the number of items to prefetch: if the number of items in the iterator is less than batchSize, no session will be created. // The default batchSize is 100, the default limit from neo-go. -func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int, contract util.Uint160, method string, args ...interface{}) error { +func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int, contract util.Uint160, method string, args ...any) error { start := time.Now() success := false defer func() { From d66bffb1919c8b6caa253b84b1f95f38afd639a0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 13 Mar 2025 09:01:42 +0300 Subject: [PATCH 1774/1943] [#1671] Use max builtin where possible gopatcH: ``` @@ var d, a expression @@ -if d < a { - d = a -} -return d +return max(d, a) @@ var d, a expression @@ -if d <= a { - d = a -} -return d +return max(d, a) ``` Signed-off-by: Evgenii Stratonikov --- .../config/engine/shard/boltdb/boltdb.go | 15 +++------------ .../config/engine/shard/pilorama/config.go | 10 ++-------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go b/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go index a51308b5b..b564d36f8 100644 --- a/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go +++ b/cmd/frostfs-node/config/engine/shard/boltdb/boltdb.go @@ -37,10 +37,7 @@ func (x *Config) Perm() fs.FileMode { // Returns 0 if the value is not a positive number. func (x *Config) MaxBatchDelay() time.Duration { d := config.DurationSafe((*config.Config)(x), "max_batch_delay") - if d < 0 { - d = 0 - } - return d + return max(d, 0) } // MaxBatchSize returns the value of "max_batch_size" config parameter. @@ -48,10 +45,7 @@ func (x *Config) MaxBatchDelay() time.Duration { // Returns 0 if the value is not a positive number. func (x *Config) MaxBatchSize() int { s := int(config.IntSafe((*config.Config)(x), "max_batch_size")) - if s < 0 { - s = 0 - } - return s + return max(s, 0) } // NoSync returns the value of "no_sync" config parameter. @@ -66,8 +60,5 @@ func (x *Config) NoSync() bool { // Returns 0 if the value is not a positive number. func (x *Config) PageSize() int { s := int(config.SizeInBytesSafe((*config.Config)(x), "page_size")) - if s < 0 { - s = 0 - } - return s + return max(s, 0) } diff --git a/cmd/frostfs-node/config/engine/shard/pilorama/config.go b/cmd/frostfs-node/config/engine/shard/pilorama/config.go index 28671ca55..5d4e8f408 100644 --- a/cmd/frostfs-node/config/engine/shard/pilorama/config.go +++ b/cmd/frostfs-node/config/engine/shard/pilorama/config.go @@ -52,10 +52,7 @@ func (x *Config) NoSync() bool { // Returns 0 if the value is not a positive number. func (x *Config) MaxBatchDelay() time.Duration { d := config.DurationSafe((*config.Config)(x), "max_batch_delay") - if d <= 0 { - d = 0 - } - return d + return max(d, 0) } // MaxBatchSize returns the value of "max_batch_size" config parameter. @@ -63,8 +60,5 @@ func (x *Config) MaxBatchDelay() time.Duration { // Returns 0 if the value is not a positive number. func (x *Config) MaxBatchSize() int { s := int(config.IntSafe((*config.Config)(x), "max_batch_size")) - if s <= 0 { - s = 0 - } - return s + return max(s, 0) } From 40536d8a0658bf374120007b13bce712422f7ba1 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 13 Mar 2025 09:09:14 +0300 Subject: [PATCH 1775/1943] [#1671] Use `fmt.Appendf` where warranted Fix gopls warnings: ``` cmd/frostfs-adm/internal/modules/morph/config/config.go:68:20-64: Replace []byte(fmt.Sprintf...) with fmt.Appendf ```` gopatch: ``` @@ var f expression @@ -[]byte(fmt.Sprintf(f, ...)) +fmt.Appendf(nil, f, ...) ``` Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/config/config.go | 6 +++--- .../internal/modules/morph/contract/dump_hashes.go | 4 ++-- cmd/frostfs-adm/internal/modules/morph/policy/policy.go | 6 +++--- cmd/frostfs-cli/modules/control/list_targets.go | 2 +- pkg/local_object_storage/metabase/reset_test.go | 2 +- scripts/populate-metabase/internal/populate.go | 7 ++----- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index 65ccc9f9f..f64cb4817 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -65,14 +65,14 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { nbuf := make([]byte, 8) copy(nbuf[:], v) n := binary.LittleEndian.Uint64(nbuf) - _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%d (int)\n", k, n))) + _, _ = tw.Write(fmt.Appendf(nil, "%s:\t%d (int)\n", k, n)) case netmap.HomomorphicHashingDisabledKey, netmap.MaintenanceModeAllowedConfig: if len(v) == 0 || len(v) > 1 { return helper.InvalidConfigValueErr(k) } - _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%t (bool)\n", k, v[0] == 1))) + _, _ = tw.Write(fmt.Appendf(nil, "%s:\t%t (bool)\n", k, v[0] == 1)) default: - _, _ = tw.Write([]byte(fmt.Sprintf("%s:\t%s (hex)\n", k, hex.EncodeToString(v)))) + _, _ = tw.Write(fmt.Appendf(nil, "%s:\t%s (hex)\n", k, hex.EncodeToString(v))) } } diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index 437e2480d..fb7e4ff62 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -219,8 +219,8 @@ func printContractInfo(cmd *cobra.Command, infos []contractDumpInfo) { if info.version == "" { info.version = "unknown" } - _, _ = tw.Write([]byte(fmt.Sprintf("%s\t(%s):\t%s\n", - info.name, info.version, info.hash.StringLE()))) + _, _ = tw.Write(fmt.Appendf(nil, "%s\t(%s):\t%s\n", + info.name, info.version, info.hash.StringLE())) } _ = tw.Flush() diff --git a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go index 686a244f0..f2932e87c 100644 --- a/cmd/frostfs-adm/internal/modules/morph/policy/policy.go +++ b/cmd/frostfs-adm/internal/modules/morph/policy/policy.go @@ -80,9 +80,9 @@ func dumpPolicyCmd(cmd *cobra.Command, _ []string) error { buf := bytes.NewBuffer(nil) tw := tabwriter.NewWriter(buf, 0, 2, 2, ' ', 0) - _, _ = tw.Write([]byte(fmt.Sprintf("Execution Fee Factor:\t%d (int)\n", execFee))) - _, _ = tw.Write([]byte(fmt.Sprintf("Fee Per Byte:\t%d (int)\n", feePerByte))) - _, _ = tw.Write([]byte(fmt.Sprintf("Storage Price:\t%d (int)\n", storagePrice))) + _, _ = tw.Write(fmt.Appendf(nil, "Execution Fee Factor:\t%d (int)\n", execFee)) + _, _ = tw.Write(fmt.Appendf(nil, "Fee Per Byte:\t%d (int)\n", feePerByte)) + _, _ = tw.Write(fmt.Appendf(nil, "Storage Price:\t%d (int)\n", storagePrice)) _ = tw.Flush() cmd.Print(buf.String()) diff --git a/cmd/frostfs-cli/modules/control/list_targets.go b/cmd/frostfs-cli/modules/control/list_targets.go index 8bd2dc9cd..3142d02e7 100644 --- a/cmd/frostfs-cli/modules/control/list_targets.go +++ b/cmd/frostfs-cli/modules/control/list_targets.go @@ -62,7 +62,7 @@ func listTargets(cmd *cobra.Command, _ []string) { tw := tabwriter.NewWriter(buf, 0, 2, 2, ' ', 0) _, _ = tw.Write([]byte("#\tName\tType\n")) for i, t := range targets { - _, _ = tw.Write([]byte(fmt.Sprintf("%s\t%s\t%s\n", strconv.Itoa(i), t.GetName(), t.GetType()))) + _, _ = tw.Write(fmt.Appendf(nil, "%s\t%s\t%s\n", strconv.Itoa(i), t.GetName(), t.GetType())) } _ = tw.Flush() cmd.Print(buf.String()) diff --git a/pkg/local_object_storage/metabase/reset_test.go b/pkg/local_object_storage/metabase/reset_test.go index 45faecc13..5f0956f0b 100644 --- a/pkg/local_object_storage/metabase/reset_test.go +++ b/pkg/local_object_storage/metabase/reset_test.go @@ -37,7 +37,7 @@ func TestResetDropsContainerBuckets(t *testing.T) { for idx := range 100 { var putPrm PutPrm putPrm.SetObject(testutil.GenerateObject()) - putPrm.SetStorageID([]byte(fmt.Sprintf("0/%d", idx))) + putPrm.SetStorageID(fmt.Appendf(nil, "0/%d", idx)) _, err := db.Put(context.Background(), putPrm) require.NoError(t, err) } diff --git a/scripts/populate-metabase/internal/populate.go b/scripts/populate-metabase/internal/populate.go index 4da23a295..fafe61eaa 100644 --- a/scripts/populate-metabase/internal/populate.go +++ b/scripts/populate-metabase/internal/populate.go @@ -31,13 +31,10 @@ func PopulateWithObjects( for range count { obj := factory() - - id := []byte(fmt.Sprintf( - "%c/%c/%c", + id := fmt.Appendf(nil, "%c/%c/%c", digits[rand.Int()%len(digits)], digits[rand.Int()%len(digits)], - digits[rand.Int()%len(digits)], - )) + digits[rand.Int()%len(digits)]) prm := meta.PutPrm{} prm.SetObject(obj) From 155d3ddb6e9d8400bdfc43cd2606ac577b6b8945 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 13 Mar 2025 09:14:14 +0300 Subject: [PATCH 1776/1943] [#1671] Use `min` builtin where possible Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/pilorama/forest.go | 5 +---- pkg/morph/client/client.go | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index 92183716c..ef284a727 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -205,10 +205,7 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI r := mergeNodeInfos(res) for i := range r { if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > *start { - finish := i + count - if len(res) < finish { - finish = len(res) - } + finish := min(len(res), i+count) last := string(findAttr(r[finish-1].Meta, AttributeFilename)) return r[i:finish], &last, nil } diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 19349ccd5..e63d926e0 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -262,10 +262,7 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int }() // Batch size for TraverseIterator() can restricted on the server-side. - traverseBatchSize := batchSize - if invoker.DefaultIteratorResultItems < traverseBatchSize { - traverseBatchSize = invoker.DefaultIteratorResultItems - } + traverseBatchSize := min(batchSize, invoker.DefaultIteratorResultItems) for { items, err := c.rpcActor.TraverseIterator(sid, &r, traverseBatchSize) if err != nil { From 460e5cbccf8fa62134ff8bbc73f886ee5ceaeb87 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 13 Mar 2025 09:24:44 +0300 Subject: [PATCH 1777/1943] [#1671] Use `slices.Delete()` where possible gopatch is missing for this one, because https://github.com/uber-go/gopatch/issues/179 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-lens/internal/tui/input.go | 4 +++- .../blobstor/internal/blobstortest/iterate.go | 3 ++- pkg/local_object_storage/metabase/lock.go | 3 ++- pkg/services/object/delete/exec.go | 3 ++- pkg/services/object/search/util.go | 3 ++- pkg/services/object/util/placement.go | 3 ++- pkg/services/object_manager/placement/traverser.go | 4 ++-- 7 files changed, 15 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-lens/internal/tui/input.go b/cmd/frostfs-lens/internal/tui/input.go index 4fdf97119..90729c119 100644 --- a/cmd/frostfs-lens/internal/tui/input.go +++ b/cmd/frostfs-lens/internal/tui/input.go @@ -1,6 +1,8 @@ package tui import ( + "slices" + "github.com/gdamore/tcell/v2" "github.com/rivo/tview" ) @@ -26,7 +28,7 @@ func (f *InputFieldWithHistory) AddToHistory(s string) { // Used history data for search prompt, so just make that data recent. if f.historyPointer != len(f.history) && s == f.history[f.historyPointer] { - f.history = append(f.history[:f.historyPointer], f.history[f.historyPointer+1:]...) + f.history = slices.Delete(f.history, f.historyPointer, f.historyPointer+1) f.history = append(f.history, s) } diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index 36b2c33f8..c11d0888b 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -3,6 +3,7 @@ package blobstortest import ( "context" "errors" + "slices" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" @@ -26,7 +27,7 @@ func TestIterate(t *testing.T, cons Constructor, minSize, maxSize uint64) { _, err := s.Delete(context.Background(), delPrm) require.NoError(t, err) - objects = append(objects[:delID], objects[delID+1:]...) + objects = slices.Delete(objects, delID, delID+1) runTestNormalHandler(t, s, objects) diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index b930a0141..aa1478423 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "slices" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" @@ -250,7 +251,7 @@ func freePotentialLocks(tx *bbolt.Tx, idCnr cid.ID, locker oid.ID) ([]oid.Addres unlockedObjects = append(unlockedObjects, addr) } else { // exclude locker - keyLockers = append(keyLockers[:i], keyLockers[i+1:]...) + keyLockers = slices.Delete(keyLockers, i, i+1) v, err = encodeList(keyLockers) if err != nil { diff --git a/pkg/services/object/delete/exec.go b/pkg/services/object/delete/exec.go index 36a17bde2..a99ba3586 100644 --- a/pkg/services/object/delete/exec.go +++ b/pkg/services/object/delete/exec.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -182,7 +183,7 @@ func (exec *execCtx) addMembers(incoming []oid.ID) { for i := range members { for j := 0; j < len(incoming); j++ { // don't use range, slice mutates in body if members[i].Equals(incoming[j]) { - incoming = append(incoming[:j], incoming[j+1:]...) + incoming = slices.Delete(incoming, j, j+1) j-- } } diff --git a/pkg/services/object/search/util.go b/pkg/services/object/search/util.go index fed168187..0be5345b9 100644 --- a/pkg/services/object/search/util.go +++ b/pkg/services/object/search/util.go @@ -2,6 +2,7 @@ package searchsvc import ( "context" + "slices" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" @@ -53,7 +54,7 @@ func (w *uniqueIDWriter) WriteIDs(list []oid.ID) error { } // exclude processed address - list = append(list[:i], list[i+1:]...) + list = slices.Delete(list, i, i+1) i-- } diff --git a/pkg/services/object/util/placement.go b/pkg/services/object/util/placement.go index 195944f92..f74b0aab9 100644 --- a/pkg/services/object/util/placement.go +++ b/pkg/services/object/util/placement.go @@ -3,6 +3,7 @@ package util import ( "context" "fmt" + "slices" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -93,7 +94,7 @@ func (p *remotePlacement) BuildPlacement(ctx context.Context, cnr cid.ID, obj *o } if p.netmapKeys.IsLocalKey(vs[i][j].PublicKey()) { - vs[i] = append(vs[i][:j], vs[i][j+1:]...) + vs[i] = slices.Delete(vs[i], j, j+1) j-- } } diff --git a/pkg/services/object_manager/placement/traverser.go b/pkg/services/object_manager/placement/traverser.go index efa4a5b06..a3f9af959 100644 --- a/pkg/services/object_manager/placement/traverser.go +++ b/pkg/services/object_manager/placement/traverser.go @@ -288,8 +288,8 @@ func (t *Traverser) Next() []Node { func (t *Traverser) skipEmptyVectors() { for i := 0; i < len(t.vectors); i++ { // don't use range, slice changes in body if len(t.vectors[i]) == 0 && t.rem[i] <= 0 || t.rem[0] == 0 { - t.vectors = append(t.vectors[:i], t.vectors[i+1:]...) - t.rem = append(t.rem[:i], t.rem[i+1:]...) + t.vectors = slices.Delete(t.vectors, i, i+1) + t.rem = slices.Delete(t.rem, i, i+1) i-- } else { break From ecb6b0793c877ae9a6d1b399a3c364323f77254d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 13 Mar 2025 09:34:59 +0300 Subject: [PATCH 1778/1943] [#1671] Use `slices.ContainsFunc()` where possible Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/helper/netmap.go | 8 +++----- pkg/network/group.go | 7 +++---- pkg/services/object/acl/v2/util_test.go | 9 ++------- pkg/services/policer/policer_test.go | 7 +++---- pkg/services/tree/service.go | 13 +++++-------- 5 files changed, 16 insertions(+), 28 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go index fb8f03783..20abaff0a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/netmap.go @@ -3,6 +3,7 @@ package helper import ( "errors" "fmt" + "slices" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" @@ -118,11 +119,8 @@ func MergeNetmapConfig(roInvoker *invoker.Invoker, md map[string]any) error { return err } for k, v := range m { - for _, key := range NetmapConfigKeys { - if k == key { - md[k] = v - break - } + if slices.Contains(NetmapConfigKeys, k) { + md[k] = v } } return nil diff --git a/pkg/network/group.go b/pkg/network/group.go index 9843b14d4..5a71e530e 100644 --- a/pkg/network/group.go +++ b/pkg/network/group.go @@ -3,6 +3,7 @@ package network import ( "errors" "fmt" + "slices" "sort" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -164,10 +165,8 @@ func WriteToNodeInfo(g AddressGroup, ni *netmap.NodeInfo) { // at least one common address. func (x AddressGroup) Intersects(x2 AddressGroup) bool { for i := range x { - for j := range x2 { - if x[i].equal(x2[j]) { - return true - } + if slices.ContainsFunc(x2, x[i].equal) { + return true } } diff --git a/pkg/services/object/acl/v2/util_test.go b/pkg/services/object/acl/v2/util_test.go index 4b19cecfe..40fce8877 100644 --- a/pkg/services/object/acl/v2/util_test.go +++ b/pkg/services/object/acl/v2/util_test.go @@ -4,6 +4,7 @@ import ( "crypto/ecdsa" "crypto/elliptic" "crypto/rand" + "slices" "testing" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/acl" @@ -91,13 +92,7 @@ func TestIsVerbCompatible(t *testing.T) { for op, list := range table { for _, verb := range verbs { - var contains bool - for _, v := range list { - if v == verb { - contains = true - break - } - } + contains := slices.Contains(list, verb) tok.ForVerb(verb) diff --git a/pkg/services/policer/policer_test.go b/pkg/services/policer/policer_test.go index cef4c36d9..049c33753 100644 --- a/pkg/services/policer/policer_test.go +++ b/pkg/services/policer/policer_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "slices" "sort" "testing" "time" @@ -226,10 +227,8 @@ func TestProcessObject(t *testing.T) { return nil, err } } - for _, i := range ti.objHolders { - if index == i { - return nil, nil - } + if slices.Contains(ti.objHolders, index) { + return nil, nil } return nil, new(apistatus.ObjectNotFound) } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index f9b7395e7..cd89d6a28 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -345,14 +345,11 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) } else { var metaValue []KeyValue for _, kv := range m.Items { - for _, attr := range b.GetAttributes() { - if kv.Key == attr { - metaValue = append(metaValue, KeyValue{ - Key: kv.Key, - Value: kv.Value, - }) - break - } + if slices.Contains(b.GetAttributes(), kv.Key) { + metaValue = append(metaValue, KeyValue{ + Key: kv.Key, + Value: kv.Value, + }) } } x.Meta = metaValue From 997759994a55cc943804d6a16f29a4703f8435a3 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 13 Mar 2025 12:01:57 +0300 Subject: [PATCH 1779/1943] [#1676] golangci: Enable gci linter Signed-off-by: Alexander Chuprov --- .golangci.yml | 6 ++++++ cmd/frostfs-adm/internal/modules/storagecfg/root.go | 1 - cmd/frostfs-cli/modules/control/locate.go | 1 - pkg/services/control/server/list_shards_for_object.go | 1 - 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index d83f36de8..f21a46248 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -22,6 +22,11 @@ linters-settings: # 'default' case is present, even if all enum members aren't listed in the # switch default-signifies-exhaustive: true + gci: + sections: + - standard + - default + custom-order: true govet: # report about shadowed variables check-shadowing: false @@ -72,6 +77,7 @@ linters: - durationcheck - exhaustive - copyloopvar + - gci - gofmt - goimports - misspell diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/root.go b/cmd/frostfs-adm/internal/modules/storagecfg/root.go index 8acbc4579..a5adea0da 100644 --- a/cmd/frostfs-adm/internal/modules/storagecfg/root.go +++ b/cmd/frostfs-adm/internal/modules/storagecfg/root.go @@ -31,7 +31,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/wallet" - "github.com/spf13/cobra" ) diff --git a/cmd/frostfs-cli/modules/control/locate.go b/cmd/frostfs-cli/modules/control/locate.go index d10e2a896..4cb4be539 100644 --- a/cmd/frostfs-cli/modules/control/locate.go +++ b/cmd/frostfs-cli/modules/control/locate.go @@ -11,7 +11,6 @@ import ( rawclient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "github.com/mr-tron/base58" "github.com/spf13/cobra" ) diff --git a/pkg/services/control/server/list_shards_for_object.go b/pkg/services/control/server/list_shards_for_object.go index 84469772f..39565ed50 100644 --- a/pkg/services/control/server/list_shards_for_object.go +++ b/pkg/services/control/server/list_shards_for_object.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/server/ctrlmessage" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "google.golang.org/grpc/codes" From ff4e9b6ae119ca234cf035c00d5c23fc4d494c5a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Mar 2025 10:18:26 +0300 Subject: [PATCH 1780/1943] [#1673] logger: Drop unused fields Signed-off-by: Dmitrii Stepanov --- pkg/util/logger/logger.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index 19d3f1ed1..a0b2728c9 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -147,8 +147,6 @@ func newJournaldLogger(prm *Prm) (*Logger, error) { lvl := zap.NewAtomicLevelAt(prm.level) c := zap.NewProductionConfig() - c.Level = lvl - c.Encoding = "console" if prm.SamplingHook != nil { c.Sampling.Hook = prm.SamplingHook } From 7893d763d1b1d5eeeb62d8a4beb2175acaa13b65 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Mar 2025 10:37:29 +0300 Subject: [PATCH 1781/1943] [#1673] logger: Add sampling for journald logger Signed-off-by: Dmitrii Stepanov --- pkg/util/logger/logger.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index a0b2728c9..2eb5e5538 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -2,6 +2,7 @@ package logger import ( "fmt" + "time" "git.frostfs.info/TrueCloudLab/zapjournald" "github.com/ssgreg/journald" @@ -166,7 +167,18 @@ func newJournaldLogger(prm *Prm) (*Logger, error) { zapjournald.SyslogPid(), }) - lZap := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), zap.AddCallerSkip(1)) + var samplerOpts []zapcore.SamplerOption + if c.Sampling.Hook != nil { + samplerOpts = append(samplerOpts, zapcore.SamplerHook(c.Sampling.Hook)) + } + samplingCore := zapcore.NewSamplerWithOptions( + coreWithContext, + time.Second, + c.Sampling.Initial, + c.Sampling.Thereafter, + samplerOpts..., + ) + lZap := zap.New(samplingCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), zap.AddCallerSkip(1)) l := &Logger{z: lZap, lvl: lvl} prm._log = l From 07a660fbc440dbd3974250608506e39c093bf6cc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 13 Mar 2025 18:01:27 +0300 Subject: [PATCH 1782/1943] [#1677] writecache: Add QoS limiter usage Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 1 + internal/logs/logs.go | 1 + pkg/local_object_storage/writecache/cache.go | 2 ++ pkg/local_object_storage/writecache/flush.go | 15 ++++++++++++++- pkg/local_object_storage/writecache/options.go | 9 +++++++++ 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index e2fe23135..2531e9173 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -930,6 +930,7 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { writecache.WithMaxCacheCount(wcRead.countLimit), writecache.WithNoSync(wcRead.noSync), writecache.WithLogger(c.log), + writecache.WithQoSLimiter(shCfg.limiter), ) } return writeCacheOpts diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 6115cdf90..3503c922e 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -514,4 +514,5 @@ const ( NotSupportedIncomingIOTagReplacedWithClient = "incoming IO tag is not supported, replaced with `client`" FailedToGetNetmapToAdjustIOTag = "failed to get netmap to adjust IO tag, replaced with `client`" FailedToValidateIncomingIOTag = "failed to validate incoming IO tag, replaced with `client`" + WriteCacheFailedToAcquireRPSQuota = "writecache failed to acquire RPS quota to flush object" ) diff --git a/pkg/local_object_storage/writecache/cache.go b/pkg/local_object_storage/writecache/cache.go index b99d73d3a..ee709ea73 100644 --- a/pkg/local_object_storage/writecache/cache.go +++ b/pkg/local_object_storage/writecache/cache.go @@ -6,6 +6,7 @@ import ( "sync" "sync/atomic" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -61,6 +62,7 @@ func New(opts ...Option) Cache { maxCacheSize: defaultMaxCacheSize, metrics: DefaultMetrics(), flushSizeLimit: defaultFlushWorkersCount * defaultMaxObjectSize, + qosLimiter: qos.NewNoopLimiter(), }, } diff --git a/pkg/local_object_storage/writecache/flush.go b/pkg/local_object_storage/writecache/flush.go index 2d07d8b32..893d27ba2 100644 --- a/pkg/local_object_storage/writecache/flush.go +++ b/pkg/local_object_storage/writecache/flush.go @@ -67,7 +67,13 @@ func (c *cache) pushToFlushQueue(ctx context.Context, fl *flushLimiter) { continue } - err := c.fsTree.IterateInfo(ctx, func(oi fstree.ObjectInfo) error { + release, err := c.qosLimiter.ReadRequest(ctx) + if err != nil { + c.log.Warn(ctx, logs.WriteCacheFailedToAcquireRPSQuota, zap.String("operation", "fstree.IterateInfo"), zap.Error(err)) + c.modeMtx.RUnlock() + continue + } + err = c.fsTree.IterateInfo(ctx, func(oi fstree.ObjectInfo) error { if err := fl.acquire(oi.DataSize); err != nil { return err } @@ -82,6 +88,7 @@ func (c *cache) pushToFlushQueue(ctx context.Context, fl *flushLimiter) { return ctx.Err() } }) + release() if err != nil { c.log.Warn(ctx, logs.BlobstorErrorOccurredDuringTheIteration, zap.Error(err)) } @@ -113,6 +120,12 @@ func (c *cache) workerFlush(ctx context.Context, fl *flushLimiter) { func (c *cache) flushIfAnObjectExistsWorker(ctx context.Context, objInfo objectInfo, fl *flushLimiter) { defer fl.release(objInfo.size) + release, err := c.qosLimiter.WriteRequest(ctx) + if err != nil { + c.log.Warn(ctx, logs.WriteCacheFailedToAcquireRPSQuota, zap.String("operation", "fstree.Get"), zap.Error(err)) + return + } + defer release() res, err := c.fsTree.Get(ctx, common.GetPrm{ Address: objInfo.addr, }) diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index f2957fe98..dbbe66c19 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -3,6 +3,7 @@ package writecache import ( "context" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "go.uber.org/zap" ) @@ -38,6 +39,8 @@ type options struct { disableBackgroundFlush bool // flushSizeLimit is total size of flushing objects. flushSizeLimit uint64 + // qosLimiter used to limit flush RPS. + qosLimiter qos.Limiter } // WithLogger sets logger. @@ -136,3 +139,9 @@ func WithFlushSizeLimit(v uint64) Option { o.flushSizeLimit = v } } + +func WithQoSLimiter(l qos.Limiter) Option { + return func(o *options) { + o.qosLimiter = l + } +} From fde2649e60ff27c4e5aa66cbd163a14bf585398e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 14 Mar 2025 12:03:42 +0300 Subject: [PATCH 1783/1943] [#1678] adm: Fix frostfs-adm morph list-subjects & list-group-subjects `include-names` for `list-subjects` returns error `invalid response subject struct` because `ListSubjects` returns only subject addresses (see frostfs-contract). Replace `include-names` with `extended` as now all subject info printed. Signed-off-by: Dmitrii Stepanov --- .../modules/morph/frostfsid/frostfsid.go | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 4fbd0bfe1..8ae606f1a 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -34,7 +34,7 @@ const ( subjectNameFlag = "subject-name" subjectKeyFlag = "subject-key" subjectAddressFlag = "subject-address" - includeNamesFlag = "include-names" + extendedFlag = "extended" groupNameFlag = "group-name" groupIDFlag = "group-id" @@ -209,7 +209,7 @@ func initFrostfsIDListSubjectsCmd() { Cmd.AddCommand(frostfsidListSubjectsCmd) frostfsidListSubjectsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidListSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace to list subjects") - frostfsidListSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") + frostfsidListSubjectsCmd.Flags().Bool(extendedFlag, false, "Whether include subject info (require additional requests)") } func initFrostfsIDCreateGroupCmd() { @@ -256,7 +256,7 @@ func initFrostfsIDListGroupSubjectsCmd() { frostfsidListGroupSubjectsCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) frostfsidListGroupSubjectsCmd.Flags().String(namespaceFlag, "", "Namespace name") frostfsidListGroupSubjectsCmd.Flags().Int64(groupIDFlag, 0, "Group id") - frostfsidListGroupSubjectsCmd.Flags().Bool(includeNamesFlag, false, "Whether include subject name (require additional requests)") + frostfsidListGroupSubjectsCmd.Flags().Bool(extendedFlag, false, "Whether include subject info (require additional requests)") } func initFrostfsIDSetKVCmd() { @@ -336,7 +336,7 @@ func frostfsidDeleteSubject(cmd *cobra.Command, _ []string) { } func frostfsidListSubjects(cmd *cobra.Command, _ []string) { - includeNames, _ := cmd.Flags().GetBool(includeNamesFlag) + extended, _ := cmd.Flags().GetBool(extendedFlag) ns := getFrostfsIDNamespace(cmd) inv, _, hash := initInvoker(cmd) reader := frostfsidrpclient.NewReader(inv, hash) @@ -349,21 +349,19 @@ func frostfsidListSubjects(cmd *cobra.Command, _ []string) { sort.Slice(subAddresses, func(i, j int) bool { return subAddresses[i].Less(subAddresses[j]) }) for _, addr := range subAddresses { - if !includeNames { + if !extended { cmd.Println(address.Uint160ToString(addr)) continue } - sessionID, it, err := reader.ListSubjects() + items, err := reader.GetSubject(addr) commonCmd.ExitOnErr(cmd, "can't get subject: %w", err) - items, err := readIterator(inv, &it, sessionID) - commonCmd.ExitOnErr(cmd, "can't read iterator: %w", err) - subj, err := frostfsidclient.ParseSubject(items) commonCmd.ExitOnErr(cmd, "can't parse subject: %w", err) - cmd.Printf("%s (%s)\n", address.Uint160ToString(addr), subj.Name) + printSubjectInfo(cmd, addr, subj) + cmd.Println() } } @@ -483,7 +481,7 @@ func frostfsidDeleteKV(cmd *cobra.Command, _ []string) { func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { ns := getFrostfsIDNamespace(cmd) groupID := getFrostfsIDGroupID(cmd) - includeNames, _ := cmd.Flags().GetBool(includeNamesFlag) + extended, _ := cmd.Flags().GetBool(extendedFlag) inv, cs, hash := initInvoker(cmd) _, err := helper.NNSResolveHash(inv, cs.Hash, helper.DomainOf(constants.FrostfsIDContract)) commonCmd.ExitOnErr(cmd, "can't get netmap contract hash: %w", err) @@ -501,7 +499,7 @@ func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { sort.Slice(subjects, func(i, j int) bool { return subjects[i].Less(subjects[j]) }) for _, subjAddr := range subjects { - if !includeNames { + if !extended { cmd.Println(address.Uint160ToString(subjAddr)) continue } @@ -510,7 +508,8 @@ func frostfsidListGroupSubjects(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "can't get subject: %w", err) subj, err := frostfsidclient.ParseSubject(items) commonCmd.ExitOnErr(cmd, "can't parse subject: %w", err) - cmd.Printf("%s (%s)\n", address.Uint160ToString(subjAddr), subj.Name) + printSubjectInfo(cmd, subjAddr, subj) + cmd.Println() } } @@ -600,3 +599,30 @@ func initInvoker(cmd *cobra.Command) (*invoker.Invoker, *state.Contract, util.Ui return inv, cs, nmHash } + +func printSubjectInfo(cmd *cobra.Command, addr util.Uint160, subj *frostfsidclient.Subject) { + cmd.Printf("Address: %s\n", address.Uint160ToString(addr)) + pk := "" + if subj.PrimaryKey != nil { + pk = subj.PrimaryKey.String() + } + cmd.Printf("Primary key: %s\n", pk) + cmd.Printf("Name: %s\n", subj.Name) + cmd.Printf("Namespace: %s\n", subj.Namespace) + if len(subj.AdditionalKeys) > 0 { + cmd.Printf("Additional keys:\n") + for _, key := range subj.AdditionalKeys { + k := "" + if key != nil { + k = key.String() + } + cmd.Printf("- %s\n", k) + } + } + if len(subj.KV) > 0 { + cmd.Printf("KV:\n") + for k, v := range subj.KV { + cmd.Printf("- %s: %s\n", k, v) + } + } +} From ef6ac751dfa849e3c32171890229b5fbbf979b7a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 14 Mar 2025 16:32:30 +0300 Subject: [PATCH 1784/1943] [#1671] Update gopls to v0.17.1 Signed-off-by: Evgenii Stratonikov --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 497dce115..538912cc1 100755 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ GOFUMPT_VERSION ?= v0.7.0 GOFUMPT_DIR ?= $(abspath $(BIN))/gofumpt GOFUMPT_VERSION_DIR ?= $(GOFUMPT_DIR)/$(GOFUMPT_VERSION) -GOPLS_VERSION ?= v0.15.1 +GOPLS_VERSION ?= v0.17.1 GOPLS_DIR ?= $(abspath $(BIN))/gopls GOPLS_VERSION_DIR ?= $(GOPLS_DIR)/$(GOPLS_VERSION) GOPLS_TEMP_FILE := $(shell mktemp) From 91c7b39232fb23ad5e46975b7a04da57214f4de0 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 14 Mar 2025 16:34:34 +0300 Subject: [PATCH 1785/1943] [#1680] go.mod: Bump go version to 1.23 Change-Id: I77f908924f675e676f0db6a57204d7c1e0df219a Signed-off-by: Evgenii Stratonikov --- .forgejo/workflows/build.yml | 2 +- .forgejo/workflows/dco.yml | 2 +- .forgejo/workflows/pre-commit.yml | 2 +- .forgejo/workflows/tests.yml | 10 +++++----- .forgejo/workflows/vulncheck.yml | 2 +- Makefile | 2 +- go.mod | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 9129d136e..d568b9607 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.22', '1.23' ] + go_versions: [ '1.23', '1.24' ] steps: - uses: actions/checkout@v3 diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 7c5af8410..190d7764a 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.24' - name: Run commit format checker uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 diff --git a/.forgejo/workflows/pre-commit.yml b/.forgejo/workflows/pre-commit.yml index b27e7a39a..c2e293175 100644 --- a/.forgejo/workflows/pre-commit.yml +++ b/.forgejo/workflows/pre-commit.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.23 + go-version: 1.24 - name: Set up Python run: | apt update diff --git a/.forgejo/workflows/tests.yml b/.forgejo/workflows/tests.yml index 4f1bebe61..f3f5432ce 100644 --- a/.forgejo/workflows/tests.yml +++ b/.forgejo/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.23' + go-version: '1.24' cache: true - name: Install linters @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_versions: [ '1.22', '1.23' ] + go_versions: [ '1.23', '1.24' ] fail-fast: false steps: - uses: actions/checkout@v3 @@ -53,7 +53,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.22' + go-version: '1.24' cache: true - name: Run tests @@ -68,7 +68,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.23' + go-version: '1.24' cache: true - name: Install staticcheck @@ -104,7 +104,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: '1.23' + go-version: '1.24' cache: true - name: Install gofumpt diff --git a/.forgejo/workflows/vulncheck.yml b/.forgejo/workflows/vulncheck.yml index 140434dfc..bc94792d8 100644 --- a/.forgejo/workflows/vulncheck.yml +++ b/.forgejo/workflows/vulncheck.yml @@ -18,7 +18,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: '1.23' + go-version: '1.24' check-latest: true - name: Install govulncheck diff --git a/Makefile b/Makefile index 538912cc1..fd2ee3b6f 100755 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" -GO_VERSION ?= 1.22 +GO_VERSION ?= 1.23 LINT_VERSION ?= 1.62.2 TRUECLOUDLAB_LINT_VERSION ?= 0.0.8 PROTOC_VERSION ?= 25.0 diff --git a/go.mod b/go.mod index 69273fda2..eeaca1645 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-node -go 1.22 +go 1.23 require ( code.gitea.io/sdk/gitea v0.17.1 From 54ef71a92ffdcdaebe3fb1685d548170c4a11006 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 14 Mar 2025 16:39:19 +0300 Subject: [PATCH 1786/1943] [#1680] Update staticcheck to 2025.1.1 Change-Id: Ie851e714afebf171c4d42d4c49b42379c2665113 Signed-off-by: Evgenii Stratonikov --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fd2ee3b6f..cd80fc72e 100755 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ PROTOC_OS_VERSION=osx-x86_64 ifeq ($(shell uname), Linux) PROTOC_OS_VERSION=linux-x86_64 endif -STATICCHECK_VERSION ?= 2024.1.1 +STATICCHECK_VERSION ?= 2025.1.1 ARCH = amd64 BIN = bin From fc743cc5373c7ccce331c2c84fb53d7dd41e4d29 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 18 Mar 2025 10:56:30 +0300 Subject: [PATCH 1787/1943] [#1684] cli: Correct description of control shards writecache seal Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/modules/control/writecache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/control/writecache.go b/cmd/frostfs-cli/modules/control/writecache.go index 80e4a0c87..d0c9a641b 100644 --- a/cmd/frostfs-cli/modules/control/writecache.go +++ b/cmd/frostfs-cli/modules/control/writecache.go @@ -24,7 +24,7 @@ var writecacheShardCmd = &cobra.Command{ var sealWritecacheShardCmd = &cobra.Command{ Use: "seal", Short: "Flush objects from write-cache and move write-cache to degraded read only mode.", - Long: "Flush all the objects from the write-cache to the main storage and move the write-cache to the degraded read only mode: write-cache will be empty and no objects will be put in it.", + Long: "Flush all the objects from the write-cache to the main storage and move the write-cache to the 'CLOSED' mode: write-cache will be empty and no objects will be put in it.", Run: sealWritecache, } From a7319bc979669dde3a67762220fdbd96077a3f94 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 18 Mar 2025 09:26:34 +0300 Subject: [PATCH 1788/1943] [#1683] metabase/test: Report allocs in benchmarkSelect() Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/select_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/local_object_storage/metabase/select_test.go b/pkg/local_object_storage/metabase/select_test.go index 5cc998311..ce2156d2e 100644 --- a/pkg/local_object_storage/metabase/select_test.go +++ b/pkg/local_object_storage/metabase/select_test.go @@ -1216,6 +1216,8 @@ func TestExpiredObjects(t *testing.T) { } func benchmarkSelect(b *testing.B, db *meta.DB, cid cidSDK.ID, fs objectSDK.SearchFilters, expected int) { + b.ReportAllocs() + var prm meta.SelectPrm prm.SetContainerID(cid) prm.SetFilters(fs) From a405fb1f39d616e37fd6d0de35c2e68d6cd3b6b8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 18 Mar 2025 09:34:47 +0300 Subject: [PATCH 1789/1943] [#1683] metabase: Check object status once in Select() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit objectStatus() is called twice for the same object: First, in selectObject() to filter removed objects. Then, again, in getObjectForSlowFilters() via db.get(). The second call will return the same result, so remove useless branch. ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ old │ status │ │ sec/op │ sec/op vs base │ Select/string_equal-8 5.022m ± 7% 3.968m ± 8% -20.98% (p=0.000 n=10) Select/string_not_equal-8 4.953m ± 9% 3.990m ± 10% -19.44% (p=0.000 n=10) Select/common_prefix-8 4.962m ± 8% 3.971m ± 9% -19.98% (p=0.000 n=10) Select/unknown-8 5.246m ± 9% 3.548m ± 5% -32.37% (p=0.000 n=10) geomean 5.045m 3.865m -23.39% │ old │ status │ │ B/op │ B/op vs base │ Select/string_equal-8 2.685Mi ± 0% 2.250Mi ± 0% -16.20% (p=0.000 n=10) Select/string_not_equal-8 2.685Mi ± 0% 2.250Mi ± 0% -16.20% (p=0.000 n=10) Select/common_prefix-8 2.685Mi ± 0% 2.250Mi ± 0% -16.20% (p=0.000 n=10) Select/unknown-8 2.677Mi ± 0% 2.243Mi ± 0% -16.24% (p=0.000 n=10) geomean 2.683Mi 2.248Mi -16.21% │ old │ status │ │ allocs/op │ allocs/op vs base │ Select/string_equal-8 69.03k ± 0% 56.02k ± 0% -18.84% (p=0.000 n=10) Select/string_not_equal-8 69.03k ± 0% 56.02k ± 0% -18.84% (p=0.000 n=10) Select/common_prefix-8 69.03k ± 0% 56.02k ± 0% -18.84% (p=0.000 n=10) Select/unknown-8 68.03k ± 0% 55.03k ± 0% -19.11% (p=0.000 n=10) geomean 68.78k 55.77k -18.90% ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/select.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 9f1b8b060..4a3b22b55 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -517,7 +517,7 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.Searc func (db *DB) getObjectForSlowFilters(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (*objectSDK.Object, bool, error) { buf := make([]byte, addressKeySize) - obj, err := db.get(tx, addr, buf, true, false, currEpoch) + obj, err := db.get(tx, addr, buf, false, false, currEpoch) if err != nil { var ecInfoError *objectSDK.ECInfoError if errors.As(err, &ecInfoError) { From a11b2d27e4011cf366e889f79cac198abdbcaa51 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Thu, 20 Feb 2025 11:05:11 +0300 Subject: [PATCH 1790/1943] [#1642] tree: Introduce `Cursor` type * Use `Cursor` as parameter for `TreeSortedByFilename` Signed-off-by: Airat Arifullin --- pkg/local_object_storage/engine/tree.go | 4 ++-- pkg/local_object_storage/pilorama/boltdb.go | 9 ++++---- pkg/local_object_storage/pilorama/forest.go | 8 +++---- .../pilorama/forest_test.go | 4 ++-- pkg/local_object_storage/pilorama/heap.go | 6 ++--- .../pilorama/interface.go | 23 ++++++++++++++++++- .../pilorama/split_test.go | 2 +- pkg/local_object_storage/shard/tree.go | 2 +- pkg/services/tree/service.go | 2 +- 9 files changed, 40 insertions(+), 20 deletions(-) diff --git a/pkg/local_object_storage/engine/tree.go b/pkg/local_object_storage/engine/tree.go index 7f70d36f7..cfd15b4d4 100644 --- a/pkg/local_object_storage/engine/tree.go +++ b/pkg/local_object_storage/engine/tree.go @@ -230,7 +230,7 @@ func (e *StorageEngine) TreeGetChildren(ctx context.Context, cid cidSDK.ID, tree } // TreeSortedByFilename implements the pilorama.Forest interface. -func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) { +func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *pilorama.Cursor, count int) ([]pilorama.MultiNodeInfo, *pilorama.Cursor, error) { ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.TreeSortedByFilename", trace.WithAttributes( attribute.String("container_id", cid.EncodeToString()), @@ -241,7 +241,7 @@ func (e *StorageEngine) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, var err error var nodes []pilorama.MultiNodeInfo - var cursor *string + var cursor *pilorama.Cursor for _, sh := range e.sortShards(cid) { nodes, cursor, err = sh.TreeSortedByFilename(ctx, cid, treeID, nodeID, last, count) if err != nil { diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 9d71d9fda..0eea60ad8 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1077,7 +1077,7 @@ func (t *boltForest) hasFewChildren(b *bbolt.Bucket, nodeIDs MultiNode, threshol } // TreeSortedByFilename implements the Forest interface. -func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeIDs MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error) { +func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeIDs MultiNode, last *Cursor, count int) ([]MultiNodeInfo, *Cursor, error) { var ( startedAt = time.Now() success = false @@ -1128,7 +1128,6 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr } t.fillSortedChildren(b, nodeIDs, h) - for info, ok := h.pop(); ok; info, ok = h.pop() { for _, id := range info.id { childInfo, err := t.getChildInfo(b, key, id) @@ -1155,7 +1154,7 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr } if len(res) != 0 { s := string(findAttr(res[len(res)-1].Meta, AttributeFilename)) - last = &s + last = NewCursor(s) } return res, last, metaerr.Wrap(err) } @@ -1166,10 +1165,10 @@ func sortByFilename(nodes []NodeInfo) { }) } -func sortAndCut(result []NodeInfo, last *string) []NodeInfo { +func sortAndCut(result []NodeInfo, last *Cursor) []NodeInfo { var lastBytes []byte if last != nil { - lastBytes = []byte(*last) + lastBytes = []byte(last.GetFilename()) } sortByFilename(result) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index ef284a727..ce8528a81 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -164,7 +164,7 @@ func (f *memoryForest) TreeGetMeta(_ context.Context, cid cid.ID, treeID string, } // TreeSortedByFilename implements the Forest interface. -func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeIDs MultiNode, start *string, count int) ([]MultiNodeInfo, *string, error) { +func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeID string, nodeIDs MultiNode, start *Cursor, count int) ([]MultiNodeInfo, *Cursor, error) { fullID := cid.String() + "/" + treeID s, ok := f.treeMap[fullID] if !ok { @@ -204,14 +204,14 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI r := mergeNodeInfos(res) for i := range r { - if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > *start { + if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > start.GetFilename() { finish := min(len(res), i+count) last := string(findAttr(r[finish-1].Meta, AttributeFilename)) - return r[i:finish], &last, nil + return r[i:finish], NewCursor(last), nil } } last := string(res[len(res)-1].Meta.GetAttr(AttributeFilename)) - return nil, &last, nil + return nil, NewCursor(last), nil } // TreeGetChildren implements the Forest interface. diff --git a/pkg/local_object_storage/pilorama/forest_test.go b/pkg/local_object_storage/pilorama/forest_test.go index de56fc82b..844084c55 100644 --- a/pkg/local_object_storage/pilorama/forest_test.go +++ b/pkg/local_object_storage/pilorama/forest_test.go @@ -273,7 +273,7 @@ func testForestTreeSortedIterationBugWithSkip(t *testing.T, s ForestStorage) { } var result []MultiNodeInfo - treeAppend := func(t *testing.T, last *string, count int) *string { + treeAppend := func(t *testing.T, last *Cursor, count int) *Cursor { res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, last, count) require.NoError(t, err) result = append(result, res...) @@ -328,7 +328,7 @@ func testForestTreeSortedIteration(t *testing.T, s ForestStorage) { } var result []MultiNodeInfo - treeAppend := func(t *testing.T, last *string, count int) *string { + treeAppend := func(t *testing.T, last *Cursor, count int) *Cursor { res, cursor, err := s.TreeSortedByFilename(context.Background(), d.CID, treeID, MultiNode{RootID}, last, count) require.NoError(t, err) result = append(result, res...) diff --git a/pkg/local_object_storage/pilorama/heap.go b/pkg/local_object_storage/pilorama/heap.go index 5a00bcf7a..70afc148a 100644 --- a/pkg/local_object_storage/pilorama/heap.go +++ b/pkg/local_object_storage/pilorama/heap.go @@ -30,13 +30,13 @@ func (h *filenameHeap) Pop() any { // fixedHeap maintains a fixed number of smallest elements started at some point. type fixedHeap struct { - start *string + start *Cursor sorted bool count int h *filenameHeap } -func newHeap(start *string, count int) *fixedHeap { +func newHeap(start *Cursor, count int) *fixedHeap { h := new(filenameHeap) heap.Init(h) @@ -50,7 +50,7 @@ func newHeap(start *string, count int) *fixedHeap { const amortizationMultiplier = 5 func (h *fixedHeap) push(id MultiNode, filename string) bool { - if h.start != nil && filename <= *h.start { + if h.start != nil && filename <= (*h.start).GetFilename() { return false } diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index 1f7e742a2..e364b008b 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -37,7 +37,7 @@ type Forest interface { TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID string, nodeID Node) ([]NodeInfo, error) // TreeSortedByFilename returns children of the node with the specified ID. The nodes are sorted by the filename attribute.. // Should return ErrTreeNotFound if the tree is not found, and empty result if the node is not in the tree. - TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID MultiNode, last *string, count int) ([]MultiNodeInfo, *string, error) + TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID MultiNode, last *Cursor, count int) ([]MultiNodeInfo, *Cursor, error) // TreeGetOpLog returns first log operation stored at or above the height. // In case no such operation is found, empty Move and nil error should be returned. TreeGetOpLog(ctx context.Context, cid cidSDK.ID, treeID string, height uint64) (Move, error) @@ -79,6 +79,27 @@ const ( AttributeVersion = "Version" ) +// Cursor keeps state between function calls for traversing nodes. +// It stores the attributes associated with a previous call, allowing subsequent operations +// to resume traversal from this point rather than starting from the beginning. +type Cursor struct { + // Last traversed filename. + filename string +} + +func NewCursor(filename string) *Cursor { + return &Cursor{ + filename: filename, + } +} + +func (c *Cursor) GetFilename() string { + if c == nil { + return "" + } + return c.filename +} + // CIDDescriptor contains container ID and information about the node position // in the list of container nodes. type CIDDescriptor struct { diff --git a/pkg/local_object_storage/pilorama/split_test.go b/pkg/local_object_storage/pilorama/split_test.go index 54c2b90a6..eecee1527 100644 --- a/pkg/local_object_storage/pilorama/split_test.go +++ b/pkg/local_object_storage/pilorama/split_test.go @@ -96,7 +96,7 @@ func testDuplicateDirectory(t *testing.T, f Forest) { require.Equal(t, []byte{8}, testGetByPath(t, "dir1/dir3/value4")) require.Equal(t, []byte{10}, testGetByPath(t, "value0")) - testSortedByFilename := func(t *testing.T, root MultiNode, last *string, batchSize int) ([]MultiNodeInfo, *string) { + testSortedByFilename := func(t *testing.T, root MultiNode, last *Cursor, batchSize int) ([]MultiNodeInfo, *Cursor) { res, last, err := f.TreeSortedByFilename(context.Background(), d.CID, treeID, root, last, batchSize) require.NoError(t, err) return res, last diff --git a/pkg/local_object_storage/shard/tree.go b/pkg/local_object_storage/shard/tree.go index e9cd5f8c1..db361a8bd 100644 --- a/pkg/local_object_storage/shard/tree.go +++ b/pkg/local_object_storage/shard/tree.go @@ -246,7 +246,7 @@ func (s *Shard) TreeGetChildren(ctx context.Context, cid cidSDK.ID, treeID strin } // TreeSortedByFilename implements the pilorama.Forest interface. -func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *string, count int) ([]pilorama.MultiNodeInfo, *string, error) { +func (s *Shard) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, treeID string, nodeID pilorama.MultiNode, last *pilorama.Cursor, count int) ([]pilorama.MultiNodeInfo, *pilorama.Cursor, error) { ctx, span := tracing.StartSpanFromContext(ctx, "Shard.TreeSortedByFilename", trace.WithAttributes( attribute.String("shard_id", s.ID().String()), diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index cd89d6a28..98c5626bd 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -412,7 +412,7 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS type stackItem struct { values []pilorama.MultiNodeInfo parent pilorama.MultiNode - last *string + last *pilorama.Cursor } func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid cidSDK.ID, b *GetSubTreeRequest_Body, forest pilorama.Forest) error { From 760b6a44ea0557489a1f16fbdbbdfb81efcc0d58 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 28 Feb 2025 15:47:29 +0300 Subject: [PATCH 1791/1943] [#1642] tree: Fix sorted getSubtree for multiversion filenames Signed-off-by: Airat Arifullin --- pkg/local_object_storage/pilorama/boltdb.go | 3 ++- pkg/local_object_storage/pilorama/forest.go | 4 ++-- pkg/local_object_storage/pilorama/heap.go | 15 +++++++++++++-- pkg/local_object_storage/pilorama/interface.go | 13 ++++++++++++- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 0eea60ad8..2ca6fdefa 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1128,6 +1128,7 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr } t.fillSortedChildren(b, nodeIDs, h) + for info, ok := h.pop(); ok; info, ok = h.pop() { for _, id := range info.id { childInfo, err := t.getChildInfo(b, key, id) @@ -1154,7 +1155,7 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr } if len(res) != 0 { s := string(findAttr(res[len(res)-1].Meta, AttributeFilename)) - last = NewCursor(s) + last = NewCursor(s, res[len(res)-1].Children[len(res[len(res)-1].Children)-1]) } return res, last, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index ce8528a81..b5320e42d 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -207,11 +207,11 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI if start == nil || string(findAttr(r[i].Meta, AttributeFilename)) > start.GetFilename() { finish := min(len(res), i+count) last := string(findAttr(r[finish-1].Meta, AttributeFilename)) - return r[i:finish], NewCursor(last), nil + return r[i:finish], NewCursor(last, 0), nil } } last := string(res[len(res)-1].Meta.GetAttr(AttributeFilename)) - return nil, NewCursor(last), nil + return nil, NewCursor(last, 0), nil } // TreeGetChildren implements the Forest interface. diff --git a/pkg/local_object_storage/pilorama/heap.go b/pkg/local_object_storage/pilorama/heap.go index 70afc148a..b035be1e1 100644 --- a/pkg/local_object_storage/pilorama/heap.go +++ b/pkg/local_object_storage/pilorama/heap.go @@ -50,8 +50,19 @@ func newHeap(start *Cursor, count int) *fixedHeap { const amortizationMultiplier = 5 func (h *fixedHeap) push(id MultiNode, filename string) bool { - if h.start != nil && filename <= (*h.start).GetFilename() { - return false + if h.start != nil { + if filename < h.start.GetFilename() { + return false + } else if filename == h.start.GetFilename() { + // A tree may have a lot of nodes with the same filename but different versions so that + // len(nodes) > batch_size. The cut nodes should be pushed into the result on repeated call + // with the same filename. + pos := slices.Index(id, h.start.GetNode()) + if pos == -1 || pos+1 >= len(id) { + return false + } + id = id[pos+1:] + } } *h.h = append(*h.h, heapInfo{id: id, filename: filename}) diff --git a/pkg/local_object_storage/pilorama/interface.go b/pkg/local_object_storage/pilorama/interface.go index e364b008b..e1f6cd8e7 100644 --- a/pkg/local_object_storage/pilorama/interface.go +++ b/pkg/local_object_storage/pilorama/interface.go @@ -85,11 +85,15 @@ const ( type Cursor struct { // Last traversed filename. filename string + + // Last traversed node. + node Node } -func NewCursor(filename string) *Cursor { +func NewCursor(filename string, node Node) *Cursor { return &Cursor{ filename: filename, + node: node, } } @@ -100,6 +104,13 @@ func (c *Cursor) GetFilename() string { return c.filename } +func (c *Cursor) GetNode() Node { + if c == nil { + return Node(0) + } + return c.node +} + // CIDDescriptor contains container ID and information about the node position // in the list of container nodes. type CIDDescriptor struct { From 39f549a7ab011c25b81f4bafa7c21f8a46cd5b26 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 10 Mar 2025 13:17:24 +0300 Subject: [PATCH 1792/1943] [#1642] tree: Intoduce a helper `LastChild` Signed-off-by: Airat Arifullin --- pkg/local_object_storage/pilorama/boltdb.go | 2 +- pkg/local_object_storage/pilorama/multinode.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index 2ca6fdefa..fc7cdaabc 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1155,7 +1155,7 @@ func (t *boltForest) TreeSortedByFilename(ctx context.Context, cid cidSDK.ID, tr } if len(res) != 0 { s := string(findAttr(res[len(res)-1].Meta, AttributeFilename)) - last = NewCursor(s, res[len(res)-1].Children[len(res[len(res)-1].Children)-1]) + last = NewCursor(s, res[len(res)-1].LastChild()) } return res, last, metaerr.Wrap(err) } diff --git a/pkg/local_object_storage/pilorama/multinode.go b/pkg/local_object_storage/pilorama/multinode.go index 106ba6ae9..36d347f10 100644 --- a/pkg/local_object_storage/pilorama/multinode.go +++ b/pkg/local_object_storage/pilorama/multinode.go @@ -25,6 +25,10 @@ func (r *MultiNodeInfo) Add(info NodeInfo) bool { return true } +func (r *MultiNodeInfo) LastChild() Node { + return r.Children[len(r.Children)-1] +} + func (n NodeInfo) ToMultiNode() MultiNodeInfo { return MultiNodeInfo{ Children: MultiNode{n.ID}, From a7ac30da9c47abe0e2fffa49de287e8849d570ed Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 12 Mar 2025 17:01:56 +0300 Subject: [PATCH 1793/1943] [#1642] tree: Refactor `getSortedSubTree` * Reuse `item` as result for `forest.TreeSortedByFilename` invocation. Signed-off-by: Airat Arifullin --- pkg/services/tree/service.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 98c5626bd..eeffec08b 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -463,14 +463,13 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid break } - nodes, last, err := forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), item.parent, item.last, batchSize) + var err error + item.values, item.last, err = forest.TreeSortedByFilename(ctx, cid, b.GetTreeId(), item.parent, item.last, batchSize) if err != nil { return err } - item.values = nodes - item.last = last - if len(nodes) == 0 { + if len(item.values) == 0 { stack = stack[:len(stack)-1] continue } From a49f0717b3fbba0ecf51ce55f49221474a8f8fb8 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 19 Mar 2025 09:22:29 +0300 Subject: [PATCH 1794/1943] [#1685] metabase: Cache frequently accessed singleton buckets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are some buckets we access almost always, to check whether an object is alive. In search we also iterate over lots of objects, and `tx.Bucket()` shows itself a lot in pprof. ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ 1 │ 2 │ │ sec/op │ sec/op vs base │ Select/string_equal-8 4.753m ± 6% 3.969m ± 14% -16.50% (p=0.000 n=10) Select/string_not_equal-8 4.247m ± 9% 3.486m ± 11% -17.93% (p=0.000 n=10) Select/common_prefix-8 4.163m ± 5% 3.323m ± 5% -20.18% (p=0.000 n=10) Select/unknown-8 3.557m ± 3% 3.064m ± 8% -13.85% (p=0.001 n=10) geomean 4.158m 3.445m -17.15% │ 1 │ 2 │ │ B/op │ B/op vs base │ Select/string_equal-8 2.250Mi ± 0% 1.907Mi ± 0% -15.24% (p=0.000 n=10) Select/string_not_equal-8 2.250Mi ± 0% 1.907Mi ± 0% -15.24% (p=0.000 n=10) Select/common_prefix-8 2.250Mi ± 0% 1.907Mi ± 0% -15.24% (p=0.000 n=10) Select/unknown-8 2.243Mi ± 0% 1.900Mi ± 0% -15.29% (p=0.000 n=10) geomean 2.248Mi 1.905Mi -15.26% │ 1 │ 2 │ │ allocs/op │ allocs/op vs base │ Select/string_equal-8 56.02k ± 0% 47.03k ± 0% -16.05% (p=0.000 n=10) Select/string_not_equal-8 56.02k ± 0% 47.03k ± 0% -16.05% (p=0.000 n=10) Select/common_prefix-8 56.02k ± 0% 47.03k ± 0% -16.05% (p=0.000 n=10) Select/unknown-8 55.03k ± 0% 46.04k ± 0% -16.34% (p=0.000 n=10) geomean 55.78k 46.78k -16.12% ``` Signed-off-by: Evgenii Stratonikov --- .../metabase/bucket_cache.go | 45 +++++++++++++++++++ pkg/local_object_storage/metabase/exists.go | 10 +++-- pkg/local_object_storage/metabase/lock.go | 6 ++- pkg/local_object_storage/metabase/select.go | 3 +- 4 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 pkg/local_object_storage/metabase/bucket_cache.go diff --git a/pkg/local_object_storage/metabase/bucket_cache.go b/pkg/local_object_storage/metabase/bucket_cache.go new file mode 100644 index 000000000..b425450af --- /dev/null +++ b/pkg/local_object_storage/metabase/bucket_cache.go @@ -0,0 +1,45 @@ +package meta + +import ( + "go.etcd.io/bbolt" +) + +type bucketCache struct { + locked *bbolt.Bucket + graveyard *bbolt.Bucket + garbage *bbolt.Bucket +} + +func newBucketCache() *bucketCache { + return &bucketCache{} +} + +func getLockedBucket(bc *bucketCache, tx *bbolt.Tx) *bbolt.Bucket { + if bc == nil { + return tx.Bucket(bucketNameLocked) + } + return getBucket(&bc.locked, tx, bucketNameLocked) +} + +func getGraveyardBucket(bc *bucketCache, tx *bbolt.Tx) *bbolt.Bucket { + if bc == nil { + return tx.Bucket(graveyardBucketName) + } + return getBucket(&bc.graveyard, tx, graveyardBucketName) +} + +func getGarbageBucket(bc *bucketCache, tx *bbolt.Tx) *bbolt.Bucket { + if bc == nil { + return tx.Bucket(garbageBucketName) + } + return getBucket(&bc.garbage, tx, garbageBucketName) +} + +func getBucket(cache **bbolt.Bucket, tx *bbolt.Tx, name []byte) *bbolt.Bucket { + if *cache != nil { + return *cache + } + + *cache = tx.Bucket(name) + return *cache +} diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index 962108a76..0b28da5c9 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -153,8 +153,12 @@ func (db *DB) exists(tx *bbolt.Tx, addr oid.Address, ecParent oid.Address, currE // - 2 if object is covered with tombstone; // - 3 if object is expired. func objectStatus(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (uint8, error) { + return objectStatusWithCache(nil, tx, addr, currEpoch) +} + +func objectStatusWithCache(bc *bucketCache, tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (uint8, error) { // locked object could not be removed/marked with GC/expired - if objectLocked(tx, addr.Container(), addr.Object()) { + if objectLockedWithCache(bc, tx, addr.Container(), addr.Object()) { return 0, nil } @@ -167,8 +171,8 @@ func objectStatus(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (uint8, erro return 3, nil } - graveyardBkt := tx.Bucket(graveyardBucketName) - garbageBkt := tx.Bucket(garbageBucketName) + graveyardBkt := getGraveyardBucket(bc, tx) + garbageBkt := getGarbageBucket(bc, tx) addrKey := addressKey(addr, make([]byte, addressKeySize)) return inGraveyardWithKey(addrKey, graveyardBkt, garbageBkt), nil } diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index aa1478423..f73c2b4f6 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -163,7 +163,11 @@ func (db *DB) FreeLockedBy(lockers []oid.Address) ([]oid.Address, error) { // checks if specified object is locked in the specified container. func objectLocked(tx *bbolt.Tx, idCnr cid.ID, idObj oid.ID) bool { - bucketLocked := tx.Bucket(bucketNameLocked) + return objectLockedWithCache(nil, tx, idCnr, idObj) +} + +func objectLockedWithCache(bc *bucketCache, tx *bbolt.Tx, idCnr cid.ID, idObj oid.ID) bool { + bucketLocked := getLockedBucket(bc, tx) if bucketLocked != nil { key := make([]byte, cidSize) idCnr.Encode(key) diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index 4a3b22b55..a95384753 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -131,6 +131,7 @@ func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters res := make([]oid.Address, 0, len(mAddr)) + bc := newBucketCache() for a, ind := range mAddr { if ind != expLen { continue // ignore objects with unmatched fast filters @@ -145,7 +146,7 @@ func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters var addr oid.Address addr.SetContainer(cnr) addr.SetObject(id) - st, err := objectStatus(tx, addr, currEpoch) + st, err := objectStatusWithCache(bc, tx, addr, currEpoch) if err != nil { return nil, err } From af5b3575d0a2a63f6ec87fb58feb28b30419110a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 20 Mar 2025 14:41:13 +0300 Subject: [PATCH 1795/1943] [#1690] qos: Do not export zero metrics counters Signed-off-by: Dmitrii Stepanov --- internal/qos/limiter.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index 8f00791c5..e92cef652 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -211,13 +211,26 @@ func (n *mClockLimiter) startMetricsCollect() { continue } metrics := n.metrics.Load().metrics - for tag, s := range n.readStats { - metrics.SetOperationTagCounters(shardID, "read", tag, s.pending.Load(), s.inProgress.Load(), s.completed.Load(), s.resourceExhausted.Load()) - } - for tag, s := range n.writeStats { - metrics.SetOperationTagCounters(shardID, "write", tag, s.pending.Load(), s.inProgress.Load(), s.completed.Load(), s.resourceExhausted.Load()) - } + exportMetrics(metrics, n.readStats, shardID, "read") + exportMetrics(metrics, n.writeStats, shardID, "write") } } }() } + +func exportMetrics(metrics Metrics, stats map[string]*stat, shardID, operation string) { + var pending uint64 + var inProgress uint64 + var completed uint64 + var resExh uint64 + for tag, s := range stats { + pending = s.pending.Load() + inProgress = s.inProgress.Load() + completed = s.completed.Load() + resExh = s.resourceExhausted.Load() + if pending == 0 && inProgress == 0 && completed == 0 && resExh == 0 { + continue + } + metrics.SetOperationTagCounters(shardID, operation, tag, pending, inProgress, completed, resExh) + } +} From 21bed3362c03730160b5f98fa3dc06c3d4d78cf2 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 20 Mar 2025 10:14:22 +0300 Subject: [PATCH 1796/1943] [#1685] metabase: Cache expired bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ master │ expired │ │ sec/op │ sec/op vs base │ Select/string_equal-8 4.007m ± 10% 3.529m ± 11% -11.94% (p=0.000 n=10) Select/string_not_equal-8 3.834m ± 12% 3.440m ± 7% -10.29% (p=0.029 n=10) Select/common_prefix-8 3.470m ± 9% 3.240m ± 6% ~ (p=0.105 n=10) Select/unknown-8 3.156m ± 3% 3.198m ± 6% ~ (p=0.631 n=10) geomean 3.602m 3.349m -7.03% │ master │ expired │ │ B/op │ B/op vs base │ Select/string_equal-8 1.907Mi ± 0% 1.885Mi ± 0% -1.18% (p=0.000 n=10) Select/string_not_equal-8 1.907Mi ± 0% 1.885Mi ± 0% -1.18% (p=0.000 n=10) Select/common_prefix-8 1.907Mi ± 0% 1.885Mi ± 0% -1.18% (p=0.000 n=10) Select/unknown-8 1.900Mi ± 0% 1.877Mi ± 0% -1.18% (p=0.000 n=10) geomean 1.905Mi 1.883Mi -1.18% │ master │ expired │ │ allocs/op │ allocs/op vs base │ Select/string_equal-8 47.03k ± 0% 46.04k ± 0% -2.12% (p=0.000 n=10) Select/string_not_equal-8 47.03k ± 0% 46.04k ± 0% -2.12% (p=0.000 n=10) Select/common_prefix-8 47.03k ± 0% 46.04k ± 0% -2.12% (p=0.000 n=10) Select/unknown-8 46.04k ± 0% 45.05k ± 0% -2.16% (p=0.000 n=10) geomean 46.78k 45.79k -2.13% ``` Change-Id: I9c7a5e1f5c8b9eb3f25a563fd74c6ad2a9d1b92e Signed-off-by: Evgenii Stratonikov --- .../metabase/bucket_cache.go | 25 ++++++++++++++++++- pkg/local_object_storage/metabase/exists.go | 2 +- pkg/local_object_storage/metabase/expired.go | 8 +++--- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/pkg/local_object_storage/metabase/bucket_cache.go b/pkg/local_object_storage/metabase/bucket_cache.go index b425450af..14c164afc 100644 --- a/pkg/local_object_storage/metabase/bucket_cache.go +++ b/pkg/local_object_storage/metabase/bucket_cache.go @@ -1,6 +1,7 @@ package meta import ( + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.etcd.io/bbolt" ) @@ -8,10 +9,11 @@ type bucketCache struct { locked *bbolt.Bucket graveyard *bbolt.Bucket garbage *bbolt.Bucket + expired map[cid.ID]*bbolt.Bucket } func newBucketCache() *bucketCache { - return &bucketCache{} + return &bucketCache{expired: make(map[cid.ID]*bbolt.Bucket)} } func getLockedBucket(bc *bucketCache, tx *bbolt.Tx) *bbolt.Bucket { @@ -43,3 +45,24 @@ func getBucket(cache **bbolt.Bucket, tx *bbolt.Tx, name []byte) *bbolt.Bucket { *cache = tx.Bucket(name) return *cache } + +func getExpiredBucket(bc *bucketCache, tx *bbolt.Tx, cnr cid.ID) *bbolt.Bucket { + if bc == nil { + bucketName := make([]byte, bucketKeySize) + bucketName = objectToExpirationEpochBucketName(cnr, bucketName) + return tx.Bucket(bucketName) + } + return getMappedBucket(bc.expired, tx, objectToExpirationEpochBucketName, cnr) +} + +func getMappedBucket(m map[cid.ID]*bbolt.Bucket, tx *bbolt.Tx, nameFunc func(cid.ID, []byte) []byte, cnr cid.ID) *bbolt.Bucket { + value, ok := m[cnr] + if ok { + return value + } + + bucketName := make([]byte, bucketKeySize) + bucketName = nameFunc(cnr, bucketName) + m[cnr] = getBucket(&value, tx, bucketName) + return value +} diff --git a/pkg/local_object_storage/metabase/exists.go b/pkg/local_object_storage/metabase/exists.go index 0b28da5c9..7bd6f90a6 100644 --- a/pkg/local_object_storage/metabase/exists.go +++ b/pkg/local_object_storage/metabase/exists.go @@ -162,7 +162,7 @@ func objectStatusWithCache(bc *bucketCache, tx *bbolt.Tx, addr oid.Address, curr return 0, nil } - expired, err := isExpired(tx, addr, currEpoch) + expired, err := isExpiredWithCache(bc, tx, addr, currEpoch) if err != nil { return 0, err } diff --git a/pkg/local_object_storage/metabase/expired.go b/pkg/local_object_storage/metabase/expired.go index 68144d8b1..a1351cb6f 100644 --- a/pkg/local_object_storage/metabase/expired.go +++ b/pkg/local_object_storage/metabase/expired.go @@ -74,9 +74,11 @@ func (db *DB) FilterExpired(ctx context.Context, epoch uint64, addresses []oid.A } func isExpired(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (bool, error) { - bucketName := make([]byte, bucketKeySize) - bucketName = objectToExpirationEpochBucketName(addr.Container(), bucketName) - b := tx.Bucket(bucketName) + return isExpiredWithCache(nil, tx, addr, currEpoch) +} + +func isExpiredWithCache(bc *bucketCache, tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (bool, error) { + b := getExpiredBucket(bc, tx, addr.Container()) if b == nil { return false, nil } From eb9df85b989f861f50891fc21732ce56e56dabc3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 20 Mar 2025 10:23:26 +0300 Subject: [PATCH 1797/1943] [#1685] metabase: Cache primary bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ expired │ primary │ │ sec/op │ sec/op vs base │ Select/string_equal-8 3.529m ± 11% 3.689m ± 7% +4.55% (p=0.023 n=10) Select/string_not_equal-8 3.440m ± 7% 3.543m ± 13% ~ (p=0.190 n=10) Select/common_prefix-8 3.240m ± 6% 3.050m ± 5% -5.85% (p=0.005 n=10) Select/unknown-8 3.198m ± 6% 2.928m ± 8% -8.44% (p=0.003 n=10) geomean 3.349m 3.287m -1.84% │ expired │ primary │ │ B/op │ B/op vs base │ Select/string_equal-8 1.885Mi ± 0% 1.786Mi ± 0% -5.23% (p=0.000 n=10) Select/string_not_equal-8 1.885Mi ± 0% 1.786Mi ± 0% -5.23% (p=0.000 n=10) Select/common_prefix-8 1.885Mi ± 0% 1.786Mi ± 0% -5.23% (p=0.000 n=10) Select/unknown-8 1.877Mi ± 0% 1.779Mi ± 0% -5.26% (p=0.000 n=10) geomean 1.883Mi 1.784Mi -5.24% │ expired │ primary │ │ allocs/op │ allocs/op vs base │ Select/string_equal-8 46.04k ± 0% 43.04k ± 0% -6.50% (p=0.000 n=10) Select/string_not_equal-8 46.04k ± 0% 43.04k ± 0% -6.50% (p=0.000 n=10) Select/common_prefix-8 46.04k ± 0% 43.04k ± 0% -6.50% (p=0.000 n=10) Select/unknown-8 45.05k ± 0% 42.05k ± 0% -6.65% (p=0.000 n=10) geomean 45.79k 42.79k -6.54% ``` Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/bucket_cache.go | 15 ++++++++++++++- pkg/local_object_storage/metabase/get.go | 15 ++++++++++----- pkg/local_object_storage/metabase/select.go | 12 ++++++------ 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/pkg/local_object_storage/metabase/bucket_cache.go b/pkg/local_object_storage/metabase/bucket_cache.go index 14c164afc..69553d55c 100644 --- a/pkg/local_object_storage/metabase/bucket_cache.go +++ b/pkg/local_object_storage/metabase/bucket_cache.go @@ -10,10 +10,14 @@ type bucketCache struct { graveyard *bbolt.Bucket garbage *bbolt.Bucket expired map[cid.ID]*bbolt.Bucket + primary map[cid.ID]*bbolt.Bucket } func newBucketCache() *bucketCache { - return &bucketCache{expired: make(map[cid.ID]*bbolt.Bucket)} + return &bucketCache{ + expired: make(map[cid.ID]*bbolt.Bucket), + primary: make(map[cid.ID]*bbolt.Bucket), + } } func getLockedBucket(bc *bucketCache, tx *bbolt.Tx) *bbolt.Bucket { @@ -55,6 +59,15 @@ func getExpiredBucket(bc *bucketCache, tx *bbolt.Tx, cnr cid.ID) *bbolt.Bucket { return getMappedBucket(bc.expired, tx, objectToExpirationEpochBucketName, cnr) } +func getPrimaryBucket(bc *bucketCache, tx *bbolt.Tx, cnr cid.ID) *bbolt.Bucket { + if bc == nil { + bucketName := make([]byte, bucketKeySize) + bucketName = primaryBucketName(cnr, bucketName) + return tx.Bucket(bucketName) + } + return getMappedBucket(bc.primary, tx, primaryBucketName, cnr) +} + func getMappedBucket(m map[cid.ID]*bbolt.Bucket, tx *bbolt.Tx, nameFunc func(cid.ID, []byte) []byte, cnr cid.ID) *bbolt.Bucket { value, ok := m[cnr] if ok { diff --git a/pkg/local_object_storage/metabase/get.go b/pkg/local_object_storage/metabase/get.go index 615add1af..821810c09 100644 --- a/pkg/local_object_storage/metabase/get.go +++ b/pkg/local_object_storage/metabase/get.go @@ -88,8 +88,12 @@ func (db *DB) Get(ctx context.Context, prm GetPrm) (res GetRes, err error) { } func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw bool, currEpoch uint64) (*objectSDK.Object, error) { + return db.getWithCache(nil, tx, addr, key, checkStatus, raw, currEpoch) +} + +func (db *DB) getWithCache(bc *bucketCache, tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw bool, currEpoch uint64) (*objectSDK.Object, error) { if checkStatus { - st, err := objectStatus(tx, addr, currEpoch) + st, err := objectStatusWithCache(bc, tx, addr, currEpoch) if err != nil { return nil, err } @@ -109,12 +113,13 @@ func (db *DB) get(tx *bbolt.Tx, addr oid.Address, key []byte, checkStatus, raw b bucketName := make([]byte, bucketKeySize) // check in primary index - data := getFromBucket(tx, primaryBucketName(cnr, bucketName), key) - if len(data) != 0 { - return obj, obj.Unmarshal(data) + if b := getPrimaryBucket(bc, tx, cnr); b != nil { + if data := b.Get(key); len(data) != 0 { + return obj, obj.Unmarshal(data) + } } - data = getFromBucket(tx, ecInfoBucketName(cnr, bucketName), key) + data := getFromBucket(tx, ecInfoBucketName(cnr, bucketName), key) if len(data) != 0 { return nil, getECInfoError(tx, cnr, data) } diff --git a/pkg/local_object_storage/metabase/select.go b/pkg/local_object_storage/metabase/select.go index a95384753..60da50671 100644 --- a/pkg/local_object_storage/metabase/select.go +++ b/pkg/local_object_storage/metabase/select.go @@ -154,7 +154,7 @@ func (db *DB) selectObjects(tx *bbolt.Tx, cnr cid.ID, fs objectSDK.SearchFilters continue // ignore removed objects } - addr, match := db.matchSlowFilters(tx, addr, group.slowFilters, currEpoch) + addr, match := db.matchSlowFilters(bc, tx, addr, group.slowFilters, currEpoch) if !match { continue // ignore objects with unmatched slow filters } @@ -452,13 +452,13 @@ func (db *DB) selectObjectID( } // matchSlowFilters return true if object header is matched by all slow filters. -func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.SearchFilters, currEpoch uint64) (oid.Address, bool) { +func (db *DB) matchSlowFilters(bc *bucketCache, tx *bbolt.Tx, addr oid.Address, f objectSDK.SearchFilters, currEpoch uint64) (oid.Address, bool) { result := addr if len(f) == 0 { return result, true } - obj, isECChunk, err := db.getObjectForSlowFilters(tx, addr, currEpoch) + obj, isECChunk, err := db.getObjectForSlowFilters(bc, tx, addr, currEpoch) if err != nil { return result, false } @@ -516,9 +516,9 @@ func (db *DB) matchSlowFilters(tx *bbolt.Tx, addr oid.Address, f objectSDK.Searc return result, true } -func (db *DB) getObjectForSlowFilters(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (*objectSDK.Object, bool, error) { +func (db *DB) getObjectForSlowFilters(bc *bucketCache, tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (*objectSDK.Object, bool, error) { buf := make([]byte, addressKeySize) - obj, err := db.get(tx, addr, buf, false, false, currEpoch) + obj, err := db.getWithCache(bc, tx, addr, buf, false, false, currEpoch) if err != nil { var ecInfoError *objectSDK.ECInfoError if errors.As(err, &ecInfoError) { @@ -528,7 +528,7 @@ func (db *DB) getObjectForSlowFilters(tx *bbolt.Tx, addr oid.Address, currEpoch continue } addr.SetObject(objID) - obj, err = db.get(tx, addr, buf, true, false, currEpoch) + obj, err = db.getWithCache(bc, tx, addr, buf, true, false, currEpoch) if err == nil { return obj, true, nil } From e8801dbf49c2407bb7cbd25d003d7a9b95fc7c3b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 20 Mar 2025 15:28:14 +0300 Subject: [PATCH 1798/1943] [#1691] metabase: Move cheaper conditions to the front in ListWithCursor() `objectLocked` call is expensive, it does IO. We may omit it if object is not expired. Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/list.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index eaef3b9ba..0b6cdf702 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -251,7 +251,7 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket } expEpoch, hasExpEpoch := hasExpirationEpoch(&o) - if !objectLocked(bkt.Tx(), cnt, obj) && hasExpEpoch && expEpoch < currEpoch { + if hasExpEpoch && expEpoch < currEpoch && !objectLocked(bkt.Tx(), cnt, obj) { continue } From 45b779615104ed790f65f4776fb59cc6c163d0f6 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 11 Mar 2025 16:57:07 +0300 Subject: [PATCH 1799/1943] [#1689] ci: Reimplement CI tasks in Jenkinsfile This commit introduces Jenkins pipeline that duplicates the features of existing Forgejo Actions workflows. Change-Id: I657a6c27373a1ed4736ae27b4fb660e0ac86012d Signed-off-by: Vitaliy Potyarkin --- .ci/Jenkinsfile | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .ci/Jenkinsfile diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile new file mode 100644 index 000000000..e21ce61c5 --- /dev/null +++ b/.ci/Jenkinsfile @@ -0,0 +1,81 @@ +def golang = ['1.23', '1.24'] +def golangDefault = "golang:${golang.last()}" + +async { + + for (version in golang) { + def go = version + + task("test/go${go}") { + container("golang:${go}") { + sh 'make test' + } + } + + task("build/go${go}") { + container("golang:${go}") { + for (app in ['cli', 'node', 'ir', 'adm', 'lens']) { + sh """ + make bin/frostfs-${app} + bin/frostfs-${app} --version + """ + } + } + } + } + + task('test/race') { + container(golangDefault) { + sh 'make test GOFLAGS="-count=1 -race"' + } + } + + task('lint') { + container(golangDefault) { + sh 'make lint-install lint' + } + } + + task('staticcheck') { + container(golangDefault) { + sh 'make staticcheck-install staticcheck-run' + } + } + + task('gopls') { + container(golangDefault) { + sh 'make gopls-install gopls-run' + } + } + + task('gofumpt') { + container(golangDefault) { + sh ''' + make fumpt-install + make fumpt + git diff --exit-code --quiet + ''' + } + } + + task('vulncheck') { + container(golangDefault) { + sh ''' + go install golang.org/x/vuln/cmd/govulncheck@latest + govulncheck ./... + ''' + } + } + + task('pre-commit') { + sh ''' + apt update + apt install -y --no-install-recommends pre-commit + ''' // TODO: Make an OCI image for pre-commit + golang? Unpack golang tarball with a library function? + withEnv(['SKIP=make-lint,go-staticcheck-repo-mod,go-unit-tests,gofumpt']) { + sh 'pre-commit run --color=always --hook-stage=manual --all-files' + } + } +} + +// TODO: dco check From affab255120dd54fc581a26fbdcac2cf9a82346a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 13 Mar 2025 10:26:08 +0300 Subject: [PATCH 1800/1943] Makefile: Add Gerrit-related targets This commit adds helper targets to easily setup an existing repo for work with Gerrit. Change-Id: I0696eb8ea84cc16a9482be6a2fb0382fe624bb96 Signed-off-by: Evgenii Stratonikov --- Makefile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Makefile b/Makefile index cd80fc72e..46168719d 100755 --- a/Makefile +++ b/Makefile @@ -186,6 +186,20 @@ test: @echo "⇒ Running go test" @GOFLAGS="$(GOFLAGS)" go test ./... +# Install Gerrit commit-msg hook +review-install: GIT_HOOK_DIR := $(shell git rev-parse --git-dir)/hooks +review-install: + @git config remote.review.url \ + || git remote add review ssh://review.frostfs.info:2222/TrueCloudLab/frostfs-node + @mkdir -p $(GIT_HOOK_DIR)/ + @curl -Lo $(GIT_HOOK_DIR)/commit-msg https://review.frostfs.info/tools/hooks/commit-msg + @chmod +x $(GIT_HOOK_DIR)/commit-msg + +# Create a PR in Gerrit +review: BRANCH ?= master +review: + @git push review HEAD:refs/for/$(BRANCH) + # Run pre-commit pre-commit-run: @pre-commit run -a --hook-stage manual From 7df2912a83f8c4b02e89d6311a10f0bdb0a6eddc Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 20 Mar 2025 17:26:35 +0300 Subject: [PATCH 1801/1943] [#1689] Makefile: Create prepare-commit-msg hook too `commit-msg` is ignored when `--no-verify` option is used, so there is no way to ignore `pre-commit` while retaining `commit-msg` hook. Ignoring pre-commit is useful, though, so we might add Change-Id in `prepare-commit-msg` hook instead. It accepts more parameters, but the first one is a file with the commit message, so we may reuse `commit-msg` hook. Change-Id: I4edb79810bbe38a5dcf7f4f07535f34c6bda0da3 Signed-off-by: Evgenii Stratonikov --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 46168719d..210ea1dc1 100755 --- a/Makefile +++ b/Makefile @@ -194,6 +194,8 @@ review-install: @mkdir -p $(GIT_HOOK_DIR)/ @curl -Lo $(GIT_HOOK_DIR)/commit-msg https://review.frostfs.info/tools/hooks/commit-msg @chmod +x $(GIT_HOOK_DIR)/commit-msg + @echo -e '#!/bin/sh\n"$$(git rev-parse --git-path hooks)"/commit-msg "$$1"' >$(GIT_HOOK_DIR)/prepare-commit-msg + @chmod +x $(GIT_HOOK_DIR)/prepare-commit-msg # Create a PR in Gerrit review: BRANCH ?= master From 60cea8c714fcbad23d86143c9c1949b09d5444e6 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 20 Mar 2025 16:22:22 +0300 Subject: [PATCH 1802/1943] [#1692] metabase/test: Fix end of iteration error check This is not good: ``` BenchmarkListWithCursor/1_item-8 --- FAIL: BenchmarkListWithCursor/1_item-8 list_test.go:63: error: end of object listing ``` Change-Id: I61b70937ce30fefaf16ebeb0cdb51bdd39096061 Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/list_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/metabase/list_test.go b/pkg/local_object_storage/metabase/list_test.go index 817b22010..02985991c 100644 --- a/pkg/local_object_storage/metabase/list_test.go +++ b/pkg/local_object_storage/metabase/list_test.go @@ -59,7 +59,7 @@ func benchmarkListWithCursor(b *testing.B, db *meta.DB, batchSize int) { for range b.N { res, err := db.ListWithCursor(context.Background(), prm) if err != nil { - if errors.Is(err, meta.ErrEndOfListing) { + if !errors.Is(err, meta.ErrEndOfListing) { b.Fatalf("error: %v", err) } prm.SetCursor(nil) From 3f4717a37fbb27ba59cc63065a83bfc18eac7a8b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 20 Mar 2025 16:35:01 +0300 Subject: [PATCH 1803/1943] [#1692] metabase: Do not allocate map in cache unless needed Change-Id: I8b1015a8c7c3df4153a08fdb788117d9f0d6c333 Signed-off-by: Evgenii Stratonikov --- .../metabase/bucket_cache.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pkg/local_object_storage/metabase/bucket_cache.go b/pkg/local_object_storage/metabase/bucket_cache.go index 69553d55c..de1479e6f 100644 --- a/pkg/local_object_storage/metabase/bucket_cache.go +++ b/pkg/local_object_storage/metabase/bucket_cache.go @@ -14,10 +14,7 @@ type bucketCache struct { } func newBucketCache() *bucketCache { - return &bucketCache{ - expired: make(map[cid.ID]*bbolt.Bucket), - primary: make(map[cid.ID]*bbolt.Bucket), - } + return &bucketCache{} } func getLockedBucket(bc *bucketCache, tx *bbolt.Tx) *bbolt.Bucket { @@ -56,7 +53,7 @@ func getExpiredBucket(bc *bucketCache, tx *bbolt.Tx, cnr cid.ID) *bbolt.Bucket { bucketName = objectToExpirationEpochBucketName(cnr, bucketName) return tx.Bucket(bucketName) } - return getMappedBucket(bc.expired, tx, objectToExpirationEpochBucketName, cnr) + return getMappedBucket(&bc.expired, tx, objectToExpirationEpochBucketName, cnr) } func getPrimaryBucket(bc *bucketCache, tx *bbolt.Tx, cnr cid.ID) *bbolt.Bucket { @@ -65,17 +62,21 @@ func getPrimaryBucket(bc *bucketCache, tx *bbolt.Tx, cnr cid.ID) *bbolt.Bucket { bucketName = primaryBucketName(cnr, bucketName) return tx.Bucket(bucketName) } - return getMappedBucket(bc.primary, tx, primaryBucketName, cnr) + return getMappedBucket(&bc.primary, tx, primaryBucketName, cnr) } -func getMappedBucket(m map[cid.ID]*bbolt.Bucket, tx *bbolt.Tx, nameFunc func(cid.ID, []byte) []byte, cnr cid.ID) *bbolt.Bucket { - value, ok := m[cnr] +func getMappedBucket(m *map[cid.ID]*bbolt.Bucket, tx *bbolt.Tx, nameFunc func(cid.ID, []byte) []byte, cnr cid.ID) *bbolt.Bucket { + value, ok := (*m)[cnr] if ok { return value } + if *m == nil { + *m = make(map[cid.ID]*bbolt.Bucket, 1) + } + bucketName := make([]byte, bucketKeySize) bucketName = nameFunc(cnr, bucketName) - m[cnr] = getBucket(&value, tx, bucketName) + (*m)[cnr] = getBucket(&value, tx, bucketName) return value } From 049a650b89554a1f4f879048edb3555be6b387ec Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 28 Jan 2025 13:29:10 +0300 Subject: [PATCH 1804/1943] [#1619] logger: Simplify `logger` config reloading Change-Id: Ide892b250304b8cdb6c279f5f728c3b35f05df54 Signed-off-by: Anton Nikiforov --- cmd/frostfs-ir/config.go | 6 ++- cmd/frostfs-ir/main.go | 2 +- cmd/frostfs-node/config.go | 41 ++++++++++---------- pkg/services/object/common/writer/ec_test.go | 2 +- pkg/util/logger/logger.go | 40 +++---------------- 5 files changed, 31 insertions(+), 60 deletions(-) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 09af08525..19b7f05d6 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -9,6 +9,7 @@ import ( configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -38,13 +39,14 @@ func reloadConfig() error { } cmode.Store(cfg.GetBool("node.kludge_compatibility_mode")) audit.Store(cfg.GetBool("audit.enabled")) + var logPrm logger.Prm err = logPrm.SetLevelString(cfg.GetString("logger.level")) if err != nil { return err } - logPrm.PrependTimestamp = cfg.GetBool("logger.timestamp") + log.Reload(logPrm) - return logPrm.Reload() + return nil } func watchForSignal(ctx context.Context, cancel func()) { diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index ade64ba84..114d8e4de 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -31,7 +31,6 @@ const ( var ( wg = new(sync.WaitGroup) intErr = make(chan error) // internal inner ring errors - logPrm = new(logger.Prm) innerRing *innerring.Server pprofCmp *pprofComponent metricsCmp *httpComponent @@ -70,6 +69,7 @@ func main() { metrics := irMetrics.NewInnerRingMetrics() + var logPrm logger.Prm err = logPrm.SetLevelString( cfg.GetString("logger.level"), ) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 2531e9173..c3c687763 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -473,7 +473,6 @@ type shared struct { // dynamicConfiguration stores parameters of the // components that supports runtime reconfigurations. type dynamicConfiguration struct { - logger *logger.Prm pprof *httpComponent metrics *httpComponent } @@ -714,7 +713,8 @@ func initCfg(appCfg *config.Config) *cfg { netState.metrics = c.metricsCollector - logPrm := c.loggerPrm() + logPrm, err := c.loggerPrm() + fatalOnErr(err) logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook() log, err := logger.NewLogger(logPrm) fatalOnErr(err) @@ -1076,26 +1076,22 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID return sh } -func (c *cfg) loggerPrm() *logger.Prm { - // check if it has been inited before - if c.dynamicConfiguration.logger == nil { - c.dynamicConfiguration.logger = new(logger.Prm) - } - +func (c *cfg) loggerPrm() (logger.Prm, error) { + var prm logger.Prm // (re)init read configuration - err := c.dynamicConfiguration.logger.SetLevelString(c.LoggerCfg.level) + err := prm.SetLevelString(c.LoggerCfg.level) if err != nil { // not expected since validation should be performed before - panic("incorrect log level format: " + c.LoggerCfg.level) + return logger.Prm{}, errors.New("incorrect log level format: " + c.LoggerCfg.level) } - err = c.dynamicConfiguration.logger.SetDestination(c.LoggerCfg.destination) + err = prm.SetDestination(c.LoggerCfg.destination) if err != nil { // not expected since validation should be performed before - panic("incorrect log destination format: " + c.LoggerCfg.destination) + return logger.Prm{}, errors.New("incorrect log destination format: " + c.LoggerCfg.destination) } - c.dynamicConfiguration.logger.PrependTimestamp = c.LoggerCfg.timestamp + prm.PrependTimestamp = c.LoggerCfg.timestamp - return c.dynamicConfiguration.logger + return prm, nil } func (c *cfg) LocalAddress() network.AddressGroup { @@ -1335,11 +1331,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { // all the components are expected to support // Logger's dynamic reconfiguration approach - // Logger - - logPrm := c.loggerPrm() - - components := c.getComponents(ctx, logPrm) + components := c.getComponents(ctx) // Object c.cfgObject.tombstoneLifetime.Store(c.ObjectCfg.tombstoneLifetime) @@ -1377,10 +1369,17 @@ func (c *cfg) reloadConfig(ctx context.Context) { c.log.Info(ctx, logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) } -func (c *cfg) getComponents(ctx context.Context, logPrm *logger.Prm) []dCmp { +func (c *cfg) getComponents(ctx context.Context) []dCmp { var components []dCmp - components = append(components, dCmp{"logger", logPrm.Reload}) + components = append(components, dCmp{"logger", func() error { + prm, err := c.loggerPrm() + if err != nil { + return err + } + c.log.Reload(prm) + return nil + }}) components = append(components, dCmp{"runtime", func() error { setRuntimeParameters(ctx, c) return nil diff --git a/pkg/services/object/common/writer/ec_test.go b/pkg/services/object/common/writer/ec_test.go index 2458e352f..d5eeddf21 100644 --- a/pkg/services/object/common/writer/ec_test.go +++ b/pkg/services/object/common/writer/ec_test.go @@ -130,7 +130,7 @@ func TestECWriter(t *testing.T) { nodeKey, err := keys.NewPrivateKey() require.NoError(t, err) - log, err := logger.NewLogger(nil) + log, err := logger.NewLogger(logger.Prm{}) require.NoError(t, err) var n nmKeys diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index 2eb5e5538..952a6f2dc 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -23,16 +23,8 @@ type Logger struct { // Parameters that have been connected to the Logger support its // configuration changing. // -// Passing Prm after a successful connection via the NewLogger, connects -// the Prm to a new instance of the Logger. -// -// See also Reload, SetLevelString. +// See also Logger.Reload, SetLevelString. type Prm struct { - // link to the created Logger - // instance; used for a runtime - // reconfiguration - _log *Logger - // support runtime rereading level zapcore.Level @@ -73,22 +65,6 @@ func (p *Prm) SetDestination(d string) error { return nil } -// Reload reloads configuration of a connected instance of the Logger. -// Returns ErrLoggerNotConnected if no connection has been performed. -// Returns any reconfiguration error from the Logger directly. -func (p Prm) Reload() error { - if p._log == nil { - // incorrect logger usage - panic("parameters are not connected to any Logger") - } - - return p._log.reload(p) -} - -func defaultPrm() *Prm { - return new(Prm) -} - // NewLogger constructs a new zap logger instance. Constructing with nil // parameters is safe: default values will be used then. // Passing non-nil parameters after a successful creation (non-error) allows @@ -100,10 +76,7 @@ func defaultPrm() *Prm { // - ISO8601 time encoding. // // Logger records a stack trace for all messages at or above fatal level. -func NewLogger(prm *Prm) (*Logger, error) { - if prm == nil { - prm = defaultPrm() - } +func NewLogger(prm Prm) (*Logger, error) { switch prm.dest { case DestinationUndefined, DestinationStdout: return newConsoleLogger(prm) @@ -114,7 +87,7 @@ func NewLogger(prm *Prm) (*Logger, error) { } } -func newConsoleLogger(prm *Prm) (*Logger, error) { +func newConsoleLogger(prm Prm) (*Logger, error) { lvl := zap.NewAtomicLevelAt(prm.level) c := zap.NewProductionConfig() @@ -139,12 +112,11 @@ func newConsoleLogger(prm *Prm) (*Logger, error) { } l := &Logger{z: lZap, lvl: lvl} - prm._log = l return l, nil } -func newJournaldLogger(prm *Prm) (*Logger, error) { +func newJournaldLogger(prm Prm) (*Logger, error) { lvl := zap.NewAtomicLevelAt(prm.level) c := zap.NewProductionConfig() @@ -181,14 +153,12 @@ func newJournaldLogger(prm *Prm) (*Logger, error) { lZap := zap.New(samplingCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), zap.AddCallerSkip(1)) l := &Logger{z: lZap, lvl: lvl} - prm._log = l return l, nil } -func (l *Logger) reload(prm Prm) error { +func (l *Logger) Reload(prm Prm) { l.lvl.SetLevel(prm.level) - return nil } func (l *Logger) WithOptions(options ...zap.Option) { From eea46a599d5e8c07d354900152ba62584b9c8d44 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 21 Mar 2025 11:20:53 +0300 Subject: [PATCH 1805/1943] [#1695] qos: Add treesync tag Tree sync is too much different from GC and rebuild to use the same tag for GC and tree sync. Change-Id: Ib44d5fa9a88daff507d759d0b0410cc9272e236f Signed-off-by: Dmitrii Stepanov --- internal/qos/grpc.go | 4 ++-- internal/qos/limiter.go | 2 +- internal/qos/stats.go | 1 + internal/qos/tags.go | 7 +++++++ internal/qos/validate.go | 1 + pkg/services/tree/service.go | 2 +- 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/internal/qos/grpc.go b/internal/qos/grpc.go index 534a1f74b..58cd9e52c 100644 --- a/internal/qos/grpc.go +++ b/internal/qos/grpc.go @@ -26,7 +26,7 @@ func NewAdjustOutgoingIOTagUnaryClientInterceptor() grpc.UnaryClientInterceptor if err != nil { tag = IOTagClient } - if tag == IOTagBackground || tag == IOTagPolicer || tag == IOTagWritecache { + if tag.IsLocal() { tag = IOTagInternal } ctx = tagging.ContextWithIOTag(ctx, tag.String()) @@ -44,7 +44,7 @@ func NewAdjustOutgoingIOTagStreamClientInterceptor() grpc.StreamClientIntercepto if err != nil { tag = IOTagClient } - if tag == IOTagBackground || tag == IOTagPolicer || tag == IOTagWritecache { + if tag.IsLocal() { tag = IOTagInternal } ctx = tagging.ContextWithIOTag(ctx, tag.String()) diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index e92cef652..82f9917a5 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -74,7 +74,7 @@ func createScheduler(config limits.OpConfig) (scheduler, error) { func converToSchedulingTags(limits []limits.IOTagConfig) map[string]scheduling.TagInfo { result := make(map[string]scheduling.TagInfo) - for _, tag := range []IOTag{IOTagClient, IOTagBackground, IOTagInternal, IOTagPolicer, IOTagWritecache} { + for _, tag := range []IOTag{IOTagClient, IOTagBackground, IOTagInternal, IOTagPolicer, IOTagWritecache, IOTagTreeSync} { result[tag.String()] = scheduling.TagInfo{ Share: defaultShare, } diff --git a/internal/qos/stats.go b/internal/qos/stats.go index f077f552b..aa4d4caf9 100644 --- a/internal/qos/stats.go +++ b/internal/qos/stats.go @@ -9,6 +9,7 @@ var statTags = map[string]struct{}{ IOTagPolicer.String(): {}, IOTagWritecache.String(): {}, IOTagCritical.String(): {}, + IOTagTreeSync.String(): {}, unknownStatsTag: {}, } diff --git a/internal/qos/tags.go b/internal/qos/tags.go index 9db45f190..2781dec76 100644 --- a/internal/qos/tags.go +++ b/internal/qos/tags.go @@ -13,6 +13,7 @@ const ( IOTagClient IOTag = "client" IOTagInternal IOTag = "internal" IOTagBackground IOTag = "background" + IOTagTreeSync IOTag = "treesync" IOTagWritecache IOTag = "writecache" IOTagPolicer IOTag = "policer" IOTagCritical IOTag = "critical" @@ -34,6 +35,8 @@ func FromRawString(s string) (IOTag, error) { return IOTagWritecache, nil case string(IOTagPolicer): return IOTagPolicer, nil + case string(IOTagTreeSync): + return IOTagTreeSync, nil default: return ioTagUnknown, fmt.Errorf("unknown tag %s", s) } @@ -50,3 +53,7 @@ func IOTagFromContext(ctx context.Context) string { } return tag } + +func (t IOTag) IsLocal() bool { + return t == IOTagBackground || t == IOTagPolicer || t == IOTagWritecache || t == IOTagTreeSync +} diff --git a/internal/qos/validate.go b/internal/qos/validate.go index 3fa4ebbd1..75bf4f4b9 100644 --- a/internal/qos/validate.go +++ b/internal/qos/validate.go @@ -47,6 +47,7 @@ func validateTags(configTags []limits.IOTagConfig) error { IOTagBackground: {}, IOTagWritecache: {}, IOTagPolicer: {}, + IOTagTreeSync: {}, } for _, t := range configTags { tag, err := FromRawString(t.Tag) diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index eeffec08b..b9bb96bab 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -85,7 +85,7 @@ func New(opts ...Option) *Service { // Start starts the service. func (s *Service) Start(ctx context.Context) { - ctx = tagging.ContextWithIOTag(ctx, qos.IOTagBackground.String()) + ctx = tagging.ContextWithIOTag(ctx, qos.IOTagTreeSync.String()) go s.replicateLoop(ctx) go s.syncLoop(ctx) From 5385f9994fd4decefbbdd82a85aecedb55671872 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 21 Mar 2025 11:59:25 +0300 Subject: [PATCH 1806/1943] [#1695] mod: Bump frostfs-observability version Change-Id: Id362b71f743ff70c8cd374030c9fa67e2566022f Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/internal/client/sdk.go | 2 +- cmd/frostfs-cli/modules/tree/client.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- pkg/network/cache/multi.go | 2 +- pkg/services/tree/cache.go | 2 +- pkg/services/tree/sync.go | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-cli/internal/client/sdk.go b/cmd/frostfs-cli/internal/client/sdk.go index 2d9c45cbd..1eadfa2e1 100644 --- a/cmd/frostfs-cli/internal/client/sdk.go +++ b/cmd/frostfs-cli/internal/client/sdk.go @@ -56,7 +56,7 @@ func GetSDKClient(ctx context.Context, cmd *cobra.Command, key *ecdsa.PrivateKey prmDial := client.PrmDial{ Endpoint: addr.URIAddr(), GRPCDialOptions: []grpc.DialOption{ - grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInteceptor()), + grpc.WithChainUnaryInterceptor(tracing.NewUnaryClientInterceptor()), grpc.WithChainStreamInterceptor(tracing.NewStreamClientInterceptor()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), }, diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index 933378df6..421b96ccd 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -33,7 +33,7 @@ func _client() (tree.TreeServiceClient, error) { opts := []grpc.DialOption{ grpc.WithChainUnaryInterceptor( - tracing.NewUnaryClientInteceptor(), + tracing.NewUnaryClientInterceptor(), ), grpc.WithChainStreamInterceptor( tracing.NewStreamClientInterceptor(), diff --git a/go.mod b/go.mod index eeaca1645..fafb4f828 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d - git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 + git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250310135838-3e7ca9403529 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9 git.frostfs.info/TrueCloudLab/hrw v1.2.1 diff --git a/go.sum b/go.sum index a8f7216a5..7818583d4 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d h1:uJ/wvuMdepbkaV8XMS5uN9B0FQWMep0CttSuDZiDhq0= git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824 h1:Mxw1c/8t96vFIUOffl28lFaHKi413oCBfLMGJmF9cFA= -git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250212111929-d34e1329c824/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 h1:fluzML8BIIabd07LyPSjc0JAV2qymWkPiFaLrXdALLA= +git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250310135838-3e7ca9403529 h1:CBreXSxGoYJAdZ1QdJPsDs1UCXGF5psinII0lxtohsc= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250310135838-3e7ca9403529/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9 h1:svCl6NDAPZ/KuQPjdVKo74RkCIANesxUPM45zQZDhSw= diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index e94fa580a..77420865a 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -66,7 +66,7 @@ func (x *multiClient) createForAddress(ctx context.Context, addr network.Address grpc.WithChainUnaryInterceptor( qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), metrics.NewUnaryClientInterceptor(), - tracing.NewUnaryClientInteceptor(), + tracing.NewUnaryClientInterceptor(), tagging.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 70f4a843b..3359af2c5 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -99,7 +99,7 @@ func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (* grpc.WithChainUnaryInterceptor( qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), metrics.NewUnaryClientInterceptor(), - tracing.NewUnaryClientInteceptor(), + tracing.NewUnaryClientInterceptor(), tagging.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 89450b739..103e2a613 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -344,7 +344,7 @@ func (*Service) createConnection(a network.Address) (*grpc.ClientConn, error) { grpc.WithChainUnaryInterceptor( qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), metrics.NewUnaryClientInterceptor(), - tracing_grpc.NewUnaryClientInteceptor(), + tracing_grpc.NewUnaryClientInterceptor(), tagging.NewUnaryClientInteceptor(), ), grpc.WithChainStreamInterceptor( From 3fa5c22ddfc036a414e66fb0e89c816e44a677dc Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 13:18:18 +0300 Subject: [PATCH 1807/1943] [#1689] Makefile: Add default reviewers via --push-option Gerrit doesn't provide an easy way to have default reviewers assigned to new change requests. However, we can use `--push-option` and mention all people from storage-core-developers group. Change-Id: Ia01f8a3c5c8eb8a1dca6efb66fbe07018f6a42c9 Signed-off-by: Evgenii Stratonikov --- Makefile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 210ea1dc1..c9147f511 100755 --- a/Makefile +++ b/Makefile @@ -200,7 +200,14 @@ review-install: # Create a PR in Gerrit review: BRANCH ?= master review: - @git push review HEAD:refs/for/$(BRANCH) + @git push review HEAD:refs/for/$(BRANCH) \ + --push-option r=e.stratonikov@yadro.com \ + --push-option r=d.stepanov@yadro.com \ + --push-option r=an.nikiforov@yadro.com \ + --push-option r=a.arifullin@yadro.com \ + --push-option r=ekaterina.lebedeva@yadro.com \ + --push-option r=a.savchuk@yadro.com \ + --push-option r=a.chuprov@yadro.com # Run pre-commit pre-commit-run: From af76350bfb2eaabb22ad19fccf1412d26a226004 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 21 Mar 2025 13:19:28 +0300 Subject: [PATCH 1808/1943] [#1695] qos: Sort tags by asc Change-Id: Ia23e392bb49d2536096de2ba07fc6f8fb7ac0489 Signed-off-by: Dmitrii Stepanov --- internal/qos/limiter.go | 2 +- internal/qos/stats.go | 6 +++--- internal/qos/tags.go | 24 ++++++++++++------------ internal/qos/validate.go | 4 ++-- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index 82f9917a5..98d254fd0 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -74,7 +74,7 @@ func createScheduler(config limits.OpConfig) (scheduler, error) { func converToSchedulingTags(limits []limits.IOTagConfig) map[string]scheduling.TagInfo { result := make(map[string]scheduling.TagInfo) - for _, tag := range []IOTag{IOTagClient, IOTagBackground, IOTagInternal, IOTagPolicer, IOTagWritecache, IOTagTreeSync} { + for _, tag := range []IOTag{IOTagBackground, IOTagClient, IOTagInternal, IOTagPolicer, IOTagTreeSync, IOTagWritecache} { result[tag.String()] = scheduling.TagInfo{ Share: defaultShare, } diff --git a/internal/qos/stats.go b/internal/qos/stats.go index aa4d4caf9..3ecfad9f9 100644 --- a/internal/qos/stats.go +++ b/internal/qos/stats.go @@ -3,13 +3,13 @@ package qos const unknownStatsTag = "unknown" var statTags = map[string]struct{}{ - IOTagClient.String(): {}, IOTagBackground.String(): {}, + IOTagClient.String(): {}, + IOTagCritical.String(): {}, IOTagInternal.String(): {}, IOTagPolicer.String(): {}, - IOTagWritecache.String(): {}, - IOTagCritical.String(): {}, IOTagTreeSync.String(): {}, + IOTagWritecache.String(): {}, unknownStatsTag: {}, } diff --git a/internal/qos/tags.go b/internal/qos/tags.go index 2781dec76..e3f7cafd6 100644 --- a/internal/qos/tags.go +++ b/internal/qos/tags.go @@ -10,33 +10,33 @@ import ( type IOTag string const ( - IOTagClient IOTag = "client" - IOTagInternal IOTag = "internal" IOTagBackground IOTag = "background" + IOTagClient IOTag = "client" + IOTagCritical IOTag = "critical" + IOTagInternal IOTag = "internal" + IOTagPolicer IOTag = "policer" IOTagTreeSync IOTag = "treesync" IOTagWritecache IOTag = "writecache" - IOTagPolicer IOTag = "policer" - IOTagCritical IOTag = "critical" ioTagUnknown IOTag = "" ) func FromRawString(s string) (IOTag, error) { switch s { - case string(IOTagCritical): - return IOTagCritical, nil - case string(IOTagClient): - return IOTagClient, nil - case string(IOTagInternal): - return IOTagInternal, nil case string(IOTagBackground): return IOTagBackground, nil - case string(IOTagWritecache): - return IOTagWritecache, nil + case string(IOTagClient): + return IOTagClient, nil + case string(IOTagCritical): + return IOTagCritical, nil + case string(IOTagInternal): + return IOTagInternal, nil case string(IOTagPolicer): return IOTagPolicer, nil case string(IOTagTreeSync): return IOTagTreeSync, nil + case string(IOTagWritecache): + return IOTagWritecache, nil default: return ioTagUnknown, fmt.Errorf("unknown tag %s", s) } diff --git a/internal/qos/validate.go b/internal/qos/validate.go index 75bf4f4b9..d4475e38b 100644 --- a/internal/qos/validate.go +++ b/internal/qos/validate.go @@ -42,12 +42,12 @@ func validateOpConfig(c limits.OpConfig) error { func validateTags(configTags []limits.IOTagConfig) error { tags := map[IOTag]tagConfig{ + IOTagBackground: {}, IOTagClient: {}, IOTagInternal: {}, - IOTagBackground: {}, - IOTagWritecache: {}, IOTagPolicer: {}, IOTagTreeSync: {}, + IOTagWritecache: {}, } for _, t := range configTags { tag, err := FromRawString(t.Tag) From 9aa486c9d814d3feed94a99d24aa1914a57b17fa Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 14:10:46 +0300 Subject: [PATCH 1809/1943] [#1689] Makefile: Create dirs with -p flag On CI there is no `bin` directory initially, so an error occurs ``` mkdir: cannot create directory '/var/cache/jenkins-agent/workspace/gerrit/frostfs-node#55-488b12-8ac3c/bin/gofumpt': No such file or directory ``` Change-Id: I43895c8f5ed7cc5c71c8025228710279f9e75e9c Signed-off-by: Evgenii Stratonikov --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index c9147f511..2c1cf89f7 100755 --- a/Makefile +++ b/Makefile @@ -115,7 +115,7 @@ protoc: # Install protoc protoc-install: @rm -rf $(PROTOBUF_DIR) - @mkdir $(PROTOBUF_DIR) + @mkdir -p $(PROTOBUF_DIR) @echo "⇒ Installing protoc... " @wget -q -O $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip 'https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_OS_VERSION).zip' @unzip -q -o $(PROTOBUF_DIR)/protoc-$(PROTOC_VERSION).zip -d $(PROTOC_DIR) @@ -169,7 +169,7 @@ imports: # Install gofumpt fumpt-install: @rm -rf $(GOFUMPT_DIR) - @mkdir $(GOFUMPT_DIR) + @mkdir -p $(GOFUMPT_DIR) @GOBIN=$(GOFUMPT_VERSION_DIR) go install mvdan.cc/gofumpt@$(GOFUMPT_VERSION) # Run gofumpt @@ -214,9 +214,9 @@ pre-commit-run: @pre-commit run -a --hook-stage manual # Install linters -lint-install: +lint-install: $(BIN) @rm -rf $(OUTPUT_LINT_DIR) - @mkdir $(OUTPUT_LINT_DIR) + @mkdir -p $(OUTPUT_LINT_DIR) @mkdir -p $(TMP_DIR) @rm -rf $(TMP_DIR)/linters @git -c advice.detachedHead=false clone --branch v$(TRUECLOUDLAB_LINT_VERSION) https://git.frostfs.info/TrueCloudLab/linters.git $(TMP_DIR)/linters @@ -235,7 +235,7 @@ lint: # Install staticcheck staticcheck-install: @rm -rf $(STATICCHECK_DIR) - @mkdir $(STATICCHECK_DIR) + @mkdir -p $(STATICCHECK_DIR) @GOBIN=$(STATICCHECK_VERSION_DIR) go install honnef.co/go/tools/cmd/staticcheck@$(STATICCHECK_VERSION) # Run staticcheck @@ -248,7 +248,7 @@ staticcheck-run: # Install gopls gopls-install: @rm -rf $(GOPLS_DIR) - @mkdir $(GOPLS_DIR) + @mkdir -p $(GOPLS_DIR) @GOBIN=$(GOPLS_VERSION_DIR) go install golang.org/x/tools/gopls@$(GOPLS_VERSION) # Run gopls From 016f2e11e371b8cbb5b6459aed33b2e3c962fc3e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 14:38:18 +0300 Subject: [PATCH 1810/1943] [#1689] Makefile: Add more restricted .SHELLFLAGS Catch more errors immediately. Change-Id: I576f1b394a2b167c78c693a794ab8cca3ac1013b Signed-off-by: Evgenii Stratonikov --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 2c1cf89f7..321365f0d 100755 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ #!/usr/bin/make -f SHELL = bash +.SHELLFLAGS = -euo pipefail -c REPO ?= $(shell go list -m) VERSION ?= $(shell git describe --tags --dirty --match "v*" --always --abbrev=8 2>/dev/null || cat VERSION 2>/dev/null || echo "develop") From d95128913123f7c3c1fbc31c79ade0d54594b2e5 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 4 Mar 2025 19:41:53 +0300 Subject: [PATCH 1811/1943] [#1294] docs: Fix description of shard switching mode Signed-off-by: Alexander Chuprov --- docs/shard-modes.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/shard-modes.md b/docs/shard-modes.md index 3b459335b..6cc4ab13c 100644 --- a/docs/shard-modes.md +++ b/docs/shard-modes.md @@ -51,10 +51,7 @@ However, all mode changing operations are idempotent. ## Automatic mode changes -Shard can automatically switch to a `degraded-read-only` mode in 3 cases: -1. If the metabase was not available or couldn't be opened/initialized during shard startup. -2. If shard error counter exceeds threshold. -3. If the metabase couldn't be reopened during SIGHUP handling. +A shard can automatically switch to `read-only` mode if its error counter exceeds the threshold. # Detach shard From 4919b6a206ad9f448fcb36ae7d574a8f7dd3ad59 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 21 Mar 2025 15:51:22 +0300 Subject: [PATCH 1812/1943] [#1689] node/config: Allow zero `max_ops` in RPC limits config The limiter allows zeros for limits, meaning "this operation is disabled". However, the config didn't allow zero due to the lack of distinction between "no value" and "zero" - cast functions read both `nil` and zero as zero. Now, the config allows a zero limit. Added tests. Managing such cases should be easier after #1610. Change-Id: Ifc840732390b2feb975f230573b34bf479406e05 Signed-off-by: Aleksey Savchuk --- cmd/frostfs-node/config/rpc/config.go | 5 ++-- cmd/frostfs-node/config/rpc/config_test.go | 26 ++++++++++++++++++- .../rpc/testdata/{node.env => no_max_ops.env} | 0 .../testdata/{node.json => no_max_ops.json} | 0 .../testdata/{node.yaml => no_max_ops.yaml} | 0 .../config/rpc/testdata/zero_max_ops.env | 4 +++ .../config/rpc/testdata/zero_max_ops.json | 19 ++++++++++++++ .../config/rpc/testdata/zero_max_ops.yaml | 9 +++++++ 8 files changed, 59 insertions(+), 4 deletions(-) rename cmd/frostfs-node/config/rpc/testdata/{node.env => no_max_ops.env} (100%) rename cmd/frostfs-node/config/rpc/testdata/{node.json => no_max_ops.json} (100%) rename cmd/frostfs-node/config/rpc/testdata/{node.yaml => no_max_ops.yaml} (100%) create mode 100644 cmd/frostfs-node/config/rpc/testdata/zero_max_ops.env create mode 100644 cmd/frostfs-node/config/rpc/testdata/zero_max_ops.json create mode 100644 cmd/frostfs-node/config/rpc/testdata/zero_max_ops.yaml diff --git a/cmd/frostfs-node/config/rpc/config.go b/cmd/frostfs-node/config/rpc/config.go index 197990d07..e0efdfde2 100644 --- a/cmd/frostfs-node/config/rpc/config.go +++ b/cmd/frostfs-node/config/rpc/config.go @@ -31,12 +31,11 @@ func Limits(c *config.Config) []LimitConfig { break } - maxOps := config.IntSafe(sc, "max_ops") - if maxOps == 0 { + if sc.Value("max_ops") == nil { panic("no max operations for method group") } - limits = append(limits, LimitConfig{methods, maxOps}) + limits = append(limits, LimitConfig{methods, config.IntSafe(sc, "max_ops")}) } return limits diff --git a/cmd/frostfs-node/config/rpc/config_test.go b/cmd/frostfs-node/config/rpc/config_test.go index 31a837cee..a6365e19f 100644 --- a/cmd/frostfs-node/config/rpc/config_test.go +++ b/cmd/frostfs-node/config/rpc/config_test.go @@ -38,7 +38,7 @@ func TestRPCSection(t *testing.T) { }) t.Run("no max operations", func(t *testing.T) { - const path = "testdata/node" + const path = "testdata/no_max_ops" fileConfigTest := func(c *config.Config) { require.Panics(t, func() { _ = Limits(c) }) @@ -50,4 +50,28 @@ func TestRPCSection(t *testing.T) { configtest.ForEnvFileType(t, path, fileConfigTest) }) }) + + t.Run("zero max operations", func(t *testing.T) { + const path = "testdata/zero_max_ops" + + fileConfigTest := func(c *config.Config) { + limits := Limits(c) + require.Len(t, limits, 2) + + limit0 := limits[0] + limit1 := limits[1] + + require.ElementsMatch(t, limit0.Methods, []string{"/neo.fs.v2.object.ObjectService/PutSingle", "/neo.fs.v2.object.ObjectService/Put"}) + require.Equal(t, limit0.MaxOps, int64(0)) + + require.ElementsMatch(t, limit1.Methods, []string{"/neo.fs.v2.object.ObjectService/Get"}) + require.Equal(t, limit1.MaxOps, int64(10000)) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) + }) } diff --git a/cmd/frostfs-node/config/rpc/testdata/node.env b/cmd/frostfs-node/config/rpc/testdata/no_max_ops.env similarity index 100% rename from cmd/frostfs-node/config/rpc/testdata/node.env rename to cmd/frostfs-node/config/rpc/testdata/no_max_ops.env diff --git a/cmd/frostfs-node/config/rpc/testdata/node.json b/cmd/frostfs-node/config/rpc/testdata/no_max_ops.json similarity index 100% rename from cmd/frostfs-node/config/rpc/testdata/node.json rename to cmd/frostfs-node/config/rpc/testdata/no_max_ops.json diff --git a/cmd/frostfs-node/config/rpc/testdata/node.yaml b/cmd/frostfs-node/config/rpc/testdata/no_max_ops.yaml similarity index 100% rename from cmd/frostfs-node/config/rpc/testdata/node.yaml rename to cmd/frostfs-node/config/rpc/testdata/no_max_ops.yaml diff --git a/cmd/frostfs-node/config/rpc/testdata/zero_max_ops.env b/cmd/frostfs-node/config/rpc/testdata/zero_max_ops.env new file mode 100644 index 000000000..ce7302b0b --- /dev/null +++ b/cmd/frostfs-node/config/rpc/testdata/zero_max_ops.env @@ -0,0 +1,4 @@ +FROSTFS_RPC_LIMITS_0_METHODS="/neo.fs.v2.object.ObjectService/PutSingle /neo.fs.v2.object.ObjectService/Put" +FROSTFS_RPC_LIMITS_0_MAX_OPS=0 +FROSTFS_RPC_LIMITS_1_METHODS="/neo.fs.v2.object.ObjectService/Get" +FROSTFS_RPC_LIMITS_1_MAX_OPS=10000 diff --git a/cmd/frostfs-node/config/rpc/testdata/zero_max_ops.json b/cmd/frostfs-node/config/rpc/testdata/zero_max_ops.json new file mode 100644 index 000000000..16a1c173f --- /dev/null +++ b/cmd/frostfs-node/config/rpc/testdata/zero_max_ops.json @@ -0,0 +1,19 @@ +{ + "rpc": { + "limits": [ + { + "methods": [ + "/neo.fs.v2.object.ObjectService/PutSingle", + "/neo.fs.v2.object.ObjectService/Put" + ], + "max_ops": 0 + }, + { + "methods": [ + "/neo.fs.v2.object.ObjectService/Get" + ], + "max_ops": 10000 + } + ] + } +} diff --git a/cmd/frostfs-node/config/rpc/testdata/zero_max_ops.yaml b/cmd/frostfs-node/config/rpc/testdata/zero_max_ops.yaml new file mode 100644 index 000000000..525d768d4 --- /dev/null +++ b/cmd/frostfs-node/config/rpc/testdata/zero_max_ops.yaml @@ -0,0 +1,9 @@ +rpc: + limits: + - methods: + - /neo.fs.v2.object.ObjectService/PutSingle + - /neo.fs.v2.object.ObjectService/Put + max_ops: 0 + - methods: + - /neo.fs.v2.object.ObjectService/Get + max_ops: 10000 From e7e91ef63407d4946f70ffa746e33d79cfea1999 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 16:31:20 +0300 Subject: [PATCH 1813/1943] [#1689] adm: Remove storagecfg subcommand It is unused and unsupported for a long time. Change-Id: I570567db4e8cb202e41286064406ad85cd0e7a39 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/root.go | 2 - .../internal/modules/storagecfg/config.go | 135 ------ .../internal/modules/storagecfg/root.go | 432 ------------------ 3 files changed, 569 deletions(-) delete mode 100644 cmd/frostfs-adm/internal/modules/storagecfg/config.go delete mode 100644 cmd/frostfs-adm/internal/modules/storagecfg/root.go diff --git a/cmd/frostfs-adm/internal/modules/root.go b/cmd/frostfs-adm/internal/modules/root.go index defd898c8..e42204b7a 100644 --- a/cmd/frostfs-adm/internal/modules/root.go +++ b/cmd/frostfs-adm/internal/modules/root.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/storagecfg" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/autocomplete" utilConfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/config" @@ -41,7 +40,6 @@ func init() { rootCmd.AddCommand(config.RootCmd) rootCmd.AddCommand(morph.RootCmd) - rootCmd.AddCommand(storagecfg.RootCmd) rootCmd.AddCommand(metabase.RootCmd) rootCmd.AddCommand(autocomplete.Command("frostfs-adm")) diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/config.go b/cmd/frostfs-adm/internal/modules/storagecfg/config.go deleted file mode 100644 index 67e3414c2..000000000 --- a/cmd/frostfs-adm/internal/modules/storagecfg/config.go +++ /dev/null @@ -1,135 +0,0 @@ -package storagecfg - -const configTemplate = `logger: - level: info # logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" - -node: - wallet: - path: {{ .Wallet.Path }} # path to a NEO wallet; ignored if key is presented - address: {{ .Wallet.Account }} # address of a NEO account in the wallet; ignored if key is presented - password: {{ .Wallet.Password }} # password for a NEO account in the wallet; ignored if key is presented - addresses: # list of addresses announced by Storage node in the Network map - - {{ .AnnouncedAddress }} - attribute_0: UN-LOCODE:{{ .Attribute.Locode }} - relay: {{ .Relay }} # start Storage node in relay mode without bootstrapping into the Network map - -grpc: - num: 1 # total number of listener endpoints - 0: - endpoint: {{ .Endpoint }} # endpoint for gRPC server - tls:{{if .TLSCert}} - enabled: true # enable TLS for a gRPC connection (min version is TLS 1.2) - certificate: {{ .TLSCert }} # path to TLS certificate - key: {{ .TLSKey }} # path to TLS key - {{- else }} - enabled: false # disable TLS for a gRPC connection - {{- end}} - -control: - authorized_keys: # list of hex-encoded public keys that have rights to use the Control Service - {{- range .AuthorizedKeys }} - - {{.}}{{end}} - grpc: - endpoint: {{.ControlEndpoint}} # endpoint that is listened by the Control Service - -morph: - dial_timeout: 20s # timeout for side chain NEO RPC client connection - cache_ttl: 15s # use TTL cache for side chain GET operations - rpc_endpoint: # side chain N3 RPC endpoints - {{- range .MorphRPC }} - - address: wss://{{.}}/ws{{end}} -{{if not .Relay }} -storage: - shard: - default: # section with the default shard parameters - metabase: - perm: 0644 # permissions for metabase files(directories: +x for current user and group) - - blobstor: - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) - depth: 2 # max depth of object tree storage in FS - small_object_size: 102400 # 100KiB, size threshold for "small" objects which are stored in key-value DB, not in FS, bytes - compress: true # turn on/off Zstandard compression (level 3) of stored objects - compression_exclude_content_types: - - audio/* - - video/* - - blobovnicza: - size: 1073741824 # approximate size limit of single blobovnicza instance, total size will be: size*width^(depth+1), bytes - depth: 1 # max depth of object tree storage in key-value DB - width: 4 # max width of object tree storage in key-value DB - opened_cache_capacity: 50 # maximum number of opened database files - opened_cache_ttl: 5m # ttl for opened database file - opened_cache_exp_interval: 15s # cache cleanup interval for expired blobovnicza's - - gc: - remover_batch_size: 200 # number of objects to be removed by the garbage collector - remover_sleep_interval: 5m # frequency of the garbage collector invocation - 0: - mode: "read-write" # mode of the shard, must be one of the: "read-write" (default), "read-only" - - metabase: - path: {{ .MetabasePath }} # path to the metabase - - blobstor: - path: {{ .BlobstorPath }} # path to the blobstor -{{end}}` - -const ( - neofsMainnetAddress = "2cafa46838e8b564468ebd868dcafdd99dce6221" - balanceMainnetAddress = "dc1ec98d9d0c5f9dfade16144defe08cffc5ca55" - neofsTestnetAddress = "b65d8243ac63983206d17e5221af0653a7266fa1" - balanceTestnetAddress = "e0420c216003747626670d1424569c17c79015bf" -) - -var n3config = map[string]struct { - MorphRPC []string - RPC []string - NeoFSContract string - BalanceContract string -}{ - "testnet": { - MorphRPC: []string{ - "rpc01.morph.testnet.fs.neo.org:51331", - "rpc02.morph.testnet.fs.neo.org:51331", - "rpc03.morph.testnet.fs.neo.org:51331", - "rpc04.morph.testnet.fs.neo.org:51331", - "rpc05.morph.testnet.fs.neo.org:51331", - "rpc06.morph.testnet.fs.neo.org:51331", - "rpc07.morph.testnet.fs.neo.org:51331", - }, - RPC: []string{ - "rpc01.testnet.n3.nspcc.ru:21331", - "rpc02.testnet.n3.nspcc.ru:21331", - "rpc03.testnet.n3.nspcc.ru:21331", - "rpc04.testnet.n3.nspcc.ru:21331", - "rpc05.testnet.n3.nspcc.ru:21331", - "rpc06.testnet.n3.nspcc.ru:21331", - "rpc07.testnet.n3.nspcc.ru:21331", - }, - NeoFSContract: neofsTestnetAddress, - BalanceContract: balanceTestnetAddress, - }, - "mainnet": { - MorphRPC: []string{ - "rpc1.morph.fs.neo.org:40341", - "rpc2.morph.fs.neo.org:40341", - "rpc3.morph.fs.neo.org:40341", - "rpc4.morph.fs.neo.org:40341", - "rpc5.morph.fs.neo.org:40341", - "rpc6.morph.fs.neo.org:40341", - "rpc7.morph.fs.neo.org:40341", - }, - RPC: []string{ - "rpc1.n3.nspcc.ru:10331", - "rpc2.n3.nspcc.ru:10331", - "rpc3.n3.nspcc.ru:10331", - "rpc4.n3.nspcc.ru:10331", - "rpc5.n3.nspcc.ru:10331", - "rpc6.n3.nspcc.ru:10331", - "rpc7.n3.nspcc.ru:10331", - }, - NeoFSContract: neofsMainnetAddress, - BalanceContract: balanceMainnetAddress, - }, -} diff --git a/cmd/frostfs-adm/internal/modules/storagecfg/root.go b/cmd/frostfs-adm/internal/modules/storagecfg/root.go deleted file mode 100644 index a5adea0da..000000000 --- a/cmd/frostfs-adm/internal/modules/storagecfg/root.go +++ /dev/null @@ -1,432 +0,0 @@ -package storagecfg - -import ( - "bytes" - "context" - "encoding/hex" - "errors" - "fmt" - "math/rand" - "net" - "net/url" - "os" - "path/filepath" - "slices" - "strconv" - "strings" - "text/template" - "time" - - netutil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" - "github.com/chzyer/readline" - "github.com/nspcc-dev/neo-go/cli/flags" - "github.com/nspcc-dev/neo-go/cli/input" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/nspcc-dev/neo-go/pkg/encoding/address" - "github.com/nspcc-dev/neo-go/pkg/encoding/fixedn" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/gas" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" - "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" - "github.com/nspcc-dev/neo-go/pkg/util" - "github.com/nspcc-dev/neo-go/pkg/wallet" - "github.com/spf13/cobra" -) - -const ( - walletFlag = "wallet" - accountFlag = "account" -) - -const ( - defaultControlEndpoint = "localhost:8090" - defaultDataEndpoint = "localhost" -) - -// RootCmd is a root command of config section. -var RootCmd = &cobra.Command{ - Use: "storage-config [-w wallet] [-a acccount] []", - Short: "Section for storage node configuration commands", - Run: storageConfig, -} - -func init() { - fs := RootCmd.Flags() - - fs.StringP(walletFlag, "w", "", "Path to wallet") - fs.StringP(accountFlag, "a", "", "Wallet account") -} - -type config struct { - AnnouncedAddress string - AuthorizedKeys []string - ControlEndpoint string - Endpoint string - TLSCert string - TLSKey string - MorphRPC []string - Attribute struct { - Locode string - } - Wallet struct { - Path string - Account string - Password string - } - Relay bool - BlobstorPath string - MetabasePath string -} - -func storageConfig(cmd *cobra.Command, args []string) { - outPath := getOutputPath(args) - - historyPath := filepath.Join(os.TempDir(), "frostfs-adm.history") - readline.SetHistoryPath(historyPath) - - var c config - - c.Wallet.Path, _ = cmd.Flags().GetString(walletFlag) - if c.Wallet.Path == "" { - c.Wallet.Path = getPath("Path to the storage node wallet: ") - } - - w, err := wallet.NewWalletFromFile(c.Wallet.Path) - fatalOnErr(err) - - fillWalletAccount(cmd, &c, w) - - accH, err := flags.ParseAddress(c.Wallet.Account) - fatalOnErr(err) - - acc := w.GetAccount(accH) - if acc == nil { - fatalOnErr(errors.New("can't find account in wallet")) - } - - c.Wallet.Password, err = input.ReadPassword(fmt.Sprintf("Enter password for %s > ", c.Wallet.Account)) - fatalOnErr(err) - - err = acc.Decrypt(c.Wallet.Password, keys.NEP2ScryptParams()) - fatalOnErr(err) - - c.AuthorizedKeys = append(c.AuthorizedKeys, hex.EncodeToString(acc.PrivateKey().PublicKey().Bytes())) - - network := readNetwork(cmd) - - c.MorphRPC = n3config[network].MorphRPC - - depositGas(cmd, acc, network) - - c.Attribute.Locode = getString("UN-LOCODE attribute in [XX YYY] format: ") - - endpoint := getDefaultEndpoint(cmd, &c) - c.Endpoint = getString(fmt.Sprintf("Listening address [%s]: ", endpoint)) - if c.Endpoint == "" { - c.Endpoint = endpoint - } - - c.ControlEndpoint = getString(fmt.Sprintf("Listening address (control endpoint) [%s]: ", defaultControlEndpoint)) - if c.ControlEndpoint == "" { - c.ControlEndpoint = defaultControlEndpoint - } - - c.TLSCert = getPath("TLS Certificate (optional): ") - if c.TLSCert != "" { - c.TLSKey = getPath("TLS Key: ") - } - - c.Relay = getConfirmation(false, "Use node as a relay? yes/[no]: ") - if !c.Relay { - p := getPath("Path to the storage directory (all available storage will be used): ") - c.BlobstorPath = filepath.Join(p, "blob") - c.MetabasePath = filepath.Join(p, "meta") - } - - out := applyTemplate(c) - fatalOnErr(os.WriteFile(outPath, out, 0o644)) - - cmd.Println("Node is ready for work! Run `frostfs-node -config " + outPath + "`") -} - -func getDefaultEndpoint(cmd *cobra.Command, c *config) string { - var addr, port string - for { - c.AnnouncedAddress = getString("Publicly announced address: ") - validator := netutil.Address{} - err := validator.FromString(c.AnnouncedAddress) - if err != nil { - cmd.Println("Incorrect address format. See https://git.frostfs.info/TrueCloudLab/frostfs-node/src/branch/master/pkg/network/address.go for details.") - continue - } - uriAddr, err := url.Parse(validator.URIAddr()) - if err != nil { - panic(fmt.Errorf("unexpected error: %w", err)) - } - addr = uriAddr.Hostname() - port = uriAddr.Port() - ip, err := net.ResolveIPAddr("ip", addr) - if err != nil { - cmd.Printf("Can't resolve IP address %s: %v\n", addr, err) - continue - } - - if !ip.IP.IsGlobalUnicast() { - cmd.Println("IP must be global unicast.") - continue - } - cmd.Printf("Resolved IP address: %s\n", ip.String()) - - _, err = strconv.ParseUint(port, 10, 16) - if err != nil { - cmd.Println("Port must be an integer.") - continue - } - - break - } - return net.JoinHostPort(defaultDataEndpoint, port) -} - -func fillWalletAccount(cmd *cobra.Command, c *config, w *wallet.Wallet) { - c.Wallet.Account, _ = cmd.Flags().GetString(accountFlag) - if c.Wallet.Account == "" { - addr := address.Uint160ToString(w.GetChangeAddress()) - c.Wallet.Account = getWalletAccount(w, fmt.Sprintf("Wallet account [%s]: ", addr)) - if c.Wallet.Account == "" { - c.Wallet.Account = addr - } - } -} - -func readNetwork(cmd *cobra.Command) string { - var network string - for { - network = getString("Choose network [mainnet]/testnet: ") - switch network { - case "": - network = "mainnet" - case "testnet", "mainnet": - default: - cmd.Println(`Network must be either "mainnet" or "testnet"`) - continue - } - break - } - return network -} - -func getOutputPath(args []string) string { - if len(args) != 0 { - return args[0] - } - outPath := getPath("File to write config at [./config.yml]: ") - if outPath == "" { - outPath = "./config.yml" - } - return outPath -} - -func getWalletAccount(w *wallet.Wallet, prompt string) string { - addrs := make([]readline.PrefixCompleterInterface, len(w.Accounts)) - for i := range w.Accounts { - addrs[i] = readline.PcItem(w.Accounts[i].Address) - } - - readline.SetAutoComplete(readline.NewPrefixCompleter(addrs...)) - defer readline.SetAutoComplete(nil) - - s, err := readline.Line(prompt) - fatalOnErr(err) - return strings.TrimSpace(s) // autocompleter can return a string with a trailing space -} - -func getString(prompt string) string { - s, err := readline.Line(prompt) - fatalOnErr(err) - if s != "" { - _ = readline.AddHistory(s) - } - return s -} - -type filenameCompleter struct{} - -func (filenameCompleter) Do(line []rune, pos int) (newLine [][]rune, length int) { - prefix := string(line[:pos]) - dir := filepath.Dir(prefix) - de, err := os.ReadDir(dir) - if err != nil { - return nil, 0 - } - - for i := range de { - name := filepath.Join(dir, de[i].Name()) - if strings.HasPrefix(name, prefix) { - tail := []rune(strings.TrimPrefix(name, prefix)) - if de[i].IsDir() { - tail = append(tail, filepath.Separator) - } - newLine = append(newLine, tail) - } - } - if pos != 0 { - return newLine, pos - len([]rune(dir)) - } - return newLine, 0 -} - -func getPath(prompt string) string { - readline.SetAutoComplete(filenameCompleter{}) - defer readline.SetAutoComplete(nil) - - p, err := readline.Line(prompt) - fatalOnErr(err) - - if p == "" { - return p - } - - _ = readline.AddHistory(p) - - abs, err := filepath.Abs(p) - if err != nil { - fatalOnErr(fmt.Errorf("can't create an absolute path: %w", err)) - } - - return abs -} - -func getConfirmation(def bool, prompt string) bool { - for { - s, err := readline.Line(prompt) - fatalOnErr(err) - - switch strings.ToLower(s) { - case "y", "yes": - return true - case "n", "no": - return false - default: - if len(s) == 0 { - return def - } - } - } -} - -func applyTemplate(c config) []byte { - tmpl, err := template.New("config").Parse(configTemplate) - fatalOnErr(err) - - b := bytes.NewBuffer(nil) - fatalOnErr(tmpl.Execute(b, c)) - - return b.Bytes() -} - -func fatalOnErr(err error) { - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func depositGas(cmd *cobra.Command, acc *wallet.Account, network string) { - sideClient := initClient(n3config[network].MorphRPC) - balanceHash, _ := util.Uint160DecodeStringLE(n3config[network].BalanceContract) - - sideActor, err := actor.NewSimple(sideClient, acc) - if err != nil { - fatalOnErr(fmt.Errorf("creating actor over side chain client: %w", err)) - } - - sideGas := nep17.NewReader(sideActor, balanceHash) - accSH := acc.Contract.ScriptHash() - - balance, err := sideGas.BalanceOf(accSH) - if err != nil { - fatalOnErr(fmt.Errorf("side chain balance: %w", err)) - } - - ok := getConfirmation(false, fmt.Sprintf("Current NeoFS balance is %s, make a deposit? y/[n]: ", - fixedn.ToString(balance, 12))) - if !ok { - return - } - - amountStr := getString("Enter amount in GAS: ") - amount, err := fixedn.FromString(amountStr, 8) - if err != nil { - fatalOnErr(fmt.Errorf("invalid amount: %w", err)) - } - - mainClient := initClient(n3config[network].RPC) - neofsHash, _ := util.Uint160DecodeStringLE(n3config[network].NeoFSContract) - - mainActor, err := actor.NewSimple(mainClient, acc) - if err != nil { - fatalOnErr(fmt.Errorf("creating actor over main chain client: %w", err)) - } - - mainGas := nep17.New(mainActor, gas.Hash) - - txHash, _, err := mainGas.Transfer(accSH, neofsHash, amount, nil) - if err != nil { - fatalOnErr(fmt.Errorf("sending TX to the NeoFS contract: %w", err)) - } - - cmd.Print("Waiting for transactions to persist.") - tick := time.NewTicker(time.Second / 2) - defer tick.Stop() - - timer := time.NewTimer(time.Second * 20) - defer timer.Stop() - - at := trigger.Application - -loop: - for { - select { - case <-tick.C: - _, err := mainClient.GetApplicationLog(txHash, &at) - if err == nil { - cmd.Print("\n") - break loop - } - cmd.Print(".") - case <-timer.C: - cmd.Printf("\nTimeout while waiting for transaction to persist.\n") - if getConfirmation(false, "Continue configuration? yes/[no]: ") { - return - } - os.Exit(1) - } - } -} - -func initClient(rpc []string) *rpcclient.Client { - var c *rpcclient.Client - var err error - - shuffled := slices.Clone(rpc) - rand.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] }) - - for _, endpoint := range shuffled { - c, err = rpcclient.New(context.Background(), "https://"+endpoint, rpcclient.Options{ - DialTimeout: time.Second * 2, - RequestTimeout: time.Second * 5, - }) - if err != nil { - continue - } - if err = c.Init(); err != nil { - continue - } - return c - } - - fatalOnErr(fmt.Errorf("can't create N3 client: %w", err)) - panic("unreachable") -} From 30099194ba8f64b5340c3c2ff3153d31ab549223 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 15:18:28 +0300 Subject: [PATCH 1814/1943] [#1689] config: Remove testnet and mainnet configs They are invalid and unsupported. There is neither mainnet nor testnet currently. Change-Id: I520363e2de0c22a584238accc253248be3eefea5 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/validate_test.go | 14 ---- config/mainnet/README.md | 28 ------- config/mainnet/config.yml | 70 ---------------- config/testnet/README.md | 129 ------------------------------ config/testnet/config.yml | 52 ------------ 5 files changed, 293 deletions(-) delete mode 100644 config/mainnet/README.md delete mode 100644 config/mainnet/config.yml delete mode 100644 config/testnet/README.md delete mode 100644 config/testnet/config.yml diff --git a/cmd/frostfs-node/validate_test.go b/cmd/frostfs-node/validate_test.go index d9c0f167f..495365cf0 100644 --- a/cmd/frostfs-node/validate_test.go +++ b/cmd/frostfs-node/validate_test.go @@ -1,7 +1,6 @@ package main import ( - "os" "path/filepath" "testing" @@ -22,17 +21,4 @@ func TestValidate(t *testing.T) { require.NoError(t, err) }) }) - - t.Run("mainnet", func(t *testing.T) { - os.Clearenv() // ENVs have priority over config files, so we do this in tests - p := filepath.Join(exampleConfigPrefix, "mainnet/config.yml") - c := config.New(p, "", config.EnvPrefix) - require.NoError(t, validateConfig(c)) - }) - t.Run("testnet", func(t *testing.T) { - os.Clearenv() // ENVs have priority over config files, so we do this in tests - p := filepath.Join(exampleConfigPrefix, "testnet/config.yml") - c := config.New(p, "", config.EnvPrefix) - require.NoError(t, validateConfig(c)) - }) } diff --git a/config/mainnet/README.md b/config/mainnet/README.md deleted file mode 100644 index 717a9b0ff..000000000 --- a/config/mainnet/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# N3 Mainnet Storage node configuration - -Here is a template for simple storage node configuration in N3 Mainnet. -Make sure to specify correct values instead of `<...>` placeholders. -Do not change `contracts` section. Run the latest frostfs-node release with -the fixed config `frostfs-node -c config.yml` - -To use NeoFS in the Mainnet, you need to deposit assets to NeoFS contract. -The contract sript hash is `2cafa46838e8b564468ebd868dcafdd99dce6221` -(N3 address `NNxVrKjLsRkWsmGgmuNXLcMswtxTGaNQLk`) - -## Tips - -Use `grpcs://` scheme in the announced address if you enable TLS in grpc server. -```yaml -node: - addresses: - - grpcs://frostfs.my.org:8080 - -grpc: - num: 1 - 0: - endpoint: frostfs.my.org:8080 - tls: - enabled: true - certificate: /path/to/cert - key: /path/to/key -``` diff --git a/config/mainnet/config.yml b/config/mainnet/config.yml deleted file mode 100644 index d86ea451f..000000000 --- a/config/mainnet/config.yml +++ /dev/null @@ -1,70 +0,0 @@ -node: - wallet: - path: - address: - password: - addresses: - - - attribute_0: UN-LOCODE: - attribute_1: Price:100000 - attribute_2: User-Agent:FrostFS\/0.9999 - -grpc: - num: 1 - 0: - endpoint: - tls: - enabled: false - -storage: - shard_num: 1 - shard: - 0: - metabase: - path: /storage/path/metabase - perm: 0600 - blobstor: - - path: /storage/path/blobovnicza - type: blobovnicza - perm: 0600 - opened_cache_capacity: 32 - depth: 1 - width: 1 - - path: /storage/path/fstree - type: fstree - perm: 0600 - depth: 4 - writecache: - enabled: false - gc: - remover_batch_size: 100 - remover_sleep_interval: 1m - -logger: - level: info - -prometheus: - enabled: true - address: localhost:9090 - shutdown_timeout: 15s - -object: - put: - remote_pool_size: 100 - local_pool_size: 100 - -morph: - rpc_endpoint: - - wss://rpc1.morph.frostfs.info:40341/ws - - wss://rpc2.morph.frostfs.info:40341/ws - - wss://rpc3.morph.frostfs.info:40341/ws - - wss://rpc4.morph.frostfs.info:40341/ws - - wss://rpc5.morph.frostfs.info:40341/ws - - wss://rpc6.morph.frostfs.info:40341/ws - - wss://rpc7.morph.frostfs.info:40341/ws - dial_timeout: 20s - -contracts: - balance: dc1ec98d9d0c5f9dfade16144defe08cffc5ca55 - container: 1b6e68d299b570e1cb7e86eadfdc06aa2e8e0cc5 - netmap: 7c5bdb23e36cc7cce95bf42f3ab9e452c2501df1 diff --git a/config/testnet/README.md b/config/testnet/README.md deleted file mode 100644 index e2cda33ec..000000000 --- a/config/testnet/README.md +++ /dev/null @@ -1,129 +0,0 @@ -# N3 Testnet Storage node configuration - -There is a prepared configuration for NeoFS Storage Node deployment in -N3 Testnet. The easiest way to deploy a Storage Node is to use the prepared -docker image and run it with docker-compose. - -## Build image - -Prepared **frostfs-storage-testnet** image is available at Docker Hub. -However, if you need to rebuild it for some reason, run -`make image-storage-testnet` command. - -``` -$ make image-storage-testnet -... -Successfully built ab0557117b02 -Successfully tagged nspccdev/neofs-storage-testnet:0.25.1 -``` - -## Deploy node - -To run a storage node in N3 Testnet environment, you should deposit GAS assets, -update docker-compose file and start the node. - -### Deposit - -The Storage Node owner should deposit GAS to NeoFS smart contract. It generates a -bit of sidechain GAS in the node's wallet. Sidechain GAS is used to send bootstrap tx. - -First, obtain GAS in N3 Testnet chain. You can do that with -[faucet](https://neowish.ngd.network) service. - -Then, make a deposit by transferring GAS to NeoFS contract in N3 Testnet. -You can provide scripthash in the `data` argument of transfer tx to make a -deposit to a specified account. Otherwise, deposit is made to the tx sender. - -NeoFS contract scripthash in N3 Testnet is `b65d8243ac63983206d17e5221af0653a7266fa1`, -so the address is `NadZ8YfvkddivcFFkztZgfwxZyKf1acpRF`. - -See a deposit example with `neo-go`. - -``` -neo-go wallet nep17 transfer -w wallet.json -r https://rpc01.testnet.n3.nspcc.ru:21331 \ ---from NXxRAFPqPstaPByndKMHuC8iGcaHgtRY3m \ ---to NadZ8YfvkddivcFFkztZgfwxZyKf1acpRF \ ---token GAS \ ---amount 1 -``` - -### Configure - -Next, configure `node_config.env` file. Change endpoints values. Both -should contain your **public** IP. - -``` -NEOFS_GRPC_0_ENDPOINT=65.52.183.157:36512 -NEOFS_NODE_ADDRESSES=65.52.183.157:36512 -``` - -Set up your [UN/LOCODE](https://unece.org/trade/cefact/unlocode-code-list-country-and-territory) -attribute. - -``` -NEOFS_GRPC_0_ENDPOINT=65.52.183.157:36512 -NEOFS_NODE_ADDRESSES=65.52.183.157:36512 -NEOFS_NODE_ATTRIBUTE_2=UN-LOCODE:RU LED -``` - -You can validate UN/LOCODE attribute in -[NeoFS LOCODE database](https://git.frostfs.info/TrueCloudLab/frostfs-locode-db/releases/tag/v0.4.0) -with frostfs-cli. - -``` -$ frostfs-cli util locode info --db ./locode_db --locode 'RU LED' -Country: Russia -Location: Saint Petersburg (ex Leningrad) -Continent: Europe -Subdivision: [SPE] Sankt-Peterburg -Coordinates: 59.53, 30.15 -``` - -It is recommended to pass the node's key as a file. To do so, convert your wallet -WIF to 32-byte hex (via `frostfs-cli` for example) and save it to a file. - -``` -// Print WIF in a 32-byte hex format -$ frostfs-cli util keyer Kwp4Q933QujZLUCcn39tzY94itNQJS4EjTp28oAMzuxMwabm3p1s -PrivateKey 11ab917cd99170cb8d0d48e78fca317564e6b3aaff7f7058952d6175cdca0f56 -PublicKey 02be8b2e837cab232168f5c3303f1b985818b7583682fb49026b8d2f43df7c1059 -WIF Kwp4Q933QujZLUCcn39tzY94itNQJS4EjTp28oAMzuxMwabm3p1s -Wallet3.0 Nfzmk7FAZmEHDhLePdgysQL2FgkJbaEMpQ -ScriptHash3.0 dffe39998f50d42f2e06807866161cd0440b4bdc -ScriptHash3.0BE dc4b0b44d01c16667880062e2fd4508f9939fedf - -// Save 32-byte hex into a file -$ echo '11ab917cd99170cb8d0d48e78fca317564e6b3aaff7f7058952d6175cdca0f56' | xxd -r -p > my_wallet.key -``` - -Then, specify the path to this file in `docker-compose.yml` -```yaml - volumes: - - frostfs_storage:/storage - - ./my_wallet.key:/node.key -``` - - -NeoFS objects will be stored on your machine. By default, docker-compose -is configured to store objects in named docker volume `frostfs_storage`. You can -specify a directory on the filesystem to store objects there. - -```yaml - volumes: - - /home/username/frostfs/rc3/storage:/storage - - ./my_wallet.key:/node.key -``` - -### Start - -Run the node with `docker-compose up` command and stop it with `docker-compose down`. - -### Debug - -To print node logs, use `docker logs frostfs-testnet`. To print debug messages in -log, set up log level to debug with this env: - -```yaml - environment: - - NEOFS_LOGGER_LEVEL=debug -``` diff --git a/config/testnet/config.yml b/config/testnet/config.yml deleted file mode 100644 index 76b36cdf6..000000000 --- a/config/testnet/config.yml +++ /dev/null @@ -1,52 +0,0 @@ -logger: - level: info - -morph: - rpc_endpoint: - - wss://rpc01.morph.testnet.frostfs.info:51331/ws - - wss://rpc02.morph.testnet.frostfs.info:51331/ws - - wss://rpc03.morph.testnet.frostfs.info:51331/ws - - wss://rpc04.morph.testnet.frostfs.info:51331/ws - - wss://rpc05.morph.testnet.frostfs.info:51331/ws - - wss://rpc06.morph.testnet.frostfs.info:51331/ws - - wss://rpc07.morph.testnet.frostfs.info:51331/ws - dial_timeout: 20s - -contracts: - balance: e0420c216003747626670d1424569c17c79015bf - container: 9dbd2b5e67568ed285c3d6f96bac4edf5e1efba0 - netmap: d4b331639799e2958d4bc5b711b469d79de94e01 - -node: - key: /node.key - attribute_0: Deployed:SelfHosted - attribute_1: User-Agent:FrostFS\/0.9999 - -prometheus: - enabled: true - address: localhost:9090 - shutdown_timeout: 15s - -storage: - shard_num: 1 - shard: - 0: - metabase: - path: /storage/metabase - perm: 0777 - blobstor: - - path: /storage/path/blobovnicza - type: blobovnicza - perm: 0600 - opened_cache_capacity: 32 - depth: 1 - width: 1 - - path: /storage/path/fstree - type: fstree - perm: 0600 - depth: 4 - writecache: - enabled: false - gc: - remover_batch_size: 100 - remover_sleep_interval: 1m From a4da1da7670b32b73e03f240d0cfcca45f33afda Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 15:40:40 +0300 Subject: [PATCH 1815/1943] [#905] morph/client: Fetch NNS hash once on init NNS contract hash is taken from the contract with ID=1. Because morph client is expected to work with the same chain, and because contract hash doesn't change on update, there is no need to fetch it from each new endpoint. Change-Id: Ic6dc18283789da076d6a0b3701139b97037714cc Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 17 +------------ pkg/morph/client/constructor.go | 5 ++++ pkg/morph/client/nns.go | 42 ++++----------------------------- 3 files changed, 10 insertions(+), 54 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index e63d926e0..a7c3c6d8d 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -60,6 +60,7 @@ type Client struct { rpcActor *actor.Actor // neo-go RPC actor gasToken *nep17.Token // neo-go GAS token wrapper rolemgmt *rolemgmt.Contract // neo-go Designation contract wrapper + nnsHash util.Uint160 // NNS contract hash acc *wallet.Account // neo account accAddr util.Uint160 // account's address @@ -94,27 +95,12 @@ type Client struct { type cache struct { m sync.RWMutex - nnsHash *util.Uint160 gKey *keys.PublicKey txHeights *lru.Cache[util.Uint256, uint32] metrics metrics.MorphCacheMetrics } -func (c *cache) nns() *util.Uint160 { - c.m.RLock() - defer c.m.RUnlock() - - return c.nnsHash -} - -func (c *cache) setNNSHash(nnsHash util.Uint160) { - c.m.Lock() - defer c.m.Unlock() - - c.nnsHash = &nnsHash -} - func (c *cache) groupKey() *keys.PublicKey { c.m.RLock() defer c.m.RUnlock() @@ -133,7 +119,6 @@ func (c *cache) invalidate() { c.m.Lock() defer c.m.Unlock() - c.nnsHash = nil c.gKey = nil c.txHeights.Purge() } diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index d061747bb..e4dcd0db7 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -145,6 +145,11 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er if cli.client == nil { return nil, ErrNoHealthyEndpoint } + cs, err := cli.client.GetContractStateByID(nnsContractID) + if err != nil { + return nil, fmt.Errorf("resolve nns hash: %w", err) + } + cli.nnsHash = cs.Hash cli.setActor(act) go cli.closeWaiter(ctx) diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index f292dccf1..b03967fdd 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -61,11 +61,7 @@ func (c *Client) NNSContractAddress(name string) (sh util.Uint160, err error) { return util.Uint160{}, ErrConnectionLost } - nnsHash, err := c.NNSHash() - if err != nil { - return util.Uint160{}, err - } - + nnsHash := c.NNSHash() sh, err = nnsResolve(c.client, nnsHash, name) if err != nil { return sh, fmt.Errorf("NNS.resolve: %w", err) @@ -74,34 +70,8 @@ func (c *Client) NNSContractAddress(name string) (sh util.Uint160, err error) { } // NNSHash returns NNS contract hash. -func (c *Client) NNSHash() (util.Uint160, error) { - c.switchLock.RLock() - defer c.switchLock.RUnlock() - - if c.inactive { - return util.Uint160{}, ErrConnectionLost - } - - success := false - startedAt := time.Now() - - defer func() { - c.cache.metrics.AddMethodDuration("NNSContractHash", success, time.Since(startedAt)) - }() - - nnsHash := c.cache.nns() - - if nnsHash == nil { - cs, err := c.client.GetContractStateByID(nnsContractID) - if err != nil { - return util.Uint160{}, fmt.Errorf("NNS contract state: %w", err) - } - - c.cache.setNNSHash(cs.Hash) - nnsHash = &cs.Hash - } - success = true - return *nnsHash, nil +func (c *Client) NNSHash() util.Uint160 { + return c.nnsHash } func nnsResolveItem(c *rpcclient.WSClient, nnsHash util.Uint160, domain string) (stackitem.Item, error) { @@ -241,11 +211,7 @@ func (c *Client) contractGroupKey() (*keys.PublicKey, error) { return gKey, nil } - nnsHash, err := c.NNSHash() - if err != nil { - return nil, err - } - + nnsHash := c.NNSHash() item, err := nnsResolveItem(c.client, nnsHash, NNSGroupKeyName) if err != nil { return nil, err From eed082459081628eef495d13b5053c07fb67ae84 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 21 Mar 2025 18:03:03 +0300 Subject: [PATCH 1816/1943] go.mod: Bump frostfs-qos version Change-Id: I8bc045b509ee1259cfad288477a0b7d045683f10 Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- pkg/network/cache/multi.go | 2 +- pkg/services/tree/cache.go | 2 +- pkg/services/tree/sync.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index fafb4f828..753bfbd29 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 - git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250310135838-3e7ca9403529 + git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250320142439-32079ad7c275 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 diff --git a/go.sum b/go.sum index 7818583d4..7f9417954 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 h1:fluzML8BIIabd07LyPSjc0JAV2qymWkPiFaLrXdALLA= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250310135838-3e7ca9403529 h1:CBreXSxGoYJAdZ1QdJPsDs1UCXGF5psinII0lxtohsc= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250310135838-3e7ca9403529/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250320142439-32079ad7c275 h1:WqWxCnCl2ekfjWja/CpGeF2rf4h0x199xhdnsm/j+E8= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250320142439-32079ad7c275/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9 h1:svCl6NDAPZ/KuQPjdVKo74RkCIANesxUPM45zQZDhSw= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= diff --git a/pkg/network/cache/multi.go b/pkg/network/cache/multi.go index 77420865a..54c1e18fb 100644 --- a/pkg/network/cache/multi.go +++ b/pkg/network/cache/multi.go @@ -67,7 +67,7 @@ func (x *multiClient) createForAddress(ctx context.Context, addr network.Address qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), metrics.NewUnaryClientInterceptor(), tracing.NewUnaryClientInterceptor(), - tagging.NewUnaryClientInteceptor(), + tagging.NewUnaryClientInterceptor(), ), grpc.WithChainStreamInterceptor( qos.NewAdjustOutgoingIOTagStreamClientInterceptor(), diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 3359af2c5..e2be2f4a2 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -100,7 +100,7 @@ func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (* qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), metrics.NewUnaryClientInterceptor(), tracing.NewUnaryClientInterceptor(), - tagging.NewUnaryClientInteceptor(), + tagging.NewUnaryClientInterceptor(), ), grpc.WithChainStreamInterceptor( qos.NewAdjustOutgoingIOTagStreamClientInterceptor(), diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 103e2a613..1c3521344 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -345,7 +345,7 @@ func (*Service) createConnection(a network.Address) (*grpc.ClientConn, error) { qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), metrics.NewUnaryClientInterceptor(), tracing_grpc.NewUnaryClientInterceptor(), - tagging.NewUnaryClientInteceptor(), + tagging.NewUnaryClientInterceptor(), ), grpc.WithChainStreamInterceptor( qos.NewAdjustOutgoingIOTagStreamClientInterceptor(), From 73e35bc88554954371555bd0e9bdf6ce99d20de7 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 19 Mar 2025 13:57:44 +0300 Subject: [PATCH 1817/1943] [#1052] object: Make `ape` middleware form request info * Move some helpers from `acl/v2` package to `ape`. Also move errors; * Introduce `Metadata`, `RequestInfo` types; * Introduce `RequestInfoExtractor` interface and its implementation. The extractor's purpose is to extract request info based on request metadata. It also validates session token; * Refactor ape service - each handler forms request info and pass necessary fields to checker. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/object.go | 5 +- pkg/services/object/ape/errors.go | 15 + pkg/services/object/ape/metadata.go | 172 ++++++++++++ pkg/services/object/ape/metadata_test.go | 164 +++++++++++ pkg/services/object/ape/service.go | 335 ++++++++++------------- pkg/services/object/ape/types.go | 8 + pkg/services/object/ape/util.go | 169 ++++++++++++ pkg/services/object/ape/util_test.go | 84 ++++++ 8 files changed, 759 insertions(+), 193 deletions(-) create mode 100644 pkg/services/object/ape/metadata.go create mode 100644 pkg/services/object/ape/metadata_test.go create mode 100644 pkg/services/object/ape/util.go create mode 100644 pkg/services/object/ape/util_test.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index ad6f4140a..652d3ad04 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -172,7 +172,7 @@ func initObjectService(c *cfg) { splitSvc := createSplitService(c, sPutV2, sGetV2, sSearchV2, sDeleteV2, sPatch) - apeSvc := createAPEService(c, splitSvc) + apeSvc := createAPEService(c, &irFetcher, splitSvc) aclSvc := createACLServiceV2(c, apeSvc, &irFetcher) @@ -439,7 +439,7 @@ func createACLServiceV2(c *cfg, apeSvc *objectAPE.Service, irFetcher *cachedIRFe ) } -func createAPEService(c *cfg, splitSvc *objectService.TransportSplitter) *objectAPE.Service { +func createAPEService(c *cfg, irFetcher *cachedIRFetcher, splitSvc *objectService.TransportSplitter) *objectAPE.Service { return objectAPE.NewService( objectAPE.NewChecker( c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalStorage(), @@ -451,6 +451,7 @@ func createAPEService(c *cfg, splitSvc *objectService.TransportSplitter) *object c.cfgObject.cnrSource, c.binPublicKey, ), + objectAPE.NewRequestInfoExtractor(c.log, c.cfgObject.cnrSource, irFetcher, c.netMapSource), splitSvc, ) } diff --git a/pkg/services/object/ape/errors.go b/pkg/services/object/ape/errors.go index 6e458b384..82e660a7f 100644 --- a/pkg/services/object/ape/errors.go +++ b/pkg/services/object/ape/errors.go @@ -7,6 +7,21 @@ import ( apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" ) +var ( + errMissingContainerID = malformedRequestError("missing container ID") + errEmptyVerificationHeader = malformedRequestError("empty verification header") + errEmptyBodySig = malformedRequestError("empty at body signature") + errInvalidSessionSig = malformedRequestError("invalid session token signature") + errInvalidSessionOwner = malformedRequestError("invalid session token owner") + errInvalidVerb = malformedRequestError("session token verb is invalid") +) + +func malformedRequestError(reason string) error { + invalidArgErr := &apistatus.InvalidArgument{} + invalidArgErr.SetMessage(reason) + return invalidArgErr +} + func toStatusErr(err error) error { var chRouterErr *checkercore.ChainRouterError if !errors.As(err, &chRouterErr) { diff --git a/pkg/services/object/ape/metadata.go b/pkg/services/object/ape/metadata.go new file mode 100644 index 000000000..b37c3b6f8 --- /dev/null +++ b/pkg/services/object/ape/metadata.go @@ -0,0 +1,172 @@ +package ape + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "strings" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +type Metadata struct { + Container cid.ID + Object *oid.ID + MetaHeader *session.RequestMetaHeader + VerificationHeader *session.RequestVerificationHeader + SessionToken *sessionSDK.Object + BearerToken *bearer.Token +} + +func (m Metadata) RequestOwner() (*user.ID, *keys.PublicKey, error) { + if m.VerificationHeader == nil { + return nil, nil, errEmptyVerificationHeader + } + + if m.BearerToken != nil && m.BearerToken.Impersonate() { + return unmarshalPublicKeyWithOwner(m.BearerToken.SigningKeyBytes()) + } + + // if session token is presented, use it as truth source + if m.SessionToken != nil { + // verify signature of session token + return ownerFromToken(m.SessionToken) + } + + // otherwise get original body signature + bodySignature := originalBodySignature(m.VerificationHeader) + if bodySignature == nil { + return nil, nil, errEmptyBodySig + } + + return unmarshalPublicKeyWithOwner(bodySignature.GetKey()) +} + +// RequestInfo contains request information extracted by request metadata. +type RequestInfo struct { + // Role defines under which role this request is executed. + // It must be represented only as a constant represented in native schema. + Role string + + ContainerOwner user.ID + + // Namespace defines to which namespace a container is belonged. + Namespace string + + // HEX-encoded sender key. + SenderKey string +} + +type RequestInfoExtractor interface { + GetRequestInfo(context.Context, Metadata, string) (RequestInfo, error) +} + +type extractor struct { + containers container.Source + + nm netmap.Source + + classifier objectCore.SenderClassifier +} + +func NewRequestInfoExtractor(log *logger.Logger, containers container.Source, irFetcher InnerRingFetcher, nm netmap.Source) RequestInfoExtractor { + return &extractor{ + containers: containers, + nm: nm, + classifier: objectCore.NewSenderClassifier(irFetcher, nm, log), + } +} + +func (e *extractor) verifySessionToken(ctx context.Context, sessionToken *sessionSDK.Object, method string) error { + currentEpoch, err := e.nm.Epoch(ctx) + if err != nil { + return errors.New("can't fetch current epoch") + } + if sessionToken.ExpiredAt(currentEpoch) { + return new(apistatus.SessionTokenExpired) + } + if sessionToken.InvalidAt(currentEpoch) { + return fmt.Errorf("malformed request: token is invalid at %d epoch)", currentEpoch) + } + if !assertVerb(*sessionToken, method) { + return errInvalidVerb + } + return nil +} + +func (e *extractor) GetRequestInfo(ctx context.Context, m Metadata, method string) (ri RequestInfo, err error) { + cnr, err := e.containers.Get(ctx, m.Container) + if err != nil { + return ri, err + } + + if m.SessionToken != nil { + if err = e.verifySessionToken(ctx, m.SessionToken, method); err != nil { + return ri, err + } + } + + ownerID, ownerKey, err := m.RequestOwner() + if err != nil { + return ri, err + } + res, err := e.classifier.Classify(ctx, ownerID, ownerKey, m.Container, cnr.Value) + if err != nil { + return ri, err + } + + ri.Role = nativeSchemaRole(res.Role) + ri.ContainerOwner = cnr.Value.Owner() + + cnrNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(cnr.Value).Zone(), ".ns") + if hasNamespace { + ri.Namespace = cnrNamespace + } + + // it is assumed that at the moment the key will be valid, + // otherwise the request would not pass validation + ri.SenderKey = hex.EncodeToString(res.Key) + + return ri, nil +} + +func readSessionToken(cnr cid.ID, obj *oid.ID, tokV2 *session.Token) (*sessionSDK.Object, error) { + var sTok *sessionSDK.Object + + if tokV2 != nil { + sTok = new(sessionSDK.Object) + + err := sTok.ReadFromV2(*tokV2) + if err != nil { + return nil, fmt.Errorf("invalid session token: %w", err) + } + + if sTok.AssertVerb(sessionSDK.VerbObjectDelete) { + // if session relates to object's removal, we don't check + // relation of the tombstone to the session here since user + // can't predict tomb's ID. + err = assertSessionRelation(*sTok, cnr, nil) + } else { + err = assertSessionRelation(*sTok, cnr, obj) + } + + if err != nil { + return nil, err + } + } + + return sTok, nil +} diff --git a/pkg/services/object/ape/metadata_test.go b/pkg/services/object/ape/metadata_test.go new file mode 100644 index 000000000..fd919008f --- /dev/null +++ b/pkg/services/object/ape/metadata_test.go @@ -0,0 +1,164 @@ +package ape + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + sigutilV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/signature" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/google/uuid" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestRequestOwner(t *testing.T) { + containerOwner, err := keys.NewPrivateKey() + require.NoError(t, err) + + userPk, err := keys.NewPrivateKey() + require.NoError(t, err) + + var userID user.ID + user.IDFromKey(&userID, userPk.PrivateKey.PublicKey) + + var userSignature refs.Signature + userSignature.SetKey(userPk.PublicKey().Bytes()) + + vh := new(sessionV2.RequestVerificationHeader) + vh.SetBodySignature(&userSignature) + + t.Run("empty verification header", func(t *testing.T) { + req := Metadata{} + checkOwner(t, req, nil, errEmptyVerificationHeader) + }) + t.Run("empty verification header signature", func(t *testing.T) { + req := Metadata{ + VerificationHeader: new(sessionV2.RequestVerificationHeader), + } + checkOwner(t, req, nil, errEmptyBodySig) + }) + t.Run("no tokens", func(t *testing.T) { + req := Metadata{ + VerificationHeader: vh, + } + checkOwner(t, req, userPk.PublicKey(), nil) + }) + + t.Run("bearer without impersonate, no session", func(t *testing.T) { + req := Metadata{ + VerificationHeader: vh, + BearerToken: newBearer(t, containerOwner, userID, false), + } + checkOwner(t, req, userPk.PublicKey(), nil) + }) + t.Run("bearer with impersonate, no session", func(t *testing.T) { + req := Metadata{ + VerificationHeader: vh, + BearerToken: newBearer(t, containerOwner, userID, true), + } + checkOwner(t, req, containerOwner.PublicKey(), nil) + }) + t.Run("bearer with impersonate, with session", func(t *testing.T) { + // To check that bearer token takes priority, use different key to sign session token. + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + req := Metadata{ + VerificationHeader: vh, + BearerToken: newBearer(t, containerOwner, userID, true), + SessionToken: newSession(t, pk), + } + checkOwner(t, req, containerOwner.PublicKey(), nil) + }) + t.Run("with session", func(t *testing.T) { + req := Metadata{ + VerificationHeader: vh, + SessionToken: newSession(t, containerOwner), + } + checkOwner(t, req, containerOwner.PublicKey(), nil) + }) + t.Run("malformed session token", func(t *testing.T) { + // This test is tricky: session token has issuer field and signature, which must correspond to each other. + // SDK prevents constructing such token in the first place, but it is still possible via API. + // Thus, construct v2 token, convert it to SDK one and pass to our function. + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + var user1 user.ID + user.IDFromKey(&user1, pk.PrivateKey.PublicKey) + + var id refs.OwnerID + id.SetValue(user1.WalletBytes()) + + raw, err := uuid.New().MarshalBinary() + require.NoError(t, err) + + var cidV2 refs.ContainerID + cidtest.ID().WriteToV2(&cidV2) + + sessionCtx := new(sessionV2.ObjectSessionContext) + sessionCtx.SetTarget(&cidV2) + + var body sessionV2.TokenBody + body.SetOwnerID(&id) + body.SetID(raw) + body.SetLifetime(new(sessionV2.TokenLifetime)) + body.SetSessionKey(pk.PublicKey().Bytes()) + body.SetContext(sessionCtx) + + var tokV2 sessionV2.Token + tokV2.SetBody(&body) + require.NoError(t, sigutilV2.SignData(&containerOwner.PrivateKey, smWrapper{Token: &tokV2})) + require.NoError(t, sigutilV2.VerifyData(smWrapper{Token: &tokV2})) + + var tok sessionSDK.Object + require.NoError(t, tok.ReadFromV2(tokV2)) + + req := Metadata{ + VerificationHeader: vh, + SessionToken: &tok, + } + checkOwner(t, req, nil, errInvalidSessionOwner) + }) +} + +type smWrapper struct { + *sessionV2.Token +} + +func (s smWrapper) ReadSignedData(data []byte) ([]byte, error) { + return s.Token.GetBody().StableMarshal(data), nil +} + +func (s smWrapper) SignedDataSize() int { + return s.Token.GetBody().StableSize() +} + +func newSession(t *testing.T, pk *keys.PrivateKey) *sessionSDK.Object { + var tok sessionSDK.Object + require.NoError(t, tok.Sign(pk.PrivateKey)) + return &tok +} + +func newBearer(t *testing.T, pk *keys.PrivateKey, user user.ID, impersonate bool) *bearer.Token { + var tok bearer.Token + tok.SetImpersonate(impersonate) + tok.ForUser(user) + require.NoError(t, tok.Sign(pk.PrivateKey)) + return &tok +} + +func checkOwner(t *testing.T, req Metadata, expected *keys.PublicKey, expectedErr error) { + _, actual, err := req.RequestOwner() + if expectedErr != nil { + require.ErrorIs(t, err, expectedErr) + return + } + + require.NoError(t, err) + require.Equal(t, expected, actual) +} diff --git a/pkg/services/object/ape/service.go b/pkg/services/object/ape/service.go index d9594a3fc..e199e2638 100644 --- a/pkg/services/object/ape/service.go +++ b/pkg/services/object/ape/service.go @@ -2,9 +2,6 @@ package ape import ( "context" - "encoding/hex" - "errors" - "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" objectSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" @@ -12,19 +9,18 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" ) -var errFailedToCastToRequestContext = errors.New("failed cast to RequestContext") - type Service struct { apeChecker Checker + extractor RequestInfoExtractor + next objectSvc.ServiceServer } @@ -64,9 +60,10 @@ func NewStorageEngineHeaderProvider(e *engine.StorageEngine, s *getsvc.Service) } } -func NewService(apeChecker Checker, next objectSvc.ServiceServer) *Service { +func NewService(apeChecker Checker, extractor RequestInfoExtractor, next objectSvc.ServiceServer) *Service { return &Service{ apeChecker: apeChecker, + extractor: extractor, next: next, } } @@ -76,15 +73,9 @@ type getStreamBasicChecker struct { apeChecker Checker - namespace string + metadata Metadata - senderKey []byte - - containerOwner user.ID - - role string - - bearerToken *bearer.Token + reqInfo RequestInfo } func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { @@ -95,15 +86,15 @@ func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { } prm := Prm{ - Namespace: g.namespace, + Namespace: g.reqInfo.Namespace, Container: cnrID, Object: objID, Header: partInit.GetHeader(), Method: nativeschema.MethodGetObject, - SenderKey: hex.EncodeToString(g.senderKey), - ContainerOwner: g.containerOwner, - Role: g.role, - BearerToken: g.bearerToken, + SenderKey: g.reqInfo.SenderKey, + ContainerOwner: g.reqInfo.ContainerOwner, + Role: g.reqInfo.Role, + BearerToken: g.metadata.BearerToken, XHeaders: resp.GetMetaHeader().GetXHeaders(), } @@ -114,69 +105,53 @@ func (g *getStreamBasicChecker) Send(resp *objectV2.GetResponse) error { return g.GetObjectStream.Send(resp) } -func requestContext(ctx context.Context) (*objectSvc.RequestContext, error) { - untyped := ctx.Value(objectSvc.RequestContextKey) - if untyped == nil { - return nil, fmt.Errorf("no key %s in context", objectSvc.RequestContextKey) - } - rc, ok := untyped.(*objectSvc.RequestContext) - if !ok { - return nil, errFailedToCastToRequestContext - } - return rc, nil -} - func (c *Service) Get(request *objectV2.GetRequest, stream objectSvc.GetObjectStream) error { - reqCtx, err := requestContext(stream.Context()) + md, err := newMetadata(request, request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { - return toStatusErr(err) + return err + } + reqInfo, err := c.extractor.GetRequestInfo(stream.Context(), md, nativeschema.MethodGetObject) + if err != nil { + return err } - return c.next.Get(request, &getStreamBasicChecker{ GetObjectStream: stream, apeChecker: c.apeChecker, - namespace: reqCtx.Namespace, - senderKey: reqCtx.SenderKey, - containerOwner: reqCtx.ContainerOwner, - role: nativeSchemaRole(reqCtx.Role), - bearerToken: reqCtx.BearerToken, + metadata: md, + reqInfo: reqInfo, }) } type putStreamBasicChecker struct { apeChecker Checker + extractor RequestInfoExtractor + next objectSvc.PutObjectStream } func (p *putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { - meta := request.GetMetaHeader() - for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { - meta = origin - } - if partInit, ok := request.GetBody().GetObjectPart().(*objectV2.PutObjectPartInit); ok { - reqCtx, err := requestContext(ctx) + md, err := newMetadata(request, partInit.GetHeader().GetContainerID(), partInit.GetObjectID()) if err != nil { - return toStatusErr(err) + return err } - - cnrID, objID, err := getAddressParamsSDK(partInit.GetHeader().GetContainerID(), partInit.GetObjectID()) + reqInfo, err := p.extractor.GetRequestInfo(ctx, md, nativeschema.MethodPutObject) if err != nil { - return toStatusErr(err) + return err } prm := Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, Header: partInit.GetHeader(), Method: nativeschema.MethodPutObject, - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - Role: nativeSchemaRole(reqCtx.Role), - BearerToken: reqCtx.BearerToken, - XHeaders: meta.GetXHeaders(), + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + Role: reqInfo.Role, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), } if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -196,6 +171,7 @@ func (c *Service) Put(ctx context.Context) (objectSvc.PutObjectStream, error) { return &putStreamBasicChecker{ apeChecker: c.apeChecker, + extractor: c.extractor, next: streamer, }, err } @@ -203,40 +179,36 @@ func (c *Service) Put(ctx context.Context) (objectSvc.PutObjectStream, error) { type patchStreamBasicChecker struct { apeChecker Checker + extractor RequestInfoExtractor + next objectSvc.PatchObjectStream nonFirstSend bool } func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.PatchRequest) error { - meta := request.GetMetaHeader() - for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { - meta = origin - } - if !p.nonFirstSend { p.nonFirstSend = true - reqCtx, err := requestContext(ctx) + md, err := newMetadata(request, request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { - return toStatusErr(err) + return err } - - cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) + reqInfo, err := p.extractor.GetRequestInfo(ctx, md, nativeschema.MethodPatchObject) if err != nil { - return toStatusErr(err) + return err } prm := Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, Method: nativeschema.MethodPatchObject, - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - Role: nativeSchemaRole(reqCtx.Role), - BearerToken: reqCtx.BearerToken, - XHeaders: meta.GetXHeaders(), + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + Role: reqInfo.Role, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), } if err := p.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -256,22 +228,17 @@ func (c *Service) Patch(ctx context.Context) (objectSvc.PatchObjectStream, error return &patchStreamBasicChecker{ apeChecker: c.apeChecker, + extractor: c.extractor, next: streamer, }, err } func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*objectV2.HeadResponse, error) { - meta := request.GetMetaHeader() - for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { - meta = origin - } - - cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) + md, err := newMetadata(request, request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { return nil, err } - - reqCtx, err := requestContext(ctx) + reqInfo, err := c.extractor.GetRequestInfo(ctx, md, nativeschema.MethodHeadObject) if err != nil { return nil, err } @@ -285,7 +252,7 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj switch headerPart := resp.GetBody().GetHeaderPart().(type) { case *objectV2.ShortHeader: cidV2 := new(refs.ContainerID) - cnrID.WriteToV2(cidV2) + md.Container.WriteToV2(cidV2) header.SetContainerID(cidV2) header.SetVersion(headerPart.GetVersion()) header.SetCreationEpoch(headerPart.GetCreationEpoch()) @@ -301,16 +268,16 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj } err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, Header: header, Method: nativeschema.MethodHeadObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - BearerToken: reqCtx.BearerToken, - XHeaders: meta.GetXHeaders(), + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), }) if err != nil { return nil, toStatusErr(err) @@ -319,32 +286,24 @@ func (c *Service) Head(ctx context.Context, request *objectV2.HeadRequest) (*obj } func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.SearchStream) error { - meta := request.GetMetaHeader() - for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { - meta = origin - } - - var cnrID cid.ID - if cnrV2 := request.GetBody().GetContainerID(); cnrV2 != nil { - if err := cnrID.ReadFromV2(*cnrV2); err != nil { - return toStatusErr(err) - } - } - - reqCtx, err := requestContext(stream.Context()) + md, err := newMetadata(request, request.GetBody().GetContainerID(), nil) if err != nil { - return toStatusErr(err) + return err + } + reqInfo, err := c.extractor.GetRequestInfo(stream.Context(), md, nativeschema.MethodSearchObject) + if err != nil { + return err } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, + Namespace: reqInfo.Namespace, + Container: md.Container, Method: nativeschema.MethodSearchObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - BearerToken: reqCtx.BearerToken, - XHeaders: meta.GetXHeaders(), + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), }) if err != nil { return toStatusErr(err) @@ -354,31 +313,25 @@ func (c *Service) Search(request *objectV2.SearchRequest, stream objectSvc.Searc } func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) (*objectV2.DeleteResponse, error) { - meta := request.GetMetaHeader() - for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { - meta = origin - } - - cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) + md, err := newMetadata(request, request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { return nil, err } - - reqCtx, err := requestContext(ctx) + reqInfo, err := c.extractor.GetRequestInfo(ctx, md, nativeschema.MethodDeleteObject) if err != nil { return nil, err } err = c.apeChecker.CheckAPE(ctx, Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, Method: nativeschema.MethodDeleteObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - BearerToken: reqCtx.BearerToken, - XHeaders: meta.GetXHeaders(), + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), }) if err != nil { return nil, toStatusErr(err) @@ -393,31 +346,25 @@ func (c *Service) Delete(ctx context.Context, request *objectV2.DeleteRequest) ( } func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.GetObjectRangeStream) error { - meta := request.GetMetaHeader() - for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { - meta = origin - } - - cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) + md, err := newMetadata(request, request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { - return toStatusErr(err) + return err } - - reqCtx, err := requestContext(stream.Context()) + reqInfo, err := c.extractor.GetRequestInfo(stream.Context(), md, nativeschema.MethodRangeObject) if err != nil { - return toStatusErr(err) + return err } err = c.apeChecker.CheckAPE(stream.Context(), Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, Method: nativeschema.MethodRangeObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - BearerToken: reqCtx.BearerToken, - XHeaders: meta.GetXHeaders(), + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), }) if err != nil { return toStatusErr(err) @@ -427,31 +374,25 @@ func (c *Service) GetRange(request *objectV2.GetRangeRequest, stream objectSvc.G } func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHashRequest) (*objectV2.GetRangeHashResponse, error) { - meta := request.GetMetaHeader() - for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { - meta = origin - } - - cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) + md, err := newMetadata(request, request.GetBody().GetAddress().GetContainerID(), request.GetBody().GetAddress().GetObjectID()) if err != nil { return nil, err } - - reqCtx, err := requestContext(ctx) + reqInfo, err := c.extractor.GetRequestInfo(ctx, md, nativeschema.MethodHashObject) if err != nil { return nil, err } prm := Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, Method: nativeschema.MethodHashObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - BearerToken: reqCtx.BearerToken, - XHeaders: meta.GetXHeaders(), + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), } resp, err := c.next.GetRangeHash(ctx, request) @@ -466,32 +407,26 @@ func (c *Service) GetRangeHash(ctx context.Context, request *objectV2.GetRangeHa } func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { - meta := request.GetMetaHeader() - for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { - meta = origin - } - - cnrID, objID, err := getAddressParamsSDK(request.GetBody().GetObject().GetHeader().GetContainerID(), request.GetBody().GetObject().GetObjectID()) + md, err := newMetadata(request, request.GetBody().GetObject().GetHeader().GetContainerID(), request.GetBody().GetObject().GetObjectID()) if err != nil { return nil, err } - - reqCtx, err := requestContext(ctx) + reqInfo, err := c.extractor.GetRequestInfo(ctx, md, nativeschema.MethodPutObject) if err != nil { return nil, err } prm := Prm{ - Namespace: reqCtx.Namespace, - Container: cnrID, - Object: objID, + Namespace: reqInfo.Namespace, + Container: md.Container, + Object: md.Object, Header: request.GetBody().GetObject().GetHeader(), Method: nativeschema.MethodPutObject, - Role: nativeSchemaRole(reqCtx.Role), - SenderKey: hex.EncodeToString(reqCtx.SenderKey), - ContainerOwner: reqCtx.ContainerOwner, - BearerToken: reqCtx.BearerToken, - XHeaders: meta.GetXHeaders(), + Role: reqInfo.Role, + SenderKey: reqInfo.SenderKey, + ContainerOwner: reqInfo.ContainerOwner, + BearerToken: md.BearerToken, + XHeaders: md.MetaHeader.GetXHeaders(), } if err = c.apeChecker.CheckAPE(ctx, prm); err != nil { @@ -501,18 +436,36 @@ func (c *Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequ return c.next.PutSingle(ctx, request) } -func getAddressParamsSDK(cidV2 *refs.ContainerID, objV2 *refs.ObjectID) (cnrID cid.ID, objID *oid.ID, err error) { - if cidV2 != nil { - if err = cnrID.ReadFromV2(*cidV2); err != nil { - return - } +type request interface { + GetMetaHeader() *session.RequestMetaHeader + GetVerificationHeader() *session.RequestVerificationHeader +} + +func newMetadata(request request, cnrV2 *refs.ContainerID, objV2 *refs.ObjectID) (md Metadata, err error) { + meta := request.GetMetaHeader() + for origin := meta.GetOrigin(); origin != nil; origin = meta.GetOrigin() { + meta = origin } - if objV2 != nil { - objID = new(oid.ID) - if err = objID.ReadFromV2(*objV2); err != nil { - return - } + cnrID, objID, err := getAddressParamsSDK(cnrV2, objV2) + if err != nil { + return + } + session, err := readSessionToken(cnrID, objID, meta.GetSessionToken()) + if err != nil { + return + } + bearer, err := originalBearerToken(request.GetMetaHeader()) + if err != nil { + return + } + + md = Metadata{ + Container: cnrID, + Object: objID, + VerificationHeader: request.GetVerificationHeader(), + SessionToken: session, + BearerToken: bearer, } return } diff --git a/pkg/services/object/ape/types.go b/pkg/services/object/ape/types.go index 46e55360d..97dbfa658 100644 --- a/pkg/services/object/ape/types.go +++ b/pkg/services/object/ape/types.go @@ -7,3 +7,11 @@ import "context" type Checker interface { CheckAPE(context.Context, Prm) error } + +// InnerRingFetcher is an interface that must provide +// Inner Ring information. +type InnerRingFetcher interface { + // InnerRingKeys must return list of public keys of + // the actual inner ring. + InnerRingKeys(ctx context.Context) ([][]byte, error) +} diff --git a/pkg/services/object/ape/util.go b/pkg/services/object/ape/util.go new file mode 100644 index 000000000..5cd2caa50 --- /dev/null +++ b/pkg/services/object/ape/util.go @@ -0,0 +1,169 @@ +package ape + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "errors" + "fmt" + + refsV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" + sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +func getAddressParamsSDK(cidV2 *refsV2.ContainerID, objV2 *refsV2.ObjectID) (cnrID cid.ID, objID *oid.ID, err error) { + if cidV2 != nil { + if err = cnrID.ReadFromV2(*cidV2); err != nil { + return + } + } else { + err = errMissingContainerID + return + } + + if objV2 != nil { + objID = new(oid.ID) + if err = objID.ReadFromV2(*objV2); err != nil { + return + } + } + return +} + +// originalBearerToken goes down to original request meta header and fetches +// bearer token from there. +func originalBearerToken(header *sessionV2.RequestMetaHeader) (*bearer.Token, error) { + for header.GetOrigin() != nil { + header = header.GetOrigin() + } + + tokV2 := header.GetBearerToken() + if tokV2 == nil { + return nil, nil + } + + var tok bearer.Token + return &tok, tok.ReadFromV2(*tokV2) +} + +func ownerFromToken(token *sessionSDK.Object) (*user.ID, *keys.PublicKey, error) { + // 1. First check signature of session token. + if !token.VerifySignature() { + return nil, nil, errInvalidSessionSig + } + + // 2. Then check if session token owner issued the session token + // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion + var tokV2 sessionV2.Token + token.WriteToV2(&tokV2) + + tokenIssuerKey, err := unmarshalPublicKey(tokV2.GetSignature().GetKey()) + if err != nil { + return nil, nil, fmt.Errorf("invalid key in session token signature: %w", err) + } + + tokenIssuer := token.Issuer() + + if !isOwnerFromKey(tokenIssuer, tokenIssuerKey) { + // TODO: #767 in this case we can issue all owner keys from frostfs.id and check once again + return nil, nil, errInvalidSessionOwner + } + + return &tokenIssuer, tokenIssuerKey, nil +} + +func originalBodySignature(v *sessionV2.RequestVerificationHeader) *refsV2.Signature { + if v == nil { + return nil + } + + for v.GetOrigin() != nil { + v = v.GetOrigin() + } + + return v.GetBodySignature() +} + +func unmarshalPublicKey(bs []byte) (*keys.PublicKey, error) { + return keys.NewPublicKeyFromBytes(bs, elliptic.P256()) +} + +func isOwnerFromKey(id user.ID, key *keys.PublicKey) bool { + if key == nil { + return false + } + + var id2 user.ID + user.IDFromKey(&id2, (ecdsa.PublicKey)(*key)) + + return id2.Equals(id) +} + +// assertVerb checks that token verb corresponds to the method. +func assertVerb(tok sessionSDK.Object, method string) bool { + switch method { + case nativeschema.MethodPutObject: + return tok.AssertVerb(sessionSDK.VerbObjectPut, sessionSDK.VerbObjectDelete, sessionSDK.VerbObjectPatch) + case nativeschema.MethodDeleteObject: + return tok.AssertVerb(sessionSDK.VerbObjectDelete) + case nativeschema.MethodGetObject: + return tok.AssertVerb(sessionSDK.VerbObjectGet) + case nativeschema.MethodHeadObject: + return tok.AssertVerb( + sessionSDK.VerbObjectHead, + sessionSDK.VerbObjectGet, + sessionSDK.VerbObjectDelete, + sessionSDK.VerbObjectRange, + sessionSDK.VerbObjectRangeHash, + sessionSDK.VerbObjectPatch, + ) + case nativeschema.MethodSearchObject: + return tok.AssertVerb(sessionSDK.VerbObjectSearch, sessionSDK.VerbObjectDelete) + case nativeschema.MethodRangeObject: + return tok.AssertVerb(sessionSDK.VerbObjectRange, sessionSDK.VerbObjectRangeHash, sessionSDK.VerbObjectPatch) + case nativeschema.MethodHashObject: + return tok.AssertVerb(sessionSDK.VerbObjectRangeHash) + case nativeschema.MethodPatchObject: + return tok.AssertVerb(sessionSDK.VerbObjectPatch) + } + return false +} + +// assertSessionRelation checks if given token describing the FrostFS session +// relates to the given container and optional object. Missing object +// means that the context isn't bound to any FrostFS object in the container. +// Returns no error iff relation is correct. Criteria: +// +// session is bound to the given container +// object is not specified or session is bound to this object +// +// Session MUST be bound to the particular container, otherwise behavior is undefined. +func assertSessionRelation(tok sessionSDK.Object, cnr cid.ID, obj *oid.ID) error { + if !tok.AssertContainer(cnr) { + return errors.New("requested container is not related to the session") + } + + if obj != nil && !tok.AssertObject(*obj) { + return errors.New("requested object is not related to the session") + } + + return nil +} + +func unmarshalPublicKeyWithOwner(rawKey []byte) (*user.ID, *keys.PublicKey, error) { + key, err := unmarshalPublicKey(rawKey) + if err != nil { + return nil, nil, fmt.Errorf("invalid signature key: %w", err) + } + + var idSender user.ID + user.IDFromKey(&idSender, (ecdsa.PublicKey)(*key)) + + return &idSender, key, nil +} diff --git a/pkg/services/object/ape/util_test.go b/pkg/services/object/ape/util_test.go new file mode 100644 index 000000000..916bce427 --- /dev/null +++ b/pkg/services/object/ape/util_test.go @@ -0,0 +1,84 @@ +package ape + +import ( + "slices" + "testing" + + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" + nativeschema "git.frostfs.info/TrueCloudLab/policy-engine/schema/native" + "github.com/stretchr/testify/require" +) + +func TestIsVerbCompatible(t *testing.T) { + table := map[string][]sessionSDK.ObjectVerb{ + nativeschema.MethodPutObject: {sessionSDK.VerbObjectPut, sessionSDK.VerbObjectDelete, sessionSDK.VerbObjectPatch}, + nativeschema.MethodDeleteObject: {sessionSDK.VerbObjectDelete}, + nativeschema.MethodGetObject: {sessionSDK.VerbObjectGet}, + nativeschema.MethodHeadObject: { + sessionSDK.VerbObjectHead, + sessionSDK.VerbObjectGet, + sessionSDK.VerbObjectDelete, + sessionSDK.VerbObjectRange, + sessionSDK.VerbObjectRangeHash, + sessionSDK.VerbObjectPatch, + }, + nativeschema.MethodRangeObject: {sessionSDK.VerbObjectRange, sessionSDK.VerbObjectRangeHash, sessionSDK.VerbObjectPatch}, + nativeschema.MethodHashObject: {sessionSDK.VerbObjectRangeHash}, + nativeschema.MethodSearchObject: {sessionSDK.VerbObjectSearch, sessionSDK.VerbObjectDelete}, + nativeschema.MethodPatchObject: {sessionSDK.VerbObjectPatch}, + } + + verbs := []sessionSDK.ObjectVerb{ + sessionSDK.VerbObjectPut, + sessionSDK.VerbObjectDelete, + sessionSDK.VerbObjectHead, + sessionSDK.VerbObjectRange, + sessionSDK.VerbObjectRangeHash, + sessionSDK.VerbObjectGet, + sessionSDK.VerbObjectSearch, + sessionSDK.VerbObjectPatch, + } + + var tok sessionSDK.Object + + for op, list := range table { + for _, verb := range verbs { + contains := slices.Contains(list, verb) + + tok.ForVerb(verb) + + require.Equal(t, contains, assertVerb(tok, op), + "%v in token, %s executing", verb, op) + } + } +} + +func TestAssertSessionRelation(t *testing.T) { + var tok sessionSDK.Object + cnr := cidtest.ID() + cnrOther := cidtest.ID() + obj := oidtest.ID() + objOther := oidtest.ID() + + // make sure ids differ, otherwise test won't work correctly + require.False(t, cnrOther.Equals(cnr)) + require.False(t, objOther.Equals(obj)) + + // bind session to the container (required) + tok.BindContainer(cnr) + + // test container-global session + require.NoError(t, assertSessionRelation(tok, cnr, nil)) + require.NoError(t, assertSessionRelation(tok, cnr, &obj)) + require.Error(t, assertSessionRelation(tok, cnrOther, nil)) + require.Error(t, assertSessionRelation(tok, cnrOther, &obj)) + + // limit the session to the particular object + tok.LimitByObjects(obj) + + // test fixed object session (here obj arg must be non-nil everywhere) + require.NoError(t, assertSessionRelation(tok, cnr, &obj)) + require.Error(t, assertSessionRelation(tok, cnr, &objOther)) +} From ccdd6cb767d728ce85e6da62b6b0fc9c760f69d8 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 19 Mar 2025 14:46:33 +0300 Subject: [PATCH 1818/1943] [#1052] object: Nuke out `acl` middleware * Remove `acl` package as it's no longer used; * Remove `RequestContext`; * Fix `cmd/frostfs-node`. Signed-off-by: Airat Arifullin --- cmd/frostfs-node/object.go | 17 +- pkg/services/object/acl/eacl/v2/eacl_test.go | 166 ---- pkg/services/object/acl/eacl/v2/headers.go | 246 ------ pkg/services/object/acl/eacl/v2/object.go | 92 --- pkg/services/object/acl/eacl/v2/opts.go | 11 - pkg/services/object/acl/eacl/v2/xheader.go | 69 -- pkg/services/object/acl/v2/errors.go | 20 - pkg/services/object/acl/v2/opts.go | 12 - pkg/services/object/acl/v2/request.go | 152 ---- pkg/services/object/acl/v2/request_test.go | 164 ---- pkg/services/object/acl/v2/service.go | 779 ------------------- pkg/services/object/acl/v2/types.go | 11 - pkg/services/object/acl/v2/util.go | 223 ------ pkg/services/object/acl/v2/util_test.go | 131 ---- pkg/services/object/request_context.go | 24 - 15 files changed, 2 insertions(+), 2115 deletions(-) delete mode 100644 pkg/services/object/acl/eacl/v2/eacl_test.go delete mode 100644 pkg/services/object/acl/eacl/v2/headers.go delete mode 100644 pkg/services/object/acl/eacl/v2/object.go delete mode 100644 pkg/services/object/acl/eacl/v2/opts.go delete mode 100644 pkg/services/object/acl/eacl/v2/xheader.go delete mode 100644 pkg/services/object/acl/v2/errors.go delete mode 100644 pkg/services/object/acl/v2/opts.go delete mode 100644 pkg/services/object/acl/v2/request.go delete mode 100644 pkg/services/object/acl/v2/request_test.go delete mode 100644 pkg/services/object/acl/v2/service.go delete mode 100644 pkg/services/object/acl/v2/types.go delete mode 100644 pkg/services/object/acl/v2/util.go delete mode 100644 pkg/services/object/acl/v2/util_test.go delete mode 100644 pkg/services/object/request_context.go diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 652d3ad04..2674be8c7 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -16,7 +16,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/cache" objectTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/object/grpc" objectService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" - v2 "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/acl/v2" objectAPE "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/ape" objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" deletesvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/delete" @@ -174,10 +173,8 @@ func initObjectService(c *cfg) { apeSvc := createAPEService(c, &irFetcher, splitSvc) - aclSvc := createACLServiceV2(c, apeSvc, &irFetcher) - var commonSvc objectService.Common - commonSvc.Init(&c.internals, aclSvc) + commonSvc.Init(&c.internals, apeSvc) respSvc := objectService.NewResponseService( &commonSvc, @@ -284,7 +281,7 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl }) } -func createInnerRingFetcher(c *cfg) v2.InnerRingFetcher { +func createInnerRingFetcher(c *cfg) objectAPE.InnerRingFetcher { return &innerRingFetcherWithNotary{ sidechain: c.cfgMorph.client, } @@ -429,16 +426,6 @@ func createSplitService(c *cfg, sPutV2 *putsvcV2.Service, sGetV2 *getsvcV2.Servi ) } -func createACLServiceV2(c *cfg, apeSvc *objectAPE.Service, irFetcher *cachedIRFetcher) v2.Service { - return v2.New( - apeSvc, - c.netMapSource, - irFetcher, - c.cfgObject.cnrSource, - v2.WithLogger(c.log), - ) -} - func createAPEService(c *cfg, irFetcher *cachedIRFetcher, splitSvc *objectService.TransportSplitter) *objectAPE.Service { return objectAPE.NewService( objectAPE.NewChecker( diff --git a/pkg/services/object/acl/eacl/v2/eacl_test.go b/pkg/services/object/acl/eacl/v2/eacl_test.go deleted file mode 100644 index 94e015abe..000000000 --- a/pkg/services/object/acl/eacl/v2/eacl_test.go +++ /dev/null @@ -1,166 +0,0 @@ -package v2 - -import ( - "context" - "crypto/ecdsa" - "errors" - "testing" - - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" - eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/stretchr/testify/require" -) - -type testLocalStorage struct { - t *testing.T - - expAddr oid.Address - - obj *objectSDK.Object - - err error -} - -func (s *testLocalStorage) Head(ctx context.Context, addr oid.Address) (*objectSDK.Object, error) { - require.True(s.t, addr.Container().Equals(s.expAddr.Container())) - require.True(s.t, addr.Object().Equals(s.expAddr.Object())) - - return s.obj, s.err -} - -func testXHeaders(strs ...string) []session.XHeader { - res := make([]session.XHeader, len(strs)/2) - - for i := 0; i < len(strs); i += 2 { - res[i/2].SetKey(strs[i]) - res[i/2].SetValue(strs[i+1]) - } - - return res -} - -func TestHeadRequest(t *testing.T) { - req := new(objectV2.HeadRequest) - - meta := new(session.RequestMetaHeader) - req.SetMetaHeader(meta) - - body := new(objectV2.HeadRequestBody) - req.SetBody(body) - - addr := oidtest.Address() - - var addrV2 refs.Address - addr.WriteToV2(&addrV2) - - body.SetAddress(&addrV2) - - xKey := "x-key" - xVal := "x-val" - xHdrs := testXHeaders( - xKey, xVal, - ) - - meta.SetXHeaders(xHdrs) - - obj := objectSDK.New() - - attrKey := "attr_key" - attrVal := "attr_val" - var attr objectSDK.Attribute - attr.SetKey(attrKey) - attr.SetValue(attrVal) - obj.SetAttributes(attr) - - table := new(eaclSDK.Table) - - priv, err := keys.NewPrivateKey() - require.NoError(t, err) - senderKey := priv.PublicKey() - - r := eaclSDK.NewRecord() - r.SetOperation(eaclSDK.OperationHead) - r.SetAction(eaclSDK.ActionDeny) - r.AddFilter(eaclSDK.HeaderFromObject, eaclSDK.MatchStringEqual, attrKey, attrVal) - r.AddFilter(eaclSDK.HeaderFromRequest, eaclSDK.MatchStringEqual, xKey, xVal) - eaclSDK.AddFormedTarget(r, eaclSDK.RoleUnknown, (ecdsa.PublicKey)(*senderKey)) - - table.AddRecord(r) - - lStorage := &testLocalStorage{ - t: t, - expAddr: addr, - obj: obj, - } - - id := addr.Object() - - newSource := func(t *testing.T) eaclSDK.TypedHeaderSource { - hdrSrc, err := NewMessageHeaderSource( - lStorage, - NewRequestXHeaderSource(req), - addr.Container(), - WithOID(&id)) - require.NoError(t, err) - return hdrSrc - } - - cnr := addr.Container() - - unit := new(eaclSDK.ValidationUnit). - WithContainerID(&cnr). - WithOperation(eaclSDK.OperationHead). - WithSenderKey(senderKey.Bytes()). - WithEACLTable(table) - - validator := eaclSDK.NewValidator() - - checkAction(t, eaclSDK.ActionDeny, validator, unit.WithHeaderSource(newSource(t))) - - meta.SetXHeaders(nil) - - checkDefaultAction(t, validator, unit.WithHeaderSource(newSource(t))) - - meta.SetXHeaders(xHdrs) - - obj.SetAttributes() - - checkDefaultAction(t, validator, unit.WithHeaderSource(newSource(t))) - - lStorage.err = errors.New("any error") - - checkDefaultAction(t, validator, unit.WithHeaderSource(newSource(t))) - - r.SetAction(eaclSDK.ActionAllow) - - rID := eaclSDK.NewRecord() - rID.SetOperation(eaclSDK.OperationHead) - rID.SetAction(eaclSDK.ActionDeny) - rID.AddObjectIDFilter(eaclSDK.MatchStringEqual, addr.Object()) - eaclSDK.AddFormedTarget(rID, eaclSDK.RoleUnknown, (ecdsa.PublicKey)(*senderKey)) - - table = eaclSDK.NewTable() - table.AddRecord(r) - table.AddRecord(rID) - - unit.WithEACLTable(table) - checkDefaultAction(t, validator, unit.WithHeaderSource(newSource(t))) -} - -func checkAction(t *testing.T, expected eaclSDK.Action, v *eaclSDK.Validator, u *eaclSDK.ValidationUnit) { - actual, fromRule := v.CalculateAction(u) - require.True(t, fromRule) - require.Equal(t, expected, actual) -} - -func checkDefaultAction(t *testing.T, v *eaclSDK.Validator, u *eaclSDK.ValidationUnit) { - actual, fromRule := v.CalculateAction(u) - require.False(t, fromRule) - require.Equal(t, eaclSDK.ActionAllow, actual) -} diff --git a/pkg/services/object/acl/eacl/v2/headers.go b/pkg/services/object/acl/eacl/v2/headers.go deleted file mode 100644 index ecb793df8..000000000 --- a/pkg/services/object/acl/eacl/v2/headers.go +++ /dev/null @@ -1,246 +0,0 @@ -package v2 - -import ( - "context" - "errors" - "fmt" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/acl" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" - refsV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" -) - -type Option func(*cfg) - -type cfg struct { - storage ObjectStorage - - msg XHeaderSource - - cnr cid.ID - obj *oid.ID -} - -type ObjectStorage interface { - Head(context.Context, oid.Address) (*objectSDK.Object, error) -} - -type Request interface { - GetMetaHeader() *session.RequestMetaHeader -} - -type Response interface { - GetMetaHeader() *session.ResponseMetaHeader -} - -type headerSource struct { - requestHeaders []eaclSDK.Header - objectHeaders []eaclSDK.Header - - incompleteObjectHeaders bool -} - -func NewMessageHeaderSource(os ObjectStorage, xhs XHeaderSource, cnrID cid.ID, opts ...Option) (eaclSDK.TypedHeaderSource, error) { - cfg := &cfg{ - storage: os, - cnr: cnrID, - msg: xhs, - } - - for i := range opts { - opts[i](cfg) - } - - if cfg.msg == nil { - return nil, errors.New("message is not provided") - } - - var res headerSource - - err := cfg.readObjectHeaders(&res) - if err != nil { - return nil, err - } - - res.requestHeaders = cfg.msg.GetXHeaders() - - return res, nil -} - -func (h headerSource) HeadersOfType(typ eaclSDK.FilterHeaderType) ([]eaclSDK.Header, bool) { - switch typ { - default: - return nil, true - case eaclSDK.HeaderFromRequest: - return h.requestHeaders, true - case eaclSDK.HeaderFromObject: - return h.objectHeaders, !h.incompleteObjectHeaders - } -} - -type xHeader session.XHeader - -func (x xHeader) Key() string { - return (*session.XHeader)(&x).GetKey() -} - -func (x xHeader) Value() string { - return (*session.XHeader)(&x).GetValue() -} - -var errMissingOID = errors.New("object ID is missing") - -func (h *cfg) readObjectHeaders(dst *headerSource) error { - switch m := h.msg.(type) { - default: - panic(fmt.Sprintf("unexpected message type %T", h.msg)) - case requestXHeaderSource: - return h.readObjectHeadersFromRequestXHeaderSource(m, dst) - case responseXHeaderSource: - return h.readObjectHeadersResponseXHeaderSource(m, dst) - } -} - -func (h *cfg) readObjectHeadersFromRequestXHeaderSource(m requestXHeaderSource, dst *headerSource) error { - switch req := m.req.(type) { - case - *objectV2.GetRequest, - *objectV2.HeadRequest: - if h.obj == nil { - return errMissingOID - } - - objHeaders, completed := h.localObjectHeaders(h.cnr, h.obj) - - dst.objectHeaders = objHeaders - dst.incompleteObjectHeaders = !completed - case - *objectV2.GetRangeRequest, - *objectV2.GetRangeHashRequest, - *objectV2.DeleteRequest: - if h.obj == nil { - return errMissingOID - } - - dst.objectHeaders = addressHeaders(h.cnr, h.obj) - case *objectV2.PutRequest: - if v, ok := req.GetBody().GetObjectPart().(*objectV2.PutObjectPartInit); ok { - oV2 := new(objectV2.Object) - oV2.SetObjectID(v.GetObjectID()) - oV2.SetHeader(v.GetHeader()) - - dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) - } - case *objectV2.PutSingleRequest: - dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(req.GetBody().GetObject()), h.cnr, h.obj) - case *objectV2.SearchRequest: - cnrV2 := req.GetBody().GetContainerID() - var cnr cid.ID - - if cnrV2 != nil { - if err := cnr.ReadFromV2(*cnrV2); err != nil { - return fmt.Errorf("can't parse container ID: %w", err) - } - } - - dst.objectHeaders = []eaclSDK.Header{cidHeader(cnr)} - } - return nil -} - -func (h *cfg) readObjectHeadersResponseXHeaderSource(m responseXHeaderSource, dst *headerSource) error { - switch resp := m.resp.(type) { - default: - objectHeaders, completed := h.localObjectHeaders(h.cnr, h.obj) - - dst.objectHeaders = objectHeaders - dst.incompleteObjectHeaders = !completed - case *objectV2.GetResponse: - if v, ok := resp.GetBody().GetObjectPart().(*objectV2.GetObjectPartInit); ok { - oV2 := new(objectV2.Object) - oV2.SetObjectID(v.GetObjectID()) - oV2.SetHeader(v.GetHeader()) - - dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) - } - case *objectV2.HeadResponse: - oV2 := new(objectV2.Object) - - var hdr *objectV2.Header - - switch v := resp.GetBody().GetHeaderPart().(type) { - case *objectV2.ShortHeader: - hdr = new(objectV2.Header) - - var idV2 refsV2.ContainerID - h.cnr.WriteToV2(&idV2) - - hdr.SetContainerID(&idV2) - hdr.SetVersion(v.GetVersion()) - hdr.SetCreationEpoch(v.GetCreationEpoch()) - hdr.SetOwnerID(v.GetOwnerID()) - hdr.SetObjectType(v.GetObjectType()) - hdr.SetPayloadLength(v.GetPayloadLength()) - case *objectV2.HeaderWithSignature: - hdr = v.GetHeader() - } - - oV2.SetHeader(hdr) - - dst.objectHeaders = headersFromObject(objectSDK.NewFromV2(oV2), h.cnr, h.obj) - } - return nil -} - -func (h *cfg) localObjectHeaders(cnr cid.ID, idObj *oid.ID) ([]eaclSDK.Header, bool) { - if idObj != nil { - var addr oid.Address - addr.SetContainer(cnr) - addr.SetObject(*idObj) - - obj, err := h.storage.Head(context.TODO(), addr) - if err == nil { - return headersFromObject(obj, cnr, idObj), true - } - } - - return addressHeaders(cnr, idObj), false -} - -func cidHeader(idCnr cid.ID) sysObjHdr { - return sysObjHdr{ - k: acl.FilterObjectContainerID, - v: idCnr.EncodeToString(), - } -} - -func oidHeader(obj oid.ID) sysObjHdr { - return sysObjHdr{ - k: acl.FilterObjectID, - v: obj.EncodeToString(), - } -} - -func ownerIDHeader(ownerID user.ID) sysObjHdr { - return sysObjHdr{ - k: acl.FilterObjectOwnerID, - v: ownerID.EncodeToString(), - } -} - -func addressHeaders(cnr cid.ID, oid *oid.ID) []eaclSDK.Header { - hh := make([]eaclSDK.Header, 0, 2) - hh = append(hh, cidHeader(cnr)) - - if oid != nil { - hh = append(hh, oidHeader(*oid)) - } - - return hh -} diff --git a/pkg/services/object/acl/eacl/v2/object.go b/pkg/services/object/acl/eacl/v2/object.go deleted file mode 100644 index 92570a3c5..000000000 --- a/pkg/services/object/acl/eacl/v2/object.go +++ /dev/null @@ -1,92 +0,0 @@ -package v2 - -import ( - "strconv" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/acl" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" - objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -type sysObjHdr struct { - k, v string -} - -func (s sysObjHdr) Key() string { - return s.k -} - -func (s sysObjHdr) Value() string { - return s.v -} - -func u64Value(v uint64) string { - return strconv.FormatUint(v, 10) -} - -func headersFromObject(obj *objectSDK.Object, cnr cid.ID, oid *oid.ID) []eaclSDK.Header { - var count int - for obj := obj; obj != nil; obj = obj.Parent() { - count += 9 + len(obj.Attributes()) - } - - res := make([]eaclSDK.Header, 0, count) - for ; obj != nil; obj = obj.Parent() { - res = append(res, - cidHeader(cnr), - // creation epoch - sysObjHdr{ - k: acl.FilterObjectCreationEpoch, - v: u64Value(obj.CreationEpoch()), - }, - // payload size - sysObjHdr{ - k: acl.FilterObjectPayloadLength, - v: u64Value(obj.PayloadSize()), - }, - // object version - sysObjHdr{ - k: acl.FilterObjectVersion, - v: obj.Version().String(), - }, - // object type - sysObjHdr{ - k: acl.FilterObjectType, - v: obj.Type().String(), - }, - ) - - if oid != nil { - res = append(res, oidHeader(*oid)) - } - - if idOwner := obj.OwnerID(); !idOwner.IsEmpty() { - res = append(res, ownerIDHeader(idOwner)) - } - - cs, ok := obj.PayloadChecksum() - if ok { - res = append(res, sysObjHdr{ - k: acl.FilterObjectPayloadHash, - v: cs.String(), - }) - } - - cs, ok = obj.PayloadHomomorphicHash() - if ok { - res = append(res, sysObjHdr{ - k: acl.FilterObjectHomomorphicHash, - v: cs.String(), - }) - } - - attrs := obj.Attributes() - for i := range attrs { - res = append(res, &attrs[i]) // only pointer attrs can implement eaclSDK.Header interface - } - } - - return res -} diff --git a/pkg/services/object/acl/eacl/v2/opts.go b/pkg/services/object/acl/eacl/v2/opts.go deleted file mode 100644 index d91a21c75..000000000 --- a/pkg/services/object/acl/eacl/v2/opts.go +++ /dev/null @@ -1,11 +0,0 @@ -package v2 - -import ( - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" -) - -func WithOID(v *oid.ID) Option { - return func(c *cfg) { - c.obj = v - } -} diff --git a/pkg/services/object/acl/eacl/v2/xheader.go b/pkg/services/object/acl/eacl/v2/xheader.go deleted file mode 100644 index ce380c117..000000000 --- a/pkg/services/object/acl/eacl/v2/xheader.go +++ /dev/null @@ -1,69 +0,0 @@ -package v2 - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" - eaclSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl" -) - -type XHeaderSource interface { - GetXHeaders() []eaclSDK.Header -} - -type requestXHeaderSource struct { - req Request -} - -func NewRequestXHeaderSource(req Request) XHeaderSource { - return requestXHeaderSource{req: req} -} - -type responseXHeaderSource struct { - resp Response - - req Request -} - -func NewResponseXHeaderSource(resp Response, req Request) XHeaderSource { - return responseXHeaderSource{resp: resp, req: req} -} - -func (s requestXHeaderSource) GetXHeaders() []eaclSDK.Header { - ln := 0 - - for meta := s.req.GetMetaHeader(); meta != nil; meta = meta.GetOrigin() { - ln += len(meta.GetXHeaders()) - } - - res := make([]eaclSDK.Header, 0, ln) - for meta := s.req.GetMetaHeader(); meta != nil; meta = meta.GetOrigin() { - x := meta.GetXHeaders() - for i := range x { - res = append(res, (xHeader)(x[i])) - } - } - - return res -} - -func (s responseXHeaderSource) GetXHeaders() []eaclSDK.Header { - ln := 0 - xHdrs := make([][]session.XHeader, 0) - - for meta := s.req.GetMetaHeader(); meta != nil; meta = meta.GetOrigin() { - x := meta.GetXHeaders() - - ln += len(x) - - xHdrs = append(xHdrs, x) - } - - res := make([]eaclSDK.Header, 0, ln) - - for i := range xHdrs { - for j := range xHdrs[i] { - res = append(res, xHeader(xHdrs[i][j])) - } - } - - return res -} diff --git a/pkg/services/object/acl/v2/errors.go b/pkg/services/object/acl/v2/errors.go deleted file mode 100644 index cd2de174a..000000000 --- a/pkg/services/object/acl/v2/errors.go +++ /dev/null @@ -1,20 +0,0 @@ -package v2 - -import ( - "fmt" -) - -const invalidRequestMessage = "malformed request" - -func malformedRequestError(reason string) error { - return fmt.Errorf("%s: %s", invalidRequestMessage, reason) -} - -var ( - errEmptyBody = malformedRequestError("empty body") - errEmptyVerificationHeader = malformedRequestError("empty verification header") - errEmptyBodySig = malformedRequestError("empty at body signature") - errInvalidSessionSig = malformedRequestError("invalid session token signature") - errInvalidSessionOwner = malformedRequestError("invalid session token owner") - errInvalidVerb = malformedRequestError("session token verb is invalid") -) diff --git a/pkg/services/object/acl/v2/opts.go b/pkg/services/object/acl/v2/opts.go deleted file mode 100644 index 15fcce884..000000000 --- a/pkg/services/object/acl/v2/opts.go +++ /dev/null @@ -1,12 +0,0 @@ -package v2 - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" -) - -// WithLogger returns option to set logger. -func WithLogger(v *logger.Logger) Option { - return func(c *cfg) { - c.log = v - } -} diff --git a/pkg/services/object/acl/v2/request.go b/pkg/services/object/acl/v2/request.go deleted file mode 100644 index 8bd34ccb3..000000000 --- a/pkg/services/object/acl/v2/request.go +++ /dev/null @@ -1,152 +0,0 @@ -package v2 - -import ( - "crypto/ecdsa" - "fmt" - - sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" -) - -// RequestInfo groups parsed version-independent (from SDK library) -// request information and raw API request. -type RequestInfo struct { - basicACL acl.Basic - requestRole acl.Role - operation acl.Op // put, get, head, etc. - cnrOwner user.ID // container owner - - // cnrNamespace defined to which namespace a container is belonged. - cnrNamespace string - - idCnr cid.ID - - // optional for some request - // e.g. Put, Search - obj *oid.ID - - senderKey []byte - - bearer *bearer.Token // bearer token of request - - srcRequest any -} - -func (r *RequestInfo) SetBasicACL(basicACL acl.Basic) { - r.basicACL = basicACL -} - -func (r *RequestInfo) SetRequestRole(requestRole acl.Role) { - r.requestRole = requestRole -} - -func (r *RequestInfo) SetSenderKey(senderKey []byte) { - r.senderKey = senderKey -} - -// Request returns raw API request. -func (r RequestInfo) Request() any { - return r.srcRequest -} - -// ContainerOwner returns owner if the container. -func (r RequestInfo) ContainerOwner() user.ID { - return r.cnrOwner -} - -func (r RequestInfo) ContainerNamespace() string { - return r.cnrNamespace -} - -// ObjectID return object ID. -func (r RequestInfo) ObjectID() *oid.ID { - return r.obj -} - -// ContainerID return container ID. -func (r RequestInfo) ContainerID() cid.ID { - return r.idCnr -} - -// CleanBearer forces cleaning bearer token information. -func (r *RequestInfo) CleanBearer() { - r.bearer = nil -} - -// Bearer returns bearer token of the request. -func (r RequestInfo) Bearer() *bearer.Token { - return r.bearer -} - -// BasicACL returns basic ACL of the container. -func (r RequestInfo) BasicACL() acl.Basic { - return r.basicACL -} - -// SenderKey returns public key of the request's sender. -func (r RequestInfo) SenderKey() []byte { - return r.senderKey -} - -// Operation returns request's operation. -func (r RequestInfo) Operation() acl.Op { - return r.operation -} - -// RequestRole returns request sender's role. -func (r RequestInfo) RequestRole() acl.Role { - return r.requestRole -} - -// MetaWithToken groups session and bearer tokens, -// verification header and raw API request. -type MetaWithToken struct { - vheader *sessionV2.RequestVerificationHeader - token *sessionSDK.Object - bearer *bearer.Token - src any -} - -// RequestOwner returns ownerID and its public key -// according to internal meta information. -func (r MetaWithToken) RequestOwner() (*user.ID, *keys.PublicKey, error) { - if r.vheader == nil { - return nil, nil, errEmptyVerificationHeader - } - - if r.bearer != nil && r.bearer.Impersonate() { - return unmarshalPublicKeyWithOwner(r.bearer.SigningKeyBytes()) - } - - // if session token is presented, use it as truth source - if r.token != nil { - // verify signature of session token - return ownerFromToken(r.token) - } - - // otherwise get original body signature - bodySignature := originalBodySignature(r.vheader) - if bodySignature == nil { - return nil, nil, errEmptyBodySig - } - - return unmarshalPublicKeyWithOwner(bodySignature.GetKey()) -} - -func unmarshalPublicKeyWithOwner(rawKey []byte) (*user.ID, *keys.PublicKey, error) { - key, err := unmarshalPublicKey(rawKey) - if err != nil { - return nil, nil, fmt.Errorf("invalid signature key: %w", err) - } - - var idSender user.ID - user.IDFromKey(&idSender, (ecdsa.PublicKey)(*key)) - - return &idSender, key, nil -} diff --git a/pkg/services/object/acl/v2/request_test.go b/pkg/services/object/acl/v2/request_test.go deleted file mode 100644 index 618af3469..000000000 --- a/pkg/services/object/acl/v2/request_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package v2 - -import ( - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" - sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" - sigutilV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/util/signature" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/google/uuid" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" - "github.com/stretchr/testify/require" -) - -func TestRequestOwner(t *testing.T) { - containerOwner, err := keys.NewPrivateKey() - require.NoError(t, err) - - userPk, err := keys.NewPrivateKey() - require.NoError(t, err) - - var userID user.ID - user.IDFromKey(&userID, userPk.PrivateKey.PublicKey) - - var userSignature refs.Signature - userSignature.SetKey(userPk.PublicKey().Bytes()) - - vh := new(sessionV2.RequestVerificationHeader) - vh.SetBodySignature(&userSignature) - - t.Run("empty verification header", func(t *testing.T) { - req := MetaWithToken{} - checkOwner(t, req, nil, errEmptyVerificationHeader) - }) - t.Run("empty verification header signature", func(t *testing.T) { - req := MetaWithToken{ - vheader: new(sessionV2.RequestVerificationHeader), - } - checkOwner(t, req, nil, errEmptyBodySig) - }) - t.Run("no tokens", func(t *testing.T) { - req := MetaWithToken{ - vheader: vh, - } - checkOwner(t, req, userPk.PublicKey(), nil) - }) - - t.Run("bearer without impersonate, no session", func(t *testing.T) { - req := MetaWithToken{ - vheader: vh, - bearer: newBearer(t, containerOwner, userID, false), - } - checkOwner(t, req, userPk.PublicKey(), nil) - }) - t.Run("bearer with impersonate, no session", func(t *testing.T) { - req := MetaWithToken{ - vheader: vh, - bearer: newBearer(t, containerOwner, userID, true), - } - checkOwner(t, req, containerOwner.PublicKey(), nil) - }) - t.Run("bearer with impersonate, with session", func(t *testing.T) { - // To check that bearer token takes priority, use different key to sign session token. - pk, err := keys.NewPrivateKey() - require.NoError(t, err) - - req := MetaWithToken{ - vheader: vh, - bearer: newBearer(t, containerOwner, userID, true), - token: newSession(t, pk), - } - checkOwner(t, req, containerOwner.PublicKey(), nil) - }) - t.Run("with session", func(t *testing.T) { - req := MetaWithToken{ - vheader: vh, - token: newSession(t, containerOwner), - } - checkOwner(t, req, containerOwner.PublicKey(), nil) - }) - t.Run("malformed session token", func(t *testing.T) { - // This test is tricky: session token has issuer field and signature, which must correspond to each other. - // SDK prevents constructing such token in the first place, but it is still possible via API. - // Thus, construct v2 token, convert it to SDK one and pass to our function. - pk, err := keys.NewPrivateKey() - require.NoError(t, err) - - var user1 user.ID - user.IDFromKey(&user1, pk.PrivateKey.PublicKey) - - var id refs.OwnerID - id.SetValue(user1.WalletBytes()) - - raw, err := uuid.New().MarshalBinary() - require.NoError(t, err) - - var cidV2 refs.ContainerID - cidtest.ID().WriteToV2(&cidV2) - - sessionCtx := new(sessionV2.ObjectSessionContext) - sessionCtx.SetTarget(&cidV2) - - var body sessionV2.TokenBody - body.SetOwnerID(&id) - body.SetID(raw) - body.SetLifetime(new(sessionV2.TokenLifetime)) - body.SetSessionKey(pk.PublicKey().Bytes()) - body.SetContext(sessionCtx) - - var tokV2 sessionV2.Token - tokV2.SetBody(&body) - require.NoError(t, sigutilV2.SignData(&containerOwner.PrivateKey, smWrapper{Token: &tokV2})) - require.NoError(t, sigutilV2.VerifyData(smWrapper{Token: &tokV2})) - - var tok sessionSDK.Object - require.NoError(t, tok.ReadFromV2(tokV2)) - - req := MetaWithToken{ - vheader: vh, - token: &tok, - } - checkOwner(t, req, nil, errInvalidSessionOwner) - }) -} - -type smWrapper struct { - *sessionV2.Token -} - -func (s smWrapper) ReadSignedData(data []byte) ([]byte, error) { - return s.Token.GetBody().StableMarshal(data), nil -} - -func (s smWrapper) SignedDataSize() int { - return s.Token.GetBody().StableSize() -} - -func newSession(t *testing.T, pk *keys.PrivateKey) *sessionSDK.Object { - var tok sessionSDK.Object - require.NoError(t, tok.Sign(pk.PrivateKey)) - return &tok -} - -func newBearer(t *testing.T, pk *keys.PrivateKey, user user.ID, impersonate bool) *bearer.Token { - var tok bearer.Token - tok.SetImpersonate(impersonate) - tok.ForUser(user) - require.NoError(t, tok.Sign(pk.PrivateKey)) - return &tok -} - -func checkOwner(t *testing.T, req MetaWithToken, expected *keys.PublicKey, expectedErr error) { - _, actual, err := req.RequestOwner() - if expectedErr != nil { - require.ErrorIs(t, err, expectedErr) - return - } - - require.NoError(t, err) - require.Equal(t, expected, actual) -} diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go deleted file mode 100644 index 86daec6cc..000000000 --- a/pkg/services/object/acl/v2/service.go +++ /dev/null @@ -1,779 +0,0 @@ -package v2 - -import ( - "context" - "errors" - "fmt" - "strings" - - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" - objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" - apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" - cnrSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "go.uber.org/zap" -) - -// Service checks basic ACL rules. -type Service struct { - *cfg - - c objectCore.SenderClassifier -} - -type putStreamBasicChecker struct { - source *Service - next object.PutObjectStream -} - -type patchStreamBasicChecker struct { - source *Service - next object.PatchObjectStream - nonFirstSend bool -} - -// Option represents Service constructor option. -type Option func(*cfg) - -type cfg struct { - log *logger.Logger - - containers container.Source - - irFetcher InnerRingFetcher - - nm netmap.Source - - next object.ServiceServer -} - -// New is a constructor for object ACL checking service. -func New(next object.ServiceServer, - nm netmap.Source, - irf InnerRingFetcher, - cs container.Source, - opts ...Option, -) Service { - cfg := &cfg{ - log: logger.NewLoggerWrapper(zap.L()), - next: next, - nm: nm, - irFetcher: irf, - containers: cs, - } - - for i := range opts { - opts[i](cfg) - } - - return Service{ - cfg: cfg, - c: objectCore.NewSenderClassifier(cfg.irFetcher, cfg.nm, cfg.log), - } -} - -// wrappedGetObjectStream propagates RequestContext into GetObjectStream's context. -// This allows to retrieve already calculated immutable request-specific values in next handler invocation. -type wrappedGetObjectStream struct { - object.GetObjectStream - - requestInfo RequestInfo -} - -func (w *wrappedGetObjectStream) Context() context.Context { - return context.WithValue(w.GetObjectStream.Context(), object.RequestContextKey, &object.RequestContext{ - Namespace: w.requestInfo.ContainerNamespace(), - ContainerOwner: w.requestInfo.ContainerOwner(), - SenderKey: w.requestInfo.SenderKey(), - Role: w.requestInfo.RequestRole(), - BearerToken: w.requestInfo.Bearer(), - }) -} - -func newWrappedGetObjectStreamStream(getObjectStream object.GetObjectStream, reqInfo RequestInfo) object.GetObjectStream { - return &wrappedGetObjectStream{ - GetObjectStream: getObjectStream, - requestInfo: reqInfo, - } -} - -// wrappedRangeStream propagates RequestContext into GetObjectRangeStream's context. -// This allows to retrieve already calculated immutable request-specific values in next handler invocation. -type wrappedRangeStream struct { - object.GetObjectRangeStream - - requestInfo RequestInfo -} - -func (w *wrappedRangeStream) Context() context.Context { - return context.WithValue(w.GetObjectRangeStream.Context(), object.RequestContextKey, &object.RequestContext{ - Namespace: w.requestInfo.ContainerNamespace(), - ContainerOwner: w.requestInfo.ContainerOwner(), - SenderKey: w.requestInfo.SenderKey(), - Role: w.requestInfo.RequestRole(), - BearerToken: w.requestInfo.Bearer(), - }) -} - -func newWrappedRangeStream(rangeStream object.GetObjectRangeStream, reqInfo RequestInfo) object.GetObjectRangeStream { - return &wrappedRangeStream{ - GetObjectRangeStream: rangeStream, - requestInfo: reqInfo, - } -} - -// wrappedSearchStream propagates RequestContext into SearchStream's context. -// This allows to retrieve already calculated immutable request-specific values in next handler invocation. -type wrappedSearchStream struct { - object.SearchStream - - requestInfo RequestInfo -} - -func (w *wrappedSearchStream) Context() context.Context { - return context.WithValue(w.SearchStream.Context(), object.RequestContextKey, &object.RequestContext{ - Namespace: w.requestInfo.ContainerNamespace(), - ContainerOwner: w.requestInfo.ContainerOwner(), - SenderKey: w.requestInfo.SenderKey(), - Role: w.requestInfo.RequestRole(), - BearerToken: w.requestInfo.Bearer(), - }) -} - -func newWrappedSearchStream(searchStream object.SearchStream, reqInfo RequestInfo) object.SearchStream { - return &wrappedSearchStream{ - SearchStream: searchStream, - requestInfo: reqInfo, - } -} - -// Get implements ServiceServer interface, makes ACL checks and calls -// next Get method in the ServiceServer pipeline. -func (b Service) Get(request *objectV2.GetRequest, stream object.GetObjectStream) error { - cnr, err := getContainerIDFromRequest(request) - if err != nil { - return err - } - - obj, err := getObjectIDFromRequestBody(request.GetBody()) - if err != nil { - return err - } - - sTok, err := originalSessionToken(request.GetMetaHeader()) - if err != nil { - return err - } - - if sTok != nil { - err = assertSessionRelation(*sTok, cnr, obj) - if err != nil { - return err - } - } - - bTok, err := originalBearerToken(request.GetMetaHeader()) - if err != nil { - return err - } - - req := MetaWithToken{ - vheader: request.GetVerificationHeader(), - token: sTok, - bearer: bTok, - src: request, - } - - reqInfo, err := b.findRequestInfo(stream.Context(), req, cnr, acl.OpObjectGet) - if err != nil { - return err - } - - reqInfo.obj = obj - - return b.next.Get(request, newWrappedGetObjectStreamStream(stream, reqInfo)) -} - -func (b Service) Put(ctx context.Context) (object.PutObjectStream, error) { - streamer, err := b.next.Put(ctx) - - return putStreamBasicChecker{ - source: &b, - next: streamer, - }, err -} - -func (b Service) Patch(ctx context.Context) (object.PatchObjectStream, error) { - streamer, err := b.next.Patch(ctx) - - return &patchStreamBasicChecker{ - source: &b, - next: streamer, - }, err -} - -func (b Service) Head( - ctx context.Context, - request *objectV2.HeadRequest, -) (*objectV2.HeadResponse, error) { - cnr, err := getContainerIDFromRequest(request) - if err != nil { - return nil, err - } - - obj, err := getObjectIDFromRequestBody(request.GetBody()) - if err != nil { - return nil, err - } - - sTok, err := originalSessionToken(request.GetMetaHeader()) - if err != nil { - return nil, err - } - - if sTok != nil { - err = assertSessionRelation(*sTok, cnr, obj) - if err != nil { - return nil, err - } - } - - bTok, err := originalBearerToken(request.GetMetaHeader()) - if err != nil { - return nil, err - } - - req := MetaWithToken{ - vheader: request.GetVerificationHeader(), - token: sTok, - bearer: bTok, - src: request, - } - - reqInfo, err := b.findRequestInfo(ctx, req, cnr, acl.OpObjectHead) - if err != nil { - return nil, err - } - - reqInfo.obj = obj - - return b.next.Head(requestContext(ctx, reqInfo), request) -} - -func (b Service) Search(request *objectV2.SearchRequest, stream object.SearchStream) error { - id, err := getContainerIDFromRequest(request) - if err != nil { - return err - } - - sTok, err := originalSessionToken(request.GetMetaHeader()) - if err != nil { - return err - } - - if sTok != nil { - err = assertSessionRelation(*sTok, id, nil) - if err != nil { - return err - } - } - - bTok, err := originalBearerToken(request.GetMetaHeader()) - if err != nil { - return err - } - - req := MetaWithToken{ - vheader: request.GetVerificationHeader(), - token: sTok, - bearer: bTok, - src: request, - } - - reqInfo, err := b.findRequestInfo(stream.Context(), req, id, acl.OpObjectSearch) - if err != nil { - return err - } - - return b.next.Search(request, newWrappedSearchStream(stream, reqInfo)) -} - -func (b Service) Delete( - ctx context.Context, - request *objectV2.DeleteRequest, -) (*objectV2.DeleteResponse, error) { - cnr, err := getContainerIDFromRequest(request) - if err != nil { - return nil, err - } - - obj, err := getObjectIDFromRequestBody(request.GetBody()) - if err != nil { - return nil, err - } - - sTok, err := originalSessionToken(request.GetMetaHeader()) - if err != nil { - return nil, err - } - - if sTok != nil { - err = assertSessionRelation(*sTok, cnr, obj) - if err != nil { - return nil, err - } - } - - bTok, err := originalBearerToken(request.GetMetaHeader()) - if err != nil { - return nil, err - } - - req := MetaWithToken{ - vheader: request.GetVerificationHeader(), - token: sTok, - bearer: bTok, - src: request, - } - - reqInfo, err := b.findRequestInfo(ctx, req, cnr, acl.OpObjectDelete) - if err != nil { - return nil, err - } - - reqInfo.obj = obj - - return b.next.Delete(requestContext(ctx, reqInfo), request) -} - -func (b Service) GetRange(request *objectV2.GetRangeRequest, stream object.GetObjectRangeStream) error { - cnr, err := getContainerIDFromRequest(request) - if err != nil { - return err - } - - obj, err := getObjectIDFromRequestBody(request.GetBody()) - if err != nil { - return err - } - - sTok, err := originalSessionToken(request.GetMetaHeader()) - if err != nil { - return err - } - - if sTok != nil { - err = assertSessionRelation(*sTok, cnr, obj) - if err != nil { - return err - } - } - - bTok, err := originalBearerToken(request.GetMetaHeader()) - if err != nil { - return err - } - - req := MetaWithToken{ - vheader: request.GetVerificationHeader(), - token: sTok, - bearer: bTok, - src: request, - } - - reqInfo, err := b.findRequestInfo(stream.Context(), req, cnr, acl.OpObjectRange) - if err != nil { - return err - } - - reqInfo.obj = obj - - return b.next.GetRange(request, newWrappedRangeStream(stream, reqInfo)) -} - -func requestContext(ctx context.Context, reqInfo RequestInfo) context.Context { - return context.WithValue(ctx, object.RequestContextKey, &object.RequestContext{ - Namespace: reqInfo.ContainerNamespace(), - ContainerOwner: reqInfo.ContainerOwner(), - SenderKey: reqInfo.SenderKey(), - Role: reqInfo.RequestRole(), - BearerToken: reqInfo.Bearer(), - }) -} - -func (b Service) GetRangeHash( - ctx context.Context, - request *objectV2.GetRangeHashRequest, -) (*objectV2.GetRangeHashResponse, error) { - cnr, err := getContainerIDFromRequest(request) - if err != nil { - return nil, err - } - - obj, err := getObjectIDFromRequestBody(request.GetBody()) - if err != nil { - return nil, err - } - - sTok, err := originalSessionToken(request.GetMetaHeader()) - if err != nil { - return nil, err - } - - if sTok != nil { - err = assertSessionRelation(*sTok, cnr, obj) - if err != nil { - return nil, err - } - } - - bTok, err := originalBearerToken(request.GetMetaHeader()) - if err != nil { - return nil, err - } - - req := MetaWithToken{ - vheader: request.GetVerificationHeader(), - token: sTok, - bearer: bTok, - src: request, - } - - reqInfo, err := b.findRequestInfo(ctx, req, cnr, acl.OpObjectHash) - if err != nil { - return nil, err - } - - reqInfo.obj = obj - - return b.next.GetRangeHash(requestContext(ctx, reqInfo), request) -} - -func (b Service) PutSingle(ctx context.Context, request *objectV2.PutSingleRequest) (*objectV2.PutSingleResponse, error) { - cnr, err := getContainerIDFromRequest(request) - if err != nil { - return nil, err - } - - idV2 := request.GetBody().GetObject().GetHeader().GetOwnerID() - if idV2 == nil { - return nil, errors.New("missing object owner") - } - - var idOwner user.ID - - err = idOwner.ReadFromV2(*idV2) - if err != nil { - return nil, fmt.Errorf("invalid object owner: %w", err) - } - - obj, err := getObjectIDFromRefObjectID(request.GetBody().GetObject().GetObjectID()) - if err != nil { - return nil, err - } - - var sTok *sessionSDK.Object - sTok, err = readSessionToken(cnr, obj, request.GetMetaHeader().GetSessionToken()) - if err != nil { - return nil, err - } - - bTok, err := originalBearerToken(request.GetMetaHeader()) - if err != nil { - return nil, err - } - - req := MetaWithToken{ - vheader: request.GetVerificationHeader(), - token: sTok, - bearer: bTok, - src: request, - } - - reqInfo, err := b.findRequestInfo(ctx, req, cnr, acl.OpObjectPut) - if err != nil { - return nil, err - } - - reqInfo.obj = obj - - return b.next.PutSingle(requestContext(ctx, reqInfo), request) -} - -func (p putStreamBasicChecker) Send(ctx context.Context, request *objectV2.PutRequest) error { - body := request.GetBody() - if body == nil { - return errEmptyBody - } - - part := body.GetObjectPart() - if part, ok := part.(*objectV2.PutObjectPartInit); ok { - cnr, err := getContainerIDFromRequest(request) - if err != nil { - return err - } - - idV2 := part.GetHeader().GetOwnerID() - if idV2 == nil { - return errors.New("missing object owner") - } - - var idOwner user.ID - - err = idOwner.ReadFromV2(*idV2) - if err != nil { - return fmt.Errorf("invalid object owner: %w", err) - } - - objV2 := part.GetObjectID() - var obj *oid.ID - - if objV2 != nil { - obj = new(oid.ID) - - err = obj.ReadFromV2(*objV2) - if err != nil { - return err - } - } - - var sTok *sessionSDK.Object - sTok, err = readSessionToken(cnr, obj, request.GetMetaHeader().GetSessionToken()) - if err != nil { - return err - } - - bTok, err := originalBearerToken(request.GetMetaHeader()) - if err != nil { - return err - } - - req := MetaWithToken{ - vheader: request.GetVerificationHeader(), - token: sTok, - bearer: bTok, - src: request, - } - - reqInfo, err := p.source.findRequestInfo(ctx, req, cnr, acl.OpObjectPut) - if err != nil { - return err - } - - reqInfo.obj = obj - - ctx = requestContext(ctx, reqInfo) - } - - return p.next.Send(ctx, request) -} - -func readSessionToken(cnr cid.ID, obj *oid.ID, tokV2 *session.Token) (*sessionSDK.Object, error) { - var sTok *sessionSDK.Object - - if tokV2 != nil { - sTok = new(sessionSDK.Object) - - err := sTok.ReadFromV2(*tokV2) - if err != nil { - return nil, fmt.Errorf("invalid session token: %w", err) - } - - if sTok.AssertVerb(sessionSDK.VerbObjectDelete) { - // if session relates to object's removal, we don't check - // relation of the tombstone to the session here since user - // can't predict tomb's ID. - err = assertSessionRelation(*sTok, cnr, nil) - } else { - err = assertSessionRelation(*sTok, cnr, obj) - } - - if err != nil { - return nil, err - } - } - - return sTok, nil -} - -func (p putStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PutResponse, error) { - return p.next.CloseAndRecv(ctx) -} - -func (p *patchStreamBasicChecker) Send(ctx context.Context, request *objectV2.PatchRequest) error { - body := request.GetBody() - if body == nil { - return errEmptyBody - } - - if !p.nonFirstSend { - p.nonFirstSend = true - - cnr, err := getContainerIDFromRequest(request) - if err != nil { - return err - } - - objV2 := request.GetBody().GetAddress().GetObjectID() - if objV2 == nil { - return errors.New("missing oid") - } - obj := new(oid.ID) - err = obj.ReadFromV2(*objV2) - if err != nil { - return err - } - - var sTok *sessionSDK.Object - sTok, err = readSessionToken(cnr, obj, request.GetMetaHeader().GetSessionToken()) - if err != nil { - return err - } - - bTok, err := originalBearerToken(request.GetMetaHeader()) - if err != nil { - return err - } - - req := MetaWithToken{ - vheader: request.GetVerificationHeader(), - token: sTok, - bearer: bTok, - src: request, - } - - reqInfo, err := p.source.findRequestInfoWithoutACLOperationAssert(ctx, req, cnr) - if err != nil { - return err - } - - reqInfo.obj = obj - - ctx = requestContext(ctx, reqInfo) - } - - return p.next.Send(ctx, request) -} - -func (p patchStreamBasicChecker) CloseAndRecv(ctx context.Context) (*objectV2.PatchResponse, error) { - return p.next.CloseAndRecv(ctx) -} - -func (b Service) findRequestInfo(ctx context.Context, req MetaWithToken, idCnr cid.ID, op acl.Op) (info RequestInfo, err error) { - cnr, err := b.containers.Get(ctx, idCnr) // fetch actual container - if err != nil { - return info, err - } - - if req.token != nil { - currentEpoch, err := b.nm.Epoch(ctx) - if err != nil { - return info, errors.New("can't fetch current epoch") - } - if req.token.ExpiredAt(currentEpoch) { - return info, new(apistatus.SessionTokenExpired) - } - if req.token.InvalidAt(currentEpoch) { - return info, fmt.Errorf("%s: token is invalid at %d epoch)", - invalidRequestMessage, currentEpoch) - } - - if !assertVerb(*req.token, op) { - return info, errInvalidVerb - } - } - - // find request role and key - ownerID, ownerKey, err := req.RequestOwner() - if err != nil { - return info, err - } - res, err := b.c.Classify(ctx, ownerID, ownerKey, idCnr, cnr.Value) - if err != nil { - return info, err - } - - info.basicACL = cnr.Value.BasicACL() - info.requestRole = res.Role - info.operation = op - info.cnrOwner = cnr.Value.Owner() - info.idCnr = idCnr - - cnrNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(cnr.Value).Zone(), ".ns") - if hasNamespace { - info.cnrNamespace = cnrNamespace - } - - // it is assumed that at the moment the key will be valid, - // otherwise the request would not pass validation - info.senderKey = res.Key - - // add bearer token if it is present in request - info.bearer = req.bearer - - info.srcRequest = req.src - - return info, nil -} - -// findRequestInfoWithoutACLOperationAssert is findRequestInfo without session token verb assert. -func (b Service) findRequestInfoWithoutACLOperationAssert(ctx context.Context, req MetaWithToken, idCnr cid.ID) (info RequestInfo, err error) { - cnr, err := b.containers.Get(ctx, idCnr) // fetch actual container - if err != nil { - return info, err - } - - if req.token != nil { - currentEpoch, err := b.nm.Epoch(ctx) - if err != nil { - return info, errors.New("can't fetch current epoch") - } - if req.token.ExpiredAt(currentEpoch) { - return info, new(apistatus.SessionTokenExpired) - } - if req.token.InvalidAt(currentEpoch) { - return info, fmt.Errorf("%s: token is invalid at %d epoch)", - invalidRequestMessage, currentEpoch) - } - } - - // find request role and key - ownerID, ownerKey, err := req.RequestOwner() - if err != nil { - return info, err - } - res, err := b.c.Classify(ctx, ownerID, ownerKey, idCnr, cnr.Value) - if err != nil { - return info, err - } - - info.basicACL = cnr.Value.BasicACL() - info.requestRole = res.Role - info.cnrOwner = cnr.Value.Owner() - info.idCnr = idCnr - - cnrNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(cnr.Value).Zone(), ".ns") - if hasNamespace { - info.cnrNamespace = cnrNamespace - } - - // it is assumed that at the moment the key will be valid, - // otherwise the request would not pass validation - info.senderKey = res.Key - - // add bearer token if it is present in request - info.bearer = req.bearer - - info.srcRequest = req.src - - return info, nil -} diff --git a/pkg/services/object/acl/v2/types.go b/pkg/services/object/acl/v2/types.go deleted file mode 100644 index 3cf10eb56..000000000 --- a/pkg/services/object/acl/v2/types.go +++ /dev/null @@ -1,11 +0,0 @@ -package v2 - -import "context" - -// InnerRingFetcher is an interface that must provide -// Inner Ring information. -type InnerRingFetcher interface { - // InnerRingKeys must return list of public keys of - // the actual inner ring. - InnerRingKeys(ctx context.Context) ([][]byte, error) -} diff --git a/pkg/services/object/acl/v2/util.go b/pkg/services/object/acl/v2/util.go deleted file mode 100644 index e02f70771..000000000 --- a/pkg/services/object/acl/v2/util.go +++ /dev/null @@ -1,223 +0,0 @@ -package v2 - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "errors" - "fmt" - - objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" - refsV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/refs" - sessionV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" - cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" - "github.com/nspcc-dev/neo-go/pkg/crypto/keys" -) - -var errMissingContainerID = errors.New("missing container ID") - -func getContainerIDFromRequest(req any) (cid.ID, error) { - var idV2 *refsV2.ContainerID - var id cid.ID - - switch v := req.(type) { - case *objectV2.GetRequest: - idV2 = v.GetBody().GetAddress().GetContainerID() - case *objectV2.PutRequest: - part, ok := v.GetBody().GetObjectPart().(*objectV2.PutObjectPartInit) - if !ok { - return cid.ID{}, errors.New("can't get container ID in chunk") - } - - idV2 = part.GetHeader().GetContainerID() - case *objectV2.HeadRequest: - idV2 = v.GetBody().GetAddress().GetContainerID() - case *objectV2.SearchRequest: - idV2 = v.GetBody().GetContainerID() - case *objectV2.DeleteRequest: - idV2 = v.GetBody().GetAddress().GetContainerID() - case *objectV2.GetRangeRequest: - idV2 = v.GetBody().GetAddress().GetContainerID() - case *objectV2.GetRangeHashRequest: - idV2 = v.GetBody().GetAddress().GetContainerID() - case *objectV2.PutSingleRequest: - idV2 = v.GetBody().GetObject().GetHeader().GetContainerID() - case *objectV2.PatchRequest: - idV2 = v.GetBody().GetAddress().GetContainerID() - default: - return cid.ID{}, errors.New("unknown request type") - } - - if idV2 == nil { - return cid.ID{}, errMissingContainerID - } - - return id, id.ReadFromV2(*idV2) -} - -// originalBearerToken goes down to original request meta header and fetches -// bearer token from there. -func originalBearerToken(header *sessionV2.RequestMetaHeader) (*bearer.Token, error) { - for header.GetOrigin() != nil { - header = header.GetOrigin() - } - - tokV2 := header.GetBearerToken() - if tokV2 == nil { - return nil, nil - } - - var tok bearer.Token - return &tok, tok.ReadFromV2(*tokV2) -} - -// originalSessionToken goes down to original request meta header and fetches -// session token from there. -func originalSessionToken(header *sessionV2.RequestMetaHeader) (*sessionSDK.Object, error) { - for header.GetOrigin() != nil { - header = header.GetOrigin() - } - - tokV2 := header.GetSessionToken() - if tokV2 == nil { - return nil, nil - } - - var tok sessionSDK.Object - - err := tok.ReadFromV2(*tokV2) - if err != nil { - return nil, fmt.Errorf("invalid session token: %w", err) - } - - return &tok, nil -} - -// getObjectIDFromRequestBody decodes oid.ID from the common interface of the -// object reference's holders. Returns an error if object ID is missing in the request. -func getObjectIDFromRequestBody(body interface{ GetAddress() *refsV2.Address }) (*oid.ID, error) { - idV2 := body.GetAddress().GetObjectID() - return getObjectIDFromRefObjectID(idV2) -} - -func getObjectIDFromRefObjectID(idV2 *refsV2.ObjectID) (*oid.ID, error) { - if idV2 == nil { - return nil, errors.New("missing object ID") - } - - var id oid.ID - - err := id.ReadFromV2(*idV2) - if err != nil { - return nil, err - } - - return &id, nil -} - -func ownerFromToken(token *sessionSDK.Object) (*user.ID, *keys.PublicKey, error) { - // 1. First check signature of session token. - if !token.VerifySignature() { - return nil, nil, errInvalidSessionSig - } - - // 2. Then check if session token owner issued the session token - // TODO(@cthulhu-rider): #468 implement and use another approach to avoid conversion - var tokV2 sessionV2.Token - token.WriteToV2(&tokV2) - - tokenIssuerKey, err := unmarshalPublicKey(tokV2.GetSignature().GetKey()) - if err != nil { - return nil, nil, fmt.Errorf("invalid key in session token signature: %w", err) - } - - tokenIssuer := token.Issuer() - - if !isOwnerFromKey(tokenIssuer, tokenIssuerKey) { - // TODO: #767 in this case we can issue all owner keys from frostfs.id and check once again - return nil, nil, errInvalidSessionOwner - } - - return &tokenIssuer, tokenIssuerKey, nil -} - -func originalBodySignature(v *sessionV2.RequestVerificationHeader) *refsV2.Signature { - if v == nil { - return nil - } - - for v.GetOrigin() != nil { - v = v.GetOrigin() - } - - return v.GetBodySignature() -} - -func unmarshalPublicKey(bs []byte) (*keys.PublicKey, error) { - return keys.NewPublicKeyFromBytes(bs, elliptic.P256()) -} - -func isOwnerFromKey(id user.ID, key *keys.PublicKey) bool { - if key == nil { - return false - } - - var id2 user.ID - user.IDFromKey(&id2, (ecdsa.PublicKey)(*key)) - - return id2.Equals(id) -} - -// assertVerb checks that token verb corresponds to op. -func assertVerb(tok sessionSDK.Object, op acl.Op) bool { - switch op { - case acl.OpObjectPut: - return tok.AssertVerb(sessionSDK.VerbObjectPut, sessionSDK.VerbObjectDelete, sessionSDK.VerbObjectPatch) - case acl.OpObjectDelete: - return tok.AssertVerb(sessionSDK.VerbObjectDelete) - case acl.OpObjectGet: - return tok.AssertVerb(sessionSDK.VerbObjectGet) - case acl.OpObjectHead: - return tok.AssertVerb( - sessionSDK.VerbObjectHead, - sessionSDK.VerbObjectGet, - sessionSDK.VerbObjectDelete, - sessionSDK.VerbObjectRange, - sessionSDK.VerbObjectRangeHash, - sessionSDK.VerbObjectPatch, - ) - case acl.OpObjectSearch: - return tok.AssertVerb(sessionSDK.VerbObjectSearch, sessionSDK.VerbObjectDelete) - case acl.OpObjectRange: - return tok.AssertVerb(sessionSDK.VerbObjectRange, sessionSDK.VerbObjectRangeHash, sessionSDK.VerbObjectPatch) - case acl.OpObjectHash: - return tok.AssertVerb(sessionSDK.VerbObjectRangeHash) - } - - return false -} - -// assertSessionRelation checks if given token describing the FrostFS session -// relates to the given container and optional object. Missing object -// means that the context isn't bound to any FrostFS object in the container. -// Returns no error iff relation is correct. Criteria: -// -// session is bound to the given container -// object is not specified or session is bound to this object -// -// Session MUST be bound to the particular container, otherwise behavior is undefined. -func assertSessionRelation(tok sessionSDK.Object, cnr cid.ID, obj *oid.ID) error { - if !tok.AssertContainer(cnr) { - return errors.New("requested container is not related to the session") - } - - if obj != nil && !tok.AssertObject(*obj) { - return errors.New("requested object is not related to the session") - } - - return nil -} diff --git a/pkg/services/object/acl/v2/util_test.go b/pkg/services/object/acl/v2/util_test.go deleted file mode 100644 index 40fce8877..000000000 --- a/pkg/services/object/acl/v2/util_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package v2 - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "slices" - "testing" - - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/acl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" - bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test" - aclsdk "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" - oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" - sessionSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" - sessiontest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session/test" - "github.com/stretchr/testify/require" -) - -func TestOriginalTokens(t *testing.T) { - sToken := sessiontest.ObjectSigned() - bToken := bearertest.Token() - - pk, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - require.NoError(t, bToken.Sign(*pk)) - - var bTokenV2 acl.BearerToken - bToken.WriteToV2(&bTokenV2) - // This line is needed because SDK uses some custom format for - // reserved filters, so `cid.ID` is not converted to string immediately. - require.NoError(t, bToken.ReadFromV2(bTokenV2)) - - var sTokenV2 session.Token - sToken.WriteToV2(&sTokenV2) - - for i := range 10 { - metaHeaders := testGenerateMetaHeader(uint32(i), &bTokenV2, &sTokenV2) - res, err := originalSessionToken(metaHeaders) - require.NoError(t, err) - require.Equal(t, sToken, res, i) - - bTok, err := originalBearerToken(metaHeaders) - require.NoError(t, err) - require.Equal(t, &bToken, bTok, i) - } -} - -func testGenerateMetaHeader(depth uint32, b *acl.BearerToken, s *session.Token) *session.RequestMetaHeader { - metaHeader := new(session.RequestMetaHeader) - metaHeader.SetBearerToken(b) - metaHeader.SetSessionToken(s) - - for range depth { - link := metaHeader - metaHeader = new(session.RequestMetaHeader) - metaHeader.SetOrigin(link) - } - - return metaHeader -} - -func TestIsVerbCompatible(t *testing.T) { - // Source: https://nspcc.ru/upload/frostfs-spec-latest.pdf#page=28 - table := map[aclsdk.Op][]sessionSDK.ObjectVerb{ - aclsdk.OpObjectPut: {sessionSDK.VerbObjectPut, sessionSDK.VerbObjectDelete}, - aclsdk.OpObjectDelete: {sessionSDK.VerbObjectDelete}, - aclsdk.OpObjectGet: {sessionSDK.VerbObjectGet}, - aclsdk.OpObjectHead: { - sessionSDK.VerbObjectHead, - sessionSDK.VerbObjectGet, - sessionSDK.VerbObjectDelete, - sessionSDK.VerbObjectRange, - sessionSDK.VerbObjectRangeHash, - }, - aclsdk.OpObjectRange: {sessionSDK.VerbObjectRange, sessionSDK.VerbObjectRangeHash}, - aclsdk.OpObjectHash: {sessionSDK.VerbObjectRangeHash}, - aclsdk.OpObjectSearch: {sessionSDK.VerbObjectSearch, sessionSDK.VerbObjectDelete}, - } - - verbs := []sessionSDK.ObjectVerb{ - sessionSDK.VerbObjectPut, - sessionSDK.VerbObjectDelete, - sessionSDK.VerbObjectHead, - sessionSDK.VerbObjectRange, - sessionSDK.VerbObjectRangeHash, - sessionSDK.VerbObjectGet, - sessionSDK.VerbObjectSearch, - } - - var tok sessionSDK.Object - - for op, list := range table { - for _, verb := range verbs { - contains := slices.Contains(list, verb) - - tok.ForVerb(verb) - - require.Equal(t, contains, assertVerb(tok, op), - "%v in token, %s executing", verb, op) - } - } -} - -func TestAssertSessionRelation(t *testing.T) { - var tok sessionSDK.Object - cnr := cidtest.ID() - cnrOther := cidtest.ID() - obj := oidtest.ID() - objOther := oidtest.ID() - - // make sure ids differ, otherwise test won't work correctly - require.False(t, cnrOther.Equals(cnr)) - require.False(t, objOther.Equals(obj)) - - // bind session to the container (required) - tok.BindContainer(cnr) - - // test container-global session - require.NoError(t, assertSessionRelation(tok, cnr, nil)) - require.NoError(t, assertSessionRelation(tok, cnr, &obj)) - require.Error(t, assertSessionRelation(tok, cnrOther, nil)) - require.Error(t, assertSessionRelation(tok, cnrOther, &obj)) - - // limit the session to the particular object - tok.LimitByObjects(obj) - - // test fixed object session (here obj arg must be non-nil everywhere) - require.NoError(t, assertSessionRelation(tok, cnr, &obj)) - require.Error(t, assertSessionRelation(tok, cnr, &objOther)) -} diff --git a/pkg/services/object/request_context.go b/pkg/services/object/request_context.go deleted file mode 100644 index eb4041f80..000000000 --- a/pkg/services/object/request_context.go +++ /dev/null @@ -1,24 +0,0 @@ -package object - -import ( - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" -) - -type RequestContextKeyT struct{} - -var RequestContextKey = RequestContextKeyT{} - -// RequestContext is a context passed between middleware handlers. -type RequestContext struct { - Namespace string - - SenderKey []byte - - ContainerOwner user.ID - - Role acl.Role - - BearerToken *bearer.Token -} From 3bb1fb744a6b09e796b6e33c80910e90ab69f67f Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 15:56:11 +0300 Subject: [PATCH 1819/1943] [#1689] morph/client: Reuse auto-generated wrappers for NNS Make code simpler, remove unused methods. Change-Id: I18807f2c14b5a96e533e5e3fc153e23c742c66c1 Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 4 ++ pkg/morph/client/nns.go | 96 ++++++-------------------------------- 2 files changed, 19 insertions(+), 81 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index a7c3c6d8d..1c12130b7 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + nnsClient "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" @@ -62,6 +63,8 @@ type Client struct { rolemgmt *rolemgmt.Contract // neo-go Designation contract wrapper nnsHash util.Uint160 // NNS contract hash + nnsReader *nnsClient.ContractReader // NNS contract wrapper + acc *wallet.Account // neo account accAddr util.Uint160 // account's address @@ -576,6 +579,7 @@ func (c *Client) setActor(act *actor.Actor) { c.rpcActor = act c.gasToken = nep17.New(act, gas.Hash) c.rolemgmt = rolemgmt.New(act) + c.nnsReader = nnsClient.NewReader(act, c.nnsHash) } func (c *Client) GetActor() *actor.Actor { diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index b03967fdd..bc00eb889 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -8,14 +8,12 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + nnsClient "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/nns" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" - "github.com/nspcc-dev/neo-go/pkg/smartcontract" "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/stackitem" - "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" ) const ( @@ -37,12 +35,8 @@ const ( NNSPolicyContractName = "policy.frostfs" ) -var ( - // ErrNNSRecordNotFound means that there is no such record in NNS contract. - ErrNNSRecordNotFound = errors.New("record has not been found in NNS contract") - - errEmptyResultStack = errors.New("returned result stack is empty") -) +// ErrNNSRecordNotFound means that there is no such record in NNS contract. +var ErrNNSRecordNotFound = errors.New("record has not been found in NNS contract") // NNSAlphabetContractName returns contract name of the alphabet contract in NNS // based on alphabet index. @@ -61,67 +55,36 @@ func (c *Client) NNSContractAddress(name string) (sh util.Uint160, err error) { return util.Uint160{}, ErrConnectionLost } - nnsHash := c.NNSHash() - sh, err = nnsResolve(c.client, nnsHash, name) + sh, err = nnsResolve(c.nnsReader, name) if err != nil { return sh, fmt.Errorf("NNS.resolve: %w", err) } return sh, nil } -// NNSHash returns NNS contract hash. -func (c *Client) NNSHash() util.Uint160 { - return c.nnsHash -} - -func nnsResolveItem(c *rpcclient.WSClient, nnsHash util.Uint160, domain string) (stackitem.Item, error) { - found, err := exists(c, nnsHash, domain) +func nnsResolveItem(r *nnsClient.ContractReader, domain string) ([]stackitem.Item, error) { + available, err := r.IsAvailable(domain) if err != nil { return nil, fmt.Errorf("check presence in NNS contract for %s: %w", domain, err) } - if !found { + if available { return nil, ErrNNSRecordNotFound } - result, err := c.InvokeFunction(nnsHash, "resolve", []smartcontract.Parameter{ - { - Type: smartcontract.StringType, - Value: domain, - }, - { - Type: smartcontract.IntegerType, - Value: big.NewInt(int64(nns.TXT)), - }, - }, nil) - if err != nil { - return nil, err - } - if result.State != vmstate.Halt.String() { - return nil, fmt.Errorf("invocation failed: %s", result.FaultException) - } - if len(result.Stack) == 0 { - return nil, errEmptyResultStack - } - return result.Stack[0], nil + return r.Resolve(domain, big.NewInt(int64(nns.TXT))) } -func nnsResolve(c *rpcclient.WSClient, nnsHash util.Uint160, domain string) (util.Uint160, error) { - res, err := nnsResolveItem(c, nnsHash, domain) +func nnsResolve(r *nnsClient.ContractReader, domain string) (util.Uint160, error) { + arr, err := nnsResolveItem(r, domain) if err != nil { return util.Uint160{}, err } - // Parse the result of resolving NNS record. - // It works with multiple formats (corresponding to multiple NNS versions). - // If array of hashes is provided, it returns only the first one. - if arr, ok := res.Value().([]stackitem.Item); ok { - if len(arr) == 0 { - return util.Uint160{}, errors.New("NNS record is missing") - } - res = arr[0] + if len(arr) == 0 { + return util.Uint160{}, errors.New("NNS record is missing") } - bs, err := res.TryBytes() + bs, err := arr[0].TryBytes() if err != nil { return util.Uint160{}, fmt.Errorf("malformed response: %w", err) } @@ -141,33 +104,6 @@ func nnsResolve(c *rpcclient.WSClient, nnsHash util.Uint160, domain string) (uti return util.Uint160{}, errors.New("no valid hashes are found") } -func exists(c *rpcclient.WSClient, nnsHash util.Uint160, domain string) (bool, error) { - result, err := c.InvokeFunction(nnsHash, "isAvailable", []smartcontract.Parameter{ - { - Type: smartcontract.StringType, - Value: domain, - }, - }, nil) - if err != nil { - return false, err - } - - if len(result.Stack) == 0 { - return false, errEmptyResultStack - } - - res := result.Stack[0] - - available, err := res.TryBool() - if err != nil { - return false, fmt.Errorf("malformed response: %w", err) - } - - // not available means that it is taken - // and, therefore, exists - return !available, nil -} - // SetGroupSignerScope makes the default signer scope include all FrostFS contracts. // Should be called for side-chain client only. func (c *Client) SetGroupSignerScope() error { @@ -211,14 +147,12 @@ func (c *Client) contractGroupKey() (*keys.PublicKey, error) { return gKey, nil } - nnsHash := c.NNSHash() - item, err := nnsResolveItem(c.client, nnsHash, NNSGroupKeyName) + arr, err := nnsResolveItem(c.nnsReader, NNSGroupKeyName) if err != nil { return nil, err } - arr, ok := item.Value().([]stackitem.Item) - if !ok || len(arr) == 0 { + if len(arr) == 0 { return nil, errors.New("NNS record is missing") } From 0a9d139e20fed1d6386f5913f2da9dc749d85d4e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 16:03:10 +0300 Subject: [PATCH 1820/1943] [#1689] morph/client: Remove notary hash field from `notaryInfo` Notary contract hash is constant. Change-Id: I7935580acbced5c9d567875ea75daa57cc259a3c Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/notary.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index dbd58a53a..4e20a3639 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -38,8 +38,7 @@ type ( alphabetSource AlphabetKeys // source of alphabet node keys to prepare witness - notary util.Uint160 - proxy util.Uint160 + proxy util.Uint160 } notaryCfg struct { @@ -102,7 +101,6 @@ func (c *Client) EnableNotarySupport(opts ...NotaryOption) error { txValidTime: cfg.txValidTime, roundTime: cfg.roundTime, alphabetSource: cfg.alphabetSource, - notary: notary.Hash, } c.notary = notaryCfg @@ -188,7 +186,7 @@ func (c *Client) DepositEndlessNotary(ctx context.Context, amount fixedn.Fixed8) func (c *Client) depositNotary(ctx context.Context, amount fixedn.Fixed8, till int64) (util.Uint256, uint32, error) { txHash, vub, err := c.gasToken.Transfer( c.accAddr, - c.notary.notary, + notary.Hash, big.NewInt(int64(amount)), []any{c.acc.PrivateKey().GetScriptHash(), till}) if err != nil { From 0c664fa804bbecab361fdec2287fa42b051c82d4 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 24 Mar 2025 11:52:39 +0300 Subject: [PATCH 1821/1943] [#1689] qos: Fix metric description Change-Id: I460fdd3713e765d57ef3ff2945b9b3776f46c164 Signed-off-by: Dmitrii Stepanov --- internal/metrics/qos.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/metrics/qos.go b/internal/metrics/qos.go index 17fb67a27..be6878142 100644 --- a/internal/metrics/qos.go +++ b/internal/metrics/qos.go @@ -15,7 +15,7 @@ func newQoSMetrics() *QoSMetrics { Namespace: namespace, Subsystem: qosSubsystem, Name: "operations_total", - Help: "Count of pending, in progree, completed and failed due of resource exhausted error operations for each shard", + Help: "Count of pending, in progress, completed and failed due of resource exhausted error operations for each shard", }, []string{shardIDLabel, operationLabel, ioTagLabel, typeLabel}), } } From 163e2e9f83a79def2d7bfda6b2b6609cfa02c1b2 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Mon, 24 Mar 2025 17:04:24 +0300 Subject: [PATCH 1822/1943] ci: Cache pre-commit installations on Jenkins Agent This change introduces a custom helper from our shared library [0] that defines ad-hoc container environment to execute CI steps in. Pre-commit installation will now be cached on Jenkins Agent: builds will tolerate network hiccups better and we will also save some run time (although on non-critical path of a parallel process). [0]: https://git.frostfs.info/TrueCloudLab/jenkins/pulls/8 Change-Id: I93b01f169c457aa35f4d8bc5b90f31b31e2bd8b2 Signed-off-by: Vitaliy Potyarkin --- .ci/Jenkinsfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index e21ce61c5..166fb9286 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -68,12 +68,14 @@ async { } task('pre-commit') { - sh ''' - apt update - apt install -y --no-install-recommends pre-commit - ''' // TODO: Make an OCI image for pre-commit + golang? Unpack golang tarball with a library function? - withEnv(['SKIP=make-lint,go-staticcheck-repo-mod,go-unit-tests,gofumpt']) { - sh 'pre-commit run --color=always --hook-stage=manual --all-files' + dockerfile(""" + FROM ${golangDefault} + RUN apt update && \ + apt install -y --no-install-recommends pre-commit + """) { + withEnv(['SKIP=make-lint,go-staticcheck-repo-mod,go-unit-tests,gofumpt']) { + sh 'pre-commit run --color=always --hook-stage=manual --all-files' + } } } } From 5470b205fd461d95c968c55ed122334c7e7f32cd Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 25 Mar 2025 10:15:51 +0300 Subject: [PATCH 1823/1943] [#1619] gc: Fix metric `frostfs_node.garbage_collector.marking_duration_seconds` Change-Id: I957f930d1babf179d0fb6de624a90f4fe9977862 Signed-off-by: Anton Nikiforov --- pkg/local_object_storage/shard/gc.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 32a377cd5..84fb6039e 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -527,7 +527,8 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { return } - release, err := s.opsLimiter.ReadRequest(ctx) + var release qos.ReleaseFunc + release, err = s.opsLimiter.ReadRequest(ctx) if err != nil { log.Error(ctx, logs.ShardIteratorOverGraveyardFailed, zap.Error(err)) s.m.RUnlock() From 9358938222bc44bd4f4638a78b74cd331cdba808 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 10 Mar 2025 11:42:55 +0300 Subject: [PATCH 1824/1943] [#1633] go.mod: Bump frostfs-sdk-go Change-Id: I50c1a0d5b88e307402a5b1b2883bb9b9a357a2c7 Signed-off-by: Anton Nikiforov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 753bfbd29..887f8bb2b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250320142439-32079ad7c275 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250307150202-749b4e9ab592 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b diff --git a/go.sum b/go.sum index 7f9417954..982fddf23 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250320142439-32079ad7c275 h1:WqWxCnCl2ekfjWja/CpGeF2rf4h0x199xhdnsm/j+E8= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250320142439-32079ad7c275/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9 h1:svCl6NDAPZ/KuQPjdVKo74RkCIANesxUPM45zQZDhSw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250306092416-69b0711d12d9/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250307150202-749b4e9ab592 h1:n7Pl8V7O1yS07J/fqdbzZjVe/mQW42a7eS0QHfgrzJw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250307150202-749b4e9ab592/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 3bbee1b554a1407e1b8c30ef0244cd4599c4299f Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 24 Mar 2025 12:32:41 +0300 Subject: [PATCH 1825/1943] [#1619] logger: Allow to set options for `zap.Logger` via `logger.Prm` Change-Id: I8eed951c25d1ecf18b0aea62c6825be65a450085 Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config.go | 16 ++++++++++------ pkg/util/logger/logger.go | 20 +++++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index c3c687763..431316258 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -108,6 +108,7 @@ type applicationConfiguration struct { level string destination string timestamp bool + options []zap.Option } ObjectCfg struct { @@ -232,6 +233,14 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { a.LoggerCfg.level = loggerconfig.Level(c) a.LoggerCfg.destination = loggerconfig.Destination(c) a.LoggerCfg.timestamp = loggerconfig.Timestamp(c) + var opts []zap.Option + if loggerconfig.ToLokiConfig(c).Enabled { + opts = []zap.Option{zap.WrapCore(func(core zapcore.Core) zapcore.Core { + lokiCore := lokicore.New(core, loggerconfig.ToLokiConfig(c)) + return lokiCore + })} + } + a.LoggerCfg.options = opts // Object @@ -718,12 +727,6 @@ func initCfg(appCfg *config.Config) *cfg { logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook() log, err := logger.NewLogger(logPrm) fatalOnErr(err) - if loggerconfig.ToLokiConfig(appCfg).Enabled { - log.WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { - lokiCore := lokicore.New(core, loggerconfig.ToLokiConfig(appCfg)) - return lokiCore - })) - } c.internals = initInternals(appCfg, log) @@ -1090,6 +1093,7 @@ func (c *cfg) loggerPrm() (logger.Prm, error) { return logger.Prm{}, errors.New("incorrect log destination format: " + c.LoggerCfg.destination) } prm.PrependTimestamp = c.LoggerCfg.timestamp + prm.Options = c.LoggerCfg.options return prm, nil } diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index 952a6f2dc..10c7e8dc9 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -36,6 +36,9 @@ type Prm struct { // PrependTimestamp specifies whether to prepend a timestamp in the log PrependTimestamp bool + + // Options for zap.Logger + Options []zap.Option } const ( @@ -103,10 +106,12 @@ func newConsoleLogger(prm Prm) (*Logger, error) { c.EncoderConfig.TimeKey = "" } - lZap, err := c.Build( + opts := []zap.Option{ zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), zap.AddCallerSkip(1), - ) + } + opts = append(opts, prm.Options...) + lZap, err := c.Build(opts...) if err != nil { return nil, err } @@ -150,7 +155,12 @@ func newJournaldLogger(prm Prm) (*Logger, error) { c.Sampling.Thereafter, samplerOpts..., ) - lZap := zap.New(samplingCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), zap.AddCallerSkip(1)) + opts := []zap.Option{ + zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), + zap.AddCallerSkip(1), + } + opts = append(opts, prm.Options...) + lZap := zap.New(samplingCore, opts...) l := &Logger{z: lZap, lvl: lvl} @@ -161,10 +171,6 @@ func (l *Logger) Reload(prm Prm) { l.lvl.SetLevel(prm.level) } -func (l *Logger) WithOptions(options ...zap.Option) { - l.z = l.z.WithOptions(options...) -} - func (l *Logger) With(fields ...zap.Field) *Logger { return &Logger{z: l.z.With(fields...)} } From 632bd8e38dd34e2438dd39c53111605dd94ef8b1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 25 Mar 2025 12:15:46 +0300 Subject: [PATCH 1826/1943] [#1696] qos: Fix internal tag adjust If request has no tag, but request's public key is netmap node's key or one of allowed internal tag keys from config, then request must use internal IO tag. Change-Id: Iff93b626941a81b088d8999b3f2947f9501dcdf8 Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/qos.go | 39 ++++-- cmd/frostfs-node/qos_test.go | 226 ++++++++++++++++++++++++++++++ internal/logs/logs.go | 2 +- pkg/core/object/fmt_test.go | 42 ++---- pkg/util/testing/netmap_source.go | 36 +++++ 5 files changed, 298 insertions(+), 47 deletions(-) create mode 100644 cmd/frostfs-node/qos_test.go create mode 100644 pkg/util/testing/netmap_source.go diff --git a/cmd/frostfs-node/qos.go b/cmd/frostfs-node/qos.go index 9663fc6ae..6394b668b 100644 --- a/cmd/frostfs-node/qos.go +++ b/cmd/frostfs-node/qos.go @@ -43,6 +43,9 @@ func initQoSService(c *cfg) { func (s *cfgQoSService) AdjustIncomingTag(ctx context.Context, requestSignPublicKey []byte) context.Context { rawTag, defined := qosTagging.IOTagFromContext(ctx) if !defined { + if s.isInternalIOTagPublicKey(ctx, requestSignPublicKey) { + return qosTagging.ContextWithIOTag(ctx, qos.IOTagInternal.String()) + } return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) } ioTag, err := qos.FromRawString(rawTag) @@ -73,20 +76,8 @@ func (s *cfgQoSService) AdjustIncomingTag(ctx context.Context, requestSignPublic s.logger.Debug(ctx, logs.FailedToValidateIncomingIOTag) return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) case qos.IOTagInternal: - for _, pk := range s.allowedInternalPubs { - if bytes.Equal(pk, requestSignPublicKey) { - return ctx - } - } - nm, err := s.netmapSource.GetNetMap(ctx, 0) - if err != nil { - s.logger.Debug(ctx, logs.FailedToGetNetmapToAdjustIOTag, zap.Error(err)) - return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) - } - for _, node := range nm.Nodes() { - if bytes.Equal(node.PublicKey(), requestSignPublicKey) { - return ctx - } + if s.isInternalIOTagPublicKey(ctx, requestSignPublicKey) { + return ctx } s.logger.Debug(ctx, logs.FailedToValidateIncomingIOTag) return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) @@ -95,3 +86,23 @@ func (s *cfgQoSService) AdjustIncomingTag(ctx context.Context, requestSignPublic return qosTagging.ContextWithIOTag(ctx, qos.IOTagClient.String()) } } + +func (s *cfgQoSService) isInternalIOTagPublicKey(ctx context.Context, publicKey []byte) bool { + for _, pk := range s.allowedInternalPubs { + if bytes.Equal(pk, publicKey) { + return true + } + } + nm, err := s.netmapSource.GetNetMap(ctx, 0) + if err != nil { + s.logger.Debug(ctx, logs.FailedToGetNetmapToAdjustIOTag, zap.Error(err)) + return false + } + for _, node := range nm.Nodes() { + if bytes.Equal(node.PublicKey(), publicKey) { + return true + } + } + + return false +} diff --git a/cmd/frostfs-node/qos_test.go b/cmd/frostfs-node/qos_test.go new file mode 100644 index 000000000..971f9eebf --- /dev/null +++ b/cmd/frostfs-node/qos_test.go @@ -0,0 +1,226 @@ +package main + +import ( + "context" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + utilTesting "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/testing" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestQoSService_Client(t *testing.T) { + t.Parallel() + s, pk := testQoSServicePrepare(t) + t.Run("IO tag client defined", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagClient.String()) + ctx = s.AdjustIncomingTag(ctx, pk.Request) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("no IO tag defined, signed with unknown key", func(t *testing.T) { + ctx := s.AdjustIncomingTag(context.Background(), pk.Request) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("no IO tag defined, signed with allowed critical key", func(t *testing.T) { + ctx := s.AdjustIncomingTag(context.Background(), pk.Critical) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("unknown IO tag, signed with unknown key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), "some IO tag we don't know") + ctx = s.AdjustIncomingTag(ctx, pk.Request) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("unknown IO tag, signed with netmap key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), "some IO tag we don't know") + ctx = s.AdjustIncomingTag(ctx, pk.NetmapNode) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("unknown IO tag, signed with allowed internal key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), "some IO tag we don't know") + ctx = s.AdjustIncomingTag(ctx, pk.Internal) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("unknown IO tag, signed with allowed critical key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), "some IO tag we don't know") + ctx = s.AdjustIncomingTag(ctx, pk.Critical) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("IO tag internal defined, signed with unknown key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagInternal.String()) + ctx = s.AdjustIncomingTag(ctx, pk.Request) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("IO tag internal defined, signed with allowed critical key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagInternal.String()) + ctx = s.AdjustIncomingTag(ctx, pk.Critical) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("IO tag critical defined, signed with unknown key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagCritical.String()) + ctx = s.AdjustIncomingTag(ctx, pk.Request) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("IO tag critical defined, signed with allowed internal key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagCritical.String()) + ctx = s.AdjustIncomingTag(ctx, pk.Internal) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) +} + +func TestQoSService_Internal(t *testing.T) { + t.Parallel() + s, pk := testQoSServicePrepare(t) + t.Run("IO tag internal defined, signed with netmap key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagInternal.String()) + ctx = s.AdjustIncomingTag(ctx, pk.NetmapNode) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagInternal.String(), tag) + }) + t.Run("IO tag internal defined, signed with allowed internal key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagInternal.String()) + ctx = s.AdjustIncomingTag(ctx, pk.Internal) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagInternal.String(), tag) + }) + t.Run("no IO tag defined, signed with netmap key", func(t *testing.T) { + ctx := s.AdjustIncomingTag(context.Background(), pk.NetmapNode) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagInternal.String(), tag) + }) + t.Run("no IO tag defined, signed with allowed internal key", func(t *testing.T) { + ctx := s.AdjustIncomingTag(context.Background(), pk.Internal) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagInternal.String(), tag) + }) +} + +func TestQoSService_Critical(t *testing.T) { + t.Parallel() + s, pk := testQoSServicePrepare(t) + t.Run("IO tag critical defined, signed with netmap key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagCritical.String()) + ctx = s.AdjustIncomingTag(ctx, pk.NetmapNode) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagCritical.String(), tag) + }) + t.Run("IO tag critical defined, signed with allowed critical key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagCritical.String()) + ctx = s.AdjustIncomingTag(ctx, pk.Critical) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagCritical.String(), tag) + }) +} + +func TestQoSService_NetmapGetError(t *testing.T) { + t.Parallel() + s, pk := testQoSServicePrepare(t) + s.netmapSource = &utilTesting.TestNetmapSource{} + t.Run("IO tag internal defined, signed with netmap key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagInternal.String()) + ctx = s.AdjustIncomingTag(ctx, pk.NetmapNode) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("IO tag critical defined, signed with netmap key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagCritical.String()) + ctx = s.AdjustIncomingTag(ctx, pk.NetmapNode) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("no IO tag defined, signed with netmap key", func(t *testing.T) { + ctx := s.AdjustIncomingTag(context.Background(), pk.NetmapNode) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) + t.Run("unknown IO tag, signed with netmap key", func(t *testing.T) { + ctx := tagging.ContextWithIOTag(context.Background(), "some IO tag we don't know") + ctx = s.AdjustIncomingTag(ctx, pk.NetmapNode) + tag, ok := tagging.IOTagFromContext(ctx) + require.True(t, ok) + require.Equal(t, qos.IOTagClient.String(), tag) + }) +} + +func testQoSServicePrepare(t *testing.T) (*cfgQoSService, *testQoSServicePublicKeys) { + nmSigner, err := keys.NewPrivateKey() + require.NoError(t, err) + + reqSigner, err := keys.NewPrivateKey() + require.NoError(t, err) + + allowedCritSigner, err := keys.NewPrivateKey() + require.NoError(t, err) + + allowedIntSigner, err := keys.NewPrivateKey() + require.NoError(t, err) + + var node netmap.NodeInfo + node.SetPublicKey(nmSigner.PublicKey().Bytes()) + nm := &netmap.NetMap{} + nm.SetEpoch(100) + nm.SetNodes([]netmap.NodeInfo{node}) + + return &cfgQoSService{ + logger: test.NewLogger(t), + netmapSource: &utilTesting.TestNetmapSource{ + Netmaps: map[uint64]*netmap.NetMap{ + 100: nm, + }, + CurrentEpoch: 100, + }, + allowedCriticalPubs: [][]byte{ + allowedCritSigner.PublicKey().Bytes(), + }, + allowedInternalPubs: [][]byte{ + allowedIntSigner.PublicKey().Bytes(), + }, + }, + &testQoSServicePublicKeys{ + NetmapNode: nmSigner.PublicKey().Bytes(), + Request: reqSigner.PublicKey().Bytes(), + Internal: allowedIntSigner.PublicKey().Bytes(), + Critical: allowedCritSigner.PublicKey().Bytes(), + } +} + +type testQoSServicePublicKeys struct { + NetmapNode []byte + Request []byte + Internal []byte + Critical []byte +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 3503c922e..5b42b25ba 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -512,7 +512,7 @@ const ( FailedToUpdateMultinetConfiguration = "failed to update multinet configuration" FailedToParseIncomingIOTag = "failed to parse incoming IO tag" NotSupportedIncomingIOTagReplacedWithClient = "incoming IO tag is not supported, replaced with `client`" - FailedToGetNetmapToAdjustIOTag = "failed to get netmap to adjust IO tag, replaced with `client`" + FailedToGetNetmapToAdjustIOTag = "failed to get netmap to adjust IO tag" FailedToValidateIncomingIOTag = "failed to validate incoming IO tag, replaced with `client`" WriteCacheFailedToAcquireRPSQuota = "writecache failed to acquire RPS quota to flush object" ) diff --git a/pkg/core/object/fmt_test.go b/pkg/core/object/fmt_test.go index 239a9f389..dc336eb34 100644 --- a/pkg/core/object/fmt_test.go +++ b/pkg/core/object/fmt_test.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + utilTesting "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/testing" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -410,11 +411,11 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { }, ), WithNetmapSource( - &testNetmapSource{ - netmaps: map[uint64]*netmap.NetMap{ + &utilTesting.TestNetmapSource{ + Netmaps: map[uint64]*netmap.NetMap{ curEpoch: currentEpochNM, }, - currentEpoch: curEpoch, + CurrentEpoch: curEpoch, }, ), WithLogger(logger.NewLoggerWrapper(zaptest.NewLogger(t))), @@ -483,12 +484,12 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { }, ), WithNetmapSource( - &testNetmapSource{ - netmaps: map[uint64]*netmap.NetMap{ + &utilTesting.TestNetmapSource{ + Netmaps: map[uint64]*netmap.NetMap{ curEpoch: currentEpochNM, curEpoch - 1: previousEpochNM, }, - currentEpoch: curEpoch, + CurrentEpoch: curEpoch, }, ), WithLogger(logger.NewLoggerWrapper(zaptest.NewLogger(t))), @@ -559,12 +560,12 @@ func TestFormatValidator_ValidateTokenIssuer(t *testing.T) { }, ), WithNetmapSource( - &testNetmapSource{ - netmaps: map[uint64]*netmap.NetMap{ + &utilTesting.TestNetmapSource{ + Netmaps: map[uint64]*netmap.NetMap{ curEpoch: currentEpochNM, curEpoch - 1: previousEpochNM, }, - currentEpoch: curEpoch, + CurrentEpoch: curEpoch, }, ), WithLogger(logger.NewLoggerWrapper(zaptest.NewLogger(t))), @@ -596,26 +597,3 @@ func (s *testContainerSource) Get(ctx context.Context, cnrID cid.ID) (*container func (s *testContainerSource) DeletionInfo(context.Context, cid.ID) (*container.DelInfo, error) { return nil, nil } - -type testNetmapSource struct { - netmaps map[uint64]*netmap.NetMap - currentEpoch uint64 -} - -func (s *testNetmapSource) GetNetMap(ctx context.Context, diff uint64) (*netmap.NetMap, error) { - if diff >= s.currentEpoch { - return nil, fmt.Errorf("invalid diff") - } - return s.GetNetMapByEpoch(ctx, s.currentEpoch-diff) -} - -func (s *testNetmapSource) GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmap.NetMap, error) { - if nm, found := s.netmaps[epoch]; found { - return nm, nil - } - return nil, fmt.Errorf("netmap not found") -} - -func (s *testNetmapSource) Epoch(ctx context.Context) (uint64, error) { - return s.currentEpoch, nil -} diff --git a/pkg/util/testing/netmap_source.go b/pkg/util/testing/netmap_source.go new file mode 100644 index 000000000..7373e538f --- /dev/null +++ b/pkg/util/testing/netmap_source.go @@ -0,0 +1,36 @@ +package testing + +import ( + "context" + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" +) + +var ( + errInvalidDiff = errors.New("invalid diff") + errNetmapNotFound = errors.New("netmap not found") +) + +type TestNetmapSource struct { + Netmaps map[uint64]*netmap.NetMap + CurrentEpoch uint64 +} + +func (s *TestNetmapSource) GetNetMap(ctx context.Context, diff uint64) (*netmap.NetMap, error) { + if diff >= s.CurrentEpoch { + return nil, errInvalidDiff + } + return s.GetNetMapByEpoch(ctx, s.CurrentEpoch-diff) +} + +func (s *TestNetmapSource) GetNetMapByEpoch(_ context.Context, epoch uint64) (*netmap.NetMap, error) { + if nm, found := s.Netmaps[epoch]; found { + return nm, nil + } + return nil, errNetmapNotFound +} + +func (s *TestNetmapSource) Epoch(context.Context) (uint64, error) { + return s.CurrentEpoch, nil +} From 0a38571a10df69da4200db8ea6c952f587c10c4d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 08:20:40 +0300 Subject: [PATCH 1827/1943] [#1689] adm: Simplify getCandidateRegisterPrice() After all the refactoring, there is no more need to have custom branch for the local client. Change-Id: I274305b0c390578fb4583759135d3e7ce58873dc Signed-off-by: Evgenii Stratonikov --- .../morph/initialize/initialize_register.go | 31 ++----------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go index 4c6607f9a..841acdc83 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go @@ -1,7 +1,6 @@ package initialize import ( - "errors" "fmt" "math/big" @@ -11,7 +10,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/io" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/neo" @@ -30,7 +28,9 @@ const ( ) func registerCandidateRange(c *helper.InitializeContext, start, end int) error { - regPrice, err := getCandidateRegisterPrice(c) + inv := invoker.New(c.Client, nil) + reader := neo.NewReader(inv) + regPrice, err := reader.GetRegisterPrice() if err != nil { return fmt.Errorf("can't fetch registration price: %w", err) } @@ -144,28 +144,3 @@ func transferNEOFinished(c *helper.InitializeContext, neoHash util.Uint160) (boo bal, err := r.BalanceOf(c.CommitteeAcc.Contract.ScriptHash()) return bal.Cmp(big.NewInt(native.NEOTotalSupply)) == -1, err } - -var errGetPriceInvalid = errors.New("`getRegisterPrice`: invalid response") - -func getCandidateRegisterPrice(c *helper.InitializeContext) (int64, error) { - switch c.Client.(type) { - case *rpcclient.Client: - inv := invoker.New(c.Client, nil) - reader := neo.NewReader(inv) - return reader.GetRegisterPrice() - default: - neoHash := neo.Hash - res, err := helper.InvokeFunction(c.Client, neoHash, "getRegisterPrice", nil, nil) - if err != nil { - return 0, err - } - if len(res.Stack) == 0 { - return 0, errGetPriceInvalid - } - bi, err := res.Stack[0].TryInteger() - if err != nil || !bi.IsInt64() { - return 0, errGetPriceInvalid - } - return bi.Int64(), nil - } -} From c2c05e222863945fdf1fb954135043cc98c5acef Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 08:21:50 +0300 Subject: [PATCH 1828/1943] [#1689] adm: Reuse ReadOnlyInvoker in registerCandidateRange() Change-Id: I544d10340825494b45a62700fa247404c18f746a Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/initialize/initialize_register.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go index 841acdc83..3efa40c50 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go @@ -11,7 +11,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" "github.com/nspcc-dev/neo-go/pkg/rpcclient/neo" "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" @@ -28,8 +27,7 @@ const ( ) func registerCandidateRange(c *helper.InitializeContext, start, end int) error { - inv := invoker.New(c.Client, nil) - reader := neo.NewReader(inv) + reader := neo.NewReader(c.ReadOnlyInvoker) regPrice, err := reader.GetRegisterPrice() if err != nil { return fmt.Errorf("can't fetch registration price: %w", err) From bce2f7bef0c31f251daeaec377b6900ec9c01c9d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 08:24:33 +0300 Subject: [PATCH 1829/1943] [#1689] adm: Reuse `neo.NewReader` helper in transferNEOFinished() Change-Id: I27980ed87436958cb4d27278e30e05da021d1506 Signed-off-by: Evgenii Stratonikov --- .../modules/morph/initialize/initialize_register.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go index 3efa40c50..46e6621d2 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go @@ -12,7 +12,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/neo" - "github.com/nspcc-dev/neo-go/pkg/rpcclient/nep17" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/util" @@ -114,7 +113,7 @@ func registerCandidates(c *helper.InitializeContext) error { func transferNEOToAlphabetContracts(c *helper.InitializeContext) error { neoHash := neo.Hash - ok, err := transferNEOFinished(c, neoHash) + ok, err := transferNEOFinished(c) if ok || err != nil { return err } @@ -137,8 +136,8 @@ func transferNEOToAlphabetContracts(c *helper.InitializeContext) error { return c.AwaitTx() } -func transferNEOFinished(c *helper.InitializeContext, neoHash util.Uint160) (bool, error) { - r := nep17.NewReader(c.ReadOnlyInvoker, neoHash) +func transferNEOFinished(c *helper.InitializeContext) (bool, error) { + r := neo.NewReader(c.ReadOnlyInvoker) bal, err := r.BalanceOf(c.CommitteeAcc.Contract.ScriptHash()) return bal.Cmp(big.NewInt(native.NEOTotalSupply)) == -1, err } From bd8ab2d84a59a1c251820a71e63d5e741fd9450e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 08:27:49 +0300 Subject: [PATCH 1830/1943] [#1689] adm: Remove useless switch in NNSIsAvailable() After all the refactorings, there is no need to have custom behaviour for local client. Change-Id: I99e297cdeffff979524b3f89d3580ab5780e7681 Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/helper/initialize.go | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go index 961ceba53..48f7a62a4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go @@ -13,7 +13,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/core/native/nativenames" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" - "github.com/nspcc-dev/neo-go/pkg/rpcclient" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" nns2 "github.com/nspcc-dev/neo-go/pkg/rpcclient/nns" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" @@ -187,19 +186,9 @@ func NNSResolveKey(inv *invoker.Invoker, nnsHash util.Uint160, domain string) (* } func NNSIsAvailable(c Client, nnsHash util.Uint160, name string) (bool, error) { - switch c.(type) { - case *rpcclient.Client: - inv := invoker.New(c, nil) - reader := nns2.NewReader(inv, nnsHash) - return reader.IsAvailable(name) - default: - b, err := unwrap.Bool(InvokeFunction(c, nnsHash, "isAvailable", []any{name}, nil)) - if err != nil { - return false, fmt.Errorf("`isAvailable`: invalid response: %w", err) - } - - return b, nil - } + inv := invoker.New(c, nil) + reader := nns2.NewReader(inv, nnsHash) + return reader.IsAvailable(name) } func CheckNotaryEnabled(c Client) error { From 60446bb66816e5d92bccf3a61022b708b6506297 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 08:31:10 +0300 Subject: [PATCH 1831/1943] [#1689] adm/helper: Use proper nns bindings import The one in `neo-go` is for another contract. Change-Id: Ia1ac2da5e419b48801afdb26df72892d77344e0d Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/helper/initialize.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go index 48f7a62a4..50b5c1ec7 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize.go @@ -6,6 +6,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" + nns2 "git.frostfs.info/TrueCloudLab/frostfs-contract/rpcclient/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" @@ -14,7 +15,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/nspcc-dev/neo-go/pkg/encoding/address" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" - nns2 "github.com/nspcc-dev/neo-go/pkg/rpcclient/nns" "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/trigger" "github.com/nspcc-dev/neo-go/pkg/util" From d00c606feed8ad776fe6df65b601b81790e7dfbe Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 21 Mar 2025 08:12:20 +0300 Subject: [PATCH 1832/1943] [#652] adm: Group independent stages in batches Each stage waits until transaction persists. This is needed to ensure the next stage will see the result of the previous one. However, some of the stages do not depend one on another, so we may execute them in parallel. `AwaitDisabled` flag is used to localize this batching on the code level. We could've removed `AwaitTx()` from respective stages, but it seems more error prone. Close #652. Change-Id: Ib9c6f6cd5e0db0f31aa1cda8e127b1fad5166336 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/helper/n3client.go | 4 +++- .../internal/modules/morph/initialize/initialize.go | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go index 3f3a66cb6..d6ca012ce 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go @@ -40,6 +40,8 @@ type ClientContext struct { CommitteeAct *actor.Actor // committee actor with the Global witness scope ReadOnlyInvoker *invoker.Invoker // R/O contract invoker, does not contain any signer SentTxs []HashVUBPair + + AwaitDisabled bool } func NewRemoteClient(v *viper.Viper) (Client, error) { @@ -120,7 +122,7 @@ func (c *ClientContext) SendTx(tx *transaction.Transaction, cmd *cobra.Command, } func (c *ClientContext) AwaitTx(cmd *cobra.Command) error { - if len(c.SentTxs) == 0 { + if len(c.SentTxs) == 0 || c.AwaitDisabled { return nil } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go index cdaf7d3bc..4d39dc662 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go @@ -39,6 +39,7 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { return err } + initCtx.AwaitDisabled = true cmd.Println("Stage 4.1: Transfer GAS to proxy contract.") if err := transferGASToProxy(initCtx); err != nil { return err @@ -55,5 +56,10 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { } cmd.Println("Stage 7: set addresses in NNS.") - return setNNS(initCtx) + if err := setNNS(initCtx); err != nil { + return err + } + + initCtx.AwaitDisabled = false + return initCtx.AwaitTx() } From a2053870e23b9d58529cd3cebb7898bff2abae15 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 20 Mar 2025 16:36:29 +0300 Subject: [PATCH 1833/1943] [#1692] metabase: Use bucket cache in ListWithCursor() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No changes in speed, but unified approach: ``` goos: linux goarch: amd64 pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz │ master │ new │ │ sec/op │ sec/op vs base │ ListWithCursor/1_item-8 6.067µ ± 8% 5.731µ ± 10% ~ (p=0.052 n=10) ListWithCursor/10_items-8 25.40µ ± 11% 26.12µ ± 13% ~ (p=0.971 n=10) ListWithCursor/100_items-8 210.7µ ± 9% 203.2µ ± 6% ~ (p=0.280 n=10) geomean 31.90µ 31.22µ -2.16% │ master │ new │ │ B/op │ B/op vs base │ ListWithCursor/1_item-8 3.287Ki ± 0% 3.287Ki ± 0% ~ (p=1.000 n=10) ¹ ListWithCursor/10_items-8 15.63Ki ± 0% 15.62Ki ± 0% ~ (p=0.328 n=10) ListWithCursor/100_items-8 138.1Ki ± 0% 138.1Ki ± 0% ~ (p=0.340 n=10) geomean 19.21Ki 19.21Ki -0.00% ¹ all samples are equal │ master │ new │ │ allocs/op │ allocs/op vs base │ ListWithCursor/1_item-8 109.0 ± 0% 109.0 ± 0% ~ (p=1.000 n=10) ¹ ListWithCursor/10_items-8 380.0 ± 0% 380.0 ± 0% ~ (p=1.000 n=10) ¹ ListWithCursor/100_items-8 3.082k ± 0% 3.082k ± 0% ~ (p=1.000 n=10) ¹ geomean 503.5 503.5 +0.00% ¹ all samples are equal ``` Change-Id: Ic11673427615053656b2a60068a6d4dbd27af2cb Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/list.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index 0b6cdf702..a1b3f4e2d 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -139,8 +139,7 @@ func (db *DB) listWithCursor(tx *bbolt.Tx, result []objectcore.Info, count int, var containerID cid.ID var offset []byte - graveyardBkt := tx.Bucket(graveyardBucketName) - garbageBkt := tx.Bucket(garbageBucketName) + bc := newBucketCache() rawAddr := make([]byte, cidSize, addressKeySize) @@ -169,7 +168,7 @@ loop: bkt := tx.Bucket(name) if bkt != nil { copy(rawAddr, cidRaw) - result, offset, cursor, err = selectNFromBucket(bkt, objType, graveyardBkt, garbageBkt, rawAddr, containerID, + result, offset, cursor, err = selectNFromBucket(bc, bkt, objType, rawAddr, containerID, result, count, cursor, threshold, currEpoch) if err != nil { return nil, nil, err @@ -204,9 +203,10 @@ loop: // selectNFromBucket similar to selectAllFromBucket but uses cursor to find // object to start selecting from. Ignores inhumed objects. -func selectNFromBucket(bkt *bbolt.Bucket, // main bucket +func selectNFromBucket( + bc *bucketCache, + bkt *bbolt.Bucket, // main bucket objType objectSDK.Type, // type of the objects stored in the main bucket - graveyardBkt, garbageBkt *bbolt.Bucket, // cached graveyard buckets cidRaw []byte, // container ID prefix, optimization cnt cid.ID, // container ID to []objectcore.Info, // listing result @@ -241,6 +241,8 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket } offset = k + graveyardBkt := getGraveyardBucket(bc, bkt.Tx()) + garbageBkt := getGarbageBucket(bc, bkt.Tx()) if inGraveyardWithKey(append(cidRaw, k...), graveyardBkt, garbageBkt) > 0 { continue } @@ -251,7 +253,7 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket } expEpoch, hasExpEpoch := hasExpirationEpoch(&o) - if hasExpEpoch && expEpoch < currEpoch && !objectLocked(bkt.Tx(), cnt, obj) { + if hasExpEpoch && expEpoch < currEpoch && !objectLockedWithCache(bc, bkt.Tx(), cnt, obj) { continue } From d144abc9771fdc8780435e09e6abba50fca81620 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 20 Mar 2025 17:55:31 +0300 Subject: [PATCH 1834/1943] [#1692] metabase: Remove useless `count` variable It is always equal to `len(to)`. Change-Id: Id7a4c26e9711216b78f96e6b2511efa0773e3471 Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/metabase/list.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/local_object_storage/metabase/list.go b/pkg/local_object_storage/metabase/list.go index a1b3f4e2d..2a0bd7f6a 100644 --- a/pkg/local_object_storage/metabase/list.go +++ b/pkg/local_object_storage/metabase/list.go @@ -219,7 +219,6 @@ func selectNFromBucket( cursor = new(Cursor) } - count := len(to) c := bkt.Cursor() k, v := c.First() @@ -231,7 +230,7 @@ func selectNFromBucket( } for ; k != nil; k, v = c.Next() { - if count >= limit { + if len(to) >= limit { break } @@ -275,7 +274,6 @@ func selectNFromBucket( a.SetContainer(cnt) a.SetObject(obj) to = append(to, objectcore.Info{Address: a, Type: objType, IsLinkingObject: isLinkingObj, ECInfo: ecInfo}) - count++ } return to, offset, cursor, nil From d432bebef45be28487db419c7a005f455a08a0ac Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 28 Mar 2025 13:16:35 +0300 Subject: [PATCH 1835/1943] [#1689] go.mod: Bump frostfs-qos version Change-Id: Iaa28da1a1e7b2f4ab7fd8ed661939eb38f4c7782 Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 887f8bb2b..2dbd1e231 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 - git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250320142439-32079ad7c275 + git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250324133647-57d895c32167 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250307150202-749b4e9ab592 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 diff --git a/go.sum b/go.sum index 982fddf23..6969fdc53 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 h1:fluzML8BIIabd07LyPSjc0JAV2qymWkPiFaLrXdALLA= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250320142439-32079ad7c275 h1:WqWxCnCl2ekfjWja/CpGeF2rf4h0x199xhdnsm/j+E8= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250320142439-32079ad7c275/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250324133647-57d895c32167 h1:NhqfqNcATndYwx413BaaYXxVJbkeu2vQOtVyxXw5xCQ= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250324133647-57d895c32167/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250307150202-749b4e9ab592 h1:n7Pl8V7O1yS07J/fqdbzZjVe/mQW42a7eS0QHfgrzJw= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250307150202-749b4e9ab592/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= From 2254c8aff5716a0de0c4367c268223ab2b0390a0 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 31 Mar 2025 10:53:25 +0300 Subject: [PATCH 1836/1943] [#1689] go.mod: Bump SDK version Change-Id: Ic946aa68c3d6da9e7d54363f8e9141c6547707d6 Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2dbd1e231..9ae49a90c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250324133647-57d895c32167 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250307150202-749b4e9ab592 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b diff --git a/go.sum b/go.sum index 6969fdc53..63925fef2 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250324133647-57d895c32167 h1:NhqfqNcATndYwx413BaaYXxVJbkeu2vQOtVyxXw5xCQ= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250324133647-57d895c32167/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250307150202-749b4e9ab592 h1:n7Pl8V7O1yS07J/fqdbzZjVe/mQW42a7eS0QHfgrzJw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250307150202-749b4e9ab592/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945 h1:zM2l316J55h9p30snl6vHBI/h0xmnuqZjnxIjRDtJZw= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 30d4692c3eab07bba7cc472c06f8cb48b0d52263 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 28 Mar 2025 19:32:26 +0300 Subject: [PATCH 1837/1943] [#1640] go.mod: Bump version for `frostfs-locode-db` Change-Id: Ic45ae77d6209c0097575fc8f89b076b22d50d149 Signed-off-by: Anton Nikiforov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9ae49a90c..76662ff5e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( code.gitea.io/sdk/gitea v0.17.1 git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 - git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d + git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.5.2 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250324133647-57d895c32167 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945 diff --git a/go.sum b/go.sum index 63925fef2..42078073b 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1 h1:k1Qw8dWUQczfo0eVXlhrq9 git.frostfs.info/TrueCloudLab/frostfs-contract v0.21.1/go.mod h1:5fSm/l5xSjGWqsPUffSdboiGFUHa7y/1S0fvxzQowN8= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk= git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU= -git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d h1:uJ/wvuMdepbkaV8XMS5uN9B0FQWMep0CttSuDZiDhq0= -git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.4.1-0.20240710074952-65761deb5c0d/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= +git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.5.2 h1:AovQs7bea0fLnYfldCZB88FkUgRj0QaHkJEbcWfgzvY= +git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.5.2/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 h1:fluzML8BIIabd07LyPSjc0JAV2qymWkPiFaLrXdALLA= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250324133647-57d895c32167 h1:NhqfqNcATndYwx413BaaYXxVJbkeu2vQOtVyxXw5xCQ= From 12a0537a7a9f5af0b4c2bd20f1d5b9f1891897d4 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 25 Mar 2025 17:15:21 +0300 Subject: [PATCH 1838/1943] [#1689] ci: Add commit checker to Jenkinsfile - Commit checker image is built from dco-go: https://git.frostfs.info/TrueCloudLab/dco-go/pulls/14 - 'pull_request_target' branch is defined in Jenkins job: https://git.frostfs.info/TrueCloudLab/jenkins/pulls/10 https://git.frostfs.info/TrueCloudLab/jenkins/pulls/11 Change-Id: Ib86c5749f9e084d736b868240c4b47014b02ba8d Signed-off-by: Vitaliy Potyarkin --- .ci/Jenkinsfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 166fb9286..4ddd36406 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -78,6 +78,10 @@ async { } } } -} -// TODO: dco check + task('dco') { + container('git.frostfs.info/truecloudlab/commit-check:master') { + sh 'FROM=pull_request_target commit-check' + } + } +} From 115aae7c34bd85f11a866d62ebe453c393f55c70 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Sat, 22 Mar 2025 01:54:06 +0300 Subject: [PATCH 1839/1943] [#1656] qos: Add tests for MaxActiveRPCLimiter Interceptors Change-Id: Ib65890ae5aec34c34e15d4ec1f05952f74f1ad26 Signed-off-by: Ekaterina Lebedeva --- internal/qos/grpc_test.go | 121 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 internal/qos/grpc_test.go diff --git a/internal/qos/grpc_test.go b/internal/qos/grpc_test.go new file mode 100644 index 000000000..d6e2a689c --- /dev/null +++ b/internal/qos/grpc_test.go @@ -0,0 +1,121 @@ +package qos_test + +import ( + "context" + "errors" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" + "git.frostfs.info/TrueCloudLab/frostfs-qos/limiting" + "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +const ( + okKey = "ok" +) + +var ( + errTest = errors.New("mock") + errResExhausted *apistatus.ResourceExhausted +) + +type mockGRPCServerStream struct { + grpc.ServerStream + + ctx context.Context +} + +func (m *mockGRPCServerStream) Context() context.Context { + return m.ctx +} + +type limiter struct { + acquired bool + released bool +} + +func (l *limiter) Acquire(key string) (limiting.ReleaseFunc, bool) { + l.acquired = true + if key != okKey { + return nil, false + } + return func() { l.released = true }, true +} + +func unaryMaxActiveRPCLimiter(ctx context.Context, lim *limiter, methodName string) error { + interceptor := qos.NewMaxActiveRPCLimiterUnaryServerInterceptor(func() limiting.Limiter { return lim }) + handler := func(ctx context.Context, req any) (any, error) { + return nil, errTest + } + _, err := interceptor(ctx, nil, &grpc.UnaryServerInfo{FullMethod: methodName}, handler) + return err +} + +func streamMaxActiveRPCLimiter(ctx context.Context, lim *limiter, methodName string) error { + interceptor := qos.NewMaxActiveRPCLimiterStreamServerInterceptor(func() limiting.Limiter { return lim }) + handler := func(srv any, stream grpc.ServerStream) error { + return errTest + } + err := interceptor(nil, &mockGRPCServerStream{ctx: ctx}, &grpc.StreamServerInfo{ + FullMethod: methodName, + }, handler) + return err +} + +func Test_MaxActiveRPCLimiter(t *testing.T) { + // UnaryServerInterceptor + t.Run("unary fail", func(t *testing.T) { + var lim limiter + + err := unaryMaxActiveRPCLimiter(context.Background(), &lim, "") + require.ErrorAs(t, err, &errResExhausted) + require.True(t, lim.acquired) + require.False(t, lim.released) + }) + t.Run("unary pass critical", func(t *testing.T) { + var lim limiter + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagCritical.String()) + + err := unaryMaxActiveRPCLimiter(ctx, &lim, "") + require.ErrorIs(t, err, errTest) + require.False(t, lim.acquired) + require.False(t, lim.released) + }) + t.Run("unary pass", func(t *testing.T) { + var lim limiter + + err := unaryMaxActiveRPCLimiter(context.Background(), &lim, okKey) + require.ErrorIs(t, err, errTest) + require.True(t, lim.acquired) + require.True(t, lim.released) + }) + // StreamServerInterceptor + t.Run("stream fail", func(t *testing.T) { + var lim limiter + + err := streamMaxActiveRPCLimiter(context.Background(), &lim, "") + require.ErrorAs(t, err, &errResExhausted) + require.True(t, lim.acquired) + require.False(t, lim.released) + }) + t.Run("stream pass critical", func(t *testing.T) { + var lim limiter + ctx := tagging.ContextWithIOTag(context.Background(), qos.IOTagCritical.String()) + + err := streamMaxActiveRPCLimiter(ctx, &lim, "") + require.ErrorIs(t, err, errTest) + require.False(t, lim.acquired) + require.False(t, lim.released) + }) + t.Run("stream pass", func(t *testing.T) { + var lim limiter + + err := streamMaxActiveRPCLimiter(context.Background(), &lim, okKey) + require.ErrorIs(t, err, errTest) + require.True(t, lim.acquired) + require.True(t, lim.released) + }) +} From dcb2b23a7d863d2e07c11fc22ebd7ceb83467486 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Sat, 22 Mar 2025 16:36:55 +0300 Subject: [PATCH 1840/1943] [#1656] qos: Add test for `SetCriticalIOTag` Interceptor Change-Id: I4a55fcb84e6f65408a1c0120ac917e49e23354a1 Signed-off-by: Ekaterina Lebedeva --- internal/qos/grpc_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/qos/grpc_test.go b/internal/qos/grpc_test.go index d6e2a689c..d4030c39e 100644 --- a/internal/qos/grpc_test.go +++ b/internal/qos/grpc_test.go @@ -19,6 +19,7 @@ const ( var ( errTest = errors.New("mock") + errWrongTag = errors.New("wrong tag") errResExhausted *apistatus.ResourceExhausted ) @@ -119,3 +120,18 @@ func Test_MaxActiveRPCLimiter(t *testing.T) { require.True(t, lim.released) }) } + +func TestSetCriticalIOTagUnaryServerInterceptor_Pass(t *testing.T) { + interceptor := qos.NewSetCriticalIOTagUnaryServerInterceptor() + called := false + handler := func(ctx context.Context, req any) (any, error) { + called = true + if tag, ok := tagging.IOTagFromContext(ctx); ok && tag == qos.IOTagCritical.String() { + return nil, nil + } + return nil, errWrongTag + } + _, err := interceptor(context.Background(), nil, nil, handler) + require.NoError(t, err) + require.True(t, called) +} From 5a13830a94ab866b8425c3a97115a29a60007650 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 31 Mar 2025 14:14:30 +0300 Subject: [PATCH 1841/1943] [#1699] mod: Bump frostfs-qos version Change-Id: Ie5e708c0ca653596c6e3346aa286618868a5aee8 Signed-off-by: Dmitrii Stepanov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 76662ff5e..23be9c822 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.5.2 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 - git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250324133647-57d895c32167 + git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 diff --git a/go.sum b/go.sum index 42078073b..e5d476ffe 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.5.2 h1:AovQs7bea0fLnYfldCZB88 git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.5.2/go.mod h1:7ZZq8iguY7qFsXajdHGmZd2AW4QbucyrJwhbsRfOfek= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 h1:fluzML8BIIabd07LyPSjc0JAV2qymWkPiFaLrXdALLA= git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250324133647-57d895c32167 h1:NhqfqNcATndYwx413BaaYXxVJbkeu2vQOtVyxXw5xCQ= -git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250324133647-57d895c32167/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47 h1:O2c3VOlaGZ862hf2ZPLBMdTG6vGJzhIgDvFEFGfntzU= +git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945 h1:zM2l316J55h9p30snl6vHBI/h0xmnuqZjnxIjRDtJZw= git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= From a5bae6c5af6252a6d340a957742c4095daeb22be Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 31 Mar 2025 14:14:58 +0300 Subject: [PATCH 1842/1943] [#1699] qos: Allow to prohibit operations for IO tag Change-Id: I2bee26885244e241d224860978b6de3526527e96 Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config/engine/config_test.go | 7 ++++--- cmd/frostfs-node/config/engine/shard/limits/config.go | 8 ++++++++ config/example/node.env | 1 + config/example/node.json | 3 ++- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 1 + internal/qos/limiter.go | 10 ++++++++-- 7 files changed, 25 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index eaf2a294e..22f26268d 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -168,9 +168,10 @@ func TestEngineSection(t *testing.T) { LimitOps: toPtr(25000), }, { - Tag: "policer", - Weight: toPtr(5), - LimitOps: toPtr(25000), + Tag: "policer", + Weight: toPtr(5), + LimitOps: toPtr(25000), + Prohibited: true, }, }) require.ElementsMatch(t, writeLimits.Tags, diff --git a/cmd/frostfs-node/config/engine/shard/limits/config.go b/cmd/frostfs-node/config/engine/shard/limits/config.go index b9b5c4382..8444d6aa8 100644 --- a/cmd/frostfs-node/config/engine/shard/limits/config.go +++ b/cmd/frostfs-node/config/engine/shard/limits/config.go @@ -84,6 +84,7 @@ type IOTagConfig struct { Weight *float64 LimitOps *float64 ReservedOps *float64 + Prohibited bool } func tags(c *config.Config) []IOTagConfig { @@ -119,6 +120,13 @@ func tags(c *config.Config) []IOTagConfig { tagConfig.ReservedOps = &r } + v = c.Value(strconv.Itoa(i) + ".prohibited") + if v != nil { + r, err := cast.ToBoolE(v) + panicOnErr(err) + tagConfig.Prohibited = r + } + result = append(result, tagConfig) } } diff --git a/config/example/node.env b/config/example/node.env index 010b6840c..b7c798ad8 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -180,6 +180,7 @@ FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_3_LIMIT_OPS=25000 FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_TAG=policer FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_WEIGHT=5 FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_LIMIT_OPS=25000 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_PROHIBITED=true FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_TAG=internal FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_WEIGHT=200 FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_LIMIT_OPS=0 diff --git a/config/example/node.json b/config/example/node.json index b26c35d2c..2f4413e4d 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -252,7 +252,8 @@ { "tag": "policer", "weight": 5, - "limit_ops": 25000 + "limit_ops": 25000, + "prohibited": true } ] }, diff --git a/config/example/node.yaml b/config/example/node.yaml index 58b687d5c..a07795da5 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -249,6 +249,7 @@ storage: - tag: policer weight: 5 limit_ops: 25000 + prohibited: true write: max_running_ops: 1000 max_waiting_ops: 100 diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 51f0a9669..3944f663f 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -359,6 +359,7 @@ limits: | `tag.weight` | `float` | 0 (no weight) | Weight for queries with the specified tag. Weights must be specified for all tags or not specified for any one. | | `tag.limit_ops` | `float` | 0 (no limit) | Operations per second rate limit for queries with the specified tag. | | `tag.reserved_ops` | `float` | 0 (no reserve) | Reserved operations per second rate for queries with the specified tag. | +| `tag.prohibited` | `bool` | false | If true, operations with this specified tag will be prohibited. | # `node` section diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index 98d254fd0..c73481c2c 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -90,6 +90,7 @@ func converToSchedulingTags(limits []limits.IOTagConfig) map[string]scheduling.T if l.ReservedOps != nil && *l.ReservedOps != 0 { v.ReservedIOPS = l.ReservedOps } + v.Prohibited = l.Prohibited result[l.Tag] = v } return result @@ -164,8 +165,7 @@ func requestArrival(ctx context.Context, s scheduler, stats map[string]*stat) (R rel, err := s.RequestArrival(ctx, tag) stat.inProgress.Add(1) if err != nil { - if errors.Is(err, scheduling.ErrMClockSchedulerRequestLimitExceeded) || - errors.Is(err, errSemaphoreLimitExceeded) { + if isResourceExhaustedErr(err) { stat.resourceExhausted.Add(1) return nil, &apistatus.ResourceExhausted{} } @@ -234,3 +234,9 @@ func exportMetrics(metrics Metrics, stats map[string]*stat, shardID, operation s metrics.SetOperationTagCounters(shardID, operation, tag, pending, inProgress, completed, resExh) } } + +func isResourceExhaustedErr(err error) bool { + return errors.Is(err, scheduling.ErrMClockSchedulerRequestLimitExceeded) || + errors.Is(err, errSemaphoreLimitExceeded) || + errors.Is(err, scheduling.ErrTagRequestsProhibited) +} From 6e1576cfdb02883a6bbc4f2800ce98f9cd06c71a Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Sat, 22 Mar 2025 16:40:31 +0300 Subject: [PATCH 1843/1943] [#1656] qos: Add tests for `AdjustOutgoingIOTag` Interceptors Change-Id: If534e756b26cf7f202039d48ecdf554b4283728b Signed-off-by: Ekaterina Lebedeva --- internal/qos/grpc_test.go | 82 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/internal/qos/grpc_test.go b/internal/qos/grpc_test.go index d4030c39e..7d0826754 100644 --- a/internal/qos/grpc_test.go +++ b/internal/qos/grpc_test.go @@ -3,6 +3,7 @@ package qos_test import ( "context" "errors" + "fmt" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" @@ -20,7 +21,9 @@ const ( var ( errTest = errors.New("mock") errWrongTag = errors.New("wrong tag") + errNoTag = errors.New("failed to get tag from context") errResExhausted *apistatus.ResourceExhausted + tags = []qos.IOTag{qos.IOTagBackground, qos.IOTagWritecache, qos.IOTagPolicer, qos.IOTagTreeSync} ) type mockGRPCServerStream struct { @@ -135,3 +138,82 @@ func TestSetCriticalIOTagUnaryServerInterceptor_Pass(t *testing.T) { require.NoError(t, err) require.True(t, called) } + +func TestAdjustOutgoingIOTagUnaryClientInterceptor(t *testing.T) { + interceptor := qos.NewAdjustOutgoingIOTagUnaryClientInterceptor() + + // check context with no value + called := false + invoker := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, opts ...grpc.CallOption) error { + called = true + if _, ok := tagging.IOTagFromContext(ctx); ok { + return fmt.Errorf("%v: expected no IO tags", errWrongTag) + } + return nil + } + require.NoError(t, interceptor(context.Background(), "", nil, nil, nil, invoker, nil)) + require.True(t, called) + + // check context for internal tag + targetTag := qos.IOTagInternal.String() + invoker = func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, opts ...grpc.CallOption) error { + raw, ok := tagging.IOTagFromContext(ctx) + if !ok { + return errNoTag + } + if raw != targetTag { + return errWrongTag + } + return nil + } + for _, tag := range tags { + ctx := tagging.ContextWithIOTag(context.Background(), tag.String()) + require.NoError(t, interceptor(ctx, "", nil, nil, nil, invoker, nil)) + } + + // check context for client tag + ctx := tagging.ContextWithIOTag(context.Background(), "") + targetTag = qos.IOTagClient.String() + require.NoError(t, interceptor(ctx, "", nil, nil, nil, invoker, nil)) +} + +func TestAdjustOutgoingIOTagStreamClientInterceptor(t *testing.T) { + interceptor := qos.NewAdjustOutgoingIOTagStreamClientInterceptor() + + // check context with no value + called := false + streamer := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) { + called = true + if _, ok := tagging.IOTagFromContext(ctx); ok { + return nil, fmt.Errorf("%v: expected no IO tags", errWrongTag) + } + return nil, nil + } + _, err := interceptor(context.Background(), nil, nil, "", streamer, nil) + require.True(t, called) + require.NoError(t, err) + + // check context for internal tag + targetTag := qos.IOTagInternal.String() + streamer = func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) { + raw, ok := tagging.IOTagFromContext(ctx) + if !ok { + return nil, errNoTag + } + if raw != targetTag { + return nil, errWrongTag + } + return nil, nil + } + for _, tag := range tags { + ctx := tagging.ContextWithIOTag(context.Background(), tag.String()) + _, err := interceptor(ctx, nil, nil, "", streamer, nil) + require.NoError(t, err) + } + + // check context for client tag + ctx := tagging.ContextWithIOTag(context.Background(), "") + targetTag = qos.IOTagClient.String() + _, err = interceptor(ctx, nil, nil, "", streamer, nil) + require.NoError(t, err) +} From bc6cc9ae2a2631b8a10d27d588c67d88e7b7041c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Apr 2025 10:55:33 +0300 Subject: [PATCH 1844/1943] [#1700] engine: Print stacks on test request limiter Change-Id: I4952769ca431d1049955823b41b99b0984b385fc Signed-off-by: Dmitrii Stepanov --- .../engine/engine_test.go | 71 ++++++++++++++++--- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 6ef3846ee..711a76100 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -2,8 +2,11 @@ package engine import ( "context" + "fmt" "path/filepath" - "sync/atomic" + "runtime/debug" + "strings" + "sync" "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" @@ -157,26 +160,74 @@ func newTestStorages(root string, smallSize uint64) ([]blobstor.SubStorage, *tes var _ qos.Limiter = (*testQoSLimiter)(nil) type testQoSLimiter struct { - t testing.TB - read atomic.Int64 - write atomic.Int64 + t testing.TB + quard sync.Mutex + id int64 + readStacks map[int64][]byte + writeStacks map[int64][]byte } func (t *testQoSLimiter) SetMetrics(qos.Metrics) {} func (t *testQoSLimiter) Close() { - require.Equal(t.t, int64(0), t.read.Load(), "read requests count after limiter close must be 0") - require.Equal(t.t, int64(0), t.write.Load(), "write requests count after limiter close must be 0") + t.quard.Lock() + defer t.quard.Unlock() + + var sb strings.Builder + var seqN int + for _, stack := range t.readStacks { + seqN++ + sb.WriteString(fmt.Sprintf("%d\n read request stack after limiter close: %s\n", seqN, string(stack))) + } + for _, stack := range t.writeStacks { + seqN++ + sb.WriteString(fmt.Sprintf("%d\n write request stack after limiter close: %s\n", seqN, string(stack))) + } + require.True(t.t, seqN == 0, sb.String()) } func (t *testQoSLimiter) ReadRequest(context.Context) (qos.ReleaseFunc, error) { - t.read.Add(1) - return func() { t.read.Add(-1) }, nil + t.quard.Lock() + defer t.quard.Unlock() + + stack := debug.Stack() + + t.id++ + id := t.id + + if t.readStacks == nil { + t.readStacks = make(map[int64][]byte) + } + t.readStacks[id] = stack + + return func() { + t.quard.Lock() + defer t.quard.Unlock() + + delete(t.readStacks, id) + }, nil } func (t *testQoSLimiter) WriteRequest(context.Context) (qos.ReleaseFunc, error) { - t.write.Add(1) - return func() { t.write.Add(-1) }, nil + t.quard.Lock() + defer t.quard.Unlock() + + stack := debug.Stack() + + t.id++ + id := t.id + + if t.writeStacks == nil { + t.writeStacks = make(map[int64][]byte) + } + t.writeStacks[id] = stack + + return func() { + t.quard.Lock() + defer t.quard.Unlock() + + delete(t.writeStacks, id) + }, nil } func (t *testQoSLimiter) SetParentID(string) {} From 27899598dc2617b859bdcd99d28fc7810329632a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Apr 2025 12:37:35 +0300 Subject: [PATCH 1845/1943] [#1700] gc: Drop Event interface There is only one event: new epoch. Change-Id: I982f3650f7bc753ff2782393625452f0f8cdcc35 Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/shards.go | 4 +- pkg/local_object_storage/shard/control.go | 24 +++-- pkg/local_object_storage/shard/gc.go | 109 +++++++--------------- pkg/local_object_storage/shard/gc_test.go | 4 +- 4 files changed, 47 insertions(+), 94 deletions(-) diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index a38c85151..dfc3b1a35 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -318,8 +318,6 @@ func (e *StorageEngine) SetShardMode(ctx context.Context, id *shard.ID, m mode.M // HandleNewEpoch notifies every shard about NewEpoch event. func (e *StorageEngine) HandleNewEpoch(ctx context.Context, epoch uint64) { - ev := shard.EventNewEpoch(epoch) - e.mtx.RLock() defer e.mtx.RUnlock() @@ -327,7 +325,7 @@ func (e *StorageEngine) HandleNewEpoch(ctx context.Context, epoch uint64) { select { case <-ctx.Done(): return - case sh.NotificationChannel() <- ev: + case sh.NotificationChannel() <- epoch: default: e.log.Debug(ctx, logs.ShardEventProcessingInProgress, zap.Uint64("epoch", epoch), zap.Stringer("shard", sh.ID())) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 6dee2da3f..19b13a8ab 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -108,19 +108,17 @@ func (s *Shard) Init(ctx context.Context) error { s.updateMetrics(ctx) s.gc = &gc{ - gcCfg: &s.gcCfg, - remover: s.removeGarbage, - stopChannel: make(chan struct{}), - eventChan: make(chan Event), - mEventHandler: map[eventType]*eventHandlers{ - eventNewEpoch: { - cancelFunc: func() {}, - handlers: []eventHandler{ - s.collectExpiredLocks, - s.collectExpiredObjects, - s.collectExpiredTombstones, - s.collectExpiredMetrics, - }, + gcCfg: &s.gcCfg, + remover: s.removeGarbage, + stopChannel: make(chan struct{}), + newEpochChan: make(chan uint64), + newEpochHandlers: &newEpochHandlers{ + cancelFunc: func() {}, + handlers: []newEpochHandler{ + s.collectExpiredLocks, + s.collectExpiredObjects, + s.collectExpiredTombstones, + s.collectExpiredMetrics, }, }, } diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 84fb6039e..82e76f1a7 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -33,41 +33,14 @@ type TombstoneSource interface { IsTombstoneAvailable(ctx context.Context, addr oid.Address, epoch uint64) bool } -// Event represents class of external events. -type Event interface { - typ() eventType -} +type newEpochHandler func(context.Context, uint64) -type eventType int - -const ( - _ eventType = iota - eventNewEpoch -) - -type newEpoch struct { - epoch uint64 -} - -func (e newEpoch) typ() eventType { - return eventNewEpoch -} - -// EventNewEpoch returns new epoch event. -func EventNewEpoch(e uint64) Event { - return newEpoch{ - epoch: e, - } -} - -type eventHandler func(context.Context, Event) - -type eventHandlers struct { +type newEpochHandlers struct { prevGroup sync.WaitGroup cancelFunc context.CancelFunc - handlers []eventHandler + handlers []newEpochHandler } type gcRunResult struct { @@ -109,10 +82,10 @@ type gc struct { remover func(context.Context) gcRunResult - // eventChan is used only for listening for the new epoch event. + // newEpochChan is used only for listening for the new epoch event. // It is ok to keep opened, we are listening for context done when writing in it. - eventChan chan Event - mEventHandler map[eventType]*eventHandlers + newEpochChan chan uint64 + newEpochHandlers *newEpochHandlers } type gcCfg struct { @@ -142,15 +115,7 @@ func defaultGCCfg() gcCfg { } func (gc *gc) init(ctx context.Context) { - sz := 0 - - for _, v := range gc.mEventHandler { - sz += len(v.handlers) - } - - if sz > 0 { - gc.workerPool = gc.workerPoolInit(sz) - } + gc.workerPool = gc.workerPoolInit(len(gc.newEpochHandlers.handlers)) ctx = tagging.ContextWithIOTag(ctx, qos.IOTagBackground.String()) gc.wg.Add(2) go gc.tickRemover(ctx) @@ -168,7 +133,7 @@ func (gc *gc) listenEvents(ctx context.Context) { case <-ctx.Done(): gc.log.Warn(ctx, logs.ShardStopEventListenerByContext) return - case event, ok := <-gc.eventChan: + case event, ok := <-gc.newEpochChan: if !ok { gc.log.Warn(ctx, logs.ShardStopEventListenerByClosedEventChannel) return @@ -179,38 +144,33 @@ func (gc *gc) listenEvents(ctx context.Context) { } } -func (gc *gc) handleEvent(ctx context.Context, event Event) { - v, ok := gc.mEventHandler[event.typ()] - if !ok { - return - } - - v.cancelFunc() - v.prevGroup.Wait() +func (gc *gc) handleEvent(ctx context.Context, epoch uint64) { + gc.newEpochHandlers.cancelFunc() + gc.newEpochHandlers.prevGroup.Wait() var runCtx context.Context - runCtx, v.cancelFunc = context.WithCancel(ctx) + runCtx, gc.newEpochHandlers.cancelFunc = context.WithCancel(ctx) - v.prevGroup.Add(len(v.handlers)) + gc.newEpochHandlers.prevGroup.Add(len(gc.newEpochHandlers.handlers)) - for i := range v.handlers { + for i := range gc.newEpochHandlers.handlers { select { case <-ctx.Done(): return default: } - h := v.handlers[i] + h := gc.newEpochHandlers.handlers[i] err := gc.workerPool.Submit(func() { - defer v.prevGroup.Done() - h(runCtx, event) + defer gc.newEpochHandlers.prevGroup.Done() + h(runCtx, epoch) }) if err != nil { gc.log.Warn(ctx, logs.ShardCouldNotSubmitGCJobToWorkerPool, zap.Error(err), ) - v.prevGroup.Done() + gc.newEpochHandlers.prevGroup.Done() } } } @@ -362,7 +322,7 @@ func (s *Shard) getExpiredObjectsParameters() (workerCount, batchSize int) { return } -func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { +func (s *Shard) collectExpiredObjects(ctx context.Context, epoch uint64) { var err error startedAt := time.Now() @@ -370,8 +330,8 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeRegular) }() - s.log.Debug(ctx, logs.ShardGCCollectingExpiredObjectsStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) - defer s.log.Debug(ctx, logs.ShardGCCollectingExpiredObjectsCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) + s.log.Debug(ctx, logs.ShardGCCollectingExpiredObjectsStarted, zap.Uint64("epoch", epoch)) + defer s.log.Debug(ctx, logs.ShardGCCollectingExpiredObjectsCompleted, zap.Uint64("epoch", epoch)) workersCount, batchSize := s.getExpiredObjectsParameters() @@ -380,7 +340,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { errGroup.Go(func() error { batch := make([]oid.Address, 0, batchSize) - expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { + expErr := s.getExpiredObjects(egCtx, epoch, func(o *meta.ExpiredObject) { if o.Type() != objectSDK.TypeTombstone && o.Type() != objectSDK.TypeLock { batch = append(batch, o.Address()) @@ -486,7 +446,7 @@ func (s *Shard) inhumeGC(ctx context.Context, addrs []oid.Address) (meta.InhumeR return s.metaBase.Inhume(ctx, inhumePrm) } -func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { +func (s *Shard) collectExpiredTombstones(ctx context.Context, epoch uint64) { var err error startedAt := time.Now() @@ -494,7 +454,6 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeTombstone) }() - epoch := e.(newEpoch).epoch log := s.log.With(zap.Uint64("epoch", epoch)) log.Debug(ctx, logs.ShardStartedExpiredTombstonesHandling) @@ -566,7 +525,7 @@ func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { } } -func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { +func (s *Shard) collectExpiredLocks(ctx context.Context, epoch uint64) { var err error startedAt := time.Now() @@ -574,8 +533,8 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { s.gc.metrics.AddExpiredObjectCollectionDuration(time.Since(startedAt), err == nil, objectTypeLock) }() - s.log.Debug(ctx, logs.ShardGCCollectingExpiredLocksStarted, zap.Uint64("epoch", e.(newEpoch).epoch)) - defer s.log.Debug(ctx, logs.ShardGCCollectingExpiredLocksCompleted, zap.Uint64("epoch", e.(newEpoch).epoch)) + s.log.Debug(ctx, logs.ShardGCCollectingExpiredLocksStarted, zap.Uint64("epoch", epoch)) + defer s.log.Debug(ctx, logs.ShardGCCollectingExpiredLocksCompleted, zap.Uint64("epoch", epoch)) workersCount, batchSize := s.getExpiredObjectsParameters() @@ -585,14 +544,14 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { errGroup.Go(func() error { batch := make([]oid.Address, 0, batchSize) - expErr := s.getExpiredObjects(egCtx, e.(newEpoch).epoch, func(o *meta.ExpiredObject) { + expErr := s.getExpiredObjects(egCtx, epoch, func(o *meta.ExpiredObject) { if o.Type() == objectSDK.TypeLock { batch = append(batch, o.Address()) if len(batch) == batchSize { expired := batch errGroup.Go(func() error { - s.expiredLocksCallback(egCtx, e.(newEpoch).epoch, expired) + s.expiredLocksCallback(egCtx, epoch, expired) return egCtx.Err() }) batch = make([]oid.Address, 0, batchSize) @@ -606,7 +565,7 @@ func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { if len(batch) > 0 { expired := batch errGroup.Go(func() error { - s.expiredLocksCallback(egCtx, e.(newEpoch).epoch, expired) + s.expiredLocksCallback(egCtx, epoch, expired) return egCtx.Err() }) } @@ -785,17 +744,15 @@ func (s *Shard) HandleDeletedLocks(ctx context.Context, lockers []oid.Address) { } } -// NotificationChannel returns channel for shard events. -func (s *Shard) NotificationChannel() chan<- Event { - return s.gc.eventChan +// NotificationChannel returns channel for new epoch events. +func (s *Shard) NotificationChannel() chan<- uint64 { + return s.gc.newEpochChan } -func (s *Shard) collectExpiredMetrics(ctx context.Context, e Event) { +func (s *Shard) collectExpiredMetrics(ctx context.Context, epoch uint64) { ctx, span := tracing.StartSpanFromContext(ctx, "shard.collectExpiredMetrics") defer span.End() - epoch := e.(newEpoch).epoch - s.log.Debug(ctx, logs.ShardGCCollectingExpiredMetricsStarted, zap.Uint64("epoch", epoch)) defer s.log.Debug(ctx, logs.ShardGCCollectingExpiredMetricsCompleted, zap.Uint64("epoch", epoch)) diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index e3670b441..f512a488a 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -69,7 +69,7 @@ func Test_GCDropsLockedExpiredSimpleObject(t *testing.T) { require.NoError(t, err) epoch.Value = 105 - sh.gc.handleEvent(context.Background(), EventNewEpoch(epoch.Value)) + sh.gc.handleEvent(context.Background(), epoch.Value) var getPrm GetPrm getPrm.SetAddress(objectCore.AddressOf(obj)) @@ -165,7 +165,7 @@ func Test_GCDropsLockedExpiredComplexObject(t *testing.T) { require.True(t, errors.As(err, &splitInfoError), "split info must be provided") epoch.Value = 105 - sh.gc.handleEvent(context.Background(), EventNewEpoch(epoch.Value)) + sh.gc.handleEvent(context.Background(), epoch.Value) _, err = sh.Get(context.Background(), getPrm) require.True(t, client.IsErrObjectNotFound(err) || IsErrObjectExpired(err), "expired complex object must be deleted on epoch after lock expires") From f62d81e26a6b4ca44fbb15fd7dd4e1343bdbed71 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Apr 2025 12:50:21 +0300 Subject: [PATCH 1846/1943] [#1700] gc: Take mode mutex in locks handlers Change-Id: I4408eae3aed936f85427b6246dcf727bd6813a0d Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 82e76f1a7..3b9ad690b 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -664,7 +664,10 @@ func (s *Shard) HandleExpiredTombstones(ctx context.Context, tss []meta.Tombston // HandleExpiredLocks unlocks all objects which were locked by lockers. // If successful, marks lockers themselves as garbage. func (s *Shard) HandleExpiredLocks(ctx context.Context, epoch uint64, lockers []oid.Address) { - if s.GetMode().NoMetabase() { + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { return } @@ -727,7 +730,10 @@ func (s *Shard) inhumeUnlockedIfExpired(ctx context.Context, epoch uint64, unloc // HandleDeletedLocks unlocks all objects which were locked by lockers. func (s *Shard) HandleDeletedLocks(ctx context.Context, lockers []oid.Address) { - if s.GetMode().NoMetabase() { + s.m.RLock() + defer s.m.RUnlock() + + if s.info.Mode.NoMetabase() { return } From 19ca9072237254f1b32b18c19e75586d8bb1c88a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 1 Apr 2025 13:16:11 +0300 Subject: [PATCH 1847/1943] [#1689] treesvc: Untie `createConnection()` from `Service` struct Change-Id: I6212de4b81afe8c2516981a7bb2fea099c7df773 Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 1c3521344..92ff97c1c 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -301,7 +301,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return false } - cc, err := s.createConnection(a) + cc, err := createConnection(a) if err != nil { s.log.Warn(ctx, logs.TreeFailedToConnectForTreeSynchronization, zap.Error(err), zap.String("address", addr)) return false @@ -339,7 +339,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return from } -func (*Service) createConnection(a network.Address) (*grpc.ClientConn, error) { +func createConnection(a network.Address) (*grpc.ClientConn, error) { return grpc.NewClient(a.URIAddr(), grpc.WithChainUnaryInterceptor( qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), From b112a92408fac6a2da5509d5c5c9a86b044ea908 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 1 Apr 2025 13:20:20 +0300 Subject: [PATCH 1848/1943] [#1689] treesvc: Create request after client is initialized Make it easier to follow. Change-Id: I40c4db77f015bb45cb25f16ce24e68188fc14380 Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/cache.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index e2be2f4a2..a74fdc5dd 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -117,13 +117,6 @@ func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (* opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } - req := &HealthcheckRequest{ - Body: &HealthcheckRequest_Body{}, - } - if err := SignMessage(req, c.key); err != nil { - return nil, err - } - cc, err := grpc.NewClient(netAddr.URIAddr(), opts...) if err != nil { return nil, err @@ -131,6 +124,14 @@ func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (* ctx, cancel := context.WithTimeout(ctx, defaultClientConnectTimeout) defer cancel() + + req := &HealthcheckRequest{ + Body: &HealthcheckRequest_Body{}, + } + if err := SignMessage(req, c.key); err != nil { + return nil, err + } + // perform some request to check connection if _, err := NewTreeServiceClient(cc).Healthcheck(ctx, req); err != nil { _ = cc.Close() From a11b54ca150a9ae8900698c95fb3fc26eed44bb3 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 1 Apr 2025 13:31:48 +0300 Subject: [PATCH 1849/1943] [#1689] treesvc: Unify gRPC client creation for cache and sync They connect to the same endpoints, the only difference is that connection for synchronization is limited in lifetime and is closed after the sync is finished. This is probably not intentional, as synchronization was implemented before cache was introduced. However, reusing dialTreeService() in sync.go has possible perfomance implications, so is avoided for now. Change-Id: I2e37befd783b4d873ff833969f932deded1195be Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/cache.go | 29 +---------------------------- pkg/services/tree/sync.go | 8 +++++--- 2 files changed, 6 insertions(+), 31 deletions(-) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index a74fdc5dd..d250f577a 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -9,15 +9,10 @@ import ( "time" internalNet "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" - "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" - metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc" - tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" - "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" "github.com/hashicorp/golang-lru/v2/simplelru" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials/insecure" ) type clientCache struct { @@ -95,29 +90,7 @@ func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (* return nil, err } - opts := []grpc.DialOption{ - grpc.WithChainUnaryInterceptor( - qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), - metrics.NewUnaryClientInterceptor(), - tracing.NewUnaryClientInterceptor(), - tagging.NewUnaryClientInterceptor(), - ), - grpc.WithChainStreamInterceptor( - qos.NewAdjustOutgoingIOTagStreamClientInterceptor(), - metrics.NewStreamClientInterceptor(), - tracing.NewStreamClientInterceptor(), - tagging.NewStreamClientInterceptor(), - ), - grpc.WithContextDialer(c.ds.GrpcContextDialer()), - grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), - grpc.WithDisableServiceConfig(), - } - - if !netAddr.IsTLSEnabled() { - opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - } - - cc, err := grpc.NewClient(netAddr.URIAddr(), opts...) + cc, err := createConnection(netAddr, grpc.WithContextDialer(c.ds.GrpcContextDialer())) if err != nil { return nil, err } diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 92ff97c1c..7107b2bad 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -339,8 +339,8 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return from } -func createConnection(a network.Address) (*grpc.ClientConn, error) { - return grpc.NewClient(a.URIAddr(), +func createConnection(a network.Address, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + defaultOpts := []grpc.DialOption{ grpc.WithChainUnaryInterceptor( qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), metrics.NewUnaryClientInterceptor(), @@ -356,7 +356,9 @@ func createConnection(a network.Address) (*grpc.ClientConn, error) { grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), grpc.WithDisableServiceConfig(), - ) + } + + return grpc.NewClient(a.URIAddr(), append(defaultOpts, opts...)...) } // ErrAlreadySyncing is returned when a service synchronization has already From 3cd808023285c226a086d7756db837f5b0559339 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 1 Apr 2025 13:35:41 +0300 Subject: [PATCH 1850/1943] [#1689] treesvc: Fix dial options for TLS connections There are two problems with the current approach: 1. For TLS connections we need different transport credentials. 2. grpc.NewClient() considers scheme from `URIAddr()` as a scheme for a resolver. `grpcs://` scheme doesn't exist, though, so the default one is picked. The default resolver (`dns://`) is in turn unable to parse the https://github.com/grpc/grpc-go/blob/5edab9e55414068e74320716117a2659c5d2174e/internal/resolver/dns/dns_resolver.go#L405 The error is `grpcs://192.168.198.248:8081:443: too many colons in address`. Both problems don't exist in the SDK code, take it from there. Change-Id: Ia1212050f539162a560796685efdc3f9cfbf80a0 Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/sync.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 7107b2bad..1a084a47f 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -3,6 +3,7 @@ package tree import ( "context" "crypto/sha256" + "crypto/tls" "errors" "fmt" "io" @@ -22,12 +23,14 @@ import ( tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" tracing_grpc "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/panjf2000/ants/v2" "go.uber.org/zap" "golang.org/x/sync/errgroup" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" ) @@ -340,6 +343,16 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, } func createConnection(a network.Address, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + host, isTLS, err := client.ParseURI(a.URIAddr()) + if err != nil { + return nil, err + } + + creds := insecure.NewCredentials() + if isTLS { + creds = credentials.NewTLS(&tls.Config{}) + } + defaultOpts := []grpc.DialOption{ grpc.WithChainUnaryInterceptor( qos.NewAdjustOutgoingIOTagUnaryClientInterceptor(), @@ -353,12 +366,12 @@ func createConnection(a network.Address, opts ...grpc.DialOption) (*grpc.ClientC tracing_grpc.NewStreamClientInterceptor(), tagging.NewStreamClientInterceptor(), ), - grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithTransportCredentials(creds), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), grpc.WithDisableServiceConfig(), } - return grpc.NewClient(a.URIAddr(), append(defaultOpts, opts...)...) + return grpc.NewClient(host, append(defaultOpts, opts...)...) } // ErrAlreadySyncing is returned when a service synchronization has already From b27f7d1d175b7d132497087497270b4bf8ecff64 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 1 Apr 2025 13:43:20 +0300 Subject: [PATCH 1851/1943] [#1689] treesvc: Use context dialer in synchronizeTree() This dialer supports source-based routing and is already used in cache. Change-Id: Ic7852edd2faea4e5d8667221e6f681cc82bb143a Signed-off-by: Evgenii Stratonikov --- pkg/services/tree/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 1a084a47f..8abdafaa7 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -304,7 +304,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return false } - cc, err := createConnection(a) + cc, err := createConnection(a, grpc.WithContextDialer(s.ds.GrpcContextDialer())) if err != nil { s.log.Warn(ctx, logs.TreeFailedToConnectForTreeSynchronization, zap.Error(err), zap.String("address", addr)) return false From bd1c18e117f7a4eb7d23e1f173fbd802252374d4 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 1 Apr 2025 13:51:53 +0300 Subject: [PATCH 1852/1943] [#1689] cli/tree: Copy dial options from the service code There should be no `grpcs://` prefix in address and credentials should be picked. Change-Id: I58cdc98b079eac2c7db7dc088f4f131794a91b9f Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/tree/client.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-cli/modules/tree/client.go b/cmd/frostfs-cli/modules/tree/client.go index 421b96ccd..d71a94b98 100644 --- a/cmd/frostfs-cli/modules/tree/client.go +++ b/cmd/frostfs-cli/modules/tree/client.go @@ -2,17 +2,19 @@ package tree import ( "context" + "crypto/tls" "fmt" - "strings" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/common" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/spf13/cobra" "github.com/spf13/viper" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" ) @@ -31,6 +33,16 @@ func _client() (tree.TreeServiceClient, error) { return nil, err } + host, isTLS, err := client.ParseURI(netAddr.URIAddr()) + if err != nil { + return nil, err + } + + creds := insecure.NewCredentials() + if isTLS { + creds = credentials.NewTLS(&tls.Config{}) + } + opts := []grpc.DialOption{ grpc.WithChainUnaryInterceptor( tracing.NewUnaryClientInterceptor(), @@ -40,13 +52,10 @@ func _client() (tree.TreeServiceClient, error) { ), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), grpc.WithDisableServiceConfig(), + grpc.WithTransportCredentials(creds), } - if !strings.HasPrefix(netAddr.URIAddr(), "grpcs:") { - opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - } - - cc, err := grpc.NewClient(netAddr.URIAddr(), opts...) + cc, err := grpc.NewClient(host, opts...) return tree.NewTreeServiceClient(cc), err } From e142d25fac6c5a3377e598639f36d7286a779a80 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 1 Apr 2025 12:56:55 +0300 Subject: [PATCH 1853/1943] [#1700] gc: Wait for handlers on GC stopping First wait for goroutine handles epoch events to not to get data race on `gc.newEpochHandlers.cancelFunc`. Then cancel handlers and wait for them. Change-Id: I71f11f8526961f8356f582a95b10eb8340c0aedd Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 3b9ad690b..9b327f6f1 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -227,6 +227,9 @@ func (gc *gc) stop(ctx context.Context) { gc.log.Info(ctx, logs.ShardWaitingForGCWorkersToStop) gc.wg.Wait() + + gc.newEpochHandlers.cancelFunc() + gc.newEpochHandlers.prevGroup.Wait() } // iterates over metabase and deletes objects From b924ecb850d7812395e1ff9427857403e1e1f663 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 31 Mar 2025 10:53:40 +0300 Subject: [PATCH 1854/1943] [#1689] object: Make patch streamer use `ApplyHeaderPatch` Change-Id: I4fb94936621544f70ef4e08815c42efaa5ba846f Signed-off-by: Airat Arifullin --- pkg/services/object/patch/streamer.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/services/object/patch/streamer.go b/pkg/services/object/patch/streamer.go index 642b9f9fa..ff13b1d3e 100644 --- a/pkg/services/object/patch/streamer.go +++ b/pkg/services/object/patch/streamer.go @@ -195,7 +195,12 @@ func (s *Streamer) Send(ctx context.Context, req *objectV2.PatchRequest) error { patch.FromV2(req.GetBody()) if !s.nonFirstSend { - err := s.patcher.ApplyAttributesPatch(ctx, patch.NewAttributes, patch.ReplaceAttributes) + err := s.patcher.ApplyHeaderPatch(ctx, + patcher.ApplyHeaderPatchPrm{ + NewSplitHeader: patch.NewSplitHeader, + NewAttributes: patch.NewAttributes, + ReplaceAttributes: patch.ReplaceAttributes, + }) if err != nil { return fmt.Errorf("patch attributes: %w", err) } From 11493d587bd31de7e8d0c52eb671a7abe934294e Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 2 Apr 2025 11:07:15 +0300 Subject: [PATCH 1855/1943] [#579] config: Change config example to be compatible with YAML 1.2 standard In accordance with the YAML 1.2 specification, octal numbers must begin with the 0o prefix. Change-Id: Icb2e83a4aa75c1eb91decd0b7c9b146aaa9fb3e2 Signed-off-by: Alexander Chuprov --- config/example/node.yaml | 8 ++++---- docs/storage-node-configuration.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/example/node.yaml b/config/example/node.yaml index a07795da5..a448ba7ce 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -148,7 +148,7 @@ storage: flush_worker_count: 30 # number of write-cache flusher threads metabase: - perm: 0644 # permissions for metabase files(directories: +x for current user and group) + perm: 0o644 # permissions for metabase files(directories: +x for current user and group) max_batch_size: 200 max_batch_delay: 20ms @@ -161,13 +161,13 @@ storage: blobstor: - size: 4m # approximate size limit of single blobovnicza instance, total size will be: size*width^(depth+1), bytes - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) + perm: 0o644 # permissions for blobstor files(directories: +x for current user and group) depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files opened_cache_ttl: 5m # ttl for opened database file opened_cache_exp_interval: 15s # cache cleanup interval for expired blobovnicza's - - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) + - perm: 0o644 # permissions for blobstor files(directories: +x for current user and group) depth: 5 # max depth of object tree storage in FS gc: @@ -291,7 +291,7 @@ storage: pilorama: path: tmp/1/blob/pilorama.db no_sync: true # USE WITH CAUTION. Return to user before pages have been persisted. - perm: 0644 # permission to use for the database file and intermediate directories + perm: 0o644 # permission to use for the database file and intermediate directories tracing: enabled: true diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 3944f663f..b5c8aadfe 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -209,7 +209,7 @@ blobstor: width: 4 - type: fstree path: /path/to/blobstor/blobovnicza - perm: 0644 + perm: 0o644 size: 4194304 depth: 1 width: 4 @@ -269,7 +269,7 @@ gc: ```yaml metabase: path: /path/to/meta.db - perm: 0644 + perm: 0o644 max_batch_size: 200 max_batch_delay: 20ms ``` From 2a6cdbdb72a3acb07948bf1e0124df98fc964925 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 31 Mar 2025 10:53:56 +0300 Subject: [PATCH 1856/1943] [#1689] cli: Add `split-header` option for object patch command * Make `split-header` option read binary- or JSON-encoded split-header; * Use `PatchHeader` instead of `PatchAttributes`. Change-Id: I50ae1bd93d4695657249dacbea981199a39e1a35 Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/internal/client/client.go | 8 +++++++- cmd/frostfs-cli/modules/object/patch.go | 25 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/internal/client/client.go b/cmd/frostfs-cli/internal/client/client.go index 3f235f070..299d0a830 100644 --- a/cmd/frostfs-cli/internal/client/client.go +++ b/cmd/frostfs-cli/internal/client/client.go @@ -858,6 +858,8 @@ type PatchObjectPrm struct { ReplaceAttribute bool + NewSplitHeader *objectSDK.SplitHeader + PayloadPatches []PayloadPatch } @@ -888,7 +890,11 @@ func Patch(ctx context.Context, prm PatchObjectPrm) (*PatchRes, error) { return nil, fmt.Errorf("init payload reading: %w", err) } - if patcher.PatchAttributes(ctx, prm.NewAttributes, prm.ReplaceAttribute) { + if patcher.PatchHeader(ctx, client.PatchHeaderPrm{ + NewSplitHeader: prm.NewSplitHeader, + NewAttributes: prm.NewAttributes, + ReplaceAttributes: prm.ReplaceAttribute, + }) { for _, pp := range prm.PayloadPatches { payloadFile, err := os.OpenFile(pp.PayloadPath, os.O_RDONLY, os.ModePerm) if err != nil { diff --git a/cmd/frostfs-cli/modules/object/patch.go b/cmd/frostfs-cli/modules/object/patch.go index d98182679..ebbde76a2 100644 --- a/cmd/frostfs-cli/modules/object/patch.go +++ b/cmd/frostfs-cli/modules/object/patch.go @@ -2,6 +2,7 @@ package object import ( "fmt" + "os" "strconv" "strings" @@ -9,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -20,6 +22,7 @@ const ( replaceAttrsFlagName = "replace-attrs" rangeFlagName = "range" payloadFlagName = "payload" + splitHeaderFlagName = "split-header" ) var objectPatchCmd = &cobra.Command{ @@ -50,6 +53,7 @@ func initObjectPatchCmd() { flags.Bool(replaceAttrsFlagName, false, "Replace object attributes by new ones.") flags.StringSlice(rangeFlagName, []string{}, "Range to which patch payload is applied. Format: offset:length") flags.StringSlice(payloadFlagName, []string{}, "Path to file with patch payload.") + flags.String(splitHeaderFlagName, "", "Path to binary or JSON-encoded split header") } func patch(cmd *cobra.Command, _ []string) { @@ -84,6 +88,8 @@ func patch(cmd *cobra.Command, _ []string) { prm.NewAttributes = newAttrs prm.ReplaceAttribute = replaceAttrs + prm.NewSplitHeader = parseSplitHeaderBinaryOrJSON(cmd) + for i := range ranges { prm.PayloadPatches = append(prm.PayloadPatches, internalclient.PayloadPatch{ Range: ranges[i], @@ -147,3 +153,22 @@ func patchPayloadPaths(cmd *cobra.Command) []string { v, _ := cmd.Flags().GetStringSlice(payloadFlagName) return v } + +func parseSplitHeaderBinaryOrJSON(cmd *cobra.Command) *objectSDK.SplitHeader { + path, _ := cmd.Flags().GetString(splitHeaderFlagName) + if path == "" { + return nil + } + + data, err := os.ReadFile(path) + commonCmd.ExitOnErr(cmd, "read file error: %w", err) + + splitHdrV2 := new(objectV2.SplitHeader) + err = splitHdrV2.Unmarshal(data) + if err != nil { + err = splitHdrV2.UnmarshalJSON(data) + commonCmd.ExitOnErr(cmd, "unmarshal error: %w", err) + } + + return objectSDK.NewSplitHeaderFromV2(splitHdrV2) +} From 634de975094b8f83b5b9eeff15b7fd2bfdd6f1fc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 3 Apr 2025 10:09:05 +0300 Subject: [PATCH 1857/1943] [#1704] metabase: Do not ignore errors by Delete Change-Id: Ie7b89071a007f53f55879ff9e7e0c25d24ad5dbf Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/metabase/delete.go | 59 ++++++++++++--------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index d338e228f..cf1d563e9 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -376,11 +376,12 @@ func parentLength(tx *bbolt.Tx, addr oid.Address) int { return len(lst) } -func delUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) { +func delUniqueIndexItem(tx *bbolt.Tx, item namedBucketItem) error { bkt := tx.Bucket(item.name) if bkt != nil { - _ = bkt.Delete(item.key) // ignore error, best effort there + return bkt.Delete(item.key) } + return nil } func delListIndexItem(tx *bbolt.Tx, item namedBucketItem) error { @@ -405,19 +406,16 @@ func delListIndexItem(tx *bbolt.Tx, item namedBucketItem) error { // if list empty, remove the key from bucket if len(lst) == 0 { - _ = bkt.Delete(item.key) // ignore error, best effort there - - return nil + return bkt.Delete(item.key) } // if list is not empty, then update it encodedLst, err := encodeList(lst) if err != nil { - return nil // ignore error, best effort there + return err } - _ = bkt.Put(item.key, encodedLst) // ignore error, best effort there - return nil + return bkt.Put(item.key, encodedLst) } func delFKBTIndexItem(tx *bbolt.Tx, item namedBucketItem) error { @@ -480,35 +478,47 @@ func delUniqueIndexes(tx *bbolt.Tx, obj *objectSDK.Object, isParent bool) error return ErrUnknownObjectType } - delUniqueIndexItem(tx, namedBucketItem{ + if err := delUniqueIndexItem(tx, namedBucketItem{ name: bucketName, key: objKey, - }) + }); err != nil { + return err + } } else { - delUniqueIndexItem(tx, namedBucketItem{ + if err := delUniqueIndexItem(tx, namedBucketItem{ name: parentBucketName(cnr, bucketName), key: objKey, - }) + }); err != nil { + return err + } } - delUniqueIndexItem(tx, namedBucketItem{ // remove from storage id index + if err := delUniqueIndexItem(tx, namedBucketItem{ // remove from storage id index name: smallBucketName(cnr, bucketName), key: objKey, - }) - delUniqueIndexItem(tx, namedBucketItem{ // remove from root index + }); err != nil { + return err + } + if err := delUniqueIndexItem(tx, namedBucketItem{ // remove from root index name: rootBucketName(cnr, bucketName), key: objKey, - }) + }); err != nil { + return err + } if expEpoch, ok := hasExpirationEpoch(obj); ok { - delUniqueIndexItem(tx, namedBucketItem{ + if err := delUniqueIndexItem(tx, namedBucketItem{ name: expEpochToObjectBucketName, key: expirationEpochKey(expEpoch, cnr, addr.Object()), - }) - delUniqueIndexItem(tx, namedBucketItem{ + }); err != nil { + return err + } + if err := delUniqueIndexItem(tx, namedBucketItem{ name: objectToExpirationEpochBucketName(cnr, make([]byte, bucketKeySize)), key: objKey, - }) + }); err != nil { + return err + } } return nil @@ -535,10 +545,12 @@ func deleteECRelatedInfo(tx *bbolt.Tx, garbageBKT *bbolt.Bucket, obj *objectSDK. // also drop EC parent root info if current EC chunk is the last one if !hasAnyChunks { - delUniqueIndexItem(tx, namedBucketItem{ + if err := delUniqueIndexItem(tx, namedBucketItem{ name: rootBucketName(cnr, make([]byte, bucketKeySize)), key: objectKey(ech.Parent(), make([]byte, objectKeySize)), - }) + }); err != nil { + return err + } } if ech.ParentSplitParentID() == nil { @@ -572,11 +584,10 @@ func deleteECRelatedInfo(tx *bbolt.Tx, garbageBKT *bbolt.Bucket, obj *objectSDK. } // drop split info - delUniqueIndexItem(tx, namedBucketItem{ + return delUniqueIndexItem(tx, namedBucketItem{ name: rootBucketName(cnr, make([]byte, bucketKeySize)), key: objectKey(*ech.ParentSplitParentID(), make([]byte, objectKeySize)), }) - return nil } func hasAnyECChunks(tx *bbolt.Tx, ech *objectSDK.ECHeader, cnr cid.ID) bool { From 50dccff7c1e16eb242bfe192ddc3b93d91c03427 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 10 Mar 2025 11:43:40 +0300 Subject: [PATCH 1858/1943] [#1633] morph/netmap: Merge node info in netmap with candidates list Applicable for both cases: when node uses local cache for netmap and when it disabled. Change-Id: I3050f537e20312a4b39e944aca763b77bd1e74c4 Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/cache.go | 292 +++++++++++++++++++----- cmd/frostfs-node/cache_test.go | 74 ++++++ cmd/frostfs-node/config/morph/config.go | 17 ++ cmd/frostfs-node/morph.go | 5 +- cmd/frostfs-node/netmap_source.go | 55 +++++ config/example/node.yaml | 3 + docs/storage-node-configuration.md | 18 +- internal/logs/logs.go | 1 + 8 files changed, 397 insertions(+), 68 deletions(-) create mode 100644 cmd/frostfs-node/netmap_source.go diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index 0fe56d2b0..38cee5837 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -1,20 +1,27 @@ package main import ( + "bytes" + "cmp" "context" + "slices" "sync" + "sync/atomic" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" - lru "github.com/hashicorp/golang-lru/v2" "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/hashicorp/golang-lru/v2/simplelru" + "go.uber.org/zap" ) type netValueReader[K any, V any] func(ctx context.Context, cid K) (V, error) @@ -110,55 +117,6 @@ func (c *ttlNetCache[K, V]) remove(key K) { hit = c.cache.Remove(key) } -// entity that provides LRU cache interface. -type lruNetCache struct { - cache *lru.Cache[uint64, *netmapSDK.NetMap] - - netRdr netValueReader[uint64, *netmapSDK.NetMap] - - metrics cacheMetrics -} - -// newNetworkLRUCache returns wrapper over netValueReader with LRU cache. -func newNetworkLRUCache(sz int, netRdr netValueReader[uint64, *netmapSDK.NetMap], metrics cacheMetrics) *lruNetCache { - cache, err := lru.New[uint64, *netmapSDK.NetMap](sz) - fatalOnErr(err) - - return &lruNetCache{ - cache: cache, - netRdr: netRdr, - metrics: metrics, - } -} - -// reads value by the key. -// -// updates the value from the network on cache miss. -// -// returned value should not be modified. -func (c *lruNetCache) get(ctx context.Context, key uint64) (*netmapSDK.NetMap, error) { - hit := false - startedAt := time.Now() - defer func() { - c.metrics.AddMethodDuration("Get", time.Since(startedAt), hit) - }() - - val, ok := c.cache.Get(key) - if ok { - hit = true - return val, nil - } - - val, err := c.netRdr(ctx, key) - if err != nil { - return nil, err - } - - c.cache.Add(key, val) - - return val, nil -} - // wrapper over TTL cache of values read from the network // that implements container storage. type ttlContainerStorage struct { @@ -200,20 +158,236 @@ func (s ttlContainerStorage) DeletionInfo(ctx context.Context, cnr cid.ID) (*con type lruNetmapSource struct { netState netmap.State - cache *lruNetCache + client rawSource + cache *simplelru.LRU[uint64, *atomic.Pointer[netmapSDK.NetMap]] + mtx sync.RWMutex + metrics cacheMetrics + log *logger.Logger + candidates atomic.Pointer[[]netmapSDK.NodeInfo] } -func newCachedNetmapStorage(s netmap.State, v netmap.Source) netmap.Source { +type rawSource interface { + GetCandidates(ctx context.Context) ([]netmapSDK.NodeInfo, error) + GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmapSDK.NetMap, error) +} + +func newCachedNetmapStorage(ctx context.Context, log *logger.Logger, + netState netmap.State, client rawSource, wg *sync.WaitGroup, d time.Duration, +) netmap.Source { const netmapCacheSize = 10 - lruNetmapCache := newNetworkLRUCache(netmapCacheSize, func(ctx context.Context, key uint64) (*netmapSDK.NetMap, error) { - return v.GetNetMapByEpoch(ctx, key) - }, metrics.NewCacheMetrics("netmap")) + cache, err := simplelru.NewLRU[uint64, *atomic.Pointer[netmapSDK.NetMap]](netmapCacheSize, nil) + fatalOnErr(err) - return &lruNetmapSource{ - netState: s, - cache: lruNetmapCache, + src := &lruNetmapSource{ + netState: netState, + client: client, + cache: cache, + log: log, + metrics: metrics.NewCacheMetrics("netmap"), } + + wg.Add(1) + go func() { + defer wg.Done() + src.updateCandidates(ctx, d) + }() + + return src +} + +// updateCandidates routine to merge netmap in cache with candidates list. +func (s *lruNetmapSource) updateCandidates(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + newCandidates, err := s.client.GetCandidates(ctx) + if err != nil { + s.log.Debug(ctx, logs.FailedToUpdateNetmapCandidates, zap.Error(err)) + timer.Reset(d) + break + } + if len(newCandidates) == 0 { + s.candidates.Store(&newCandidates) + timer.Reset(d) + break + } + slices.SortFunc(newCandidates, func(n1 netmapSDK.NodeInfo, n2 netmapSDK.NodeInfo) int { + return cmp.Compare(n1.Hash(), n2.Hash()) + }) + + // Check once state changed + v := s.candidates.Load() + if v == nil { + s.candidates.Store(&newCandidates) + s.mergeCacheWithCandidates(newCandidates) + timer.Reset(d) + break + } + ret := slices.CompareFunc(*v, newCandidates, func(n1 netmapSDK.NodeInfo, n2 netmapSDK.NodeInfo) int { + if !bytes.Equal(n1.PublicKey(), n2.PublicKey()) || + uint32(n1.Status()) != uint32(n2.Status()) || + slices.Compare(n1.ExternalAddresses(), n2.ExternalAddresses()) != 0 { + return 1 + } + var ne1 []string + n1.IterateNetworkEndpoints(func(s string) bool { + ne1 = append(ne1, s) + return false + }) + var ne2 []string + n2.IterateNetworkEndpoints(func(s string) bool { + ne2 = append(ne2, s) + return false + }) + return slices.Compare(ne1, ne2) + }) + if ret != 0 { + s.candidates.Store(&newCandidates) + s.mergeCacheWithCandidates(newCandidates) + } + timer.Reset(d) + } + } +} + +func (s *lruNetmapSource) mergeCacheWithCandidates(candidates []netmapSDK.NodeInfo) { + s.mtx.Lock() + tmp := s.cache.Values() + s.mtx.Unlock() + for _, pointer := range tmp { + nm := pointer.Load() + updates := getNetMapNodesToUpdate(nm, candidates) + if len(updates) > 0 { + nm = nm.Clone() + mergeNetmapWithCandidates(updates, nm) + pointer.Store(nm) + } + } +} + +// reads value by the key. +// +// updates the value from the network on cache miss. +// +// returned value should not be modified. +func (s *lruNetmapSource) get(ctx context.Context, key uint64) (*netmapSDK.NetMap, error) { + hit := false + startedAt := time.Now() + defer func() { + s.metrics.AddMethodDuration("Get", time.Since(startedAt), hit) + }() + + s.mtx.RLock() + val, ok := s.cache.Get(key) + s.mtx.RUnlock() + if ok { + hit = true + return val.Load(), nil + } + + s.mtx.Lock() + defer s.mtx.Unlock() + + val, ok = s.cache.Get(key) + if ok { + hit = true + return val.Load(), nil + } + + nm, err := s.client.GetNetMapByEpoch(ctx, key) + if err != nil { + return nil, err + } + v := s.candidates.Load() + if v != nil { + updates := getNetMapNodesToUpdate(nm, *v) + if len(updates) > 0 { + mergeNetmapWithCandidates(updates, nm) + } + } + + p := atomic.Pointer[netmapSDK.NetMap]{} + p.Store(nm) + s.cache.Add(key, &p) + + return nm, nil +} + +// mergeNetmapWithCandidates updates nodes state in the provided netmap with state in the list of candidates. +func mergeNetmapWithCandidates(updates []nodeToUpdate, nm *netmapSDK.NetMap) { + for _, v := range updates { + if v.status != netmapSDK.UnspecifiedState { + nm.Nodes()[v.netmapIndex].SetStatus(v.status) + } + if v.externalAddresses != nil { + nm.Nodes()[v.netmapIndex].SetExternalAddresses(v.externalAddresses...) + } + if v.endpoints != nil { + nm.Nodes()[v.netmapIndex].SetNetworkEndpoints(v.endpoints...) + } + } +} + +type nodeToUpdate struct { + netmapIndex int + status netmapSDK.NodeState + externalAddresses []string + endpoints []string +} + +// getNetMapNodesToUpdate checks for the changes between provided netmap and the list of candidates. +func getNetMapNodesToUpdate(nm *netmapSDK.NetMap, candidates []netmapSDK.NodeInfo) []nodeToUpdate { + var res []nodeToUpdate + for i := range nm.Nodes() { + for _, cnd := range candidates { + if bytes.Equal(nm.Nodes()[i].PublicKey(), cnd.PublicKey()) { + var tmp nodeToUpdate + var update bool + + if cnd.Status() != nm.Nodes()[i].Status() && + (cnd.Status() == netmapSDK.Online || cnd.Status() == netmapSDK.Maintenance) { + update = true + tmp.status = cnd.Status() + } + + externalAddresses := cnd.ExternalAddresses() + if externalAddresses != nil && + slices.Compare(externalAddresses, nm.Nodes()[i].ExternalAddresses()) != 0 { + update = true + tmp.externalAddresses = externalAddresses + } + + nodeEndpoints := make([]string, 0, nm.Nodes()[i].NumberOfNetworkEndpoints()) + nm.Nodes()[i].IterateNetworkEndpoints(func(s string) bool { + nodeEndpoints = append(nodeEndpoints, s) + return false + }) + candidateEndpoints := make([]string, 0, cnd.NumberOfNetworkEndpoints()) + cnd.IterateNetworkEndpoints(func(s string) bool { + candidateEndpoints = append(candidateEndpoints, s) + return false + }) + if slices.Compare(nodeEndpoints, candidateEndpoints) != 0 { + update = true + tmp.endpoints = candidateEndpoints + } + + if update { + tmp.netmapIndex = i + res = append(res, tmp) + } + + break + } + } + } + return res } func (s *lruNetmapSource) GetNetMap(ctx context.Context, diff uint64) (*netmapSDK.NetMap, error) { @@ -225,7 +399,7 @@ func (s *lruNetmapSource) GetNetMapByEpoch(ctx context.Context, epoch uint64) (* } func (s *lruNetmapSource) getNetMapByEpoch(ctx context.Context, epoch uint64) (*netmapSDK.NetMap, error) { - val, err := s.cache.get(ctx, epoch) + val, err := s.get(ctx, epoch) if err != nil { return nil, err } diff --git a/cmd/frostfs-node/cache_test.go b/cmd/frostfs-node/cache_test.go index b1601aa67..24286826f 100644 --- a/cmd/frostfs-node/cache_test.go +++ b/cmd/frostfs-node/cache_test.go @@ -3,9 +3,11 @@ package main import ( "context" "errors" + "sync" "testing" "time" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/stretchr/testify/require" ) @@ -59,3 +61,75 @@ func testNetValueReader(_ context.Context, key string) (time.Time, error) { type noopCacheMetricts struct{} func (m *noopCacheMetricts) AddMethodDuration(method string, d time.Duration, hit bool) {} + +type rawSrc struct{} + +func (r *rawSrc) GetCandidates(_ context.Context) ([]netmapSDK.NodeInfo, error) { + node0 := netmapSDK.NodeInfo{} + node0.SetPublicKey([]byte{byte(1)}) + node0.SetStatus(netmapSDK.Online) + node0.SetExternalAddresses("1", "0") + node0.SetNetworkEndpoints("1", "0") + + node1 := netmapSDK.NodeInfo{} + node1.SetPublicKey([]byte{byte(1)}) + node1.SetStatus(netmapSDK.Online) + node1.SetExternalAddresses("1", "0") + node1.SetNetworkEndpoints("1", "0") + + return []netmapSDK.NodeInfo{node0, node1}, nil +} + +func (r *rawSrc) GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmapSDK.NetMap, error) { + nm := netmapSDK.NetMap{} + nm.SetEpoch(1) + + node0 := netmapSDK.NodeInfo{} + node0.SetPublicKey([]byte{byte(1)}) + node0.SetStatus(netmapSDK.Maintenance) + node0.SetExternalAddresses("0") + node0.SetNetworkEndpoints("0") + + node1 := netmapSDK.NodeInfo{} + node1.SetPublicKey([]byte{byte(1)}) + node1.SetStatus(netmapSDK.Maintenance) + node1.SetExternalAddresses("0") + node1.SetNetworkEndpoints("0") + + nm.SetNodes([]netmapSDK.NodeInfo{node0, node1}) + + return &nm, nil +} + +type st struct{} + +func (s *st) CurrentEpoch() uint64 { + return 1 +} + +func TestNetmapStorage(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + wg := sync.WaitGroup{} + cache := newCachedNetmapStorage(ctx, nil, &st{}, &rawSrc{}, &wg, time.Millisecond*50) + + nm, err := cache.GetNetMapByEpoch(ctx, 1) + require.NoError(t, err) + require.True(t, nm.Nodes()[0].Status() == netmapSDK.Maintenance) + require.True(t, len(nm.Nodes()[0].ExternalAddresses()) == 1) + require.True(t, nm.Nodes()[0].NumberOfNetworkEndpoints() == 1) + + require.Eventually(t, func() bool { + nm, err := cache.GetNetMapByEpoch(ctx, 1) + require.NoError(t, err) + for _, node := range nm.Nodes() { + if !(node.Status() == netmapSDK.Online && len(node.ExternalAddresses()) == 2 && + node.NumberOfNetworkEndpoints() == 2) { + return false + } + } + return true + }, time.Second*5, time.Millisecond*10) + + cancel() + wg.Wait() +} diff --git a/cmd/frostfs-node/config/morph/config.go b/cmd/frostfs-node/config/morph/config.go index d089870ea..a9f774d18 100644 --- a/cmd/frostfs-node/config/morph/config.go +++ b/cmd/frostfs-node/config/morph/config.go @@ -33,6 +33,9 @@ const ( // ContainerCacheSizeDefault represents the default size for the container cache. ContainerCacheSizeDefault = 100 + + // PollCandidatesTimeoutDefault is a default poll timeout for netmap candidates. + PollCandidatesTimeoutDefault = 20 * time.Second ) var errNoMorphEndpoints = errors.New("no morph chain RPC endpoints, see `morph.rpc_endpoint` section") @@ -154,3 +157,17 @@ func FrostfsIDCacheSize(c *config.Config) uint32 { } return config.Uint32Safe(c.Sub(subsection), "frostfsid_cache_size") } + +// NetmapCandidatesPollInterval returns the value of "netmap.candidates.poll_interval" config parameter +// from "morph" section. +// +// Returns PollCandidatesTimeoutDefault if the value is not positive duration. +func NetmapCandidatesPollInterval(c *config.Config) time.Duration { + v := config.DurationSafe(c.Sub(subsection). + Sub("netmap").Sub("candidates"), "poll_interval") + if v > 0 { + return v + } + + return PollCandidatesTimeoutDefault +} diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 657e22389..d3c0f7b81 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -60,10 +60,11 @@ func (c *cfg) initMorphComponents(ctx context.Context) { } if c.cfgMorph.cacheTTL < 0 { - netmapSource = wrap + netmapSource = newRawNetmapStorage(wrap) } else { // use RPC node as source of netmap (with caching) - netmapSource = newCachedNetmapStorage(c.cfgNetmap.state, wrap) + netmapSource = newCachedNetmapStorage(ctx, c.log, c.cfgNetmap.state, wrap, &c.wg, + morphconfig.NetmapCandidatesPollInterval(c.appCfg)) } c.netMapSource = netmapSource diff --git a/cmd/frostfs-node/netmap_source.go b/cmd/frostfs-node/netmap_source.go new file mode 100644 index 000000000..e6be9cdf5 --- /dev/null +++ b/cmd/frostfs-node/netmap_source.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + netmapClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" +) + +type rawNetmapSource struct { + client *netmapClient.Client +} + +func newRawNetmapStorage(client *netmapClient.Client) netmap.Source { + return &rawNetmapSource{ + client: client, + } +} + +func (s *rawNetmapSource) GetNetMap(ctx context.Context, diff uint64) (*netmapSDK.NetMap, error) { + nm, err := s.client.GetNetMap(ctx, diff) + if err != nil { + return nil, err + } + candidates, err := s.client.GetCandidates(ctx) + if err != nil { + return nil, err + } + updates := getNetMapNodesToUpdate(nm, candidates) + if len(updates) > 0 { + mergeNetmapWithCandidates(updates, nm) + } + return nm, nil +} + +func (s *rawNetmapSource) GetNetMapByEpoch(ctx context.Context, epoch uint64) (*netmapSDK.NetMap, error) { + nm, err := s.client.GetNetMapByEpoch(ctx, epoch) + if err != nil { + return nil, err + } + candidates, err := s.client.GetCandidates(ctx) + if err != nil { + return nil, err + } + updates := getNetMapNodesToUpdate(nm, candidates) + if len(updates) > 0 { + mergeNetmapWithCandidates(updates, nm) + } + return nm, nil +} + +func (s *rawNetmapSource) Epoch(ctx context.Context) (uint64, error) { + return s.client.Epoch(ctx) +} diff --git a/config/example/node.yaml b/config/example/node.yaml index a448ba7ce..0b6c7b12c 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -95,6 +95,9 @@ morph: - address: wss://rpc2.morph.frostfs.info:40341/ws priority: 2 ape_chain_cache_size: 100000 + netmap: + candidates: + poll_interval: 20s apiclient: dial_timeout: 15s # timeout for FrostFS API client connection diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index b5c8aadfe..5fe011ece 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -148,15 +148,19 @@ morph: - address: wss://rpc2.morph.frostfs.info:40341/ws priority: 2 switch_interval: 2m + netmap: + candidates: + poll_interval: 20s ``` -| Parameter | Type | Default value | Description | -| ---------------------- | --------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `dial_timeout` | `duration` | `5s` | Timeout for dialing connections to N3 RPCs. | -| `cache_ttl` | `duration` | Morph block time | Sidechain cache TTL value (min interval between similar calls).
Negative value disables caching.
Cached entities: containers, container lists, eACL tables. | -| `rpc_endpoint` | list of [endpoint descriptions](#rpc_endpoint-subsection) | | Array of endpoint descriptions. | -| `switch_interval` | `duration` | `2m` | Time interval between the attempts to connect to the highest priority RPC node if the connection is not established yet. | -| `ape_chain_cache_size` | `int` | `10000` | Size of the morph cache for APE chains. | +| Parameter | Type | Default value | Description | +|-----------------------------------|-----------------------------------------------------------|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `dial_timeout` | `duration` | `5s` | Timeout for dialing connections to N3 RPCs. | +| `cache_ttl` | `duration` | Morph block time | Sidechain cache TTL value (min interval between similar calls).
Negative value disables caching.
Cached entities: containers, container lists, eACL tables. | +| `rpc_endpoint` | list of [endpoint descriptions](#rpc_endpoint-subsection) | | Array of endpoint descriptions. | +| `switch_interval` | `duration` | `2m` | Time interval between the attempts to connect to the highest priority RPC node if the connection is not established yet. | +| `ape_chain_cache_size` | `int` | `10000` | Size of the morph cache for APE chains. | +| `netmap.candidates.poll_interval` | `duration` | `20s` | Timeout to set up frequency of merge candidates to netmap with netmap in local cache. | ## `rpc_endpoint` subsection | Parameter | Type | Default value | Description | diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 5b42b25ba..3a3ceb150 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -515,4 +515,5 @@ const ( FailedToGetNetmapToAdjustIOTag = "failed to get netmap to adjust IO tag" FailedToValidateIncomingIOTag = "failed to validate incoming IO tag, replaced with `client`" WriteCacheFailedToAcquireRPSQuota = "writecache failed to acquire RPS quota to flush object" + FailedToUpdateNetmapCandidates = "update netmap candidates failed" ) From 272128e61fd20c53aa0a3b02da8bdc77f7779e26 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 13:32:11 +0300 Subject: [PATCH 1859/1943] Revert "[#652] adm: Group independent stages in batches" This reverts commit d00c606feed8ad776fe6df65b601b81790e7dfbe. There are internal dependencies inside the last stage: first, we register NNS root, only then register add records. Revert for now, will revert back after more testing. Change-Id: I760632b5628caf04849d4a64c714cf286051f357 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-adm/internal/modules/morph/helper/n3client.go | 4 +--- .../internal/modules/morph/initialize/initialize.go | 8 +------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go index d6ca012ce..3f3a66cb6 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/n3client.go @@ -40,8 +40,6 @@ type ClientContext struct { CommitteeAct *actor.Actor // committee actor with the Global witness scope ReadOnlyInvoker *invoker.Invoker // R/O contract invoker, does not contain any signer SentTxs []HashVUBPair - - AwaitDisabled bool } func NewRemoteClient(v *viper.Viper) (Client, error) { @@ -122,7 +120,7 @@ func (c *ClientContext) SendTx(tx *transaction.Transaction, cmd *cobra.Command, } func (c *ClientContext) AwaitTx(cmd *cobra.Command) error { - if len(c.SentTxs) == 0 || c.AwaitDisabled { + if len(c.SentTxs) == 0 { return nil } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go index 4d39dc662..cdaf7d3bc 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize.go @@ -39,7 +39,6 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { return err } - initCtx.AwaitDisabled = true cmd.Println("Stage 4.1: Transfer GAS to proxy contract.") if err := transferGASToProxy(initCtx); err != nil { return err @@ -56,10 +55,5 @@ func initializeSideChainCmd(cmd *cobra.Command, _ []string) error { } cmd.Println("Stage 7: set addresses in NNS.") - if err := setNNS(initCtx); err != nil { - return err - } - - initCtx.AwaitDisabled = false - return initCtx.AwaitTx() + return setNNS(initCtx) } From 2938498b52369bc3abaac3ab1f8e790bba414e8e Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 15:23:41 +0300 Subject: [PATCH 1860/1943] [#1689] adm: Fix NNS root availability check After https://git.frostfs.info/TrueCloudLab/frostfs-contract/pulls/117 we allow checking for root domain availability directly. Before this commit, NNSRootRegistered() has always returned true, so the actual root registration happened as a side-effect of the following code, because NNS registers all parent domains, if they are missing. Change-Id: Icf98f130e77d31b4af7b69697989183c1c8f6a56 Signed-off-by: Evgenii Stratonikov --- .../internal/modules/morph/helper/initialize_ctx.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go index 8e5615baa..27052697f 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go @@ -21,6 +21,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/actor" "github.com/nspcc-dev/neo-go/pkg/rpcclient/management" + "github.com/nspcc-dev/neo-go/pkg/rpcclient/unwrap" "github.com/nspcc-dev/neo-go/pkg/smartcontract/callflag" "github.com/nspcc-dev/neo-go/pkg/smartcontract/context" "github.com/nspcc-dev/neo-go/pkg/smartcontract/manifest" @@ -28,7 +29,6 @@ import ( "github.com/nspcc-dev/neo-go/pkg/util" "github.com/nspcc-dev/neo-go/pkg/vm/emit" "github.com/nspcc-dev/neo-go/pkg/vm/opcode" - "github.com/nspcc-dev/neo-go/pkg/vm/vmstate" "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -524,12 +524,8 @@ func (c *InitializeContext) NNSRegisterDomainScript(nnsHash, expectedHash util.U } func (c *InitializeContext) NNSRootRegistered(nnsHash util.Uint160, zone string) (bool, error) { - res, err := c.CommitteeAct.Call(nnsHash, "isAvailable", "name."+zone) - if err != nil { - return false, err - } - - return res.State == vmstate.Halt.String(), nil + avail, err := unwrap.Bool(c.CommitteeAct.Call(nnsHash, "isAvailable", zone)) + return !avail, err } func (c *InitializeContext) IsUpdated(ctrHash util.Uint160, cs *ContractState) bool { From 5350632e01f2b01f8068dbfec2b9829ae8721519 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Fri, 4 Apr 2025 12:48:22 +0300 Subject: [PATCH 1861/1943] [#1705] engine/test: Increase evacuation timeout This test was flaky in CI probably because of runner load fluctuations. Let's increase the timeout and see if the flakiness goes away. (close #1705) Change-Id: I76f96e3d6f4adb3d5de0e27b8ee6b47685236277 Signed-off-by: Vitaliy Potyarkin --- pkg/local_object_storage/engine/evacuate_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index bd5222b78..359e49481 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -208,7 +208,7 @@ func testWaitForEvacuationCompleted(t *testing.T, e *StorageEngine) *EvacuationS st, err = e.GetEvacuationState(context.Background()) require.NoError(t, err) return st.ProcessingStatus() == EvacuateProcessStateCompleted - }, 3*time.Second, 10*time.Millisecond) + }, 6*time.Second, 10*time.Millisecond) return st } From fbc623f34e5d50d3ca16f1b63ba0ac5fbf011a0b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 2 Apr 2025 10:47:42 +0300 Subject: [PATCH 1862/1943] [#1701] go.mod: Bump `policy-engine` version Change-Id: I7aa359bf235034d6459275d366a276d9930fa227 Signed-off-by: Airat Arifullin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 23be9c822..9ed5e4187 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945 git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 - git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b + git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 github.com/VictoriaMetrics/easyproto v0.1.4 diff --git a/go.sum b/go.sum index e5d476ffe..9727d8786 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/96 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972/go.mod h1:2hM42MBrlhvN6XToaW6OWNk5ZLcu1FhaukGgxtfpDDI= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20241015133823-8aee80dbdc07 h1:gPaqGsk6gSWQyNVjaStydfUz6Z/loHc9XyvGrJ5qSPY= git.frostfs.info/TrueCloudLab/neoneo-go v0.106.1-0.20241015133823-8aee80dbdc07/go.mod h1:bZyJexBlrja4ngxiBgo8by5pVHuAbhg9l09/8yVGDyg= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b h1:M50kdfrf/h8c3cz0bJ2AEUcbXvAlPFVC1Wp1WkfZ/8E= -git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240822104152-a3bc3099bd5b/go.mod h1:GZTk55RI4dKzsK6BCn5h2xxE28UHNfgoq/NJxW/LQ6A= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88 h1:V0a7ia84ZpSM2YxpJq1SKLQfeYmsqFWqcxwweBHJIzc= +git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88/go.mod h1:GZTk55RI4dKzsK6BCn5h2xxE28UHNfgoq/NJxW/LQ6A= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 h1:M2KR3iBj7WpY3hP10IevfIB9MURr4O9mwVfJ+SjT3HA= git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+sZ8rsHrP8Rr/jYPNRc= git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjqZzS4gsb4UA= From 979d4bb2aec9ce74a45b83f00defa532e5d764db Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Wed, 2 Apr 2025 11:15:40 +0300 Subject: [PATCH 1863/1943] [#1701] tree: Form `$Tree:ID` resource property for APE * Make `verifyClient`, `checkAPE` receive `treeID` from request body; * Make `newAPERequest` set `$Tree:ID` property * Add unit-test to check if a rule for `$Tree:ID` works Close #1701 Change-Id: I834fed366e8adfd4b5c07bf50aac09af6239991b Signed-off-by: Airat Arifullin --- pkg/services/tree/ape.go | 12 +++++--- pkg/services/tree/ape_test.go | 43 +++++++++++++++++++++++++++-- pkg/services/tree/service.go | 12 ++++---- pkg/services/tree/signature.go | 4 +-- pkg/services/tree/signature_test.go | 36 ++++++++++++------------ 5 files changed, 76 insertions(+), 31 deletions(-) diff --git a/pkg/services/tree/ape.go b/pkg/services/tree/ape.go index c4b03cbe6..58757ff6d 100644 --- a/pkg/services/tree/ape.go +++ b/pkg/services/tree/ape.go @@ -22,7 +22,7 @@ import ( ) func (s *Service) newAPERequest(ctx context.Context, namespace string, - cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, + cid cid.ID, treeID string, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, ) (aperequest.Request, error) { schemaMethod, err := converter.SchemaMethodFromACLOperation(operation) if err != nil { @@ -53,15 +53,19 @@ func (s *Service) newAPERequest(ctx context.Context, namespace string, resourceName = fmt.Sprintf(nativeschema.ResourceFormatNamespaceContainerObjects, namespace, cid.EncodeToString()) } + resProps := map[string]string{ + nativeschema.ProperyKeyTreeID: treeID, + } + return aperequest.NewRequest( schemaMethod, - aperequest.NewResource(resourceName, make(map[string]string)), + aperequest.NewResource(resourceName, resProps), reqProps, ), nil } func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, - container *core.Container, cid cid.ID, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, + container *core.Container, cid cid.ID, treeID string, operation acl.Op, role acl.Role, publicKey *keys.PublicKey, ) error { namespace := "" cntNamespace, hasNamespace := strings.CutSuffix(cnrSDK.ReadDomain(container.Value).Zone(), ".ns") @@ -69,7 +73,7 @@ func (s *Service) checkAPE(ctx context.Context, bt *bearer.Token, namespace = cntNamespace } - request, err := s.newAPERequest(ctx, namespace, cid, operation, role, publicKey) + request, err := s.newAPERequest(ctx, namespace, cid, treeID, operation, role, publicKey) if err != nil { return fmt.Errorf("failed to create ape request: %w", err) } diff --git a/pkg/services/tree/ape_test.go b/pkg/services/tree/ape_test.go index 0afc7660a..7b209fd47 100644 --- a/pkg/services/tree/ape_test.go +++ b/pkg/services/tree/ape_test.go @@ -107,6 +107,45 @@ func TestCheckAPE(t *testing.T) { cid := cid.ID{} _ = cid.DecodeString(containerID) + t.Run("treeID rule", func(t *testing.T) { + los := inmemory.NewInmemoryLocalStorage() + mcs := inmemory.NewInmemoryMorphRuleChainStorage() + fid := newFrostfsIDProviderMock(t) + s := Service{ + cfg: cfg{ + frostfsidSubjectProvider: fid, + }, + apeChecker: checkercore.New(los, mcs, fid, &stMock{}), + } + + mcs.AddMorphRuleChain(chain.Ingress, engine.ContainerTarget(containerID), &chain.Chain{ + Rules: []chain.Rule{ + { + Status: chain.QuotaLimitReached, + Actions: chain.Actions{Names: []string{nativeschema.MethodGetObject}}, + Resources: chain.Resources{ + Names: []string{fmt.Sprintf(nativeschema.ResourceFormatRootContainerObjects, containerID)}, + }, + Condition: []chain.Condition{ + { + Op: chain.CondStringEquals, + Kind: chain.KindResource, + Key: nativeschema.ProperyKeyTreeID, + Value: versionTreeID, + }, + }, + }, + }, + MatchType: chain.MatchTypeFirstMatch, + }) + + err := s.checkAPE(context.Background(), nil, rootCnr, cid, versionTreeID, acl.OpObjectGet, acl.RoleOwner, senderPrivateKey.PublicKey()) + + var chErr *checkercore.ChainRouterError + require.ErrorAs(t, err, &chErr) + require.Equal(t, chain.QuotaLimitReached, chErr.Status()) + }) + t.Run("put non-tombstone rule won't affect tree remove", func(t *testing.T) { los := inmemory.NewInmemoryLocalStorage() mcs := inmemory.NewInmemoryMorphRuleChainStorage() @@ -152,7 +191,7 @@ func TestCheckAPE(t *testing.T) { MatchType: chain.MatchTypeFirstMatch, }) - err := s.checkAPE(context.Background(), nil, rootCnr, cid, acl.OpObjectDelete, acl.RoleOwner, senderPrivateKey.PublicKey()) + err := s.checkAPE(context.Background(), nil, rootCnr, cid, versionTreeID, acl.OpObjectDelete, acl.RoleOwner, senderPrivateKey.PublicKey()) require.NoError(t, err) }) @@ -201,7 +240,7 @@ func TestCheckAPE(t *testing.T) { MatchType: chain.MatchTypeFirstMatch, }) - err := s.checkAPE(context.Background(), nil, rootCnr, cid, acl.OpObjectPut, acl.RoleOwner, senderPrivateKey.PublicKey()) + err := s.checkAPE(context.Background(), nil, rootCnr, cid, versionTreeID, acl.OpObjectPut, acl.RoleOwner, senderPrivateKey.PublicKey()) require.NoError(t, err) }) } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index b9bb96bab..85bb03a28 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -117,7 +117,7 @@ func (s *Service) Add(ctx context.Context, req *AddRequest) (*AddResponse, error return nil, err } - err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectPut) + err := s.verifyClient(ctx, req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectPut) if err != nil { return nil, err } @@ -161,7 +161,7 @@ func (s *Service) AddByPath(ctx context.Context, req *AddByPathRequest) (*AddByP return nil, err } - err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectPut) + err := s.verifyClient(ctx, req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectPut) if err != nil { return nil, err } @@ -217,7 +217,7 @@ func (s *Service) Remove(ctx context.Context, req *RemoveRequest) (*RemoveRespon return nil, err } - err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectDelete) + err := s.verifyClient(ctx, req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectDelete) if err != nil { return nil, err } @@ -262,7 +262,7 @@ func (s *Service) Move(ctx context.Context, req *MoveRequest) (*MoveResponse, er return nil, err } - err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectPut) + err := s.verifyClient(ctx, req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectPut) if err != nil { return nil, err } @@ -306,7 +306,7 @@ func (s *Service) GetNodeByPath(ctx context.Context, req *GetNodeByPathRequest) return nil, err } - err := s.verifyClient(ctx, req, cid, b.GetBearerToken(), acl.OpObjectGet) + err := s.verifyClient(ctx, req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectGet) if err != nil { return nil, err } @@ -377,7 +377,7 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS return err } - err := s.verifyClient(srv.Context(), req, cid, b.GetBearerToken(), acl.OpObjectGet) + err := s.verifyClient(srv.Context(), req, cid, req.GetBody().GetTreeId(), b.GetBearerToken(), acl.OpObjectGet) if err != nil { return err } diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index d15438e81..89056056d 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -38,7 +38,7 @@ var ( // Operation must be one of: // - 1. ObjectPut; // - 2. ObjectGet. -func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, rawBearer []byte, op acl.Op) error { +func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, treeID string, rawBearer []byte, op acl.Op) error { err := verifyMessage(req) if err != nil { return err @@ -64,7 +64,7 @@ func (s *Service) verifyClient(ctx context.Context, req message, cid cidSDK.ID, return fmt.Errorf("can't get request role: %w", err) } - if err = s.checkAPE(ctx, bt, cnr, cid, op, role, pubKey); err != nil { + if err = s.checkAPE(ctx, bt, cnr, cid, treeID, op, role, pubKey); err != nil { return apeErr(err) } return nil diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 97f8a727a..947de8e58 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -31,6 +31,8 @@ import ( "github.com/stretchr/testify/require" ) +const versionTreeID = "version" + type dummyNetmapSource struct { netmap.Source } @@ -168,26 +170,26 @@ func TestMessageSign(t *testing.T) { cnr.Value.SetBasicACL(acl.PublicRW) t.Run("missing signature, no panic", func(t *testing.T) { - require.Error(t, s.verifyClient(context.Background(), req, cid2, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid2, versionTreeID, nil, op)) }) require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, nil, op)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, nil, op)) t.Run("invalid CID", func(t *testing.T) { - require.Error(t, s.verifyClient(context.Background(), req, cid2, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid2, versionTreeID, nil, op)) }) cnr.Value.SetBasicACL(acl.Private) t.Run("extension disabled", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid2, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid2, versionTreeID, nil, op)) }) t.Run("invalid key", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, nil, op)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, nil, op)) }) t.Run("bearer", func(t *testing.T) { @@ -200,7 +202,7 @@ func TestMessageSign(t *testing.T) { t.Run("invalid bearer", func(t *testing.T) { req.Body.BearerToken = []byte{0xFF} require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("invalid bearer CID", func(t *testing.T) { @@ -209,7 +211,7 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bt.Marshal() require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("invalid bearer owner", func(t *testing.T) { bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) @@ -217,7 +219,7 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bt.Marshal() require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("invalid bearer signature", func(t *testing.T) { bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) @@ -229,7 +231,7 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bv2.StableMarshal(nil) require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) t.Run("impersonate", func(t *testing.T) { @@ -241,8 +243,8 @@ func TestMessageSign(t *testing.T) { req.Body.BearerToken = bt.Marshal() require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) @@ -252,18 +254,18 @@ func TestMessageSign(t *testing.T) { t.Run("put and get", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) t.Run("only get", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[2].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) t.Run("none", func(t *testing.T) { require.NoError(t, SignMessage(req, &privs[3].PrivateKey)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.Error(t, s.verifyClient(context.Background(), req, cid1, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) }) } From 52367dc9b259b6044f0fc7fda09e4339b7170f3d Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 20:07:23 +0300 Subject: [PATCH 1864/1943] [#1689] go.mod: Update sdk-go Change-Id: I72052fe11e66e4c77f4aef6cb2c0f038aa7b0d1f Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9ed5e4187..5ed4a90be 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.5.2 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945 + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88 diff --git a/go.sum b/go.sum index 9727d8786..38aba9bde 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47 h1:O2c3VOlaGZ862hf2ZPLBMdTG6vGJzhIgDvFEFGfntzU= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945 h1:zM2l316J55h9p30snl6vHBI/h0xmnuqZjnxIjRDtJZw= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250326101739-4d36a49d3945/go.mod h1:aQpPWfG8oyfJ2X+FenPTJpSRWZjwcP5/RAtkW+/VEX8= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d h1:ZLKDupw362Ciing7kdIZhDYGMyo2QZyJ6sS/8X9QWJ0= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d/go.mod h1:2PWt5GwJTnhjHp+mankcfCeAJBMn7puxPm+RS+lliVk= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= From 3d771aa21c8fb961c53816566db77b3b0274dc40 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 5 Apr 2025 08:56:22 +0300 Subject: [PATCH 1865/1943] [#1689] morph: Remove `frostfsError` type It has no custom methods defined, only adds `frostfs error: ` prefix to the error message. The utility of this prefix is debatable, failed invocations already have `invocation failed` prefix. Change-Id: If25ebb3679497f3f10acde43b596c81d52351907 Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/client.go | 18 ++---------------- pkg/morph/client/notary.go | 8 +++----- pkg/morph/client/util.go | 2 +- pkg/morph/client/waiter.go | 2 +- 4 files changed, 7 insertions(+), 23 deletions(-) diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 1c12130b7..aab058d27 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -151,20 +151,6 @@ func (e *notHaltStateError) Error() string { ) } -// implementation of error interface for FrostFS-specific errors. -type frostfsError struct { - err error -} - -func (e frostfsError) Error() string { - return fmt.Sprintf("frostfs error: %v", e.err) -} - -// wraps FrostFS-specific error into frostfsError. Arg must not be nil. -func wrapFrostFSError(err error) error { - return frostfsError{err} -} - // Invoke invokes contract method by sending transaction into blockchain. // Returns valid until block value. // Supported args types: int64, string, util.Uint160, []byte and bool. @@ -228,7 +214,7 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, batchSize int if err != nil { return err } else if val.State != HaltState { - return wrapFrostFSError(¬HaltStateError{state: val.State, exception: val.FaultException}) + return ¬HaltStateError{state: val.State, exception: val.FaultException} } arr, sid, r, err := unwrap.ArrayAndSessionIterator(val, err) @@ -292,7 +278,7 @@ func (c *Client) TestInvoke(contract util.Uint160, method string, args ...any) ( } if val.State != HaltState { - return nil, wrapFrostFSError(¬HaltStateError{state: val.State, exception: val.FaultException}) + return nil, ¬HaltStateError{state: val.State, exception: val.FaultException} } success = true diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 4e20a3639..448702613 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -461,7 +461,7 @@ func (c *Client) notaryInvoke(ctx context.Context, committee, invokedByAlpha boo mainH, fbH, untilActual, err := nAct.Notarize(nAct.MakeTunedCall(contract, method, nil, func(r *result.Invoke, t *transaction.Transaction) error { if r.State != vmstate.Halt.String() { - return wrapFrostFSError(¬HaltStateError{state: r.State, exception: r.FaultException}) + return ¬HaltStateError{state: r.State, exception: r.FaultException} } t.ValidUntilBlock = until @@ -608,8 +608,7 @@ func (c *Client) notaryMultisigAccount(ir []*keys.PublicKey, committee, invokedB multisigAccount = wallet.NewAccountFromPrivateKey(c.acc.PrivateKey()) err := multisigAccount.ConvertMultisig(m, ir) if err != nil { - // wrap error as FrostFS-specific since the call is not related to any client - return nil, wrapFrostFSError(fmt.Errorf("convert account to inner ring multisig wallet: %w", err)) + return nil, fmt.Errorf("convert account to inner ring multisig wallet: %w", err) } } else { // alphabet multisig redeem script is @@ -617,8 +616,7 @@ func (c *Client) notaryMultisigAccount(ir []*keys.PublicKey, committee, invokedB // inner ring multiaddress witness multisigAccount, err = notary.FakeMultisigAccount(m, ir) if err != nil { - // wrap error as FrostFS-specific since the call is not related to any client - return nil, wrapFrostFSError(fmt.Errorf("make inner ring multisig wallet: %w", err)) + return nil, fmt.Errorf("make inner ring multisig wallet: %w", err) } } diff --git a/pkg/morph/client/util.go b/pkg/morph/client/util.go index f68d39beb..f7b6705a8 100644 --- a/pkg/morph/client/util.go +++ b/pkg/morph/client/util.go @@ -98,7 +98,7 @@ func StringFromStackItem(param stackitem.Item) (string, error) { func addFeeCheckerModifier(add int64) func(r *result.Invoke, t *transaction.Transaction) error { return func(r *result.Invoke, t *transaction.Transaction) error { if r.State != HaltState { - return wrapFrostFSError(¬HaltStateError{state: r.State, exception: r.FaultException}) + return ¬HaltStateError{state: r.State, exception: r.FaultException} } t.SystemFee += add diff --git a/pkg/morph/client/waiter.go b/pkg/morph/client/waiter.go index 5b9d2cbe0..87fcf84b8 100644 --- a/pkg/morph/client/waiter.go +++ b/pkg/morph/client/waiter.go @@ -47,5 +47,5 @@ func (c *Client) WaitTxHalt(ctx context.Context, vub uint32, h util.Uint256) err if res.VMState.HasFlag(vmstate.Halt) { return nil } - return wrapFrostFSError(¬HaltStateError{state: res.VMState.String(), exception: res.FaultException}) + return ¬HaltStateError{state: res.VMState.String(), exception: res.FaultException} } From d933609084a23595e36bf48e5c226664d315bc23 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 5 Apr 2025 09:07:03 +0300 Subject: [PATCH 1866/1943] [#1689] client/netmap: Refactor Client.config() There are problems with that code: - explicit casts, - `ErrConfigNotFound` which is not a part of a public API, - hand-rolled assertions, even though neo-go already has everything we need. So, remove the error, use `stackitem/Item.Try*()` methods for conversions. Note, that readUint64Config() returns an error if the parameter is missing. This is likely an error, but this behaviour is preserved in this PR: `TryInteger()` returns error when applied to `Null`. By contract, `TryBool()` returns false for `Null`, so this PR introduces no functional changes. Refs https://github.com/nspcc-dev/neo-go/blob/82c7a50b8a308698e1440a716d34232094f9f55f/pkg/vm/stackitem/item.go#L418 Change-Id: I445d28a7c6b5abb9a2bb97b57c0cc42d617e16f7 Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/netmap/config.go | 46 +++++++------------------------ 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index fcdb70e3f..78063b857 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -2,7 +2,6 @@ package netmap import ( "context" - "errors" "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" @@ -106,29 +105,27 @@ func (c *Client) MaintenanceModeAllowed(ctx context.Context) (bool, error) { } func (c *Client) readUInt64Config(ctx context.Context, key string) (uint64, error) { - v, err := c.config(ctx, []byte(key), IntegerAssert) + v, err := c.config(ctx, []byte(key)) if err != nil { return 0, fmt.Errorf("read netconfig value '%s': %w", key, err) } - // IntegerAssert is guaranteed to return int64 if the error is nil. - return uint64(v.(int64)), nil + bi, err := v.TryInteger() + if err != nil { + return 0, err + } + return bi.Uint64(), nil } // reads boolean value by the given key from the FrostFS network configuration // stored in the Sidechain. Returns false if key is not presented. func (c *Client) readBoolConfig(ctx context.Context, key string) (bool, error) { - v, err := c.config(ctx, []byte(key), BoolAssert) + v, err := c.config(ctx, []byte(key)) if err != nil { - if errors.Is(err, ErrConfigNotFound) { - return false, nil - } - return false, fmt.Errorf("read netconfig value '%s': %w", key, err) } - // BoolAssert is guaranteed to return bool if the error is nil. - return v.(bool), nil + return v.TryBool() } // SetConfigPrm groups parameters of SetConfig operation. @@ -277,15 +274,11 @@ func bytesToBool(val []byte) bool { return false } -// ErrConfigNotFound is returned when the requested key was not found -// in the network config (returned value is `Null`). -var ErrConfigNotFound = errors.New("config value not found") - // config performs the test invoke of get config value // method of FrostFS Netmap contract. // // Returns ErrConfigNotFound if config key is not found in the contract. -func (c *Client) config(ctx context.Context, key []byte, assert func(stackitem.Item) (any, error)) (any, error) { +func (c *Client) config(ctx context.Context, key []byte) (stackitem.Item, error) { prm := client.TestInvokePrm{} prm.SetMethod(configMethod) prm.SetArgs(key) @@ -301,26 +294,7 @@ func (c *Client) config(ctx context.Context, key []byte, assert func(stackitem.I configMethod, ln) } - if _, ok := items[0].(stackitem.Null); ok { - return nil, ErrConfigNotFound - } - - return assert(items[0]) -} - -// IntegerAssert converts stack item to int64. -func IntegerAssert(item stackitem.Item) (any, error) { - return client.IntFromStackItem(item) -} - -// StringAssert converts stack item to string. -func StringAssert(item stackitem.Item) (any, error) { - return client.StringFromStackItem(item) -} - -// BoolAssert converts stack item to bool. -func BoolAssert(item stackitem.Item) (any, error) { - return client.BoolFromStackItem(item) + return items[0], nil } // iterateRecords iterates over all config records and passes them to f. From c4f941a5f5217024ed8ba3cebda22d3518bd18f7 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Sat, 5 Apr 2025 09:16:25 +0300 Subject: [PATCH 1867/1943] [#1689] client/netmap: Remove useless error-handling No functional changes. Change-Id: I3a53c992c3ce5e8c6db252abb09aa40626142a97 Signed-off-by: Evgenii Stratonikov --- pkg/morph/client/netmap/config.go | 42 +++++-------------------------- 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/pkg/morph/client/netmap/config.go b/pkg/morph/client/netmap/config.go index 78063b857..3f6aed506 100644 --- a/pkg/morph/client/netmap/config.go +++ b/pkg/morph/client/netmap/config.go @@ -25,44 +25,24 @@ const ( // MaxObjectSize receives max object size configuration // value through the Netmap contract call. func (c *Client) MaxObjectSize(ctx context.Context) (uint64, error) { - objectSize, err := c.readUInt64Config(ctx, MaxObjectSizeConfig) - if err != nil { - return 0, err - } - - return objectSize, nil + return c.readUInt64Config(ctx, MaxObjectSizeConfig) } // EpochDuration returns number of sidechain blocks per one FrostFS epoch. func (c *Client) EpochDuration(ctx context.Context) (uint64, error) { - epochDuration, err := c.readUInt64Config(ctx, EpochDurationConfig) - if err != nil { - return 0, err - } - - return epochDuration, nil + return c.readUInt64Config(ctx, EpochDurationConfig) } // ContainerFee returns fee paid by container owner to each alphabet node // for container registration. func (c *Client) ContainerFee(ctx context.Context) (uint64, error) { - fee, err := c.readUInt64Config(ctx, ContainerFeeConfig) - if err != nil { - return 0, err - } - - return fee, nil + return c.readUInt64Config(ctx, ContainerFeeConfig) } // ContainerAliasFee returns additional fee paid by container owner to each // alphabet node for container nice name registration. func (c *Client) ContainerAliasFee(ctx context.Context) (uint64, error) { - fee, err := c.readUInt64Config(ctx, ContainerAliasFeeConfig) - if err != nil { - return 0, err - } - - return fee, nil + return c.readUInt64Config(ctx, ContainerAliasFeeConfig) } // HomomorphicHashDisabled returns global configuration value of homomorphic hashing @@ -76,23 +56,13 @@ func (c *Client) HomomorphicHashDisabled(ctx context.Context) (bool, error) { // InnerRingCandidateFee returns global configuration value of fee paid by // node to be in inner ring candidates list. func (c *Client) InnerRingCandidateFee(ctx context.Context) (uint64, error) { - fee, err := c.readUInt64Config(ctx, IrCandidateFeeConfig) - if err != nil { - return 0, err - } - - return fee, nil + return c.readUInt64Config(ctx, IrCandidateFeeConfig) } // WithdrawFee returns global configuration value of fee paid by user to // withdraw assets from FrostFS contract. func (c *Client) WithdrawFee(ctx context.Context) (uint64, error) { - fee, err := c.readUInt64Config(ctx, WithdrawFeeConfig) - if err != nil { - return 0, err - } - - return fee, nil + return c.readUInt64Config(ctx, WithdrawFeeConfig) } // MaintenanceModeAllowed reads admission of "maintenance" state from the From 0c5d74729c62d535959965952fd88fe3fbdeffb2 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 13:43:30 +0300 Subject: [PATCH 1868/1943] [#1679] node: Fix 'revive' warning Change-Id: I74ff6332b10f17a329c5d108d01d43002e92aafd Signed-off-by: Alexander Chuprov --- pkg/util/ape/parser.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/util/ape/parser.go b/pkg/util/ape/parser.go index a34a17f6f..6f114d45b 100644 --- a/pkg/util/ape/parser.go +++ b/pkg/util/ape/parser.go @@ -174,11 +174,11 @@ func parseStatus(lexeme string) (apechain.Status, error) { case "deny": if !found { return apechain.AccessDenied, nil - } else if strings.EqualFold(expression, "QuotaLimitReached") { - return apechain.QuotaLimitReached, nil - } else { - return 0, fmt.Errorf("%w: %s", errUnknownStatusDetail, expression) } + if strings.EqualFold(expression, "QuotaLimitReached") { + return apechain.QuotaLimitReached, nil + } + return 0, fmt.Errorf("%w: %s", errUnknownStatusDetail, expression) case "allow": if found { return 0, errUnknownStatusDetail From b4b053cecd4c85361eee345b4b628e98412d270e Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 13:46:49 +0300 Subject: [PATCH 1869/1943] [#1679] node: Fix 'gocognit' warning Change-Id: I6e2a278af51869c05c306c2910ba85130e39532e Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/engine/lock.go | 29 ++++++++++++++++--------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 5d43e59df..18fff9cad 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -84,17 +84,11 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo var siErr *objectSDK.SplitInfoError var eiErr *objectSDK.ECInfoError if errors.As(err, &eiErr) { - eclocked := []oid.ID{locked} - for _, chunk := range eiErr.ECInfo().Chunks { - var objID oid.ID - err = objID.ReadFromV2(chunk.ID) - if err != nil { - e.reportShardError(ctx, sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), - zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) - return false - } - eclocked = append(eclocked, objID) + eclocked, ok := e.checkECLocked(ctx, sh, idCnr, locker, locked, eiErr) + if !ok { + return false } + err = sh.Lock(ctx, idCnr, locker, eclocked) if err != nil { e.reportShardError(ctx, sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), @@ -137,3 +131,18 @@ func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, lo }) return } + +func (e *StorageEngine) checkECLocked(ctx context.Context, sh hashedShard, idCnr cid.ID, locker, locked oid.ID, eiErr *objectSDK.ECInfoError) ([]oid.ID, bool) { + eclocked := []oid.ID{locked} + for _, chunk := range eiErr.ECInfo().Chunks { + var objID oid.ID + err := objID.ReadFromV2(chunk.ID) + if err != nil { + e.reportShardError(ctx, sh, "could not lock object in shard", err, zap.Stringer("container_id", idCnr), + zap.Stringer("locker_id", locker), zap.Stringer("locked_id", locked)) + return nil, false + } + eclocked = append(eclocked, objID) + } + return eclocked, true +} From 9b5c1da40f2eac063ce38ca05d9211deba541234 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 14:21:44 +0300 Subject: [PATCH 1870/1943] [#1679] linter: Bump 'golangci-lint' to v1.64.8 - Removed deprecated config option 'linters.govet.check-shadowing', replaced with enabling the 'shadow' linter. - Removed usage of deprecated 'tenv' linter, replaced by 'usetesting'. Change-Id: Ib1bd1ec83b0fd55a47e405b290bc2bc967b9389c Signed-off-by: Alexander Chuprov --- .golangci.yml | 7 ++----- Makefile | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index f21a46248..18de49425 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -27,9 +27,6 @@ linters-settings: - standard - default custom-order: true - govet: - # report about shadowed variables - check-shadowing: false staticcheck: checks: ["all", "-SA1019"] # TODO Enable SA1019 after deprecated warning are fixed. funlen: @@ -60,8 +57,8 @@ linters-settings: linters: enable: # mandatory linters - - govet - revive + - predeclared # some default golangci-lint linters - errcheck @@ -94,8 +91,8 @@ linters: - testifylint - protogetter - intrange - - tenv - unconvert - unparam + - usetesting disable-all: true fast: false diff --git a/Makefile b/Makefile index 321365f0d..5b55c9eec 100755 --- a/Makefile +++ b/Makefile @@ -9,8 +9,8 @@ HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.23 -LINT_VERSION ?= 1.62.2 -TRUECLOUDLAB_LINT_VERSION ?= 0.0.8 +LINT_VERSION ?= 1.64.8 +TRUECLOUDLAB_LINT_VERSION ?= 0.0.10 PROTOC_VERSION ?= 25.0 PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-sdk-go) PROTOC_OS_VERSION=osx-x86_64 From 923f0acf8f03a485c665f9b610bd5f96b5c0200d Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 31 Mar 2025 05:12:22 +0300 Subject: [PATCH 1871/1943] [#1564] cli: Fix output of `object nodes` command The object nodes command misleadingly reported the number of "found data objects" as if it matched the actual expected amount, which could be incorrect for EC objects. Updated the output wording to explicitly distinguish between currently available data objects and total objects per the EC schema. Change-Id: Ib36b89db58ae66d8978baf5a16b59435db9a068d Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-cli/modules/object/nodes.go | 59 ++++++++++++++++++------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index bc34b370d..734b557a4 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -48,6 +48,12 @@ type ecHeader struct { parent oid.ID } +type objectCounter struct { + sync.Mutex + total uint32 + isECcounted bool +} + type objectPlacement struct { requiredNodes []netmapSDK.NodeInfo confirmedNodes []netmapSDK.NodeInfo @@ -56,6 +62,7 @@ type objectPlacement struct { type objectNodesResult struct { errors []error placements map[oid.ID]objectPlacement + total uint32 } type ObjNodesDataObject struct { @@ -106,18 +113,18 @@ func objectNodes(cmd *cobra.Command, _ []string) { pk := key.GetOrGenerate(cmd) cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC) - objects := getPhyObjects(cmd, cnrID, objID, cli, pk) + objects, count := getPhyObjects(cmd, cnrID, objID, cli, pk) placementPolicy, netmap := getPlacementPolicyAndNetmap(cmd, cnrID, cli) result := getRequiredPlacement(cmd, objects, placementPolicy, netmap) - getActualPlacement(cmd, netmap, pk, objects, result) + getActualPlacement(cmd, netmap, pk, objects, count, result) printPlacement(cmd, objID, objects, result) } -func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) []phyObject { +func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) ([]phyObject, int) { var addrObj oid.Address addrObj.SetContainer(cnrID) addrObj.SetObject(objID) @@ -145,7 +152,7 @@ func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.C parent: res.Header().ECHeader().Parent(), } } - return []phyObject{obj} + return []phyObject{obj}, 1 } var errSplitInfo *objectSDK.SplitInfoError @@ -155,29 +162,34 @@ func getPhyObjects(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.C var ecInfoError *objectSDK.ECInfoError if errors.As(err, &ecInfoError) { - return getECObjectChunks(cmd, cnrID, objID, ecInfoError) + return getECObjectChunks(cmd, cnrID, objID, ecInfoError), 1 } commonCmd.ExitOnErr(cmd, "failed to get object info: %w", err) - return nil + return nil, 0 } -func getComplexObjectParts(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) []phyObject { - members := getCompexObjectMembers(cmd, cnrID, objID, cli, prmHead, errSplitInfo) - return flattenComplexMembersIfECContainer(cmd, cnrID, members, prmHead) +func getComplexObjectParts(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) ([]phyObject, int) { + members, total := getCompexObjectMembers(cmd, cnrID, objID, cli, prmHead, errSplitInfo) + return flattenComplexMembersIfECContainer(cmd, cnrID, members, prmHead), total } -func getCompexObjectMembers(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) []oid.ID { +func getCompexObjectMembers(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, prmHead internalclient.HeadObjectPrm, errSplitInfo *objectSDK.SplitInfoError) ([]oid.ID, int) { + var total int splitInfo := errSplitInfo.SplitInfo() if members, ok := tryGetSplitMembersByLinkingObject(cmd, splitInfo, prmHead, cnrID); ok { - return members + if total = len(members); total > 0 { + total-- // linking object is not data object + } + return members, total } if members, ok := tryGetSplitMembersBySplitID(cmd, splitInfo, cli, cnrID); ok { - return members + return members, len(members) } - return tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnrID, objID) + members := tryRestoreChainInReverse(cmd, splitInfo, prmHead, cli, cnrID, objID) + return members, len(members) } func flattenComplexMembersIfECContainer(cmd *cobra.Command, cnrID cid.ID, members []oid.ID, prmHead internalclient.HeadObjectPrm) []phyObject { @@ -383,8 +395,11 @@ func getECRequiredPlacementInternal(cmd *cobra.Command, object phyObject, placem } } -func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa.PrivateKey, objects []phyObject, result *objectNodesResult) { +func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa.PrivateKey, objects []phyObject, count int, result *objectNodesResult) { resultMtx := &sync.Mutex{} + counter := &objectCounter{ + total: uint32(count), + } candidates := getNodesToCheckObjectExistance(cmd, netmap, result) @@ -401,7 +416,7 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa. for _, object := range objects { eg.Go(func() error { - stored, err := isObjectStoredOnNode(egCtx, cmd, object.containerID, object.objectID, cli, pk) + stored, err := isObjectStoredOnNode(egCtx, cmd, object.containerID, object.objectID, cli, pk, counter) resultMtx.Lock() defer resultMtx.Unlock() if err == nil && stored { @@ -420,6 +435,7 @@ func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, pk *ecdsa. } commonCmd.ExitOnErr(cmd, "failed to get actual placement: %w", eg.Wait()) + result.total = counter.total } func getNodesToCheckObjectExistance(cmd *cobra.Command, netmap *netmapSDK.NetMap, result *objectNodesResult) []netmapSDK.NodeInfo { @@ -478,7 +494,7 @@ func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.N return cli, nil } -func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey) (bool, error) { +func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.Client, pk *ecdsa.PrivateKey, counter *objectCounter) (bool, error) { var addrObj oid.Address addrObj.SetContainer(cnrID) addrObj.SetObject(objID) @@ -493,6 +509,14 @@ func isObjectStoredOnNode(ctx context.Context, cmd *cobra.Command, cnrID cid.ID, res, err := internalclient.HeadObject(ctx, prmHead) if err == nil && res != nil { + if res.Header().ECHeader() != nil { + counter.Lock() + defer counter.Unlock() + if !counter.isECcounted { + counter.total *= res.Header().ECHeader().Total() + } + counter.isECcounted = true + } return true, nil } var notFound *apistatus.ObjectNotFound @@ -512,7 +536,8 @@ func printPlacement(cmd *cobra.Command, objID oid.ID, objects []phyObject, resul } func printObjectNodesAsText(cmd *cobra.Command, objID oid.ID, objects []phyObject, result *objectNodesResult) { - fmt.Fprintf(cmd.OutOrStdout(), "Object %s stores payload in %d data objects:\n", objID.EncodeToString(), len(objects)) + fmt.Fprintf(cmd.OutOrStdout(), "Object %s stores payload in %d data objects\n", objID.EncodeToString(), result.total) + fmt.Fprintf(cmd.OutOrStdout(), "Found %d:\n", len(objects)) for _, object := range objects { fmt.Fprintf(cmd.OutOrStdout(), "- %s\n", object.objectID) From 6f7b6b65f3b8cc3f372d900ee41242f0624b5b67 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 16:54:58 +0300 Subject: [PATCH 1872/1943] [#1689] linter: Fix staticcheck warning: 'embedded field can be simplified' Change-Id: I8f454f7d09973cdea096495c3949b88cdd01102e Signed-off-by: Alexander Chuprov --- .../modules/morph/contract/dump_hashes.go | 2 +- cmd/frostfs-lens/internal/tui/input.go | 8 ++-- cmd/frostfs-lens/internal/tui/ui.go | 2 +- cmd/frostfs-node/apemanager.go | 2 +- cmd/frostfs-node/container.go | 6 +-- cmd/frostfs-node/metrics.go | 28 ++++++------- cmd/frostfs-node/object.go | 6 +-- cmd/frostfs-node/pprof.go | 26 ++++++------ cmd/frostfs-node/tree.go | 4 +- pkg/local_object_storage/blobstor/blobstor.go | 2 +- pkg/local_object_storage/engine/container.go | 4 +- pkg/local_object_storage/engine/control.go | 2 +- pkg/local_object_storage/engine/inhume.go | 4 +- pkg/local_object_storage/engine/shards.go | 2 +- pkg/local_object_storage/pilorama/boltdb.go | 6 +-- pkg/local_object_storage/pilorama/forest.go | 4 +- pkg/local_object_storage/pilorama/inmemory.go | 10 ++--- pkg/local_object_storage/shard/control.go | 4 +- pkg/local_object_storage/shard/gc.go | 4 +- pkg/local_object_storage/shard/id.go | 2 +- pkg/local_object_storage/shard/shard.go | 42 +++++++++---------- pkg/services/control/server/evacuate_async.go | 2 +- pkg/services/object/common/writer/common.go | 6 +-- pkg/services/object/get/assemble.go | 2 +- pkg/services/object/get/get.go | 2 +- pkg/services/object/get/v2/streamer.go | 6 +-- pkg/services/object/patch/service.go | 2 +- pkg/services/object/put/single.go | 22 +++++----- pkg/services/policer/ec.go | 14 +++---- .../session/storage/persistent/storage.go | 2 +- pkg/services/tree/cache.go | 6 +-- pkg/services/tree/replicator.go | 2 +- pkg/services/tree/service.go | 2 +- pkg/services/tree/sync.go | 4 +- 34 files changed, 121 insertions(+), 121 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index fb7e4ff62..7630a226e 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -242,7 +242,7 @@ func fillContractVersion(cmd *cobra.Command, c helper.Client, infos []contractDu script := sub.Bytes() emit.Instruction(bw.BinWriter, opcode.TRY, []byte{byte(3 + len(script) + 2), 0}) - bw.BinWriter.WriteBytes(script) + bw.WriteBytes(script) emit.Instruction(bw.BinWriter, opcode.ENDTRY, []byte{2 + 1}) emit.Opcodes(bw.BinWriter, opcode.PUSH0) } diff --git a/cmd/frostfs-lens/internal/tui/input.go b/cmd/frostfs-lens/internal/tui/input.go index 90729c119..471514e5d 100644 --- a/cmd/frostfs-lens/internal/tui/input.go +++ b/cmd/frostfs-lens/internal/tui/input.go @@ -53,17 +53,17 @@ func (f *InputFieldWithHistory) InputHandler() func(event *tcell.EventKey, setFo f.historyPointer++ // Stop iterating over history. if f.historyPointer == len(f.history) { - f.InputField.SetText(f.currentContent) + f.SetText(f.currentContent) return } - f.InputField.SetText(f.history[f.historyPointer]) + f.SetText(f.history[f.historyPointer]) case tcell.KeyUp: if len(f.history) == 0 { return } // Start iterating over history. if f.historyPointer == len(f.history) { - f.currentContent = f.InputField.GetText() + f.currentContent = f.GetText() } // End of history. if f.historyPointer == 0 { @@ -71,7 +71,7 @@ func (f *InputFieldWithHistory) InputHandler() func(event *tcell.EventKey, setFo } // Iterate to least recent prompts. f.historyPointer-- - f.InputField.SetText(f.history[f.historyPointer]) + f.SetText(f.history[f.historyPointer]) default: f.InputField.InputHandler()(event, func(tview.Primitive) {}) } diff --git a/cmd/frostfs-lens/internal/tui/ui.go b/cmd/frostfs-lens/internal/tui/ui.go index bcc082821..bd7540b01 100644 --- a/cmd/frostfs-lens/internal/tui/ui.go +++ b/cmd/frostfs-lens/internal/tui/ui.go @@ -482,7 +482,7 @@ func (ui *UI) handleInputOnSearching(event *tcell.EventKey) { ui.searchBar.InputHandler()(event, func(tview.Primitive) {}) } - ui.Box.MouseHandler() + ui.MouseHandler() } func (ui *UI) WithPrompt(prompt string) error { diff --git a/cmd/frostfs-node/apemanager.go b/cmd/frostfs-node/apemanager.go index e761a1b14..513314712 100644 --- a/cmd/frostfs-node/apemanager.go +++ b/cmd/frostfs-node/apemanager.go @@ -14,7 +14,7 @@ import ( func initAPEManagerService(c *cfg) { contractStorage := ape_contract.NewProxyVerificationContractStorage( morph.NewSwitchRPCGuardedActor(c.cfgMorph.client), - c.shared.key, + c.key, c.cfgMorph.proxyScriptHash, c.cfgObject.cfgAccessPolicyEngine.policyContractHash) diff --git a/cmd/frostfs-node/container.go b/cmd/frostfs-node/container.go index 012012297..bdb280d87 100644 --- a/cmd/frostfs-node/container.go +++ b/cmd/frostfs-node/container.go @@ -32,7 +32,7 @@ func initContainerService(_ context.Context, c *cfg) { wrap, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0) fatalOnErr(err) - c.shared.cnrClient = wrap + c.cnrClient = wrap cnrSrc := cntClient.AsContainerSource(wrap) @@ -47,7 +47,7 @@ func initContainerService(_ context.Context, c *cfg) { frostfsIDSubjectProvider = newMorphFrostfsIDCache(frostfsIDSubjectProvider, int(cacheSize), c.cfgMorph.cacheTTL, metrics.NewCacheMetrics("frostfs_id")) } - c.shared.frostfsidClient = frostfsIDSubjectProvider + c.frostfsidClient = frostfsIDSubjectProvider c.cfgContainer.containerBatchSize = containerconfig.ContainerBatchSize(c.appCfg) defaultChainRouter := engine.NewDefaultChainRouterWithLocalOverrides( @@ -57,7 +57,7 @@ func initContainerService(_ context.Context, c *cfg) { service := containerService.NewSignService( &c.key.PrivateKey, containerService.NewAPEServer(defaultChainRouter, cnrRdr, - newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, c.shared.frostfsidClient, + newCachedIRFetcher(createInnerRingFetcher(c)), c.netMapSource, c.frostfsidClient, containerService.NewSplitterService( c.cfgContainer.containerBatchSize, c.respSvc, containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc)), diff --git a/cmd/frostfs-node/metrics.go b/cmd/frostfs-node/metrics.go index 19b4af51f..d9ca01e70 100644 --- a/cmd/frostfs-node/metrics.go +++ b/cmd/frostfs-node/metrics.go @@ -8,38 +8,38 @@ import ( func metricsComponent(c *cfg) (*httpComponent, bool) { var updated bool // check if it has been inited before - if c.dynamicConfiguration.metrics == nil { - c.dynamicConfiguration.metrics = new(httpComponent) - c.dynamicConfiguration.metrics.cfg = c - c.dynamicConfiguration.metrics.name = "metrics" - c.dynamicConfiguration.metrics.handler = metrics.Handler() + if c.metrics == nil { + c.metrics = new(httpComponent) + c.metrics.cfg = c + c.metrics.name = "metrics" + c.metrics.handler = metrics.Handler() updated = true } // (re)init read configuration enabled := metricsconfig.Enabled(c.appCfg) - if enabled != c.dynamicConfiguration.metrics.enabled { - c.dynamicConfiguration.metrics.enabled = enabled + if enabled != c.metrics.enabled { + c.metrics.enabled = enabled updated = true } address := metricsconfig.Address(c.appCfg) - if address != c.dynamicConfiguration.metrics.address { - c.dynamicConfiguration.metrics.address = address + if address != c.metrics.address { + c.metrics.address = address updated = true } dur := metricsconfig.ShutdownTimeout(c.appCfg) - if dur != c.dynamicConfiguration.metrics.shutdownDur { - c.dynamicConfiguration.metrics.shutdownDur = dur + if dur != c.metrics.shutdownDur { + c.metrics.shutdownDur = dur updated = true } - return c.dynamicConfiguration.metrics, updated + return c.metrics, updated } func enableMetricsSvc(c *cfg) { - c.shared.metricsSvc.Enable() + c.metricsSvc.Enable() } func disableMetricsSvc(c *cfg) { - c.shared.metricsSvc.Disable() + c.metricsSvc.Disable() } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 2674be8c7..527746d26 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -186,9 +186,9 @@ func initObjectService(c *cfg) { respSvc, ) - c.shared.metricsSvc = objectService.NewMetricCollector( + c.metricsSvc = objectService.NewMetricCollector( signSvc, c.metricsCollector.ObjectService(), metricsconfig.Enabled(c.appCfg)) - qosService := objectService.NewQoSObjectService(c.shared.metricsSvc, &c.cfgQoSService) + qosService := objectService.NewQoSObjectService(c.metricsSvc, &c.cfgQoSService) auditSvc := objectService.NewAuditService(qosService, c.log, c.audit) server := objectTransportGRPC.New(auditSvc) @@ -432,7 +432,7 @@ func createAPEService(c *cfg, irFetcher *cachedIRFetcher, splitSvc *objectServic c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.LocalStorage(), c.cfgObject.cfgAccessPolicyEngine.accessPolicyEngine.MorphRuleChainStorage(), objectAPE.NewStorageEngineHeaderProvider(c.cfgObject.cfgLocalStorage.localStorage, c.cfgObject.getSvc), - c.shared.frostfsidClient, + c.frostfsidClient, c.netMapSource, c.cfgNetmap.state, c.cfgObject.cnrSource, diff --git a/cmd/frostfs-node/pprof.go b/cmd/frostfs-node/pprof.go index 5b40c8a88..e4da8119f 100644 --- a/cmd/frostfs-node/pprof.go +++ b/cmd/frostfs-node/pprof.go @@ -18,33 +18,33 @@ func initProfilerService(ctx context.Context, c *cfg) { func pprofComponent(c *cfg) (*httpComponent, bool) { var updated bool // check if it has been inited before - if c.dynamicConfiguration.pprof == nil { - c.dynamicConfiguration.pprof = new(httpComponent) - c.dynamicConfiguration.pprof.cfg = c - c.dynamicConfiguration.pprof.name = "pprof" - c.dynamicConfiguration.pprof.handler = httputil.Handler() - c.dynamicConfiguration.pprof.preReload = tuneProfilers + if c.pprof == nil { + c.pprof = new(httpComponent) + c.pprof.cfg = c + c.pprof.name = "pprof" + c.pprof.handler = httputil.Handler() + c.pprof.preReload = tuneProfilers updated = true } // (re)init read configuration enabled := profilerconfig.Enabled(c.appCfg) - if enabled != c.dynamicConfiguration.pprof.enabled { - c.dynamicConfiguration.pprof.enabled = enabled + if enabled != c.pprof.enabled { + c.pprof.enabled = enabled updated = true } address := profilerconfig.Address(c.appCfg) - if address != c.dynamicConfiguration.pprof.address { - c.dynamicConfiguration.pprof.address = address + if address != c.pprof.address { + c.pprof.address = address updated = true } dur := profilerconfig.ShutdownTimeout(c.appCfg) - if dur != c.dynamicConfiguration.pprof.shutdownDur { - c.dynamicConfiguration.pprof.shutdownDur = dur + if dur != c.pprof.shutdownDur { + c.pprof.shutdownDur = dur updated = true } - return c.dynamicConfiguration.pprof, updated + return c.pprof, updated } func tuneProfilers(c *cfg) { diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index 65414f0ca..67d9c9df0 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -51,9 +51,9 @@ func initTreeService(c *cfg) { c.treeService = tree.New( tree.WithContainerSource(cnrSource{ src: c.cfgObject.cnrSource, - cli: c.shared.cnrClient, + cli: c.cnrClient, }), - tree.WithFrostfsidSubjectProvider(c.shared.frostfsidClient), + tree.WithFrostfsidSubjectProvider(c.frostfsidClient), tree.WithNetmapSource(c.netMapSource), tree.WithPrivateKey(&c.key.PrivateKey), tree.WithLogger(c.log), diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index f850f48b4..d7c333349 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -153,5 +153,5 @@ func WithMetrics(m Metrics) Option { } func (b *BlobStor) Compressor() *compression.Config { - return &b.cfg.compression + return &b.compression } diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 3160d7f83..03a299b93 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -74,7 +74,7 @@ func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) var csPrm shard.ContainerSizePrm csPrm.SetContainerID(prm.cnr) - csRes, err := sh.Shard.ContainerSize(ctx, csPrm) + csRes, err := sh.ContainerSize(ctx, csPrm) if err != nil { e.reportShardError(ctx, sh, "can't get container size", err, zap.Stringer("container_id", prm.cnr)) @@ -119,7 +119,7 @@ func (e *StorageEngine) listContainers(ctx context.Context) ListContainersRes { uniqueIDs := make(map[string]cid.ID) e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - res, err := sh.Shard.ListContainers(ctx, shard.ListContainersPrm{}) + res, err := sh.ListContainers(ctx, shard.ListContainersPrm{}) if err != nil { e.reportShardError(ctx, sh, "can't get list of containers", err) return false diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 7caa515d4..96b53581e 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -77,7 +77,7 @@ func (e *StorageEngine) Init(ctx context.Context) error { errCh := make(chan shardInitError, len(e.shards)) var eg errgroup.Group - if e.cfg.lowMem && e.anyShardRequiresRefill() { + if e.lowMem && e.anyShardRequiresRefill() { eg.SetLimit(1) } diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index c8ee33b53..6ec9a4ef0 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -227,7 +227,7 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e var outErr error e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { - locked, err = h.Shard.IsLocked(ctx, addr) + locked, err = h.IsLocked(ctx, addr) if err != nil { e.reportShardError(ctx, h, "can't check object's lockers", err, zap.Stringer("address", addr)) outErr = err @@ -256,7 +256,7 @@ func (e *StorageEngine) GetLocks(ctx context.Context, addr oid.Address) ([]oid.I var outErr error e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { - locks, err := h.Shard.GetLocks(ctx, addr) + locks, err := h.GetLocks(ctx, addr) if err != nil { e.reportShardError(ctx, h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr)) outErr = err diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index dfc3b1a35..6e6c08bb5 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -118,7 +118,7 @@ func (e *StorageEngine) AddShard(ctx context.Context, opts ...shard.Option) (*sh return nil, fmt.Errorf("add %s shard: %w", sh.ID().String(), err) } - e.cfg.metrics.SetMode(sh.ID().String(), sh.GetMode()) + e.metrics.SetMode(sh.ID().String(), sh.GetMode()) return sh.ID(), nil } diff --git a/pkg/local_object_storage/pilorama/boltdb.go b/pkg/local_object_storage/pilorama/boltdb.go index fc7cdaabc..897b37ea0 100644 --- a/pkg/local_object_storage/pilorama/boltdb.go +++ b/pkg/local_object_storage/pilorama/boltdb.go @@ -1582,12 +1582,12 @@ func (t *boltForest) moveFromBytes(m *Move, data []byte) error { func (t *boltForest) logFromBytes(lm *Move, data []byte) error { lm.Child = binary.LittleEndian.Uint64(data) lm.Parent = binary.LittleEndian.Uint64(data[8:]) - return lm.Meta.FromBytes(data[16:]) + return lm.FromBytes(data[16:]) } func (t *boltForest) logToBytes(lm *Move) []byte { w := io.NewBufBinWriter() - size := 8 + 8 + lm.Meta.Size() + 1 + size := 8 + 8 + lm.Size() + 1 // if lm.HasOld { // size += 8 + lm.Old.Meta.Size() // } @@ -1595,7 +1595,7 @@ func (t *boltForest) logToBytes(lm *Move) []byte { w.Grow(size) w.WriteU64LE(lm.Child) w.WriteU64LE(lm.Parent) - lm.Meta.EncodeBinary(w.BinWriter) + lm.EncodeBinary(w.BinWriter) // w.WriteBool(lm.HasOld) // if lm.HasOld { // w.WriteU64LE(lm.Old.Parent) diff --git a/pkg/local_object_storage/pilorama/forest.go b/pkg/local_object_storage/pilorama/forest.go index b5320e42d..ebfd0bcc0 100644 --- a/pkg/local_object_storage/pilorama/forest.go +++ b/pkg/local_object_storage/pilorama/forest.go @@ -177,7 +177,7 @@ func (f *memoryForest) TreeSortedByFilename(_ context.Context, cid cid.ID, treeI var res []NodeInfo for _, nodeID := range nodeIDs { - children := s.tree.getChildren(nodeID) + children := s.getChildren(nodeID) for _, childID := range children { var found bool for _, kv := range s.infoMap[childID].Meta.Items { @@ -222,7 +222,7 @@ func (f *memoryForest) TreeGetChildren(_ context.Context, cid cid.ID, treeID str return nil, ErrTreeNotFound } - children := s.tree.getChildren(nodeID) + children := s.getChildren(nodeID) res := make([]NodeInfo, 0, len(children)) for _, childID := range children { res = append(res, NodeInfo{ diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index ce7b3db1e..f74d12a1b 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -35,9 +35,9 @@ func newMemoryTree() *memoryTree { // undo un-does op and changes s in-place. func (s *memoryTree) undo(op *move) { if op.HasOld { - s.tree.infoMap[op.Child] = op.Old + s.infoMap[op.Child] = op.Old } else { - delete(s.tree.infoMap, op.Child) + delete(s.infoMap, op.Child) } } @@ -83,8 +83,8 @@ func (s *memoryTree) do(op *Move) move { }, } - shouldPut := !s.tree.isAncestor(op.Child, op.Parent) - p, ok := s.tree.infoMap[op.Child] + shouldPut := !s.isAncestor(op.Child, op.Parent) + p, ok := s.infoMap[op.Child] if ok { lm.HasOld = true lm.Old = p @@ -100,7 +100,7 @@ func (s *memoryTree) do(op *Move) move { p.Meta = m p.Parent = op.Parent - s.tree.infoMap[op.Child] = p + s.infoMap[op.Child] = p return lm } diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 19b13a8ab..72e650c5e 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -214,8 +214,8 @@ func (s *Shard) refillMetabase(ctx context.Context) error { } eg, egCtx := errgroup.WithContext(ctx) - if s.cfg.refillMetabaseWorkersCount > 0 { - eg.SetLimit(s.cfg.refillMetabaseWorkersCount) + if s.refillMetabaseWorkersCount > 0 { + eg.SetLimit(s.refillMetabaseWorkersCount) } var completedCount uint64 diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 9b327f6f1..19b6e2d12 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -320,8 +320,8 @@ func (s *Shard) getGarbage(ctx context.Context) ([]oid.Address, error) { } func (s *Shard) getExpiredObjectsParameters() (workerCount, batchSize int) { - workerCount = max(minExpiredWorkers, s.gc.gcCfg.expiredCollectorWorkerCount) - batchSize = max(minExpiredBatchSize, s.gc.gcCfg.expiredCollectorBatchSize) + workerCount = max(minExpiredWorkers, s.gc.expiredCollectorWorkerCount) + batchSize = max(minExpiredBatchSize, s.gc.expiredCollectorBatchSize) return } diff --git a/pkg/local_object_storage/shard/id.go b/pkg/local_object_storage/shard/id.go index b233b705c..7391adef2 100644 --- a/pkg/local_object_storage/shard/id.go +++ b/pkg/local_object_storage/shard/id.go @@ -45,7 +45,7 @@ func (s *Shard) UpdateID(ctx context.Context) (err error) { } shardID := s.info.ID.String() - s.cfg.metricsWriter.SetShardID(shardID) + s.metricsWriter.SetShardID(shardID) if s.writeCache != nil && s.writeCache.GetMetrics() != nil { s.writeCache.GetMetrics().SetShardID(shardID) } diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index b9ec05f01..304a6bf9d 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -218,7 +218,7 @@ func WithWriteCache(use bool) Option { // hasWriteCache returns bool if write cache exists on shards. func (s *Shard) hasWriteCache() bool { - return s.cfg.useWriteCache + return s.useWriteCache } // NeedRefillMetabase returns true if metabase is needed to be refilled. @@ -379,15 +379,15 @@ func WithLimiter(l qos.Limiter) Option { } func (s *Shard) fillInfo() { - s.cfg.info.MetaBaseInfo = s.metaBase.DumpInfo() - s.cfg.info.BlobStorInfo = s.blobStor.DumpInfo() - s.cfg.info.Mode = s.GetMode() + s.info.MetaBaseInfo = s.metaBase.DumpInfo() + s.info.BlobStorInfo = s.blobStor.DumpInfo() + s.info.Mode = s.GetMode() - if s.cfg.useWriteCache { - s.cfg.info.WriteCacheInfo = s.writeCache.DumpInfo() + if s.useWriteCache { + s.info.WriteCacheInfo = s.writeCache.DumpInfo() } if s.pilorama != nil { - s.cfg.info.PiloramaInfo = s.pilorama.DumpInfo() + s.info.PiloramaInfo = s.pilorama.DumpInfo() } } @@ -454,57 +454,57 @@ func (s *Shard) updateMetrics(ctx context.Context) { s.setContainerObjectsCount(contID.EncodeToString(), logical, count.Logic) s.setContainerObjectsCount(contID.EncodeToString(), user, count.User) } - s.cfg.metricsWriter.SetMode(s.info.Mode) + s.metricsWriter.SetMode(s.info.Mode) } // incObjectCounter increment both physical and logical object // counters. func (s *Shard) incObjectCounter(cnrID cid.ID, isUser bool) { - s.cfg.metricsWriter.IncObjectCounter(physical) - s.cfg.metricsWriter.IncObjectCounter(logical) - s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), physical) - s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), logical) + s.metricsWriter.IncObjectCounter(physical) + s.metricsWriter.IncObjectCounter(logical) + s.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), physical) + s.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), logical) if isUser { - s.cfg.metricsWriter.IncObjectCounter(user) - s.cfg.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), user) + s.metricsWriter.IncObjectCounter(user) + s.metricsWriter.IncContainerObjectsCount(cnrID.EncodeToString(), user) } } func (s *Shard) decObjectCounterBy(typ string, v uint64) { if v > 0 { - s.cfg.metricsWriter.AddToObjectCounter(typ, -int(v)) + s.metricsWriter.AddToObjectCounter(typ, -int(v)) } } func (s *Shard) setObjectCounterBy(typ string, v uint64) { if v > 0 { - s.cfg.metricsWriter.SetObjectCounter(typ, v) + s.metricsWriter.SetObjectCounter(typ, v) } } func (s *Shard) decContainerObjectCounter(byCnr map[cid.ID]meta.ObjectCounters) { for cnrID, count := range byCnr { if count.Phy > 0 { - s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), physical, count.Phy) + s.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), physical, count.Phy) } if count.Logic > 0 { - s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), logical, count.Logic) + s.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), logical, count.Logic) } if count.User > 0 { - s.cfg.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), user, count.User) + s.metricsWriter.SubContainerObjectsCount(cnrID.EncodeToString(), user, count.User) } } } func (s *Shard) addToContainerSize(cnr string, size int64) { if size != 0 { - s.cfg.metricsWriter.AddToContainerSize(cnr, size) + s.metricsWriter.AddToContainerSize(cnr, size) } } func (s *Shard) addToPayloadSize(size int64) { if size != 0 { - s.cfg.metricsWriter.AddToPayloadSize(size) + s.metricsWriter.AddToPayloadSize(size) } } diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index 7469ea74e..f3ba9015e 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -220,7 +220,7 @@ func (s *Server) replicateTreeToNode(ctx context.Context, forest pilorama.Forest TreeId: treeID, Operation: &tree.LogMove{ ParentId: op.Parent, - Meta: op.Meta.Bytes(), + Meta: op.Bytes(), ChildId: op.Child, }, }, diff --git a/pkg/services/object/common/writer/common.go b/pkg/services/object/common/writer/common.go index 1998e9638..6593d3ca0 100644 --- a/pkg/services/object/common/writer/common.go +++ b/pkg/services/object/common/writer/common.go @@ -29,7 +29,7 @@ func (c *Config) NewNodeIterator(opts []placement.Option) *NodeIterator { } func (n *NodeIterator) ForEachNode(ctx context.Context, f func(context.Context, NodeDescriptor) error) error { - traverser, err := placement.NewTraverser(ctx, n.Traversal.Opts...) + traverser, err := placement.NewTraverser(ctx, n.Opts...) if err != nil { return fmt.Errorf("could not create object placement traverser: %w", err) } @@ -56,7 +56,7 @@ func (n *NodeIterator) ForEachNode(ctx context.Context, f func(context.Context, } // perform additional container broadcast if needed - if n.Traversal.submitPrimaryPlacementFinish() { + if n.submitPrimaryPlacementFinish() { err := n.ForEachNode(ctx, f) if err != nil { n.cfg.Logger.Error(ctx, logs.PutAdditionalContainerBroadcastFailure, zap.Error(err)) @@ -101,7 +101,7 @@ func (n *NodeIterator) forEachAddress(ctx context.Context, traverser *placement. // in subsequent container broadcast. Note that we don't // process this node during broadcast if primary placement // on it failed. - n.Traversal.submitProcessed(addr, item) + n.submitProcessed(addr, item) } wg.Wait() diff --git a/pkg/services/object/get/assemble.go b/pkg/services/object/get/assemble.go index e164627d2..e80132489 100644 --- a/pkg/services/object/get/assemble.go +++ b/pkg/services/object/get/assemble.go @@ -146,5 +146,5 @@ func (r *request) getObjectWithIndependentRequest(ctx context.Context, prm Reque detachedExecutor.execute(ctx) - return detachedExecutor.statusError.err + return detachedExecutor.err } diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 557e9a028..9676fd914 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -87,7 +87,7 @@ func (s *Service) get(ctx context.Context, prm RequestParameters) error { exec.execute(ctx) - return exec.statusError.err + return exec.err } func (exec *request) execute(ctx context.Context) { diff --git a/pkg/services/object/get/v2/streamer.go b/pkg/services/object/get/v2/streamer.go index 98207336c..0d73bcd4d 100644 --- a/pkg/services/object/get/v2/streamer.go +++ b/pkg/services/object/get/v2/streamer.go @@ -24,14 +24,14 @@ func (s *streamObjectWriter) WriteHeader(_ context.Context, obj *objectSDK.Objec p.SetHeader(objV2.GetHeader()) p.SetSignature(objV2.GetSignature()) - return s.GetObjectStream.Send(newResponse(p)) + return s.Send(newResponse(p)) } func (s *streamObjectWriter) WriteChunk(_ context.Context, chunk []byte) error { p := new(objectV2.GetObjectPartChunk) p.SetChunk(chunk) - return s.GetObjectStream.Send(newResponse(p)) + return s.Send(newResponse(p)) } func newResponse(p objectV2.GetObjectPart) *objectV2.GetResponse { @@ -46,7 +46,7 @@ func newResponse(p objectV2.GetObjectPart) *objectV2.GetResponse { } func (s *streamObjectRangeWriter) WriteChunk(_ context.Context, chunk []byte) error { - return s.GetObjectRangeStream.Send(newRangeResponse(chunk)) + return s.Send(newRangeResponse(chunk)) } func newRangeResponse(p []byte) *objectV2.GetRangeResponse { diff --git a/pkg/services/object/patch/service.go b/pkg/services/object/patch/service.go index 953f82b48..5d298bfed 100644 --- a/pkg/services/object/patch/service.go +++ b/pkg/services/object/patch/service.go @@ -28,7 +28,7 @@ func NewService(cfg *objectwriter.Config, // Patch calls internal service and returns v2 object streamer. func (s *Service) Patch() (object.PatchObjectStream, error) { - nodeKey, err := s.Config.KeyStorage.GetKey(nil) + nodeKey, err := s.KeyStorage.GetKey(nil) if err != nil { return nil, err } diff --git a/pkg/services/object/put/single.go b/pkg/services/object/put/single.go index f3c2dca1a..90f473254 100644 --- a/pkg/services/object/put/single.go +++ b/pkg/services/object/put/single.go @@ -102,7 +102,7 @@ func (s *Service) validarePutSingleSize(ctx context.Context, obj *objectSDK.Obje return target.ErrWrongPayloadSize } - maxAllowedSize := s.Config.MaxSizeSrc.MaxObjectSize(ctx) + maxAllowedSize := s.MaxSizeSrc.MaxObjectSize(ctx) if obj.PayloadSize() > maxAllowedSize { return target.ErrExceedingMaxSize } @@ -166,13 +166,13 @@ func (s *Service) saveToNodes(ctx context.Context, obj *objectSDK.Object, req *o } func (s *Service) saveToREPReplicas(ctx context.Context, placement putSinglePlacement, obj *objectSDK.Object, localOnly bool, req *objectAPI.PutSingleRequest, meta object.ContentMeta) error { - iter := s.Config.NewNodeIterator(placement.placementOptions) + iter := s.NewNodeIterator(placement.placementOptions) iter.ExtraBroadcastEnabled = objectwriter.NeedAdditionalBroadcast(obj, localOnly) iter.ResetSuccessAfterOnBroadcast = placement.resetSuccessAfterOnBroadcast signer := &putSingleRequestSigner{ req: req, - keyStorage: s.Config.KeyStorage, + keyStorage: s.KeyStorage, signer: &sync.Once{}, } @@ -186,13 +186,13 @@ func (s *Service) saveToECReplicas(ctx context.Context, placement putSinglePlace if err != nil { return err } - key, err := s.Config.KeyStorage.GetKey(nil) + key, err := s.KeyStorage.GetKey(nil) if err != nil { return err } signer := &putSingleRequestSigner{ req: req, - keyStorage: s.Config.KeyStorage, + keyStorage: s.KeyStorage, signer: &sync.Once{}, } @@ -225,7 +225,7 @@ func (s *Service) getPutSinglePlacementOptions(ctx context.Context, obj *objectS if !ok { return result, errors.New("missing container ID") } - cnrInfo, err := s.Config.ContainerSource.Get(ctx, cnrID) + cnrInfo, err := s.ContainerSource.Get(ctx, cnrID) if err != nil { return result, fmt.Errorf("could not get container by ID: %w", err) } @@ -249,14 +249,14 @@ func (s *Service) getPutSinglePlacementOptions(ctx context.Context, obj *objectS } result.placementOptions = append(result.placementOptions, placement.ForObject(objID)) - latestNetmap, err := netmap.GetLatestNetworkMap(ctx, s.Config.NetmapSource) + latestNetmap, err := netmap.GetLatestNetworkMap(ctx, s.NetmapSource) if err != nil { return result, fmt.Errorf("could not get latest network map: %w", err) } builder := placement.NewNetworkMapBuilder(latestNetmap) if localOnly { result.placementOptions = append(result.placementOptions, placement.SuccessAfter(1)) - builder = svcutil.NewLocalPlacement(builder, s.Config.NetmapKeys) + builder = svcutil.NewLocalPlacement(builder, s.NetmapKeys) } result.placementOptions = append(result.placementOptions, placement.UseBuilder(builder)) return result, nil @@ -273,7 +273,7 @@ func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *objectwrite client.NodeInfoFromNetmapElement(&info, nodeDesc.Info) - c, err := s.Config.ClientConstructor.Get(info) + c, err := s.ClientConstructor.Get(info) if err != nil { return fmt.Errorf("could not create SDK client %s: %w", info.AddressGroup(), err) } @@ -283,7 +283,7 @@ func (s *Service) saveToPlacementNode(ctx context.Context, nodeDesc *objectwrite func (s *Service) saveLocal(ctx context.Context, obj *objectSDK.Object, meta object.ContentMeta, container containerSDK.Container) error { localTarget := &objectwriter.LocalTarget{ - Storage: s.Config.LocalStore, + Storage: s.LocalStore, Container: container, } return localTarget.WriteObject(ctx, obj, meta) @@ -317,7 +317,7 @@ func (s *Service) redirectPutSingleRequest(ctx context.Context, if err != nil { objID, _ := obj.ID() cnrID, _ := obj.ContainerID() - s.Config.Logger.Warn(ctx, logs.PutSingleRedirectFailure, + s.Logger.Warn(ctx, logs.PutSingleRedirectFailure, zap.Error(err), zap.Stringer("address", addr), zap.Stringer("object_id", objID), diff --git a/pkg/services/policer/ec.go b/pkg/services/policer/ec.go index 1ee31d480..fbdeb3148 100644 --- a/pkg/services/policer/ec.go +++ b/pkg/services/policer/ec.go @@ -101,7 +101,7 @@ func (p *Policer) processECContainerECObject(ctx context.Context, objInfo object func (p *Policer) processECChunk(ctx context.Context, objInfo objectcore.Info, nodes []netmap.NodeInfo) ecChunkProcessResult { var removeLocalChunk bool requiredNode := nodes[int(objInfo.ECInfo.Index)%(len(nodes))] - if p.cfg.netmapKeys.IsLocalKey(requiredNode.PublicKey()) { + if p.netmapKeys.IsLocalKey(requiredNode.PublicKey()) { // current node is required node, we are happy return ecChunkProcessResult{ validPlacement: true, @@ -185,7 +185,7 @@ func (p *Policer) collectRequiredECChunks(nodes []netmap.NodeInfo, objInfo objec if uint32(i) == objInfo.ECInfo.Total { break } - if p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) { + if p.netmapKeys.IsLocalKey(n.PublicKey()) { requiredChunkIndexes[uint32(i)] = []netmap.NodeInfo{} } } @@ -210,7 +210,7 @@ func (p *Policer) resolveLocalECChunks(ctx context.Context, parentAddress oid.Ad func (p *Policer) resolveRemoteECChunks(ctx context.Context, parentAddress oid.Address, nodes []netmap.NodeInfo, required map[uint32][]netmap.NodeInfo, indexToObjectID map[uint32]oid.ID) bool { var eiErr *objectSDK.ECInfoError for _, n := range nodes { - if p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) { + if p.netmapKeys.IsLocalKey(n.PublicKey()) { continue } _, err := p.remoteHeader(ctx, n, parentAddress, true) @@ -260,7 +260,7 @@ func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info return } var err error - if p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) { + if p.netmapKeys.IsLocalKey(n.PublicKey()) { _, err = p.localHeader(ctx, parentAddress) } else { _, err = p.remoteHeader(ctx, n, parentAddress, true) @@ -283,7 +283,7 @@ func (p *Policer) adjustECPlacement(ctx context.Context, objInfo objectcore.Info } } else if client.IsErrObjectAlreadyRemoved(err) { restore = false - } else if !p.cfg.netmapKeys.IsLocalKey(n.PublicKey()) && uint32(idx) < objInfo.ECInfo.Total { + } else if !p.netmapKeys.IsLocalKey(n.PublicKey()) && uint32(idx) < objInfo.ECInfo.Total { p.log.Warn(ctx, logs.PolicerCouldNotGetObjectFromNodeMoving, zap.String("node", hex.EncodeToString(n.PublicKey())), zap.Stringer("object", parentAddress), zap.Error(err)) p.replicator.HandleReplicationTask(ctx, replicator.Task{ NumCopies: 1, @@ -343,7 +343,7 @@ func (p *Policer) restoreECObject(ctx context.Context, objInfo objectcore.Info, pID, _ := part.ID() addr.SetObject(pID) targetNode := nodes[idx%len(nodes)] - if p.cfg.netmapKeys.IsLocalKey(targetNode.PublicKey()) { + if p.netmapKeys.IsLocalKey(targetNode.PublicKey()) { p.replicator.HandleLocalPutTask(ctx, replicator.Task{ Addr: addr, Obj: part, @@ -371,7 +371,7 @@ func (p *Policer) collectExistedChunks(ctx context.Context, objInfo objectcore.I var obj *objectSDK.Object var err error for _, node := range nodes { - if p.cfg.netmapKeys.IsLocalKey(node.PublicKey()) { + if p.netmapKeys.IsLocalKey(node.PublicKey()) { obj, err = p.localObject(egCtx, objID) } else { obj, err = p.remoteObject(egCtx, node, objID) diff --git a/pkg/services/session/storage/persistent/storage.go b/pkg/services/session/storage/persistent/storage.go index d312ea0ea..132d62445 100644 --- a/pkg/services/session/storage/persistent/storage.go +++ b/pkg/services/session/storage/persistent/storage.go @@ -64,7 +64,7 @@ func NewTokenStore(path string, opts ...Option) (*TokenStore, error) { // enable encryption if it // was configured so if cfg.privateKey != nil { - rawKey := make([]byte, (cfg.privateKey.Curve.Params().N.BitLen()+7)/8) + rawKey := make([]byte, (cfg.privateKey.Params().N.BitLen()+7)/8) cfg.privateKey.D.FillBytes(rawKey) c, err := aes.NewCipher(rawKey) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index d250f577a..462c8554f 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -48,7 +48,7 @@ func (c *clientCache) init(pk *ecdsa.PrivateKey, ds *internalNet.DialerSource) { func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceClient, error) { c.Lock() - ccInt, ok := c.LRU.Get(netmapAddr) + ccInt, ok := c.Get(netmapAddr) c.Unlock() if ok { @@ -71,9 +71,9 @@ func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceCl c.Lock() if err != nil { - c.LRU.Add(netmapAddr, cacheItem{cc: nil, lastTry: lastTry}) + c.Add(netmapAddr, cacheItem{cc: nil, lastTry: lastTry}) } else { - c.LRU.Add(netmapAddr, cacheItem{cc: cc, lastTry: lastTry}) + c.Add(netmapAddr, cacheItem{cc: cc, lastTry: lastTry}) } c.Unlock() diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 164815c76..01a4ffde0 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -202,7 +202,7 @@ func newApplyRequest(op *movePair) *ApplyRequest { TreeId: op.treeID, Operation: &LogMove{ ParentId: op.op.Parent, - Meta: op.op.Meta.Bytes(), + Meta: op.op.Bytes(), ChildId: op.op.Child, }, }, diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 85bb03a28..a00c8c1cd 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -687,7 +687,7 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) Body: &GetOpLogResponse_Body{ Operation: &LogMove{ ParentId: lm.Parent, - Meta: lm.Meta.Bytes(), + Meta: lm.Bytes(), ChildId: lm.Child, }, }, diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 8abdafaa7..c3796fbd4 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -245,7 +245,7 @@ func (s *Service) startStream(ctx context.Context, cid cid.ID, treeID string, Parent: lm.GetParentId(), Child: lm.GetChildId(), } - if err := m.Meta.FromBytes(lm.GetMeta()); err != nil { + if err := m.FromBytes(lm.GetMeta()); err != nil { return err } select { @@ -415,7 +415,7 @@ func (s *Service) syncLoop(ctx context.Context) { start := time.Now() - cnrs, err := s.cfg.cnrSource.List(ctx) + cnrs, err := s.cnrSource.List(ctx) if err != nil { s.log.Error(ctx, logs.TreeCouldNotFetchContainers, zap.Error(err)) s.metrics.AddSyncDuration(time.Since(start), false) From b0ef737a74d9e0b789be24db0830fa927e7078e0 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 16:58:21 +0300 Subject: [PATCH 1873/1943] [#1689] linter: Fix testifylint warning: 'len: use require.Len' Change-Id: I7a08f09c169ac237647dcb20b0737f1c51c441ad Signed-off-by: Alexander Chuprov --- .../blobstor/internal/blobstortest/iterate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go index c11d0888b..d54c54f59 100644 --- a/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go +++ b/pkg/local_object_storage/blobstor/internal/blobstortest/iterate.go @@ -50,7 +50,7 @@ func runTestNormalHandler(t *testing.T, s common.Storage, objects []objectDesc) _, err := s.Iterate(context.Background(), iterPrm) require.NoError(t, err) - require.Equal(t, len(objects), len(seen)) + require.Len(t, objects, len(seen)) for i := range objects { d, ok := seen[objects[i].addr.String()] require.True(t, ok) From f4696e8964786dd5489fdce1456eccab010d02b8 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:03:19 +0300 Subject: [PATCH 1874/1943] [#1689] linter: Fix staticcheck warning: 'Use fmt.Fprintf(...) instead of WriteString(fmt.Sprintf(...))' Change-Id: I253ab717885cb01b4a2e471147e883ee351be277 Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/modules/control/evacuation.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-cli/modules/control/evacuation.go b/cmd/frostfs-cli/modules/control/evacuation.go index 8032bf09a..b8d7eb046 100644 --- a/cmd/frostfs-cli/modules/control/evacuation.go +++ b/cmd/frostfs-cli/modules/control/evacuation.go @@ -296,7 +296,7 @@ func appendEstimation(sb *strings.Builder, resp *control.GetShardEvacuationStatu leftSeconds := avgObjEvacuationTimeSeconds * objectsLeft leftMinutes := int(leftSeconds / 60) - sb.WriteString(fmt.Sprintf(" Estimated time left: %d minutes.", leftMinutes)) + fmt.Fprintf(sb, " Estimated time left: %d minutes.", leftMinutes) } func appendDuration(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { @@ -305,20 +305,20 @@ func appendDuration(sb *strings.Builder, resp *control.GetShardEvacuationStatusR hour := int(duration.Seconds() / 3600) minute := int(duration.Seconds()/60) % 60 second := int(duration.Seconds()) % 60 - sb.WriteString(fmt.Sprintf(" Duration: %02d:%02d:%02d.", hour, minute, second)) + fmt.Fprintf(sb, " Duration: %02d:%02d:%02d.", hour, minute, second) } } func appendStartedAt(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { if resp.GetBody().GetStartedAt() != nil { startedAt := time.Unix(resp.GetBody().GetStartedAt().GetValue(), 0).UTC() - sb.WriteString(fmt.Sprintf(" Started at: %s UTC.", startedAt.Format(time.RFC3339))) + fmt.Fprintf(sb, " Started at: %s UTC.", startedAt.Format(time.RFC3339)) } } func appendError(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { if len(resp.GetBody().GetErrorMessage()) > 0 { - sb.WriteString(fmt.Sprintf(" Error: %s.", resp.GetBody().GetErrorMessage())) + fmt.Fprintf(sb, " Error: %s.", resp.GetBody().GetErrorMessage()) } } @@ -332,7 +332,7 @@ func appendStatus(sb *strings.Builder, resp *control.GetShardEvacuationStatusRes default: status = "undefined" } - sb.WriteString(fmt.Sprintf(" Status: %s.", status)) + fmt.Fprintf(sb, " Status: %s.", status) } func appendShardIDs(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { @@ -350,14 +350,14 @@ func appendShardIDs(sb *strings.Builder, resp *control.GetShardEvacuationStatusR } func appendCounts(sb *strings.Builder, resp *control.GetShardEvacuationStatusResponse) { - sb.WriteString(fmt.Sprintf(" Evacuated %d objects out of %d, failed to evacuate: %d, skipped: %d; evacuated %d trees out of %d, failed to evacuate: %d.", + fmt.Fprintf(sb, " Evacuated %d objects out of %d, failed to evacuate: %d, skipped: %d; evacuated %d trees out of %d, failed to evacuate: %d.", resp.GetBody().GetEvacuatedObjects(), resp.GetBody().GetTotalObjects(), resp.GetBody().GetFailedObjects(), resp.GetBody().GetSkippedObjects(), resp.GetBody().GetEvacuatedTrees(), resp.GetBody().GetTotalTrees(), - resp.GetBody().GetFailedTrees())) + resp.GetBody().GetFailedTrees()) } func initControlEvacuationShardCmd() { From 4c03561aa20c2bbc2e311c9438fd7979ff1d96db Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 7 Apr 2025 18:48:36 +0300 Subject: [PATCH 1875/1943] [#1693] internal/assert: Add `False` and `NoError` checks Change-Id: Ib3ab1671eeff8e8917673513477f158cadbb4287 Signed-off-by: Ekaterina Lebedeva --- internal/assert/cond.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/assert/cond.go b/internal/assert/cond.go index 701036fa8..c6a034f94 100644 --- a/internal/assert/cond.go +++ b/internal/assert/cond.go @@ -1,9 +1,25 @@ package assert -import "strings" +import ( + "fmt" + "strings" +) func True(cond bool, details ...string) { if !cond { panic(strings.Join(details, " ")) } } + +func False(cond bool, details ...string) { + if cond { + panic(strings.Join(details, " ")) + } +} + +func NoError(err error, details ...string) { + if err != nil { + content := fmt.Sprintf("BUG: %v: %s", err, strings.Join(details, " ")) + panic(content) + } +} From 0e1b01b15ff21c07d809dbed00f23c6b2377bf4e Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Wed, 2 Apr 2025 19:06:10 +0300 Subject: [PATCH 1876/1943] [#1693] cli/adm: Replace conditional panics with asserts Change-Id: I3a46f7ac6d9e4ff51bb490e6fcfc07957418f1a7 Signed-off-by: Ekaterina Lebedeva --- .../internal/modules/morph/balance/balance.go | 9 +++----- .../modules/morph/container/container.go | 5 ++-- .../internal/modules/morph/contract/deploy.go | 10 ++++---- .../modules/morph/contract/dump_hashes.go | 9 +++----- .../modules/morph/helper/initialize_ctx.go | 23 ++++++++----------- .../modules/morph/helper/local_client.go | 9 +++----- .../morph/initialize/initialize_nns.go | 5 ++-- .../morph/initialize/initialize_register.go | 5 ++-- 8 files changed, 28 insertions(+), 47 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go index be42f2aa5..23dba14f4 100644 --- a/cmd/frostfs-adm/internal/modules/morph/balance/balance.go +++ b/cmd/frostfs-adm/internal/modules/morph/balance/balance.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -161,9 +162,7 @@ func printAlphabetContractBalances(cmd *cobra.Command, c helper.Client, inv *inv helper.GetAlphabetNNSDomain(i), int64(nns.TXT)) } - if w.Err != nil { - panic(w.Err) - } + assert.NoError(w.Err) alphaRes, err := c.InvokeScript(w.Bytes(), nil) if err != nil { @@ -226,9 +225,7 @@ func fetchBalances(c *invoker.Invoker, gasHash util.Uint160, accounts []accBalan for i := range accounts { emit.AppCall(w.BinWriter, gasHash, "balanceOf", callflag.ReadStates, accounts[i].scriptHash) } - if w.Err != nil { - panic(w.Err) - } + assert.NoError(w.Err) res, err := c.Run(w.Bytes()) if err != nil || res.State != vmstate.Halt.String() || len(res.Stack) != len(accounts) { diff --git a/cmd/frostfs-adm/internal/modules/morph/container/container.go b/cmd/frostfs-adm/internal/modules/morph/container/container.go index e72dc15e9..79685f111 100644 --- a/cmd/frostfs-adm/internal/modules/morph/container/container.go +++ b/cmd/frostfs-adm/internal/modules/morph/container/container.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" "github.com/nspcc-dev/neo-go/pkg/io" @@ -235,9 +236,7 @@ func restoreOrPutContainers(containers []Container, isOK func([]byte) bool, cmd putContainer(bw, ch, cnt) - if bw.Err != nil { - panic(bw.Err) - } + assert.NoError(bw.Err) if err := wCtx.SendConsensusTx(bw.Bytes()); err != nil { return err diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go index 5adb480da..543b5fcb3 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/deploy.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "github.com/nspcc-dev/neo-go/cli/cmdargs" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/encoding/address" @@ -120,9 +121,7 @@ func deployContractCmd(cmd *cobra.Command, args []string) error { } } - if writer.Err != nil { - panic(fmt.Errorf("BUG: can't create deployment script: %w", writer.Err)) - } + assert.NoError(writer.Err, "can't create deployment script") if err := c.SendCommitteeTx(writer.Bytes(), false); err != nil { return err @@ -173,9 +172,8 @@ func registerNNS(nnsCs *state.Contract, c *helper.InitializeContext, zone string domain, int64(nns.TXT), address.Uint160ToString(cs.Hash)) } - if bw.Err != nil { - panic(fmt.Errorf("BUG: can't create deployment script: %w", writer.Err)) - } else if bw.Len() != start { + assert.NoError(bw.Err, "can't create deployment script") + if bw.Len() != start { writer.WriteBytes(bw.Bytes()) emit.Opcodes(writer.BinWriter, opcode.LDSFLD0, opcode.PUSH1, opcode.PACK) emit.AppCallNoArgs(writer.BinWriter, nnsCs.Hash, "setPrice", callflag.All) diff --git a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go index 7630a226e..fde58fd2b 100644 --- a/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go +++ b/cmd/frostfs-adm/internal/modules/morph/contract/dump_hashes.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/io" "github.com/nspcc-dev/neo-go/pkg/rpcclient/invoker" @@ -236,9 +237,7 @@ func fillContractVersion(cmd *cobra.Command, c helper.Client, infos []contractDu } else { sub.Reset() emit.AppCall(sub.BinWriter, infos[i].hash, "version", callflag.NoneFlag) - if sub.Err != nil { - panic(fmt.Errorf("BUG: can't create version script: %w", bw.Err)) - } + assert.NoError(sub.Err, "can't create version script") script := sub.Bytes() emit.Instruction(bw.BinWriter, opcode.TRY, []byte{byte(3 + len(script) + 2), 0}) @@ -248,9 +247,7 @@ func fillContractVersion(cmd *cobra.Command, c helper.Client, infos []contractDu } } emit.Opcodes(bw.BinWriter, opcode.NOP) // for the last ENDTRY target - if bw.Err != nil { - panic(fmt.Errorf("BUG: can't create version script: %w", bw.Err)) - } + assert.NoError(bw.Err, "can't create version script") res, err := c.InvokeScript(bw.Bytes(), nil) if err != nil { diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go index 27052697f..da5ffedae 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/initialize_ctx.go @@ -13,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -375,9 +376,7 @@ func (c *InitializeContext) sendMultiTx(script []byte, tryGroup bool, withConsen } act, err = actor.New(c.Client, signers) } else { - if withConsensus { - panic("BUG: should never happen") - } + assert.False(withConsensus, "BUG: should never happen") act, err = c.CommitteeAct, nil } if err != nil { @@ -411,11 +410,9 @@ func (c *InitializeContext) MultiSignAndSend(tx *transaction.Transaction, accTyp func (c *InitializeContext) MultiSign(tx *transaction.Transaction, accType string) error { version, err := c.Client.GetVersion() - if err != nil { - // error appears only if client - // has not been initialized - panic(err) - } + // error appears only if client + // has not been initialized + assert.NoError(err) network := version.Protocol.Network // Use parameter context to avoid dealing with signature order. @@ -447,12 +444,12 @@ func (c *InitializeContext) MultiSign(tx *transaction.Transaction, accType strin for i := range tx.Signers { if tx.Signers[i].Account == h { + assert.True(i <= len(tx.Scripts), "BUG: invalid signing order") if i < len(tx.Scripts) { tx.Scripts[i] = *w - } else if i == len(tx.Scripts) { + } + if i == len(tx.Scripts) { tx.Scripts = append(tx.Scripts, *w) - } else { - panic("BUG: invalid signing order") } return nil } @@ -510,9 +507,7 @@ func (c *InitializeContext) NNSRegisterDomainScript(nnsHash, expectedHash util.U int64(constants.DefaultExpirationTime), constants.NNSTtlDefVal) emit.Opcodes(bw.BinWriter, opcode.ASSERT) - if bw.Err != nil { - panic(bw.Err) - } + assert.NoError(bw.Err) return bw.Bytes(), false, nil } diff --git a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go index d0a05d5c7..46611c177 100644 --- a/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go +++ b/cmd/frostfs-adm/internal/modules/morph/helper/local_client.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/config" "github.com/nspcc-dev/neo-go/pkg/core" @@ -316,9 +317,7 @@ func (l *LocalClient) SendRawTransaction(tx *transaction.Transaction) (util.Uint func (l *LocalClient) putTransactions() error { // 1. Prepare new block. lastBlock, err := l.bc.GetBlock(l.bc.CurrentBlockHash()) - if err != nil { - panic(err) - } + assert.NoError(err) defer func() { l.transactions = l.transactions[:0] }() b := &block.Block{ @@ -359,9 +358,7 @@ func InvokeFunction(c Client, h util.Uint160, method string, parameters []any, s w := io.NewBufBinWriter() emit.Array(w.BinWriter, parameters...) emit.AppCallNoArgs(w.BinWriter, h, method, callflag.All) - if w.Err != nil { - panic(fmt.Sprintf("BUG: invalid parameters for '%s': %v", method, w.Err)) - } + assert.True(w.Err == nil, fmt.Sprintf("BUG: invalid parameters for '%s': %v", method, w.Err)) return c.InvokeScript(w.Bytes(), signers) } diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go index e127ca545..176356378 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_nns.go @@ -7,6 +7,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" morphClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -111,9 +112,7 @@ func wrapRegisterScriptWithPrice(w *io.BufBinWriter, nnsHash util.Uint160, s []b emit.Opcodes(w.BinWriter, opcode.LDSFLD0, opcode.PUSH1, opcode.PACK) emit.AppCallNoArgs(w.BinWriter, nnsHash, "setPrice", callflag.All) - if w.Err != nil { - panic(fmt.Errorf("BUG: can't wrap register script: %w", w.Err)) - } + assert.NoError(w.Err, "can't wrap register script") } func nnsRegisterDomain(c *helper.InitializeContext, nnsHash, expectedHash util.Uint160, domain string) error { diff --git a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go index 46e6621d2..7b7597d91 100644 --- a/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go +++ b/cmd/frostfs-adm/internal/modules/morph/initialize/initialize_register.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/constants" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph/helper" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "github.com/nspcc-dev/neo-go/pkg/core/native" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -39,9 +40,7 @@ func registerCandidateRange(c *helper.InitializeContext, start, end int) error { emit.Opcodes(w.BinWriter, opcode.ASSERT) } emit.AppCall(w.BinWriter, neo.Hash, "setRegisterPrice", callflag.States, regPrice) - if w.Err != nil { - panic(fmt.Sprintf("BUG: %v", w.Err)) - } + assert.NoError(w.Err) signers := []actor.SignerAccount{{ Signer: c.GetSigner(false, c.CommitteeAcc), From 766d9ec46b7b7da10dbbd4bf257db6f5f799a968 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 7 Apr 2025 19:25:57 +0300 Subject: [PATCH 1877/1943] [#1693] cli/lens: Replace conditional panics with asserts Change-Id: Id827da0cd9eef66efd806be6c9bc61044175a971 Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-lens/internal/schema/common/schema.go | 8 +++----- cmd/frostfs-lens/internal/tui/records.go | 5 ++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-lens/internal/schema/common/schema.go b/cmd/frostfs-lens/internal/schema/common/schema.go index 9bad19032..077a68785 100644 --- a/cmd/frostfs-lens/internal/schema/common/schema.go +++ b/cmd/frostfs-lens/internal/schema/common/schema.go @@ -3,6 +3,8 @@ package common import ( "errors" "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" ) type FilterResult byte @@ -71,11 +73,7 @@ func (fp FallbackParser) ToParser() Parser { func (p Parser) ToFallbackParser() FallbackParser { return func(key, value []byte) (SchemaEntry, Parser) { entry, next, err := p(key, value) - if err != nil { - panic(fmt.Errorf( - "couldn't use that parser as a fallback parser, it returned an error: %w", err, - )) - } + assert.NoError(err, "couldn't use that parser as a fallback parser") return entry, next } } diff --git a/cmd/frostfs-lens/internal/tui/records.go b/cmd/frostfs-lens/internal/tui/records.go index 5f61df884..a4d392ab3 100644 --- a/cmd/frostfs-lens/internal/tui/records.go +++ b/cmd/frostfs-lens/internal/tui/records.go @@ -8,6 +8,7 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal/schema/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "github.com/gdamore/tcell/v2" "github.com/rivo/tview" ) @@ -94,9 +95,7 @@ func (v *RecordsView) Mount(ctx context.Context) error { } func (v *RecordsView) Unmount() { - if v.onUnmount == nil { - panic("try to unmount not mounted component") - } + assert.False(v.onUnmount == nil, "try to unmount not mounted component") v.onUnmount() v.onUnmount = nil } From 17cba3387e926c50ae47a224af47f3f6668134f4 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 8 Apr 2025 19:57:43 +0300 Subject: [PATCH 1878/1943] [#1332] cli/playground: Prevent prompt artifacts by writing to 'readline' stdout Change-Id: I1c3cbb0b762f29c0995d3f6fc79bae5246ee7bc3 Signed-off-by: Alexander Chuprov --- .../modules/container/policy_playground.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index dcd755510..825e1d222 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -19,8 +19,9 @@ import ( ) type policyPlaygroundREPL struct { - cmd *cobra.Command - nodes map[string]netmap.NodeInfo + cmd *cobra.Command + nodes map[string]netmap.NodeInfo + console *readline.Instance } func newPolicyPlaygroundREPL(cmd *cobra.Command) *policyPlaygroundREPL { @@ -40,7 +41,7 @@ func (repl *policyPlaygroundREPL) handleLs(args []string) error { node.IterateAttributes(func(k, v string) { attrs = append(attrs, fmt.Sprintf("%s:%q", k, v)) }) - fmt.Printf("\t%2d: id=%s attrs={%v}\n", i, id, strings.Join(attrs, " ")) + fmt.Fprintf(repl.console, "\t%2d: id=%s attrs={%v}\n", i, id, strings.Join(attrs, " ")) i++ } return nil @@ -147,7 +148,7 @@ func (repl *policyPlaygroundREPL) handleEval(args []string) error { for _, node := range ns { ids = append(ids, hex.EncodeToString(node.PublicKey())) } - fmt.Printf("\t%2d: %v\n", i+1, ids) + fmt.Fprintf(repl.console, "\t%2d: %v\n", i+1, ids) } return nil @@ -208,6 +209,7 @@ func (repl *policyPlaygroundREPL) run() error { if err != nil { return fmt.Errorf("error initializing readline: %w", err) } + repl.console = rl defer rl.Close() var exit bool @@ -232,10 +234,10 @@ func (repl *policyPlaygroundREPL) run() error { cmd := parts[0] if handler, exists := cmdHandlers[cmd]; exists { if err := handler(parts[1:]); err != nil { - fmt.Printf("error: %v\n", err) + fmt.Fprintf(repl.console, "error: %v\n", err) } } else { - fmt.Printf("error: unknown command %q\n", cmd) + fmt.Fprintf(repl.console, "error: unknown command %q\n", cmd) } } } From 46fd5e17b2366c22dba259d1349c6e3fa8662cb6 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 8 Apr 2025 20:23:22 +0300 Subject: [PATCH 1879/1943] [#1332] cli/playground: Add help Change-Id: I6160cfddf427b161619e4b96ceec8396b75c4d08 Signed-off-by: Alexander Chuprov --- .../modules/container/policy_playground.go | 119 ++++++++++++++++-- 1 file changed, 109 insertions(+), 10 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 825e1d222..f747b3252 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -5,7 +5,9 @@ import ( "encoding/json" "errors" "fmt" + "maps" "os" + "slices" "strings" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" @@ -154,6 +156,23 @@ func (repl *policyPlaygroundREPL) handleEval(args []string) error { return nil } +func (repl *policyPlaygroundREPL) handleHelp(args []string) error { + if len(args) != 0 { + if _, ok := commands[args[0]]; !ok { + return fmt.Errorf("unknown command: %q", args[0]) + } + fmt.Fprintln(repl.console, commands[args[0]].usage) + return nil + } + + commandList := slices.Collect(maps.Keys(commands)) + slices.Sort(commandList) + for _, command := range commandList { + fmt.Fprintf(repl.console, "%s: %s\n", command, commands[command].descriprion) + } + return nil +} + func (repl *policyPlaygroundREPL) netMap() netmap.NetMap { var nm netmap.NetMap var nodes []netmap.NodeInfo @@ -164,15 +183,82 @@ func (repl *policyPlaygroundREPL) netMap() netmap.NetMap { return nm } -var policyPlaygroundCompleter = readline.NewPrefixCompleter( - readline.PcItem("list"), - readline.PcItem("ls"), - readline.PcItem("add"), - readline.PcItem("load"), - readline.PcItem("remove"), - readline.PcItem("rm"), - readline.PcItem("eval"), -) +type commandDescription struct { + descriprion string + usage string +} + +var commands = map[string]commandDescription{ + "list": { + descriprion: "Display all nodes in the netmap", + usage: `Display all nodes in the netmap +Example of usage: + list + 1: id=03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae attrs={Continent:"Europe" Country:"Poland"} + 2: id=02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3 attrs={Continent:"Antarctica" Country:"Heard Island"} +`, + }, + + "ls": { + descriprion: "Display all nodes in the netmap", + usage: `Display all nodes in the netmap +Example of usage: + ls + 1: id=03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae attrs={Continent:"Europe" Country:"Poland"} + 2: id=02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3 attrs={Continent:"Antarctica" Country:"Heard Island"} +`, + }, + + "add": { + descriprion: "Add a new node: add attr=value", + usage: `Add a new node +Example of usage: + add 03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae continent:Europe country:Poland`, + }, + + "load": { + descriprion: "Load netmap from file: load ", + usage: `Load netmap from file +Example of usage: + load "netmap.json" +File format (netmap.json): +{ + "03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae": { + "continent": "Europe", + "country": "Poland" + }, + "02ac920cd7df0b61b289072e6b946e2da4e1a31b9ab1c621bb475e30fa4ab102c3": { + "continent": "Antarctica", + "country": "Heard Island" + } +}`, + }, + + "remove": { + descriprion: "Remove a node: remove ", + usage: `Remove a node +Example of usage: + remove 03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae`, + }, + + "rm": { + descriprion: "Remove a node: rm ", + usage: `Remove a node +Example of usage: + rm 03ff65b6ae79134a4dce9d0d39d3851e9bab4ee97abf86e81e1c5bbc50cd2826ae`, + }, + + "eval": { + descriprion: "Evaluate a policy: eval ", + usage: `Evaluate a policy +Example of usage: + eval REP 2`, + }, + + "help": { + descriprion: "Show available commands", + }, +} func (repl *policyPlaygroundREPL) run() error { if len(viper.GetString(commonflags.RPC)) > 0 { @@ -199,12 +285,25 @@ func (repl *policyPlaygroundREPL) run() error { "remove": repl.handleRemove, "rm": repl.handleRemove, "eval": repl.handleEval, + "help": repl.handleHelp, } + var cfgCompleter []readline.PrefixCompleterInterface + var helpSubItems []readline.PrefixCompleterInterface + + for name := range commands { + if name != "help" { + cfgCompleter = append(cfgCompleter, readline.PcItem(name)) + helpSubItems = append(helpSubItems, readline.PcItem(name)) + } + } + + cfgCompleter = append(cfgCompleter, readline.PcItem("help", helpSubItems...)) + completer := readline.NewPrefixCompleter(cfgCompleter...) rl, err := readline.NewEx(&readline.Config{ Prompt: "> ", InterruptPrompt: "^C", - AutoComplete: policyPlaygroundCompleter, + AutoComplete: completer, }) if err != nil { return fmt.Errorf("error initializing readline: %w", err) From faec499b38a8ff06bc5c38fa31b4e75139d09d7d Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:19:34 +0300 Subject: [PATCH 1880/1943] [#1689] linter: Fix staticcheck warning: 'variable naming format' Change-Id: I8f8b63a6a5f9b6feb7c91f70fe8ac092575b145c Signed-off-by: Alexander Chuprov --- .../blobstor/blobovniczatree/manager.go | 26 +++++++++---------- .../blobstor/blobovniczatree/rebuild.go | 4 +-- pkg/local_object_storage/pilorama/inmemory.go | 4 +-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go index f2f9509ad..6438f715b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/manager.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/manager.go @@ -141,8 +141,8 @@ func (b *sharedDB) SystemPath() string { return b.path } -// levelDbManager stores pointers of the sharedDB's for the leaf directory of the blobovnicza tree. -type levelDbManager struct { +// levelDBManager stores pointers of the sharedDB's for the leaf directory of the blobovnicza tree. +type levelDBManager struct { dbMtx *sync.RWMutex databases map[uint64]*sharedDB @@ -157,8 +157,8 @@ type levelDbManager struct { func newLevelDBManager(options []blobovnicza.Option, rootPath string, lvlPath string, readOnly bool, metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlag *atomic.Bool, log *logger.Logger, -) *levelDbManager { - result := &levelDbManager{ +) *levelDBManager { + result := &levelDBManager{ databases: make(map[uint64]*sharedDB), dbMtx: &sync.RWMutex{}, @@ -173,7 +173,7 @@ func newLevelDBManager(options []blobovnicza.Option, rootPath string, lvlPath st return result } -func (m *levelDbManager) GetByIndex(idx uint64) *sharedDB { +func (m *levelDBManager) GetByIndex(idx uint64) *sharedDB { res := m.getDBIfExists(idx) if res != nil { return res @@ -181,14 +181,14 @@ func (m *levelDbManager) GetByIndex(idx uint64) *sharedDB { return m.getOrCreateDB(idx) } -func (m *levelDbManager) getDBIfExists(idx uint64) *sharedDB { +func (m *levelDBManager) getDBIfExists(idx uint64) *sharedDB { m.dbMtx.RLock() defer m.dbMtx.RUnlock() return m.databases[idx] } -func (m *levelDbManager) getOrCreateDB(idx uint64) *sharedDB { +func (m *levelDBManager) getOrCreateDB(idx uint64) *sharedDB { m.dbMtx.Lock() defer m.dbMtx.Unlock() @@ -202,7 +202,7 @@ func (m *levelDbManager) getOrCreateDB(idx uint64) *sharedDB { return db } -func (m *levelDbManager) hasAnyDB() bool { +func (m *levelDBManager) hasAnyDB() bool { m.dbMtx.RLock() defer m.dbMtx.RUnlock() @@ -213,7 +213,7 @@ func (m *levelDbManager) hasAnyDB() bool { // // The blobovnicza opens at the first request, closes after the last request. type dbManager struct { - levelToManager map[string]*levelDbManager + levelToManager map[string]*levelDBManager levelToManagerGuard *sync.RWMutex closedFlag *atomic.Bool dbCounter *openDBCounter @@ -231,7 +231,7 @@ func newDBManager(rootPath string, options []blobovnicza.Option, readOnly bool, options: options, readOnly: readOnly, metrics: metrics, - levelToManager: make(map[string]*levelDbManager), + levelToManager: make(map[string]*levelDBManager), levelToManagerGuard: &sync.RWMutex{}, log: log, closedFlag: &atomic.Bool{}, @@ -266,7 +266,7 @@ func (m *dbManager) Close() { m.dbCounter.WaitUntilAllClosed() } -func (m *dbManager) getLevelManager(lvlPath string) *levelDbManager { +func (m *dbManager) getLevelManager(lvlPath string) *levelDBManager { result := m.getLevelManagerIfExists(lvlPath) if result != nil { return result @@ -274,14 +274,14 @@ func (m *dbManager) getLevelManager(lvlPath string) *levelDbManager { return m.getOrCreateLevelManager(lvlPath) } -func (m *dbManager) getLevelManagerIfExists(lvlPath string) *levelDbManager { +func (m *dbManager) getLevelManagerIfExists(lvlPath string) *levelDBManager { m.levelToManagerGuard.RLock() defer m.levelToManagerGuard.RUnlock() return m.levelToManager[lvlPath] } -func (m *dbManager) getOrCreateLevelManager(lvlPath string) *levelDbManager { +func (m *dbManager) getOrCreateLevelManager(lvlPath string) *levelDBManager { m.levelToManagerGuard.Lock() defer m.levelToManagerGuard.Unlock() diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index 7ef3317fd..d2eef2074 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -328,7 +328,7 @@ func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobo return nil } -func (b *Blobovniczas) dropDB(ctx context.Context, path string, shDb *sharedDB) (bool, error) { +func (b *Blobovniczas) dropDB(ctx context.Context, path string, shDB *sharedDB) (bool, error) { select { case <-ctx.Done(): return false, ctx.Err() @@ -341,7 +341,7 @@ func (b *Blobovniczas) dropDB(ctx context.Context, path string, shDb *sharedDB) b.dbFilesGuard.Lock() defer b.dbFilesGuard.Unlock() - if err := shDb.CloseAndRemoveFile(ctx); err != nil { + if err := shDB.CloseAndRemoveFile(ctx); err != nil { return false, err } b.commondbManager.CleanResources(path) diff --git a/pkg/local_object_storage/pilorama/inmemory.go b/pkg/local_object_storage/pilorama/inmemory.go index f74d12a1b..28b7faec8 100644 --- a/pkg/local_object_storage/pilorama/inmemory.go +++ b/pkg/local_object_storage/pilorama/inmemory.go @@ -192,7 +192,7 @@ func (t tree) getByPath(attr string, path []string, latest bool) []Node { } var nodes []Node - var lastTs Timestamp + var lastTS Timestamp children := t.getChildren(curNode) for i := range children { @@ -200,7 +200,7 @@ func (t tree) getByPath(attr string, path []string, latest bool) []Node { fileName := string(info.Meta.GetAttr(attr)) if fileName == path[len(path)-1] { if latest { - if info.Meta.Time >= lastTs { + if info.Meta.Time >= lastTS { nodes = append(nodes[:0], children[i]) } } else { From c274bbeb7c0b6957ccc5d3f507b5b5432f91ffd6 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:25:59 +0300 Subject: [PATCH 1881/1943] [#1689] linter: Fix staticcheck warning: 'methods on the same type should have the same receiver name' Change-Id: I25e9432987f73061c1506a184a82065e37885861 Signed-off-by: Alexander Chuprov --- pkg/services/object/get/get.go | 42 +++++++++++++++--------------- pkg/services/object/put/service.go | 4 +-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/pkg/services/object/get/get.go b/pkg/services/object/get/get.go index 9676fd914..3a50308c2 100644 --- a/pkg/services/object/get/get.go +++ b/pkg/services/object/get/get.go @@ -90,48 +90,48 @@ func (s *Service) get(ctx context.Context, prm RequestParameters) error { return exec.err } -func (exec *request) execute(ctx context.Context) { - exec.log.Debug(ctx, logs.ServingRequest) +func (r *request) execute(ctx context.Context) { + r.log.Debug(ctx, logs.ServingRequest) // perform local operation - exec.executeLocal(ctx) + r.executeLocal(ctx) - exec.analyzeStatus(ctx, true) + r.analyzeStatus(ctx, true) } -func (exec *request) analyzeStatus(ctx context.Context, execCnr bool) { +func (r *request) analyzeStatus(ctx context.Context, execCnr bool) { // analyze local result - switch exec.status { + switch r.status { case statusOK: - exec.log.Debug(ctx, logs.OperationFinishedSuccessfully) + r.log.Debug(ctx, logs.OperationFinishedSuccessfully) case statusINHUMED: - exec.log.Debug(ctx, logs.GetRequestedObjectWasMarkedAsRemoved) + r.log.Debug(ctx, logs.GetRequestedObjectWasMarkedAsRemoved) case statusVIRTUAL: - exec.log.Debug(ctx, logs.GetRequestedObjectIsVirtual) - exec.assemble(ctx) + r.log.Debug(ctx, logs.GetRequestedObjectIsVirtual) + r.assemble(ctx) case statusOutOfRange: - exec.log.Debug(ctx, logs.GetRequestedRangeIsOutOfObjectBounds) + r.log.Debug(ctx, logs.GetRequestedRangeIsOutOfObjectBounds) case statusEC: - exec.log.Debug(ctx, logs.GetRequestedObjectIsEC) - if exec.isRaw() && execCnr { - exec.executeOnContainer(ctx) - exec.analyzeStatus(ctx, false) + r.log.Debug(ctx, logs.GetRequestedObjectIsEC) + if r.isRaw() && execCnr { + r.executeOnContainer(ctx) + r.analyzeStatus(ctx, false) } - exec.assembleEC(ctx) + r.assembleEC(ctx) default: - exec.log.Debug(ctx, logs.OperationFinishedWithError, - zap.Error(exec.err), + r.log.Debug(ctx, logs.OperationFinishedWithError, + zap.Error(r.err), ) var errAccessDenied *apistatus.ObjectAccessDenied - if execCnr && errors.As(exec.err, &errAccessDenied) { + if execCnr && errors.As(r.err, &errAccessDenied) { // Local get can't return access denied error, so this error was returned by // write to the output stream. So there is no need to try to find object on other nodes. return } if execCnr { - exec.executeOnContainer(ctx) - exec.analyzeStatus(ctx, false) + r.executeOnContainer(ctx) + r.analyzeStatus(ctx, false) } } } diff --git a/pkg/services/object/put/service.go b/pkg/services/object/put/service.go index 099486b3f..7aeb5857d 100644 --- a/pkg/services/object/put/service.go +++ b/pkg/services/object/put/service.go @@ -56,8 +56,8 @@ func NewService(ks *objutil.KeyStorage, } } -func (p *Service) Put() (*Streamer, error) { +func (s *Service) Put() (*Streamer, error) { return &Streamer{ - Config: p.Config, + Config: s.Config, }, nil } From 2394ae6ce006acf85e3914fc29c3dd284be9ce3c Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:28:27 +0300 Subject: [PATCH 1882/1943] [#1689] linter: Fix staticcheck warning: 'could lift into loop condition' Change-Id: I4ff3cda54861d857740203d6994872998a22d5d5 Signed-off-by: Alexander Chuprov --- pkg/services/object/get/container.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pkg/services/object/get/container.go b/pkg/services/object/get/container.go index 0ee8aed53..dfb31133c 100644 --- a/pkg/services/object/get/container.go +++ b/pkg/services/object/get/container.go @@ -28,16 +28,7 @@ func (r *request) executeOnContainer(ctx context.Context) { localStatus := r.status - for { - if r.processCurrentEpoch(ctx, localStatus) { - break - } - - // check the maximum depth has been reached - if lookupDepth == 0 { - break - } - + for !r.processCurrentEpoch(ctx, localStatus) && lookupDepth != 0 { lookupDepth-- // go to the previous epoch From dfdeedfc6fe2b557fe71d58960f954f30be2c339 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:32:23 +0300 Subject: [PATCH 1883/1943] [#1689] linter: Fix staticcheck warning: 'could apply De Morgan's law' Change-Id: Ife03172bad7d517dc99771250c3308a9fc0916b3 Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/modules/object/range.go | 2 +- pkg/morph/event/notary_preparator.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/range.go b/cmd/frostfs-cli/modules/object/range.go index be4fee4cf..6ec508ae2 100644 --- a/cmd/frostfs-cli/modules/object/range.go +++ b/cmd/frostfs-cli/modules/object/range.go @@ -154,7 +154,7 @@ func printECInfoErr(cmd *cobra.Command, err error) bool { if ok { toJSON, _ := cmd.Flags().GetBool(commonflags.JSON) toProto, _ := cmd.Flags().GetBool("proto") - if !(toJSON || toProto) { + if !toJSON && !toProto { cmd.PrintErrln("Object is erasure-encoded, ec information received.") } printECInfo(cmd, errECInfo.ECInfo()) diff --git a/pkg/morph/event/notary_preparator.go b/pkg/morph/event/notary_preparator.go index 40f5984a9..b11973646 100644 --- a/pkg/morph/event/notary_preparator.go +++ b/pkg/morph/event/notary_preparator.go @@ -199,8 +199,8 @@ func (p Preparator) validateNotaryRequest(nr *payload.P2PNotaryRequest) error { // neo-go API) // // this check prevents notary flow recursion - if !(len(nr.MainTransaction.Scripts[1].InvocationScript) == 0 || - bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript)) { // compatibility with old version + if len(nr.MainTransaction.Scripts[1].InvocationScript) != 0 && + !bytes.Equal(nr.MainTransaction.Scripts[1].InvocationScript, p.dummyInvocationScript) { // compatibility with old version return ErrTXAlreadyHandled } @@ -364,8 +364,8 @@ func (p Preparator) validateWitnesses(w []transaction.Witness, alphaKeys keys.Pu // the last one must be a placeholder for notary contract witness last := len(w) - 1 - if !(len(w[last].InvocationScript) == 0 || // https://github.com/nspcc-dev/neo-go/pull/2981 - bytes.Equal(w[last].InvocationScript, p.dummyInvocationScript)) || // compatibility with old version + if (len(w[last].InvocationScript) != 0 && // https://github.com/nspcc-dev/neo-go/pull/2981 + !bytes.Equal(w[last].InvocationScript, p.dummyInvocationScript)) || // compatibility with old version len(w[last].VerificationScript) != 0 { return errIncorrectNotaryPlaceholder } From 4f9d237042233c8ae155ae3d2de4ea3911f8ce49 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Mon, 7 Apr 2025 17:36:54 +0300 Subject: [PATCH 1884/1943] [#1689] linter: Fix staticcheck warning: 'probably want to use time.Time.Equal instead' Change-Id: Idb119d3f4f167c9e42ed48633d301185589553ed Signed-off-by: Alexander Chuprov --- pkg/local_object_storage/engine/evacuate_limiter.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go index c74134500..3dd7494be 100644 --- a/pkg/local_object_storage/engine/evacuate_limiter.go +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -95,8 +95,7 @@ func (s *EvacuationState) StartedAt() *time.Time { if s == nil { return nil } - defaultTime := time.Time{} - if s.startedAt == defaultTime { + if s.startedAt.IsZero() { return nil } return &s.startedAt @@ -106,8 +105,7 @@ func (s *EvacuationState) FinishedAt() *time.Time { if s == nil { return nil } - defaultTime := time.Time{} - if s.finishedAt == defaultTime { + if s.finishedAt.IsZero() { return nil } return &s.finishedAt From fe29ed043a7d5e6e27256cf52490e1ec7ceedb40 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 8 Apr 2025 15:22:00 +0300 Subject: [PATCH 1885/1943] [#1689] linter: Fix staticcheck warning: 'could use tagged switch on *' Change-Id: Ia340ce1ccdd223eb87f7aefabfba62b7055f344d Signed-off-by: Alexander Chuprov --- pkg/services/object/ape/checker.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/services/object/ape/checker.go b/pkg/services/object/ape/checker.go index ee46a6fe4..b96757def 100644 --- a/pkg/services/object/ape/checker.go +++ b/pkg/services/object/ape/checker.go @@ -76,9 +76,10 @@ var errMissingOID = errors.New("object ID is not set") // CheckAPE prepares an APE-request and checks if it is permitted by policies. func (c *checkerImpl) CheckAPE(ctx context.Context, prm Prm) error { // APE check is ignored for some inter-node requests. - if prm.Role == nativeschema.PropertyValueContainerRoleContainer { + switch prm.Role { + case nativeschema.PropertyValueContainerRoleContainer: return nil - } else if prm.Role == nativeschema.PropertyValueContainerRoleIR { + case nativeschema.PropertyValueContainerRoleIR: switch prm.Method { case nativeschema.MethodGetObject, nativeschema.MethodHeadObject, From aed84b567caac5b2825888621ca170ee9dae0974 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Tue, 8 Apr 2025 15:23:51 +0300 Subject: [PATCH 1886/1943] [#1689] linter: Bump 'golangci-lint' to v2.0.2 Change-Id: Ib546af43845014785f0debce429a37d62e616539 Signed-off-by: Alexander Chuprov --- .golangci.yml | 179 ++++++++++++++++++++++++++------------------------ Makefile | 4 +- 2 files changed, 94 insertions(+), 89 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 18de49425..3ac4eb651 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,98 +1,103 @@ -# This file contains all available configuration options -# with their default values. - -# options for analysis running +version: "2" run: - # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 20m - - # include test files or not, default is true tests: false - -# output configuration options output: - # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" formats: - - format: tab - -# all available settings of specific linters -linters-settings: - exhaustive: - # indicates that switch statements are to be considered exhaustive if a - # 'default' case is present, even if all enum members aren't listed in the - # switch - default-signifies-exhaustive: true - gci: - sections: - - standard - - default - custom-order: true - staticcheck: - checks: ["all", "-SA1019"] # TODO Enable SA1019 after deprecated warning are fixed. - funlen: - lines: 80 # default 60 - statements: 60 # default 40 - gocognit: - min-complexity: 40 # default 30 - importas: - no-unaliased: true - no-extra-aliases: false - alias: - pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object - alias: objectSDK - unused: - field-writes-are-uses: false - exported-fields-are-used: false - local-variables-are-used: false - custom: - truecloudlab-linters: - path: bin/linters/external_linters.so - original-url: git.frostfs.info/TrueCloudLab/linters.git - settings: - noliteral: - target-methods : ["reportFlushError", "reportError"] - disable-packages: ["codes", "err", "res","exec"] - constants-package: "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" - + tab: + path: stdout + colors: false linters: + default: none enable: - # mandatory linters - - revive - - predeclared - - # some default golangci-lint linters - - errcheck - - gosimple - - godot - - ineffassign - - staticcheck - - typecheck - - unused - - # extra linters - bidichk - - durationcheck - - exhaustive + - containedctx + - contextcheck - copyloopvar + - durationcheck + - errcheck + - exhaustive + - funlen + - gocognit + - godot + - importas + - ineffassign + - intrange + - misspell + - perfsprint + - predeclared + - protogetter + - reassign + - revive + - staticcheck + - testifylint + - truecloudlab-linters + - unconvert + - unparam + - unused + - usetesting + - whitespace + settings: + exhaustive: + default-signifies-exhaustive: true + funlen: + lines: 80 + statements: 60 + gocognit: + min-complexity: 40 + importas: + alias: + - pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object + alias: objectSDK + no-unaliased: true + no-extra-aliases: false + staticcheck: + checks: + - all + - -QF1002 + unused: + field-writes-are-uses: false + exported-fields-are-used: false + local-variables-are-used: false + custom: + truecloudlab-linters: + path: bin/linters/external_linters.so + original-url: git.frostfs.info/TrueCloudLab/linters.git + settings: + noliteral: + constants-package: git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs + disable-packages: + - codes + - err + - res + - exec + target-methods: + - reportFlushError + - reportError + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: - gci - gofmt - goimports - - misspell - - predeclared - - reassign - - whitespace - - containedctx - - funlen - - gocognit - - contextcheck - - importas - - truecloudlab-linters - - perfsprint - - testifylint - - protogetter - - intrange - - unconvert - - unparam - - usetesting - disable-all: true - fast: false + settings: + gci: + sections: + - standard + - default + custom-order: true + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/Makefile b/Makefile index 5b55c9eec..575eaae6f 100755 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ HUB_IMAGE ?= git.frostfs.info/truecloudlab/frostfs HUB_TAG ?= "$(shell echo ${VERSION} | sed 's/^v//')" GO_VERSION ?= 1.23 -LINT_VERSION ?= 1.64.8 +LINT_VERSION ?= 2.0.2 TRUECLOUDLAB_LINT_VERSION ?= 0.0.10 PROTOC_VERSION ?= 25.0 PROTOGEN_FROSTFS_VERSION ?= $(shell go list -f '{{.Version}}' -m git.frostfs.info/TrueCloudLab/frostfs-sdk-go) @@ -224,7 +224,7 @@ lint-install: $(BIN) @@make -C $(TMP_DIR)/linters lib CGO_ENABLED=1 OUT_DIR=$(OUTPUT_LINT_DIR) @rm -rf $(TMP_DIR)/linters @rmdir $(TMP_DIR) 2>/dev/null || true - @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install -trimpath github.com/golangci/golangci-lint/cmd/golangci-lint@v$(LINT_VERSION) + @CGO_ENABLED=1 GOBIN=$(LINT_DIR) go install -trimpath github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v$(LINT_VERSION) # Run linters lint: From f93b96c60114440128782ca9d0a228a80a501ae1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Apr 2025 12:35:34 +0300 Subject: [PATCH 1887/1943] [#1712] adm: Add `maintenance zombie` commands Change-Id: I1b73e561a8daad67d0a8ffc0d293cbdd09aaab6b Signed-off-by: Dmitrii Stepanov --- .../internal/modules/maintenance/root.go | 15 + .../modules/maintenance/zombie/key.go | 70 +++++ .../modules/maintenance/zombie/list.go | 31 ++ .../modules/maintenance/zombie/morph.go | 46 +++ .../modules/maintenance/zombie/quarantine.go | 154 ++++++++++ .../modules/maintenance/zombie/remove.go | 55 ++++ .../modules/maintenance/zombie/restore.go | 69 +++++ .../modules/maintenance/zombie/root.go | 125 ++++++++ .../modules/maintenance/zombie/scan.go | 281 ++++++++++++++++++ .../maintenance/zombie/storage_engine.go | 203 +++++++++++++ cmd/frostfs-adm/internal/modules/root.go | 2 + 11 files changed, 1051 insertions(+) create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/root.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/key.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/list.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/morph.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/quarantine.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/remove.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/restore.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/scan.go create mode 100644 cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go diff --git a/cmd/frostfs-adm/internal/modules/maintenance/root.go b/cmd/frostfs-adm/internal/modules/maintenance/root.go new file mode 100644 index 000000000..d67b70d2a --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/root.go @@ -0,0 +1,15 @@ +package maintenance + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/maintenance/zombie" + "github.com/spf13/cobra" +) + +var RootCmd = &cobra.Command{ + Use: "maintenance", + Short: "Section for maintenance commands", +} + +func init() { + RootCmd.AddCommand(zombie.Cmd) +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/key.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/key.go new file mode 100644 index 000000000..1b66889aa --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/key.go @@ -0,0 +1,70 @@ +package zombie + +import ( + "crypto/ecdsa" + "fmt" + "os" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "github.com/nspcc-dev/neo-go/cli/flags" + "github.com/nspcc-dev/neo-go/cli/input" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/nspcc-dev/neo-go/pkg/wallet" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func getPrivateKey(cmd *cobra.Command, appCfg *config.Config) *ecdsa.PrivateKey { + keyDesc := viper.GetString(walletFlag) + if keyDesc == "" { + return &nodeconfig.Key(appCfg).PrivateKey + } + data, err := os.ReadFile(keyDesc) + commonCmd.ExitOnErr(cmd, "open wallet file: %w", err) + + priv, err := keys.NewPrivateKeyFromBytes(data) + if err != nil { + w, err := wallet.NewWalletFromFile(keyDesc) + commonCmd.ExitOnErr(cmd, "provided key is incorrect, only wallet or binary key supported: %w", err) + return fromWallet(cmd, w, viper.GetString(addressFlag)) + } + return &priv.PrivateKey +} + +func fromWallet(cmd *cobra.Command, w *wallet.Wallet, addrStr string) *ecdsa.PrivateKey { + var ( + addr util.Uint160 + err error + ) + + if addrStr == "" { + addr = w.GetChangeAddress() + } else { + addr, err = flags.ParseAddress(addrStr) + commonCmd.ExitOnErr(cmd, "--address option must be specified and valid: %w", err) + } + + acc := w.GetAccount(addr) + if acc == nil { + commonCmd.ExitOnErr(cmd, "--address option must be specified and valid: %w", fmt.Errorf("can't find wallet account for %s", addrStr)) + } + + pass, err := getPassword() + commonCmd.ExitOnErr(cmd, "invalid password for the encrypted key: %w", err) + + commonCmd.ExitOnErr(cmd, "can't decrypt account: %w", acc.Decrypt(pass, keys.NEP2ScryptParams())) + + return &acc.PrivateKey().PrivateKey +} + +func getPassword() (string, error) { + // this check allows empty passwords + if viper.IsSet("password") { + return viper.GetString("password"), nil + } + + return input.ReadPassword("Enter password > ") +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/list.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/list.go new file mode 100644 index 000000000..f73f33db9 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/list.go @@ -0,0 +1,31 @@ +package zombie + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" +) + +func list(cmd *cobra.Command, _ []string) { + configFile, _ := cmd.Flags().GetString(commonflags.ConfigFlag) + configDir, _ := cmd.Flags().GetString(commonflags.ConfigDirFlag) + appCfg := config.New(configFile, configDir, config.EnvPrefix) + storageEngine := newEngine(cmd, appCfg) + q := createQuarantine(cmd, storageEngine.DumpInfo()) + var containerID *cid.ID + if cidStr, _ := cmd.Flags().GetString(cidFlag); cidStr != "" { + containerID = &cid.ID{} + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", containerID.DecodeString(cidStr)) + } + + commonCmd.ExitOnErr(cmd, "iterate over quarantine: %w", q.Iterate(cmd.Context(), func(a oid.Address) error { + if containerID != nil && a.Container() != *containerID { + return nil + } + cmd.Println(a.EncodeToString()) + return nil + })) +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/morph.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/morph.go new file mode 100644 index 000000000..cd3a64499 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/morph.go @@ -0,0 +1,46 @@ +package zombie + +import ( + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" + nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" + cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + netmapClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + "github.com/spf13/cobra" +) + +func createMorphClient(cmd *cobra.Command, appCfg *config.Config) *client.Client { + addresses := morphconfig.RPCEndpoint(appCfg) + if len(addresses) == 0 { + commonCmd.ExitOnErr(cmd, "create morph client: %w", errors.New("no morph endpoints found")) + } + key := nodeconfig.Key(appCfg) + cli, err := client.New(cmd.Context(), + key, + client.WithDialTimeout(morphconfig.DialTimeout(appCfg)), + client.WithEndpoints(addresses...), + client.WithSwitchInterval(morphconfig.SwitchInterval(appCfg)), + ) + commonCmd.ExitOnErr(cmd, "create morph client: %w", err) + return cli +} + +func createContainerClient(cmd *cobra.Command, morph *client.Client) *cntClient.Client { + hs, err := morph.NNSContractAddress(client.NNSContainerContractName) + commonCmd.ExitOnErr(cmd, "resolve container contract hash: %w", err) + cc, err := cntClient.NewFromMorph(morph, hs, 0) + commonCmd.ExitOnErr(cmd, "create morph container client: %w", err) + return cc +} + +func createNetmapClient(cmd *cobra.Command, morph *client.Client) *netmapClient.Client { + hs, err := morph.NNSContractAddress(client.NNSNetmapContractName) + commonCmd.ExitOnErr(cmd, "resolve netmap contract hash: %w", err) + cli, err := netmapClient.NewFromMorph(morph, hs, 0) + commonCmd.ExitOnErr(cmd, "create morph netmap client: %w", err) + return cli +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/quarantine.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/quarantine.go new file mode 100644 index 000000000..27f83aec7 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/quarantine.go @@ -0,0 +1,154 @@ +package zombie + +import ( + "context" + "fmt" + "math" + "os" + "path/filepath" + "strings" + "sync" + + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + objectcore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" +) + +type quarantine struct { + // mtx protects current field. + mtx sync.Mutex + current int + trees []*fstree.FSTree +} + +func createQuarantine(cmd *cobra.Command, engineInfo engine.Info) *quarantine { + var paths []string + for _, sh := range engineInfo.Shards { + var storagePaths []string + for _, st := range sh.BlobStorInfo.SubStorages { + storagePaths = append(storagePaths, st.Path) + } + if len(storagePaths) == 0 { + continue + } + paths = append(paths, filepath.Join(commonPath(storagePaths), "quarantine")) + } + q, err := newQuarantine(paths) + commonCmd.ExitOnErr(cmd, "create quarantine: %w", err) + return q +} + +func commonPath(paths []string) string { + if len(paths) == 0 { + return "" + } + if len(paths) == 1 { + return paths[0] + } + minLen := math.MaxInt + for _, p := range paths { + if len(p) < minLen { + minLen = len(p) + } + } + + var sb strings.Builder + for i := range minLen { + for _, path := range paths[1:] { + if paths[0][i] != path[i] { + return sb.String() + } + } + sb.WriteByte(paths[0][i]) + } + return sb.String() +} + +func newQuarantine(paths []string) (*quarantine, error) { + var q quarantine + for i := range paths { + f := fstree.New( + fstree.WithDepth(1), + fstree.WithDirNameLen(1), + fstree.WithPath(paths[i]), + fstree.WithPerm(os.ModePerm), + ) + if err := f.Open(mode.ComponentReadWrite); err != nil { + return nil, fmt.Errorf("open fstree %s: %w", paths[i], err) + } + if err := f.Init(); err != nil { + return nil, fmt.Errorf("init fstree %s: %w", paths[i], err) + } + q.trees = append(q.trees, f) + } + return &q, nil +} + +func (q *quarantine) Get(ctx context.Context, a oid.Address) (*objectSDK.Object, error) { + for i := range q.trees { + res, err := q.trees[i].Get(ctx, common.GetPrm{Address: a}) + if err != nil { + continue + } + return res.Object, nil + } + return nil, &apistatus.ObjectNotFound{} +} + +func (q *quarantine) Delete(ctx context.Context, a oid.Address) error { + for i := range q.trees { + _, err := q.trees[i].Delete(ctx, common.DeletePrm{Address: a}) + if err != nil { + continue + } + return nil + } + return &apistatus.ObjectNotFound{} +} + +func (q *quarantine) Put(ctx context.Context, obj *objectSDK.Object) error { + data, err := obj.Marshal() + if err != nil { + return err + } + + var prm common.PutPrm + prm.Address = objectcore.AddressOf(obj) + prm.Object = obj + prm.RawData = data + + q.mtx.Lock() + current := q.current + q.current = (q.current + 1) % len(q.trees) + q.mtx.Unlock() + + _, err = q.trees[current].Put(ctx, prm) + return err +} + +func (q *quarantine) Iterate(ctx context.Context, f func(oid.Address) error) error { + var prm common.IteratePrm + prm.Handler = func(elem common.IterationElement) error { + return f(elem.Address) + } + for i := range q.trees { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + _, err := q.trees[i].Iterate(ctx, prm) + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/remove.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/remove.go new file mode 100644 index 000000000..0b8f2f172 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/remove.go @@ -0,0 +1,55 @@ +package zombie + +import ( + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" +) + +func remove(cmd *cobra.Command, _ []string) { + configFile, _ := cmd.Flags().GetString(commonflags.ConfigFlag) + configDir, _ := cmd.Flags().GetString(commonflags.ConfigDirFlag) + appCfg := config.New(configFile, configDir, config.EnvPrefix) + storageEngine := newEngine(cmd, appCfg) + q := createQuarantine(cmd, storageEngine.DumpInfo()) + + var containerID cid.ID + cidStr, _ := cmd.Flags().GetString(cidFlag) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", containerID.DecodeString(cidStr)) + + var objectID *oid.ID + oidStr, _ := cmd.Flags().GetString(oidFlag) + if oidStr != "" { + objectID = &oid.ID{} + commonCmd.ExitOnErr(cmd, "decode object ID string: %w", objectID.DecodeString(oidStr)) + } + + if objectID != nil { + var addr oid.Address + addr.SetContainer(containerID) + addr.SetObject(*objectID) + removeObject(cmd, q, addr) + } else { + commonCmd.ExitOnErr(cmd, "iterate over quarantine: %w", q.Iterate(cmd.Context(), func(addr oid.Address) error { + if addr.Container() != containerID { + return nil + } + removeObject(cmd, q, addr) + return nil + })) + } +} + +func removeObject(cmd *cobra.Command, q *quarantine, addr oid.Address) { + err := q.Delete(cmd.Context(), addr) + if errors.Is(err, new(apistatus.ObjectNotFound)) { + return + } + commonCmd.ExitOnErr(cmd, "remove object from quarantine: %w", err) +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/restore.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/restore.go new file mode 100644 index 000000000..f179c7c2d --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/restore.go @@ -0,0 +1,69 @@ +package zombie + +import ( + "crypto/sha256" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" +) + +func restore(cmd *cobra.Command, _ []string) { + configFile, _ := cmd.Flags().GetString(commonflags.ConfigFlag) + configDir, _ := cmd.Flags().GetString(commonflags.ConfigDirFlag) + appCfg := config.New(configFile, configDir, config.EnvPrefix) + storageEngine := newEngine(cmd, appCfg) + q := createQuarantine(cmd, storageEngine.DumpInfo()) + morphClient := createMorphClient(cmd, appCfg) + cnrCli := createContainerClient(cmd, morphClient) + + var containerID cid.ID + cidStr, _ := cmd.Flags().GetString(cidFlag) + commonCmd.ExitOnErr(cmd, "decode container ID string: %w", containerID.DecodeString(cidStr)) + + var objectID *oid.ID + oidStr, _ := cmd.Flags().GetString(oidFlag) + if oidStr != "" { + objectID = &oid.ID{} + commonCmd.ExitOnErr(cmd, "decode object ID string: %w", objectID.DecodeString(oidStr)) + } + + if objectID != nil { + var addr oid.Address + addr.SetContainer(containerID) + addr.SetObject(*objectID) + restoreObject(cmd, storageEngine, q, addr, cnrCli) + } else { + commonCmd.ExitOnErr(cmd, "iterate over quarantine: %w", q.Iterate(cmd.Context(), func(addr oid.Address) error { + if addr.Container() != containerID { + return nil + } + restoreObject(cmd, storageEngine, q, addr, cnrCli) + return nil + })) + } +} + +func restoreObject(cmd *cobra.Command, storageEngine *engine.StorageEngine, q *quarantine, addr oid.Address, cnrCli *cntClient.Client) { + obj, err := q.Get(cmd.Context(), addr) + commonCmd.ExitOnErr(cmd, "get object from quarantine: %w", err) + rawCID := make([]byte, sha256.Size) + + cid := addr.Container() + cid.Encode(rawCID) + cnr, err := cnrCli.Get(cmd.Context(), rawCID) + commonCmd.ExitOnErr(cmd, "get container: %w", err) + + putPrm := engine.PutPrm{ + Object: obj, + IsIndexedContainer: containerCore.IsIndexedContainer(cnr.Value), + } + commonCmd.ExitOnErr(cmd, "put object to storage engine: %w", storageEngine.Put(cmd.Context(), putPrm)) + commonCmd.ExitOnErr(cmd, "remove object from quarantine: %w", q.Delete(cmd.Context(), addr)) +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go new file mode 100644 index 000000000..9ef18f7f8 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go @@ -0,0 +1,125 @@ +package zombie + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + flagBatchSize = "batch-size" + flagBatchSizeUsage = "Objects iteration batch size" + cidFlag = "cid" + cidFlagUsage = "Container ID" + oidFlag = "oid" + oidFlagUsage = "Object ID" + walletFlag = "wallet" + walletFlagShorthand = "w" + walletFlagUsage = "Path to the wallet or binary key" + addressFlag = "address" + addressFlagUsage = "Address of wallet account" + moveFlag = "move" + moveFlagUsage = "Move objects from storage engine to quarantine" +) + +var ( + Cmd = &cobra.Command{ + Use: "zombie", + Short: "Zombie objects related commands", + } + scanCmd = &cobra.Command{ + Use: "scan", + Short: "Scan storage engine for zombie objects and move them to quarantine", + Long: "", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) + _ = viper.BindPFlag(commonflags.ConfigFlag, cmd.Flags().Lookup(commonflags.ConfigFlag)) + _ = viper.BindPFlag(commonflags.ConfigDirFlag, cmd.Flags().Lookup(commonflags.ConfigDirFlag)) + _ = viper.BindPFlag(walletFlag, cmd.Flags().Lookup(walletFlag)) + _ = viper.BindPFlag(addressFlag, cmd.Flags().Lookup(addressFlag)) + _ = viper.BindPFlag(flagBatchSize, cmd.Flags().Lookup(flagBatchSize)) + _ = viper.BindPFlag(moveFlag, cmd.Flags().Lookup(moveFlag)) + }, + Run: scan, + } + listCmd = &cobra.Command{ + Use: "list", + Short: "List zombie objects from quarantine", + Long: "", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.ConfigFlag, cmd.Flags().Lookup(commonflags.ConfigFlag)) + _ = viper.BindPFlag(commonflags.ConfigDirFlag, cmd.Flags().Lookup(commonflags.ConfigDirFlag)) + _ = viper.BindPFlag(cidFlag, cmd.Flags().Lookup(cidFlag)) + }, + Run: list, + } + restoreCmd = &cobra.Command{ + Use: "restore", + Short: "Restore zombie objects from quarantine", + Long: "", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.ConfigFlag, cmd.Flags().Lookup(commonflags.ConfigFlag)) + _ = viper.BindPFlag(commonflags.ConfigDirFlag, cmd.Flags().Lookup(commonflags.ConfigDirFlag)) + _ = viper.BindPFlag(cidFlag, cmd.Flags().Lookup(cidFlag)) + _ = viper.BindPFlag(oidFlag, cmd.Flags().Lookup(oidFlag)) + }, + Run: restore, + } + removeCmd = &cobra.Command{ + Use: "remove", + Short: "Remove zombie objects from quarantine", + Long: "", + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlag(commonflags.ConfigFlag, cmd.Flags().Lookup(commonflags.ConfigFlag)) + _ = viper.BindPFlag(commonflags.ConfigDirFlag, cmd.Flags().Lookup(commonflags.ConfigDirFlag)) + _ = viper.BindPFlag(cidFlag, cmd.Flags().Lookup(cidFlag)) + _ = viper.BindPFlag(oidFlag, cmd.Flags().Lookup(oidFlag)) + }, + Run: remove, + } +) + +func init() { + initScanCmd() + initListCmd() + initRestoreCmd() + initRemoveCmd() +} + +func initScanCmd() { + Cmd.AddCommand(scanCmd) + + scanCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) + scanCmd.Flags().StringP(commonflags.ConfigFlag, commonflags.ConfigFlagShorthand, "", commonflags.ConfigFlagUsage) + scanCmd.Flags().String(commonflags.ConfigDirFlag, "", commonflags.ConfigDirFlagUsage) + scanCmd.Flags().Uint32(flagBatchSize, 1000, flagBatchSizeUsage) + scanCmd.Flags().StringP(walletFlag, walletFlagShorthand, "", walletFlagUsage) + scanCmd.Flags().String(addressFlag, "", addressFlagUsage) + scanCmd.Flags().Bool(moveFlag, false, moveFlagUsage) +} + +func initListCmd() { + Cmd.AddCommand(listCmd) + + listCmd.Flags().StringP(commonflags.ConfigFlag, commonflags.ConfigFlagShorthand, "", commonflags.ConfigFlagUsage) + listCmd.Flags().String(commonflags.ConfigDirFlag, "", commonflags.ConfigDirFlagUsage) + listCmd.Flags().String(cidFlag, "", cidFlagUsage) +} + +func initRestoreCmd() { + Cmd.AddCommand(restoreCmd) + + restoreCmd.Flags().StringP(commonflags.ConfigFlag, commonflags.ConfigFlagShorthand, "", commonflags.ConfigFlagUsage) + restoreCmd.Flags().String(commonflags.ConfigDirFlag, "", commonflags.ConfigDirFlagUsage) + restoreCmd.Flags().String(cidFlag, "", cidFlagUsage) + restoreCmd.Flags().String(oidFlag, "", oidFlagUsage) +} + +func initRemoveCmd() { + Cmd.AddCommand(removeCmd) + + removeCmd.Flags().StringP(commonflags.ConfigFlag, commonflags.ConfigFlagShorthand, "", commonflags.ConfigFlagUsage) + removeCmd.Flags().String(commonflags.ConfigDirFlag, "", commonflags.ConfigDirFlagUsage) + removeCmd.Flags().String(cidFlag, "", cidFlagUsage) + removeCmd.Flags().String(oidFlag, "", oidFlagUsage) +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/scan.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/scan.go new file mode 100644 index 000000000..268ec4911 --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/scan.go @@ -0,0 +1,281 @@ +package zombie + +import ( + "context" + "crypto/ecdsa" + "crypto/sha256" + "errors" + "fmt" + "sync" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + apiclientconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/apiclient" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + clientCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client" + netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/cache" + clientSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" +) + +func scan(cmd *cobra.Command, _ []string) { + configFile, _ := cmd.Flags().GetString(commonflags.ConfigFlag) + configDir, _ := cmd.Flags().GetString(commonflags.ConfigDirFlag) + appCfg := config.New(configFile, configDir, config.EnvPrefix) + batchSize, _ := cmd.Flags().GetUint32(flagBatchSize) + if batchSize == 0 { + commonCmd.ExitOnErr(cmd, "invalid batch size: %w", errors.New("batch size must be positive value")) + } + move, _ := cmd.Flags().GetBool(moveFlag) + + storageEngine := newEngine(cmd, appCfg) + morphClient := createMorphClient(cmd, appCfg) + cnrCli := createContainerClient(cmd, morphClient) + nmCli := createNetmapClient(cmd, morphClient) + q := createQuarantine(cmd, storageEngine.DumpInfo()) + pk := getPrivateKey(cmd, appCfg) + + epoch, err := nmCli.Epoch(cmd.Context()) + commonCmd.ExitOnErr(cmd, "read epoch from morph: %w", err) + + nm, err := nmCli.GetNetMapByEpoch(cmd.Context(), epoch) + commonCmd.ExitOnErr(cmd, "read netmap from morph: %w", err) + + cmd.Printf("Epoch: %d\n", nm.Epoch()) + cmd.Printf("Nodes in the netmap: %d\n", len(nm.Nodes())) + + ps := &processStatus{ + statusCount: make(map[status]uint64), + } + + stopCh := make(chan struct{}) + start := time.Now() + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case <-cmd.Context().Done(): + return + case <-stopCh: + return + case <-tick.C: + fmt.Printf("Objects processed: %d; Time elapsed: %s\n", ps.total(), time.Since(start)) + } + } + }() + go func() { + defer wg.Done() + err = scanStorageEngine(cmd, batchSize, storageEngine, ps, appCfg, cnrCli, nmCli, q, pk, move) + close(stopCh) + }() + wg.Wait() + commonCmd.ExitOnErr(cmd, "scan storage engine for zombie objects: %w", err) + + cmd.Println() + cmd.Println("Status description:") + cmd.Println("undefined -- nothing is clear") + cmd.Println("found -- object is found in cluster") + cmd.Println("quarantine -- object is not found in cluster") + cmd.Println() + for status, count := range ps.statusCount { + cmd.Printf("Status: %s, Count: %d\n", status, count) + } +} + +type status string + +const ( + statusUndefined status = "undefined" + statusFound status = "found" + statusQuarantine status = "quarantine" +) + +func checkAddr(ctx context.Context, cnrCli *cntClient.Client, nmCli *netmap.Client, cc *cache.ClientCache, obj object.Info) (status, error) { + rawCID := make([]byte, sha256.Size) + cid := obj.Address.Container() + cid.Encode(rawCID) + + cnr, err := cnrCli.Get(ctx, rawCID) + if err != nil { + var errContainerNotFound *apistatus.ContainerNotFound + if errors.As(err, &errContainerNotFound) { + // Policer will deal with this object. + return statusFound, nil + } + return statusUndefined, fmt.Errorf("read container %s from morph: %w", cid, err) + } + nm, err := nmCli.NetMap(ctx) + if err != nil { + return statusUndefined, fmt.Errorf("read netmap from morph: %w", err) + } + + nodes, err := nm.ContainerNodes(cnr.Value.PlacementPolicy(), rawCID) + if err != nil { + // Not enough nodes, check all netmap nodes. + nodes = append([][]netmap.NodeInfo{}, nm.Nodes()) + } + + objID := obj.Address.Object() + cnrID := obj.Address.Container() + local := true + raw := false + if obj.ECInfo != nil { + objID = obj.ECInfo.ParentID + local = false + raw = true + } + prm := clientSDK.PrmObjectHead{ + ObjectID: &objID, + ContainerID: &cnrID, + Local: local, + Raw: raw, + } + + var ni clientCore.NodeInfo + for i := range nodes { + for j := range nodes[i] { + if err := clientCore.NodeInfoFromRawNetmapElement(&ni, netmapCore.Node(nodes[i][j])); err != nil { + return statusUndefined, fmt.Errorf("parse node info: %w", err) + } + c, err := cc.Get(ni) + if err != nil { + continue + } + res, err := c.ObjectHead(ctx, prm) + if err != nil { + var errECInfo *objectSDK.ECInfoError + if raw && errors.As(err, &errECInfo) { + return statusFound, nil + } + continue + } + if err := apistatus.ErrFromStatus(res.Status()); err != nil { + continue + } + return statusFound, nil + } + } + + if cnr.Value.PlacementPolicy().NumberOfReplicas() == 1 && cnr.Value.PlacementPolicy().ReplicaDescriptor(0).NumberOfObjects() == 1 { + return statusFound, nil + } + return statusQuarantine, nil +} + +func scanStorageEngine(cmd *cobra.Command, batchSize uint32, storageEngine *engine.StorageEngine, ps *processStatus, + appCfg *config.Config, cnrCli *cntClient.Client, nmCli *netmap.Client, q *quarantine, pk *ecdsa.PrivateKey, move bool, +) error { + cc := cache.NewSDKClientCache(cache.ClientCacheOpts{ + DialTimeout: apiclientconfig.DialTimeout(appCfg), + StreamTimeout: apiclientconfig.StreamTimeout(appCfg), + ReconnectTimeout: apiclientconfig.ReconnectTimeout(appCfg), + Key: pk, + AllowExternal: apiclientconfig.AllowExternal(appCfg), + }) + ctx := cmd.Context() + + var cursor *engine.Cursor + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + var prm engine.ListWithCursorPrm + prm.WithCursor(cursor) + prm.WithCount(batchSize) + + res, err := storageEngine.ListWithCursor(ctx, prm) + if err != nil { + if errors.Is(err, engine.ErrEndOfListing) { + return nil + } + return fmt.Errorf("list with cursor: %w", err) + } + + cursor = res.Cursor() + addrList := res.AddressList() + eg, egCtx := errgroup.WithContext(ctx) + eg.SetLimit(int(batchSize)) + + for i := range addrList { + addr := addrList[i] + eg.Go(func() error { + result, err := checkAddr(egCtx, cnrCli, nmCli, cc, addr) + if err != nil { + return fmt.Errorf("check object %s status: %w", addr.Address, err) + } + ps.add(result) + + if !move && result == statusQuarantine { + cmd.Println(addr) + return nil + } + + if result == statusQuarantine { + return moveToQuarantine(egCtx, storageEngine, q, addr.Address) + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return fmt.Errorf("process objects batch: %w", err) + } + } +} + +func moveToQuarantine(ctx context.Context, storageEngine *engine.StorageEngine, q *quarantine, addr oid.Address) error { + var getPrm engine.GetPrm + getPrm.WithAddress(addr) + res, err := storageEngine.Get(ctx, getPrm) + if err != nil { + return fmt.Errorf("get object %s from storage engine: %w", addr, err) + } + + if err := q.Put(ctx, res.Object()); err != nil { + return fmt.Errorf("put object %s to quarantine: %w", addr, err) + } + + var delPrm engine.DeletePrm + delPrm.WithForceRemoval() + delPrm.WithAddress(addr) + + if err = storageEngine.Delete(ctx, delPrm); err != nil { + return fmt.Errorf("delete object %s from storage engine: %w", addr, err) + } + return nil +} + +type processStatus struct { + guard sync.RWMutex + statusCount map[status]uint64 + count uint64 +} + +func (s *processStatus) add(st status) { + s.guard.Lock() + defer s.guard.Unlock() + s.statusCount[st]++ + s.count++ +} + +func (s *processStatus) total() uint64 { + s.guard.RLock() + defer s.guard.RUnlock() + return s.count +} diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go new file mode 100644 index 000000000..5851e049c --- /dev/null +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go @@ -0,0 +1,203 @@ +package zombie + +import ( + "context" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + engineconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine" + shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard" + blobovniczaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza" + fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "github.com/panjf2000/ants/v2" + "github.com/spf13/cobra" + "go.etcd.io/bbolt" + "go.uber.org/zap" +) + +func newEngine(cmd *cobra.Command, c *config.Config) *engine.StorageEngine { + ngOpts := storageEngineOptions(c) + shardOpts := shardOptions(cmd, c) + e := engine.New(ngOpts...) + for _, opts := range shardOpts { + _, err := e.AddShard(cmd.Context(), opts...) + commonCmd.ExitOnErr(cmd, "iterate shards from config: %w", err) + } + commonCmd.ExitOnErr(cmd, "open storage engine: %w", e.Open(cmd.Context())) + commonCmd.ExitOnErr(cmd, "init storage engine: %w", e.Init(cmd.Context())) + return e +} + +func storageEngineOptions(c *config.Config) []engine.Option { + return []engine.Option{ + engine.WithErrorThreshold(engineconfig.ShardErrorThreshold(c)), + engine.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + engine.WithLowMemoryConsumption(engineconfig.EngineLowMemoryConsumption(c)), + } +} + +func shardOptions(cmd *cobra.Command, c *config.Config) [][]shard.Option { + var result [][]shard.Option + err := engineconfig.IterateShards(c, false, func(sh *shardconfig.Config) error { + result = append(result, getShardOpts(cmd, c, sh)) + return nil + }) + commonCmd.ExitOnErr(cmd, "iterate shards from config: %w", err) + return result +} + +func getShardOpts(cmd *cobra.Command, c *config.Config, sh *shardconfig.Config) []shard.Option { + wc, wcEnabled := getWriteCacheOpts(sh) + return []shard.Option{ + shard.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + shard.WithRefillMetabase(sh.RefillMetabase()), + shard.WithRefillMetabaseWorkersCount(sh.RefillMetabaseWorkersCount()), + shard.WithMode(sh.Mode()), + shard.WithBlobStorOptions(getBlobstorOpts(cmd.Context(), sh)...), + shard.WithMetaBaseOptions(getMetabaseOpts(sh)...), + shard.WithPiloramaOptions(getPiloramaOpts(c, sh)...), + shard.WithWriteCache(wcEnabled), + shard.WithWriteCacheOptions(wc), + shard.WithRemoverBatchSize(sh.GC().RemoverBatchSize()), + shard.WithGCRemoverSleepInterval(sh.GC().RemoverSleepInterval()), + shard.WithExpiredCollectorBatchSize(sh.GC().ExpiredCollectorBatchSize()), + shard.WithExpiredCollectorWorkerCount(sh.GC().ExpiredCollectorWorkerCount()), + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + commonCmd.ExitOnErr(cmd, "init GC pool: %w", err) + return pool + }), + shard.WithLimiter(qos.NewNoopLimiter()), + } +} + +func getWriteCacheOpts(sh *shardconfig.Config) ([]writecache.Option, bool) { + if wc := sh.WriteCache(); wc != nil && wc.Enabled() { + var result []writecache.Option + result = append(result, + writecache.WithPath(wc.Path()), + writecache.WithFlushSizeLimit(wc.MaxFlushingObjectsSize()), + writecache.WithMaxObjectSize(wc.MaxObjectSize()), + writecache.WithFlushWorkersCount(wc.WorkerCount()), + writecache.WithMaxCacheSize(wc.SizeLimit()), + writecache.WithMaxCacheCount(wc.CountLimit()), + writecache.WithNoSync(wc.NoSync()), + writecache.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + writecache.WithQoSLimiter(qos.NewNoopLimiter()), + ) + return result, true + } + return nil, false +} + +func getPiloramaOpts(c *config.Config, sh *shardconfig.Config) []pilorama.Option { + var piloramaOpts []pilorama.Option + if config.BoolSafe(c.Sub("tree"), "enabled") { + pr := sh.Pilorama() + piloramaOpts = append(piloramaOpts, + pilorama.WithPath(pr.Path()), + pilorama.WithPerm(pr.Perm()), + pilorama.WithNoSync(pr.NoSync()), + pilorama.WithMaxBatchSize(pr.MaxBatchSize()), + pilorama.WithMaxBatchDelay(pr.MaxBatchDelay()), + ) + } + return piloramaOpts +} + +func getMetabaseOpts(sh *shardconfig.Config) []meta.Option { + return []meta.Option{ + meta.WithPath(sh.Metabase().Path()), + meta.WithPermissions(sh.Metabase().BoltDB().Perm()), + meta.WithMaxBatchSize(sh.Metabase().BoltDB().MaxBatchSize()), + meta.WithMaxBatchDelay(sh.Metabase().BoltDB().MaxBatchDelay()), + meta.WithBoltDBOptions(&bbolt.Options{ + Timeout: 100 * time.Millisecond, + }), + meta.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + meta.WithEpochState(&epochState{}), + } +} + +func getBlobstorOpts(ctx context.Context, sh *shardconfig.Config) []blobstor.Option { + result := []blobstor.Option{ + blobstor.WithCompressObjects(sh.Compress()), + blobstor.WithUncompressableContentTypes(sh.UncompressableContentTypes()), + blobstor.WithCompressibilityEstimate(sh.EstimateCompressibility()), + blobstor.WithCompressibilityEstimateThreshold(sh.EstimateCompressibilityThreshold()), + blobstor.WithStorages(getSubStorages(ctx, sh)), + blobstor.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + } + + return result +} + +func getSubStorages(ctx context.Context, sh *shardconfig.Config) []blobstor.SubStorage { + var ss []blobstor.SubStorage + for _, storage := range sh.BlobStor().Storages() { + switch storage.Type() { + case blobovniczatree.Type: + sub := blobovniczaconfig.From((*config.Config)(storage)) + blobTreeOpts := []blobovniczatree.Option{ + blobovniczatree.WithRootPath(storage.Path()), + blobovniczatree.WithPermissions(storage.Perm()), + blobovniczatree.WithBlobovniczaSize(sub.Size()), + blobovniczatree.WithBlobovniczaShallowDepth(sub.ShallowDepth()), + blobovniczatree.WithBlobovniczaShallowWidth(sub.ShallowWidth()), + blobovniczatree.WithOpenedCacheSize(sub.OpenedCacheSize()), + blobovniczatree.WithOpenedCacheTTL(sub.OpenedCacheTTL()), + blobovniczatree.WithOpenedCacheExpInterval(sub.OpenedCacheExpInterval()), + blobovniczatree.WithInitWorkerCount(sub.InitWorkerCount()), + blobovniczatree.WithWaitBeforeDropDB(sub.RebuildDropTimeout()), + blobovniczatree.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + blobovniczatree.WithObjectSizeLimit(sh.SmallSizeLimit()), + } + + ss = append(ss, blobstor.SubStorage{ + Storage: blobovniczatree.NewBlobovniczaTree(ctx, blobTreeOpts...), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return uint64(len(data)) < sh.SmallSizeLimit() + }, + }) + case fstree.Type: + sub := fstreeconfig.From((*config.Config)(storage)) + fstreeOpts := []fstree.Option{ + fstree.WithPath(storage.Path()), + fstree.WithPerm(storage.Perm()), + fstree.WithDepth(sub.Depth()), + fstree.WithNoSync(sub.NoSync()), + fstree.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + } + + ss = append(ss, blobstor.SubStorage{ + Storage: fstree.New(fstreeOpts...), + Policy: func(_ *objectSDK.Object, _ []byte) bool { + return true + }, + }) + default: + // should never happen, that has already + // been handled: when the config was read + } + } + return ss +} + +type epochState struct{} + +func (epochState) CurrentEpoch() uint64 { + return 0 +} diff --git a/cmd/frostfs-adm/internal/modules/root.go b/cmd/frostfs-adm/internal/modules/root.go index e42204b7a..cc8225c7a 100644 --- a/cmd/frostfs-adm/internal/modules/root.go +++ b/cmd/frostfs-adm/internal/modules/root.go @@ -5,6 +5,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/commonflags" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/maintenance" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-adm/internal/modules/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" @@ -41,6 +42,7 @@ func init() { rootCmd.AddCommand(config.RootCmd) rootCmd.AddCommand(morph.RootCmd) rootCmd.AddCommand(metabase.RootCmd) + rootCmd.AddCommand(maintenance.RootCmd) rootCmd.AddCommand(autocomplete.Command("frostfs-adm")) rootCmd.AddCommand(gendoc.Command(rootCmd, gendoc.Options{})) From f7779adf71babe6c06a91172e3ef95c947c1f1a7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Apr 2025 15:32:23 +0300 Subject: [PATCH 1888/1943] [#1712] core: Extend object info string with EC header Closes #1712 Change-Id: Ief4a960f7dece3359763113270d1ff5155f3f19e Signed-off-by: Dmitrii Stepanov --- pkg/core/object/info.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/core/object/info.go b/pkg/core/object/info.go index 67c9a3188..aab12ebf9 100644 --- a/pkg/core/object/info.go +++ b/pkg/core/object/info.go @@ -13,6 +13,13 @@ type ECInfo struct { Total uint32 } +func (v *ECInfo) String() string { + if v == nil { + return "" + } + return fmt.Sprintf("parent ID: %s, index: %d, total %d", v.ParentID, v.Index, v.Total) +} + // Info groups object address with its FrostFS // object info. type Info struct { @@ -23,5 +30,5 @@ type Info struct { } func (v Info) String() string { - return fmt.Sprintf("address: %s, type: %s, is linking: %t", v.Address, v.Type, v.IsLinkingObject) + return fmt.Sprintf("address: %s, type: %s, is linking: %t, EC header: %s", v.Address, v.Type, v.IsLinkingObject, v.ECInfo) } From 6730e27ae71a97af5ca33392ce6ab5e9e3257173 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 9 Apr 2025 16:29:48 +0300 Subject: [PATCH 1889/1943] [#1712] adm: Drop rpc-endpoint flag from `zombie scan` Morph addresses from config are used. Change-Id: Id99f91defbbff442c308f30d219b9824b4c871de Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go index 9ef18f7f8..c8fd9e5e5 100644 --- a/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/root.go @@ -32,7 +32,6 @@ var ( Short: "Scan storage engine for zombie objects and move them to quarantine", Long: "", PreRun: func(cmd *cobra.Command, _ []string) { - _ = viper.BindPFlag(commonflags.EndpointFlag, cmd.Flags().Lookup(commonflags.EndpointFlag)) _ = viper.BindPFlag(commonflags.ConfigFlag, cmd.Flags().Lookup(commonflags.ConfigFlag)) _ = viper.BindPFlag(commonflags.ConfigDirFlag, cmd.Flags().Lookup(commonflags.ConfigDirFlag)) _ = viper.BindPFlag(walletFlag, cmd.Flags().Lookup(walletFlag)) @@ -89,7 +88,6 @@ func init() { func initScanCmd() { Cmd.AddCommand(scanCmd) - scanCmd.Flags().StringP(commonflags.EndpointFlag, commonflags.EndpointFlagShort, "", commonflags.EndpointFlagDesc) scanCmd.Flags().StringP(commonflags.ConfigFlag, commonflags.ConfigFlagShorthand, "", commonflags.ConfigFlagUsage) scanCmd.Flags().String(commonflags.ConfigDirFlag, "", commonflags.ConfigDirFlagUsage) scanCmd.Flags().Uint32(flagBatchSize, 1000, flagBatchSizeUsage) From dcfd89544925719560343521ceb9a792d195cb51 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 8 Apr 2025 10:21:14 +0300 Subject: [PATCH 1890/1943] [#1710] object: Implement `Unwrap()` for `errIncompletePut` * When sign service calls `SignResponse`, it tries to set v2 status to response by unwrapping an error to the possible depth. This wasn't applicable for `errIncompletePut` so far as it didn't implement `Unwrap()`. Thus, it wasn't able to find a correct status set in error. Change-Id: I280c1806a008176854c55f13bf8688e5736ef941 Signed-off-by: Airat Arifullin --- pkg/services/object/common/writer/distributed.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/services/object/common/writer/distributed.go b/pkg/services/object/common/writer/distributed.go index f7486eae7..fff58aca7 100644 --- a/pkg/services/object/common/writer/distributed.go +++ b/pkg/services/object/common/writer/distributed.go @@ -95,6 +95,10 @@ func (x errIncompletePut) Error() string { return commonMsg } +func (x errIncompletePut) Unwrap() error { + return x.singleErr +} + // WriteObject implements the transformer.ObjectWriter interface. func (t *distributedWriter) WriteObject(ctx context.Context, obj *objectSDK.Object) error { t.obj = obj From 64c1392513f43b95bbe3f2afb6bab0a8edd03f89 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 8 Apr 2025 10:45:46 +0300 Subject: [PATCH 1891/1943] [#1710] object: Sign response even if `CloseAndRecv` returns error * Sign service wraps an error with status and sign a response even if error occurs from `CloseAndRecv` in `Put` and `Patch` methods. Close #1710 Change-Id: I7e1d8fe00db53607fa6e04ebec9a29b87349f8a1 Signed-off-by: Airat Arifullin --- pkg/services/object/sign.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/services/object/sign.go b/pkg/services/object/sign.go index 2b44227a5..fd8e926dd 100644 --- a/pkg/services/object/sign.go +++ b/pkg/services/object/sign.go @@ -96,7 +96,8 @@ func (s *putStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.PutRes } else { resp, err = s.stream.CloseAndRecv(ctx) if err != nil { - return nil, fmt.Errorf("could not close stream and receive response: %w", err) + err = fmt.Errorf("could not close stream and receive response: %w", err) + resp = new(object.PutResponse) } } @@ -132,7 +133,8 @@ func (s *patchStreamSigner) CloseAndRecv(ctx context.Context) (resp *object.Patc } else { resp, err = s.stream.CloseAndRecv(ctx) if err != nil { - return nil, fmt.Errorf("could not close stream and receive response: %w", err) + err = fmt.Errorf("could not close stream and receive response: %w", err) + resp = new(object.PatchResponse) } } From e06ecacf57e14630f1cd783af0c88854c26f41cc Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 10 Apr 2025 16:04:04 +0300 Subject: [PATCH 1892/1943] [#1705] engine: Use condition var for evacuation unit tests To know exactly when the evacuation was completed, a conditional variable was added. Closes #1705 Change-Id: I86f6d7d2ad2b9759905b6b5e9341008cb74f5dfd Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/engine.go | 16 +++++++++++----- .../engine/evacuate_limiter.go | 6 +++++- .../engine/evacuate_test.go | 18 +++++++++++++----- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index a915c9bd6..1d33c0592 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -212,12 +212,18 @@ func New(opts ...Option) *StorageEngine { opts[i](c) } + evLimMtx := &sync.RWMutex{} + evLimCond := sync.NewCond(evLimMtx) + return &StorageEngine{ - cfg: c, - shards: make(map[string]hashedShard), - closeCh: make(chan struct{}), - setModeCh: make(chan setModeRequest), - evacuateLimiter: &evacuationLimiter{}, + cfg: c, + shards: make(map[string]hashedShard), + closeCh: make(chan struct{}), + setModeCh: make(chan setModeRequest), + evacuateLimiter: &evacuationLimiter{ + guard: evLimMtx, + statusCond: evLimCond, + }, } } diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go index 3dd7494be..b75e8686d 100644 --- a/pkg/local_object_storage/engine/evacuate_limiter.go +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -139,7 +139,8 @@ type evacuationLimiter struct { eg *errgroup.Group cancel context.CancelFunc - guard sync.RWMutex + guard *sync.RWMutex + statusCond *sync.Cond // used in unit tests } func (l *evacuationLimiter) TryStart(ctx context.Context, shardIDs []string, result *EvacuateShardRes) (*errgroup.Group, context.Context, error) { @@ -165,6 +166,7 @@ func (l *evacuationLimiter) TryStart(ctx context.Context, shardIDs []string, res startedAt: time.Now().UTC(), result: result, } + l.statusCond.Broadcast() return l.eg, egCtx, nil } @@ -180,6 +182,7 @@ func (l *evacuationLimiter) Complete(err error) { l.state.processState = EvacuateProcessStateCompleted l.state.errMessage = errMsq l.state.finishedAt = time.Now().UTC() + l.statusCond.Broadcast() l.eg = nil } @@ -214,6 +217,7 @@ func (l *evacuationLimiter) ResetEvacuationStatus() error { l.state = EvacuationState{} l.eg = nil l.cancel = nil + l.statusCond.Broadcast() return nil } diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 359e49481..f2ba7d994 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -204,11 +204,10 @@ func TestEvacuateShardObjects(t *testing.T) { func testWaitForEvacuationCompleted(t *testing.T, e *StorageEngine) *EvacuationState { var st *EvacuationState var err error - require.Eventually(t, func() bool { - st, err = e.GetEvacuationState(context.Background()) - require.NoError(t, err) - return st.ProcessingStatus() == EvacuateProcessStateCompleted - }, 6*time.Second, 10*time.Millisecond) + e.evacuateLimiter.waitForCompleted() + st, err = e.GetEvacuationState(context.Background()) + require.NoError(t, err) + require.Equal(t, EvacuateProcessStateCompleted, st.ProcessingStatus()) return st } @@ -817,3 +816,12 @@ func TestEvacuateShardObjectsRepOneOnlyBench(t *testing.T) { t.Logf("evacuate took %v\n", time.Since(start)) require.NoError(t, err) } + +func (l *evacuationLimiter) waitForCompleted() { + l.guard.Lock() + defer l.guard.Unlock() + + for l.state.processState != EvacuateProcessStateCompleted { + l.statusCond.Wait() + } +} From dfe2f9956a82c6bee62e443e9e5178d6584ade64 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Mon, 24 Mar 2025 15:32:25 +0300 Subject: [PATCH 1893/1943] [#1619] logger: Filter entries by tags provided in config Change-Id: Ia2a79d6cb2a5eb263fb2e6db3f9cf9f2a7d57118 Signed-off-by: Anton Nikiforov --- cmd/frostfs-ir/config.go | 23 +++++- cmd/frostfs-ir/main.go | 4 ++ cmd/frostfs-node/config.go | 10 ++- cmd/frostfs-node/config/logger/config.go | 16 +++++ cmd/frostfs-node/validate.go | 5 ++ pkg/util/logger/logger.go | 92 +++++++++++++++++++----- pkg/util/logger/tag_string.go | 24 +++++++ pkg/util/logger/tags.go | 75 +++++++++++++++++++ 8 files changed, 231 insertions(+), 18 deletions(-) create mode 100644 pkg/util/logger/tag_string.go create mode 100644 pkg/util/logger/tags.go diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index 19b7f05d6..13a747ba6 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -4,12 +4,14 @@ import ( "context" "os" "os/signal" + "strconv" "syscall" configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" + "github.com/spf13/cast" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -44,11 +46,30 @@ func reloadConfig() error { if err != nil { return err } - log.Reload(logPrm) + err = logPrm.SetTags(loggerTags()) + if err != nil { + return err + } + logger.UpdateLevelForTags(logPrm) return nil } +func loggerTags() [][]string { + var res [][]string + for i := 0; ; i++ { + var item []string + index := strconv.FormatInt(int64(i), 10) + names := cast.ToString(cfg.Get("logger.tags." + index + ".names")) + if names == "" { + break + } + item = append(item, names, cast.ToString(cfg.Get("logger.tags."+index+".level"))) + res = append(res, item) + } + return res +} + func watchForSignal(ctx context.Context, cancel func()) { ch := make(chan os.Signal, 1) signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 114d8e4de..799feb784 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -80,10 +80,14 @@ func main() { exitErr(err) logPrm.SamplingHook = metrics.LogMetrics().GetSamplingHook() logPrm.PrependTimestamp = cfg.GetBool("logger.timestamp") + err = logPrm.SetTags(loggerTags()) + exitErr(err) log, err = logger.NewLogger(logPrm) exitErr(err) + logger.UpdateLevelForTags(logPrm) + ctx, cancel := context.WithCancel(context.Background()) pprofCmp = newPprofComponent() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 431316258..88e45d848 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -109,6 +109,7 @@ type applicationConfiguration struct { destination string timestamp bool options []zap.Option + tags [][]string } ObjectCfg struct { @@ -241,6 +242,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { })} } a.LoggerCfg.options = opts + a.LoggerCfg.tags = loggerconfig.Tags(c) // Object @@ -727,6 +729,7 @@ func initCfg(appCfg *config.Config) *cfg { logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook() log, err := logger.NewLogger(logPrm) fatalOnErr(err) + logger.UpdateLevelForTags(logPrm) c.internals = initInternals(appCfg, log) @@ -1094,6 +1097,11 @@ func (c *cfg) loggerPrm() (logger.Prm, error) { } prm.PrependTimestamp = c.LoggerCfg.timestamp prm.Options = c.LoggerCfg.options + err = prm.SetTags(c.LoggerCfg.tags) + if err != nil { + // not expected since validation should be performed before + return logger.Prm{}, errors.New("incorrect allowed tags format: " + c.LoggerCfg.destination) + } return prm, nil } @@ -1381,7 +1389,7 @@ func (c *cfg) getComponents(ctx context.Context) []dCmp { if err != nil { return err } - c.log.Reload(prm) + logger.UpdateLevelForTags(prm) return nil }}) components = append(components, dCmp{"runtime", func() error { diff --git a/cmd/frostfs-node/config/logger/config.go b/cmd/frostfs-node/config/logger/config.go index ba9eeea2b..20f373184 100644 --- a/cmd/frostfs-node/config/logger/config.go +++ b/cmd/frostfs-node/config/logger/config.go @@ -2,6 +2,7 @@ package loggerconfig import ( "os" + "strconv" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" @@ -60,6 +61,21 @@ func Timestamp(c *config.Config) bool { return config.BoolSafe(c.Sub(subsection), "timestamp") } +// Tags returns the value of "tags" config parameter from "logger" section. +func Tags(c *config.Config) [][]string { + var res [][]string + sub := c.Sub(subsection).Sub("tags") + for i := 0; ; i++ { + s := sub.Sub(strconv.FormatInt(int64(i), 10)) + names := config.StringSafe(s, "names") + if names == "" { + break + } + res = append(res, []string{names, config.StringSafe(s, "level")}) + } + return res +} + // ToLokiConfig extracts loki config. func ToLokiConfig(c *config.Config) loki.Config { hostname, _ := os.Hostname() diff --git a/cmd/frostfs-node/validate.go b/cmd/frostfs-node/validate.go index ae52b9e4a..22d2e0aa9 100644 --- a/cmd/frostfs-node/validate.go +++ b/cmd/frostfs-node/validate.go @@ -30,6 +30,11 @@ func validateConfig(c *config.Config) error { return fmt.Errorf("invalid logger destination: %w", err) } + err = loggerPrm.SetTags(loggerconfig.Tags(c)) + if err != nil { + return fmt.Errorf("invalid list of allowed tags: %w", err) + } + // shard configuration validation shardNum := 0 diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index 10c7e8dc9..276847be1 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -13,8 +13,10 @@ import ( // Logger represents a component // for writing messages to log. type Logger struct { - z *zap.Logger - lvl zap.AtomicLevel + z *zap.Logger + c zapcore.Core + t Tag + w bool } // Prm groups Logger's parameters. @@ -39,6 +41,9 @@ type Prm struct { // Options for zap.Logger Options []zap.Option + + // map of tag's bit masks to log level, overrides lvl + tl map[Tag]zapcore.Level } const ( @@ -68,6 +73,12 @@ func (p *Prm) SetDestination(d string) error { return nil } +// SetTags parses list of tags with log level. +func (p *Prm) SetTags(tags [][]string) (err error) { + p.tl, err = parseTags(tags) + return err +} + // NewLogger constructs a new zap logger instance. Constructing with nil // parameters is safe: default values will be used then. // Passing non-nil parameters after a successful creation (non-error) allows @@ -91,10 +102,8 @@ func NewLogger(prm Prm) (*Logger, error) { } func newConsoleLogger(prm Prm) (*Logger, error) { - lvl := zap.NewAtomicLevelAt(prm.level) - c := zap.NewProductionConfig() - c.Level = lvl + c.Level = zap.NewAtomicLevelAt(zap.DebugLevel) c.Encoding = "console" if prm.SamplingHook != nil { c.Sampling.Hook = prm.SamplingHook @@ -115,15 +124,13 @@ func newConsoleLogger(prm Prm) (*Logger, error) { if err != nil { return nil, err } - - l := &Logger{z: lZap, lvl: lvl} + l := &Logger{z: lZap, c: lZap.Core()} + l = l.WithTag(TagMain) return l, nil } func newJournaldLogger(prm Prm) (*Logger, error) { - lvl := zap.NewAtomicLevelAt(prm.level) - c := zap.NewProductionConfig() if prm.SamplingHook != nil { c.Sampling.Hook = prm.SamplingHook @@ -137,7 +144,7 @@ func newJournaldLogger(prm Prm) (*Logger, error) { encoder := zapjournald.NewPartialEncoder(zapcore.NewConsoleEncoder(c.EncoderConfig), zapjournald.SyslogFields) - core := zapjournald.NewCore(lvl, encoder, &journald.Journal{}, zapjournald.SyslogFields) + core := zapjournald.NewCore(zap.NewAtomicLevelAt(zap.DebugLevel), encoder, &journald.Journal{}, zapjournald.SyslogFields) coreWithContext := core.With([]zapcore.Field{ zapjournald.SyslogFacility(zapjournald.LogDaemon), zapjournald.SyslogIdentifier(), @@ -161,22 +168,75 @@ func newJournaldLogger(prm Prm) (*Logger, error) { } opts = append(opts, prm.Options...) lZap := zap.New(samplingCore, opts...) - - l := &Logger{z: lZap, lvl: lvl} + l := &Logger{z: lZap, c: lZap.Core()} + l = l.WithTag(TagMain) return l, nil } -func (l *Logger) Reload(prm Prm) { - l.lvl.SetLevel(prm.level) +// With create a child logger with new fields, don't affect the parent. +// Throws panic if tag is unset. +func (l *Logger) With(fields ...zap.Field) *Logger { + if l.t == 0 { + panic("tag is unset") + } + c := *l + c.z = l.z.With(fields...) + // With called under the logger + c.w = true + return &c } -func (l *Logger) With(fields ...zap.Field) *Logger { - return &Logger{z: l.z.With(fields...)} +type core struct { + c zapcore.Core + l zap.AtomicLevel +} + +func (c *core) Enabled(lvl zapcore.Level) bool { + return c.l.Enabled(lvl) +} + +func (c *core) With(fields []zapcore.Field) zapcore.Core { + clone := *c + clone.c = clone.c.With(fields) + return &clone +} + +func (c *core) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + return c.c.Check(e, ce) +} + +func (c *core) Write(e zapcore.Entry, fields []zapcore.Field) error { + return c.c.Write(e, fields) +} + +func (c *core) Sync() error { + return c.c.Sync() +} + +// WithTag is an equivalent of calling [NewLogger] with the same parameters for the current logger. +// Throws panic if provided unsupported tag. +func (l *Logger) WithTag(tag Tag) *Logger { + if tag == 0 || tag > Tag(len(_Tag_index)-1) { + panic("unsupported tag " + tag.String()) + } + if l.w { + panic("unsupported operation for the logger's state") + } + c := *l + c.t = tag + c.z = l.z.WithOptions(zap.WrapCore(func(zapcore.Core) zapcore.Core { + return &core{ + c: l.c.With([]zap.Field{zap.String("tag", tag.String())}), + l: tagToLogLevel[tag], + } + })) + return &c } func NewLoggerWrapper(z *zap.Logger) *Logger { return &Logger{ z: z.WithOptions(zap.AddCallerSkip(1)), + t: TagMain, } } diff --git a/pkg/util/logger/tag_string.go b/pkg/util/logger/tag_string.go new file mode 100644 index 000000000..80cb712eb --- /dev/null +++ b/pkg/util/logger/tag_string.go @@ -0,0 +1,24 @@ +// Code generated by "stringer -type Tag -linecomment"; DO NOT EDIT. + +package logger + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[TagMain-1] +} + +const _Tag_name = "main" + +var _Tag_index = [...]uint8{0, 4} + +func (i Tag) String() string { + i -= 1 + if i >= Tag(len(_Tag_index)-1) { + return "Tag(" + strconv.FormatInt(int64(i+1), 10) + ")" + } + return _Tag_name[_Tag_index[i]:_Tag_index[i+1]] +} diff --git a/pkg/util/logger/tags.go b/pkg/util/logger/tags.go new file mode 100644 index 000000000..06abcabe5 --- /dev/null +++ b/pkg/util/logger/tags.go @@ -0,0 +1,75 @@ +package logger + +import ( + "fmt" + "strings" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +//go:generate stringer -type Tag -linecomment + +type Tag uint8 + +const ( + _ Tag = iota // + TagMain // main + + defaultLevel = zapcore.InfoLevel +) + +var ( + tagToLogLevel = map[Tag]zap.AtomicLevel{} + stringToTag = map[string]Tag{} +) + +func init() { + for i := TagMain; i <= Tag(len(_Tag_index)-1); i++ { + tagToLogLevel[i] = zap.NewAtomicLevelAt(defaultLevel) + stringToTag[i.String()] = i + } +} + +// parseTags returns: +// - map(always instantiated) of tag to custom log level for that tag; +// - error if it occurred(map is empty). +func parseTags(raw [][]string) (map[Tag]zapcore.Level, error) { + m := make(map[Tag]zapcore.Level) + if len(raw) == 0 { + return m, nil + } + for _, item := range raw { + str, level := item[0], item[1] + if len(level) == 0 { + // It is not necessary to parse tags without level, + // because default log level will be used. + continue + } + var l zapcore.Level + err := l.UnmarshalText([]byte(level)) + if err != nil { + return nil, err + } + tmp := strings.Split(str, ",") + for _, tagStr := range tmp { + tag, ok := stringToTag[strings.TrimSpace(tagStr)] + if !ok { + return nil, fmt.Errorf("unsupported tag %s", str) + } + m[tag] = l + } + } + return m, nil +} + +func UpdateLevelForTags(prm Prm) { + for k, v := range tagToLogLevel { + nk, ok := prm.tl[k] + if ok { + v.SetLevel(nk) + } else { + v.SetLevel(prm.level) + } + } +} From 12fc7850dd78d6ea0818f90d90388de8e42d190c Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 29 Jan 2025 14:37:17 +0300 Subject: [PATCH 1894/1943] [#1619] logger: Set tags for `ir` components Change-Id: Ifab575bc2a3cd83c9001cd68fffaf94c91494043 Signed-off-by: Anton Nikiforov --- config/example/ir.env | 2 ++ config/example/ir.yaml | 3 +++ pkg/innerring/initialization.go | 16 ++++++++-------- pkg/innerring/innerring.go | 2 +- pkg/util/logger/tag_string.go | 8 ++++++-- pkg/util/logger/tags.go | 8 ++++++-- 6 files changed, 26 insertions(+), 13 deletions(-) diff --git a/config/example/ir.env b/config/example/ir.env index ebd91c243..c13044a6e 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -1,5 +1,7 @@ FROSTFS_IR_LOGGER_LEVEL=info FROSTFS_IR_LOGGER_TIMESTAMP=true +FROSTFS_IR_LOGGER_TAGS_0_NAMES="main, morph" +FROSTFS_IR_LOGGER_TAGS_0_LEVEL="debug" FROSTFS_IR_WALLET_PATH=/path/to/wallet.json FROSTFS_IR_WALLET_ADDRESS=NUHtW3eM6a4mmFCgyyr4rj4wygsTKB88XX diff --git a/config/example/ir.yaml b/config/example/ir.yaml index 49f9fd324..a4a006550 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -3,6 +3,9 @@ logger: level: info # Logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" timestamp: true + tags: + - names: "main, morph" # Possible values: `main`, `morph`, `grpc_svc`, `ir`, `processor`. + level: debug wallet: path: /path/to/wallet.json # Path to NEP-6 NEO wallet file diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index f7b71dbe6..3d236641e 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -50,7 +50,7 @@ func (s *Server) initNetmapProcessor(ctx context.Context, cfg *viper.Viper, var err error s.netmapProcessor, err = netmap.New(&netmap.Params{ - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, PoolSize: poolSize, NetmapClient: netmap.NewNetmapClient(s.netmapClient), @@ -159,7 +159,7 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli } else { // create governance processor governanceProcessor, err := governance.New(&governance.Params{ - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, FrostFSClient: frostfsCli, AlphabetState: s, @@ -225,7 +225,7 @@ func (s *Server) initAlphabetProcessor(ctx context.Context, cfg *viper.Viper) er // create alphabet processor s.alphabetProcessor, err = alphabet.New(&alphabet.Params{ ParsedWallets: parsedWallets, - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, PoolSize: poolSize, AlphabetContracts: s.contracts.alphabet, @@ -247,7 +247,7 @@ func (s *Server) initContainerProcessor(ctx context.Context, cfg *viper.Viper, c s.log.Debug(ctx, logs.ContainerContainerWorkerPool, zap.Int("size", poolSize)) // container processor containerProcessor, err := cont.New(&cont.Params{ - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, PoolSize: poolSize, AlphabetState: s, @@ -268,7 +268,7 @@ func (s *Server) initBalanceProcessor(ctx context.Context, cfg *viper.Viper, fro s.log.Debug(ctx, logs.BalanceBalanceWorkerPool, zap.Int("size", poolSize)) // create balance processor balanceProcessor, err := balance.New(&balance.Params{ - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, PoolSize: poolSize, FrostFSClient: frostfsCli, @@ -291,7 +291,7 @@ func (s *Server) initFrostFSMainnetProcessor(ctx context.Context, cfg *viper.Vip s.log.Debug(ctx, logs.FrostFSFrostfsWorkerPool, zap.Int("size", poolSize)) frostfsProcessor, err := frostfs.New(&frostfs.Params{ - Log: s.log, + Log: s.log.WithTag(logger.TagProcessor), Metrics: s.irMetrics, PoolSize: poolSize, FrostFSContract: s.contracts.frostfs, @@ -342,7 +342,7 @@ func (s *Server) initGRPCServer(ctx context.Context, cfg *viper.Viper, log *logg controlSvc := controlsrv.NewAuditService(controlsrv.New(p, s.netmapClient, s.containerClient, controlsrv.WithAllowedKeys(authKeys), - ), log, audit) + ), log.WithTag(logger.TagGrpcSvc), audit) grpcControlSrv := grpc.NewServer() control.RegisterControlServiceServer(grpcControlSrv, controlSvc) @@ -458,7 +458,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- } morphChain := &chainParams{ - log: s.log, + log: s.log.WithTag(logger.TagMorph), cfg: cfg, key: s.key, name: morphPrefix, diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index ae5661905..3a5137261 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -339,7 +339,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan ) (*Server, error) { var err error server := &Server{ - log: log, + log: log.WithTag(logger.TagIr), irMetrics: metrics, cmode: cmode, } diff --git a/pkg/util/logger/tag_string.go b/pkg/util/logger/tag_string.go index 80cb712eb..a25b3c445 100644 --- a/pkg/util/logger/tag_string.go +++ b/pkg/util/logger/tag_string.go @@ -9,11 +9,15 @@ func _() { // Re-run the stringer command to generate them again. var x [1]struct{} _ = x[TagMain-1] + _ = x[TagMorph-2] + _ = x[TagGrpcSvc-3] + _ = x[TagIr-4] + _ = x[TagProcessor-5] } -const _Tag_name = "main" +const _Tag_name = "mainmorphgrpc_svcirprocessor" -var _Tag_index = [...]uint8{0, 4} +var _Tag_index = [...]uint8{0, 4, 9, 17, 19, 28} func (i Tag) String() string { i -= 1 diff --git a/pkg/util/logger/tags.go b/pkg/util/logger/tags.go index 06abcabe5..51a6f62da 100644 --- a/pkg/util/logger/tags.go +++ b/pkg/util/logger/tags.go @@ -13,8 +13,12 @@ import ( type Tag uint8 const ( - _ Tag = iota // - TagMain // main + _ Tag = iota // + TagMain // main + TagMorph // morph + TagGrpcSvc // grpc_svc + TagIr // ir + TagProcessor // processor defaultLevel = zapcore.InfoLevel ) From 8e87cbee17d3374c55a536bc14f02acb752f9d5f Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Fri, 11 Apr 2025 14:44:19 +0300 Subject: [PATCH 1895/1943] [#1689] ci: Move commit checker out of Jenkinsfile Commit checker is now configured globally for all Gerrit repositories: https://git.frostfs.info/TrueCloudLab/jenkins/pulls/16 This allows us to execute commit-checker independently from the rest of CI suite and re-check commit message format without rerunning other tests. Change-Id: Ib8f899b856482a5dc5d03861171585415ff6b452 Signed-off-by: Vitaliy Potyarkin --- .ci/Jenkinsfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 4ddd36406..4234de160 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -78,10 +78,4 @@ async { } } } - - task('dco') { - container('git.frostfs.info/truecloudlab/commit-check:master') { - sh 'FROM=pull_request_target commit-check' - } - } } From 0d36e93169495b82e0cd23ff0c681c67c020c5e6 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 9 Apr 2025 16:27:03 +0300 Subject: [PATCH 1896/1943] [#1332] cli/playground: Move command handler selection to separate function Change-Id: I2dcbd85e61960c3cf141b815edab174e308ef858 Signed-off-by: Alexander Chuprov --- .../modules/container/policy_playground.go | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index f747b3252..a9dea13b1 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -260,6 +260,28 @@ Example of usage: }, } +func (repl *policyPlaygroundREPL) handleCommand(args []string) error { + if len(args) == 0 { + return nil + } + + switch args[0] { + case "list", "ls": + return repl.handleLs(args[1:]) + case "add": + return repl.handleAdd(args[1:]) + case "load": + return repl.handleLoad(args[1:]) + case "remove", "rm": + return repl.handleRemove(args[1:]) + case "eval": + return repl.handleEval(args[1:]) + case "help": + return repl.handleHelp(args[1:]) + } + return fmt.Errorf("unknown command %q", args[0]) +} + func (repl *policyPlaygroundREPL) run() error { if len(viper.GetString(commonflags.RPC)) > 0 { key := key.GetOrGenerate(repl.cmd) @@ -277,17 +299,6 @@ func (repl *policyPlaygroundREPL) run() error { } } - cmdHandlers := map[string]func([]string) error{ - "list": repl.handleLs, - "ls": repl.handleLs, - "add": repl.handleAdd, - "load": repl.handleLoad, - "remove": repl.handleRemove, - "rm": repl.handleRemove, - "eval": repl.handleEval, - "help": repl.handleHelp, - } - var cfgCompleter []readline.PrefixCompleterInterface var helpSubItems []readline.PrefixCompleterInterface @@ -326,17 +337,8 @@ func (repl *policyPlaygroundREPL) run() error { } exit = false - parts := strings.Fields(line) - if len(parts) == 0 { - continue - } - cmd := parts[0] - if handler, exists := cmdHandlers[cmd]; exists { - if err := handler(parts[1:]); err != nil { - fmt.Fprintf(repl.console, "error: %v\n", err) - } - } else { - fmt.Fprintf(repl.console, "error: unknown command %q\n", cmd) + if err := repl.handleCommand(strings.Fields(line)); err != nil { + fmt.Fprintf(repl.console, "error: %v\n", err) } } } From 29b4fbe451bb695724e529e757e339f235dd8759 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 9 Apr 2025 16:31:48 +0300 Subject: [PATCH 1897/1943] [#1332] cli/playground: Add 'netmap-config' flag Change-Id: I4342fb9a6da2a05c18ae4e0ad9f0c71550efc5ef Signed-off-by: Alexander Chuprov --- .../modules/container/policy_playground.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index a9dea13b1..377cf43c6 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -299,6 +299,11 @@ func (repl *policyPlaygroundREPL) run() error { } } + if len(viper.GetString(netmapConfigPath)) > 0 { + err := repl.handleLoad([]string{viper.GetString(netmapConfigPath)}) + commonCmd.ExitOnErr(repl.cmd, "load netmap config error: %w", err) + } + var cfgCompleter []readline.PrefixCompleterInterface var helpSubItems []readline.PrefixCompleterInterface @@ -354,6 +359,14 @@ If a wallet and endpoint is provided, the initial netmap data will be loaded fro }, } +const ( + netmapConfigPath = "netmap-config" + netmapConfigUsage = "Path to the netmap configuration file" +) + func initContainerPolicyPlaygroundCmd() { commonflags.Init(policyPlaygroundCmd) + policyPlaygroundCmd.Flags().String(netmapConfigPath, "", netmapConfigUsage) + + _ = viper.BindPFlag(netmapConfigPath, policyPlaygroundCmd.Flags().Lookup(netmapConfigPath)) } From 3be33b7117900ba1dc715e330545ca6ff5d44dc4 Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Wed, 9 Apr 2025 16:34:16 +0300 Subject: [PATCH 1898/1943] [#1706] cli/playground: Mention 'help' in error message for invalid commands Change-Id: Ica1112b907919a6d19fa1bf683f2a952c4c638e4 Signed-off-by: Alexander Chuprov --- cmd/frostfs-cli/modules/container/policy_playground.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 377cf43c6..2cc1107ef 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -279,7 +279,7 @@ func (repl *policyPlaygroundREPL) handleCommand(args []string) error { case "help": return repl.handleHelp(args[1:]) } - return fmt.Errorf("unknown command %q", args[0]) + return fmt.Errorf("unknown command %q. See 'help' for assistance", args[0]) } func (repl *policyPlaygroundREPL) run() error { From 5aaa3df533c02763db0dfef0ad5fd63cb0bebc46 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 11 Apr 2025 14:56:07 +0300 Subject: [PATCH 1899/1943] [#1700] config: Move config struct to qos package Change-Id: Ie642fff5cd1702cda00425628e11f3fd8c514798 Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 2 +- cmd/frostfs-node/config/engine/config_test.go | 36 ++++++---- .../config/engine/shard/limits/config.go | 66 ++++++------------- config/example/node.env | 6 ++ config/example/node.json | 10 +++ config/example/node.yaml | 6 ++ internal/qos/config.go | 31 +++++++++ internal/qos/limiter.go | 15 ++--- internal/qos/validate.go | 12 ++-- 9 files changed, 109 insertions(+), 75 deletions(-) create mode 100644 internal/qos/config.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 88e45d848..298049158 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -385,7 +385,7 @@ func (a *applicationConfiguration) setGCConfig(target *shardCfg, source *shardco } func (a *applicationConfiguration) setLimiter(target *shardCfg, source *shardconfig.Config) error { - limitsConfig := source.Limits() + limitsConfig := source.Limits().ToConfig() limiter, err := qos.NewLimiter(limitsConfig) if err != nil { return err diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 22f26268d..34613ad9e 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -11,10 +11,10 @@ import ( blobovniczaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza" fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" gcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/gc" - limitsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" ) @@ -135,8 +135,8 @@ func TestEngineSection(t *testing.T) { require.Equal(t, mode.ReadOnly, sc.Mode()) require.Equal(t, 100, sc.RefillMetabaseWorkersCount()) - readLimits := limits.Read() - writeLimits := limits.Write() + readLimits := limits.ToConfig().Read + writeLimits := limits.ToConfig().Write require.Equal(t, 30*time.Second, readLimits.IdleTimeout) require.Equal(t, int64(10_000), readLimits.MaxRunningOps) require.Equal(t, int64(1_000), readLimits.MaxWaitingOps) @@ -144,7 +144,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, int64(1_000), writeLimits.MaxRunningOps) require.Equal(t, int64(100), writeLimits.MaxWaitingOps) require.ElementsMatch(t, readLimits.Tags, - []limitsconfig.IOTagConfig{ + []qos.IOTagConfig{ { Tag: "internal", Weight: toPtr(20), @@ -173,9 +173,14 @@ func TestEngineSection(t *testing.T) { LimitOps: toPtr(25000), Prohibited: true, }, + { + Tag: "treesync", + Weight: toPtr(5), + LimitOps: toPtr(25), + }, }) require.ElementsMatch(t, writeLimits.Tags, - []limitsconfig.IOTagConfig{ + []qos.IOTagConfig{ { Tag: "internal", Weight: toPtr(200), @@ -203,6 +208,11 @@ func TestEngineSection(t *testing.T) { Weight: toPtr(50), LimitOps: toPtr(2500), }, + { + Tag: "treesync", + Weight: toPtr(50), + LimitOps: toPtr(100), + }, }) case 1: require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path()) @@ -259,14 +269,14 @@ func TestEngineSection(t *testing.T) { require.Equal(t, mode.ReadWrite, sc.Mode()) require.Equal(t, shardconfig.RefillMetabaseWorkersCountDefault, sc.RefillMetabaseWorkersCount()) - readLimits := limits.Read() - writeLimits := limits.Write() - require.Equal(t, limitsconfig.DefaultIdleTimeout, readLimits.IdleTimeout) - require.Equal(t, limitsconfig.NoLimit, readLimits.MaxRunningOps) - require.Equal(t, limitsconfig.NoLimit, readLimits.MaxWaitingOps) - require.Equal(t, limitsconfig.DefaultIdleTimeout, writeLimits.IdleTimeout) - require.Equal(t, limitsconfig.NoLimit, writeLimits.MaxRunningOps) - require.Equal(t, limitsconfig.NoLimit, writeLimits.MaxWaitingOps) + readLimits := limits.ToConfig().Read + writeLimits := limits.ToConfig().Write + require.Equal(t, qos.DefaultIdleTimeout, readLimits.IdleTimeout) + require.Equal(t, qos.NoLimit, readLimits.MaxRunningOps) + require.Equal(t, qos.NoLimit, readLimits.MaxWaitingOps) + require.Equal(t, qos.DefaultIdleTimeout, writeLimits.IdleTimeout) + require.Equal(t, qos.NoLimit, writeLimits.MaxRunningOps) + require.Equal(t, qos.NoLimit, writeLimits.MaxWaitingOps) require.Equal(t, 0, len(readLimits.Tags)) require.Equal(t, 0, len(writeLimits.Tags)) } diff --git a/cmd/frostfs-node/config/engine/shard/limits/config.go b/cmd/frostfs-node/config/engine/shard/limits/config.go index 8444d6aa8..ccd1e0000 100644 --- a/cmd/frostfs-node/config/engine/shard/limits/config.go +++ b/cmd/frostfs-node/config/engine/shard/limits/config.go @@ -1,19 +1,13 @@ package limits import ( - "math" "strconv" - "time" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "github.com/spf13/cast" ) -const ( - NoLimit int64 = math.MaxInt64 - DefaultIdleTimeout = 5 * time.Minute -) - // From wraps config section into Config. func From(c *config.Config) *Config { return (*Config)(c) @@ -23,36 +17,43 @@ func From(c *config.Config) *Config { // which provides access to Shard's limits configurations. type Config config.Config -// Read returns the value of "read" limits config section. -func (x *Config) Read() OpConfig { +func (x *Config) ToConfig() qos.LimiterConfig { + result := qos.LimiterConfig{ + Read: x.read(), + Write: x.write(), + } + panicOnErr(result.Validate()) + return result +} + +func (x *Config) read() qos.OpConfig { return x.parse("read") } -// Write returns the value of "write" limits config section. -func (x *Config) Write() OpConfig { +func (x *Config) write() qos.OpConfig { return x.parse("write") } -func (x *Config) parse(sub string) OpConfig { +func (x *Config) parse(sub string) qos.OpConfig { c := (*config.Config)(x).Sub(sub) - var result OpConfig + var result qos.OpConfig if s := config.Int(c, "max_waiting_ops"); s > 0 { result.MaxWaitingOps = s } else { - result.MaxWaitingOps = NoLimit + result.MaxWaitingOps = qos.NoLimit } if s := config.Int(c, "max_running_ops"); s > 0 { result.MaxRunningOps = s } else { - result.MaxRunningOps = NoLimit + result.MaxRunningOps = qos.NoLimit } if s := config.DurationSafe(c, "idle_timeout"); s > 0 { result.IdleTimeout = s } else { - result.IdleTimeout = DefaultIdleTimeout + result.IdleTimeout = qos.DefaultIdleTimeout } result.Tags = tags(c) @@ -60,43 +61,16 @@ func (x *Config) parse(sub string) OpConfig { return result } -type OpConfig struct { - // MaxWaitingOps returns the value of "max_waiting_ops" config parameter. - // - // Equals NoLimit if the value is not a positive number. - MaxWaitingOps int64 - // MaxRunningOps returns the value of "max_running_ops" config parameter. - // - // Equals NoLimit if the value is not a positive number. - MaxRunningOps int64 - // IdleTimeout returns the value of "idle_timeout" config parameter. - // - // Equals DefaultIdleTimeout if the value is not a valid duration. - IdleTimeout time.Duration - // Tags returns the value of "tags" config parameter. - // - // Equals nil if the value is not a valid tags config slice. - Tags []IOTagConfig -} - -type IOTagConfig struct { - Tag string - Weight *float64 - LimitOps *float64 - ReservedOps *float64 - Prohibited bool -} - -func tags(c *config.Config) []IOTagConfig { +func tags(c *config.Config) []qos.IOTagConfig { c = c.Sub("tags") - var result []IOTagConfig + var result []qos.IOTagConfig for i := 0; ; i++ { tag := config.String(c, strconv.Itoa(i)+".tag") if tag == "" { return result } - var tagConfig IOTagConfig + var tagConfig qos.IOTagConfig tagConfig.Tag = tag v := c.Value(strconv.Itoa(i) + ".weight") diff --git a/config/example/node.env b/config/example/node.env index b7c798ad8..dfb250341 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -181,6 +181,9 @@ FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_TAG=policer FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_WEIGHT=5 FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_LIMIT_OPS=25000 FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_4_PROHIBITED=true +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_5_TAG=treesync +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_5_WEIGHT=5 +FROSTFS_STORAGE_SHARD_0_LIMITS_READ_TAGS_5_LIMIT_OPS=25 FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_TAG=internal FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_WEIGHT=200 FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_0_LIMIT_OPS=0 @@ -198,6 +201,9 @@ FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_3_LIMIT_OPS=2500 FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_4_TAG=policer FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_4_WEIGHT=50 FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_4_LIMIT_OPS=2500 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_5_TAG=treesync +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_5_WEIGHT=50 +FROSTFS_STORAGE_SHARD_0_LIMITS_WRITE_TAGS_5_LIMIT_OPS=100 ## 1 shard ### Flag to refill Metabase from BlobStor diff --git a/config/example/node.json b/config/example/node.json index 2f4413e4d..0b061a3d4 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -254,6 +254,11 @@ "weight": 5, "limit_ops": 25000, "prohibited": true + }, + { + "tag": "treesync", + "weight": 5, + "limit_ops": 25 } ] }, @@ -288,6 +293,11 @@ "tag": "policer", "weight": 50, "limit_ops": 2500 + }, + { + "tag": "treesync", + "weight": 50, + "limit_ops": 100 } ] } diff --git a/config/example/node.yaml b/config/example/node.yaml index 0b6c7b12c..46e4ebdbe 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -253,6 +253,9 @@ storage: weight: 5 limit_ops: 25000 prohibited: true + - tag: treesync + weight: 5 + limit_ops: 25 write: max_running_ops: 1000 max_waiting_ops: 100 @@ -275,6 +278,9 @@ storage: - tag: policer weight: 50 limit_ops: 2500 + - tag: treesync + weight: 50 + limit_ops: 100 1: writecache: diff --git a/internal/qos/config.go b/internal/qos/config.go new file mode 100644 index 000000000..d90b403b5 --- /dev/null +++ b/internal/qos/config.go @@ -0,0 +1,31 @@ +package qos + +import ( + "math" + "time" +) + +const ( + NoLimit int64 = math.MaxInt64 + DefaultIdleTimeout = 5 * time.Minute +) + +type LimiterConfig struct { + Read OpConfig + Write OpConfig +} + +type OpConfig struct { + MaxWaitingOps int64 + MaxRunningOps int64 + IdleTimeout time.Duration + Tags []IOTagConfig +} + +type IOTagConfig struct { + Tag string + Weight *float64 + LimitOps *float64 + ReservedOps *float64 + Prohibited bool +} diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index c73481c2c..5851d7626 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -8,7 +8,6 @@ import ( "sync/atomic" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" "git.frostfs.info/TrueCloudLab/frostfs-qos/scheduling" "git.frostfs.info/TrueCloudLab/frostfs-qos/tagging" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" @@ -37,15 +36,15 @@ type scheduler interface { Close() } -func NewLimiter(c *limits.Config) (Limiter, error) { - if err := validateConfig(c); err != nil { +func NewLimiter(c LimiterConfig) (Limiter, error) { + if err := c.Validate(); err != nil { return nil, err } - readScheduler, err := createScheduler(c.Read()) + readScheduler, err := createScheduler(c.Read) if err != nil { return nil, fmt.Errorf("create read scheduler: %w", err) } - writeScheduler, err := createScheduler(c.Write()) + writeScheduler, err := createScheduler(c.Write) if err != nil { return nil, fmt.Errorf("create write scheduler: %w", err) } @@ -63,8 +62,8 @@ func NewLimiter(c *limits.Config) (Limiter, error) { return l, nil } -func createScheduler(config limits.OpConfig) (scheduler, error) { - if len(config.Tags) == 0 && config.MaxWaitingOps == limits.NoLimit { +func createScheduler(config OpConfig) (scheduler, error) { + if len(config.Tags) == 0 && config.MaxWaitingOps == NoLimit { return newSemaphoreScheduler(config.MaxRunningOps), nil } return scheduling.NewMClock( @@ -72,7 +71,7 @@ func createScheduler(config limits.OpConfig) (scheduler, error) { converToSchedulingTags(config.Tags), config.IdleTimeout) } -func converToSchedulingTags(limits []limits.IOTagConfig) map[string]scheduling.TagInfo { +func converToSchedulingTags(limits []IOTagConfig) map[string]scheduling.TagInfo { result := make(map[string]scheduling.TagInfo) for _, tag := range []IOTag{IOTagBackground, IOTagClient, IOTagInternal, IOTagPolicer, IOTagTreeSync, IOTagWritecache} { result[tag.String()] = scheduling.TagInfo{ diff --git a/internal/qos/validate.go b/internal/qos/validate.go index d4475e38b..70f1f24e8 100644 --- a/internal/qos/validate.go +++ b/internal/qos/validate.go @@ -4,8 +4,6 @@ import ( "errors" "fmt" "math" - - "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/limits" ) var errWeightsMustBeSpecified = errors.New("invalid weights: weights must be specified for all tags or not specified for any") @@ -14,17 +12,17 @@ type tagConfig struct { Shares, Limit, Reserved *float64 } -func validateConfig(c *limits.Config) error { - if err := validateOpConfig(c.Read()); err != nil { +func (c *LimiterConfig) Validate() error { + if err := validateOpConfig(c.Read); err != nil { return fmt.Errorf("limits 'read' section validation error: %w", err) } - if err := validateOpConfig(c.Write()); err != nil { + if err := validateOpConfig(c.Write); err != nil { return fmt.Errorf("limits 'write' section validation error: %w", err) } return nil } -func validateOpConfig(c limits.OpConfig) error { +func validateOpConfig(c OpConfig) error { if c.MaxRunningOps <= 0 { return fmt.Errorf("invalid 'max_running_ops = %d': must be greater than zero", c.MaxRunningOps) } @@ -40,7 +38,7 @@ func validateOpConfig(c limits.OpConfig) error { return nil } -func validateTags(configTags []limits.IOTagConfig) error { +func validateTags(configTags []IOTagConfig) error { tags := map[IOTag]tagConfig{ IOTagBackground: {}, IOTagClient: {}, From e80632884a2a89c6a5edfcb46cc1a939c9413028 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 11 Apr 2025 16:21:27 +0300 Subject: [PATCH 1900/1943] [#1700] config: Drop redundant check Target config created on level above, so limiter is always nil. Change-Id: I1896baae5b9ddeed339a7d2b022a9a886589d362 Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 298049158..b167439e0 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -390,9 +390,6 @@ func (a *applicationConfiguration) setLimiter(target *shardCfg, source *shardcon if err != nil { return err } - if target.limiter != nil { - target.limiter.Close() - } target.limiter = limiter return nil } From fd37cea443df4aad4b3a61db0e31fbfa0ae94c33 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 11 Apr 2025 17:08:44 +0300 Subject: [PATCH 1901/1943] [#1700] engine: Drop unused block execution methods `BlockExecution` and `ResumeExecution` were used only by unit test. So drop them and simplify code. Change-Id: Ib3de324617e8a27fc1f015542ac5e94df5c60a6e Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/control.go | 67 +++---------------- .../engine/control_test.go | 40 ----------- pkg/local_object_storage/engine/engine.go | 5 +- 3 files changed, 10 insertions(+), 102 deletions(-) diff --git a/pkg/local_object_storage/engine/control.go b/pkg/local_object_storage/engine/control.go index 96b53581e..bf1649f6e 100644 --- a/pkg/local_object_storage/engine/control.go +++ b/pkg/local_object_storage/engine/control.go @@ -22,10 +22,6 @@ type shardInitError struct { // Open opens all StorageEngine's components. func (e *StorageEngine) Open(ctx context.Context) error { - return e.open(ctx) -} - -func (e *StorageEngine) open(ctx context.Context) error { e.mtx.Lock() defer e.mtx.Unlock() @@ -149,11 +145,11 @@ var errClosed = errors.New("storage engine is closed") func (e *StorageEngine) Close(ctx context.Context) error { close(e.closeCh) defer e.wg.Wait() - return e.setBlockExecErr(ctx, errClosed) + return e.closeEngine(ctx) } // closes all shards. Never returns an error, shard errors are logged. -func (e *StorageEngine) close(ctx context.Context) error { +func (e *StorageEngine) closeAllShards(ctx context.Context) error { e.mtx.RLock() defer e.mtx.RUnlock() @@ -176,70 +172,23 @@ func (e *StorageEngine) execIfNotBlocked(op func() error) error { e.blockExec.mtx.RLock() defer e.blockExec.mtx.RUnlock() - if e.blockExec.err != nil { - return e.blockExec.err + if e.blockExec.closed { + return errClosed } return op() } -// sets the flag of blocking execution of all data operations according to err: -// - err != nil, then blocks the execution. If exec wasn't blocked, calls close method -// (if err == errClosed => additionally releases pools and does not allow to resume executions). -// - otherwise, resumes execution. If exec was blocked, calls open method. -// -// Can be called concurrently with exec. In this case it waits for all executions to complete. -func (e *StorageEngine) setBlockExecErr(ctx context.Context, err error) error { +func (e *StorageEngine) closeEngine(ctx context.Context) error { e.blockExec.mtx.Lock() defer e.blockExec.mtx.Unlock() - prevErr := e.blockExec.err - - wasClosed := errors.Is(prevErr, errClosed) - if wasClosed { + if e.blockExec.closed { return errClosed } - e.blockExec.err = err - - if err == nil { - if prevErr != nil { // block -> ok - return e.open(ctx) - } - } else if prevErr == nil { // ok -> block - return e.close(ctx) - } - - // otherwise do nothing - - return nil -} - -// BlockExecution blocks the execution of any data-related operation. All blocked ops will return err. -// To resume the execution, use ResumeExecution method. -// -// Сan be called regardless of the fact of the previous blocking. If execution wasn't blocked, releases all resources -// similar to Close. Can be called concurrently with Close and any data related method (waits for all executions -// to complete). Returns error if any Close has been called before. -// -// Must not be called concurrently with either Open or Init. -// -// Note: technically passing nil error will resume the execution, otherwise, it is recommended to call ResumeExecution -// for this. -func (e *StorageEngine) BlockExecution(err error) error { - return e.setBlockExecErr(context.Background(), err) -} - -// ResumeExecution resumes the execution of any data-related operation. -// To block the execution, use BlockExecution method. -// -// Сan be called regardless of the fact of the previous blocking. If execution was blocked, prepares all resources -// similar to Open. Can be called concurrently with Close and any data related method (waits for all executions -// to complete). Returns error if any Close has been called before. -// -// Must not be called concurrently with either Open or Init. -func (e *StorageEngine) ResumeExecution() error { - return e.setBlockExecErr(context.Background(), nil) + e.blockExec.closed = true + return e.closeAllShards(ctx) } type ReConfiguration struct { diff --git a/pkg/local_object_storage/engine/control_test.go b/pkg/local_object_storage/engine/control_test.go index a0e658aeb..4ff0ed5ec 100644 --- a/pkg/local_object_storage/engine/control_test.go +++ b/pkg/local_object_storage/engine/control_test.go @@ -2,7 +2,6 @@ package engine import ( "context" - "errors" "fmt" "io/fs" "os" @@ -12,17 +11,14 @@ import ( "testing" "time" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" - cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" ) @@ -163,42 +159,6 @@ func testEngineFailInitAndReload(t *testing.T, degradedMode bool, opts []shard.O require.Equal(t, 1, shardCount) } -func TestExecBlocks(t *testing.T) { - e := testNewEngine(t).setShardsNum(t, 2).prepare(t).engine // number doesn't matter in this test, 2 is several but not many - - // put some object - obj := testutil.GenerateObjectWithCID(cidtest.ID()) - - addr := object.AddressOf(obj) - - require.NoError(t, Put(context.Background(), e, obj, false)) - - // block executions - errBlock := errors.New("block exec err") - - require.NoError(t, e.BlockExecution(errBlock)) - - // try to exec some op - _, err := Head(context.Background(), e, addr) - require.ErrorIs(t, err, errBlock) - - // resume executions - require.NoError(t, e.ResumeExecution()) - - _, err = Head(context.Background(), e, addr) // can be any data-related op - require.NoError(t, err) - - // close - require.NoError(t, e.Close(context.Background())) - - // try exec after close - _, err = Head(context.Background(), e, addr) - require.Error(t, err) - - // try to resume - require.Error(t, e.ResumeExecution()) -} - func TestPersistentShardID(t *testing.T) { dir := t.TempDir() diff --git a/pkg/local_object_storage/engine/engine.go b/pkg/local_object_storage/engine/engine.go index 1d33c0592..376d545d3 100644 --- a/pkg/local_object_storage/engine/engine.go +++ b/pkg/local_object_storage/engine/engine.go @@ -33,9 +33,8 @@ type StorageEngine struct { wg sync.WaitGroup blockExec struct { - mtx sync.RWMutex - - err error + mtx sync.RWMutex + closed bool } evacuateLimiter *evacuationLimiter } From f37babdc5482334a3c573f09c7b43f43943d7dfa Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 14 Apr 2025 09:51:50 +0300 Subject: [PATCH 1902/1943] [#1700] shard: Lock shard's mode mutex on close To prevent race between GC handlers and close. Change-Id: I06219230964f000f666a56158d3563c760518c3b Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/control.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index 72e650c5e..d489b8b0d 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -363,6 +363,7 @@ func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object // Close releases all Shard's components. func (s *Shard) Close(ctx context.Context) error { + unlock := s.lockExclusive() if s.rb != nil { s.rb.Stop(ctx, s.log) } @@ -388,15 +389,19 @@ func (s *Shard) Close(ctx context.Context) error { } } + if s.opsLimiter != nil { + s.opsLimiter.Close() + } + + unlock() + + // GC waits for handlers and remover to complete. Handlers may try to lock shard's lock. + // So to prevent deadlock GC stopping is outside of exclusive lock. // If Init/Open was unsuccessful gc can be nil. if s.gc != nil { s.gc.stop(ctx) } - if s.opsLimiter != nil { - s.opsLimiter.Close() - } - return lastErr } From 48930ec452bc957a1919ee66f91606937cd08261 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Fri, 4 Apr 2025 16:59:22 +0300 Subject: [PATCH 1903/1943] [#1703] cli: Allow reading RPC endpoint from config file Allowed reading an RPC endpoint from a configuration file when getting current epoch in the `object lock` and `bearer create` commands. Close #1703 Change-Id: Iea8509dff2893a02cb63f695d7f532eecd743ed8 Signed-off-by: Aleksey Savchuk --- cmd/frostfs-cli/modules/bearer/create.go | 3 ++- cmd/frostfs-cli/modules/object/lock.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-cli/modules/bearer/create.go b/cmd/frostfs-cli/modules/bearer/create.go index a86506c37..0927788ba 100644 --- a/cmd/frostfs-cli/modules/bearer/create.go +++ b/cmd/frostfs-cli/modules/bearer/create.go @@ -44,6 +44,7 @@ is set to current epoch + n. _ = viper.BindPFlag(commonflags.WalletPath, ff.Lookup(commonflags.WalletPath)) _ = viper.BindPFlag(commonflags.Account, ff.Lookup(commonflags.Account)) + _ = viper.BindPFlag(commonflags.RPC, ff.Lookup(commonflags.RPC)) }, } @@ -81,7 +82,7 @@ func createToken(cmd *cobra.Command, _ []string) { commonCmd.ExitOnErr(cmd, "can't parse --"+notValidBeforeFlag+" flag: %w", err) if iatRelative || expRelative || nvbRelative { - endpoint, _ := cmd.Flags().GetString(commonflags.RPC) + endpoint := viper.GetString(commonflags.RPC) if len(endpoint) == 0 { commonCmd.ExitOnErr(cmd, "can't fetch current epoch: %w", fmt.Errorf("'%s' flag value must be specified", commonflags.RPC)) } diff --git a/cmd/frostfs-cli/modules/object/lock.go b/cmd/frostfs-cli/modules/object/lock.go index 53dd01868..d67db9f0d 100644 --- a/cmd/frostfs-cli/modules/object/lock.go +++ b/cmd/frostfs-cli/modules/object/lock.go @@ -18,6 +18,7 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/spf13/cobra" + "github.com/spf13/viper" ) // object lock command. @@ -78,7 +79,7 @@ var objectLockCmd = &cobra.Command{ ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - endpoint, _ := cmd.Flags().GetString(commonflags.RPC) + endpoint := viper.GetString(commonflags.RPC) currEpoch, err := internalclient.GetCurrentEpoch(ctx, cmd, endpoint) commonCmd.ExitOnErr(cmd, "Request current epoch: %w", err) From 0712c113dee45c81dfacb882f688c18ea39c1ee3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 15 Apr 2025 10:06:05 +0300 Subject: [PATCH 1904/1943] [#1700] gc: Fix deadlock `HandleExpiredLocks` gets read lock, then `shard.Close` tries to acquire write lock, but `HandleExpiredLocks` calls `inhumeUnlockedIfExpired` or `selectExpired`, that try to acquire read lock again. Change-Id: Ib2ed015e859328045b5a542a4f569e5e0ff8b05b Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/shard/gc.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 19b6e2d12..a262a52cb 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -391,6 +391,16 @@ func (s *Shard) handleExpiredObjects(ctx context.Context, expired []oid.Address) return } + s.handleExpiredObjectsUnsafe(ctx, expired) +} + +func (s *Shard) handleExpiredObjectsUnsafe(ctx context.Context, expired []oid.Address) { + select { + case <-ctx.Done(): + return + default: + } + expired, err := s.getExpiredWithLinked(ctx, expired) if err != nil { s.log.Warn(ctx, logs.ShardGCFailedToGetExpiredWithLinked, zap.Error(err)) @@ -611,13 +621,6 @@ func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, onExpiredFo } func (s *Shard) selectExpired(ctx context.Context, epoch uint64, addresses []oid.Address) ([]oid.Address, error) { - s.m.RLock() - defer s.m.RUnlock() - - if s.info.Mode.NoMetabase() { - return nil, ErrDegradedMode - } - release, err := s.opsLimiter.ReadRequest(ctx) if err != nil { return nil, err @@ -728,7 +731,7 @@ func (s *Shard) inhumeUnlockedIfExpired(ctx context.Context, epoch uint64, unloc return } - s.handleExpiredObjects(ctx, expiredUnlocked) + s.handleExpiredObjectsUnsafe(ctx, expiredUnlocked) } // HandleDeletedLocks unlocks all objects which were locked by lockers. From 56d09a99579a59b41f194f4c90a1dd582a9c88bb Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Fri, 7 Feb 2025 17:09:08 +0300 Subject: [PATCH 1905/1943] [#1640] object: Add priority metric based on geo distance Change-Id: I3a7ea4fc4807392bf50e6ff1389c61367c953074 Signed-off-by: Anton Nikiforov --- cmd/frostfs-node/config.go | 17 +- cmd/frostfs-node/config/node/config.go | 5 + config/example/node.env | 1 + config/example/node.json | 3 +- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 53 ++++--- .../object_manager/placement/metrics.go | 150 +++++++++++++++++- .../placement/traverser_test.go | 49 ++++++ 8 files changed, 241 insertions(+), 38 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index b167439e0..8ceef2c31 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -247,15 +247,16 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error { // Object a.ObjectCfg.tombstoneLifetime = objectconfig.TombstoneLifetime(c) - var pm []placement.Metric - for _, raw := range objectconfig.Get(c).Priority() { - m, err := placement.ParseMetric(raw) - if err != nil { - return err - } - pm = append(pm, m) + locodeDBPath := nodeconfig.LocodeDBPath(c) + parser, err := placement.NewMetricsParser(locodeDBPath) + if err != nil { + return fmt.Errorf("metrics parser creation: %w", err) } - a.ObjectCfg.priorityMetrics = pm + m, err := parser.ParseMetrics(objectconfig.Get(c).Priority()) + if err != nil { + return fmt.Errorf("parse metrics: %w", err) + } + a.ObjectCfg.priorityMetrics = m // Storage Engine diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index 969d77396..18aa254f1 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -217,3 +217,8 @@ func (l PersistentPolicyRulesConfig) NoSync() bool { func CompatibilityMode(c *config.Config) bool { return config.BoolSafe(c.Sub(subsection), "kludge_compatibility_mode") } + +// LocodeDBPath returns path to LOCODE database. +func LocodeDBPath(c *config.Config) string { + return config.String(c.Sub(subsection), "locode_db_path") +} diff --git a/config/example/node.env b/config/example/node.env index dfb250341..b501d3836 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -23,6 +23,7 @@ FROSTFS_NODE_ATTRIBUTE_1="UN-LOCODE:RU MSK" FROSTFS_NODE_RELAY=true FROSTFS_NODE_PERSISTENT_SESSIONS_PATH=/sessions FROSTFS_NODE_PERSISTENT_STATE_PATH=/state +FROSTFS_NODE_LOCODE_DB_PATH=/path/to/locode/db # Tree service section FROSTFS_TREE_ENABLED=true diff --git a/config/example/node.json b/config/example/node.json index 0b061a3d4..b02f43f60 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -37,7 +37,8 @@ }, "persistent_state": { "path": "/state" - } + }, + "locode_db_path": "/path/to/locode/db" }, "grpc": { "0": { diff --git a/config/example/node.yaml b/config/example/node.yaml index 46e4ebdbe..ba32adb82 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -36,6 +36,7 @@ node: path: /sessions # path to persistent session tokens file of Storage node (default: in-memory sessions) persistent_state: path: /state # path to persistent state file of Storage node + "locode_db_path": "/path/to/locode/db" grpc: - endpoint: s01.frostfs.devenv:8080 # endpoint for gRPC server diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 5fe011ece..248b54ea4 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -12,22 +12,23 @@ There are some custom types used for brevity: # Structure -| Section | Description | -|------------------------|---------------------------------------------------------------------| -| `logger` | [Logging parameters](#logger-section) | -| `pprof` | [PProf configuration](#pprof-section) | -| `prometheus` | [Prometheus metrics configuration](#prometheus-section) | -| `control` | [Control service configuration](#control-section) | -| `contracts` | [Override FrostFS contracts hashes](#contracts-section) | -| `morph` | [N3 blockchain client configuration](#morph-section) | -| `apiclient` | [FrostFS API client configuration](#apiclient-section) | -| `policer` | [Policer service configuration](#policer-section) | -| `replicator` | [Replicator service configuration](#replicator-section) | -| `storage` | [Storage engine configuration](#storage-section) | -| `runtime` | [Runtime configuration](#runtime-section) | -| `audit` | [Audit configuration](#audit-section) | -| `multinet` | [Multinet configuration](#multinet-section) | -| `qos` | [QoS configuration](#qos-section) | +| Section | Description | +|--------------|---------------------------------------------------------| +| `node` | [Node parameters](#node-section) | +| `logger` | [Logging parameters](#logger-section) | +| `pprof` | [PProf configuration](#pprof-section) | +| `prometheus` | [Prometheus metrics configuration](#prometheus-section) | +| `control` | [Control service configuration](#control-section) | +| `contracts` | [Override FrostFS contracts hashes](#contracts-section) | +| `morph` | [N3 blockchain client configuration](#morph-section) | +| `apiclient` | [FrostFS API client configuration](#apiclient-section) | +| `policer` | [Policer service configuration](#policer-section) | +| `replicator` | [Replicator service configuration](#replicator-section) | +| `storage` | [Storage engine configuration](#storage-section) | +| `runtime` | [Runtime configuration](#runtime-section) | +| `audit` | [Audit configuration](#audit-section) | +| `multinet` | [Multinet configuration](#multinet-section) | +| `qos` | [QoS configuration](#qos-section) | # `control` section ```yaml @@ -384,17 +385,19 @@ node: path: /sessions persistent_state: path: /state + locode_db_path: "/path/to/locode/db" ``` -| Parameter | Type | Default value | Description | -|-----------------------|---------------------------------------------------------------|---------------|-------------------------------------------------------------------------| -| `key` | `string` | | Path to the binary-encoded private key. | -| `wallet` | [Wallet config](#wallet-subsection) | | Wallet configuration. Has no effect if `key` is provided. | -| `addresses` | `[]string` | | Addresses advertised in the netmap. | -| `attribute` | `[]string` | | Node attributes as a list of key-value pairs in `:` format. | -| `relay` | `bool` | | Enable relay mode. | -| `persistent_sessions` | [Persistent sessions config](#persistent_sessions-subsection) | | Persistent session token store configuration. | -| `persistent_state` | [Persistent state config](#persistent_state-subsection) | | Persistent state configuration. | +| Parameter | Type | Default value | Description | +|-----------------------|---------------------------------------------------------------|---------------|-----------------------------------------------------------------------------------------------------| +| `key` | `string` | | Path to the binary-encoded private key. | +| `wallet` | [Wallet config](#wallet-subsection) | | Wallet configuration. Has no effect if `key` is provided. | +| `addresses` | `[]string` | | Addresses advertised in the netmap. | +| `attribute` | `[]string` | | Node attributes as a list of key-value pairs in `:` format. | +| `relay` | `bool` | | Enable relay mode. | +| `persistent_sessions` | [Persistent sessions config](#persistent_sessions-subsection) | | Persistent session token store configuration. | +| `persistent_state` | [Persistent state config](#persistent_state-subsection) | | Persistent state configuration. | +| `locode_db_path` | `string` | empty | Path to UN/LOCODE [database](https://git.frostfs.info/TrueCloudLab/frostfs-locode-db/) for FrostFS. | ## `wallet` subsection N3 wallet configuration. diff --git a/pkg/services/object_manager/placement/metrics.go b/pkg/services/object_manager/placement/metrics.go index 45e6df339..0f24a9d96 100644 --- a/pkg/services/object_manager/placement/metrics.go +++ b/pkg/services/object_manager/placement/metrics.go @@ -2,24 +2,90 @@ package placement import ( "errors" + "fmt" + "maps" + "math" "strings" + "sync" + "sync/atomic" + locodedb "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db" + locodebolt "git.frostfs.info/TrueCloudLab/frostfs-locode-db/pkg/locode/db/boltdb" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) const ( attrPrefix = "$attribute:" + + geoDistance = "$geoDistance" ) type Metric interface { CalculateValue(*netmap.NodeInfo, *netmap.NodeInfo) int } -func ParseMetric(raw string) (Metric, error) { - if attr, found := strings.CutPrefix(raw, attrPrefix); found { - return NewAttributeMetric(attr), nil +type metricsParser struct { + locodeDBPath string + locodes map[string]locodedb.Point +} + +type MetricParser interface { + ParseMetrics([]string) ([]Metric, error) +} + +func NewMetricsParser(locodeDBPath string) (MetricParser, error) { + return &metricsParser{ + locodeDBPath: locodeDBPath, + }, nil +} + +func (p *metricsParser) initLocodes() error { + if len(p.locodes) != 0 { + return nil } - return nil, errors.New("unsupported priority metric") + if len(p.locodeDBPath) > 0 { + p.locodes = make(map[string]locodedb.Point) + locodeDB := locodebolt.New(locodebolt.Prm{ + Path: p.locodeDBPath, + }, + locodebolt.ReadOnly(), + ) + err := locodeDB.Open() + if err != nil { + return err + } + defer locodeDB.Close() + err = locodeDB.IterateOverLocodes(func(k string, v locodedb.Point) { + p.locodes[k] = v + }) + if err != nil { + return err + } + return nil + } + return errors.New("set path to locode database") +} + +func (p *metricsParser) ParseMetrics(priority []string) ([]Metric, error) { + var metrics []Metric + for _, raw := range priority { + if attr, found := strings.CutPrefix(raw, attrPrefix); found { + metrics = append(metrics, NewAttributeMetric(attr)) + } else if raw == geoDistance { + err := p.initLocodes() + if err != nil { + return nil, err + } + if len(p.locodes) == 0 { + return nil, fmt.Errorf("provide locodes database for metric %s", raw) + } + m := NewGeoDistanceMetric(p.locodes) + metrics = append(metrics, m) + } else { + return nil, fmt.Errorf("unsupported priority metric %s", raw) + } + } + return metrics, nil } // attributeMetric describes priority metric based on attribute. @@ -41,3 +107,79 @@ func (am *attributeMetric) CalculateValue(from *netmap.NodeInfo, to *netmap.Node func NewAttributeMetric(attr string) Metric { return &attributeMetric{attribute: attr} } + +// geoDistanceMetric describes priority metric based on attribute. +type geoDistanceMetric struct { + locodes map[string]locodedb.Point + distance *atomic.Pointer[map[string]int] + mtx sync.Mutex +} + +func NewGeoDistanceMetric(locodes map[string]locodedb.Point) Metric { + d := atomic.Pointer[map[string]int]{} + m := make(map[string]int) + d.Store(&m) + gm := &geoDistanceMetric{ + locodes: locodes, + distance: &d, + } + return gm +} + +// CalculateValue return distance in kilometers between current node and provided, +// if coordinates for provided node found. In other case return math.MaxInt. +func (gm *geoDistanceMetric) CalculateValue(from *netmap.NodeInfo, to *netmap.NodeInfo) int { + fl := from.LOCODE() + tl := to.LOCODE() + if fl == tl { + return 0 + } + m := gm.distance.Load() + if v, ok := (*m)[fl+tl]; ok { + return v + } + return gm.calculateDistance(fl, tl) +} + +func (gm *geoDistanceMetric) calculateDistance(from, to string) int { + gm.mtx.Lock() + defer gm.mtx.Unlock() + od := gm.distance.Load() + if v, ok := (*od)[from+to]; ok { + return v + } + nd := maps.Clone(*od) + var dist int + pointFrom, okFrom := gm.locodes[from] + pointTo, okTo := gm.locodes[to] + if okFrom && okTo { + dist = int(distance(pointFrom.Latitude(), pointFrom.Longitude(), pointTo.Latitude(), pointTo.Longitude())) + } else { + dist = math.MaxInt + } + nd[from+to] = dist + gm.distance.Store(&nd) + + return dist +} + +// distance return amount of KM between two points. +// Parameters are latitude and longitude of point 1 and 2 in decimal degrees. +// Original implementation can be found here https://www.geodatasource.com/developers/go. +func distance(lt1 float64, ln1 float64, lt2 float64, ln2 float64) float64 { + radLat1 := math.Pi * lt1 / 180 + radLat2 := math.Pi * lt2 / 180 + radTheta := math.Pi * (ln1 - ln2) / 180 + + dist := math.Sin(radLat1)*math.Sin(radLat2) + math.Cos(radLat1)*math.Cos(radLat2)*math.Cos(radTheta) + + if dist > 1 { + dist = 1 + } + + dist = math.Acos(dist) + dist = dist * 180 / math.Pi + dist = dist * 60 * 1.1515 * 1.609344 + + return dist +} diff --git a/pkg/services/object_manager/placement/traverser_test.go b/pkg/services/object_manager/placement/traverser_test.go index 9c825bf19..d1370f21e 100644 --- a/pkg/services/object_manager/placement/traverser_test.go +++ b/pkg/services/object_manager/placement/traverser_test.go @@ -601,4 +601,53 @@ func TestTraverserPriorityMetrics(t *testing.T) { next = tr.Next() require.Nil(t, next) }) + + t.Run("one rep one geo metric", func(t *testing.T) { + t.Skip() + selectors := []int{2} + replicas := []int{2} + + nodes, cnr := testPlacement(selectors, replicas) + + // Node_0, PK - ip4/0.0.0.0/tcp/0 + nodes[0][0].SetAttribute("UN-LOCODE", "RU MOW") + // Node_1, PK - ip4/0.0.0.0/tcp/1 + nodes[0][1].SetAttribute("UN-LOCODE", "RU LED") + + sdkNode := testNode(2) + sdkNode.SetAttribute("UN-LOCODE", "FI HEL") + + nodesCopy := copyVectors(nodes) + + parser, err := NewMetricsParser("/path/to/locode_db") + require.NoError(t, err) + m, err := parser.ParseMetrics([]string{geoDistance}) + require.NoError(t, err) + + tr, err := NewTraverser(context.Background(), + ForContainer(cnr), + UseBuilder(&testBuilder{ + vectors: nodesCopy, + }), + WithoutSuccessTracking(), + WithPriorityMetrics(m), + WithNodeState(&nodeState{ + node: &sdkNode, + }), + ) + require.NoError(t, err) + + // Without priority metric `$geoDistance` the order will be: + // [ {Node_0 RU MOW}, {Node_1 RU LED}] + // With priority metric `$geoDistance` the order should be: + // [ {Node_1 RU LED}, {Node_0 RU MOW}] + next := tr.Next() + require.NotNil(t, next) + require.Equal(t, 2, len(next)) + require.Equal(t, "/ip4/0.0.0.0/tcp/1", string(next[0].PublicKey())) + require.Equal(t, "/ip4/0.0.0.0/tcp/0", string(next[1].PublicKey())) + + next = tr.Next() + require.Nil(t, next) + }) } From bf06c4fb4b935410a4ea9b610f4e12f2fe28cb2a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 20:22:05 +0300 Subject: [PATCH 1906/1943] [#1689] Remove deprecated NodeInfo.IterateNetworkEndpoints() Change-Id: Ic78f18aed11fab34ee3147ceea657296b89fe60c Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/netmap/nodeinfo.go | 4 +- cmd/frostfs-cli/modules/object/nodes.go | 11 ++---- cmd/frostfs-node/cache.go | 22 ++--------- cmd/frostfs-node/netmap.go | 6 ++- cmd/internal/common/netmap.go | 4 +- pkg/core/netmap/nodes.go | 6 ++- pkg/network/validation.go | 6 ++- pkg/services/replicator/pull.go | 7 +--- pkg/services/tree/redirect.go | 41 +++++++++++-------- pkg/services/tree/replicator.go | 46 ++++++++++++---------- pkg/services/tree/sync.go | 12 +++--- 11 files changed, 83 insertions(+), 82 deletions(-) diff --git a/cmd/frostfs-cli/modules/netmap/nodeinfo.go b/cmd/frostfs-cli/modules/netmap/nodeinfo.go index ae4bb329a..316d18d2b 100644 --- a/cmd/frostfs-cli/modules/netmap/nodeinfo.go +++ b/cmd/frostfs-cli/modules/netmap/nodeinfo.go @@ -62,9 +62,9 @@ func prettyPrintNodeInfo(cmd *cobra.Command, i netmap.NodeInfo) { cmd.Println("state:", stateWord) - netmap.IterateNetworkEndpoints(i, func(s string) { + for s := range i.NetworkEndpoints() { cmd.Println("address:", s) - }) + } i.IterateAttributes(func(key, value string) { cmd.Printf("attribute: %s=%s\n", key, value) diff --git a/cmd/frostfs-cli/modules/object/nodes.go b/cmd/frostfs-cli/modules/object/nodes.go index 734b557a4..476238651 100644 --- a/cmd/frostfs-cli/modules/object/nodes.go +++ b/cmd/frostfs-cli/modules/object/nodes.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "sync" internalclient "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/client" @@ -460,17 +461,11 @@ func createClient(ctx context.Context, cmd *cobra.Command, candidate netmapSDK.N var cli *client.Client var addresses []string if preferInternal, _ := cmd.Flags().GetBool(preferInternalAddressesFlag); preferInternal { - candidate.IterateNetworkEndpoints(func(s string) bool { - addresses = append(addresses, s) - return false - }) + addresses = slices.AppendSeq(addresses, candidate.NetworkEndpoints()) addresses = append(addresses, candidate.ExternalAddresses()...) } else { addresses = append(addresses, candidate.ExternalAddresses()...) - candidate.IterateNetworkEndpoints(func(s string) bool { - addresses = append(addresses, s) - return false - }) + addresses = slices.AppendSeq(addresses, candidate.NetworkEndpoints()) } var lastErr error diff --git a/cmd/frostfs-node/cache.go b/cmd/frostfs-node/cache.go index 38cee5837..e5df0a22d 100644 --- a/cmd/frostfs-node/cache.go +++ b/cmd/frostfs-node/cache.go @@ -235,16 +235,8 @@ func (s *lruNetmapSource) updateCandidates(ctx context.Context, d time.Duration) slices.Compare(n1.ExternalAddresses(), n2.ExternalAddresses()) != 0 { return 1 } - var ne1 []string - n1.IterateNetworkEndpoints(func(s string) bool { - ne1 = append(ne1, s) - return false - }) - var ne2 []string - n2.IterateNetworkEndpoints(func(s string) bool { - ne2 = append(ne2, s) - return false - }) + ne1 := slices.Collect(n1.NetworkEndpoints()) + ne2 := slices.Collect(n2.NetworkEndpoints()) return slices.Compare(ne1, ne2) }) if ret != 0 { @@ -364,15 +356,9 @@ func getNetMapNodesToUpdate(nm *netmapSDK.NetMap, candidates []netmapSDK.NodeInf } nodeEndpoints := make([]string, 0, nm.Nodes()[i].NumberOfNetworkEndpoints()) - nm.Nodes()[i].IterateNetworkEndpoints(func(s string) bool { - nodeEndpoints = append(nodeEndpoints, s) - return false - }) + nodeEndpoints = slices.AppendSeq(nodeEndpoints, nm.Nodes()[i].NetworkEndpoints()) candidateEndpoints := make([]string, 0, cnd.NumberOfNetworkEndpoints()) - cnd.IterateNetworkEndpoints(func(s string) bool { - candidateEndpoints = append(candidateEndpoints, s) - return false - }) + candidateEndpoints = slices.AppendSeq(candidateEndpoints, cnd.NetworkEndpoints()) if slices.Compare(nodeEndpoints, candidateEndpoints) != 0 { update = true tmp.endpoints = candidateEndpoints diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 0e90e7707..6d57edcce 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -124,7 +124,11 @@ func nodeKeyFromNetmap(c *cfg) []byte { func (c *cfg) iterateNetworkAddresses(f func(string) bool) { ni, ok := c.cfgNetmap.state.getNodeInfo() if ok { - ni.IterateNetworkEndpoints(f) + for s := range ni.NetworkEndpoints() { + if f(s) { + return + } + } } } diff --git a/cmd/internal/common/netmap.go b/cmd/internal/common/netmap.go index f550552d2..334b662f8 100644 --- a/cmd/internal/common/netmap.go +++ b/cmd/internal/common/netmap.go @@ -27,9 +27,9 @@ func PrettyPrintNodeInfo(cmd *cobra.Command, node netmap.NodeInfo, cmd.Printf("%sNode %d: %s %s ", indent, index+1, hex.EncodeToString(node.PublicKey()), strState) - netmap.IterateNetworkEndpoints(node, func(endpoint string) { + for endpoint := range node.NetworkEndpoints() { cmd.Printf("%s ", endpoint) - }) + } cmd.Println() if !short { diff --git a/pkg/core/netmap/nodes.go b/pkg/core/netmap/nodes.go index b0c9e1f9e..f01c07b19 100644 --- a/pkg/core/netmap/nodes.go +++ b/pkg/core/netmap/nodes.go @@ -17,7 +17,11 @@ func (x Node) PublicKey() []byte { // IterateAddresses iterates over all announced network addresses // and passes them into f. Handler MUST NOT be nil. func (x Node) IterateAddresses(f func(string) bool) { - (netmap.NodeInfo)(x).IterateNetworkEndpoints(f) + for s := range (netmap.NodeInfo)(x).NetworkEndpoints() { + if f(s) { + return + } + } } // NumberOfAddresses returns number of announced network addresses. diff --git a/pkg/network/validation.go b/pkg/network/validation.go index 92f650119..73a3ef8d7 100644 --- a/pkg/network/validation.go +++ b/pkg/network/validation.go @@ -35,7 +35,11 @@ var ( type NodeEndpointsIterator netmap.NodeInfo func (x NodeEndpointsIterator) IterateAddresses(f func(string) bool) { - (netmap.NodeInfo)(x).IterateNetworkEndpoints(f) + for s := range (netmap.NodeInfo)(x).NetworkEndpoints() { + if f(s) { + return + } + } } func (x NodeEndpointsIterator) NumberOfAddresses() int { diff --git a/pkg/services/replicator/pull.go b/pkg/services/replicator/pull.go index bb38c72ad..216fe4919 100644 --- a/pkg/services/replicator/pull.go +++ b/pkg/services/replicator/pull.go @@ -3,6 +3,7 @@ package replicator import ( "context" "errors" + "slices" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" @@ -42,11 +43,7 @@ func (p *Replicator) HandlePullTask(ctx context.Context, task Task) { if err == nil { break } - var endpoints []string - node.IterateNetworkEndpoints(func(s string) bool { - endpoints = append(endpoints, s) - return false - }) + endpoints := slices.Collect(node.NetworkEndpoints()) p.log.Error(ctx, logs.ReplicatorCouldNotGetObjectFromRemoteStorage, zap.Stringer("object", task.Addr), zap.Error(err), diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index d92c749a8..3dcdc4fc7 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -41,24 +41,15 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo var called bool for _, n := range cntNodes { var stop bool - n.IterateNetworkEndpoints(func(endpoint string) bool { - ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.IterateNetworkEndpoints", - trace.WithAttributes( - attribute.String("endpoint", endpoint), - )) - defer span.End() - - c, err := s.cache.get(ctx, endpoint) - if err != nil { - return false + for endpoint := range n.NetworkEndpoints() { + stop = s.execOnClient(ctx, endpoint, func(c TreeServiceClient) bool { + called = true + return f(c) + }) + if called { + break } - - s.log.Debug(ctx, logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint)) - - called = true - stop = f(c) - return true - }) + } if stop { return nil } @@ -68,3 +59,19 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo } return nil } + +func (s *Service) execOnClient(ctx context.Context, endpoint string, f func(TreeServiceClient) bool) bool { + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.IterateNetworkEndpoints", + trace.WithAttributes( + attribute.String("endpoint", endpoint), + )) + defer span.End() + + c, err := s.cache.get(ctx, endpoint) + if err != nil { + return false + } + + s.log.Debug(ctx, logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint)) + return f(c) +} diff --git a/pkg/services/tree/replicator.go b/pkg/services/tree/replicator.go index 01a4ffde0..ee40884eb 100644 --- a/pkg/services/tree/replicator.go +++ b/pkg/services/tree/replicator.go @@ -89,29 +89,13 @@ func (s *Service) ReplicateTreeOp(ctx context.Context, n netmapSDK.NodeInfo, req var lastErr error var lastAddr string - n.IterateNetworkEndpoints(func(addr string) bool { - ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.HandleReplicationTaskOnEndpoint", - trace.WithAttributes( - attribute.String("public_key", hex.EncodeToString(n.PublicKey())), - attribute.String("address", addr), - ), - ) - defer span.End() - + for addr := range n.NetworkEndpoints() { lastAddr = addr - - c, err := s.cache.get(ctx, addr) - if err != nil { - lastErr = fmt.Errorf("can't create client: %w", err) - return false + lastErr = s.apply(ctx, n, addr, req) + if lastErr == nil { + break } - - ctx, cancel := context.WithTimeout(ctx, s.replicatorTimeout) - _, lastErr = c.Apply(ctx, req) - cancel() - - return lastErr == nil - }) + } if lastErr != nil { if errors.Is(lastErr, errRecentlyFailed) { @@ -130,6 +114,26 @@ func (s *Service) ReplicateTreeOp(ctx context.Context, n netmapSDK.NodeInfo, req return nil } +func (s *Service) apply(ctx context.Context, n netmapSDK.NodeInfo, addr string, req *ApplyRequest) error { + ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.HandleReplicationTaskOnEndpoint", + trace.WithAttributes( + attribute.String("public_key", hex.EncodeToString(n.PublicKey())), + attribute.String("address", addr), + ), + ) + defer span.End() + + c, err := s.cache.get(ctx, addr) + if err != nil { + return fmt.Errorf("can't create client: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, s.replicatorTimeout) + _, err = c.Apply(ctx, req) + cancel() + return err +} + func (s *Service) replicateLoop(ctx context.Context) { for range s.replicatorWorkerCount { go s.replicationWorker(ctx) diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index c3796fbd4..32297f9d7 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -297,27 +297,27 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, for i, n := range nodes { errGroup.Go(func() error { var nodeSynced bool - n.IterateNetworkEndpoints(func(addr string) bool { + for addr := range n.NetworkEndpoints() { var a network.Address if err := a.FromString(addr); err != nil { s.log.Warn(ctx, logs.TreeFailedToParseAddressForTreeSynchronization, zap.Error(err), zap.String("address", addr)) - return false + continue } cc, err := createConnection(a, grpc.WithContextDialer(s.ds.GrpcContextDialer())) if err != nil { s.log.Warn(ctx, logs.TreeFailedToConnectForTreeSynchronization, zap.Error(err), zap.String("address", addr)) - return false + continue } - defer cc.Close() err = s.startStream(egCtx, cid, treeID, from, cc, nodeOperationStreams[i]) if err != nil { s.log.Warn(ctx, logs.TreeFailedToRunTreeSynchronizationForSpecificNode, zap.Error(err), zap.String("address", addr)) } nodeSynced = err == nil - return true - }) + _ = cc.Close() + break + } close(nodeOperationStreams[i]) if !nodeSynced { allNodesSynced.Store(false) From e65d578ba93f18c5d4276300fe660f01941f630c Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 20:24:46 +0300 Subject: [PATCH 1907/1943] [#1689] Remove deprecated NodeInfo.IterateAttributes() Change-Id: Ibd07302079efe148903aa6177759232a28616736 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-cli/modules/container/policy_playground.go | 4 ++-- cmd/frostfs-cli/modules/netmap/nodeinfo.go | 4 ++-- cmd/internal/common/netmap.go | 4 ++-- pkg/util/attributes/parser_test.go | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/policy_playground.go b/cmd/frostfs-cli/modules/container/policy_playground.go index 2cc1107ef..cf4862b4a 100644 --- a/cmd/frostfs-cli/modules/container/policy_playground.go +++ b/cmd/frostfs-cli/modules/container/policy_playground.go @@ -40,9 +40,9 @@ func (repl *policyPlaygroundREPL) handleLs(args []string) error { i := 1 for id, node := range repl.nodes { var attrs []string - node.IterateAttributes(func(k, v string) { + for k, v := range node.Attributes() { attrs = append(attrs, fmt.Sprintf("%s:%q", k, v)) - }) + } fmt.Fprintf(repl.console, "\t%2d: id=%s attrs={%v}\n", i, id, strings.Join(attrs, " ")) i++ } diff --git a/cmd/frostfs-cli/modules/netmap/nodeinfo.go b/cmd/frostfs-cli/modules/netmap/nodeinfo.go index 316d18d2b..5da66dcd9 100644 --- a/cmd/frostfs-cli/modules/netmap/nodeinfo.go +++ b/cmd/frostfs-cli/modules/netmap/nodeinfo.go @@ -66,7 +66,7 @@ func prettyPrintNodeInfo(cmd *cobra.Command, i netmap.NodeInfo) { cmd.Println("address:", s) } - i.IterateAttributes(func(key, value string) { + for key, value := range i.Attributes() { cmd.Printf("attribute: %s=%s\n", key, value) - }) + } } diff --git a/cmd/internal/common/netmap.go b/cmd/internal/common/netmap.go index 334b662f8..5dd1a060e 100644 --- a/cmd/internal/common/netmap.go +++ b/cmd/internal/common/netmap.go @@ -33,9 +33,9 @@ func PrettyPrintNodeInfo(cmd *cobra.Command, node netmap.NodeInfo, cmd.Println() if !short { - node.IterateAttributes(func(key, value string) { + for key, value := range node.Attributes() { cmd.Printf("%s\t%s: %s\n", indent, key, value) - }) + } } } diff --git a/pkg/util/attributes/parser_test.go b/pkg/util/attributes/parser_test.go index 547c8d50b..66581878a 100644 --- a/pkg/util/attributes/parser_test.go +++ b/pkg/util/attributes/parser_test.go @@ -23,12 +23,12 @@ func testAttributeMap(t *testing.T, mSrc, mExp map[string]string) { mExp = mSrc } - node.IterateAttributes(func(key, value string) { + for key, value := range node.Attributes() { v, ok := mExp[key] require.True(t, ok) require.Equal(t, value, v) delete(mExp, key) - }) + } require.Empty(t, mExp) } From 2d1232ce6de2f9291528c1c3e325bb8a0683de10 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Fri, 4 Apr 2025 20:31:35 +0300 Subject: [PATCH 1908/1943] [#1689] network,core/netmap: Replace Iterate*() functions with iterators Change-Id: I4842a3160d74c56d99ea9465d4be2f0662080605 Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config/node/config.go | 10 ++++------ pkg/core/client/util.go | 3 ++- pkg/core/netmap/nodes.go | 12 +++++++++++- pkg/network/group.go | 18 +++++++++--------- pkg/network/group_test.go | 8 ++++---- pkg/network/validation.go | 9 +++------ 6 files changed, 33 insertions(+), 27 deletions(-) diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index 18aa254f1..81b191e96 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -3,7 +3,9 @@ package nodeconfig import ( "fmt" "io/fs" + "iter" "os" + "slices" "strconv" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" @@ -88,12 +90,8 @@ func Wallet(c *config.Config) *keys.PrivateKey { type stringAddressGroup []string -func (x stringAddressGroup) IterateAddresses(f func(string) bool) { - for i := range x { - if f(x[i]) { - break - } - } +func (x stringAddressGroup) Addresses() iter.Seq[string] { + return slices.Values(x) } func (x stringAddressGroup) NumberOfAddresses() int { diff --git a/pkg/core/client/util.go b/pkg/core/client/util.go index d4bc0cf68..91ee5c6c3 100644 --- a/pkg/core/client/util.go +++ b/pkg/core/client/util.go @@ -3,6 +3,7 @@ package client import ( "bytes" "fmt" + "iter" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -19,7 +20,7 @@ func nodeInfoFromKeyAddr(dst *NodeInfo, k []byte, a, external network.AddressGro // Args must not be nil. func NodeInfoFromRawNetmapElement(dst *NodeInfo, info interface { PublicKey() []byte - IterateAddresses(func(string) bool) + Addresses() iter.Seq[string] NumberOfAddresses() int ExternalAddresses() []string }, diff --git a/pkg/core/netmap/nodes.go b/pkg/core/netmap/nodes.go index f01c07b19..e58e42634 100644 --- a/pkg/core/netmap/nodes.go +++ b/pkg/core/netmap/nodes.go @@ -1,6 +1,10 @@ package netmap -import "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" +import ( + "iter" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" +) // Node is a named type of netmap.NodeInfo which provides interface needed // in the current repository. Node is expected to be used everywhere instead @@ -14,8 +18,14 @@ func (x Node) PublicKey() []byte { return (netmap.NodeInfo)(x).PublicKey() } +// Addresses returns an iterator over all announced network addresses. +func (x Node) Addresses() iter.Seq[string] { + return (netmap.NodeInfo)(x).NetworkEndpoints() +} + // IterateAddresses iterates over all announced network addresses // and passes them into f. Handler MUST NOT be nil. +// Deprecated: use [Node.Addresses] instead. func (x Node) IterateAddresses(f func(string) bool) { for s := range (netmap.NodeInfo)(x).NetworkEndpoints() { if f(s) { diff --git a/pkg/network/group.go b/pkg/network/group.go index 5a71e530e..0044fb2d4 100644 --- a/pkg/network/group.go +++ b/pkg/network/group.go @@ -3,6 +3,7 @@ package network import ( "errors" "fmt" + "iter" "slices" "sort" @@ -68,9 +69,8 @@ func (x AddressGroup) Swap(i, j int) { // MultiAddressIterator is an interface of network address group. type MultiAddressIterator interface { - // IterateAddresses must iterate over network addresses and pass each one - // to the handler until it returns true. - IterateAddresses(func(string) bool) + // Addresses must return an iterator over network addresses. + Addresses() iter.Seq[string] // NumberOfAddresses must return number of addresses in group. NumberOfAddresses() int @@ -131,19 +131,19 @@ func (x *AddressGroup) FromIterator(iter MultiAddressIterator) error { // iterateParsedAddresses parses each address from MultiAddressIterator and passes it to f // until 1st parsing failure or f's error. func iterateParsedAddresses(iter MultiAddressIterator, f func(s Address) error) (err error) { - iter.IterateAddresses(func(s string) bool { + for s := range iter.Addresses() { var a Address err = a.FromString(s) if err != nil { - err = fmt.Errorf("could not parse address from string: %w", err) - return true + return fmt.Errorf("could not parse address from string: %w", err) } err = f(a) - - return err != nil - }) + if err != nil { + return err + } + } return } diff --git a/pkg/network/group_test.go b/pkg/network/group_test.go index 5b335fa52..d08264533 100644 --- a/pkg/network/group_test.go +++ b/pkg/network/group_test.go @@ -1,6 +1,8 @@ package network import ( + "iter" + "slices" "sort" "testing" @@ -58,10 +60,8 @@ func TestAddressGroup_FromIterator(t *testing.T) { type testIterator []string -func (t testIterator) IterateAddresses(f func(string) bool) { - for i := range t { - f(t[i]) - } +func (t testIterator) Addresses() iter.Seq[string] { + return slices.Values(t) } func (t testIterator) NumberOfAddresses() int { diff --git a/pkg/network/validation.go b/pkg/network/validation.go index 73a3ef8d7..b5157f28f 100644 --- a/pkg/network/validation.go +++ b/pkg/network/validation.go @@ -2,6 +2,7 @@ package network import ( "errors" + "iter" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" ) @@ -34,12 +35,8 @@ var ( // MultiAddressIterator. type NodeEndpointsIterator netmap.NodeInfo -func (x NodeEndpointsIterator) IterateAddresses(f func(string) bool) { - for s := range (netmap.NodeInfo)(x).NetworkEndpoints() { - if f(s) { - return - } - } +func (x NodeEndpointsIterator) Addresses() iter.Seq[string] { + return (netmap.NodeInfo)(x).NetworkEndpoints() } func (x NodeEndpointsIterator) NumberOfAddresses() int { From 98308d0cad312c34f2575d05651405de58c5b2f3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 14 Apr 2025 15:39:56 +0300 Subject: [PATCH 1909/1943] [#1715] blobstor: Allow to specify custom compression level Change-Id: I140c39b9dceaaeb58767061b131777af22242b19 Signed-off-by: Dmitrii Stepanov --- .../maintenance/zombie/storage_engine.go | 2 ++ cmd/frostfs-node/config.go | 4 +++ cmd/frostfs-node/config/engine/config_test.go | 2 ++ .../config/engine/shard/config.go | 10 ++++++ config/example/node.env | 1 + config/example/node.json | 1 + config/example/node.yaml | 5 +-- docs/storage-node-configuration.md | 3 +- internal/assert/cond.go | 4 +++ internal/logs/logs.go | 1 + pkg/local_object_storage/blobstor/blobstor.go | 6 ++++ .../blobstor/compression/compress.go | 34 ++++++++++++++++++- pkg/local_object_storage/blobstor/control.go | 5 +++ 13 files changed, 74 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go index 5851e049c..fe75a6f6f 100644 --- a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go @@ -13,6 +13,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -135,6 +136,7 @@ func getMetabaseOpts(sh *shardconfig.Config) []meta.Option { func getBlobstorOpts(ctx context.Context, sh *shardconfig.Config) []blobstor.Option { result := []blobstor.Option{ blobstor.WithCompressObjects(sh.Compress()), + blobstor.WithCompressionLevel(compression.Level(sh.CompressionLevel())), blobstor.WithUncompressableContentTypes(sh.UncompressableContentTypes()), blobstor.WithCompressibilityEstimate(sh.EstimateCompressibility()), blobstor.WithCompressibilityEstimateThreshold(sh.EstimateCompressibilityThreshold()), diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 8ceef2c31..f32953c58 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -40,6 +40,7 @@ import ( netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -129,6 +130,7 @@ type applicationConfiguration struct { type shardCfg struct { compress bool + compressionLevel compression.Level estimateCompressibility bool estimateCompressibilityThreshold float64 @@ -273,6 +275,7 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, source *s target.refillMetabaseWorkersCount = source.RefillMetabaseWorkersCount() target.mode = source.Mode() target.compress = source.Compress() + target.compressionLevel = compression.Level(source.CompressionLevel()) target.estimateCompressibility = source.EstimateCompressibility() target.estimateCompressibilityThreshold = source.EstimateCompressibilityThreshold() target.uncompressableContentType = source.UncompressableContentTypes() @@ -1027,6 +1030,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID blobstoreOpts := []blobstor.Option{ blobstor.WithCompressObjects(shCfg.compress), + blobstor.WithCompressionLevel(shCfg.compressionLevel), blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), blobstor.WithCompressibilityEstimate(shCfg.estimateCompressibility), blobstor.WithCompressibilityEstimateThreshold(shCfg.estimateCompressibilityThreshold), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 34613ad9e..afadb4c99 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -101,6 +101,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, 10*time.Millisecond, meta.BoltDB().MaxBatchDelay()) require.Equal(t, true, sc.Compress()) + require.Equal(t, "fastest", sc.CompressionLevel()) require.Equal(t, []string{"audio/*", "video/*"}, sc.UncompressableContentTypes()) require.Equal(t, true, sc.EstimateCompressibility()) require.Equal(t, float64(0.7), sc.EstimateCompressibilityThreshold()) @@ -237,6 +238,7 @@ func TestEngineSection(t *testing.T) { require.Equal(t, 20*time.Millisecond, meta.BoltDB().MaxBatchDelay()) require.Equal(t, false, sc.Compress()) + require.Equal(t, "", sc.CompressionLevel()) require.Equal(t, []string(nil), sc.UncompressableContentTypes()) require.EqualValues(t, 102400, sc.SmallSizeLimit()) diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go index e50d56b95..14e91f01f 100644 --- a/cmd/frostfs-node/config/engine/shard/config.go +++ b/cmd/frostfs-node/config/engine/shard/config.go @@ -37,6 +37,16 @@ func (x *Config) Compress() bool { ) } +// CompressionLevel returns the value of "compression_level" config parameter. +// +// Returns empty string if the value is not a valid string. +func (x *Config) CompressionLevel() string { + return config.StringSafe( + (*config.Config)(x), + "compression_level", + ) +} + // UncompressableContentTypes returns the value of "compress_skip_content_types" config parameter. // // Returns nil if a the value is missing or is invalid. diff --git a/config/example/node.env b/config/example/node.env index b501d3836..fb4fc69dd 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -123,6 +123,7 @@ FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_SIZE=100 FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_DELAY=10ms ### Blobstor config FROSTFS_STORAGE_SHARD_0_COMPRESS=true +FROSTFS_STORAGE_SHARD_0_COMPRESSION_LEVEL=fastest FROSTFS_STORAGE_SHARD_0_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*" FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY=true FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY_THRESHOLD=0.7 diff --git a/config/example/node.json b/config/example/node.json index b02f43f60..1e9dd7835 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -184,6 +184,7 @@ "max_batch_delay": "10ms" }, "compress": true, + "compression_level": "fastest", "compression_exclude_content_types": [ "audio/*", "video/*" ], diff --git a/config/example/node.yaml b/config/example/node.yaml index ba32adb82..26c6e248c 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -160,7 +160,7 @@ storage: max_batch_delay: 5ms # maximum delay for a batch of operations to be executed max_batch_size: 100 # maximum amount of operations in a single batch - compress: false # turn on/off zstd(level 3) compression of stored objects + compress: false # turn on/off zstd compression of stored objects small_object_size: 100 kb # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes blobstor: @@ -202,7 +202,8 @@ storage: max_batch_size: 100 max_batch_delay: 10ms - compress: true # turn on/off zstd(level 3) compression of stored objects + compress: true # turn on/off zstd compression of stored objects + compression_level: fastest compression_exclude_content_types: - audio/* - video/* diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 248b54ea4..19f6ee66d 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -188,6 +188,7 @@ The following table describes configuration for each shard. | Parameter | Type | Default value | Description | | ------------------------------------------------ | ------------------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `compress` | `bool` | `false` | Flag to enable compression. | +| `compression_level` | `string` | `optimal` | Compression level. Available values are `optimal`, `fastest`, `smallest_size`. | | `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). | | `compression_estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. | | `compression_estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. | @@ -199,7 +200,7 @@ The following table describes configuration for each shard. | `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | | `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. | | `gc` | [GC config](#gc-subsection) | | GC configuration. | -| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. | +| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. | ### `blobstor` subsection diff --git a/internal/assert/cond.go b/internal/assert/cond.go index c6a034f94..113d2eba9 100644 --- a/internal/assert/cond.go +++ b/internal/assert/cond.go @@ -23,3 +23,7 @@ func NoError(err error, details ...string) { panic(content) } } + +func Fail(details ...string) { + panic(strings.Join(details, " ")) +} diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 3a3ceb150..dedc7e12c 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -516,4 +516,5 @@ const ( FailedToValidateIncomingIOTag = "failed to validate incoming IO tag, replaced with `client`" WriteCacheFailedToAcquireRPSQuota = "writecache failed to acquire RPS quota to flush object" FailedToUpdateNetmapCandidates = "update netmap candidates failed" + UnknownCompressionLevelDefaultWillBeUsed = "unknown compression level, 'optimal' will be used" ) diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index d7c333349..7a9568aff 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -109,6 +109,12 @@ func WithCompressObjects(comp bool) Option { } } +func WithCompressionLevel(level compression.Level) Option { + return func(c *cfg) { + c.compression.Level = level + } +} + // WithCompressibilityEstimate returns an option to use // normilized compressibility estimate to decide compress // data or not. diff --git a/pkg/local_object_storage/blobstor/compression/compress.go b/pkg/local_object_storage/blobstor/compression/compress.go index 85ab47692..8a86b6982 100644 --- a/pkg/local_object_storage/blobstor/compression/compress.go +++ b/pkg/local_object_storage/blobstor/compression/compress.go @@ -4,15 +4,26 @@ import ( "bytes" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" "github.com/klauspost/compress" "github.com/klauspost/compress/zstd" ) +type Level string + +const ( + LevelDefault Level = "" + LevelOptimal Level = "optimal" + LevelFastest Level = "fastest" + LevelSmallestSize Level = "smallest_size" +) + // Config represents common compression-related configuration. type Config struct { Enabled bool UncompressableContentTypes []string + Level Level UseCompressEstimation bool CompressEstimationThreshold float64 @@ -30,7 +41,7 @@ func (c *Config) Init() error { var err error if c.Enabled { - c.encoder, err = zstd.NewWriter(nil) + c.encoder, err = zstd.NewWriter(nil, zstd.WithEncoderLevel(c.compressionLevel())) if err != nil { return err } @@ -116,3 +127,24 @@ func (c *Config) Close() error { } return err } + +func (c *Config) HasValidCompressionLevel() bool { + return c.Level == LevelDefault || + c.Level == LevelOptimal || + c.Level == LevelFastest || + c.Level == LevelSmallestSize +} + +func (c *Config) compressionLevel() zstd.EncoderLevel { + switch c.Level { + case LevelDefault, LevelOptimal: + return zstd.SpeedDefault + case LevelFastest: + return zstd.SpeedFastest + case LevelSmallestSize: + return zstd.SpeedBestCompression + default: + assert.Fail("unknown compression level", string(c.Level)) + return zstd.SpeedDefault + } +} diff --git a/pkg/local_object_storage/blobstor/control.go b/pkg/local_object_storage/blobstor/control.go index 93316be02..0418eedd0 100644 --- a/pkg/local_object_storage/blobstor/control.go +++ b/pkg/local_object_storage/blobstor/control.go @@ -6,6 +6,7 @@ import ( "fmt" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "go.uber.org/zap" ) @@ -53,6 +54,10 @@ var ErrInitBlobovniczas = errors.New("failure on blobovnicza initialization stag func (b *BlobStor) Init(ctx context.Context) error { b.log.Debug(ctx, logs.BlobstorInitializing) + if !b.compression.HasValidCompressionLevel() { + b.log.Warn(ctx, logs.UnknownCompressionLevelDefaultWillBeUsed, zap.String("level", string(b.compression.Level))) + b.compression.Level = compression.LevelDefault + } if err := b.compression.Init(); err != nil { return err } From 8c746a914ac917a16b0ab3098e70fda7e15dcbb0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 14 Apr 2025 16:18:08 +0300 Subject: [PATCH 1910/1943] [#1715] compression: Decouple Config and Compressor Refactoring. Change-Id: Ide2e1378f30c39045d4bacd13a902331bd4f764f Signed-off-by: Dmitrii Stepanov --- .../blobstor/blobovniczatree/blobovnicza.go | 4 ++-- .../blobstor/blobovniczatree/option.go | 2 +- pkg/local_object_storage/blobstor/blobstor.go | 4 ++-- .../blobstor/common/storage.go | 4 ++-- .../blobstor/compression/bench_test.go | 16 +++++++++----- .../blobstor/compression/compress.go | 22 +++++++++++-------- .../blobstor/fstree/fstree.go | 18 +++++++-------- .../blobstor/memstore/control.go | 4 ++-- .../blobstor/memstore/option.go | 2 +- .../blobstor/teststore/option.go | 8 +++---- .../blobstor/teststore/teststore.go | 4 ++-- .../writecache/writecache.go | 2 +- 12 files changed, 49 insertions(+), 41 deletions(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index d9e99d0d1..3e8b9f07b 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -158,11 +158,11 @@ func (b *Blobovniczas) Path() string { } // SetCompressor implements common.Storage. -func (b *Blobovniczas) SetCompressor(cc *compression.Config) { +func (b *Blobovniczas) SetCompressor(cc *compression.Compressor) { b.compression = cc } -func (b *Blobovniczas) Compressor() *compression.Config { +func (b *Blobovniczas) Compressor() *compression.Compressor { return b.compression } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 0e1b2022e..2f6d31b4e 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -19,7 +19,7 @@ type cfg struct { openedCacheSize int blzShallowDepth uint64 blzShallowWidth uint64 - compression *compression.Config + compression *compression.Compressor blzOpts []blobovnicza.Option reportError func(context.Context, string, error) // reportError is the function called when encountering disk errors. metrics Metrics diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index 7a9568aff..cf67c6fe9 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -41,7 +41,7 @@ type SubStorageInfo struct { type Option func(*cfg) type cfg struct { - compression compression.Config + compression compression.Compressor log *logger.Logger storage []SubStorage metrics Metrics @@ -158,6 +158,6 @@ func WithMetrics(m Metrics) Option { } } -func (b *BlobStor) Compressor() *compression.Config { +func (b *BlobStor) Compressor() *compression.Compressor { return &b.compression } diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index 6ecef48cd..e35c35e60 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -18,8 +18,8 @@ type Storage interface { Path() string ObjectsCount(ctx context.Context) (uint64, error) - SetCompressor(cc *compression.Config) - Compressor() *compression.Config + SetCompressor(cc *compression.Compressor) + Compressor() *compression.Compressor // SetReportErrorFunc allows to provide a function to be called on disk errors. // This function MUST be called before Open. diff --git a/pkg/local_object_storage/blobstor/compression/bench_test.go b/pkg/local_object_storage/blobstor/compression/bench_test.go index 9f70f8ec2..445a0494b 100644 --- a/pkg/local_object_storage/blobstor/compression/bench_test.go +++ b/pkg/local_object_storage/blobstor/compression/bench_test.go @@ -11,7 +11,7 @@ import ( ) func BenchmarkCompression(b *testing.B) { - c := Config{Enabled: true} + c := Compressor{Config: Config{Enabled: true}} require.NoError(b, c.Init()) for _, size := range []int{128, 1024, 32 * 1024, 32 * 1024 * 1024} { @@ -33,7 +33,7 @@ func BenchmarkCompression(b *testing.B) { } } -func benchWith(b *testing.B, c Config, data []byte) { +func benchWith(b *testing.B, c Compressor, data []byte) { b.ResetTimer() b.ReportAllocs() for range b.N { @@ -56,8 +56,10 @@ func BenchmarkCompressionRealVSEstimate(b *testing.B) { b.Run("estimate", func(b *testing.B) { b.ResetTimer() - c := &Config{ - Enabled: true, + c := &Compressor{ + Config: Config{ + Enabled: true, + }, } require.NoError(b, c.Init()) @@ -76,8 +78,10 @@ func BenchmarkCompressionRealVSEstimate(b *testing.B) { b.Run("compress", func(b *testing.B) { b.ResetTimer() - c := &Config{ - Enabled: true, + c := &Compressor{ + Config: Config{ + Enabled: true, + }, } require.NoError(b, c.Init()) diff --git a/pkg/local_object_storage/blobstor/compression/compress.go b/pkg/local_object_storage/blobstor/compression/compress.go index 8a86b6982..efe84ea2a 100644 --- a/pkg/local_object_storage/blobstor/compression/compress.go +++ b/pkg/local_object_storage/blobstor/compression/compress.go @@ -19,6 +19,13 @@ const ( LevelSmallestSize Level = "smallest_size" ) +type Compressor struct { + Config + + encoder *zstd.Encoder + decoder *zstd.Decoder +} + // Config represents common compression-related configuration. type Config struct { Enabled bool @@ -27,9 +34,6 @@ type Config struct { UseCompressEstimation bool CompressEstimationThreshold float64 - - encoder *zstd.Encoder - decoder *zstd.Decoder } // zstdFrameMagic contains first 4 bytes of any compressed object @@ -37,7 +41,7 @@ type Config struct { var zstdFrameMagic = []byte{0x28, 0xb5, 0x2f, 0xfd} // Init initializes compression routines. -func (c *Config) Init() error { +func (c *Compressor) Init() error { var err error if c.Enabled { @@ -84,7 +88,7 @@ func (c *Config) NeedsCompression(obj *objectSDK.Object) bool { // Decompress decompresses data if it starts with the magic // and returns data untouched otherwise. -func (c *Config) Decompress(data []byte) ([]byte, error) { +func (c *Compressor) Decompress(data []byte) ([]byte, error) { if len(data) < 4 || !bytes.Equal(data[:4], zstdFrameMagic) { return data, nil } @@ -93,7 +97,7 @@ func (c *Config) Decompress(data []byte) ([]byte, error) { // Compress compresses data if compression is enabled // and returns data untouched otherwise. -func (c *Config) Compress(data []byte) []byte { +func (c *Compressor) Compress(data []byte) []byte { if c == nil || !c.Enabled { return data } @@ -107,7 +111,7 @@ func (c *Config) Compress(data []byte) []byte { return c.compress(data) } -func (c *Config) compress(data []byte) []byte { +func (c *Compressor) compress(data []byte) []byte { maxSize := c.encoder.MaxEncodedSize(len(data)) compressed := c.encoder.EncodeAll(data, make([]byte, 0, maxSize)) if len(data) < len(compressed) { @@ -117,7 +121,7 @@ func (c *Config) compress(data []byte) []byte { } // Close closes encoder and decoder, returns any error occurred. -func (c *Config) Close() error { +func (c *Compressor) Close() error { var err error if c.encoder != nil { err = c.encoder.Close() @@ -135,7 +139,7 @@ func (c *Config) HasValidCompressionLevel() bool { c.Level == LevelSmallestSize } -func (c *Config) compressionLevel() zstd.EncoderLevel { +func (c *Compressor) compressionLevel() zstd.EncoderLevel { switch c.Level { case LevelDefault, LevelOptimal: return zstd.SpeedDefault diff --git a/pkg/local_object_storage/blobstor/fstree/fstree.go b/pkg/local_object_storage/blobstor/fstree/fstree.go index 031b385b2..112741ab4 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree.go @@ -45,7 +45,7 @@ type FSTree struct { log *logger.Logger - *compression.Config + compressor *compression.Compressor Depth uint64 DirNameLen int @@ -82,7 +82,7 @@ func New(opts ...Option) *FSTree { Permissions: 0o700, RootPath: "./", }, - Config: nil, + compressor: nil, Depth: 4, DirNameLen: DirNameLen, metrics: &noopMetrics{}, @@ -196,7 +196,7 @@ func (t *FSTree) iterate(ctx context.Context, depth uint64, curPath []string, pr } if err == nil { - data, err = t.Decompress(data) + data, err = t.compressor.Decompress(data) } if err != nil { if prm.IgnoreErrors { @@ -405,7 +405,7 @@ func (t *FSTree) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, err return common.PutRes{}, err } if !prm.DontCompress { - prm.RawData = t.Compress(prm.RawData) + prm.RawData = t.compressor.Compress(prm.RawData) } size = len(prm.RawData) @@ -448,7 +448,7 @@ func (t *FSTree) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, err } } - data, err = t.Decompress(data) + data, err = t.compressor.Decompress(data) if err != nil { return common.GetRes{}, err } @@ -597,12 +597,12 @@ func (t *FSTree) Path() string { } // SetCompressor implements common.Storage. -func (t *FSTree) SetCompressor(cc *compression.Config) { - t.Config = cc +func (t *FSTree) SetCompressor(cc *compression.Compressor) { + t.compressor = cc } -func (t *FSTree) Compressor() *compression.Config { - return t.Config +func (t *FSTree) Compressor() *compression.Compressor { + return t.compressor } // SetReportErrorFunc implements common.Storage. diff --git a/pkg/local_object_storage/blobstor/memstore/control.go b/pkg/local_object_storage/blobstor/memstore/control.go index 95a916662..3df96a1c3 100644 --- a/pkg/local_object_storage/blobstor/memstore/control.go +++ b/pkg/local_object_storage/blobstor/memstore/control.go @@ -16,7 +16,7 @@ func (s *memstoreImpl) Init() error func (s *memstoreImpl) Close(context.Context) error { return nil } func (s *memstoreImpl) Type() string { return Type } func (s *memstoreImpl) Path() string { return s.rootPath } -func (s *memstoreImpl) SetCompressor(cc *compression.Config) { s.compression = cc } -func (s *memstoreImpl) Compressor() *compression.Config { return s.compression } +func (s *memstoreImpl) SetCompressor(cc *compression.Compressor) { s.compression = cc } +func (s *memstoreImpl) Compressor() *compression.Compressor { return s.compression } func (s *memstoreImpl) SetReportErrorFunc(func(context.Context, string, error)) {} func (s *memstoreImpl) SetParentID(string) {} diff --git a/pkg/local_object_storage/blobstor/memstore/option.go b/pkg/local_object_storage/blobstor/memstore/option.go index 97a03993d..7605af4e5 100644 --- a/pkg/local_object_storage/blobstor/memstore/option.go +++ b/pkg/local_object_storage/blobstor/memstore/option.go @@ -7,7 +7,7 @@ import ( type cfg struct { rootPath string readOnly bool - compression *compression.Config + compression *compression.Compressor } func defaultConfig() *cfg { diff --git a/pkg/local_object_storage/blobstor/teststore/option.go b/pkg/local_object_storage/blobstor/teststore/option.go index fb1188751..3a38ecf82 100644 --- a/pkg/local_object_storage/blobstor/teststore/option.go +++ b/pkg/local_object_storage/blobstor/teststore/option.go @@ -17,8 +17,8 @@ type cfg struct { Type func() string Path func() string - SetCompressor func(cc *compression.Config) - Compressor func() *compression.Config + SetCompressor func(cc *compression.Compressor) + Compressor func() *compression.Compressor SetReportErrorFunc func(f func(context.Context, string, error)) Get func(common.GetPrm) (common.GetRes, error) @@ -45,11 +45,11 @@ func WithClose(f func() error) Option { return func(c *cfg) { c func WithType(f func() string) Option { return func(c *cfg) { c.overrides.Type = f } } func WithPath(f func() string) Option { return func(c *cfg) { c.overrides.Path = f } } -func WithSetCompressor(f func(*compression.Config)) Option { +func WithSetCompressor(f func(*compression.Compressor)) Option { return func(c *cfg) { c.overrides.SetCompressor = f } } -func WithCompressor(f func() *compression.Config) Option { +func WithCompressor(f func() *compression.Compressor) Option { return func(c *cfg) { c.overrides.Compressor = f } } diff --git a/pkg/local_object_storage/blobstor/teststore/teststore.go b/pkg/local_object_storage/blobstor/teststore/teststore.go index 626ba0023..190b6a876 100644 --- a/pkg/local_object_storage/blobstor/teststore/teststore.go +++ b/pkg/local_object_storage/blobstor/teststore/teststore.go @@ -116,7 +116,7 @@ func (s *TestStore) Path() string { } } -func (s *TestStore) SetCompressor(cc *compression.Config) { +func (s *TestStore) SetCompressor(cc *compression.Compressor) { s.mu.RLock() defer s.mu.RUnlock() switch { @@ -129,7 +129,7 @@ func (s *TestStore) SetCompressor(cc *compression.Config) { } } -func (s *TestStore) Compressor() *compression.Config { +func (s *TestStore) Compressor() *compression.Compressor { s.mu.RLock() defer s.mu.RUnlock() switch { diff --git a/pkg/local_object_storage/writecache/writecache.go b/pkg/local_object_storage/writecache/writecache.go index 70b17eb8e..7ed511318 100644 --- a/pkg/local_object_storage/writecache/writecache.go +++ b/pkg/local_object_storage/writecache/writecache.go @@ -52,7 +52,7 @@ type Cache interface { // MainStorage is the interface of the underlying storage of Cache implementations. type MainStorage interface { - Compressor() *compression.Config + Compressor() *compression.Compressor Exists(context.Context, common.ExistsPrm) (common.ExistsRes, error) Put(context.Context, common.PutPrm) (common.PutRes, error) } From 0ee7467da5c9cda8c398a011fa00c86657b7cbc3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 14 Apr 2025 17:36:51 +0300 Subject: [PATCH 1911/1943] [#1715] config: Add `compression` config section To group all `compression_*` parameters together. Change-Id: I11ad9600f731903753fef1adfbc0328ef75bbf87 Signed-off-by: Dmitrii Stepanov --- .../maintenance/zombie/storage_engine.go | 7 +-- cmd/frostfs-node/config.go | 18 +----- cmd/frostfs-node/config/engine/config_test.go | 17 +++--- .../config/engine/shard/config.go | 56 ++++++------------- config/example/node.env | 2 +- config/example/node.json | 20 ++++--- config/example/node.yaml | 18 +++--- docs/storage-node-configuration.md | 51 +++++++++++------ pkg/local_object_storage/blobstor/blobstor.go | 47 +--------------- .../blobstor/blobstor_test.go | 11 +++- .../blobstor/compression/compress.go | 8 +-- .../blobstor/iterate_test.go | 5 +- 12 files changed, 105 insertions(+), 155 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go index fe75a6f6f..c54b331f3 100644 --- a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go @@ -13,7 +13,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" @@ -135,11 +134,7 @@ func getMetabaseOpts(sh *shardconfig.Config) []meta.Option { func getBlobstorOpts(ctx context.Context, sh *shardconfig.Config) []blobstor.Option { result := []blobstor.Option{ - blobstor.WithCompressObjects(sh.Compress()), - blobstor.WithCompressionLevel(compression.Level(sh.CompressionLevel())), - blobstor.WithUncompressableContentTypes(sh.UncompressableContentTypes()), - blobstor.WithCompressibilityEstimate(sh.EstimateCompressibility()), - blobstor.WithCompressibilityEstimateThreshold(sh.EstimateCompressibilityThreshold()), + blobstor.WithCompression(sh.Compression()), blobstor.WithStorages(getSubStorages(ctx, sh)), blobstor.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index f32953c58..f80401b5b 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -129,13 +129,9 @@ type applicationConfiguration struct { } type shardCfg struct { - compress bool - compressionLevel compression.Level - estimateCompressibility bool - estimateCompressibilityThreshold float64 + compression compression.Config smallSizeObjectLimit uint64 - uncompressableContentType []string refillMetabase bool refillMetabaseWorkersCount int mode shardmode.Mode @@ -274,11 +270,7 @@ func (a *applicationConfiguration) updateShardConfig(c *config.Config, source *s target.refillMetabase = source.RefillMetabase() target.refillMetabaseWorkersCount = source.RefillMetabaseWorkersCount() target.mode = source.Mode() - target.compress = source.Compress() - target.compressionLevel = compression.Level(source.CompressionLevel()) - target.estimateCompressibility = source.EstimateCompressibility() - target.estimateCompressibilityThreshold = source.EstimateCompressibilityThreshold() - target.uncompressableContentType = source.UncompressableContentTypes() + target.compression = source.Compression() target.smallSizeObjectLimit = source.SmallSizeLimit() a.setShardWriteCacheConfig(&target, source) @@ -1029,11 +1021,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID ss := c.getSubstorageOpts(ctx, shCfg) blobstoreOpts := []blobstor.Option{ - blobstor.WithCompressObjects(shCfg.compress), - blobstor.WithCompressionLevel(shCfg.compressionLevel), - blobstor.WithUncompressableContentTypes(shCfg.uncompressableContentType), - blobstor.WithCompressibilityEstimate(shCfg.estimateCompressibility), - blobstor.WithCompressibilityEstimateThreshold(shCfg.estimateCompressibilityThreshold), + blobstor.WithCompression(shCfg.compression), blobstor.WithStorages(ss), blobstor.WithLogger(c.log), } diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index afadb4c99..401c54edc 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -15,6 +15,7 @@ import ( writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" ) @@ -100,11 +101,11 @@ func TestEngineSection(t *testing.T) { require.Equal(t, 100, meta.BoltDB().MaxBatchSize()) require.Equal(t, 10*time.Millisecond, meta.BoltDB().MaxBatchDelay()) - require.Equal(t, true, sc.Compress()) - require.Equal(t, "fastest", sc.CompressionLevel()) - require.Equal(t, []string{"audio/*", "video/*"}, sc.UncompressableContentTypes()) - require.Equal(t, true, sc.EstimateCompressibility()) - require.Equal(t, float64(0.7), sc.EstimateCompressibilityThreshold()) + require.Equal(t, true, sc.Compression().Enabled) + require.Equal(t, compression.LevelFastest, sc.Compression().Level) + require.Equal(t, []string{"audio/*", "video/*"}, sc.Compression().UncompressableContentTypes) + require.Equal(t, true, sc.Compression().EstimateCompressibility) + require.Equal(t, float64(0.7), sc.Compression().EstimateCompressibilityThreshold) require.EqualValues(t, 102400, sc.SmallSizeLimit()) require.Equal(t, 2, len(ss)) @@ -237,9 +238,9 @@ func TestEngineSection(t *testing.T) { require.Equal(t, 200, meta.BoltDB().MaxBatchSize()) require.Equal(t, 20*time.Millisecond, meta.BoltDB().MaxBatchDelay()) - require.Equal(t, false, sc.Compress()) - require.Equal(t, "", sc.CompressionLevel()) - require.Equal(t, []string(nil), sc.UncompressableContentTypes()) + require.Equal(t, false, sc.Compression().Enabled) + require.Equal(t, compression.LevelDefault, sc.Compression().Level) + require.Equal(t, []string(nil), sc.Compression().UncompressableContentTypes) require.EqualValues(t, 102400, sc.SmallSizeLimit()) require.Equal(t, 2, len(ss)) diff --git a/cmd/frostfs-node/config/engine/shard/config.go b/cmd/frostfs-node/config/engine/shard/config.go index 14e91f01f..d42646da7 100644 --- a/cmd/frostfs-node/config/engine/shard/config.go +++ b/cmd/frostfs-node/config/engine/shard/config.go @@ -8,6 +8,7 @@ import ( metabaseconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/metabase" piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama" writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" ) @@ -27,52 +28,27 @@ func From(c *config.Config) *Config { return (*Config)(c) } -// Compress returns the value of "compress" config parameter. -// -// Returns false if the value is not a valid bool. -func (x *Config) Compress() bool { - return config.BoolSafe( - (*config.Config)(x), - "compress", - ) -} - -// CompressionLevel returns the value of "compression_level" config parameter. -// -// Returns empty string if the value is not a valid string. -func (x *Config) CompressionLevel() string { - return config.StringSafe( - (*config.Config)(x), - "compression_level", - ) -} - -// UncompressableContentTypes returns the value of "compress_skip_content_types" config parameter. -// -// Returns nil if a the value is missing or is invalid. -func (x *Config) UncompressableContentTypes() []string { - return config.StringSliceSafe( - (*config.Config)(x), - "compression_exclude_content_types") -} - -// EstimateCompressibility returns the value of "estimate_compressibility" config parameter. -// -// Returns false if the value is not a valid bool. -func (x *Config) EstimateCompressibility() bool { - return config.BoolSafe( - (*config.Config)(x), - "compression_estimate_compressibility", - ) +func (x *Config) Compression() compression.Config { + cc := (*config.Config)(x).Sub("compression") + if cc == nil { + return compression.Config{} + } + return compression.Config{ + Enabled: config.BoolSafe(cc, "enabled"), + UncompressableContentTypes: config.StringSliceSafe(cc, "exclude_content_types"), + Level: compression.Level(config.StringSafe(cc, "level")), + EstimateCompressibility: config.BoolSafe(cc, "estimate_compressibility"), + EstimateCompressibilityThreshold: estimateCompressibilityThreshold(cc), + } } // EstimateCompressibilityThreshold returns the value of "estimate_compressibility_threshold" config parameter. // // Returns EstimateCompressibilityThresholdDefault if the value is not defined, not valid float or not in range [0.0; 1.0]. -func (x *Config) EstimateCompressibilityThreshold() float64 { +func estimateCompressibilityThreshold(c *config.Config) float64 { v := config.FloatOrDefault( - (*config.Config)(x), - "compression_estimate_compressibility_threshold", + c, + "estimate_compressibility_threshold", EstimateCompressibilityThresholdDefault) if v < 0.0 || v > 1.0 { return EstimateCompressibilityThresholdDefault diff --git a/config/example/node.env b/config/example/node.env index fb4fc69dd..9d054fe78 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -122,7 +122,7 @@ FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644 FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_SIZE=100 FROSTFS_STORAGE_SHARD_0_METABASE_MAX_BATCH_DELAY=10ms ### Blobstor config -FROSTFS_STORAGE_SHARD_0_COMPRESS=true +FROSTFS_STORAGE_SHARD_0_COMPRESSION_ENABLED=true FROSTFS_STORAGE_SHARD_0_COMPRESSION_LEVEL=fastest FROSTFS_STORAGE_SHARD_0_COMPRESSION_EXCLUDE_CONTENT_TYPES="audio/* video/*" FROSTFS_STORAGE_SHARD_0_COMPRESSION_ESTIMATE_COMPRESSIBILITY=true diff --git a/config/example/node.json b/config/example/node.json index 1e9dd7835..110e99ee8 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -183,13 +183,15 @@ "max_batch_size": 100, "max_batch_delay": "10ms" }, - "compress": true, - "compression_level": "fastest", - "compression_exclude_content_types": [ - "audio/*", "video/*" - ], - "compression_estimate_compressibility": true, - "compression_estimate_compressibility_threshold": 0.7, + "compression": { + "enabled": true, + "level": "fastest", + "exclude_content_types": [ + "audio/*", "video/*" + ], + "estimate_compressibility": true, + "estimate_compressibility_threshold": 0.7 + }, "small_object_size": 102400, "blobstor": [ { @@ -323,7 +325,9 @@ "max_batch_size": 200, "max_batch_delay": "20ms" }, - "compress": false, + "compression": { + "enabled": false + }, "small_object_size": 102400, "blobstor": [ { diff --git a/config/example/node.yaml b/config/example/node.yaml index 26c6e248c..de5eaa133 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -160,7 +160,8 @@ storage: max_batch_delay: 5ms # maximum delay for a batch of operations to be executed max_batch_size: 100 # maximum amount of operations in a single batch - compress: false # turn on/off zstd compression of stored objects + compression: + enabled: false # turn on/off zstd compression of stored objects small_object_size: 100 kb # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes blobstor: @@ -202,13 +203,14 @@ storage: max_batch_size: 100 max_batch_delay: 10ms - compress: true # turn on/off zstd compression of stored objects - compression_level: fastest - compression_exclude_content_types: - - audio/* - - video/* - compression_estimate_compressibility: true - compression_estimate_compressibility_threshold: 0.7 + compression: + enabled: true # turn on/off zstd compression of stored objects + level: fastest + exclude_content_types: + - audio/* + - video/* + estimate_compressibility: true + estimate_compressibility_threshold: 0.7 blobstor: - type: blobovnicza diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 19f6ee66d..1494d2fca 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -185,22 +185,41 @@ Contains configuration for each shard. Keys must be consecutive numbers starting `default` subsection has the same format and specifies defaults for missing values. The following table describes configuration for each shard. -| Parameter | Type | Default value | Description | -| ------------------------------------------------ | ------------------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `compress` | `bool` | `false` | Flag to enable compression. | -| `compression_level` | `string` | `optimal` | Compression level. Available values are `optimal`, `fastest`, `smallest_size`. | -| `compression_exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). | -| `compression_estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. | -| `compression_estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. | -| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` | -| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. | -| `resync_metabase_worker_count` | `int` | `1000` | Count of concurrent workers to resync metabase. | -| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. | -| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. | -| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | -| `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. | -| `gc` | [GC config](#gc-subsection) | | GC configuration. | -| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. | +| Parameter | Type | Default value | Description | +| ------------------------------ | --------------------------------------------- | ------------- | --------------------------------------------------------------------------------------------------------- | +| `compression` | [Compression config](#compression-subsection) | | Compression config. | +| `mode` | `string` | `read-write` | Shard Mode.
Possible values: `read-write`, `read-only`, `degraded`, `degraded-read-only`, `disabled` | +| `resync_metabase` | `bool` | `false` | Flag to enable metabase resync on start. | +| `resync_metabase_worker_count` | `int` | `1000` | Count of concurrent workers to resync metabase. | +| `writecache` | [Writecache config](#writecache-subsection) | | Write-cache configuration. | +| `metabase` | [Metabase config](#metabase-subsection) | | Metabase configuration. | +| `blobstor` | [Blobstor config](#blobstor-subsection) | | Blobstor configuration. | +| `small_object_size` | `size` | `1M` | Maximum size of an object stored in blobovnicza tree. | +| `gc` | [GC config](#gc-subsection) | | GC configuration. | +| `limits` | [Shard limits config](#limits-subsection) | | Shard limits configuration. | + +### `compression` subsection + +Contains compression config. + +```yaml +compression: + enabled: true + level: smallest_size + exclude_content_types: + - audio/* + - video/* + estimate_compressibility: true + estimate_compressibility_threshold: 0.7 +``` + +| Parameter | Type | Default value | Description | +| ------------------------------------ | ---------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `enabled` | `bool` | `false` | Flag to enable compression. | +| `level` | `string` | `optimal` | Compression level. Available values are `optimal`, `fastest`, `smallest_size`. | +| `exclude_content_types` | `[]string` | | List of content-types to disable compression for. Content-type is taken from `Content-Type` object attribute. Each element can contain a star `*` as a first (last) character, which matches any prefix (suffix). | +| `estimate_compressibility` | `bool` | `false` | If `true`, then noramalized compressibility estimation is used to decide compress data or not. | +| `estimate_compressibility_threshold` | `float` | `0.1` | Normilized compressibility estimate threshold: data will compress if estimation if greater than this value. | ### `blobstor` subsection diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index cf67c6fe9..edb2c1946 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -95,52 +95,9 @@ func WithLogger(l *logger.Logger) Option { } } -// WithCompressObjects returns option to toggle -// compression of the stored objects. -// -// If true, Zstandard algorithm is used for data compression. -// -// If compressor (decompressor) creation failed, -// the uncompressed option will be used, and the error -// is recorded in the provided log. -func WithCompressObjects(comp bool) Option { +func WithCompression(comp compression.Config) Option { return func(c *cfg) { - c.compression.Enabled = comp - } -} - -func WithCompressionLevel(level compression.Level) Option { - return func(c *cfg) { - c.compression.Level = level - } -} - -// WithCompressibilityEstimate returns an option to use -// normilized compressibility estimate to decide compress -// data or not. -// -// See https://github.com/klauspost/compress/blob/v1.17.2/compressible.go#L5 -func WithCompressibilityEstimate(v bool) Option { - return func(c *cfg) { - c.compression.UseCompressEstimation = v - } -} - -// WithCompressibilityEstimateThreshold returns an option to set -// normilized compressibility estimate threshold. -// -// See https://github.com/klauspost/compress/blob/v1.17.2/compressible.go#L5 -func WithCompressibilityEstimateThreshold(threshold float64) Option { - return func(c *cfg) { - c.compression.CompressEstimationThreshold = threshold - } -} - -// WithUncompressableContentTypes returns option to disable decompression -// for specific content types as seen by object.AttributeContentType attribute. -func WithUncompressableContentTypes(values []string) Option { - return func(c *cfg) { - c.compression.UncompressableContentTypes = values + c.compression.Config = comp } } diff --git a/pkg/local_object_storage/blobstor/blobstor_test.go b/pkg/local_object_storage/blobstor/blobstor_test.go index 6cc56fa3b..6ddeb6f00 100644 --- a/pkg/local_object_storage/blobstor/blobstor_test.go +++ b/pkg/local_object_storage/blobstor/blobstor_test.go @@ -9,6 +9,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -51,7 +52,9 @@ func TestCompression(t *testing.T) { newBlobStor := func(t *testing.T, compress bool) *BlobStor { bs := New( - WithCompressObjects(compress), + WithCompression(compression.Config{ + Enabled: compress, + }), WithStorages(defaultStorages(dir, smallSizeLimit))) require.NoError(t, bs.Open(context.Background(), mode.ReadWrite)) require.NoError(t, bs.Init(context.Background())) @@ -113,8 +116,10 @@ func TestBlobstor_needsCompression(t *testing.T) { dir := t.TempDir() bs := New( - WithCompressObjects(compress), - WithUncompressableContentTypes(ct), + WithCompression(compression.Config{ + Enabled: compress, + UncompressableContentTypes: ct, + }), WithStorages([]SubStorage{ { Storage: blobovniczatree.NewBlobovniczaTree( diff --git a/pkg/local_object_storage/blobstor/compression/compress.go b/pkg/local_object_storage/blobstor/compression/compress.go index efe84ea2a..c76cec9a1 100644 --- a/pkg/local_object_storage/blobstor/compression/compress.go +++ b/pkg/local_object_storage/blobstor/compression/compress.go @@ -32,8 +32,8 @@ type Config struct { UncompressableContentTypes []string Level Level - UseCompressEstimation bool - CompressEstimationThreshold float64 + EstimateCompressibility bool + EstimateCompressibilityThreshold float64 } // zstdFrameMagic contains first 4 bytes of any compressed object @@ -101,9 +101,9 @@ func (c *Compressor) Compress(data []byte) []byte { if c == nil || !c.Enabled { return data } - if c.UseCompressEstimation { + if c.EstimateCompressibility { estimated := compress.Estimate(data) - if estimated >= c.CompressEstimationThreshold { + if estimated >= c.EstimateCompressibilityThreshold { return c.compress(data) } return data diff --git a/pkg/local_object_storage/blobstor/iterate_test.go b/pkg/local_object_storage/blobstor/iterate_test.go index ccfa510fe..2786321a8 100644 --- a/pkg/local_object_storage/blobstor/iterate_test.go +++ b/pkg/local_object_storage/blobstor/iterate_test.go @@ -8,6 +8,7 @@ import ( "testing" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/memstore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/teststore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" @@ -24,7 +25,9 @@ func TestIterateObjects(t *testing.T) { // create BlobStor instance blobStor := New( WithStorages(defaultStorages(p, smalSz)), - WithCompressObjects(true), + WithCompression(compression.Config{ + Enabled: true, + }), ) defer os.RemoveAll(p) From 410b6f70bae0e44da29d328ec3ba532042617c8b Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Tue, 15 Apr 2025 17:56:47 +0300 Subject: [PATCH 1912/1943] [#1716] cli: Return trace ID on operation failure Close #1716 Change-Id: I293d0cc6b7331517e8cde42eae07d65384976da5 Signed-off-by: Aleksey Savchuk --- cmd/internal/common/exit.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/internal/common/exit.go b/cmd/internal/common/exit.go index b8acf0143..13f447af4 100644 --- a/cmd/internal/common/exit.go +++ b/cmd/internal/common/exit.go @@ -51,8 +51,13 @@ func ExitOnErr(cmd *cobra.Command, errFmt string, err error) { } cmd.PrintErrln(err) - if cmd.PersistentPostRun != nil { - cmd.PersistentPostRun(cmd, nil) + for p := cmd; p != nil; p = p.Parent() { + if p.PersistentPostRun != nil { + p.PersistentPostRun(cmd, nil) + if !cobra.EnableTraverseRunHooks { + break + } + } } os.Exit(code) } From a285d8924f87da6af50ad3fc28b5ac5726fc92e8 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Mon, 14 Apr 2025 12:38:52 +0300 Subject: [PATCH 1913/1943] [#1693] node: Replace conditional panics with asserts Change-Id: I5024705fd1693d00cb9241235030a73984c2a7e1 Signed-off-by: Ekaterina Lebedeva --- cmd/frostfs-node/netmap.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 6d57edcce..82b799e4c 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -8,6 +8,7 @@ import ( "net" "sync/atomic" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" @@ -104,9 +105,7 @@ func (s *networkState) getNodeInfo() (res netmapSDK.NodeInfo, ok bool) { v := s.nodeInfo.Load() if v != nil { res, ok = v.(netmapSDK.NodeInfo) - if !ok { - panic(fmt.Sprintf("unexpected value in atomic node info state: %T", v)) - } + assert.True(ok, fmt.Sprintf("unexpected value in atomic node info state: %T", v)) } return From fc6abe30b892491fce0053492220e2ec91e11236 Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 8 Apr 2025 16:47:52 +0300 Subject: [PATCH 1914/1943] [#1693] storage: Replace conditional panics with asserts Change-Id: I9d8ccde3c71fca716856c7bfc53da20ee0542f20 Signed-off-by: Ekaterina Lebedeva --- .../blobstor/fstree/counter.go | 17 +++++++---------- pkg/local_object_storage/metabase/lock.go | 5 ++--- pkg/local_object_storage/metabase/util.go | 5 ++--- pkg/local_object_storage/writecache/limiter.go | 16 ++++++---------- 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/counter.go b/pkg/local_object_storage/blobstor/fstree/counter.go index b5dbc9e40..3caee7ee1 100644 --- a/pkg/local_object_storage/blobstor/fstree/counter.go +++ b/pkg/local_object_storage/blobstor/fstree/counter.go @@ -2,6 +2,8 @@ package fstree import ( "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" ) // FileCounter used to count files in FSTree. The implementation must be thread-safe. @@ -52,16 +54,11 @@ func (c *SimpleCounter) Dec(size uint64) { c.mtx.Lock() defer c.mtx.Unlock() - if c.count > 0 { - c.count-- - } else { - panic("fstree.SimpleCounter: invalid count") - } - if c.size >= size { - c.size -= size - } else { - panic("fstree.SimpleCounter: invalid size") - } + assert.True(c.count > 0, "fstree.SimpleCounter: invalid count") + c.count-- + + assert.True(c.size >= size, "fstree.SimpleCounter: invalid size") + c.size -= size } func (c *SimpleCounter) CountSize() (uint64, uint64) { diff --git a/pkg/local_object_storage/metabase/lock.go b/pkg/local_object_storage/metabase/lock.go index f73c2b4f6..f4cb9e53b 100644 --- a/pkg/local_object_storage/metabase/lock.go +++ b/pkg/local_object_storage/metabase/lock.go @@ -7,6 +7,7 @@ import ( "slices" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -63,9 +64,7 @@ func (db *DB) Lock(ctx context.Context, cnr cid.ID, locker oid.ID, locked []oid. return ErrReadOnlyMode } - if len(locked) == 0 { - panic("empty locked list") - } + assert.False(len(locked) == 0, "empty locked list") err := db.lockInternal(locked, cnr, locker) success = err == nil diff --git a/pkg/local_object_storage/metabase/util.go b/pkg/local_object_storage/metabase/util.go index 80851f1c4..4ad83332b 100644 --- a/pkg/local_object_storage/metabase/util.go +++ b/pkg/local_object_storage/metabase/util.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" @@ -278,9 +279,7 @@ func objectKey(obj oid.ID, key []byte) []byte { // // firstIrregularObjectType(tx, cnr, obj) usage allows getting object type. func firstIrregularObjectType(tx *bbolt.Tx, idCnr cid.ID, objs ...[]byte) objectSDK.Type { - if len(objs) == 0 { - panic("empty object list in firstIrregularObjectType") - } + assert.False(len(objs) == 0, "empty object list in firstIrregularObjectType") var keys [2][1 + cidSize]byte diff --git a/pkg/local_object_storage/writecache/limiter.go b/pkg/local_object_storage/writecache/limiter.go index ddc4101be..0e020b36e 100644 --- a/pkg/local_object_storage/writecache/limiter.go +++ b/pkg/local_object_storage/writecache/limiter.go @@ -3,6 +3,8 @@ package writecache import ( "errors" "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" ) var errLimiterClosed = errors.New("acquire failed: limiter closed") @@ -45,17 +47,11 @@ func (l *flushLimiter) release(size uint64) { l.cond.L.Lock() defer l.cond.L.Unlock() - if l.size >= size { - l.size -= size - } else { - panic("flushLimiter: invalid size") - } + assert.True(l.size >= size, "flushLimiter: invalid size") + l.size -= size - if l.count > 0 { - l.count-- - } else { - panic("flushLimiter: invalid count") - } + assert.True(l.count > 0, "flushLimiter: invalid count") + l.count-- l.cond.Broadcast() } From bc045b29e21b458c9d3f244f05a16bce9869059e Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 8 Apr 2025 16:54:05 +0300 Subject: [PATCH 1915/1943] [#1693] services: Replace conditional panics with asserts Change-Id: Ic79609e6ad867caa88ad245b3014aa7fc32e05a8 Signed-off-by: Ekaterina Lebedeva --- pkg/services/netmap/executor.go | 11 +++++++---- pkg/services/object_manager/tombstone/constructor.go | 5 ++--- .../object_manager/tombstone/source/source.go | 5 ++--- pkg/services/policer/policer.go | 6 +++--- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pkg/services/netmap/executor.go b/pkg/services/netmap/executor.go index 44101a153..1b92fdaad 100644 --- a/pkg/services/netmap/executor.go +++ b/pkg/services/netmap/executor.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/version" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/util/response" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/netmap" @@ -46,10 +47,12 @@ type NetworkInfo interface { } func NewExecutionService(s NodeState, v versionsdk.Version, netInfo NetworkInfo, respSvc *response.Service) Server { - if s == nil || netInfo == nil || !version.IsValid(v) || respSvc == nil { - // this should never happen, otherwise it programmers bug - panic("can't create netmap execution service") - } + // this should never happen, otherwise it's a programmer's bug + msg := "BUG: can't create netmap execution service" + assert.False(s == nil, msg, "node state is nil") + assert.False(netInfo == nil, msg, "network info is nil") + assert.False(respSvc == nil, msg, "response service is nil") + assert.True(version.IsValid(v), msg, "invalid version") res := &executorSvc{ state: s, diff --git a/pkg/services/object_manager/tombstone/constructor.go b/pkg/services/object_manager/tombstone/constructor.go index 67ddf316f..2147a32fe 100644 --- a/pkg/services/object_manager/tombstone/constructor.go +++ b/pkg/services/object_manager/tombstone/constructor.go @@ -3,6 +3,7 @@ package tombstone import ( "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "go.uber.org/zap" @@ -49,9 +50,7 @@ func NewChecker(oo ...Option) *ExpirationChecker { panicOnNil(cfg.tsSource, "Tombstone source") cache, err := lru.New[string, uint64](cfg.cacheSize) - if err != nil { - panic(fmt.Errorf("could not create LRU cache with %d size: %w", cfg.cacheSize, err)) - } + assert.NoError(err, fmt.Sprintf("could not create LRU cache with %d size", cfg.cacheSize)) return &ExpirationChecker{ cache: cache, diff --git a/pkg/services/object_manager/tombstone/source/source.go b/pkg/services/object_manager/tombstone/source/source.go index 1ff07b05a..975941847 100644 --- a/pkg/services/object_manager/tombstone/source/source.go +++ b/pkg/services/object_manager/tombstone/source/source.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/util" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" @@ -38,9 +39,7 @@ func (s *TombstoneSourcePrm) SetGetService(v *getsvc.Service) { // Panics if any of the provided options does not allow // constructing a valid tombstone local Source. func NewSource(p TombstoneSourcePrm) Source { - if p.s == nil { - panic("Tombstone source: nil object service") - } + assert.False(p.s == nil, "Tombstone source: nil object service") return Source(p) } diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index 4e8bacfec..d18b71a23 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -1,9 +1,11 @@ package policer import ( + "fmt" "sync" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" "go.uber.org/zap" @@ -57,9 +59,7 @@ func New(opts ...Option) *Policer { c.log = c.log.With(zap.String("component", "Object Policer")) cache, err := lru.New[oid.Address, time.Time](int(c.cacheSize)) - if err != nil { - panic(err) - } + assert.NoError(err, fmt.Sprintf("could not create LRU cache with %d size", c.cacheSize)) return &Policer{ cfg: c, From 5dd8d7e87ab20ff3d7e1c4c2d93982a1ae79381a Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 8 Apr 2025 16:54:51 +0300 Subject: [PATCH 1916/1943] [#1693] network: Replace conditional panics with asserts Change-Id: Icba39aa2ed0048d63c6efed398273627e1e4fbbe Signed-off-by: Ekaterina Lebedeva --- pkg/network/address.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/network/address.go b/pkg/network/address.go index cb83a813d..4643eef15 100644 --- a/pkg/network/address.go +++ b/pkg/network/address.go @@ -2,11 +2,11 @@ package network import ( "errors" - "fmt" "net" "net/url" "strings" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/rpc/client" "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" @@ -44,11 +44,9 @@ func (a Address) equal(addr Address) bool { // See also FromString. func (a Address) URIAddr() string { _, host, err := manet.DialArgs(a.ma) - if err != nil { - // the only correct way to construct Address is AddressFromString - // which makes this error appear unexpected - panic(fmt.Errorf("could not get host addr: %w", err)) - } + // the only correct way to construct Address is AddressFromString + // which makes this error appear unexpected + assert.NoError(err, "could not get host addr") if !a.IsTLSEnabled() { return host From e45382b0c18ef6f5b116df4cd89f7813d271d54d Mon Sep 17 00:00:00 2001 From: Ekaterina Lebedeva Date: Tue, 8 Apr 2025 16:55:56 +0300 Subject: [PATCH 1917/1943] [#1693] util: Replace conditional panics with asserts Change-Id: I13b566cde3e6d43d8a75aa2e9b28e63b597adff9 Signed-off-by: Ekaterina Lebedeva --- pkg/util/keyer/dashboard.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/util/keyer/dashboard.go b/pkg/util/keyer/dashboard.go index b2942b52a..6337039a9 100644 --- a/pkg/util/keyer/dashboard.go +++ b/pkg/util/keyer/dashboard.go @@ -6,6 +6,7 @@ import ( "os" "text/tabwriter" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" "github.com/mr-tron/base58" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" @@ -104,9 +105,7 @@ func (d Dashboard) PrettyPrint(uncompressed, useHex bool) { func base58ToHex(data string) string { val, err := base58.Decode(data) - if err != nil { - panic("produced incorrect base58 value") - } + assert.NoError(err, "produced incorrect base58 value") return hex.EncodeToString(val) } From 36fb15b9a4b1b7771b2011c745222326ee6e9299 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Wed, 16 Apr 2025 11:55:00 +0300 Subject: [PATCH 1918/1943] [#1689] engine: Return error if object is locked during inhume Return `object is locked` error if object doesn't exists but is locked, since the locked index may be populated even when the object itself doesn't exist. Change-Id: If1a145c6efead9873acd33bb4fd22cf6175cbabd Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/engine/inhume.go | 11 ++-- .../engine/inhume_test.go | 55 +++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 6ec9a4ef0..e5a2396e1 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -186,10 +186,6 @@ func (e *StorageEngine) findShards(ctx context.Context, addr oid.Address, checkL default: } - if !objectExists { - return - } - if checkLocked { if isLocked, err := sh.IsLocked(ctx, addr); err != nil { e.log.Warn(ctx, logs.EngineRemovingAnObjectWithoutFullLockingCheck, @@ -202,6 +198,13 @@ func (e *StorageEngine) findShards(ctx context.Context, addr oid.Address, checkL } } + // This exit point must come after checking if the object is locked, + // since the locked index may be populated even if the object doesn't + // exist. + if !objectExists { + return + } + ids = append(ids, sh.ID().String()) // Continue if it's a root object. diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 10cebfb52..0601a43f2 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -242,3 +242,58 @@ func benchmarkInhumeMultipart(b *testing.B, numShards, numObjects int) { b.StopTimer() } } + +func TestInhumeIfObjectDoesntExist(t *testing.T) { + t.Run("object is locked", func(t *testing.T) { + t.Run("inhume without tombstone", func(t *testing.T) { + testInhumeLockedIfObjectDoesntExist(t, false, false) + }) + t.Run("inhume with tombstone", func(t *testing.T) { + testInhumeLockedIfObjectDoesntExist(t, true, false) + }) + t.Run("force inhume without tombstone", func(t *testing.T) { + testInhumeLockedIfObjectDoesntExist(t, false, true) + }) + t.Run("force inhume with tombstone", func(t *testing.T) { + testInhumeLockedIfObjectDoesntExist(t, true, true) + }) + }) +} + +func testInhumeLockedIfObjectDoesntExist(t *testing.T, withTombstone, withForce bool) { + t.Parallel() + + var ( + errLocked *apistatus.ObjectLocked + inhumePrm InhumePrm + ctx = context.Background() + container = cidtest.ID() + object = oidtest.Address() + lock = oidtest.ID() + tombstone = oidtest.Address() + ) + object.SetContainer(container) + tombstone.SetContainer(container) + + engine := testNewEngine(t).setShardsNum(t, 4).prepare(t).engine + defer func() { require.NoError(t, engine.Close(ctx)) }() + + err := engine.Lock(ctx, container, lock, []oid.ID{object.Object()}) + require.NoError(t, err) + + if withTombstone { + inhumePrm.WithTarget(tombstone, object) + } else { + inhumePrm.MarkAsGarbage(object) + } + if withForce { + inhumePrm.WithForceRemoval() + } + + err = engine.Inhume(ctx, inhumePrm) + if withForce { + require.NoError(t, err) + } else { + require.ErrorAs(t, err, &errLocked) + } +} From 100eb8b654b55286caa5bf4aa18c358eab12fd26 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 29 Jan 2025 15:39:28 +0300 Subject: [PATCH 1919/1943] [#1619] logger: Set tags for `node` components Change-Id: I55ffcce9d2a74fdd47621674739b07f2e20199e3 Signed-off-by: Anton Nikiforov --- .../maintenance/zombie/storage_engine.go | 3 ++- cmd/frostfs-node/config.go | 13 ++++----- cmd/frostfs-node/config/logger/config_test.go | 3 +++ cmd/frostfs-node/morph.go | 14 +++++----- cmd/frostfs-node/object.go | 14 +++++----- cmd/frostfs-node/session.go | 3 ++- cmd/frostfs-node/tree.go | 3 ++- config/example/node.env | 2 ++ config/example/node.json | 8 +++++- config/example/node.yaml | 3 +++ docs/storage-node-configuration.md | 16 ++++++++--- .../blobovnicza/blobovnicza.go | 2 +- .../blobovniczatree/concurrency_test.go | 3 ++- .../blobstor/blobovniczatree/exists_test.go | 3 ++- .../blobstor/blobovniczatree/generic_test.go | 6 +++-- .../blobstor/blobovniczatree/option.go | 11 +++++--- .../blobovniczatree/rebuild_failover_test.go | 3 ++- .../blobstor/blobovniczatree/rebuild_test.go | 27 ++++++++++++------- pkg/local_object_storage/blobstor/blobstor.go | 2 +- .../blobstor/fstree/option.go | 3 +-- .../engine/engine_test.go | 3 ++- .../shard/gc_internal_test.go | 3 ++- pkg/local_object_storage/shard/lock_test.go | 3 ++- pkg/local_object_storage/shard/range_test.go | 3 ++- pkg/local_object_storage/shard/shard.go | 2 +- pkg/local_object_storage/shard/shard_test.go | 3 ++- .../writecache/options.go | 3 +-- pkg/services/object/delete/service.go | 2 +- pkg/services/object/get/service.go | 2 +- pkg/services/object/get/v2/service.go | 2 +- pkg/services/object/search/service.go | 2 +- pkg/services/policer/policer.go | 3 --- pkg/services/replicator/replicator.go | 3 --- pkg/services/session/executor.go | 5 +--- pkg/util/logger/logger.go | 1 + pkg/util/logger/tag_string.go | 19 +++++++++++-- pkg/util/logger/tags.go | 27 ++++++++++++++----- 37 files changed, 151 insertions(+), 77 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go index c54b331f3..5be34d502 100644 --- a/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go +++ b/cmd/frostfs-adm/internal/modules/maintenance/zombie/storage_engine.go @@ -159,7 +159,8 @@ func getSubStorages(ctx context.Context, sh *shardconfig.Config) []blobstor.SubS blobovniczatree.WithOpenedCacheExpInterval(sub.OpenedCacheExpInterval()), blobovniczatree.WithInitWorkerCount(sub.InitWorkerCount()), blobovniczatree.WithWaitBeforeDropDB(sub.RebuildDropTimeout()), - blobovniczatree.WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + blobovniczatree.WithBlobovniczaLogger(logger.NewLoggerWrapper(zap.NewNop())), + blobovniczatree.WithBlobovniczaTreeLogger(logger.NewLoggerWrapper(zap.NewNop())), blobovniczatree.WithObjectSizeLimit(sh.SmallSizeLimit()), } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index f80401b5b..fae1ca1ca 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -891,7 +891,7 @@ func (c *cfg) engineOpts() []engine.Option { opts = append(opts, engine.WithErrorThreshold(c.EngineCfg.errorThreshold), - engine.WithLogger(c.log), + engine.WithLogger(c.log.WithTag(logger.TagEngine)), engine.WithLowMemoryConsumption(c.EngineCfg.lowMem), ) @@ -928,7 +928,7 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { writecache.WithMaxCacheSize(wcRead.sizeLimit), writecache.WithMaxCacheCount(wcRead.countLimit), writecache.WithNoSync(wcRead.noSync), - writecache.WithLogger(c.log), + writecache.WithLogger(c.log.WithTag(logger.TagWriteCache)), writecache.WithQoSLimiter(shCfg.limiter), ) } @@ -968,7 +968,8 @@ func (c *cfg) getSubstorageOpts(ctx context.Context, shCfg shardCfg) []blobstor. blobovniczatree.WithOpenedCacheExpInterval(sRead.openedCacheExpInterval), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), blobovniczatree.WithWaitBeforeDropDB(sRead.rebuildDropTimeout), - blobovniczatree.WithLogger(c.log), + blobovniczatree.WithBlobovniczaLogger(c.log.WithTag(logger.TagBlobovnicza)), + blobovniczatree.WithBlobovniczaTreeLogger(c.log.WithTag(logger.TagBlobovniczaTree)), blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit), } @@ -991,7 +992,7 @@ func (c *cfg) getSubstorageOpts(ctx context.Context, shCfg shardCfg) []blobstor. fstree.WithPerm(sRead.perm), fstree.WithDepth(sRead.depth), fstree.WithNoSync(sRead.noSync), - fstree.WithLogger(c.log), + fstree.WithLogger(c.log.WithTag(logger.TagFSTree)), } if c.metricsCollector != nil { fstreeOpts = append(fstreeOpts, @@ -1023,7 +1024,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID blobstoreOpts := []blobstor.Option{ blobstor.WithCompression(shCfg.compression), blobstor.WithStorages(ss), - blobstor.WithLogger(c.log), + blobstor.WithLogger(c.log.WithTag(logger.TagBlobstor)), } if c.metricsCollector != nil { blobstoreOpts = append(blobstoreOpts, blobstor.WithMetrics(lsmetrics.NewBlobstoreMetrics(c.metricsCollector.Blobstore()))) @@ -1048,7 +1049,7 @@ func (c *cfg) getShardOpts(ctx context.Context, shCfg shardCfg) shardOptsWithID var sh shardOptsWithID sh.configID = shCfg.id() sh.shOpts = []shard.Option{ - shard.WithLogger(c.log), + shard.WithLogger(c.log.WithTag(logger.TagShard)), shard.WithRefillMetabase(shCfg.refillMetabase), shard.WithRefillMetabaseWorkersCount(shCfg.refillMetabaseWorkersCount), shard.WithMode(shCfg.mode), diff --git a/cmd/frostfs-node/config/logger/config_test.go b/cmd/frostfs-node/config/logger/config_test.go index ffe8ac693..796ad529e 100644 --- a/cmd/frostfs-node/config/logger/config_test.go +++ b/cmd/frostfs-node/config/logger/config_test.go @@ -22,6 +22,9 @@ func TestLoggerSection_Level(t *testing.T) { require.Equal(t, "debug", loggerconfig.Level(c)) require.Equal(t, "journald", loggerconfig.Destination(c)) require.Equal(t, true, loggerconfig.Timestamp(c)) + tags := loggerconfig.Tags(c) + require.Equal(t, "main, morph", tags[0][0]) + require.Equal(t, "debug", tags[0][1]) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index d3c0f7b81..917cf6fc0 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" netmapEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/subscriber" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" "github.com/nspcc-dev/neo-go/pkg/core/block" "github.com/nspcc-dev/neo-go/pkg/core/state" @@ -84,7 +85,7 @@ func initMorphClient(ctx context.Context, c *cfg) { cli, err := client.New(ctx, c.key, client.WithDialTimeout(morphconfig.DialTimeout(c.appCfg)), - client.WithLogger(c.log), + client.WithLogger(c.log.WithTag(logger.TagMorph)), client.WithMetrics(c.metricsCollector.MorphClientMetrics()), client.WithEndpoints(addresses...), client.WithConnLostCallback(func() { @@ -165,6 +166,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { err error subs subscriber.Subscriber ) + log := c.log.WithTag(logger.TagMorph) fromSideChainBlock, err := c.persistate.UInt32(persistateSideChainLastBlockKey) if err != nil { @@ -173,14 +175,14 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { } subs, err = subscriber.New(ctx, &subscriber.Params{ - Log: c.log, + Log: log, StartFromBlock: fromSideChainBlock, Client: c.cfgMorph.client, }) fatalOnErr(err) lis, err := event.NewListener(event.ListenerParams{ - Logger: c.log, + Logger: log, Subscriber: subs, }) fatalOnErr(err) @@ -198,7 +200,7 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { setNetmapNotificationParser(c, newEpochNotification, func(src *state.ContainedNotificationEvent) (event.Event, error) { res, err := netmapEvent.ParseNewEpoch(src) if err == nil { - c.log.Info(ctx, logs.FrostFSNodeNewEpochEventFromSidechain, + log.Info(ctx, logs.FrostFSNodeNewEpochEventFromSidechain, zap.Uint64("number", res.(netmapEvent.NewEpoch).EpochNumber()), ) } @@ -209,11 +211,11 @@ func listenMorphNotifications(ctx context.Context, c *cfg) { registerNotificationHandlers(c.cfgContainer.scriptHash, lis, c.cfgContainer.parsers, c.cfgContainer.subscribers) registerBlockHandler(lis, func(ctx context.Context, block *block.Block) { - c.log.Debug(ctx, logs.FrostFSNodeNewBlock, zap.Uint32("index", block.Index)) + log.Debug(ctx, logs.FrostFSNodeNewBlock, zap.Uint32("index", block.Index)) err = c.persistate.SetUInt32(persistateSideChainLastBlockKey, block.Index) if err != nil { - c.log.Warn(ctx, logs.FrostFSNodeCantUpdatePersistentState, + log.Warn(ctx, logs.FrostFSNodeCantUpdatePersistentState, zap.String("chain", "side"), zap.Uint32("block_index", block.Index)) } diff --git a/cmd/frostfs-node/object.go b/cmd/frostfs-node/object.go index 527746d26..c33c02b3f 100644 --- a/cmd/frostfs-node/object.go +++ b/cmd/frostfs-node/object.go @@ -31,6 +31,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object_manager/placement" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/policer" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/replicator" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" objectGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object/grpc" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -217,9 +218,8 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl } remoteReader := objectService.NewRemoteReader(keyStorage, clientConstructor) - pol := policer.New( - policer.WithLogger(c.log), + policer.WithLogger(c.log.WithTag(logger.TagPolicer)), policer.WithKeySpaceIterator(&keySpaceIterator{ng: ls}), policer.WithBuryFunc(buryFn), policer.WithContainerSource(c.cfgObject.cnrSource), @@ -291,7 +291,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa ls := c.cfgObject.cfgLocalStorage.localStorage return replicator.New( - replicator.WithLogger(c.log), + replicator.WithLogger(c.log.WithTag(logger.TagReplicator)), replicator.WithPutTimeout( replicatorconfig.PutTimeout(c.appCfg), ), @@ -348,7 +348,7 @@ func createSearchSvc(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Trav c.netMapSource, keyStorage, containerSource, - searchsvc.WithLogger(c.log), + searchsvc.WithLogger(c.log.WithTag(logger.TagSearchSvc)), ) } @@ -374,7 +374,7 @@ func createGetService(c *cfg, keyStorage *util.KeyStorage, traverseGen *util.Tra ), coreConstructor, containerSource, - getsvc.WithLogger(c.log)) + getsvc.WithLogger(c.log.WithTag(logger.TagGetSvc))) } func createGetServiceV2(c *cfg, sGet *getsvc.Service, keyStorage *util.KeyStorage) *getsvcV2.Service { @@ -385,7 +385,7 @@ func createGetServiceV2(c *cfg, sGet *getsvc.Service, keyStorage *util.KeyStorag c.netMapSource, c, c.cfgObject.cnrSource, - getsvcV2.WithLogger(c.log), + getsvcV2.WithLogger(c.log.WithTag(logger.TagGetSvc)), ) } @@ -402,7 +402,7 @@ func createDeleteService(c *cfg, keyStorage *util.KeyStorage, sGet *getsvc.Servi cfg: c, }, keyStorage, - deletesvc.WithLogger(c.log), + deletesvc.WithLogger(c.log.WithTag(logger.TagDeleteSvc)), ) } diff --git a/cmd/frostfs-node/session.go b/cmd/frostfs-node/session.go index 2f3c9cbfe..fbfe3f5e6 100644 --- a/cmd/frostfs-node/session.go +++ b/cmd/frostfs-node/session.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage/persistent" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/session/storage/temporary" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session" sessionGRPC "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/session/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" @@ -55,7 +56,7 @@ func initSessionService(c *cfg) { server := sessionTransportGRPC.New( sessionSvc.NewSignService( &c.key.PrivateKey, - sessionSvc.NewExecutionService(c.privateTokenStore, c.respSvc, c.log), + sessionSvc.NewExecutionService(c.privateTokenStore, c.respSvc, c.log.WithTag(logger.TagSessionSvc)), ), ) diff --git a/cmd/frostfs-node/tree.go b/cmd/frostfs-node/tree.go index 67d9c9df0..62af45389 100644 --- a/cmd/frostfs-node/tree.go +++ b/cmd/frostfs-node/tree.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "go.uber.org/zap" "google.golang.org/grpc" @@ -56,7 +57,7 @@ func initTreeService(c *cfg) { tree.WithFrostfsidSubjectProvider(c.frostfsidClient), tree.WithNetmapSource(c.netMapSource), tree.WithPrivateKey(&c.key.PrivateKey), - tree.WithLogger(c.log), + tree.WithLogger(c.log.WithTag(logger.TagTreeSvc)), tree.WithStorage(c.cfgObject.cfgLocalStorage.localStorage), tree.WithContainerCacheSize(treeConfig.CacheSize()), tree.WithReplicationTimeout(treeConfig.ReplicationTimeout()), diff --git a/config/example/node.env b/config/example/node.env index 9d054fe78..e7d7a6cc8 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -1,6 +1,8 @@ FROSTFS_LOGGER_LEVEL=debug FROSTFS_LOGGER_DESTINATION=journald FROSTFS_LOGGER_TIMESTAMP=true +FROSTFS_LOGGER_TAGS_0_NAMES="main, morph" +FROSTFS_LOGGER_TAGS_0_LEVEL="debug" FROSTFS_PPROF_ENABLED=true FROSTFS_PPROF_ADDRESS=localhost:6060 diff --git a/config/example/node.json b/config/example/node.json index 110e99ee8..3f7854d98 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -2,7 +2,13 @@ "logger": { "level": "debug", "destination": "journald", - "timestamp": true + "timestamp": true, + "tags": [ + { + "names": "main, morph", + "level": "debug" + } + ] }, "pprof": { "enabled": true, diff --git a/config/example/node.yaml b/config/example/node.yaml index de5eaa133..32f0cba67 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -2,6 +2,9 @@ logger: level: debug # logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" destination: journald # logger destination: one of "stdout" (default), "journald" timestamp: true + tags: + - names: "main, morph" + level: debug systemdnotify: enabled: true diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 1494d2fca..3e770457b 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -112,11 +112,21 @@ Contains logger parameters. ```yaml logger: level: info + tags: + - names: "main, morph" + level: debug ``` -| Parameter | Type | Default value | Description | -|-----------|----------|---------------|---------------------------------------------------------------------------------------------------| -| `level` | `string` | `info` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal` | +| Parameter | Type | Default value | Description | +|-----------|-----------------------------------------------|---------------|---------------------------------------------------------------------------------------------------| +| `level` | `string` | `info` | Logging level.
Possible values: `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal` | +| `tags` | list of [tags descriptions](#tags-subsection) | | Array of tags description. | + +## `tags` subsection +| Parameter | Type | Default value | Description | +|-----------|----------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `names` | `string` | | List of components divided by `,`.
Possible values: `main`, `morph`, `grpcsvc`, `ir`, `processor`, `engine`, `blobovnicza`, `blobstor`, `fstree`, `gc`, `shard`, `writecache`, `deletesvc`, `getsvc`, `searchsvc`, `sessionsvc`, `treesvc`, `policer`, `replicator`. | +| `level` | `string` | | Logging level for the components from `names`, overrides default logging level. | # `contracts` section Contains override values for FrostFS side-chain contract hashes. Most of the time contract diff --git a/pkg/local_object_storage/blobovnicza/blobovnicza.go b/pkg/local_object_storage/blobovnicza/blobovnicza.go index 08ef8b86c..a6c40f9fa 100644 --- a/pkg/local_object_storage/blobovnicza/blobovnicza.go +++ b/pkg/local_object_storage/blobovnicza/blobovnicza.go @@ -110,7 +110,7 @@ func WithFullSizeLimit(lim uint64) Option { // WithLogger returns an option to specify Blobovnicza's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = l.With(zap.String("component", "Blobovnicza")) + c.log = l } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go index ec9743b57..f87f4a144 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/concurrency_test.go @@ -19,7 +19,8 @@ func TestBlobovniczaTree_Concurrency(t *testing.T) { st := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(1024), WithBlobovniczaShallowWidth(10), WithBlobovniczaShallowDepth(1), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go index 5414140f0..df2b4ffe5 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go @@ -19,7 +19,8 @@ func TestExistsInvalidStorageID(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(1024), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go index d390ecf1d..9244d765c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/generic_test.go @@ -15,7 +15,8 @@ func TestGeneric(t *testing.T) { helper := func(t *testing.T, dir string) common.Storage { return NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), @@ -43,7 +44,8 @@ func TestControl(t *testing.T) { newTree := func(t *testing.T) common.Storage { return NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(maxObjectSize), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index 2f6d31b4e..5f268b0f2 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -63,10 +63,15 @@ func initConfig(c *cfg) { } } -func WithLogger(l *logger.Logger) Option { +func WithBlobovniczaTreeLogger(log *logger.Logger) Option { return func(c *cfg) { - c.log = l - c.blzOpts = append(c.blzOpts, blobovnicza.WithLogger(l)) + c.log = log + } +} + +func WithBlobovniczaLogger(log *logger.Logger) Option { + return func(c *cfg) { + c.blzOpts = append(c.blzOpts, blobovnicza.WithLogger(log)) } } diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go index 8832603c4..4146ef260 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go @@ -140,7 +140,8 @@ func testRebuildFailoverObjectDeletedFromSource(t *testing.T) { func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object, mustUpdateStorageID bool) { b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(2), WithBlobovniczaShallowDepth(2), diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go index 9c971bfb6..a7a99fec3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go @@ -50,7 +50,8 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), WithBlobovniczaShallowWidth(1), // single directory WithBlobovniczaShallowDepth(1), @@ -106,7 +107,8 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), WithBlobovniczaShallowWidth(1), // single directory WithBlobovniczaShallowDepth(1), @@ -160,7 +162,8 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), WithBlobovniczaShallowWidth(1), // single directory WithBlobovniczaShallowDepth(1), @@ -231,7 +234,8 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), WithBlobovniczaShallowWidth(1), // single directory WithBlobovniczaShallowDepth(1), @@ -262,7 +266,8 @@ func TestBlobovniczaTreeFillPercentRebuild(t *testing.T) { require.NoError(t, b.Close(context.Background())) b = NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), WithBlobovniczaShallowWidth(1), WithBlobovniczaShallowDepth(1), @@ -304,7 +309,8 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(64*1024), // 64KB object size limit WithBlobovniczaShallowWidth(5), WithBlobovniczaShallowDepth(2), // depth = 2 @@ -332,7 +338,8 @@ func TestBlobovniczaTreeRebuildLargeObject(t *testing.T) { b = NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(32*1024), // 32KB object size limit WithBlobovniczaShallowWidth(5), WithBlobovniczaShallowDepth(3), // depth = 3 @@ -374,7 +381,8 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta dir := t.TempDir() b := NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(sourceWidth), WithBlobovniczaShallowDepth(sourceDepth), @@ -415,7 +423,8 @@ func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, ta b = NewBlobovniczaTree( context.Background(), - WithLogger(test.NewLogger(t)), + WithBlobovniczaLogger(test.NewLogger(t)), + WithBlobovniczaTreeLogger(test.NewLogger(t)), WithObjectSizeLimit(2048), WithBlobovniczaShallowWidth(targetWidth), WithBlobovniczaShallowDepth(targetDepth), diff --git a/pkg/local_object_storage/blobstor/blobstor.go b/pkg/local_object_storage/blobstor/blobstor.go index edb2c1946..ceaf2538a 100644 --- a/pkg/local_object_storage/blobstor/blobstor.go +++ b/pkg/local_object_storage/blobstor/blobstor.go @@ -91,7 +91,7 @@ func WithStorages(st []SubStorage) Option { // WithLogger returns option to specify BlobStor's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = l.With(zap.String("component", "BlobStor")) + c.log = l } } diff --git a/pkg/local_object_storage/blobstor/fstree/option.go b/pkg/local_object_storage/blobstor/fstree/option.go index 7155ddcbb..6f2ac87e1 100644 --- a/pkg/local_object_storage/blobstor/fstree/option.go +++ b/pkg/local_object_storage/blobstor/fstree/option.go @@ -4,7 +4,6 @@ import ( "io/fs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" ) type Option func(*FSTree) @@ -53,6 +52,6 @@ func WithFileCounter(c FileCounter) Option { func WithLogger(l *logger.Logger) Option { return func(f *FSTree) { - f.log = l.With(zap.String("component", "FSTree")) + f.log = l } } diff --git a/pkg/local_object_storage/engine/engine_test.go b/pkg/local_object_storage/engine/engine_test.go index 711a76100..fc6d9ee9c 100644 --- a/pkg/local_object_storage/engine/engine_test.go +++ b/pkg/local_object_storage/engine/engine_test.go @@ -116,7 +116,8 @@ func newStorages(t testing.TB, root string, smallSize uint64) []blobstor.SubStor blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1), blobovniczatree.WithPermissions(0o700), - blobovniczatree.WithLogger(test.NewLogger(t))), + blobovniczatree.WithBlobovniczaLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaTreeLogger(test.NewLogger(t))), Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < smallSize }, diff --git a/pkg/local_object_storage/shard/gc_internal_test.go b/pkg/local_object_storage/shard/gc_internal_test.go index 9998bbae2..54d2f1510 100644 --- a/pkg/local_object_storage/shard/gc_internal_test.go +++ b/pkg/local_object_storage/shard/gc_internal_test.go @@ -37,7 +37,8 @@ func Test_ObjectNotFoundIfNotDeletedFromMetabase(t *testing.T) { { Storage: blobovniczatree.NewBlobovniczaTree( context.Background(), - blobovniczatree.WithLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaTreeLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index 5caf3641f..3878a65cd 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -28,9 +28,10 @@ func TestShard_Lock(t *testing.T) { var sh *Shard rootPath := t.TempDir() + l := logger.NewLoggerWrapper(zap.NewNop()) opts := []Option{ WithID(NewIDFromBytes([]byte{})), - WithLogger(logger.NewLoggerWrapper(zap.NewNop())), + WithLogger(l), WithBlobStorOptions( blobstor.WithStorages([]blobstor.SubStorage{ { diff --git a/pkg/local_object_storage/shard/range_test.go b/pkg/local_object_storage/shard/range_test.go index 146e834cc..06fe9f511 100644 --- a/pkg/local_object_storage/shard/range_test.go +++ b/pkg/local_object_storage/shard/range_test.go @@ -79,7 +79,8 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) { { Storage: blobovniczatree.NewBlobovniczaTree( context.Background(), - blobovniczatree.WithLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaTreeLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(t.TempDir(), "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index 304a6bf9d..d89b56266 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -205,7 +205,7 @@ func WithPiloramaOptions(opts ...pilorama.Option) Option { func WithLogger(l *logger.Logger) Option { return func(c *cfg) { c.log = l - c.gcCfg.log = l + c.gcCfg.log = l.WithTag(logger.TagGC) } } diff --git a/pkg/local_object_storage/shard/shard_test.go b/pkg/local_object_storage/shard/shard_test.go index f9ee34488..84be71c4d 100644 --- a/pkg/local_object_storage/shard/shard_test.go +++ b/pkg/local_object_storage/shard/shard_test.go @@ -60,7 +60,8 @@ func newCustomShard(t testing.TB, enableWriteCache bool, o shardOptions) *Shard { Storage: blobovniczatree.NewBlobovniczaTree( context.Background(), - blobovniczatree.WithLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaLogger(test.NewLogger(t)), + blobovniczatree.WithBlobovniczaTreeLogger(test.NewLogger(t)), blobovniczatree.WithRootPath(filepath.Join(o.rootPath, "blob", "blobovnicza")), blobovniczatree.WithBlobovniczaShallowDepth(1), blobovniczatree.WithBlobovniczaShallowWidth(1)), diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index dbbe66c19..a4f98ad06 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -5,7 +5,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" ) // Option represents write-cache configuration option. @@ -46,7 +45,7 @@ type options struct { // WithLogger sets logger. func WithLogger(log *logger.Logger) Option { return func(o *options) { - o.log = log.With(zap.String("component", "WriteCache")) + o.log = log } } diff --git a/pkg/services/object/delete/service.go b/pkg/services/object/delete/service.go index 867d3f4ef..1c4d7d585 100644 --- a/pkg/services/object/delete/service.go +++ b/pkg/services/object/delete/service.go @@ -92,6 +92,6 @@ func New(gs *getsvc.Service, // WithLogger returns option to specify Delete service's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = l.With(zap.String("component", "objectSDK.Delete service")) + c.log = l } } diff --git a/pkg/services/object/get/service.go b/pkg/services/object/get/service.go index 9ec10b5f2..a103f5a7f 100644 --- a/pkg/services/object/get/service.go +++ b/pkg/services/object/get/service.go @@ -53,6 +53,6 @@ func New( // WithLogger returns option to specify Get service's logger. func WithLogger(l *logger.Logger) Option { return func(s *Service) { - s.log = l.With(zap.String("component", "Object.Get service")) + s.log = l } } diff --git a/pkg/services/object/get/v2/service.go b/pkg/services/object/get/v2/service.go index fc483b74b..0ec8912fd 100644 --- a/pkg/services/object/get/v2/service.go +++ b/pkg/services/object/get/v2/service.go @@ -145,6 +145,6 @@ func (s *Service) Head(ctx context.Context, req *objectV2.HeadRequest) (*objectV func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = l.With(zap.String("component", "Object.Get V2 service")) + c.log = l } } diff --git a/pkg/services/object/search/service.go b/pkg/services/object/search/service.go index e1aeca957..56fe56468 100644 --- a/pkg/services/object/search/service.go +++ b/pkg/services/object/search/service.go @@ -94,6 +94,6 @@ func New(e *engine.StorageEngine, // WithLogger returns option to specify Get service's logger. func WithLogger(l *logger.Logger) Option { return func(c *cfg) { - c.log = l.With(zap.String("component", "Object.Search service")) + c.log = l } } diff --git a/pkg/services/policer/policer.go b/pkg/services/policer/policer.go index d18b71a23..c91e7cc7c 100644 --- a/pkg/services/policer/policer.go +++ b/pkg/services/policer/policer.go @@ -8,7 +8,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/assert" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" lru "github.com/hashicorp/golang-lru/v2" - "go.uber.org/zap" ) type objectsInWork struct { @@ -56,8 +55,6 @@ func New(opts ...Option) *Policer { opts[i](c) } - c.log = c.log.With(zap.String("component", "Object Policer")) - cache, err := lru.New[oid.Address, time.Time](int(c.cacheSize)) assert.NoError(err, fmt.Sprintf("could not create LRU cache with %d size", c.cacheSize)) diff --git a/pkg/services/replicator/replicator.go b/pkg/services/replicator/replicator.go index 6910fa5af..a940cef37 100644 --- a/pkg/services/replicator/replicator.go +++ b/pkg/services/replicator/replicator.go @@ -7,7 +7,6 @@ import ( objectwriter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/common/writer" getsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/get" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "go.uber.org/zap" ) // Replicator represents the utility that replicates @@ -45,8 +44,6 @@ func New(opts ...Option) *Replicator { opts[i](c) } - c.log = c.log.With(zap.String("component", "Object Replicator")) - return &Replicator{ cfg: c, } diff --git a/pkg/services/session/executor.go b/pkg/services/session/executor.go index 12b221613..f0591de71 100644 --- a/pkg/services/session/executor.go +++ b/pkg/services/session/executor.go @@ -33,10 +33,7 @@ func NewExecutionService(exec ServiceExecutor, respSvc *response.Service, l *log } func (s *executorSvc) Create(ctx context.Context, req *session.CreateRequest) (*session.CreateResponse, error) { - s.log.Debug(ctx, logs.ServingRequest, - zap.String("component", "SessionService"), - zap.String("request", "Create"), - ) + s.log.Debug(ctx, logs.ServingRequest, zap.String("request", "Create")) respBody, err := s.exec.Create(ctx, req.GetBody()) if err != nil { diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index 276847be1..a1998cb1a 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -238,5 +238,6 @@ func NewLoggerWrapper(z *zap.Logger) *Logger { return &Logger{ z: z.WithOptions(zap.AddCallerSkip(1)), t: TagMain, + c: z.Core(), } } diff --git a/pkg/util/logger/tag_string.go b/pkg/util/logger/tag_string.go index a25b3c445..1b98f2e62 100644 --- a/pkg/util/logger/tag_string.go +++ b/pkg/util/logger/tag_string.go @@ -13,11 +13,26 @@ func _() { _ = x[TagGrpcSvc-3] _ = x[TagIr-4] _ = x[TagProcessor-5] + _ = x[TagEngine-6] + _ = x[TagBlobovnicza-7] + _ = x[TagBlobovniczaTree-8] + _ = x[TagBlobstor-9] + _ = x[TagFSTree-10] + _ = x[TagGC-11] + _ = x[TagShard-12] + _ = x[TagWriteCache-13] + _ = x[TagDeleteSvc-14] + _ = x[TagGetSvc-15] + _ = x[TagSearchSvc-16] + _ = x[TagSessionSvc-17] + _ = x[TagTreeSvc-18] + _ = x[TagPolicer-19] + _ = x[TagReplicator-20] } -const _Tag_name = "mainmorphgrpc_svcirprocessor" +const _Tag_name = "mainmorphgrpcsvcirprocessorengineblobovniczablobovniczatreeblobstorfstreegcshardwritecachedeletesvcgetsvcsearchsvcsessionsvctreesvcpolicerreplicator" -var _Tag_index = [...]uint8{0, 4, 9, 17, 19, 28} +var _Tag_index = [...]uint8{0, 4, 9, 16, 18, 27, 33, 44, 59, 67, 73, 75, 80, 90, 99, 105, 114, 124, 131, 138, 148} func (i Tag) String() string { i -= 1 diff --git a/pkg/util/logger/tags.go b/pkg/util/logger/tags.go index 51a6f62da..a5386707e 100644 --- a/pkg/util/logger/tags.go +++ b/pkg/util/logger/tags.go @@ -13,12 +13,27 @@ import ( type Tag uint8 const ( - _ Tag = iota // - TagMain // main - TagMorph // morph - TagGrpcSvc // grpc_svc - TagIr // ir - TagProcessor // processor + _ Tag = iota // + TagMain // main + TagMorph // morph + TagGrpcSvc // grpcsvc + TagIr // ir + TagProcessor // processor + TagEngine // engine + TagBlobovnicza // blobovnicza + TagBlobovniczaTree // blobovniczatree + TagBlobstor // blobstor + TagFSTree // fstree + TagGC // gc + TagShard // shard + TagWriteCache // writecache + TagDeleteSvc // deletesvc + TagGetSvc // getsvc + TagSearchSvc // searchsvc + TagSessionSvc // sessionsvc + TagTreeSvc // treesvc + TagPolicer // policer + TagReplicator // replicator defaultLevel = zapcore.InfoLevel ) From 86264e4e20af3eb9f90ae65487a5b5e91eb8a28d Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 4 Feb 2025 09:11:36 +0300 Subject: [PATCH 1920/1943] [#1619] logger: Add benchmark Change-Id: I49e90e8a3689a755755afd0638b327a6b1884795 Signed-off-by: Anton Nikiforov --- pkg/util/logger/logger_test.go | 118 +++++++++++++++++++++++++++++ pkg/util/logger/logger_test.result | 46 +++++++++++ 2 files changed, 164 insertions(+) create mode 100644 pkg/util/logger/logger_test.go create mode 100644 pkg/util/logger/logger_test.result diff --git a/pkg/util/logger/logger_test.go b/pkg/util/logger/logger_test.go new file mode 100644 index 000000000..b867ee6cc --- /dev/null +++ b/pkg/util/logger/logger_test.go @@ -0,0 +1,118 @@ +package logger + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func BenchmarkLogger(b *testing.B) { + ctx := context.Background() + m := map[string]Prm{} + + prm := Prm{} + require.NoError(b, prm.SetLevelString("debug")) + m["logging enabled"] = prm + + prm = Prm{} + require.NoError(b, prm.SetLevelString("error")) + m["logging disabled"] = prm + + prm = Prm{} + require.NoError(b, prm.SetLevelString("error")) + require.NoError(b, prm.SetTags([][]string{{"main", "debug"}, {"morph", "debug"}})) + m["logging enabled via tags"] = prm + + prm = Prm{} + require.NoError(b, prm.SetLevelString("debug")) + require.NoError(b, prm.SetTags([][]string{{"main", "error"}, {"morph", "debug"}})) + m["logging disabled via tags"] = prm + + for k, v := range m { + b.Run(k, func(b *testing.B) { + logger, err := createLogger(v) + require.NoError(b, err) + UpdateLevelForTags(v) + b.ResetTimer() + b.ReportAllocs() + for range b.N { + logger.Info(ctx, "test info") + } + }) + } +} + +type testCore struct { + core zapcore.Core +} + +func (c *testCore) Enabled(lvl zapcore.Level) bool { + return c.core.Enabled(lvl) +} + +func (c *testCore) With(fields []zapcore.Field) zapcore.Core { + c.core = c.core.With(fields) + return c +} + +func (c *testCore) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + return ce.AddCore(e, c) +} + +func (c *testCore) Write(zapcore.Entry, []zapcore.Field) error { + return nil +} + +func (c *testCore) Sync() error { + return c.core.Sync() +} + +func createLogger(prm Prm) (*Logger, error) { + prm.Options = []zap.Option{zap.WrapCore(func(core zapcore.Core) zapcore.Core { + tc := testCore{core: core} + return &tc + })} + return NewLogger(prm) +} + +func TestLoggerOutput(t *testing.T) { + obs, logs := observer.New(zap.NewAtomicLevelAt(zap.DebugLevel)) + + prm := Prm{} + require.NoError(t, prm.SetLevelString("debug")) + prm.Options = []zap.Option{zap.WrapCore(func(zapcore.Core) zapcore.Core { + return obs + })} + loggerMain, err := NewLogger(prm) + require.NoError(t, err) + UpdateLevelForTags(prm) + + loggerMainWith := loggerMain.With(zap.String("key", "value")) + + require.Panics(t, func() { + loggerMainWith.WithTag(TagShard) + }) + loggerShard := loggerMain.WithTag(TagShard) + loggerShard = loggerShard.With(zap.String("key1", "value1")) + + loggerMorph := loggerMain.WithTag(TagMorph) + loggerMorph = loggerMorph.With(zap.String("key2", "value2")) + + ctx := context.Background() + loggerMain.Debug(ctx, "main") + loggerMainWith.Debug(ctx, "main with") + loggerShard.Debug(ctx, "shard") + loggerMorph.Debug(ctx, "morph") + + require.Len(t, logs.All(), 4) + require.Len(t, logs.FilterFieldKey("key").All(), 1) + require.Len(t, logs.FilterFieldKey("key1").All(), 1) + require.Len(t, logs.FilterFieldKey("key2").All(), 1) + require.Len(t, logs.FilterField(zap.String("tag", TagMain.String())).All(), 2) + require.Len(t, logs.FilterField(zap.String("tag", TagShard.String())).All(), 1) + require.Len(t, logs.FilterField(zap.String("tag", TagMorph.String())).All(), 1) +} diff --git a/pkg/util/logger/logger_test.result b/pkg/util/logger/logger_test.result new file mode 100644 index 000000000..612fa2967 --- /dev/null +++ b/pkg/util/logger/logger_test.result @@ -0,0 +1,46 @@ +goos: linux +goarch: amd64 +pkg: git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger +cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz +BenchmarkLogger/logging_enabled-8 10000 1156 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1124 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1106 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1096 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1071 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1081 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1074 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1134 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1123 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled-8 10000 1144 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.15 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.54 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.22 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.22 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 17.01 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.31 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.61 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.17 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 16.26 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled-8 10000 21.02 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1146 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1086 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1113 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1157 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1069 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1073 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1096 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1092 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1060 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_enabled_via_tags-8 10000 1153 ns/op 240 B/op 1 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.23 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.39 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.47 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.62 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.53 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.53 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.74 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.20 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 17.06 ns/op 0 B/op 0 allocs/op +BenchmarkLogger/logging_disabled_via_tags-8 10000 16.60 ns/op 0 B/op 0 allocs/op +PASS +ok git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger 0.260s From cf48069fd8a12df6c0013d60cc0ce0a89420241b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:26:20 +0300 Subject: [PATCH 1921/1943] [#1718] linter: Resolve gocritic's appendAssign linter See https://go-critic.com/overview#appendassign for details. Change-Id: I991979ea680af25e2cec9097fa12b1c4eebc6c1d Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/modules/object/util.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-cli/modules/object/util.go b/cmd/frostfs-cli/modules/object/util.go index 3955f8ee1..8e4e8b287 100644 --- a/cmd/frostfs-cli/modules/object/util.go +++ b/cmd/frostfs-cli/modules/object/util.go @@ -262,13 +262,8 @@ func OpenSessionViaClient(cmd *cobra.Command, dst SessionPrm, cli *client.Client if _, ok := dst.(*internal.DeleteObjectPrm); ok { common.PrintVerbose(cmd, "Collecting relatives of the removal object...") - rels := collectObjectRelatives(cmd, cli, cnr, *obj) - - if len(rels) == 0 { - objs = []oid.ID{*obj} - } else { - objs = append(rels, *obj) - } + objs = collectObjectRelatives(cmd, cli, cnr, *obj) + objs = append(objs, *obj) } } From 2075e09cedad3150a91216524c014846edb3cb71 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:28:21 +0300 Subject: [PATCH 1922/1943] [#1718] linter: Resolve gocritic's unslice linter See https://go-critic.com/overview#unslice for details. Change-Id: I6d21e8ce1c9bae56099dc203f5080b0e3ea0c1ef Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-adm/internal/modules/morph/config/config.go | 2 +- cmd/frostfs-lens/internal/schema/writecache/parsers.go | 2 +- pkg/local_object_storage/metabase/delete.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/config/config.go b/cmd/frostfs-adm/internal/modules/morph/config/config.go index f64cb4817..c17fb62ff 100644 --- a/cmd/frostfs-adm/internal/modules/morph/config/config.go +++ b/cmd/frostfs-adm/internal/modules/morph/config/config.go @@ -63,7 +63,7 @@ func dumpNetworkConfig(cmd *cobra.Command, _ []string) error { netmap.MaxObjectSizeConfig, netmap.WithdrawFeeConfig, netmap.MaxECDataCountConfig, netmap.MaxECParityCountConfig: nbuf := make([]byte, 8) - copy(nbuf[:], v) + copy(nbuf, v) n := binary.LittleEndian.Uint64(nbuf) _, _ = tw.Write(fmt.Appendf(nil, "%s:\t%d (int)\n", k, n)) case netmap.HomomorphicHashingDisabledKey, netmap.MaintenanceModeAllowedConfig: diff --git a/cmd/frostfs-lens/internal/schema/writecache/parsers.go b/cmd/frostfs-lens/internal/schema/writecache/parsers.go index 7d70b27b2..3bfe2608b 100644 --- a/cmd/frostfs-lens/internal/schema/writecache/parsers.go +++ b/cmd/frostfs-lens/internal/schema/writecache/parsers.go @@ -57,7 +57,7 @@ func DefaultRecordParser(key, value []byte) (common.SchemaEntry, common.Parser, r.addr.SetContainer(cnr) r.addr.SetObject(obj) - r.data = value[:] + r.data = value return &r, nil, nil } diff --git a/pkg/local_object_storage/metabase/delete.go b/pkg/local_object_storage/metabase/delete.go index cf1d563e9..9a5a6e574 100644 --- a/pkg/local_object_storage/metabase/delete.go +++ b/pkg/local_object_storage/metabase/delete.go @@ -363,12 +363,12 @@ func (db *DB) deleteObject( func parentLength(tx *bbolt.Tx, addr oid.Address) int { bucketName := make([]byte, bucketKeySize) - bkt := tx.Bucket(parentBucketName(addr.Container(), bucketName[:])) + bkt := tx.Bucket(parentBucketName(addr.Container(), bucketName)) if bkt == nil { return 0 } - lst, err := decodeList(bkt.Get(objectKey(addr.Object(), bucketName[:]))) + lst, err := decodeList(bkt.Get(objectKey(addr.Object(), bucketName))) if err != nil { return 0 } From d2114759aaaf4e99aa83c1185c0a5d184a83b97a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:38:06 +0300 Subject: [PATCH 1923/1943] [#1718] linter: Resolve gocritic's typeSwitchVar linter See https://go-critic.com/overview#typeswitchvar for details Change-Id: Ic29db32c9b080576ab51dd484b4376114e9e775c Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-lens/internal/tui/ui.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-lens/internal/tui/ui.go b/cmd/frostfs-lens/internal/tui/ui.go index bd7540b01..cc6b7859e 100644 --- a/cmd/frostfs-lens/internal/tui/ui.go +++ b/cmd/frostfs-lens/internal/tui/ui.go @@ -460,11 +460,11 @@ func (ui *UI) handleInputOnSearching(event *tcell.EventKey) { return } - switch ui.mountedPage.(type) { + switch v := ui.mountedPage.(type) { case *BucketsView: ui.moveNextPage(NewBucketsView(ui, res)) case *RecordsView: - bucket := ui.mountedPage.(*RecordsView).bucket + bucket := v.bucket ui.moveNextPage(NewRecordsView(ui, bucket, res)) } From 8d499f03fee98825d183d18a7ef492a6a019b25a Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:40:31 +0300 Subject: [PATCH 1924/1943] [#1718] linter: Resolve gocritic's elseif linter See https://go-critic.com/overview#elseif for details. Change-Id: I8fd3edfacaeea2b0a83917575d545af7e7ab4d13 Signed-off-by: Dmitrii Stepanov --- pkg/services/container/ape.go | 6 ++---- pkg/services/object/common/target/target.go | 6 ++---- pkg/services/object_manager/tombstone/checker.go | 6 ++---- pkg/services/tree/service.go | 6 ++---- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/pkg/services/container/ape.go b/pkg/services/container/ape.go index e1fbe3960..01bd825d7 100644 --- a/pkg/services/container/ape.go +++ b/pkg/services/container/ape.go @@ -655,10 +655,8 @@ func (ac *apeChecker) namespaceByOwner(ctx context.Context, owner *refs.OwnerID) subject, err := ac.frostFSIDClient.GetSubject(ctx, addr) if err == nil { namespace = subject.Namespace - } else { - if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { - return "", fmt.Errorf("get subject error: %w", err) - } + } else if !strings.Contains(err.Error(), frostfsidcore.SubjectNotFoundErrorMessage) { + return "", fmt.Errorf("get subject error: %w", err) } return namespace, nil } diff --git a/pkg/services/object/common/target/target.go b/pkg/services/object/common/target/target.go index b2ae79dbc..f2bd907db 100644 --- a/pkg/services/object/common/target/target.go +++ b/pkg/services/object/common/target/target.go @@ -89,10 +89,8 @@ func newTrustedTarget(ctx context.Context, prm *objectwriter.Params) (transforme if !ownerObj.Equals(ownerSession) { return nil, fmt.Errorf("session token is missing but object owner id (%s) is different from the default key (%s)", ownerObj, ownerSession) } - } else { - if !ownerObj.Equals(sessionInfo.Owner) { - return nil, fmt.Errorf("different token issuer and object owner identifiers %s/%s", sessionInfo.Owner, ownerObj) - } + } else if !ownerObj.Equals(sessionInfo.Owner) { + return nil, fmt.Errorf("different token issuer and object owner identifiers %s/%s", sessionInfo.Owner, ownerObj) } if prm.SignRequestPrivateKey == nil { diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go index a4e36c2dc..e5f001d5a 100644 --- a/pkg/services/object_manager/tombstone/checker.go +++ b/pkg/services/object_manager/tombstone/checker.go @@ -61,10 +61,8 @@ func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Addr logs.TombstoneCouldNotGetTheTombstoneTheSource, zap.Error(err), ) - } else { - if ts != nil { - return g.handleTS(ctx, addrStr, ts, epoch) - } + } else if ts != nil { + return g.handleTS(ctx, addrStr, ts, epoch) } // requested tombstone not diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index a00c8c1cd..5349dd13e 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -436,10 +436,8 @@ func getSortedSubTree(ctx context.Context, srv TreeService_GetSubTreeServer, cid } if ms == nil { ms = m.Items - } else { - if len(m.Items) != 1 { - return status.Error(codes.InvalidArgument, "multiple non-internal nodes provided") - } + } else if len(m.Items) != 1 { + return status.Error(codes.InvalidArgument, "multiple non-internal nodes provided") } ts = append(ts, m.Time) ps = append(ps, p) From 64900f87e1872dea2794b02ddbc888f171a24a77 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:44:00 +0300 Subject: [PATCH 1925/1943] [#1718] linter: Resolve gocritic's singleCaseSwitch linter See https://go-critic.com/overview#singlecaseswitch for details. Change-Id: Ied7885f83b4116969771de6f91bc5e1e3b2a4f1e Signed-off-by: Dmitrii Stepanov --- .../blobstor/fstree/fstree_write_generic.go | 9 +++------ pkg/services/control/ir/server/server.go | 3 +-- pkg/util/http/server.go | 3 +-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go index 07a618b0a..6d633dad6 100644 --- a/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go +++ b/pkg/local_object_storage/blobstor/fstree/fstree_write_generic.go @@ -67,12 +67,9 @@ func (w *genericWriter) writeAndRename(tmpPath, p string, data []byte) error { err := w.writeFile(tmpPath, data) if err != nil { var pe *fs.PathError - if errors.As(err, &pe) { - switch pe.Err { - case syscall.ENOSPC: - err = common.ErrNoSpace - _ = os.RemoveAll(tmpPath) - } + if errors.As(err, &pe) && errors.Is(pe.Err, syscall.ENOSPC) { + err = common.ErrNoSpace + _ = os.RemoveAll(tmpPath) } return err } diff --git a/pkg/services/control/ir/server/server.go b/pkg/services/control/ir/server/server.go index c2a4f88a6..0cfca71c1 100644 --- a/pkg/services/control/ir/server/server.go +++ b/pkg/services/control/ir/server/server.go @@ -35,8 +35,7 @@ func panicOnPrmValue(n string, v any) { // the parameterized private key. func New(prm Prm, netmapClient *netmap.Client, containerClient *container.Client, opts ...Option) *Server { // verify required parameters - switch { - case prm.healthChecker == nil: + if prm.healthChecker == nil { panicOnPrmValue("health checker", prm.healthChecker) } diff --git a/pkg/util/http/server.go b/pkg/util/http/server.go index 923412a7f..2589ab786 100644 --- a/pkg/util/http/server.go +++ b/pkg/util/http/server.go @@ -76,8 +76,7 @@ func New(prm HTTPSrvPrm, opts ...Option) *Server { o(c) } - switch { - case c.shutdownTimeout <= 0: + if c.shutdownTimeout <= 0 { panicOnOptValue("shutdown timeout", c.shutdownTimeout) } From b88fe8c4a7f842c65d98a1edf6b83e54a7f2c3b3 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:45:04 +0300 Subject: [PATCH 1926/1943] [#1718] linter: Resolve gocritic's assignOp linter See https://go-critic.com/overview#assignop for details. Change-Id: I839446846437c8c74c119d8b5669f5b866c247dc Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go index d2eef2074..a840275b8 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go @@ -226,7 +226,7 @@ func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.M func (b *Blobovniczas) addRebuildTempFile(ctx context.Context, path string) (func(), error) { sysPath := filepath.Join(b.rootPath, path) - sysPath = sysPath + rebuildSuffix + sysPath += rebuildSuffix _, err := os.OpenFile(sysPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, b.perm) if err != nil { return nil, err From ca8b01667fac7107a8451843a4b21c656dfc7527 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:46:31 +0300 Subject: [PATCH 1927/1943] [#1718] linter: Resolve gocritic's unlambda linter See https://go-critic.com/overview#unlambda for details. Change-Id: Iccb2d293ce31a302fcbb2c3f9c55c9b3fa554db5 Signed-off-by: Dmitrii Stepanov --- pkg/services/object/get/v2/util.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/services/object/get/v2/util.go b/pkg/services/object/get/v2/util.go index bfa7fd619..4b7dcc530 100644 --- a/pkg/services/object/get/v2/util.go +++ b/pkg/services/object/get/v2/util.go @@ -182,9 +182,7 @@ func (s *Service) toHashRangePrm(req *objectV2.GetRangeHashRequest) (*getsvc.Ran default: return nil, errUnknownChechsumType(t) case refs.SHA256: - p.SetHashGenerator(func() hash.Hash { - return sha256.New() - }) + p.SetHashGenerator(sha256.New) case refs.TillichZemor: p.SetHashGenerator(func() hash.Hash { return tz.New() From 3cbff575351a9a8ffff384e920d06da56a6c9703 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 14:47:32 +0300 Subject: [PATCH 1928/1943] [#1718] linter: Enable gocritic linter See https://go-critic.com/overview#checkers-from-the-diagnostic-group for list of default enabled checkers. `ifElseChain` disabled as it generates doubtful issues. Change-Id: I5937b116d9af8b3cdf8b06451c4904d0b3f67f68 Signed-off-by: Dmitrii Stepanov --- .golangci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 3ac4eb651..e3ec09f60 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -18,6 +18,7 @@ linters: - exhaustive - funlen - gocognit + - gocritic - godot - importas - ineffassign @@ -44,6 +45,9 @@ linters: statements: 60 gocognit: min-complexity: 40 + gocritic: + disabled-checks: + - ifElseChain importas: alias: - pkg: git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object From c84013a85408a5403c2cc8f6f9cf98fb7bf79a7b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 16 Apr 2025 15:09:43 +0300 Subject: [PATCH 1929/1943] [#1719] cli: Fix TTL description Change-Id: I051a27af57a74304713c1f832dc31dbaeb10cbc6 Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-cli/internal/commonflags/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-cli/internal/commonflags/api.go b/cmd/frostfs-cli/internal/commonflags/api.go index 88321176f..6ed21e107 100644 --- a/cmd/frostfs-cli/internal/commonflags/api.go +++ b/cmd/frostfs-cli/internal/commonflags/api.go @@ -9,7 +9,7 @@ const ( TTL = "ttl" TTLShorthand = "" TTLDefault = 2 - TTLUsage = "TTL value in request meta header" + TTLUsage = "The maximum number of intermediate nodes in the request route" XHeadersKey = "xhdr" XHeadersShorthand = "x" From 1a738792c00518eff82b8643135c4bd4a40e0710 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Thu, 17 Apr 2025 16:15:12 +0300 Subject: [PATCH 1930/1943] [#1722] node, ir: Fix documentation for logger tags Close #1722. Change-Id: Iee9b3d96a31353622e1680d2d596ca6a9ee104b4 Signed-off-by: Anton Nikiforov --- config/example/ir.yaml | 2 +- docs/storage-node-configuration.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/example/ir.yaml b/config/example/ir.yaml index a4a006550..ed53f014b 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -4,7 +4,7 @@ logger: level: info # Logger level: one of "debug", "info" (default), "warn", "error", "dpanic", "panic", "fatal" timestamp: true tags: - - names: "main, morph" # Possible values: `main`, `morph`, `grpc_svc`, `ir`, `processor`. + - names: "main, morph" # Possible values: `main`, `morph`, `grpcsvc`, `ir`, `processor`. level: debug wallet: diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 3e770457b..14ebb53b3 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -123,10 +123,10 @@ logger: | `tags` | list of [tags descriptions](#tags-subsection) | | Array of tags description. | ## `tags` subsection -| Parameter | Type | Default value | Description | -|-----------|----------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `names` | `string` | | List of components divided by `,`.
Possible values: `main`, `morph`, `grpcsvc`, `ir`, `processor`, `engine`, `blobovnicza`, `blobstor`, `fstree`, `gc`, `shard`, `writecache`, `deletesvc`, `getsvc`, `searchsvc`, `sessionsvc`, `treesvc`, `policer`, `replicator`. | -| `level` | `string` | | Logging level for the components from `names`, overrides default logging level. | +| Parameter | Type | Default value | Description | +|-----------|----------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `names` | `string` | | List of components divided by `,`.
Possible values: `main`, `engine`, `blobovnicza`, `blobovniczatree`, `blobstor`, `fstree`, `gc`, `shard`, `writecache`, `deletesvc`, `getsvc`, `searchsvc`, `sessionsvc`, `treesvc`, `policer`, `replicator`. | +| `level` | `string` | | Logging level for the components from `names`, overrides default logging level. | # `contracts` section Contains override values for FrostFS side-chain contract hashes. Most of the time contract From 6567ceaf132c3295b8f011920d3718c989168b2f Mon Sep 17 00:00:00 2001 From: Alexander Chuprov Date: Thu, 17 Apr 2025 13:20:18 +0300 Subject: [PATCH 1931/1943] [#1702] tree: Support reloading 'tree.authorized_keys' on SIGHUP Allows updating the list of public keys authorized to invoke 'Tree service' operations without restarting the node. Change-Id: I01b6e05875b7ae3f3218062eb12bf9755e87f2a3 Signed-off-by: Alexander Chuprov --- cmd/frostfs-node/config.go | 7 +++++++ pkg/services/tree/options.go | 8 +++----- pkg/services/tree/service.go | 14 ++++++++++++++ pkg/services/tree/signature.go | 4 ++-- pkg/services/tree/signature_test.go | 1 + 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index fae1ca1ca..b688acfde 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -30,6 +30,7 @@ import ( objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object" replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator" tracingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tracing" + treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" internalNet "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" @@ -1403,6 +1404,12 @@ func (c *cfg) getComponents(ctx context.Context) []dCmp { } return err }}) + if c.treeService != nil { + components = append(components, dCmp{"tree", func() error { + c.treeService.ReloadAuthorizedKeys(treeconfig.Tree(c.appCfg).AuthorizedKeys()) + return nil + }}) + } if cmp, updated := metricsComponent(c); updated { if cmp.enabled { cmp.preReload = enableMetricsSvc diff --git a/pkg/services/tree/options.go b/pkg/services/tree/options.go index 4ad760846..56cbcc081 100644 --- a/pkg/services/tree/options.go +++ b/pkg/services/tree/options.go @@ -3,6 +3,7 @@ package tree import ( "context" "crypto/ecdsa" + "sync/atomic" "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" @@ -41,7 +42,7 @@ type cfg struct { replicatorWorkerCount int replicatorTimeout time.Duration containerCacheSize int - authorizedKeys [][]byte + authorizedKeys atomic.Pointer[[][]byte] syncBatchSize int localOverrideStorage policyengine.LocalOverrideStorage @@ -147,10 +148,7 @@ func WithMetrics(v MetricsRegister) Option { // keys that have rights to use Tree service. func WithAuthorizedKeys(keys keys.PublicKeys) Option { return func(c *cfg) { - c.authorizedKeys = nil - for _, key := range keys { - c.authorizedKeys = append(c.authorizedKeys, key.Bytes()) - } + c.authorizedKeys.Store(fromPublicKeys(keys)) } } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index 5349dd13e..a4bc0c97c 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -17,6 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl" cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/panjf2000/ants/v2" "go.uber.org/zap" "google.golang.org/grpc/codes" @@ -59,6 +60,7 @@ func New(opts ...Option) *Service { s.replicatorTimeout = defaultReplicatorSendTimeout s.syncBatchSize = defaultSyncBatchSize s.metrics = defaultMetricsRegister{} + s.authorizedKeys.Store(&[][]byte{}) for i := range opts { opts[i](&s.cfg) @@ -782,3 +784,15 @@ func (s *Service) Healthcheck(context.Context, *HealthcheckRequest) (*Healthchec return new(HealthcheckResponse), nil } + +func (s *Service) ReloadAuthorizedKeys(newKeys keys.PublicKeys) { + s.authorizedKeys.Store(fromPublicKeys(newKeys)) +} + +func fromPublicKeys(keys keys.PublicKeys) *[][]byte { + buff := make([][]byte, len(keys)) + for i, k := range keys { + buff[i] = k.Bytes() + } + return &buff +} diff --git a/pkg/services/tree/signature.go b/pkg/services/tree/signature.go index 89056056d..8221a4546 100644 --- a/pkg/services/tree/signature.go +++ b/pkg/services/tree/signature.go @@ -95,8 +95,8 @@ func (s *Service) isAuthorized(req message, op acl.Op) (bool, error) { } key := sign.GetKey() - for i := range s.authorizedKeys { - if bytes.Equal(s.authorizedKeys[i], key) { + for _, currentKey := range *s.authorizedKeys.Load() { + if bytes.Equal(currentKey, key) { return true, nil } } diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index 947de8e58..f5659d5e2 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -152,6 +152,7 @@ func TestMessageSign(t *testing.T) { apeChecker: checkercore.New(e.LocalStorage(), e.MorphRuleChainStorage(), frostfsidProvider, dummyEpochSource{}), } + s.cfg.authorizedKeys.Store(&[][]byte{}) rawCID1 := make([]byte, sha256.Size) cid1.Encode(rawCID1) From 4bcb67263076baebb2192c49c4d9c6e8d46627cb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 18 Apr 2025 18:28:36 +0300 Subject: [PATCH 1932/1943] [#1707] tree: Pass tracing context in `forEachNode` Change-Id: I884dcd215bd3934f9b4ea43dcc77e38f9dadcf10 Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/redirect.go | 14 +++++++------- pkg/services/tree/service.go | 8 ++++---- pkg/services/tree/sync.go | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/services/tree/redirect.go b/pkg/services/tree/redirect.go index 3dcdc4fc7..647f8cb30 100644 --- a/pkg/services/tree/redirect.go +++ b/pkg/services/tree/redirect.go @@ -19,8 +19,8 @@ var errNoSuitableNode = errors.New("no node was found to execute the request") func relayUnary[Req any, Resp any](ctx context.Context, s *Service, ns []netmapSDK.NodeInfo, req *Req, callback func(TreeServiceClient, context.Context, *Req, ...grpc.CallOption) (*Resp, error)) (*Resp, error) { var resp *Resp var outErr error - err := s.forEachNode(ctx, ns, func(c TreeServiceClient) bool { - resp, outErr = callback(c, ctx, req) + err := s.forEachNode(ctx, ns, func(fCtx context.Context, c TreeServiceClient) bool { + resp, outErr = callback(c, fCtx, req) return true }) if err != nil { @@ -31,7 +31,7 @@ func relayUnary[Req any, Resp any](ctx context.Context, s *Service, ns []netmapS // forEachNode executes callback for each node in the container until true is returned. // Returns errNoSuitableNode if there was no successful attempt to dial any node. -func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo, f func(c TreeServiceClient) bool) error { +func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo, f func(context.Context, TreeServiceClient) bool) error { for _, n := range cntNodes { if bytes.Equal(n.PublicKey(), s.rawPub) { return nil @@ -42,9 +42,9 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo for _, n := range cntNodes { var stop bool for endpoint := range n.NetworkEndpoints() { - stop = s.execOnClient(ctx, endpoint, func(c TreeServiceClient) bool { + stop = s.execOnClient(ctx, endpoint, func(fCtx context.Context, c TreeServiceClient) bool { called = true - return f(c) + return f(fCtx, c) }) if called { break @@ -60,7 +60,7 @@ func (s *Service) forEachNode(ctx context.Context, cntNodes []netmapSDK.NodeInfo return nil } -func (s *Service) execOnClient(ctx context.Context, endpoint string, f func(TreeServiceClient) bool) bool { +func (s *Service) execOnClient(ctx context.Context, endpoint string, f func(context.Context, TreeServiceClient) bool) bool { ctx, span := tracing.StartSpanFromContext(ctx, "TreeService.IterateNetworkEndpoints", trace.WithAttributes( attribute.String("endpoint", endpoint), @@ -73,5 +73,5 @@ func (s *Service) execOnClient(ctx context.Context, endpoint string, f func(Tree } s.log.Debug(ctx, logs.TreeRedirectingTreeServiceQuery, zap.String("endpoint", endpoint)) - return f(c) + return f(ctx, c) } diff --git a/pkg/services/tree/service.go b/pkg/services/tree/service.go index a4bc0c97c..3994d6973 100644 --- a/pkg/services/tree/service.go +++ b/pkg/services/tree/service.go @@ -391,8 +391,8 @@ func (s *Service) GetSubTree(req *GetSubTreeRequest, srv TreeService_GetSubTreeS if pos < 0 { var cli TreeService_GetSubTreeClient var outErr error - err = s.forEachNode(srv.Context(), ns, func(c TreeServiceClient) bool { - cli, outErr = c.GetSubTree(srv.Context(), req) + err = s.forEachNode(srv.Context(), ns, func(fCtx context.Context, c TreeServiceClient) bool { + cli, outErr = c.GetSubTree(fCtx, req) return true }) if err != nil { @@ -655,8 +655,8 @@ func (s *Service) GetOpLog(req *GetOpLogRequest, srv TreeService_GetOpLogServer) if pos < 0 { var cli TreeService_GetOpLogClient var outErr error - err := s.forEachNode(srv.Context(), ns, func(c TreeServiceClient) bool { - cli, outErr = c.GetOpLog(srv.Context(), req) + err := s.forEachNode(srv.Context(), ns, func(fCtx context.Context, c TreeServiceClient) bool { + cli, outErr = c.GetOpLog(fCtx, req) return true }) if err != nil { diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index 32297f9d7..d4040337d 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -76,8 +76,8 @@ func (s *Service) synchronizeAllTrees(ctx context.Context, cid cid.ID) error { var treesToSync []string var outErr error - err = s.forEachNode(ctx, nodes, func(c TreeServiceClient) bool { - resp, outErr = c.TreeList(ctx, req) + err = s.forEachNode(ctx, nodes, func(fCtx context.Context, c TreeServiceClient) bool { + resp, outErr = c.TreeList(fCtx, req) if outErr != nil { return false } From a27e0035083b8e66ee446c949f98405c1ac2791d Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 21 Apr 2025 10:53:51 +0300 Subject: [PATCH 1933/1943] [#1709] qos: Add context.Done check before schedule request Do not push request to schedule queue, if context is already cancelled. Change-Id: Ieef837a7d423158e3dbb0c3b4efecaa20744c845 Signed-off-by: Dmitrii Stepanov --- internal/qos/limiter.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/qos/limiter.go b/internal/qos/limiter.go index 5851d7626..2d7de32fc 100644 --- a/internal/qos/limiter.go +++ b/internal/qos/limiter.go @@ -149,6 +149,11 @@ func (n *mClockLimiter) WriteRequest(ctx context.Context) (ReleaseFunc, error) { } func requestArrival(ctx context.Context, s scheduler, stats map[string]*stat) (ReleaseFunc, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } tag, ok := tagging.IOTagFromContext(ctx) if !ok { tag = IOTagClient.String() From 3a441f072ffb955c6513eb7dd7733da311590ad7 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 21 Apr 2025 12:13:32 +0300 Subject: [PATCH 1934/1943] [#1709] shard: Check if context canceled for shard iteration If context has already been canceled, then there is no need to check other shards. At the same time, it is necessary to avoid handling context cancellation in each handler. Therefore, the context check has been moved to the shard iteration method, which now returns an error. Change-Id: I70030ace36593ce7d2b8376bee39fe82e9dbf88f Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + pkg/local_object_storage/engine/container.go | 27 ++++--- pkg/local_object_storage/engine/delete.go | 12 +-- pkg/local_object_storage/engine/exists.go | 6 +- pkg/local_object_storage/engine/get.go | 16 ++-- pkg/local_object_storage/engine/head.go | 6 +- pkg/local_object_storage/engine/inhume.go | 77 ++++++++++++-------- pkg/local_object_storage/engine/lock.go | 16 +++- pkg/local_object_storage/engine/put.go | 6 +- pkg/local_object_storage/engine/range.go | 16 ++-- pkg/local_object_storage/engine/select.go | 30 +++++--- pkg/local_object_storage/engine/shards.go | 22 +++++- 12 files changed, 149 insertions(+), 86 deletions(-) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index dedc7e12c..626372f43 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -198,6 +198,7 @@ const ( EngineInterruptProcessingTheExpiredLocks = "interrupt processing the expired locks" EngineInterruptGettingLockers = "can't get object's lockers" EngineInterruptProcessingTheDeletedLocks = "interrupt processing the deleted locks" + EngineInterruptProcessingTheExpiredTombstones = "interrupt processing the expired tombstones" EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly = "failed to move shard in degraded-read-only mode, moving to read-only" EngineFailedToMoveShardInReadonlyMode = "failed to move shard in read-only mode" EngineShardIsMovedInReadonlyModeDueToErrorThreshold = "shard is moved in read-only mode due to error threshold" diff --git a/pkg/local_object_storage/engine/container.go b/pkg/local_object_storage/engine/container.go index 03a299b93..e0617a832 100644 --- a/pkg/local_object_storage/engine/container.go +++ b/pkg/local_object_storage/engine/container.go @@ -48,8 +48,9 @@ func (e *StorageEngine) ContainerSize(ctx context.Context, prm ContainerSizePrm) defer elapsed("ContainerSize", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res = e.containerSize(ctx, prm) - return nil + var csErr error + res, csErr = e.containerSize(ctx, prm) + return csErr }) return @@ -69,8 +70,9 @@ func ContainerSize(ctx context.Context, e *StorageEngine, id cid.ID) (uint64, er return res.Size(), nil } -func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) (res ContainerSizeRes) { - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { +func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) (ContainerSizeRes, error) { + var res ContainerSizeRes + err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { var csPrm shard.ContainerSizePrm csPrm.SetContainerID(prm.cnr) @@ -86,7 +88,7 @@ func (e *StorageEngine) containerSize(ctx context.Context, prm ContainerSizePrm) return false }) - return + return res, err } // ListContainers returns a unique container IDs presented in the engine objects. @@ -96,8 +98,9 @@ func (e *StorageEngine) ListContainers(ctx context.Context, _ ListContainersPrm) defer elapsed("ListContainers", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res = e.listContainers(ctx) - return nil + var lcErr error + res, lcErr = e.listContainers(ctx) + return lcErr }) return @@ -115,10 +118,10 @@ func ListContainers(ctx context.Context, e *StorageEngine) ([]cid.ID, error) { return res.Containers(), nil } -func (e *StorageEngine) listContainers(ctx context.Context) ListContainersRes { +func (e *StorageEngine) listContainers(ctx context.Context) (ListContainersRes, error) { uniqueIDs := make(map[string]cid.ID) - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { res, err := sh.ListContainers(ctx, shard.ListContainersPrm{}) if err != nil { e.reportShardError(ctx, sh, "can't get list of containers", err) @@ -133,7 +136,9 @@ func (e *StorageEngine) listContainers(ctx context.Context) ListContainersRes { } return false - }) + }); err != nil { + return ListContainersRes{}, err + } result := make([]cid.ID, 0, len(uniqueIDs)) for _, v := range uniqueIDs { @@ -142,5 +147,5 @@ func (e *StorageEngine) listContainers(ctx context.Context) ListContainersRes { return ListContainersRes{ containers: result, - } + }, nil } diff --git a/pkg/local_object_storage/engine/delete.go b/pkg/local_object_storage/engine/delete.go index 5e5f65fa2..223cdbc48 100644 --- a/pkg/local_object_storage/engine/delete.go +++ b/pkg/local_object_storage/engine/delete.go @@ -71,7 +71,7 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) error { // Removal of a big object is done in multiple stages: // 1. Remove the parent object. If it is locked or already removed, return immediately. // 2. Otherwise, search for all objects with a particular SplitID and delete them too. - e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { + if err := e.iterateOverSortedShards(ctx, prm.addr, func(_ int, sh hashedShard) (stop bool) { var existsPrm shard.ExistsPrm existsPrm.Address = prm.addr @@ -116,20 +116,22 @@ func (e *StorageEngine) delete(ctx context.Context, prm DeletePrm) error { // If a parent object is removed we should set GC mark on each shard. return splitInfo == nil - }) + }); err != nil { + return err + } if locked.is { return new(apistatus.ObjectLocked) } if splitInfo != nil { - e.deleteChildren(ctx, prm.addr, prm.forceRemoval, splitInfo.SplitID()) + return e.deleteChildren(ctx, prm.addr, prm.forceRemoval, splitInfo.SplitID()) } return nil } -func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, force bool, splitID *objectSDK.SplitID) { +func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, force bool, splitID *objectSDK.SplitID) error { var fs objectSDK.SearchFilters fs.AddSplitIDFilter(objectSDK.MatchStringEqual, splitID) @@ -142,7 +144,7 @@ func (e *StorageEngine) deleteChildren(ctx context.Context, addr oid.Address, fo inhumePrm.ForceRemoval() } - e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { + return e.iterateOverSortedShards(ctx, addr, func(_ int, sh hashedShard) (stop bool) { res, err := sh.Select(ctx, selectPrm) if err != nil { e.log.Warn(ctx, logs.EngineErrorDuringSearchingForObjectChildren, diff --git a/pkg/local_object_storage/engine/exists.go b/pkg/local_object_storage/engine/exists.go index 9d2b1c1b7..7dac9eb97 100644 --- a/pkg/local_object_storage/engine/exists.go +++ b/pkg/local_object_storage/engine/exists.go @@ -18,7 +18,7 @@ func (e *StorageEngine) exists(ctx context.Context, shPrm shard.ExistsPrm) (bool exists := false locked := false - e.iterateOverSortedShards(shPrm.Address, func(_ int, sh hashedShard) (stop bool) { + if err := e.iterateOverSortedShards(ctx, shPrm.Address, func(_ int, sh hashedShard) (stop bool) { res, err := sh.Exists(ctx, shPrm) if err != nil { if client.IsErrObjectAlreadyRemoved(err) { @@ -50,7 +50,9 @@ func (e *StorageEngine) exists(ctx context.Context, shPrm shard.ExistsPrm) (bool } return false - }) + }); err != nil { + return false, false, err + } if alreadyRemoved { return false, false, new(apistatus.ObjectAlreadyRemoved) diff --git a/pkg/local_object_storage/engine/get.go b/pkg/local_object_storage/engine/get.go index 74c64bbb6..0694c53f3 100644 --- a/pkg/local_object_storage/engine/get.go +++ b/pkg/local_object_storage/engine/get.go @@ -78,7 +78,9 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { Engine: e, } - it.tryGetWithMeta(ctx) + if err := it.tryGetWithMeta(ctx); err != nil { + return GetRes{}, err + } if it.SplitInfo != nil { return GetRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(it.SplitInfo)) @@ -97,7 +99,9 @@ func (e *StorageEngine) get(ctx context.Context, prm GetPrm) (GetRes, error) { return GetRes{}, it.OutError } - it.tryGetFromBlobstore(ctx) + if err := it.tryGetFromBlobstore(ctx); err != nil { + return GetRes{}, err + } if it.Object == nil { return GetRes{}, it.OutError @@ -133,8 +137,8 @@ type getShardIterator struct { ecInfoErr *objectSDK.ECInfoError } -func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { - i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { +func (i *getShardIterator) tryGetWithMeta(ctx context.Context) error { + return i.Engine.iterateOverSortedShards(ctx, i.Address, func(_ int, sh hashedShard) (stop bool) { noMeta := sh.GetMode().NoMetabase() i.ShardPrm.SetIgnoreMeta(noMeta) @@ -187,13 +191,13 @@ func (i *getShardIterator) tryGetWithMeta(ctx context.Context) { }) } -func (i *getShardIterator) tryGetFromBlobstore(ctx context.Context) { +func (i *getShardIterator) tryGetFromBlobstore(ctx context.Context) error { // If the object is not found but is present in metabase, // try to fetch it from blobstor directly. If it is found in any // blobstor, increase the error counter for the shard which contains the meta. i.ShardPrm.SetIgnoreMeta(true) - i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { + return i.Engine.iterateOverSortedShards(ctx, i.Address, func(_ int, sh hashedShard) (stop bool) { if sh.GetMode().NoMetabase() { // Already visited. return false diff --git a/pkg/local_object_storage/engine/head.go b/pkg/local_object_storage/engine/head.go index d6892f129..d436dd411 100644 --- a/pkg/local_object_storage/engine/head.go +++ b/pkg/local_object_storage/engine/head.go @@ -82,7 +82,7 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) shPrm.SetAddress(prm.addr) shPrm.SetRaw(prm.raw) - e.iterateOverSortedShards(prm.addr, func(_ int, sh hashedShard) (stop bool) { + if err := e.iterateOverSortedShards(ctx, prm.addr, func(_ int, sh hashedShard) (stop bool) { shPrm.ShardLooksBad = sh.errorCount.Load() >= e.errorsThreshold res, err := sh.Head(ctx, shPrm) if err != nil { @@ -123,7 +123,9 @@ func (e *StorageEngine) head(ctx context.Context, prm HeadPrm) (HeadRes, error) } head = res.Object() return true - }) + }); err != nil { + return HeadRes{}, err + } if head != nil { return HeadRes{head: head}, nil diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index e5a2396e1..e13f04927 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -158,7 +158,7 @@ func (e *StorageEngine) findShards(ctx context.Context, addr oid.Address, checkL objectExists bool ) - e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { + if err := e.iterateOverSortedShards(ctx, addr, func(_ int, sh hashedShard) (stop bool) { objectExists = false prm.Address = addr @@ -209,7 +209,9 @@ func (e *StorageEngine) findShards(ctx context.Context, addr oid.Address, checkL // Continue if it's a root object. return !isRootObject - }) + }); err != nil { + return nil, err + } if retErr != nil { return nil, retErr @@ -229,7 +231,7 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e var err error var outErr error - e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(h hashedShard) (stop bool) { locked, err = h.IsLocked(ctx, addr) if err != nil { e.reportShardError(ctx, h, "can't check object's lockers", err, zap.Stringer("address", addr)) @@ -238,7 +240,9 @@ func (e *StorageEngine) IsLocked(ctx context.Context, addr oid.Address) (bool, e } return locked - }) + }); err != nil { + return false, err + } if locked { return locked, nil @@ -258,7 +262,7 @@ func (e *StorageEngine) GetLocks(ctx context.Context, addr oid.Address) ([]oid.I var allLocks []oid.ID var outErr error - e.iterateOverUnsortedShards(func(h hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(h hashedShard) (stop bool) { locks, err := h.GetLocks(ctx, addr) if err != nil { e.reportShardError(ctx, h, logs.EngineInterruptGettingLockers, err, zap.Stringer("address", addr)) @@ -266,7 +270,9 @@ func (e *StorageEngine) GetLocks(ctx context.Context, addr oid.Address) ([]oid.I } allLocks = append(allLocks, locks...) return false - }) + }); err != nil { + return nil, err + } if len(allLocks) > 0 { return allLocks, nil } @@ -274,20 +280,23 @@ func (e *StorageEngine) GetLocks(ctx context.Context, addr oid.Address) ([]oid.I } func (e *StorageEngine) processExpiredTombstones(ctx context.Context, addrs []meta.TombstonedObject) { - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { sh.HandleExpiredTombstones(ctx, addrs) select { case <-ctx.Done(): + e.log.Info(ctx, logs.EngineInterruptProcessingTheExpiredTombstones, zap.Error(ctx.Err())) return true default: return false } - }) + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingTheExpiredTombstones, zap.Error(err)) + } } func (e *StorageEngine) processExpiredLocks(ctx context.Context, epoch uint64, lockers []oid.Address) { - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { sh.HandleExpiredLocks(ctx, epoch, lockers) select { @@ -297,11 +306,13 @@ func (e *StorageEngine) processExpiredLocks(ctx context.Context, epoch uint64, l default: return false } - }) + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingTheExpiredLocks, zap.Error(err)) + } } func (e *StorageEngine) processDeletedLocks(ctx context.Context, lockers []oid.Address) { - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { sh.HandleDeletedLocks(ctx, lockers) select { @@ -311,26 +322,25 @@ func (e *StorageEngine) processDeletedLocks(ctx context.Context, lockers []oid.A default: return false } - }) + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingTheDeletedLocks, zap.Error(err)) + } } func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid.ID) { if len(ids) == 0 { return } - idMap, err := e.selectNonExistentIDs(ctx, ids) if err != nil { return } - if len(idMap) == 0 { return } - var failed bool var prm shard.ContainerSizePrm - e.iterateOverUnsortedShards(func(sh hashedShard) bool { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) bool { select { case <-ctx.Done(): e.log.Info(ctx, logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(ctx.Err())) @@ -357,13 +367,15 @@ func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid } return len(idMap) == 0 - }) - + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(err)) + return + } if failed || len(idMap) == 0 { return } - e.iterateOverUnsortedShards(func(sh hashedShard) bool { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) bool { select { case <-ctx.Done(): e.log.Info(ctx, logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(ctx.Err())) @@ -381,12 +393,13 @@ func (e *StorageEngine) processZeroSizeContainers(ctx context.Context, ids []cid } return false - }) - + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingZeroSizeContainers, zap.Error(err)) + return + } if failed { return } - for id := range idMap { e.metrics.DeleteContainerSize(id.EncodeToString()) } @@ -396,19 +409,16 @@ func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []ci if len(ids) == 0 { return } - idMap, err := e.selectNonExistentIDs(ctx, ids) if err != nil { return } - if len(idMap) == 0 { return } - var failed bool var prm shard.ContainerCountPrm - e.iterateOverUnsortedShards(func(sh hashedShard) bool { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) bool { select { case <-ctx.Done(): e.log.Info(ctx, logs.EngineInterruptProcessingZeroCountContainers, zap.Error(ctx.Err())) @@ -435,13 +445,15 @@ func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []ci } return len(idMap) == 0 - }) - + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingZeroCountContainers, zap.Error(err)) + return + } if failed || len(idMap) == 0 { return } - e.iterateOverUnsortedShards(func(sh hashedShard) bool { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) bool { select { case <-ctx.Done(): e.log.Info(ctx, logs.EngineInterruptProcessingZeroCountContainers, zap.Error(ctx.Err())) @@ -459,12 +471,13 @@ func (e *StorageEngine) processZeroCountContainers(ctx context.Context, ids []ci } return false - }) - + }); err != nil { + e.log.Info(ctx, logs.EngineInterruptProcessingZeroCountContainers, zap.Error(err)) + return + } if failed { return } - for id := range idMap { e.metrics.DeleteContainerCount(id.EncodeToString()) } diff --git a/pkg/local_object_storage/engine/lock.go b/pkg/local_object_storage/engine/lock.go index 18fff9cad..3b0cf74f9 100644 --- a/pkg/local_object_storage/engine/lock.go +++ b/pkg/local_object_storage/engine/lock.go @@ -41,11 +41,19 @@ func (e *StorageEngine) Lock(ctx context.Context, idCnr cid.ID, locker oid.ID, l func (e *StorageEngine) lock(ctx context.Context, idCnr cid.ID, locker oid.ID, locked []oid.ID) error { for i := range locked { - switch e.lockSingle(ctx, idCnr, locker, locked[i], true) { + st, err := e.lockSingle(ctx, idCnr, locker, locked[i], true) + if err != nil { + return err + } + switch st { case 1: return logicerr.Wrap(new(apistatus.LockNonRegularObject)) case 0: - switch e.lockSingle(ctx, idCnr, locker, locked[i], false) { + st, err = e.lockSingle(ctx, idCnr, locker, locked[i], false) + if err != nil { + return err + } + switch st { case 1: return logicerr.Wrap(new(apistatus.LockNonRegularObject)) case 0: @@ -61,13 +69,13 @@ func (e *StorageEngine) lock(ctx context.Context, idCnr cid.ID, locker oid.ID, l // - 0: fail // - 1: locking irregular object // - 2: ok -func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, locked oid.ID, checkExists bool) (status uint8) { +func (e *StorageEngine) lockSingle(ctx context.Context, idCnr cid.ID, locker, locked oid.ID, checkExists bool) (status uint8, retErr error) { // code is pretty similar to inhumeAddr, maybe unify? root := false var addrLocked oid.Address addrLocked.SetContainer(idCnr) addrLocked.SetObject(locked) - e.iterateOverSortedShards(addrLocked, func(_ int, sh hashedShard) (stop bool) { + retErr = e.iterateOverSortedShards(ctx, addrLocked, func(_ int, sh hashedShard) (stop bool) { defer func() { // if object is root we continue since information about it // can be presented in other shards diff --git a/pkg/local_object_storage/engine/put.go b/pkg/local_object_storage/engine/put.go index b348d13a2..10cf5ffd5 100644 --- a/pkg/local_object_storage/engine/put.go +++ b/pkg/local_object_storage/engine/put.go @@ -96,7 +96,7 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { } var shRes putToShardRes - e.iterateOverSortedShards(addr, func(_ int, sh hashedShard) (stop bool) { + if err := e.iterateOverSortedShards(ctx, addr, func(_ int, sh hashedShard) (stop bool) { e.mtx.RLock() _, ok := e.shards[sh.ID().String()] e.mtx.RUnlock() @@ -106,7 +106,9 @@ func (e *StorageEngine) put(ctx context.Context, prm PutPrm) error { } shRes = e.putToShard(ctx, sh, addr, prm.Object, prm.IsIndexedContainer) return shRes.status != putToShardUnknown - }) + }); err != nil { + return err + } switch shRes.status { case putToShardUnknown: return errPutShard diff --git a/pkg/local_object_storage/engine/range.go b/pkg/local_object_storage/engine/range.go index a468cf594..7ec4742d8 100644 --- a/pkg/local_object_storage/engine/range.go +++ b/pkg/local_object_storage/engine/range.go @@ -93,7 +93,9 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error Engine: e, } - it.tryGetWithMeta(ctx) + if err := it.tryGetWithMeta(ctx); err != nil { + return RngRes{}, err + } if it.SplitInfo != nil { return RngRes{}, logicerr.Wrap(objectSDK.NewSplitInfoError(it.SplitInfo)) @@ -109,7 +111,9 @@ func (e *StorageEngine) getRange(ctx context.Context, prm RngPrm) (RngRes, error return RngRes{}, it.OutError } - it.tryGetFromBlobstor(ctx) + if err := it.tryGetFromBlobstor(ctx); err != nil { + return RngRes{}, err + } if it.Object == nil { return RngRes{}, it.OutError @@ -157,8 +161,8 @@ type getRangeShardIterator struct { Engine *StorageEngine } -func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) { - i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { +func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) error { + return i.Engine.iterateOverSortedShards(ctx, i.Address, func(_ int, sh hashedShard) (stop bool) { noMeta := sh.GetMode().NoMetabase() i.HasDegraded = i.HasDegraded || noMeta i.ShardPrm.SetIgnoreMeta(noMeta) @@ -209,13 +213,13 @@ func (i *getRangeShardIterator) tryGetWithMeta(ctx context.Context) { }) } -func (i *getRangeShardIterator) tryGetFromBlobstor(ctx context.Context) { +func (i *getRangeShardIterator) tryGetFromBlobstor(ctx context.Context) error { // If the object is not found but is present in metabase, // try to fetch it from blobstor directly. If it is found in any // blobstor, increase the error counter for the shard which contains the meta. i.ShardPrm.SetIgnoreMeta(true) - i.Engine.iterateOverSortedShards(i.Address, func(_ int, sh hashedShard) (stop bool) { + return i.Engine.iterateOverSortedShards(ctx, i.Address, func(_ int, sh hashedShard) (stop bool) { if sh.GetMode().NoMetabase() { // Already processed it without a metabase. return false diff --git a/pkg/local_object_storage/engine/select.go b/pkg/local_object_storage/engine/select.go index fc8b4a9a7..4243a5481 100644 --- a/pkg/local_object_storage/engine/select.go +++ b/pkg/local_object_storage/engine/select.go @@ -54,14 +54,15 @@ func (e *StorageEngine) Select(ctx context.Context, prm SelectPrm) (res SelectRe defer elapsed("Select", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res = e._select(ctx, prm) - return nil + var sErr error + res, sErr = e._select(ctx, prm) + return sErr }) return } -func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) SelectRes { +func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) (SelectRes, error) { addrList := make([]oid.Address, 0) uniqueMap := make(map[string]struct{}) @@ -69,7 +70,7 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) SelectRes { shPrm.SetContainerID(prm.cnr, prm.indexedContainer) shPrm.SetFilters(prm.filters) - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { res, err := sh.Select(ctx, shPrm) if err != nil { e.reportShardError(ctx, sh, "could not select objects from shard", err) @@ -84,11 +85,13 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) SelectRes { } return false - }) + }); err != nil { + return SelectRes{}, err + } return SelectRes{ addrList: addrList, - } + }, nil } // List returns `limit` available physically storage object addresses in engine. @@ -98,20 +101,21 @@ func (e *StorageEngine) _select(ctx context.Context, prm SelectPrm) SelectRes { func (e *StorageEngine) List(ctx context.Context, limit uint64) (res SelectRes, err error) { defer elapsed("List", e.metrics.AddMethodDuration)() err = e.execIfNotBlocked(func() error { - res = e.list(ctx, limit) - return nil + var lErr error + res, lErr = e.list(ctx, limit) + return lErr }) return } -func (e *StorageEngine) list(ctx context.Context, limit uint64) SelectRes { +func (e *StorageEngine) list(ctx context.Context, limit uint64) (SelectRes, error) { addrList := make([]oid.Address, 0, limit) uniqueMap := make(map[string]struct{}) ln := uint64(0) // consider iterating over shuffled shards - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { + if err := e.iterateOverUnsortedShards(ctx, func(sh hashedShard) (stop bool) { res, err := sh.List(ctx) // consider limit result of shard iterator if err != nil { e.reportShardError(ctx, sh, "could not select objects from shard", err) @@ -130,11 +134,13 @@ func (e *StorageEngine) list(ctx context.Context, limit uint64) SelectRes { } return false - }) + }); err != nil { + return SelectRes{}, err + } return SelectRes{ addrList: addrList, - } + }, nil } // Select selects objects from local storage using provided filters. diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 6e6c08bb5..69067c500 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -280,20 +280,32 @@ func (e *StorageEngine) unsortedShards() []hashedShard { return shards } -func (e *StorageEngine) iterateOverSortedShards(addr oid.Address, handler func(int, hashedShard) (stop bool)) { +func (e *StorageEngine) iterateOverSortedShards(ctx context.Context, addr oid.Address, handler func(int, hashedShard) (stop bool)) error { for i, sh := range e.sortShards(addr) { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } if handler(i, sh) { break } } + return nil } -func (e *StorageEngine) iterateOverUnsortedShards(handler func(hashedShard) (stop bool)) { +func (e *StorageEngine) iterateOverUnsortedShards(ctx context.Context, handler func(hashedShard) (stop bool)) error { for _, sh := range e.unsortedShards() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } if handler(sh) { break } } + return nil } // SetShardMode sets mode of the shard with provided identifier. @@ -433,7 +445,7 @@ func (e *StorageEngine) ListShardsForObject(ctx context.Context, obj oid.Address var siErr *objectSDK.SplitInfoError var ecErr *objectSDK.ECInfoError - e.iterateOverUnsortedShards(func(hs hashedShard) (stop bool) { + if itErr := e.iterateOverUnsortedShards(ctx, func(hs hashedShard) (stop bool) { res, exErr := hs.Exists(ctx, prm) if exErr != nil { if client.IsErrObjectAlreadyRemoved(exErr) { @@ -463,6 +475,8 @@ func (e *StorageEngine) ListShardsForObject(ctx context.Context, obj oid.Address info = append(info, hs.DumpInfo()) } return false - }) + }); itErr != nil { + return nil, itErr + } return info, err } From 908b08108d7781bad2054d449c427389f2dfe9f3 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 21 Apr 2025 11:59:21 +0300 Subject: [PATCH 1935/1943] [#1689] engine/test: Fix `TestInhumeIfObjectDoesntExist` test Removed an invalid test case which used exclusive options, added object status check after removal. Change-Id: I4551c0e4532fb669ee6c72871dc4bd34707d8469 Signed-off-by: Aleksey Savchuk --- .../engine/inhume_test.go | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 0601a43f2..fa73dcad5 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -11,6 +11,7 @@ import ( meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" objectV2 "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" @@ -251,21 +252,23 @@ func TestInhumeIfObjectDoesntExist(t *testing.T) { t.Run("inhume with tombstone", func(t *testing.T) { testInhumeLockedIfObjectDoesntExist(t, true, false) }) - t.Run("force inhume without tombstone", func(t *testing.T) { + t.Run("force inhume", func(t *testing.T) { testInhumeLockedIfObjectDoesntExist(t, false, true) }) - t.Run("force inhume with tombstone", func(t *testing.T) { - testInhumeLockedIfObjectDoesntExist(t, true, true) - }) }) } func testInhumeLockedIfObjectDoesntExist(t *testing.T, withTombstone, withForce bool) { t.Parallel() + // Due to the tests design it is possible to set both the options, + // however removal with tombstone and force removal are exclusive. + require.False(t, withTombstone && withForce) + var ( errLocked *apistatus.ObjectLocked inhumePrm InhumePrm + headPrm HeadPrm ctx = context.Background() container = cidtest.ID() object = oidtest.Address() @@ -291,9 +294,17 @@ func testInhumeLockedIfObjectDoesntExist(t *testing.T, withTombstone, withForce } err = engine.Inhume(ctx, inhumePrm) - if withForce { - require.NoError(t, err) - } else { + if !withForce { require.ErrorAs(t, err, &errLocked) + return + } + require.NoError(t, err) + + headPrm.WithAddress(object) + _, err = engine.Head(ctx, headPrm) + if withTombstone { + require.True(t, client.IsErrObjectAlreadyRemoved(err)) + } else { + require.True(t, client.IsErrObjectNotFound(err)) } } From 487cb34c5dcf21bf385d8ddf8a397652260101a8 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 21 Apr 2025 13:26:32 +0300 Subject: [PATCH 1936/1943] [#1689] engine/test: Refactor `TestInhumeIfObjectDoesntExist` test - Use the same storage engine in multiple parallel tests - Move `Lock`, `Inhume`, `Head` calls to separate functions Change-Id: I00849c1f068f0ab8d92061719d67d6fe786200db Signed-off-by: Aleksey Savchuk --- .../engine/inhume_test.go | 84 ++++++++++--------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index fa73dcad5..813c329f2 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -245,66 +245,74 @@ func benchmarkInhumeMultipart(b *testing.B, numShards, numObjects int) { } func TestInhumeIfObjectDoesntExist(t *testing.T) { + const numShards = 4 + + engine := testNewEngine(t).setShardsNum(t, numShards).prepare(t).engine + t.Cleanup(func() { require.NoError(t, engine.Close(context.Background())) }) + t.Run("object is locked", func(t *testing.T) { t.Run("inhume without tombstone", func(t *testing.T) { - testInhumeLockedIfObjectDoesntExist(t, false, false) + testInhumeLockedIfObjectDoesntExist(t, engine, false, false) }) t.Run("inhume with tombstone", func(t *testing.T) { - testInhumeLockedIfObjectDoesntExist(t, true, false) + testInhumeLockedIfObjectDoesntExist(t, engine, true, false) }) t.Run("force inhume", func(t *testing.T) { - testInhumeLockedIfObjectDoesntExist(t, false, true) + testInhumeLockedIfObjectDoesntExist(t, engine, false, true) }) }) } -func testInhumeLockedIfObjectDoesntExist(t *testing.T, withTombstone, withForce bool) { +func testInhumeLockedIfObjectDoesntExist(t *testing.T, e *StorageEngine, withTombstone, withForce bool) { t.Parallel() - // Due to the tests design it is possible to set both the options, - // however removal with tombstone and force removal are exclusive. - require.False(t, withTombstone && withForce) + object := oidtest.Address() + require.NoError(t, testLockObject(e, object)) - var ( - errLocked *apistatus.ObjectLocked - inhumePrm InhumePrm - headPrm HeadPrm - ctx = context.Background() - container = cidtest.ID() - object = oidtest.Address() - lock = oidtest.ID() - tombstone = oidtest.Address() - ) - object.SetContainer(container) - tombstone.SetContainer(container) - - engine := testNewEngine(t).setShardsNum(t, 4).prepare(t).engine - defer func() { require.NoError(t, engine.Close(ctx)) }() - - err := engine.Lock(ctx, container, lock, []oid.ID{object.Object()}) - require.NoError(t, err) - - if withTombstone { - inhumePrm.WithTarget(tombstone, object) - } else { - inhumePrm.MarkAsGarbage(object) - } - if withForce { - inhumePrm.WithForceRemoval() - } - - err = engine.Inhume(ctx, inhumePrm) + err := testInhumeObject(t, e, object, withTombstone, withForce) if !withForce { + var errLocked *apistatus.ObjectLocked require.ErrorAs(t, err, &errLocked) return } require.NoError(t, err) - headPrm.WithAddress(object) - _, err = engine.Head(ctx, headPrm) + err = testHeadObject(e, object) if withTombstone { require.True(t, client.IsErrObjectAlreadyRemoved(err)) } else { require.True(t, client.IsErrObjectNotFound(err)) } } + +func testLockObject(e *StorageEngine, obj oid.Address) error { + return e.Lock(context.Background(), obj.Container(), oidtest.ID(), []oid.ID{obj.Object()}) +} + +func testInhumeObject(t testing.TB, e *StorageEngine, obj oid.Address, withTombstone, withForce bool) error { + tombstone := oidtest.Address() + tombstone.SetContainer(obj.Container()) + + // Due to the tests design it is possible to set both the options, + // however removal with tombstone and force removal are exclusive. + require.False(t, withTombstone && withForce) + + var inhumePrm InhumePrm + if withTombstone { + inhumePrm.WithTarget(tombstone, obj) + } else { + inhumePrm.MarkAsGarbage(obj) + } + if withForce { + inhumePrm.WithForceRemoval() + } + return e.Inhume(context.Background(), inhumePrm) +} + +func testHeadObject(e *StorageEngine, obj oid.Address) error { + var headPrm HeadPrm + headPrm.WithAddress(obj) + + _, err := e.Head(context.Background(), headPrm) + return err +} From 77b8545601f13ba4e596542be58b7e6f9c8e36d1 Mon Sep 17 00:00:00 2001 From: Aleksey Savchuk Date: Mon, 21 Apr 2025 13:31:54 +0300 Subject: [PATCH 1937/1943] [#1689] engine: Fix removal of objects not found on node Ensured correct object status if the object is not found on a node. Fixed regression introduced in #1450. Besides an object not being found on any shard, it also important to remove it anyway in order to populate the metabase indexes because they are responsible for the correct object status, i.e., the status will be `object not found` without the indexes, the status will be `object is already removed` with the indexes. Change-Id: I6237fbc0f8bb0c4f2a51ada3a68f52950050e660 Signed-off-by: Aleksey Savchuk --- pkg/local_object_storage/engine/inhume.go | 100 +++++++++++++++--- .../engine/inhume_test.go | 24 +++++ 2 files changed, 107 insertions(+), 17 deletions(-) diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index e13f04927..e5f7072e2 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -74,7 +74,7 @@ func (e *StorageEngine) Inhume(ctx context.Context, prm InhumePrm) error { } func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) error { - addrsPerShard, err := e.groupObjectsByShard(ctx, prm.addrs, !prm.forceRemoval) + addrsPerShard, notFoundObjects, err := e.groupObjectsByShard(ctx, prm.addrs, !prm.forceRemoval) if err != nil { return err } @@ -84,8 +84,6 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) error { shPrm.ForceRemoval() } - var errLocked *apistatus.ObjectLocked - for shardID, addrs := range addrsPerShard { if prm.tombstone != nil { shPrm.SetTarget(*prm.tombstone, addrs...) @@ -103,39 +101,107 @@ func (e *StorageEngine) inhume(ctx context.Context, prm InhumePrm) error { } if _, err := sh.Inhume(ctx, shPrm); err != nil { - switch { - case errors.As(err, &errLocked): - case errors.Is(err, shard.ErrLockObjectRemoval): - case errors.Is(err, shard.ErrReadOnlyMode): - case errors.Is(err, shard.ErrDegradedMode): - default: - e.reportShardError(ctx, sh, "couldn't inhume object in shard", err) - } + e.reportInhumeError(ctx, err, sh) return err } } - return nil + return e.inhumeNotFoundObjects(ctx, notFoundObjects, prm) +} + +func (e *StorageEngine) reportInhumeError(ctx context.Context, err error, hs hashedShard) { + if err == nil { + return + } + + var errLocked *apistatus.ObjectLocked + switch { + case errors.As(err, &errLocked): + case errors.Is(err, shard.ErrLockObjectRemoval): + case errors.Is(err, shard.ErrReadOnlyMode): + case errors.Is(err, shard.ErrDegradedMode): + default: + e.reportShardError(ctx, hs, "couldn't inhume object in shard", err) + } +} + +// inhumeNotFoundObjects removes object which are not found on any shard. +// +// Besides an object not being found on any shard, it is also important to +// remove it anyway in order to populate the metabase indexes because they are +// responsible for the correct object status, i.e., the status will be `object +// not found` without the indexes, the status will be `object is already +// removed` with the indexes. +// +// It is suggested to evenly remove those objects on each shard with the batch +// size equal to 1 + floor(number of objects / number of shards). +func (e *StorageEngine) inhumeNotFoundObjects(ctx context.Context, addrs []oid.Address, prm InhumePrm) error { + if len(addrs) == 0 { + return nil + } + + var shPrm shard.InhumePrm + if prm.forceRemoval { + shPrm.ForceRemoval() + } + + numObjectsPerShard := 1 + len(addrs)/len(e.shards) + + var inhumeErr error + itErr := e.iterateOverUnsortedShards(ctx, func(hs hashedShard) (stop bool) { + numObjects := min(numObjectsPerShard, len(addrs)) + + if numObjects == 0 { + return true + } + + if prm.tombstone != nil { + shPrm.SetTarget(*prm.tombstone, addrs[:numObjects]...) + } else { + shPrm.MarkAsGarbage(addrs[:numObjects]...) + } + addrs = addrs[numObjects:] + + _, inhumeErr = hs.Inhume(ctx, shPrm) + e.reportInhumeError(ctx, inhumeErr, hs) + return inhumeErr != nil + }) + if inhumeErr != nil { + return inhumeErr + } + return itErr } // groupObjectsByShard groups objects based on the shard(s) they are stored on. // // If checkLocked is set, [apistatus.ObjectLocked] will be returned if any of // the objects are locked. -func (e *StorageEngine) groupObjectsByShard(ctx context.Context, addrs []oid.Address, checkLocked bool) (map[string][]oid.Address, error) { - groups := make(map[string][]oid.Address) +// +// Returns two sets of objects: found objects which are grouped per shard and +// not found object. Not found objects are objects which are not found on any +// shard. This can happen if a node is a container node but doesn't participate +// in a replica group of the object. +func (e *StorageEngine) groupObjectsByShard(ctx context.Context, addrs []oid.Address, checkLocked bool) (groups map[string][]oid.Address, notFoundObjects []oid.Address, err error) { + groups = make(map[string][]oid.Address) + var ids []string for _, addr := range addrs { - ids, err := e.findShards(ctx, addr, checkLocked) + ids, err = e.findShards(ctx, addr, checkLocked) if err != nil { - return nil, err + return } + + if len(ids) == 0 { + notFoundObjects = append(notFoundObjects, addr) + continue + } + for _, id := range ids { groups[id] = append(groups[id], addr) } } - return groups, nil + return } // findShards determines the shard(s) where the object is stored. diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index 813c329f2..0e268cd23 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -250,6 +250,16 @@ func TestInhumeIfObjectDoesntExist(t *testing.T) { engine := testNewEngine(t).setShardsNum(t, numShards).prepare(t).engine t.Cleanup(func() { require.NoError(t, engine.Close(context.Background())) }) + t.Run("inhume without tombstone", func(t *testing.T) { + testInhumeIfObjectDoesntExist(t, engine, false, false) + }) + t.Run("inhume with tombstone", func(t *testing.T) { + testInhumeIfObjectDoesntExist(t, engine, true, false) + }) + t.Run("force inhume", func(t *testing.T) { + testInhumeIfObjectDoesntExist(t, engine, false, true) + }) + t.Run("object is locked", func(t *testing.T) { t.Run("inhume without tombstone", func(t *testing.T) { testInhumeLockedIfObjectDoesntExist(t, engine, false, false) @@ -263,6 +273,20 @@ func TestInhumeIfObjectDoesntExist(t *testing.T) { }) } +func testInhumeIfObjectDoesntExist(t *testing.T, e *StorageEngine, withTombstone, withForce bool) { + t.Parallel() + + object := oidtest.Address() + require.NoError(t, testInhumeObject(t, e, object, withTombstone, withForce)) + + err := testHeadObject(e, object) + if withTombstone { + require.True(t, client.IsErrObjectAlreadyRemoved(err)) + } else { + require.True(t, client.IsErrObjectNotFound(err)) + } +} + func testInhumeLockedIfObjectDoesntExist(t *testing.T, e *StorageEngine, withTombstone, withForce bool) { t.Parallel() From 6bdbe6a18b2fc666af67837a3b5ce6434ddbea96 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 22 Apr 2025 17:37:29 +0300 Subject: [PATCH 1938/1943] [#1689] adm: Fix `printSubjectInfo` method * Fix print format for primary and additional subject keys from frostfsid contract. Since the format corresponds to `neo-go wallet dump-keys` output format. Change-Id: I9ae9fd43bfb378970786b97bd3d9d7f739466ae6 Signed-off-by: Airat Arifullin --- .../internal/modules/morph/frostfsid/frostfsid.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go index 8ae606f1a..7f777db98 100644 --- a/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go +++ b/cmd/frostfs-adm/internal/modules/morph/frostfsid/frostfsid.go @@ -1,6 +1,7 @@ package frostfsid import ( + "encoding/hex" "errors" "fmt" "math/big" @@ -604,7 +605,7 @@ func printSubjectInfo(cmd *cobra.Command, addr util.Uint160, subj *frostfsidclie cmd.Printf("Address: %s\n", address.Uint160ToString(addr)) pk := "" if subj.PrimaryKey != nil { - pk = subj.PrimaryKey.String() + pk = hex.EncodeToString(subj.PrimaryKey.Bytes()) } cmd.Printf("Primary key: %s\n", pk) cmd.Printf("Name: %s\n", subj.Name) @@ -614,7 +615,7 @@ func printSubjectInfo(cmd *cobra.Command, addr util.Uint160, subj *frostfsidclie for _, key := range subj.AdditionalKeys { k := "" if key != nil { - k = key.String() + k = hex.EncodeToString(key.Bytes()) } cmd.Printf("- %s\n", k) } From b0f39dca16c9564c6a68da874ceab705288dde9a Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 22 Apr 2025 18:14:00 +0300 Subject: [PATCH 1939/1943] [#1721] object: Make `CheckAPE` always validate bearer token * The bearer token must always be validated, regardless of whether it has been impersonated; * Fix unit-tests for tree service which check verification with bearer token. Close #1721 Change-Id: I5f715c498ae10b2e758244e60b8f21849328a04f Signed-off-by: Airat Arifullin --- pkg/services/common/ape/checker.go | 12 ++++++++---- pkg/services/tree/signature_test.go | 28 +++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index c9b0b7363..a2e628144 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -73,14 +73,18 @@ func New(localOverrideStorage policyengine.LocalOverrideStorage, morphChainStora // CheckAPE performs the common policy-engine check logic on a prepared request. func (c *checkerCoreImpl) CheckAPE(ctx context.Context, prm CheckPrm) error { var cr policyengine.ChainRouter - if prm.BearerToken != nil && !prm.BearerToken.Impersonate() { + if prm.BearerToken != nil { var err error if err = isValidBearer(prm.BearerToken, prm.ContainerOwner, prm.Container, prm.PublicKey, c.State); err != nil { return fmt.Errorf("bearer validation error: %w", err) } - cr, err = router.BearerChainFeedRouter(c.LocalOverrideStorage, c.MorphChainStorage, prm.BearerToken.APEOverride()) - if err != nil { - return fmt.Errorf("create chain router error: %w", err) + if prm.BearerToken.Impersonate() { + cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.MorphChainStorage, c.LocalOverrideStorage) + } else { + cr, err = router.BearerChainFeedRouter(c.LocalOverrideStorage, c.MorphChainStorage, prm.BearerToken.APEOverride()) + if err != nil { + return fmt.Errorf("create chain router error: %w", err) + } } } else { cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.MorphChainStorage, c.LocalOverrideStorage) diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index f5659d5e2..dd37b4191 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -238,14 +238,40 @@ func TestMessageSign(t *testing.T) { t.Run("impersonate", func(t *testing.T) { cnr.Value.SetBasicACL(acl.PublicRWExtended) var bt bearer.Token + bt.SetExp(10) bt.SetImpersonate(true) + bt.SetAPEOverride(bearer.APEOverride{ + Target: ape.ChainTarget{ + TargetType: ape.TargetTypeContainer, + Name: cid1.EncodeToString(), + }, + Chains: []ape.Chain{}, + }) + require.NoError(t, bt.Sign(privs[0].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + }) + + t.Run("impersonate but invalid signer", func(t *testing.T) { + var bt bearer.Token + bt.SetExp(10) + bt.SetImpersonate(true) + bt.SetAPEOverride(bearer.APEOverride{ + Target: ape.ChainTarget{ + TargetType: ape.TargetTypeContainer, + Name: cid1.EncodeToString(), + }, + Chains: []ape.Chain{}, + }) require.NoError(t, bt.Sign(privs[1].PrivateKey)) req.Body.BearerToken = bt.Marshal() require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) - require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) + require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectGet)) }) bt := testBearerToken(cid1, privs[1].PublicKey(), privs[2].PublicKey()) From 5b6cba04cb5c3fb0589caa6e26bdc67c8ef183e2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 18 Apr 2025 18:17:00 +0300 Subject: [PATCH 1940/1943] [#1689] node: Drop `node.relay` config It is not used and not tested properly, so drop it. Change-Id: I7c90c7391ecb4be17459415d209811ba1a693f7a Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/attributes.go | 4 --- cmd/frostfs-node/config.go | 27 +++++---------------- cmd/frostfs-node/config/node/config.go | 8 ------ cmd/frostfs-node/config/node/config_test.go | 5 ---- cmd/frostfs-node/netmap.go | 20 +++++---------- config/example/node.env | 1 - config/example/node.json | 1 - config/example/node.yaml | 2 -- docs/storage-node-configuration.md | 2 -- 9 files changed, 12 insertions(+), 58 deletions(-) diff --git a/cmd/frostfs-node/attributes.go b/cmd/frostfs-node/attributes.go index 64c3beba7..ce8ae9662 100644 --- a/cmd/frostfs-node/attributes.go +++ b/cmd/frostfs-node/attributes.go @@ -6,9 +6,5 @@ import ( ) func parseAttributes(c *cfg) { - if nodeconfig.Relay(c.appCfg) { - return - } - fatalOnErr(attributes.ReadNodeAttributes(&c.cfgNodeInfo.localInfo, nodeconfig.Attributes(c.appCfg))) } diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index b688acfde..96274e625 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -651,7 +651,6 @@ type cfgNetmap struct { state *networkState - needBootstrap bool reBoostrapTurnedOff *atomic.Bool // managed by control service in runtime } @@ -710,11 +709,9 @@ func initCfg(appCfg *config.Config) *cfg { key := nodeconfig.Key(appCfg) - relayOnly := nodeconfig.Relay(appCfg) - netState := newNetworkState() - c.shared = initShared(appCfg, key, netState, relayOnly) + c.shared = initShared(appCfg, key, netState) netState.metrics = c.metricsCollector @@ -734,7 +731,7 @@ func initCfg(appCfg *config.Config) *cfg { c.cfgFrostfsID = initFrostfsID(appCfg) - c.cfgNetmap = initNetmap(appCfg, netState, relayOnly) + c.cfgNetmap = initNetmap(appCfg, netState) c.cfgGRPC = initCfgGRPC() @@ -780,12 +777,8 @@ func initSdNotify(appCfg *config.Config) bool { return false } -func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkState, relayOnly bool) shared { - var netAddr network.AddressGroup - - if !relayOnly { - netAddr = nodeconfig.BootstrapAddresses(appCfg) - } +func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkState) shared { + netAddr := nodeconfig.BootstrapAddresses(appCfg) persistate, err := state.NewPersistentStorage(nodeconfig.PersistentState(appCfg).Path()) fatalOnErr(err) @@ -836,18 +829,15 @@ func internalNetConfig(appCfg *config.Config, m metrics.MultinetMetrics) interna return result } -func initNetmap(appCfg *config.Config, netState *networkState, relayOnly bool) cfgNetmap { +func initNetmap(appCfg *config.Config, netState *networkState) cfgNetmap { netmapWorkerPool, err := ants.NewPool(notificationHandlerPoolSize) fatalOnErr(err) - var reBootstrapTurnedOff atomic.Bool - reBootstrapTurnedOff.Store(relayOnly) return cfgNetmap{ scriptHash: contractsconfig.Netmap(appCfg), state: netState, workerPool: netmapWorkerPool, - needBootstrap: !relayOnly, - reBoostrapTurnedOff: &reBootstrapTurnedOff, + reBoostrapTurnedOff: &atomic.Bool{}, } } @@ -1256,11 +1246,6 @@ func (c *cfg) bootstrap(ctx context.Context) error { return bootstrapOnline(ctx, c) } -// needBootstrap checks if local node should be registered in network on bootup. -func (c *cfg) needBootstrap() bool { - return c.cfgNetmap.needBootstrap -} - type dCmp struct { name string reloadFunc func() error diff --git a/cmd/frostfs-node/config/node/config.go b/cmd/frostfs-node/config/node/config.go index 81b191e96..c50718c5f 100644 --- a/cmd/frostfs-node/config/node/config.go +++ b/cmd/frostfs-node/config/node/config.go @@ -131,14 +131,6 @@ func Attributes(c *config.Config) (attrs []string) { return } -// Relay returns the value of "relay" config parameter -// from "node" section. -// -// Returns false if the value is not set. -func Relay(c *config.Config) bool { - return config.BoolSafe(c.Sub(subsection), "relay") -} - // PersistentSessions returns structure that provides access to "persistent_sessions" // subsection of "node" section. func PersistentSessions(c *config.Config) PersistentSessionsConfig { diff --git a/cmd/frostfs-node/config/node/config_test.go b/cmd/frostfs-node/config/node/config_test.go index 7b9adecf4..9af1dc038 100644 --- a/cmd/frostfs-node/config/node/config_test.go +++ b/cmd/frostfs-node/config/node/config_test.go @@ -29,12 +29,10 @@ func TestNodeSection(t *testing.T) { ) attribute := Attributes(empty) - relay := Relay(empty) persisessionsPath := PersistentSessions(empty).Path() persistatePath := PersistentState(empty).Path() require.Empty(t, attribute) - require.Equal(t, false, relay) require.Equal(t, "", persisessionsPath) require.Equal(t, PersistentStatePathDefault, persistatePath) }) @@ -45,7 +43,6 @@ func TestNodeSection(t *testing.T) { key := Key(c) addrs := BootstrapAddresses(c) attributes := Attributes(c) - relay := Relay(c) wKey := Wallet(c) persisessionsPath := PersistentSessions(c).Path() persistatePath := PersistentState(c).Path() @@ -87,8 +84,6 @@ func TestNodeSection(t *testing.T) { return false }) - require.Equal(t, true, relay) - require.Len(t, attributes, 2) require.Equal(t, "Price:11", attributes[0]) require.Equal(t, "UN-LOCODE:RU MSK", attributes[1]) diff --git a/cmd/frostfs-node/netmap.go b/cmd/frostfs-node/netmap.go index 82b799e4c..7dfb4fe12 100644 --- a/cmd/frostfs-node/netmap.go +++ b/cmd/frostfs-node/netmap.go @@ -187,7 +187,7 @@ func addNewEpochNotificationHandlers(c *cfg) { c.updateContractNodeInfo(ctx, e) - if !c.needBootstrap() || c.cfgNetmap.reBoostrapTurnedOff.Load() { // fixes #470 + if c.cfgNetmap.reBoostrapTurnedOff.Load() { // fixes #470 return } @@ -209,14 +209,12 @@ func addNewEpochNotificationHandlers(c *cfg) { // bootstrapNode adds current node to the Network map. // Must be called after initNetmapService. func bootstrapNode(ctx context.Context, c *cfg) { - if c.needBootstrap() { - if c.IsMaintenance() { - c.log.Info(ctx, logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) - return - } - err := c.bootstrap(ctx) - fatalOnErrDetails("bootstrap error", err) + if c.IsMaintenance() { + c.log.Info(ctx, logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap) + return } + err := c.bootstrap(ctx) + fatalOnErrDetails("bootstrap error", err) } func addNetmapNotificationHandler(c *cfg, sTyp string, h event.Handler) { @@ -352,8 +350,6 @@ func addNewEpochAsyncNotificationHandler(c *cfg, h event.Handler) { ) } -var errRelayBootstrap = errors.New("setting netmap status is forbidden in relay mode") - func (c *cfg) SetNetmapStatus(ctx context.Context, st control.NetmapStatus) error { switch st { default: @@ -365,10 +361,6 @@ func (c *cfg) SetNetmapStatus(ctx context.Context, st control.NetmapStatus) erro c.stopMaintenance(ctx) - if !c.needBootstrap() { - return errRelayBootstrap - } - if st == control.NetmapStatus_ONLINE { c.cfgNetmap.reBoostrapTurnedOff.Store(false) return bootstrapOnline(ctx, c) diff --git a/config/example/node.env b/config/example/node.env index e7d7a6cc8..9a2426358 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -22,7 +22,6 @@ FROSTFS_NODE_WALLET_PASSWORD=password FROSTFS_NODE_ADDRESSES="s01.frostfs.devenv:8080 /dns4/s02.frostfs.devenv/tcp/8081 grpc://127.0.0.1:8082 grpcs://localhost:8083" FROSTFS_NODE_ATTRIBUTE_0=Price:11 FROSTFS_NODE_ATTRIBUTE_1="UN-LOCODE:RU MSK" -FROSTFS_NODE_RELAY=true FROSTFS_NODE_PERSISTENT_SESSIONS_PATH=/sessions FROSTFS_NODE_PERSISTENT_STATE_PATH=/state FROSTFS_NODE_LOCODE_DB_PATH=/path/to/locode/db diff --git a/config/example/node.json b/config/example/node.json index 3f7854d98..6b7a9c2c6 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -37,7 +37,6 @@ ], "attribute_0": "Price:11", "attribute_1": "UN-LOCODE:RU MSK", - "relay": true, "persistent_sessions": { "path": "/sessions" }, diff --git a/config/example/node.yaml b/config/example/node.yaml index 32f0cba67..2d4bc90fb 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -34,7 +34,6 @@ node: - grpcs://localhost:8083 attribute_0: "Price:11" attribute_1: UN-LOCODE:RU MSK - relay: true # start Storage node in relay mode without bootstrapping into the Network map persistent_sessions: path: /sessions # path to persistent session tokens file of Storage node (default: in-memory sessions) persistent_state: @@ -141,7 +140,6 @@ rpc: max_ops: 10000 storage: - # note: shard configuration can be omitted for relay node (see `node.relay`) shard_ro_error_threshold: 100 # amount of errors to occur before shard is made read-only (default: 0, ignore errors) shard: diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 14ebb53b3..da9fdfed0 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -410,7 +410,6 @@ node: - "Price:11" - "UN-LOCODE:RU MSK" - "key:value" - relay: false persistent_sessions: path: /sessions persistent_state: @@ -424,7 +423,6 @@ node: | `wallet` | [Wallet config](#wallet-subsection) | | Wallet configuration. Has no effect if `key` is provided. | | `addresses` | `[]string` | | Addresses advertised in the netmap. | | `attribute` | `[]string` | | Node attributes as a list of key-value pairs in `:` format. | -| `relay` | `bool` | | Enable relay mode. | | `persistent_sessions` | [Persistent sessions config](#persistent_sessions-subsection) | | Persistent session token store configuration. | | `persistent_state` | [Persistent state config](#persistent_state-subsection) | | Persistent state configuration. | | `locode_db_path` | `string` | empty | Path to UN/LOCODE [database](https://git.frostfs.info/TrueCloudLab/frostfs-locode-db/) for FrostFS. | From 652237500fb9e1618eb66310d3f2a51c599a033e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 24 Apr 2025 16:54:47 +0300 Subject: [PATCH 1941/1943] [#1689] tree: Dial tree service after create connection on sync Background trees sync creates grpc connection with `grpc.WithDefaultCallOptions(grpc.WaitForReady(true))` option. When grpc connection created with this option, client will wait until a connection becomes available or the RPC's deadline is reached. As background sync has no timeout in context, so in case of client is in TRANSIENT_FAILURE RPC call will hang forever. Change-Id: I17c8c1d2779bb81c541f47dd0e558e0b8ed2e7c1 Signed-off-by: Dmitrii Stepanov --- pkg/services/tree/cache.go | 36 ++++++------------------------------ pkg/services/tree/sync.go | 28 +++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/pkg/services/tree/cache.go b/pkg/services/tree/cache.go index 462c8554f..a11700771 100644 --- a/pkg/services/tree/cache.go +++ b/pkg/services/tree/cache.go @@ -66,7 +66,12 @@ func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceCl } } - cc, err := c.dialTreeService(ctx, netmapAddr) + var netAddr network.Address + if err := netAddr.FromString(netmapAddr); err != nil { + return nil, err + } + + cc, err := dialTreeService(ctx, netAddr, c.key, c.ds) lastTry := time.Now() c.Lock() @@ -83,32 +88,3 @@ func (c *clientCache) get(ctx context.Context, netmapAddr string) (TreeServiceCl return NewTreeServiceClient(cc), nil } - -func (c *clientCache) dialTreeService(ctx context.Context, netmapAddr string) (*grpc.ClientConn, error) { - var netAddr network.Address - if err := netAddr.FromString(netmapAddr); err != nil { - return nil, err - } - - cc, err := createConnection(netAddr, grpc.WithContextDialer(c.ds.GrpcContextDialer())) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithTimeout(ctx, defaultClientConnectTimeout) - defer cancel() - - req := &HealthcheckRequest{ - Body: &HealthcheckRequest_Body{}, - } - if err := SignMessage(req, c.key); err != nil { - return nil, err - } - - // perform some request to check connection - if _, err := NewTreeServiceClient(cc).Healthcheck(ctx, req); err != nil { - _ = cc.Close() - return nil, err - } - return cc, nil -} diff --git a/pkg/services/tree/sync.go b/pkg/services/tree/sync.go index d4040337d..af355639f 100644 --- a/pkg/services/tree/sync.go +++ b/pkg/services/tree/sync.go @@ -2,6 +2,7 @@ package tree import ( "context" + "crypto/ecdsa" "crypto/sha256" "crypto/tls" "errors" @@ -14,6 +15,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/net" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/qos" containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama" @@ -304,7 +306,7 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, continue } - cc, err := createConnection(a, grpc.WithContextDialer(s.ds.GrpcContextDialer())) + cc, err := dialTreeService(ctx, a, s.key, s.ds) if err != nil { s.log.Warn(ctx, logs.TreeFailedToConnectForTreeSynchronization, zap.Error(err), zap.String("address", addr)) continue @@ -342,6 +344,30 @@ func (s *Service) synchronizeTree(ctx context.Context, cid cid.ID, from uint64, return from } +func dialTreeService(ctx context.Context, netAddr network.Address, key *ecdsa.PrivateKey, ds *net.DialerSource) (*grpc.ClientConn, error) { + cc, err := createConnection(netAddr, grpc.WithContextDialer(ds.GrpcContextDialer())) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(ctx, defaultClientConnectTimeout) + defer cancel() + + req := &HealthcheckRequest{ + Body: &HealthcheckRequest_Body{}, + } + if err := SignMessage(req, key); err != nil { + return nil, err + } + + // perform some request to check connection + if _, err := NewTreeServiceClient(cc).Healthcheck(ctx, req); err != nil { + _ = cc.Close() + return nil, err + } + return cc, nil +} + func createConnection(a network.Address, opts ...grpc.DialOption) (*grpc.ClientConn, error) { host, isTLS, err := client.ParseURI(a.URIAddr()) if err != nil { From 8e2f919df0fe4525bf25fcbe1fa11f6af642637e Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 28 Apr 2025 20:07:56 +0300 Subject: [PATCH 1942/1943] [#1689] go.mod: Bump SDK version * Fix `APEOverride` method usage in ape checker. * Fix linter errors: factor out deprecated methods and packages. Change-Id: I8c939f4c58c2a4e3c4e795c7224d935d40ce6f24 Signed-off-by: Airat Arifullin --- cmd/frostfs-cli/modules/container/get.go | 4 +- cmd/frostfs-cli/modules/container/list.go | 4 +- go.mod | 24 ++++----- go.sum | 52 +++++++++---------- pkg/core/container/util.go | 4 +- .../internal/testutil/generators.go | 7 +-- .../internal/testutil/object.go | 2 +- pkg/services/common/ape/checker.go | 5 +- .../populate-metabase/internal/generate.go | 5 +- 9 files changed, 55 insertions(+), 52 deletions(-) diff --git a/cmd/frostfs-cli/modules/container/get.go b/cmd/frostfs-cli/modules/container/get.go index 8c4ab14f8..fac6eb2cd 100644 --- a/cmd/frostfs-cli/modules/container/get.go +++ b/cmd/frostfs-cli/modules/container/get.go @@ -93,9 +93,9 @@ func prettyPrintContainer(cmd *cobra.Command, cnr container.Container, jsonEncod cmd.Println("created:", container.CreatedAt(cnr)) cmd.Println("attributes:") - cnr.IterateAttributes(func(key, val string) { + for key, val := range cnr.Attributes() { cmd.Printf("\t%s=%s\n", key, val) - }) + } cmd.Println("placement policy:") commonCmd.ExitOnErr(cmd, "write policy: %w", cnr.PlacementPolicy().WriteStringTo((*stringWriter)(cmd))) diff --git a/cmd/frostfs-cli/modules/container/list.go b/cmd/frostfs-cli/modules/container/list.go index bbb8da840..e4a023d91 100644 --- a/cmd/frostfs-cli/modules/container/list.go +++ b/cmd/frostfs-cli/modules/container/list.go @@ -102,9 +102,9 @@ func printContainer(cmd *cobra.Command, prmGet internalclient.GetContainerPrm, i cmd.Println(id.String()) if flagVarListPrintAttr { - cnr.IterateUserAttributes(func(key, val string) { + for key, val := range cnr.Attributes() { cmd.Printf(" %s: %s\n", key, val) - }) + } } } diff --git a/go.mod b/go.mod index 5ed4a90be..fb45c3874 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.frostfs.info/TrueCloudLab/frostfs-node -go 1.23 +go 1.23.0 require ( code.gitea.io/sdk/gitea v0.17.1 @@ -9,7 +9,7 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-locode-db v0.5.2 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248 git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47 - git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d + git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250428134706-8822aedbbbaa git.frostfs.info/TrueCloudLab/hrw v1.2.1 git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20250402100642-acd94d200f88 @@ -28,7 +28,7 @@ require ( github.com/klauspost/compress v1.17.4 github.com/mailru/easyjson v0.7.7 github.com/mr-tron/base58 v1.2.0 - github.com/multiformats/go-multiaddr v0.14.0 + github.com/multiformats/go-multiaddr v0.15.0 github.com/nspcc-dev/neo-go v0.106.3 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.9.0 @@ -44,10 +44,9 @@ require ( go.opentelemetry.io/otel v1.31.0 go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/sync v0.10.0 - golang.org/x/sys v0.28.0 - golang.org/x/term v0.27.0 + golang.org/x/sync v0.12.0 + golang.org/x/sys v0.31.0 + golang.org/x/term v0.30.0 google.golang.org/grpc v1.69.2 google.golang.org/protobuf v1.36.1 gopkg.in/yaml.v3 v3.0.1 @@ -86,9 +85,9 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/ipfs/go-cid v0.4.1 // indirect + github.com/ipfs/go-cid v0.5.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/klauspost/reedsolomon v1.12.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -124,13 +123,14 @@ require ( go.opentelemetry.io/otel/sdk v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect golang.org/x/net v0.30.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/text v0.23.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - lukechampine.com/blake3 v1.2.1 // indirect + lukechampine.com/blake3 v1.4.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 38aba9bde..acc26af36 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20250321063246-93b681a20248/go.mod h1:kbwB4v2o6RyOfCo9kEFeUDZIX3LKhmS0yXPrtvzkQ1g= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47 h1:O2c3VOlaGZ862hf2ZPLBMdTG6vGJzhIgDvFEFGfntzU= git.frostfs.info/TrueCloudLab/frostfs-qos v0.0.0-20250331080422-b5ed0b6eff47/go.mod h1:PCijYq4oa8vKtIEcUX6jRiszI6XAW+nBwU+T1kB4d1U= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d h1:ZLKDupw362Ciing7kdIZhDYGMyo2QZyJ6sS/8X9QWJ0= -git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250404152210-6458c11e833d/go.mod h1:2PWt5GwJTnhjHp+mankcfCeAJBMn7puxPm+RS+lliVk= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250428134706-8822aedbbbaa h1:ttJxiw5+Wti3outhaPFaLGwCinmUTQgyVQfD/sIU5sg= +git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20250428134706-8822aedbbbaa/go.mod h1:mimnb6yQUBLLQ8PboNc5ZP8iz4VMhFRKrfZcjfR9CVs= git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc= git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM= git.frostfs.info/TrueCloudLab/multinet v0.0.0-20241015075604-6cb0d80e0972 h1:/960fWeyn2AFHwQUwDsWB3sbP6lTEnFnMzLMM6tx6N8= @@ -145,14 +145,14 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= -github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= +github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/reedsolomon v1.12.1 h1:NhWgum1efX1x58daOBGCFWcxtEhOhXKKl1HAPQUp03Q= github.com/klauspost/reedsolomon v1.12.1/go.mod h1:nEi5Kjb6QqtbofI6s+cbG/j1da11c96IBYBSnVGtuBs= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -192,8 +192,8 @@ github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aG github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU= -github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4= +github.com/multiformats/go-multiaddr v0.15.0 h1:zB/HeaI/apcZiTDwhY5YqMvNVl/oQYvs3XySU+qeAVo= +github.com/multiformats/go-multiaddr v0.15.0/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= @@ -324,15 +324,15 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -353,8 +353,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -381,16 +381,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -398,15 +398,15 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= +golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -445,7 +445,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= -lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +lukechampine.com/blake3 v1.4.0 h1:xDbKOZCVbnZsfzM6mHSYcGRHZ3YrLDzqz8XnV4uaD5w= +lukechampine.com/blake3 v1.4.0/go.mod h1:MQJNQCTnR+kwOP/JEZSxj3MaQjp80FOFSNMMHXcSeX0= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/pkg/core/container/util.go b/pkg/core/container/util.go index a24b36944..61c568052 100644 --- a/pkg/core/container/util.go +++ b/pkg/core/container/util.go @@ -26,10 +26,10 @@ func WasRemoved(ctx context.Context, s Source, cid cid.ID) (bool, error) { // IsIndexedContainer returns True if container attributes should be indexed. func IsIndexedContainer(cnr containerSDK.Container) bool { var isS3Container bool - cnr.IterateAttributes(func(key, _ string) { + for key := range cnr.Attributes() { if key == ".s3-location-constraint" { isS3Container = true } - }) + } return !isS3Container } diff --git a/pkg/local_object_storage/internal/testutil/generators.go b/pkg/local_object_storage/internal/testutil/generators.go index 383c596af..52b199b0b 100644 --- a/pkg/local_object_storage/internal/testutil/generators.go +++ b/pkg/local_object_storage/internal/testutil/generators.go @@ -1,7 +1,9 @@ package testutil import ( + cryptorand "crypto/rand" "encoding/binary" + "math/rand" "sync/atomic" "testing" @@ -9,7 +11,6 @@ import ( objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" - "golang.org/x/exp/rand" ) // AddressGenerator is the interface of types that generate object addresses. @@ -61,7 +62,7 @@ var _ ObjectGenerator = &SeqObjGenerator{} func generateObjectWithOIDWithCIDWithSize(oid oid.ID, cid cid.ID, sz uint64) *objectSDK.Object { data := make([]byte, sz) - _, _ = rand.Read(data) + _, _ = cryptorand.Read(data) obj := GenerateObjectWithCIDWithPayload(cid, data) obj.SetID(oid) return obj @@ -82,7 +83,7 @@ var _ ObjectGenerator = &RandObjGenerator{} func (g *RandObjGenerator) Next() *objectSDK.Object { var id oid.ID - _, _ = rand.Read(id[:]) + _, _ = cryptorand.Read(id[:]) return generateObjectWithOIDWithCIDWithSize(id, cid.ID{}, g.ObjSize) } diff --git a/pkg/local_object_storage/internal/testutil/object.go b/pkg/local_object_storage/internal/testutil/object.go index 60e9211d5..1087e40be 100644 --- a/pkg/local_object_storage/internal/testutil/object.go +++ b/pkg/local_object_storage/internal/testutil/object.go @@ -1,6 +1,7 @@ package testutil import ( + "crypto/rand" "crypto/sha256" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" @@ -11,7 +12,6 @@ import ( usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "git.frostfs.info/TrueCloudLab/tzhash/tz" - "golang.org/x/exp/rand" ) const defaultDataSize = 32 diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index a2e628144..02c1e2ee1 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -81,7 +81,8 @@ func (c *checkerCoreImpl) CheckAPE(ctx context.Context, prm CheckPrm) error { if prm.BearerToken.Impersonate() { cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.MorphChainStorage, c.LocalOverrideStorage) } else { - cr, err = router.BearerChainFeedRouter(c.LocalOverrideStorage, c.MorphChainStorage, prm.BearerToken.APEOverride()) + override, _ := prm.BearerToken.APEOverride() + cr, err = router.BearerChainFeedRouter(c.LocalOverrideStorage, c.MorphChainStorage, override) if err != nil { return fmt.Errorf("create chain router error: %w", err) } @@ -130,7 +131,7 @@ func isValidBearer(token *bearer.Token, ownerCnr user.ID, cntID cid.ID, publicKe } // Check for ape overrides defined in the bearer token. - apeOverride := token.APEOverride() + apeOverride, _ := token.APEOverride() if len(apeOverride.Chains) > 0 && apeOverride.Target.TargetType != ape.TargetTypeContainer { return fmt.Errorf("%w: %s", errInvalidTargetType, apeOverride.Target.TargetType.ToV2().String()) } diff --git a/scripts/populate-metabase/internal/generate.go b/scripts/populate-metabase/internal/generate.go index f2f8881cf..39a420358 100644 --- a/scripts/populate-metabase/internal/generate.go +++ b/scripts/populate-metabase/internal/generate.go @@ -1,8 +1,10 @@ package internal import ( + cryptorand "crypto/rand" "crypto/sha256" "fmt" + "math/rand" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/checksum" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" @@ -14,14 +16,13 @@ import ( usertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user/test" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" "git.frostfs.info/TrueCloudLab/tzhash/tz" - "golang.org/x/exp/rand" ) func GeneratePayloadPool(count uint, size uint) [][]byte { var pool [][]byte for range count { payload := make([]byte, size) - _, _ = rand.Read(payload) + _, _ = cryptorand.Read(payload) pool = append(pool, payload) } From 64b46746e409a40a93a41c030a646d69fbf45282 Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Mon, 28 Apr 2025 20:10:09 +0300 Subject: [PATCH 1943/1943] [#1689] ape: Fix validation for overrides in bearer * APE-overrides are optional for bearer. So, it should validate only set override; * Bearer can set overrides for containers, not only the one container - validation expects for any target type for set override. Basically, APE-overrides for all container must be set for namespace target; * Add unit-test cases to check bearer token validation. Change-Id: I6b8e19eb73d24f8cd8799bf99b6c551287da67d9 Signed-off-by: Airat Arifullin --- pkg/services/common/ape/checker.go | 32 ++++++++------- pkg/services/tree/signature_test.go | 61 +++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/pkg/services/common/ape/checker.go b/pkg/services/common/ape/checker.go index 02c1e2ee1..fcd3efa44 100644 --- a/pkg/services/common/ape/checker.go +++ b/pkg/services/common/ape/checker.go @@ -20,7 +20,6 @@ import ( ) var ( - errInvalidTargetType = errors.New("bearer token defines non-container target override") errBearerExpired = errors.New("bearer token has expired") errBearerInvalidSignature = errors.New("bearer token has invalid signature") errBearerInvalidContainerID = errors.New("bearer token was created for another container") @@ -81,7 +80,10 @@ func (c *checkerCoreImpl) CheckAPE(ctx context.Context, prm CheckPrm) error { if prm.BearerToken.Impersonate() { cr = policyengine.NewDefaultChainRouterWithLocalOverrides(c.MorphChainStorage, c.LocalOverrideStorage) } else { - override, _ := prm.BearerToken.APEOverride() + override, isSet := prm.BearerToken.APEOverride() + if !isSet { + return errors.New("expected for override within bearer") + } cr, err = router.BearerChainFeedRouter(c.LocalOverrideStorage, c.MorphChainStorage, override) if err != nil { return fmt.Errorf("create chain router error: %w", err) @@ -131,19 +133,19 @@ func isValidBearer(token *bearer.Token, ownerCnr user.ID, cntID cid.ID, publicKe } // Check for ape overrides defined in the bearer token. - apeOverride, _ := token.APEOverride() - if len(apeOverride.Chains) > 0 && apeOverride.Target.TargetType != ape.TargetTypeContainer { - return fmt.Errorf("%w: %s", errInvalidTargetType, apeOverride.Target.TargetType.ToV2().String()) - } - - // Then check if container is either empty or equal to the container in the request. - var targetCnr cid.ID - err := targetCnr.DecodeString(apeOverride.Target.Name) - if err != nil { - return fmt.Errorf("invalid cid format: %s", apeOverride.Target.Name) - } - if !cntID.Equals(targetCnr) { - return errBearerInvalidContainerID + if apeOverride, isSet := token.APEOverride(); isSet { + switch apeOverride.Target.TargetType { + case ape.TargetTypeContainer: + var targetCnr cid.ID + err := targetCnr.DecodeString(apeOverride.Target.Name) + if err != nil { + return fmt.Errorf("invalid cid format: %s", apeOverride.Target.Name) + } + if !cntID.Equals(targetCnr) { + return errBearerInvalidContainerID + } + default: + } } // Then check if container owner signed this token. diff --git a/pkg/services/tree/signature_test.go b/pkg/services/tree/signature_test.go index dd37b4191..13a5c1395 100644 --- a/pkg/services/tree/signature_test.go +++ b/pkg/services/tree/signature_test.go @@ -235,6 +235,48 @@ func TestMessageSign(t *testing.T) { require.Error(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) }) + t.Run("omit override within bt", func(t *testing.T) { + t.Run("personated", func(t *testing.T) { + bt := testBearerTokenNoOverride() + require.NoError(t, bt.Sign(privs[0].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + + require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) + require.ErrorContains(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut), "expected for override") + }) + + t.Run("impersonated", func(t *testing.T) { + bt := testBearerTokenNoOverride() + bt.SetImpersonate(true) + require.NoError(t, bt.Sign(privs[0].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + + require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) + require.NoError(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut)) + }) + }) + + t.Run("invalid override within bearer token", func(t *testing.T) { + t.Run("personated", func(t *testing.T) { + bt := testBearerTokenCorruptOverride(privs[1].PublicKey(), privs[2].PublicKey()) + require.NoError(t, bt.Sign(privs[0].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + + require.NoError(t, SignMessage(req, &privs[1].PrivateKey)) + require.ErrorContains(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut), "invalid cid") + }) + + t.Run("impersonated", func(t *testing.T) { + bt := testBearerTokenCorruptOverride(privs[1].PublicKey(), privs[2].PublicKey()) + bt.SetImpersonate(true) + require.NoError(t, bt.Sign(privs[0].PrivateKey)) + req.Body.BearerToken = bt.Marshal() + + require.NoError(t, SignMessage(req, &privs[0].PrivateKey)) + require.ErrorContains(t, s.verifyClient(context.Background(), req, cid1, versionTreeID, req.GetBody().GetBearerToken(), acl.OpObjectPut), "invalid cid") + }) + }) + t.Run("impersonate", func(t *testing.T) { cnr.Value.SetBasicACL(acl.PublicRWExtended) var bt bearer.Token @@ -311,6 +353,25 @@ func testBearerToken(cid cid.ID, forPutGet, forGet *keys.PublicKey) bearer.Token return b } +func testBearerTokenCorruptOverride(forPutGet, forGet *keys.PublicKey) bearer.Token { + var b bearer.Token + b.SetExp(currentEpoch + 1) + b.SetAPEOverride(bearer.APEOverride{ + Target: ape.ChainTarget{ + TargetType: ape.TargetTypeContainer, + }, + Chains: []ape.Chain{{Raw: testChain(forPutGet, forGet).Bytes()}}, + }) + + return b +} + +func testBearerTokenNoOverride() bearer.Token { + var b bearer.Token + b.SetExp(currentEpoch + 1) + return b +} + func testChain(forPutGet, forGet *keys.PublicKey) *chain.Chain { ruleGet := chain.Rule{ Status: chain.Allow,